From 7c808317dc7fee894fb7bd871576e6936cda3416 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Fri, 11 Feb 2022 12:24:40 +0000 Subject: [PATCH] clang format apply --- .clang-format | 22 + datatypes/mcs_data_condition.h | 47 +- datatypes/mcs_datatype.cpp | 1584 +- datatypes/mcs_datatype.h | 2564 +-- datatypes/mcs_datatype_basic.h | 30 +- datatypes/mcs_decimal.cpp | 1261 +- datatypes/mcs_decimal.h | 1416 +- datatypes/mcs_double.h | 21 +- datatypes/mcs_float128.h | 1023 +- datatypes/mcs_int128.cpp | 163 +- datatypes/mcs_int128.h | 440 +- datatypes/mcs_int64.h | 121 +- datatypes/mcs_longdouble.h | 21 +- datatypes/mcs_numeric_limits.h | 21 +- datatypes/mcs_string.h | 44 +- datatypes/numericliteral.h | 391 +- dbcon/ddlpackage/altertable.cpp | 246 +- dbcon/ddlpackage/columndef.cpp | 152 +- dbcon/ddlpackage/createindex.cpp | 35 +- dbcon/ddlpackage/createtable.cpp | 31 +- dbcon/ddlpackage/ddl-gram-win.cpp | 6081 +++--- dbcon/ddlpackage/ddl-gram-win.h | 239 +- dbcon/ddlpackage/ddl-scan-win.cpp | 3048 ++- dbcon/ddlpackage/ddlpkg.cpp | 214 +- dbcon/ddlpackage/ddlpkg.h | 1859 +- dbcon/ddlpackage/dropindex.cpp | 19 +- dbcon/ddlpackage/droppartition.cpp | 28 +- dbcon/ddlpackage/droptable.cpp | 28 +- dbcon/ddlpackage/gramtest.cpp | 69 +- dbcon/ddlpackage/markpartition.cpp | 28 +- dbcon/ddlpackage/old-tdriver.cpp | 978 +- dbcon/ddlpackage/restorepartition.cpp | 28 +- dbcon/ddlpackage/serialize.cpp | 1887 +- dbcon/ddlpackage/sqlparser.cpp | 106 +- dbcon/ddlpackage/sqlparser.h | 110 +- dbcon/ddlpackage/sqlstatement.cpp | 14 +- dbcon/ddlpackage/sqlstatementlist.cpp | 39 +- dbcon/ddlpackage/tabledef.cpp | 343 +- dbcon/ddlpackage/tdriver.cpp | 1581 +- dbcon/ddlpackageproc/altertableprocessor.cpp | 4118 ++-- dbcon/ddlpackageproc/altertableprocessor.h | 221 +- dbcon/ddlpackageproc/createindexprocessor.cpp | 849 +- dbcon/ddlpackageproc/createindexprocessor.h | 26 +- dbcon/ddlpackageproc/createtableprocessor.cpp | 1515 +- dbcon/ddlpackageproc/createtableprocessor.h | 30 +- dbcon/ddlpackageproc/ddlindexpopulator.cpp | 794 +- dbcon/ddlpackageproc/ddlindexpopulator.h | 474 +- dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 2049 +- dbcon/ddlpackageproc/ddlpackageprocessor.h | 1367 +- .../ddlpackageprocessorfactory.cpp | 38 +- .../ddlpackageprocessorfactory.h | 37 +- dbcon/ddlpackageproc/dropindexprocessor.cpp | 120 +- dbcon/ddlpackageproc/dropindexprocessor.h | 22 +- .../ddlpackageproc/droppartitionprocessor.cpp | 638 +- dbcon/ddlpackageproc/droppartitionprocessor.h | 26 +- dbcon/ddlpackageproc/droptableprocessor.cpp | 2453 +-- dbcon/ddlpackageproc/droptableprocessor.h | 48 +- .../ddlpackageproc/markpartitionprocessor.cpp | 484 +- dbcon/ddlpackageproc/markpartitionprocessor.h | 28 +- dbcon/ddlpackageproc/mydriver.cpp | 1006 +- dbcon/ddlpackageproc/resource.h | 8 +- .../restorepartitionprocessor.cpp | 485 +- .../restorepartitionprocessor.h | 26 +- dbcon/ddlpackageproc/tdriver.cpp | 1888 +- dbcon/dmlpackage/calpontdmlfactory.cpp | 296 +- dbcon/dmlpackage/calpontdmlfactory.h | 54 +- dbcon/dmlpackage/calpontdmlpackage.cpp | 82 +- dbcon/dmlpackage/calpontdmlpackage.h | 620 +- dbcon/dmlpackage/commanddmlpackage.cpp | 88 +- dbcon/dmlpackage/commanddmlpackage.h | 96 +- dbcon/dmlpackage/deletedmlpackage.cpp | 249 +- dbcon/dmlpackage/deletedmlpackage.h | 101 +- dbcon/dmlpackage/dml-gram-win.cpp | 4926 ++--- dbcon/dmlpackage/dml-gram-win.h | 257 +- dbcon/dmlpackage/dml-scan-win.cpp | 2829 ++- dbcon/dmlpackage/dmlcolumn.cpp | 136 +- dbcon/dmlpackage/dmlcolumn.h | 225 +- dbcon/dmlpackage/dmlobject.cpp | 14 +- dbcon/dmlpackage/dmlobject.h | 66 +- dbcon/dmlpackage/dmlpackage.h | 11 +- dbcon/dmlpackage/dmlparser.cpp | 130 +- dbcon/dmlpackage/dmlparser.h | 101 +- dbcon/dmlpackage/dmlpkg.cpp | 1263 +- dbcon/dmlpackage/dmlpkg.h | 1096 +- dbcon/dmlpackage/dmltable.cpp | 120 +- dbcon/dmlpackage/dmltable.h | 152 +- dbcon/dmlpackage/gramtest.cpp | 74 +- dbcon/dmlpackage/insertdmlpackage.cpp | 389 +- dbcon/dmlpackage/insertdmlpackage.h | 134 +- dbcon/dmlpackage/mysqldmlstatement.cpp | 9 +- dbcon/dmlpackage/mysqldmlstatement.h | 26 +- dbcon/dmlpackage/oracledmlstatement.cpp | 10 +- dbcon/dmlpackage/oracledmlstatement.h | 26 +- dbcon/dmlpackage/row.cpp | 109 +- dbcon/dmlpackage/row.h | 132 +- dbcon/dmlpackage/tdriver.cpp | 568 +- dbcon/dmlpackage/updatedmlpackage.cpp | 328 +- dbcon/dmlpackage/updatedmlpackage.h | 103 +- dbcon/dmlpackage/vendordmlstatement.cpp | 63 +- dbcon/dmlpackage/vendordmlstatement.h | 347 +- dbcon/dmlpackageproc/autoincrementdata.cpp | 58 +- dbcon/dmlpackageproc/autoincrementdata.h | 35 +- .../commandpackageprocessor.cpp | 1933 +- .../dmlpackageproc/commandpackageprocessor.h | 57 +- .../dmlpackageproc/deletepackageprocessor.cpp | 1522 +- dbcon/dmlpackageproc/deletepackageprocessor.h | 71 +- dbcon/dmlpackageproc/dmlpackageprocessor.cpp | 1327 +- dbcon/dmlpackageproc/dmlpackageprocessor.h | 855 +- .../dmlpackageprocessorfactory.cpp | 43 +- .../dmlpackageprocessorfactory.h | 38 +- .../dmlpackageproc/insertpackageprocessor.cpp | 685 +- dbcon/dmlpackageproc/insertpackageprocessor.h | 44 +- dbcon/dmlpackageproc/resource.h | 8 +- dbcon/dmlpackageproc/tablelockdata.cpp | 61 +- dbcon/dmlpackageproc/tablelockdata.h | 38 +- dbcon/dmlpackageproc/tdriver.cpp | 1752 +- .../dmlpackageproc/updatepackageprocessor.cpp | 1734 +- dbcon/dmlpackageproc/updatepackageprocessor.h | 63 +- dbcon/doc/calpontsystemcatalog.h | 50 +- dbcon/execplan/aggregatecolumn.cpp | 996 +- dbcon/execplan/aggregatecolumn.h | 738 +- dbcon/execplan/arithmeticcolumn.cpp | 582 +- dbcon/execplan/arithmeticcolumn.h | 404 +- dbcon/execplan/arithmeticoperator.cpp | 107 +- dbcon/execplan/arithmeticoperator.h | 750 +- dbcon/execplan/blocksize.h | 1 - dbcon/execplan/btdriver.cpp | 279 +- dbcon/execplan/calpontexecutionplan.cpp | 11 +- dbcon/execplan/calpontexecutionplan.h | 224 +- .../execplan/calpontexecutionplanfactory.cpp | 14 +- dbcon/execplan/calpontexecutionplanfactory.h | 20 +- dbcon/execplan/calpontselectexecutionplan.cpp | 1224 +- dbcon/execplan/calpontselectexecutionplan.h | 1743 +- dbcon/execplan/calpontsystemcatalog.cpp | 6586 +++--- dbcon/execplan/calpontsystemcatalog.h | 2047 +- dbcon/execplan/clientrotator.cpp | 460 +- dbcon/execplan/clientrotator.h | 218 +- dbcon/execplan/columnresult.h | 129 +- dbcon/execplan/constantcolumn.cpp | 521 +- dbcon/execplan/constantcolumn.h | 722 +- dbcon/execplan/constantfilter.cpp | 333 +- dbcon/execplan/constantfilter.h | 337 +- dbcon/execplan/existsfilter.cpp | 122 +- dbcon/execplan/existsfilter.h | 187 +- dbcon/execplan/exp_templates.h | 991 +- dbcon/execplan/expressionparser.cpp | 525 +- dbcon/execplan/expressionparser.h | 150 +- dbcon/execplan/filter.cpp | 59 +- dbcon/execplan/filter.h | 200 +- dbcon/execplan/functioncolumn.cpp | 607 +- dbcon/execplan/functioncolumn.h | 527 +- dbcon/execplan/groupconcatcolumn.cpp | 135 +- dbcon/execplan/groupconcatcolumn.h | 179 +- dbcon/execplan/intervalcolumn.cpp | 46 +- dbcon/execplan/intervalcolumn.h | 159 +- dbcon/execplan/logicoperator.cpp | 54 +- dbcon/execplan/logicoperator.h | 237 +- .../execplan/mcsanalyzetableexecutionplan.cpp | 181 +- dbcon/execplan/mcsanalyzetableexecutionplan.h | 267 +- dbcon/execplan/mysqlexecutionplan.cpp | 9 +- dbcon/execplan/mysqlexecutionplan.h | 127 +- dbcon/execplan/njlcolumnresult.h | 1 - dbcon/execplan/objectidmanager.cpp | 60 +- dbcon/execplan/objectidmanager.h | 116 +- dbcon/execplan/objectreader.cpp | 412 +- dbcon/execplan/objectreader.h | 204 +- dbcon/execplan/operator.cpp | 409 +- dbcon/execplan/operator.h | 367 +- dbcon/execplan/oracleexecutionplan.cpp | 9 +- dbcon/execplan/oracleexecutionplan.h | 127 +- dbcon/execplan/outerjoinonfilter.cpp | 106 +- dbcon/execplan/outerjoinonfilter.h | 151 +- dbcon/execplan/parsetree.h | 780 +- dbcon/execplan/predicateoperator.cpp | 1139 +- dbcon/execplan/predicateoperator.h | 222 +- dbcon/execplan/pseudocolumn.cpp | 293 +- dbcon/execplan/pseudocolumn.h | 180 +- dbcon/execplan/range.cpp | 9 +- dbcon/execplan/range.h | 127 +- dbcon/execplan/returnedcolumn.cpp | 300 +- dbcon/execplan/returnedcolumn.h | 540 +- dbcon/execplan/rowcolumn.cpp | 147 +- dbcon/execplan/rowcolumn.h | 221 +- dbcon/execplan/selectfilter.cpp | 218 +- dbcon/execplan/selectfilter.h | 248 +- dbcon/execplan/sessionmanager.cpp | 157 +- dbcon/execplan/sessionmanager.h | 251 +- dbcon/execplan/sessionmonitor.cpp | 625 +- dbcon/execplan/sessionmonitor.h | 461 +- dbcon/execplan/simplecolumn.cpp | 992 +- dbcon/execplan/simplecolumn.h | 625 +- dbcon/execplan/simplecolumn_decimal.h | 279 +- dbcon/execplan/simplecolumn_int.h | 276 +- dbcon/execplan/simplecolumn_uint.h | 294 +- dbcon/execplan/simplefilter.cpp | 1191 +- dbcon/execplan/simplefilter.h | 368 +- dbcon/execplan/simplescalarfilter.cpp | 199 +- dbcon/execplan/simplescalarfilter.h | 212 +- dbcon/execplan/tdriver-oid.cpp | 327 +- dbcon/execplan/tdriver-sm.cpp | 357 +- dbcon/execplan/tdriver.cpp | 2756 ++- dbcon/execplan/tdriver1.cpp | 162 +- dbcon/execplan/tdriver10.cpp | 197 +- dbcon/execplan/tdriver11.cpp | 189 +- dbcon/execplan/tdriver12.cpp | 329 +- dbcon/execplan/tdriver13.cpp | 340 +- dbcon/execplan/tdriver14.cpp | 307 +- dbcon/execplan/tdriver15.cpp | 480 +- dbcon/execplan/tdriver16.cpp | 459 +- dbcon/execplan/tdriver17.cpp | 324 +- dbcon/execplan/tdriver18.cpp | 371 +- dbcon/execplan/tdriver19.cpp | 656 +- dbcon/execplan/tdriver2.cpp | 265 +- dbcon/execplan/tdriver20.cpp | 444 +- dbcon/execplan/tdriver21.cpp | 573 +- dbcon/execplan/tdriver22.cpp | 657 +- dbcon/execplan/tdriver23.cpp | 185 +- dbcon/execplan/tdriver3.cpp | 177 +- dbcon/execplan/tdriver4.cpp | 217 +- dbcon/execplan/tdriver5.cpp | 179 +- dbcon/execplan/tdriver6.cpp | 128 +- dbcon/execplan/tdriver7.cpp | 325 +- dbcon/execplan/tdriver8.cpp | 268 +- dbcon/execplan/tdriver9.cpp | 240 +- dbcon/execplan/treenode.cpp | 61 +- dbcon/execplan/treenode.h | 1837 +- dbcon/execplan/treenodeimpl.cpp | 56 +- dbcon/execplan/treenodeimpl.h | 152 +- dbcon/execplan/udafcolumn.cpp | 69 +- dbcon/execplan/udafcolumn.h | 145 +- dbcon/execplan/vendorexecutionplan.cpp | 9 +- dbcon/execplan/vendorexecutionplan.h | 20 +- dbcon/execplan/wf_frame.h | 85 +- dbcon/execplan/windowfunctioncolumn.cpp | 1087 +- dbcon/execplan/windowfunctioncolumn.h | 353 +- dbcon/joblist/anydatalist.cpp | 200 +- dbcon/joblist/bandeddl.h | 465 +- dbcon/joblist/batchprimitiveprocessor-jl.cpp | 2324 +- dbcon/joblist/batchprimitiveprocessor-jl.h | 524 +- dbcon/joblist/bpp-jl.h | 1 - dbcon/joblist/bucketdl.h | 1174 +- dbcon/joblist/columncommand-jl.cpp | 341 +- dbcon/joblist/columncommand-jl.h | 154 +- dbcon/joblist/command-jl.cpp | 38 +- dbcon/joblist/command-jl.h | 140 +- dbcon/joblist/crossenginestep.cpp | 1216 +- dbcon/joblist/crossenginestep.h | 351 +- dbcon/joblist/datalist.h | 204 +- dbcon/joblist/datalistimpl.h | 267 +- dbcon/joblist/dictstep-jl.cpp | 84 +- dbcon/joblist/dictstep-jl.h | 66 +- dbcon/joblist/diskjoinstep.cpp | 997 +- dbcon/joblist/diskjoinstep.h | 213 +- dbcon/joblist/distributedenginecomm.cpp | 1882 +- dbcon/joblist/distributedenginecomm.h | 359 +- dbcon/joblist/elementcompression.h | 640 +- dbcon/joblist/elementtype.cpp | 80 +- dbcon/joblist/elementtype.h | 794 +- dbcon/joblist/errorinfo.h | 34 +- dbcon/joblist/expressionstep.cpp | 1123 +- dbcon/joblist/expressionstep.h | 402 +- dbcon/joblist/fifo.h | 753 +- dbcon/joblist/filtercommand-jl.cpp | 45 +- dbcon/joblist/filtercommand-jl.h | 53 +- dbcon/joblist/filterstep.cpp | 312 +- dbcon/joblist/groupconcat.cpp | 1543 +- dbcon/joblist/groupconcat.h | 200 +- dbcon/joblist/jl_logger.cpp | 58 +- dbcon/joblist/jl_logger.h | 70 +- dbcon/joblist/jlf_common.cpp | 1124 +- dbcon/joblist/jlf_common.h | 562 +- dbcon/joblist/jlf_execplantojoblist.cpp | 5686 +++-- dbcon/joblist/jlf_execplantojoblist.h | 46 +- dbcon/joblist/jlf_graphics.cpp | 703 +- dbcon/joblist/jlf_graphics.h | 8 +- dbcon/joblist/jlf_subquery.cpp | 1192 +- dbcon/joblist/jlf_subquery.h | 11 +- dbcon/joblist/jlf_tuplejoblist.cpp | 7861 ++++--- dbcon/joblist/jlf_tuplejoblist.h | 149 +- dbcon/joblist/joblist.cpp | 1893 +- dbcon/joblist/joblist.h | 339 +- dbcon/joblist/joblistfactory.cpp | 3808 ++-- dbcon/joblist/joblistfactory.h | 35 +- dbcon/joblist/joblisttypes.h | 40 +- dbcon/joblist/jobstep.cpp | 274 +- dbcon/joblist/jobstep.h | 894 +- dbcon/joblist/jobstepassociation.cpp | 10 +- dbcon/joblist/largedatalist.h | 1686 +- dbcon/joblist/largehashjoin.cpp | 1920 +- dbcon/joblist/largehashjoin.h | 800 +- dbcon/joblist/lbidlist.cpp | 1334 +- dbcon/joblist/lbidlist.h | 168 +- dbcon/joblist/limitedorderby.cpp | 402 +- dbcon/joblist/limitedorderby.h | 49 +- dbcon/joblist/orderby-tests.cpp | 603 +- dbcon/joblist/passthrucommand-jl.cpp | 73 +- dbcon/joblist/passthrucommand-jl.h | 40 +- dbcon/joblist/passthrustep.cpp | 187 +- dbcon/joblist/pcolscan.cpp | 1980 +- dbcon/joblist/pcolstep.cpp | 2470 ++- dbcon/joblist/pdictionary.cpp | 836 +- dbcon/joblist/pdictionaryscan.cpp | 1390 +- dbcon/joblist/primitivemsg.cpp | 25 +- dbcon/joblist/primitivemsg.h | 1006 +- dbcon/joblist/primitivestep.h | 2717 ++- dbcon/joblist/pseudocc-jl.cpp | 122 +- dbcon/joblist/pseudocc-jl.h | 34 +- dbcon/joblist/resource.h | 8 +- dbcon/joblist/resourcedistributor.cpp | 172 +- dbcon/joblist/resourcedistributor.h | 134 +- dbcon/joblist/resourcemanager.cpp | 729 +- dbcon/joblist/resourcemanager.h | 988 +- dbcon/joblist/rowestimator.cpp | 1029 +- dbcon/joblist/rowestimator.h | 200 +- dbcon/joblist/rtscommand-jl.cpp | 77 +- dbcon/joblist/rtscommand-jl.h | 52 +- dbcon/joblist/subquerystep.cpp | 661 +- dbcon/joblist/subquerystep.h | 416 +- dbcon/joblist/subquerytransformer.cpp | 996 +- dbcon/joblist/subquerytransformer.h | 320 +- dbcon/joblist/tablecolumn.cpp | 421 +- dbcon/joblist/tablecolumn.h | 159 +- dbcon/joblist/tdriver-agg.cpp | 1754 +- dbcon/joblist/tdriver-bdl.cpp | 752 +- dbcon/joblist/tdriver-bru.cpp | 1265 +- dbcon/joblist/tdriver-datalist.cpp | 2289 +- dbcon/joblist/tdriver-dec.cpp | 268 +- dbcon/joblist/tdriver-deliver.cpp | 980 +- dbcon/joblist/tdriver-filter.cpp | 986 +- dbcon/joblist/tdriver-function.cpp | 2650 ++- dbcon/joblist/tdriver-hashjoin.cpp | 1281 +- dbcon/joblist/tdriver-index.cpp | 568 +- dbcon/joblist/tdriver-jobstep.cpp | 1837 +- dbcon/joblist/tdriver-pdict.cpp | 172 +- dbcon/joblist/tdriver-zdl.cpp | 1602 +- dbcon/joblist/tdriver1.cpp | 59 +- dbcon/joblist/tdriver2.cpp | 108 +- dbcon/joblist/threadsafequeue.h | 453 +- dbcon/joblist/timeset.h | 329 +- dbcon/joblist/timestamp.cpp | 94 +- dbcon/joblist/timestamp.h | 196 +- dbcon/joblist/tuple-bps.cpp | 5124 +++-- dbcon/joblist/tupleaggregatestep.cpp | 11121 +++++----- dbcon/joblist/tupleaggregatestep.h | 335 +- dbcon/joblist/tupleannexstep.cpp | 1912 +- dbcon/joblist/tupleannexstep.h | 253 +- dbcon/joblist/tupleconstantstep.cpp | 1503 +- dbcon/joblist/tupleconstantstep.h | 240 +- dbcon/joblist/tuplehashjoin.cpp | 3045 ++- dbcon/joblist/tuplehashjoin.h | 1147 +- dbcon/joblist/tuplehavingstep.cpp | 513 +- dbcon/joblist/tuplehavingstep.h | 127 +- dbcon/joblist/tupleunion.cpp | 2459 +-- dbcon/joblist/tupleunion.h | 281 +- dbcon/joblist/unique32generator.cpp | 33 +- dbcon/joblist/unique32generator.h | 29 +- dbcon/joblist/virtualtable.cpp | 171 +- dbcon/joblist/virtualtable.h | 145 +- dbcon/joblist/windowfunctionstep.cpp | 2550 ++- dbcon/joblist/windowfunctionstep.h | 287 +- dbcon/mysql/ha_autoi.cpp | 266 +- dbcon/mysql/ha_exists_sub.cpp | 177 +- dbcon/mysql/ha_from_sub.cpp | 687 +- dbcon/mysql/ha_in_sub.cpp | 441 +- dbcon/mysql/ha_mcs.cpp | 1424 +- dbcon/mysql/ha_mcs.h | 447 +- dbcon/mysql/ha_mcs_client_udfs.cpp | 1497 +- dbcon/mysql/ha_mcs_datatype.h | 293 +- dbcon/mysql/ha_mcs_ddl.cpp | 4911 ++--- dbcon/mysql/ha_mcs_dml.cpp | 1578 +- dbcon/mysql/ha_mcs_execplan.cpp | 18005 ++++++++-------- dbcon/mysql/ha_mcs_impl.cpp | 9063 ++++---- dbcon/mysql/ha_mcs_impl.h | 39 +- dbcon/mysql/ha_mcs_impl_if.h | 585 +- dbcon/mysql/ha_mcs_logging.h | 25 +- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 185 +- dbcon/mysql/ha_mcs_opt_rewrites.h | 4 +- dbcon/mysql/ha_mcs_partition.cpp | 2463 ++- dbcon/mysql/ha_mcs_pushdown.cpp | 1394 +- dbcon/mysql/ha_mcs_pushdown.h | 71 +- dbcon/mysql/ha_mcs_sysvars.cpp | 686 +- dbcon/mysql/ha_mcs_sysvars.h | 27 +- dbcon/mysql/ha_mcs_udf.cpp | 321 +- dbcon/mysql/ha_pseudocolumn.cpp | 725 +- dbcon/mysql/ha_scalar_sub.cpp | 484 +- dbcon/mysql/ha_select_sub.cpp | 112 +- dbcon/mysql/ha_subquery.h | 287 +- dbcon/mysql/ha_tzinfo.cpp | 355 +- dbcon/mysql/ha_tzinfo.h | 3 +- dbcon/mysql/ha_view.cpp | 235 +- dbcon/mysql/ha_view.h | 45 +- dbcon/mysql/ha_window_function.cpp | 1524 +- dbcon/mysql/idb_mysql.h | 15 +- dbcon/mysql/is_columnstore.h | 111 +- dbcon/mysql/is_columnstore_columns.cpp | 244 +- dbcon/mysql/is_columnstore_extents.cpp | 459 +- dbcon/mysql/is_columnstore_files.cpp | 462 +- dbcon/mysql/is_columnstore_tables.cpp | 116 +- dbcon/mysql/sm.cpp | 815 +- dbcon/mysql/sm.h | 342 +- dbcon/mysql/tdriver1.cpp | 121 +- dbcon/mysql/versionnumber.h | 1 - ddlproc/ddlproc.cpp | 281 +- ddlproc/ddlproc.h | 3 +- ddlproc/ddlprocessor.cpp | 1551 +- ddlproc/ddlprocessor.h | 120 +- ddlproc/resource.h | 8 +- ddlproc/tdriver.cpp | 68 +- dmlproc/batchinsertprocessor.cpp | 759 +- dmlproc/batchinsertprocessor.h | 98 +- dmlproc/dmlproc.cpp | 1010 +- dmlproc/dmlproc.h | 18 +- dmlproc/dmlprocessor.cpp | 3331 +-- dmlproc/dmlprocessor.h | 460 +- dmlproc/dmlresultbuffer.cpp | 69 +- dmlproc/dmlresultbuffer.h | 102 +- dmlproc/resource.h | 8 +- exemgr/activestatementcounter.cpp | 41 +- exemgr/activestatementcounter.h | 53 +- exemgr/femsghandler.cpp | 118 +- exemgr/femsghandler.h | 34 +- exemgr/main.cpp | 2926 ++- exemgr/resource.h | 8 +- oam/oamcpp/liboamcpp.cpp | 1438 +- oam/oamcpp/liboamcpp.h | 355 +- oam/oamcpp/oamcache.cpp | 289 +- oam/oamcpp/oamcache.h | 75 +- oam/oamcpp/resource.h | 8 +- oam/replaytxnlog/replaytxnlog.cpp | 1522 +- oam/replaytxnlog/replaytxnlog.h | 238 +- oamapps/columnstoreDB/columnstoreDB.cpp | 509 +- .../columnstoreSupport/columnstoreSupport.cpp | 1531 +- oamapps/columnstoreSupport/mcsSupportUtil.cpp | 1052 +- oamapps/columnstoreSupport/mcsSupportUtil.h | 8 +- oamapps/postConfigure/mycnfUpgrade.cpp | 386 +- .../replaytransactionlog.cpp | 196 +- oamapps/replayTransactionLog/tdriver.cpp | 37 +- oamapps/sessionWalker/sessionwalker.cpp | 140 +- oamapps/sessionWalker/tdriver.cpp | 256 +- primitives/blockcache/bcTest.cpp | 580 +- primitives/blockcache/blockcacheclient.cpp | 1 - primitives/blockcache/blockcacheclient.h | 231 +- .../blockcache/blockrequestprocessor.cpp | 331 +- primitives/blockcache/blockrequestprocessor.h | 239 +- .../blockcache/fileblockrequestqueue.cpp | 59 +- primitives/blockcache/fileblockrequestqueue.h | 108 +- primitives/blockcache/filebuffer.cpp | 67 +- primitives/blockcache/filebuffer.h | 215 +- primitives/blockcache/filebuffermgr.cpp | 1229 +- primitives/blockcache/filebuffermgr.h | 261 +- primitives/blockcache/filerequest.cpp | 102 +- primitives/blockcache/filerequest.h | 482 +- primitives/blockcache/fsutils.cpp | 103 +- primitives/blockcache/fsutils.h | 1 - primitives/blockcache/iomanager.cpp | 2097 +- primitives/blockcache/iomanager.h | 165 +- primitives/blockcache/stats.cpp | 312 +- primitives/blockcache/stats.h | 58 +- primitives/blockcache/tdriver.cpp | 518 +- primitives/linux-port/column.cpp | 2682 ++- primitives/linux-port/dictionary.cpp | 1059 +- primitives/linux-port/index.cpp | 1961 +- primitives/linux-port/pp-scan-unittest.cpp | 9693 ++++----- primitives/linux-port/primitiveprocessor.cpp | 37 +- primitives/linux-port/primitiveprocessor.h | 780 +- primitives/linux-port/print_dictblock.cpp | 101 +- primitives/linux-port/print_indexlist.cpp | 212 +- .../linux-port/print_indextree_subblock.cpp | 72 +- .../primproc/batchprimitiveprocessor.cpp | 4438 ++-- primitives/primproc/batchprimitiveprocessor.h | 627 +- primitives/primproc/bpp.h | 1 - primitives/primproc/bppseeder.cpp | 526 +- primitives/primproc/bppseeder.h | 73 +- primitives/primproc/bppsendthread.cpp | 420 +- primitives/primproc/bppsendthread.h | 205 +- primitives/primproc/columncommand.cpp | 1564 +- primitives/primproc/columncommand.h | 428 +- primitives/primproc/command.cpp | 176 +- primitives/primproc/command.h | 151 +- primitives/primproc/dictstep.cpp | 938 +- primitives/primproc/dictstep.h | 202 +- primitives/primproc/filtercommand.cpp | 706 +- primitives/primproc/filtercommand.h | 193 +- primitives/primproc/logger.cpp | 74 +- primitives/primproc/passthrucommand.cpp | 176 +- primitives/primproc/passthrucommand.h | 52 +- primitives/primproc/pp_logger.h | 47 +- primitives/primproc/primitiveserver.cpp | 4066 ++-- primitives/primproc/primitiveserver.h | 193 +- primitives/primproc/primproc.cpp | 992 +- primitives/primproc/primproc.h | 125 +- primitives/primproc/pseudocc.cpp | 314 +- primitives/primproc/pseudocc.h | 124 +- primitives/primproc/resource.h | 8 +- primitives/primproc/rtscommand.cpp | 242 +- primitives/primproc/rtscommand.h | 77 +- primitives/primproc/tdriver-umsocksel.cpp | 272 +- primitives/primproc/tdriver.cpp | 1412 +- primitives/primproc/udf.cpp | 40 +- primitives/primproc/umsocketselector.cpp | 651 +- primitives/primproc/umsocketselector.h | 386 +- storage-manager/include/messageFormat.h | 224 +- storage-manager/src/AppendTask.cpp | 150 +- storage-manager/src/AppendTask.h | 21 +- storage-manager/src/Cache.cpp | 421 +- storage-manager/src/Cache.h | 147 +- .../src/ClientRequestProcessor.cpp | 30 +- storage-manager/src/ClientRequestProcessor.h | 21 +- storage-manager/src/CloudStorage.cpp | 88 +- storage-manager/src/CloudStorage.h | 56 +- storage-manager/src/Config.cpp | 279 +- storage-manager/src/Config.h | 62 +- storage-manager/src/CopyTask.cpp | 95 +- storage-manager/src/CopyTask.h | 21 +- storage-manager/src/Downloader.cpp | 300 +- storage-manager/src/Downloader.h | 141 +- storage-manager/src/IOCoordinator.cpp | 2703 +-- storage-manager/src/IOCoordinator.h | 169 +- storage-manager/src/ListDirectoryTask.cpp | 194 +- storage-manager/src/ListDirectoryTask.h | 25 +- storage-manager/src/LocalStorage.cpp | 377 +- storage-manager/src/LocalStorage.h | 55 +- storage-manager/src/MetadataFile.cpp | 781 +- storage-manager/src/MetadataFile.h | 211 +- storage-manager/src/OpenTask.cpp | 105 +- storage-manager/src/OpenTask.h | 21 +- storage-manager/src/Ownership.cpp | 509 +- storage-manager/src/Ownership.h | 77 +- storage-manager/src/PingTask.cpp | 48 +- storage-manager/src/PingTask.h | 21 +- storage-manager/src/PosixTask.cpp | 286 +- storage-manager/src/PosixTask.h | 67 +- storage-manager/src/PrefixCache.cpp | 1026 +- storage-manager/src/PrefixCache.h | 242 +- storage-manager/src/ProcessTask.cpp | 133 +- storage-manager/src/ProcessTask.h | 36 +- storage-manager/src/RWLock.cpp | 115 +- storage-manager/src/RWLock.h | 55 +- storage-manager/src/ReadTask.cpp | 133 +- storage-manager/src/ReadTask.h | 17 +- storage-manager/src/Replicator.cpp | 808 +- storage-manager/src/Replicator.h | 65 +- storage-manager/src/S3Storage.cpp | 1241 +- storage-manager/src/S3Storage.h | 113 +- storage-manager/src/SMLogging.cpp | 58 +- storage-manager/src/SMLogging.h | 21 +- storage-manager/src/SessionManager.cpp | 828 +- storage-manager/src/SessionManager.h | 69 +- storage-manager/src/StatTask.cpp | 91 +- storage-manager/src/StatTask.h | 21 +- storage-manager/src/SyncTask.cpp | 44 +- storage-manager/src/SyncTask.h | 17 +- storage-manager/src/Synchronizer.cpp | 1466 +- storage-manager/src/Synchronizer.h | 187 +- storage-manager/src/ThreadPool.cpp | 230 +- storage-manager/src/ThreadPool.h | 114 +- storage-manager/src/TruncateTask.cpp | 89 +- storage-manager/src/TruncateTask.h | 21 +- storage-manager/src/UnlinkTask.cpp | 92 +- storage-manager/src/UnlinkTask.h | 21 +- storage-manager/src/Utilities.cpp | 113 +- storage-manager/src/Utilities.h | 72 +- storage-manager/src/WriteTask.cpp | 146 +- storage-manager/src/WriteTask.h | 21 +- storage-manager/src/main.cpp | 201 +- storage-manager/src/smcat.cpp | 226 +- storage-manager/src/smls.cpp | 249 +- storage-manager/src/smput.cpp | 254 +- storage-manager/src/smrm.cpp | 146 +- storage-manager/src/testS3Connection.cpp | 65 +- storage-manager/src/unit_tests.cpp | 3131 +-- tests/arithmeticoperator-tests.cpp | 6 +- tests/col16block.h | 1166 +- tests/col1block.h | 1166 +- tests/col2block.h | 1166 +- tests/col4block.h | 1166 +- tests/col8block.h | 1166 +- tests/col_double_block.h | 1167 +- tests/col_float_block.h | 1166 +- tests/col_neg_double.h | 1167 +- tests/col_neg_float.h | 1166 +- tests/comparators-tests.cpp | 723 +- tests/compression-tests.cpp | 134 +- tests/dataconvert-tests.cpp | 1067 +- tests/mcs_decimal-tests.cpp | 2537 ++- tests/primitives_column_scan_and_filter.cpp | 191 +- tests/primitives_scan_bench.cpp | 44 +- tests/rebuild-em-tests.cpp | 192 +- tests/rowgroup-tests.cpp | 445 +- tests/shared_components_tests.cpp | 3222 ++- tools/bincvt/li2bin.cpp | 356 +- tools/cfread/cfread.cpp | 274 +- tools/cfread/mtread.cpp | 417 +- tools/clearShm/main.cpp | 290 +- tools/cleartablelock/cleartablelock.cpp | 1058 +- tools/cleartablelock/cleartablelockthread.cpp | 290 +- tools/cleartablelock/cleartablelockthread.h | 164 +- tools/configMgt/autoConfigure.cpp | 3483 +-- tools/cplogger/main.cpp | 108 +- tools/dbbuilder/dbbuilder.cpp | 431 +- tools/dbbuilder/dbbuilder.h | 8 +- tools/dbbuilder/systemcatalog.cpp | 933 +- tools/dbbuilder/systemcatalog.h | 15 +- tools/dbbuilder/tpchpopulate.cpp | 277 +- tools/dbbuilder/tpchpopulate.h | 27 +- tools/dbbuilder/tpchschema.cpp | 489 +- tools/dbbuilder/tpchschema.h | 30 +- tools/dbloadxml/colxml.cpp | 294 +- tools/dbloadxml/inputmgr.cpp | 339 +- tools/dbloadxml/inputmgr.h | 52 +- tools/dbloadxml/resource.h | 8 +- tools/ddlcleanup/ddlcleanup.cpp | 36 +- tools/ddldriver/ddldriver.cpp | 141 +- tools/dmldriver/dmldriver.cpp | 276 +- tools/dmldriver/dmlif.cpp | 553 +- tools/dmldriver/dmlif.h | 48 +- tools/dmldriver/tpchrf2.cpp | 77 +- tools/dmldriver/tpchrf2.h | 33 +- tools/editem/editem.cpp | 1559 +- tools/fixdate/fixdate.cpp | 152 +- tools/getConfig/main.cpp | 210 +- tools/idb_comp/main.cpp | 374 +- tools/idbmeminfo/idbmeminfo.cpp | 213 +- tools/passwd/cskeys.cpp | 116 +- tools/passwd/cspasswd.cpp | 334 +- tools/passwd/secrets.cpp | 896 +- tools/passwd/secrets.h | 23 +- tools/pingproc/pingproc.cpp | 3745 ++-- tools/qfe/bison-win.cpp | 2902 ++- tools/qfe/bison-win.h | 58 +- tools/qfe/cli.cpp | 229 +- tools/qfe/cseputils.cpp | 199 +- tools/qfe/cseputils.h | 28 +- tools/qfe/ddlstmts.cpp | 70 +- tools/qfe/ddlstmts.h | 3 +- tools/qfe/lex-win.cpp | 2013 +- tools/qfe/parsequery.cpp | 85 +- tools/qfe/parsequery.h | 1 - tools/qfe/returnedrows.cpp | 105 +- tools/qfe/returnedrows.h | 2 - tools/qfe/sendcsep.cpp | 52 +- tools/qfe/sendcsep.h | 8 +- tools/qfe/server.cpp | 511 +- tools/qfe/socketio.cpp | 253 +- tools/qfe/socketio.h | 6 +- tools/qfe/socktype.h | 1 - .../BRM_saves_em_system_tables_blob.h | 567 +- tools/rebuildEM/main.cpp | 215 +- tools/rebuildEM/rebuildEM.cpp | 699 +- tools/rebuildEM/rebuildEM.h | 287 +- tools/rgprint/rgprint.cpp | 25 +- tools/sendPlan/sendplan.cpp | 587 +- tools/setConfig/main.cpp | 197 +- tools/valprint/valprint.cpp | 32 +- tools/viewtablelock/viewtablelock.cpp | 589 +- utils/batchloader/batchloader.cpp | 928 +- utils/batchloader/batchloader.h | 154 +- utils/cacheutils/cacheutils.cpp | 406 +- utils/cacheutils/cacheutils.h | 3 +- utils/cloudio/SMComm.cpp | 360 +- utils/cloudio/SMComm.h | 91 +- utils/cloudio/SMDataFile.cpp | 140 +- utils/cloudio/SMDataFile.h | 57 +- utils/cloudio/SMFileFactory.cpp | 153 +- utils/cloudio/SMFileFactory.h | 7 +- utils/cloudio/SMFileSystem.cpp | 113 +- utils/cloudio/SMFileSystem.h | 35 +- utils/cloudio/SocketPool.cpp | 401 +- utils/cloudio/SocketPool.h | 43 +- utils/cloudio/cloud_plugin.cpp | 14 +- utils/cloudio/component_test.cpp | 416 +- utils/cloudio/end_to_end_test.cpp | 335 +- utils/cloudio/sm_exceptions.h | 13 +- utils/common/MonitorProcMem.cpp | 157 +- utils/common/MonitorProcMem.h | 132 +- utils/common/any.hpp | 518 +- utils/common/atomicops.h | 148 +- utils/common/branchpred.h | 5 +- utils/common/cgroupconfigurator.cpp | 681 +- utils/common/cgroupconfigurator.h | 68 +- utils/common/checks.h | 46 +- utils/common/collation.h | 234 +- utils/common/columnwidth.h | 124 +- utils/common/conststring.h | 51 +- utils/common/crashtrace.cpp | 44 +- utils/common/fixedallocator.cpp | 92 +- utils/common/fixedallocator.h | 152 +- utils/common/genericparser.h | 244 +- utils/common/hasher.h | 584 +- utils/common/hashfamily.h | 46 +- utils/common/mcs_basic_types.h | 8 +- utils/common/nullvaluemanip.cpp | 363 +- utils/common/nullvaluemanip.h | 7 +- utils/common/pipe.h | 50 +- utils/common/poolallocator.cpp | 95 +- utils/common/poolallocator.h | 166 +- utils/common/robin_hood.h | 4142 ++-- utils/common/service.h | 142 +- utils/common/simd_sse.h | 1606 +- utils/common/simpleallocator.h | 288 +- utils/common/spinlock.h | 23 +- utils/common/statistics.cpp | 606 +- utils/common/statistics.h | 111 +- utils/common/stlpoolallocator.h | 144 +- utils/common/syncstream.h | 158 +- utils/common/threadnaming.cpp | 22 +- utils/common/threadnaming.h | 6 +- utils/common/utils_utf8.cpp | 15 +- utils/common/utils_utf8.h | 138 +- utils/common/vlarray.h | 155 +- utils/common/widedecimalutils.h | 42 +- utils/compress/idbcompress.cpp | 632 +- utils/compress/idbcompress.h | 682 +- utils/configcpp/configcpp.cpp | 806 +- utils/configcpp/configcpp.h | 316 +- utils/configcpp/configstream.cpp | 35 +- utils/configcpp/configstream.h | 40 +- utils/configcpp/md5/md32_common.h | 358 +- utils/configcpp/md5/md5.h | 39 +- utils/configcpp/md5/md5_locl.h | 96 +- utils/configcpp/resource.h | 8 +- utils/configcpp/tdriver.cpp | 625 +- utils/configcpp/tdriver2.cpp | 130 +- utils/configcpp/writeonce.h | 159 +- utils/configcpp/xmlparser.cpp | 335 +- utils/configcpp/xmlparser.h | 46 +- utils/dataconvert/dataconvert.cpp | 5638 +++-- utils/dataconvert/dataconvert.h | 2079 +- utils/dataconvert/tdriver.cpp | 2298 +- utils/ddlcleanup/ddlcleanuputil.cpp | 389 +- utils/ddlcleanup/ddlcleanuputil.h | 2 - utils/funcexp/func_abs.cpp | 68 +- utils/funcexp/func_add_time.cpp | 424 +- utils/funcexp/func_ascii.cpp | 28 +- utils/funcexp/func_between.cpp | 598 +- utils/funcexp/func_bitwise.cpp | 714 +- utils/funcexp/func_case.cpp | 1087 +- utils/funcexp/func_cast.cpp | 2663 ++- utils/funcexp/func_ceil.cpp | 1021 +- utils/funcexp/func_char.cpp | 240 +- utils/funcexp/func_char_length.cpp | 146 +- utils/funcexp/func_coalesce.cpp | 280 +- utils/funcexp/func_concat.cpp | 50 +- utils/funcexp/func_concat_oracle.cpp | 57 +- utils/funcexp/func_concat_ws.cpp | 77 +- utils/funcexp/func_conv.cpp | 344 +- utils/funcexp/func_convert_tz.cpp | 310 +- utils/funcexp/func_crc32.cpp | 51 +- utils/funcexp/func_date.cpp | 197 +- utils/funcexp/func_date_add.cpp | 1445 +- utils/funcexp/func_date_format.cpp | 658 +- utils/funcexp/func_day.cpp | 207 +- utils/funcexp/func_dayname.cpp | 267 +- utils/funcexp/func_dayofweek.cpp | 249 +- utils/funcexp/func_dayofyear.cpp | 238 +- utils/funcexp/func_decode.cpp | 80 +- utils/funcexp/func_decode_oracle.cpp | 883 +- utils/funcexp/func_div.cpp | 78 +- utils/funcexp/func_elt.cpp | 98 +- utils/funcexp/func_encode.cpp | 83 +- utils/funcexp/func_exp.cpp | 118 +- utils/funcexp/func_extract.cpp | 358 +- utils/funcexp/func_find_in_set.cpp | 150 +- utils/funcexp/func_floor.cpp | 872 +- utils/funcexp/func_from_days.cpp | 61 +- utils/funcexp/func_from_unixtime.cpp | 274 +- utils/funcexp/func_get_format.cpp | 133 +- utils/funcexp/func_greatest.cpp | 266 +- utils/funcexp/func_hex.cpp | 173 +- utils/funcexp/func_hour.cpp | 218 +- utils/funcexp/func_idbpartition.cpp | 36 +- utils/funcexp/func_if.cpp | 350 +- utils/funcexp/func_ifnull.cpp | 237 +- utils/funcexp/func_in.cpp | 614 +- utils/funcexp/func_inet_aton.cpp | 372 +- utils/funcexp/func_inet_ntoa.cpp | 272 +- utils/funcexp/func_insert.cpp | 103 +- utils/funcexp/func_instr.cpp | 96 +- utils/funcexp/func_isnull.cpp | 74 +- utils/funcexp/func_last_day.cpp | 298 +- utils/funcexp/func_lcase.cpp | 49 +- utils/funcexp/func_least.cpp | 233 +- utils/funcexp/func_left.cpp | 69 +- utils/funcexp/func_length.cpp | 35 +- utils/funcexp/func_lpad.cpp | 165 +- utils/funcexp/func_ltrim.cpp | 103 +- utils/funcexp/func_ltrim_oracle.cpp | 111 +- utils/funcexp/func_makedate.cpp | 248 +- utils/funcexp/func_maketime.cpp | 231 +- utils/funcexp/func_math.cpp | 4037 ++-- utils/funcexp/func_md5.cpp | 492 +- utils/funcexp/func_microsecond.cpp | 172 +- utils/funcexp/func_minute.cpp | 186 +- utils/funcexp/func_mod.cpp | 877 +- utils/funcexp/func_month.cpp | 206 +- utils/funcexp/func_monthname.cpp | 267 +- utils/funcexp/func_nullif.cpp | 1766 +- utils/funcexp/func_period_add.cpp | 106 +- utils/funcexp/func_period_diff.cpp | 120 +- utils/funcexp/func_pow.cpp | 123 +- utils/funcexp/func_quarter.cpp | 197 +- utils/funcexp/func_quote.cpp | 87 +- utils/funcexp/func_rand.cpp | 84 +- utils/funcexp/func_regexp.cpp | 362 +- utils/funcexp/func_repeat.cpp | 78 +- utils/funcexp/func_replace.cpp | 257 +- utils/funcexp/func_replace_oracle.cpp | 251 +- utils/funcexp/func_reverse.cpp | 103 +- utils/funcexp/func_right.cpp | 66 +- utils/funcexp/func_round.cpp | 1203 +- utils/funcexp/func_rpad.cpp | 166 +- utils/funcexp/func_rtrim.cpp | 223 +- utils/funcexp/func_rtrim_oracle.cpp | 227 +- utils/funcexp/func_sec_to_time.cpp | 300 +- utils/funcexp/func_second.cpp | 200 +- utils/funcexp/func_sha.cpp | 458 +- utils/funcexp/func_sign.cpp | 61 +- utils/funcexp/func_space.cpp | 44 +- utils/funcexp/func_str_to_date.cpp | 406 +- utils/funcexp/func_strcmp.cpp | 58 +- utils/funcexp/func_substr.cpp | 112 +- utils/funcexp/func_substring_index.cpp | 304 +- utils/funcexp/func_sysdate.cpp | 90 +- utils/funcexp/func_time.cpp | 240 +- utils/funcexp/func_time_format.cpp | 372 +- utils/funcexp/func_time_to_sec.cpp | 291 +- utils/funcexp/func_timediff.cpp | 542 +- utils/funcexp/func_timestampdiff.cpp | 277 +- utils/funcexp/func_to_days.cpp | 207 +- utils/funcexp/func_trim.cpp | 239 +- utils/funcexp/func_trim_oracle.cpp | 238 +- utils/funcexp/func_truncate.cpp | 1213 +- utils/funcexp/func_ucase.cpp | 58 +- utils/funcexp/func_unhex.cpp | 100 +- utils/funcexp/func_unix_timestamp.cpp | 321 +- utils/funcexp/func_week.cpp | 254 +- utils/funcexp/func_weekday.cpp | 248 +- utils/funcexp/func_year.cpp | 202 +- utils/funcexp/func_yearweek.cpp | 261 +- utils/funcexp/funcexp.cpp | 877 +- utils/funcexp/funcexp.h | 108 +- utils/funcexp/funcexpwrapper.cpp | 89 +- utils/funcexp/funcexpwrapper.h | 41 +- utils/funcexp/funchelpers.h | 1059 +- utils/funcexp/functor.cpp | 441 +- utils/funcexp/functor.h | 365 +- utils/funcexp/functor_all.h | 777 +- utils/funcexp/functor_bool.h | 437 +- utils/funcexp/functor_dtm.h | 715 +- utils/funcexp/functor_export.h | 74 +- utils/funcexp/functor_int.h | 964 +- utils/funcexp/functor_real.h | 1198 +- utils/funcexp/functor_str.h | 1445 +- utils/funcexp/sha.h | 88 +- utils/funcexp/sql_crypt.cpp | 54 +- utils/funcexp/sql_crypt.h | 36 +- utils/funcexp/tdriver.cpp | 573 +- utils/funcexp/timeextract.h | 933 +- utils/funcexp/utf8.h | 3 +- utils/funcexp/utf8/checked.h | 493 +- utils/funcexp/utf8/core.h | 313 +- utils/funcexp/utf8/unchecked.h | 294 +- utils/idbdatafile/BufferedFile.cpp | 317 +- utils/idbdatafile/BufferedFile.h | 43 +- utils/idbdatafile/BufferedFileFactory.h | 15 +- utils/idbdatafile/FileFactoryBase.h | 12 +- utils/idbdatafile/IDBDataFile.cpp | 38 +- utils/idbdatafile/IDBDataFile.h | 344 +- utils/idbdatafile/IDBFactory.cpp | 125 +- utils/idbdatafile/IDBFactory.h | 122 +- utils/idbdatafile/IDBFileSystem.cpp | 6 +- utils/idbdatafile/IDBFileSystem.h | 208 +- utils/idbdatafile/IDBLogger.cpp | 229 +- utils/idbdatafile/IDBLogger.h | 53 +- utils/idbdatafile/IDBPolicy.cpp | 398 +- utils/idbdatafile/IDBPolicy.h | 285 +- utils/idbdatafile/PosixFileSystem.cpp | 382 +- utils/idbdatafile/PosixFileSystem.h | 32 +- utils/idbdatafile/UnbufferedFile.cpp | 364 +- utils/idbdatafile/UnbufferedFile.h | 41 +- utils/idbdatafile/UnbufferedFileFactory.h | 15 +- utils/idbdatafile/largefile64.h | 12 +- utils/idbdatafile/tdriver.cpp | 2544 +-- utils/idbdatafile/tdriver1.cpp | 11 +- utils/idbdatafile/utility.h | 151 +- utils/joiner/btree.h | 4611 ++-- utils/joiner/btree_container.h | 732 +- utils/joiner/btree_map.h | 143 +- utils/joiner/btree_set.h | 130 +- utils/joiner/joiner.cpp | 57 +- utils/joiner/joiner.h | 265 +- utils/joiner/joinpartition.cpp | 1224 +- utils/joiner/joinpartition.h | 248 +- utils/joiner/tuplejoiner.cpp | 3182 ++- utils/joiner/tuplejoiner.h | 917 +- utils/libmysql_client/libmysql_client.cpp | 139 +- utils/libmysql_client/libmysql_client.h | 111 +- utils/loggingcpp/errorcodes.cpp | 125 +- utils/loggingcpp/errorcodes.h | 123 +- utils/loggingcpp/exceptclasses.h | 317 +- utils/loggingcpp/idberrorinfo.cpp | 223 +- utils/loggingcpp/idberrorinfo.h | 41 +- utils/loggingcpp/logger.cpp | 107 +- utils/loggingcpp/logger.h | 82 +- utils/loggingcpp/loggingid.h | 30 +- utils/loggingcpp/message.cpp | 216 +- utils/loggingcpp/messagelog.cpp | 279 +- utils/loggingcpp/messagelog.h | 165 +- utils/loggingcpp/messageobj.h | 208 +- utils/loggingcpp/sqllogger.cpp | 42 +- utils/loggingcpp/sqllogger.h | 26 +- utils/loggingcpp/stopwatch.cpp | 259 +- utils/loggingcpp/stopwatch.h | 107 +- utils/loggingcpp/tdriver.cpp | 645 +- utils/messageqcpp/bytestream.cpp | 747 +- utils/messageqcpp/bytestream.h | 989 +- utils/messageqcpp/bytestreampool.cpp | 80 +- utils/messageqcpp/bytestreampool.h | 36 +- utils/messageqcpp/cli.cpp | 43 +- utils/messageqcpp/compressed_iss.cpp | 279 +- utils/messageqcpp/compressed_iss.h | 44 +- utils/messageqcpp/inetstreamsocket.cpp | 1574 +- utils/messageqcpp/inetstreamsocket.h | 350 +- utils/messageqcpp/iosocket.cpp | 57 +- utils/messageqcpp/iosocket.h | 326 +- utils/messageqcpp/messagequeue.cpp | 428 +- utils/messageqcpp/messagequeue.h | 395 +- utils/messageqcpp/messagequeuepool.cpp | 255 +- utils/messageqcpp/messagequeuepool.h | 40 +- utils/messageqcpp/serializeable.h | 33 +- utils/messageqcpp/serversocket.h | 164 +- utils/messageqcpp/socket.h | 240 +- utils/messageqcpp/socketclosed.h | 38 +- utils/messageqcpp/socketparms.cpp | 36 +- utils/messageqcpp/socketparms.h | 158 +- utils/messageqcpp/srv.cpp | 45 +- utils/messageqcpp/tdriver.cpp | 3117 ++- utils/multicast/impl.cpp | 5638 +++-- utils/multicast/impl.h | 31 +- utils/multicast/mc_fifo.h | 14 +- utils/multicast/multicast.cpp | 49 +- utils/multicast/multicast.h | 141 +- utils/multicast/socklib.h | 54 +- utils/multicast/statistics.h | 11 +- utils/multicast/threads.h | 148 +- utils/multicast/udp-sender.h | 11 +- utils/multicast/udpc-protoc.h | 176 +- utils/multicast/udpcast.h | 132 +- utils/multicast/util.h | 13 +- utils/querystats/querystats.cpp | 358 +- utils/querystats/querystats.h | 181 +- utils/querytele/QueryTeleService.cpp | 1191 +- utils/querytele/QueryTeleService.h | 677 +- utils/querytele/querystepparms.h | 22 +- utils/querytele/querytele.cpp | 22 +- utils/querytele/querytele.h | 1 - utils/querytele/querytele_constants.cpp | 4 +- utils/querytele/querytele_constants.h | 8 +- utils/querytele/querytele_types.cpp | 2394 +- utils/querytele/querytele_types.h | 1393 +- utils/querytele/queryteleclient.cpp | 307 +- utils/querytele/queryteleclient.h | 77 +- utils/querytele/queryteleprotoimpl.cpp | 585 +- utils/querytele/queryteleprotoimpl.h | 34 +- utils/querytele/queryteleserverparms.h | 28 +- utils/querytele/serverdriver.cpp | 260 +- utils/querytele/tdriver.cpp | 61 +- utils/querytele/telestats.h | 285 +- utils/regr/corr.cpp | 212 +- utils/regr/corr.h | 39 +- utils/regr/covar_pop.cpp | 160 +- utils/regr/covar_pop.h | 39 +- utils/regr/covar_samp.cpp | 169 +- utils/regr/covar_samp.h | 39 +- utils/regr/moda.cpp | 806 +- utils/regr/moda.h | 368 +- utils/regr/modamysql.cpp | 271 +- utils/regr/regr_avgx.cpp | 128 +- utils/regr/regr_avgx.h | 39 +- utils/regr/regr_avgy.cpp | 124 +- utils/regr/regr_avgy.h | 39 +- utils/regr/regr_count.cpp | 126 +- utils/regr/regr_count.h | 39 +- utils/regr/regr_intercept.cpp | 181 +- utils/regr/regr_intercept.h | 39 +- utils/regr/regr_r2.cpp | 214 +- utils/regr/regr_r2.h | 39 +- utils/regr/regr_slope.cpp | 184 +- utils/regr/regr_slope.h | 39 +- utils/regr/regr_sxx.cpp | 144 +- utils/regr/regr_sxx.h | 39 +- utils/regr/regr_sxy.cpp | 154 +- utils/regr/regr_sxy.h | 39 +- utils/regr/regr_syy.cpp | 144 +- utils/regr/regr_syy.h | 39 +- utils/regr/regrmysql.cpp | 2580 ++- utils/rowgroup/rowaggregation.cpp | 7457 ++++--- utils/rowgroup/rowaggregation.h | 1427 +- utils/rowgroup/rowgroup.cpp | 2623 ++- utils/rowgroup/rowgroup.h | 2630 ++- utils/rowgroup/rowstorage.cpp | 673 +- utils/rowgroup/rowstorage.h | 67 +- utils/rwlock/rwlock.cpp | 1046 +- utils/rwlock/rwlock.h | 324 +- utils/rwlock/rwlock_local.cpp | 280 +- utils/rwlock/rwlock_local.h | 220 +- utils/rwlock/tdriver-rw.cpp | 408 +- utils/rwlock/tdriver.cpp | 243 +- utils/startup/installdir.cpp | 105 +- utils/startup/installdir.h | 32 +- utils/testbc/blockcacheclient.cpp | 44 +- utils/testbc/blockcacheclient.h | 100 +- utils/testbc/blockrequestprocessor.cpp | 250 +- utils/testbc/blockrequestprocessor.h | 136 +- utils/testbc/fileblockrequestqueue.cpp | 60 +- utils/testbc/fileblockrequestqueue.h | 108 +- utils/testbc/filebuffer.cpp | 58 +- utils/testbc/filebuffer.h | 217 +- utils/testbc/filebuffermgr.cpp | 385 +- utils/testbc/filebuffermgr.h | 204 +- utils/testbc/filerequest.cpp | 86 +- utils/testbc/filerequest.h | 377 +- utils/testbc/iomanager.cpp | 393 +- utils/testbc/iomanager.h | 103 +- utils/testbc/logger.cpp | 47 +- utils/testbc/logger.h | 42 +- utils/testbc/stats.cpp | 310 +- utils/testbc/stats.h | 50 +- utils/testbc/testbc.cpp | 435 +- utils/testbc/testbc2.cpp | 425 +- utils/testbc/testbc3.cpp | 498 +- utils/threadpool/prioritythreadpool.cpp | 465 +- utils/threadpool/prioritythreadpool.h | 206 +- utils/threadpool/tdriver.cpp | 131 +- utils/threadpool/threadpool.cpp | 815 +- utils/threadpool/threadpool.h | 583 +- utils/threadpool/tp.cpp | 167 +- utils/threadpool/weightedthreadpool.cpp | 562 +- utils/threadpool/weightedthreadpool.h | 351 +- utils/threadpool/wtp.cpp | 65 +- utils/udfsdk/allnull.cpp | 75 +- utils/udfsdk/allnull.h | 223 +- utils/udfsdk/avg_mode.cpp | 209 +- utils/udfsdk/avg_mode.h | 345 +- utils/udfsdk/avgx.cpp | 128 +- utils/udfsdk/avgx.h | 39 +- utils/udfsdk/distinct_count.cpp | 92 +- utils/udfsdk/distinct_count.h | 317 +- utils/udfsdk/mcsv1_udaf.cpp | 336 +- utils/udfsdk/mcsv1_udaf.h | 1275 +- utils/udfsdk/median.cpp | 287 +- utils/udfsdk/median.h | 345 +- utils/udfsdk/resource.h | 8 +- utils/udfsdk/ssq.cpp | 145 +- utils/udfsdk/ssq.h | 273 +- utils/udfsdk/udfmysql.cpp | 856 +- utils/udfsdk/udfsdk.cpp | 409 +- utils/udfsdk/udfsdk.h | 426 +- utils/windowfunction/framebound.cpp | 57 +- utils/windowfunction/framebound.h | 179 +- utils/windowfunction/frameboundrange.cpp | 581 +- utils/windowfunction/frameboundrange.h | 285 +- utils/windowfunction/frameboundrow.cpp | 116 +- utils/windowfunction/frameboundrow.h | 184 +- utils/windowfunction/idborderby.cpp | 1308 +- utils/windowfunction/idborderby.h | 484 +- utils/windowfunction/wf_count.cpp | 207 +- utils/windowfunction/wf_count.h | 37 +- utils/windowfunction/wf_lead_lag.cpp | 449 +- utils/windowfunction/wf_lead_lag.h | 49 +- utils/windowfunction/wf_min_max.cpp | 233 +- utils/windowfunction/wf_min_max.h | 36 +- utils/windowfunction/wf_nth_value.cpp | 387 +- utils/windowfunction/wf_nth_value.h | 45 +- utils/windowfunction/wf_ntile.cpp | 141 +- utils/windowfunction/wf_ntile.h | 37 +- utils/windowfunction/wf_percentile.cpp | 576 +- utils/windowfunction/wf_percentile.h | 39 +- utils/windowfunction/wf_ranking.cpp | 147 +- utils/windowfunction/wf_ranking.h | 35 +- utils/windowfunction/wf_row_number.cpp | 44 +- utils/windowfunction/wf_row_number.h | 33 +- utils/windowfunction/wf_stats.cpp | 330 +- utils/windowfunction/wf_stats.h | 40 +- utils/windowfunction/wf_sum_avg.cpp | 414 +- utils/windowfunction/wf_sum_avg.h | 52 +- utils/windowfunction/wf_udaf.cpp | 2131 +- utils/windowfunction/wf_udaf.h | 117 +- utils/windowfunction/windowframe.cpp | 51 +- utils/windowfunction/windowframe.h | 157 +- utils/windowfunction/windowfunction.cpp | 358 +- utils/windowfunction/windowfunction.h | 112 +- utils/windowfunction/windowfunctiontype.cpp | 1097 +- utils/windowfunction/windowfunctiontype.h | 405 +- utils/winport/WinSyslog.cpp | 450 +- utils/winport/WinSyslog.h | 106 +- utils/winport/bootstrap.cpp | 487 +- utils/winport/fixup.cpp | 956 +- utils/winport/glob.h | 51 +- utils/winport/grepit.cpp | 24 +- utils/winport/idb_getopt.cpp | 260 +- utils/winport/idbregistry.cpp | 38 +- utils/winport/main.cpp | 121 +- utils/winport/poll.h | 106 +- utils/winport/resource.h | 8 +- utils/winport/sedit.cpp | 27 +- utils/winport/sedit.h | 2 +- utils/winport/sys/time.h | 1 - utils/winport/sysinfo.cpp | 114 +- utils/winport/sysinfo.h | 28 +- utils/winport/syslog.h | 69 +- utils/winport/unistd.h | 74 +- utils/winport/winfinidb.cpp | 760 +- utils/winport/winport.cpp | 178 +- versioning/BRM/autoincrementmanager.cpp | 127 +- versioning/BRM/autoincrementmanager.h | 59 +- versioning/BRM/autoinctest.cpp | 218 +- versioning/BRM/blockresolutionmanager.cpp | 177 +- versioning/BRM/blockresolutionmanager.h | 118 +- versioning/BRM/brmshmimpl.cpp | 283 +- versioning/BRM/brmshmimpl.h | 61 +- versioning/BRM/brmtypes.cpp | 594 +- versioning/BRM/brmtypes.h | 530 +- versioning/BRM/copylocks.cpp | 384 +- versioning/BRM/copylocks.h | 161 +- versioning/BRM/copylocktest.cpp | 121 +- versioning/BRM/cvt_em.cpp | 391 +- versioning/BRM/dbrm.cpp | 6105 +++--- versioning/BRM/dbrm.h | 1811 +- versioning/BRM/dbrmctl.cpp | 179 +- versioning/BRM/extentmap.cpp | 8589 ++++---- versioning/BRM/extentmap.h | 1509 +- versioning/BRM/lbidresourcegraph.cpp | 442 +- versioning/BRM/lbidresourcegraph.h | 41 +- versioning/BRM/load_brm.cpp | 128 +- versioning/BRM/load_brm64.cpp | 50 +- versioning/BRM/load_brm_from_file.cpp | 280 +- versioning/BRM/load_em.cpp | 44 +- versioning/BRM/lock_grabber.cpp | 84 +- versioning/BRM/lock_state.cpp | 44 +- versioning/BRM/logicalpartition.cpp | 32 +- versioning/BRM/logicalpartition.h | 59 +- versioning/BRM/masterdbrmnode.cpp | 4118 ++-- versioning/BRM/masterdbrmnode.h | 305 +- versioning/BRM/masternode.cpp | 221 +- versioning/BRM/mastersegmenttable.cpp | 258 +- versioning/BRM/mastersegmenttable.h | 249 +- versioning/BRM/oidserver.cpp | 1058 +- versioning/BRM/oidserver.h | 265 +- versioning/BRM/reset_locks.cpp | 224 +- versioning/BRM/resource.h | 8 +- versioning/BRM/resourcenode.cpp | 42 +- versioning/BRM/resourcenode.h | 56 +- versioning/BRM/rgnode.cpp | 54 +- versioning/BRM/rgnode.h | 41 +- versioning/BRM/rollback.cpp | 101 +- versioning/BRM/rwlockmonitor.cpp | 191 +- versioning/BRM/rwlockmonitor.h | 34 +- versioning/BRM/save_brm.cpp | 118 +- versioning/BRM/sessionmanagerserver.cpp | 503 +- versioning/BRM/sessionmanagerserver.h | 351 +- versioning/BRM/shmkeys.cpp | 46 +- versioning/BRM/shmkeys.h | 56 +- versioning/BRM/slavecomm.cpp | 3207 ++- versioning/BRM/slavecomm.h | 145 +- versioning/BRM/slavedbrmnode.cpp | 2181 +- versioning/BRM/slavedbrmnode.h | 727 +- versioning/BRM/slavenode.cpp | 220 +- versioning/BRM/tablelockserver.cpp | 376 +- versioning/BRM/tablelockserver.h | 43 +- versioning/BRM/tablelocktest.cpp | 324 +- versioning/BRM/tdriver-brm.cpp | 4452 ++-- versioning/BRM/tdriver-dbrm.cpp | 2078 +- versioning/BRM/tdriver-dbrm2.cpp | 3502 ++- versioning/BRM/tdriver-load.cpp | 1622 +- versioning/BRM/tdriver.cpp | 3596 ++- versioning/BRM/tracer.cpp | 125 +- versioning/BRM/tracer.h | 228 +- versioning/BRM/transactionnode.cpp | 21 +- versioning/BRM/transactionnode.h | 37 +- versioning/BRM/undoable.cpp | 27 +- versioning/BRM/undoable.h | 19 +- versioning/BRM/vbbm.cpp | 1626 +- versioning/BRM/vbbm.h | 244 +- versioning/BRM/vss.cpp | 1923 +- versioning/BRM/vss.h | 280 +- writeengine/bulk/cpimport.cpp | 2372 +- writeengine/bulk/parsetest.cpp | 357 +- writeengine/bulk/resource.h | 8 +- writeengine/bulk/we_brmreporter.cpp | 487 +- writeengine/bulk/we_brmreporter.h | 160 +- writeengine/bulk/we_bulkload.cpp | 2325 +- writeengine/bulk/we_bulkload.h | 629 +- writeengine/bulk/we_bulkloadbuffer.cpp | 5857 +++-- writeengine/bulk/we_bulkloadbuffer.h | 621 +- writeengine/bulk/we_bulkstatus.cpp | 8 +- writeengine/bulk/we_bulkstatus.h | 47 +- writeengine/bulk/we_colbuf.cpp | 134 +- writeengine/bulk/we_colbuf.h | 144 +- writeengine/bulk/we_colbufcompressed.cpp | 1148 +- writeengine/bulk/we_colbufcompressed.h | 115 +- writeengine/bulk/we_colbufmgr.cpp | 925 +- writeengine/bulk/we_colbufmgr.h | 352 +- writeengine/bulk/we_colbufmgrdctnry.cpp | 158 +- writeengine/bulk/we_colbufsec.cpp | 69 +- writeengine/bulk/we_colbufsec.h | 118 +- writeengine/bulk/we_colextinf.cpp | 474 +- writeengine/bulk/we_colextinf.h | 302 +- writeengine/bulk/we_colopbulk.cpp | 50 +- writeengine/bulk/we_colopbulk.h | 27 +- writeengine/bulk/we_columnautoinc.cpp | 454 +- writeengine/bulk/we_columnautoinc.h | 94 +- writeengine/bulk/we_columninfo.cpp | 2700 ++- writeengine/bulk/we_columninfo.h | 822 +- writeengine/bulk/we_columninfocompressed.cpp | 788 +- writeengine/bulk/we_columninfocompressed.h | 88 +- writeengine/bulk/we_extentstripealloc.cpp | 362 +- writeengine/bulk/we_extentstripealloc.h | 196 +- writeengine/bulk/we_tableinfo.cpp | 3728 ++-- writeengine/bulk/we_tableinfo.h | 966 +- writeengine/bulk/we_tempxmlgendata.cpp | 32 +- writeengine/bulk/we_tempxmlgendata.h | 27 +- writeengine/bulk/we_workers.cpp | 724 +- writeengine/client/we_clients.cpp | 801 +- writeengine/client/we_clients.h | 227 +- writeengine/client/we_ddlcommandclient.cpp | 87 +- writeengine/client/we_ddlcommandclient.h | 33 +- writeengine/client/we_dmlcommandclient.cpp | 10 +- writeengine/client/we_dmlcommandclient.h | 17 +- writeengine/dictionary/qdctnry.cpp | 1451 +- writeengine/dictionary/tdctnry.cpp | 1183 +- writeengine/dictionary/we_dctnry.cpp | 1975 +- writeengine/dictionary/we_dctnry.h | 504 +- writeengine/dictionary/we_dctnrystore.cpp | 161 +- writeengine/dictionary/we_dctnrystore.h | 297 +- writeengine/index/qindex.cpp | 2257 +- writeengine/index/tindex.cpp | 8114 ++++--- writeengine/index/we_freemgr.cpp | 2637 +-- writeengine/index/we_freemgr.h | 143 +- writeengine/index/we_indexlist.cpp | 391 +- writeengine/index/we_indexlist.h | 788 +- writeengine/index/we_indexlist_common.cpp | 1663 +- .../index/we_indexlist_find_delete.cpp | 1687 +- .../index/we_indexlist_multiple_narray.cpp | 1047 +- writeengine/index/we_indexlist_narray.cpp | 474 +- .../index/we_indexlist_update_hdr_sub.cpp | 396 +- writeengine/index/we_indextree.cpp | 1711 +- writeengine/index/we_indextree.h | 496 +- writeengine/redistribute/we_redistribute.cpp | 44 +- writeengine/redistribute/we_redistribute.h | 22 +- .../redistribute/we_redistributecontrol.cpp | 1090 +- .../redistribute/we_redistributecontrol.h | 109 +- .../we_redistributecontrolthread.cpp | 1330 +- .../we_redistributecontrolthread.h | 114 +- writeengine/redistribute/we_redistributedef.h | 244 +- .../we_redistributeworkerthread.cpp | 2436 +-- .../we_redistributeworkerthread.h | 138 +- writeengine/resource.h | 8 +- writeengine/server/resource.h | 8 +- writeengine/server/we_brmrprtparser.cpp | 204 +- writeengine/server/we_brmrprtparser.h | 26 +- writeengine/server/we_cleartablelockcmd.cpp | 164 +- writeengine/server/we_cleartablelockcmd.h | 47 +- writeengine/server/we_cpifeederthread.cpp | 163 +- writeengine/server/we_cpifeederthread.h | 87 +- writeengine/server/we_dataloader.cpp | 2260 +- writeengine/server/we_dataloader.h | 294 +- writeengine/server/we_ddlcommandproc.cpp | 9138 ++++---- writeengine/server/we_ddlcommandproc.h | 154 +- writeengine/server/we_ddlcommon.h | 346 +- writeengine/server/we_dmlcommandproc.cpp | 8863 ++++---- writeengine/server/we_dmlcommandproc.h | 165 +- writeengine/server/we_getfilesizes.cpp | 597 +- writeengine/server/we_getfilesizes.h | 68 +- writeengine/server/we_message_handlers.h | 10 +- writeengine/server/we_messages.h | 175 +- writeengine/server/we_msg1.cpp | 12 +- writeengine/server/we_msg2.cpp | 12 +- writeengine/server/we_observer.cpp | 54 +- writeengine/server/we_observer.h | 41 +- writeengine/server/we_readthread.cpp | 1496 +- writeengine/server/we_readthread.h | 136 +- writeengine/server/we_server.cpp | 437 +- writeengine/shared/tconfig.cpp | 123 +- writeengine/shared/we_blockop.cpp | 153 +- writeengine/shared/we_blockop.h | 157 +- writeengine/shared/we_brm.cpp | 2566 ++- writeengine/shared/we_brm.h | 1001 +- writeengine/shared/we_bulkrollbackfile.cpp | 548 +- writeengine/shared/we_bulkrollbackfile.h | 254 +- .../shared/we_bulkrollbackfilecompressed.cpp | 1512 +- .../shared/we_bulkrollbackfilecompressed.h | 172 +- .../we_bulkrollbackfilecompressedhdfs.cpp | 255 +- .../we_bulkrollbackfilecompressedhdfs.h | 146 +- writeengine/shared/we_bulkrollbackmgr.cpp | 2863 ++- writeengine/shared/we_bulkrollbackmgr.h | 301 +- writeengine/shared/we_cache.cpp | 525 +- writeengine/shared/we_cache.h | 292 +- writeengine/shared/we_chunkmanager.cpp | 3860 ++-- writeengine/shared/we_chunkmanager.h | 477 +- writeengine/shared/we_config.cpp | 568 +- writeengine/shared/we_config.h | 250 +- writeengine/shared/we_confirmhdfsdbfile.cpp | 1217 +- writeengine/shared/we_confirmhdfsdbfile.h | 145 +- writeengine/shared/we_convertor.cpp | 1196 +- writeengine/shared/we_convertor.h | 178 +- writeengine/shared/we_dbfileop.cpp | 474 +- writeengine/shared/we_dbfileop.h | 292 +- writeengine/shared/we_dbrootextenttracker.cpp | 720 +- writeengine/shared/we_dbrootextenttracker.h | 277 +- writeengine/shared/we_define.cpp | 488 +- writeengine/shared/we_define.h | 530 +- writeengine/shared/we_fileop.cpp | 3533 ++- writeengine/shared/we_fileop.h | 839 +- writeengine/shared/we_index.h | 570 +- writeengine/shared/we_log.cpp | 213 +- writeengine/shared/we_log.h | 116 +- writeengine/shared/we_macro.h | 43 +- writeengine/shared/we_obj.h | 109 +- writeengine/shared/we_rbmetawriter.cpp | 2013 +- writeengine/shared/we_rbmetawriter.h | 404 +- writeengine/shared/we_simplesyslog.cpp | 96 +- writeengine/shared/we_simplesyslog.h | 56 +- writeengine/shared/we_stats.cpp | 181 +- writeengine/shared/we_stats.h | 247 +- writeengine/shared/we_type.h | 754 +- writeengine/shared/we_typeext.h | 85 +- writeengine/splitter/resource.h | 8 +- writeengine/splitter/we_brmupdater.cpp | 1037 +- writeengine/splitter/we_brmupdater.h | 89 +- writeengine/splitter/we_cmdargs.cpp | 3149 ++- writeengine/splitter/we_cmdargs.h | 595 +- writeengine/splitter/we_filereadthread.cpp | 937 +- writeengine/splitter/we_filereadthread.h | 227 +- writeengine/splitter/we_respreadthread.cpp | 24 +- writeengine/splitter/we_respreadthread.h | 22 +- writeengine/splitter/we_sdhandler.cpp | 4116 ++-- writeengine/splitter/we_sdhandler.h | 612 +- writeengine/splitter/we_splclient.cpp | 577 +- writeengine/splitter/we_splclient.h | 663 +- writeengine/splitter/we_splitterapp.cpp | 771 +- writeengine/splitter/we_splitterapp.h | 90 +- writeengine/splitter/we_tablelockgrabber.cpp | 125 +- writeengine/splitter/we_tablelockgrabber.h | 38 +- writeengine/splitter/we_xmlgetter.cpp | 373 +- writeengine/splitter/we_xmlgetter.h | 52 +- writeengine/wrapper/tdriver.cpp | 5678 +++-- writeengine/wrapper/we_colop.cpp | 3002 ++- writeengine/wrapper/we_colop.h | 479 +- writeengine/wrapper/we_colopcompress.cpp | 154 +- writeengine/wrapper/we_colopcompress.h | 233 +- writeengine/wrapper/we_dctnrycompress.cpp | 116 +- writeengine/wrapper/we_dctnrycompress.h | 213 +- writeengine/wrapper/we_tablemetadata.cpp | 89 +- writeengine/wrapper/we_tablemetadata.h | 91 +- writeengine/wrapper/writeengine.cpp | 11215 +++++----- writeengine/wrapper/writeengine.h | 1325 +- writeengine/xml/txml.cpp | 77 +- writeengine/xml/we_xmlgendata.cpp | 88 +- writeengine/xml/we_xmlgendata.h | 109 +- writeengine/xml/we_xmlgenproc.cpp | 664 +- writeengine/xml/we_xmlgenproc.h | 133 +- writeengine/xml/we_xmljob.cpp | 2115 +- writeengine/xml/we_xmljob.h | 197 +- writeengine/xml/we_xmlop.cpp | 211 +- writeengine/xml/we_xmlop.h | 131 +- writeengine/xml/we_xmltag.h | 149 +- 1367 files changed, 394342 insertions(+), 413129 deletions(-) create mode 100644 .clang-format mode change 100755 => 100644 dbcon/joblist/windowfunctionstep.cpp mode change 100755 => 100644 dbcon/mysql/ha_mcs_execplan.cpp mode change 100755 => 100644 storage-manager/include/messageFormat.h mode change 100755 => 100644 utils/cloudio/end_to_end_test.cpp mode change 100755 => 100644 utils/common/any.hpp mode change 100755 => 100644 utils/rowgroup/rowaggregation.cpp mode change 100755 => 100644 utils/udfsdk/mcsv1_udaf.cpp mode change 100755 => 100644 utils/udfsdk/mcsv1_udaf.h diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..20ca4cdc8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,22 @@ +--- +BasedOnStyle: Google +IndentWidth: 2 +BreakBeforeBraces: Allman +ColumnLimit: 110 + +--- +Language: Cpp +AccessModifierOffset: -1 +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +BreakConstructorInitializersBeforeComma: true +ConstructorInitializerIndentWidth: 1 +DerivePointerAlignment: false +IndentCaseLabels: true +NamespaceIndentation: None +PointerAlignment: Left +SortIncludes: false +Standard: Auto diff --git a/datatypes/mcs_data_condition.h b/datatypes/mcs_data_condition.h index 597daf911..c7cbbc93b 100644 --- a/datatypes/mcs_data_condition.h +++ b/datatypes/mcs_data_condition.h @@ -15,13 +15,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef MCS_DATA_CONDITION_H #define MCS_DATA_CONDITION_H namespace datatypes { - /* A subset of SQL Conditions related to data processing. SQLSTATE terminology is used for categories: @@ -31,36 +29,39 @@ namespace datatypes */ class DataCondition { -public: + public: enum Code { // Code Value SQLSTATE - S_SUCCESS = 0, // 00000 - W_STRING_DATA_RIGHT_TRUNCATION = 1 << 1, // 01004 - X_STRING_DATA_RIGHT_TRUNCATION = 1 << 16, // 22001 - X_NUMERIC_VALUE_OUT_OF_RANGE = 1 << 17, // 22003 - X_INVALID_CHARACTER_VALUE_FOR_CAST = 1 << 18, // 22018 + S_SUCCESS = 0, // 00000 + W_STRING_DATA_RIGHT_TRUNCATION = 1 << 1, // 01004 + X_STRING_DATA_RIGHT_TRUNCATION = 1 << 16, // 22001 + X_NUMERIC_VALUE_OUT_OF_RANGE = 1 << 17, // 22003 + X_INVALID_CHARACTER_VALUE_FOR_CAST = 1 << 18, // 22018 }; - DataCondition() - :mError(S_SUCCESS) - { } - DataCondition(Code code) - :mError(code) - { } - DataCondition & operator|=(Code code) + DataCondition() : mError(S_SUCCESS) { - mError= (Code) (mError | code); + } + DataCondition(Code code) : mError(code) + { + } + DataCondition& operator|=(Code code) + { + mError = (Code)(mError | code); return *this; } DataCondition operator&(Code rhs) const { - return DataCondition((Code) (mError & rhs)); + return DataCondition((Code)(mError & rhs)); + } + operator Code() const + { + return mError; } - operator Code () const { return mError; } // Adjust a sigened integer of any size to the range [-absMaxVal , +absMaxVal] - template - void adjustSIntXRange(T & val, T absMaxVal) + template + void adjustSIntXRange(T& val, T absMaxVal) { if (val > absMaxVal) { @@ -74,10 +75,10 @@ public: } } -private: + private: Code mError; }; -} // namespace datatypes +} // namespace datatypes -#endif // MCS_DATA_CONDITION_H +#endif // MCS_DATA_CONDITION_H diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index efcc5f4f4..c33269dbb 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -39,7 +39,6 @@ using namespace std; #include - #include "dataconvert.h" using namespace dataconvert; @@ -56,236 +55,198 @@ using namespace dataconvert; namespace datatypes { - -int128_t -SystemCatalog::TypeAttributesStd::decimal128FromString( - const std::string& value, bool *saturate) const +int128_t SystemCatalog::TypeAttributesStd::decimal128FromString(const std::string& value, + bool* saturate) const { int128_t result = 0; bool pushWarning = false; bool noRoundup = false; - dataconvert::number_int_value(value, - SystemCatalog::DECIMAL, - *this, - pushWarning, - noRoundup, - result, - saturate); + dataconvert::number_int_value(value, SystemCatalog::DECIMAL, *this, pushWarning, noRoundup, + result, saturate); return result; } - -const string & TypeHandlerSInt8::name() const +const string& TypeHandlerSInt8::name() const { - static const string xname= "TINYINT"; + static const string xname = "TINYINT"; return xname; } - -const string & TypeHandlerUInt8::name() const +const string& TypeHandlerUInt8::name() const { - static const string xname= "UTINYINT"; + static const string xname = "UTINYINT"; return xname; } - -const string & TypeHandlerSInt16::name() const +const string& TypeHandlerSInt16::name() const { - static const string xname= "SMALLINT"; + static const string xname = "SMALLINT"; return xname; } - -const string & TypeHandlerUInt16::name() const +const string& TypeHandlerUInt16::name() const { - static const string xname= "USMALLINT"; + static const string xname = "USMALLINT"; return xname; } - -const string & TypeHandlerSInt24::name() const +const string& TypeHandlerSInt24::name() const { - static const string xname= "MEDINT"; + static const string xname = "MEDINT"; return xname; } - -const string & TypeHandlerUInt24::name() const +const string& TypeHandlerUInt24::name() const { - static const string xname= "UMEDINT"; + static const string xname = "UMEDINT"; return xname; } - -const string & TypeHandlerSInt32::name() const +const string& TypeHandlerSInt32::name() const { - static const string xname= "INT"; + static const string xname = "INT"; return xname; } - -const string & TypeHandlerUInt32::name() const +const string& TypeHandlerUInt32::name() const { - static const string xname= "UINT"; + static const string xname = "UINT"; return xname; } - -const string & TypeHandlerSInt64::name() const +const string& TypeHandlerSInt64::name() const { - static const string xname= "BIGINT"; + static const string xname = "BIGINT"; return xname; } - -const string & TypeHandlerUInt64::name() const +const string& TypeHandlerUInt64::name() const { - static const string xname= "UBIGINT"; + static const string xname = "UBIGINT"; return xname; } - -const string & TypeHandlerSFloat::name() const +const string& TypeHandlerSFloat::name() const { - static const string xname= "FLOAT"; + static const string xname = "FLOAT"; return xname; } - -const string & TypeHandlerUFloat::name() const +const string& TypeHandlerUFloat::name() const { - static const string xname= "UFLOAT"; + static const string xname = "UFLOAT"; return xname; } - -const string & TypeHandlerSDouble::name() const +const string& TypeHandlerSDouble::name() const { - static const string xname= "DOUBLE"; + static const string xname = "DOUBLE"; return xname; } - -const string & TypeHandlerUDouble::name() const +const string& TypeHandlerUDouble::name() const { - static const string xname= "UDOUBLE"; + static const string xname = "UDOUBLE"; return xname; } - -const string & TypeHandlerSLongDouble::name() const +const string& TypeHandlerSLongDouble::name() const { - static const string xname= "LONGDOUBLE"; + static const string xname = "LONGDOUBLE"; return xname; } - -const string & TypeHandlerSDecimal64::name() const +const string& TypeHandlerSDecimal64::name() const { - static const string xname= "DECIMAL"; + static const string xname = "DECIMAL"; return xname; } - -const string & TypeHandlerUDecimal64::name() const +const string& TypeHandlerUDecimal64::name() const { - static const string xname= "UDECIMAL"; + static const string xname = "UDECIMAL"; return xname; } - -const string & TypeHandlerSDecimal128::name() const +const string& TypeHandlerSDecimal128::name() const { - static const string xname= "DECIMAL"; + static const string xname = "DECIMAL"; return xname; } - -const string & TypeHandlerUDecimal128::name() const +const string& TypeHandlerUDecimal128::name() const { - static const string xname= "UDECIMAL"; + static const string xname = "UDECIMAL"; return xname; } - -const string & TypeHandlerDate::name() const +const string& TypeHandlerDate::name() const { - static const string xname= "DATE"; + static const string xname = "DATE"; return xname; } - -const string & TypeHandlerDatetime::name() const +const string& TypeHandlerDatetime::name() const { - static const string xname= "DATETIME"; + static const string xname = "DATETIME"; return xname; } - -const string & TypeHandlerTime::name() const +const string& TypeHandlerTime::name() const { - static const string xname= "TIME"; + static const string xname = "TIME"; return xname; } - -const string & TypeHandlerTimestamp::name() const +const string& TypeHandlerTimestamp::name() const { - static const string xname= "TIMESTAMP"; + static const string xname = "TIMESTAMP"; return xname; } - -const string & TypeHandlerChar::name() const +const string& TypeHandlerChar::name() const { - static const string xname= "CHAR"; + static const string xname = "CHAR"; return xname; } - -const string & TypeHandlerVarchar::name() const +const string& TypeHandlerVarchar::name() const { - static const string xname= "VARCHAR"; + static const string xname = "VARCHAR"; return xname; } - -const string & TypeHandlerVarbinary::name() const +const string& TypeHandlerVarbinary::name() const { - static const string xname= "VARBINARY"; + static const string xname = "VARBINARY"; return xname; } - -const string & TypeHandlerBlob::name() const +const string& TypeHandlerBlob::name() const { - static const string xname= "BLOB"; + static const string xname = "BLOB"; return xname; } - -const string & TypeHandlerClob::name() const +const string& TypeHandlerClob::name() const { - static const string xname= "CLOB"; + static const string xname = "CLOB"; return xname; } - -const string & TypeHandlerText::name() const +const string& TypeHandlerText::name() const { - static const string xname= "TEXT"; + static const string xname = "TEXT"; return xname; } - -const string & TypeHandlerBit::name() const +const string& TypeHandlerBit::name() const { - static const string xname= "BIT"; + static const string xname = "BIT"; return xname; } - -TypeHandlerBit mcs_type_handler_bit; +TypeHandlerBit mcs_type_handler_bit; TypeHandlerSInt8 mcs_type_handler_sint8; TypeHandlerSInt16 mcs_type_handler_sint16; @@ -299,11 +260,11 @@ TypeHandlerUInt24 mcs_type_handler_uint24; TypeHandlerUInt32 mcs_type_handler_uint32; TypeHandlerUInt64 mcs_type_handler_uint64; -TypeHandlerSFloat mcs_type_handler_sfloat; +TypeHandlerSFloat mcs_type_handler_sfloat; TypeHandlerSDouble mcs_type_handler_sdouble; TypeHandlerSLongDouble mcs_type_handler_slongdouble; -TypeHandlerUFloat mcs_type_handler_ufloat; +TypeHandlerUFloat mcs_type_handler_ufloat; TypeHandlerUDouble mcs_type_handler_udouble; TypeHandlerSDecimal64 mcs_type_handler_sdecimal64; @@ -312,85 +273,78 @@ TypeHandlerUDecimal64 mcs_type_handler_udecimal64; TypeHandlerSDecimal128 mcs_type_handler_sdecimal128; TypeHandlerUDecimal128 mcs_type_handler_udecimal128; -TypeHandlerDate mcs_type_handler_date; -TypeHandlerTime mcs_type_handler_time; -TypeHandlerDatetime mcs_type_handler_datetime; +TypeHandlerDate mcs_type_handler_date; +TypeHandlerTime mcs_type_handler_time; +TypeHandlerDatetime mcs_type_handler_datetime; TypeHandlerTimestamp mcs_type_handler_timestamp; - -TypeHandlerChar mcs_type_handler_char; -TypeHandlerVarchar mcs_type_handler_varchar; -TypeHandlerText mcs_type_handler_text; -TypeHandlerClob mcs_type_handler_clob; +TypeHandlerChar mcs_type_handler_char; +TypeHandlerVarchar mcs_type_handler_varchar; +TypeHandlerText mcs_type_handler_text; +TypeHandlerClob mcs_type_handler_clob; TypeHandlerVarbinary mcs_type_handler_varbinary; -TypeHandlerBlob mcs_type_handler_blob; +TypeHandlerBlob mcs_type_handler_blob; - -const TypeHandler * -TypeHandler::find(SystemCatalog::ColDataType typeCode, - const SystemCatalog::TypeAttributesStd &ct) +const TypeHandler* TypeHandler::find(SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd& ct) { - switch (typeCode) { - case SystemCatalog::BIT: return &mcs_type_handler_bit; - case SystemCatalog::TINYINT: return &mcs_type_handler_sint8; - case SystemCatalog::SMALLINT: return &mcs_type_handler_sint16; - case SystemCatalog::MEDINT: return &mcs_type_handler_sint24; - case SystemCatalog::INT: return &mcs_type_handler_sint32; - case SystemCatalog::BIGINT: return &mcs_type_handler_sint64; - case SystemCatalog::UTINYINT: return &mcs_type_handler_uint8; - case SystemCatalog::USMALLINT: return &mcs_type_handler_uint16; - case SystemCatalog::UMEDINT: return &mcs_type_handler_uint24; - case SystemCatalog::UINT: return &mcs_type_handler_uint32; - case SystemCatalog::UBIGINT: return &mcs_type_handler_uint64; - case SystemCatalog::FLOAT: return &mcs_type_handler_sfloat; - case SystemCatalog::DOUBLE: return &mcs_type_handler_sdouble; - case SystemCatalog::LONGDOUBLE: return &mcs_type_handler_slongdouble; - case SystemCatalog::UFLOAT: return &mcs_type_handler_ufloat; - case SystemCatalog::UDOUBLE: return &mcs_type_handler_udouble; + switch (typeCode) + { + case SystemCatalog::BIT: return &mcs_type_handler_bit; + case SystemCatalog::TINYINT: return &mcs_type_handler_sint8; + case SystemCatalog::SMALLINT: return &mcs_type_handler_sint16; + case SystemCatalog::MEDINT: return &mcs_type_handler_sint24; + case SystemCatalog::INT: return &mcs_type_handler_sint32; + case SystemCatalog::BIGINT: return &mcs_type_handler_sint64; + case SystemCatalog::UTINYINT: return &mcs_type_handler_uint8; + case SystemCatalog::USMALLINT: return &mcs_type_handler_uint16; + case SystemCatalog::UMEDINT: return &mcs_type_handler_uint24; + case SystemCatalog::UINT: return &mcs_type_handler_uint32; + case SystemCatalog::UBIGINT: return &mcs_type_handler_uint64; + case SystemCatalog::FLOAT: return &mcs_type_handler_sfloat; + case SystemCatalog::DOUBLE: return &mcs_type_handler_sdouble; + case SystemCatalog::LONGDOUBLE: return &mcs_type_handler_slongdouble; + case SystemCatalog::UFLOAT: return &mcs_type_handler_ufloat; + case SystemCatalog::UDOUBLE: return &mcs_type_handler_udouble; - case SystemCatalog::DECIMAL: - if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) - return &mcs_type_handler_sdecimal64; - else - return &mcs_type_handler_sdecimal128; + case SystemCatalog::DECIMAL: + if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_sdecimal64; + else + return &mcs_type_handler_sdecimal128; - case SystemCatalog::UDECIMAL: - if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) - return &mcs_type_handler_udecimal64; - else - return &mcs_type_handler_udecimal128; + case SystemCatalog::UDECIMAL: + if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_udecimal64; + else + return &mcs_type_handler_udecimal128; - case SystemCatalog::TIME: return &mcs_type_handler_time; - case SystemCatalog::DATE: return &mcs_type_handler_date; - case SystemCatalog::DATETIME: return &mcs_type_handler_datetime; - case SystemCatalog::TIMESTAMP: return &mcs_type_handler_timestamp; - case SystemCatalog::CHAR: return &mcs_type_handler_char; - case SystemCatalog::VARCHAR: return &mcs_type_handler_varchar; - case SystemCatalog::TEXT: return &mcs_type_handler_text; - case SystemCatalog::CLOB: return &mcs_type_handler_clob; - case SystemCatalog::VARBINARY: return &mcs_type_handler_varbinary; - case SystemCatalog::BLOB: return &mcs_type_handler_blob; + case SystemCatalog::TIME: return &mcs_type_handler_time; + case SystemCatalog::DATE: return &mcs_type_handler_date; + case SystemCatalog::DATETIME: return &mcs_type_handler_datetime; + case SystemCatalog::TIMESTAMP: return &mcs_type_handler_timestamp; + case SystemCatalog::CHAR: return &mcs_type_handler_char; + case SystemCatalog::VARCHAR: return &mcs_type_handler_varchar; + case SystemCatalog::TEXT: return &mcs_type_handler_text; + case SystemCatalog::CLOB: return &mcs_type_handler_clob; + case SystemCatalog::VARBINARY: return &mcs_type_handler_varbinary; + case SystemCatalog::BLOB: return &mcs_type_handler_blob; - case SystemCatalog::NUM_OF_COL_DATA_TYPE: - case SystemCatalog::STRINT: - case SystemCatalog::UNDEFINED: - break; + case SystemCatalog::NUM_OF_COL_DATA_TYPE: + case SystemCatalog::STRINT: + case SystemCatalog::UNDEFINED: break; } return NULL; } - -const TypeHandler * -SystemCatalog::TypeHolderStd::typeHandler() const +const TypeHandler* SystemCatalog::TypeHolderStd::typeHandler() const { return TypeHandler::find(colDataType, *this); } - -boost::any -SystemCatalog::TypeHolderStd::getNullValueForType() const +boost::any SystemCatalog::TypeHolderStd::getNullValueForType() const { - const TypeHandler *h= typeHandler(); + const TypeHandler* h = typeHandler(); if (!h) { throw std::runtime_error("getNullValueForType: unkown column data type"); @@ -399,105 +353,92 @@ SystemCatalog::TypeHolderStd::getNullValueForType() const return h->getNullValueForType(*this); } - -const TypeHandler * -TypeHandler::find_by_ddltype(const ddlpackage::ColumnType &ct) +const TypeHandler* TypeHandler::find_by_ddltype(const ddlpackage::ColumnType& ct) { - switch (ct.fType) { - case ddlpackage::DDL_CHAR: return &mcs_type_handler_char; - case ddlpackage::DDL_VARCHAR: return &mcs_type_handler_varchar; - case ddlpackage::DDL_VARBINARY: return &mcs_type_handler_varbinary; - case ddlpackage::DDL_BIT: return &mcs_type_handler_bit; + switch (ct.fType) + { + case ddlpackage::DDL_CHAR: return &mcs_type_handler_char; + case ddlpackage::DDL_VARCHAR: return &mcs_type_handler_varchar; + case ddlpackage::DDL_VARBINARY: return &mcs_type_handler_varbinary; + case ddlpackage::DDL_BIT: return &mcs_type_handler_bit; - case ddlpackage::DDL_REAL: - case ddlpackage::DDL_DECIMAL: - case ddlpackage::DDL_NUMERIC: - case ddlpackage::DDL_NUMBER: + case ddlpackage::DDL_REAL: + case ddlpackage::DDL_DECIMAL: + case ddlpackage::DDL_NUMERIC: + case ddlpackage::DDL_NUMBER: - if (ct.fLength < datatypes::MAXDECIMALWIDTH) - return &mcs_type_handler_sdecimal64; - return &mcs_type_handler_sdecimal128; + if (ct.fLength < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_sdecimal64; + return &mcs_type_handler_sdecimal128; - case ddlpackage::DDL_FLOAT: return &mcs_type_handler_sfloat; - case ddlpackage::DDL_DOUBLE: return &mcs_type_handler_sdouble; + case ddlpackage::DDL_FLOAT: return &mcs_type_handler_sfloat; + case ddlpackage::DDL_DOUBLE: return &mcs_type_handler_sdouble; - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: return &mcs_type_handler_sint32; + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: return &mcs_type_handler_sint32; - case ddlpackage::DDL_BIGINT: return &mcs_type_handler_sint64; - case ddlpackage::DDL_MEDINT: return &mcs_type_handler_sint24; - case ddlpackage::DDL_SMALLINT: return &mcs_type_handler_sint16; - case ddlpackage::DDL_TINYINT: return &mcs_type_handler_sint8; + case ddlpackage::DDL_BIGINT: return &mcs_type_handler_sint64; + case ddlpackage::DDL_MEDINT: return &mcs_type_handler_sint24; + case ddlpackage::DDL_SMALLINT: return &mcs_type_handler_sint16; + case ddlpackage::DDL_TINYINT: return &mcs_type_handler_sint8; - case ddlpackage::DDL_DATE: return &mcs_type_handler_date; - case ddlpackage::DDL_DATETIME: return &mcs_type_handler_datetime; - case ddlpackage::DDL_TIME: return &mcs_type_handler_time; - case ddlpackage::DDL_TIMESTAMP: return &mcs_type_handler_timestamp; + case ddlpackage::DDL_DATE: return &mcs_type_handler_date; + case ddlpackage::DDL_DATETIME: return &mcs_type_handler_datetime; + case ddlpackage::DDL_TIME: return &mcs_type_handler_time; + case ddlpackage::DDL_TIMESTAMP: return &mcs_type_handler_timestamp; - case ddlpackage::DDL_CLOB: return &mcs_type_handler_clob; - case ddlpackage::DDL_BLOB: return &mcs_type_handler_blob; - case ddlpackage::DDL_TEXT: return &mcs_type_handler_text; + case ddlpackage::DDL_CLOB: return &mcs_type_handler_clob; + case ddlpackage::DDL_BLOB: return &mcs_type_handler_blob; + case ddlpackage::DDL_TEXT: return &mcs_type_handler_text; - case ddlpackage::DDL_UNSIGNED_TINYINT: return &mcs_type_handler_uint8; - case ddlpackage::DDL_UNSIGNED_SMALLINT: return &mcs_type_handler_uint16; - case ddlpackage::DDL_UNSIGNED_MEDINT: return &mcs_type_handler_uint24; - case ddlpackage::DDL_UNSIGNED_INT: return &mcs_type_handler_uint32; - case ddlpackage::DDL_UNSIGNED_BIGINT: return &mcs_type_handler_uint64; + case ddlpackage::DDL_UNSIGNED_TINYINT: return &mcs_type_handler_uint8; + case ddlpackage::DDL_UNSIGNED_SMALLINT: return &mcs_type_handler_uint16; + case ddlpackage::DDL_UNSIGNED_MEDINT: return &mcs_type_handler_uint24; + case ddlpackage::DDL_UNSIGNED_INT: return &mcs_type_handler_uint32; + case ddlpackage::DDL_UNSIGNED_BIGINT: return &mcs_type_handler_uint64; - case ddlpackage::DDL_UNSIGNED_DECIMAL: - case ddlpackage::DDL_UNSIGNED_NUMERIC: + case ddlpackage::DDL_UNSIGNED_DECIMAL: + case ddlpackage::DDL_UNSIGNED_NUMERIC: - if (ct.fLength < datatypes::MAXDECIMALWIDTH) + if (ct.fLength < datatypes::MAXDECIMALWIDTH) return &mcs_type_handler_udecimal64; - return &mcs_type_handler_udecimal128; + return &mcs_type_handler_udecimal128; - case ddlpackage::DDL_UNSIGNED_FLOAT: return &mcs_type_handler_ufloat; - case ddlpackage::DDL_UNSIGNED_DOUBLE: return &mcs_type_handler_udouble; - - case ddlpackage::DDL_INVALID_DATATYPE: - break; + case ddlpackage::DDL_UNSIGNED_FLOAT: return &mcs_type_handler_ufloat; + case ddlpackage::DDL_UNSIGNED_DOUBLE: return &mcs_type_handler_udouble; + + case ddlpackage::DDL_INVALID_DATATYPE: break; } return NULL; } - /****************************************************************************/ - -int TypeHandlerDate::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerDate::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t intColVal = row.getUintField<4>(pos); return f->store_date(intColVal); } - -int TypeHandlerDatetime::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerDatetime::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t intColVal = row.getUintField<8>(pos); return f->store_datetime(intColVal); } - -int TypeHandlerTime::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerTime::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t intColVal = row.getUintField<8>(pos); return f->store_time(intColVal); } - -int TypeHandlerTimestamp::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerTimestamp::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t intColVal = row.getUintField<8>(pos); return f->store_timestamp(intColVal); } - -int TypeHandlerStr::storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerStr::storeValueToFieldCharVarchar(rowgroup::Row& row, int pos, StoreField* f) const { int64_t intColVal; switch (f->colWidth()) @@ -516,148 +457,110 @@ int TypeHandlerStr::storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, case 8: { - //make sure we don't send strlen off into the weeds... + // make sure we don't send strlen off into the weeds... intColVal = row.getUintField<8>(pos); char tmp[256]; memcpy(tmp, &intColVal, 8); tmp[8] = 0; return f->store_string(tmp, strlen(tmp)); } - default: - return f->storeConstString(row.getConstString(pos)); + default: return f->storeConstString(row.getConstString(pos)); } } - -int TypeHandlerVarbinary::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerVarbinary::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { uint32_t l; const uint8_t* p = row.getVarBinaryField(l, pos); - return f->store_varbinary((const char *) p, l); + return f->store_varbinary((const char*)p, l); } - -int TypeHandlerSInt64::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerSInt64::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t val = row.getIntField<8>(pos); return f->store_xlonglong(val); } - -int TypeHandlerUInt64::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerUInt64::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { uint64_t val = row.getUintField<8>(pos); return f->store_xlonglong(static_cast(val)); } - -int TypeHandlerInt::storeValueToFieldSInt32(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerInt::storeValueToFieldSInt32(rowgroup::Row& row, int pos, StoreField* f) const { int64_t val = row.getIntField<4>(pos); return f->store_xlonglong(val); } - -int TypeHandlerInt::storeValueToFieldUInt32(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerInt::storeValueToFieldUInt32(rowgroup::Row& row, int pos, StoreField* f) const { uint64_t val = row.getUintField<4>(pos); return f->store_xlonglong(static_cast(val)); } - -int TypeHandlerSInt16::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerSInt16::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t val = row.getIntField<2>(pos); return f->store_xlonglong(val); } - -int TypeHandlerUInt16::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerUInt16::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { uint64_t val = row.getUintField<2>(pos); return f->store_xlonglong(static_cast(val)); } - -int TypeHandlerSInt8::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerSInt8::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { int64_t val = row.getIntField<1>(pos); return f->store_xlonglong(val); } - -int TypeHandlerUInt8::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerUInt8::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { uint64_t val = row.getUintField<1>(pos); return f->store_xlonglong(static_cast(val)); } - /* In this case, we're trying to load a double output column with float data. This is the case when you do sum(floatcol), e.g. */ -int TypeHandlerReal::storeValueToFieldXFloat(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerReal::storeValueToFieldXFloat(rowgroup::Row& row, int pos, StoreField* f) const { float dl = row.getFloatField(pos); return f->store_float(dl); } - -int TypeHandlerReal::storeValueToFieldXDouble(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerReal::storeValueToFieldXDouble(rowgroup::Row& row, int pos, StoreField* f) const { double dl = row.getDoubleField(pos); return f->store_double(dl); } - -int TypeHandlerSLongDouble::storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerSLongDouble::storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const { long double dl = row.getLongDoubleField(pos); return f->store_long_double(dl); } - -int TypeHandlerXDecimal::storeValueToField64(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerXDecimal::storeValueToField64(rowgroup::Row& row, int pos, StoreField* f) const { - return f->store_decimal64(datatypes::Decimal(row.getIntField(pos), - f->scale(), - f->precision())); + return f->store_decimal64(datatypes::Decimal(row.getIntField(pos), f->scale(), f->precision())); } - -int TypeHandlerXDecimal::storeValueToField128(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerXDecimal::storeValueToField128(rowgroup::Row& row, int pos, StoreField* f) const { int128_t* decPtr = row.getBinaryField(pos); - return f->store_decimal128(datatypes::Decimal(0, - f->scale(), - f->precision(), - decPtr)); + return f->store_decimal128(datatypes::Decimal(0, f->scale(), f->precision(), decPtr)); } - -int TypeHandlerStr::storeValueToFieldBlobText(rowgroup::Row &row, int pos, - StoreField *f) const +int TypeHandlerStr::storeValueToFieldBlobText(rowgroup::Row& row, int pos, StoreField* f) const { - return f->store_lob((const char*) row.getVarBinaryField(pos), - row.getVarBinaryLength(pos)); + return f->store_lob((const char*)row.getVarBinaryField(pos), row.getVarBinaryLength(pos)); } - /* int TypeHandlerBinary::storeValueToField(rowgroup::Row &row, int pos, StoreField *f) const @@ -675,43 +578,29 @@ int TypeHandlerBinary::storeValueToField(rowgroup::Row &row, int pos, storeNumericField(f, intColVal, colType); */ - /****************************************************************************/ -string TypeHandlerDate::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerDate::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { return DataConvert::dateToString(v.toSInt64()); } - -string TypeHandlerDatetime::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerDatetime::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { return DataConvert::datetimeToString(v.toSInt64()); } -string TypeHandlerTimestamp::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerTimestamp::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { return DataConvert::timestampToString(v.toSInt64(), v.tzname()); } - -string TypeHandlerTime::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerTime::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { return DataConvert::timeToString(v.toSInt64()); } - -string TypeHandlerChar::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerChar::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { // swap again to retain the string byte order ostringstream oss; @@ -720,10 +609,7 @@ string TypeHandlerChar::format(const SimpleValue &v, return oss.str(); } - -string TypeHandlerVarchar::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerVarchar::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { // swap again to retain the string byte order ostringstream oss; @@ -732,39 +618,26 @@ string TypeHandlerVarchar::format(const SimpleValue &v, return oss.str(); } - -string TypeHandlerInt::formatSInt64(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerInt::formatSInt64(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { ostringstream oss; oss << v.toSInt64(); return oss.str(); } - -string TypeHandlerInt::formatUInt64(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerInt::formatUInt64(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { ostringstream oss; oss << static_cast(v.toSInt64()); return oss.str(); } - -string TypeHandlerVarbinary::format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerVarbinary::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { return "N/A"; } - -string -TypeHandlerXDecimal::format64(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerXDecimal::format64(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { idbassert(isValidXDecimal64(attr)); if (attr.scale > 0) @@ -777,65 +650,51 @@ TypeHandlerXDecimal::format64(const SimpleValue &v, return oss.str(); } - -string -TypeHandlerXDecimal::format128(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) - const +string TypeHandlerXDecimal::format128(const SimpleValue& v, + const SystemCatalog::TypeAttributesStd& attr) const { idbassert(isValidXDecimal128(attr)); datatypes::Decimal dec(0, attr.scale, attr.precision, v.toSInt128()); return dec.toString(true); } - /****************************************************************************/ -class ostringstreamL: public ostringstream +class ostringstreamL : public ostringstream { -public: + public: ostringstreamL() { setf(ios::left, ios::adjustfield); } }; - -string TypeHandler::formatPartitionInfoSInt64( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &pi) const +string TypeHandler::formatPartitionInfoSInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& pi) const { ostringstreamL output; if (pi.isEmptyOrNullSInt64()) - output << setw(30) << "Empty/Null" - << setw(30) << "Empty/Null"; + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; else - output << setw(30) << format(SimpleValueSInt64(pi.min), attr) - << setw(30) << format(SimpleValueSInt64(pi.max), attr); + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) << setw(30) + << format(SimpleValueSInt64(pi.max), attr); return output.str(); } - -string TypeHandler::formatPartitionInfoUInt64( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &pi) const +string TypeHandler::formatPartitionInfoUInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& pi) const { ostringstreamL output; if (pi.isEmptyOrNullUInt64()) - output << setw(30) << "Empty/Null" - << setw(30) << "Empty/Null"; + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; else - output << setw(30) << format(SimpleValueSInt64(pi.min), attr) - << setw(30) << format(SimpleValueSInt64(pi.max), attr); + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) << setw(30) + << format(SimpleValueSInt64(pi.max), attr); return output.str(); } - - -string -TypeHandlerXDecimal::formatPartitionInfo128( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &pi) const +string TypeHandlerXDecimal::formatPartitionInfo128(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& pi) const { ostringstreamL output; if (pi.isEmptyOrNullSInt128()) @@ -847,11 +706,8 @@ TypeHandlerXDecimal::formatPartitionInfo128( return output.str(); } - -string -TypeHandlerStr::formatPartitionInfoSmallCharVarchar( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &pi) const +string TypeHandlerStr::formatPartitionInfoSmallCharVarchar(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& pi) const { ostringstreamL output; int64_t maxLimit = std::numeric_limits::max(); @@ -859,18 +715,15 @@ TypeHandlerStr::formatPartitionInfoSmallCharVarchar( maxLimit = uint64ToStr(maxLimit); minLimit = uint64ToStr(minLimit); if (pi.min == maxLimit && pi.max == minLimit) - output << setw(30) << "Empty/Null" - << setw(30) << "Empty/Null"; - else - output << setw(30) << format(SimpleValueSInt64(pi.min), attr) - << setw(30) << format(SimpleValueSInt64(pi.max), attr); - return output.str(); + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) << setw(30) + << format(SimpleValueSInt64(pi.max), attr); + return output.str(); } - -string -TypeHandlerChar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &pi) const +string TypeHandlerChar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& pi) const { // char column order swap for compare in subsequent loop if (attr.colWidth <= 8) @@ -878,10 +731,8 @@ TypeHandlerChar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &att return formatPartitionInfoSInt64(attr, pi); } - -string -TypeHandlerVarchar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &pi) const +string TypeHandlerVarchar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& pi) const { // varchar column order swap for compare in subsequent loop if (attr.colWidth <= 7) @@ -889,188 +740,172 @@ TypeHandlerVarchar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd & return formatPartitionInfoSInt64(attr, pi); } - /****************************************************************************/ -execplan::SimpleColumn * -TypeHandlerSInt8::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerSInt8::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { if (ct.scale == 0) - return new execplan::SimpleColumn_INT<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_INT<1>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); ct.colDataType = SystemCatalog::DECIMAL; - return new execplan::SimpleColumn_Decimal<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_Decimal<1>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerSInt16::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerSInt16::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { if (ct.scale == 0) - return new execplan::SimpleColumn_INT<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_INT<2>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); ct.colDataType = SystemCatalog::DECIMAL; - return new execplan::SimpleColumn_Decimal<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_Decimal<2>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerSInt24::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerSInt24::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { if (ct.scale == 0) - return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); ct.colDataType = SystemCatalog::DECIMAL; - return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerSInt32::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerSInt32::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { if (ct.scale == 0) - return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); ct.colDataType = SystemCatalog::DECIMAL; - return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerSInt64::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerSInt64::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { if (ct.scale == 0) - return new execplan::SimpleColumn_INT<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_INT<8>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); ct.colDataType = SystemCatalog::DECIMAL; - return new execplan::SimpleColumn_Decimal<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_Decimal<8>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerUInt8::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerUInt8::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { // QQ: why scale is not checked (unlike SInt1)? - return new execplan::SimpleColumn_UINT<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_UINT<1>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerUInt16::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerUInt16::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - return new execplan::SimpleColumn_UINT<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_UINT<2>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerUInt24::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerUInt24::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerUInt32::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerUInt32::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerUInt64::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerUInt64::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - return new execplan::SimpleColumn_UINT<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + return new execplan::SimpleColumn_UINT<8>(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerReal::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerReal::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - // QQ - return new execplan::SimpleColumn(name.db(), name.table(), name.column(), - prm.columnStore(), prm.sessionid()); + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerXDecimal::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerXDecimal::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - // QQ - return new execplan::SimpleColumn(name.db(), name.table(), name.column(), - prm.columnStore(), prm.sessionid()); + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - -execplan::SimpleColumn * -TypeHandlerStr::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerStr::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - // QQ - return new execplan::SimpleColumn(name.db(), name.table(), name.column(), - prm.columnStore(), prm.sessionid()); + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - - -execplan::SimpleColumn * -TypeHandlerTemporal::newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) const +execplan::SimpleColumn* TypeHandlerTemporal::newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const { - // QQ - return new execplan::SimpleColumn(name.db(), name.table(), name.column(), - prm.columnStore(), prm.sessionid()); + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), prm.columnStore(), + prm.sessionid()); } - /****************************************************************************/ -class SimpleConverter: public boost::any +class SimpleConverter : public boost::any { - bool &initPushWarning() + bool& initPushWarning() { - m_pushWarning= false; + m_pushWarning = false; return m_pushWarning; } bool m_pushWarning; -public: - SimpleConverter(const SessionParam &sp, - const TypeHandler *h, - const SystemCatalog::TypeAttributesStd &attr, - const char *str) - :boost::any(h->convertFromString(attr, - ConvertFromStringParam(sp.tzname(), true, false), - str, initPushWarning())) - { } + + public: + SimpleConverter(const SessionParam& sp, const TypeHandler* h, const SystemCatalog::TypeAttributesStd& attr, + const char* str) + : boost::any( + h->convertFromString(attr, ConvertFromStringParam(sp.tzname(), true, false), str, initPushWarning())) + { + } round_style_t roundStyle() const { return m_pushWarning ? round_style_t::POS : round_style_t::NONE; } - round_style_t roundStyle(const char *str) const + round_style_t roundStyle(const char* str) const { return m_pushWarning ? roundStyleDetect(str) : round_style_t::NONE; } - static round_style_t roundStyleDetect(const char *str) + static round_style_t roundStyleDetect(const char* str) { // get rid of leading white spaces and parentheses string data(str); @@ -1082,191 +917,163 @@ public: } return (data[0] == '-') ? round_style_t::NEG : round_style_t::POS; } - int64_t to_sint64() const { return boost::any_cast(*this); } - uint64_t to_uint64() const { return boost::any_cast(*this); } - uint32_t to_uint32() const { return boost::any_cast(*this); } - int128_t to_sint128() const { return boost::any_cast(*this); } + int64_t to_sint64() const + { + return boost::any_cast(*this); + } + uint64_t to_uint64() const + { + return boost::any_cast(*this); + } + uint32_t to_uint32() const + { + return boost::any_cast(*this); + } + int128_t to_sint128() const + { + return boost::any_cast(*this); + } }; - -class SimpleConverterSNumeric: public SimpleConverter +class SimpleConverterSNumeric : public SimpleConverter { -public: - SimpleConverterSNumeric(const SessionParam &sp, - const TypeHandler *h, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, - round_style_t &rf) - :SimpleConverter(sp, h, attr, str) + public: + SimpleConverterSNumeric(const SessionParam& sp, const TypeHandler* h, + const SystemCatalog::TypeAttributesStd& attr, const char* str, round_style_t& rf) + : SimpleConverter(sp, h, attr, str) { rf = roundStyle(str); } }; - -template -SimpleValue -toSimpleValueSInt(const SessionParam &sp, - const TypeHandler *h, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) +template +SimpleValue toSimpleValueSInt(const SessionParam& sp, const TypeHandler* h, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverterSNumeric anyVal(sp, h, attr, str, rf); return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); } - -SimpleValue -TypeHandlerSInt8::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerSInt8::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueSInt(sp, this, attr, str, rf); } - -SimpleValue -TypeHandlerSInt16::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerSInt16::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueSInt(sp, this, attr, str, rf); } - -SimpleValue -TypeHandlerSInt24::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerSInt24::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueSInt(sp, this, attr, str, rf); } - -SimpleValue -TypeHandlerSInt32::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerSInt32::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueSInt(sp, this, attr, str, rf); } - -SimpleValue -TypeHandlerSInt64::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerSInt64::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueSInt(sp, this, attr, str, rf); } - -template -SimpleValue toSimpleValueUInt(const SessionParam &sp, - const TypeHandler *h, - const SystemCatalog::TypeAttributesStd &attr, - const char *str) +template +SimpleValue toSimpleValueUInt(const SessionParam& sp, const TypeHandler* h, + const SystemCatalog::TypeAttributesStd& attr, const char* str) { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, h, attr, str); return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); } - -SimpleValue -TypeHandlerUInt8::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerUInt8::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueUInt(sp, this, attr, str); } - -SimpleValue -TypeHandlerUInt16::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerUInt16::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueUInt(sp, this, attr, str); } - -SimpleValue -TypeHandlerUInt24::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerUInt24::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueUInt(sp, this, attr, str); } - -SimpleValue -TypeHandlerUInt32::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerUInt32::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueUInt(sp, this, attr, str); } - -SimpleValue -TypeHandlerUInt64::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerUInt64::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { return toSimpleValueUInt(sp, this, attr, str); } - -SimpleValue -TypeHandlerDate::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerDate::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, this, attr, str); return SimpleValueSInt64(static_cast(anyVal.to_uint32())); } - -SimpleValue -TypeHandlerDatetime::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerDatetime::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, this, attr, str); return SimpleValueSInt64(static_cast(anyVal.to_uint64())); } - -SimpleValue -TypeHandlerTimestamp::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerTimestamp::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, this, attr, str); return SimpleValueTimestamp(anyVal.to_uint64(), sp.tzname()); } - -SimpleValue -TypeHandlerTime::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerTime::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, this, attr, str); return SimpleValueSInt64(anyVal.to_sint64()); } - -SimpleValue -TypeHandlerXDecimal::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerXDecimal::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { if (attr.colWidth <= SystemCatalog::EIGHT_BYTE) { @@ -1295,33 +1102,28 @@ TypeHandlerXDecimal::toSimpleValue(const SessionParam &sp, } } - -SimpleValue -TypeHandlerStr::toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const +SimpleValue TypeHandlerStr::toSimpleValue(const SessionParam& sp, + const SystemCatalog::TypeAttributesStd& attr, const char* str, + round_style_t& rf) const { SimpleConverter anyVal(sp, this, attr, str); - rf= anyVal.roundStyle(); + rf = anyVal.roundStyle(); string i = boost::any_cast(anyVal); // bug 1932, pad nulls up to the size of v i.resize(sizeof(int64_t), 0); - return SimpleValueSInt64(static_cast(uint64ToStr(*((uint64_t*) i.data())))); + return SimpleValueSInt64(static_cast(uint64ToStr(*((uint64_t*)i.data())))); } - /****************************************************************************/ -MinMaxPartitionInfo::MinMaxPartitionInfo(const BRM::EMEntry &entry) - :m_status(entry.status == BRM::EXTENTOUTOFSERVICE ? ET_DISABLED : EXPL_NULL) -{ } +MinMaxPartitionInfo::MinMaxPartitionInfo(const BRM::EMEntry& entry) + : m_status(entry.status == BRM::EXTENTOUTOFSERVICE ? ET_DISABLED : EXPL_NULL) +{ +} - -MinMaxPartitionInfo -TypeHandler::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const +MinMaxPartitionInfo TypeHandler::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + BRM::DBRM& em, const BRM::EMEntry& entry, + int* state) const { int32_t seqNum; MinMaxPartitionInfo partInfo(entry); @@ -1329,12 +1131,8 @@ TypeHandler::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr return partInfo; } - -MinMaxPartitionInfo -TypeHandlerXDecimal::getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const +MinMaxPartitionInfo TypeHandlerXDecimal::getExtentPartitionInfo64( + const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, const BRM::EMEntry& entry, int* state) const { int32_t seqNum; MinMaxPartitionInfo partInfo(entry); @@ -1342,12 +1140,8 @@ TypeHandlerXDecimal::getExtentPartitionInfo64(const SystemCatalog::TypeAttribute return partInfo; } - -MinMaxPartitionInfo -TypeHandlerXDecimal::getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const +MinMaxPartitionInfo TypeHandlerXDecimal::getExtentPartitionInfo128( + const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, const BRM::EMEntry& entry, int* state) const { int32_t seqNum; MinMaxPartitionInfo partInfo(entry); @@ -1355,12 +1149,9 @@ TypeHandlerXDecimal::getExtentPartitionInfo128(const SystemCatalog::TypeAttribut return partInfo; } - -MinMaxPartitionInfo -TypeHandlerChar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const +MinMaxPartitionInfo TypeHandlerChar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + BRM::DBRM& em, const BRM::EMEntry& entry, + int* state) const { int32_t seqNum; MinMaxPartitionInfo partInfo(entry); @@ -1374,12 +1165,9 @@ TypeHandlerChar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd & return partInfo; } - -MinMaxPartitionInfo -TypeHandlerVarchar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const +MinMaxPartitionInfo TypeHandlerVarchar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + BRM::DBRM& em, const BRM::EMEntry& entry, + int* state) const { int32_t seqNum; MinMaxPartitionInfo partInfo(entry); @@ -1393,17 +1181,12 @@ TypeHandlerVarchar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesSt return partInfo; } - /****************************************************************************/ - -string TypeHandler::PrintPartitionValueSInt64( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, - round_style_t rfMax) const +string TypeHandler::PrintPartitionValueSInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, + const SimpleValue& startVal, round_style_t rfMin, + const SimpleValue& endVal, round_style_t rfMax) const { if (!partInfo.isSuitableSInt64(startVal, rfMin, endVal, rfMax)) return ""; @@ -1412,19 +1195,15 @@ string TypeHandler::PrintPartitionValueSInt64( if (partInfo.min > partInfo.max) oss << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; else - oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) - << setw(30) << format(SimpleValueSInt64(partInfo.max), attr); + oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) << setw(30) + << format(SimpleValueSInt64(partInfo.max), attr); return oss.str(); } - -string TypeHandler::PrintPartitionValueUInt64( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, - round_style_t rfMax) const +string TypeHandler::PrintPartitionValueUInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, + const SimpleValue& startVal, round_style_t rfMin, + const SimpleValue& endVal, round_style_t rfMax) const { if (!partInfo.isSuitableUInt64(startVal, rfMin, endVal, rfMax)) return ""; @@ -1433,19 +1212,15 @@ string TypeHandler::PrintPartitionValueUInt64( if (static_cast(partInfo.min) > static_cast(partInfo.max)) oss << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; else - oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) - << setw(30) << format(SimpleValueSInt64(partInfo.max), attr); + oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) << setw(30) + << format(SimpleValueSInt64(partInfo.max), attr); return oss.str(); } - -string TypeHandlerXDecimal::PrintPartitionValue128( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, - round_style_t rfMax) const +string TypeHandlerXDecimal::PrintPartitionValue128(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, + const SimpleValue& startVal, round_style_t rfMin, + const SimpleValue& endVal, round_style_t rfMax) const { if (!partInfo.isSuitableSInt128(startVal, rfMin, endVal, rfMax)) return ""; @@ -1462,99 +1237,77 @@ string TypeHandlerXDecimal::PrintPartitionValue128( /****************************************************************************/ - -boost::any -TypeHandlerSInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { char tinyintvalue = joblist::TINYINTNULL; boost::any value = tinyintvalue; return value; } - -boost::any -TypeHandlerUInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint8_t utinyintvalue = joblist::UTINYINTNULL; boost::any value = utinyintvalue; return value; } - -boost::any -TypeHandlerSInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { short smallintvalue = joblist::SMALLINTNULL; boost::any value = smallintvalue; return value; } - -boost::any -TypeHandlerUInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint16_t usmallintvalue = joblist::USMALLINTNULL; boost::any value = usmallintvalue; return value; } - -boost::any -TypeHandlerSInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { int intvalue = joblist::INTNULL; boost::any value = intvalue; return value; } - -boost::any -TypeHandlerSInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { int intvalue = joblist::INTNULL; boost::any value = intvalue; return value; } - -boost::any -TypeHandlerUInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint32_t uintvalue = joblist::UINTNULL; boost::any value = uintvalue; return value; } - -boost::any -TypeHandlerUInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint32_t uintvalue = joblist::UINTNULL; boost::any value = uintvalue; return value; } - -boost::any -TypeHandlerSInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { long long bigint = joblist::BIGINTNULL; boost::any value = bigint; return value; } - -boost::any -TypeHandlerUInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint64_t ubigint = joblist::UBIGINTNULL; boost::any value = ubigint; return value; } - -boost::any -TypeHandlerXDecimal::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerXDecimal::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { if (LIKELY(attr.colWidth == 16)) { @@ -1592,9 +1345,7 @@ TypeHandlerXDecimal::getNullValueForType(const SystemCatalog::TypeAttributesStd return value; } - -boost::any -TypeHandlerSFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint32_t jlfloatnull = joblist::FLOATNULL; float* fp = reinterpret_cast(&jlfloatnull); @@ -1602,9 +1353,7 @@ TypeHandlerSFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &a return value; } - -boost::any -TypeHandlerUFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint32_t jlfloatnull = joblist::FLOATNULL; float* fp = reinterpret_cast(&jlfloatnull); @@ -1612,9 +1361,7 @@ TypeHandlerUFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &a return value; } - -boost::any -TypeHandlerSDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerSDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint64_t jldoublenull = joblist::DOUBLENULL; double* dp = reinterpret_cast(&jldoublenull); @@ -1622,9 +1369,7 @@ TypeHandlerSDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd & return value; } - -boost::any -TypeHandlerUDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerUDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint64_t jldoublenull = joblist::DOUBLENULL; double* dp = reinterpret_cast(&jldoublenull); @@ -1632,58 +1377,48 @@ TypeHandlerUDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd & return value; } - -boost::any -TypeHandlerDate::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerDate::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint32_t d = joblist::DATENULL; boost::any value = d; return value; } - -boost::any -TypeHandlerDatetime::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerDatetime::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint64_t d = joblist::DATETIMENULL; boost::any value = d; return value; } - -boost::any -TypeHandlerTime::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerTime::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { int64_t d = joblist::TIMENULL; boost::any value = d; return value; } - -boost::any -TypeHandlerTimestamp::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerTimestamp::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { uint64_t d = joblist::TIMESTAMPNULL; boost::any value = d; return value; } - -boost::any -TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { switch (attr.colWidth) { case 1: { - //charnull = joblist::CHAR1NULL; + // charnull = joblist::CHAR1NULL; std::string charnull = "\376"; boost::any value = charnull; return value; } case 2: { - //charnull = joblist::CHAR2NULL; + // charnull = joblist::CHAR2NULL; std::string charnull = "\377\376"; boost::any value = charnull; return value; @@ -1691,7 +1426,7 @@ TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd &att case 3: case 4: { - //charnull = joblist::CHAR4NULL; + // charnull = joblist::CHAR4NULL; std::string charnull = "\377\377\377\376"; boost::any value = charnull; return value; @@ -1701,7 +1436,7 @@ TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd &att case 7: case 8: { - //charnull = joblist::CHAR8NULL; + // charnull = joblist::CHAR8NULL; std::string charnull = "\377\377\377\377\377\377\377\376"; boost::any value = charnull; return value; @@ -1712,15 +1447,13 @@ TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd &att return value; } - -boost::any -TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd& attr) const { switch (attr.colWidth) { case 1: { - //charnull = joblist::CHAR2NULL; + // charnull = joblist::CHAR2NULL; std::string charnull = "\377\376"; boost::any value = charnull; return value; @@ -1728,7 +1461,7 @@ TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttribut case 2: case 3: { - //charnull = joblist::CHAR4NULL; + // charnull = joblist::CHAR4NULL; std::string charnull = "\377\377\377\376"; boost::any value = charnull; return value; @@ -1738,7 +1471,7 @@ TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttribut case 6: case 7: { - //charnull = joblist::CHAR8NULL; + // charnull = joblist::CHAR8NULL; std::string charnull = "\377\377\377\377\377\377\377\376"; boost::any value = charnull; return value; @@ -1749,17 +1482,14 @@ TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttribut return value; } -boost::any -TypeHandlerBlob::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerBlob::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { WriteEngine::Token nullToken; boost::any value = nullToken; return value; } - -boost::any -TypeHandlerVarbinary::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +boost::any TypeHandlerVarbinary::getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const { WriteEngine::Token nullToken; boost::any value = nullToken; @@ -1768,452 +1498,350 @@ TypeHandlerVarbinary::getNullValueForType(const SystemCatalog::TypeAttributesStd /****************************************************************************/ -boost::any -TypeHandlerBit::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerBit::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { - return dataconvert::DataConvert::StringToBit(colType, prm, data, pushWarning); + return dataconvert::DataConvert::StringToBit(colType, prm, data, pushWarning); } - -boost::any -TypeHandlerSInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { int64_t val64; dataconvert::number_int_value(data, SystemCatalog::TINYINT, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (char) val64; + boost::any value = (char)val64; return value; } - -boost::any -TypeHandlerSInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { int64_t val64; dataconvert::number_int_value(data, SystemCatalog::SMALLINT, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (short) val64; + boost::any value = (short)val64; return value; } - -boost::any -TypeHandlerSInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { int64_t val64; dataconvert::number_int_value(data, SystemCatalog::MEDINT, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (int) val64; + boost::any value = (int)val64; return value; } - -boost::any -TypeHandlerSInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { int64_t val64; dataconvert::number_int_value(data, SystemCatalog::INT, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (int) val64; + boost::any value = (int)val64; return value; } - -boost::any -TypeHandlerSInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { int64_t val64; dataconvert::number_int_value(data, SystemCatalog::BIGINT, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (long long) val64; + boost::any value = (long long)val64; return value; } - -boost::any -TypeHandlerUInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { - boost::any value = (uint8_t)dataconvert::number_uint_value(data, SystemCatalog::UTINYINT, colType, pushWarning, prm.noRoundup()); + boost::any value = (uint8_t)dataconvert::number_uint_value(data, SystemCatalog::UTINYINT, colType, + pushWarning, prm.noRoundup()); return value; } - -boost::any -TypeHandlerUInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { - boost::any value = (uint16_t)dataconvert::number_uint_value(data, SystemCatalog::USMALLINT, colType, pushWarning, prm.noRoundup()); + boost::any value = (uint16_t)dataconvert::number_uint_value(data, SystemCatalog::USMALLINT, colType, + pushWarning, prm.noRoundup()); return value; } - -boost::any -TypeHandlerUInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { - boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UMEDINT, colType, pushWarning, prm.noRoundup()); + boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UMEDINT, colType, + pushWarning, prm.noRoundup()); return value; } - -boost::any -TypeHandlerUInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { - boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UINT, colType, pushWarning, prm.noRoundup()); + boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UINT, colType, pushWarning, + prm.noRoundup()); return value; } - -boost::any -TypeHandlerUInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { - boost::any value = (uint64_t)dataconvert::number_uint_value(data, SystemCatalog::UBIGINT, colType, pushWarning, prm.noRoundup()); + boost::any value = (uint64_t)dataconvert::number_uint_value(data, SystemCatalog::UBIGINT, colType, + pushWarning, prm.noRoundup()); return value; } - -boost::any -TypeHandlerSFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToFloat(SystemCatalog::FLOAT, data, pushWarning); } - -boost::any -TypeHandlerUFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToFloat(SystemCatalog::UFLOAT, data, pushWarning); } - -boost::any -TypeHandlerSDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToDouble(SystemCatalog::DOUBLE, data, pushWarning); } - -boost::any -TypeHandlerUDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToDouble(SystemCatalog::UDOUBLE, data, pushWarning); } - -boost::any -TypeHandlerDate::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerDate::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToDate(data, pushWarning); } - -boost::any -TypeHandlerDatetime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerDatetime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToDatetime(data, pushWarning); } - -boost::any -TypeHandlerTime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerTime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToTime(colType, data, pushWarning); } -boost::any -TypeHandlerTimestamp::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerTimestamp::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToTimestamp(prm, data, pushWarning); } - -boost::any -TypeHandlerChar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerChar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToString(colType, data, pushWarning); } - -boost::any -TypeHandlerVarchar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerVarchar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToString(colType, data, pushWarning); } - -boost::any -TypeHandlerText::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerText::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { return dataconvert::DataConvert::StringToString(colType, data, pushWarning); } - -boost::any -TypeHandlerClob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerClob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { boost::any value = data; return value; } - -boost::any -TypeHandlerBlob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerBlob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { boost::any value = data; return value; } - -boost::any -TypeHandlerVarbinary::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerVarbinary::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning) const { boost::any value = data; return value; } - -boost::any -TypeHandlerSDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) const { return dataconvert::DataConvert::StringToSDecimal(colType, prm, data, pushWarning); } - -boost::any -TypeHandlerUDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) const { return dataconvert::DataConvert::StringToUDecimal(colType, prm, data, pushWarning); } - -boost::any -TypeHandlerSDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerSDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) const { return dataconvert::DataConvert::StringToSDecimal(colType, prm, data, pushWarning); } - -boost::any -TypeHandlerUDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& data, - bool& pushWarning) const +boost::any TypeHandlerUDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) const { return dataconvert::DataConvert::StringToUDecimal(colType, prm, data, pushWarning); } - /****************************************************************************/ -const uint8_t* -getEmptyTypeHandlerSInt8() +const uint8_t* getEmptyTypeHandlerSInt8() { const static uint8_t TINYINTEMPTYROW = joblist::TINYINTEMPTYROW; return &TINYINTEMPTYROW; } - -const uint8_t* -TypeHandlerSInt8::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const + +const uint8_t* TypeHandlerSInt8::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerSInt8(); } -const uint8_t* -getEmptyTypeHandlerSInt16() +const uint8_t* getEmptyTypeHandlerSInt16() { const static uint16_t SMALLINTEMPTYROW = joblist::SMALLINTEMPTYROW; return reinterpret_cast(&SMALLINTEMPTYROW); } - -const uint8_t* -TypeHandlerSInt16::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const + +const uint8_t* TypeHandlerSInt16::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerSInt16(); } -const uint8_t* -getEmptyTypeHandlerSInt32() +const uint8_t* getEmptyTypeHandlerSInt32() { const static uint32_t INTEMPTYROW = joblist::INTEMPTYROW; return reinterpret_cast(&INTEMPTYROW); } -const uint8_t* -TypeHandlerSInt24::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSInt24::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerSInt32(); } -const uint8_t* -TypeHandlerSInt32::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSInt32::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerSInt32(); } -const uint8_t* -getEmptyTypeHandlerSInt64() +const uint8_t* getEmptyTypeHandlerSInt64() { const static uint64_t BIGINTEMPTYROW = joblist::BIGINTEMPTYROW; return reinterpret_cast(&BIGINTEMPTYROW); } -const uint8_t* -TypeHandlerSInt64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSInt64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerSInt64(); } -const uint8_t* -TypeHandlerUInt8::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUInt8::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { const static uint8_t UTINYINTEMPTYROW = joblist::UTINYINTEMPTYROW; return reinterpret_cast(&UTINYINTEMPTYROW); } -const uint8_t* -TypeHandlerUInt16::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUInt16::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { const static uint16_t USMALLINTEMPTYROW = joblist::USMALLINTEMPTYROW; return reinterpret_cast(&USMALLINTEMPTYROW); } -const uint8_t* -getEmptyTypeHandlerUInt32() +const uint8_t* getEmptyTypeHandlerUInt32() { const static uint32_t UINTEMPTYROW = joblist::UINTEMPTYROW; return reinterpret_cast(&UINTEMPTYROW); } -const uint8_t* -TypeHandlerUInt24::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUInt24::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerUInt32(); } -const uint8_t* -TypeHandlerUInt32::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUInt32::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerUInt32(); } -const uint8_t* -TypeHandlerUInt64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUInt64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { const static uint64_t UBIGINTEMPTYROW = joblist::UBIGINTEMPTYROW; return reinterpret_cast(&UBIGINTEMPTYROW); } -const uint8_t* -getEmptyTypeHandlerFloat(const SystemCatalog::TypeAttributesStd &attr) +const uint8_t* getEmptyTypeHandlerFloat(const SystemCatalog::TypeAttributesStd& attr) { const static uint32_t FLOATEMPTYROW = joblist::FLOATEMPTYROW; return reinterpret_cast(&FLOATEMPTYROW); } -const uint8_t* -TypeHandlerUFloat::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUFloat::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerFloat(attr); } -const uint8_t* -TypeHandlerSFloat::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSFloat::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerFloat(attr); } -const uint8_t* -getEmptyTypeHandlerDouble(const SystemCatalog::TypeAttributesStd &attr) +const uint8_t* getEmptyTypeHandlerDouble(const SystemCatalog::TypeAttributesStd& attr) { const static uint64_t DOUBLEEMPTYROW = joblist::DOUBLEEMPTYROW; return reinterpret_cast(&DOUBLEEMPTYROW); } -const uint8_t* -TypeHandlerUDouble::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUDouble::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerDouble(attr); } -const uint8_t* -TypeHandlerSDouble::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSDouble::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerDouble(attr); } @@ -2222,10 +1850,9 @@ TypeHandlerSDouble::getEmptyValueForType(const SystemCatalog::TypeAttributesStd // args // attr - width, precision and scale // offset - offset value to reduce width for VARCHAR by 1 -const uint8_t* -getEmptyTypeHandlerStr(const SystemCatalog::TypeAttributesStd &attr, int8_t offset) +const uint8_t* getEmptyTypeHandlerStr(const SystemCatalog::TypeAttributesStd& attr, int8_t offset) { - const static uint8_t CHAR1EMPTYROW = joblist::CHAR1EMPTYROW; + const static uint8_t CHAR1EMPTYROW = joblist::CHAR1EMPTYROW; const static uint16_t CHAR2EMPTYROW = joblist::CHAR2EMPTYROW; const static uint32_t CHAR4EMPTYROW = joblist::CHAR4EMPTYROW; const static uint64_t CHAR8EMPTYROW = joblist::CHAR8EMPTYROW; @@ -2240,20 +1867,17 @@ getEmptyTypeHandlerStr(const SystemCatalog::TypeAttributesStd &attr, int8_t offs return reinterpret_cast(&CHAR1EMPTYROW); } -const uint8_t* -TypeHandlerStr::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerStr::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerStr(attr, 0); } -const uint8_t* -TypeHandlerVarchar::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerVarchar::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyTypeHandlerStr(attr, -1); } -inline const uint8_t* -getEmptyValueForDecimal64(const SystemCatalog::TypeAttributesStd &attr) +inline const uint8_t* getEmptyValueForDecimal64(const SystemCatalog::TypeAttributesStd& attr) { if (attr.colWidth <= 1) return getEmptyTypeHandlerSInt8(); @@ -2265,34 +1889,30 @@ getEmptyValueForDecimal64(const SystemCatalog::TypeAttributesStd &attr) return getEmptyTypeHandlerSInt64(); } -const uint8_t* -TypeHandlerSDecimal64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSDecimal64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyValueForDecimal64(attr); } -const uint8_t* -TypeHandlerUDecimal64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUDecimal64::getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const { return getEmptyValueForDecimal64(attr); } -const uint8_t* -TypeHandlerSDecimal128::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerSDecimal128::getEmptyValueForType( + const SystemCatalog::TypeAttributesStd& attr) const { return reinterpret_cast(&datatypes::Decimal128Empty); } -const uint8_t* -TypeHandlerUDecimal128::getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const +const uint8_t* TypeHandlerUDecimal128::getEmptyValueForType( + const SystemCatalog::TypeAttributesStd& attr) const { return reinterpret_cast(&datatypes::Decimal128Empty); } - /****************************************************************************/ -} // end of namespace datatypes - +} // end of namespace datatypes // vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index fbaa47e05..19bab0237 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -29,165 +29,203 @@ #include "mcs_longdouble.h" #ifdef _MSC_VER -typedef int mcs_sint32_t; +typedef int mcs_sint32_t; #else typedef int32_t mcs_sint32_t; #endif - // Because including my_sys.h in a Columnstore header causes too many conflicts struct charset_info_st; typedef const struct charset_info_st CHARSET_INFO; - #ifdef _MSC_VER #define __attribute__(x) #endif namespace { -const int64_t MIN_TINYINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -126; -const int64_t MAX_TINYINT __attribute__ ((unused)) = std::numeric_limits::max(); // 127; -const int64_t MIN_SMALLINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -32766; -const int64_t MAX_SMALLINT __attribute__ ((unused)) = std::numeric_limits::max(); // 32767; -const int64_t MIN_MEDINT __attribute__ ((unused)) = -(1ULL << 23); // -8388608; -const int64_t MAX_MEDINT __attribute__ ((unused)) = (1ULL << 23) - 1; // 8388607; -const int64_t MIN_INT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -2147483646; -const int64_t MAX_INT __attribute__ ((unused)) = std::numeric_limits::max(); // 2147483647; -const int64_t MIN_BIGINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -9223372036854775806LL; -const int64_t MAX_BIGINT __attribute__ ((unused)) = std::numeric_limits::max(); // 9223372036854775807 +const int64_t MIN_TINYINT __attribute__((unused)) = std::numeric_limits::min() + 2; // -126; +const int64_t MAX_TINYINT __attribute__((unused)) = std::numeric_limits::max(); // 127; +const int64_t MIN_SMALLINT __attribute__((unused)) = std::numeric_limits::min() + 2; // -32766; +const int64_t MAX_SMALLINT __attribute__((unused)) = std::numeric_limits::max(); // 32767; +const int64_t MIN_MEDINT __attribute__((unused)) = -(1ULL << 23); // -8388608; +const int64_t MAX_MEDINT __attribute__((unused)) = (1ULL << 23) - 1; // 8388607; +const int64_t MIN_INT __attribute__((unused)) = std::numeric_limits::min() + 2; // -2147483646; +const int64_t MAX_INT __attribute__((unused)) = std::numeric_limits::max(); // 2147483647; +const int64_t MIN_BIGINT __attribute__((unused)) = + std::numeric_limits::min() + 2; // -9223372036854775806LL; +const int64_t MAX_BIGINT __attribute__((unused)) = + std::numeric_limits::max(); // 9223372036854775807 -const uint64_t MIN_UINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UTINYINT __attribute__ ((unused)) = 0; -const uint64_t MIN_USMALLINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UMEDINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UBIGINT __attribute__ ((unused)) = 0; -const uint64_t MAX_UINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 4294967293 -const uint64_t MAX_UTINYINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 253; -const uint64_t MAX_USMALLINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 65533; -const uint64_t MAX_UMEDINT __attribute__ ((unused)) = (1ULL << 24) - 1; // 16777215 -const uint64_t MAX_UBIGINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 18446744073709551613 +const uint64_t MIN_UINT __attribute__((unused)) = 0; +const uint64_t MIN_UTINYINT __attribute__((unused)) = 0; +const uint64_t MIN_USMALLINT __attribute__((unused)) = 0; +const uint64_t MIN_UMEDINT __attribute__((unused)) = 0; +const uint64_t MIN_UBIGINT __attribute__((unused)) = 0; +const uint64_t MAX_UINT __attribute__((unused)) = std::numeric_limits::max() - 2; // 4294967293 +const uint64_t MAX_UTINYINT __attribute__((unused)) = std::numeric_limits::max() - 2; // 253; +const uint64_t MAX_USMALLINT __attribute__((unused)) = std::numeric_limits::max() - 2; // 65533; +const uint64_t MAX_UMEDINT __attribute__((unused)) = (1ULL << 24) - 1; // 16777215 +const uint64_t MAX_UBIGINT __attribute__((unused)) = + std::numeric_limits::max() - 2; // 18446744073709551613 -const float MAX_FLOAT __attribute__ ((unused)) = std::numeric_limits::max(); // 3.402823466385289e+38 -const float MIN_FLOAT __attribute__ ((unused)) = -std::numeric_limits::max(); -const double MAX_DOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); // 1.7976931348623157e+308 -const double MIN_DOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); -const long double MAX_LONGDOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); // 1.7976931348623157e+308 -const long double MIN_LONGDOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); +const float MAX_FLOAT __attribute__((unused)) = std::numeric_limits::max(); // 3.402823466385289e+38 +const float MIN_FLOAT __attribute__((unused)) = -std::numeric_limits::max(); +const double MAX_DOUBLE __attribute__((unused)) = + std::numeric_limits::max(); // 1.7976931348623157e+308 +const double MIN_DOUBLE __attribute__((unused)) = -std::numeric_limits::max(); +const long double MAX_LONGDOUBLE __attribute__((unused)) = + std::numeric_limits::max(); // 1.7976931348623157e+308 +const long double MIN_LONGDOUBLE __attribute__((unused)) = -std::numeric_limits::max(); -const uint64_t AUTOINCR_SATURATED __attribute__ ((unused)) = std::numeric_limits::max(); -} - - -using namespace std; // e.g. string +const uint64_t AUTOINCR_SATURATED __attribute__((unused)) = std::numeric_limits::max(); +} // namespace +using namespace std; // e.g. string namespace ddlpackage { - struct ColumnType; +struct ColumnType; }; - namespace BRM { - struct EMEntry; - class DBRM; -}; - +struct EMEntry; +class DBRM; +}; // namespace BRM namespace rowgroup { - class Row; +class Row; }; - namespace execplan { - class SimpleColumn; +class SimpleColumn; }; - - namespace datatypes { - template struct make_unsigned { - typedef struct { } type; + typedef struct + { + } type; }; -template<> struct make_unsigned { typedef uint8_t type; }; -template<> struct make_unsigned { typedef uint16_t type; }; -template<> struct make_unsigned { typedef uint32_t type; }; -template<> struct make_unsigned { typedef uint64_t type; }; -template<> struct make_unsigned { typedef uint128_t type; }; +template <> +struct make_unsigned +{ + typedef uint8_t type; +}; +template <> +struct make_unsigned +{ + typedef uint16_t type; +}; +template <> +struct make_unsigned +{ + typedef uint32_t type; +}; +template <> +struct make_unsigned +{ + typedef uint64_t type; +}; +template <> +struct make_unsigned +{ + typedef uint128_t type; +}; -template +template struct _WidthToSIntegralType { - typedef T type; + typedef T type; }; template -struct WidthToSIntegralType: _WidthToSIntegralType { }; +struct WidthToSIntegralType : _WidthToSIntegralType +{ +}; template <> -struct WidthToSIntegralType<1>: _WidthToSIntegralType<1, int8_t> { }; +struct WidthToSIntegralType<1> : _WidthToSIntegralType<1, int8_t> +{ +}; template <> -struct WidthToSIntegralType<2>: _WidthToSIntegralType<2, int16_t> { }; +struct WidthToSIntegralType<2> : _WidthToSIntegralType<2, int16_t> +{ +}; template <> -struct WidthToSIntegralType<4>: _WidthToSIntegralType<4, int32_t> { }; +struct WidthToSIntegralType<4> : _WidthToSIntegralType<4, int32_t> +{ +}; template <> -struct WidthToSIntegralType<8>: _WidthToSIntegralType<8, int64_t> { }; +struct WidthToSIntegralType<8> : _WidthToSIntegralType<8, int64_t> +{ +}; template <> -struct WidthToSIntegralType<16>: _WidthToSIntegralType<16, int128_t> { }; +struct WidthToSIntegralType<16> : _WidthToSIntegralType<16, int128_t> +{ +}; class SystemCatalog { -public: - + public: /** the set of Calpont column widths * */ - enum ColWidth { ONE_BIT, ONE_BYTE, TWO_BYTE, THREE_BYTE, FOUR_BYTE, FIVE_BYTE, SIX_BYTE, SEVEN_BYTE, EIGHT_BYTE }; - + enum ColWidth + { + ONE_BIT, + ONE_BYTE, + TWO_BYTE, + THREE_BYTE, + FOUR_BYTE, + FIVE_BYTE, + SIX_BYTE, + SEVEN_BYTE, + EIGHT_BYTE + }; /** the set of Calpont column data types * */ enum ColDataType { - BIT, /*!< BIT type */ - TINYINT, /*!< TINYINT type */ - CHAR, /*!< CHAR type */ - SMALLINT, /*!< SMALLINT type */ - DECIMAL, /*!< DECIMAL type */ - MEDINT, /*!< MEDINT type */ - INT, /*!< INT type */ - FLOAT, /*!< FLOAT type */ - DATE, /*!< DATE type */ - BIGINT, /*!< BIGINT type */ - DOUBLE, /*!< DOUBLE type */ - DATETIME, /*!< DATETIME type */ - VARCHAR, /*!< VARCHAR type */ - VARBINARY, /*!< VARBINARY type */ - CLOB, /*!< CLOB type */ - BLOB, /*!< BLOB type */ - UTINYINT, /*!< Unsigned TINYINT type */ - USMALLINT, /*!< Unsigned SMALLINT type */ - UDECIMAL, /*!< Unsigned DECIMAL type */ - UMEDINT, /*!< Unsigned MEDINT type */ - UINT, /*!< Unsigned INT type */ - UFLOAT, /*!< Unsigned FLOAT type */ - UBIGINT, /*!< Unsigned BIGINT type */ - UDOUBLE, /*!< Unsigned DOUBLE type */ - TEXT, /*!< TEXT type */ - TIME, /*!< TIME type */ - TIMESTAMP, /*!< TIMESTAMP type */ - NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ - LONGDOUBLE, /* @bug3241, dev and variance calculation only */ - STRINT, /* @bug3532, string as int for fast comparison */ - UNDEFINED, /*!< Undefined - used in UDAF API */ + BIT, /*!< BIT type */ + TINYINT, /*!< TINYINT type */ + CHAR, /*!< CHAR type */ + SMALLINT, /*!< SMALLINT type */ + DECIMAL, /*!< DECIMAL type */ + MEDINT, /*!< MEDINT type */ + INT, /*!< INT type */ + FLOAT, /*!< FLOAT type */ + DATE, /*!< DATE type */ + BIGINT, /*!< BIGINT type */ + DOUBLE, /*!< DOUBLE type */ + DATETIME, /*!< DATETIME type */ + VARCHAR, /*!< VARCHAR type */ + VARBINARY, /*!< VARBINARY type */ + CLOB, /*!< CLOB type */ + BLOB, /*!< BLOB type */ + UTINYINT, /*!< Unsigned TINYINT type */ + USMALLINT, /*!< Unsigned SMALLINT type */ + UDECIMAL, /*!< Unsigned DECIMAL type */ + UMEDINT, /*!< Unsigned MEDINT type */ + UINT, /*!< Unsigned INT type */ + UFLOAT, /*!< Unsigned FLOAT type */ + UBIGINT, /*!< Unsigned BIGINT type */ + UDOUBLE, /*!< Unsigned DOUBLE type */ + TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ + TIMESTAMP, /*!< TIMESTAMP type */ + NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ + LONGDOUBLE, /* @bug3241, dev and variance calculation only */ + STRINT, /* @bug3532, string as int for fast comparison */ + UNDEFINED, /*!< Undefined - used in UDAF API */ }; // XXX: It is assumed here that ALL TYPES have width, scale and precision. @@ -197,103 +235,95 @@ public: // XXX: Current hierarchy is not all that straightforward to work with. class TypeAttributesStd { - public: - int32_t colWidth; - int32_t scale; //number after decimal points - int32_t precision; - TypeAttributesStd() - :colWidth(0), - scale(0), - precision(-1) - {} - TypeAttributesStd(int32_t w, int32_t s, int32_t p) - :colWidth(w), - scale(s), - precision(p) - {} - /** - @brief Convenience method to get int128 from a std::string. - */ - int128_t decimal128FromString(const std::string& value, bool *saturate = 0) const; + public: + int32_t colWidth; + int32_t scale; // number after decimal points + int32_t precision; + TypeAttributesStd() : colWidth(0), scale(0), precision(-1) + { + } + TypeAttributesStd(int32_t w, int32_t s, int32_t p) : colWidth(w), scale(s), precision(p) + { + } + /** + @brief Convenience method to get int128 from a std::string. + */ + int128_t decimal128FromString(const std::string& value, bool* saturate = 0) const; - /** - @brief The method sets the legacy scale and precision of a wide decimal - column which is the result of an arithmetic operation. - */ - inline void setDecimalScalePrecisionLegacy(unsigned int p, unsigned int s) + /** + @brief The method sets the legacy scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + inline void setDecimalScalePrecisionLegacy(unsigned int p, unsigned int s) + { + scale = s; + + if (s == 0) + precision = p - 1; + else + precision = p - s; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + inline void setDecimalScalePrecision(unsigned int p, unsigned int s) + { + colWidth = (p > INT64MAXPRECISION) ? MAXDECIMALWIDTH : MAXLEGACYWIDTH; + + precision = (p > INT128MAXPRECISION) ? INT128MAXPRECISION : p; + + scale = s; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation, based on a heuristic. + */ + inline void setDecimalScalePrecisionHeuristic(unsigned int p, unsigned int s) + { + unsigned int diff = 0; + + if (p > INT128MAXPRECISION) { - scale = s; - - if (s == 0) - precision = p - 1; - else - precision = p - s; + precision = INT128MAXPRECISION; + diff = p - INT128MAXPRECISION; + } + else + { + precision = p; } - /** - @brief The method sets the scale and precision of a wide decimal - column which is the result of an arithmetic operation. - */ - inline void setDecimalScalePrecision(unsigned int p, unsigned int s) + scale = s; + + if (diff != 0) { - colWidth = (p > INT64MAXPRECISION) ? - MAXDECIMALWIDTH : MAXLEGACYWIDTH; + scale = s - (int)(diff * (38.0 / 65.0)); - precision = (p > INT128MAXPRECISION) ? - INT128MAXPRECISION : p; - - scale = s; - } - - /** - @brief The method sets the scale and precision of a wide decimal - column which is the result of an arithmetic operation, based on a heuristic. - */ - inline void setDecimalScalePrecisionHeuristic(unsigned int p, unsigned int s) - { - unsigned int diff = 0; - - if (p > INT128MAXPRECISION) - { - precision = INT128MAXPRECISION; - diff = p - INT128MAXPRECISION; - } - else - { - precision = p; - } - - scale = s; - - if (diff != 0) - { - scale = s - (int)(diff * (38.0/65.0)); - - if (scale < 0) - scale = 0; - } - } - - /** - @brief The method returns true if the column precision - belongs to a wide decimal range. - */ - inline bool isWideDecimalPrecision() const - { - return ((precision > INT64MAXPRECISION) && - (precision <= INT128MAXPRECISION)); + if (scale < 0) + scale = 0; } + } + /** + @brief The method returns true if the column precision + belongs to a wide decimal range. + */ + inline bool isWideDecimalPrecision() const + { + return ((precision > INT64MAXPRECISION) && (precision <= INT128MAXPRECISION)); + } }; - class TypeHolderStd: public TypeAttributesStd + class TypeHolderStd : public TypeAttributesStd { - public: + public: ColDataType colDataType; - TypeHolderStd() - :colDataType(MEDINT) - { } - const class TypeHandler *typeHandler() const; + TypeHolderStd() : colDataType(MEDINT) + { + } + const class TypeHandler* typeHandler() const; boost::any getNullValueForType() const; /** @@ -302,86 +332,72 @@ public: */ inline bool isWideDecimalType() const { - return (colDataType == DECIMAL || - colDataType == UDECIMAL) && - colWidth == MAXDECIMALWIDTH; + return (colDataType == DECIMAL || colDataType == UDECIMAL) && colWidth == MAXDECIMALWIDTH; } inline bool isWide() const { - return colWidth > static_cast(MAXLEGACYWIDTH); + return colWidth > static_cast(MAXLEGACYWIDTH); } inline bool isNarrow() const { - return colWidth <= static_cast(MAXLEGACYWIDTH); + return colWidth <= static_cast(MAXLEGACYWIDTH); } bool isUnsignedInteger() const { - switch (colDataType) - { - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - return true; - default: - return false; - } + switch (colDataType) + { + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: return true; + default: return false; + } } bool isSignedInteger() const { - switch (colDataType) - { - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - return true; + switch (colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: return true; - default: - return false; - } + default: return false; + } } bool canReturnXInt64() const { - switch (colDataType) - { - case datatypes::SystemCatalog::BIT: - case datatypes::SystemCatalog::VARBINARY: - case datatypes::SystemCatalog::CLOB: - case datatypes::SystemCatalog::BLOB: - case datatypes::SystemCatalog::NUM_OF_COL_DATA_TYPE: - case datatypes::SystemCatalog::LONGDOUBLE: - case datatypes::SystemCatalog::STRINT: - case datatypes::SystemCatalog::UNDEFINED: - return false; - default: - break; - } - return true; + switch (colDataType) + { + case datatypes::SystemCatalog::BIT: + case datatypes::SystemCatalog::VARBINARY: + case datatypes::SystemCatalog::CLOB: + case datatypes::SystemCatalog::BLOB: + case datatypes::SystemCatalog::NUM_OF_COL_DATA_TYPE: + case datatypes::SystemCatalog::LONGDOUBLE: + case datatypes::SystemCatalog::STRINT: + case datatypes::SystemCatalog::UNDEFINED: return false; + default: break; + } + return true; } - }; - }; - /** @brief The method detects whether decimal type is wide using datatype and width. */ -static inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType &dt, - const int32_t width) +static inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType& dt, const int32_t width) { - return width == MAXDECIMALWIDTH && - (dt == SystemCatalog::DECIMAL || - dt == SystemCatalog::UDECIMAL); + return width == MAXDECIMALWIDTH && (dt == SystemCatalog::DECIMAL || dt == SystemCatalog::UDECIMAL); } /** convenience function to determine if column type is a char @@ -389,21 +405,16 @@ static inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType */ inline bool isCharType(const datatypes::SystemCatalog::ColDataType type) { - return (datatypes::SystemCatalog::VARCHAR == type || - datatypes::SystemCatalog::CHAR == type || - datatypes::SystemCatalog::BLOB == type || - datatypes::SystemCatalog::TEXT == type); + return (datatypes::SystemCatalog::VARCHAR == type || datatypes::SystemCatalog::CHAR == type || + datatypes::SystemCatalog::BLOB == type || datatypes::SystemCatalog::TEXT == type); } - inline bool typeHasCollation(const datatypes::SystemCatalog::ColDataType type) { - return datatypes::SystemCatalog::VARCHAR == type || - datatypes::SystemCatalog::CHAR == type || - datatypes::SystemCatalog::TEXT == type; + return datatypes::SystemCatalog::VARCHAR == type || datatypes::SystemCatalog::CHAR == type || + datatypes::SystemCatalog::TEXT == type; } - /** convenience function to determine if column type is a * numeric type */ @@ -426,11 +437,9 @@ inline bool isNumeric(const datatypes::SystemCatalog::ColDataType type) case datatypes::SystemCatalog::UBIGINT: case datatypes::SystemCatalog::UFLOAT: case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::UDECIMAL: - return true; + case datatypes::SystemCatalog::UDECIMAL: return true; - default: - return false; + default: return false; } } @@ -447,15 +456,12 @@ inline bool isInteger(const datatypes::SystemCatalog::ColDataType type) case datatypes::SystemCatalog::USMALLINT: case datatypes::SystemCatalog::UMEDINT: case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - return true; + case datatypes::SystemCatalog::UBIGINT: return true; - default: - return false; + default: return false; } } - inline bool isLongDouble(const datatypes::SystemCatalog::ColDataType type) { return type == datatypes::SystemCatalog::LONGDOUBLE; @@ -463,8 +469,7 @@ inline bool isLongDouble(const datatypes::SystemCatalog::ColDataType type) inline bool isDecimal(const datatypes::SystemCatalog::ColDataType type) { - return (type == datatypes::SystemCatalog::DECIMAL || - type == datatypes::SystemCatalog::UDECIMAL); + return (type == datatypes::SystemCatalog::DECIMAL || type == datatypes::SystemCatalog::UDECIMAL); } /** convenience function to determine if column type is an @@ -478,11 +483,9 @@ inline bool isUnsigned(const datatypes::SystemCatalog::ColDataType type) case datatypes::SystemCatalog::USMALLINT: case datatypes::SystemCatalog::UMEDINT: case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - return true; + case datatypes::SystemCatalog::UBIGINT: return true; - default: - return false; + default: return false; } } @@ -494,11 +497,9 @@ inline bool isSignedInteger(const datatypes::SystemCatalog::ColDataType type) case datatypes::SystemCatalog::SMALLINT: case datatypes::SystemCatalog::MEDINT: case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - return true; + case datatypes::SystemCatalog::BIGINT: return true; - default: - return false; + default: return false; } } @@ -508,35 +509,43 @@ inline bool isSignedInteger(const datatypes::SystemCatalog::ColDataType type) */ inline bool hasUnderlyingWideDecimalForSumAndAvg(datatypes::SystemCatalog::ColDataType type) { - return datatypes::isSignedInteger(type) || datatypes::isUnsigned(type); + return datatypes::isSignedInteger(type) || datatypes::isUnsigned(type); } -} // end of namespace datatypes - +} // end of namespace datatypes namespace datatypes { - static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; class ConstString { - const char *m_str; + const char* m_str; size_t m_length; -public: - ConstString(const char *str, size_t length) - :m_str(str), m_length(length) - { } - const char *str() const { return m_str; } - const char *end() const { return m_str + m_length; } - size_t length() const { return m_length; } - void bin2hex(char *o) + + public: + ConstString(const char* str, size_t length) : m_str(str), m_length(length) { - static const char hexdig[] = { '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; - const char *e= end(); - for (const char *s= m_str; s < e; s++) + } + const char* str() const + { + return m_str; + } + const char* end() const + { + return m_str + m_length; + } + size_t length() const + { + return m_length; + } + void bin2hex(char* o) + { + static const char hexdig[] = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + const char* e = end(); + for (const char* s = m_str; s < e; s++) { *o++ = hexdig[*s >> 4]; *o++ = hexdig[*s & 0xf]; @@ -544,107 +553,120 @@ public: } }; - -enum class round_style_t: uint8_t +enum class round_style_t : uint8_t { - NONE= 0, - POS= 0x01, - NEG= 0x80 + NONE = 0, + POS = 0x01, + NEG = 0x80 }; - class SessionParam { - const char *m_tzname; -public: - SessionParam(const char *tzname) - :m_tzname(tzname) - { } - const char *tzname() const + const char* m_tzname; + + public: + SessionParam(const char* tzname) : m_tzname(tzname) + { + } + const char* tzname() const { return m_tzname; } }; - class ConvertFromStringParam { const std::string& m_timeZone; const bool m_noRoundup; const bool m_isUpdate; -public: - ConvertFromStringParam(const std::string& timeZone, - bool noRoundup, bool isUpdate) - :m_timeZone(timeZone), - m_noRoundup(noRoundup), - m_isUpdate(isUpdate) - { } - const std::string& timeZone() const { return m_timeZone; } - bool noRoundup() const { return m_noRoundup; } - bool isUpdate() const { return m_isUpdate; } -}; + public: + ConvertFromStringParam(const std::string& timeZone, bool noRoundup, bool isUpdate) + : m_timeZone(timeZone), m_noRoundup(noRoundup), m_isUpdate(isUpdate) + { + } + const std::string& timeZone() const + { + return m_timeZone; + } + bool noRoundup() const + { + return m_noRoundup; + } + bool isUpdate() const + { + return m_isUpdate; + } +}; class SimpleValue { - int64_t m_sint64; + int64_t m_sint64; int128_t m_sint128; - const char *m_tzname; -public: - SimpleValue(const int64_t sint64, - const int128_t &sint128, - const char *tzname) - :m_sint64(sint64), m_sint128(sint128), m_tzname(tzname) - { } - SimpleValue() - :m_sint64(0), m_sint128(0), m_tzname(0) - { } - int64_t toSInt64() const { return m_sint64; } - uint64_t toUInt64() const { return static_cast(m_sint64); } - int128_t toSInt128() const { return m_sint128; } - const char *tzname() const { return m_tzname; } + const char* m_tzname; + + public: + SimpleValue(const int64_t sint64, const int128_t& sint128, const char* tzname) + : m_sint64(sint64), m_sint128(sint128), m_tzname(tzname) + { + } + SimpleValue() : m_sint64(0), m_sint128(0), m_tzname(0) + { + } + int64_t toSInt64() const + { + return m_sint64; + } + uint64_t toUInt64() const + { + return static_cast(m_sint64); + } + int128_t toSInt128() const + { + return m_sint128; + } + const char* tzname() const + { + return m_tzname; + } }; - -class SimpleValueSInt64: public SimpleValue +class SimpleValueSInt64 : public SimpleValue { -public: - SimpleValueSInt64(int64_t value) - :SimpleValue(value, 0, NULL) - { } + public: + SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, NULL) + { + } }; - -class SimpleValueUInt64: public SimpleValue +class SimpleValueUInt64 : public SimpleValue { -public: - SimpleValueUInt64(uint64_t value) - :SimpleValue(static_cast(value), 0, NULL) - { } + public: + SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, NULL) + { + } }; - -class SimpleValueSInt128: public SimpleValue +class SimpleValueSInt128 : public SimpleValue { -public: - SimpleValueSInt128(int128_t value) - :SimpleValue(0, value, NULL) - { } + public: + SimpleValueSInt128(int128_t value) : SimpleValue(0, value, NULL) + { + } }; - -class SimpleValueTimestamp: public SimpleValue +class SimpleValueTimestamp : public SimpleValue { -public: - SimpleValueTimestamp(uint64_t value, const char *tzname) - :SimpleValue(static_cast(value), 0, tzname) - { } + public: + SimpleValueTimestamp(uint64_t value, const char* tzname) + : SimpleValue(static_cast(value), 0, tzname) + { + } }; - class MinMaxInfo { -public: + public: int64_t min; int64_t max; union @@ -657,9 +679,7 @@ public: int128_t int128Max; int64_t max_; }; - MinMaxInfo() - :min((uint64_t)0x8000000000000001ULL), - max((uint64_t) - 0x8000000000000001LL) + MinMaxInfo() : min((uint64_t)0x8000000000000001ULL), max((uint64_t)-0x8000000000000001LL) { int128Min = datatypes::minInt128; int128Max = datatypes::maxInt128; @@ -681,9 +701,9 @@ public: MinMaxInfo tmp; tmp.min = static_cast(std::numeric_limits::max()); tmp.max = static_cast(std::numeric_limits::min()); - tmp.int128Max = datatypes::minInt128; // please bear in mind that DECIMAL(38) UNSIGNED - // has representable range 0..10^38-1 which is well - // inside int128 representable range. + tmp.int128Max = datatypes::minInt128; // please bear in mind that DECIMAL(38) UNSIGNED + // has representable range 0..10^38-1 which is well + // inside int128 representable range. tmp.int128Min = datatypes::maxInt128; return tmp; } @@ -703,7 +723,8 @@ public: * A more general approach to check non-nullness of the range than * explicit comparison with invalid bounds. */ - static bool isRangeInvalid(const MinMaxInfo& mm, datatypes::SystemCatalog::ColDataType colType, int colWidth) + static bool isRangeInvalid(const MinMaxInfo& mm, datatypes::SystemCatalog::ColDataType colType, + int colWidth) { if (colWidth > 8) { @@ -718,8 +739,7 @@ public: } bool isEmptyOrNullSInt64() const { - return min == std::numeric_limits::max() && - max == std::numeric_limits::min(); + return min == std::numeric_limits::max() && max == std::numeric_limits::min(); } bool isEmptyOrNullUInt64() const { @@ -728,49 +748,46 @@ public: } bool isEmptyOrNullSInt128() const { - return int128Min == datatypes::maxInt128 && - int128Max == datatypes::minInt128; + return int128Min == datatypes::maxInt128 && int128Max == datatypes::minInt128; } - void widenSInt64(const MinMaxInfo &partInfo) + void widenSInt64(const MinMaxInfo& partInfo) { min = partInfo.min < min ? partInfo.min : min; max = partInfo.max > max ? partInfo.max : max; } - void widenUInt64(const MinMaxInfo &partInfo) + void widenUInt64(const MinMaxInfo& partInfo) { - min = static_cast(static_cast(partInfo.min) < static_cast(min) ? partInfo.min : min); - max = static_cast(static_cast(partInfo.max) > static_cast(max) ? partInfo.max : max); + min = static_cast( + static_cast(partInfo.min) < static_cast(min) ? partInfo.min : min); + max = static_cast( + static_cast(partInfo.max) > static_cast(max) ? partInfo.max : max); } - void widenSInt128(const MinMaxInfo &partInfo) + void widenSInt128(const MinMaxInfo& partInfo) { int128Min = partInfo.int128Min < int128Min ? partInfo.int128Min : int128Min; int128Max = partInfo.int128Max > int128Max ? partInfo.int128Max : int128Max; } - static MinMaxInfo widenSInt64(const MinMaxInfo &a, - const MinMaxInfo &b) + static MinMaxInfo widenSInt64(const MinMaxInfo& a, const MinMaxInfo& b) { - MinMaxInfo tmp= a; + MinMaxInfo tmp = a; tmp.widenSInt64(b); return tmp; } - static MinMaxInfo widenUInt64(const MinMaxInfo &a, - const MinMaxInfo &b) + static MinMaxInfo widenUInt64(const MinMaxInfo& a, const MinMaxInfo& b) { MinMaxInfo tmp(a); tmp.widenUInt64(b); return tmp; } - static MinMaxInfo widenSInt128(const MinMaxInfo &a, - const MinMaxInfo &b) + static MinMaxInfo widenSInt128(const MinMaxInfo& a, const MinMaxInfo& b) { - MinMaxInfo tmp= a; + MinMaxInfo tmp = a; tmp.widenSInt128(b); return tmp; } }; - -class MinMaxPartitionInfo: public MinMaxInfo +class MinMaxPartitionInfo : public MinMaxInfo { enum status_flag_t : uint64_t { @@ -779,24 +796,28 @@ class MinMaxPartitionInfo: public MinMaxInfo CPINVALID = 0x0004 }; uint64_t m_status; -public: - MinMaxPartitionInfo() - :m_status(0) - { }; - MinMaxPartitionInfo(const BRM::EMEntry &entry); - void set_invalid() { m_status|= CPINVALID; } - bool is_invalid() const { return m_status & CPINVALID; } - bool is_disabled() const { return m_status & ET_DISABLED; } - bool isSuitableSInt64(const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + public: + MinMaxPartitionInfo() : m_status(0){}; + MinMaxPartitionInfo(const BRM::EMEntry& entry); + void set_invalid() + { + m_status |= CPINVALID; + } + bool is_invalid() const + { + return m_status & CPINVALID; + } + bool is_disabled() const + { + return m_status & ET_DISABLED; + } + + bool isSuitableSInt64(const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const { - if (min >= startVal.toSInt64() && - max <= endVal.toSInt64() && - !(min == std::numeric_limits::max() && - max == std::numeric_limits::min())) + if (min >= startVal.toSInt64() && max <= endVal.toSInt64() && + !(min == std::numeric_limits::max() && max == std::numeric_limits::min())) { if (rfMin == round_style_t::POS && min == startVal.toSInt64()) return false; @@ -808,15 +829,12 @@ public: } return false; } - bool isSuitableUInt64(const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitableUInt64(const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const { if (static_cast(min) >= startVal.toUInt64() && static_cast(max) <= endVal.toUInt64() && - !(static_cast(min) == std::numeric_limits::max() && - max == 0)) + !(static_cast(min) == std::numeric_limits::max() && max == 0)) { if (rfMin == round_style_t::POS && min == startVal.toSInt64()) return false; @@ -827,15 +845,11 @@ public: } return false; } - bool isSuitableSInt128(const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitableSInt128(const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const { - if (int128Min >= startVal.toSInt128() && - int128Max <= endVal.toSInt128() && - !(int128Min == datatypes::maxInt128 && - int128Max == datatypes::minInt128)) + if (int128Min >= startVal.toSInt128() && int128Max <= endVal.toSInt128() && + !(int128Min == datatypes::maxInt128 && int128Max == datatypes::minInt128)) { if (rfMin == round_style_t::POS && int128Min == startVal.toSInt128()) return false; @@ -849,64 +863,85 @@ public: } }; - class ColBatchWriter { - FILE *m_filePtr; + FILE* m_filePtr; char m_delimiter; char m_enclosed_by; -public: - ColBatchWriter(FILE *f, - char delimiter, - char enclosed_by) - :m_filePtr(f), - m_delimiter(delimiter), - m_enclosed_by(enclosed_by) - { } - FILE *filePtr() const { return m_filePtr; } - char delimiter() const { return m_delimiter; } - char enclosed_by() const { return m_enclosed_by; } -}; + public: + ColBatchWriter(FILE* f, char delimiter, char enclosed_by) + : m_filePtr(f), m_delimiter(delimiter), m_enclosed_by(enclosed_by) + { + } + FILE* filePtr() const + { + return m_filePtr; + } + char delimiter() const + { + return m_delimiter; + } + char enclosed_by() const + { + return m_enclosed_by; + } +}; class SimpleColumnParam { uint32_t m_sessionid; bool m_columnStore; -public: - SimpleColumnParam(uint32_t sessionid, bool columnStore) - :m_sessionid(sessionid), - m_columnStore(columnStore) - { } - uint32_t sessionid() const { return m_sessionid; } - bool columnStore() const { return m_columnStore; } - void columnStore(bool value) { m_columnStore= value; } -}; + public: + SimpleColumnParam(uint32_t sessionid, bool columnStore) : m_sessionid(sessionid), m_columnStore(columnStore) + { + } + uint32_t sessionid() const + { + return m_sessionid; + } + bool columnStore() const + { + return m_columnStore; + } + void columnStore(bool value) + { + m_columnStore = value; + } +}; class DatabaseQualifiedColumnName { std::string m_db; std::string m_table; std::string m_column; -public: - DatabaseQualifiedColumnName(const std::string &db, - const std::string &table, - const std::string &column) - :m_db(db), - m_table(table), - m_column(column) - { } - const std::string &db() const { return m_db; } - const std::string &table() const { return m_table; } - const std::string &column() const { return m_column; } -}; + public: + DatabaseQualifiedColumnName(const std::string& db, const std::string& table, const std::string& column) + : m_db(db), m_table(table), m_column(column) + { + } + const std::string& db() const + { + return m_db; + } + const std::string& table() const + { + return m_table; + } + const std::string& column() const + { + return m_column; + } +}; class StoreField { -public: - virtual ~StoreField() {} + public: + virtual ~StoreField() + { + } virtual int32_t colWidth() const = 0; virtual int32_t precision() const = 0; virtual int32_t scale() const = 0; @@ -914,290 +949,239 @@ public: virtual int store_datetime(int64_t val) = 0; virtual int store_time(int64_t val) = 0; virtual int store_timestamp(int64_t val) = 0; - virtual int store_string(const char *str, size_t length) = 0; - virtual int store_varbinary(const char *str, size_t length) = 0; + virtual int store_string(const char* str, size_t length) = 0; + virtual int store_varbinary(const char* str, size_t length) = 0; virtual int store_xlonglong(int64_t val) = 0; virtual int store_float(float val) = 0; virtual int store_double(double val) = 0; virtual int store_long_double(long double val) = 0; virtual int store_decimal64(const datatypes::Decimal& dec) = 0; virtual int store_decimal128(const datatypes::Decimal& dec) = 0; - virtual int store_lob(const char *str, size_t length) = 0; + virtual int store_lob(const char* str, size_t length) = 0; - int storeConstString(const utils::ConstString &str) + int storeConstString(const utils::ConstString& str) { - return store_string(str.str(), str.length()); + return store_string(str.str(), str.length()); } }; - class WriteBatchField { -public: - virtual ~WriteBatchField() { } - virtual size_t ColWriteBatchDate(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchDatetime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchTime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchTimestamp(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchChar(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchVarchar(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchSInt64(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchUInt64(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchSInt32(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchUInt32(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchSInt16(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchUInt16(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchSInt8(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchUInt8(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchXFloat(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchXDouble(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchSLongDouble(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchXDecimal(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchVarbinary(const unsigned char *buf0, bool nullVal, ColBatchWriter &ci) = 0; - virtual size_t ColWriteBatchBlob(const unsigned char *buf0, bool nullVal, ColBatchWriter &ci) = 0; + public: + virtual ~WriteBatchField() + { + } + virtual size_t ColWriteBatchDate(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchDatetime(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchTime(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchTimestamp(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchChar(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchVarchar(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchSInt64(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchUInt64(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchSInt32(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchUInt32(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchSInt16(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchUInt16(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchSInt8(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchUInt8(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchXFloat(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchXDouble(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchSLongDouble(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchXDecimal(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchVarbinary(const unsigned char* buf0, bool nullVal, ColBatchWriter& ci) = 0; + virtual size_t ColWriteBatchBlob(const unsigned char* buf0, bool nullVal, ColBatchWriter& ci) = 0; }; - class TypeHandler { -public: + public: using code_t = datatypes::SystemCatalog::ColDataType; -protected: - std::string formatPartitionInfoSInt64(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const; - std::string formatPartitionInfoUInt64(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const; - std::string PrintPartitionValueSInt64(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + protected: + std::string formatPartitionInfoSInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const; + std::string formatPartitionInfoUInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const; + + std::string PrintPartitionValueSInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const; - std::string PrintPartitionValueUInt64(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + std::string PrintPartitionValueUInt64(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const; -public: - static const TypeHandler *find(SystemCatalog::ColDataType typeCode, - const SystemCatalog::TypeAttributesStd &attr); - static const TypeHandler *find_by_ddltype(const ddlpackage::ColumnType &ct); - virtual ~TypeHandler() { } - virtual const string & name() const= 0; - virtual const string print(const SystemCatalog::TypeAttributesStd &attr) const + public: + static const TypeHandler* find(SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd& attr); + static const TypeHandler* find_by_ddltype(const ddlpackage::ColumnType& ct); + virtual ~TypeHandler() + { + } + virtual const string& name() const = 0; + virtual const string print(const SystemCatalog::TypeAttributesStd& attr) const { return name(); } - virtual code_t code() const= 0; - virtual bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const + virtual code_t code() const = 0; + virtual bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const { return false; } - virtual uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) - const + virtual uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd& attr) const { return 30; } - virtual size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const= 0; - virtual int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const= 0; + virtual size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const = 0; + virtual int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const = 0; - virtual std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const = 0; + virtual std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const = 0; - virtual std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const= 0; - virtual execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const= 0; + virtual std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const = 0; + virtual execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const = 0; virtual SimpleValue getMinValueSimple() const { - return SimpleValue(std::numeric_limits::min(), - std::numeric_limits::min(), - 0); + return SimpleValue(std::numeric_limits::min(), std::numeric_limits::min(), 0); } virtual SimpleValue getMaxValueSimple() const { - return SimpleValue(std::numeric_limits::max(), - std::numeric_limits::max(), - 0); + return SimpleValue(std::numeric_limits::max(), std::numeric_limits::max(), 0); } - virtual SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const= 0; - virtual MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const + virtual SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const = 0; + virtual MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const { return MinMaxInfo::widenSInt64(a, b); } - virtual MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const; - virtual string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + virtual MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + BRM::DBRM& em, const BRM::EMEntry& entry, + int* state) const; + virtual string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const { - return PrintPartitionValueSInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueSInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - virtual bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, - round_style_t rfMax) const + virtual bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& part, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const { return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); } - virtual boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const = 0; + virtual boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const = 0; virtual boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const = 0; - virtual const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) - const = 0; - + virtual const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const = 0; }; - // QQ: perhaps not needed yet -class TypeHandlerBit: public TypeHandler +class TypeHandlerBit : public TypeHandler { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::BIT; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { - idbassert(0); // QQ + idbassert(0); // QQ return 0; } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { - idbassert(0); // QQ + idbassert(0); // QQ return 1; } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { - return "0"; // QQ + return "0"; // QQ } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { idbassert(0); return "Error"; } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override { idbassert(0); return NULL; } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override { idbassert(0); return SimpleValue(); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { - //TODO: How to communicate with write engine? + // TODO: How to communicate with write engine? return boost::any(); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { idbassert(0); return nullptr; } - }; - -class TypeHandlerInt: public TypeHandler +class TypeHandlerInt : public TypeHandler { -protected: - int storeValueToFieldSInt32(rowgroup::Row &row, int pos, - StoreField *f) const; - int storeValueToFieldUInt32(rowgroup::Row &row, int pos, - StoreField *f) const; - std::string formatSInt64(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const; - std::string formatUInt64(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const; - + protected: + int storeValueToFieldSInt32(rowgroup::Row& row, int pos, StoreField* f) const; + int storeValueToFieldUInt32(rowgroup::Row& row, int pos, StoreField* f) const; + std::string formatSInt64(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const; + std::string formatUInt64(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const; }; - -class TypeHandlerSInt8: public TypeHandlerInt +class TypeHandlerSInt8 : public TypeHandlerInt { -public: - const string &name() const override; + public: + const string& name() const override; code_t code() const override { return SystemCatalog::TINYINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchSInt8(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatSInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { return SimpleValueSInt64(std::numeric_limits::min()); @@ -1206,55 +1190,45 @@ public: { return SimpleValueSInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSInt16: public TypeHandlerInt +class TypeHandlerSInt16 : public TypeHandlerInt { -public: - const string &name() const override; + public: + const string& name() const override; code_t code() const override { return SystemCatalog::SMALLINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchSInt16(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatSInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { return SimpleValueSInt64(std::numeric_limits::min()); @@ -1263,57 +1237,47 @@ public: { return SimpleValueSInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSInt24: public TypeHandlerInt +class TypeHandlerSInt24 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::MEDINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchSInt32(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldSInt32(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatSInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { return SimpleValueSInt64(MIN_MEDINT); @@ -1322,57 +1286,47 @@ class TypeHandlerSInt24: public TypeHandlerInt { return SimpleValueSInt64(MAX_MEDINT); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSInt32: public TypeHandlerInt +class TypeHandlerSInt32 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::INT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchSInt32(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldSInt32(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatSInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { return SimpleValueSInt64(std::numeric_limits::min()); @@ -1381,54 +1335,44 @@ class TypeHandlerSInt32: public TypeHandlerInt { return SimpleValueSInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSInt64: public TypeHandlerInt +class TypeHandlerSInt64 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::BIGINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchSInt64(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatSInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { return SimpleValueSInt64(std::numeric_limits::min()); @@ -1437,54 +1381,44 @@ class TypeHandlerSInt64: public TypeHandlerInt { return SimpleValueSInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUInt8: public TypeHandlerInt +class TypeHandlerUInt8 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::UTINYINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchUInt8(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatUInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoUInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { @@ -1494,332 +1428,260 @@ class TypeHandlerUInt8: public TypeHandlerInt { return SimpleValueUInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenUInt64(a, b); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueUInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueUInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUInt16: public TypeHandlerInt +class TypeHandlerUInt16 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::USMALLINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchUInt16(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatUInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoUInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { - return SimpleValueUInt64(0); + return SimpleValueUInt64(0); } SimpleValue getMaxValueSimple() const override { return SimpleValueUInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenUInt64(a, b); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueUInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueUInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUInt24: public TypeHandlerInt +class TypeHandlerUInt24 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::UMEDINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchUInt32(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldUInt32(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatUInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoUInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { - return SimpleValueUInt64(0); + return SimpleValueUInt64(0); } SimpleValue getMaxValueSimple() const override { - return SimpleValueUInt64(MAX_UMEDINT); + return SimpleValueUInt64(MAX_UMEDINT); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenUInt64(a, b); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueUInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueUInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUInt32: public TypeHandlerInt +class TypeHandlerUInt32 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::UINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchUInt32(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldUInt32(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatUInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoUInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { - return SimpleValueUInt64(0); + return SimpleValueUInt64(0); } SimpleValue getMaxValueSimple() const override { - return SimpleValueUInt64(std::numeric_limits::max()); + return SimpleValueUInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenUInt64(a, b); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueUInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueUInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUInt64: public TypeHandlerInt +class TypeHandlerUInt64 : public TypeHandlerInt { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::BIGINT; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchUInt64(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return formatUInt64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoUInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; SimpleValue getMinValueSimple() const override { return SimpleValueUInt64(0); @@ -1828,134 +1690,99 @@ class TypeHandlerUInt64: public TypeHandlerInt { return SimpleValueUInt64(std::numeric_limits::max()); } - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenUInt64(a, b); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueUInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueUInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerXDecimal: public TypeHandler +class TypeHandlerXDecimal : public TypeHandler { -protected: - - static bool isValidXDecimal64(const SystemCatalog::TypeAttributesStd &attr) + protected: + static bool isValidXDecimal64(const SystemCatalog::TypeAttributesStd& attr) { return attr.colWidth <= 8; } - static bool isValidXDecimal128(const SystemCatalog::TypeAttributesStd &attr) + static bool isValidXDecimal128(const SystemCatalog::TypeAttributesStd& attr) { return attr.colWidth == 16; } - int storeValueToField64(rowgroup::Row &row, int pos, - StoreField *f) const; - int storeValueToField128(rowgroup::Row &row, int pos, - StoreField *f) const; + int storeValueToField64(rowgroup::Row& row, int pos, StoreField* f) const; + int storeValueToField128(rowgroup::Row& row, int pos, StoreField* f) const; - std::string format64(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const; - std::string format128(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const; - std::string formatPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const; + std::string format64(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const; + std::string format128(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const; + std::string formatPartitionInfo128(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& i) const; - MinMaxPartitionInfo getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const; - MinMaxPartitionInfo getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const; - string PrintPartitionValue128(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, - round_style_t rfMax) const; + MinMaxPartitionInfo getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const; + MinMaxPartitionInfo getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const; + string PrintPartitionValue128(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const; -public: - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + public: + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchXDecimal(buf, nullVal, writer); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSDecimal64: public TypeHandlerXDecimal +class TypeHandlerSDecimal64 : public TypeHandlerXDecimal { -public: - const string &name() const override; + public: + const string& name() const override; code_t code() const override { return SystemCatalog::DECIMAL; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) - const override + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd& attr) const override { return 30; } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToField64(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return format64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } @@ -1967,76 +1794,61 @@ public: { return SimpleValueSInt64(std::numeric_limits::max()); } - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenSInt64(a, b); } - MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const override + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const override { return getExtentPartitionInfo64(attr, em, entry, state); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueSInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueSInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUDecimal64: public TypeHandlerXDecimal +class TypeHandlerUDecimal64 : public TypeHandlerXDecimal { -public: - const string &name() const override; + public: + const string& name() const override; code_t code() const override { return SystemCatalog::UDECIMAL; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) - const override + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd& attr) const override { return 30; } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToField64(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return format64(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } @@ -2048,159 +1860,127 @@ public: { return SimpleValueUInt64(std::numeric_limits::max()); } - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenSInt64(a, b); } - MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const override + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const override { return getExtentPartitionInfo64(attr, em, entry, state); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { - return PrintPartitionValueSInt64(attr, partInfo, - startVal, rfMin, - endVal, rfMax); + return PrintPartitionValueSInt64(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSDecimal128: public TypeHandlerXDecimal +class TypeHandlerSDecimal128 : public TypeHandlerXDecimal { -public: - const string &name() const override; + public: + const string& name() const override; code_t code() const override { return SystemCatalog::DECIMAL; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) - const override + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd& attr) const override { return Decimal::MAXLENGTH16BYTES; } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToField128(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return format128(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfo128(attr, i); } SimpleValue getMinValueSimple() const override { - return SimpleValue(std::numeric_limits::min(), - datatypes::minInt128, - 0); + return SimpleValue(std::numeric_limits::min(), datatypes::minInt128, 0); } SimpleValue getMaxValueSimple() const override { - return SimpleValue(std::numeric_limits::max(), - datatypes::maxInt128, - 0); + return SimpleValue(std::numeric_limits::max(), datatypes::maxInt128, 0); } - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenSInt128(a, b); } - MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const override + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const override { return getExtentPartitionInfo128(attr, em, entry, state); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return PrintPartitionValue128(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUDecimal128: public TypeHandlerXDecimal +class TypeHandlerUDecimal128 : public TypeHandlerXDecimal { -public: - const string &name() const override; + public: + const string& name() const override; code_t code() const override { return SystemCatalog::UDECIMAL; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) - const override + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd& attr) const override { return Decimal::MAXLENGTH16BYTES; } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToField128(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { return format128(v, attr); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfo128(attr, i); } @@ -2212,677 +1992,537 @@ public: { return SimpleValueSInt128(-1); } - MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &a, - const MinMaxInfo &b) const override + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd& attr, const MinMaxInfo& a, + const MinMaxInfo& b) const override { return MinMaxInfo::widenSInt128(a, b); } - MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const override + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const override { return getExtentPartitionInfo128(attr, em, entry, state); } - string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &partInfo, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxPartitionInfo& partInfo, const SimpleValue& startVal, + round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return PrintPartitionValue128(attr, partInfo, startVal, rfMin, endVal, rfMax); } - bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxPartitionInfo &part, - const SimpleValue &startVal, - round_style_t rfMin, - const SimpleValue &endVal, + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd& attr, const MinMaxPartitionInfo& part, + const SimpleValue& startVal, round_style_t rfMin, const SimpleValue& endVal, round_style_t rfMax) const override { return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerReal: public TypeHandler +class TypeHandlerReal : public TypeHandler { -public: - int storeValueToFieldXFloat(rowgroup::Row &row, int pos, - StoreField *f) const; - int storeValueToFieldXDouble(rowgroup::Row &row, int pos, - StoreField *f) const; - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override + public: + int storeValueToFieldXFloat(rowgroup::Row& row, int pos, StoreField* f) const; + int storeValueToFieldXDouble(rowgroup::Row& row, int pos, StoreField* f) const; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override { - return SimpleValue(); // QQ: real types were not handled in IDB_format() + return SimpleValue(); // QQ: real types were not handled in IDB_format() } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { - return "0"; // QQ + return "0"; // QQ } }; - -class TypeHandlerSFloat: public TypeHandlerReal +class TypeHandlerSFloat : public TypeHandlerReal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::FLOAT; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchXFloat(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldXFloat(row, pos, f); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSDouble: public TypeHandlerReal +class TypeHandlerSDouble : public TypeHandlerReal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::DOUBLE; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchXDouble(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldXDouble(row, pos, f); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUFloat: public TypeHandlerReal +class TypeHandlerUFloat : public TypeHandlerReal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::UFLOAT; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchXFloat(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldXFloat(row, pos, f); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerUDouble: public TypeHandlerReal +class TypeHandlerUDouble : public TypeHandlerReal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::UDOUBLE; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchXDouble(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldXDouble(row, pos, f); } - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerSLongDouble: public TypeHandlerReal +class TypeHandlerSLongDouble : public TypeHandlerReal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::LONGDOUBLE; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchSLongDouble(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { idbassert(0); return "Error"; } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { // QQ: DDLPackageProcessor::getNullValueForType() did not handle LONGDOUBLE return boost::any(); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override { throw logging::QueryDataExcept("convertColumnData: unknown column data type.", logging::dataTypeErr); return boost::any(); } - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { idbassert(0); return nullptr; } }; -const uint8_t* getEmptyTypeHandlerStr(const SystemCatalog::TypeAttributesStd &attr, int8_t offset); +const uint8_t* getEmptyTypeHandlerStr(const SystemCatalog::TypeAttributesStd& attr, int8_t offset); -class TypeHandlerStr: public TypeHandler +class TypeHandlerStr : public TypeHandler { -protected: - std::string formatPartitionInfoSmallCharVarchar( - const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const; - boost::any getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd &attr) - const; -public: - int storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, - StoreField *f) const; - int storeValueToFieldBlobText(rowgroup::Row &row, int pos, - StoreField *f) const; - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + protected: + std::string formatPartitionInfoSmallCharVarchar(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const; + boost::any getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd& attr) const; + + public: + int storeValueToFieldCharVarchar(rowgroup::Row& row, int pos, StoreField* f) const; + int storeValueToFieldBlobText(rowgroup::Row& row, int pos, StoreField* f) const; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { // QQ: Check with Roman if this correct: return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerChar: public TypeHandlerStr +class TypeHandlerChar : public TypeHandlerStr { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::CHAR; } - const string print(const SystemCatalog::TypeAttributesStd &attr) const override + const string print(const SystemCatalog::TypeAttributesStd& attr) const override { ostringstream oss; - oss << name () << "(" << attr.colWidth << ")"; + oss << name() << "(" << attr.colWidth << ")"; return oss.str(); } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return attr.colWidth <= 8; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchChar(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldCharVarchar(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override; - MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override; + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerVarchar: public TypeHandlerStr +class TypeHandlerVarchar : public TypeHandlerStr { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::VARCHAR; } - const string print(const SystemCatalog::TypeAttributesStd &attr) const override + const string print(const SystemCatalog::TypeAttributesStd& attr) const override { ostringstream oss; - oss << name () << "(" << attr.colWidth << ")"; + oss << name() << "(" << attr.colWidth << ")"; return oss.str(); } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return attr.colWidth <= 7; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchVarchar(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldCharVarchar(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override; - MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - BRM::DBRM &em, - const BRM::EMEntry &entry, - int *state) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override; + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, BRM::DBRM& em, + const BRM::EMEntry& entry, int* state) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { return getNullValueForTypeVarcharText(attr); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; }; - -class TypeHandlerVarbinary: public TypeHandlerStr +class TypeHandlerVarbinary : public TypeHandlerStr { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::VARBINARY; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchVarbinary(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerBlob: public TypeHandlerStr +class TypeHandlerBlob : public TypeHandlerStr { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::BLOB; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchBlob(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldBlobText(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { - return "0"; // QQ + return "0"; // QQ } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerText: public TypeHandlerStr +class TypeHandlerText : public TypeHandlerStr { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::TEXT; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchBlob(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { return storeValueToFieldBlobText(row, pos, f); } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { - return "0"; // QQ + return "0"; // QQ } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { return getNullValueForTypeVarcharText(attr); } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerClob: public TypeHandlerStr +class TypeHandlerClob : public TypeHandlerStr { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::CLOB; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { - idbassert(0); // QQ + idbassert(0); // QQ return 0; } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override { - idbassert(0); // QQ + idbassert(0); // QQ return 1; } - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override { - return "0"; // QQ + return "0"; // QQ } - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { - return boost::any(); // QQ + return boost::any(); // QQ } boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerTemporal: public TypeHandler +class TypeHandlerTemporal : public TypeHandler { -public: - std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, - const MinMaxInfo &i) - const override + public: + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd& attr, + const MinMaxInfo& i) const override { return formatPartitionInfoSInt64(attr, i); } - execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, - SystemCatalog::TypeHolderStd &ct, - const SimpleColumnParam &prm) - const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd &attr) const override + execplan::SimpleColumn* newSimpleColumn(const DatabaseQualifiedColumnName& name, + SystemCatalog::TypeHolderStd& ct, + const SimpleColumnParam& prm) const override; + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& attr) const override { return getEmptyTypeHandlerStr(attr, 0); } }; - -class TypeHandlerDate: public TypeHandlerTemporal +class TypeHandlerDate : public TypeHandlerTemporal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::DATE; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchDate(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerDatetime: public TypeHandlerTemporal +class TypeHandlerDatetime : public TypeHandlerTemporal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::DATETIME; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchDatetime(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerTime: public TypeHandlerTemporal +class TypeHandlerTime : public TypeHandlerTemporal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::TIME; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchTime(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; - -class TypeHandlerTimestamp: public TypeHandlerTemporal +class TypeHandlerTimestamp : public TypeHandlerTemporal { - const string &name() const override; + const string& name() const override; code_t code() const override { return SystemCatalog::TIMESTAMP; } - bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + bool CP_type(const SystemCatalog::TypeAttributesStd& attr) const override { return true; } - size_t ColWriteBatch(WriteBatchField *field, - const unsigned char *buf, - bool nullVal, - ColBatchWriter & writer) const override + size_t ColWriteBatch(WriteBatchField* field, const unsigned char* buf, bool nullVal, + ColBatchWriter& writer) const override { return field->ColWriteBatchTimestamp(buf, nullVal, writer); } - int storeValueToField(rowgroup::Row &row, int pos, - StoreField *f) const override; - std::string format(const SimpleValue &v, - const SystemCatalog::TypeAttributesStd &attr) const override; - SimpleValue toSimpleValue(const SessionParam &sp, - const SystemCatalog::TypeAttributesStd &attr, - const char *str, round_style_t & rf) const override; - boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) - const override; + int storeValueToField(rowgroup::Row& row, int pos, StoreField* f) const override; + std::string format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const override; + SimpleValue toSimpleValue(const SessionParam& sp, const SystemCatalog::TypeAttributesStd& attr, + const char* str, round_style_t& rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd& attr) const override; boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, - const ConvertFromStringParam& prm, - const std::string& str, + const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; }; -}// end of namespace datatypes +} // end of namespace datatypes -#endif //MCS_DATATYPE_H_INCLUDED +#endif // MCS_DATATYPE_H_INCLUDED // vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype_basic.h b/datatypes/mcs_datatype_basic.h index 9450692f3..5ba6cfc17 100644 --- a/datatypes/mcs_datatype_basic.h +++ b/datatypes/mcs_datatype_basic.h @@ -26,13 +26,11 @@ namespace datatypes { - - // Convert a positive floating point value to // a signed or unsigned integer with rounding // SRC - a floating point data type (float, double, float128_t) // DST - a signed or unsigned integer data type (int32_t, uint64_t, int128_t, etc) -template +template DST positiveXFloatToXIntRound(SRC value, DST limit) { SRC tmp = value + 0.5; @@ -41,12 +39,11 @@ DST positiveXFloatToXIntRound(SRC value, DST limit) return static_cast(tmp); } - // Convert a negative floating point value to // a signed integer with rounding // SRC - a floating point data type (float, double, float128_t) // DST - a signed integer data type (int32_t, int64_t, int128_t, etc) -template +template DST negativeXFloatToXIntRound(SRC value, DST limit) { SRC tmp = value - 0.5; @@ -55,41 +52,32 @@ DST negativeXFloatToXIntRound(SRC value, DST limit) return static_cast(tmp); } - // Convert a floating point value to ColumnStore int64_t // Magic values cannot be returned. -template +template int64_t xFloatToMCSSInt64Round(SRC value) { if (value > 0) - return positiveXFloatToXIntRound( - value, - numeric_limits::max()); + return positiveXFloatToXIntRound(value, numeric_limits::max()); if (value < 0) - return negativeXFloatToXIntRound( - value, - numeric_limits::min() + 2); + return negativeXFloatToXIntRound(value, numeric_limits::min() + 2); return 0; } - // Convert a floating point value to ColumnStore uint64_t // Magic values cannot be returned. -template +template uint64_t xFloatToMCSUInt64Round(SRC value) { if (value > 0) - return positiveXFloatToXIntRound( - value, - numeric_limits::max() - 2); + return positiveXFloatToXIntRound(value, numeric_limits::max() - 2); if (value < 0) return negativeXFloatToXIntRound(value, 0); return 0; } +} // end of namespace datatypes -} //end of namespace datatypes - -#endif // MCS_DATATYPE_BASIC_H_INCLUDED +#endif // MCS_DATATYPE_BASIC_H_INCLUDED // vim:ts=2 sw=2: diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 5ede71e9f..a5b74f409 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -16,7 +16,7 @@ MA 02110-1301, USA. */ #include -#include +#include #include "utils/common/branchpred.h" #include "mcs_decimal.h" @@ -24,667 +24,614 @@ namespace datatypes { - template - void addSubtractExecute(const Decimal& l, - const Decimal& r, - Decimal& result, - BinaryOperation op, - OpOverflowCheck opOverflowCheck, - MultiplicationOverflowCheck mulOverflowCheck) +template +void addSubtractExecute(const Decimal& l, const Decimal& r, Decimal& result, BinaryOperation op, + OpOverflowCheck opOverflowCheck, MultiplicationOverflowCheck mulOverflowCheck) +{ + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) ? r.s128Value : r.value; + + if (result.scale == l.scale && result.scale == r.scale) + { + opOverflowCheck(lValue, rValue); + result.s128Value = op(lValue, rValue); + return; + } + + if (result.scale > l.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, result.scale - l.scale); + mulOverflowCheck(lValue, scaleMultiplier); + lValue *= scaleMultiplier; + } + else if (result.scale < l.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, l.scale - result.scale); + lValue = (int128_t)(lValue > 0 ? (float128_t)lValue / scaleMultiplier + 0.5 + : (float128_t)lValue / scaleMultiplier - 0.5); + } + + if (result.scale > r.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, result.scale - r.scale); + mulOverflowCheck(rValue, scaleMultiplier); + rValue *= scaleMultiplier; + } + else if (result.scale < r.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, r.scale - result.scale); + rValue = (int128_t)(rValue > 0 ? (float128_t)rValue / scaleMultiplier + 0.5 + : (float128_t)rValue / scaleMultiplier - 0.5); + } + + // We assume there is no way that lValue or rValue calculations + // give an overflow and this is an incorrect assumption. + opOverflowCheck(lValue, rValue); + + result.s128Value = op(lValue, rValue); +} + +template +void divisionExecute(const Decimal& l, const Decimal& r, Decimal& result, OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) +{ + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) ? r.s128Value : r.value; + + opOverflowCheck(lValue, rValue); + if (result.scale >= l.scale - r.scale) + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, result.scale - (l.scale - r.scale)); + + // TODO How do we check overflow of (int128_t)((float128_t)lValue / rValue * scaleMultiplier) ? + + result.s128Value = (int128_t)(((lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) + ? (float128_t)lValue / rValue * scaleMultiplier + 0.5 + : (float128_t)lValue / rValue * scaleMultiplier - 0.5)); + } + else + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, (l.scale - r.scale) - result.scale); + + result.s128Value = (int128_t)(((lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) + ? (float128_t)lValue / rValue / scaleMultiplier + 0.5 + : (float128_t)lValue / rValue / scaleMultiplier - 0.5)); + } +} + +template +void multiplicationExecute(const Decimal& l, const Decimal& r, Decimal& result, + OpOverflowCheck opOverflowCheck, MultiplicationOverflowCheck mulOverflowCheck) +{ + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) ? r.s128Value : r.value; + + if (lValue == 0 || rValue == 0) + { + result.s128Value = 0; + return; + } + + if (result.scale >= l.scale + r.scale) + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, result.scale - (l.scale + r.scale)); + + opOverflowCheck(lValue, rValue, result.s128Value); + opOverflowCheck(result.s128Value, scaleMultiplier, result.s128Value); + } + else + { + unsigned int diff = l.scale + r.scale - result.scale; + + int128_t scaleMultiplierL, scaleMultiplierR; + + getScaleDivisor(scaleMultiplierL, diff / 2); + getScaleDivisor(scaleMultiplierR, diff - (diff / 2)); + + lValue = (int128_t)(((lValue > 0) ? (float128_t)lValue / scaleMultiplierL + 0.5 + : (float128_t)lValue / scaleMultiplierL - 0.5)); + + rValue = (int128_t)(((rValue > 0) ? (float128_t)rValue / scaleMultiplierR + 0.5 + : (float128_t)rValue / scaleMultiplierR - 0.5)); + + opOverflowCheck(lValue, rValue, result.s128Value); + ; + } +} + +Decimal::Decimal(const char* str, size_t length, DataCondition& convError, int8_t s, uint8_t p) + : scale(s), precision(p) +{ + literal::Converter conv(str, length, convError); + // We don't check "convErr" here. Let the caller do it. + // Let's just convert what has been parsed. + + // Remove redundant leading integral and trailing fractional digits + conv.normalize(); + if (isTSInt128ByPrecision()) + { + s128Value = conv.toPackedSDecimal((literal::scale_t)scale, convError); + int128_t max_number_decimal = mcs_pow_10_128[precision - 19] - 1; + convError.adjustSIntXRange(s128Value, max_number_decimal); + } + else + { + value = conv.toPackedSDecimal((literal::scale_t)scale, convError); + int64_t max_number_decimal = (int64_t)mcs_pow_10[precision] - 1; + convError.adjustSIntXRange(value, max_number_decimal); + } +} + +int Decimal::compare(const Decimal& l, const Decimal& r) +{ + int128_t divisorL, divisorR; + getScaleDivisor(divisorL, l.scale); + getScaleDivisor(divisorR, r.scale); + + lldiv_t_128 d1 = lldiv128(l.s128Value, divisorL); + lldiv_t_128 d2 = lldiv128(r.s128Value, divisorR); + + int ret = 0; + + if (d1.quot > d2.quot) + { + ret = 1; + } + else if (d1.quot < d2.quot) + { + ret = -1; + } + else + { + // rem carries the value's sign, but needs to be normalized. + int64_t s = l.scale - r.scale; + int128_t divisor; + getScaleDivisor(divisor, std::abs(s)); + + if (s < 0) { - int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) - ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) - ? r.s128Value : r.value; - - if (result.scale == l.scale && result.scale == r.scale) - { - opOverflowCheck(lValue, rValue); - result.s128Value = op(lValue, rValue); - return; - } - - if (result.scale > l.scale) - { - int128_t scaleMultiplier; - getScaleDivisor(scaleMultiplier, result.scale - l.scale); - mulOverflowCheck(lValue, scaleMultiplier); - lValue *= scaleMultiplier; - } - else if (result.scale < l.scale) - { - int128_t scaleMultiplier; - getScaleDivisor(scaleMultiplier, l.scale - result.scale); - lValue = (int128_t) (lValue > 0 ? - (float128_t)lValue / scaleMultiplier + 0.5 : - (float128_t)lValue / scaleMultiplier - 0.5); - } - - if (result.scale > r.scale) - { - int128_t scaleMultiplier; - getScaleDivisor(scaleMultiplier, result.scale - r.scale); - mulOverflowCheck(rValue, scaleMultiplier); - rValue *= scaleMultiplier; - } - else if (result.scale < r.scale) - { - int128_t scaleMultiplier; - getScaleDivisor(scaleMultiplier, r.scale - result.scale); - rValue = (int128_t) (rValue > 0 ? - (float128_t)rValue / scaleMultiplier + 0.5 : - (float128_t)rValue / scaleMultiplier - 0.5); - } - - // We assume there is no way that lValue or rValue calculations - // give an overflow and this is an incorrect assumption. - opOverflowCheck(lValue, rValue); - - result.s128Value = op(lValue, rValue); + if ((d1.rem * divisor) > d2.rem) + ret = 1; + else if ((d1.rem * divisor) < d2.rem) + ret = -1; } - - template - void divisionExecute(const Decimal& l, - const Decimal& r, - Decimal& result, - OpOverflowCheck opOverflowCheck, - MultiplicationOverflowCheck mulOverflowCheck) + else { - int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) - ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) - ? r.s128Value : r.value; - - opOverflowCheck(lValue, rValue); - if (result.scale >= l.scale - r.scale) - { - int128_t scaleMultiplier; - - getScaleDivisor(scaleMultiplier, result.scale - (l.scale - r.scale)); - - // TODO How do we check overflow of (int128_t)((float128_t)lValue / rValue * scaleMultiplier) ? - - result.s128Value = (int128_t)(( (lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) ? - (float128_t)lValue / rValue * scaleMultiplier + 0.5 : - (float128_t)lValue / rValue * scaleMultiplier - 0.5)); - } - else - { - int128_t scaleMultiplier; - - getScaleDivisor(scaleMultiplier, (l.scale - r.scale) - result.scale); - - result.s128Value = (int128_t)(( (lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) ? - (float128_t)lValue / rValue / scaleMultiplier + 0.5 : - (float128_t)lValue / rValue / scaleMultiplier - 0.5)); - } + if (d1.rem > (d2.rem * divisor)) + ret = 1; + else if (d1.rem < (d2.rem * divisor)) + ret = -1; } + } - template - void multiplicationExecute(const Decimal& l, - const Decimal& r, - Decimal& result, - OpOverflowCheck opOverflowCheck, - MultiplicationOverflowCheck mulOverflowCheck) + return ret; +} + +// no overflow check +template <> +void Decimal::addition(const Decimal& l, const Decimal& r, Decimal& result) +{ + std::plus add; + NoOverflowCheck noOverflowCheck; + addSubtractExecute(l, r, result, add, noOverflowCheck, noOverflowCheck); +} + +// with overflow check +template <> +void Decimal::addition(const Decimal& l, const Decimal& r, Decimal& result) +{ + std::plus add; + AdditionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + addSubtractExecute(l, r, result, add, overflowCheck, mulOverflowCheck); +} + +// no overflow check +template <> +void Decimal::addition(const Decimal& l, const Decimal& r, Decimal& result) +{ + if (result.scale == l.scale && result.scale == r.scale) + { + result.value = l.value + r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + lValue *= mcs_pow_10[result.scale - l.scale]; + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 + : (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + rValue *= mcs_pow_10[result.scale - r.scale]; + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 + : (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + result.value = lValue + rValue; +} + +// with overflow check +template <> +void Decimal::addition(const Decimal& l, const Decimal& r, Decimal& result) +{ + AdditionOverflowCheck additionOverflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale == l.scale && result.scale == r.scale) + { + additionOverflowCheck(l.value, r.value); + result.value = l.value + r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 + : (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 + : (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + additionOverflowCheck(lValue, rValue); + result.value = lValue + rValue; +} + +// no overflow check +template <> +void Decimal::subtraction(const Decimal& l, const Decimal& r, Decimal& result) +{ + std::minus subtract; + NoOverflowCheck noOverflowCheck; + addSubtractExecute(l, r, result, subtract, noOverflowCheck, noOverflowCheck); +} + +// with overflow check +template <> +void Decimal::subtraction(const Decimal& l, const Decimal& r, Decimal& result) +{ + std::minus subtract; + SubtractionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + addSubtractExecute(l, r, result, subtract, overflowCheck, mulOverflowCheck); +} + +// no overflow check +template <> +void Decimal::subtraction(const Decimal& l, const Decimal& r, Decimal& result) +{ + if (result.scale == l.scale && result.scale == r.scale) + { + result.value = l.value - r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + lValue *= mcs_pow_10[result.scale - l.scale]; + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 + : (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + rValue *= mcs_pow_10[result.scale - r.scale]; + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 + : (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + result.value = lValue - rValue; +} + +// with overflow check +template <> +void Decimal::subtraction(const Decimal& l, const Decimal& r, Decimal& result) +{ + SubtractionOverflowCheck subtractionOverflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale == l.scale && result.scale == r.scale) + { + subtractionOverflowCheck(l.value, r.value); + result.value = l.value - r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 + : (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 + : (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + subtractionOverflowCheck(lValue, rValue); + result.value = lValue - rValue; +} + +// no overflow check +template <> +void Decimal::division(const Decimal& l, const Decimal& r, Decimal& result) +{ + NoOverflowCheck noOverflowCheck; + divisionExecute(l, r, result, noOverflowCheck, noOverflowCheck); +} + +// With overflow check +template <> +void Decimal::division(const Decimal& l, const Decimal& r, Decimal& result) +{ + DivisionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + divisionExecute(l, r, result, overflowCheck, mulOverflowCheck); +} + +// no overflow check +// We rely on the zero check from ArithmeticOperator::execute +template <> +void Decimal::division(const Decimal& l, const Decimal& r, Decimal& result) +{ + if (result.scale >= l.scale - r.scale) + result.value = (int64_t)(( + (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) + ? (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 + : (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( + (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) + ? (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 + : (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); +} + +// With overflow check +template <> +void Decimal::division(const Decimal& l, const Decimal& r, Decimal& result) +{ + DivisionOverflowCheck divisionOverflowCheck; + + divisionOverflowCheck(l.value, r.value); + + if (result.scale >= l.scale - r.scale) + // TODO How do we check overflow of (int64_t)((long double)l.value / r.value * mcs_pow_10[result.scale - + // (l.scale - r.scale)]) ? + result.value = (int64_t)(( + (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) + ? (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 + : (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( + (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) + ? (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 + : (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); +} + +// no overflow check +template <> +void Decimal::multiplication(const Decimal& l, const Decimal& r, Decimal& result) +{ + MultiplicationNoOverflowCheck noOverflowCheck; + multiplicationExecute(l, r, result, noOverflowCheck, noOverflowCheck); +} + +// With overflow check +template <> +void Decimal::multiplication(const Decimal& l, const Decimal& r, Decimal& result) +{ + MultiplicationOverflowCheck mulOverflowCheck; + multiplicationExecute(l, r, result, mulOverflowCheck, mulOverflowCheck); +} + +// no overflow check +template <> +void Decimal::multiplication(const Decimal& l, const Decimal& r, Decimal& result) +{ + if (result.scale >= l.scale + r.scale) + result.value = l.value * r.value * mcs_pow_10[result.scale - (l.scale + r.scale)]; + else + result.value = + (int64_t)(((l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) + ? (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 + : (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); +} + +// With overflow check +template <> +void Decimal::multiplication(const Decimal& l, const Decimal& r, Decimal& result) +{ + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale >= l.scale + r.scale) + { + mulOverflowCheck(l.value, r.value, result.value); + mulOverflowCheck(result.value, mcs_pow_10[result.scale - (l.scale + r.scale)], result.value); + } + else + { + mulOverflowCheck(l.value, r.value, result.value); + + result.value = (int64_t)(( + (result.value > 0) ? (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 + : (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); + } +} + +// Writes integer part of a Decimal using int128 argument provided +uint8_t Decimal::writeIntPart(const int128_t& x, char* buf, const uint8_t buflen) const +{ + char* p = buf; + int128_t intPart = x; + int128_t high = 0, mid = 0, low = 0; + uint64_t maxUint64divisor = 10000000000000000000ULL; + + // Assuming scale = [0, 56] + switch (scale / datatypes::maxPowOf10) + { + case 2: // scale = [38, 56] + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + low = intPart; + break; + case 1: // scale = [19, 37] + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + mid = intPart / maxUint64divisor; + break; + case 0: // scale = [0, 18] + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + intPart /= maxUint64divisor; + mid = intPart % maxUint64divisor; + high = intPart / maxUint64divisor; + break; + default: throw logging::QueryDataExcept("Decimal::writeIntPart() bad scale", logging::formatErr); + } + + p += printPodParts(p, high, mid, low); + uint8_t written = p - buf; + if (buflen <= written) + { + throw logging::QueryDataExcept("Decimal::writeIntPart() char buffer overflow.", logging::formatErr); + } + + return written; +} + +uint8_t Decimal::writeFractionalPart(const int128_t& x, char* buf, const uint8_t buflen) const +{ + int128_t scaleDivisor = 1; + char* p = buf; + + switch (scale / datatypes::maxPowOf10) + { + case 2: + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + break; + case 1: + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + // fallthrough + case 0: scaleDivisor *= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + } + + int128_t fractionalPart = x % scaleDivisor; + + // divide by the base until we have non-zero quotient + scaleDivisor /= 10; + + while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0) + { + *p++ = '0'; + scaleDivisor /= 10; + } + size_t written = p - buf; + ; + p += TSInt128::writeIntPart(fractionalPart, p, buflen - written); + return p - buf; +} + +// The method writes Decimal based on TSInt128 with scale provided. +// It first writes sign, then extracts integer part +// prints delimiter and then decimal part. +std::string Decimal::toStringTSInt128WithScale() const +{ + char buf[Decimal::MAXLENGTH16BYTES]; + uint8_t left = sizeof(buf); + char* p = buf; + int128_t tempValue = s128Value; + // sign + if (tempValue < static_cast(0)) + { + *p++ = '-'; + tempValue *= -1; + left--; + } + + // integer part + p += writeIntPart(tempValue, p, left); + + // decimal delimiter + *p++ = '.'; + // decimal part + left = sizeof(buf) - (p - buf); + p += writeFractionalPart(tempValue, p, left); + uint8_t written = p - buf; + + if (sizeof(buf) <= written) + { + throw logging::QueryDataExcept("Decimal::toString() char buffer overflow.", logging::formatErr); + } + + *p = '\0'; + + return std::string(buf); +} + +std::string Decimal::toStringTSInt64() const +{ + char buf[Decimal::MAXLENGTH8BYTES]; + uint64_t divisor = scaleDivisor(scale); + uint64_t uvalue = value < 0 ? (uint64_t)-value : (uint64_t)value; + uint64_t intg = uvalue / divisor; + uint64_t frac = uvalue % divisor; + int nbytes = snprintf(buf, sizeof(buf), "%s%" PRIu64, value < 0 ? "-" : "", intg); + if (scale > 0) + snprintf(buf + nbytes, sizeof(buf) - nbytes, ".%.*" PRIu64, (int)scale, frac); + return std::string(buf); +} + +// Dispatcher method for toString() implementations +std::string Decimal::toString(bool hasTSInt128) const +{ + // There must be no empty at this point though + if (isNull()) + { + return std::string("NULL"); + } + + if (LIKELY(hasTSInt128 || isTSInt128ByPrecision())) + { + if (scale) { - int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) - ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) - ? r.s128Value : r.value; - - if (lValue == 0 || rValue == 0) - { - result.s128Value = 0; - return; - } - - if (result.scale >= l.scale + r.scale) - { - int128_t scaleMultiplier; - - getScaleDivisor(scaleMultiplier, result.scale - (l.scale + r.scale)); - - opOverflowCheck(lValue, rValue, result.s128Value); - opOverflowCheck(result.s128Value, scaleMultiplier, result.s128Value); - } - else - { - unsigned int diff = l.scale + r.scale - result.scale; - - int128_t scaleMultiplierL, scaleMultiplierR; - - getScaleDivisor(scaleMultiplierL, diff / 2); - getScaleDivisor(scaleMultiplierR, diff - (diff / 2)); - - lValue = (int128_t)(( (lValue > 0) ? - (float128_t)lValue / scaleMultiplierL + 0.5 : - (float128_t)lValue / scaleMultiplierL - 0.5)); - - rValue = (int128_t)(( (rValue > 0) ? - (float128_t)rValue / scaleMultiplierR + 0.5 : - (float128_t)rValue / scaleMultiplierR - 0.5)); - - opOverflowCheck(lValue, rValue, result.s128Value);; - } + return toStringTSInt128WithScale(); } + return TSInt128::toString(); + } + // TSInt64 Decimal + if (scale) + { + return toStringTSInt64(); + } + return std::to_string(value); +} - Decimal::Decimal(const char *str, size_t length, DataCondition & convError, - int8_t s, uint8_t p) - :scale(s), - precision(p) - { - literal::Converter conv(str, length, convError); - // We don't check "convErr" here. Let the caller do it. - // Let's just convert what has been parsed. - - // Remove redundant leading integral and trailing fractional digits - conv.normalize(); - if (isTSInt128ByPrecision()) - { - s128Value = conv.toPackedSDecimal((literal::scale_t) scale, convError); - int128_t max_number_decimal = mcs_pow_10_128[precision - 19] - 1; - convError.adjustSIntXRange(s128Value, max_number_decimal); - } - else - { - value = conv.toPackedSDecimal((literal::scale_t) scale, convError); - int64_t max_number_decimal = (int64_t) mcs_pow_10[precision] - 1; - convError.adjustSIntXRange(value, max_number_decimal); - } - } - - int Decimal::compare(const Decimal& l, const Decimal& r) - { - int128_t divisorL, divisorR; - getScaleDivisor(divisorL, l.scale); - getScaleDivisor(divisorR, r.scale); - - lldiv_t_128 d1 = lldiv128(l.s128Value, divisorL); - lldiv_t_128 d2 = lldiv128(r.s128Value, divisorR); - - int ret = 0; - - if (d1.quot > d2.quot) - { - ret = 1; - } - else if (d1.quot < d2.quot) - { - ret = -1; - } - else - { - // rem carries the value's sign, but needs to be normalized. - int64_t s = l.scale - r.scale; - int128_t divisor; - getScaleDivisor(divisor, std::abs(s)); - - if (s < 0) - { - if ((d1.rem * divisor) > d2.rem) - ret = 1; - else if ((d1.rem * divisor) < d2.rem) - ret = -1; - } - else - { - if (d1.rem > (d2.rem * divisor)) - ret = 1; - else if (d1.rem < (d2.rem * divisor)) - ret = -1; - } - } - - return ret; - } - - // no overflow check - template<> - void Decimal::addition(const Decimal& l, - const Decimal& r, Decimal& result) - { - std::plus add; - NoOverflowCheck noOverflowCheck; - addSubtractExecute(l, r, result, add, noOverflowCheck, noOverflowCheck); - } - - // with overflow check - template<> - void Decimal::addition(const Decimal& l, - const Decimal& r, Decimal& result) - { - std::plus add; - AdditionOverflowCheck overflowCheck; - MultiplicationOverflowCheck mulOverflowCheck; - addSubtractExecute(l, r, result, add, overflowCheck, mulOverflowCheck); - } - - // no overflow check - template<> - void Decimal::addition(const Decimal& l, - const Decimal& r, Decimal& result) - { - if (result.scale == l.scale && result.scale == r.scale) - { - result.value = l.value + r.value; - return; - } - - int64_t lValue = l.value, rValue = r.value; - - if (result.scale > l.scale) - lValue *= mcs_pow_10[result.scale - l.scale]; - else if (result.scale < l.scale) - lValue = (int64_t)(lValue > 0 ? - (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : - (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); - - if (result.scale > r.scale) - rValue *= mcs_pow_10[result.scale - r.scale]; - else if (result.scale < r.scale) - rValue = (int64_t)(rValue > 0 ? - (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : - (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); - - result.value = lValue + rValue; - } - - // with overflow check - template<> - void Decimal::addition(const Decimal& l, - const Decimal& r, Decimal& result) - { - AdditionOverflowCheck additionOverflowCheck; - MultiplicationOverflowCheck mulOverflowCheck; - - if (result.scale == l.scale && result.scale == r.scale) - { - additionOverflowCheck(l.value, r.value); - result.value = l.value + r.value; - return; - } - - int64_t lValue = l.value, rValue = r.value; - - if (result.scale > l.scale) - mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); - else if (result.scale < l.scale) - lValue = (int64_t)(lValue > 0 ? - (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : - (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); - - if (result.scale > r.scale) - mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); - else if (result.scale < r.scale) - rValue = (int64_t)(rValue > 0 ? - (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : - (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); - - additionOverflowCheck(lValue, rValue); - result.value = lValue + rValue; - } - - // no overflow check - template<> - void Decimal::subtraction(const Decimal& l, - const Decimal& r, Decimal& result) - { - std::minus subtract; - NoOverflowCheck noOverflowCheck; - addSubtractExecute(l, r, result, subtract, noOverflowCheck, noOverflowCheck); - } - - // with overflow check - template<> - void Decimal::subtraction(const Decimal& l, - const Decimal& r, Decimal& result) - { - std::minus subtract; - SubtractionOverflowCheck overflowCheck; - MultiplicationOverflowCheck mulOverflowCheck; - addSubtractExecute(l, r, result, subtract, overflowCheck, mulOverflowCheck); - } - - // no overflow check - template<> - void Decimal::subtraction(const Decimal& l, - const Decimal& r, Decimal& result) - { - if (result.scale == l.scale && result.scale == r.scale) - { - result.value = l.value - r.value; - return; - } - - int64_t lValue = l.value, rValue = r.value; - - if (result.scale > l.scale) - lValue *= mcs_pow_10[result.scale - l.scale]; - else if (result.scale < l.scale) - lValue = (int64_t)(lValue > 0 ? - (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : - (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); - - if (result.scale > r.scale) - rValue *= mcs_pow_10[result.scale - r.scale]; - else if (result.scale < r.scale) - rValue = (int64_t)(rValue > 0 ? - (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : - (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); - - result.value = lValue - rValue; - } - - // with overflow check - template<> - void Decimal::subtraction(const Decimal& l, - const Decimal& r, Decimal& result) - { - SubtractionOverflowCheck subtractionOverflowCheck; - MultiplicationOverflowCheck mulOverflowCheck; - - if (result.scale == l.scale && result.scale == r.scale) - { - subtractionOverflowCheck(l.value, r.value); - result.value = l.value - r.value; - return; - } - - int64_t lValue = l.value, rValue = r.value; - - if (result.scale > l.scale) - mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); - else if (result.scale < l.scale) - lValue = (int64_t)(lValue > 0 ? - (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : - (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); - - if (result.scale > r.scale) - mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); - else if (result.scale < r.scale) - rValue = (int64_t)(rValue > 0 ? - (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : - (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); - - subtractionOverflowCheck(lValue, rValue); - result.value = lValue - rValue; - } - - // no overflow check - template<> - void Decimal::division(const Decimal& l, - const Decimal& r, Decimal& result) - { - NoOverflowCheck noOverflowCheck; - divisionExecute(l, r, result, noOverflowCheck, noOverflowCheck); - } - - // With overflow check - template<> - void Decimal::division(const Decimal& l, - const Decimal& r, Decimal& result) - { - DivisionOverflowCheck overflowCheck; - MultiplicationOverflowCheck mulOverflowCheck; - divisionExecute(l, r, result, overflowCheck, mulOverflowCheck); - } - - // no overflow check - // We rely on the zero check from ArithmeticOperator::execute - template<> - void Decimal::division(const Decimal& l, - const Decimal& r, Decimal& result) - { - if (result.scale >= l.scale - r.scale) - result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? - (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : - (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); - else - result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? - (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : - (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); - } - - // With overflow check - template<> - void Decimal::division(const Decimal& l, - const Decimal& r, Decimal& result) - { - DivisionOverflowCheck divisionOverflowCheck; - - divisionOverflowCheck(l.value, r.value); - - if (result.scale >= l.scale - r.scale) - // TODO How do we check overflow of (int64_t)((long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)]) ? - result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? - (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : - (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); - else - result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? - (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : - (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); - } - - // no overflow check - template<> - void Decimal::multiplication(const Decimal& l, - const Decimal& r, Decimal& result) - { - MultiplicationNoOverflowCheck noOverflowCheck; - multiplicationExecute(l, r, result, noOverflowCheck, noOverflowCheck); - } - - // With overflow check - template<> - void Decimal::multiplication(const Decimal& l, - const Decimal& r, Decimal& result) - { - MultiplicationOverflowCheck mulOverflowCheck; - multiplicationExecute(l, r, result, mulOverflowCheck, mulOverflowCheck); - } - - // no overflow check - template<> - void Decimal::multiplication(const Decimal& l, - const Decimal& r, Decimal& result) - { - if (result.scale >= l.scale + r.scale) - result.value = l.value * r.value * mcs_pow_10[result.scale - (l.scale + r.scale)]; - else - result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? - (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 : - (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); - } - - // With overflow check - template<> - void Decimal::multiplication(const Decimal& l, - const Decimal& r, Decimal& result) - { - MultiplicationOverflowCheck mulOverflowCheck; - - if (result.scale >= l.scale + r.scale) - { - mulOverflowCheck(l.value, r.value, result.value); - mulOverflowCheck(result.value, mcs_pow_10[result.scale - (l.scale + r.scale)], result.value); - } - else - { - mulOverflowCheck(l.value, r.value, result.value); - - result.value = (int64_t)(( (result.value > 0) ? - (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 : - (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); - } - } - - // Writes integer part of a Decimal using int128 argument provided - uint8_t Decimal::writeIntPart(const int128_t& x, - char* buf, - const uint8_t buflen) const - { - char* p = buf; - int128_t intPart = x; - int128_t high = 0, mid = 0, low = 0; - uint64_t maxUint64divisor = 10000000000000000000ULL; - - // Assuming scale = [0, 56] - switch (scale / datatypes::maxPowOf10) - { - case 2: // scale = [38, 56] - intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - low = intPart; - break; - case 1: // scale = [19, 37] - intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; - low = intPart % maxUint64divisor; - mid = intPart / maxUint64divisor; - break; - case 0: // scale = [0, 18] - intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; - low = intPart % maxUint64divisor; - intPart /= maxUint64divisor; - mid = intPart % maxUint64divisor; - high = intPart / maxUint64divisor; - break; - default: - throw logging::QueryDataExcept("Decimal::writeIntPart() bad scale", - logging::formatErr); - } - - p += printPodParts(p, high, mid, low); - uint8_t written = p - buf; - if (buflen <= written) - { - throw logging::QueryDataExcept("Decimal::writeIntPart() char buffer overflow.", - logging::formatErr); - } - - return written; - } - - uint8_t Decimal::writeFractionalPart(const int128_t& x, - char* buf, - const uint8_t buflen) const - { - int128_t scaleDivisor = 1; - char* p = buf; - - switch (scale / datatypes::maxPowOf10) - { - case 2: - scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - break; - case 1: - scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - //fallthrough - case 0: - scaleDivisor *= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; - } - - int128_t fractionalPart = x % scaleDivisor; - - // divide by the base until we have non-zero quotient - scaleDivisor /= 10; - - while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0) - { - *p++ = '0'; - scaleDivisor /= 10; - } - size_t written = p - buf;; - p += TSInt128::writeIntPart(fractionalPart, p, buflen - written); - return p - buf; - } - - // The method writes Decimal based on TSInt128 with scale provided. - // It first writes sign, then extracts integer part - // prints delimiter and then decimal part. - std::string Decimal::toStringTSInt128WithScale() const - { - char buf[Decimal::MAXLENGTH16BYTES]; - uint8_t left = sizeof(buf); - char* p = buf; - int128_t tempValue = s128Value; - // sign - if (tempValue < static_cast(0)) - { - *p++ = '-'; - tempValue *= -1; - left--; - } - - // integer part - p += writeIntPart(tempValue, p, left); - - // decimal delimiter - *p++ = '.'; - // decimal part - left = sizeof(buf) - (p - buf); - p += writeFractionalPart(tempValue, p, left); - uint8_t written = p - buf; - - if (sizeof(buf) <= written) - { - throw logging::QueryDataExcept("Decimal::toString() char buffer overflow.", - logging::formatErr); - } - - *p = '\0'; - - return std::string(buf); - } - - std::string Decimal::toStringTSInt64() const - { - char buf[Decimal::MAXLENGTH8BYTES]; - uint64_t divisor = scaleDivisor(scale); - uint64_t uvalue = value < 0 ? (uint64_t) -value : (uint64_t) value; - uint64_t intg = uvalue / divisor; - uint64_t frac = uvalue % divisor; - int nbytes = snprintf(buf, sizeof(buf), "%s%" PRIu64, - value < 0 ? "-" : "", intg); - if (scale > 0) - snprintf(buf + nbytes, sizeof(buf) - nbytes, ".%.*" PRIu64, - (int) scale, frac); - return std::string(buf); - } - - // Dispatcher method for toString() implementations - std::string Decimal::toString(bool hasTSInt128) const - { - // There must be no empty at this point though - if (isNull()) - { - return std::string("NULL"); - } - - if(LIKELY(hasTSInt128 || isTSInt128ByPrecision())) - { - if (scale) - { - return toStringTSInt128WithScale(); - } - return TSInt128::toString(); - } - // TSInt64 Decimal - if (scale) - { - return toStringTSInt64(); - } - return std::to_string(value); - } - - std::ostream& operator<<(std::ostream& os, const Decimal& dec) - { - os << dec.toString(); - return os; - } -} // end of namespace +std::ostream& operator<<(std::ostream& os, const Decimal& dec) +{ + os << dec.toString(); + return os; +} +} // namespace datatypes diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 1c5216d19..73e2a7905 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -32,10 +32,9 @@ #include "branchpred.h" #include "mcs_data_condition.h" - namespace datatypes { - class Decimal; +class Decimal; } // A class by Fabio Fernandes pulled off of stackoverflow @@ -43,52 +42,85 @@ namespace datatypes // Ex: int128_t i128 = 12345678901234567890123456789_xxl namespace detail_xxl { - constexpr uint8_t hexval(char c) - { return c>='a' ? (10+c-'a') : c>='A' ? (10+c-'A') : c-'0'; } - - template - constexpr uint128_t lit_eval() { return V; } - - template - constexpr uint128_t lit_eval() { - static_assert( BASE!=16 || sizeof...(Cs) <= 32-1, "Literal too large for BASE=16"); - static_assert( BASE!=10 || sizeof...(Cs) <= 39-1, "Literal too large for BASE=10"); - static_assert( BASE!=8 || sizeof...(Cs) <= 44-1, "Literal too large for BASE=8"); - static_assert( BASE!=2 || sizeof...(Cs) <= 128-1, "Literal too large for BASE=2"); - return lit_eval(); - } - - template struct LitEval - {static constexpr uint128_t eval() {return lit_eval<10,0,Cs...>();} }; - - template struct LitEval<'0','x',Cs...> - {static constexpr uint128_t eval() {return lit_eval<16,0,Cs...>();} }; - - template struct LitEval<'0','b',Cs...> - {static constexpr uint128_t eval() {return lit_eval<2,0,Cs...>();} }; - - template struct LitEval<'0',Cs...> - {static constexpr uint128_t eval() {return lit_eval<8,0,Cs...>();} }; - - template - constexpr uint128_t operator "" _xxl() {return LitEval::eval();} +constexpr uint8_t hexval(char c) +{ + return c >= 'a' ? (10 + c - 'a') : c >= 'A' ? (10 + c - 'A') : c - '0'; } -template -constexpr uint128_t operator "" _xxl() {return ::detail_xxl::operator "" _xxl();} +template +constexpr uint128_t lit_eval() +{ + return V; +} + +template +constexpr uint128_t lit_eval() +{ + static_assert(BASE != 16 || sizeof...(Cs) <= 32 - 1, "Literal too large for BASE=16"); + static_assert(BASE != 10 || sizeof...(Cs) <= 39 - 1, "Literal too large for BASE=10"); + static_assert(BASE != 8 || sizeof...(Cs) <= 44 - 1, "Literal too large for BASE=8"); + static_assert(BASE != 2 || sizeof...(Cs) <= 128 - 1, "Literal too large for BASE=2"); + return lit_eval(); +} + +template +struct LitEval +{ + static constexpr uint128_t eval() + { + return lit_eval<10, 0, Cs...>(); + } +}; + +template +struct LitEval<'0', 'x', Cs...> +{ + static constexpr uint128_t eval() + { + return lit_eval<16, 0, Cs...>(); + } +}; + +template +struct LitEval<'0', 'b', Cs...> +{ + static constexpr uint128_t eval() + { + return lit_eval<2, 0, Cs...>(); + } +}; + +template +struct LitEval<'0', Cs...> +{ + static constexpr uint128_t eval() + { + return lit_eval<8, 0, Cs...>(); + } +}; + +template +constexpr uint128_t operator"" _xxl() +{ + return LitEval::eval(); +} +} // namespace detail_xxl + +template +constexpr uint128_t operator"" _xxl() +{ + return ::detail_xxl::operator"" _xxl(); +} namespace datatypes { - constexpr uint32_t MAXDECIMALWIDTH = 16U; constexpr uint8_t INT64MAXPRECISION = 18U; constexpr uint8_t INT128MAXPRECISION = 38U; constexpr uint32_t MAXLEGACYWIDTH = 8U; constexpr uint8_t MAXSCALEINC4AVG = 4U; - -const uint64_t mcs_pow_10[20] = -{ +const uint64_t mcs_pow_10[20] = { 1ULL, 10ULL, 100ULL, @@ -110,8 +142,7 @@ const uint64_t mcs_pow_10[20] = 1000000000000000000ULL, 10000000000000000000ULL, }; -const int128_t mcs_pow_10_128[20] = -{ +const int128_t mcs_pow_10_128[20] = { 10000000000000000000_xxl, 100000000000000000000_xxl, 1000000000000000000000_xxl, @@ -134,853 +165,806 @@ const int128_t mcs_pow_10_128[20] = 100000000000000000000000000000000000000_xxl, }; -constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; +constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10) / sizeof(mcs_pow_10[0]) - 1; constexpr int128_t Decimal128Null = TSInt128::NullValue; constexpr int128_t Decimal128Empty = TSInt128::EmptyValue; - -template +template T scaleDivisor(const uint32_t scale) { - if (scale < 19) - return (T) mcs_pow_10[scale]; - if (scale > 39) - { - std::string msg = "scaleDivisor called with a wrong scale: " + std::to_string(scale); - throw std::invalid_argument(msg); - } - return (T) mcs_pow_10_128[scale - 19]; + if (scale < 19) + return (T)mcs_pow_10[scale]; + if (scale > 39) + { + std::string msg = "scaleDivisor called with a wrong scale: " + std::to_string(scale); + throw std::invalid_argument(msg); + } + return (T)mcs_pow_10_128[scale - 19]; } - // Decomposed Decimal representation // T - storage data type (int64_t, int128_t) -templateclass DecomposedDecimal +template +class DecomposedDecimal { - T mDivisor; - T mIntegral; - T mFractional; -public: - DecomposedDecimal(T value, uint32_t scale) - :mDivisor(scaleDivisor(scale)), - mIntegral(value / mDivisor), - mFractional(value % mDivisor) - { } - T toSIntRound() const - { - T frac2 = 2 * mFractional; - if (frac2 >= mDivisor) - return mIntegral + 1; - if (frac2 <= -mDivisor) - return mIntegral - 1; - return mIntegral; - } - T toSIntRoundPositive() const - { - T frac2 = 2 * mFractional; - if (frac2 >= mDivisor) - return mIntegral + 1; - return mIntegral; - } - T toSIntFloor() const - { - return mFractional < 0 ? mIntegral - 1 : mIntegral; - } - T toSIntCeil() const - { - return mFractional > 0 ? mIntegral + 1 : mIntegral; - } + T mDivisor; + T mIntegral; + T mFractional; + + public: + DecomposedDecimal(T value, uint32_t scale) + : mDivisor(scaleDivisor(scale)), mIntegral(value / mDivisor), mFractional(value % mDivisor) + { + } + T toSIntRound() const + { + T frac2 = 2 * mFractional; + if (frac2 >= mDivisor) + return mIntegral + 1; + if (frac2 <= -mDivisor) + return mIntegral - 1; + return mIntegral; + } + T toSIntRoundPositive() const + { + T frac2 = 2 * mFractional; + if (frac2 >= mDivisor) + return mIntegral + 1; + return mIntegral; + } + T toSIntFloor() const + { + return mFractional < 0 ? mIntegral - 1 : mIntegral; + } + T toSIntCeil() const + { + return mFractional > 0 ? mIntegral + 1 : mIntegral; + } }; - -template -T applySignedScale(const T & val, int32_t scale) +template +T applySignedScale(const T& val, int32_t scale) { - return scale < 0 ? - val / datatypes::scaleDivisor((uint32_t) -scale) : - val * datatypes::scaleDivisor((uint32_t) scale); + return scale < 0 ? val / datatypes::scaleDivisor((uint32_t)-scale) + : val * datatypes::scaleDivisor((uint32_t)scale); } /** @brief The function to produce scale multiplier/divisor for wide decimals. */ -template +template inline void getScaleDivisor(T& divisor, const int8_t scale) { - if (scale < 0) - { - std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale); - throw std::invalid_argument(msg); - } - divisor = scaleDivisor((uint32_t) scale); + if (scale < 0) + { + std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale); + throw std::invalid_argument(msg); + } + divisor = scaleDivisor((uint32_t)scale); } struct lldiv_t_128 { - int128_t quot; - int128_t rem; - lldiv_t_128() : quot(0), rem(0) {} - lldiv_t_128(const int128_t& a_quot, const int128_t& a_rem) - : quot(a_quot), rem(a_rem) {} + int128_t quot; + int128_t rem; + lldiv_t_128() : quot(0), rem(0) + { + } + lldiv_t_128(const int128_t& a_quot, const int128_t& a_rem) : quot(a_quot), rem(a_rem) + { + } }; inline lldiv_t_128 lldiv128(const int128_t& dividend, const int128_t& divisor) { - if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0)) - return lldiv_t_128(); + if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0)) + return lldiv_t_128(); - return lldiv_t_128(dividend / divisor, dividend % divisor); + return lldiv_t_128(dividend / divisor, dividend % divisor); } - // TODO: derive it from TSInt64 eventually class TDecimal64 { -public: - int64_t value; -public: - static constexpr uint8_t MAXLENGTH8BYTES = 23; + public: + int64_t value; -public: - TDecimal64() - :value(0) - { } - explicit TDecimal64(int64_t val) - :value(val) - { } - explicit TDecimal64(const TSInt64 &val) - :value(static_cast(val)) - { } - // Divide to the scale divisor with rounding - int64_t toSInt64Round(uint32_t scale) const - { - return DecomposedDecimal(value, scale).toSIntRound(); - } - uint64_t toUInt64Round(uint32_t scale) const - { - return value < 0 ? - 0 : - static_cast(toSInt64Round(scale)); - } + public: + static constexpr uint8_t MAXLENGTH8BYTES = 23; - int64_t toSInt64Floor(uint32_t scale) const - { - return DecomposedDecimal(value, scale).toSIntFloor(); - } + public: + TDecimal64() : value(0) + { + } + explicit TDecimal64(int64_t val) : value(val) + { + } + explicit TDecimal64(const TSInt64& val) : value(static_cast(val)) + { + } + // Divide to the scale divisor with rounding + int64_t toSInt64Round(uint32_t scale) const + { + return DecomposedDecimal(value, scale).toSIntRound(); + } + uint64_t toUInt64Round(uint32_t scale) const + { + return value < 0 ? 0 : static_cast(toSInt64Round(scale)); + } - int64_t toSInt64Ceil(uint32_t scale) const - { - return DecomposedDecimal(value, scale).toSIntCeil(); - } + int64_t toSInt64Floor(uint32_t scale) const + { + return DecomposedDecimal(value, scale).toSIntFloor(); + } - // Convert to an arbitrary floating point data type, - // e.g. float, double, long double - template - T toXFloat(uint32_t scale) const - { - return (T) value / scaleDivisor(scale); - } + int64_t toSInt64Ceil(uint32_t scale) const + { + return DecomposedDecimal(value, scale).toSIntCeil(); + } + + // Convert to an arbitrary floating point data type, + // e.g. float, double, long double + template + T toXFloat(uint32_t scale) const + { + return (T)value / scaleDivisor(scale); + } }; - -class TDecimal128: public TSInt128 +class TDecimal128 : public TSInt128 { -public: - static constexpr uint8_t MAXLENGTH16BYTES = TSInt128::maxLength(); - static constexpr int128_t minInt128 = TFloat128::minInt128; - static constexpr int128_t maxInt128 = TFloat128::maxInt128; + public: + static constexpr uint8_t MAXLENGTH16BYTES = TSInt128::maxLength(); + static constexpr int128_t minInt128 = TFloat128::minInt128; + static constexpr int128_t maxInt128 = TFloat128::maxInt128; - static inline bool isWideDecimalNullValue(const int128_t& val) - { - return (val == TSInt128::NullValue); - } + static inline bool isWideDecimalNullValue(const int128_t& val) + { + return (val == TSInt128::NullValue); + } - static inline bool isWideDecimalEmptyValue(const int128_t& val) - { - return (val == TSInt128::EmptyValue); - } + static inline bool isWideDecimalEmptyValue(const int128_t& val) + { + return (val == TSInt128::EmptyValue); + } - static inline void setWideDecimalNullValue(int128_t& val) - { - val = TSInt128::NullValue; - } + static inline void setWideDecimalNullValue(int128_t& val) + { + val = TSInt128::NullValue; + } - static inline void setWideDecimalEmptyValue(int128_t& val) - { - val = TSInt128::EmptyValue; - } + static inline void setWideDecimalEmptyValue(int128_t& val) + { + val = TSInt128::EmptyValue; + } -public: - TDecimal128() - { } - explicit TDecimal128(const int128_t &val) - :TSInt128(val) - { } - explicit TDecimal128(const TSInt128& val) - :TSInt128(val) - { } - explicit TDecimal128(const int128_t* valPtr) - :TSInt128(valPtr) - { } - uint64_t toUInt64Round(uint32_t scale) const - { - if (s128Value <= 0) - return 0; - int128_t intg = DecomposedDecimal(s128Value, scale). - toSIntRoundPositive(); - return intg > numeric_limits::max() ? numeric_limits::max() : - static_cast(intg); - } + public: + TDecimal128() + { + } + explicit TDecimal128(const int128_t& val) : TSInt128(val) + { + } + explicit TDecimal128(const TSInt128& val) : TSInt128(val) + { + } + explicit TDecimal128(const int128_t* valPtr) : TSInt128(valPtr) + { + } + uint64_t toUInt64Round(uint32_t scale) const + { + if (s128Value <= 0) + return 0; + int128_t intg = DecomposedDecimal(s128Value, scale).toSIntRoundPositive(); + return intg > numeric_limits::max() ? numeric_limits::max() + : static_cast(intg); + } - int128_t toSInt128Floor(uint32_t scale) const - { - return DecomposedDecimal(s128Value, scale).toSIntFloor(); - } + int128_t toSInt128Floor(uint32_t scale) const + { + return DecomposedDecimal(s128Value, scale).toSIntFloor(); + } - int128_t toSInt128Ceil(uint32_t scale) const - { - return DecomposedDecimal(s128Value, scale).toSIntCeil(); - } + int128_t toSInt128Ceil(uint32_t scale) const + { + return DecomposedDecimal(s128Value, scale).toSIntCeil(); + } }; - // @brief The class for Decimal related operations // The methods and operators implemented in this class are // scale and precision aware. // We should eventually move the members "scale" and "precision" // into a separate class DecimalMeta and derive Decimal from it. -class Decimal: public TDecimal128, public TDecimal64 +class Decimal : public TDecimal128, public TDecimal64 { - public: - /** - @brief Compares two Decimal taking scale into account. - */ - static int compare(const Decimal& l, const Decimal& r); - /** - @brief Addition template that supports overflow check and - two internal representations of decimal. - */ - template - static void addition(const Decimal& l, - const Decimal& r, - Decimal& result); + public: + /** + @brief Compares two Decimal taking scale into account. + */ + static int compare(const Decimal& l, const Decimal& r); + /** + @brief Addition template that supports overflow check and + two internal representations of decimal. + */ + template + static void addition(const Decimal& l, const Decimal& r, Decimal& result); - /** - @brief Subtraction template that supports overflow check and - two internal representations of decimal. - */ - template - static void subtraction(const Decimal& l, - const Decimal& r, - Decimal& result); + /** + @brief Subtraction template that supports overflow check and + two internal representations of decimal. + */ + template + static void subtraction(const Decimal& l, const Decimal& r, Decimal& result); - /** - @brief Division template that supports overflow check and - two internal representations of decimal. - */ - template - static void division(const Decimal& l, - const Decimal& r, - Decimal& result); + /** + @brief Division template that supports overflow check and + two internal representations of decimal. + */ + template + static void division(const Decimal& l, const Decimal& r, Decimal& result); - /** - @brief Multiplication template that supports overflow check and - two internal representations of decimal. - */ - template - static void multiplication(const Decimal& l, - const Decimal& r, - Decimal& result); + /** + @brief Multiplication template that supports overflow check and + two internal representations of decimal. + */ + template + static void multiplication(const Decimal& l, const Decimal& r, Decimal& result); - /** - @brief The method detects whether decimal type is wide - using precision. - */ - static inline bool isWideDecimalTypeByPrecision(const int32_t precision) - { - return precision > INT64MAXPRECISION - && precision <= INT128MAXPRECISION; - } + /** + @brief The method detects whether decimal type is wide + using precision. + */ + static inline bool isWideDecimalTypeByPrecision(const int32_t precision) + { + return precision > INT64MAXPRECISION && precision <= INT128MAXPRECISION; + } - /** - @brief MDB increases scale by up to 4 digits calculating avg() - */ - static inline void setScalePrecision4Avg( - unsigned int& precision, - unsigned int& scale) - { - uint32_t scaleAvailable = INT128MAXPRECISION - scale; - uint32_t precisionAvailable = INT128MAXPRECISION - precision; - scale += (scaleAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : scaleAvailable; - precision += (precisionAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : precisionAvailable; - } + /** + @brief MDB increases scale by up to 4 digits calculating avg() + */ + static inline void setScalePrecision4Avg(unsigned int& precision, unsigned int& scale) + { + uint32_t scaleAvailable = INT128MAXPRECISION - scale; + uint32_t precisionAvailable = INT128MAXPRECISION - precision; + scale += (scaleAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : scaleAvailable; + precision += (precisionAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : precisionAvailable; + } - Decimal(): scale(0), precision(0) - { - } + Decimal() : scale(0), precision(0) + { + } - Decimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : - TDecimal128(val128), - TDecimal64(val), - scale(s), - precision(p) - { } + Decimal(int64_t val, int8_t s, uint8_t p, const int128_t& val128 = 0) + : TDecimal128(val128), TDecimal64(val), scale(s), precision(p) + { + } - Decimal(const TSInt64 &val, int8_t s, uint8_t p) : - TDecimal64(val), - scale(s), - precision(p) - { } + Decimal(const TSInt64& val, int8_t s, uint8_t p) : TDecimal64(val), scale(s), precision(p) + { + } - Decimal(int64_t unused, int8_t s, uint8_t p, const int128_t* val128Ptr) : - TDecimal128(val128Ptr), - TDecimal64(unused), - scale(s), - precision(p) - { } + Decimal(int64_t unused, int8_t s, uint8_t p, const int128_t* val128Ptr) + : TDecimal128(val128Ptr), TDecimal64(unused), scale(s), precision(p) + { + } - Decimal(const TSInt128& val128, int8_t s, uint8_t p) : - TDecimal128(val128), - scale(s), - precision(p) - { } + Decimal(const TSInt128& val128, int8_t s, uint8_t p) : TDecimal128(val128), scale(s), precision(p) + { + } + Decimal(const char* str, size_t length, DataCondition& error, int8_t s, uint8_t p); - Decimal(const char *str, size_t length, DataCondition & error, - int8_t s, uint8_t p); + int decimalComp(const Decimal& d) const + { + lldiv_t d1 = lldiv(value, static_cast(mcs_pow_10[scale])); + lldiv_t d2 = lldiv(d.value, static_cast(mcs_pow_10[d.scale])); - int decimalComp(const Decimal& d) const - { - lldiv_t d1 = lldiv(value, static_cast(mcs_pow_10[scale])); - lldiv_t d2 = lldiv(d.value, static_cast(mcs_pow_10[d.scale])); + int ret = 0; - int ret = 0; + if (d1.quot > d2.quot) + { + ret = 1; + } + else if (d1.quot < d2.quot) + { + ret = -1; + } + else + { + // rem carries the value's sign, but needs to be normalized. + int64_t s = scale - d.scale; - if (d1.quot > d2.quot) - { - ret = 1; - } - else if (d1.quot < d2.quot) - { - ret = -1; - } - else - { - // rem carries the value's sign, but needs to be normalized. - int64_t s = scale - d.scale; + if (s < 0) + { + if ((d1.rem * static_cast(mcs_pow_10[-s])) > d2.rem) + ret = 1; + else if ((d1.rem * static_cast(mcs_pow_10[-s])) < d2.rem) + ret = -1; + } + else + { + if (d1.rem > (d2.rem * static_cast(mcs_pow_10[s]))) + ret = 1; + else if (d1.rem < (d2.rem * static_cast(mcs_pow_10[s]))) + ret = -1; + } + } - if (s < 0) - { - if ((d1.rem * static_cast(mcs_pow_10[-s])) > d2.rem) - ret = 1; - else if ((d1.rem * static_cast(mcs_pow_10[-s])) < d2.rem) - ret = -1; - } - else - { - if (d1.rem > (d2.rem * static_cast(mcs_pow_10[s]))) - ret = 1; - else if (d1.rem < (d2.rem * static_cast(mcs_pow_10[s]))) - ret = -1; - } - } + return ret; + } - return ret; - } + inline TSInt128 toTSInt128() const + { + return TSInt128(s128Value); + } - inline TSInt128 toTSInt128() const - { - return TSInt128(s128Value); - } + inline TFloat128 toTFloat128() const + { + return TFloat128(s128Value); + } - inline TFloat128 toTFloat128() const - { - return TFloat128(s128Value); - } + inline double toDouble() const + { + int128_t scaleDivisor; + getScaleDivisor(scaleDivisor, scale); + datatypes::TFloat128 tmpval((float128_t)s128Value / scaleDivisor); + return static_cast(tmpval); + } - inline double toDouble() const - { - int128_t scaleDivisor; - getScaleDivisor(scaleDivisor, scale); - datatypes::TFloat128 tmpval((float128_t) s128Value / scaleDivisor); - return static_cast(tmpval); - } + inline operator double() const + { + return toDouble(); + } - inline operator double() const - { - return toDouble(); - } + inline float toFloat() const + { + int128_t scaleDivisor; + getScaleDivisor(scaleDivisor, scale); + datatypes::TFloat128 tmpval((float128_t)s128Value / scaleDivisor); + return static_cast(tmpval); + } - inline float toFloat() const - { - int128_t scaleDivisor; - getScaleDivisor(scaleDivisor, scale); - datatypes::TFloat128 tmpval((float128_t) s128Value / scaleDivisor); - return static_cast(tmpval); - } + inline operator float() const + { + return toFloat(); + } - inline operator float() const - { - return toFloat(); - } + inline long double toLongDouble() const + { + int128_t scaleDivisor; + getScaleDivisor(scaleDivisor, scale); + datatypes::TFloat128 tmpval((float128_t)s128Value / scaleDivisor); + return static_cast(tmpval); + } - inline long double toLongDouble() const - { - int128_t scaleDivisor; - getScaleDivisor(scaleDivisor, scale); - datatypes::TFloat128 tmpval((float128_t) s128Value / scaleDivisor); - return static_cast(tmpval); - } + inline operator long double() const + { + return toLongDouble(); + } - inline operator long double() const - { - return toLongDouble(); - } - - // This method returns integral part as a TSInt128 and - // fractional part as a TFloat128 - inline std::pair getIntegralAndDividedFractional() const - { - int128_t scaleDivisor; - getScaleDivisor(scaleDivisor, scale); - return std::make_pair(TSInt128(s128Value / scaleDivisor), - TFloat128((float128_t)(s128Value % scaleDivisor) / scaleDivisor)); - } + // This method returns integral part as a TSInt128 and + // fractional part as a TFloat128 + inline std::pair getIntegralAndDividedFractional() const + { + int128_t scaleDivisor; + getScaleDivisor(scaleDivisor, scale); + return std::make_pair(TSInt128(s128Value / scaleDivisor), + TFloat128((float128_t)(s128Value % scaleDivisor) / scaleDivisor)); + } - // This method returns integral part as a TSInt128 and - // fractional part as a TFloat128 - inline std::pair getIntegralAndFractional() const - { - int128_t scaleDivisor; - getScaleDivisor(scaleDivisor, scale); - return std::make_pair(TSInt128(s128Value / scaleDivisor), - TSInt128(s128Value % scaleDivisor)); - } + // This method returns integral part as a TSInt128 and + // fractional part as a TFloat128 + inline std::pair getIntegralAndFractional() const + { + int128_t scaleDivisor; + getScaleDivisor(scaleDivisor, scale); + return std::make_pair(TSInt128(s128Value / scaleDivisor), TSInt128(s128Value % scaleDivisor)); + } - inline std::tuple getIntegralFractionalAndDivisor() const - { - int128_t scaleDivisor; - getScaleDivisor(scaleDivisor, scale); - return std::make_tuple(TSInt128(s128Value / scaleDivisor), - TSInt128(s128Value % scaleDivisor), - TSInt128(scaleDivisor)); - } + inline std::tuple getIntegralFractionalAndDivisor() const + { + int128_t scaleDivisor; + getScaleDivisor(scaleDivisor, scale); + return std::make_tuple(TSInt128(s128Value / scaleDivisor), TSInt128(s128Value % scaleDivisor), + TSInt128(scaleDivisor)); + } - inline TSInt128 getIntegralPart() const - { - int128_t scaleDivisor = 0; - if(LIKELY(utils::is_nonnegative(scale))) - { - return TSInt128(getIntegralPartNonNegativeScale(scaleDivisor)); - } - return TSInt128(getIntegralPartNegativeScale(scaleDivisor)); - } + inline TSInt128 getIntegralPart() const + { + int128_t scaleDivisor = 0; + if (LIKELY(utils::is_nonnegative(scale))) + { + return TSInt128(getIntegralPartNonNegativeScale(scaleDivisor)); + } + return TSInt128(getIntegralPartNegativeScale(scaleDivisor)); + } - inline TSInt128 getPosNegRoundedIntegralPart(const uint8_t roundingFactor = 0) const - { - int128_t flooredScaleDivisor = 0; - int128_t roundedValue = getIntegralPartNonNegativeScale(flooredScaleDivisor); - int128_t ceiledScaleDivisor = (flooredScaleDivisor <= 10) ? 1 : (flooredScaleDivisor / 10); - int128_t leftO = (s128Value - roundedValue * flooredScaleDivisor) / ceiledScaleDivisor; - if (utils::is_nonnegative(roundedValue) && leftO > roundingFactor) - { - roundedValue++; - } - if (utils::is_negative(roundedValue) && leftO < -roundingFactor) - { - roundedValue--; - } + inline TSInt128 getPosNegRoundedIntegralPart(const uint8_t roundingFactor = 0) const + { + int128_t flooredScaleDivisor = 0; + int128_t roundedValue = getIntegralPartNonNegativeScale(flooredScaleDivisor); + int128_t ceiledScaleDivisor = (flooredScaleDivisor <= 10) ? 1 : (flooredScaleDivisor / 10); + int128_t leftO = (s128Value - roundedValue * flooredScaleDivisor) / ceiledScaleDivisor; + if (utils::is_nonnegative(roundedValue) && leftO > roundingFactor) + { + roundedValue++; + } + if (utils::is_negative(roundedValue) && leftO < -roundingFactor) + { + roundedValue--; + } - return TSInt128(roundedValue); - } + return TSInt128(roundedValue); + } - int64_t decimal64ToSInt64Round() const - { - return TDecimal64::toSInt64Round((uint32_t) scale); - } - uint64_t decimal64ToUInt64Round() const - { - return TDecimal64::toUInt64Round((uint32_t) scale); - } + int64_t decimal64ToSInt64Round() const + { + return TDecimal64::toSInt64Round((uint32_t)scale); + } + uint64_t decimal64ToUInt64Round() const + { + return TDecimal64::toUInt64Round((uint32_t)scale); + } - template T decimal64ToXFloat() const - { - return TDecimal64::toXFloat((uint32_t) scale); - } + template + T decimal64ToXFloat() const + { + return TDecimal64::toXFloat((uint32_t)scale); + } - int64_t toSInt64Round() const - { - return isWideDecimalTypeByPrecision(precision) ? - static_cast(getPosNegRoundedIntegralPart(4)) : - TDecimal64::toSInt64Round((uint32_t) scale); - } - uint64_t toUInt64Round() const - { - return isWideDecimalTypeByPrecision(precision) ? - TDecimal128::toUInt64Round((uint32_t) scale) : - TDecimal64::toUInt64Round((uint32_t) scale); - } + int64_t toSInt64Round() const + { + return isWideDecimalTypeByPrecision(precision) ? static_cast(getPosNegRoundedIntegralPart(4)) + : TDecimal64::toSInt64Round((uint32_t)scale); + } + uint64_t toUInt64Round() const + { + return isWideDecimalTypeByPrecision(precision) ? TDecimal128::toUInt64Round((uint32_t)scale) + : TDecimal64::toUInt64Round((uint32_t)scale); + } - // FLOOR related routines - int64_t toSInt64Floor() const - { - return isWideDecimalTypeByPrecision(precision) ? - static_cast(TSInt128(TDecimal128::toSInt128Floor((uint32_t) scale))) : - TDecimal64::toSInt64Floor((uint32_t) scale); - } + // FLOOR related routines + int64_t toSInt64Floor() const + { + return isWideDecimalTypeByPrecision(precision) + ? static_cast(TSInt128(TDecimal128::toSInt128Floor((uint32_t)scale))) + : TDecimal64::toSInt64Floor((uint32_t)scale); + } - uint64_t toUInt64Floor() const - { - return isWideDecimalTypeByPrecision(precision) ? - static_cast(TSInt128(TDecimal128::toSInt128Floor((uint32_t) scale))) : - static_cast(TSInt64(TDecimal64::toSInt64Floor((uint32_t) scale))); - } + uint64_t toUInt64Floor() const + { + return isWideDecimalTypeByPrecision(precision) + ? static_cast(TSInt128(TDecimal128::toSInt128Floor((uint32_t)scale))) + : static_cast(TSInt64(TDecimal64::toSInt64Floor((uint32_t)scale))); + } - Decimal floor() const - { - return isWideDecimalTypeByPrecision(precision)? - Decimal(TSInt128(TDecimal128::toSInt128Floor((uint32_t) scale)), 0, precision) : - Decimal(TSInt64(TDecimal64::toSInt64Floor((uint32_t) scale)), 0, precision); - } + Decimal floor() const + { + return isWideDecimalTypeByPrecision(precision) + ? Decimal(TSInt128(TDecimal128::toSInt128Floor((uint32_t)scale)), 0, precision) + : Decimal(TSInt64(TDecimal64::toSInt64Floor((uint32_t)scale)), 0, precision); + } - // CEIL related routines - int64_t toSInt64Ceil() const - { - return isWideDecimalTypeByPrecision(precision) ? - static_cast(TSInt128(TDecimal128::toSInt128Ceil((uint32_t) scale))) : - static_cast(TSInt64(TDecimal64::toSInt64Ceil((uint32_t) scale))); - } + // CEIL related routines + int64_t toSInt64Ceil() const + { + return isWideDecimalTypeByPrecision(precision) + ? static_cast(TSInt128(TDecimal128::toSInt128Ceil((uint32_t)scale))) + : static_cast(TSInt64(TDecimal64::toSInt64Ceil((uint32_t)scale))); + } - uint64_t toUInt64Ceil() const - { - return isWideDecimalTypeByPrecision(precision) ? - static_cast(TSInt128(TDecimal128::toSInt128Ceil((uint32_t) scale))) : - static_cast(TSInt64(TDecimal64::toSInt64Ceil((uint32_t) scale))); - } + uint64_t toUInt64Ceil() const + { + return isWideDecimalTypeByPrecision(precision) + ? static_cast(TSInt128(TDecimal128::toSInt128Ceil((uint32_t)scale))) + : static_cast(TSInt64(TDecimal64::toSInt64Ceil((uint32_t)scale))); + } - Decimal ceil() const - { - return isWideDecimalTypeByPrecision(precision) ? - Decimal(TSInt128(TDecimal128::toSInt128Ceil((uint32_t) scale)), 0, precision) : - Decimal(TSInt64(TDecimal64::toSInt64Ceil((uint32_t) scale)), 0, precision); - } + Decimal ceil() const + { + return isWideDecimalTypeByPrecision(precision) + ? Decimal(TSInt128(TDecimal128::toSInt128Ceil((uint32_t)scale)), 0, precision) + : Decimal(TSInt64(TDecimal64::toSInt64Ceil((uint32_t)scale)), 0, precision); + } - // MOD operator for an integer divisor to be used - // for integer rhs - inline TSInt128 operator%(const TSInt128& div) const - { - if (!isScaled()) - { - return TSInt128(s128Value % div.getValue()); - } - // Scale the value and calculate - // (LHS.value % RHS.value) * LHS.scaleMultiplier + LHS.scale_div_remainder - auto integralFractionalDivisor = getIntegralFractionalAndDivisor(); - return (std::get<0>(integralFractionalDivisor) % div.getValue()) * std::get<2>(integralFractionalDivisor) + std::get<1>(integralFractionalDivisor); - } + // MOD operator for an integer divisor to be used + // for integer rhs + inline TSInt128 operator%(const TSInt128& div) const + { + if (!isScaled()) + { + return TSInt128(s128Value % div.getValue()); + } + // Scale the value and calculate + // (LHS.value % RHS.value) * LHS.scaleMultiplier + LHS.scale_div_remainder + auto integralFractionalDivisor = getIntegralFractionalAndDivisor(); + return (std::get<0>(integralFractionalDivisor) % div.getValue()) * + std::get<2>(integralFractionalDivisor) + + std::get<1>(integralFractionalDivisor); + } - template - bool cmpOperatorTemplate(const Decimal& rhs) const - { - Op128 op128; - Op64 op64; - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return op128(s128Value, rhs.s128Value); - else - return op64(datatypes::Decimal::compare(*this, rhs), 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { + template + bool cmpOperatorTemplate(const Decimal& rhs) const + { + Op128 op128; + Op64 op64; + if (precision > datatypes::INT64MAXPRECISION && rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return op128(s128Value, rhs.s128Value); + else + return op64(datatypes::Decimal::compare(*this, rhs), 0); + } + else if (precision > datatypes::INT64MAXPRECISION && rhs.precision <= datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + { + return op128(s128Value, (int128_t)rhs.value); + } + else + { + // comp_op(compare(l,r),0) + return op64( + datatypes::Decimal::compare(*this, Decimal(TSInt128(rhs.value), rhs.scale, rhs.precision)), 0); + } + } + else if (precision <= datatypes::INT64MAXPRECISION && rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + { + return op128((int128_t)value, rhs.s128Value); + } + else + { + // comp_op(compare(l,r),0) + return op64(datatypes::Decimal::compare(Decimal(TSInt128(value), scale, precision), rhs), 0); + } + } + else + { + if (scale == rhs.scale) + return op64(value, rhs.value); + else + return op64(decimalComp(rhs), 0); + } + } - if (scale == rhs.scale) - { - return op128(s128Value, (int128_t) rhs.value); - } - else - { - // comp_op(compare(l,r),0) - return op64(datatypes::Decimal::compare( - *this, - Decimal(TSInt128(rhs.value), - rhs.scale, - rhs.precision)), - 0); - } - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - { - return op128((int128_t) value, rhs.s128Value); - } - else - { - // comp_op(compare(l,r),0) - return op64(datatypes::Decimal::compare( - Decimal( - TSInt128(value), - scale, - precision), - rhs), - 0); - } - } - else - { - if (scale == rhs.scale) - return op64(value, rhs.value); - else - return op64(decimalComp(rhs), 0); - } - } + bool operator==(const Decimal& rhs) const + { + return cmpOperatorTemplate, std::equal_to>(rhs); + } - bool operator==(const Decimal& rhs) const - { - return cmpOperatorTemplate, - std::equal_to>(rhs); - } + bool operator>(const Decimal& rhs) const + { + return cmpOperatorTemplate, std::greater>(rhs); + } - bool operator>(const Decimal& rhs) const - { - return cmpOperatorTemplate, - std::greater>(rhs); - } + bool operator>=(const Decimal& rhs) const + { + return cmpOperatorTemplate, std::greater_equal>(rhs); + } - bool operator>=(const Decimal& rhs) const - { - return cmpOperatorTemplate, - std::greater_equal>(rhs); - } + bool operator<(const Decimal& rhs) const + { + return !this->operator>=(rhs); + } - bool operator<(const Decimal& rhs) const - { - return !this->operator>=(rhs); - } + bool operator<=(const Decimal& rhs) const + { + return !this->operator>(rhs); + } - bool operator<=(const Decimal& rhs) const - { - return !this->operator>(rhs); - } + bool operator!=(const Decimal& rhs) const + { + return !this->operator==(rhs); + } - bool operator!=(const Decimal& rhs) const - { - return !this->operator==(rhs); - } + Decimal integralWideRound(const int128_t& scaleDivisor = 0) const + { + int128_t scaleDivisorInt = scaleDivisor; + if (UNLIKELY(!scaleDivisorInt)) + { + datatypes::getScaleDivisor(scaleDivisorInt, scale); + } + lldiv_t div = lldiv(s128Value, scaleDivisorInt); - Decimal integralWideRound(const int128_t& scaleDivisor = 0) const - { - int128_t scaleDivisorInt = scaleDivisor; - if(UNLIKELY(!scaleDivisorInt)) - { - datatypes::getScaleDivisor(scaleDivisorInt, scale); - } - lldiv_t div = lldiv(s128Value, scaleDivisorInt); + if (datatypes::abs(div.rem) * 2 >= scaleDivisorInt) + { + return Decimal(value, scale, precision, (div.quot < 0) ? div.quot-- : div.quot++); + } + return Decimal(value, scale, precision, div.quot); + } - if (datatypes::abs(div.rem) * 2 >= scaleDivisorInt) - { - return Decimal(value, - scale, - precision, - (div.quot < 0) ? div.quot-- : div.quot++); - } - return Decimal(value, - scale, - precision, - div.quot); - } + inline bool isTSInt128ByPrecision() const + { + return precision > INT64MAXPRECISION && precision <= INT128MAXPRECISION; + } - inline bool isTSInt128ByPrecision() const - { - return precision > INT64MAXPRECISION - && precision <= INT128MAXPRECISION; - } - - inline bool isScaled() const - { - return scale != 0; - } + inline bool isScaled() const + { + return scale != 0; + } - // hasTSInt128 explicitly tells to print int128 out in cases - // where precision can't detect decimal type properly, e.g. - // DECIMAL(10)/DECIMAL(38) - std::string toString(bool hasTSInt128 = false) const; - friend std::ostream& operator<<(std::ostream& os, const Decimal& dec); + // hasTSInt128 explicitly tells to print int128 out in cases + // where precision can't detect decimal type properly, e.g. + // DECIMAL(10)/DECIMAL(38) + std::string toString(bool hasTSInt128 = false) const; + friend std::ostream& operator<<(std::ostream& os, const Decimal& dec); - int8_t scale; // 0~38 - uint8_t precision; // 1~38 + int8_t scale; // 0~38 + uint8_t precision; // 1~38 - // STRICTLY for unit tests!!! - void setTSInt64Value(const int64_t x) { value = x; } - void setTSInt128Value(const int128_t& x) { s128Value = x; } - void setScale(const uint8_t x) { scale = x; } + // STRICTLY for unit tests!!! + void setTSInt64Value(const int64_t x) + { + value = x; + } + void setTSInt128Value(const int128_t& x) + { + s128Value = x; + } + void setScale(const uint8_t x) + { + scale = x; + } - private: - uint8_t writeIntPart(const int128_t& x, - char* buf, - const uint8_t buflen) const; - uint8_t writeFractionalPart(const int128_t& x, - char* buf, - const uint8_t buflen) const; - std::string toStringTSInt128WithScale() const; - std::string toStringTSInt64() const; + private: + uint8_t writeIntPart(const int128_t& x, char* buf, const uint8_t buflen) const; + uint8_t writeFractionalPart(const int128_t& x, char* buf, const uint8_t buflen) const; + std::string toStringTSInt128WithScale() const; + std::string toStringTSInt64() const; - inline int128_t getIntegralPartNonNegativeScale(int128_t& scaleDivisor) const - { - getScaleDivisor(scaleDivisor, scale); - return s128Value / scaleDivisor; - } + inline int128_t getIntegralPartNonNegativeScale(int128_t& scaleDivisor) const + { + getScaleDivisor(scaleDivisor, scale); + return s128Value / scaleDivisor; + } - inline int128_t getIntegralPartNegativeScale(int128_t& scaleDivisor) const - { - getScaleDivisor(scaleDivisor, -scale); - // Calls for overflow check - return s128Value * scaleDivisor; - } -}; //end of Decimal + inline int128_t getIntegralPartNegativeScale(int128_t& scaleDivisor) const + { + getScaleDivisor(scaleDivisor, -scale); + // Calls for overflow check + return s128Value * scaleDivisor; + } +}; // end of Decimal /** @brief The structure contains an overflow check for int128 division. */ -struct DivisionOverflowCheck { - void operator()(const int128_t& x, const int128_t& y) +struct DivisionOverflowCheck +{ + void operator()(const int128_t& x, const int128_t& y) + { + if (x == Decimal::minInt128 && y == -1) { - if (x == Decimal::minInt128 && y == -1) - { - throw logging::OperationOverflowExcept( - "Decimal::division produces an overflow."); - } + throw logging::OperationOverflowExcept("Decimal::division produces an overflow."); } - void operator()(const int64_t x, const int64_t y) + } + void operator()(const int64_t x, const int64_t y) + { + if (x == std::numeric_limits::min() && y == -1) { - if (x == std::numeric_limits::min() && y == -1) - { - throw logging::OperationOverflowExcept( - "Decimal::division produces an overflow."); - } + throw logging::OperationOverflowExcept("Decimal::division produces an overflow."); } + } }; // // @brief The structure contains an overflow check for int128 // and int64_t multiplication. // -struct MultiplicationOverflowCheck { - void operator()(const int128_t& x, const int128_t& y) +struct MultiplicationOverflowCheck +{ + void operator()(const int128_t& x, const int128_t& y) + { + int128_t tempR = 0; + this->operator()(x, y, tempR); + } + bool operator()(const int128_t& x, const int128_t& y, int128_t& r) + { + volatile int128_t z = x * y; + if (z / y != x) { - int128_t tempR = 0; - this->operator()(x, y, tempR); - } - bool operator()(const int128_t& x, const int128_t& y, int128_t& r) - { - volatile int128_t z = x * y; - if (z / y != x) - { - throw logging::OperationOverflowExcept( - "Decimal::multiplication or scale multiplication \ + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ produces an overflow."); - } - r = z; - return true; } - void operator()(const int64_t x, const int64_t y) + r = z; + return true; + } + void operator()(const int64_t x, const int64_t y) + { + if (x * y / y != x) { - if (x * y / y != x) - { - throw logging::OperationOverflowExcept( - "Decimal::multiplication or scale multiplication \ + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ produces an overflow."); - } } - bool operator()(const int64_t x, const int64_t y, int64_t& r) + } + bool operator()(const int64_t x, const int64_t y, int64_t& r) + { + if ((r = x * y) / y != x) { - if ((r = x * y) / y != x) - { - throw logging::OperationOverflowExcept( - "Decimal::multiplication or scale multiplication \ + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ produces an overflow."); - } - return true; } + return true; + } }; /** @brief The strucuture runs an empty overflow check for int128 multiplication operation. */ -struct MultiplicationNoOverflowCheck { - void operator()(const int128_t& x, const int128_t& y, int128_t& r) - { - r = x * y; - } +struct MultiplicationNoOverflowCheck +{ + void operator()(const int128_t& x, const int128_t& y, int128_t& r) + { + r = x * y; + } }; /** @brief The structure contains an overflow check for int128 and int64 addition. */ -struct AdditionOverflowCheck { - void operator()(const int128_t& x, const int128_t& y) +struct AdditionOverflowCheck +{ + void operator()(const int128_t& x, const int128_t& y) + { + if ((y > 0 && x > Decimal::maxInt128 - y) || (y < 0 && x < Decimal::minInt128 - y)) { - if ((y > 0 && x > Decimal::maxInt128 - y) - || (y < 0 && x < Decimal::minInt128 - y)) - { - throw logging::OperationOverflowExcept( - "Decimal::addition produces an overflow."); - } + throw logging::OperationOverflowExcept("Decimal::addition produces an overflow."); } - void operator()(const int64_t x, const int64_t y) + } + void operator()(const int64_t x, const int64_t y) + { + if ((y > 0 && x > std::numeric_limits::max() - y) || + (y < 0 && x < std::numeric_limits::min() - y)) { - if ((y > 0 && x > std::numeric_limits::max() - y) - || (y < 0 && x < std::numeric_limits::min() - y)) - { - throw logging::OperationOverflowExcept( - "Decimal::addition produces an overflow."); - } + throw logging::OperationOverflowExcept("Decimal::addition produces an overflow."); } + } }; /** @brief The structure contains an overflow check for int128 subtraction. */ -struct SubtractionOverflowCheck { - void operator()(const int128_t& x, const int128_t& y) +struct SubtractionOverflowCheck +{ + void operator()(const int128_t& x, const int128_t& y) + { + if ((y > 0 && x < Decimal::minInt128 + y) || (y < 0 && x > Decimal::maxInt128 + y)) { - if ((y > 0 && x < Decimal::minInt128 + y) - || (y < 0 && x > Decimal::maxInt128 + y)) - { - throw logging::OperationOverflowExcept( - "Decimal::subtraction produces an overflow."); - } + throw logging::OperationOverflowExcept("Decimal::subtraction produces an overflow."); } - void operator()(const int64_t x, const int64_t y) + } + void operator()(const int64_t x, const int64_t y) + { + if ((y > 0 && x < std::numeric_limits::min() + y) || + (y < 0 && x > std::numeric_limits::max() + y)) { - if ((y > 0 && x < std::numeric_limits::min() + y) - || (y < 0 && x > std::numeric_limits::max() + y)) - { - throw logging::OperationOverflowExcept( - "Decimal::subtraction produces an overflow."); - } + throw logging::OperationOverflowExcept("Decimal::subtraction produces an overflow."); } + } }; /** @brief The strucuture runs an empty overflow check for int128 operation. */ -struct NoOverflowCheck { - void operator()(const int128_t& x, const int128_t& y) - { - return; - } +struct NoOverflowCheck +{ + void operator()(const int128_t& x, const int128_t& y) + { + return; + } }; -} //end of namespace +} // namespace datatypes #endif diff --git a/datatypes/mcs_double.h b/datatypes/mcs_double.h index 04bfbf3c9..e34bf7aaa 100644 --- a/datatypes/mcs_double.h +++ b/datatypes/mcs_double.h @@ -23,17 +23,21 @@ namespace datatypes { - class TDouble { -protected: + protected: double mValue; -public: - TDouble(): mValue(0) { } - explicit TDouble(double value): mValue(value) { } + public: + TDouble() : mValue(0) + { + } - explicit operator double () const + explicit TDouble(double value) : mValue(value) + { + } + + explicit operator double() const { return mValue; } @@ -49,8 +53,7 @@ public: } }; +} // end of namespace datatypes -} //end of namespace datatypes - -#endif // MCS_DOUBLE_H_INCLUDED +#endif // MCS_DOUBLE_H_INCLUDED // vim:ts=2 sw=2: diff --git a/datatypes/mcs_float128.h b/datatypes/mcs_float128.h index 068d9205c..058ad1ed7 100644 --- a/datatypes/mcs_float128.h +++ b/datatypes/mcs_float128.h @@ -34,7 +34,6 @@ using float128_t = __float128; namespace datatypes { - /* Main union type we use to manipulate the floating-point type. */ typedef union { @@ -42,12 +41,12 @@ typedef union struct { - unsigned mantissa3:32; - unsigned mantissa2:32; - unsigned mantissa1:32; - unsigned mantissa0:16; - unsigned exponent:15; - unsigned negative:1; + unsigned mantissa3 : 32; + unsigned mantissa2 : 32; + unsigned mantissa1 : 32; + unsigned mantissa0 : 16; + unsigned exponent : 15; + unsigned negative : 1; } ieee; struct @@ -66,571 +65,665 @@ typedef union struct { - unsigned mantissa3:32; - unsigned mantissa2:32; - unsigned mantissa1:32; - unsigned mantissa0:15; - unsigned quiet_nan:1; - unsigned exponent:15; - unsigned negative:1; + unsigned mantissa3 : 32; + unsigned mantissa2 : 32; + unsigned mantissa1 : 32; + unsigned mantissa0 : 15; + unsigned quiet_nan : 1; + unsigned exponent : 15; + unsigned negative : 1; } ieee_nan; } mcs_ieee854_float128; /* Get two 64 bit ints from a long double. */ -#define MCS_GET_FLT128_WORDS64(ix0,ix1,d) \ -do { \ - mcs_ieee854_float128 u; \ - u.value = (d); \ - (ix0) = u.words64.high; \ - (ix1) = u.words64.low; \ -} while (0) +#define MCS_GET_FLT128_WORDS64(ix0, ix1, d) \ + do \ + { \ + mcs_ieee854_float128 u; \ + u.value = (d); \ + (ix0) = u.words64.high; \ + (ix1) = u.words64.low; \ + } while (0) /* Set a long double from two 64 bit ints. */ -#define MCS_SET_FLT128_WORDS64(d,ix0,ix1) \ -do { \ - mcs_ieee854_float128 u; \ - u.words64.high = (ix0); \ - u.words64.low = (ix1); \ - (d) = u.value; \ -} while (0) +#define MCS_SET_FLT128_WORDS64(d, ix0, ix1) \ + do \ + { \ + mcs_ieee854_float128 u; \ + u.words64.high = (ix0); \ + u.words64.low = (ix1); \ + (d) = u.value; \ + } while (0) class TSInt128; class TFloat128; using int128_t = __int128; -static const float128_t mcs_fl_one = 1.0, mcs_fl_Zero[] = {0.0, -0.0,}; +static const float128_t mcs_fl_one = 1.0, mcs_fl_Zero[] = { + 0.0, + -0.0, +}; // Copy from boost::multiprecision::float128 -template<> class numeric_limits { - public: - static constexpr bool is_specialized = true; - static float128_t max() - { - return mcs_ieee854_float128{ .ieee = {0xffffffff, - 0xffffffff, - 0xffffffff, - 0xffff, - 0x7ffe, - 0x0}}.value; - } - static float128_t min() - { - return mcs_ieee854_float128{ .ieee = {0x0, - 0x0, - 0x0, - 0x0, - 0x1, - 0x0}}.value; - } - static float128_t denorm_min() - { - return mcs_ieee854_float128{ .ieee = {0x1, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0}}.value; - } - static float128_t lowest() { return -max(); } - static constexpr int digits = 113; - static constexpr int digits10 = 33; - static constexpr int max_digits10 = 36; - static constexpr bool is_signed = true; - static constexpr bool is_integer = false; - static constexpr bool is_exact = false; - static constexpr int radix = 2; - static float128_t round_error() { return 0.5; } - static constexpr int min_exponent = -16381; - static constexpr int min_exponent10 = min_exponent * 301L / 1000L; - static constexpr int max_exponent = 16384; - static constexpr int max_exponent10 = max_exponent * 301L / 1000L; - static constexpr bool has_infinity = true; - static constexpr bool has_quiet_NaN = true; - static float128_t quiet_NaN() { return 1.0 / 0.0; } - static constexpr bool has_signaling_NaN = false; - static constexpr bool has_denorm_loss = true; - static float128_t infinity() { return 1.0 / 0.0; } - static float128_t signaling_NaN() { return 0; } - static constexpr bool is_iec559 = true; - static constexpr bool is_bounded = false; - static constexpr bool is_modulo = false; - static constexpr bool traps = false; - static constexpr bool tinyness_before = false; +template <> +class numeric_limits +{ + public: + static constexpr bool is_specialized = true; + static float128_t max() + { + return mcs_ieee854_float128{.ieee = {0xffffffff, 0xffffffff, 0xffffffff, 0xffff, 0x7ffe, 0x0}}.value; + } + static float128_t min() + { + return mcs_ieee854_float128{.ieee = {0x0, 0x0, 0x0, 0x0, 0x1, 0x0}}.value; + } + static float128_t denorm_min() + { + return mcs_ieee854_float128{.ieee = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0}}.value; + } + static float128_t lowest() + { + return -max(); + } + static constexpr int digits = 113; + static constexpr int digits10 = 33; + static constexpr int max_digits10 = 36; + static constexpr bool is_signed = true; + static constexpr bool is_integer = false; + static constexpr bool is_exact = false; + static constexpr int radix = 2; + static float128_t round_error() + { + return 0.5; + } + static constexpr int min_exponent = -16381; + static constexpr int min_exponent10 = min_exponent * 301L / 1000L; + static constexpr int max_exponent = 16384; + static constexpr int max_exponent10 = max_exponent * 301L / 1000L; + static constexpr bool has_infinity = true; + static constexpr bool has_quiet_NaN = true; + static float128_t quiet_NaN() + { + return 1.0 / 0.0; + } + static constexpr bool has_signaling_NaN = false; + static constexpr bool has_denorm_loss = true; + static float128_t infinity() + { + return 1.0 / 0.0; + } + static float128_t signaling_NaN() + { + return 0; + } + static constexpr bool is_iec559 = true; + static constexpr bool is_bounded = false; + static constexpr bool is_modulo = false; + static constexpr bool traps = false; + static constexpr bool tinyness_before = false; }; // Type defined integral types // for templates template -struct get_integral_type { +struct get_integral_type +{ typedef T type; }; -template<> -struct get_integral_type{ +template <> +struct get_integral_type +{ typedef float128_t type; }; -template<> -struct get_integral_type{ +template <> +struct get_integral_type +{ typedef int128_t type; }; class TFloat128 { - public: - static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; - static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + public: + static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; + static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; - static constexpr uint16_t MAXLENGTH16BYTES = 42; - // A variety of ctors for aligned and unaligned arguments - TFloat128(): value(0) { } + static constexpr uint16_t MAXLENGTH16BYTES = 42; + // A variety of ctors for aligned and unaligned arguments + TFloat128() : value(0) + { + } - // aligned argument - TFloat128(const float128_t& x) { value = x; } - TFloat128(const int128_t& x) { value = static_cast(x); } + // aligned argument + TFloat128(const float128_t& x) + { + value = x; + } + TFloat128(const int128_t& x) + { + value = static_cast(x); + } - // fmodq(x,y) taken from libquadmath - // Return x mod y in exact arithmetic - // Method: shift and subtract - static float128_t fmodq (float128_t& x, float128_t& y) + // fmodq(x,y) taken from libquadmath + // Return x mod y in exact arithmetic + // Method: shift and subtract + static float128_t fmodq(float128_t& x, float128_t& y) + { + int64_t n, hx, hy, hz, ix, iy, sx, i; + uint64_t lx, ly, lz; + + MCS_GET_FLT128_WORDS64(hx, lx, x); + MCS_GET_FLT128_WORDS64(hy, ly, y); + sx = hx & 0x8000000000000000ULL; /* sign of x */ + hx ^= sx; /* |x| */ + hy &= 0x7fffffffffffffffLL; /* |y| */ + + /* purge off exception values */ + if ((hy | ly) == 0 || (hx >= 0x7fff000000000000LL) || /* y=0,or x not finite */ + ((hy | ((ly | -ly) >> 63)) > 0x7fff000000000000LL)) /* or y is NaN */ + return (x * y) / (x * y); + if (hx <= hy) { - int64_t n,hx,hy,hz,ix,iy,sx,i; - uint64_t lx,ly,lz; - - MCS_GET_FLT128_WORDS64(hx,lx,x); - MCS_GET_FLT128_WORDS64(hy,ly,y); - sx = hx&0x8000000000000000ULL; /* sign of x */ - hx ^=sx; /* |x| */ - hy &= 0x7fffffffffffffffLL; /* |y| */ - - /* purge off exception values */ - if((hy|ly)==0||(hx>=0x7fff000000000000LL)|| /* y=0,or x not finite */ - ((hy|((ly|-ly)>>63))>0x7fff000000000000LL)) /* or y is NaN */ - return (x*y)/(x*y); - if(hx<=hy) { - if((hx>63]; /* |x|=|y| return x*0*/ - } - - /* determine ix = ilogb(x) */ - if(hx<0x0001000000000000LL) { /* subnormal x */ - if(hx==0) { - for (ix = -16431, i=lx; i>0; i<<=1) ix -=1; - } else { - for (ix = -16382, i=hx<<15; i>0; i<<=1) ix -=1; - } - } else ix = (hx>>48)-0x3fff; - - /* determine iy = ilogb(y) */ - if(hy<0x0001000000000000LL) { /* subnormal y */ - if(hy==0) { - for (iy = -16431, i=ly; i>0; i<<=1) iy -=1; - } else { - for (iy = -16382, i=hy<<15; i>0; i<<=1) iy -=1; - } - } else iy = (hy>>48)-0x3fff; - - /* set up {hx,lx}, {hy,ly} and align y to x */ - if(ix >= -16382) - hx = 0x0001000000000000LL|(0x0000ffffffffffffLL&hx); - else { /* subnormal x, shift x to normal */ - n = -16382-ix; - if(n<=63) { - hx = (hx<>(64-n)); - lx <<= n; - } else { - hx = lx<<(n-64); - lx = 0; - } - } - if(iy >= -16382) - hy = 0x0001000000000000LL|(0x0000ffffffffffffLL&hy); - else { /* subnormal y, shift y to normal */ - n = -16382-iy; - if(n<=63) { - hy = (hy<>(64-n)); - ly <<= n; - } else { - hy = ly<<(n-64); - ly = 0; - } - } - - /* fix point fmod */ - n = ix - iy; - while(n--) { - hz=hx-hy;lz=lx-ly; if(lx>63); lx = lx+lx;} - else { - if((hz|lz)==0) /* return sign(x)*0 */ - return mcs_fl_Zero[(uint64_t)sx>>63]; - hx = hz+hz+(lz>>63); lx = lz+lz; - } - } - hz=hx-hy;lz=lx-ly; if(lx=0) {hx=hz;lx=lz;} - - /* convert back to floating value and restore the sign */ - if((hx|lx)==0) /* return sign(x)*0 */ - return mcs_fl_Zero[(uint64_t)sx>>63]; - while(hx<0x0001000000000000LL) { /* normalize x */ - hx = hx+hx+(lx>>63); lx = lx+lx; - iy -= 1; - } - if(iy>= -16382) { /* normalize output */ - hx = ((hx-0x0001000000000000LL)|((iy+16383)<<48)); - MCS_SET_FLT128_WORDS64(x,hx|sx,lx); - } else { /* subnormal output */ - n = -16382 - iy; - if(n<=48) { - lx = (lx>>n)|((uint64_t)hx<<(64-n)); - hx >>= n; - } else if (n<=63) { - lx = (hx<<(64-n))|(lx>>n); hx = sx; - } else { - lx = hx>>(n-64); hx = sx; - } - MCS_SET_FLT128_WORDS64(x,hx|sx,lx); - x *= mcs_fl_one; /* create necessary signal */ - } - return x; /* exact output */ + if ((hx < hy) || (lx < ly)) + return x; /* |x|<|y| return x */ + if (lx == ly) + return mcs_fl_Zero[(uint64_t)sx >> 63]; /* |x|=|y| return x*0*/ } - // The f() returns float128_t power p - // taken from boost::multiprecision - static inline float128_t pown(const float128_t& x, const int p) - { - const bool isneg = (x < 0); - const bool isnan = (x != x); - const bool isinf = ((!isneg) ? bool(+x > (datatypes::numeric_limits::max)()) - : bool(-x > (datatypes::numeric_limits::max)())); - - if(isnan) { return x; } - - if(isinf) { return datatypes::numeric_limits::quiet_NaN(); } - - const bool x_is_neg = (x < 0); - const float128_t abs_x = (x_is_neg ? -x : x); - - if(p < static_cast(0)) + /* determine ix = ilogb(x) */ + if (hx < 0x0001000000000000LL) + { /* subnormal x */ + if (hx == 0) { - if(abs_x < (datatypes::numeric_limits::min)()) - { - return (x_is_neg ? -datatypes::numeric_limits::infinity() - : +datatypes::numeric_limits::infinity()); - } - else - { - return float128_t(1) / pown(x, static_cast(-p)); - } - } - - if(p == static_cast(0)) - { - return float128_t(1); + for (ix = -16431, i = lx; i > 0; i <<= 1) + ix -= 1; } else { - if(p == static_cast(1)) { return x; } + for (ix = -16382, i = hx << 15; i > 0; i <<= 1) + ix -= 1; + } + } + else + ix = (hx >> 48) - 0x3fff; - if(abs_x > (datatypes::numeric_limits::max)()) - { - return (x_is_neg ? -datatypes::numeric_limits::infinity() - : +datatypes::numeric_limits::infinity()); - } + /* determine iy = ilogb(y) */ + if (hy < 0x0001000000000000LL) + { /* subnormal y */ + if (hy == 0) + { + for (iy = -16431, i = ly; i > 0; i <<= 1) + iy -= 1; + } + else + { + for (iy = -16382, i = hy << 15; i > 0; i <<= 1) + iy -= 1; + } + } + else + iy = (hy >> 48) - 0x3fff; - if (p == static_cast(2)) { return (x * x); } - else if(p == static_cast(3)) { return ((x * x) * x); } - else if(p == static_cast(4)) { const float128_t x2 = (x * x); return (x2 * x2); } - else - { - // The variable xn stores the binary powers of x. - float128_t result(((p % int(2)) != int(0)) ? x : float128_t(1)); - float128_t xn (x); - - int p2 = p; - - while(int(p2 /= 2) != int(0)) - { - // Square xn for each binary power. - xn *= xn; - - const bool has_binary_power = (int(p2 % int(2)) != int(0)); - - if(has_binary_power) - { - // Multiply the result with each binary power contained in the exponent. - result *= xn; - } - } - - return result; - } + /* set up {hx,lx}, {hy,ly} and align y to x */ + if (ix >= -16382) + hx = 0x0001000000000000LL | (0x0000ffffffffffffLL & hx); + else + { /* subnormal x, shift x to normal */ + n = -16382 - ix; + if (n <= 63) + { + hx = (hx << n) | (lx >> (64 - n)); + lx <<= n; + } + else + { + hx = lx << (n - 64); + lx = 0; + } + } + if (iy >= -16382) + hy = 0x0001000000000000LL | (0x0000ffffffffffffLL & hy); + else + { /* subnormal y, shift y to normal */ + n = -16382 - iy; + if (n <= 63) + { + hy = (hy << n) | (ly >> (64 - n)); + ly <<= n; + } + else + { + hy = ly << (n - 64); + ly = 0; } } - // fromString conversion for float128_t - // algo is taken from - // boost/math/cstdfloat/cstdfloat_iostream.hpp:convert_from_string() - static float128_t fromString(const std::string& str) + /* fix point fmod */ + n = ix - iy; + while (n--) { - float128_t value = 0; - const char* p = str.c_str(); - - if((p == static_cast(0U)) || (*p == static_cast(0))) + hz = hx - hy; + lz = lx - ly; + if (lx < ly) + hz -= 1; + if (hz < 0) { - return value; + hx = hx + hx + (lx >> 63); + lx = lx + lx; + } + else + { + if ((hz | lz) == 0) /* return sign(x)*0 */ + return mcs_fl_Zero[(uint64_t)sx >> 63]; + hx = hz + hz + (lz >> 63); + lx = lz + lz; + } + } + hz = hx - hy; + lz = lx - ly; + if (lx < ly) + hz -= 1; + if (hz >= 0) + { + hx = hz; + lx = lz; + } + + /* convert back to floating value and restore the sign */ + if ((hx | lx) == 0) /* return sign(x)*0 */ + return mcs_fl_Zero[(uint64_t)sx >> 63]; + while (hx < 0x0001000000000000LL) + { /* normalize x */ + hx = hx + hx + (lx >> 63); + lx = lx + lx; + iy -= 1; + } + if (iy >= -16382) + { /* normalize output */ + hx = ((hx - 0x0001000000000000LL) | ((iy + 16383) << 48)); + MCS_SET_FLT128_WORDS64(x, hx | sx, lx); + } + else + { /* subnormal output */ + n = -16382 - iy; + if (n <= 48) + { + lx = (lx >> n) | ((uint64_t)hx << (64 - n)); + hx >>= n; + } + else if (n <= 63) + { + lx = (hx << (64 - n)) | (lx >> n); + hx = sx; + } + else + { + lx = hx >> (n - 64); + hx = sx; + } + MCS_SET_FLT128_WORDS64(x, hx | sx, lx); + x *= mcs_fl_one; /* create necessary signal */ + } + return x; /* exact output */ + } + + // The f() returns float128_t power p + // taken from boost::multiprecision + static inline float128_t pown(const float128_t& x, const int p) + { + const bool isneg = (x < 0); + const bool isnan = (x != x); + const bool isinf = ((!isneg) ? bool(+x > (datatypes::numeric_limits::max)()) + : bool(-x > (datatypes::numeric_limits::max)())); + + if (isnan) + { + return x; + } + + if (isinf) + { + return datatypes::numeric_limits::quiet_NaN(); + } + + const bool x_is_neg = (x < 0); + const float128_t abs_x = (x_is_neg ? -x : x); + + if (p < static_cast(0)) + { + if (abs_x < (datatypes::numeric_limits::min)()) + { + return (x_is_neg ? -datatypes::numeric_limits::infinity() + : +datatypes::numeric_limits::infinity()); + } + else + { + return float128_t(1) / pown(x, static_cast(-p)); + } + } + + if (p == static_cast(0)) + { + return float128_t(1); + } + else + { + if (p == static_cast(1)) + { + return x; } - bool is_neg = false; - bool is_neg_expon = false; - - constexpr int ten = 10; - - int expon = 0; - int digits_seen = 0; - - constexpr int max_digits10 = datatypes::numeric_limits::max_digits10 + 1; - - if(*p == static_cast('+')) + if (abs_x > (datatypes::numeric_limits::max)()) { - ++p; - } - else if(*p == static_cast('-')) - { - is_neg = true; - ++p; + return (x_is_neg ? -datatypes::numeric_limits::infinity() + : +datatypes::numeric_limits::infinity()); } - const bool isnan = ((std::strcmp(p, "nan") == 0) || (std::strcmp(p, "NaN") == 0) || (std::strcmp(p, "NAN") == 0)); - - if(isnan) + if (p == static_cast(2)) { - value = datatypes::numeric_limits::infinity(); - if (is_neg) + return (x * x); + } + else if (p == static_cast(3)) + { + return ((x * x) * x); + } + else if (p == static_cast(4)) + { + const float128_t x2 = (x * x); + return (x2 * x2); + } + else + { + // The variable xn stores the binary powers of x. + float128_t result(((p % int(2)) != int(0)) ? x : float128_t(1)); + float128_t xn(x); + + int p2 = p; + + while (int(p2 /= 2) != int(0)) { - value = -value; - } - return value; - } + // Square xn for each binary power. + xn *= xn; - const bool isinf = ((std::strcmp(p, "inf") == 0) || (std::strcmp(p, "Inf") == 0) || (std::strcmp(p, "INF") == 0)); + const bool has_binary_power = (int(p2 % int(2)) != int(0)); - if(isinf) - { - value = datatypes::numeric_limits::infinity(); - if (is_neg) - { - value = -value; - } - return value; - } - - // Grab all the leading digits before the decimal point. - while(std::isdigit(*p)) - { - value *= ten; - value += static_cast(*p - '0'); - ++p; - ++digits_seen; - } - - if(*p == static_cast('.')) - { - // Grab everything after the point, stop when we've seen - // enough digits, even if there are actually more available. - - ++p; - - while(std::isdigit(*p)) - { - value *= ten; - value += static_cast(*p - '0'); - ++p; - --expon; - - if(++digits_seen > max_digits10) + if (has_binary_power) { - break; + // Multiply the result with each binary power contained in the exponent. + result *= xn; } } - while(std::isdigit(*p)) - { - ++p; - } + return result; } + } + } - // Parse the exponent. - if((*p == static_cast('e')) || (*p == static_cast('E'))) - { - ++p; + // fromString conversion for float128_t + // algo is taken from + // boost/math/cstdfloat/cstdfloat_iostream.hpp:convert_from_string() + static float128_t fromString(const std::string& str) + { + float128_t value = 0; + const char* p = str.c_str(); - if(*p == static_cast('+')) - { - ++p; - } - else if(*p == static_cast('-')) - { - is_neg_expon = true; - ++p; - } + if ((p == static_cast(0U)) || (*p == static_cast(0))) + { + return value; + } - int e2 = 0; + bool is_neg = false; + bool is_neg_expon = false; - while(std::isdigit(*p)) - { - e2 *= 10; - e2 += (*p - '0'); - ++p; - } + constexpr int ten = 10; - if(is_neg_expon) - { - e2 = -e2; - } + int expon = 0; + int digits_seen = 0; - expon += e2; - } + constexpr int max_digits10 = datatypes::numeric_limits::max_digits10 + 1; - if(expon) - { - // Scale by 10^expon. Note that 10^expon can be outside the range - // of our number type, even though the result is within range. - // If that looks likely, then split the calculation in two parts. - float128_t t; - t = ten; + if (*p == static_cast('+')) + { + ++p; + } + else if (*p == static_cast('-')) + { + is_neg = true; + ++p; + } - if(expon > (datatypes::numeric_limits::min_exponent10 + 2)) - { - t = TFloat128::pown(t, expon); - value *= t; - } - else - { - t = TFloat128::pown(t, (expon + digits_seen + 1)); - value *= t; - t = ten; - t = TFloat128::pown(t, (-digits_seen - 1)); - value *= t; - } - } + const bool isnan = + ((std::strcmp(p, "nan") == 0) || (std::strcmp(p, "NaN") == 0) || (std::strcmp(p, "NAN") == 0)); - if(is_neg) + if (isnan) + { + value = datatypes::numeric_limits::infinity(); + if (is_neg) { value = -value; } return value; } - // Method returns max length of a string representation - static constexpr uint8_t maxLength() + const bool isinf = + ((std::strcmp(p, "inf") == 0) || (std::strcmp(p, "Inf") == 0) || (std::strcmp(p, "INF") == 0)); + + if (isinf) { - return TFloat128::MAXLENGTH16BYTES; + value = datatypes::numeric_limits::infinity(); + if (is_neg) + { + value = -value; + } + return value; } - inline int128_t toTSInt128() const + // Grab all the leading digits before the decimal point. + while (std::isdigit(*p)) { - if (value > static_cast(maxInt128)) - return maxInt128; - else if (value < static_cast(minInt128)) - return minInt128; - - return static_cast(value); + value *= ten; + value += static_cast(*p - '0'); + ++p; + ++digits_seen; } - inline operator int128_t() const + if (*p == static_cast('.')) { - return toTSInt128(); + // Grab everything after the point, stop when we've seen + // enough digits, even if there are actually more available. + + ++p; + + while (std::isdigit(*p)) + { + value *= ten; + value += static_cast(*p - '0'); + ++p; + --expon; + + if (++digits_seen > max_digits10) + { + break; + } + } + + while (std::isdigit(*p)) + { + ++p; + } } - inline operator double() const + // Parse the exponent. + if ((*p == static_cast('e')) || (*p == static_cast('E'))) { - return toDouble(); + ++p; + + if (*p == static_cast('+')) + { + ++p; + } + else if (*p == static_cast('-')) + { + is_neg_expon = true; + ++p; + } + + int e2 = 0; + + while (std::isdigit(*p)) + { + e2 *= 10; + e2 += (*p - '0'); + ++p; + } + + if (is_neg_expon) + { + e2 = -e2; + } + + expon += e2; } - inline double toDouble() const + if (expon) { - if (value > static_cast(DBL_MAX)) - return DBL_MAX; - else if (value < -static_cast(DBL_MAX)) - return -DBL_MAX; + // Scale by 10^expon. Note that 10^expon can be outside the range + // of our number type, even though the result is within range. + // If that looks likely, then split the calculation in two parts. + float128_t t; + t = ten; - return static_cast(value); + if (expon > (datatypes::numeric_limits::min_exponent10 + 2)) + { + t = TFloat128::pown(t, expon); + value *= t; + } + else + { + t = TFloat128::pown(t, (expon + digits_seen + 1)); + value *= t; + t = ten; + t = TFloat128::pown(t, (-digits_seen - 1)); + value *= t; + } } - inline operator long double() const + if (is_neg) { - return toLongDouble(); + value = -value; } + return value; + } - inline operator float() const - { - return toFloat(); - } + // Method returns max length of a string representation + static constexpr uint8_t maxLength() + { + return TFloat128::MAXLENGTH16BYTES; + } - inline float toFloat() const - { - if (value > static_cast(FLT_MAX)) - return FLT_MAX; - else if (value < -static_cast(FLT_MAX)) - return -FLT_MAX; + inline int128_t toTSInt128() const + { + if (value > static_cast(maxInt128)) + return maxInt128; + else if (value < static_cast(minInt128)) + return minInt128; - return static_cast(value); - } + return static_cast(value); + } - inline int64_t toTSInt64() const - { - if (value > static_cast(INT64_MAX)) - return INT64_MAX; - else if (value < static_cast(INT64_MIN)) - return INT64_MIN; + inline operator int128_t() const + { + return toTSInt128(); + } - return static_cast(value); - } + inline operator double() const + { + return toDouble(); + } - inline operator int64_t() const - { - return toTSInt64(); - } + inline double toDouble() const + { + if (value > static_cast(DBL_MAX)) + return DBL_MAX; + else if (value < -static_cast(DBL_MAX)) + return -DBL_MAX; - inline uint64_t toTUInt64() const - { - if (value > static_cast(UINT64_MAX)) - return UINT64_MAX; - else if (value < 0) - return 0; + return static_cast(value); + } - return static_cast(value); - } + inline operator long double() const + { + return toLongDouble(); + } - inline operator uint64_t() const - { - return toTUInt64(); - } + inline operator float() const + { + return toFloat(); + } - inline TFloat128 operator+(const TFloat128& rhs) const - { - return TFloat128(value + rhs.value); - } + inline float toFloat() const + { + if (value > static_cast(FLT_MAX)) + return FLT_MAX; + else if (value < -static_cast(FLT_MAX)) + return -FLT_MAX; - inline long double toLongDouble() const - { - if (value > static_cast(LDBL_MAX)) - return LDBL_MAX; - else if (value < -static_cast(LDBL_MAX)) - return -LDBL_MAX; + return static_cast(value); + } - return static_cast(value); - } - private: - float128_t value; + inline int64_t toTSInt64() const + { + if (value > static_cast(INT64_MAX)) + return INT64_MAX; + else if (value < static_cast(INT64_MIN)) + return INT64_MIN; + + return static_cast(value); + } + + inline operator int64_t() const + { + return toTSInt64(); + } + + inline uint64_t toTUInt64() const + { + if (value > static_cast(UINT64_MAX)) + return UINT64_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + + inline operator uint64_t() const + { + return toTUInt64(); + } + + inline TFloat128 operator+(const TFloat128& rhs) const + { + return TFloat128(value + rhs.value); + } + + inline long double toLongDouble() const + { + if (value > static_cast(LDBL_MAX)) + return LDBL_MAX; + else if (value < -static_cast(LDBL_MAX)) + return -LDBL_MAX; + + return static_cast(value); + } + + private: + float128_t value; }; -} //end of namespace +} // namespace datatypes -#endif // MCS_TSFLOAT128_H_INCLUDED +#endif // MCS_TSFLOAT128_H_INCLUDED // vim:ts=2 sw=2: - diff --git a/datatypes/mcs_int128.cpp b/datatypes/mcs_int128.cpp index fb6df19fd..13329fc41 100644 --- a/datatypes/mcs_int128.cpp +++ b/datatypes/mcs_int128.cpp @@ -14,8 +14,8 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ + MA 02110-1301, USA. +*/ #include @@ -24,98 +24,93 @@ namespace datatypes { - uint8_t TSInt128::printPodParts(char* buf, - const int128_t& high, - const int128_t& mid, - const int128_t& low) const - { - char* p = buf; - // pod[0] is low 8 bytes, pod[1] is high 8 bytes - const uint64_t* high_pod = reinterpret_cast(&high); - const uint64_t* mid_pod = reinterpret_cast(&mid); - const uint64_t* low_pod = reinterpret_cast(&low); +uint8_t TSInt128::printPodParts(char* buf, const int128_t& high, const int128_t& mid, + const int128_t& low) const +{ + char* p = buf; + // pod[0] is low 8 bytes, pod[1] is high 8 bytes + const uint64_t* high_pod = reinterpret_cast(&high); + const uint64_t* mid_pod = reinterpret_cast(&mid); + const uint64_t* low_pod = reinterpret_cast(&low); - if (high_pod[0] != 0) - { - p += sprintf(p, "%lu", high_pod[0]); - p += sprintf(p, "%019lu", mid_pod[0]); - p += sprintf(p, "%019lu", low_pod[0]); - } - else if (mid_pod[0] != 0) - { - p += sprintf(p, "%lu", mid_pod[0]); - p += sprintf(p, "%019lu", low_pod[0]); - } - else - { - p += sprintf(p, "%lu", low_pod[0]); - } - return p - buf; + if (high_pod[0] != 0) + { + p += sprintf(p, "%lu", high_pod[0]); + p += sprintf(p, "%019lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else if (mid_pod[0] != 0) + { + p += sprintf(p, "%lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else + { + p += sprintf(p, "%lu", low_pod[0]); + } + return p - buf; +} + +// This method writes unsigned integer representation of TSInt128 +uint8_t TSInt128::writeIntPart(const int128_t& x, char* buf, const uint8_t buflen) const +{ + char* p = buf; + int128_t high = 0, mid = 0, low = 0; + uint64_t maxUint64divisor = 10000000000000000000ULL; + + low = x % maxUint64divisor; + int128_t value = x / maxUint64divisor; + mid = value % maxUint64divisor; + high = value / maxUint64divisor; + + p += printPodParts(p, high, mid, low); + uint8_t written = p - buf; + if (buflen <= written) + { + throw logging::QueryDataExcept("TSInt128::writeIntPart() char buffer overflow.", logging::formatErr); } - // This method writes unsigned integer representation of TSInt128 - uint8_t TSInt128::writeIntPart(const int128_t& x, - char* buf, - const uint8_t buflen) const + return written; +} + +// conversion to std::string +std::string TSInt128::toString() const +{ + if (isNull()) { - char* p = buf; - int128_t high = 0, mid = 0, low = 0; - uint64_t maxUint64divisor = 10000000000000000000ULL; - - low = x % maxUint64divisor; - int128_t value = x / maxUint64divisor; - mid = value % maxUint64divisor; - high = value / maxUint64divisor; - - p += printPodParts(p, high, mid, low); - uint8_t written = p - buf; - if (buflen <= written) - { - throw logging::QueryDataExcept("TSInt128::writeIntPart() char buffer overflow.", - logging::formatErr); - } - - return written; + return std::string("NULL"); } - // conversion to std::string - std::string TSInt128::toString() const + if (isEmpty()) { - if (isNull()) - { - return std::string("NULL"); - } + return std::string("EMPTY"); + } - if (isEmpty()) - { - return std::string("EMPTY"); - } - - int128_t tempValue = s128Value; - char buf[TSInt128::MAXLENGTH16BYTES]; - uint8_t left = sizeof(buf); - char* p = buf; - // sign - if (tempValue < static_cast(0)) - { - *p++ = '-'; - tempValue *= -1; - left--; - } - // integer part - // reduce the size by one to account for \0 + int128_t tempValue = s128Value; + char buf[TSInt128::MAXLENGTH16BYTES]; + uint8_t left = sizeof(buf); + char* p = buf; + // sign + if (tempValue < static_cast(0)) + { + *p++ = '-'; + tempValue *= -1; left--; - p += writeIntPart(tempValue, p, left); - *p = '\0'; - - return std::string(buf); } + // integer part + // reduce the size by one to account for \0 + left--; + p += writeIntPart(tempValue, p, left); + *p = '\0'; - std::ostream& operator<<(std::ostream& os, const TSInt128& x) - { - os << x.toString(); - return os; - } + return std::string(buf); +} -} // end of namespace datatypes +std::ostream& operator<<(std::ostream& os, const TSInt128& x) +{ + os << x.toString(); + return os; +} + +} // end of namespace datatypes // vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h index 4a23033c0..02df0c7e2 100644 --- a/datatypes/mcs_int128.h +++ b/datatypes/mcs_int128.h @@ -28,100 +28,73 @@ // Inline asm has three argument lists: output, input and clobber list #ifdef __aarch64__ - #define MACRO_VALUE_PTR_128(dst, \ - dst_restrictions, \ - src, \ - src_restrictions, \ - clobb) \ - ::memcpy((dst), &(src), sizeof(int128_t)); - #define MACRO_PTR_PTR_128(dst, \ - dst_restrictions, \ - src, \ - src_restrictions, \ - clobb) \ - ::memcpy((dst), (src), sizeof(int128_t)); +#define MACRO_VALUE_PTR_128(dst, dst_restrictions, src, src_restrictions, clobb) \ + ::memcpy((dst), &(src), sizeof(int128_t)); +#define MACRO_PTR_PTR_128(dst, dst_restrictions, src, src_restrictions, clobb) \ + ::memcpy((dst), (src), sizeof(int128_t)); #elif defined(__GNUC__) && (__GNUC___ > 7) || defined(__clang__) - #define MACRO_VALUE_PTR_128(dst, \ - dst_restrictions, \ - src, \ - src_restrictions, \ - clobb) \ - __asm__ volatile("movups %1,%0" \ - :dst_restrictions ( *(dst) ) \ - :src_restrictions ( (src) ) \ - :clobb \ - ); - #define MACRO_PTR_PTR_128(dst, \ - dst_restrictions, \ - src, \ - src_restrictions, \ - clobb) \ - ::memcpy((dst), (src), sizeof(int128_t)); +#define MACRO_VALUE_PTR_128(dst, dst_restrictions, src, src_restrictions, clobb) \ + __asm__ volatile("movups %1,%0" : dst_restrictions(*(dst)) : src_restrictions((src)) : clobb); +#define MACRO_PTR_PTR_128(dst, dst_restrictions, src, src_restrictions, clobb) \ + ::memcpy((dst), (src), sizeof(int128_t)); #else - #define MACRO_VALUE_PTR_128(dst, \ - dst_restrictions, \ - src, \ - src_restrictions, \ - clobb) \ - __asm__ volatile("movups %1,%0" \ - :dst_restrictions ( *(dst) ) \ - :src_restrictions ( (src) ) \ - :clobb \ - ); - #define MACRO_PTR_PTR_128(dst, \ - dst_restrictions, \ - src, \ - src_restrictions, \ - clobb) \ - __asm__ volatile("movdqu %1,%%xmm0;" \ - "movups %%xmm0,%0;" \ - :dst_restrictions ( *(dst) ) \ - :src_restrictions ( *(src) ) \ - :"memory", clobb \ - ); +#define MACRO_VALUE_PTR_128(dst, dst_restrictions, src, src_restrictions, clobb) \ + __asm__ volatile("movups %1,%0" : dst_restrictions(*(dst)) : src_restrictions((src)) : clobb); +#define MACRO_PTR_PTR_128(dst, dst_restrictions, src, src_restrictions, clobb) \ + __asm__ volatile( \ + "movdqu %1,%%xmm0;" \ + "movups %%xmm0,%0;" \ + : dst_restrictions(*(dst)) \ + : src_restrictions(*(src)) \ + : "memory", clobb); #endif namespace datatypes { - using int128_t = __int128; using uint128_t = unsigned __int128; - // Type traits template -struct is_allowed_numeric { +struct is_allowed_numeric +{ static const bool value = false; }; template <> -struct is_allowed_numeric { +struct is_allowed_numeric +{ static const bool value = true; }; template <> -struct is_allowed_numeric { +struct is_allowed_numeric +{ static const bool value = true; }; template -struct is_int128_t { +struct is_int128_t +{ static const bool value = false; }; -template<> -struct is_int128_t { +template <> +struct is_int128_t +{ static const bool value = true; }; template -struct is_uint128_t { +struct is_uint128_t +{ static const bool value = false; }; -template<> -struct is_uint128_t { +template <> +struct is_uint128_t +{ static const bool value = true; }; @@ -132,218 +105,223 @@ inline int128_t abs(int128_t x) class TSInt128 { - public: - static constexpr uint8_t MAXLENGTH16BYTES = 42; - static constexpr int128_t NullValue = int128_t(0x8000000000000000LL) << 64; - static constexpr int128_t EmptyValue = (int128_t(0x8000000000000000LL) << 64) + 1; + public: + static constexpr uint8_t MAXLENGTH16BYTES = 42; + static constexpr int128_t NullValue = int128_t(0x8000000000000000LL) << 64; + static constexpr int128_t EmptyValue = (int128_t(0x8000000000000000LL) << 64) + 1; - // A variety of ctors for aligned and unaligned arguments - TSInt128(): s128Value(0) { } + // A variety of ctors for aligned and unaligned arguments + TSInt128() : s128Value(0) + { + } - // Copy ctor - TSInt128(const TSInt128& other): s128Value(other.s128Value) { } + // Copy ctor + TSInt128(const TSInt128& other) : s128Value(other.s128Value) + { + } - TSInt128& operator=(const TSInt128& other) - { - s128Value = other.s128Value; - return *this; - } + TSInt128& operator=(const TSInt128& other) + { + s128Value = other.s128Value; + return *this; + } - // aligned argument - explicit TSInt128(const int128_t& x) { s128Value = x; } + // aligned argument + explicit TSInt128(const int128_t& x) + { + s128Value = x; + } - // unaligned argument - explicit TSInt128(const int128_t* x) { assignPtrPtr(&s128Value, x); } + // unaligned argument + explicit TSInt128(const int128_t* x) + { + assignPtrPtr(&s128Value, x); + } - // unaligned argument - explicit TSInt128(const unsigned char* x) { assignPtrPtr(&s128Value, x); } + // unaligned argument + explicit TSInt128(const unsigned char* x) + { + assignPtrPtr(&s128Value, x); + } - // Method returns max length of a string representation - static constexpr uint8_t maxLength() - { - return TSInt128::MAXLENGTH16BYTES; - } + // Method returns max length of a string representation + static constexpr uint8_t maxLength() + { + return TSInt128::MAXLENGTH16BYTES; + } - // Checks if the value is NULL - inline bool isNull() const - { - return s128Value == NullValue; - } + // Checks if the value is NULL + inline bool isNull() const + { + return s128Value == NullValue; + } - // Checks if the value is Empty - inline bool isEmpty() const - { - return s128Value == EmptyValue; - } + // Checks if the value is Empty + inline bool isEmpty() const + { + return s128Value == EmptyValue; + } - // The method copies 16 bytes from one memory cell - // into another using memcpy or SIMD. - // memcpy in gcc >= 7 is replaced with SIMD instructions - template - static inline void assignPtrPtr(D* dst, const S* src) - { - MACRO_PTR_PTR_128(dst, "=m", src, "m", "xmm0") - } + // The method copies 16 bytes from one memory cell + // into another using memcpy or SIMD. + // memcpy in gcc >= 7 is replaced with SIMD instructions + template + static inline void assignPtrPtr(D* dst, const S* src) + { + MACRO_PTR_PTR_128(dst, "=m", src, "m", "xmm0") + } - template - static inline void storeUnaligned(D* dst, const int128_t& src) - { - MACRO_VALUE_PTR_128(dst, "=m", src, "x", "memory") - } + template + static inline void storeUnaligned(D* dst, const int128_t& src) + { + MACRO_VALUE_PTR_128(dst, "=m", src, "x", "memory") + } - // operators - template::value> > - inline bool operator<(const T& x) const - { - return s128Value < static_cast(x); - } + // operators + template ::value> > + inline bool operator<(const T& x) const + { + return s128Value < static_cast(x); + } - template::value> > - inline bool operator==(const T& x) const - { - return s128Value == static_cast(x); - } + template ::value> > + inline bool operator==(const T& x) const + { + return s128Value == static_cast(x); + } - inline operator double() const - { - return toDouble(); - } + inline operator double() const + { + return toDouble(); + } - inline double toDouble() const - { - return static_cast(s128Value); - } + inline double toDouble() const + { + return static_cast(s128Value); + } - inline operator long double() const - { - return toLongDouble(); - } + inline operator long double() const + { + return toLongDouble(); + } - inline long double toLongDouble() const - { - return static_cast(s128Value); - } + inline long double toLongDouble() const + { + return static_cast(s128Value); + } - inline operator int32_t() const - { - if (s128Value > static_cast(INT32_MAX)) - return INT32_MAX; - if (s128Value < static_cast(INT32_MIN)) - return INT32_MIN; + inline operator int32_t() const + { + if (s128Value > static_cast(INT32_MAX)) + return INT32_MAX; + if (s128Value < static_cast(INT32_MIN)) + return INT32_MIN; - return static_cast(s128Value); - } + return static_cast(s128Value); + } - inline operator uint32_t() const - { - if (s128Value > static_cast(UINT32_MAX)) - return UINT32_MAX; - if (s128Value < 0) - return 0; + inline operator uint32_t() const + { + if (s128Value > static_cast(UINT32_MAX)) + return UINT32_MAX; + if (s128Value < 0) + return 0; - return static_cast(s128Value); - } + return static_cast(s128Value); + } - inline operator int64_t() const - { - if (s128Value > static_cast(INT64_MAX)) - return INT64_MAX; - if (s128Value < static_cast(INT64_MIN)) - return INT64_MIN; + inline operator int64_t() const + { + if (s128Value > static_cast(INT64_MAX)) + return INT64_MAX; + if (s128Value < static_cast(INT64_MIN)) + return INT64_MIN; - return static_cast(s128Value); - } + return static_cast(s128Value); + } - inline operator uint64_t() const - { - if (s128Value > static_cast(UINT64_MAX)) - return UINT64_MAX; - if (s128Value < 0) - return 0; + inline operator uint64_t() const + { + if (s128Value > static_cast(UINT64_MAX)) + return UINT64_MAX; + if (s128Value < 0) + return 0; - return static_cast(s128Value); - } + return static_cast(s128Value); + } - inline operator TFloat128() const - { - return toTFloat128(); - } + inline operator TFloat128() const + { + return toTFloat128(); + } - // unaligned argument - inline TSInt128& operator=(const int128_t* x) - { - assignPtrPtr(&s128Value, x); - return *this; - } + // unaligned argument + inline TSInt128& operator=(const int128_t* x) + { + assignPtrPtr(&s128Value, x); + return *this; + } - inline TSInt128 operator%(const int64_t& rhs) const - { - return TSInt128(s128Value % rhs); - } + inline TSInt128 operator%(const int64_t& rhs) const + { + return TSInt128(s128Value % rhs); + } - inline TSInt128 operator%(const int128_t& rhs) const - { - return TSInt128(s128Value % rhs); - } + inline TSInt128 operator%(const int128_t& rhs) const + { + return TSInt128(s128Value % rhs); + } - inline TSInt128 operator*(const TSInt128& rhs) const - { - return TSInt128(s128Value * rhs.s128Value); - } + inline TSInt128 operator*(const TSInt128& rhs) const + { + return TSInt128(s128Value * rhs.s128Value); + } - inline TSInt128 operator+(const TSInt128& rhs) const - { - return TSInt128(s128Value + rhs.s128Value); - } + inline TSInt128 operator+(const TSInt128& rhs) const + { + return TSInt128(s128Value + rhs.s128Value); + } - inline bool operator>(const TSInt128& rhs) const - { - return s128Value > rhs.s128Value; - } + inline bool operator>(const TSInt128& rhs) const + { + return s128Value > rhs.s128Value; + } - inline bool operator<(const TSInt128& rhs) const - { - return s128Value < rhs.s128Value; - } + inline bool operator<(const TSInt128& rhs) const + { + return s128Value < rhs.s128Value; + } - inline bool operator!=(const TSInt128& rhs) const - { - return s128Value != rhs.getValue(); - } + inline bool operator!=(const TSInt128& rhs) const + { + return s128Value != rhs.getValue(); + } - inline TFloat128 toTFloat128() const - { - return TFloat128(s128Value); - } + inline TFloat128 toTFloat128() const + { + return TFloat128(s128Value); + } - inline const int128_t& getValue() const - { - return s128Value; - } + inline const int128_t& getValue() const + { + return s128Value; + } - // print int128_t parts represented as PODs - uint8_t printPodParts(char* buf, - const int128_t& high, - const int128_t& mid, - const int128_t& low) const; + // print int128_t parts represented as PODs + uint8_t printPodParts(char* buf, const int128_t& high, const int128_t& mid, const int128_t& low) const; - // writes integer part of dec into a buffer - uint8_t writeIntPart(const int128_t& x, - char* buf, - const uint8_t buflen) const; + // writes integer part of dec into a buffer + uint8_t writeIntPart(const int128_t& x, char* buf, const uint8_t buflen) const; - // string representation of TSInt128 - std::string toString() const; + // string representation of TSInt128 + std::string toString() const; - friend std::ostream& operator<<(std::ostream& os, const TSInt128& x); + friend std::ostream& operator<<(std::ostream& os, const TSInt128& x); - int128_t s128Value; - }; // end of class + int128_t s128Value; +}; // end of class +} // end of namespace datatypes -} //end of namespace datatypes - -#endif // MCS_TSINT128_H_INCLUDED +#endif // MCS_TSINT128_H_INCLUDED // vim:ts=2 sw=2: diff --git a/datatypes/mcs_int64.h b/datatypes/mcs_int64.h index 431a7315c..c44f1169a 100644 --- a/datatypes/mcs_int64.h +++ b/datatypes/mcs_int64.h @@ -14,8 +14,8 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ + MA 02110-1301, USA. +*/ #ifndef MCS_INT64_H_INCLUDED #define MCS_INT64_H_INCLUDED @@ -23,155 +23,162 @@ namespace datatypes { - - class TNullFlag { -protected: + protected: bool mIsNull; -public: - explicit TNullFlag(bool val) :mIsNull(val) { } + + public: + explicit TNullFlag(bool val) : mIsNull(val) + { + } bool isNull() const { return mIsNull; } }; - class TUInt64 { -protected: + protected: uint64_t mValue; -public: - TUInt64(): mValue(0) { } - explicit TUInt64(uint64_t value): mValue(value) { } + public: + TUInt64() : mValue(0) + { + } - explicit operator uint64_t () const + explicit TUInt64(uint64_t value) : mValue(value) + { + } + + explicit operator uint64_t() const { return mValue; } void store(uint8_t* dst) const { - *(uint64_t*) dst = mValue; + *(uint64_t*)dst = mValue; } }; - class TSInt64 { -protected: + protected: int64_t mValue; -public: - TSInt64(): mValue(0) { } - explicit TSInt64(int64_t value): mValue(value) { } + public: + TSInt64() : mValue(0) + { + } - explicit operator int64_t () const + explicit TSInt64(int64_t value) : mValue(value) + { + } + + explicit operator int64_t() const { return mValue; } - explicit operator uint64_t () const + explicit operator uint64_t() const { return mValue < 0 ? 0 : static_cast(mValue); } }; - - -class TUInt64Null: public TUInt64, public TNullFlag +class TUInt64Null : public TUInt64, public TNullFlag { -public: + public: + TUInt64Null() : TNullFlag(true) + { + } - TUInt64Null(): TNullFlag(true) { } + explicit TUInt64Null(uint64_t value, bool isNull = false) : TUInt64(value), TNullFlag(isNull) + { + } - explicit TUInt64Null(uint64_t value, bool isNull = false): - TUInt64(value), TNullFlag(isNull) - { } - - explicit operator uint64_t () const + explicit operator uint64_t() const { idbassert(!mIsNull); return mValue; } - uint64_t nullSafeValue(bool & isNullRef) const + uint64_t nullSafeValue(bool& isNullRef) const { return (isNullRef = isNull()) ? 0 : mValue; } - TUInt64Null operator&(const TUInt64Null &rhs) const + TUInt64Null operator&(const TUInt64Null& rhs) const { return TUInt64Null(mValue & rhs.mValue, mIsNull || rhs.mIsNull); } - TUInt64Null operator|(const TUInt64Null &rhs) const + TUInt64Null operator|(const TUInt64Null& rhs) const { return TUInt64Null(mValue | rhs.mValue, mIsNull || rhs.mIsNull); } - TUInt64Null operator^(const TUInt64Null &rhs) const + TUInt64Null operator^(const TUInt64Null& rhs) const { return TUInt64Null(mValue ^ rhs.mValue, mIsNull || rhs.mIsNull); } - TUInt64Null MariaDBShiftLeft(const TUInt64Null &rhs) const + TUInt64Null MariaDBShiftLeft(const TUInt64Null& rhs) const { return TUInt64Null(rhs.mValue >= 64 ? 0 : mValue << rhs.mValue, mIsNull || rhs.mIsNull); } - TUInt64Null MariaDBShiftRight(const TUInt64Null &rhs) const + TUInt64Null MariaDBShiftRight(const TUInt64Null& rhs) const { return TUInt64Null(rhs.mValue >= 64 ? 0 : mValue >> rhs.mValue, mIsNull || rhs.mIsNull); } }; - -class TSInt64Null: public TSInt64, public TNullFlag +class TSInt64Null : public TSInt64, public TNullFlag { -public: + public: + TSInt64Null() : TNullFlag(true) + { + } - TSInt64Null(): TNullFlag(true) { } + explicit TSInt64Null(int64_t value, bool isNull = false) : TSInt64(value), TNullFlag(isNull) + { + } - explicit TSInt64Null(int64_t value, bool isNull = false): - TSInt64(value), TNullFlag(isNull) - { } - - explicit operator int64_t () const + explicit operator int64_t() const { idbassert(!mIsNull); return mValue; } - int64_t nullSafeValue(bool & isNullRef) const + int64_t nullSafeValue(bool& isNullRef) const { return (isNullRef = isNull()) ? 0 : mValue; } - TSInt64Null operator&(const TSInt64Null &rhs) const + TSInt64Null operator&(const TSInt64Null& rhs) const { return TSInt64Null(mValue & rhs.mValue, mIsNull || rhs.mIsNull); } - TSInt64Null operator|(const TSInt64Null &rhs) const + TSInt64Null operator|(const TSInt64Null& rhs) const { return TSInt64Null(mValue | rhs.mValue, mIsNull || rhs.mIsNull); } - TSInt64Null operator^(const TSInt64Null &rhs) const + TSInt64Null operator^(const TSInt64Null& rhs) const { return TSInt64Null(mValue ^ rhs.mValue, mIsNull || rhs.mIsNull); } - TSInt64Null MariaDBShiftLeft(const TUInt64Null &rhs) const + TSInt64Null MariaDBShiftLeft(const TUInt64Null& rhs) const { if (isNull() || rhs.isNull()) return TSInt64Null(); - return TSInt64Null((uint64_t) rhs >= 64 ? 0 : mValue << (uint64_t) rhs, false); + return TSInt64Null((uint64_t)rhs >= 64 ? 0 : mValue << (uint64_t)rhs, false); } - TSInt64Null MariaDBShiftRight(const TUInt64Null &rhs) const + TSInt64Null MariaDBShiftRight(const TUInt64Null& rhs) const { if (isNull() || rhs.isNull()) return TSInt64Null(); - return TSInt64Null((uint64_t) rhs >= 64 ? 0 : mValue >> (uint64_t) rhs, false); + return TSInt64Null((uint64_t)rhs >= 64 ? 0 : mValue >> (uint64_t)rhs, false); } }; +} // end of namespace datatypes -} //end of namespace datatypes - -#endif // MCS_INT64_H_INCLUDED +#endif // MCS_INT64_H_INCLUDED // vim:ts=2 sw=2: diff --git a/datatypes/mcs_longdouble.h b/datatypes/mcs_longdouble.h index 60caa3982..55215642e 100644 --- a/datatypes/mcs_longdouble.h +++ b/datatypes/mcs_longdouble.h @@ -23,17 +23,21 @@ namespace datatypes { - class TLongDouble { -protected: + protected: long double mValue; -public: - TLongDouble(): mValue(0) { } - explicit TLongDouble(long double value): mValue(value) { } + public: + TLongDouble() : mValue(0) + { + } - explicit operator long double () const + explicit TLongDouble(long double value) : mValue(value) + { + } + + explicit operator long double() const { return mValue; } @@ -49,8 +53,7 @@ public: } }; +} // end of namespace datatypes -} //end of namespace datatypes - -#endif // MCS_LONGDOUBLE_H_INCLUDED +#endif // MCS_LONGDOUBLE_H_INCLUDED // vim:ts=2 sw=2: diff --git a/datatypes/mcs_numeric_limits.h b/datatypes/mcs_numeric_limits.h index 48fe545db..a12f68b6d 100644 --- a/datatypes/mcs_numeric_limits.h +++ b/datatypes/mcs_numeric_limits.h @@ -22,18 +22,24 @@ namespace datatypes { - template struct numeric_limits { - static constexpr T min() { return std::numeric_limits::min(); } - static constexpr T max() { return std::numeric_limits::max(); } + static constexpr T min() + { + return std::numeric_limits::min(); + } + static constexpr T max() + { + return std::numeric_limits::max(); + } }; using int128_t = __int128; using uint128_t = unsigned __int128; -template<> struct numeric_limits +template <> +struct numeric_limits { static constexpr int128_t min() { @@ -45,7 +51,8 @@ template<> struct numeric_limits } }; -template<> struct numeric_limits +template <> +struct numeric_limits { static constexpr uint128_t min() { @@ -57,6 +64,6 @@ template<> struct numeric_limits } }; -} // namespace datatypes +} // namespace datatypes -#endif // MCS_NUMERIC_LIMITS_H_INCLUDED +#endif // MCS_NUMERIC_LIMITS_H_INCLUDED diff --git a/datatypes/mcs_string.h b/datatypes/mcs_string.h index 15720cedb..a6023d9b4 100644 --- a/datatypes/mcs_string.h +++ b/datatypes/mcs_string.h @@ -15,39 +15,35 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef MCS_DATATYPES_STRING_H #define MCS_DATATYPES_STRING_H #include "conststring.h" -#include "collation.h" // class Charset +#include "collation.h" // class Charset namespace datatypes { - class TCharShort { - int64_t mValue; -public: - TCharShort(int64_t value) - :mValue(value) - { } - utils::ConstString toConstString(uint32_t width) const - { - utils::ConstString res = utils::ConstString((const char *) &mValue, width); - return res.rtrimZero(); - } - static int strnncollsp(const datatypes::Charset &cs, int64_t a, int64_t b, uint32_t width) - { - datatypes::TCharShort sa(a); - datatypes::TCharShort sb(b); - return cs.strnncollsp(sa.toConstString(width), - sb.toConstString(width)); - } + int64_t mValue; + + public: + TCharShort(int64_t value) : mValue(value) + { + } + utils::ConstString toConstString(uint32_t width) const + { + utils::ConstString res = utils::ConstString((const char*)&mValue, width); + return res.rtrimZero(); + } + static int strnncollsp(const datatypes::Charset& cs, int64_t a, int64_t b, uint32_t width) + { + datatypes::TCharShort sa(a); + datatypes::TCharShort sb(b); + return cs.strnncollsp(sa.toConstString(width), sb.toConstString(width)); + } }; +} // namespace datatypes -} // namespace datatypes - - -#endif // MCS_DATATYPES_STRING_H +#endif // MCS_DATATYPES_STRING_H diff --git a/datatypes/numericliteral.h b/datatypes/numericliteral.h index 2aeb58ab3..701d9402f 100644 --- a/datatypes/numericliteral.h +++ b/datatypes/numericliteral.h @@ -15,33 +15,26 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef NUMERICLITERAL_H #define NUMERICLITERAL_H #include "genericparser.h" #include "mcs_datatype.h" - namespace literal { - -using utils::ConstString; -using genericparser::Parser; using datatypes::DataCondition; +using genericparser::Parser; +using utils::ConstString; typedef uint32_t scale_t; - - -template -class Converter: public Parser, - public A +template +class Converter : public Parser, public A { -public: - Converter(const char *str, size_t length, DataCondition & error) - :Parser(str, length), - A(&Parser::skipLeadingSpaces()) + public: + Converter(const char* str, size_t length, DataCondition& error) + : Parser(str, length), A(&Parser::skipLeadingSpaces()) { if (Parser::syntaxError()) { @@ -57,15 +50,14 @@ public: '1e' - exponent marker was not followed by , expect '1e1' '1e+' - in , was not followed by a digit, expect '1e+1' */ - error|=(DataCondition::X_INVALID_CHARACTER_VALUE_FOR_CAST); + error |= (DataCondition::X_INVALID_CHARACTER_VALUE_FOR_CAST); } } - Converter(const std::string & str, DataCondition &error) - :Converter(str.data(), str.length(), error) - { } + Converter(const std::string& str, DataCondition& error) : Converter(str.data(), str.length(), error) + { + } }; - /* SQL Standard definition for @@ -144,70 +136,83 @@ Grammar */ - // // Terminal symbols // -class Period: public ConstString +class Period : public ConstString { -public: - explicit Period(Parser *p) - :ConstString(p->tokenChar('.')) - { } - bool isNull() const { return mStr == nullptr; } + public: + explicit Period(Parser* p) : ConstString(p->tokenChar('.')) + { + } + bool isNull() const + { + return mStr == nullptr; + } }; - -class ExponentMarker: public ConstString +class ExponentMarker : public ConstString { -public: - explicit ExponentMarker(Parser *p) - :ConstString(p->tokenAnyCharOf('e', 'E')) - { } - bool isNull() const { return mStr == nullptr; } + public: + explicit ExponentMarker(Parser* p) : ConstString(p->tokenAnyCharOf('e', 'E')) + { + } + bool isNull() const + { + return mStr == nullptr; + } }; - -class Sign: public ConstString +class Sign : public ConstString { -public: - explicit Sign(): ConstString(NULL, 0) { } - explicit Sign(const ConstString &str) - :ConstString(str) - { } - explicit Sign(Parser *p) - :ConstString(p->tokenAnyCharOf('+', '-')) - { } - static Sign empty(Parser *p) + public: + explicit Sign() : ConstString(NULL, 0) + { + } + explicit Sign(const ConstString& str) : ConstString(str) + { + } + explicit Sign(Parser* p) : ConstString(p->tokenAnyCharOf('+', '-')) + { + } + static Sign empty(Parser* p) { return Sign(p->tokStartConstString()); } - bool isNull() const { return mStr == nullptr; } - bool negative() const { return eq('-'); } + bool isNull() const + { + return mStr == nullptr; + } + bool negative() const + { + return eq('-'); + } }; - -class Digits: public ConstString +class Digits : public ConstString { -public: - explicit Digits() - :ConstString(NULL, 0) - { } - explicit Digits(const char *str, size_t length) - :ConstString(str, length) - { } - explicit Digits(const ConstString &str) - :ConstString(str) - { } - explicit Digits(Parser *p) - :ConstString(p->tokenDigits()) - { } - bool isNull() const { return mStr == nullptr; } + public: + explicit Digits() : ConstString(NULL, 0) + { + } + explicit Digits(const char* str, size_t length) : ConstString(str, length) + { + } + explicit Digits(const ConstString& str) : ConstString(str) + { + } + explicit Digits(Parser* p) : ConstString(p->tokenDigits()) + { + } + bool isNull() const + { + return mStr == nullptr; + } void skipLeadingZeroDigits() { - for ( ; mLength > 0 && mStr[0] == '0'; ) + for (; mLength > 0 && mStr[0] == '0';) { mStr++; mLength--; @@ -215,33 +220,32 @@ public: } void skipTrailingZeroDigits() { - for ( ; mLength > 0 && mStr[mLength - 1] == '0' ; ) + for (; mLength > 0 && mStr[mLength - 1] == '0';) mLength--; } }; - // // Non-terminal symbols // // ::= ... -class UnsignedInteger: public Digits +class UnsignedInteger : public Digits { -public: - explicit UnsignedInteger() - :Digits() - { } - explicit UnsignedInteger(const char *str, size_t length) - :Digits(str, length) - { } - explicit UnsignedInteger(const ConstString &str) - :Digits(str) - { } - explicit UnsignedInteger(Parser *p) - :Digits(p) - { } - static UnsignedInteger empty(const Parser *p) + public: + explicit UnsignedInteger() : Digits() + { + } + explicit UnsignedInteger(const char* str, size_t length) : Digits(str, length) + { + } + explicit UnsignedInteger(const ConstString& str) : Digits(str) + { + } + explicit UnsignedInteger(Parser* p) : Digits(p) + { + } + static UnsignedInteger empty(const Parser* p) { return UnsignedInteger(p->tokStartConstString()); } @@ -250,182 +254,177 @@ public: return UnsignedInteger(str(), length() > len ? len : length()); } - template - T toXIntPositiveContinue(T start, DataCondition & error) const + template + T toXIntPositiveContinue(T start, DataCondition& error) const { - const char *e = end(); + const char* e = end(); T val = start; - for (const char *s= mStr; s < e; s++) + for (const char* s = mStr; s < e; s++) { constexpr T cutoff = datatypes::numeric_limits::max() / 10; if (val > cutoff) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return datatypes::numeric_limits::max(); } - val*= 10; + val *= 10; T newval = val + (s[0] - '0'); if (newval < val) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return datatypes::numeric_limits::max(); } val = newval; } return val; } - template - T toXIntPositive(DataCondition & error) const + template + T toXIntPositive(DataCondition& error) const { return toXIntPositiveContinue(0, error); } - template - T toSIntNegativeContinue(T start, DataCondition & error) const + template + T toSIntNegativeContinue(T start, DataCondition& error) const { - const char *e = end(); + const char* e = end(); T val = start; - for (const char *s= mStr; s < e; s++) + for (const char* s = mStr; s < e; s++) { constexpr T cutoff = datatypes::numeric_limits::min() / 10; if (val < cutoff) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return datatypes::numeric_limits::min(); } - val*= 10; + val *= 10; T newval = val - (s[0] - '0'); if (newval > val) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return datatypes::numeric_limits::min(); } val = newval; } return val; } - template - T toSIntNegative(DataCondition & error) const + template + T toSIntNegative(DataCondition& error) const { return toSIntNegativeContinue(0, error); } - template - T toXIntPositiveRoundAwayFromZeroContinue(T start, bool round, DataCondition & error) const + template + T toXIntPositiveRoundAwayFromZeroContinue(T start, bool round, DataCondition& error) const { T val = toXIntPositiveContinue(start, error); if (val == datatypes::numeric_limits::max() && round) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return val; } return val + round; } - template - T toXIntPositiveRoundAwayFromZero(bool round, DataCondition & error) const + template + T toXIntPositiveRoundAwayFromZero(bool round, DataCondition& error) const { return toXIntPositiveRoundAwayFromZeroContinue(0, round, error); } }; - // := [] -class SignedInteger: public Parser::DD2OM +class SignedInteger : public Parser::DD2OM { -public: + public: using DD2OM::DD2OM; - bool isNull() const { return UnsignedInteger::isNull(); } + bool isNull() const + { + return UnsignedInteger::isNull(); + } - template T abs(DataCondition & error) const + template + T abs(DataCondition& error) const { return toXIntPositive(error); } - template T toSInt(DataCondition & error) const + template + T toSInt(DataCondition& error) const { - return negative() ? - toSIntNegative(error) : - toXIntPositive(error); + return negative() ? toSIntNegative(error) : toXIntPositive(error); } }; - // E -class EExponent: public Parser::UD2MM +class EExponent : public Parser::UD2MM { -public: + public: using UD2MM::UD2MM; }; - // -class ExactUnsignedNumericLiteralFractionAlone: public Parser::UD2MM +class ExactUnsignedNumericLiteralFractionAlone : public Parser::UD2MM { -public: + public: using UD2MM::UD2MM; }; - // [ ] -class PeriodOptUnsignedInteger: public Parser::UD2MO +class PeriodOptUnsignedInteger : public Parser::UD2MO { -public: + public: using UD2MO::UD2MO; - static PeriodOptUnsignedInteger empty(Parser *p) + static PeriodOptUnsignedInteger empty(Parser* p) { return PeriodOptUnsignedInteger(UnsignedInteger(p->tokStartConstString())); } - const PeriodOptUnsignedInteger & fraction() const + const PeriodOptUnsignedInteger& fraction() const { return *this; } }; - // := -class IntegralUnsignedInteger: public UnsignedInteger +class IntegralUnsignedInteger : public UnsignedInteger { -public: - explicit IntegralUnsignedInteger(Parser *p) - :UnsignedInteger(p) - { } - const UnsignedInteger & integral() const + public: + explicit IntegralUnsignedInteger(Parser* p) : UnsignedInteger(p) + { + } + const UnsignedInteger& integral() const { return *this; } }; - // [ [ ] ] -class ExactUnsignedNumericLiteralIntegralOptFraction: - public Parser::DD2MO +class ExactUnsignedNumericLiteralIntegralOptFraction + : public Parser::DD2MO { -public: + public: using DD2MO::DD2MO; }; - // A container for integral and fractional parts class UnsignedIntegerDecimal { -protected: + protected: UnsignedInteger mIntegral; UnsignedInteger mFraction; -public: - explicit UnsignedIntegerDecimal(const UnsignedInteger &intg, - const UnsignedInteger &frac) - :mIntegral(intg), - mFraction(frac) - { } - explicit UnsignedIntegerDecimal(const ExactUnsignedNumericLiteralFractionAlone &rhs) - :mFraction(rhs) - { } - explicit UnsignedIntegerDecimal(const ExactUnsignedNumericLiteralIntegralOptFraction &rhs) - :mIntegral(rhs.integral()), - mFraction(rhs.fraction()) - { } + + public: + explicit UnsignedIntegerDecimal(const UnsignedInteger& intg, const UnsignedInteger& frac) + : mIntegral(intg), mFraction(frac) + { + } + explicit UnsignedIntegerDecimal(const ExactUnsignedNumericLiteralFractionAlone& rhs) : mFraction(rhs) + { + } + explicit UnsignedIntegerDecimal(const ExactUnsignedNumericLiteralIntegralOptFraction& rhs) + : mIntegral(rhs.integral()), mFraction(rhs.fraction()) + { + } size_t IntFracDigits() const { @@ -443,51 +442,55 @@ public: mFraction.skipTrailingZeroDigits(); } - template T toXIntPositive(DataCondition & error) const + template + T toXIntPositive(DataCondition& error) const { T val = mIntegral.toXIntPositive(error); return mFraction.toXIntPositiveContinue(val, error); } - template T toXIntPositiveRoundAwayFromZero(bool roundUp, DataCondition & error) const + template + T toXIntPositiveRoundAwayFromZero(bool roundUp, DataCondition& error) const { T val = mIntegral.toXIntPositive(error); return mFraction.toXIntPositiveRoundAwayFromZeroContinue(val, roundUp, error); } - template T toXIntPositiveScaleUp(size_t scale, DataCondition & error) const + template + T toXIntPositiveScaleUp(size_t scale, DataCondition& error) const { T val = toXIntPositive(error); if (val == datatypes::numeric_limits::max()) return val; - for ( ; scale ; scale--) + for (; scale; scale--) { constexpr T cutoff = datatypes::numeric_limits::max() / 10; if (val > cutoff) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return datatypes::numeric_limits::max(); } - val*= 10; + val *= 10; } return val; } - template T toXIntPositiveRound(DataCondition & error) const + template + T toXIntPositiveRound(DataCondition& error) const { bool roundUp = mFraction.length() && mFraction.str()[0] >= '5'; return mIntegral.toXIntPositiveRoundAwayFromZero(roundUp, error); } - template T toXIntPositiveRoundExp(uint64_t absExp, bool negExp, - DataCondition & error) const + template + T toXIntPositiveRoundExp(uint64_t absExp, bool negExp, DataCondition& error) const { if (absExp == 0) return toXIntPositiveRound(error); if (negExp) { - if (mIntegral.length() == absExp) // 567.8e-3 -> 0.5678 -> 1 + if (mIntegral.length() == absExp) // 567.8e-3 -> 0.5678 -> 1 return mIntegral.str()[0] >= '5' ? 1 : 0; if (mIntegral.length() < absExp) // 123e-4 -> 0.0123 return 0; @@ -499,7 +502,7 @@ public: } // Positive exponent: 123.456e2 - if (mFraction.length() >= absExp) // 123.456e2 -> 12345.6 -> 12346 + if (mFraction.length() >= absExp) // 123.456e2 -> 12345.6 -> 12346 { bool roundUp = mFraction.length() > absExp && mFraction.str()[absExp] >= '5'; UnsignedIntegerDecimal tmp(mIntegral, mFraction.left(absExp)); @@ -510,42 +513,38 @@ public: size_t diff = absExp - mFraction.length(); return toXIntPositiveScaleUp(diff, error); } - }; - // := // [ ] // | [ [ ] ] -class ExactUnsignedNumericLiteral: - public Parser::Choice2 +class ExactUnsignedNumericLiteral + : public Parser::Choice2 { -public: + public: using Choice2::Choice2; }; - // ::= [ E ] -class UnsignedNumericLiteral: public Parser::DM2MO +class UnsignedNumericLiteral : public Parser::DM2MO { -public: + public: using DM2MO::DM2MO; void normalize() { ExactUnsignedNumericLiteral::normalize(); mB.skipLeadingZeroDigits(); } - const SignedInteger & exponent() const + const SignedInteger& exponent() const { return mB; } - template - T toXIntPositiveRound(DataCondition & error) const + template + T toXIntPositiveRound(DataCondition& error) const { size_t availableDigits = IntFracDigits(); if (!availableDigits) @@ -554,35 +553,36 @@ public: return ExactUnsignedNumericLiteral::toXIntPositiveRoundExp(absexp, exponent().negative(), error); } - template - T toPackedDecimalPositive(scale_t scale, DataCondition & error) const + template + T toPackedDecimalPositive(scale_t scale, DataCondition& error) const { size_t availableDigits = IntFracDigits(); if (!availableDigits) return 0; int64_t exp = exponent().toSInt(error); if (exp <= datatypes::numeric_limits::max() - scale) - exp+= scale; + exp += scale; if (exp < 0) { if (exp == datatypes::numeric_limits::min()) - exp++; // Avoid undefined behaviour in the unary minus below: - return ExactUnsignedNumericLiteral::toXIntPositiveRoundExp((uint64_t) -exp, true, error); + exp++; // Avoid undefined behaviour in the unary minus below: + return ExactUnsignedNumericLiteral::toXIntPositiveRoundExp((uint64_t)-exp, true, error); } - return ExactUnsignedNumericLiteral::toXIntPositiveRoundExp((uint64_t) exp, false, error); + return ExactUnsignedNumericLiteral::toXIntPositiveRoundExp((uint64_t)exp, false, error); } - }; - // ::= [ ] -class SignedNumericLiteral: public Parser::DD2OM +class SignedNumericLiteral : public Parser::DD2OM { -public: + public: using DD2OM::DD2OM; - bool isNull() const { return UnsignedNumericLiteral::isNull(); } + bool isNull() const + { + return UnsignedNumericLiteral::isNull(); + } - template + template T toUIntXRound() const { if (negative()) @@ -590,31 +590,30 @@ public: return UnsignedNumericLiteral::toXIntPositiveRound(); } - template - T toPackedUDecimal(scale_t scale, DataCondition & error) const + template + T toPackedUDecimal(scale_t scale, DataCondition& error) const { if (negative()) return 0; return UnsignedNumericLiteral::toPackedDecimalPositive(scale, error); } - template - T toPackedSDecimal(scale_t scale, DataCondition & error) const + template + T toPackedSDecimal(scale_t scale, DataCondition& error) const { if (!negative()) return UnsignedNumericLiteral::toPackedDecimalPositive(scale, error); typedef typename datatypes::make_unsigned::type UT; UT absval = UnsignedNumericLiteral::toPackedDecimalPositive(scale, error); - if (absval >= (UT) datatypes::numeric_limits::min()) + if (absval >= (UT)datatypes::numeric_limits::min()) { - error|= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; + error |= DataCondition::X_NUMERIC_VALUE_OUT_OF_RANGE; return datatypes::numeric_limits::min(); } - return - (T) absval; + return -(T)absval; } }; +} // namespace literal -} // namespace literal - -#endif // NUMERICLITERAL_H +#endif // NUMERICLITERAL_H diff --git a/dbcon/ddlpackage/altertable.cpp b/dbcon/ddlpackage/altertable.cpp index b90988917..fcf2f8357 100644 --- a/dbcon/ddlpackage/altertable.cpp +++ b/dbcon/ddlpackage/altertable.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: altertable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: altertable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -29,169 +29,139 @@ namespace ddlpackage { using namespace std; -AlterTableStatement::AlterTableStatement(QualifiedName* qName, AlterTableActionList* ataList): - fTableName(qName), - fActions(*ataList) +AlterTableStatement::AlterTableStatement(QualifiedName* qName, AlterTableActionList* ataList) + : fTableName(qName), fActions(*ataList) { - delete ataList; + delete ataList; } AlterTableStatement::~AlterTableStatement() { - delete fTableName; - AlterTableActionList::iterator itr; + delete fTableName; + AlterTableActionList::iterator itr; - for (itr = fActions.begin(); itr != fActions.end(); ++itr) - { - delete *itr; - } + for (itr = fActions.begin(); itr != fActions.end(); ++itr) + { + delete *itr; + } } std::ostream& AlterTableStatement::put(std::ostream& os) const { - AlterTableActionList::const_iterator itr; - os << "Alter Table " << *fTableName << endl; + AlterTableActionList::const_iterator itr; + os << "Alter Table " << *fTableName << endl; - for (itr = fActions.begin(); itr != fActions.end(); ++itr) - { - os << **itr << endl; - } + for (itr = fActions.begin(); itr != fActions.end(); ++itr) + { + os << **itr << endl; + } - return os; + return os; } - - /** @brief Format to ostream. Diagnostic. */ std::ostream& AlterTableAction::put(std::ostream& os) const { - os << "AlterTableAction put stub"; - return os; + os << "AlterTableAction put stub"; + return os; } /** @brief Invokes the virtual function put, to dispatch to subclass - ostream writers. */ + ostream writers. */ std::ostream& operator<<(std::ostream& os, const AlterTableAction& ata) { - return ata.put(os); + return ata.put(os); } - AtaAddColumn::~AtaAddColumn() { - delete fColumnDef; + delete fColumnDef; } - /** @brief ostream output */ std::ostream& AtaAddColumn::put(std::ostream& os) const { - os << "Add Column" << endl; - os << *fColumnDef << endl; - return os; + os << "Add Column" << endl; + os << *fColumnDef << endl; + return os; } - - -AtaDropColumn::AtaDropColumn(std::string columnName, DDL_REFERENTIAL_ACTION dropBehavior) : - fColumnName(columnName), - fDropBehavior(dropBehavior) +AtaDropColumn::AtaDropColumn(std::string columnName, DDL_REFERENTIAL_ACTION dropBehavior) + : fColumnName(columnName), fDropBehavior(dropBehavior) { } std::ostream& AtaDropColumn::put(std::ostream& os) const { - os << "Drop Column: " << fColumnName << " " - << ReferentialActionStrings[fDropBehavior]; - return os; + os << "Drop Column: " << fColumnName << " " << ReferentialActionStrings[fDropBehavior]; + return os; } - AtaModifyColumnType::~AtaModifyColumnType() { - delete fColumnType; + delete fColumnType; } std::ostream& AtaModifyColumnType::put(std::ostream& os) const { - os << "Modify column type: " << fName << " " << *fColumnType; + os << "Modify column type: " << fName << " " << *fColumnType; - return os; + return os; } - - AtaRenameColumn::~AtaRenameColumn() { - delete fNewType; + delete fNewType; } std::ostream& AtaRenameColumn::put(std::ostream& os) const { - os << "Rename Column: " << fName << " -> " << fNewName << " (" << *fNewType << ')'; - return os; + os << "Rename Column: " << fName << " -> " << fNewName << " (" << *fNewType << ')'; + return os; } - - -AtaSetColumnDefault::AtaSetColumnDefault(const char* colName, ColumnDefaultValue* defaultValue) : - fColumnName(colName), - fDefaultValue(defaultValue) +AtaSetColumnDefault::AtaSetColumnDefault(const char* colName, ColumnDefaultValue* defaultValue) + : fColumnName(colName), fDefaultValue(defaultValue) { } - AtaSetColumnDefault::~AtaSetColumnDefault() { - delete fDefaultValue; + delete fDefaultValue; } - std::ostream& AtaSetColumnDefault::put(std::ostream& os) const { - os << "Set Column Default: " << fColumnName << " " - << *fDefaultValue << endl; - return os; + os << "Set Column Default: " << fColumnName << " " << *fDefaultValue << endl; + return os; } - - - -AtaDropColumnDefault::AtaDropColumnDefault(const char* colName) : - fColumnName(colName) +AtaDropColumnDefault::AtaDropColumnDefault(const char* colName) : fColumnName(colName) { } std::ostream& AtaDropColumnDefault::put(std::ostream& os) const { - os << "Drop Column Default: " << fColumnName << " "; - return os; + os << "Drop Column Default: " << fColumnName << " "; + return os; } - - - -AtaRenameTable::AtaRenameTable(QualifiedName* qualifiedName) : - fQualifiedName(qualifiedName) +AtaRenameTable::AtaRenameTable(QualifiedName* qualifiedName) : fQualifiedName(qualifiedName) { } AtaRenameTable::~AtaRenameTable() { - delete fQualifiedName; + delete fQualifiedName; } - std::ostream& AtaRenameTable::put(std::ostream& os) const { - os << "Rename Table: " << *fQualifiedName << endl; - return os; + os << "Rename Table: " << *fQualifiedName << endl; + return os; } - - -AtaTableComment::AtaTableComment(const char* tableComment) : - fTableComment(tableComment) +AtaTableComment::AtaTableComment(const char* tableComment) : fTableComment(tableComment) { } @@ -199,61 +169,54 @@ AtaTableComment::~AtaTableComment() { } - std::ostream& AtaTableComment::put(std::ostream& os) const { - os << "TableComment: " << fTableComment << endl; - return os; + os << "TableComment: " << fTableComment << endl; + return os; } - AtaAddColumns::~AtaAddColumns() { - ColumnDefList::iterator itr; + ColumnDefList::iterator itr; - for (itr = fColumns.begin(); itr != fColumns.end(); itr++) - delete *itr; + for (itr = fColumns.begin(); itr != fColumns.end(); itr++) + delete *itr; } - AtaAddColumns::AtaAddColumns(TableElementList* tableElements) { - /* It is convenient to reuse the grammar rules for - table_element_list, and we do. So, it is possible for - there to be errant table constraint defs in the input list. - We ignore them. That is all we are doing here. - */ - ColumnDef* column; - TableElementList::const_iterator itr; + /* It is convenient to reuse the grammar rules for + table_element_list, and we do. So, it is possible for + there to be errant table constraint defs in the input list. + We ignore them. That is all we are doing here. + */ + ColumnDef* column; + TableElementList::const_iterator itr; - for (itr = tableElements->begin(); - itr != tableElements->end(); - ++itr) + for (itr = tableElements->begin(); itr != tableElements->end(); ++itr) + { + column = dynamic_cast(*itr); + + if (0 != column) { - column = dynamic_cast(*itr); - - if (0 != column) - { - fColumns.push_back(column); - } + fColumns.push_back(column); } + } - delete tableElements; + delete tableElements; } std::ostream& AtaAddColumns::put(std::ostream& os) const { - os << "Add Columns: " << endl; - ColumnDefList::const_iterator itr; + os << "Add Columns: " << endl; + ColumnDefList::const_iterator itr; - for (itr = fColumns.begin(); - itr != fColumns.end(); - ++itr) - { - os << **itr << endl; - } + for (itr = fColumns.begin(); itr != fColumns.end(); ++itr) + { + os << **itr << endl; + } - return os; + return os; } ////////////////////////// @@ -261,63 +224,52 @@ AtaDropColumns::~AtaDropColumns() { } - AtaDropColumns::AtaDropColumns(ColumnNameList* columnNames) { - fColumns = *columnNames; - delete columnNames; + fColumns = *columnNames; + delete columnNames; } std::ostream& AtaDropColumns::put(std::ostream& os) const { - os << "Drop Columns: " << endl; - ColumnNameList::const_iterator itr; + os << "Drop Columns: " << endl; + ColumnNameList::const_iterator itr; - for (itr = fColumns.begin(); - itr != fColumns.end(); - ++itr) - { - os << *itr << endl; - } + for (itr = fColumns.begin(); itr != fColumns.end(); ++itr) + { + os << *itr << endl; + } - return os; + return os; } - - - -AtaAddTableConstraint::AtaAddTableConstraint(TableConstraintDef* tableConstraint) : - fTableConstraint(tableConstraint) +AtaAddTableConstraint::AtaAddTableConstraint(TableConstraintDef* tableConstraint) + : fTableConstraint(tableConstraint) { } AtaAddTableConstraint::~AtaAddTableConstraint() { - delete fTableConstraint; + delete fTableConstraint; } - std::ostream& AtaAddTableConstraint::put(std::ostream& os) const { - os << "Add Table Constraint:" << endl; - os << *fTableConstraint << endl; - return os; + os << "Add Table Constraint:" << endl; + os << *fTableConstraint << endl; + return os; } - - - -AtaDropTableConstraint::AtaDropTableConstraint -(const char* constraintName, DDL_REFERENTIAL_ACTION dropBehavior) : - fConstraintName(constraintName), - fDropBehavior(dropBehavior) +AtaDropTableConstraint::AtaDropTableConstraint(const char* constraintName, + DDL_REFERENTIAL_ACTION dropBehavior) + : fConstraintName(constraintName), fDropBehavior(dropBehavior) { } std::ostream& AtaDropTableConstraint::put(std::ostream& os) const { - os << "Drop Table Constraint: " << fConstraintName; - return os; + os << "Drop Table Constraint: " << fConstraintName; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/columndef.cpp b/dbcon/ddlpackage/columndef.cpp index 5c65e4a68..6174d0377 100644 --- a/dbcon/ddlpackage/columndef.cpp +++ b/dbcon/ddlpackage/columndef.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: columndef.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: columndef.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -30,131 +30,113 @@ namespace ddlpackage { - using namespace std; ColumnDef::~ColumnDef() { - delete fType; - delete fDefaultValue; - ColumnConstraintList::iterator itr; + delete fType; + delete fDefaultValue; + ColumnConstraintList::iterator itr; - for (itr = fConstraints.begin(); itr != fConstraints.end(); ++itr) - { - delete *itr; - } + for (itr = fConstraints.begin(); itr != fConstraints.end(); ++itr) + { + delete *itr; + } } ColumnDef::ColumnDef(const char* name, ColumnType* columnType, ColumnConstraintList* constraints, - ColumnDefaultValue* defaultValue, const char* comment ) : - SchemaObject(name), - fType(columnType), - fDefaultValue(defaultValue) + ColumnDefaultValue* defaultValue, const char* comment) + : SchemaObject(name), fType(columnType), fDefaultValue(defaultValue) { - if (constraints) - { - fConstraints = *constraints; - delete constraints; - } + if (constraints) + { + fConstraints = *constraints; + delete constraints; + } - if ( comment ) - fComment = comment; + if (comment) + fComment = comment; } - ostream& operator<<(ostream& os, const ColumnType& columnType) { - os << setw(12) << left << DDLDatatypeString[columnType.fType] - << "[" - << "L=" << setw(2) << columnType.fLength << "," - << "P=" << setw(2) << columnType.fPrecision << "," - << "S=" << setw(2) << columnType.fScale << "," - << "T=" << setw(2) << columnType.fWithTimezone - << "]"; - return os; + os << setw(12) << left << DDLDatatypeString[columnType.fType] << "[" + << "L=" << setw(2) << columnType.fLength << "," + << "P=" << setw(2) << columnType.fPrecision << "," + << "S=" << setw(2) << columnType.fScale << "," + << "T=" << setw(2) << columnType.fWithTimezone << "]"; + return os; } - ostream& operator<<(ostream& os, const ColumnDef& column) { - os << "Column: " << column.fName << " " << *column.fType; + os << "Column: " << column.fName << " " << *column.fType; - if (column.fDefaultValue) - { - os << " def="; + if (column.fDefaultValue) + { + os << " def="; - if (column.fDefaultValue->fNull) - os << "NULL"; - else - os << column.fDefaultValue->fValue; - } + if (column.fDefaultValue->fNull) + os << "NULL"; + else + os << column.fDefaultValue->fValue; + } - os << endl << " " << column.fConstraints.size() - << " constraints "; + os << endl << " " << column.fConstraints.size() << " constraints "; - ColumnConstraintList::const_iterator itr; + ColumnConstraintList::const_iterator itr; - for (itr = column.fConstraints.begin(); - itr != column.fConstraints.end(); - ++itr) - { - ColumnConstraintDef* con = *itr; - os << *con; - } + for (itr = column.fConstraints.begin(); itr != column.fConstraints.end(); ++itr) + { + ColumnConstraintDef* con = *itr; + os << *con; + } - - - return os; + return os; } - ostream& operator<<(ostream& os, const ColumnConstraintDef& con) { - os << " Constraint: " - << con.fName << " " - << ConstraintString[con.fConstraintType] << " " - << "defer=" << con.fDeferrable << " " - << ConstraintAttrStrings[con.fCheckTime] << " "; + os << " Constraint: " << con.fName << " " << ConstraintString[con.fConstraintType] << " " + << "defer=" << con.fDeferrable << " " << ConstraintAttrStrings[con.fCheckTime] << " "; - if (!con.fCheck.empty()) - os << "check=" << "\"" << con.fCheck << "\""; + if (!con.fCheck.empty()) + os << "check=" + << "\"" << con.fCheck << "\""; - return os; + return os; } std::ostream& operator<<(std::ostream& os, const ColumnDefList& clist) { - ColumnDefList::const_iterator itr; + ColumnDefList::const_iterator itr; - for (itr = clist.begin(); itr != clist.end(); ++itr) - { - os << **itr; - } + for (itr = clist.begin(); itr != clist.end(); ++itr) + { + os << **itr; + } - return os; + return os; } - -ColumnDefaultValue::ColumnDefaultValue(const char* value) : - fNull(false) +ColumnDefaultValue::ColumnDefaultValue(const char* value) : fNull(false) { - if (0 == value) - fNull = true; - else - fValue = value; + if (0 == value) + fNull = true; + else + fValue = value; } - std::ostream& operator<<(std::ostream& os, const ColumnDefaultValue& defaultValue) { - os << " def="; + os << " def="; - if (defaultValue.fNull) - os << "NULL"; - else - os << defaultValue.fValue; + if (defaultValue.fNull) + os << "NULL"; + else + os << defaultValue.fValue; - return os; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/createindex.cpp b/dbcon/ddlpackage/createindex.cpp index 0881c7939..6a15eea0d 100644 --- a/dbcon/ddlpackage/createindex.cpp +++ b/dbcon/ddlpackage/createindex.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: createindex.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: createindex.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "ddlpkg.h" @@ -27,35 +27,28 @@ namespace ddlpackage { using namespace std; -CreateIndexStatement::CreateIndexStatement(): - fIndexName(NULL), - fTableName(NULL), - fColumnNames(), - fUnique(false) +CreateIndexStatement::CreateIndexStatement() + : fIndexName(NULL), fTableName(NULL), fColumnNames(), fUnique(false) { } CreateIndexStatement::CreateIndexStatement(QualifiedName* indexName, QualifiedName* tableName, - ColumnNameList* columnNames, bool unique) : - fIndexName(indexName), - fTableName(tableName), - fColumnNames(*columnNames), - fUnique(unique) + ColumnNameList* columnNames, bool unique) + : fIndexName(indexName), fTableName(tableName), fColumnNames(*columnNames), fUnique(unique) { - delete columnNames; + delete columnNames; } CreateIndexStatement::~CreateIndexStatement() { - delete fIndexName; - delete fTableName; + delete fIndexName; + delete fTableName; } std::ostream& CreateIndexStatement::put(std::ostream& os) const { - os << "Create Index: " << *fIndexName << " on " << *fTableName - << fColumnNames << endl; - return os; + os << "Create Index: " << *fIndexName << " on " << *fTableName << fColumnNames << endl; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/createtable.cpp b/dbcon/ddlpackage/createtable.cpp index 2ffd167dd..2d0d84512 100644 --- a/dbcon/ddlpackage/createtable.cpp +++ b/dbcon/ddlpackage/createtable.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: createtable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: createtable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include @@ -29,33 +29,28 @@ namespace ddlpackage { - using namespace std; -CreateTableStatement::CreateTableStatement() : - fTableDef(0) +CreateTableStatement::CreateTableStatement() : fTableDef(0) { } -CreateTableStatement::CreateTableStatement(TableDef* tableDef) : - fTableDef(tableDef) +CreateTableStatement::CreateTableStatement(TableDef* tableDef) : fTableDef(tableDef) { } - CreateTableStatement::~CreateTableStatement() { - if (fTableDef) - { - delete fTableDef; - } + if (fTableDef) + { + delete fTableDef; + } } /** \brief Put to ostream. */ ostream& CreateTableStatement::put(ostream& os) const { - os << "CreateTable " - << *fTableDef; - return os; -} + os << "CreateTable " << *fTableDef; + return os; } +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/ddl-gram-win.cpp b/dbcon/ddlpackage/ddl-gram-win.cpp index cf16912bd..e66b2d724 100644 --- a/dbcon/ddlpackage/ddl-gram-win.cpp +++ b/dbcon/ddlpackage/ddl-gram-win.cpp @@ -64,18 +64,16 @@ #define YYLSP_NEEDED 0 /* Substitute the variable and function names. */ -#define yyparse ddlparse -#define yylex ddllex -#define yyerror ddlerror -#define yylval ddllval -#define yychar ddlchar -#define yydebug ddldebug -#define yynerrs ddlnerrs - +#define yyparse ddlparse +#define yylex ddllex +#define yyerror ddlerror +#define yylval ddllval +#define yychar ddlchar +#define yydebug ddldebug +#define yynerrs ddlnerrs /* Copy the first part of user declarations. */ - #include "sqlparser.h" #ifdef _MSC_VER @@ -91,176 +89,161 @@ using namespace ddlpackage; static ParseTree* parseTree; static std::string db_schema; int ddllex(); -void ddlerror (char const* error); +void ddlerror(char const* error); char* copy_string(const char* str); - - /* Enabling traces. */ #ifndef YYDEBUG -# define YYDEBUG 1 +#define YYDEBUG 1 #endif /* Enabling verbose error messages. */ #ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 +#undef YYERROR_VERBOSE +#define YYERROR_VERBOSE 1 #else -# define YYERROR_VERBOSE 0 +#define YYERROR_VERBOSE 0 #endif /* Enabling the token table. */ #ifndef YYTOKEN_TABLE -# define YYTOKEN_TABLE 0 +#define YYTOKEN_TABLE 0 #endif - /* Tokens. */ #ifndef YYTOKENTYPE -# define YYTOKENTYPE +#define YYTOKENTYPE /* Put the tokens into the symbol table, so that GDB and other debuggers know about them. */ enum yytokentype { - ACTION = 258, - ADD = 259, - ALTER = 260, - AUTO_INCREMENT = 261, - BIGINT = 262, - BIT = 263, - IDB_BLOB = 264, - CASCADE = 265, - IDB_CHAR = 266, - CHARACTER = 267, - CHECK = 268, - CLOB = 269, - COLUMN = 270, - COLUMNS = 271, - COMMENT = 272, - CONSTRAINT = 273, - CONSTRAINTS = 274, - CREATE = 275, - CURRENT_USER = 276, - DATETIME = 277, - DEC = 278, - DECIMAL = 279, - DEFAULT = 280, - DEFERRABLE = 281, - DEFERRED = 282, - IDB_DELETE = 283, - DROP = 284, - ENGINE = 285, - FOREIGN = 286, - FULL = 287, - IMMEDIATE = 288, - INDEX = 289, - INITIALLY = 290, - IDB_INT = 291, - INTEGER = 292, - KEY = 293, - MATCH = 294, - MAX_ROWS = 295, - MIN_ROWS = 296, - MODIFY = 297, - NO = 298, - NOT = 299, - NULL_TOK = 300, - NUMBER = 301, - NUMERIC = 302, - ON = 303, - PARTIAL = 304, - PRECISION = 305, - PRIMARY = 306, - REFERENCES = 307, - RENAME = 308, - RESTRICT = 309, - SET = 310, - SMALLINT = 311, - TABLE = 312, - TIME = 313, - TINYINT = 314, - TO = 315, - UNIQUE = 316, - UNSIGNED = 317, - UPDATE = 318, - USER = 319, - SESSION_USER = 320, - SYSTEM_USER = 321, - VARCHAR = 322, - VARBINARY = 323, - VARYING = 324, - WITH = 325, - ZONE = 326, - DOUBLE = 327, - IDB_FLOAT = 328, - REAL = 329, - CHARSET = 330, - IDB_IF = 331, - EXISTS = 332, - CHANGE = 333, - TRUNCATE = 334, - IDENT = 335, - FCONST = 336, - SCONST = 337, - CP_SEARCH_CONDITION_TEXT = 338, - ICONST = 339, - DATE = 340 + ACTION = 258, + ADD = 259, + ALTER = 260, + AUTO_INCREMENT = 261, + BIGINT = 262, + BIT = 263, + IDB_BLOB = 264, + CASCADE = 265, + IDB_CHAR = 266, + CHARACTER = 267, + CHECK = 268, + CLOB = 269, + COLUMN = 270, + COLUMNS = 271, + COMMENT = 272, + CONSTRAINT = 273, + CONSTRAINTS = 274, + CREATE = 275, + CURRENT_USER = 276, + DATETIME = 277, + DEC = 278, + DECIMAL = 279, + DEFAULT = 280, + DEFERRABLE = 281, + DEFERRED = 282, + IDB_DELETE = 283, + DROP = 284, + ENGINE = 285, + FOREIGN = 286, + FULL = 287, + IMMEDIATE = 288, + INDEX = 289, + INITIALLY = 290, + IDB_INT = 291, + INTEGER = 292, + KEY = 293, + MATCH = 294, + MAX_ROWS = 295, + MIN_ROWS = 296, + MODIFY = 297, + NO = 298, + NOT = 299, + NULL_TOK = 300, + NUMBER = 301, + NUMERIC = 302, + ON = 303, + PARTIAL = 304, + PRECISION = 305, + PRIMARY = 306, + REFERENCES = 307, + RENAME = 308, + RESTRICT = 309, + SET = 310, + SMALLINT = 311, + TABLE = 312, + TIME = 313, + TINYINT = 314, + TO = 315, + UNIQUE = 316, + UNSIGNED = 317, + UPDATE = 318, + USER = 319, + SESSION_USER = 320, + SYSTEM_USER = 321, + VARCHAR = 322, + VARBINARY = 323, + VARYING = 324, + WITH = 325, + ZONE = 326, + DOUBLE = 327, + IDB_FLOAT = 328, + REAL = 329, + CHARSET = 330, + IDB_IF = 331, + EXISTS = 332, + CHANGE = 333, + TRUNCATE = 334, + IDENT = 335, + FCONST = 336, + SCONST = 337, + CP_SEARCH_CONDITION_TEXT = 338, + ICONST = 339, + DATE = 340 }; #endif - - -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED typedef union YYSTYPE { - - - ddlpackage::AlterTableStatement* alterTableStmt; - ddlpackage::AlterTableAction* ata; - ddlpackage::AlterTableActionList* ataList; - ddlpackage::DDL_CONSTRAINT_ATTRIBUTES cattr; - std::pair* tableOption; - const char* columnOption; - ddlpackage::ColumnConstraintDef* columnConstraintDef; - ddlpackage::ColumnNameList* columnNameList; - ddlpackage::ColumnType* columnType; - ddlpackage::ConstraintAttributes* constraintAttributes; - ddlpackage::ColumnConstraintList* constraintList; - ddlpackage::DDL_CONSTRAINTS constraintType; - double dval; - bool flag; - int ival; - ddlpackage::QualifiedName* qualifiedName; - ddlpackage::SchemaObject* schemaObject; - ddlpackage::SqlStatement* sqlStmt; - ddlpackage::SqlStatementList* sqlStmtList; - const char* str; - ddlpackage::TableConstraintDef* tableConstraint; - ddlpackage::TableElementList* tableElementList; - ddlpackage::TableOptionMap* tableOptionMap; - ddlpackage::ColumnDefaultValue* colDefault; - ddlpackage::DDL_MATCH_TYPE matchType; - ddlpackage::DDL_REFERENTIAL_ACTION refActionCode; - ddlpackage::ReferentialAction* refAction; - - + ddlpackage::AlterTableStatement* alterTableStmt; + ddlpackage::AlterTableAction* ata; + ddlpackage::AlterTableActionList* ataList; + ddlpackage::DDL_CONSTRAINT_ATTRIBUTES cattr; + std::pair* tableOption; + const char* columnOption; + ddlpackage::ColumnConstraintDef* columnConstraintDef; + ddlpackage::ColumnNameList* columnNameList; + ddlpackage::ColumnType* columnType; + ddlpackage::ConstraintAttributes* constraintAttributes; + ddlpackage::ColumnConstraintList* constraintList; + ddlpackage::DDL_CONSTRAINTS constraintType; + double dval; + bool flag; + int ival; + ddlpackage::QualifiedName* qualifiedName; + ddlpackage::SchemaObject* schemaObject; + ddlpackage::SqlStatement* sqlStmt; + ddlpackage::SqlStatementList* sqlStmtList; + const char* str; + ddlpackage::TableConstraintDef* tableConstraint; + ddlpackage::TableElementList* tableElementList; + ddlpackage::TableOptionMap* tableOptionMap; + ddlpackage::ColumnDefaultValue* colDefault; + ddlpackage::DDL_MATCH_TYPE matchType; + ddlpackage::DDL_REFERENTIAL_ACTION refActionCode; + ddlpackage::ReferentialAction* refAction; } YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 +#define YYSTYPE_IS_TRIVIAL 1 +#define yystype YYSTYPE /* obsolescent; will be withdrawn */ +#define YYSTYPE_IS_DECLARED 1 #endif - /* Copy the second part of user declarations. */ - - - - - #ifdef short -# undef short +#undef short #endif #ifdef YYTYPE_UINT8 @@ -271,8 +254,7 @@ typedef unsigned char yytype_uint8; #ifdef YYTYPE_INT8 typedef YYTYPE_INT8 yytype_int8; -#elif (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) +#elif (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) typedef signed char yytype_int8; #else typedef short int yytype_int8; @@ -291,810 +273,674 @@ typedef short int yytype_int16; #endif #ifndef YYSIZE_T -# ifdef __SIZE_TYPE__ -# define YYSIZE_T __SIZE_TYPE__ -# elif defined size_t -# define YYSIZE_T size_t -# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# define YYSIZE_T size_t -# else -# define YYSIZE_T unsigned int -# endif +#ifdef __SIZE_TYPE__ +#define YYSIZE_T __SIZE_TYPE__ +#elif defined size_t +#define YYSIZE_T size_t +#elif !defined YYSIZE_T && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +#include /* INFRINGES ON USER NAME SPACE */ +#define YYSIZE_T size_t +#else +#define YYSIZE_T unsigned int +#endif #endif -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) +#define YYSIZE_MAXIMUM ((YYSIZE_T)-1) #ifndef YY_ -# if YYENABLE_NLS -# if ENABLE_NLS -# include /* INFRINGES ON USER NAME SPACE */ -# define YY_(msgid) dgettext ("bison-runtime", msgid) -# endif -# endif -# ifndef YY_ -# define YY_(msgid) msgid -# endif +#if YYENABLE_NLS +#if ENABLE_NLS +#include /* INFRINGES ON USER NAME SPACE */ +#define YY_(msgid) dgettext("bison-runtime", msgid) +#endif +#endif +#ifndef YY_ +#define YY_(msgid) msgid +#endif #endif /* Suppress unused-variable warnings by "using" E. */ -#if ! defined lint || defined __GNUC__ -# define YYUSE(e) ((void) (e)) +#if !defined lint || defined __GNUC__ +#define YYUSE(e) ((void)(e)) #else -# define YYUSE(e) /* empty */ +#define YYUSE(e) /* empty */ #endif /* Identity function, used to suppress warnings about constant conditions. */ #ifndef lint -# define YYID(n) (n) +#define YYID(n) (n) #else -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static int -YYID (int yyi) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static int YYID(int yyi) #else -static int -YYID (yyi) -int yyi; +static int YYID(yyi) int yyi; #endif { - return yyi; + return yyi; } #endif -#if ! defined yyoverflow || YYERROR_VERBOSE +#if !defined yyoverflow || YYERROR_VERBOSE /* The parser invokes alloca or malloc; define the necessary symbols. */ -# ifdef YYSTACK_USE_ALLOCA -# if YYSTACK_USE_ALLOCA -# ifdef __GNUC__ -# define YYSTACK_ALLOC __builtin_alloca -# elif defined __BUILTIN_VA_ARG_INCR -# include /* INFRINGES ON USER NAME SPACE */ -# elif defined _AIX -# define YYSTACK_ALLOC __alloca -# elif defined _MSC_VER -# include /* INFRINGES ON USER NAME SPACE */ -# define alloca _alloca -# else -# define YYSTACK_ALLOC alloca -# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# endif -# endif -# endif +#ifdef YYSTACK_USE_ALLOCA +#if YYSTACK_USE_ALLOCA +#ifdef __GNUC__ +#define YYSTACK_ALLOC __builtin_alloca +#elif defined __BUILTIN_VA_ARG_INCR +#include /* INFRINGES ON USER NAME SPACE */ +#elif defined _AIX +#define YYSTACK_ALLOC __alloca +#elif defined _MSC_VER +#include /* INFRINGES ON USER NAME SPACE */ +#define alloca _alloca +#else +#define YYSTACK_ALLOC alloca +#if !defined _ALLOCA_H && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +#include /* INFRINGES ON USER NAME SPACE */ +#ifndef _STDLIB_H +#define _STDLIB_H 1 +#endif +#endif +#endif +#endif +#endif -# ifdef YYSTACK_ALLOC +#ifdef YYSTACK_ALLOC /* Pacify GCC's `empty if-body' warning. */ -# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0)) -# ifndef YYSTACK_ALLOC_MAXIMUM +#define YYSTACK_FREE(Ptr) \ + do \ + { /* empty */ \ + ; \ + } while (YYID(0)) +#ifndef YYSTACK_ALLOC_MAXIMUM /* The OS might guarantee only one guard page at the bottom of the stack, and a page size can be as small as 4096 bytes. So we cannot safely invoke alloca (N) if N exceeds 4096. Use a slightly smaller number to allow for a few compiler-allocated temporary stack slots. */ -# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ -# endif -# else -# define YYSTACK_ALLOC YYMALLOC -# define YYSTACK_FREE YYFREE -# ifndef YYSTACK_ALLOC_MAXIMUM -# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM -# endif -# if (defined __cplusplus && ! defined _STDLIB_H \ - && ! ((defined YYMALLOC || defined malloc) \ - && (defined YYFREE || defined free))) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# ifndef YYMALLOC -# define YYMALLOC malloc -# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void* malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# ifndef YYFREE -# define YYFREE free -# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void free (void*); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# endif +#define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +#endif +#else +#define YYSTACK_ALLOC YYMALLOC +#define YYSTACK_FREE YYFREE +#ifndef YYSTACK_ALLOC_MAXIMUM +#define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +#endif +#if (defined __cplusplus && !defined _STDLIB_H && \ + !((defined YYMALLOC || defined malloc) && (defined YYFREE || defined free))) +#include /* INFRINGES ON USER NAME SPACE */ +#ifndef _STDLIB_H +#define _STDLIB_H 1 +#endif +#endif +#ifndef YYMALLOC +#define YYMALLOC malloc +#if !defined malloc && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +void* malloc(YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +#endif +#endif +#ifndef YYFREE +#define YYFREE free +#if !defined free && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +void free(void*); /* INFRINGES ON USER NAME SPACE */ +#endif +#endif +#endif #endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - -#if (! defined yyoverflow \ - && (! defined __cplusplus \ - || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) +#if (!defined yyoverflow && (!defined __cplusplus || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) /* A type that is properly aligned for any stack member. */ union yyalloc { - yytype_int16 yyss_alloc; - YYSTYPE yyvs_alloc; + yytype_int16 yyss_alloc; + YYSTYPE yyvs_alloc; }; /* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) +#define YYSTACK_GAP_MAXIMUM (sizeof(union yyalloc) - 1) /* The size of an array large to enough to hold all stacks, each with N elements. */ -# define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ - + YYSTACK_GAP_MAXIMUM) +#define YYSTACK_BYTES(N) ((N) * (sizeof(yytype_int16) + sizeof(YYSTYPE)) + YYSTACK_GAP_MAXIMUM) /* Copy COUNT objects from FROM to TO. The source and destination do not overlap. */ -# ifndef YYCOPY -# if defined __GNUC__ && 1 < __GNUC__ -# define YYCOPY(To, From, Count) \ - __builtin_memcpy (To, From, (Count) * sizeof (*(From))) -# else -# define YYCOPY(To, From, Count) \ - do \ - { \ - YYSIZE_T yyi; \ - for (yyi = 0; yyi < (Count); yyi++) \ - (To)[yyi] = (From)[yyi]; \ - } \ - while (YYID (0)) -# endif -# endif +#ifndef YYCOPY +#if defined __GNUC__ && 1 < __GNUC__ +#define YYCOPY(To, From, Count) __builtin_memcpy(To, From, (Count) * sizeof(*(From))) +#else +#define YYCOPY(To, From, Count) \ + do \ + { \ + YYSIZE_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (To)[yyi] = (From)[yyi]; \ + } while (YYID(0)) +#endif +#endif /* Relocate STACK from its old location to the new one. The local variables YYSIZE and YYSTACKSIZE give the old and new number of elements in the stack, and YYPTR gives the new location of the stack. Advance YYPTR to a properly aligned location for the next stack. */ -# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ - do \ - { \ - YYSIZE_T yynewbytes; \ - YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ - Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ - } \ - while (YYID (0)) +#define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYSIZE_T yynewbytes; \ + YYCOPY(&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * sizeof(*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / sizeof(*yyptr); \ + } while (YYID(0)) #endif /* YYFINAL -- State number of the termination state. */ -#define YYFINAL 23 +#define YYFINAL 23 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 387 +#define YYLAST 387 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 93 +#define YYNTOKENS 93 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 76 +#define YYNNTS 76 /* YYNRULES -- Number of rules. */ -#define YYNRULES 229 +#define YYNRULES 229 /* YYNRULES -- Number of states. */ -#define YYNSTATES 396 +#define YYNSTATES 396 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ -#define YYUNDEFTOK 2 -#define YYMAXUTOK 340 +#define YYUNDEFTOK 2 +#define YYMAXUTOK 340 -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) +#define YYTRANSLATE(YYX) ((unsigned int)(YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) /* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ -static const yytype_uint8 yytranslate[] = -{ - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 92, - 87, 88, 2, 2, 89, 2, 91, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 86, - 2, 90, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, - 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, - 85 -}; +static const yytype_uint8 yytranslate[] = { + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 92, 87, 88, 2, 2, 89, 2, 91, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 86, 2, 90, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85}; #if YYDEBUG /* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in YYRHS. */ -static const yytype_uint16 yyprhs[] = -{ - 0, 0, 3, 5, 9, 11, 13, 15, 17, 19, - 21, 23, 24, 29, 36, 39, 40, 44, 54, 65, - 67, 68, 77, 85, 89, 90, 94, 97, 99, 103, - 105, 107, 112, 116, 118, 119, 121, 123, 125, 130, - 132, 136, 139, 141, 154, 156, 157, 160, 162, 164, - 166, 167, 170, 173, 175, 176, 178, 179, 183, 187, - 189, 192, 195, 198, 200, 202, 205, 209, 213, 217, - 221, 224, 228, 233, 239, 244, 251, 253, 257, 260, - 262, 264, 266, 268, 270, 272, 274, 276, 280, 285, - 290, 296, 302, 309, 315, 322, 329, 337, 343, 350, - 357, 365, 372, 380, 387, 395, 403, 412, 420, 429, - 434, 437, 440, 444, 446, 450, 452, 455, 459, 464, - 470, 472, 474, 476, 479, 483, 488, 494, 499, 504, - 510, 517, 523, 530, 536, 537, 539, 542, 545, 548, - 551, 554, 557, 559, 561, 563, 565, 567, 569, 571, - 574, 577, 582, 584, 585, 588, 591, 593, 594, 596, - 599, 602, 605, 607, 610, 612, 614, 619, 623, 625, - 627, 632, 637, 643, 649, 654, 659, 661, 663, 666, - 670, 673, 677, 680, 684, 687, 690, 693, 696, 699, - 703, 707, 711, 715, 719, 723, 729, 730, 734, 735, - 738, 742, 745, 749, 752, 756, 760, 766, 767, 769, - 771, 773, 775, 777, 781, 785, 786, 790, 791, 795, - 800, 806, 811, 817, 819, 821, 822, 828, 834, 836 -}; +static const yytype_uint16 yyprhs[] = { + 0, 0, 3, 5, 9, 11, 13, 15, 17, 19, 21, 23, 24, 29, 36, 39, 40, 44, 54, 65, 67, + 68, 77, 85, 89, 90, 94, 97, 99, 103, 105, 107, 112, 116, 118, 119, 121, 123, 125, 130, 132, 136, + 139, 141, 154, 156, 157, 160, 162, 164, 166, 167, 170, 173, 175, 176, 178, 179, 183, 187, 189, 192, 195, + 198, 200, 202, 205, 209, 213, 217, 221, 224, 228, 233, 239, 244, 251, 253, 257, 260, 262, 264, 266, 268, + 270, 272, 274, 276, 280, 285, 290, 296, 302, 309, 315, 322, 329, 337, 343, 350, 357, 365, 372, 380, 387, + 395, 403, 412, 420, 429, 434, 437, 440, 444, 446, 450, 452, 455, 459, 464, 470, 472, 474, 476, 479, 483, + 488, 494, 499, 504, 510, 517, 523, 530, 536, 537, 539, 542, 545, 548, 551, 554, 557, 559, 561, 563, 565, + 567, 569, 571, 574, 577, 582, 584, 585, 588, 591, 593, 594, 596, 599, 602, 605, 607, 610, 612, 614, 619, + 623, 625, 627, 632, 637, 643, 649, 654, 659, 661, 663, 666, 670, 673, 677, 680, 684, 687, 690, 693, 696, + 699, 703, 707, 711, 715, 719, 723, 729, 730, 734, 735, 738, 742, 745, 749, 752, 756, 760, 766, 767, 769, + 771, 773, 775, 777, 781, 785, 786, 790, 791, 795, 800, 806, 811, 817, 819, 821, 822, 828, 834, 836}; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ -static const yytype_int16 yyrhs[] = -{ - 94, 0, -1, 95, -1, 95, 86, 96, -1, 96, - -1, 125, -1, 102, -1, 99, -1, 97, -1, 100, - -1, 104, -1, -1, 29, 57, 98, 134, -1, 29, - 57, 98, 134, 10, 19, -1, 76, 77, -1, -1, - 29, 34, 134, -1, 20, 34, 134, 48, 134, 87, - 111, 88, 101, -1, 20, 61, 34, 134, 48, 134, - 87, 111, 88, 101, -1, 123, -1, -1, 20, 57, - 103, 133, 87, 105, 88, 123, -1, 20, 57, 103, - 133, 87, 105, 88, -1, 76, 44, 77, -1, -1, - 79, 57, 134, -1, 79, 134, -1, 106, -1, 105, - 89, 106, -1, 139, -1, 107, -1, 18, 108, 109, - 145, -1, 108, 109, 145, -1, 137, -1, -1, 110, - -1, 113, -1, 151, -1, 112, 87, 111, 88, -1, - 136, -1, 111, 89, 136, -1, 51, 38, -1, 61, - -1, 31, 38, 87, 111, 88, 52, 133, 87, 111, - 88, 114, 116, -1, 115, -1, -1, 39, 115, -1, - 32, -1, 49, -1, 117, -1, -1, 120, 118, -1, - 121, 119, -1, 121, -1, -1, 120, -1, -1, 48, - 63, 122, -1, 48, 28, 122, -1, 10, -1, 55, - 45, -1, 55, 25, -1, 43, 3, -1, 54, -1, - 124, -1, 123, 124, -1, 30, 90, 80, -1, 40, - 90, 84, -1, 41, 90, 84, -1, 17, 90, 152, - -1, 17, 152, -1, 6, 90, 84, -1, 25, 75, - 90, 80, -1, 25, 11, 55, 90, 80, -1, 5, - 57, 133, 126, -1, 5, 57, 133, 126, 17, 152, - -1, 127, -1, 126, 89, 127, -1, 126, 127, -1, - 135, -1, 165, -1, 167, -1, 131, -1, 130, -1, - 132, -1, 128, -1, 129, -1, 42, 136, 142, -1, - 42, 15, 136, 142, -1, 78, 136, 136, 142, -1, - 78, 136, 136, 142, 138, -1, 78, 15, 136, 136, - 142, -1, 78, 15, 136, 136, 142, 138, -1, 78, - 136, 136, 142, 143, -1, 78, 15, 136, 136, 142, - 143, -1, 78, 136, 136, 142, 143, 138, -1, 78, - 15, 136, 136, 142, 143, 138, -1, 78, 136, 136, - 142, 141, -1, 78, 15, 136, 136, 142, 141, -1, - 78, 136, 136, 142, 141, 138, -1, 78, 15, 136, - 136, 142, 141, 138, -1, 78, 136, 136, 142, 143, - 141, -1, 78, 15, 136, 136, 142, 143, 141, -1, - 78, 136, 136, 142, 141, 143, -1, 78, 15, 136, - 136, 142, 141, 143, -1, 78, 136, 136, 142, 143, - 141, 138, -1, 78, 15, 136, 136, 142, 143, 141, - 138, -1, 78, 136, 136, 142, 141, 143, 138, -1, - 78, 15, 136, 136, 142, 141, 143, 138, -1, 29, - 18, 137, 166, -1, 4, 107, -1, 53, 133, -1, - 53, 60, 133, -1, 134, -1, 80, 91, 80, -1, - 80, -1, 4, 139, -1, 4, 15, 139, -1, 4, - 87, 105, 88, -1, 4, 15, 87, 105, 88, -1, - 85, -1, 80, -1, 80, -1, 17, 152, -1, 136, - 142, 140, -1, 136, 142, 140, 143, -1, 136, 142, - 140, 141, 143, -1, 136, 142, 140, 141, -1, 136, - 142, 140, 138, -1, 136, 142, 140, 143, 138, -1, - 136, 142, 140, 141, 143, 138, -1, 136, 142, 140, - 143, 141, -1, 136, 142, 140, 143, 141, 138, -1, - 136, 142, 140, 141, 138, -1, -1, 45, -1, 25, - 161, -1, 25, 45, -1, 25, 64, -1, 25, 21, - -1, 25, 65, -1, 25, 66, -1, 153, -1, 154, - -1, 155, -1, 162, -1, 9, -1, 14, -1, 144, - -1, 143, 144, -1, 150, 145, -1, 18, 137, 150, - 145, -1, 146, -1, -1, 149, 147, -1, 26, 149, - -1, 148, -1, -1, 26, -1, 35, 27, -1, 35, - 33, -1, 44, 45, -1, 61, -1, 51, 38, -1, - 6, -1, 151, -1, 13, 87, 83, 88, -1, 92, - 82, 92, -1, 12, -1, 11, -1, 12, 87, 84, - 88, -1, 11, 87, 84, 88, -1, 12, 69, 87, - 84, 88, -1, 11, 69, 87, 84, 88, -1, 67, - 87, 84, 88, -1, 68, 87, 84, 88, -1, 156, - -1, 159, -1, 47, 157, -1, 47, 157, 62, -1, - 24, 157, -1, 24, 157, 62, -1, 46, 157, -1, - 46, 157, 62, -1, 37, 158, -1, 36, 158, -1, - 56, 158, -1, 59, 158, -1, 7, 158, -1, 37, - 158, 62, -1, 36, 158, 62, -1, 56, 158, 62, - -1, 59, 158, 62, -1, 7, 158, 62, -1, 87, - 84, 88, -1, 87, 84, 89, 84, 88, -1, -1, - 87, 84, 88, -1, -1, 72, 160, -1, 72, 160, - 62, -1, 74, 160, -1, 74, 160, 62, -1, 73, - 160, -1, 73, 160, 62, -1, 87, 84, 88, -1, - 87, 84, 89, 84, 88, -1, -1, 84, -1, 152, - -1, 81, -1, 22, -1, 85, -1, 58, 163, 164, - -1, 87, 84, 88, -1, -1, 70, 58, 71, -1, - -1, 29, 136, 166, -1, 29, 15, 136, 166, -1, - 29, 15, 87, 111, 88, -1, 29, 87, 111, 88, - -1, 29, 16, 87, 111, 88, -1, 10, -1, 54, - -1, -1, 5, 168, 136, 55, 141, -1, 5, 168, - 136, 29, 25, -1, 15, -1, -1 -}; +static const yytype_int16 yyrhs[] = { + 94, 0, -1, 95, -1, 95, 86, 96, -1, 96, -1, 125, -1, 102, -1, 99, -1, 97, -1, 100, -1, + 104, -1, -1, 29, 57, 98, 134, -1, 29, 57, 98, 134, 10, 19, -1, 76, 77, -1, -1, 29, 34, + 134, -1, 20, 34, 134, 48, 134, 87, 111, 88, 101, -1, 20, 61, 34, 134, 48, 134, 87, 111, 88, + 101, -1, 123, -1, -1, 20, 57, 103, 133, 87, 105, 88, 123, -1, 20, 57, 103, 133, 87, 105, 88, + -1, 76, 44, 77, -1, -1, 79, 57, 134, -1, 79, 134, -1, 106, -1, 105, 89, 106, -1, 139, -1, + 107, -1, 18, 108, 109, 145, -1, 108, 109, 145, -1, 137, -1, -1, 110, -1, 113, -1, 151, -1, 112, + 87, 111, 88, -1, 136, -1, 111, 89, 136, -1, 51, 38, -1, 61, -1, 31, 38, 87, 111, 88, 52, + 133, 87, 111, 88, 114, 116, -1, 115, -1, -1, 39, 115, -1, 32, -1, 49, -1, 117, -1, -1, 120, + 118, -1, 121, 119, -1, 121, -1, -1, 120, -1, -1, 48, 63, 122, -1, 48, 28, 122, -1, 10, -1, + 55, 45, -1, 55, 25, -1, 43, 3, -1, 54, -1, 124, -1, 123, 124, -1, 30, 90, 80, -1, 40, + 90, 84, -1, 41, 90, 84, -1, 17, 90, 152, -1, 17, 152, -1, 6, 90, 84, -1, 25, 75, 90, + 80, -1, 25, 11, 55, 90, 80, -1, 5, 57, 133, 126, -1, 5, 57, 133, 126, 17, 152, -1, 127, + -1, 126, 89, 127, -1, 126, 127, -1, 135, -1, 165, -1, 167, -1, 131, -1, 130, -1, 132, -1, 128, + -1, 129, -1, 42, 136, 142, -1, 42, 15, 136, 142, -1, 78, 136, 136, 142, -1, 78, 136, 136, 142, + 138, -1, 78, 15, 136, 136, 142, -1, 78, 15, 136, 136, 142, 138, -1, 78, 136, 136, 142, 143, -1, + 78, 15, 136, 136, 142, 143, -1, 78, 136, 136, 142, 143, 138, -1, 78, 15, 136, 136, 142, 143, 138, + -1, 78, 136, 136, 142, 141, -1, 78, 15, 136, 136, 142, 141, -1, 78, 136, 136, 142, 141, 138, -1, + 78, 15, 136, 136, 142, 141, 138, -1, 78, 136, 136, 142, 143, 141, -1, 78, 15, 136, 136, 142, 143, + 141, -1, 78, 136, 136, 142, 141, 143, -1, 78, 15, 136, 136, 142, 141, 143, -1, 78, 136, 136, 142, + 143, 141, 138, -1, 78, 15, 136, 136, 142, 143, 141, 138, -1, 78, 136, 136, 142, 141, 143, 138, -1, + 78, 15, 136, 136, 142, 141, 143, 138, -1, 29, 18, 137, 166, -1, 4, 107, -1, 53, 133, -1, 53, + 60, 133, -1, 134, -1, 80, 91, 80, -1, 80, -1, 4, 139, -1, 4, 15, 139, -1, 4, 87, 105, + 88, -1, 4, 15, 87, 105, 88, -1, 85, -1, 80, -1, 80, -1, 17, 152, -1, 136, 142, 140, -1, + 136, 142, 140, 143, -1, 136, 142, 140, 141, 143, -1, 136, 142, 140, 141, -1, 136, 142, 140, 138, -1, + 136, 142, 140, 143, 138, -1, 136, 142, 140, 141, 143, 138, -1, 136, 142, 140, 143, 141, -1, 136, 142, + 140, 143, 141, 138, -1, 136, 142, 140, 141, 138, -1, -1, 45, -1, 25, 161, -1, 25, 45, -1, 25, + 64, -1, 25, 21, -1, 25, 65, -1, 25, 66, -1, 153, -1, 154, -1, 155, -1, 162, -1, 9, -1, + 14, -1, 144, -1, 143, 144, -1, 150, 145, -1, 18, 137, 150, 145, -1, 146, -1, -1, 149, 147, -1, + 26, 149, -1, 148, -1, -1, 26, -1, 35, 27, -1, 35, 33, -1, 44, 45, -1, 61, -1, 51, 38, + -1, 6, -1, 151, -1, 13, 87, 83, 88, -1, 92, 82, 92, -1, 12, -1, 11, -1, 12, 87, 84, + 88, -1, 11, 87, 84, 88, -1, 12, 69, 87, 84, 88, -1, 11, 69, 87, 84, 88, -1, 67, 87, + 84, 88, -1, 68, 87, 84, 88, -1, 156, -1, 159, -1, 47, 157, -1, 47, 157, 62, -1, 24, 157, + -1, 24, 157, 62, -1, 46, 157, -1, 46, 157, 62, -1, 37, 158, -1, 36, 158, -1, 56, 158, -1, + 59, 158, -1, 7, 158, -1, 37, 158, 62, -1, 36, 158, 62, -1, 56, 158, 62, -1, 59, 158, 62, + -1, 7, 158, 62, -1, 87, 84, 88, -1, 87, 84, 89, 84, 88, -1, -1, 87, 84, 88, -1, -1, + 72, 160, -1, 72, 160, 62, -1, 74, 160, -1, 74, 160, 62, -1, 73, 160, -1, 73, 160, 62, -1, + 87, 84, 88, -1, 87, 84, 89, 84, 88, -1, -1, 84, -1, 152, -1, 81, -1, 22, -1, 85, -1, + 58, 163, 164, -1, 87, 84, 88, -1, -1, 70, 58, 71, -1, -1, 29, 136, 166, -1, 29, 15, 136, + 166, -1, 29, 15, 87, 111, 88, -1, 29, 87, 111, 88, -1, 29, 16, 87, 111, 88, -1, 10, -1, + 54, -1, -1, 5, 168, 136, 55, 141, -1, 5, 168, 136, 29, 25, -1, 15, -1, -1}; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ -static const yytype_uint16 yyrline[] = -{ - 0, 202, 202, 207, 217, 234, 235, 236, 237, 238, - 239, 240, 244, 245, 252, 253, 257, 263, 268, 276, - 277, 281, 286, 293, 294, 298, 299, 303, 309, 317, - 318, 322, 328, 336, 337, 341, 342, 343, 347, 357, - 362, 370, 371, 375, 382, 383, 387, 391, 392, 396, - 397, 401, 407, 416, 417, 421, 422, 426, 430, 434, - 435, 436, 437, 438, 442, 448, 457, 459, 461, 463, - 465, 467, 472, 474, 478, 482, 489, 501, 506, 514, - 515, 516, 517, 518, 519, 520, 521, 526, 529, 535, - 537, 539, 541, 543, 545, 547, 549, 551, 553, 555, - 557, 559, 561, 563, 565, 567, 569, 571, 573, 578, - 585, 589, 590, 594, 598, 599, 611, 612, 613, 614, - 618, 619, 623, 627, 630, 634, 638, 642, 646, 650, - 654, 658, 662, 666, 672, 675, 679, 683, 684, 685, - 686, 687, 691, 692, 693, 694, 695, 700, 709, 714, - 722, 733, 748, 749, 753, 758, 772, 773, 777, 788, - 789, 793, 794, 795, 796, 797, 801, 805, 809, 814, - 819, 824, 829, 835, 841, 849, 857, 858, 862, 868, - 874, 880, 886, 892, 898, 903, 908, 913, 918, 923, - 928, 933, 938, 943, 951, 952, 953, 957, 958, 962, - 967, 972, 977, 982, 987, 995, 997, 998, 1002, 1003, - 1004, 1008, 1014, 1020, 1029, 1030, 1034, 1035, 1039, 1040, - 1041, 1042, 1043, 1047, 1048, 1049, 1053, 1054, 1058, 1059 -}; +static const yytype_uint16 yyrline[] = { + 0, 202, 202, 207, 217, 234, 235, 236, 237, 238, 239, 240, 244, 245, 252, 253, 257, + 263, 268, 276, 277, 281, 286, 293, 294, 298, 299, 303, 309, 317, 318, 322, 328, 336, + 337, 341, 342, 343, 347, 357, 362, 370, 371, 375, 382, 383, 387, 391, 392, 396, 397, + 401, 407, 416, 417, 421, 422, 426, 430, 434, 435, 436, 437, 438, 442, 448, 457, 459, + 461, 463, 465, 467, 472, 474, 478, 482, 489, 501, 506, 514, 515, 516, 517, 518, 519, + 520, 521, 526, 529, 535, 537, 539, 541, 543, 545, 547, 549, 551, 553, 555, 557, 559, + 561, 563, 565, 567, 569, 571, 573, 578, 585, 589, 590, 594, 598, 599, 611, 612, 613, + 614, 618, 619, 623, 627, 630, 634, 638, 642, 646, 650, 654, 658, 662, 666, 672, 675, + 679, 683, 684, 685, 686, 687, 691, 692, 693, 694, 695, 700, 709, 714, 722, 733, 748, + 749, 753, 758, 772, 773, 777, 788, 789, 793, 794, 795, 796, 797, 801, 805, 809, 814, + 819, 824, 829, 835, 841, 849, 857, 858, 862, 868, 874, 880, 886, 892, 898, 903, 908, + 913, 918, 923, 928, 933, 938, 943, 951, 952, 953, 957, 958, 962, 967, 972, 977, 982, + 987, 995, 997, 998, 1002, 1003, 1004, 1008, 1014, 1020, 1029, 1030, 1034, 1035, 1039, 1040, 1041, + 1042, 1043, 1047, 1048, 1049, 1053, 1054, 1058, 1059}; #endif #if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. First, the terminals, then, starting at YYNTOKENS, nonterminals. */ -static const char* const yytname[] = -{ - "$end", "error", "$undefined", "ACTION", "ADD", "ALTER", - "AUTO_INCREMENT", "BIGINT", "BIT", "IDB_BLOB", "CASCADE", "IDB_CHAR", - "CHARACTER", "CHECK", "CLOB", "COLUMN", "COLUMNS", "COMMENT", - "CONSTRAINT", "CONSTRAINTS", "CREATE", "CURRENT_USER", "DATETIME", "DEC", - "DECIMAL", "DEFAULT", "DEFERRABLE", "DEFERRED", "IDB_DELETE", "DROP", - "ENGINE", "FOREIGN", "FULL", "IMMEDIATE", "INDEX", "INITIALLY", - "IDB_INT", "INTEGER", "KEY", "MATCH", "MAX_ROWS", "MIN_ROWS", "MODIFY", - "NO", "NOT", "NULL_TOK", "NUMBER", "NUMERIC", "ON", "PARTIAL", - "PRECISION", "PRIMARY", "REFERENCES", "RENAME", "RESTRICT", "SET", - "SMALLINT", "TABLE", "TIME", "TINYINT", "TO", "UNIQUE", "UNSIGNED", - "UPDATE", "USER", "SESSION_USER", "SYSTEM_USER", "VARCHAR", "VARBINARY", - "VARYING", "WITH", "ZONE", "DOUBLE", "IDB_FLOAT", "REAL", "CHARSET", - "IDB_IF", "EXISTS", "CHANGE", "TRUNCATE", "IDENT", "FCONST", "SCONST", - "CP_SEARCH_CONDITION_TEXT", "ICONST", "DATE", "';'", "'('", "')'", "','", - "'='", "'.'", "'\\''", "$accept", "stmtblock", "stmtmulti", "stmt", - "drop_table_statement", "opt_if_exists", "drop_index_statement", - "create_index_statement", "opt_table_options", "create_table_statement", - "opt_if_not_exists", "trunc_table_statement", "table_element_list", - "table_element", "table_constraint_def", "opt_constraint_name", - "table_constraint", "unique_constraint_def", "column_name_list", - "unique_specifier", "referential_constraint_def", "opt_match_type", - "match_type", "opt_referential_triggered_action", - "referential_triggered_action", "opt_delete_rule", "opt_update_rule", - "update_rule", "delete_rule", "referential_action", "table_options", - "table_option", "alter_table_statement", "alter_table_actions", - "alter_table_action", "modify_column", "rename_column", - "drop_table_constraint_def", "add_table_constraint_def", - "ata_rename_table", "table_name", "qualified_name", "ata_add_column", - "column_name", "constraint_name", "column_option", "column_def", - "opt_null_tok", "default_clause", "data_type", "column_qualifier_list", - "column_constraint_def", "opt_constraint_attributes", - "constraint_attributes", "opt_deferrability_clause", - "deferrability_clause", "constraint_check_time", "column_constraint", - "check_constraint_def", "string_literal", "character_string_type", - "binary_string_type", "numeric_type", "exact_numeric_type", - "opt_precision_scale", "opt_display_width", "approximate_numeric_type", - "opt_display_precision_scale_null", "literal", "datetime_type", - "opt_time_precision", "opt_with_time_zone", "drop_column_def", - "drop_behavior", "alter_column_def", "opt_column", 0 -}; +static const char* const yytname[] = {"$end", + "error", + "$undefined", + "ACTION", + "ADD", + "ALTER", + "AUTO_INCREMENT", + "BIGINT", + "BIT", + "IDB_BLOB", + "CASCADE", + "IDB_CHAR", + "CHARACTER", + "CHECK", + "CLOB", + "COLUMN", + "COLUMNS", + "COMMENT", + "CONSTRAINT", + "CONSTRAINTS", + "CREATE", + "CURRENT_USER", + "DATETIME", + "DEC", + "DECIMAL", + "DEFAULT", + "DEFERRABLE", + "DEFERRED", + "IDB_DELETE", + "DROP", + "ENGINE", + "FOREIGN", + "FULL", + "IMMEDIATE", + "INDEX", + "INITIALLY", + "IDB_INT", + "INTEGER", + "KEY", + "MATCH", + "MAX_ROWS", + "MIN_ROWS", + "MODIFY", + "NO", + "NOT", + "NULL_TOK", + "NUMBER", + "NUMERIC", + "ON", + "PARTIAL", + "PRECISION", + "PRIMARY", + "REFERENCES", + "RENAME", + "RESTRICT", + "SET", + "SMALLINT", + "TABLE", + "TIME", + "TINYINT", + "TO", + "UNIQUE", + "UNSIGNED", + "UPDATE", + "USER", + "SESSION_USER", + "SYSTEM_USER", + "VARCHAR", + "VARBINARY", + "VARYING", + "WITH", + "ZONE", + "DOUBLE", + "IDB_FLOAT", + "REAL", + "CHARSET", + "IDB_IF", + "EXISTS", + "CHANGE", + "TRUNCATE", + "IDENT", + "FCONST", + "SCONST", + "CP_SEARCH_CONDITION_TEXT", + "ICONST", + "DATE", + "';'", + "'('", + "')'", + "','", + "'='", + "'.'", + "'\\''", + "$accept", + "stmtblock", + "stmtmulti", + "stmt", + "drop_table_statement", + "opt_if_exists", + "drop_index_statement", + "create_index_statement", + "opt_table_options", + "create_table_statement", + "opt_if_not_exists", + "trunc_table_statement", + "table_element_list", + "table_element", + "table_constraint_def", + "opt_constraint_name", + "table_constraint", + "unique_constraint_def", + "column_name_list", + "unique_specifier", + "referential_constraint_def", + "opt_match_type", + "match_type", + "opt_referential_triggered_action", + "referential_triggered_action", + "opt_delete_rule", + "opt_update_rule", + "update_rule", + "delete_rule", + "referential_action", + "table_options", + "table_option", + "alter_table_statement", + "alter_table_actions", + "alter_table_action", + "modify_column", + "rename_column", + "drop_table_constraint_def", + "add_table_constraint_def", + "ata_rename_table", + "table_name", + "qualified_name", + "ata_add_column", + "column_name", + "constraint_name", + "column_option", + "column_def", + "opt_null_tok", + "default_clause", + "data_type", + "column_qualifier_list", + "column_constraint_def", + "opt_constraint_attributes", + "constraint_attributes", + "opt_deferrability_clause", + "deferrability_clause", + "constraint_check_time", + "column_constraint", + "check_constraint_def", + "string_literal", + "character_string_type", + "binary_string_type", + "numeric_type", + "exact_numeric_type", + "opt_precision_scale", + "opt_display_width", + "approximate_numeric_type", + "opt_display_precision_scale_null", + "literal", + "datetime_type", + "opt_time_precision", + "opt_with_time_zone", + "drop_column_def", + "drop_behavior", + "alter_column_def", + "opt_column", + 0}; #endif -# ifdef YYPRINT +#ifdef YYPRINT /* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to token YYLEX-NUM. */ -static const yytype_uint16 yytoknum[] = -{ - 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, - 335, 336, 337, 338, 339, 340, 59, 40, 41, 44, - 61, 46, 39 -}; -# endif +static const yytype_uint16 yytoknum[] = { + 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, + 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, + 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 59, 40, 41, 44, 61, 46, 39}; +#endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = -{ - 0, 93, 94, 95, 95, 96, 96, 96, 96, 96, - 96, 96, 97, 97, 98, 98, 99, 100, 100, 101, - 101, 102, 102, 103, 103, 104, 104, 105, 105, 106, - 106, 107, 107, 108, 108, 109, 109, 109, 110, 111, - 111, 112, 112, 113, 114, 114, 115, 115, 115, 116, - 116, 117, 117, 118, 118, 119, 119, 120, 121, 122, - 122, 122, 122, 122, 123, 123, 124, 124, 124, 124, - 124, 124, 124, 124, 125, 125, 126, 126, 126, 127, - 127, 127, 127, 127, 127, 127, 127, 128, 128, 129, - 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, - 129, 129, 129, 129, 129, 129, 129, 129, 129, 130, - 131, 132, 132, 133, 134, 134, 135, 135, 135, 135, - 136, 136, 137, 138, 139, 139, 139, 139, 139, 139, - 139, 139, 139, 139, 140, 140, 141, 141, 141, 141, - 141, 141, 142, 142, 142, 142, 142, 142, 143, 143, - 144, 144, 145, 145, 146, 146, 147, 147, 148, 149, - 149, 150, 150, 150, 150, 150, 151, 152, 153, 153, - 153, 153, 153, 153, 153, 154, 155, 155, 156, 156, - 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, - 156, 156, 156, 156, 157, 157, 157, 158, 158, 159, - 159, 159, 159, 159, 159, 160, 160, 160, 161, 161, - 161, 162, 162, 162, 163, 163, 164, 164, 165, 165, - 165, 165, 165, 166, 166, 166, 167, 167, 168, 168 -}; +static const yytype_uint8 yyr1[] = { + 0, 93, 94, 95, 95, 96, 96, 96, 96, 96, 96, 96, 97, 97, 98, 98, 99, 100, 100, 101, 101, + 102, 102, 103, 103, 104, 104, 105, 105, 106, 106, 107, 107, 108, 108, 109, 109, 109, 110, 111, 111, 112, + 112, 113, 114, 114, 115, 115, 115, 116, 116, 117, 117, 118, 118, 119, 119, 120, 121, 122, 122, 122, 122, + 122, 123, 123, 124, 124, 124, 124, 124, 124, 124, 124, 125, 125, 126, 126, 126, 127, 127, 127, 127, 127, + 127, 127, 127, 128, 128, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, 129, + 129, 129, 129, 129, 130, 131, 132, 132, 133, 134, 134, 135, 135, 135, 135, 136, 136, 137, 138, 139, 139, + 139, 139, 139, 139, 139, 139, 139, 139, 140, 140, 141, 141, 141, 141, 141, 141, 142, 142, 142, 142, 142, + 142, 143, 143, 144, 144, 145, 145, 146, 146, 147, 147, 148, 149, 149, 150, 150, 150, 150, 150, 151, 152, + 153, 153, 153, 153, 153, 153, 153, 154, 155, 155, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 157, 157, 157, 158, 158, 159, 159, 159, 159, 159, 159, 160, 160, 160, 161, 161, + 161, 162, 162, 162, 163, 163, 164, 164, 165, 165, 165, 165, 165, 166, 166, 166, 167, 167, 168, 168}; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = -{ - 0, 2, 1, 3, 1, 1, 1, 1, 1, 1, - 1, 0, 4, 6, 2, 0, 3, 9, 10, 1, - 0, 8, 7, 3, 0, 3, 2, 1, 3, 1, - 1, 4, 3, 1, 0, 1, 1, 1, 4, 1, - 3, 2, 1, 12, 1, 0, 2, 1, 1, 1, - 0, 2, 2, 1, 0, 1, 0, 3, 3, 1, - 2, 2, 2, 1, 1, 2, 3, 3, 3, 3, - 2, 3, 4, 5, 4, 6, 1, 3, 2, 1, - 1, 1, 1, 1, 1, 1, 1, 3, 4, 4, - 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, - 7, 6, 7, 6, 7, 7, 8, 7, 8, 4, - 2, 2, 3, 1, 3, 1, 2, 3, 4, 5, - 1, 1, 1, 2, 3, 4, 5, 4, 4, 5, - 6, 5, 6, 5, 0, 1, 2, 2, 2, 2, - 2, 2, 1, 1, 1, 1, 1, 1, 1, 2, - 2, 4, 1, 0, 2, 2, 1, 0, 1, 2, - 2, 2, 1, 2, 1, 1, 4, 3, 1, 1, - 4, 4, 5, 5, 4, 4, 1, 1, 2, 3, - 2, 3, 2, 3, 2, 2, 2, 2, 2, 3, - 3, 3, 3, 3, 3, 5, 0, 3, 0, 2, - 3, 2, 3, 2, 3, 3, 5, 0, 1, 1, - 1, 1, 1, 3, 3, 0, 3, 0, 3, 4, - 5, 4, 5, 1, 1, 0, 5, 5, 1, 0 -}; +static const yytype_uint8 yyr2[] = { + 0, 2, 1, 3, 1, 1, 1, 1, 1, 1, 1, 0, 4, 6, 2, 0, 3, 9, 10, 1, 0, 8, 7, 3, 0, 3, 2, 1, 3, 1, 1, 4, 3, + 1, 0, 1, 1, 1, 4, 1, 3, 2, 1, 12, 1, 0, 2, 1, 1, 1, 0, 2, 2, 1, 0, 1, 0, 3, 3, 1, 2, 2, 2, 1, 1, 2, + 3, 3, 3, 3, 2, 3, 4, 5, 4, 6, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 3, 4, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, + 6, 7, 6, 7, 6, 7, 7, 8, 7, 8, 4, 2, 2, 3, 1, 3, 1, 2, 3, 4, 5, 1, 1, 1, 2, 3, 4, 5, 4, 4, 5, 6, 5, + 6, 5, 0, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 4, 1, 0, 2, 2, 1, 0, 1, 2, 2, 2, 1, 2, 1, + 1, 4, 3, 1, 1, 4, 4, 5, 5, 4, 4, 1, 1, 2, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 5, 0, 3, + 0, 2, 3, 2, 3, 2, 3, 3, 5, 0, 1, 1, 1, 1, 1, 3, 3, 0, 3, 0, 3, 4, 5, 4, 5, 1, 1, 0, 5, 5, 1, 0}; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state STATE-NUM when YYTABLE doesn't specify something else to do. Zero means the default is an error. */ -static const yytype_uint8 yydefact[] = -{ - 11, 0, 0, 0, 0, 0, 2, 4, 8, 7, - 9, 6, 10, 5, 0, 0, 24, 0, 0, 15, - 0, 115, 26, 1, 11, 0, 113, 0, 0, 0, - 0, 16, 0, 0, 25, 0, 3, 34, 229, 0, - 0, 0, 0, 74, 76, 85, 86, 83, 82, 84, - 79, 80, 81, 0, 0, 0, 0, 14, 12, 114, - 0, 34, 121, 120, 34, 110, 0, 0, 33, 116, - 228, 0, 0, 0, 0, 121, 0, 225, 0, 0, - 0, 111, 0, 0, 0, 0, 78, 0, 23, 34, - 0, 0, 34, 117, 122, 0, 0, 27, 30, 29, - 0, 0, 0, 42, 153, 35, 0, 36, 37, 198, - 146, 169, 168, 147, 211, 196, 198, 198, 196, 196, - 198, 215, 198, 0, 0, 207, 207, 207, 212, 134, - 142, 143, 144, 176, 177, 145, 0, 0, 225, 0, - 225, 0, 39, 223, 224, 218, 0, 87, 112, 0, - 0, 0, 75, 77, 0, 0, 0, 13, 0, 153, - 118, 34, 0, 0, 41, 0, 0, 32, 152, 157, - 0, 0, 188, 0, 0, 0, 0, 0, 180, 185, - 184, 182, 178, 186, 0, 217, 187, 0, 0, 0, - 199, 203, 201, 135, 124, 0, 0, 0, 219, 0, - 109, 221, 0, 88, 0, 89, 0, 0, 22, 0, - 119, 31, 28, 0, 0, 155, 159, 160, 158, 154, - 156, 0, 0, 193, 0, 0, 0, 0, 0, 181, - 190, 189, 183, 179, 191, 0, 0, 213, 192, 0, - 0, 0, 200, 204, 202, 164, 0, 0, 0, 0, - 0, 162, 128, 127, 125, 148, 153, 165, 227, 226, - 220, 222, 40, 91, 90, 97, 93, 167, 20, 0, - 0, 0, 0, 0, 0, 21, 64, 0, 166, 0, - 38, 197, 0, 171, 0, 170, 194, 0, 214, 0, - 174, 175, 205, 0, 123, 0, 139, 137, 138, 140, - 141, 210, 208, 209, 136, 161, 163, 133, 126, 129, - 131, 149, 150, 92, 98, 94, 99, 103, 95, 101, - 17, 19, 0, 0, 70, 0, 0, 0, 0, 0, - 65, 20, 0, 173, 172, 0, 216, 0, 153, 130, - 132, 100, 104, 96, 102, 107, 105, 71, 69, 0, - 0, 66, 67, 68, 18, 0, 195, 206, 151, 108, - 106, 0, 72, 0, 73, 0, 0, 45, 47, 0, - 48, 50, 44, 46, 0, 43, 49, 54, 56, 0, - 0, 0, 51, 53, 0, 52, 55, 59, 0, 63, - 0, 58, 57, 62, 61, 60 -}; +static const yytype_uint8 yydefact[] = { + 11, 0, 0, 0, 0, 0, 2, 4, 8, 7, 9, 6, 10, 5, 0, 0, 24, 0, 0, 15, 0, + 115, 26, 1, 11, 0, 113, 0, 0, 0, 0, 16, 0, 0, 25, 0, 3, 34, 229, 0, 0, 0, + 0, 74, 76, 85, 86, 83, 82, 84, 79, 80, 81, 0, 0, 0, 0, 14, 12, 114, 0, 34, 121, + 120, 34, 110, 0, 0, 33, 116, 228, 0, 0, 0, 0, 121, 0, 225, 0, 0, 0, 111, 0, 0, + 0, 0, 78, 0, 23, 34, 0, 0, 34, 117, 122, 0, 0, 27, 30, 29, 0, 0, 0, 42, 153, + 35, 0, 36, 37, 198, 146, 169, 168, 147, 211, 196, 198, 198, 196, 196, 198, 215, 198, 0, 0, 207, + 207, 207, 212, 134, 142, 143, 144, 176, 177, 145, 0, 0, 225, 0, 225, 0, 39, 223, 224, 218, 0, + 87, 112, 0, 0, 0, 75, 77, 0, 0, 0, 13, 0, 153, 118, 34, 0, 0, 41, 0, 0, 32, + 152, 157, 0, 0, 188, 0, 0, 0, 0, 0, 180, 185, 184, 182, 178, 186, 0, 217, 187, 0, 0, + 0, 199, 203, 201, 135, 124, 0, 0, 0, 219, 0, 109, 221, 0, 88, 0, 89, 0, 0, 22, 0, + 119, 31, 28, 0, 0, 155, 159, 160, 158, 154, 156, 0, 0, 193, 0, 0, 0, 0, 0, 181, 190, + 189, 183, 179, 191, 0, 0, 213, 192, 0, 0, 0, 200, 204, 202, 164, 0, 0, 0, 0, 0, 162, + 128, 127, 125, 148, 153, 165, 227, 226, 220, 222, 40, 91, 90, 97, 93, 167, 20, 0, 0, 0, 0, + 0, 0, 21, 64, 0, 166, 0, 38, 197, 0, 171, 0, 170, 194, 0, 214, 0, 174, 175, 205, 0, + 123, 0, 139, 137, 138, 140, 141, 210, 208, 209, 136, 161, 163, 133, 126, 129, 131, 149, 150, 92, 98, + 94, 99, 103, 95, 101, 17, 19, 0, 0, 70, 0, 0, 0, 0, 0, 65, 20, 0, 173, 172, 0, + 216, 0, 153, 130, 132, 100, 104, 96, 102, 107, 105, 71, 69, 0, 0, 66, 67, 68, 18, 0, 195, + 206, 151, 108, 106, 0, 72, 0, 73, 0, 0, 45, 47, 0, 48, 50, 44, 46, 0, 43, 49, 54, + 56, 0, 0, 0, 51, 53, 0, 52, 55, 59, 0, 63, 0, 58, 57, 62, 61, 60}; /* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int16 yydefgoto[] = -{ - -1, 5, 6, 7, 8, 33, 9, 10, 320, 11, - 29, 12, 96, 97, 98, 66, 104, 105, 141, 106, - 107, 371, 372, 375, 376, 382, 385, 377, 378, 391, - 321, 276, 13, 43, 44, 45, 46, 47, 48, 49, - 25, 26, 50, 142, 68, 252, 99, 194, 253, 129, - 254, 255, 167, 168, 219, 220, 169, 256, 257, 152, - 130, 131, 132, 133, 178, 172, 134, 190, 304, 135, - 185, 237, 51, 145, 52, 71 - }; +static const yytype_int16 yydefgoto[] = { + -1, 5, 6, 7, 8, 33, 9, 10, 320, 11, 29, 12, 96, 97, 98, 66, 104, 105, 141, + 106, 107, 371, 372, 375, 376, 382, 385, 377, 378, 391, 321, 276, 13, 43, 44, 45, 46, 47, + 48, 49, 25, 26, 50, 142, 68, 252, 99, 194, 253, 129, 254, 255, 167, 168, 219, 220, 169, + 256, 257, 152, 130, 131, 132, 133, 178, 172, 134, 190, 304, 135, 185, 237, 51, 145, 52, 71}; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ #define YYPACT_NINF -270 -static const yytype_int16 yypact[] = -{ - 34, -44, 107, 30, 53, 40, -21, -270, -270, -270, - -270, -270, -270, -270, 9, 9, 20, 74, 9, 42, - 9, 58, -270, -270, 34, 3, -270, 117, 127, 9, - 9, -270, 97, 9, -270, 104, -270, 35, 172, 82, - 21, 43, 27, 100, -270, -270, -270, -270, -270, -270, - -270, -270, -270, 9, 114, 109, 174, -270, 210, -270, - 168, 145, 115, -270, 29, -270, 124, 136, -270, -270, - -270, -56, 178, 141, 145, -270, -56, 23, -56, 136, - 9, -270, -56, -56, 162, 3, -270, 150, -270, 29, - 9, 237, 29, -270, -270, 124, 123, -270, -270, -270, - 173, 226, 228, -270, 76, -270, 207, -270, -270, 208, - -270, -59, 83, -270, -270, 211, 208, 208, 211, 211, - 208, 212, 208, 213, 214, 215, 215, 215, -270, 244, - -270, -270, -270, -270, -270, -270, 38, -56, 23, -56, - 23, 154, -270, -270, -270, -270, 136, -270, -270, -56, - 136, 221, -270, -270, -56, 179, 217, -270, 181, 76, - -270, 29, 222, 219, -270, 261, 130, -270, -270, 271, - -56, 223, 246, 224, 225, 227, 229, 231, 248, 250, - 254, 255, 256, 257, 236, 251, 260, 239, 240, 241, - 265, 266, 267, -270, 180, 305, 306, 183, -270, 185, - -270, -270, -56, -270, 136, 180, 242, 187, 209, -56, - -270, -270, -270, 245, -56, -270, -270, -270, -270, -270, - -270, 189, 247, -270, 252, 249, 258, 253, 193, -270, - -270, -270, -270, -270, -270, 259, 274, -270, -270, 262, - 263, 195, -270, -270, -270, -270, 162, 145, 135, 293, - 301, -270, -270, 200, 180, -270, 76, -270, -270, -270, - -270, -270, -270, 180, -270, 200, 180, -270, 209, 264, - 98, 0, 268, 269, 270, 209, -270, 197, -270, 199, - -270, -270, 273, -270, 275, -270, -270, 272, -270, 277, - -270, -270, -270, 278, -270, 201, -270, -270, -270, -270, - -270, -270, -270, -270, -270, -270, -270, -270, 200, -270, - 323, -270, -270, -270, 200, 180, -270, 200, -270, 323, - -270, 209, 280, 162, -270, 288, 276, 285, 283, 284, - -270, 209, 292, -270, -270, 281, -270, 282, 76, -270, - -270, -270, 200, -270, 323, -270, -270, -270, -270, 286, - 291, -270, -270, -270, -270, 9, -270, -270, -270, -270, - -270, 294, -270, 290, -270, -56, 202, 191, -270, 191, - -270, 297, -270, -270, -1, -270, -270, 298, 304, 192, - 192, 321, -270, -270, 309, -270, -270, -270, 350, -270, - 99, -270, -270, -270, -270, -270 -}; +static const yytype_int16 yypact[] = { + 34, -44, 107, 30, 53, 40, -21, -270, -270, -270, -270, -270, -270, -270, 9, 9, 20, + 74, 9, 42, 9, 58, -270, -270, 34, 3, -270, 117, 127, 9, 9, -270, 97, 9, + -270, 104, -270, 35, 172, 82, 21, 43, 27, 100, -270, -270, -270, -270, -270, -270, -270, + -270, -270, 9, 114, 109, 174, -270, 210, -270, 168, 145, 115, -270, 29, -270, 124, 136, + -270, -270, -270, -56, 178, 141, 145, -270, -56, 23, -56, 136, 9, -270, -56, -56, 162, + 3, -270, 150, -270, 29, 9, 237, 29, -270, -270, 124, 123, -270, -270, -270, 173, 226, + 228, -270, 76, -270, 207, -270, -270, 208, -270, -59, 83, -270, -270, 211, 208, 208, 211, + 211, 208, 212, 208, 213, 214, 215, 215, 215, -270, 244, -270, -270, -270, -270, -270, -270, + 38, -56, 23, -56, 23, 154, -270, -270, -270, -270, 136, -270, -270, -56, 136, 221, -270, + -270, -56, 179, 217, -270, 181, 76, -270, 29, 222, 219, -270, 261, 130, -270, -270, 271, + -56, 223, 246, 224, 225, 227, 229, 231, 248, 250, 254, 255, 256, 257, 236, 251, 260, + 239, 240, 241, 265, 266, 267, -270, 180, 305, 306, 183, -270, 185, -270, -270, -56, -270, + 136, 180, 242, 187, 209, -56, -270, -270, -270, 245, -56, -270, -270, -270, -270, -270, -270, + 189, 247, -270, 252, 249, 258, 253, 193, -270, -270, -270, -270, -270, -270, 259, 274, -270, + -270, 262, 263, 195, -270, -270, -270, -270, 162, 145, 135, 293, 301, -270, -270, 200, 180, + -270, 76, -270, -270, -270, -270, -270, -270, 180, -270, 200, 180, -270, 209, 264, 98, 0, + 268, 269, 270, 209, -270, 197, -270, 199, -270, -270, 273, -270, 275, -270, -270, 272, -270, + 277, -270, -270, -270, 278, -270, 201, -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, + -270, -270, 200, -270, 323, -270, -270, -270, 200, 180, -270, 200, -270, 323, -270, 209, 280, + 162, -270, 288, 276, 285, 283, 284, -270, 209, 292, -270, -270, 281, -270, 282, 76, -270, + -270, -270, 200, -270, 323, -270, -270, -270, -270, 286, 291, -270, -270, -270, -270, 9, -270, + -270, -270, -270, -270, 294, -270, 290, -270, -56, 202, 191, -270, 191, -270, 297, -270, -270, + -1, -270, -270, 298, 304, 192, 192, 321, -270, -270, 309, -270, -270, -270, 350, -270, 99, + -270, -270, -270, -270, -270}; /* YYPGOTO[NTERM-NUM]. */ -static const yytype_int16 yypgoto[] = -{ - -270, -270, -270, 331, -270, -270, -270, -270, 26, -270, - -270, -270, 62, 218, 336, 314, 287, -270, -136, -270, - -270, -270, 11, -270, -270, -270, -270, 6, 1, 7, - 175, -269, -270, -270, -28, -270, -270, -270, -270, -270, - -29, 5, -270, -23, -70, -183, 79, -270, -175, -74, - -179, -223, -157, -270, -270, -270, 216, 90, -5, -202, - -270, -270, -270, -270, 161, 116, -270, 166, -270, -270, - -270, -270, -270, 119, -270, -270 - }; +static const yytype_int16 yypgoto[] = { + -270, -270, -270, 331, -270, -270, -270, -270, 26, -270, -270, -270, 62, 218, 336, 314, + 287, -270, -136, -270, -270, -270, 11, -270, -270, -270, -270, 6, 1, 7, 175, -269, + -270, -270, -28, -270, -270, -270, -270, -270, -29, 5, -270, -23, -70, -183, 79, -270, + -175, -74, -179, -223, -157, -270, -270, -270, 216, 90, -5, -202, -270, -270, -270, -270, + 161, 116, -270, 166, -270, -270, -270, -270, -270, 119, -270, -270}; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ #define YYTABLE_NINF -123 -static const yytype_int16 yytable[] = -{ - 55, 197, 211, 199, 140, 147, 330, 37, 38, 22, - 173, 325, 81, 14, 67, 86, 77, 79, 207, 83, - 27, 259, 264, 31, 75, 34, 266, 379, 174, 63, - 265, 311, 39, 143, 221, 56, 78, 67, 58, 1, - 23, 67, 82, 311, 294, 40, 303, 61, 136, 138, - 60, 148, 330, 61, 2, 146, 41, 153, 87, 149, - 150, 108, 380, 3, 18, 24, 67, 195, 324, 67, - 307, 309, 203, 277, 308, 326, 205, 144, 279, 310, - 313, 42, 316, 318, 315, 311, 317, 19, 314, 21, - 108, 319, 311, 196, 311, 156, 28, 72, 73, 312, - 74, 75, 165, 80, 37, 38, 63, 75, 30, 62, - 20, 166, 63, 4, 63, 62, 69, 84, 32, 311, - 63, 348, 64, 21, 394, 339, 204, 340, -122, 39, - 263, 341, 343, 21, 345, 342, 346, 100, 67, 93, - 344, 15, 40, 109, 395, 110, -122, 111, 112, 35, - 113, 155, 175, 41, 158, 101, 296, 216, 114, 359, - 115, 360, 75, 217, 16, 53, -122, 63, 17, 76, - 176, 54, 116, 117, 57, 102, -122, 295, 42, 262, - 297, 358, 118, 119, 59, 103, 245, 70, 323, 85, - 151, 88, 120, 100, 121, 122, 89, 246, 247, 298, - 299, 300, 387, 123, 124, 248, 245, 245, 125, 126, - 127, 160, 161, 100, 100, 269, 301, 246, 247, 302, - 91, 128, 90, 368, 249, 94, 270, 151, 139, 366, - 369, 250, 179, 180, 271, 388, 183, 154, 186, 272, - 370, 251, 201, 202, 249, 249, 389, 390, 75, 273, - 274, 250, 250, 63, 151, 92, 157, 198, 75, 200, - 162, 251, 251, 63, 163, 137, 164, 208, 161, 210, - 161, 260, 202, 261, 202, 268, 202, 280, 202, 181, - 182, 286, 287, 292, 293, 331, 202, 332, 202, 193, - 367, 202, 191, 192, 170, 171, 166, 218, 177, 184, - 187, 188, 189, 206, 209, 213, 214, 222, 223, 225, - 229, 224, 230, 227, 226, 228, 231, 232, 233, 234, - 235, 236, 238, 239, 240, 241, 363, 242, 243, 244, - 258, 248, 289, 278, 267, 281, 282, 283, 305, 306, - 246, 285, 284, 349, 355, 374, 381, 288, 336, 379, - 290, 291, 384, 393, 322, 36, 335, 354, 327, 328, - 329, 333, 337, 334, 347, 351, 350, 352, 353, 356, - 357, 362, 380, 65, 364, 95, 361, 365, 383, 212, - 373, 215, 159, 275, 386, 338, 0, 392 -}; +static const yytype_int16 yytable[] = { + 55, 197, 211, 199, 140, 147, 330, 37, 38, 22, 173, 325, 81, 14, 67, 86, 77, 79, 207, 83, + 27, 259, 264, 31, 75, 34, 266, 379, 174, 63, 265, 311, 39, 143, 221, 56, 78, 67, 58, 1, + 23, 67, 82, 311, 294, 40, 303, 61, 136, 138, 60, 148, 330, 61, 2, 146, 41, 153, 87, 149, + 150, 108, 380, 3, 18, 24, 67, 195, 324, 67, 307, 309, 203, 277, 308, 326, 205, 144, 279, 310, + 313, 42, 316, 318, 315, 311, 317, 19, 314, 21, 108, 319, 311, 196, 311, 156, 28, 72, 73, 312, + 74, 75, 165, 80, 37, 38, 63, 75, 30, 62, 20, 166, 63, 4, 63, 62, 69, 84, 32, 311, + 63, 348, 64, 21, 394, 339, 204, 340, -122, 39, 263, 341, 343, 21, 345, 342, 346, 100, 67, 93, + 344, 15, 40, 109, 395, 110, -122, 111, 112, 35, 113, 155, 175, 41, 158, 101, 296, 216, 114, 359, + 115, 360, 75, 217, 16, 53, -122, 63, 17, 76, 176, 54, 116, 117, 57, 102, -122, 295, 42, 262, + 297, 358, 118, 119, 59, 103, 245, 70, 323, 85, 151, 88, 120, 100, 121, 122, 89, 246, 247, 298, + 299, 300, 387, 123, 124, 248, 245, 245, 125, 126, 127, 160, 161, 100, 100, 269, 301, 246, 247, 302, + 91, 128, 90, 368, 249, 94, 270, 151, 139, 366, 369, 250, 179, 180, 271, 388, 183, 154, 186, 272, + 370, 251, 201, 202, 249, 249, 389, 390, 75, 273, 274, 250, 250, 63, 151, 92, 157, 198, 75, 200, + 162, 251, 251, 63, 163, 137, 164, 208, 161, 210, 161, 260, 202, 261, 202, 268, 202, 280, 202, 181, + 182, 286, 287, 292, 293, 331, 202, 332, 202, 193, 367, 202, 191, 192, 170, 171, 166, 218, 177, 184, + 187, 188, 189, 206, 209, 213, 214, 222, 223, 225, 229, 224, 230, 227, 226, 228, 231, 232, 233, 234, + 235, 236, 238, 239, 240, 241, 363, 242, 243, 244, 258, 248, 289, 278, 267, 281, 282, 283, 305, 306, + 246, 285, 284, 349, 355, 374, 381, 288, 336, 379, 290, 291, 384, 393, 322, 36, 335, 354, 327, 328, + 329, 333, 337, 334, 347, 351, 350, 352, 353, 356, 357, 362, 380, 65, 364, 95, 361, 365, 383, 212, + 373, 215, 159, 275, 386, 338, 0, 392}; -static const yytype_int16 yycheck[] = -{ - 29, 137, 159, 139, 74, 79, 275, 4, 5, 4, - 69, 11, 41, 57, 37, 43, 39, 40, 154, 42, - 15, 196, 205, 18, 80, 20, 205, 28, 87, 85, - 205, 254, 29, 10, 170, 30, 15, 60, 33, 5, - 0, 64, 15, 266, 246, 42, 248, 18, 71, 72, - 15, 80, 321, 18, 20, 78, 53, 85, 53, 82, - 83, 66, 63, 29, 34, 86, 89, 29, 270, 92, - 253, 254, 146, 209, 253, 75, 150, 54, 214, 254, - 263, 78, 265, 266, 263, 308, 265, 57, 263, 80, - 95, 266, 315, 55, 317, 90, 76, 15, 16, 256, - 18, 80, 26, 60, 4, 5, 85, 80, 34, 80, - 57, 35, 85, 79, 85, 80, 37, 17, 76, 342, - 85, 323, 87, 80, 25, 308, 149, 310, 13, 29, - 204, 314, 315, 80, 317, 314, 319, 13, 161, 60, - 315, 34, 42, 7, 45, 9, 31, 11, 12, 91, - 14, 89, 69, 53, 92, 31, 21, 27, 22, 342, - 24, 344, 80, 33, 57, 48, 51, 85, 61, 87, - 87, 44, 36, 37, 77, 51, 61, 247, 78, 202, - 45, 338, 46, 47, 80, 61, 6, 15, 90, 89, - 92, 77, 56, 13, 58, 59, 87, 17, 18, 64, - 65, 66, 10, 67, 68, 25, 6, 6, 72, 73, - 74, 88, 89, 13, 13, 6, 81, 17, 18, 84, - 10, 85, 48, 32, 44, 80, 17, 92, 87, 365, - 39, 51, 116, 117, 25, 43, 120, 87, 122, 30, - 49, 61, 88, 89, 44, 44, 54, 55, 80, 40, - 41, 51, 51, 85, 92, 87, 19, 138, 80, 140, - 87, 61, 61, 85, 38, 87, 38, 88, 89, 88, - 89, 88, 89, 88, 89, 88, 89, 88, 89, 118, - 119, 88, 89, 88, 89, 88, 89, 88, 89, 45, - 88, 89, 126, 127, 87, 87, 35, 26, 87, 87, - 87, 87, 87, 82, 87, 83, 87, 84, 62, 84, - 62, 87, 62, 84, 87, 84, 62, 62, 62, 62, - 84, 70, 62, 84, 84, 84, 355, 62, 62, 62, - 25, 25, 58, 88, 92, 88, 84, 88, 45, 38, - 17, 88, 84, 55, 52, 48, 48, 88, 71, 28, - 88, 88, 48, 3, 90, 24, 84, 331, 90, 90, - 90, 88, 84, 88, 84, 80, 90, 84, 84, 88, - 88, 80, 63, 37, 80, 61, 90, 87, 377, 161, - 369, 165, 95, 208, 378, 295, -1, 380 -}; +static const yytype_int16 yycheck[] = { + 29, 137, 159, 139, 74, 79, 275, 4, 5, 4, 69, 11, 41, 57, 37, 43, 39, 40, 154, 42, 15, + 196, 205, 18, 80, 20, 205, 28, 87, 85, 205, 254, 29, 10, 170, 30, 15, 60, 33, 5, 0, 64, + 15, 266, 246, 42, 248, 18, 71, 72, 15, 80, 321, 18, 20, 78, 53, 85, 53, 82, 83, 66, 63, + 29, 34, 86, 89, 29, 270, 92, 253, 254, 146, 209, 253, 75, 150, 54, 214, 254, 263, 78, 265, 266, + 263, 308, 265, 57, 263, 80, 95, 266, 315, 55, 317, 90, 76, 15, 16, 256, 18, 80, 26, 60, 4, + 5, 85, 80, 34, 80, 57, 35, 85, 79, 85, 80, 37, 17, 76, 342, 85, 323, 87, 80, 25, 308, + 149, 310, 13, 29, 204, 314, 315, 80, 317, 314, 319, 13, 161, 60, 315, 34, 42, 7, 45, 9, 31, + 11, 12, 91, 14, 89, 69, 53, 92, 31, 21, 27, 22, 342, 24, 344, 80, 33, 57, 48, 51, 85, + 61, 87, 87, 44, 36, 37, 77, 51, 61, 247, 78, 202, 45, 338, 46, 47, 80, 61, 6, 15, 90, + 89, 92, 77, 56, 13, 58, 59, 87, 17, 18, 64, 65, 66, 10, 67, 68, 25, 6, 6, 72, 73, + 74, 88, 89, 13, 13, 6, 81, 17, 18, 84, 10, 85, 48, 32, 44, 80, 17, 92, 87, 365, 39, + 51, 116, 117, 25, 43, 120, 87, 122, 30, 49, 61, 88, 89, 44, 44, 54, 55, 80, 40, 41, 51, + 51, 85, 92, 87, 19, 138, 80, 140, 87, 61, 61, 85, 38, 87, 38, 88, 89, 88, 89, 88, 89, + 88, 89, 88, 89, 88, 89, 118, 119, 88, 89, 88, 89, 88, 89, 88, 89, 45, 88, 89, 126, 127, + 87, 87, 35, 26, 87, 87, 87, 87, 87, 82, 87, 83, 87, 84, 62, 84, 62, 87, 62, 84, 87, + 84, 62, 62, 62, 62, 84, 70, 62, 84, 84, 84, 355, 62, 62, 62, 25, 25, 58, 88, 92, 88, + 84, 88, 45, 38, 17, 88, 84, 55, 52, 48, 48, 88, 71, 28, 88, 88, 48, 3, 90, 24, 84, + 331, 90, 90, 90, 88, 84, 88, 84, 80, 90, 84, 84, 88, 88, 80, 63, 37, 80, 61, 90, 87, + 377, 161, 369, 165, 95, 208, 378, 295, -1, 380}; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = -{ - 0, 5, 20, 29, 79, 94, 95, 96, 97, 99, - 100, 102, 104, 125, 57, 34, 57, 61, 34, 57, - 57, 80, 134, 0, 86, 133, 134, 134, 76, 103, - 34, 134, 76, 98, 134, 91, 96, 4, 5, 29, - 42, 53, 78, 126, 127, 128, 129, 130, 131, 132, - 135, 165, 167, 48, 44, 133, 134, 77, 134, 80, - 15, 18, 80, 85, 87, 107, 108, 136, 137, 139, - 15, 168, 15, 16, 18, 80, 87, 136, 15, 136, - 60, 133, 15, 136, 17, 89, 127, 134, 77, 87, - 48, 10, 87, 139, 80, 108, 105, 106, 107, 139, - 13, 31, 51, 61, 109, 110, 112, 113, 151, 7, - 9, 11, 12, 14, 22, 24, 36, 37, 46, 47, - 56, 58, 59, 67, 68, 72, 73, 74, 85, 142, - 153, 154, 155, 156, 159, 162, 136, 87, 136, 87, - 137, 111, 136, 10, 54, 166, 136, 142, 133, 136, - 136, 92, 152, 127, 87, 105, 134, 19, 105, 109, - 88, 89, 87, 38, 38, 26, 35, 145, 146, 149, - 87, 87, 158, 69, 87, 69, 87, 87, 157, 158, - 158, 157, 157, 158, 87, 163, 158, 87, 87, 87, - 160, 160, 160, 45, 140, 29, 55, 111, 166, 111, - 166, 88, 89, 142, 136, 142, 82, 111, 88, 87, - 88, 145, 106, 83, 87, 149, 27, 33, 26, 147, - 148, 111, 84, 62, 87, 84, 87, 84, 84, 62, - 62, 62, 62, 62, 62, 84, 70, 164, 62, 84, - 84, 84, 62, 62, 62, 6, 17, 18, 25, 44, - 51, 61, 138, 141, 143, 144, 150, 151, 25, 141, - 88, 88, 136, 142, 138, 141, 143, 92, 88, 6, - 17, 25, 30, 40, 41, 123, 124, 111, 88, 111, - 88, 88, 84, 88, 84, 88, 88, 89, 88, 58, - 88, 88, 88, 89, 152, 137, 21, 45, 64, 65, - 66, 81, 84, 152, 161, 45, 38, 138, 143, 138, - 141, 144, 145, 138, 141, 143, 138, 143, 138, 141, - 101, 123, 90, 90, 152, 11, 75, 90, 90, 90, - 124, 88, 88, 88, 88, 84, 71, 84, 150, 138, - 138, 138, 143, 138, 141, 138, 138, 84, 152, 55, - 90, 80, 84, 84, 101, 52, 88, 88, 145, 138, - 138, 90, 80, 133, 80, 87, 111, 88, 32, 39, - 49, 114, 115, 115, 48, 116, 117, 120, 121, 28, - 63, 48, 118, 121, 48, 119, 120, 10, 43, 54, - 55, 122, 122, 3, 25, 45 -}; +static const yytype_uint8 yystos[] = { + 0, 5, 20, 29, 79, 94, 95, 96, 97, 99, 100, 102, 104, 125, 57, 34, 57, 61, 34, 57, 57, + 80, 134, 0, 86, 133, 134, 134, 76, 103, 34, 134, 76, 98, 134, 91, 96, 4, 5, 29, 42, 53, + 78, 126, 127, 128, 129, 130, 131, 132, 135, 165, 167, 48, 44, 133, 134, 77, 134, 80, 15, 18, 80, + 85, 87, 107, 108, 136, 137, 139, 15, 168, 15, 16, 18, 80, 87, 136, 15, 136, 60, 133, 15, 136, + 17, 89, 127, 134, 77, 87, 48, 10, 87, 139, 80, 108, 105, 106, 107, 139, 13, 31, 51, 61, 109, + 110, 112, 113, 151, 7, 9, 11, 12, 14, 22, 24, 36, 37, 46, 47, 56, 58, 59, 67, 68, 72, + 73, 74, 85, 142, 153, 154, 155, 156, 159, 162, 136, 87, 136, 87, 137, 111, 136, 10, 54, 166, 136, + 142, 133, 136, 136, 92, 152, 127, 87, 105, 134, 19, 105, 109, 88, 89, 87, 38, 38, 26, 35, 145, + 146, 149, 87, 87, 158, 69, 87, 69, 87, 87, 157, 158, 158, 157, 157, 158, 87, 163, 158, 87, 87, + 87, 160, 160, 160, 45, 140, 29, 55, 111, 166, 111, 166, 88, 89, 142, 136, 142, 82, 111, 88, 87, + 88, 145, 106, 83, 87, 149, 27, 33, 26, 147, 148, 111, 84, 62, 87, 84, 87, 84, 84, 62, 62, + 62, 62, 62, 62, 84, 70, 164, 62, 84, 84, 84, 62, 62, 62, 6, 17, 18, 25, 44, 51, 61, + 138, 141, 143, 144, 150, 151, 25, 141, 88, 88, 136, 142, 138, 141, 143, 92, 88, 6, 17, 25, 30, + 40, 41, 123, 124, 111, 88, 111, 88, 88, 84, 88, 84, 88, 88, 89, 88, 58, 88, 88, 88, 89, + 152, 137, 21, 45, 64, 65, 66, 81, 84, 152, 161, 45, 38, 138, 143, 138, 141, 144, 145, 138, 141, + 143, 138, 143, 138, 141, 101, 123, 90, 90, 152, 11, 75, 90, 90, 90, 124, 88, 88, 88, 88, 84, + 71, 84, 150, 138, 138, 138, 143, 138, 141, 138, 138, 84, 152, 55, 90, 80, 84, 84, 101, 52, 88, + 88, 145, 138, 138, 90, 80, 133, 80, 87, 111, 88, 32, 39, 49, 114, 115, 115, 48, 116, 117, 120, + 121, 28, 63, 48, 118, 121, 48, 119, 120, 10, 43, 54, 55, 122, 122, 3, 25, 45}; -#define yyerrok (yyerrstatus = 0) -#define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 - -#define YYACCEPT goto yyacceptlab -#define YYABORT goto yyabortlab -#define YYERROR goto yyerrorlab +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) +#define YYEMPTY (-2) +#define YYEOF 0 +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab /* Like YYERROR except do call yyerror. This remains here temporarily to ease the transition to the new meaning of YYERROR, for GCC. Once GCC version 2 has supplanted version 1, this can go. */ -#define YYFAIL goto yyerrlab +#define YYFAIL goto yyerrlab -#define YYRECOVERING() (!!yyerrstatus) +#define YYRECOVERING() (!!yyerrstatus) -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY && yylen == 1) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - yytoken = YYTRANSLATE (yychar); \ - YYPOPSTACK (1); \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (YYID (0)) - - -#define YYTERROR 1 -#define YYERRCODE 256 +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY && yylen == 1) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + yytoken = YYTRANSLATE(yychar); \ + YYPOPSTACK(1); \ + goto yybackup; \ + } \ + else \ + { \ + yyerror(YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (YYID(0)) +#define YYTERROR 1 +#define YYERRCODE 256 /* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. If N is 0, then set CURRENT to the empty location which ends @@ -1102,136 +948,120 @@ while (YYID (0)) #define YYRHSLOC(Rhs, K) ((Rhs)[K]) #ifndef YYLLOC_DEFAULT -# define YYLLOC_DEFAULT(Current, Rhs, N) \ - do \ - if (YYID (N)) \ - { \ - (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ - (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ - (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ - (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ - } \ - else \ - { \ - (Current).first_line = (Current).last_line = \ - YYRHSLOC (Rhs, 0).last_line; \ - (Current).first_column = (Current).last_column = \ - YYRHSLOC (Rhs, 0).last_column; \ - } \ - while (YYID (0)) +#define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (YYID(N)) \ + { \ + (Current).first_line = YYRHSLOC(Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC(Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC(Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC(Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = YYRHSLOC(Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = YYRHSLOC(Rhs, 0).last_column; \ + } \ + while (YYID(0)) #endif - /* YY_LOCATION_PRINT -- Print the location on the stream. This macro was not mandated originally: define only if we know we won't break user code: when these are the locations we know. */ #ifndef YY_LOCATION_PRINT -# if YYLTYPE_IS_TRIVIAL -# define YY_LOCATION_PRINT(File, Loc) \ - fprintf (File, "%d.%d-%d.%d", \ - (Loc).first_line, (Loc).first_column, \ - (Loc).last_line, (Loc).last_column) -# else -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -# endif +#if YYLTYPE_IS_TRIVIAL +#define YY_LOCATION_PRINT(File, Loc) \ + fprintf(File, "%d.%d-%d.%d", (Loc).first_line, (Loc).first_column, (Loc).last_line, (Loc).last_column) +#else +#define YY_LOCATION_PRINT(File, Loc) ((void)0) +#endif #endif - /* YYLEX -- calling `yylex' with the right arguments. */ #ifdef YYLEX_PARAM -# define YYLEX yylex (YYLEX_PARAM) +#define YYLEX yylex(YYLEX_PARAM) #else -# define YYLEX yylex () +#define YYLEX yylex() #endif /* Enable debugging if requested. */ #if YYDEBUG -# ifndef YYFPRINTF -# include /* INFRINGES ON USER NAME SPACE */ -# define YYFPRINTF fprintf -# endif +#ifndef YYFPRINTF +#include /* INFRINGES ON USER NAME SPACE */ +#define YYFPRINTF fprintf +#endif -# define YYDPRINTF(Args) \ -do { \ - if (yydebug) \ - YYFPRINTF Args; \ -} while (YYID (0)) - -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ -do { \ - if (yydebug) \ - { \ - YYFPRINTF (stderr, "%s ", Title); \ - yy_symbol_print (stderr, \ - Type, Value); \ - YYFPRINTF (stderr, "\n"); \ - } \ -} while (YYID (0)) +#define YYDPRINTF(Args) \ + do \ + { \ + if (yydebug) \ + YYFPRINTF Args; \ + } while (YYID(0)) +#define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ + do \ + { \ + if (yydebug) \ + { \ + YYFPRINTF(stderr, "%s ", Title); \ + yy_symbol_print(stderr, Type, Value); \ + YYFPRINTF(stderr, "\n"); \ + } \ + } while (YYID(0)) /*--------------------------------. | Print this symbol on YYOUTPUT. | `--------------------------------*/ /*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_value_print (FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_symbol_value_print(FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) #else -static void -yy_symbol_value_print (yyoutput, yytype, yyvaluep) -FILE* yyoutput; +static void yy_symbol_value_print(yyoutput, yytype, yyvaluep) FILE* yyoutput; int yytype; YYSTYPE const* const yyvaluep; #endif { - if (!yyvaluep) - return; + if (!yyvaluep) + return; -# ifdef YYPRINT +#ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); + if (yytype < YYNTOKENS) + YYPRINT(yyoutput, yytoknum[yytype], *yyvaluep); -# else - YYUSE (yyoutput); -# endif +#else + YYUSE(yyoutput); +#endif - switch (yytype) - { - default: - break; - } + switch (yytype) + { + default: break; + } } - /*--------------------------------. | Print this symbol on YYOUTPUT. | `--------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_print (FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_symbol_print(FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) #else -static void -yy_symbol_print (yyoutput, yytype, yyvaluep) -FILE* yyoutput; +static void yy_symbol_print(yyoutput, yytype, yyvaluep) FILE* yyoutput; int yytype; YYSTYPE const* const yyvaluep; #endif { - if (yytype < YYNTOKENS) - YYFPRINTF (yyoutput, "token %s (", yytname[yytype]); - else - YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]); + if (yytype < YYNTOKENS) + YYFPRINTF(yyoutput, "token %s (", yytname[yytype]); + else + YYFPRINTF(yyoutput, "nterm %s (", yytname[yytype]); - yy_symbol_value_print (yyoutput, yytype, yyvaluep); - YYFPRINTF (yyoutput, ")"); + yy_symbol_value_print(yyoutput, yytype, yyvaluep); + YYFPRINTF(yyoutput, ")"); } /*------------------------------------------------------------------. @@ -1239,87 +1069,76 @@ YYSTYPE const* const yyvaluep; | TOP (included). | `------------------------------------------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_stack_print (yytype_int16* yybottom, yytype_int16* yytop) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_stack_print(yytype_int16* yybottom, yytype_int16* yytop) #else -static void -yy_stack_print (yybottom, yytop) -yytype_int16* yybottom; +static void yy_stack_print(yybottom, yytop) yytype_int16* yybottom; yytype_int16* yytop; #endif { - YYFPRINTF (stderr, "Stack now"); + YYFPRINTF(stderr, "Stack now"); - for (; yybottom <= yytop; yybottom++) - { - int yybot = *yybottom; - YYFPRINTF (stderr, " %d", yybot); - } + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF(stderr, " %d", yybot); + } - YYFPRINTF (stderr, "\n"); + YYFPRINTF(stderr, "\n"); } -# define YY_STACK_PRINT(Bottom, Top) \ -do { \ - if (yydebug) \ - yy_stack_print ((Bottom), (Top)); \ -} while (YYID (0)) - +#define YY_STACK_PRINT(Bottom, Top) \ + do \ + { \ + if (yydebug) \ + yy_stack_print((Bottom), (Top)); \ + } while (YYID(0)) /*------------------------------------------------. | Report that the YYRULE is going to be reduced. | `------------------------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_reduce_print (YYSTYPE* yyvsp, int yyrule) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_reduce_print(YYSTYPE* yyvsp, int yyrule) #else -static void -yy_reduce_print (yyvsp, yyrule) -YYSTYPE* yyvsp; +static void yy_reduce_print(yyvsp, yyrule) YYSTYPE* yyvsp; int yyrule; #endif { - int yynrhs = yyr2[yyrule]; - int yyi; - unsigned long int yylno = yyrline[yyrule]; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", - yyrule - 1, yylno); + int yynrhs = yyr2[yyrule]; + int yyi; + unsigned long int yylno = yyrline[yyrule]; + YYFPRINTF(stderr, "Reducing stack by rule %d (line %lu):\n", yyrule - 1, yylno); - /* The symbols being reduced. */ - for (yyi = 0; yyi < yynrhs; yyi++) - { - YYFPRINTF (stderr, " $%d = ", yyi + 1); - yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi], - &(yyvsp[(yyi + 1) - (yynrhs)]) - ); - YYFPRINTF (stderr, "\n"); - } + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF(stderr, " $%d = ", yyi + 1); + yy_symbol_print(stderr, yyrhs[yyprhs[yyrule] + yyi], &(yyvsp[(yyi + 1) - (yynrhs)])); + YYFPRINTF(stderr, "\n"); + } } -# define YY_REDUCE_PRINT(Rule) \ -do { \ - if (yydebug) \ - yy_reduce_print (yyvsp, Rule); \ -} while (YYID (0)) +#define YY_REDUCE_PRINT(Rule) \ + do \ + { \ + if (yydebug) \ + yy_reduce_print(yyvsp, Rule); \ + } while (YYID(0)) /* Nonzero means print parse trace. It is left uninitialized so that multiple parsers can coexist. */ int yydebug; #else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) -# define YY_STACK_PRINT(Bottom, Top) -# define YY_REDUCE_PRINT(Rule) +#define YYDPRINTF(Args) +#define YY_SYMBOL_PRINT(Title, Type, Value, Location) +#define YY_STACK_PRINT(Bottom, Top) +#define YY_REDUCE_PRINT(Rule) #endif /* !YYDEBUG */ - /* YYINITDEPTH -- initial size of the parser's stacks. */ -#ifndef YYINITDEPTH -# define YYINITDEPTH 200 +#ifndef YYINITDEPTH +#define YYINITDEPTH 200 #endif /* YYMAXDEPTH -- maximum size the stacks can grow to (effective only @@ -1330,67 +1149,57 @@ int yydebug; evaluated with infinite-precision integer arithmetic. */ #ifndef YYMAXDEPTH -# define YYMAXDEPTH 10000 +#define YYMAXDEPTH 10000 #endif - - #if YYERROR_VERBOSE -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else -/* Return the length of YYSTR. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static YYSIZE_T -yystrlen (const char* yystr) +#ifndef yystrlen +#if defined __GLIBC__ && defined _STRING_H +#define yystrlen strlen #else -static YYSIZE_T -yystrlen (yystr) -const char* yystr; +/* Return the length of YYSTR. */ +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static YYSIZE_T yystrlen(const char* yystr) +#else +static YYSIZE_T yystrlen(yystr) const char* yystr; #endif { - YYSIZE_T yylen; + YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; + for (yylen = 0; yystr[yylen]; yylen++) + continue; - return yylen; + return yylen; } -# endif -# endif +#endif +#endif -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else +#ifndef yystpcpy +#if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +#define yystpcpy stpcpy +#else /* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in YYDEST. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static char* -yystpcpy (char* yydest, const char* yysrc) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static char* yystpcpy(char* yydest, const char* yysrc) #else -static char* -yystpcpy (yydest, yysrc) -char* yydest; +static char* yystpcpy(yydest, yysrc) char* yydest; const char* yysrc; #endif { - char* yyd = yydest; - const char* yys = yysrc; + char* yyd = yydest; + const char* yys = yysrc; - while ((*yyd++ = *yys++) != '\0') - continue; + while ((*yyd++ = *yys++) != '\0') + continue; - return yyd - 1; + return yyd - 1; } -# endif -# endif +#endif +#endif -# ifndef yytnamerr +#ifndef yytnamerr /* Copy to YYRES the contents of YYSTR after stripping away unnecessary quotes and backslashes, so that it's suitable for yyerror. The heuristic is that double-quoting is unnecessary unless the string @@ -1398,50 +1207,47 @@ const char* yysrc; backslash-backslash). YYSTR is taken from yytname. If YYRES is null, do not copy; instead, return the length of what the result would have been. */ -static YYSIZE_T -yytnamerr (char* yyres, const char* yystr) +static YYSIZE_T yytnamerr(char* yyres, const char* yystr) { - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const* yyp = yystr; + if (*yystr == '"') + { + YYSIZE_T yyn = 0; + char const* yyp = yystr; - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': goto do_not_strip_quotes; - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; + /* Fall through. */ + default: + if (yyres) + yyres[yyn] = *yyp; - yyn++; - break; + yyn++; + break; - case '"': - if (yyres) - yyres[yyn] = '\0'; + case '"': + if (yyres) + yyres[yyn] = '\0'; - return yyn; - } + return yyn; + } -do_not_strip_quotes: - ; - } + do_not_strip_quotes:; + } - if (! yyres) - return yystrlen (yystr); + if (!yyres) + return yystrlen(yystr); - return yystpcpy (yyres, yystr) - yyres; + return yystpcpy(yyres, yystr) - yyres; } -# endif +#endif /* Copy into YYRESULT an error message about the unexpected token YYCHAR while in state YYSTATE. Return the number of bytes copied, @@ -1450,25 +1256,27 @@ do_not_strip_quotes: copied. As a special case, return 0 if an ordinary "syntax error" message will do. Return YYSIZE_MAXIMUM if overflow occurs during size calculation. */ -static YYSIZE_T -yysyntax_error (char* yyresult, int yystate, int yychar) +static YYSIZE_T yysyntax_error(char* yyresult, int yystate, int yychar) { - int yyn = yypact[yystate]; + int yyn = yypact[yystate]; - if (! (YYPACT_NINF < yyn && yyn <= YYLAST)) - return 0; - else + if (!(YYPACT_NINF < yyn && yyn <= YYLAST)) + return 0; + else + { + int yytype = YYTRANSLATE(yychar); + YYSIZE_T yysize0 = yytnamerr(0, yytname[yytype]); + YYSIZE_T yysize = yysize0; + YYSIZE_T yysize1; + int yysize_overflow = 0; + enum { - int yytype = YYTRANSLATE (yychar); - YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]); - YYSIZE_T yysize = yysize0; - YYSIZE_T yysize1; - int yysize_overflow = 0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - char const* yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - int yyx; + YYERROR_VERBOSE_ARGS_MAXIMUM = 5 + }; + char const* yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; + int yyx; -# if 0 +#if 0 /* This is so xgettext sees the translatable formats that are constructed on the fly. */ YY_("syntax error, unexpected %s"); @@ -1476,134 +1284,124 @@ yysyntax_error (char* yyresult, int yystate, int yychar) YY_("syntax error, unexpected %s, expecting %s or %s"); YY_("syntax error, unexpected %s, expecting %s or %s or %s"); YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); -# endif - char* yyfmt; - char const* yyf; - static char const yyunexpected[] = "syntax error, unexpected %s"; - static char const yyexpecting[] = ", expecting %s"; - static char const yyor[] = " or %s"; - char yyformat[sizeof yyunexpected - + sizeof yyexpecting - 1 - + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) - * (sizeof yyor - 1))]; - char const* yyprefix = yyexpecting; +#endif + char* yyfmt; + char const* yyf; + static char const yyunexpected[] = "syntax error, unexpected %s"; + static char const yyexpecting[] = ", expecting %s"; + static char const yyor[] = " or %s"; + char yyformat[sizeof yyunexpected + sizeof yyexpecting - 1 + + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) * (sizeof yyor - 1))]; + char const* yyprefix = yyexpecting; - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. */ - int yyxbegin = yyn < 0 ? -yyn : 0; + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. */ + int yyxbegin = yyn < 0 ? -yyn : 0; - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yycount = 1; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yycount = 1; - yyarg[0] = yytname[yytype]; - yyfmt = yystpcpy (yyformat, yyunexpected); + yyarg[0] = yytname[yytype]; + yyfmt = yystpcpy(yyformat, yyunexpected); - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - yyformat[sizeof yyunexpected - 1] = '\0'; - break; - } - - yyarg[yycount++] = yytname[yyx]; - yysize1 = yysize + yytnamerr (0, yytname[yyx]); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - yyfmt = yystpcpy (yyfmt, yyprefix); - yyprefix = yyor; - } - - yyf = YY_(yyformat); - yysize1 = yysize + yystrlen (yyf); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - - if (yysize_overflow) - return YYSIZE_MAXIMUM; - - if (yyresult) + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) + { + if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) { - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - char* yyp = yyresult; - int yyi = 0; - - while ((*yyp = *yyf) != '\0') - { - if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) - { - yyp += yytnamerr (yyp, yyarg[yyi++]); - yyf += 2; - } - else - { - yyp++; - yyf++; - } - } + yycount = 1; + yysize = yysize0; + yyformat[sizeof yyunexpected - 1] = '\0'; + break; } - return yysize; + yyarg[yycount++] = yytname[yyx]; + yysize1 = yysize + yytnamerr(0, yytname[yyx]); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + yyfmt = yystpcpy(yyfmt, yyprefix); + yyprefix = yyor; + } + + yyf = YY_(yyformat); + yysize1 = yysize + yystrlen(yyf); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + + if (yysize_overflow) + return YYSIZE_MAXIMUM; + + if (yyresult) + { + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + char* yyp = yyresult; + int yyi = 0; + + while ((*yyp = *yyf) != '\0') + { + if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) + { + yyp += yytnamerr(yyp, yyarg[yyi++]); + yyf += 2; + } + else + { + yyp++; + yyf++; + } + } } + + return yysize; + } } #endif /* YYERROR_VERBOSE */ - /*-----------------------------------------------. | Release the memory associated to this symbol. | `-----------------------------------------------*/ /*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yydestruct (const char* yymsg, int yytype, YYSTYPE* yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yydestruct(const char* yymsg, int yytype, YYSTYPE* yyvaluep) #else -static void -yydestruct (yymsg, yytype, yyvaluep) -const char* yymsg; +static void yydestruct(yymsg, yytype, yyvaluep) const char* yymsg; int yytype; YYSTYPE* yyvaluep; #endif { - YYUSE (yyvaluep); + YYUSE(yyvaluep); - if (!yymsg) - yymsg = "Deleting"; + if (!yymsg) + yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + YY_SYMBOL_PRINT(yymsg, yytype, yyvaluep, yylocationp); - switch (yytype) - { - - default: - break; - } + switch (yytype) + { + default: break; + } } /* Prevent warnings from -Wmissing-prototypes. */ #ifdef YYPARSE_PARAM #if defined __STDC__ || defined __cplusplus -int yyparse (void* YYPARSE_PARAM); +int yyparse(void* YYPARSE_PARAM); #else -int yyparse (); +int yyparse(); #endif #else /* ! YYPARSE_PARAM */ #if defined __STDC__ || defined __cplusplus -int yyparse (void); +int yyparse(void); #else -int yyparse (); +int yyparse(); #endif #endif /* ! YYPARSE_PARAM */ - /* The lookahead symbol. */ int yychar; @@ -1613,2043 +1411,2174 @@ YYSTYPE yylval; /* Number of syntax errors so far. */ int yynerrs; - - /*-------------------------. | yyparse or yypush_parse. | `-------------------------*/ #ifdef YYPARSE_PARAM -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void* YYPARSE_PARAM) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +int yyparse(void* YYPARSE_PARAM) #else -int -yyparse (YYPARSE_PARAM) -void* YYPARSE_PARAM; +int yyparse(YYPARSE_PARAM) void* YYPARSE_PARAM; #endif #else /* ! YYPARSE_PARAM */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +int yyparse(void) #else -int -yyparse () +int yyparse() #endif #endif { + int yystate; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus; + /* The stacks and their tools: + `yyss': related to states. + `yyvs': related to semantic values. - int yystate; - /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; + Refer to the stacks thru separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ - /* The stacks and their tools: - `yyss': related to states. - `yyvs': related to semantic values. + /* The state stack. */ + yytype_int16 yyssa[YYINITDEPTH]; + yytype_int16* yyss; + yytype_int16* yyssp; - Refer to the stacks thru separate pointers, to allow yyoverflow - to reallocate them elsewhere. */ + /* The semantic value stack. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE* yyvs; + YYSTYPE* yyvsp; - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16* yyss; - yytype_int16* yyssp; + YYSIZE_T yystacksize; - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE* yyvs; - YYSTYPE* yyvsp; - - YYSIZE_T yystacksize; - - int yyn; - int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken; - /* The variables used to return semantic value and location from the - action routines. */ - YYSTYPE yyval; + int yyn; + int yyresult; + /* Lookahead token as an internal (translated) token number. */ + int yytoken; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; #if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char* yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char* yymsg = yymsgbuf; + YYSIZE_T yymsg_alloc = sizeof yymsgbuf; #endif -#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) - /* The number of symbols on the RHS of the reduced rule. - Keep to zero when no symbol should be popped. */ - int yylen = 0; + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; - yytoken = 0; - yyss = yyssa; - yyvs = yyvsa; - yystacksize = YYINITDEPTH; + yytoken = 0; + yyss = yyssa; + yyvs = yyvsa; + yystacksize = YYINITDEPTH; - YYDPRINTF ((stderr, "Starting parse\n")); + YYDPRINTF((stderr, "Starting parse\n")); - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; - yychar = YYEMPTY; /* Cause a token to be read. */ + yystate = 0; + yyerrstatus = 0; + yynerrs = 0; + yychar = YYEMPTY; /* Cause a token to be read. */ - /* Initialize stack pointers. - Waste one element of value and location stack - so that they stay on the same level as the state stack. - The wasted elements are never initialized. */ - yyssp = yyss; - yyvsp = yyvs; + /* Initialize stack pointers. + Waste one element of value and location stack + so that they stay on the same level as the state stack. + The wasted elements are never initialized. */ + yyssp = yyss; + yyvsp = yyvs; - goto yysetstate; + goto yysetstate; - /*------------------------------------------------------------. - | yynewstate -- Push a new state, which is found in yystate. | - `------------------------------------------------------------*/ + /*------------------------------------------------------------. + | yynewstate -- Push a new state, which is found in yystate. | + `------------------------------------------------------------*/ yynewstate: - /* In all cases, when you get here, the value and location stacks - have just been pushed. So pushing a state here evens the stacks. */ - yyssp++; + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; yysetstate: - *yyssp = yystate; + *yyssp = yystate; - if (yyss + yystacksize - 1 <= yyssp) - { - /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; + if (yyss + yystacksize - 1 <= yyssp) + { + /* Get the current used size of the three stacks, in elements. */ + YYSIZE_T yysize = yyssp - yyss + 1; #ifdef yyoverflow - { - /* Give user a chance to reallocate the stack. Use copies of - these so that the &'s don't force the real ones into - memory. */ - YYSTYPE* yyvs1 = yyvs; - yytype_int16* yyss1 = yyss; + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + YYSTYPE* yyvs1 = yyvs; + yytype_int16* yyss1 = yyss; - /* Each stack pointer address is followed by the size of the - data in use in that stack, in bytes. This used to be a - conditional around just the two extra args, but that might - be undefined if yyoverflow is a macro. */ - yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), - &yystacksize); + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow(YY_("memory exhausted"), &yyss1, yysize * sizeof(*yyssp), &yyvs1, yysize * sizeof(*yyvsp), + &yystacksize); - yyss = yyss1; - yyvs = yyvs1; - } + yyss = yyss1; + yyvs = yyvs1; + } #else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE +#ifndef YYSTACK_RELOCATE + goto yyexhaustedlab; +#else + + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + goto yyexhaustedlab; + + yystacksize *= 2; + + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yytype_int16* yyss1 = yyss; + union yyalloc* yyptr = (union yyalloc*)YYSTACK_ALLOC(YYSTACK_BYTES(yystacksize)); + + if (!yyptr) goto yyexhaustedlab; -# else - /* Extend the stack our own way. */ - if (YYMAXDEPTH <= yystacksize) - goto yyexhaustedlab; + YYSTACK_RELOCATE(yyss_alloc, yyss); + YYSTACK_RELOCATE(yyvs_alloc, yyvs); +#undef YYSTACK_RELOCATE - yystacksize *= 2; - - if (YYMAXDEPTH < yystacksize) - yystacksize = YYMAXDEPTH; - - { - yytype_int16* yyss1 = yyss; - union yyalloc* yyptr = - (union yyalloc*) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); - - if (! yyptr) - goto yyexhaustedlab; - - YYSTACK_RELOCATE (yyss_alloc, yyss); - YYSTACK_RELOCATE (yyvs_alloc, yyvs); -# undef YYSTACK_RELOCATE - - if (yyss1 != yyssa) - YYSTACK_FREE (yyss1); - } -# endif + if (yyss1 != yyssa) + YYSTACK_FREE(yyss1); + } +#endif #endif /* no yyoverflow */ - yyssp = yyss + yysize - 1; - yyvsp = yyvs + yysize - 1; + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); + YYDPRINTF((stderr, "Stack size increased to %lu\n", (unsigned long int)yystacksize)); - if (yyss + yystacksize - 1 <= yyssp) - YYABORT; - } + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YYDPRINTF((stderr, "Entering state %d\n", yystate)); - if (yystate == YYFINAL) - YYACCEPT; + if (yystate == YYFINAL) + YYACCEPT; - goto yybackup; + goto yybackup; - /*-----------. - | yybackup. | - `-----------*/ + /*-----------. + | yybackup. | + `-----------*/ yybackup: - /* Do appropriate processing given the current state. Read a - lookahead token if we need one and don't already have one. */ + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ - /* First try to decide what to do without reference to lookahead token. */ - yyn = yypact[yystate]; + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; - if (yyn == YYPACT_NINF) - goto yydefault; + if (yyn == YYPACT_NINF) + goto yydefault; - /* Not known => get a lookahead token if don't already have one. */ + /* Not known => get a lookahead token if don't already have one. */ + + /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ + if (yychar == YYEMPTY) + { + YYDPRINTF((stderr, "Reading a token: ")); + yychar = YYLEX; + } + + if (yychar <= YYEOF) + { + yychar = yytoken = YYEOF; + YYDPRINTF((stderr, "Now at end of input.\n")); + } + else + { + yytoken = YYTRANSLATE(yychar); + YY_SYMBOL_PRINT("Next token is", yytoken, &yylval, &yylloc); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + + yyn = yytable[yyn]; + + if (yyn <= 0) + { + if (yyn == 0 || yyn == YYTABLE_NINF) + goto yyerrlab; + + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT("Shifting", yytoken, &yylval, &yylloc); + + /* Discard the shifted token. */ + yychar = YYEMPTY; + + yystate = yyn; + *++yyvsp = yylval; + + goto yynewstate; + + /*-----------------------------------------------------------. + | yydefault -- do the default action for the current state. | + `-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + + if (yyn == 0) + goto yyerrlab; + + goto yyreduce; + + /*-----------------------------. + | yyreduce -- Do a reduction. | + `-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + `$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1 - yylen]; + + YY_REDUCE_PRINT(yyn); + + switch (yyn) + { + case 2: - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ - if (yychar == YYEMPTY) { - YYDPRINTF ((stderr, "Reading a token: ")); - yychar = YYLEX; + parseTree = (yyvsp[(1) - (1)].sqlStmtList); + ; } + break; + + case 3: + + { + if ((yyvsp[(3) - (3)].sqlStmt) != NULL) + { + (yyvsp[(1) - (3)].sqlStmtList)->push_back((yyvsp[(3) - (3)].sqlStmt)); + (yyval.sqlStmtList) = (yyvsp[(1) - (3)].sqlStmtList); + } + else + { + (yyval.sqlStmtList) = (yyvsp[(1) - (3)].sqlStmtList); + } + + ; + } + break; + + case 4: + + { + /* The user is supposed to supply a ParseTree* via grammar_init. + So, it is already there. */ + if ((yyvsp[(1) - (1)].sqlStmt) != NULL) + { + (yyval.sqlStmtList) = parseTree; + (yyval.sqlStmtList)->push_back((yyvsp[(1) - (1)].sqlStmt)); + } + else + { + (yyval.sqlStmtList) = NULL; + } + + ; + } + break; + + case 11: + + { + (yyval.sqlStmt) = NULL; + ; + } + break; + + case 12: + + { + (yyval.sqlStmt) = new DropTableStatement((yyvsp[(4) - (4)].qualifiedName), false); + ; + } + break; + + case 13: + + { + { + (yyval.sqlStmt) = new DropTableStatement((yyvsp[(4) - (6)].qualifiedName), true); + }; + } + break; + + case 14: + + { + (yyval.str) = NULL; + ; + } + break; + + case 15: + + { + (yyval.str) = NULL; + ; + } + break; + + case 16: + + { + (yyval.sqlStmt) = new DropIndexStatement((yyvsp[(3) - (3)].qualifiedName)); + ; + } + break; + + case 17: + + { + (yyval.sqlStmt) = + new CreateIndexStatement((yyvsp[(3) - (9)].qualifiedName), (yyvsp[(5) - (9)].qualifiedName), + (yyvsp[(7) - (9)].columnNameList), false); + delete (yyvsp[(9) - (9)].tableOptionMap); + ; + } + break; + + case 18: + + { + (yyval.sqlStmt) = + new CreateIndexStatement((yyvsp[(4) - (10)].qualifiedName), (yyvsp[(6) - (10)].qualifiedName), + (yyvsp[(8) - (10)].columnNameList), true); + delete (yyvsp[(10) - (10)].tableOptionMap); + ; + } + break; + + case 20: + + { + (yyval.tableOptionMap) = NULL; + ; + } + break; + + case 21: + + { + (yyval.sqlStmt) = new CreateTableStatement(new TableDef((yyvsp[(4) - (8)].qualifiedName), + (yyvsp[(6) - (8)].tableElementList), + (yyvsp[(8) - (8)].tableOptionMap))); + ; + } + break; + + case 22: + + { + (yyval.sqlStmt) = new CreateTableStatement( + new TableDef((yyvsp[(4) - (7)].qualifiedName), (yyvsp[(6) - (7)].tableElementList), NULL)); + ; + } + break; + + case 23: + + { + (yyval.str) = NULL; + ; + } + break; + + case 24: + + { + (yyval.str) = NULL; + ; + } + break; + + case 25: + + { + (yyval.sqlStmt) = new TruncTableStatement((yyvsp[(3) - (3)].qualifiedName)); + ; + } + break; + + case 26: + + { + { + (yyval.sqlStmt) = new TruncTableStatement((yyvsp[(2) - (2)].qualifiedName)); + }; + } + break; + + case 27: + + { + (yyval.tableElementList) = new TableElementList(); + (yyval.tableElementList)->push_back((yyvsp[(1) - (1)].schemaObject)); + ; + } + break; + + case 28: + + { + (yyval.tableElementList) = (yyvsp[(1) - (3)].tableElementList); + (yyval.tableElementList)->push_back((yyvsp[(3) - (3)].schemaObject)); + ; + } + break; + + case 31: + + { + (yyval.schemaObject) = (yyvsp[(3) - (4)].schemaObject); + (yyvsp[(3) - (4)].schemaObject)->fName = (yyvsp[(2) - (4)].str); + ; + } + break; + + case 32: + + { + (yyval.schemaObject) = (yyvsp[(2) - (3)].schemaObject); + (yyvsp[(2) - (3)].schemaObject)->fName = (yyvsp[(1) - (3)].str); + ; + } + break; + + case 33: + + { + (yyval.str) = (yyvsp[(1) - (1)].str); + ; + } + break; + + case 34: + + { + (yyval.str) = "noname"; + ; + } + break; + + case 37: + + { + (yyval.schemaObject) = new TableCheckConstraintDef((yyvsp[(1) - (1)].str)); + ; + } + break; + + case 38: + + { + if ((yyvsp[(1) - (4)].constraintType) == DDL_UNIQUE) + (yyval.schemaObject) = new TableUniqueConstraintDef((yyvsp[(3) - (4)].columnNameList)); + else if ((yyvsp[(1) - (4)].constraintType) == DDL_PRIMARY_KEY) + (yyval.schemaObject) = new TablePrimaryKeyConstraintDef((yyvsp[(3) - (4)].columnNameList)); + + ; + } + break; + + case 39: + + { + (yyval.columnNameList) = new vector; + (yyval.columnNameList)->push_back((yyvsp[(1) - (1)].str)); + ; + } + break; + + case 40: + + { + (yyval.columnNameList) = (yyvsp[(1) - (3)].columnNameList); + (yyval.columnNameList)->push_back((yyvsp[(3) - (3)].str)); + ; + } + break; + + case 41: + + { + (yyval.constraintType) = DDL_PRIMARY_KEY; + ; + } + break; + + case 42: + + { + (yyval.constraintType) = DDL_UNIQUE; + ; + } + break; + + case 43: + + { + (yyval.schemaObject) = new TableReferencesConstraintDef( + (yyvsp[(4) - (12)].columnNameList), (yyvsp[(7) - (12)].qualifiedName), + (yyvsp[(9) - (12)].columnNameList), (yyvsp[(11) - (12)].matchType), (yyvsp[(12) - (12)].refAction)); + ; + } + break; + + case 45: + + { + (yyval.matchType) = DDL_FULL; + ; + } + break; + + case 46: + + { + (yyval.matchType) = (yyvsp[(2) - (2)].matchType); + ; + } + break; + + case 47: + + { + (yyval.matchType) = DDL_FULL; + ; + } + break; + + case 48: + + { + (yyval.matchType) = DDL_PARTIAL; + ; + } + break; + + case 50: + + { + (yyval.refAction) = NULL; + ; + } + break; + + case 51: + + { + (yyval.refAction) = new ReferentialAction(); + (yyval.refAction)->fOnUpdate = (yyvsp[(1) - (2)].refActionCode); + (yyval.refAction)->fOnDelete = (yyvsp[(2) - (2)].refActionCode); + ; + } + break; + + case 52: + + { + (yyval.refAction) = new ReferentialAction(); + (yyval.refAction)->fOnUpdate = (yyvsp[(2) - (2)].refActionCode); + (yyval.refAction)->fOnDelete = (yyvsp[(1) - (2)].refActionCode); + ; + } + break; + + case 54: + + { + (yyval.refActionCode) = DDL_NO_ACTION; + ; + } + break; + + case 56: + + { + (yyval.refActionCode) = DDL_NO_ACTION; + ; + } + break; + + case 57: + + { + (yyval.refActionCode) = (yyvsp[(3) - (3)].refActionCode); + ; + } + break; + + case 58: + + { + (yyval.refActionCode) = (yyvsp[(3) - (3)].refActionCode); + ; + } + break; + + case 59: + + { + (yyval.refActionCode) = DDL_CASCADE; + ; + } + break; + + case 60: + + { + (yyval.refActionCode) = DDL_SET_NULL; + ; + } + break; + + case 61: + + { + (yyval.refActionCode) = DDL_SET_DEFAULT; + ; + } + break; + + case 62: + + { + (yyval.refActionCode) = DDL_NO_ACTION; + ; + } + break; + + case 63: + + { + (yyval.refActionCode) = DDL_RESTRICT; + ; + } + break; + + case 64: + + { + (yyval.tableOptionMap) = new TableOptionMap(); + (*(yyval.tableOptionMap))[(yyvsp[(1) - (1)].tableOption)->first] = + (yyvsp[(1) - (1)].tableOption)->second; + delete (yyvsp[(1) - (1)].tableOption); + ; + } + break; + + case 65: + + { + (yyval.tableOptionMap) = (yyvsp[(1) - (2)].tableOptionMap); + (*(yyval.tableOptionMap))[(yyvsp[(2) - (2)].tableOption)->first] = + (yyvsp[(2) - (2)].tableOption)->second; + delete (yyvsp[(2) - (2)].tableOption); + ; + } + break; + + case 66: + + { + (yyval.tableOption) = new pair("engine", (yyvsp[(3) - (3)].str)); + ; + } + break; + + case 67: + + { + (yyval.tableOption) = new pair("max_rows", (yyvsp[(3) - (3)].str)); + ; + } + break; + + case 68: + + { + (yyval.tableOption) = new pair("min_rows", (yyvsp[(3) - (3)].str)); + ; + } + break; + + case 69: + + { + (yyval.tableOption) = new pair("comment", (yyvsp[(3) - (3)].str)); + ; + } + break; + + case 70: + + { + (yyval.tableOption) = new pair("comment", (yyvsp[(2) - (2)].str)); + ; + } + break; + + case 71: + + { + (yyval.tableOption) = new pair("auto_increment", (yyvsp[(3) - (3)].str)); + ; + } + break; + + case 72: + + { + (yyval.tableOption) = new pair("default charset", (yyvsp[(4) - (4)].str)); + ; + } + break; + + case 73: + + { + (yyval.tableOption) = new pair("default charset", (yyvsp[(5) - (5)].str)); + ; + } + break; + + case 74: + + { + (yyval.sqlStmt) = new AlterTableStatement((yyvsp[(3) - (4)].qualifiedName), (yyvsp[(4) - (4)].ataList)); + ; + } + break; + + case 75: + + { + (yyval.sqlStmt) = new AlterTableStatement((yyvsp[(3) - (6)].qualifiedName), (yyvsp[(4) - (6)].ataList)); + ; + } + break; + + case 76: + + { + if ((yyvsp[(1) - (1)].ata) != NULL) + { + (yyval.ataList) = new AlterTableActionList(); + (yyval.ataList)->push_back((yyvsp[(1) - (1)].ata)); + } + else + { + /* An alter_table_statement requires at least one action. + So, this shouldn't happen. */ + (yyval.ataList) = NULL; + } + + ; + } + break; + + case 77: + + { + (yyval.ataList) = (yyvsp[(1) - (3)].ataList); + (yyval.ataList)->push_back((yyvsp[(3) - (3)].ata)); + ; + } + break; + + case 78: + + { + (yyval.ataList) = (yyvsp[(1) - (2)].ataList); + (yyval.ataList)->push_back((yyvsp[(2) - (2)].ata)); + ; + } + break; + + case 87: + + { + (yyval.ata) = new AtaModifyColumnType((yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].columnType)); + ; + } + break; + + case 88: + + { + (yyval.ata) = new AtaModifyColumnType((yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].columnType)); + ; + } + break; + + case 89: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (4)].str), (yyvsp[(3) - (4)].str), + (yyvsp[(4) - (4)].columnType), NULL); + ; + } + break; + + case 90: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (5)].str), (yyvsp[(3) - (5)].str), + (yyvsp[(4) - (5)].columnType), (yyvsp[(5) - (5)].columnOption)); + ; + } + break; + + case 91: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (5)].str), (yyvsp[(4) - (5)].str), + (yyvsp[(5) - (5)].columnType), NULL); + ; + } + break; + + case 92: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].str), + (yyvsp[(5) - (6)].columnType), (yyvsp[(6) - (6)].columnOption)); + ; + } + break; + + case 93: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(2) - (5)].str), (yyvsp[(3) - (5)].str), (yyvsp[(4) - (5)].columnType), + (yyvsp[(5) - (5)].constraintList), NULL); + ; + } + break; + + case 94: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].str), (yyvsp[(5) - (6)].columnType), + (yyvsp[(6) - (6)].constraintList), NULL); + ; + } + break; + + case 95: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), + (yyvsp[(5) - (6)].constraintList), NULL, (yyvsp[(6) - (6)].columnOption)); + ; + } + break; + + case 96: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), + (yyvsp[(6) - (7)].constraintList), NULL, (yyvsp[(7) - (7)].columnOption)); + ; + } + break; + + case 97: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (5)].str), (yyvsp[(3) - (5)].str), + (yyvsp[(4) - (5)].columnType), NULL, (yyvsp[(5) - (5)].colDefault)); + ; + } + break; + + case 98: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].str), + (yyvsp[(5) - (6)].columnType), NULL, (yyvsp[(6) - (6)].colDefault)); + ; + } + break; + + case 99: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), + NULL, (yyvsp[(5) - (6)].colDefault), (yyvsp[(6) - (6)].columnOption)); + ; + } + break; + + case 100: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), + NULL, (yyvsp[(6) - (7)].colDefault), (yyvsp[(7) - (7)].columnOption)); + ; + } + break; + + case 101: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), + (yyvsp[(5) - (6)].constraintList), (yyvsp[(6) - (6)].colDefault)); + ; + } + break; + + case 102: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), + (yyvsp[(6) - (7)].constraintList), (yyvsp[(7) - (7)].colDefault)); + ; + } + break; + + case 103: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), + (yyvsp[(6) - (6)].constraintList), (yyvsp[(5) - (6)].colDefault)); + ; + } + break; + + case 104: + + { + (yyval.ata) = + new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), + (yyvsp[(7) - (7)].constraintList), (yyvsp[(6) - (7)].colDefault)); + ; + } + break; + + case 105: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (7)].str), (yyvsp[(3) - (7)].str), + (yyvsp[(4) - (7)].columnType), (yyvsp[(5) - (7)].constraintList), + (yyvsp[(6) - (7)].colDefault), (yyvsp[(7) - (7)].columnOption)); + ; + } + break; + + case 106: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (8)].str), (yyvsp[(4) - (8)].str), + (yyvsp[(5) - (8)].columnType), (yyvsp[(6) - (8)].constraintList), + (yyvsp[(7) - (8)].colDefault), (yyvsp[(8) - (8)].columnOption)); + ; + } + break; + + case 107: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (7)].str), (yyvsp[(3) - (7)].str), + (yyvsp[(4) - (7)].columnType), (yyvsp[(6) - (7)].constraintList), + (yyvsp[(5) - (7)].colDefault), (yyvsp[(7) - (7)].columnOption)); + ; + } + break; + + case 108: + + { + (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (8)].str), (yyvsp[(4) - (8)].str), + (yyvsp[(5) - (8)].columnType), (yyvsp[(7) - (8)].constraintList), + (yyvsp[(6) - (8)].colDefault), (yyvsp[(8) - (8)].columnOption)); + ; + } + break; + + case 109: + + { + (yyval.ata) = new AtaDropTableConstraint((yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].refActionCode)); + ; + } + break; + + case 110: + + { + (yyval.ata) = + new AtaAddTableConstraint(dynamic_cast((yyvsp[(2) - (2)].schemaObject))); + ; + } + break; + + case 111: + + { + (yyval.ata) = new AtaRenameTable((yyvsp[(2) - (2)].qualifiedName)); + ; + } + break; + + case 112: + + { + (yyval.ata) = new AtaRenameTable((yyvsp[(3) - (3)].qualifiedName)); + ; + } + break; + + case 114: + + { + (yyval.qualifiedName) = new QualifiedName((yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)); + ; + } + break; + + case 115: + + { + if (db_schema.size()) + (yyval.qualifiedName) = new QualifiedName((char*)db_schema.c_str(), (yyvsp[(1) - (1)].str)); + else + (yyval.qualifiedName) = new QualifiedName((yyvsp[(1) - (1)].str)); + + ; + } + break; + + case 116: + + { + (yyval.ata) = new AtaAddColumn(dynamic_cast((yyvsp[(2) - (2)].schemaObject))); + ; + } + break; + + case 117: + + { + (yyval.ata) = new AtaAddColumn(dynamic_cast((yyvsp[(3) - (3)].schemaObject))); + ; + } + break; + + case 118: + + { + (yyval.ata) = new AtaAddColumns((yyvsp[(3) - (4)].tableElementList)); + ; + } + break; + + case 119: + + { + (yyval.ata) = new AtaAddColumns((yyvsp[(4) - (5)].tableElementList)); + ; + } + break; + + case 123: + + { + (yyval.columnOption) = (yyvsp[(2) - (2)].str); + ; + } + break; + + case 124: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (3)].str), (yyvsp[(2) - (3)].columnType), NULL, NULL); + ; + } + break; + + case 125: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (4)].str), (yyvsp[(2) - (4)].columnType), + (yyvsp[(4) - (4)].constraintList), NULL); + ; + } + break; + + case 126: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), + (yyvsp[(5) - (5)].constraintList), (yyvsp[(4) - (5)].colDefault)); + ; + } + break; + + case 127: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (4)].str), (yyvsp[(2) - (4)].columnType), NULL, + (yyvsp[(4) - (4)].colDefault), NULL); + ; + } + break; + + case 128: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (4)].str), (yyvsp[(2) - (4)].columnType), NULL, NULL, + (yyvsp[(4) - (4)].columnOption)); + ; + } + break; + + case 129: + + { + (yyval.schemaObject) = + new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), + (yyvsp[(4) - (5)].constraintList), NULL, (yyvsp[(5) - (5)].columnOption)); + ; + } + break; + + case 130: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (6)].str), (yyvsp[(2) - (6)].columnType), + (yyvsp[(5) - (6)].constraintList), (yyvsp[(4) - (6)].colDefault), + (yyvsp[(6) - (6)].columnOption)); + ; + } + break; + + case 131: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), + (yyvsp[(4) - (5)].constraintList), (yyvsp[(5) - (5)].colDefault)); + ; + } + break; + + case 132: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (6)].str), (yyvsp[(2) - (6)].columnType), + (yyvsp[(4) - (6)].constraintList), (yyvsp[(5) - (6)].colDefault), + (yyvsp[(6) - (6)].columnOption)); + ; + } + break; + + case 133: + + { + (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), NULL, + (yyvsp[(4) - (5)].colDefault), (yyvsp[(5) - (5)].columnOption)); + ; + } + break; + + case 136: + + { + (yyval.colDefault) = new ColumnDefaultValue((yyvsp[(2) - (2)].str)); + ; + } + break; + + case 137: + + { + (yyval.colDefault) = new ColumnDefaultValue(NULL); + ; + } + break; + + case 138: + + { + (yyval.colDefault) = new ColumnDefaultValue("$USER"); + ; + } + break; + + case 139: + + { + (yyval.colDefault) = new ColumnDefaultValue("$CURRENT_USER"); + ; + } + break; + + case 140: + + { + (yyval.colDefault) = new ColumnDefaultValue("$SESSION_USER"); + ; + } + break; + + case 141: + + { + (yyval.colDefault) = new ColumnDefaultValue("$SYSTEM_USER"); + ; + } + break; + + case 146: + + { + (yyval.columnType) = new ColumnType(DDL_BLOB); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_BLOB]; + ; + } + break; + + case 147: + + { + (yyval.columnType) = new ColumnType(DDL_CLOB); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_CLOB]; + ; + } + break; + + case 148: + + { + (yyval.constraintList) = new ColumnConstraintList(); + (yyval.constraintList)->push_back((yyvsp[(1) - (1)].columnConstraintDef)); + ; + } + break; + + case 149: + + { + (yyval.constraintList) = (yyvsp[(1) - (2)].constraintList); + (yyval.constraintList)->push_back((yyvsp[(2) - (2)].columnConstraintDef)); + ; + } + break; + + case 150: + + { + (yyval.columnConstraintDef) = (yyvsp[(1) - (2)].columnConstraintDef); + + if ((yyvsp[(2) - (2)].constraintAttributes) != NULL) + { + (yyvsp[(1) - (2)].columnConstraintDef)->fDeferrable = + (yyvsp[(2) - (2)].constraintAttributes)->fDeferrable; + (yyvsp[(1) - (2)].columnConstraintDef)->fCheckTime = + (yyvsp[(2) - (2)].constraintAttributes)->fCheckTime; + } + + ; + } + break; + + case 151: + + { + (yyval.columnConstraintDef) = (yyvsp[(3) - (4)].columnConstraintDef); + (yyvsp[(3) - (4)].columnConstraintDef)->fName = (yyvsp[(2) - (4)].str); + + if ((yyvsp[(4) - (4)].constraintAttributes) != NULL) + { + (yyvsp[(3) - (4)].columnConstraintDef)->fDeferrable = + (yyvsp[(4) - (4)].constraintAttributes)->fDeferrable; + (yyvsp[(3) - (4)].columnConstraintDef)->fCheckTime = + (yyvsp[(4) - (4)].constraintAttributes)->fCheckTime; + } + + ; + } + break; + + case 152: + + { + (yyval.constraintAttributes) = (yyvsp[(1) - (1)].constraintAttributes); + ; + } + break; + + case 153: + + { + (yyval.constraintAttributes) = NULL; + ; + } + break; + + case 154: + + { + (yyval.constraintAttributes) = + new ConstraintAttributes((yyvsp[(1) - (2)].cattr), ((yyvsp[(2) - (2)].cattr) != 0)); + ; + } + break; + + case 155: + + { + (yyval.constraintAttributes) = new ConstraintAttributes((yyvsp[(2) - (2)].cattr), true); + ; + } + break; + + case 157: + + { + (yyval.cattr) = DDL_NON_DEFERRABLE; + ; + } + break; + + case 158: + + { + (yyval.cattr) = DDL_DEFERRABLE; + ; + } + break; + + case 159: + + { + (yyval.cattr) = DDL_INITIALLY_DEFERRED; + ; + } + break; + + case 160: + + { + (yyval.cattr) = DDL_INITIALLY_IMMEDIATE; + ; + } + break; + + case 161: + + { + (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_NOT_NULL); + ; + } + break; + + case 162: + + { + (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_UNIQUE); + ; + } + break; + + case 163: + + { + (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_PRIMARY_KEY); + ; + } + break; + + case 164: + + { + (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_AUTO_INCREMENT); + ; + } + break; + + case 165: + + { + (yyval.columnConstraintDef) = new ColumnConstraintDef((yyvsp[(1) - (1)].str)); + ; + } + break; + + case 166: + + { + (yyval.str) = (yyvsp[(3) - (4)].str); + ; + } + break; + + case 167: + + { + (yyval.str) = (yyvsp[(2) - (3)].str); + ; + } + break; + + case 168: + + { + (yyval.columnType) = new ColumnType(DDL_CHAR); + (yyval.columnType)->fLength = 1; + ; + } + break; + + case 169: + + { + (yyval.columnType) = new ColumnType(DDL_CHAR); + (yyval.columnType)->fLength = 1; + ; + } + break; + + case 170: + + { + (yyval.columnType) = new ColumnType(DDL_CHAR); + (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); + ; + } + break; + + case 171: + + { + (yyval.columnType) = new ColumnType(DDL_CHAR); + (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); + ; + } + break; + + case 172: + + { + (yyval.columnType) = new ColumnType(DDL_VARCHAR); + (yyval.columnType)->fLength = atoi((yyvsp[(4) - (5)].str)); + ; + } + break; + + case 173: + + { + (yyval.columnType) = new ColumnType(DDL_VARCHAR); + (yyval.columnType)->fLength = atoi((yyvsp[(4) - (5)].str)); + ; + } + break; + + case 174: + + { + (yyval.columnType) = new ColumnType(DDL_VARCHAR); + (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); + ; + } + break; + + case 175: + + { + (yyval.columnType) = new ColumnType(DDL_VARBINARY); + (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); + ; + } + break; + + case 178: + + { + (yyvsp[(2) - (2)].columnType)->fType = DDL_NUMERIC; + (yyvsp[(2) - (2)].columnType)->fLength = DDLDatatypeLength[DDL_NUMERIC]; + (yyval.columnType) = (yyvsp[(2) - (2)].columnType); + ; + } + break; + + case 179: + + { + (yyvsp[(2) - (3)].columnType)->fType = DDL_UNSIGNED_NUMERIC; + (yyvsp[(2) - (3)].columnType)->fLength = DDLDatatypeLength[DDL_UNSIGNED_NUMERIC]; + (yyval.columnType) = (yyvsp[(2) - (3)].columnType); + ; + } + break; + + case 180: + + { + (yyvsp[(2) - (2)].columnType)->fType = DDL_DECIMAL; + /* $2->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ + (yyval.columnType) = (yyvsp[(2) - (2)].columnType); + ; + } + break; + + case 181: + + { + (yyvsp[(2) - (3)].columnType)->fType = DDL_UNSIGNED_DECIMAL; + /* $3->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ + (yyval.columnType) = (yyvsp[(2) - (3)].columnType); + ; + } + break; + + case 182: + + { + (yyvsp[(2) - (2)].columnType)->fType = DDL_DECIMAL; + (yyvsp[(2) - (2)].columnType)->fLength = DDLDatatypeLength[DDL_DECIMAL]; + (yyval.columnType) = (yyvsp[(2) - (2)].columnType); + ; + } + break; + + case 183: + + { + (yyvsp[(2) - (3)].columnType)->fType = DDL_UNSIGNED_DECIMAL; + (yyvsp[(2) - (3)].columnType)->fLength = DDLDatatypeLength[DDL_UNSIGNED_DECIMAL]; + (yyval.columnType) = (yyvsp[(2) - (3)].columnType); + ; + } + break; + + case 184: + + { + (yyval.columnType) = new ColumnType(DDL_INT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; + ; + } + break; + + case 185: + + { + (yyval.columnType) = new ColumnType(DDL_INT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; + ; + } + break; + + case 186: + + { + (yyval.columnType) = new ColumnType(DDL_SMALLINT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_SMALLINT]; + ; + } + break; + + case 187: + + { + (yyval.columnType) = new ColumnType(DDL_TINYINT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_TINYINT]; + ; + } + break; + + case 188: + + { + (yyval.columnType) = new ColumnType(DDL_BIGINT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_BIGINT]; + ; + } + break; + + case 189: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_INT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; + ; + } + break; + + case 190: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_INT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; + ; + } + break; + + case 191: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_SMALLINT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_SMALLINT]; + ; + } + break; + + case 192: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_TINYINT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_TINYINT]; + ; + } + break; + + case 193: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_BIGINT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_BIGINT]; + ; + } + break; + + case 194: + + { + (yyval.columnType) = new ColumnType(atoi((yyvsp[(2) - (3)].str)), 0); + ; + } + break; + + case 195: + + { + (yyval.columnType) = new ColumnType(atoi((yyvsp[(2) - (5)].str)), atoi((yyvsp[(4) - (5)].str))); + ; + } + break; + + case 196: + + { + (yyval.columnType) = new ColumnType(10, 0); + ; + } + break; + + case 197: + + { + (yyval.str) = NULL; + ; + } + break; + + case 198: + + { + (yyval.str) = NULL; + ; + } + break; + + case 199: + + { + (yyval.columnType) = new ColumnType(DDL_DOUBLE); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; + ; + } + break; + + case 200: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_DOUBLE); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; + ; + } + break; + + case 201: + + { + (yyval.columnType) = new ColumnType(DDL_DOUBLE); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; + ; + } + break; + + case 202: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_DOUBLE); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; + ; + } + break; + + case 203: + + { + (yyval.columnType) = new ColumnType(DDL_FLOAT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_FLOAT]; + ; + } + break; + + case 204: + + { + (yyval.columnType) = new ColumnType(DDL_UNSIGNED_FLOAT); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_FLOAT]; + ; + } + break; + + case 205: + + { + (yyval.str) = NULL; + ; + } + break; + + case 206: + + { + (yyval.str) = NULL; + ; + } + break; + + case 207: + + { + (yyval.str) = NULL; + ; + } + break; + + case 211: + + { + (yyval.columnType) = new ColumnType(DDL_DATETIME); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DATETIME]; + ; + } + break; + + case 212: + + { + (yyval.columnType) = new ColumnType(DDL_DATE); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DATE]; + ; + } + break; + + case 213: + + { + (yyval.columnType) = new ColumnType(DDL_DATETIME); + (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DATETIME]; + (yyval.columnType)->fPrecision = (yyvsp[(2) - (3)].ival); + (yyval.columnType)->fWithTimezone = (yyvsp[(3) - (3)].flag); + ; + } + break; + + case 214: + + { + (yyval.ival) = atoi((yyvsp[(2) - (3)].str)); + ; + } + break; + + case 215: + + { + (yyval.ival) = -1; + ; + } + break; + + case 216: + + { + (yyval.flag) = true; + ; + } + break; + + case 217: + + { + (yyval.flag) = false; + ; + } + break; + + case 218: + + { + (yyval.ata) = new AtaDropColumn((yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].refActionCode)); + ; + } + break; + + case 219: + + { + (yyval.ata) = new AtaDropColumn((yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].refActionCode)); + ; + } + break; + + case 220: + + { + (yyval.ata) = new AtaDropColumns((yyvsp[(4) - (5)].columnNameList)); + ; + } + break; + + case 221: + + { + (yyval.ata) = new AtaDropColumns((yyvsp[(3) - (4)].columnNameList)); + ; + } + break; + + case 222: + + { + (yyval.ata) = new AtaDropColumns((yyvsp[(4) - (5)].columnNameList)); + ; + } + break; + + case 223: + + { + (yyval.refActionCode) = DDL_CASCADE; + ; + } + break; + + case 224: + + { + (yyval.refActionCode) = DDL_RESTRICT; + ; + } + break; + + case 225: + + { + (yyval.refActionCode) = DDL_NO_ACTION; + ; + } + break; + + case 226: + + { + (yyval.ata) = new AtaSetColumnDefault((yyvsp[(3) - (5)].str), (yyvsp[(5) - (5)].colDefault)); + ; + } + break; + + case 227: + + { + (yyval.ata) = new AtaDropColumnDefault((yyvsp[(3) - (5)].str)); + ; + } + break; + + default: break; + } + + YY_SYMBOL_PRINT("-> $$ =", yyr1[yyn], &yyval, &yyloc); + + YYPOPSTACK(yylen); + yylen = 0; + YY_STACK_PRINT(yyss, yyssp); + + *++yyvsp = yyval; + + /* Now `shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + + yyn = yyr1[yyn]; + + yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; + + if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) + yystate = yytable[yystate]; + else + yystate = yydefgoto[yyn - YYNTOKENS]; + + goto yynewstate; + + /*------------------------------------. + | yyerrlab -- here on detecting error | + `------------------------------------*/ +yyerrlab: + + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; +#if !YYERROR_VERBOSE + yyerror(YY_("syntax error")); +#else + { + YYSIZE_T yysize = yysyntax_error(0, yystate, yychar); + + if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) + { + YYSIZE_T yyalloc = 2 * yysize; + + if (!(yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) + yyalloc = YYSTACK_ALLOC_MAXIMUM; + + if (yymsg != yymsgbuf) + YYSTACK_FREE(yymsg); + + yymsg = (char*)YYSTACK_ALLOC(yyalloc); + + if (yymsg) + yymsg_alloc = yyalloc; + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + } + } + + if (0 < yysize && yysize <= yymsg_alloc) + { + (void)yysyntax_error(yymsg, yystate, yychar); + yyerror(yymsg); + } + else + { + yyerror(YY_("syntax error")); + + if (yysize != 0) + goto yyexhaustedlab; + } + } +#endif + } + + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ if (yychar <= YYEOF) { - yychar = yytoken = YYEOF; - YYDPRINTF ((stderr, "Now at end of input.\n")); + /* Return failure if at end of input. */ + if (yychar == YYEOF) + YYABORT; } else { - yytoken = YYTRANSLATE (yychar); - YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); + yydestruct("Error: discarding", yytoken, &yylval); + yychar = YYEMPTY; } + } - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken; + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; - if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) - goto yydefault; - - yyn = yytable[yyn]; - - if (yyn <= 0) - { - if (yyn == 0 || yyn == YYTABLE_NINF) - goto yyerrlab; - - yyn = -yyn; - goto yyreduce; - } - - /* Count tokens shifted since error; after three, turn off error - status. */ - if (yyerrstatus) - yyerrstatus--; - - /* Shift the lookahead token. */ - YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - - yystate = yyn; - *++yyvsp = yylval; - - goto yynewstate; - - - /*-----------------------------------------------------------. - | yydefault -- do the default action for the current state. | - `-----------------------------------------------------------*/ -yydefault: - yyn = yydefact[yystate]; - - if (yyn == 0) - goto yyerrlab; - - goto yyreduce; - - - /*-----------------------------. - | yyreduce -- Do a reduction. | - `-----------------------------*/ -yyreduce: - /* yyn is the number of a rule to reduce with. */ - yylen = yyr2[yyn]; - - /* If YYLEN is nonzero, implement the default value of the action: - `$$ = $1'. - - Otherwise, the following line sets YYVAL to garbage. - This behavior is undocumented and Bison - users should not rely upon it. Assigning to YYVAL - unconditionally makes the parser a bit smaller, and it avoids a - GCC warning that YYVAL may be used uninitialized. */ - yyval = yyvsp[1 - yylen]; - - - YY_REDUCE_PRINT (yyn); - - switch (yyn) - { - case 2: - - { - parseTree = (yyvsp[(1) - (1)].sqlStmtList); ; - } - break; - - case 3: - - { - if ((yyvsp[(3) - (3)].sqlStmt) != NULL) - { - (yyvsp[(1) - (3)].sqlStmtList)->push_back((yyvsp[(3) - (3)].sqlStmt)); - (yyval.sqlStmtList) = (yyvsp[(1) - (3)].sqlStmtList); - } - else - { - (yyval.sqlStmtList) = (yyvsp[(1) - (3)].sqlStmtList); - } - - ; - } - break; - - case 4: - - { - /* The user is supposed to supply a ParseTree* via grammar_init. - So, it is already there. */ - if ((yyvsp[(1) - (1)].sqlStmt) != NULL) - { - (yyval.sqlStmtList) = parseTree; - (yyval.sqlStmtList)->push_back((yyvsp[(1) - (1)].sqlStmt)); - } - else - { - (yyval.sqlStmtList) = NULL; - } - - ; - } - break; - - case 11: - - { - (yyval.sqlStmt) = NULL; ; - } - break; - - case 12: - - { - (yyval.sqlStmt) = new DropTableStatement((yyvsp[(4) - (4)].qualifiedName), false);; - } - break; - - case 13: - - { - { - (yyval.sqlStmt) = new DropTableStatement((yyvsp[(4) - (6)].qualifiedName), true); - } - ; - } - break; - - case 14: - - { - (yyval.str) = NULL;; - } - break; - - case 15: - - { - (yyval.str) = NULL;; - } - break; - - case 16: - - { - (yyval.sqlStmt) = new DropIndexStatement((yyvsp[(3) - (3)].qualifiedName));; - } - break; - - case 17: - - { - (yyval.sqlStmt) = new CreateIndexStatement((yyvsp[(3) - (9)].qualifiedName), (yyvsp[(5) - (9)].qualifiedName), (yyvsp[(7) - (9)].columnNameList), false); - delete (yyvsp[(9) - (9)].tableOptionMap); - ; - } - break; - - case 18: - - { - (yyval.sqlStmt) = new CreateIndexStatement((yyvsp[(4) - (10)].qualifiedName), (yyvsp[(6) - (10)].qualifiedName), (yyvsp[(8) - (10)].columnNameList), true); - delete (yyvsp[(10) - (10)].tableOptionMap); - ; - } - break; - - case 20: - - { - (yyval.tableOptionMap) = NULL;; - } - break; - - case 21: - - { - (yyval.sqlStmt) = new CreateTableStatement(new TableDef((yyvsp[(4) - (8)].qualifiedName), (yyvsp[(6) - (8)].tableElementList), (yyvsp[(8) - (8)].tableOptionMap))); - ; - } - break; - - case 22: - - { - (yyval.sqlStmt) = new CreateTableStatement(new TableDef((yyvsp[(4) - (7)].qualifiedName), (yyvsp[(6) - (7)].tableElementList), NULL)); - ; - } - break; - - case 23: - - { - (yyval.str) = NULL;; - } - break; - - case 24: - - { - (yyval.str) = NULL;; - } - break; - - case 25: - - { - (yyval.sqlStmt) = new TruncTableStatement((yyvsp[(3) - (3)].qualifiedName));; - } - break; - - case 26: - - { - { - (yyval.sqlStmt) = new TruncTableStatement((yyvsp[(2) - (2)].qualifiedName)); - } ; - } - break; - - case 27: - - { - (yyval.tableElementList) = new TableElementList(); - (yyval.tableElementList)->push_back((yyvsp[(1) - (1)].schemaObject)); - ; - } - break; - - case 28: - - { - (yyval.tableElementList) = (yyvsp[(1) - (3)].tableElementList); - (yyval.tableElementList)->push_back((yyvsp[(3) - (3)].schemaObject)); - ; - } - break; - - case 31: - - { - (yyval.schemaObject) = (yyvsp[(3) - (4)].schemaObject); - (yyvsp[(3) - (4)].schemaObject)->fName = (yyvsp[(2) - (4)].str); - ; - } - break; - - case 32: - - { - (yyval.schemaObject) = (yyvsp[(2) - (3)].schemaObject); - (yyvsp[(2) - (3)].schemaObject)->fName = (yyvsp[(1) - (3)].str); - ; - } - break; - - case 33: - - { - (yyval.str) = (yyvsp[(1) - (1)].str);; - } - break; - - case 34: - - { - (yyval.str) = "noname";; - } - break; - - case 37: - - { - (yyval.schemaObject) = new TableCheckConstraintDef((yyvsp[(1) - (1)].str));; - } - break; - - case 38: - - { - if ((yyvsp[(1) - (4)].constraintType) == DDL_UNIQUE) - (yyval.schemaObject) = new TableUniqueConstraintDef((yyvsp[(3) - (4)].columnNameList)); - else if ((yyvsp[(1) - (4)].constraintType) == DDL_PRIMARY_KEY) - (yyval.schemaObject) = new TablePrimaryKeyConstraintDef((yyvsp[(3) - (4)].columnNameList)); - - ; - } - break; - - case 39: - - { - (yyval.columnNameList) = new vector; - (yyval.columnNameList)->push_back((yyvsp[(1) - (1)].str)); - ; - } - break; - - case 40: - - { - (yyval.columnNameList) = (yyvsp[(1) - (3)].columnNameList); - (yyval.columnNameList)->push_back((yyvsp[(3) - (3)].str)); - ; - } - break; - - case 41: - - { - (yyval.constraintType) = DDL_PRIMARY_KEY;; - } - break; - - case 42: - - { - (yyval.constraintType) = DDL_UNIQUE;; - } - break; - - case 43: - - { - (yyval.schemaObject) = new TableReferencesConstraintDef((yyvsp[(4) - (12)].columnNameList), (yyvsp[(7) - (12)].qualifiedName), (yyvsp[(9) - (12)].columnNameList), (yyvsp[(11) - (12)].matchType), (yyvsp[(12) - (12)].refAction)); - ; - } - break; - - case 45: - - { - (yyval.matchType) = DDL_FULL;; - } - break; - - case 46: - - { - (yyval.matchType) = (yyvsp[(2) - (2)].matchType);; - } - break; - - case 47: - - { - (yyval.matchType) = DDL_FULL;; - } - break; - - case 48: - - { - (yyval.matchType) = DDL_PARTIAL;; - } - break; - - case 50: - - { - (yyval.refAction) = NULL;; - } - break; - - case 51: - - { - (yyval.refAction) = new ReferentialAction(); - (yyval.refAction)->fOnUpdate = (yyvsp[(1) - (2)].refActionCode); - (yyval.refAction)->fOnDelete = (yyvsp[(2) - (2)].refActionCode); - ; - } - break; - - case 52: - - { - (yyval.refAction) = new ReferentialAction(); - (yyval.refAction)->fOnUpdate = (yyvsp[(2) - (2)].refActionCode); - (yyval.refAction)->fOnDelete = (yyvsp[(1) - (2)].refActionCode); - ; - } - break; - - case 54: - - { - (yyval.refActionCode) = DDL_NO_ACTION;; - } - break; - - case 56: - - { - (yyval.refActionCode) = DDL_NO_ACTION;; - } - break; - - case 57: - - { - (yyval.refActionCode) = (yyvsp[(3) - (3)].refActionCode);; - } - break; - - case 58: - - { - (yyval.refActionCode) = (yyvsp[(3) - (3)].refActionCode);; - } - break; - - case 59: - - { - (yyval.refActionCode) = DDL_CASCADE;; - } - break; - - case 60: - - { - (yyval.refActionCode) = DDL_SET_NULL;; - } - break; - - case 61: - - { - (yyval.refActionCode) = DDL_SET_DEFAULT;; - } - break; - - case 62: - - { - (yyval.refActionCode) = DDL_NO_ACTION;; - } - break; - - case 63: - - { - (yyval.refActionCode) = DDL_RESTRICT;; - } - break; - - case 64: - - { - (yyval.tableOptionMap) = new TableOptionMap(); - (*(yyval.tableOptionMap))[(yyvsp[(1) - (1)].tableOption)->first] = (yyvsp[(1) - (1)].tableOption)->second; - delete (yyvsp[(1) - (1)].tableOption); - ; - } - break; - - case 65: - - { - (yyval.tableOptionMap) = (yyvsp[(1) - (2)].tableOptionMap); - (*(yyval.tableOptionMap))[(yyvsp[(2) - (2)].tableOption)->first] = (yyvsp[(2) - (2)].tableOption)->second; - delete (yyvsp[(2) - (2)].tableOption); - ; - } - break; - - case 66: - - { - (yyval.tableOption) = new pair("engine", (yyvsp[(3) - (3)].str));; - } - break; - - case 67: - - { - (yyval.tableOption) = new pair("max_rows", (yyvsp[(3) - (3)].str));; - } - break; - - case 68: - - { - (yyval.tableOption) = new pair("min_rows", (yyvsp[(3) - (3)].str));; - } - break; - - case 69: - - { - (yyval.tableOption) = new pair("comment", (yyvsp[(3) - (3)].str));; - } - break; - - case 70: - - { - (yyval.tableOption) = new pair("comment", (yyvsp[(2) - (2)].str));; - } - break; - - case 71: - - { - (yyval.tableOption) = new pair("auto_increment", (yyvsp[(3) - (3)].str)); - ; - } - break; - - case 72: - - { - (yyval.tableOption) = new pair("default charset", (yyvsp[(4) - (4)].str));; - } - break; - - case 73: - - { - (yyval.tableOption) = new pair("default charset", (yyvsp[(5) - (5)].str));; - } - break; - - case 74: - - { - (yyval.sqlStmt) = new AlterTableStatement((yyvsp[(3) - (4)].qualifiedName), (yyvsp[(4) - (4)].ataList)); - ; - } - break; - - case 75: - - { - (yyval.sqlStmt) = new AlterTableStatement((yyvsp[(3) - (6)].qualifiedName), (yyvsp[(4) - (6)].ataList)); - ; - } - break; - - case 76: - - { - if ((yyvsp[(1) - (1)].ata) != NULL) - { - (yyval.ataList) = new AlterTableActionList(); - (yyval.ataList)->push_back((yyvsp[(1) - (1)].ata)); - } - else - { - /* An alter_table_statement requires at least one action. - So, this shouldn't happen. */ - (yyval.ataList) = NULL; - } - - ; - } - break; - - case 77: - - { - (yyval.ataList) = (yyvsp[(1) - (3)].ataList); - (yyval.ataList)->push_back((yyvsp[(3) - (3)].ata)); - ; - } - break; - - case 78: - - { - (yyval.ataList) = (yyvsp[(1) - (2)].ataList); - (yyval.ataList)->push_back((yyvsp[(2) - (2)].ata)); - ; - } - break; - - case 87: - - { - (yyval.ata) = new AtaModifyColumnType((yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].columnType));; - } - break; - - case 88: - - { - (yyval.ata) = new AtaModifyColumnType((yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].columnType));; - } - break; - - case 89: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (4)].str), (yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].columnType), NULL);; - } - break; - - case 90: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (5)].str), (yyvsp[(3) - (5)].str), (yyvsp[(4) - (5)].columnType), (yyvsp[(5) - (5)].columnOption));; - } - break; - - case 91: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (5)].str), (yyvsp[(4) - (5)].str), (yyvsp[(5) - (5)].columnType), NULL);; - } - break; - - case 92: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].str), (yyvsp[(5) - (6)].columnType), (yyvsp[(6) - (6)].columnOption));; - } - break; - - case 93: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (5)].str), (yyvsp[(3) - (5)].str), (yyvsp[(4) - (5)].columnType), (yyvsp[(5) - (5)].constraintList), NULL);; - } - break; - - case 94: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].str), (yyvsp[(5) - (6)].columnType), (yyvsp[(6) - (6)].constraintList), NULL);; - } - break; - - case 95: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), (yyvsp[(5) - (6)].constraintList), NULL, (yyvsp[(6) - (6)].columnOption));; - } - break; - - case 96: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), (yyvsp[(6) - (7)].constraintList), NULL, (yyvsp[(7) - (7)].columnOption));; - } - break; - - case 97: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (5)].str), (yyvsp[(3) - (5)].str), (yyvsp[(4) - (5)].columnType), NULL, (yyvsp[(5) - (5)].colDefault));; - } - break; - - case 98: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].str), (yyvsp[(5) - (6)].columnType), NULL, (yyvsp[(6) - (6)].colDefault));; - } - break; - - case 99: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), NULL, (yyvsp[(5) - (6)].colDefault), (yyvsp[(6) - (6)].columnOption));; - } - break; - - case 100: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), NULL, (yyvsp[(6) - (7)].colDefault), (yyvsp[(7) - (7)].columnOption));; - } - break; - - case 101: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), (yyvsp[(5) - (6)].constraintList), (yyvsp[(6) - (6)].colDefault));; - } - break; - - case 102: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), (yyvsp[(6) - (7)].constraintList), (yyvsp[(7) - (7)].colDefault));; - } - break; - - case 103: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (6)].str), (yyvsp[(3) - (6)].str), (yyvsp[(4) - (6)].columnType), (yyvsp[(6) - (6)].constraintList), (yyvsp[(5) - (6)].colDefault));; - } - break; - - case 104: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].str), (yyvsp[(5) - (7)].columnType), (yyvsp[(7) - (7)].constraintList), (yyvsp[(6) - (7)].colDefault));; - } - break; - - case 105: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (7)].str), (yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].columnType), (yyvsp[(5) - (7)].constraintList), (yyvsp[(6) - (7)].colDefault), (yyvsp[(7) - (7)].columnOption));; - } - break; - - case 106: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (8)].str), (yyvsp[(4) - (8)].str), (yyvsp[(5) - (8)].columnType), (yyvsp[(6) - (8)].constraintList), (yyvsp[(7) - (8)].colDefault), (yyvsp[(8) - (8)].columnOption));; - } - break; - - case 107: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(2) - (7)].str), (yyvsp[(3) - (7)].str), (yyvsp[(4) - (7)].columnType), (yyvsp[(6) - (7)].constraintList), (yyvsp[(5) - (7)].colDefault), (yyvsp[(7) - (7)].columnOption));; - } - break; - - case 108: - - { - (yyval.ata) = new AtaRenameColumn((yyvsp[(3) - (8)].str), (yyvsp[(4) - (8)].str), (yyvsp[(5) - (8)].columnType), (yyvsp[(7) - (8)].constraintList), (yyvsp[(6) - (8)].colDefault), (yyvsp[(8) - (8)].columnOption));; - } - break; - - case 109: - - { - (yyval.ata) = new AtaDropTableConstraint((yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].refActionCode)); - ; - } - break; - - case 110: - - { - (yyval.ata) = new AtaAddTableConstraint(dynamic_cast((yyvsp[(2) - (2)].schemaObject)));; - } - break; - - case 111: - - { - (yyval.ata) = new AtaRenameTable((yyvsp[(2) - (2)].qualifiedName));; - } - break; - - case 112: - - { - (yyval.ata) = new AtaRenameTable((yyvsp[(3) - (3)].qualifiedName));; - } - break; - - case 114: - - { - (yyval.qualifiedName) = new QualifiedName((yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str));; - } - break; - - case 115: - - { - if (db_schema.size()) - (yyval.qualifiedName) = new QualifiedName((char*)db_schema.c_str(), (yyvsp[(1) - (1)].str)); - else - (yyval.qualifiedName) = new QualifiedName((yyvsp[(1) - (1)].str)); - - ; - } - break; - - case 116: - - { - (yyval.ata) = new AtaAddColumn(dynamic_cast((yyvsp[(2) - (2)].schemaObject)));; - } - break; - - case 117: - - { - (yyval.ata) = new AtaAddColumn(dynamic_cast((yyvsp[(3) - (3)].schemaObject)));; - } - break; - - case 118: - - { - (yyval.ata) = new AtaAddColumns((yyvsp[(3) - (4)].tableElementList));; - } - break; - - case 119: - - { - (yyval.ata) = new AtaAddColumns((yyvsp[(4) - (5)].tableElementList));; - } - break; - - case 123: - - { - (yyval.columnOption) = (yyvsp[(2) - (2)].str);; - } - break; - - case 124: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (3)].str), (yyvsp[(2) - (3)].columnType), NULL, NULL ); - ; - } - break; - - case 125: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (4)].str), (yyvsp[(2) - (4)].columnType), (yyvsp[(4) - (4)].constraintList), NULL); - ; - } - break; - - case 126: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), (yyvsp[(5) - (5)].constraintList), (yyvsp[(4) - (5)].colDefault)); - ; - } - break; - - case 127: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (4)].str), (yyvsp[(2) - (4)].columnType), NULL, (yyvsp[(4) - (4)].colDefault), NULL); - ; - } - break; - - case 128: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (4)].str), (yyvsp[(2) - (4)].columnType), NULL, NULL, (yyvsp[(4) - (4)].columnOption) ); - ; - } - break; - - case 129: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), (yyvsp[(4) - (5)].constraintList), NULL, (yyvsp[(5) - (5)].columnOption)); - ; - } - break; - - case 130: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (6)].str), (yyvsp[(2) - (6)].columnType), (yyvsp[(5) - (6)].constraintList), (yyvsp[(4) - (6)].colDefault), (yyvsp[(6) - (6)].columnOption)); - ; - } - break; - - case 131: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), (yyvsp[(4) - (5)].constraintList), (yyvsp[(5) - (5)].colDefault)); - ; - } - break; - - case 132: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (6)].str), (yyvsp[(2) - (6)].columnType), (yyvsp[(4) - (6)].constraintList), (yyvsp[(5) - (6)].colDefault), (yyvsp[(6) - (6)].columnOption)); - ; - } - break; - - case 133: - - { - (yyval.schemaObject) = new ColumnDef((yyvsp[(1) - (5)].str), (yyvsp[(2) - (5)].columnType), NULL, (yyvsp[(4) - (5)].colDefault), (yyvsp[(5) - (5)].columnOption)); - ; - } - break; - - case 136: - - { - (yyval.colDefault) = new ColumnDefaultValue((yyvsp[(2) - (2)].str)); - ; - } - break; - - case 137: - - { - (yyval.colDefault) = new ColumnDefaultValue(NULL);; - } - break; - - case 138: - - { - (yyval.colDefault) = new ColumnDefaultValue("$USER");; - } - break; - - case 139: - - { - (yyval.colDefault) = new ColumnDefaultValue("$CURRENT_USER");; - } - break; - - case 140: - - { - (yyval.colDefault) = new ColumnDefaultValue("$SESSION_USER");; - } - break; - - case 141: - - { - (yyval.colDefault) = new ColumnDefaultValue("$SYSTEM_USER");; - } - break; - - case 146: - - { - (yyval.columnType) = new ColumnType(DDL_BLOB); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_BLOB]; - ; - } - break; - - case 147: - - { - (yyval.columnType) = new ColumnType(DDL_CLOB); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_CLOB]; - ; - } - break; - - case 148: - - { - (yyval.constraintList) = new ColumnConstraintList(); - (yyval.constraintList)->push_back((yyvsp[(1) - (1)].columnConstraintDef)); - ; - } - break; - - case 149: - - { - (yyval.constraintList) = (yyvsp[(1) - (2)].constraintList); - (yyval.constraintList)->push_back((yyvsp[(2) - (2)].columnConstraintDef)); - ; - } - break; - - case 150: - - { - (yyval.columnConstraintDef) = (yyvsp[(1) - (2)].columnConstraintDef); - - if ((yyvsp[(2) - (2)].constraintAttributes) != NULL) - { - (yyvsp[(1) - (2)].columnConstraintDef)->fDeferrable = (yyvsp[(2) - (2)].constraintAttributes)->fDeferrable; - (yyvsp[(1) - (2)].columnConstraintDef)->fCheckTime = (yyvsp[(2) - (2)].constraintAttributes)->fCheckTime; - } - - ; - } - break; - - case 151: - - { - (yyval.columnConstraintDef) = (yyvsp[(3) - (4)].columnConstraintDef); - (yyvsp[(3) - (4)].columnConstraintDef)->fName = (yyvsp[(2) - (4)].str); - - if ((yyvsp[(4) - (4)].constraintAttributes) != NULL) - { - (yyvsp[(3) - (4)].columnConstraintDef)->fDeferrable = (yyvsp[(4) - (4)].constraintAttributes)->fDeferrable; - (yyvsp[(3) - (4)].columnConstraintDef)->fCheckTime = (yyvsp[(4) - (4)].constraintAttributes)->fCheckTime; - } - - ; - } - break; - - case 152: - - { - (yyval.constraintAttributes) = (yyvsp[(1) - (1)].constraintAttributes);; - } - break; - - case 153: - - { - (yyval.constraintAttributes) = NULL;; - } - break; - - case 154: - - { - (yyval.constraintAttributes) = new ConstraintAttributes((yyvsp[(1) - (2)].cattr), ((yyvsp[(2) - (2)].cattr) != 0)); - ; - } - break; - - case 155: - - { - (yyval.constraintAttributes) = new ConstraintAttributes((yyvsp[(2) - (2)].cattr), true); - ; - } - break; - - case 157: - - { - (yyval.cattr) = DDL_NON_DEFERRABLE;; - } - break; - - case 158: - - { - (yyval.cattr) = DDL_DEFERRABLE;; - } - break; - - case 159: - - { - (yyval.cattr) = DDL_INITIALLY_DEFERRED;; - } - break; - - case 160: - - { - (yyval.cattr) = DDL_INITIALLY_IMMEDIATE;; - } - break; - - case 161: - - { - (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_NOT_NULL);; - } - break; - - case 162: - - { - (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_UNIQUE);; - } - break; - - case 163: - - { - (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_PRIMARY_KEY);; - } - break; - - case 164: - - { - (yyval.columnConstraintDef) = new ColumnConstraintDef(DDL_AUTO_INCREMENT);; - } - break; - - case 165: - - { - (yyval.columnConstraintDef) = new ColumnConstraintDef((yyvsp[(1) - (1)].str));; - } - break; - - case 166: - - { - (yyval.str) = (yyvsp[(3) - (4)].str);; - } - break; - - case 167: - - { - (yyval.str) = (yyvsp[(2) - (3)].str);; - } - break; - - case 168: - - { - (yyval.columnType) = new ColumnType(DDL_CHAR); - (yyval.columnType)->fLength = 1; - ; - } - break; - - case 169: - - { - (yyval.columnType) = new ColumnType(DDL_CHAR); - (yyval.columnType)->fLength = 1; - ; - } - break; - - case 170: - - { - (yyval.columnType) = new ColumnType(DDL_CHAR); - (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); - ; - } - break; - - case 171: - - { - (yyval.columnType) = new ColumnType(DDL_CHAR); - (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); - ; - } - break; - - case 172: - - { - (yyval.columnType) = new ColumnType(DDL_VARCHAR); - (yyval.columnType)->fLength = atoi((yyvsp[(4) - (5)].str)); - ; - } - break; - - case 173: - - { - (yyval.columnType) = new ColumnType(DDL_VARCHAR); - (yyval.columnType)->fLength = atoi((yyvsp[(4) - (5)].str)); - ; - } - break; - - case 174: - - { - (yyval.columnType) = new ColumnType(DDL_VARCHAR); - (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); - ; - } - break; - - case 175: - - { - (yyval.columnType) = new ColumnType(DDL_VARBINARY); - (yyval.columnType)->fLength = atoi((yyvsp[(3) - (4)].str)); - ; - } - break; - - case 178: - - { - (yyvsp[(2) - (2)].columnType)->fType = DDL_NUMERIC; - (yyvsp[(2) - (2)].columnType)->fLength = DDLDatatypeLength[DDL_NUMERIC]; - (yyval.columnType) = (yyvsp[(2) - (2)].columnType); - ; - } - break; - - case 179: - - { - (yyvsp[(2) - (3)].columnType)->fType = DDL_UNSIGNED_NUMERIC; - (yyvsp[(2) - (3)].columnType)->fLength = DDLDatatypeLength[DDL_UNSIGNED_NUMERIC]; - (yyval.columnType) = (yyvsp[(2) - (3)].columnType); - ; - } - break; - - case 180: - - { - (yyvsp[(2) - (2)].columnType)->fType = DDL_DECIMAL; - /* $2->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ - (yyval.columnType) = (yyvsp[(2) - (2)].columnType); - ; - } - break; - - case 181: - - { - (yyvsp[(2) - (3)].columnType)->fType = DDL_UNSIGNED_DECIMAL; - /* $3->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ - (yyval.columnType) = (yyvsp[(2) - (3)].columnType); - ; - } - break; - - case 182: - - { - (yyvsp[(2) - (2)].columnType)->fType = DDL_DECIMAL; - (yyvsp[(2) - (2)].columnType)->fLength = DDLDatatypeLength[DDL_DECIMAL]; - (yyval.columnType) = (yyvsp[(2) - (2)].columnType); - ; - } - break; - - case 183: - - { - (yyvsp[(2) - (3)].columnType)->fType = DDL_UNSIGNED_DECIMAL; - (yyvsp[(2) - (3)].columnType)->fLength = DDLDatatypeLength[DDL_UNSIGNED_DECIMAL]; - (yyval.columnType) = (yyvsp[(2) - (3)].columnType); - ; - } - break; - - case 184: - - { - (yyval.columnType) = new ColumnType(DDL_INT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; - ; - } - break; - - case 185: - - { - (yyval.columnType) = new ColumnType(DDL_INT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; - ; - } - break; - - case 186: - - { - (yyval.columnType) = new ColumnType(DDL_SMALLINT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_SMALLINT]; - ; - } - break; - - case 187: - - { - (yyval.columnType) = new ColumnType(DDL_TINYINT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_TINYINT]; - ; - } - break; - - case 188: - - { - (yyval.columnType) = new ColumnType(DDL_BIGINT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_BIGINT]; - ; - } - break; - - case 189: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_INT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; - ; - } - break; - - case 190: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_INT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_INT]; - ; - } - break; - - case 191: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_SMALLINT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_SMALLINT]; - ; - } - break; - - case 192: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_TINYINT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_TINYINT]; - ; - } - break; - - case 193: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_BIGINT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_BIGINT]; - ; - } - break; - - case 194: - - { - (yyval.columnType) = new ColumnType(atoi((yyvsp[(2) - (3)].str)), 0);; - } - break; - - case 195: - - { - (yyval.columnType) = new ColumnType(atoi((yyvsp[(2) - (5)].str)), atoi((yyvsp[(4) - (5)].str)));; - } - break; - - case 196: - - { - (yyval.columnType) = new ColumnType(10, 0);; - } - break; - - case 197: - - { - (yyval.str) = NULL;; - } - break; - - case 198: - - { - (yyval.str) = NULL;; - } - break; - - case 199: - - { - (yyval.columnType) = new ColumnType(DDL_DOUBLE); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; - ; - } - break; - - case 200: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_DOUBLE); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; - ; - } - break; - - case 201: - - { - (yyval.columnType) = new ColumnType(DDL_DOUBLE); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; - ; - } - break; - - case 202: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_DOUBLE); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DOUBLE]; - ; - } - break; - - case 203: - - { - (yyval.columnType) = new ColumnType(DDL_FLOAT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_FLOAT]; - ; - } - break; - - case 204: - - { - (yyval.columnType) = new ColumnType(DDL_UNSIGNED_FLOAT); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_FLOAT]; - ; - } - break; - - case 205: - - { - (yyval.str) = NULL;; - } - break; - - case 206: - - { - (yyval.str) = NULL;; - } - break; - - case 207: - - { - (yyval.str) = NULL;; - } - break; - - case 211: - - { - (yyval.columnType) = new ColumnType(DDL_DATETIME); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DATETIME]; - ; - } - break; - - case 212: - - { - (yyval.columnType) = new ColumnType(DDL_DATE); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DATE]; - ; - } - break; - - case 213: - - { - (yyval.columnType) = new ColumnType(DDL_DATETIME); - (yyval.columnType)->fLength = DDLDatatypeLength[DDL_DATETIME]; - (yyval.columnType)->fPrecision = (yyvsp[(2) - (3)].ival); - (yyval.columnType)->fWithTimezone = (yyvsp[(3) - (3)].flag); - ; - } - break; - - case 214: - - { - (yyval.ival) = atoi((yyvsp[(2) - (3)].str));; - } - break; - - case 215: - - { - (yyval.ival) = -1;; - } - break; - - case 216: - - { - (yyval.flag) = true;; - } - break; - - case 217: - - { - (yyval.flag) = false;; - } - break; - - case 218: - - { - (yyval.ata) = new AtaDropColumn((yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].refActionCode));; - } - break; - - case 219: - - { - (yyval.ata) = new AtaDropColumn((yyvsp[(3) - (4)].str), (yyvsp[(4) - (4)].refActionCode));; - } - break; - - case 220: - - { - (yyval.ata) = new AtaDropColumns((yyvsp[(4) - (5)].columnNameList));; - } - break; - - case 221: - - { - (yyval.ata) = new AtaDropColumns((yyvsp[(3) - (4)].columnNameList));; - } - break; - - case 222: - - { - (yyval.ata) = new AtaDropColumns((yyvsp[(4) - (5)].columnNameList));; - } - break; - - case 223: - - { - (yyval.refActionCode) = DDL_CASCADE;; - } - break; - - case 224: - - { - (yyval.refActionCode) = DDL_RESTRICT;; - } - break; - - case 225: - - { - (yyval.refActionCode) = DDL_NO_ACTION;; - } - break; - - case 226: - - { - (yyval.ata) = new AtaSetColumnDefault((yyvsp[(3) - (5)].str), (yyvsp[(5) - (5)].colDefault));; - } - break; - - case 227: - - { - (yyval.ata) = new AtaDropColumnDefault((yyvsp[(3) - (5)].str));; - } - break; - - - - default: - break; - } - - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); - - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - - *++yyvsp = yyval; - - /* Now `shift' the result of the reduction. Determine what state - that goes to, based on the state we popped back to and the rule - number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; - - goto yynewstate; - - - /*------------------------------------. - | yyerrlab -- here on detecting error | - `------------------------------------*/ -yyerrlab: - - /* If not already recovering from an error, report this error. */ - if (!yyerrstatus) - { - ++yynerrs; -#if ! YYERROR_VERBOSE - yyerror (YY_("syntax error")); -#else - { - YYSIZE_T yysize = yysyntax_error (0, yystate, yychar); - - if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) - { - YYSIZE_T yyalloc = 2 * yysize; - - if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) - yyalloc = YYSTACK_ALLOC_MAXIMUM; - - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); - - yymsg = (char*) YYSTACK_ALLOC (yyalloc); - - if (yymsg) - yymsg_alloc = yyalloc; - else - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - } - } - - if (0 < yysize && yysize <= yymsg_alloc) - { - (void) yysyntax_error (yymsg, yystate, yychar); - yyerror (yymsg); - } - else - { - yyerror (YY_("syntax error")); - - if (yysize != 0) - goto yyexhaustedlab; - } - } -#endif - } - - - - if (yyerrstatus == 3) - { - /* If just tried and failed to reuse lookahead token after an - error, discard it. */ - - if (yychar <= YYEOF) - { - /* Return failure if at end of input. */ - if (yychar == YYEOF) - YYABORT; - } - else - { - yydestruct ("Error: discarding", - yytoken, &yylval); - yychar = YYEMPTY; - } - } - - /* Else will try to reuse lookahead token after shifting the error - token. */ - goto yyerrlab1; - - - /*---------------------------------------------------. - | yyerrorlab -- error raised explicitly by YYERROR. | - `---------------------------------------------------*/ + /*---------------------------------------------------. + | yyerrorlab -- error raised explicitly by YYERROR. | + `---------------------------------------------------*/ yyerrorlab: - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; + /* Pacify compilers like GCC when the user code never invokes + YYERROR and the label yyerrorlab therefore never appears in user + code. */ + if (/*CONSTCOND*/ 0) + goto yyerrorlab; - /* Do not reclaim the symbols of the rule which action triggered - this YYERROR. */ - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - yystate = *yyssp; - goto yyerrlab1; + /* Do not reclaim the symbols of the rule which action triggered + this YYERROR. */ + YYPOPSTACK(yylen); + yylen = 0; + YY_STACK_PRINT(yyss, yyssp); + yystate = *yyssp; + goto yyerrlab1; - - /*-------------------------------------------------------------. - | yyerrlab1 -- common code for both syntax error and YYERROR. | - `-------------------------------------------------------------*/ + /*-------------------------------------------------------------. + | yyerrlab1 -- common code for both syntax error and YYERROR. | + `-------------------------------------------------------------*/ yyerrlab1: - yyerrstatus = 3; /* Each real token shifted decrements this. */ + yyerrstatus = 3; /* Each real token shifted decrements this. */ - for (;;) + for (;;) + { + yyn = yypact[yystate]; + + if (yyn != YYPACT_NINF) { - yyn = yypact[yystate]; + yyn += YYTERROR; - if (yyn != YYPACT_NINF) - { - yyn += YYTERROR; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + { + yyn = yytable[yyn]; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) - { - yyn = yytable[yyn]; - - if (0 < yyn) - break; - } - } - - /* Pop the current state because it cannot handle the error token. */ - if (yyssp == yyss) - YYABORT; - - - yydestruct ("Error: popping", - yystos[yystate], yyvsp); - YYPOPSTACK (1); - yystate = *yyssp; - YY_STACK_PRINT (yyss, yyssp); + if (0 < yyn) + break; + } } - *++yyvsp = yylval; + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + yydestruct("Error: popping", yystos[yystate], yyvsp); + YYPOPSTACK(1); + yystate = *yyssp; + YY_STACK_PRINT(yyss, yyssp); + } - /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + *++yyvsp = yylval; - yystate = yyn; - goto yynewstate; + /* Shift the error token. */ + YY_SYMBOL_PRINT("Shifting", yystos[yyn], yyvsp, yylsp); + yystate = yyn; + goto yynewstate; - /*-------------------------------------. - | yyacceptlab -- YYACCEPT comes here. | - `-------------------------------------*/ + /*-------------------------------------. + | yyacceptlab -- YYACCEPT comes here. | + `-------------------------------------*/ yyacceptlab: - yyresult = 0; - goto yyreturn; + yyresult = 0; + goto yyreturn; - /*-----------------------------------. - | yyabortlab -- YYABORT comes here. | - `-----------------------------------*/ + /*-----------------------------------. + | yyabortlab -- YYABORT comes here. | + `-----------------------------------*/ yyabortlab: - yyresult = 1; - goto yyreturn; + yyresult = 1; + goto yyreturn; #if !defined(yyoverflow) || YYERROR_VERBOSE - /*-------------------------------------------------. - | yyexhaustedlab -- memory exhaustion comes here. | - `-------------------------------------------------*/ + /*-------------------------------------------------. + | yyexhaustedlab -- memory exhaustion comes here. | + `-------------------------------------------------*/ yyexhaustedlab: - yyerror (YY_("memory exhausted")); - yyresult = 2; - /* Fall through. */ + yyerror(YY_("memory exhausted")); + yyresult = 2; + /* Fall through. */ #endif yyreturn: - if (yychar != YYEMPTY) - yydestruct ("Cleanup: discarding lookahead", - yytoken, &yylval); + if (yychar != YYEMPTY) + yydestruct("Cleanup: discarding lookahead", yytoken, &yylval); - /* Do not reclaim the symbols of the rule which action triggered - this YYABORT or YYACCEPT. */ - YYPOPSTACK (yylen); - YY_STACK_PRINT (yyss, yyssp); + /* Do not reclaim the symbols of the rule which action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK(yylen); + YY_STACK_PRINT(yyss, yyssp); - while (yyssp != yyss) - { - yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp); - YYPOPSTACK (1); - } + while (yyssp != yyss) + { + yydestruct("Cleanup: popping", yystos[*yyssp], yyvsp); + YYPOPSTACK(1); + } #ifndef yyoverflow - if (yyss != yyssa) - YYSTACK_FREE (yyss); + if (yyss != yyssa) + YYSTACK_FREE(yyss); #endif #if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); + if (yymsg != yymsgbuf) + YYSTACK_FREE(yymsg); #endif - /* Make sure YYID is used. */ - return YYID (yyresult); + /* Make sure YYID is used. */ + return YYID(yyresult); } - - - - void grammar_init(ParseTree* _parseTree, bool debug) { - parseTree = _parseTree; + parseTree = _parseTree; - if (debug) - yydebug = 1; + if (debug) + yydebug = 1; } void set_schema(std::string schema) { - db_schema = schema; + db_schema = schema; } - diff --git a/dbcon/ddlpackage/ddl-gram-win.h b/dbcon/ddlpackage/ddl-gram-win.h index adcb6ca30..12ad84b86 100644 --- a/dbcon/ddlpackage/ddl-gram-win.h +++ b/dbcon/ddlpackage/ddl-gram-win.h @@ -32,143 +32,134 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ - /* Tokens. */ #ifndef YYTOKENTYPE -# define YYTOKENTYPE +#define YYTOKENTYPE /* Put the tokens into the symbol table, so that GDB and other debuggers know about them. */ enum yytokentype { - ACTION = 258, - ADD = 259, - ALTER = 260, - AUTO_INCREMENT = 261, - BIGINT = 262, - BIT = 263, - IDB_BLOB = 264, - CASCADE = 265, - IDB_CHAR = 266, - CHARACTER = 267, - CHECK = 268, - CLOB = 269, - COLUMN = 270, - COLUMNS = 271, - COMMENT = 272, - CONSTRAINT = 273, - CONSTRAINTS = 274, - CREATE = 275, - CURRENT_USER = 276, - DATETIME = 277, - DEC = 278, - DECIMAL = 279, - DEFAULT = 280, - DEFERRABLE = 281, - DEFERRED = 282, - IDB_DELETE = 283, - DROP = 284, - ENGINE = 285, - FOREIGN = 286, - FULL = 287, - IMMEDIATE = 288, - INDEX = 289, - INITIALLY = 290, - IDB_INT = 291, - INTEGER = 292, - KEY = 293, - MATCH = 294, - MAX_ROWS = 295, - MIN_ROWS = 296, - MODIFY = 297, - NO = 298, - NOT = 299, - NULL_TOK = 300, - NUMBER = 301, - NUMERIC = 302, - ON = 303, - PARTIAL = 304, - PRECISION = 305, - PRIMARY = 306, - REFERENCES = 307, - RENAME = 308, - RESTRICT = 309, - SET = 310, - SMALLINT = 311, - TABLE = 312, - TIME = 313, - TINYINT = 314, - TO = 315, - UNIQUE = 316, - UNSIGNED = 317, - UPDATE = 318, - USER = 319, - SESSION_USER = 320, - SYSTEM_USER = 321, - VARCHAR = 322, - VARBINARY = 323, - VARYING = 324, - WITH = 325, - ZONE = 326, - DOUBLE = 327, - IDB_FLOAT = 328, - REAL = 329, - CHARSET = 330, - IDB_IF = 331, - EXISTS = 332, - CHANGE = 333, - TRUNCATE = 334, - IDENT = 335, - FCONST = 336, - SCONST = 337, - CP_SEARCH_CONDITION_TEXT = 338, - ICONST = 339, - DATE = 340 + ACTION = 258, + ADD = 259, + ALTER = 260, + AUTO_INCREMENT = 261, + BIGINT = 262, + BIT = 263, + IDB_BLOB = 264, + CASCADE = 265, + IDB_CHAR = 266, + CHARACTER = 267, + CHECK = 268, + CLOB = 269, + COLUMN = 270, + COLUMNS = 271, + COMMENT = 272, + CONSTRAINT = 273, + CONSTRAINTS = 274, + CREATE = 275, + CURRENT_USER = 276, + DATETIME = 277, + DEC = 278, + DECIMAL = 279, + DEFAULT = 280, + DEFERRABLE = 281, + DEFERRED = 282, + IDB_DELETE = 283, + DROP = 284, + ENGINE = 285, + FOREIGN = 286, + FULL = 287, + IMMEDIATE = 288, + INDEX = 289, + INITIALLY = 290, + IDB_INT = 291, + INTEGER = 292, + KEY = 293, + MATCH = 294, + MAX_ROWS = 295, + MIN_ROWS = 296, + MODIFY = 297, + NO = 298, + NOT = 299, + NULL_TOK = 300, + NUMBER = 301, + NUMERIC = 302, + ON = 303, + PARTIAL = 304, + PRECISION = 305, + PRIMARY = 306, + REFERENCES = 307, + RENAME = 308, + RESTRICT = 309, + SET = 310, + SMALLINT = 311, + TABLE = 312, + TIME = 313, + TINYINT = 314, + TO = 315, + UNIQUE = 316, + UNSIGNED = 317, + UPDATE = 318, + USER = 319, + SESSION_USER = 320, + SYSTEM_USER = 321, + VARCHAR = 322, + VARBINARY = 323, + VARYING = 324, + WITH = 325, + ZONE = 326, + DOUBLE = 327, + IDB_FLOAT = 328, + REAL = 329, + CHARSET = 330, + IDB_IF = 331, + EXISTS = 332, + CHANGE = 333, + TRUNCATE = 334, + IDENT = 335, + FCONST = 336, + SCONST = 337, + CP_SEARCH_CONDITION_TEXT = 338, + ICONST = 339, + DATE = 340 }; #endif - - -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED typedef union YYSTYPE { - - - ddlpackage::AlterTableStatement* alterTableStmt; - ddlpackage::AlterTableAction* ata; - ddlpackage::AlterTableActionList* ataList; - ddlpackage::DDL_CONSTRAINT_ATTRIBUTES cattr; - std::pair* tableOption; - const char* columnOption; - ddlpackage::ColumnConstraintDef* columnConstraintDef; - ddlpackage::ColumnNameList* columnNameList; - ddlpackage::ColumnType* columnType; - ddlpackage::ConstraintAttributes* constraintAttributes; - ddlpackage::ColumnConstraintList* constraintList; - ddlpackage::DDL_CONSTRAINTS constraintType; - double dval; - bool flag; - int ival; - ddlpackage::QualifiedName* qualifiedName; - ddlpackage::SchemaObject* schemaObject; - ddlpackage::SqlStatement* sqlStmt; - ddlpackage::SqlStatementList* sqlStmtList; - const char* str; - ddlpackage::TableConstraintDef* tableConstraint; - ddlpackage::TableElementList* tableElementList; - ddlpackage::TableOptionMap* tableOptionMap; - ddlpackage::ColumnDefaultValue* colDefault; - ddlpackage::DDL_MATCH_TYPE matchType; - ddlpackage::DDL_REFERENTIAL_ACTION refActionCode; - ddlpackage::ReferentialAction* refAction; - - + ddlpackage::AlterTableStatement* alterTableStmt; + ddlpackage::AlterTableAction* ata; + ddlpackage::AlterTableActionList* ataList; + ddlpackage::DDL_CONSTRAINT_ATTRIBUTES cattr; + std::pair* tableOption; + const char* columnOption; + ddlpackage::ColumnConstraintDef* columnConstraintDef; + ddlpackage::ColumnNameList* columnNameList; + ddlpackage::ColumnType* columnType; + ddlpackage::ConstraintAttributes* constraintAttributes; + ddlpackage::ColumnConstraintList* constraintList; + ddlpackage::DDL_CONSTRAINTS constraintType; + double dval; + bool flag; + int ival; + ddlpackage::QualifiedName* qualifiedName; + ddlpackage::SchemaObject* schemaObject; + ddlpackage::SqlStatement* sqlStmt; + ddlpackage::SqlStatementList* sqlStmtList; + const char* str; + ddlpackage::TableConstraintDef* tableConstraint; + ddlpackage::TableElementList* tableElementList; + ddlpackage::TableOptionMap* tableOptionMap; + ddlpackage::ColumnDefaultValue* colDefault; + ddlpackage::DDL_MATCH_TYPE matchType; + ddlpackage::DDL_REFERENTIAL_ACTION refActionCode; + ddlpackage::ReferentialAction* refAction; } YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 +#define YYSTYPE_IS_TRIVIAL 1 +#define yystype YYSTYPE /* obsolescent; will be withdrawn */ +#define YYSTYPE_IS_DECLARED 1 #endif extern YYSTYPE ddllval; - - diff --git a/dbcon/ddlpackage/ddl-scan-win.cpp b/dbcon/ddlpackage/ddl-scan-win.cpp index 937355094..11e0fbb00 100644 --- a/dbcon/ddlpackage/ddl-scan-win.cpp +++ b/dbcon/ddlpackage/ddl-scan-win.cpp @@ -27,7 +27,6 @@ #include - /* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */ #ifdef c_plusplus #ifndef __cplusplus @@ -35,7 +34,6 @@ #endif #endif - #ifdef __cplusplus #include @@ -47,19 +45,19 @@ /* The "const" storage-class-modifier is valid. */ #define YY_USE_CONST -#else /* ! __cplusplus */ +#else /* ! __cplusplus */ #if __STDC__ #define YY_USE_PROTOS #define YY_USE_CONST -#endif /* __STDC__ */ -#endif /* ! __cplusplus */ +#endif /* __STDC__ */ +#endif /* ! __cplusplus */ #ifdef __TURBOC__ -#pragma warn -rch -#pragma warn -use +#pragma warn - rch +#pragma warn - use #include #include #define YY_USE_CONST @@ -72,7 +70,6 @@ #define yyconst #endif - #ifdef YY_USE_PROTOS #define YY_PROTO(proto) proto #else @@ -87,7 +84,7 @@ * we want to instead treat it as an 8-bit unsigned char, hence the * double cast. */ -#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) +#define YY_SC_TO_UI(c) ((unsigned int)(unsigned char)c) /* Enter a start condition. This macro really ought to take a parameter, * but we do it the disgusting crufty way forced on us by the ()-less @@ -106,7 +103,7 @@ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin ) +#define YY_NEW_FILE yyrestart(yyin) #define YY_END_OF_BUFFER_CHAR 0 @@ -116,7 +113,7 @@ typedef struct yy_buffer_state* YY_BUFFER_STATE; extern int yyleng; -extern FILE* yyin, *yyout; +extern FILE *yyin, *yyout; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 @@ -138,18 +135,17 @@ extern FILE* yyin, *yyout; /* Return all but the first 'n' matched characters back to the input stream. */ -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - *yy_cp = yy_hold_char; \ - YY_RESTORE_YY_MORE_OFFSET \ - yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ - YY_DO_BEFORE_ACTION; /* set up yytext again */ \ - } \ - while ( 0 ) +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + *yy_cp = yy_hold_char; \ + YY_RESTORE_YY_MORE_OFFSET \ + yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } while (0) -#define unput(c) yyunput( c, yytext_ptr ) +#define unput(c) yyunput(c, yytext_ptr) /* The following is because we cannot portably get our hands on size_t * (without autoconf's help, which isn't available because we want @@ -157,61 +153,60 @@ extern FILE* yyin, *yyout; */ typedef unsigned int yy_size_t; - struct yy_buffer_state { - FILE* yy_input_file; + FILE* yy_input_file; - char* yy_ch_buf; /* input buffer */ - char* yy_buf_pos; /* current position in input buffer */ + char* yy_ch_buf; /* input buffer */ + char* yy_buf_pos; /* current position in input buffer */ - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - yy_size_t yy_buf_size; + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + yy_size_t yy_buf_size; - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - int yy_n_chars; + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; - int yy_buffer_status; + int yy_buffer_status; #define YY_BUFFER_NEW 0 #define YY_BUFFER_NORMAL 1 - /* When an EOF's been seen but there's still some text to process - * then we mark the buffer as YY_EOF_PENDING, to indicate that we - * shouldn't try reading from the input source any more. We might - * still have a bunch of tokens to match, though, because of - * possible backing-up. - * - * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by - * just pointing yyin at a new input file. - */ + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ #define YY_BUFFER_EOF_PENDING 2 }; @@ -223,389 +218,238 @@ static YY_BUFFER_STATE yy_current_buffer = 0; */ #define YY_CURRENT_BUFFER yy_current_buffer - /* yy_hold_char holds the character lost when yytext is formed. */ static char yy_hold_char; -static int yy_n_chars; /* number of characters read into yy_ch_buf */ - +static int yy_n_chars; /* number of characters read into yy_ch_buf */ int yyleng; /* Points to current character in buffer. */ -static char* yy_c_buf_p = (char*) 0; -static int yy_init = 1; /* whether we need to initialize */ -static int yy_start = 0; /* start state number */ +static char* yy_c_buf_p = (char*)0; +static int yy_init = 1; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ /* Flag which is used to allow yywrap()'s to do buffer switches * instead of setting up a fresh yyin. A bit of a hack ... */ static int yy_did_buffer_switch_on_eof; -void yyrestart YY_PROTO(( FILE* input_file )); +void yyrestart YY_PROTO((FILE * input_file)); -void yy_switch_to_buffer YY_PROTO(( YY_BUFFER_STATE new_buffer )); -void yy_load_buffer_state YY_PROTO(( void )); -YY_BUFFER_STATE yy_create_buffer YY_PROTO(( FILE* file, int size )); -void yy_delete_buffer YY_PROTO(( YY_BUFFER_STATE b )); -void yy_init_buffer YY_PROTO(( YY_BUFFER_STATE b, FILE* file )); -void yy_flush_buffer YY_PROTO(( YY_BUFFER_STATE b )); -#define YY_FLUSH_BUFFER yy_flush_buffer( yy_current_buffer ) +void yy_switch_to_buffer YY_PROTO((YY_BUFFER_STATE new_buffer)); +void yy_load_buffer_state YY_PROTO((void)); +YY_BUFFER_STATE yy_create_buffer YY_PROTO((FILE * file, int size)); +void yy_delete_buffer YY_PROTO((YY_BUFFER_STATE b)); +void yy_init_buffer YY_PROTO((YY_BUFFER_STATE b, FILE* file)); +void yy_flush_buffer YY_PROTO((YY_BUFFER_STATE b)); +#define YY_FLUSH_BUFFER yy_flush_buffer(yy_current_buffer) -YY_BUFFER_STATE yy_scan_buffer YY_PROTO(( char* base, yy_size_t size )); -YY_BUFFER_STATE yy_scan_string YY_PROTO(( yyconst char* yy_str )); -YY_BUFFER_STATE yy_scan_bytes YY_PROTO(( yyconst char* bytes, int len )); +YY_BUFFER_STATE yy_scan_buffer YY_PROTO((char* base, yy_size_t size)); +YY_BUFFER_STATE yy_scan_string YY_PROTO((yyconst char* yy_str)); +YY_BUFFER_STATE yy_scan_bytes YY_PROTO((yyconst char* bytes, int len)); -static void* yy_flex_alloc YY_PROTO(( yy_size_t )); -static void* yy_flex_realloc YY_PROTO(( void*, yy_size_t )); -static void yy_flex_free YY_PROTO(( void* )); +static void* yy_flex_alloc YY_PROTO((yy_size_t)); +static void* yy_flex_realloc YY_PROTO((void*, yy_size_t)); +static void yy_flex_free YY_PROTO((void*)); #define yy_new_buffer yy_create_buffer -#define yy_set_interactive(is_interactive) \ - { \ - if ( ! yy_current_buffer ) \ - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ - yy_current_buffer->yy_is_interactive = is_interactive; \ - } +#define yy_set_interactive(is_interactive) \ + { \ + if (!yy_current_buffer) \ + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); \ + yy_current_buffer->yy_is_interactive = is_interactive; \ + } -#define yy_set_bol(at_bol) \ - { \ - if ( ! yy_current_buffer ) \ - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ - yy_current_buffer->yy_at_bol = at_bol; \ - } +#define yy_set_bol(at_bol) \ + { \ + if (!yy_current_buffer) \ + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); \ + yy_current_buffer->yy_at_bol = at_bol; \ + } #define YY_AT_BOL() (yy_current_buffer->yy_at_bol) - #define yywrap() 1 #define YY_SKIP_YYWRAP typedef unsigned char YY_CHAR; -FILE* yyin = (FILE*) 0, *yyout = (FILE*) 0; +FILE *yyin = (FILE*)0, *yyout = (FILE*)0; typedef int yy_state_type; extern char* yytext; #define yytext_ptr yytext -static yy_state_type yy_get_previous_state YY_PROTO(( void )); -static yy_state_type yy_try_NUL_trans YY_PROTO(( yy_state_type current_state )); -static int yy_get_next_buffer YY_PROTO(( void )); -static void yy_fatal_error YY_PROTO(( yyconst char msg[] )); +static yy_state_type yy_get_previous_state YY_PROTO((void)); +static yy_state_type yy_try_NUL_trans YY_PROTO((yy_state_type current_state)); +static int yy_get_next_buffer YY_PROTO((void)); +static void yy_fatal_error YY_PROTO((yyconst char msg[])); /* Done after the current pattern has been matched and before the * corresponding action - sets up yytext. */ -#define YY_DO_BEFORE_ACTION \ - yytext_ptr = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ - yy_hold_char = *yy_cp; \ - *yy_cp = '\0'; \ - yy_c_buf_p = yy_cp; +#define YY_DO_BEFORE_ACTION \ + yytext_ptr = yy_bp; \ + yyleng = (int)(yy_cp - yy_bp); \ + yy_hold_char = *yy_cp; \ + *yy_cp = '\0'; \ + yy_c_buf_p = yy_cp; #define YY_NUM_RULES 85 #define YY_END_OF_BUFFER 86 -static yyconst short int yy_accept[393] = -{ - 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 86, 85, 81, 80, 83, 13, 83, 83, 83, 16, - 82, 84, 82, 82, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 82, 82, 82, 82, 82, 82, 10, - 12, 85, 11, 85, 14, 15, 81, 0, 16, 81, - 17, 17, 16, 0, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 82, 75, 82, 82, 82, 82, 82, - 82, 48, 82, 52, 82, 82, 82, 82, 82, 82, - 82, 82, 65, 82, 82, 82, 82, 82, 0, 11, +static yyconst short int yy_accept[393] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 85, 81, 80, 83, 13, 83, 83, 83, 16, 82, 84, 82, 82, 82, + 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 10, 12, 85, 11, 85, 14, 15, 81, 0, 16, 81, 17, + 17, 16, 0, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 75, 82, 82, + 82, 82, 82, 82, 48, 82, 52, 82, 82, 82, 82, 82, 82, 82, 82, 65, 82, 82, 82, 82, 82, 0, 11, - 0, 14, 81, 17, 0, 18, 82, 2, 82, 82, - 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, - 29, 82, 82, 82, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 41, 43, 82, 82, 82, 82, 49, - 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, - 61, 82, 82, 82, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 82, 82, 82, 82, 6, 82, 82, - 82, 82, 82, 82, 26, 82, 82, 82, 82, 82, - 34, 82, 82, 82, 82, 37, 82, 82, 82, 82, - 82, 82, 82, 82, 50, 82, 82, 82, 82, 73, + 0, 14, 81, 17, 0, 18, 82, 2, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 29, 82, 82, 82, 82, 82, + 82, 82, 82, 82, 82, 82, 82, 41, 43, 82, 82, 82, 82, 49, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 61, 82, + 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 6, 82, 82, 82, 82, 82, 82, 26, 82, 82, 82, + 82, 82, 34, 82, 82, 82, 82, 37, 82, 82, 82, 82, 82, 82, 82, 82, 50, 82, 82, 82, 82, 73, - 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, - 82, 82, 69, 82, 82, 82, 3, 82, 82, 82, - 82, 82, 82, 9, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 82, 82, 82, 82, 71, 82, 82, - 39, 82, 82, 44, 82, 82, 82, 82, 82, 82, - 82, 82, 82, 82, 82, 82, 82, 63, 82, 82, - 82, 82, 82, 82, 82, 1, 82, 8, 82, 77, - 82, 82, 20, 82, 82, 24, 82, 82, 82, 82, - 82, 33, 72, 35, 76, 82, 82, 82, 82, 82, - 82, 47, 82, 82, 82, 82, 82, 57, 82, 82, + 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 69, 82, 82, 82, 3, 82, 82, 82, 82, 82, 82, 9, 82, 82, + 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 82, 71, 82, 82, 39, 82, 82, 44, 82, 82, 82, 82, 82, 82, 82, 82, + 82, 82, 82, 82, 82, 63, 82, 82, 82, 82, 82, 82, 82, 1, 82, 8, 82, 77, 82, 82, 20, 82, 82, 24, 82, 82, + 82, 82, 82, 33, 72, 35, 76, 82, 82, 82, 82, 82, 82, 47, 82, 82, 82, 82, 82, 57, 82, 82, - 82, 82, 82, 82, 66, 82, 68, 82, 82, 82, - 5, 82, 74, 21, 19, 82, 82, 82, 28, 30, - 82, 82, 36, 82, 82, 42, 82, 82, 51, 53, - 82, 55, 82, 82, 82, 82, 82, 64, 82, 82, - 82, 70, 82, 82, 82, 82, 27, 82, 32, 82, - 82, 45, 46, 82, 82, 58, 82, 62, 82, 78, - 67, 82, 82, 7, 82, 82, 82, 38, 40, 54, - 82, 82, 82, 79, 82, 22, 82, 31, 56, 82, - 82, 82, 23, 82, 82, 60, 82, 25, 59, 82, - 4, 0 + 82, 82, 82, 82, 66, 82, 68, 82, 82, 82, 5, 82, 74, 21, 19, 82, 82, 82, 28, 30, 82, 82, 36, 82, 82, 42, + 82, 82, 51, 53, 82, 55, 82, 82, 82, 82, 82, 64, 82, 82, 82, 70, 82, 82, 82, 82, 27, 82, 32, 82, 82, 45, + 46, 82, 82, 58, 82, 62, 82, 78, 67, 82, 82, 7, 82, 82, 82, 38, 40, 54, 82, 82, 82, 79, 82, 22, 82, 31, + 56, 82, 82, 82, 23, 82, 82, 60, 82, 25, 59, 82, 4, 0 -} ; +}; -static yyconst int yy_ec[256] = -{ - 0, - 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, - 1, 2, 4, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 1, 1, 1, 5, 6, 1, 7, 8, - 9, 6, 10, 6, 11, 12, 6, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 6, 6, 6, - 6, 6, 1, 1, 16, 17, 18, 19, 20, 21, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 25, - 6, 1, 6, 6, 14, 15, 16, 17, 18, 19, +static yyconst int yy_ec[256] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 2, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 5, 6, 1, 7, 8, 9, 6, 10, 6, 11, 12, 6, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 6, 6, 6, 6, 6, 1, 1, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 25, 6, 1, 6, 6, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - 40, 25, 1, 1, 1, 1, 1, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 25, 1, 1, 1, 1, + 1, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25 -} ; + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25}; -static yyconst int yy_meta[41] = -{ - 0, - 1, 1, 2, 2, 3, 1, 1, 1, 1, 1, - 1, 1, 3, 3, 1, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 -} ; +static yyconst int yy_meta[41] = {0, 1, 1, 2, 2, 3, 1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}; -static yyconst short int yy_base[400] = -{ - 0, - 0, 0, 33, 35, 428, 427, 428, 427, 426, 425, - 431, 438, 43, 46, 438, 438, 39, 42, 417, 44, - 0, 438, 40, 405, 45, 49, 31, 36, 56, 408, - 64, 53, 398, 57, 406, 67, 75, 67, 409, 438, - 438, 415, 438, 416, 438, 438, 100, 409, 80, 0, - 93, 96, 98, 104, 0, 386, 401, 384, 383, 395, - 382, 103, 93, 395, 381, 378, 106, 376, 381, 388, - 385, 378, 374, 379, 0, 377, 106, 364, 93, 374, - 383, 366, 107, 0, 367, 116, 110, 103, 383, 364, - 380, 367, 0, 359, 118, 375, 373, 359, 382, 438, +static yyconst short int yy_base[400] = { + 0, 0, 0, 33, 35, 428, 427, 428, 427, 426, 425, 431, 438, 43, 46, 438, 438, 39, 42, 417, 44, + 0, 438, 40, 405, 45, 49, 31, 36, 56, 408, 64, 53, 398, 57, 406, 67, 75, 67, 409, 438, 438, + 415, 438, 416, 438, 438, 100, 409, 80, 0, 93, 96, 98, 104, 0, 386, 401, 384, 383, 395, 382, 103, + 93, 395, 381, 378, 106, 376, 381, 388, 385, 378, 374, 379, 0, 377, 106, 364, 93, 374, 383, 366, 107, + 0, 367, 116, 110, 103, 383, 364, 380, 367, 0, 359, 118, 375, 373, 359, 382, 438, - 383, 438, 0, 130, 376, 375, 363, 0, 366, 355, - 360, 365, 116, 364, 345, 352, 345, 362, 344, 356, - 351, 131, 354, 356, 341, 347, 336, 353, 348, 340, - 346, 345, 329, 343, 0, 344, 347, 346, 335, 0, - 331, 337, 321, 337, 326, 326, 332, 335, 315, 315, - 0, 321, 312, 319, 305, 315, 311, 318, 325, 307, - 136, 309, 305, 323, 307, 319, 312, 130, 307, 304, - 311, 295, 294, 308, 292, 298, 289, 291, 288, 295, - 0, 292, 285, 284, 294, 0, 298, 277, 291, 292, - 290, 279, 278, 289, 0, 276, 284, 283, 290, 0, + 383, 438, 0, 130, 376, 375, 363, 0, 366, 355, 360, 365, 116, 364, 345, 352, 345, 362, 344, 356, 351, + 131, 354, 356, 341, 347, 336, 353, 348, 340, 346, 345, 329, 343, 0, 344, 347, 346, 335, 0, 331, 337, + 321, 337, 326, 326, 332, 335, 315, 315, 0, 321, 312, 319, 305, 315, 311, 318, 325, 307, 136, 309, 305, + 323, 307, 319, 312, 130, 307, 304, 311, 295, 294, 308, 292, 298, 289, 291, 288, 295, 0, 292, 285, 284, + 294, 0, 298, 277, 291, 292, 290, 279, 278, 289, 0, 276, 284, 283, 290, 0, - 272, 276, 270, 278, 274, 280, 279, 274, 279, 260, - 273, 259, 0, 269, 269, 262, 0, 266, 254, 269, - 267, 268, 265, 0, 255, 254, 249, 261, 251, 255, - 262, 250, 243, 255, 254, 253, 238, 0, 249, 246, - 0, 253, 248, 0, 237, 236, 225, 240, 247, 228, - 228, 240, 239, 234, 227, 232, 227, 0, 225, 237, - 232, 222, 230, 220, 232, 0, 218, 0, 226, 0, - 210, 209, 209, 207, 225, 0, 205, 211, 211, 202, - 139, 0, 0, 0, 0, 207, 219, 207, 200, 194, - 193, 0, 212, 202, 204, 187, 197, 0, 207, 195, + 272, 276, 270, 278, 274, 280, 279, 274, 279, 260, 273, 259, 0, 269, 269, 262, 0, 266, 254, 269, 267, + 268, 265, 0, 255, 254, 249, 261, 251, 255, 262, 250, 243, 255, 254, 253, 238, 0, 249, 246, 0, 253, + 248, 0, 237, 236, 225, 240, 247, 228, 228, 240, 239, 234, 227, 232, 227, 0, 225, 237, 232, 222, 230, + 220, 232, 0, 218, 0, 226, 0, 210, 209, 209, 207, 225, 0, 205, 211, 211, 202, 139, 0, 0, 0, + 0, 207, 219, 207, 200, 194, 193, 0, 212, 202, 204, 187, 197, 0, 207, 195, - 194, 208, 186, 185, 0, 199, 0, 202, 184, 198, - 0, 195, 0, 0, 0, 190, 199, 192, 0, 0, - 194, 191, 0, 174, 181, 0, 173, 172, 0, 0, - 175, 0, 186, 168, 188, 166, 164, 0, 179, 179, - 164, 0, 163, 162, 165, 157, 0, 165, 0, 171, - 150, 0, 0, 160, 168, 0, 151, 0, 152, 0, - 0, 145, 164, 0, 148, 147, 143, 0, 0, 0, - 128, 127, 140, 0, 130, 123, 136, 0, 0, 128, - 96, 92, 0, 64, 53, 0, 45, 0, 0, 36, - 0, 438, 164, 167, 170, 59, 173, 176, 179 + 194, 208, 186, 185, 0, 199, 0, 202, 184, 198, 0, 195, 0, 0, 0, 190, 199, 192, 0, 0, 194, + 191, 0, 174, 181, 0, 173, 172, 0, 0, 175, 0, 186, 168, 188, 166, 164, 0, 179, 179, 164, 0, + 163, 162, 165, 157, 0, 165, 0, 171, 150, 0, 0, 160, 168, 0, 151, 0, 152, 0, 0, 145, 164, + 0, 148, 147, 143, 0, 0, 0, 128, 127, 140, 0, 130, 123, 136, 0, 0, 128, 96, 92, 0, 64, + 53, 0, 45, 0, 0, 36, 0, 438, 164, 167, 170, 59, 173, 176, 179 -} ; +}; -static yyconst short int yy_def[400] = -{ - 0, - 392, 1, 393, 393, 394, 394, 395, 395, 393, 393, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 396, 392, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 392, - 392, 397, 392, 398, 392, 392, 392, 392, 392, 399, - 392, 392, 392, 392, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 397, 392, +static yyconst short int yy_def[400] = { + 0, 392, 1, 393, 393, 394, 394, 395, 395, 393, 393, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, + 396, 392, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 392, 392, + 397, 392, 398, 392, 392, 392, 392, 392, 399, 392, 392, 392, 392, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 397, 392, - 398, 392, 399, 392, 392, 392, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 398, 392, 399, 392, 392, 392, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, - 396, 0, 392, 392, 392, 392, 392, 392, 392 + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 0, 392, 392, 392, 392, 392, 392, 392 -} ; +}; -static yyconst short int yy_nxt[479] = -{ - 0, - 12, 13, 14, 13, 12, 15, 16, 15, 15, 17, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 21, 21, 29, 21, 30, 21, 31, 32, 33, - 34, 21, 35, 36, 37, 38, 39, 21, 21, 21, - 40, 41, 40, 41, 47, 47, 47, 47, 47, 47, - 48, 49, 50, 48, 49, 52, 53, 56, 57, 70, - 61, 55, 72, 54, 66, 73, 58, 62, 67, 71, - 391, 74, 85, 390, 63, 59, 75, 64, 68, 79, - 65, 69, 82, 76, 77, 389, 88, 80, 83, 86, - 91, 48, 49, 81, 89, 95, 388, 96, 92, 54, +static yyconst short int yy_nxt[479] = { + 0, 12, 13, 14, 13, 12, 15, 16, 15, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 21, 21, 29, 21, 30, 21, 31, 32, 33, 34, 21, 35, 36, 37, 38, 39, 21, 21, 21, 40, + 41, 40, 41, 47, 47, 47, 47, 47, 47, 48, 49, 50, 48, 49, 52, 53, 56, 57, 70, 61, 55, + 72, 54, 66, 73, 58, 62, 67, 71, 391, 74, 85, 390, 63, 59, 75, 64, 68, 79, 65, 69, 82, + 76, 77, 389, 88, 80, 83, 86, 91, 48, 49, 81, 89, 95, 388, 96, 92, 54, - 97, 47, 47, 47, 93, 51, 90, 94, 104, 52, - 53, 387, 54, 105, 105, 54, 106, 54, 113, 115, - 116, 117, 114, 121, 132, 146, 122, 136, 386, 133, - 147, 137, 123, 141, 142, 144, 150, 151, 148, 145, - 134, 157, 104, 149, 167, 222, 177, 385, 168, 54, - 178, 158, 214, 215, 321, 384, 383, 382, 322, 381, - 380, 379, 378, 223, 12, 12, 12, 42, 42, 42, - 44, 44, 44, 99, 99, 99, 101, 101, 101, 103, - 377, 103, 376, 375, 374, 373, 372, 371, 370, 369, - 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, + 97, 47, 47, 47, 93, 51, 90, 94, 104, 52, 53, 387, 54, 105, 105, 54, 106, 54, 113, 115, 116, + 117, 114, 121, 132, 146, 122, 136, 386, 133, 147, 137, 123, 141, 142, 144, 150, 151, 148, 145, 134, 157, + 104, 149, 167, 222, 177, 385, 168, 54, 178, 158, 214, 215, 321, 384, 383, 382, 322, 381, 380, 379, 378, + 223, 12, 12, 12, 42, 42, 42, 44, 44, 44, 99, 99, 99, 101, 101, 101, 103, 377, 103, 376, 375, + 374, 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, - 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, - 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, - 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, - 328, 327, 326, 325, 324, 323, 320, 319, 318, 317, - 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, - 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, - 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, - 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, - 276, 275, 274, 273, 272, 271, 270, 269, 268, 267, - 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, + 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, + 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 320, 319, 318, 317, 316, 315, + 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, + 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, + 272, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, - 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, - 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, - 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, - 226, 225, 224, 221, 220, 219, 218, 217, 216, 213, - 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, - 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, - 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, - 182, 181, 180, 179, 176, 175, 174, 173, 172, 171, - 170, 169, 166, 165, 164, 163, 162, 106, 106, 102, - 100, 161, 160, 159, 156, 155, 154, 153, 152, 143, + 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, + 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 221, 220, 219, 218, 217, 216, 213, 212, 211, + 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, + 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 176, 175, 174, 173, 172, 171, 170, 169, 166, 165, + 164, 163, 162, 106, 106, 102, 100, 161, 160, 159, 156, 155, 154, 153, 152, 143, - 140, 139, 138, 135, 131, 130, 129, 128, 127, 126, - 125, 124, 120, 119, 118, 112, 111, 110, 109, 108, - 107, 51, 102, 100, 98, 87, 84, 78, 60, 51, - 392, 46, 46, 45, 45, 43, 43, 11, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392 -} ; + 140, 139, 138, 135, 131, 130, 129, 128, 127, 126, 125, 124, 120, 119, 118, 112, 111, 110, 109, 108, 107, + 51, 102, 100, 98, 87, 84, 78, 60, 51, 392, 46, 46, 45, 45, 43, 43, 11, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392}; -static yyconst short int yy_chk[479] = -{ - 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 3, 3, 4, 4, 13, 13, 13, 14, 14, 14, - 17, 17, 18, 18, 18, 20, 20, 23, 23, 27, - 25, 396, 28, 20, 26, 28, 23, 25, 26, 27, - 390, 28, 34, 387, 25, 23, 29, 25, 26, 31, - 25, 26, 32, 29, 29, 385, 36, 31, 32, 34, - 37, 49, 49, 31, 36, 38, 384, 38, 37, 49, +static yyconst short int yy_chk[479] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, + 3, 4, 4, 13, 13, 13, 14, 14, 14, 17, 17, 18, 18, 18, 20, 20, 23, 23, 27, 25, 396, + 28, 20, 26, 28, 23, 25, 26, 27, 390, 28, 34, 387, 25, 23, 29, 25, 26, 31, 25, 26, 32, + 29, 29, 385, 36, 31, 32, 34, 37, 49, 49, 31, 36, 38, 384, 38, 37, 49, - 38, 47, 47, 47, 37, 51, 36, 37, 52, 53, - 53, 382, 51, 54, 54, 52, 54, 53, 62, 63, - 63, 63, 62, 67, 77, 87, 67, 79, 381, 77, - 87, 79, 67, 83, 83, 86, 88, 88, 87, 86, - 77, 95, 104, 87, 113, 168, 122, 380, 113, 104, - 122, 95, 161, 161, 281, 377, 376, 375, 281, 373, - 372, 371, 367, 168, 393, 393, 393, 394, 394, 394, - 395, 395, 395, 397, 397, 397, 398, 398, 398, 399, - 366, 399, 365, 363, 362, 359, 357, 355, 354, 351, - 350, 348, 346, 345, 344, 343, 341, 340, 339, 337, + 38, 47, 47, 47, 37, 51, 36, 37, 52, 53, 53, 382, 51, 54, 54, 52, 54, 53, 62, 63, 63, + 63, 62, 67, 77, 87, 67, 79, 381, 77, 87, 79, 67, 83, 83, 86, 88, 88, 87, 86, 77, 95, + 104, 87, 113, 168, 122, 380, 113, 104, 122, 95, 161, 161, 281, 377, 376, 375, 281, 373, 372, 371, 367, + 168, 393, 393, 393, 394, 394, 394, 395, 395, 395, 397, 397, 397, 398, 398, 398, 399, 366, 399, 365, 363, + 362, 359, 357, 355, 354, 351, 350, 348, 346, 345, 344, 343, 341, 340, 339, 337, - 336, 335, 334, 333, 331, 328, 327, 325, 324, 322, - 321, 318, 317, 316, 312, 310, 309, 308, 306, 304, - 303, 302, 301, 300, 299, 297, 296, 295, 294, 293, - 291, 290, 289, 288, 287, 286, 280, 279, 278, 277, - 275, 274, 273, 272, 271, 269, 267, 265, 264, 263, - 262, 261, 260, 259, 257, 256, 255, 254, 253, 252, - 251, 250, 249, 248, 247, 246, 245, 243, 242, 240, - 239, 237, 236, 235, 234, 233, 232, 231, 230, 229, - 228, 227, 226, 225, 223, 222, 221, 220, 219, 218, - 216, 215, 214, 212, 211, 210, 209, 208, 207, 206, + 336, 335, 334, 333, 331, 328, 327, 325, 324, 322, 321, 318, 317, 316, 312, 310, 309, 308, 306, 304, 303, + 302, 301, 300, 299, 297, 296, 295, 294, 293, 291, 290, 289, 288, 287, 286, 280, 279, 278, 277, 275, 274, + 273, 272, 271, 269, 267, 265, 264, 263, 262, 261, 260, 259, 257, 256, 255, 254, 253, 252, 251, 250, 249, + 248, 247, 246, 245, 243, 242, 240, 239, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, + 223, 222, 221, 220, 219, 218, 216, 215, 214, 212, 211, 210, 209, 208, 207, 206, - 205, 204, 203, 202, 201, 199, 198, 197, 196, 194, - 193, 192, 191, 190, 189, 188, 187, 185, 184, 183, - 182, 180, 179, 178, 177, 176, 175, 174, 173, 172, - 171, 170, 169, 167, 166, 165, 164, 163, 162, 160, - 159, 158, 157, 156, 155, 154, 153, 152, 150, 149, - 148, 147, 146, 145, 144, 143, 142, 141, 139, 138, - 137, 136, 134, 133, 132, 131, 130, 129, 128, 127, - 126, 125, 124, 123, 121, 120, 119, 118, 117, 116, - 115, 114, 112, 111, 110, 109, 107, 106, 105, 101, - 99, 98, 97, 96, 94, 92, 91, 90, 89, 85, + 205, 204, 203, 202, 201, 199, 198, 197, 196, 194, 193, 192, 191, 190, 189, 188, 187, 185, 184, 183, 182, + 180, 179, 178, 177, 176, 175, 174, 173, 172, 171, 170, 169, 167, 166, 165, 164, 163, 162, 160, 159, 158, + 157, 156, 155, 154, 153, 152, 150, 149, 148, 147, 146, 145, 144, 143, 142, 141, 139, 138, 137, 136, 134, + 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 121, 120, 119, 118, 117, 116, 115, 114, 112, 111, + 110, 109, 107, 106, 105, 101, 99, 98, 97, 96, 94, 92, 91, 90, 89, 85, - 82, 81, 80, 78, 76, 74, 73, 72, 71, 70, - 69, 68, 66, 65, 64, 61, 60, 59, 58, 57, - 56, 48, 44, 42, 39, 35, 33, 30, 24, 19, - 11, 10, 9, 8, 7, 6, 5, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 392, 392 -} ; + 82, 81, 80, 78, 76, 74, 73, 72, 71, 70, 69, 68, 66, 65, 64, 61, 60, 59, 58, 57, 56, + 48, 44, 42, 39, 35, 33, 30, 24, 19, 11, 10, 9, 8, 7, 6, 5, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, 392}; static yy_state_type yy_last_accepting_state; static char* yy_last_accepting_cpos; @@ -654,7 +498,7 @@ void ddlerror(char* s); static YY_BUFFER_STATE scanbufhandle; static char* scanbuf; -static char* scanner_copy (char* str); +static char* scanner_copy(char* str); #define YY_NO_UNPUT 1 /* %option header-file="ddl-scan.h" */ @@ -666,36 +510,35 @@ static char* scanner_copy (char* str); #define endquote 4 - /* Macros after this point can all be overridden by user definitions in * section 1. */ #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap YY_PROTO(( void )); +extern "C" int yywrap YY_PROTO((void)); #else -extern int yywrap YY_PROTO(( void )); +extern int yywrap YY_PROTO((void)); #endif #endif #ifndef YY_NO_UNPUT -static void yyunput YY_PROTO(( int c, char* buf_ptr )); +static void yyunput YY_PROTO((int c, char* buf_ptr)); #endif #ifndef yytext_ptr -static void yy_flex_strncpy YY_PROTO(( char*, yyconst char*, int )); +static void yy_flex_strncpy YY_PROTO((char*, yyconst char*, int)); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen YY_PROTO(( yyconst char* )); +static int yy_flex_strlen YY_PROTO((yyconst char*)); #endif #ifndef YY_NO_INPUT #ifdef __cplusplus -static int yyinput YY_PROTO(( void )); +static int yyinput YY_PROTO((void)); #else -static int input YY_PROTO(( void )); +static int input YY_PROTO((void)); #endif #endif @@ -704,13 +547,13 @@ static int yy_start_stack_ptr = 0; static int yy_start_stack_depth = 0; static int* yy_start_stack = 0; #ifndef YY_NO_PUSH_STATE -static void yy_push_state YY_PROTO(( int new_state )); +static void yy_push_state YY_PROTO((int new_state)); #endif #ifndef YY_NO_POP_STATE -static void yy_pop_state YY_PROTO(( void )); +static void yy_pop_state YY_PROTO((void)); #endif #ifndef YY_NO_TOP_STATE -static int yy_top_state YY_PROTO(( void )); +static int yy_top_state YY_PROTO((void)); #endif #else @@ -745,29 +588,27 @@ YY_MALLOC_DECL /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO (void) fwrite( yytext, yyleng, 1, yyout ) +#define ECHO (void)fwrite(yytext, yyleng, 1, yyout) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, * is returned in "result". */ #ifndef YY_INPUT -#define YY_INPUT(buf,result,max_size) \ - if ( yy_current_buffer->yy_is_interactive ) \ - { \ - int c = '*', n; \ - for ( n = 0; n < max_size && \ - (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ - buf[n] = (char) c; \ - if ( c == '\n' ) \ - buf[n++] = (char) c; \ - if ( c == EOF && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); \ - result = n; \ - } \ - else if ( ((result = fread( buf, 1, max_size, yyin )) == 0) \ - && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); +#define YY_INPUT(buf, result, max_size) \ + if (yy_current_buffer->yy_is_interactive) \ + { \ + int c = '*', n; \ + for (n = 0; n < max_size && (c = getc(yyin)) != EOF && c != '\n'; ++n) \ + buf[n] = (char)c; \ + if (c == '\n') \ + buf[n++] = (char)c; \ + if (c == EOF && ferror(yyin)) \ + YY_FATAL_ERROR("input in flex scanner failed"); \ + result = n; \ + } \ + else if (((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin)) \ + YY_FATAL_ERROR("input in flex scanner failed"); #endif /* No semi-colon after return; correct usage is to write "yyterminate();" - @@ -785,14 +626,14 @@ YY_MALLOC_DECL /* Report a fatal error. */ #ifndef YY_FATAL_ERROR -#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#define YY_FATAL_ERROR(msg) yy_fatal_error(msg) #endif /* Default declaration of generated scanner - a define so the user can * easily add parameters. */ #ifndef YY_DECL -#define YY_DECL int yylex YY_PROTO(( void )) +#define YY_DECL int yylex YY_PROTO((void)) #endif /* Code executed at the beginning of each rule, after yytext and yyleng @@ -807,683 +648,846 @@ YY_MALLOC_DECL #define YY_BREAK break; #endif -#define YY_RULE_SETUP \ - YY_USER_ACTION +#define YY_RULE_SETUP YY_USER_ACTION YY_DECL { - register yy_state_type yy_current_state; - register char* yy_cp, *yy_bp; - register int yy_act; + register yy_state_type yy_current_state; + register char *yy_cp, *yy_bp; + register int yy_act; - - - - if ( yy_init ) - { - yy_init = 0; + if (yy_init) + { + yy_init = 0; #ifdef YY_USER_INIT - YY_USER_INIT; + YY_USER_INIT; #endif - if ( ! yy_start ) - yy_start = 1; /* first start state */ + if (!yy_start) + yy_start = 1; /* first start state */ - if ( ! yyin ) - yyin = stdin; + if (!yyin) + yyin = stdin; - if ( ! yyout ) - yyout = stdout; + if (!yyout) + yyout = stdout; - if ( ! yy_current_buffer ) - yy_current_buffer = - yy_create_buffer( yyin, YY_BUF_SIZE ); + if (!yy_current_buffer) + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); - yy_load_buffer_state(); + yy_load_buffer_state(); + } + + while (1) /* loops until end-of-file is reached */ + { + yy_cp = yy_c_buf_p; + + /* Support of yytext. */ + *yy_cp = yy_hold_char; + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = yy_start; + yy_match: + + do + { + register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + + if (yy_accept[yy_current_state]) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; + + if (yy_current_state >= 393) + yy_c = yy_meta[(unsigned int)yy_c]; + } + + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + ++yy_cp; + } while (yy_base[yy_current_state] != 438); + + yy_find_action: + yy_act = yy_accept[yy_current_state]; + + if (yy_act == 0) + { + /* have to back up */ + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + yy_act = yy_accept[yy_current_state]; } - while ( 1 ) /* loops until end-of-file is reached */ + YY_DO_BEFORE_ACTION; + + do_action: /* This label is used only to access EOF actions. */ + + switch (yy_act) { - yy_cp = yy_c_buf_p; - - /* Support of yytext. */ + /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ *yy_cp = yy_hold_char; + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + goto yy_find_action; - /* yy_bp points to the position in yy_ch_buf of the start of - * the current run. + case 1: + YY_RULE_SETUP + { + return ACTION; + } + YY_BREAK + + case 2: + YY_RULE_SETUP + { + return ADD; + } + YY_BREAK + + case 3: + YY_RULE_SETUP + { + return ALTER; + } + YY_BREAK + + case 4: + YY_RULE_SETUP + { + return AUTO_INCREMENT; + } + YY_BREAK + + case 5: + YY_RULE_SETUP + { + return CASCADE; + } + YY_BREAK + + case 6: + YY_RULE_SETUP + { + return IDB_CHAR; + } + YY_BREAK + + case 7: + YY_RULE_SETUP + { + return IDB_CHAR; + } + YY_BREAK + + case 8: + YY_RULE_SETUP + { + return BIGINT; + } + YY_BREAK + + case 9: + YY_RULE_SETUP + { + BEGIN(check1); + return CHECK; + } + YY_BREAK + + case 10: + YY_RULE_SETUP + { + BEGIN(check2); + return '('; + } + YY_BREAK + + case 11: + *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ + yy_c_buf_p = yy_cp -= 1; + YY_DO_BEFORE_ACTION; /* set up yytext again */ + YY_RULE_SETUP + { + BEGIN(check1); + ddllval.str = scanner_copy(ddltext); + return CP_SEARCH_CONDITION_TEXT; + } + YY_BREAK + + case 12: + YY_RULE_SETUP + { + BEGIN(0); + return ')'; + } + YY_BREAK + + case 13: + YY_RULE_SETUP + { + BEGIN(inquote); + return yytext[0]; + } + YY_BREAK + + case 14: + *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ + yy_c_buf_p = yy_cp -= 1; + YY_DO_BEFORE_ACTION; /* set up yytext again */ + YY_RULE_SETUP + { + BEGIN(endquote); + ddllval.str = scanner_copy(ddltext); + return SCONST; + } + YY_BREAK + + case 15: + YY_RULE_SETUP + { + BEGIN(0); + return yytext[0]; + } + YY_BREAK + + case 16: + YY_RULE_SETUP + { + ddllval.str = scanner_copy(ddltext); + return ICONST; + } + YY_BREAK + + case 17: + YY_RULE_SETUP + { + ddllval.str = scanner_copy(ddltext); + return FCONST; + } + YY_BREAK + + case 18: + YY_RULE_SETUP + { + ddllval.str = scanner_copy(ddltext); + return FCONST; + } + YY_BREAK + + case 19: + YY_RULE_SETUP + { + return COMMENT; + } + YY_BREAK + + case 20: + YY_RULE_SETUP + { + return COLUMN; + } + YY_BREAK + + case 21: + YY_RULE_SETUP + { + return COLUMNS; + } + YY_BREAK + + case 22: + YY_RULE_SETUP + { + return CONSTRAINT; + } + YY_BREAK + + case 23: + YY_RULE_SETUP + { + return CONSTRAINTS; + } + YY_BREAK + + case 24: + YY_RULE_SETUP + { + return CREATE; + } + YY_BREAK + + case 25: + YY_RULE_SETUP + { + return CURRENT_USER; + } + YY_BREAK + + case 26: + YY_RULE_SETUP + { + ddllval.str = strdup("date"); + return DATE; + } + YY_BREAK + + case 27: + YY_RULE_SETUP + { + return DATETIME; + } + YY_BREAK + + case 28: + YY_RULE_SETUP + { + return DECIMAL; + } + YY_BREAK + + case 29: + YY_RULE_SETUP + { + return DECIMAL; + } + YY_BREAK + + case 30: + YY_RULE_SETUP + { + return DEFAULT; + } + YY_BREAK + + case 31: + YY_RULE_SETUP + { + return DEFERRABLE; + } + YY_BREAK + + case 32: + YY_RULE_SETUP + { + return DEFERRED; + } + YY_BREAK + + case 33: + YY_RULE_SETUP + { + return IDB_DELETE; + } + YY_BREAK + + case 34: + YY_RULE_SETUP + { + return DROP; + } + YY_BREAK + + case 35: + YY_RULE_SETUP + { + return ENGINE; + } + YY_BREAK + + case 36: + YY_RULE_SETUP + { + return FOREIGN; + } + YY_BREAK + + case 37: + YY_RULE_SETUP + { + return FULL; + } + YY_BREAK + + case 38: + YY_RULE_SETUP + { + return IMMEDIATE; + } + YY_BREAK + + case 39: + YY_RULE_SETUP + { + return INDEX; + } + YY_BREAK + + case 40: + YY_RULE_SETUP + { + return INITIALLY; + } + YY_BREAK + + case 41: + YY_RULE_SETUP + { + return IDB_INT; + } + YY_BREAK + + case 42: + YY_RULE_SETUP + { + return INTEGER; + } + YY_BREAK + + case 43: + YY_RULE_SETUP + { + return KEY; + } + YY_BREAK + + case 44: + YY_RULE_SETUP + { + return MATCH; + } + YY_BREAK + + case 45: + YY_RULE_SETUP + { + return MAX_ROWS; + } + YY_BREAK + + case 46: + YY_RULE_SETUP + { + return MIN_ROWS; + } + YY_BREAK + + case 47: + YY_RULE_SETUP + { + return MODIFY; + } + YY_BREAK + + case 48: + YY_RULE_SETUP + { + return NO; + } + YY_BREAK + + case 49: + YY_RULE_SETUP + { + return NOT; + } + YY_BREAK + + case 50: + YY_RULE_SETUP + { + return NULL_TOK; + } + YY_BREAK + + case 51: + YY_RULE_SETUP + { + return NUMERIC; + } + YY_BREAK + + case 52: + YY_RULE_SETUP + { + return ON; + } + YY_BREAK + + case 53: + YY_RULE_SETUP + { + return PARTIAL; + } + YY_BREAK + + case 54: + YY_RULE_SETUP + { + return PRECISION; + } + YY_BREAK + + case 55: + YY_RULE_SETUP + { + return PRIMARY; + } + YY_BREAK + + case 56: + YY_RULE_SETUP + { + return REFERENCES; + } + YY_BREAK + + case 57: + YY_RULE_SETUP + { + return RENAME; + } + YY_BREAK + + case 58: + YY_RULE_SETUP + { + return RESTRICT; + } + YY_BREAK + + case 59: + YY_RULE_SETUP + { + return SESSION_USER; + } + YY_BREAK + + case 60: + YY_RULE_SETUP + { + return SYSTEM_USER; + } + YY_BREAK + + case 61: + YY_RULE_SETUP + { + return SET; + } + YY_BREAK + + case 62: + YY_RULE_SETUP + { + return SMALLINT; + } + YY_BREAK + + case 63: + YY_RULE_SETUP + { + return TABLE; + } + YY_BREAK + + case 64: + YY_RULE_SETUP + { + return TINYINT; + } + YY_BREAK + + case 65: + YY_RULE_SETUP + { + return TO; + } + YY_BREAK + + case 66: + YY_RULE_SETUP + { + return UNIQUE; + } + YY_BREAK + + case 67: + YY_RULE_SETUP + { + return UNSIGNED; + } + YY_BREAK + + case 68: + YY_RULE_SETUP + { + return UPDATE; + } + YY_BREAK + + case 69: + YY_RULE_SETUP + { + return USER; + } + YY_BREAK + + case 70: + YY_RULE_SETUP + { + return VARCHAR; + } + YY_BREAK + + case 71: + YY_RULE_SETUP + { + return IDB_FLOAT; + } + YY_BREAK + + case 72: + YY_RULE_SETUP + { + return DOUBLE; + } + YY_BREAK + + case 73: + YY_RULE_SETUP + { + return REAL; + } + YY_BREAK + + case 74: + YY_RULE_SETUP + { + return CHARSET; + } + YY_BREAK + + case 75: + YY_RULE_SETUP + { + return IDB_IF; + } + YY_BREAK + + case 76: + YY_RULE_SETUP + { + return EXISTS; + } + YY_BREAK + + case 77: + YY_RULE_SETUP + { + return CHANGE; + } + YY_BREAK + + case 78: + YY_RULE_SETUP + { + return TRUNCATE; + } + YY_BREAK + + case 79: + YY_RULE_SETUP + { + return VARBINARY; + } + YY_BREAK + + case 80: + YY_RULE_SETUP + { + lineno++; + } + YY_BREAK + + case 81: + YY_RULE_SETUP + { + /* ignore */ + } + YY_BREAK + + case 82: + YY_RULE_SETUP + { + ddllval.str = scanner_copy(ddltext); + return IDENT; + } + YY_BREAK + + case 83: + YY_RULE_SETUP + { + return ddltext[0]; + } + YY_BREAK + + case 84: + YY_RULE_SETUP + { + /* ignore */ + } + YY_BREAK + + case 85: + YY_RULE_SETUP + ECHO; + YY_BREAK + + case YY_STATE_EOF(INITIAL): + case YY_STATE_EOF(check1): + case YY_STATE_EOF(check2): + case YY_STATE_EOF(inquote): + case YY_STATE_EOF(endquote): yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int)(yy_cp - yytext_ptr) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = yy_hold_char; + YY_RESTORE_YY_MORE_OFFSET + + if (yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between yy_current_buffer and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + yy_n_chars = yy_current_buffer->yy_n_chars; + yy_current_buffer->yy_input_file = yyin; + yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). */ - yy_bp = yy_cp; - - yy_current_state = yy_start; -yy_match: - - do + if (yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars]) { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + /* This was really a NUL. */ + yy_state_type yy_next_state; - if ( yy_accept[yy_current_state] ) - { - yy_last_accepting_state = yy_current_state; - yy_last_accepting_cpos = yy_cp; - } + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; + yy_current_state = yy_get_previous_state(); - if ( yy_current_state >= 393 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - ++yy_cp; - } - while ( yy_base[yy_current_state] != 438 ); + yy_next_state = yy_try_NUL_trans(yy_current_state); -yy_find_action: - yy_act = yy_accept[yy_current_state]; + yy_bp = yytext_ptr + YY_MORE_ADJ; - if ( yy_act == 0 ) - { - /* have to back up */ - yy_cp = yy_last_accepting_cpos; - yy_current_state = yy_last_accepting_state; - yy_act = yy_accept[yy_current_state]; + if (yy_next_state) + { + /* Consume the NUL. */ + yy_cp = ++yy_c_buf_p; + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = yy_c_buf_p; + goto yy_find_action; + } } - YY_DO_BEFORE_ACTION; - - -do_action: /* This label is used only to access EOF actions. */ - - - switch ( yy_act ) - { - /* beginning of action switch */ - case 0: /* must back up */ - /* undo the effects of YY_DO_BEFORE_ACTION */ - *yy_cp = yy_hold_char; - yy_cp = yy_last_accepting_cpos; - yy_current_state = yy_last_accepting_state; - goto yy_find_action; - - case 1: - YY_RULE_SETUP - {return ACTION;} - YY_BREAK - - case 2: - YY_RULE_SETUP - {return ADD;} - YY_BREAK - - case 3: - YY_RULE_SETUP - {return ALTER;} - YY_BREAK - - case 4: - YY_RULE_SETUP - {return AUTO_INCREMENT;} - YY_BREAK - - case 5: - YY_RULE_SETUP - {return CASCADE;} - YY_BREAK - - case 6: - YY_RULE_SETUP - {return IDB_CHAR;} - YY_BREAK - - case 7: - YY_RULE_SETUP - {return IDB_CHAR;} - YY_BREAK - - case 8: - YY_RULE_SETUP - {return BIGINT;} - YY_BREAK - - case 9: - YY_RULE_SETUP - {BEGIN(check1); return CHECK;} - YY_BREAK - - case 10: - YY_RULE_SETUP - {BEGIN(check2); return '(';} - YY_BREAK - - case 11: - *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ - yy_c_buf_p = yy_cp -= 1; - YY_DO_BEFORE_ACTION; /* set up yytext again */ - YY_RULE_SETUP - {BEGIN(check1); ddllval.str = scanner_copy(ddltext); return CP_SEARCH_CONDITION_TEXT;} - YY_BREAK - - case 12: - YY_RULE_SETUP - {BEGIN(0); return ')';} - YY_BREAK - - case 13: - YY_RULE_SETUP - {BEGIN(inquote); return yytext[0];} - YY_BREAK - - case 14: - *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ - yy_c_buf_p = yy_cp -= 1; - YY_DO_BEFORE_ACTION; /* set up yytext again */ - YY_RULE_SETUP - {BEGIN(endquote); ddllval.str = scanner_copy(ddltext); return SCONST;} - YY_BREAK - - case 15: - YY_RULE_SETUP - {BEGIN(0); return yytext[0];} - YY_BREAK - - case 16: - YY_RULE_SETUP - {ddllval.str = scanner_copy(ddltext); return ICONST;} - YY_BREAK - - case 17: - YY_RULE_SETUP - {ddllval.str = scanner_copy(ddltext); return FCONST;} - YY_BREAK - - case 18: - YY_RULE_SETUP - {ddllval.str = scanner_copy(ddltext); return FCONST;} - YY_BREAK - - case 19: - YY_RULE_SETUP - {return COMMENT;} - YY_BREAK - - case 20: - YY_RULE_SETUP - {return COLUMN;} - YY_BREAK - - case 21: - YY_RULE_SETUP - {return COLUMNS;} - YY_BREAK - - case 22: - YY_RULE_SETUP - {return CONSTRAINT;} - YY_BREAK - - case 23: - YY_RULE_SETUP - {return CONSTRAINTS;} - YY_BREAK - - case 24: - YY_RULE_SETUP - {return CREATE;} - YY_BREAK - - case 25: - YY_RULE_SETUP - {return CURRENT_USER;} - YY_BREAK - - case 26: - YY_RULE_SETUP - {ddllval.str = strdup("date"); return DATE;} - YY_BREAK - - case 27: - YY_RULE_SETUP - {return DATETIME;} - YY_BREAK - - case 28: - YY_RULE_SETUP - {return DECIMAL;} - YY_BREAK - - case 29: - YY_RULE_SETUP - {return DECIMAL;} - YY_BREAK - - case 30: - YY_RULE_SETUP - {return DEFAULT;} - YY_BREAK - - case 31: - YY_RULE_SETUP - {return DEFERRABLE;} - YY_BREAK - - case 32: - YY_RULE_SETUP - {return DEFERRED;} - YY_BREAK - - case 33: - YY_RULE_SETUP - {return IDB_DELETE;} - YY_BREAK - - case 34: - YY_RULE_SETUP - {return DROP;} - YY_BREAK - - case 35: - YY_RULE_SETUP - {return ENGINE;} - YY_BREAK - - case 36: - YY_RULE_SETUP - {return FOREIGN;} - YY_BREAK - - case 37: - YY_RULE_SETUP - {return FULL;} - YY_BREAK - - case 38: - YY_RULE_SETUP - {return IMMEDIATE;} - YY_BREAK - - case 39: - YY_RULE_SETUP - {return INDEX;} - YY_BREAK - - case 40: - YY_RULE_SETUP - {return INITIALLY;} - YY_BREAK - - case 41: - YY_RULE_SETUP - {return IDB_INT;} - YY_BREAK - - case 42: - YY_RULE_SETUP - {return INTEGER;} - YY_BREAK - - case 43: - YY_RULE_SETUP - {return KEY;} - YY_BREAK - - case 44: - YY_RULE_SETUP - {return MATCH;} - YY_BREAK - - case 45: - YY_RULE_SETUP - {return MAX_ROWS;} - YY_BREAK - - case 46: - YY_RULE_SETUP - {return MIN_ROWS;} - YY_BREAK - - case 47: - YY_RULE_SETUP - {return MODIFY;} - YY_BREAK - - case 48: - YY_RULE_SETUP - {return NO;} - YY_BREAK - - case 49: - YY_RULE_SETUP - {return NOT;} - YY_BREAK - - case 50: - YY_RULE_SETUP - {return NULL_TOK;} - YY_BREAK - - case 51: - YY_RULE_SETUP - {return NUMERIC;} - YY_BREAK - - case 52: - YY_RULE_SETUP - {return ON;} - YY_BREAK - - case 53: - YY_RULE_SETUP - {return PARTIAL;} - YY_BREAK - - case 54: - YY_RULE_SETUP - {return PRECISION;} - YY_BREAK - - case 55: - YY_RULE_SETUP - {return PRIMARY;} - YY_BREAK - - case 56: - YY_RULE_SETUP - {return REFERENCES;} - YY_BREAK - - case 57: - YY_RULE_SETUP - {return RENAME;} - YY_BREAK - - case 58: - YY_RULE_SETUP - {return RESTRICT;} - YY_BREAK - - case 59: - YY_RULE_SETUP - {return SESSION_USER;} - YY_BREAK - - case 60: - YY_RULE_SETUP - {return SYSTEM_USER;} - YY_BREAK - - case 61: - YY_RULE_SETUP - {return SET;} - YY_BREAK - - case 62: - YY_RULE_SETUP - {return SMALLINT;} - YY_BREAK - - case 63: - YY_RULE_SETUP - {return TABLE;} - YY_BREAK - - case 64: - YY_RULE_SETUP - {return TINYINT;} - YY_BREAK - - case 65: - YY_RULE_SETUP - {return TO;} - YY_BREAK - - case 66: - YY_RULE_SETUP - {return UNIQUE;} - YY_BREAK - - case 67: - YY_RULE_SETUP - {return UNSIGNED;} - YY_BREAK - - case 68: - YY_RULE_SETUP - {return UPDATE;} - YY_BREAK - - case 69: - YY_RULE_SETUP - {return USER;} - YY_BREAK - - case 70: - YY_RULE_SETUP - {return VARCHAR;} - YY_BREAK - - case 71: - YY_RULE_SETUP - {return IDB_FLOAT;} - YY_BREAK - - case 72: - YY_RULE_SETUP - {return DOUBLE;} - YY_BREAK - - case 73: - YY_RULE_SETUP - {return REAL;} - YY_BREAK - - case 74: - YY_RULE_SETUP - {return CHARSET;} - YY_BREAK - - case 75: - YY_RULE_SETUP - {return IDB_IF;} - YY_BREAK - - case 76: - YY_RULE_SETUP - {return EXISTS;} - YY_BREAK - - case 77: - YY_RULE_SETUP - {return CHANGE;} - YY_BREAK - - case 78: - YY_RULE_SETUP - {return TRUNCATE;} - YY_BREAK - - case 79: - YY_RULE_SETUP - {return VARBINARY;} - YY_BREAK - - case 80: - YY_RULE_SETUP - { lineno++;} - YY_BREAK - - case 81: - YY_RULE_SETUP - { - /* ignore */ - } - YY_BREAK - - case 82: - YY_RULE_SETUP - {ddllval.str = scanner_copy(ddltext); return IDENT;} - YY_BREAK - - case 83: - YY_RULE_SETUP - { - return ddltext[0]; - } - YY_BREAK - - case 84: - YY_RULE_SETUP - { - /* ignore */ - } - YY_BREAK - - case 85: - YY_RULE_SETUP - ECHO; - YY_BREAK - - case YY_STATE_EOF(INITIAL): - case YY_STATE_EOF(check1): - case YY_STATE_EOF(check2): - case YY_STATE_EOF(inquote): - case YY_STATE_EOF(endquote): - yyterminate(); - - case YY_END_OF_BUFFER: + else + switch (yy_get_next_buffer()) + { + case EOB_ACT_END_OF_FILE: { - /* Amount of text matched not including the EOB char. */ - int yy_amount_of_matched_text = (int) (yy_cp - yytext_ptr) - 1; + yy_did_buffer_switch_on_eof = 0; - /* Undo the effects of YY_DO_BEFORE_ACTION. */ - *yy_cp = yy_hold_char; - YY_RESTORE_YY_MORE_OFFSET - - if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW ) - { - /* We're scanning a new file or input source. It's - * possible that this happened because the user - * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure - * consistency between yy_current_buffer and our - * globals. Here is the right place to do so, because - * this is the first action (other than possibly a - * back-up) that will match for the new input source. - */ - yy_n_chars = yy_current_buffer->yy_n_chars; - yy_current_buffer->yy_input_file = yyin; - yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; - } - - /* Note that here we test for yy_c_buf_p "<=" to the position - * of the first EOB in the buffer, since yy_c_buf_p will - * already have been incremented past the NUL character - * (since all states make transitions on EOB to the - * end-of-buffer state). Contrast this with the test - * in input(). + if (yywrap()) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. */ - if ( yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars] ) - { - /* This was really a NUL. */ - yy_state_type yy_next_state; + yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; - yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } - yy_current_state = yy_get_previous_state(); + else + { + if (!yy_did_buffer_switch_on_eof) + YY_NEW_FILE; + } - /* Okay, we're now positioned to make the NUL - * transition. We couldn't have - * yy_get_previous_state() go ahead and do it - * for us because it doesn't know how to deal - * with the possibility of jamming (and we don't - * want to build jamming into it because then it - * will run more slowly). - */ - - yy_next_state = yy_try_NUL_trans( yy_current_state ); - - yy_bp = yytext_ptr + YY_MORE_ADJ; - - if ( yy_next_state ) - { - /* Consume the NUL. */ - yy_cp = ++yy_c_buf_p; - yy_current_state = yy_next_state; - goto yy_match; - } - - else - { - yy_cp = yy_c_buf_p; - goto yy_find_action; - } - } - - else switch ( yy_get_next_buffer() ) - { - case EOB_ACT_END_OF_FILE: - { - yy_did_buffer_switch_on_eof = 0; - - if ( yywrap() ) - { - /* Note: because we've taken care in - * yy_get_next_buffer() to have set up - * yytext, we can now set up - * yy_c_buf_p so that if some total - * hoser (like flex itself) wants to - * call the scanner after we return the - * YY_NULL, it'll still work - another - * YY_NULL will get returned. - */ - yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; - - yy_act = YY_STATE_EOF(YY_START); - goto do_action; - } - - else - { - if ( ! yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; - } - - break; - } - - case EOB_ACT_CONTINUE_SCAN: - yy_c_buf_p = - yytext_ptr + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state(); - - yy_cp = yy_c_buf_p; - yy_bp = yytext_ptr + YY_MORE_ADJ; - goto yy_match; - - case EOB_ACT_LAST_MATCH: - yy_c_buf_p = - &yy_current_buffer->yy_ch_buf[yy_n_chars]; - - yy_current_state = yy_get_previous_state(); - - yy_cp = yy_c_buf_p; - yy_bp = yytext_ptr + YY_MORE_ADJ; - goto yy_find_action; - } - - break; + break; } - default: - YY_FATAL_ERROR( - "fatal flex scanner internal error--no action found" ); - } /* end of action switch */ - } /* end of scanning one token */ -} /* end of yylex */ + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + yy_c_buf_p = &yy_current_buffer->yy_ch_buf[yy_n_chars]; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_find_action; + } + + break; + } + + default: YY_FATAL_ERROR("fatal flex scanner internal error--no action found"); + } /* end of action switch */ + } /* end of scanning one token */ +} /* end of yylex */ /* yy_get_next_buffer - try to read in a new buffer * @@ -1495,169 +1499,158 @@ do_action: /* This label is used only to access EOF actions. */ static int yy_get_next_buffer() { - register char* dest = yy_current_buffer->yy_ch_buf; - register char* source = yytext_ptr; - register int number_to_move, i; - int ret_val; + register char* dest = yy_current_buffer->yy_ch_buf; + register char* source = yytext_ptr; + register int number_to_move, i; + int ret_val; - if ( yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1] ) - YY_FATAL_ERROR( - "fatal flex scanner internal error--end of buffer missed" ); + if (yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1]) + YY_FATAL_ERROR("fatal flex scanner internal error--end of buffer missed"); - if ( yy_current_buffer->yy_fill_buffer == 0 ) + if (yy_current_buffer->yy_fill_buffer == 0) + { + /* Don't try to fill the buffer, so this is an EOF. */ + if (yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1) { - /* Don't try to fill the buffer, so this is an EOF. */ - if ( yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1 ) - { - /* We matched a single character, the EOB, so - * treat this as a final EOF. - */ - return EOB_ACT_END_OF_FILE; - } - - else - { - /* We matched some text prior to the EOB, first - * process it. - */ - return EOB_ACT_LAST_MATCH; - } + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; } - /* Try to read more data. */ - - /* First move last chars to start of buffer. */ - number_to_move = (int) (yy_c_buf_p - yytext_ptr) - 1; - - for ( i = 0; i < number_to_move; ++i ) - *(dest++) = *(source++); - - if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING ) - /* don't do the read, it's not guaranteed to return an EOF, - * just force an EOF - */ - yy_current_buffer->yy_n_chars = yy_n_chars = 0; - else { - int num_to_read = - yy_current_buffer->yy_buf_size - number_to_move - 1; + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } - while ( num_to_read <= 0 ) - { - /* Not enough room in the buffer - grow it. */ + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int)(yy_c_buf_p - yytext_ptr) - 1; + + for (i = 0; i < number_to_move; ++i) + *(dest++) = *(source++); + + if (yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + yy_current_buffer->yy_n_chars = yy_n_chars = 0; + + else + { + int num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; + + while (num_to_read <= 0) + { + /* Not enough room in the buffer - grow it. */ #ifdef YY_USES_REJECT - YY_FATAL_ERROR( - "input buffer overflow, can't enlarge buffer because scanner uses REJECT" ); + YY_FATAL_ERROR("input buffer overflow, can't enlarge buffer because scanner uses REJECT"); #else - /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = yy_current_buffer; + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = yy_current_buffer; - int yy_c_buf_p_offset = - (int) (yy_c_buf_p - b->yy_ch_buf); + int yy_c_buf_p_offset = (int)(yy_c_buf_p - b->yy_ch_buf); - if ( b->yy_is_our_buffer ) - { - int new_size = b->yy_buf_size * 2; + if (b->yy_is_our_buffer) + { + int new_size = b->yy_buf_size * 2; - if ( new_size <= 0 ) - b->yy_buf_size += b->yy_buf_size / 8; - else - b->yy_buf_size *= 2; + if (new_size <= 0) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; - b->yy_ch_buf = (char*) - /* Include room in for 2 EOB chars. */ - yy_flex_realloc( (void*) b->yy_ch_buf, - b->yy_buf_size + 2 ); - } - else - /* Can't grow it, we don't own it. */ - b->yy_ch_buf = 0; + b->yy_ch_buf = (char*) + /* Include room in for 2 EOB chars. */ + yy_flex_realloc((void*)b->yy_ch_buf, b->yy_buf_size + 2); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = 0; - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( - "fatal error - scanner input buffer overflow" ); + if (!b->yy_ch_buf) + YY_FATAL_ERROR("fatal error - scanner input buffer overflow"); - yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; + yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; - num_to_read = yy_current_buffer->yy_buf_size - - number_to_move - 1; + num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; #endif - } - - if ( num_to_read > YY_READ_BUF_SIZE ) - num_to_read = YY_READ_BUF_SIZE; - - /* Read in more data. */ - YY_INPUT( (&yy_current_buffer->yy_ch_buf[number_to_move]), - yy_n_chars, num_to_read ); - - yy_current_buffer->yy_n_chars = yy_n_chars; } - if ( yy_n_chars == 0 ) - { - if ( number_to_move == YY_MORE_ADJ ) - { - ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin ); - } + if (num_to_read > YY_READ_BUF_SIZE) + num_to_read = YY_READ_BUF_SIZE; - else - { - ret_val = EOB_ACT_LAST_MATCH; - yy_current_buffer->yy_buffer_status = - YY_BUFFER_EOF_PENDING; - } + /* Read in more data. */ + YY_INPUT((&yy_current_buffer->yy_ch_buf[number_to_move]), yy_n_chars, num_to_read); + + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + if (yy_n_chars == 0) + { + if (number_to_move == YY_MORE_ADJ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart(yyin); } else - ret_val = EOB_ACT_CONTINUE_SCAN; + { + ret_val = EOB_ACT_LAST_MATCH; + yy_current_buffer->yy_buffer_status = YY_BUFFER_EOF_PENDING; + } + } - yy_n_chars += number_to_move; - yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; - yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; + else + ret_val = EOB_ACT_CONTINUE_SCAN; - yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + yy_n_chars += number_to_move; + yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; + yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; - return ret_val; + yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + + return ret_val; } - /* yy_get_previous_state - get the state just before the EOB char was reached */ static yy_state_type yy_get_previous_state() { - register yy_state_type yy_current_state; - register char* yy_cp; + register yy_state_type yy_current_state; + register char* yy_cp; - yy_current_state = yy_start; + yy_current_state = yy_start; - for ( yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp ) + for (yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp) + { + register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + + if (yy_accept[yy_current_state]) { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); - - if ( yy_accept[yy_current_state] ) - { - yy_last_accepting_state = yy_current_state; - yy_last_accepting_cpos = yy_cp; - } - - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - - if ( yy_current_state >= 393 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; } - return yy_current_state; -} + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; + if (yy_current_state >= 393) + yy_c = yy_meta[(unsigned int)yy_c]; + } + + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + } + + return yy_current_state; +} /* yy_try_NUL_trans - try to make a transition on the NUL character * @@ -1666,83 +1659,75 @@ static yy_state_type yy_get_previous_state() */ #ifdef YY_USE_PROTOS -static yy_state_type yy_try_NUL_trans( yy_state_type yy_current_state ) +static yy_state_type yy_try_NUL_trans(yy_state_type yy_current_state) #else -static yy_state_type yy_try_NUL_trans( yy_current_state ) -yy_state_type yy_current_state; +static yy_state_type yy_try_NUL_trans(yy_current_state) yy_state_type yy_current_state; #endif { - register int yy_is_jam; - register char* yy_cp = yy_c_buf_p; + register int yy_is_jam; + register char* yy_cp = yy_c_buf_p; - register YY_CHAR yy_c = 1; + register YY_CHAR yy_c = 1; - if ( yy_accept[yy_current_state] ) - { - yy_last_accepting_state = yy_current_state; - yy_last_accepting_cpos = yy_cp; - } + if (yy_accept[yy_current_state]) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; - if ( yy_current_state >= 393 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } + if (yy_current_state >= 393) + yy_c = yy_meta[(unsigned int)yy_c]; + } - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 392); + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + yy_is_jam = (yy_current_state == 392); - return yy_is_jam ? 0 : yy_current_state; + return yy_is_jam ? 0 : yy_current_state; } - #ifndef YY_NO_UNPUT #ifdef YY_USE_PROTOS -static void yyunput( int c, register char* yy_bp ) +static void yyunput(int c, register char* yy_bp) #else -static void yyunput( c, yy_bp ) -int c; +static void yyunput(c, yy_bp) int c; register char* yy_bp; #endif { - register char* yy_cp = yy_c_buf_p; + register char* yy_cp = yy_c_buf_p; - /* undo effects of setting up yytext */ - *yy_cp = yy_hold_char; + /* undo effects of setting up yytext */ + *yy_cp = yy_hold_char; - if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) - { - /* need to shift things up to make room */ - /* +2 for EOB chars. */ - register int number_to_move = yy_n_chars + 2; - register char* dest = &yy_current_buffer->yy_ch_buf[ - yy_current_buffer->yy_buf_size + 2]; - register char* source = - &yy_current_buffer->yy_ch_buf[number_to_move]; + if (yy_cp < yy_current_buffer->yy_ch_buf + 2) + { + /* need to shift things up to make room */ + /* +2 for EOB chars. */ + register int number_to_move = yy_n_chars + 2; + register char* dest = &yy_current_buffer->yy_ch_buf[yy_current_buffer->yy_buf_size + 2]; + register char* source = &yy_current_buffer->yy_ch_buf[number_to_move]; - while ( source > yy_current_buffer->yy_ch_buf ) - *--dest = *--source; + while (source > yy_current_buffer->yy_ch_buf) + *--dest = *--source; - yy_cp += (int) (dest - source); - yy_bp += (int) (dest - source); - yy_current_buffer->yy_n_chars = - yy_n_chars = yy_current_buffer->yy_buf_size; + yy_cp += (int)(dest - source); + yy_bp += (int)(dest - source); + yy_current_buffer->yy_n_chars = yy_n_chars = yy_current_buffer->yy_buf_size; - if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) - YY_FATAL_ERROR( "flex scanner push-back overflow" ); - } + if (yy_cp < yy_current_buffer->yy_ch_buf + 2) + YY_FATAL_ERROR("flex scanner push-back overflow"); + } - *--yy_cp = (char) c; + *--yy_cp = (char)c; - - yytext_ptr = yy_bp; - yy_hold_char = *yy_cp; - yy_c_buf_p = yy_cp; + yytext_ptr = yy_bp; + yy_hold_char = *yy_cp; + yy_c_buf_p = yy_cp; } -#endif /* ifndef YY_NO_UNPUT */ - +#endif /* ifndef YY_NO_UNPUT */ #ifdef __cplusplus static int yyinput() @@ -1750,396 +1735,365 @@ static int yyinput() static int input() #endif { - int c; + int c; - *yy_c_buf_p = yy_hold_char; + *yy_c_buf_p = yy_hold_char; - if ( *yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) + if (*yy_c_buf_p == YY_END_OF_BUFFER_CHAR) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if (yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars]) + /* This was really a NUL. */ + *yy_c_buf_p = '\0'; + + else { - /* yy_c_buf_p now points to the character we want to return. - * If this occurs *before* the EOB characters, then it's a - * valid NUL; if not, then we've hit the end of the buffer. - */ - if ( yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars] ) - /* This was really a NUL. */ - *yy_c_buf_p = '\0'; + /* need more input */ + int offset = yy_c_buf_p - yytext_ptr; + ++yy_c_buf_p; - else + switch (yy_get_next_buffer()) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart(yyin); + + /* fall through */ + + case EOB_ACT_END_OF_FILE: { - /* need more input */ - int offset = yy_c_buf_p - yytext_ptr; - ++yy_c_buf_p; + if (yywrap()) + return EOF; - switch ( yy_get_next_buffer() ) - { - case EOB_ACT_LAST_MATCH: - /* This happens because yy_g_n_b() - * sees that we've accumulated a - * token and flags that we need to - * try matching the token before - * proceeding. But for input(), - * there's no matching to consider. - * So convert the EOB_ACT_LAST_MATCH - * to EOB_ACT_END_OF_FILE. - */ - - /* Reset buffer status. */ - yyrestart( yyin ); - - /* fall through */ - - case EOB_ACT_END_OF_FILE: - { - if ( yywrap() ) - return EOF; - - if ( ! yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; + if (!yy_did_buffer_switch_on_eof) + YY_NEW_FILE; #ifdef __cplusplus - return yyinput(); + return yyinput(); #else - return input(); + return input(); #endif - } - - case EOB_ACT_CONTINUE_SCAN: - yy_c_buf_p = yytext_ptr + offset; - break; - } } + + case EOB_ACT_CONTINUE_SCAN: yy_c_buf_p = yytext_ptr + offset; break; + } } + } - c = *(unsigned char*) yy_c_buf_p; /* cast for 8-bit char's */ - *yy_c_buf_p = '\0'; /* preserve yytext */ - yy_hold_char = *++yy_c_buf_p; + c = *(unsigned char*)yy_c_buf_p; /* cast for 8-bit char's */ + *yy_c_buf_p = '\0'; /* preserve yytext */ + yy_hold_char = *++yy_c_buf_p; - - return c; + return c; } - #ifdef YY_USE_PROTOS -void yyrestart( FILE* input_file ) +void yyrestart(FILE* input_file) #else -void yyrestart( input_file ) -FILE* input_file; +void yyrestart(input_file) FILE* input_file; #endif { - if ( ! yy_current_buffer ) - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); + if (!yy_current_buffer) + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); - yy_init_buffer( yy_current_buffer, input_file ); - yy_load_buffer_state(); + yy_init_buffer(yy_current_buffer, input_file); + yy_load_buffer_state(); } - #ifdef YY_USE_PROTOS -void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +void yy_switch_to_buffer(YY_BUFFER_STATE new_buffer) #else -void yy_switch_to_buffer( new_buffer ) -YY_BUFFER_STATE new_buffer; +void yy_switch_to_buffer(new_buffer) YY_BUFFER_STATE new_buffer; #endif { - if ( yy_current_buffer == new_buffer ) - return; + if (yy_current_buffer == new_buffer) + return; - if ( yy_current_buffer ) - { - /* Flush out information for old buffer. */ - *yy_c_buf_p = yy_hold_char; - yy_current_buffer->yy_buf_pos = yy_c_buf_p; - yy_current_buffer->yy_n_chars = yy_n_chars; - } + if (yy_current_buffer) + { + /* Flush out information for old buffer. */ + *yy_c_buf_p = yy_hold_char; + yy_current_buffer->yy_buf_pos = yy_c_buf_p; + yy_current_buffer->yy_n_chars = yy_n_chars; + } - yy_current_buffer = new_buffer; - yy_load_buffer_state(); + yy_current_buffer = new_buffer; + yy_load_buffer_state(); - /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe - * to go ahead and always set it. - */ - yy_did_buffer_switch_on_eof = 1; + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yy_did_buffer_switch_on_eof = 1; } - #ifdef YY_USE_PROTOS -void yy_load_buffer_state( void ) +void yy_load_buffer_state(void) #else void yy_load_buffer_state() #endif { - yy_n_chars = yy_current_buffer->yy_n_chars; - yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; - yyin = yy_current_buffer->yy_input_file; - yy_hold_char = *yy_c_buf_p; + yy_n_chars = yy_current_buffer->yy_n_chars; + yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; + yyin = yy_current_buffer->yy_input_file; + yy_hold_char = *yy_c_buf_p; } - #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_create_buffer( FILE* file, int size ) +YY_BUFFER_STATE yy_create_buffer(FILE* file, int size) #else -YY_BUFFER_STATE yy_create_buffer( file, size ) -FILE* file; +YY_BUFFER_STATE yy_create_buffer(file, size) FILE* file; int size; #endif { - YY_BUFFER_STATE b; + YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE)yy_flex_alloc(sizeof(struct yy_buffer_state)); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + if (!b) + YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - b->yy_buf_size = size; + b->yy_buf_size = size; - /* yy_ch_buf has to be 2 characters longer than the size given because - * we need to put in 2 end-of-buffer characters. - */ - b->yy_ch_buf = (char*) yy_flex_alloc( b->yy_buf_size + 2 ); + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char*)yy_flex_alloc(b->yy_buf_size + 2); - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + if (!b->yy_ch_buf) + YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - b->yy_is_our_buffer = 1; + b->yy_is_our_buffer = 1; - yy_init_buffer( b, file ); + yy_init_buffer(b, file); - return b; + return b; } - #ifdef YY_USE_PROTOS -void yy_delete_buffer( YY_BUFFER_STATE b ) +void yy_delete_buffer(YY_BUFFER_STATE b) #else -void yy_delete_buffer( b ) -YY_BUFFER_STATE b; +void yy_delete_buffer(b) YY_BUFFER_STATE b; #endif { - if ( ! b ) - return; + if (!b) + return; - if ( b == yy_current_buffer ) - yy_current_buffer = (YY_BUFFER_STATE) 0; + if (b == yy_current_buffer) + yy_current_buffer = (YY_BUFFER_STATE)0; - if ( b->yy_is_our_buffer ) - yy_flex_free( (void*) b->yy_ch_buf ); + if (b->yy_is_our_buffer) + yy_flex_free((void*)b->yy_ch_buf); - yy_flex_free( (void*) b ); + yy_flex_free((void*)b); } - #ifndef YY_ALWAYS_INTERACTIVE #ifndef YY_NEVER_INTERACTIVE -extern int isatty YY_PROTO(( int )); +extern int isatty YY_PROTO((int)); #endif #endif #ifdef YY_USE_PROTOS -void yy_init_buffer( YY_BUFFER_STATE b, FILE* file ) +void yy_init_buffer(YY_BUFFER_STATE b, FILE* file) #else -void yy_init_buffer( b, file ) -YY_BUFFER_STATE b; +void yy_init_buffer(b, file) YY_BUFFER_STATE b; FILE* file; #endif - { - yy_flush_buffer( b ); + yy_flush_buffer(b); - b->yy_input_file = file; - b->yy_fill_buffer = 1; + b->yy_input_file = file; + b->yy_fill_buffer = 1; #if YY_ALWAYS_INTERACTIVE - b->yy_is_interactive = 1; + b->yy_is_interactive = 1; #else #if YY_NEVER_INTERACTIVE - b->yy_is_interactive = 0; + b->yy_is_interactive = 0; #else - b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; + b->yy_is_interactive = file ? (isatty(fileno(file)) > 0) : 0; #endif #endif } - #ifdef YY_USE_PROTOS -void yy_flush_buffer( YY_BUFFER_STATE b ) +void yy_flush_buffer(YY_BUFFER_STATE b) #else -void yy_flush_buffer( b ) -YY_BUFFER_STATE b; +void yy_flush_buffer(b) YY_BUFFER_STATE b; #endif { - if ( ! b ) - return; + if (!b) + return; - b->yy_n_chars = 0; + b->yy_n_chars = 0; - /* We always need two end-of-buffer characters. The first causes - * a transition to the end-of-buffer state. The second causes - * a jam in that state. - */ - b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; - b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; - b->yy_buf_pos = &b->yy_ch_buf[0]; + b->yy_buf_pos = &b->yy_ch_buf[0]; - b->yy_at_bol = 1; - b->yy_buffer_status = YY_BUFFER_NEW; + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; - if ( b == yy_current_buffer ) - yy_load_buffer_state(); + if (b == yy_current_buffer) + yy_load_buffer_state(); } - #ifndef YY_NO_SCAN_BUFFER #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_buffer( char* base, yy_size_t size ) +YY_BUFFER_STATE yy_scan_buffer(char* base, yy_size_t size) #else -YY_BUFFER_STATE yy_scan_buffer( base, size ) -char* base; +YY_BUFFER_STATE yy_scan_buffer(base, size) char* base; yy_size_t size; #endif { - YY_BUFFER_STATE b; + YY_BUFFER_STATE b; - if ( size < 2 || - base[size - 2] != YY_END_OF_BUFFER_CHAR || - base[size - 1] != YY_END_OF_BUFFER_CHAR ) - /* They forgot to leave room for the EOB's. */ - return 0; + if (size < 2 || base[size - 2] != YY_END_OF_BUFFER_CHAR || base[size - 1] != YY_END_OF_BUFFER_CHAR) + /* They forgot to leave room for the EOB's. */ + return 0; - b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE)yy_flex_alloc(sizeof(struct yy_buffer_state)); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + if (!b) + YY_FATAL_ERROR("out of dynamic memory in yy_scan_buffer()"); - b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ - b->yy_buf_pos = b->yy_ch_buf = base; - b->yy_is_our_buffer = 0; - b->yy_input_file = 0; - b->yy_n_chars = b->yy_buf_size; - b->yy_is_interactive = 0; - b->yy_at_bol = 1; - b->yy_fill_buffer = 0; - b->yy_buffer_status = YY_BUFFER_NEW; + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = 0; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b ); + yy_switch_to_buffer(b); - return b; + return b; } #endif - #ifndef YY_NO_SCAN_STRING #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_string( yyconst char* yy_str ) +YY_BUFFER_STATE yy_scan_string(yyconst char* yy_str) #else -YY_BUFFER_STATE yy_scan_string( yy_str ) -yyconst char* yy_str; +YY_BUFFER_STATE yy_scan_string(yy_str) yyconst char* yy_str; #endif { - int len; + int len; - for ( len = 0; yy_str[len]; ++len ) - ; + for (len = 0; yy_str[len]; ++len) + ; - return yy_scan_bytes( yy_str, len ); + return yy_scan_bytes(yy_str, len); } #endif - #ifndef YY_NO_SCAN_BYTES #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_bytes( yyconst char* bytes, int len ) +YY_BUFFER_STATE yy_scan_bytes(yyconst char* bytes, int len) #else -YY_BUFFER_STATE yy_scan_bytes( bytes, len ) -yyconst char* bytes; +YY_BUFFER_STATE yy_scan_bytes(bytes, len) yyconst char* bytes; int len; #endif { - YY_BUFFER_STATE b; - char* buf; - yy_size_t n; - int i; + YY_BUFFER_STATE b; + char* buf; + yy_size_t n; + int i; - /* Get memory for full buffer, including space for trailing EOB's. */ - n = len + 2; - buf = (char*) yy_flex_alloc( n ); + /* Get memory for full buffer, including space for trailing EOB's. */ + n = len + 2; + buf = (char*)yy_flex_alloc(n); - if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + if (!buf) + YY_FATAL_ERROR("out of dynamic memory in yy_scan_bytes()"); - for ( i = 0; i < len; ++i ) - buf[i] = bytes[i]; + for (i = 0; i < len; ++i) + buf[i] = bytes[i]; - buf[len] = buf[len + 1] = YY_END_OF_BUFFER_CHAR; + buf[len] = buf[len + 1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n ); + b = yy_scan_buffer(buf, n); - if ( ! b ) - YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + if (!b) + YY_FATAL_ERROR("bad buffer in yy_scan_bytes()"); - /* It's okay to grow etc. this buffer, and we should throw it - * away when we're done. - */ - b->yy_is_our_buffer = 1; + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; - return b; + return b; } #endif - #ifndef YY_NO_PUSH_STATE #ifdef YY_USE_PROTOS -static void yy_push_state( int new_state ) +static void yy_push_state(int new_state) #else -static void yy_push_state( new_state ) -int new_state; +static void yy_push_state(new_state) int new_state; #endif { - if ( yy_start_stack_ptr >= yy_start_stack_depth ) - { - yy_size_t new_size; + if (yy_start_stack_ptr >= yy_start_stack_depth) + { + yy_size_t new_size; - yy_start_stack_depth += YY_START_STACK_INCR; - new_size = yy_start_stack_depth * sizeof( int ); + yy_start_stack_depth += YY_START_STACK_INCR; + new_size = yy_start_stack_depth * sizeof(int); - if ( ! yy_start_stack ) - yy_start_stack = (int*) yy_flex_alloc( new_size ); + if (!yy_start_stack) + yy_start_stack = (int*)yy_flex_alloc(new_size); - else - yy_start_stack = (int*) yy_flex_realloc( - (void*) yy_start_stack, new_size ); + else + yy_start_stack = (int*)yy_flex_realloc((void*)yy_start_stack, new_size); - if ( ! yy_start_stack ) - YY_FATAL_ERROR( - "out of memory expanding start-condition stack" ); - } + if (!yy_start_stack) + YY_FATAL_ERROR("out of memory expanding start-condition stack"); + } - yy_start_stack[yy_start_stack_ptr++] = YY_START; + yy_start_stack[yy_start_stack_ptr++] = YY_START; - BEGIN(new_state); + BEGIN(new_state); } #endif - #ifndef YY_NO_POP_STATE static void yy_pop_state() { - if ( --yy_start_stack_ptr < 0 ) - YY_FATAL_ERROR( "start-condition stack underflow" ); + if (--yy_start_stack_ptr < 0) + YY_FATAL_ERROR("start-condition stack underflow"); - BEGIN(yy_start_stack[yy_start_stack_ptr]); + BEGIN(yy_start_stack[yy_start_stack_ptr]); } #endif - #ifndef YY_NO_TOP_STATE static int yy_top_state() { - return yy_start_stack[yy_start_stack_ptr - 1]; + return yy_start_stack[yy_start_stack_ptr - 1]; } #endif @@ -2148,121 +2102,109 @@ static int yy_top_state() #endif #ifdef YY_USE_PROTOS -static void yy_fatal_error( yyconst char msg[] ) +static void yy_fatal_error(yyconst char msg[]) #else -static void yy_fatal_error( msg ) -char msg[]; +static void yy_fatal_error(msg) char msg[]; #endif { - (void) fprintf( stderr, "%s\n", msg ); - exit( YY_EXIT_FAILURE ); + (void)fprintf(stderr, "%s\n", msg); + exit(YY_EXIT_FAILURE); } - - /* Redefine yyless() so it works in section 3 code. */ #undef yyless -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - yytext[yyleng] = yy_hold_char; \ - yy_c_buf_p = yytext + n; \ - yy_hold_char = *yy_c_buf_p; \ - *yy_c_buf_p = '\0'; \ - yyleng = n; \ - } \ - while ( 0 ) - +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yytext[yyleng] = yy_hold_char; \ + yy_c_buf_p = yytext + n; \ + yy_hold_char = *yy_c_buf_p; \ + *yy_c_buf_p = '\0'; \ + yyleng = n; \ + } while (0) /* Internal utility routines. */ #ifndef yytext_ptr #ifdef YY_USE_PROTOS -static void yy_flex_strncpy( char* s1, yyconst char* s2, int n ) +static void yy_flex_strncpy(char* s1, yyconst char* s2, int n) #else -static void yy_flex_strncpy( s1, s2, n ) -char* s1; +static void yy_flex_strncpy(s1, s2, n) char* s1; yyconst char* s2; int n; #endif { - register int i; + register int i; - for ( i = 0; i < n; ++i ) - s1[i] = s2[i]; + for (i = 0; i < n; ++i) + s1[i] = s2[i]; } #endif #ifdef YY_NEED_STRLEN #ifdef YY_USE_PROTOS -static int yy_flex_strlen( yyconst char* s ) +static int yy_flex_strlen(yyconst char* s) #else -static int yy_flex_strlen( s ) -yyconst char* s; +static int yy_flex_strlen(s) yyconst char* s; #endif { - register int n; + register int n; - for ( n = 0; s[n]; ++n ) - ; + for (n = 0; s[n]; ++n) + ; - return n; + return n; } #endif +#ifdef YY_USE_PROTOS +static void* yy_flex_alloc(yy_size_t size) +#else +static void* yy_flex_alloc(size) yy_size_t size; +#endif +{ + return (void*)malloc(size); +} #ifdef YY_USE_PROTOS -static void* yy_flex_alloc( yy_size_t size ) +static void* yy_flex_realloc(void* ptr, yy_size_t size) #else -static void* yy_flex_alloc( size ) +static void* yy_flex_realloc(ptr, size) void* ptr; yy_size_t size; #endif { - return (void*) malloc( size ); + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return (void*)realloc((char*)ptr, size); } #ifdef YY_USE_PROTOS -static void* yy_flex_realloc( void* ptr, yy_size_t size ) +static void yy_flex_free(void* ptr) #else -static void* yy_flex_realloc( ptr, size ) -void* ptr; -yy_size_t size; +static void yy_flex_free(ptr) void* ptr; #endif { - /* The cast to (char *) in the following accommodates both - * implementations that use char* generic pointers, and those - * that use void* generic pointers. It works with the latter - * because both ANSI C and C++ allow castless assignment from - * any pointer type to void*, and deal with argument conversions - * as though doing an assignment. - */ - return (void*) realloc( (char*) ptr, size ); -} - -#ifdef YY_USE_PROTOS -static void yy_flex_free( void* ptr ) -#else -static void yy_flex_free( ptr ) -void* ptr; -#endif -{ - free( ptr ); + free(ptr); } #if YY_MAIN int main() { - yylex(); - return 0; + yylex(); + return 0; } #endif - void ddlerror(char const* s) { - printf("yyerror: %d: %s at %s\n", lineno, s, yytext); + printf("yyerror: %d: %s at %s\n", lineno, s, yytext); } typedef std::vector valbuf_t; @@ -2274,61 +2216,59 @@ static valbuf_t valbuf; */ void scanner_init(const char* str) { - size_t slen = strlen(str); + size_t slen = strlen(str); - /* - * Might be left over after ereport() - */ - if (YY_CURRENT_BUFFER) - yy_delete_buffer(YY_CURRENT_BUFFER); + /* + * Might be left over after ereport() + */ + if (YY_CURRENT_BUFFER) + yy_delete_buffer(YY_CURRENT_BUFFER); - /* - * Make a scan buffer with special termination needed by flex. - */ - scanbuf = (char*)malloc(slen + 2); - memcpy(scanbuf, str, slen); - scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR; - scanbufhandle = yy_scan_buffer(scanbuf, slen + 2); + /* + * Make a scan buffer with special termination needed by flex. + */ + scanbuf = (char*)malloc(slen + 2); + memcpy(scanbuf, str, slen); + scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR; + scanbufhandle = yy_scan_buffer(scanbuf, slen + 2); - BEGIN(INITIAL); + BEGIN(INITIAL); - valbuf.clear(); + valbuf.clear(); } - /* * Called after parsing is done to clean up after scanner_init() */ - void scanner_finish(void) { - char* str; + char* str; - yy_delete_buffer(scanbufhandle); - free(scanbuf); - unsigned int i; + yy_delete_buffer(scanbufhandle); + free(scanbuf); + unsigned int i; - for (i = 0; i < valbuf.size(); i++) + for (i = 0; i < valbuf.size(); i++) + { + str = valbuf[i]; + + if (str) { - str = valbuf[i]; - - if (str) - { -// std::cout << "valbuf:(" << str << ")" << std::endl; - free(valbuf[i]); - } + // std::cout << "valbuf:(" << str << ")" << std::endl; + free(valbuf[i]); } + } - valbuf.clear(); + valbuf.clear(); } -char* scanner_copy (char* str) +char* scanner_copy(char* str) { - char* nv = strdup(str); + char* nv = strdup(str); - if (nv) - valbuf.push_back(nv); + if (nv) + valbuf.push_back(nv); - return nv; + return nv; } diff --git a/dbcon/ddlpackage/ddlpkg.cpp b/dbcon/ddlpackage/ddlpkg.cpp index 7e04e6f93..49b8abbc4 100644 --- a/dbcon/ddlpackage/ddlpkg.cpp +++ b/dbcon/ddlpackage/ddlpkg.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ddlpkg.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ddlpkg.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include @@ -32,167 +32,135 @@ namespace ddlpackage { using namespace std; -QualifiedName::QualifiedName(const char* name): - fName(name) +QualifiedName::QualifiedName(const char* name) : fName(name) { } -QualifiedName::QualifiedName(const char* schema, const char* name): - fName(name), - fSchema(schema) +QualifiedName::QualifiedName(const char* schema, const char* name) : fName(name), fSchema(schema) { } -QualifiedName::QualifiedName(const char* catalog, const char* schema, const char* name): - fCatalog(catalog), - fName(name), - fSchema(schema) +QualifiedName::QualifiedName(const char* catalog, const char* schema, const char* name) + : fCatalog(catalog), fName(name), fSchema(schema) { } ostream& operator<<(ostream& os, const QualifiedName& qname) { - if (!qname.fCatalog.empty()) - os << qname.fCatalog << "."; + if (!qname.fCatalog.empty()) + os << qname.fCatalog << "."; - if (!qname.fSchema.empty()) - os << qname.fSchema << "."; + if (!qname.fSchema.empty()) + os << qname.fSchema << "."; - os << qname.fName; - return os; + os << qname.fName; + return os; } -ColumnType::ColumnType(int prec, int scale) : - fType(DDL_INVALID_DATATYPE), - fLength(0), - fPrecision(prec), - fScale(scale), - fWithTimezone(false), - fCharset(NULL), - fExplicitLength(false) +ColumnType::ColumnType(int prec, int scale) + : fType(DDL_INVALID_DATATYPE) + , fLength(0) + , fPrecision(prec) + , fScale(scale) + , fWithTimezone(false) + , fCharset(NULL) + , fExplicitLength(false) { - fLength = utils::widthByPrecision(fPrecision); + fLength = utils::widthByPrecision(fPrecision); } -ColumnType::ColumnType(int type) : - fType(type), - fLength(0), - fScale(0), - fWithTimezone(false), - fCharset(NULL), - fExplicitLength(false) +ColumnType::ColumnType(int type) + : fType(type), fLength(0), fScale(0), fWithTimezone(false), fCharset(NULL), fExplicitLength(false) { - switch ( type ) - { - case DDL_TINYINT: - case DDL_UNSIGNED_TINYINT: - fPrecision = 3; - break; + switch (type) + { + case DDL_TINYINT: + case DDL_UNSIGNED_TINYINT: fPrecision = 3; break; - case DDL_SMALLINT: - case DDL_UNSIGNED_SMALLINT: - fPrecision = 5; - break; + case DDL_SMALLINT: + case DDL_UNSIGNED_SMALLINT: fPrecision = 5; break; - case DDL_MEDINT: - fPrecision = 7; - break; + case DDL_MEDINT: fPrecision = 7; break; - case DDL_UNSIGNED_MEDINT: - fPrecision = 8; - break; + case DDL_UNSIGNED_MEDINT: fPrecision = 8; break; - case DDL_INT: - case DDL_UNSIGNED_INT: - fPrecision = 10; - break; + case DDL_INT: + case DDL_UNSIGNED_INT: fPrecision = 10; break; - case DDL_BIGINT: - fPrecision = 19; - break; + case DDL_BIGINT: fPrecision = 19; break; - case DDL_UNSIGNED_BIGINT: - fPrecision = 20; - break; + case DDL_UNSIGNED_BIGINT: fPrecision = 20; break; - default: - fPrecision = 10; - break; - } + default: fPrecision = 10; break; + } } -ColumnConstraintDef::ColumnConstraintDef(DDL_CONSTRAINTS type) : - SchemaObject(), - fDeferrable(false), - fCheckTime(DDL_INITIALLY_IMMEDIATE), - fConstraintType(type), - fCheck("") +ColumnConstraintDef::ColumnConstraintDef(DDL_CONSTRAINTS type) + : SchemaObject(), fDeferrable(false), fCheckTime(DDL_INITIALLY_IMMEDIATE), fConstraintType(type), fCheck("") { } -ColumnConstraintDef::ColumnConstraintDef(const char* check) : - SchemaObject(), - fDeferrable(false), - fCheckTime(DDL_INITIALLY_IMMEDIATE), - fConstraintType(DDL_CHECK), - fCheck(check) +ColumnConstraintDef::ColumnConstraintDef(const char* check) + : SchemaObject() + , fDeferrable(false) + , fCheckTime(DDL_INITIALLY_IMMEDIATE) + , fConstraintType(DDL_CHECK) + , fCheck(check) { } - -AtaAddColumn::AtaAddColumn(ColumnDef* columnDef) : - fColumnDef(columnDef) +AtaAddColumn::AtaAddColumn(ColumnDef* columnDef) : fColumnDef(columnDef) { } ostream& operator<<(ostream& os, const ReferentialAction& ref) { - os << "ref action: u=" << ReferentialActionStrings[ref.fOnUpdate] << " " - << "d=" << ReferentialActionStrings[ref.fOnDelete]; - return os; + os << "ref action: u=" << ReferentialActionStrings[ref.fOnUpdate] << " " + << "d=" << ReferentialActionStrings[ref.fOnDelete]; + return os; } void ColumnDef::convertDecimal() { - //@Bug 2089 decimal precision default to 10 if 0 is used. - if (fType->fPrecision <= 0) - fType->fPrecision = 10; + //@Bug 2089 decimal precision default to 10 if 0 is used. + if (fType->fPrecision <= 0) + fType->fPrecision = 10; - if (fType->fPrecision == -1 || fType->fPrecision == 0) - { - fType->fType = DDL_BIGINT; - fType->fLength = 8; - fType->fScale = 0; - } - else if ((fType->fPrecision > 0) && (fType->fPrecision < 3)) - { - fType->fType = DDL_TINYINT; - fType->fLength = 1; - } + if (fType->fPrecision == -1 || fType->fPrecision == 0) + { + fType->fType = DDL_BIGINT; + fType->fLength = 8; + fType->fScale = 0; + } + else if ((fType->fPrecision > 0) && (fType->fPrecision < 3)) + { + fType->fType = DDL_TINYINT; + fType->fLength = 1; + } - else if (fType->fPrecision < 5 && (fType->fPrecision > 2)) - { - fType->fType = DDL_SMALLINT; - fType->fLength = 2; - } - else if (fType->fPrecision > 4 && fType->fPrecision < 7) - { - fType->fType = DDL_MEDINT; - fType->fLength = 4; - } - else if (fType->fPrecision > 6 && fType->fPrecision < 10) - { - fType->fType = DDL_INT; - fType->fLength = 4; - } - else if (fType->fPrecision > 9 && fType->fPrecision < 19) - { - fType->fType = DDL_BIGINT; - fType->fLength = 8; - } - else if (fType->fPrecision > 18 && fType->fPrecision < 39) - { - fType->fType = DDL_DECIMAL; - fType->fLength = 16; - } + else if (fType->fPrecision < 5 && (fType->fPrecision > 2)) + { + fType->fType = DDL_SMALLINT; + fType->fLength = 2; + } + else if (fType->fPrecision > 4 && fType->fPrecision < 7) + { + fType->fType = DDL_MEDINT; + fType->fLength = 4; + } + else if (fType->fPrecision > 6 && fType->fPrecision < 10) + { + fType->fType = DDL_INT; + fType->fLength = 4; + } + else if (fType->fPrecision > 9 && fType->fPrecision < 19) + { + fType->fType = DDL_BIGINT; + fType->fLength = 8; + } + else if (fType->fPrecision > 18 && fType->fPrecision < 39) + { + fType->fType = DDL_DECIMAL; + fType->fLength = 16; + } } -} // end of namespace +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 83dfa57fe..baa5bd42c 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ddlpkg.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ddlpkg.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file * * Class definitions for ddl parser objects. These are the things @@ -99,286 +99,247 @@ std::ostream& operator<<(std::ostream& os, const ColumnNameList& columnNames); std::ostream& operator<<(std::ostream& os, const ReferentialAction& ref); std::ostream& operator<<(std::ostream& os, const TableDef& tableDef); - /** @brief Verb List -* Make sure to keep the enum and string list in-sync -*/ + * Make sure to keep the enum and string list in-sync + */ enum DDL_VERBS { - DDL_CREATE, - DDL_ALTER, - DDL_DROP, - DDL_INVALID_VERB + DDL_CREATE, + DDL_ALTER, + DDL_DROP, + DDL_INVALID_VERB }; /** @brief Subject List -* Make sure to keep the enum and string list in-sync -*/ + * Make sure to keep the enum and string list in-sync + */ enum DDL_SUBJECTS { - DDL_TABLE, - DDL_INDEX, - DDL_INVALID_SUBJECT + DDL_TABLE, + DDL_INDEX, + DDL_INVALID_SUBJECT }; enum DDL_CONSTRAINT_ATTRIBUTES { - DDL_DEFERRABLE, - DDL_NON_DEFERRABLE, - DDL_INITIALLY_IMMEDIATE, - DDL_INITIALLY_DEFERRED, - DDL_INVALID_ATTRIBUTE -}; - -const std::string ConstraintAttrStrings[] = -{ - "deferrable", - "non-deferrable", - "initially-immediate", - "initially-deferred", - "invalid" + DDL_DEFERRABLE, + DDL_NON_DEFERRABLE, + DDL_INITIALLY_IMMEDIATE, + DDL_INITIALLY_DEFERRED, + DDL_INVALID_ATTRIBUTE }; +const std::string ConstraintAttrStrings[] = {"deferrable", "non-deferrable", "initially-immediate", + "initially-deferred", "invalid"}; enum DDL_REFERENTIAL_ACTION { - DDL_CASCADE, - DDL_SET_NULL, - DDL_SET_DEFAULT, - DDL_NO_ACTION, - DDL_RESTRICT, - DDL_INVALID_REFERENTIAL_ACTION + DDL_CASCADE, + DDL_SET_NULL, + DDL_SET_DEFAULT, + DDL_NO_ACTION, + DDL_RESTRICT, + DDL_INVALID_REFERENTIAL_ACTION }; -const std::string ReferentialActionStrings[] = -{ - "cascade", - "set_null", - "set_default", - "no_action", - "invalid_action" -}; +const std::string ReferentialActionStrings[] = {"cascade", "set_null", "set_default", "no_action", + "invalid_action"}; enum DDL_MATCH_TYPE { - DDL_FULL, - DDL_PARTIAL, - DDL_INVALID_MATCH_TYPE -}; - -const std::string MatchTypeStrings[] = -{ - "full", - "partial", - "invalid_match_type" + DDL_FULL, + DDL_PARTIAL, + DDL_INVALID_MATCH_TYPE }; +const std::string MatchTypeStrings[] = {"full", "partial", "invalid_match_type"}; /** @brief Constraint List * Make sure to keep the enum and string list in-sync */ enum DDL_CONSTRAINTS { - DDL_PRIMARY_KEY, - DDL_FOREIGN_KEY, - DDL_CHECK, - DDL_UNIQUE, - DDL_REFERENCES, - DDL_NOT_NULL, - DDL_AUTO_INCREMENT, - DDL_INVALID_CONSTRAINT + DDL_PRIMARY_KEY, + DDL_FOREIGN_KEY, + DDL_CHECK, + DDL_UNIQUE, + DDL_REFERENCES, + DDL_NOT_NULL, + DDL_AUTO_INCREMENT, + DDL_INVALID_CONSTRAINT }; /** @brief */ -const std::string ConstraintString[] = -{ - "primary", - "foreign", - "check", - "unique", - "references", - "not_null", - "auto_increment" - "" -}; +const std::string ConstraintString[] = {"primary", + "foreign", + "check", + "unique", + "references", + "not_null", + "auto_increment" + ""}; /** @brief Datatype List * Make sure to keep the enum, string, and length list in-sync */ enum DDL_DATATYPES { - DDL_BIT, - DDL_TINYINT, - DDL_CHAR, - DDL_SMALLINT, - DDL_DECIMAL, - DDL_MEDINT, - DDL_INT, - DDL_FLOAT, - DDL_DATE, - DDL_BIGINT, - DDL_DOUBLE, - DDL_DATETIME, - DDL_VARCHAR, - DDL_VARBINARY, - DDL_CLOB, - DDL_BLOB, - DDL_REAL, - DDL_NUMERIC, - DDL_NUMBER, - DDL_INTEGER, - DDL_UNSIGNED_TINYINT, - DDL_UNSIGNED_SMALLINT, - DDL_UNSIGNED_MEDINT, - DDL_UNSIGNED_INT, - DDL_UNSIGNED_BIGINT, - DDL_UNSIGNED_DECIMAL, - DDL_UNSIGNED_FLOAT, - DDL_UNSIGNED_DOUBLE, - DDL_UNSIGNED_NUMERIC, - DDL_TEXT, - DDL_TIME, - DDL_TIMESTAMP, - DDL_INVALID_DATATYPE + DDL_BIT, + DDL_TINYINT, + DDL_CHAR, + DDL_SMALLINT, + DDL_DECIMAL, + DDL_MEDINT, + DDL_INT, + DDL_FLOAT, + DDL_DATE, + DDL_BIGINT, + DDL_DOUBLE, + DDL_DATETIME, + DDL_VARCHAR, + DDL_VARBINARY, + DDL_CLOB, + DDL_BLOB, + DDL_REAL, + DDL_NUMERIC, + DDL_NUMBER, + DDL_INTEGER, + DDL_UNSIGNED_TINYINT, + DDL_UNSIGNED_SMALLINT, + DDL_UNSIGNED_MEDINT, + DDL_UNSIGNED_INT, + DDL_UNSIGNED_BIGINT, + DDL_UNSIGNED_DECIMAL, + DDL_UNSIGNED_FLOAT, + DDL_UNSIGNED_DOUBLE, + DDL_UNSIGNED_NUMERIC, + DDL_TEXT, + DDL_TIME, + DDL_TIMESTAMP, + DDL_INVALID_DATATYPE }; /** @brief Datatype string list */ -const std::string DDLDatatypeString[] = -{ - "bit", - "tinyint", - "char", - "smallint", - "decimal", - "medint", - "integer", - "float", - "date", - "bigint", - "double", - "datetime", - "varchar", - "varbinary", - "clob", - "blob", - "real", - "numeric", - "number", - "integer", - "unsigned-tinyint", - "unsigned-smallint", - "unsigned-medint", - "unsigned-int", - "unsigned-bigint", - "unsigned-decimal", - "unsigned-float", - "unsigned-double", - "unsigned-numeric", - "text", - "time", - "timestamp", - "" -}; +const std::string DDLDatatypeString[] = {"bit", + "tinyint", + "char", + "smallint", + "decimal", + "medint", + "integer", + "float", + "date", + "bigint", + "double", + "datetime", + "varchar", + "varbinary", + "clob", + "blob", + "real", + "numeric", + "number", + "integer", + "unsigned-tinyint", + "unsigned-smallint", + "unsigned-medint", + "unsigned-int", + "unsigned-bigint", + "unsigned-decimal", + "unsigned-float", + "unsigned-double", + "unsigned-numeric", + "text", + "time", + "timestamp", + ""}; /** @brief Alter table action string list */ -const std::string AlterActionString[] = -{ - "AtaAddColumn", - "AtaAddColumns", - "AtaDropColumn", - "AtaDropColumns", - "AtaAddTableConstraint", - "AtaSetColumnDefault", - "AtaDropColumnDefault", - "AtaDropTableConstraint", - "AtaRenameTable", - "AtaModifyColumnType", - "AtaRenameColumn", - "AtaTableComment" -}; +const std::string AlterActionString[] = { + "AtaAddColumn", "AtaAddColumns", "AtaDropColumn", "AtaDropColumns", + "AtaAddTableConstraint", "AtaSetColumnDefault", "AtaDropColumnDefault", "AtaDropTableConstraint", + "AtaRenameTable", "AtaModifyColumnType", "AtaRenameColumn", "AtaTableComment"}; /** @brief Datatype Length list * */ -const int DDLDatatypeLength[] = -{ - 1, // BIT - 1, // TINYINT - 1, // CHAR - 2, // SMALLINT - 2, // DECIMAL - 4, // MEDINT - 4, // INT - 4, // FLOAT - 4, // DATE - 8, // BIGINT - 8, // DOUBLE - 8, // DATETIME - 8, // VARCHAR - 8, // VARBINAR - 8, // CLOB - 8, // BLOB - 4, // REAL - 2, // NUMERIC - 4, // NUMBER - 4, // INTEGER - 1, // UNSIGNED_TINYINT, - 2, // UNSIGNED_SMALLINT, - 4, // UNSIGNED_MEDINT, - 4, // UNSIGNED_INT, - 8, // UNSIGNED_BIGINT, - 2, // UNSIGNED_DECIMAL, - 4, // UNSIGNED_FLOAT, - 8, // UNSIGNED_DOUBLE, - 2, // UNSIGNED_NUMERIC, - 8, // TEXT - 8, // TIME - 8, // TIMESTAMP - -1 // INVALID LENGTH +const int DDLDatatypeLength[] = { + 1, // BIT + 1, // TINYINT + 1, // CHAR + 2, // SMALLINT + 2, // DECIMAL + 4, // MEDINT + 4, // INT + 4, // FLOAT + 4, // DATE + 8, // BIGINT + 8, // DOUBLE + 8, // DATETIME + 8, // VARCHAR + 8, // VARBINAR + 8, // CLOB + 8, // BLOB + 4, // REAL + 2, // NUMERIC + 4, // NUMBER + 4, // INTEGER + 1, // UNSIGNED_TINYINT, + 2, // UNSIGNED_SMALLINT, + 4, // UNSIGNED_MEDINT, + 4, // UNSIGNED_INT, + 8, // UNSIGNED_BIGINT, + 2, // UNSIGNED_DECIMAL, + 4, // UNSIGNED_FLOAT, + 8, // UNSIGNED_DOUBLE, + 2, // UNSIGNED_NUMERIC, + 8, // TEXT + 8, // TIME + 8, // TIMESTAMP + -1 // INVALID LENGTH }; enum DDL_SERIAL_TYPE { - DDL_TABLE_DEF, - DDL_COLUMN_DEF, - DDL_COLUMN_CONSTRAINT_DEF, - DDL_TABLE_CONSTRAINT_DEF, - DDL_SQL_STATEMENT_LIST, - DDL_CREATE_TABLE_STATEMENT, - DDL_CREATE_INDEX, - DDL_ALTER_TABLE_STATEMENT, - DDL_ATA_ADD_COLUMN, - DDL_ATA_ADD_COLUMNS, - DDL_ATA_DROP_COLUMN, - DDL_ATA_ADD_TABLE_CONSTRAINT, - DDL_ATA_SET_COLUMN_DEFAULT, - DDL_ATA_DROP_COLUMN_DEFAULT, - DDL_ATA_DROP_TABLE_CONSTRAINT, - DDL_ATA_RENAME_TABLE, - DDL_ATA_RENAME_COLUMN, - DDL_ATA_MODIFY_COLUMN_TYPE, - DDL_ATA_TABLE_COMMENT, - DDL_COLUMN_TYPE, - DDL_COLUMN_DEFAULT_VALUE, - DDL_TABLE_UNIQUE_CONSTRAINT_DEF, - DDL_TABLE_PRIMARY_CONSTRAINT_DEF, - DDL_REF_ACTION, - DDL_TABLE_REFERENCES_CONSTRAINT_DEF, - DDL_TABLE_CHECK_CONSTRAINT_DEF, - DDL_QUALIFIED_NAME, - DDL_CONSTRAINT_ATTRIBUTES_DEF, - DDL_DROP_INDEX_STATEMENT, - DDL_DROP_TABLE_STATEMENT, - DDL_ATA_DROP_COLUMNS, - DDL_NULL, - DDL_INVALID_SERIAL_TYPE, - DDL_TRUNC_TABLE_STATEMENT, - DDL_MARK_PARTITION_STATEMENT, - DDL_RESTORE_PARTITION_STATEMENT, - DDL_DROP_PARTITION_STATEMENT + DDL_TABLE_DEF, + DDL_COLUMN_DEF, + DDL_COLUMN_CONSTRAINT_DEF, + DDL_TABLE_CONSTRAINT_DEF, + DDL_SQL_STATEMENT_LIST, + DDL_CREATE_TABLE_STATEMENT, + DDL_CREATE_INDEX, + DDL_ALTER_TABLE_STATEMENT, + DDL_ATA_ADD_COLUMN, + DDL_ATA_ADD_COLUMNS, + DDL_ATA_DROP_COLUMN, + DDL_ATA_ADD_TABLE_CONSTRAINT, + DDL_ATA_SET_COLUMN_DEFAULT, + DDL_ATA_DROP_COLUMN_DEFAULT, + DDL_ATA_DROP_TABLE_CONSTRAINT, + DDL_ATA_RENAME_TABLE, + DDL_ATA_RENAME_COLUMN, + DDL_ATA_MODIFY_COLUMN_TYPE, + DDL_ATA_TABLE_COMMENT, + DDL_COLUMN_TYPE, + DDL_COLUMN_DEFAULT_VALUE, + DDL_TABLE_UNIQUE_CONSTRAINT_DEF, + DDL_TABLE_PRIMARY_CONSTRAINT_DEF, + DDL_REF_ACTION, + DDL_TABLE_REFERENCES_CONSTRAINT_DEF, + DDL_TABLE_CHECK_CONSTRAINT_DEF, + DDL_QUALIFIED_NAME, + DDL_CONSTRAINT_ATTRIBUTES_DEF, + DDL_DROP_INDEX_STATEMENT, + DDL_DROP_TABLE_STATEMENT, + DDL_ATA_DROP_COLUMNS, + DDL_NULL, + DDL_INVALID_SERIAL_TYPE, + DDL_TRUNC_TABLE_STATEMENT, + DDL_MARK_PARTITION_STATEMENT, + DDL_RESTORE_PARTITION_STATEMENT, + DDL_DROP_PARTITION_STATEMENT }; - /** @brief An abstract base for TableDef, ColumnDef, ... * * The primary purpose of this class is to provide a unified type @@ -392,24 +353,21 @@ enum DDL_SERIAL_TYPE */ struct SchemaObject { - virtual ~SchemaObject() - {} + virtual ~SchemaObject() + { + } - SchemaObject(std::string name): - fName(name) - {} + SchemaObject(std::string name) : fName(name) + { + } - SchemaObject() : - fName("unnamed") - {} - - std::string fName; + SchemaObject() : fName("unnamed") + { + } + std::string fName; }; - - - /** @brief SqlStatement represents a toplevel * syntactic element such as a create table or alter table SQL * statement. @@ -420,46 +378,41 @@ struct SchemaObject */ struct SqlStatement { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs) = 0; - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const = 0; - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const = 0; + EXPORT SqlStatement(); - EXPORT SqlStatement(); + EXPORT virtual ~SqlStatement(); - EXPORT virtual ~SqlStatement(); + /** @brief The session ID assigned to this stmt by the front end (in theory) + * + * XXXPAT: need to fix this. It should be type execplan::SessionManager::SID, but + * that causes a circular dependency in the header files. Should unravel that at + * some point. + * + * Right now this var is initialized from a counter. At some point we need + * to serialize/unserialize it from a byte stream. + */ + uint32_t fSessionID; - /** @brief The session ID assigned to this stmt by the front end (in theory) - * - * XXXPAT: need to fix this. It should be type execplan::SessionManager::SID, but - * that causes a circular dependency in the header files. Should unravel that at - * some point. - * - * Right now this var is initialized from a counter. At some point we need - * to serialize/unserialize it from a byte stream. - */ - uint32_t fSessionID; + /** @brief The original sql string + */ + std::string fSql; - /** @brief The original sql string - */ - std::string fSql; - - /** @brief the default schema (owner that will be used when not specified) - */ - std::string fOwner; - - - uint32_t fTableWithAutoi; // has autoincrement column? + /** @brief the default schema (owner that will be used when not specified) + */ + std::string fOwner; + uint32_t fTableWithAutoi; // has autoincrement column? }; - - /** @brief Collects SqlStatements so that we can support the * parsing of sqltext containing multiple statements. * @@ -469,30 +422,27 @@ struct SqlStatement */ struct SqlStatementList { - SqlStatementList() - {} + SqlStatementList() + { + } - SqlStatement* operator[](int i) const - { - return fList[i]; - } + SqlStatement* operator[](int i) const + { + return fList[i]; + } + virtual ~SqlStatementList(); - virtual ~SqlStatementList(); + /** @brief Add a statement to the underlying container. */ + void push_back(SqlStatement* v); - /** @brief Add a statement to the underlying container. */ - void push_back(SqlStatement* v); - - std::vector fList; - std::string fSqlText; - -private: - SqlStatementList(const SqlStatementList& x); + std::vector fList; + std::string fSqlText; + private: + SqlStatementList(const SqlStatementList& x); }; - - /** @brief Stores catalog, schema, object names. * * ddl.y does not yet support catalog. So, expect catalog @@ -500,107 +450,100 @@ private: */ struct QualifiedName { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + QualifiedName() + { + } - QualifiedName() - {} + EXPORT QualifiedName(const char* name); + EXPORT QualifiedName(const char* name, const char* schema); + EXPORT QualifiedName(const char* name, const char* schema, const char* catalog); - EXPORT QualifiedName(const char* name); - EXPORT QualifiedName(const char* name, const char* schema); - EXPORT QualifiedName(const char* name, const char* schema, const char* catalog); + virtual ~QualifiedName() + { + } - virtual ~QualifiedName() - {} - - std::string fCatalog; - std::string fName; - std::string fSchema; + std::string fCatalog; + std::string fName; + std::string fSchema; }; - - /** TableDef represents a table definition. */ struct TableDef : public SchemaObject { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + TableDef() : fQualifiedName(0) + { + } - TableDef() : fQualifiedName(0) - {} + EXPORT TableDef(QualifiedName* name, TableElementList* elements, TableOptionMap* options); - EXPORT TableDef(QualifiedName* name, TableElementList* elements, TableOptionMap* options); + /** @brief TableDef ctor. + * ctor + */ + TableDef(QualifiedName* name, ColumnDefList columns, TableConstraintDefList constraints, + int tableWithAutoinc) + : fQualifiedName(name), fColumns(columns), fConstraints(constraints) - /** @brief TableDef ctor. - * ctor - */ - TableDef( QualifiedName* name, - ColumnDefList columns, - TableConstraintDefList constraints, int tableWithAutoinc) : - fQualifiedName (name), - fColumns (columns), - fConstraints (constraints) + { + } + EXPORT virtual ~TableDef(); - {} + QualifiedName* fQualifiedName; - EXPORT virtual ~TableDef(); + ColumnDefList fColumns; + TableConstraintDefList fConstraints; - QualifiedName* fQualifiedName; - - ColumnDefList fColumns; - TableConstraintDefList fConstraints; - - TableOptionMap fOptions; + TableOptionMap fOptions; }; - - /** @brief Represents the create table statement * * @note It takes possession of the TableDef given to it. */ struct CreateTableStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - EXPORT CreateTableStatement(); + /** @brief Ctor for deserialization */ + EXPORT CreateTableStatement(); - /** @brief You can't have a CreateTableStatement without a - table defintion */ - EXPORT CreateTableStatement(TableDef* tableDef); + /** @brief You can't have a CreateTableStatement without a + table defintion */ + EXPORT CreateTableStatement(TableDef* tableDef); - EXPORT virtual ~CreateTableStatement(); + EXPORT virtual ~CreateTableStatement(); - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - std::string schemaName() const - { - if (!fTableDef || !fTableDef->fQualifiedName) return "UNKNOWN"; + std::string schemaName() const + { + if (!fTableDef || !fTableDef->fQualifiedName) + return "UNKNOWN"; - return fTableDef->fQualifiedName->fSchema; - } - - TableDef* fTableDef; ///< The table defintion. + return fTableDef->fQualifiedName->fSchema; + } + TableDef* fTableDef; ///< The table defintion. }; - /** * @brief The subforms of alter table are represented as * subclasses of AlterTableAction @@ -615,445 +558,436 @@ struct CreateTableStatement : public SqlStatement */ struct AlterTableAction { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs) = 0; - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs) = 0; - /** @brief Ctor for deserialization */ - AlterTableAction() - {} + /** @brief Ctor for deserialization */ + AlterTableAction() + { + } - virtual ~AlterTableAction() - {} + virtual ~AlterTableAction() + { + } - /** @brief QualifiedName of the focal table for this - statement. */ - // QualifiedName *fTableName; + /** @brief QualifiedName of the focal table for this + statement. */ + // QualifiedName *fTableName; - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; }; - - - /** @brief Represents alter table add column forms. */ struct AtaAddColumn : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaAddColumn() : fColumnDef(0) {} + /** @brief Ctor for deserialization */ + AtaAddColumn() : fColumnDef(0) + { + } - /** @brief You can't add a column without specifying a column - definition. */ - AtaAddColumn(ColumnDef* columnDef); + /** @brief You can't add a column without specifying a column + definition. */ + AtaAddColumn(ColumnDef* columnDef); - virtual ~AtaAddColumn(); + virtual ~AtaAddColumn(); - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief The focal column definition. */ - ColumnDef* fColumnDef; + /** @brief The focal column definition. */ + ColumnDef* fColumnDef; }; - /** @brief Represents the table_element_list style of add column - which is not part of SQL-92. + which is not part of SQL-92. */ struct AtaAddColumns : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaAddColumns() - {} + /** @brief Ctor for deserialization */ + AtaAddColumns() + { + } - AtaAddColumns(TableElementList* tableElements); + AtaAddColumns(TableElementList* tableElements); - virtual ~AtaAddColumns(); + virtual ~AtaAddColumns(); - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - ColumnDefList fColumns; + ColumnDefList fColumns; }; - /** @brief Represents the table_element_list style of drop column - which is not part of SQL-92. + which is not part of SQL-92. */ struct AtaDropColumns : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaDropColumns() - {} + /** @brief Ctor for deserialization */ + AtaDropColumns() + { + } - EXPORT AtaDropColumns(ColumnNameList* tableElements); + EXPORT AtaDropColumns(ColumnNameList* tableElements); - EXPORT virtual ~AtaDropColumns(); + EXPORT virtual ~AtaDropColumns(); - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - ColumnNameList fColumns; + ColumnNameList fColumns; }; - - /** AtaAddTableConstraint */ struct AtaAddTableConstraint : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaAddTableConstraint() : fTableConstraint(0) - {} + /** @brief Ctor for deserialization */ + AtaAddTableConstraint() : fTableConstraint(0) + { + } - AtaAddTableConstraint(TableConstraintDef* tableConstraint); + AtaAddTableConstraint(TableConstraintDef* tableConstraint); - virtual ~AtaAddTableConstraint(); + virtual ~AtaAddTableConstraint(); - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - TableConstraintDef* fTableConstraint; + TableConstraintDef* fTableConstraint; }; - - /** @brief alter table drop column. */ struct AtaDropColumn : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaDropColumn() - {} + /** @brief Ctor for deserialization */ + AtaDropColumn() + { + } - /** @brief Ctor for parser construction */ - EXPORT AtaDropColumn(std::string columnName, DDL_REFERENTIAL_ACTION dropBehavior); + /** @brief Ctor for parser construction */ + EXPORT AtaDropColumn(std::string columnName, DDL_REFERENTIAL_ACTION dropBehavior); - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - virtual ~AtaDropColumn() - {} - std::string fColumnName; - DDL_REFERENTIAL_ACTION fDropBehavior; + virtual ~AtaDropColumn() + { + } + std::string fColumnName; + DDL_REFERENTIAL_ACTION fDropBehavior; }; - - /** @brief alter table set column default */ struct AtaSetColumnDefault : AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - AtaSetColumnDefault() : fDefaultValue(0) {} + AtaSetColumnDefault() : fDefaultValue(0) + { + } - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - virtual ~AtaSetColumnDefault(); + virtual ~AtaSetColumnDefault(); - AtaSetColumnDefault(const char* colName, ColumnDefaultValue* defaultValue); - - std::string fColumnName; - ColumnDefaultValue* fDefaultValue; + AtaSetColumnDefault(const char* colName, ColumnDefaultValue* defaultValue); + std::string fColumnName; + ColumnDefaultValue* fDefaultValue; }; - - /** @brief alter table drop column default. */ struct AtaDropColumnDefault : AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaDropColumnDefault() - {} + /** @brief Ctor for deserialization */ + AtaDropColumnDefault() + { + } - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - virtual ~AtaDropColumnDefault() - {} + virtual ~AtaDropColumnDefault() + { + } - /** @brief Ctor for parser construction */ - AtaDropColumnDefault(const char* colName); + /** @brief Ctor for parser construction */ + AtaDropColumnDefault(const char* colName); - std::string fColumnName; + std::string fColumnName; }; - /** @brief alter table drop table constraint. */ struct AtaDropTableConstraint : AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Ctor for deserialization */ + AtaDropTableConstraint() + { + } - /** @brief Ctor for deserialization */ - AtaDropTableConstraint() - {} + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + virtual ~AtaDropTableConstraint() + { + } - virtual ~AtaDropTableConstraint() - {} + AtaDropTableConstraint(const char* constraintName, DDL_REFERENTIAL_ACTION dropBehavior); - AtaDropTableConstraint(const char* constraintName, DDL_REFERENTIAL_ACTION dropBehavior); - - std::string fConstraintName; - DDL_REFERENTIAL_ACTION fDropBehavior; + std::string fConstraintName; + DDL_REFERENTIAL_ACTION fDropBehavior; }; - - /** alter table rename */ struct AtaRenameTable : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaRenameTable() : fQualifiedName(0) {} - AtaRenameTable(QualifiedName* qualifiedName); + /** @brief Ctor for deserialization */ + AtaRenameTable() : fQualifiedName(0) + { + } + AtaRenameTable(QualifiedName* qualifiedName); - /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + std::ostream& put(std::ostream& os) const; - virtual ~AtaRenameTable(); + virtual ~AtaRenameTable(); - QualifiedName* fQualifiedName; + QualifiedName* fQualifiedName; }; /** alter table comment */ struct AtaTableComment : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaTableComment() : fTableComment("") - {} - AtaTableComment(const char* tableComment); + /** @brief Ctor for deserialization */ + AtaTableComment() : fTableComment("") + { + } + AtaTableComment(const char* tableComment); - /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + std::ostream& put(std::ostream& os) const; - virtual ~AtaTableComment(); + virtual ~AtaTableComment(); - std::string fTableComment; + std::string fTableComment; }; - - /** @brief alter table modify column */ struct AtaModifyColumnType : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaModifyColumnType() : fColumnType(0) {} + /** @brief Ctor for deserialization */ + AtaModifyColumnType() : fColumnType(0) + { + } - /** @brief Ctor for parser construction */ - AtaModifyColumnType(const char* name, ColumnType* columnType) : - fColumnType(columnType), - fName(name) - {} + /** @brief Ctor for parser construction */ + AtaModifyColumnType(const char* name, ColumnType* columnType) : fColumnType(columnType), fName(name) + { + } - AtaModifyColumnType(QualifiedName* qualifiedName); + AtaModifyColumnType(QualifiedName* qualifiedName); - /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + std::ostream& put(std::ostream& os) const; - virtual ~AtaModifyColumnType(); + virtual ~AtaModifyColumnType(); - ColumnType* fColumnType; + ColumnType* fColumnType; - std::string fName; + std::string fName; }; - - /** @brief alter table rename column */ struct AtaRenameColumn : public AlterTableAction { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - AtaRenameColumn() : fNewType(0), fDefaultValue(0) { } + /** @brief Ctor for deserialization */ + AtaRenameColumn() : fNewType(0), fDefaultValue(0) + { + } - AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, const char* comment = NULL) : - fName(name), - fNewName(newName), - fNewType(newType) - { - if (comment) - fComment = comment; + AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, const char* comment = NULL) + : fName(name), fNewName(newName), fNewType(newType) + { + if (comment) + fComment = comment; - fDefaultValue = 0; - } + fDefaultValue = 0; + } - AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, ColumnConstraintList* constraint_list, - ColumnDefaultValue* defaultValue, const char* comment = NULL) : - fName(name), - fNewName(newName), - fNewType(newType), - fDefaultValue(defaultValue) - { - if (constraint_list) - fConstraints = *constraint_list; + AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, + ColumnConstraintList* constraint_list, ColumnDefaultValue* defaultValue, + const char* comment = NULL) + : fName(name), fNewName(newName), fNewType(newType), fDefaultValue(defaultValue) + { + if (constraint_list) + fConstraints = *constraint_list; - //if (defaultValue) - //{ - //fDefaultValue = defaultValue; - //} + // if (defaultValue) + //{ + // fDefaultValue = defaultValue; + //} - if (comment) - fComment = comment; - } + if (comment) + fComment = comment; + } - AtaRenameColumn(QualifiedName* qualifiedName); + AtaRenameColumn(QualifiedName* qualifiedName); - /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + std::ostream& put(std::ostream& os) const; - virtual ~AtaRenameColumn(); + virtual ~AtaRenameColumn(); - std::string fName; ///< current column name - std::string fNewName; ///< new column name - ColumnType* fNewType; - /** @brief Zero or more constraints. */ - ColumnConstraintList fConstraints; + std::string fName; ///< current column name + std::string fNewName; ///< new column name + ColumnType* fNewType; + /** @brief Zero or more constraints. */ + ColumnConstraintList fConstraints; - /** @brief NULL if there was no DEFAULT clause */ - ColumnDefaultValue* fDefaultValue; - std::string fComment; + /** @brief NULL if there was no DEFAULT clause */ + ColumnDefaultValue* fDefaultValue; + std::string fComment; }; - - - /** @brief Stores the type information for a column. */ struct ColumnType { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief For deserialization. */ - ColumnType() : fCharset(NULL), fExplicitLength(false) - {} + /** @brief For deserialization. */ + ColumnType() : fCharset(NULL), fExplicitLength(false) + { + } - friend std::ostream& operator<<(std::ostream& os, const ColumnType& ac); + friend std::ostream& operator<<(std::ostream& os, const ColumnType& ac); - /** @brief This constructor is used by the parser to construct the - ColumnType when a precision/scale clause is encountered. */ + /** @brief This constructor is used by the parser to construct the + ColumnType when a precision/scale clause is encountered. */ - EXPORT ColumnType(int prec, int scale); + EXPORT ColumnType(int prec, int scale); - /** @brief Used in cases where we don't need to create an - object until we have seen all it's parts. */ + /** @brief Used in cases where we don't need to create an + object until we have seen all it's parts. */ - EXPORT ColumnType(int type); + EXPORT ColumnType(int type); - virtual ~ColumnType() - {} + virtual ~ColumnType() + { + } - /** @brief Type code from DDL_DATATYPES */ - int fType; + /** @brief Type code from DDL_DATATYPES */ + int fType; - /** @brief Length of datatype in bytes */ - long fLength; + /** @brief Length of datatype in bytes */ + long fLength; - /** @brief SQL precision. This is the number of digits in the representation. */ - int fPrecision; + /** @brief SQL precision. This is the number of digits in the representation. */ + int fPrecision; - /** @brief SQL scale. This is is the number of digits to the - right of the decimal point. */ - int fScale; + /** @brief SQL scale. This is is the number of digits to the + right of the decimal point. */ + int fScale; - /** @brief SQL "with timezone" specifier */ - bool fWithTimezone; + /** @brief SQL "with timezone" specifier */ + bool fWithTimezone; - int fCompressiontype; + int fCompressiontype; - std::string fAutoincrement; + std::string fAutoincrement; - uint64_t fNextvalue; + uint64_t fNextvalue; - /** @brief Column charset (CHAR, VARCHAR and TEXT only) */ - const char* fCharset; - - /** @brief Is the TEXT column has explicit defined length, ie TEXT(1717) */ - bool fExplicitLength; + /** @brief Column charset (CHAR, VARCHAR and TEXT only) */ + const char* fCharset; + /** @brief Is the TEXT column has explicit defined length, ie TEXT(1717) */ + bool fExplicitLength; }; - - /** @brief A column constraint definition. * * Since we aren't supporting references constraint specifications @@ -1063,313 +997,284 @@ struct ColumnType */ struct ColumnConstraintDef : public SchemaObject { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + ColumnConstraintDef() + { + } - ColumnConstraintDef() - {} + /** @brief Constructs as check constraint. */ + EXPORT ColumnConstraintDef(const char* check); - /** @brief Constructs as check constraint. */ - EXPORT ColumnConstraintDef(const char* check); + /** @brief Constructs as other constraint. */ + EXPORT ColumnConstraintDef(DDL_CONSTRAINTS type); + virtual ~ColumnConstraintDef() + { + } - /** @brief Constructs as other constraint. */ - EXPORT ColumnConstraintDef(DDL_CONSTRAINTS type); + /** @brief Whether deferrable. */ + bool fDeferrable; - virtual ~ColumnConstraintDef() - {} + /** @brief Immediate or defferred */ + DDL_CONSTRAINT_ATTRIBUTES fCheckTime; - /** @brief Whether deferrable. */ - bool fDeferrable; + /** @brief Distinguish kinds of constraints */ + DDL_CONSTRAINTS fConstraintType; - /** @brief Immediate or defferred */ - DDL_CONSTRAINT_ATTRIBUTES fCheckTime; - - /** @brief Distinguish kinds of constraints */ - DDL_CONSTRAINTS fConstraintType; - - /** @brief Stores foo from check(foo) */ - std::string fCheck; + /** @brief Stores foo from check(foo) */ + std::string fCheck; }; - - - /** @brief Represents a columns default value. */ struct ColumnDefaultValue { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); + ColumnDefaultValue() + { + } - ColumnDefaultValue() - {} + ColumnDefaultValue(const char* value); - ColumnDefaultValue(const char* value); + virtual ~ColumnDefaultValue() + { + } - virtual ~ColumnDefaultValue() - {} + /** @brief Is NULL the default value? */ + bool fNull; - - /** @brief Is NULL the default value? */ - bool fNull; - - /** @brief Specified default value as a string. */ - std::string fValue; + /** @brief Specified default value as a string. */ + std::string fValue; }; - - /** @brief Represents a column definition. */ struct ColumnDef : public SchemaObject { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief For deserialization. */ + ColumnDef() : fType(0) + { + } - /** @brief For deserialization. */ - ColumnDef() : fType(0) {} + EXPORT virtual ~ColumnDef(); - EXPORT virtual ~ColumnDef(); + /** @brief Parser ctor. */ + EXPORT ColumnDef(const char* name, ColumnType* type, ColumnConstraintList* constraint_list, + ColumnDefaultValue* defaultValue, const char* comment = NULL); - /** @brief Parser ctor. */ - EXPORT ColumnDef(const char* name, - ColumnType* type, - ColumnConstraintList* constraint_list, - ColumnDefaultValue* defaultValue, const char* comment = NULL); + /** @brief ColumnDef ctor. + * ctor */ + ColumnDef(const char* name, ColumnType* type, ColumnConstraintList constraints, + ColumnDefaultValue* defaultValue = NULL, const char* comment = NULL) + : SchemaObject(name), fType(type), fConstraints(constraints), fDefaultValue(defaultValue) + { + } - /** @brief ColumnDef ctor. - * ctor */ - ColumnDef(const char* name, - ColumnType* type, - ColumnConstraintList constraints, - ColumnDefaultValue* defaultValue = NULL, const char* comment = NULL) : - SchemaObject(name), - fType (type), - fConstraints (constraints), - fDefaultValue (defaultValue) - {} + void convertDecimal(); - void convertDecimal(); + /** @brief Never NULL since all Columns must have a type. */ + ColumnType* fType; - /** @brief Never NULL since all Columns must have a type. */ - ColumnType* fType; + /** @brief Zero or more constraints. */ + ColumnConstraintList fConstraints; - /** @brief Zero or more constraints. */ - ColumnConstraintList fConstraints; + /** @brief NULL if there was no DEFAULT clause */ + ColumnDefaultValue* fDefaultValue; - /** @brief NULL if there was no DEFAULT clause */ - ColumnDefaultValue* fDefaultValue; - - std::string fComment; + std::string fComment; }; - - /** @brief Abstract base for table constraint definitions. */ struct TableConstraintDef : public SchemaObject { - /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() = 0; + /** @brief Return DDL_SERIAL code */ + virtual DDL_SERIAL_TYPE getSerialType() = 0; - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs) = 0; - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs) = 0; + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs) = 0; + TableConstraintDef(); - TableConstraintDef(); + TableConstraintDef(DDL_CONSTRAINTS cType); - TableConstraintDef(DDL_CONSTRAINTS cType); + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; - - virtual ~TableConstraintDef() - {} - // std::string fName; - DDL_CONSTRAINTS fConstraintType; + virtual ~TableConstraintDef() + { + } + // std::string fName; + DDL_CONSTRAINTS fConstraintType; }; - - - /** @brief Unique table constraint. */ struct TableUniqueConstraintDef : public TableConstraintDef { - /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() - { - return DDL_TABLE_UNIQUE_CONSTRAINT_DEF; - } + /** @brief Return DDL_SERIAL code */ + virtual DDL_SERIAL_TYPE getSerialType() + { + return DDL_TABLE_UNIQUE_CONSTRAINT_DEF; + } + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + TableUniqueConstraintDef() : TableConstraintDef(DDL_UNIQUE) + { + } + TableUniqueConstraintDef(ColumnNameList* columns); + virtual ~TableUniqueConstraintDef() + { + } - TableUniqueConstraintDef() : TableConstraintDef(DDL_UNIQUE) - {} + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - TableUniqueConstraintDef(ColumnNameList* columns); - virtual ~TableUniqueConstraintDef() - {} - - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; - - ColumnNameList fColumnNameList; + ColumnNameList fColumnNameList; }; - - /** @brief Primary key table constraint. */ struct TablePrimaryKeyConstraintDef : public TableConstraintDef { - /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() - { - return DDL_TABLE_PRIMARY_CONSTRAINT_DEF; - } + /** @brief Return DDL_SERIAL code */ + virtual DDL_SERIAL_TYPE getSerialType() + { + return DDL_TABLE_PRIMARY_CONSTRAINT_DEF; + } - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + TablePrimaryKeyConstraintDef() : TableConstraintDef(DDL_PRIMARY_KEY) + { + } - TablePrimaryKeyConstraintDef() : TableConstraintDef(DDL_PRIMARY_KEY) - {} + EXPORT TablePrimaryKeyConstraintDef(ColumnNameList* columns); - EXPORT TablePrimaryKeyConstraintDef(ColumnNameList* columns); + virtual ~TablePrimaryKeyConstraintDef() + { + } - virtual ~TablePrimaryKeyConstraintDef() - {} + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; - - ColumnNameList fColumnNameList; + ColumnNameList fColumnNameList; }; - - /** @brief ReferentialAction specifies what to do about table - relationships when elements are updated and deleted. + relationships when elements are updated and deleted. */ struct ReferentialAction { - virtual ~ReferentialAction() - {} + virtual ~ReferentialAction() + { + } - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - - DDL_REFERENTIAL_ACTION fOnUpdate; - DDL_REFERENTIAL_ACTION fOnDelete; + DDL_REFERENTIAL_ACTION fOnUpdate; + DDL_REFERENTIAL_ACTION fOnDelete; }; - - /** @brief TableReferencesConstraintDef represents a foreign key constraint. */ struct TableReferencesConstraintDef : public TableConstraintDef { - /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() - { - return DDL_TABLE_REFERENCES_CONSTRAINT_DEF; - } + /** @brief Return DDL_SERIAL code */ + virtual DDL_SERIAL_TYPE getSerialType() + { + return DDL_TABLE_REFERENCES_CONSTRAINT_DEF; + } - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); + TableReferencesConstraintDef() : TableConstraintDef(DDL_REFERENCES), fTableName(0), fRefAction(0) + { + } - TableReferencesConstraintDef() : - TableConstraintDef(DDL_REFERENCES), - fTableName(0), - fRefAction(0) - {} + TableReferencesConstraintDef(ColumnNameList* columns, QualifiedName* fTableName, + ColumnNameList* foreignColumns, DDL_MATCH_TYPE matchType, + ReferentialAction* refAction); - TableReferencesConstraintDef - (ColumnNameList* columns, - QualifiedName* fTableName, - ColumnNameList* foreignColumns, - DDL_MATCH_TYPE matchType, - ReferentialAction* refAction); + virtual ~TableReferencesConstraintDef(); - virtual ~TableReferencesConstraintDef(); + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; - - ColumnNameList fColumns; - QualifiedName* fTableName; - ColumnNameList fForeignColumns; - DDL_MATCH_TYPE fMatchType; - ReferentialAction* fRefAction; + ColumnNameList fColumns; + QualifiedName* fTableName; + ColumnNameList fForeignColumns; + DDL_MATCH_TYPE fMatchType; + ReferentialAction* fRefAction; }; - - /** @brief Table check constraint. */ struct TableCheckConstraintDef : public TableConstraintDef { - /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() - { - return DDL_TABLE_CHECK_CONSTRAINT_DEF; - } + /** @brief Return DDL_SERIAL code */ + virtual DDL_SERIAL_TYPE getSerialType() + { + return DDL_TABLE_CHECK_CONSTRAINT_DEF; + } - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); + TableCheckConstraintDef() : TableConstraintDef(DDL_CHECK) + { + } - TableCheckConstraintDef() : TableConstraintDef(DDL_CHECK) - {} + TableCheckConstraintDef(const char* check); - TableCheckConstraintDef(const char* check); + /** @brief Dump to stdout. */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; - - virtual ~TableCheckConstraintDef() - {} - std::string fCheck; + virtual ~TableCheckConstraintDef() + { + } + std::string fCheck; }; - - /** @brief Represents the alter table command. * * All forms of alter_table_statements are represented as @@ -1378,67 +1283,66 @@ struct TableCheckConstraintDef : public TableConstraintDef struct AlterTableStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + AlterTableStatement() : fTableName(0) + { + } - AlterTableStatement() : fTableName(0) - {} + EXPORT AlterTableStatement(QualifiedName* qName, AlterTableActionList* ataList); - EXPORT AlterTableStatement(QualifiedName* qName, AlterTableActionList* ataList); + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Delete members. */ + EXPORT virtual ~AlterTableStatement(); - /** @brief Delete members. */ - EXPORT virtual ~AlterTableStatement(); + std::string schemaName() const + { + if (!fTableName) + return "UNKNOWN"; - std::string schemaName() const - { - if (!fTableName) return "UNKNOWN"; + return fTableName->fSchema; + } - return fTableName->fSchema; - } - - QualifiedName* fTableName; - AlterTableActionList fActions; - std::string fTimeZone; + QualifiedName* fTableName; + AlterTableActionList fActions; + std::string fTimeZone; }; - - /** @brief This is used during parsing when constraint attributes - are recognized. This is always before the parent column def - is recognized. That's why we need a separate structure during - parsing. When the column def is recognized, we'll just copy - these into the Column structure. + are recognized. This is always before the parent column def + is recognized. That's why we need a separate structure during + parsing. When the column def is recognized, we'll just copy + these into the Column structure. */ struct ConstraintAttributes { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); + ConstraintAttributes() + { + } - ConstraintAttributes() - {} + ConstraintAttributes(DDL_CONSTRAINT_ATTRIBUTES checkTime, bool deferrable) + : fCheckTime(checkTime), fDeferrable(deferrable) + { + } - ConstraintAttributes(DDL_CONSTRAINT_ATTRIBUTES checkTime, bool deferrable) : - fCheckTime(checkTime), - fDeferrable(deferrable) - {} + virtual ~ConstraintAttributes() + { + } - virtual ~ConstraintAttributes() - {} - - - DDL_CONSTRAINT_ATTRIBUTES fCheckTime; - bool fDeferrable; + DDL_CONSTRAINT_ATTRIBUTES fCheckTime; + bool fDeferrable; }; /** @brief CreateIndex represents the CreateIndex operation. @@ -1448,115 +1352,117 @@ struct ConstraintAttributes */ struct CreateIndexStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); - CreateIndexStatement(); - CreateIndexStatement(QualifiedName* qualifiedName1, QualifiedName* qualifiedName2, - ColumnNameList* columnNames, bool unique); + CreateIndexStatement(); + CreateIndexStatement(QualifiedName* qualifiedName1, QualifiedName* qualifiedName2, + ColumnNameList* columnNames, bool unique); - /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + std::ostream& put(std::ostream& os) const; - virtual ~CreateIndexStatement(); + virtual ~CreateIndexStatement(); - QualifiedName* fIndexName; - QualifiedName* fTableName; - ColumnNameList fColumnNames; - bool fUnique; + QualifiedName* fIndexName; + QualifiedName* fTableName; + ColumnNameList fColumnNames; + bool fUnique; }; /** @brief DropIndexStatement represents the drop index operation. */ struct DropIndexStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + virtual int serialize(messageqcpp::ByteStream& bs); + DropIndexStatement() : fIndexName(0) + { + } + DropIndexStatement(QualifiedName* qualifiedName); - DropIndexStatement() : fIndexName(0) - {} - DropIndexStatement(QualifiedName* qualifiedName); + /** @brief Dump to stdout. */ + std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + virtual ~DropIndexStatement(); - virtual ~DropIndexStatement(); - - QualifiedName* fIndexName; + QualifiedName* fIndexName; }; /** @brief DropTableStatement represents the drop table operation */ struct DropTableStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + DropTableStatement() : fTableName(0) + { + } + EXPORT DropTableStatement(QualifiedName* qualifiedName, bool cascade); - DropTableStatement() : fTableName(0) - {} - EXPORT DropTableStatement(QualifiedName* qualifiedName, bool cascade); + /** @brief Dump to stdout. */ + EXPORT std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - EXPORT std::ostream& put(std::ostream& os) const; + virtual ~DropTableStatement() + { + delete fTableName; + } - virtual ~DropTableStatement() - { - delete fTableName; - } + std::string schemaName() const + { + if (!fTableName) + return "UNKNOWN"; - std::string schemaName() const - { - if (!fTableName) return "UNKNOWN"; + return fTableName->fSchema; + } - return fTableName->fSchema; - } - - QualifiedName* fTableName; - bool fCascade; + QualifiedName* fTableName; + bool fCascade; }; /** @brief TruncTableStatement represents the drop table operation */ struct TruncTableStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + TruncTableStatement() : fTableName(0) + { + } + EXPORT TruncTableStatement(QualifiedName* qualifiedName); - TruncTableStatement() : fTableName(0) - {} - EXPORT TruncTableStatement(QualifiedName* qualifiedName); + /** @brief Dump to stdout. */ + EXPORT std::ostream& put(std::ostream& os) const; - /** @brief Dump to stdout. */ - EXPORT std::ostream& put(std::ostream& os) const; + virtual ~TruncTableStatement() + { + delete fTableName; + } - virtual ~TruncTableStatement() - { - delete fTableName; - } + std::string schemaName() const + { + if (!fTableName) + return "UNKNOWN"; - std::string schemaName() const - { - if (!fTableName) return "UNKNOWN"; + return fTableName->fSchema; + } - return fTableName->fSchema; - } - - QualifiedName* fTableName; + QualifiedName* fTableName; }; /** @brief Represents the mark partition out of service statement @@ -1564,29 +1470,30 @@ struct TruncTableStatement : public SqlStatement */ struct MarkPartitionStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - MarkPartitionStatement() : fTableName(0) - {} + /** @brief Ctor for deserialization */ + MarkPartitionStatement() : fTableName(0) + { + } - /** @brief You can't have a CreateTableStatement without a table defintion */ - EXPORT MarkPartitionStatement(QualifiedName* qualifiedName); + /** @brief You can't have a CreateTableStatement without a table defintion */ + EXPORT MarkPartitionStatement(QualifiedName* qualifiedName); - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - virtual ~MarkPartitionStatement() - { - delete fTableName; - } + virtual ~MarkPartitionStatement() + { + delete fTableName; + } - QualifiedName* fTableName; ///< The table defintion - std::set fPartitions; // partition numbers + QualifiedName* fTableName; ///< The table defintion + std::set fPartitions; // partition numbers }; /** @brief Represents the mark partition out of service statement @@ -1594,28 +1501,29 @@ struct MarkPartitionStatement : public SqlStatement */ struct RestorePartitionStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - RestorePartitionStatement() : fTableName(0) - {} + /** @brief Ctor for deserialization */ + RestorePartitionStatement() : fTableName(0) + { + } - EXPORT RestorePartitionStatement(QualifiedName* qualifiedName); + EXPORT RestorePartitionStatement(QualifiedName* qualifiedName); - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - virtual ~RestorePartitionStatement() - { - delete fTableName; - } + virtual ~RestorePartitionStatement() + { + delete fTableName; + } - QualifiedName* fTableName; ///< The table name. - std::set fPartitions; // partition numbers + QualifiedName* fTableName; ///< The table name. + std::set fPartitions; // partition numbers }; /** @brief Represents the mark partition out of service statement @@ -1623,31 +1531,32 @@ struct RestorePartitionStatement : public SqlStatement */ struct DropPartitionStatement : public SqlStatement { - /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); - /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - /** @brief Ctor for deserialization */ - DropPartitionStatement() : fTableName(0) - {} + /** @brief Ctor for deserialization */ + DropPartitionStatement() : fTableName(0) + { + } - EXPORT DropPartitionStatement(QualifiedName* qualifiedName); + EXPORT DropPartitionStatement(QualifiedName* qualifiedName); - /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + /** @brief Dump to stdout. */ + EXPORT virtual std::ostream& put(std::ostream& os) const; - virtual ~DropPartitionStatement() - { - delete fTableName; - } + virtual ~DropPartitionStatement() + { + delete fTableName; + } - QualifiedName* fTableName; ///< The table name. - std::set fPartitions; // partition numbers + QualifiedName* fTableName; ///< The table name. + std::set fPartitions; // partition numbers }; -} +} // namespace ddlpackage #undef EXPORT diff --git a/dbcon/ddlpackage/dropindex.cpp b/dbcon/ddlpackage/dropindex.cpp index 4a20dae6d..7e4e929ed 100644 --- a/dbcon/ddlpackage/dropindex.cpp +++ b/dbcon/ddlpackage/dropindex.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dropindex.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dropindex.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "ddlpkg.h" @@ -29,18 +29,17 @@ using namespace std; DropIndexStatement::~DropIndexStatement() { - delete fIndexName; + delete fIndexName; } -DropIndexStatement::DropIndexStatement(QualifiedName* qualifiedName) : - fIndexName(qualifiedName) +DropIndexStatement::DropIndexStatement(QualifiedName* qualifiedName) : fIndexName(qualifiedName) { } std::ostream& DropIndexStatement::put(std::ostream& os) const { - os << "Drop Index: " << *fIndexName << endl; - return os; + os << "Drop Index: " << *fIndexName << endl; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/droppartition.cpp b/dbcon/ddlpackage/droppartition.cpp index b89094cb1..c0a7ed975 100644 --- a/dbcon/ddlpackage/droppartition.cpp +++ b/dbcon/ddlpackage/droppartition.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: markpartition.cpp 6566 2010-04-27 18:02:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: markpartition.cpp 6566 2010-04-27 18:02:51Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -29,23 +29,21 @@ using namespace std; namespace ddlpackage { - -DropPartitionStatement::DropPartitionStatement(QualifiedName* qualifiedName) : - fTableName(qualifiedName) +DropPartitionStatement::DropPartitionStatement(QualifiedName* qualifiedName) : fTableName(qualifiedName) { } ostream& DropPartitionStatement::put(ostream& os) const { - os << "Mark partitions out of service: " << *fTableName << endl; - os << " partitions: "; - set::const_iterator it; + os << "Mark partitions out of service: " << *fTableName << endl; + os << " partitions: "; + set::const_iterator it; - for (it = fPartitions.begin(); it != fPartitions.end(); ++it) - os << (*it) << " "; + for (it = fPartitions.begin(); it != fPartitions.end(); ++it) + os << (*it) << " "; - os << endl; - return os; + os << endl; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/droptable.cpp b/dbcon/ddlpackage/droptable.cpp index c22863b25..89fb7e984 100644 --- a/dbcon/ddlpackage/droptable.cpp +++ b/dbcon/ddlpackage/droptable.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: droptable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: droptable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -29,28 +29,26 @@ using namespace std; namespace ddlpackage { - -DropTableStatement::DropTableStatement(QualifiedName* qualifiedName, bool cascade) : - fTableName(qualifiedName), - fCascade(cascade) +DropTableStatement::DropTableStatement(QualifiedName* qualifiedName, bool cascade) + : fTableName(qualifiedName), fCascade(cascade) { } ostream& DropTableStatement::put(ostream& os) const { - os << "Drop Table: " << *fTableName << " " << "C=" << fCascade << endl; - return os; + os << "Drop Table: " << *fTableName << " " + << "C=" << fCascade << endl; + return os; } -TruncTableStatement::TruncTableStatement(QualifiedName* qualifiedName) : - fTableName(qualifiedName) +TruncTableStatement::TruncTableStatement(QualifiedName* qualifiedName) : fTableName(qualifiedName) { } ostream& TruncTableStatement::put(ostream& os) const { - os << "Truncate Table: " << *fTableName << endl; - return os; + os << "Truncate Table: " << *fTableName << endl; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/gramtest.cpp b/dbcon/ddlpackage/gramtest.cpp index 9bf01a3fb..5dd48af8e 100644 --- a/dbcon/ddlpackage/gramtest.cpp +++ b/dbcon/ddlpackage/gramtest.cpp @@ -31,48 +31,47 @@ namespace po = boost::program_options; int main(int argc, char* argv[]) { - string sqlfile; - int count; + string sqlfile; + int count; - po::options_description desc ("Allowed options"); - desc.add_options () - ("help", "produce help message") - ("bisond", /* po::value (),*/ "Have bison produce debug output") - ("count", po::value (), "number of runs") - ("sql", po::value < string > (), "sql file"); - po::variables_map vm; - po::store (po::parse_command_line (argc, argv, desc), vm); - po::notify (vm); + po::options_description desc("Allowed options"); + desc.add_options()("help", "produce help message")( + "bisond", + /* po::value (),*/ "Have bison produce debug output")("count", po::value(), + "number of runs")("sql", + po::value(), + "sql file"); + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); - if (vm.count ("sql")) - sqlfile = vm["sql"].as (); + if (vm.count("sql")) + sqlfile = vm["sql"].as(); + if (vm.count("count")) + count = vm["count"].as(); - if (vm.count("count")) - count = vm["count"].as(); + SqlFileParser parser; - SqlFileParser parser; + if (vm.count("bisond")) + parser.SetDebug(true); - if (vm.count ("bisond")) - parser.SetDebug(true); + parser.Parse(sqlfile); - parser.Parse(sqlfile); + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); + cout << "Parser succeeded." << endl; + cout << ptree.fList.size() << " " + << "SQL statements" << endl; + cout << ptree; + cout << endl; + } + else + { + cout << "Parser failed." << endl; + } - cout << "Parser succeeded." << endl; - cout << ptree.fList.size() << " " << "SQL statements" << endl; - cout << ptree; - cout << endl; - } - else - { - cout << "Parser failed." << endl; - } - - return parser.Good() ? 0 : -1; + return parser.Good() ? 0 : -1; } - - diff --git a/dbcon/ddlpackage/markpartition.cpp b/dbcon/ddlpackage/markpartition.cpp index 2f221afca..41b8bc6eb 100644 --- a/dbcon/ddlpackage/markpartition.cpp +++ b/dbcon/ddlpackage/markpartition.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: markpartition.cpp 6566 2010-04-27 18:02:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: markpartition.cpp 6566 2010-04-27 18:02:51Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -29,23 +29,21 @@ using namespace std; namespace ddlpackage { - -MarkPartitionStatement::MarkPartitionStatement(QualifiedName* qualifiedName) : - fTableName(qualifiedName) +MarkPartitionStatement::MarkPartitionStatement(QualifiedName* qualifiedName) : fTableName(qualifiedName) { } ostream& MarkPartitionStatement::put(ostream& os) const { - os << "Mark partition out of service: " << *fTableName; - os << " partitions: "; - set::const_iterator it; + os << "Mark partition out of service: " << *fTableName; + os << " partitions: "; + set::const_iterator it; - for (it = fPartitions.begin(); it != fPartitions.end(); ++it) - os << (*it) << " "; + for (it = fPartitions.begin(); it != fPartitions.end(); ++it) + os << (*it) << " "; - os << endl; - return os; + os << endl; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/old-tdriver.cpp b/dbcon/ddlpackage/old-tdriver.cpp index d98afbc79..2ad2d2c9f 100644 --- a/dbcon/ddlpackage/old-tdriver.cpp +++ b/dbcon/ddlpackage/old-tdriver.cpp @@ -41,7 +41,6 @@ #include "dropobjectddlpackage.h" #include "messagequeue.h" - using namespace std; using namespace ddlpackage; using namespace messageqcpp; @@ -52,679 +51,636 @@ std::string itoa(const int i); class DDLWriteReadTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DDLWriteReadTest); - CPPUNIT_TEST_SUITE( DDLWriteReadTest ); + CPPUNIT_TEST(test_write_read_create_table_object); + CPPUNIT_TEST(test_write_read_create_index_object); + CPPUNIT_TEST(test_write_read_alter_table_object); + CPPUNIT_TEST(test_write_read_drop_table_object); + CPPUNIT_TEST(test_write_read_drop_index_object); - CPPUNIT_TEST( test_write_read_create_table_object ); - CPPUNIT_TEST( test_write_read_create_index_object ); - CPPUNIT_TEST( test_write_read_alter_table_object ); - CPPUNIT_TEST( test_write_read_drop_table_object ); - CPPUNIT_TEST( test_write_read_drop_index_object ); + CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } - CPPUNIT_TEST_SUITE_END(); + void tearDown() + { + } + // CREATE TABLE + void test_write_read_create_table_object() + { + ByteStream bytestream; -private: + std::string ddl_statement = + "CREATE TABLE calpont.PART(p_partkey int not null, p_a varchar(55), p_b decimal(8,2) unique, p_c int " + "default 1, p_d varchar(25) default 'unknown' check (varchar = 'a' ), foreign key (p_partkey) " + "references Customers(p_partkey)) engine=infinidb;"; + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); -public: - void setUp() - { + CPPUNIT_ASSERT(0 != pDDLStatement); - } + pDDLStatement->populateFromDDLStatement(ddl_statement); - void tearDown() - { + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - } - // CREATE TABLE - void test_write_read_create_table_object() - { - ByteStream bytestream; + CPPUNIT_ASSERT(0 != pDDLPackage); - std::string ddl_statement = "CREATE TABLE calpont.PART(p_partkey int not null, p_a varchar(55), p_b decimal(8,2) unique, p_c int default 1, p_d varchar(25) default 'unknown' check (varchar = 'a' ), foreign key (p_partkey) references Customers(p_partkey)) engine=infinidb;"; + // parse the data + pDDLPackage->Parse(); - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + write_create_table_object(bytestream, pDDLPackage); - CPPUNIT_ASSERT( 0 != pDDLStatement ); + delete pDDLStatement; - pDDLStatement->populateFromDDLStatement( ddl_statement ); + delete pDDLPackage; - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + read_create_table_object(bytestream); + } + void write_create_table_object(ByteStream& bs, CalpontDDLPackage* pDDLPackage) + { + pDDLPackage->Write(bs); + } - CPPUNIT_ASSERT( 0 != pDDLPackage ); + void read_create_table_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - // parse the data - pDDLPackage->Parse(); + CPPUNIT_ASSERT(DDL_CREATE == package_type); - write_create_table_object( bytestream, pDDLPackage ); + CreateObjectDDLPackage* pObject = new CreateObjectDDLPackage(); - delete pDDLStatement; + pObject->Read(bs); - delete pDDLPackage; + delete pObject; + } - read_create_table_object( bytestream ); + // CREATE INDEX + void test_write_read_create_index_object() + { + ByteStream bytestream; - } - void write_create_table_object( ByteStream& bs, CalpontDDLPackage* pDDLPackage ) - { + std::string ddl_statement = "CREATE INDEX calpont.index1 ON calpont.PART(p_partkey,p_a) engine=infinidb;"; - pDDLPackage->Write( bs ); - } + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - void read_create_table_object( ByteStream& bs ) - { + CPPUNIT_ASSERT(0 != pDDLStatement); - ByteStream::byte package_type; - bs >> package_type; + pDDLStatement->populateFromDDLStatement(ddl_statement); - CPPUNIT_ASSERT( DDL_CREATE == package_type ); + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - CreateObjectDDLPackage* pObject = new CreateObjectDDLPackage(); + CPPUNIT_ASSERT(0 != pDDLPackage); - pObject->Read( bs ); + // parse the data + pDDLPackage->Parse(); - delete pObject; + write_create_index_object(bytestream, pDDLPackage); - } + delete pDDLStatement; - // CREATE INDEX - void test_write_read_create_index_object() - { - ByteStream bytestream; + delete pDDLPackage; - std::string ddl_statement = "CREATE INDEX calpont.index1 ON calpont.PART(p_partkey,p_a) engine=infinidb;"; + read_create_index_object(bytestream); + } + void write_create_index_object(ByteStream& bs, CalpontDDLPackage* pDDLPackage) + { + pDDLPackage->Write(bs); + } - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + void read_create_index_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - CPPUNIT_ASSERT( 0 != pDDLStatement ); + CPPUNIT_ASSERT(DDL_CREATE == package_type); - pDDLStatement->populateFromDDLStatement( ddl_statement ); + CreateObjectDDLPackage* pObject = new CreateObjectDDLPackage(); - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + pObject->Read(bs); - CPPUNIT_ASSERT( 0 != pDDLPackage ); + delete pObject; + } - // parse the data - pDDLPackage->Parse(); + // ALTER TABLE + void test_write_read_alter_table_object() + { + ByteStream bytestream; - write_create_index_object( bytestream, pDDLPackage ); + std::string ddl_statement = "ALTER TABLE calpont.PART_a ADD id int engine=infinidb;"; - delete pDDLStatement; + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - delete pDDLPackage; + CPPUNIT_ASSERT(0 != pDDLStatement); - read_create_index_object( bytestream ); + pDDLStatement->populateFromDDLStatement(ddl_statement); - } - void write_create_index_object( ByteStream& bs, CalpontDDLPackage* pDDLPackage ) - { + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - pDDLPackage->Write( bs ); - } + CPPUNIT_ASSERT(0 != pDDLPackage); - void read_create_index_object( ByteStream& bs ) - { + // parse the data + pDDLPackage->Parse(); - ByteStream::byte package_type; - bs >> package_type; + write_alter_table_object(bytestream, pDDLPackage); - CPPUNIT_ASSERT( DDL_CREATE == package_type ); + delete pDDLStatement; - CreateObjectDDLPackage* pObject = new CreateObjectDDLPackage(); + delete pDDLPackage; - pObject->Read( bs ); + read_alter_table_object(bytestream); + } + void write_alter_table_object(ByteStream& bs, CalpontDDLPackage* pDDLPackage) + { + pDDLPackage->Write(bs); + } - delete pObject; + void read_alter_table_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - } + CPPUNIT_ASSERT(DDL_ALTER == package_type); - // ALTER TABLE - void test_write_read_alter_table_object() - { - ByteStream bytestream; + AlterObjectDDLPackage* pObject = new AlterObjectDDLPackage(); - std::string ddl_statement = "ALTER TABLE calpont.PART_a ADD id int engine=infinidb;"; + pObject->Read(bs); - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + delete pObject; + } - CPPUNIT_ASSERT( 0 != pDDLStatement ); + // DROP TABLE + void test_write_read_drop_table_object() + { + ByteStream bytestream; - pDDLStatement->populateFromDDLStatement( ddl_statement ); + std::string ddl_statement = "DROP TABLE calpont.PART_a CASCADE CONSTRAINTS engine=infinidb;"; - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - CPPUNIT_ASSERT( 0 != pDDLPackage ); + CPPUNIT_ASSERT(0 != pDDLStatement); - // parse the data - pDDLPackage->Parse(); + pDDLStatement->populateFromDDLStatement(ddl_statement); - write_alter_table_object( bytestream, pDDLPackage ); + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - delete pDDLStatement; + CPPUNIT_ASSERT(0 != pDDLPackage); - delete pDDLPackage; + // parse the data + pDDLPackage->Parse(); - read_alter_table_object( bytestream ); + write_drop_table_object(bytestream, pDDLPackage); - } - void write_alter_table_object( ByteStream& bs, CalpontDDLPackage* pDDLPackage ) - { + delete pDDLStatement; - pDDLPackage->Write( bs ); - } + delete pDDLPackage; - void read_alter_table_object( ByteStream& bs ) - { + read_drop_table_object(bytestream); + } + void write_drop_table_object(ByteStream& bs, CalpontDDLPackage* pDDLPackage) + { + pDDLPackage->Write(bs); + } - ByteStream::byte package_type; - bs >> package_type; + void read_drop_table_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - CPPUNIT_ASSERT( DDL_ALTER == package_type ); + CPPUNIT_ASSERT(DDL_DROP == package_type); - AlterObjectDDLPackage* pObject = new AlterObjectDDLPackage(); + DropObjectDDLPackage* pObject = new DropObjectDDLPackage(); - pObject->Read( bs ); + pObject->Read(bs); - delete pObject; + delete pObject; + } - } + // DROP INDEX + void test_write_read_drop_index_object() + { + ByteStream bytestream; - // DROP TABLE - void test_write_read_drop_table_object() - { - ByteStream bytestream; + std::string ddl_statement = "DROP INDEX calpont.INDEX_1 on calpont.PART engine=infinidb;"; - std::string ddl_statement = "DROP TABLE calpont.PART_a CASCADE CONSTRAINTS engine=infinidb;"; + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + CPPUNIT_ASSERT(0 != pDDLStatement); - CPPUNIT_ASSERT( 0 != pDDLStatement ); + pDDLStatement->populateFromDDLStatement(ddl_statement); - pDDLStatement->populateFromDDLStatement( ddl_statement ); + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + CPPUNIT_ASSERT(0 != pDDLPackage); - CPPUNIT_ASSERT( 0 != pDDLPackage ); + // parse the data + pDDLPackage->Parse(); - // parse the data - pDDLPackage->Parse(); + write_drop_index_object(bytestream, pDDLPackage); - write_drop_table_object( bytestream, pDDLPackage ); + delete pDDLStatement; - delete pDDLStatement; + delete pDDLPackage; - delete pDDLPackage; + read_drop_index_object(bytestream); + } + void write_drop_index_object(ByteStream& bs, CalpontDDLPackage* pDDLPackage) + { + pDDLPackage->Write(bs); + } - read_drop_table_object( bytestream ); + void read_drop_index_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - } - void write_drop_table_object( ByteStream& bs, CalpontDDLPackage* pDDLPackage ) - { + CPPUNIT_ASSERT(DDL_DROP == package_type); - pDDLPackage->Write( bs ); - } + DropObjectDDLPackage* pObject = new DropObjectDDLPackage(); - void read_drop_table_object( ByteStream& bs ) - { - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DDL_DROP == package_type ); - - DropObjectDDLPackage* pObject = new DropObjectDDLPackage(); - - pObject->Read( bs ); - - delete pObject; - - } - - // DROP INDEX - void test_write_read_drop_index_object() - { - ByteStream bytestream; - - std::string ddl_statement = "DROP INDEX calpont.INDEX_1 on calpont.PART engine=infinidb;"; - - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - - CPPUNIT_ASSERT( 0 != pDDLStatement ); - - pDDLStatement->populateFromDDLStatement( ddl_statement ); - - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - - CPPUNIT_ASSERT( 0 != pDDLPackage ); - - // parse the data - pDDLPackage->Parse(); - - write_drop_index_object( bytestream, pDDLPackage ); - - delete pDDLStatement; - - delete pDDLPackage; - - read_drop_index_object( bytestream ); - - } - void write_drop_index_object( ByteStream& bs, CalpontDDLPackage* pDDLPackage ) - { - - pDDLPackage->Write( bs ); - } - - void read_drop_index_object( ByteStream& bs ) - { - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DDL_DROP == package_type ); - - DropObjectDDLPackage* pObject = new DropObjectDDLPackage(); - - pObject->Read( bs ); - - delete pObject; - - } + pObject->Read(bs); + delete pObject; + } }; // PARSE STATEMENT TESTS - class DDLCreateTableParserTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DDLCreateTableParserTest); - CPPUNIT_TEST_SUITE( DDLCreateTableParserTest ); + CPPUNIT_TEST(create_t1); - CPPUNIT_TEST( create_t1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: + void tearDown() + { + } -public: - void setUp() + void create_t1() + { + ifstream fin("sql/examples/create-table.sql"); + + CPPUNIT_ASSERT(fin != NULL); + + // read CREATE TABLE statements from buffer and parse + for (;;) { + string fileBuffer; + char Buffer[64000]; + string::size_type pos_begin; + + fin.getline(Buffer, 64000, ';'); + + fileBuffer = Buffer; + + string::size_type pos = fileBuffer.find("create ", 0); + string::size_type pos1 = fileBuffer.find("CREATE ", 0); + + if (pos == string::npos && pos1 == string::npos) + // end of file + break; + + if (pos < pos1) + pos_begin = pos; + else + pos_begin = pos1; + + std::string DDLStatement = fileBuffer.substr(pos_begin, 64000); + + fileBuffer.append(";"); + + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + + pDDLStatement->populateFromDDLStatement(DDLStatement); + + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + + // parse the data + pDDLPackage->Parse(); + + delete pDDLStatement; + + delete pDDLPackage; } - void tearDown() - { - } - - void create_t1() - { - ifstream fin("sql/examples/create-table.sql"); - - CPPUNIT_ASSERT( fin != NULL ); - - // read CREATE TABLE statements from buffer and parse - for (;;) - { - string fileBuffer; - char Buffer[64000]; - string::size_type pos_begin; - - fin.getline (Buffer, 64000, ';'); - - fileBuffer = Buffer; - - string::size_type pos = fileBuffer.find ("create ", 0); - string::size_type pos1 = fileBuffer.find ("CREATE ", 0); - - if (pos == string::npos && pos1 == string::npos ) - // end of file - break; - - if (pos < pos1) - pos_begin = pos; - else - pos_begin = pos1; - - std::string DDLStatement = fileBuffer.substr (pos_begin, 64000); - - fileBuffer.append(";"); - - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - - pDDLStatement->populateFromDDLStatement( DDLStatement ); - - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - - // parse the data - pDDLPackage->Parse(); - - delete pDDLStatement; - - delete pDDLPackage; - } - - fin.close(); - } + fin.close(); + } }; class DDLCreateIndexParserTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DDLCreateIndexParserTest); - CPPUNIT_TEST_SUITE( DDLCreateIndexParserTest ); + CPPUNIT_TEST(createIndex_t1); - CPPUNIT_TEST( createIndex_t1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: + void tearDown() + { + } -public: - void setUp() + void createIndex_t1() + { + ifstream fin("sql/examples/create-index.sql"); + + CPPUNIT_ASSERT(fin != NULL); + + // read CREATE INDEX statements from buffer and parse + for (;;) { + string fileBuffer; + char Buffer[64000]; + string::size_type pos_begin; + + fin.getline(Buffer, 64000, ';'); + + fileBuffer = Buffer; + + string::size_type pos = fileBuffer.find("create ", 0); + string::size_type pos1 = fileBuffer.find("CREATE ", 0); + + if (pos == string::npos && pos1 == string::npos) + // end of file + break; + + if (pos < pos1) + pos_begin = pos; + else + pos_begin = pos1; + + std::string DDLStatement = fileBuffer.substr(pos_begin, 64000); + + fileBuffer.append(";"); + + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + + pDDLStatement->populateFromDDLStatement(DDLStatement); + + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + + // parse the data + pDDLPackage->Parse(); + + delete pDDLStatement; + + delete pDDLPackage; } - void tearDown() - { - } - - void createIndex_t1() - { - ifstream fin("sql/examples/create-index.sql"); - - CPPUNIT_ASSERT( fin != NULL ); - - // read CREATE INDEX statements from buffer and parse - for (;;) - { - string fileBuffer; - char Buffer[64000]; - string::size_type pos_begin; - - fin.getline (Buffer, 64000, ';'); - - fileBuffer = Buffer; - - string::size_type pos = fileBuffer.find ("create ", 0); - string::size_type pos1 = fileBuffer.find ("CREATE ", 0); - - if (pos == string::npos && pos1 == string::npos ) - // end of file - break; - - if (pos < pos1) - pos_begin = pos; - else - pos_begin = pos1; - - std::string DDLStatement = fileBuffer.substr (pos_begin, 64000); - - fileBuffer.append(";"); - - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - - pDDLStatement->populateFromDDLStatement( DDLStatement ); - - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - - // parse the data - pDDLPackage->Parse(); - - delete pDDLStatement; - - delete pDDLPackage; - } - - fin.close(); - } - + fin.close(); + } }; - class DDLAlterTableParserTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DDLAlterTableParserTest); - CPPUNIT_TEST_SUITE( DDLAlterTableParserTest ); + CPPUNIT_TEST(alter_t1); - CPPUNIT_TEST( alter_t1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: + void tearDown() + { + } -public: - void setUp() + void alter_t1() + { + ifstream fin("sql/examples/alter-table.sql"); + + CPPUNIT_ASSERT(fin != NULL); + + // read ALTER TABLE statements from buffer and parse + for (;;) { + string fileBuffer; + char Buffer[64000]; + string::size_type pos_begin; + + fin.getline(Buffer, 64000, ';'); + + fileBuffer = Buffer; + + string::size_type pos = fileBuffer.find("alter ", 0); + string::size_type pos1 = fileBuffer.find("ALTER ", 0); + + if (pos == string::npos && pos1 == string::npos) + // end of file + break; + + if (pos < pos1) + pos_begin = pos; + else + pos_begin = pos1; + + std::string DDLStatement = fileBuffer.substr(pos_begin, 64000); + + fileBuffer.append(";"); + + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + + pDDLStatement->populateFromDDLStatement(DDLStatement); + + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + + // parse the data + pDDLPackage->Parse(); + + delete pDDLStatement; + + delete pDDLPackage; } - void tearDown() - { - } - - void alter_t1() - { - ifstream fin("sql/examples/alter-table.sql"); - - CPPUNIT_ASSERT( fin != NULL ); - - // read ALTER TABLE statements from buffer and parse - for (;;) - { - string fileBuffer; - char Buffer[64000]; - string::size_type pos_begin; - - fin.getline (Buffer, 64000, ';'); - - fileBuffer = Buffer; - - string::size_type pos = fileBuffer.find ("alter ", 0); - string::size_type pos1 = fileBuffer.find ("ALTER ", 0); - - if (pos == string::npos && pos1 == string::npos ) - // end of file - break; - - if (pos < pos1) - pos_begin = pos; - else - pos_begin = pos1; - - std::string DDLStatement = fileBuffer.substr (pos_begin, 64000); - - fileBuffer.append(";"); - - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - - pDDLStatement->populateFromDDLStatement( DDLStatement ); - - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - - // parse the data - pDDLPackage->Parse(); - - delete pDDLStatement; - - delete pDDLPackage; - } - - fin.close(); - } + fin.close(); + } }; - class DDLDropTableParserTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DDLDropTableParserTest); - CPPUNIT_TEST_SUITE( DDLDropTableParserTest ); + CPPUNIT_TEST(drop_t1); - CPPUNIT_TEST( drop_t1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: + void tearDown() + { + } -public: - void setUp() + void drop_t1() + { + ifstream fin("sql/examples/drop-table.sql"); + + CPPUNIT_ASSERT(fin != NULL); + + // read DROP TABLE statements from buffer and parse + for (;;) { + string fileBuffer; + char Buffer[64000]; + string::size_type pos_begin; + fin.getline(Buffer, 64000, ';'); + + fileBuffer = Buffer; + + string::size_type pos = fileBuffer.find("drop ", 0); + string::size_type pos1 = fileBuffer.find("DROP ", 0); + + if (pos == string::npos && pos1 == string::npos) + // end of file + break; + + if (pos < pos1) + pos_begin = pos; + else + pos_begin = pos1; + + std::string DDLStatement = fileBuffer.substr(pos_begin, 64000); + + fileBuffer.append(";"); + + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + + pDDLStatement->populateFromDDLStatement(DDLStatement); + + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + + // parse the data + pDDLPackage->Parse(); + + delete pDDLStatement; + + delete pDDLPackage; } - void tearDown() - { - } - - void drop_t1() - { - ifstream fin("sql/examples/drop-table.sql"); - - CPPUNIT_ASSERT( fin != NULL ); - - // read DROP TABLE statements from buffer and parse - for (;;) - { - string fileBuffer; - char Buffer[64000]; - string::size_type pos_begin; - - fin.getline (Buffer, 64000, ';'); - - fileBuffer = Buffer; - - string::size_type pos = fileBuffer.find ("drop ", 0); - string::size_type pos1 = fileBuffer.find ("DROP ", 0); - - if (pos == string::npos && pos1 == string::npos ) - // end of file - break; - - if (pos < pos1) - pos_begin = pos; - else - pos_begin = pos1; - - std::string DDLStatement = fileBuffer.substr (pos_begin, 64000); - - fileBuffer.append(";"); - - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - - pDDLStatement->populateFromDDLStatement( DDLStatement ); - - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - - // parse the data - pDDLPackage->Parse(); - - delete pDDLStatement; - - delete pDDLPackage; - } - - fin.close(); - } - + fin.close(); + } }; class DDLDropIndexParserTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DDLDropIndexParserTest); - CPPUNIT_TEST_SUITE( DDLDropIndexParserTest ); + CPPUNIT_TEST(dropIndex_t1); - CPPUNIT_TEST( dropIndex_t1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + std::string fDDLStatement; -private: - std::string fDDLStatement; + public: + void setUp() + { + } -public: - void setUp() + void tearDown() + { + } + + void dropIndex_t1() + { + ifstream fin("sql/examples/drop-index.sql"); + + CPPUNIT_ASSERT(fin != NULL); + + // read DROP INDEX statements from buffer and parse + for (;;) { + string fileBuffer; + char Buffer[64000]; + string::size_type pos_begin; + + fin.getline(Buffer, 64000, ';'); + + fileBuffer = Buffer; + + string::size_type pos = fileBuffer.find("drop ", 0); + string::size_type pos1 = fileBuffer.find("DROP ", 0); + + if (pos == string::npos && pos1 == string::npos) + // end of file + break; + + if (pos < pos1) + pos_begin = pos; + else + pos_begin = pos1; + + std::string DDLStatement = fileBuffer.substr(pos_begin, 64000); + + fileBuffer.append(";"); + + MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); + + pDDLStatement->populateFromDDLStatement(DDLStatement); + + CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); + + // parse the data + pDDLPackage->Parse(); + + delete pDDLStatement; + + delete pDDLPackage; } - void tearDown() - { - } - - void dropIndex_t1() - { - ifstream fin("sql/examples/drop-index.sql"); - - CPPUNIT_ASSERT( fin != NULL ); - - // read DROP INDEX statements from buffer and parse - for (;;) - { - string fileBuffer; - char Buffer[64000]; - string::size_type pos_begin; - - fin.getline (Buffer, 64000, ';'); - - fileBuffer = Buffer; - - string::size_type pos = fileBuffer.find ("drop ", 0); - string::size_type pos1 = fileBuffer.find ("DROP ", 0); - - if (pos == string::npos && pos1 == string::npos ) - // end of file - break; - - if (pos < pos1) - pos_begin = pos; - else - pos_begin = pos1; - - std::string DDLStatement = fileBuffer.substr (pos_begin, 64000); - - fileBuffer.append(";"); - - MySQLDDLStatement* pDDLStatement = new MySQLDDLStatement(); - - pDDLStatement->populateFromDDLStatement( DDLStatement ); - - CalpontDDLPackage* pDDLPackage = CalpontDDLFactory::makeCalpontDDLPackage(*pDDLStatement); - - // parse the data - pDDLPackage->Parse(); - - delete pDDLStatement; - - delete pDDLPackage; - } - - fin.close(); - } - + fin.close(); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( DDLDropIndexParserTest ); -CPPUNIT_TEST_SUITE_REGISTRATION( DDLDropTableParserTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DDLDropIndexParserTest); +CPPUNIT_TEST_SUITE_REGISTRATION(DDLDropTableParserTest); // DOESN'T NOT PARSE CPPUNIT_TEST_SUITE_REGISTRATION( DDLAlterTableParserTest ); -CPPUNIT_TEST_SUITE_REGISTRATION( DDLCreateIndexParserTest ); -CPPUNIT_TEST_SUITE_REGISTRATION( DDLCreateTableParserTest ); -CPPUNIT_TEST_SUITE_REGISTRATION( DDLWriteReadTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DDLCreateIndexParserTest); +CPPUNIT_TEST_SUITE_REGISTRATION(DDLCreateTableParserTest); +CPPUNIT_TEST_SUITE_REGISTRATION(DDLWriteReadTest); #include #include int main(int argc, char* argv[]) { + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - - return (wasSuccessful ? 0 : 1); - + return (wasSuccessful ? 0 : 1); } string itoa(const int i) { - stringstream ss; - ss << i; - return ss.str(); + stringstream ss; + ss << i; + return ss.str(); } - diff --git a/dbcon/ddlpackage/restorepartition.cpp b/dbcon/ddlpackage/restorepartition.cpp index 9f101707f..6d8c04306 100644 --- a/dbcon/ddlpackage/restorepartition.cpp +++ b/dbcon/ddlpackage/restorepartition.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: markpartition.cpp 6566 2010-04-27 18:02:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: markpartition.cpp 6566 2010-04-27 18:02:51Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -29,23 +29,21 @@ using namespace std; namespace ddlpackage { - -RestorePartitionStatement::RestorePartitionStatement(QualifiedName* qualifiedName) : - fTableName(qualifiedName) +RestorePartitionStatement::RestorePartitionStatement(QualifiedName* qualifiedName) : fTableName(qualifiedName) { } ostream& RestorePartitionStatement::put(ostream& os) const { - os << "Mark partition out of service: " << *fTableName; - os << " partitions: "; - set::const_iterator it; + os << "Mark partition out of service: " << *fTableName; + os << " partitions: "; + set::const_iterator it; - for (it = fPartitions.begin(); it != fPartitions.end(); ++it) - os << (*it) << " "; + for (it = fPartitions.begin(); it != fPartitions.end(); ++it) + os << (*it) << " "; - os << endl; - return os; + os << endl; + return os; } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/serialize.cpp b/dbcon/ddlpackage/serialize.cpp index f81cef077..c719e8f96 100644 --- a/dbcon/ddlpackage/serialize.cpp +++ b/dbcon/ddlpackage/serialize.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: serialize.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: serialize.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -35,32 +35,31 @@ using namespace messageqcpp; using namespace std; using namespace ddlpackage; -template +template void write_vec(vector& v, ByteStream& bs) { - bs << (quadbyte) v.size(); - typename vector::const_iterator itr; + bs << (quadbyte)v.size(); + typename vector::const_iterator itr; - for (itr = v.begin(); itr != v.end(); ++itr) - (*itr)->serialize(bs); + for (itr = v.begin(); itr != v.end(); ++itr) + (*itr)->serialize(bs); } -template +template void read_vec(vector& v, ByteStream& bs) { - T* x; - quadbyte count; - bs >> count; + T* x; + quadbyte count; + bs >> count; - while (count--) - { - x = new T; - x->unserialize(bs); - v.push_back(x); - } + while (count--) + { + x = new T; + x->unserialize(bs); + v.push_back(x); + } } - /////////////////////////////////////// /// CreateTableStatement Serialization /////////////////////////////////////// @@ -68,43 +67,41 @@ void read_vec(vector& v, ByteStream& bs) /** @brief Construct from Bytestream */ int CreateTableStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fTableDef = new TableDef(); - fTableDef->unserialize( bytestream ); - bytestream >> fSessionID; - bytestream >> fSql; - bytestream >> fOwner; - bytestream >> fTableWithAutoi; - return ret; + fTableDef = new TableDef(); + fTableDef->unserialize(bytestream); + bytestream >> fSessionID; + bytestream >> fSql; + bytestream >> fOwner; + bytestream >> fTableWithAutoi; + return ret; } /** @brief Serialize to ByteStream */ int CreateTableStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_CREATE_TABLE_STATEMENT; + bytestream << (quadbyte)DDL_CREATE_TABLE_STATEMENT; - // write table def - fTableDef->serialize( bytestream ); + // write table def + fTableDef->serialize(bytestream); - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write the original ddl statement. - bytestream << fSql; + // write the original ddl statement. + bytestream << fSql; - // write the owner (default schema). - bytestream << fOwner; + // write the owner (default schema). + bytestream << fOwner; - bytestream << fTableWithAutoi; + bytestream << fTableWithAutoi; - return ret; + return ret; } - - /////////////////////////////////////// /// AlterTableStatement Serialization /////////////////////////////////////// @@ -112,144 +109,140 @@ int CreateTableStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AlterTableStatement::unserialize(ByteStream& bytestream) { - int ret = 1; - AlterTableAction* ata; - messageqcpp::ByteStream::quadbyte type; + int ret = 1; + AlterTableAction* ata; + messageqcpp::ByteStream::quadbyte type; - fTableName = new QualifiedName(); + fTableName = new QualifiedName(); - // read table name - fTableName->unserialize( bytestream ); + // read table name + fTableName->unserialize(bytestream); - bytestream >> fTimeZone; + bytestream >> fTimeZone; - // read alter action list - quadbyte action_count; - bytestream >> action_count; + // read alter action list + quadbyte action_count; + bytestream >> action_count; - for ( unsigned int i = 0; i < action_count; i++ ) + for (unsigned int i = 0; i < action_count; i++) + { + // read action type + bytestream >> type; + + switch (type) { - // read action type - bytestream >> type; + case DDL_ATA_ADD_COLUMN: + ata = new AtaAddColumn(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - switch (type) - { - case DDL_ATA_ADD_COLUMN: - ata = new AtaAddColumn(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_ADD_COLUMNS: + ata = new AtaAddColumns(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_ADD_COLUMNS: - ata = new AtaAddColumns(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_DROP_COLUMN: + ata = new AtaDropColumn(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_DROP_COLUMN: - ata = new AtaDropColumn(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_DROP_COLUMNS: + ata = new AtaDropColumns(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_DROP_COLUMNS: - ata = new AtaDropColumns(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_ADD_TABLE_CONSTRAINT: + ata = new AtaAddTableConstraint(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_ADD_TABLE_CONSTRAINT: - ata = new AtaAddTableConstraint(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_SET_COLUMN_DEFAULT: + ata = new AtaSetColumnDefault(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_SET_COLUMN_DEFAULT: - ata = new AtaSetColumnDefault(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_DROP_COLUMN_DEFAULT: + ata = new AtaDropColumnDefault(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_DROP_COLUMN_DEFAULT: - ata = new AtaDropColumnDefault(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_DROP_TABLE_CONSTRAINT: + ata = new AtaDropTableConstraint(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_DROP_TABLE_CONSTRAINT: - ata = new AtaDropTableConstraint(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_RENAME_TABLE: + ata = new AtaRenameTable(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_RENAME_TABLE: - ata = new AtaRenameTable(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_RENAME_COLUMN: + ata = new AtaRenameColumn(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_RENAME_COLUMN: - ata = new AtaRenameColumn(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_MODIFY_COLUMN_TYPE: + ata = new AtaModifyColumnType(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_MODIFY_COLUMN_TYPE: - ata = new AtaModifyColumnType(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; + case DDL_ATA_TABLE_COMMENT: + ata = new AtaTableComment(); + ata->unserialize(bytestream); + fActions.push_back(ata); + break; - case DDL_ATA_TABLE_COMMENT: - ata = new AtaTableComment(); - ata->unserialize(bytestream); - fActions.push_back( ata ); - break; - - default: - throw ("Bad typecode for AlterTableAction"); - break; - } - - bytestream >> fSessionID; - bytestream >> fSql; - bytestream >> fOwner; - bytestream >> fTableWithAutoi; + default: throw("Bad typecode for AlterTableAction"); break; } - return ret; + bytestream >> fSessionID; + bytestream >> fSql; + bytestream >> fOwner; + bytestream >> fTableWithAutoi; + } + + return ret; } /** @brief Serialize to ByteStream */ int AlterTableStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ALTER_TABLE_STATEMENT; + bytestream << (quadbyte)DDL_ALTER_TABLE_STATEMENT; - // write table name - fTableName->serialize( bytestream ); + // write table name + fTableName->serialize(bytestream); - bytestream << fTimeZone; + bytestream << fTimeZone; - write_vec(fActions, bytestream); + write_vec(fActions, bytestream); - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write original ddl statement. - bytestream << fSql; + // write original ddl statement. + bytestream << fSql; - // write the owner (default schema). - bytestream << fOwner; + // write the owner (default schema). + bytestream << fOwner; - bytestream << fTableWithAutoi; + bytestream << fTableWithAutoi; - return ret; + return ret; } - - /////////////////////////////////////// /// CreateIndexStatement Serialization /////////////////////////////////////// @@ -257,81 +250,75 @@ int AlterTableStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int CreateIndexStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // read the index and schema name - fIndexName = new QualifiedName(); - fIndexName->unserialize( bytestream ); + // read the index and schema name + fIndexName = new QualifiedName(); + fIndexName->unserialize(bytestream); - // read the table and schema name - fTableName = new QualifiedName(); - fTableName->unserialize( bytestream ); + // read the table and schema name + fTableName = new QualifiedName(); + fTableName->unserialize(bytestream); + quadbyte column_count; + bytestream >> column_count; - quadbyte column_count; - bytestream >> column_count; + std::string columnname; + for (unsigned int i = 0; i < column_count; i++) + { + bytestream >> columnname; + fColumnNames.push_back(columnname); + } - std::string columnname; + // read unique flag + quadbyte unique; + bytestream >> unique; + fUnique = (unique != 0); + bytestream >> fSessionID; + bytestream >> fSql; + bytestream >> fOwner; - for ( unsigned int i = 0; i < column_count; i++ ) - { - bytestream >> columnname; - fColumnNames.push_back( columnname ); - } - - // read unique flag - quadbyte unique; - bytestream >> unique; - fUnique = (unique != 0); - bytestream >> fSessionID; - bytestream >> fSql; - bytestream >> fOwner; - - return ret; + return ret; } /** @brief Serialize to ByteStream */ int CreateIndexStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_CREATE_INDEX; + bytestream << (quadbyte)DDL_CREATE_INDEX; - // write the index and schema name - fIndexName->serialize( bytestream ); + // write the index and schema name + fIndexName->serialize(bytestream); - // write the table and schema name - fTableName->serialize( bytestream ); + // write the table and schema name + fTableName->serialize(bytestream); - // write column name list - bytestream << (quadbyte) fColumnNames.size(); - ColumnNameList::const_iterator itr; + // write column name list + bytestream << (quadbyte)fColumnNames.size(); + ColumnNameList::const_iterator itr; - for (itr = fColumnNames.begin(); - itr != fColumnNames.end(); - ++itr) - { - bytestream << *itr; - } + for (itr = fColumnNames.begin(); itr != fColumnNames.end(); ++itr) + { + bytestream << *itr; + } - // write Unique flag - bytestream << (quadbyte) fUnique; + // write Unique flag + bytestream << (quadbyte)fUnique; - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write original ddl - bytestream << fSql; + // write original ddl + bytestream << fSql; - // write the owner (default schema). - bytestream << fOwner; + // write the owner (default schema). + bytestream << fOwner; - return ret; + return ret; } - - /////////////////////////////////////// /// DropIndexStatement Serialization /////////////////////////////////////// @@ -339,47 +326,46 @@ int CreateIndexStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int DropIndexStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fIndexName = new QualifiedName(); + fIndexName = new QualifiedName(); - // read the table and schema name - fIndexName->unserialize(bytestream); + // read the table and schema name + fIndexName->unserialize(bytestream); - // read the sessionID - bytestream >> fSessionID; + // read the sessionID + bytestream >> fSessionID; - // read the original ddlACK - bytestream >> fSql; + // read the original ddlACK + bytestream >> fSql; - // read the owner (default schema) - bytestream >> fOwner; - return ret; + // read the owner (default schema) + bytestream >> fOwner; + return ret; } /** @brief Serialize to ByteStream */ int DropIndexStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_DROP_INDEX_STATEMENT; + bytestream << (quadbyte)DDL_DROP_INDEX_STATEMENT; - // write the table and schema name - fIndexName->serialize( bytestream ); + // write the table and schema name + fIndexName->serialize(bytestream); - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write ddl statement - bytestream << fSql; + // write ddl statement + bytestream << fSql; - // write the owner (default schema) - bytestream << fOwner; + // write the owner (default schema) + bytestream << fOwner; - return ret; + return ret; } - /////////////////////////////////////// /// DropTableStatement Serialization /////////////////////////////////////// @@ -387,57 +373,57 @@ int DropIndexStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int DropTableStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - //cout << endl << "DropTableStatement unserialize testing started" << endl; + // cout << endl << "DropTableStatement unserialize testing started" << endl; - fTableName = new QualifiedName(); - fTableName->unserialize(bytestream); + fTableName = new QualifiedName(); + fTableName->unserialize(bytestream); - messageqcpp::ByteStream::quadbyte cascade; + messageqcpp::ByteStream::quadbyte cascade; - // read cascade flag - bytestream >> cascade; + // read cascade flag + bytestream >> cascade; - // read the sessionID - bytestream >> fSessionID; + // read the sessionID + bytestream >> fSessionID; - // read the original ddl - bytestream >> fSql; + // read the original ddl + bytestream >> fSql; - // read the owner (default schema) - bytestream >> fOwner; + // read the owner (default schema) + bytestream >> fOwner; - fCascade = (cascade != 0); + fCascade = (cascade != 0); - return ret; + return ret; } /** @brief Serialize to ByteStream */ int DropTableStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - //cout << "DropTableStatement serialize testing started" << endl; + // cout << "DropTableStatement serialize testing started" << endl; - bytestream << (quadbyte) DDL_DROP_TABLE_STATEMENT; + bytestream << (quadbyte)DDL_DROP_TABLE_STATEMENT; - // write the table and schema name - fTableName->serialize( bytestream ); + // write the table and schema name + fTableName->serialize(bytestream); - // read cascade flag - bytestream << (quadbyte) fCascade; + // read cascade flag + bytestream << (quadbyte)fCascade; - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write original ddl - bytestream << fSql; + // write original ddl + bytestream << fSql; - // write the owner (default schema) - bytestream << fOwner; + // write the owner (default schema) + bytestream << fOwner; - return ret; + return ret; } /////////////////////////////////////// @@ -447,50 +433,49 @@ int DropTableStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TruncTableStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - //cout << endl << "TruncTableStatement unserialize testing started" << endl; + // cout << endl << "TruncTableStatement unserialize testing started" << endl; - fTableName = new QualifiedName(); - fTableName->unserialize(bytestream); + fTableName = new QualifiedName(); + fTableName->unserialize(bytestream); - // read the sessionID - bytestream >> fSessionID; + // read the sessionID + bytestream >> fSessionID; - // read the original ddl - bytestream >> fSql; + // read the original ddl + bytestream >> fSql; - // read the owner (default schema) - bytestream >> fOwner; + // read the owner (default schema) + bytestream >> fOwner; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TruncTableStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - //cout << "TruncTableStatement serialize testing started" << endl; + // cout << "TruncTableStatement serialize testing started" << endl; - bytestream << (quadbyte) DDL_TRUNC_TABLE_STATEMENT; + bytestream << (quadbyte)DDL_TRUNC_TABLE_STATEMENT; - // write the table and schema name - fTableName->serialize( bytestream ); + // write the table and schema name + fTableName->serialize(bytestream); - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write original ddl - bytestream << fSql; + // write original ddl + bytestream << fSql; - // write the owner (default schema) - bytestream << fOwner; + // write the owner (default schema) + bytestream << fOwner; - return ret; + return ret; } - /////////////////////////////////////// /// MarkPartitionStatement Serialization /////////////////////////////////////// @@ -498,61 +483,60 @@ int TruncTableStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int MarkPartitionStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fTableName = new QualifiedName(); - fTableName->unserialize(bytestream); + fTableName = new QualifiedName(); + fTableName->unserialize(bytestream); - // read the sessionID - bytestream >> fSessionID; + // read the sessionID + bytestream >> fSessionID; - // read the original ddl - bytestream >> fSql; + // read the original ddl + bytestream >> fSql; - // read the owner (default schema) - bytestream >> fOwner; + // read the owner (default schema) + bytestream >> fOwner; - fPartitions.clear(); - uint32_t size = 0; - bytestream >> size; - BRM::LogicalPartition part; + fPartitions.clear(); + uint32_t size = 0; + bytestream >> size; + BRM::LogicalPartition part; - for (uint32_t i = 0; i < size; i++) - { - part.unserialize(bytestream); - fPartitions.insert(part); - } + for (uint32_t i = 0; i < size; i++) + { + part.unserialize(bytestream); + fPartitions.insert(part); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int MarkPartitionStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; + bytestream << (quadbyte)DDL_MARK_PARTITION_STATEMENT; - bytestream << (quadbyte) DDL_MARK_PARTITION_STATEMENT; + // write the table and schema name + fTableName->serialize(bytestream); - // write the table and schema name - fTableName->serialize( bytestream ); + // write sessionid + bytestream << fSessionID; - // write sessionid - bytestream << fSessionID; + // write original ddl + bytestream << fSql; - // write original ddl - bytestream << fSql; + // write the owner (default schema) + bytestream << fOwner; - // write the owner (default schema) - bytestream << fOwner; + bytestream << (uint32_t)fPartitions.size(); + set::iterator it; - bytestream << (uint32_t)fPartitions.size(); - set::iterator it; + for (it = fPartitions.begin(); it != fPartitions.end(); ++it) + (*it).serialize(bytestream); - for (it = fPartitions.begin(); it != fPartitions.end(); ++it) - (*it).serialize(bytestream); - - return ret; + return ret; } /////////////////////////////////////// @@ -562,58 +546,58 @@ int MarkPartitionStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int DropPartitionStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fTableName = new QualifiedName(); - fTableName->unserialize(bytestream); + fTableName = new QualifiedName(); + fTableName->unserialize(bytestream); - // read the sessionID - bytestream >> fSessionID; + // read the sessionID + bytestream >> fSessionID; - // read the original ddl - bytestream >> fSql; + // read the original ddl + bytestream >> fSql; - // read the owner (default schema) - bytestream >> fOwner; + // read the owner (default schema) + bytestream >> fOwner; - uint32_t size = 0; - bytestream >> size; - BRM::LogicalPartition part; + uint32_t size = 0; + bytestream >> size; + BRM::LogicalPartition part; - for (uint32_t i = 0; i < size; i++) - { - part.unserialize(bytestream); - fPartitions.insert(part); - } + for (uint32_t i = 0; i < size; i++) + { + part.unserialize(bytestream); + fPartitions.insert(part); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int DropPartitionStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_DROP_PARTITION_STATEMENT; + bytestream << (quadbyte)DDL_DROP_PARTITION_STATEMENT; - // write the table and schema name - fTableName->serialize( bytestream ); + // write the table and schema name + fTableName->serialize(bytestream); - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write original ddl - bytestream << fSql; + // write original ddl + bytestream << fSql; - // write the owner (default schema) - bytestream << fOwner; - bytestream << (uint32_t)fPartitions.size(); - set::iterator it; + // write the owner (default schema) + bytestream << fOwner; + bytestream << (uint32_t)fPartitions.size(); + set::iterator it; - for (it = fPartitions.begin(); it != fPartitions.end(); ++it) - (*it).serialize(bytestream); + for (it = fPartitions.begin(); it != fPartitions.end(); ++it) + (*it).serialize(bytestream); - return ret; + return ret; } /////////////////////////////////////// @@ -623,59 +607,59 @@ int DropPartitionStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int RestorePartitionStatement::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fTableName = new QualifiedName(); - fTableName->unserialize(bytestream); + fTableName = new QualifiedName(); + fTableName->unserialize(bytestream); - // read the sessionID - bytestream >> fSessionID; + // read the sessionID + bytestream >> fSessionID; - // read the original ddl - bytestream >> fSql; + // read the original ddl + bytestream >> fSql; - // read the owner (default schema) - bytestream >> fOwner; + // read the owner (default schema) + bytestream >> fOwner; - uint32_t size = 0; - bytestream >> size; - BRM::LogicalPartition part; + uint32_t size = 0; + bytestream >> size; + BRM::LogicalPartition part; - for (uint32_t i = 0; i < size; i++) - { - part.unserialize(bytestream); - fPartitions.insert(part); - } + for (uint32_t i = 0; i < size; i++) + { + part.unserialize(bytestream); + fPartitions.insert(part); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int RestorePartitionStatement::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_RESTORE_PARTITION_STATEMENT; + bytestream << (quadbyte)DDL_RESTORE_PARTITION_STATEMENT; - // write the table and schema name - fTableName->serialize( bytestream ); + // write the table and schema name + fTableName->serialize(bytestream); - // write sessionid - bytestream << fSessionID; + // write sessionid + bytestream << fSessionID; - // write original ddl - bytestream << fSql; + // write original ddl + bytestream << fSql; - // write the owner (default schema) - bytestream << fOwner; + // write the owner (default schema) + bytestream << fOwner; - bytestream << (uint32_t)fPartitions.size(); - set::iterator it; + bytestream << (uint32_t)fPartitions.size(); + set::iterator it; - for (it = fPartitions.begin(); it != fPartitions.end(); ++it) - (*it).serialize(bytestream); + for (it = fPartitions.begin(); it != fPartitions.end(); ++it) + (*it).serialize(bytestream); - return ret; + return ret; } /////////////////////////////////////// @@ -685,32 +669,30 @@ int RestorePartitionStatement::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaAddColumn::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fColumnDef = new ColumnDef(); + fColumnDef = new ColumnDef(); - // read column - fColumnDef->unserialize( bytestream ); + // read column + fColumnDef->unserialize(bytestream); - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaAddColumn::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write type code - bytestream << (quadbyte) DDL_ATA_ADD_COLUMN; + // write type code + bytestream << (quadbyte)DDL_ATA_ADD_COLUMN; - // write column - fColumnDef->serialize( bytestream ); + // write column + fColumnDef->serialize(bytestream); - return ret; + return ret; } - - /////////////////////////////////////// /// AtaAddColumns Serialization /////////////////////////////////////// @@ -718,24 +700,24 @@ int AtaAddColumn::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaAddColumns::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - read_vec(fColumns, bytestream); + read_vec(fColumns, bytestream); - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaAddColumns::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write type code - bytestream << (quadbyte) DDL_ATA_ADD_COLUMNS; + // write type code + bytestream << (quadbyte)DDL_ATA_ADD_COLUMNS; - write_vec(fColumns, bytestream); + write_vec(fColumns, bytestream); - return ret; + return ret; } /////////////////////////////////////// @@ -745,43 +727,41 @@ int AtaAddColumns::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaDropColumns::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - quadbyte count; - bytestream >> count; - string colName; + quadbyte count; + bytestream >> count; + string colName; - while (count--) - { - bytestream >> colName; - fColumns.push_back(colName); - } + while (count--) + { + bytestream >> colName; + fColumns.push_back(colName); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaDropColumns::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write type code - bytestream << (quadbyte) DDL_ATA_DROP_COLUMNS; + // write type code + bytestream << (quadbyte)DDL_ATA_DROP_COLUMNS; - bytestream << (quadbyte) fColumns.size(); + bytestream << (quadbyte)fColumns.size(); - ColumnNameList::const_iterator itr; + ColumnNameList::const_iterator itr; - for (itr = fColumns.begin(); itr != fColumns.end(); itr++) - { - bytestream << *itr; - } + for (itr = fColumns.begin(); itr != fColumns.end(); itr++) + { + bytestream << *itr; + } - return ret; + return ret; } - - /////////////////////////////////////// /// AtaAddTableConstraint Serialization /////////////////////////////////////// @@ -789,51 +769,50 @@ int AtaDropColumns::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaAddTableConstraint::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - quadbyte ctype; + quadbyte ctype; - bytestream >> ctype; + bytestream >> ctype; - switch (ctype) - { - case DDL_TABLE_UNIQUE_CONSTRAINT_DEF: - fTableConstraint = new TableUniqueConstraintDef; - fTableConstraint->unserialize(bytestream); - break; + switch (ctype) + { + case DDL_TABLE_UNIQUE_CONSTRAINT_DEF: + fTableConstraint = new TableUniqueConstraintDef; + fTableConstraint->unserialize(bytestream); + break; - case DDL_TABLE_PRIMARY_CONSTRAINT_DEF: - fTableConstraint = new TablePrimaryKeyConstraintDef; - fTableConstraint->unserialize(bytestream); - break; + case DDL_TABLE_PRIMARY_CONSTRAINT_DEF: + fTableConstraint = new TablePrimaryKeyConstraintDef; + fTableConstraint->unserialize(bytestream); + break; - case DDL_TABLE_CHECK_CONSTRAINT_DEF: - fTableConstraint = new TableCheckConstraintDef; - fTableConstraint->unserialize(bytestream); - break; + case DDL_TABLE_CHECK_CONSTRAINT_DEF: + fTableConstraint = new TableCheckConstraintDef; + fTableConstraint->unserialize(bytestream); + break; - case DDL_TABLE_REFERENCES_CONSTRAINT_DEF: - fTableConstraint = new TableReferencesConstraintDef; - fTableConstraint->unserialize(bytestream); - break; - } + case DDL_TABLE_REFERENCES_CONSTRAINT_DEF: + fTableConstraint = new TableReferencesConstraintDef; + fTableConstraint->unserialize(bytestream); + break; + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaAddTableConstraint::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ATA_ADD_TABLE_CONSTRAINT; - bytestream << (quadbyte) fTableConstraint->getSerialType(); - fTableConstraint->serialize( bytestream ); + bytestream << (quadbyte)DDL_ATA_ADD_TABLE_CONSTRAINT; + bytestream << (quadbyte)fTableConstraint->getSerialType(); + fTableConstraint->serialize(bytestream); - return ret; + return ret; } - /////////////////////////////////////// /// AtaDropColumn Serialization /////////////////////////////////////// @@ -841,30 +820,29 @@ int AtaAddTableConstraint::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaDropColumn::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream >> fColumnName; - quadbyte action; - bytestream >> action; - fDropBehavior = (DDL_REFERENTIAL_ACTION) action; + bytestream >> fColumnName; + quadbyte action; + bytestream >> action; + fDropBehavior = (DDL_REFERENTIAL_ACTION)action; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaDropColumn::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write type code - bytestream << (quadbyte) DDL_ATA_DROP_COLUMN; - bytestream << fColumnName; - bytestream << (quadbyte) fDropBehavior; + // write type code + bytestream << (quadbyte)DDL_ATA_DROP_COLUMN; + bytestream << fColumnName; + bytestream << (quadbyte)fDropBehavior; - return ret; + return ret; } - /////////////////////////////////////// /// AtaSetColumnDefault Serialization /////////////////////////////////////// @@ -872,29 +850,27 @@ int AtaDropColumn::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaSetColumnDefault::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream >> fColumnName; - fDefaultValue = new ColumnDefaultValue(); - fDefaultValue->unserialize(bytestream); + bytestream >> fColumnName; + fDefaultValue = new ColumnDefaultValue(); + fDefaultValue->unserialize(bytestream); - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaSetColumnDefault::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ATA_SET_COLUMN_DEFAULT; - bytestream << fColumnName; - fDefaultValue->serialize( bytestream ); + bytestream << (quadbyte)DDL_ATA_SET_COLUMN_DEFAULT; + bytestream << fColumnName; + fDefaultValue->serialize(bytestream); - return ret; + return ret; } - - /////////////////////////////////////// /// AtaDropColumnDefault Serialization /////////////////////////////////////// @@ -902,27 +878,25 @@ int AtaSetColumnDefault::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaDropColumnDefault::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // read column name - bytestream >> fColumnName; + // read column name + bytestream >> fColumnName; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaDropColumnDefault::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ATA_DROP_COLUMN_DEFAULT; - bytestream << fColumnName; + bytestream << (quadbyte)DDL_ATA_DROP_COLUMN_DEFAULT; + bytestream << fColumnName; - return ret; + return ret; } - - /////////////////////////////////////// /// AtaDropTableConstraint Serialization /////////////////////////////////////// @@ -930,31 +904,30 @@ int AtaDropColumnDefault::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaDropTableConstraint::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // read table constraint - bytestream >> fConstraintName; - quadbyte action; - bytestream >> action; + // read table constraint + bytestream >> fConstraintName; + quadbyte action; + bytestream >> action; - fDropBehavior = (DDL_REFERENTIAL_ACTION) action; + fDropBehavior = (DDL_REFERENTIAL_ACTION)action; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaDropTableConstraint::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ATA_DROP_TABLE_CONSTRAINT; - bytestream << fConstraintName; - bytestream << (quadbyte) fDropBehavior; + bytestream << (quadbyte)DDL_ATA_DROP_TABLE_CONSTRAINT; + bytestream << fConstraintName; + bytestream << (quadbyte)fDropBehavior; - return ret; + return ret; } - /////////////////////////////////////// /// AtaTableComment Serialization /////////////////////////////////////// @@ -962,27 +935,25 @@ int AtaDropTableConstraint::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaTableComment::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // read table constraint - bytestream >> fTableComment; + // read table constraint + bytestream >> fTableComment; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaTableComment::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ATA_TABLE_COMMENT; - bytestream << fTableComment; + bytestream << (quadbyte)DDL_ATA_TABLE_COMMENT; + bytestream << fTableComment; - return ret; + return ret; } - - /////////////////////////////////////// /// AtaRenameTable Serialization /////////////////////////////////////// @@ -990,32 +961,30 @@ int AtaTableComment::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaRenameTable::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fQualifiedName = new QualifiedName(); + fQualifiedName = new QualifiedName(); - // read the table and schema name - fQualifiedName->unserialize( bytestream ); + // read the table and schema name + fQualifiedName->unserialize(bytestream); - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaRenameTable::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write type code - bytestream << (quadbyte) DDL_ATA_RENAME_TABLE; + // write type code + bytestream << (quadbyte)DDL_ATA_RENAME_TABLE; - // write the table and schema name - fQualifiedName->serialize( bytestream ); + // write the table and schema name + fQualifiedName->serialize(bytestream); - return ret; + return ret; } - - /////////////////////////////////////// /// AtaModifyColumnType Serialization /////////////////////////////////////// @@ -1023,36 +992,34 @@ int AtaRenameTable::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaModifyColumnType::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - fColumnType = new ColumnType(); + fColumnType = new ColumnType(); - // read column type and name - fColumnType->unserialize( bytestream ); + // read column type and name + fColumnType->unserialize(bytestream); - bytestream >> fName; + bytestream >> fName; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaModifyColumnType::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write type code - bytestream << (quadbyte) DDL_ATA_MODIFY_COLUMN_TYPE; + // write type code + bytestream << (quadbyte)DDL_ATA_MODIFY_COLUMN_TYPE; - // write column type and name - fColumnType->serialize( bytestream ); + // write column type and name + fColumnType->serialize(bytestream); - bytestream << fName; + bytestream << fName; - return ret; + return ret; } - - /////////////////////////////////////// /// AtaRenameColumn Serialization /////////////////////////////////////// @@ -1060,70 +1027,68 @@ int AtaModifyColumnType::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int AtaRenameColumn::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // read column names - bytestream >> fName; - bytestream >> fNewName; + // read column names + bytestream >> fName; + bytestream >> fNewName; - if (!fNewType) - fNewType = new ColumnType(DDL_INT); + if (!fNewType) + fNewType = new ColumnType(DDL_INT); - fNewType->unserialize(bytestream); + fNewType->unserialize(bytestream); - read_vec(fConstraints, bytestream); + read_vec(fConstraints, bytestream); - // read default value. It might not be there since the parser does - // not make one unless specified. + // read default value. It might not be there since the parser does + // not make one unless specified. - quadbyte type; - bytestream >> type; + quadbyte type; + bytestream >> type; - if (type == DDL_NULL) - { - fDefaultValue = 0; - } - else - { - fDefaultValue = new ColumnDefaultValue(); - fDefaultValue->unserialize(bytestream); - } + if (type == DDL_NULL) + { + fDefaultValue = 0; + } + else + { + fDefaultValue = new ColumnDefaultValue(); + fDefaultValue->unserialize(bytestream); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int AtaRenameColumn::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << (quadbyte) DDL_ATA_RENAME_COLUMN; - bytestream << fName; - bytestream << fNewName; + bytestream << (quadbyte)DDL_ATA_RENAME_COLUMN; + bytestream << fName; + bytestream << fNewName; - if (!fNewType) - fNewType = new ColumnType(DDL_INT); + if (!fNewType) + fNewType = new ColumnType(DDL_INT); - fNewType->serialize(bytestream); + fNewType->serialize(bytestream); - // serialize column constraints. - write_vec(fConstraints, bytestream); + // serialize column constraints. + write_vec(fConstraints, bytestream); - if (0 == fDefaultValue) - { - bytestream << (quadbyte)DDL_NULL; - } - else - { - bytestream << (quadbyte)DDL_COLUMN_DEFAULT_VALUE; - fDefaultValue->serialize( bytestream ); - } + if (0 == fDefaultValue) + { + bytestream << (quadbyte)DDL_NULL; + } + else + { + bytestream << (quadbyte)DDL_COLUMN_DEFAULT_VALUE; + fDefaultValue->serialize(bytestream); + } - return ret; + return ret; } - - /////////////////////////////////////// /// ColumnType Serialization /////////////////////////////////////// @@ -1131,72 +1096,70 @@ int AtaRenameColumn::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int ColumnType::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte ftype; - messageqcpp::ByteStream::quadbyte length; - messageqcpp::ByteStream::quadbyte precision; - messageqcpp::ByteStream::quadbyte scale; - messageqcpp::ByteStream::quadbyte withtimezone; - messageqcpp::ByteStream::quadbyte compressiontype; - std::string autoincrement; - messageqcpp::ByteStream::octbyte nextVal; + messageqcpp::ByteStream::quadbyte ftype; + messageqcpp::ByteStream::quadbyte length; + messageqcpp::ByteStream::quadbyte precision; + messageqcpp::ByteStream::quadbyte scale; + messageqcpp::ByteStream::quadbyte withtimezone; + messageqcpp::ByteStream::quadbyte compressiontype; + std::string autoincrement; + messageqcpp::ByteStream::octbyte nextVal; - // read column types - bytestream >> ftype; - bytestream >> length; - bytestream >> precision; - bytestream >> scale; - bytestream >> withtimezone; - bytestream >> compressiontype; - bytestream >> autoincrement; - bytestream >> nextVal; + // read column types + bytestream >> ftype; + bytestream >> length; + bytestream >> precision; + bytestream >> scale; + bytestream >> withtimezone; + bytestream >> compressiontype; + bytestream >> autoincrement; + bytestream >> nextVal; - fType = ftype; - fLength = length; - fPrecision = precision; - fScale = scale; - fWithTimezone = (withtimezone != 0); - fCompressiontype = compressiontype; - fAutoincrement = autoincrement; - fNextvalue = nextVal; + fType = ftype; + fLength = length; + fPrecision = precision; + fScale = scale; + fWithTimezone = (withtimezone != 0); + fCompressiontype = compressiontype; + fAutoincrement = autoincrement; + fNextvalue = nextVal; -// cout << "BS length = " << bytestream.length() << endl; + // cout << "BS length = " << bytestream.length() << endl; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int ColumnType::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte ftype = fType; - messageqcpp::ByteStream::quadbyte length = fLength; - messageqcpp::ByteStream::quadbyte precision = fPrecision; - messageqcpp::ByteStream::quadbyte scale = fScale; - messageqcpp::ByteStream::quadbyte withtimezone = fWithTimezone; - messageqcpp::ByteStream::quadbyte compressiontype = fCompressiontype; - std::string autoincrement = fAutoincrement; - messageqcpp::ByteStream::octbyte nextVal = fNextvalue; + messageqcpp::ByteStream::quadbyte ftype = fType; + messageqcpp::ByteStream::quadbyte length = fLength; + messageqcpp::ByteStream::quadbyte precision = fPrecision; + messageqcpp::ByteStream::quadbyte scale = fScale; + messageqcpp::ByteStream::quadbyte withtimezone = fWithTimezone; + messageqcpp::ByteStream::quadbyte compressiontype = fCompressiontype; + std::string autoincrement = fAutoincrement; + messageqcpp::ByteStream::octbyte nextVal = fNextvalue; - // write column types - bytestream << ftype; - bytestream << length; - bytestream << precision; - bytestream << scale; - bytestream << withtimezone; - bytestream << compressiontype; - bytestream << autoincrement; - bytestream << nextVal; + // write column types + bytestream << ftype; + bytestream << length; + bytestream << precision; + bytestream << scale; + bytestream << withtimezone; + bytestream << compressiontype; + bytestream << autoincrement; + bytestream << nextVal; -// cout << "BS length = " << bytestream.length() << endl; + // cout << "BS length = " << bytestream.length() << endl; - return ret; + return ret; } - - /////////////////////////////////////// /// ColumnConstraintDef Serialization /////////////////////////////////////// @@ -1204,47 +1167,45 @@ int ColumnType::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int ColumnConstraintDef::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte deferrable; - messageqcpp::ByteStream::quadbyte checktime; - messageqcpp::ByteStream::quadbyte constrainttype; + messageqcpp::ByteStream::quadbyte deferrable; + messageqcpp::ByteStream::quadbyte checktime; + messageqcpp::ByteStream::quadbyte constrainttype; - // read constaint defs - bytestream >> fName; - bytestream >> deferrable; - bytestream >> checktime; - bytestream >> constrainttype; - bytestream >> fCheck; + // read constaint defs + bytestream >> fName; + bytestream >> deferrable; + bytestream >> checktime; + bytestream >> constrainttype; + bytestream >> fCheck; - fDeferrable = (deferrable != 0); - fCheckTime = (DDL_CONSTRAINT_ATTRIBUTES) checktime; - fConstraintType = (DDL_CONSTRAINTS) constrainttype; + fDeferrable = (deferrable != 0); + fCheckTime = (DDL_CONSTRAINT_ATTRIBUTES)checktime; + fConstraintType = (DDL_CONSTRAINTS)constrainttype; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int ColumnConstraintDef::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte deferrable = fDeferrable; - messageqcpp::ByteStream::quadbyte checktime = fCheckTime; - messageqcpp::ByteStream::quadbyte constrainttype = fConstraintType; + messageqcpp::ByteStream::quadbyte deferrable = fDeferrable; + messageqcpp::ByteStream::quadbyte checktime = fCheckTime; + messageqcpp::ByteStream::quadbyte constrainttype = fConstraintType; - // write constaint defs - bytestream << fName; - bytestream << deferrable; - bytestream << checktime; - bytestream << constrainttype; - bytestream << fCheck; + // write constaint defs + bytestream << fName; + bytestream << deferrable; + bytestream << checktime; + bytestream << constrainttype; + bytestream << fCheck; - return ret; + return ret; } - - /////////////////////////////////////// /// ColumnDefaultValue Serialization /////////////////////////////////////// @@ -1252,33 +1213,31 @@ int ColumnConstraintDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int ColumnDefaultValue::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // read update and delete actions - quadbyte qb; + // read update and delete actions + quadbyte qb; - bytestream >> qb; - fNull = (qb != 0); + bytestream >> qb; + fNull = (qb != 0); - bytestream >> fValue; + bytestream >> fValue; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int ColumnDefaultValue::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - // write update and delete actions - bytestream << (quadbyte)fNull; - bytestream << fValue; + // write update and delete actions + bytestream << (quadbyte)fNull; + bytestream << fValue; - return ret; + return ret; } - - /////////////////////////////////////// /// ColumnDef Serialization /////////////////////////////////////// @@ -1286,67 +1245,64 @@ int ColumnDefaultValue::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int ColumnDef::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream >> fName; + bytestream >> fName; - // read column type - fType = new ColumnType(); - fType->unserialize( bytestream ); + // read column type + fType = new ColumnType(); + fType->unserialize(bytestream); - read_vec(fConstraints, bytestream); + read_vec(fConstraints, bytestream); - // read default value. It might not be there since the parser does - // not make one unless specified. + // read default value. It might not be there since the parser does + // not make one unless specified. - quadbyte type; - bytestream >> type; + quadbyte type; + bytestream >> type; - if (type == DDL_NULL) - { - fDefaultValue = 0; - } - else - { - fDefaultValue = new ColumnDefaultValue(); - fDefaultValue->unserialize(bytestream); - } + if (type == DDL_NULL) + { + fDefaultValue = 0; + } + else + { + fDefaultValue = new ColumnDefaultValue(); + fDefaultValue->unserialize(bytestream); + } -// cout << "BS length = " << bytestream.length() << endl; - return ret; + // cout << "BS length = " << bytestream.length() << endl; + return ret; } /** @brief Serialize to ByteStream */ int ColumnDef::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << fName; + bytestream << fName; - // write column type - fType->serialize( bytestream ); + // write column type + fType->serialize(bytestream); - // serialize column constraints. - write_vec(fConstraints, bytestream); + // serialize column constraints. + write_vec(fConstraints, bytestream); - if (0 == fDefaultValue) - { - bytestream << (quadbyte)DDL_NULL; - } - else - { - bytestream << (quadbyte)DDL_COLUMN_DEFAULT_VALUE; - fDefaultValue->serialize( bytestream ); - } + if (0 == fDefaultValue) + { + bytestream << (quadbyte)DDL_NULL; + } + else + { + bytestream << (quadbyte)DDL_COLUMN_DEFAULT_VALUE; + fDefaultValue->serialize(bytestream); + } + // cout << "BS length = " << bytestream.length() << endl; -// cout << "BS length = " << bytestream.length() << endl; - - return ret; + return ret; } - - /////////////////////////////////////// /// TableConstraintDef Serialization /////////////////////////////////////// @@ -1354,36 +1310,31 @@ int ColumnDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TableConstraintDef::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte constrainttype; + messageqcpp::ByteStream::quadbyte constrainttype; - // read constraint def - bytestream >> constrainttype; + // read constraint def + bytestream >> constrainttype; - fConstraintType = (DDL_CONSTRAINTS) constrainttype; + fConstraintType = (DDL_CONSTRAINTS)constrainttype; - - - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TableConstraintDef::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte constrainttype = fConstraintType; + messageqcpp::ByteStream::quadbyte constrainttype = fConstraintType; - // write constraint def - bytestream << constrainttype; + // write constraint def + bytestream << constrainttype; - - return ret; + return ret; } - - /////////////////////////////////////// /// TableUniqueConstraintDef Serialization /////////////////////////////////////// @@ -1391,44 +1342,42 @@ int TableConstraintDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TableUniqueConstraintDef::unserialize(ByteStream& bytestream) { - int ret = 1; - quadbyte count; + int ret = 1; + quadbyte count; - bytestream >> fName; - bytestream >> count; + bytestream >> fName; + bytestream >> count; - string str; + string str; - while (count-- > 0) - { - bytestream >> str; - fColumnNameList.push_back(str); - } + while (count-- > 0) + { + bytestream >> str; + fColumnNameList.push_back(str); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TableUniqueConstraintDef::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << fName; + bytestream << fName; - bytestream << (quadbyte) fColumnNameList.size(); + bytestream << (quadbyte)fColumnNameList.size(); - ColumnNameList::const_iterator itr; + ColumnNameList::const_iterator itr; - for (itr = fColumnNameList.begin(); itr != fColumnNameList.end(); ++itr) - { - bytestream << *itr; - } + for (itr = fColumnNameList.begin(); itr != fColumnNameList.end(); ++itr) + { + bytestream << *itr; + } - return ret; + return ret; } - - /////////////////////////////////////// /// TablePrimaryKeyConstraintDef Serialization /////////////////////////////////////// @@ -1436,44 +1385,42 @@ int TableUniqueConstraintDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TablePrimaryKeyConstraintDef::unserialize(ByteStream& bytestream) { - int ret = 1; - quadbyte count; + int ret = 1; + quadbyte count; - bytestream >> fName; - bytestream >> count; + bytestream >> fName; + bytestream >> count; - string str; + string str; - while (count-- > 0) - { - bytestream >> str; - fColumnNameList.push_back(str); - } + while (count-- > 0) + { + bytestream >> str; + fColumnNameList.push_back(str); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TablePrimaryKeyConstraintDef::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << fName; + bytestream << fName; - bytestream << (quadbyte) fColumnNameList.size(); + bytestream << (quadbyte)fColumnNameList.size(); - ColumnNameList::const_iterator itr; + ColumnNameList::const_iterator itr; - for (itr = fColumnNameList.begin(); itr != fColumnNameList.end(); ++itr) - { - bytestream << *itr; - } + for (itr = fColumnNameList.begin(); itr != fColumnNameList.end(); ++itr) + { + bytestream << *itr; + } - return ret; + return ret; } - - /////////////////////////////////////// /// ReferentialAction Serialization /////////////////////////////////////// @@ -1481,40 +1428,36 @@ int TablePrimaryKeyConstraintDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int ReferentialAction::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte onupdate; - messageqcpp::ByteStream::quadbyte ondelete; + messageqcpp::ByteStream::quadbyte onupdate; + messageqcpp::ByteStream::quadbyte ondelete; - // read check - bytestream >> onupdate; - bytestream >> ondelete; + // read check + bytestream >> onupdate; + bytestream >> ondelete; - fOnUpdate = (DDL_REFERENTIAL_ACTION) onupdate; - fOnDelete = (DDL_REFERENTIAL_ACTION) ondelete; + fOnUpdate = (DDL_REFERENTIAL_ACTION)onupdate; + fOnDelete = (DDL_REFERENTIAL_ACTION)ondelete; - - return ret; + return ret; } /** @brief Serialize to ByteStream */ int ReferentialAction::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte onupdate = fOnUpdate; - messageqcpp::ByteStream::quadbyte ondelete = fOnDelete; + messageqcpp::ByteStream::quadbyte onupdate = fOnUpdate; + messageqcpp::ByteStream::quadbyte ondelete = fOnDelete; - // write update and delete actions - bytestream << onupdate; - bytestream << ondelete; + // write update and delete actions + bytestream << onupdate; + bytestream << ondelete; - - return ret; + return ret; } - - /////////////////////////////////////// /// TableReferencesConstraintDef Serialization /////////////////////////////////////// @@ -1522,101 +1465,99 @@ int ReferentialAction::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TableReferencesConstraintDef::unserialize(ByteStream& bytestream) { - string str; - int ret = 1; - quadbyte count; + string str; + int ret = 1; + quadbyte count; - // Name - bytestream >> fName; + // Name + bytestream >> fName; - // Local columns - bytestream >> count; + // Local columns + bytestream >> count; - while (count-- > 0) - { - bytestream >> str; - fColumns.push_back(str); - } + while (count-- > 0) + { + bytestream >> str; + fColumns.push_back(str); + } - // Table name - fTableName = new QualifiedName; - fTableName->unserialize(bytestream); + // Table name + fTableName = new QualifiedName; + fTableName->unserialize(bytestream); - // Foreign columns - bytestream >> count; + // Foreign columns + bytestream >> count; - while (count-- > 0) - { - bytestream >> str; - fForeignColumns.push_back(str); - } + while (count-- > 0) + { + bytestream >> str; + fForeignColumns.push_back(str); + } - // Match type - quadbyte matchType; - bytestream >> matchType; - fMatchType = (DDL_MATCH_TYPE) matchType; + // Match type + quadbyte matchType; + bytestream >> matchType; + fMatchType = (DDL_MATCH_TYPE)matchType; - // Ref Action - quadbyte sertype; - bytestream >> sertype; + // Ref Action + quadbyte sertype; + bytestream >> sertype; - if (sertype == DDL_NULL) - { - fRefAction = 0; - } - else - { - fRefAction = new ReferentialAction(); - fRefAction->unserialize(bytestream); - } + if (sertype == DDL_NULL) + { + fRefAction = 0; + } + else + { + fRefAction = new ReferentialAction(); + fRefAction->unserialize(bytestream); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TableReferencesConstraintDef::serialize(ByteStream& bytestream) { - int ret = 1; - messageqcpp::ByteStream::quadbyte size; - bytestream << fName; + int ret = 1; + messageqcpp::ByteStream::quadbyte size; + bytestream << fName; - // local columns - size = fColumns.size(); - bytestream << size; - ColumnNameList::const_iterator itr; + // local columns + size = fColumns.size(); + bytestream << size; + ColumnNameList::const_iterator itr; - for (itr = fColumns.begin(); itr != fColumns.end(); ++itr) - bytestream << *itr; + for (itr = fColumns.begin(); itr != fColumns.end(); ++itr) + bytestream << *itr; - // Table name - fTableName->serialize(bytestream); + // Table name + fTableName->serialize(bytestream); - // Foreign columns - size = fForeignColumns.size(); - bytestream << size; + // Foreign columns + size = fForeignColumns.size(); + bytestream << size; - for (itr = fForeignColumns.begin(); itr != fForeignColumns.end(); ++itr) - bytestream << *itr; + for (itr = fForeignColumns.begin(); itr != fForeignColumns.end(); ++itr) + bytestream << *itr; - // Match type - bytestream << (quadbyte) fMatchType; + // Match type + bytestream << (quadbyte)fMatchType; - // Ref action - if (0 == fRefAction) - { - bytestream << (quadbyte) DDL_NULL; - } - else - { - bytestream << (quadbyte) DDL_REF_ACTION; - fRefAction->serialize(bytestream); - } + // Ref action + if (0 == fRefAction) + { + bytestream << (quadbyte)DDL_NULL; + } + else + { + bytestream << (quadbyte)DDL_REF_ACTION; + fRefAction->serialize(bytestream); + } - return ret; + return ret; } - - /////////////////////////////////////// /// TableCheckConstraintDef Serialization /////////////////////////////////////// @@ -1624,27 +1565,25 @@ int TableReferencesConstraintDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TableCheckConstraintDef::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream >> fName; - bytestream >> fCheck; + bytestream >> fName; + bytestream >> fCheck; - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TableCheckConstraintDef::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << fName; - bytestream << fCheck; + bytestream << fName; + bytestream << fCheck; - return ret; + return ret; } - - /////////////////////////////////////// /// TableDef Serialization /////////////////////////////////////// @@ -1652,114 +1591,108 @@ int TableCheckConstraintDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int TableDef::unserialize(ByteStream& bytestream) { - int ret = 1; - messageqcpp::ByteStream::quadbyte type; + int ret = 1; + messageqcpp::ByteStream::quadbyte type; - // table name - fQualifiedName = new QualifiedName(); - fQualifiedName->unserialize(bytestream); + // table name + fQualifiedName = new QualifiedName(); + fQualifiedName->unserialize(bytestream); - // ColumnDefs - read_vec(fColumns, bytestream); + // ColumnDefs + read_vec(fColumns, bytestream); - // read column constraint list - quadbyte count; - bytestream >> count; - TableConstraintDef* constraint; + // read column constraint list + quadbyte count; + bytestream >> count; + TableConstraintDef* constraint; - while (count-- > 0) + while (count-- > 0) + { + bytestream >> type; + + switch (type) { - bytestream >> type; + case DDL_TABLE_UNIQUE_CONSTRAINT_DEF: + constraint = new TableUniqueConstraintDef(); + constraint->unserialize(bytestream); + fConstraints.push_back(constraint); + break; - switch (type) - { - case DDL_TABLE_UNIQUE_CONSTRAINT_DEF: - constraint = new TableUniqueConstraintDef(); - constraint->unserialize(bytestream); - fConstraints.push_back( constraint ); - break; + case DDL_TABLE_PRIMARY_CONSTRAINT_DEF: + constraint = new TablePrimaryKeyConstraintDef(); + constraint->unserialize(bytestream); + fConstraints.push_back(constraint); + break; - case DDL_TABLE_PRIMARY_CONSTRAINT_DEF: - constraint = new TablePrimaryKeyConstraintDef(); - constraint->unserialize(bytestream); - fConstraints.push_back( constraint ); - break; + case DDL_TABLE_REFERENCES_CONSTRAINT_DEF: + constraint = new TableReferencesConstraintDef(); + constraint->unserialize(bytestream); + fConstraints.push_back(constraint); + break; - case DDL_TABLE_REFERENCES_CONSTRAINT_DEF: - constraint = new TableReferencesConstraintDef(); - constraint->unserialize(bytestream); - fConstraints.push_back( constraint ); - break; + case DDL_TABLE_CHECK_CONSTRAINT_DEF: + constraint = new TableCheckConstraintDef(); + constraint->unserialize(bytestream); + fConstraints.push_back(constraint); + break; - case DDL_TABLE_CHECK_CONSTRAINT_DEF: - constraint = new TableCheckConstraintDef(); - constraint->unserialize(bytestream); - fConstraints.push_back( constraint ); - break; - - default: - throw ("Bad typecode for TableConstraintDef"); - break; - } + default: throw("Bad typecode for TableConstraintDef"); break; } + } - // read option maps list - bytestream >> count; + // read option maps list + bytestream >> count; - for ( unsigned int i = 0; i < count; i++ ) - { - // read option map - string map, map1; - bytestream >> map; - bytestream >> map1; - fOptions.insert( pair (map, map1) ); - } + for (unsigned int i = 0; i < count; i++) + { + // read option map + string map, map1; + bytestream >> map; + bytestream >> map1; + fOptions.insert(pair(map, map1)); + } - return ret; + return ret; } /** @brief Serialize to ByteStream */ int TableDef::serialize(ByteStream& bytestream) { - int ret = 1; - messageqcpp::ByteStream::quadbyte size; - // table name - fQualifiedName->serialize( bytestream ); + int ret = 1; + messageqcpp::ByteStream::quadbyte size; + // table name + fQualifiedName->serialize(bytestream); - // ColumnDef's - write_vec(fColumns, bytestream); + // ColumnDef's + write_vec(fColumns, bytestream); - // write table constraint list - size = fConstraints.size(); - bytestream << size; + // write table constraint list + size = fConstraints.size(); + bytestream << size; - TableConstraintDefList::const_iterator itr; + TableConstraintDefList::const_iterator itr; - for (itr = fConstraints.begin(); - itr != fConstraints.end(); - ++itr) - { - bytestream << (quadbyte) (*itr)->getSerialType(); - (*itr)->serialize( bytestream ); - } + for (itr = fConstraints.begin(); itr != fConstraints.end(); ++itr) + { + bytestream << (quadbyte)(*itr)->getSerialType(); + (*itr)->serialize(bytestream); + } - // serialize TableOptions - size = fOptions.size(); - bytestream << size; + // serialize TableOptions + size = fOptions.size(); + bytestream << size; - pair oval; - TableOptionMap::const_iterator itr2; + pair oval; + TableOptionMap::const_iterator itr2; - for (itr2 = fOptions.begin(); - itr2 != fOptions.end(); - ++itr2) - { - oval = *itr2; - bytestream << oval.first; - bytestream << oval.second; - } + for (itr2 = fOptions.begin(); itr2 != fOptions.end(); ++itr2) + { + oval = *itr2; + bytestream << oval.first; + bytestream << oval.second; + } - return ret; + return ret; } /////////////////////////////////////// @@ -1769,68 +1702,60 @@ int TableDef::serialize(ByteStream& bytestream) /** @brief Construct from Bytestream */ int QualifiedName::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream >> fCatalog; - bytestream >> fSchema; - bytestream >> fName; + bytestream >> fCatalog; + bytestream >> fSchema; + bytestream >> fName; - - return ret; + return ret; } /** @brief Serialize to ByteStream */ int QualifiedName::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - bytestream << fCatalog; - bytestream << fSchema; - bytestream << fName; + bytestream << fCatalog; + bytestream << fSchema; + bytestream << fName; - return ret; + return ret; } - /////////////////////////////////////// /// ConstraintAttributes Serialization /////////////////////////////////////// - /** @brief Construct from Bytestream */ int ConstraintAttributes::unserialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte checktime; - messageqcpp::ByteStream::quadbyte deferrable; + messageqcpp::ByteStream::quadbyte checktime; + messageqcpp::ByteStream::quadbyte deferrable; - // read the checktime and deferrable flag - bytestream >> checktime; - bytestream >> deferrable; + // read the checktime and deferrable flag + bytestream >> checktime; + bytestream >> deferrable; - fCheckTime = (DDL_CONSTRAINT_ATTRIBUTES) checktime; - fDeferrable = (deferrable != 0); + fCheckTime = (DDL_CONSTRAINT_ATTRIBUTES)checktime; + fDeferrable = (deferrable != 0); - - return ret; + return ret; } /** @brief Serialize to ByteStream */ int ConstraintAttributes::serialize(ByteStream& bytestream) { - int ret = 1; + int ret = 1; - messageqcpp::ByteStream::quadbyte checktime = fCheckTime; - messageqcpp::ByteStream::quadbyte deferrable = fDeferrable; + messageqcpp::ByteStream::quadbyte checktime = fCheckTime; + messageqcpp::ByteStream::quadbyte deferrable = fDeferrable; - // write the checktime and deferrable flag - bytestream << checktime; - bytestream << deferrable; + // write the checktime and deferrable flag + bytestream << checktime; + bytestream << deferrable; - - return ret; + return ret; } - - - diff --git a/dbcon/ddlpackage/sqlparser.cpp b/dbcon/ddlpackage/sqlparser.cpp index c8ee3586f..81a8367ce 100644 --- a/dbcon/ddlpackage/sqlparser.cpp +++ b/dbcon/ddlpackage/sqlparser.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: sqlparser.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: sqlparser.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -45,104 +45,94 @@ namespace ddlpackage { using namespace std; -SqlParser::SqlParser() : - fStatus(-1), - fDebug(false), - x(&fParseTree) +SqlParser::SqlParser() : fStatus(-1), fDebug(false), x(&fParseTree) { } - void SqlParser::SetDebug(bool debug) { - fDebug = debug; + fDebug = debug; } void SqlParser::setDefaultSchema(std::string schema) { - x.fDBSchema = schema; + x.fDBSchema = schema; } void SqlParser::setDefaultCharset(const CHARSET_INFO* default_charset) { - x.default_table_charset = default_charset; + x.default_table_charset = default_charset; } int SqlParser::Parse(const char* sqltext) { - ddllex_init_extra(&scanData, &x.scanner); - scanner_init(sqltext, x.scanner); - fStatus = ddlparse(&x); - return fStatus; + ddllex_init_extra(&scanData, &x.scanner); + scanner_init(sqltext, x.scanner); + fStatus = ddlparse(&x); + return fStatus; } - const ParseTree& SqlParser::GetParseTree(void) { - if (!Good()) - { - throw logic_error("The ParseTree is invalid"); - } + if (!Good()) + { + throw logic_error("The ParseTree is invalid"); + } - return fParseTree; + return fParseTree; } - bool SqlParser::Good() { - return fStatus == 0; + return fStatus == 0; } - SqlParser::~SqlParser() { - scanner_finish(x.scanner); // free scanner allocated memory - ddllex_destroy(x.scanner); + scanner_finish(x.scanner); // free scanner allocated memory + ddllex_destroy(x.scanner); } - -SqlFileParser::SqlFileParser() : - SqlParser() +SqlFileParser::SqlFileParser() : SqlParser() { } - int SqlFileParser::Parse(const string& sqlfile) { - fStatus = -1; + fStatus = -1; - ifstream ifsql; - ifsql.open(sqlfile.c_str()); + ifstream ifsql; + ifsql.open(sqlfile.c_str()); - if (!ifsql.is_open()) - { - perror(sqlfile.c_str()); - return fStatus; - } + if (!ifsql.is_open()) + { + perror(sqlfile.c_str()); + return fStatus; + } - char sqlbuf[1024 * 1024]; - unsigned length; - ifsql.seekg (0, ios::end); - length = ifsql.tellg(); - ifsql.seekg (0, ios::beg); + char sqlbuf[1024 * 1024]; + unsigned length; + ifsql.seekg(0, ios::end); + length = ifsql.tellg(); + ifsql.seekg(0, ios::beg); - if (length > sizeof(sqlbuf) - 1) - { - throw length_error("SqlFileParser has file size hard limit of 16K."); - } + if (length > sizeof(sqlbuf) - 1) + { + throw length_error("SqlFileParser has file size hard limit of 16K."); + } - std::streamsize rcount; - rcount = ifsql.readsome(sqlbuf, sizeof(sqlbuf) - 1); + std::streamsize rcount; + rcount = ifsql.readsome(sqlbuf, sizeof(sqlbuf) - 1); - if (rcount < 0) - return fStatus; + if (rcount < 0) + return fStatus; - sqlbuf[rcount] = 0; + sqlbuf[rcount] = 0; - //cout << endl << sqlfile << "(" << rcount << ")" << endl; - //cout << "----------------------" << endl; - //cout << sqlbuf << endl; + // cout << endl << sqlfile << "(" << rcount << ")" << endl; + // cout << "----------------------" << endl; + // cout << sqlbuf << endl; - return SqlParser::Parse(sqlbuf); -} + return SqlParser::Parse(sqlbuf); } +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/sqlparser.h b/dbcon/ddlpackage/sqlparser.h index ac1f0a0ff..6e477a449 100644 --- a/dbcon/ddlpackage/sqlparser.h +++ b/dbcon/ddlpackage/sqlparser.h @@ -17,17 +17,17 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: sqlparser.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: sqlparser.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file * * This contains a class wrapper for the Bison parsing machinery. */ #include -#include "collation.h" // CHARSET_INFO +#include "collation.h" // CHARSET_INFO #include "ddlpkg.h" #if defined(_MSC_VER) && defined(xxxDDLPKGSQLPARSER_DLLEXPORT) @@ -38,7 +38,6 @@ namespace ddlpackage { - typedef SqlStatementList ParseTree; /** @brief SqlParser is a class interface around the Bison parser @@ -54,7 +53,7 @@ typedef SqlStatementList ParseTree; or SqlFileParser parser; - parser.setDefaultSchema("tpch"); + parser.setDefaultSchema("tpch"); parser.Parse(sqlFileName); if (parser.Good()) { @@ -76,78 +75,77 @@ typedef std::vector valbuf_t; struct scan_data { - /* Handles to the buffer that the lexer uses internally */ - char* scanbuf; - void* scanbufhandle; // This is a YY_BUFFER_STATE defined in ddl-scan.cpp - valbuf_t valbuf; + /* Handles to the buffer that the lexer uses internally */ + char* scanbuf; + void* scanbufhandle; // This is a YY_BUFFER_STATE defined in ddl-scan.cpp + valbuf_t valbuf; }; struct pass_to_bison { - ParseTree* fParseTree; - std::string fDBSchema; - void* scanner; - const CHARSET_INFO* default_table_charset; + ParseTree* fParseTree; + std::string fDBSchema; + void* scanner; + const CHARSET_INFO* default_table_charset; - pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL), default_table_charset(NULL) {}; + pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL), default_table_charset(NULL){}; }; class SqlParser { -public: - EXPORT SqlParser(void); + public: + EXPORT SqlParser(void); - EXPORT virtual ~SqlParser(); + EXPORT virtual ~SqlParser(); - EXPORT int Parse(const char* sqltext); + EXPORT int Parse(const char* sqltext); - /** @brief Return the ParseTree if state is Good. Otherwise - * throw a logic_error. - */ - EXPORT const ParseTree& GetParseTree(void); + /** @brief Return the ParseTree if state is Good. Otherwise + * throw a logic_error. + */ + EXPORT const ParseTree& GetParseTree(void); - /** @brief Tells whether current state resulted from a good - * parse. - */ - EXPORT bool Good(void); + /** @brief Tells whether current state resulted from a good + * parse. + */ + EXPORT bool Good(void); - /** @brief Control bison debugging - */ - EXPORT void SetDebug(bool debug); + /** @brief Control bison debugging + */ + EXPORT void SetDebug(bool debug); - /** @brief Set the default schema to use if it is not - * supplied in the DDL statement - * - * @param schema the default schema - */ - EXPORT void setDefaultSchema(std::string schema); + /** @brief Set the default schema to use if it is not + * supplied in the DDL statement + * + * @param schema the default schema + */ + EXPORT void setDefaultSchema(std::string schema); - /** @brief Set the default table charset. Can be overriden by column - * or table options - * - * @param default_charset the default CHARSET_INFO pointer - */ - EXPORT void setDefaultCharset(const CHARSET_INFO* default_charset); + /** @brief Set the default table charset. Can be overriden by column + * or table options + * + * @param default_charset the default CHARSET_INFO pointer + */ + EXPORT void setDefaultCharset(const CHARSET_INFO* default_charset); -protected: - ParseTree fParseTree; - std::string fDBSchema; - int fStatus; ///< return from yyparse() stored here. - bool fDebug; ///< Turn on bison debugging. - scan_data scanData; - pass_to_bison x; + protected: + ParseTree fParseTree; + std::string fDBSchema; + int fStatus; ///< return from yyparse() stored here. + bool fDebug; ///< Turn on bison debugging. + scan_data scanData; + pass_to_bison x; }; - /** SqlFileParser is a testing device. - */ + */ class SqlFileParser : public SqlParser { -public: - SqlFileParser(); - int Parse(const std::string& fileName); + public: + SqlFileParser(); + int Parse(const std::string& fileName); }; -} +} // namespace ddlpackage #undef EXPORT diff --git a/dbcon/ddlpackage/sqlstatement.cpp b/dbcon/ddlpackage/sqlstatement.cpp index 53e26596b..be94dcd37 100644 --- a/dbcon/ddlpackage/sqlstatement.cpp +++ b/dbcon/ddlpackage/sqlstatement.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: sqlstatement.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: sqlstatement.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -33,7 +33,7 @@ static uint32_t sessionID = 1; SqlStatement::SqlStatement() { - fSessionID = sessionID; + fSessionID = sessionID; } SqlStatement::~SqlStatement() @@ -42,6 +42,6 @@ SqlStatement::~SqlStatement() ostream& operator<<(ostream& os, const SqlStatement& stmt) { - return stmt.put(os); -} + return stmt.put(os); } +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/sqlstatementlist.cpp b/dbcon/ddlpackage/sqlstatementlist.cpp index 343355251..ad50ebf95 100644 --- a/dbcon/ddlpackage/sqlstatementlist.cpp +++ b/dbcon/ddlpackage/sqlstatementlist.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: sqlstatementlist.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: sqlstatementlist.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" @@ -29,35 +29,32 @@ namespace ddlpackage { using namespace std; - ostream& operator<<(ostream& os, const SqlStatementList& ssl) { - vector::const_iterator itr; + vector::const_iterator itr; - for (itr = ssl.fList.begin(); itr != ssl.fList.end(); ++itr) - { - SqlStatement& stmt = **itr; - os << stmt; - } + for (itr = ssl.fList.begin(); itr != ssl.fList.end(); ++itr) + { + SqlStatement& stmt = **itr; + os << stmt; + } - return os; + return os; } - void SqlStatementList::push_back(SqlStatement* v) { - fList.push_back(v); + fList.push_back(v); } - SqlStatementList::~SqlStatementList() { - vector::iterator itr; + vector::iterator itr; - for (itr = fList.begin(); itr != fList.end(); ++itr) - { - delete *itr; - } + for (itr = fList.begin(); itr != fList.end(); ++itr) + { + delete *itr; + } } -} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/tabledef.cpp b/dbcon/ddlpackage/tabledef.cpp index 03c132222..f4b8cf49a 100644 --- a/dbcon/ddlpackage/tabledef.cpp +++ b/dbcon/ddlpackage/tabledef.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: tabledef.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: tabledef.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #define DDLPKG_DLLEXPORT @@ -30,293 +30,252 @@ namespace ddlpackage { using namespace std; - TableDef::~TableDef() { + { + ColumnDefList::iterator itr; + + for (itr = fColumns.begin(); itr != fColumns.end(); itr++) { - ColumnDefList::iterator itr; - - for (itr = fColumns.begin(); itr != fColumns.end(); itr++) - { - delete *itr; - } + delete *itr; } + } + { + TableConstraintDefList::iterator itr; + + for (itr = fConstraints.begin(); itr != fConstraints.end(); itr++) { - TableConstraintDefList::iterator itr; - - for (itr = fConstraints.begin(); itr != fConstraints.end(); itr++) - { - delete *itr; - } + delete *itr; } + } - delete fQualifiedName; + delete fQualifiedName; } - -TableDef::TableDef(QualifiedName* name, TableElementList* elements, TableOptionMap* options) : - fQualifiedName(name) +TableDef::TableDef(QualifiedName* name, TableElementList* elements, TableOptionMap* options) + : fQualifiedName(name) { - if (options) + if (options) + { + fOptions = *options; + delete options; + } + + ColumnDef* column; + TableConstraintDef* constraint; + + /* When parsing, it is necessary to collect ColumnDefs and + TableConstraintDefs as TableElements. Here we separate + them out into separately typed lists. + */ + TableElementList::iterator itr; + + for (itr = elements->begin(); itr != elements->end(); ++itr) + { + column = dynamic_cast(*itr); + + if (column) { - fOptions = *options; - delete options; + fColumns.push_back(column); } - - ColumnDef* column; - TableConstraintDef* constraint; - - /* When parsing, it is necessary to collect ColumnDefs and - TableConstraintDefs as TableElements. Here we separate - them out into separately typed lists. - */ - TableElementList::iterator itr; - - for (itr = elements->begin(); itr != elements->end(); ++itr) + else { - column = dynamic_cast(*itr); + constraint = dynamic_cast(*itr); - if (column) - { - fColumns.push_back(column); - } - else - { - constraint = dynamic_cast(*itr); - - if (constraint) - { - fConstraints.push_back(constraint); - } - } + if (constraint) + { + fConstraints.push_back(constraint); + } } + } - delete elements; + delete elements; } - /** \brief Put to ostream. */ ostream& operator<<(ostream& os, const TableDef& tableDef) { - os << "CreateTable "; + os << "CreateTable "; - if (tableDef.fQualifiedName->fSchema != "") - //cout << tableDef.fQualifiedName->fSchema << "."; - os << tableDef.fQualifiedName->fName - << " " << tableDef.fConstraints.size() - << " table constraints" - << endl; + if (tableDef.fQualifiedName->fSchema != "") + // cout << tableDef.fQualifiedName->fSchema << "."; + os << tableDef.fQualifiedName->fName << " " << tableDef.fConstraints.size() << " table constraints" + << endl; + { + ColumnDefList::const_iterator itr; + + for (itr = tableDef.fColumns.begin(); itr != tableDef.fColumns.end(); ++itr) { - ColumnDefList::const_iterator itr; - - for (itr = tableDef.fColumns.begin(); - itr != tableDef.fColumns.end(); ++itr) - { - ColumnDef* col = *itr; - os << *col << endl; - } + ColumnDef* col = *itr; + os << *col << endl; } + } + { + TableConstraintDefList::const_iterator itr; + for (itr = tableDef.fConstraints.begin(); itr != tableDef.fConstraints.end(); ++itr) { - TableConstraintDefList::const_iterator itr; - - for (itr = tableDef.fConstraints.begin(); - itr != tableDef.fConstraints.end(); - ++itr) - { - os << (**itr); - } + os << (**itr); } + } - pair oval; + pair oval; + TableOptionMap::const_iterator oitr; + os << "Table Options" << endl; + + if (!tableDef.fOptions.empty()) + { TableOptionMap::const_iterator oitr; - os << "Table Options" << endl; - if (!tableDef.fOptions.empty()) + for (oitr = tableDef.fOptions.begin(); oitr != tableDef.fOptions.end(); ++oitr) { - TableOptionMap::const_iterator oitr; - - for (oitr = tableDef.fOptions.begin(); - oitr != tableDef.fOptions.end(); ++oitr) - { - oval = *oitr; - os << " " << oval.first << "=" << oval.second << endl; - } + oval = *oitr; + os << " " << oval.first << "=" << oval.second << endl; } + } - return os; + return os; } - - - ostream& operator<<(ostream& os, const TableConstraintDef& constraint) { - return constraint.put(os); + return constraint.put(os); } std::ostream& TableConstraintDef::put(std::ostream& os) const { - os << "No!!!" << endl; + os << "No!!!" << endl; - return os; + return os; } - -TableConstraintDef::TableConstraintDef(DDL_CONSTRAINTS cType) : - fConstraintType(cType) +TableConstraintDef::TableConstraintDef(DDL_CONSTRAINTS cType) : fConstraintType(cType) { } -TableConstraintDef::TableConstraintDef() : - fConstraintType(DDL_INVALID_CONSTRAINT) +TableConstraintDef::TableConstraintDef() : fConstraintType(DDL_INVALID_CONSTRAINT) { } -TableCheckConstraintDef::TableCheckConstraintDef(const char* check) : - TableConstraintDef(DDL_CHECK), - fCheck(check) +TableCheckConstraintDef::TableCheckConstraintDef(const char* check) + : TableConstraintDef(DDL_CHECK), fCheck(check) { } std::ostream& TableCheckConstraintDef::put(std::ostream& os) const { - os << "Constraint: " - << ConstraintString[fConstraintType] << " "; - os << "\"" << fCheck << "\""; - os << endl; + os << "Constraint: " << ConstraintString[fConstraintType] << " "; + os << "\"" << fCheck << "\""; + os << endl; - return os; + return os; } - -TableUniqueConstraintDef::TableUniqueConstraintDef(ColumnNameList* columns) : - TableConstraintDef(DDL_UNIQUE), - fColumnNameList(*columns) +TableUniqueConstraintDef::TableUniqueConstraintDef(ColumnNameList* columns) + : TableConstraintDef(DDL_UNIQUE), fColumnNameList(*columns) { - delete columns; + delete columns; } std::ostream& TableUniqueConstraintDef::put(std::ostream& os) const { - os << "Constraint: " - << fName << " " - << ConstraintString[fConstraintType] << " "; + os << "Constraint: " << fName << " " << ConstraintString[fConstraintType] << " "; - ColumnNameList::const_iterator itr; - os << "("; + ColumnNameList::const_iterator itr; + os << "("; - for (itr = fColumnNameList.begin(); - itr != fColumnNameList.end(); - ++itr) - { - os << *itr << " "; - } + for (itr = fColumnNameList.begin(); itr != fColumnNameList.end(); ++itr) + { + os << *itr << " "; + } - os << ")"; - return os; + os << ")"; + return os; } -TablePrimaryKeyConstraintDef::TablePrimaryKeyConstraintDef(ColumnNameList* columns) : - TableConstraintDef(DDL_PRIMARY_KEY), - fColumnNameList(*columns) +TablePrimaryKeyConstraintDef::TablePrimaryKeyConstraintDef(ColumnNameList* columns) + : TableConstraintDef(DDL_PRIMARY_KEY), fColumnNameList(*columns) { - delete columns; + delete columns; } std::ostream& TablePrimaryKeyConstraintDef::put(std::ostream& os) const { - os << "Constraint: " - << fName << " " - << ConstraintString[fConstraintType] << " "; + os << "Constraint: " << fName << " " << ConstraintString[fConstraintType] << " "; - ColumnNameList::const_iterator itr; - os << "("; + ColumnNameList::const_iterator itr; + os << "("; - for (itr = fColumnNameList.begin(); - itr != fColumnNameList.end(); - ++itr) - { - os << *itr << " "; - } + for (itr = fColumnNameList.begin(); itr != fColumnNameList.end(); ++itr) + { + os << *itr << " "; + } - os << ")"; + os << ")"; - return os; + return os; } -TableReferencesConstraintDef::TableReferencesConstraintDef -(ColumnNameList* columns, - QualifiedName* tableName, - ColumnNameList* foreignColumns, - DDL_MATCH_TYPE matchType, - ReferentialAction* refAction) : - TableConstraintDef(DDL_REFERENCES), - fColumns(*columns), - fTableName(tableName), - fForeignColumns(*foreignColumns), - fMatchType(matchType), - fRefAction(refAction) +TableReferencesConstraintDef::TableReferencesConstraintDef(ColumnNameList* columns, QualifiedName* tableName, + ColumnNameList* foreignColumns, + DDL_MATCH_TYPE matchType, + ReferentialAction* refAction) + : TableConstraintDef(DDL_REFERENCES) + , fColumns(*columns) + , fTableName(tableName) + , fForeignColumns(*foreignColumns) + , fMatchType(matchType) + , fRefAction(refAction) { - delete columns; - delete foreignColumns; + delete columns; + delete foreignColumns; } std::ostream& TableReferencesConstraintDef::put(std::ostream& os) const { - os << "Constraint: " - << fName << " " - << ConstraintString[fConstraintType] << " "; + os << "Constraint: " << fName << " " << ConstraintString[fConstraintType] << " "; - ColumnNameList::const_iterator itr; - os << "lcols ("; + ColumnNameList::const_iterator itr; + os << "lcols ("; - for (itr = fColumns.begin(); - itr != fColumns.end(); - ++itr) - { - os << *itr << " "; - } + for (itr = fColumns.begin(); itr != fColumns.end(); ++itr) + { + os << *itr << " "; + } - os << ")"; + os << ")"; - os << " ftable=" << *fTableName; + os << " ftable=" << *fTableName; - os << " "; + os << " "; - os << "fcols ("; + os << "fcols ("; - for (itr = fForeignColumns.begin(); - itr != fForeignColumns.end(); - ++itr) - { - os << *itr << " "; - } + for (itr = fForeignColumns.begin(); itr != fForeignColumns.end(); ++itr) + { + os << *itr << " "; + } - os << ")"; + os << ")"; - return os; + return os; } std::ostream& operator<<(std::ostream& os, const ColumnNameList& columnNames) { - ColumnNameList::const_iterator itr; - os << '('; + ColumnNameList::const_iterator itr; + os << '('; - for (itr = columnNames.begin(); - itr != columnNames.end(); - ++itr) - { - os << *itr << " "; - } + for (itr = columnNames.begin(); itr != columnNames.end(); ++itr) + { + os << *itr << " "; + } - os << ')'; - return os; + os << ')'; + return os; } - TableReferencesConstraintDef::~TableReferencesConstraintDef() { - delete fTableName; - delete fRefAction; -} - + delete fTableName; + delete fRefAction; } +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/tdriver.cpp b/dbcon/ddlpackage/tdriver.cpp index 913f3278c..8b41151aa 100644 --- a/dbcon/ddlpackage/tdriver.cpp +++ b/dbcon/ddlpackage/tdriver.cpp @@ -46,883 +46,857 @@ std::string itoa(const int i); bool parse_file(char* fileName) { - SqlFileParser parser; - parser.Parse(fileName); - return parser.Good(); + SqlFileParser parser; + parser.Parse(fileName); + return parser.Good(); } class ParserTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(ParserTest); - CPPUNIT_TEST(atac01); - CPPUNIT_TEST(atac05); - CPPUNIT_TEST(atac09); - CPPUNIT_TEST(atmcdd01); - CPPUNIT_TEST(atrt01); - CPPUNIT_TEST(ct02); - CPPUNIT_TEST(ct06); - CPPUNIT_TEST(ct10); - CPPUNIT_TEST(dt02); - CPPUNIT_TEST(atac02); - CPPUNIT_TEST(atac06); - CPPUNIT_TEST(atatc01); - CPPUNIT_TEST(atmcsd01); - CPPUNIT_TEST(ci01); - CPPUNIT_TEST(ct03); - CPPUNIT_TEST(ct07); - CPPUNIT_TEST(ct11); - CPPUNIT_TEST(empty_stmt); - CPPUNIT_TEST(atac03); - CPPUNIT_TEST(atac07); - CPPUNIT_TEST(atdc01); - CPPUNIT_TEST(atdc02); - CPPUNIT_TEST(atdc03); - CPPUNIT_TEST(atmct01); - CPPUNIT_TEST(atmct02); - CPPUNIT_TEST(comment); - CPPUNIT_TEST(ct04); - CPPUNIT_TEST(ct08); - CPPUNIT_TEST(di01); - CPPUNIT_TEST(atac04); - CPPUNIT_TEST(atac08); - CPPUNIT_TEST(atdtc01); - CPPUNIT_TEST(atdtc02); - CPPUNIT_TEST(atrc01); - CPPUNIT_TEST(ct01); - CPPUNIT_TEST(ct05); - CPPUNIT_TEST(ct09); - CPPUNIT_TEST(dt01); - CPPUNIT_TEST(foo); + CPPUNIT_TEST_SUITE(ParserTest); + CPPUNIT_TEST(atac01); + CPPUNIT_TEST(atac05); + CPPUNIT_TEST(atac09); + CPPUNIT_TEST(atmcdd01); + CPPUNIT_TEST(atrt01); + CPPUNIT_TEST(ct02); + CPPUNIT_TEST(ct06); + CPPUNIT_TEST(ct10); + CPPUNIT_TEST(dt02); + CPPUNIT_TEST(atac02); + CPPUNIT_TEST(atac06); + CPPUNIT_TEST(atatc01); + CPPUNIT_TEST(atmcsd01); + CPPUNIT_TEST(ci01); + CPPUNIT_TEST(ct03); + CPPUNIT_TEST(ct07); + CPPUNIT_TEST(ct11); + CPPUNIT_TEST(empty_stmt); + CPPUNIT_TEST(atac03); + CPPUNIT_TEST(atac07); + CPPUNIT_TEST(atdc01); + CPPUNIT_TEST(atdc02); + CPPUNIT_TEST(atdc03); + CPPUNIT_TEST(atmct01); + CPPUNIT_TEST(atmct02); + CPPUNIT_TEST(comment); + CPPUNIT_TEST(ct04); + CPPUNIT_TEST(ct08); + CPPUNIT_TEST(di01); + CPPUNIT_TEST(atac04); + CPPUNIT_TEST(atac08); + CPPUNIT_TEST(atdtc01); + CPPUNIT_TEST(atdtc02); + CPPUNIT_TEST(atrc01); + CPPUNIT_TEST(ct01); + CPPUNIT_TEST(ct05); + CPPUNIT_TEST(ct09); + CPPUNIT_TEST(dt01); + CPPUNIT_TEST(foo); - CPPUNIT_TEST_SUITE_END(); -private: + CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } + void tearDown() + { + } -public: - void setUp() - { - } - void tearDown() - { - } + void foo() + { + SqlParser p; + p.Parse("drop table foo"); + cout << p.GetParseTree(); + } - void foo() - { - SqlParser p; - p.Parse("drop table foo"); - cout << p.GetParseTree(); - } - - void atac01() - { - CPPUNIT_ASSERT(parse_file("sql/atac01.sql")); - } - void atac05() - { - CPPUNIT_ASSERT(parse_file("sql/atac05.sql")); - } - void atac09() - { - CPPUNIT_ASSERT(parse_file("sql/atac09.sql")); - } - void atmcdd01() - { - CPPUNIT_ASSERT(parse_file("sql/atmcdd01.sql")); - } - void atrt01() - { - CPPUNIT_ASSERT(parse_file("sql/atrt01.sql")); - } - void ct02() - { - CPPUNIT_ASSERT(parse_file("sql/ct02.sql")); - } - void ct06() - { - CPPUNIT_ASSERT(parse_file("sql/ct06.sql")); - } - void ct10() - { - CPPUNIT_ASSERT(parse_file("sql/ct10.sql")); - } - void dt02() - { - CPPUNIT_ASSERT(parse_file("sql/dt02.sql")); - } - void atac02() - { - CPPUNIT_ASSERT(parse_file("sql/atac02.sql")); - } - void atac06() - { - CPPUNIT_ASSERT(parse_file("sql/atac06.sql")); - } - void atatc01() - { - CPPUNIT_ASSERT(parse_file("sql/atatc01.sql")); - } - void atmcsd01() - { - CPPUNIT_ASSERT(parse_file("sql/atmcsd01.sql")); - } - void ci01() - { - CPPUNIT_ASSERT(parse_file("sql/ci01.sql")); - } - void ct03() - { - CPPUNIT_ASSERT(parse_file("sql/ct03.sql")); - } - void ct07() - { - CPPUNIT_ASSERT(parse_file("sql/ct07.sql")); - } - void ct11() - { - CPPUNIT_ASSERT(parse_file("sql/ct11.sql")); - } - void empty_stmt() - { - CPPUNIT_ASSERT(parse_file("sql/empty-stmt.sql")); - } - void atac03() - { - CPPUNIT_ASSERT(parse_file("sql/atac03.sql")); - } - void atac07() - { - CPPUNIT_ASSERT(parse_file("sql/atac07.sql")); - } - void atdc01() - { - CPPUNIT_ASSERT(parse_file("sql/atdc01.sql")); - } - void atdc02() - { - CPPUNIT_ASSERT(parse_file("sql/atdc02.sql")); - } - void atdc03() - { - CPPUNIT_ASSERT(parse_file("sql/atdc03.sql")); - } - void atmct01() - { - CPPUNIT_ASSERT(parse_file("sql/atmct01.sql")); - } - void atmct02() - { - CPPUNIT_ASSERT(parse_file("sql/atmct02.sql")); - } - void comment() - { - CPPUNIT_ASSERT(parse_file("sql/comment.sql")); - } - void ct04() - { - CPPUNIT_ASSERT(parse_file("sql/ct04.sql")); - } - void ct08() - { - CPPUNIT_ASSERT(parse_file("sql/ct08.sql")); - } - void di01() - { - CPPUNIT_ASSERT(parse_file("sql/di01.sql")); - } - void atac04() - { - CPPUNIT_ASSERT(parse_file("sql/atac04.sql")); - } - void atac08() - { - CPPUNIT_ASSERT(parse_file("sql/atac08.sql")); - } - void atdtc01() - { - CPPUNIT_ASSERT(parse_file("sql/atdtc01.sql")); - } - void atdtc02() - { - CPPUNIT_ASSERT(parse_file("sql/atdtc02.sql")); - } - void atrc01() - { - CPPUNIT_ASSERT(parse_file("sql/atrc01.sql")); - } - void ct01() - { - CPPUNIT_ASSERT(parse_file("sql/ct01.sql")); - } - void ct05() - { - CPPUNIT_ASSERT(parse_file("sql/ct05.sql")); - } - void ct09() - { - CPPUNIT_ASSERT(parse_file("sql/ct09.sql")); - } - void dt01() - { - CPPUNIT_ASSERT(parse_file("sql/dt01.sql")); - } + void atac01() + { + CPPUNIT_ASSERT(parse_file("sql/atac01.sql")); + } + void atac05() + { + CPPUNIT_ASSERT(parse_file("sql/atac05.sql")); + } + void atac09() + { + CPPUNIT_ASSERT(parse_file("sql/atac09.sql")); + } + void atmcdd01() + { + CPPUNIT_ASSERT(parse_file("sql/atmcdd01.sql")); + } + void atrt01() + { + CPPUNIT_ASSERT(parse_file("sql/atrt01.sql")); + } + void ct02() + { + CPPUNIT_ASSERT(parse_file("sql/ct02.sql")); + } + void ct06() + { + CPPUNIT_ASSERT(parse_file("sql/ct06.sql")); + } + void ct10() + { + CPPUNIT_ASSERT(parse_file("sql/ct10.sql")); + } + void dt02() + { + CPPUNIT_ASSERT(parse_file("sql/dt02.sql")); + } + void atac02() + { + CPPUNIT_ASSERT(parse_file("sql/atac02.sql")); + } + void atac06() + { + CPPUNIT_ASSERT(parse_file("sql/atac06.sql")); + } + void atatc01() + { + CPPUNIT_ASSERT(parse_file("sql/atatc01.sql")); + } + void atmcsd01() + { + CPPUNIT_ASSERT(parse_file("sql/atmcsd01.sql")); + } + void ci01() + { + CPPUNIT_ASSERT(parse_file("sql/ci01.sql")); + } + void ct03() + { + CPPUNIT_ASSERT(parse_file("sql/ct03.sql")); + } + void ct07() + { + CPPUNIT_ASSERT(parse_file("sql/ct07.sql")); + } + void ct11() + { + CPPUNIT_ASSERT(parse_file("sql/ct11.sql")); + } + void empty_stmt() + { + CPPUNIT_ASSERT(parse_file("sql/empty-stmt.sql")); + } + void atac03() + { + CPPUNIT_ASSERT(parse_file("sql/atac03.sql")); + } + void atac07() + { + CPPUNIT_ASSERT(parse_file("sql/atac07.sql")); + } + void atdc01() + { + CPPUNIT_ASSERT(parse_file("sql/atdc01.sql")); + } + void atdc02() + { + CPPUNIT_ASSERT(parse_file("sql/atdc02.sql")); + } + void atdc03() + { + CPPUNIT_ASSERT(parse_file("sql/atdc03.sql")); + } + void atmct01() + { + CPPUNIT_ASSERT(parse_file("sql/atmct01.sql")); + } + void atmct02() + { + CPPUNIT_ASSERT(parse_file("sql/atmct02.sql")); + } + void comment() + { + CPPUNIT_ASSERT(parse_file("sql/comment.sql")); + } + void ct04() + { + CPPUNIT_ASSERT(parse_file("sql/ct04.sql")); + } + void ct08() + { + CPPUNIT_ASSERT(parse_file("sql/ct08.sql")); + } + void di01() + { + CPPUNIT_ASSERT(parse_file("sql/di01.sql")); + } + void atac04() + { + CPPUNIT_ASSERT(parse_file("sql/atac04.sql")); + } + void atac08() + { + CPPUNIT_ASSERT(parse_file("sql/atac08.sql")); + } + void atdtc01() + { + CPPUNIT_ASSERT(parse_file("sql/atdtc01.sql")); + } + void atdtc02() + { + CPPUNIT_ASSERT(parse_file("sql/atdtc02.sql")); + } + void atrc01() + { + CPPUNIT_ASSERT(parse_file("sql/atrc01.sql")); + } + void ct01() + { + CPPUNIT_ASSERT(parse_file("sql/ct01.sql")); + } + void ct05() + { + CPPUNIT_ASSERT(parse_file("sql/ct05.sql")); + } + void ct09() + { + CPPUNIT_ASSERT(parse_file("sql/ct09.sql")); + } + void dt01() + { + CPPUNIT_ASSERT(parse_file("sql/dt01.sql")); + } }; - -template +template void u_sertest(T* x) { - ByteStream bs; - stringstream s1, s2; - auto_ptr y(new T); + ByteStream bs; + stringstream s1, s2; + auto_ptr y(new T); - x->serialize(bs); - y->unserialize(bs); + x->serialize(bs); + y->unserialize(bs); - s1 << *x; - s2 << *y; + s1 << *x; + s2 << *y; - cout << "String Compare" << endl; - cout << "------------------" << endl; - cout << s1.str() << endl - << s2.str() << endl; - cout << "------------------" << endl; + cout << "String Compare" << endl; + cout << "------------------" << endl; + cout << s1.str() << endl << s2.str() << endl; + cout << "------------------" << endl; - CPPUNIT_ASSERT(s1.str() == s2.str()); + CPPUNIT_ASSERT(s1.str() == s2.str()); } - /** @brief Just like u_sertest except that a typecode is pulled off - the ByteStream before the unserialize. */ -template + the ByteStream before the unserialize. */ +template void t_sertest(T* x) { - ByteStream bs; - stringstream s1, s2; - auto_ptr y(new T); + ByteStream bs; + stringstream s1, s2; + auto_ptr y(new T); - x->serialize(bs); + x->serialize(bs); - quadbyte type; - bs >> type; + quadbyte type; + bs >> type; - y->unserialize(bs); + y->unserialize(bs); - s1 << *x; - s2 << *y; + s1 << *x; + s2 << *y; - cout << "String Compare" << endl; - cout << "------------------" << endl; - cout << s1.str() << endl - << s2.str() << endl; - cout << "------------------" << endl; + cout << "String Compare" << endl; + cout << "------------------" << endl; + cout << s1.str() << endl << s2.str() << endl; + cout << "------------------" << endl; - CPPUNIT_ASSERT(s1.str() == s2.str()); + CPPUNIT_ASSERT(s1.str() == s2.str()); } - class SerializeTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(SerializeTest); + CPPUNIT_TEST(qname); + CPPUNIT_TEST(columntype); + CPPUNIT_TEST(columndefaultvalue); + CPPUNIT_TEST(columnconstraintdef); + CPPUNIT_TEST(columndef_01); + CPPUNIT_TEST(referentialaction); + CPPUNIT_TEST(tablecheckconstraint); + CPPUNIT_TEST(tableuniqueconstraint); + CPPUNIT_TEST(tableprimarykeyconstraint); + CPPUNIT_TEST(tablereferencesconstraint); + CPPUNIT_TEST(tabledef); - CPPUNIT_TEST_SUITE(SerializeTest); - CPPUNIT_TEST(qname); - CPPUNIT_TEST(columntype); - CPPUNIT_TEST(columndefaultvalue); - CPPUNIT_TEST(columnconstraintdef); - CPPUNIT_TEST(columndef_01); - CPPUNIT_TEST(referentialaction); - CPPUNIT_TEST(tablecheckconstraint); - CPPUNIT_TEST(tableuniqueconstraint); - CPPUNIT_TEST(tableprimarykeyconstraint); - CPPUNIT_TEST(tablereferencesconstraint); - CPPUNIT_TEST(tabledef); + CPPUNIT_TEST(createtable); + CPPUNIT_TEST(droptable); + CPPUNIT_TEST(createindex); + CPPUNIT_TEST(dropindex); - CPPUNIT_TEST(createtable); - CPPUNIT_TEST(droptable); - CPPUNIT_TEST(createindex); - CPPUNIT_TEST(dropindex); + CPPUNIT_TEST(altertableaddcolumn); + CPPUNIT_TEST(altertableaddcolumns); + CPPUNIT_TEST(altertabledropcolumns); + CPPUNIT_TEST(altertableaddtableconstraint); + CPPUNIT_TEST(altertabledropcolumn); - CPPUNIT_TEST(altertableaddcolumn); - CPPUNIT_TEST(altertableaddcolumns); - CPPUNIT_TEST(altertabledropcolumns); - CPPUNIT_TEST(altertableaddtableconstraint); - CPPUNIT_TEST(altertabledropcolumn); + CPPUNIT_TEST(altertablesetcolumndefault); + CPPUNIT_TEST(altertabledropcolumndefault); + CPPUNIT_TEST(altertabledroptableconstraint); - CPPUNIT_TEST(altertablesetcolumndefault); - CPPUNIT_TEST(altertabledropcolumndefault); - CPPUNIT_TEST(altertabledroptableconstraint); + CPPUNIT_TEST(altertablerenametable); - CPPUNIT_TEST(altertablerenametable); + CPPUNIT_TEST(altertablemodifycolumntype); - CPPUNIT_TEST(altertablemodifycolumntype); + CPPUNIT_TEST(altertablerenamecolumn); - CPPUNIT_TEST(altertablerenamecolumn); + CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } - CPPUNIT_TEST_SUITE_END(); + void tearDown() + { + } -private: + void altertablerenamecolumn() + { + SqlFileParser p; + p.Parse("sql/atrc01.sql"); - -public: - void setUp() + if (p.Good()) { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); } + } + void altertablemodifycolumntype() + { + SqlFileParser p; + p.Parse("sql/atmct01.sql"); - void tearDown() + if (p.Good()) { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); } + } - void altertablerenamecolumn() + void altertablerenametable() + { + SqlFileParser p; + p.Parse("sql/atrt01.sql"); + + if (p.Good()) { - SqlFileParser p; - p.Parse("sql/atrc01.sql"); + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); + } + } - if (p.Good()) + void altertabledroptableconstraint() + { + SqlFileParser p; + p.Parse("sql/atdtc01.sql"); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); + } + } + + void altertabledropcolumndefault() + { + SqlFileParser p; + p.Parse("sql/atmcdd01.sql"); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); + } + } + + void altertablesetcolumndefault() + { + SqlFileParser p; + p.Parse("sql/atmcsd01.sql"); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); + } + } + + void altertabledropcolumn() + { + SqlFileParser p; + p.Parse("sql/atdc01.sql"); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); + } + } + + void altertableaddtableconstraint() + { + SqlFileParser p; + p.Parse("sql/atatc01.sql"); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + AlterTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + t_sertest(stmt); + } + } + + void altertableaddcolumn() + { + cout << "Serialize test: AtaAddColumn" << endl; + auto_ptr ata(new AtaAddColumn); + + vector files; + files.push_back("sql/ct01.sql"); + files.push_back("sql/ct02.sql"); + files.push_back("sql/ct03.sql"); + files.push_back("sql/ct04.sql"); + files.push_back("sql/ct05.sql"); + files.push_back("sql/ct06.sql"); + files.push_back("sql/ct07.sql"); + files.push_back("sql/ct08.sql"); + files.push_back("sql/ct09.sql"); + files.push_back("sql/ct10.sql"); + files.push_back("sql/ct11.sql"); + + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) + { + SqlFileParser p; + p.Parse(*itr); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + CreateTableStatement* ct = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(ct); + + ColumnDefList* columns = &(ct->fTableDef->fColumns); + + ColumnDefList::const_iterator itr; + + for (itr = columns->begin(); itr != columns->end(); ++itr) { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); + ata->fColumnDef = *itr; + t_sertest(ata.get()); + // We borrowed the column def from the parse + // tree. Null it to avoid double free. + ata->fColumnDef = 0; } + } } + } + void altertableaddcolumns() + { + cout << "Serialize test: AtaAddColumnS" << endl; + auto_ptr ata(new AtaAddColumns); - void altertablemodifycolumntype() + vector files; + files.push_back("sql/ct01.sql"); + files.push_back("sql/ct02.sql"); + files.push_back("sql/ct03.sql"); + files.push_back("sql/ct04.sql"); + files.push_back("sql/ct05.sql"); + files.push_back("sql/ct06.sql"); + files.push_back("sql/ct07.sql"); + files.push_back("sql/ct08.sql"); + files.push_back("sql/ct09.sql"); + files.push_back("sql/ct10.sql"); + files.push_back("sql/ct11.sql"); + + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) { - SqlFileParser p; - p.Parse("sql/atmct01.sql"); + SqlFileParser p; + p.Parse(*itr); - if (p.Good()) + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + CreateTableStatement* ct = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(ct); + + ata->fColumns = ct->fTableDef->fColumns; + t_sertest(ata.get()); + // We borrowed the column defs from the parse + // tree. Null it to avoid double free. + ata->fColumns.clear(); + } + } + } + + void altertabledropcolumns() + { + ColumnNameList* names = new ColumnNameList; + names->push_back("c1"); + names->push_back("c2"); + auto_ptr stmt(new AtaDropColumns(names)); + t_sertest(stmt.get()); + } + + void qname() + { + cout << "Serialize test: QualifiedName" << endl; + auto_ptr name(new QualifiedName("one", "two", "three")); + u_sertest(name.get()); + } + + void columntype() + { + cout << "Serialize test: ColumnType" << endl; + + auto_ptr type(new ColumnType); + type->fType = 1; + type->fLength = 2; + type->fPrecision = 3; + type->fScale = 4; + type->fWithTimezone = true; + u_sertest(type.get()); + } + + void columndefaultvalue() + { + cout << "Serialize test: ColumnDefaultValue" << endl; + auto_ptr dval(new ColumnDefaultValue); + dval->fNull = false; + dval->fValue = "1.234"; + u_sertest(dval.get()); + } + + void columnconstraintdef() + { + cout << "Serialize test: ColumnConstraintDef" << endl; + auto_ptr con(new ColumnConstraintDef); + con->fDeferrable = true; + con->fCheckTime = DDL_INITIALLY_DEFERRED; + con->fConstraintType = DDL_CHECK; + con->fCheck = "thecheck"; + u_sertest(con.get()); + } + + /** @brief Can we serialize ColumDefs? + * + * Here we leverage the parser and the ostream operators for + * parser objects to test serialization for ColumnDefs. We use + * the parser to build parse trees for some create table + * statements. Then we push the ColumnDefs through a ByteStream + * and compare the results as strings. In other words, if C is a + * ColumnDef in the parser constructed tree, we write C to the + * ByteStream and then unserialize it back out making a new ColumnDef, + * C'. We then format C and C' to respective stringstreams and + * then compare the strings. If they are equal, chances are good + * the serialization is OK. + */ + + void columndef_01() + { + cout << "columndef_01" << endl; + + vector files; + files.push_back("sql/ct01.sql"); + files.push_back("sql/ct02.sql"); + files.push_back("sql/ct03.sql"); + files.push_back("sql/ct04.sql"); + files.push_back("sql/ct05.sql"); + files.push_back("sql/ct06.sql"); + files.push_back("sql/ct07.sql"); + files.push_back("sql/ct08.sql"); + files.push_back("sql/ct09.sql"); + files.push_back("sql/ct10.sql"); + files.push_back("sql/ct11.sql"); + + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) + { + cout << "* * * Checking columndef serialization for " << *itr << endl; + SqlFileParser p; + p.Parse(*itr); + + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + CreateTableStatement* ct = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(ct); + + cout << "Parsed CreateTable:" << endl; + cout << *ct << endl; + + ColumnDefList* columns = &(ct->fTableDef->fColumns); + + ColumnDefList::const_iterator itr; + + for (itr = columns->begin(); itr != columns->end(); ++itr) { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); + u_sertest(*itr); } + } } + } + void referentialaction() + { + cout << "Serialize test: ReferentialAction" << endl; + auto_ptr ref(new ReferentialAction); + ref->fOnUpdate = DDL_CASCADE; + ref->fOnDelete = DDL_SET_NULL; + u_sertest(ref.get()); + } - void altertablerenametable() + void tablecheckconstraint() + { + cout << "Serialize test: TableCheckConstraintDef" << endl; + auto_ptr tc(new TableCheckConstraintDef); + tc->fName = "fooby"; + tc->fCheck = "check constraint text"; + u_sertest(tc.get()); + } + + void tableuniqueconstraint() + { + cout << "Serialize test: TableUniqueConstraintDef" << endl; + auto_ptr tc(new TableUniqueConstraintDef); + tc->fName = "fooby"; + tc->fColumnNameList.push_back("C1"); + tc->fColumnNameList.push_back("C2"); + tc->fColumnNameList.push_back("C3"); + u_sertest(tc.get()); + } + + void tableprimarykeyconstraint() + { + cout << "Serialize test: TablePrimaryKeyConstraintDef" << endl; + auto_ptr tc(new TablePrimaryKeyConstraintDef); + tc->fName = "fooby"; + tc->fColumnNameList.push_back("C1"); + tc->fColumnNameList.push_back("C2"); + tc->fColumnNameList.push_back("C3"); + u_sertest(tc.get()); + } + + void tablereferencesconstraint() + { + cout << "Serialize test: TableReferencesConstraintDef" << endl; + auto_ptr tc(new TableReferencesConstraintDef); + + tc->fName = "fooby"; + tc->fColumns.push_back("C1"); + tc->fColumns.push_back("C2"); + tc->fColumns.push_back("C3"); + + QualifiedName* qname = new QualifiedName; + qname->fCatalog = ""; + qname->fSchema = "calpont"; + qname->fName = "table_fooby"; + tc->fTableName = qname; + + tc->fForeignColumns.push_back("F1"); + tc->fForeignColumns.push_back("F2"); + tc->fForeignColumns.push_back("F3"); + + tc->fMatchType = DDL_FULL; + + ReferentialAction* ref = new ReferentialAction; + ref->fOnUpdate = DDL_CASCADE; + ref->fOnDelete = DDL_NO_ACTION; + tc->fRefAction = ref; + + u_sertest(tc.get()); + } + + void tabledef() + { + cout << "tabledef" << endl; + + vector files; + files.push_back("sql/ct01.sql"); + files.push_back("sql/ct02.sql"); + files.push_back("sql/ct03.sql"); + files.push_back("sql/ct04.sql"); + files.push_back("sql/ct05.sql"); + files.push_back("sql/ct06.sql"); + files.push_back("sql/ct07.sql"); + files.push_back("sql/ct08.sql"); + files.push_back("sql/ct09.sql"); + files.push_back("sql/ct10.sql"); + files.push_back("sql/ct11.sql"); + + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) { - SqlFileParser p; - p.Parse("sql/atrt01.sql"); + cout << "* * * Checking tabledef serialization for " << *itr << endl; + SqlFileParser p; + p.Parse(*itr); - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); - } + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + CreateTableStatement* ct = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(ct); + + cout << "Parsed CreateTable:" << endl; + cout << *ct << endl; + + u_sertest(ct->fTableDef); + } } + } - void altertabledroptableconstraint() + void createtable() + { + vector files; + files.push_back("sql/ct01.sql"); + files.push_back("sql/ct02.sql"); + files.push_back("sql/ct03.sql"); + files.push_back("sql/ct04.sql"); + files.push_back("sql/ct05.sql"); + files.push_back("sql/ct06.sql"); + files.push_back("sql/ct07.sql"); + files.push_back("sql/ct08.sql"); + files.push_back("sql/ct09.sql"); + files.push_back("sql/ct10.sql"); + files.push_back("sql/ct11.sql"); + + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) { - SqlFileParser p; - p.Parse("sql/atdtc01.sql"); + cout << "* * * Checking CreateTableStatement serialization for " << *itr << endl; + SqlFileParser p; + p.setDefaultSchema("tpch"); + p.Parse(*itr); - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); - } + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + CreateTableStatement* ct = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(ct); + + cout << "Parsed CreateTable:" << endl; + cout << *ct << endl; + + t_sertest(ct); + } } + } - void altertabledropcolumndefault() + void droptable() + { + vector files; + files.push_back("sql/dt01.sql"); + files.push_back("sql/dt02.sql"); + + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) { - SqlFileParser p; - p.Parse("sql/atmcdd01.sql"); + cout << "* * * Checking DropTableStatement serialization for " << *itr << endl; + SqlFileParser p; + p.Parse(*itr); - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); - } + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + DropTableStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + + cout << "Parsed DropTable:" << endl; + cout << *stmt << endl; + + t_sertest(stmt); + } } + } + void createindex() + { + vector files; + files.push_back("sql/ci01.sql"); + files.push_back("sql/ci02.sql"); - void altertablesetcolumndefault() + vector::const_iterator itr; + + for (itr = files.begin(); itr != files.end(); ++itr) { - SqlFileParser p; - p.Parse("sql/atmcsd01.sql"); + cout << "* * * Checking Create Index serialization for " << *itr << endl; + SqlFileParser p; + p.Parse(*itr); - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); - } + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + CreateIndexStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + + cout << "Parsed:" << endl; + cout << *stmt << endl; + + t_sertest(stmt); + } } + } + void dropindex() + { + vector files; + files.push_back("sql/di01.sql"); + vector::const_iterator itr; - void altertabledropcolumn() + for (itr = files.begin(); itr != files.end(); ++itr) { - SqlFileParser p; - p.Parse("sql/atdc01.sql"); + cout << "* * * Checking Drop Index serialization for " << *itr << endl; + SqlFileParser p; + p.Parse(*itr); - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); - } - } - - void altertableaddtableconstraint() - { - SqlFileParser p; - p.Parse("sql/atatc01.sql"); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - AlterTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - t_sertest(stmt); - } - } - - - void altertableaddcolumn() - { - cout << "Serialize test: AtaAddColumn" << endl; - auto_ptr ata(new AtaAddColumn); - - vector files; - files.push_back("sql/ct01.sql"); - files.push_back("sql/ct02.sql"); - files.push_back("sql/ct03.sql"); - files.push_back("sql/ct04.sql"); - files.push_back("sql/ct05.sql"); - files.push_back("sql/ct06.sql"); - files.push_back("sql/ct07.sql"); - files.push_back("sql/ct08.sql"); - files.push_back("sql/ct09.sql"); - files.push_back("sql/ct10.sql"); - files.push_back("sql/ct11.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - CreateTableStatement* ct = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(ct); - - ColumnDefList* columns = &(ct->fTableDef->fColumns); - - ColumnDefList::const_iterator itr; - - for (itr = columns->begin(); - itr != columns->end(); - ++itr) - { - ata->fColumnDef = *itr; - t_sertest(ata.get()); - // We borrowed the column def from the parse - // tree. Null it to avoid double free. - ata->fColumnDef = 0; - } - } - } - } - - void altertableaddcolumns() - { - cout << "Serialize test: AtaAddColumnS" << endl; - auto_ptr ata(new AtaAddColumns); - - vector files; - files.push_back("sql/ct01.sql"); - files.push_back("sql/ct02.sql"); - files.push_back("sql/ct03.sql"); - files.push_back("sql/ct04.sql"); - files.push_back("sql/ct05.sql"); - files.push_back("sql/ct06.sql"); - files.push_back("sql/ct07.sql"); - files.push_back("sql/ct08.sql"); - files.push_back("sql/ct09.sql"); - files.push_back("sql/ct10.sql"); - files.push_back("sql/ct11.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - CreateTableStatement* ct = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(ct); - - ata->fColumns = ct->fTableDef->fColumns; - t_sertest(ata.get()); - // We borrowed the column defs from the parse - // tree. Null it to avoid double free. - ata->fColumns.clear(); - } - } - } - - - void altertabledropcolumns() - { - ColumnNameList* names = new ColumnNameList; - names->push_back("c1"); - names->push_back("c2"); - auto_ptr stmt(new AtaDropColumns(names)); - t_sertest(stmt.get()); - } - - - void qname() - { - cout << "Serialize test: QualifiedName" << endl; - auto_ptr name(new QualifiedName("one", "two", "three")); - u_sertest(name.get()); - } - - - void columntype() - { - cout << "Serialize test: ColumnType" << endl; - - auto_ptr type(new ColumnType); - type->fType = 1; - type->fLength = 2; - type->fPrecision = 3; - type->fScale = 4; - type->fWithTimezone = true; - u_sertest(type.get()); - } - - void columndefaultvalue() - { - cout << "Serialize test: ColumnDefaultValue" << endl; - auto_ptr dval(new ColumnDefaultValue); - dval->fNull = false; - dval->fValue = "1.234"; - u_sertest(dval.get()); - } - - void columnconstraintdef() - { - cout << "Serialize test: ColumnConstraintDef" << endl; - auto_ptr con(new ColumnConstraintDef); - con->fDeferrable = true; - con->fCheckTime = DDL_INITIALLY_DEFERRED; - con->fConstraintType = DDL_CHECK; - con->fCheck = "thecheck"; - u_sertest(con.get()); - } - - /** @brief Can we serialize ColumDefs? - * - * Here we leverage the parser and the ostream operators for - * parser objects to test serialization for ColumnDefs. We use - * the parser to build parse trees for some create table - * statements. Then we push the ColumnDefs through a ByteStream - * and compare the results as strings. In other words, if C is a - * ColumnDef in the parser constructed tree, we write C to the - * ByteStream and then unserialize it back out making a new ColumnDef, - * C'. We then format C and C' to respective stringstreams and - * then compare the strings. If they are equal, chances are good - * the serialization is OK. - */ - - void columndef_01() - { - cout << "columndef_01" << endl; - - vector files; - files.push_back("sql/ct01.sql"); - files.push_back("sql/ct02.sql"); - files.push_back("sql/ct03.sql"); - files.push_back("sql/ct04.sql"); - files.push_back("sql/ct05.sql"); - files.push_back("sql/ct06.sql"); - files.push_back("sql/ct07.sql"); - files.push_back("sql/ct08.sql"); - files.push_back("sql/ct09.sql"); - files.push_back("sql/ct10.sql"); - files.push_back("sql/ct11.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - cout << "* * * Checking columndef serialization for " << *itr << endl; - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - CreateTableStatement* ct = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(ct); - - cout << "Parsed CreateTable:" << endl; - cout << *ct << endl; - - ColumnDefList* columns = &(ct->fTableDef->fColumns); - - ColumnDefList::const_iterator itr; - - for (itr = columns->begin(); - itr != columns->end(); - ++itr) - { - u_sertest(*itr); - } - } - } - - } - - void referentialaction() - { - cout << "Serialize test: ReferentialAction" << endl; - auto_ptr ref(new ReferentialAction); - ref->fOnUpdate = DDL_CASCADE; - ref->fOnDelete = DDL_SET_NULL; - u_sertest(ref.get()); - } - - void tablecheckconstraint() - { - cout << "Serialize test: TableCheckConstraintDef" << endl; - auto_ptr tc(new TableCheckConstraintDef); - tc->fName = "fooby"; - tc->fCheck = "check constraint text"; - u_sertest(tc.get()); - } - - void tableuniqueconstraint() - { - cout << "Serialize test: TableUniqueConstraintDef" << endl; - auto_ptr tc(new TableUniqueConstraintDef); - tc->fName = "fooby"; - tc->fColumnNameList.push_back("C1"); - tc->fColumnNameList.push_back("C2"); - tc->fColumnNameList.push_back("C3"); - u_sertest(tc.get()); - } - - void tableprimarykeyconstraint() - { - cout << "Serialize test: TablePrimaryKeyConstraintDef" << endl; - auto_ptr tc(new TablePrimaryKeyConstraintDef); - tc->fName = "fooby"; - tc->fColumnNameList.push_back("C1"); - tc->fColumnNameList.push_back("C2"); - tc->fColumnNameList.push_back("C3"); - u_sertest(tc.get()); - } - - void tablereferencesconstraint() - { - cout << "Serialize test: TableReferencesConstraintDef" << endl; - auto_ptr tc(new TableReferencesConstraintDef); - - tc->fName = "fooby"; - tc->fColumns.push_back("C1"); - tc->fColumns.push_back("C2"); - tc->fColumns.push_back("C3"); - - QualifiedName* qname = new QualifiedName; - qname->fCatalog = ""; - qname->fSchema = "calpont"; - qname->fName = "table_fooby"; - tc->fTableName = qname; - - - tc->fForeignColumns.push_back("F1"); - tc->fForeignColumns.push_back("F2"); - tc->fForeignColumns.push_back("F3"); - - tc->fMatchType = DDL_FULL; - - ReferentialAction* ref = new ReferentialAction; - ref->fOnUpdate = DDL_CASCADE; - ref->fOnDelete = DDL_NO_ACTION; - tc->fRefAction = ref; - - u_sertest(tc.get()); - } - - void tabledef() - { - cout << "tabledef" << endl; - - vector files; - files.push_back("sql/ct01.sql"); - files.push_back("sql/ct02.sql"); - files.push_back("sql/ct03.sql"); - files.push_back("sql/ct04.sql"); - files.push_back("sql/ct05.sql"); - files.push_back("sql/ct06.sql"); - files.push_back("sql/ct07.sql"); - files.push_back("sql/ct08.sql"); - files.push_back("sql/ct09.sql"); - files.push_back("sql/ct10.sql"); - files.push_back("sql/ct11.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - cout << "* * * Checking tabledef serialization for " << *itr << endl; - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - CreateTableStatement* ct = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(ct); - - cout << "Parsed CreateTable:" << endl; - cout << *ct << endl; - - u_sertest(ct->fTableDef); - } - } - - } - - - void createtable() - { - vector files; - files.push_back("sql/ct01.sql"); - files.push_back("sql/ct02.sql"); - files.push_back("sql/ct03.sql"); - files.push_back("sql/ct04.sql"); - files.push_back("sql/ct05.sql"); - files.push_back("sql/ct06.sql"); - files.push_back("sql/ct07.sql"); - files.push_back("sql/ct08.sql"); - files.push_back("sql/ct09.sql"); - files.push_back("sql/ct10.sql"); - files.push_back("sql/ct11.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - cout << "* * * Checking CreateTableStatement serialization for " << *itr << endl; - SqlFileParser p; - p.setDefaultSchema("tpch"); - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - CreateTableStatement* ct = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(ct); - - cout << "Parsed CreateTable:" << endl; - cout << *ct << endl; - - t_sertest(ct); - } - } - } - - void droptable() - { - vector files; - files.push_back("sql/dt01.sql"); - files.push_back("sql/dt02.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - cout << "* * * Checking DropTableStatement serialization for " << *itr << endl; - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - DropTableStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - - cout << "Parsed DropTable:" << endl; - cout << *stmt << endl; - - t_sertest(stmt); - } - } - } - void createindex() - { - vector files; - files.push_back("sql/ci01.sql"); - files.push_back("sql/ci02.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - cout << "* * * Checking Create Index serialization for " << *itr << endl; - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - CreateIndexStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - - cout << "Parsed:" << endl; - cout << *stmt << endl; - - t_sertest(stmt); - } - } - } - void dropindex() - { - vector files; - files.push_back("sql/di01.sql"); - - vector::const_iterator itr; - - for (itr = files.begin(); itr != files.end(); ++itr) - { - cout << "* * * Checking Drop Index serialization for " << *itr << endl; - SqlFileParser p; - p.Parse(*itr); - - if (p.Good()) - { - const ParseTree& stmts = p.GetParseTree(); - DropIndexStatement* stmt = dynamic_cast(stmts[0]); - CPPUNIT_ASSERT(stmt); - - cout << "Parsed:" << endl; - cout << *stmt << endl; - - t_sertest(stmt); - } - } + if (p.Good()) + { + const ParseTree& stmts = p.GetParseTree(); + DropIndexStatement* stmt = dynamic_cast(stmts[0]); + CPPUNIT_ASSERT(stmt); + + cout << "Parsed:" << endl; + cout << *stmt << endl; + + t_sertest(stmt); + } } + } }; CPPUNIT_TEST_SUITE_REGISTRATION(SerializeTest); @@ -933,21 +907,18 @@ CPPUNIT_TEST_SUITE_REGISTRATION(ParserTest); int main(int argc, char* argv[]) { + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - - bool wasSuccessful = runner.run( "", false ); - - return (wasSuccessful ? 0 : 1); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } string itoa(const int i) { - stringstream ss; - ss << i; - return ss.str(); + stringstream ss; + ss << i; + return ss.str(); } - diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index b08fc9181..0e321e063 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -62,440 +62,464 @@ using namespace cacheutils; #include "IDBPolicy.h" using namespace idbdatafile; - #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" - // for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to 'typeid' +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" +// for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to +// 'typeid' #endif - -//TODO: this should be in a common header somewhere +// TODO: this should be in a common header somewhere struct extentInfo { - uint16_t dbRoot; - uint32_t partition; - uint16_t segment; - bool operator==(const extentInfo& rhs) const - { - return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment); - } - bool operator!=(const extentInfo& rhs) const - { - return !(*this == rhs); - } + uint16_t dbRoot; + uint32_t partition; + uint16_t segment; + bool operator==(const extentInfo& rhs) const + { + return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment); + } + bool operator!=(const extentInfo& rhs) const + { + return !(*this == rhs); + } }; namespace { - bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType& newType) { - switch (colType.colDataType) - { - case (CalpontSystemCatalog::BIT): - if (newType.fType == DDL_BIT) return true; + switch (colType.colDataType) + { + case (CalpontSystemCatalog::BIT): + if (newType.fType == DDL_BIT) + return true; - break; + break; - case (CalpontSystemCatalog::TINYINT): - if (newType.fType == DDL_TINYINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::TINYINT): + if (newType.fType == DDL_TINYINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - //@Bug 5443 Not allow user change data type. - //Not sure this is possible... - //if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && - // colType.scale == newType.fScale) return true; - break; + //@Bug 5443 Not allow user change data type. + // Not sure this is possible... + // if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && + // colType.scale == newType.fScale) return true; + break; - case (CalpontSystemCatalog::UTINYINT): - if (newType.fType == DDL_UNSIGNED_TINYINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::UTINYINT): + if (newType.fType == DDL_UNSIGNED_TINYINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::CHAR): - if (newType.fType == DDL_CHAR && colType.colWidth == newType.fLength) return true; + case (CalpontSystemCatalog::CHAR): + if (newType.fType == DDL_CHAR && colType.colWidth == newType.fLength) + return true; - break; + break; - case (CalpontSystemCatalog::SMALLINT): - if (newType.fType == DDL_SMALLINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::SMALLINT): + if (newType.fType == DDL_SMALLINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - //if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && - // colType.scale == newType.fScale) return true; - break; + // if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && + // colType.scale == newType.fScale) return true; + break; - case (CalpontSystemCatalog::USMALLINT): - if (newType.fType == DDL_UNSIGNED_SMALLINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::USMALLINT): + if (newType.fType == DDL_UNSIGNED_SMALLINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::DECIMAL): - if ((newType.fType == DDL_DECIMAL || newType.fType == DDL_NUMERIC) && - colType.precision == newType.fPrecision && colType.scale == newType.fScale) - return true; + case (CalpontSystemCatalog::DECIMAL): + if ((newType.fType == DDL_DECIMAL || newType.fType == DDL_NUMERIC) && + colType.precision == newType.fPrecision && colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::UDECIMAL): - if ((newType.fType == DDL_UNSIGNED_DECIMAL || newType.fType == DDL_UNSIGNED_NUMERIC) && - colType.precision == newType.fPrecision && colType.scale == newType.fScale) - return true; + case (CalpontSystemCatalog::UDECIMAL): + if ((newType.fType == DDL_UNSIGNED_DECIMAL || newType.fType == DDL_UNSIGNED_NUMERIC) && + colType.precision == newType.fPrecision && colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::MEDINT): - if (newType.fType == DDL_MEDINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::MEDINT): + if (newType.fType == DDL_MEDINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - //@Bug 5443 Not allow user change data type. - //if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && - // colType.scale == newType.fScale) return true; - break; + //@Bug 5443 Not allow user change data type. + // if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && + // colType.scale == newType.fScale) return true; + break; - case (CalpontSystemCatalog::UMEDINT): - if (newType.fType == DDL_UNSIGNED_MEDINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::UMEDINT): + if (newType.fType == DDL_UNSIGNED_MEDINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::INT): - if (newType.fType == DDL_INT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::INT): + if (newType.fType == DDL_INT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - //if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && - // colType.scale == newType.fScale) return true; - break; + // if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && + // colType.scale == newType.fScale) return true; + break; - case (CalpontSystemCatalog::UINT): - if (newType.fType == DDL_UNSIGNED_INT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::UINT): + if (newType.fType == DDL_UNSIGNED_INT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::FLOAT): - if (newType.fType == DDL_FLOAT) return true; + case (CalpontSystemCatalog::FLOAT): + if (newType.fType == DDL_FLOAT) + return true; - break; + break; - case (CalpontSystemCatalog::UFLOAT): - if (newType.fType == DDL_UNSIGNED_FLOAT) return true; + case (CalpontSystemCatalog::UFLOAT): + if (newType.fType == DDL_UNSIGNED_FLOAT) + return true; - break; + break; - case (CalpontSystemCatalog::DATE): - if (newType.fType == DDL_DATE) return true; + case (CalpontSystemCatalog::DATE): + if (newType.fType == DDL_DATE) + return true; - break; + break; - case (CalpontSystemCatalog::BIGINT): - if (newType.fType == DDL_BIGINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::BIGINT): + if (newType.fType == DDL_BIGINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - //@Bug 5443 Not allow user change data type. - //decimal is mapped to bigint in syscat - //if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && - // colType.scale == newType.fScale) return true; - break; + //@Bug 5443 Not allow user change data type. + // decimal is mapped to bigint in syscat + // if (newType.fType == DDL_DECIMAL && colType.precision == newType.fPrecision && + // colType.scale == newType.fScale) return true; + break; - case (CalpontSystemCatalog::UBIGINT): - if (newType.fType == DDL_UNSIGNED_BIGINT && colType.precision == newType.fPrecision && - colType.scale == newType.fScale) return true; + case (CalpontSystemCatalog::UBIGINT): + if (newType.fType == DDL_UNSIGNED_BIGINT && colType.precision == newType.fPrecision && + colType.scale == newType.fScale) + return true; - break; + break; - case (CalpontSystemCatalog::DOUBLE): - if (newType.fType == DDL_DOUBLE) return true; + case (CalpontSystemCatalog::DOUBLE): + if (newType.fType == DDL_DOUBLE) + return true; - break; + break; - case (CalpontSystemCatalog::UDOUBLE): - if (newType.fType == DDL_UNSIGNED_DOUBLE) return true; + case (CalpontSystemCatalog::UDOUBLE): + if (newType.fType == DDL_UNSIGNED_DOUBLE) + return true; - break; + break; - case (CalpontSystemCatalog::DATETIME): - if (newType.fType == DDL_DATETIME) return true; + case (CalpontSystemCatalog::DATETIME): + if (newType.fType == DDL_DATETIME) + return true; - break; + break; - case (CalpontSystemCatalog::TIMESTAMP): - if (newType.fType == DDL_TIMESTAMP) return true; + case (CalpontSystemCatalog::TIMESTAMP): + if (newType.fType == DDL_TIMESTAMP) + return true; - break; + break; - case (CalpontSystemCatalog::TIME): - if (newType.fType == DDL_TIME) return true; + case (CalpontSystemCatalog::TIME): + if (newType.fType == DDL_TIME) + return true; - break; + break; - case (CalpontSystemCatalog::VARCHAR): - if (newType.fType == DDL_VARCHAR && colType.colWidth == newType.fLength) return true; + case (CalpontSystemCatalog::VARCHAR): + if (newType.fType == DDL_VARCHAR && colType.colWidth == newType.fLength) + return true; - break; + break; - case (CalpontSystemCatalog::VARBINARY): - if (newType.fType == DDL_VARBINARY && colType.colWidth == newType.fLength) return true; + case (CalpontSystemCatalog::VARBINARY): + if (newType.fType == DDL_VARBINARY && colType.colWidth == newType.fLength) + return true; - break; + break; - case (CalpontSystemCatalog::CLOB): - break; + case (CalpontSystemCatalog::CLOB): break; - case (CalpontSystemCatalog::BLOB): - if (newType.fType == DDL_BLOB && colType.colWidth == newType.fLength) return true; + case (CalpontSystemCatalog::BLOB): + if (newType.fType == DDL_BLOB && colType.colWidth == newType.fLength) + return true; - break; + break; - case (CalpontSystemCatalog::TEXT): - if (newType.fType == DDL_TEXT && colType.colWidth == newType.fLength) return true; + case (CalpontSystemCatalog::TEXT): + if (newType.fType == DDL_TEXT && colType.colWidth == newType.fLength) + return true; - break; + break; - default: - break; - } + default: break; + } - return false; + return false; } - bool comptypesAreCompat(int oldCtype, int newCtype) { - switch (oldCtype) - { - case 1: - case 2: - return (newCtype == 1 || newCtype == 2); + switch (oldCtype) + { + case 1: + case 2: return (newCtype == 1 || newCtype == 2); - default: - break; - } + default: break; + } - return (oldCtype == newCtype); + return (oldCtype == newCtype); } -} +} // namespace namespace ddlpackageprocessor { - -AlterTableProcessor::DDLResult AlterTableProcessor::processPackage(ddlpackage::AlterTableStatement& alterTableStmt) +AlterTableProcessor::DDLResult AlterTableProcessor::processPackage( + ddlpackage::AlterTableStatement& alterTableStmt) { - SUMMARY_INFO("AlterTableProcessor::processPackage"); + SUMMARY_INFO("AlterTableProcessor::processPackage"); - DDLResult result; - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - result.result = NO_ERROR; - std::string err; - uint64_t tableLockId = 0; - DETAIL_INFO(alterTableStmt); - int rc = 0; - rc = fDbrm->isReadWrite(); + DDLResult result; + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + result.result = NO_ERROR; + std::string err; + uint64_t tableLockId = 0; + DETAIL_INFO(alterTableStmt); + int rc = 0; + rc = fDbrm->isReadWrite(); - if (rc != 0 ) + if (rc != 0) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = ALTER_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + //@Bug 4538. Log the sql statement before grabbing tablelock + string stmt = alterTableStmt.fSql + "|" + (alterTableStmt.fTableName)->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, alterTableStmt.fSessionID, txnID.id); + + VERBOSE_INFO("Getting current txnID"); + OamCache* oamcache = OamCache::makeOamCache(); + std::vector moduleIds = oamcache->getModuleIds(); + uint64_t uniqueId = 0; + + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(9); + args.add(ex.what()); + message.format(args); + result.result = ALTER_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + catch (...) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = ALTER_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + fWEClient->addQueue(uniqueId); + + try + { + // check table lock + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(alterTableStmt.fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(alterTableStmt.fSessionID); + CalpontSystemCatalog::TableName tableName; + tableName.schema = (alterTableStmt.fTableName)->fSchema; + tableName.table = (alterTableStmt.fTableName)->fName; + execplan::CalpontSystemCatalog::ROPair roPair; + roPair = systemCatalogPtr->tableRID(tableName); + + uint32_t processID = ::getpid(); + int32_t txnid = txnID.id; + int32_t sessionId = alterTableStmt.fSessionID; + std::string processName("DDLProc"); + int i = 0; + + std::vector pms; + + for (unsigned i = 0; i < moduleIds.size(); i++) { - logging::Message::Args args; - logging::Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); - message.format(args); - result.result = ALTER_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; + pms.push_back((uint32_t)moduleIds[i]); } - //@Bug 4538. Log the sql statement before grabbing tablelock - string stmt = alterTableStmt.fSql + "|" + (alterTableStmt.fTableName)->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, alterTableStmt.fSessionID, txnID.id); - - VERBOSE_INFO("Getting current txnID"); - OamCache* oamcache = OamCache::makeOamCache(); - std::vector moduleIds = oamcache->getModuleIds(); - uint64_t uniqueId = 0; - - //Bug 5070. Added exception handling - try - { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(9); - args.add(ex.what()); - message.format(args); - result.result = ALTER_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - catch ( ... ) - { - logging::Message::Args args; - logging::Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = ALTER_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - fWEClient->addQueue(uniqueId); - try { - //check table lock - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(alterTableStmt.fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(alterTableStmt.fSessionID); - CalpontSystemCatalog::TableName tableName; - tableName.schema = (alterTableStmt.fTableName)->fSchema; - tableName.table = (alterTableStmt.fTableName)->fName; - execplan::CalpontSystemCatalog::ROPair roPair; - roPair = systemCatalogPtr->tableRID(tableName); + tableLockId = + fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } - uint32_t processID = ::getpid(); - int32_t txnid = txnID.id; - int32_t sessionId = alterTableStmt.fSessionID; - std::string processName("DDLProc"); - int i = 0; + if (tableLockId == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; - std::vector pms; + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - for (unsigned i = 0; i < moduleIds.size(); i++) - { - pms.push_back((uint32_t)moduleIds[i]); - } - - try - { - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if ( tableLockId == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { + for (; i < numTries; i++) + { #ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); + Sleep(rm_ts.tv_sec * 1000); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif - try - { - processID = ::getpid(); - txnid = txnID.id; - sessionId = alterTableStmt.fSessionID;; - processName = "DDLProc"; - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (tableLockId > 0) - break; - } - - if (i >= numTries) //error out - { - logging::Message::Args args; - string strOp("alter"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add(sessionId); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - } + try + { + processID = ::getpid(); + txnid = txnID.id; + sessionId = alterTableStmt.fSessionID; + ; + processName = "DDLProc"; + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, + BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); } - ddlpackage::AlterTableActionList actionList = alterTableStmt.fActions; - AlterTableActionList::const_iterator action_iterator = actionList.begin(); + if (tableLockId > 0) + break; + } - while (action_iterator != actionList.end()) + if (i >= numTries) // error out + { + logging::Message::Args args; + string strOp("alter"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add(sessionId); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + } + } + + ddlpackage::AlterTableActionList actionList = alterTableStmt.fActions; + AlterTableActionList::const_iterator action_iterator = actionList.begin(); + + while (action_iterator != actionList.end()) + { + std::string s(typeid(*(*action_iterator)).name()); + + if (s.find(AlterActionString[0]) != string::npos) + { + // bug 827:change AtaAddColumn to AtaAddColumns + // Add a column + // Bug 1192 + ddlpackage::ColumnDef* columnDefPtr = 0; + ddlpackage::AtaAddColumn* addColumnPtr = dynamic_cast(*action_iterator); + + if (addColumnPtr) { - std::string s(typeid(*(*action_iterator)).name()); + columnDefPtr = addColumnPtr->fColumnDef; + } + else + { + ddlpackage::AtaAddColumns& addColumns = *(dynamic_cast(*action_iterator)); + columnDefPtr = addColumns.fColumns[0]; + } - if (s.find(AlterActionString[0]) != string::npos) - { - //bug 827:change AtaAddColumn to AtaAddColumns - //Add a column - //Bug 1192 - ddlpackage::ColumnDef* columnDefPtr = 0; - ddlpackage::AtaAddColumn* addColumnPtr = dynamic_cast (*action_iterator); + addColumn(alterTableStmt.fSessionID, txnID.id, result, columnDefPtr, *(alterTableStmt.fTableName), + uniqueId); - if (addColumnPtr) - { - columnDefPtr = addColumnPtr->fColumnDef; - } - else - { - ddlpackage::AtaAddColumns& addColumns = *(dynamic_cast (*action_iterator)); - columnDefPtr = addColumns.fColumns[0]; - } + if (result.result != NO_ERROR) + { + err = "AlterTable: add column failed"; + throw std::runtime_error(err); + } + } + else if (s.find(AlterActionString[6]) != string::npos) + { + // Drop Column Default + dropColumnDefault(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), + *(alterTableStmt.fTableName), uniqueId); - addColumn (alterTableStmt.fSessionID, txnID.id, result, columnDefPtr, *(alterTableStmt.fTableName), uniqueId); - - if (result.result != NO_ERROR) - { - err = "AlterTable: add column failed"; - throw std::runtime_error(err); - } - - } - else if (s.find(AlterActionString[6]) != string::npos) - { - //Drop Column Default - dropColumnDefault (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); - - if (result.result != NO_ERROR) - { - err = "AlterTable: drop column default failed"; - throw std::runtime_error(err); - } - } - else if (s.find(AlterActionString[3]) != string::npos) - { - //Drop Columns - dropColumns (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); - } - else if (s.find(AlterActionString[2]) != string::npos) - { - //Drop a column - dropColumn (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); - - } + if (result.result != NO_ERROR) + { + err = "AlterTable: drop column default failed"; + throw std::runtime_error(err); + } + } + else if (s.find(AlterActionString[3]) != string::npos) + { + // Drop Columns + dropColumns(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), *(alterTableStmt.fTableName), + uniqueId); + } + else if (s.find(AlterActionString[2]) != string::npos) + { + // Drop a column + dropColumn(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), *(alterTableStmt.fTableName), uniqueId); + } #if 0 else if (s.find(AlterActionString[4]) != string::npos) @@ -506,12 +530,13 @@ AlterTableProcessor::DDLResult AlterTableProcessor::processPackage(ddlpackage::A } #endif - else if (s.find(AlterActionString[5]) != string::npos) - { - //Set Column Default - setColumnDefault (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); - - } + else if (s.find(AlterActionString[5]) != string::npos) + { + // Set Column Default + setColumnDefault(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), + *(alterTableStmt.fTableName), uniqueId); + } #if 0 else if (s.find(AlterActionString[7]) != string::npos) @@ -523,1255 +548,1257 @@ AlterTableProcessor::DDLResult AlterTableProcessor::processPackage(ddlpackage::A #endif - else if (s.find(AlterActionString[8]) != string::npos) - { - //Rename Table - renameTable (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); + else if (s.find(AlterActionString[8]) != string::npos) + { + // Rename Table + renameTable(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), *(alterTableStmt.fTableName), + uniqueId); + } - } + else if (s.find(AlterActionString[10]) != string::npos) + { + // Rename a Column + renameColumn(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), *(alterTableStmt.fTableName), + uniqueId); + } + else if (s.find(AlterActionString[11]) != string::npos) + { + // Table Comment + tableComment(alterTableStmt.fSessionID, txnID.id, result, + *(dynamic_cast(*action_iterator)), *(alterTableStmt.fTableName), + uniqueId); + } + else + { + throw std::runtime_error("Altertable: Error in the action type"); + } - else if (s.find(AlterActionString[10]) != string::npos) - { - //Rename a Column - renameColumn (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); - - } - else if (s.find(AlterActionString[11]) != string::npos) - { - // Table Comment - tableComment (alterTableStmt.fSessionID, txnID.id, result, *(dynamic_cast (*action_iterator)), *(alterTableStmt.fTableName), uniqueId); - } - else - { - throw std::runtime_error("Altertable: Error in the action type"); - - } - - ++action_iterator; - - } - - // Log the DDL statement. - logging::logDDL(alterTableStmt.fSessionID, txnID.id, alterTableStmt.fSql, alterTableStmt.fOwner); - - DETAIL_INFO("Commiting transaction"); - commitTransaction(uniqueId, txnID); - fSessionManager.committed(txnID); + ++action_iterator; } - catch (std::exception& ex) + + // Log the DDL statement. + logging::logDDL(alterTableStmt.fSessionID, txnID.id, alterTableStmt.fSql, alterTableStmt.fOwner); + + DETAIL_INFO("Commiting transaction"); + commitTransaction(uniqueId, txnID); + fSessionManager.committed(txnID); + } + catch (std::exception& ex) + { + rollBackAlter(ex.what(), txnID, alterTableStmt.fSessionID, result, uniqueId); + } + catch (...) + { + rollBackAlter("encountered unknown exception. ", txnID, alterTableStmt.fSessionID, result, uniqueId); + } + + // release table lock + try + { + (void)fDbrm->releaseTableLock(tableLockId); + // cout << "table lock " << tableLockId << " is released" << endl; + } + catch (std::exception&) + { + if (result.result == NO_ERROR) { - rollBackAlter(ex.what(), txnID, alterTableStmt.fSessionID, result, uniqueId); - } - catch (...) - { - rollBackAlter("encountered unknown exception. ", txnID, alterTableStmt.fSessionID, result, uniqueId); + logging::Message::Args args; + logging::Message message(1); + args.add("Table lock is not released due to "); + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + args.add(""); + args.add(""); + message.format(args); + result.result = ALTER_ERROR; + result.message = message; } + } - //release table lock - try - { - (void)fDbrm->releaseTableLock(tableLockId); - //cout << "table lock " << tableLockId << " is released" << endl; - } - catch (std::exception&) - { - if (result.result == NO_ERROR) - { - logging::Message::Args args; - logging::Message message(1); - args.add("Table lock is not released due to "); - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - args.add(""); - args.add(""); - message.format(args); - result.result = ALTER_ERROR; - result.message = message; - } - } - - fWEClient->removeQueue(uniqueId); - return result; - + fWEClient->removeQueue(uniqueId); + return result; } -void AlterTableProcessor::rollBackAlter(const string& error, BRM::TxnID txnID, - int sessionId, DDLResult& result, uint64_t uniqueId) +void AlterTableProcessor::rollBackAlter(const string& error, BRM::TxnID txnID, int sessionId, + DDLResult& result, uint64_t uniqueId) { - DETAIL_INFO("Rolling back transaction"); - cerr << "AltertableProcessor::processPackage: " << error << endl; + DETAIL_INFO("Rolling back transaction"); + cerr << "AltertableProcessor::processPackage: " << error << endl; - logging::Message::Args args; - logging::Message message(1); - args.add("Alter table Failed: "); - args.add(error); - args.add(""); - args.add(""); - message.format(args); + logging::Message::Args args; + logging::Message message(1); + args.add("Alter table Failed: "); + args.add(error); + args.add(""); + args.add(""); + message.format(args); - rollBackTransaction( uniqueId, txnID, sessionId); - fSessionManager.rolledback(txnID); + rollBackTransaction(uniqueId, txnID, sessionId); + fSessionManager.rolledback(txnID); + result.result = ALTER_ERROR; + result.message = message; +} + +void AlterTableProcessor::addColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::ColumnDef* columnDefPtr, + ddlpackage::QualifiedName& inTableName, const uint64_t uniqueId) +{ + std::string err("AlterTableProcessor::addColumn "); + SUMMARY_INFO(err); + // Allocate an object ID for the column we are about to create, non systables only + VERBOSE_INFO("Allocating object ID for a column"); + ByteStream bs; + ByteStream::byte tmp8; + int rc = 0; + std::string errorMsg; + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + bool isDict = false; + //@Bug 4111. Check whether the column exists in calpont systable + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableColName tableColName; + tableColName.schema = inTableName.fSchema; + tableColName.table = inTableName.fName; + tableColName.column = columnDefPtr->fName; + CalpontSystemCatalog::OID columnOid; + + try + { + columnOid = systemCatalogPtr->lookupOID(tableColName); + } + catch (std::exception& ex) + { result.result = ALTER_ERROR; - result.message = message; -} + err += ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + result.result = ALTER_ERROR; + err += "Unknown exception caught"; + throw std::runtime_error(err); + } -void AlterTableProcessor::addColumn (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::ColumnDef* columnDefPtr, ddlpackage::QualifiedName& inTableName, const uint64_t uniqueId) -{ - std::string err("AlterTableProcessor::addColumn "); - SUMMARY_INFO(err); - // Allocate an object ID for the column we are about to create, non systables only - VERBOSE_INFO("Allocating object ID for a column"); - ByteStream bs; - ByteStream::byte tmp8; - int rc = 0; - std::string errorMsg; - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - bool isDict = false; - //@Bug 4111. Check whether the column exists in calpont systable - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableColName tableColName; - tableColName.schema = inTableName.fSchema; - tableColName.table = inTableName.fName; - tableColName.column = columnDefPtr->fName; - CalpontSystemCatalog::OID columnOid; + if (columnOid > 0) // Column exists already + { + err = err + "Internal add column error for " + tableColName.schema + "." + tableColName.table + "." + + tableColName.column + ". Column exists already. Your table is probably out-of-sync"; + throw std::runtime_error(err); + } + + if ((columnDefPtr->fType->fType == CalpontSystemCatalog::CHAR && columnDefPtr->fType->fLength > 8) || + (columnDefPtr->fType->fType == CalpontSystemCatalog::VARCHAR && columnDefPtr->fType->fLength > 7) || + (columnDefPtr->fType->fType == CalpontSystemCatalog::VARBINARY && columnDefPtr->fType->fLength > 7) || + (columnDefPtr->fType->fType == CalpontSystemCatalog::BLOB)) + { + isDict = true; + } + + // Find out where syscolumn are + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot "); + + int pmNum = 1; + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + + boost::shared_ptr bsIn; + // Will create files on each PM as needed. + // BUG931 + // In order to fill up the new column with data an existing column is selected as reference + ColumnList columns; + getColumnsForTable(sessionID, inTableName.fSchema, inTableName.fName, columns); + ColumnList::const_iterator column_iterator; + column_iterator = columns.begin(); + + if (inTableName.fSchema != CALPONT_SCHEMA) + { try { - columnOid = systemCatalogPtr->lookupOID(tableColName); + execplan::ObjectIDManager fObjectIDManager; + + if (isDict) + { + fStartingColOID = fObjectIDManager.allocOIDs(2); + } + else + fStartingColOID = fObjectIDManager.allocOIDs(1); } catch (std::exception& ex) { - result.result = ALTER_ERROR; - err += ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - result.result = ALTER_ERROR; - err += "Unknown exception caught"; - throw std::runtime_error(err); + result.result = ALTER_ERROR; + err += ex.what(); + throw std::runtime_error(err); } - if (columnOid > 0) //Column exists already + // cout << "new oid is " << fStartingColOID << endl; + } + else + { + // Add columns to SYSTABLE and SYSCOLUMN + if ((inTableName.fName == SYSTABLE_TABLE) && (columnDefPtr->fName == AUTOINC_COL)) { - err = err + "Internal add column error for " + tableColName.schema + "." + tableColName.table + "." + tableColName.column - + ". Column exists already. Your table is probably out-of-sync"; - - throw std::runtime_error(err); + fStartingColOID = OID_SYSTABLE_AUTOINCREMENT; } - - if ((columnDefPtr->fType->fType == CalpontSystemCatalog::CHAR && columnDefPtr->fType->fLength > 8) || - (columnDefPtr->fType->fType == CalpontSystemCatalog::VARCHAR && columnDefPtr->fType->fLength > 7) || - (columnDefPtr->fType->fType == CalpontSystemCatalog::VARBINARY && columnDefPtr->fType->fLength > 7) || - (columnDefPtr->fType->fType == CalpontSystemCatalog::BLOB)) + else if ((inTableName.fName == SYSCOLUMN_TABLE) && (columnDefPtr->fName == COMPRESSIONTYPE_COL)) { - isDict = true; + fStartingColOID = OID_SYSCOLUMN_COMPRESSIONTYPE; } - - //Find out where syscolumn are - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot "); - - int pmNum = 1; - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; - - boost::shared_ptr bsIn; - //Will create files on each PM as needed. - //BUG931 - //In order to fill up the new column with data an existing column is selected as reference - ColumnList columns; - getColumnsForTable(sessionID, inTableName.fSchema, inTableName.fName, columns); - ColumnList::const_iterator column_iterator; - column_iterator = columns.begin(); - - if (inTableName.fSchema != CALPONT_SCHEMA) + else if ((inTableName.fName == SYSCOLUMN_TABLE) && (columnDefPtr->fName == NEXTVALUE_COL)) { - try - { - execplan::ObjectIDManager fObjectIDManager; - - if (isDict) - { - fStartingColOID = fObjectIDManager.allocOIDs(2); - } - else - fStartingColOID = fObjectIDManager.allocOIDs(1); - } - catch (std::exception& ex) - { - result.result = ALTER_ERROR; - err += ex.what(); - throw std::runtime_error(err); - } - - //cout << "new oid is " << fStartingColOID << endl; + fStartingColOID = OID_SYSCOLUMN_NEXTVALUE; } else { - // Add columns to SYSTABLE and SYSCOLUMN - if ((inTableName.fName == SYSTABLE_TABLE) && - (columnDefPtr->fName == AUTOINC_COL)) - { - fStartingColOID = OID_SYSTABLE_AUTOINCREMENT; - } - else if ((inTableName.fName == SYSCOLUMN_TABLE) && - (columnDefPtr->fName == COMPRESSIONTYPE_COL)) - { - fStartingColOID = OID_SYSCOLUMN_COMPRESSIONTYPE; - } - else if ((inTableName.fName == SYSCOLUMN_TABLE) && - (columnDefPtr->fName == NEXTVALUE_COL)) - { - fStartingColOID = OID_SYSCOLUMN_NEXTVALUE; - } - else - { - throw std::runtime_error("Error adding column to calpontsys table"); - } - - columnDefPtr->fType->fCompressiontype = 0; - columnDefPtr->fType->fAutoincrement = "n"; - columnDefPtr->fType->fNextvalue = 0; - cerr << "updating calpontsys...using static OID " << fStartingColOID << endl; + throw std::runtime_error("Error adding column to calpontsys table"); } - fColumnNum = 1; - //Find the position for the last column - //@Bug 1358 - CalpontSystemCatalog::TableName tableName; - tableName.schema = inTableName.fSchema; - tableName.table = inTableName.fName; - std::set outOfSerPar; - CalpontSystemCatalog::ROPair ropair; - bool autoincrement = false; + columnDefPtr->fType->fCompressiontype = 0; + columnDefPtr->fType->fAutoincrement = "n"; + columnDefPtr->fType->fNextvalue = 0; + cerr << "updating calpontsys...using static OID " << fStartingColOID << endl; + } - try + fColumnNum = 1; + // Find the position for the last column + //@Bug 1358 + CalpontSystemCatalog::TableName tableName; + tableName.schema = inTableName.fSchema; + tableName.table = inTableName.fName; + std::set outOfSerPar; + CalpontSystemCatalog::ROPair ropair; + bool autoincrement = false; + + try + { + ropair = systemCatalogPtr->tableRID(tableName); + + if (ropair.objnum < 0) { - ropair = systemCatalogPtr->tableRID(tableName); - - if (ropair.objnum < 0) - { - err = "No such table: " + tableName.table; - throw std::runtime_error(err); - } - - int totalColumns = systemCatalogPtr->colNumbers(tableName); - ColumnDefList aColumnList; - aColumnList.push_back(columnDefPtr); - bool alterFlag = true; - - // MCOL-66 The DBRM can't handle concurrent DDL - boost::mutex::scoped_lock lk(dbrmMutex); - - if (inTableName.fSchema != CALPONT_SCHEMA) - { - VERBOSE_INFO("Writing meta data to SYSCOL"); //send to WES to process - bs.restart(); - bs << (ByteStream::byte) WE_SVR_WRITE_SYSCOLUMN; - bs << uniqueId; - bs << sessionID; - bs << (uint32_t) txnID; - bs << inTableName.fSchema; - bs << inTableName.fName; - bs << (uint32_t) fStartingColOID; - - if (isDict) - bs << (uint32_t) (fStartingColOID + 1); - else - bs << (uint32_t) 0; - - bs << (uint8_t) alterFlag; - bs << (uint32_t) totalColumns; - columnDefPtr->serialize(bs); - - //send to WES to process - try - { - fWEClient->write(bs, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; - } - - if (rc != 0) - throw std::runtime_error(errorMsg); - - } - - if ((columnDefPtr->fType->fAutoincrement).compare("y") == 0) - { - //update systable autoincrement column - sysOid = 1001; - //Find out where systable is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot "); - - pmNum = (*dbRootPMMap)[dbRoot]; - bs.restart(); - bs << (ByteStream::byte) WE_SVR_UPDATE_SYSTABLE_AUTO; - bs << uniqueId; - bs << sessionID; - bs << (uint32_t) txnID; - bs << inTableName.fSchema; - bs << inTableName.fName; - bs << (uint32_t) 1; - - //send to WES to process - try - { - fWEClient->write(bs, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; - } - - if (rc != 0) - throw std::runtime_error(errorMsg); - - //start a sequence in controller - fDbrm->startAISequence(fStartingColOID, columnDefPtr->fType->fNextvalue, columnDefPtr->fType->fLength, - convertDataType(columnDefPtr->fType->fType)); - } - - //@Bug 4176. save oids to a log file for cleanup after fail over. - std::vector oidList; - - if (isDict) - { - oidList.push_back(fStartingColOID); - oidList.push_back(fStartingColOID + 1); - } - else - oidList.push_back(fStartingColOID); - - - createWriteDropLogFile( ropair.objnum, uniqueId, oidList ); - - //@Bug 1358,1427 Always use the first column in the table, not the first one in columnlist to prevent random result - //@Bug 4182. Use widest column as reference column - - //Find the widest column - unsigned int colpos = 0; - int maxColwidth = 0; - - for (colpos = 0; colpos < columns.size(); colpos++) - { - if (columns[colpos].colType.colWidth > maxColwidth) - maxColwidth = columns[colpos].colType.colWidth; - } - - while (column_iterator != columns.end()) - { - if (column_iterator->colType.colWidth == maxColwidth) - { - //If there is atleast one existing column then use that as a reference to initialize the new column rows. - //get dbroot information - - //fDbrm->getStartExtent((*column_iterator).oid, dbroot, partitionNum, true); - - rc = fDbrm->getOutOfServicePartitions(column_iterator->oid, outOfSerPar); - - if (rc != 0) - { - string errorMsg; - BRM::errString(rc, errorMsg); - ostringstream oss; - oss << "getOutOfServicePartitions failed due to " << errorMsg; - throw std::runtime_error(oss.str()); - } - - - int dataType1; - dataType1 = convertDataType(columnDefPtr->fType->fType); - - if (dataType1 == CalpontSystemCatalog::DECIMAL || - dataType1 == CalpontSystemCatalog::UDECIMAL) - { - columnDefPtr->convertDecimal(); - } - - CalpontSystemCatalog::ColDataType dataType = convertDataType(columnDefPtr->fType->fType); - - if ((columnDefPtr->fType->fAutoincrement).compare("y") == 0) - { - autoincrement = true; - } - - //send to all WES to add the new column - bs.restart(); - bs << (ByteStream::byte) WE_SVR_FILL_COLUMN; - bs << uniqueId; - bs << (uint32_t) txnID; - bs << (uint32_t) fStartingColOID; - - if (isDict) - bs << (uint32_t) (fStartingColOID + 1); - else - bs << (uint32_t) 0; - - //new column info - bs << (ByteStream::byte) dataType; - bs << (ByteStream::byte) autoincrement; - bs << (uint32_t) columnDefPtr->fType->fLength; - bs << (uint32_t) columnDefPtr->fType->fScale; - bs << (uint32_t) columnDefPtr->fType->fPrecision; - std::string tmpStr(""); - - if (columnDefPtr->fDefaultValue) - { - tmpStr = columnDefPtr->fDefaultValue->fValue; - } - - bs << tmpStr; - bs << (ByteStream::byte) columnDefPtr->fType->fCompressiontype; - //ref column info - bs << (uint32_t) column_iterator->oid; - bs << (ByteStream::byte) column_iterator->colType.colDataType; - bs << (uint32_t) column_iterator->colType.colWidth; - bs << (ByteStream::byte) column_iterator->colType.compressionType; - bs << fTimeZone; - //cout << "sending command fillcolumn " << endl; - uint32_t msgRecived = 0; - fWEClient->write_to_all(bs); - bsIn.reset(new ByteStream()); - - while (1) - { - if (msgRecived == fPMCount) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - break; - } - else - { - *bsIn >> tmp8; - *bsIn >> errorMsg; - rc = tmp8; - - //cout << "Got error code from WES " << rc << endl; - if (rc != 0) - break; - else - msgRecived++; - } - } - - if (rc != 0) //delete the newly created files before erroring out - { - bs.restart(); - bs << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; - bs << uniqueId; - bs << (uint32_t) oidList.size(); - - for (uint32_t i = 0; i < oidList.size(); i++) - { - bs << (uint32_t) oidList[i]; - } - - uint32_t msgRecived = 0; - - try - { - fWEClient->write_to_all(bs); - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while dropping column files"; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - break; - } - else - msgRecived++; - } - } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while dropping column files."; - } - - if ( rc == 0 ) - { - fWEClient->removeQueue(uniqueId); - deleteLogFile(DROPTABLE_LOG, ropair.objnum, uniqueId); - fWEClient->addQueue(uniqueId); - } - - throw std::runtime_error(errorMsg); - } - - //Update nextVal - break; - } - - //Update nextVal - column_iterator++; - } - } - catch (std::exception& ex) - { - if (result.result != CREATE_ERROR) - result.result = ALTER_ERROR; - - err += ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - result.result = ALTER_ERROR; - err += "Unknown exception caught"; - throw std::runtime_error(err); + err = "No such table: " + tableName.table; + throw std::runtime_error(err); } - //update SYSCOLUMN of the new next value - if (autoincrement) - { - DBRM aDbrm; - aDbrm.getAILock(fStartingColOID); - uint64_t nextValInController; - bool validNextVal = aDbrm.getAIValue(fStartingColOID, &nextValInController); - - if (validNextVal) - { - WE_DDLCommandClient ddlClient; - uint8_t rc = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - rc = ddlClient.UpdateSyscolumnNextval(fStartingColOID, nextValInController, txnID); - else - rc = ddlClient.UpdateSyscolumnNextval(fStartingColOID, nextValInController, sessionID); - - aDbrm.releaseAILock(fStartingColOID); - - if (rc != 0) - throw std::runtime_error("Update SYSCAT next value failed"); - } - else - { - aDbrm.releaseAILock(fStartingColOID); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT)); - } - } - - std::vector oidList; - oidList.push_back(fStartingColOID); - - if (outOfSerPar.size() > 0) - rc = fDbrm->markPartitionForDeletion(oidList, outOfSerPar, errorMsg); - - if (rc != 0) - { - ostringstream oss; - oss << "Mark partition for deletition failed due to " << errorMsg; - throw std::runtime_error(oss.str()); - } - - fWEClient->removeQueue(uniqueId); - deleteLogFile(DROPTABLE_LOG, ropair.objnum, uniqueId); - fWEClient->addQueue(uniqueId); -} - -void AlterTableProcessor::dropColumn (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropColumn& ataDropColumn, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) -{ - // 1. Get the OIDs for the column - // 2. Get the OIDs for the dictionary - // 3. Remove the column from SYSCOLUMN - // 4. update systable if the dropped column is autoincrement column - // 5. update column position for affected columns - // 6. Remove the files - - SUMMARY_INFO("AlterTableProcessor::dropColumn"); - VERBOSE_INFO("Finding object IDs for the column"); - CalpontSystemCatalog::TableColName tableColName; - CalpontSystemCatalog::TableName tableName; - tableName.schema = fTableName.fSchema; - tableName.table = fTableName.fName; - tableColName.schema = fTableName.fSchema; - tableColName.table = fTableName.fName; - tableColName.column = ataDropColumn.fColumnName; - execplan::CalpontSystemCatalog::DictOIDList dictOIDList; - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - //@Bug 1358 - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - std::string err; - execplan::CalpontSystemCatalog::ROPair roPair; - CalpontSystemCatalog::OID oid; - CalpontSystemCatalog::ColType colType; - - try - { - roPair = systemCatalogPtr->tableRID( tableName ); - - oid = systemCatalogPtr->lookupOID(tableColName); - colType = systemCatalogPtr->colType(oid); - } - catch (std::exception& ex) - { - throw std::runtime_error(ex.what()); - } - - int colPos = colType.colPosition; - ByteStream bytestream; - bytestream << (ByteStream::byte)WE_SVR_DELETE_SYSCOLUMN_ROW; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID; - bytestream << fTableName.fSchema; - bytestream << fTableName.fName; - bytestream << ataDropColumn.fColumnName; - - std::string errorMsg; - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - ByteStream::byte rc = 0; - //Find out where syscolumn is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot "); - - int pmNum = 1; - - boost::shared_ptr bsIn; - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; + int totalColumns = systemCatalogPtr->colNumbers(tableName); + ColumnDefList aColumnList; + aColumnList.push_back(columnDefPtr); + bool alterFlag = true; // MCOL-66 The DBRM can't handle concurrent DDL boost::mutex::scoped_lock lk(dbrmMutex); - try + if (inTableName.fSchema != CALPONT_SCHEMA) { - fWEClient->write(bytestream, (uint32_t)pmNum); -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column sending WE_SVR_DELETE_SYSCOLUMN_ROW to pm " << pmNum << endl; -#endif + VERBOSE_INFO("Writing meta data to SYSCOL"); // send to WES to process + bs.restart(); + bs << (ByteStream::byte)WE_SVR_WRITE_SYSCOLUMN; + bs << uniqueId; + bs << sessionID; + bs << (uint32_t)txnID; + bs << inTableName.fSchema; + bs << inTableName.fName; + bs << (uint32_t)fStartingColOID; + + if (isDict) + bs << (uint32_t)(fStartingColOID + 1); + else + bs << (uint32_t)0; + + bs << (uint8_t)alterFlag; + bs << (uint32_t)totalColumns; + columnDefPtr->serialize(bs); + + // send to WES to process + try + { + fWEClient->write(bs, (uint32_t)pmNum); while (1) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; - break; + *bsIn >> errorMsg; } + + break; + } } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got exception" << ex.what() << endl; -#endif + } + catch (runtime_error& ex) // write error + { rc = NETWORK_ERROR; errorMsg = ex.what(); - } - catch (...) - { + } + catch (...) + { rc = NETWORK_ERROR; errorMsg = " Unknown exception caught while updating SYSTABLE."; -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got unknown exception" << endl; -#endif + } + + if (rc != 0) + throw std::runtime_error(errorMsg); } - if (rc != 0) + if ((columnDefPtr->fType->fAutoincrement).compare("y") == 0) + { + // update systable autoincrement column + sysOid = 1001; + // Find out where systable is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot "); + + pmNum = (*dbRootPMMap)[dbRoot]; + bs.restart(); + bs << (ByteStream::byte)WE_SVR_UPDATE_SYSTABLE_AUTO; + bs << uniqueId; + bs << sessionID; + bs << (uint32_t)txnID; + bs << inTableName.fSchema; + bs << inTableName.fName; + bs << (uint32_t)1; + + // send to WES to process + try + { + fWEClient->write(bs, (uint32_t)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; + } + + if (rc != 0) throw std::runtime_error(errorMsg); - //Update SYSTABLE - if (colType.autoincrement) + // start a sequence in controller + fDbrm->startAISequence(fStartingColOID, columnDefPtr->fType->fNextvalue, columnDefPtr->fType->fLength, + convertDataType(columnDefPtr->fType->fType)); + } + + //@Bug 4176. save oids to a log file for cleanup after fail over. + std::vector oidList; + + if (isDict) { - sysOid = 1001; - //Find out where systable is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + oidList.push_back(fStartingColOID); + oidList.push_back(fStartingColOID + 1); + } + else + oidList.push_back(fStartingColOID); + + createWriteDropLogFile(ropair.objnum, uniqueId, oidList); + + //@Bug 1358,1427 Always use the first column in the table, not the first one in columnlist to prevent + //random result + //@Bug 4182. Use widest column as reference column + + // Find the widest column + unsigned int colpos = 0; + int maxColwidth = 0; + + for (colpos = 0; colpos < columns.size(); colpos++) + { + if (columns[colpos].colType.colWidth > maxColwidth) + maxColwidth = columns[colpos].colType.colWidth; + } + + while (column_iterator != columns.end()) + { + if (column_iterator->colType.colWidth == maxColwidth) + { + // If there is atleast one existing column then use that as a reference to initialize the new column + // rows. get dbroot information + + // fDbrm->getStartExtent((*column_iterator).oid, dbroot, partitionNum, true); + + rc = fDbrm->getOutOfServicePartitions(column_iterator->oid, outOfSerPar); if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot "); - - pmNum = (*dbRootPMMap)[dbRoot]; - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSTABLE_AUTO; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID; - bytestream << fTableName.fSchema; - bytestream << fTableName.fName; - bytestream << (uint32_t) 0; //autoincrement off - - try { - fWEClient->write(bytestream, (uint32_t)pmNum); -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column sending WE_SVR_UPDATE_SYSTABLE_AUTO to pm " << pmNum << endl; -#endif + string errorMsg; + BRM::errString(rc, errorMsg); + ostringstream oss; + oss << "getOutOfServicePartitions failed due to " << errorMsg; + throw std::runtime_error(oss.str()); + } + + int dataType1; + dataType1 = convertDataType(columnDefPtr->fType->fType); + + if (dataType1 == CalpontSystemCatalog::DECIMAL || dataType1 == CalpontSystemCatalog::UDECIMAL) + { + columnDefPtr->convertDecimal(); + } + + CalpontSystemCatalog::ColDataType dataType = convertDataType(columnDefPtr->fType->fType); + + if ((columnDefPtr->fType->fAutoincrement).compare("y") == 0) + { + autoincrement = true; + } + + // send to all WES to add the new column + bs.restart(); + bs << (ByteStream::byte)WE_SVR_FILL_COLUMN; + bs << uniqueId; + bs << (uint32_t)txnID; + bs << (uint32_t)fStartingColOID; + + if (isDict) + bs << (uint32_t)(fStartingColOID + 1); + else + bs << (uint32_t)0; + + // new column info + bs << (ByteStream::byte)dataType; + bs << (ByteStream::byte)autoincrement; + bs << (uint32_t)columnDefPtr->fType->fLength; + bs << (uint32_t)columnDefPtr->fType->fScale; + bs << (uint32_t)columnDefPtr->fType->fPrecision; + std::string tmpStr(""); + + if (columnDefPtr->fDefaultValue) + { + tmpStr = columnDefPtr->fDefaultValue->fValue; + } + + bs << tmpStr; + bs << (ByteStream::byte)columnDefPtr->fType->fCompressiontype; + // ref column info + bs << (uint32_t)column_iterator->oid; + bs << (ByteStream::byte)column_iterator->colType.colDataType; + bs << (uint32_t)column_iterator->colType.colWidth; + bs << (ByteStream::byte)column_iterator->colType.compressionType; + bs << fTimeZone; + // cout << "sending command fillcolumn " << endl; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bs); + bsIn.reset(new ByteStream()); + + while (1) + { + if (msgRecived == fPMCount) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + break; + } + else + { + *bsIn >> tmp8; + *bsIn >> errorMsg; + rc = tmp8; + + // cout << "Got error code from WES " << rc << endl; + if (rc != 0) + break; + else + msgRecived++; + } + } + + if (rc != 0) // delete the newly created files before erroring out + { + bs.restart(); + bs << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; + bs << uniqueId; + bs << (uint32_t)oidList.size(); + + for (uint32_t i = 0; i < oidList.size(); i++) + { + bs << (uint32_t)oidList[i]; + } + + uint32_t msgRecived = 0; + + try + { + fWEClient->write_to_all(bs); + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; while (1) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; - break; - } - } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got exception" << ex.what() << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got unknown exception" << endl; -#endif - } - - if (rc != 0) - throw std::runtime_error(errorMsg); - } - - //Update column position - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_COLPOS; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID; - bytestream << fTableName.fSchema; - bytestream << fTableName.fName; - bytestream << (uint32_t) colPos; - sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot "); - - pmNum = (*dbRootPMMap)[dbRoot]; - - try - { - fWEClient->write(bytestream, (uint32_t)pmNum); -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column sending WE_SVR_UPDATE_SYSTABLE_AUTO to pm " << pmNum << endl; -#endif - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; - break; - } - } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got exception" << ex.what() << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got unknown exception" << endl; -#endif - } - - if (rc != 0) - throw std::runtime_error(errorMsg); - - //commit the transaction. - BRM::TxnID aTxnID; - aTxnID.id = txnID; - aTxnID.valid = true; - commitTransaction(uniqueId, aTxnID); - - // Bug 4208 Drop the PrimProcFDCache before droping the column files - // FOr Windows, this ensures (most likely) that the column files have - // no open handles to hinder the deletion of the files. - rc = cacheutils::dropPrimProcFdCache(); - - VERBOSE_INFO("Removing column files"); - //Drop files - std::vector oidList; - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; - bytestream << uniqueId; - - if (colType.ddn.dictOID > 3000) //@bug 4847. need to take care varchar(8) - { - bytestream << (uint32_t) 2; - bytestream << (uint32_t) oid; - bytestream << (uint32_t) colType.ddn.dictOID; - oidList.push_back(oid); - oidList.push_back( colType.ddn.dictOID); - } - else - { - bytestream << (uint32_t) 1; - bytestream << (uint32_t) oid; - oidList.push_back(oid); - } - - //Save the oids to a file - uint32_t msgRecived = 0; - bool fileDropped = true; - - try - { - createWriteDropLogFile( roPair.objnum, uniqueId, oidList ); - //@Bug 4811. Need to send to all PMs - fWEClient->write_to_all(bytestream); -#ifdef IDB_DDL_DEBUG - cout << "Alter table drop column sending WE_SVR_UPDATE_SYSTABLE_AUTO to pm " << pmNum << endl; -#endif - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) + if (msgRecived == fWEClient->getPmCount()) break; - fWEClient->read(uniqueId, bsIn); + fWEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error - { + if (bsIn->length() == 0) // read error + { rc = NETWORK_ERROR; errorMsg = "Lost connection to Write Engine Server while dropping column files"; break; - } - else - { + } + else + { *bsIn >> tmp8; rc = tmp8; if (rc != 0) { - *bsIn >> errorMsg; - fileDropped = false; - break; + *bsIn >> errorMsg; + break; } else - msgRecived++; + msgRecived++; + } } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while dropping column files."; + } + + if (rc == 0) + { + fWEClient->removeQueue(uniqueId); + deleteLogFile(DROPTABLE_LOG, ropair.objnum, uniqueId); + fWEClient->addQueue(uniqueId); + } + + throw std::runtime_error(errorMsg); } + + // Update nextVal + break; + } + + // Update nextVal + column_iterator++; } - catch (runtime_error& ex) //write error + } + catch (std::exception& ex) + { + if (result.result != CREATE_ERROR) + result.result = ALTER_ERROR; + + err += ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + result.result = ALTER_ERROR; + err += "Unknown exception caught"; + throw std::runtime_error(err); + } + + // update SYSCOLUMN of the new next value + if (autoincrement) + { + DBRM aDbrm; + aDbrm.getAILock(fStartingColOID); + uint64_t nextValInController; + bool validNextVal = aDbrm.getAIValue(fStartingColOID, &nextValInController); + + if (validNextVal) { + WE_DDLCommandClient ddlClient; + uint8_t rc = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + rc = ddlClient.UpdateSyscolumnNextval(fStartingColOID, nextValInController, txnID); + else + rc = ddlClient.UpdateSyscolumnNextval(fStartingColOID, nextValInController, sessionID); + + aDbrm.releaseAILock(fStartingColOID); + + if (rc != 0) + throw std::runtime_error("Update SYSCAT next value failed"); + } + else + { + aDbrm.releaseAILock(fStartingColOID); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT)); + } + } + + std::vector oidList; + oidList.push_back(fStartingColOID); + + if (outOfSerPar.size() > 0) + rc = fDbrm->markPartitionForDeletion(oidList, outOfSerPar, errorMsg); + + if (rc != 0) + { + ostringstream oss; + oss << "Mark partition for deletition failed due to " << errorMsg; + throw std::runtime_error(oss.str()); + } + + fWEClient->removeQueue(uniqueId); + deleteLogFile(DROPTABLE_LOG, ropair.objnum, uniqueId); + fWEClient->addQueue(uniqueId); +} + +void AlterTableProcessor::dropColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::AtaDropColumn& ataDropColumn, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) +{ + // 1. Get the OIDs for the column + // 2. Get the OIDs for the dictionary + // 3. Remove the column from SYSCOLUMN + // 4. update systable if the dropped column is autoincrement column + // 5. update column position for affected columns + // 6. Remove the files + + SUMMARY_INFO("AlterTableProcessor::dropColumn"); + VERBOSE_INFO("Finding object IDs for the column"); + CalpontSystemCatalog::TableColName tableColName; + CalpontSystemCatalog::TableName tableName; + tableName.schema = fTableName.fSchema; + tableName.table = fTableName.fName; + tableColName.schema = fTableName.fSchema; + tableColName.table = fTableName.fName; + tableColName.column = ataDropColumn.fColumnName; + execplan::CalpontSystemCatalog::DictOIDList dictOIDList; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + //@Bug 1358 + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + std::string err; + execplan::CalpontSystemCatalog::ROPair roPair; + CalpontSystemCatalog::OID oid; + CalpontSystemCatalog::ColType colType; + + try + { + roPair = systemCatalogPtr->tableRID(tableName); + + oid = systemCatalogPtr->lookupOID(tableColName); + colType = systemCatalogPtr->colType(oid); + } + catch (std::exception& ex) + { + throw std::runtime_error(ex.what()); + } + + int colPos = colType.colPosition; + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_SVR_DELETE_SYSCOLUMN_ROW; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID; + bytestream << fTableName.fSchema; + bytestream << fTableName.fName; + bytestream << ataDropColumn.fColumnName; + + std::string errorMsg; + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + ByteStream::byte rc = 0; + // Find out where syscolumn is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot "); + + int pmNum = 1; + + boost::shared_ptr bsIn; + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + + // MCOL-66 The DBRM can't handle concurrent DDL + boost::mutex::scoped_lock lk(dbrmMutex); + + try + { + fWEClient->write(bytestream, (uint32_t)pmNum); #ifdef IDB_DDL_DEBUG - cout << "Alter table drop column got exception" << ex.what() << endl; + cout << "Alter table drop column sending WE_SVR_DELETE_SYSCOLUMN_ROW to pm " << pmNum << endl; #endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) + + while (1) { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while dropping column files."; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + break; + } + } + } + catch (runtime_error& ex) // write error + { #ifdef IDB_DDL_DEBUG - cout << "create table got unknown exception" << endl; + cout << "Alter table drop column got exception" << ex.what() << endl; #endif - } + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column got unknown exception" << endl; +#endif + } - //@Bug 3860 - rc = cacheutils::dropPrimProcFdCache(); - //Flush primProc cache - rc = cacheutils::flushOIDsFromCache( oidList ); - //Delete extents from extent map - rc = fDbrm->deleteOIDs(oidList); + if (rc != 0) + throw std::runtime_error(errorMsg); - if (fileDropped) - { - fWEClient->removeQueue(uniqueId); - deleteLogFile(DROPTABLE_LOG, roPair.objnum, uniqueId); - fWEClient->addQueue(uniqueId); - } -} - - -void AlterTableProcessor::dropColumns (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropColumns& ataDropColumns, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) -{ - SUMMARY_INFO("AlterTableProcessor::dropColumns"); - ddlpackage::ColumnNameList colList = ataDropColumns.fColumns; - ddlpackage::ColumnNameList::const_iterator col_iter = colList.begin(); - - std::string err; - - try - { - while (col_iter != colList.end()) - { - ddlpackage::AtaDropColumn ataDropColumn; - - ataDropColumn.fColumnName = *col_iter; - - dropColumn (sessionID, txnID, result, ataDropColumn, fTableName, uniqueId); - - if (result.result != NO_ERROR) - { - DETAIL_INFO("dropColumns::dropColumn failed"); - return; - } - - col_iter++; - } - } - catch (std::exception& ex) - { - err = ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "dropColumns:Unknown exception caught"; - throw std::runtime_error(err); - } -} - -void AlterTableProcessor::addTableConstraint (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, ddlpackage::AtaAddTableConstraint& ataAddTableConstraint, ddlpackage::QualifiedName& fTableName) -{ - /*TODO: Check if existing row satisfy the constraint. - If not, the constraint will not be added. */ - SUMMARY_INFO("AlterTableProcessor::addTableConstraint"); - ddlpackage::TableConstraintDefList constrainList; - constrainList.push_back(ataAddTableConstraint.fTableConstraint); - VERBOSE_INFO("Writing table constraint meta data to SYSCONSTRAINT"); - //bool alterFlag = true; - std::string err; - - try - { - //writeTableSysConstraintMetaData(sessionID, txnID, result, constrainList, fTableName, alterFlag); - VERBOSE_INFO("Writing table constraint meta data to SYSCONSTRAINTCOL"); - //writeTableSysConstraintColMetaData(sessionID, txnID, result,constrainList, fTableName, alterFlag); - } - catch (std::exception& ex) - { - err = ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "addTableConstraint:Unknown exception caught"; - throw std::runtime_error(err); - } -} - -void AlterTableProcessor::setColumnDefault (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaSetColumnDefault& ataSetColumnDefault, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) -{ - SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); - /*Steps: - 1. Update SYSCOLUMN for default value change - */ - SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); - ByteStream bs; - std::string errorMsg; - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - ByteStream::byte rc = 0; - //Find out where syscolumns + // Update SYSTABLE + if (colType.autoincrement) + { + sysOid = 1001; + // Find out where systable is rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); + throw std::runtime_error("Error while calling getSysCatDBRoot "); - int pmNum = 1; - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); pmNum = (*dbRootPMMap)[dbRoot]; + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSTABLE_AUTO; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID; + bytestream << fTableName.fSchema; + bytestream << fTableName.fName; + bytestream << (uint32_t)0; // autoincrement off - boost::shared_ptr bsIn; - - string err; - - - //Update SYSCOLUMN - bs.restart(); - bs << (ByteStream::byte) WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL; - bs << uniqueId; - bs << sessionID; - bs << (uint32_t) txnID; - bs << fTableName.fSchema; - bs << fTableName.fName; - bs << ataSetColumnDefault.fColumnName; - string defaultValue(""); - - if (ataSetColumnDefault.fDefaultValue) - defaultValue = ataSetColumnDefault.fDefaultValue->fValue; - - bs << defaultValue; - - //send to WES to process try { - fWEClient->write(bs, (uint32_t)pmNum); + fWEClient->write(bytestream, (uint32_t)pmNum); +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column sending WE_SVR_UPDATE_SYSTABLE_AUTO to pm " << pmNum << endl; +#endif - while (1) + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + break; + } + } } - catch (runtime_error& ex) //write error + catch (runtime_error& ex) // write error { - rc = NETWORK_ERROR; - errorMsg = ex.what(); +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); } catch (...) { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column got unknown exception" << endl; +#endif } if (rc != 0) - throw std::runtime_error(errorMsg); + throw std::runtime_error(errorMsg); + } + + // Update column position + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_COLPOS; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID; + bytestream << fTableName.fSchema; + bytestream << fTableName.fName; + bytestream << (uint32_t)colPos; + sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot "); + + pmNum = (*dbRootPMMap)[dbRoot]; + + try + { + fWEClient->write(bytestream, (uint32_t)pmNum); +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column sending WE_SVR_UPDATE_SYSTABLE_AUTO to pm " << pmNum << endl; +#endif + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + break; + } + } + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column got unknown exception" << endl; +#endif + } + + if (rc != 0) + throw std::runtime_error(errorMsg); + + // commit the transaction. + BRM::TxnID aTxnID; + aTxnID.id = txnID; + aTxnID.valid = true; + commitTransaction(uniqueId, aTxnID); + + // Bug 4208 Drop the PrimProcFDCache before droping the column files + // FOr Windows, this ensures (most likely) that the column files have + // no open handles to hinder the deletion of the files. + rc = cacheutils::dropPrimProcFdCache(); + + VERBOSE_INFO("Removing column files"); + // Drop files + std::vector oidList; + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; + bytestream << uniqueId; + + if (colType.ddn.dictOID > 3000) //@bug 4847. need to take care varchar(8) + { + bytestream << (uint32_t)2; + bytestream << (uint32_t)oid; + bytestream << (uint32_t)colType.ddn.dictOID; + oidList.push_back(oid); + oidList.push_back(colType.ddn.dictOID); + } + else + { + bytestream << (uint32_t)1; + bytestream << (uint32_t)oid; + oidList.push_back(oid); + } + + // Save the oids to a file + uint32_t msgRecived = 0; + bool fileDropped = true; + + try + { + createWriteDropLogFile(roPair.objnum, uniqueId, oidList); + //@Bug 4811. Need to send to all PMs + fWEClient->write_to_all(bytestream); +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column sending WE_SVR_UPDATE_SYSTABLE_AUTO to pm " << pmNum << endl; +#endif + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while dropping column files"; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fileDropped = false; + break; + } + else + msgRecived++; + } + } + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DDL_DEBUG + cout << "Alter table drop column got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while dropping column files."; +#ifdef IDB_DDL_DEBUG + cout << "create table got unknown exception" << endl; +#endif + } + + //@Bug 3860 + rc = cacheutils::dropPrimProcFdCache(); + // Flush primProc cache + rc = cacheutils::flushOIDsFromCache(oidList); + // Delete extents from extent map + rc = fDbrm->deleteOIDs(oidList); + + if (fileDropped) + { + fWEClient->removeQueue(uniqueId); + deleteLogFile(DROPTABLE_LOG, roPair.objnum, uniqueId); + fWEClient->addQueue(uniqueId); + } } -void AlterTableProcessor::dropColumnDefault (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropColumnDefault& ataDropColumnDefault, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) + +void AlterTableProcessor::dropColumns(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::AtaDropColumns& ataDropColumns, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) { - SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); - /*Steps: - 1. Update SYSCOLUMN for default value change - */ - SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); - ByteStream bs; - std::string errorMsg; - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - ByteStream::byte rc = 0; - //Find out where syscolumn is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + SUMMARY_INFO("AlterTableProcessor::dropColumns"); + ddlpackage::ColumnNameList colList = ataDropColumns.fColumns; + ddlpackage::ColumnNameList::const_iterator col_iter = colList.begin(); - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); + std::string err; - int pmNum = 1; - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; - - boost::shared_ptr bsIn; - - string err; - - - //Update SYSCOLUMN - bs.restart(); - bs << (ByteStream::byte) WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL; - bs << uniqueId; - bs << sessionID; - bs << (uint32_t) txnID; - bs << fTableName.fSchema; - bs << fTableName.fName; - bs << ataDropColumnDefault.fColumnName; - string defaultValue(""); - bs << defaultValue; - - //send to WES to process - try + try + { + while (col_iter != colList.end()) { - fWEClient->write(bs, (uint32_t)pmNum); + ddlpackage::AtaDropColumn ataDropColumn; - while (1) + ataDropColumn.fColumnName = *col_iter; + + dropColumn(sessionID, txnID, result, ataDropColumn, fTableName, uniqueId); + + if (result.result != NO_ERROR) + { + DETAIL_INFO("dropColumns::dropColumn failed"); + return; + } + + col_iter++; + } + } + catch (std::exception& ex) + { + err = ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "dropColumns:Unknown exception caught"; + throw std::runtime_error(err); + } +} + +void AlterTableProcessor::addTableConstraint(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, + ddlpackage::AtaAddTableConstraint& ataAddTableConstraint, + ddlpackage::QualifiedName& fTableName) +{ + /*TODO: Check if existing row satisfy the constraint. + If not, the constraint will not be added. */ + SUMMARY_INFO("AlterTableProcessor::addTableConstraint"); + ddlpackage::TableConstraintDefList constrainList; + constrainList.push_back(ataAddTableConstraint.fTableConstraint); + VERBOSE_INFO("Writing table constraint meta data to SYSCONSTRAINT"); + // bool alterFlag = true; + std::string err; + + try + { + // writeTableSysConstraintMetaData(sessionID, txnID, result, constrainList, fTableName, alterFlag); + VERBOSE_INFO("Writing table constraint meta data to SYSCONSTRAINTCOL"); + // writeTableSysConstraintColMetaData(sessionID, txnID, result,constrainList, fTableName, alterFlag); + } + catch (std::exception& ex) + { + err = ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "addTableConstraint:Unknown exception caught"; + throw std::runtime_error(err); + } +} + +void AlterTableProcessor::setColumnDefault(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, + ddlpackage::AtaSetColumnDefault& ataSetColumnDefault, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) +{ + SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); + /*Steps: + 1. Update SYSCOLUMN for default value change + */ + SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); + ByteStream bs; + std::string errorMsg; + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + ByteStream::byte rc = 0; + // Find out where syscolumns + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot"); + + int pmNum = 1; + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + + boost::shared_ptr bsIn; + + string err; + + // Update SYSCOLUMN + bs.restart(); + bs << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL; + bs << uniqueId; + bs << sessionID; + bs << (uint32_t)txnID; + bs << fTableName.fSchema; + bs << fTableName.fName; + bs << ataSetColumnDefault.fColumnName; + string defaultValue(""); + + if (ataSetColumnDefault.fDefaultValue) + defaultValue = ataSetColumnDefault.fDefaultValue->fValue; + + bs << defaultValue; + + // send to WES to process + try + { + fWEClient->write(bs, (uint32_t)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } + *bsIn >> errorMsg; } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; - } - if (rc != 0) - throw std::runtime_error(errorMsg); + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; + } + + if (rc != 0) + throw std::runtime_error(errorMsg); +} +void AlterTableProcessor::dropColumnDefault(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, + ddlpackage::AtaDropColumnDefault& ataDropColumnDefault, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) +{ + SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); + /*Steps: + 1. Update SYSCOLUMN for default value change + */ + SUMMARY_INFO("AlterTableProcessor::setColumnDefault"); + ByteStream bs; + std::string errorMsg; + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + ByteStream::byte rc = 0; + // Find out where syscolumn is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot"); + + int pmNum = 1; + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + + boost::shared_ptr bsIn; + + string err; + + // Update SYSCOLUMN + bs.restart(); + bs << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL; + bs << uniqueId; + bs << sessionID; + bs << (uint32_t)txnID; + bs << fTableName.fSchema; + bs << fTableName.fName; + bs << ataDropColumnDefault.fColumnName; + string defaultValue(""); + bs << defaultValue; + + // send to WES to process + try + { + fWEClient->write(bs, (uint32_t)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; + } + + if (rc != 0) + throw std::runtime_error(errorMsg); } #if 0 @@ -1862,656 +1889,657 @@ void AlterTableProcessor::dropTableConstraint (uint32_t sessionID, execplan::Cal } #endif -void AlterTableProcessor::renameTable (uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, ddlpackage::AtaRenameTable& ataRenameTable, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) +void AlterTableProcessor::renameTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::AtaRenameTable& ataRenameTable, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) { - /*Steps: - 1. Update SYSTABLE (table name) - 2. Update SYSCOLUMN (table name) - */ - SUMMARY_INFO("AlterTableProcessor::renameTable"); + /*Steps: + 1. Update SYSTABLE (table name) + 2. Update SYSCOLUMN (table name) + */ + SUMMARY_INFO("AlterTableProcessor::renameTable"); - //@Bug 4599. Check whether the new table exists in infinidb - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - execplan::CalpontSystemCatalog::TableName tableName; - tableName.schema = fTableName.fSchema; - tableName.table = ataRenameTable.fQualifiedName->fName; - execplan::CalpontSystemCatalog::ROPair roPair; + //@Bug 4599. Check whether the new table exists in infinidb + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + execplan::CalpontSystemCatalog::TableName tableName; + tableName.schema = fTableName.fSchema; + tableName.table = ataRenameTable.fQualifiedName->fName; + execplan::CalpontSystemCatalog::ROPair roPair; + roPair.objnum = 0; + + try + { + roPair = systemCatalogPtr->tableRID(tableName); + } + catch (...) + { roPair.objnum = 0; + } - try - { - roPair = systemCatalogPtr->tableRID(tableName); - } - catch (...) - { - roPair.objnum = 0; - } + if (roPair.objnum >= 3000) + throw std::runtime_error("The new tablename is already in use."); - if (roPair.objnum >= 3000) - throw std::runtime_error("The new tablename is already in use."); + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSTABLE_TABLENAME; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID; + bytestream << fTableName.fSchema; + bytestream << fTableName.fName; + bytestream << ataRenameTable.fQualifiedName->fName; - ByteStream bytestream; - bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSTABLE_TABLENAME; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID; - bytestream << fTableName.fSchema; - bytestream << fTableName.fName; - bytestream << ataRenameTable.fQualifiedName->fName; + std::string errorMsg; + uint16_t dbRoot; + BRM::OID_t sysOid = 1001; + ByteStream::byte rc = 0; + // Find out where systable is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - std::string errorMsg; - uint16_t dbRoot; - BRM::OID_t sysOid = 1001; - ByteStream::byte rc = 0; - //Find out where systable is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot"); - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); + int pmNum = 1; - int pmNum = 1; + boost::shared_ptr bsIn; + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; - boost::shared_ptr bsIn; - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; - - try - { - fWEClient->write(bytestream, (uint32_t)pmNum); + try + { + fWEClient->write(bytestream, (uint32_t)pmNum); #ifdef IDB_DDL_DEBUG - cout << "Rename table sending WE_SVR_UPDATE_SYSTABLE_TABLENAME to pm " << pmNum << endl; + cout << "Rename table sending WE_SVR_UPDATE_SYSTABLE_TABLENAME to pm " << pmNum << endl; #endif - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; - break; - } - } - } - catch (runtime_error& ex) //write error + while (1) { -#ifdef IDB_DDL_DEBUG - cout << "create table got exception" << ex.what() << endl; -#endif + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { rc = NETWORK_ERROR; - errorMsg = ex.what(); + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + break; + } } - catch (...) + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DDL_DEBUG + cout << "create table got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; +#ifdef IDB_DDL_DEBUG + cout << "create table got unknown exception" << endl; +#endif + } + + if (rc != 0) + throw std::runtime_error(errorMsg); + + // update SYSCOLUMN + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_TABLENAME; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID; + bytestream << fTableName.fSchema; + bytestream << fTableName.fName; + bytestream << ataRenameTable.fQualifiedName->fName; + sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot"); + + pmNum = (*dbRootPMMap)[dbRoot]; + + try + { + fWEClient->write(bytestream, (unsigned)pmNum); +#ifdef IDB_DDL_DEBUG + cout << "Rename table sending WE_SVR_UPDATE_SYSCOLUMN_TABLENAME to pm " << pmNum << endl; +#endif + + while (1) { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; -#ifdef IDB_DDL_DEBUG - cout << "create table got unknown exception" << endl; -#endif + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + break; + } } - - if (rc != 0) - throw std::runtime_error(errorMsg); - - //update SYSCOLUMN - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_TABLENAME; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID; - bytestream << fTableName.fSchema; - bytestream << fTableName.fName; - bytestream << ataRenameTable.fQualifiedName->fName; - sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); - - pmNum = (*dbRootPMMap)[dbRoot]; - - try - { - fWEClient->write(bytestream, (unsigned)pmNum); + } + catch (runtime_error& ex) // write error + { #ifdef IDB_DDL_DEBUG - cout << "Rename table sending WE_SVR_UPDATE_SYSCOLUMN_TABLENAME to pm " << pmNum << endl; + cout << "create table got exception" << ex.what() << endl; #endif - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; - break; - } - } - } - catch (runtime_error& ex) //write error - { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; #ifdef IDB_DDL_DEBUG - cout << "create table got exception" << ex.what() << endl; + cout << "create table got unknown exception" << endl; #endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; -#ifdef IDB_DDL_DEBUG - cout << "create table got unknown exception" << endl; -#endif - } + } - if (rc != 0) - throw std::runtime_error(errorMsg); + if (rc != 0) + throw std::runtime_error(errorMsg); } void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, ddlpackage::AtaTableComment& ataTableComment, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) { - // Currently only process autoincrement values in table comments during alter - SUMMARY_INFO("AlterTableProcessor::tableComment"); - uint64_t nextVal; - BRM::OID_t sysOid = 1001; - ByteStream::byte rc = 0; - boost::shared_ptr bsIn; - uint16_t dbRoot; - std::string errorMsg; - int pmNum = 1; - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; + // Currently only process autoincrement values in table comments during alter + SUMMARY_INFO("AlterTableProcessor::tableComment"); + uint64_t nextVal; + BRM::OID_t sysOid = 1001; + ByteStream::byte rc = 0; + boost::shared_ptr bsIn; + uint16_t dbRoot; + std::string errorMsg; + int pmNum = 1; + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot"); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - boost::algorithm::to_upper(ataTableComment.fTableComment); - boost::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", boost::regex_constants::extended); - boost::match_results what; - std::string::const_iterator start, end; - start = ataTableComment.fTableComment.begin(); - end = ataTableComment.fTableComment.end(); - boost::match_flag_type flags = boost::match_default; + boost::algorithm::to_upper(ataTableComment.fTableComment); + boost::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", boost::regex_constants::extended); + boost::match_results what; + std::string::const_iterator start, end; + start = ataTableComment.fTableComment.begin(); + end = ataTableComment.fTableComment.end(); + boost::match_flag_type flags = boost::match_default; - if (boost::regex_search(start, end, what, compat, flags) && what[0].matched) + if (boost::regex_search(start, end, what, compat, flags) && what[0].matched) + { + std::string params(&(*(what[0].second))); + char* ep = NULL; + const char* str = params.c_str(); + errno = 0; + nextVal = strtoull(str, &ep, 10); + + if ((ep == str) || (*ep != '\0') || (errno != 0)) { - std::string params(&(*(what[0].second))); - char* ep = NULL; - const char* str = params.c_str(); - errno = 0; - nextVal = strtoull(str, &ep, 10); - - if ((ep == str) || (*ep != '\0') || (errno != 0)) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); - } - - // Checks if zero and throws appropriate error (despite message name) - // negative checks are below - if (nextVal == 0) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)); - } - } - else - { - // Generic table comment, we don't need to do anything - return; + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); } - // Get the OID for autoinc (if exists) - CalpontSystemCatalog::TableName tableName; - tableName.schema = fTableName.fSchema; - tableName.table = fTableName.fName; - CalpontSystemCatalog::TableInfo tblInfo = systemCatalogPtr->tableInfo(tableName); - - if (tblInfo.tablewithautoincr != 1) + // Checks if zero and throws appropriate error (despite message name) + // negative checks are below + if (nextVal == 0) { - throw std::runtime_error("Table does not have an autoincrement column"); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)); } + } + else + { + // Generic table comment, we don't need to do anything + return; + } - int32_t oid = systemCatalogPtr->autoColumOid(tableName); + // Get the OID for autoinc (if exists) + CalpontSystemCatalog::TableName tableName; + tableName.schema = fTableName.fSchema; + tableName.table = fTableName.fName; + CalpontSystemCatalog::TableInfo tblInfo = systemCatalogPtr->tableInfo(tableName); - CalpontSystemCatalog::ColType type = systemCatalogPtr->colType(oid); + if (tblInfo.tablewithautoincr != 1) + { + throw std::runtime_error("Table does not have an autoincrement column"); + } - bool validated = true; - bool negative = false; + int32_t oid = systemCatalogPtr->autoColumOid(tableName); - switch (type.colDataType) + CalpontSystemCatalog::ColType type = systemCatalogPtr->colType(oid); + + bool validated = true; + bool negative = false; + + switch (type.colDataType) + { + case CalpontSystemCatalog::BIGINT: + if (static_cast(nextVal) > MAX_BIGINT) + validated = false; + + if (static_cast(nextVal) < 1) + negative = true; + + break; + + case CalpontSystemCatalog::UBIGINT: + if (nextVal > MAX_UBIGINT) + validated = false; + + break; + + case CalpontSystemCatalog::INT: + if (static_cast(nextVal) > MAX_INT) + validated = false; + + if (static_cast(nextVal) < 1) + negative = true; + + break; + + case CalpontSystemCatalog::UINT: + if (nextVal > MAX_UINT) + validated = false; + + break; + + case CalpontSystemCatalog::MEDINT: + if (static_cast(nextVal) > MAX_MEDINT) + validated = false; + + if (static_cast(nextVal) < 1) + negative = true; + + break; + + case CalpontSystemCatalog::UMEDINT: + if (nextVal > MAX_UMEDINT) + validated = false; + + break; + + case CalpontSystemCatalog::SMALLINT: + if (static_cast(nextVal) > MAX_SMALLINT) + validated = false; + + if (static_cast(nextVal) < 1) + negative = true; + + break; + + case CalpontSystemCatalog::USMALLINT: + if (nextVal > MAX_USMALLINT) + validated = false; + + break; + + case CalpontSystemCatalog::TINYINT: + if (static_cast(nextVal) > MAX_TINYINT) + validated = false; + + if (static_cast(nextVal) < 1) + negative = true; + + break; + + case CalpontSystemCatalog::UTINYINT: + if (nextVal > MAX_UTINYINT) + validated = false; + + break; + + default: break; + } + + if (!validated) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); + } + + if (negative) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)); + } + + fDbrm->resetAISequence(oid, nextVal); + ByteStream bs; + bs.restart(); + bs << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL; + bs << uniqueId; + bs << oid; + bs << nextVal; + bs << sessionID; + + try + { + fWEClient->write(bs, (uint32_t)pmNum); + + while (1) { - case CalpontSystemCatalog::BIGINT: - if (static_cast(nextVal) > MAX_BIGINT) - validated = false; + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - if (static_cast(nextVal) < 1) - negative = true; - - break; - - case CalpontSystemCatalog::UBIGINT: - if (nextVal > MAX_UBIGINT) - validated = false; - - break; - - case CalpontSystemCatalog::INT: - if (static_cast(nextVal) > MAX_INT) - validated = false; - - if (static_cast(nextVal) < 1) - negative = true; - - break; - - case CalpontSystemCatalog::UINT: - if (nextVal > MAX_UINT) - validated = false; - - break; - - case CalpontSystemCatalog::MEDINT: - if (static_cast(nextVal) > MAX_MEDINT) - validated = false; - - if (static_cast(nextVal) < 1) - negative = true; - - break; - - case CalpontSystemCatalog::UMEDINT: - if (nextVal > MAX_UMEDINT) - validated = false; - - break; - - case CalpontSystemCatalog::SMALLINT: - if (static_cast(nextVal) > MAX_SMALLINT) - validated = false; - - if (static_cast(nextVal) < 1) - negative = true; - - break; - - case CalpontSystemCatalog::USMALLINT: - if (nextVal > MAX_USMALLINT) - validated = false; - - break; - - case CalpontSystemCatalog::TINYINT: - if (static_cast(nextVal) > MAX_TINYINT) - validated = false; - - if (static_cast(nextVal) < 1) - negative = true; - - break; - - case CalpontSystemCatalog::UTINYINT: - if (nextVal > MAX_UTINYINT) - validated = false; - - break; - - default: - break; - } - - if (!validated) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); - } - - if (negative) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)); - } - - fDbrm->resetAISequence(oid, nextVal); - ByteStream bs; - bs.restart(); - bs << (ByteStream::byte) WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL; - bs << uniqueId; - bs << oid; - bs << nextVal; - bs << sessionID; - - try - { - fWEClient->write(bs, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; - break; - } - } - } - catch (runtime_error& ex) //write error - { + if (bsIn->length() == 0) // read error + { rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + break; + } } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; + } - if (rc != 0) - throw std::runtime_error(errorMsg); - + if (rc != 0) + throw std::runtime_error(errorMsg); } void AlterTableProcessor::renameColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, ddlpackage::AtaRenameColumn& ataRenameColumn, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId) { - /*Steps: - 1. Update SYSCOLUMN for name, autoincrement, nextval change - 2. Update SYSTABLE if column is autoincrement column - */ - SUMMARY_INFO("AlterTableProcessor::renameColumn"); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + /*Steps: + 1. Update SYSCOLUMN for name, autoincrement, nextval change + 2. Update SYSTABLE if column is autoincrement column + */ + SUMMARY_INFO("AlterTableProcessor::renameColumn"); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - ByteStream bs; - std::string errorMsg; - uint16_t dbRoot; - BRM::OID_t sysOid = 1001; - ByteStream::byte rc = 0; - //Find out where systable is + ByteStream bs; + std::string errorMsg; + uint16_t dbRoot; + BRM::OID_t sysOid = 1001; + ByteStream::byte rc = 0; + // Find out where systable is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + throw std::runtime_error("Error while calling getSysCatDBRoot"); + + int pmNum = 1; + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + boost::shared_ptr bsIn; + + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::TableColName tableColName; + tableColName.schema = fTableName.fSchema; + tableColName.table = fTableName.fName; + tableColName.column = ataRenameColumn.fName; + CalpontSystemCatalog::ROPair ropair; + + string err; + + try + { + // This gives us the rid in syscolumn that we want to update + tableName.schema = tableColName.schema; + tableName.table = tableColName.table; + ropair = systemCatalogPtr->tableRID(tableName); + + if (ropair.objnum < 0) + { + ostringstream oss; + oss << "No such table: " << tableName; + throw std::runtime_error(oss.str().c_str()); + } + + ropair = systemCatalogPtr->columnRID(tableColName); + + if (ropair.objnum < 0) + { + ostringstream oss; + oss << "No such column: " << tableColName; + throw std::runtime_error(oss.str().c_str()); + } + + CalpontSystemCatalog::ColType colType = systemCatalogPtr->colType(ropair.objnum); + + if (!typesAreSame(colType, *ataRenameColumn.fNewType)) + { + ostringstream oss; + oss << "Changing the datatype of a column is not supported"; + throw std::runtime_error(oss.str().c_str()); + } + + //@Bug 3746 Check whether the change is about the compression type + if (!comptypesAreCompat(colType.compressionType, (*ataRenameColumn.fNewType).fCompressiontype)) + { + ostringstream oss; + oss << "The compression type of an existing column cannot be changed."; + throw std::runtime_error(oss.str().c_str()); + } + + // Check whether SYSTABLE needs to be updated + + CalpontSystemCatalog::TableInfo tblInfo = systemCatalogPtr->tableInfo(tableName); + + if (((tblInfo.tablewithautoincr == 1) && (colType.autoincrement) && + (ataRenameColumn.fNewType->fAutoincrement.compare("n") == 0)) || + ((tblInfo.tablewithautoincr == 0) && (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0))) + { + // update systable autoincrement column + bs.restart(); + bs << (ByteStream::byte)WE_SVR_UPDATE_SYSTABLE_AUTO; + bs << uniqueId; + bs << sessionID; + bs << (uint32_t)txnID; + bs << fTableName.fSchema; + bs << fTableName.fName; + + if (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0) + bs << (uint32_t)1; + else + bs << (uint32_t)0; + + // send to WES to process + try + { + fWEClient->write(bs, (uint32_t)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; + } + + if (rc != 0) + throw std::runtime_error(errorMsg); + + // change a sequence in controller + if ((!(tblInfo.tablewithautoincr == 1) || (colType.autoincrement)) && + (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0)) + { + fDbrm->startAISequence(ropair.objnum, ataRenameColumn.fNewType->fNextvalue, + ataRenameColumn.fNewType->fLength, + convertDataType(ataRenameColumn.fNewType->fType)); + // Reset it in case there is a sequence already + fDbrm->resetAISequence(ropair.objnum, ataRenameColumn.fNewType->fNextvalue); + } + } + else if ((tblInfo.tablewithautoincr == 1) && (colType.autoincrement) && + (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0)) + { + } + else + { + fDbrm->resetAISequence(ropair.objnum, 0); + } + + // Update SYSCOLUMN + bs.restart(); + bs << (ByteStream::byte)WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN; + bs << uniqueId; + bs << sessionID; + bs << (uint32_t)txnID; + bs << fTableName.fSchema; + bs << fTableName.fName; + bs << ataRenameColumn.fName; + bs << ataRenameColumn.fNewName; + bs << ataRenameColumn.fNewType->fAutoincrement; + //@Bug 5913. for autoincrement column, find the next value from SYSCOLUMN + long long nextVal = ataRenameColumn.fNewType->fNextvalue; + + if ((tblInfo.tablewithautoincr == 1) && (colType.autoincrement) && + (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0)) + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + + bs << (uint64_t)nextVal; + uint32_t nullable = 1; + string defaultValue(""); + + if (ataRenameColumn.fConstraints.size() > 0) + { + for (uint32_t j = 0; j < ataRenameColumn.fConstraints.size(); j++) + { + if (ataRenameColumn.fConstraints[j]->fConstraintType == DDL_NOT_NULL) + { + nullable = 0; + break; + } + } + } + + bs << nullable; + + if (ataRenameColumn.fDefaultValue) + defaultValue = ataRenameColumn.fDefaultValue->fValue; + + bs << defaultValue; + sysOid = 1021; + // Find out where syscolumn is rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); + throw std::runtime_error("Error while calling getSysCatDBRoot"); - int pmNum = 1; - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); pmNum = (*dbRootPMMap)[dbRoot]; - boost::shared_ptr bsIn; - - - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::TableColName tableColName; - tableColName.schema = fTableName.fSchema; - tableColName.table = fTableName.fName; - tableColName.column = ataRenameColumn.fName; - CalpontSystemCatalog::ROPair ropair; - - string err; + // send to WES to process try { - //This gives us the rid in syscolumn that we want to update - tableName.schema = tableColName.schema; - tableName.table = tableColName.table; - ropair = systemCatalogPtr->tableRID(tableName); + fWEClient->write(bs, (uint32_t)pmNum); - if (ropair.objnum < 0) + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - ostringstream oss; - oss << "No such table: " << tableName; - throw std::runtime_error(oss.str().c_str()); + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; } - - ropair = systemCatalogPtr->columnRID(tableColName); - - if (ropair.objnum < 0) - { - ostringstream oss; - oss << "No such column: " << tableColName; - throw std::runtime_error(oss.str().c_str()); - } - - CalpontSystemCatalog::ColType colType = systemCatalogPtr->colType(ropair.objnum); - - if (!typesAreSame(colType, *ataRenameColumn.fNewType)) - { - ostringstream oss; - oss << "Changing the datatype of a column is not supported"; - throw std::runtime_error(oss.str().c_str()); - } - - //@Bug 3746 Check whether the change is about the compression type - if (!comptypesAreCompat(colType.compressionType, (*ataRenameColumn.fNewType).fCompressiontype)) - { - ostringstream oss; - oss << "The compression type of an existing column cannot be changed."; - throw std::runtime_error(oss.str().c_str()); - } - - //Check whether SYSTABLE needs to be updated - - CalpontSystemCatalog::TableInfo tblInfo = systemCatalogPtr->tableInfo(tableName); - - if (((tblInfo.tablewithautoincr == 1) && (colType.autoincrement) && (ataRenameColumn.fNewType->fAutoincrement.compare("n") == 0)) || - ((tblInfo.tablewithautoincr == 0) && (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0))) - { - //update systable autoincrement column - bs.restart(); - bs << (ByteStream::byte) WE_SVR_UPDATE_SYSTABLE_AUTO; - bs << uniqueId; - bs << sessionID; - bs << (uint32_t) txnID; - bs << fTableName.fSchema; - bs << fTableName.fName; - - if (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0) - bs << (uint32_t) 1; - else - bs << (uint32_t) 0; - - //send to WES to process - try - { - fWEClient->write(bs, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; - } - - if (rc != 0) - throw std::runtime_error(errorMsg); - - //change a sequence in controller - if ((!(tblInfo.tablewithautoincr == 1) || (colType.autoincrement)) && (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0)) - { - fDbrm->startAISequence(ropair.objnum, ataRenameColumn.fNewType->fNextvalue, ataRenameColumn.fNewType->fLength, - convertDataType(ataRenameColumn.fNewType->fType)); - //Reset it in case there is a sequence already - fDbrm->resetAISequence(ropair.objnum, ataRenameColumn.fNewType->fNextvalue); - } - - } - else if ((tblInfo.tablewithautoincr == 1) && (colType.autoincrement) && (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0)) - {} else { - fDbrm->resetAISequence(ropair.objnum, 0); + *bsIn >> rc; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; } - - //Update SYSCOLUMN - bs.restart(); - bs << (ByteStream::byte) WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN; - bs << uniqueId; - bs << sessionID; - bs << (uint32_t) txnID; - bs << fTableName.fSchema; - bs << fTableName.fName; - bs << ataRenameColumn.fName; - bs << ataRenameColumn.fNewName; - bs << ataRenameColumn.fNewType->fAutoincrement; - //@Bug 5913. for autoincrement column, find the next value from SYSCOLUMN - long long nextVal = ataRenameColumn.fNewType->fNextvalue; - - if ((tblInfo.tablewithautoincr == 1) && (colType.autoincrement) && (ataRenameColumn.fNewType->fAutoincrement.compare("y") == 0)) - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - - - bs << (uint64_t) nextVal; - uint32_t nullable = 1; - string defaultValue(""); - - if (ataRenameColumn.fConstraints.size() > 0) - { - for (uint32_t j = 0; j < ataRenameColumn.fConstraints.size(); j++) - { - if (ataRenameColumn.fConstraints[j]->fConstraintType == DDL_NOT_NULL) - { - nullable = 0; - break; - } - } - } - - bs << nullable; - - if (ataRenameColumn.fDefaultValue) - defaultValue = ataRenameColumn.fDefaultValue->fValue; - - bs << defaultValue; - sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - throw std::runtime_error("Error while calling getSysCatDBRoot"); - - pmNum = (*dbRootPMMap)[dbRoot]; - - //send to WES to process - try - { - fWEClient->write(bs, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = " Unknown exception caught while updating SYSTABLE."; - } - - if (rc != 0) - throw std::runtime_error(errorMsg); + } } - catch (std::exception& ex) + catch (runtime_error& ex) // write error { - err = ex.what(); - throw std::runtime_error(err); + rc = NETWORK_ERROR; + errorMsg = ex.what(); } catch (...) { - err = "renameColumn:Unknown exception caught"; - throw std::runtime_error(err); + rc = NETWORK_ERROR; + errorMsg = " Unknown exception caught while updating SYSTABLE."; } + if (rc != 0) + throw std::runtime_error(errorMsg); + } + catch (std::exception& ex) + { + err = ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "renameColumn:Unknown exception caught"; + throw std::runtime_error(err); + } } -} +} // namespace ddlpackageprocessor // vim:ts=4 sw=4: #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif diff --git a/dbcon/ddlpackageproc/altertableprocessor.h b/dbcon/ddlpackageproc/altertableprocessor.h index 645b003d0..35c09e296 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.h +++ b/dbcon/ddlpackageproc/altertableprocessor.h @@ -40,123 +40,126 @@ namespace ddlpackageprocessor */ class AlterTableProcessor : public DDLPackageProcessor { -public: - AlterTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process an alter table statement - * - * @param alterTableStmt the AlterTableStatement - */ - EXPORT DDLResult processPackage(ddlpackage::AlterTableStatement& alterTableStmt); - /** @brief add a physical column file - * - * @param result the result of the operation - * @param addColumn the AtaAddColumn object - * @param fTableName the QualifiedName of the table - */ - EXPORT void addColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::ColumnDef* columnDefPtr, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); + public: + AlterTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process an alter table statement + * + * @param alterTableStmt the AlterTableStatement + */ + EXPORT DDLResult processPackage(ddlpackage::AlterTableStatement& alterTableStmt); + /** @brief add a physical column file + * + * @param result the result of the operation + * @param addColumn the AtaAddColumn object + * @param fTableName the QualifiedName of the table + */ + EXPORT void addColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, + ddlpackage::ColumnDef* columnDefPtr, ddlpackage::QualifiedName& fTableName, + const uint64_t uniqueId); - /** @brief drop a column - * - * @param result the result of the operation - * @param ataDropColumn the AtaDropColumn object - * @param fTableName the QualifiedName for the table - */ - EXPORT void dropColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropColumn& ataDropColumn, + /** @brief drop a column + * + * @param result the result of the operation + * @param ataDropColumn the AtaDropColumn object + * @param fTableName the QualifiedName for the table + */ + EXPORT void dropColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, + ddlpackage::AtaDropColumn& ataDropColumn, ddlpackage::QualifiedName& fTableName, + const uint64_t uniqueId); + + /** @brief drop columns + * + * @param result the result of the operation + * @param ataDropColumns the AtaDropColumn object + * @param fTableName the QualifiedName for the table + */ + EXPORT void dropColumns(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, + ddlpackage::AtaDropColumns& ataDropColumns, ddlpackage::QualifiedName& fTableName, + const uint64_t uniqueId); + + /** @brief add table constraint + * + * @param result the result of the operation + * @param ataAddTableConstraint the AtaDropColumn object + * @param fTableName the QualifiedName for the table + */ + EXPORT void addTableConstraint(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::AtaAddTableConstraint& ataAddTableConstraint, + ddlpackage::QualifiedName& fTableName); + + /** @brief set column default + * + * @param result the result of the operation + * @param ataSetColumnDefault the AtaSetColumnDefault object + * @param fTableName the QualifiedName for the table + */ + EXPORT void setColumnDefault(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::AtaSetColumnDefault& ataSetColumnDefault, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); + + /** @brief drop column default + * + * @param result the result of the operation + * @param ataDropColumnDefault the AtaDropColumnDefault object + * @param fTableName the QualifiedName for the table + */ + EXPORT void dropColumnDefault(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, ddlpackage::AtaDropColumnDefault& ataDropColumnDefault, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); + + /** @brief drop table constraint + * + * @param result the result of the operation + * @param ataDropTableConstraint the AtaDropTableConstraint object + * @param fTableName the QualifiedName for the table + */ + EXPORT void dropTableConstraint(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLResult& result, + ddlpackage::AtaDropTableConstraint& ataDropTableConstraint, + ddlpackage::QualifiedName& fTableName); + /** @brief rename a table + * + * @param result the result of the operation + * @param ataRenameTable the AtaRenameTable object + * @param fTableName the QualifiedName for the table + */ + EXPORT void renameTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, + ddlpackage::AtaRenameTable& ataRenameTable, ddlpackage::QualifiedName& fTableName, + const uint64_t uniqueId); + + /** @brief rename a column + * + * @param result the result of the operation + * @param ataRenameColumn the AtaRenameColumn object + * @param fTableName the QualifiedName for the table + */ + EXPORT void renameColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, + ddlpackage::AtaRenameColumn& ataRenameColumn, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - /** @brief drop columns - * - * @param result the result of the operation - * @param ataDropColumns the AtaDropColumn object - * @param fTableName the QualifiedName for the table - */ - EXPORT void dropColumns(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropColumns& ataDropColumns, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId ); + /** @brief change a table autoincrement via a comment + * + * @param result the result of the operation + * @param ataTableComment the AtaTableComment object + * @param fTableName the QualifiedName for the table + */ + EXPORT void tableComment(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, + ddlpackage::AtaTableComment& ataTableComment, + ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - /** @brief add table constraint - * - * @param result the result of the operation - * @param ataAddTableConstraint the AtaDropColumn object - * @param fTableName the QualifiedName for the table - */ - EXPORT void addTableConstraint(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaAddTableConstraint& ataAddTableConstraint, - ddlpackage::QualifiedName& fTableName ); + std::string fTimeZone; - /** @brief set column default - * - * @param result the result of the operation - * @param ataSetColumnDefault the AtaSetColumnDefault object - * @param fTableName the QualifiedName for the table - */ - EXPORT void setColumnDefault(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaSetColumnDefault& ataSetColumnDefault, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId ); - - /** @brief drop column default - * - * @param result the result of the operation - * @param ataDropColumnDefault the AtaDropColumnDefault object - * @param fTableName the QualifiedName for the table - */ - EXPORT void dropColumnDefault(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropColumnDefault& ataDropColumnDefault, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId ); - - /** @brief drop table constraint - * - * @param result the result of the operation - * @param ataDropTableConstraint the AtaDropTableConstraint object - * @param fTableName the QualifiedName for the table - */ - EXPORT void dropTableConstraint(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaDropTableConstraint& ataDropTableConstraint, - ddlpackage::QualifiedName& fTableName ); - /** @brief rename a table - * - * @param result the result of the operation - * @param ataRenameTable the AtaRenameTable object - * @param fTableName the QualifiedName for the table - */ - EXPORT void renameTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, - DDLResult& result, ddlpackage::AtaRenameTable& ataRenameTable, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - - /** @brief rename a column - * - * @param result the result of the operation - * @param ataRenameColumn the AtaRenameColumn object - * @param fTableName the QualifiedName for the table - */ - EXPORT void renameColumn(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaRenameColumn& ataRenameColumn, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - - /** @brief change a table autoincrement via a comment - * - * @param result the result of the operation - * @param ataTableComment the AtaTableComment object - * @param fTableName the QualifiedName for the table - */ - EXPORT void tableComment(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - ddlpackage::AtaTableComment& ataTableComment, - ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - - std::string fTimeZone; - -protected: - void rollBackAlter(const std::string& error, BRM::TxnID txnID, int sessionId, DDLResult& result, uint64_t uniqueId); - -private: + protected: + void rollBackAlter(const std::string& error, BRM::TxnID txnID, int sessionId, DDLResult& result, + uint64_t uniqueId); + private: }; -} //namespace ddlpackageprocessor +} // namespace ddlpackageprocessor #undef EXPORT -#endif //ALTERTABLEPROCESSOR_H +#endif // ALTERTABLEPROCESSOR_H diff --git a/dbcon/ddlpackageproc/createindexprocessor.cpp b/dbcon/ddlpackageproc/createindexprocessor.cpp index 92787f02f..706d4339e 100644 --- a/dbcon/ddlpackageproc/createindexprocessor.cpp +++ b/dbcon/ddlpackageproc/createindexprocessor.cpp @@ -37,446 +37,451 @@ using namespace logging; using namespace BRM; namespace ddlpackageprocessor { - -CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage::CreateIndexStatement& createIndexStmt) +CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage( + ddlpackage::CreateIndexStatement& createIndexStmt) { - /* - get OIDs for the list & tree files - commit the current transaction - start a new transaction - create the index in the metadata - create the index on the WE - end the transaction - */ - SUMMARY_INFO("CreateIndexProcesssor::processPackage"); + /* + get OIDs for the list & tree files + commit the current transaction + start a new transaction + create the index in the metadata + create the index on the WE + end the transaction + */ + SUMMARY_INFO("CreateIndexProcesssor::processPackage"); - DDLResult result; - result.result = NO_ERROR; + DDLResult result; + result.result = NO_ERROR; - DETAIL_INFO(createIndexStmt); + DETAIL_INFO(createIndexStmt); - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - /*Check whether the table exists already. If not, it is assumed from primary key creating. - This is based on the assumption that Front end is already error out if the user trys to - create index on non-existing table. */ - CalpontSystemCatalog::TableName tableName; - tableName.schema = (createIndexStmt.fTableName)->fSchema; - tableName.table = (createIndexStmt.fTableName)->fName; - CalpontSystemCatalog::ROPair roPair; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog( createIndexStmt.fSessionID ); - - try - { - roPair = systemCatalogPtr->tableRID( tableName ); - } - catch (exception& ex) - { - // store primary key name in fPKName - fPKName = createIndexStmt.fIndexName->fName; - return result; - - } - catch (...) - { - return result; - } - - if ( roPair.objnum < 3000 ) - { - return result; - } + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + /*Check whether the table exists already. If not, it is assumed from primary key creating. + This is based on the assumption that Front end is already error out if the user trys to + create index on non-existing table. */ + CalpontSystemCatalog::TableName tableName; + tableName.schema = (createIndexStmt.fTableName)->fSchema; + tableName.table = (createIndexStmt.fTableName)->fName; + CalpontSystemCatalog::ROPair roPair; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(createIndexStmt.fSessionID); + try + { + roPair = systemCatalogPtr->tableRID(tableName); + } + catch (exception& ex) + { + // store primary key name in fPKName fPKName = createIndexStmt.fIndexName->fName; - int err = 0; - - - SQLLogger logger(createIndexStmt.fSql, fDDLLoggingId, createIndexStmt.fSessionID, txnID.id); - - VERBOSE_INFO("Allocating object IDs for columns"); - -// int oidbase = fObjectIDManager.allocOIDs(2); -// fIdxOID.listOID = oidbase; -// fIdxOID.treeOID = ++oidbase; - - - VERBOSE_INFO("Starting a new transaction"); - - ddlpackage::DDL_CONSTRAINTS type = createIndexStmt.fUnique ? ddlpackage::DDL_UNIQUE : ddlpackage::DDL_INVALID_CONSTRAINT; - - VERBOSE_INFO("Writing meta data to SYSINDEX"); - bool multicol = false; - - if ( createIndexStmt.fColumnNames.size() > 1 ) - { - multicol = true; - } - - //validate index columns - CalpontSystemCatalog::TableColName tableColName; - tableColName.schema = (createIndexStmt.fTableName)->fSchema; - tableColName.table = (createIndexStmt.fTableName)->fName; - CalpontSystemCatalog::OID oid; - CalpontSystemCatalog::ColType colType; - ColumnNameList::const_iterator colIter; - int totalWidth = 0; - DDLIndexPopulator pop(&fWriteEngine, &fSessionManager, createIndexStmt.fSessionID, txnID.id, result, - fIdxOID, createIndexStmt.fColumnNames, *createIndexStmt.fTableName, - type, getDebugLevel()); - - if ( multicol) - { - for ( colIter = createIndexStmt.fColumnNames.begin(); colIter != createIndexStmt.fColumnNames.end(); colIter++) - { - tableColName.column = *colIter; - - roPair = systemCatalogPtr->columnRID( tableColName ); - oid = systemCatalogPtr->lookupOID( tableColName ); - colType = systemCatalogPtr->colType (oid ); - totalWidth += (pop.isDictionaryType(colType)) ? 8 : colType.colWidth; - } - - if ( totalWidth > 32 ) - { - stringstream ss; - ss << totalWidth; - DETAIL_INFO("Total indexed column width greater than 32: " + ss.str()); - logging::Message::Args args; - logging::Message message(9); - args.add("Error creating index: "); - args.add("Total indexed column width"); - args.add("greater than 32. "); - message.format( args ); - - result.result = CREATE_ERROR; - result.message = message; - return result; - } - } - - try - { - //writeSysIndexMetaData(createIndexStmt.fSessionID, txnID.id, result, *createIndexStmt.fTableName, type, createIndexStmt.fIndexName->fName, multicol); - - //fIdxOID values are set in writeSysIndexMetaData. - pop.setIdxOID(fIdxOID); - - VERBOSE_INFO("Writing meta data to SYSINDEXCOL"); - //writeSysIndexColMetaData(createIndexStmt.fSessionID, txnID.id, result,*createIndexStmt.fTableName, createIndexStmt.fColumnNames, createIndexStmt.fIndexName->fName ); - - if (createIndexStmt.fUnique) - { - VERBOSE_INFO("Writing column constraint meta data to SYSCONSTRAINT"); - WriteEngine::ColStruct colStruct; - WriteEngine::ColTuple colTuple; - WriteEngine::ColStructList colStructs; - WriteEngine::ColTupleList colTuples; - WriteEngine::ColValueList colValuesList; - WriteEngine::RIDList ridList; - - DDLColumn column; - - CalpontSystemCatalog::TableName sysConsTableName; - sysConsTableName.schema = CALPONT_SCHEMA; - sysConsTableName.table = SYSCONSTRAINT_TABLE; - - bool isNull = false; - int error = 0; - - // get the columns for the SYSCONSTRAINT table - ColumnList sysConsColumns; - ColumnList::const_iterator sysCons_iterator; - getColumnsForTable(createIndexStmt.fSessionID, sysConsTableName.schema, sysConsTableName.table, sysConsColumns); - sysCons_iterator = sysConsColumns.begin(); - std::string idxData; - - while ( sysCons_iterator != sysConsColumns.end() ) - { - column = *sysCons_iterator; - isNull = false; - - if (CONSTRAINTNAME_COL == column.tableColName.column) - { - idxData = createIndexStmt.fIndexName->fName; - colTuple.data = idxData; - } - else if (SCHEMA_COL == column.tableColName.column) - { - idxData = (createIndexStmt.fTableName)->fSchema; - colTuple.data = idxData; - } - else if (TABLENAME_COL == column.tableColName.column) - { - idxData = (createIndexStmt.fTableName)->fName; - colTuple.data = idxData; - } - else if (CONSTRAINTTYPE_COL == column.tableColName.column) - { - std::string consType; - char constraint_type = getConstraintCode(type); - consType += constraint_type; - colTuple.data = consType; - } - else if (CONSTRAINTPRIM_COL == column.tableColName.column) - { - colTuple.data =column.colType.getNullValueForType(); - isNull = true; - } - else if (CONSTRAINTTEXT_COL == column.tableColName.column) - { - colTuple.data = column.colType.getNullValueForType(); - isNull = true; - } - else if (INDEXNAME_COL == column.tableColName.column) - { - idxData = createIndexStmt.fIndexName->fName; - colTuple.data = idxData; - } - else - { - colTuple.data = column.colType.getNullValueForType(); - isNull = true; - } - - colStruct.dataOid = column.oid; - - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; - colStruct.colDataType = column.colType.colDataType; - - if (column.colType.colWidth > 8 && !isNull) - { - colTuple.data = tokenizeData(txnID.id, result, column.colType, colTuple.data); - } - - colStructs.push_back( colStruct ); - - colTuples.push_back( colTuple ); - - colValuesList.push_back( colTuples ); - - colTuples.pop_back(); - ++sysCons_iterator; - } - - if (colStructs.size() != 0) - { - //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); - //error = fWriteEngine.insertColumnRec( txnID.id, colStructs, colValuesList, ridList ); - if ( error != WriteEngine::NO_ERROR ) - { - - return rollBackCreateIndex(errorString( "WE: Error inserting Column Record: ", error), txnID, createIndexStmt.fSessionID); -// logging::Message::Args args; -// logging::Message message(9); -// args.add("Error updating: "); -// args.add("calpont.sysconstraint"); -// args.add("error number: "); -// args.add( error ); -// message.format( args ); -// -// result.result = CREATE_ERROR; -// result.message = message; - } - else - { - result.result = NO_ERROR; - } - } - - VERBOSE_INFO("Writing column constraint meta data to SYSCONSTRAINTCOL"); - WriteEngine::ColStruct colStructCol; - WriteEngine::ColTuple colTupleCol; - WriteEngine::ColStructList colStructsCol; - WriteEngine::ColTupleList colTuplesCol; - WriteEngine::ColValueList colValuesListCol; - CalpontSystemCatalog::TableName sysConsColTableName; - sysConsColTableName.schema = CALPONT_SCHEMA; - sysConsColTableName.table = SYSCONSTRAINTCOL_TABLE; - colValuesList.clear(); - colTuples.clear(); - isNull = false; - error = 0; - // get the columns for the SYSCONSTRAINTCOL table - ColumnList sysConsColColumns; - ColumnList::const_iterator sysConsCol_iterator; - getColumnsForTable(createIndexStmt.fSessionID, sysConsColTableName.schema, sysConsColTableName.table, sysConsColColumns); - // write sysconstraintcol - sysConsCol_iterator = sysConsColColumns.begin(); - std::string colData; - - while ( sysConsCol_iterator != sysConsColColumns.end() ) - { - column = *sysConsCol_iterator; - - isNull = false; - - if (SCHEMA_COL == column.tableColName.column) - { - colData = (createIndexStmt.fTableName)->fSchema; - colTupleCol.data = colData; - } - else if (TABLENAME_COL == column.tableColName.column) - { - colData = (createIndexStmt.fTableName)->fName; - colTupleCol.data = colData; - } - else if (COLNAME_COL == column.tableColName.column) - { - colData = createIndexStmt.fColumnNames[0]; - colTupleCol.data = colData; - - } - else if (CONSTRAINTNAME_COL == column.tableColName.column) - { - colData = createIndexStmt.fIndexName->fName; - colTupleCol.data = colData; - } - else - { - colTupleCol.data = column.colType.getNullValueForType(); - isNull = true; - } - - colStructCol.dataOid = column.oid; - colStructCol.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStructCol.tokenFlag = false; - colStructCol.tokenFlag = column.colType.colWidth > 8 ? true : false; - colStructCol.colDataType = column.colType.colDataType; - - if (column.colType.colWidth > 8 && !isNull) - { - colTupleCol.data = tokenizeData(txnID.id, result, column.colType, colTupleCol.data); - } - - colStructsCol.push_back( colStructCol ); - - colTuplesCol.push_back( colTupleCol ); - - colValuesListCol.push_back( colTuplesCol ); - - colTuplesCol.pop_back(); - - ++sysConsCol_iterator; - } - - if (colStructsCol.size() != 0) - { - //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); - //error = fWriteEngine.insertColumnRec( txnID.id, colStructsCol, colValuesListCol, ridList ); - if ( error != WriteEngine::NO_ERROR ) - { - return rollBackCreateIndex(errorString( "WE: Error inserting Column Record: ", error), txnID, createIndexStmt.fSessionID); - - /* logging::Message::Args args; - logging::Message message(9); - args.add("Error updating: "); - args.add("calpont.sysconstraintcol"); - args.add("error number: "); - args.add( error ); - message.format( args ); - - result.result = CREATE_ERROR; - result.message = message;*/ - } - else - { - result.result = NO_ERROR; - } - } - } - - VERBOSE_INFO("Creating index files"); - err = fWriteEngine.createIndex( txnID.id, fIdxOID.treeOID, fIdxOID.listOID ); - - if (err) - { - return rollBackCreateIndex(errorString("Write engine failed to create the new index. ", err), txnID, createIndexStmt.fSessionID); - } - - // new if BULK_LOAD close - err = pop.populateIndex(result); - - if ( err ) - { - return rollBackCreateIndex(errorString("Failed to populate index with current data. ", err), txnID, createIndexStmt.fSessionID); - } - - - // Log the DDL statement. - logging::logDDL(createIndexStmt.fSessionID, txnID.id, createIndexStmt.fSql, createIndexStmt.fOwner); - - DETAIL_INFO("Commiting transaction"); - err = fWriteEngine.commit( txnID.id ); - - if (err) - { - return rollBackCreateIndex(errorString("Failed to commit the create index transaction. ", err), txnID, createIndexStmt.fSessionID); - } - - fSessionManager.committed(txnID); -// original if BULK_LOAD close } - } // try - - catch (exception& ex) - { - result = rollBackCreateIndex(ex.what(), txnID, createIndexStmt.fSessionID); - } - catch (...) - { - string msg("CreateIndexProcessor::processPackage: caught unknown exception!"); - result = rollBackCreateIndex(msg, txnID, createIndexStmt.fSessionID); - } - return result; + } + catch (...) + { + return result; + } + + if (roPair.objnum < 3000) + { + return result; + } + + fPKName = createIndexStmt.fIndexName->fName; + int err = 0; + + SQLLogger logger(createIndexStmt.fSql, fDDLLoggingId, createIndexStmt.fSessionID, txnID.id); + + VERBOSE_INFO("Allocating object IDs for columns"); + + // int oidbase = fObjectIDManager.allocOIDs(2); + // fIdxOID.listOID = oidbase; + // fIdxOID.treeOID = ++oidbase; + + VERBOSE_INFO("Starting a new transaction"); + + ddlpackage::DDL_CONSTRAINTS type = + createIndexStmt.fUnique ? ddlpackage::DDL_UNIQUE : ddlpackage::DDL_INVALID_CONSTRAINT; + + VERBOSE_INFO("Writing meta data to SYSINDEX"); + bool multicol = false; + + if (createIndexStmt.fColumnNames.size() > 1) + { + multicol = true; + } + + // validate index columns + CalpontSystemCatalog::TableColName tableColName; + tableColName.schema = (createIndexStmt.fTableName)->fSchema; + tableColName.table = (createIndexStmt.fTableName)->fName; + CalpontSystemCatalog::OID oid; + CalpontSystemCatalog::ColType colType; + ColumnNameList::const_iterator colIter; + int totalWidth = 0; + DDLIndexPopulator pop(&fWriteEngine, &fSessionManager, createIndexStmt.fSessionID, txnID.id, result, + fIdxOID, createIndexStmt.fColumnNames, *createIndexStmt.fTableName, type, + getDebugLevel()); + + if (multicol) + { + for (colIter = createIndexStmt.fColumnNames.begin(); colIter != createIndexStmt.fColumnNames.end(); + colIter++) + { + tableColName.column = *colIter; + + roPair = systemCatalogPtr->columnRID(tableColName); + oid = systemCatalogPtr->lookupOID(tableColName); + colType = systemCatalogPtr->colType(oid); + totalWidth += (pop.isDictionaryType(colType)) ? 8 : colType.colWidth; + } + + if (totalWidth > 32) + { + stringstream ss; + ss << totalWidth; + DETAIL_INFO("Total indexed column width greater than 32: " + ss.str()); + logging::Message::Args args; + logging::Message message(9); + args.add("Error creating index: "); + args.add("Total indexed column width"); + args.add("greater than 32. "); + message.format(args); + + result.result = CREATE_ERROR; + result.message = message; + return result; + } + } + + try + { + // writeSysIndexMetaData(createIndexStmt.fSessionID, txnID.id, result, *createIndexStmt.fTableName, type, + // createIndexStmt.fIndexName->fName, multicol); + + // fIdxOID values are set in writeSysIndexMetaData. + pop.setIdxOID(fIdxOID); + + VERBOSE_INFO("Writing meta data to SYSINDEXCOL"); + // writeSysIndexColMetaData(createIndexStmt.fSessionID, txnID.id, result,*createIndexStmt.fTableName, + // createIndexStmt.fColumnNames, createIndexStmt.fIndexName->fName ); + + if (createIndexStmt.fUnique) + { + VERBOSE_INFO("Writing column constraint meta data to SYSCONSTRAINT"); + WriteEngine::ColStruct colStruct; + WriteEngine::ColTuple colTuple; + WriteEngine::ColStructList colStructs; + WriteEngine::ColTupleList colTuples; + WriteEngine::ColValueList colValuesList; + WriteEngine::RIDList ridList; + + DDLColumn column; + + CalpontSystemCatalog::TableName sysConsTableName; + sysConsTableName.schema = CALPONT_SCHEMA; + sysConsTableName.table = SYSCONSTRAINT_TABLE; + + bool isNull = false; + int error = 0; + + // get the columns for the SYSCONSTRAINT table + ColumnList sysConsColumns; + ColumnList::const_iterator sysCons_iterator; + getColumnsForTable(createIndexStmt.fSessionID, sysConsTableName.schema, sysConsTableName.table, + sysConsColumns); + sysCons_iterator = sysConsColumns.begin(); + std::string idxData; + + while (sysCons_iterator != sysConsColumns.end()) + { + column = *sysCons_iterator; + isNull = false; + + if (CONSTRAINTNAME_COL == column.tableColName.column) + { + idxData = createIndexStmt.fIndexName->fName; + colTuple.data = idxData; + } + else if (SCHEMA_COL == column.tableColName.column) + { + idxData = (createIndexStmt.fTableName)->fSchema; + colTuple.data = idxData; + } + else if (TABLENAME_COL == column.tableColName.column) + { + idxData = (createIndexStmt.fTableName)->fName; + colTuple.data = idxData; + } + else if (CONSTRAINTTYPE_COL == column.tableColName.column) + { + std::string consType; + char constraint_type = getConstraintCode(type); + consType += constraint_type; + colTuple.data = consType; + } + else if (CONSTRAINTPRIM_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + isNull = true; + } + else if (CONSTRAINTTEXT_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + isNull = true; + } + else if (INDEXNAME_COL == column.tableColName.column) + { + idxData = createIndexStmt.fIndexName->fName; + colTuple.data = idxData; + } + else + { + colTuple.data = column.colType.getNullValueForType(); + isNull = true; + } + + colStruct.dataOid = column.oid; + + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; + colStruct.colDataType = column.colType.colDataType; + + if (column.colType.colWidth > 8 && !isNull) + { + colTuple.data = tokenizeData(txnID.id, result, column.colType, colTuple.data); + } + + colStructs.push_back(colStruct); + + colTuples.push_back(colTuple); + + colValuesList.push_back(colTuples); + + colTuples.pop_back(); + ++sysCons_iterator; + } + + if (colStructs.size() != 0) + { + // fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); + // error = fWriteEngine.insertColumnRec( txnID.id, colStructs, colValuesList, ridList ); + if (error != WriteEngine::NO_ERROR) + { + return rollBackCreateIndex(errorString("WE: Error inserting Column Record: ", error), txnID, + createIndexStmt.fSessionID); + // logging::Message::Args args; + // logging::Message message(9); + // args.add("Error updating: "); + // args.add("calpont.sysconstraint"); + // args.add("error number: "); + // args.add( error ); + // message.format( args ); + // + // result.result = CREATE_ERROR; + // result.message = message; + } + else + { + result.result = NO_ERROR; + } + } + + VERBOSE_INFO("Writing column constraint meta data to SYSCONSTRAINTCOL"); + WriteEngine::ColStruct colStructCol; + WriteEngine::ColTuple colTupleCol; + WriteEngine::ColStructList colStructsCol; + WriteEngine::ColTupleList colTuplesCol; + WriteEngine::ColValueList colValuesListCol; + CalpontSystemCatalog::TableName sysConsColTableName; + sysConsColTableName.schema = CALPONT_SCHEMA; + sysConsColTableName.table = SYSCONSTRAINTCOL_TABLE; + colValuesList.clear(); + colTuples.clear(); + isNull = false; + error = 0; + // get the columns for the SYSCONSTRAINTCOL table + ColumnList sysConsColColumns; + ColumnList::const_iterator sysConsCol_iterator; + getColumnsForTable(createIndexStmt.fSessionID, sysConsColTableName.schema, sysConsColTableName.table, + sysConsColColumns); + // write sysconstraintcol + sysConsCol_iterator = sysConsColColumns.begin(); + std::string colData; + + while (sysConsCol_iterator != sysConsColColumns.end()) + { + column = *sysConsCol_iterator; + + isNull = false; + + if (SCHEMA_COL == column.tableColName.column) + { + colData = (createIndexStmt.fTableName)->fSchema; + colTupleCol.data = colData; + } + else if (TABLENAME_COL == column.tableColName.column) + { + colData = (createIndexStmt.fTableName)->fName; + colTupleCol.data = colData; + } + else if (COLNAME_COL == column.tableColName.column) + { + colData = createIndexStmt.fColumnNames[0]; + colTupleCol.data = colData; + } + else if (CONSTRAINTNAME_COL == column.tableColName.column) + { + colData = createIndexStmt.fIndexName->fName; + colTupleCol.data = colData; + } + else + { + colTupleCol.data = column.colType.getNullValueForType(); + isNull = true; + } + + colStructCol.dataOid = column.oid; + colStructCol.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStructCol.tokenFlag = false; + colStructCol.tokenFlag = column.colType.colWidth > 8 ? true : false; + colStructCol.colDataType = column.colType.colDataType; + + if (column.colType.colWidth > 8 && !isNull) + { + colTupleCol.data = tokenizeData(txnID.id, result, column.colType, colTupleCol.data); + } + + colStructsCol.push_back(colStructCol); + + colTuplesCol.push_back(colTupleCol); + + colValuesListCol.push_back(colTuplesCol); + + colTuplesCol.pop_back(); + + ++sysConsCol_iterator; + } + + if (colStructsCol.size() != 0) + { + // fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); + // error = fWriteEngine.insertColumnRec( txnID.id, colStructsCol, colValuesListCol, ridList ); + if (error != WriteEngine::NO_ERROR) + { + return rollBackCreateIndex(errorString("WE: Error inserting Column Record: ", error), txnID, + createIndexStmt.fSessionID); + + /* logging::Message::Args args; + logging::Message message(9); + args.add("Error updating: "); + args.add("calpont.sysconstraintcol"); + args.add("error number: "); + args.add( error ); + message.format( args ); + + result.result = CREATE_ERROR; + result.message = message;*/ + } + else + { + result.result = NO_ERROR; + } + } + } + + VERBOSE_INFO("Creating index files"); + err = fWriteEngine.createIndex(txnID.id, fIdxOID.treeOID, fIdxOID.listOID); + + if (err) + { + return rollBackCreateIndex(errorString("Write engine failed to create the new index. ", err), txnID, + createIndexStmt.fSessionID); + } + + // new if BULK_LOAD close + err = pop.populateIndex(result); + + if (err) + { + return rollBackCreateIndex(errorString("Failed to populate index with current data. ", err), txnID, + createIndexStmt.fSessionID); + } + + // Log the DDL statement. + logging::logDDL(createIndexStmt.fSessionID, txnID.id, createIndexStmt.fSql, createIndexStmt.fOwner); + + DETAIL_INFO("Commiting transaction"); + err = fWriteEngine.commit(txnID.id); + + if (err) + { + return rollBackCreateIndex(errorString("Failed to commit the create index transaction. ", err), txnID, + createIndexStmt.fSessionID); + } + + fSessionManager.committed(txnID); + // original if BULK_LOAD close } + } // try + + catch (exception& ex) + { + result = rollBackCreateIndex(ex.what(), txnID, createIndexStmt.fSessionID); + } + catch (...) + { + string msg("CreateIndexProcessor::processPackage: caught unknown exception!"); + result = rollBackCreateIndex(msg, txnID, createIndexStmt.fSessionID); + } + + return result; } -string CreateIndexProcessor::errorString(const string& msg, int error) +string CreateIndexProcessor::errorString(const string& msg, int error) { - WriteEngine::WErrorCodes ec; - return string(msg + ec.errorString(error)); + WriteEngine::WErrorCodes ec; + return string(msg + ec.errorString(error)); } - -CreateIndexProcessor::DDLResult CreateIndexProcessor::rollBackCreateIndex(const string& error, BRM::TxnID& txnID, int sessionId) +CreateIndexProcessor::DDLResult CreateIndexProcessor::rollBackCreateIndex(const string& error, + BRM::TxnID& txnID, int sessionId) { - cerr << "CreatetableProcessor::processPackage: " << error << endl; - DETAIL_INFO(error); - logging::Message::Args args; - logging::Message message(1); - args.add("Create Index Failed: "); - args.add( error ); - args.add(""); - args.add(""); - message.format( args ); - DDLResult result; - result.result = CREATE_ERROR; - result.message = message; - rollBackIndex(txnID, sessionId); - return result; + cerr << "CreatetableProcessor::processPackage: " << error << endl; + DETAIL_INFO(error); + logging::Message::Args args; + logging::Message message(1); + args.add("Create Index Failed: "); + args.add(error); + args.add(""); + args.add(""); + message.format(args); + DDLResult result; + result.result = CREATE_ERROR; + result.message = message; + rollBackIndex(txnID, sessionId); + return result; } void CreateIndexProcessor::rollBackIndex(BRM::TxnID& txnID, int sessionId) { - fWriteEngine.rollbackTran(txnID.id, sessionId); - fWriteEngine.dropIndex(txnID.id, fIdxOID.listOID, fIdxOID.treeOID); + fWriteEngine.rollbackTran(txnID.id, sessionId); + fWriteEngine.dropIndex(txnID.id, fIdxOID.listOID, fIdxOID.treeOID); - try - { - //execplan::ObjectIDManager fObjectIDManager; - //fObjectIDManager.returnOIDs(fIdxOID.treeOID, fIdxOID.listOID); - } - catch ( exception& ex ) - { + try + { + // execplan::ObjectIDManager fObjectIDManager; + // fObjectIDManager.returnOIDs(fIdxOID.treeOID, fIdxOID.listOID); + } + catch (exception& ex) + { + } + catch (...) + { + } - } - catch (... ) - { } - - fSessionManager.rolledback(txnID); + fSessionManager.rolledback(txnID); } - -} +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/createindexprocessor.h b/dbcon/ddlpackageproc/createindexprocessor.h index a09290556..8e5920b88 100644 --- a/dbcon/ddlpackageproc/createindexprocessor.h +++ b/dbcon/ddlpackageproc/createindexprocessor.h @@ -35,20 +35,20 @@ namespace ddlpackageprocessor */ class CreateIndexProcessor : public DDLPackageProcessor { -public: - /** @brief process a create index statement - * - * @param createIndexStmt the create index statement - */ - DDLResult processPackage(ddlpackage::CreateIndexStatement& createIndexStmt); + public: + /** @brief process a create index statement + * + * @param createIndexStmt the create index statement + */ + DDLResult processPackage(ddlpackage::CreateIndexStatement& createIndexStmt); -protected: - DDLResult rollBackCreateIndex(const std::string& error, BRM::TxnID& txnID, int sessionId); - void rollBackIndex(BRM::TxnID& txnID); - std::string errorString(const std::string& msg, int error); -private: + protected: + DDLResult rollBackCreateIndex(const std::string& error, BRM::TxnID& txnID, int sessionId); + void rollBackIndex(BRM::TxnID& txnID); + std::string errorString(const std::string& msg, int error); + private: }; -} //namespace ddlpackageprocessor -#endif //CREATEINDEXPROCESSOR_H +} // namespace ddlpackageprocessor +#endif // CREATEINDEXPROCESSOR_H diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index e8f615c93..f51d03958 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -47,792 +47,787 @@ using namespace logging; namespace ddlpackageprocessor { - CreateTableProcessor::DDLResult CreateTableProcessor::processPackage( ddlpackage::CreateTableStatement& createTableStmt) { - SUMMARY_INFO("CreateTableProcessor::processPackage"); - - DDLResult result; - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - result.result = NO_ERROR; - int rc1 = 0; - rc1 = fDbrm->isReadWrite(); - - if (rc1 != 0 ) - { - Message::Args args; - Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - DETAIL_INFO(createTableStmt); - ddlpackage::TableDef& tableDef = *createTableStmt.fTableDef; - //If schema = CALPONTSYS, do not create table - - if (tableDef.fQualifiedName->fSchema == CALPONT_SCHEMA) - { - //release the transaction - fSessionManager.rolledback(txnID); - return result; - } - - // Commit current transaction. - // all DDL statements cause an implicut commit - VERBOSE_INFO("Getting current txnID"); - - //Check whether the table is existed already - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(createTableStmt.fSessionID); - execplan::CalpontSystemCatalog::TableName tableName; - tableName.schema = tableDef.fQualifiedName->fSchema; - tableName.table = tableDef.fQualifiedName->fName; - execplan::CalpontSystemCatalog::ROPair roPair; - roPair.objnum = 0; - ByteStream::byte rc = 0; - - /** @Bug 217 */ - /** @Bug 225 */ - try - { - roPair = systemCatalogPtr->tableRID(tableName); - } - catch (IDBExcept& ie) - { - // TODO: What is and is not an error here? - if (ie.errorCode() == ERR_DATA_OFFLINE) - { - //release transaction - fSessionManager.rolledback(txnID); - // Return the error for display to user - Message::Args args; - Message message(9); - args.add(ie.what()); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - return result; - } - else if ( ie.errorCode() == ERR_TABLE_NOT_IN_CATALOG) - { - roPair.objnum = 0; - } - else //error out - { - //release transaction - fSessionManager.rolledback(txnID); - // Return the error for display to user - Message::Args args; - Message message(9); - args.add(ie.what()); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - return result; - } - } - catch (std::exception& ex) //error out - { - //release transaction - fSessionManager.rolledback(txnID); - // Return the error for display to user - Message::Args args; - Message message(9); - args.add(ex.what()); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - return result; - } - catch (...) //error out - { - //release transaction - fSessionManager.rolledback(txnID); - // Return the error for display to user - Message::Args args; - Message message(9); - args.add("Unknown exception caught when checking if the table name is already in use."); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - return result; - } - - //This is a current db bug, it should not turn OID is it cannot find - if (roPair.objnum >= 3000) - { -#ifdef _MSC_VER - //FIXME: Why do we need to do this??? - systemCatalogPtr->flushCache(); - - try - { - roPair = systemCatalogPtr->tableRID(tableName); - } - catch (...) - { - roPair.objnum = 0; - } - - if (roPair.objnum < 3000) - goto keepGoing; - -#endif - Message::Args args; - Message message(9); - args.add("Internal create table error for"); - args.add(tableName.toString()); - args.add(": table already exists"); - args.add("(your schema is probably out-of-sync)"); - message.format(args); - - result.result = CREATE_ERROR; - result.message = message; - //release the transaction - fSessionManager.rolledback(txnID); - return result; - } - -#ifdef _MSC_VER -keepGoing: -#endif - // Start a new transaction - VERBOSE_INFO("Starting a new transaction"); - - string stmt = createTableStmt.fSql + "|" + tableDef.fQualifiedName->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, createTableStmt.fSessionID, txnID.id); - - - std::string err; - execplan::ObjectIDManager fObjectIDManager; - OamCache* oamcache = OamCache::makeOamCache(); - string errorMsg; - //get a unique number - uint64_t uniqueId = 0; - - //Bug 5070. Added exception handling - try - { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - Message::Args args; - Message message(9); - args.add(ex.what()); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - catch ( ... ) - { - Message::Args args; - Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = CREATE_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - fWEClient->addQueue(uniqueId); - - try - { - //Allocate tableoid table identification - VERBOSE_INFO("Allocating object ID for table"); - // Allocate a object ID for each column we are about to create - VERBOSE_INFO("Allocating object IDs for columns"); - uint32_t numColumns = tableDef.fColumns.size(); - uint32_t numDictCols = 0; - - for (unsigned i = 0; i < numColumns; i++) - { - int dataType; - dataType = convertDataType(tableDef.fColumns[i]->fType->fType); - - if ( (dataType == CalpontSystemCatalog::CHAR && tableDef.fColumns[i]->fType->fLength > 8) || - (dataType == CalpontSystemCatalog::VARCHAR && tableDef.fColumns[i]->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::VARBINARY && tableDef.fColumns[i]->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::BLOB && tableDef.fColumns[i]->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::TEXT && tableDef.fColumns[i]->fType->fLength > 7) ) - numDictCols++; - } - - fStartingColOID = fObjectIDManager.allocOIDs(numColumns + numDictCols + 1); //include column, oids,dictionary oids and tableoid -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Create table allocOIDs got the starting oid " << fStartingColOID << endl; -#endif - - if (fStartingColOID < 0) - { - result.result = CREATE_ERROR; - errorMsg = "Error in getting objectid from oidmanager."; - Message::Args args; - Message message(9); - args.add("(1)Create table failed due to "); - args.add(errorMsg); - message.format(args); - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - // Write the table metadata to the systemtable - VERBOSE_INFO("Writing meta data to SYSTABLE"); - ByteStream bytestream; - bytestream << (ByteStream::byte)WE_SVR_WRITE_SYSTABLE; - bytestream << uniqueId; - bytestream << (uint32_t) createTableStmt.fSessionID; - bytestream << (uint32_t)txnID.id; - bytestream << (uint32_t)fStartingColOID; - bytestream << (uint32_t)createTableStmt.fTableWithAutoi; - uint16_t dbRoot; - BRM::OID_t sysOid = 1001; - //Find out where systable is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - { - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("Error while calling getSysCatDBRoot "); - args.add(errorMsg); - message.format(args); - result.message = message; - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - int pmNum = 1; - bytestream << (uint32_t)dbRoot; - tableDef.serialize(bytestream); - boost::shared_ptr bsIn; - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; - // MCOL-66 The DBRM can't handle concurrent DDL - boost::mutex::scoped_lock lk(dbrmMutex); - - try - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " create table sending We_SVR_WRITE_SYSTABLE to pm " << pmNum << endl; -#endif - fWEClient->write(bytestream, (unsigned)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - errorMsg.clear(); - *bsIn >> errorMsg; -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << "Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; -#endif - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " create table got exception" << ex.what() << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; -#ifdef IDB_DDL_DEBUG - cout << "create table got unknown exception" << endl; -#endif - } - - if (rc != 0) - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; -#endif - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("(2)Create table failed due to "); - args.add(errorMsg); - message.format( args ); - result.message = message; - - if (rc != NETWORK_ERROR) - { - rollBackTransaction( uniqueId, txnID, createTableStmt.fSessionID ); //What to do with the error code - } - - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - VERBOSE_INFO("Writing meta data to SYSCOLUMN"); - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATE_SYSCOLUMN; - bytestream << uniqueId; - bytestream << (uint32_t) createTableStmt.fSessionID; - bytestream << (uint32_t)txnID.id; - bytestream << numColumns; - - for (unsigned i = 0; i < numColumns; ++i) - { - bytestream << (uint32_t)(fStartingColOID + i + 1); - } - - bytestream << numDictCols; - - for (unsigned i = 0; i < numDictCols; ++i) - { - bytestream << (uint32_t)(fStartingColOID + numColumns + i + 1); - } - - uint8_t alterFlag = 0; - int colPos = 0; - bytestream << (ByteStream::byte)alterFlag; - bytestream << (uint32_t)colPos; - - sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - { - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("Error while calling getSysCatDBRoot "); - args.add(errorMsg); - message.format(args); - result.message = message; - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - bytestream << (uint32_t)dbRoot; - tableDef.serialize(bytestream); - pmNum = (*dbRootPMMap)[dbRoot]; - - try - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " create table sending WE_SVR_WRITE_CREATE_SYSCOLUMN to pm " << pmNum << endl; -#endif - fWEClient->write(bytestream, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - errorMsg.clear(); - *bsIn >> errorMsg; -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << "Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; -#endif - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " create table got exception" << ex.what() << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " create table got unknown exception" << endl; -#endif - } - - if (rc != 0) - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Create table WE_SVR_WRITE_CREATE_SYSCOLUMN: " << errorMsg << endl; -#endif - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("(3)Create table failed due to "); - args.add(errorMsg); - message.format( args ); - result.message = message; - - if (rc != NETWORK_ERROR) - { - rollBackTransaction( uniqueId, txnID, createTableStmt.fSessionID ); //What to do with the error code - } - - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - - //Get the number of tables in the database, the current table is included. - int tableCount = systemCatalogPtr->getTableCount(); - - //Calculate which dbroot the columns should start - DBRootConfigList dbRootList = oamcache->getDBRootNums(); - - uint16_t useDBRootIndex = tableCount % dbRootList.size(); - //Find out the dbroot# corresponding the useDBRootIndex from oam - uint16_t useDBRoot = dbRootList[useDBRootIndex]; - - VERBOSE_INFO("Creating column files"); - ColumnDef* colDefPtr; - ddlpackage::ColumnDefList tableDefCols = tableDef.fColumns; - ColumnDefList::const_iterator iter = tableDefCols.begin(); - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES; - bytestream << uniqueId; - bytestream << (uint32_t)txnID.id; - bytestream << (numColumns + numDictCols); - unsigned colNum = 0; - unsigned dictNum = 0; - - while (iter != tableDefCols.end()) - { - colDefPtr = *iter; - - CalpontSystemCatalog::ColDataType dataType = convertDataType(colDefPtr->fType->fType); - - if (dataType == CalpontSystemCatalog::DECIMAL || - dataType == CalpontSystemCatalog::UDECIMAL) - { - if (colDefPtr->fType->fPrecision == -1 || colDefPtr->fType->fPrecision == 0) - { - colDefPtr->fType->fLength = 8; - } - else if ((colDefPtr->fType->fPrecision > 0) && (colDefPtr->fType->fPrecision < 3)) - { - colDefPtr->fType->fLength = 1; - } - - else if (colDefPtr->fType->fPrecision < 5 && (colDefPtr->fType->fPrecision > 2)) - { - colDefPtr->fType->fLength = 2; - } - else if (colDefPtr->fType->fPrecision > 4 && colDefPtr->fType->fPrecision < 10) - { - colDefPtr->fType->fLength = 4; - } - else if (colDefPtr->fType->fPrecision > 9 && colDefPtr->fType->fPrecision < 19) - { - colDefPtr->fType->fLength = 8; - } - else if (colDefPtr->fType->fPrecision > 18 && colDefPtr->fType->fPrecision < 39) - { - colDefPtr->fType->fLength = 16; - } - } - - bytestream << (fStartingColOID + (colNum++) + 1); - bytestream << (uint8_t) dataType; - bytestream << (uint8_t) false; - - bytestream << (uint32_t) colDefPtr->fType->fLength; - bytestream << (uint16_t) useDBRoot; - bytestream << (uint32_t) colDefPtr->fType->fCompressiontype; - - if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || - (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::TEXT && colDefPtr->fType->fLength > 7) ) - { - bytestream << (uint32_t) (fStartingColOID + numColumns + (dictNum++) + 1); - bytestream << (uint8_t) dataType; - bytestream << (uint8_t) true; - bytestream << (uint32_t) colDefPtr->fType->fLength; - bytestream << (uint16_t) useDBRoot; - bytestream << (uint32_t) colDefPtr->fType->fCompressiontype; - } - - ++iter; - } - - //@Bug 4176. save oids to a log file for cleanup after fail over. - std::vector oidList; - - for (unsigned i = 0; i < numColumns; ++i) - { - oidList.push_back(fStartingColOID + i + 1); - } - - bytestream << numDictCols; - - for (unsigned i = 0; i < numDictCols; ++i) - { - oidList.push_back(fStartingColOID + numColumns + i + 1); - } - - try - { - createWriteDropLogFile( fStartingColOID, uniqueId, oidList ); - } - catch (std::exception& ex) - { - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("(4)Create table failed due to "); - args.add(ex.what()); - message.format( args ); - result.message = message; - - if (rc != NETWORK_ERROR) - { - rollBackTransaction( uniqueId, txnID, createTableStmt.fSessionID ); //What to do with the error code - } - - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - pmNum = (*dbRootPMMap)[useDBRoot]; - - try - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " create table sending WE_SVR_WRITE_CREATETABLEFILES to pm " << pmNum << endl; -#endif - fWEClient->write(bytestream, pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - errorMsg.clear(); - *bsIn >> errorMsg; -#ifdef IDB_DDL_DEBUG - cout << "Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; -#endif - } - - break; - } - } - - if (rc != 0) - { - //drop the newly created files - bytestream.restart(); - bytestream << (ByteStream::byte) WE_SVR_WRITE_DROPFILES; - bytestream << uniqueId; - bytestream << (uint32_t)(numColumns + numDictCols); - - for (unsigned i = 0; i < (numColumns + numDictCols); i++) - { - bytestream << (uint32_t)(fStartingColOID + i + 1); - } - - fWEClient->write(bytestream, pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - break; - } - else - { - break; - } - } - - //@Bug 5464. Delete from extent map. - fDbrm->deleteOIDs(oidList); - - } - } - catch (runtime_error&) - { - errorMsg = "Lost connection to Write Engine Server"; - } - - if (rc != 0) - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; -#endif - rollBackTransaction( uniqueId, txnID, createTableStmt.fSessionID); //What to do with the error code - fSessionManager.rolledback(txnID); - } - else - { - commitTransaction(uniqueId, txnID); - fSessionManager.committed(txnID); - fWEClient->removeQueue(uniqueId); - deleteLogFile(DROPTABLE_LOG, fStartingColOID, uniqueId); - } - - // Log the DDL statement. - logDDL(createTableStmt.fSessionID, txnID.id, createTableStmt.fSql, createTableStmt.fOwner); - } - catch (std::exception& ex) - { - result.result = CREATE_ERROR; - Message::Args args; - Message message(9); - args.add("(5)Create table failed due to "); - args.add(ex.what()); - message.format( args ); - result.message = message; - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - return result; - } - - //fWEClient->removeQueue(uniqueId); - if (rc != 0) - { - result.result = CREATE_ERROR; - Message::Args args; - Message message(9); - args.add("(6)Create table failed due to "); - args.add(errorMsg); - message.format( args ); - result.message = message; - } - - return result; -} - -void CreateTableProcessor::rollBackCreateTable(const string& error, BRM::TxnID txnID, int sessionId, - ddlpackage::TableDef& tableDef, DDLResult& result) -{ - cerr << "CreatetableProcessor::processPackage: " << error << endl; - + SUMMARY_INFO("CreateTableProcessor::processPackage"); + + DDLResult result; + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + result.result = NO_ERROR; + int rc1 = 0; + rc1 = fDbrm->isReadWrite(); + + if (rc1 != 0) + { Message::Args args; - Message message(1); - args.add("(7)Create table Failed: "); - args.add(error); - args.add(""); - args.add(""); + Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + DETAIL_INFO(createTableStmt); + ddlpackage::TableDef& tableDef = *createTableStmt.fTableDef; + // If schema = CALPONTSYS, do not create table + + if (tableDef.fQualifiedName->fSchema == CALPONT_SCHEMA) + { + // release the transaction + fSessionManager.rolledback(txnID); + return result; + } + + // Commit current transaction. + // all DDL statements cause an implicut commit + VERBOSE_INFO("Getting current txnID"); + + // Check whether the table is existed already + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(createTableStmt.fSessionID); + execplan::CalpontSystemCatalog::TableName tableName; + tableName.schema = tableDef.fQualifiedName->fSchema; + tableName.table = tableDef.fQualifiedName->fName; + execplan::CalpontSystemCatalog::ROPair roPair; + roPair.objnum = 0; + ByteStream::byte rc = 0; + + /** @Bug 217 */ + /** @Bug 225 */ + try + { + roPair = systemCatalogPtr->tableRID(tableName); + } + catch (IDBExcept& ie) + { + // TODO: What is and is not an error here? + if (ie.errorCode() == ERR_DATA_OFFLINE) + { + // release transaction + fSessionManager.rolledback(txnID); + // Return the error for display to user + Message::Args args; + Message message(9); + args.add(ie.what()); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + return result; + } + else if (ie.errorCode() == ERR_TABLE_NOT_IN_CATALOG) + { + roPair.objnum = 0; + } + else // error out + { + // release transaction + fSessionManager.rolledback(txnID); + // Return the error for display to user + Message::Args args; + Message message(9); + args.add(ie.what()); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + return result; + } + } + catch (std::exception& ex) // error out + { + // release transaction + fSessionManager.rolledback(txnID); + // Return the error for display to user + Message::Args args; + Message message(9); + args.add(ex.what()); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + return result; + } + catch (...) // error out + { + // release transaction + fSessionManager.rolledback(txnID); + // Return the error for display to user + Message::Args args; + Message message(9); + args.add("Unknown exception caught when checking if the table name is already in use."); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + return result; + } + + // This is a current db bug, it should not turn OID is it cannot find + if (roPair.objnum >= 3000) + { +#ifdef _MSC_VER + // FIXME: Why do we need to do this??? + systemCatalogPtr->flushCache(); + + try + { + roPair = systemCatalogPtr->tableRID(tableName); + } + catch (...) + { + roPair.objnum = 0; + } + + if (roPair.objnum < 3000) + goto keepGoing; + +#endif + Message::Args args; + Message message(9); + args.add("Internal create table error for"); + args.add(tableName.toString()); + args.add(": table already exists"); + args.add("(your schema is probably out-of-sync)"); message.format(args); result.result = CREATE_ERROR; result.message = message; + // release the transaction + fSessionManager.rolledback(txnID); + return result; + } - fWriteEngine.rollbackTran(txnID.id, sessionId); +#ifdef _MSC_VER +keepGoing: +#endif + // Start a new transaction + VERBOSE_INFO("Starting a new transaction"); - size_t size = tableDef.fColumns.size(); + string stmt = createTableStmt.fSql + "|" + tableDef.fQualifiedName->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, createTableStmt.fSessionID, txnID.id); - for (size_t i = 0; i < size; ++i) + std::string err; + execplan::ObjectIDManager fObjectIDManager; + OamCache* oamcache = OamCache::makeOamCache(); + string errorMsg; + // get a unique number + uint64_t uniqueId = 0; + + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + Message::Args args; + Message message(9); + args.add(ex.what()); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + catch (...) + { + Message::Args args; + Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = CREATE_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + fWEClient->addQueue(uniqueId); + + try + { + // Allocate tableoid table identification + VERBOSE_INFO("Allocating object ID for table"); + // Allocate a object ID for each column we are about to create + VERBOSE_INFO("Allocating object IDs for columns"); + uint32_t numColumns = tableDef.fColumns.size(); + uint32_t numDictCols = 0; + + for (unsigned i = 0; i < numColumns; i++) { - fWriteEngine.dropColumn(txnID.id, fStartingColOID + i); + int dataType; + dataType = convertDataType(tableDef.fColumns[i]->fType->fType); + + if ((dataType == CalpontSystemCatalog::CHAR && tableDef.fColumns[i]->fType->fLength > 8) || + (dataType == CalpontSystemCatalog::VARCHAR && tableDef.fColumns[i]->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::VARBINARY && tableDef.fColumns[i]->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::BLOB && tableDef.fColumns[i]->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::TEXT && tableDef.fColumns[i]->fType->fLength > 7)) + numDictCols++; + } + + fStartingColOID = fObjectIDManager.allocOIDs(numColumns + numDictCols + + 1); // include column, oids,dictionary oids and tableoid +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Create table allocOIDs got the starting oid " << fStartingColOID << endl; +#endif + + if (fStartingColOID < 0) + { + result.result = CREATE_ERROR; + errorMsg = "Error in getting objectid from oidmanager."; + Message::Args args; + Message message(9); + args.add("(1)Create table failed due to "); + args.add(errorMsg); + message.format(args); + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + // Write the table metadata to the systemtable + VERBOSE_INFO("Writing meta data to SYSTABLE"); + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_SVR_WRITE_SYSTABLE; + bytestream << uniqueId; + bytestream << (uint32_t)createTableStmt.fSessionID; + bytestream << (uint32_t)txnID.id; + bytestream << (uint32_t)fStartingColOID; + bytestream << (uint32_t)createTableStmt.fTableWithAutoi; + uint16_t dbRoot; + BRM::OID_t sysOid = 1001; + // Find out where systable is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + { + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("Error while calling getSysCatDBRoot "); + args.add(errorMsg); + message.format(args); + result.message = message; + // release transaction + fSessionManager.rolledback(txnID); + return result; + } + + int pmNum = 1; + bytestream << (uint32_t)dbRoot; + tableDef.serialize(bytestream); + boost::shared_ptr bsIn; + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + // MCOL-66 The DBRM can't handle concurrent DDL + boost::mutex::scoped_lock lk(dbrmMutex); + + try + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " create table sending We_SVR_WRITE_SYSTABLE to pm " << pmNum << endl; +#endif + fWEClient->write(bytestream, (unsigned)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + errorMsg.clear(); + *bsIn >> errorMsg; +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << "Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; +#endif + } + + break; + } + } + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " create table got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; +#ifdef IDB_DDL_DEBUG + cout << "create table got unknown exception" << endl; +#endif + } + + if (rc != 0) + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; +#endif + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("(2)Create table failed due to "); + args.add(errorMsg); + message.format(args); + result.message = message; + + if (rc != NETWORK_ERROR) + { + rollBackTransaction(uniqueId, txnID, createTableStmt.fSessionID); // What to do with the error code + } + + // release transaction + fSessionManager.rolledback(txnID); + return result; + } + + VERBOSE_INFO("Writing meta data to SYSCOLUMN"); + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATE_SYSCOLUMN; + bytestream << uniqueId; + bytestream << (uint32_t)createTableStmt.fSessionID; + bytestream << (uint32_t)txnID.id; + bytestream << numColumns; + + for (unsigned i = 0; i < numColumns; ++i) + { + bytestream << (uint32_t)(fStartingColOID + i + 1); + } + + bytestream << numDictCols; + + for (unsigned i = 0; i < numDictCols; ++i) + { + bytestream << (uint32_t)(fStartingColOID + numColumns + i + 1); + } + + uint8_t alterFlag = 0; + int colPos = 0; + bytestream << (ByteStream::byte)alterFlag; + bytestream << (uint32_t)colPos; + + sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + { + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("Error while calling getSysCatDBRoot "); + args.add(errorMsg); + message.format(args); + result.message = message; + // release transaction + fSessionManager.rolledback(txnID); + return result; + } + + bytestream << (uint32_t)dbRoot; + tableDef.serialize(bytestream); + pmNum = (*dbRootPMMap)[dbRoot]; + + try + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " create table sending WE_SVR_WRITE_CREATE_SYSCOLUMN to pm " << pmNum << endl; +#endif + fWEClient->write(bytestream, (uint32_t)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + errorMsg.clear(); + *bsIn >> errorMsg; +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << "Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; +#endif + } + + break; + } + } + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " create table got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " create table got unknown exception" << endl; +#endif + } + + if (rc != 0) + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Create table WE_SVR_WRITE_CREATE_SYSCOLUMN: " << errorMsg << endl; +#endif + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("(3)Create table failed due to "); + args.add(errorMsg); + message.format(args); + result.message = message; + + if (rc != NETWORK_ERROR) + { + rollBackTransaction(uniqueId, txnID, createTableStmt.fSessionID); // What to do with the error code + } + + // release transaction + fSessionManager.rolledback(txnID); + return result; + } + + // Get the number of tables in the database, the current table is included. + int tableCount = systemCatalogPtr->getTableCount(); + + // Calculate which dbroot the columns should start + DBRootConfigList dbRootList = oamcache->getDBRootNums(); + + uint16_t useDBRootIndex = tableCount % dbRootList.size(); + // Find out the dbroot# corresponding the useDBRootIndex from oam + uint16_t useDBRoot = dbRootList[useDBRootIndex]; + + VERBOSE_INFO("Creating column files"); + ColumnDef* colDefPtr; + ddlpackage::ColumnDefList tableDefCols = tableDef.fColumns; + ColumnDefList::const_iterator iter = tableDefCols.begin(); + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES; + bytestream << uniqueId; + bytestream << (uint32_t)txnID.id; + bytestream << (numColumns + numDictCols); + unsigned colNum = 0; + unsigned dictNum = 0; + + while (iter != tableDefCols.end()) + { + colDefPtr = *iter; + + CalpontSystemCatalog::ColDataType dataType = convertDataType(colDefPtr->fType->fType); + + if (dataType == CalpontSystemCatalog::DECIMAL || dataType == CalpontSystemCatalog::UDECIMAL) + { + if (colDefPtr->fType->fPrecision == -1 || colDefPtr->fType->fPrecision == 0) + { + colDefPtr->fType->fLength = 8; + } + else if ((colDefPtr->fType->fPrecision > 0) && (colDefPtr->fType->fPrecision < 3)) + { + colDefPtr->fType->fLength = 1; + } + + else if (colDefPtr->fType->fPrecision < 5 && (colDefPtr->fType->fPrecision > 2)) + { + colDefPtr->fType->fLength = 2; + } + else if (colDefPtr->fType->fPrecision > 4 && colDefPtr->fType->fPrecision < 10) + { + colDefPtr->fType->fLength = 4; + } + else if (colDefPtr->fType->fPrecision > 9 && colDefPtr->fType->fPrecision < 19) + { + colDefPtr->fType->fLength = 8; + } + else if (colDefPtr->fType->fPrecision > 18 && colDefPtr->fType->fPrecision < 39) + { + colDefPtr->fType->fLength = 16; + } + } + + bytestream << (fStartingColOID + (colNum++) + 1); + bytestream << (uint8_t)dataType; + bytestream << (uint8_t) false; + + bytestream << (uint32_t)colDefPtr->fType->fLength; + bytestream << (uint16_t)useDBRoot; + bytestream << (uint32_t)colDefPtr->fType->fCompressiontype; + + if ((dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || + (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::TEXT && colDefPtr->fType->fLength > 7)) + { + bytestream << (uint32_t)(fStartingColOID + numColumns + (dictNum++) + 1); + bytestream << (uint8_t)dataType; + bytestream << (uint8_t) true; + bytestream << (uint32_t)colDefPtr->fType->fLength; + bytestream << (uint16_t)useDBRoot; + bytestream << (uint32_t)colDefPtr->fType->fCompressiontype; + } + + ++iter; + } + + //@Bug 4176. save oids to a log file for cleanup after fail over. + std::vector oidList; + + for (unsigned i = 0; i < numColumns; ++i) + { + oidList.push_back(fStartingColOID + i + 1); + } + + bytestream << numDictCols; + + for (unsigned i = 0; i < numDictCols; ++i) + { + oidList.push_back(fStartingColOID + numColumns + i + 1); } try { - execplan::ObjectIDManager fObjectIDManager; - fObjectIDManager.returnOID(fTableOID); - fObjectIDManager.returnOIDs(fStartingColOID, - fStartingColOID + tableDef.fColumns.size() - 1); + createWriteDropLogFile(fStartingColOID, uniqueId, oidList); } catch (std::exception& ex) { - Message::Args args; - Message message(6); - args.add(ex.what()); - message.format(args); - result.message = message; - result.result = CREATE_ERROR; + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("(4)Create table failed due to "); + args.add(ex.what()); + message.format(args); + result.message = message; + + if (rc != NETWORK_ERROR) + { + rollBackTransaction(uniqueId, txnID, createTableStmt.fSessionID); // What to do with the error code + } + + // release transaction + fSessionManager.rolledback(txnID); + return result; } - catch (...) + + pmNum = (*dbRootPMMap)[useDBRoot]; + + try { - Message::Args args; - Message message(6); - args.add("Unknown exception"); - message.format(args); - result.message = message; - result.result = CREATE_ERROR; - //cout << "returnOIDs error" << endl; +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " create table sending WE_SVR_WRITE_CREATETABLEFILES to pm " << pmNum << endl; +#endif + fWEClient->write(bytestream, pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + errorMsg.clear(); + *bsIn >> errorMsg; +#ifdef IDB_DDL_DEBUG + cout << "Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; +#endif + } + + break; + } + } + + if (rc != 0) + { + // drop the newly created files + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; + bytestream << uniqueId; + bytestream << (uint32_t)(numColumns + numDictCols); + + for (unsigned i = 0; i < (numColumns + numDictCols); i++) + { + bytestream << (uint32_t)(fStartingColOID + i + 1); + } + + fWEClient->write(bytestream, pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + break; + } + else + { + break; + } + } + + //@Bug 5464. Delete from extent map. + fDbrm->deleteOIDs(oidList); + } } - - DictionaryOIDList::const_iterator dictoid_iter = fDictionaryOIDList.begin(); - - while (dictoid_iter != fDictionaryOIDList.end()) + catch (runtime_error&) { - DictOID dictOID = *dictoid_iter; - fWriteEngine.dropDctnry(txnID.id, dictOID.dictOID, dictOID.treeOID, dictOID.listOID); - //fObjectIDManager.returnOID(dictOID.dictOID); - - ++dictoid_iter; + errorMsg = "Lost connection to Write Engine Server"; } + if (rc != 0) + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Create table We_SVR_WRITE_CREATETABLEFILES: " << errorMsg << endl; +#endif + rollBackTransaction(uniqueId, txnID, createTableStmt.fSessionID); // What to do with the error code + fSessionManager.rolledback(txnID); + } + else + { + commitTransaction(uniqueId, txnID); + fSessionManager.committed(txnID); + fWEClient->removeQueue(uniqueId); + deleteLogFile(DROPTABLE_LOG, fStartingColOID, uniqueId); + } + + // Log the DDL statement. + logDDL(createTableStmt.fSessionID, txnID.id, createTableStmt.fSql, createTableStmt.fOwner); + } + catch (std::exception& ex) + { + result.result = CREATE_ERROR; + Message::Args args; + Message message(9); + args.add("(5)Create table failed due to "); + args.add(ex.what()); + message.format(args); + result.message = message; fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + return result; + } + + // fWEClient->removeQueue(uniqueId); + if (rc != 0) + { + result.result = CREATE_ERROR; + Message::Args args; + Message message(9); + args.add("(6)Create table failed due to "); + args.add(errorMsg); + message.format(args); + result.message = message; + } + + return result; } -} // namespace ddlpackageprocessor +void CreateTableProcessor::rollBackCreateTable(const string& error, BRM::TxnID txnID, int sessionId, + ddlpackage::TableDef& tableDef, DDLResult& result) +{ + cerr << "CreatetableProcessor::processPackage: " << error << endl; + + Message::Args args; + Message message(1); + args.add("(7)Create table Failed: "); + args.add(error); + args.add(""); + args.add(""); + message.format(args); + + result.result = CREATE_ERROR; + result.message = message; + + fWriteEngine.rollbackTran(txnID.id, sessionId); + + size_t size = tableDef.fColumns.size(); + + for (size_t i = 0; i < size; ++i) + { + fWriteEngine.dropColumn(txnID.id, fStartingColOID + i); + } + + try + { + execplan::ObjectIDManager fObjectIDManager; + fObjectIDManager.returnOID(fTableOID); + fObjectIDManager.returnOIDs(fStartingColOID, fStartingColOID + tableDef.fColumns.size() - 1); + } + catch (std::exception& ex) + { + Message::Args args; + Message message(6); + args.add(ex.what()); + message.format(args); + result.message = message; + result.result = CREATE_ERROR; + } + catch (...) + { + Message::Args args; + Message message(6); + args.add("Unknown exception"); + message.format(args); + result.message = message; + result.result = CREATE_ERROR; + // cout << "returnOIDs error" << endl; + } + + DictionaryOIDList::const_iterator dictoid_iter = fDictionaryOIDList.begin(); + + while (dictoid_iter != fDictionaryOIDList.end()) + { + DictOID dictOID = *dictoid_iter; + fWriteEngine.dropDctnry(txnID.id, dictOID.dictOID, dictOID.treeOID, dictOID.listOID); + // fObjectIDManager.returnOID(dictOID.dictOID); + + ++dictoid_iter; + } + + fSessionManager.rolledback(txnID); +} + +} // namespace ddlpackageprocessor // vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/createtableprocessor.h b/dbcon/ddlpackageproc/createtableprocessor.h index 3a601efb7..f67e05c96 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.h +++ b/dbcon/ddlpackageproc/createtableprocessor.h @@ -34,31 +34,31 @@ namespace ddlpackageprocessor { - /** @brief specialization of a DDLPackageProcessor * for interacting with the Write Engine * to process create table ddl statements. */ class CreateTableProcessor : public DDLPackageProcessor { -public: + public: + CreateTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process a create table statement + * + * @param createTableStmt the CreateTableStatement + */ + EXPORT DDLResult processPackage(ddlpackage::CreateTableStatement& createTableStmt); - CreateTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process a create table statement - * - * @param createTableStmt the CreateTableStatement - */ - EXPORT DDLResult processPackage(ddlpackage::CreateTableStatement& createTableStmt); - -protected: - void rollBackCreateTable(const std::string& error, BRM::TxnID txnID, int sessionId, ddlpackage::TableDef& tableDef, DDLResult& result); - -private: + protected: + void rollBackCreateTable(const std::string& error, BRM::TxnID txnID, int sessionId, + ddlpackage::TableDef& tableDef, DDLResult& result); + private: }; -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor #undef EXPORT -#endif // CREATETABLEPROCESSOR_H +#endif // CREATETABLEPROCESSOR_H diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.cpp b/dbcon/ddlpackageproc/ddlindexpopulator.cpp index be2549084..734aa3ff1 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.cpp +++ b/dbcon/ddlpackageproc/ddlindexpopulator.cpp @@ -51,197 +51,194 @@ using namespace messageqcpp; namespace ddlpackageprocessor { - bool DDLIndexPopulator::populateIndex(DDLPackageProcessor::DDLResult& result) { - if (makeIndexStructs() ) - insertIndex(); + if (makeIndexStructs()) + insertIndex(); - result = fResult; - return NO_ERROR != fResult.result; + result = fResult; + return NO_ERROR != fResult.result; } - -bool DDLIndexPopulator::makeIndexStructs( ) +bool DDLIndexPopulator::makeIndexStructs() { - CalpontSelectExecutionPlan csep; - makeCsep(csep); - ResourceManager* rm; + CalpontSelectExecutionPlan csep; + makeCsep(csep); + ResourceManager* rm; - if (! fEC) + if (!fEC) + { + fEC = DistributedEngineComm::instance(rm); + fEC->Open(); + } + + SJLP jbl = joblist::JobListFactory::makeJobList(&csep, rm); + + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + csc->identity(CalpontSystemCatalog::EC); + + jbl->putEngineComm(fEC); + /* + ResultManager * result = jbl->GetResultManager(); + result->setRunning(1); + jbl->Execute(); */ + jbl->doQuery(); + + CalpontSystemCatalog::TableName tableName; + tableName.schema = fTable.fSchema; + tableName.table = fTable.fName; + + CalpontSystemCatalog::OID tableOid = (csc->tableRID(tableName)).objnum; + CalpontSystemCatalog::NJLSysDataList sysDataList; + + for (;;) + { + TableBand band; + band = jbl->projectTable(tableOid); + + if (band.getRowCount() == 0) { - fEC = DistributedEngineComm::instance(rm); - fEC->Open(); + // No more bands, table is done + break; } - SJLP jbl = joblist::JobListFactory::makeJobList(&csep, rm); + band.convertToSysDataList(sysDataList, csc); + break; + } - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog( fSessionID ); - csc->identity(CalpontSystemCatalog::EC); + // size_t cnt = fColNames.size(); + size_t i = 0; + vector::const_iterator it; + vector::const_iterator oid_iter; - jbl->putEngineComm(fEC); - /* - ResultManager * result = jbl->GetResultManager(); - result->setRunning(1); - jbl->Execute(); */ - jbl->doQuery(); + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if (isUnique()) + fUniqueColResultList.push_back(*it); - CalpontSystemCatalog::TableName tableName; - tableName.schema = fTable.fSchema; - tableName.table = fTable.fName; - - CalpontSystemCatalog::OID tableOid = (csc->tableRID ( tableName )).objnum; - CalpontSystemCatalog::NJLSysDataList sysDataList; - - for (;;) + for (oid_iter = fOidList.begin(); oid_iter != fOidList.end(); oid_iter++) { - TableBand band; - band = jbl->projectTable(tableOid); - - if (band.getRowCount() == 0) - { - // No more bands, table is done - break; - } - - band.convertToSysDataList(sysDataList, csc); - break; + if ((*it)->ColumnOID() == *oid_iter) + { + CalpontSystemCatalog::ColType coltype = makeIdxStruct(*it, fColNames.size(), csc); + addColumnData(*it, coltype, i); + } } - //size_t cnt = fColNames.size(); - size_t i = 0; - vector::const_iterator it; - vector::const_iterator oid_iter; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if (isUnique()) - fUniqueColResultList.push_back(*it); - - for ( oid_iter = fOidList.begin(); oid_iter != fOidList.end(); oid_iter++ ) - { - if ( (*it)->ColumnOID() == *oid_iter ) - { - CalpontSystemCatalog::ColType coltype = makeIdxStruct(*it, fColNames.size(), csc); - addColumnData(*it, coltype, i); - } - } - - i++; - } - - return (fIdxValueList.size() && NO_ERROR == fResult.result ); + i++; + } + return (fIdxValueList.size() && NO_ERROR == fResult.result); } - - -void DDLIndexPopulator::makeCsep(CalpontSelectExecutionPlan& csep) +void DDLIndexPopulator::makeCsep(CalpontSelectExecutionPlan& csep) { + csep.sessionID(fSessionID); - csep.sessionID(fSessionID); + csep.txnID(fTxnID); + csep.verID(fSessionManager->verID()); - csep.txnID(fTxnID); - csep.verID(fSessionManager->verID()); + CalpontSelectExecutionPlan::ReturnedColumnList colList; + CalpontSelectExecutionPlan::ColumnMap colMap; + CalpontSystemCatalog::TableColName tableColName; + CalpontSystemCatalog::OID oid; + tableColName.schema = fTable.fSchema; + tableColName.table = fTable.fName; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + string tableName(fTable.fSchema + "." + fTable.fName + "."); - CalpontSelectExecutionPlan::ReturnedColumnList colList; - CalpontSelectExecutionPlan::ColumnMap colMap; - CalpontSystemCatalog::TableColName tableColName; - CalpontSystemCatalog::OID oid; - tableColName.schema = fTable.fSchema; - tableColName.table = fTable.fName; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog( fSessionID ); - string tableName(fTable.fSchema + "." + fTable.fName + "."); + ColumnNameList::const_iterator cend = fColNames.end(); - ColumnNameList::const_iterator cend = fColNames.end(); + for (ColumnNameList::const_iterator cname = fColNames.begin(); cname != cend; ++cname) + { + string fullColName(tableName + *cname); + SRCP srcp(new SimpleColumn(fullColName, fSessionID)); + colList.push_back(srcp); + tableColName.column = *cname; + oid = csc->lookupOID(tableColName); + fOidList.push_back(oid); + colMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(fullColName, srcp)); + } - for (ColumnNameList::const_iterator cname = fColNames.begin(); cname != cend; ++cname) - { - string fullColName(tableName + *cname); - SRCP srcp(new SimpleColumn (fullColName, fSessionID)); - colList.push_back(srcp); - tableColName.column = *cname; - oid = csc->lookupOID( tableColName ); - fOidList.push_back( oid ); - colMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(fullColName, srcp)); - } - - csep.columnMap (colMap); - csep.returnedCols (colList); + csep.columnMap(colMap); + csep.returnedCols(colList); } - -CalpontSystemCatalog::ColType DDLIndexPopulator::makeIdxStruct(const ColumnResult* cr, size_t cols, boost::shared_ptr csc ) +CalpontSystemCatalog::ColType DDLIndexPopulator::makeIdxStruct(const ColumnResult* cr, size_t cols, + boost::shared_ptr csc) { - IdxStruct idx; - idx.treeOid = fIdxOID.treeOID; - idx.listOid = fIdxOID.listOID; - idx.multiColFlag = cols > 1; - CalpontSystemCatalog::ColType coltype = csc->colType(cr->ColumnOID()); - idx.idxDataType = static_cast(coltype.colDataType); + IdxStruct idx; + idx.treeOid = fIdxOID.treeOID; + idx.listOid = fIdxOID.listOID; + idx.multiColFlag = cols > 1; + CalpontSystemCatalog::ColType coltype = csc->colType(cr->ColumnOID()); + idx.idxDataType = static_cast(coltype.colDataType); - if (isDictionaryType(coltype) ) - { - idx.idxWidth = fTOKENSIZE; - idx.idxType = WR_CHAR; - }//@bug 410: index sizes are either 1, 4 or 8 - else if (exeplan::isCharType(coltype)) - { - if (1 == coltype.colWidth) idx.idxWidth = 1; - else idx.idxWidth = (coltype.colWidth > 4) ? 8 : 4; + if (isDictionaryType(coltype)) + { + idx.idxWidth = fTOKENSIZE; + idx.idxType = WR_CHAR; + } //@bug 410: index sizes are either 1, 4 or 8 + else if (exeplan::isCharType(coltype)) + { + if (1 == coltype.colWidth) + idx.idxWidth = 1; + else + idx.idxWidth = (coltype.colWidth > 4) ? 8 : 4; - idx.idxType = WR_CHAR; + idx.idxType = WR_CHAR; + } + else + idx.idxWidth = coltype.colWidth; + + fIdxStructList.push_back(idx); + return coltype; +} + +void DDLIndexPopulator::addColumnData(const execplan::ColumnResult* cr, + const CalpontSystemCatalog::ColType colType, int added) +{ + WriteEngine::IdxTupleList tupleList; + WriteEngine::IdxTuple tuple; + + for (int i = 0; i < cr->dataCount(); ++i) + { + WriteEngine::IdxTuple tuple; + convertColData(cr, i, colType, tuple); + + if (checkConstraints(tuple, colType, i, added)) + { + tupleList.push_back(tuple); + + if (!added) + fRidList.push_back(cr->GetRid(i)); } else - idx.idxWidth = coltype.colWidth; + break; + } - fIdxStructList.push_back(idx); - return coltype; + if (tupleList.size()) + fIdxValueList.push_back(tupleList); } -void DDLIndexPopulator::addColumnData(const execplan::ColumnResult* cr, const CalpontSystemCatalog::ColType colType, int added) +void DDLIndexPopulator::convertColData(const execplan::ColumnResult* cr, int idx, + const CalpontSystemCatalog::ColType& colType, + WriteEngine::IdxTuple& tuple) { - WriteEngine::IdxTupleList tupleList; - WriteEngine::IdxTuple tuple; - - for (int i = 0; i < cr->dataCount(); ++i) - { - - WriteEngine::IdxTuple tuple ; - convertColData( cr, i, colType, tuple); - - if (checkConstraints( tuple, colType, i, added)) - { - tupleList.push_back(tuple); - - if (! added ) - fRidList.push_back(cr->GetRid(i)); - } - else - break; - } - - if (tupleList.size()) - fIdxValueList.push_back(tupleList); + if (isDictionaryType(colType)) + { + /* tuple.data = tokenizeData ( colType, cr->GetStringData(idx) );*/ + /* tuple.data = tokenizeData ( cr->GetRid(idx) );*/ + tuple.data = convertTokenData(cr->GetStringData(idx)); + } + else + tuple.data = convertData(colType, cr, idx); } - - -void DDLIndexPopulator::convertColData(const execplan::ColumnResult* cr, int idx, const CalpontSystemCatalog::ColType& colType, WriteEngine::IdxTuple& tuple) +boost::any DDLIndexPopulator::convertTokenData(const std::string& data) { - if (isDictionaryType(colType)) - { - /* tuple.data = tokenizeData ( colType, cr->GetStringData(idx) );*/ - /* tuple.data = tokenizeData ( cr->GetRid(idx) );*/ - tuple.data = convertTokenData(cr->GetStringData(idx)); - } - else tuple.data = convertData( colType, cr, idx); -} - -boost::any DDLIndexPopulator::convertTokenData( const std::string& data ) -{ - string strData((size_t)fTOKENSIZE < data.length() ? data.substr(0, fTOKENSIZE) : data); - return strData; + string strData((size_t)fTOKENSIZE < data.length() ? data.substr(0, fTOKENSIZE) : data); + return strData; } #if 0 @@ -269,331 +266,304 @@ bool DDLIndexPopulator::openColumnFile(WriteEngine::OID oid) #endif // Workaround to get original column token and not "retokenize" the string value -boost::any DDLIndexPopulator::tokenizeData( WriteEngine::RID rid ) +boost::any DDLIndexPopulator::tokenizeData(WriteEngine::RID rid) { - int64_t byteOffset = rid * fTOKENSIZE; - ByteStream::byte inbuf[fTOKENSIZE]; - fColumnFile.seekg(byteOffset, ios::beg); - fColumnFile.read(reinterpret_cast(inbuf), fTOKENSIZE); + int64_t byteOffset = rid * fTOKENSIZE; + ByteStream::byte inbuf[fTOKENSIZE]; + fColumnFile.seekg(byteOffset, ios::beg); + fColumnFile.read(reinterpret_cast(inbuf), fTOKENSIZE); - WriteEngine::Token token; - memcpy(&token, inbuf, fTOKENSIZE); - return token; + WriteEngine::Token token; + memcpy(&token, inbuf, fTOKENSIZE); + return token; } - -boost::any DDLIndexPopulator::tokenizeData( const execplan::CalpontSystemCatalog::ColType& colType, const std::string& data ) +boost::any DDLIndexPopulator::tokenizeData(const execplan::CalpontSystemCatalog::ColType& colType, + const std::string& data) { - WriteEngine::DctnryTuple dictTuple; + WriteEngine::DctnryTuple dictTuple; - if ( data.length() > (unsigned int)colType.colWidth ) + if (data.length() > (unsigned int)colType.colWidth) + { + logError("Insert value is too large for column"); + } + else + { + WriteEngine::DctnryStruct dictStruct; + dictStruct.treeOid = colType.ddn.treeOID; + dictStruct.listOid = colType.ddn.listOID; + dictStruct.dctnryOid = colType.ddn.dictOID; + dictTuple.sigValue = data.c_str(); + dictTuple.sigSize = data.length(); + int error = NO_ERROR; + + if (NO_ERROR != (error = fWriteEngine->tokenize(fTxnID, dictStruct, dictTuple))) { - logError("Insert value is too large for column"); + logError("Tokenization failed", error); } - else - { - WriteEngine::DctnryStruct dictStruct; - dictStruct.treeOid = colType.ddn.treeOID; - dictStruct.listOid = colType.ddn.listOID; - dictStruct.dctnryOid = colType.ddn.dictOID; - dictTuple.sigValue = data.c_str(); - dictTuple.sigSize = data.length(); - int error = NO_ERROR; + } - if ( NO_ERROR != (error = fWriteEngine->tokenize( fTxnID, dictStruct, dictTuple)) ) - { - logError("Tokenization failed", error); - } - } - - return dictTuple.token; + return dictTuple.token; } - - -boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& colType, const execplan::ColumnResult* cr, int idx ) +boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& colType, + const execplan::ColumnResult* cr, int idx) { - uint64_t data = cr->GetData(idx); + uint64_t data = cr->GetData(idx); - switch ( colType.colDataType ) + switch (colType.colDataType) + { + case CalpontSystemCatalog::BIT: + case execplan::CalpontSystemCatalog::TINYINT: return *reinterpret_cast(&data); + + case execplan::CalpontSystemCatalog::SMALLINT: return *reinterpret_cast(&data); + + case execplan::CalpontSystemCatalog::DATE: // @bug 375 + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: return *reinterpret_cast(&data); + + case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::BIGINT: return *reinterpret_cast(&data); + + case execplan::CalpontSystemCatalog::DECIMAL: { - case CalpontSystemCatalog::BIT: - case execplan::CalpontSystemCatalog::TINYINT: - return *reinterpret_cast(&data); + if (colType.colWidth <= CalpontSystemCatalog::FOUR_BYTE) + return *reinterpret_cast(&data); - case execplan::CalpontSystemCatalog::SMALLINT: - return *reinterpret_cast(&data); + else if (colType.colWidth <= 9) + return *reinterpret_cast(&data); - case execplan::CalpontSystemCatalog::DATE: // @bug 375 - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - return *reinterpret_cast(&data); - - case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::BIGINT: - return *reinterpret_cast(&data); - - case execplan::CalpontSystemCatalog::DECIMAL: - { - if (colType.colWidth <= CalpontSystemCatalog::FOUR_BYTE) return *reinterpret_cast(&data); - - else if (colType.colWidth <= 9) return *reinterpret_cast(&data); - - else return *reinterpret_cast(&data); - } - - case execplan::CalpontSystemCatalog::FLOAT: - return *reinterpret_cast(&data); - - case execplan::CalpontSystemCatalog::DOUBLE: - return *reinterpret_cast(&data); - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - { - string strData(cr->GetStringData(idx) ); - return *reinterpret_cast(&strData); - } - - default: - break; + else + return *reinterpret_cast(&data); } - logError("Invalid column type"); - throw std::runtime_error("Invalid data"); + case execplan::CalpontSystemCatalog::FLOAT: return *reinterpret_cast(&data); - return *reinterpret_cast(&data); + case execplan::CalpontSystemCatalog::DOUBLE: return *reinterpret_cast(&data); + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + { + string strData(cr->GetStringData(idx)); + return *reinterpret_cast(&strData); + } + + default: break; + } + + logError("Invalid column type"); + throw std::runtime_error("Invalid data"); + + return *reinterpret_cast(&data); } - -void DDLIndexPopulator::insertIndex( ) +void DDLIndexPopulator::insertIndex() { -// @bug 359 use bulk load build - int rc = (1 < fIdxStructList.size()) ? - (void)0 - : (void)0; - - if (rc) - logError("Error inserting index values", rc ); + // @bug 359 use bulk load build + int rc = (1 < fIdxStructList.size()) ? (void)0 : (void)0; + if (rc) + logError("Error inserting index values", rc); } bool DDLIndexPopulator::isDictionaryType(const CalpontSystemCatalog::ColType& colType) { - return ( (CalpontSystemCatalog::CHAR == colType.colDataType && 8 < colType.colWidth ) - || (CalpontSystemCatalog::VARCHAR == colType.colDataType && 7 < colType.colWidth ) - || (CalpontSystemCatalog::DECIMAL == colType.colDataType && 18 < colType.precision )); - + return ((CalpontSystemCatalog::CHAR == colType.colDataType && 8 < colType.colWidth) || + (CalpontSystemCatalog::VARCHAR == colType.colDataType && 7 < colType.colWidth) || + (CalpontSystemCatalog::DECIMAL == colType.colDataType && 18 < colType.precision)); } -bool DDLIndexPopulator::checkConstraints( const IdxTuple& data, const CalpontSystemCatalog::ColType& ctype, int i, int column) +bool DDLIndexPopulator::checkConstraints(const IdxTuple& data, const CalpontSystemCatalog::ColType& ctype, + int i, int column) { + switch (fConstraint) + { + case DDL_INVALID_CONSTRAINT: return true; - switch ( fConstraint ) - { - case DDL_INVALID_CONSTRAINT: - return true; + case DDL_UNIQUE: + case DDL_PRIMARY_KEY: + if ((size_t)column + 1 < fColNames.size()) + return true; - case DDL_UNIQUE: - case DDL_PRIMARY_KEY: - if ((size_t)column + 1 < fColNames.size() ) - return true; + return checkUnique(i, ctype); - return checkUnique( i, ctype ); + case DDL_NOT_NULL: return checkNotNull(data, ctype); - case DDL_NOT_NULL: - return checkNotNull( data, ctype ); - - case DDL_CHECK: - return checkCheck( data, ctype ); - - default: - return true; //? - } + case DDL_CHECK: return checkCheck(data, ctype); + default: return true; //? + } } // Check if the row of data at idx is already in fUniqueColResultList -bool DDLIndexPopulator::checkUnique( int idx, const CalpontSystemCatalog::ColType& colType ) +bool DDLIndexPopulator::checkUnique(int idx, const CalpontSystemCatalog::ColType& colType) { - if (0 == idx) - return true; + if (0 == idx) + return true; - //Get row of data as each column result data at idx - size_t indexSize = fColNames.size(); - vector rowIntData(indexSize); - vector rowStrData(indexSize); + // Get row of data as each column result data at idx + size_t indexSize = fColNames.size(); + vector rowIntData(indexSize); + vector rowStrData(indexSize); - for (size_t i = 0; i < indexSize; ++i) + for (size_t i = 0; i < indexSize; ++i) + { + // if ( isStringType(fUniqueColResultList[i]->columnType()) ) + if (isStringType(colType.colDataType)) + rowStrData[i] = fUniqueColResultList[i]->GetStringData(idx); + else + rowIntData[i] = fUniqueColResultList[i]->GetData(idx); + } + + // check if each value in the idx row is equal to each value in a previous row + // i is the row; j is the column. + bool unique = true; + + for (int i = 0; i < idx && unique; ++i) + { + bool equal = true; + + for (size_t j = 0; j < indexSize && equal; ++j) { - //if ( isStringType(fUniqueColResultList[i]->columnType()) ) - if ( isStringType(colType.colDataType) ) - rowStrData[i] = fUniqueColResultList[i]->GetStringData(idx); - else - rowIntData[i] = fUniqueColResultList[i]->GetData(idx); + if (isStringType(colType.colDataType)) + { + equal = fUniqueColResultList[j]->GetStringData(i) == rowStrData[j]; + } + else + { + equal = (static_cast(fUniqueColResultList[j]->GetData(i)) == rowIntData[j]); + } } - //check if each value in the idx row is equal to each value in a previous row - // i is the row; j is the column. - bool unique = true; + unique = !equal; + } - for (int i = 0; i < idx && unique; ++i) - { - bool equal = true; + if (!unique) + { + stringstream ss; + ss << idx; + logError("Unique Constraint violated on row: " + ss.str()); + } - for (size_t j = 0; j < indexSize && equal; ++j) - { - if ( isStringType(colType.colDataType) ) - { - equal = fUniqueColResultList[j]->GetStringData(i) == rowStrData[j]; - } - else - { - equal = (static_cast(fUniqueColResultList[j]->GetData(i)) == rowIntData[j]); - } - } - - unique = ! equal; - } - - if (! unique) - { - stringstream ss; - ss << idx; - logError("Unique Constraint violated on row: " + ss.str() ); - } - - return unique; + return unique; } - bool DDLIndexPopulator::checkNotNull(const IdxTuple& data, const CalpontSystemCatalog::ColType& colType) { + any nullvalue = DDLNullValueForType(colType); + bool isNull = false; - any nullvalue = DDLNullValueForType(colType); - bool isNull = false; + switch (colType.colDataType) + { + case CalpontSystemCatalog::BIT: break; - switch ( colType.colDataType ) + case execplan::CalpontSystemCatalog::TINYINT: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + + case execplan::CalpontSystemCatalog::SMALLINT: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + + case execplan::CalpontSystemCatalog::BIGINT: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + + case execplan::CalpontSystemCatalog::DECIMAL: { - case CalpontSystemCatalog::BIT: - break; + if (colType.colWidth <= CalpontSystemCatalog::FOUR_BYTE) + isNull = any_cast(data.data) == any_cast(nullvalue); + else if (colType.colWidth <= 9) + isNull = any_cast(data.data) == any_cast(nullvalue); + else if (colType.colWidth <= 18) + isNull = any_cast(data.data) == any_cast(nullvalue); + else + isNull = + compareToken(any_cast(data.data), any_cast(nullvalue)); - case execplan::CalpontSystemCatalog::TINYINT: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::BIGINT: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - { - if (colType.colWidth <= CalpontSystemCatalog::FOUR_BYTE) - isNull = any_cast(data.data) == any_cast(nullvalue); - else if (colType.colWidth <= 9) - isNull = any_cast(data.data) == any_cast(nullvalue); - else if (colType.colWidth <= 18) - isNull = any_cast(data.data) == any_cast(nullvalue); - else - isNull = compareToken(any_cast(data.data), any_cast(nullvalue)); - - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::DATE: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - isNull = any_cast(data.data) == any_cast(nullvalue); - break; - - case execplan::CalpontSystemCatalog::CHAR: - { - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - isNull = any_cast(data.data) == any_cast(nullvalue); - else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - isNull = any_cast(data.data) == any_cast(nullvalue); - else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - isNull = any_cast(data.data) == any_cast(nullvalue); - else - isNull = compareToken(any_cast(data.data), any_cast(nullvalue)); - - break; - - } - - case execplan::CalpontSystemCatalog::VARCHAR: - { - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - isNull = any_cast(data.data) == any_cast(nullvalue); - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - isNull = any_cast(data.data) == any_cast(nullvalue); - else - isNull = compareToken(any_cast(data.data), any_cast(nullvalue)); - - break; - } - - default: - throw std::runtime_error("getNullValueForType: unkown column data type"); + break; } - if (isNull) - logError("Null value not allowed in index"); + case execplan::CalpontSystemCatalog::FLOAT: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; - return ! isNull; + case execplan::CalpontSystemCatalog::DOUBLE: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + case execplan::CalpontSystemCatalog::DATE: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + isNull = any_cast(data.data) == any_cast(nullvalue); + break; + + case execplan::CalpontSystemCatalog::CHAR: + { + if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + isNull = any_cast(data.data) == any_cast(nullvalue); + else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + isNull = any_cast(data.data) == any_cast(nullvalue); + else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) + isNull = any_cast(data.data) == any_cast(nullvalue); + else + isNull = + compareToken(any_cast(data.data), any_cast(nullvalue)); + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + { + if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + isNull = any_cast(data.data) == any_cast(nullvalue); + else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) + isNull = any_cast(data.data) == any_cast(nullvalue); + else + isNull = + compareToken(any_cast(data.data), any_cast(nullvalue)); + + break; + } + + default: throw std::runtime_error("getNullValueForType: unkown column data type"); + } + + if (isNull) + logError("Null value not allowed in index"); + + return !isNull; } -void DDLIndexPopulator::logError(const string& msg, int error) +void DDLIndexPopulator::logError(const string& msg, int error) { + Message::Args args; + Message message(9); + args.add((string)__FILE__ + ": "); + args.add(msg); - Message::Args args; - Message message(9); - args.add((string)__FILE__ + ": "); - args.add(msg); + if (error) + { + args.add("Error number: "); + args.add(error); + } - if (error) - { - args.add("Error number: "); - args.add(error); - } + message.format(args); - message.format( args ); - - fResult.result = DDLPackageProcessor::CREATE_ERROR; - fResult.message = message; + fResult.result = DDLPackageProcessor::CREATE_ERROR; + fResult.message = message; } - -} //namespace - - - - - - +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.h b/dbcon/ddlpackageproc/ddlindexpopulator.h index 71910c32d..1570d2221 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.h +++ b/dbcon/ddlpackageproc/ddlindexpopulator.h @@ -40,7 +40,6 @@ #include #include - #include "joblistfactory.h" namespace joblist @@ -50,253 +49,258 @@ class DistributedEngineComm; namespace ddlpackageprocessor { - /** @brief Populate an new Index * implementation of a DDLPopulator */ class DDLIndexPopulator { + public: + /** @brief constructor + * + */ + DDLIndexPopulator(WriteEngine::WriteEngineWrapper* writeEngine, execplan::SessionManager* sessionManager, + uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + DDLPackageProcessor::DDLResult& result, const DDLPackageProcessor::IndexOID& idxOID, + const ddlpackage::ColumnNameList& colNames, const ddlpackage::QualifiedName& table, + const ddlpackage::DDL_CONSTRAINTS constraint, const DDLPackageProcessor::DebugLevel debug) + : fWriteEngine(writeEngine) + , fSessionManager(sessionManager) + , fSessionID(sessionID) + , fTxnID(txnID) + , fResult(result) + , fIdxOID(idxOID) + , fColNames(colNames) + , fTable(table) + , fDebugLevel(debug) + , fEC(0) + , fRidList() + , fIdxStructList() + , fIdxValueList() + , + /* fTOKENSIZE(sizeof(WriteEngine::Token) ) {}*/ + fConstraint(constraint) + , fUniqueColResultList() + { + } -public: - /** @brief constructor - * - */ - DDLIndexPopulator(WriteEngine::WriteEngineWrapper* writeEngine, - execplan::SessionManager* sessionManager, - uint32_t sessionID, - execplan::CalpontSystemCatalog::SCN txnID, - DDLPackageProcessor::DDLResult& result, - const DDLPackageProcessor::IndexOID& idxOID, - const ddlpackage::ColumnNameList& colNames, - const ddlpackage::QualifiedName& table, - const ddlpackage::DDL_CONSTRAINTS constraint, - const DDLPackageProcessor::DebugLevel debug): - fWriteEngine(writeEngine), fSessionManager(sessionManager), - fSessionID(sessionID), fTxnID(txnID), fResult(result), - fIdxOID(idxOID), fColNames(colNames), fTable(table), fDebugLevel(debug), - fEC(0), fRidList(), fIdxStructList(), fIdxValueList(), - /* fTOKENSIZE(sizeof(WriteEngine::Token) ) {}*/ - fConstraint(constraint), fUniqueColResultList() {} + /** @brief destructor + */ + virtual ~DDLIndexPopulator(){}; + /** @brief Is it required to debug + */ + const bool isDebug(const DDLPackageProcessor::DebugLevel level) const + { + return level <= fDebugLevel; + } - /** @brief destructor - */ - virtual ~DDLIndexPopulator() { }; + /** @brief Get debug level + */ + const DDLPackageProcessor::DebugLevel getDebugLevel() const + { + return fDebugLevel; + } + /** @brief set distributedEngineComm pointer ( for + * loading index). + */ + void setEngineComm(joblist::DistributedEngineComm* ec) + { + fEC = ec; + } + void setIdxOID(const DDLPackageProcessor::IndexOID& idxOID) + { + fIdxOID = idxOID; + } - /** @brief Is it required to debug - */ - const bool isDebug( const DDLPackageProcessor::DebugLevel level ) const + DDLPackageProcessor::DDLResult getResult() const + { + return fResult; + } + + /** @brief add data to the index from the statement + * + * populate the newly made index with data in the index columns. + * returns if there was an error. + */ + bool populateIndex(DDLPackageProcessor::DDLResult& result); + /** @brief returns if dictionary type + * + * determines if coltype is dictionary type based on type and size + */ + bool isDictionaryType(const execplan::CalpontSystemCatalog::ColType& ctype); + + void setConstraint(ddlpackage::DDL_CONSTRAINTS constraint); + + protected: + /** @brief make the structures to update the index + * + * builds and executes a query to retrieve all the data from the index columns + * to make the structures required by WriteEngine::updateIndex + * returns if there is data and no error + */ + bool makeIndexStructs(); + + /** @brief make the IdxStruct + * + * Fills in the values from the column result and calpont system catalog for + * the WriteEngine::IdxStruct + */ + execplan::CalpontSystemCatalog::ColType makeIdxStruct( + const execplan::ColumnResult* cr, size_t cols, boost::shared_ptr csc); + + /** @brief add the column result data to the value list + * + * Check contraints on each data item and adds it to a tuple list. + * Adds the rid to the rid list if it is the first column (not been added) + * Adds the completed tuple list to the fValueList + */ + void addColumnData(const execplan::ColumnResult* cr, const execplan::CalpontSystemCatalog::ColType ctype, + int added); + + /** @brief insert data into index. + * + * updates the index with the data using the appropriate write engine method + * based on multi column. Sets result to error if there is one. + */ + void insertIndex(); + + private: + DDLIndexPopulator(const DDLIndexPopulator&); + void operator=(const DDLIndexPopulator&); + /** @brief makes Calpont Select Execution Plan + * + * builds csep to select data from all columns from fColNames + */ + void makeCsep(execplan::CalpontSelectExecutionPlan& csep); + + /** @brief return if ColumnResult string type + * + * Uses same logic as ColumnResult from type to return getStringData (true) + * or getData (false). + */ + + bool isStringType(int type) const + { + return (type == execplan::CalpontSystemCatalog::CHAR || type == execplan::CalpontSystemCatalog::VARCHAR || + type == execplan::CalpontSystemCatalog::FLOAT || type == execplan::CalpontSystemCatalog::DOUBLE || + type == execplan::CalpontSystemCatalog::UFLOAT || + type == execplan::CalpontSystemCatalog::UDOUBLE); + } + + /** @brief converts column result data + * + * Converts or tokenizes data, depending on column type + */ + void convertColData(const execplan::ColumnResult* cr, int idx, + const execplan::CalpontSystemCatalog::ColType& cType, WriteEngine::IdxTuple& tuple); + + /** @brief converts non token data to its original type + */ + boost::any convertData(const execplan::CalpontSystemCatalog::ColType& colType, + const execplan::ColumnResult* cr, int idx); + + /** @brief returns token for string data + * + * There are two methods, the one using the rid is a workaround. + * Use the method that passes string data when WriteEngine::tokenize is + * able to return an existing token for a string. The rid method reads + * the column file directly. + */ + boost::any tokenizeData(const execplan::CalpontSystemCatalog::ColType& colType, const std::string& data); + + boost::any tokenizeData(WriteEngine::RID rid); + + /** @brief convert token data + * + * Indexes will use the first 8 bytes of a token type value instead + * of a token. + */ + + boost::any convertTokenData(const std::string& data); + + /** @brief opens the column file for the oid + * + * This method is needed only as long as the class is using the rid + * tokenizeData method. The fColumnFile and this method are no longer + * needed when the WriteEngine::tokenize method can be used. + */ + // bool openColumnFile(WriteEngine::OID oid); + + /** @brief returns if data violated its constraint + * + * checks data according to contraint in coltype and sets result to error + * if constraint violated. Returns if no error. + */ + bool checkConstraints(const WriteEngine::IdxTuple& data, + const execplan::CalpontSystemCatalog::ColType& ctype, int i, int column); + + /** @brief returns if data not null + * + * Returns false if data is null and sets result to error + */ + bool checkNotNull(const WriteEngine::IdxTuple& data, const execplan::CalpontSystemCatalog::ColType& ctype); + + /** @brief returns if data is not unique + * + * Returns false if data row is found more than once in columns and sets result to error + */ + bool checkUnique(int i, const execplan::CalpontSystemCatalog::ColType& colType); + + bool checkCheck(const WriteEngine::IdxTuple& data, + const execplan::CalpontSystemCatalog::ColType& ctype) const + { + return true; + } + + bool isUnique() + { + return ddlpackage::DDL_PRIMARY_KEY == fConstraint || ddlpackage::DDL_UNIQUE == fConstraint; + } + /** @brief logs error and message + * + * Updates result with message and sets it to CREATE_ERROR + */ + void logError(const std::string& msg, int error = 0); + + bool compareToken(const WriteEngine::Token& first, const WriteEngine::Token& second) const + { + return (first.op == second.op && first.fbo == second.fbo && first.spare == second.spare); + } + + WriteEngine::WriteEngineWrapper* fWriteEngine; + execplan::SessionManager* fSessionManager; + uint32_t fSessionID; + execplan::CalpontSystemCatalog::SCN fTxnID; + DDLPackageProcessor::DDLResult fResult; + DDLPackageProcessor::IndexOID fIdxOID; + ddlpackage::ColumnNameList fColNames; + ddlpackage::QualifiedName fTable; + DDLPackageProcessor::DebugLevel fDebugLevel; + joblist::DistributedEngineComm* fEC; + WriteEngine::RIDList fRidList; + WriteEngine::IdxStructList fIdxStructList; + WriteEngine::IdxValueList fIdxValueList; + + ddlpackage::DDL_CONSTRAINTS fConstraint; + std::vector fUniqueColResultList; + std::vector fOidList; + std::ifstream fColumnFile; + static const int fTOKENSIZE = 8; + + struct DDLNullValueForType : DDLPackageProcessor + { + DDLNullValueForType(const execplan::CalpontSystemCatalog::ColType& ctype) + : DDLPackageProcessor(), fType(ctype) { - return level <= fDebugLevel; } - - /** @brief Get debug level - */ - const DDLPackageProcessor::DebugLevel getDebugLevel() const + boost::any operator()(execplan::CalpontSystemCatalog::ColType& ctype) { - return fDebugLevel; + return ctype.getNullValueForType(); } - - - /** @brief set distributedEngineComm pointer ( for - * loading index). - */ - void setEngineComm(joblist::DistributedEngineComm* ec) - { - fEC = ec; - } - void setIdxOID(const DDLPackageProcessor::IndexOID& idxOID) - { - fIdxOID = idxOID; - } - - DDLPackageProcessor::DDLResult getResult() const - { - return fResult; - } - - - /** @brief add data to the index from the statement - * - * populate the newly made index with data in the index columns. - * returns if there was an error. - */ - bool populateIndex(DDLPackageProcessor::DDLResult& result); - /** @brief returns if dictionary type - * - * determines if coltype is dictionary type based on type and size - */ - bool isDictionaryType(const execplan::CalpontSystemCatalog::ColType& ctype); - - void setConstraint(ddlpackage::DDL_CONSTRAINTS constraint); - -protected: - - /** @brief make the structures to update the index - * - * builds and executes a query to retrieve all the data from the index columns - * to make the structures required by WriteEngine::updateIndex - * returns if there is data and no error - */ - bool makeIndexStructs(); - - /** @brief make the IdxStruct - * - * Fills in the values from the column result and calpont system catalog for - * the WriteEngine::IdxStruct - */ - execplan::CalpontSystemCatalog::ColType makeIdxStruct(const execplan::ColumnResult* cr, size_t cols, boost::shared_ptr csc ); - - /** @brief add the column result data to the value list - * - * Check contraints on each data item and adds it to a tuple list. - * Adds the rid to the rid list if it is the first column (not been added) - * Adds the completed tuple list to the fValueList - */ - void addColumnData(const execplan::ColumnResult* cr, const execplan::CalpontSystemCatalog::ColType ctype, int added); - - /** @brief insert data into index. - * - * updates the index with the data using the appropriate write engine method - * based on multi column. Sets result to error if there is one. - */ - void insertIndex(); - - -private: - DDLIndexPopulator(const DDLIndexPopulator& ); - void operator=(const DDLIndexPopulator& ); - /** @brief makes Calpont Select Execution Plan - * - * builds csep to select data from all columns from fColNames - */ - void makeCsep(execplan::CalpontSelectExecutionPlan& csep); - - /** @brief return if ColumnResult string type - * - * Uses same logic as ColumnResult from type to return getStringData (true) - * or getData (false). - */ - - bool isStringType(int type) const - { - return (type == execplan::CalpontSystemCatalog::CHAR - || type == execplan::CalpontSystemCatalog::VARCHAR - || type == execplan::CalpontSystemCatalog::FLOAT - || type == execplan::CalpontSystemCatalog::DOUBLE - || type == execplan::CalpontSystemCatalog::UFLOAT - || type == execplan::CalpontSystemCatalog::UDOUBLE ); - } - - /** @brief converts column result data - * - * Converts or tokenizes data, depending on column type - */ - void convertColData(const execplan::ColumnResult* cr, int idx, const execplan::CalpontSystemCatalog::ColType& cType, WriteEngine::IdxTuple& tuple); - - /** @brief converts non token data to its original type - */ - boost::any convertData(const execplan::CalpontSystemCatalog::ColType& colType, const execplan::ColumnResult* cr, int idx ); - - /** @brief returns token for string data - * - * There are two methods, the one using the rid is a workaround. - * Use the method that passes string data when WriteEngine::tokenize is - * able to return an existing token for a string. The rid method reads - * the column file directly. - */ - boost::any tokenizeData( const execplan::CalpontSystemCatalog::ColType& colType, const std::string& data ); - - boost::any tokenizeData( WriteEngine::RID rid ); - - /** @brief convert token data - * - * Indexes will use the first 8 bytes of a token type value instead - * of a token. - */ - - boost::any convertTokenData( const std::string& data ); - - /** @brief opens the column file for the oid - * - * This method is needed only as long as the class is using the rid - * tokenizeData method. The fColumnFile and this method are no longer - * needed when the WriteEngine::tokenize method can be used. - */ - //bool openColumnFile(WriteEngine::OID oid); - - /** @brief returns if data violated its constraint - * - * checks data according to contraint in coltype and sets result to error - * if constraint violated. Returns if no error. - */ - bool checkConstraints(const WriteEngine::IdxTuple& data, const execplan::CalpontSystemCatalog::ColType& ctype, int i, int column); - - /** @brief returns if data not null - * - * Returns false if data is null and sets result to error - */ - bool checkNotNull(const WriteEngine::IdxTuple& data, const execplan::CalpontSystemCatalog::ColType& ctype); - - /** @brief returns if data is not unique - * - * Returns false if data row is found more than once in columns and sets result to error - */ - bool checkUnique(int i, const execplan::CalpontSystemCatalog::ColType& colType ); - - bool checkCheck( const WriteEngine::IdxTuple& data, const execplan::CalpontSystemCatalog::ColType& ctype) const - { - return true; - } - - bool isUnique() - { - return ddlpackage::DDL_PRIMARY_KEY == fConstraint || ddlpackage::DDL_UNIQUE == fConstraint; - } - /** @brief logs error and message - * - * Updates result with message and sets it to CREATE_ERROR - */ - void logError(const std::string& msg, int error = 0); - - bool compareToken(const WriteEngine::Token& first, const WriteEngine::Token& second) const - { - return (first.op == second.op && first.fbo == second.fbo && first.spare == second.spare); - } - - WriteEngine::WriteEngineWrapper* fWriteEngine; - execplan::SessionManager* fSessionManager; - uint32_t fSessionID; - execplan::CalpontSystemCatalog::SCN fTxnID; - DDLPackageProcessor::DDLResult fResult; - DDLPackageProcessor::IndexOID fIdxOID; - ddlpackage::ColumnNameList fColNames; - ddlpackage::QualifiedName fTable; - DDLPackageProcessor::DebugLevel fDebugLevel; - joblist::DistributedEngineComm* fEC; - WriteEngine::RIDList fRidList; - WriteEngine::IdxStructList fIdxStructList; - WriteEngine::IdxValueList fIdxValueList; - - ddlpackage::DDL_CONSTRAINTS fConstraint; - std::vector fUniqueColResultList; - std::vector fOidList; - std::ifstream fColumnFile; - static const int fTOKENSIZE = 8; - - struct DDLNullValueForType : DDLPackageProcessor - { - DDLNullValueForType(const execplan::CalpontSystemCatalog::ColType& ctype) - : DDLPackageProcessor(), fType(ctype) {} - boost::any operator()(execplan::CalpontSystemCatalog::ColType& ctype) - { - return ctype.getNullValueForType(); - } - const execplan::CalpontSystemCatalog::ColType& fType; - }; - + const execplan::CalpontSystemCatalog::ColType& fType; + }; }; -} -#endif //DDLPINDEXPOPULATOR_H +} // namespace ddlpackageprocessor +#endif // DDLPINDEXPOPULATOR_H diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index fb3e1e78e..3740035d6 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -67,11 +67,11 @@ const SOP opne(new Operator("<>")); const SOP opor(new Operator("or")); const SOP opand(new Operator("and")); const SOP opis(new Operator("is")); -} +} // namespace #include -using boost::lexical_cast; using boost::any_cast; +using boost::lexical_cast; using namespace std; using namespace execplan; @@ -83,1138 +83,1037 @@ boost::mutex DDLPackageProcessor::dbrmMutex; DDLPackageProcessor::~DDLPackageProcessor() { - //cout << "in DDLPackageProcessor destructor " << this << endl; - delete fWEClient; + // cout << "in DDLPackageProcessor destructor " << this << endl; + delete fWEClient; } -void DDLPackageProcessor::getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, - ColumnList& colList) +void DDLPackageProcessor::getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, + ColumnList& colList) { + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = table; + std::string err; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = table; - std::string err; + try + { + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - try + const CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName); + + CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); + + while (rid_iterator != ridList.end()) { - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair roPair = *rid_iterator; - const CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName); + DDLColumn column; + column.oid = roPair.objnum; + column.colType = systemCatalogPtr->colType(column.oid); + column.tableColName = systemCatalogPtr->colName(column.oid); - CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); - - while (rid_iterator != ridList.end()) - { - CalpontSystemCatalog::ROPair roPair = *rid_iterator; - - DDLColumn column; - column.oid = roPair.objnum; - column.colType = systemCatalogPtr->colType(column.oid); - column.tableColName = systemCatalogPtr->colName(column.oid); - - colList.push_back(column); - - ++rid_iterator; - } + colList.push_back(column); + ++rid_iterator; } - catch (std::exception& ex) - { - - err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " + schema + '.' + table + ": " + ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!" ; - throw std::runtime_error(err); - } - + } + catch (std::exception& ex) + { + err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " + schema + '.' + table + + ": " + ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!"; + throw std::runtime_error(err); + } } execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType(int dataType) { - execplan::CalpontSystemCatalog::ColDataType colDataType; + execplan::CalpontSystemCatalog::ColDataType colDataType; - switch (dataType) - { - case ddlpackage::DDL_CHAR: - colDataType = CalpontSystemCatalog::CHAR; - break; + switch (dataType) + { + case ddlpackage::DDL_CHAR: colDataType = CalpontSystemCatalog::CHAR; break; - case ddlpackage::DDL_VARCHAR: - colDataType = CalpontSystemCatalog::VARCHAR; - break; + case ddlpackage::DDL_VARCHAR: colDataType = CalpontSystemCatalog::VARCHAR; break; - case ddlpackage::DDL_VARBINARY: - colDataType = CalpontSystemCatalog::VARBINARY; - break; + case ddlpackage::DDL_VARBINARY: colDataType = CalpontSystemCatalog::VARBINARY; break; - case ddlpackage::DDL_BIT: - colDataType = CalpontSystemCatalog::BIT; - break; + case ddlpackage::DDL_BIT: colDataType = CalpontSystemCatalog::BIT; break; - case ddlpackage::DDL_REAL: - case ddlpackage::DDL_DECIMAL: - case ddlpackage::DDL_NUMERIC: - case ddlpackage::DDL_NUMBER: - colDataType = CalpontSystemCatalog::DECIMAL; - break; + case ddlpackage::DDL_REAL: + case ddlpackage::DDL_DECIMAL: + case ddlpackage::DDL_NUMERIC: + case ddlpackage::DDL_NUMBER: colDataType = CalpontSystemCatalog::DECIMAL; break; - case ddlpackage::DDL_FLOAT: - colDataType = CalpontSystemCatalog::FLOAT; - break; + case ddlpackage::DDL_FLOAT: colDataType = CalpontSystemCatalog::FLOAT; break; - case ddlpackage::DDL_DOUBLE: - colDataType = CalpontSystemCatalog::DOUBLE; - break; + case ddlpackage::DDL_DOUBLE: colDataType = CalpontSystemCatalog::DOUBLE; break; - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: - colDataType = CalpontSystemCatalog::INT; - break; + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: colDataType = CalpontSystemCatalog::INT; break; - case ddlpackage::DDL_BIGINT: - colDataType = CalpontSystemCatalog::BIGINT; - break; + case ddlpackage::DDL_BIGINT: colDataType = CalpontSystemCatalog::BIGINT; break; - case ddlpackage::DDL_MEDINT: - colDataType = CalpontSystemCatalog::MEDINT; - break; + case ddlpackage::DDL_MEDINT: colDataType = CalpontSystemCatalog::MEDINT; break; - case ddlpackage::DDL_SMALLINT: - colDataType = CalpontSystemCatalog::SMALLINT; - break; + case ddlpackage::DDL_SMALLINT: colDataType = CalpontSystemCatalog::SMALLINT; break; - case ddlpackage::DDL_TINYINT: - colDataType = CalpontSystemCatalog::TINYINT; - break; + case ddlpackage::DDL_TINYINT: colDataType = CalpontSystemCatalog::TINYINT; break; - case ddlpackage::DDL_UNSIGNED_DECIMAL: - case ddlpackage::DDL_UNSIGNED_NUMERIC: - colDataType = CalpontSystemCatalog::UDECIMAL; - break; + case ddlpackage::DDL_UNSIGNED_DECIMAL: + case ddlpackage::DDL_UNSIGNED_NUMERIC: colDataType = CalpontSystemCatalog::UDECIMAL; break; - case ddlpackage::DDL_UNSIGNED_FLOAT: - colDataType = CalpontSystemCatalog::UFLOAT; - break; + case ddlpackage::DDL_UNSIGNED_FLOAT: colDataType = CalpontSystemCatalog::UFLOAT; break; - case ddlpackage::DDL_UNSIGNED_DOUBLE: - colDataType = CalpontSystemCatalog::UDOUBLE; - break; + case ddlpackage::DDL_UNSIGNED_DOUBLE: colDataType = CalpontSystemCatalog::UDOUBLE; break; - case ddlpackage::DDL_UNSIGNED_INT: - colDataType = CalpontSystemCatalog::UINT; - break; + case ddlpackage::DDL_UNSIGNED_INT: colDataType = CalpontSystemCatalog::UINT; break; - case ddlpackage::DDL_UNSIGNED_BIGINT: - colDataType = CalpontSystemCatalog::UBIGINT; - break; + case ddlpackage::DDL_UNSIGNED_BIGINT: colDataType = CalpontSystemCatalog::UBIGINT; break; - case ddlpackage::DDL_UNSIGNED_MEDINT: - colDataType = CalpontSystemCatalog::UMEDINT; - break; + case ddlpackage::DDL_UNSIGNED_MEDINT: colDataType = CalpontSystemCatalog::UMEDINT; break; - case ddlpackage::DDL_UNSIGNED_SMALLINT: - colDataType = CalpontSystemCatalog::USMALLINT; - break; + case ddlpackage::DDL_UNSIGNED_SMALLINT: colDataType = CalpontSystemCatalog::USMALLINT; break; - case ddlpackage::DDL_UNSIGNED_TINYINT: - colDataType = CalpontSystemCatalog::UTINYINT; - break; + case ddlpackage::DDL_UNSIGNED_TINYINT: colDataType = CalpontSystemCatalog::UTINYINT; break; - case ddlpackage::DDL_DATE: - colDataType = CalpontSystemCatalog::DATE; - break; + case ddlpackage::DDL_DATE: colDataType = CalpontSystemCatalog::DATE; break; - case ddlpackage::DDL_DATETIME: - colDataType = CalpontSystemCatalog::DATETIME; - break; + case ddlpackage::DDL_DATETIME: colDataType = CalpontSystemCatalog::DATETIME; break; - case ddlpackage::DDL_TIME: - colDataType = CalpontSystemCatalog::TIME; - break; + case ddlpackage::DDL_TIME: colDataType = CalpontSystemCatalog::TIME; break; - case ddlpackage::DDL_TIMESTAMP: - colDataType = CalpontSystemCatalog::TIMESTAMP; - break; + case ddlpackage::DDL_TIMESTAMP: colDataType = CalpontSystemCatalog::TIMESTAMP; break; - case ddlpackage::DDL_CLOB: - colDataType = CalpontSystemCatalog::CLOB; - break; + case ddlpackage::DDL_CLOB: colDataType = CalpontSystemCatalog::CLOB; break; - case ddlpackage::DDL_BLOB: - colDataType = CalpontSystemCatalog::BLOB; - break; + case ddlpackage::DDL_BLOB: colDataType = CalpontSystemCatalog::BLOB; break; - case ddlpackage::DDL_TEXT: - colDataType = CalpontSystemCatalog::TEXT; - break; - - default: - throw runtime_error("Unsupported datatype!"); + case ddlpackage::DDL_TEXT: colDataType = CalpontSystemCatalog::TEXT; break; - } + default: throw runtime_error("Unsupported datatype!"); + } - return colDataType; + return colDataType; } -std::string DDLPackageProcessor::buildTableConstraintName(const int oid, - ddlpackage::DDL_CONSTRAINTS type) +std::string DDLPackageProcessor::buildTableConstraintName(const int oid, ddlpackage::DDL_CONSTRAINTS type) { - std::stringstream oid_number; - oid_number << oid; - std::string indexName; - std::string prefix; + std::stringstream oid_number; + oid_number << oid; + std::string indexName; + std::string prefix; - switch (type) - { - case ddlpackage::DDL_PRIMARY_KEY: - //prefix = "pk_"; - // @note this name is supplied by the previous create index statement - // generated by Oracle. Use Oracle's PK name instead of making up our own - indexName = fPKName; - break; + switch (type) + { + case ddlpackage::DDL_PRIMARY_KEY: + // prefix = "pk_"; + // @note this name is supplied by the previous create index statement + // generated by Oracle. Use Oracle's PK name instead of making up our own + indexName = fPKName; + break; - case ddlpackage::DDL_FOREIGN_KEY: - case ddlpackage::DDL_REFERENCES: - prefix = "fk_"; - break; + case ddlpackage::DDL_FOREIGN_KEY: + case ddlpackage::DDL_REFERENCES: prefix = "fk_"; break; - case ddlpackage::DDL_UNIQUE: - prefix = "uk_"; - break; + case ddlpackage::DDL_UNIQUE: prefix = "uk_"; break; - case ddlpackage::DDL_CHECK: - prefix = "ck_"; - break; + case ddlpackage::DDL_CHECK: prefix = "ck_"; break; - case ddlpackage::DDL_NOT_NULL: - prefix = "nk_"; - break; + case ddlpackage::DDL_NOT_NULL: prefix = "nk_"; break; - default: - throw runtime_error("Unsupported constraint type!"); - break; - } + default: throw runtime_error("Unsupported constraint type!"); break; + } - if (type != ddlpackage::DDL_PRIMARY_KEY) - indexName = prefix + oid_number.str(); + if (type != ddlpackage::DDL_PRIMARY_KEY) + indexName = prefix + oid_number.str(); - boost::to_lower(indexName); + boost::to_lower(indexName); - return indexName; + return indexName; } std::string DDLPackageProcessor::buildColumnConstraintName(const std::string& schema, - const std::string& table, - const std::string& column, - ddlpackage::DDL_CONSTRAINTS type) + const std::string& table, + const std::string& column, + ddlpackage::DDL_CONSTRAINTS type) { - std::string indexName; + std::string indexName; - std::string prefix; + std::string prefix; - switch (type) - { - case ddlpackage::DDL_PRIMARY_KEY: - prefix = "pk_"; - break; + switch (type) + { + case ddlpackage::DDL_PRIMARY_KEY: prefix = "pk_"; break; - case ddlpackage::DDL_FOREIGN_KEY: - case ddlpackage::DDL_REFERENCES: - prefix = "fk_"; - break; + case ddlpackage::DDL_FOREIGN_KEY: + case ddlpackage::DDL_REFERENCES: prefix = "fk_"; break; - case ddlpackage::DDL_UNIQUE: - prefix = "uk_"; - break; + case ddlpackage::DDL_UNIQUE: prefix = "uk_"; break; - case ddlpackage::DDL_CHECK: - prefix = "ck_"; - break; + case ddlpackage::DDL_CHECK: prefix = "ck_"; break; - case ddlpackage::DDL_NOT_NULL: - prefix = "nk_"; - break; + case ddlpackage::DDL_NOT_NULL: prefix = "nk_"; break; - default: - throw runtime_error("Unsupported constraint type!"); - break; - } + default: throw runtime_error("Unsupported constraint type!"); break; + } - indexName = prefix + schema + "_" + table + "_" + column; + indexName = prefix + schema + "_" + table + "_" + column; - boost::to_lower(indexName); + boost::to_lower(indexName); - return indexName; + return indexName; } char DDLPackageProcessor::getConstraintCode(ddlpackage::DDL_CONSTRAINTS type) { - char constraint_type; + char constraint_type; - switch (type) - { - case ddlpackage::DDL_PRIMARY_KEY: - constraint_type = 'p'; - break; + switch (type) + { + case ddlpackage::DDL_PRIMARY_KEY: constraint_type = 'p'; break; - case ddlpackage::DDL_REFERENCES: - case ddlpackage::DDL_FOREIGN_KEY: - constraint_type = 'f'; - break; + case ddlpackage::DDL_REFERENCES: + case ddlpackage::DDL_FOREIGN_KEY: constraint_type = 'f'; break; - case ddlpackage::DDL_UNIQUE: - constraint_type = 'u'; - break; + case ddlpackage::DDL_UNIQUE: constraint_type = 'u'; break; - case ddlpackage::DDL_CHECK: - constraint_type = 'c'; - break; + case ddlpackage::DDL_CHECK: constraint_type = 'c'; break; - case ddlpackage::DDL_NOT_NULL: - constraint_type = 'n'; - break; + case ddlpackage::DDL_NOT_NULL: constraint_type = 'n'; break; - default: - constraint_type = '0'; - break; - } + default: constraint_type = '0'; break; + } - return constraint_type; + return constraint_type; } - bool DDLPackageProcessor::isIndexConstraint(ddlpackage::DDL_CONSTRAINTS type) { - bool indexConstraint = false; + bool indexConstraint = false; - switch (type) - { - case ddlpackage::DDL_PRIMARY_KEY: + switch (type) + { + case ddlpackage::DDL_PRIMARY_KEY: - // @bug fix for #418, #416. Do not insert into sysindex - //case ddlpackage::DDL_REFERENCES: - case ddlpackage::DDL_UNIQUE: - indexConstraint = true; - break; + // @bug fix for #418, #416. Do not insert into sysindex + // case ddlpackage::DDL_REFERENCES: + case ddlpackage::DDL_UNIQUE: indexConstraint = true; break; - default: - break; + default: break; + } - } - - return indexConstraint; + return indexConstraint; } void DDLPackageProcessor::getColumnReferences(ddlpackage::TableConstraintDef& tableConstraint, - ddlpackage::ColumnNameList& columns) + ddlpackage::ColumnNameList& columns) { - - switch (tableConstraint.fConstraintType) + switch (tableConstraint.fConstraintType) + { + case ddlpackage::DDL_PRIMARY_KEY: { - case ddlpackage::DDL_PRIMARY_KEY: - { - ddlpackage::TablePrimaryKeyConstraintDef& pkConstraint = - dynamic_cast(tableConstraint); + ddlpackage::TablePrimaryKeyConstraintDef& pkConstraint = + dynamic_cast(tableConstraint); - columns = pkConstraint.fColumnNameList; - } - break; - - case ddlpackage::DDL_REFERENCES: - case ddlpackage::DDL_FOREIGN_KEY: - { - ddlpackage::TableReferencesConstraintDef& fkConstraint = - dynamic_cast(tableConstraint); - - columns = fkConstraint.fColumns; - } - break; - - case ddlpackage::DDL_UNIQUE: - { - ddlpackage::TableUniqueConstraintDef& ukConstraint = - dynamic_cast(tableConstraint); - columns = ukConstraint.fColumnNameList; - } - break; - - default: - break; + columns = pkConstraint.fColumnNameList; } + break; + case ddlpackage::DDL_REFERENCES: + case ddlpackage::DDL_FOREIGN_KEY: + { + ddlpackage::TableReferencesConstraintDef& fkConstraint = + dynamic_cast(tableConstraint); + + columns = fkConstraint.fColumns; + } + break; + + case ddlpackage::DDL_UNIQUE: + { + ddlpackage::TableUniqueConstraintDef& ukConstraint = + dynamic_cast(tableConstraint); + columns = ukConstraint.fColumnNameList; + } + break; + + default: break; + } } -boost::any DDLPackageProcessor::tokenizeData(execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - const execplan::CalpontSystemCatalog::ColType& colType, - const boost::any& data) +boost::any DDLPackageProcessor::tokenizeData(execplan::CalpontSystemCatalog::SCN txnID, + const DDLResult& result, + const execplan::CalpontSystemCatalog::ColType& colType, + const boost::any& data) { - std::string err("DDLPackageProcessor::tokenizeData "); - SUMMARY_INFO(err); - boost::any value; + std::string err("DDLPackageProcessor::tokenizeData "); + SUMMARY_INFO(err); + boost::any value; - if (result.result == NO_ERROR) + if (result.result == NO_ERROR) + { + try { + std::string str; - try - { - std::string str; + if (data.type() == typeid(int)) + str = lexical_cast(any_cast(data)); + else if (data.type() == typeid(float)) + str = lexical_cast(any_cast(data)); + else if (data.type() == typeid(long long)) + str = lexical_cast(any_cast(data)); + else if (data.type() == typeid(double)) + str = lexical_cast(any_cast(data)); + else if (data.type() == typeid(bool)) + str = lexical_cast(any_cast(data)); + else if (data.type() == typeid(short)) + str = lexical_cast(any_cast(data)); + else if (data.type() == typeid(char)) + str = lexical_cast(any_cast(data)); + else + str = any_cast(data); - if (data.type() == typeid(int)) - str = lexical_cast(any_cast(data)); - else if (data.type() == typeid(float)) - str = lexical_cast(any_cast(data)); - else if (data.type() == typeid(long long)) - str = lexical_cast(any_cast(data)); - else if (data.type() == typeid(double)) - str = lexical_cast(any_cast(data)); - else if (data.type() == typeid(bool)) - str = lexical_cast(any_cast(data)); - else if (data.type() == typeid(short)) - str = lexical_cast(any_cast(data)); - else if (data.type() == typeid(char)) - str = lexical_cast(any_cast(data)); - else - str = any_cast(data); + // Tokenize the data value + WriteEngine::DctnryStruct dictStruct; + dictStruct.dctnryOid = colType.ddn.dictOID; + // added for multifiles per oid + dictStruct.columnOid = colType.columnOID; + WriteEngine::DctnryTuple dictTuple; + dictTuple.sigValue = (unsigned char*)str.c_str(); + dictTuple.sigSize = str.length(); + int error = NO_ERROR; - //Tokenize the data value - WriteEngine::DctnryStruct dictStruct; - dictStruct.dctnryOid = colType.ddn.dictOID; - //added for multifiles per oid - dictStruct.columnOid = colType.columnOID; - WriteEngine::DctnryTuple dictTuple; - dictTuple.sigValue = (unsigned char*)str.c_str(); - dictTuple.sigSize = str.length(); - int error = NO_ERROR; + if (NO_ERROR != + (error = fWriteEngine.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file + { + WErrorCodes ec; + throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); + } - if (NO_ERROR != (error = fWriteEngine.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file - { - WErrorCodes ec; - throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); - } + WriteEngine::Token aToken = dictTuple.token; - WriteEngine::Token aToken = dictTuple.token; - - value = aToken; - - } - catch (std::exception& ex) - { - err += ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err += "Unknown exception caught, tokenizeData failed."; - throw std::runtime_error(err); - - } + value = aToken; } + catch (std::exception& ex) + { + err += ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err += "Unknown exception caught, tokenizeData failed."; + throw std::runtime_error(err); + } + } - return value; + return value; } void DDLPackageProcessor::flushPrimprocCache(std::vector& oidList) { - SUMMARY_INFO("DDLPackageProcessor::flushPrimprocCache"); + SUMMARY_INFO("DDLPackageProcessor::flushPrimprocCache"); - int err = 0; - CalpontSystemCatalog::ROPair roPair; - std::vector::const_iterator iter = oidList.begin(); - std::string error; - BRM::DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - LBIDRange_v::iterator it; - BRM::BlockList_t blockList; - execplan::CalpontSystemCatalog::SCN verID = 0; + int err = 0; + CalpontSystemCatalog::ROPair roPair; + std::vector::const_iterator iter = oidList.begin(); + std::string error; + BRM::DBRM dbrm; + BRM::LBIDRange_v lbidRanges; + LBIDRange_v::iterator it; + BRM::BlockList_t blockList; + execplan::CalpontSystemCatalog::SCN verID = 0; - try + try + { + while (iter != oidList.end()) { - while (iter != oidList.end()) + WriteEngine::OID oid = *iter; + + if (oid < 3000) + { + ++iter; + continue; + } + + //@Bug 1708 Flush primproc cache for associated lbids. + err = dbrm.lookup(oid, lbidRanges); + + if (err) + { + error = "DBRM lookUp error."; + throw std::runtime_error(error); + } + + blockList.clear(); + + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + for (LBID_t lbid = it->start; lbid < (it->start + it->size); lbid++) { - WriteEngine::OID oid = *iter; - - if (oid < 3000) - { - ++iter; - continue; - } - - //@Bug 1708 Flush primproc cache for associated lbids. - err = dbrm.lookup(oid, lbidRanges); - - if (err) - { - error = "DBRM lookUp error."; - throw std::runtime_error(error); - } - - blockList.clear(); - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - for (LBID_t lbid = it->start; lbid < (it->start + it->size); lbid++) - { - blockList.push_back(BRM::LVP_t(lbid, verID)); - } - } - - //Need find a more efficient way to do this. - err = cacheutils::flushPrimProcBlocks (blockList); - //No need to handle this error as the error comes from timeout, not real error - (void)err; - - ++iter; + blockList.push_back(BRM::LVP_t(lbid, verID)); } + } + + // Need find a more efficient way to do this. + err = cacheutils::flushPrimProcBlocks(blockList); + // No need to handle this error as the error comes from timeout, not real error + (void)err; + + ++iter; } - catch (std::exception& ex) - { - error = ex.what(); - throw std::runtime_error(error); - } - catch (...) - { - error = "Unknown exception caught"; - throw std::runtime_error(error); - } + } + catch (std::exception& ex) + { + error = ex.what(); + throw std::runtime_error(error); + } + catch (...) + { + error = "Unknown exception caught"; + throw std::runtime_error(error); + } } - - -void DDLPackageProcessor::removeFiles(const uint64_t uniqueId, std::vector& oidList) +void DDLPackageProcessor::removeFiles(const uint64_t uniqueId, + std::vector& oidList) { - SUMMARY_INFO("DDLPackageProcessor::removeFiles"); - ByteStream bytestream; - boost::shared_ptr bsIn; - fWEClient->addQueue(uniqueId); - bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; - bytestream << uniqueId; - bytestream << (uint32_t) oidList.size(); + SUMMARY_INFO("DDLPackageProcessor::removeFiles"); + ByteStream bytestream; + boost::shared_ptr bsIn; + fWEClient->addQueue(uniqueId); + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; + bytestream << uniqueId; + bytestream << (uint32_t)oidList.size(); - for (unsigned i = 0; i < oidList.size(); i++) + for (unsigned i = 0; i < oidList.size(); i++) + { + bytestream << (uint32_t)oidList[i]; + } + + uint32_t msgRecived = 0; + ByteStream::byte rc = 0; + ByteStream::byte tmp8; + string errorMsg; + + try + { + fWEClient->write_to_all(bytestream); + + bsIn.reset(new ByteStream()); + + while (1) { - bytestream << (uint32_t) oidList[i]; - } + if (msgRecived == fWEClient->getPmCount()) + break; - uint32_t msgRecived = 0; - ByteStream::byte rc = 0; - ByteStream::byte tmp8; - string errorMsg; + fWEClient->read(uniqueId, bsIn); - try - { - fWEClient->write_to_all(bytestream); + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Network error while deleting files."; + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; - bsIn.reset(new ByteStream()); - - while (1) + if (rc != 0) { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Network error while deleting files."; - fWEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; } + else + msgRecived++; + } } - catch (std::exception& ex) - { - fWEClient->removeQueue(uniqueId); - throw std::runtime_error(ex.what()); - } - catch (...) - { - fWEClient->removeQueue(uniqueId); - throw std::runtime_error("Unknown error caught while deleting files."); - } - + } + catch (std::exception& ex) + { fWEClient->removeQueue(uniqueId); + throw std::runtime_error(ex.what()); + } + catch (...) + { + fWEClient->removeQueue(uniqueId); + throw std::runtime_error("Unknown error caught while deleting files."); + } - if ( rc != 0) - { - throw std::runtime_error(errorMsg); - } + fWEClient->removeQueue(uniqueId); + + if (rc != 0) + { + throw std::runtime_error(errorMsg); + } } void DDLPackageProcessor::createFiles(CalpontSystemCatalog::TableName aTableName, const int useDBRoot, const uint64_t uniqueId, const uint32_t numOids) { - SUMMARY_INFO("DDLPackageProcessor::createFiles"); - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(1); - CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(aTableName); - fWEClient->addQueue(uniqueId); - CalpontSystemCatalog::ColType colType; - ByteStream bytestream; - boost::shared_ptr bsIn; - bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES; - bytestream << (uint32_t)1; - bytestream << uniqueId; - bytestream << numOids; + SUMMARY_INFO("DDLPackageProcessor::createFiles"); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(1); + CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(aTableName); + fWEClient->addQueue(uniqueId); + CalpontSystemCatalog::ColType colType; + ByteStream bytestream; + boost::shared_ptr bsIn; + bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES; + bytestream << (uint32_t)1; + bytestream << uniqueId; + bytestream << numOids; - for (unsigned col = 0; col < ridList.size(); col++) + for (unsigned col = 0; col < ridList.size(); col++) + { + colType = systemCatalogPtr->colType(ridList[col].objnum); + bytestream << (uint32_t)ridList[col].objnum; + bytestream << (uint8_t)colType.colDataType; + bytestream << (uint8_t) false; + bytestream << (uint32_t)colType.colWidth; + bytestream << (uint16_t)useDBRoot; + bytestream << (uint32_t)colType.compressionType; + + if (colType.ddn.dictOID > 3000) { - colType = systemCatalogPtr->colType(ridList[col].objnum); - bytestream << (uint32_t) ridList[col].objnum; - bytestream << (uint8_t) colType.colDataType; - bytestream << (uint8_t) false; - bytestream << (uint32_t) colType.colWidth; - bytestream << (uint16_t) useDBRoot; - bytestream << (uint32_t) colType.compressionType; - - if (colType.ddn.dictOID > 3000) - { - bytestream << (uint32_t) colType.ddn.dictOID; - bytestream << (uint8_t) colType.colDataType; - bytestream << (uint8_t) true; - bytestream << (uint32_t) colType.colWidth; - bytestream << (uint16_t) useDBRoot; - bytestream << (uint32_t) colType.compressionType; - } + bytestream << (uint32_t)colType.ddn.dictOID; + bytestream << (uint8_t)colType.colDataType; + bytestream << (uint8_t) true; + bytestream << (uint32_t)colType.colWidth; + bytestream << (uint16_t)useDBRoot; + bytestream << (uint32_t)colType.compressionType; } + } - ByteStream::byte rc = 0; - ByteStream::byte tmp8; - string errorMsg; + ByteStream::byte rc = 0; + ByteStream::byte tmp8; + string errorMsg; - try + try + { + OamCache* oamcache = OamCache::makeOamCache(); + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + int pmNum = (*dbRootPMMap)[useDBRoot]; + + fWEClient->write(bytestream, (uint32_t)pmNum); + bsIn.reset(new ByteStream()); + + while (1) { - OamCache* oamcache = OamCache::makeOamCache(); - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - int pmNum = (*dbRootPMMap)[useDBRoot]; + fWEClient->read(uniqueId, bsIn); - fWEClient->write(bytestream, (uint32_t)pmNum); - bsIn.reset(new ByteStream()); - - while (1) - { - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Network error while creating files."; - fWEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (std::exception& ex) - { + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Network error while creating files."; fWEClient->removeQueue(uniqueId); - throw std::runtime_error(ex.what()); - } - catch (...) - { - fWEClient->removeQueue(uniqueId); - throw std::runtime_error("Unknown error caught while creating files."); - } + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } + } + catch (std::exception& ex) + { fWEClient->removeQueue(uniqueId); + throw std::runtime_error(ex.what()); + } + catch (...) + { + fWEClient->removeQueue(uniqueId); + throw std::runtime_error("Unknown error caught while creating files."); + } - if ( rc != 0) - throw std::runtime_error(errorMsg); + fWEClient->removeQueue(uniqueId); + + if (rc != 0) + throw std::runtime_error(errorMsg); } void DDLPackageProcessor::removePartitionFiles(std::vector& oidList, - const PartitionNums& partitions, - uint64_t uniqueId) + const PartitionNums& partitions, uint64_t uniqueId) { - SUMMARY_INFO("DDLPackageProcessor::removeFiles"); + SUMMARY_INFO("DDLPackageProcessor::removeFiles"); - ByteStream::byte rc = 0; - std::string errorMsg; + ByteStream::byte rc = 0; + std::string errorMsg; - //get a unique number. - fWEClient->addQueue(uniqueId); - // Write the tables metadata to the system catalog - VERBOSE_INFO("Remove Partition Files"); - ByteStream bs; - bs << (ByteStream::byte)WE_SVR_DROP_PARTITIONS; - bs << uniqueId; - bs << (uint32_t)oidList.size(); - PartitionNums::const_iterator partIt; - vector partInfos; + // get a unique number. + fWEClient->addQueue(uniqueId); + // Write the tables metadata to the system catalog + VERBOSE_INFO("Remove Partition Files"); + ByteStream bs; + bs << (ByteStream::byte)WE_SVR_DROP_PARTITIONS; + bs << uniqueId; + bs << (uint32_t)oidList.size(); + PartitionNums::const_iterator partIt; + vector partInfos; - for (uint32_t i = 0; i < oidList.size(); i++) + for (uint32_t i = 0; i < oidList.size(); i++) + { + bs << (uint32_t)oidList[i]; + + // add oid to LogicalPartition to form PartitionInfo + for (partIt = partitions.begin(); partIt != partitions.end(); ++partIt) { - bs << (uint32_t)oidList[i]; - - // add oid to LogicalPartition to form PartitionInfo - for (partIt = partitions.begin(); partIt != partitions.end(); ++partIt) - { - PartitionInfo pi; - pi.lp = (*partIt); - pi.oid = oidList[i]; - partInfos.push_back(pi); - } + PartitionInfo pi; + pi.lp = (*partIt); + pi.oid = oidList[i]; + partInfos.push_back(pi); } + } - bs << (uint32_t)partInfos.size(); + bs << (uint32_t)partInfos.size(); - for (uint32_t i = 0; i < partInfos.size(); i++) - partInfos[i].serialize(bs); + for (uint32_t i = 0; i < partInfos.size(); i++) + partInfos[i].serialize(bs); - fWEClient->write_to_all(bs); - uint32_t pmCount = fWEClient->getPmCount(); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); + fWEClient->write_to_all(bs); + uint32_t pmCount = fWEClient->getPmCount(); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); - while (pmCount) + while (pmCount) + { + bsIn->restart(); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - bsIn->restart(); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while dropping partitions"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - break; - } - - pmCount--; - } + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while dropping partitions"; + break; } - - if (rc) + else { - WErrorCodes ec; - errorMsg = "WE: Error removing files " + ec.errorString(rc); - rc = cacheutils::dropPrimProcFdCache(); - fWEClient->removeQueue(uniqueId); - throw std::runtime_error(errorMsg); - } + *bsIn >> rc; - //@Bug 2171,3327. Drop PrimProc fd cache + if (rc != 0) + { + *bsIn >> errorMsg; + break; + } + + pmCount--; + } + } + + if (rc) + { + WErrorCodes ec; + errorMsg = "WE: Error removing files " + ec.errorString(rc); rc = cacheutils::dropPrimProcFdCache(); fWEClient->removeQueue(uniqueId); + throw std::runtime_error(errorMsg); + } + + //@Bug 2171,3327. Drop PrimProc fd cache + rc = cacheutils::dropPrimProcFdCache(); + fWEClient->removeQueue(uniqueId); } void DDLPackageProcessor::removeExtents(std::vector& oidList) { - SUMMARY_INFO("DDLPackageProcessor::removeExtents"); - int err = 0; - err = fDbrm->deleteOIDs(oidList); - - if (err) - { - string errMsg; - BRM::errString(err, errMsg); - throw std::runtime_error(errMsg); - } + SUMMARY_INFO("DDLPackageProcessor::removeExtents"); + int err = 0; + err = fDbrm->deleteOIDs(oidList); + if (err) + { + string errMsg; + BRM::errString(err, errMsg); + throw std::runtime_error(errMsg); + } } void DDLPackageProcessor::createWriteDropLogFile(execplan::CalpontSystemCatalog::OID tableOid, - uint64_t uniqueId, std::vector& oidList) + uint64_t uniqueId, + std::vector& oidList) { - SUMMARY_INFO("DDLPackageProcessor::createWriteDropLogFile"); - //For shared nothing, the meta files are created under data1 with controllernode. - OamCache* oamcache = OamCache::makeOamCache(); - std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); - OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); - int parentId = atoi(OAMParentModuleName.c_str()); - ByteStream bytestream; - uint8_t rc = 0; - std::string errorMsg; - boost::shared_ptr bsIn; - bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPTABLE; - bytestream << uniqueId; - bytestream << (uint32_t)tableOid; - bytestream << (uint32_t) oidList.size(); + SUMMARY_INFO("DDLPackageProcessor::createWriteDropLogFile"); + // For shared nothing, the meta files are created under data1 with controllernode. + OamCache* oamcache = OamCache::makeOamCache(); + std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); + OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); + int parentId = atoi(OAMParentModuleName.c_str()); + ByteStream bytestream; + uint8_t rc = 0; + std::string errorMsg; + boost::shared_ptr bsIn; + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPTABLE; + bytestream << uniqueId; + bytestream << (uint32_t)tableOid; + bytestream << (uint32_t)oidList.size(); - for (uint32_t i = 0; i < oidList.size(); i++) + for (uint32_t i = 0; i < oidList.size(); i++) + { + bytestream << (uint32_t)oidList[i]; + } + + try + { + fWEClient->write(bytestream, (uint32_t)parentId); + + while (1) { - bytestream << (uint32_t)oidList[i]; - } + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - try - { - fWEClient->write(bytestream, (uint32_t)parentId); + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while writting drop table Log"; + break; + } + else + { + *bsIn >> rc; - while (1) + if (rc != 0) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while writting drop table Log"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } + *bsIn >> errorMsg; } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = "Got unknown exception while writting drop table Log." ; - } - if ( rc != 0) - throw std::runtime_error(errorMsg); + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = "Got unknown exception while writting drop table Log."; + } + + if (rc != 0) + throw std::runtime_error(errorMsg); } -void DDLPackageProcessor::deleteLogFile(LogFileType fileType, execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId) +void DDLPackageProcessor::deleteLogFile(LogFileType fileType, execplan::CalpontSystemCatalog::OID tableOid, + uint64_t uniqueId) { - SUMMARY_INFO("DDLPackageProcessor::deleteLogFile"); - OamCache* oamcache = OamCache::makeOamCache(); - std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); - OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); - int parentId = atoi(OAMParentModuleName.c_str()); - ByteStream bytestream; - uint8_t rc = 0; - std::string errorMsg; - fWEClient->addQueue(uniqueId); - boost::shared_ptr bsIn; - bytestream << (ByteStream::byte)WE_SVR_DELETE_DDLLOG; - bytestream << uniqueId; - bytestream << (uint32_t) fileType; - bytestream << (uint32_t)tableOid; + SUMMARY_INFO("DDLPackageProcessor::deleteLogFile"); + OamCache* oamcache = OamCache::makeOamCache(); + std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); + OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); + int parentId = atoi(OAMParentModuleName.c_str()); + ByteStream bytestream; + uint8_t rc = 0; + std::string errorMsg; + fWEClient->addQueue(uniqueId); + boost::shared_ptr bsIn; + bytestream << (ByteStream::byte)WE_SVR_DELETE_DDLLOG; + bytestream << uniqueId; + bytestream << (uint32_t)fileType; + bytestream << (uint32_t)tableOid; - try + try + { + fWEClient->write(bytestream, (uint32_t)parentId); + + while (1) { - fWEClient->write(bytestream, (uint32_t)parentId); + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - while (1) + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while deleting DDL log"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while deleting DDL log"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } + *bsIn >> errorMsg; } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = "Got unknown exception while deleting DDL Log." ; - } - fWEClient->removeQueue(uniqueId); - - if ( rc != 0) - { - throw std::runtime_error(errorMsg); + break; + } } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = "Got unknown exception while deleting DDL Log."; + } + + fWEClient->removeQueue(uniqueId); + + if (rc != 0) + { + throw std::runtime_error(errorMsg); + } } void DDLPackageProcessor::fetchLogFile(TableLogInfo& tableLogInfos, uint64_t uniqueId) { - SUMMARY_INFO("DDLPackageProcessor::fetchLogFile"); - OamCache* oamcache = OamCache::makeOamCache(); - std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); + SUMMARY_INFO("DDLPackageProcessor::fetchLogFile"); + OamCache* oamcache = OamCache::makeOamCache(); + std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); - //Use a sensible default so that substr doesn't throw... - if (OAMParentModuleName.empty()) - OAMParentModuleName = "pm1"; + // Use a sensible default so that substr doesn't throw... + if (OAMParentModuleName.empty()) + OAMParentModuleName = "pm1"; - int parentId = atoi(OAMParentModuleName.substr(2, OAMParentModuleName.length()).c_str()); - ByteStream bytestream; - uint8_t rc = 0; - uint32_t tmp32, tableOid, numOids, numPartitions; - LogFileType logFileType; - std::string errorMsg; - fWEClient->addQueue(uniqueId); - boost::shared_ptr bsIn; - bytestream << (ByteStream::byte)WE_SVR_FETCH_DDL_LOGS; - bytestream << uniqueId; + int parentId = atoi(OAMParentModuleName.substr(2, OAMParentModuleName.length()).c_str()); + ByteStream bytestream; + uint8_t rc = 0; + uint32_t tmp32, tableOid, numOids, numPartitions; + LogFileType logFileType; + std::string errorMsg; + fWEClient->addQueue(uniqueId); + boost::shared_ptr bsIn; + bytestream << (ByteStream::byte)WE_SVR_FETCH_DDL_LOGS; + bytestream << uniqueId; - try + try + { + fWEClient->write(bytestream, (uint32_t)parentId); + + while (1) { - fWEClient->write(bytestream, (uint32_t)parentId); + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - while (1) + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while deleting DDL log"; + break; + } + else + { + *bsIn >> rc; + *bsIn >> errorMsg; + + while (bsIn->length() > 0) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); + *bsIn >> tmp32; + tableOid = tmp32; + *bsIn >> tmp32; + logFileType = (LogFileType)tmp32; + *bsIn >> tmp32; + numOids = tmp32; + OidList oidsList; + PartitionNums partitionNums; - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while deleting DDL log"; - break; - } - else - { - *bsIn >> rc; - *bsIn >> errorMsg; + for (unsigned i = 0; i < numOids; i++) + { + *bsIn >> tmp32; + oidsList.push_back(tmp32); + } - while ( bsIn->length() > 0 ) - { - *bsIn >> tmp32; - tableOid = tmp32; - *bsIn >> tmp32; - logFileType = (LogFileType)tmp32; - *bsIn >> tmp32; - numOids = tmp32; - OidList oidsList; - PartitionNums partitionNums; + *bsIn >> tmp32; + numPartitions = tmp32; + BRM::LogicalPartition lp; - for (unsigned i = 0; i < numOids; i++) - { - *bsIn >> tmp32; - oidsList.push_back(tmp32); - } + for (unsigned i = 0; i < numPartitions; i++) + { + lp.unserialize(*bsIn); + partitionNums.insert(lp); + } - *bsIn >> tmp32; - numPartitions = tmp32; - BRM::LogicalPartition lp; - - for (unsigned i = 0; i < numPartitions; i++) - { - lp.unserialize(*bsIn); - partitionNums.insert(lp); - } - - //build the tableloginfo - LogInfo aLog; - aLog.fileType = logFileType; - aLog.oids = oidsList; - aLog.partitionNums = partitionNums; - tableLogInfos[tableOid] = aLog; - } - - break; - } + // build the tableloginfo + LogInfo aLog; + aLog.fileType = logFileType; + aLog.oids = oidsList; + aLog.partitionNums = partitionNums; + tableLogInfos[tableOid] = aLog; } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = "Got unknown exception while fetching DDL Log." ; - } - fWEClient->removeQueue(uniqueId); + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = "Got unknown exception while fetching DDL Log."; + } - if ( rc != 0) - throw std::runtime_error(errorMsg); + fWEClient->removeQueue(uniqueId); + if (rc != 0) + throw std::runtime_error(errorMsg); } -void DDLPackageProcessor::createWritePartitionLogFile(execplan::CalpontSystemCatalog::OID tableOid, - const PartitionNums& partitionNums, - std::vector& oidList, uint64_t uniqueId) +void DDLPackageProcessor::createWritePartitionLogFile( + execplan::CalpontSystemCatalog::OID tableOid, const PartitionNums& partitionNums, + std::vector& oidList, uint64_t uniqueId) { - SUMMARY_INFO("DDLPackageProcessor::createWritePartitionLogFile"); - fWEClient->addQueue(uniqueId); - OamCache* oamcache = OamCache::makeOamCache(); - std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); - OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); - int parentId = atoi(OAMParentModuleName.c_str()); - boost::shared_ptr bsIn; - ByteStream bytestream; - std::string errorMsg; - uint8_t rc = 0; - bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPPARTITION; - bytestream << uniqueId; - bytestream << (uint32_t)tableOid; - bytestream << (uint32_t) partitionNums.size(); - PartitionNums::const_iterator it; + SUMMARY_INFO("DDLPackageProcessor::createWritePartitionLogFile"); + fWEClient->addQueue(uniqueId); + OamCache* oamcache = OamCache::makeOamCache(); + std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); + OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); + int parentId = atoi(OAMParentModuleName.c_str()); + boost::shared_ptr bsIn; + ByteStream bytestream; + std::string errorMsg; + uint8_t rc = 0; + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPPARTITION; + bytestream << uniqueId; + bytestream << (uint32_t)tableOid; + bytestream << (uint32_t)partitionNums.size(); + PartitionNums::const_iterator it; - for (it = partitionNums.begin(); it != partitionNums.end(); ++it) - (*it).serialize(bytestream); + for (it = partitionNums.begin(); it != partitionNums.end(); ++it) + (*it).serialize(bytestream); - bytestream << (uint32_t) oidList.size(); + bytestream << (uint32_t)oidList.size(); - for (uint32_t i = 0; i < oidList.size(); i++) + for (uint32_t i = 0; i < oidList.size(); i++) + { + bytestream << (uint32_t)oidList[i]; + } + + try + { + fWEClient->write(bytestream, (uint32_t)parentId); + + while (1) { - bytestream << (uint32_t)oidList[i]; - } + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - try - { - fWEClient->write(bytestream, (uint32_t)parentId); + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while writing DDL drop partition log"; + break; + } + else + { + *bsIn >> rc; - while (1) + if (rc != 0) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while writing DDL drop partition log"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } + *bsIn >> errorMsg; } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = "Got unknown exception while writting truncate Log." ; - } - fWEClient->removeQueue(uniqueId); + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = "Got unknown exception while writting truncate Log."; + } - if ( rc != 0) - throw std::runtime_error(errorMsg); + fWEClient->removeQueue(uniqueId); + + if (rc != 0) + throw std::runtime_error(errorMsg); } -void DDLPackageProcessor::createWriteTruncateTableLogFile(execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId, std::vector& oidList) +void DDLPackageProcessor::createWriteTruncateTableLogFile( + execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId, + std::vector& oidList) { - SUMMARY_INFO("DDLPackageProcessor::createWriteTruncateTableLogFile"); - //For shared nothing, the meta files are created under data1 with controllernode. - OamCache* oamcache = OamCache::makeOamCache(); - std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); - OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); - int parentId = atoi(OAMParentModuleName.c_str()); - ByteStream bytestream; - uint8_t rc = 0; - std::string errorMsg; - boost::shared_ptr bsIn; - bytestream << (ByteStream::byte)WE_SVR_WRITE_TRUNCATE; - bytestream << uniqueId; - bytestream << (uint32_t)tableOid; - bytestream << (uint32_t) oidList.size(); + SUMMARY_INFO("DDLPackageProcessor::createWriteTruncateTableLogFile"); + // For shared nothing, the meta files are created under data1 with controllernode. + OamCache* oamcache = OamCache::makeOamCache(); + std::string OAMParentModuleName = oamcache->getOAMParentModuleName(); + OAMParentModuleName = OAMParentModuleName.substr(2, OAMParentModuleName.length()); + int parentId = atoi(OAMParentModuleName.c_str()); + ByteStream bytestream; + uint8_t rc = 0; + std::string errorMsg; + boost::shared_ptr bsIn; + bytestream << (ByteStream::byte)WE_SVR_WRITE_TRUNCATE; + bytestream << uniqueId; + bytestream << (uint32_t)tableOid; + bytestream << (uint32_t)oidList.size(); - for (uint32_t i = 0; i < oidList.size(); i++) + for (uint32_t i = 0; i < oidList.size(); i++) + { + bytestream << (uint32_t)oidList[i]; + } + + try + { + fWEClient->write(bytestream, (uint32_t)parentId); + + while (1) { - bytestream << (uint32_t)oidList[i]; - } + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - try - { - fWEClient->write(bytestream, (uint32_t)parentId); + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while writing truncate table log"; + break; + } + else + { + *bsIn >> rc; - while (1) + if (rc != 0) { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while writing truncate table log"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } + *bsIn >> errorMsg; } - } - catch (runtime_error& ex) //write error - { - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; - errorMsg = "Got unknown exception while writting truncate table Log." ; - } - if ( rc != 0) - throw std::runtime_error(errorMsg); + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = "Got unknown exception while writting truncate table Log."; + } + + if (rc != 0) + throw std::runtime_error(errorMsg); } #if 0 @@ -1362,234 +1261,234 @@ void DDLPackageProcessor::updateSyscolumns(execplan::CalpontSystemCatalog::SCN t void DDLPackageProcessor::returnOIDs(execplan::CalpontSystemCatalog::RIDList& ridList, execplan::CalpontSystemCatalog::DictOIDList& dictOIDList) { - CalpontSystemCatalog::ROPair roPair; - CalpontSystemCatalog::RIDList::const_iterator col_iter = ridList.begin(); - std::string err; + CalpontSystemCatalog::ROPair roPair; + CalpontSystemCatalog::RIDList::const_iterator col_iter = ridList.begin(); + std::string err; - try + try + { + execplan::ObjectIDManager fObjectIDManager; + + while (col_iter != ridList.end()) { - execplan::ObjectIDManager fObjectIDManager; + roPair = *col_iter; - while (col_iter != ridList.end()) - { - roPair = *col_iter; + if (roPair.objnum < 3000) + { + ++col_iter; + continue; + } - if (roPair.objnum < 3000) - { - ++col_iter; - continue; - } - - fObjectIDManager.returnOID(roPair.objnum); - ++col_iter; - } - - CalpontSystemCatalog::DictOID dictOID; - CalpontSystemCatalog::DictOIDList::const_iterator dict_iter = dictOIDList.begin(); - - while (dict_iter != dictOIDList.end()) - { - dictOID = *dict_iter; - - if (dictOID.dictOID < 3000) - { - ++dict_iter; - continue; - } - - fObjectIDManager.returnOID(dictOID.dictOID); - ++dict_iter; - } + fObjectIDManager.returnOID(roPair.objnum); + ++col_iter; } - catch (std::exception& ex) + + CalpontSystemCatalog::DictOID dictOID; + CalpontSystemCatalog::DictOIDList::const_iterator dict_iter = dictOIDList.begin(); + + while (dict_iter != dictOIDList.end()) { - err = ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "returnOIDs:Unknown exception caught"; - throw std::runtime_error(err); + dictOID = *dict_iter; + + if (dictOID.dictOID < 3000) + { + ++dict_iter; + continue; + } + + fObjectIDManager.returnOID(dictOID.dictOID); + ++dict_iter; } + } + catch (std::exception& ex) + { + err = ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "returnOIDs:Unknown exception caught"; + throw std::runtime_error(err); + } } -void DDLPackageProcessor::findColumnData(uint32_t sessionID, execplan::CalpontSystemCatalog::TableName& systableName, - const std::string& colName, - DDLColumn& sysCol) +void DDLPackageProcessor::findColumnData(uint32_t sessionID, + execplan::CalpontSystemCatalog::TableName& systableName, + const std::string& colName, DDLColumn& sysCol) { - ColumnList columns; - ColumnList::const_iterator column_iterator; - std::string err; + ColumnList columns; + ColumnList::const_iterator column_iterator; + std::string err; - try + try + { + getColumnsForTable(sessionID, systableName.schema, systableName.table, columns); + column_iterator = columns.begin(); + + while (column_iterator != columns.end()) { - getColumnsForTable(sessionID, systableName.schema, systableName.table, columns); - column_iterator = columns.begin(); + sysCol = *column_iterator; + boost::to_lower(sysCol.tableColName.column); - while (column_iterator != columns.end()) - { - sysCol = *column_iterator; - boost::to_lower(sysCol.tableColName.column); + if (colName == sysCol.tableColName.column) + { + break; + } - if (colName == sysCol.tableColName.column) - { - break; - } - - ++column_iterator; - } - } - catch (std::exception& ex) - { - err = ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "findColumnData:Unknown exception caught"; - throw std::runtime_error(err); + ++column_iterator; } + } + catch (std::exception& ex) + { + err = ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "findColumnData:Unknown exception caught"; + throw std::runtime_error(err); + } } void DDLPackageProcessor::cleanString(string& s) { - string::size_type pos = s.find_first_not_of(" "); + string::size_type pos = s.find_first_not_of(" "); - //stripe off space and ' or '' at beginning and end - if (pos < s.length()) + // stripe off space and ' or '' at beginning and end + if (pos < s.length()) + { + s = s.substr(pos, s.length() - pos); + + if ((pos = s.find_last_of(" ")) < s.length()) { - s = s.substr(pos, s.length() - pos); - - if ((pos = s.find_last_of(" ")) < s.length()) - { - s = s.substr(0, pos); - } - + s = s.substr(0, pos); } + } - if (s[0] == '\'') - { - s = s.substr(1, s.length() - 2); + if (s[0] == '\'') + { + s = s.substr(1, s.length() - 2); - if (s[0] == '\'') - s = s.substr(1, s.length() - 2); - } + if (s[0] == '\'') + s = s.substr(1, s.length() - 2); + } } void DDLPackageProcessor::convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, - unsigned& segment, unsigned filesPerColumnPartition, - unsigned extentsPerSegmentFile, unsigned extentRows, - unsigned startDBRoot, unsigned dbrootCnt) + unsigned& segment, unsigned filesPerColumnPartition, + unsigned extentsPerSegmentFile, unsigned extentRows, + unsigned startDBRoot, unsigned dbrootCnt) { - partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); + partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); - segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % filesPerColumnPartition; + segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % + filesPerColumnPartition; - dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; + dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; - //Calculate the relative rid for this segment file - uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); - idbassert(relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); - uint32_t numExtentsInThisPart = relRidInPartition / extentRows; - unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; - uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; - rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; + // Calculate the relative rid for this segment file + uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * + (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); + idbassert(relRidInPartition <= + (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); + uint32_t numExtentsInThisPart = relRidInPartition / extentRows; + unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; + uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; + rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; } int DDLPackageProcessor::rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID) { - ByteStream bytestream; - bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BLOCKS; + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_SVR_ROLLBACK_BLOCKS; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID.id; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + int rc = 0; + ByteStream::byte tmp8; + std::string errorMsg; + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; + } + else + msgRecived++; + } + } + + if ((msgRecived == fWEClient->getPmCount()) && (rc == 0)) + { + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_ROLLBACK_VERSION; bytestream << uniqueId; bytestream << sessionID; bytestream << (uint32_t)txnID.id; - uint32_t msgRecived = 0; fWEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; bsIn.reset(new ByteStream()); - int rc = 0; - ByteStream::byte tmp8; - std::string errorMsg; + msgRecived = 0; while (1) { - if (msgRecived == fWEClient->getPmCount()) - break; + if (msgRecived == fWEClient->getPmCount()) + break; - fWEClient->read(uniqueId, bsIn); + fWEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error + if (bsIn->length() == 0) // read error + { + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) { - rc = NETWORK_ERROR; - fWEClient->removeQueue(uniqueId); - break; + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; } else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } + msgRecived++; + } } + } - if ((msgRecived == fWEClient->getPmCount()) && (rc == 0)) - { - bytestream.restart(); - bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_VERSION; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t) txnID.id; - fWEClient->write_to_all(bytestream); - bsIn.reset(new ByteStream()); - msgRecived = 0; - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - fWEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - - } - } - } - - return rc; + return rc; } int DDLPackageProcessor::commitTransaction(uint64_t uniqueId, BRM::TxnID txnID) { - int rc = fDbrm->vbCommit(txnID.id); - return rc; + int rc = fDbrm->vbCommit(txnID.id); + return rc; } -} // namespace +} // namespace ddlpackageprocessor // vim:ts=4 sw=4: - diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 84d50d805..d2dd84c79 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -57,610 +57,634 @@ //#define IDB_DDL_DEBUG namespace ddlpackageprocessor { +#define SUMMARY_INFO(message) \ + if (isDebug(SUMMARY)) \ + { \ + std::cerr << message << std::endl; \ + } -#define SUMMARY_INFO( message ) \ - if ( isDebug(SUMMARY) ) \ - { \ - std::cerr << message << std::endl; \ - } +#define DETAIL_INFO(message) \ + if (isDebug(DETAIL)) \ + { \ + std::cerr << message << std::endl; \ + } -#define DETAIL_INFO( message ) \ - if ( isDebug(DETAIL) ) \ - { \ - std::cerr << message << std::endl; \ - } - -#define VERBOSE_INFO( message ) \ - if ( isDebug(VERBOSE) ) \ - { \ - std::cerr << message << std::endl; \ - } +#define VERBOSE_INFO(message) \ + if (isDebug(VERBOSE)) \ + { \ + std::cerr << message << std::endl; \ + } /** @brief base class that defines the common interface and * implementation of a DDLPacakageProcessor */ class DDLPackageProcessor { + public: + /** @brief Result code + */ + enum ResultCode + { + NO_ERROR, + CREATE_ERROR, + ALTER_ERROR, + DROP_ERROR, + TRUNC_ERROR, + TOKENIZATION_ERROR, + NOT_ACCEPTING_PACKAGES, + PK_NOTNULL_ERROR, + WARNING, + USER_ERROR, + NETWORK_ERROR, + PARTITION_WARNING, + WARN_NO_PARTITION, + DROP_TABLE_NOT_IN_CATALOG_ERROR + }; -public: + enum DebugLevel /** @brief Debug level type enumeration */ + { + NONE = 0, /** @brief No debug info */ + SUMMARY = 1, /** @brief Summary level debug info */ + DETAIL = 2, /** @brief A little detail debug info */ + VERBOSE = 3, /** @brief Detailed debug info */ + }; - /** @brief Result code + enum LogFileType + { + DROPTABLE_LOG, + DROPPART_LOG, + TRUNCATE_LOG + }; + typedef std::vector OidList; + typedef std::set PartitionNums; + struct LogInfo + { + LogFileType fileType; + OidList oids; + PartitionNums partitionNums; + }; + typedef std::map TableLogInfo; + + /** @brief the result of dml operations + */ + struct DDLResult + { + /** @brief the result code */ - enum ResultCode { NO_ERROR, CREATE_ERROR, ALTER_ERROR, DROP_ERROR, TRUNC_ERROR, - TOKENIZATION_ERROR, NOT_ACCEPTING_PACKAGES, PK_NOTNULL_ERROR, WARNING, USER_ERROR, NETWORK_ERROR, PARTITION_WARNING, - WARN_NO_PARTITION, DROP_TABLE_NOT_IN_CATALOG_ERROR - }; - - enum DebugLevel /** @brief Debug level type enumeration */ - { - NONE = 0, /** @brief No debug info */ - SUMMARY = 1, /** @brief Summary level debug info */ - DETAIL = 2, /** @brief A little detail debug info */ - VERBOSE = 3, /** @brief Detailed debug info */ - }; - - enum LogFileType { DROPTABLE_LOG, DROPPART_LOG, TRUNCATE_LOG}; - typedef std::vector OidList; - typedef std::set PartitionNums; - struct LogInfo - { - LogFileType fileType; - OidList oids; - PartitionNums partitionNums; - }; - typedef std::map TableLogInfo; - - /** @brief the result of dml operations + ResultCode result; + /** @brief the error message if result != NO_ERROR */ - struct DDLResult + logging::Message message; + }; + + /** @brief a structure to hold ddlcolumn attributes + */ + struct DDLColumn + { + execplan::CalpontSystemCatalog::OID oid; + execplan::CalpontSystemCatalog::ColType colType; + execplan::CalpontSystemCatalog::TableColName tableColName; + }; + + /** @brief a list of DDLColumns + */ + typedef std::vector ColumnList; + + /** @brief a strcuture to hold index object + */ + struct IndexOID + { + int listOID; + int treeOID; + }; + + /** @brief a vector of index object ids + */ + typedef std::vector IndexOIDList; + + /** @brief a structure to hold a dictionary's + * object ids + */ + struct DictOID + { + int dictOID; + int listOID; + int treeOID; + int colWidth; + int compressionType; + }; + /** @brief a structure to hold a date + */ + + struct Date + { + unsigned spare : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; + // NULL column value = 0xFFFFFFFE + EXPORT Date() { - /** @brief the result code - */ - ResultCode result; - /** @brief the error message if result != NO_ERROR - */ - logging::Message message; - }; - - /** @brief a structure to hold ddlcolumn attributes - */ - struct DDLColumn - { - execplan::CalpontSystemCatalog::OID oid; - execplan::CalpontSystemCatalog::ColType colType; - execplan:: CalpontSystemCatalog::TableColName tableColName; - }; - - /** @brief a list of DDLColumns - */ - typedef std::vector ColumnList; - - /** @brief a strcuture to hold index object - */ - struct IndexOID - { - int listOID; - int treeOID; - }; - - /** @brief a vector of index object ids - */ - typedef std::vector IndexOIDList; - - /** @brief a structure to hold a dictionary's - * object ids - */ - struct DictOID - { - int dictOID; - int listOID; - int treeOID; - int colWidth; - int compressionType; - }; - /** @brief a structure to hold a date - */ - - struct Date - { - unsigned spare : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; - // NULL column value = 0xFFFFFFFE - EXPORT Date( ) - { - year = 0xFFFF; - month = 0xF; - day = 0x3F; - spare = 0x3E; - } - }; - /* - struct Date - { - int year : 16; - int month : 4; - int day : 6; - int spare : 6; - Date( ) { year = 0; month = 0; day = 0; spare = 0;} - }; */ - /** @brief a structure to hold a datetime - */ - struct dateTime - { - unsigned msecond : 20; - unsigned second : 6; - unsigned minute : 6; - unsigned hour : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; - // NULL column value = 0xFFFFFFFFFFFFFFFE - EXPORT dateTime( ) - { - year = 0xFFFF; - month = 0xF; - day = 0x3F; - hour = 0x3F; - minute = 0x3F; - second = 0x3F; - msecond = 0xFFFFE; - } - }; - /* - struct dateTime - { - int year : 16; - int month : 4; - int day : 6; - int hour : 6; - int minute : 6; - int second : 6; - int msecond : 20; - dateTime( ) { year = 0; month = 0; day = 0; hour = 0; minute = 0; second = 0; msecond = 0; } - } - ; */ - /** @brief a vector of dictionary object ids - */ - typedef std::vector DictionaryOIDList; - - /** the type of a list of ColumnResult as returned from getSysData - */ - typedef std::vector NJLSysDataVector; - struct NJLSysDataList - { - NJLSysDataVector sysDataVec; - EXPORT NJLSysDataList() {}; - EXPORT ~NJLSysDataList(); - NJLSysDataVector::const_iterator begin() - { - return sysDataVec.begin(); - } - NJLSysDataVector::const_iterator end() - { - return sysDataVec.end(); - } - void push_back(execplan::ColumnResult* cr) - { - sysDataVec.push_back(cr); - } - unsigned int size() - { - return static_cast(sysDataVec.size()); - } - int findColumn(const execplan::CalpontSystemCatalog::OID& columnOID) - { - for (uint32_t i = 0; i < sysDataVec.size(); i++) - { - if (sysDataVec[i]->ColumnOID() == columnOID) - { - return i; - } - } - - return -1; - } - }; - - - /** @brief constructor - */ - DDLPackageProcessor(BRM::DBRM* aDbrm) : fStartingColOID(0), fDDLLoggingId(23), fDebugLevel( NONE ) - { - fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DDLPROC); - fPMCount = fWEClient->getPmCount(); - fDbrm = aDbrm; - //std::cout << "in DDLPackageProcessor constructor " << this << std::endl; + year = 0xFFFF; + month = 0xF; + day = 0x3F; + spare = 0x3E; } - - /** @brief destructor - */ - EXPORT virtual ~DDLPackageProcessor(); - - - /** @brief Is it required to debug - */ - bool isDebug( const DebugLevel level ) const + }; + /* + struct Date + { + int year : 16; + int month : 4; + int day : 6; + int spare : 6; + Date( ) { year = 0; month = 0; day = 0; spare = 0;} + }; */ + /** @brief a structure to hold a datetime + */ + struct dateTime + { + unsigned msecond : 20; + unsigned second : 6; + unsigned minute : 6; + unsigned hour : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; + // NULL column value = 0xFFFFFFFFFFFFFFFE + EXPORT dateTime() { - return level <= fDebugLevel; + year = 0xFFFF; + month = 0xF; + day = 0x3F; + hour = 0x3F; + minute = 0x3F; + second = 0x3F; + msecond = 0xFFFFE; } + }; + /* + struct dateTime + { + int year : 16; + int month : 4; + int day : 6; + int hour : 6; + int minute : 6; + int second : 6; + int msecond : 20; + dateTime( ) { year = 0; month = 0; day = 0; hour = 0; minute = 0; second = 0; msecond = 0; } + } + ; */ + /** @brief a vector of dictionary object ids + */ + typedef std::vector DictionaryOIDList; - /** @brief Get debug level - */ - DebugLevel getDebugLevel() const + /** the type of a list of ColumnResult as returned from getSysData + */ + typedef std::vector NJLSysDataVector; + struct NJLSysDataList + { + NJLSysDataVector sysDataVec; + EXPORT NJLSysDataList(){}; + EXPORT ~NJLSysDataList(); + NJLSysDataVector::const_iterator begin() { - return fDebugLevel; + return sysDataVec.begin(); } - - /** @brief Set debug level - */ - void setDebugLevel( const DebugLevel level ) + NJLSysDataVector::const_iterator end() { - fDebugLevel = level; + return sysDataVec.end(); } - - /** @brief Get index oid that was allocated during index creation - */ - IndexOID getIndexOID() const + void push_back(execplan::ColumnResult* cr) { - return fIdxOID; + sysDataVec.push_back(cr); } - - /** @brief Get starting column oid that was allocated during table - * creation. - */ - int getStartingColumnOID() const + unsigned int size() { - return fStartingColOID; + return static_cast(sysDataVec.size()); } - - /** @brief access and mutator of fPKName */ - const std::string PKName() const + int findColumn(const execplan::CalpontSystemCatalog::OID& columnOID) { - return fPKName; + for (uint32_t i = 0; i < sysDataVec.size(); i++) + { + if (sysDataVec[i]->ColumnOID() == columnOID) + { + return i; + } + } + + return -1; } - void PKName (const std::string PKName) - { - fPKName = PKName; - } - /** @brief Flush primproc cache for associated lbids. - * - * @param oidList the list of OIDs for - * which the lbids for those files will be removed - */ - EXPORT void flushPrimprocCache( std::vector& oidList ); + }; - /** @brief remove the physical files - * - * @param txnID the transaction id - * @param oidList the list of OIDs for - * which the files should be removed - */ - EXPORT void removeFiles(const uint64_t uniqueId, std::vector& oidList); + /** @brief constructor + */ + DDLPackageProcessor(BRM::DBRM* aDbrm) : fStartingColOID(0), fDDLLoggingId(23), fDebugLevel(NONE) + { + fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DDLPROC); + fPMCount = fWEClient->getPmCount(); + fDbrm = aDbrm; + // std::cout << "in DDLPackageProcessor constructor " << this << std::endl; + } - EXPORT void createFiles(execplan::CalpontSystemCatalog::TableName aTableName, const int useDBRoot, const uint64_t uniqueId, const uint32_t numOids); + /** @brief destructor + */ + EXPORT virtual ~DDLPackageProcessor(); - /** @brief remove the physical files for the specified partition - * - * @param oidList the list of OIDs for - * which the files should be removed - * @param partition number - */ - EXPORT void removePartitionFiles(std::vector& oidList, - const PartitionNums& partitions, - uint64_t uniqueId); + /** @brief Is it required to debug + */ + bool isDebug(const DebugLevel level) const + { + return level <= fDebugLevel; + } - /** @brief remove the extents from extent map - * - * @param txnID the transaction id - * @param result the result of the operation - * @param oidList the list of OIDs for - * which the extents should be removed - */ - EXPORT void removeExtents(std::vector& oidList); + /** @brief Get debug level + */ + DebugLevel getDebugLevel() const + { + return fDebugLevel; + } + /** @brief Set debug level + */ + void setDebugLevel(const DebugLevel level) + { + fDebugLevel = level; + } - /** @brief create and open log file to log a table information - * - * @param tableOid the oid of the table - * @param tableName the shcema, table name - */ - EXPORT void createWriteDropLogFile(execplan::CalpontSystemCatalog::OID tableOid, - uint64_t uniqueId, std::vector& oidList); + /** @brief Get index oid that was allocated during index creation + */ + IndexOID getIndexOID() const + { + return fIdxOID; + } - /** @brief create and open log file to log a table partition information - * - * @param tableOid the oid of the table - * @param tableName the shcema, table name - * @param partition the partition number to be dropped - */ - EXPORT void createWritePartitionLogFile(execplan::CalpontSystemCatalog::OID tableOid, - const PartitionNums& partitionNums, - std::vector& oidList, - uint64_t uniqueId); + /** @brief Get starting column oid that was allocated during table + * creation. + */ + int getStartingColumnOID() const + { + return fStartingColOID; + } - // EXPORT void createOpenTruncateTableLogFile(execplan::CalpontSystemCatalog::OID tableOid, execplan::CalpontSystemCatalog::TableName tableName); + /** @brief access and mutator of fPKName */ + const std::string PKName() const + { + return fPKName; + } + void PKName(const std::string PKName) + { + fPKName = PKName; + } + /** @brief Flush primproc cache for associated lbids. + * + * @param oidList the list of OIDs for + * which the lbids for those files will be removed + */ + EXPORT void flushPrimprocCache(std::vector& oidList); - /** @brief create and open log file to log a truncae table information - * - * @param tableOid the oid of the table - * @param tableName the shcema, table name - */ - EXPORT void createWriteTruncateTableLogFile(execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId, std::vector& oidList); + /** @brief remove the physical files + * + * @param txnID the transaction id + * @param oidList the list of OIDs for + * which the files should be removed + */ + EXPORT void removeFiles(const uint64_t uniqueId, std::vector& oidList); + EXPORT void createFiles(execplan::CalpontSystemCatalog::TableName aTableName, const int useDBRoot, + const uint64_t uniqueId, const uint32_t numOids); - /** @brief delete log file - * - */ - EXPORT void deleteLogFile(LogFileType fileType, execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId); + /** @brief remove the physical files for the specified partition + * + * @param oidList the list of OIDs for + * which the files should be removed + * @param partition number + */ + EXPORT void removePartitionFiles(std::vector& oidList, + const PartitionNums& partitions, uint64_t uniqueId); - /** @brief fetch log file infomation - * - */ - EXPORT void fetchLogFile(TableLogInfo& tableLogInfos, uint64_t uniqueId); + /** @brief remove the extents from extent map + * + * @param txnID the transaction id + * @param result the result of the operation + * @param oidList the list of OIDs for + * which the extents should be removed + */ + EXPORT void removeExtents(std::vector& oidList); - BRM::TxnID fTxnid; + /** @brief create and open log file to log a table information + * + * @param tableOid the oid of the table + * @param tableName the shcema, table name + */ + EXPORT void createWriteDropLogFile(execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId, + std::vector& oidList); -protected: - /** @brief get a list of DDLColumns for the given schema.table - * - * @param schema the schema the table belongs to - * @param table the table name - * @param colList will contain the list of columns on return - */ - EXPORT void getColumnsForTable( uint32_t sessionID, std::string schema, std::string table, - ColumnList& colList ); + /** @brief create and open log file to log a table partition information + * + * @param tableOid the oid of the table + * @param tableName the shcema, table name + * @param partition the partition number to be dropped + */ + EXPORT void createWritePartitionLogFile(execplan::CalpontSystemCatalog::OID tableOid, + const PartitionNums& partitionNums, + std::vector& oidList, + uint64_t uniqueId); - /** @brief convert parsed ddl data type to a system catalog data type - * - * @param dateType the parsed ddl data type - */ - execplan::CalpontSystemCatalog::ColDataType convertDataType(int dataType); + // EXPORT void createOpenTruncateTableLogFile(execplan::CalpontSystemCatalog::OID tableOid, + // execplan::CalpontSystemCatalog::TableName tableName); - /** @brief return a tokenized value for the supplied data value - * - * @param result the result of the operation - * @param colType the column type - * @param data the value to tokenize - */ - boost::any tokenizeData(execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - const execplan::CalpontSystemCatalog::ColType& colType, - const boost::any& data ); + /** @brief create and open log file to log a truncae table information + * + * @param tableOid the oid of the table + * @param tableName the shcema, table name + */ + EXPORT void createWriteTruncateTableLogFile(execplan::CalpontSystemCatalog::OID tableOid, uint64_t uniqueId, + std::vector& oidList); - /** @brief does the supplied constraint type require an index - * - * @param type the constraint type - */ - bool isIndexConstraint(ddlpackage::DDL_CONSTRAINTS type); + /** @brief delete log file + * + */ + EXPORT void deleteLogFile(LogFileType fileType, execplan::CalpontSystemCatalog::OID tableOid, + uint64_t uniqueId); - /** @brief get the char code for the constraint type - * - * @param type the constraint type - */ - char getConstraintCode(ddlpackage::DDL_CONSTRAINTS type); + /** @brief fetch log file infomation + * + */ + EXPORT void fetchLogFile(TableLogInfo& tableLogInfos, uint64_t uniqueId); - /** @brief get the column refrences for the given table constraint - * - * @param tableConstraint the table constraint - * @param columns on return will contain the list of columns - */ - void getColumnReferences(ddlpackage::TableConstraintDef& tableConstraint, - ddlpackage::ColumnNameList& columns); + BRM::TxnID fTxnid; - /** @brief build a table constraint name - * - * @param schema the schema the table belongs to - * @param table the name of the table - * @param constraintNumber the constraint number - * @param type the constraint type - */ - std::string buildTableConstraintName(int oid, - ddlpackage::DDL_CONSTRAINTS type); + protected: + /** @brief get a list of DDLColumns for the given schema.table + * + * @param schema the schema the table belongs to + * @param table the table name + * @param colList will contain the list of columns on return + */ + EXPORT void getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, + ColumnList& colList); - /** @brief build a column constraint name - * - * @param schema the schema the table belongs to - * @param table the name of the table - * @param column the column name - * @param type the constraint type - */ - std::string buildColumnConstraintName(const std::string& schema, - const std::string& table, - const std::string& column, - ddlpackage::DDL_CONSTRAINTS type); + /** @brief convert parsed ddl data type to a system catalog data type + * + * @param dateType the parsed ddl data type + */ + execplan::CalpontSystemCatalog::ColDataType convertDataType(int dataType); + /** @brief return a tokenized value for the supplied data value + * + * @param result the result of the operation + * @param colType the column type + * @param data the value to tokenize + */ + boost::any tokenizeData(execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, + const execplan::CalpontSystemCatalog::ColType& colType, const boost::any& data); - /** @brief write the tables meta data to the SYSTABLE table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableDef the table definition - */ - //void writeSysTableMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - // ddlpackage::TableDef& tableDef, uint32_t tableWithAutoi=0); + /** @brief does the supplied constraint type require an index + * + * @param type the constraint type + */ + bool isIndexConstraint(ddlpackage::DDL_CONSTRAINTS type); - /** @brief write the table columns meta data to the SYSCOLUMN table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableDefCols the table columns definition - * @param qualifiedName the name of catalog, schema, object names - */ - //void writeSysColumnMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - // ddlpackage::ColumnDefList& tableDefCols, - // ddlpackage::QualifiedName& qualifiedName, int colpos, bool alterFlag=false ); + /** @brief get the char code for the constraint type + * + * @param type the constraint type + */ + char getConstraintCode(ddlpackage::DDL_CONSTRAINTS type); - /** @brief write the table constraint meta data to the SYSCONSTRAINT table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param constraintList the table constrain list - * @param qualifiedName the name of catalog, schema, object names - */ -// void writeTableSysConstraintMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, -// ddlpackage::TableConstraintDefList& constraintList, ddlpackage::QualifiedName& -// qualifiedName, bool alterFlag=false ); - /** @brief write the table constraint meta data to the SYSCONSTRAINTCOL table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param constraintList the table constrain list - * @param qualifiedName the name of catalog, schema, object names - */ -// void writeTableSysConstraintColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, -// const DDLResult& result, ddlpackage::TableConstraintDefList& constraintList, -// ddlpackage::QualifiedName& qualifiedName, bool alterFlag=false ); + /** @brief get the column refrences for the given table constraint + * + * @param tableConstraint the table constraint + * @param columns on return will contain the list of columns + */ + void getColumnReferences(ddlpackage::TableConstraintDef& tableConstraint, + ddlpackage::ColumnNameList& columns); - /** @brief write the column constraint meta data to the SYSCONTRAINT table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param constraintList the table constrain list - * @param qualifiedName the name of catalog, schema, object names - */ -// void writeColumnSysConstraintMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, -// ddlpackage::ColumnDefList& tableDefCols, -// ddlpackage::QualifiedName& qualifiedName ); + /** @brief build a table constraint name + * + * @param schema the schema the table belongs to + * @param table the name of the table + * @param constraintNumber the constraint number + * @param type the constraint type + */ + std::string buildTableConstraintName(int oid, ddlpackage::DDL_CONSTRAINTS type); - /** @brief write the column constraint meta data to the SYSCONTRAINTCOL table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableDef the table definition - */ -// void writeColumnSysConstraintColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, -// const DDLResult& result, ddlpackage::ColumnDefList& tableDefCols, -// ddlpackage::QualifiedName& qualifiedName); + /** @brief build a column constraint name + * + * @param schema the schema the table belongs to + * @param table the name of the table + * @param column the column name + * @param type the constraint type + */ + std::string buildColumnConstraintName(const std::string& schema, const std::string& table, + const std::string& column, ddlpackage::DDL_CONSTRAINTS type); + /** @brief write the tables meta data to the SYSTABLE table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableDef the table definition + */ + // void writeSysTableMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const + // DDLResult& result, + // ddlpackage::TableDef& tableDef, uint32_t tableWithAutoi=0); - /** @brief write the index meta data to the SYSINDEX table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableDef the table definiton - * @param consDef the table constraint - * @param indexName name of the index - */ - // void writeSysIndexMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - // ddlpackage::QualifiedName& qualifiedName, - // ddlpackage::DDL_CONSTRAINTS type, - // std::string& indexName, bool multicol, bool alterFlag=false); + /** @brief write the table columns meta data to the SYSCOLUMN table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableDefCols the table columns definition + * @param qualifiedName the name of catalog, schema, object names + */ + // void writeSysColumnMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const + // DDLResult& result, + // ddlpackage::ColumnDefList& tableDefCols, + // ddlpackage::QualifiedName& qualifiedName, int colpos, bool alterFlag=false ); - /** @brief write the index meta data to the SYSINDEXCOL table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableDef the table definiton - * @param constraintCols the list of columns in this index - * @param indexName name of the index - */ - // void writeSysIndexColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, -//ddlpackage::QualifiedName& qualifiedName, - // ddlpackage::ColumnNameList& constraintCols, - // std::string& indexName, bool alterFlag=false); + /** @brief write the table constraint meta data to the SYSCONSTRAINT table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param constraintList the table constrain list + * @param qualifiedName the name of catalog, schema, object names + */ + // void writeTableSysConstraintMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // DDLResult& result, + // ddlpackage::TableConstraintDefList& constraintList, + // ddlpackage::QualifiedName& + // qualifiedName, bool alterFlag=false ); + /** @brief write the table constraint meta data to the SYSCONSTRAINTCOL table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param constraintList the table constrain list + * @param qualifiedName the name of catalog, schema, object names + */ + // void writeTableSysConstraintColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // const DDLResult& result, ddlpackage::TableConstraintDefList& + // constraintList, ddlpackage::QualifiedName& qualifiedName, bool + // alterFlag=false ); - /** @brief remove all indexes for the supplied table from the SYSINDEX table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableName the qualified name of the table - */ - //void removeSysIndexMetaDataForTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, - // DDLResult& result, ddlpackage::QualifiedName& tableName); + /** @brief write the column constraint meta data to the SYSCONTRAINT table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param constraintList the table constrain list + * @param qualifiedName the name of catalog, schema, object names + */ + // void writeColumnSysConstraintMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // const DDLResult& result, + // ddlpackage::ColumnDefList& tableDefCols, + // ddlpackage::QualifiedName& qualifiedName ); - /** @brief remove all index columns for the supplied table from the SYSINDEXCOL table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableName the qualified name of the table - */ - // void removeSysIndexColMetaDataForTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, - // DDLResult& result, ddlpackage::QualifiedName& tableName); + /** @brief write the column constraint meta data to the SYSCONTRAINTCOL table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableDef the table definition + */ + // void writeColumnSysConstraintColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN + // txnID, + // const DDLResult& result, ddlpackage::ColumnDefList& tableDefCols, + // ddlpackage::QualifiedName& qualifiedName); - /** @brief remove an index from the SYSINDEX table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param indexName the qualified name of the index - */ - // void removeSysIndexMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& indexName); + /** @brief write the index meta data to the SYSINDEX table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableDef the table definiton + * @param consDef the table constraint + * @param indexName name of the index + */ + // void writeSysIndexMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const + // DDLResult& result, + // ddlpackage::QualifiedName& qualifiedName, + // ddlpackage::DDL_CONSTRAINTS type, + // std::string& indexName, bool multicol, bool alterFlag=false); - /** @brief remove index columns from the SYSINDEXCOL table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param indexName the qualified name of the index - */ - // void removeSysIndexColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& indexName); + /** @brief write the index meta data to the SYSINDEXCOL table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableDef the table definiton + * @param constraintCols the list of columns in this index + * @param indexName name of the index + */ + // void writeSysIndexColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const + // DDLResult& result, + // ddlpackage::QualifiedName& qualifiedName, + // ddlpackage::ColumnNameList& constraintCols, + // std::string& indexName, bool alterFlag=false); - /** @brief remove the table meta data from the SYSTABLE table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableName the qualified name of the table to remove - */ - // void removeSysTableMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& tableName); + /** @brief remove all indexes for the supplied table from the SYSINDEX table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableName the qualified name of the table + */ + // void removeSysIndexMetaDataForTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // DDLResult& result, ddlpackage::QualifiedName& tableName); - /** @brief remove the column meta data from the SYSCOLUMN table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableName the qualified name of the table whose columns - * are to be removed - */ - // void removeSysColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& tableName); + /** @brief remove all index columns for the supplied table from the SYSINDEXCOL table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableName the qualified name of the table + */ + // void removeSysIndexColMetaDataForTable(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // DDLResult& result, ddlpackage::QualifiedName& tableName); - /** @brief remove the column meta data from the SYSCOLUMN table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param columnInfo the qualified name of the column - * to be removed - */ - // void removeColSysColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& columnInfo); + /** @brief remove an index from the SYSINDEX table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param indexName the qualified name of the index + */ + // void removeSysIndexMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& + // result, + // ddlpackage::QualifiedName& indexName); - /** @brief remove the constraint meta data from the SYSCONSTRAINT table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableName the qualified name of the table whose constraints - * are to be removed - */ -// void removeSysContraintMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& tableName); + /** @brief remove index columns from the SYSINDEXCOL table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param indexName the qualified name of the index + */ + // void removeSysIndexColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& + // result, + // ddlpackage::QualifiedName& indexName); - /** @brief remove the constraint meta data from the SYSCONSTRAINT table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param indexName the index name to be removed - */ -// void removeSysIndexMetaDataForIndex(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, -// DDLResult& result, -// execplan::CalpontSystemCatalog::IndexNameList& indexNameList); + /** @brief remove the table meta data from the SYSTABLE table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableName the qualified name of the table to remove + */ + // void removeSysTableMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& + // result, + // ddlpackage::QualifiedName& tableName); - /** @brief remove the column constraint meta data from the SYSCONSTRAINTCOL table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param tableName the qualified name of the table whose column constraints - * are to be removed - */ - // void removeSysConstraintColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, - // ddlpackage::QualifiedName& tableName); - /** @brief remove the column constraint meta data from the SYSCONSTRAINT table - * - * @param txnID the transaction id - * @param result the result of the operation - * @param constrintNames the names of the constraints - * are to be removed - */ + /** @brief remove the column meta data from the SYSCOLUMN table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableName the qualified name of the table whose columns + * are to be removed + */ + // void removeSysColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& + // result, + // ddlpackage::QualifiedName& tableName); + + /** @brief remove the column meta data from the SYSCOLUMN table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param columnInfo the qualified name of the column + * to be removed + */ + // void removeColSysColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& + // result, + // ddlpackage::QualifiedName& columnInfo); + + /** @brief remove the constraint meta data from the SYSCONSTRAINT table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableName the qualified name of the table whose constraints + * are to be removed + */ + // void removeSysContraintMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // DDLResult& result, + // ddlpackage::QualifiedName& tableName); + + /** @brief remove the constraint meta data from the SYSCONSTRAINT table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param indexName the index name to be removed + */ + // void removeSysIndexMetaDataForIndex(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // DDLResult& result, + // execplan::CalpontSystemCatalog::IndexNameList& indexNameList); + + /** @brief remove the column constraint meta data from the SYSCONSTRAINTCOL table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param tableName the qualified name of the table whose column constraints + * are to be removed + */ + // void removeSysConstraintColMetaData(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + // DDLResult& result, + // ddlpackage::QualifiedName& tableName); + /** @brief remove the column constraint meta data from the SYSCONSTRAINT table + * + * @param txnID the transaction id + * @param result the result of the operation + * @param constrintNames the names of the constraints + * are to be removed + */ #if 0 void removeSysContraintMetaDataForConstraint(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, DDLResult& result, @@ -714,171 +738,158 @@ protected: execplan::CalpontSystemCatalog::IndexOIDList& idxOIDList); #endif - /** @brief return the OIDs used by the database objects - * - * @param ridList the list of column OIDs to return - * @param dictOIDList the list of dictionary OIDs to return - */ - void returnOIDs(execplan::CalpontSystemCatalog::RIDList& ridList, - execplan::CalpontSystemCatalog::DictOIDList& dictOIDList); + /** @brief return the OIDs used by the database objects + * + * @param ridList the list of column OIDs to return + * @param dictOIDList the list of dictionary OIDs to return + */ + void returnOIDs(execplan::CalpontSystemCatalog::RIDList& ridList, + execplan::CalpontSystemCatalog::DictOIDList& dictOIDList); - /** - * @brief convert a columns data, represnted as a string, to it's native - * data type - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - //boost::any convertColumnData( execplan::CalpontSystemCatalog::ColType colType, - // const std::string& data ); + /** + * @brief convert a columns data, represnted as a string, to it's native + * data type + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + // boost::any convertColumnData( execplan::CalpontSystemCatalog::ColType colType, + // const std::string& data ); - /** - * @brief Find the given column in the given system catalog table and - * return it as a DDLColumn object - * - * @param systableName the table to find the column in - * @param colName the name of the column to find in the table - * @param sysCol on success the returned sysCol object - */ - void findColumnData(uint32_t sessionID, execplan::CalpontSystemCatalog::TableName& systableName, - const std::string& colName, DDLColumn& sysCol ); + /** + * @brief Find the given column in the given system catalog table and + * return it as a DDLColumn object + * + * @param systableName the table to find the column in + * @param colName the name of the column to find in the table + * @param sysCol on success the returned sysCol object + */ + void findColumnData(uint32_t sessionID, execplan::CalpontSystemCatalog::TableName& systableName, + const std::string& colName, DDLColumn& sysCol); - /** @brief remove the supplied row from the supplied system catalog table - * - * @param result the result of the operation - * @param sysCatalogTableName the qualified name of the system catalog table - * @param rid the id of the row to remove - */ - void removeRowFromSysCatalog(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - ddlpackage::QualifiedName& sysCatalogTableName, WriteEngine::RID& rid); + /** @brief remove the supplied row from the supplied system catalog table + * + * @param result the result of the operation + * @param sysCatalogTableName the qualified name of the system catalog table + * @param rid the id of the row to remove + */ + void removeRowFromSysCatalog(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + const DDLResult& result, ddlpackage::QualifiedName& sysCatalogTableName, + WriteEngine::RID& rid); - /** @brief validate reference constraint for altering existing table - * - * @param sessionID session ID - * @param result the result of the operation - * @param tcn the column which has the foreign key constraint - * @param refIndexName the index name of the referenced primary key constraint - * @return true if violation - */ - bool referenceConstraintViolation(uint32_t sessionID, - DDLResult& result, - execplan::CalpontSystemCatalog::TableColName tcn, - execplan::CalpontSystemCatalog::IndexName refIndexName); + /** @brief validate reference constraint for altering existing table + * + * @param sessionID session ID + * @param result the result of the operation + * @param tcn the column which has the foreign key constraint + * @param refIndexName the index name of the referenced primary key constraint + * @return true if violation + */ + bool referenceConstraintViolation(uint32_t sessionID, DDLResult& result, + execplan::CalpontSystemCatalog::TableColName tcn, + execplan::CalpontSystemCatalog::IndexName refIndexName); - /** @brief validate PK constraint (not null part) for altering existing table - * - * @param sessionID session ID - * @param result the result of the operation - * @param qualifiedName schema.table name - * @param constraintCols the columns associated with the primary key - * @return true if violation - */ - bool PKConstraintViolation(uint32_t sessionID, - DDLResult& result, - ddlpackage::QualifiedName& qualifiedName, - ddlpackage::ColumnNameList& constraintCols); - /** @brief validate check constraint for altering existing table - * - * @param sessionID session ID - * @param result the result of the operation - * @param qualifiedName schema.table name - * @param checkConstraint the constraint text string - * @return true if violation - */ - bool checkConstraintViolation(uint32_t sessionID, - DDLResult& result, - ddlpackage::QualifiedName& qualifiedName, - std::string& checkConstraint); + /** @brief validate PK constraint (not null part) for altering existing table + * + * @param sessionID session ID + * @param result the result of the operation + * @param qualifiedName schema.table name + * @param constraintCols the columns associated with the primary key + * @return true if violation + */ + bool PKConstraintViolation(uint32_t sessionID, DDLResult& result, ddlpackage::QualifiedName& qualifiedName, + ddlpackage::ColumnNameList& constraintCols); + /** @brief validate check constraint for altering existing table + * + * @param sessionID session ID + * @param result the result of the operation + * @param qualifiedName schema.table name + * @param checkConstraint the constraint text string + * @return true if violation + */ + bool checkConstraintViolation(uint32_t sessionID, DDLResult& result, + ddlpackage::QualifiedName& qualifiedName, std::string& checkConstraint); + /** @brief remove the supplied rows from the supplied system catalog table + * + * @param result the result of the operation + * @param sysCatalogTableName the qualified name of the system catalog table + * @param colRidList the list of row ids to remove + */ + void removeRowsFromSysCatalog(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, + const DDLResult& result, ddlpackage::QualifiedName& sysCatalogTableName, + execplan::CalpontSystemCatalog::RIDList& colRidList); - /** @brief remove the supplied rows from the supplied system catalog table - * - * @param result the result of the operation - * @param sysCatalogTableName the qualified name of the system catalog table - * @param colRidList the list of row ids to remove - */ - void removeRowsFromSysCatalog(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const DDLResult& result, - ddlpackage::QualifiedName& sysCatalogTableName, - execplan::CalpontSystemCatalog::RIDList& colRidList); + WriteEngine::WriteEngineWrapper fWriteEngine; + BRM::DBRM* fDbrm; - WriteEngine::WriteEngineWrapper fWriteEngine; + execplan::SessionManager fSessionManager; + uint32_t fPMCount; + WriteEngine::WEClients* fWEClient; - BRM::DBRM* fDbrm; + DictionaryOIDList fDictionaryOIDList; - execplan::SessionManager fSessionManager; - uint32_t fPMCount; - WriteEngine::WEClients* fWEClient; + // store oids used during table and index creation + // for external reference + int fTableOID; + int fStartingColOID; + int fColumnNum; + IndexOID fIdxOID; + std::ofstream fDDLLogFile; + std::string fDDLLogFileName; + std::string fPKName; // primary key name supplied by Oracle. Oracle will issue + // two separate DDL statements for a create table with primary + // key DDL, with the 1st one being create index and the 2nd + // one being create table. This PK name will be stored here + // when creatindexprocessor gets the create index statement. This + // is to make sure Calpont use the same system primary key name as Oracle + unsigned const fDDLLoggingId; - DictionaryOIDList fDictionaryOIDList; + // std::ofstream fDDLLogFile; + // std::string fDDLLogFileName; - // store oids used during table and index creation - // for external reference - int fTableOID; - int fStartingColOID; - int fColumnNum; - IndexOID fIdxOID; - std::ofstream fDDLLogFile; - std::string fDDLLogFileName; - - std::string fPKName; // primary key name supplied by Oracle. Oracle will issue - // two separate DDL statements for a create table with primary - // key DDL, with the 1st one being create index and the 2nd - // one being create table. This PK name will be stored here - // when creatindexprocessor gets the create index statement. This - // is to make sure Calpont use the same system primary key name as Oracle - unsigned const fDDLLoggingId; - - //std::ofstream fDDLLogFile; - //std::string fDDLLogFileName; - - /** @brief convert absolute rid to relative rid in a segement file - * - * @param rid in:the absolute rid out: relative rid in a segement file - * @param dbRoot,partition, segment the extent information obtained from rid - * @param filesPerColumnPartition,extentRows, extentsPerSegmentFile the extent map parameters - * @param startDBRoot the dbroot this table starts - * @param dbrootCnt the number of dbroot in db - */ - void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, - unsigned& segment, unsigned filesPerColumnPartition, - unsigned extentsPerSegmentFile, unsigned extentRows, - unsigned startDBRoot, unsigned dbrootCnt); - - int rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID); - int commitTransaction(uint64_t uniqueId, BRM::TxnID txnID); - - // MCOL-66 The DBRM can't handle concurrent DDL - static boost::mutex dbrmMutex; -private: - /** @brief clean beginning and ending glitches and spaces from string - * - * @param s string to be cleaned - */ - void cleanString(std::string& s); - //std::string fDDLLogFileName; - DebugLevel fDebugLevel; // internal use debug level + /** @brief convert absolute rid to relative rid in a segement file + * + * @param rid in:the absolute rid out: relative rid in a segement file + * @param dbRoot,partition, segment the extent information obtained from rid + * @param filesPerColumnPartition,extentRows, extentsPerSegmentFile the extent map parameters + * @param startDBRoot the dbroot this table starts + * @param dbrootCnt the number of dbroot in db + */ + void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, unsigned& segment, + unsigned filesPerColumnPartition, unsigned extentsPerSegmentFile, + unsigned extentRows, unsigned startDBRoot, unsigned dbrootCnt); + int rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID); + int commitTransaction(uint64_t uniqueId, BRM::TxnID txnID); + // MCOL-66 The DBRM can't handle concurrent DDL + static boost::mutex dbrmMutex; + private: + /** @brief clean beginning and ending glitches and spaces from string + * + * @param s string to be cleaned + */ + void cleanString(std::string& s); + // std::string fDDLLogFileName; + DebugLevel fDebugLevel; // internal use debug level }; /** @brief helper template function to do safe from string to type conversions - * - */ + * + */ template -bool from_string(T& t, - const std::string& s, - std::ios_base & (*f)(std::ios_base&)) +bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&)) { - std::istringstream iss(s); - return !(iss >> f >> t).fail(); + std::istringstream iss(s); + return !(iss >> f >> t).fail(); } -} +} // namespace ddlpackageprocessor #undef EXPORT -#endif //DDLPACKAGEPROCESSOR_H +#endif // DDLPACKAGEPROCESSOR_H // vim:ts=4 sw=4: - diff --git a/dbcon/ddlpackageproc/ddlpackageprocessorfactory.cpp b/dbcon/ddlpackageproc/ddlpackageprocessorfactory.cpp index 279e93ce2..d5928ae90 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessorfactory.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessorfactory.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ddlpackageprocessorfactory.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ddlpackageprocessorfactory.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "ddlpackageprocessorfactory.h" #include "ddlpackage.h" #include "alterpackageprocessor.h" @@ -30,29 +30,21 @@ using namespace ddlpackage; namespace ddlpackageprocessor { - -DDLPackageProcessor* DDLPackageProcessorFactory:: -makePackageProcessor(int packageType, ddlpackage::CalpontDDLPackage& cpackage) +DDLPackageProcessor* DDLPackageProcessorFactory::makePackageProcessor(int packageType, + ddlpackage::CalpontDDLPackage& cpackage) { - DDLPackageProcessor* ddlProcPtr = 0; + DDLPackageProcessor* ddlProcPtr = 0; - switch ( packageType ) - { - case DDL_CREATE: - ddlProcPtr = new CreatePackageProcessor(); - break; + switch (packageType) + { + case DDL_CREATE: ddlProcPtr = new CreatePackageProcessor(); break; - case DDL_ALTER: - ddlProcPtr = new AlterPackageProcessor(); - break; + case DDL_ALTER: ddlProcPtr = new AlterPackageProcessor(); break; - case DDL_DROP: - ddlProcPtr = new DropPackageProcessor(); - break; + case DDL_DROP: ddlProcPtr = new DropPackageProcessor(); break; + } - } - - return ddlProcPtr; + return ddlProcPtr; } -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/ddlpackageprocessorfactory.h b/dbcon/ddlpackageproc/ddlpackageprocessorfactory.h index dfdc995d8..1641cc4d0 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessorfactory.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessorfactory.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ddlpackageprocessorfactory.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ddlpackageprocessorfactory.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DDLPACKAGEPROCESSORFACTORY_H #define DDLPACKAGEPROCESSORFACTORY_H @@ -27,31 +27,24 @@ #include "ddlpkg.h" #include "ddlpackageprocessor.h" - namespace ddlpackageprocessor { - /** @brief create a ddlPackageProcessor object from a CalpontddlPackage object * */ class DDLPackageProcessorFactory { + public: + /** @brief static ddlPackageProcessor constructor method + * + * @param packageType the ddl Package type + * @param cpackage the CalpontddlPackage from which the ddlPackageProcessor is constructed + */ + static DDLPackageProcessor* makePackageProcessor(int packageType, ddlpackage::CalpontDDLPackage& cpackage); -public: - - /** @brief static ddlPackageProcessor constructor method - * - * @param packageType the ddl Package type - * @param cpackage the CalpontddlPackage from which the ddlPackageProcessor is constructed - */ - static DDLPackageProcessor* - makePackageProcessor( int packageType, ddlpackage::CalpontDDLPackage& cpackage ); - -protected: - -private: + protected: + private: }; -} -#endif //DDLPACKAGEPROCESSORFACTORY_H - +} // namespace ddlpackageprocessor +#endif // DDLPACKAGEPROCESSORFACTORY_H diff --git a/dbcon/ddlpackageproc/dropindexprocessor.cpp b/dbcon/ddlpackageproc/dropindexprocessor.cpp index 0a00f05ec..e1b5720c8 100644 --- a/dbcon/ddlpackageproc/dropindexprocessor.cpp +++ b/dbcon/ddlpackageproc/dropindexprocessor.cpp @@ -30,85 +30,85 @@ using namespace logging; namespace ddlpackageprocessor { -DropIndexProcessor::DDLResult DropIndexProcessor::processPackage(ddlpackage::DropIndexStatement& dropIndexStmt) +DropIndexProcessor::DDLResult DropIndexProcessor::processPackage( + ddlpackage::DropIndexStatement& dropIndexStmt) { - SUMMARY_INFO("DropIndexProcessor::processPackage"); + SUMMARY_INFO("DropIndexProcessor::processPackage"); - boost::shared_ptr sysCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog( dropIndexStmt.fSessionID ); - CalpontSystemCatalog::IndexName indexName; - CalpontSystemCatalog::IndexOID indexOID; + boost::shared_ptr sysCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(dropIndexStmt.fSessionID); + CalpontSystemCatalog::IndexName indexName; + CalpontSystemCatalog::IndexOID indexOID; - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; - DDLResult result; - result.result = NO_ERROR; + DDLResult result; + result.result = NO_ERROR; - int err = 0; + int err = 0; - VERBOSE_INFO(dropIndexStmt); + VERBOSE_INFO(dropIndexStmt); - SQLLogger logger(dropIndexStmt.fSql, fDDLLoggingId, dropIndexStmt.fSessionID, txnID.id); + SQLLogger logger(dropIndexStmt.fSql, fDDLLoggingId, dropIndexStmt.fSessionID, txnID.id); - indexName.schema = dropIndexStmt.fIndexName->fSchema; - indexName.index = dropIndexStmt.fIndexName->fName; - //Look up table name from indexname. Oracle will error out if same constraintname or indexname exists. - CalpontSystemCatalog::TableName tableName = sysCatalogPtr->lookupTableForIndex (dropIndexStmt.fIndexName->fName, dropIndexStmt.fIndexName->fSchema ); - indexName.table = tableName.table; - indexOID = sysCatalogPtr->lookupIndexNbr(indexName); + indexName.schema = dropIndexStmt.fIndexName->fSchema; + indexName.index = dropIndexStmt.fIndexName->fName; + // Look up table name from indexname. Oracle will error out if same constraintname or indexname exists. + CalpontSystemCatalog::TableName tableName = + sysCatalogPtr->lookupTableForIndex(dropIndexStmt.fIndexName->fName, dropIndexStmt.fIndexName->fSchema); + indexName.table = tableName.table; + indexOID = sysCatalogPtr->lookupIndexNbr(indexName); - VERBOSE_INFO("Removing the SYSINDEX meta data"); - removeSysIndexMetaData(dropIndexStmt.fSessionID, txnID.id, result, *dropIndexStmt.fIndexName); + VERBOSE_INFO("Removing the SYSINDEX meta data"); + removeSysIndexMetaData(dropIndexStmt.fSessionID, txnID.id, result, *dropIndexStmt.fIndexName); - if (result.result != NO_ERROR) - { - DETAIL_INFO("writeSysIndexMetaData failed"); - goto rollback; - } + if (result.result != NO_ERROR) + { + DETAIL_INFO("writeSysIndexMetaData failed"); + goto rollback; + } - VERBOSE_INFO("Removing the SYSINDEXCOL meta data"); - removeSysIndexColMetaData(dropIndexStmt.fSessionID, txnID.id, result, *dropIndexStmt.fIndexName); + VERBOSE_INFO("Removing the SYSINDEXCOL meta data"); + removeSysIndexColMetaData(dropIndexStmt.fSessionID, txnID.id, result, *dropIndexStmt.fIndexName); - if (result.result != NO_ERROR) - { - DETAIL_INFO("writeSysIndexMetaData failed"); - goto rollback; - } + if (result.result != NO_ERROR) + { + DETAIL_INFO("writeSysIndexMetaData failed"); + goto rollback; + } + VERBOSE_INFO("Removing the index files"); + err = fWriteEngine.dropIndex(txnID.id, indexOID.objnum, indexOID.listOID); - VERBOSE_INFO("Removing the index files"); - err = fWriteEngine.dropIndex(txnID.id, indexOID.objnum, indexOID.listOID); + if (err) + { + DETAIL_INFO("WriteEngine dropIndex failed"); + goto rollback; + } - if (err) - { - DETAIL_INFO("WriteEngine dropIndex failed"); - goto rollback; - } + // Log the DDL statement + logging::logDDL(dropIndexStmt.fSessionID, txnID.id, dropIndexStmt.fSql, dropIndexStmt.fOwner); - // Log the DDL statement - logging::logDDL(dropIndexStmt.fSessionID, txnID.id, dropIndexStmt.fSql, dropIndexStmt.fOwner); + // register the changes + err = fWriteEngine.commit(txnID.id); - // register the changes - err = fWriteEngine.commit( txnID.id ); + if (err) + { + DETAIL_INFO("Failed to commit the drop index transaction"); + goto rollback; + } - if (err) - { - DETAIL_INFO("Failed to commit the drop index transaction"); - goto rollback; - } - - fSessionManager.committed(txnID); - //fObjectIDManager.returnOID(indexOID.objnum); - //fObjectIDManager.returnOID(indexOID.listOID); - return result; + fSessionManager.committed(txnID); + // fObjectIDManager.returnOID(indexOID.objnum); + // fObjectIDManager.returnOID(indexOID.listOID); + return result; rollback: - fWriteEngine.rollbackTran(txnID.id, dropIndexStmt.fSessionID); - fSessionManager.rolledback(txnID); - return result; + fWriteEngine.rollbackTran(txnID.id, dropIndexStmt.fSessionID); + fSessionManager.rolledback(txnID); + return result; } - - -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/dropindexprocessor.h b/dbcon/ddlpackageproc/dropindexprocessor.h index 34a17de21..62f801837 100644 --- a/dbcon/ddlpackageproc/dropindexprocessor.h +++ b/dbcon/ddlpackageproc/dropindexprocessor.h @@ -34,18 +34,16 @@ namespace ddlpackageprocessor */ class DropIndexProcessor : public DDLPackageProcessor { -public: - /** @brief process a drop index statement - * - * @param dropIndexStmt the drop index statement - */ - DDLResult processPackage(ddlpackage::DropIndexStatement& dropIndexStmt); - -protected: - -private: + public: + /** @brief process a drop index statement + * + * @param dropIndexStmt the drop index statement + */ + DDLResult processPackage(ddlpackage::DropIndexStatement& dropIndexStmt); + protected: + private: }; -} //namespace ddlpackageprocessor -#endif //DROPINDEXPROCESSOR_H +} // namespace ddlpackageprocessor +#endif // DROPINDEXPROCESSOR_H diff --git a/dbcon/ddlpackageproc/droppartitionprocessor.cpp b/dbcon/ddlpackageproc/droppartitionprocessor.cpp index 25f475df9..83894f83c 100644 --- a/dbcon/ddlpackageproc/droppartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/droppartitionprocessor.cpp @@ -37,344 +37,344 @@ using namespace oam; namespace ddlpackageprocessor { - -DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackage(ddlpackage::DropPartitionStatement& dropPartitionStmt) +DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackage( + ddlpackage::DropPartitionStatement& dropPartitionStmt) { - SUMMARY_INFO("DropPartitionProcessor::processPackage"); + SUMMARY_INFO("DropPartitionProcessor::processPackage"); - DDLResult result; - result.result = NO_ERROR; - std::string err; - VERBOSE_INFO(dropPartitionStmt); + DDLResult result; + result.result = NO_ERROR; + std::string err; + VERBOSE_INFO(dropPartitionStmt); - // Commit current transaction. - // all DDL statements cause an implicit commit - VERBOSE_INFO("Getting current txnID"); + // Commit current transaction. + // all DDL statements cause an implicit commit + VERBOSE_INFO("Getting current txnID"); - int rc = 0; - rc = fDbrm->isReadWrite(); - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; + int rc = 0; + rc = fDbrm->isReadWrite(); + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; - if (rc != 0 ) + if (rc != 0) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + std::vector oidList; + CalpontSystemCatalog::RIDList tableColRidList; + CalpontSystemCatalog::DictOIDList dictOIDList; + execplan::CalpontSystemCatalog::ROPair roPair; + uint32_t processID = 0; + uint64_t uniqueID = 0; + uint32_t sessionID = dropPartitionStmt.fSessionID; + std::string processName("DDLProc"); + uint64_t uniqueId = 0; + + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(9); + args.add(ex.what()); + message.format(args); + result.result = ALTER_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + catch (...) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = ALTER_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + string stmt = dropPartitionStmt.fSql + "|" + dropPartitionStmt.fTableName->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, sessionID, txnID.id); + + try + { + // check table lock + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(dropPartitionStmt.fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(dropPartitionStmt.fSessionID); + CalpontSystemCatalog::TableName tableName; + tableName.schema = dropPartitionStmt.fTableName->fSchema; + tableName.table = dropPartitionStmt.fTableName->fName; + roPair = systemCatalogPtr->tableRID(tableName); + + //@Bug 3054 check for system catalog + if (roPair.objnum < 3000) { - logging::Message::Args args; - logging::Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; + throw std::runtime_error("Drop partition cannot be operated on Calpont system catalog."); } + int i = 0; + processID = ::getpid(); + oam::OamCache* oamcache = OamCache::makeOamCache(); + std::vector pmList = oamcache->getModuleIds(); + std::vector pms; - std::vector oidList; - CalpontSystemCatalog::RIDList tableColRidList; - CalpontSystemCatalog::DictOIDList dictOIDList; - execplan::CalpontSystemCatalog::ROPair roPair; - uint32_t processID = 0; - uint64_t uniqueID = 0; - uint32_t sessionID = dropPartitionStmt.fSessionID; - std::string processName("DDLProc"); - uint64_t uniqueId = 0; - - //Bug 5070. Added exception handling - try + for (unsigned i = 0; i < pmList.size(); i++) { - uniqueId = fDbrm->getUnique64(); + pms.push_back((uint32_t)pmList[i]); } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(9); - args.add(ex.what()); - message.format(args); - result.result = ALTER_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - catch ( ... ) - { - logging::Message::Args args; - logging::Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = ALTER_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - string stmt = dropPartitionStmt.fSql + "|" + dropPartitionStmt.fTableName->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, sessionID, txnID.id); try { - //check table lock - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(dropPartitionStmt.fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(dropPartitionStmt.fSessionID); - CalpontSystemCatalog::TableName tableName; - tableName.schema = dropPartitionStmt.fTableName->fSchema; - tableName.table = dropPartitionStmt.fTableName->fName; - roPair = systemCatalogPtr->tableRID( tableName ); - - //@Bug 3054 check for system catalog - if ( roPair.objnum < 3000 ) - { - throw std::runtime_error("Drop partition cannot be operated on Calpont system catalog."); - } - - int i = 0; - processID = ::getpid(); - oam::OamCache* oamcache = OamCache::makeOamCache(); - std::vector pmList = oamcache->getModuleIds(); - std::vector pms; - - for (unsigned i = 0; i < pmList.size(); i++) - { - pms.push_back((uint32_t)pmList[i]); - } - - try - { - uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, (int32_t*)&txnID.id, BRM::LOADING ); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - // no need to release lock. dbrm un-hold the lock - fSessionManager.rolledback(txnID); - return result; - } - - if ( uniqueID == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - // reset - sessionID = dropPartitionStmt.fSessionID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - processID = ::getpid(); - processName = "DDLProc"; - - try - { - uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, (int32_t*)&txnID.id, BRM::LOADING ); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; - } - - if (uniqueID > 0) - break; - } - - if (i >= numTries) //error out - { - result.result = DROP_ERROR; - logging::Message::Args args; - string strOp("drop partition"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add((uint64_t)sessionID); - result.message = Message(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - fSessionManager.rolledback(txnID); - return result; - } - } - - // 1. Get the OIDs for the columns - // 2. Get the OIDs for the dictionaries - // 3. Save the OIDs to a log file - // 4. Disable the extents from extentmap for the partition - // 5. Remove the column and dictionary files for the partition - // 6. Flush PrimProc Cache - // 7. Remove the extents from extentmap for the partition - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = dropPartitionStmt.fTableName->fSchema; - userTableName.table = dropPartitionStmt.fTableName->fName; - - tableColRidList = systemCatalogPtr->columnRIDs( userTableName ); - - dictOIDList = systemCatalogPtr->dictOIDs( userTableName ); - - //Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format - for ( unsigned i = 0; i < tableColRidList.size(); i++ ) - { - if ( tableColRidList[i].objnum > 3000 ) - oidList.push_back( tableColRidList[i].objnum ); - } - - for ( unsigned i = 0; i < dictOIDList.size(); i++ ) - { - if ( dictOIDList[i].dictOID > 3000 ) - oidList.push_back( dictOIDList[i].dictOID ); - } - - //Mark the partition disabled from extent map - string emsg; - rc = fDbrm->markPartitionForDeletion( oidList, dropPartitionStmt.fPartitions, emsg); - - if (rc != 0 && rc != BRM::ERR_PARTITION_DISABLED && - rc != BRM::ERR_INVALID_OP_LAST_PARTITION && - rc != BRM::ERR_NOT_EXIST_PARTITION) - { - throw std::runtime_error(emsg); - } - - set markedPartitions; - set outOfServicePartitions; - - // only log partitions that are successfully marked disabled. - rc = fDbrm->getOutOfServicePartitions(oidList[0], outOfServicePartitions); - - if (rc != 0) - { - string errorMsg; - BRM::errString(rc, errorMsg); - ostringstream oss; - oss << "getOutOfServicePartitions failed due to " << errorMsg; - throw std::runtime_error(oss.str()); - } - - set::iterator it; - - for (it = dropPartitionStmt.fPartitions.begin(); it != dropPartitionStmt.fPartitions.end(); ++it) - { - if (outOfServicePartitions.find(*it) != outOfServicePartitions.end()) - markedPartitions.insert(*it); - } - - //Save the oids to a file - createWritePartitionLogFile( roPair.objnum, markedPartitions, oidList, uniqueId); - - VERBOSE_INFO("Removing files"); - removePartitionFiles( oidList, markedPartitions, uniqueId ); - //Flush PrimProc cache for those lbids - rc = cacheutils::flushPartition( oidList, markedPartitions ); - - //Remove the partition from extent map - emsg.clear(); - rc = fDbrm->deletePartition( oidList, dropPartitionStmt.fPartitions, emsg); - - if ( rc != 0 ) - throw std::runtime_error(emsg); - } - catch (exception& ex) - { - cerr << "DropPartitionProcessor::processPackage: " << ex.what() << endl; - - logging::Message::Args args; - logging::Message message(ex.what()); - - if (rc == BRM::ERR_TABLE_NOT_LOCKED) - result.result = USER_ERROR; - else if (rc == BRM::ERR_NOT_EXIST_PARTITION || rc == BRM::ERR_INVALID_OP_LAST_PARTITION) - result.result = PARTITION_WARNING; - else if (rc == BRM::ERR_NO_PARTITION_PERFORMED) - result.result = WARN_NO_PARTITION; - else - result.result = DROP_ERROR; - - result.message = message; - - try - { - fDbrm->releaseTableLock(uniqueID); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - } - - fSessionManager.rolledback(txnID); - return result; - } - catch (...) - { - cerr << "DropPartitionProcessor::processPackage: caught unknown exception!" << endl; - - logging::Message::Args args; - logging::Message message(1); - args.add("Drop partition failed: "); - args.add( "encountered unkown exception" ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = DROP_ERROR; - result.message = message; - - try - { - fDbrm->releaseTableLock(uniqueID); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - } - - fSessionManager.rolledback(txnID); - return result; - } - - // Log the DDL statement - logging::logDDL(dropPartitionStmt.fSessionID, txnID.id, dropPartitionStmt.fSql, dropPartitionStmt.fOwner); - - //Remove the log file - //release the transaction - try - { - fDbrm->releaseTableLock(uniqueID); - deleteLogFile(DROPPART_LOG, roPair.objnum, uniqueId); + uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, + (int32_t*)&txnID.id, BRM::LOADING); } catch (std::exception&) { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + // no need to release lock. dbrm un-hold the lock + fSessionManager.rolledback(txnID); + return result; } - fSessionManager.committed(txnID); + if (uniqueID == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + // reset + sessionID = dropPartitionStmt.fSessionID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + processID = ::getpid(); + processName = "DDLProc"; + + try + { + uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, + (int32_t*)&txnID.id, BRM::LOADING); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; + } + + if (uniqueID > 0) + break; + } + + if (i >= numTries) // error out + { + result.result = DROP_ERROR; + logging::Message::Args args; + string strOp("drop partition"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add((uint64_t)sessionID); + result.message = Message(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + fSessionManager.rolledback(txnID); + return result; + } + } + + // 1. Get the OIDs for the columns + // 2. Get the OIDs for the dictionaries + // 3. Save the OIDs to a log file + // 4. Disable the extents from extentmap for the partition + // 5. Remove the column and dictionary files for the partition + // 6. Flush PrimProc Cache + // 7. Remove the extents from extentmap for the partition + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = dropPartitionStmt.fTableName->fSchema; + userTableName.table = dropPartitionStmt.fTableName->fName; + + tableColRidList = systemCatalogPtr->columnRIDs(userTableName); + + dictOIDList = systemCatalogPtr->dictOIDs(userTableName); + + // Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format + for (unsigned i = 0; i < tableColRidList.size(); i++) + { + if (tableColRidList[i].objnum > 3000) + oidList.push_back(tableColRidList[i].objnum); + } + + for (unsigned i = 0; i < dictOIDList.size(); i++) + { + if (dictOIDList[i].dictOID > 3000) + oidList.push_back(dictOIDList[i].dictOID); + } + + // Mark the partition disabled from extent map + string emsg; + rc = fDbrm->markPartitionForDeletion(oidList, dropPartitionStmt.fPartitions, emsg); + + if (rc != 0 && rc != BRM::ERR_PARTITION_DISABLED && rc != BRM::ERR_INVALID_OP_LAST_PARTITION && + rc != BRM::ERR_NOT_EXIST_PARTITION) + { + throw std::runtime_error(emsg); + } + + set markedPartitions; + set outOfServicePartitions; + + // only log partitions that are successfully marked disabled. + rc = fDbrm->getOutOfServicePartitions(oidList[0], outOfServicePartitions); + + if (rc != 0) + { + string errorMsg; + BRM::errString(rc, errorMsg); + ostringstream oss; + oss << "getOutOfServicePartitions failed due to " << errorMsg; + throw std::runtime_error(oss.str()); + } + + set::iterator it; + + for (it = dropPartitionStmt.fPartitions.begin(); it != dropPartitionStmt.fPartitions.end(); ++it) + { + if (outOfServicePartitions.find(*it) != outOfServicePartitions.end()) + markedPartitions.insert(*it); + } + + // Save the oids to a file + createWritePartitionLogFile(roPair.objnum, markedPartitions, oidList, uniqueId); + + VERBOSE_INFO("Removing files"); + removePartitionFiles(oidList, markedPartitions, uniqueId); + // Flush PrimProc cache for those lbids + rc = cacheutils::flushPartition(oidList, markedPartitions); + + // Remove the partition from extent map + emsg.clear(); + rc = fDbrm->deletePartition(oidList, dropPartitionStmt.fPartitions, emsg); + + if (rc != 0) + throw std::runtime_error(emsg); + } + catch (exception& ex) + { + cerr << "DropPartitionProcessor::processPackage: " << ex.what() << endl; + + logging::Message::Args args; + logging::Message message(ex.what()); + + if (rc == BRM::ERR_TABLE_NOT_LOCKED) + result.result = USER_ERROR; + else if (rc == BRM::ERR_NOT_EXIST_PARTITION || rc == BRM::ERR_INVALID_OP_LAST_PARTITION) + result.result = PARTITION_WARNING; + else if (rc == BRM::ERR_NO_PARTITION_PERFORMED) + result.result = WARN_NO_PARTITION; + else + result.result = DROP_ERROR; + + result.message = message; + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + } + + fSessionManager.rolledback(txnID); return result; + } + catch (...) + { + cerr << "DropPartitionProcessor::processPackage: caught unknown exception!" << endl; + + logging::Message::Args args; + logging::Message message(1); + args.add("Drop partition failed: "); + args.add("encountered unkown exception"); + args.add(""); + args.add(""); + message.format(args); + + result.result = DROP_ERROR; + result.message = message; + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + } + + fSessionManager.rolledback(txnID); + return result; + } + + // Log the DDL statement + logging::logDDL(dropPartitionStmt.fSessionID, txnID.id, dropPartitionStmt.fSql, dropPartitionStmt.fOwner); + + // Remove the log file + // release the transaction + try + { + fDbrm->releaseTableLock(uniqueID); + deleteLogFile(DROPPART_LOG, roPair.objnum, uniqueId); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; + } + + fSessionManager.committed(txnID); + return result; } -} +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/droppartitionprocessor.h b/dbcon/ddlpackageproc/droppartitionprocessor.h index 96747d022..5cc097f35 100644 --- a/dbcon/ddlpackageproc/droppartitionprocessor.h +++ b/dbcon/ddlpackageproc/droppartitionprocessor.h @@ -40,21 +40,21 @@ namespace ddlpackageprocessor */ class DropPartitionProcessor : public DDLPackageProcessor { -public: - DropPartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process a drop table statement - * - * @param dropTableStmt the drop table statement - */ - EXPORT DDLResult processPackage(ddlpackage::DropPartitionStatement& dropPartitionStmt); - -protected: - -private: + public: + DropPartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process a drop table statement + * + * @param dropTableStmt the drop table statement + */ + EXPORT DDLResult processPackage(ddlpackage::DropPartitionStatement& dropPartitionStmt); + protected: + private: }; -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor #undef EXPORT -#endif //DROPPARTITIONEPROCESSOR_H +#endif // DROPPARTITIONEPROCESSOR_H diff --git a/dbcon/ddlpackageproc/droptableprocessor.cpp b/dbcon/ddlpackageproc/droptableprocessor.cpp index 769bab407..e1ca785d9 100644 --- a/dbcon/ddlpackageproc/droptableprocessor.cpp +++ b/dbcon/ddlpackageproc/droptableprocessor.cpp @@ -50,547 +50,981 @@ using namespace oam; namespace ddlpackageprocessor { - -DropTableProcessor::DDLResult DropTableProcessor::processPackage(ddlpackage::DropTableStatement& dropTableStmt) +DropTableProcessor::DDLResult DropTableProcessor::processPackage( + ddlpackage::DropTableStatement& dropTableStmt) { - SUMMARY_INFO("DropTableProcessor::processPackage"); + SUMMARY_INFO("DropTableProcessor::processPackage"); - DDLResult result; - result.result = NO_ERROR; - std::string err; - VERBOSE_INFO(dropTableStmt); + DDLResult result; + result.result = NO_ERROR; + std::string err; + VERBOSE_INFO(dropTableStmt); - // Commit current transaction. - // all DDL statements cause an implicit commit - VERBOSE_INFO("Getting current txnID"); - ByteStream::byte rc = 0; - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - int rc1 = 0; - rc1 = fDbrm->isReadWrite(); + // Commit current transaction. + // all DDL statements cause an implicit commit + VERBOSE_INFO("Getting current txnID"); + ByteStream::byte rc = 0; + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + int rc1 = 0; + rc1 = fDbrm->isReadWrite(); - if (rc1 != 0 ) - { - Message::Args args; - Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } + if (rc1 != 0) + { + Message::Args args; + Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } - string stmt = dropTableStmt.fSql + "|" + dropTableStmt.fTableName->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, dropTableStmt.fSessionID, txnID.id); + string stmt = dropTableStmt.fSql + "|" + dropTableStmt.fTableName->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, dropTableStmt.fSessionID, txnID.id); - std::vector oidList; - CalpontSystemCatalog::RIDList tableColRidList; - CalpontSystemCatalog::DictOIDList dictOIDList; - execplan::CalpontSystemCatalog::ROPair roPair; - std::string errorMsg; - ByteStream bytestream; - uint64_t uniqueId = 0; + std::vector oidList; + CalpontSystemCatalog::RIDList tableColRidList; + CalpontSystemCatalog::DictOIDList dictOIDList; + execplan::CalpontSystemCatalog::ROPair roPair; + std::string errorMsg; + ByteStream bytestream; + uint64_t uniqueId = 0; - //Bug 5070. Added exception handling - try - { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - Message::Args args; - Message message(9); - args.add(ex.what()); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - catch ( ... ) - { - Message::Args args; - Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + Message::Args args; + Message message(9); + args.add(ex.what()); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + catch (...) + { + Message::Args args; + Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } - fWEClient->addQueue(uniqueId); - int pmNum = 1; - boost::shared_ptr bsIn; - uint64_t tableLockId = 0; - OamCache* oamcache = OamCache::makeOamCache(); - std::vector moduleIds = oamcache->getModuleIds(); + fWEClient->addQueue(uniqueId); + int pmNum = 1; + boost::shared_ptr bsIn; + uint64_t tableLockId = 0; + OamCache* oamcache = OamCache::makeOamCache(); + std::vector moduleIds = oamcache->getModuleIds(); - // MCOL-66 The DBRM can't handle concurrent DDL - boost::mutex::scoped_lock lk(dbrmMutex); + // MCOL-66 The DBRM can't handle concurrent DDL + boost::mutex::scoped_lock lk(dbrmMutex); + + try + { + // check table lock + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(dropTableStmt.fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(dropTableStmt.fSessionID); + CalpontSystemCatalog::TableName tableName; + tableName.schema = dropTableStmt.fTableName->fSchema; + tableName.table = dropTableStmt.fTableName->fName; try { - //check table lock - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(dropTableStmt.fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(dropTableStmt.fSessionID); - CalpontSystemCatalog::TableName tableName; - tableName.schema = dropTableStmt.fTableName->fSchema; - tableName.table = dropTableStmt.fTableName->fName; - - try - { - roPair = systemCatalogPtr->tableRID(tableName); - } - catch (IDBExcept& ie) - { - if (ie.errorCode() == ERR_TABLE_NOT_IN_CATALOG) - { - Message::Args args; - Message message(1); - args.add("Table does not exist in ColumnStore."); - message.format(args); - result.result = DROP_TABLE_NOT_IN_CATALOG_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - else - { - result.result = DROP_ERROR; - Message::Args args; - Message message(9); - args.add("Drop table failed due to "); - args.add(ie.what()); - message.format(args); - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - } - - uint32_t processID = ::getpid(); - int32_t txnid = txnID.id; - int32_t sessionId = dropTableStmt.fSessionID; - std::string processName("DDLProc"); - int i = 0; - - std::vector pms; - - for (unsigned i = 0; i < moduleIds.size(); i++) - { - pms.push_back((uint32_t)moduleIds[i]); - } - - try - { - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if ( tableLockId == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = WriteEngine::Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - - try - { - processID = ::getpid(); - txnid = txnID.id; - sessionId = dropTableStmt.fSessionID;; - processName = "DDLProc"; - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (tableLockId > 0) - break; - } - - if (i >= numTries) //error out - { - Message::Args args; - string strOp("drop table"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add(sessionId); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - } - } - - // 1. Get the OIDs for the columns - // 2. Get the OIDs for the dictionaries - // 3. Save the OIDs to a log file - // 4. Remove the Table from SYSTABLE - // 5. Remove the columns from SYSCOLUMN - // 6. Commit the changes made to systables - // 7. Flush PrimProc Cache - // 8. Update extent map - // 9. Remove the column and dictionary files - // 10.Return the OIDs - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = dropTableStmt.fTableName->fSchema; - userTableName.table = dropTableStmt.fTableName->fName; - - tableColRidList = systemCatalogPtr->columnRIDs( userTableName ); - - dictOIDList = systemCatalogPtr->dictOIDs( userTableName ); - Oam oam; - - //Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format - for ( unsigned i = 0; i < tableColRidList.size(); i++ ) - { - if ( tableColRidList[i].objnum > 3000 ) - oidList.push_back( tableColRidList[i].objnum ); - } - - for ( unsigned i = 0; i < dictOIDList.size(); i++ ) - { - if ( dictOIDList[i].dictOID > 3000 ) - oidList.push_back( dictOIDList[i].dictOID ); - } - - //get a unique number - VERBOSE_INFO("Removing the SYSTABLE meta data"); -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Removing the SYSTABLEs meta data" << endl; -#endif - bytestream << (ByteStream::byte)WE_SVR_DELETE_SYSTABLE; - bytestream << uniqueId; - bytestream << (uint32_t) dropTableStmt.fSessionID; - bytestream << (uint32_t)txnID.id; - bytestream << dropTableStmt.fTableName->fSchema; - bytestream << dropTableStmt.fTableName->fName; - - //Find out where systable is - BRM::OID_t sysOid = 1001; - ByteStream::byte rc = 0; - - uint16_t dbRoot; - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - { - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("Error while calling getSysCatDBRoot"); - args.add(errorMsg); - result.message = message; - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbRoot]; - - try - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table sending WE_SVR_DELETE_SYSTABLES to pm " << pmNum << endl; -#endif - //cout << "deleting systable entries with txnid " << txnID.id << endl; - fWEClient->write(bytestream, (uint32_t)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table got exception" << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table got unknown exception" << endl; -#endif - } - - if (rc != 0) - { - cout << fTxnid.id << " Error in dropping table from systables(" << (int)rc << ") " << errorMsg.c_str() << endl; - Message::Args args; - Message message(9); - args.add("Error in dropping table from systables."); - args.add(errorMsg); - message.format(args); - result.result = (ResultCode)rc; - result.message = message; - //release table lock and session - fSessionManager.rolledback(txnID); - (void)fDbrm->releaseTableLock(tableLockId); - fWEClient->removeQueue(uniqueId); - return result; - } - - //remove from syscolumn - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_DELETE_SYSCOLUMN; - bytestream << uniqueId; - bytestream << (uint32_t) dropTableStmt.fSessionID; - bytestream << (uint32_t)txnID.id; - bytestream << dropTableStmt.fTableName->fSchema; - bytestream << dropTableStmt.fTableName->fName; - - //Find out where syscolumn is - sysOid = 1021; - rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); - - if (rc != 0) - { - result.result = (ResultCode) rc; - Message::Args args; - Message message(9); - args.add("Error while calling getSysCatDBRoot"); - args.add(errorMsg); - result.message = message; - //release transaction - fSessionManager.rolledback(txnID); - return result; - } - - pmNum = (*dbRootPMMap)[dbRoot]; - - try - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table sending WE_SVR_DELETE_SYSCOLUMN to pm " << pmNum << endl; -#endif - fWEClient->write(bytestream, (unsigned)pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> rc; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - } - catch (runtime_error& ex) //write error - { -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table got exception" << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - rc = NETWORK_ERROR; -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table got unknown exception" << endl; -#endif - } - - if (rc != 0) - { - cout << fTxnid.id << " Error in dropping column from systables(" << (int)rc << ") " << errorMsg.c_str() << endl; - Message::Args args; - Message message(9); - args.add("Error in dropping column from systables."); - args.add(errorMsg); - message.format(args); - result.result = (ResultCode)rc; - result.message = message; - //release table lock and session - fSessionManager.rolledback(txnID); - (void)fDbrm->releaseTableLock(tableLockId); - fWEClient->removeQueue(uniqueId); - return result; - } - - rc = commitTransaction(uniqueId, txnID); - - if (rc != 0) - { - cout << txnID.id << " rolledback transaction " << " and valid is " << txnID.valid << endl; - fSessionManager.rolledback(txnID); - } - else - { - cout << txnID.id << " commiting transaction " << txnID.id << " and valid is " << txnID.valid << endl; - fSessionManager.committed(txnID); - } - - if (rc != 0) - { - Message::Args args; - Message message(9); - ostringstream oss; - oss << " Commit failed with error code " << rc; - args.add(oss.str()); - fSessionManager.rolledback(txnID); - (void)fDbrm->releaseTableLock(tableLockId); - message.format(args); - result.result = (ResultCode)rc; - result.message = message; - fWEClient->removeQueue(uniqueId); - return result; - } - - // Log the DDL statement - logDDL(dropTableStmt.fSessionID, txnID.id, dropTableStmt.fSql, dropTableStmt.fOwner); + roPair = systemCatalogPtr->tableRID(tableName); } - catch (std::exception& ex) + catch (IDBExcept& ie) { + if (ie.errorCode() == ERR_TABLE_NOT_IN_CATALOG) + { + Message::Args args; + Message message(1); + args.add("Table does not exist in ColumnStore."); + message.format(args); + result.result = DROP_TABLE_NOT_IN_CATALOG_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + else + { result.result = DROP_ERROR; Message::Args args; Message message(9); args.add("Drop table failed due to "); - args.add(ex.what()); - fSessionManager.rolledback(txnID); - - try - { - (void)fDbrm->releaseTableLock(tableLockId); - } - catch (std::exception&) - { - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - message.format( args ); + args.add(ie.what()); + message.format(args); result.message = message; - fWEClient->removeQueue(uniqueId); + fSessionManager.rolledback(txnID); return result; + } } - catch (...) + + uint32_t processID = ::getpid(); + int32_t txnid = txnID.id; + int32_t sessionId = dropTableStmt.fSessionID; + std::string processName("DDLProc"); + int i = 0; + + std::vector pms; + + for (unsigned i = 0; i < moduleIds.size(); i++) { - result.result = DROP_ERROR; - errorMsg = "Error in getting information from system catalog or from dbrm."; - Message::Args args; - Message message(9); - args.add("Drop table failed due to "); - args.add(errorMsg); - fSessionManager.rolledback(txnID); - - try - { - (void)fDbrm->releaseTableLock(tableLockId); - } - catch (std::exception&) - { - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - message.format( args ); - result.message = message; - fWEClient->removeQueue(uniqueId); - return result; + pms.push_back((uint32_t)moduleIds[i]); } try { - (void)fDbrm->releaseTableLock(tableLockId); + tableLockId = + fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING); } catch (std::exception&) { - result.result = DROP_ERROR; - Message::Args args; - Message message(9); - args.add("Drop table failed due to "); - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - fSessionManager.rolledback(txnID); - message.format( args ); - result.message = message; - fWEClient->removeQueue(uniqueId); - return result; + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); } - //Save the oids to a file + if (tableLockId == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = WriteEngine::Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + + try + { + processID = ::getpid(); + txnid = txnID.id; + sessionId = dropTableStmt.fSessionID; + ; + processName = "DDLProc"; + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, + BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + if (tableLockId > 0) + break; + } + + if (i >= numTries) // error out + { + Message::Args args; + string strOp("drop table"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add(sessionId); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + } + } + + // 1. Get the OIDs for the columns + // 2. Get the OIDs for the dictionaries + // 3. Save the OIDs to a log file + // 4. Remove the Table from SYSTABLE + // 5. Remove the columns from SYSCOLUMN + // 6. Commit the changes made to systables + // 7. Flush PrimProc Cache + // 8. Update extent map + // 9. Remove the column and dictionary files + // 10.Return the OIDs + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = dropTableStmt.fTableName->fSchema; + userTableName.table = dropTableStmt.fTableName->fName; + + tableColRidList = systemCatalogPtr->columnRIDs(userTableName); + + dictOIDList = systemCatalogPtr->dictOIDs(userTableName); + Oam oam; + + // Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format + for (unsigned i = 0; i < tableColRidList.size(); i++) + { + if (tableColRidList[i].objnum > 3000) + oidList.push_back(tableColRidList[i].objnum); + } + + for (unsigned i = 0; i < dictOIDList.size(); i++) + { + if (dictOIDList[i].dictOID > 3000) + oidList.push_back(dictOIDList[i].dictOID); + } + + // get a unique number + VERBOSE_INFO("Removing the SYSTABLE meta data"); +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Removing the SYSTABLEs meta data" << endl; +#endif + bytestream << (ByteStream::byte)WE_SVR_DELETE_SYSTABLE; + bytestream << uniqueId; + bytestream << (uint32_t)dropTableStmt.fSessionID; + bytestream << (uint32_t)txnID.id; + bytestream << dropTableStmt.fTableName->fSchema; + bytestream << dropTableStmt.fTableName->fName; + + // Find out where systable is + BRM::OID_t sysOid = 1001; + ByteStream::byte rc = 0; + + uint16_t dbRoot; + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + { + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("Error while calling getSysCatDBRoot"); + args.add(errorMsg); + result.message = message; + // release transaction + fSessionManager.rolledback(txnID); + return result; + } + + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbRoot]; + try { - createWriteDropLogFile( roPair.objnum, uniqueId, oidList ); +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table sending WE_SVR_DELETE_SYSTABLES to pm " << pmNum << endl; +#endif + // cout << "deleting systable entries with txnid " << txnID.id << endl; + fWEClient->write(bytestream, (uint32_t)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } } - catch (std::exception& ex) + catch (runtime_error& ex) // write error { - result.result = WARNING; - Message::Args args; - Message message(9); - args.add("Drop table failed due to "); - args.add(ex.what()); - message.format(args); - result.message = message; - fSessionManager.rolledback(txnID); +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table got exception" << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table got unknown exception" << endl; +#endif + } + + if (rc != 0) + { + cout << fTxnid.id << " Error in dropping table from systables(" << (int)rc << ") " << errorMsg.c_str() + << endl; + Message::Args args; + Message message(9); + args.add("Error in dropping table from systables."); + args.add(errorMsg); + message.format(args); + result.result = (ResultCode)rc; + result.message = message; + // release table lock and session + fSessionManager.rolledback(txnID); + (void)fDbrm->releaseTableLock(tableLockId); + fWEClient->removeQueue(uniqueId); + return result; + } + + // remove from syscolumn + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_DELETE_SYSCOLUMN; + bytestream << uniqueId; + bytestream << (uint32_t)dropTableStmt.fSessionID; + bytestream << (uint32_t)txnID.id; + bytestream << dropTableStmt.fTableName->fSchema; + bytestream << dropTableStmt.fTableName->fName; + + // Find out where syscolumn is + sysOid = 1021; + rc = fDbrm->getSysCatDBRoot(sysOid, dbRoot); + + if (rc != 0) + { + result.result = (ResultCode)rc; + Message::Args args; + Message message(9); + args.add("Error while calling getSysCatDBRoot"); + args.add(errorMsg); + result.message = message; + // release transaction + fSessionManager.rolledback(txnID); + return result; + } + + pmNum = (*dbRootPMMap)[dbRoot]; + + try + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table sending WE_SVR_DELETE_SYSCOLUMN to pm " << pmNum << endl; +#endif + fWEClient->write(bytestream, (unsigned)pmNum); + + while (1) + { + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> rc; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table got exception" << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + rc = NETWORK_ERROR; +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table got unknown exception" << endl; +#endif + } + + if (rc != 0) + { + cout << fTxnid.id << " Error in dropping column from systables(" << (int)rc << ") " << errorMsg.c_str() + << endl; + Message::Args args; + Message message(9); + args.add("Error in dropping column from systables."); + args.add(errorMsg); + message.format(args); + result.result = (ResultCode)rc; + result.message = message; + // release table lock and session + fSessionManager.rolledback(txnID); + (void)fDbrm->releaseTableLock(tableLockId); + fWEClient->removeQueue(uniqueId); + return result; + } + + rc = commitTransaction(uniqueId, txnID); + + if (rc != 0) + { + cout << txnID.id << " rolledback transaction " + << " and valid is " << txnID.valid << endl; + fSessionManager.rolledback(txnID); + } + else + { + cout << txnID.id << " commiting transaction " << txnID.id << " and valid is " << txnID.valid << endl; + fSessionManager.committed(txnID); + } + + if (rc != 0) + { + Message::Args args; + Message message(9); + ostringstream oss; + oss << " Commit failed with error code " << rc; + args.add(oss.str()); + fSessionManager.rolledback(txnID); + (void)fDbrm->releaseTableLock(tableLockId); + message.format(args); + result.result = (ResultCode)rc; + result.message = message; + fWEClient->removeQueue(uniqueId); + return result; + } + + // Log the DDL statement + logDDL(dropTableStmt.fSessionID, txnID.id, dropTableStmt.fSql, dropTableStmt.fOwner); + } + catch (std::exception& ex) + { + result.result = DROP_ERROR; + Message::Args args; + Message message(9); + args.add("Drop table failed due to "); + args.add(ex.what()); + fSessionManager.rolledback(txnID); + + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + message.format(args); + result.message = message; + fWEClient->removeQueue(uniqueId); + return result; + } + catch (...) + { + result.result = DROP_ERROR; + errorMsg = "Error in getting information from system catalog or from dbrm."; + Message::Args args; + Message message(9); + args.add("Drop table failed due to "); + args.add(errorMsg); + fSessionManager.rolledback(txnID); + + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + message.format(args); + result.message = message; + fWEClient->removeQueue(uniqueId); + return result; + } + + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + Message::Args args; + Message message(9); + args.add("Drop table failed due to "); + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + fSessionManager.rolledback(txnID); + message.format(args); + result.message = message; + fWEClient->removeQueue(uniqueId); + return result; + } + + // Save the oids to a file + try + { + createWriteDropLogFile(roPair.objnum, uniqueId, oidList); + } + catch (std::exception& ex) + { + result.result = WARNING; + Message::Args args; + Message message(9); + args.add("Drop table failed due to "); + args.add(ex.what()); + message.format(args); + result.message = message; + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + return result; + } + + // Bug 4208 Drop the PrimProcFDCache before droping the column files + // FOr Windows, this ensures (most likely) that the column files have + // no open handles to hinder the deletion of the files. + rc = cacheutils::dropPrimProcFdCache(); + + // Drop files + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; + bytestream << uniqueId; + bytestream << (uint32_t)oidList.size(); + + for (unsigned i = 0; i < oidList.size(); i++) + { + bytestream << (uint32_t)oidList[i]; + } + +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table removing column files" << endl; +#endif + uint32_t msgRecived = 0; + + try + { + fWEClient->write_to_all(bytestream); + + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; fWEClient->removeQueue(uniqueId); - return result; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; + } + else + msgRecived++; + } + } + } + catch (std::exception& ex) + { + result.result = WARNING; + Message::Args args; + Message message(9); + args.add("Drop table failed due to "); + args.add(ex.what()); + message.format(args); + result.message = message; + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + return result; + } + catch (...) + { + result.result = WARNING; + errorMsg = "Error in getting information from system catalog or from dbrm."; + Message::Args args; + Message message(9); + args.add("Drop table failed due to "); + args.add(errorMsg); + message.format(args); + result.message = message; + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + return result; + } + + // Drop PrimProc FD cache + rc = cacheutils::dropPrimProcFdCache(); + // Flush primProc cache + rc = cacheutils::flushOIDsFromCache(oidList); + // Delete extents from extent map +#ifdef IDB_DDL_DEBUG + cout << fTxnid.id << " Drop table deleteOIDs" << endl; +#endif + rc = fDbrm->deleteOIDs(oidList); + + if (rc != 0) + { + Message::Args args; + Message message(1); + args.add("Table dropped with warning "); + args.add("Remove from extent map failed."); + args.add(""); + args.add(""); + message.format(args); + + result.result = WARNING; + result.message = message; + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + return result; + } + + // Remove the log file + fWEClient->removeQueue(uniqueId); + deleteLogFile(DROPTABLE_LOG, roPair.objnum, uniqueId); + // release the transaction + // fSessionManager.committed(txnID); + returnOIDs(tableColRidList, dictOIDList); + return result; +} + +TruncTableProcessor::DDLResult TruncTableProcessor::processPackage( + ddlpackage::TruncTableStatement& truncTableStmt) +{ + SUMMARY_INFO("TruncTableProcessor::processPackage"); + // 1. lock the table + // 2. Get the OIDs for the columns + // 3. Get the OIDs for the dictionaries + // 4. Save the OIDs + // 5. Disable all partitions + // 6. Remove the column and dictionary files + // 7. Flush PrimProc Cache + // 8. Update extent map + // 9. Use the OIDs to create new column and dictionary files with abbreviate extent + // 10 Update next value if the table has autoincrement column + + DDLResult result; + result.result = NO_ERROR; + std::string err; + VERBOSE_INFO(truncTableStmt); + + // @Bug 4150. Check dbrm status before doing anything to the table. + int rc = 0; + rc = fDbrm->isReadWrite(); + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + + if (rc != 0) + { + Message::Args args; + Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + //@Bug 5765 log the schema. + string stmt = truncTableStmt.fSql + "|" + truncTableStmt.fTableName->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, truncTableStmt.fSessionID, txnID.id); + + std::vector columnOidList; + std::vector allOidList; + CalpontSystemCatalog::RIDList tableColRidList; + CalpontSystemCatalog::DictOIDList dictOIDList; + execplan::CalpontSystemCatalog::ROPair roPair; + std::string processName("DDLProc"); + uint32_t processID = ::getpid(); + ; + int32_t txnid = txnID.id; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(truncTableStmt.fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(truncTableStmt.fSessionID); + CalpontSystemCatalog::TableInfo tableInfo; + uint64_t uniqueId = 0; + + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + Message::Args args; + Message message(9); + args.add(ex.what()); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + catch (...) + { + Message::Args args; + Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + fWEClient->addQueue(uniqueId); + int pmNum = 1; + boost::shared_ptr bsIn; + string errorMsg; + uint32_t autoIncColOid = 0; + uint64_t tableLockId = 0; + OamCache* oamcache = OamCache::makeOamCache(); + std::vector moduleIds = oamcache->getModuleIds(); + + try + { + // check table lock + + CalpontSystemCatalog::TableName tableName; + tableName.schema = truncTableStmt.fTableName->fSchema; + tableName.table = truncTableStmt.fTableName->fName; + roPair = systemCatalogPtr->tableRID(tableName); + int32_t sessionId = truncTableStmt.fSessionID; + std::string processName("DDLProc"); + int i = 0; + + std::vector pms; + + for (unsigned i = 0; i < moduleIds.size(); i++) + { + pms.push_back((uint32_t)moduleIds[i]); + } + + try + { + tableLockId = + fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + if (tableLockId == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + + try + { + processID = ::getpid(); + txnid = txnID.id; + sessionId = truncTableStmt.fSessionID; + processName = "DDLProc"; + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, + BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + if (tableLockId > 0) + break; + } + + if (i >= numTries) // error out + { + Message::Args args; + args.add(processName); + args.add((uint64_t)processID); + args.add(sessionId); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + } + } + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = truncTableStmt.fTableName->fSchema; + userTableName.table = truncTableStmt.fTableName->fName; + + tableColRidList = systemCatalogPtr->columnRIDs(userTableName); + + dictOIDList = systemCatalogPtr->dictOIDs(userTableName); + + for (unsigned i = 0; i < tableColRidList.size(); i++) + { + if (tableColRidList[i].objnum > 3000) + { + columnOidList.push_back(tableColRidList[i].objnum); + allOidList.push_back(tableColRidList[i].objnum); + } + } + + for (unsigned i = 0; i < dictOIDList.size(); i++) + { + if (dictOIDList[i].dictOID > 3000) + allOidList.push_back(dictOIDList[i].dictOID); + } + + // Check whether the table has autoincrement column + tableInfo = systemCatalogPtr->tableInfo(userTableName); + } + catch (std::exception& ex) + { + cerr << "TruncateTableProcessor::processPackage: " << ex.what() << endl; + + Message::Args args; + Message message(9); + args.add("Truncate table failed: "); + args.add(ex.what()); + args.add(""); + fSessionManager.rolledback(txnID); + + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + fWEClient->removeQueue(uniqueId); + message.format(args); + + result.result = TRUNC_ERROR; + result.message = message; + return result; + } + catch (...) + { + cerr << "TruncateTableProcessor::processPackage: caught unknown exception!" << endl; + + Message::Args args; + Message message(1); + args.add("Truncate table failed: "); + args.add("encountered unkown exception"); + args.add(""); + + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + fWEClient->removeQueue(uniqueId); + message.format(args); + + result.result = TRUNC_ERROR; + result.message = message; + return result; + } + + // Save the oids to a file + try + { + createWriteTruncateTableLogFile(roPair.objnum, uniqueId, allOidList); + } + catch (std::exception& ex) + { + Message::Args args; + Message message(9); + args.add("Truncate table failed due to "); + args.add(ex.what()); + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + message.format(args); + + //@bug 4515 Release the tablelock as nothing has done to this table. + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + } + + result.result = TRUNC_ERROR; + result.message = message; + return result; + } + + ByteStream bytestream; + ByteStream::byte tmp8; + + // MCOL-66 The DBRM can't handle concurrent DDL + boost::mutex::scoped_lock lk(dbrmMutex); + + try + { + // Disable extents first + int rc1 = fDbrm->markAllPartitionForDeletion(allOidList); + + if (rc1 != 0) + { + string errMsg; + BRM::errString(rc, errMsg); + throw std::runtime_error(errMsg); } // Bug 4208 Drop the PrimProcFDCache before droping the column files @@ -598,760 +1032,339 @@ DropTableProcessor::DDLResult DropTableProcessor::processPackage(ddlpackage::Dro // no open handles to hinder the deletion of the files. rc = cacheutils::dropPrimProcFdCache(); - //Drop files - bytestream.restart(); + VERBOSE_INFO("Removing files"); bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; bytestream << uniqueId; - bytestream << (uint32_t) oidList.size(); + bytestream << (uint32_t)allOidList.size(); - for (unsigned i = 0; i < oidList.size(); i++) + for (unsigned i = 0; i < allOidList.size(); i++) { - bytestream << (uint32_t) oidList[i]; + bytestream << (uint32_t)allOidList[i]; } -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table removing column files" << endl; -#endif uint32_t msgRecived = 0; try { - fWEClient->write_to_all(bytestream); + fWEClient->write_to_all(bytestream); + bsIn.reset(new ByteStream()); + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; + } + else + msgRecived++; + } + } + } + catch (std::exception& ex) + { + Message::Args args; + Message message(9); + args.add("Truncate table failed due to "); + args.add(ex.what()); + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + message.format(args); + + result.result = TRUNC_ERROR; + result.message = message; + deleteLogFile(TRUNCATE_LOG, roPair.objnum, uniqueId); + return result; + } + catch (...) + { + result.result = DROP_ERROR; + errorMsg = "Error in getting information from system catalog or from dbrm."; + Message::Args args; + Message message(9); + args.add("Truncate table failed due to "); + args.add(errorMsg); + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + message.format(args); + + result.result = TRUNC_ERROR; + result.message = message; + deleteLogFile(TRUNCATE_LOG, roPair.objnum, uniqueId); + return result; + } + + // Drop PrimProc FD cache + rc = cacheutils::dropPrimProcFdCache(); + // Flush primProc cache + rc = cacheutils::flushOIDsFromCache(allOidList); + // Delete extents from extent map + rc = fDbrm->deleteOIDs(allOidList); + + if (rc != 0) + { + Message::Args args; + Message message(1); + args.add("Table truncated with warning "); + args.add("Remove from extent map failed."); + args.add(""); + args.add(""); + message.format(args); + + result.result = WARNING; + result.message = message; + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + return result; + } + + // Get the number of tables in the database, the current table is included. + int tableCount = systemCatalogPtr->getTableCount(); + Oam oam; + // Calculate which dbroot the columns should start + DBRootConfigList dbRootList = oamcache->getDBRootNums(); + + uint16_t useDBRootIndex = tableCount % dbRootList.size(); + // Find out the dbroot# corresponding the useDBRootIndex from oam + uint16_t useDBRoot = dbRootList[useDBRootIndex]; + + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES; + bytestream << uniqueId; + bytestream << (uint32_t)txnID.id; + uint32_t numOids = columnOidList.size() + dictOIDList.size(); + bytestream << numOids; + CalpontSystemCatalog::ColType colType; + + for (unsigned col = 0; col < columnOidList.size(); col++) + { + colType = systemCatalogPtr->colType(columnOidList[col]); + + if (colType.autoincrement) + autoIncColOid = colType.columnOID; + + bytestream << (uint32_t)columnOidList[col]; + bytestream << (uint8_t)colType.colDataType; + bytestream << (uint8_t) false; + bytestream << (uint32_t)colType.colWidth; + bytestream << (uint16_t)useDBRoot; + bytestream << (uint32_t)colType.compressionType; + } + + for (unsigned col = 0; col < dictOIDList.size(); col++) + { + colType = systemCatalogPtr->colTypeDct(dictOIDList[col].dictOID); + bytestream << (uint32_t)dictOIDList[col].dictOID; + bytestream << (uint8_t)colType.colDataType; + bytestream << (uint8_t) true; + bytestream << (uint32_t)colType.colWidth; + bytestream << (uint16_t)useDBRoot; + bytestream << (uint32_t)colType.compressionType; + } + + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[useDBRoot]; + + try + { +#ifdef IDB_DDL_DEBUG + cout << "Truncate table sending We_SVR_WRITE_CREATETABLEFILES to pm " << pmNum << endl; +#endif + fWEClient->write(bytestream, pmNum); + + while (1) + { bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + } + + break; + } + } + + if (rc != 0) + { + // drop the newly created files + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; + bytestream << uniqueId; + bytestream << (uint32_t)(allOidList.size()); + + for (unsigned i = 0; i < (allOidList.size()); i++) + { + bytestream << (uint32_t)(allOidList[i]); + } + + fWEClient->write(bytestream, pmNum); while (1) { - if (msgRecived == fWEClient->getPmCount()) - break; + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - fWEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } + if (bsIn->length() == 0) // read error + { + break; + } + else + { + *bsIn >> tmp8; + // rc = tmp8; + break; + } } - } - catch (std::exception& ex) - { - result.result = WARNING; + Message::Args args; - Message message(9); - args.add("Drop table failed due to "); - args.add(ex.what()); - message.format(args); - result.message = message; - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - return result; - } - catch (...) - { - result.result = WARNING; - errorMsg = "Error in getting information from system catalog or from dbrm."; - Message::Args args; - Message message(9); - args.add("Drop table failed due to "); + Message message(1); + args.add("Truncate table failed."); args.add(errorMsg); - message.format(args); - result.message = message; - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - return result; - } - - //Drop PrimProc FD cache - rc = cacheutils::dropPrimProcFdCache(); - //Flush primProc cache - rc = cacheutils::flushOIDsFromCache( oidList ); - //Delete extents from extent map -#ifdef IDB_DDL_DEBUG - cout << fTxnid.id << " Drop table deleteOIDs" << endl; -#endif - rc = fDbrm->deleteOIDs(oidList); - - if (rc != 0) - { - Message::Args args; - Message message(1); - args.add("Table dropped with warning "); - args.add( "Remove from extent map failed." ); args.add(""); - args.add(""); - message.format( args ); - - result.result = WARNING; - result.message = message; - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - return result; - } - - //Remove the log file - fWEClient->removeQueue(uniqueId); - deleteLogFile(DROPTABLE_LOG, roPair.objnum, uniqueId); - //release the transaction - //fSessionManager.committed(txnID); - returnOIDs( tableColRidList, dictOIDList ); - return result; - -} - -TruncTableProcessor::DDLResult TruncTableProcessor::processPackage(ddlpackage::TruncTableStatement& truncTableStmt) -{ - SUMMARY_INFO("TruncTableProcessor::processPackage"); - // 1. lock the table - // 2. Get the OIDs for the columns - // 3. Get the OIDs for the dictionaries - // 4. Save the OIDs - // 5. Disable all partitions - // 6. Remove the column and dictionary files - // 7. Flush PrimProc Cache - // 8. Update extent map - // 9. Use the OIDs to create new column and dictionary files with abbreviate extent - // 10 Update next value if the table has autoincrement column - - DDLResult result; - result.result = NO_ERROR; - std::string err; - VERBOSE_INFO(truncTableStmt); - - // @Bug 4150. Check dbrm status before doing anything to the table. - int rc = 0; - rc = fDbrm->isReadWrite(); - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - - if (rc != 0 ) - { - Message::Args args; - Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - //@Bug 5765 log the schema. - string stmt = truncTableStmt.fSql + "|" + truncTableStmt.fTableName->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, truncTableStmt.fSessionID, txnID.id); - - std::vector columnOidList; - std::vector allOidList; - CalpontSystemCatalog::RIDList tableColRidList; - CalpontSystemCatalog::DictOIDList dictOIDList; - execplan::CalpontSystemCatalog::ROPair roPair; - std::string processName("DDLProc"); - uint32_t processID = ::getpid();; - int32_t txnid = txnID.id; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(truncTableStmt.fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(truncTableStmt.fSessionID); - CalpontSystemCatalog::TableInfo tableInfo; - uint64_t uniqueId = 0; - - //Bug 5070. Added exception handling - try - { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - Message::Args args; - Message message(9); - args.add(ex.what()); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - catch ( ... ) - { - Message::Args args; - Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; - } - - fWEClient->addQueue(uniqueId); - int pmNum = 1; - boost::shared_ptr bsIn; - string errorMsg; - uint32_t autoIncColOid = 0; - uint64_t tableLockId = 0; - OamCache* oamcache = OamCache::makeOamCache(); - std::vector moduleIds = oamcache->getModuleIds(); - - try - { - //check table lock - - CalpontSystemCatalog::TableName tableName; - tableName.schema = truncTableStmt.fTableName->fSchema; - tableName.table = truncTableStmt.fTableName->fName; - roPair = systemCatalogPtr->tableRID( tableName ); - int32_t sessionId = truncTableStmt.fSessionID; - std::string processName("DDLProc"); - int i = 0; - - std::vector pms; - - for (unsigned i = 0; i < moduleIds.size(); i++) - { - pms.push_back((uint32_t)moduleIds[i]); - } - - try - { - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if ( tableLockId == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - - try - { - processID = ::getpid(); - txnid = txnID.id; - sessionId = truncTableStmt.fSessionID; - processName = "DDLProc"; - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnid, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (tableLockId > 0) - break; - } - - if (i >= numTries) //error out - { - Message::Args args; - args.add(processName); - args.add((uint64_t)processID); - args.add(sessionId); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - } - } - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = truncTableStmt.fTableName->fSchema; - userTableName.table = truncTableStmt.fTableName->fName; - - tableColRidList = systemCatalogPtr->columnRIDs( userTableName ); - - dictOIDList = systemCatalogPtr->dictOIDs( userTableName ); - - for ( unsigned i = 0; i < tableColRidList.size(); i++ ) - { - if ( tableColRidList[i].objnum > 3000 ) - { - columnOidList.push_back( tableColRidList[i].objnum ); - allOidList.push_back( tableColRidList[i].objnum ); - } - } - - for ( unsigned i = 0; i < dictOIDList.size(); i++ ) - { - if ( dictOIDList[i].dictOID > 3000 ) - allOidList.push_back( dictOIDList[i].dictOID ); - } - - //Check whether the table has autoincrement column - tableInfo = systemCatalogPtr->tableInfo(userTableName); - } - catch (std::exception& ex) - { - cerr << "TruncateTableProcessor::processPackage: " << ex.what() << endl; - - Message::Args args; - Message message(9); - args.add("Truncate table failed: "); - args.add( ex.what() ); - args.add(""); - fSessionManager.rolledback(txnID); - - try - { - (void)fDbrm->releaseTableLock(tableLockId); - } - catch (std::exception&) - { - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - fWEClient->removeQueue(uniqueId); - message.format( args ); result.result = TRUNC_ERROR; result.message = message; - return result; - } - catch (...) - { - cerr << "TruncateTableProcessor::processPackage: caught unknown exception!" << endl; - - Message::Args args; - Message message(1); - args.add("Truncate table failed: "); - args.add( "encountered unkown exception" ); - args.add(""); - - try - { - (void)fDbrm->releaseTableLock(tableLockId); - } - catch (std::exception&) - { - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - fWEClient->removeQueue(uniqueId); - message.format( args ); - - result.result = TRUNC_ERROR; - result.message = message; - return result; - } - - //Save the oids to a file - try - { - createWriteTruncateTableLogFile( roPair.objnum, uniqueId, allOidList); - } - catch (std::exception& ex) - { - Message::Args args; - Message message(9); - args.add("Truncate table failed due to "); - args.add(ex.what()); + // rc = fSessionManager.setTableLock( roPair.objnum, truncTableStmt.fSessionID, processID, + // processName, false ); fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - message.format( args ); - - //@bug 4515 Release the tablelock as nothing has done to this table. - try - { - (void)fDbrm->releaseTableLock(tableLockId); - } - catch (std::exception&) {} - - result.result = TRUNC_ERROR; - result.message = message; return result; + } } - - ByteStream bytestream; - ByteStream::byte tmp8; - - // MCOL-66 The DBRM can't handle concurrent DDL - boost::mutex::scoped_lock lk(dbrmMutex); - - try + catch (runtime_error&) { - //Disable extents first - int rc1 = fDbrm->markAllPartitionForDeletion( allOidList); - - if (rc1 != 0) - { - string errMsg; - BRM::errString(rc, errMsg); - throw std::runtime_error(errMsg); - } - - // Bug 4208 Drop the PrimProcFDCache before droping the column files - // FOr Windows, this ensures (most likely) that the column files have - // no open handles to hinder the deletion of the files. - rc = cacheutils::dropPrimProcFdCache(); - - VERBOSE_INFO("Removing files"); - bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; - bytestream << uniqueId; - bytestream << (uint32_t) allOidList.size(); - - for (unsigned i = 0; i < allOidList.size(); i++) - { - bytestream << (uint32_t) allOidList[i]; - } - - uint32_t msgRecived = 0; - - try - { - fWEClient->write_to_all(bytestream); - - bsIn.reset(new ByteStream()); - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - fWEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } - } - } - catch (std::exception& ex) - { - Message::Args args; - Message message(9); - args.add("Truncate table failed due to "); - args.add(ex.what()); - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - message.format( args ); - - result.result = TRUNC_ERROR; - result.message = message; - deleteLogFile(TRUNCATE_LOG, roPair.objnum, uniqueId); - return result; - } - catch (...) - { - result.result = DROP_ERROR; - errorMsg = "Error in getting information from system catalog or from dbrm."; - Message::Args args; - Message message(9); - args.add("Truncate table failed due to "); - args.add(errorMsg); - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - message.format( args ); - - result.result = TRUNC_ERROR; - result.message = message; - deleteLogFile(TRUNCATE_LOG, roPair.objnum, uniqueId); - return result; - } - - //Drop PrimProc FD cache - rc = cacheutils::dropPrimProcFdCache(); - //Flush primProc cache - rc = cacheutils::flushOIDsFromCache( allOidList ); - //Delete extents from extent map - rc = fDbrm->deleteOIDs(allOidList); - - if (rc != 0) - { - Message::Args args; - Message message(1); - args.add("Table truncated with warning "); - args.add( "Remove from extent map failed." ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = WARNING; - result.message = message; - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - return result; - } - - //Get the number of tables in the database, the current table is included. - int tableCount = systemCatalogPtr->getTableCount(); - Oam oam; - //Calculate which dbroot the columns should start - DBRootConfigList dbRootList = oamcache->getDBRootNums(); - - uint16_t useDBRootIndex = tableCount % dbRootList.size(); - //Find out the dbroot# corresponding the useDBRootIndex from oam - uint16_t useDBRoot = dbRootList[useDBRootIndex]; - - bytestream.restart(); - bytestream << (ByteStream::byte) WE_SVR_WRITE_CREATETABLEFILES; - bytestream << uniqueId; - bytestream << (uint32_t)txnID.id; - uint32_t numOids = columnOidList.size() + dictOIDList.size(); - bytestream << numOids; - CalpontSystemCatalog::ColType colType; - - for (unsigned col = 0; col < columnOidList.size(); col++) - { - colType = systemCatalogPtr->colType(columnOidList[col]); - - if (colType.autoincrement) - autoIncColOid = colType.columnOID; - - bytestream << (uint32_t)columnOidList[col]; - bytestream << (uint8_t) colType.colDataType; - bytestream << (uint8_t) false; - bytestream << (uint32_t) colType.colWidth; - bytestream << (uint16_t) useDBRoot; - bytestream << (uint32_t) colType.compressionType; - } - - for (unsigned col = 0; col < dictOIDList.size(); col++) - { - colType = systemCatalogPtr->colTypeDct(dictOIDList[col].dictOID); - bytestream << (uint32_t) dictOIDList[col].dictOID; - bytestream << (uint8_t) colType.colDataType; - bytestream << (uint8_t) true; - bytestream << (uint32_t) colType.colWidth; - bytestream << (uint16_t) useDBRoot; - bytestream << (uint32_t) colType.compressionType; - } - - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[useDBRoot]; - - try - { -#ifdef IDB_DDL_DEBUG - cout << "Truncate table sending We_SVR_WRITE_CREATETABLEFILES to pm " << pmNum << endl; -#endif - fWEClient->write(bytestream, pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - } - - break; - } - } - - if (rc != 0) - { - //drop the newly created files - bytestream.restart(); - bytestream << (ByteStream::byte) WE_SVR_WRITE_DROPFILES; - bytestream << uniqueId; - bytestream << (uint32_t)(allOidList.size()); - - for (unsigned i = 0; i < (allOidList.size()); i++) - { - bytestream << (uint32_t)(allOidList[i]); - } - - fWEClient->write(bytestream, pmNum); - - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - break; - } - else - { - *bsIn >> tmp8; - //rc = tmp8; - break; - } - } - - Message::Args args; - Message message(1); - args.add( "Truncate table failed." ); - args.add( errorMsg); - args.add(""); - message.format( args ); - - result.result = TRUNC_ERROR; - result.message = message; - //rc = fSessionManager.setTableLock( roPair.objnum, truncTableStmt.fSessionID, processID, processName, false ); - fSessionManager.rolledback(txnID); - return result; - } - } - catch (runtime_error&) - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server"; - } + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server"; } + } #ifdef _MSC_VER - catch (std::exception&) - { - //FIXME: Windows can't delete a file that's still open by another process - } + catch (std::exception&) + { + // FIXME: Windows can't delete a file that's still open by another process + } #else - catch (std::exception& ex) - { - Message::Args args; - Message message(1); - args.add( "Truncate table failed." ); - args.add( ex.what() ); - args.add(""); - message.format( args ); + catch (std::exception& ex) + { + Message::Args args; + Message message(1); + args.add("Truncate table failed."); + args.add(ex.what()); + args.add(""); + message.format(args); - result.result = TRUNC_ERROR; - result.message = message; - //rc = fSessionManager.setTableLock( roPair.objnum, truncTableStmt.fSessionID, processID, processName, false ); - fSessionManager.rolledback(txnID); - return result; - } + result.result = TRUNC_ERROR; + result.message = message; + // rc = fSessionManager.setTableLock( roPair.objnum, truncTableStmt.fSessionID, processID, processName, + // false ); + fSessionManager.rolledback(txnID); + return result; + } #endif - catch ( ... ) - { - Message::Args args; - Message message(1); - args.add("Truncate table failed: "); - args.add( "Remove column files failed." ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = TRUNC_ERROR; - result.message = message; - //rc = fSessionManager.setTableLock( roPair.objnum, truncTableStmt.fSessionID, processID, processName, false ); - fSessionManager.rolledback(txnID); - return result; - } - - if (rc != 0) - { - rollBackTransaction( uniqueId, txnID, truncTableStmt.fSessionID); //What to do with the error code - fSessionManager.rolledback(txnID); - } - - //Check whether the table has autoincrement column - if (tableInfo.tablewithautoincr == 1) - { - //reset nextvalue to 1 - WE_DDLCommandClient commandClient; - rc = commandClient.UpdateSyscolumnNextval(autoIncColOid, 1); - } - - // Log the DDL statement - logDDL(truncTableStmt.fSessionID, txnID.id, truncTableStmt.fSql, truncTableStmt.fOwner); - - try - { - (void)fDbrm->releaseTableLock(tableLockId); - } - catch (std::exception&) - { - Message::Args args; - Message message(1); - args.add("Table truncated with warning "); - args.add( "Release table failed." ); - args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - args.add(""); - message.format( args ); - - result.result = WARNING; - result.message = message; - fSessionManager.rolledback(txnID); - fWEClient->removeQueue(uniqueId); - } - - //release the transaction - fSessionManager.committed(txnID); - fWEClient->removeQueue(uniqueId); - - //Remove the log file - try - { - deleteLogFile(TRUNCATE_LOG, roPair.objnum, uniqueId); - } - catch ( ... ) - { - } + catch (...) + { + Message::Args args; + Message message(1); + args.add("Truncate table failed: "); + args.add("Remove column files failed."); + args.add(""); + args.add(""); + message.format(args); + result.result = TRUNC_ERROR; + result.message = message; + // rc = fSessionManager.setTableLock( roPair.objnum, truncTableStmt.fSessionID, processID, processName, + // false ); + fSessionManager.rolledback(txnID); return result; + } + + if (rc != 0) + { + rollBackTransaction(uniqueId, txnID, truncTableStmt.fSessionID); // What to do with the error code + fSessionManager.rolledback(txnID); + } + + // Check whether the table has autoincrement column + if (tableInfo.tablewithautoincr == 1) + { + // reset nextvalue to 1 + WE_DDLCommandClient commandClient; + rc = commandClient.UpdateSyscolumnNextval(autoIncColOid, 1); + } + + // Log the DDL statement + logDDL(truncTableStmt.fSessionID, txnID.id, truncTableStmt.fSql, truncTableStmt.fOwner); + + try + { + (void)fDbrm->releaseTableLock(tableLockId); + } + catch (std::exception&) + { + Message::Args args; + Message message(1); + args.add("Table truncated with warning "); + args.add("Release table failed."); + args.add(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + args.add(""); + message.format(args); + + result.result = WARNING; + result.message = message; + fSessionManager.rolledback(txnID); + fWEClient->removeQueue(uniqueId); + } + + // release the transaction + fSessionManager.committed(txnID); + fWEClient->removeQueue(uniqueId); + + // Remove the log file + try + { + deleteLogFile(TRUNCATE_LOG, roPair.objnum, uniqueId); + } + catch (...) + { + } + + return result; } -} //namespace ddlpackageprocessor +} // namespace ddlpackageprocessor // vim:ts=4 sw=4: - diff --git a/dbcon/ddlpackageproc/droptableprocessor.h b/dbcon/ddlpackageproc/droptableprocessor.h index 4b7a3369b..bb358e87b 100644 --- a/dbcon/ddlpackageproc/droptableprocessor.h +++ b/dbcon/ddlpackageproc/droptableprocessor.h @@ -40,18 +40,18 @@ namespace ddlpackageprocessor */ class DropTableProcessor : public DDLPackageProcessor { -public: - DropTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process a drop table statement - * - * @param dropTableStmt the drop table statement - */ - EXPORT DDLResult processPackage(ddlpackage::DropTableStatement& dropTableStmt); - -protected: - -private: + public: + DropTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process a drop table statement + * + * @param dropTableStmt the drop table statement + */ + EXPORT DDLResult processPackage(ddlpackage::DropTableStatement& dropTableStmt); + protected: + private: }; /** @brief specialization of a DDLPacakageProcessor @@ -60,22 +60,22 @@ private: */ class TruncTableProcessor : public DDLPackageProcessor { -public: - TruncTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process a truncate table statement - * - * @param truncTableStmt the truncate table statement - */ - EXPORT DDLResult processPackage(ddlpackage::TruncTableStatement& truncTableStmt); - -protected: - -private: + public: + TruncTableProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process a truncate table statement + * + * @param truncTableStmt the truncate table statement + */ + EXPORT DDLResult processPackage(ddlpackage::TruncTableStatement& truncTableStmt); + protected: + private: }; -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor #undef EXPORT -#endif //DROPTABLEPROCESSOR_H +#endif // DROPTABLEPROCESSOR_H diff --git a/dbcon/ddlpackageproc/markpartitionprocessor.cpp b/dbcon/ddlpackageproc/markpartitionprocessor.cpp index 5613377ca..55487ce9d 100644 --- a/dbcon/ddlpackageproc/markpartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/markpartitionprocessor.cpp @@ -34,264 +34,266 @@ using namespace oam; namespace ddlpackageprocessor { - -MarkPartitionProcessor::DDLResult MarkPartitionProcessor::processPackage(ddlpackage::MarkPartitionStatement& markPartitionStmt) +MarkPartitionProcessor::DDLResult MarkPartitionProcessor::processPackage( + ddlpackage::MarkPartitionStatement& markPartitionStmt) { - SUMMARY_INFO("RestorePartitionProcessor::processPackage"); + SUMMARY_INFO("RestorePartitionProcessor::processPackage"); - DDLResult result; - result.result = NO_ERROR; - std::string err; - VERBOSE_INFO(markPartitionStmt); + DDLResult result; + result.result = NO_ERROR; + std::string err; + VERBOSE_INFO(markPartitionStmt); - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; - int rc = 0; - rc = fDbrm->isReadWrite(); + int rc = 0; + rc = fDbrm->isReadWrite(); - if (rc != 0 ) + if (rc != 0) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + std::vector oidList; + CalpontSystemCatalog::RIDList tableColRidList; + CalpontSystemCatalog::DictOIDList dictOIDList; + std::string processName("DDLProc"); + + string stmt = markPartitionStmt.fSql + "|" + markPartitionStmt.fTableName->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, markPartitionStmt.fSessionID, txnID.id); + + uint32_t processID = 0; + uint64_t uniqueID = 0; + uint32_t sessionID = markPartitionStmt.fSessionID; + execplan::CalpontSystemCatalog::ROPair roPair; + + try + { + // check table lock + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(markPartitionStmt.fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(markPartitionStmt.fSessionID); + CalpontSystemCatalog::TableName tableName; + tableName.schema = markPartitionStmt.fTableName->fSchema; + tableName.table = markPartitionStmt.fTableName->fName; + roPair = systemCatalogPtr->tableRID(tableName); + + //@Bug 3054 check for system catalog + if (roPair.objnum < 3000) { - logging::Message::Args args; - logging::Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; + throw std::runtime_error("Drop partition cannot be operated on Calpont system catalog."); } - std::vector oidList; - CalpontSystemCatalog::RIDList tableColRidList; - CalpontSystemCatalog::DictOIDList dictOIDList; - std::string processName("DDLProc"); + int i = 0; + processID = ::getpid(); + oam::OamCache* oamcache = OamCache::makeOamCache(); + std::vector pmList = oamcache->getModuleIds(); + std::vector pms; - string stmt = markPartitionStmt.fSql + "|" + markPartitionStmt.fTableName->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, markPartitionStmt.fSessionID, txnID.id); - - uint32_t processID = 0; - uint64_t uniqueID = 0; - uint32_t sessionID = markPartitionStmt.fSessionID; - execplan::CalpontSystemCatalog::ROPair roPair; + for (unsigned i = 0; i < pmList.size(); i++) + { + pms.push_back((uint32_t)pmList[i]); + } try { - //check table lock - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(markPartitionStmt.fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(markPartitionStmt.fSessionID); - CalpontSystemCatalog::TableName tableName; - tableName.schema = markPartitionStmt.fTableName->fSchema; - tableName.table = markPartitionStmt.fTableName->fName; - roPair = systemCatalogPtr->tableRID( tableName ); - - //@Bug 3054 check for system catalog - if ( roPair.objnum < 3000 ) - { - throw std::runtime_error("Drop partition cannot be operated on Calpont system catalog."); - } - - int i = 0; - processID = ::getpid(); - oam::OamCache* oamcache = OamCache::makeOamCache(); - std::vector pmList = oamcache->getModuleIds(); - std::vector pms; - - for (unsigned i = 0; i < pmList.size(); i++) - { - pms.push_back((uint32_t)pmList[i]); - } - - try - { - uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, (int32_t*)&txnID.id, BRM::LOADING ); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; - } - - if ( uniqueID == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - // reset - sessionID = markPartitionStmt.fSessionID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - processID = ::getpid(); - processName = "DDLProc"; - - try - { - uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, (int32_t*)&txnID.id, BRM::LOADING ); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; - } - - if (uniqueID > 0) - break; - } - - if (i >= numTries) //error out - { - result.result = DROP_ERROR; - logging::Message::Args args; - args.add(processName); - args.add((uint64_t)processID); - args.add((uint64_t)sessionID); - result.message = Message(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - fSessionManager.rolledback(txnID); - return result; - } - } - - // 1. Get the OIDs for the columns - // 2. Get the OIDs for the dictionaries - // 3. Save the OIDs to a log file - // 4. Remove the extents from extentmap - // 5. Flush PrimProc Cache - // 6. Remove the column and dictionary files for the partition - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = markPartitionStmt.fTableName->fSchema; - userTableName.table = markPartitionStmt.fTableName->fName; - - tableColRidList = systemCatalogPtr->columnRIDs( userTableName ); - - dictOIDList = systemCatalogPtr->dictOIDs( userTableName ); - - //Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format - for ( unsigned i = 0; i < tableColRidList.size(); i++ ) - { - if ( tableColRidList[i].objnum > 3000 ) - oidList.push_back( tableColRidList[i].objnum ); - } - - for ( unsigned i = 0; i < dictOIDList.size(); i++ ) - { - if ( dictOIDList[i].dictOID > 3000 ) - oidList.push_back( dictOIDList[i].dictOID ); - } - - //Remove the partition from extent map - string emsg; - rc = fDbrm->markPartitionForDeletion( oidList, markPartitionStmt.fPartitions, emsg); - - if ( rc != 0 ) - { - throw std::runtime_error(emsg); - } - } - catch (exception& ex) - { - logging::Message::Args args; - logging::Message message(ex.what()); - - if (rc == BRM::ERR_TABLE_NOT_LOCKED) - result.result = USER_ERROR; - else if (rc == BRM::ERR_PARTITION_DISABLED || rc == BRM::ERR_INVALID_OP_LAST_PARTITION || - rc == BRM::ERR_NOT_EXIST_PARTITION) - result.result = PARTITION_WARNING; - else if (rc == BRM::ERR_NO_PARTITION_PERFORMED) - result.result = WARN_NO_PARTITION; - else - result.result = DROP_ERROR; - - result.message = message; - - try - { - fDbrm->releaseTableLock(uniqueID); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - } - - fSessionManager.rolledback(txnID); - return result; - } - catch (...) - { - //cerr << "MarkPartitionProcessor::processPackage: caught unknown exception!" << endl; - - logging::Message::Args args; - logging::Message message(1); - args.add("Disable partition failed: "); - args.add( "encountered unkown exception" ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = DROP_ERROR; - result.message = message; - - try - { - fDbrm->releaseTableLock(uniqueID); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - } - - fSessionManager.rolledback(txnID); - return result; - } - - // Log the DDL statement - logging::logDDL(markPartitionStmt.fSessionID, 0, markPartitionStmt.fSql, markPartitionStmt.fOwner); - - try - { - fDbrm->releaseTableLock(uniqueID); + uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, + (int32_t*)&txnID.id, BRM::LOADING); } catch (std::exception&) { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; } - fSessionManager.committed(txnID); + if (uniqueID == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + // reset + sessionID = markPartitionStmt.fSessionID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + processID = ::getpid(); + processName = "DDLProc"; + + try + { + uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, + (int32_t*)&txnID.id, BRM::LOADING); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; + } + + if (uniqueID > 0) + break; + } + + if (i >= numTries) // error out + { + result.result = DROP_ERROR; + logging::Message::Args args; + args.add(processName); + args.add((uint64_t)processID); + args.add((uint64_t)sessionID); + result.message = Message(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + fSessionManager.rolledback(txnID); + return result; + } + } + + // 1. Get the OIDs for the columns + // 2. Get the OIDs for the dictionaries + // 3. Save the OIDs to a log file + // 4. Remove the extents from extentmap + // 5. Flush PrimProc Cache + // 6. Remove the column and dictionary files for the partition + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = markPartitionStmt.fTableName->fSchema; + userTableName.table = markPartitionStmt.fTableName->fName; + + tableColRidList = systemCatalogPtr->columnRIDs(userTableName); + + dictOIDList = systemCatalogPtr->dictOIDs(userTableName); + + // Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format + for (unsigned i = 0; i < tableColRidList.size(); i++) + { + if (tableColRidList[i].objnum > 3000) + oidList.push_back(tableColRidList[i].objnum); + } + + for (unsigned i = 0; i < dictOIDList.size(); i++) + { + if (dictOIDList[i].dictOID > 3000) + oidList.push_back(dictOIDList[i].dictOID); + } + + // Remove the partition from extent map + string emsg; + rc = fDbrm->markPartitionForDeletion(oidList, markPartitionStmt.fPartitions, emsg); + + if (rc != 0) + { + throw std::runtime_error(emsg); + } + } + catch (exception& ex) + { + logging::Message::Args args; + logging::Message message(ex.what()); + + if (rc == BRM::ERR_TABLE_NOT_LOCKED) + result.result = USER_ERROR; + else if (rc == BRM::ERR_PARTITION_DISABLED || rc == BRM::ERR_INVALID_OP_LAST_PARTITION || + rc == BRM::ERR_NOT_EXIST_PARTITION) + result.result = PARTITION_WARNING; + else if (rc == BRM::ERR_NO_PARTITION_PERFORMED) + result.result = WARN_NO_PARTITION; + else + result.result = DROP_ERROR; + + result.message = message; + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + } + + fSessionManager.rolledback(txnID); return result; + } + catch (...) + { + // cerr << "MarkPartitionProcessor::processPackage: caught unknown exception!" << endl; + + logging::Message::Args args; + logging::Message message(1); + args.add("Disable partition failed: "); + args.add("encountered unkown exception"); + args.add(""); + args.add(""); + message.format(args); + + result.result = DROP_ERROR; + result.message = message; + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + } + + fSessionManager.rolledback(txnID); + return result; + } + + // Log the DDL statement + logging::logDDL(markPartitionStmt.fSessionID, 0, markPartitionStmt.fSql, markPartitionStmt.fOwner); + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; + } + + fSessionManager.committed(txnID); + return result; } -} +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/markpartitionprocessor.h b/dbcon/ddlpackageproc/markpartitionprocessor.h index 411bf45b3..2bea225c4 100644 --- a/dbcon/ddlpackageproc/markpartitionprocessor.h +++ b/dbcon/ddlpackageproc/markpartitionprocessor.h @@ -34,30 +34,28 @@ namespace ddlpackageprocessor { - /** @brief specialization of a DDLPackageProcessor * for interacting with the Write Engine * to process create table ddl statements. */ class MarkPartitionProcessor : public DDLPackageProcessor { -public: - MarkPartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process a create table statement - * - * @param createTableStmt the CreateTableStatement - */ - EXPORT DDLResult processPackage(ddlpackage::MarkPartitionStatement& MarkPartitionStmt); - -protected: - - -private: + public: + MarkPartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process a create table statement + * + * @param createTableStmt the CreateTableStatement + */ + EXPORT DDLResult processPackage(ddlpackage::MarkPartitionStatement& MarkPartitionStmt); + protected: + private: }; -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor #undef EXPORT -#endif // MARKPARTITIONPROCESSOR_H +#endif // MARKPARTITIONPROCESSOR_H diff --git a/dbcon/ddlpackageproc/mydriver.cpp b/dbcon/ddlpackageproc/mydriver.cpp index f6427b43c..ed844df53 100644 --- a/dbcon/ddlpackageproc/mydriver.cpp +++ b/dbcon/ddlpackageproc/mydriver.cpp @@ -60,547 +60,579 @@ using namespace joblist; class PopulateIndexTest { -public: + public: + PopulateIndexTest(DistributedEngineComm* ec) : fEC(ec) + { + } + DistributedEngineComm* fEC; - PopulateIndexTest(DistributedEngineComm* ec) : fEC(ec) { } - DistributedEngineComm* fEC; + void test_createindex() + { + cout << "Begining create index test ... " << endl; + std::string sqlbuf = "CREATE INDEX test1_idx ON tpch.nation (n_nationkey)"; + cout << sqlbuf << endl; - void test_createindex() + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining create index test ... " << endl; - std::string sqlbuf = "CREATE INDEX test1_idx ON tpch.nation (n_nationkey)"; - cout << sqlbuf << endl; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + cout << "Parser succeeded." << endl; + cout << ptree.fList.size() << " " + << "SQL statements" << endl; + cout << ptree.fSqlText << endl; - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); + try + { + CreateIndexProcessor processor; + processor.setDebugLevel(CreateIndexProcessor::VERBOSE); + SqlStatement& stmt = *ptree.fList[0]; + CreateIndexProcessor::DDLResult result; + DISPLAY(stmt.fSessionID); - cout << "Parser succeeded." << endl; - cout << ptree.fList.size() << " " << "SQL statements" << endl; - cout << ptree.fSqlText << endl; + result = processor.processPackage(dynamic_cast(stmt)); - try - { - CreateIndexProcessor processor; - processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - SqlStatement& stmt = *ptree.fList[0]; - CreateIndexProcessor::DDLResult result; - DISPLAY(stmt.fSessionID); - - result = processor.processPackage(dynamic_cast(stmt)); - - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - throw; - } - } + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } + } - void test_createuniqueindex() + void test_createuniqueindex() + { + cout << "Begining create unique index test ..." << endl; + std::string sqlbuf = "CREATE UNIQUE INDEX test2_idx ON tpch.nation (n_name)"; + cout << sqlbuf << endl; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining create unique index test ..." << endl; - std::string sqlbuf = "CREATE UNIQUE INDEX test2_idx ON tpch.nation (n_name)"; - cout << sqlbuf << endl; + const ParseTree& ptree = parser.GetParseTree(); + cout << ptree.fSqlText << endl; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + CreateIndexProcessor processor; + processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - cout << ptree.fSqlText << endl; + SqlStatement& stmt = *ptree.fList[0]; + CreateIndexProcessor::DDLResult result; - try - { - CreateIndexProcessor processor; - processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - CreateIndexProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - throw; - } - } + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } + } - void test_createindextest(std::string& sqlbuf) + void test_createindextest(std::string& sqlbuf) + { + cout << "Begining create index test ..." << endl; + cout << sqlbuf << endl; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining create index test ..." << endl; - cout << sqlbuf << endl; + const ParseTree& ptree = parser.GetParseTree(); + cout << ptree.fSqlText << endl; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + CreateIndexProcessor processor; + processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - cout << ptree.fSqlText << endl; - - try - { - CreateIndexProcessor processor; - processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - CreateIndexProcessor::DDLResult result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + CreateIndexProcessor::DDLResult result = + processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } + } + void test_createtabletest(const string& sqlbuf) + { + cout << "Begining create table test: " << sqlbuf << endl; + SqlParser parser; + parser.Parse(sqlbuf.c_str()); - void test_createtabletest(const string& sqlbuf) + if (parser.Good()) { - cout << "Begining create table test: " << sqlbuf << endl; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + const ParseTree& ptree = parser.GetParseTree(); + cout << ptree.fSqlText << endl; - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - cout << ptree.fSqlText << endl; + try + { + CreateTableProcessor processor; + processor.setDebugLevel(CreateTableProcessor::VERBOSE); - try - { - CreateTableProcessor processor; - processor.setDebugLevel(CreateTableProcessor::VERBOSE); + SqlStatement& stmt = *ptree.fList[0]; + CreateTableProcessor::DDLResult result; - SqlStatement& stmt = *ptree.fList[0]; - CreateTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - throw; - } - } + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } + } - void test_altertable_addtableconstraint(std::string& sqlbuf) + void test_altertable_addtableconstraint(std::string& sqlbuf) + { + cout << "Begining Alter Table add table constraint test ... " << endl; + cout << sqlbuf << endl; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table add table constraint test ... " << endl; - cout << sqlbuf << endl; + const ParseTree& ptree = parser.GetParseTree(); + cout << ptree.fSqlText << endl; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - cout << ptree.fSqlText << endl; + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - throw; - } - } + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } + } - void test_altertable_addtablenullconstraint() + void test_altertable_addtablenullconstraint() + { + // sql syntax error? (Does not build index test.) + cout << "Begining Alter Table add table not null constraint test ... " << endl; + std::string sqlbuf = "ALTER TABLE tpch.region add CONSTRAINT not null(r_regionkey);"; + cout << sqlbuf << endl; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { -//sql syntax error? (Does not build index test.) - cout << "Begining Alter Table add table not null constraint test ... " << endl; - std::string sqlbuf = "ALTER TABLE tpch.region add CONSTRAINT not null(r_regionkey);"; - cout << sqlbuf << endl; + const ParseTree& ptree = parser.GetParseTree(); + cout << ptree.fSqlText << endl; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - cout << ptree.fSqlText << endl; + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - throw; - } - } + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } - - + } }; - -int main( int argc, char** argv) +int main(int argc, char** argv) { - int DoAll = 0; - int Do1 = 0; - int Do2 = 0; - int Do3 = 0; - int Do4 = 0; - int Do5 = 0; - int Do6 = 0; - int Do7 = 0; - int Do8 = 0; - int Do9 = 0; - int Do10 = 0; - int Do11 = 0; - int Do12 = 0; - int Do13 = 0; - int Do14 = 0; - int Do15 = 0; - int Do16 = 0; - int Do17 = 0; - int Do18 = 0; - int Do19 = 0; - int Do20 = 0; - int Do21 = 0; - int Do22 = 0; - int Do23 = 0; - int Do24 = 0; - int Do25 = 0; - int Do26 = 0; - int Do27 = 0; - int Do28 = 0; - int Do29 = 0; - int Do30 = 0; - int Do31 = 0; - int Do32 = 0; - int Do33 = 0; - int Do34 = 0; - int Do35 = 0; - int Do36 = 0; + int DoAll = 0; + int Do1 = 0; + int Do2 = 0; + int Do3 = 0; + int Do4 = 0; + int Do5 = 0; + int Do6 = 0; + int Do7 = 0; + int Do8 = 0; + int Do9 = 0; + int Do10 = 0; + int Do11 = 0; + int Do12 = 0; + int Do13 = 0; + int Do14 = 0; + int Do15 = 0; + int Do16 = 0; + int Do17 = 0; + int Do18 = 0; + int Do19 = 0; + int Do20 = 0; + int Do21 = 0; + int Do22 = 0; + int Do23 = 0; + int Do24 = 0; + int Do25 = 0; + int Do26 = 0; + int Do27 = 0; + int Do28 = 0; + int Do29 = 0; + int Do30 = 0; + int Do31 = 0; + int Do32 = 0; + int Do33 = 0; + int Do34 = 0; + int Do35 = 0; + int Do36 = 0; - cout << "Driver Test starting with " << argc << " parameters." << endl; + cout << "Driver Test starting with " << argc << " parameters." << endl; - for (int i = 0; i < argc; i++) - { - cout << "Arg " << i << ": " << argv[i] << endl; - } + for (int i = 0; i < argc; i++) + { + cout << "Arg " << i << ": " << argv[i] << endl; + } - if (argc > 1) - { - if (strcmp(argv[1], "All") == 0) DoAll = 1; - else if (strcmp(argv[1], "t1") == 0) Do1 = 1; - else if (strcmp(argv[1], "t2") == 0) Do2 = 1; - else if (strcmp(argv[1], "t3") == 0) Do3 = 1; - else if (strcmp(argv[1], "t4") == 0) Do4 = 1; - else if (strcmp(argv[1], "t5") == 0) Do5 = 1; - else if (strcmp(argv[1], "t6") == 0) Do6 = 1; - else if (strcmp(argv[1], "t7") == 0) Do7 = 1; - else if (strcmp(argv[1], "t8") == 0) Do8 = 1; - else if (strcmp(argv[1], "t9") == 0) Do9 = 1; - else if (strcmp(argv[1], "t10") == 0) Do10 = 1; - else if (strcmp(argv[1], "t11") == 0) Do11 = 1; - else if (strcmp(argv[1], "t12") == 0) Do12 = 1; - else if (strcmp(argv[1], "t13") == 0) Do13 = 1; - else if (strcmp(argv[1], "t14") == 0) Do14 = 1; - else if (strcmp(argv[1], "t15") == 0) Do15 = 1; - else if (strcmp(argv[1], "t16") == 0) Do16 = 1; - else if (strcmp(argv[1], "t17") == 0) Do17 = 1; - else if (strcmp(argv[1], "t18") == 0) Do18 = 1; - else if (strcmp(argv[1], "t19") == 0) Do19 = 1; - else if (strcmp(argv[1], "t20") == 0) Do20 = 1; - else if (strcmp(argv[1], "t21") == 0) Do21 = 1; - else if (strcmp(argv[1], "t22") == 0) Do22 = 1; - else if (strcmp(argv[1], "t23") == 0) Do23 = 1; - else if (strcmp(argv[1], "t24") == 0) Do24 = 1; - else if (strcmp(argv[1], "t25") == 0) Do25 = 1; - else if (strcmp(argv[1], "t26") == 0) Do26 = 1; - else if (strcmp(argv[1], "t27") == 0) Do27 = 1; - else if (strcmp(argv[1], "t28") == 0) Do28 = 1; - else if (strcmp(argv[1], "t29") == 0) Do29 = 1; - else if (strcmp(argv[1], "t30") == 0) Do30 = 1; - else if (strcmp(argv[1], "t31") == 0) Do31 = 1; - else if (strcmp(argv[1], "t32") == 0) Do32 = 1; - else if (strcmp(argv[1], "t33") == 0) Do33 = 1; - else if (strcmp(argv[1], "t34") == 0) Do34 = 1; - else if (strcmp(argv[1], "t35") == 0) Do35 = 1; - else if (strcmp(argv[1], "t36") == 0) Do35 = 1; + if (argc > 1) + { + if (strcmp(argv[1], "All") == 0) + DoAll = 1; + else if (strcmp(argv[1], "t1") == 0) + Do1 = 1; + else if (strcmp(argv[1], "t2") == 0) + Do2 = 1; + else if (strcmp(argv[1], "t3") == 0) + Do3 = 1; + else if (strcmp(argv[1], "t4") == 0) + Do4 = 1; + else if (strcmp(argv[1], "t5") == 0) + Do5 = 1; + else if (strcmp(argv[1], "t6") == 0) + Do6 = 1; + else if (strcmp(argv[1], "t7") == 0) + Do7 = 1; + else if (strcmp(argv[1], "t8") == 0) + Do8 = 1; + else if (strcmp(argv[1], "t9") == 0) + Do9 = 1; + else if (strcmp(argv[1], "t10") == 0) + Do10 = 1; + else if (strcmp(argv[1], "t11") == 0) + Do11 = 1; + else if (strcmp(argv[1], "t12") == 0) + Do12 = 1; + else if (strcmp(argv[1], "t13") == 0) + Do13 = 1; + else if (strcmp(argv[1], "t14") == 0) + Do14 = 1; + else if (strcmp(argv[1], "t15") == 0) + Do15 = 1; + else if (strcmp(argv[1], "t16") == 0) + Do16 = 1; + else if (strcmp(argv[1], "t17") == 0) + Do17 = 1; + else if (strcmp(argv[1], "t18") == 0) + Do18 = 1; + else if (strcmp(argv[1], "t19") == 0) + Do19 = 1; + else if (strcmp(argv[1], "t20") == 0) + Do20 = 1; + else if (strcmp(argv[1], "t21") == 0) + Do21 = 1; + else if (strcmp(argv[1], "t22") == 0) + Do22 = 1; + else if (strcmp(argv[1], "t23") == 0) + Do23 = 1; + else if (strcmp(argv[1], "t24") == 0) + Do24 = 1; + else if (strcmp(argv[1], "t25") == 0) + Do25 = 1; + else if (strcmp(argv[1], "t26") == 0) + Do26 = 1; + else if (strcmp(argv[1], "t27") == 0) + Do27 = 1; + else if (strcmp(argv[1], "t28") == 0) + Do28 = 1; + else if (strcmp(argv[1], "t29") == 0) + Do29 = 1; + else if (strcmp(argv[1], "t30") == 0) + Do30 = 1; + else if (strcmp(argv[1], "t31") == 0) + Do31 = 1; + else if (strcmp(argv[1], "t32") == 0) + Do32 = 1; + else if (strcmp(argv[1], "t33") == 0) + Do33 = 1; + else if (strcmp(argv[1], "t34") == 0) + Do34 = 1; + else if (strcmp(argv[1], "t35") == 0) + Do35 = 1; + else if (strcmp(argv[1], "t36") == 0) + Do35 = 1; + } + PopulateIndexTest pit(DistributedEngineComm::instance()); + boost::timer theTimer; - } + if (DoAll) + { + cout << "Starting all tests" << endl; - PopulateIndexTest pit(DistributedEngineComm::instance()); - boost::timer theTimer; + pit.test_createindex(); + pit.test_createuniqueindex(); + std::string altsql = "ALTER TABLE tpch.region add CONSTRAINT test1_cstr unique(r_name);"; + pit.test_altertable_addtableconstraint(altsql); + pit.test_altertable_addtablenullconstraint(); - if (DoAll) - { - cout << "Starting all tests" << endl; + cout << "Finished all tests" << endl; + } + else if (Do1) + { + pit.test_createindex(); - pit.test_createindex(); - pit.test_createuniqueindex(); - std::string altsql = "ALTER TABLE tpch.region add CONSTRAINT test1_cstr unique(r_name);"; - pit.test_altertable_addtableconstraint(altsql); - pit.test_altertable_addtablenullconstraint(); + cout << "Finished create index test" << endl; + } + else if (Do2) + { + pit.test_createuniqueindex(); - cout << "Finished all tests" << endl; - } - else if (Do1) - { - pit.test_createindex(); + cout << "Finished create unique index test" << endl; + } + else if (Do3) + { + std::string sqlbuf = "ALTER TABLE tpch.region add CONSTRAINT test1r5_cstr unique(r_name);"; + pit.test_altertable_addtableconstraint(sqlbuf); + cout << "Finished add table constraint test" << endl; + } + else if (Do4) + { + std::string sql("CREATE INDEX test4_idx ON tpch.part (p_size)"); + pit.test_createindextest(sql); - cout << "Finished create index test" << endl; - } - else if (Do2) - { - pit.test_createuniqueindex(); + cout << "Finished " << sql << endl; + } + else if (Do5) + { + std::string sql("CREATE INDEX test5_idx ON tpch.part (p_name)"); + pit.test_createindextest(sql); - cout << "Finished create unique index test" << endl; - } - else if (Do3) - { - std::string sqlbuf = "ALTER TABLE tpch.region add CONSTRAINT test1r5_cstr unique(r_name);"; - pit.test_altertable_addtableconstraint(sqlbuf); - cout << "Finished add table constraint test" << endl; - } - else if (Do4) - { - std::string sql("CREATE INDEX test4_idx ON tpch.part (p_size)"); - pit.test_createindextest(sql); + cout << "Finished " << sql << endl; + } + else if (Do6) + { + std::string sql("CREATE INDEX test6_idx ON tpch.orders (o_orderkey, o_custkey)"); + pit.test_createindextest(sql); - cout << "Finished " << sql << endl; - } - else if (Do5) - { - std::string sql("CREATE INDEX test5_idx ON tpch.part (p_name)"); - pit.test_createindextest(sql); + cout << "Finished " << sql << endl; + } + else if (Do7) + { + std::string sqlbuf = "ALTER TABLE tpch.supplier add CONSTRAINT tests2_cstr unique(s_name);"; + pit.test_altertable_addtableconstraint(sqlbuf); + cout << "Finished add table constraint test" << endl; + } + else if (Do8) + { + std::string sqlbuf = "ALTER TABLE tpch.partsupp add CONSTRAINT testps1_cstr unique(ps_partkey);"; + pit.test_altertable_addtableconstraint(sqlbuf); + cout << "Finished add table constraint test: should fail ps_partkey is not unique" << endl; + } + else if (Do9) + { + std::string sql("CREATE INDEX test7_idx ON tpch.customer (c_custkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do10) + { + std::string sql("CREATE INDEX test8_idx ON tpch.supplier(s_phone)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do11) + { + std::string sql("CREATE INDEX test9_idx ON tpch.part (p_retailprice)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do12) + { + std::string sql("CREATE INDEX test10_idx ON tpch.customer (c_acctbal)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do13) + { + std::string sql("CREATE UNIQUE INDEX test11_idx ON tpch.orders (o_clerk)"); + pit.test_createindextest(sql); + cout << "Finished add table index test: should fail" << endl; + } + else if (Do14) + { + std::string sql("CREATE INDEX test12_idx ON tpch.lineitem (l_returnflag)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do15) + { + std::string sql("CREATE INDEX test13_idx ON tpch.lineitem (l_linestatus)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do16) + { + std::string sql("CREATE INDEX multi_4idx ON tpch.region (r_regionkey, r_name)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do17) + { + std::string sql("CREATE INDEX multi_5idx ON tpch.orders (o_orderkey, o_orderstatus)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do18) + { + std::string sql("CREATE UNIQUE INDEX orderkey_idx ON tpch.orders (o_orderkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do19) + { + std::string sql("CREATE UNIQUE INDEX partkey_idx ON tpch.part (p_partkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do20) + { + std::string sql("CREATE INDEX lorderkey_idx ON tpch.lineitem (l_orderkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do21) + { + std::string sql("CREATE INDEX lpartkey1_idx ON tpch.lineitem (l_partkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do22) + { + std::string sql("CREATE INDEX suppkey1_idx ON tpch.lineitem (l_suppkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do23) + { + std::string sql("CREATE INDEX n_regionkey_id ON tpch.nation (n_regionkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do24) + { + std::string sql( + "CREATE INDEX multi_cust_idx ON tpch.customer (c_name, c_address, c_phone, c_mktsegment)"); + pit.test_createindextest(sql); + cout << "Finished add table index test: " << sql << endl; + } + else if (Do25) + { + std::string sql( + "CREATE INDEX multi_part_no_idx ON tpch.part (p_name, p_mfgr, p_brand, p_container, p_size)"); + pit.test_createindextest(sql); + cout << "Finished add table index test: " << sql << endl; + } + else if (Do26) + { + std::string sql("CREATE INDEX o_date_idx ON tpch.orders (o_orderdate)"); + pit.test_createindextest(sql); + cout << "Finished add table index test: " << sql << endl; + } + else if (Do27) + { + std::string sql("CREATE INDEX multi_order_idx ON tpch.orders (o_orderkey, o_orderstatus, o_orderdate)"); + pit.test_createindextest(sql); + cout << "Finished add table index test: " << sql << endl; + } + else if (Do28) + { + string sql("create table tpch.tablea( c1 integer, c2 char);"); + pit.test_createtabletest(sql); + std::string sqlbuf = "ALTER TABLE tpch.tablea add CONSTRAINT tablea_cstr1 unique(c2);"; + pit.test_altertable_addtableconstraint(sqlbuf); + cout << "Finished add table constraint test" << endl; + } + else if (Do29) + { + std::string sqlbuf = "ALTER TABLE tpch.nation add CONSTRAINT testn1_cstr unique(n_regionkey);"; + pit.test_altertable_addtableconstraint(sqlbuf); + cout << "Finished add table constraint test: should fail n_regionkey is not unique" << endl; + } + else if (Do30) + { + std::string sql("CREATE UNIQUE INDEX multicstr_1idx ON tpch.region (r_regionkey, r_name)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do31) + { + std::string sql("CREATE UNIQUE INDEX multicsto_1idx ON tpch.orders (o_orderkey, o_orderstatus)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do32) + { + std::string sql("CREATE UNIQUE INDEX multicstn_1idx ON tpch.nation (n_nationkey, n_regionkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do33) + { + std::string sql("CREATE UNIQUE INDEX multicstps_1idx ON tpch.partsupp (ps_partkey, ps_suppkey)"); + pit.test_createindextest(sql); + cout << "Finished add table index test" << endl; + } + else if (Do34) + { + std::string sql("CREATE UNIQUE INDEX multicsto_2idx ON tpch.orders (o_orderstatus, o_orderpriority)"); + pit.test_createindextest(sql); + cout << "Finished add table index test: should fail" << endl; + } + else if (Do35) + { + std::string sql("ALTER TABLE tpch.nation add CONSTRAINT testn2_cstr unique(n_nationkey);"); + pit.test_altertable_addtableconstraint(sql); + cout << "Finished add table constraint test" << endl; + } + else if (Do36) + { + pit.test_altertable_addtablenullconstraint(); - cout << "Finished " << sql << endl; - } - else if (Do6) - { - std::string sql("CREATE INDEX test6_idx ON tpch.orders (o_orderkey, o_custkey)"); - pit.test_createindextest(sql); - - cout << "Finished " << sql << endl; - } - else if (Do7) - { - std::string sqlbuf = "ALTER TABLE tpch.supplier add CONSTRAINT tests2_cstr unique(s_name);"; - pit.test_altertable_addtableconstraint(sqlbuf); - cout << "Finished add table constraint test" << endl; - } - else if (Do8) - { - std::string sqlbuf = "ALTER TABLE tpch.partsupp add CONSTRAINT testps1_cstr unique(ps_partkey);"; - pit.test_altertable_addtableconstraint(sqlbuf); - cout << "Finished add table constraint test: should fail ps_partkey is not unique" << endl; - } - else if (Do9) - { - std::string sql("CREATE INDEX test7_idx ON tpch.customer (c_custkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do10) - { - std::string sql("CREATE INDEX test8_idx ON tpch.supplier(s_phone)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do11) - { - std::string sql("CREATE INDEX test9_idx ON tpch.part (p_retailprice)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do12) - { - std::string sql("CREATE INDEX test10_idx ON tpch.customer (c_acctbal)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do13) - { - std::string sql("CREATE UNIQUE INDEX test11_idx ON tpch.orders (o_clerk)"); - pit.test_createindextest(sql); - cout << "Finished add table index test: should fail" << endl; - } - else if (Do14) - { - std::string sql("CREATE INDEX test12_idx ON tpch.lineitem (l_returnflag)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do15) - { - std::string sql("CREATE INDEX test13_idx ON tpch.lineitem (l_linestatus)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do16) - { - std::string sql("CREATE INDEX multi_4idx ON tpch.region (r_regionkey, r_name)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do17) - { - std::string sql("CREATE INDEX multi_5idx ON tpch.orders (o_orderkey, o_orderstatus)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do18) - { - std::string sql("CREATE UNIQUE INDEX orderkey_idx ON tpch.orders (o_orderkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do19) - { - std::string sql("CREATE UNIQUE INDEX partkey_idx ON tpch.part (p_partkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do20) - { - std::string sql("CREATE INDEX lorderkey_idx ON tpch.lineitem (l_orderkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do21) - { - std::string sql("CREATE INDEX lpartkey1_idx ON tpch.lineitem (l_partkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do22) - { - std::string sql("CREATE INDEX suppkey1_idx ON tpch.lineitem (l_suppkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do23) - { - std::string sql("CREATE INDEX n_regionkey_id ON tpch.nation (n_regionkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do24) - { - std::string sql("CREATE INDEX multi_cust_idx ON tpch.customer (c_name, c_address, c_phone, c_mktsegment)"); - pit.test_createindextest(sql); - cout << "Finished add table index test: " << sql << endl; - } - else if (Do25) - { - std::string sql("CREATE INDEX multi_part_no_idx ON tpch.part (p_name, p_mfgr, p_brand, p_container, p_size)"); - pit.test_createindextest(sql); - cout << "Finished add table index test: " << sql << endl; - } - else if (Do26) - { - std::string sql("CREATE INDEX o_date_idx ON tpch.orders (o_orderdate)"); - pit.test_createindextest(sql); - cout << "Finished add table index test: " << sql << endl; - } - else if (Do27) - { - std::string sql("CREATE INDEX multi_order_idx ON tpch.orders (o_orderkey, o_orderstatus, o_orderdate)"); - pit.test_createindextest(sql); - cout << "Finished add table index test: " << sql << endl; - } - else if (Do28) - { - string sql("create table tpch.tablea( c1 integer, c2 char);"); - pit.test_createtabletest(sql); - std::string sqlbuf = "ALTER TABLE tpch.tablea add CONSTRAINT tablea_cstr1 unique(c2);"; - pit.test_altertable_addtableconstraint(sqlbuf); - cout << "Finished add table constraint test" << endl; - - } - else if (Do29) - { - std::string sqlbuf = "ALTER TABLE tpch.nation add CONSTRAINT testn1_cstr unique(n_regionkey);"; - pit.test_altertable_addtableconstraint(sqlbuf); - cout << "Finished add table constraint test: should fail n_regionkey is not unique" << endl; - } - else if (Do30) - { - std::string sql("CREATE UNIQUE INDEX multicstr_1idx ON tpch.region (r_regionkey, r_name)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do31) - { - std::string sql("CREATE UNIQUE INDEX multicsto_1idx ON tpch.orders (o_orderkey, o_orderstatus)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do32) - { - std::string sql("CREATE UNIQUE INDEX multicstn_1idx ON tpch.nation (n_nationkey, n_regionkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do33) - { - std::string sql("CREATE UNIQUE INDEX multicstps_1idx ON tpch.partsupp (ps_partkey, ps_suppkey)"); - pit.test_createindextest(sql); - cout << "Finished add table index test" << endl; - } - else if (Do34) - { - std::string sql("CREATE UNIQUE INDEX multicsto_2idx ON tpch.orders (o_orderstatus, o_orderpriority)"); - pit.test_createindextest(sql); - cout << "Finished add table index test: should fail" << endl; - } - else if (Do35) - { - std::string sql("ALTER TABLE tpch.nation add CONSTRAINT testn2_cstr unique(n_nationkey);"); - pit.test_altertable_addtableconstraint(sql); - cout << "Finished add table constraint test" << endl; - } - else if (Do36) - { - pit.test_altertable_addtablenullconstraint(); - - cout << "Finished add table not null constraint test" << endl; - } - else - { - cout << "No Test Selected!" << endl << endl; - - cout << "All" << endl; - cout << "t1" << endl; - cout << "t2" << endl; - cout << "t3" << endl; - cout << "t4" << endl; - cout << endl; - } - - cout << "Create index test took :" << theTimer.elapsed() << " seconds to complete." << endl; + cout << "Finished add table not null constraint test" << endl; + } + else + { + cout << "No Test Selected!" << endl << endl; + cout << "All" << endl; + cout << "t1" << endl; + cout << "t2" << endl; + cout << "t3" << endl; + cout << "t4" << endl; + cout << endl; + } + cout << "Create index test took :" << theTimer.elapsed() << " seconds to complete." << endl; } - diff --git a/dbcon/ddlpackageproc/resource.h b/dbcon/ddlpackageproc/resource.h index 50f97d594..9aeb56a1b 100644 --- a/dbcon/ddlpackageproc/resource.h +++ b/dbcon/ddlpackageproc/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/dbcon/ddlpackageproc/restorepartitionprocessor.cpp b/dbcon/ddlpackageproc/restorepartitionprocessor.cpp index 181b313fa..cb535d302 100644 --- a/dbcon/ddlpackageproc/restorepartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/restorepartitionprocessor.cpp @@ -34,263 +34,266 @@ using namespace WriteEngine; namespace ddlpackageprocessor { - -RestorePartitionProcessor::DDLResult RestorePartitionProcessor::processPackage(ddlpackage::RestorePartitionStatement& restorePartitionStmt) +RestorePartitionProcessor::DDLResult RestorePartitionProcessor::processPackage( + ddlpackage::RestorePartitionStatement& restorePartitionStmt) { - SUMMARY_INFO("RestorePartitionProcessor::processPackage"); + SUMMARY_INFO("RestorePartitionProcessor::processPackage"); - DDLResult result; - result.result = NO_ERROR; - std::string err; - VERBOSE_INFO(restorePartitionStmt); + DDLResult result; + result.result = NO_ERROR; + std::string err; + VERBOSE_INFO(restorePartitionStmt); - BRM::TxnID txnID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; + BRM::TxnID txnID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; - int rc = 0; - rc = fDbrm->isReadWrite(); + int rc = 0; + rc = fDbrm->isReadWrite(); - if (rc != 0 ) + if (rc != 0) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unable to execute the statement due to DBRM is read only"); + message.format(args); + result.result = DROP_ERROR; + result.message = message; + fSessionManager.rolledback(txnID); + return result; + } + + std::vector oidList; + CalpontSystemCatalog::RIDList tableColRidList; + CalpontSystemCatalog::DictOIDList dictOIDList; + std::string processName("DDLProc"); + + string stmt = restorePartitionStmt.fSql + "|" + restorePartitionStmt.fTableName->fSchema + "|"; + SQLLogger logger(stmt, fDDLLoggingId, restorePartitionStmt.fSessionID, txnID.id); + + uint32_t processID = 0; + uint64_t uniqueID = 0; + uint32_t sessionID = restorePartitionStmt.fSessionID; + execplan::CalpontSystemCatalog::ROPair roPair; + + try + { + // check table lock + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(restorePartitionStmt.fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(restorePartitionStmt.fSessionID); + CalpontSystemCatalog::TableName tableName; + tableName.schema = restorePartitionStmt.fTableName->fSchema; + tableName.table = restorePartitionStmt.fTableName->fName; + roPair = systemCatalogPtr->tableRID(tableName); + + //@Bug 3054 check for system catalog + if (roPair.objnum < 3000) { - logging::Message::Args args; - logging::Message message(9); - args.add("Unable to execute the statement due to DBRM is read only"); - message.format(args); - result.result = DROP_ERROR; - result.message = message; - fSessionManager.rolledback(txnID); - return result; + throw std::runtime_error("Drop partition cannot be operated on Calpont system catalog."); } - std::vector oidList; - CalpontSystemCatalog::RIDList tableColRidList; - CalpontSystemCatalog::DictOIDList dictOIDList; - std::string processName("DDLProc"); + int i = 0; + processID = ::getpid(); + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + std::vector pmList = oamcache->getModuleIds(); + std::vector pms; - string stmt = restorePartitionStmt.fSql + "|" + restorePartitionStmt.fTableName->fSchema + "|"; - SQLLogger logger(stmt, fDDLLoggingId, restorePartitionStmt.fSessionID, txnID.id); - - uint32_t processID = 0; - uint64_t uniqueID = 0; - uint32_t sessionID = restorePartitionStmt.fSessionID; - execplan::CalpontSystemCatalog::ROPair roPair; + for (unsigned i = 0; i < pmList.size(); i++) + { + pms.push_back((uint32_t)pmList[i]); + } try { - //check table lock - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(restorePartitionStmt.fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(restorePartitionStmt.fSessionID); - CalpontSystemCatalog::TableName tableName; - tableName.schema = restorePartitionStmt.fTableName->fSchema; - tableName.table = restorePartitionStmt.fTableName->fName; - roPair = systemCatalogPtr->tableRID( tableName ); - - //@Bug 3054 check for system catalog - if ( roPair.objnum < 3000 ) - { - throw std::runtime_error("Drop partition cannot be operated on Calpont system catalog."); - } - - int i = 0; - processID = ::getpid(); - oam::OamCache* oamcache = oam::OamCache::makeOamCache(); - std::vector pmList = oamcache->getModuleIds(); - std::vector pms; - - for (unsigned i = 0; i < pmList.size(); i++) - { - pms.push_back((uint32_t)pmList[i]); - } - - try - { - uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, (int32_t*)&txnID.id, BRM::LOADING ); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; - } - - if ( uniqueID == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - // reset - sessionID = restorePartitionStmt.fSessionID; - txnID.id = fTxnid.id; - txnID.valid = fTxnid.valid; - processID = ::getpid(); - processName = "DDLProc"; - - try - { - uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, (int32_t*)&txnID.id, BRM::LOADING ); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; - } - - if (uniqueID > 0) - break; - } - - if (i >= numTries) //error out - { - result.result = DROP_ERROR; - logging::Message::Args args; - string strOp("restore partition"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add((uint64_t)sessionID); - result.message = Message(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - fSessionManager.rolledback(txnID); - return result; - } - } - - // 1. Get the OIDs for the columns - // 2. Get the OIDs for the dictionaries - // 3. Save the OIDs to a log file - // 4. Remove the extents from extentmap - // 5. Flush PrimProc Cache - // 6. Remove the column and dictionary files for the partition - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = restorePartitionStmt.fTableName->fSchema; - userTableName.table = restorePartitionStmt.fTableName->fName; - - tableColRidList = systemCatalogPtr->columnRIDs( userTableName ); - - dictOIDList = systemCatalogPtr->dictOIDs( userTableName ); - - //Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format - for ( unsigned i = 0; i < tableColRidList.size(); i++ ) - { - if ( tableColRidList[i].objnum > 3000 ) - oidList.push_back( tableColRidList[i].objnum ); - } - - for ( unsigned i = 0; i < dictOIDList.size(); i++ ) - { - if ( dictOIDList[i].dictOID > 3000 ) - oidList.push_back( dictOIDList[i].dictOID ); - } - - //Remove the partition from extent map - string emsg; - rc = fDbrm->restorePartition( oidList, restorePartitionStmt.fPartitions, emsg); - - if ( rc != 0 ) - { - throw std::runtime_error(emsg); - } - } - catch (exception& ex) - { - logging::Message::Args args; - logging::Message message(ex.what()); - - if (( rc == BRM::ERR_NOT_EXIST_PARTITION) || (rc == BRM::ERR_INVALID_OP_LAST_PARTITION) || - (rc == BRM::ERR_PARTITION_DISABLED) || (rc == BRM::ERR_TABLE_NOT_LOCKED)) - result.result = USER_ERROR; - else if (rc == BRM::ERR_PARTITION_ENABLED) - result.result = PARTITION_WARNING; - else - result.result = DROP_ERROR; - - result.message = message; - - try - { - fDbrm->releaseTableLock(uniqueID); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - } - - fSessionManager.rolledback(txnID); - return result; - } - catch (...) - { - //cerr << "RestorePartitionProcessor::processPackage: caught unknown exception!" << endl; - - logging::Message::Args args; - logging::Message message(1); - args.add("Enable partition: "); - args.add( "encountered unkown exception" ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = DROP_ERROR; - result.message = message; - - try - { - fDbrm->releaseTableLock(uniqueID); - } - catch (std::exception&) - { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - } - - fSessionManager.rolledback(txnID); - return result; - } - - // Log the DDL statement - logging::logDDL(restorePartitionStmt.fSessionID, txnID.id, restorePartitionStmt.fSql, restorePartitionStmt.fOwner); - - try - { - fDbrm->releaseTableLock(uniqueID); + uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, + (int32_t*)&txnID.id, BRM::LOADING); } catch (std::exception&) { - result.result = DROP_ERROR; - result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - fSessionManager.rolledback(txnID); - return result; + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; } - fSessionManager.committed(txnID); + if (uniqueID == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + // reset + sessionID = restorePartitionStmt.fSessionID; + txnID.id = fTxnid.id; + txnID.valid = fTxnid.valid; + processID = ::getpid(); + processName = "DDLProc"; + + try + { + uniqueID = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, (int32_t*)&sessionID, + (int32_t*)&txnID.id, BRM::LOADING); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; + } + + if (uniqueID > 0) + break; + } + + if (i >= numTries) // error out + { + result.result = DROP_ERROR; + logging::Message::Args args; + string strOp("restore partition"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add((uint64_t)sessionID); + result.message = Message(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + fSessionManager.rolledback(txnID); + return result; + } + } + + // 1. Get the OIDs for the columns + // 2. Get the OIDs for the dictionaries + // 3. Save the OIDs to a log file + // 4. Remove the extents from extentmap + // 5. Flush PrimProc Cache + // 6. Remove the column and dictionary files for the partition + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = restorePartitionStmt.fTableName->fSchema; + userTableName.table = restorePartitionStmt.fTableName->fName; + + tableColRidList = systemCatalogPtr->columnRIDs(userTableName); + + dictOIDList = systemCatalogPtr->dictOIDs(userTableName); + + // Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format + for (unsigned i = 0; i < tableColRidList.size(); i++) + { + if (tableColRidList[i].objnum > 3000) + oidList.push_back(tableColRidList[i].objnum); + } + + for (unsigned i = 0; i < dictOIDList.size(); i++) + { + if (dictOIDList[i].dictOID > 3000) + oidList.push_back(dictOIDList[i].dictOID); + } + + // Remove the partition from extent map + string emsg; + rc = fDbrm->restorePartition(oidList, restorePartitionStmt.fPartitions, emsg); + + if (rc != 0) + { + throw std::runtime_error(emsg); + } + } + catch (exception& ex) + { + logging::Message::Args args; + logging::Message message(ex.what()); + + if ((rc == BRM::ERR_NOT_EXIST_PARTITION) || (rc == BRM::ERR_INVALID_OP_LAST_PARTITION) || + (rc == BRM::ERR_PARTITION_DISABLED) || (rc == BRM::ERR_TABLE_NOT_LOCKED)) + result.result = USER_ERROR; + else if (rc == BRM::ERR_PARTITION_ENABLED) + result.result = PARTITION_WARNING; + else + result.result = DROP_ERROR; + + result.message = message; + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + } + + fSessionManager.rolledback(txnID); return result; + } + catch (...) + { + // cerr << "RestorePartitionProcessor::processPackage: caught unknown exception!" << endl; + + logging::Message::Args args; + logging::Message message(1); + args.add("Enable partition: "); + args.add("encountered unkown exception"); + args.add(""); + args.add(""); + message.format(args); + + result.result = DROP_ERROR; + result.message = message; + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + } + + fSessionManager.rolledback(txnID); + return result; + } + + // Log the DDL statement + logging::logDDL(restorePartitionStmt.fSessionID, txnID.id, restorePartitionStmt.fSql, + restorePartitionStmt.fOwner); + + try + { + fDbrm->releaseTableLock(uniqueID); + } + catch (std::exception&) + { + result.result = DROP_ERROR; + result.message = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + fSessionManager.rolledback(txnID); + return result; + } + + fSessionManager.committed(txnID); + return result; } -} +} // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/restorepartitionprocessor.h b/dbcon/ddlpackageproc/restorepartitionprocessor.h index a61849836..06a4f2b1f 100644 --- a/dbcon/ddlpackageproc/restorepartitionprocessor.h +++ b/dbcon/ddlpackageproc/restorepartitionprocessor.h @@ -40,21 +40,21 @@ namespace ddlpackageprocessor */ class RestorePartitionProcessor : public DDLPackageProcessor { -public: - RestorePartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) {} - /** @brief process a drop table statement - * - * @param dropTableStmt the drop table statement - */ - EXPORT DDLResult processPackage(ddlpackage::RestorePartitionStatement& RestorePartitionStmt); - -protected: - -private: + public: + RestorePartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) + { + } + /** @brief process a drop table statement + * + * @param dropTableStmt the drop table statement + */ + EXPORT DDLResult processPackage(ddlpackage::RestorePartitionStatement& RestorePartitionStmt); + protected: + private: }; -} // namespace ddlpackageprocessor +} // namespace ddlpackageprocessor #undef EXPORT -#endif //RESTOREPARTITIONEPROCESSOR_H +#endif // RESTOREPARTITIONEPROCESSOR_H diff --git a/dbcon/ddlpackageproc/tdriver.cpp b/dbcon/ddlpackageproc/tdriver.cpp index 4684aac8a..316d3e4ed 100644 --- a/dbcon/ddlpackageproc/tdriver.cpp +++ b/dbcon/ddlpackageproc/tdriver.cpp @@ -55,1086 +55,1086 @@ using namespace WriteEngine; #include "messagelog.h" class SystemCatalogBuilder { -public: - static void build() - { - ColumnOp colOp; - Column curCol; - WriteEngine::WriteEngineWrapper fWriteEngine; - WriteEngine::TxnID txnID = 0; - int rc; - - remove(); - - colOp.initColumn( curCol ); - // SYSTABLE - - cout << "Creating System Catalog..." << endl; - - // TableName - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1001 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2001, 2002, 2003); - // Schema - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1002 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2004, 2005, 2006); - // CreateDate - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1003 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastUpdateDate - rc = colOp.createColumn( curCol, 3, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1004 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // INIT - rc = colOp.createColumn( curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1005 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // NEXT - rc = colOp.createColumn( curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1006 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //SYSCOLUMN - // Shema - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1007 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2007, 2008, 2009); - - // TableName - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1008 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2010, 2011, 2012); - - // ColumnName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1009 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2013, 2014, 2015); - // ObjectID - rc = colOp.createColumn( curCol, 3, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1010 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DictOID - rc = colOp.createColumn( curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1011 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ListOID - rc = colOp.createColumn( curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1012 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // TreeOID - rc = colOp.createColumn( curCol, 6, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1013 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DataType - rc = colOp.createColumn( curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1014 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ColumnLength - rc = colOp.createColumn( curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1015 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ColumnPos - rc = colOp.createColumn( curCol, 9, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1016 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastUpdate - rc = colOp.createColumn( curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1017 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DefaultValue - rc = colOp.createColumn( curCol, 11, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1018 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2016, 2017, 2018); - // Nullable - rc = colOp.createColumn( curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1019 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // Scale - rc = colOp.createColumn( curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1020 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // Precision - rc = colOp.createColumn( curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1021 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // AutoInc - rc = colOp.createColumn( curCol, 15, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1022 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // SYSCONSTRAINT - - // ConstraintName - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1023 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2019, 2020, 2021); - // Schema - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1024 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2022, 2023, 2024); - // TableName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1025 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2025, 2026, 2027); - - // ConstraintType - rc = colOp.createColumn( curCol, 3, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1026 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ConstraintPrim - rc = colOp.createColumn( curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1027 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2028, 2029, 2030); - - // ConstraintText - rc = colOp.createColumn( curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1028 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2031, 2032, 2033); - - // ConstraintStatus - rc = colOp.createColumn( curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1029 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2034, 2035, 2036); - - // IndexName - rc = colOp.createColumn( curCol, 7, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1030 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2037, 2038, 2039); - - //SYSCONSTRAINTCOL - // Schema - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1031 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2040, 2041, 2042); - - // TableName - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1032 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2043, 2044, 2045); - - // ColumnName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1033 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2046, 2047, 2048); - - // ConstraintName - rc = colOp.createColumn( curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1034 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2049, 2050, 2051); - - // SYSINDEX - // Schema - rc = colOp.createColumn( curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1035 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2052, 2053, 2054); - - //TableName - rc = colOp.createColumn( curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1036 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2055, 2056, 2057); - - // IndexName - rc = colOp.createColumn( curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1037 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2058, 2059, 2060); - - // ListOID - rc = colOp.createColumn( curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1038 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // TreeOID - rc = colOp.createColumn( curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1039 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + public: + static void build() + { + ColumnOp colOp; + Column curCol; + WriteEngine::WriteEngineWrapper fWriteEngine; + WriteEngine::TxnID txnID = 0; + int rc; + + remove(); + + colOp.initColumn(curCol); + // SYSTABLE + + cout << "Creating System Catalog..." << endl; + + // TableName + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1001); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2001, 2002, 2003); + // Schema + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1002); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2004, 2005, 2006); + // CreateDate + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1003); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastUpdateDate + rc = colOp.createColumn(curCol, 3, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1004); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // INIT + rc = colOp.createColumn(curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1005); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // NEXT + rc = colOp.createColumn(curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1006); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // SYSCOLUMN + // Shema + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1007); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2007, 2008, 2009); + + // TableName + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1008); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2010, 2011, 2012); + + // ColumnName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1009); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2013, 2014, 2015); + // ObjectID + rc = colOp.createColumn(curCol, 3, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1010); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DictOID + rc = colOp.createColumn(curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1011); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ListOID + rc = colOp.createColumn(curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1012); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // TreeOID + rc = colOp.createColumn(curCol, 6, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1013); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DataType + rc = colOp.createColumn(curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1014); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ColumnLength + rc = colOp.createColumn(curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1015); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ColumnPos + rc = colOp.createColumn(curCol, 9, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1016); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastUpdate + rc = colOp.createColumn(curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1017); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DefaultValue + rc = colOp.createColumn(curCol, 11, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1018); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2016, 2017, 2018); + // Nullable + rc = colOp.createColumn(curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1019); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Scale + rc = colOp.createColumn(curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1020); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Precision + rc = colOp.createColumn(curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1021); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // AutoInc + rc = colOp.createColumn(curCol, 15, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1022); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // SYSCONSTRAINT + + // ConstraintName + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1023); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2019, 2020, 2021); + // Schema + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1024); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2022, 2023, 2024); + // TableName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1025); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2025, 2026, 2027); + + // ConstraintType + rc = colOp.createColumn(curCol, 3, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1026); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ConstraintPrim + rc = colOp.createColumn(curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1027); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2028, 2029, 2030); + + // ConstraintText + rc = colOp.createColumn(curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1028); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2031, 2032, 2033); + + // ConstraintStatus + rc = colOp.createColumn(curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1029); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2034, 2035, 2036); + + // IndexName + rc = colOp.createColumn(curCol, 7, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1030); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2037, 2038, 2039); + + // SYSCONSTRAINTCOL + // Schema + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1031); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2040, 2041, 2042); + + // TableName + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1032); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2043, 2044, 2045); + + // ColumnName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1033); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2046, 2047, 2048); + + // ConstraintName + rc = colOp.createColumn(curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1034); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2049, 2050, 2051); + + // SYSINDEX + // Schema + rc = colOp.createColumn(curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1035); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2052, 2053, 2054); + + // TableName + rc = colOp.createColumn(curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1036); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2055, 2056, 2057); + + // IndexName + rc = colOp.createColumn(curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1037); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2058, 2059, 2060); + + // ListOID + rc = colOp.createColumn(curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1038); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // TreeOID + rc = colOp.createColumn(curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1039); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // IndexType + rc = colOp.createColumn(curCol, 9, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1040); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // CreateDate + rc = colOp.createColumn(curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1041); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastUpdateDate + rc = colOp.createColumn(curCol, 11, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1042); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // RecordCount + rc = colOp.createColumn(curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1043); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // TreeLevel + rc = colOp.createColumn(curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1044); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LeafCount + rc = colOp.createColumn(curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1045); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DistinctKeys + rc = colOp.createColumn(curCol, 15, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1046); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LeafBlocks + rc = colOp.createColumn(curCol, 16, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1047); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // AvgLeafCount + rc = colOp.createColumn(curCol, 17, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1048); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // AvgDataBlock + rc = colOp.createColumn(curCol, 18, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1049); + CPPUNIT_ASSERT(rc == NO_ERROR); - // IndexType - rc = colOp.createColumn( curCol, 9, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1040 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // SampleSize + rc = colOp.createColumn(curCol, 19, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1050); + CPPUNIT_ASSERT(rc == NO_ERROR); - // CreateDate - rc = colOp.createColumn( curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1041 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastUpdateDate - rc = colOp.createColumn( curCol, 11, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1042 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // RecordCount - rc = colOp.createColumn( curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1043 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // TreeLevel - rc = colOp.createColumn( curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1044 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LeafCount - rc = colOp.createColumn( curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1045 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DistinctKeys - rc = colOp.createColumn( curCol, 15, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1046 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LeafBlocks - rc = colOp.createColumn( curCol, 16, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1047 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // AvgLeafCount - rc = colOp.createColumn( curCol, 17, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1048 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // AvgDataBlock - rc = colOp.createColumn( curCol, 18, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1049 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // SampleSize - rc = colOp.createColumn( curCol, 19, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1050 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ClusterFactor - rc = colOp.createColumn( curCol, 20, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1051 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastAnalysisDate - rc = colOp.createColumn( curCol, 21, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1052 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // SYSINDEXCOL - // Schema - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1053 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2061, 2062, 2063); - - // TableName - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1054 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2064, 2065, 2066); - - // ColumnName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1055 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2067, 2068, 2069); - - // IndexName - rc = colOp.createColumn( curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1056 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2070, 2071, 2072); - - // ColumnPos - rc = colOp.createColumn( curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1057 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - static void remove() - { - ColumnOp colOp; - - for ( int i = 1001; i <= 1057; i++ ) - colOp.deleteFile( i ); - } + // ClusterFactor + rc = colOp.createColumn(curCol, 20, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1051); + CPPUNIT_ASSERT(rc == NO_ERROR); + // LastAnalysisDate + rc = colOp.createColumn(curCol, 21, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1052); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // SYSINDEXCOL + // Schema + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1053); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2061, 2062, 2063); + + // TableName + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1054); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2064, 2065, 2066); + + // ColumnName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1055); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2067, 2068, 2069); + + // IndexName + rc = colOp.createColumn(curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1056); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2070, 2071, 2072); + + // ColumnPos + rc = colOp.createColumn(curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1057); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + static void remove() + { + ColumnOp colOp; + + for (int i = 1001; i <= 1057; i++) + colOp.deleteFile(i); + } }; void destroySemaphores() { - key_t semkey; - int sems, err; + key_t semkey; + int sems, err; - semkey = 0x2149bdd2; - sems = semget(semkey, 2, 0666); + semkey = 0x2149bdd2; + sems = semget(semkey, 2, 0666); - if (sems != -1) - { - err = semctl(sems, 0, IPC_RMID); + if (sems != -1) + { + err = semctl(sems, 0, IPC_RMID); - if (err == -1) - perror("tdriver: semctl"); - } + if (err == -1) + perror("tdriver: semctl"); + } } - - void destroyShmseg() { - key_t shmkey; - int shms, err; + key_t shmkey; + int shms, err; - shmkey = 0x2149bdd2; - shms = shmget(shmkey, 0, 0666); + shmkey = 0x2149bdd2; + shms = shmget(shmkey, 0, 0666); - if (shms != -1) + if (shms != -1) + { + err = shmctl(shms, IPC_RMID, NULL); + + if (err == -1 && errno != EINVAL) { - err = shmctl(shms, IPC_RMID, NULL); - - if (err == -1 && errno != EINVAL) - { - perror("tdriver: shmctl"); - return; - } + perror("tdriver: shmctl"); + return; } + } } void setUp() { - destroySemaphores(); - destroyShmseg(); - unlink("/tmp/oidbitmap"); - SystemCatalogBuilder::build(); + destroySemaphores(); + destroyShmseg(); + unlink("/tmp/oidbitmap"); + SystemCatalogBuilder::build(); } void tearDown() { - destroySemaphores(); - destroyShmseg(); - unlink("/tmp/oidbitmap"); + destroySemaphores(); + destroyShmseg(); + unlink("/tmp/oidbitmap"); } class DDLPackageProcessorTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(DDLPackageProcessorTest); + CPPUNIT_TEST_SUITE(DDLPackageProcessorTest); - /* - CPPUNIT_TEST( test_createtable ); - CPPUNIT_TEST( test_createtable_region ); - CPPUNIT_TEST( test_createindex ); - CPPUNIT_TEST( test_createuniqueindex ); + /* + CPPUNIT_TEST( test_createtable ); + CPPUNIT_TEST( test_createtable_region ); + CPPUNIT_TEST( test_createindex ); + CPPUNIT_TEST( test_createuniqueindex ); - CPPUNIT_TEST( test_altertable_renamecolumn ); - CPPUNIT_TEST( test_altertable_renamecolumnwithcons ); + CPPUNIT_TEST( test_altertable_renamecolumn ); + CPPUNIT_TEST( test_altertable_renamecolumnwithcons ); - CPPUNIT_TEST( test_altertable_addacolumn ); - CPPUNIT_TEST( test_altertable_addacolumnconstrain ); - CPPUNIT_TEST( test_altertable_addacolumnunique ); - CPPUNIT_TEST( test_altertable_dropacolumn ); - CPPUNIT_TEST( test_altertable_dropcolumns ); - CPPUNIT_TEST( test_altertable_addcolumns ); - CPPUNIT_TEST( test_altertable_addtableconstraint ); - CPPUNIT_TEST( test_altertable_droptableconstraint ); - CPPUNIT_TEST( test_altertable_setcolumndefault ); - CPPUNIT_TEST( test_altertable_dropcolumndefault ); - CPPUNIT_TEST( test_altertable_renametable ); - CPPUNIT_TEST( test_dropindex ); - CPPUNIT_TEST( test_droptable ); */ + CPPUNIT_TEST( test_altertable_addacolumn ); + CPPUNIT_TEST( test_altertable_addacolumnconstrain ); + CPPUNIT_TEST( test_altertable_addacolumnunique ); + CPPUNIT_TEST( test_altertable_dropacolumn ); + CPPUNIT_TEST( test_altertable_dropcolumns ); + CPPUNIT_TEST( test_altertable_addcolumns ); + CPPUNIT_TEST( test_altertable_addtableconstraint ); + CPPUNIT_TEST( test_altertable_droptableconstraint ); + CPPUNIT_TEST( test_altertable_setcolumndefault ); + CPPUNIT_TEST( test_altertable_dropcolumndefault ); + CPPUNIT_TEST( test_altertable_renametable ); + CPPUNIT_TEST( test_dropindex ); + CPPUNIT_TEST( test_droptable ); */ - CPPUNIT_TEST_SUITE_END(); -private: + CPPUNIT_TEST_SUITE_END(); -public: + private: + public: + /* + * destroySemaphores() and destroyShmseg() will print error messages + * if there are no objects to destroy. That's OK. + */ + void test_createtable_region() + { + cout << "Begining create region table testing..." << endl; + std::string sqlbuf = + "create table region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; + SqlParser parser; + parser.Parse(sqlbuf.c_str()); - /* - * destroySemaphores() and destroyShmseg() will print error messages - * if there are no objects to destroy. That's OK. - */ - void test_createtable_region() + if (parser.Good()) { - cout << "Begining create region table testing..." << endl; - std::string sqlbuf = "create table region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + const ParseTree& ptree = parser.GetParseTree(); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); + try + { + CreateTableProcessor processor; + processor.setDebugLevel(CreateTableProcessor::VERBOSE); - try - { - CreateTableProcessor processor; - processor.setDebugLevel(CreateTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - CreateTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + CreateTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_createtable() + tearDown(); + } + + void test_createtable() + { + // setUp(); + // removeSystemCatalog(); + // createSystemCatalog(); + cout << "Begining create table testing..." << endl; + std::string sqlbuf = + "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) default 'helloworld', p_mfgr " + "char(6), p_brand char(10) , p_type varchar(25) default 'foobar' , p_size integer , p_container " + "char(10) ,p_retailprice integer , p_comment varchar(23), CONSTRAINT PK_PART PRIMARY KEY(p_partkey) " + ")"; + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - //setUp(); - //removeSystemCatalog(); - //createSystemCatalog(); - cout << "Begining create table testing..." << endl; - std::string sqlbuf = "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) default 'helloworld', p_mfgr char(6), p_brand char(10) , p_type varchar(25) default 'foobar' , p_size integer , p_container char(10) ,p_retailprice integer , p_comment varchar(23), CONSTRAINT PK_PART PRIMARY KEY(p_partkey) )"; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + const ParseTree& ptree = parser.GetParseTree(); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); + try + { + CreateTableProcessor processor; + processor.setDebugLevel(CreateTableProcessor::VERBOSE); - try - { - CreateTableProcessor processor; - processor.setDebugLevel(CreateTableProcessor::VERBOSE); + SqlStatement& stmt = *ptree.fList[0]; - SqlStatement& stmt = *ptree.fList[0]; + CreateTableStatement& ct = dynamic_cast(stmt); - CreateTableStatement& ct = dynamic_cast(stmt); + cout << "Parsed CreateTable:" << endl; + cout << ct << endl; - cout << "Parsed CreateTable:" << endl; - cout << ct << endl; - - CreateTableProcessor::DDLResult result; - - result = processor.processPackage(ct); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + CreateTableProcessor::DDLResult result; + result = processor.processPackage(ct); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_createindex() + tearDown(); + } + + void test_createindex() + { + cout << "Begining create index test ... " << endl; + std::string sqlbuf = "CREATE INDEX test_idx ON tpch.part (p_size)"; + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining create index test ... " << endl; - std::string sqlbuf = "CREATE INDEX test_idx ON tpch.part (p_size)"; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + const ParseTree& ptree = parser.GetParseTree(); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); + try + { + CreateIndexProcessor processor; + processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - try - { - CreateIndexProcessor processor; - processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - CreateIndexProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + CreateIndexProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_createuniqueindex() + tearDown(); + } + + void test_createuniqueindex() + { + cout << "Begining create unique index test ..." << endl; + std::string sqlbuf = "CREATE UNIQUE INDEX test_idx ON tpch.part (p_mfgr)"; + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining create unique index test ..." << endl; - std::string sqlbuf = "CREATE UNIQUE INDEX test_idx ON tpch.part (p_mfgr)"; - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + const ParseTree& ptree = parser.GetParseTree(); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); + try + { + CreateIndexProcessor processor; + processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - try - { - CreateIndexProcessor processor; - processor.setDebugLevel(CreateIndexProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - CreateIndexProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } - - tearDown(); - } - void test_altertable_addacolumn() - { - cout << "Begoning Alter Table Add column test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part ADD COLUMN c3 char(50)"; - - SqlParser parser; - parser.Parse(sqlbuf.c_str()); - - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + CreateIndexProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_addacolumnconstrain() + tearDown(); + } + void test_altertable_addacolumn() + { + cout << "Begoning Alter Table Add column test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part ADD COLUMN c3 char(50)"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table add column constraint test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part ADD COLUMN c3 char(50) NOT NULL"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_addacolumnunique() + tearDown(); + } + + void test_altertable_addacolumnconstrain() + { + cout << "Begining Alter Table add column constraint test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part ADD COLUMN c3 char(50) NOT NULL"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table add column with constraint test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part ADD COLUMN c3 char(50) UNIQUE"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } - - tearDown(); - } - void test_altertable_dropacolumn() - { - cout << "Begining Alter Table drop a column test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part DROP p_size "; - - SqlParser parser; - parser.Parse(sqlbuf.c_str()); - - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_dropcolumns() + tearDown(); + } + + void test_altertable_addacolumnunique() + { + cout << "Begining Alter Table add column with constraint test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part ADD COLUMN c3 char(50) UNIQUE"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table drop columns test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part DROP p_size, DROP p_type "; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_addcolumns() + tearDown(); + } + void test_altertable_dropacolumn() + { + cout << "Begining Alter Table drop a column test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part DROP p_size "; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table add columns test ..." << endl; - std::string sqlbuf = "ALTER TABLE test ADD COLUMN c3 char(50), ADD COLUMN c4 char(10), ADD COLUMN C5 int;"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } - - tearDown(); - } - void test_dropindex() - { - cout << "Begining Alter Table drop index test ..." << endl; - std::string sqlbuf = "drop index tpch.test_idx"; - - SqlParser parser; - parser.Parse(sqlbuf.c_str()); - - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - DropIndexProcessor processor; - processor.setDebugLevel(DropIndexProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - DropIndexProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } - - tearDown(); - } - void test_altertable_renamecolumn() - { - cout << "Begining Alter Table rename a column ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part RENAME COLUMN p_size TO size;"; - - SqlParser parser; - parser.Parse(sqlbuf.c_str()); - - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } + tearDown(); + } - void test_altertable_renamecolumnwithcons() + void test_altertable_dropcolumns() + { + cout << "Begining Alter Table drop columns test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part DROP p_size, DROP p_type "; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table rename a column with constraints ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part RENAME COLUMN p_partkey TO partlkey;"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } - - tearDown(); - } - void test_altertable_addtableconstraint() - { - cout << "Begining Alter Table add table constraint test ... " << endl; - std::string sqlbuf = "ALTER TABLE tpch.part add CONSTRAINT unique(p_type);"; - - SqlParser parser; - parser.Parse(sqlbuf.c_str()); - - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_droptableconstraint() + tearDown(); + } + + void test_altertable_addcolumns() + { + cout << "Begining Alter Table add columns test ..." << endl; + std::string sqlbuf = + "ALTER TABLE test ADD COLUMN c3 char(50), ADD COLUMN c4 char(10), ADD COLUMN C5 int;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table drop table constraint test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part drop constraint pk_tpch cascade;"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_setcolumndefault() + tearDown(); + } + void test_dropindex() + { + cout << "Begining Alter Table drop index test ..." << endl; + std::string sqlbuf = "drop index tpch.test_idx"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table set column default test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part alter p_partkey set default 3 ;"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + DropIndexProcessor processor; + processor.setDebugLevel(DropIndexProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } - - tearDown(); - } - void test_altertable_dropcolumndefault() - { - cout << "Begining Alter Table drop column default test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.part ALTER COLUMN p_type DROP DEFAULT; ;"; - - SqlParser parser; - parser.Parse(sqlbuf.c_str()); - - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + DropIndexProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_altertable_renametable() + tearDown(); + } + void test_altertable_renamecolumn() + { + cout << "Begining Alter Table rename a column ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part RENAME COLUMN p_size TO size;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining Alter Table rename a table test ..." << endl; - std::string sqlbuf = "ALTER TABLE tpch.region rename to tpch.ready ;"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - AlterTableProcessor processor; - processor.setDebugLevel(AlterTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - AlterTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } - void test_droptable() + tearDown(); + } + + void test_altertable_renamecolumnwithcons() + { + cout << "Begining Alter Table rename a column with constraints ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part RENAME COLUMN p_partkey TO partlkey;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) { - cout << "Begining drop table test ..." << endl; - std::string sqlbuf = "drop table tpch.part"; + const ParseTree& ptree = parser.GetParseTree(); - SqlParser parser; - parser.Parse(sqlbuf.c_str()); + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); - if (parser.Good()) - { - const ParseTree& ptree = parser.GetParseTree(); - - try - { - DropTableProcessor processor; - processor.setDebugLevel(DropTableProcessor::VERBOSE); - - SqlStatement& stmt = *ptree.fList[0]; - DropTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - tearDown(); - throw; - } - } + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { tearDown(); + throw; + } } + tearDown(); + } + void test_altertable_addtableconstraint() + { + cout << "Begining Alter Table add table constraint test ... " << endl; + std::string sqlbuf = "ALTER TABLE tpch.part add CONSTRAINT unique(p_type);"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); + + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); + + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + tearDown(); + throw; + } + } + + tearDown(); + } + + void test_altertable_droptableconstraint() + { + cout << "Begining Alter Table drop table constraint test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part drop constraint pk_tpch cascade;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); + + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); + + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + tearDown(); + throw; + } + } + + tearDown(); + } + + void test_altertable_setcolumndefault() + { + cout << "Begining Alter Table set column default test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part alter p_partkey set default 3 ;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); + + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); + + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + tearDown(); + throw; + } + } + + tearDown(); + } + void test_altertable_dropcolumndefault() + { + cout << "Begining Alter Table drop column default test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.part ALTER COLUMN p_type DROP DEFAULT; ;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); + + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); + + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + tearDown(); + throw; + } + } + + tearDown(); + } + + void test_altertable_renametable() + { + cout << "Begining Alter Table rename a table test ..." << endl; + std::string sqlbuf = "ALTER TABLE tpch.region rename to tpch.ready ;"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); + + try + { + AlterTableProcessor processor; + processor.setDebugLevel(AlterTableProcessor::VERBOSE); + + SqlStatement& stmt = *ptree.fList[0]; + AlterTableProcessor::DDLResult result; + + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + tearDown(); + throw; + } + } + + tearDown(); + } + + void test_droptable() + { + cout << "Begining drop table test ..." << endl; + std::string sqlbuf = "drop table tpch.part"; + + SqlParser parser; + parser.Parse(sqlbuf.c_str()); + + if (parser.Good()) + { + const ParseTree& ptree = parser.GetParseTree(); + + try + { + DropTableProcessor processor; + processor.setDebugLevel(DropTableProcessor::VERBOSE); + + SqlStatement& stmt = *ptree.fList[0]; + DropTableProcessor::DDLResult result; + + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + tearDown(); + throw; + } + } + + tearDown(); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( DDLPackageProcessorTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DDLPackageProcessorTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - // Uncomment before running tests - //setUp(); - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + // Uncomment before running tests + // setUp(); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/dmlpackage/calpontdmlfactory.cpp b/dbcon/dmlpackage/calpontdmlfactory.cpp index c16800d59..ab4531206 100644 --- a/dbcon/dmlpackage/calpontdmlfactory.cpp +++ b/dbcon/dmlpackage/calpontdmlfactory.cpp @@ -46,183 +46,183 @@ namespace dmlpackage { boost::mutex CalpontDMLFactory::fParserLock; -dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackage(dmlpackage::VendorDMLStatement& vpackage, - std::string defaultSchema /*= ""*/) +dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackage( + dmlpackage::VendorDMLStatement& vpackage, std::string defaultSchema /*= ""*/) { - CalpontDMLPackage* packagePtr = 0; + CalpontDMLPackage* packagePtr = 0; - try + try + { + std::string dmlStatement = vpackage.get_DMLStatement(); + //@Bug 2680. DMLParser is not thread safe. + boost::mutex::scoped_lock lk(fParserLock); + DMLParser parser; + + if (defaultSchema.size()) { - std::string dmlStatement = vpackage.get_DMLStatement(); - //@Bug 2680. DMLParser is not thread safe. - boost::mutex::scoped_lock lk(fParserLock); - DMLParser parser; - - if (defaultSchema.size()) - { - parser.setDefaultSchema(defaultSchema); - } - - parser.parse(dmlStatement.c_str()); - - if (parser.good()) - { - - const ParseTree& ptree = parser.getParseTree(); - SqlStatement* statementPtr = ptree[0]; - - int dmlStatementType = statementPtr->getStatementType(); - - switch (dmlStatementType) - { - case DML_INSERT: - packagePtr = new InsertDMLPackage(statementPtr->getSchemaName(), statementPtr->getTableName(), - ptree.fSqlText, vpackage.get_SessionID() ); - packagePtr->set_SQLStatement(dmlStatement); - (void)packagePtr->buildFromSqlStatement(*statementPtr); - break; - - case DML_UPDATE: - packagePtr = new UpdateDMLPackage(statementPtr->getSchemaName(), statementPtr->getTableName(), - ptree.fSqlText, vpackage.get_SessionID() ); - packagePtr->set_SQLStatement(dmlStatement); - (void)packagePtr->buildFromSqlStatement(*statementPtr); - break; - - case DML_DELETE: - packagePtr = new DeleteDMLPackage(statementPtr->getSchemaName(), statementPtr->getTableName(), - ptree.fSqlText, vpackage.get_SessionID() ); - packagePtr->set_SQLStatement(dmlStatement); - (void)packagePtr->buildFromSqlStatement(*statementPtr); - break; - - case DML_COMMAND: - packagePtr = new CommandDMLPackage(ptree.fSqlText, vpackage.get_SessionID()); - (void)packagePtr->buildFromSqlStatement(*statementPtr); - break; - - default: - cerr << "makeCalpontDMLPackage: invalid statement type" << endl; - break; - - } - - } - } - catch (std::exception& ex) - { - cerr << "makeCalpontDMLPackage:" << ex.what() << endl; - } - catch (...) - { - cerr << "makeCalpontDMLPackage: caught unknown exception!" << endl; + parser.setDefaultSchema(defaultSchema); } - return packagePtr; + parser.parse(dmlStatement.c_str()); + if (parser.good()) + { + const ParseTree& ptree = parser.getParseTree(); + SqlStatement* statementPtr = ptree[0]; + + int dmlStatementType = statementPtr->getStatementType(); + + switch (dmlStatementType) + { + case DML_INSERT: + packagePtr = new InsertDMLPackage(statementPtr->getSchemaName(), statementPtr->getTableName(), + ptree.fSqlText, vpackage.get_SessionID()); + packagePtr->set_SQLStatement(dmlStatement); + (void)packagePtr->buildFromSqlStatement(*statementPtr); + break; + + case DML_UPDATE: + packagePtr = new UpdateDMLPackage(statementPtr->getSchemaName(), statementPtr->getTableName(), + ptree.fSqlText, vpackage.get_SessionID()); + packagePtr->set_SQLStatement(dmlStatement); + (void)packagePtr->buildFromSqlStatement(*statementPtr); + break; + + case DML_DELETE: + packagePtr = new DeleteDMLPackage(statementPtr->getSchemaName(), statementPtr->getTableName(), + ptree.fSqlText, vpackage.get_SessionID()); + packagePtr->set_SQLStatement(dmlStatement); + (void)packagePtr->buildFromSqlStatement(*statementPtr); + break; + + case DML_COMMAND: + packagePtr = new CommandDMLPackage(ptree.fSqlText, vpackage.get_SessionID()); + (void)packagePtr->buildFromSqlStatement(*statementPtr); + break; + + default: cerr << "makeCalpontDMLPackage: invalid statement type" << endl; break; + } + } + } + catch (std::exception& ex) + { + cerr << "makeCalpontDMLPackage:" << ex.what() << endl; + } + catch (...) + { + cerr << "makeCalpontDMLPackage: caught unknown exception!" << endl; + } + + return packagePtr; } -dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromBuffer(dmlpackage::VendorDMLStatement& vpackage) +dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromBuffer( + dmlpackage::VendorDMLStatement& vpackage) { - CalpontDMLPackage* packagePtr = 0; + CalpontDMLPackage* packagePtr = 0; - try + try + { + int dmlStatementType = vpackage.get_DMLStatementType(); + + switch (dmlStatementType) { - int dmlStatementType = vpackage.get_DMLStatementType(); + case DML_INSERT: + packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), + vpackage.get_DMLStatement(), vpackage.get_SessionID()); + (void)packagePtr->buildFromBuffer(vpackage.get_DataBuffer(), vpackage.get_Columns(), + vpackage.get_Rows()); + break; - switch (dmlStatementType) - { - case DML_INSERT: - packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID()); - (void)packagePtr->buildFromBuffer(vpackage.get_DataBuffer - (), vpackage.get_Columns(), vpackage.get_Rows()); - break; + case DML_UPDATE: + packagePtr = new UpdateDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), + vpackage.get_DMLStatement(), vpackage.get_SessionID()); + (void)packagePtr->buildFromBuffer(vpackage.get_DataBuffer(), vpackage.get_Columns(), + vpackage.get_Rows()); + break; - case DML_UPDATE: - packagePtr = new UpdateDMLPackage(vpackage.get_SchemaName(), - vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID()); - (void)packagePtr->buildFromBuffer(vpackage.get_DataBuffer - (), vpackage.get_Columns(), vpackage.get_Rows()); - break; + case DML_DELETE: + packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), + vpackage.get_DMLStatement(), vpackage.get_SessionID()); + (void)packagePtr->buildFromBuffer(vpackage.get_DataBuffer(), vpackage.get_Columns(), + vpackage.get_Rows()); + break; - case DML_DELETE: - packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), - vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID()); - (void)packagePtr->buildFromBuffer(vpackage.get_DataBuffer - (), vpackage.get_Columns(), vpackage.get_Rows()); - break; + case DML_COMMAND: + packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID()); - case DML_COMMAND: - packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID() ); + break; - break; - - default: - cerr << "makeCalpontDMLPackage: invalid statement type" << endl; - break; - } - } - catch (std::exception& ex) - { - cerr << "makeCalpontDMLPackage:" << ex.what() << endl; - } - catch (...) - { - cerr << "makeCalpontDMLPackage: caught unknown exception!" << endl; + default: cerr << "makeCalpontDMLPackage: invalid statement type" << endl; break; } + } + catch (std::exception& ex) + { + cerr << "makeCalpontDMLPackage:" << ex.what() << endl; + } + catch (...) + { + cerr << "makeCalpontDMLPackage: caught unknown exception!" << endl; + } - return packagePtr; + return packagePtr; } -dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlpackage::VendorDMLStatement& vpackage) +dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer( + dmlpackage::VendorDMLStatement& vpackage) { - CalpontDMLPackage* packagePtr = 0; + CalpontDMLPackage* packagePtr = 0; - try + try + { + int dmlStatementType = vpackage.get_DMLStatementType(); + + switch (dmlStatementType) { - int dmlStatementType = vpackage.get_DMLStatementType(); + case DML_INSERT: + packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), + vpackage.get_DMLStatement(), vpackage.get_SessionID()); + (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), + vpackage.get_Columns(), vpackage.get_Rows(), + vpackage.get_nullValues()); + break; - switch (dmlStatementType) - { - case DML_INSERT: - packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID()); - (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues()); - break; + case DML_COMMAND: + packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID()); + break; - case DML_COMMAND: - packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID() ); - break; + case DML_DELETE: + packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), + vpackage.get_DMLStatement(), vpackage.get_SessionID()); + (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), + vpackage.get_Columns(), vpackage.get_Rows(), + vpackage.get_nullValues()); + break; - case DML_DELETE: - packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), - vpackage.get_DMLStatement(), vpackage.get_SessionID() ); - (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues()); - break; - - default: - cerr << "makeCalpontDMLPackage: invalid statement type" << endl; - break; - } - } - catch (std::exception& ex) - { - cerr << "makeCalpontDMLPackage:" << ex.what() << endl; - } - catch (...) - { - cerr << "makeCalpontDMLPackage: caught unknown exception!" << endl; + default: cerr << "makeCalpontDMLPackage: invalid statement type" << endl; break; } + } + catch (std::exception& ex) + { + cerr << "makeCalpontDMLPackage:" << ex.what() << endl; + } + catch (...) + { + cerr << "makeCalpontDMLPackage: caught unknown exception!" << endl; + } - return packagePtr; + return packagePtr; } -dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontUpdatePackageFromMysqlBuffer(dmlpackage::VendorDMLStatement& vpackage, dmlpackage::UpdateSqlStatement& updateStmt) +dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontUpdatePackageFromMysqlBuffer( + dmlpackage::VendorDMLStatement& vpackage, dmlpackage::UpdateSqlStatement& updateStmt) { - CalpontDMLPackage* packagePtr = new UpdateDMLPackage((updateStmt.fNamePtr)->fSchema, (updateStmt.fNamePtr)->fName, - vpackage.get_DMLStatement(), vpackage.get_SessionID() ); - UpdateDMLPackage* updatePkgPtr = dynamic_cast(packagePtr); - updatePkgPtr->buildUpdateFromMysqlBuffer(updateStmt); - packagePtr = dynamic_cast(updatePkgPtr); - return packagePtr; + CalpontDMLPackage* packagePtr = + new UpdateDMLPackage((updateStmt.fNamePtr)->fSchema, (updateStmt.fNamePtr)->fName, + vpackage.get_DMLStatement(), vpackage.get_SessionID()); + UpdateDMLPackage* updatePkgPtr = dynamic_cast(packagePtr); + updatePkgPtr->buildUpdateFromMysqlBuffer(updateStmt); + packagePtr = dynamic_cast(updatePkgPtr); + return packagePtr; } -} //namespace dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/calpontdmlfactory.h b/dbcon/dmlpackage/calpontdmlfactory.h index 78a5bb76f..5c311ce5a 100644 --- a/dbcon/dmlpackage/calpontdmlfactory.h +++ b/dbcon/dmlpackage/calpontdmlfactory.h @@ -36,40 +36,40 @@ #endif namespace dmlpackage { - class CalpontDMLFactory { - /** @brief a concrete implementation responsible for creating - * the proper concrete realization of a CalpontDMLPackage - * given a VendorDMLStatement. - */ -public: + /** @brief a concrete implementation responsible for creating + * the proper concrete realization of a CalpontDMLPackage + * given a VendorDMLStatement. + */ + public: + /** @brief factory method + * + * @param vpackage the VendorDMLStatement + * @param defaultSchema the default schema to be used for DML statements + */ + EXPORT static dmlpackage::CalpontDMLPackage* makeCalpontDMLPackage(dmlpackage::VendorDMLStatement& vpackage, + std::string defaultSchema = ""); - /** @brief factory method - * - * @param vpackage the VendorDMLStatement - * @param defaultSchema the default schema to be used for DML statements - */ - EXPORT static dmlpackage::CalpontDMLPackage* makeCalpontDMLPackage (dmlpackage::VendorDMLStatement& vpackage, - std::string defaultSchema = "" ); + /** @brief old factory method! + * + * @param vpackage the VendorDMLStatement + */ + EXPORT static dmlpackage::CalpontDMLPackage* makeCalpontDMLPackageFromBuffer( + dmlpackage::VendorDMLStatement& vpackage); - /** @brief old factory method! - * - * @param vpackage the VendorDMLStatement - */ - EXPORT static dmlpackage::CalpontDMLPackage* makeCalpontDMLPackageFromBuffer(dmlpackage::VendorDMLStatement& vpackage); + EXPORT static dmlpackage::CalpontDMLPackage* makeCalpontDMLPackageFromMysqlBuffer( + dmlpackage::VendorDMLStatement& vpackage); + static dmlpackage::CalpontDMLPackage* makeCalpontUpdatePackageFromMysqlBuffer( + dmlpackage::VendorDMLStatement& vpackage, dmlpackage::UpdateSqlStatement& updateStmt); - EXPORT static dmlpackage::CalpontDMLPackage* makeCalpontDMLPackageFromMysqlBuffer(dmlpackage::VendorDMLStatement& vpackage); - static dmlpackage::CalpontDMLPackage* makeCalpontUpdatePackageFromMysqlBuffer(dmlpackage::VendorDMLStatement& vpackage, dmlpackage::UpdateSqlStatement& updateStmt); - -protected: - -private: - static boost::mutex fParserLock; + protected: + private: + static boost::mutex fParserLock; }; -} +} // namespace dmlpackage #undef EXPORT -#endif //CALPONTDMLFACTORY_H +#endif // CALPONTDMLFACTORY_H diff --git a/dbcon/dmlpackage/calpontdmlpackage.cpp b/dbcon/dmlpackage/calpontdmlpackage.cpp index 51554362c..37cd1e486 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.cpp +++ b/dbcon/dmlpackage/calpontdmlpackage.cpp @@ -31,59 +31,75 @@ namespace dmlpackage */ CalpontDMLPackage::CalpontDMLPackage() - : fPlan(new messageqcpp::ByteStream()), - fTable(0), fHasFilter(0), fLogging(true), fIsInsertSelect(false), - fIsBatchInsert(false), fIsCacheInsert(false), fIsAutocommitOn(false), fIsWarnToError(false), fTableOid(0) + : fPlan(new messageqcpp::ByteStream()) + , fTable(0) + , fHasFilter(0) + , fLogging(true) + , fIsInsertSelect(false) + , fIsBatchInsert(false) + , fIsCacheInsert(false) + , fIsAutocommitOn(false) + , fIsWarnToError(false) + , fTableOid(0) { - } -CalpontDMLPackage::CalpontDMLPackage( std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID ) - : fSchemaName(schemaName), fTableName( tableName ), fDMLStatement( dmlStatement ), - fSessionID(sessionID), fPlan(new messageqcpp::ByteStream()), fTable(0), fHasFilter(false), fLogging(true), fIsInsertSelect(false), - fIsBatchInsert(false), fIsCacheInsert(false), fIsAutocommitOn(false), fIsWarnToError(false), fTableOid(0) +CalpontDMLPackage::CalpontDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID) + : fSchemaName(schemaName) + , fTableName(tableName) + , fDMLStatement(dmlStatement) + , fSessionID(sessionID) + , fPlan(new messageqcpp::ByteStream()) + , fTable(0) + , fHasFilter(false) + , fLogging(true) + , fIsInsertSelect(false) + , fIsBatchInsert(false) + , fIsCacheInsert(false) + , fIsAutocommitOn(false) + , fIsWarnToError(false) + , fTableOid(0) { - } CalpontDMLPackage::~CalpontDMLPackage() { - if ( 0 != fTable ) - delete fTable; + if (0 != fTable) + delete fTable; } /* * strip off whitespaces from a string */ -std::string CalpontDMLPackage::StripLeadingWhitespace( std::string value ) +std::string CalpontDMLPackage::StripLeadingWhitespace(std::string value) { - for (;;) + for (;;) + { + string::size_type pos = value.find(' ', 0); + + if (pos == 0) { - string::size_type pos = value.find (' ', 0); - - if (pos == 0) - { - value = value.substr (pos + 1, 10000); - } - else - { - // no more whitespace - break; - } + value = value.substr(pos + 1, 10000); } + else + { + // no more whitespace + break; + } + } - return value; + return value; } void CalpontDMLPackage::initializeTable() { - if (0 == fTable) - { - fTable = new DMLTable(); - fTable->set_SchemaName(fSchemaName); - fTable->set_TableName(fTableName); - } + if (0 == fTable) + { + fTable = new DMLTable(); + fTable->set_SchemaName(fSchemaName); + fTable->set_TableName(fTableName); + } } -} // namespace dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/calpontdmlpackage.h b/dbcon/dmlpackage/calpontdmlpackage.h index af25e2524..2f43fc885 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.h +++ b/dbcon/dmlpackage/calpontdmlpackage.h @@ -41,357 +41,355 @@ namespace dmlpackage */ class CalpontDMLPackage { + public: + /** @brief ctor + */ + CalpontDMLPackage(); -public: - /** @brief ctor - */ - CalpontDMLPackage(); + /** @brief ctor + * + * @param schemaName the schema of the table being operated on + * @param tableName the name of the table being operated on + * @param dmlStatement the dml statement + * @param sessionID the session id + */ + CalpontDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, int sessionID); - /** @brief ctor - * - * @param schemaName the schema of the table being operated on - * @param tableName the name of the table being operated on - * @param dmlStatement the dml statement - * @param sessionID the session id - */ - CalpontDMLPackage( std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID ); + /** @brief dtor + */ + virtual ~CalpontDMLPackage(); - /** @brief dtor - */ - virtual ~CalpontDMLPackage(); + /** @brief write a CalpontDMLPackage to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + virtual int write(messageqcpp::ByteStream& bytestream) = 0; - /** @brief write a CalpontDMLPackage to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - virtual int write( messageqcpp::ByteStream& bytestream ) = 0; + /** @brief read a CalpontDMLPackage from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + virtual int read(messageqcpp::ByteStream& bytestream) = 0; - /** @brief read a CalpontDMLPackage from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - virtual int read( messageqcpp::ByteStream& bytestream ) = 0; + /** @brief build a CalpontDMLPackage from a string buffer + * + * @param buffer the row buffer + * @param columns the number of columns in the buffer + * @param rows the number of rows in the buffer + */ + virtual int buildFromBuffer(std::string& buffer, int columns, int rows) = 0; - /** @brief build a CalpontDMLPackage from a string buffer - * - * @param buffer the row buffer - * @param columns the number of columns in the buffer - * @param rows the number of rows in the buffer - */ - virtual int buildFromBuffer( std::string& buffer, int columns, int rows ) = 0; + /** @brief build a CalpontDMLPackage from a parsed SqlStatement + * + * @param sqlStatement the parsed SqlStatement + */ + virtual int buildFromSqlStatement(SqlStatement& sqlStatement) = 0; - /** @brief build a CalpontDMLPackage from a parsed SqlStatement - * - * @param sqlStatement the parsed SqlStatement - */ - virtual int buildFromSqlStatement( SqlStatement& sqlStatement ) = 0; + /** @brief build a CalpontDMLPackage from valuelist built from mysql table fields + * + * @param tableValuesMap the value list for each column in the table + * @param colNameList the column name for each column + * @param columns number of columns in the table + * @param rows number of rows to be touched + */ + virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, + int rows, NullValuesBitset& nullValues) = 0; - /** @brief build a CalpontDMLPackage from valuelist built from mysql table fields - * - * @param tableValuesMap the value list for each column in the table - * @param colNameList the column name for each column - * @param columns number of columns in the table - * @param rows number of rows to be touched - */ - virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues) = 0; + /** @brief get the table object + */ + DMLTable* get_Table() + { + return fTable; + } - /** @brief get the table object - */ - DMLTable* get_Table() - { - return fTable; - } + /** @brief set the DML statement (the parsed statement) + * + * @param statement the dml statement to set + */ + void set_DMLStatement(const std::string& statement) + { + fDMLStatement = statement; + } - /** @brief set the DML statement (the parsed statement) - * - * @param statement the dml statement to set - */ - void set_DMLStatement( const std::string& statement ) - { - fDMLStatement = statement; - } + /** @brief get the DML statement (the parsed statement) + */ + const std::string get_DMLStatement() const + { + return fDMLStatement; + } - /** @brief get the DML statement (the parsed statement) - */ - const std::string get_DMLStatement() const - { - return fDMLStatement; - } + /** @brief set the SQL statement (the original SQL statement) + * + * @param statement the SQL statement to set (the original SQL statement with quotes) + */ + void set_SQLStatement(const std::string& statement) + { + fSQLStatement = statement; + } - /** @brief set the SQL statement (the original SQL statement) - * - * @param statement the SQL statement to set (the original SQL statement with quotes) - */ - void set_SQLStatement( const std::string& statement ) - { - fSQLStatement = statement; - } + /** @brief get the SQL statement (the original SQL statement) + */ + const std::string get_SQLStatement() const + { + return fSQLStatement; + } - /** @brief get the SQL statement (the original SQL statement) - */ - const std::string get_SQLStatement() const - { - return fSQLStatement; - } + /** @brief get the logging flag + */ + bool get_Logging() const + { + return fLogging; + } - /** @brief get the logging flag - */ - bool get_Logging() const - { - return fLogging; - } + /** @brief set the logging flag + * + * @param logging the logging flag to set + */ + void set_Logging(bool logging) + { + fLogging = logging; + } - /** @brief set the logging flag - * - * @param logging the logging flag to set - */ - void set_Logging( bool logging ) - { - fLogging = logging; - } + /** @brief get the logending flag + */ + bool get_Logending() const + { + return fLogending; + } - /** @brief get the logending flag - */ - bool get_Logending() const - { - return fLogending; - } + /** @brief set the logending flag + * + * @param logending the logending flag to set + */ + void set_Logending(bool logending) + { + fLogending = logending; + } - /** @brief set the logending flag - * - * @param logending the logending flag to set - */ - void set_Logending( bool logending ) - { - fLogending = logending; - } + /** @brief get the isFromCol flag + */ + bool get_IsFromCol() const + { + return fIsFromCol; + } - /** @brief get the isFromCol flag - */ - bool get_IsFromCol() const - { - return fIsFromCol; - } + /** @brief set the update column from column flag + * + * @param logging the logging flag to set + */ + void set_IsFromCol(bool isFromCol) + { + fIsFromCol = isFromCol; + } + /** @brief set the Table name + * + * @param tableName the name to set + */ + void set_TableName(std::string& tableName) + { + fTableName = tableName; - /** @brief set the update column from column flag - * - * @param logging the logging flag to set - */ - void set_IsFromCol ( bool isFromCol ) - { - fIsFromCol = isFromCol; - } - /** @brief set the Table name - * - * @param tableName the name to set - */ - void set_TableName( std::string& tableName ) - { - fTableName = tableName; + if (fTable != 0) + fTable->set_TableName(tableName); + } - if (fTable != 0) - fTable->set_TableName(tableName); - } + /** @brief get the Table name + */ + const std::string get_TableName() const + { + return fTableName; + } - /** @brief get the Table name - */ - const std::string get_TableName() const - { - return fTableName; - } + /** @brief set the Schema name + * + * @param the schema to set + */ + void set_SchemaName(std::string& schemaName) + { + fSchemaName = schemaName; - /** @brief set the Schema name - * - * @param the schema to set - */ - void set_SchemaName( std::string& schemaName ) - { - fSchemaName = schemaName; + if (fTable != 0) + fTable->set_SchemaName(schemaName); + } - if (fTable != 0) - fTable->set_SchemaName(schemaName); - } + /** @brief get the Schema name + */ + const std::string get_SchemaName() const + { + return fSchemaName; + } - /** @brief get the Schema name - */ - const std::string get_SchemaName() const - { - return fSchemaName; - } + /** @brief set the timezone + * + * @param the timezone to set + */ + void set_TimeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } - /** @brief set the timezone - * - * @param the timezone to set - */ - void set_TimeZone( const std::string& timeZone ) - { - fTimeZone = timeZone; - } + /** @brief get the timezone + */ + const std::string get_TimeZone() const + { + return fTimeZone; + } - /** @brief get the timezone - */ - const std::string get_TimeZone() const - { - return fTimeZone; - } + /** @brief does this dml statement have a filter + */ + bool HasFilter() const + { + return fHasFilter; + } + void HasFilter(bool hasFilter) + { + fHasFilter = hasFilter; + } - /** @brief does this dml statement have a filter - */ - bool HasFilter() const - { - return fHasFilter; - } - void HasFilter( bool hasFilter) - { - fHasFilter = hasFilter; - } + /** @brief get the filter statement + */ + const std::string get_QueryString() const + { + return fQueryString; + } - /** @brief get the filter statement - */ - const std::string get_QueryString() const - { - return fQueryString; - } + /** @brief set the sessionID associated with this package + */ + void set_SessionID(int sessionID) + { + fSessionID = sessionID; + } - /** @brief set the sessionID associated with this package - */ - void set_SessionID( int sessionID ) - { - fSessionID = sessionID; - } + /** @brief get the sessionID associated with this package + */ + int get_SessionID() const + { + return fSessionID; + } - /** @brief get the sessionID associated with this package - */ - int get_SessionID() const - { - return fSessionID; - } + /** @brief set the transaction ID associated with this package + */ + void set_TxnID(execplan::CalpontSystemCatalog::SCN txnID) + { + fTxnId = txnID; + } - /** @brief set the transaction ID associated with this package - */ - void set_TxnID( execplan::CalpontSystemCatalog::SCN txnID ) - { - fTxnId = txnID; - } + /** @brief get the transaction ID associated with this package + */ + execplan::CalpontSystemCatalog::SCN get_TxnID() const + { + return fTxnId; + } + /** @brief set the chunkmanager associated with this package + */ + void set_ChunkManager(WriteEngine::ChunkManager* cm) + { + fCM = cm; + } - /** @brief get the transaction ID associated with this package - */ - execplan::CalpontSystemCatalog::SCN get_TxnID() const - { - return fTxnId; - } - /** @brief set the chunkmanager associated with this package - */ - void set_ChunkManager( WriteEngine::ChunkManager* cm ) - { - fCM = cm; - } + /** @brief get the chunkmanager associated with this package + */ + WriteEngine::ChunkManager* get_ChunkManager() const + { + return fCM; + } - /** @brief get the chunkmanager associated with this package - */ - WriteEngine::ChunkManager* get_ChunkManager() const - { - return fCM; - } + /** @brief get the ExecutionPlan associated with this package + */ + boost::shared_ptr get_ExecutionPlan() + { + return fPlan; + } - /** @brief get the ExecutionPlan associated with this package - */ - boost::shared_ptr get_ExecutionPlan() - { - return fPlan; - } + bool get_isInsertSelect() + { + return fIsInsertSelect; + } + void set_isInsertSelect(const bool isInsertSelect) + { + fIsInsertSelect = isInsertSelect; + } - bool get_isInsertSelect() - { - return fIsInsertSelect; - } - void set_isInsertSelect( const bool isInsertSelect ) - { - fIsInsertSelect = isInsertSelect; - } + bool get_isBatchInsert() + { + return fIsBatchInsert; + } + void set_isBatchInsert(const bool isBatchInsert) + { + fIsBatchInsert = isBatchInsert; + } - bool get_isBatchInsert() - { - return fIsBatchInsert; - } - void set_isBatchInsert( const bool isBatchInsert ) - { - fIsBatchInsert = isBatchInsert; - } + bool get_isCacheInsert() + { + return fIsCacheInsert; + } + void set_isCacheInsert(const bool isCacheInsert) + { + fIsCacheInsert = isCacheInsert; + } - bool get_isCacheInsert() - { - return fIsCacheInsert; - } - void set_isCacheInsert( const bool isCacheInsert ) - { - fIsCacheInsert = isCacheInsert; - } + bool get_isAutocommitOn() + { + return fIsAutocommitOn; + } + void set_isAutocommitOn(const bool isAutocommitOn) + { + fIsAutocommitOn = isAutocommitOn; + } - bool get_isAutocommitOn() - { - return fIsAutocommitOn; - } - void set_isAutocommitOn( const bool isAutocommitOn ) - { - fIsAutocommitOn = isAutocommitOn; - } + bool get_isWarnToError() + { + return fIsWarnToError; + } + void set_isWarnToError(const bool isWarnToError) + { + fIsWarnToError = isWarnToError; + } - bool get_isWarnToError() - { - return fIsWarnToError; - } - void set_isWarnToError( const bool isWarnToError ) - { - fIsWarnToError = isWarnToError; - } + uint32_t getTableOid() + { + return fTableOid; + } + void setTableOid(const uint32_t tableOid) + { + fTableOid = tableOid; + } - uint32_t getTableOid() - { - return fTableOid; - } - void setTableOid( const uint32_t tableOid ) - { - fTableOid = tableOid; - } + void uuid(const boost::uuids::uuid& uuid) + { + fUuid = uuid; + } + const boost::uuids::uuid& uuid() const + { + return fUuid; + } - void uuid(const boost::uuids::uuid& uuid) - { - fUuid = uuid; - } - const boost::uuids::uuid& uuid() const - { - return fUuid; - } + protected: + void initializeTable(); -protected: - - void initializeTable(); - - std::string fSchemaName; - std::string fTimeZone; - std::string fTableName; - std::string fDMLStatement; - std::string fSQLStatement; - std::string fQueryString; - int fSessionID; - boost::uuids::uuid fUuid; - execplan::CalpontSystemCatalog::SCN fTxnId; - boost::shared_ptr fPlan; - DMLTable* fTable; - bool fHasFilter; - bool fLogging; - bool fLogending; - bool fIsFromCol; - std::string StripLeadingWhitespace( std::string value ); - bool fIsInsertSelect; - bool fIsBatchInsert; - bool fIsCacheInsert; - bool fIsAutocommitOn; - bool fIsWarnToError; - uint32_t fTableOid; - WriteEngine::ChunkManager* fCM; + std::string fSchemaName; + std::string fTimeZone; + std::string fTableName; + std::string fDMLStatement; + std::string fSQLStatement; + std::string fQueryString; + int fSessionID; + boost::uuids::uuid fUuid; + execplan::CalpontSystemCatalog::SCN fTxnId; + boost::shared_ptr fPlan; + DMLTable* fTable; + bool fHasFilter; + bool fLogging; + bool fLogending; + bool fIsFromCol; + std::string StripLeadingWhitespace(std::string value); + bool fIsInsertSelect; + bool fIsBatchInsert; + bool fIsCacheInsert; + bool fIsAutocommitOn; + bool fIsWarnToError; + uint32_t fTableOid; + WriteEngine::ChunkManager* fCM; }; -} -#endif //CALPONTDMLPACKAGE_H +} // namespace dmlpackage +#endif // CALPONTDMLPACKAGE_H diff --git a/dbcon/dmlpackage/commanddmlpackage.cpp b/dbcon/dmlpackage/commanddmlpackage.cpp index 02669232e..cf9847396 100644 --- a/dbcon/dmlpackage/commanddmlpackage.cpp +++ b/dbcon/dmlpackage/commanddmlpackage.cpp @@ -31,70 +31,72 @@ using namespace std; #undef COMMANDDMLPKG_DLLEXPORT namespace dmlpackage { - CommandDMLPackage::CommandDMLPackage() -{} +{ +} -CommandDMLPackage::CommandDMLPackage( std::string dmlStatement, int sessionID) - : CalpontDMLPackage( "", "", dmlStatement, sessionID) -{} +CommandDMLPackage::CommandDMLPackage(std::string dmlStatement, int sessionID) + : CalpontDMLPackage("", "", dmlStatement, sessionID) +{ +} CommandDMLPackage::~CommandDMLPackage() -{} +{ +} int CommandDMLPackage::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::byte package_type = DML_COMMAND; - bytestream << package_type; + messageqcpp::ByteStream::byte package_type = DML_COMMAND; + bytestream << package_type; - messageqcpp::ByteStream::quadbyte session_id = fSessionID; - bytestream << session_id; + messageqcpp::ByteStream::quadbyte session_id = fSessionID; + bytestream << session_id; - bytestream << fUuid; + bytestream << fUuid; - bytestream << fDMLStatement; - bytestream << fSQLStatement; // for cleartablelock, this is table lockID - bytestream << (uint8_t)fLogging; - bytestream << fSchemaName; - bytestream << fTimeZone; - bytestream << fTableName; - bytestream << fTableOid; - bytestream << static_cast(fIsAutocommitOn); - bytestream << static_cast(fIsBatchInsert); - return retval; + bytestream << fDMLStatement; + bytestream << fSQLStatement; // for cleartablelock, this is table lockID + bytestream << (uint8_t)fLogging; + bytestream << fSchemaName; + bytestream << fTimeZone; + bytestream << fTableName; + bytestream << fTableOid; + bytestream << static_cast(fIsAutocommitOn); + bytestream << static_cast(fIsBatchInsert); + return retval; } int CommandDMLPackage::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::quadbyte session_id; - bytestream >> session_id; - fSessionID = session_id; - bytestream >> fUuid; + messageqcpp::ByteStream::quadbyte session_id; + bytestream >> session_id; + fSessionID = session_id; + bytestream >> fUuid; - bytestream >> fDMLStatement; - bytestream >> fSQLStatement; // for cleartablelock, this is table lockID - uint8_t logging; - bytestream >> logging; - fLogging = (logging != 0); - bytestream >> fSchemaName; - bytestream >> fTimeZone; - bytestream >> fTableName; - bytestream >> fTableOid; - bytestream >> reinterpret_cast< messageqcpp::ByteStream::byte&>(fIsAutocommitOn); - bytestream >> reinterpret_cast< messageqcpp::ByteStream::byte&>(fIsBatchInsert); - return retval; + bytestream >> fDMLStatement; + bytestream >> fSQLStatement; // for cleartablelock, this is table lockID + uint8_t logging; + bytestream >> logging; + fLogging = (logging != 0); + bytestream >> fSchemaName; + bytestream >> fTimeZone; + bytestream >> fTableName; + bytestream >> fTableOid; + bytestream >> reinterpret_cast(fIsAutocommitOn); + bytestream >> reinterpret_cast(fIsBatchInsert); + return retval; } int CommandDMLPackage::buildFromSqlStatement(SqlStatement& sqlStatement) { - CommandSqlStatement& cmdStmt = dynamic_cast(sqlStatement); - fDMLStatement = cmdStmt.fCommandText; + CommandSqlStatement& cmdStmt = dynamic_cast(sqlStatement); + fDMLStatement = cmdStmt.fCommandText; - return 1; + return 1; } -} // namespace dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/commanddmlpackage.h b/dbcon/dmlpackage/commanddmlpackage.h index 2ddcdde29..cde03ae84 100644 --- a/dbcon/dmlpackage/commanddmlpackage.h +++ b/dbcon/dmlpackage/commanddmlpackage.h @@ -41,64 +41,62 @@ namespace dmlpackage */ class CommandDMLPackage : public CalpontDMLPackage { + public: + /** @brief ctor + */ + EXPORT CommandDMLPackage(); -public: - /** @brief ctor - */ - EXPORT CommandDMLPackage(); + /** @brief ctor + */ + EXPORT CommandDMLPackage(std::string dmlStatement, int sessionID); - /** @brief ctor - */ - EXPORT CommandDMLPackage( std::string dmlStatement, int sessionID ); + /** @brief dtor + */ + EXPORT virtual ~CommandDMLPackage(); - /** @brief dtor - */ - EXPORT virtual ~CommandDMLPackage(); + /** @brief write a CommandDMLPackage to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + EXPORT int write(messageqcpp::ByteStream& bytestream); - /** @brief write a CommandDMLPackage to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + /** @brief read CommandDMLPackage from bytestream + * + * @param bytestream the ByteStream to read from + */ + EXPORT int read(messageqcpp::ByteStream& bytestream); + /** @brief do nothing + * + * @param buffer + * @param columns the number of columns in the buffer + * @param rows the number of rows in the buffer + */ + inline int buildFromBuffer(std::string& buffer, int columns = 0, int rows = 0) + { + return 1; + }; - /** @brief read CommandDMLPackage from bytestream - * - * @param bytestream the ByteStream to read from - */ - EXPORT int read(messageqcpp::ByteStream& bytestream); - /** @brief do nothing - * - * @param buffer - * @param columns the number of columns in the buffer - * @param rows the number of rows in the buffer - */ - inline int buildFromBuffer(std::string& buffer, int columns = 0, int rows = 0) - { - return 1; - }; + /** @brief build a CommandDMLPackage from a CommandSqlStatement + */ + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - /** @brief build a CommandDMLPackage from a CommandSqlStatement - */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - - /** @brief build a InsertDMLPackage from MySQL buffer - * - * @param colNameList, tableValuesMap - * @param rows the number of rows in the buffer - */ - int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues) - { - return 1; - }; - -protected: - -private: + /** @brief build a InsertDMLPackage from MySQL buffer + * + * @param colNameList, tableValuesMap + * @param rows the number of rows in the buffer + */ + int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, + NullValuesBitset& nullValues) + { + return 1; + }; + protected: + private: }; -} +} // namespace dmlpackage #undef EXPORT -#endif //COMMANDDMLPACKAGE_H +#endif // COMMANDDMLPACKAGE_H diff --git a/dbcon/dmlpackage/deletedmlpackage.cpp b/dbcon/dmlpackage/deletedmlpackage.cpp index af0a77bde..2ab6c6f2a 100644 --- a/dbcon/dmlpackage/deletedmlpackage.cpp +++ b/dbcon/dmlpackage/deletedmlpackage.cpp @@ -32,107 +32,109 @@ using namespace std; namespace dmlpackage { - DeleteDMLPackage::DeleteDMLPackage() -{} +{ +} -DeleteDMLPackage::DeleteDMLPackage(std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID) - : CalpontDMLPackage( schemaName, tableName, dmlStatement, sessionID ) -{} +DeleteDMLPackage::DeleteDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID) + : CalpontDMLPackage(schemaName, tableName, dmlStatement, sessionID) +{ +} DeleteDMLPackage::~DeleteDMLPackage() -{} +{ +} int DeleteDMLPackage::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::byte package_type = DML_DELETE; - bytestream << package_type; + messageqcpp::ByteStream::byte package_type = DML_DELETE; + bytestream << package_type; - messageqcpp::ByteStream::quadbyte session_id = fSessionID; - bytestream << session_id; + messageqcpp::ByteStream::quadbyte session_id = fSessionID; + bytestream << session_id; - /* if(fPlan != 0) - fHasFilter = true; - else - fHasFilter = false; - */ - messageqcpp::ByteStream::quadbyte hasFilter = fHasFilter; - bytestream << hasFilter; + /* if(fPlan != 0) + fHasFilter = true; + else + fHasFilter = false; + */ + messageqcpp::ByteStream::quadbyte hasFilter = fHasFilter; + bytestream << hasFilter; - bytestream << fUuid; + bytestream << fUuid; - bytestream << fDMLStatement; - bytestream << fSQLStatement; - bytestream << fSchemaName; - bytestream << fTimeZone; + bytestream << fDMLStatement; + bytestream << fSQLStatement; + bytestream << fSchemaName; + bytestream << fTimeZone; - if (fTable != 0) - { - retval = fTable->write(bytestream); - } + if (fTable != 0) + { + retval = fTable->write(bytestream); + } - if (fHasFilter) - { - bytestream += *(fPlan.get()); - } + if (fHasFilter) + { + bytestream += *(fPlan.get()); + } - return retval; + return retval; } /** * */ int DeleteDMLPackage::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::quadbyte session_id; - messageqcpp::ByteStream::quadbyte hasFilter; + messageqcpp::ByteStream::quadbyte session_id; + messageqcpp::ByteStream::quadbyte hasFilter; - bytestream >> session_id; - fSessionID = session_id; + bytestream >> session_id; + fSessionID = session_id; - bytestream >> hasFilter; - fHasFilter = (hasFilter != 0); + bytestream >> hasFilter; + fHasFilter = (hasFilter != 0); - bytestream >> fUuid; + bytestream >> fUuid; - std::string dmlStatement; - bytestream >> fDMLStatement; - bytestream >> fSQLStatement; - bytestream >> fSchemaName; - bytestream >> fTimeZone; + std::string dmlStatement; + bytestream >> fDMLStatement; + bytestream >> fSQLStatement; + bytestream >> fSchemaName; + bytestream >> fTimeZone; - fTable = new DMLTable(); - retval = fTable->read(bytestream); + fTable = new DMLTable(); + retval = fTable->read(bytestream); - if (fHasFilter) - { - fPlan.reset(new messageqcpp::ByteStream(bytestream)); - } + if (fHasFilter) + { + fPlan.reset(new messageqcpp::ByteStream(bytestream)); + } - return retval; + return retval; } int DeleteDMLPackage::buildFromSqlStatement(SqlStatement& sqlStatement) { - int retval = 1; + int retval = 1; - DeleteSqlStatement& deleteStmt = dynamic_cast(sqlStatement); + DeleteSqlStatement& deleteStmt = dynamic_cast(sqlStatement); - initializeTable(); + initializeTable(); - if (0 != deleteStmt.fWhereClausePtr) - { - fHasFilter = true; - fQueryString = deleteStmt.getQueryString(); - } + if (0 != deleteStmt.fWhereClausePtr) + { + fHasFilter = true; + fQueryString = deleteStmt.getQueryString(); + } - // else all rows are deleted + // else all rows are deleted - return retval; + return retval; } /** @@ -141,84 +143,83 @@ int DeleteDMLPackage::buildFromSqlStatement(SqlStatement& sqlStatement) int DeleteDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows) { #ifdef DML_PACKAGE_DEBUG - //cout << "The data buffer received: " << buffer << endl; + // cout << "The data buffer received: " << buffer << endl; #endif - int retval = 1; + int retval = 1; - initializeTable(); + initializeTable(); - std::vector dataList; - typedef boost::tokenizer > tokenizer; - boost::char_separator sep(":"); - tokenizer tokens(buffer, sep); + std::vector dataList; + typedef boost::tokenizer > tokenizer; + boost::char_separator sep(":"); + tokenizer tokens(buffer, sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) - { - dataList.push_back(StripLeadingWhitespace(*tok_iter)); + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + dataList.push_back(StripLeadingWhitespace(*tok_iter)); + } - } + int n = 0; - int n = 0; - - for (int i = 0; i < rows; i++) - { - //get a new row - Row aRow; - //Row *aRowPtr = new Row(); - //std::string colName; - //std::string colValue; - //get row ID from the buffer - std::string rowid = dataList[n++]; - aRow.set_RowID(atoll(rowid.c_str())); - //aRowPtr->set_RowID(atol(rowid.c_str())); + for (int i = 0; i < rows; i++) + { + // get a new row + Row aRow; + // Row *aRowPtr = new Row(); + // std::string colName; + // std::string colValue; + // get row ID from the buffer + std::string rowid = dataList[n++]; + aRow.set_RowID(atoll(rowid.c_str())); + // aRowPtr->set_RowID(atol(rowid.c_str())); #ifdef DML_PACKAGE_DEBUG - //cout << "The row ID is " << rowid << endl; + // cout << "The row ID is " << rowid << endl; #endif - /* - for (int j = 0; j < columns; j++) - { - //Build a column list - colName = dataList[n++]; - colValue = dataList[n++]; - #ifdef DML_PACKAGE_DEBUG - cout << "The column data: " << colName << " " << colValue << endl; - #endif - DMLColumn* aColumn = new DMLColumn(colName, colValue); - (aRowPtr->get_ColumnList()).push_back(aColumn); - } - //build a row list for a table - fTable->get_RowList().push_back(aRowPtr); - */ + /* + for (int j = 0; j < columns; j++) + { + //Build a column list + colName = dataList[n++]; + colValue = dataList[n++]; + #ifdef DML_PACKAGE_DEBUG + cout << "The column data: " << colName << " " << colValue << endl; + #endif + DMLColumn* aColumn = new DMLColumn(colName, colValue); + (aRowPtr->get_ColumnList()).push_back(aColumn); } + //build a row list for a table + fTable->get_RowList().push_back(aRowPtr); + */ + } - return retval; - + return retval; } -int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues ) +int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, + int columns, int rows, NullValuesBitset& nullValues) { - int retval = 1; + int retval = 1; - initializeTable(); - //The row already built from MySql parser. - /* Row *aRowPtr = new Row(); - std::string colName; - std::vector colValList; - for (int j = 0; j < columns; j++) - { - //Build a column list - colName = colNameList[j]; + initializeTable(); + // The row already built from MySql parser. + /* Row *aRowPtr = new Row(); + std::string colName; + std::vector colValList; + for (int j = 0; j < columns; j++) + { + //Build a column list + colName = colNameList[j]; - colValList = tableValuesMap[j]; + colValList = tableValuesMap[j]; - DMLColumn* aColumn = new DMLColumn(colName, colValList, false); - (aRowPtr->get_ColumnList()).push_back(aColumn); - } - //build a row list for a table - fTable->get_RowList().push_back(aRowPtr); */ - return retval; + DMLColumn* aColumn = new DMLColumn(colName, colValList, false); + (aRowPtr->get_ColumnList()).push_back(aColumn); + } + //build a row list for a table + fTable->get_RowList().push_back(aRowPtr); */ + return retval; } -} // namespace dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/deletedmlpackage.h b/dbcon/dmlpackage/deletedmlpackage.h index 562b73931..4658853f8 100644 --- a/dbcon/dmlpackage/deletedmlpackage.h +++ b/dbcon/dmlpackage/deletedmlpackage.h @@ -41,66 +41,63 @@ namespace dmlpackage */ class DeleteDMLPackage : public CalpontDMLPackage { + public: + /** @brief ctor + */ + EXPORT DeleteDMLPackage(); -public: + /** @brief ctor + * + * @param schemaName the schema of the table being operated on + * @param tableName the name of the table being operated on + * @param dmlStatement the dml statement + * @param sessionID the session ID + */ + EXPORT DeleteDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID); - /** @brief ctor - */ - EXPORT DeleteDMLPackage(); + /** @brief dtor + */ + EXPORT virtual ~DeleteDMLPackage(); - /** @brief ctor - * - * @param schemaName the schema of the table being operated on - * @param tableName the name of the table being operated on - * @param dmlStatement the dml statement - * @param sessionID the session ID - */ - EXPORT DeleteDMLPackage( std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID ); + /** @brief write a DeleteDMLPackage to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + EXPORT int write(messageqcpp::ByteStream& bytestream); - /** @brief dtor - */ - EXPORT virtual ~DeleteDMLPackage(); + /** @brief read a DeleteDMLPackage from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + EXPORT int read(messageqcpp::ByteStream& bytestream); - /** @brief write a DeleteDMLPackage to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + /** @brief build a DeleteDMLPackage from a string buffer + * + * @param buffer [rowId, columnName, colValue] + * @param columns the number of columns in the buffer + * @param rows the number of rows in the buffer + */ + EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); - /** @brief read a DeleteDMLPackage from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - EXPORT int read(messageqcpp::ByteStream& bytestream); - - /** @brief build a DeleteDMLPackage from a string buffer - * - * @param buffer [rowId, columnName, colValue] - * @param columns the number of columns in the buffer - * @param rows the number of rows in the buffer - */ - EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); - - /** @brief build a DeleteDMLPackage from a parsed DeleteSqlStatement - * - * @param sqlStatement the parsed DeleteSqlStatement - */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - /** @brief build a InsertDMLPackage from MySQL buffer - * - * @param colNameList, tableValuesMap - * @param rows the number of rows in the buffer - */ - EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues); - -protected: - -private: + /** @brief build a DeleteDMLPackage from a parsed DeleteSqlStatement + * + * @param sqlStatement the parsed DeleteSqlStatement + */ + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); + /** @brief build a InsertDMLPackage from MySQL buffer + * + * @param colNameList, tableValuesMap + * @param rows the number of rows in the buffer + */ + EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, + int rows, NullValuesBitset& nullValues); + protected: + private: }; -} +} // namespace dmlpackage #undef EXPORT -#endif //DELETEDMLPACKAGE_H +#endif // DELETEDMLPACKAGE_H diff --git a/dbcon/dmlpackage/dml-gram-win.cpp b/dbcon/dmlpackage/dml-gram-win.cpp index d224d89b1..0cdd2d9f9 100644 --- a/dbcon/dmlpackage/dml-gram-win.cpp +++ b/dbcon/dmlpackage/dml-gram-win.cpp @@ -64,18 +64,16 @@ #define YYLSP_NEEDED 0 /* Substitute the variable and function names. */ -#define yyparse dmlparse -#define yylex dmllex -#define yyerror dmlerror -#define yylval dmllval -#define yychar dmlchar -#define yydebug dmldebug -#define yynerrs dmlnerrs - +#define yyparse dmlparse +#define yylex dmllex +#define yyerror dmlerror +#define yylval dmllval +#define yychar dmlchar +#define yydebug dmldebug +#define yynerrs dmlnerrs /* Copy the first part of user declarations. */ - #include #include "dmlparser.h" @@ -93,11 +91,10 @@ using namespace dmlpackage; int dmllex(); -void dmlerror (char const* error); +void dmlerror(char const* error); namespace dmlpackage { - /* The user is expect to pass a ParseTree* to grammar_init */ static ParseTree* parseTree; typedef std::vector copybuf_t; @@ -105,185 +102,169 @@ static copybuf_t copy_buffer; static std::string default_schema; char* copy_string(const char* str); -} - - - +} // namespace dmlpackage /* Enabling traces. */ #ifndef YYDEBUG -# define YYDEBUG 1 +#define YYDEBUG 1 #endif /* Enabling verbose error messages. */ #ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 +#undef YYERROR_VERBOSE +#define YYERROR_VERBOSE 1 #else -# define YYERROR_VERBOSE 0 +#define YYERROR_VERBOSE 0 #endif /* Enabling the token table. */ #ifndef YYTOKEN_TABLE -# define YYTOKEN_TABLE 0 +#define YYTOKEN_TABLE 0 #endif - /* Tokens. */ #ifndef YYTOKENTYPE -# define YYTOKENTYPE +#define YYTOKENTYPE /* Put the tokens into the symbol table, so that GDB and other debuggers know about them. */ enum yytokentype { - NAME = 258, - STRING = 259, - INTNUM = 260, - APPROXNUM = 261, - SELECT = 262, - ALL = 263, - DISTINCT = 264, - NULLX = 265, - USER = 266, - INDICATOR = 267, - AMMSC = 268, - PARAMETER = 269, - ANY = 270, - SOME = 271, - OR = 272, - AND = 273, - NOT = 274, - COMPARISON = 275, - UMINUS = 276, - AS = 277, - ASC = 278, - AUTHORIZATION = 279, - BETWEEN = 280, - BY = 281, - CHARACTER = 282, - CHECK = 283, - CLOSE = 284, - COMMIT = 285, - CONTINUE = 286, - CREATE = 287, - CURRENT = 288, - CURSOR = 289, - IDB_DECIMAL = 290, - DECLARE = 291, - DEFAULT = 292, - DELETE = 293, - DESC = 294, - IDB_DOUBLE = 295, - ESCAPE = 296, - EXISTS = 297, - FETCH = 298, - IDB_FLOAT = 299, - FOR = 300, - FOREIGN = 301, - FOUND = 302, - FROM = 303, - GOTO = 304, - GRANT = 305, - IDB_GROUP = 306, - HAVING = 307, - IN = 308, - INSERT = 309, - INTEGER = 310, - INTO = 311, - IS = 312, - KEY = 313, - LANGUAGE = 314, - LIKE = 315, - NUMERIC = 316, - OF = 317, - ON = 318, - OPEN = 319, - OPTION = 320, - ORDER = 321, - PRECISION = 322, - PRIMARY = 323, - PRIVILEGES = 324, - PROCEDURE = 325, - PUBLIC = 326, - REAL = 327, - REFERENCES = 328, - ROLLBACK = 329, - SCHEMA = 330, - SET = 331, - SMALLINT = 332, - SQLCODE = 333, - SQLERROR = 334, - TABLE = 335, - TO = 336, - UNION = 337, - UNIQUE = 338, - UPDATE = 339, - VALUES = 340, - VIEW = 341, - WHENEVER = 342, - WHERE = 343, - WITH = 344, - WORK = 345 + NAME = 258, + STRING = 259, + INTNUM = 260, + APPROXNUM = 261, + SELECT = 262, + ALL = 263, + DISTINCT = 264, + NULLX = 265, + USER = 266, + INDICATOR = 267, + AMMSC = 268, + PARAMETER = 269, + ANY = 270, + SOME = 271, + OR = 272, + AND = 273, + NOT = 274, + COMPARISON = 275, + UMINUS = 276, + AS = 277, + ASC = 278, + AUTHORIZATION = 279, + BETWEEN = 280, + BY = 281, + CHARACTER = 282, + CHECK = 283, + CLOSE = 284, + COMMIT = 285, + CONTINUE = 286, + CREATE = 287, + CURRENT = 288, + CURSOR = 289, + IDB_DECIMAL = 290, + DECLARE = 291, + DEFAULT = 292, + DELETE = 293, + DESC = 294, + IDB_DOUBLE = 295, + ESCAPE = 296, + EXISTS = 297, + FETCH = 298, + IDB_FLOAT = 299, + FOR = 300, + FOREIGN = 301, + FOUND = 302, + FROM = 303, + GOTO = 304, + GRANT = 305, + IDB_GROUP = 306, + HAVING = 307, + IN = 308, + INSERT = 309, + INTEGER = 310, + INTO = 311, + IS = 312, + KEY = 313, + LANGUAGE = 314, + LIKE = 315, + NUMERIC = 316, + OF = 317, + ON = 318, + OPEN = 319, + OPTION = 320, + ORDER = 321, + PRECISION = 322, + PRIMARY = 323, + PRIVILEGES = 324, + PROCEDURE = 325, + PUBLIC = 326, + REAL = 327, + REFERENCES = 328, + ROLLBACK = 329, + SCHEMA = 330, + SET = 331, + SMALLINT = 332, + SQLCODE = 333, + SQLERROR = 334, + TABLE = 335, + TO = 336, + UNION = 337, + UNIQUE = 338, + UPDATE = 339, + VALUES = 340, + VIEW = 341, + WHENEVER = 342, + WHERE = 343, + WITH = 344, + WORK = 345 }; #endif - - -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED typedef union YYSTYPE { - - - int intval; - double floatval; - char* strval; - int subtok; - dmlpackage::SqlStatementList* sqlStmtList; - dmlpackage::SqlStatement* sqlStmt; - dmlpackage::TableName* tblName; - dmlpackage::ColumnNameList* colNameList; - dmlpackage::ValuesOrQuery* valsOrQuery; - dmlpackage::ValuesList* valsList; - dmlpackage::QuerySpec* querySpec; - dmlpackage::TableNameList* tableNameList; - dmlpackage::TableExpression* tableExpression; - dmlpackage::WhereClause* whereClause; - dmlpackage::SearchCondition* searchCondition; - dmlpackage::ExistanceTestPredicate* existPredicate; - dmlpackage::AllOrAnyPredicate* allOrAnyPredicate; - dmlpackage::InPredicate* inPredicate; - dmlpackage::NullTestPredicate* nullTestPredicate; - dmlpackage::LikePredicate* likePredicate; - dmlpackage::BetweenPredicate* betweenPredicate; - dmlpackage::ComparisonPredicate* comparisonPredicate; - dmlpackage::Predicate* predicate; - dmlpackage::FromClause* fromClause; - dmlpackage::SelectFilter* selectFilter; - dmlpackage::GroupByClause* groupByClause; - dmlpackage::HavingClause* havingClause; - dmlpackage::Escape* escape; - dmlpackage::AtomList* atomList; - dmlpackage::ColumnAssignment* colAssignment; - dmlpackage::ColumnAssignmentList* colAssignmentList; - - + int intval; + double floatval; + char* strval; + int subtok; + dmlpackage::SqlStatementList* sqlStmtList; + dmlpackage::SqlStatement* sqlStmt; + dmlpackage::TableName* tblName; + dmlpackage::ColumnNameList* colNameList; + dmlpackage::ValuesOrQuery* valsOrQuery; + dmlpackage::ValuesList* valsList; + dmlpackage::QuerySpec* querySpec; + dmlpackage::TableNameList* tableNameList; + dmlpackage::TableExpression* tableExpression; + dmlpackage::WhereClause* whereClause; + dmlpackage::SearchCondition* searchCondition; + dmlpackage::ExistanceTestPredicate* existPredicate; + dmlpackage::AllOrAnyPredicate* allOrAnyPredicate; + dmlpackage::InPredicate* inPredicate; + dmlpackage::NullTestPredicate* nullTestPredicate; + dmlpackage::LikePredicate* likePredicate; + dmlpackage::BetweenPredicate* betweenPredicate; + dmlpackage::ComparisonPredicate* comparisonPredicate; + dmlpackage::Predicate* predicate; + dmlpackage::FromClause* fromClause; + dmlpackage::SelectFilter* selectFilter; + dmlpackage::GroupByClause* groupByClause; + dmlpackage::HavingClause* havingClause; + dmlpackage::Escape* escape; + dmlpackage::AtomList* atomList; + dmlpackage::ColumnAssignment* colAssignment; + dmlpackage::ColumnAssignmentList* colAssignmentList; } YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 +#define YYSTYPE_IS_TRIVIAL 1 +#define yystype YYSTYPE /* obsolescent; will be withdrawn */ +#define YYSTYPE_IS_DECLARED 1 #endif - /* Copy the second part of user declarations. */ - - - - - #ifdef short -# undef short +#undef short #endif #ifdef YYTYPE_UINT8 @@ -294,8 +275,7 @@ typedef unsigned char yytype_uint8; #ifdef YYTYPE_INT8 typedef YYTYPE_INT8 yytype_int8; -#elif (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) +#elif (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) typedef signed char yytype_int8; #else typedef short int yytype_int8; @@ -314,839 +294,708 @@ typedef short int yytype_int16; #endif #ifndef YYSIZE_T -# ifdef __SIZE_TYPE__ -# define YYSIZE_T __SIZE_TYPE__ -# elif defined size_t -# define YYSIZE_T size_t -# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# define YYSIZE_T size_t -# else -# define YYSIZE_T unsigned int -# endif +#ifdef __SIZE_TYPE__ +#define YYSIZE_T __SIZE_TYPE__ +#elif defined size_t +#define YYSIZE_T size_t +#elif !defined YYSIZE_T && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +#include /* INFRINGES ON USER NAME SPACE */ +#define YYSIZE_T size_t +#else +#define YYSIZE_T unsigned int +#endif #endif -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) +#define YYSIZE_MAXIMUM ((YYSIZE_T)-1) #ifndef YY_ -# if YYENABLE_NLS -# if ENABLE_NLS -# include /* INFRINGES ON USER NAME SPACE */ -# define YY_(msgid) dgettext ("bison-runtime", msgid) -# endif -# endif -# ifndef YY_ -# define YY_(msgid) msgid -# endif +#if YYENABLE_NLS +#if ENABLE_NLS +#include /* INFRINGES ON USER NAME SPACE */ +#define YY_(msgid) dgettext("bison-runtime", msgid) +#endif +#endif +#ifndef YY_ +#define YY_(msgid) msgid +#endif #endif /* Suppress unused-variable warnings by "using" E. */ -#if ! defined lint || defined __GNUC__ -# define YYUSE(e) ((void) (e)) +#if !defined lint || defined __GNUC__ +#define YYUSE(e) ((void)(e)) #else -# define YYUSE(e) /* empty */ +#define YYUSE(e) /* empty */ #endif /* Identity function, used to suppress warnings about constant conditions. */ #ifndef lint -# define YYID(n) (n) +#define YYID(n) (n) #else -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static int -YYID (int yyi) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static int YYID(int yyi) #else -static int -YYID (yyi) -int yyi; +static int YYID(yyi) int yyi; #endif { - return yyi; + return yyi; } #endif -#if ! defined yyoverflow || YYERROR_VERBOSE +#if !defined yyoverflow || YYERROR_VERBOSE /* The parser invokes alloca or malloc; define the necessary symbols. */ -# ifdef YYSTACK_USE_ALLOCA -# if YYSTACK_USE_ALLOCA -# ifdef __GNUC__ -# define YYSTACK_ALLOC __builtin_alloca -# elif defined __BUILTIN_VA_ARG_INCR -# include /* INFRINGES ON USER NAME SPACE */ -# elif defined _AIX -# define YYSTACK_ALLOC __alloca -# elif defined _MSC_VER -# include /* INFRINGES ON USER NAME SPACE */ -# define alloca _alloca -# else -# define YYSTACK_ALLOC alloca -# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# endif -# endif -# endif +#ifdef YYSTACK_USE_ALLOCA +#if YYSTACK_USE_ALLOCA +#ifdef __GNUC__ +#define YYSTACK_ALLOC __builtin_alloca +#elif defined __BUILTIN_VA_ARG_INCR +#include /* INFRINGES ON USER NAME SPACE */ +#elif defined _AIX +#define YYSTACK_ALLOC __alloca +#elif defined _MSC_VER +#include /* INFRINGES ON USER NAME SPACE */ +#define alloca _alloca +#else +#define YYSTACK_ALLOC alloca +#if !defined _ALLOCA_H && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +#include /* INFRINGES ON USER NAME SPACE */ +#ifndef _STDLIB_H +#define _STDLIB_H 1 +#endif +#endif +#endif +#endif +#endif -# ifdef YYSTACK_ALLOC +#ifdef YYSTACK_ALLOC /* Pacify GCC's `empty if-body' warning. */ -# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0)) -# ifndef YYSTACK_ALLOC_MAXIMUM +#define YYSTACK_FREE(Ptr) \ + do \ + { /* empty */ \ + ; \ + } while (YYID(0)) +#ifndef YYSTACK_ALLOC_MAXIMUM /* The OS might guarantee only one guard page at the bottom of the stack, and a page size can be as small as 4096 bytes. So we cannot safely invoke alloca (N) if N exceeds 4096. Use a slightly smaller number to allow for a few compiler-allocated temporary stack slots. */ -# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ -# endif -# else -# define YYSTACK_ALLOC YYMALLOC -# define YYSTACK_FREE YYFREE -# ifndef YYSTACK_ALLOC_MAXIMUM -# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM -# endif -# if (defined __cplusplus && ! defined _STDLIB_H \ - && ! ((defined YYMALLOC || defined malloc) \ - && (defined YYFREE || defined free))) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# ifndef YYMALLOC -# define YYMALLOC malloc -# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void* malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# ifndef YYFREE -# define YYFREE free -# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void free (void*); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# endif +#define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +#endif +#else +#define YYSTACK_ALLOC YYMALLOC +#define YYSTACK_FREE YYFREE +#ifndef YYSTACK_ALLOC_MAXIMUM +#define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +#endif +#if (defined __cplusplus && !defined _STDLIB_H && \ + !((defined YYMALLOC || defined malloc) && (defined YYFREE || defined free))) +#include /* INFRINGES ON USER NAME SPACE */ +#ifndef _STDLIB_H +#define _STDLIB_H 1 +#endif +#endif +#ifndef YYMALLOC +#define YYMALLOC malloc +#if !defined malloc && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +void* malloc(YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +#endif +#endif +#ifndef YYFREE +#define YYFREE free +#if !defined free && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +void free(void*); /* INFRINGES ON USER NAME SPACE */ +#endif +#endif +#endif #endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - -#if (! defined yyoverflow \ - && (! defined __cplusplus \ - || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) +#if (!defined yyoverflow && (!defined __cplusplus || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) /* A type that is properly aligned for any stack member. */ union yyalloc { - yytype_int16 yyss_alloc; - YYSTYPE yyvs_alloc; + yytype_int16 yyss_alloc; + YYSTYPE yyvs_alloc; }; /* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) +#define YYSTACK_GAP_MAXIMUM (sizeof(union yyalloc) - 1) /* The size of an array large to enough to hold all stacks, each with N elements. */ -# define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ - + YYSTACK_GAP_MAXIMUM) +#define YYSTACK_BYTES(N) ((N) * (sizeof(yytype_int16) + sizeof(YYSTYPE)) + YYSTACK_GAP_MAXIMUM) /* Copy COUNT objects from FROM to TO. The source and destination do not overlap. */ -# ifndef YYCOPY -# if defined __GNUC__ && 1 < __GNUC__ -# define YYCOPY(To, From, Count) \ - __builtin_memcpy (To, From, (Count) * sizeof (*(From))) -# else -# define YYCOPY(To, From, Count) \ - do \ - { \ - YYSIZE_T yyi; \ - for (yyi = 0; yyi < (Count); yyi++) \ - (To)[yyi] = (From)[yyi]; \ - } \ - while (YYID (0)) -# endif -# endif +#ifndef YYCOPY +#if defined __GNUC__ && 1 < __GNUC__ +#define YYCOPY(To, From, Count) __builtin_memcpy(To, From, (Count) * sizeof(*(From))) +#else +#define YYCOPY(To, From, Count) \ + do \ + { \ + YYSIZE_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (To)[yyi] = (From)[yyi]; \ + } while (YYID(0)) +#endif +#endif /* Relocate STACK from its old location to the new one. The local variables YYSIZE and YYSTACKSIZE give the old and new number of elements in the stack, and YYPTR gives the new location of the stack. Advance YYPTR to a properly aligned location for the next stack. */ -# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ - do \ - { \ - YYSIZE_T yynewbytes; \ - YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ - Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ - } \ - while (YYID (0)) +#define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYSIZE_T yynewbytes; \ + YYCOPY(&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * sizeof(*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / sizeof(*yyptr); \ + } while (YYID(0)) #endif /* YYFINAL -- State number of the termination state. */ -#define YYFINAL 47 +#define YYFINAL 47 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 523 +#define YYLAST 523 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 100 +#define YYNTOKENS 100 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 92 +#define YYNNTS 92 /* YYNRULES -- Number of rules. */ -#define YYNRULES 221 +#define YYNRULES 221 /* YYNRULES -- Number of states. */ -#define YYNSTATES 415 +#define YYNSTATES 415 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ -#define YYUNDEFTOK 2 -#define YYMAXUTOK 345 +#define YYUNDEFTOK 2 +#define YYMAXUTOK 345 -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) +#define YYTRANSLATE(YYX) ((unsigned int)(YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) /* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ -static const yytype_uint8 yytranslate[] = -{ - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 96, 97, 23, 21, 98, 22, 99, 24, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 95, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 25, 26, 27, 28, - 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, - 89, 90, 91, 92, 93, 94 -}; +static const yytype_uint8 yytranslate[] = { + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 96, 97, 23, 21, 98, 22, 99, 24, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 95, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94}; #if YYDEBUG /* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in YYRHS. */ -static const yytype_uint16 yyprhs[] = -{ - 0, 0, 3, 6, 10, 12, 18, 19, 21, 23, - 26, 28, 30, 32, 39, 41, 45, 47, 49, 53, - 54, 57, 60, 64, 69, 72, 75, 78, 83, 86, - 92, 97, 103, 111, 122, 127, 129, 133, 141, 142, - 146, 147, 151, 159, 160, 164, 167, 169, 171, 173, - 177, 179, 181, 183, 186, 189, 191, 195, 197, 199, - 201, 208, 209, 213, 215, 219, 222, 225, 226, 228, - 230, 232, 234, 236, 238, 240, 242, 244, 246, 248, - 250, 252, 254, 257, 260, 262, 270, 275, 280, 286, - 291, 293, 295, 299, 301, 303, 306, 309, 311, 318, - 319, 321, 323, 332, 334, 338, 342, 346, 352, 354, - 358, 360, 361, 363, 365, 369, 374, 376, 380, 385, - 387, 389, 394, 397, 399, 403, 405, 408, 411, 412, - 416, 418, 422, 423, 426, 430, 434, 437, 441, 443, - 445, 447, 449, 451, 453, 455, 457, 461, 465, 472, - 478, 484, 489, 490, 493, 498, 502, 509, 515, 522, - 528, 530, 534, 539, 541, 543, 545, 548, 555, 559, - 563, 567, 571, 574, 577, 579, 581, 583, 587, 589, - 593, 595, 597, 599, 601, 604, 608, 613, 619, 625, - 630, 632, 634, 636, 638, 642, 644, 646, 650, 656, - 658, 663, 665, 670, 677, 679, 684, 691, 693, 695, - 697, 702, 704, 707, 709, 711, 713, 715, 717, 722, - 726, 729 -}; +static const yytype_uint16 yyprhs[] = { + 0, 0, 3, 6, 10, 12, 18, 19, 21, 23, 26, 28, 30, 32, 39, 41, 45, 47, 49, 53, 54, + 57, 60, 64, 69, 72, 75, 78, 83, 86, 92, 97, 103, 111, 122, 127, 129, 133, 141, 142, 146, 147, + 151, 159, 160, 164, 167, 169, 171, 173, 177, 179, 181, 183, 186, 189, 191, 195, 197, 199, 201, 208, 209, + 213, 215, 219, 222, 225, 226, 228, 230, 232, 234, 236, 238, 240, 242, 244, 246, 248, 250, 252, 254, 257, + 260, 262, 270, 275, 280, 286, 291, 293, 295, 299, 301, 303, 306, 309, 311, 318, 319, 321, 323, 332, 334, + 338, 342, 346, 352, 354, 358, 360, 361, 363, 365, 369, 374, 376, 380, 385, 387, 389, 394, 397, 399, 403, + 405, 408, 411, 412, 416, 418, 422, 423, 426, 430, 434, 437, 441, 443, 445, 447, 449, 451, 453, 455, 457, + 461, 465, 472, 478, 484, 489, 490, 493, 498, 502, 509, 515, 522, 528, 530, 534, 539, 541, 543, 545, 548, + 555, 559, 563, 567, 571, 574, 577, 579, 581, 583, 587, 589, 593, 595, 597, 599, 601, 604, 608, 613, 619, + 625, 630, 632, 634, 636, 638, 642, 644, 646, 650, 656, 658, 663, 665, 670, 677, 679, 684, 691, 693, 695, + 697, 702, 704, 707, 709, 711, 713, 715, 717, 722, 726, 729}; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ -static const yytype_int16 yyrhs[] = -{ - 101, 0, -1, 102, 95, -1, 101, 102, 95, -1, - 103, -1, 36, 79, 28, 190, 104, -1, -1, 105, - -1, 106, -1, 105, 106, -1, 107, -1, 115, -1, - 118, -1, 36, 84, 182, 96, 108, 97, -1, 109, - -1, 108, 98, 109, -1, 110, -1, 113, -1, 186, - 185, 111, -1, -1, 111, 112, -1, 19, 10, -1, - 19, 10, 87, -1, 19, 10, 72, 62, -1, 41, - 181, -1, 41, 10, -1, 41, 11, -1, 32, 96, - 163, 97, -1, 77, 182, -1, 77, 182, 96, 114, - 97, -1, 87, 96, 114, 97, -1, 72, 62, 96, - 114, 97, -1, 50, 62, 96, 114, 97, 77, 182, - -1, 50, 62, 96, 114, 97, 77, 182, 96, 114, - 97, -1, 32, 96, 163, 97, -1, 186, -1, 114, - 98, 186, -1, 36, 90, 182, 117, 26, 153, 116, - -1, -1, 93, 32, 69, -1, -1, 96, 114, 97, - -1, 54, 120, 67, 182, 85, 123, 119, -1, -1, - 93, 54, 69, -1, 8, 73, -1, 8, -1, 121, - -1, 122, -1, 121, 98, 122, -1, 7, -1, 58, - -1, 42, -1, 88, 117, -1, 77, 117, -1, 124, - -1, 123, 98, 124, -1, 75, -1, 190, -1, 125, - -1, 40, 187, 38, 49, 151, 126, -1, -1, 70, - 30, 127, -1, 128, -1, 127, 98, 128, -1, 5, - 129, -1, 184, 129, -1, -1, 27, -1, 43, -1, - 130, -1, 131, -1, 132, -1, 133, -1, 134, -1, - 135, -1, 136, -1, 140, -1, 141, -1, 142, -1, - 144, -1, 147, -1, 33, 187, -1, 34, 94, -1, - 34, -1, 42, 52, 182, 92, 37, 66, 187, -1, - 42, 52, 182, 150, -1, 47, 187, 60, 148, -1, - 58, 60, 183, 117, 137, -1, 89, 96, 138, 97, - -1, 153, -1, 139, -1, 138, 98, 139, -1, 178, - -1, 10, -1, 68, 187, -1, 78, 94, -1, 78, - -1, 7, 143, 154, 60, 148, 155, -1, -1, 8, - -1, 9, -1, 88, 182, 80, 145, 92, 37, 66, - 187, -1, 146, -1, 145, 98, 146, -1, 186, 20, - 176, -1, 186, 20, 10, -1, 88, 182, 80, 145, - 150, -1, 149, -1, 148, 98, 149, -1, 179, -1, - -1, 159, -1, 152, -1, 151, 86, 152, -1, 151, - 86, 8, 152, -1, 153, -1, 96, 151, 97, -1, - 7, 143, 154, 155, -1, 177, -1, 23, -1, 156, - 150, 160, 162, -1, 52, 157, -1, 158, -1, 157, - 98, 158, -1, 182, -1, 182, 189, -1, 92, 163, - -1, -1, 55, 30, 161, -1, 184, -1, 161, 98, - 184, -1, -1, 56, 163, -1, 163, 17, 163, -1, - 163, 18, 163, -1, 19, 163, -1, 96, 163, 97, - -1, 164, -1, 165, -1, 166, -1, 167, -1, 169, - -1, 170, -1, 172, -1, 174, -1, 176, 20, 176, - -1, 176, 20, 175, -1, 176, 19, 29, 176, 18, - 176, -1, 176, 29, 176, 18, 176, -1, 176, 19, - 64, 178, 168, -1, 176, 64, 178, 168, -1, -1, - 45, 178, -1, 184, 61, 19, 10, -1, 184, 61, - 10, -1, 176, 19, 57, 96, 175, 97, -1, 176, - 57, 96, 175, 97, -1, 176, 19, 57, 96, 171, - 97, -1, 176, 57, 96, 171, 97, -1, 178, -1, - 171, 98, 178, -1, 176, 20, 173, 175, -1, 15, - -1, 8, -1, 16, -1, 46, 175, -1, 96, 7, - 143, 154, 155, 97, -1, 176, 21, 176, -1, 176, - 22, 176, -1, 176, 23, 176, -1, 176, 24, 176, - -1, 21, 176, -1, 22, 176, -1, 178, -1, 184, - -1, 180, -1, 96, 176, 97, -1, 176, -1, 177, - 98, 176, -1, 179, -1, 181, -1, 11, -1, 188, - -1, 188, 188, -1, 188, 12, 188, -1, 13, 96, - 23, 97, -1, 13, 96, 9, 184, 97, -1, 13, - 96, 8, 176, 97, -1, 13, 96, 176, 97, -1, - 4, -1, 5, -1, 6, -1, 183, -1, 3, 99, - 3, -1, 3, -1, 3, -1, 3, 99, 3, -1, - 3, 99, 3, 99, 3, -1, 31, -1, 31, 96, - 5, 97, -1, 65, -1, 65, 96, 5, 97, -1, - 65, 96, 5, 98, 5, 97, -1, 39, -1, 39, - 96, 5, 97, -1, 39, 96, 5, 98, 5, 97, - -1, 59, -1, 81, -1, 48, -1, 48, 96, 5, - 97, -1, 76, -1, 44, 71, -1, 3, -1, 3, - -1, 14, -1, 3, -1, 3, -1, 91, 19, 51, - 191, -1, 91, 83, 191, -1, 53, 3, -1, 35, - -1 -}; +static const yytype_int16 yyrhs[] = { + 101, 0, -1, 102, 95, -1, 101, 102, 95, -1, 103, -1, 36, 79, 28, 190, 104, -1, -1, 105, -1, + 106, -1, 105, 106, -1, 107, -1, 115, -1, 118, -1, 36, 84, 182, 96, 108, 97, -1, 109, -1, 108, + 98, 109, -1, 110, -1, 113, -1, 186, 185, 111, -1, -1, 111, 112, -1, 19, 10, -1, 19, 10, 87, + -1, 19, 10, 72, 62, -1, 41, 181, -1, 41, 10, -1, 41, 11, -1, 32, 96, 163, 97, -1, 77, + 182, -1, 77, 182, 96, 114, 97, -1, 87, 96, 114, 97, -1, 72, 62, 96, 114, 97, -1, 50, 62, + 96, 114, 97, 77, 182, -1, 50, 62, 96, 114, 97, 77, 182, 96, 114, 97, -1, 32, 96, 163, 97, + -1, 186, -1, 114, 98, 186, -1, 36, 90, 182, 117, 26, 153, 116, -1, -1, 93, 32, 69, -1, -1, + 96, 114, 97, -1, 54, 120, 67, 182, 85, 123, 119, -1, -1, 93, 54, 69, -1, 8, 73, -1, 8, + -1, 121, -1, 122, -1, 121, 98, 122, -1, 7, -1, 58, -1, 42, -1, 88, 117, -1, 77, 117, -1, + 124, -1, 123, 98, 124, -1, 75, -1, 190, -1, 125, -1, 40, 187, 38, 49, 151, 126, -1, -1, 70, + 30, 127, -1, 128, -1, 127, 98, 128, -1, 5, 129, -1, 184, 129, -1, -1, 27, -1, 43, -1, 130, + -1, 131, -1, 132, -1, 133, -1, 134, -1, 135, -1, 136, -1, 140, -1, 141, -1, 142, -1, 144, -1, + 147, -1, 33, 187, -1, 34, 94, -1, 34, -1, 42, 52, 182, 92, 37, 66, 187, -1, 42, 52, 182, + 150, -1, 47, 187, 60, 148, -1, 58, 60, 183, 117, 137, -1, 89, 96, 138, 97, -1, 153, -1, 139, + -1, 138, 98, 139, -1, 178, -1, 10, -1, 68, 187, -1, 78, 94, -1, 78, -1, 7, 143, 154, 60, + 148, 155, -1, -1, 8, -1, 9, -1, 88, 182, 80, 145, 92, 37, 66, 187, -1, 146, -1, 145, 98, + 146, -1, 186, 20, 176, -1, 186, 20, 10, -1, 88, 182, 80, 145, 150, -1, 149, -1, 148, 98, 149, + -1, 179, -1, -1, 159, -1, 152, -1, 151, 86, 152, -1, 151, 86, 8, 152, -1, 153, -1, 96, 151, + 97, -1, 7, 143, 154, 155, -1, 177, -1, 23, -1, 156, 150, 160, 162, -1, 52, 157, -1, 158, -1, + 157, 98, 158, -1, 182, -1, 182, 189, -1, 92, 163, -1, -1, 55, 30, 161, -1, 184, -1, 161, 98, + 184, -1, -1, 56, 163, -1, 163, 17, 163, -1, 163, 18, 163, -1, 19, 163, -1, 96, 163, 97, -1, + 164, -1, 165, -1, 166, -1, 167, -1, 169, -1, 170, -1, 172, -1, 174, -1, 176, 20, 176, -1, 176, + 20, 175, -1, 176, 19, 29, 176, 18, 176, -1, 176, 29, 176, 18, 176, -1, 176, 19, 64, 178, 168, + -1, 176, 64, 178, 168, -1, -1, 45, 178, -1, 184, 61, 19, 10, -1, 184, 61, 10, -1, 176, 19, + 57, 96, 175, 97, -1, 176, 57, 96, 175, 97, -1, 176, 19, 57, 96, 171, 97, -1, 176, 57, 96, + 171, 97, -1, 178, -1, 171, 98, 178, -1, 176, 20, 173, 175, -1, 15, -1, 8, -1, 16, -1, 46, + 175, -1, 96, 7, 143, 154, 155, 97, -1, 176, 21, 176, -1, 176, 22, 176, -1, 176, 23, 176, -1, + 176, 24, 176, -1, 21, 176, -1, 22, 176, -1, 178, -1, 184, -1, 180, -1, 96, 176, 97, -1, 176, + -1, 177, 98, 176, -1, 179, -1, 181, -1, 11, -1, 188, -1, 188, 188, -1, 188, 12, 188, -1, 13, + 96, 23, 97, -1, 13, 96, 9, 184, 97, -1, 13, 96, 8, 176, 97, -1, 13, 96, 176, 97, -1, + 4, -1, 5, -1, 6, -1, 183, -1, 3, 99, 3, -1, 3, -1, 3, -1, 3, 99, 3, -1, 3, + 99, 3, 99, 3, -1, 31, -1, 31, 96, 5, 97, -1, 65, -1, 65, 96, 5, 97, -1, 65, 96, + 5, 98, 5, 97, -1, 39, -1, 39, 96, 5, 97, -1, 39, 96, 5, 98, 5, 97, -1, 59, -1, + 81, -1, 48, -1, 48, 96, 5, 97, -1, 76, -1, 44, 71, -1, 3, -1, 3, -1, 14, -1, 3, + -1, 3, -1, 91, 19, 51, 191, -1, 91, 83, 191, -1, 53, 3, -1, 35, -1}; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ -static const yytype_uint16 yyrline[] = -{ - 0, 244, 244, 256, 267, 271, 274, 276, 280, 281, - 285, 286, 287, 291, 295, 296, 300, 301, 305, 308, - 310, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 326, 327, 328, 330, 332, 336, 341, 349, 353, 355, - 359, 360, 364, 368, 370, 374, 375, 376, 380, 381, - 385, 386, 387, 388, 389, 394, 395, 399, 400, 405, - 410, 413, 415, 419, 420, 424, 425, 428, 430, 431, - 436, 440, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 454, 458, 462, 470, 478, 485, 489, 499, - 503, 510, 515, 523, 524, 528, 532, 536, 543, 549, - 550, 551, 555, 563, 568, 576, 583, 593, 600, 601, - 605, 609, 610, 616, 617, 618, 622, 623, 627, 639, - 640, 644, 658, 666, 671, 679, 680, 684, 692, 693, - 701, 706, 714, 715, 725, 732, 739, 745, 749, 758, - 762, 766, 770, 774, 778, 782, 789, 796, 806, 815, - 827, 835, 846, 847, 855, 861, 870, 877, 884, 892, - 903, 908, 916, 928, 929, 930, 934, 942, 956, 963, - 970, 977, 984, 990, 996, 997, 998, 999, 1003, 1009, - 1017, 1018, 1019, 1023, 1024, 1031, 1043, 1051, 1062, 1072, - 1083, 1084, 1085, 1091, 1095, 1096, 1107, 1108, 1115, 1129, - 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, - 1140, 1141, 1142, 1149, 1152, 1156, 1159, 1162, 1166, 1167, - 1170, 1171 -}; +static const yytype_uint16 yyrline[] = { + 0, 244, 244, 256, 267, 271, 274, 276, 280, 281, 285, 286, 287, 291, 295, 296, + 300, 301, 305, 308, 310, 314, 315, 316, 317, 318, 319, 320, 321, 322, 326, 327, + 328, 330, 332, 336, 341, 349, 353, 355, 359, 360, 364, 368, 370, 374, 375, 376, + 380, 381, 385, 386, 387, 388, 389, 394, 395, 399, 400, 405, 410, 413, 415, 419, + 420, 424, 425, 428, 430, 431, 436, 440, 441, 442, 443, 444, 445, 446, 447, 448, + 449, 450, 454, 458, 462, 470, 478, 485, 489, 499, 503, 510, 515, 523, 524, 528, + 532, 536, 543, 549, 550, 551, 555, 563, 568, 576, 583, 593, 600, 601, 605, 609, + 610, 616, 617, 618, 622, 623, 627, 639, 640, 644, 658, 666, 671, 679, 680, 684, + 692, 693, 701, 706, 714, 715, 725, 732, 739, 745, 749, 758, 762, 766, 770, 774, + 778, 782, 789, 796, 806, 815, 827, 835, 846, 847, 855, 861, 870, 877, 884, 892, + 903, 908, 916, 928, 929, 930, 934, 942, 956, 963, 970, 977, 984, 990, 996, 997, + 998, 999, 1003, 1009, 1017, 1018, 1019, 1023, 1024, 1031, 1043, 1051, 1062, 1072, 1083, 1084, + 1085, 1091, 1095, 1096, 1107, 1108, 1115, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, + 1138, 1139, 1140, 1141, 1142, 1149, 1152, 1156, 1159, 1162, 1166, 1167, 1170, 1171}; #endif #if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. First, the terminals, then, starting at YYNTOKENS, nonterminals. */ -static const char* const yytname[] = -{ - "$end", "error", "$undefined", "NAME", "STRING", "INTNUM", "APPROXNUM", - "SELECT", "ALL", "DISTINCT", "NULLX", "USER", "INDICATOR", "AMMSC", - "PARAMETER", "ANY", "SOME", "OR", "AND", "NOT", "COMPARISON", "'+'", - "'-'", "'*'", "'/'", "UMINUS", "AS", "ASC", "AUTHORIZATION", "BETWEEN", - "BY", "CHARACTER", "CHECK", "CLOSE", "COMMIT", "CONTINUE", "CREATE", - "CURRENT", "CURSOR", "IDB_DECIMAL", "DECLARE", "DEFAULT", "DELETE", - "DESC", "IDB_DOUBLE", "ESCAPE", "EXISTS", "FETCH", "IDB_FLOAT", "FOR", - "FOREIGN", "FOUND", "FROM", "GOTO", "GRANT", "IDB_GROUP", "HAVING", "IN", - "INSERT", "INTEGER", "INTO", "IS", "KEY", "LANGUAGE", "LIKE", "NUMERIC", - "OF", "ON", "OPEN", "OPTION", "ORDER", "PRECISION", "PRIMARY", - "PRIVILEGES", "PROCEDURE", "PUBLIC", "REAL", "REFERENCES", "ROLLBACK", - "SCHEMA", "SET", "SMALLINT", "SQLCODE", "SQLERROR", "TABLE", "TO", - "UNION", "UNIQUE", "UPDATE", "VALUES", "VIEW", "WHENEVER", "WHERE", - "WITH", "WORK", "';'", "'('", "')'", "','", "'.'", "$accept", "sql_list", - "sql", "schema", "opt_schema_element_list", "schema_element_list", - "schema_element", "base_table_def", "base_table_element_commalist", - "base_table_element", "column_def", "column_def_opt_list", - "column_def_opt", "table_constraint_def", "column_commalist", "view_def", - "opt_with_check_option", "opt_column_commalist", "privilege_def", - "opt_with_grant_option", "privileges", "operation_commalist", - "operation", "grantee_commalist", "grantee", "cursor_def", - "opt_order_by_clause", "ordering_spec_commalist", "ordering_spec", - "opt_asc_desc", "manipulative_statement", "close_statement", - "commit_statement", "delete_statement_positioned", - "delete_statement_searched", "fetch_statement", "insert_statement", - "values_or_query_spec", "insert_atom_commalist", "insert_atom", - "open_statement", "rollback_statement", "select_statement", - "opt_all_distinct", "update_statement_positioned", - "assignment_commalist", "assignment", "update_statement_searched", - "target_commalist", "target", "opt_where_clause", "query_exp", - "query_term", "query_spec", "selection", "table_exp", "from_clause", - "table_ref_commalist", "table_ref", "where_clause", - "opt_group_by_clause", "column_ref_commalist", "opt_having_clause", - "search_condition", "predicate", "comparison_predicate", - "between_predicate", "like_predicate", "opt_escape", "test_for_null", - "in_predicate", "atom_commalist", "all_or_any_predicate", "any_all_some", - "existence_test", "subquery", "scalar_exp", "scalar_exp_commalist", - "atom", "parameter_ref", "function_ref", "literal", "table", - "table_name", "column_ref", "data_type", "column", "cursor", "parameter", - "range_variable", "user", "when_action", 0 -}; +static const char* const yytname[] = {"$end", + "error", + "$undefined", + "NAME", + "STRING", + "INTNUM", + "APPROXNUM", + "SELECT", + "ALL", + "DISTINCT", + "NULLX", + "USER", + "INDICATOR", + "AMMSC", + "PARAMETER", + "ANY", + "SOME", + "OR", + "AND", + "NOT", + "COMPARISON", + "'+'", + "'-'", + "'*'", + "'/'", + "UMINUS", + "AS", + "ASC", + "AUTHORIZATION", + "BETWEEN", + "BY", + "CHARACTER", + "CHECK", + "CLOSE", + "COMMIT", + "CONTINUE", + "CREATE", + "CURRENT", + "CURSOR", + "IDB_DECIMAL", + "DECLARE", + "DEFAULT", + "DELETE", + "DESC", + "IDB_DOUBLE", + "ESCAPE", + "EXISTS", + "FETCH", + "IDB_FLOAT", + "FOR", + "FOREIGN", + "FOUND", + "FROM", + "GOTO", + "GRANT", + "IDB_GROUP", + "HAVING", + "IN", + "INSERT", + "INTEGER", + "INTO", + "IS", + "KEY", + "LANGUAGE", + "LIKE", + "NUMERIC", + "OF", + "ON", + "OPEN", + "OPTION", + "ORDER", + "PRECISION", + "PRIMARY", + "PRIVILEGES", + "PROCEDURE", + "PUBLIC", + "REAL", + "REFERENCES", + "ROLLBACK", + "SCHEMA", + "SET", + "SMALLINT", + "SQLCODE", + "SQLERROR", + "TABLE", + "TO", + "UNION", + "UNIQUE", + "UPDATE", + "VALUES", + "VIEW", + "WHENEVER", + "WHERE", + "WITH", + "WORK", + "';'", + "'('", + "')'", + "','", + "'.'", + "$accept", + "sql_list", + "sql", + "schema", + "opt_schema_element_list", + "schema_element_list", + "schema_element", + "base_table_def", + "base_table_element_commalist", + "base_table_element", + "column_def", + "column_def_opt_list", + "column_def_opt", + "table_constraint_def", + "column_commalist", + "view_def", + "opt_with_check_option", + "opt_column_commalist", + "privilege_def", + "opt_with_grant_option", + "privileges", + "operation_commalist", + "operation", + "grantee_commalist", + "grantee", + "cursor_def", + "opt_order_by_clause", + "ordering_spec_commalist", + "ordering_spec", + "opt_asc_desc", + "manipulative_statement", + "close_statement", + "commit_statement", + "delete_statement_positioned", + "delete_statement_searched", + "fetch_statement", + "insert_statement", + "values_or_query_spec", + "insert_atom_commalist", + "insert_atom", + "open_statement", + "rollback_statement", + "select_statement", + "opt_all_distinct", + "update_statement_positioned", + "assignment_commalist", + "assignment", + "update_statement_searched", + "target_commalist", + "target", + "opt_where_clause", + "query_exp", + "query_term", + "query_spec", + "selection", + "table_exp", + "from_clause", + "table_ref_commalist", + "table_ref", + "where_clause", + "opt_group_by_clause", + "column_ref_commalist", + "opt_having_clause", + "search_condition", + "predicate", + "comparison_predicate", + "between_predicate", + "like_predicate", + "opt_escape", + "test_for_null", + "in_predicate", + "atom_commalist", + "all_or_any_predicate", + "any_all_some", + "existence_test", + "subquery", + "scalar_exp", + "scalar_exp_commalist", + "atom", + "parameter_ref", + "function_ref", + "literal", + "table", + "table_name", + "column_ref", + "data_type", + "column", + "cursor", + "parameter", + "range_variable", + "user", + "when_action", + 0}; #endif -# ifdef YYPRINT +#ifdef YYPRINT /* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to token YYLEX-NUM. */ -static const yytype_uint16 yytoknum[] = -{ - 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 43, 45, 42, 47, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, - 341, 342, 343, 344, 345, 59, 40, 41, 44, 46 -}; -# endif +static const yytype_uint16 yytoknum[] = { + 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 43, 45, 42, 47, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, + 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, + 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 59, 40, 41, 44, 46}; +#endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = -{ - 0, 100, 101, 101, 102, 103, 104, 104, 105, 105, - 106, 106, 106, 107, 108, 108, 109, 109, 110, 111, - 111, 112, 112, 112, 112, 112, 112, 112, 112, 112, - 113, 113, 113, 113, 113, 114, 114, 115, 116, 116, - 117, 117, 118, 119, 119, 120, 120, 120, 121, 121, - 122, 122, 122, 122, 122, 123, 123, 124, 124, 102, - 125, 126, 126, 127, 127, 128, 128, 129, 129, 129, - 102, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 131, 132, 132, 133, 134, 135, 136, 137, - 137, 138, 138, 139, 139, 140, 141, 141, 142, 143, - 143, 143, 144, 145, 145, 146, 146, 147, 148, 148, - 149, 150, 150, 151, 151, 151, 152, 152, 153, 154, - 154, 155, 156, 157, 157, 158, 158, 159, 160, 160, - 161, 161, 162, 162, 163, 163, 163, 163, 163, 164, - 164, 164, 164, 164, 164, 164, 165, 165, 166, 166, - 167, 167, 168, 168, 169, 169, 170, 170, 170, 170, - 171, 171, 172, 173, 173, 173, 174, 175, 176, 176, - 176, 176, 176, 176, 176, 176, 176, 176, 177, 177, - 178, 178, 178, 179, 179, 179, 180, 180, 180, 180, - 181, 181, 181, 182, 183, 183, 184, 184, 184, 185, - 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, - 185, 185, 185, 186, 187, 188, 189, 190, 102, 102, - 191, 191 -}; +static const yytype_uint8 yyr1[] = { + 0, 100, 101, 101, 102, 103, 104, 104, 105, 105, 106, 106, 106, 107, 108, 108, 109, 109, 110, 111, 111, + 112, 112, 112, 112, 112, 112, 112, 112, 112, 113, 113, 113, 113, 113, 114, 114, 115, 116, 116, 117, 117, + 118, 119, 119, 120, 120, 120, 121, 121, 122, 122, 122, 122, 122, 123, 123, 124, 124, 102, 125, 126, 126, + 127, 127, 128, 128, 129, 129, 129, 102, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 131, 132, + 132, 133, 134, 135, 136, 137, 137, 138, 138, 139, 139, 140, 141, 141, 142, 143, 143, 143, 144, 145, 145, + 146, 146, 147, 148, 148, 149, 150, 150, 151, 151, 151, 152, 152, 153, 154, 154, 155, 156, 157, 157, 158, + 158, 159, 160, 160, 161, 161, 162, 162, 163, 163, 163, 163, 163, 164, 164, 164, 164, 164, 164, 164, 165, + 165, 166, 166, 167, 167, 168, 168, 169, 169, 170, 170, 170, 170, 171, 171, 172, 173, 173, 173, 174, 175, + 176, 176, 176, 176, 176, 176, 176, 176, 176, 176, 177, 177, 178, 178, 178, 179, 179, 179, 180, 180, 180, + 180, 181, 181, 181, 182, 183, 183, 184, 184, 184, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, + 185, 185, 185, 186, 187, 188, 189, 190, 102, 102, 191, 191}; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = -{ - 0, 2, 2, 3, 1, 5, 0, 1, 1, 2, - 1, 1, 1, 6, 1, 3, 1, 1, 3, 0, - 2, 2, 3, 4, 2, 2, 2, 4, 2, 5, - 4, 5, 7, 10, 4, 1, 3, 7, 0, 3, - 0, 3, 7, 0, 3, 2, 1, 1, 1, 3, - 1, 1, 1, 2, 2, 1, 3, 1, 1, 1, - 6, 0, 3, 1, 3, 2, 2, 0, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 2, 2, 1, 7, 4, 4, 5, 4, - 1, 1, 3, 1, 1, 2, 2, 1, 6, 0, - 1, 1, 8, 1, 3, 3, 3, 5, 1, 3, - 1, 0, 1, 1, 3, 4, 1, 3, 4, 1, - 1, 4, 2, 1, 3, 1, 2, 2, 0, 3, - 1, 3, 0, 2, 3, 3, 2, 3, 1, 1, - 1, 1, 1, 1, 1, 1, 3, 3, 6, 5, - 5, 4, 0, 2, 4, 3, 6, 5, 6, 5, - 1, 3, 4, 1, 1, 1, 2, 6, 3, 3, - 3, 3, 2, 2, 1, 1, 1, 3, 1, 3, - 1, 1, 1, 1, 2, 3, 4, 5, 5, 4, - 1, 1, 1, 1, 3, 1, 1, 3, 5, 1, - 4, 1, 4, 6, 1, 4, 6, 1, 1, 1, - 4, 1, 2, 1, 1, 1, 1, 1, 4, 3, - 2, 1 -}; +static const yytype_uint8 yyr2[] = { + 0, 2, 2, 3, 1, 5, 0, 1, 1, 2, 1, 1, 1, 6, 1, 3, 1, 1, 3, 0, 2, 2, 3, 4, 2, 2, 2, 4, 2, 5, 4, 5, + 7, 10, 4, 1, 3, 7, 0, 3, 0, 3, 7, 0, 3, 2, 1, 1, 1, 3, 1, 1, 1, 2, 2, 1, 3, 1, 1, 1, 6, 0, 3, 1, + 3, 2, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 7, 4, 4, 5, 4, 1, 1, 3, 1, 1, 2, + 2, 1, 6, 0, 1, 1, 8, 1, 3, 3, 3, 5, 1, 3, 1, 0, 1, 1, 3, 4, 1, 3, 4, 1, 1, 4, 2, 1, 3, 1, 2, 2, + 0, 3, 1, 3, 0, 2, 3, 3, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 6, 5, 5, 4, 0, 2, 4, 3, 6, 5, 6, 5, + 1, 3, 4, 1, 1, 1, 2, 6, 3, 3, 3, 3, 2, 2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2, 3, 4, 5, 5, 4, 1, 1, + 1, 1, 3, 1, 1, 3, 5, 1, 4, 1, 4, 6, 1, 4, 6, 1, 1, 1, 4, 1, 2, 1, 1, 1, 1, 1, 4, 3, 2, 1}; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state STATE-NUM when YYTABLE doesn't specify something else to do. Zero means the default is an error. */ -static const yytype_uint8 yydefact[] = -{ - 0, 99, 0, 84, 0, 0, 0, 0, 0, 0, - 97, 0, 0, 0, 0, 4, 59, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 79, 80, 81, 100, - 101, 0, 214, 82, 83, 0, 0, 0, 0, 0, - 95, 96, 195, 0, 193, 0, 0, 1, 0, 2, - 196, 190, 191, 192, 182, 0, 215, 0, 0, 120, - 0, 0, 178, 119, 174, 180, 176, 181, 175, 183, - 0, 0, 111, 0, 40, 0, 0, 0, 221, 0, - 219, 3, 0, 0, 172, 173, 0, 0, 0, 0, - 0, 0, 0, 0, 184, 217, 6, 0, 0, 86, - 112, 87, 108, 110, 0, 0, 194, 213, 111, 103, - 0, 218, 220, 197, 0, 0, 0, 0, 177, 0, - 168, 169, 170, 171, 179, 185, 0, 0, 5, 7, - 8, 10, 11, 12, 99, 0, 61, 113, 116, 0, - 0, 0, 0, 127, 138, 139, 140, 141, 142, 143, - 144, 145, 0, 175, 0, 0, 35, 0, 88, 90, - 0, 0, 107, 0, 0, 0, 0, 186, 189, 0, - 98, 111, 0, 0, 50, 46, 52, 51, 40, 40, - 0, 47, 48, 9, 0, 0, 0, 0, 60, 136, - 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 109, 41, 0, 0, 0, 104, 106, - 105, 198, 188, 187, 122, 123, 125, 0, 128, 0, - 40, 45, 54, 53, 0, 0, 0, 117, 0, 0, - 114, 85, 99, 137, 134, 135, 0, 0, 0, 164, - 163, 165, 0, 0, 147, 146, 0, 0, 152, 155, - 0, 36, 94, 0, 91, 93, 0, 0, 216, 126, - 0, 132, 0, 0, 0, 49, 118, 67, 62, 63, - 67, 115, 0, 0, 0, 152, 162, 0, 0, 0, - 160, 0, 151, 154, 89, 0, 102, 124, 0, 0, - 121, 0, 0, 0, 0, 0, 14, 16, 17, 0, - 0, 0, 68, 69, 65, 0, 66, 0, 0, 0, - 0, 150, 149, 159, 0, 157, 153, 92, 129, 130, - 133, 0, 0, 0, 0, 13, 0, 199, 204, 0, - 209, 207, 201, 211, 208, 19, 38, 57, 43, 55, - 58, 64, 0, 148, 158, 156, 161, 0, 0, 0, - 0, 0, 15, 0, 0, 212, 0, 0, 18, 0, - 37, 0, 0, 42, 167, 131, 34, 0, 0, 30, - 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, - 0, 56, 0, 31, 200, 205, 0, 210, 202, 0, - 21, 0, 25, 26, 24, 28, 39, 44, 0, 0, - 0, 0, 22, 0, 0, 32, 206, 203, 23, 27, - 0, 0, 29, 0, 33 -}; +static const yytype_uint8 yydefact[] = { + 0, 99, 0, 84, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 4, 59, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 100, 101, 0, 214, 82, 83, 0, 0, 0, 0, 0, 95, 96, + 195, 0, 193, 0, 0, 1, 0, 2, 196, 190, 191, 192, 182, 0, 215, 0, 0, 120, 0, 0, 178, + 119, 174, 180, 176, 181, 175, 183, 0, 0, 111, 0, 40, 0, 0, 0, 221, 0, 219, 3, 0, 0, + 172, 173, 0, 0, 0, 0, 0, 0, 0, 0, 184, 217, 6, 0, 0, 86, 112, 87, 108, 110, 0, + 0, 194, 213, 111, 103, 0, 218, 220, 197, 0, 0, 0, 0, 177, 0, 168, 169, 170, 171, 179, 185, + 0, 0, 5, 7, 8, 10, 11, 12, 99, 0, 61, 113, 116, 0, 0, 0, 0, 127, 138, 139, 140, + 141, 142, 143, 144, 145, 0, 175, 0, 0, 35, 0, 88, 90, 0, 0, 107, 0, 0, 0, 0, 186, + 189, 0, 98, 111, 0, 0, 50, 46, 52, 51, 40, 40, 0, 47, 48, 9, 0, 0, 0, 0, 60, + 136, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 41, 0, 0, 0, 104, 106, + 105, 198, 188, 187, 122, 123, 125, 0, 128, 0, 40, 45, 54, 53, 0, 0, 0, 117, 0, 0, 114, + 85, 99, 137, 134, 135, 0, 0, 0, 164, 163, 165, 0, 0, 147, 146, 0, 0, 152, 155, 0, 36, + 94, 0, 91, 93, 0, 0, 216, 126, 0, 132, 0, 0, 0, 49, 118, 67, 62, 63, 67, 115, 0, + 0, 0, 152, 162, 0, 0, 0, 160, 0, 151, 154, 89, 0, 102, 124, 0, 0, 121, 0, 0, 0, + 0, 0, 14, 16, 17, 0, 0, 0, 68, 69, 65, 0, 66, 0, 0, 0, 0, 150, 149, 159, 0, + 157, 153, 92, 129, 130, 133, 0, 0, 0, 0, 13, 0, 199, 204, 0, 209, 207, 201, 211, 208, 19, + 38, 57, 43, 55, 58, 64, 0, 148, 158, 156, 161, 0, 0, 0, 0, 0, 15, 0, 0, 212, 0, + 0, 18, 0, 37, 0, 0, 42, 167, 131, 34, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, + 20, 0, 0, 56, 0, 31, 200, 205, 0, 210, 202, 0, 21, 0, 25, 26, 24, 28, 39, 44, 0, + 0, 0, 0, 22, 0, 0, 32, 206, 203, 23, 27, 0, 0, 29, 0, 33}; /* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int16 yydefgoto[] = -{ - -1, 13, 14, 15, 128, 129, 130, 131, 295, 296, - 297, 358, 378, 298, 155, 132, 360, 105, 133, 363, - 180, 181, 182, 338, 339, 16, 188, 268, 269, 304, - 17, 18, 19, 20, 21, 22, 23, 158, 253, 254, - 24, 25, 26, 31, 27, 108, 109, 28, 101, 102, - 99, 136, 137, 138, 61, 170, 171, 214, 215, 100, - 261, 318, 290, 143, 144, 145, 146, 147, 282, 148, - 149, 278, 150, 243, 151, 192, 152, 63, 64, 65, - 66, 67, 216, 44, 68, 335, 156, 33, 69, 259, - 340, 80 - }; +static const yytype_int16 yydefgoto[] = { + -1, 13, 14, 15, 128, 129, 130, 131, 295, 296, 297, 358, 378, 298, 155, 132, 360, 105, 133, + 363, 180, 181, 182, 338, 339, 16, 188, 268, 269, 304, 17, 18, 19, 20, 21, 22, 23, 158, + 253, 254, 24, 25, 26, 31, 27, 108, 109, 28, 101, 102, 99, 136, 137, 138, 61, 170, 171, + 214, 215, 100, 261, 318, 290, 143, 144, 145, 146, 147, 282, 148, 149, 278, 150, 243, 151, 192, + 152, 63, 64, 65, 66, 67, 216, 44, 68, 335, 156, 33, 69, 259, 340, 80}; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ #define YYPACT_NINF -270 -static const yytype_int16 yypact[] = -{ - 352, 101, 20, -63, -25, 20, 30, 20, 90, 20, - 59, 185, 81, 337, 104, -270, -270, -270, -270, -270, - -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, - -270, 286, -270, -270, -270, 191, 202, 185, 156, 185, - -270, -270, 138, 162, -270, 196, -3, -270, 154, -270, - 168, -270, -270, -270, -270, 160, -270, 320, 320, -270, - 320, 192, 341, 172, -270, -270, -270, -270, -270, 153, - 274, 232, 206, 269, 214, 299, 317, -3, -270, 324, - -270, -270, 340, 16, -270, -270, 135, 269, 320, 320, - 320, 320, 320, 269, -270, -270, 91, 21, 100, -270, - -270, 234, -270, -270, 317, -1, -270, -270, 97, -270, - 336, -270, -270, 261, 320, 366, 243, 149, -270, 78, - 199, 199, -270, -270, 341, -270, 127, 66, -270, 91, - -270, -270, -270, -270, 101, 21, 82, -270, -270, 282, - 306, 295, 282, 213, -270, -270, -270, -270, -270, -270, - -270, -270, 326, 332, 269, 178, -270, 300, -270, -270, - 222, 317, -270, 73, 395, 161, 312, -270, -270, 185, - -270, 311, 185, 185, -270, 346, -270, -270, 214, 214, - 357, 316, -270, -270, 286, 50, 392, 33, -270, -270, - 20, 422, -270, 26, 181, 282, 282, 157, 258, 320, - 335, 407, 122, -270, -270, 317, 347, 367, -270, -270, - 341, -270, -270, -270, 344, -270, 451, 282, 400, 360, - 214, -270, -270, -270, 185, 121, 405, -270, 175, 21, - -270, -270, 101, -270, 440, -270, 320, 363, 407, -270, - -270, -270, 308, 295, -270, 341, 414, 42, 415, -270, - 452, -270, -270, 208, -270, -270, 20, 185, -270, -270, - 431, 408, 94, 437, 380, -270, -270, 150, 368, -270, - 150, -270, 286, 423, 42, 415, -270, 320, 219, 371, - -270, 407, -270, -270, -270, 347, -270, -270, 366, 282, - -270, 373, 409, 410, 374, 238, -270, -270, -270, 358, - 460, 32, -270, -270, -270, 175, -270, 405, 320, 241, - 376, -270, 341, -270, 407, -270, -270, -270, 377, -270, - 213, 282, 378, 381, 317, -270, 94, 382, 383, 411, - 384, -270, 385, -270, -270, -270, 390, -270, 141, -270, - -270, -270, 379, 341, -270, -270, -270, 366, 52, 317, - 317, 270, -270, 479, 480, -270, 481, 482, 188, 456, - -270, 435, 32, -270, -270, -270, -270, 303, 310, -270, - 393, 329, 394, 351, 483, 396, 370, 185, -270, 425, - 426, -270, 419, -270, -270, -270, 492, -270, -270, 493, - -12, 282, -270, -270, -270, 403, -270, -270, 185, 404, - 406, 438, -270, 54, 317, 412, -270, -270, -270, -270, - 353, 317, -270, 355, -270 -}; +static const yytype_int16 yypact[] = { + 352, 101, 20, -63, -25, 20, 30, 20, 90, 20, 59, 185, 81, 337, 104, -270, -270, + -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, -270, 286, -270, -270, + -270, 191, 202, 185, 156, 185, -270, -270, 138, 162, -270, 196, -3, -270, 154, -270, 168, + -270, -270, -270, -270, 160, -270, 320, 320, -270, 320, 192, 341, 172, -270, -270, -270, -270, + -270, 153, 274, 232, 206, 269, 214, 299, 317, -3, -270, 324, -270, -270, 340, 16, -270, + -270, 135, 269, 320, 320, 320, 320, 320, 269, -270, -270, 91, 21, 100, -270, -270, 234, + -270, -270, 317, -1, -270, -270, 97, -270, 336, -270, -270, 261, 320, 366, 243, 149, -270, + 78, 199, 199, -270, -270, 341, -270, 127, 66, -270, 91, -270, -270, -270, -270, 101, 21, + 82, -270, -270, 282, 306, 295, 282, 213, -270, -270, -270, -270, -270, -270, -270, -270, 326, + 332, 269, 178, -270, 300, -270, -270, 222, 317, -270, 73, 395, 161, 312, -270, -270, 185, + -270, 311, 185, 185, -270, 346, -270, -270, 214, 214, 357, 316, -270, -270, 286, 50, 392, + 33, -270, -270, 20, 422, -270, 26, 181, 282, 282, 157, 258, 320, 335, 407, 122, -270, + -270, 317, 347, 367, -270, -270, 341, -270, -270, -270, 344, -270, 451, 282, 400, 360, 214, + -270, -270, -270, 185, 121, 405, -270, 175, 21, -270, -270, 101, -270, 440, -270, 320, 363, + 407, -270, -270, -270, 308, 295, -270, 341, 414, 42, 415, -270, 452, -270, -270, 208, -270, + -270, 20, 185, -270, -270, 431, 408, 94, 437, 380, -270, -270, 150, 368, -270, 150, -270, + 286, 423, 42, 415, -270, 320, 219, 371, -270, 407, -270, -270, -270, 347, -270, -270, 366, + 282, -270, 373, 409, 410, 374, 238, -270, -270, -270, 358, 460, 32, -270, -270, -270, 175, + -270, 405, 320, 241, 376, -270, 341, -270, 407, -270, -270, -270, 377, -270, 213, 282, 378, + 381, 317, -270, 94, 382, 383, 411, 384, -270, 385, -270, -270, -270, 390, -270, 141, -270, + -270, -270, 379, 341, -270, -270, -270, 366, 52, 317, 317, 270, -270, 479, 480, -270, 481, + 482, 188, 456, -270, 435, 32, -270, -270, -270, -270, 303, 310, -270, 393, 329, 394, 351, + 483, 396, 370, 185, -270, 425, 426, -270, 419, -270, -270, -270, 492, -270, -270, 493, -12, + 282, -270, -270, -270, 403, -270, -270, 185, 404, 406, 438, -270, 54, 317, 412, -270, -270, + -270, -270, 353, 317, -270, 355, -270}; /* YYPGOTO[NTERM-NUM]. */ -static const yytype_int16 yypgoto[] = -{ - -270, -270, 489, -270, -270, -270, 375, -270, -270, 179, - -270, -270, -270, -270, -269, -270, -270, -163, -270, -270, - -270, -270, 281, -270, 145, -270, -270, -270, 204, 240, - -270, -270, -270, -270, -270, -270, -270, -270, -270, 226, - -270, -270, -270, -131, -270, -270, 354, -270, 427, 359, - -56, 386, -136, -103, -170, -216, -270, -270, 255, -270, - -270, -270, -270, -134, -270, -270, -270, -270, 242, -270, - -270, 244, -270, -270, -270, 8, -24, -270, -189, 61, - -270, 140, -11, 484, -97, -270, -72, 4, 47, -270, - 449, 443 - }; +static const yytype_int16 yypgoto[] = { + -270, -270, 489, -270, -270, -270, 375, -270, -270, 179, -270, -270, -270, -270, -269, -270, + -270, -163, -270, -270, -270, -270, 281, -270, 145, -270, -270, -270, 204, 240, -270, -270, + -270, -270, -270, -270, -270, -270, -270, 226, -270, -270, -270, -131, -270, -270, 354, -270, + 427, 359, -56, 386, -136, -103, -170, -216, -270, -270, 255, -270, -270, -270, -270, -134, + -270, -270, -270, -270, 242, -270, -270, 244, -270, -270, -270, 8, -24, -270, -189, 61, + -270, 140, -11, 484, -97, -270, -72, 4, 47, -270, 449, 443}; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ #define YYTABLE_NINF -1 -static const yytype_uint16 yytable[] = -{ - 43, 153, 159, 184, 110, 189, 134, 62, 193, 36, - 266, 38, 248, 40, 226, 222, 223, 255, 166, 50, - 51, 52, 53, 32, 114, 115, 72, 54, 134, 55, - 56, 34, 78, 84, 85, 95, 86, 57, 58, 116, - 134, 229, 153, 195, 196, 153, 51, 52, 53, 275, - 79, 230, 162, 54, 35, 351, 56, 263, 280, 117, - 401, 234, 235, 153, 120, 121, 122, 123, 124, 195, - 196, 195, 196, 174, 175, 402, 50, 51, 52, 53, - 367, 368, 37, 209, 54, 280, 55, 56, 157, 110, - 165, 342, 316, 271, 57, 58, 255, 107, 153, 153, - 45, 272, 307, 50, 51, 52, 53, 337, 176, 29, - 30, 54, 60, 55, 56, 218, 94, 135, 194, 139, - 153, 57, 58, 233, 177, 346, 291, 126, 174, 135, - 169, 270, 249, 251, 103, 410, 187, 140, 191, 210, - 125, 250, 413, 178, 292, 127, 141, 227, 103, 366, - 39, 409, 186, 41, 179, 320, 88, 89, 90, 91, - 62, 219, 220, 176, 46, 93, 293, 56, 187, 60, - 88, 89, 90, 91, 245, 246, 154, 302, 50, 177, - 267, 294, 88, 89, 90, 91, 236, 348, 42, 160, - 299, 319, 153, 303, 231, 161, 142, 336, 178, 49, - 197, 198, 88, 89, 90, 91, 244, 374, 270, 179, - 199, 172, 273, 264, 237, 103, 73, 173, 86, 70, - 375, 238, 90, 91, 153, 50, 51, 52, 53, 376, - 195, 196, 118, 54, 361, 55, 56, 75, 200, 362, - 71, 139, 76, 57, 58, 201, 168, 77, 62, 81, - 365, 276, 87, 312, 299, 279, 83, 403, 212, 207, - 286, 50, 51, 52, 53, 377, 239, 82, 141, 54, - 92, 55, 56, 240, 241, 204, 205, 95, 118, 57, - 58, 97, 310, 56, 343, 50, 51, 52, 53, 50, - 51, 52, 53, 54, 153, 55, 56, 54, 98, 55, - 56, 139, 106, 57, 58, 284, 285, 57, 58, 59, - 104, 50, 51, 52, 53, 232, 313, 314, 142, 54, - 107, 55, 56, 50, 51, 52, 53, 112, 141, 57, - 58, 54, 154, 55, 56, 325, 326, 47, 344, 314, - 167, 57, 58, 113, 1, 197, 198, 88, 89, 90, - 91, 51, 52, 53, 242, 199, 163, 252, 54, 1, - 164, 56, 88, 89, 90, 91, 395, 369, 205, 50, - 2, 3, 190, 4, 51, 52, 53, 5, 142, 6, - 392, 393, 60, 200, 7, 2, 3, 405, 4, 327, - 201, 191, 5, 202, 6, 8, 206, 328, 211, 7, - 382, 205, 329, 217, 60, 9, 330, 383, 205, 213, - 8, 51, 52, 53, 225, 10, 60, 331, 54, 221, - 9, 56, 228, 332, 224, 11, 385, 386, 12, 232, - 10, 247, 277, 256, 333, 88, 89, 90, 91, 334, - 11, 308, 257, 12, 88, 89, 90, 91, 388, 389, - 412, 205, 414, 205, 258, 260, 262, 169, 196, 274, - 281, 288, 283, 300, 289, 301, 305, 134, 315, 321, - 324, 322, 323, 345, 349, 347, 364, 350, 353, 354, - 356, 357, 355, 359, 370, 371, 372, 373, 379, 380, - 384, 387, 391, 390, 396, 397, 398, 399, 400, 404, - 408, 406, 48, 407, 183, 352, 265, 381, 411, 341, - 306, 317, 287, 203, 119, 208, 394, 311, 309, 96, - 111, 185, 0, 74 -}; +static const yytype_uint16 yytable[] = { + 43, 153, 159, 184, 110, 189, 134, 62, 193, 36, 266, 38, 248, 40, 226, 222, 223, 255, 166, 50, 51, + 52, 53, 32, 114, 115, 72, 54, 134, 55, 56, 34, 78, 84, 85, 95, 86, 57, 58, 116, 134, 229, + 153, 195, 196, 153, 51, 52, 53, 275, 79, 230, 162, 54, 35, 351, 56, 263, 280, 117, 401, 234, 235, + 153, 120, 121, 122, 123, 124, 195, 196, 195, 196, 174, 175, 402, 50, 51, 52, 53, 367, 368, 37, 209, + 54, 280, 55, 56, 157, 110, 165, 342, 316, 271, 57, 58, 255, 107, 153, 153, 45, 272, 307, 50, 51, + 52, 53, 337, 176, 29, 30, 54, 60, 55, 56, 218, 94, 135, 194, 139, 153, 57, 58, 233, 177, 346, + 291, 126, 174, 135, 169, 270, 249, 251, 103, 410, 187, 140, 191, 210, 125, 250, 413, 178, 292, 127, 141, + 227, 103, 366, 39, 409, 186, 41, 179, 320, 88, 89, 90, 91, 62, 219, 220, 176, 46, 93, 293, 56, + 187, 60, 88, 89, 90, 91, 245, 246, 154, 302, 50, 177, 267, 294, 88, 89, 90, 91, 236, 348, 42, + 160, 299, 319, 153, 303, 231, 161, 142, 336, 178, 49, 197, 198, 88, 89, 90, 91, 244, 374, 270, 179, + 199, 172, 273, 264, 237, 103, 73, 173, 86, 70, 375, 238, 90, 91, 153, 50, 51, 52, 53, 376, 195, + 196, 118, 54, 361, 55, 56, 75, 200, 362, 71, 139, 76, 57, 58, 201, 168, 77, 62, 81, 365, 276, + 87, 312, 299, 279, 83, 403, 212, 207, 286, 50, 51, 52, 53, 377, 239, 82, 141, 54, 92, 55, 56, + 240, 241, 204, 205, 95, 118, 57, 58, 97, 310, 56, 343, 50, 51, 52, 53, 50, 51, 52, 53, 54, + 153, 55, 56, 54, 98, 55, 56, 139, 106, 57, 58, 284, 285, 57, 58, 59, 104, 50, 51, 52, 53, + 232, 313, 314, 142, 54, 107, 55, 56, 50, 51, 52, 53, 112, 141, 57, 58, 54, 154, 55, 56, 325, + 326, 47, 344, 314, 167, 57, 58, 113, 1, 197, 198, 88, 89, 90, 91, 51, 52, 53, 242, 199, 163, + 252, 54, 1, 164, 56, 88, 89, 90, 91, 395, 369, 205, 50, 2, 3, 190, 4, 51, 52, 53, 5, + 142, 6, 392, 393, 60, 200, 7, 2, 3, 405, 4, 327, 201, 191, 5, 202, 6, 8, 206, 328, 211, + 7, 382, 205, 329, 217, 60, 9, 330, 383, 205, 213, 8, 51, 52, 53, 225, 10, 60, 331, 54, 221, + 9, 56, 228, 332, 224, 11, 385, 386, 12, 232, 10, 247, 277, 256, 333, 88, 89, 90, 91, 334, 11, + 308, 257, 12, 88, 89, 90, 91, 388, 389, 412, 205, 414, 205, 258, 260, 262, 169, 196, 274, 281, 288, + 283, 300, 289, 301, 305, 134, 315, 321, 324, 322, 323, 345, 349, 347, 364, 350, 353, 354, 356, 357, 355, + 359, 370, 371, 372, 373, 379, 380, 384, 387, 391, 390, 396, 397, 398, 399, 400, 404, 408, 406, 48, 407, + 183, 352, 265, 381, 411, 341, 306, 317, 287, 203, 119, 208, 394, 311, 309, 96, 111, 185, 0, 74}; -static const yytype_int16 yycheck[] = -{ - 11, 98, 105, 134, 76, 139, 7, 31, 142, 5, - 226, 7, 201, 9, 184, 178, 179, 206, 115, 3, - 4, 5, 6, 3, 8, 9, 37, 11, 7, 13, - 14, 94, 35, 57, 58, 3, 60, 21, 22, 23, - 7, 8, 139, 17, 18, 142, 4, 5, 6, 238, - 53, 187, 108, 11, 79, 324, 14, 220, 247, 83, - 72, 195, 196, 160, 88, 89, 90, 91, 92, 17, - 18, 17, 18, 7, 8, 87, 3, 4, 5, 6, - 349, 350, 52, 10, 11, 274, 13, 14, 89, 161, - 114, 307, 281, 229, 21, 22, 285, 3, 195, 196, - 19, 232, 272, 3, 4, 5, 6, 75, 42, 8, - 9, 11, 96, 13, 14, 171, 69, 96, 142, 19, - 217, 21, 22, 97, 58, 314, 32, 36, 7, 96, - 52, 228, 10, 205, 73, 404, 86, 37, 96, 163, - 93, 19, 411, 77, 50, 54, 46, 97, 87, 97, - 60, 97, 70, 94, 88, 289, 21, 22, 23, 24, - 184, 172, 173, 42, 83, 12, 72, 14, 86, 96, - 21, 22, 23, 24, 198, 199, 98, 27, 3, 58, - 5, 87, 21, 22, 23, 24, 29, 321, 3, 92, - 262, 288, 289, 43, 190, 98, 96, 300, 77, 95, - 19, 20, 21, 22, 23, 24, 198, 19, 305, 88, - 29, 84, 236, 224, 57, 154, 60, 90, 242, 28, - 32, 64, 23, 24, 321, 3, 4, 5, 6, 41, - 17, 18, 97, 11, 93, 13, 14, 99, 57, 98, - 38, 19, 80, 21, 22, 64, 97, 51, 272, 95, - 347, 243, 60, 277, 326, 247, 96, 391, 97, 37, - 256, 3, 4, 5, 6, 77, 8, 99, 46, 11, - 98, 13, 14, 15, 16, 97, 98, 3, 97, 21, - 22, 49, 274, 14, 308, 3, 4, 5, 6, 3, - 4, 5, 6, 11, 391, 13, 14, 11, 92, 13, - 14, 19, 3, 21, 22, 97, 98, 21, 22, 23, - 96, 3, 4, 5, 6, 7, 97, 98, 96, 11, - 3, 13, 14, 3, 4, 5, 6, 3, 46, 21, - 22, 11, 98, 13, 14, 97, 98, 0, 97, 98, - 97, 21, 22, 3, 7, 19, 20, 21, 22, 23, - 24, 4, 5, 6, 96, 29, 20, 10, 11, 7, - 99, 14, 21, 22, 23, 24, 377, 97, 98, 3, - 33, 34, 66, 36, 4, 5, 6, 40, 96, 42, - 10, 11, 96, 57, 47, 33, 34, 398, 36, 31, - 64, 96, 40, 61, 42, 58, 96, 39, 3, 47, - 97, 98, 44, 92, 96, 68, 48, 97, 98, 97, - 58, 4, 5, 6, 98, 78, 96, 59, 11, 73, - 68, 14, 30, 65, 67, 88, 97, 98, 91, 7, - 78, 96, 18, 66, 76, 21, 22, 23, 24, 81, - 88, 18, 98, 91, 21, 22, 23, 24, 97, 98, - 97, 98, 97, 98, 3, 55, 96, 52, 18, 96, - 45, 30, 10, 26, 56, 85, 98, 7, 97, 96, - 96, 62, 62, 97, 96, 98, 97, 96, 96, 96, - 96, 96, 71, 93, 5, 5, 5, 5, 32, 54, - 97, 97, 96, 10, 69, 69, 77, 5, 5, 96, - 62, 97, 13, 97, 129, 326, 225, 362, 96, 305, - 270, 285, 257, 154, 87, 161, 376, 275, 274, 70, - 77, 135, -1, 39 -}; +static const yytype_int16 yycheck[] = { + 11, 98, 105, 134, 76, 139, 7, 31, 142, 5, 226, 7, 201, 9, 184, 178, 179, 206, 115, 3, 4, + 5, 6, 3, 8, 9, 37, 11, 7, 13, 14, 94, 35, 57, 58, 3, 60, 21, 22, 23, 7, 8, + 139, 17, 18, 142, 4, 5, 6, 238, 53, 187, 108, 11, 79, 324, 14, 220, 247, 83, 72, 195, 196, + 160, 88, 89, 90, 91, 92, 17, 18, 17, 18, 7, 8, 87, 3, 4, 5, 6, 349, 350, 52, 10, + 11, 274, 13, 14, 89, 161, 114, 307, 281, 229, 21, 22, 285, 3, 195, 196, 19, 232, 272, 3, 4, + 5, 6, 75, 42, 8, 9, 11, 96, 13, 14, 171, 69, 96, 142, 19, 217, 21, 22, 97, 58, 314, + 32, 36, 7, 96, 52, 228, 10, 205, 73, 404, 86, 37, 96, 163, 93, 19, 411, 77, 50, 54, 46, + 97, 87, 97, 60, 97, 70, 94, 88, 289, 21, 22, 23, 24, 184, 172, 173, 42, 83, 12, 72, 14, + 86, 96, 21, 22, 23, 24, 198, 199, 98, 27, 3, 58, 5, 87, 21, 22, 23, 24, 29, 321, 3, + 92, 262, 288, 289, 43, 190, 98, 96, 300, 77, 95, 19, 20, 21, 22, 23, 24, 198, 19, 305, 88, + 29, 84, 236, 224, 57, 154, 60, 90, 242, 28, 32, 64, 23, 24, 321, 3, 4, 5, 6, 41, 17, + 18, 97, 11, 93, 13, 14, 99, 57, 98, 38, 19, 80, 21, 22, 64, 97, 51, 272, 95, 347, 243, + 60, 277, 326, 247, 96, 391, 97, 37, 256, 3, 4, 5, 6, 77, 8, 99, 46, 11, 98, 13, 14, + 15, 16, 97, 98, 3, 97, 21, 22, 49, 274, 14, 308, 3, 4, 5, 6, 3, 4, 5, 6, 11, + 391, 13, 14, 11, 92, 13, 14, 19, 3, 21, 22, 97, 98, 21, 22, 23, 96, 3, 4, 5, 6, + 7, 97, 98, 96, 11, 3, 13, 14, 3, 4, 5, 6, 3, 46, 21, 22, 11, 98, 13, 14, 97, + 98, 0, 97, 98, 97, 21, 22, 3, 7, 19, 20, 21, 22, 23, 24, 4, 5, 6, 96, 29, 20, + 10, 11, 7, 99, 14, 21, 22, 23, 24, 377, 97, 98, 3, 33, 34, 66, 36, 4, 5, 6, 40, + 96, 42, 10, 11, 96, 57, 47, 33, 34, 398, 36, 31, 64, 96, 40, 61, 42, 58, 96, 39, 3, + 47, 97, 98, 44, 92, 96, 68, 48, 97, 98, 97, 58, 4, 5, 6, 98, 78, 96, 59, 11, 73, + 68, 14, 30, 65, 67, 88, 97, 98, 91, 7, 78, 96, 18, 66, 76, 21, 22, 23, 24, 81, 88, + 18, 98, 91, 21, 22, 23, 24, 97, 98, 97, 98, 97, 98, 3, 55, 96, 52, 18, 96, 45, 30, + 10, 26, 56, 85, 98, 7, 97, 96, 96, 62, 62, 97, 96, 98, 97, 96, 96, 96, 96, 96, 71, + 93, 5, 5, 5, 5, 32, 54, 97, 97, 96, 10, 69, 69, 77, 5, 5, 96, 62, 97, 13, 97, + 129, 326, 225, 362, 96, 305, 270, 285, 257, 154, 87, 161, 376, 275, 274, 70, 77, 135, -1, 39}; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = -{ - 0, 7, 33, 34, 36, 40, 42, 47, 58, 68, - 78, 88, 91, 101, 102, 103, 125, 130, 131, 132, - 133, 134, 135, 136, 140, 141, 142, 144, 147, 8, - 9, 143, 3, 187, 94, 79, 187, 52, 187, 60, - 187, 94, 3, 182, 183, 19, 83, 0, 102, 95, - 3, 4, 5, 6, 11, 13, 14, 21, 22, 23, - 96, 154, 176, 177, 178, 179, 180, 181, 184, 188, - 28, 38, 182, 60, 183, 99, 80, 51, 35, 53, - 191, 95, 99, 96, 176, 176, 176, 60, 21, 22, - 23, 24, 98, 12, 188, 3, 190, 49, 92, 150, - 159, 148, 149, 179, 96, 117, 3, 3, 145, 146, - 186, 191, 3, 3, 8, 9, 23, 176, 97, 148, - 176, 176, 176, 176, 176, 188, 36, 54, 104, 105, - 106, 107, 115, 118, 7, 96, 151, 152, 153, 19, - 37, 46, 96, 163, 164, 165, 166, 167, 169, 170, - 172, 174, 176, 184, 98, 114, 186, 89, 137, 153, - 92, 98, 150, 20, 99, 176, 184, 97, 97, 52, - 155, 156, 84, 90, 7, 8, 42, 58, 77, 88, - 120, 121, 122, 106, 143, 151, 70, 86, 126, 163, - 66, 96, 175, 163, 176, 17, 18, 19, 20, 29, - 57, 64, 61, 149, 97, 98, 96, 37, 146, 10, - 176, 3, 97, 97, 157, 158, 182, 92, 150, 182, - 182, 73, 117, 117, 67, 98, 154, 97, 30, 8, - 152, 187, 7, 97, 163, 163, 29, 57, 64, 8, - 15, 16, 96, 173, 175, 176, 176, 96, 178, 10, - 19, 186, 10, 138, 139, 178, 66, 98, 3, 189, - 55, 160, 96, 117, 182, 122, 155, 5, 127, 128, - 184, 152, 143, 176, 96, 178, 175, 18, 171, 175, - 178, 45, 168, 10, 97, 98, 187, 158, 30, 56, - 162, 32, 50, 72, 87, 108, 109, 110, 113, 186, - 26, 85, 27, 43, 129, 98, 129, 154, 18, 171, - 175, 168, 176, 97, 98, 97, 178, 139, 161, 184, - 163, 96, 62, 62, 96, 97, 98, 31, 39, 44, - 48, 59, 65, 76, 81, 185, 153, 75, 123, 124, - 190, 128, 155, 176, 97, 97, 178, 98, 163, 96, - 96, 114, 109, 96, 96, 71, 96, 96, 111, 93, - 116, 93, 98, 119, 97, 184, 97, 114, 114, 97, - 5, 5, 5, 5, 19, 32, 41, 77, 112, 32, - 54, 124, 97, 97, 97, 97, 98, 97, 97, 98, - 10, 96, 10, 11, 181, 182, 69, 69, 77, 5, - 5, 72, 87, 163, 96, 182, 97, 97, 62, 97, - 114, 96, 97, 114, 97 -}; +static const yytype_uint8 yystos[] = { + 0, 7, 33, 34, 36, 40, 42, 47, 58, 68, 78, 88, 91, 101, 102, 103, 125, 130, 131, 132, 133, + 134, 135, 136, 140, 141, 142, 144, 147, 8, 9, 143, 3, 187, 94, 79, 187, 52, 187, 60, 187, 94, + 3, 182, 183, 19, 83, 0, 102, 95, 3, 4, 5, 6, 11, 13, 14, 21, 22, 23, 96, 154, 176, + 177, 178, 179, 180, 181, 184, 188, 28, 38, 182, 60, 183, 99, 80, 51, 35, 53, 191, 95, 99, 96, + 176, 176, 176, 60, 21, 22, 23, 24, 98, 12, 188, 3, 190, 49, 92, 150, 159, 148, 149, 179, 96, + 117, 3, 3, 145, 146, 186, 191, 3, 3, 8, 9, 23, 176, 97, 148, 176, 176, 176, 176, 176, 188, + 36, 54, 104, 105, 106, 107, 115, 118, 7, 96, 151, 152, 153, 19, 37, 46, 96, 163, 164, 165, 166, + 167, 169, 170, 172, 174, 176, 184, 98, 114, 186, 89, 137, 153, 92, 98, 150, 20, 99, 176, 184, 97, + 97, 52, 155, 156, 84, 90, 7, 8, 42, 58, 77, 88, 120, 121, 122, 106, 143, 151, 70, 86, 126, + 163, 66, 96, 175, 163, 176, 17, 18, 19, 20, 29, 57, 64, 61, 149, 97, 98, 96, 37, 146, 10, + 176, 3, 97, 97, 157, 158, 182, 92, 150, 182, 182, 73, 117, 117, 67, 98, 154, 97, 30, 8, 152, + 187, 7, 97, 163, 163, 29, 57, 64, 8, 15, 16, 96, 173, 175, 176, 176, 96, 178, 10, 19, 186, + 10, 138, 139, 178, 66, 98, 3, 189, 55, 160, 96, 117, 182, 122, 155, 5, 127, 128, 184, 152, 143, + 176, 96, 178, 175, 18, 171, 175, 178, 45, 168, 10, 97, 98, 187, 158, 30, 56, 162, 32, 50, 72, + 87, 108, 109, 110, 113, 186, 26, 85, 27, 43, 129, 98, 129, 154, 18, 171, 175, 168, 176, 97, 98, + 97, 178, 139, 161, 184, 163, 96, 62, 62, 96, 97, 98, 31, 39, 44, 48, 59, 65, 76, 81, 185, + 153, 75, 123, 124, 190, 128, 155, 176, 97, 97, 178, 98, 163, 96, 96, 114, 109, 96, 96, 71, 96, + 96, 111, 93, 116, 93, 98, 119, 97, 184, 97, 114, 114, 97, 5, 5, 5, 5, 19, 32, 41, 77, + 112, 32, 54, 124, 97, 97, 97, 97, 98, 97, 97, 98, 10, 96, 10, 11, 181, 182, 69, 69, 77, + 5, 5, 72, 87, 163, 96, 182, 97, 97, 62, 97, 114, 96, 97, 114, 97}; -#define yyerrok (yyerrstatus = 0) -#define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 - -#define YYACCEPT goto yyacceptlab -#define YYABORT goto yyabortlab -#define YYERROR goto yyerrorlab +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) +#define YYEMPTY (-2) +#define YYEOF 0 +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab /* Like YYERROR except do call yyerror. This remains here temporarily to ease the transition to the new meaning of YYERROR, for GCC. Once GCC version 2 has supplanted version 1, this can go. */ -#define YYFAIL goto yyerrlab +#define YYFAIL goto yyerrlab -#define YYRECOVERING() (!!yyerrstatus) +#define YYRECOVERING() (!!yyerrstatus) -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY && yylen == 1) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - yytoken = YYTRANSLATE (yychar); \ - YYPOPSTACK (1); \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (YYID (0)) - - -#define YYTERROR 1 -#define YYERRCODE 256 +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY && yylen == 1) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + yytoken = YYTRANSLATE(yychar); \ + YYPOPSTACK(1); \ + goto yybackup; \ + } \ + else \ + { \ + yyerror(YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (YYID(0)) +#define YYTERROR 1 +#define YYERRCODE 256 /* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. If N is 0, then set CURRENT to the empty location which ends @@ -1154,136 +1003,120 @@ while (YYID (0)) #define YYRHSLOC(Rhs, K) ((Rhs)[K]) #ifndef YYLLOC_DEFAULT -# define YYLLOC_DEFAULT(Current, Rhs, N) \ - do \ - if (YYID (N)) \ - { \ - (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ - (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ - (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ - (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ - } \ - else \ - { \ - (Current).first_line = (Current).last_line = \ - YYRHSLOC (Rhs, 0).last_line; \ - (Current).first_column = (Current).last_column = \ - YYRHSLOC (Rhs, 0).last_column; \ - } \ - while (YYID (0)) +#define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (YYID(N)) \ + { \ + (Current).first_line = YYRHSLOC(Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC(Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC(Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC(Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = YYRHSLOC(Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = YYRHSLOC(Rhs, 0).last_column; \ + } \ + while (YYID(0)) #endif - /* YY_LOCATION_PRINT -- Print the location on the stream. This macro was not mandated originally: define only if we know we won't break user code: when these are the locations we know. */ #ifndef YY_LOCATION_PRINT -# if YYLTYPE_IS_TRIVIAL -# define YY_LOCATION_PRINT(File, Loc) \ - fprintf (File, "%d.%d-%d.%d", \ - (Loc).first_line, (Loc).first_column, \ - (Loc).last_line, (Loc).last_column) -# else -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -# endif +#if YYLTYPE_IS_TRIVIAL +#define YY_LOCATION_PRINT(File, Loc) \ + fprintf(File, "%d.%d-%d.%d", (Loc).first_line, (Loc).first_column, (Loc).last_line, (Loc).last_column) +#else +#define YY_LOCATION_PRINT(File, Loc) ((void)0) +#endif #endif - /* YYLEX -- calling `yylex' with the right arguments. */ #ifdef YYLEX_PARAM -# define YYLEX yylex (YYLEX_PARAM) +#define YYLEX yylex(YYLEX_PARAM) #else -# define YYLEX yylex () +#define YYLEX yylex() #endif /* Enable debugging if requested. */ #if YYDEBUG -# ifndef YYFPRINTF -# include /* INFRINGES ON USER NAME SPACE */ -# define YYFPRINTF fprintf -# endif +#ifndef YYFPRINTF +#include /* INFRINGES ON USER NAME SPACE */ +#define YYFPRINTF fprintf +#endif -# define YYDPRINTF(Args) \ -do { \ - if (yydebug) \ - YYFPRINTF Args; \ -} while (YYID (0)) - -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ -do { \ - if (yydebug) \ - { \ - YYFPRINTF (stderr, "%s ", Title); \ - yy_symbol_print (stderr, \ - Type, Value); \ - YYFPRINTF (stderr, "\n"); \ - } \ -} while (YYID (0)) +#define YYDPRINTF(Args) \ + do \ + { \ + if (yydebug) \ + YYFPRINTF Args; \ + } while (YYID(0)) +#define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ + do \ + { \ + if (yydebug) \ + { \ + YYFPRINTF(stderr, "%s ", Title); \ + yy_symbol_print(stderr, Type, Value); \ + YYFPRINTF(stderr, "\n"); \ + } \ + } while (YYID(0)) /*--------------------------------. | Print this symbol on YYOUTPUT. | `--------------------------------*/ /*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_value_print (FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_symbol_value_print(FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) #else -static void -yy_symbol_value_print (yyoutput, yytype, yyvaluep) -FILE* yyoutput; +static void yy_symbol_value_print(yyoutput, yytype, yyvaluep) FILE* yyoutput; int yytype; YYSTYPE const* const yyvaluep; #endif { - if (!yyvaluep) - return; + if (!yyvaluep) + return; -# ifdef YYPRINT +#ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); + if (yytype < YYNTOKENS) + YYPRINT(yyoutput, yytoknum[yytype], *yyvaluep); -# else - YYUSE (yyoutput); -# endif +#else + YYUSE(yyoutput); +#endif - switch (yytype) - { - default: - break; - } + switch (yytype) + { + default: break; + } } - /*--------------------------------. | Print this symbol on YYOUTPUT. | `--------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_print (FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_symbol_print(FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) #else -static void -yy_symbol_print (yyoutput, yytype, yyvaluep) -FILE* yyoutput; +static void yy_symbol_print(yyoutput, yytype, yyvaluep) FILE* yyoutput; int yytype; YYSTYPE const* const yyvaluep; #endif { - if (yytype < YYNTOKENS) - YYFPRINTF (yyoutput, "token %s (", yytname[yytype]); - else - YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]); + if (yytype < YYNTOKENS) + YYFPRINTF(yyoutput, "token %s (", yytname[yytype]); + else + YYFPRINTF(yyoutput, "nterm %s (", yytname[yytype]); - yy_symbol_value_print (yyoutput, yytype, yyvaluep); - YYFPRINTF (yyoutput, ")"); + yy_symbol_value_print(yyoutput, yytype, yyvaluep); + YYFPRINTF(yyoutput, ")"); } /*------------------------------------------------------------------. @@ -1291,87 +1124,76 @@ YYSTYPE const* const yyvaluep; | TOP (included). | `------------------------------------------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_stack_print (yytype_int16* yybottom, yytype_int16* yytop) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_stack_print(yytype_int16* yybottom, yytype_int16* yytop) #else -static void -yy_stack_print (yybottom, yytop) -yytype_int16* yybottom; +static void yy_stack_print(yybottom, yytop) yytype_int16* yybottom; yytype_int16* yytop; #endif { - YYFPRINTF (stderr, "Stack now"); + YYFPRINTF(stderr, "Stack now"); - for (; yybottom <= yytop; yybottom++) - { - int yybot = *yybottom; - YYFPRINTF (stderr, " %d", yybot); - } + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF(stderr, " %d", yybot); + } - YYFPRINTF (stderr, "\n"); + YYFPRINTF(stderr, "\n"); } -# define YY_STACK_PRINT(Bottom, Top) \ -do { \ - if (yydebug) \ - yy_stack_print ((Bottom), (Top)); \ -} while (YYID (0)) - +#define YY_STACK_PRINT(Bottom, Top) \ + do \ + { \ + if (yydebug) \ + yy_stack_print((Bottom), (Top)); \ + } while (YYID(0)) /*------------------------------------------------. | Report that the YYRULE is going to be reduced. | `------------------------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_reduce_print (YYSTYPE* yyvsp, int yyrule) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_reduce_print(YYSTYPE* yyvsp, int yyrule) #else -static void -yy_reduce_print (yyvsp, yyrule) -YYSTYPE* yyvsp; +static void yy_reduce_print(yyvsp, yyrule) YYSTYPE* yyvsp; int yyrule; #endif { - int yynrhs = yyr2[yyrule]; - int yyi; - unsigned long int yylno = yyrline[yyrule]; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", - yyrule - 1, yylno); + int yynrhs = yyr2[yyrule]; + int yyi; + unsigned long int yylno = yyrline[yyrule]; + YYFPRINTF(stderr, "Reducing stack by rule %d (line %lu):\n", yyrule - 1, yylno); - /* The symbols being reduced. */ - for (yyi = 0; yyi < yynrhs; yyi++) - { - YYFPRINTF (stderr, " $%d = ", yyi + 1); - yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi], - &(yyvsp[(yyi + 1) - (yynrhs)]) - ); - YYFPRINTF (stderr, "\n"); - } + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF(stderr, " $%d = ", yyi + 1); + yy_symbol_print(stderr, yyrhs[yyprhs[yyrule] + yyi], &(yyvsp[(yyi + 1) - (yynrhs)])); + YYFPRINTF(stderr, "\n"); + } } -# define YY_REDUCE_PRINT(Rule) \ -do { \ - if (yydebug) \ - yy_reduce_print (yyvsp, Rule); \ -} while (YYID (0)) +#define YY_REDUCE_PRINT(Rule) \ + do \ + { \ + if (yydebug) \ + yy_reduce_print(yyvsp, Rule); \ + } while (YYID(0)) /* Nonzero means print parse trace. It is left uninitialized so that multiple parsers can coexist. */ int yydebug; #else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) -# define YY_STACK_PRINT(Bottom, Top) -# define YY_REDUCE_PRINT(Rule) +#define YYDPRINTF(Args) +#define YY_SYMBOL_PRINT(Title, Type, Value, Location) +#define YY_STACK_PRINT(Bottom, Top) +#define YY_REDUCE_PRINT(Rule) #endif /* !YYDEBUG */ - /* YYINITDEPTH -- initial size of the parser's stacks. */ -#ifndef YYINITDEPTH -# define YYINITDEPTH 200 +#ifndef YYINITDEPTH +#define YYINITDEPTH 200 #endif /* YYMAXDEPTH -- maximum size the stacks can grow to (effective only @@ -1382,67 +1204,57 @@ int yydebug; evaluated with infinite-precision integer arithmetic. */ #ifndef YYMAXDEPTH -# define YYMAXDEPTH 10000 +#define YYMAXDEPTH 10000 #endif - - #if YYERROR_VERBOSE -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else -/* Return the length of YYSTR. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static YYSIZE_T -yystrlen (const char* yystr) +#ifndef yystrlen +#if defined __GLIBC__ && defined _STRING_H +#define yystrlen strlen #else -static YYSIZE_T -yystrlen (yystr) -const char* yystr; +/* Return the length of YYSTR. */ +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static YYSIZE_T yystrlen(const char* yystr) +#else +static YYSIZE_T yystrlen(yystr) const char* yystr; #endif { - YYSIZE_T yylen; + YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; + for (yylen = 0; yystr[yylen]; yylen++) + continue; - return yylen; + return yylen; } -# endif -# endif +#endif +#endif -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else +#ifndef yystpcpy +#if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +#define yystpcpy stpcpy +#else /* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in YYDEST. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static char* -yystpcpy (char* yydest, const char* yysrc) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static char* yystpcpy(char* yydest, const char* yysrc) #else -static char* -yystpcpy (yydest, yysrc) -char* yydest; +static char* yystpcpy(yydest, yysrc) char* yydest; const char* yysrc; #endif { - char* yyd = yydest; - const char* yys = yysrc; + char* yyd = yydest; + const char* yys = yysrc; - while ((*yyd++ = *yys++) != '\0') - continue; + while ((*yyd++ = *yys++) != '\0') + continue; - return yyd - 1; + return yyd - 1; } -# endif -# endif +#endif +#endif -# ifndef yytnamerr +#ifndef yytnamerr /* Copy to YYRES the contents of YYSTR after stripping away unnecessary quotes and backslashes, so that it's suitable for yyerror. The heuristic is that double-quoting is unnecessary unless the string @@ -1450,50 +1262,47 @@ const char* yysrc; backslash-backslash). YYSTR is taken from yytname. If YYRES is null, do not copy; instead, return the length of what the result would have been. */ -static YYSIZE_T -yytnamerr (char* yyres, const char* yystr) +static YYSIZE_T yytnamerr(char* yyres, const char* yystr) { - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const* yyp = yystr; + if (*yystr == '"') + { + YYSIZE_T yyn = 0; + char const* yyp = yystr; - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': goto do_not_strip_quotes; - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; + /* Fall through. */ + default: + if (yyres) + yyres[yyn] = *yyp; - yyn++; - break; + yyn++; + break; - case '"': - if (yyres) - yyres[yyn] = '\0'; + case '"': + if (yyres) + yyres[yyn] = '\0'; - return yyn; - } + return yyn; + } -do_not_strip_quotes: - ; - } + do_not_strip_quotes:; + } - if (! yyres) - return yystrlen (yystr); + if (!yyres) + return yystrlen(yystr); - return yystpcpy (yyres, yystr) - yyres; + return yystpcpy(yyres, yystr) - yyres; } -# endif +#endif /* Copy into YYRESULT an error message about the unexpected token YYCHAR while in state YYSTATE. Return the number of bytes copied, @@ -1502,25 +1311,27 @@ do_not_strip_quotes: copied. As a special case, return 0 if an ordinary "syntax error" message will do. Return YYSIZE_MAXIMUM if overflow occurs during size calculation. */ -static YYSIZE_T -yysyntax_error (char* yyresult, int yystate, int yychar) +static YYSIZE_T yysyntax_error(char* yyresult, int yystate, int yychar) { - int yyn = yypact[yystate]; + int yyn = yypact[yystate]; - if (! (YYPACT_NINF < yyn && yyn <= YYLAST)) - return 0; - else + if (!(YYPACT_NINF < yyn && yyn <= YYLAST)) + return 0; + else + { + int yytype = YYTRANSLATE(yychar); + YYSIZE_T yysize0 = yytnamerr(0, yytname[yytype]); + YYSIZE_T yysize = yysize0; + YYSIZE_T yysize1; + int yysize_overflow = 0; + enum { - int yytype = YYTRANSLATE (yychar); - YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]); - YYSIZE_T yysize = yysize0; - YYSIZE_T yysize1; - int yysize_overflow = 0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - char const* yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - int yyx; + YYERROR_VERBOSE_ARGS_MAXIMUM = 5 + }; + char const* yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; + int yyx; -# if 0 +#if 0 /* This is so xgettext sees the translatable formats that are constructed on the fly. */ YY_("syntax error, unexpected %s"); @@ -1528,134 +1339,124 @@ yysyntax_error (char* yyresult, int yystate, int yychar) YY_("syntax error, unexpected %s, expecting %s or %s"); YY_("syntax error, unexpected %s, expecting %s or %s or %s"); YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); -# endif - char* yyfmt; - char const* yyf; - static char const yyunexpected[] = "syntax error, unexpected %s"; - static char const yyexpecting[] = ", expecting %s"; - static char const yyor[] = " or %s"; - char yyformat[sizeof yyunexpected - + sizeof yyexpecting - 1 - + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) - * (sizeof yyor - 1))]; - char const* yyprefix = yyexpecting; +#endif + char* yyfmt; + char const* yyf; + static char const yyunexpected[] = "syntax error, unexpected %s"; + static char const yyexpecting[] = ", expecting %s"; + static char const yyor[] = " or %s"; + char yyformat[sizeof yyunexpected + sizeof yyexpecting - 1 + + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) * (sizeof yyor - 1))]; + char const* yyprefix = yyexpecting; - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. */ - int yyxbegin = yyn < 0 ? -yyn : 0; + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. */ + int yyxbegin = yyn < 0 ? -yyn : 0; - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yycount = 1; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yycount = 1; - yyarg[0] = yytname[yytype]; - yyfmt = yystpcpy (yyformat, yyunexpected); + yyarg[0] = yytname[yytype]; + yyfmt = yystpcpy(yyformat, yyunexpected); - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - yyformat[sizeof yyunexpected - 1] = '\0'; - break; - } - - yyarg[yycount++] = yytname[yyx]; - yysize1 = yysize + yytnamerr (0, yytname[yyx]); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - yyfmt = yystpcpy (yyfmt, yyprefix); - yyprefix = yyor; - } - - yyf = YY_(yyformat); - yysize1 = yysize + yystrlen (yyf); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - - if (yysize_overflow) - return YYSIZE_MAXIMUM; - - if (yyresult) + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) + { + if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) { - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - char* yyp = yyresult; - int yyi = 0; - - while ((*yyp = *yyf) != '\0') - { - if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) - { - yyp += yytnamerr (yyp, yyarg[yyi++]); - yyf += 2; - } - else - { - yyp++; - yyf++; - } - } + yycount = 1; + yysize = yysize0; + yyformat[sizeof yyunexpected - 1] = '\0'; + break; } - return yysize; + yyarg[yycount++] = yytname[yyx]; + yysize1 = yysize + yytnamerr(0, yytname[yyx]); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + yyfmt = yystpcpy(yyfmt, yyprefix); + yyprefix = yyor; + } + + yyf = YY_(yyformat); + yysize1 = yysize + yystrlen(yyf); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + + if (yysize_overflow) + return YYSIZE_MAXIMUM; + + if (yyresult) + { + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + char* yyp = yyresult; + int yyi = 0; + + while ((*yyp = *yyf) != '\0') + { + if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) + { + yyp += yytnamerr(yyp, yyarg[yyi++]); + yyf += 2; + } + else + { + yyp++; + yyf++; + } + } } + + return yysize; + } } #endif /* YYERROR_VERBOSE */ - /*-----------------------------------------------. | Release the memory associated to this symbol. | `-----------------------------------------------*/ /*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yydestruct (const char* yymsg, int yytype, YYSTYPE* yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yydestruct(const char* yymsg, int yytype, YYSTYPE* yyvaluep) #else -static void -yydestruct (yymsg, yytype, yyvaluep) -const char* yymsg; +static void yydestruct(yymsg, yytype, yyvaluep) const char* yymsg; int yytype; YYSTYPE* yyvaluep; #endif { - YYUSE (yyvaluep); + YYUSE(yyvaluep); - if (!yymsg) - yymsg = "Deleting"; + if (!yymsg) + yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + YY_SYMBOL_PRINT(yymsg, yytype, yyvaluep, yylocationp); - switch (yytype) - { - - default: - break; - } + switch (yytype) + { + default: break; + } } /* Prevent warnings from -Wmissing-prototypes. */ #ifdef YYPARSE_PARAM #if defined __STDC__ || defined __cplusplus -int yyparse (void* YYPARSE_PARAM); +int yyparse(void* YYPARSE_PARAM); #else -int yyparse (); +int yyparse(); #endif #else /* ! YYPARSE_PARAM */ #if defined __STDC__ || defined __cplusplus -int yyparse (void); +int yyparse(void); #else -int yyparse (); +int yyparse(); #endif #endif /* ! YYPARSE_PARAM */ - /* The lookahead symbol. */ int yychar; @@ -1665,1512 +1466,1499 @@ YYSTYPE yylval; /* Number of syntax errors so far. */ int yynerrs; - - /*-------------------------. | yyparse or yypush_parse. | `-------------------------*/ #ifdef YYPARSE_PARAM -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void* YYPARSE_PARAM) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +int yyparse(void* YYPARSE_PARAM) #else -int -yyparse (YYPARSE_PARAM) -void* YYPARSE_PARAM; +int yyparse(YYPARSE_PARAM) void* YYPARSE_PARAM; #endif #else /* ! YYPARSE_PARAM */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +int yyparse(void) #else -int -yyparse () +int yyparse() #endif #endif { + int yystate; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus; + /* The stacks and their tools: + `yyss': related to states. + `yyvs': related to semantic values. - int yystate; - /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; + Refer to the stacks thru separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ - /* The stacks and their tools: - `yyss': related to states. - `yyvs': related to semantic values. + /* The state stack. */ + yytype_int16 yyssa[YYINITDEPTH]; + yytype_int16* yyss; + yytype_int16* yyssp; - Refer to the stacks thru separate pointers, to allow yyoverflow - to reallocate them elsewhere. */ + /* The semantic value stack. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE* yyvs; + YYSTYPE* yyvsp; - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16* yyss; - yytype_int16* yyssp; + YYSIZE_T yystacksize; - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE* yyvs; - YYSTYPE* yyvsp; - - YYSIZE_T yystacksize; - - int yyn; - int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken; - /* The variables used to return semantic value and location from the - action routines. */ - YYSTYPE yyval; + int yyn; + int yyresult; + /* Lookahead token as an internal (translated) token number. */ + int yytoken; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; #if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char* yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char* yymsg = yymsgbuf; + YYSIZE_T yymsg_alloc = sizeof yymsgbuf; #endif -#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) - /* The number of symbols on the RHS of the reduced rule. - Keep to zero when no symbol should be popped. */ - int yylen = 0; + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; - yytoken = 0; - yyss = yyssa; - yyvs = yyvsa; - yystacksize = YYINITDEPTH; + yytoken = 0; + yyss = yyssa; + yyvs = yyvsa; + yystacksize = YYINITDEPTH; - YYDPRINTF ((stderr, "Starting parse\n")); + YYDPRINTF((stderr, "Starting parse\n")); - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; - yychar = YYEMPTY; /* Cause a token to be read. */ + yystate = 0; + yyerrstatus = 0; + yynerrs = 0; + yychar = YYEMPTY; /* Cause a token to be read. */ - /* Initialize stack pointers. - Waste one element of value and location stack - so that they stay on the same level as the state stack. - The wasted elements are never initialized. */ - yyssp = yyss; - yyvsp = yyvs; + /* Initialize stack pointers. + Waste one element of value and location stack + so that they stay on the same level as the state stack. + The wasted elements are never initialized. */ + yyssp = yyss; + yyvsp = yyvs; - goto yysetstate; + goto yysetstate; - /*------------------------------------------------------------. - | yynewstate -- Push a new state, which is found in yystate. | - `------------------------------------------------------------*/ + /*------------------------------------------------------------. + | yynewstate -- Push a new state, which is found in yystate. | + `------------------------------------------------------------*/ yynewstate: - /* In all cases, when you get here, the value and location stacks - have just been pushed. So pushing a state here evens the stacks. */ - yyssp++; + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; yysetstate: - *yyssp = yystate; + *yyssp = yystate; - if (yyss + yystacksize - 1 <= yyssp) - { - /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; + if (yyss + yystacksize - 1 <= yyssp) + { + /* Get the current used size of the three stacks, in elements. */ + YYSIZE_T yysize = yyssp - yyss + 1; #ifdef yyoverflow - { - /* Give user a chance to reallocate the stack. Use copies of - these so that the &'s don't force the real ones into - memory. */ - YYSTYPE* yyvs1 = yyvs; - yytype_int16* yyss1 = yyss; + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + YYSTYPE* yyvs1 = yyvs; + yytype_int16* yyss1 = yyss; - /* Each stack pointer address is followed by the size of the - data in use in that stack, in bytes. This used to be a - conditional around just the two extra args, but that might - be undefined if yyoverflow is a macro. */ - yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), - &yystacksize); + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow(YY_("memory exhausted"), &yyss1, yysize * sizeof(*yyssp), &yyvs1, yysize * sizeof(*yyvsp), + &yystacksize); - yyss = yyss1; - yyvs = yyvs1; - } + yyss = yyss1; + yyvs = yyvs1; + } #else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE +#ifndef YYSTACK_RELOCATE + goto yyexhaustedlab; +#else + + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + goto yyexhaustedlab; + + yystacksize *= 2; + + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yytype_int16* yyss1 = yyss; + union yyalloc* yyptr = (union yyalloc*)YYSTACK_ALLOC(YYSTACK_BYTES(yystacksize)); + + if (!yyptr) goto yyexhaustedlab; -# else - /* Extend the stack our own way. */ - if (YYMAXDEPTH <= yystacksize) - goto yyexhaustedlab; + YYSTACK_RELOCATE(yyss_alloc, yyss); + YYSTACK_RELOCATE(yyvs_alloc, yyvs); +#undef YYSTACK_RELOCATE - yystacksize *= 2; - - if (YYMAXDEPTH < yystacksize) - yystacksize = YYMAXDEPTH; - - { - yytype_int16* yyss1 = yyss; - union yyalloc* yyptr = - (union yyalloc*) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); - - if (! yyptr) - goto yyexhaustedlab; - - YYSTACK_RELOCATE (yyss_alloc, yyss); - YYSTACK_RELOCATE (yyvs_alloc, yyvs); -# undef YYSTACK_RELOCATE - - if (yyss1 != yyssa) - YYSTACK_FREE (yyss1); - } -# endif + if (yyss1 != yyssa) + YYSTACK_FREE(yyss1); + } +#endif #endif /* no yyoverflow */ - yyssp = yyss + yysize - 1; - yyvsp = yyvs + yysize - 1; + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); + YYDPRINTF((stderr, "Stack size increased to %lu\n", (unsigned long int)yystacksize)); - if (yyss + yystacksize - 1 <= yyssp) - YYABORT; - } + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YYDPRINTF((stderr, "Entering state %d\n", yystate)); - if (yystate == YYFINAL) - YYACCEPT; + if (yystate == YYFINAL) + YYACCEPT; - goto yybackup; + goto yybackup; - /*-----------. - | yybackup. | - `-----------*/ + /*-----------. + | yybackup. | + `-----------*/ yybackup: - /* Do appropriate processing given the current state. Read a - lookahead token if we need one and don't already have one. */ + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ - /* First try to decide what to do without reference to lookahead token. */ - yyn = yypact[yystate]; + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; - if (yyn == YYPACT_NINF) - goto yydefault; + if (yyn == YYPACT_NINF) + goto yydefault; - /* Not known => get a lookahead token if don't already have one. */ + /* Not known => get a lookahead token if don't already have one. */ + + /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ + if (yychar == YYEMPTY) + { + YYDPRINTF((stderr, "Reading a token: ")); + yychar = YYLEX; + } + + if (yychar <= YYEOF) + { + yychar = yytoken = YYEOF; + YYDPRINTF((stderr, "Now at end of input.\n")); + } + else + { + yytoken = YYTRANSLATE(yychar); + YY_SYMBOL_PRINT("Next token is", yytoken, &yylval, &yylloc); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + + yyn = yytable[yyn]; + + if (yyn <= 0) + { + if (yyn == 0 || yyn == YYTABLE_NINF) + goto yyerrlab; + + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT("Shifting", yytoken, &yylval, &yylloc); + + /* Discard the shifted token. */ + yychar = YYEMPTY; + + yystate = yyn; + *++yyvsp = yylval; + + goto yynewstate; + + /*-----------------------------------------------------------. + | yydefault -- do the default action for the current state. | + `-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + + if (yyn == 0) + goto yyerrlab; + + goto yyreduce; + + /*-----------------------------. + | yyreduce -- Do a reduction. | + `-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + `$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1 - yylen]; + + YY_REDUCE_PRINT(yyn); + + switch (yyn) + { + case 2: - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ - if (yychar == YYEMPTY) { - YYDPRINTF ((stderr, "Reading a token: ")); - yychar = YYLEX; + if ((yyvsp[(1) - (2)].sqlStmt) != NULL) + { + (yyval.sqlStmtList) = parseTree; + (yyval.sqlStmtList)->push_back((yyvsp[(1) - (2)].sqlStmt)); + } + else + { + (yyval.sqlStmtList) = NULL; + } + + ; } + break; + + case 3: + + { + if ((yyvsp[(1) - (3)].sqlStmtList) != NULL) + { + parseTree = (yyvsp[(1) - (3)].sqlStmtList); + } + + ; + } + break; + + case 4: + + { + (yyval.sqlStmt) = NULL; + ; + } + break; + + case 35: + + { + (yyval.colNameList) = new ColumnNameList; + (yyval.colNameList)->push_back((yyvsp[(1) - (1)].strval)); + ; + } + break; + + case 36: + + { + (yyval.colNameList) = (yyvsp[(1) - (3)].colNameList); + (yyval.colNameList)->push_back((yyvsp[(3) - (3)].strval)); + ; + } + break; + + case 40: + + { + (yyval.colNameList) = NULL; + ; + } + break; + + case 41: + + { + (yyval.colNameList) = (yyvsp[(2) - (3)].colNameList); + ; + } + break; + + case 59: + + { + (yyval.sqlStmt) = NULL; + ; + } + break; + + case 82: + + { + ; + } + break; + + case 83: + + { + (yyval.sqlStmt) = new CommandSqlStatement("COMMIT"); + ; + } + break; + + case 84: + + { + (yyval.sqlStmt) = new CommandSqlStatement("COMMIT"); + ; + } + break; + + case 85: + + { + (yyval.sqlStmt) = new DeleteSqlStatement((yyvsp[(3) - (7)].tblName)); + + ; + } + break; + + case 86: + + { + (yyval.sqlStmt) = new DeleteSqlStatement((yyvsp[(3) - (4)].tblName), (yyvsp[(4) - (4)].whereClause)); + ; + } + break; + + case 87: + + { + ; + } + break; + + case 88: + + { + if (NULL == (yyvsp[(4) - (5)].colNameList)) + (yyval.sqlStmt) = new InsertSqlStatement((yyvsp[(3) - (5)].tblName), (yyvsp[(5) - (5)].valsOrQuery)); + else + (yyval.sqlStmt) = new InsertSqlStatement((yyvsp[(3) - (5)].tblName), (yyvsp[(4) - (5)].colNameList), + (yyvsp[(5) - (5)].valsOrQuery)); + + ; + } + break; + + case 89: + + { + (yyval.valsOrQuery) = new ValuesOrQuery((yyvsp[(3) - (4)].valsList)); + ; + } + break; + + case 90: + + { + (yyval.valsOrQuery) = new ValuesOrQuery((yyvsp[(1) - (1)].querySpec)); + ; + } + break; + + case 91: + + { + (yyval.valsList) = new ValuesList; + (yyval.valsList)->push_back((yyvsp[(1) - (1)].strval)); + ; + } + break; + + case 92: + + { + (yyval.valsList) = (yyvsp[(1) - (3)].valsList); + (yyval.valsList)->push_back((yyvsp[(3) - (3)].strval)); + ; + } + break; + + case 95: + + { + ; + } + break; + + case 96: + + { + (yyval.sqlStmt) = new CommandSqlStatement("ROLLBACK"); + ; + } + break; + + case 97: + + { + (yyval.sqlStmt) = new CommandSqlStatement("ROLLBACK"); + ; + } + break; + + case 98: + + { + ; + } + break; + + case 99: + + { + (yyval.strval) = NULL; + ; + } + break; + + case 100: + + { + (yyval.strval) = (yyvsp[(1) - (1)].strval); + ; + } + break; + + case 101: + + { + (yyval.strval) = (yyvsp[(1) - (1)].strval); + ; + } + break; + + case 102: + + { + (yyval.sqlStmt) = + new UpdateSqlStatement((yyvsp[(2) - (8)].tblName), (yyvsp[(4) - (8)].colAssignmentList)); + ; + } + break; + + case 103: + + { + (yyval.colAssignmentList) = new ColumnAssignmentList(); + (yyval.colAssignmentList)->push_back((yyvsp[(1) - (1)].colAssignment)); + ; + } + break; + + case 104: + + { + (yyval.colAssignmentList) = (yyvsp[(1) - (3)].colAssignmentList); + (yyval.colAssignmentList)->push_back((yyvsp[(3) - (3)].colAssignment)); + ; + } + break; + + case 105: + + { + (yyval.colAssignment) = new ColumnAssignment(); + (yyval.colAssignment)->fColumn = (yyvsp[(1) - (3)].strval); + (yyval.colAssignment)->fOperator = (yyvsp[(2) - (3)].strval); + (yyval.colAssignment)->fScalarExpression = (yyvsp[(3) - (3)].strval); + ; + } + break; + + case 106: + + { + (yyval.colAssignment) = new ColumnAssignment(); + (yyval.colAssignment)->fColumn = (yyvsp[(1) - (3)].strval); + (yyval.colAssignment)->fOperator = (yyvsp[(2) - (3)].strval); + (yyval.colAssignment)->fScalarExpression = (yyvsp[(3) - (3)].strval); + ; + } + break; + + case 107: + + { + (yyval.sqlStmt) = new UpdateSqlStatement( + (yyvsp[(2) - (5)].tblName), (yyvsp[(4) - (5)].colAssignmentList), (yyvsp[(5) - (5)].whereClause)); + ; + } + break; + + case 111: + + { + (yyval.whereClause) = NULL; + ; + } + break; + + case 112: + + { + (yyval.whereClause) = (yyvsp[(1) - (1)].whereClause); + ; + } + break; + + case 118: + + { + (yyval.querySpec) = new QuerySpec(); + + if (NULL != (yyvsp[(2) - (4)].strval)) + (yyval.querySpec)->fOptionAllOrDistinct = (yyvsp[(2) - (4)].strval); + + (yyval.querySpec)->fSelectFilterPtr = (yyvsp[(3) - (4)].selectFilter); + (yyval.querySpec)->fTableExpressionPtr = (yyvsp[(4) - (4)].tableExpression); + + ; + } + break; + + case 119: + + { + (yyval.selectFilter) = new SelectFilter((yyvsp[(1) - (1)].colNameList)); + ; + } + break; + + case 120: + + { + (yyval.selectFilter) = new SelectFilter(); + ; + } + break; + + case 121: + + { + (yyval.tableExpression) = new TableExpression(); + (yyval.tableExpression)->fFromClausePtr = (yyvsp[(1) - (4)].fromClause); + (yyval.tableExpression)->fWhereClausePtr = (yyvsp[(2) - (4)].whereClause); + (yyval.tableExpression)->fGroupByPtr = (yyvsp[(3) - (4)].groupByClause); + (yyval.tableExpression)->fHavingPtr = (yyvsp[(4) - (4)].havingClause); + ; + } + break; + + case 122: + + { + (yyval.fromClause) = new FromClause(); + (yyval.fromClause)->fTableListPtr = (yyvsp[(2) - (2)].tableNameList); + ; + } + break; + + case 123: + + { + (yyval.tableNameList) = new TableNameList(); + (yyval.tableNameList)->push_back((yyvsp[(1) - (1)].tblName)); + ; + } + break; + + case 124: + + { + (yyval.tableNameList) = (yyvsp[(1) - (3)].tableNameList); + (yyval.tableNameList)->push_back((yyvsp[(3) - (3)].tblName)); + ; + } + break; + + case 127: + + { + (yyval.whereClause) = new WhereClause(); + (yyval.whereClause)->fSearchConditionPtr = (yyvsp[(2) - (2)].searchCondition); + ; + } + break; + + case 128: + + { + (yyval.groupByClause) = NULL; + ; + } + break; + + case 129: + + { + (yyval.groupByClause) = new GroupByClause(); + (yyval.groupByClause)->fColumnNamesListPtr = (yyvsp[(3) - (3)].colNameList); + ; + } + break; + + case 130: + + { + (yyval.colNameList) = new ColumnNameList(); + (yyval.colNameList)->push_back((yyvsp[(1) - (1)].strval)); + ; + } + break; + + case 131: + + { + (yyval.colNameList) = (yyvsp[(1) - (3)].colNameList); + (yyval.colNameList)->push_back((yyvsp[(3) - (3)].strval)); + ; + } + break; + + case 132: + + { + (yyval.havingClause) = NULL; + ; + } + break; + + case 133: + + { + (yyval.havingClause) = new HavingClause(); + (yyval.havingClause)->fSearchConditionPtr = (yyvsp[(2) - (2)].searchCondition); + ; + } + break; + + case 134: + + { + (yyval.searchCondition) = new SearchCondition; + (yyval.searchCondition)->fLHSearchConditionPtr = (yyvsp[(1) - (3)].searchCondition); + (yyval.searchCondition)->fOperator = "OR"; + (yyval.searchCondition)->fRHSearchConditionPtr = (yyvsp[(3) - (3)].searchCondition); + ; + } + break; + + case 135: + + { + (yyval.searchCondition) = new SearchCondition; + (yyval.searchCondition)->fLHSearchConditionPtr = (yyvsp[(1) - (3)].searchCondition); + (yyval.searchCondition)->fOperator = "AND"; + (yyval.searchCondition)->fRHSearchConditionPtr = (yyvsp[(3) - (3)].searchCondition); + ; + } + break; + + case 136: + + { + (yyval.searchCondition) = new SearchCondition; + (yyval.searchCondition)->fOperator = "NOT"; + (yyval.searchCondition)->fRHSearchConditionPtr = (yyvsp[(2) - (2)].searchCondition); + ; + } + break; + + case 137: + + { + (yyval.searchCondition) = new SearchCondition; + ; + } + break; + + case 138: + + { + (yyval.searchCondition) = new SearchCondition; + (yyval.searchCondition)->fPredicatePtr = (yyvsp[(1) - (1)].predicate); + ; + } + break; + + case 139: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].comparisonPredicate); + ; + } + break; + + case 140: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].betweenPredicate); + ; + } + break; + + case 141: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].likePredicate); + ; + } + break; + + case 142: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].nullTestPredicate); + ; + } + break; + + case 143: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].inPredicate); + ; + } + break; + + case 144: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].allOrAnyPredicate); + ; + } + break; + + case 145: + + { + (yyval.predicate) = (yyvsp[(1) - (1)].existPredicate); + ; + } + break; + + case 146: + + { + (yyval.comparisonPredicate) = new ComparisonPredicate(); + (yyval.comparisonPredicate)->fLHScalarExpression = (yyvsp[(1) - (3)].strval); + (yyval.comparisonPredicate)->fOperator = (yyvsp[(2) - (3)].strval); + (yyval.comparisonPredicate)->fRHScalarExpression = (yyvsp[(3) - (3)].strval); + ; + } + break; + + case 147: + + { + (yyval.comparisonPredicate) = new ComparisonPredicate(); + (yyval.comparisonPredicate)->fLHScalarExpression = (yyvsp[(1) - (3)].strval); + (yyval.comparisonPredicate)->fOperator = (yyvsp[(2) - (3)].strval); + (yyval.comparisonPredicate)->fSubQuerySpec = (yyvsp[(3) - (3)].querySpec); + ; + } + break; + + case 148: + + { + (yyval.betweenPredicate) = new BetweenPredicate(); + (yyval.betweenPredicate)->fLHScalarExpression = (yyvsp[(1) - (6)].strval); + (yyval.betweenPredicate)->fOperator1 = "NOT BETWEEN"; + (yyval.betweenPredicate)->fRH1ScalarExpression = (yyvsp[(4) - (6)].strval); + (yyval.betweenPredicate)->fOperator2 = "AND"; + (yyval.betweenPredicate)->fRH2ScalarExpression = (yyvsp[(6) - (6)].strval); + ; + } + break; + + case 149: + + { + (yyval.betweenPredicate) = new BetweenPredicate(); + (yyval.betweenPredicate)->fLHScalarExpression = (yyvsp[(1) - (5)].strval); + (yyval.betweenPredicate)->fOperator1 = "BETWEEN"; + (yyval.betweenPredicate)->fRH1ScalarExpression = (yyvsp[(3) - (5)].strval); + (yyval.betweenPredicate)->fOperator2 = "AND"; + (yyval.betweenPredicate)->fRH2ScalarExpression = (yyvsp[(5) - (5)].strval); + ; + } + break; + + case 150: + + { + (yyval.likePredicate) = new LikePredicate(); + (yyval.likePredicate)->fLHScalarExpression = (yyvsp[(1) - (5)].strval); + (yyval.likePredicate)->fOperator = "NOT LIKE"; + (yyval.likePredicate)->fAtom = (yyvsp[(4) - (5)].strval); + (yyval.likePredicate)->fOptionalEscapePtr = (yyvsp[(5) - (5)].escape); + ; + } + break; + + case 151: + + { + (yyval.likePredicate) = new LikePredicate(); + (yyval.likePredicate)->fLHScalarExpression = (yyvsp[(1) - (4)].strval); + (yyval.likePredicate)->fOperator = "LIKE"; + (yyval.likePredicate)->fAtom = (yyvsp[(3) - (4)].strval); + (yyval.likePredicate)->fOptionalEscapePtr = (yyvsp[(4) - (4)].escape); + ; + } + break; + + case 152: + + { + (yyval.escape) = NULL; + ; + } + break; + + case 153: + + { + (yyval.escape) = new Escape(); + (yyval.escape)->fEscapeChar = (yyvsp[(2) - (2)].strval); + ; + } + break; + + case 154: + + { + (yyval.nullTestPredicate) = new NullTestPredicate(); + (yyval.nullTestPredicate)->fOperator = "IS NOT NULL"; + (yyval.nullTestPredicate)->fColumnRef = (yyvsp[(1) - (4)].strval); + ; + } + break; + + case 155: + + { + (yyval.nullTestPredicate) = new NullTestPredicate(); + (yyval.nullTestPredicate)->fOperator = "IS NULL"; + (yyval.nullTestPredicate)->fColumnRef = (yyvsp[(1) - (3)].strval); + ; + } + break; + + case 156: + + { + (yyval.inPredicate) = new InPredicate(); + (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (6)].strval); + (yyval.inPredicate)->fOperator = "NOT IN"; + (yyval.inPredicate)->fSubQuerySpecPtr = (yyvsp[(5) - (6)].querySpec); + ; + } + break; + + case 157: + + { + (yyval.inPredicate) = new InPredicate(); + (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (5)].strval); + (yyval.inPredicate)->fOperator = "IN"; + (yyval.inPredicate)->fSubQuerySpecPtr = (yyvsp[(4) - (5)].querySpec); + ; + } + break; + + case 158: + + { + (yyval.inPredicate) = new InPredicate(); + (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (6)].strval); + (yyval.inPredicate)->fOperator = "NOT IN"; + (yyval.inPredicate)->fAtomList = *(yyvsp[(5) - (6)].atomList); + delete (yyvsp[(5) - (6)].atomList); + ; + } + break; + + case 159: + + { + (yyval.inPredicate) = new InPredicate(); + (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (5)].strval); + (yyval.inPredicate)->fOperator = "IN"; + (yyval.inPredicate)->fAtomList = *(yyvsp[(4) - (5)].atomList); + delete (yyvsp[(4) - (5)].atomList); + ; + } + break; + + case 160: + + { + (yyval.atomList) = new AtomList(); + (yyval.atomList)->push_back((yyvsp[(1) - (1)].strval)); + ; + } + break; + + case 161: + + { + (yyval.atomList) = (yyvsp[(1) - (3)].atomList); + (yyval.atomList)->push_back((yyvsp[(3) - (3)].strval)); + ; + } + break; + + case 162: + + { + (yyval.allOrAnyPredicate) = new AllOrAnyPredicate(); + (yyval.allOrAnyPredicate)->fScalarExpression = (yyvsp[(1) - (4)].strval); + (yyval.allOrAnyPredicate)->fOperator = (yyvsp[(2) - (4)].strval); + (yyval.allOrAnyPredicate)->fAnyAllSome = (yyvsp[(3) - (4)].strval); + (yyval.allOrAnyPredicate)->fSubQuerySpecPtr = (yyvsp[(4) - (4)].querySpec); + + ; + } + break; + + case 166: + + { + (yyval.existPredicate) = new ExistanceTestPredicate(); + (yyval.existPredicate)->fSubQuerySpecPtr = (yyvsp[(2) - (2)].querySpec); + ; + } + break; + + case 167: + + { + (yyval.querySpec) = new QuerySpec(); + + if (NULL != (yyvsp[(3) - (6)].strval)) + (yyval.querySpec)->fOptionAllOrDistinct = (yyvsp[(3) - (6)].strval); + + (yyval.querySpec)->fSelectFilterPtr = (yyvsp[(4) - (6)].selectFilter); + (yyval.querySpec)->fTableExpressionPtr = (yyvsp[(5) - (6)].tableExpression); + ; + } + break; + + case 168: + + { + std::string str = (yyvsp[(1) - (3)].strval); + str += " + "; + str += (yyvsp[(3) - (3)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 169: + + { + std::string str = (yyvsp[(1) - (3)].strval); + str += " - "; + str += (yyvsp[(3) - (3)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 170: + + { + std::string str = (yyvsp[(1) - (3)].strval); + str += " * "; + str += (yyvsp[(3) - (3)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 171: + + { + std::string str = (yyvsp[(1) - (3)].strval); + str += " / "; + str += (yyvsp[(3) - (3)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 172: + + { + std::string str = "+ "; + str += (yyvsp[(2) - (2)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 173: + + { + std::string str = "- "; + str += (yyvsp[(2) - (2)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 177: + + { + (yyval.strval) = (yyvsp[(2) - (3)].strval); + ; + } + break; + + case 178: + + { + (yyval.colNameList) = new ColumnNameList; + (yyval.colNameList)->push_back((yyvsp[(1) - (1)].strval)); + + ; + } + break; + + case 179: + + { + (yyval.colNameList) = (yyvsp[(1) - (3)].colNameList); + (yyval.colNameList)->push_back((yyvsp[(3) - (3)].strval)); + ; + } + break; + + case 184: + + { + std::string str = (yyvsp[(1) - (2)].strval); + str += " "; + str += (yyvsp[(2) - (2)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 185: + + { + std::string str = (yyvsp[(1) - (3)].strval); + str += " "; + str += (yyvsp[(2) - (3)].strval); + str += " "; + str += (yyvsp[(3) - (3)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 186: + + { + std::string str = (yyvsp[(1) - (4)].strval); + str += "("; + str += "*"; + str += ")"; + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 187: + + { + std::string str = (yyvsp[(1) - (5)].strval); + str += "("; + str += (yyvsp[(3) - (5)].strval); + str += " "; + str += (yyvsp[(4) - (5)].strval); + str += ")"; + (yyval.strval) = copy_string(str.c_str()); + + ; + } + break; + + case 188: + + { + std::string str = (yyvsp[(1) - (5)].strval); + str += "("; + str += (yyvsp[(3) - (5)].strval); + str += " "; + str += (yyvsp[(4) - (5)].strval); + str += ")"; + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 189: + + { + std::string str = (yyvsp[(1) - (4)].strval); + str += "("; + str += (yyvsp[(3) - (4)].strval); + str += ")"; + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 194: + + { + (yyval.tblName) = new TableName((yyvsp[(1) - (3)].strval), (yyvsp[(3) - (3)].strval)); + ; + } + break; + + case 195: + + { + if (default_schema.size()) + (yyval.tblName) = new TableName((char*)default_schema.c_str(), (yyvsp[(1) - (1)].strval)); + else + (yyval.tblName) = new TableName((yyvsp[(1) - (1)].strval)); + + ; + } + break; + + case 197: + + { + std::string str = (yyvsp[(1) - (3)].strval); + str += "."; + str += (yyvsp[(3) - (3)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 198: + + { + std::string str = (yyvsp[(1) - (5)].strval); + str += "."; + str += (yyvsp[(3) - (5)].strval); + str += "."; + str += (yyvsp[(5) - (5)].strval); + (yyval.strval) = copy_string(str.c_str()); + ; + } + break; + + case 218: + + { + (yyval.sqlStmt) = NULL; + ; + } + break; + + case 219: + + { + (yyval.sqlStmt) = NULL; + ; + } + break; + + default: break; + } + + YY_SYMBOL_PRINT("-> $$ =", yyr1[yyn], &yyval, &yyloc); + + YYPOPSTACK(yylen); + yylen = 0; + YY_STACK_PRINT(yyss, yyssp); + + *++yyvsp = yyval; + + /* Now `shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + + yyn = yyr1[yyn]; + + yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; + + if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) + yystate = yytable[yystate]; + else + yystate = yydefgoto[yyn - YYNTOKENS]; + + goto yynewstate; + + /*------------------------------------. + | yyerrlab -- here on detecting error | + `------------------------------------*/ +yyerrlab: + + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; +#if !YYERROR_VERBOSE + yyerror(YY_("syntax error")); +#else + { + YYSIZE_T yysize = yysyntax_error(0, yystate, yychar); + + if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) + { + YYSIZE_T yyalloc = 2 * yysize; + + if (!(yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) + yyalloc = YYSTACK_ALLOC_MAXIMUM; + + if (yymsg != yymsgbuf) + YYSTACK_FREE(yymsg); + + yymsg = (char*)YYSTACK_ALLOC(yyalloc); + + if (yymsg) + yymsg_alloc = yyalloc; + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + } + } + + if (0 < yysize && yysize <= yymsg_alloc) + { + (void)yysyntax_error(yymsg, yystate, yychar); + yyerror(yymsg); + } + else + { + yyerror(YY_("syntax error")); + + if (yysize != 0) + goto yyexhaustedlab; + } + } +#endif + } + + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ if (yychar <= YYEOF) { - yychar = yytoken = YYEOF; - YYDPRINTF ((stderr, "Now at end of input.\n")); + /* Return failure if at end of input. */ + if (yychar == YYEOF) + YYABORT; } else { - yytoken = YYTRANSLATE (yychar); - YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); + yydestruct("Error: discarding", yytoken, &yylval); + yychar = YYEMPTY; } + } - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken; + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; - if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) - goto yydefault; - - yyn = yytable[yyn]; - - if (yyn <= 0) - { - if (yyn == 0 || yyn == YYTABLE_NINF) - goto yyerrlab; - - yyn = -yyn; - goto yyreduce; - } - - /* Count tokens shifted since error; after three, turn off error - status. */ - if (yyerrstatus) - yyerrstatus--; - - /* Shift the lookahead token. */ - YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - - yystate = yyn; - *++yyvsp = yylval; - - goto yynewstate; - - - /*-----------------------------------------------------------. - | yydefault -- do the default action for the current state. | - `-----------------------------------------------------------*/ -yydefault: - yyn = yydefact[yystate]; - - if (yyn == 0) - goto yyerrlab; - - goto yyreduce; - - - /*-----------------------------. - | yyreduce -- Do a reduction. | - `-----------------------------*/ -yyreduce: - /* yyn is the number of a rule to reduce with. */ - yylen = yyr2[yyn]; - - /* If YYLEN is nonzero, implement the default value of the action: - `$$ = $1'. - - Otherwise, the following line sets YYVAL to garbage. - This behavior is undocumented and Bison - users should not rely upon it. Assigning to YYVAL - unconditionally makes the parser a bit smaller, and it avoids a - GCC warning that YYVAL may be used uninitialized. */ - yyval = yyvsp[1 - yylen]; - - - YY_REDUCE_PRINT (yyn); - - switch (yyn) - { - case 2: - - { - if ((yyvsp[(1) - (2)].sqlStmt) != NULL) - { - (yyval.sqlStmtList) = parseTree; - (yyval.sqlStmtList)->push_back((yyvsp[(1) - (2)].sqlStmt)); - } - else - { - (yyval.sqlStmtList) = NULL; - } - - ; - } - break; - - case 3: - - { - - if ((yyvsp[(1) - (3)].sqlStmtList) != NULL) - { - parseTree = (yyvsp[(1) - (3)].sqlStmtList); - } - - ; - } - break; - - case 4: - - { - (yyval.sqlStmt) = NULL; ; - } - break; - - case 35: - - { - (yyval.colNameList) = new ColumnNameList; - (yyval.colNameList)->push_back((yyvsp[(1) - (1)].strval)); - ; - } - break; - - case 36: - - { - (yyval.colNameList) = (yyvsp[(1) - (3)].colNameList); - (yyval.colNameList)->push_back((yyvsp[(3) - (3)].strval)); - ; - } - break; - - case 40: - - { - (yyval.colNameList) = NULL; ; - } - break; - - case 41: - - { - (yyval.colNameList) = (yyvsp[(2) - (3)].colNameList);; - } - break; - - case 59: - - { - (yyval.sqlStmt) = NULL; ; - } - break; - - case 82: - - { ;} - break; - - case 83: - - { - (yyval.sqlStmt) = new CommandSqlStatement("COMMIT"); - ; - } - break; - - case 84: - - { - (yyval.sqlStmt) = new CommandSqlStatement("COMMIT"); - ; - } - break; - - case 85: - - { - (yyval.sqlStmt) = new DeleteSqlStatement((yyvsp[(3) - (7)].tblName)); - - ; - } - break; - - case 86: - - { - (yyval.sqlStmt) = new DeleteSqlStatement((yyvsp[(3) - (4)].tblName), (yyvsp[(4) - (4)].whereClause)); - ; - } - break; - - case 87: - - { ;} - break; - - case 88: - - { - if (NULL == (yyvsp[(4) - (5)].colNameList)) - (yyval.sqlStmt) = new InsertSqlStatement((yyvsp[(3) - (5)].tblName), (yyvsp[(5) - (5)].valsOrQuery)); - else - (yyval.sqlStmt) = new InsertSqlStatement((yyvsp[(3) - (5)].tblName), (yyvsp[(4) - (5)].colNameList), (yyvsp[(5) - (5)].valsOrQuery)); - - ; - } - break; - - case 89: - - { - (yyval.valsOrQuery) = new ValuesOrQuery((yyvsp[(3) - (4)].valsList)); - ; - } - break; - - case 90: - - { - (yyval.valsOrQuery) = new ValuesOrQuery((yyvsp[(1) - (1)].querySpec)); - ; - } - break; - - case 91: - - { - (yyval.valsList) = new ValuesList; - (yyval.valsList)->push_back((yyvsp[(1) - (1)].strval)); - ; - } - break; - - case 92: - - { - (yyval.valsList) = (yyvsp[(1) - (3)].valsList); - (yyval.valsList)->push_back((yyvsp[(3) - (3)].strval)); - ; - } - break; - - case 95: - - { ;} - break; - - case 96: - - { - (yyval.sqlStmt) = new CommandSqlStatement("ROLLBACK"); - ; - } - break; - - case 97: - - { - (yyval.sqlStmt) = new CommandSqlStatement("ROLLBACK"); - ; - } - break; - - case 98: - - { ;} - break; - - case 99: - - { - (yyval.strval) = NULL; ; - } - break; - - case 100: - - { - (yyval.strval) = (yyvsp[(1) - (1)].strval); ; - } - break; - - case 101: - - { - (yyval.strval) = (yyvsp[(1) - (1)].strval); ; - } - break; - - case 102: - - { - (yyval.sqlStmt) = new UpdateSqlStatement((yyvsp[(2) - (8)].tblName), (yyvsp[(4) - (8)].colAssignmentList)); - ; - } - break; - - case 103: - - { - (yyval.colAssignmentList) = new ColumnAssignmentList(); - (yyval.colAssignmentList)->push_back((yyvsp[(1) - (1)].colAssignment)); - ; - } - break; - - case 104: - - { - (yyval.colAssignmentList) = (yyvsp[(1) - (3)].colAssignmentList); - (yyval.colAssignmentList)->push_back((yyvsp[(3) - (3)].colAssignment)); - ; - } - break; - - case 105: - - { - (yyval.colAssignment) = new ColumnAssignment(); - (yyval.colAssignment)->fColumn = (yyvsp[(1) - (3)].strval); - (yyval.colAssignment)->fOperator = (yyvsp[(2) - (3)].strval); - (yyval.colAssignment)->fScalarExpression = (yyvsp[(3) - (3)].strval); - ; - } - break; - - case 106: - - { - (yyval.colAssignment) = new ColumnAssignment(); - (yyval.colAssignment)->fColumn = (yyvsp[(1) - (3)].strval); - (yyval.colAssignment)->fOperator = (yyvsp[(2) - (3)].strval); - (yyval.colAssignment)->fScalarExpression = (yyvsp[(3) - (3)].strval); - ; - } - break; - - case 107: - - { - (yyval.sqlStmt) = new UpdateSqlStatement((yyvsp[(2) - (5)].tblName), (yyvsp[(4) - (5)].colAssignmentList), (yyvsp[(5) - (5)].whereClause)); - ; - } - break; - - case 111: - - { - (yyval.whereClause) = NULL; ; - } - break; - - case 112: - - { - (yyval.whereClause) = (yyvsp[(1) - (1)].whereClause); ; - } - break; - - case 118: - - { - (yyval.querySpec) = new QuerySpec(); - - if (NULL != (yyvsp[(2) - (4)].strval)) - (yyval.querySpec)->fOptionAllOrDistinct = (yyvsp[(2) - (4)].strval); - - (yyval.querySpec)->fSelectFilterPtr = (yyvsp[(3) - (4)].selectFilter); - (yyval.querySpec)->fTableExpressionPtr = (yyvsp[(4) - (4)].tableExpression); - - ; - } - break; - - case 119: - - { - (yyval.selectFilter) = new SelectFilter((yyvsp[(1) - (1)].colNameList)); ; - } - break; - - case 120: - - { - (yyval.selectFilter) = new SelectFilter(); ; - } - break; - - case 121: - - { - (yyval.tableExpression) = new TableExpression(); - (yyval.tableExpression)->fFromClausePtr = (yyvsp[(1) - (4)].fromClause); - (yyval.tableExpression)->fWhereClausePtr = (yyvsp[(2) - (4)].whereClause); - (yyval.tableExpression)->fGroupByPtr = (yyvsp[(3) - (4)].groupByClause); - (yyval.tableExpression)->fHavingPtr = (yyvsp[(4) - (4)].havingClause); - ; - } - break; - - case 122: - - { - (yyval.fromClause) = new FromClause(); - (yyval.fromClause)->fTableListPtr = (yyvsp[(2) - (2)].tableNameList); - ; - } - break; - - case 123: - - { - (yyval.tableNameList) = new TableNameList(); - (yyval.tableNameList)->push_back((yyvsp[(1) - (1)].tblName)); - ; - } - break; - - case 124: - - { - (yyval.tableNameList) = (yyvsp[(1) - (3)].tableNameList); - (yyval.tableNameList)->push_back((yyvsp[(3) - (3)].tblName)); - ; - } - break; - - case 127: - - { - (yyval.whereClause) = new WhereClause(); - (yyval.whereClause)->fSearchConditionPtr = (yyvsp[(2) - (2)].searchCondition); - ; - } - break; - - case 128: - - { - (yyval.groupByClause) = NULL; ; - } - break; - - case 129: - - { - (yyval.groupByClause) = new GroupByClause(); - (yyval.groupByClause)->fColumnNamesListPtr = (yyvsp[(3) - (3)].colNameList); - ; - } - break; - - case 130: - - { - (yyval.colNameList) = new ColumnNameList(); - (yyval.colNameList)->push_back((yyvsp[(1) - (1)].strval)); - ; - } - break; - - case 131: - - { - (yyval.colNameList) = (yyvsp[(1) - (3)].colNameList); - (yyval.colNameList)->push_back((yyvsp[(3) - (3)].strval)); - ; - } - break; - - case 132: - - { - (yyval.havingClause) = NULL; ; - } - break; - - case 133: - - { - (yyval.havingClause) = new HavingClause(); - (yyval.havingClause)->fSearchConditionPtr = (yyvsp[(2) - (2)].searchCondition); - ; - } - break; - - case 134: - - { - (yyval.searchCondition) = new SearchCondition; - (yyval.searchCondition)->fLHSearchConditionPtr = (yyvsp[(1) - (3)].searchCondition); - (yyval.searchCondition)->fOperator = "OR"; - (yyval.searchCondition)->fRHSearchConditionPtr = (yyvsp[(3) - (3)].searchCondition); - ; - } - break; - - case 135: - - { - (yyval.searchCondition) = new SearchCondition; - (yyval.searchCondition)->fLHSearchConditionPtr = (yyvsp[(1) - (3)].searchCondition); - (yyval.searchCondition)->fOperator = "AND"; - (yyval.searchCondition)->fRHSearchConditionPtr = (yyvsp[(3) - (3)].searchCondition); - ; - } - break; - - case 136: - - { - (yyval.searchCondition) = new SearchCondition; - (yyval.searchCondition)->fOperator = "NOT"; - (yyval.searchCondition)->fRHSearchConditionPtr = (yyvsp[(2) - (2)].searchCondition); - ; - } - break; - - case 137: - - { - (yyval.searchCondition) = new SearchCondition; - ; - } - break; - - case 138: - - { - - (yyval.searchCondition) = new SearchCondition; - (yyval.searchCondition)->fPredicatePtr = (yyvsp[(1) - (1)].predicate); - ; - } - break; - - case 139: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].comparisonPredicate); - ; - } - break; - - case 140: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].betweenPredicate); - ; - } - break; - - case 141: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].likePredicate); - ; - } - break; - - case 142: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].nullTestPredicate); - ; - } - break; - - case 143: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].inPredicate); - ; - } - break; - - case 144: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].allOrAnyPredicate); - ; - } - break; - - case 145: - - { - (yyval.predicate) = (yyvsp[(1) - (1)].existPredicate); - ; - } - break; - - case 146: - - { - (yyval.comparisonPredicate) = new ComparisonPredicate(); - (yyval.comparisonPredicate)->fLHScalarExpression = (yyvsp[(1) - (3)].strval); - (yyval.comparisonPredicate)->fOperator = (yyvsp[(2) - (3)].strval); - (yyval.comparisonPredicate)->fRHScalarExpression = (yyvsp[(3) - (3)].strval); - ; - } - break; - - case 147: - - { - (yyval.comparisonPredicate) = new ComparisonPredicate(); - (yyval.comparisonPredicate)->fLHScalarExpression = (yyvsp[(1) - (3)].strval); - (yyval.comparisonPredicate)->fOperator = (yyvsp[(2) - (3)].strval); - (yyval.comparisonPredicate)->fSubQuerySpec = (yyvsp[(3) - (3)].querySpec); - ; - } - break; - - case 148: - - { - (yyval.betweenPredicate) = new BetweenPredicate(); - (yyval.betweenPredicate)->fLHScalarExpression = (yyvsp[(1) - (6)].strval); - (yyval.betweenPredicate)->fOperator1 = "NOT BETWEEN"; - (yyval.betweenPredicate)->fRH1ScalarExpression = (yyvsp[(4) - (6)].strval); - (yyval.betweenPredicate)->fOperator2 = "AND"; - (yyval.betweenPredicate)->fRH2ScalarExpression = (yyvsp[(6) - (6)].strval); - ; - } - break; - - case 149: - - { - (yyval.betweenPredicate) = new BetweenPredicate(); - (yyval.betweenPredicate)->fLHScalarExpression = (yyvsp[(1) - (5)].strval); - (yyval.betweenPredicate)->fOperator1 = "BETWEEN"; - (yyval.betweenPredicate)->fRH1ScalarExpression = (yyvsp[(3) - (5)].strval); - (yyval.betweenPredicate)->fOperator2 = "AND"; - (yyval.betweenPredicate)->fRH2ScalarExpression = (yyvsp[(5) - (5)].strval); - ; - } - break; - - case 150: - - { - (yyval.likePredicate) = new LikePredicate(); - (yyval.likePredicate)->fLHScalarExpression = (yyvsp[(1) - (5)].strval); - (yyval.likePredicate)->fOperator = "NOT LIKE"; - (yyval.likePredicate)->fAtom = (yyvsp[(4) - (5)].strval); - (yyval.likePredicate)->fOptionalEscapePtr = (yyvsp[(5) - (5)].escape); - ; - } - break; - - case 151: - - { - (yyval.likePredicate) = new LikePredicate(); - (yyval.likePredicate)->fLHScalarExpression = (yyvsp[(1) - (4)].strval); - (yyval.likePredicate)->fOperator = "LIKE"; - (yyval.likePredicate)->fAtom = (yyvsp[(3) - (4)].strval); - (yyval.likePredicate)->fOptionalEscapePtr = (yyvsp[(4) - (4)].escape); - ; - } - break; - - case 152: - - { - (yyval.escape) = NULL; ; - } - break; - - case 153: - - { - (yyval.escape) = new Escape(); - (yyval.escape)->fEscapeChar = (yyvsp[(2) - (2)].strval); - ; - } - break; - - case 154: - - { - (yyval.nullTestPredicate) = new NullTestPredicate(); - (yyval.nullTestPredicate)->fOperator = "IS NOT NULL"; - (yyval.nullTestPredicate)->fColumnRef = (yyvsp[(1) - (4)].strval); - ; - } - break; - - case 155: - - { - (yyval.nullTestPredicate) = new NullTestPredicate(); - (yyval.nullTestPredicate)->fOperator = "IS NULL"; - (yyval.nullTestPredicate)->fColumnRef = (yyvsp[(1) - (3)].strval); - ; - } - break; - - case 156: - - { - (yyval.inPredicate) = new InPredicate(); - (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (6)].strval); - (yyval.inPredicate)->fOperator = "NOT IN"; - (yyval.inPredicate)->fSubQuerySpecPtr = (yyvsp[(5) - (6)].querySpec); - ; - } - break; - - case 157: - - { - (yyval.inPredicate) = new InPredicate(); - (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (5)].strval); - (yyval.inPredicate)->fOperator = "IN"; - (yyval.inPredicate)->fSubQuerySpecPtr = (yyvsp[(4) - (5)].querySpec); - ; - } - break; - - case 158: - - { - (yyval.inPredicate) = new InPredicate(); - (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (6)].strval); - (yyval.inPredicate)->fOperator = "NOT IN"; - (yyval.inPredicate)->fAtomList = *(yyvsp[(5) - (6)].atomList); - delete (yyvsp[(5) - (6)].atomList); - ; - } - break; - - case 159: - - { - (yyval.inPredicate) = new InPredicate(); - (yyval.inPredicate)->fScalarExpression = (yyvsp[(1) - (5)].strval); - (yyval.inPredicate)->fOperator = "IN"; - (yyval.inPredicate)->fAtomList = *(yyvsp[(4) - (5)].atomList); - delete (yyvsp[(4) - (5)].atomList); - ; - } - break; - - case 160: - - { - (yyval.atomList) = new AtomList(); - (yyval.atomList)->push_back((yyvsp[(1) - (1)].strval)); - ; - } - break; - - case 161: - - { - (yyval.atomList) = (yyvsp[(1) - (3)].atomList); - (yyval.atomList)->push_back((yyvsp[(3) - (3)].strval)); - ; - } - break; - - case 162: - - { - (yyval.allOrAnyPredicate) = new AllOrAnyPredicate(); - (yyval.allOrAnyPredicate)->fScalarExpression = (yyvsp[(1) - (4)].strval); - (yyval.allOrAnyPredicate)->fOperator = (yyvsp[(2) - (4)].strval); - (yyval.allOrAnyPredicate)->fAnyAllSome = (yyvsp[(3) - (4)].strval); - (yyval.allOrAnyPredicate)->fSubQuerySpecPtr = (yyvsp[(4) - (4)].querySpec); - - ; - } - break; - - case 166: - - { - (yyval.existPredicate) = new ExistanceTestPredicate(); - (yyval.existPredicate)->fSubQuerySpecPtr = (yyvsp[(2) - (2)].querySpec); - ; - } - break; - - case 167: - - { - (yyval.querySpec) = new QuerySpec(); - - if (NULL != (yyvsp[(3) - (6)].strval)) - (yyval.querySpec)->fOptionAllOrDistinct = (yyvsp[(3) - (6)].strval); - - (yyval.querySpec)->fSelectFilterPtr = (yyvsp[(4) - (6)].selectFilter); - (yyval.querySpec)->fTableExpressionPtr = (yyvsp[(5) - (6)].tableExpression); - ; - } - break; - - case 168: - - { - std::string str = (yyvsp[(1) - (3)].strval); - str += " + "; - str += (yyvsp[(3) - (3)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 169: - - { - std::string str = (yyvsp[(1) - (3)].strval); - str += " - "; - str += (yyvsp[(3) - (3)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 170: - - { - std::string str = (yyvsp[(1) - (3)].strval); - str += " * "; - str += (yyvsp[(3) - (3)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 171: - - { - std::string str = (yyvsp[(1) - (3)].strval); - str += " / "; - str += (yyvsp[(3) - (3)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 172: - - { - std::string str = "+ "; - str += (yyvsp[(2) - (2)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 173: - - { - std::string str = "- "; - str += (yyvsp[(2) - (2)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 177: - - { - (yyval.strval) = (yyvsp[(2) - (3)].strval); ; - } - break; - - case 178: - - { - (yyval.colNameList) = new ColumnNameList; - (yyval.colNameList)->push_back((yyvsp[(1) - (1)].strval)); - - ; - } - break; - - case 179: - - { - (yyval.colNameList) = (yyvsp[(1) - (3)].colNameList); - (yyval.colNameList)->push_back((yyvsp[(3) - (3)].strval)); - ; - } - break; - - case 184: - - { - std::string str = (yyvsp[(1) - (2)].strval); - str += " "; - str += (yyvsp[(2) - (2)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 185: - - { - std::string str = (yyvsp[(1) - (3)].strval); - str += " "; - str += (yyvsp[(2) - (3)].strval); - str += " "; - str += (yyvsp[(3) - (3)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 186: - - { - std::string str = (yyvsp[(1) - (4)].strval); - str += "("; - str += "*"; - str += ")"; - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 187: - - { - std::string str = (yyvsp[(1) - (5)].strval); - str += "("; - str += (yyvsp[(3) - (5)].strval); - str += " "; - str += (yyvsp[(4) - (5)].strval); - str += ")"; - (yyval.strval) = copy_string(str.c_str()); - - ; - } - break; - - case 188: - - { - std::string str = (yyvsp[(1) - (5)].strval); - str += "("; - str += (yyvsp[(3) - (5)].strval); - str += " "; - str += (yyvsp[(4) - (5)].strval); - str += ")"; - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 189: - - { - std::string str = (yyvsp[(1) - (4)].strval); - str += "("; - str += (yyvsp[(3) - (4)].strval); - str += ")"; - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 194: - - { - (yyval.tblName) = new TableName((yyvsp[(1) - (3)].strval), (yyvsp[(3) - (3)].strval));; - } - break; - - case 195: - - { - if (default_schema.size()) - (yyval.tblName) = new TableName((char*)default_schema.c_str(), (yyvsp[(1) - (1)].strval)); - else - (yyval.tblName) = new TableName((yyvsp[(1) - (1)].strval)); - - ; - } - break; - - case 197: - - { - std::string str = (yyvsp[(1) - (3)].strval); - str += "."; - str += (yyvsp[(3) - (3)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 198: - - { - std::string str = (yyvsp[(1) - (5)].strval); - str += "."; - str += (yyvsp[(3) - (5)].strval); - str += "."; - str += (yyvsp[(5) - (5)].strval); - (yyval.strval) = copy_string(str.c_str()); - ; - } - break; - - case 218: - - { - (yyval.sqlStmt) = NULL; ; - } - break; - - case 219: - - { - (yyval.sqlStmt) = NULL; ; - } - break; - - - - default: - break; - } - - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); - - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - - *++yyvsp = yyval; - - /* Now `shift' the result of the reduction. Determine what state - that goes to, based on the state we popped back to and the rule - number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; - - goto yynewstate; - - - /*------------------------------------. - | yyerrlab -- here on detecting error | - `------------------------------------*/ -yyerrlab: - - /* If not already recovering from an error, report this error. */ - if (!yyerrstatus) - { - ++yynerrs; -#if ! YYERROR_VERBOSE - yyerror (YY_("syntax error")); -#else - { - YYSIZE_T yysize = yysyntax_error (0, yystate, yychar); - - if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) - { - YYSIZE_T yyalloc = 2 * yysize; - - if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) - yyalloc = YYSTACK_ALLOC_MAXIMUM; - - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); - - yymsg = (char*) YYSTACK_ALLOC (yyalloc); - - if (yymsg) - yymsg_alloc = yyalloc; - else - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - } - } - - if (0 < yysize && yysize <= yymsg_alloc) - { - (void) yysyntax_error (yymsg, yystate, yychar); - yyerror (yymsg); - } - else - { - yyerror (YY_("syntax error")); - - if (yysize != 0) - goto yyexhaustedlab; - } - } -#endif - } - - - - if (yyerrstatus == 3) - { - /* If just tried and failed to reuse lookahead token after an - error, discard it. */ - - if (yychar <= YYEOF) - { - /* Return failure if at end of input. */ - if (yychar == YYEOF) - YYABORT; - } - else - { - yydestruct ("Error: discarding", - yytoken, &yylval); - yychar = YYEMPTY; - } - } - - /* Else will try to reuse lookahead token after shifting the error - token. */ - goto yyerrlab1; - - - /*---------------------------------------------------. - | yyerrorlab -- error raised explicitly by YYERROR. | - `---------------------------------------------------*/ + /*---------------------------------------------------. + | yyerrorlab -- error raised explicitly by YYERROR. | + `---------------------------------------------------*/ yyerrorlab: - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; + /* Pacify compilers like GCC when the user code never invokes + YYERROR and the label yyerrorlab therefore never appears in user + code. */ + if (/*CONSTCOND*/ 0) + goto yyerrorlab; - /* Do not reclaim the symbols of the rule which action triggered - this YYERROR. */ - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - yystate = *yyssp; - goto yyerrlab1; + /* Do not reclaim the symbols of the rule which action triggered + this YYERROR. */ + YYPOPSTACK(yylen); + yylen = 0; + YY_STACK_PRINT(yyss, yyssp); + yystate = *yyssp; + goto yyerrlab1; - - /*-------------------------------------------------------------. - | yyerrlab1 -- common code for both syntax error and YYERROR. | - `-------------------------------------------------------------*/ + /*-------------------------------------------------------------. + | yyerrlab1 -- common code for both syntax error and YYERROR. | + `-------------------------------------------------------------*/ yyerrlab1: - yyerrstatus = 3; /* Each real token shifted decrements this. */ + yyerrstatus = 3; /* Each real token shifted decrements this. */ - for (;;) + for (;;) + { + yyn = yypact[yystate]; + + if (yyn != YYPACT_NINF) { - yyn = yypact[yystate]; + yyn += YYTERROR; - if (yyn != YYPACT_NINF) - { - yyn += YYTERROR; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + { + yyn = yytable[yyn]; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) - { - yyn = yytable[yyn]; - - if (0 < yyn) - break; - } - } - - /* Pop the current state because it cannot handle the error token. */ - if (yyssp == yyss) - YYABORT; - - - yydestruct ("Error: popping", - yystos[yystate], yyvsp); - YYPOPSTACK (1); - yystate = *yyssp; - YY_STACK_PRINT (yyss, yyssp); + if (0 < yyn) + break; + } } - *++yyvsp = yylval; + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + yydestruct("Error: popping", yystos[yystate], yyvsp); + YYPOPSTACK(1); + yystate = *yyssp; + YY_STACK_PRINT(yyss, yyssp); + } - /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + *++yyvsp = yylval; - yystate = yyn; - goto yynewstate; + /* Shift the error token. */ + YY_SYMBOL_PRINT("Shifting", yystos[yyn], yyvsp, yylsp); + yystate = yyn; + goto yynewstate; - /*-------------------------------------. - | yyacceptlab -- YYACCEPT comes here. | - `-------------------------------------*/ + /*-------------------------------------. + | yyacceptlab -- YYACCEPT comes here. | + `-------------------------------------*/ yyacceptlab: - yyresult = 0; - goto yyreturn; + yyresult = 0; + goto yyreturn; - /*-----------------------------------. - | yyabortlab -- YYABORT comes here. | - `-----------------------------------*/ + /*-----------------------------------. + | yyabortlab -- YYABORT comes here. | + `-----------------------------------*/ yyabortlab: - yyresult = 1; - goto yyreturn; + yyresult = 1; + goto yyreturn; #if !defined(yyoverflow) || YYERROR_VERBOSE - /*-------------------------------------------------. - | yyexhaustedlab -- memory exhaustion comes here. | - `-------------------------------------------------*/ + /*-------------------------------------------------. + | yyexhaustedlab -- memory exhaustion comes here. | + `-------------------------------------------------*/ yyexhaustedlab: - yyerror (YY_("memory exhausted")); - yyresult = 2; - /* Fall through. */ + yyerror(YY_("memory exhausted")); + yyresult = 2; + /* Fall through. */ #endif yyreturn: - if (yychar != YYEMPTY) - yydestruct ("Cleanup: discarding lookahead", - yytoken, &yylval); + if (yychar != YYEMPTY) + yydestruct("Cleanup: discarding lookahead", yytoken, &yylval); - /* Do not reclaim the symbols of the rule which action triggered - this YYABORT or YYACCEPT. */ - YYPOPSTACK (yylen); - YY_STACK_PRINT (yyss, yyssp); + /* Do not reclaim the symbols of the rule which action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK(yylen); + YY_STACK_PRINT(yyss, yyssp); - while (yyssp != yyss) - { - yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp); - YYPOPSTACK (1); - } + while (yyssp != yyss) + { + yydestruct("Cleanup: popping", yystos[*yyssp], yyvsp); + YYPOPSTACK(1); + } #ifndef yyoverflow - if (yyss != yyssa) - YYSTACK_FREE (yyss); + if (yyss != yyssa) + YYSTACK_FREE(yyss); #endif #if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); + if (yymsg != yymsgbuf) + YYSTACK_FREE(yymsg); #endif - /* Make sure YYID is used. */ - return YYID (yyresult); + /* Make sure YYID is used. */ + return YYID(yyresult); } - - - - using namespace dmlpackage; namespace dmlpackage { - void grammar_init(ParseTree* _parseTree, bool debug) { - parseTree = _parseTree; + parseTree = _parseTree; - if (debug) - yydebug = 1; + if (debug) + yydebug = 1; } void free_copybuffer() { + unsigned int i; - unsigned int i; + for (i = 0; i < copy_buffer.size(); i++) + { + if (copy_buffer[i]) + free(copy_buffer[i]); + } - for (i = 0; i < copy_buffer.size(); i++) - { - if (copy_buffer[i]) - free(copy_buffer[i]); - } - - copy_buffer.clear(); + copy_buffer.clear(); } char* copy_string(const char* str) { - char* nv = strdup(str); + char* nv = strdup(str); - if (nv) - copy_buffer.push_back(nv); + if (nv) + copy_buffer.push_back(nv); - return nv; + return nv; } void set_defaultSchema(std::string schema) { - default_schema = schema; -} - + default_schema = schema; } +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dml-gram-win.h b/dbcon/dmlpackage/dml-gram-win.h index 457adfe1d..da61c3855 100644 --- a/dbcon/dmlpackage/dml-gram-win.h +++ b/dbcon/dmlpackage/dml-gram-win.h @@ -32,152 +32,143 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ - /* Tokens. */ #ifndef YYTOKENTYPE -# define YYTOKENTYPE +#define YYTOKENTYPE /* Put the tokens into the symbol table, so that GDB and other debuggers know about them. */ enum yytokentype { - NAME = 258, - STRING = 259, - INTNUM = 260, - APPROXNUM = 261, - SELECT = 262, - ALL = 263, - DISTINCT = 264, - NULLX = 265, - USER = 266, - INDICATOR = 267, - AMMSC = 268, - PARAMETER = 269, - ANY = 270, - SOME = 271, - OR = 272, - AND = 273, - NOT = 274, - COMPARISON = 275, - UMINUS = 276, - AS = 277, - ASC = 278, - AUTHORIZATION = 279, - BETWEEN = 280, - BY = 281, - CHARACTER = 282, - CHECK = 283, - CLOSE = 284, - COMMIT = 285, - CONTINUE = 286, - CREATE = 287, - CURRENT = 288, - CURSOR = 289, - IDB_DECIMAL = 290, - DECLARE = 291, - DEFAULT = 292, - DELETE = 293, - DESC = 294, - IDB_DOUBLE = 295, - ESCAPE = 296, - EXISTS = 297, - FETCH = 298, - IDB_FLOAT = 299, - FOR = 300, - FOREIGN = 301, - FOUND = 302, - FROM = 303, - GOTO = 304, - GRANT = 305, - IDB_GROUP = 306, - HAVING = 307, - IN = 308, - INSERT = 309, - INTEGER = 310, - INTO = 311, - IS = 312, - KEY = 313, - LANGUAGE = 314, - LIKE = 315, - NUMERIC = 316, - OF = 317, - ON = 318, - OPEN = 319, - OPTION = 320, - ORDER = 321, - PRECISION = 322, - PRIMARY = 323, - PRIVILEGES = 324, - PROCEDURE = 325, - PUBLIC = 326, - REAL = 327, - REFERENCES = 328, - ROLLBACK = 329, - SCHEMA = 330, - SET = 331, - SMALLINT = 332, - SQLCODE = 333, - SQLERROR = 334, - TABLE = 335, - TO = 336, - UNION = 337, - UNIQUE = 338, - UPDATE = 339, - VALUES = 340, - VIEW = 341, - WHENEVER = 342, - WHERE = 343, - WITH = 344, - WORK = 345 + NAME = 258, + STRING = 259, + INTNUM = 260, + APPROXNUM = 261, + SELECT = 262, + ALL = 263, + DISTINCT = 264, + NULLX = 265, + USER = 266, + INDICATOR = 267, + AMMSC = 268, + PARAMETER = 269, + ANY = 270, + SOME = 271, + OR = 272, + AND = 273, + NOT = 274, + COMPARISON = 275, + UMINUS = 276, + AS = 277, + ASC = 278, + AUTHORIZATION = 279, + BETWEEN = 280, + BY = 281, + CHARACTER = 282, + CHECK = 283, + CLOSE = 284, + COMMIT = 285, + CONTINUE = 286, + CREATE = 287, + CURRENT = 288, + CURSOR = 289, + IDB_DECIMAL = 290, + DECLARE = 291, + DEFAULT = 292, + DELETE = 293, + DESC = 294, + IDB_DOUBLE = 295, + ESCAPE = 296, + EXISTS = 297, + FETCH = 298, + IDB_FLOAT = 299, + FOR = 300, + FOREIGN = 301, + FOUND = 302, + FROM = 303, + GOTO = 304, + GRANT = 305, + IDB_GROUP = 306, + HAVING = 307, + IN = 308, + INSERT = 309, + INTEGER = 310, + INTO = 311, + IS = 312, + KEY = 313, + LANGUAGE = 314, + LIKE = 315, + NUMERIC = 316, + OF = 317, + ON = 318, + OPEN = 319, + OPTION = 320, + ORDER = 321, + PRECISION = 322, + PRIMARY = 323, + PRIVILEGES = 324, + PROCEDURE = 325, + PUBLIC = 326, + REAL = 327, + REFERENCES = 328, + ROLLBACK = 329, + SCHEMA = 330, + SET = 331, + SMALLINT = 332, + SQLCODE = 333, + SQLERROR = 334, + TABLE = 335, + TO = 336, + UNION = 337, + UNIQUE = 338, + UPDATE = 339, + VALUES = 340, + VIEW = 341, + WHENEVER = 342, + WHERE = 343, + WITH = 344, + WORK = 345 }; #endif - - -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED typedef union YYSTYPE { - - - int intval; - double floatval; - char* strval; - int subtok; - dmlpackage::SqlStatementList* sqlStmtList; - dmlpackage::SqlStatement* sqlStmt; - dmlpackage::TableName* tblName; - dmlpackage::ColumnNameList* colNameList; - dmlpackage::ValuesOrQuery* valsOrQuery; - dmlpackage::ValuesList* valsList; - dmlpackage::QuerySpec* querySpec; - dmlpackage::TableNameList* tableNameList; - dmlpackage::TableExpression* tableExpression; - dmlpackage::WhereClause* whereClause; - dmlpackage::SearchCondition* searchCondition; - dmlpackage::ExistanceTestPredicate* existPredicate; - dmlpackage::AllOrAnyPredicate* allOrAnyPredicate; - dmlpackage::InPredicate* inPredicate; - dmlpackage::NullTestPredicate* nullTestPredicate; - dmlpackage::LikePredicate* likePredicate; - dmlpackage::BetweenPredicate* betweenPredicate; - dmlpackage::ComparisonPredicate* comparisonPredicate; - dmlpackage::Predicate* predicate; - dmlpackage::FromClause* fromClause; - dmlpackage::SelectFilter* selectFilter; - dmlpackage::GroupByClause* groupByClause; - dmlpackage::HavingClause* havingClause; - dmlpackage::Escape* escape; - dmlpackage::AtomList* atomList; - dmlpackage::ColumnAssignment* colAssignment; - dmlpackage::ColumnAssignmentList* colAssignmentList; - - + int intval; + double floatval; + char* strval; + int subtok; + dmlpackage::SqlStatementList* sqlStmtList; + dmlpackage::SqlStatement* sqlStmt; + dmlpackage::TableName* tblName; + dmlpackage::ColumnNameList* colNameList; + dmlpackage::ValuesOrQuery* valsOrQuery; + dmlpackage::ValuesList* valsList; + dmlpackage::QuerySpec* querySpec; + dmlpackage::TableNameList* tableNameList; + dmlpackage::TableExpression* tableExpression; + dmlpackage::WhereClause* whereClause; + dmlpackage::SearchCondition* searchCondition; + dmlpackage::ExistanceTestPredicate* existPredicate; + dmlpackage::AllOrAnyPredicate* allOrAnyPredicate; + dmlpackage::InPredicate* inPredicate; + dmlpackage::NullTestPredicate* nullTestPredicate; + dmlpackage::LikePredicate* likePredicate; + dmlpackage::BetweenPredicate* betweenPredicate; + dmlpackage::ComparisonPredicate* comparisonPredicate; + dmlpackage::Predicate* predicate; + dmlpackage::FromClause* fromClause; + dmlpackage::SelectFilter* selectFilter; + dmlpackage::GroupByClause* groupByClause; + dmlpackage::HavingClause* havingClause; + dmlpackage::Escape* escape; + dmlpackage::AtomList* atomList; + dmlpackage::ColumnAssignment* colAssignment; + dmlpackage::ColumnAssignmentList* colAssignmentList; } YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 +#define YYSTYPE_IS_TRIVIAL 1 +#define yystype YYSTYPE /* obsolescent; will be withdrawn */ +#define YYSTYPE_IS_DECLARED 1 #endif extern YYSTYPE dmllval; - - diff --git a/dbcon/dmlpackage/dml-scan-win.cpp b/dbcon/dmlpackage/dml-scan-win.cpp index 7999f205d..695357345 100644 --- a/dbcon/dmlpackage/dml-scan-win.cpp +++ b/dbcon/dmlpackage/dml-scan-win.cpp @@ -27,7 +27,6 @@ #include - /* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */ #ifdef c_plusplus #ifndef __cplusplus @@ -35,7 +34,6 @@ #endif #endif - #ifdef __cplusplus #include @@ -47,19 +45,19 @@ /* The "const" storage-class-modifier is valid. */ #define YY_USE_CONST -#else /* ! __cplusplus */ +#else /* ! __cplusplus */ #if __STDC__ #define YY_USE_PROTOS #define YY_USE_CONST -#endif /* __STDC__ */ -#endif /* ! __cplusplus */ +#endif /* __STDC__ */ +#endif /* ! __cplusplus */ #ifdef __TURBOC__ -#pragma warn -rch -#pragma warn -use +#pragma warn - rch +#pragma warn - use #include #include #define YY_USE_CONST @@ -72,7 +70,6 @@ #define yyconst #endif - #ifdef YY_USE_PROTOS #define YY_PROTO(proto) proto #else @@ -87,7 +84,7 @@ * we want to instead treat it as an 8-bit unsigned char, hence the * double cast. */ -#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) +#define YY_SC_TO_UI(c) ((unsigned int)(unsigned char)c) /* Enter a start condition. This macro really ought to take a parameter, * but we do it the disgusting crufty way forced on us by the ()-less @@ -106,7 +103,7 @@ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin ) +#define YY_NEW_FILE yyrestart(yyin) #define YY_END_OF_BUFFER_CHAR 0 @@ -116,7 +113,7 @@ typedef struct yy_buffer_state* YY_BUFFER_STATE; extern int yyleng; -extern FILE* yyin, *yyout; +extern FILE *yyin, *yyout; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 @@ -138,18 +135,17 @@ extern FILE* yyin, *yyout; /* Return all but the first 'n' matched characters back to the input stream. */ -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - *yy_cp = yy_hold_char; \ - YY_RESTORE_YY_MORE_OFFSET \ - yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ - YY_DO_BEFORE_ACTION; /* set up yytext again */ \ - } \ - while ( 0 ) +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + *yy_cp = yy_hold_char; \ + YY_RESTORE_YY_MORE_OFFSET \ + yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } while (0) -#define unput(c) yyunput( c, yytext_ptr ) +#define unput(c) yyunput(c, yytext_ptr) /* The following is because we cannot portably get our hands on size_t * (without autoconf's help, which isn't available because we want @@ -157,61 +153,60 @@ extern FILE* yyin, *yyout; */ typedef unsigned int yy_size_t; - struct yy_buffer_state { - FILE* yy_input_file; + FILE* yy_input_file; - char* yy_ch_buf; /* input buffer */ - char* yy_buf_pos; /* current position in input buffer */ + char* yy_ch_buf; /* input buffer */ + char* yy_buf_pos; /* current position in input buffer */ - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - yy_size_t yy_buf_size; + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + yy_size_t yy_buf_size; - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - int yy_n_chars; + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; - int yy_buffer_status; + int yy_buffer_status; #define YY_BUFFER_NEW 0 #define YY_BUFFER_NORMAL 1 - /* When an EOF's been seen but there's still some text to process - * then we mark the buffer as YY_EOF_PENDING, to indicate that we - * shouldn't try reading from the input source any more. We might - * still have a bunch of tokens to match, though, because of - * possible backing-up. - * - * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by - * just pointing yyin at a new input file. - */ + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ #define YY_BUFFER_EOF_PENDING 2 }; @@ -223,390 +218,241 @@ static YY_BUFFER_STATE yy_current_buffer = 0; */ #define YY_CURRENT_BUFFER yy_current_buffer - /* yy_hold_char holds the character lost when yytext is formed. */ static char yy_hold_char; -static int yy_n_chars; /* number of characters read into yy_ch_buf */ - +static int yy_n_chars; /* number of characters read into yy_ch_buf */ int yyleng; /* Points to current character in buffer. */ -static char* yy_c_buf_p = (char*) 0; -static int yy_init = 1; /* whether we need to initialize */ -static int yy_start = 0; /* start state number */ +static char* yy_c_buf_p = (char*)0; +static int yy_init = 1; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ /* Flag which is used to allow yywrap()'s to do buffer switches * instead of setting up a fresh yyin. A bit of a hack ... */ static int yy_did_buffer_switch_on_eof; -void yyrestart YY_PROTO(( FILE* input_file )); +void yyrestart YY_PROTO((FILE * input_file)); -void yy_switch_to_buffer YY_PROTO(( YY_BUFFER_STATE new_buffer )); -void yy_load_buffer_state YY_PROTO(( void )); -YY_BUFFER_STATE yy_create_buffer YY_PROTO(( FILE* file, int size )); -void yy_delete_buffer YY_PROTO(( YY_BUFFER_STATE b )); -void yy_init_buffer YY_PROTO(( YY_BUFFER_STATE b, FILE* file )); -void yy_flush_buffer YY_PROTO(( YY_BUFFER_STATE b )); -#define YY_FLUSH_BUFFER yy_flush_buffer( yy_current_buffer ) +void yy_switch_to_buffer YY_PROTO((YY_BUFFER_STATE new_buffer)); +void yy_load_buffer_state YY_PROTO((void)); +YY_BUFFER_STATE yy_create_buffer YY_PROTO((FILE * file, int size)); +void yy_delete_buffer YY_PROTO((YY_BUFFER_STATE b)); +void yy_init_buffer YY_PROTO((YY_BUFFER_STATE b, FILE* file)); +void yy_flush_buffer YY_PROTO((YY_BUFFER_STATE b)); +#define YY_FLUSH_BUFFER yy_flush_buffer(yy_current_buffer) -YY_BUFFER_STATE yy_scan_buffer YY_PROTO(( char* base, yy_size_t size )); -YY_BUFFER_STATE yy_scan_string YY_PROTO(( yyconst char* yy_str )); -YY_BUFFER_STATE yy_scan_bytes YY_PROTO(( yyconst char* bytes, int len )); +YY_BUFFER_STATE yy_scan_buffer YY_PROTO((char* base, yy_size_t size)); +YY_BUFFER_STATE yy_scan_string YY_PROTO((yyconst char* yy_str)); +YY_BUFFER_STATE yy_scan_bytes YY_PROTO((yyconst char* bytes, int len)); -static void* yy_flex_alloc YY_PROTO(( yy_size_t )); -static void* yy_flex_realloc YY_PROTO(( void*, yy_size_t )); -static void yy_flex_free YY_PROTO(( void* )); +static void* yy_flex_alloc YY_PROTO((yy_size_t)); +static void* yy_flex_realloc YY_PROTO((void*, yy_size_t)); +static void yy_flex_free YY_PROTO((void*)); #define yy_new_buffer yy_create_buffer -#define yy_set_interactive(is_interactive) \ - { \ - if ( ! yy_current_buffer ) \ - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ - yy_current_buffer->yy_is_interactive = is_interactive; \ - } +#define yy_set_interactive(is_interactive) \ + { \ + if (!yy_current_buffer) \ + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); \ + yy_current_buffer->yy_is_interactive = is_interactive; \ + } -#define yy_set_bol(at_bol) \ - { \ - if ( ! yy_current_buffer ) \ - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ - yy_current_buffer->yy_at_bol = at_bol; \ - } +#define yy_set_bol(at_bol) \ + { \ + if (!yy_current_buffer) \ + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); \ + yy_current_buffer->yy_at_bol = at_bol; \ + } #define YY_AT_BOL() (yy_current_buffer->yy_at_bol) - #define yywrap() 1 #define YY_SKIP_YYWRAP typedef unsigned char YY_CHAR; -FILE* yyin = (FILE*) 0, *yyout = (FILE*) 0; +FILE *yyin = (FILE*)0, *yyout = (FILE*)0; typedef int yy_state_type; extern char* yytext; #define yytext_ptr yytext -static yy_state_type yy_get_previous_state YY_PROTO(( void )); -static yy_state_type yy_try_NUL_trans YY_PROTO(( yy_state_type current_state )); -static int yy_get_next_buffer YY_PROTO(( void )); -static void yy_fatal_error YY_PROTO(( yyconst char msg[] )); +static yy_state_type yy_get_previous_state YY_PROTO((void)); +static yy_state_type yy_try_NUL_trans YY_PROTO((yy_state_type current_state)); +static int yy_get_next_buffer YY_PROTO((void)); +static void yy_fatal_error YY_PROTO((yyconst char msg[])); /* Done after the current pattern has been matched and before the * corresponding action - sets up yytext. */ -#define YY_DO_BEFORE_ACTION \ - yytext_ptr = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ - yy_hold_char = *yy_cp; \ - *yy_cp = '\0'; \ - yy_c_buf_p = yy_cp; +#define YY_DO_BEFORE_ACTION \ + yytext_ptr = yy_bp; \ + yyleng = (int)(yy_cp - yy_bp); \ + yy_hold_char = *yy_cp; \ + *yy_cp = '\0'; \ + yy_c_buf_p = yy_cp; #define YY_NUM_RULES 103 #define YY_END_OF_BUFFER 104 -static yyconst short int yy_accept[394] = -{ - 0, - 0, 0, 0, 0, 0, 0, 104, 103, 101, 100, - 97, 90, 90, 90, 90, 93, 103, 86, 84, 87, - 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 103, 98, 99, 101, 95, 93, 102, 95, - 94, 93, 0, 92, 88, 85, 89, 91, 91, 91, - 9, 91, 91, 91, 13, 91, 91, 91, 91, 91, - 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 41, 46, 91, 91, 91, 91, 91, 91, - 91, 53, 54, 91, 57, 91, 91, 91, 91, 91, +static yyconst short int yy_accept[394] = { + 0, 0, 0, 0, 0, 0, 0, 104, 103, 101, 100, 97, 90, 90, 90, 90, 93, 103, 86, 84, 87, 91, 91, 91, + 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 103, 98, 99, 101, 95, + 93, 102, 95, 94, 93, 0, 92, 88, 85, 89, 91, 91, 91, 9, 91, 91, 91, 13, 91, 91, 91, 91, 91, 91, + 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 41, 46, 91, 91, 91, 91, 91, 91, 91, 53, 54, 91, 57, + 91, 91, 91, 91, 91, - 91, 91, 91, 91, 91, 73, 91, 91, 91, 91, - 91, 91, 91, 91, 0, 98, 102, 94, 0, 96, - 92, 1, 2, 8, 10, 91, 3, 91, 91, 91, - 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 91, 91, 91, 91, 33, 91, 91, 0, - 91, 91, 91, 91, 91, 91, 44, 47, 91, 91, - 5, 4, 50, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 91, 91, 91, 91, 68, 91, 91, 91, - 6, 91, 91, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 14, 91, 91, 91, 91, 91, 91, 91, + 91, 91, 91, 91, 91, 73, 91, 91, 91, 91, 91, 91, 91, 91, 0, 98, 102, 94, 0, 96, 92, 1, 2, 8, + 10, 91, 3, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 33, 91, + 91, 0, 91, 91, 91, 91, 91, 91, 44, 47, 91, 91, 5, 4, 50, 91, 91, 91, 91, 91, 91, 91, 91, 91, + 91, 91, 91, 91, 68, 91, 91, 91, 6, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, 14, 91, 91, 91, + 91, 91, 91, 91, - 91, 91, 91, 91, 91, 26, 91, 91, 91, 91, - 91, 91, 91, 91, 36, 0, 37, 91, 91, 91, - 91, 91, 91, 45, 91, 49, 51, 91, 55, 91, - 91, 91, 91, 91, 91, 91, 64, 91, 91, 91, - 91, 70, 91, 91, 91, 91, 91, 77, 91, 79, - 91, 91, 82, 83, 91, 91, 91, 15, 16, 91, - 91, 7, 91, 91, 91, 91, 91, 91, 91, 91, - 91, 91, 91, 31, 32, 91, 35, 37, 38, 39, - 91, 91, 91, 91, 91, 91, 91, 58, 91, 91, - 91, 91, 91, 91, 91, 91, 91, 91, 72, 74, + 91, 91, 91, 91, 91, 26, 91, 91, 91, 91, 91, 91, 91, 91, 36, 0, 37, 91, 91, 91, 91, 91, 91, 45, + 91, 49, 51, 91, 55, 91, 91, 91, 91, 91, 91, 91, 64, 91, 91, 91, 91, 70, 91, 91, 91, 91, 91, 77, + 91, 79, 91, 91, 82, 83, 91, 91, 91, 15, 16, 91, 91, 7, 91, 91, 91, 91, 91, 91, 91, 91, 91, 91, + 91, 31, 32, 91, 35, 37, 38, 39, 91, 91, 91, 91, 91, 91, 91, 58, 91, 91, 91, 91, 91, 91, 91, 91, + 91, 91, 72, 74, - 91, 91, 91, 91, 81, 91, 91, 91, 17, 91, - 19, 91, 21, 91, 91, 91, 25, 91, 28, 29, - 30, 91, 40, 91, 43, 91, 91, 91, 56, 91, - 91, 91, 91, 63, 91, 91, 67, 91, 91, 75, - 76, 78, 91, 91, 12, 91, 91, 20, 22, 23, - 24, 91, 34, 91, 44, 91, 52, 91, 60, 91, - 91, 91, 91, 91, 71, 91, 91, 91, 18, 27, - 91, 48, 91, 91, 91, 91, 66, 69, 80, 91, - 14, 42, 59, 91, 62, 91, 91, 61, 65, 91, - 91, 11, 0 + 91, 91, 91, 91, 81, 91, 91, 91, 17, 91, 19, 91, 21, 91, 91, 91, 25, 91, 28, 29, 30, 91, 40, 91, + 43, 91, 91, 91, 56, 91, 91, 91, 91, 63, 91, 91, 67, 91, 91, 75, 76, 78, 91, 91, 12, 91, 91, 20, + 22, 23, 24, 91, 34, 91, 44, 91, 52, 91, 60, 91, 91, 91, 91, 91, 71, 91, 91, 91, 18, 27, 91, 48, + 91, 91, 91, 91, 66, 69, 80, 91, 14, 42, 59, 91, 62, 91, 91, 61, 65, 91, 91, 11, 0 -} ; +}; -static yyconst int yy_ec[256] = -{ - 0, - 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, - 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 1, 1, 1, 1, 1, 1, 5, 6, - 6, 6, 7, 6, 8, 9, 6, 10, 10, 10, - 10, 10, 10, 10, 10, 10, 10, 11, 6, 12, - 13, 14, 1, 1, 16, 17, 18, 19, 20, 21, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 1, 1, 1, 1, 15, 1, 16, 17, 18, 19, +static yyconst int yy_ec[256] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 5, 6, 6, 6, 7, 6, 8, 9, 6, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 11, 6, 12, 13, 14, 1, 1, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 1, 1, 1, 1, 15, 1, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 -} ; + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1}; -static yyconst int yy_meta[42] = -{ - 0, - 1, 1, 2, 1, 1, 1, 1, 1, 1, 3, - 1, 1, 1, 1, 3, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4 -} ; +static yyconst int yy_meta[42] = {0, 1, 1, 2, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 3, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}; -static yyconst short int yy_base[401] = -{ - 0, - 0, 0, 425, 424, 423, 422, 426, 431, 40, 431, - 431, 431, 36, 39, 415, 41, 0, 39, 431, 411, - 28, 23, 43, 47, 20, 48, 39, 407, 51, 0, - 402, 58, 67, 54, 65, 56, 67, 73, 72, 77, - 83, 89, 416, 431, 431, 54, 410, 105, 0, 90, - 106, 108, 113, 0, 431, 431, 431, 0, 392, 103, - 400, 382, 394, 380, 0, 111, 384, 101, 393, 379, - 114, 377, 374, 391, 384, 372, 376, 100, 375, 122, - 122, 367, 115, 0, 363, 373, 375, 361, 370, 363, - 112, 0, 0, 124, 378, 131, 379, 126, 368, 118, +static yyconst short int yy_base[401] = { + 0, 0, 0, 425, 424, 423, 422, 426, 431, 40, 431, 431, 431, 36, 39, 415, 41, 0, 39, 431, 411, + 28, 23, 43, 47, 20, 48, 39, 407, 51, 0, 402, 58, 67, 54, 65, 56, 67, 73, 72, 77, 83, + 89, 416, 431, 431, 54, 410, 105, 0, 90, 106, 108, 113, 0, 431, 431, 431, 0, 392, 103, 400, 382, + 394, 380, 0, 111, 384, 101, 393, 379, 114, 377, 374, 391, 384, 372, 376, 100, 375, 122, 122, 367, 115, + 0, 363, 373, 375, 361, 370, 363, 112, 0, 0, 124, 378, 131, 379, 126, 368, 118, - 378, 365, 365, 363, 373, 0, 365, 369, 367, 359, - 365, 364, 348, 349, 376, 431, 0, 136, 370, 369, - 0, 0, 0, 0, 0, 355, 0, 339, 343, 357, - 340, 345, 337, 342, 354, 129, 140, 353, 348, 349, - 331, 348, 348, 329, 344, 345, 340, 330, 330, 152, - 327, 327, 319, 330, 329, 332, 138, 0, 329, 330, - 0, 0, 0, 322, 328, 318, 322, 325, 326, 67, - 325, 315, 314, 320, 312, 318, 0, 310, 316, 317, - 0, 307, 139, 317, 299, 295, 292, 137, 306, 302, - 297, 306, 309, 298, 303, 298, 297, 285, 284, 298, + 378, 365, 365, 363, 373, 0, 365, 369, 367, 359, 365, 364, 348, 349, 376, 431, 0, 136, 370, 369, 0, + 0, 0, 0, 0, 355, 0, 339, 343, 357, 340, 345, 337, 342, 354, 129, 140, 353, 348, 349, 331, 348, + 348, 329, 344, 345, 340, 330, 330, 152, 327, 327, 319, 330, 329, 332, 138, 0, 329, 330, 0, 0, 0, + 322, 328, 318, 322, 325, 326, 67, 325, 315, 314, 320, 312, 318, 0, 310, 316, 317, 0, 307, 139, 317, + 299, 295, 292, 137, 306, 302, 297, 306, 309, 298, 303, 298, 297, 285, 284, 298, - 287, 288, 299, 278, 278, 0, 288, 284, 279, 274, - 285, 272, 282, 286, 0, 274, 0, 268, 271, 272, - 282, 266, 276, 0, 261, 0, 0, 263, 0, 265, - 261, 269, 276, 267, 270, 265, 0, 255, 270, 268, - 258, 0, 254, 263, 253, 245, 245, 0, 259, 0, - 258, 257, 0, 0, 243, 255, 256, 0, 0, 238, - 243, 0, 251, 241, 236, 252, 234, 239, 245, 235, - 243, 242, 227, 0, 0, 238, 0, 431, 0, 0, - 237, 242, 222, 236, 239, 230, 224, 0, 218, 218, - 223, 230, 230, 227, 230, 210, 220, 224, 0, 0, + 287, 288, 299, 278, 278, 0, 288, 284, 279, 274, 285, 272, 282, 286, 0, 274, 0, 268, 271, 272, 282, + 266, 276, 0, 261, 0, 0, 263, 0, 265, 261, 269, 276, 267, 270, 265, 0, 255, 270, 268, 258, 0, + 254, 263, 253, 245, 245, 0, 259, 0, 258, 257, 0, 0, 243, 255, 256, 0, 0, 238, 243, 0, 251, + 241, 236, 252, 234, 239, 245, 235, 243, 242, 227, 0, 0, 238, 0, 431, 0, 0, 237, 242, 222, 236, + 239, 230, 224, 0, 218, 218, 223, 230, 230, 227, 230, 210, 220, 224, 0, 0, - 222, 221, 206, 202, 0, 214, 208, 201, 0, 199, - 0, 199, 0, 206, 212, 196, 0, 212, 0, 0, - 0, 200, 0, 193, 0, 194, 204, 207, 0, 200, - 183, 202, 185, 0, 191, 201, 0, 189, 197, 0, - 0, 0, 196, 174, 0, 194, 193, 0, 0, 0, - 0, 177, 0, 181, 0, 190, 0, 179, 0, 186, - 170, 168, 159, 149, 0, 150, 166, 148, 0, 0, - 147, 0, 150, 158, 157, 156, 0, 0, 0, 140, - 0, 0, 0, 140, 0, 139, 148, 0, 0, 135, - 131, 0, 431, 187, 191, 56, 193, 197, 201, 203 + 222, 221, 206, 202, 0, 214, 208, 201, 0, 199, 0, 199, 0, 206, 212, 196, 0, 212, 0, 0, 0, + 200, 0, 193, 0, 194, 204, 207, 0, 200, 183, 202, 185, 0, 191, 201, 0, 189, 197, 0, 0, 0, + 196, 174, 0, 194, 193, 0, 0, 0, 0, 177, 0, 181, 0, 190, 0, 179, 0, 186, 170, 168, 159, + 149, 0, 150, 166, 148, 0, 0, 147, 0, 150, 158, 157, 156, 0, 0, 0, 140, 0, 0, 0, 140, + 0, 139, 148, 0, 0, 135, 131, 0, 431, 187, 191, 56, 193, 197, 201, 203 -} ; +}; -static yyconst short int yy_def[401] = -{ - 0, - 393, 1, 394, 394, 395, 395, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 396, 393, 393, 393, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 398, 393, 393, 393, 393, 393, 399, 393, - 393, 393, 393, 400, 393, 393, 393, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, +static yyconst short int yy_def[401] = { + 0, 393, 1, 394, 394, 395, 395, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 396, 393, 393, 393, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 398, 393, 393, 393, 393, 393, 399, 393, 393, 393, 393, 400, 393, 393, 393, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 398, 393, 399, 393, 393, 393, - 400, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 393, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 398, 393, 399, 393, 393, 393, 400, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 393, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 393, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 393, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 393, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 393, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, - 397, 397, 0, 393, 393, 393, 393, 393, 393, 393 + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 0, 393, 393, 393, 393, 393, 393, 393 -} ; +}; -static yyconst short int yy_nxt[473] = -{ - 0, - 8, 9, 10, 9, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 8, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - 36, 30, 37, 38, 39, 40, 41, 42, 30, 30, - 30, 46, 64, 46, 47, 48, 49, 47, 48, 51, - 52, 55, 56, 74, 59, 46, 60, 46, 75, 54, - 53, 61, 65, 62, 63, 66, 71, 76, 80, 67, - 72, 81, 68, 86, 77, 69, 73, 78, 70, 83, - 79, 87, 88, 90, 84, 92, 98, 105, 96, 91, - 89, 97, 100, 93, 233, 94, 99, 95, 110, 50, +static yyconst short int yy_nxt[473] = { + 0, 8, 9, 10, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 8, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 30, 37, 38, 39, 40, 41, 42, 30, 30, 30, + 46, 64, 46, 47, 48, 49, 47, 48, 51, 52, 55, 56, 74, 59, 46, 60, 46, 75, 54, 53, 61, + 65, 62, 63, 66, 71, 76, 80, 67, 72, 81, 68, 86, 77, 69, 73, 78, 70, 83, 79, 87, 88, + 90, 84, 92, 98, 105, 96, 91, 89, 97, 100, 93, 233, 94, 99, 95, 110, 50, - 101, 106, 102, 234, 103, 107, 111, 108, 104, 53, - 109, 112, 113, 51, 52, 118, 51, 52, 114, 119, - 119, 123, 120, 150, 53, 53, 129, 53, 132, 133, - 130, 137, 147, 155, 138, 148, 134, 152, 164, 165, - 139, 173, 124, 166, 176, 118, 174, 140, 156, 157, - 169, 153, 177, 150, 170, 53, 151, 223, 167, 392, - 171, 200, 201, 202, 391, 251, 203, 224, 245, 252, - 246, 390, 389, 388, 387, 386, 385, 384, 383, 382, - 381, 380, 379, 378, 377, 376, 216, 43, 43, 43, - 43, 8, 8, 8, 8, 58, 58, 115, 115, 115, + 101, 106, 102, 234, 103, 107, 111, 108, 104, 53, 109, 112, 113, 51, 52, 118, 51, 52, 114, 119, 119, + 123, 120, 150, 53, 53, 129, 53, 132, 133, 130, 137, 147, 155, 138, 148, 134, 152, 164, 165, 139, 173, + 124, 166, 176, 118, 174, 140, 156, 157, 169, 153, 177, 150, 170, 53, 151, 223, 167, 392, 171, 200, 201, + 202, 391, 251, 203, 224, 245, 252, 246, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, + 377, 376, 216, 43, 43, 43, 43, 8, 8, 8, 8, 58, 58, 115, 115, 115, - 115, 117, 375, 117, 117, 121, 121, 374, 373, 372, - 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, - 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, - 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, - 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, - 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, - 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, - 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, - 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, - 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, + 115, 117, 375, 117, 117, 121, 121, 374, 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, + 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, + 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, + 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, + 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, - 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, - 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, - 261, 260, 259, 258, 257, 256, 255, 254, 253, 250, - 249, 248, 247, 244, 243, 242, 241, 240, 239, 238, - 237, 236, 235, 232, 231, 230, 229, 228, 227, 226, - 225, 222, 221, 220, 219, 218, 217, 215, 214, 213, - 212, 211, 210, 209, 208, 207, 206, 205, 204, 199, - 198, 197, 196, 195, 194, 193, 192, 191, 120, 120, - 116, 190, 189, 188, 187, 186, 185, 184, 183, 182, - 181, 180, 179, 178, 175, 172, 168, 163, 162, 161, + 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, + 260, 259, 258, 257, 256, 255, 254, 253, 250, 249, 248, 247, 244, 243, 242, 241, 240, 239, 238, 237, 236, + 235, 232, 231, 230, 229, 228, 227, 226, 225, 222, 221, 220, 219, 218, 217, 215, 214, 213, 212, 211, 210, + 209, 208, 207, 206, 205, 204, 199, 198, 197, 196, 195, 194, 193, 192, 191, 120, 120, 116, 190, 189, 188, + 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 175, 172, 168, 163, 162, 161, - 160, 159, 158, 154, 149, 146, 145, 144, 143, 142, - 141, 136, 135, 131, 128, 127, 126, 125, 122, 50, - 116, 85, 82, 57, 50, 393, 45, 45, 44, 44, - 7, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393 -} ; + 160, 159, 158, 154, 149, 146, 145, 144, 143, 142, 141, 136, 135, 131, 128, 127, 126, 125, 122, 50, 116, + 85, 82, 57, 50, 393, 45, 45, 44, 44, 7, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, + 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, + 393, 393, 393, 393, 393, 393, 393, 393, 393}; -static yyconst short int yy_chk[473] = -{ - 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 9, 22, 9, 13, 13, 14, 14, 14, 16, - 16, 18, 18, 25, 21, 46, 21, 46, 25, 396, - 16, 21, 22, 21, 21, 23, 24, 26, 27, 23, - 24, 27, 23, 32, 26, 23, 24, 26, 23, 29, - 26, 32, 33, 34, 29, 35, 37, 39, 36, 34, - 33, 36, 38, 35, 170, 35, 37, 35, 41, 50, +static yyconst short int yy_chk[473] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 9, 22, 9, 13, 13, 14, 14, 14, 16, 16, 18, 18, 25, 21, 46, 21, 46, 25, 396, 16, 21, + 22, 21, 21, 23, 24, 26, 27, 23, 24, 27, 23, 32, 26, 23, 24, 26, 23, 29, 26, 32, 33, + 34, 29, 35, 37, 39, 36, 34, 33, 36, 38, 35, 170, 35, 37, 35, 41, 50, - 38, 39, 38, 170, 38, 40, 41, 40, 38, 50, - 40, 42, 42, 48, 48, 51, 52, 52, 42, 53, - 53, 60, 53, 80, 48, 51, 66, 52, 68, 68, - 66, 71, 78, 83, 71, 78, 68, 81, 91, 91, - 71, 98, 60, 94, 100, 118, 98, 71, 83, 83, - 96, 81, 100, 150, 96, 118, 80, 157, 94, 391, - 96, 136, 136, 137, 390, 188, 137, 157, 183, 188, - 183, 387, 386, 384, 380, 376, 375, 374, 373, 371, - 368, 367, 366, 364, 363, 362, 150, 394, 394, 394, - 394, 395, 395, 395, 395, 397, 397, 398, 398, 398, + 38, 39, 38, 170, 38, 40, 41, 40, 38, 50, 40, 42, 42, 48, 48, 51, 52, 52, 42, 53, 53, + 60, 53, 80, 48, 51, 66, 52, 68, 68, 66, 71, 78, 83, 71, 78, 68, 81, 91, 91, 71, 98, + 60, 94, 100, 118, 98, 71, 83, 83, 96, 81, 100, 150, 96, 118, 80, 157, 94, 391, 96, 136, 136, + 137, 390, 188, 137, 157, 183, 188, 183, 387, 386, 384, 380, 376, 375, 374, 373, 371, 368, 367, 366, 364, + 363, 362, 150, 394, 394, 394, 394, 395, 395, 395, 395, 397, 397, 398, 398, 398, - 398, 399, 361, 399, 399, 400, 400, 360, 358, 356, - 354, 352, 347, 346, 344, 343, 339, 338, 336, 335, - 333, 332, 331, 330, 328, 327, 326, 324, 322, 318, - 316, 315, 314, 312, 310, 308, 307, 306, 304, 303, - 302, 301, 298, 297, 296, 295, 294, 293, 292, 291, - 290, 289, 287, 286, 285, 284, 283, 282, 281, 276, - 273, 272, 271, 270, 269, 268, 267, 266, 265, 264, - 263, 261, 260, 257, 256, 255, 252, 251, 249, 247, - 246, 245, 244, 243, 241, 240, 239, 238, 236, 235, - 234, 233, 232, 231, 230, 228, 225, 223, 222, 221, + 398, 399, 361, 399, 399, 400, 400, 360, 358, 356, 354, 352, 347, 346, 344, 343, 339, 338, 336, 335, 333, + 332, 331, 330, 328, 327, 326, 324, 322, 318, 316, 315, 314, 312, 310, 308, 307, 306, 304, 303, 302, 301, + 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 287, 286, 285, 284, 283, 282, 281, 276, 273, 272, 271, + 270, 269, 268, 267, 266, 265, 264, 263, 261, 260, 257, 256, 255, 252, 251, 249, 247, 246, 245, 244, 243, + 241, 240, 239, 238, 236, 235, 234, 233, 232, 231, 230, 228, 225, 223, 222, 221, - 220, 219, 218, 216, 214, 213, 212, 211, 210, 209, - 208, 207, 205, 204, 203, 202, 201, 200, 199, 198, - 197, 196, 195, 194, 193, 192, 191, 190, 189, 187, - 186, 185, 184, 182, 180, 179, 178, 176, 175, 174, - 173, 172, 171, 169, 168, 167, 166, 165, 164, 160, - 159, 156, 155, 154, 153, 152, 151, 149, 148, 147, - 146, 145, 144, 143, 142, 141, 140, 139, 138, 135, - 134, 133, 132, 131, 130, 129, 128, 126, 120, 119, - 115, 114, 113, 112, 111, 110, 109, 108, 107, 105, - 104, 103, 102, 101, 99, 97, 95, 90, 89, 88, + 220, 219, 218, 216, 214, 213, 212, 211, 210, 209, 208, 207, 205, 204, 203, 202, 201, 200, 199, 198, 197, + 196, 195, 194, 193, 192, 191, 190, 189, 187, 186, 185, 184, 182, 180, 179, 178, 176, 175, 174, 173, 172, + 171, 169, 168, 167, 166, 165, 164, 160, 159, 156, 155, 154, 153, 152, 151, 149, 148, 147, 146, 145, 144, + 143, 142, 141, 140, 139, 138, 135, 134, 133, 132, 131, 130, 129, 128, 126, 120, 119, 115, 114, 113, 112, + 111, 110, 109, 108, 107, 105, 104, 103, 102, 101, 99, 97, 95, 90, 89, 88, - 87, 86, 85, 82, 79, 77, 76, 75, 74, 73, - 72, 70, 69, 67, 64, 63, 62, 61, 59, 47, - 43, 31, 28, 20, 15, 7, 6, 5, 4, 3, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393 -} ; + 87, 86, 85, 82, 79, 77, 76, 75, 74, 73, 72, 70, 69, 67, 64, 63, 62, 61, 59, 47, 43, + 31, 28, 20, 15, 7, 6, 5, 4, 3, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, + 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, + 393, 393, 393, 393, 393, 393, 393, 393, 393}; static yy_state_type yy_last_accepting_state; static char* yy_last_accepting_cpos; @@ -655,53 +501,54 @@ namespace dmlpackage { int lineno = 1; - /* Handles to the buffer that the lexer uses internally */ static YY_BUFFER_STATE scanbufhandle; static char* scanbuf; -static char* scanner_copy (char* str); - +static char* scanner_copy(char* str); /* macro to save the text and return a token */ -#define TOK(name) { dmllval.strval = scanner_copy(dmltext); return name; } -} +#define TOK(name) \ + { \ + dmllval.strval = scanner_copy(dmltext); \ + return name; \ + } +} // namespace dmlpackage #define YY_NO_UNPUT 1 #define inquote 1 #define endquote 2 - /* Macros after this point can all be overridden by user definitions in * section 1. */ #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap YY_PROTO(( void )); +extern "C" int yywrap YY_PROTO((void)); #else -extern int yywrap YY_PROTO(( void )); +extern int yywrap YY_PROTO((void)); #endif #endif #ifndef YY_NO_UNPUT -static void yyunput YY_PROTO(( int c, char* buf_ptr )); +static void yyunput YY_PROTO((int c, char* buf_ptr)); #endif #ifndef yytext_ptr -static void yy_flex_strncpy YY_PROTO(( char*, yyconst char*, int )); +static void yy_flex_strncpy YY_PROTO((char*, yyconst char*, int)); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen YY_PROTO(( yyconst char* )); +static int yy_flex_strlen YY_PROTO((yyconst char*)); #endif #ifndef YY_NO_INPUT #ifdef __cplusplus -static int yyinput YY_PROTO(( void )); +static int yyinput YY_PROTO((void)); #else -static int input YY_PROTO(( void )); +static int input YY_PROTO((void)); #endif #endif @@ -710,13 +557,13 @@ static int yy_start_stack_ptr = 0; static int yy_start_stack_depth = 0; static int* yy_start_stack = 0; #ifndef YY_NO_PUSH_STATE -static void yy_push_state YY_PROTO(( int new_state )); +static void yy_push_state YY_PROTO((int new_state)); #endif #ifndef YY_NO_POP_STATE -static void yy_pop_state YY_PROTO(( void )); +static void yy_pop_state YY_PROTO((void)); #endif #ifndef YY_NO_TOP_STATE -static int yy_top_state YY_PROTO(( void )); +static int yy_top_state YY_PROTO((void)); #endif #else @@ -751,29 +598,27 @@ YY_MALLOC_DECL /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO (void) fwrite( yytext, yyleng, 1, yyout ) +#define ECHO (void)fwrite(yytext, yyleng, 1, yyout) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, * is returned in "result". */ #ifndef YY_INPUT -#define YY_INPUT(buf,result,max_size) \ - if ( yy_current_buffer->yy_is_interactive ) \ - { \ - int c = '*', n; \ - for ( n = 0; n < max_size && \ - (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ - buf[n] = (char) c; \ - if ( c == '\n' ) \ - buf[n++] = (char) c; \ - if ( c == EOF && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); \ - result = n; \ - } \ - else if ( ((result = fread( buf, 1, max_size, yyin )) == 0) \ - && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); +#define YY_INPUT(buf, result, max_size) \ + if (yy_current_buffer->yy_is_interactive) \ + { \ + int c = '*', n; \ + for (n = 0; n < max_size && (c = getc(yyin)) != EOF && c != '\n'; ++n) \ + buf[n] = (char)c; \ + if (c == '\n') \ + buf[n++] = (char)c; \ + if (c == EOF && ferror(yyin)) \ + YY_FATAL_ERROR("input in flex scanner failed"); \ + result = n; \ + } \ + else if (((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin)) \ + YY_FATAL_ERROR("input in flex scanner failed"); #endif /* No semi-colon after return; correct usage is to write "yyterminate();" - @@ -791,14 +636,14 @@ YY_MALLOC_DECL /* Report a fatal error. */ #ifndef YY_FATAL_ERROR -#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#define YY_FATAL_ERROR(msg) yy_fatal_error(msg) #endif /* Default declaration of generated scanner - a define so the user can * easily add parameters. */ #ifndef YY_DECL -#define YY_DECL int yylex YY_PROTO(( void )) +#define YY_DECL int yylex YY_PROTO((void)) #endif /* Code executed at the beginning of each rule, after yytext and yyleng @@ -813,659 +658,642 @@ YY_MALLOC_DECL #define YY_BREAK break; #endif -#define YY_RULE_SETUP \ - YY_USER_ACTION +#define YY_RULE_SETUP YY_USER_ACTION YY_DECL { - register yy_state_type yy_current_state; - register char* yy_cp, *yy_bp; - register int yy_act; + register yy_state_type yy_current_state; + register char *yy_cp, *yy_bp; + register int yy_act; - - - - if ( yy_init ) - { - yy_init = 0; + if (yy_init) + { + yy_init = 0; #ifdef YY_USER_INIT - YY_USER_INIT; + YY_USER_INIT; #endif - if ( ! yy_start ) - yy_start = 1; /* first start state */ + if (!yy_start) + yy_start = 1; /* first start state */ - if ( ! yyin ) - yyin = stdin; + if (!yyin) + yyin = stdin; - if ( ! yyout ) - yyout = stdout; + if (!yyout) + yyout = stdout; - if ( ! yy_current_buffer ) - yy_current_buffer = - yy_create_buffer( yyin, YY_BUF_SIZE ); + if (!yy_current_buffer) + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); - yy_load_buffer_state(); + yy_load_buffer_state(); + } + + while (1) /* loops until end-of-file is reached */ + { + yy_cp = yy_c_buf_p; + + /* Support of yytext. */ + *yy_cp = yy_hold_char; + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = yy_start; + yy_match: + + do + { + register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + + if (yy_accept[yy_current_state]) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; + + if (yy_current_state >= 394) + yy_c = yy_meta[(unsigned int)yy_c]; + } + + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + ++yy_cp; + } while (yy_base[yy_current_state] != 431); + + yy_find_action: + yy_act = yy_accept[yy_current_state]; + + if (yy_act == 0) + { + /* have to back up */ + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + yy_act = yy_accept[yy_current_state]; } - while ( 1 ) /* loops until end-of-file is reached */ + YY_DO_BEFORE_ACTION; + + do_action: /* This label is used only to access EOF actions. */ + + switch (yy_act) { - yy_cp = yy_c_buf_p; - - /* Support of yytext. */ + /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ *yy_cp = yy_hold_char; + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + goto yy_find_action; - /* yy_bp points to the position in yy_ch_buf of the start of - * the current run. - */ - yy_bp = yy_cp; + case 1: + YY_RULE_SETUP + TOK(ALL) + YY_BREAK + case 2: + YY_RULE_SETUP + TOK(AND) + YY_BREAK + case 3: + YY_RULE_SETUP + TOK(AMMSC) + YY_BREAK + case 4: + YY_RULE_SETUP + TOK(AMMSC) + YY_BREAK + case 5: + YY_RULE_SETUP + TOK(AMMSC) + YY_BREAK + case 6: + YY_RULE_SETUP + TOK(AMMSC) + YY_BREAK + case 7: + YY_RULE_SETUP + TOK(AMMSC) + YY_BREAK + case 8: + YY_RULE_SETUP + TOK(ANY) + YY_BREAK + case 9: + YY_RULE_SETUP + TOK(AS) + YY_BREAK + case 10: + YY_RULE_SETUP + TOK(ASC) + YY_BREAK + case 11: + YY_RULE_SETUP + TOK(AUTHORIZATION) + YY_BREAK + case 12: + YY_RULE_SETUP + TOK(BETWEEN) + YY_BREAK + case 13: + YY_RULE_SETUP + TOK(BY) + YY_BREAK + case 14: + YY_RULE_SETUP + TOK(CHARACTER) + YY_BREAK + case 15: + YY_RULE_SETUP + TOK(CHECK) + YY_BREAK + case 16: + YY_RULE_SETUP + TOK(CLOSE) + YY_BREAK + case 17: + YY_RULE_SETUP + TOK(COMMIT) + YY_BREAK + case 18: + YY_RULE_SETUP + TOK(CONTINUE) + YY_BREAK + case 19: + YY_RULE_SETUP + TOK(CREATE) + YY_BREAK + case 20: + YY_RULE_SETUP + TOK(CURRENT) + YY_BREAK + case 21: + YY_RULE_SETUP + TOK(CURSOR) + YY_BREAK + case 22: + YY_RULE_SETUP + TOK(IDB_DECIMAL) + YY_BREAK + case 23: + YY_RULE_SETUP + TOK(DECLARE) + YY_BREAK + case 24: + YY_RULE_SETUP + TOK(DEFAULT) + YY_BREAK + case 25: + YY_RULE_SETUP + TOK(DELETE) + YY_BREAK + case 26: + YY_RULE_SETUP + TOK(DESC) + YY_BREAK + case 27: + YY_RULE_SETUP + TOK(ALL) + YY_BREAK + case 28: + YY_RULE_SETUP + TOK(IDB_DOUBLE) + YY_BREAK + case 29: + YY_RULE_SETUP + TOK(ESCAPE) + YY_BREAK + case 30: + YY_RULE_SETUP + TOK(EXISTS) + YY_BREAK + case 31: + YY_RULE_SETUP + TOK(FETCH) + YY_BREAK + case 32: + YY_RULE_SETUP + TOK(IDB_FLOAT) + YY_BREAK + case 33: + YY_RULE_SETUP + TOK(FOR) + YY_BREAK + case 34: + YY_RULE_SETUP + TOK(FOREIGN) + YY_BREAK + case 35: + YY_RULE_SETUP + TOK(FOUND) + YY_BREAK + case 36: + YY_RULE_SETUP + TOK(FROM) + YY_BREAK + case 37: + YY_RULE_SETUP + TOK(GOTO) + YY_BREAK + case 38: + YY_RULE_SETUP + TOK(GRANT) + YY_BREAK + case 39: + YY_RULE_SETUP + TOK(IDB_GROUP) + YY_BREAK + case 40: + YY_RULE_SETUP + TOK(HAVING) + YY_BREAK + case 41: + YY_RULE_SETUP + TOK(IN) + YY_BREAK + case 42: + YY_RULE_SETUP + TOK(INDICATOR) + YY_BREAK + case 43: + YY_RULE_SETUP + TOK(INSERT) + YY_BREAK + case 44: + YY_RULE_SETUP + TOK(INTEGER) + YY_BREAK + case 45: + YY_RULE_SETUP + TOK(INTO) + YY_BREAK + case 46: + YY_RULE_SETUP + TOK(IS) + YY_BREAK + case 47: + YY_RULE_SETUP + TOK(KEY) + YY_BREAK + case 48: + YY_RULE_SETUP + TOK(LANGUAGE) + YY_BREAK + case 49: + YY_RULE_SETUP + TOK(LIKE) + YY_BREAK + case 50: + YY_RULE_SETUP + TOK(NOT) + YY_BREAK + case 51: + YY_RULE_SETUP + TOK(NULLX) + YY_BREAK + case 52: + YY_RULE_SETUP + TOK(NUMERIC) + YY_BREAK + case 53: + YY_RULE_SETUP + TOK(OF) + YY_BREAK + case 54: + YY_RULE_SETUP + TOK(ON) + YY_BREAK + case 55: + YY_RULE_SETUP + TOK(OPEN) + YY_BREAK + case 56: + YY_RULE_SETUP + TOK(OPTION) + YY_BREAK + case 57: + YY_RULE_SETUP + TOK(OR) + YY_BREAK + case 58: + YY_RULE_SETUP + TOK(ORDER) + YY_BREAK + case 59: + YY_RULE_SETUP + TOK(PRECISION) + YY_BREAK + case 60: + YY_RULE_SETUP + TOK(PRIMARY) + YY_BREAK + case 61: + YY_RULE_SETUP + TOK(PRIVILEGES) + YY_BREAK + case 62: + YY_RULE_SETUP + TOK(PROCEDURE) + YY_BREAK + case 63: + YY_RULE_SETUP + TOK(PUBLIC) + YY_BREAK + case 64: + YY_RULE_SETUP + TOK(REAL) + YY_BREAK + case 65: + YY_RULE_SETUP + TOK(REFERENCES) + YY_BREAK + case 66: + YY_RULE_SETUP + TOK(ROLLBACK) + YY_BREAK + case 67: + YY_RULE_SETUP + TOK(SELECT) + YY_BREAK + case 68: + YY_RULE_SETUP + TOK(SET) + YY_BREAK + case 69: + YY_RULE_SETUP + TOK(SMALLINT) + YY_BREAK + case 70: + YY_RULE_SETUP + TOK(SOME) + YY_BREAK + case 71: + YY_RULE_SETUP + TOK(SQLCODE) + YY_BREAK + case 72: + YY_RULE_SETUP + TOK(TABLE) + YY_BREAK + case 73: + YY_RULE_SETUP + TOK(TO) + YY_BREAK + case 74: + YY_RULE_SETUP + TOK(UNION) + YY_BREAK + case 75: + YY_RULE_SETUP + TOK(UNIQUE) + YY_BREAK + case 76: + YY_RULE_SETUP + TOK(UPDATE) + YY_BREAK + case 77: + YY_RULE_SETUP + TOK(USER) + YY_BREAK + case 78: + YY_RULE_SETUP + TOK(VALUES) + YY_BREAK + case 79: + YY_RULE_SETUP + TOK(VIEW) + YY_BREAK + case 80: + YY_RULE_SETUP + TOK(WHENEVER) + YY_BREAK + case 81: + YY_RULE_SETUP + TOK(WHERE) + YY_BREAK + case 82: + YY_RULE_SETUP + TOK(WITH) + YY_BREAK + case 83: + YY_RULE_SETUP + TOK(WORK) + YY_BREAK - yy_current_state = yy_start; -yy_match: - - do - { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; - - if ( yy_accept[yy_current_state] ) - { - yy_last_accepting_state = yy_current_state; - yy_last_accepting_cpos = yy_cp; - } - - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - - if ( yy_current_state >= 394 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - ++yy_cp; - } - while ( yy_base[yy_current_state] != 431 ); - -yy_find_action: - yy_act = yy_accept[yy_current_state]; - - if ( yy_act == 0 ) - { - /* have to back up */ - yy_cp = yy_last_accepting_cpos; - yy_current_state = yy_last_accepting_state; - yy_act = yy_accept[yy_current_state]; - } - - YY_DO_BEFORE_ACTION; - - -do_action: /* This label is used only to access EOF actions. */ - - - switch ( yy_act ) - { - /* beginning of action switch */ - case 0: /* must back up */ - /* undo the effects of YY_DO_BEFORE_ACTION */ - *yy_cp = yy_hold_char; - yy_cp = yy_last_accepting_cpos; - yy_current_state = yy_last_accepting_state; - goto yy_find_action; - - case 1: - YY_RULE_SETUP - TOK(ALL) - YY_BREAK - case 2: - YY_RULE_SETUP - TOK(AND) - YY_BREAK - case 3: - YY_RULE_SETUP - TOK(AMMSC) - YY_BREAK - case 4: - YY_RULE_SETUP - TOK(AMMSC) - YY_BREAK - case 5: - YY_RULE_SETUP - TOK(AMMSC) - YY_BREAK - case 6: - YY_RULE_SETUP - TOK(AMMSC) - YY_BREAK - case 7: - YY_RULE_SETUP - TOK(AMMSC) - YY_BREAK - case 8: - YY_RULE_SETUP - TOK(ANY) - YY_BREAK - case 9: - YY_RULE_SETUP - TOK(AS) - YY_BREAK - case 10: - YY_RULE_SETUP - TOK(ASC) - YY_BREAK - case 11: - YY_RULE_SETUP - TOK(AUTHORIZATION) - YY_BREAK - case 12: - YY_RULE_SETUP - TOK(BETWEEN) - YY_BREAK - case 13: - YY_RULE_SETUP - TOK(BY) - YY_BREAK - case 14: - YY_RULE_SETUP - TOK(CHARACTER) - YY_BREAK - case 15: - YY_RULE_SETUP - TOK(CHECK) - YY_BREAK - case 16: - YY_RULE_SETUP - TOK(CLOSE) - YY_BREAK - case 17: - YY_RULE_SETUP - TOK(COMMIT) - YY_BREAK - case 18: - YY_RULE_SETUP - TOK(CONTINUE) - YY_BREAK - case 19: - YY_RULE_SETUP - TOK(CREATE) - YY_BREAK - case 20: - YY_RULE_SETUP - TOK(CURRENT) - YY_BREAK - case 21: - YY_RULE_SETUP - TOK(CURSOR) - YY_BREAK - case 22: - YY_RULE_SETUP - TOK(IDB_DECIMAL) - YY_BREAK - case 23: - YY_RULE_SETUP - TOK(DECLARE) - YY_BREAK - case 24: - YY_RULE_SETUP - TOK(DEFAULT) - YY_BREAK - case 25: - YY_RULE_SETUP - TOK(DELETE) - YY_BREAK - case 26: - YY_RULE_SETUP - TOK(DESC) - YY_BREAK - case 27: - YY_RULE_SETUP - TOK(ALL) - YY_BREAK - case 28: - YY_RULE_SETUP - TOK(IDB_DOUBLE) - YY_BREAK - case 29: - YY_RULE_SETUP - TOK(ESCAPE) - YY_BREAK - case 30: - YY_RULE_SETUP - TOK(EXISTS) - YY_BREAK - case 31: - YY_RULE_SETUP - TOK(FETCH) - YY_BREAK - case 32: - YY_RULE_SETUP - TOK(IDB_FLOAT) - YY_BREAK - case 33: - YY_RULE_SETUP - TOK(FOR) - YY_BREAK - case 34: - YY_RULE_SETUP - TOK(FOREIGN) - YY_BREAK - case 35: - YY_RULE_SETUP - TOK(FOUND) - YY_BREAK - case 36: - YY_RULE_SETUP - TOK(FROM) - YY_BREAK - case 37: - YY_RULE_SETUP - TOK(GOTO) - YY_BREAK - case 38: - YY_RULE_SETUP - TOK(GRANT) - YY_BREAK - case 39: - YY_RULE_SETUP - TOK(IDB_GROUP) - YY_BREAK - case 40: - YY_RULE_SETUP - TOK(HAVING) - YY_BREAK - case 41: - YY_RULE_SETUP - TOK(IN) - YY_BREAK - case 42: - YY_RULE_SETUP - TOK(INDICATOR) - YY_BREAK - case 43: - YY_RULE_SETUP - TOK(INSERT) - YY_BREAK - case 44: - YY_RULE_SETUP - TOK(INTEGER) - YY_BREAK - case 45: - YY_RULE_SETUP - TOK(INTO) - YY_BREAK - case 46: - YY_RULE_SETUP - TOK(IS) - YY_BREAK - case 47: - YY_RULE_SETUP - TOK(KEY) - YY_BREAK - case 48: - YY_RULE_SETUP - TOK(LANGUAGE) - YY_BREAK - case 49: - YY_RULE_SETUP - TOK(LIKE) - YY_BREAK - case 50: - YY_RULE_SETUP - TOK(NOT) - YY_BREAK - case 51: - YY_RULE_SETUP - TOK(NULLX) - YY_BREAK - case 52: - YY_RULE_SETUP - TOK(NUMERIC) - YY_BREAK - case 53: - YY_RULE_SETUP - TOK(OF) - YY_BREAK - case 54: - YY_RULE_SETUP - TOK(ON) - YY_BREAK - case 55: - YY_RULE_SETUP - TOK(OPEN) - YY_BREAK - case 56: - YY_RULE_SETUP - TOK(OPTION) - YY_BREAK - case 57: - YY_RULE_SETUP - TOK(OR) - YY_BREAK - case 58: - YY_RULE_SETUP - TOK(ORDER) - YY_BREAK - case 59: - YY_RULE_SETUP - TOK(PRECISION) - YY_BREAK - case 60: - YY_RULE_SETUP - TOK(PRIMARY) - YY_BREAK - case 61: - YY_RULE_SETUP - TOK(PRIVILEGES) - YY_BREAK - case 62: - YY_RULE_SETUP - TOK(PROCEDURE) - YY_BREAK - case 63: - YY_RULE_SETUP - TOK(PUBLIC) - YY_BREAK - case 64: - YY_RULE_SETUP - TOK(REAL) - YY_BREAK - case 65: - YY_RULE_SETUP - TOK(REFERENCES) - YY_BREAK - case 66: - YY_RULE_SETUP - TOK(ROLLBACK) - YY_BREAK - case 67: - YY_RULE_SETUP - TOK(SELECT) - YY_BREAK - case 68: - YY_RULE_SETUP - TOK(SET) - YY_BREAK - case 69: - YY_RULE_SETUP - TOK(SMALLINT) - YY_BREAK - case 70: - YY_RULE_SETUP - TOK(SOME) - YY_BREAK - case 71: - YY_RULE_SETUP - TOK(SQLCODE) - YY_BREAK - case 72: - YY_RULE_SETUP - TOK(TABLE) - YY_BREAK - case 73: - YY_RULE_SETUP - TOK(TO) - YY_BREAK - case 74: - YY_RULE_SETUP - TOK(UNION) - YY_BREAK - case 75: - YY_RULE_SETUP - TOK(UNIQUE) - YY_BREAK - case 76: - YY_RULE_SETUP - TOK(UPDATE) - YY_BREAK - case 77: - YY_RULE_SETUP - TOK(USER) - YY_BREAK - case 78: - YY_RULE_SETUP - TOK(VALUES) - YY_BREAK - case 79: - YY_RULE_SETUP - TOK(VIEW) - YY_BREAK - case 80: - YY_RULE_SETUP - TOK(WHENEVER) - YY_BREAK - case 81: - YY_RULE_SETUP - TOK(WHERE) - YY_BREAK - case 82: - YY_RULE_SETUP - TOK(WITH) - YY_BREAK - case 83: - YY_RULE_SETUP - TOK(WORK) - YY_BREAK - - /* punctuation */ - case 84: - case 85: - case 86: - case 87: - case 88: - case 89: - YY_RULE_SETUP - TOK(COMPARISON) - YY_BREAK - case 90: - YY_RULE_SETUP - { TOK(yytext[0]) } - YY_BREAK + /* punctuation */ + case 84: + case 85: + case 86: + case 87: + case 88: + case 89: + YY_RULE_SETUP + TOK(COMPARISON) + YY_BREAK + case 90: + YY_RULE_SETUP{TOK(yytext[0])} YY_BREAK /* names */ - case 91: - YY_RULE_SETUP - { TOK(NAME) } - YY_BREAK + case 91 : YY_RULE_SETUP{TOK(NAME)} YY_BREAK - /* parameters */ - case 92: - YY_RULE_SETUP - { - return PARAMETER; - } - YY_BREAK + /* parameters */ + case 92 : YY_RULE_SETUP + { + return PARAMETER; + } + YY_BREAK - /* numbers */ - case 93: - case 94: - case 95: - YY_RULE_SETUP - { TOK(INTNUM) } - YY_BREAK + /* numbers */ + case 93: + case 94: + case 95: + YY_RULE_SETUP{TOK(INTNUM)} YY_BREAK - case 96: - YY_RULE_SETUP - { TOK(APPROXNUM) } - YY_BREAK + case 96 : YY_RULE_SETUP{TOK(APPROXNUM)} YY_BREAK - case 97: - YY_RULE_SETUP - {BEGIN(inquote);} - YY_BREAK + case 97 : YY_RULE_SETUP + { + BEGIN(inquote); + } + YY_BREAK - case 98: - *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ - yy_c_buf_p = yy_cp -= 1; - YY_DO_BEFORE_ACTION; /* set up yytext again */ - YY_RULE_SETUP - {BEGIN(endquote); TOK(STRING) } - YY_BREAK + case 98: + *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ + yy_c_buf_p = yy_cp -= 1; + YY_DO_BEFORE_ACTION; /* set up yytext again */ + YY_RULE_SETUP + { + BEGIN(endquote); + TOK(STRING) + } + YY_BREAK - case 99: - YY_RULE_SETUP - {BEGIN(0);} - YY_BREAK + case 99: + YY_RULE_SETUP + { + BEGIN(0); + } + YY_BREAK - /* @bug 1870. Since MySQL parser will error out all the unterminated string, we don't actually need it here. */ - /* '[^'\n]*$ { dmlerror("Unterminated string"); } */ - case 100: - YY_RULE_SETUP - { lineno++;} - YY_BREAK + /* @bug 1870. Since MySQL parser will error out all the unterminated string, we don't actually need it + * here. */ + /* '[^'\n]*$ { dmlerror("Unterminated string"); } */ + case 100: + YY_RULE_SETUP + { + lineno++; + } + YY_BREAK - case 101: - YY_RULE_SETUP - ; /* white space */ - YY_BREAK + case 101: + YY_RULE_SETUP; /* white space */ + YY_BREAK - case 102: - YY_RULE_SETUP - ; /* comment */ - YY_BREAK + case 102: + YY_RULE_SETUP; /* comment */ + YY_BREAK - case 103: - YY_RULE_SETUP - ECHO; - YY_BREAK + case 103: + YY_RULE_SETUP + ECHO; + YY_BREAK - case YY_STATE_EOF(INITIAL): - case YY_STATE_EOF(inquote): - case YY_STATE_EOF(endquote): - yyterminate(); + case YY_STATE_EOF(INITIAL): + case YY_STATE_EOF(inquote): + case YY_STATE_EOF(endquote): yyterminate(); - case YY_END_OF_BUFFER: + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int)(yy_cp - yytext_ptr) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = yy_hold_char; + YY_RESTORE_YY_MORE_OFFSET + + if (yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between yy_current_buffer and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + yy_n_chars = yy_current_buffer->yy_n_chars; + yy_current_buffer->yy_input_file = yyin; + yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if (yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars]) + { + /* This was really a NUL. */ + yy_state_type yy_next_state; + + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans(yy_current_state); + + yy_bp = yytext_ptr + YY_MORE_ADJ; + + if (yy_next_state) + { + /* Consume the NUL. */ + yy_cp = ++yy_c_buf_p; + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = yy_c_buf_p; + goto yy_find_action; + } + } + + else + switch (yy_get_next_buffer()) + { + case EOB_ACT_END_OF_FILE: { - /* Amount of text matched not including the EOB char. */ - int yy_amount_of_matched_text = (int) (yy_cp - yytext_ptr) - 1; + yy_did_buffer_switch_on_eof = 0; - /* Undo the effects of YY_DO_BEFORE_ACTION. */ - *yy_cp = yy_hold_char; - YY_RESTORE_YY_MORE_OFFSET - - if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW ) - { - /* We're scanning a new file or input source. It's - * possible that this happened because the user - * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure - * consistency between yy_current_buffer and our - * globals. Here is the right place to do so, because - * this is the first action (other than possibly a - * back-up) that will match for the new input source. - */ - yy_n_chars = yy_current_buffer->yy_n_chars; - yy_current_buffer->yy_input_file = yyin; - yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; - } - - /* Note that here we test for yy_c_buf_p "<=" to the position - * of the first EOB in the buffer, since yy_c_buf_p will - * already have been incremented past the NUL character - * (since all states make transitions on EOB to the - * end-of-buffer state). Contrast this with the test - * in input(). + if (yywrap()) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. */ - if ( yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars] ) - { - /* This was really a NUL. */ - yy_state_type yy_next_state; + yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; - yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } - yy_current_state = yy_get_previous_state(); + else + { + if (!yy_did_buffer_switch_on_eof) + YY_NEW_FILE; + } - /* Okay, we're now positioned to make the NUL - * transition. We couldn't have - * yy_get_previous_state() go ahead and do it - * for us because it doesn't know how to deal - * with the possibility of jamming (and we don't - * want to build jamming into it because then it - * will run more slowly). - */ - - yy_next_state = yy_try_NUL_trans( yy_current_state ); - - yy_bp = yytext_ptr + YY_MORE_ADJ; - - if ( yy_next_state ) - { - /* Consume the NUL. */ - yy_cp = ++yy_c_buf_p; - yy_current_state = yy_next_state; - goto yy_match; - } - - else - { - yy_cp = yy_c_buf_p; - goto yy_find_action; - } - } - - else switch ( yy_get_next_buffer() ) - { - case EOB_ACT_END_OF_FILE: - { - yy_did_buffer_switch_on_eof = 0; - - if ( yywrap() ) - { - /* Note: because we've taken care in - * yy_get_next_buffer() to have set up - * yytext, we can now set up - * yy_c_buf_p so that if some total - * hoser (like flex itself) wants to - * call the scanner after we return the - * YY_NULL, it'll still work - another - * YY_NULL will get returned. - */ - yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; - - yy_act = YY_STATE_EOF(YY_START); - goto do_action; - } - - else - { - if ( ! yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; - } - - break; - } - - case EOB_ACT_CONTINUE_SCAN: - yy_c_buf_p = - yytext_ptr + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state(); - - yy_cp = yy_c_buf_p; - yy_bp = yytext_ptr + YY_MORE_ADJ; - goto yy_match; - - case EOB_ACT_LAST_MATCH: - yy_c_buf_p = - &yy_current_buffer->yy_ch_buf[yy_n_chars]; - - yy_current_state = yy_get_previous_state(); - - yy_cp = yy_c_buf_p; - yy_bp = yytext_ptr + YY_MORE_ADJ; - goto yy_find_action; - } - - break; + break; } - default: - YY_FATAL_ERROR( - "fatal flex scanner internal error--no action found" ); - } /* end of action switch */ - } /* end of scanning one token */ -} /* end of yylex */ + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + yy_c_buf_p = &yy_current_buffer->yy_ch_buf[yy_n_chars]; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_find_action; + } + + break; + } + + default: YY_FATAL_ERROR("fatal flex scanner internal error--no action found"); + } /* end of action switch */ + } /* end of scanning one token */ +} /* end of yylex */ /* yy_get_next_buffer - try to read in a new buffer * @@ -1477,169 +1305,158 @@ do_action: /* This label is used only to access EOF actions. */ static int yy_get_next_buffer() { - register char* dest = yy_current_buffer->yy_ch_buf; - register char* source = yytext_ptr; - register int number_to_move, i; - int ret_val; + register char* dest = yy_current_buffer->yy_ch_buf; + register char* source = yytext_ptr; + register int number_to_move, i; + int ret_val; - if ( yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1] ) - YY_FATAL_ERROR( - "fatal flex scanner internal error--end of buffer missed" ); + if (yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1]) + YY_FATAL_ERROR("fatal flex scanner internal error--end of buffer missed"); - if ( yy_current_buffer->yy_fill_buffer == 0 ) + if (yy_current_buffer->yy_fill_buffer == 0) + { + /* Don't try to fill the buffer, so this is an EOF. */ + if (yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1) { - /* Don't try to fill the buffer, so this is an EOF. */ - if ( yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1 ) - { - /* We matched a single character, the EOB, so - * treat this as a final EOF. - */ - return EOB_ACT_END_OF_FILE; - } - - else - { - /* We matched some text prior to the EOB, first - * process it. - */ - return EOB_ACT_LAST_MATCH; - } + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; } - /* Try to read more data. */ - - /* First move last chars to start of buffer. */ - number_to_move = (int) (yy_c_buf_p - yytext_ptr) - 1; - - for ( i = 0; i < number_to_move; ++i ) - *(dest++) = *(source++); - - if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING ) - /* don't do the read, it's not guaranteed to return an EOF, - * just force an EOF - */ - yy_current_buffer->yy_n_chars = yy_n_chars = 0; - else { - int num_to_read = - yy_current_buffer->yy_buf_size - number_to_move - 1; + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } - while ( num_to_read <= 0 ) - { - /* Not enough room in the buffer - grow it. */ + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int)(yy_c_buf_p - yytext_ptr) - 1; + + for (i = 0; i < number_to_move; ++i) + *(dest++) = *(source++); + + if (yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + yy_current_buffer->yy_n_chars = yy_n_chars = 0; + + else + { + int num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; + + while (num_to_read <= 0) + { + /* Not enough room in the buffer - grow it. */ #ifdef YY_USES_REJECT - YY_FATAL_ERROR( - "input buffer overflow, can't enlarge buffer because scanner uses REJECT" ); + YY_FATAL_ERROR("input buffer overflow, can't enlarge buffer because scanner uses REJECT"); #else - /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = yy_current_buffer; + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = yy_current_buffer; - int yy_c_buf_p_offset = - (int) (yy_c_buf_p - b->yy_ch_buf); + int yy_c_buf_p_offset = (int)(yy_c_buf_p - b->yy_ch_buf); - if ( b->yy_is_our_buffer ) - { - int new_size = b->yy_buf_size * 2; + if (b->yy_is_our_buffer) + { + int new_size = b->yy_buf_size * 2; - if ( new_size <= 0 ) - b->yy_buf_size += b->yy_buf_size / 8; - else - b->yy_buf_size *= 2; + if (new_size <= 0) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; - b->yy_ch_buf = (char*) - /* Include room in for 2 EOB chars. */ - yy_flex_realloc( (void*) b->yy_ch_buf, - b->yy_buf_size + 2 ); - } - else - /* Can't grow it, we don't own it. */ - b->yy_ch_buf = 0; + b->yy_ch_buf = (char*) + /* Include room in for 2 EOB chars. */ + yy_flex_realloc((void*)b->yy_ch_buf, b->yy_buf_size + 2); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = 0; - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( - "fatal error - scanner input buffer overflow" ); + if (!b->yy_ch_buf) + YY_FATAL_ERROR("fatal error - scanner input buffer overflow"); - yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; + yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; - num_to_read = yy_current_buffer->yy_buf_size - - number_to_move - 1; + num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; #endif - } - - if ( num_to_read > YY_READ_BUF_SIZE ) - num_to_read = YY_READ_BUF_SIZE; - - /* Read in more data. */ - YY_INPUT( (&yy_current_buffer->yy_ch_buf[number_to_move]), - yy_n_chars, num_to_read ); - - yy_current_buffer->yy_n_chars = yy_n_chars; } - if ( yy_n_chars == 0 ) - { - if ( number_to_move == YY_MORE_ADJ ) - { - ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin ); - } + if (num_to_read > YY_READ_BUF_SIZE) + num_to_read = YY_READ_BUF_SIZE; - else - { - ret_val = EOB_ACT_LAST_MATCH; - yy_current_buffer->yy_buffer_status = - YY_BUFFER_EOF_PENDING; - } + /* Read in more data. */ + YY_INPUT((&yy_current_buffer->yy_ch_buf[number_to_move]), yy_n_chars, num_to_read); + + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + if (yy_n_chars == 0) + { + if (number_to_move == YY_MORE_ADJ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart(yyin); } else - ret_val = EOB_ACT_CONTINUE_SCAN; + { + ret_val = EOB_ACT_LAST_MATCH; + yy_current_buffer->yy_buffer_status = YY_BUFFER_EOF_PENDING; + } + } - yy_n_chars += number_to_move; - yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; - yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; + else + ret_val = EOB_ACT_CONTINUE_SCAN; - yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + yy_n_chars += number_to_move; + yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; + yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; - return ret_val; + yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + + return ret_val; } - /* yy_get_previous_state - get the state just before the EOB char was reached */ static yy_state_type yy_get_previous_state() { - register yy_state_type yy_current_state; - register char* yy_cp; + register yy_state_type yy_current_state; + register char* yy_cp; - yy_current_state = yy_start; + yy_current_state = yy_start; - for ( yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp ) + for (yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp) + { + register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + + if (yy_accept[yy_current_state]) { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); - - if ( yy_accept[yy_current_state] ) - { - yy_last_accepting_state = yy_current_state; - yy_last_accepting_cpos = yy_cp; - } - - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - - if ( yy_current_state >= 394 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; } - return yy_current_state; -} + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; + if (yy_current_state >= 394) + yy_c = yy_meta[(unsigned int)yy_c]; + } + + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + } + + return yy_current_state; +} /* yy_try_NUL_trans - try to make a transition on the NUL character * @@ -1648,83 +1465,75 @@ static yy_state_type yy_get_previous_state() */ #ifdef YY_USE_PROTOS -static yy_state_type yy_try_NUL_trans( yy_state_type yy_current_state ) +static yy_state_type yy_try_NUL_trans(yy_state_type yy_current_state) #else -static yy_state_type yy_try_NUL_trans( yy_current_state ) -yy_state_type yy_current_state; +static yy_state_type yy_try_NUL_trans(yy_current_state) yy_state_type yy_current_state; #endif { - register int yy_is_jam; - register char* yy_cp = yy_c_buf_p; + register int yy_is_jam; + register char* yy_cp = yy_c_buf_p; - register YY_CHAR yy_c = 1; + register YY_CHAR yy_c = 1; - if ( yy_accept[yy_current_state] ) - { - yy_last_accepting_state = yy_current_state; - yy_last_accepting_cpos = yy_cp; - } + if (yy_accept[yy_current_state]) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; - if ( yy_current_state >= 394 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } + if (yy_current_state >= 394) + yy_c = yy_meta[(unsigned int)yy_c]; + } - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 393); + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + yy_is_jam = (yy_current_state == 393); - return yy_is_jam ? 0 : yy_current_state; + return yy_is_jam ? 0 : yy_current_state; } - #ifndef YY_NO_UNPUT #ifdef YY_USE_PROTOS -static void yyunput( int c, register char* yy_bp ) +static void yyunput(int c, register char* yy_bp) #else -static void yyunput( c, yy_bp ) -int c; +static void yyunput(c, yy_bp) int c; register char* yy_bp; #endif { - register char* yy_cp = yy_c_buf_p; + register char* yy_cp = yy_c_buf_p; - /* undo effects of setting up yytext */ - *yy_cp = yy_hold_char; + /* undo effects of setting up yytext */ + *yy_cp = yy_hold_char; - if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) - { - /* need to shift things up to make room */ - /* +2 for EOB chars. */ - register int number_to_move = yy_n_chars + 2; - register char* dest = &yy_current_buffer->yy_ch_buf[ - yy_current_buffer->yy_buf_size + 2]; - register char* source = - &yy_current_buffer->yy_ch_buf[number_to_move]; + if (yy_cp < yy_current_buffer->yy_ch_buf + 2) + { + /* need to shift things up to make room */ + /* +2 for EOB chars. */ + register int number_to_move = yy_n_chars + 2; + register char* dest = &yy_current_buffer->yy_ch_buf[yy_current_buffer->yy_buf_size + 2]; + register char* source = &yy_current_buffer->yy_ch_buf[number_to_move]; - while ( source > yy_current_buffer->yy_ch_buf ) - *--dest = *--source; + while (source > yy_current_buffer->yy_ch_buf) + *--dest = *--source; - yy_cp += (int) (dest - source); - yy_bp += (int) (dest - source); - yy_current_buffer->yy_n_chars = - yy_n_chars = yy_current_buffer->yy_buf_size; + yy_cp += (int)(dest - source); + yy_bp += (int)(dest - source); + yy_current_buffer->yy_n_chars = yy_n_chars = yy_current_buffer->yy_buf_size; - if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) - YY_FATAL_ERROR( "flex scanner push-back overflow" ); - } + if (yy_cp < yy_current_buffer->yy_ch_buf + 2) + YY_FATAL_ERROR("flex scanner push-back overflow"); + } - *--yy_cp = (char) c; + *--yy_cp = (char)c; - - yytext_ptr = yy_bp; - yy_hold_char = *yy_cp; - yy_c_buf_p = yy_cp; + yytext_ptr = yy_bp; + yy_hold_char = *yy_cp; + yy_c_buf_p = yy_cp; } -#endif /* ifndef YY_NO_UNPUT */ - +#endif /* ifndef YY_NO_UNPUT */ #ifdef __cplusplus static int yyinput() @@ -1732,396 +1541,365 @@ static int yyinput() static int input() #endif { - int c; + int c; - *yy_c_buf_p = yy_hold_char; + *yy_c_buf_p = yy_hold_char; - if ( *yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) + if (*yy_c_buf_p == YY_END_OF_BUFFER_CHAR) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if (yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars]) + /* This was really a NUL. */ + *yy_c_buf_p = '\0'; + + else { - /* yy_c_buf_p now points to the character we want to return. - * If this occurs *before* the EOB characters, then it's a - * valid NUL; if not, then we've hit the end of the buffer. - */ - if ( yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars] ) - /* This was really a NUL. */ - *yy_c_buf_p = '\0'; + /* need more input */ + int offset = yy_c_buf_p - yytext_ptr; + ++yy_c_buf_p; - else + switch (yy_get_next_buffer()) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart(yyin); + + /* fall through */ + + case EOB_ACT_END_OF_FILE: { - /* need more input */ - int offset = yy_c_buf_p - yytext_ptr; - ++yy_c_buf_p; + if (yywrap()) + return EOF; - switch ( yy_get_next_buffer() ) - { - case EOB_ACT_LAST_MATCH: - /* This happens because yy_g_n_b() - * sees that we've accumulated a - * token and flags that we need to - * try matching the token before - * proceeding. But for input(), - * there's no matching to consider. - * So convert the EOB_ACT_LAST_MATCH - * to EOB_ACT_END_OF_FILE. - */ - - /* Reset buffer status. */ - yyrestart( yyin ); - - /* fall through */ - - case EOB_ACT_END_OF_FILE: - { - if ( yywrap() ) - return EOF; - - if ( ! yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; + if (!yy_did_buffer_switch_on_eof) + YY_NEW_FILE; #ifdef __cplusplus - return yyinput(); + return yyinput(); #else - return input(); + return input(); #endif - } - - case EOB_ACT_CONTINUE_SCAN: - yy_c_buf_p = yytext_ptr + offset; - break; - } } + + case EOB_ACT_CONTINUE_SCAN: yy_c_buf_p = yytext_ptr + offset; break; + } } + } - c = *(unsigned char*) yy_c_buf_p; /* cast for 8-bit char's */ - *yy_c_buf_p = '\0'; /* preserve yytext */ - yy_hold_char = *++yy_c_buf_p; + c = *(unsigned char*)yy_c_buf_p; /* cast for 8-bit char's */ + *yy_c_buf_p = '\0'; /* preserve yytext */ + yy_hold_char = *++yy_c_buf_p; - - return c; + return c; } - #ifdef YY_USE_PROTOS -void yyrestart( FILE* input_file ) +void yyrestart(FILE* input_file) #else -void yyrestart( input_file ) -FILE* input_file; +void yyrestart(input_file) FILE* input_file; #endif { - if ( ! yy_current_buffer ) - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); + if (!yy_current_buffer) + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); - yy_init_buffer( yy_current_buffer, input_file ); - yy_load_buffer_state(); + yy_init_buffer(yy_current_buffer, input_file); + yy_load_buffer_state(); } - #ifdef YY_USE_PROTOS -void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +void yy_switch_to_buffer(YY_BUFFER_STATE new_buffer) #else -void yy_switch_to_buffer( new_buffer ) -YY_BUFFER_STATE new_buffer; +void yy_switch_to_buffer(new_buffer) YY_BUFFER_STATE new_buffer; #endif { - if ( yy_current_buffer == new_buffer ) - return; + if (yy_current_buffer == new_buffer) + return; - if ( yy_current_buffer ) - { - /* Flush out information for old buffer. */ - *yy_c_buf_p = yy_hold_char; - yy_current_buffer->yy_buf_pos = yy_c_buf_p; - yy_current_buffer->yy_n_chars = yy_n_chars; - } + if (yy_current_buffer) + { + /* Flush out information for old buffer. */ + *yy_c_buf_p = yy_hold_char; + yy_current_buffer->yy_buf_pos = yy_c_buf_p; + yy_current_buffer->yy_n_chars = yy_n_chars; + } - yy_current_buffer = new_buffer; - yy_load_buffer_state(); + yy_current_buffer = new_buffer; + yy_load_buffer_state(); - /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe - * to go ahead and always set it. - */ - yy_did_buffer_switch_on_eof = 1; + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yy_did_buffer_switch_on_eof = 1; } - #ifdef YY_USE_PROTOS -void yy_load_buffer_state( void ) +void yy_load_buffer_state(void) #else void yy_load_buffer_state() #endif { - yy_n_chars = yy_current_buffer->yy_n_chars; - yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; - yyin = yy_current_buffer->yy_input_file; - yy_hold_char = *yy_c_buf_p; + yy_n_chars = yy_current_buffer->yy_n_chars; + yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; + yyin = yy_current_buffer->yy_input_file; + yy_hold_char = *yy_c_buf_p; } - #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_create_buffer( FILE* file, int size ) +YY_BUFFER_STATE yy_create_buffer(FILE* file, int size) #else -YY_BUFFER_STATE yy_create_buffer( file, size ) -FILE* file; +YY_BUFFER_STATE yy_create_buffer(file, size) FILE* file; int size; #endif { - YY_BUFFER_STATE b; + YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE)yy_flex_alloc(sizeof(struct yy_buffer_state)); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + if (!b) + YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - b->yy_buf_size = size; + b->yy_buf_size = size; - /* yy_ch_buf has to be 2 characters longer than the size given because - * we need to put in 2 end-of-buffer characters. - */ - b->yy_ch_buf = (char*) yy_flex_alloc( b->yy_buf_size + 2 ); + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char*)yy_flex_alloc(b->yy_buf_size + 2); - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + if (!b->yy_ch_buf) + YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - b->yy_is_our_buffer = 1; + b->yy_is_our_buffer = 1; - yy_init_buffer( b, file ); + yy_init_buffer(b, file); - return b; + return b; } - #ifdef YY_USE_PROTOS -void yy_delete_buffer( YY_BUFFER_STATE b ) +void yy_delete_buffer(YY_BUFFER_STATE b) #else -void yy_delete_buffer( b ) -YY_BUFFER_STATE b; +void yy_delete_buffer(b) YY_BUFFER_STATE b; #endif { - if ( ! b ) - return; + if (!b) + return; - if ( b == yy_current_buffer ) - yy_current_buffer = (YY_BUFFER_STATE) 0; + if (b == yy_current_buffer) + yy_current_buffer = (YY_BUFFER_STATE)0; - if ( b->yy_is_our_buffer ) - yy_flex_free( (void*) b->yy_ch_buf ); + if (b->yy_is_our_buffer) + yy_flex_free((void*)b->yy_ch_buf); - yy_flex_free( (void*) b ); + yy_flex_free((void*)b); } - #ifndef YY_ALWAYS_INTERACTIVE #ifndef YY_NEVER_INTERACTIVE -extern int isatty YY_PROTO(( int )); +extern int isatty YY_PROTO((int)); #endif #endif #ifdef YY_USE_PROTOS -void yy_init_buffer( YY_BUFFER_STATE b, FILE* file ) +void yy_init_buffer(YY_BUFFER_STATE b, FILE* file) #else -void yy_init_buffer( b, file ) -YY_BUFFER_STATE b; +void yy_init_buffer(b, file) YY_BUFFER_STATE b; FILE* file; #endif - { - yy_flush_buffer( b ); + yy_flush_buffer(b); - b->yy_input_file = file; - b->yy_fill_buffer = 1; + b->yy_input_file = file; + b->yy_fill_buffer = 1; #if YY_ALWAYS_INTERACTIVE - b->yy_is_interactive = 1; + b->yy_is_interactive = 1; #else #if YY_NEVER_INTERACTIVE - b->yy_is_interactive = 0; + b->yy_is_interactive = 0; #else - b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; + b->yy_is_interactive = file ? (isatty(fileno(file)) > 0) : 0; #endif #endif } - #ifdef YY_USE_PROTOS -void yy_flush_buffer( YY_BUFFER_STATE b ) +void yy_flush_buffer(YY_BUFFER_STATE b) #else -void yy_flush_buffer( b ) -YY_BUFFER_STATE b; +void yy_flush_buffer(b) YY_BUFFER_STATE b; #endif { - if ( ! b ) - return; + if (!b) + return; - b->yy_n_chars = 0; + b->yy_n_chars = 0; - /* We always need two end-of-buffer characters. The first causes - * a transition to the end-of-buffer state. The second causes - * a jam in that state. - */ - b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; - b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; - b->yy_buf_pos = &b->yy_ch_buf[0]; + b->yy_buf_pos = &b->yy_ch_buf[0]; - b->yy_at_bol = 1; - b->yy_buffer_status = YY_BUFFER_NEW; + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; - if ( b == yy_current_buffer ) - yy_load_buffer_state(); + if (b == yy_current_buffer) + yy_load_buffer_state(); } - #ifndef YY_NO_SCAN_BUFFER #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_buffer( char* base, yy_size_t size ) +YY_BUFFER_STATE yy_scan_buffer(char* base, yy_size_t size) #else -YY_BUFFER_STATE yy_scan_buffer( base, size ) -char* base; +YY_BUFFER_STATE yy_scan_buffer(base, size) char* base; yy_size_t size; #endif { - YY_BUFFER_STATE b; + YY_BUFFER_STATE b; - if ( size < 2 || - base[size - 2] != YY_END_OF_BUFFER_CHAR || - base[size - 1] != YY_END_OF_BUFFER_CHAR ) - /* They forgot to leave room for the EOB's. */ - return 0; + if (size < 2 || base[size - 2] != YY_END_OF_BUFFER_CHAR || base[size - 1] != YY_END_OF_BUFFER_CHAR) + /* They forgot to leave room for the EOB's. */ + return 0; - b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE)yy_flex_alloc(sizeof(struct yy_buffer_state)); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + if (!b) + YY_FATAL_ERROR("out of dynamic memory in yy_scan_buffer()"); - b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ - b->yy_buf_pos = b->yy_ch_buf = base; - b->yy_is_our_buffer = 0; - b->yy_input_file = 0; - b->yy_n_chars = b->yy_buf_size; - b->yy_is_interactive = 0; - b->yy_at_bol = 1; - b->yy_fill_buffer = 0; - b->yy_buffer_status = YY_BUFFER_NEW; + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = 0; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b ); + yy_switch_to_buffer(b); - return b; + return b; } #endif - #ifndef YY_NO_SCAN_STRING #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_string( yyconst char* yy_str ) +YY_BUFFER_STATE yy_scan_string(yyconst char* yy_str) #else -YY_BUFFER_STATE yy_scan_string( yy_str ) -yyconst char* yy_str; +YY_BUFFER_STATE yy_scan_string(yy_str) yyconst char* yy_str; #endif { - int len; + int len; - for ( len = 0; yy_str[len]; ++len ) - ; + for (len = 0; yy_str[len]; ++len) + ; - return yy_scan_bytes( yy_str, len ); + return yy_scan_bytes(yy_str, len); } #endif - #ifndef YY_NO_SCAN_BYTES #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_bytes( yyconst char* bytes, int len ) +YY_BUFFER_STATE yy_scan_bytes(yyconst char* bytes, int len) #else -YY_BUFFER_STATE yy_scan_bytes( bytes, len ) -yyconst char* bytes; +YY_BUFFER_STATE yy_scan_bytes(bytes, len) yyconst char* bytes; int len; #endif { - YY_BUFFER_STATE b; - char* buf; - yy_size_t n; - int i; + YY_BUFFER_STATE b; + char* buf; + yy_size_t n; + int i; - /* Get memory for full buffer, including space for trailing EOB's. */ - n = len + 2; - buf = (char*) yy_flex_alloc( n ); + /* Get memory for full buffer, including space for trailing EOB's. */ + n = len + 2; + buf = (char*)yy_flex_alloc(n); - if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + if (!buf) + YY_FATAL_ERROR("out of dynamic memory in yy_scan_bytes()"); - for ( i = 0; i < len; ++i ) - buf[i] = bytes[i]; + for (i = 0; i < len; ++i) + buf[i] = bytes[i]; - buf[len] = buf[len + 1] = YY_END_OF_BUFFER_CHAR; + buf[len] = buf[len + 1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n ); + b = yy_scan_buffer(buf, n); - if ( ! b ) - YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + if (!b) + YY_FATAL_ERROR("bad buffer in yy_scan_bytes()"); - /* It's okay to grow etc. this buffer, and we should throw it - * away when we're done. - */ - b->yy_is_our_buffer = 1; + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; - return b; + return b; } #endif - #ifndef YY_NO_PUSH_STATE #ifdef YY_USE_PROTOS -static void yy_push_state( int new_state ) +static void yy_push_state(int new_state) #else -static void yy_push_state( new_state ) -int new_state; +static void yy_push_state(new_state) int new_state; #endif { - if ( yy_start_stack_ptr >= yy_start_stack_depth ) - { - yy_size_t new_size; + if (yy_start_stack_ptr >= yy_start_stack_depth) + { + yy_size_t new_size; - yy_start_stack_depth += YY_START_STACK_INCR; - new_size = yy_start_stack_depth * sizeof( int ); + yy_start_stack_depth += YY_START_STACK_INCR; + new_size = yy_start_stack_depth * sizeof(int); - if ( ! yy_start_stack ) - yy_start_stack = (int*) yy_flex_alloc( new_size ); + if (!yy_start_stack) + yy_start_stack = (int*)yy_flex_alloc(new_size); - else - yy_start_stack = (int*) yy_flex_realloc( - (void*) yy_start_stack, new_size ); + else + yy_start_stack = (int*)yy_flex_realloc((void*)yy_start_stack, new_size); - if ( ! yy_start_stack ) - YY_FATAL_ERROR( - "out of memory expanding start-condition stack" ); - } + if (!yy_start_stack) + YY_FATAL_ERROR("out of memory expanding start-condition stack"); + } - yy_start_stack[yy_start_stack_ptr++] = YY_START; + yy_start_stack[yy_start_stack_ptr++] = YY_START; - BEGIN(new_state); + BEGIN(new_state); } #endif - #ifndef YY_NO_POP_STATE static void yy_pop_state() { - if ( --yy_start_stack_ptr < 0 ) - YY_FATAL_ERROR( "start-condition stack underflow" ); + if (--yy_start_stack_ptr < 0) + YY_FATAL_ERROR("start-condition stack underflow"); - BEGIN(yy_start_stack[yy_start_stack_ptr]); + BEGIN(yy_start_stack[yy_start_stack_ptr]); } #endif - #ifndef YY_NO_TOP_STATE static int yy_top_state() { - return yy_start_stack[yy_start_stack_ptr - 1]; + return yy_start_stack[yy_start_stack_ptr - 1]; } #endif @@ -2130,114 +1908,103 @@ static int yy_top_state() #endif #ifdef YY_USE_PROTOS -static void yy_fatal_error( yyconst char msg[] ) +static void yy_fatal_error(yyconst char msg[]) #else -static void yy_fatal_error( msg ) -char msg[]; +static void yy_fatal_error(msg) char msg[]; #endif { - (void) fprintf( stderr, "%s\n", msg ); - exit( YY_EXIT_FAILURE ); + (void)fprintf(stderr, "%s\n", msg); + exit(YY_EXIT_FAILURE); } - - /* Redefine yyless() so it works in section 3 code. */ #undef yyless -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - yytext[yyleng] = yy_hold_char; \ - yy_c_buf_p = yytext + n; \ - yy_hold_char = *yy_c_buf_p; \ - *yy_c_buf_p = '\0'; \ - yyleng = n; \ - } \ - while ( 0 ) - +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yytext[yyleng] = yy_hold_char; \ + yy_c_buf_p = yytext + n; \ + yy_hold_char = *yy_c_buf_p; \ + *yy_c_buf_p = '\0'; \ + yyleng = n; \ + } while (0) /* Internal utility routines. */ #ifndef yytext_ptr #ifdef YY_USE_PROTOS -static void yy_flex_strncpy( char* s1, yyconst char* s2, int n ) +static void yy_flex_strncpy(char* s1, yyconst char* s2, int n) #else -static void yy_flex_strncpy( s1, s2, n ) -char* s1; +static void yy_flex_strncpy(s1, s2, n) char* s1; yyconst char* s2; int n; #endif { - register int i; + register int i; - for ( i = 0; i < n; ++i ) - s1[i] = s2[i]; + for (i = 0; i < n; ++i) + s1[i] = s2[i]; } #endif #ifdef YY_NEED_STRLEN #ifdef YY_USE_PROTOS -static int yy_flex_strlen( yyconst char* s ) +static int yy_flex_strlen(yyconst char* s) #else -static int yy_flex_strlen( s ) -yyconst char* s; +static int yy_flex_strlen(s) yyconst char* s; #endif { - register int n; + register int n; - for ( n = 0; s[n]; ++n ) - ; + for (n = 0; s[n]; ++n) + ; - return n; + return n; } #endif +#ifdef YY_USE_PROTOS +static void* yy_flex_alloc(yy_size_t size) +#else +static void* yy_flex_alloc(size) yy_size_t size; +#endif +{ + return (void*)malloc(size); +} #ifdef YY_USE_PROTOS -static void* yy_flex_alloc( yy_size_t size ) +static void* yy_flex_realloc(void* ptr, yy_size_t size) #else -static void* yy_flex_alloc( size ) +static void* yy_flex_realloc(ptr, size) void* ptr; yy_size_t size; #endif { - return (void*) malloc( size ); + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return (void*)realloc((char*)ptr, size); } #ifdef YY_USE_PROTOS -static void* yy_flex_realloc( void* ptr, yy_size_t size ) +static void yy_flex_free(void* ptr) #else -static void* yy_flex_realloc( ptr, size ) -void* ptr; -yy_size_t size; +static void yy_flex_free(ptr) void* ptr; #endif { - /* The cast to (char *) in the following accommodates both - * implementations that use char* generic pointers, and those - * that use void* generic pointers. It works with the latter - * because both ANSI C and C++ allow castless assignment from - * any pointer type to void*, and deal with argument conversions - * as though doing an assignment. - */ - return (void*) realloc( (char*) ptr, size ); -} - -#ifdef YY_USE_PROTOS -static void yy_flex_free( void* ptr ) -#else -static void yy_flex_free( ptr ) -void* ptr; -#endif -{ - free( ptr ); + free(ptr); } #if YY_MAIN int main() { - yylex(); - return 0; + yylex(); + return 0; } #endif @@ -2245,17 +2012,16 @@ using namespace dmlpackage; void dmlerror(char const* s) { - printf("yyerror: %d: %s at %s\n", lineno, s, yytext); + printf("yyerror: %d: %s at %s\n", lineno, s, yytext); } namespace dmlpackage { - static valbuf_t valbuf; valbuf_t get_valbuffer(void) { - return valbuf; + return valbuf; } /* @@ -2263,64 +2029,61 @@ valbuf_t get_valbuffer(void) */ void scanner_init(const char* str) { - size_t slen = strlen(str); + size_t slen = strlen(str); - /* - * Might be left over after ereport() - */ - if (YY_CURRENT_BUFFER) - yy_delete_buffer(YY_CURRENT_BUFFER); + /* + * Might be left over after ereport() + */ + if (YY_CURRENT_BUFFER) + yy_delete_buffer(YY_CURRENT_BUFFER); - /* - * Make a scan buffer with special termination needed by flex. - */ - scanbuf = (char*)malloc(slen + 2); - memcpy(scanbuf, str, slen); - scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR; - scanbufhandle = yy_scan_buffer(scanbuf, slen + 2); + /* + * Make a scan buffer with special termination needed by flex. + */ + scanbuf = (char*)malloc(slen + 2); + memcpy(scanbuf, str, slen); + scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR; + scanbufhandle = yy_scan_buffer(scanbuf, slen + 2); - BEGIN(INITIAL); + BEGIN(INITIAL); - - valbuf.clear(); + valbuf.clear(); } - /* * Called after parsing is done to clean up after scanner_init() */ - void scanner_finish(void) { - char* str; + char* str; - yy_delete_buffer(scanbufhandle); - free(scanbuf); - unsigned int i; + yy_delete_buffer(scanbufhandle); + free(scanbuf); + unsigned int i; - for (i = 0; i < valbuf.size(); i++) + for (i = 0; i < valbuf.size(); i++) + { + str = valbuf[i]; + + if (str) { - str = valbuf[i]; - - if (str) - { - //std::cout << "valbuf:(" << str << ")" << std::endl; - free(valbuf[i]); - } + // std::cout << "valbuf:(" << str << ")" << std::endl; + free(valbuf[i]); } + } - valbuf.clear(); + valbuf.clear(); } -char* scanner_copy (char* str) +char* scanner_copy(char* str) { - char* nv = strdup(str); + char* nv = strdup(str); - if (nv) - valbuf.push_back(nv); + if (nv) + valbuf.push_back(nv); - return nv; + return nv; } -} +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dmlcolumn.cpp b/dbcon/dmlpackage/dmlcolumn.cpp index 82ec79f61..417c218f2 100644 --- a/dbcon/dmlpackage/dmlcolumn.cpp +++ b/dbcon/dmlpackage/dmlcolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlcolumn.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlcolumn.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #define DMLPKGCOLUMN_DLLEXPORT #include "dmlcolumn.h" @@ -31,95 +31,93 @@ namespace dmlpackage { - - DMLColumn::DMLColumn() -{} - -DMLColumn::DMLColumn(std::string name, std::string value, bool isFromCol, uint32_t funcScale, bool isNULL) { - fName = name; - fData = value; - - if (( strcasecmp(value.c_str(), "NULL") == 0) || (value.length() == 0) ) - { - isNULL = true; - } - - fisNULL = isNULL; - fIsFromCol = isFromCol; - fFuncScale = funcScale; - } -DMLColumn::DMLColumn(std::string name, std::vector& valueList, bool isFromCol, uint32_t funcScale, bool isNULL) +DMLColumn::DMLColumn(std::string name, std::string value, bool isFromCol, uint32_t funcScale, bool isNULL) { - fName = name; - fColValuesList = valueList; - fisNULL = isNULL; - fIsFromCol = isFromCol; - fFuncScale = funcScale; + fName = name; + fData = value; + + if ((strcasecmp(value.c_str(), "NULL") == 0) || (value.length() == 0)) + { + isNULL = true; + } + + fisNULL = isNULL; + fIsFromCol = isFromCol; + fFuncScale = funcScale; +} + +DMLColumn::DMLColumn(std::string name, std::vector& valueList, bool isFromCol, + uint32_t funcScale, bool isNULL) +{ + fName = name; + fColValuesList = valueList; + fisNULL = isNULL; + fIsFromCol = isFromCol; + fFuncScale = funcScale; } DMLColumn::~DMLColumn() -{} +{ +} int DMLColumn::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; - bytestream >> fName; - bytestream >> reinterpret_cast(fisNULL); - uint32_t vectorSize; - bytestream >> vectorSize; + int retval = 1; + bytestream >> fName; + bytestream >> reinterpret_cast(fisNULL); + uint32_t vectorSize; + bytestream >> vectorSize; - if (vectorSize > 0 ) + if (vectorSize > 0) + { + for (uint32_t i = 0; i < vectorSize; i++) { - for ( uint32_t i = 0; i < vectorSize; i++ ) - { - std::string dataStr; - bytestream >> dataStr; - // if ( !fisNULL && (dataStr.length() == 0 )) - // dataStr = (char) 0; - - fColValuesList.push_back( dataStr); - } + std::string dataStr; + bytestream >> dataStr; + // if ( !fisNULL && (dataStr.length() == 0 )) + // dataStr = (char) 0; + fColValuesList.push_back(dataStr); } - else - bytestream >> fData; //deprecated. + } + else + bytestream >> fData; // deprecated. - if ( (fColValuesList.size() < 1) && (fColValuesList.size() > 0) ) //deprecated. - fData = fColValuesList[0] ; //deprecated. + if ((fColValuesList.size() < 1) && (fColValuesList.size() > 0)) // deprecated. + fData = fColValuesList[0]; // deprecated. - //bytestream >> reinterpret_cast(fisNULL); - bytestream >> reinterpret_cast(fIsFromCol); - bytestream >> (uint32_t&) fFuncScale; - return retval; + // bytestream >> reinterpret_cast(fisNULL); + bytestream >> reinterpret_cast(fIsFromCol); + bytestream >> (uint32_t&)fFuncScale; + return retval; } int DMLColumn::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; - bytestream << fName; - bytestream << static_cast(fisNULL); - uint32_t vectorSize = fColValuesList.size(); - bytestream << vectorSize; + int retval = 1; + bytestream << fName; + bytestream << static_cast(fisNULL); + uint32_t vectorSize = fColValuesList.size(); + bytestream << vectorSize; - if (vectorSize > 0 ) + if (vectorSize > 0) + { + for (uint32_t i = 0; i < vectorSize; i++) { - for ( uint32_t i = 0; i < vectorSize; i++ ) - { - bytestream << fColValuesList[i]; - } - + bytestream << fColValuesList[i]; } - else - bytestream << fData; //deprecated. + } + else + bytestream << fData; // deprecated. - //bytestream << static_cast(fisNULL); - bytestream << static_cast(fIsFromCol); - bytestream << (uint32_t)fFuncScale; - return retval; + // bytestream << static_cast(fisNULL); + bytestream << static_cast(fIsFromCol); + bytestream << (uint32_t)fFuncScale; + return retval; } -} //namespace dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dmlcolumn.h b/dbcon/dmlpackage/dmlcolumn.h index ffed035ae..30c96501d 100644 --- a/dbcon/dmlpackage/dmlcolumn.h +++ b/dbcon/dmlpackage/dmlcolumn.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlcolumn.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlcolumn.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DMLCOLUMN_H @@ -39,133 +39,130 @@ namespace dmlpackage { /** @brief concrete implementation of a DMLObject - * Specifically for representing a database column - */ + * Specifically for representing a database column + */ class DMLColumn : public DMLObject { + public: + /** @brief ctor + */ + EXPORT DMLColumn(); -public: - /** @brief ctor - */ - EXPORT DMLColumn(); + /** @brief ctor + */ - /** @brief ctor - */ + EXPORT DMLColumn(std::string name, std::string value, bool isFromCol = false, uint32_t funcScale = 0, + bool isNULL = false); + /** @brief new ctor + * isNUll is currently not in use. It supposed to indicate whether each value is null or not. + */ - EXPORT DMLColumn(std::string name, std::string value, bool isFromCol = false, uint32_t funcScale = 0, bool isNULL = false); - /** @brief new ctor - * isNUll is currently not in use. It supposed to indicate whether each value is null or not. - */ + EXPORT DMLColumn(std::string name, std::vector& valueList, bool isFromCol = false, + uint32_t funcScale = 0, bool isNULL = false); - EXPORT DMLColumn(std::string name, std::vector& valueList, bool isFromCol = false, uint32_t funcScale = 0, bool isNULL = false ); + /** @brief dtor + */ + EXPORT ~DMLColumn(); - /** @brief dtor - */ - EXPORT ~DMLColumn(); + /** @brief read a DMLColumn from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + EXPORT int read(messageqcpp::ByteStream& bytestream); - /** @brief read a DMLColumn from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + /** @brief write a DML column to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + EXPORT int write(messageqcpp::ByteStream& bytestream); - /** @brief write a DML column to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + /** @brief get the data for the column + */ + const std::string get_Data() const + { + return fData; + } - /** @brief get the data for the column - */ - const std::string get_Data() const - { - return fData; - } + const std::vector& get_DataVector() const + { + return fColValuesList; + } + /** @brief get the data for the column + */ + bool get_isnull() const + { + return fisNULL; + } + /** @brief get the fIsFromCol data for the column + */ + bool get_isFromCol() const + { + return fIsFromCol; + } + /** @brief get the fFuncScale data for the column + */ + uint32_t get_funcScale() const + { + return fFuncScale; + } + /** @brief get the column name + */ + const std::string get_Name() const + { + return fName; + } + /** @brief set the column name + */ + EXPORT void set_Name(std::string name) + { + boost::algorithm::to_lower(name); + fName = name; + } + /** @brief set the NULL flag + */ + void set_isnull(bool isNULL) + { + fisNULL = isNULL; + } + /** @brief set the fIsFromCol flag + */ + void set_isFromCol(bool isFromCol) + { + fIsFromCol = isFromCol; + } + /** @brief set the fFuncScale + */ + void set_funcScale(uint32_t funcScale) + { + fFuncScale = funcScale; + } + void set_Data(std::string data) + { + fData = data; + } - const std::vector& get_DataVector() const - { - return fColValuesList; - } - - /** @brief get the data for the column - */ - bool get_isnull() const - { - return fisNULL; - } - /** @brief get the fIsFromCol data for the column - */ - bool get_isFromCol() const - { - return fIsFromCol; - } - /** @brief get the fFuncScale data for the column - */ - uint32_t get_funcScale() const - { - return fFuncScale; - } - /** @brief get the column name - */ - const std::string get_Name() const - { - return fName; - } - /** @brief set the column name - */ - EXPORT void set_Name( std::string name) - { - boost::algorithm::to_lower(name); - fName = name; - } - /** @brief set the NULL flag - */ - void set_isnull( bool isNULL) - { - fisNULL = isNULL; - } - /** @brief set the fIsFromCol flag - */ - void set_isFromCol( bool isFromCol) - { - fIsFromCol = isFromCol; - } - /** @brief set the fFuncScale - */ - void set_funcScale( uint32_t funcScale) - { - fFuncScale = funcScale; - } - void set_Data ( std::string data) - { - fData = data; - } - - void set_DataVector ( std::vector& dataVec) - { - fColValuesList = dataVec; - } - -protected: - -private: - std::string fName; - std::string fData; - std::vector fColValuesList; - bool fisNULL; - bool fIsFromCol; - uint32_t fFuncScale; + void set_DataVector(std::vector& dataVec) + { + fColValuesList = dataVec; + } + protected: + private: + std::string fName; + std::string fData; + std::vector fColValuesList; + bool fisNULL; + bool fIsFromCol; + uint32_t fFuncScale; }; /** @brief a vector of DMLColumns - */ -typedef std::vectorColumnList; + */ +typedef std::vector ColumnList; -} +} // namespace dmlpackage #undef EXPORT -#endif //DMLCOLUMN_H - +#endif // DMLCOLUMN_H diff --git a/dbcon/dmlpackage/dmlobject.cpp b/dbcon/dmlpackage/dmlobject.cpp index 625d13dc6..88958b4ac 100644 --- a/dbcon/dmlpackage/dmlobject.cpp +++ b/dbcon/dmlpackage/dmlobject.cpp @@ -16,24 +16,20 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlobject.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlobject.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "dmlobject.h" namespace dmlpackage { - DMLObject::DMLObject() { - } DMLObject::~DMLObject() { - } -} // namespace dmlpackage - +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dmlobject.h b/dbcon/dmlpackage/dmlobject.h index 929389b89..75d8fa4fa 100644 --- a/dbcon/dmlpackage/dmlobject.h +++ b/dbcon/dmlpackage/dmlobject.h @@ -16,57 +16,49 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlobject.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlobject.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ - #ifndef DMLOBJECT_H #define DMLOBJECT_H #include -#include"bytestream.h" - +#include "bytestream.h" namespace dmlpackage { /** @brief an abstract class that represents - * a database object to be inserted, updated or - * deleted by a DML statement - */ + * a database object to be inserted, updated or + * deleted by a DML statement + */ class DMLObject { + public: + /** @brief ctor + */ + DMLObject(); -public: + /** @brief dtor + */ + virtual ~DMLObject(); - /** @brief ctor - */ - DMLObject(); + /** @brief read a DMLObject from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + virtual int read(messageqcpp::ByteStream& bytestream) = 0; - /** @brief dtor - */ - virtual ~DMLObject(); - - /** @brief read a DMLObject from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - virtual int read(messageqcpp::ByteStream& bytestream) = 0; - - /** @brief write a DMLObject to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - virtual int write(messageqcpp::ByteStream& bytestream) = 0; - -protected: - -private: + /** @brief write a DMLObject to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + virtual int write(messageqcpp::ByteStream& bytestream) = 0; + protected: + private: }; -} -#endif //DMLOBJECT_H - - +} // namespace dmlpackage +#endif // DMLOBJECT_H diff --git a/dbcon/dmlpackage/dmlpackage.h b/dbcon/dmlpackage/dmlpackage.h index cafc6776c..ff39afa38 100644 --- a/dbcon/dmlpackage/dmlpackage.h +++ b/dbcon/dmlpackage/dmlpackage.h @@ -28,11 +28,10 @@ namespace dmlpackage { - -#define DML_DEBUG 0 // debug flag 0 for off, 1 for on +#define DML_DEBUG 0 // debug flag 0 for off, 1 for on const std::string nullValue = "nvl"; -//const size_t maxThreads = 100; -//const size_t queueSize = 200; -} // namespace dmlpackage -#endif //DMLPACKAGE_H +// const size_t maxThreads = 100; +// const size_t queueSize = 200; +} // namespace dmlpackage +#endif // DMLPACKAGE_H diff --git a/dbcon/dmlpackage/dmlparser.cpp b/dbcon/dmlpackage/dmlparser.cpp index b21caee07..236444796 100644 --- a/dbcon/dmlpackage/dmlparser.cpp +++ b/dbcon/dmlpackage/dmlparser.cpp @@ -54,118 +54,116 @@ valbuf_t get_valbuffer(void); void free_copybuffer(); void set_defaultSchema(std::string schema); -DMLParser::DMLParser() : - fStatus(-1), fDebug(false) -{} +DMLParser::DMLParser() : fStatus(-1), fDebug(false) +{ +} DMLParser::~DMLParser() { - scanner_finish(scanner); - dmllex_destroy(scanner); + scanner_finish(scanner); + dmllex_destroy(scanner); } void DMLParser::setDebug(bool debug) { - fDebug = true; + fDebug = true; } int DMLParser::parse(const char* dmltext) { - dmllex_init_extra(&scanData, &scanner); - scanner_init(dmltext, scanner); - grammar_init(&fParseTree, fDebug); - fStatus = dmlparse(scanner); + dmllex_init_extra(&scanData, &scanner); + scanner_init(dmltext, scanner); + grammar_init(&fParseTree, fDebug); + fStatus = dmlparse(scanner); - if (fStatus == 0) + if (fStatus == 0) + { + char* str; + valbuf_t valueBuffer = get_valbuffer(); + + for (unsigned int i = 0; i < valueBuffer.size(); i++) { - char* str; - valbuf_t valueBuffer = get_valbuffer(); + str = valueBuffer[i]; - for (unsigned int i = 0; i < valueBuffer.size(); i++) - { - str = valueBuffer[i]; + if (str) + { + if (i > 0) + fParseTree.fSqlText += " "; - if (str) - { - if (i > 0) - fParseTree.fSqlText += " "; - - fParseTree.fSqlText += str; - } - } + fParseTree.fSqlText += str; + } } + } - free_copybuffer(); - return fStatus; + free_copybuffer(); + return fStatus; } const ParseTree& DMLParser::getParseTree() { - if (!good()) - { - throw logic_error("The ParseTree is invalid"); - } - - return fParseTree; + if (!good()) + { + throw logic_error("The ParseTree is invalid"); + } + return fParseTree; } bool DMLParser::good() { - return fStatus == 0; + return fStatus == 0; } void DMLParser::setDefaultSchema(std::string schema) { - set_defaultSchema(schema); + set_defaultSchema(schema); } -DMLFileParser::DMLFileParser() - : DMLParser() -{} +DMLFileParser::DMLFileParser() : DMLParser() +{ +} int DMLFileParser::parse(const string& fileName) { - fStatus = -1; + fStatus = -1; - ifstream ifdml; - ifdml.open(fileName.c_str()); + ifstream ifdml; + ifdml.open(fileName.c_str()); - if (!ifdml.is_open()) - { - perror(fileName.c_str()); - return fStatus; - } + if (!ifdml.is_open()) + { + perror(fileName.c_str()); + return fStatus; + } - char dmlbuf[1024 * 1024]; - unsigned length; - ifdml.seekg(0, ios::end); - length = ifdml.tellg(); - ifdml.seekg(0, ios::beg); + char dmlbuf[1024 * 1024]; + unsigned length; + ifdml.seekg(0, ios::end); + length = ifdml.tellg(); + ifdml.seekg(0, ios::beg); - if (length > sizeof(dmlbuf) - 1) - { - throw length_error("DMLFileParser has file size hard limit of 16K."); - } + if (length > sizeof(dmlbuf) - 1) + { + throw length_error("DMLFileParser has file size hard limit of 16K."); + } - std::streamsize rcount; - rcount = ifdml.readsome(dmlbuf, sizeof(dmlbuf) - 1); + std::streamsize rcount; + rcount = ifdml.readsome(dmlbuf, sizeof(dmlbuf) - 1); - if (rcount < 0) - return fStatus; + if (rcount < 0) + return fStatus; - dmlbuf[rcount] = 0; + dmlbuf[rcount] = 0; - // cout << endl << fileName << "(" << rcount << ")" << endl; - //cout << "-----------------------------" << endl; - //cout << dmlbuf << endl; + // cout << endl << fileName << "(" << rcount << ")" << endl; + // cout << "-----------------------------" << endl; + // cout << dmlbuf << endl; - return DMLParser::parse(dmlbuf); + return DMLParser::parse(dmlbuf); } void end_sql(void) { +} /* end_sql */ -} /* end_sql */ - -} // dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dmlparser.h b/dbcon/dmlpackage/dmlparser.h index 2587b015e..056c232c8 100644 --- a/dbcon/dmlpackage/dmlparser.h +++ b/dbcon/dmlpackage/dmlparser.h @@ -33,7 +33,6 @@ namespace dmlpackage { typedef std::vector valbuf_t; - typedef SqlStatementList ParseTree; // instance data for the parser @@ -41,55 +40,54 @@ typedef std::vector valbuf_t; struct scan_data { - /* Handles to the buffer that the lexer uses internally */ - char* scanbuf; - void* scanbufhandle; // This is a YY_BUFFER_STATE defined in ddl-scan.cpp - valbuf_t valbuf; + /* Handles to the buffer that the lexer uses internally */ + char* scanbuf; + void* scanbufhandle; // This is a YY_BUFFER_STATE defined in ddl-scan.cpp + valbuf_t valbuf; }; /** @brief BISON parser wrapper class */ class DMLParser { -public: - /** @brief ctor - */ - DMLParser(); + public: + /** @brief ctor + */ + DMLParser(); - /** @brief dtor - */ - virtual ~DMLParser(); + /** @brief dtor + */ + virtual ~DMLParser(); - /** @brief parse the supplied dml statement - * - * @param dmltext the dml statement to parse - */ - int parse(const char* dmltext); + /** @brief parse the supplied dml statement + * + * @param dmltext the dml statement to parse + */ + int parse(const char* dmltext); - /** @brief get the parse tree - */ - const ParseTree& getParseTree(); + /** @brief get the parse tree + */ + const ParseTree& getParseTree(); - void setDefaultSchema(std::string schema); + void setDefaultSchema(std::string schema); - /** @brief was the parse successful - */ - bool good(); + /** @brief was the parse successful + */ + bool good(); - /** @brief put the parser in debug mode so as to dump - * diagnostic information - */ - void setDebug(bool debug); + /** @brief put the parser in debug mode so as to dump + * diagnostic information + */ + void setDebug(bool debug); -protected: - ParseTree fParseTree; - int fStatus; - bool fDebug; - void* scanner; // yyscan_t * needed for re-entrant flex scanner - scan_data scanData; - -private: + protected: + ParseTree fParseTree; + int fStatus; + bool fDebug; + void* scanner; // yyscan_t * needed for re-entrant flex scanner + scan_data scanData; + private: }; /** @brief specialization of the DMLParser class @@ -98,23 +96,22 @@ private: */ class DMLFileParser : public DMLParser { -public: - /** @brief ctor - */ - DMLFileParser(); + public: + /** @brief ctor + */ + DMLFileParser(); - /** @brief parse the dml statement contained in the - * supplied file - * - * @param fileName the fully qualified file name to open - * and parse the contents of - */ - int parse(const std::string& fileName); + /** @brief parse the dml statement contained in the + * supplied file + * + * @param fileName the fully qualified file name to open + * and parse the contents of + */ + int parse(const std::string& fileName); -protected: - -private: + protected: + private: }; -} -#endif // DMLPARSER_H +} // namespace dmlpackage +#endif // DMLPARSER_H diff --git a/dbcon/dmlpackage/dmlpkg.cpp b/dbcon/dmlpackage/dmlpkg.cpp index 5219d9a09..829373eae 100644 --- a/dbcon/dmlpackage/dmlpkg.cpp +++ b/dbcon/dmlpackage/dmlpkg.cpp @@ -27,1266 +27,1215 @@ namespace dmlpackage using namespace std; /** SqlStatementList - */ + */ ostream& operator<<(ostream& os, const SqlStatementList& ssl) { - vector::const_iterator itr; + vector::const_iterator itr; - for (itr = ssl.fList.begin(); itr != ssl.fList.end(); ++itr) - { - SqlStatement& stmt = **itr; - os << stmt; - } + for (itr = ssl.fList.begin(); itr != ssl.fList.end(); ++itr) + { + SqlStatement& stmt = **itr; + os << stmt; + } - return os; + return os; } void SqlStatementList::push_back(SqlStatement* v) { - fList.push_back(v); + fList.push_back(v); } SqlStatementList::~SqlStatementList() { - vector::iterator itr; + vector::iterator itr; - for (itr = fList.begin(); itr != fList.end(); ++itr) - { - delete *itr; - } + for (itr = fList.begin(); itr != fList.end(); ++itr) + { + delete *itr; + } } /** SqlStatement - */ -SqlStatement::SqlStatement() - : fNamePtr(0) + */ +SqlStatement::SqlStatement() : fNamePtr(0) { - } SqlStatement::~SqlStatement() { - if (0 != fNamePtr) - delete fNamePtr; + if (0 != fNamePtr) + delete fNamePtr; } ostream& operator<<(ostream& os, const SqlStatement& stmt) { - return stmt.put(os); + return stmt.put(os); } std::string SqlStatement::getSchemaName() const { - std::string schema_name; + std::string schema_name; - if (0 != fNamePtr) - schema_name = fNamePtr->fSchema; + if (0 != fNamePtr) + schema_name = fNamePtr->fSchema; - return schema_name; + return schema_name; } std::string SqlStatement::getTableName() const { - std::string table_name; + std::string table_name; - if (0 != fNamePtr) - table_name = fNamePtr->fName; + if (0 != fNamePtr) + table_name = fNamePtr->fName; - return table_name; + return table_name; } /** InsertSqlStatement */ -InsertSqlStatement::InsertSqlStatement() - : fValuesOrQueryPtr(0) +InsertSqlStatement::InsertSqlStatement() : fValuesOrQueryPtr(0) { - } -InsertSqlStatement::InsertSqlStatement(TableName* tableNamePtr, ValuesOrQuery* valsOrQueryPtr) - : fValuesOrQueryPtr(valsOrQueryPtr) +InsertSqlStatement::InsertSqlStatement(TableName* tableNamePtr, ValuesOrQuery* valsOrQueryPtr) + : fValuesOrQueryPtr(valsOrQueryPtr) { - fNamePtr = tableNamePtr; + fNamePtr = tableNamePtr; } InsertSqlStatement::InsertSqlStatement(TableName* tableNamePtr, ColumnNameList* columnNamesPtr, ValuesOrQuery* valsOrQueryPtr) - : fValuesOrQueryPtr(valsOrQueryPtr) + : fValuesOrQueryPtr(valsOrQueryPtr) { - fNamePtr = tableNamePtr; - fColumnList = *columnNamesPtr; - delete columnNamesPtr; + fNamePtr = tableNamePtr; + fColumnList = *columnNamesPtr; + delete columnNamesPtr; } InsertSqlStatement::~InsertSqlStatement() { - if (0 != fValuesOrQueryPtr) - delete fValuesOrQueryPtr; + if (0 != fValuesOrQueryPtr) + delete fValuesOrQueryPtr; } ostream& InsertSqlStatement::put(ostream& os) const { - os << "Insert " << endl; + os << "Insert " << endl; - if (0 != fNamePtr ) - { - fNamePtr->put(os); - } + if (0 != fNamePtr) + { + fNamePtr->put(os); + } - ColumnNameList::const_iterator itr; + ColumnNameList::const_iterator itr; - for (itr = fColumnList.begin(); itr != fColumnList.end(); ++itr) - { - os << *itr << endl; - } + for (itr = fColumnList.begin(); itr != fColumnList.end(); ++itr) + { + os << *itr << endl; + } - if (0 != fValuesOrQueryPtr ) - { - fValuesOrQueryPtr->put(os); + if (0 != fValuesOrQueryPtr) + { + fValuesOrQueryPtr->put(os); + } - } - - return os; + return os; } string InsertSqlStatement::getQueryString() const { - std::string query_string; + std::string query_string; - if (0 != fValuesOrQueryPtr) - { - query_string = fValuesOrQueryPtr->getQueryString(); - } + if (0 != fValuesOrQueryPtr) + { + query_string = fValuesOrQueryPtr->getQueryString(); + } - return query_string; + return query_string; } /** UpdateSqlStatement */ -UpdateSqlStatement::UpdateSqlStatement() - : fColAssignmentListPtr(0), fWhereClausePtr(0) +UpdateSqlStatement::UpdateSqlStatement() : fColAssignmentListPtr(0), fWhereClausePtr(0) { - } -UpdateSqlStatement::UpdateSqlStatement(TableName* tableNamePtr, - ColumnAssignmentList* colAssignmentListPtr, WhereClause* whereClausePtr /*=0*/) - : fColAssignmentListPtr(colAssignmentListPtr), - fWhereClausePtr(whereClausePtr) +UpdateSqlStatement::UpdateSqlStatement(TableName* tableNamePtr, ColumnAssignmentList* colAssignmentListPtr, + WhereClause* whereClausePtr /*=0*/) + : fColAssignmentListPtr(colAssignmentListPtr), fWhereClausePtr(whereClausePtr) { - fNamePtr = tableNamePtr; + fNamePtr = tableNamePtr; } UpdateSqlStatement::~UpdateSqlStatement() { - if (0 != fColAssignmentListPtr) + if (0 != fColAssignmentListPtr) + { + ColumnAssignmentList::iterator iter = fColAssignmentListPtr->begin(); + + while (iter != fColAssignmentListPtr->end()) { - ColumnAssignmentList::iterator iter = fColAssignmentListPtr->begin(); - - while (iter != fColAssignmentListPtr->end()) - { - delete *iter; - ++iter; - } - - fColAssignmentListPtr->clear(); - delete fColAssignmentListPtr; + delete *iter; + ++iter; } - if (0 != fWhereClausePtr) - { - delete fWhereClausePtr; - } + fColAssignmentListPtr->clear(); + delete fColAssignmentListPtr; + } + + if (0 != fWhereClausePtr) + { + delete fWhereClausePtr; + } } ostream& UpdateSqlStatement::put(ostream& os) const { - os << "Update " << endl; + os << "Update " << endl; - if (0 != fNamePtr) + if (0 != fNamePtr) + { + fNamePtr->put(os); + } + + if (0 != fColAssignmentListPtr) + { + os << "SET " << endl; + ColumnAssignmentList::const_iterator iter = fColAssignmentListPtr->begin(); + + while (iter != fColAssignmentListPtr->end()) { - fNamePtr->put(os); + ColumnAssignment* cola = *iter; + cola->put(os); + ++iter; } + } - if (0 != fColAssignmentListPtr) - { - os << "SET " << endl; - ColumnAssignmentList::const_iterator iter = fColAssignmentListPtr->begin(); + if (0 != fWhereClausePtr) + { + fWhereClausePtr->put(os); + } - while (iter != fColAssignmentListPtr->end()) - { - ColumnAssignment* cola = *iter; - cola->put(os); - ++iter; - } - } - - if (0 != fWhereClausePtr) - { - fWhereClausePtr->put(os); - } - - return os; + return os; } string UpdateSqlStatement::getQueryString() const { - std::string query_string; + std::string query_string; - if (0 != fColAssignmentListPtr) + if (0 != fColAssignmentListPtr) + { + query_string += "SET "; + ColumnAssignmentList::const_iterator iter = fColAssignmentListPtr->begin(); + + while (iter != fColAssignmentListPtr->end()) { - query_string += "SET "; - ColumnAssignmentList::const_iterator iter = fColAssignmentListPtr->begin(); + ColumnAssignment* cola = *iter; + query_string += cola->getColumnAssignmentString(); - while (iter != fColAssignmentListPtr->end()) - { - ColumnAssignment* cola = *iter; - query_string += cola->getColumnAssignmentString(); + ++iter; - ++iter; - - if (iter != fColAssignmentListPtr->end()) - query_string += ","; - } + if (iter != fColAssignmentListPtr->end()) + query_string += ","; } + } - if (0 != fWhereClausePtr) - { - query_string += " "; - query_string += fWhereClausePtr->getWhereClauseString(); - } + if (0 != fWhereClausePtr) + { + query_string += " "; + query_string += fWhereClausePtr->getWhereClauseString(); + } - return query_string; + return query_string; } /** DeleteSqlStatement */ -DeleteSqlStatement::DeleteSqlStatement() - : fWhereClausePtr(0) +DeleteSqlStatement::DeleteSqlStatement() : fWhereClausePtr(0) { - } DeleteSqlStatement::DeleteSqlStatement(TableName* tableNamePtr, WhereClause* whereClausePtr /*=0*/) - : fWhereClausePtr(whereClausePtr) + : fWhereClausePtr(whereClausePtr) { - fNamePtr = tableNamePtr; - + fNamePtr = tableNamePtr; } DeleteSqlStatement::~DeleteSqlStatement() { - if (0 != fWhereClausePtr) - { - delete fWhereClausePtr; - } - + if (0 != fWhereClausePtr) + { + delete fWhereClausePtr; + } } ostream& DeleteSqlStatement::put(ostream& os) const { - os << "Delete " << endl; + os << "Delete " << endl; - if (0 != fNamePtr) - { - fNamePtr->put(os); - } + if (0 != fNamePtr) + { + fNamePtr->put(os); + } - if (0 != fWhereClausePtr) - { - fWhereClausePtr->put(os); - } + if (0 != fWhereClausePtr) + { + fWhereClausePtr->put(os); + } - return os; + return os; } string DeleteSqlStatement::getQueryString() const { - std::string query_string; + std::string query_string; - if (0 != fWhereClausePtr) - { - query_string += fWhereClausePtr->getWhereClauseString(); - } + if (0 != fWhereClausePtr) + { + query_string += fWhereClausePtr->getWhereClauseString(); + } - return query_string; + return query_string; } /** CommandSqlStatement */ -CommandSqlStatement::CommandSqlStatement(std::string command) - : fCommandText(command) +CommandSqlStatement::CommandSqlStatement(std::string command) : fCommandText(command) { - } ostream& CommandSqlStatement::put(ostream& os) const { - os << fCommandText << endl; + os << fCommandText << endl; - return os; + return os; } string CommandSqlStatement::getQueryString() const { - return fCommandText; + return fCommandText; } /** TableName */ TableName::TableName() { - } TableName::TableName(char* name) { - fName = name; - + fName = name; } TableName::TableName(char* schema, char* name) { - fSchema = schema; - fName = name; + fSchema = schema; + fName = name; } ostream& TableName::put(ostream& os) const { - if (fSchema != "") - os << fSchema << "."; + if (fSchema != "") + os << fSchema << "."; - os << fName << endl; - return os; + os << fName << endl; + return os; } /** ColumnAssignment */ ostream& ColumnAssignment::put(ostream& os) const { - os << fColumn << endl; - os << fOperator << endl; - os << fScalarExpression << endl; - return os; + os << fColumn << endl; + os << fOperator << endl; + os << fScalarExpression << endl; + return os; } string ColumnAssignment::getColumnAssignmentString() const { - std::string column_assignment = fColumn; - column_assignment += " "; - column_assignment += fOperator; - column_assignment += " "; - column_assignment += fScalarExpression; + std::string column_assignment = fColumn; + column_assignment += " "; + column_assignment += fOperator; + column_assignment += " "; + column_assignment += fScalarExpression; - return column_assignment; + return column_assignment; } /** ValuesOrQuery */ -ValuesOrQuery::ValuesOrQuery() - : fQuerySpecPtr(0) +ValuesOrQuery::ValuesOrQuery() : fQuerySpecPtr(0) { } -ValuesOrQuery::ValuesOrQuery(ValuesList* valuesPtr) - : fQuerySpecPtr(0) +ValuesOrQuery::ValuesOrQuery(ValuesList* valuesPtr) : fQuerySpecPtr(0) { - fValuesList = *valuesPtr; - delete valuesPtr; + fValuesList = *valuesPtr; + delete valuesPtr; } ValuesOrQuery::ValuesOrQuery(QuerySpec* querySpecPtr) { - fQuerySpecPtr = querySpecPtr; + fQuerySpecPtr = querySpecPtr; } ValuesOrQuery::~ValuesOrQuery() { - if (0 != fQuerySpecPtr) - delete fQuerySpecPtr; + if (0 != fQuerySpecPtr) + delete fQuerySpecPtr; } ostream& ValuesOrQuery::put(ostream& os) const { - ValuesList::const_iterator iter = fValuesList.begin(); + ValuesList::const_iterator iter = fValuesList.begin(); - while ( iter != fValuesList.end() ) - { - os << *iter << endl; - ++iter; - } + while (iter != fValuesList.end()) + { + os << *iter << endl; + ++iter; + } - if (0 != fQuerySpecPtr) - { - fQuerySpecPtr->put(os); - } + if (0 != fQuerySpecPtr) + { + fQuerySpecPtr->put(os); + } - return os; + return os; } string ValuesOrQuery::getQueryString() const { - std::string query_string; + std::string query_string; - if (0 != fQuerySpecPtr) - { - query_string = fQuerySpecPtr->getQueryString(); - } + if (0 != fQuerySpecPtr) + { + query_string = fQuerySpecPtr->getQueryString(); + } - return query_string; + return query_string; } /** QuerySpec */ -QuerySpec::QuerySpec() - : fSelectFilterPtr(0), fTableExpressionPtr(0) +QuerySpec::QuerySpec() : fSelectFilterPtr(0), fTableExpressionPtr(0) { } QuerySpec::QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression) - : fSelectFilterPtr(selectFilter), fTableExpressionPtr(tableExpression) + : fSelectFilterPtr(selectFilter), fTableExpressionPtr(tableExpression) { - } -QuerySpec::QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression, - char* allOrDistinct) - : fSelectFilterPtr(selectFilter), fTableExpressionPtr(tableExpression) +QuerySpec::QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression, char* allOrDistinct) + : fSelectFilterPtr(selectFilter), fTableExpressionPtr(tableExpression) { - fOptionAllOrDistinct = allOrDistinct; + fOptionAllOrDistinct = allOrDistinct; } QuerySpec::~QuerySpec() { - if (0 != fSelectFilterPtr) - delete fSelectFilterPtr; + if (0 != fSelectFilterPtr) + delete fSelectFilterPtr; - if (0 != fTableExpressionPtr) - delete fTableExpressionPtr; + if (0 != fTableExpressionPtr) + delete fTableExpressionPtr; } ostream& QuerySpec::put(ostream& os) const { - if (0 != fSelectFilterPtr) - { - fSelectFilterPtr->put(os); - } + if (0 != fSelectFilterPtr) + { + fSelectFilterPtr->put(os); + } - if (0 != fTableExpressionPtr) - { - fTableExpressionPtr->put(os); - } + if (0 != fTableExpressionPtr) + { + fTableExpressionPtr->put(os); + } - if (fOptionAllOrDistinct != "") - { - os << fOptionAllOrDistinct << endl; - } + if (fOptionAllOrDistinct != "") + { + os << fOptionAllOrDistinct << endl; + } - return os; + return os; } string QuerySpec::getQueryString() const { - std::string query_string; + std::string query_string; - if (0 != fSelectFilterPtr) - { - query_string += fSelectFilterPtr->getSelectString(); - } + if (0 != fSelectFilterPtr) + { + query_string += fSelectFilterPtr->getSelectString(); + } - if (0 != fTableExpressionPtr) - { - query_string += " "; - query_string += fTableExpressionPtr->getTableExpressionString(); - } + if (0 != fTableExpressionPtr) + { + query_string += " "; + query_string += fTableExpressionPtr->getTableExpressionString(); + } - if (fOptionAllOrDistinct != "") - { - query_string += " "; - query_string += fOptionAllOrDistinct; - } + if (fOptionAllOrDistinct != "") + { + query_string += " "; + query_string += fOptionAllOrDistinct; + } - return query_string; + return query_string; } /** SelectFilter */ -SelectFilter::SelectFilter() - : fColumnList(0) +SelectFilter::SelectFilter() : fColumnList(0) { } SelectFilter::SelectFilter(ColumnNameList* columnListPtr) { - fColumnList = *columnListPtr; + fColumnList = *columnListPtr; - delete columnListPtr; + delete columnListPtr; } SelectFilter::~SelectFilter() { - } ostream& SelectFilter::put(ostream& os) const { - os << "SELECT" << endl; - ColumnNameList::const_iterator iter = fColumnList.begin(); + os << "SELECT" << endl; + ColumnNameList::const_iterator iter = fColumnList.begin(); - while ( iter != fColumnList.end() ) - { - os << *iter << endl; - ++iter; - } + while (iter != fColumnList.end()) + { + os << *iter << endl; + ++iter; + } - if (0 == fColumnList.size()) - os << "*" << endl; + if (0 == fColumnList.size()) + os << "*" << endl; - return os; + return os; } string SelectFilter::getSelectString() const { - std::string select_filter = "SELECT "; - ColumnNameList::const_iterator iter = fColumnList.begin(); + std::string select_filter = "SELECT "; + ColumnNameList::const_iterator iter = fColumnList.begin(); - while ( iter != fColumnList.end() ) - { - select_filter += *iter; - ++iter; + while (iter != fColumnList.end()) + { + select_filter += *iter; + ++iter; - if (iter != fColumnList.end()) - select_filter += ","; - } + if (iter != fColumnList.end()) + select_filter += ","; + } - if (0 == fColumnList.size()) - select_filter += "*"; + if (0 == fColumnList.size()) + select_filter += "*"; - return select_filter; + return select_filter; } /** TableExpression */ -TableExpression::TableExpression() - : fFromClausePtr(0), fWhereClausePtr(0), - fGroupByPtr(0), fHavingPtr(0) +TableExpression::TableExpression() : fFromClausePtr(0), fWhereClausePtr(0), fGroupByPtr(0), fHavingPtr(0) { - } TableExpression::TableExpression(FromClause* fromClausePtr, WhereClause* whereClausePtr, GroupByClause* groupByPtr, HavingClause* havingPtr) - : fFromClausePtr(fromClausePtr), fWhereClausePtr(whereClausePtr), - fGroupByPtr(groupByPtr), fHavingPtr(havingPtr) + : fFromClausePtr(fromClausePtr) + , fWhereClausePtr(whereClausePtr) + , fGroupByPtr(groupByPtr) + , fHavingPtr(havingPtr) { - } TableExpression::~TableExpression() { - if (0 != fFromClausePtr) - delete fFromClausePtr; + if (0 != fFromClausePtr) + delete fFromClausePtr; - if (0 != fWhereClausePtr) - delete fWhereClausePtr; + if (0 != fWhereClausePtr) + delete fWhereClausePtr; - if (0 != fGroupByPtr) - delete fGroupByPtr; - - if (0 != fHavingPtr) - delete fHavingPtr; + if (0 != fGroupByPtr) + delete fGroupByPtr; + if (0 != fHavingPtr) + delete fHavingPtr; } ostream& TableExpression::put(ostream& os) const { - if (0 != fFromClausePtr) - { - fFromClausePtr->put(os); - } + if (0 != fFromClausePtr) + { + fFromClausePtr->put(os); + } - if (0 != fWhereClausePtr) - { - fWhereClausePtr->put(os); - } + if (0 != fWhereClausePtr) + { + fWhereClausePtr->put(os); + } - if (0 != fGroupByPtr) - { - fGroupByPtr->put(os); - } + if (0 != fGroupByPtr) + { + fGroupByPtr->put(os); + } - if (0 != fHavingPtr) - { - fHavingPtr->put(os); - } + if (0 != fHavingPtr) + { + fHavingPtr->put(os); + } - return os; + return os; } string TableExpression::getTableExpressionString() const { - std::string table_expression; + std::string table_expression; - if (0 != fFromClausePtr) - { - table_expression += fFromClausePtr->getFromClauseString(); - } + if (0 != fFromClausePtr) + { + table_expression += fFromClausePtr->getFromClauseString(); + } - if (0 != fWhereClausePtr) - { - table_expression += " "; - table_expression += fWhereClausePtr->getWhereClauseString(); - } + if (0 != fWhereClausePtr) + { + table_expression += " "; + table_expression += fWhereClausePtr->getWhereClauseString(); + } - if (0 != fGroupByPtr) - { - table_expression += " "; - table_expression += fGroupByPtr->getGroupByClauseString(); - } + if (0 != fGroupByPtr) + { + table_expression += " "; + table_expression += fGroupByPtr->getGroupByClauseString(); + } - if (0 != fHavingPtr) - { - table_expression += " "; - table_expression += fHavingPtr->getHavingClauseString(); - } + if (0 != fHavingPtr) + { + table_expression += " "; + table_expression += fHavingPtr->getHavingClauseString(); + } - return table_expression; + return table_expression; } /** FromClause */ -FromClause::FromClause() - : fTableListPtr(0) +FromClause::FromClause() : fTableListPtr(0) { } FromClause::FromClause(TableNameList* tableNameList) { - fTableListPtr = tableNameList; + fTableListPtr = tableNameList; } FromClause::~FromClause() { - if (0 != fTableListPtr) + if (0 != fTableListPtr) + { + TableNameList::iterator iter = fTableListPtr->begin(); + + while (iter != fTableListPtr->end()) { - TableNameList::iterator iter = fTableListPtr->begin(); - - while ( iter != fTableListPtr->end() ) - { - TableName* tableNamePtr = *iter; - delete tableNamePtr; - ++iter; - } - - fTableListPtr->clear(); - delete fTableListPtr; + TableName* tableNamePtr = *iter; + delete tableNamePtr; + ++iter; } + + fTableListPtr->clear(); + delete fTableListPtr; + } } ostream& FromClause::put(ostream& os) const { - os << "FROM" << endl; + os << "FROM" << endl; - if (0 != fTableListPtr) + if (0 != fTableListPtr) + { + TableNameList::const_iterator iter = fTableListPtr->begin(); + + while (iter != fTableListPtr->end()) { - TableNameList::const_iterator iter = fTableListPtr->begin(); - - while ( iter != fTableListPtr->end() ) - { - TableName* tableNamePtr = *iter; - tableNamePtr->put(os); - ++iter; - } + TableName* tableNamePtr = *iter; + tableNamePtr->put(os); + ++iter; } + } - return os; + return os; } string FromClause::getFromClauseString() const { - std::string from_clause = "FROM "; + std::string from_clause = "FROM "; - if (0 != fTableListPtr) + if (0 != fTableListPtr) + { + TableNameList::const_iterator iter = fTableListPtr->begin(); + + while (iter != fTableListPtr->end()) { - TableNameList::const_iterator iter = fTableListPtr->begin(); + TableName* tableNamePtr = *iter; - while (iter != fTableListPtr->end()) - { - TableName* tableNamePtr = *iter; + if (tableNamePtr->fSchema != "") + { + from_clause += tableNamePtr->fSchema; + from_clause += "."; + } - if (tableNamePtr->fSchema != "") - { - from_clause += tableNamePtr->fSchema; - from_clause += "."; - } + from_clause += tableNamePtr->fName; - from_clause += tableNamePtr->fName; + ++iter; - ++iter; - - if (iter != fTableListPtr->end()) - from_clause += ","; - } + if (iter != fTableListPtr->end()) + from_clause += ","; } + } - return from_clause; + return from_clause; } /** WhereClause */ -WhereClause::WhereClause() - : fSearchConditionPtr(0) +WhereClause::WhereClause() : fSearchConditionPtr(0) { - } WhereClause::~WhereClause() { - if (0 != fSearchConditionPtr) - { - delete fSearchConditionPtr; - } + if (0 != fSearchConditionPtr) + { + delete fSearchConditionPtr; + } } ostream& WhereClause::put(ostream& os) const { - os << "WHERE" << endl; + os << "WHERE" << endl; - if (0 != fSearchConditionPtr) - { - fSearchConditionPtr->put(os); - } + if (0 != fSearchConditionPtr) + { + fSearchConditionPtr->put(os); + } - return os; + return os; } string WhereClause::getWhereClauseString() const { - std::string where_clause = "WHERE"; + std::string where_clause = "WHERE"; - if (0 != fSearchConditionPtr) - { - where_clause += " "; - where_clause += fSearchConditionPtr->getSearchConditionString(); - } + if (0 != fSearchConditionPtr) + { + where_clause += " "; + where_clause += fSearchConditionPtr->getSearchConditionString(); + } - return where_clause; + return where_clause; } /** HavingClause */ -HavingClause::HavingClause() - : fSearchConditionPtr(0) +HavingClause::HavingClause() : fSearchConditionPtr(0) { } HavingClause::~HavingClause() { - if (fSearchConditionPtr != 0) - { - delete fSearchConditionPtr; - } + if (fSearchConditionPtr != 0) + { + delete fSearchConditionPtr; + } } ostream& HavingClause::put(ostream& os) const { - os << "HAVING" << endl; + os << "HAVING" << endl; - if (0 != fSearchConditionPtr) - { - fSearchConditionPtr->put(os); - } + if (0 != fSearchConditionPtr) + { + fSearchConditionPtr->put(os); + } - return os; + return os; } string HavingClause::getHavingClauseString() const { - std::string having_clause = "HAVING"; + std::string having_clause = "HAVING"; - if (0 != fSearchConditionPtr) - { - having_clause += " "; - having_clause += fSearchConditionPtr->getSearchConditionString(); - } + if (0 != fSearchConditionPtr) + { + having_clause += " "; + having_clause += fSearchConditionPtr->getSearchConditionString(); + } - return having_clause; + return having_clause; } /** GroupByClause */ -GroupByClause::GroupByClause() - : fColumnNamesListPtr(0) +GroupByClause::GroupByClause() : fColumnNamesListPtr(0) { } GroupByClause::~GroupByClause() { - if (0 != fColumnNamesListPtr) - { - fColumnNamesListPtr->clear(); - delete fColumnNamesListPtr; - } + if (0 != fColumnNamesListPtr) + { + fColumnNamesListPtr->clear(); + delete fColumnNamesListPtr; + } } ostream& GroupByClause::put(ostream& os) const { - os << "GROUP BY" << endl; + os << "GROUP BY" << endl; - if (0 != fColumnNamesListPtr) + if (0 != fColumnNamesListPtr) + { + ColumnNameList::const_iterator iter = fColumnNamesListPtr->begin(); + + if (iter != fColumnNamesListPtr->end()) { - ColumnNameList::const_iterator iter = fColumnNamesListPtr->begin(); - - if (iter != fColumnNamesListPtr->end()) - { - os << *iter << endl; - ++iter; - } + os << *iter << endl; + ++iter; } + } - return os; + return os; } string GroupByClause::getGroupByClauseString() const { - std::string group_by_clause = "GROUP BY "; + std::string group_by_clause = "GROUP BY "; - if (0 != fColumnNamesListPtr) + if (0 != fColumnNamesListPtr) + { + ColumnNameList::const_iterator iter = fColumnNamesListPtr->begin(); + + if (iter != fColumnNamesListPtr->end()) { - ColumnNameList::const_iterator iter = fColumnNamesListPtr->begin(); + group_by_clause += *iter; + ++iter; - if (iter != fColumnNamesListPtr->end()) - { - group_by_clause += *iter; - ++iter; - - if (iter != fColumnNamesListPtr->end()) - group_by_clause += ","; - } + if (iter != fColumnNamesListPtr->end()) + group_by_clause += ","; } + } - return group_by_clause; + return group_by_clause; } /** Escape */ ostream& Escape::put(ostream& os) const { - os << "ESCAPE" << endl; + os << "ESCAPE" << endl; - os << fEscapeChar << endl; - return os; + os << fEscapeChar << endl; + return os; } /** SearchCondition */ -SearchCondition::SearchCondition() - : fPredicatePtr(0), fLHSearchConditionPtr(0), - fRHSearchConditionPtr(0) +SearchCondition::SearchCondition() : fPredicatePtr(0), fLHSearchConditionPtr(0), fRHSearchConditionPtr(0) { } SearchCondition::~SearchCondition() { - if (0 != fPredicatePtr) - { - delete fPredicatePtr; - } + if (0 != fPredicatePtr) + { + delete fPredicatePtr; + } - if (0 != fLHSearchConditionPtr) - { - delete fLHSearchConditionPtr; - } - - if (0 != fRHSearchConditionPtr) - { - delete fRHSearchConditionPtr; - } + if (0 != fLHSearchConditionPtr) + { + delete fLHSearchConditionPtr; + } + if (0 != fRHSearchConditionPtr) + { + delete fRHSearchConditionPtr; + } } ostream& SearchCondition::put(ostream& os) const { - if (0 != fPredicatePtr) - { - fPredicatePtr->put(os); - } + if (0 != fPredicatePtr) + { + fPredicatePtr->put(os); + } - if (0 != fLHSearchConditionPtr) - { - fLHSearchConditionPtr->put(os); - } + if (0 != fLHSearchConditionPtr) + { + fLHSearchConditionPtr->put(os); + } - if (0 != fRHSearchConditionPtr) - { - os << fOperator << endl; - fRHSearchConditionPtr->put(os); - } + if (0 != fRHSearchConditionPtr) + { + os << fOperator << endl; + fRHSearchConditionPtr->put(os); + } - return os; + return os; } string SearchCondition::getSearchConditionString() const { - std::string search_condition; + std::string search_condition; - if (0 != fPredicatePtr) - { - search_condition += fPredicatePtr->getPredicateString(); - } + if (0 != fPredicatePtr) + { + search_condition += fPredicatePtr->getPredicateString(); + } - if (0 != fLHSearchConditionPtr) - { - search_condition += fLHSearchConditionPtr->getSearchConditionString(); - search_condition += " "; - } + if (0 != fLHSearchConditionPtr) + { + search_condition += fLHSearchConditionPtr->getSearchConditionString(); + search_condition += " "; + } - if (0 != fRHSearchConditionPtr) - { - search_condition += fOperator; - search_condition += " "; - search_condition += fRHSearchConditionPtr->getSearchConditionString(); - } + if (0 != fRHSearchConditionPtr) + { + search_condition += fOperator; + search_condition += " "; + search_condition += fRHSearchConditionPtr->getSearchConditionString(); + } - return search_condition; + return search_condition; } /** ExistanceTestPredicate */ -ExistanceTestPredicate::ExistanceTestPredicate() - : Predicate(EXIST_PREDICATE), fSubQuerySpecPtr(0) +ExistanceTestPredicate::ExistanceTestPredicate() : Predicate(EXIST_PREDICATE), fSubQuerySpecPtr(0) { } ExistanceTestPredicate::~ExistanceTestPredicate() { - if (0 != fSubQuerySpecPtr) - { - delete fSubQuerySpecPtr; - } + if (0 != fSubQuerySpecPtr) + { + delete fSubQuerySpecPtr; + } } ostream& ExistanceTestPredicate::put(ostream& os) const { - //cout << "EXISTS" << endl; - //cout << "(" << endl; - if (0 != fSubQuerySpecPtr) - { - fSubQuerySpecPtr->put(os); - } + // cout << "EXISTS" << endl; + // cout << "(" << endl; + if (0 != fSubQuerySpecPtr) + { + fSubQuerySpecPtr->put(os); + } - //cout << ")" << endl; - return os; + // cout << ")" << endl; + return os; } string ExistanceTestPredicate::getPredicateString() const { - std::string exists_predicate = "EXISTS"; - exists_predicate += "("; + std::string exists_predicate = "EXISTS"; + exists_predicate += "("; - if (0 != fSubQuerySpecPtr) - { - exists_predicate += " "; - exists_predicate += fSubQuerySpecPtr->getQueryString(); - } + if (0 != fSubQuerySpecPtr) + { + exists_predicate += " "; + exists_predicate += fSubQuerySpecPtr->getQueryString(); + } - exists_predicate += ")"; - return exists_predicate; + exists_predicate += ")"; + return exists_predicate; } /** AllOrAnyPredicate */ -AllOrAnyPredicate::AllOrAnyPredicate() - : Predicate(ALLORANY_PREDICATE), fSubQuerySpecPtr(0) +AllOrAnyPredicate::AllOrAnyPredicate() : Predicate(ALLORANY_PREDICATE), fSubQuerySpecPtr(0) { - } AllOrAnyPredicate::~AllOrAnyPredicate() { - if (0 != fSubQuerySpecPtr) - { - delete fSubQuerySpecPtr; - } + if (0 != fSubQuerySpecPtr) + { + delete fSubQuerySpecPtr; + } } ostream& AllOrAnyPredicate::put(ostream& os) const { - os << fScalarExpression << endl; - os << fOperator; - os << fAnyAllSome; + os << fScalarExpression << endl; + os << fOperator; + os << fAnyAllSome; - if (0 != fSubQuerySpecPtr) - { - fSubQuerySpecPtr->put(os); - } + if (0 != fSubQuerySpecPtr) + { + fSubQuerySpecPtr->put(os); + } - return os; + return os; } string AllOrAnyPredicate::getPredicateString() const { - std::string all_or_any = fScalarExpression; - all_or_any += " "; - all_or_any += fOperator; - all_or_any += " "; - all_or_any += fAnyAllSome; + std::string all_or_any = fScalarExpression; + all_or_any += " "; + all_or_any += fOperator; + all_or_any += " "; + all_or_any += fAnyAllSome; - if (0 != fSubQuerySpecPtr) - { - all_or_any += fSubQuerySpecPtr->getQueryString(); - } + if (0 != fSubQuerySpecPtr) + { + all_or_any += fSubQuerySpecPtr->getQueryString(); + } - return all_or_any; + return all_or_any; } /** InPredicate */ -InPredicate::InPredicate() - : Predicate(IN_PREDICATE), fSubQuerySpecPtr(0) +InPredicate::InPredicate() : Predicate(IN_PREDICATE), fSubQuerySpecPtr(0) { - } InPredicate::~InPredicate() { - if (0 != fSubQuerySpecPtr) - { - delete fSubQuerySpecPtr; - } + if (0 != fSubQuerySpecPtr) + { + delete fSubQuerySpecPtr; + } } ostream& InPredicate::put(ostream& os) const { - os << fScalarExpression << endl; - os << fOperator << endl; + os << fScalarExpression << endl; + os << fOperator << endl; - os << "(" << endl; - AtomList::const_iterator iter = fAtomList.begin(); + os << "(" << endl; + AtomList::const_iterator iter = fAtomList.begin(); - while (iter != fAtomList.end()) - { - os << *iter << endl; - ++iter; - } + while (iter != fAtomList.end()) + { + os << *iter << endl; + ++iter; + } - if (0 != fSubQuerySpecPtr) - { - fSubQuerySpecPtr->put(os); - } + if (0 != fSubQuerySpecPtr) + { + fSubQuerySpecPtr->put(os); + } - os << ")" << endl; + os << ")" << endl; - return os; + return os; } string InPredicate::getPredicateString() const { - std::string in_predicate = fScalarExpression; - in_predicate += " "; - in_predicate += fOperator; - in_predicate += " "; - in_predicate += "("; + std::string in_predicate = fScalarExpression; + in_predicate += " "; + in_predicate += fOperator; + in_predicate += " "; + in_predicate += "("; - AtomList::const_iterator iter = fAtomList.begin(); + AtomList::const_iterator iter = fAtomList.begin(); - while (iter != fAtomList.end()) - { - in_predicate += *iter; - ++iter; + while (iter != fAtomList.end()) + { + in_predicate += *iter; + ++iter; - if (iter != fAtomList.end()) - in_predicate += ","; - } + if (iter != fAtomList.end()) + in_predicate += ","; + } - if (0 != fSubQuerySpecPtr) - { - in_predicate += fSubQuerySpecPtr->getQueryString(); - } + if (0 != fSubQuerySpecPtr) + { + in_predicate += fSubQuerySpecPtr->getQueryString(); + } - in_predicate += ")"; + in_predicate += ")"; - return in_predicate; + return in_predicate; } /** NullTestPredicate */ -NullTestPredicate::NullTestPredicate() - : Predicate(NULLTEST_PREDICATE) +NullTestPredicate::NullTestPredicate() : Predicate(NULLTEST_PREDICATE) { } NullTestPredicate::~NullTestPredicate() { - } ostream& NullTestPredicate::put(ostream& os) const { - os << fColumnRef; - os << fOperator; + os << fColumnRef; + os << fOperator; - return os; + return os; } string NullTestPredicate::getPredicateString() const { - std::string null_test_predicate = fColumnRef; - null_test_predicate += " "; - null_test_predicate += fOperator; + std::string null_test_predicate = fColumnRef; + null_test_predicate += " "; + null_test_predicate += fOperator; - return null_test_predicate; + return null_test_predicate; } /** LikePredicate */ -LikePredicate::LikePredicate() - : Predicate(LIKE_PREDICATE), fOptionalEscapePtr(0) +LikePredicate::LikePredicate() : Predicate(LIKE_PREDICATE), fOptionalEscapePtr(0) { - } LikePredicate::~LikePredicate() { - if (0 != fOptionalEscapePtr) - delete fOptionalEscapePtr; + if (0 != fOptionalEscapePtr) + delete fOptionalEscapePtr; } ostream& LikePredicate::put(ostream& os) const { - os << fLHScalarExpression << endl; - os << fAtom << endl; + os << fLHScalarExpression << endl; + os << fAtom << endl; - if (0 != fOptionalEscapePtr) - fOptionalEscapePtr->put(os); + if (0 != fOptionalEscapePtr) + fOptionalEscapePtr->put(os); - return os; + return os; } string LikePredicate::getPredicateString() const { - std::string like_predicate = fLHScalarExpression; + std::string like_predicate = fLHScalarExpression; + like_predicate += " "; + like_predicate += fAtom; + + if (0 != fOptionalEscapePtr) + { like_predicate += " "; - like_predicate += fAtom; + like_predicate += fOptionalEscapePtr->fEscapeChar; + } - if (0 != fOptionalEscapePtr) - { - like_predicate += " "; - like_predicate += fOptionalEscapePtr->fEscapeChar; - } - - return like_predicate; + return like_predicate; } /** BetweenPredicate */ -BetweenPredicate::BetweenPredicate() - : Predicate(BETWEEN_PREDICATE) +BetweenPredicate::BetweenPredicate() : Predicate(BETWEEN_PREDICATE) { } BetweenPredicate::~BetweenPredicate() { - } ostream& BetweenPredicate::put(ostream& os) const { + os << fLHScalarExpression << endl; + os << fOperator1 << endl; + os << fRH1ScalarExpression << endl; + os << fOperator2 << endl; + os << fRH2ScalarExpression << endl; - os << fLHScalarExpression << endl; - os << fOperator1 << endl; - os << fRH1ScalarExpression << endl; - os << fOperator2 << endl; - os << fRH2ScalarExpression << endl; - - return os; + return os; } string BetweenPredicate::getPredicateString() const { - std::string between_predicate = fLHScalarExpression; - between_predicate += " "; - between_predicate += fOperator1; - between_predicate += " "; - between_predicate += fRH1ScalarExpression; - between_predicate += " "; - between_predicate += fOperator2; - between_predicate += " "; - between_predicate += fRH2ScalarExpression; + std::string between_predicate = fLHScalarExpression; + between_predicate += " "; + between_predicate += fOperator1; + between_predicate += " "; + between_predicate += fRH1ScalarExpression; + between_predicate += " "; + between_predicate += fOperator2; + between_predicate += " "; + between_predicate += fRH2ScalarExpression; - return between_predicate; + return between_predicate; } /** ComparisonPredicate */ -ComparisonPredicate::ComparisonPredicate() - : Predicate(COMPARE_PREDICATE), fSubQuerySpec(0) +ComparisonPredicate::ComparisonPredicate() : Predicate(COMPARE_PREDICATE), fSubQuerySpec(0) { } ComparisonPredicate::~ComparisonPredicate() { - if (0 != fSubQuerySpec) - delete fSubQuerySpec; + if (0 != fSubQuerySpec) + delete fSubQuerySpec; } ostream& ComparisonPredicate::put(ostream& os) const { - os << fLHScalarExpression << endl; - os << fOperator << endl; - os << fRHScalarExpression << endl; + os << fLHScalarExpression << endl; + os << fOperator << endl; + os << fRHScalarExpression << endl; - if (0 != fSubQuerySpec) - fSubQuerySpec->put(os); + if (0 != fSubQuerySpec) + fSubQuerySpec->put(os); - return os; + return os; } string ComparisonPredicate::getPredicateString() const { - std::string comparison_predicate = fLHScalarExpression; - comparison_predicate += " "; - comparison_predicate += fOperator; - comparison_predicate += " "; - comparison_predicate += fRHScalarExpression; + std::string comparison_predicate = fLHScalarExpression; + comparison_predicate += " "; + comparison_predicate += fOperator; + comparison_predicate += " "; + comparison_predicate += fRHScalarExpression; - if (0 != fSubQuerySpec) - { - comparison_predicate += " "; - comparison_predicate += fSubQuerySpec->getQueryString(); - } + if (0 != fSubQuerySpec) + { + comparison_predicate += " "; + comparison_predicate += fSubQuerySpec->getQueryString(); + } - return comparison_predicate; + return comparison_predicate; } /** Predicate */ -Predicate::Predicate() - : fPredicateType(INVALID_PREDICATE) +Predicate::Predicate() : fPredicateType(INVALID_PREDICATE) { } -Predicate::Predicate(PREDICATE_TYPE predicateType) - : fPredicateType(predicateType) +Predicate::Predicate(PREDICATE_TYPE predicateType) : fPredicateType(predicateType) { - } Predicate::~Predicate() { - } ostream& Predicate::put(ostream& os) const { - return os; + return os; } string Predicate::getPredicateString() const { - std::string predicate; + std::string predicate; - return predicate; + return predicate; } -} //namespace dmlpackage +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dmlpkg.h b/dbcon/dmlpackage/dmlpkg.h index f7377f8a3..eddb18960 100644 --- a/dbcon/dmlpackage/dmlpkg.h +++ b/dbcon/dmlpackage/dmlpkg.h @@ -34,7 +34,6 @@ namespace dmlpackage { - class DeleteSqlStatement; class UpdateSqlStatement; class InsertSqlStatement; @@ -81,25 +80,25 @@ std::ostream& operator<<(std::ostream& os, const SqlStatement& stmt); */ enum PREDICATE_TYPE { - COMPARE_PREDICATE, - BETWEEN_PREDICATE, - LIKE_PREDICATE, - NULLTEST_PREDICATE, - IN_PREDICATE, - ALLORANY_PREDICATE, - EXIST_PREDICATE, - INVALID_PREDICATE + COMPARE_PREDICATE, + BETWEEN_PREDICATE, + LIKE_PREDICATE, + NULLTEST_PREDICATE, + IN_PREDICATE, + ALLORANY_PREDICATE, + EXIST_PREDICATE, + INVALID_PREDICATE }; /** @brief DML Statement types */ enum DML_TYPE { - DML_INSERT, - DML_UPDATE, - DML_DELETE, - DML_COMMAND, - DML_INVALID_TYPE + DML_INSERT, + DML_UPDATE, + DML_DELETE, + DML_COMMAND, + DML_INVALID_TYPE }; /** @brief SqlStatement represents a toplevel @@ -112,40 +111,39 @@ enum DML_TYPE */ class SqlStatement { -public: - /** @brief ctor - */ - SqlStatement(); + public: + /** @brief ctor + */ + SqlStatement(); - /** @brief dtor - */ - virtual ~SqlStatement(); + /** @brief dtor + */ + virtual ~SqlStatement(); - /** @brief dump to stdout. - */ - virtual std::ostream& put(std::ostream& os) const = 0; + /** @brief dump to stdout. + */ + virtual std::ostream& put(std::ostream& os) const = 0; - /** @brief get the query string associated with the - * SqlStatement - */ - virtual std::string getQueryString() const = 0; + /** @brief get the query string associated with the + * SqlStatement + */ + virtual std::string getQueryString() const = 0; - /** @brief get the statement type - */ - virtual int getStatementType() const = 0; + /** @brief get the statement type + */ + virtual int getStatementType() const = 0; - /** @brief get the schema name from the - * TableName data member - */ - virtual std::string getSchemaName() const; + /** @brief get the schema name from the + * TableName data member + */ + virtual std::string getSchemaName() const; - /** @brief get the table name from the - * TableName data member - */ - virtual std::string getTableName() const; - - TableName* fNamePtr; + /** @brief get the table name from the + * TableName data member + */ + virtual std::string getTableName() const; + TableName* fNamePtr; }; /** @brief Collects SqlStatements so that we can support the @@ -157,37 +155,37 @@ public: */ class SqlStatementList { -public: - /** @brief ctor - */ - SqlStatementList() - {} + public: + /** @brief ctor + */ + SqlStatementList() + { + } - /** @brief dtor - */ - virtual ~SqlStatementList(); + /** @brief dtor + */ + virtual ~SqlStatementList(); - /** @brief get the SqlStatement at the given index - * - * @param i the index - */ - SqlStatement* operator[](int i) const - { - return fList[i]; - } + /** @brief get the SqlStatement at the given index + * + * @param i the index + */ + SqlStatement* operator[](int i) const + { + return fList[i]; + } - /** @brief push the supplied SqlStatement pointer onto the list - * - * @param v a pointer to a SqlStatement - */ - void push_back(SqlStatement* v); + /** @brief push the supplied SqlStatement pointer onto the list + * + * @param v a pointer to a SqlStatement + */ + void push_back(SqlStatement* v); - std::vector fList; - std::string fSqlText; - -private: - SqlStatementList(const SqlStatementList& x); + std::vector fList; + std::string fSqlText; + private: + SqlStatementList(const SqlStatementList& x); }; /** @brief The representation of a parsed @@ -198,48 +196,47 @@ private: */ class InsertSqlStatement : public SqlStatement { -public: - /** @brief ctor - */ - InsertSqlStatement(); + public: + /** @brief ctor + */ + InsertSqlStatement(); - /** @brief ctor - * - * @param tableNamePtr pointer to a TableName object - * @param valsOrQueryPtr pointer to a ValuesOrQueryObject - */ - InsertSqlStatement(TableName* tableNamePtr, ValuesOrQuery* valsOrQueryPtr); + /** @brief ctor + * + * @param tableNamePtr pointer to a TableName object + * @param valsOrQueryPtr pointer to a ValuesOrQueryObject + */ + InsertSqlStatement(TableName* tableNamePtr, ValuesOrQuery* valsOrQueryPtr); - /** @brief ctor - * - * @param tableNamePtr pointer to a TableName object - * @param columnNamesPtr pointer to ColumnNamesList object - * @param valsOrQueryPtr pointer to ValuesOrQueryObject - */ - InsertSqlStatement(TableName* tableNamePtr, ColumnNameList* columnNamesPtr, - ValuesOrQuery* valsOrQueryPtr); + /** @brief ctor + * + * @param tableNamePtr pointer to a TableName object + * @param columnNamesPtr pointer to ColumnNamesList object + * @param valsOrQueryPtr pointer to ValuesOrQueryObject + */ + InsertSqlStatement(TableName* tableNamePtr, ColumnNameList* columnNamesPtr, ValuesOrQuery* valsOrQueryPtr); - /** @brief dtor - */ - virtual ~InsertSqlStatement(); + /** @brief dtor + */ + virtual ~InsertSqlStatement(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get a string representation of the query spec - */ - virtual std::string getQueryString() const; + /** @brief get a string representation of the query spec + */ + virtual std::string getQueryString() const; - /** @brief get the statement type - DML_INSERT - */ - inline virtual int getStatementType() const - { - return DML_INSERT; - } + /** @brief get the statement type - DML_INSERT + */ + inline virtual int getStatementType() const + { + return DML_INSERT; + } - ValuesOrQuery* fValuesOrQueryPtr; - ColumnNameList fColumnList; + ValuesOrQuery* fValuesOrQueryPtr; + ColumnNameList fColumnList; }; /** @brief The representation of a parsed @@ -250,43 +247,43 @@ public: */ class UpdateSqlStatement : public SqlStatement { -public: - /** @brief ctor - */ - UpdateSqlStatement(); + public: + /** @brief ctor + */ + UpdateSqlStatement(); - /** @brief ctor - * - * @param tableNamePtr pointer to a TableName object - * @param colAssignmentPtr pointer to a ColumnAssignmentList object - * @param whereClausePtr pointer to a WhereClause object - default 0 - */ - UpdateSqlStatement(TableName* tableNamePtr, ColumnAssignmentList* colAssignmentListPtr, - WhereClause* whereClausePtr = 0); + /** @brief ctor + * + * @param tableNamePtr pointer to a TableName object + * @param colAssignmentPtr pointer to a ColumnAssignmentList object + * @param whereClausePtr pointer to a WhereClause object - default 0 + */ + UpdateSqlStatement(TableName* tableNamePtr, ColumnAssignmentList* colAssignmentListPtr, + WhereClause* whereClausePtr = 0); - /** @brief dtor - */ - virtual ~UpdateSqlStatement(); + /** @brief dtor + */ + virtual ~UpdateSqlStatement(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the - * SET assignment_commalist opt_where_clause - * statement - */ - virtual std::string getQueryString() const; + /** @brief get the string representation of the + * SET assignment_commalist opt_where_clause + * statement + */ + virtual std::string getQueryString() const; - /** @brief get the statement type - DML_UPDATE - */ - inline virtual int getStatementType() const - { - return DML_UPDATE; - } + /** @brief get the statement type - DML_UPDATE + */ + inline virtual int getStatementType() const + { + return DML_UPDATE; + } - ColumnAssignmentList* fColAssignmentListPtr; - WhereClause* fWhereClausePtr; + ColumnAssignmentList* fColAssignmentListPtr; + WhereClause* fWhereClausePtr; }; /** @brief The representation of a parsed @@ -297,38 +294,38 @@ public: */ class DeleteSqlStatement : public SqlStatement { -public: - /** @brief ctor - */ - DeleteSqlStatement(); + public: + /** @brief ctor + */ + DeleteSqlStatement(); - /** @brief ctor - * - * @param tableNamePtr pointer to a TableName object - * @param whereClausePtr pointer to a WhereClause object - default = 0 - */ - DeleteSqlStatement(TableName* tableNamePtr, WhereClause* whereClausePtr = 0); + /** @brief ctor + * + * @param tableNamePtr pointer to a TableName object + * @param whereClausePtr pointer to a WhereClause object - default = 0 + */ + DeleteSqlStatement(TableName* tableNamePtr, WhereClause* whereClausePtr = 0); - /** @brief dtor - */ - virtual ~DeleteSqlStatement(); + /** @brief dtor + */ + virtual ~DeleteSqlStatement(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the WHERE clause - */ - virtual std::string getQueryString() const; + /** @brief get the string representation of the WHERE clause + */ + virtual std::string getQueryString() const; - /** @brief get the statement type - DML_DELETE - */ - inline virtual int getStatementType() const - { - return DML_DELETE; - } + /** @brief get the statement type - DML_DELETE + */ + inline virtual int getStatementType() const + { + return DML_DELETE; + } - WhereClause* fWhereClausePtr; + WhereClause* fWhereClausePtr; }; /** @brief The representation of a parsed @@ -344,29 +341,29 @@ public: */ class CommandSqlStatement : public SqlStatement { -public: - /** @brief ctor - * - * @param command the COMMIT or ROLLBACK string - */ - CommandSqlStatement(std::string command); + public: + /** @brief ctor + * + * @param command the COMMIT or ROLLBACK string + */ + CommandSqlStatement(std::string command); - /** @brief get the statement type - DML_COMMAND - */ - inline virtual int getStatementType() const - { - return DML_COMMAND; - } + /** @brief get the statement type - DML_COMMAND + */ + inline virtual int getStatementType() const + { + return DML_COMMAND; + } - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the COMMIT or ROLLBACK string - */ - virtual std::string getQueryString() const; + /** @brief get the COMMIT or ROLLBACK string + */ + virtual std::string getQueryString() const; - std::string fCommandText; + std::string fCommandText; }; /** @brief Stores schema, object names. @@ -374,30 +371,30 @@ public: */ class TableName { -public: - /** @brief ctor - */ - TableName(); + public: + /** @brief ctor + */ + TableName(); - /** @brief ctor - * - * @param name the table name - */ - TableName(char* name); + /** @brief ctor + * + * @param name the table name + */ + TableName(char* name); - /** @brief ctor - * - * @param schema the schema name - * @param name the table name - */ - TableName(char* shema, char* name); + /** @brief ctor + * + * @param schema the schema name + * @param name the table name + */ + TableName(char* shema, char* name); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - std::string fName; - std::string fSchema; + std::string fName; + std::string fSchema; }; /** @brief Stores a column assignment @@ -408,30 +405,31 @@ public: */ class ColumnAssignment { -public: - explicit ColumnAssignment( - std::string const& column, - std::string const& op = "=", - std::string const& expr = "") : - fColumn(column), fOperator(op), fScalarExpression(expr), - fFromCol(false), fFuncScale(0), fIsNull(false) - {}; + public: + explicit ColumnAssignment(std::string const& column, std::string const& op = "=", + std::string const& expr = "") + : fColumn(column) + , fOperator(op) + , fScalarExpression(expr) + , fFromCol(false) + , fFuncScale(0) + , fIsNull(false){}; - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of - * the column assignment - */ - std::string getColumnAssignmentString() const; + /** @brief get the string representation of + * the column assignment + */ + std::string getColumnAssignmentString() const; - std::string fColumn; - std::string fOperator; - std::string fScalarExpression; - bool fFromCol; - uint32_t fFuncScale; - bool fIsNull; + std::string fColumn; + std::string fOperator; + std::string fScalarExpression; + bool fFromCol; + uint32_t fFuncScale; + bool fIsNull; }; /** @brief Stores a value list or a query specification @@ -442,38 +440,38 @@ public: */ class ValuesOrQuery { -public: - /** @brief ctor - */ - ValuesOrQuery(); + public: + /** @brief ctor + */ + ValuesOrQuery(); - /** @brief ctor - * - * @param valuesPtr pointer to a ValuesList object - */ - ValuesOrQuery(ValuesList* valuesPtr); + /** @brief ctor + * + * @param valuesPtr pointer to a ValuesList object + */ + ValuesOrQuery(ValuesList* valuesPtr); - /** @brief ctor - * - * @param querySpecPtr pointer to a QuerySpec object - */ - ValuesOrQuery(QuerySpec* querySpecPtr); + /** @brief ctor + * + * @param querySpecPtr pointer to a QuerySpec object + */ + ValuesOrQuery(QuerySpec* querySpecPtr); - /** @brief dtor - */ - ~ValuesOrQuery(); + /** @brief dtor + */ + ~ValuesOrQuery(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string reperesentation of - * the ValuesList or the QuerySpec - */ - std::string getQueryString() const; + /** @brief get the string reperesentation of + * the ValuesList or the QuerySpec + */ + std::string getQueryString() const; - ValuesList fValuesList; - QuerySpec* fQuerySpecPtr; + ValuesList fValuesList; + QuerySpec* fQuerySpecPtr; }; /** @brief Stores a SELECT filter @@ -484,30 +482,30 @@ public: */ class SelectFilter { -public: - /** @brief ctor - */ - SelectFilter(); + public: + /** @brief ctor + */ + SelectFilter(); - /** @brief ctor - * - * @param columnListPtr pointer to a ColumnNameList object - */ - SelectFilter(ColumnNameList* columnListPtr); + /** @brief ctor + * + * @param columnListPtr pointer to a ColumnNameList object + */ + SelectFilter(ColumnNameList* columnListPtr); - /** @brief dtor - */ - ~SelectFilter(); + /** @brief dtor + */ + ~SelectFilter(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string represntation of the SELECT statement - */ - std::string getSelectString() const; + /** @brief get the string represntation of the SELECT statement + */ + std::string getSelectString() const; - ColumnNameList fColumnList; + ColumnNameList fColumnList; }; /** @brief Stores a FROM clause @@ -517,30 +515,30 @@ public: */ class FromClause { -public: - /** @brief ctor - */ - FromClause(); + public: + /** @brief ctor + */ + FromClause(); - /** @brief ctor - * - * @param tableNameList pointer to a TableNameList object - */ - FromClause(TableNameList* tableNameList); + /** @brief ctor + * + * @param tableNameList pointer to a TableNameList object + */ + FromClause(TableNameList* tableNameList); - /** @brief dtor - */ - ~FromClause(); + /** @brief dtor + */ + ~FromClause(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the FROM clause - */ - std::string getFromClauseString() const; + /** @brief get the string representation of the FROM clause + */ + std::string getFromClauseString() const; - TableNameList* fTableListPtr; + TableNameList* fTableListPtr; }; /** @brief Stores a WHERE clause @@ -550,25 +548,24 @@ public: */ class WhereClause { -public: - /** @brief ctor - */ - WhereClause(); + public: + /** @brief ctor + */ + WhereClause(); - /** @brief dtor - */ - ~WhereClause(); + /** @brief dtor + */ + ~WhereClause(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the WHERE clause - */ - std::string getWhereClauseString() const; - - SearchCondition* fSearchConditionPtr; + /** @brief get the string representation of the WHERE clause + */ + std::string getWhereClauseString() const; + SearchCondition* fSearchConditionPtr; }; /** @brief Stores a GROUP BY clause @@ -579,24 +576,24 @@ public: */ class GroupByClause { -public: - /** @brief ctor - */ - GroupByClause(); + public: + /** @brief ctor + */ + GroupByClause(); - /** @brief dtor - */ - ~GroupByClause(); + /** @brief dtor + */ + ~GroupByClause(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the GROUP BY clause - */ - std::string getGroupByClauseString() const; + /** @brief get the string representation of the GROUP BY clause + */ + std::string getGroupByClauseString() const; - ColumnNameList* fColumnNamesListPtr; + ColumnNameList* fColumnNamesListPtr; }; /** @brief Stores a HAVING clause @@ -607,24 +604,24 @@ public: */ class HavingClause { -public: - /** @brief ctor - */ - HavingClause(); + public: + /** @brief ctor + */ + HavingClause(); - /** @brief dtor - */ - ~HavingClause(); + /** @brief dtor + */ + ~HavingClause(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the HAVING clause - */ - std::string getHavingClauseString() const; + /** @brief get the string representation of the HAVING clause + */ + std::string getHavingClauseString() const; - SearchCondition* fSearchConditionPtr; + SearchCondition* fSearchConditionPtr; }; /** @brief Stores an ESCAPE sequence @@ -636,12 +633,12 @@ public: */ class Escape { -public: - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + public: + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - std::string fEscapeChar; + std::string fEscapeChar; }; /** @brief The base class representaion of a parsed SQL predicate @@ -657,30 +654,30 @@ public: */ class Predicate { -public: - /** @brief ctor - */ - Predicate(); + public: + /** @brief ctor + */ + Predicate(); - /** @brief ctor - * - * @param predicateType the PREDICATE_TYPE - */ - Predicate(PREDICATE_TYPE predicateType); + /** @brief ctor + * + * @param predicateType the PREDICATE_TYPE + */ + Predicate(PREDICATE_TYPE predicateType); - /** @brief dtor - */ - virtual ~Predicate(); + /** @brief dtor + */ + virtual ~Predicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @param get the string representation of the predicate - */ - virtual std::string getPredicateString() const; + /** @param get the string representation of the predicate + */ + virtual std::string getPredicateString() const; - PREDICATE_TYPE fPredicateType; + PREDICATE_TYPE fPredicateType; }; /** @brief The representation of a parsed @@ -692,30 +689,30 @@ public: */ class ComparisonPredicate : public Predicate { -public: - /** @brief ctor - */ - ComparisonPredicate(); + public: + /** @brief ctor + */ + ComparisonPredicate(); - /** @brief dtor - */ - ~ComparisonPredicate(); + /** @brief dtor + */ + ~ComparisonPredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the COMPARISON - * predicate - */ - virtual std::string getPredicateString() const; + /** @brief get the string representation of the COMPARISON + * predicate + */ + virtual std::string getPredicateString() const; - std::string fLHScalarExpression; - std::string fRHScalarExpression; + std::string fLHScalarExpression; + std::string fRHScalarExpression; - std::string fOperator; + std::string fOperator; - QuerySpec* fSubQuerySpec; + QuerySpec* fSubQuerySpec; }; /** @brief The representation of a parsed @@ -727,30 +724,30 @@ public: */ class BetweenPredicate : public Predicate { -public: - /** @brief ctor - */ - BetweenPredicate(); + public: + /** @brief ctor + */ + BetweenPredicate(); - /** @brief dtor - */ - ~BetweenPredicate(); + /** @brief dtor + */ + ~BetweenPredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the BETWEEN - * predicate - */ - virtual std::string getPredicateString() const; + /** @brief get the string representation of the BETWEEN + * predicate + */ + virtual std::string getPredicateString() const; - std::string fLHScalarExpression; - std::string fRH1ScalarExpression; - std::string fRH2ScalarExpression; + std::string fLHScalarExpression; + std::string fRH1ScalarExpression; + std::string fRH2ScalarExpression; - std::string fOperator1; - std::string fOperator2; + std::string fOperator1; + std::string fOperator2; }; /** @brief The representation of a parsed @@ -762,29 +759,29 @@ public: */ class LikePredicate : public Predicate { -public: - /** @brief ctor - */ - LikePredicate(); + public: + /** @brief ctor + */ + LikePredicate(); - /** @brief dtor - */ - ~LikePredicate(); + /** @brief dtor + */ + ~LikePredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the LIKE - * predicate - */ - virtual std::string getPredicateString() const; + /** @brief get the string representation of the LIKE + * predicate + */ + virtual std::string getPredicateString() const; - std::string fLHScalarExpression; - std::string fAtom; + std::string fLHScalarExpression; + std::string fAtom; - std::string fOperator; - Escape* fOptionalEscapePtr; + std::string fOperator; + Escape* fOptionalEscapePtr; }; /** @brief The representation of a parsed @@ -796,27 +793,27 @@ public: */ class NullTestPredicate : public Predicate { -public: - /** @brief ctor - */ - NullTestPredicate(); + public: + /** @brief ctor + */ + NullTestPredicate(); - /** @brief dtor - */ - ~NullTestPredicate(); + /** @brief dtor + */ + ~NullTestPredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the NULL test - * predicate - */ - std::string getPredicateString() const; + /** @brief get the string representation of the NULL test + * predicate + */ + std::string getPredicateString() const; - std::string fColumnRef; + std::string fColumnRef; - std::string fOperator; + std::string fOperator; }; /** @brief The representation of a parsed @@ -830,29 +827,29 @@ public: */ class InPredicate : public Predicate { -public: - /** @brief ctor - */ - InPredicate(); + public: + /** @brief ctor + */ + InPredicate(); - /** @brief dtor - */ - ~InPredicate(); + /** @brief dtor + */ + ~InPredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the IN - * predicate - */ - virtual std::string getPredicateString() const; + /** @brief get the string representation of the IN + * predicate + */ + virtual std::string getPredicateString() const; - std::string fScalarExpression; - std::string fOperator; + std::string fScalarExpression; + std::string fOperator; - AtomList fAtomList; - QuerySpec* fSubQuerySpecPtr; + AtomList fAtomList; + QuerySpec* fSubQuerySpecPtr; }; /** @brief The representation of a parsed @@ -863,29 +860,29 @@ public: */ class AllOrAnyPredicate : public Predicate { -public: - /** @brief ctor - */ - AllOrAnyPredicate(); + public: + /** @brief ctor + */ + AllOrAnyPredicate(); - /** @brief dtor - */ - ~AllOrAnyPredicate(); + /** @brief dtor + */ + ~AllOrAnyPredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the - * ALL or ANY predicate - */ - virtual std::string getPredicateString() const; + /** @brief get the string representation of the + * ALL or ANY predicate + */ + virtual std::string getPredicateString() const; - std::string fScalarExpression; - std::string fOperator; - std::string fAnyAllSome; + std::string fScalarExpression; + std::string fOperator; + std::string fAnyAllSome; - QuerySpec* fSubQuerySpecPtr; + QuerySpec* fSubQuerySpecPtr; }; /** @brief The representation of a parsed @@ -896,25 +893,25 @@ public: */ class ExistanceTestPredicate : public Predicate { -public: - /** @brief ctor - */ - ExistanceTestPredicate(); + public: + /** @brief ctor + */ + ExistanceTestPredicate(); - /** @brief dtor - */ - ~ExistanceTestPredicate(); + /** @brief dtor + */ + ~ExistanceTestPredicate(); - /** @brief dump to stdout - */ - virtual std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + virtual std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the EXISTS - * predicate - */ - virtual std::string getPredicateString() const; + /** @brief get the string representation of the EXISTS + * predicate + */ + virtual std::string getPredicateString() const; - QuerySpec* fSubQuerySpecPtr; + QuerySpec* fSubQuerySpecPtr; }; /** @brief The representation of a parsed @@ -929,29 +926,29 @@ public: */ class SearchCondition { -public: - /** @brief ctor - */ - SearchCondition(); + public: + /** @brief ctor + */ + SearchCondition(); - /** @brief dtor - */ - ~SearchCondition(); + /** @brief dtor + */ + ~SearchCondition(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the striong representation of the - * search condition - */ - std::string getSearchConditionString() const; + /** @brief get the striong representation of the + * search condition + */ + std::string getSearchConditionString() const; - Predicate* fPredicatePtr; - SearchCondition* fLHSearchConditionPtr; - SearchCondition* fRHSearchConditionPtr; + Predicate* fPredicatePtr; + SearchCondition* fLHSearchConditionPtr; + SearchCondition* fRHSearchConditionPtr; - std::string fOperator; + std::string fOperator; }; /** @brief The representation of a parsed @@ -965,38 +962,38 @@ public: */ class TableExpression { -public: - /** @brief ctor - */ - TableExpression(); + public: + /** @brief ctor + */ + TableExpression(); - /** @brief ctor - * - * @param fromClausePtr pointer to a FromClause object - * @param whereClausePtr pointer to a WhereClause object - * @param groupByPtr pointer to a GroupByClause object - * @param havingPtr pointer to a HavingClause object - */ - TableExpression(FromClause* fromClausePtr, WhereClause* whereClausePtr, - GroupByClause* groupByPtr, HavingClause* havingPtr); + /** @brief ctor + * + * @param fromClausePtr pointer to a FromClause object + * @param whereClausePtr pointer to a WhereClause object + * @param groupByPtr pointer to a GroupByClause object + * @param havingPtr pointer to a HavingClause object + */ + TableExpression(FromClause* fromClausePtr, WhereClause* whereClausePtr, GroupByClause* groupByPtr, + HavingClause* havingPtr); - /** @brief dtor - */ - ~TableExpression(); + /** @brief dtor + */ + ~TableExpression(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the - * table expression - */ - std::string getTableExpressionString() const; + /** @brief get the string representation of the + * table expression + */ + std::string getTableExpressionString() const; - FromClause* fFromClausePtr; - WhereClause* fWhereClausePtr; - GroupByClause* fGroupByPtr; - HavingClause* fHavingPtr; + FromClause* fFromClausePtr; + WhereClause* fWhereClausePtr; + GroupByClause* fGroupByPtr; + HavingClause* fHavingPtr; }; /** @brief The representation of a parsed @@ -1007,45 +1004,44 @@ public: */ class QuerySpec { -public: - /** @brief ctor - */ - QuerySpec(); + public: + /** @brief ctor + */ + QuerySpec(); - /** @brief ctor - * - * @param selectFilter pointer to a SelectFilter object - * @param tableExpression pointer to a TableExpression object - */ - QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression); + /** @brief ctor + * + * @param selectFilter pointer to a SelectFilter object + * @param tableExpression pointer to a TableExpression object + */ + QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression); - /** @brief ctor - * - * @param selectFilter pointer to a SelectFilter object - * @param tableExpression pointer to a TableExpression object - * @param allOrDistinct pointer to a ALL or DISTINCT string - */ - QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression, - char* allOrDistinct); + /** @brief ctor + * + * @param selectFilter pointer to a SelectFilter object + * @param tableExpression pointer to a TableExpression object + * @param allOrDistinct pointer to a ALL or DISTINCT string + */ + QuerySpec(SelectFilter* selectFilter, TableExpression* tableExpression, char* allOrDistinct); - /** @brief dtor - */ - ~QuerySpec(); + /** @brief dtor + */ + ~QuerySpec(); - /** @brief dump to stdout - */ - std::ostream& put(std::ostream& os) const; + /** @brief dump to stdout + */ + std::ostream& put(std::ostream& os) const; - /** @brief get the string representation of the - * query specification - */ - std::string getQueryString() const; + /** @brief get the string representation of the + * query specification + */ + std::string getQueryString() const; - SelectFilter* fSelectFilterPtr; - TableExpression* fTableExpressionPtr; + SelectFilter* fSelectFilterPtr; + TableExpression* fTableExpressionPtr; - std::string fOptionAllOrDistinct; + std::string fOptionAllOrDistinct; }; -} // namespace dmlpackage -#endif // DMLPKG_H +} // namespace dmlpackage +#endif // DMLPKG_H diff --git a/dbcon/dmlpackage/dmltable.cpp b/dbcon/dmlpackage/dmltable.cpp index 0d8fd5646..0022e0d9d 100644 --- a/dbcon/dmlpackage/dmltable.cpp +++ b/dbcon/dmlpackage/dmltable.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmltable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmltable.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "dmltable.h" using namespace std; @@ -27,93 +27,91 @@ using namespace std; namespace dmlpackage { DMLTable::DMLTable() -{} +{ +} DMLTable::~DMLTable() { + try + { + RowList::iterator it = fRows.begin(); - try + while (it != fRows.end()) { - RowList::iterator it = fRows.begin(); - - while (it != fRows.end()) - { - delete *it; - it++; - } + delete *it; + it++; } - catch (...) - { - cout << "failed to delete the table rows" << endl; - } - + } + catch (...) + { + cout << "failed to delete the table rows" << endl; + } } int DMLTable::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - // read the table name - bytestream >> fName; + // read the table name + bytestream >> fName; - // read the schema name - bytestream >> fSchema; + // read the schema name + bytestream >> fSchema; - messageqcpp::ByteStream::quadbyte rowNum; - bytestream >> rowNum; + messageqcpp::ByteStream::quadbyte rowNum; + bytestream >> rowNum; - for (unsigned int i = 0; i < rowNum; i++) - { - Row* aRow = new Row(); - retval = aRow->read(bytestream); - fRows.push_back(aRow); - } + for (unsigned int i = 0; i < rowNum; i++) + { + Row* aRow = new Row(); + retval = aRow->read(bytestream); + fRows.push_back(aRow); + } - return retval; + return retval; } void DMLTable::readMetaData(messageqcpp::ByteStream& bytestream) { - // read the table name - bytestream >> fName; + // read the table name + bytestream >> fName; - // read the schema name - bytestream >> fSchema; + // read the schema name + bytestream >> fSchema; } void DMLTable::readRowData(messageqcpp::ByteStream& bytestream) { - messageqcpp::ByteStream::quadbyte rowNum; - bytestream >> rowNum; + messageqcpp::ByteStream::quadbyte rowNum; + bytestream >> rowNum; - for (unsigned int i = 0; i < rowNum; i++) - { - Row* aRow = new Row(); - aRow->read(bytestream); - fRows.push_back(aRow); - } + for (unsigned int i = 0; i < rowNum; i++) + { + Row* aRow = new Row(); + aRow->read(bytestream); + fRows.push_back(aRow); + } } int DMLTable::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; - //write table name and schma name to the bytestream - bytestream << fName; - bytestream << fSchema; - messageqcpp::ByteStream::quadbyte rowNum; - rowNum = fRows.size(); - bytestream << rowNum; - //write the row list - RowList::iterator rowListPtr; - rowListPtr = fRows.begin(); + int retval = 1; + // write table name and schma name to the bytestream + bytestream << fName; + bytestream << fSchema; + messageqcpp::ByteStream::quadbyte rowNum; + rowNum = fRows.size(); + bytestream << rowNum; + // write the row list + RowList::iterator rowListPtr; + rowListPtr = fRows.begin(); - for (; rowListPtr != fRows.end(); ++rowListPtr) - { - retval = (*rowListPtr)->write(bytestream); - } + for (; rowListPtr != fRows.end(); ++rowListPtr) + { + retval = (*rowListPtr)->write(bytestream); + } - return retval; + return retval; } -} // namespace dmlpackage - +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/dmltable.h b/dbcon/dmlpackage/dmltable.h index 8847318f4..561d4fd9f 100644 --- a/dbcon/dmlpackage/dmltable.h +++ b/dbcon/dmlpackage/dmltable.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmltable.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmltable.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DMLTABLE_H @@ -34,96 +34,88 @@ namespace dmlpackage { /** @brief concrete implementation of a DMLObject - * Specifically for representing a database table - */ + * Specifically for representing a database table + */ class DMLTable : public DMLObject { + public: + /** @brief ctor + */ + DMLTable(); -public: + /** @brief dtor + */ + ~DMLTable(); - /** @brief ctor - */ - DMLTable(); + /** @brief get the schema name + */ + inline const std::string get_SchemaName() const + { + return fSchema; + } - /** @brief dtor - */ - ~DMLTable(); + /** @brief set the schema name + */ + inline void set_SchemaName(std::string& sName) + { + fSchema = sName; + } - /** @brief get the schema name - */ - inline const std::string get_SchemaName() const - { - return fSchema; - } + /** @brief get the table name + */ + inline const std::string get_TableName() const + { + return fName; + } - /** @brief set the schema name - */ - inline void set_SchemaName( std::string& sName ) - { - fSchema = sName; - } + /** @brief set the table name + */ + inline void set_TableName(std::string& tName) + { + fName = tName; + } - /** @brief get the table name - */ - inline const std::string get_TableName() const - { - return fName; - } + /** @brief get the row list + */ + inline RowList& get_RowList() + { + return fRows; + } - /** @brief set the table name - */ - inline void set_TableName( std::string& tName ) - { - fName = tName; - } + /** @brief read a DMLTable from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + int read(messageqcpp::ByteStream& bytestream); - /** @brief get the row list - */ - inline RowList& get_RowList() - { - return fRows; - } + /** @brief read a DMLTable metadata from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + void readMetaData(messageqcpp::ByteStream& bytestream); - /** @brief read a DMLTable from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - int read(messageqcpp::ByteStream& bytestream); + /** @brief read a DMLTable row data from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + void readRowData(messageqcpp::ByteStream& bytestream); + /** @brief write a DMLTable to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + int write(messageqcpp::ByteStream& bytestream); - /** @brief read a DMLTable metadata from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - void readMetaData(messageqcpp::ByteStream& bytestream); - - - /** @brief read a DMLTable row data from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - void readRowData(messageqcpp::ByteStream& bytestream); - - - /** @brief write a DMLTable to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - int write(messageqcpp::ByteStream& bytestream); - - -protected: - -private: - std::string fName; - RowList fRows; - std::string fSchema; - /*Copy and copy assignment constructor */ - DMLTable(const DMLTable&); - DMLTable& operator=(const DMLTable&); + protected: + private: + std::string fName; + RowList fRows; + std::string fSchema; + /*Copy and copy assignment constructor */ + DMLTable(const DMLTable&); + DMLTable& operator=(const DMLTable&); }; } // namespace dmlpackage -#endif //DMLTABLE_H - +#endif // DMLTABLE_H diff --git a/dbcon/dmlpackage/gramtest.cpp b/dbcon/dmlpackage/gramtest.cpp index 2df56453e..44f10042c 100644 --- a/dbcon/dmlpackage/gramtest.cpp +++ b/dbcon/dmlpackage/gramtest.cpp @@ -32,52 +32,54 @@ using namespace dmlpackage; int main(int argc, char* argv[]) { - string sqlfile; - int count; + string sqlfile; + int count; - po::options_description desc ("Allowed options"); - desc.add_options () - ("help", "produce help message") - ("bisond", /* po::value (),*/ "Have bison produce debug output") - ("count", po::value (), "number of runs") - ("sql", po::value < string > (), "sql file"); - po::variables_map vm; - po::store (po::parse_command_line (argc, argv, desc), vm); - po::notify (vm); + po::options_description desc("Allowed options"); + desc.add_options()("help", "produce help message")( + "bisond", + /* po::value (),*/ "Have bison produce debug output")("count", po::value(), + "number of runs")("sql", + po::value(), + "sql file"); + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); - if (vm.count ("sql")) - sqlfile = vm["sql"].as (); + if (vm.count("sql")) + sqlfile = vm["sql"].as(); - if (vm.count("count")) - count = vm["count"].as(); + if (vm.count("count")) + count = vm["count"].as(); - DMLFileParser parser; + DMLFileParser parser; - if (vm.count ("bisond")) - parser.setDebug(true); + if (vm.count("bisond")) + parser.setDebug(true); - parser.parse(sqlfile); + parser.parse(sqlfile); - if (parser.good()) - { - const ParseTree& ptree = parser.getParseTree(); + if (parser.good()) + { + const ParseTree& ptree = parser.getParseTree(); - cout << "Parser succeeded." << endl; - cout << ptree.fList.size() << " " << "SQL statements" << endl; - cout << ptree.fSqlText << endl; - cout << ptree; + cout << "Parser succeeded." << endl; + cout << ptree.fList.size() << " " + << "SQL statements" << endl; + cout << ptree.fSqlText << endl; + cout << ptree; - SqlStatement* statementPtr = ptree[0]; + SqlStatement* statementPtr = ptree[0]; - if (statementPtr) - cout << statementPtr->getQueryString(); + if (statementPtr) + cout << statementPtr->getQueryString(); - cout << endl; - } - else - { - cout << "Parser failed." << endl; - } + cout << endl; + } + else + { + cout << "Parser failed." << endl; + } - return parser.good() ? 0 : -1; + return parser.good() ? 0 : -1; } diff --git a/dbcon/dmlpackage/insertdmlpackage.cpp b/dbcon/dmlpackage/insertdmlpackage.cpp index bdc81a454..6d4693d4d 100644 --- a/dbcon/dmlpackage/insertdmlpackage.cpp +++ b/dbcon/dmlpackage/insertdmlpackage.cpp @@ -35,273 +35,270 @@ using namespace std; namespace dmlpackage { - InsertDMLPackage::InsertDMLPackage() -{} +{ +} -InsertDMLPackage::InsertDMLPackage( std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID ) - : CalpontDMLPackage( schemaName, tableName, dmlStatement, sessionID ) -{} +InsertDMLPackage::InsertDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID) + : CalpontDMLPackage(schemaName, tableName, dmlStatement, sessionID) +{ +} InsertDMLPackage::~InsertDMLPackage() -{} +{ +} int InsertDMLPackage::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::byte package_type = DML_INSERT; - bytestream << package_type; + messageqcpp::ByteStream::byte package_type = DML_INSERT; + bytestream << package_type; - messageqcpp::ByteStream::quadbyte session_id = fSessionID; - bytestream << session_id; + messageqcpp::ByteStream::quadbyte session_id = fSessionID; + bytestream << session_id; - bytestream << fUuid; + bytestream << fUuid; - bytestream << fDMLStatement; - bytestream << fDMLStatement; - bytestream << fSchemaName; - bytestream << fTimeZone; - bytestream << (uint8_t)fLogging; - bytestream << (uint8_t)fLogending; + bytestream << fDMLStatement; + bytestream << fDMLStatement; + bytestream << fSchemaName; + bytestream << fTimeZone; + bytestream << (uint8_t)fLogging; + bytestream << (uint8_t)fLogending; - bytestream << fTableOid; - bytestream << static_cast(fIsInsertSelect); - bytestream << static_cast(fIsBatchInsert); - bytestream << static_cast(fIsCacheInsert); - bytestream << static_cast(fIsAutocommitOn); + bytestream << fTableOid; + bytestream << static_cast(fIsInsertSelect); + bytestream << static_cast(fIsBatchInsert); + bytestream << static_cast(fIsCacheInsert); + bytestream << static_cast(fIsAutocommitOn); - if (fTable != 0) - { - retval = fTable->write(bytestream); - } + if (fTable != 0) + { + retval = fTable->write(bytestream); + } - return retval; + return retval; } int InsertDMLPackage::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::quadbyte session_id; - bytestream >> session_id; - fSessionID = session_id; - bytestream >> fUuid; + messageqcpp::ByteStream::quadbyte session_id; + bytestream >> session_id; + fSessionID = session_id; + bytestream >> fUuid; - std::string dmlStatement; - bytestream >> fDMLStatement; - bytestream >> fSQLStatement; - bytestream >> fSchemaName; - bytestream >> fTimeZone; - uint8_t logging; - bytestream >> logging; - fLogging = (logging != 0); - uint8_t logending; - bytestream >> logending; - fLogending = (logending != 0); + std::string dmlStatement; + bytestream >> fDMLStatement; + bytestream >> fSQLStatement; + bytestream >> fSchemaName; + bytestream >> fTimeZone; + uint8_t logging; + bytestream >> logging; + fLogging = (logging != 0); + uint8_t logending; + bytestream >> logending; + fLogending = (logending != 0); - bytestream >> fTableOid; - bytestream >> reinterpret_cast(fIsInsertSelect); - bytestream >> reinterpret_cast(fIsBatchInsert); - bytestream >> reinterpret_cast(fIsCacheInsert); - bytestream >> reinterpret_cast(fIsAutocommitOn); + bytestream >> fTableOid; + bytestream >> reinterpret_cast(fIsInsertSelect); + bytestream >> reinterpret_cast(fIsBatchInsert); + bytestream >> reinterpret_cast(fIsCacheInsert); + bytestream >> reinterpret_cast(fIsAutocommitOn); - fTable = new DMLTable(); - retval = fTable->read(bytestream); - return retval; + fTable = new DMLTable(); + retval = fTable->read(bytestream); + return retval; } void InsertDMLPackage::readMetaData(messageqcpp::ByteStream& bytestream) { - messageqcpp::ByteStream::quadbyte session_id; - bytestream >> session_id; - fSessionID = session_id; - bytestream >> fUuid; + messageqcpp::ByteStream::quadbyte session_id; + bytestream >> session_id; + fSessionID = session_id; + bytestream >> fUuid; - std::string dmlStatement; - bytestream >> fDMLStatement; - bytestream >> fSQLStatement; - bytestream >> fSchemaName; - bytestream >> fTimeZone; - uint8_t logging; - bytestream >> logging; - fLogging = (logging != 0); - uint8_t logending; - bytestream >> logending; - fLogending = (logending != 0); + std::string dmlStatement; + bytestream >> fDMLStatement; + bytestream >> fSQLStatement; + bytestream >> fSchemaName; + bytestream >> fTimeZone; + uint8_t logging; + bytestream >> logging; + fLogging = (logging != 0); + uint8_t logending; + bytestream >> logending; + fLogending = (logending != 0); - bytestream >> fTableOid; - bytestream >> reinterpret_cast(fIsInsertSelect); - bytestream >> reinterpret_cast(fIsBatchInsert); - bytestream >> reinterpret_cast(fIsCacheInsert); - bytestream >> reinterpret_cast(fIsAutocommitOn); + bytestream >> fTableOid; + bytestream >> reinterpret_cast(fIsInsertSelect); + bytestream >> reinterpret_cast(fIsBatchInsert); + bytestream >> reinterpret_cast(fIsCacheInsert); + bytestream >> reinterpret_cast(fIsAutocommitOn); - fTable = new DMLTable(); - fTable->readMetaData(bytestream); + fTable = new DMLTable(); + fTable->readMetaData(bytestream); } // Has to be called after InsertDMLPackage::readMetaData() void InsertDMLPackage::readRowData(messageqcpp::ByteStream& bytestream) { - fTable->readRowData(bytestream); + fTable->readRowData(bytestream); } int InsertDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows) { #ifdef DML_PACKAGE_DEBUG - // cout << "The data buffer received: " << buffer << endl; + // cout << "The data buffer received: " << buffer << endl; #endif - int retval = 1; + int retval = 1; - initializeTable(); + initializeTable(); - std::vector dataList; - typedef boost::tokenizer > tokenizer; - boost::char_separator sep(","); - tokenizer tokens(buffer, sep); + std::vector dataList; + typedef boost::tokenizer > tokenizer; + boost::char_separator sep(","); + tokenizer tokens(buffer, sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) - { - dataList.push_back(StripLeadingWhitespace(*tok_iter)); + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + dataList.push_back(StripLeadingWhitespace(*tok_iter)); + } - } + int n = 0; - int n = 0; - - for (int i = 0; i < rows; i++) - { - //get a new row - Row* aRowPtr = new Row(); - std::string colName; - std::string colValue; - - for (int j = 0; j < columns; j++) - { - //Build a column list - colName = dataList[n]; - n++; - colValue = dataList[n]; - n++; -#ifdef DML_PACKAGE_DEBUG - //cout << "The column data: " << colName << " " << colValue << endl; -#endif - DMLColumn* aColumn = new DMLColumn(colName, colValue, false); - (aRowPtr->get_ColumnList()).push_back(aColumn); - } - - //build a row list for a table - fTable->get_RowList().push_back(aRowPtr); - } - - return retval; -} - -int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues ) -{ - int retval = 1; - - initializeTable(); + for (int i = 0; i < rows; i++) + { + // get a new row Row* aRowPtr = new Row(); std::string colName; - std::vector colValList; + std::string colValue; for (int j = 0; j < columns; j++) { - //Build a column list - colName = colNameList[j]; - - colValList = tableValuesMap[j]; - - DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]); - (aRowPtr->get_ColumnList()).push_back(aColumn); + // Build a column list + colName = dataList[n]; + n++; + colValue = dataList[n]; + n++; +#ifdef DML_PACKAGE_DEBUG + // cout << "The column data: " << colName << " " << colValue << endl; +#endif + DMLColumn* aColumn = new DMLColumn(colName, colValue, false); + (aRowPtr->get_ColumnList()).push_back(aColumn); } - //build a row list for a table + // build a row list for a table fTable->get_RowList().push_back(aRowPtr); - aRowPtr = NULL; - delete aRowPtr; - return retval; + } + + return retval; +} + +int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, + int columns, int rows, NullValuesBitset& nullValues) +{ + int retval = 1; + + initializeTable(); + Row* aRowPtr = new Row(); + std::string colName; + std::vector colValList; + + for (int j = 0; j < columns; j++) + { + // Build a column list + colName = colNameList[j]; + + colValList = tableValuesMap[j]; + + DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]); + (aRowPtr->get_ColumnList()).push_back(aColumn); + } + + // build a row list for a table + fTable->get_RowList().push_back(aRowPtr); + aRowPtr = NULL; + delete aRowPtr; + return retval; } int InsertDMLPackage::buildFromSqlStatement(SqlStatement& sqlStatement) { + int retval = 1; - int retval = 1; + InsertSqlStatement& insertStmt = dynamic_cast(sqlStatement); - InsertSqlStatement& insertStmt = dynamic_cast(sqlStatement); + if (!insertStmt.fValuesOrQueryPtr) + throw runtime_error("insertStmt.fValuesOrQueryPtr == NULL"); - if (!insertStmt.fValuesOrQueryPtr) - throw runtime_error("insertStmt.fValuesOrQueryPtr == NULL"); + initializeTable(); + bool isNULL = false; - initializeTable(); - bool isNULL = false; + // only if we don't have a select statement + if (0 == insertStmt.fValuesOrQueryPtr->fQuerySpecPtr) + { + ColumnNameList columnNameList = insertStmt.fColumnList; - // only if we don't have a select statement - if (0 == insertStmt.fValuesOrQueryPtr->fQuerySpecPtr) + if (columnNameList.size()) { + ValuesList valuesList = insertStmt.fValuesOrQueryPtr->fValuesList; - ColumnNameList columnNameList = insertStmt.fColumnList; + if (columnNameList.size() != valuesList.size()) + { + throw logic_error("Column names and values count mismatch!"); + } - if (columnNameList.size()) - { + Row* aRow = new Row(); - ValuesList valuesList = insertStmt.fValuesOrQueryPtr->fValuesList; - - if (columnNameList.size() != valuesList.size()) - { - throw logic_error("Column names and values count mismatch!"); - } - - Row* aRow = new Row(); - - for (unsigned int i = 0; i < columnNameList.size(); i++) - { - DMLColumn* aColumn = new DMLColumn(columnNameList[i], valuesList[i], isNULL); - (aRow->get_ColumnList()).push_back(aColumn); - } - - fTable->get_RowList().push_back(aRow); - - } - else - { - ValuesList valuesList = insertStmt.fValuesOrQueryPtr->fValuesList; - ValuesList::const_iterator iter = valuesList.begin(); - Row* aRow = new Row(); - std::string colName = ""; - std::string colValue; - - while (iter != valuesList.end()) - { - colValue = *iter; - - if ( strcasecmp(colValue.c_str(), "NULL") == 0) - { - isNULL = true; - } - else - { - isNULL = false; - } - - DMLColumn* aColumn = new DMLColumn(colName, colValue, isNULL); - (aRow->get_ColumnList()).push_back(aColumn); - - ++iter; - } - - fTable->get_RowList().push_back(aRow); - } + for (unsigned int i = 0; i < columnNameList.size(); i++) + { + DMLColumn* aColumn = new DMLColumn(columnNameList[i], valuesList[i], isNULL); + (aRow->get_ColumnList()).push_back(aColumn); + } + fTable->get_RowList().push_back(aRow); } else { - fHasFilter = true; - fQueryString = insertStmt.getQueryString(); - } + ValuesList valuesList = insertStmt.fValuesOrQueryPtr->fValuesList; + ValuesList::const_iterator iter = valuesList.begin(); + Row* aRow = new Row(); + std::string colName = ""; + std::string colValue; - return retval; + while (iter != valuesList.end()) + { + colValue = *iter; + + if (strcasecmp(colValue.c_str(), "NULL") == 0) + { + isNULL = true; + } + else + { + isNULL = false; + } + + DMLColumn* aColumn = new DMLColumn(colName, colValue, isNULL); + (aRow->get_ColumnList()).push_back(aColumn); + + ++iter; + } + + fTable->get_RowList().push_back(aRow); + } + } + else + { + fHasFilter = true; + fQueryString = insertStmt.getQueryString(); + } + + return retval; } } // namespace dmlpackage diff --git a/dbcon/dmlpackage/insertdmlpackage.h b/dbcon/dmlpackage/insertdmlpackage.h index a2b76d8fd..e748d9032 100644 --- a/dbcon/dmlpackage/insertdmlpackage.h +++ b/dbcon/dmlpackage/insertdmlpackage.h @@ -37,89 +37,87 @@ namespace dmlpackage { /** @brief concrete implementation of a CalpontDMLPackage - * Specifically for representing INSERT DML Statements - */ + * Specifically for representing INSERT DML Statements + */ class InsertDMLPackage : public CalpontDMLPackage { + public: + /** @brief ctor + */ + EXPORT InsertDMLPackage(); -public: - /** @brief ctor - */ - EXPORT InsertDMLPackage(); + /** @brief ctor + * + * @param schemaName the schema of the table being operated on + * @param tableName the table name of the table being operated on + * @param dmlStatement the dml statement + * @param sessionID the session id + */ + EXPORT InsertDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID); - /** @brief ctor - * - * @param schemaName the schema of the table being operated on - * @param tableName the table name of the table being operated on - * @param dmlStatement the dml statement - * @param sessionID the session id - */ - EXPORT InsertDMLPackage(std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID ); + /** @brief dtor + */ + EXPORT virtual ~InsertDMLPackage(); - /** @brief dtor - */ - EXPORT virtual ~InsertDMLPackage(); + /** @brief write a InsertDMLPackage to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + EXPORT int write(messageqcpp::ByteStream& bytestream); - /** @brief write a InsertDMLPackage to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + /** @brief read InsertDMLPackage from bytestream + * + * @param bytestream the ByteStream to read from + */ + EXPORT int read(messageqcpp::ByteStream& bytestream); - /** @brief read InsertDMLPackage from bytestream - * - * @param bytestream the ByteStream to read from - */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + /** @brief read InsertDMLPackage metadata from bytestream + * + * @param bytestream the ByteStream to read from + */ + EXPORT void readMetaData(messageqcpp::ByteStream& bytestream); - /** @brief read InsertDMLPackage metadata from bytestream - * - * @param bytestream the ByteStream to read from - */ - EXPORT void readMetaData(messageqcpp::ByteStream& bytestream); + /** @brief read InsertDMLPackage row data from bytestream + * + * @param bytestream the ByteStream to read from + */ + EXPORT void readRowData(messageqcpp::ByteStream& bytestream); - /** @brief read InsertDMLPackage row data from bytestream - * - * @param bytestream the ByteStream to read from - */ - EXPORT void readRowData(messageqcpp::ByteStream& bytestream); + /** @brief build a InsertDMLPackage from a string buffer + * + * @param buffer + * @param columns the number of columns in the buffer + * @param rows the number of rows in the buffer + */ + EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); - /** @brief build a InsertDMLPackage from a string buffer - * - * @param buffer - * @param columns the number of columns in the buffer - * @param rows the number of rows in the buffer - */ - EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); + /** @brief build a InsertDMLPackage from MySQL buffer + * + * @param tableValuesMap the value list for each column in the table + * @param colNameList the column name for each column + * @param columns number of columns in the table + * @param rows number of rows to be touched + */ + EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, + int rows, NullValuesBitset& nullValues); - /** @brief build a InsertDMLPackage from MySQL buffer - * - * @param tableValuesMap the value list for each column in the table - * @param colNameList the column name for each column - * @param columns number of columns in the table - * @param rows number of rows to be touched - */ - EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues); + /** @brief build a InsertDMLPackage from a InsertSqlStatement + * + * @param sqlStmt the InsertSqlStatement + */ + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - /** @brief build a InsertDMLPackage from a InsertSqlStatement - * - * @param sqlStmt the InsertSqlStatement - */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - - /** @brief Dump the InsertDMLPackage for debugging purposes - */ - EXPORT void Dump(); - -protected: - -private: + /** @brief Dump the InsertDMLPackage for debugging purposes + */ + EXPORT void Dump(); + protected: + private: }; -} +} // namespace dmlpackage #undef EXPORT -#endif //INSERTDMLPACKAGE_H +#endif // INSERTDMLPACKAGE_H diff --git a/dbcon/dmlpackage/mysqldmlstatement.cpp b/dbcon/dmlpackage/mysqldmlstatement.cpp index 2a02464e5..e57e9da9e 100644 --- a/dbcon/dmlpackage/mysqldmlstatement.cpp +++ b/dbcon/dmlpackage/mysqldmlstatement.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: mysqldmlstatement.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: mysqldmlstatement.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "mysqldmlstatement.h" /** @@ -28,4 +28,3 @@ /** * Methods */ - diff --git a/dbcon/dmlpackage/mysqldmlstatement.h b/dbcon/dmlpackage/mysqldmlstatement.h index 76340798b..8890a1ad0 100644 --- a/dbcon/dmlpackage/mysqldmlstatement.h +++ b/dbcon/dmlpackage/mysqldmlstatement.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: mysqldmlstatement.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: mysqldmlstatement.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef MYSQLDMLSTATEMENT_H #define MYSQLDMLSTATEMENT_H @@ -33,15 +33,13 @@ namespace dmlpackage */ class MySQLDMLStatement : public VendorDMLStatement { + public: + MySQLDMLStatement() : VendorDMLStatement("", 0) + { + } -public: - MySQLDMLStatement() : VendorDMLStatement("", 0) {} - -protected: - -private: - + protected: + private: }; -} -#endif //MYSQLDMLSTATEMENT_H - +} // namespace dmlpackage +#endif // MYSQLDMLSTATEMENT_H diff --git a/dbcon/dmlpackage/oracledmlstatement.cpp b/dbcon/dmlpackage/oracledmlstatement.cpp index b79ccad9d..d21865578 100644 --- a/dbcon/dmlpackage/oracledmlstatement.cpp +++ b/dbcon/dmlpackage/oracledmlstatement.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: oracledmlstatement.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: oracledmlstatement.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "oracledmlstatement.h" /** @@ -28,5 +28,3 @@ /** * Methods */ - - diff --git a/dbcon/dmlpackage/oracledmlstatement.h b/dbcon/dmlpackage/oracledmlstatement.h index e5bc248e3..d205bce1a 100644 --- a/dbcon/dmlpackage/oracledmlstatement.h +++ b/dbcon/dmlpackage/oracledmlstatement.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: oracledmlstatement.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: oracledmlstatement.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef ORACLEDMLSTATEMENT_H @@ -34,15 +34,13 @@ namespace dmlpackage */ class OracleDMLStatement : public VendorDMLStatement { + public: + OracleDMLStatement() : VendorDMLStatement("", 0) + { + } -public: - OracleDMLStatement() : VendorDMLStatement("", 0) {} - -protected: - -private: - + protected: + private: }; -} -#endif //ORACLEDMLSTATEMENT_H - +} // namespace dmlpackage +#endif // ORACLEDMLSTATEMENT_H diff --git a/dbcon/dmlpackage/row.cpp b/dbcon/dmlpackage/row.cpp index 45a02c37d..1f62041bf 100644 --- a/dbcon/dmlpackage/row.cpp +++ b/dbcon/dmlpackage/row.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: row.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: row.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #define DMLPKGROW_DLLEXPORT @@ -28,83 +28,78 @@ namespace dmlpackage { - -Row::Row() - : fRowID(std::numeric_limits::max()) -{} +Row::Row() : fRowID(std::numeric_limits::max()) +{ +} Row::~Row() { - for ( unsigned int i = 0; i < fColumnList.size(); i++) - { - delete fColumnList[i]; - } + for (unsigned int i = 0; i < fColumnList.size(); i++) + { + delete fColumnList[i]; + } - fColumnList.clear(); + fColumnList.clear(); } Row::Row(const Row& row) { - for (unsigned int i = 0; i < row.fColumnList.size(); i++) - { - const DMLColumn* aColumn = row.get_ColumnAt(i); - DMLColumn* newColumn = new DMLColumn(aColumn->get_Name(), aColumn->get_Data()); - fColumnList.push_back(newColumn); - } + for (unsigned int i = 0; i < row.fColumnList.size(); i++) + { + const DMLColumn* aColumn = row.get_ColumnAt(i); + DMLColumn* newColumn = new DMLColumn(aColumn->get_Name(), aColumn->get_Data()); + fColumnList.push_back(newColumn); + } - fRowID = row.fRowID; + fRowID = row.fRowID; } int Row::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; - messageqcpp::ByteStream::octbyte rowID; - bytestream >> rowID; - set_RowID(rowID); - messageqcpp::ByteStream::quadbyte col_count; - bytestream >> col_count; + int retval = 1; + messageqcpp::ByteStream::octbyte rowID; + bytestream >> rowID; + set_RowID(rowID); + messageqcpp::ByteStream::quadbyte col_count; + bytestream >> col_count; - for (unsigned int i = 0; i < col_count; i++) - { - DMLColumn* aColumn = new DMLColumn(); - retval = aColumn->read(bytestream); - fColumnList.push_back(aColumn); - } + for (unsigned int i = 0; i < col_count; i++) + { + DMLColumn* aColumn = new DMLColumn(); + retval = aColumn->read(bytestream); + fColumnList.push_back(aColumn); + } - return retval; + return retval; } - int Row::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; - messageqcpp::ByteStream::octbyte rowID = fRowID; - bytestream << rowID; - ColumnList::iterator colListPtr; - colListPtr = fColumnList.begin(); - messageqcpp::ByteStream::quadbyte col_count = fColumnList.size(); - bytestream << col_count; + int retval = 1; + messageqcpp::ByteStream::octbyte rowID = fRowID; + bytestream << rowID; + ColumnList::iterator colListPtr; + colListPtr = fColumnList.begin(); + messageqcpp::ByteStream::quadbyte col_count = fColumnList.size(); + bytestream << col_count; - for (; colListPtr != fColumnList.end(); ++colListPtr) - { - retval = (*colListPtr)->write(bytestream); - } + for (; colListPtr != fColumnList.end(); ++colListPtr) + { + retval = (*colListPtr)->write(bytestream); + } - return retval; + return retval; } -const DMLColumn* Row::get_ColumnAt( unsigned int index ) const +const DMLColumn* Row::get_ColumnAt(unsigned int index) const { - const DMLColumn* columnPtr = 0; + const DMLColumn* columnPtr = 0; - if ( index < fColumnList.size() ) - { - columnPtr = fColumnList[index]; - } + if (index < fColumnList.size()) + { + columnPtr = fColumnList[index]; + } - return columnPtr; + return columnPtr; } -} // namespace dmlpackage - - - +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/row.h b/dbcon/dmlpackage/row.h index 43c9a99aa..0b49a6c41 100644 --- a/dbcon/dmlpackage/row.h +++ b/dbcon/dmlpackage/row.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: row.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: row.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef ROW_H @@ -39,85 +39,81 @@ namespace dmlpackage { /** @brief concrete implementation of a DMLObject - * Specifically for representing a table row - */ + * Specifically for representing a table row + */ class Row : public DMLObject { -public: + public: + /** @brief ctor + */ + EXPORT Row(); - /** @brief ctor - */ - EXPORT Row(); + /** @brief dtor + */ + EXPORT ~Row(); - /** @brief dtor - */ - EXPORT ~Row(); + /** @brief copy constructor + */ + EXPORT Row(const Row&); - /** @brief copy constructor - */ - EXPORT Row(const Row&); + /** @brief read a Row from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + EXPORT int read(messageqcpp::ByteStream& bytestream); - /** @brief read a Row from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + /** @brief write a Row to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + EXPORT int write(messageqcpp::ByteStream& bytestream); - /** @brief write a Row to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + /** @brief get the list of columns in the row + */ + inline ColumnList& get_ColumnList() + { + return fColumnList; + } - /** @brief get the list of columns in the row - */ - inline ColumnList& get_ColumnList() - { - return fColumnList; - } + /** @brief get the row id + */ + inline WriteEngine::RID get_RowID() const + { + return fRowID; + } - /** @brief get the row id - */ - inline WriteEngine::RID get_RowID() const - { - return fRowID; - } + /** @brief set the row id + */ + inline void set_RowID(WriteEngine::RID rowId) + { + fRowID = rowId; + } - /** @brief set the row id - */ - inline void set_RowID(WriteEngine::RID rowId) - { - fRowID = rowId; - } + /** @brief get the number of columns + */ + inline unsigned int get_NumberOfColumns() const + { + return static_cast(fColumnList.size()); + } - /** @brief get the number of columns - */ - inline unsigned int get_NumberOfColumns() const - { - return static_cast(fColumnList.size()); - } - - /** @brief get the column at the specified index - * - * @param index the index of the column to get - */ - EXPORT const DMLColumn* get_ColumnAt( unsigned int index ) const; - -protected: - -private: - WriteEngine::RID fRowID; - ColumnList fColumnList; - Row& operator=(const Row&); + /** @brief get the column at the specified index + * + * @param index the index of the column to get + */ + EXPORT const DMLColumn* get_ColumnAt(unsigned int index) const; + protected: + private: + WriteEngine::RID fRowID; + ColumnList fColumnList; + Row& operator=(const Row&); }; /** @brief a vector of Rows - */ -typedef std::vectorRowList; -} + */ +typedef std::vector RowList; +} // namespace dmlpackage #undef EXPORT -#endif //ROW_H - +#endif // ROW_H diff --git a/dbcon/dmlpackage/tdriver.cpp b/dbcon/dmlpackage/tdriver.cpp index cff229e17..a07098091 100644 --- a/dbcon/dmlpackage/tdriver.cpp +++ b/dbcon/dmlpackage/tdriver.cpp @@ -43,351 +43,345 @@ using namespace messageqcpp; bool parse_file(char* fileName) { - DMLFileParser parser; - parser.parse(fileName); - bool good = parser.good(); + DMLFileParser parser; + parser.parse(fileName); + bool good = parser.good(); - if (good) - { - const ParseTree& ptree = parser.getParseTree(); + if (good) + { + const ParseTree& ptree = parser.getParseTree(); - cout << "Parser succeeded." << endl; - cout << ptree.fList.size() << " " << "SQL statements" << endl; - cout << ptree.fSqlText << endl; - cout << ptree; + cout << "Parser succeeded." << endl; + cout << ptree.fList.size() << " " + << "SQL statements" << endl; + cout << ptree.fSqlText << endl; + cout << ptree; - SqlStatement* statementPtr = ptree[0]; + SqlStatement* statementPtr = ptree[0]; - if (statementPtr) - cout << statementPtr->getQueryString(); + if (statementPtr) + cout << statementPtr->getQueryString(); - cout << endl; - } + cout << endl; + } - - return good; + return good; } class DMLParserTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE( DMLParserTest ); - CPPUNIT_TEST( test_i01 ); - CPPUNIT_TEST( test_i02 ); - CPPUNIT_TEST( test_i03 ); - CPPUNIT_TEST( test_i04 ); - CPPUNIT_TEST( test_u01 ); - CPPUNIT_TEST( test_u02 ); - CPPUNIT_TEST( test_d01 ); - CPPUNIT_TEST( test_d02 ); - CPPUNIT_TEST( test_d03 ); - CPPUNIT_TEST( test_d04 ); - CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST_SUITE(DMLParserTest); + CPPUNIT_TEST(test_i01); + CPPUNIT_TEST(test_i02); + CPPUNIT_TEST(test_i03); + CPPUNIT_TEST(test_i04); + CPPUNIT_TEST(test_u01); + CPPUNIT_TEST(test_u02); + CPPUNIT_TEST(test_d01); + CPPUNIT_TEST(test_d02); + CPPUNIT_TEST(test_d03); + CPPUNIT_TEST(test_d04); + CPPUNIT_TEST_SUITE_END(); -private: + private: + public: + void setUp() + { + } -public: - void setUp() {} + void tearDown() + { + } - void tearDown() {} + void test_i01() + { + CPPUNIT_ASSERT(parse_file("sql/i01.sql")); + } - void test_i01() - { - CPPUNIT_ASSERT(parse_file("sql/i01.sql")); - } + void test_i02() + { + CPPUNIT_ASSERT(parse_file("sql/i02.sql")); + } - void test_i02() - { - CPPUNIT_ASSERT(parse_file("sql/i02.sql")); - } + void test_i03() + { + CPPUNIT_ASSERT(parse_file("sql/i03.sql")); + } - void test_i03() - { - CPPUNIT_ASSERT(parse_file("sql/i03.sql")); - } + void test_i04() + { + CPPUNIT_ASSERT(parse_file("sql/i04.sql")); + } - void test_i04() - { - CPPUNIT_ASSERT(parse_file("sql/i04.sql")); - } + void test_u01() + { + CPPUNIT_ASSERT(parse_file("sql/u01.sql")); + } - void test_u01() - { - CPPUNIT_ASSERT(parse_file("sql/u01.sql")); - } + void test_u02() + { + CPPUNIT_ASSERT(parse_file("sql/u02.sql")); + } - void test_u02() - { - CPPUNIT_ASSERT(parse_file("sql/u02.sql")); - } + void test_d01() + { + CPPUNIT_ASSERT(parse_file("sql/d01.sql")); + } - void test_d01() - { - CPPUNIT_ASSERT(parse_file("sql/d01.sql")); - } + void test_d02() + { + CPPUNIT_ASSERT(parse_file("sql/d02.sql")); + } - void test_d02() - { - CPPUNIT_ASSERT(parse_file("sql/d02.sql")); - } + void test_d03() + { + CPPUNIT_ASSERT(parse_file("sql/d03.sql")); + } - void test_d03() - { - CPPUNIT_ASSERT(parse_file("sql/d03.sql")); - } - - void test_d04() - { - CPPUNIT_ASSERT(parse_file("sql/d04.sql")); - } + void test_d04() + { + CPPUNIT_ASSERT(parse_file("sql/d04.sql")); + } }; class DMLTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DMLTest); + // CPPUNIT_TEST( test_direct_insert ); + // CPPUNIT_TEST( test_query_insert ); + CPPUNIT_TEST(test_direct_update); + // CPPUNIT_TEST( test_query_update ); + // CPPUNIT_TEST( test_delete_all ); + // CPPUNIT_TEST( test_delete_query ); + // CPPUNIT_TEST( test_commit ); + // CPPUNIT_TEST( test_rollback ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE( DMLTest ); - // CPPUNIT_TEST( test_direct_insert ); - // CPPUNIT_TEST( test_query_insert ); - CPPUNIT_TEST( test_direct_update ); - // CPPUNIT_TEST( test_query_update ); - // CPPUNIT_TEST( test_delete_all ); - // CPPUNIT_TEST( test_delete_query ); - // CPPUNIT_TEST( test_commit ); - // CPPUNIT_TEST( test_rollback ); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: - void setUp() {} + void tearDown() + { + } - void tearDown() {} + void test_direct_insert() + { + ByteStream bytestream; + std::string dmlStatement = "INSERT INTO tpch.supplier (supplier_id, supplier_name) VALUES(24553, 'IBM');"; + cout << dmlStatement << endl; - void test_direct_insert() + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_insert_object(bytestream); + } + + void test_query_insert() + { + ByteStream bytestream; + std::string dmlStatement = + "INSERT INTO supplier (supplier_id, supplier_name) SELECT account_no, name FROM customers WHERE city " + "= 'Newark';"; + cout << dmlStatement << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + if (pDMLPackage->HasFilter()) { - - ByteStream bytestream; - std::string dmlStatement = "INSERT INTO tpch.supplier (supplier_id, supplier_name) VALUES(24553, 'IBM');"; - cout << dmlStatement << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_insert_object(bytestream); - + cout << "This INSERT statement has a filter:" << endl; + cout << pDMLPackage->get_QueryString() << endl; } - void test_query_insert() + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_insert_object(bytestream); + } + + void write_DML_object(ByteStream& bs, CalpontDMLPackage* pDMLPackage) + { + pDMLPackage->write(bs); + } + + void read_insert_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; + + CPPUNIT_ASSERT(DML_INSERT == package_type); + + InsertDMLPackage* pObject = new InsertDMLPackage(); + + pObject->read(bs); + + delete pObject; + } + + void test_delete_all() + { + ByteStream bytestream; + std::string dmlStatement = "DELETE FROM tpch.part;"; + + cout << dmlStatement << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_delete_object(bytestream); + } + + void test_delete_query() + { + ByteStream bytestream; + std::string dmlStatement = "DELETE FROM tpch.supplier WHERE supplier_name = 'IBM';"; + + cout << dmlStatement << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + if (pDMLPackage->HasFilter()) { - ByteStream bytestream; - std::string dmlStatement = "INSERT INTO supplier (supplier_id, supplier_name) SELECT account_no, name FROM customers WHERE city = 'Newark';"; - cout << dmlStatement << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - if ( pDMLPackage->HasFilter() ) - { - cout << "This INSERT statement has a filter:" << endl; - cout << pDMLPackage->get_QueryString() << endl; - } - - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_insert_object(bytestream); - + cout << "This DELETE statement has a filter:" << endl; + cout << pDMLPackage->get_QueryString() << endl; } - void write_DML_object( ByteStream& bs, CalpontDMLPackage* pDMLPackage ) - { + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_delete_object(bytestream); + } - pDMLPackage->write( bs ); + void read_delete_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; + + CPPUNIT_ASSERT(DML_DELETE == package_type); + + DeleteDMLPackage* pObject = new DeleteDMLPackage(); + + pObject->read(bs); + + delete pObject; + } + + void test_direct_update() + { + ByteStream bytestream; + std::string dmlStatement = "UPDATE tpch.part SET p_partno = 1, p_name = 'joe' where p_partno=2;"; + cout << dmlStatement << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_update_object(bytestream); + } + + void test_query_update() + { + ByteStream bytestream; + std::string dmlStatement = + "UPDATE tpch.supplier SET supplier_name='joe',supplier_state='ca' WHERE EXISTS ( SELECT " + "customer.name FROM customers WHERE customers.customer_id = supplier.supplier_id);"; + cout << dmlStatement << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + if (pDMLPackage->HasFilter()) + { + cout << "This UPDATE statement has a filter:" << endl; + cout << pDMLPackage->get_QueryString() << endl; } - void read_insert_object( ByteStream& bs ) - { + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_update_object(bytestream); + } - ByteStream::byte package_type; - bs >> package_type; + void read_update_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - CPPUNIT_ASSERT( DML_INSERT == package_type ); + CPPUNIT_ASSERT(DML_UPDATE == package_type); - InsertDMLPackage* pObject = new InsertDMLPackage(); + UpdateDMLPackage* pObject = new UpdateDMLPackage(); - pObject->read( bs ); + pObject->read(bs); - delete pObject; + delete pObject; + } - } + void test_commit() + { + ByteStream bytestream; + std::string dmlStatement = "COMMIT;"; + cout << dmlStatement << endl; - void test_delete_all() - { + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_command_object(bytestream); + } - ByteStream bytestream; - std::string dmlStatement = "DELETE FROM tpch.part;"; + void test_rollback() + { + ByteStream bytestream; + std::string dmlStatement = "ROLLBACK;"; + cout << dmlStatement << endl; - cout << dmlStatement << endl; + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + write_DML_object(bytestream, pDMLPackage); + delete pDMLPackage; + read_command_object(bytestream); + } - VendorDMLStatement dmlStmt(dmlStatement, 1); + void read_command_object(ByteStream& bs) + { + ByteStream::byte package_type; + bs >> package_type; - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_delete_object(bytestream); - } + CPPUNIT_ASSERT(DML_COMMAND == package_type); - void test_delete_query() - { - ByteStream bytestream; - std::string dmlStatement = "DELETE FROM tpch.supplier WHERE supplier_name = 'IBM';"; + CommandDMLPackage* pObject = new CommandDMLPackage(); - cout << dmlStatement << endl; + pObject->read(bs); - VendorDMLStatement dmlStmt(dmlStatement, 1); - - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - if (pDMLPackage->HasFilter()) - { - cout << "This DELETE statement has a filter:" << endl; - cout << pDMLPackage->get_QueryString() << endl; - } - - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_delete_object(bytestream); - - } - - void read_delete_object( ByteStream& bs ) - { - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_DELETE == package_type ); - - DeleteDMLPackage* pObject = new DeleteDMLPackage(); - - pObject->read( bs ); - - delete pObject; - - } - - void test_direct_update() - { - ByteStream bytestream; - std::string dmlStatement = "UPDATE tpch.part SET p_partno = 1, p_name = 'joe' where p_partno=2;"; - cout << dmlStatement << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_update_object(bytestream); - } - - void test_query_update() - { - - ByteStream bytestream; - std::string dmlStatement = "UPDATE tpch.supplier SET supplier_name='joe',supplier_state='ca' WHERE EXISTS ( SELECT customer.name FROM customers WHERE customers.customer_id = supplier.supplier_id);"; - cout << dmlStatement << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - if (pDMLPackage->HasFilter()) - { - cout << "This UPDATE statement has a filter:" << endl; - cout << pDMLPackage->get_QueryString() << endl; - } - - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_update_object(bytestream); - } - - void read_update_object( ByteStream& bs ) - { - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_UPDATE == package_type ); - - UpdateDMLPackage* pObject = new UpdateDMLPackage(); - - pObject->read( bs ); - - delete pObject; - - } - - void test_commit() - { - ByteStream bytestream; - std::string dmlStatement = "COMMIT;"; - cout << dmlStatement << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_command_object(bytestream); - - } - - void test_rollback() - { - ByteStream bytestream; - std::string dmlStatement = "ROLLBACK;"; - cout << dmlStatement << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - write_DML_object(bytestream, pDMLPackage); - delete pDMLPackage; - read_command_object(bytestream); - } - - void read_command_object( ByteStream& bs ) - { - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_COMMAND == package_type ); - - CommandDMLPackage* pObject = new CommandDMLPackage(); - - pObject->read( bs ); - - delete pObject; - - } + delete pObject; + } }; -//CPPUNIT_TEST_SUITE_REGISTRATION( DMLParserTest ); -CPPUNIT_TEST_SUITE_REGISTRATION( DMLTest ); +// CPPUNIT_TEST_SUITE_REGISTRATION( DMLParserTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DMLTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/dmlpackage/updatedmlpackage.cpp b/dbcon/dmlpackage/updatedmlpackage.cpp index 23b42a8dd..2400e6b5a 100644 --- a/dbcon/dmlpackage/updatedmlpackage.cpp +++ b/dbcon/dmlpackage/updatedmlpackage.cpp @@ -32,129 +32,130 @@ using namespace std; namespace dmlpackage { - UpdateDMLPackage::UpdateDMLPackage() -{} +{ +} -UpdateDMLPackage::UpdateDMLPackage(std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID) - : CalpontDMLPackage( schemaName, tableName, dmlStatement, sessionID) -{} +UpdateDMLPackage::UpdateDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID) + : CalpontDMLPackage(schemaName, tableName, dmlStatement, sessionID) +{ +} UpdateDMLPackage::~UpdateDMLPackage() -{} +{ +} int UpdateDMLPackage::write(messageqcpp::ByteStream& bytestream) { - int retval = 1; - messageqcpp::ByteStream::byte package_type = DML_UPDATE; - bytestream << package_type; + int retval = 1; + messageqcpp::ByteStream::byte package_type = DML_UPDATE; + bytestream << package_type; - messageqcpp::ByteStream::quadbyte session_id = fSessionID; - bytestream << session_id; - /* - if(fPlan != 0) - fHasFilter = true; - else - fHasFilter = false; - */ - messageqcpp::ByteStream::quadbyte hasFilter = fHasFilter; - bytestream << hasFilter; + messageqcpp::ByteStream::quadbyte session_id = fSessionID; + bytestream << session_id; + /* + if(fPlan != 0) + fHasFilter = true; + else + fHasFilter = false; + */ + messageqcpp::ByteStream::quadbyte hasFilter = fHasFilter; + bytestream << hasFilter; - bytestream << fUuid; + bytestream << fUuid; - bytestream << fDMLStatement; - bytestream << fSQLStatement; - bytestream << fSchemaName; - bytestream << fTimeZone; - bytestream << (uint8_t)fIsFromCol; + bytestream << fDMLStatement; + bytestream << fSQLStatement; + bytestream << fSchemaName; + bytestream << fTimeZone; + bytestream << (uint8_t)fIsFromCol; - if (fTable != 0) - { - retval = fTable->write(bytestream); - } + if (fTable != 0) + { + retval = fTable->write(bytestream); + } - if (fHasFilter) - { - bytestream += *(fPlan.get()); - } + if (fHasFilter) + { + bytestream += *(fPlan.get()); + } - return retval; + return retval; } /** * */ int UpdateDMLPackage::read(messageqcpp::ByteStream& bytestream) { - int retval = 1; + int retval = 1; - messageqcpp::ByteStream::quadbyte session_id; - messageqcpp::ByteStream::quadbyte hasFilter; + messageqcpp::ByteStream::quadbyte session_id; + messageqcpp::ByteStream::quadbyte hasFilter; - bytestream >> session_id; - fSessionID = session_id; + bytestream >> session_id; + fSessionID = session_id; - bytestream >> hasFilter; - fHasFilter = (hasFilter != 0); + bytestream >> hasFilter; + fHasFilter = (hasFilter != 0); - bytestream >> fUuid; + bytestream >> fUuid; - std::string dmlStatement; - bytestream >> fDMLStatement; - bytestream >> fSQLStatement; - bytestream >> fSchemaName; - bytestream >> fTimeZone; - uint8_t isFromCol; - bytestream >> isFromCol; - fIsFromCol = (isFromCol != 0); - fTable = new DMLTable(); - retval = fTable->read(bytestream); + std::string dmlStatement; + bytestream >> fDMLStatement; + bytestream >> fSQLStatement; + bytestream >> fSchemaName; + bytestream >> fTimeZone; + uint8_t isFromCol; + bytestream >> isFromCol; + fIsFromCol = (isFromCol != 0); + fTable = new DMLTable(); + retval = fTable->read(bytestream); - if (fHasFilter) - { - fPlan.reset(new messageqcpp::ByteStream(bytestream)); - } + if (fHasFilter) + { + fPlan.reset(new messageqcpp::ByteStream(bytestream)); + } - return retval; + return retval; } int UpdateDMLPackage::buildFromSqlStatement(SqlStatement& sqlStatement) { + int retval = 1; - int retval = 1; + UpdateSqlStatement& updateStmt = dynamic_cast(sqlStatement); - UpdateSqlStatement& updateStmt = dynamic_cast(sqlStatement); + if (!updateStmt.fColAssignmentListPtr) + throw runtime_error("updateStmt.fColAssignmentPtr == NULL"); - if (!updateStmt.fColAssignmentListPtr) - throw runtime_error("updateStmt.fColAssignmentPtr == NULL"); + initializeTable(); - initializeTable(); + // Since there is no filter, all rows are updated - // Since there is no filter, all rows are updated + // Push one row always and let the filter happen on the proc side. + Row* rowPtr = new Row(); + ColumnAssignmentList::const_iterator iter = updateStmt.fColAssignmentListPtr->begin(); - // Push one row always and let the filter happen on the proc side. - Row* rowPtr = new Row(); - ColumnAssignmentList::const_iterator iter = updateStmt.fColAssignmentListPtr->begin(); + while (iter != updateStmt.fColAssignmentListPtr->end()) + { + ColumnAssignment* colaPtr = *iter; + DMLColumn* colPtr = new DMLColumn(colaPtr->fColumn, colaPtr->fScalarExpression); + rowPtr->get_ColumnList().push_back(colPtr); - while (iter != updateStmt.fColAssignmentListPtr->end()) - { - ColumnAssignment* colaPtr = *iter; - DMLColumn* colPtr = new DMLColumn(colaPtr->fColumn, colaPtr->fScalarExpression); - rowPtr->get_ColumnList().push_back(colPtr); + ++iter; + } - ++iter; - } + fTable->get_RowList().push_back(rowPtr); - fTable->get_RowList().push_back(rowPtr); + if (0 != updateStmt.fWhereClausePtr) + { + // We need to filter the rows...get row ids + fHasFilter = true; + fQueryString = updateStmt.getQueryString(); + } - if (0 != updateStmt.fWhereClausePtr) - { - // We need to filter the rows...get row ids - fHasFilter = true; - fQueryString = updateStmt.getQueryString(); - } - - return retval; + return retval; } /** @@ -163,109 +164,108 @@ int UpdateDMLPackage::buildFromSqlStatement(SqlStatement& sqlStatement) int UpdateDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows) { #ifdef DML_PACKAGE_DEBUG - //cout << "The data buffer received: " << buffer << endl; + // cout << "The data buffer received: " << buffer << endl; #endif - int retval = 1; + int retval = 1; - initializeTable(); + initializeTable(); - std::vector dataList; - typedef boost::tokenizer > tokenizer; - boost::char_separator sep(":,"); - tokenizer tokens(buffer, sep); + std::vector dataList; + typedef boost::tokenizer > tokenizer; + boost::char_separator sep(":,"); + tokenizer tokens(buffer, sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) - { - dataList.push_back(StripLeadingWhitespace(*tok_iter)); + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + dataList.push_back(StripLeadingWhitespace(*tok_iter)); + } - } + int n = 0; - int n = 0; - - for (int i = 0; i < rows; i++) - { - //get a new row - Row* aRowPtr = new Row(); - std::string colName; - std::string colValue; - //get row ID from the buffer - std::string rowid = dataList[n++]; - aRowPtr->set_RowID(atoll(rowid.c_str())); -#ifdef DML_PACKAGE_DEBUG - - //cout << "The row ID is " << rowid << endl; -#endif - - for (int j = 0; j < columns; j++) - { - //Build a column list - colName = dataList[n++]; - colValue = dataList[n++]; -#ifdef DML_PACKAGE_DEBUG - - //cout << "The column data: " << colName << " " << colValue << endl; -#endif - - DMLColumn* aColumn = new DMLColumn(colName, colValue); - (aRowPtr->get_ColumnList()).push_back(aColumn); - } - - //build a row list for a table - fTable->get_RowList().push_back(aRowPtr); - } - - return retval; -} -int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues ) -{ - int retval = 1; - - initializeTable(); + for (int i = 0; i < rows; i++) + { + // get a new row Row* aRowPtr = new Row(); std::string colName; - std::vector colValList; + std::string colValue; + // get row ID from the buffer + std::string rowid = dataList[n++]; + aRowPtr->set_RowID(atoll(rowid.c_str())); +#ifdef DML_PACKAGE_DEBUG + + // cout << "The row ID is " << rowid << endl; +#endif for (int j = 0; j < columns; j++) { - //Build a column list - colName = colNameList[j]; + // Build a column list + colName = dataList[n++]; + colValue = dataList[n++]; +#ifdef DML_PACKAGE_DEBUG - colValList = tableValuesMap[j]; + // cout << "The column data: " << colName << " " << colValue << endl; +#endif - DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]); - (aRowPtr->get_ColumnList()).push_back(aColumn); + DMLColumn* aColumn = new DMLColumn(colName, colValue); + (aRowPtr->get_ColumnList()).push_back(aColumn); } - //build a row list for a table + // build a row list for a table fTable->get_RowList().push_back(aRowPtr); - return retval; -} + } -void UpdateDMLPackage::buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt) + return retval; +} +int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, + int columns, int rows, NullValuesBitset& nullValues) { + int retval = 1; - if (!updateStmt.fColAssignmentListPtr) - throw runtime_error("updateStmt.fColAssignmentPtr == NULL"); + initializeTable(); + Row* aRowPtr = new Row(); + std::string colName; + std::vector colValList; - initializeTable(); + for (int j = 0; j < columns; j++) + { + // Build a column list + colName = colNameList[j]; - // Since there is no filter, all rows are updated + colValList = tableValuesMap[j]; - // Push one row always and let the filter happen on the proc side. - Row* rowPtr = new Row(); - ColumnAssignmentList::const_iterator iter = updateStmt.fColAssignmentListPtr->begin(); + DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]); + (aRowPtr->get_ColumnList()).push_back(aColumn); + } - while (iter != updateStmt.fColAssignmentListPtr->end()) - { - ColumnAssignment* colaPtr = *iter; - DMLColumn* colPtr = new DMLColumn(colaPtr->fColumn, colaPtr->fScalarExpression, colaPtr->fFromCol, colaPtr->fFuncScale, - colaPtr->fIsNull); - rowPtr->get_ColumnList().push_back(colPtr); - - ++iter; - } - - fTable->get_RowList().push_back(rowPtr); + // build a row list for a table + fTable->get_RowList().push_back(aRowPtr); + return retval; } -} // namespace dmlpackage + +void UpdateDMLPackage::buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt) +{ + if (!updateStmt.fColAssignmentListPtr) + throw runtime_error("updateStmt.fColAssignmentPtr == NULL"); + + initializeTable(); + + // Since there is no filter, all rows are updated + + // Push one row always and let the filter happen on the proc side. + Row* rowPtr = new Row(); + ColumnAssignmentList::const_iterator iter = updateStmt.fColAssignmentListPtr->begin(); + + while (iter != updateStmt.fColAssignmentListPtr->end()) + { + ColumnAssignment* colaPtr = *iter; + DMLColumn* colPtr = new DMLColumn(colaPtr->fColumn, colaPtr->fScalarExpression, colaPtr->fFromCol, + colaPtr->fFuncScale, colaPtr->fIsNull); + rowPtr->get_ColumnList().push_back(colPtr); + + ++iter; + } + + fTable->get_RowList().push_back(rowPtr); +} +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/updatedmlpackage.h b/dbcon/dmlpackage/updatedmlpackage.h index f5aac0097..252c4befa 100644 --- a/dbcon/dmlpackage/updatedmlpackage.h +++ b/dbcon/dmlpackage/updatedmlpackage.h @@ -41,68 +41,65 @@ namespace dmlpackage */ class UpdateDMLPackage : public CalpontDMLPackage { + public: + /** @brief ctor + */ + EXPORT UpdateDMLPackage(); -public: - /** @brief ctor - */ - EXPORT UpdateDMLPackage(); + /** @brief ctor + * + * @param schemaName the schema of the table being operated on + * @param tableName the name of the table being operated on + * @param dmlStatement the dml statement + * @param sessionID the session ID + */ + EXPORT UpdateDMLPackage(std::string schemaName, std::string tableName, std::string dmlStatement, + int sessionID); - /** @brief ctor - * - * @param schemaName the schema of the table being operated on - * @param tableName the name of the table being operated on - * @param dmlStatement the dml statement - * @param sessionID the session ID - */ - EXPORT UpdateDMLPackage( std::string schemaName, std::string tableName, - std::string dmlStatement, int sessionID ); + /** @brief dtor + */ + EXPORT virtual ~UpdateDMLPackage(); - /** @brief dtor - */ - EXPORT virtual ~UpdateDMLPackage(); + /** @brief write a UpdateDMLPackage to a ByteStream + * + * @param bytestream the ByteStream to write to + */ + EXPORT int write(messageqcpp::ByteStream& bytestream); - /** @brief write a UpdateDMLPackage to a ByteStream - * - * @param bytestream the ByteStream to write to - */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + /** @brief read a UpdateDMLPackage from a ByteStream + * + * @param bytestream the ByteStream to read from + */ + EXPORT int read(messageqcpp::ByteStream& bytestream); - /** @brief read a UpdateDMLPackage from a ByteStream - * - * @param bytestream the ByteStream to read from - */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + /** @brief build a UpdateDMLPackage from a string buffer + * + * @param buffer + * @param columns the number of columns in the buffer + * @param rows the number of rows in the buffer + */ + EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); - /** @brief build a UpdateDMLPackage from a string buffer - * - * @param buffer - * @param columns the number of columns in the buffer - * @param rows the number of rows in the buffer - */ - EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); + /** @brief build a UpdateDMLPackage from a parsed UpdateSqlStatement + * + * @param sqlStatement the parsed UpdateSqlStatement + */ + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - /** @brief build a UpdateDMLPackage from a parsed UpdateSqlStatement - * - * @param sqlStatement the parsed UpdateSqlStatement - */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); - - /** @brief build a InsertDMLPackage from MySQL buffer - * - * @param colNameList, tableValuesMap - * @param rows the number of rows in the buffer - */ - EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues); - void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt ); - - -protected: - -private: + /** @brief build a InsertDMLPackage from MySQL buffer + * + * @param colNameList, tableValuesMap + * @param rows the number of rows in the buffer + */ + EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, + int rows, NullValuesBitset& nullValues); + void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt); + protected: + private: }; -} +} // namespace dmlpackage #undef EXPORT -#endif //UPDATEDMLPACKAGE_H +#endif // UPDATEDMLPACKAGE_H diff --git a/dbcon/dmlpackage/vendordmlstatement.cpp b/dbcon/dmlpackage/vendordmlstatement.cpp index 33f3fd08f..6776d41a0 100644 --- a/dbcon/dmlpackage/vendordmlstatement.cpp +++ b/dbcon/dmlpackage/vendordmlstatement.cpp @@ -29,31 +29,56 @@ using namespace std; namespace dmlpackage { - VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int sessionID) - : fDMLStatement(dmlstatement), fSessionID(sessionID), fLogging(true), fLogending(true) -{} + : fDMLStatement(dmlstatement), fSessionID(sessionID), fLogging(true), fLogending(true) +{ +} VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, int sessionID) - : fDMLStatement(dmlstatement), fDMLStatementType(stmttype), fSessionID(sessionID), fLogging(true), fLogending(true) -{} + : fDMLStatement(dmlstatement) + , fDMLStatementType(stmttype) + , fSessionID(sessionID) + , fLogging(true) + , fLogending(true) +{ +} -VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, - std::string tName, std::string schema, - int rows, int columns, std::string buf, +VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, + std::string schema, int rows, int columns, std::string buf, int sessionID) - : fDMLStatement(dmlstatement), fDMLStatementType(stmttype), - fTableName(tName), fSchema(schema), fRows(rows), fColumns(columns), - fDataBuffer(buf), fSessionID(sessionID), fLogging(true), fLogending(true) -{} + : fDMLStatement(dmlstatement) + , fDMLStatementType(stmttype) + , fTableName(tName) + , fSchema(schema) + , fRows(rows) + , fColumns(columns) + , fDataBuffer(buf) + , fSessionID(sessionID) + , fLogging(true) + , fLogending(true) +{ +} -VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, - ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID) - : fDMLStatement(dmlstatement), fDMLStatementType(stmttype), - fTableName(tName), fSchema(schema), fRows(rows), fColumns(columns), - fColNameList(colNameList), fTableValuesMap(tableValuesMap), fNullValues(nullValues), fSessionID(sessionID), fLogging(true), fLogending(true) -{} +VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, + std::string schema, int rows, int columns, ColNameList& colNameList, + TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, + int sessionID) + : fDMLStatement(dmlstatement) + , fDMLStatementType(stmttype) + , fTableName(tName) + , fSchema(schema) + , fRows(rows) + , fColumns(columns) + , fColNameList(colNameList) + , fTableValuesMap(tableValuesMap) + , fNullValues(nullValues) + , fSessionID(sessionID) + , fLogging(true) + , fLogending(true) +{ +} VendorDMLStatement::~VendorDMLStatement() -{} +{ } +} // namespace dmlpackage diff --git a/dbcon/dmlpackage/vendordmlstatement.h b/dbcon/dmlpackage/vendordmlstatement.h index 4b9dd4b71..1d721c0e7 100644 --- a/dbcon/dmlpackage/vendordmlstatement.h +++ b/dbcon/dmlpackage/vendordmlstatement.h @@ -46,208 +46,205 @@ typedef std::bitset<4096> NullValuesBitset; */ class VendorDMLStatement { + public: + /** @brief ctor + */ + EXPORT VendorDMLStatement(std::string dmlstatement, int sessionID); -public: - /** @brief ctor - */ - EXPORT VendorDMLStatement(std::string dmlstatement, int sessionID); + /** @brief ctor + */ + EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, int sessionID); - /** @brief ctor - */ - EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, int sessionID); + /** @brief old ctor! + */ + EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, + int rows, int columns, std::string buf, int sessionID); - /** @brief old ctor! - */ - EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, - std::string schema, int rows, int columns, std::string buf, - int sessionID); + /** @brief ctor for mysql + */ + EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, + int rows, int columns, ColNameList& colNameList, TableValuesMap& tableValuesMap, + NullValuesBitset& nullValues, int sessionID); - /** @brief ctor for mysql - */ - EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, - ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID); + /** @brief destructor + */ + EXPORT ~VendorDMLStatement(); - /** @brief destructor - */ - EXPORT ~VendorDMLStatement(); + /** @brief Get the table name + */ + inline std::string get_TableName() const + { + return fTableName; + } - /** @brief Get the table name - */ - inline std::string get_TableName() const - { - return fTableName; - } + /** @brief Set the table name + */ + inline void set_TableName(std::string value) + { + fTableName = value; + } - /** @brief Set the table name - */ - inline void set_TableName( std::string value ) - { - fTableName = value; - } + /** @brief Get the schema name + */ + inline std::string get_SchemaName() const + { + return fSchema; + } - /** @brief Get the schema name - */ - inline std::string get_SchemaName() const - { - return fSchema; - } + /** @brief Set the schema name + */ + inline void set_SchemaName(std::string value) + { + fSchema = value; + } - /** @brief Set the schema name - */ - inline void set_SchemaName( std::string value ) - { - fSchema = value; - } + /** @brief Get the DML statVendorDMLStatement classement type + */ + inline int get_DMLStatementType() const + { + return fDMLStatementType; + } - /** @brief Get the DML statVendorDMLStatement classement type - */ - inline int get_DMLStatementType() const - { - return fDMLStatementType; - } + /** @brief Set the DML statement type + */ + inline void set_DMLStatementType(int value) + { + fDMLStatementType = value; + } - /** @brief Set the DML statement type - */ - inline void set_DMLStatementType( int value ) - { - fDMLStatementType = value; - } + /** @brief Get the DML statement + */ + inline const std::string get_DMLStatement() const + { + return fDMLStatement; + } - /** @brief Get the DML statement - */ - inline const std::string get_DMLStatement() const - { - return fDMLStatement; - } + /** @brief Set the DML statVendorDMLStatement classement + */ + inline void set_DMLStatement(std::string dmlStatement) + { + fDMLStatement = dmlStatement; + } - /** @brief Set the DML statVendorDMLStatement classement - */ - inline void set_DMLStatement( std::string dmlStatement ) - { - fDMLStatement = dmlStatement; - } + /** @brief Get the number of rows + */ + inline int get_Rows() const + { + return fRows; + } - /** @brief Get the number of rows - */ - inline int get_Rows() const - { - return fRows; - } + /** @brief Set the number of rows + */ + inline void set_Rows(int value) + { + fRows = value; + } - /** @brief Set the number of rows - */ - inline void set_Rows( int value ) - { - fRows = value; - } + /** @brief Get the number of columns + */ + inline int get_Columns() const + { + return fColumns; + } - /** @brief Get the number of columns - */ - inline int get_Columns() const - { - return fColumns; - } + /** @brief Set the number of columns + */ + inline void set_Columns(int value) + { + fColumns = value; + } - /** @brief Set the number of columns - */ - inline void set_Columns( int value ) - { - fColumns = value; - } + /** @brief Get the data buffer + */ + inline std::string& get_DataBuffer() + { + return fDataBuffer; + } - /** @brief Get the data buffer - */ - inline std::string& get_DataBuffer() - { - return fDataBuffer; - } + /** @brief Set the data buffer + */ + inline void set_DataBuffer(std::string value) + { + fDataBuffer = value; + } + /** @brief Get the session ID + */ + inline int get_SessionID() + { + return fSessionID; + } - /** @brief Set the data buffer - */ - inline void set_DataBuffer( std::string value ) - { - fDataBuffer = value; - } - /** @brief Get the session ID - */ - inline int get_SessionID() - { - return fSessionID; - } + inline NullValuesBitset& get_nullValues() + { + return fNullValues; + } - inline NullValuesBitset& get_nullValues() - { - return fNullValues; - } + /** @brief Set the session ID + */ + inline void set_SessionID(int value) + { + fSessionID = value; + } - /** @brief Set the session ID - */ - inline void set_SessionID( int value ) - { - fSessionID = value; - } + inline ColNameList& get_ColNames() + { + return fColNameList; + } + inline TableValuesMap& get_values() + { + return fTableValuesMap; + } + /** @brief get the logging flag + */ + inline bool get_Logging() const + { + return fLogging; + } - inline ColNameList& get_ColNames() - { - return fColNameList; - } - inline TableValuesMap& get_values() - { - return fTableValuesMap; - } - /** @brief get the logging flag - */ - inline bool get_Logging() const - { - return fLogging; - } + /** @brief set the logging flag + * + * @param logging the logging flag to set + */ + inline void set_Logging(bool logging) + { + fLogging = logging; + } - /** @brief set the logging flag - * - * @param logging the logging flag to set - */ - inline void set_Logging( bool logging ) - { - fLogging = logging; - } + /** @brief get the logging flag + */ + inline bool get_Logending() const + { + return fLogending; + } - /** @brief get the logging flag - */ - inline bool get_Logending() const - { - return fLogending; - } - - /** @brief set the logending flag - * - * @param logending the logending flag to set - */ - inline void set_Logending( bool logending ) - { - fLogending = logending; - } - -protected: - -private: - std::string fDMLStatement; - int fDMLStatementType; - std::string fTableName; - std::string fSchema; - int fRows; - int fColumns; - std::string fDataBuffer; - ColNameList fColNameList; - TableValuesMap fTableValuesMap; - NullValuesBitset fNullValues; - int fSessionID; - bool fLogging; - bool fLogending; + /** @brief set the logending flag + * + * @param logending the logending flag to set + */ + inline void set_Logending(bool logending) + { + fLogending = logending; + } + protected: + private: + std::string fDMLStatement; + int fDMLStatementType; + std::string fTableName; + std::string fSchema; + int fRows; + int fColumns; + std::string fDataBuffer; + ColNameList fColNameList; + TableValuesMap fTableValuesMap; + NullValuesBitset fNullValues; + int fSessionID; + bool fLogging; + bool fLogending; }; -} +} // namespace dmlpackage #undef EXPORT -#endif //VENDORDMLSTATEMENT_H +#endif // VENDORDMLSTATEMENT_H diff --git a/dbcon/dmlpackageproc/autoincrementdata.cpp b/dbcon/dmlpackageproc/autoincrementdata.cpp index bfeedc1c6..3a71efaf6 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.cpp +++ b/dbcon/dmlpackageproc/autoincrementdata.cpp @@ -37,31 +37,31 @@ AutoincrementData::AutoincDataMap AutoincrementData::fAutoincDataMap; /* static */ AutoincrementData* AutoincrementData::makeAutoincrementData(uint32_t sessionID) { - boost::mutex::scoped_lock lock(map_mutex); - AutoincrementData* instance; - AutoincDataMap::const_iterator it = fAutoincDataMap.find(sessionID); + boost::mutex::scoped_lock lock(map_mutex); + AutoincrementData* instance; + AutoincDataMap::const_iterator it = fAutoincDataMap.find(sessionID); - if (it == fAutoincDataMap.end()) - { - instance = new AutoincrementData(); - fAutoincDataMap[sessionID] = instance; - return instance; - } + if (it == fAutoincDataMap.end()) + { + instance = new AutoincrementData(); + fAutoincDataMap[sessionID] = instance; + return instance; + } - return it->second; + return it->second; } /* static */ void AutoincrementData::removeAutoincrementData(uint32_t sessionID) { - boost::mutex::scoped_lock lock(map_mutex); - AutoincDataMap::iterator it = fAutoincDataMap.find(sessionID); + boost::mutex::scoped_lock lock(map_mutex); + AutoincDataMap::iterator it = fAutoincDataMap.find(sessionID); - if (it != fAutoincDataMap.end()) - { - delete (*it).second; - fAutoincDataMap.erase(it); - } + if (it != fAutoincDataMap.end()) + { + delete (*it).second; + fAutoincDataMap.erase(it); + } } AutoincrementData::AutoincrementData() @@ -73,28 +73,28 @@ AutoincrementData::~AutoincrementData() void AutoincrementData::setNextValue(uint32_t columnOid, long long nextValue) { - boost::mutex::scoped_lock lk(fOIDnextvalLock); - fOidNextValueMap[columnOid] = nextValue; + boost::mutex::scoped_lock lk(fOIDnextvalLock); + fOidNextValueMap[columnOid] = nextValue; } long long AutoincrementData::getNextValue(uint32_t columnOid) { - boost::mutex::scoped_lock lk(fOIDnextvalLock); - long long nextValue = 0; - OIDNextValue::iterator it = fOidNextValueMap.find(columnOid); + boost::mutex::scoped_lock lk(fOIDnextvalLock); + long long nextValue = 0; + OIDNextValue::iterator it = fOidNextValueMap.find(columnOid); - if (it != fOidNextValueMap.end()) - { - nextValue = it->second; - } + if (it != fOidNextValueMap.end()) + { + nextValue = it->second; + } - return nextValue; + return nextValue; } AutoincrementData::OIDNextValue& AutoincrementData::getOidNextValueMap() { - boost::mutex::scoped_lock lk(fOIDnextvalLock); + boost::mutex::scoped_lock lk(fOIDnextvalLock); - return fOidNextValueMap; + return fOidNextValueMap; } // vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/autoincrementdata.h b/dbcon/dmlpackageproc/autoincrementdata.h index d3e68b344..17be0648d 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.h +++ b/dbcon/dmlpackageproc/autoincrementdata.h @@ -27,28 +27,27 @@ #include #include - class AutoincrementData { -public: - typedef std::map AutoincDataMap; - typedef std::map OIDNextValue; - static AutoincrementData* makeAutoincrementData(uint32_t sessionID = 0); - static void removeAutoincrementData(uint32_t sessionID = 0); - void setNextValue(uint32_t columnOid, long long nextValue); - long long getNextValue(uint32_t columnOid); - OIDNextValue& getOidNextValueMap(); + public: + typedef std::map AutoincDataMap; + typedef std::map OIDNextValue; + static AutoincrementData* makeAutoincrementData(uint32_t sessionID = 0); + static void removeAutoincrementData(uint32_t sessionID = 0); + void setNextValue(uint32_t columnOid, long long nextValue); + long long getNextValue(uint32_t columnOid); + OIDNextValue& getOidNextValueMap(); -private: - /** Constuctors */ - explicit AutoincrementData(); - explicit AutoincrementData(const AutoincrementData& rhs); - ~AutoincrementData(); + private: + /** Constuctors */ + explicit AutoincrementData(); + explicit AutoincrementData(const AutoincrementData& rhs); + ~AutoincrementData(); - static boost::mutex map_mutex; - static AutoincDataMap fAutoincDataMap; - OIDNextValue fOidNextValueMap; - boost::mutex fOIDnextvalLock; + static boost::mutex map_mutex; + static AutoincDataMap fAutoincDataMap; + OIDNextValue fOidNextValueMap; + boost::mutex fOIDnextvalLock; }; #endif diff --git a/dbcon/dmlpackageproc/commandpackageprocessor.cpp b/dbcon/dmlpackageproc/commandpackageprocessor.cpp index e41ff3067..46fe32e69 100644 --- a/dbcon/dmlpackageproc/commandpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/commandpackageprocessor.cpp @@ -48,533 +48,532 @@ using namespace BRM; namespace dmlpackageprocessor { // Tracks active cleartablelock commands by storing set of table lock IDs -/*static*/ std::set -CommandPackageProcessor::fActiveClearTableLockCmds; -/*static*/ boost::mutex -CommandPackageProcessor::fActiveClearTableLockCmdMutex; +/*static*/ std::set CommandPackageProcessor::fActiveClearTableLockCmds; +/*static*/ boost::mutex CommandPackageProcessor::fActiveClearTableLockCmdMutex; -DMLPackageProcessor::DMLResult -CommandPackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) +DMLPackageProcessor::DMLResult CommandPackageProcessor::processPackage( + dmlpackage::CalpontDMLPackage& cpackage) { - SUMMARY_INFO("CommandPackageProcessor::processPackage"); + SUMMARY_INFO("CommandPackageProcessor::processPackage"); - DMLResult result; - result.result = NO_ERROR; + DMLResult result; + result.result = NO_ERROR; - VERBOSE_INFO("Processing Command DML Package..."); - std::string stmt = cpackage.get_DMLStatement(); - boost::algorithm::to_upper(stmt); - trim(stmt); - fSessionID = cpackage.get_SessionID(); - BRM::TxnID txnid = fSessionManager.getTxnID(cpackage.get_SessionID()); - uint64_t uniqueId = 0; + VERBOSE_INFO("Processing Command DML Package..."); + std::string stmt = cpackage.get_DMLStatement(); + boost::algorithm::to_upper(stmt); + trim(stmt); + fSessionID = cpackage.get_SessionID(); + BRM::TxnID txnid = fSessionManager.getTxnID(cpackage.get_SessionID()); + uint64_t uniqueId = 0; - //Bug 5070. Added exception handling - try + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(9); + args.add(ex.what()); + message.format(args); + result.result = COMMAND_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + catch (...) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unknown error occurred while getting unique number."); + message.format(args); + result.result = COMMAND_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + + string errorMsg; + bool queRemoved = false; + logging::LoggingID lid(20); + logging::MessageLog ml(lid); + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + Logger logger(logid.fSubsysID); + + if (stmt != "CLEANUP") + { + args1.add("Start SQL statement: "); + args1.add(stmt); + msg.format(args1); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + } + + // fWEClient->addQueue(uniqueId); + try + { + // set-up the transaction + if ((stmt == "COMMIT") || (stmt == "ROLLBACK")) { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(9); - args.add(ex.what()); - message.format(args); - result.result = COMMAND_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; - } - catch ( ... ) - { - logging::Message::Args args; - logging::Message message(9); - args.add("Unknown error occurred while getting unique number."); - message.format(args); - result.result = COMMAND_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; - } + // SQLLogger sqlLogger(stmt, DMLLoggingId, cpackage.get_SessionID(), txnid.id); - string errorMsg; - bool queRemoved = false; - logging::LoggingID lid(20); - logging::MessageLog ml(lid); - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - Logger logger(logid.fSubsysID); + if ((txnid.valid)) + { + vector lbidList; + fDbrm->getUncommittedExtentLBIDs(static_cast(txnid.id), lbidList); + bool cpInvalidated = false; - if (stmt != "CLEANUP") - { - args1.add("Start SQL statement: "); - args1.add(stmt); - msg.format( args1 ); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - } - - //fWEClient->addQueue(uniqueId); - try - { - // set-up the transaction - if ( (stmt == "COMMIT") || (stmt == "ROLLBACK") ) + // cout << "get a valid txnid " << txnid.id << " and stmt is " << stmt << " and isBachinsert is " << + // cpackage.get_isBatchInsert() << endl; + if ((stmt == "COMMIT") && (cpackage.get_isBatchInsert())) { - //SQLLogger sqlLogger(stmt, DMLLoggingId, cpackage.get_SessionID(), txnid.id); + // update syscolumn for the next value + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + CalpontSystemCatalog::TableName tableName; + tableName = systemCatalogPtr->tableName(cpackage.getTableOid()); - if ((txnid.valid)) + try + { + uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + + if (nextVal != AUTOINCR_SATURATED) // need to update syscolumn { - vector lbidList; - fDbrm->getUncommittedExtentLBIDs(static_cast(txnid.id), lbidList); - bool cpInvalidated = false; + // get autoincrement column oid + int32_t columnOid = systemCatalogPtr->autoColumOid(tableName); + // get the current nextVal from controller + scoped_ptr aDbrm(new DBRM()); + uint64_t nextValInController = 0; + bool validNextVal = false; + aDbrm->getAILock(columnOid); + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); // in case it has changed + validNextVal = aDbrm->getAIValue(columnOid, &nextValInController); - //cout << "get a valid txnid " << txnid.id << " and stmt is " << stmt << " and isBachinsert is " << cpackage.get_isBatchInsert() << endl; - if ((stmt == "COMMIT") && (cpackage.get_isBatchInsert())) + if ((validNextVal) && (nextValInController > nextVal)) + { + fWEClient->removeQueue(uniqueId); + queRemoved = true; + WE_DDLCommandClient ddlClient; + uint8_t rc = ddlClient.UpdateSyscolumnNextval(columnOid, nextValInController); + //@bug 5894. Need release lock. + aDbrm->releaseAILock(columnOid); + + if (rc != 0) + throw std::runtime_error("Error in UpdateSyscolumnNextval"); + } + else + aDbrm->releaseAILock(columnOid); + } + } + catch (std::exception& ex) + { + // Rollback transaction + rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); + fSessionManager.rolledback(txnid); + throw std::runtime_error(ex.what()); + } + + // systemCatalogPtr->updateColinfoCache(nextValMap); + int weRc = 0; + + if (cpackage.get_isAutocommitOn()) + { + weRc = commitBatchAutoOnTransaction(uniqueId, txnid, cpackage.getTableOid(), errorMsg); + + if (weRc != 0) + BRM::errString(weRc, errorMsg); + + cpInvalidated = true; + } + else + { + weRc = fDbrm->vbCommit(txnid.id); + + if (weRc != 0) + BRM::errString(weRc, errorMsg); + + // weRc = commitBatchAutoOffTransaction(uniqueId, txnid, cpackage.getTableOid(), errorMsg); + } + + if (weRc != 0) + { + throw std::runtime_error(errorMsg); + } + + logging::logCommand(cpackage.get_SessionID(), txnid.id, "COMMIT;"); + fSessionManager.committed(txnid); + // cout << "releasing transaction id for batchinsert" << txnid.id << endl; + } + else if (stmt == "COMMIT") + { + // cout << "success in commitTransaction" << endl; + // update syscolumn for the next value + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + CalpontSystemCatalog::TableName tableName; + uint32_t tableOid = cpackage.getTableOid(); + std::vector tableLocks = fDbrm->getAllTableLocks(); + + if (tableOid == 0) // special case: transaction commit for autocommit off and not following a dml + // statement immediately + { + TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); + TablelockData::OIDTablelock tablelockMap = tablelockData->getOidTablelockMap(); + TablelockData::OIDTablelock::iterator iter; + + if (!tablelockMap.empty()) + { + for (unsigned k = 0; k < tableLocks.size(); k++) + { + iter = tablelockMap.find(tableLocks[k].tableOID); + + if (iter != tablelockMap.end()) { - //update syscolumn for the next value - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); - CalpontSystemCatalog::TableName tableName; - tableName = systemCatalogPtr->tableName(cpackage.getTableOid()); + tableName = systemCatalogPtr->tableName(tableLocks[k].tableOID); + try + { + uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + + if (nextVal != AUTOINCR_SATURATED) // neet to update syscolumn + { + // get autoincrement column oid + int32_t columnOid = systemCatalogPtr->autoColumOid(tableName); + // get the current nextVal from controller + scoped_ptr aDbrm(new DBRM()); + uint64_t nextValInController = 0; + bool validNextVal = false; + aDbrm->getAILock(columnOid); + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); // in case it has changed + validNextVal = aDbrm->getAIValue(columnOid, &nextValInController); + + if ((validNextVal) && (nextValInController > (uint64_t)nextVal)) + { + fWEClient->removeQueue(uniqueId); + + queRemoved = true; + WE_DDLCommandClient ddlClient; + uint8_t rc = + ddlClient.UpdateSyscolumnNextval(columnOid, nextValInController, fSessionID); + aDbrm->releaseAILock(columnOid); + + if (rc != 0) + { + // for now + fSessionManager.committed(txnid); + throw std::runtime_error("Error in UpdateSyscolumnNextval"); + } + } + else + aDbrm->releaseAILock(columnOid); + } + } + catch (std::exception& ex) + { + // Rollback transaction, release tablelock + rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); + fDbrm->releaseTableLock(iter->second); + fSessionManager.rolledback(txnid); + throw std::runtime_error(ex.what()); + } + } + } + } + } + else + { + if (tableOid >= 3000) + { + tableName = systemCatalogPtr->tableName(tableOid); + + try + { + uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + + if (nextVal != AUTOINCR_SATURATED) // need to update syscolumn + { + // get autoincrement column oid + int32_t columnOid = systemCatalogPtr->autoColumOid(tableName); + // get the current nextVal from controller + scoped_ptr aDbrm(new DBRM()); + uint64_t nextValInController = 0; + bool validNextVal = false; + aDbrm->getAILock(columnOid); + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); // in case it has changed + validNextVal = aDbrm->getAIValue(columnOid, &nextValInController); + + if ((validNextVal) && (nextValInController > (uint64_t)nextVal)) + { + fWEClient->removeQueue(uniqueId); + + queRemoved = true; + WE_DDLCommandClient ddlClient; + uint8_t rc = ddlClient.UpdateSyscolumnNextval(columnOid, nextValInController, fSessionID); + aDbrm->releaseAILock(columnOid); + + if (rc != 0) + { + // for now + fSessionManager.committed(txnid); + throw std::runtime_error("Error in UpdateSyscolumnNextval"); + } + } + else + aDbrm->releaseAILock(columnOid); + } + } + catch (std::exception& ex) + { + // Rollback transaction, release tablelock + rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); + + for (unsigned k = 0; k < tableLocks.size(); k++) + { + if (tableLocks[k].tableOID == tableOid) + { try { - uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - - if (nextVal != AUTOINCR_SATURATED) //need to update syscolumn - { - //get autoincrement column oid - int32_t columnOid = systemCatalogPtr->autoColumOid(tableName); - //get the current nextVal from controller - scoped_ptr aDbrm(new DBRM()); - uint64_t nextValInController = 0; - bool validNextVal = false; - aDbrm->getAILock(columnOid); - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); //in case it has changed - validNextVal = aDbrm->getAIValue(columnOid, &nextValInController); - - if ((validNextVal) && (nextValInController > nextVal)) - { - fWEClient->removeQueue(uniqueId); - queRemoved = true; - WE_DDLCommandClient ddlClient; - uint8_t rc = ddlClient.UpdateSyscolumnNextval(columnOid, nextValInController); - //@bug 5894. Need release lock. - aDbrm->releaseAILock(columnOid); - - if (rc != 0) - throw std::runtime_error("Error in UpdateSyscolumnNextval"); - } - else - aDbrm->releaseAILock(columnOid); - } + fDbrm->releaseTableLock(tableLocks[k].id); } - catch (std::exception& ex) + catch (std::exception&) { - //Rollback transaction - rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); - fSessionManager.rolledback( txnid ); - throw std::runtime_error(ex.what()); } - - //systemCatalogPtr->updateColinfoCache(nextValMap); - int weRc = 0; - - if (cpackage.get_isAutocommitOn()) - { - weRc = commitBatchAutoOnTransaction(uniqueId, txnid, cpackage.getTableOid(), errorMsg); - - if (weRc != 0) - BRM::errString(weRc, errorMsg); - - cpInvalidated = true; - } - else - { - weRc = fDbrm->vbCommit(txnid.id); - - if (weRc != 0) - BRM::errString(weRc, errorMsg); - - //weRc = commitBatchAutoOffTransaction(uniqueId, txnid, cpackage.getTableOid(), errorMsg); - } - - if (weRc != 0) - { - throw std::runtime_error(errorMsg); - } - - logging::logCommand(cpackage.get_SessionID(), txnid.id, "COMMIT;"); - fSessionManager.committed( txnid ); - //cout << "releasing transaction id for batchinsert" << txnid.id << endl; - } - else if (stmt == "COMMIT") - { - //cout << "success in commitTransaction" << endl; - //update syscolumn for the next value - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); - CalpontSystemCatalog::TableName tableName; - uint32_t tableOid = cpackage.getTableOid(); - std::vector tableLocks = fDbrm->getAllTableLocks(); - - if (tableOid == 0) //special case: transaction commit for autocommit off and not following a dml statement immediately - { - TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - TablelockData::OIDTablelock tablelockMap = tablelockData->getOidTablelockMap(); - TablelockData::OIDTablelock::iterator iter; - - if (!tablelockMap.empty()) - { - for ( unsigned k = 0; k < tableLocks.size(); k++) - { - iter = tablelockMap.find(tableLocks[k].tableOID); - - if ( iter != tablelockMap.end() ) - { - tableName = systemCatalogPtr->tableName(tableLocks[k].tableOID); - - try - { - uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - - if (nextVal != AUTOINCR_SATURATED) //neet to update syscolumn - { - //get autoincrement column oid - int32_t columnOid = systemCatalogPtr->autoColumOid(tableName); - //get the current nextVal from controller - scoped_ptr aDbrm(new DBRM()); - uint64_t nextValInController = 0; - bool validNextVal = false; - aDbrm->getAILock(columnOid); - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); //in case it has changed - validNextVal = aDbrm->getAIValue(columnOid, &nextValInController); - - if ((validNextVal) && (nextValInController > (uint64_t)nextVal)) - { - fWEClient->removeQueue(uniqueId); - - queRemoved = true; - WE_DDLCommandClient ddlClient; - uint8_t rc = ddlClient.UpdateSyscolumnNextval(columnOid, nextValInController, fSessionID); - aDbrm->releaseAILock(columnOid); - - if (rc != 0) - { - //for now - fSessionManager.committed( txnid ); - throw std::runtime_error("Error in UpdateSyscolumnNextval"); - } - } - else - aDbrm->releaseAILock(columnOid); - } - - } - catch (std::exception& ex) - { - //Rollback transaction, release tablelock - rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); - fDbrm->releaseTableLock(iter->second); - fSessionManager.rolledback( txnid ); - throw std::runtime_error(ex.what()); - } - } - } - } - } - else - { - if (tableOid >= 3000) - { - tableName = systemCatalogPtr->tableName(tableOid); - - try - { - uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - - if (nextVal != AUTOINCR_SATURATED) //need to update syscolumn - { - //get autoincrement column oid - int32_t columnOid = systemCatalogPtr->autoColumOid(tableName); - //get the current nextVal from controller - scoped_ptr aDbrm(new DBRM()); - uint64_t nextValInController = 0; - bool validNextVal = false; - aDbrm->getAILock(columnOid); - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); //in case it has changed - validNextVal = aDbrm->getAIValue(columnOid, &nextValInController); - - if ((validNextVal) && (nextValInController > (uint64_t)nextVal)) - { - fWEClient->removeQueue(uniqueId); - - queRemoved = true; - WE_DDLCommandClient ddlClient; - uint8_t rc = ddlClient.UpdateSyscolumnNextval(columnOid, nextValInController, fSessionID); - aDbrm->releaseAILock(columnOid); - - if (rc != 0) - { - //for now - fSessionManager.committed( txnid ); - throw std::runtime_error("Error in UpdateSyscolumnNextval"); - } - } - else - aDbrm->releaseAILock(columnOid); - } - } - catch (std::exception& ex) - { - //Rollback transaction, release tablelock - rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); - - for ( unsigned k = 0; k < tableLocks.size(); k++) - { - if ( tableLocks[k].tableOID == tableOid ) - { - try - { - fDbrm->releaseTableLock(tableLocks[k].id); - } - catch (std::exception&) - {} - } - } - - fSessionManager.rolledback( txnid ); - throw std::runtime_error(ex.what()); - } - } - } - - int weRc = commitTransaction(uniqueId, txnid ); - logging::logCommand(cpackage.get_SessionID(), txnid.id, "COMMIT;"); - - if (weRc != WriteEngine::NO_ERROR) - { - //cout << "Got an error in commitTransaction" << endl; - WErrorCodes ec; - ostringstream oss; - oss << "COMMIT failed: " << ec.errorString(weRc); - throw std::runtime_error(oss.str()); - } - - fSessionManager.committed( txnid ); - //cout << "commit releasing transaction id " << txnid.id << endl; - } - else if ((stmt == "ROLLBACK") && (cpackage.get_isBatchInsert())) - { - int weRc = 0; - - - //version rollback, Bulkrollback - weRc = rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); - - if (weRc == 0) - { - //@Bug 4560 invalidate cp first as bulkrollback will truncate the newly added lbids. - fDbrm->invalidateUncommittedExtentLBIDs(0, true, &lbidList); - cpInvalidated = true; - weRc = rollBackBatchAutoOnTransaction(uniqueId, txnid, fSessionID, cpackage.getTableOid(), errorMsg); - - } - else - { - throw std::runtime_error(errorMsg); - } - - logging::logCommand(cpackage.get_SessionID(), txnid.id, "ROLLBACK;"); - - if (weRc != 0) - { - //@Bug 4524. Don't set to readonly. Just error out. - WErrorCodes ec; - ostringstream oss; - oss << "ROLLBACK batch insert failed due to: " << "(" << weRc << ")" << ec.errorString(weRc); - //Log to error log - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logErrorMessage( message1 ); - throw std::runtime_error(oss.str()); - } - - fSessionManager.rolledback( txnid ); - //cout << "batch rollback releasing transaction id " << txnid.id << endl; - } - else if (stmt == "ROLLBACK") - { - std::string errorMsg(""); - logging::logCommand(cpackage.get_SessionID(), txnid.id, "ROLLBACK;"); - int weRc = rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); - - if (weRc != 0) - { - //cout << "Rollback failed" << endl; - //@Bug 4524. Don't set to readonly. Just error out. - ostringstream oss; - oss << "ROLLBACK failed due to: " << errorMsg; - //Log to error log - logging::Message::Args args2; - logging::Message message2(2); - args2.add(oss.str()); - message2.format( args2 ); - ml.logErrorMessage( message2 ); - throw std::runtime_error(oss.str()); - } - - fSessionManager.rolledback( txnid ); - //cout << "Rollback releasing transaction id " << txnid.id << endl; + } } - if (!cpInvalidated) - { - fDbrm->invalidateUncommittedExtentLBIDs(0, stmt == "ROLLBACK", &lbidList); - } + fSessionManager.rolledback(txnid); + throw std::runtime_error(ex.what()); + } } + } + + int weRc = commitTransaction(uniqueId, txnid); + logging::logCommand(cpackage.get_SessionID(), txnid.id, "COMMIT;"); + + if (weRc != WriteEngine::NO_ERROR) + { + // cout << "Got an error in commitTransaction" << endl; + WErrorCodes ec; + ostringstream oss; + oss << "COMMIT failed: " << ec.errorString(weRc); + throw std::runtime_error(oss.str()); + } + + fSessionManager.committed(txnid); + // cout << "commit releasing transaction id " << txnid.id << endl; } - else if (stmt == "CLEANUP") + else if ((stmt == "ROLLBACK") && (cpackage.get_isBatchInsert())) { - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog - (cpackage.get_SessionID()); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog - (cpackage.get_SessionID() | 0x80000000); + int weRc = 0; + + // version rollback, Bulkrollback + weRc = rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); + + if (weRc == 0) + { + //@Bug 4560 invalidate cp first as bulkrollback will truncate the newly added lbids. + fDbrm->invalidateUncommittedExtentLBIDs(0, true, &lbidList); + cpInvalidated = true; + weRc = + rollBackBatchAutoOnTransaction(uniqueId, txnid, fSessionID, cpackage.getTableOid(), errorMsg); + } + else + { + throw std::runtime_error(errorMsg); + } + + logging::logCommand(cpackage.get_SessionID(), txnid.id, "ROLLBACK;"); + + if (weRc != 0) + { + //@Bug 4524. Don't set to readonly. Just error out. + WErrorCodes ec; + ostringstream oss; + oss << "ROLLBACK batch insert failed due to: " + << "(" << weRc << ")" << ec.errorString(weRc); + // Log to error log + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logErrorMessage(message1); + throw std::runtime_error(oss.str()); + } + + fSessionManager.rolledback(txnid); + // cout << "batch rollback releasing transaction id " << txnid.id << endl; } - else if (stmt == "VIEWTABLELOCK") + else if (stmt == "ROLLBACK") { - viewTableLock( cpackage, result ); - } - else if (stmt == "CLEARTABLELOCK") - { - clearTableLock( uniqueId, cpackage, result ); - } - else if ( !cpackage.get_Logging()) - { - BRM::TxnID txnid = fSessionManager.getTxnID(cpackage.get_SessionID()); - logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_DMLStatement() + ";", cpackage.get_SchemaName()); - SQLLogger sqlLogger(cpackage.get_DMLStatement(), DMLLoggingId, fSessionID, txnid.id); - //cout << "commandpackageprocessor Logging " << cpackage.get_DMLStatement()+ ";" << endl; - } - else - { - std::string err = "Unknown command."; - SUMMARY_INFO(err); - throw std::runtime_error(err); + std::string errorMsg(""); + logging::logCommand(cpackage.get_SessionID(), txnid.id, "ROLLBACK;"); + int weRc = rollBackTransaction(uniqueId, txnid, fSessionID, errorMsg); + + if (weRc != 0) + { + // cout << "Rollback failed" << endl; + //@Bug 4524. Don't set to readonly. Just error out. + ostringstream oss; + oss << "ROLLBACK failed due to: " << errorMsg; + // Log to error log + logging::Message::Args args2; + logging::Message message2(2); + args2.add(oss.str()); + message2.format(args2); + ml.logErrorMessage(message2); + throw std::runtime_error(oss.str()); + } + + fSessionManager.rolledback(txnid); + // cout << "Rollback releasing transaction id " << txnid.id << endl; } + if (!cpInvalidated) + { + fDbrm->invalidateUncommittedExtentLBIDs(0, stmt == "ROLLBACK", &lbidList); + } + } } - catch ( logging::IDBExcept& noTable) //@Bug 2606 catch no table found exception + else if (stmt == "CLEANUP") { - cerr << "CommandPackageProcessor::processPackage: " << noTable.what() << endl; - - result.result = COMMAND_ERROR; - result.message = Message(noTable.what()); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(cpackage.get_SessionID()); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(cpackage.get_SessionID() | 0x80000000); } - catch (std::exception& ex) + else if (stmt == "VIEWTABLELOCK") { - cerr << "CommandPackageProcessor::processPackage: " << ex.what() << endl; - - logging::Message::Args args; - logging::Message message(1); - args.add( ex.what() ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = COMMAND_ERROR; - result.message = message; + viewTableLock(cpackage, result); } - catch (...) + else if (stmt == "CLEARTABLELOCK") { - cerr << "CommandPackageProcessor::processPackage: caught unknown exception!" << endl; - logging::Message::Args args; - logging::Message message(1); - args.add( "Command Failed: "); - args.add( "encountered unkown exception" ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = COMMAND_ERROR; - result.message = message; + clearTableLock(uniqueId, cpackage, result); } - - if (!queRemoved) - fWEClient->removeQueue(uniqueId); - - //release table lock - if ((result.result == NO_ERROR) && ((stmt == "COMMIT") || (stmt == "ROLLBACK")) ) + else if (!cpackage.get_Logging()) { - TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - TablelockData::OIDTablelock tablelockMap = tablelockData->getOidTablelockMap(); - bool lockReleased = true; + BRM::TxnID txnid = fSessionManager.getTxnID(cpackage.get_SessionID()); + logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_DMLStatement() + ";", + cpackage.get_SchemaName()); + SQLLogger sqlLogger(cpackage.get_DMLStatement(), DMLLoggingId, fSessionID, txnid.id); + // cout << "commandpackageprocessor Logging " << cpackage.get_DMLStatement()+ ";" << endl; + } + else + { + std::string err = "Unknown command."; + SUMMARY_INFO(err); + throw std::runtime_error(err); + } + } + catch (logging::IDBExcept& noTable) //@Bug 2606 catch no table found exception + { + cerr << "CommandPackageProcessor::processPackage: " << noTable.what() << endl; - if (!tablelockMap.empty()) + result.result = COMMAND_ERROR; + result.message = Message(noTable.what()); + } + catch (std::exception& ex) + { + cerr << "CommandPackageProcessor::processPackage: " << ex.what() << endl; + + logging::Message::Args args; + logging::Message message(1); + args.add(ex.what()); + args.add(""); + args.add(""); + message.format(args); + + result.result = COMMAND_ERROR; + result.message = message; + } + catch (...) + { + cerr << "CommandPackageProcessor::processPackage: caught unknown exception!" << endl; + logging::Message::Args args; + logging::Message message(1); + args.add("Command Failed: "); + args.add("encountered unkown exception"); + args.add(""); + args.add(""); + message.format(args); + + result.result = COMMAND_ERROR; + result.message = message; + } + + if (!queRemoved) + fWEClient->removeQueue(uniqueId); + + // release table lock + if ((result.result == NO_ERROR) && ((stmt == "COMMIT") || (stmt == "ROLLBACK"))) + { + TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); + TablelockData::OIDTablelock tablelockMap = tablelockData->getOidTablelockMap(); + bool lockReleased = true; + + if (!tablelockMap.empty()) + { + TablelockData::OIDTablelock::iterator it = tablelockMap.begin(); + + while (it != tablelockMap.end()) + { + try { - TablelockData::OIDTablelock::iterator it = tablelockMap.begin(); + lockReleased = fDbrm->releaseTableLock(it->second); + // cout << "releasing tablelock " << it->second << endl; + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(1); + args.add(ex.what()); + args.add(""); + args.add(""); + message.format(args); - while (it != tablelockMap.end()) - { - - try - { - lockReleased = fDbrm->releaseTableLock(it->second); - //cout << "releasing tablelock " << it->second << endl; - } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(1); - args.add( ex.what() ); - args.add(""); - args.add(""); - message.format( args ); - - result.result = COMMAND_ERROR; - result.message = message; - } - - if (!lockReleased) //log an error - { - ostringstream os; - os << "tablelock for table oid " << it->first << " is not found"; - logging::Message::Args args; - logging::Message message(1); - args.add( os.str()); - args.add(""); - args.add(""); - message.format( args ); - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - - ml.logErrorMessage(message); - } - - //cout << "tablelock " << it->second << " is released" << endl; - it++; - } - - //@Bug 3557. Clean tablelock cache after commit/rollback. - TablelockData::removeTablelockData(cpackage.get_SessionID()); + result.result = COMMAND_ERROR; + result.message = message; } + if (!lockReleased) // log an error + { + ostringstream os; + os << "tablelock for table oid " << it->first << " is not found"; + logging::Message::Args args; + logging::Message message(1); + args.add(os.str()); + args.add(""); + args.add(""); + message.format(args); + logging::LoggingID lid(21); + logging::MessageLog ml(lid); + + ml.logErrorMessage(message); + } + + // cout << "tablelock " << it->second << " is released" << endl; + it++; + } + + //@Bug 3557. Clean tablelock cache after commit/rollback. + TablelockData::removeTablelockData(cpackage.get_SessionID()); } + } - VERBOSE_INFO("Finished processing Command DML Package"); - //LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args2; - logging::Message msg1(1); + VERBOSE_INFO("Finished processing Command DML Package"); + // LoggingID logid( DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args2; + logging::Message msg1(1); - if (stmt != "CLEANUP") - { - args2.add("End SQL statement"); - msg1.format( args2 ); - //Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); - } + if (stmt != "CLEANUP") + { + args2.add("End SQL statement"); + msg1.format(args2); + // Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); + } - return result; + return result; } //------------------------------------------------------------------------------ @@ -582,536 +581,512 @@ CommandPackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) // specified table. // This function closely resembles printTableLocks in viewtablelock.cpp. //------------------------------------------------------------------------------ -void CommandPackageProcessor::viewTableLock( - const dmlpackage::CalpontDMLPackage& cpackage, - DMLPackageProcessor::DMLResult& result) +void CommandPackageProcessor::viewTableLock(const dmlpackage::CalpontDMLPackage& cpackage, + DMLPackageProcessor::DMLResult& result) { - // Initialize System Catalog object used to get table name - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::TableName tableName; - tableName.schema = cpackage.get_SchemaName(); - tableName.table = cpackage.get_TableName(); - execplan::CalpontSystemCatalog::ROPair roPair; + // Initialize System Catalog object used to get table name + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::TableName tableName; + tableName.schema = cpackage.get_SchemaName(); + tableName.table = cpackage.get_TableName(); + execplan::CalpontSystemCatalog::ROPair roPair; - roPair = systemCatalogPtr->tableRID( tableName ); + roPair = systemCatalogPtr->tableRID(tableName); - // Get list of table locks for the requested table - std::vector tableLocks; - tableLocks = fDbrm->getAllTableLocks(); + // Get list of table locks for the requested table + std::vector tableLocks; + tableLocks = fDbrm->getAllTableLocks(); - // Make preliminary pass through the table locks in order to determine our - // output column widths based on the data. Min column widths are based on - // the width of the column heading (except for the 'state' column). - uint64_t maxLockID = 0; - uint32_t maxPID = 0; - int32_t maxSessionID = 0; - int32_t minSessionID = 0; - int32_t maxTxnID = 0; + // Make preliminary pass through the table locks in order to determine our + // output column widths based on the data. Min column widths are based on + // the width of the column heading (except for the 'state' column). + uint64_t maxLockID = 0; + uint32_t maxPID = 0; + int32_t maxSessionID = 0; + int32_t minSessionID = 0; + int32_t maxTxnID = 0; - unsigned int lockIDColumnWidth = 6; // "LockID" - unsigned int ownerColumnWidth = 7; // "Process" - unsigned int pidColumnWidth = 3; // "PID" - unsigned int sessionIDColumnWidth = 7; // "Session" - unsigned int txnIDColumnWidth = 3; // "Txn" - unsigned int createTimeColumnWidth = 12; // "CreationTime" - unsigned int pmColumnWidth = 7; // "DBRoots" - std::vector createTimes; - char cTimeBuffer[1024]; + unsigned int lockIDColumnWidth = 6; // "LockID" + unsigned int ownerColumnWidth = 7; // "Process" + unsigned int pidColumnWidth = 3; // "PID" + unsigned int sessionIDColumnWidth = 7; // "Session" + unsigned int txnIDColumnWidth = 3; // "Txn" + unsigned int createTimeColumnWidth = 12; // "CreationTime" + unsigned int pmColumnWidth = 7; // "DBRoots" + std::vector createTimes; + char cTimeBuffer[1024]; - for (unsigned idx = 0; idx < tableLocks.size(); idx++) + for (unsigned idx = 0; idx < tableLocks.size(); idx++) + { + if (tableLocks[idx].id > maxLockID) + maxLockID = tableLocks[idx].id; + + if (tableLocks[idx].ownerName.length() > ownerColumnWidth) + ownerColumnWidth = tableLocks[idx].ownerName.length(); + + if (tableLocks[idx].ownerPID > maxPID) + maxPID = tableLocks[idx].ownerPID; + + if (tableLocks[idx].ownerSessionID > maxSessionID) + maxSessionID = tableLocks[idx].ownerSessionID; + + if (tableLocks[idx].ownerSessionID < minSessionID) + minSessionID = tableLocks[idx].ownerSessionID; + + if (tableLocks[idx].ownerTxnID > maxTxnID) + maxTxnID = tableLocks[idx].ownerTxnID; + + ctime_r(&tableLocks[idx].creationTime, cTimeBuffer); + cTimeBuffer[strlen(cTimeBuffer) - 1] = '\0'; // strip trailing '\n' + std::string cTimeStr(cTimeBuffer); + + if (cTimeStr.length() > createTimeColumnWidth) + createTimeColumnWidth = cTimeStr.length(); + + createTimes.push_back(cTimeStr); + + std::ostringstream pms; // It is dbroots now + + for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) { - if (tableLocks[idx].id > maxLockID) - maxLockID = tableLocks[idx].id; + if (k > 0) + pms << ','; - if (tableLocks[idx].ownerName.length() > ownerColumnWidth) - ownerColumnWidth = tableLocks[idx].ownerName.length(); - - if (tableLocks[idx].ownerPID > maxPID) - maxPID = tableLocks[idx].ownerPID; - - if (tableLocks[idx].ownerSessionID > maxSessionID) - maxSessionID = tableLocks[idx].ownerSessionID; - - if (tableLocks[idx].ownerSessionID < minSessionID) - minSessionID = tableLocks[idx].ownerSessionID; - - if (tableLocks[idx].ownerTxnID > maxTxnID) - maxTxnID = tableLocks[idx].ownerTxnID; - - ctime_r( &tableLocks[idx].creationTime, cTimeBuffer ); - cTimeBuffer[ strlen(cTimeBuffer) - 1 ] = '\0'; // strip trailing '\n' - std::string cTimeStr( cTimeBuffer ); - - if (cTimeStr.length() > createTimeColumnWidth) - createTimeColumnWidth = cTimeStr.length(); - - createTimes.push_back( cTimeStr ); - - std::ostringstream pms; //It is dbroots now - - for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) - { - if (k > 0) - pms << ','; - - pms << tableLocks[idx].dbrootList[k]; - } - - if (pms.str().length() > pmColumnWidth) - pmColumnWidth = pms.str().length(); + pms << tableLocks[idx].dbrootList[k]; } - ownerColumnWidth += 2; - pmColumnWidth += 2; - createTimeColumnWidth += 2; + if (pms.str().length() > pmColumnWidth) + pmColumnWidth = pms.str().length(); + } - std::ostringstream idString; - idString << maxLockID; + ownerColumnWidth += 2; + pmColumnWidth += 2; + createTimeColumnWidth += 2; - if (idString.str().length() > lockIDColumnWidth) - lockIDColumnWidth = idString.str().length(); + std::ostringstream idString; + idString << maxLockID; - lockIDColumnWidth += 2; + if (idString.str().length() > lockIDColumnWidth) + lockIDColumnWidth = idString.str().length(); - std::ostringstream pidString; - pidString << maxPID; + lockIDColumnWidth += 2; - if (pidString.str().length() > pidColumnWidth) - pidColumnWidth = pidString.str().length(); + std::ostringstream pidString; + pidString << maxPID; - pidColumnWidth += 2; + if (pidString.str().length() > pidColumnWidth) + pidColumnWidth = pidString.str().length(); - const std::string sessionNoneStr("BulkLoad"); - std::ostringstream sessionString; - sessionString << maxSessionID; + pidColumnWidth += 2; - if (sessionString.str().length() > sessionIDColumnWidth) - sessionIDColumnWidth = sessionString.str().length(); + const std::string sessionNoneStr("BulkLoad"); + std::ostringstream sessionString; + sessionString << maxSessionID; - if ((minSessionID < 0) && - (sessionNoneStr.length() > sessionIDColumnWidth)) - sessionIDColumnWidth = sessionNoneStr.length(); + if (sessionString.str().length() > sessionIDColumnWidth) + sessionIDColumnWidth = sessionString.str().length(); - sessionIDColumnWidth += 2; + if ((minSessionID < 0) && (sessionNoneStr.length() > sessionIDColumnWidth)) + sessionIDColumnWidth = sessionNoneStr.length(); - const std::string txnNoneStr("n/a"); - std::ostringstream txnString; - txnString << maxTxnID; + sessionIDColumnWidth += 2; - if (txnString.str().length() > txnIDColumnWidth) - txnIDColumnWidth = txnString.str().length(); + const std::string txnNoneStr("n/a"); + std::ostringstream txnString; + txnString << maxTxnID; - txnIDColumnWidth += 2; + if (txnString.str().length() > txnIDColumnWidth) + txnIDColumnWidth = txnString.str().length(); - // Make second pass through the table locks to build our result. - // Keep in mind the same table could have more than 1 lock - // (on different PMs), so we don't exit loop after "first" match. - bool found = false; - ostringstream os; + txnIDColumnWidth += 2; - for (unsigned idx = 0; idx < tableLocks.size(); idx++) + // Make second pass through the table locks to build our result. + // Keep in mind the same table could have more than 1 lock + // (on different PMs), so we don't exit loop after "first" match. + bool found = false; + ostringstream os; + + for (unsigned idx = 0; idx < tableLocks.size(); idx++) + { + if (roPair.objnum == (CalpontSystemCatalog::OID)tableLocks[idx].tableOID) { - if (roPair.objnum == (CalpontSystemCatalog::OID) - tableLocks[idx].tableOID) - { - std::ostringstream pms; + std::ostringstream pms; - for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) - { - if (k > 0) - pms << ','; + for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) + { + if (k > 0) + pms << ','; - pms << tableLocks[idx].dbrootList[k]; - } + pms << tableLocks[idx].dbrootList[k]; + } - if (found) // write newline between lines - { - os << endl; - } - else // write the column headers before the first entry - { - os.setf(ios::left, ios::adjustfield); - os << setw(lockIDColumnWidth) << "LockID" << - setw(ownerColumnWidth) << "Process" << - setw(pidColumnWidth) << "PID" << - setw(sessionIDColumnWidth) << "Session" << - setw(txnIDColumnWidth) << "Txn" << - setw(createTimeColumnWidth) << "CreationTime" << - setw(9) << "State" << - setw(pmColumnWidth) << "DBRoots" << endl; - } + if (found) // write newline between lines + { + os << endl; + } + else // write the column headers before the first entry + { + os.setf(ios::left, ios::adjustfield); + os << setw(lockIDColumnWidth) << "LockID" << setw(ownerColumnWidth) << "Process" + << setw(pidColumnWidth) << "PID" << setw(sessionIDColumnWidth) << "Session" + << setw(txnIDColumnWidth) << "Txn" << setw(createTimeColumnWidth) << "CreationTime" << setw(9) + << "State" << setw(pmColumnWidth) << "DBRoots" << endl; + } - os << " " << - setw(lockIDColumnWidth) << tableLocks[idx].id << - setw(ownerColumnWidth) << tableLocks[idx].ownerName << - setw(pidColumnWidth) << tableLocks[idx].ownerPID; + os << " " << setw(lockIDColumnWidth) << tableLocks[idx].id << setw(ownerColumnWidth) + << tableLocks[idx].ownerName << setw(pidColumnWidth) << tableLocks[idx].ownerPID; - // Log session ID, or "BulkLoad" if session is -1 - if (tableLocks[idx].ownerSessionID < 0) - os << setw(sessionIDColumnWidth) << sessionNoneStr; - else - os << setw(sessionIDColumnWidth) << - tableLocks[idx].ownerSessionID; + // Log session ID, or "BulkLoad" if session is -1 + if (tableLocks[idx].ownerSessionID < 0) + os << setw(sessionIDColumnWidth) << sessionNoneStr; + else + os << setw(sessionIDColumnWidth) << tableLocks[idx].ownerSessionID; - // Log txn ID, or "n/a" if txn is -1 - if (tableLocks[idx].ownerTxnID < 0) - os << setw(txnIDColumnWidth) << txnNoneStr; - else - os << setw(txnIDColumnWidth) << - tableLocks[idx].ownerTxnID; + // Log txn ID, or "n/a" if txn is -1 + if (tableLocks[idx].ownerTxnID < 0) + os << setw(txnIDColumnWidth) << txnNoneStr; + else + os << setw(txnIDColumnWidth) << tableLocks[idx].ownerTxnID; - os << setw(createTimeColumnWidth) << - createTimes[idx] << - setw(9) << ((tableLocks[idx].state == BRM::LOADING) ? - "LOADING" : "CLEANUP") << - setw(pmColumnWidth) << pms.str(); + os << setw(createTimeColumnWidth) << createTimes[idx] << setw(9) + << ((tableLocks[idx].state == BRM::LOADING) ? "LOADING" : "CLEANUP") << setw(pmColumnWidth) + << pms.str(); - found = true; - } // end of displaying a table lock match - } // end of loop through all table locks + found = true; + } // end of displaying a table lock match + } // end of loop through all table locks - if (!found) - { - os << " Table " << tableName.schema << "." << - tableName.table << " is not locked by any process."; - } + if (!found) + { + os << " Table " << tableName.schema << "." << tableName.table << " is not locked by any process."; + } - result.tableLockInfo = os.str(); + result.tableLockInfo = os.str(); } //------------------------------------------------------------------------------ // Process cleartablelock command; execute bulk rollback and release table lock // for the specified table. //------------------------------------------------------------------------------ -void CommandPackageProcessor::clearTableLock( uint64_t uniqueId, - const dmlpackage::CalpontDMLPackage& cpackage, - DMLPackageProcessor::DMLResult& result) +void CommandPackageProcessor::clearTableLock(uint64_t uniqueId, const dmlpackage::CalpontDMLPackage& cpackage, + DMLPackageProcessor::DMLResult& result) { - CalpontSystemCatalog::TableName tableName; - tableName.schema = cpackage.get_SchemaName(); - tableName.table = cpackage.get_TableName (); + CalpontSystemCatalog::TableName tableName; + tableName.schema = cpackage.get_SchemaName(); + tableName.table = cpackage.get_TableName(); - // Get the Table lock ID that is passed in the SQL statement attribute. - // This is a kludge we may want to consider changing. Could add a table - // lock ID attribute to the CalpontDMLPackage object. - std::istringstream lockIDString( cpackage.get_SQLStatement() ); - uint64_t tableLockID; - lockIDString >> tableLockID; + // Get the Table lock ID that is passed in the SQL statement attribute. + // This is a kludge we may want to consider changing. Could add a table + // lock ID attribute to the CalpontDMLPackage object. + std::istringstream lockIDString(cpackage.get_SQLStatement()); + uint64_t tableLockID; + lockIDString >> tableLockID; - //----------------------------------------------------- start of syslog code - // - // Log initiation of cleartablelock to syslog - // - const std::string APPLNAME("cleartablelock SQL cmd"); - const int SUBSYSTEM_ID = 21; // dmlpackageproc - const int INIT_MSG_NUM = logging::M0088; - logging::Message::Args msgArgs; - logging::Message logMsg1( INIT_MSG_NUM ); - msgArgs.add( APPLNAME ); - msgArgs.add( tableName.toString() ); - msgArgs.add( tableLockID ); - logMsg1.format( msgArgs ); - logging::LoggingID lid( SUBSYSTEM_ID ); - logging::MessageLog ml( lid ); - ml.logInfoMessage( logMsg1 ); - //------------------------------------------------------- end of syslog code + //----------------------------------------------------- start of syslog code + // + // Log initiation of cleartablelock to syslog + // + const std::string APPLNAME("cleartablelock SQL cmd"); + const int SUBSYSTEM_ID = 21; // dmlpackageproc + const int INIT_MSG_NUM = logging::M0088; + logging::Message::Args msgArgs; + logging::Message logMsg1(INIT_MSG_NUM); + msgArgs.add(APPLNAME); + msgArgs.add(tableName.toString()); + msgArgs.add(tableLockID); + logMsg1.format(msgArgs); + logging::LoggingID lid(SUBSYSTEM_ID); + logging::MessageLog ml(lid); + ml.logInfoMessage(logMsg1); + //------------------------------------------------------- end of syslog code - boost::shared_ptr bsIn; - messageqcpp::ByteStream bsOut; - bool lockGrabbed = false; - bool bErrFlag = false; - bool bRemoveMetaErrFlag = false; - std::ostringstream combinedErrMsg; + boost::shared_ptr bsIn; + messageqcpp::ByteStream bsOut; + bool lockGrabbed = false; + bool bErrFlag = false; + bool bRemoveMetaErrFlag = false; + std::ostringstream combinedErrMsg; - try + try + { + // Make sure BRM is in READ-WRITE state before starting + int brmRc = fDbrm->isReadWrite(); + + if (brmRc != BRM::ERR_OK) { - // Make sure BRM is in READ-WRITE state before starting - int brmRc = fDbrm->isReadWrite( ); + std::string brmErrMsg; + BRM::errString(brmRc, brmErrMsg); + std::ostringstream oss; + oss << "Failed BRM status check: " << brmErrMsg; + throw std::runtime_error(oss.str()); + } - if (brmRc != BRM::ERR_OK) + BRM::TableLockInfo lockInfo; + establishTableLockToClear(tableLockID, lockInfo); + lockGrabbed = true; + + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + oam::OamCache::dbRootPMMap_t dbRootPmMap = oamCache->getDBRootToPMMap(); + std::map::const_iterator mapIter; + std::set pmSet; + + // Construct relevant list of PMs based on the DBRoots associated + // with the tableLock. + for (unsigned int k = 0; k < lockInfo.dbrootList.size(); k++) + { + mapIter = dbRootPmMap->find(lockInfo.dbrootList[k]); + + if (mapIter != dbRootPmMap->end()) + { + int pm = mapIter->second; + pmSet.insert(pm); + } + else + { + std::ostringstream oss; + oss << "DBRoot " << lockInfo.dbrootList[k] << " does not map to a PM. Cannot perform rollback"; + throw std::runtime_error(oss.str()); + } + } + + std::vector pmList; + + for (std::set::const_iterator setIter = pmSet.begin(); setIter != pmSet.end(); ++setIter) + { + pmList.push_back(*setIter); + } + + std::cout << "cleartablelock rollback for table lock " << tableLockID << " being forwarded to PM(s): "; + + for (unsigned int k = 0; k < pmList.size(); k++) + { + if (k > 0) + std::cout << ", "; + + std::cout << pmList[k]; + } + + std::cout << std::endl; + + // Perform bulk rollback if state is in LOADING state + if (lockInfo.state == BRM::LOADING) + { + fWEClient->addQueue(uniqueId); + + //------------------------------------------------------------------ + // Send rollback msg to the writeengine server for every PM. + // We send to each PM, instead of just to PMs in the tablelock's + // PM list, just in case a DBRoot has been moved to a different PM. + //------------------------------------------------------------------ + // std::cout << "cleartablelock rollback: tableLock-" << tableLockID << + // ": uniqueID-" << uniqueId << + // ": oid-" << lockInfo.tableOID << + // "; name-" << tableName.toString() << + // "; app-" << APPLNAME << std::endl; + + bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK; + bsOut << uniqueId; + bsOut << tableLockID; + bsOut << lockInfo.tableOID; + bsOut << tableName.toString(); + bsOut << APPLNAME; + + for (unsigned j = 0; j < pmList.size(); j++) + { + fWEClient->write(bsOut, pmList[j]); + } + + // Wait for "all" the responses, and accumulate any/all errors + unsigned int pmMsgCnt = 0; + + while (pmMsgCnt < pmList.size()) + { + std::string rollbackErrMsg; + bsIn.reset(new messageqcpp::ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) { - std::string brmErrMsg; - BRM::errString( brmRc, brmErrMsg ); - std::ostringstream oss; - oss << "Failed BRM status check: " << brmErrMsg; - throw std::runtime_error( oss.str() ); + bRemoveMetaErrFlag = true; + + if (combinedErrMsg.str().length() > 0) + combinedErrMsg << std::endl; + + combinedErrMsg << "Network error, PM rollback; "; } - - BRM::TableLockInfo lockInfo; - establishTableLockToClear( tableLockID, lockInfo ); - lockGrabbed = true; - - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - oam::OamCache::dbRootPMMap_t dbRootPmMap = oamCache->getDBRootToPMMap(); - std::map::const_iterator mapIter; - std::set pmSet; - - // Construct relevant list of PMs based on the DBRoots associated - // with the tableLock. - for (unsigned int k = 0; k < lockInfo.dbrootList.size(); k++) + else { - mapIter = dbRootPmMap->find( lockInfo.dbrootList[k] ); + messageqcpp::ByteStream::byte rc; + uint16_t pmNum; + *bsIn >> rc; + *bsIn >> rollbackErrMsg; + *bsIn >> pmNum; - if (mapIter != dbRootPmMap->end()) - { - int pm = mapIter->second; - pmSet.insert( pm ); - } + // std::cout << "cleartablelock rollback response from PM"<< + // pmNum << "; rc-" << (int)rc << + // "; errMsg: {" << rollbackErrMsg << '}' << std::endl; + + if (rc != 0) + { + bRemoveMetaErrFlag = true; + + if (combinedErrMsg.str().empty()) + combinedErrMsg << "Rollback error; "; else - { - std::ostringstream oss; - oss << "DBRoot " << lockInfo.dbrootList[k] << - " does not map to a PM. Cannot perform rollback"; - throw std::runtime_error( oss.str() ); - } + combinedErrMsg << std::endl; + + combinedErrMsg << "[PM" << pmNum << "] " << rollbackErrMsg; + } } - std::vector pmList; + pmMsgCnt++; + } // end of while loop to process all responses to bulk rollback - for (std::set::const_iterator setIter = pmSet.begin(); - setIter != pmSet.end(); - ++setIter) - { - pmList.push_back( *setIter ); - } - - std::cout << "cleartablelock rollback for table lock " << tableLockID << - " being forwarded to PM(s): "; - - for (unsigned int k = 0; k < pmList.size(); k++) - { - if (k > 0) - std::cout << ", "; - - std::cout << pmList[k]; - } - - std::cout << std::endl; - - // Perform bulk rollback if state is in LOADING state - if (lockInfo.state == BRM::LOADING) - { - fWEClient->addQueue(uniqueId); - - //------------------------------------------------------------------ - // Send rollback msg to the writeengine server for every PM. - // We send to each PM, instead of just to PMs in the tablelock's - // PM list, just in case a DBRoot has been moved to a different PM. - //------------------------------------------------------------------ -// std::cout << "cleartablelock rollback: tableLock-" << tableLockID << -// ": uniqueID-" << uniqueId << -// ": oid-" << lockInfo.tableOID << -// "; name-" << tableName.toString() << -// "; app-" << APPLNAME << std::endl; - - bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK; - bsOut << uniqueId; - bsOut << tableLockID; - bsOut << lockInfo.tableOID; - bsOut << tableName.toString(); - bsOut << APPLNAME; - - for (unsigned j = 0; j < pmList.size(); j++) - { - fWEClient->write(bsOut, pmList[j]); - } - - // Wait for "all" the responses, and accumulate any/all errors - unsigned int pmMsgCnt = 0; - - while (pmMsgCnt < pmList.size()) - { - std::string rollbackErrMsg; - bsIn.reset(new messageqcpp::ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if (bsIn->length() == 0) - { - bRemoveMetaErrFlag = true; - - if (combinedErrMsg.str().length() > 0) - combinedErrMsg << std::endl; - - combinedErrMsg << "Network error, PM rollback; "; - } - else - { - messageqcpp::ByteStream::byte rc; - uint16_t pmNum; - *bsIn >> rc; - *bsIn >> rollbackErrMsg; - *bsIn >> pmNum; - -// std::cout << "cleartablelock rollback response from PM"<< -// pmNum << "; rc-" << (int)rc << -// "; errMsg: {" << rollbackErrMsg << '}' << std::endl; - - if (rc != 0) - { - bRemoveMetaErrFlag = true; - - if (combinedErrMsg.str().empty()) - combinedErrMsg << "Rollback error; "; - else - combinedErrMsg << std::endl; - - combinedErrMsg << "[PM" << pmNum << "] " << - rollbackErrMsg; - } - } - - pmMsgCnt++; - } // end of while loop to process all responses to bulk rollback - - // If no errors so far, then change state to CLEANUP state. - // We ignore the return stateChange flag. - if (!bErrFlag) - { - fDbrm->changeState( tableLockID, BRM::CLEANUP ); - } - } // end of (lockInfo.state == BRM::LOADING) - - // If no errors so far, then: - // 1. delete meta data backup rollback files - // 2. finally release table lock - if (!bErrFlag) - { - bsOut.reset(); - - //------------------------------------------------------------------ - // Delete meta data backup rollback files - //------------------------------------------------------------------ -// std::cout << "cleartablelock cleanup: uniqueID-" << uniqueId << -// ": oid-" << lockInfo.tableOID << std::endl; - - bsOut << (messageqcpp::ByteStream::byte) - WE_SVR_DML_BULKROLLBACK_CLEANUP; - bsOut << uniqueId; - bsOut << lockInfo.tableOID; - - for (unsigned j = 0; j < pmList.size(); j++) - { - fWEClient->write(bsOut, pmList[j]); - } - - // Wait for "all" the responses, and accumulate any/all errors - unsigned int pmMsgCnt = 0; - - while (pmMsgCnt < pmList.size()) - { - std::string fileDeleteErrMsg; - bsIn.reset(new messageqcpp::ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if (bsIn->length() == 0) - { - bRemoveMetaErrFlag = true; - - if (combinedErrMsg.str().length() > 0) - combinedErrMsg << std::endl; - - combinedErrMsg << "Network error, PM rollback cleanup; "; - } - else - { - messageqcpp::ByteStream::byte rc; - uint16_t pmNum; - *bsIn >> rc; - *bsIn >> fileDeleteErrMsg; - *bsIn >> pmNum; - -// std::cout << "cleartablelock cleanup response from PM" << -// pmNum << "; rc-" << (int)rc << -// "; errMsg: {" << fileDeleteErrMsg << '}' << std::endl; - - if (rc != 0) - { - bRemoveMetaErrFlag = true; - - if (combinedErrMsg.str().empty()) - combinedErrMsg << "Cleanup error; "; - else - combinedErrMsg << std::endl; - - combinedErrMsg << "[PM" << pmNum << "] " << - fileDeleteErrMsg; - } - } - - pmMsgCnt++; - } // end of while loop to process all responses to rollback cleanup - - // We ignore return release flag from releaseTableLock(). - if (!bErrFlag) - { - fDbrm->releaseTableLock( tableLockID ); - } - } - } - catch (std::exception& ex) - { - bErrFlag = true; - combinedErrMsg << ex.what(); - } + // If no errors so far, then change state to CLEANUP state. + // We ignore the return stateChange flag. + if (!bErrFlag) + { + fDbrm->changeState(tableLockID, BRM::CLEANUP); + } + } // end of (lockInfo.state == BRM::LOADING) + // If no errors so far, then: + // 1. delete meta data backup rollback files + // 2. finally release table lock if (!bErrFlag) { - std::ostringstream oss; - oss << "Table lock " << tableLockID << " for table " << - tableName.toString() << " is cleared."; + bsOut.reset(); - //@Bug 4517. Release tablelock if remove meta files failed. - if (bRemoveMetaErrFlag) + //------------------------------------------------------------------ + // Delete meta data backup rollback files + //------------------------------------------------------------------ + // std::cout << "cleartablelock cleanup: uniqueID-" << uniqueId << + // ": oid-" << lockInfo.tableOID << std::endl; + + bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK_CLEANUP; + bsOut << uniqueId; + bsOut << lockInfo.tableOID; + + for (unsigned j = 0; j < pmList.size(); j++) + { + fWEClient->write(bsOut, pmList[j]); + } + + // Wait for "all" the responses, and accumulate any/all errors + unsigned int pmMsgCnt = 0; + + while (pmMsgCnt < pmList.size()) + { + std::string fileDeleteErrMsg; + bsIn.reset(new messageqcpp::ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) { - oss << " Warning: " << combinedErrMsg.str(); + bRemoveMetaErrFlag = true; + + if (combinedErrMsg.str().length() > 0) + combinedErrMsg << std::endl; + + combinedErrMsg << "Network error, PM rollback cleanup; "; + } + else + { + messageqcpp::ByteStream::byte rc; + uint16_t pmNum; + *bsIn >> rc; + *bsIn >> fileDeleteErrMsg; + *bsIn >> pmNum; + + // std::cout << "cleartablelock cleanup response from PM" << + // pmNum << "; rc-" << (int)rc << + // "; errMsg: {" << fileDeleteErrMsg << '}' << std::endl; + + if (rc != 0) + { + bRemoveMetaErrFlag = true; + + if (combinedErrMsg.str().empty()) + combinedErrMsg << "Cleanup error; "; + else + combinedErrMsg << std::endl; + + combinedErrMsg << "[PM" << pmNum << "] " << fileDeleteErrMsg; + } } - result.tableLockInfo = oss.str(); + pmMsgCnt++; + } // end of while loop to process all responses to rollback cleanup + + // We ignore return release flag from releaseTableLock(). + if (!bErrFlag) + { + fDbrm->releaseTableLock(tableLockID); + } } - else + } + catch (std::exception& ex) + { + bErrFlag = true; + combinedErrMsg << ex.what(); + } + + if (!bErrFlag) + { + std::ostringstream oss; + oss << "Table lock " << tableLockID << " for table " << tableName.toString() << " is cleared."; + + //@Bug 4517. Release tablelock if remove meta files failed. + if (bRemoveMetaErrFlag) { - std::ostringstream oss; - oss << "Table lock " << tableLockID << " for table " << - tableName.toString() << " cannot be cleared. " << - combinedErrMsg.str(); - result.tableLockInfo = oss.str(); + oss << " Warning: " << combinedErrMsg.str(); } - // Remove tableLockID out of the active cleartableLock command list - if (lockGrabbed) - { - boost::mutex::scoped_lock lock( fActiveClearTableLockCmdMutex ); - fActiveClearTableLockCmds.erase( tableLockID ); - } + result.tableLockInfo = oss.str(); + } + else + { + std::ostringstream oss; + oss << "Table lock " << tableLockID << " for table " << tableName.toString() << " cannot be cleared. " + << combinedErrMsg.str(); + result.tableLockInfo = oss.str(); + } - //----------------------------------------------------- start of syslog code - // - // Log completion of cleartablelock to syslog - // - const int COMP_MSG_NUM = logging::M0089; - msgArgs.reset(); - logging::Message logMsg2( COMP_MSG_NUM ); - msgArgs.add( APPLNAME ); - msgArgs.add( tableName.toString() ); - msgArgs.add( tableLockID ); - std::string finalStatus; + // Remove tableLockID out of the active cleartableLock command list + if (lockGrabbed) + { + boost::mutex::scoped_lock lock(fActiveClearTableLockCmdMutex); + fActiveClearTableLockCmds.erase(tableLockID); + } - if (!bErrFlag) - { - finalStatus = "Completed successfully"; - } - else - { - finalStatus = "Encountered errors: "; - finalStatus += combinedErrMsg.str(); - } + //----------------------------------------------------- start of syslog code + // + // Log completion of cleartablelock to syslog + // + const int COMP_MSG_NUM = logging::M0089; + msgArgs.reset(); + logging::Message logMsg2(COMP_MSG_NUM); + msgArgs.add(APPLNAME); + msgArgs.add(tableName.toString()); + msgArgs.add(tableLockID); + std::string finalStatus; - msgArgs.add( finalStatus ); - logMsg2.format( msgArgs ); - ml.logInfoMessage( logMsg2 ); - //------------------------------------------------------- end of syslog code + if (!bErrFlag) + { + finalStatus = "Completed successfully"; + } + else + { + finalStatus = "Encountered errors: "; + finalStatus += combinedErrMsg.str(); + } + + msgArgs.add(finalStatus); + logMsg2.format(msgArgs); + ml.logInfoMessage(logMsg2); + //------------------------------------------------------- end of syslog code } //------------------------------------------------------------------------------ @@ -1119,54 +1094,50 @@ void CommandPackageProcessor::clearTableLock( uint64_t uniqueId, // lock to ourselves if we can. If the lock is still in use by another process // or by another DML cleartablelock thread, then we error out with exception. //------------------------------------------------------------------------------ -void CommandPackageProcessor::establishTableLockToClear( uint64_t tableLockID, - BRM::TableLockInfo& lockInfo ) +void CommandPackageProcessor::establishTableLockToClear(uint64_t tableLockID, BRM::TableLockInfo& lockInfo) { - boost::mutex::scoped_lock lock( fActiveClearTableLockCmdMutex ); + boost::mutex::scoped_lock lock(fActiveClearTableLockCmdMutex); - // Get current table lock info - bool getLockInfo = fDbrm->getTableLockInfo(tableLockID, &lockInfo); + // Get current table lock info + bool getLockInfo = fDbrm->getTableLockInfo(tableLockID, &lockInfo); - if (!getLockInfo) + if (!getLockInfo) + { + throw std::runtime_error(std::string("Lock does not exist.")); + } + + std::string processName("DMLProc clearTableLock"); + uint32_t processID = ::getpid(); + + // See if another thread is executing a cleartablelock cmd for this table + if ((lockInfo.ownerName == processName) && (lockInfo.ownerPID == processID)) + { + std::set::const_iterator it = fActiveClearTableLockCmds.find(tableLockID); + + if (it != fActiveClearTableLockCmds.end()) { - throw std::runtime_error( std::string("Lock does not exist.") ); + throw std::runtime_error( + std::string("Lock in use. " + "DML is executing another cleartablelock MySQL cmd.")); } + } + else + { + // Take over ownership of stale lock. + // Use "DMLProc clearTableLock" as process name to differentiate + // from a DMLProc lock used for inserts, updates, etc. + int32_t sessionID = fSessionID; + int32_t txnid = -1; + bool ownerChanged = fDbrm->changeOwner(tableLockID, processName, processID, sessionID, txnid); - std::string processName("DMLProc clearTableLock"); - uint32_t processID = ::getpid(); - - // See if another thread is executing a cleartablelock cmd for this table - if ((lockInfo.ownerName == processName) && - (lockInfo.ownerPID == processID)) + if (!ownerChanged) { - std::set::const_iterator it = - fActiveClearTableLockCmds.find( tableLockID ); - - if (it != fActiveClearTableLockCmds.end()) - { - throw std::runtime_error( std::string( "Lock in use. " - "DML is executing another cleartablelock MySQL cmd.") ); - } + throw std::runtime_error(std::string("Unable to grab lock; lock not found or still in use.")); } - else - { - // Take over ownership of stale lock. - // Use "DMLProc clearTableLock" as process name to differentiate - // from a DMLProc lock used for inserts, updates, etc. - int32_t sessionID = fSessionID; - int32_t txnid = -1; - bool ownerChanged = fDbrm->changeOwner( - tableLockID, processName, processID, sessionID, txnid); + } - if (!ownerChanged) - { - throw std::runtime_error( std::string( - "Unable to grab lock; lock not found or still in use.") ); - } - } - - // Add this cleartablelock command to the list of active cleartablelock cmds - fActiveClearTableLockCmds.insert( tableLockID ); + // Add this cleartablelock command to the list of active cleartablelock cmds + fActiveClearTableLockCmds.insert(tableLockID); } -} // namespace dmlpackageprocessor +} // namespace dmlpackageprocessor diff --git a/dbcon/dmlpackageproc/commandpackageprocessor.h b/dbcon/dmlpackageproc/commandpackageprocessor.h index b0180051f..fc272fd41 100644 --- a/dbcon/dmlpackageproc/commandpackageprocessor.h +++ b/dbcon/dmlpackageproc/commandpackageprocessor.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: commandpackageprocessor.h 9302 2013-03-07 16:06:59Z chao $ -* -* -***********************************************************************/ + * $Id: commandpackageprocessor.h 9302 2013-03-07 16:06:59Z chao $ + * + * + ***********************************************************************/ /** @file */ #ifndef COMMANDPACKAGEPROCESSOR_H @@ -43,39 +43,34 @@ namespace dmlpackageprocessor { /** @brief concrete implementation of a DMLPackageProcessor. - * Specifically for interacting with the Write Engine to - * process INSERT dml statements. - */ + * Specifically for interacting with the Write Engine to + * process INSERT dml statements. + */ class CommandPackageProcessor : public DMLPackageProcessor { + public: + CommandPackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) + { + } + /** @brief process an CommandDMLPackage + * + * @param cpackage the CommandDMLPackage to process + */ + EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); -public: - CommandPackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) {} - /** @brief process an CommandDMLPackage - * - * @param cpackage the CommandDMLPackage to process - */ - EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); + protected: + private: + void viewTableLock(const dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result); + void clearTableLock(uint64_t uniqueId, const dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result); + void establishTableLockToClear(uint64_t tableLockID, BRM::TableLockInfo& lockInfo); -protected: - -private: - void viewTableLock(const dmlpackage::CalpontDMLPackage& cpackage, - DMLResult& result ); - void clearTableLock(uint64_t uniqueId, - const dmlpackage::CalpontDMLPackage& cpackage, - DMLResult& result ); - void establishTableLockToClear(uint64_t tableLockID, - BRM::TableLockInfo& lockInfo); - - // Tracks active cleartablelock commands by storing set of table lock IDs - static std::set fActiveClearTableLockCmds; - static boost::mutex fActiveClearTableLockCmdMutex; + // Tracks active cleartablelock commands by storing set of table lock IDs + static std::set fActiveClearTableLockCmds; + static boost::mutex fActiveClearTableLockCmdMutex; }; -} +} // namespace dmlpackageprocessor #undef EXPORT -#endif //COMMANDPACKAGEPROCESSOR_H - +#endif // COMMANDPACKAGEPROCESSOR_H diff --git a/dbcon/dmlpackageproc/deletepackageprocessor.cpp b/dbcon/dmlpackageproc/deletepackageprocessor.cpp index 66ae687ea..92295cec7 100644 --- a/dbcon/dmlpackageproc/deletepackageprocessor.cpp +++ b/dbcon/dmlpackageproc/deletepackageprocessor.cpp @@ -57,549 +57,719 @@ using namespace messageqcpp; using namespace oam; namespace dmlpackageprocessor { -DMLPackageProcessor::DMLResult -DeletePackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) +DMLPackageProcessor::DMLResult DeletePackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) { - SUMMARY_INFO("DeletePackageProcessor::processPackage"); + SUMMARY_INFO("DeletePackageProcessor::processPackage"); - DMLResult result; - result.result = NO_ERROR; - BRM::TxnID txnid; - // set-up the transaction - txnid.id = cpackage.get_TxnID(); - txnid.valid = true; - fSessionID = cpackage.get_SessionID(); - //StopWatch timer; - VERBOSE_INFO("DeletePackageProcessor is processing CalpontDMLPackage ..."); - TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - uint64_t uniqueId = 0; + DMLResult result; + result.result = NO_ERROR; + BRM::TxnID txnid; + // set-up the transaction + txnid.id = cpackage.get_TxnID(); + txnid.valid = true; + fSessionID = cpackage.get_SessionID(); + // StopWatch timer; + VERBOSE_INFO("DeletePackageProcessor is processing CalpontDMLPackage ..."); + TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); + uint64_t uniqueId = 0; - //Bug 5070. Added exception handling - try + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(9); + args.add(ex.what()); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + catch (...) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + + uint64_t tableLockId = 0; + // get the table object from the package + DMLTable* tablePtr = cpackage.get_Table(); + std::string schemaName = tablePtr->get_SchemaName(); + std::string tableName = tablePtr->get_TableName(); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + CalpontSystemCatalog::TableName aTableName; + aTableName.table = tableName; + aTableName.schema = schemaName; + fWEClient->addQueue(uniqueId); + execplan::CalpontSystemCatalog::ROPair roPair; + + try + { + string stmt = cpackage.get_SQLStatement() + "|" + schemaName + "|"; + SQLLogger sqlLogger(stmt, DMLLoggingId, fSessionID, txnid.id); + + if (0 != tablePtr) { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(9); - args.add(ex.what()); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; - } - catch ( ... ) - { - logging::Message::Args args; - logging::Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; - } + roPair = csc->tableRID(aTableName); - uint64_t tableLockId = 0; - // get the table object from the package - DMLTable* tablePtr = cpackage.get_Table(); - std::string schemaName = tablePtr->get_SchemaName(); - std::string tableName = tablePtr->get_TableName(); - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog( fSessionID ); - CalpontSystemCatalog::TableName aTableName; - aTableName.table = tableName; - aTableName.schema = schemaName; - fWEClient->addQueue(uniqueId); - execplan::CalpontSystemCatalog::ROPair roPair; + tableLockId = tablelockData->getTablelockId( + roPair.objnum); // check whether this table is locked already for this session - try - { - string stmt = cpackage.get_SQLStatement() + "|" + schemaName + "|"; - SQLLogger sqlLogger(stmt, DMLLoggingId, fSessionID, txnid.id); + if (tableLockId == 0) + { + // cout << "tablelock is not found in cache " << endl; + uint32_t processID = ::getpid(); + int32_t txnId = txnid.id; + std::string processName("DMLProc"); + int32_t sessionId = fSessionID; + int i = 0; + OamCache* oamcache = OamCache::makeOamCache(); + std::vector pmList = oamcache->getModuleIds(); + std::vector pms; - if ( 0 != tablePtr ) + for (unsigned i = 0; i < pmList.size(); i++) { + pms.push_back((uint32_t)pmList[i]); + } - roPair = csc->tableRID(aTableName); + try + { + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, + BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } - tableLockId = tablelockData->getTablelockId(roPair.objnum); //check whether this table is locked already for this session + if (tableLockId == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 30; // try 30 times (3 seconds) + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; - if (tableLockId == 0) - { - //cout << "tablelock is not found in cache " << endl; - uint32_t processID = ::getpid(); - int32_t txnId = txnid.id; - std::string processName("DMLProc"); - int32_t sessionId = fSessionID; - int i = 0; - OamCache* oamcache = OamCache::makeOamCache(); - std::vector pmList = oamcache->getModuleIds(); - std::vector pms; + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - for (unsigned i = 0; i < pmList.size(); i++) - { - pms.push_back((uint32_t)pmList[i]); - } - - try - { - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if ( tableLockId == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 30; // try 30 times (3 seconds) - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { + for (; i < numTries; i++) + { #ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); + Sleep(rm_ts.tv_sec * 1000); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif - try - { - processID = ::getpid(); - txnId = txnid.id; - sessionId = fSessionID; - processName = "DMLProc"; - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (tableLockId > 0) - break; - } - - if (i >= numTries) //error out - { - result.result = DELETE_ERROR; - logging::Message::Args args; - string strOp("delete"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add(sessionId); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - } - } - } - - //cout << " tablelock is obtained with id " << tableLockId << endl; - tablelockData->setTablelock(roPair.objnum, tableLockId); - //@Bug 4491 start AI sequence for autoincrement column - const CalpontSystemCatalog::RIDList ridList = csc->columnRIDs(aTableName); - CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); - CalpontSystemCatalog::ColType colType; - - while (rid_iterator != ridList.end()) + try { - // If user hit ctrl+c in the mysql console, fRollbackPending will be true. - if (fRollbackPending) - { - result.result = JOB_CANCELED; - break; - } - - CalpontSystemCatalog::ROPair roPair = *rid_iterator; - colType = csc->colType(roPair.objnum); - - if (colType.autoincrement) - { - try - { - uint64_t nextVal = csc->nextAutoIncrValue(aTableName); - fDbrm->startAISequence(roPair.objnum, nextVal, colType.colWidth, colType.colDataType); - break; //Only one autoincrement column per table - } - catch (std::exception& ex) - { - result.result = DELETE_ERROR; - throw std::runtime_error(ex.what()); - } - } - - ++rid_iterator; + processID = ::getpid(); + txnId = txnid.id; + sessionId = fSessionID; + processName = "DMLProc"; + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, + &txnId, BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); } - uint64_t rowsProcessed = 0; + if (tableLockId > 0) + break; + } - rowsProcessed = fixUpRows(cpackage, result, uniqueId, roPair.objnum); - - //@Bug 4994 Cancelled job is not error - if (result.result == JOB_CANCELED) - throw std::runtime_error("Query execution was interrupted"); - - if ((result.result != 0) && (result.result != DMLPackageProcessor::IDBRANGE_WARNING)) - throw std::runtime_error(result.message.msg()); - - result.rowCount = rowsProcessed; - - // Log the DML statement. - logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement(), cpackage.get_SchemaName()); - } - } - catch (exception& ex) - { - cerr << "DeletePackageProcessor::processPackage: " << ex.what() << endl; - - //@Bug 4994 Cancelled job is not error - if (result.result == 0) - { + if (i >= numTries) // error out + { result.result = DELETE_ERROR; + logging::Message::Args args; + string strOp("delete"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add(sessionId); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + } + } + } + + // cout << " tablelock is obtained with id " << tableLockId << endl; + tablelockData->setTablelock(roPair.objnum, tableLockId); + //@Bug 4491 start AI sequence for autoincrement column + const CalpontSystemCatalog::RIDList ridList = csc->columnRIDs(aTableName); + CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); + CalpontSystemCatalog::ColType colType; + + while (rid_iterator != ridList.end()) + { + // If user hit ctrl+c in the mysql console, fRollbackPending will be true. + if (fRollbackPending) + { + result.result = JOB_CANCELED; + break; } - result.message = Message(ex.what()); + CalpontSystemCatalog::ROPair roPair = *rid_iterator; + colType = csc->colType(roPair.objnum); + + if (colType.autoincrement) + { + try + { + uint64_t nextVal = csc->nextAutoIncrValue(aTableName); + fDbrm->startAISequence(roPair.objnum, nextVal, colType.colWidth, colType.colDataType); + break; // Only one autoincrement column per table + } + catch (std::exception& ex) + { + result.result = DELETE_ERROR; + throw std::runtime_error(ex.what()); + } + } + + ++rid_iterator; + } + + uint64_t rowsProcessed = 0; + + rowsProcessed = fixUpRows(cpackage, result, uniqueId, roPair.objnum); + + //@Bug 4994 Cancelled job is not error + if (result.result == JOB_CANCELED) + throw std::runtime_error("Query execution was interrupted"); + + if ((result.result != 0) && (result.result != DMLPackageProcessor::IDBRANGE_WARNING)) + throw std::runtime_error(result.message.msg()); + + result.rowCount = rowsProcessed; + + // Log the DML statement. + logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement(), + cpackage.get_SchemaName()); } - catch (...) + } + catch (exception& ex) + { + cerr << "DeletePackageProcessor::processPackage: " << ex.what() << endl; + + //@Bug 4994 Cancelled job is not error + if (result.result == 0) { - cerr << "DeletePackageProcessor::processPackage: caught unknown exception!" << endl; + result.result = DELETE_ERROR; + } + + result.message = Message(ex.what()); + } + catch (...) + { + cerr << "DeletePackageProcessor::processPackage: caught unknown exception!" << endl; + logging::Message::Args args; + logging::Message message(7); + args.add("Delete Failed: "); + args.add("encountered unknown exception"); + args.add(result.message.msg()); + args.add(""); + message.format(args); + + result.result = DELETE_ERROR; + result.message = message; + } + + // timer.finish(); + //@Bug 1886,2870 Flush VM cache only once per statement. + std::map oids; + int rc = 0; + + if (result.result == NO_ERROR) + { + rc = flushDataFiles(result.result, oids, uniqueId, txnid, roPair.objnum); + + if (rc != NO_ERROR) + { + cerr << "UpdatePackageProcessor::processPackage: write data to disk failed" << endl; + + if (!fRollbackPending) + { logging::Message::Args args; logging::Message message(7); - args.add( "Delete Failed: "); - args.add( "encountered unknown exception" ); - args.add(result.message.msg()); + args.add("Delete Failed: "); + args.add("error when writing data to disk"); args.add(""); - message.format( args ); + args.add(""); + message.format(args); - result.result = DELETE_ERROR; + result.result = UPDATE_ERROR; result.message = message; - } + } - //timer.finish(); - //@Bug 1886,2870 Flush VM cache only once per statement. - std::map oids; - int rc = 0; + result.rowCount = 0; + rc = endTransaction(uniqueId, txnid, false); - if (result.result == NO_ERROR) - { - rc = flushDataFiles( result.result, oids, uniqueId, txnid, roPair.objnum); + if ((rc != NO_ERROR) && (!fRollbackPending)) + { + logging::Message::Args args; + logging::Message message(7); + args.add("Delete Failed: "); + args.add("error when cleaning up data files"); + args.add(""); + args.add(""); + message.format(args); - if (rc != NO_ERROR) - { - cerr << "UpdatePackageProcessor::processPackage: write data to disk failed" << endl; - - if (!fRollbackPending) - { - logging::Message::Args args; - logging::Message message(7); - args.add("Delete Failed: "); - args.add("error when writing data to disk"); - args.add(""); - args.add(""); - message.format(args); - - result.result = UPDATE_ERROR; - result.message = message; - } - - result.rowCount = 0; - rc = endTransaction(uniqueId, txnid, false); - - if ( (rc != NO_ERROR) && (!fRollbackPending)) - { - logging::Message::Args args; - logging::Message message(7); - args.add("Delete Failed: "); - args.add("error when cleaning up data files"); - args.add(""); - args.add(""); - message.format(args); - - result.result = UPDATE_ERROR; - result.message = message; - result.rowCount = 0; - - } - } - else - { - if (fRollbackPending) - rc = endTransaction(uniqueId, txnid, false); - else - rc = endTransaction(uniqueId, txnid, true); - } + result.result = UPDATE_ERROR; + result.message = message; + result.rowCount = 0; + } } else { - rc = flushDataFiles( result.result, oids, uniqueId, txnid, roPair.objnum); - result.rowCount = 0; + if (fRollbackPending) rc = endTransaction(uniqueId, txnid, false); + else + rc = endTransaction(uniqueId, txnid, true); + } + } + else + { + rc = flushDataFiles(result.result, oids, uniqueId, txnid, roPair.objnum); + result.rowCount = 0; + rc = endTransaction(uniqueId, txnid, false); + } + + if (fRollbackPending) + { + result.result = JOB_CANCELED; + logging::Message::Args args1; + args1.add("Query execution was interrupted"); + result.message.format(args1); + } + + fWEClient->removeQueue(uniqueId); + + VERBOSE_INFO("Finished Processing Delete DML Package"); + return result; +} + +uint64_t DeletePackageProcessor::fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, + const uint64_t uniqueId, const uint32_t tableOid) +{ + ByteStream msg, msgBk, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + boost::scoped_ptr rowGroup; + uint64_t rowsProcessed = 0; + uint32_t dbroot = 1; + bool metaData = false; + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + std::vector fPMs = oamCache->getModuleIds(); + std::map pmStateDel; + string emsg; + string emsgStr; + bool err = false; + + // boost::scoped_ptr fExeMgr; + // fExeMgr.reset( new messageqcpp::MessageQueueClient("ExeMgr1")); + try + { + for (unsigned i = 0; i < fPMs.size(); i++) + { + pmStateDel[fPMs[i]] = true; } + fExeMgr->write(msg); + fExeMgr->write(*(cpackage.get_ExecutionPlan())); + // cout << "sending to ExeMgr plan with length " << (cpackage.get_ExecutionPlan())->length() << endl; + msg.restart(); + emsgBs.restart(); + msg = fExeMgr->read(); // error handling + + if (msg.length() == 4) + { + msg >> qb; + + if (qb != 0) + err = true; + } + else + { + qb = 999; + err = true; + } + + if (err) + { + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: Error from ExeMgr"); + args.add((int)qb); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + return 0; + } + + emsgBs = fExeMgr->read(); + + if (emsgBs.length() == 0) + { + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: "); + args.add("Lost connection to ExeMgr"); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + return 0; + } + + emsgBs >> emsgStr; + + while (true) + { + if (fRollbackPending) + { + result.result = JOB_CANCELED; + err = true; + break; + } + + msg.restart(); + msgBk.restart(); + msg = fExeMgr->read(); + msgBk = msg; + + if (msg.length() == 0) + { + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: "); + args.add("Lost connection to ExeMgr"); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + // return rowsProcessed; + break; + } + else + { + if (rowGroup.get() == NULL) + { + // This is meta data, need to send to all PMs. + metaData = true; + // cout << "sending meta data" << endl; + err = processRowgroup(msgBk, result, uniqueId, cpackage, pmStateDel, metaData, dbroot); + rowGroup.reset(new rowgroup::RowGroup()); + rowGroup->deserialize(msg); + qb = 100; + msg.restart(); + msg << qb; + // cout << "Projecting rows" << endl; + fExeMgr->write(msg); + metaData = false; + continue; + } + + // XXXST: take out the 'true' when all jobsteps have been made st-compatible + rgData.deserialize(msg, true); + rowGroup->setData(&rgData); + // rowGroup->setData(const_cast(msg.buf())); + err = (rowGroup->getStatus() != 0); + + if (err) + { + // msgBk.advance(rowGroup->getDataSize()); + string errorMsg; + msg >> errorMsg; + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: "); + args.add(errorMsg); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + DMLResult tmpResult; + receiveAll(tmpResult, uniqueId, fPMs, pmStateDel, tableOid); + //@Bug 4358 get rid of broken pipe error. + // msg.restart(); + // msg << qb; + // fExeMgr->write(msg); + // return rowsProcessed; + break; + } + + if (rowGroup->getRGData() == NULL) + { + msg.restart(); + } + + if (rowGroup->getRowCount() == 0) // done fetching + { + err = receiveAll(result, uniqueId, fPMs, pmStateDel, tableOid); + // return rowsProcessed; + break; + } + + if (rowGroup->getBaseRid() == (uint64_t)(-1)) + { + continue; // @bug4247, not valid row ids, may from small side outer + } + + dbroot = rowGroup->getDBRoot(); + err = processRowgroup(msgBk, result, uniqueId, cpackage, pmStateDel, metaData, dbroot); + + if (err) + { + DMLResult tmpResult; + receiveAll(tmpResult, uniqueId, fPMs, pmStateDel, tableOid); + //@Bug 4358 get rid of broken pipe error. + // msg.restart(); + // msg << qb; + // fExeMgr->write(msg); + // return rowsProcessed; + break; + } + + rowsProcessed += rowGroup->getRowCount(); + } + } if (fRollbackPending) { - result.result = JOB_CANCELED; - logging::Message::Args args1; - args1.add("Query execution was interrupted"); - result.message.format(args1); + err = true; + // Response to user + cerr << "DeletePackageProcessor::processPackage::fixupRows Rollback Pending" << endl; + + //@Bug 4994 Cancelled job is not error + result.result = JOB_CANCELED; + // Log + LoggingID logid(DMLLoggingId, fSessionID, cpackage.get_TxnID()); + logging::Message::Args args1; + logging::Message msg1(1); + args1.add("SQL statement canceled by user"); + msg1.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); + + // Clean out the pipe; + DMLResult tmpResult; + receiveAll(tmpResult, uniqueId, fPMs, pmStateDel, tableOid); } - fWEClient->removeQueue(uniqueId); + if (!err) + { + // get stats from ExeMgr + qb = 3; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + msg = fExeMgr->read(); + msg >> result.queryStats; + msg >> result.extendedStats; + msg >> result.miniStats; + result.stats.unserialize(msg); + } - VERBOSE_INFO("Finished Processing Delete DML Package"); - return result; + //@Bug 4358 get rid of broken pipe error by sending a dummy bs. + if (err) + { + msg.restart(); + msg << qb; + fExeMgr->write(msg); + } + + return rowsProcessed; + } + catch (runtime_error& ex) + { + cerr << "DeletePackageProcessor::processPackage::fixupRows" << ex.what() << endl; + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: "); + args.add(ex.what()); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + qb = 0; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + return rowsProcessed; + } + catch (...) + { + cerr << "DeletePackageProcessor::processPackage::fixupRows" << endl; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("Unknown error caught when communicating with ExeMgr"); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + qb = 0; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + return rowsProcessed; + } + + return rowsProcessed; } -uint64_t DeletePackageProcessor::fixUpRows (dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, const uint64_t uniqueId, const uint32_t tableOid) +bool DeletePackageProcessor::processRowgroup(ByteStream& aRowGroup, DMLResult& result, + const uint64_t uniqueId, dmlpackage::CalpontDMLPackage& cpackage, + std::map& pmStateDel, bool isMeta, + uint32_t dbroot) { - ByteStream msg, msgBk, emsgBs; - rowgroup::RGData rgData; - ByteStream::quadbyte qb = 4; - msg << qb; - boost::scoped_ptr rowGroup; - uint64_t rowsProcessed = 0; - uint32_t dbroot = 1; - bool metaData = false; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - std::vector fPMs = oamCache->getModuleIds(); - std::map pmStateDel; - string emsg; - string emsgStr; - bool err = false; + bool rc = false; + // cout << "Get dbroot " << dbroot << endl; + int pmNum = (*fDbRootPMMap)[dbroot]; + DMLTable* tablePtr = cpackage.get_Table(); + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_SVR_DELETE; + bytestream << uniqueId; + bytestream << (ByteStream::quadbyte)pmNum; + bytestream << uint32_t(cpackage.get_SessionID()); + bytestream << (ByteStream::quadbyte)cpackage.get_TxnID(); + bytestream << tablePtr->get_SchemaName(); + bytestream << tablePtr->get_TableName(); + bytestream += aRowGroup; + // cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; + uint32_t msgRecived = 0; + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; + string errorMsg; + ByteStream::quadbyte tmp32; + uint64_t blocksChanged = 0; - //boost::scoped_ptr fExeMgr; - //fExeMgr.reset( new messageqcpp::MessageQueueClient("ExeMgr1")); + if (isMeta) // send to all PMs + { + fWEClient->write_to_all(bytestream); + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = true; + break; + } + else + { + *bsIn >> tmp8; + rc = (tmp8 != 0); + + if (rc != 0) + { + *bsIn >> errorMsg; + break; + } + else + msgRecived++; + } + } + + return rc; + } + + if (pmStateDel[pmNum]) + { try { - for (unsigned i = 0; i < fPMs.size(); i++) + fWEClient->write(bytestream, (uint32_t)pmNum); + // cout << "sent tp pm " << pmNum<read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - pmStateDel[fPMs[i]] = true; - } - - fExeMgr->write(msg); - fExeMgr->write(*(cpackage.get_ExecutionPlan())); - //cout << "sending to ExeMgr plan with length " << (cpackage.get_ExecutionPlan())->length() << endl; - msg.restart(); - emsgBs.restart(); - msg = fExeMgr->read(); //error handling - - if (msg.length() == 4) - { - msg >> qb; - - if (qb != 0) - err = true; + rc = true; + errorMsg = "Lost connection to Write Engine Server while deleting"; + throw std::runtime_error(errorMsg); } else { - qb = 999; - err = true; + *bsIn >> tmp8; + rc = (tmp8 != 0); + *bsIn >> errorMsg; + *bsIn >> tmp32; + *bsIn >> blocksChanged; + result.stats.fBlocksChanged += blocksChanged; + result.stats.fErrorNo = tmp8; + + // cout << "received from pm " << (uint32_t)tmp32 << " and rc = " << rc << endl; + pmStateDel[tmp32] = true; + + if (rc != 0) + { + throw std::runtime_error(errorMsg); + } + + if (tmp32 == (uint32_t)pmNum) + { + fWEClient->write(bytestream, (uint32_t)pmNum); + pmStateDel[pmNum] = false; + break; + } } - - if (err) - { - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: Error from ExeMgr"); - args.add((int)qb); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - return 0; - } - - emsgBs = fExeMgr->read(); - - if (emsgBs.length() == 0) - { - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add("Lost connection to ExeMgr"); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - return 0; - } - - emsgBs >> emsgStr; - - while (true) - { - if (fRollbackPending) - { - result.result = JOB_CANCELED; - err = true; - break; - } - - msg.restart(); - msgBk.restart(); - msg = fExeMgr->read(); - msgBk = msg; - - if ( msg.length() == 0 ) - { - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add("Lost connection to ExeMgr"); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - //return rowsProcessed; - break; - } - else - { - if (rowGroup.get() == NULL) - { - //This is meta data, need to send to all PMs. - metaData = true; - //cout << "sending meta data" << endl; - err = processRowgroup(msgBk, result, uniqueId, cpackage, pmStateDel, metaData, dbroot); - rowGroup.reset(new rowgroup::RowGroup()); - rowGroup->deserialize(msg); - qb = 100; - msg.restart(); - msg << qb; - //cout << "Projecting rows" << endl; - fExeMgr->write(msg); - metaData = false; - continue; - } - - // XXXST: take out the 'true' when all jobsteps have been made st-compatible - rgData.deserialize(msg, true); - rowGroup->setData(&rgData); - //rowGroup->setData(const_cast(msg.buf())); - err = (rowGroup->getStatus() != 0); - - if (err) - { - //msgBk.advance(rowGroup->getDataSize()); - string errorMsg; - msg >> errorMsg; - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add(errorMsg); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - DMLResult tmpResult; - receiveAll( tmpResult, uniqueId, fPMs, pmStateDel, tableOid); - //@Bug 4358 get rid of broken pipe error. - //msg.restart(); - //msg << qb; - //fExeMgr->write(msg); - //return rowsProcessed; - break; - } - - if (rowGroup->getRGData() == NULL) - { - msg.restart(); - } - - if (rowGroup->getRowCount() == 0) //done fetching - { - err = receiveAll( result, uniqueId, fPMs, pmStateDel, tableOid); - //return rowsProcessed; - break; - } - - if (rowGroup->getBaseRid() == (uint64_t) (-1)) - { - continue; // @bug4247, not valid row ids, may from small side outer - } - - dbroot = rowGroup->getDBRoot(); - err = processRowgroup(msgBk, result, uniqueId, cpackage, pmStateDel, metaData, dbroot); - - if (err) - { - DMLResult tmpResult; - receiveAll( tmpResult, uniqueId, fPMs, pmStateDel, tableOid); - //@Bug 4358 get rid of broken pipe error. - //msg.restart(); - //msg << qb; - //fExeMgr->write(msg); - //return rowsProcessed; - break; - } - - rowsProcessed += rowGroup->getRowCount(); - } - } - - if (fRollbackPending) - { - err = true; - // Response to user - cerr << "DeletePackageProcessor::processPackage::fixupRows Rollback Pending" << endl; - - //@Bug 4994 Cancelled job is not error - result.result = JOB_CANCELED; - // Log - LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID()); - logging::Message::Args args1; - logging::Message msg1(1); - args1.add("SQL statement canceled by user"); - msg1.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); - - // Clean out the pipe; - DMLResult tmpResult; - receiveAll( tmpResult, uniqueId, fPMs, pmStateDel, tableOid); - } - - if (!err) - { - // get stats from ExeMgr - qb = 3; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - msg = fExeMgr->read(); - msg >> result.queryStats; - msg >> result.extendedStats; - msg >> result.miniStats; - result.stats.unserialize(msg); - } - - //@Bug 4358 get rid of broken pipe error by sending a dummy bs. - if (err) - { - msg.restart(); - msg << qb; - fExeMgr->write(msg); - } - - return rowsProcessed; - } - catch (runtime_error& ex) - { - cerr << "DeletePackageProcessor::processPackage::fixupRows" << ex.what() << endl; + } + catch (runtime_error& ex) // write error + { + rc = true; logging::Message::Args args; logging::Message message(2); args.add("Delete Failed: "); @@ -607,306 +777,138 @@ uint64_t DeletePackageProcessor::fixUpRows (dmlpackage::CalpontDMLPackage& cpack message.format(args); result.result = DELETE_ERROR; result.message = message; - qb = 0; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - return rowsProcessed; - } - catch ( ... ) - { - cerr << "DeletePackageProcessor::processPackage::fixupRows" << endl; + break; + } + catch (...) + { + rc = true; logging::Message::Args args; logging::Message message(2); - args.add("Update Failed: "); - args.add("Unknown error caught when communicating with ExeMgr"); + args.add("Delete Failed: "); + args.add("Unknown error caught when communicating with WES"); message.format(args); result.result = DELETE_ERROR; result.message = message; - qb = 0; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - return rowsProcessed; + break; + } } + } - return rowsProcessed; -} - -bool DeletePackageProcessor::processRowgroup(ByteStream& aRowGroup, DMLResult& result, const uint64_t uniqueId, - dmlpackage::CalpontDMLPackage& cpackage, std::map& pmStateDel, bool isMeta, uint32_t dbroot) -{ - bool rc = false; - //cout << "Get dbroot " << dbroot << endl; - int pmNum = (*fDbRootPMMap)[dbroot]; - DMLTable* tablePtr = cpackage.get_Table(); - ByteStream bytestream; - bytestream << (ByteStream::byte)WE_SVR_DELETE; - bytestream << uniqueId; - bytestream << (ByteStream::quadbyte) pmNum; - bytestream << uint32_t(cpackage.get_SessionID()); - bytestream << (ByteStream::quadbyte) cpackage.get_TxnID(); - bytestream << tablePtr->get_SchemaName(); - bytestream << tablePtr->get_TableName(); - bytestream += aRowGroup; - //cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; - uint32_t msgRecived = 0; - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - string errorMsg; - ByteStream::quadbyte tmp32; - uint64_t blocksChanged = 0; - - if (isMeta) //send to all PMs - { - fWEClient->write_to_all(bytestream); - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = true; - break; - } - else - { - *bsIn >> tmp8; - rc = (tmp8 != 0); - - if (rc != 0) - { - *bsIn >> errorMsg; - break; - } - else - msgRecived++; - } - } - - return rc; - } - - if (pmStateDel[pmNum]) - { - try - { - fWEClient->write(bytestream, (uint32_t)pmNum); - //cout << "sent tp pm " << pmNum<read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = true; - errorMsg = "Lost connection to Write Engine Server while deleting"; - throw std::runtime_error(errorMsg); - } - else - { - *bsIn >> tmp8; - rc = (tmp8 != 0); - *bsIn >> errorMsg; - *bsIn >> tmp32; - *bsIn >> blocksChanged; - result.stats.fBlocksChanged += blocksChanged; - result.stats.fErrorNo = tmp8; - - //cout << "received from pm " << (uint32_t)tmp32 << " and rc = " << rc << endl; - pmStateDel[tmp32] = true; - - if (rc != 0) - { - throw std::runtime_error(errorMsg); - } - - if ( tmp32 == (uint32_t)pmNum ) - { - fWEClient->write(bytestream, (uint32_t)pmNum); - pmStateDel[pmNum] = false; - break; - } - } - } - catch (runtime_error& ex) //write error - { - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add(ex.what()); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - break; - } - catch (...) - { - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add("Unknown error caught when communicating with WES"); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - break; - } - } - } - - return rc; + return rc; } bool DeletePackageProcessor::receiveAll(DMLResult& result, const uint64_t uniqueId, std::vector& fPMs, std::map& pmStateDel, const uint32_t tableOid) { - //check how many message we need to receive - uint32_t messagesNotReceived = 0; - bool err = false; + // check how many message we need to receive + uint32_t messagesNotReceived = 0; + bool err = false; - for (unsigned i = 0; i < fPMs.size(); i++) + for (unsigned i = 0; i < fPMs.size(); i++) + { + if (!pmStateDel[fPMs[i]]) + messagesNotReceived++; + } + + boost::shared_ptr bsIn; + uint32_t msgReceived = 0; + ByteStream::byte tmp8; + string errorMsg; + + if (messagesNotReceived > 0) + { + LoggingID logid(DMLLoggingId, fSessionID, fSessionID); + + if (messagesNotReceived > fWEClient->getPmCount()) { - if (!pmStateDel[fPMs[i]]) - messagesNotReceived++; + logging::Message::Args args1; + logging::Message msg(1); + args1.add("Delete outstanding messages exceed PM count , need to receive messages:PMcount = "); + ostringstream oss; + oss << messagesNotReceived << ":" << fPMCount; + args1.add(oss.str()); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_ERROR, msg, logid); + err = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("One of WriteEngineServer went away."); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + return err; } - boost::shared_ptr bsIn; - uint32_t msgReceived = 0; - ByteStream::byte tmp8; - string errorMsg; + bsIn.reset(new ByteStream()); + ByteStream::quadbyte tmp32; + uint64_t blocksChanged = 0; - if (messagesNotReceived > 0) + while (1) { - LoggingID logid( DMLLoggingId, fSessionID, fSessionID); + if (msgReceived == messagesNotReceived) + break; - if ( messagesNotReceived > fWEClient->getPmCount()) + bsIn.reset(new ByteStream()); + + try + { + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Delete outstanding messages exceed PM count , need to receive messages:PMcount = "); - ostringstream oss; - oss << messagesNotReceived << ":" << fPMCount; - args1.add(oss.str()); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_ERROR, msg, logid); - err = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add("One of WriteEngineServer went away."); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - return err; + err = true; + errorMsg = "Lost connection to Write Engine Server while deleting"; + throw std::runtime_error(errorMsg); } - - bsIn.reset(new ByteStream()); - ByteStream::quadbyte tmp32; - uint64_t blocksChanged = 0; - - while (1) + else { - if (msgReceived == messagesNotReceived) - break; + *bsIn >> tmp8; + err = (tmp8 != 0); + *bsIn >> errorMsg; + *bsIn >> tmp32; + *bsIn >> blocksChanged; + // cout << "Received response from pm " << tmp32 << endl; + pmStateDel[tmp32] = true; - bsIn.reset(new ByteStream()); + if (err) + { + throw std::runtime_error(errorMsg); + } - try - { - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - err = true; - errorMsg = "Lost connection to Write Engine Server while deleting"; - throw std::runtime_error(errorMsg); - } - else - { - *bsIn >> tmp8; - err = (tmp8 != 0); - *bsIn >> errorMsg; - *bsIn >> tmp32; - *bsIn >> blocksChanged; - //cout << "Received response from pm " << tmp32 << endl; - pmStateDel[tmp32] = true; - - if (err) - { - throw std::runtime_error(errorMsg); - } - - msgReceived++; - result.stats.fBlocksChanged += blocksChanged; - result.stats.fErrorNo = tmp8; - } - } - catch (runtime_error& ex) //write error - { - err = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add(ex.what()); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - break; - } - catch (...) - { - err = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Delete Failed: "); - args.add("Unknown error caught when communicating with WES"); - message.format(args); - result.result = DELETE_ERROR; - result.message = message; - break; - } + msgReceived++; + result.stats.fBlocksChanged += blocksChanged; + result.stats.fErrorNo = tmp8; } + } + catch (runtime_error& ex) // write error + { + err = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: "); + args.add(ex.what()); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + break; + } + catch (...) + { + err = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Delete Failed: "); + args.add("Unknown error caught when communicating with WES"); + message.format(args); + result.result = DELETE_ERROR; + result.message = message; + break; + } } + } - return err; + return err; } -} // namespace dmlpackageprocessor +} // namespace dmlpackageprocessor diff --git a/dbcon/dmlpackageproc/deletepackageprocessor.h b/dbcon/dmlpackageproc/deletepackageprocessor.h index 392cfa2ec..1c6c244a5 100644 --- a/dbcon/dmlpackageproc/deletepackageprocessor.h +++ b/dbcon/dmlpackageproc/deletepackageprocessor.h @@ -46,47 +46,46 @@ namespace dmlpackageprocessor */ class DeletePackageProcessor : public DMLPackageProcessor { + public: + DeletePackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) + { + } + /** @brief process a DeleteDMLPackage + * + * @param cpackage the delete dml package to process + */ + EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); -public: + protected: + private: + /** @brief delete a row + * + * @param txnID the transaction id + * @param tablePtr a pointer to the table that is being operated on + * @param rowIDList upon return containts the row ids of the rows deleted + * @param colOldValuesList upon return contains the values the were delete + * @param result upon return will containt the result of the operation + bool deleteRows(execplan::CalpontSystemCatalog::SCN txnID, dmlpackage::DMLTable* tablePtr, + WriteEngine::RIDList& rowIDList, WriteEngine::ColValueList& colOldValuesList, + DMLResult& result); + */ + bool processRowgroup(messageqcpp::ByteStream& aRowGroup, DMLResult& result, const uint64_t uniqueId, + dmlpackage::CalpontDMLPackage& cpackage, std::map& pmStateDel, + bool isMeta = false, uint32_t dbroot = 1); - DeletePackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) {} - /** @brief process a DeleteDMLPackage - * - * @param cpackage the delete dml package to process - */ - EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); - -protected: - -private: - - /** @brief delete a row - * - * @param txnID the transaction id - * @param tablePtr a pointer to the table that is being operated on - * @param rowIDList upon return containts the row ids of the rows deleted - * @param colOldValuesList upon return contains the values the were delete - * @param result upon return will containt the result of the operation - bool deleteRows(execplan::CalpontSystemCatalog::SCN txnID, dmlpackage::DMLTable* tablePtr, - WriteEngine::RIDList& rowIDList, WriteEngine::ColValueList& colOldValuesList, - DMLResult& result); - */ - bool processRowgroup(messageqcpp::ByteStream& aRowGroup, DMLResult& result, const uint64_t uniqueId, dmlpackage::CalpontDMLPackage& cpackage, std::map& pmStateDel, - bool isMeta = false, uint32_t dbroot = 1); - - - /** @brief add all rows if we have no filter for the delete - * - * @param tablePtr a pointer to the table that is being operated on - */ - uint64_t fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, const uint64_t uniqueId, const uint32_t tableOid); - bool receiveAll(DMLResult& result, const uint64_t uniqueId, std::vector& fPMs, std::map& pmStateDel, const uint32_t tableOid); - - //bandListsByExtent bandListsMap; + /** @brief add all rows if we have no filter for the delete + * + * @param tablePtr a pointer to the table that is being operated on + */ + uint64_t fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, const uint64_t uniqueId, + const uint32_t tableOid); + bool receiveAll(DMLResult& result, const uint64_t uniqueId, std::vector& fPMs, + std::map& pmStateDel, const uint32_t tableOid); + // bandListsByExtent bandListsMap; }; -} // namespace dmlpackageprocessor +} // namespace dmlpackageprocessor #undef EXPORT diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index 1972fd838..b5217b0e6 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -61,44 +61,43 @@ const SOP opeq(new Operator("=")); const SOP opne(new Operator("<>")); const SOP opor(new Operator("or")); const SOP opand(new Operator("and")); -} +} // namespace namespace dmlpackageprocessor { - DMLPackageProcessor::~DMLPackageProcessor() { - //cout << "In DMLPackageProcessor destructor " << this << endl; - if (fWEClient) - delete fWEClient; + // cout << "In DMLPackageProcessor destructor " << this << endl; + if (fWEClient) + delete fWEClient; - if (fExeMgr) - delete fExeMgr; + if (fExeMgr) + delete fExeMgr; } //@bug 397 void DMLPackageProcessor::cleanString(string& s) { - string::size_type pos = s.find_first_not_of(" "); + string::size_type pos = s.find_first_not_of(" "); - //stripe off space and ' or '' at beginning and end - if ( pos < s.length() ) + // stripe off space and ' or '' at beginning and end + if (pos < s.length()) + { + s = s.substr(pos, s.length() - pos); + + if ((pos = s.find_last_of(" ")) < s.length()) { - s = s.substr( pos, s.length() - pos ); - - if ( (pos = s.find_last_of(" ")) < s.length()) - { - s = s.substr(0, pos ); - } + s = s.substr(0, pos); } + } - if ( s[0] == '\'') - { - s = s.substr(1, s.length() - 2); + if (s[0] == '\'') + { + s = s.substr(1, s.length() - 2); - if ( s[0] == '\'') - s = s.substr(1, s.length() - 2); - } + if (s[0] == '\'') + s = s.substr(1, s.length() - 2); + } } #if 0 boost::any DMLPackageProcessor::tokenizeData( execplan::CalpontSystemCatalog::SCN txnID, @@ -165,751 +164,753 @@ boost::any DMLPackageProcessor::tokenizeData( execplan::CalpontSystemCatalog::SC return value; } #endif -void DMLPackageProcessor::getColumnsForTable(uint32_t sessionID, std::string schema, - std::string table, dmlpackage::ColumnList& colList) +void DMLPackageProcessor::getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, + dmlpackage::ColumnList& colList) { + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = table; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = table; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true); - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog( sessionID ); - CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true); + CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); - CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); + while (rid_iterator != ridList.end()) + { + CalpontSystemCatalog::ROPair roPair = *rid_iterator; + DMLColumn* columnPtr = new DMLColumn(); + CalpontSystemCatalog::TableColName tblColName = systemCatalogPtr->colName(roPair.objnum); + columnPtr->set_Name(tblColName.column); - while (rid_iterator != ridList.end()) - { - CalpontSystemCatalog::ROPair roPair = *rid_iterator; - DMLColumn* columnPtr = new DMLColumn(); - CalpontSystemCatalog::TableColName tblColName = systemCatalogPtr->colName( roPair.objnum ); - columnPtr->set_Name(tblColName.column); - - colList.push_back(columnPtr); - - ++rid_iterator; - } + colList.push_back(columnPtr); + ++rid_iterator; + } } char* DMLPackageProcessor::strlower(char* in) { - char* p = in; + char* p = in; - if (p) + if (p) + { + while (*p) { - while (*p) - { - *p = tolower(*p); - p++; - } + *p = tolower(*p); + p++; } + } - return in; + return in; } void DMLPackageProcessor::convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, - unsigned& segment, unsigned filesPerColumnPartition, unsigned extentsPerSegmentFile, unsigned extentRows, - unsigned startDBRoot, unsigned dbrootCnt, const unsigned startPartitionNum) + unsigned& segment, unsigned filesPerColumnPartition, + unsigned extentsPerSegmentFile, unsigned extentRows, + unsigned startDBRoot, unsigned dbrootCnt, + const unsigned startPartitionNum) { - partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); + partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); - segment = (((rid % ( filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / - extentRows)) % filesPerColumnPartition; + segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % + filesPerColumnPartition; - dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; + dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; - //Calculate the relative rid for this segment file - uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * - (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); - idbassert(relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * - (uint64_t)extentRows); - uint32_t numExtentsInThisPart = relRidInPartition / extentRows; - unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; - uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; - rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; + // Calculate the relative rid for this segment file + uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * + (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); + idbassert(relRidInPartition <= + (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); + uint32_t numExtentsInThisPart = relRidInPartition / extentRows; + unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; + uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; + rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; } - string DMLPackageProcessor::projectTableErrCodeToMsg(uint32_t ec) { - if (ec < 1000) // pre IDB error code - { - ErrorCodes ecObj; - string errMsg("Statement failed."); - errMsg += ecObj.errorString(ec).substr(150); // substr removes ErrorCodes::fPreamble - return errMsg; - } + if (ec < 1000) // pre IDB error code + { + ErrorCodes ecObj; + string errMsg("Statement failed."); + errMsg += ecObj.errorString(ec).substr(150); // substr removes ErrorCodes::fPreamble + return errMsg; + } - // IDB error - return IDBErrorInfo::instance()->errorMsg(ec); + // IDB error + return IDBErrorInfo::instance()->errorMsg(ec); } -bool DMLPackageProcessor::validateVarbinaryVal( std::string& inStr) +bool DMLPackageProcessor::validateVarbinaryVal(std::string& inStr) { - bool invalid = false; + bool invalid = false; - for (unsigned i = 0; i < inStr.length(); i++) + for (unsigned i = 0; i < inStr.length(); i++) + { + if (!isxdigit(inStr[i])) { - if (!isxdigit(inStr[i])) - { - invalid = true; - break; - } - + invalid = true; + break; } + } - return invalid; + return invalid; } int DMLPackageProcessor::commitTransaction(uint64_t uniqueId, BRM::TxnID txnID) { - int rc = fDbrm->vbCommit(txnID.id); - return rc; + int rc = fDbrm->vbCommit(txnID.id); + return rc; } int DMLPackageProcessor::rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, - std::string& errorMsg) + std::string& errorMsg) { - std::vector lbidList; - std::vector lbidRangeList; - BRM::LBIDRange range; - int rc = 0; - //Check BRM status before processing. - rc = fDbrm->isReadWrite(); - - if (rc != 0 ) - { - std::string brmMsg; - errorMsg = "Can't read DBRM isReadWrite [ "; - BRM::errString(rc, brmMsg); - errorMsg += brmMsg; - errorMsg += "]"; - return rc; - } - - ByteStream bytestream; - fWEClient->addQueue(uniqueId); - //cout << "adding to queue with id " << uniqueId << endl; - bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BLOCKS; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID.id; - uint32_t msgRecived = 0; - - try - { - fWEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Network error reading WEClient"; - fWEClient->removeQueue(uniqueId); - //cout << "erroring out remove queue id " << uniqueId << endl; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - char szrc[20]; - *bsIn >> errorMsg; - errorMsg += " (WriteEngine returns error "; - sprintf(szrc, "%d", rc); - errorMsg += szrc; - errorMsg += ")"; - fWEClient->removeQueue(uniqueId); - cout << "erroring out remove queue id " << uniqueId << endl; - break; - } - else - msgRecived++; - } - } - } - catch (std::exception& e) - { - rc = NETWORK_ERROR; - errorMsg = "Network error occured when rolling back blocks"; - errorMsg += e.what(); - fWEClient->removeQueue(uniqueId); - cout << "erroring out remove queue id " << uniqueId << endl; - //delete fWEClient; - return rc; - } - catch ( ... ) - { - rc = NETWORK_ERROR; - errorMsg = "Unknown exception caught while rolling back transaction."; - fWEClient->removeQueue(uniqueId); - cout << "erroring out remove queue id " << uniqueId << endl; - //delete fWEClient; - return rc; - } - - if (rc != 0) - { - //delete fWEClient; - return rc; - } - - fWEClient->removeQueue(uniqueId); - //delete fWEClient; -// cout << "success. remove queue id " << uniqueId << endl; - rc = fDbrm->getUncommittedLBIDs(txnID.id, lbidList); - - if (rc != 0 ) - { - std::string brmMsg; - errorMsg = "DBRM getUncommittedLBIDs [ "; - BRM::errString(rc, brmMsg); - errorMsg += brmMsg; - errorMsg += "]"; - return rc; - } - - for (size_t i = 0; i < lbidList.size(); i++) - { - range.start = lbidList[i]; - range.size = 1; - lbidRangeList.push_back(range); - } - - rc = fDbrm->vbRollback(txnID.id, lbidRangeList); - - if (rc != 0 ) - { - std::string brmMsg; - errorMsg = "DBRM vbRollback [ "; - BRM::errString(rc, brmMsg); - errorMsg += brmMsg; - errorMsg += "]"; - return rc; - } + std::vector lbidList; + std::vector lbidRangeList; + BRM::LBIDRange range; + int rc = 0; + // Check BRM status before processing. + rc = fDbrm->isReadWrite(); + if (rc != 0) + { + std::string brmMsg; + errorMsg = "Can't read DBRM isReadWrite [ "; + BRM::errString(rc, brmMsg); + errorMsg += brmMsg; + errorMsg += "]"; return rc; -} + } -int DMLPackageProcessor::commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, std::string& errorMsg) -{ - //collect hwm info from all pms and set them here. remove table metadata if all successful - ByteStream bytestream; - fWEClient->addQueue(uniqueId); - bytestream << (ByteStream::byte)WE_SVR_COMMIT_BATCH_AUTO_ON; - bytestream << uniqueId; - bytestream << (uint32_t) txnID.id; - bytestream << tableOid; - bytestream << fSessionID; + ByteStream bytestream; + fWEClient->addQueue(uniqueId); + // cout << "adding to queue with id " << uniqueId << endl; + bytestream << (ByteStream::byte)WE_SVR_ROLLBACK_BLOCKS; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID.id; + uint32_t msgRecived = 0; - uint32_t msgRecived = 0; + try + { fWEClient->write_to_all(bytestream); boost::shared_ptr bsIn; bsIn.reset(new ByteStream()); - int rc = 0; ByteStream::byte tmp8; - typedef std::vector BulkSetHWMArgs; - std::vector hwmArgsAllPms; while (1) { - if (msgRecived == fWEClient->getPmCount()) - break; + if (msgRecived == fWEClient->getPmCount()) + break; - fWEClient->read(uniqueId, bsIn); + fWEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Network error reading WEClient"; + fWEClient->removeQueue(uniqueId); + // cout << "erroring out remove queue id " << uniqueId << endl; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) { - rc = NETWORK_ERROR; - fWEClient->removeQueue(uniqueId); - break; + char szrc[20]; + *bsIn >> errorMsg; + errorMsg += " (WriteEngine returns error "; + sprintf(szrc, "%d", rc); + errorMsg += szrc; + errorMsg += ")"; + fWEClient->removeQueue(uniqueId); + cout << "erroring out remove queue id " << uniqueId << endl; + break; } else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - { - //get hwm info - *bsIn >> errorMsg; - BulkSetHWMArgs setHWMArgs; - //cout << "received from WES bytestream length = " << bsIn->length() << endl; - deserializeInlineVector(*(bsIn.get()), setHWMArgs); - //cout << "get hwm info from WES size " << setHWMArgs.size() << endl; - hwmArgsAllPms.push_back(setHWMArgs); - msgRecived++; - } - } + msgRecived++; + } } + } + catch (std::exception& e) + { + rc = NETWORK_ERROR; + errorMsg = "Network error occured when rolling back blocks"; + errorMsg += e.what(); + fWEClient->removeQueue(uniqueId); + cout << "erroring out remove queue id " << uniqueId << endl; + // delete fWEClient; + return rc; + } + catch (...) + { + rc = NETWORK_ERROR; + errorMsg = "Unknown exception caught while rolling back transaction."; + fWEClient->removeQueue(uniqueId); + cout << "erroring out remove queue id " << uniqueId << endl; + // delete fWEClient; + return rc; + } - if (rc != 0) - return rc; + if (rc != 0) + { + // delete fWEClient; + return rc; + } - //set hwm - std::vector allHwm; - BulkSetHWMArgs::const_iterator itor; + fWEClient->removeQueue(uniqueId); + // delete fWEClient; + // cout << "success. remove queue id " << uniqueId << endl; + rc = fDbrm->getUncommittedLBIDs(txnID.id, lbidList); - //cout << "total hwmArgsAllPms size " << hwmArgsAllPms.size() << endl; - for (unsigned i = 0; i < fWEClient->getPmCount(); i++) + if (rc != 0) + { + std::string brmMsg; + errorMsg = "DBRM getUncommittedLBIDs [ "; + BRM::errString(rc, brmMsg); + errorMsg += brmMsg; + errorMsg += "]"; + return rc; + } + + for (size_t i = 0; i < lbidList.size(); i++) + { + range.start = lbidList[i]; + range.size = 1; + lbidRangeList.push_back(range); + } + + rc = fDbrm->vbRollback(txnID.id, lbidRangeList); + + if (rc != 0) + { + std::string brmMsg; + errorMsg = "DBRM vbRollback [ "; + BRM::errString(rc, brmMsg); + errorMsg += brmMsg; + errorMsg += "]"; + return rc; + } + + return rc; +} + +int DMLPackageProcessor::commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, + const uint32_t tableOid, std::string& errorMsg) +{ + // collect hwm info from all pms and set them here. remove table metadata if all successful + ByteStream bytestream; + fWEClient->addQueue(uniqueId); + bytestream << (ByteStream::byte)WE_SVR_COMMIT_BATCH_AUTO_ON; + bytestream << uniqueId; + bytestream << (uint32_t)txnID.id; + bytestream << tableOid; + bytestream << fSessionID; + + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + int rc = 0; + ByteStream::byte tmp8; + typedef std::vector BulkSetHWMArgs; + std::vector hwmArgsAllPms; + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - itor = hwmArgsAllPms[i].begin(); - - while (itor != hwmArgsAllPms[i].end()) - { - allHwm.push_back(*itor); - //cout << "received hwm info: " << itor->oid << ":" << itor->hwm << endl; - itor++; - } + rc = NETWORK_ERROR; + fWEClient->removeQueue(uniqueId); + break; } + else + { + *bsIn >> tmp8; + rc = tmp8; - //set CP data before hwm. + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; + } + else + { + // get hwm info + *bsIn >> errorMsg; + BulkSetHWMArgs setHWMArgs; + // cout << "received from WES bytestream length = " << bsIn->length() << endl; + deserializeInlineVector(*(bsIn.get()), setHWMArgs); + // cout << "get hwm info from WES size " << setHWMArgs.size() << endl; + hwmArgsAllPms.push_back(setHWMArgs); + msgRecived++; + } + } + } - //cout << "setting hwm allHwm size " << allHwm.size() << endl; - BRM::CPInfoList_t cpInfos; + if (rc != 0) + return rc; - std::vector mergeCPDataArgs; - rc = fDbrm->bulkSetHWMAndCP(allHwm, cpInfos, mergeCPDataArgs, txnID.id); - fDbrm->takeSnapshot(); - //Set tablelock to rollforward remove meta files + // set hwm + std::vector allHwm; + BulkSetHWMArgs::const_iterator itor; - if (rc != 0) - return rc; + // cout << "total hwmArgsAllPms size " << hwmArgsAllPms.size() << endl; + for (unsigned i = 0; i < fWEClient->getPmCount(); i++) + { + itor = hwmArgsAllPms[i].begin(); + while (itor != hwmArgsAllPms[i].end()) + { + allHwm.push_back(*itor); + // cout << "received hwm info: " << itor->oid << ":" << itor->hwm << endl; + itor++; + } + } + + // set CP data before hwm. + + // cout << "setting hwm allHwm size " << allHwm.size() << endl; + BRM::CPInfoList_t cpInfos; + + std::vector mergeCPDataArgs; + rc = fDbrm->bulkSetHWMAndCP(allHwm, cpInfos, mergeCPDataArgs, txnID.id); + fDbrm->takeSnapshot(); + // Set tablelock to rollforward remove meta files + + if (rc != 0) + return rc; + + bool stateChanged = true; + TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); + uint64_t tablelockId = tablelockData->getTablelockId(tableOid); + + try + { + stateChanged = fDbrm->changeState(tablelockId, BRM::CLEANUP); + } + catch (std::exception&) + { + errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + stateChanged = false; + } + + if (!stateChanged) + return rc; + + bytestream.restart(); + //@Bug 4517 Remove meta data failure doesn't stop tablelock releasing. + bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META; + bytestream << uniqueId; + bytestream << tableOid; + msgRecived = 0; + fWEClient->write_to_all(bytestream); + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + msgRecived++; + } + } + + return rc; +} + +int DMLPackageProcessor::rollBackBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, + uint32_t sessionID, const uint32_t tableOid, + std::string& errorMsg) +{ + // Bulkrollback, rollback blocks, vbrollback, change state, remove meta file + // cout << "In rollBackBatchAutoOnTransaction" << endl; + std::vector tableLocks; + tableLocks = fDbrm->getAllTableLocks(); + // cout << " Got all tablelocks" << endl; + unsigned idx = 0; + string ownerName("DMLProc batchinsert"); + uint64_t tableLockId = 0; + int rc = 0; + + for (; idx < tableLocks.size(); idx++) + { + if ((tableLocks[idx].ownerName == ownerName) && (tableLocks[idx].tableOID == tableOid)) + { + tableLockId = tableLocks[idx].id; + break; + } + } + + if ((tableLockId == 0) || (tableOid == 0)) + { + // table is not locked by DMLProc. Could happen if we failed to get lock + // while inserting. Not an error during rollback, but we don't + // want to do anything. + return rc; + } + + // cout << "sending to WES" << endl; + ByteStream bytestream; + fWEClient->addQueue(uniqueId); + // cout << "adding queue id " << uniqueId << endl; + bytestream << (ByteStream::byte)WE_SVR_ROLLBACK_BATCH_AUTO_ON; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << tableLockId; + bytestream << tableOid; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; + + // cout << "waiting for reply from WES" << endl; + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + fWEClient->removeQueue(uniqueId); + // cout << "erroring out remove queue id " << uniqueId << endl; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + // cout << "erroring out remove queue id " << uniqueId << endl; + break; + } + else + msgRecived++; + } + } + + if (rc == 0) // change table lock state + { bool stateChanged = true; - TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - uint64_t tablelockId = tablelockData->getTablelockId(tableOid); + // cout << "changing tablelock state" << endl; try { - stateChanged = fDbrm->changeState(tablelockId, BRM::CLEANUP); + stateChanged = fDbrm->changeState(tableLockId, BRM::CLEANUP); } catch (std::exception&) { - errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - stateChanged = false; - } - - if (!stateChanged) - return rc; - - bytestream.restart(); - //@Bug 4517 Remove meta data failure doesn't stop tablelock releasing. - bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META; - bytestream << uniqueId; - bytestream << tableOid; - msgRecived = 0; - fWEClient->write_to_all(bytestream); - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - fWEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - msgRecived++; - } - } - - return rc; - - -} - -int DMLPackageProcessor::rollBackBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, - const uint32_t tableOid, std::string& errorMsg) -{ - //Bulkrollback, rollback blocks, vbrollback, change state, remove meta file - //cout << "In rollBackBatchAutoOnTransaction" << endl; - std::vector tableLocks; - tableLocks = fDbrm->getAllTableLocks(); - //cout << " Got all tablelocks" << endl; - unsigned idx = 0; - string ownerName ("DMLProc batchinsert"); - uint64_t tableLockId = 0; - int rc = 0; - - for (; idx < tableLocks.size(); idx++) - { - if ((tableLocks[idx].ownerName == ownerName) && (tableLocks[idx].tableOID == tableOid)) - { - tableLockId = tableLocks[idx].id; - break; - } - } - - if ((tableLockId == 0) || (tableOid == 0)) - { - // table is not locked by DMLProc. Could happen if we failed to get lock - // while inserting. Not an error during rollback, but we don't - // want to do anything. - return rc; - } - - //cout << "sending to WES" << endl; - ByteStream bytestream; - fWEClient->addQueue(uniqueId); - //cout << "adding queue id " << uniqueId << endl; - bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BATCH_AUTO_ON; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << tableLockId; - bytestream << tableOid; - uint32_t msgRecived = 0; - fWEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - - //cout << "waiting for reply from WES" << endl; - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - fWEClient->removeQueue(uniqueId); - //cout << "erroring out remove queue id " << uniqueId << endl; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - //cout << "erroring out remove queue id " << uniqueId << endl; - break; - } - else - msgRecived++; - } - } - - if (rc == 0) //change table lock state - { - bool stateChanged = true; - - //cout << "changing tablelock state" << endl; - try - { - stateChanged = fDbrm->changeState(tableLockId, BRM::CLEANUP); - } - catch (std::exception&) - { - errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - stateChanged = false; - } - - if (!stateChanged) - { - rc = 1; - } - } - - if ( rc != 0 ) - return rc; - - bytestream.restart(); - bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META; - bytestream << uniqueId; - bytestream << tableOid; - msgRecived = 0; - fWEClient->write_to_all(bytestream); - - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - fWEClient->removeQueue(uniqueId); - //cout << "erroring out remove queue id " << uniqueId << endl; - break; - } - else - { - *bsIn >> tmp8; - msgRecived++; - } - } - - fWEClient->removeQueue(uniqueId); - return rc; -} - -int DMLPackageProcessor::commitBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, std::string& errorMsg) -{ - std::vector tableLocks; - tableLocks = fDbrm->getAllTableLocks(); - //cout << " Got all tablelocks" << endl; - unsigned idx = 0; - string ownerName ("DMLProc batchinsert"); - uint64_t tableLockId = 0; - int rc = 0; - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - - for (; idx < tableLocks.size(); idx++) - { - if ((tableLocks[idx].ownerName == ownerName) && (tableLocks[idx].tableOID == tableOid)) - { - tableLockId = tableLocks[idx].id; - break; - } - } - - if ((tableLockId == 0) || (tableOid == 0)) - { - // table is not locked by DMLProc. Could happen if we failed to get lock - // while inserting. Not an error during rollback, but we don't - // want to do anything. - return rc; - } - - bool stateChanged = true; - - //cout << "changing tablelock state" << endl; - try - { - stateChanged = fDbrm->changeState(tableLockId, BRM::CLEANUP); - } - catch (std::exception&) - { - errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); - stateChanged = false; + errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + stateChanged = false; } if (!stateChanged) { - rc = 1; + rc = 1; } + } - if ( rc != 0 ) - return rc; + if (rc != 0) + return rc; - ByteStream bytestream; - fWEClient->addQueue(uniqueId); - bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META; - bytestream << uniqueId; - bytestream << tableOid; - uint32_t msgRecived = 0; - fWEClient->write_to_all(bytestream); + bytestream.restart(); + bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META; + bytestream << uniqueId; + bytestream << tableOid; + msgRecived = 0; + fWEClient->write_to_all(bytestream); + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + fWEClient->removeQueue(uniqueId); + // cout << "erroring out remove queue id " << uniqueId << endl; + break; + } + else + { + *bsIn >> tmp8; + msgRecived++; + } + } + + fWEClient->removeQueue(uniqueId); + return rc; +} + +int DMLPackageProcessor::commitBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, + const uint32_t tableOid, std::string& errorMsg) +{ + std::vector tableLocks; + tableLocks = fDbrm->getAllTableLocks(); + // cout << " Got all tablelocks" << endl; + unsigned idx = 0; + string ownerName("DMLProc batchinsert"); + uint64_t tableLockId = 0; + int rc = 0; + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; + + for (; idx < tableLocks.size(); idx++) + { + if ((tableLocks[idx].ownerName == ownerName) && (tableLocks[idx].tableOID == tableOid)) + { + tableLockId = tableLocks[idx].id; + break; + } + } + + if ((tableLockId == 0) || (tableOid == 0)) + { + // table is not locked by DMLProc. Could happen if we failed to get lock + // while inserting. Not an error during rollback, but we don't + // want to do anything. + return rc; + } + + bool stateChanged = true; + + // cout << "changing tablelock state" << endl; + try + { + stateChanged = fDbrm->changeState(tableLockId, BRM::CLEANUP); + } + catch (std::exception&) + { + errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE); + stateChanged = false; + } + + if (!stateChanged) + { + rc = 1; + } + + if (rc != 0) + return rc; + + ByteStream bytestream; + fWEClient->addQueue(uniqueId); + bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META; + bytestream << uniqueId; + bytestream << tableOid; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + msgRecived++; + } + } + + fWEClient->removeQueue(uniqueId); + return rc; +} + +int DMLPackageProcessor::rollBackBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, + uint32_t sessionID, const uint32_t tableOid, + std::string& errorMsg) +{ + ByteStream bytestream; + fWEClient->addQueue(uniqueId); + bytestream << (ByteStream::byte)WE_SVR_ROLLBACK_BATCH_AUTO_OFF; + bytestream << uniqueId; + bytestream << sessionID; + bytestream << (uint32_t)txnID.id; + bytestream << tableOid; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + int rc = 0; + ByteStream::byte tmp8; + + while (1) + { + if (msgRecived == fWEClient->getPmCount()) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; + } + else + msgRecived++; + } + } + + return rc; +} + +int DMLPackageProcessor::flushDataFiles(int rcIn, std::map& columnOids, uint64_t uniqueId, + BRM::TxnID txnID, uint32_t tableOid) +{ + // cout <<"in flushDataFiles" << endl; + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_SVR_FLUSH_FILES; + bytestream << uniqueId; + bytestream << (uint32_t)rcIn; + bytestream << (uint32_t)txnID.id; + bytestream << tableOid; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + int rc = 0; + ByteStream::byte tmp8; + std::string errorMsg; + + try + { while (1) { - if (msgRecived == fWEClient->getPmCount()) - break; + if (msgRecived == fWEClient->getPmCount()) + break; - fWEClient->read(uniqueId, bsIn); + fWEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) { - fWEClient->removeQueue(uniqueId); - break; + *bsIn >> errorMsg; + break; } else - { - *bsIn >> tmp8; - msgRecived++; - } + msgRecived++; + } } + } + catch (std::exception&) + { + } - fWEClient->removeQueue(uniqueId); - return rc; + return rc; } -int DMLPackageProcessor::rollBackBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, - const uint32_t tableOid, std::string& errorMsg) +int DMLPackageProcessor::endTransaction(uint64_t uniqueId, BRM::TxnID txnID, bool success) { - ByteStream bytestream; - fWEClient->addQueue(uniqueId); - bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BATCH_AUTO_OFF; - bytestream << uniqueId; - bytestream << sessionID; - bytestream << (uint32_t)txnID.id; - bytestream << tableOid; - uint32_t msgRecived = 0; - fWEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - int rc = 0; - ByteStream::byte tmp8; + // cout <<"in flushDataFiles" << endl; + ByteStream bytestream; + bytestream << (ByteStream::byte)WE_END_TRANSACTION; + bytestream << uniqueId; + bytestream << (uint32_t)txnID.id; + bytestream << (ByteStream::byte)success; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + int rc = 0; + ByteStream::byte tmp8; + std::string errorMsg; + try + { while (1) { - if (msgRecived == fWEClient->getPmCount()) - break; + if (msgRecived == fWEClient->getPmCount()) + break; - fWEClient->read(uniqueId, bsIn); + fWEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) { - rc = NETWORK_ERROR; - fWEClient->removeQueue(uniqueId); - break; + *bsIn >> errorMsg; + break; } else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } + msgRecived++; + } } + } + catch (std::exception&) + { + } - return rc; -} - -int DMLPackageProcessor::flushDataFiles (int rcIn, std::map& columnOids, uint64_t uniqueId, BRM::TxnID txnID, uint32_t tableOid) -{ -//cout <<"in flushDataFiles" << endl; - ByteStream bytestream; - bytestream << (ByteStream::byte) WE_SVR_FLUSH_FILES; - bytestream << uniqueId; - bytestream << (uint32_t) rcIn; - bytestream << (uint32_t)txnID.id; - bytestream << tableOid; - uint32_t msgRecived = 0; - fWEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - int rc = 0; - ByteStream::byte tmp8; - std::string errorMsg; - - try - { - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - break; - } - else - msgRecived++; - } - } - } - catch (std::exception&) - { - } - - return rc; -} - -int DMLPackageProcessor::endTransaction (uint64_t uniqueId, BRM::TxnID txnID, bool success) -{ -//cout <<"in flushDataFiles" << endl; - ByteStream bytestream; - bytestream << (ByteStream::byte) WE_END_TRANSACTION; - bytestream << uniqueId; - bytestream << (uint32_t)txnID.id; - bytestream << (ByteStream::byte)success; - uint32_t msgRecived = 0; - fWEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - int rc = 0; - ByteStream::byte tmp8; - std::string errorMsg; - - try - { - while (1) - { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - break; - } - else - msgRecived++; - } - } - } - catch (std::exception&) - { - } - - return rc; -} + return rc; } +} // namespace dmlpackageprocessor // vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.h b/dbcon/dmlpackageproc/dmlpackageprocessor.h index 05c428781..a010f625d 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.h +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.h @@ -54,23 +54,23 @@ namespace dmlpackageprocessor { typedef std::vector dicStrValues; -#define SUMMARY_INFO( message ) \ - if ( isDebug(SUMMARY) ) \ - { \ - std::cerr << message << std::endl; \ - } +#define SUMMARY_INFO(message) \ + if (isDebug(SUMMARY)) \ + { \ + std::cerr << message << std::endl; \ + } -#define DETAIL_INFO( message ) \ - if ( isDebug(DETAIL) ) \ - { \ - std::cerr << message << std::endl; \ - } +#define DETAIL_INFO(message) \ + if (isDebug(DETAIL)) \ + { \ + std::cerr << message << std::endl; \ + } -#define VERBOSE_INFO( message ) \ - if ( isDebug(VERBOSE) ) \ - { \ - std::cerr << message << std::endl; \ - } +#define VERBOSE_INFO(message) \ + if (isDebug(VERBOSE)) \ + { \ + std::cerr << message << std::endl; \ + } typedef std::vector rids; @@ -79,476 +79,475 @@ typedef std::vector rids; */ class DMLPackageProcessor { + public: + /** @brief Result code + */ + enum ResultCode + { + NO_ERROR, + INSERT_ERROR, + NETWORK_ERROR, + NOTNULL_VIOLATION, + CHECK_VIOLATION, + DELETE_ERROR, + UPDATE_ERROR, + INDEX_UPDATE_ERROR, + COMMAND_ERROR, + TOKEN_ERROR, + NOT_ACCEPTING_PACKAGES, + DEAD_LOCK_ERROR, + REFERENCE_VIOLATION, + IDBRANGE_WARNING, + VB_OVERFLOW_ERROR, + ACTIVE_TRANSACTION_ERROR, + TABLE_LOCK_ERROR, + JOB_ERROR, + JOB_CANCELED + }; -public: - /** @brief Result code + enum DebugLevel /** @brief Debug level type enumeration */ + { + NONE = 0, /** @brief No debug info */ + SUMMARY = 1, /** @brief Summary level debug info */ + DETAIL = 2, /** @brief A little detail debug info */ + VERBOSE = 3, /** @brief Detailed debug info */ + }; + + /** @brief the result of dml operations + */ + struct DMLResult + { + /** @brief the result code */ - enum ResultCode - { - NO_ERROR, INSERT_ERROR, NETWORK_ERROR, NOTNULL_VIOLATION, - CHECK_VIOLATION, DELETE_ERROR, UPDATE_ERROR, INDEX_UPDATE_ERROR, - COMMAND_ERROR, TOKEN_ERROR, NOT_ACCEPTING_PACKAGES, DEAD_LOCK_ERROR, REFERENCE_VIOLATION, - IDBRANGE_WARNING, VB_OVERFLOW_ERROR, ACTIVE_TRANSACTION_ERROR, TABLE_LOCK_ERROR, JOB_ERROR, JOB_CANCELED - }; - - enum DebugLevel /** @brief Debug level type enumeration */ - { - NONE = 0, /** @brief No debug info */ - SUMMARY = 1, /** @brief Summary level debug info */ - DETAIL = 2, /** @brief A little detail debug info */ - VERBOSE = 3, /** @brief Detailed debug info */ - }; - - /** @brief the result of dml operations + ResultCode result; + /** @brief the error message if result != NO_ERROR */ - struct DMLResult - { - /** @brief the result code - */ - ResultCode result; - /** @brief the error message if result != NO_ERROR - */ - logging::Message message; - /** @brief the rowCount - */ - long long rowCount; - std::string tableLockInfo; - // query stats; - std::string queryStats; - std::string extendedStats; - std::string miniStats; - querystats::QueryStats stats; - - DMLResult(): result(NO_ERROR), rowCount(0) {}; - }; - /** @brief a structure to hold a date + logging::Message message; + /** @brief the rowCount */ - struct Date - { - unsigned spare : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; - // NULL column value = 0xFFFFFFFE - Date( ) - { - year = 0xFFFF; - month = 0xF; - day = 0x3F; - spare = 0x3E; - } - }; - /** @brief a structure to hold a datetime - */ - struct dateTime - { - unsigned msecond : 20; - unsigned second : 6; - unsigned minute : 6; - unsigned hour : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; - // NULL column value = 0xFFFFFFFFFFFFFFFE - dateTime( ) - { - year = 0xFFFF; - month = 0xF; - day = 0x3F; - hour = 0x3F; - minute = 0x3F; - second = 0x3F; - msecond = 0xFFFFE; - } - }; + long long rowCount; + std::string tableLockInfo; + // query stats; + std::string queryStats; + std::string extendedStats; + std::string miniStats; + querystats::QueryStats stats; - /** @brief ctor - */ - DMLPackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : fEC(0), DMLLoggingId(21), fRollbackPending(false), fDebugLevel(NONE) + DMLResult() : result(NO_ERROR), rowCount(0){}; + }; + /** @brief a structure to hold a date + */ + struct Date + { + unsigned spare : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; + // NULL column value = 0xFFFFFFFE + Date() { - try - { - fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DMLPROC); - //std::cout << "In DMLPackageProcessor constructor " << this << std::endl; - fPMCount = fWEClient->getPmCount(); - } - catch (...) - { - std::cout << "Cannot make connection to WES" << std::endl; - } + year = 0xFFFF; + month = 0xF; + day = 0x3F; + spare = 0x3E; + } + }; + /** @brief a structure to hold a datetime + */ + struct dateTime + { + unsigned msecond : 20; + unsigned second : 6; + unsigned minute : 6; + unsigned hour : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; + // NULL column value = 0xFFFFFFFFFFFFFFFE + dateTime() + { + year = 0xFFFF; + month = 0xF; + day = 0x3F; + hour = 0x3F; + minute = 0x3F; + second = 0x3F; + msecond = 0xFFFFE; + } + }; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - fDbRootPMMap = oamCache->getDBRootToPMMap(); - fDbrm = aDbrm; - fSessionID = sid; - fExeMgr = new execplan::ClientRotator(fSessionID, "ExeMgr"); - //std::cout << " fSessionID is " << fSessionID << std::endl; - fExeMgr->connect(0.005); + /** @brief ctor + */ + DMLPackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) + : fEC(0), DMLLoggingId(21), fRollbackPending(false), fDebugLevel(NONE) + { + try + { + fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DMLPROC); + // std::cout << "In DMLPackageProcessor constructor " << this << std::endl; + fPMCount = fWEClient->getPmCount(); + } + catch (...) + { + std::cout << "Cannot make connection to WES" << std::endl; } + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + fDbRootPMMap = oamCache->getDBRootToPMMap(); + fDbrm = aDbrm; + fSessionID = sid; + fExeMgr = new execplan::ClientRotator(fSessionID, "ExeMgr"); + // std::cout << " fSessionID is " << fSessionID << std::endl; + fExeMgr->connect(0.005); + } + /** @brief destructor + */ + EXPORT virtual ~DMLPackageProcessor(); - /** @brief destructor - */ - EXPORT virtual ~DMLPackageProcessor(); + /** @brief Is it required to debug + */ + inline bool isDebug(const DebugLevel level) const + { + return level <= fDebugLevel; + } - /** @brief Is it required to debug - */ - inline bool isDebug( const DebugLevel level ) const - { - return level <= fDebugLevel; - } + /** + * @brief Get debug level + */ + inline DebugLevel getDebugLevel() const + { + return fDebugLevel; + } + // int rollBackTransaction(uint64_t uniqueId, uint32_t txnID, uint32_t sessionID, std::string & errorMsg); + /** + * @brief Set debug level + */ + inline void setDebugLevel(const DebugLevel level) + { + fDebugLevel = level; + } - /** - * @brief Get debug level - */ - inline DebugLevel getDebugLevel() const - { - return fDebugLevel; - } - //int rollBackTransaction(uint64_t uniqueId, uint32_t txnID, uint32_t sessionID, std::string & errorMsg); - /** - * @brief Set debug level - */ - inline void setDebugLevel( const DebugLevel level ) - { - fDebugLevel = level; - } + /** + * @brief Set the Distributed Engine Comm object + */ + inline void setEngineComm(joblist::DistributedEngineComm* ec) + { + fEC = ec; + } - /** - * @brief Set the Distributed Engine Comm object - */ - inline void setEngineComm(joblist::DistributedEngineComm* ec) - { - fEC = ec; - } + /** @brief process the dml package + * + * @param cpackage the CalpontDMLPackage to process + */ + virtual DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage) = 0; - /** @brief process the dml package - * - * @param cpackage the CalpontDMLPackage to process - */ - virtual DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage) = 0; + inline void setRM(joblist::ResourceManager* frm) + { + fRM = frm; + }; - inline void setRM ( joblist::ResourceManager* frm) - { - fRM = frm; - }; + EXPORT int rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, + std::string& errorMsg); - EXPORT int rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, std::string& errorMsg); + EXPORT int rollBackBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, + const uint32_t tableOid, std::string& errorMsg); + /** + * @brief convert a columns data, represnted as a string, to it's native + * data type + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + // static boost::any convertColumnData( execplan::CalpontSystemCatalog::ColType colType, + // const std::string& dataOrig ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + // static std::string dateToString( int datevalue ); + /** @brief validate numerical string + * + * @param result the result structure + * @param data + */ - EXPORT int rollBackBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, const uint32_t tableOid, std::string& errorMsg); - /** - * @brief convert a columns data, represnted as a string, to it's native - * data type - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - //static boost::any convertColumnData( execplan::CalpontSystemCatalog::ColType colType, -// const std::string& dataOrig ); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - //static std::string dateToString( int datevalue ); - /** @brief validate numerical string - * - * @param result the result structure - * @param data - */ + // static bool numer_value( std::string data ); + /** @brief check whether the given year is leap year + * + * @param year + */ + // static bool isLeapYear ( int year); - //static bool numer_value( std::string data ); - /** @brief check whether the given year is leap year - * - * @param year - */ - //static bool isLeapYear ( int year); + /** @brief check whether the given date valid + * + * @param year, month, day + */ + // static bool isDateValid ( int day, int month, int year); - /** @brief check whether the given date valid - * - * @param year, month, day - */ - //static bool isDateValid ( int day, int month, int year); + /** @brief check whether the given datetime valid + * + * @param hour, minute, second, microSecond + */ + // static bool isDateTimeValid ( int hour, int minute, int second, int microSecond); - /** @brief check whether the given datetime valid - * - * @param hour, minute, second, microSecond - */ - //static bool isDateTimeValid ( int hour, int minute, int second, int microSecond); + /** @brief convert month from string to integer + * + * @param month + */ + // static int convertMonth (std::string month); - /** @brief convert month from string to integer - * - * @param month - */ - //static int convertMonth (std::string month); + /** @brief tokenize date or datetime string + * + * @param data + */ + // static void tokenTime (std::string data, std::vector& dataList); - /** @brief tokenize date or datetime string - * - * @param data - */ - //static void tokenTime (std::string data, std::vector& dataList); + /** @brief Access the rollback pending flag + */ + bool getRollbackPending() + { + return fRollbackPending; + } - /** @brief Access the rollback pending flag - */ - bool getRollbackPending() - { - return fRollbackPending; - } + /** @brief Set the rollback pending flag + */ + void setRollbackPending(bool rollback) + { + fRollbackPending = rollback; + } - /** @brief Set the rollback pending flag - */ - void setRollbackPending(bool rollback) - { - fRollbackPending = rollback; - } + protected: + /** @brief update the indexes on the target table + * + * @param schema the schema name + * @param table the table name + * @param rows the list of rows + * @param result the result structure + * @param updateFlag 0: delete, 1: update, 2 insert + * @param colNameList the updated column names, only valid for update SQL statement + */ + bool updateIndexes(uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const std::string& schema, + const std::string& table, const dmlpackage::RowList& rows, DMLResult& result, + std::vector ridList, WriteEngine::ColValueList& colValuesList, + std::vector& colNameList, const char updateFlag, + std::vector& dicStrValCols); -protected: - /** @brief update the indexes on the target table - * - * @param schema the schema name - * @param table the table name - * @param rows the list of rows - * @param result the result structure - * @param updateFlag 0: delete, 1: update, 2 insert - * @param colNameList the updated column names, only valid for update SQL statement - */ - bool updateIndexes( uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnID, const std::string& schema, - const std::string& table, const dmlpackage::RowList& rows, DMLResult& result, - std::vector ridList, - WriteEngine::ColValueList& colValuesList, std::vector& colNameList, const char updateFlag, - std::vector& dicStrValCols ); + /** @brief delete the indexes on the target table + * + * @param schema the schema name + * @param table the table name + * @param rows the list of rows + * @param result the result structure + */ + bool deleteIndexes(const std::string& schema, const std::string& table, const dmlpackage::RowList& rows, + DMLResult& result); - /** @brief delete the indexes on the target table - * - * @param schema the schema name - * @param table the table name - * @param rows the list of rows - * @param result the result structure - */ - bool deleteIndexes( const std::string& schema, const std::string& table, - const dmlpackage::RowList& rows, DMLResult& result ); + /** @brief validate that none of the columns in the supplied row(s) violate + * any defined constraints + * + * @param schema the schema name + * @param table the table name + * @param rows the lists of rows to check column constraints on + * @param result the result structure + */ + bool violatesConstraints(uint32_t sessionID, const std::string& schema, const std::string& table, + const dmlpackage::RowList& rows, DMLResult& result); - /** @brief validate that none of the columns in the supplied row(s) violate - * any defined constraints - * - * @param schema the schema name - * @param table the table name - * @param rows the lists of rows to check column constraints on - * @param result the result structure - */ - bool violatesConstraints( uint32_t sessionID, const std::string& schema, const std::string& table, - const dmlpackage::RowList& rows, DMLResult& result ); + /** @brief validate that the column does not violate a unique constraint + * + * @param column the column to validate + * @param result the result structure + */ + bool violatesUniqueConstraint(const dmlpackage::RowList& rows, + const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, + unsigned int sessionID, DMLResult& result); - /** @brief validate that the column does not violate a unique constraint - * - * @param column the column to validate - * @param result the result structure - */ - bool violatesUniqueConstraint( const dmlpackage::RowList& rows, + /** @brief validate that the column does not violate a check constraint + * + * @param column the column to validate + * @param result the result structure + */ + bool violatesCheckConstraint(const dmlpackage::RowList& rows, + const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, + unsigned int sessionID, unsigned int colOffset, DMLResult& result); + + /** @brief validate that the column does not violate a not null constraint + * + * @param type the columns database type + * @param column the column to validate + * @param result the result structure + */ + bool violatesNotNullConstraint(const dmlpackage::RowList& rows, unsigned int colOffset, DMLResult& result); + + /** @brief validate that the column does not violate a reference (foreign key) constraint + * + * @param rows the row set to validate + * @param colOffset the offset of this column in the row + * @param constraintInfo the column constraint infomation + * @param result the result structure + */ + bool violatesReferenceConstraint(const dmlpackage::RowList& rows, unsigned int colOffset, const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, - unsigned int sessionID, DMLResult& result); - /** @brief validate that the column does not violate a check constraint - * - * @param column the column to validate - * @param result the result structure - */ - bool violatesCheckConstraint( const dmlpackage::RowList& rows, - const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, - unsigned int sessionID, unsigned int colOffset, - DMLResult& result ); + /** @brief validate that non of the rows deleted from this table violate a reference + * (foreign key) constraint of the reference table. + * + * @param schema the schema name + * @param table the table name + * @param rows the lists of rows to check column constraints on + * @param result the result structure + */ + bool violatesPKRefConnstraint(uint32_t sessionID, const std::string& schema, const std::string& table, + const dmlpackage::RowList& rows, + const WriteEngine::ColValueList& oldValueList, DMLResult& result); - /** @brief validate that the column does not violate a not null constraint - * - * @param type the columns database type - * @param column the column to validate - * @param result the result structure - */ - bool violatesNotNullConstraint( const dmlpackage::RowList& rows, unsigned int colOffset, - DMLResult& result ); + bool violatesPKRefConnstraint(uint32_t sessionID, const std::string& schema, const std::string& table, + std::vector& rowIDList, std::vector& oldValueList, + DMLResult& result); - /** @brief validate that the column does not violate a reference (foreign key) constraint - * - * @param rows the row set to validate - * @param colOffset the offset of this column in the row - * @param constraintInfo the column constraint infomation - * @param result the result structure - */ - bool violatesReferenceConstraint( const dmlpackage::RowList& rows, - unsigned int colOffset, - const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, - DMLResult& result ); + /** @brief validate that the rows deleted does not violate a reference (foreign key) constraint + * + * @param rows the row set to validate + * @param colOffset the offset of this column in the row + * @param constraintInfo the column constraint infomation + * @param result the result structure + */ + bool violatesReferenceConstraint_PK(const WriteEngine::ColValueList& oldValueList, + const execplan::CalpontSystemCatalog::ColType& colType, + unsigned int colOffset, + const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, + DMLResult& result); - /** @brief validate that non of the rows deleted from this table violate a reference - * (foreign key) constraint of the reference table. - * - * @param schema the schema name - * @param table the table name - * @param rows the lists of rows to check column constraints on - * @param result the result structure - */ - bool violatesPKRefConnstraint ( uint32_t sessionID, - const std::string& schema, - const std::string& table, - const dmlpackage::RowList& rows, - const WriteEngine::ColValueList& oldValueList, - DMLResult& result ); + bool violatesReferenceConstraint_PK(std::vector& oldValueList, const int totalRows, + const execplan::CalpontSystemCatalog::ColType& colType, + unsigned int colOffset, + const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, + DMLResult& result); - bool violatesPKRefConnstraint ( uint32_t sessionID, - const std::string& schema, - const std::string& table, - std::vector& rowIDList, - std::vector& oldValueList, - DMLResult& result ); + /** @brief validate that none of the columns in the update row(s) violate + * any reference constraints of the foreign key table + * + * @param schema the schema name + * @param table the table name + * @param rows the lists of rows to check column constraints on + * @param result the result structure + */ + bool violatesUpdtRefConstraints(uint32_t sessionID, const std::string& schema, const std::string& table, + const dmlpackage::RowList& rows, DMLResult& result); - /** @brief validate that the rows deleted does not violate a reference (foreign key) constraint - * - * @param rows the row set to validate - * @param colOffset the offset of this column in the row - * @param constraintInfo the column constraint infomation - * @param result the result structure - */ - bool violatesReferenceConstraint_PK( const WriteEngine::ColValueList& oldValueList, - const execplan::CalpontSystemCatalog::ColType& colType, - unsigned int colOffset, - const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, - DMLResult& result ); + /** @brief validate that the column does not violate a reference (foreign key) constraint + * + * @param rows the row set to validate + * @param colOffset the offset of this column in the row + * @param constraintInfo the column constraint infomation + * @param result the result structure + */ + bool violatesReferenceConstraint_updt(const dmlpackage::RowList& rows, unsigned int colOffset, + const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, + DMLResult& result); - bool violatesReferenceConstraint_PK( std::vector& oldValueList, - const int totalRows, - const execplan::CalpontSystemCatalog::ColType& colType, - unsigned int colOffset, - const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, - DMLResult& result ); + /** @brief get the column list for the supplied table + * + * @param schema the schema name + * @param table the table name + * @param colList ColumnList to fill with the columns for the supplied table + */ + void getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, + dmlpackage::ColumnList& colList); - /** @brief validate that none of the columns in the update row(s) violate - * any reference constraints of the foreign key table - * - * @param schema the schema name - * @param table the table name - * @param rows the lists of rows to check column constraints on - * @param result the result structure - */ - bool violatesUpdtRefConstraints( uint32_t sessionID, - const std::string& schema, - const std::string& table, - const dmlpackage::RowList& rows, - DMLResult& result ); + /** @brief convert absolute rid to relative rid in a segement file + * + * @param rid in:the absolute rid out: relative rid in a segement file + * @param dbRoot,partition, segment the extent information obtained from rid + * @param filesPerColumnPartition,extentRows, extentsPerSegmentFile the extent map parameters + * @param startDBRoot the dbroot this table starts + * @param dbrootCnt the number of dbroot in db + */ + void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, unsigned& segment, + unsigned filesPerColumnPartition, unsigned extentsPerSegmentFile, + unsigned extentRows, unsigned startDBRoot, unsigned dbrootCnt, + const unsigned startPartitionNum); - - /** @brief validate that the column does not violate a reference (foreign key) constraint - * - * @param rows the row set to validate - * @param colOffset the offset of this column in the row - * @param constraintInfo the column constraint infomation - * @param result the result structure - */ - bool violatesReferenceConstraint_updt( const dmlpackage::RowList& rows, - unsigned int colOffset, - const execplan::CalpontSystemCatalog::ConstraintInfo& constraintInfo, - DMLResult& result ); - - - /** @brief get the column list for the supplied table - * - * @param schema the schema name - * @param table the table name - * @param colList ColumnList to fill with the columns for the supplied table - */ - void getColumnsForTable( uint32_t sessionID, std::string schema, std::string table, dmlpackage::ColumnList& colList ); - - /** @brief convert absolute rid to relative rid in a segement file - * - * @param rid in:the absolute rid out: relative rid in a segement file - * @param dbRoot,partition, segment the extent information obtained from rid - * @param filesPerColumnPartition,extentRows, extentsPerSegmentFile the extent map parameters - * @param startDBRoot the dbroot this table starts - * @param dbrootCnt the number of dbroot in db - */ - void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, - unsigned& segment, unsigned filesPerColumnPartition, - unsigned extentsPerSegmentFile, unsigned extentRows, - unsigned startDBRoot, unsigned dbrootCnt, - const unsigned startPartitionNum ); - - inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType ) + inline bool isDictCol(execplan::CalpontSystemCatalog::ColType colType) + { + if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || + ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || + ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) || + (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || + (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || + (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) { - if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) - || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) - || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) - || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) - { - return true; - } - else - return false; + return true; } + else + return false; + } + /** @brief convert an error code to a string + * + * @param ec in:the error code received + * @returns error string + */ + std::string projectTableErrCodeToMsg(uint32_t ec); - /** @brief convert an error code to a string - * - * @param ec in:the error code received - * @returns error string - */ - std::string projectTableErrCodeToMsg(uint32_t ec); + // bool validateNextValue(execplan::CalpontSystemCatalog::ColType colType, int64_t value, bool & + // offByOne); -// bool validateNextValue(execplan::CalpontSystemCatalog::ColType colType, int64_t value, bool & offByOne); + bool validateVarbinaryVal(std::string& inStr); + int commitTransaction(uint64_t uniqueId, BRM::TxnID txnID); + int commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, + std::string& errorMsg); + int commitBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, + std::string& errorMsg); + int rollBackBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, + const uint32_t tableOid, std::string& errorMsg); + int flushDataFiles(int rc, std::map& columnOids, uint64_t uniqueId, BRM::TxnID txnID, + uint32_t tableOid); + int endTransaction(uint64_t uniqueId, BRM::TxnID txnID, bool success); - bool validateVarbinaryVal( std::string& inStr); - int commitTransaction(uint64_t uniqueId, BRM::TxnID txnID); - int commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, std::string& errorMsg); - int commitBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, std::string& errorMsg); - int rollBackBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID, const uint32_t tableOid, std::string& errorMsg); - int flushDataFiles (int rc, std::map& columnOids, uint64_t uniqueId, BRM::TxnID txnID, uint32_t tableOid); - int endTransaction (uint64_t uniqueId, BRM::TxnID txnID, bool success); + /** @brief the Session Manager interface + */ + execplan::SessionManager fSessionManager; + joblist::DistributedEngineComm* fEC; + joblist::ResourceManager* fRM; + char* strlower(char* in); + uint32_t fSessionID; + const unsigned DMLLoggingId; + uint32_t fPMCount; + WriteEngine::WEClients* fWEClient; + BRM::DBRM* fDbrm; + boost::shared_ptr > fDbRootPMMap; + oam::Oam fOam; + bool fRollbackPending; // When set, any derived object should stop what it's doing and cleanup in + // preparation for a Rollback + execplan::ClientRotator* fExeMgr; - /** @brief the Session Manager interface - */ - execplan::SessionManager fSessionManager; - joblist::DistributedEngineComm* fEC; - joblist::ResourceManager* fRM; - char* strlower(char* in); - uint32_t fSessionID; - const unsigned DMLLoggingId; - uint32_t fPMCount; - WriteEngine::WEClients* fWEClient; - BRM::DBRM* fDbrm; - boost::shared_ptr > fDbRootPMMap; - oam::Oam fOam; - bool fRollbackPending; // When set, any derived object should stop what it's doing and cleanup in preparation for a Rollback - execplan::ClientRotator* fExeMgr; - -private: - - /** @brief clean beginning and ending glitches and spaces from string - * - * @param s string to be cleaned - */ - void cleanString(std::string& s); - - DebugLevel fDebugLevel; // internal use debug level + private: + /** @brief clean beginning and ending glitches and spaces from string + * + * @param s string to be cleaned + */ + void cleanString(std::string& s); + DebugLevel fDebugLevel; // internal use debug level }; /** @brief helper template function to do safe from string to type conversions * */ template -bool from_string(T& t, - const std::string& s, - std::ios_base & (*f)(std::ios_base&)) +bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&)) { - std::istringstream iss(s); - return !(iss >> f >> t).fail(); + std::istringstream iss(s); + return !(iss >> f >> t).fail(); } -} +} // namespace dmlpackageprocessor #undef EXPORT -#endif //DMLPACKAGEPROCESSOR_H +#endif // DMLPACKAGEPROCESSOR_H diff --git a/dbcon/dmlpackageproc/dmlpackageprocessorfactory.cpp b/dbcon/dmlpackageproc/dmlpackageprocessorfactory.cpp index 5023cb806..70537a56f 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessorfactory.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessorfactory.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlpackageprocessorfactory.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlpackageprocessorfactory.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #define DMLPKGPROCFACTORY_DLLEXPORT #include "dmlpackageprocessorfactory.h" #undef DMLPKGPROCFACTORY_DLLEXPORT @@ -33,34 +33,23 @@ using namespace dmlpackage; namespace dmlpackageprocessor { - -DMLPackageProcessor* DMLPackageProcessorFactory:: -makePackageProcessor(int packageType, dmlpackage::CalpontDMLPackage& cpackage) +DMLPackageProcessor* DMLPackageProcessorFactory::makePackageProcessor(int packageType, + dmlpackage::CalpontDMLPackage& cpackage) { - DMLPackageProcessor* dmlProcPtr = 0; + DMLPackageProcessor* dmlProcPtr = 0; - switch ( packageType ) - { - case DML_INSERT: - dmlProcPtr = new InsertPackageProcessor(); - break; + switch (packageType) + { + case DML_INSERT: dmlProcPtr = new InsertPackageProcessor(); break; - case DML_DELETE: - dmlProcPtr = new DeletePackageProcessor(); - break; + case DML_DELETE: dmlProcPtr = new DeletePackageProcessor(); break; - case DML_UPDATE: - dmlProcPtr = new UpdatePackageProcessor(); - break; + case DML_UPDATE: dmlProcPtr = new UpdatePackageProcessor(); break; - case DML_COMMAND: - dmlProcPtr = new CommandPackageProcessor(); - break; + case DML_COMMAND: dmlProcPtr = new CommandPackageProcessor(); break; + } - } - - return dmlProcPtr; + return dmlProcPtr; } -} // namespace dmlpackageprocessor - +} // namespace dmlpackageprocessor diff --git a/dbcon/dmlpackageproc/dmlpackageprocessorfactory.h b/dbcon/dmlpackageproc/dmlpackageprocessorfactory.h index 3c46fde65..9f7120e62 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessorfactory.h +++ b/dbcon/dmlpackageproc/dmlpackageprocessorfactory.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlpackageprocessorfactory.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlpackageprocessorfactory.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DMLPACKAGEPROCESSORFACTORY_H #define DMLPACKAGEPROCESSORFACTORY_H @@ -35,32 +35,26 @@ namespace dmlpackageprocessor { - /** @brief create a DMLPackageProcessor object from a CalpontDMLPackage object * */ class DMLPackageProcessorFactory { + public: + /** @brief static DMLPackageProcessor constructor method + * + * @param packageType the DML Package type + * @param cpackage the CalpontDMLPackage from which the DMLPackageProcessor is constructed + */ + EXPORT static DMLPackageProcessor* makePackageProcessor(int packageType, + dmlpackage::CalpontDMLPackage& cpackage); -public: - - /** @brief static DMLPackageProcessor constructor method - * - * @param packageType the DML Package type - * @param cpackage the CalpontDMLPackage from which the DMLPackageProcessor is constructed - */ - EXPORT static DMLPackageProcessor* - makePackageProcessor( int packageType, dmlpackage::CalpontDMLPackage& cpackage ); - -protected: - -private: - + protected: + private: }; -} +} // namespace dmlpackageprocessor #undef EXPORT -#endif //DMLPACKAGEPROCESSORFACTORY_H - +#endif // DMLPACKAGEPROCESSORFACTORY_H diff --git a/dbcon/dmlpackageproc/insertpackageprocessor.cpp b/dbcon/dmlpackageproc/insertpackageprocessor.cpp index 74795a655..fb844b696 100644 --- a/dbcon/dmlpackageproc/insertpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/insertpackageprocessor.cpp @@ -50,385 +50,386 @@ using namespace messageqcpp; namespace dmlpackageprocessor { - DMLPackageProcessor::DMLResult InsertPackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) { - SUMMARY_INFO("InsertPackageProcessor::processPackage"); + SUMMARY_INFO("InsertPackageProcessor::processPackage"); - DMLResult result; - result.result = NO_ERROR; - BRM::TxnID txnid; - // set-up the transaction - txnid.id = cpackage.get_TxnID(); - txnid.valid = true; - fSessionID = cpackage.get_SessionID(); - DMLTable* tablePtr = cpackage.get_Table(); + DMLResult result; + result.result = NO_ERROR; + BRM::TxnID txnid; + // set-up the transaction + txnid.id = cpackage.get_TxnID(); + txnid.valid = true; + fSessionID = cpackage.get_SessionID(); + DMLTable* tablePtr = cpackage.get_Table(); - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Start SQL statement: "); - ostringstream oss; - oss << cpackage.get_SQLStatement() << "; |" << tablePtr->get_SchemaName() << "|"; - args1.add(oss.str()); + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("Start SQL statement: "); + ostringstream oss; + oss << cpackage.get_SQLStatement() << "; |" << tablePtr->get_SchemaName() << "|"; + args1.add(oss.str()); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - //WriteEngine::ChunkManager* cm = cpackage.get_ChunkManager(); - //fWriteEngine.setChunkManager(cm); - //std::map oids; - VERBOSE_INFO("Processing Insert DML Package..."); - uint64_t uniqueId = 0; + msg.format(args1); + Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + // WriteEngine::ChunkManager* cm = cpackage.get_ChunkManager(); + // fWriteEngine.setChunkManager(cm); + // std::map oids; + VERBOSE_INFO("Processing Insert DML Package..."); + uint64_t uniqueId = 0; - //Bug 5070. Added exception handling - try + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(9); + args.add(ex.what()); + message.format(args); + result.result = INSERT_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + catch (...) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = INSERT_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + + uint64_t tableLockId = 0; + int rc = 0; + std::string errorMsg; + OamCache* oamcache = OamCache::makeOamCache(); + std::vector moduleIds = oamcache->getModuleIds(); + std::vector pms; + + try + { + for (unsigned int i = 0; i < moduleIds.size(); i++) { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(9); - args.add(ex.what()); - message.format(args); - result.result = INSERT_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; - } - catch ( ... ) - { - logging::Message::Args args; - logging::Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = INSERT_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; + pms.push_back((uint32_t)moduleIds[i]); } - uint64_t tableLockId = 0; - int rc = 0; - std::string errorMsg; - OamCache* oamcache = OamCache::makeOamCache(); - std::vector moduleIds = oamcache->getModuleIds(); - std::vector pms; + // cout << "single insert get transaction id " << txnid.id << endl; + // get the table object from the package + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + // cout << "DMLProc using syscatptr:sessionid = " << systemCatalogPtr <<":" << fSessionID<< endl; + CalpontSystemCatalog::TableName tableName; + execplan::CalpontSystemCatalog::ROPair roPair; + TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - try + if (0 != tablePtr) { - for (unsigned int i = 0; i < moduleIds.size(); i++) + // check table lock + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + systemCatalogPtr->sessionID(fSessionID); + tableName.schema = tablePtr->get_SchemaName(); + tableName.table = tablePtr->get_TableName(); + roPair = systemCatalogPtr->tableRID(tableName); + + tableLockId = tablelockData->getTablelockId( + roPair.objnum); // check whether this table is locked already for this session + + if (tableLockId == 0) + { + // cout << "tablelock is not found in cache, getting from dbrm" << endl; + uint32_t processID = ::getpid(); + int32_t txnId = txnid.id; + int32_t sessionId = fSessionID; + std::string processName("DMLProc"); + int i = 0; + + try { - pms.push_back((uint32_t)moduleIds[i]); + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, + BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); } - //cout << "single insert get transaction id " << txnid.id << endl; - // get the table object from the package - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); - //cout << "DMLProc using syscatptr:sessionid = " << systemCatalogPtr <<":" << fSessionID<< endl; - CalpontSystemCatalog::TableName tableName; - execplan::CalpontSystemCatalog::ROPair roPair; - TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - - if (0 != tablePtr) + if (tableLockId == 0) { - //check table lock - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - systemCatalogPtr->sessionID(fSessionID); - tableName.schema = tablePtr->get_SchemaName(); - tableName.table = tablePtr->get_TableName(); - roPair = systemCatalogPtr->tableRID( tableName ); + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; - tableLockId = tablelockData->getTablelockId(roPair.objnum); //check whether this table is locked already for this session + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - if (tableLockId == 0) - { - //cout << "tablelock is not found in cache, getting from dbrm" << endl; - uint32_t processID = ::getpid(); - int32_t txnId = txnid.id; - int32_t sessionId = fSessionID; - std::string processName("DMLProc"); - int i = 0; - - try - { - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if ( tableLockId == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { + for (; i < numTries; i++) + { #ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); + Sleep(rm_ts.tv_sec * 1000); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif - try - { - processID = ::getpid(); - txnId = txnid.id; - sessionId = fSessionID; - processName = "DMLProc"; - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (tableLockId > 0) - break; - } - - if (i >= numTries) //error out - { - result.result = INSERT_ERROR; - logging::Message::Args args; - string strOp("insert"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add(sessionId); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - } - } - } - - //cout << " tablelock is obtained with id " << tableLockId << endl; - tablelockData->setTablelock(roPair.objnum, tableLockId); - - int pmNum = 0; - - // Select PM to receive the row. - // 1. Get BRM information - // 2. Find the DBRoot with the fewest in-service blocks. - // DBRoots having no blocks are excluded - // 3. Map the selected DBRoot to the corresponding PM - CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true); - std::vector allInfo (pms.size()); - - for (unsigned i = 0; i < pms.size(); i++) - { - rc = fDbrm->getDbRootHWMInfo((ridList[0].objnum), pms[i], allInfo[i]); - - if ( rc != 0 ) //@Bug 4760. - { - result.result = INSERT_ERROR; - ostringstream oss; - oss << "Error getting extent information for table " << tableName.table; - throw std::runtime_error(oss.str()); - } - } - - // Find DBRoot with fewest blocks; if all DBRoots - // have 0 blocks, then we select the first DBRoot - BRM::EmDbRootHWMInfo tmp; - bool tmpSet = false; - - for (unsigned i = 0; i < allInfo.size(); i++) - { - BRM::EmDbRootHWMInfo_v emDbRootHWMInfos = allInfo[i]; - - for (unsigned j = 0; j < emDbRootHWMInfos.size(); j++) - { - if (!tmpSet) - { - tmp = emDbRootHWMInfos[j]; - tmpSet = true; - } - else if (emDbRootHWMInfos[j].totalBlocks > 0) - { - if ((emDbRootHWMInfos[j].totalBlocks < tmp.totalBlocks) || - (tmp.totalBlocks == 0)) - { - tmp = emDbRootHWMInfos[j]; - } - } - } - } - - // Select the PM to receive the row - uint32_t dbroot; - - if (tmpSet) - { - dbroot = tmp.dbRoot; - boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); - pmNum = (*dbRootPMMap)[dbroot]; - - //@Bug 4760. validate pm value - if (pmNum == 0) - { - result.result = INSERT_ERROR; - ostringstream oss; - oss << "Error mapping extent/DBRoot to PM for table " << tableName.table; - throw std::runtime_error(oss.str()); - } - } - else - { - result.result = INSERT_ERROR; - ostringstream oss; - oss << "There is no extent information for table " << tableName.table; - throw std::runtime_error(oss.str()); - } - - //This is for single insert only. Batch insert is handled in dmlprocessor. - //cout << "fWEClient = " << fWEClient << endl; - fWEClient->addQueue(uniqueId); - ByteStream bytestream; - bytestream << (uint8_t)WE_SVR_SINGLE_INSERT; - bytestream << uniqueId; - bytestream << (uint32_t)txnid.id; - bytestream << dbroot; - cpackage.write(bytestream); - boost::shared_ptr bsIn; - - ByteStream::byte rc1; - try { - fWEClient->write(bytestream, (uint32_t)pmNum); -#ifdef IDB_DML_DEBUG - cout << "Single insert sending WE_SVR_SINGLE_INSERT to pm " << pmNum << endl; -#endif - - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = NETWORK_ERROR; - errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; - } - else - { - *bsIn >> rc1; - - if (rc1 != 0) - { - *bsIn >> errorMsg; - rc = rc1; - } - } - + processID = ::getpid(); + txnId = txnid.id; + sessionId = fSessionID; + processName = "DMLProc"; + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, + &txnId, BRM::LOADING); } - catch (runtime_error& ex) //write error + catch (std::exception&) { -#ifdef IDB_DML_DEBUG - cout << "Single insert got exception" << ex.what() << endl; -#endif - rc = NETWORK_ERROR; - errorMsg = ex.what(); - } - catch (...) - { - errorMsg = "Caught ... exception during single row insert"; - rc = NETWORK_ERROR; -#ifdef IDB_DML_DEBUG - cout << "Single insert got unknown exception" << endl; -#endif + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); } - // Log the insert statement. - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement() + ";", cpackage.get_SchemaName()); - } - } - catch (exception& ex) - { - cerr << "InsertPackageProcessor::processPackage: " << ex.what() << endl; + if (tableLockId > 0) + break; + } - logging::Message::Args args; - logging::Message message(1); - args.add("Insert Failed: "); - args.add(ex.what()); - args.add(""); - args.add(""); - message.format(args); - - if ( result.result != VB_OVERFLOW_ERROR ) - { + if (i >= numTries) // error out + { result.result = INSERT_ERROR; - result.message = message; - errorMsg = ex.what(); + logging::Message::Args args; + string strOp("insert"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add(sessionId); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + } } - } - catch (...) - { - cerr << "InsertPackageProcessor::processPackage: caught unknown exception!" << endl; - logging::Message::Args args; - logging::Message message(1); - args.add("Insert Failed: "); - args.add("encountered unkown exception"); - args.add(""); - args.add(""); - message.format(args); + } + // cout << " tablelock is obtained with id " << tableLockId << endl; + tablelockData->setTablelock(roPair.objnum, tableLockId); + + int pmNum = 0; + + // Select PM to receive the row. + // 1. Get BRM information + // 2. Find the DBRoot with the fewest in-service blocks. + // DBRoots having no blocks are excluded + // 3. Map the selected DBRoot to the corresponding PM + CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true); + std::vector allInfo(pms.size()); + + for (unsigned i = 0; i < pms.size(); i++) + { + rc = fDbrm->getDbRootHWMInfo((ridList[0].objnum), pms[i], allInfo[i]); + + if (rc != 0) //@Bug 4760. + { + result.result = INSERT_ERROR; + ostringstream oss; + oss << "Error getting extent information for table " << tableName.table; + throw std::runtime_error(oss.str()); + } + } + + // Find DBRoot with fewest blocks; if all DBRoots + // have 0 blocks, then we select the first DBRoot + BRM::EmDbRootHWMInfo tmp; + bool tmpSet = false; + + for (unsigned i = 0; i < allInfo.size(); i++) + { + BRM::EmDbRootHWMInfo_v emDbRootHWMInfos = allInfo[i]; + + for (unsigned j = 0; j < emDbRootHWMInfos.size(); j++) + { + if (!tmpSet) + { + tmp = emDbRootHWMInfos[j]; + tmpSet = true; + } + else if (emDbRootHWMInfos[j].totalBlocks > 0) + { + if ((emDbRootHWMInfos[j].totalBlocks < tmp.totalBlocks) || (tmp.totalBlocks == 0)) + { + tmp = emDbRootHWMInfos[j]; + } + } + } + } + + // Select the PM to receive the row + uint32_t dbroot; + + if (tmpSet) + { + dbroot = tmp.dbRoot; + boost::shared_ptr > dbRootPMMap = oamcache->getDBRootToPMMap(); + pmNum = (*dbRootPMMap)[dbroot]; + + //@Bug 4760. validate pm value + if (pmNum == 0) + { + result.result = INSERT_ERROR; + ostringstream oss; + oss << "Error mapping extent/DBRoot to PM for table " << tableName.table; + throw std::runtime_error(oss.str()); + } + } + else + { result.result = INSERT_ERROR; - result.message = message; - } + ostringstream oss; + oss << "There is no extent information for table " << tableName.table; + throw std::runtime_error(oss.str()); + } - if (( rc != 0) && (rc != IDBRANGE_WARNING)) - { - logging::Message::Args args; - logging::Message message(1); - args.add("Insert Failed: "); - args.add(errorMsg); - args.add(""); - args.add(""); - message.format(args); - result.result = INSERT_ERROR; - result.message = message; - } - else if (rc == IDBRANGE_WARNING) - { - logging::Message::Args args; - logging::Message message(1); - args.add(errorMsg); - args.add(""); - args.add(""); - message.format(args); - result.result = IDBRANGE_WARNING; - result.message = message; - } + // This is for single insert only. Batch insert is handled in dmlprocessor. + // cout << "fWEClient = " << fWEClient << endl; + fWEClient->addQueue(uniqueId); + ByteStream bytestream; + bytestream << (uint8_t)WE_SVR_SINGLE_INSERT; + bytestream << uniqueId; + bytestream << (uint32_t)txnid.id; + bytestream << dbroot; + cpackage.write(bytestream); + boost::shared_ptr bsIn; - fWEClient->removeQueue(uniqueId); - VERBOSE_INFO("Finished Processing Insert DML Package"); - return result; + ByteStream::byte rc1; + + try + { + fWEClient->write(bytestream, (uint32_t)pmNum); +#ifdef IDB_DML_DEBUG + cout << "Single insert sending WE_SVR_SINGLE_INSERT to pm " << pmNum << endl; +#endif + + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = NETWORK_ERROR; + errorMsg = "Lost connection to Write Engine Server while updating SYSTABLES"; + } + else + { + *bsIn >> rc1; + + if (rc1 != 0) + { + *bsIn >> errorMsg; + rc = rc1; + } + } + } + catch (runtime_error& ex) // write error + { +#ifdef IDB_DML_DEBUG + cout << "Single insert got exception" << ex.what() << endl; +#endif + rc = NETWORK_ERROR; + errorMsg = ex.what(); + } + catch (...) + { + errorMsg = "Caught ... exception during single row insert"; + rc = NETWORK_ERROR; +#ifdef IDB_DML_DEBUG + cout << "Single insert got unknown exception" << endl; +#endif + } + + // Log the insert statement. + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement"); + msg.format(args1); + Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement() + ";", + cpackage.get_SchemaName()); + } + } + catch (exception& ex) + { + cerr << "InsertPackageProcessor::processPackage: " << ex.what() << endl; + + logging::Message::Args args; + logging::Message message(1); + args.add("Insert Failed: "); + args.add(ex.what()); + args.add(""); + args.add(""); + message.format(args); + + if (result.result != VB_OVERFLOW_ERROR) + { + result.result = INSERT_ERROR; + result.message = message; + errorMsg = ex.what(); + } + } + catch (...) + { + cerr << "InsertPackageProcessor::processPackage: caught unknown exception!" << endl; + logging::Message::Args args; + logging::Message message(1); + args.add("Insert Failed: "); + args.add("encountered unkown exception"); + args.add(""); + args.add(""); + message.format(args); + + result.result = INSERT_ERROR; + result.message = message; + } + + if ((rc != 0) && (rc != IDBRANGE_WARNING)) + { + logging::Message::Args args; + logging::Message message(1); + args.add("Insert Failed: "); + args.add(errorMsg); + args.add(""); + args.add(""); + message.format(args); + result.result = INSERT_ERROR; + result.message = message; + } + else if (rc == IDBRANGE_WARNING) + { + logging::Message::Args args; + logging::Message message(1); + args.add(errorMsg); + args.add(""); + args.add(""); + message.format(args); + result.result = IDBRANGE_WARNING; + result.message = message; + } + + fWEClient->removeQueue(uniqueId); + VERBOSE_INFO("Finished Processing Insert DML Package"); + return result; } -} // namespace dmlpackageprocessor +} // namespace dmlpackageprocessor // vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/insertpackageprocessor.h b/dbcon/dmlpackageproc/insertpackageprocessor.h index 75458fffb..5d71f725d 100644 --- a/dbcon/dmlpackageproc/insertpackageprocessor.h +++ b/dbcon/dmlpackageproc/insertpackageprocessor.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: insertpackageprocessor.h 9473 2013-05-02 15:15:44Z dcathey $ -* -* -***********************************************************************/ + * $Id: insertpackageprocessor.h 9473 2013-05-02 15:15:44Z dcathey $ + * + * + ***********************************************************************/ /** @file */ #ifndef INSERTPACKAGEPROCESSOR_H @@ -41,31 +41,27 @@ namespace dmlpackageprocessor { /** @brief concrete implementation of a DMLPackageProcessor. - * Specifically for interacting with the Write Engine to - * process INSERT dml statements. - */ + * Specifically for interacting with the Write Engine to + * process INSERT dml statements. + */ class InsertPackageProcessor : public DMLPackageProcessor { + public: + InsertPackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) + { + } + /** @brief process an InsertDMLPackage + * + * @param cpackage the InsertDMLPackage to process + */ + EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); -public: - InsertPackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) - { - } - /** @brief process an InsertDMLPackage - * - * @param cpackage the InsertDMLPackage to process - */ - EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); - -protected: - -private: - + protected: + private: }; -} +} // namespace dmlpackageprocessor #undef EXPORT -#endif //INSERTPACKAGEPROCESSOR_H - +#endif // INSERTPACKAGEPROCESSOR_H diff --git a/dbcon/dmlpackageproc/resource.h b/dbcon/dmlpackageproc/resource.h index 371529752..9cdcbea14 100644 --- a/dbcon/dmlpackageproc/resource.h +++ b/dbcon/dmlpackageproc/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/dbcon/dmlpackageproc/tablelockdata.cpp b/dbcon/dmlpackageproc/tablelockdata.cpp index 15645fdcc..d5f94ec60 100644 --- a/dbcon/dmlpackageproc/tablelockdata.cpp +++ b/dbcon/dmlpackageproc/tablelockdata.cpp @@ -32,7 +32,6 @@ using namespace boost; namespace dmlpackageprocessor { - /*static*/ boost::mutex TablelockData::map_mutex; /*static*/ @@ -40,31 +39,31 @@ TablelockData::TablelockDataMap TablelockData::fTablelockDataMap; /* static */ TablelockData* TablelockData::makeTablelockData(uint32_t sessionID) { - boost::mutex::scoped_lock lock(map_mutex); - TablelockData* instance; - TablelockDataMap::const_iterator it = fTablelockDataMap.find(sessionID); + boost::mutex::scoped_lock lock(map_mutex); + TablelockData* instance; + TablelockDataMap::const_iterator it = fTablelockDataMap.find(sessionID); - if (it == fTablelockDataMap.end()) - { - instance = new TablelockData(); - fTablelockDataMap[sessionID] = instance; - return instance; - } + if (it == fTablelockDataMap.end()) + { + instance = new TablelockData(); + fTablelockDataMap[sessionID] = instance; + return instance; + } - return it->second; + return it->second; } /* static */ void TablelockData::removeTablelockData(uint32_t sessionID) { - boost::mutex::scoped_lock lock(map_mutex); - TablelockDataMap::iterator it = fTablelockDataMap.find(sessionID); + boost::mutex::scoped_lock lock(map_mutex); + TablelockDataMap::iterator it = fTablelockDataMap.find(sessionID); - if (it != fTablelockDataMap.end()) - { - delete (*it).second; - fTablelockDataMap.erase(it); - } + if (it != fTablelockDataMap.end()) + { + delete (*it).second; + fTablelockDataMap.erase(it); + } } TablelockData::TablelockData() @@ -76,30 +75,30 @@ TablelockData::~TablelockData() void TablelockData::setTablelock(uint32_t tableOid, uint64_t tablelockId) { - boost::mutex::scoped_lock lk(fOIDTablelock); - fOIDTablelockMap[tableOid] = tablelockId; + boost::mutex::scoped_lock lk(fOIDTablelock); + fOIDTablelockMap[tableOid] = tablelockId; } uint64_t TablelockData::getTablelockId(uint32_t tableOid) { - boost::mutex::scoped_lock lk(fOIDTablelock); - uint64_t tablelockId = 0; - OIDTablelock::iterator it = fOIDTablelockMap.find(tableOid); + boost::mutex::scoped_lock lk(fOIDTablelock); + uint64_t tablelockId = 0; + OIDTablelock::iterator it = fOIDTablelockMap.find(tableOid); - if (it != fOIDTablelockMap.end()) - { - tablelockId = it->second; - } + if (it != fOIDTablelockMap.end()) + { + tablelockId = it->second; + } - return tablelockId; + return tablelockId; } TablelockData::OIDTablelock& TablelockData::getOidTablelockMap() { - boost::mutex::scoped_lock lk(fOIDTablelock); + boost::mutex::scoped_lock lk(fOIDTablelock); - return fOIDTablelockMap; + return fOIDTablelockMap; } -} +} // namespace dmlpackageprocessor // vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/tablelockdata.h b/dbcon/dmlpackageproc/tablelockdata.h index 29a0fbb1d..a5910d849 100644 --- a/dbcon/dmlpackageproc/tablelockdata.h +++ b/dbcon/dmlpackageproc/tablelockdata.h @@ -35,34 +35,32 @@ namespace dmlpackageprocessor { - class TablelockData { -public: - typedef std::map TablelockDataMap; - typedef std::map OIDTablelock; - EXPORT static TablelockData* makeTablelockData(uint32_t sessionID = 0); - EXPORT static void removeTablelockData(uint32_t sessionID = 0); - EXPORT void setTablelock(uint32_t tableOid, uint64_t tablelockId); - EXPORT uint64_t getTablelockId(uint32_t tableOid); - OIDTablelock& getOidTablelockMap(); + public: + typedef std::map TablelockDataMap; + typedef std::map OIDTablelock; + EXPORT static TablelockData* makeTablelockData(uint32_t sessionID = 0); + EXPORT static void removeTablelockData(uint32_t sessionID = 0); + EXPORT void setTablelock(uint32_t tableOid, uint64_t tablelockId); + EXPORT uint64_t getTablelockId(uint32_t tableOid); + OIDTablelock& getOidTablelockMap(); -private: - /** Constuctors */ - explicit TablelockData(); - explicit TablelockData(const TablelockData& rhs); - ~TablelockData(); + private: + /** Constuctors */ + explicit TablelockData(); + explicit TablelockData(const TablelockData& rhs); + ~TablelockData(); - static boost::mutex map_mutex; - static TablelockDataMap fTablelockDataMap; - OIDTablelock fOIDTablelockMap; - boost::mutex fOIDTablelock; + static boost::mutex map_mutex; + static TablelockDataMap fTablelockDataMap; + OIDTablelock fOIDTablelockMap; + boost::mutex fOIDTablelock; }; -} +} // namespace dmlpackageprocessor #undef EXPORT #endif // vim:ts=4 sw=4: - diff --git a/dbcon/dmlpackageproc/tdriver.cpp b/dbcon/dmlpackageproc/tdriver.cpp index 3cf9732d4..44d1159ea 100644 --- a/dbcon/dmlpackageproc/tdriver.cpp +++ b/dbcon/dmlpackageproc/tdriver.cpp @@ -43,7 +43,7 @@ using namespace ddlpackage; #include "createtableprocessor.h" //#include "createindexprocessor.h" -//using namespace ddlpackageprocessor; +// using namespace ddlpackageprocessor; #include "writeengine.h" #include "we_colop.h" @@ -66,964 +66,956 @@ using namespace dmlpackageprocessor; using namespace dmlpackage; using namespace std; - class SystemCatalogBuilder { -public: - static void build() - { - ColumnOp colOp; - Column curCol; - WriteEngine::WriteEngineWrapper fWriteEngine; - WriteEngine::TxnID txnID = 0; - int rc; - - remove(); - - cout << "Creating System Catalog..." << endl; - - colOp.initColumn( curCol ); - // SYSTABLE - - // TableName - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1001 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2001, 2002, 2003); - // Schema - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1002 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2004, 2005, 2006); - // CreateDate - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1003 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastUpdateDate - rc = colOp.createColumn( curCol, 3, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1004 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // INIT - rc = colOp.createColumn( curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1005 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // NEXT - rc = colOp.createColumn( curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1006 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //SYSCOLUMN - // Shema - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1007 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2007, 2008, 2009); - - // TableName - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1008 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2010, 2011, 2012); - - // ColumnName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1009 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2013, 2014, 2015); - // ObjectID - rc = colOp.createColumn( curCol, 3, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1010 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DictOID - rc = colOp.createColumn( curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1011 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ListOID - rc = colOp.createColumn( curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1012 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // TreeOID - rc = colOp.createColumn( curCol, 6, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1013 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DataType - rc = colOp.createColumn( curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1014 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ColumnLength - rc = colOp.createColumn( curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1015 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ColumnPos - rc = colOp.createColumn( curCol, 9, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1016 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastUpdate - rc = colOp.createColumn( curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1017 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DefaultValue - rc = colOp.createColumn( curCol, 11, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1018 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2016, 2017, 2018); - // Nullable - rc = colOp.createColumn( curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1019 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // Scale - rc = colOp.createColumn( curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1020 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // Precision - rc = colOp.createColumn( curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1021 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // AutoInc - rc = colOp.createColumn( curCol, 15, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1022 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // SYSCONSTRAINT - - // ConstraintName - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1023 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2019, 2020, 2021); - // Schema - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1024 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2022, 2023, 2024); - // TableName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1025 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2025, 2026, 2027); - - // ConstraintType - rc = colOp.createColumn( curCol, 3, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1026 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ConstraintPrim - rc = colOp.createColumn( curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1027 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2028, 2029, 2030); - - // ConstraintText - rc = colOp.createColumn( curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1028 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2031, 2032, 2033); - - // ConstraintStatus - rc = colOp.createColumn( curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1029 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2034, 2035, 2036); - - // IndexName - rc = colOp.createColumn( curCol, 7, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1030 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2037, 2038, 2039); - - //SYSCONSTRAINTCOL - // Schema - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1031 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2040, 2041, 2042); - - // TableName - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1032 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2043, 2044, 2045); - - // ColumnName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1033 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2046, 2047, 2048); - - // ConstraintName - rc = colOp.createColumn( curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1034 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2049, 2050, 2051); - - // SYSINDEX - // Schema - rc = colOp.createColumn( curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1035 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2052, 2053, 2054); - - //TableName - rc = colOp.createColumn( curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1036 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2055, 2056, 2057); - - // IndexName - rc = colOp.createColumn( curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1037 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2058, 2059, 2060); - - // ListOID - rc = colOp.createColumn( curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1038 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // TreeOID - rc = colOp.createColumn( curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1039 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // IndexType - rc = colOp.createColumn( curCol, 9, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1040 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // CreateDate - rc = colOp.createColumn( curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1041 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastUpdateDate - rc = colOp.createColumn( curCol, 11, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1042 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // RecordCount - rc = colOp.createColumn( curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1043 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // TreeLevel - rc = colOp.createColumn( curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1044 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LeafCount - rc = colOp.createColumn( curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1045 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // DistinctKeys - rc = colOp.createColumn( curCol, 15, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1046 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LeafBlocks - rc = colOp.createColumn( curCol, 16, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1047 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // AvgLeafCount - rc = colOp.createColumn( curCol, 17, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1048 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // AvgDataBlock - rc = colOp.createColumn( curCol, 18, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1049 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // SampleSize - rc = colOp.createColumn( curCol, 19, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1050 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // ClusterFactor - rc = colOp.createColumn( curCol, 20, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1051 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // LastAnalysisDate - rc = colOp.createColumn( curCol, 21, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1052 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // SYSINDEXCOL - // Schema - rc = colOp.createColumn( curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1053 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2061, 2062, 2063); - - // TableName - rc = colOp.createColumn( curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1054 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2064, 2065, 2066); - - // ColumnName - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1055 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2067, 2068, 2069); - - // IndexName - rc = colOp.createColumn( curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1056 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - cout << "Creating Dictionary..." << endl; - //Dictionary files - rc = fWriteEngine.createStore (txnID, 2070, 2071, 2072); - - // ColumnPos - rc = colOp.createColumn( curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1057 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - static void remove() - { - ColumnOp colOp; - - for ( int i = 1001; i <= 1057; i++ ) - colOp.deleteFile( i ); - } - + public: + static void build() + { + ColumnOp colOp; + Column curCol; + WriteEngine::WriteEngineWrapper fWriteEngine; + WriteEngine::TxnID txnID = 0; + int rc; + + remove(); + + cout << "Creating System Catalog..." << endl; + + colOp.initColumn(curCol); + // SYSTABLE + + // TableName + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1001); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2001, 2002, 2003); + // Schema + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1002); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2004, 2005, 2006); + // CreateDate + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1003); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastUpdateDate + rc = colOp.createColumn(curCol, 3, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1004); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // INIT + rc = colOp.createColumn(curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1005); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // NEXT + rc = colOp.createColumn(curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1006); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // SYSCOLUMN + // Shema + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1007); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2007, 2008, 2009); + + // TableName + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1008); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2010, 2011, 2012); + + // ColumnName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1009); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2013, 2014, 2015); + // ObjectID + rc = colOp.createColumn(curCol, 3, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1010); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DictOID + rc = colOp.createColumn(curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1011); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ListOID + rc = colOp.createColumn(curCol, 5, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1012); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // TreeOID + rc = colOp.createColumn(curCol, 6, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1013); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DataType + rc = colOp.createColumn(curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1014); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ColumnLength + rc = colOp.createColumn(curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1015); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ColumnPos + rc = colOp.createColumn(curCol, 9, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1016); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastUpdate + rc = colOp.createColumn(curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1017); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DefaultValue + rc = colOp.createColumn(curCol, 11, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1018); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2016, 2017, 2018); + // Nullable + rc = colOp.createColumn(curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1019); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Scale + rc = colOp.createColumn(curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1020); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Precision + rc = colOp.createColumn(curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1021); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // AutoInc + rc = colOp.createColumn(curCol, 15, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1022); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // SYSCONSTRAINT + + // ConstraintName + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1023); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2019, 2020, 2021); + // Schema + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1024); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2022, 2023, 2024); + // TableName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1025); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2025, 2026, 2027); + + // ConstraintType + rc = colOp.createColumn(curCol, 3, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1026); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ConstraintPrim + rc = colOp.createColumn(curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1027); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2028, 2029, 2030); + + // ConstraintText + rc = colOp.createColumn(curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1028); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2031, 2032, 2033); + + // ConstraintStatus + rc = colOp.createColumn(curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1029); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2034, 2035, 2036); + + // IndexName + rc = colOp.createColumn(curCol, 7, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1030); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2037, 2038, 2039); + + // SYSCONSTRAINTCOL + // Schema + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1031); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2040, 2041, 2042); + + // TableName + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1032); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2043, 2044, 2045); + + // ColumnName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1033); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2046, 2047, 2048); + + // ConstraintName + rc = colOp.createColumn(curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1034); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2049, 2050, 2051); + + // SYSINDEX + // Schema + rc = colOp.createColumn(curCol, 4, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1035); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2052, 2053, 2054); + + // TableName + rc = colOp.createColumn(curCol, 5, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1036); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2055, 2056, 2057); + + // IndexName + rc = colOp.createColumn(curCol, 6, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1037); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2058, 2059, 2060); + + // ListOID + rc = colOp.createColumn(curCol, 7, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1038); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // TreeOID + rc = colOp.createColumn(curCol, 8, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1039); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // IndexType + rc = colOp.createColumn(curCol, 9, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR, 1040); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // CreateDate + rc = colOp.createColumn(curCol, 10, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1041); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastUpdateDate + rc = colOp.createColumn(curCol, 11, 8, WriteEngine::DATE, WriteEngine::WR_CHAR, 1042); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // RecordCount + rc = colOp.createColumn(curCol, 12, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1043); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // TreeLevel + rc = colOp.createColumn(curCol, 13, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1044); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LeafCount + rc = colOp.createColumn(curCol, 14, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1045); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // DistinctKeys + rc = colOp.createColumn(curCol, 15, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1046); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LeafBlocks + rc = colOp.createColumn(curCol, 16, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1047); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // AvgLeafCount + rc = colOp.createColumn(curCol, 17, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1048); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // AvgDataBlock + rc = colOp.createColumn(curCol, 18, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1049); + CPPUNIT_ASSERT(rc == NO_ERROR); + // SampleSize + rc = colOp.createColumn(curCol, 19, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1050); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // ClusterFactor + rc = colOp.createColumn(curCol, 20, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1051); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // LastAnalysisDate + rc = colOp.createColumn(curCol, 21, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1052); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // SYSINDEXCOL + // Schema + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1053); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2061, 2062, 2063); + + // TableName + rc = colOp.createColumn(curCol, 1, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1054); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2064, 2065, 2066); + + // ColumnName + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1055); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2067, 2068, 2069); + + // IndexName + rc = colOp.createColumn(curCol, 3, 8, WriteEngine::VARCHAR, WriteEngine::WR_CHAR, 1056); + CPPUNIT_ASSERT(rc == NO_ERROR); + + cout << "Creating Dictionary..." << endl; + // Dictionary files + rc = fWriteEngine.createStore(txnID, 2070, 2071, 2072); + + // ColumnPos + rc = colOp.createColumn(curCol, 4, 4, WriteEngine::INT, WriteEngine::WR_CHAR, 1057); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + static void remove() + { + ColumnOp colOp; + + for (int i = 1001; i <= 1057; i++) + colOp.deleteFile(i); + } }; class TpchDBBuilder { -public: - static void build() + public: + static void build() + { + cout << "Creating tpch database..." << endl; + + std::string createStmt = + "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; + buildTable(createStmt); + + createStmt = + "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT " + "NULL,n_comment varchar(152))"; + buildTable(createStmt); + + createStmt = + "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), " + " c_nationkey integer ,c_phone char(15) ,c_acctbal integer , c_mktsegment char(8) , c_comment " + "varchar(117))"; + buildTable(createStmt); + + createStmt = + "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), " + "o_totalprice integer , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), " + "o_shippriority integer,o_comment varchar(79))"; + buildTable(createStmt); + + createStmt = + "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey " + "integer NOT NULL, l_linenumber integer NOT NULL, l_quantity integer NOT NULL,l_extendedprice " + "integer NOT NULL, l_discount integer NOT NULL, l_tax integer NOT NULL, l_returnflag char(1) , " + "l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct " + "char(25), l_shipmode char(10),l_comment varchar(44))"; + buildTable(createStmt); + + createStmt = + "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , " + "ps_supplycost integer , ps_comment varchar(199))"; + buildTable(createStmt); + + createStmt = + "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), " + "s_nationkey integer , s_phone char(15) , s_acctbal integer, s_comment varchar(101))"; + buildTable(createStmt); + + createStmt = + "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand " + "char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice integer , " + "p_comment varchar(23), PRIMARY KEY(p_partkey))"; + buildTable(createStmt); + } + + static void buildTable(std::string createText) + { + ddlpackage::SqlParser parser; + parser.Parse(createText.c_str()); + + if (parser.Good()) { + const ddlpackage::ParseTree& ptree = parser.GetParseTree(); - cout << "Creating tpch database..." << endl; + try + { + ddlpackageprocessor::CreateTableProcessor processor; + processor.setDebugLevel(ddlpackageprocessor::CreateTableProcessor::VERBOSE); - std::string createStmt = "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; - buildTable(createStmt); - - createStmt = "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT NULL,n_comment varchar(152))"; - buildTable(createStmt); - - createStmt = "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), c_nationkey integer ,c_phone char(15) ,c_acctbal integer , c_mktsegment char(8) , c_comment varchar(117))"; - buildTable(createStmt); - - createStmt = "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), o_totalprice integer , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), o_shippriority integer,o_comment varchar(79))"; - buildTable(createStmt); - - createStmt = "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity integer NOT NULL,l_extendedprice integer NOT NULL, l_discount integer NOT NULL, l_tax integer NOT NULL, l_returnflag char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; - buildTable(createStmt); - - createStmt = "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , ps_supplycost integer , ps_comment varchar(199))"; - buildTable(createStmt); - - createStmt = "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), s_nationkey integer , s_phone char(15) , s_acctbal integer, s_comment varchar(101))"; - buildTable(createStmt); - - createStmt = "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice integer , p_comment varchar(23), PRIMARY KEY(p_partkey))"; - buildTable(createStmt); + ddlpackage::SqlStatement& stmt = *ptree.fList[0]; + ddlpackageprocessor::CreateTableProcessor::DDLResult result; + result = processor.processPackage(dynamic_cast(stmt)); + std::cout << "return: " << result.result << std::endl; + } + catch (...) + { + throw; + } } - - static void buildTable(std::string createText) - { - - ddlpackage::SqlParser parser; - parser.Parse(createText.c_str()); - - if (parser.Good()) - { - const ddlpackage::ParseTree& ptree = parser.GetParseTree(); - - try - { - ddlpackageprocessor::CreateTableProcessor processor; - processor.setDebugLevel(ddlpackageprocessor::CreateTableProcessor::VERBOSE); - - ddlpackage::SqlStatement& stmt = *ptree.fList[0]; - ddlpackageprocessor::CreateTableProcessor::DDLResult result; - - result = processor.processPackage(dynamic_cast(stmt)); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - - throw; - } - } - } + } }; - /* * destroySemaphores() and destroyShmseg() will print error messages * if there are no objects to destroy. That's OK. */ void destroySemaphores() { - key_t semkey; - int sems, err; + key_t semkey; + int sems, err; - semkey = 0x2149bdd2; - sems = semget(semkey, 2, 0666); + semkey = 0x2149bdd2; + sems = semget(semkey, 2, 0666); - if (sems != -1) - { - err = semctl(sems, 0, IPC_RMID); + if (sems != -1) + { + err = semctl(sems, 0, IPC_RMID); - if (err == -1) - perror("tdriver: semctl"); - } + if (err == -1) + perror("tdriver: semctl"); + } } void destroyShmseg() { - key_t shmkey; - int shms, err; + key_t shmkey; + int shms, err; - shmkey = 0x2149bdd2; - shms = shmget(shmkey, 0, 0666); + shmkey = 0x2149bdd2; + shms = shmget(shmkey, 0, 0666); - if (shms != -1) + if (shms != -1) + { + err = shmctl(shms, IPC_RMID, NULL); + + if (err == -1 && errno != EINVAL) { - err = shmctl(shms, IPC_RMID, NULL); - - if (err == -1 && errno != EINVAL) - { - perror("tdriver: shmctl"); - return; - } + perror("tdriver: shmctl"); + return; } + } } void setUp() { - destroySemaphores(); - destroyShmseg(); - unlink("/tmp/oidbitmap"); - execplan::ObjectIDManager fObjectIDManager; - int dummyOid = fObjectIDManager.allocOIDs(5); - SystemCatalogBuilder::build(); - - TpchDBBuilder::build(); + destroySemaphores(); + destroyShmseg(); + unlink("/tmp/oidbitmap"); + execplan::ObjectIDManager fObjectIDManager; + int dummyOid = fObjectIDManager.allocOIDs(5); + SystemCatalogBuilder::build(); + TpchDBBuilder::build(); } void tearDown() { - destroySemaphores(); - destroyShmseg(); - unlink("/tmp/oidbitmap"); + destroySemaphores(); + destroyShmseg(); + unlink("/tmp/oidbitmap"); } - - class DMLPackageProcessorTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DMLPackageProcessorTest); + /* + CPPUNIT_TEST( test_insert_package1); + CPPUNIT_TEST( test_insert_package2); + CPPUNIT_TEST( test_insert_package3); + CPPUNIT_TEST( test_insert_NULL ); + CPPUNIT_TEST( test_update_package); + CPPUNIT_TEST( test_delete_package); + CPPUNIT_TEST( test_insert_package_fail); + CPPUNIT_TEST( test_update_package_fail); + CPPUNIT_TEST( test_delete_package_fail); + CPPUNIT_TEST( test_command_package ); + */ + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE( DMLPackageProcessorTest ); - /* - CPPUNIT_TEST( test_insert_package1); - CPPUNIT_TEST( test_insert_package2); - CPPUNIT_TEST( test_insert_package3); - CPPUNIT_TEST( test_insert_NULL ); - CPPUNIT_TEST( test_update_package); - CPPUNIT_TEST( test_delete_package); - CPPUNIT_TEST( test_insert_package_fail); - CPPUNIT_TEST( test_update_package_fail); - CPPUNIT_TEST( test_delete_package_fail); - CPPUNIT_TEST( test_command_package ); - */ - CPPUNIT_TEST_SUITE_END(); + private: + public: + void test_insert_package1() + { + cout << "Begin insert package test" << endl; -private: + ByteStream bytestream; + std::string dmlStatement = "insert into tpch.part(p_partkey,p_name,p_mfgr) values(3, 'widget', 'acmea');"; + cout << "\nDML statement:" << dmlStatement.c_str() << endl; -public: + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); - void test_insert_package1() + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_insert_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_insert_object(bytestream); + + cout << "\nEnd insert package test" << endl; + } + + void test_insert_package2() + { + cout << "Begin insert package test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "insert into tpch.part(p_partkey,p_name,p_mfgr) values(1, 'widget', 'acmea');"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_insert_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_insert_object(bytestream); + + cout << "\nEnd insert package test" << endl; + } + void test_insert_package3() + { + cout << "Begin insert package test" << endl; + + ByteStream bytestream; + std::string dmlStatement = + "insert into tpch.part values(6, 'name', 'mfgr', 'brand', 'type', 6, 'container', 12, 'comment');"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_insert_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_insert_object(bytestream); + + cout << "\nEnd insert package test" << endl; + } + void test_insert_NULL() + { + cout << "Begin insert package test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "insert into tpch.part(p_partkey,p_name,p_mfgr) values(2, 'widget2', NULL);"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_insert_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_insert_object(bytestream); + + cout << "\nEnd insert package test" << endl; + } + + void test_insert_package_fail() + { + cout << "Begin insert package failure test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "insert into tpch.region(r_regionkey, r_name) values(1,2);"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_insert_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_insert_object(bytestream); + + cout << "\nEnd insert package test" << endl; + } + + void write_insert_object(ByteStream& bs, CalpontDMLPackage* pDMLPackage) + { + cout << "Writing out package over bytestream" << endl; + pDMLPackage->write(bs); + } + + void processs_insert_object(ByteStream& bs) + { + cout << "Reading in package from bytestream" << endl; + + ByteStream::byte package_type; + bs >> package_type; + + CPPUNIT_ASSERT(DML_INSERT == package_type); + + InsertDMLPackage* pObject = new InsertDMLPackage(); + + pObject->read(bs); + + cout << "Success reading package from bytestream" << endl; + + DMLPackageProcessor* pkgProcPtr = + DMLPackageProcessorFactory::makePackageProcessor(package_type, *pObject); + + CPPUNIT_ASSERT(0 != pkgProcPtr); + + pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); + DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage(*pObject); + + if (DMLPackageProcessor::NO_ERROR != result.result) { + cout << "Insert failed!" << endl; + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - cout << "Begin insert package test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "insert into tpch.part(p_partkey,p_name,p_mfgr) values(3, 'widget', 'acmea');"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_insert_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_insert_object( bytestream ); - - cout << "\nEnd insert package test" << endl; - + ml.logDebugMessage(result.message); } - void test_insert_package2() + delete pkgProcPtr; + delete pObject; + } + + void test_delete_package() + { + cout << "Begin delete package test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "delete from tpch.part;"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_delete_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_delete_object(bytestream); + + cout << "\nEnd delete package test" << endl; + } + + void test_delete_package_fail() + { + cout << "Begin delete package failure test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "delete from tpch.region;"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_delete_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_delete_object(bytestream); + + cout << "\nEnd delete package test" << endl; + } + void write_delete_object(ByteStream& bs, CalpontDMLPackage* pDMLPackage) + { + cout << "Writing out package over bytestream" << endl; + pDMLPackage->write(bs); + } + + void processs_delete_object(ByteStream& bs) + { + cout << "Reading in package from bytestream" << endl; + + ByteStream::byte package_type; + bs >> package_type; + + CPPUNIT_ASSERT(DML_DELETE == package_type); + + DeleteDMLPackage* pObject = new DeleteDMLPackage(); + + pObject->read(bs); + + cout << "Success reading package from bytestream" << endl; + + DMLPackageProcessor* pkgProcPtr = + DMLPackageProcessorFactory::makePackageProcessor(package_type, *pObject); + + CPPUNIT_ASSERT(0 != pkgProcPtr); + + pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); + DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage(*pObject); + + if (DMLPackageProcessor::NO_ERROR != result.result) { + cout << "Delete failed!" << endl; + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - cout << "Begin insert package test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "insert into tpch.part(p_partkey,p_name,p_mfgr) values(1, 'widget', 'acmea');"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_insert_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_insert_object( bytestream ); - - cout << "\nEnd insert package test" << endl; - - } - void test_insert_package3() - { - - cout << "Begin insert package test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "insert into tpch.part values(6, 'name', 'mfgr', 'brand', 'type', 6, 'container', 12, 'comment');"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_insert_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_insert_object( bytestream ); - - cout << "\nEnd insert package test" << endl; - - } - void test_insert_NULL() - { - - cout << "Begin insert package test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "insert into tpch.part(p_partkey,p_name,p_mfgr) values(2, 'widget2', NULL);"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_insert_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_insert_object( bytestream ); - - cout << "\nEnd insert package test" << endl; - + ml.logDebugMessage(result.message); } - void test_insert_package_fail() + delete pkgProcPtr; + delete pObject; + } + + void test_update_package() + { + cout << "Begin update package test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "update tpch.part set p_mfgr = 'bigco';"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_update_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_update_object(bytestream); + + cout << "\nEnd update package test" << endl; + } + + void test_command_package() + { + std::vector commands; + commands.push_back("COMMIT;"); + commands.push_back("ROLLBACK;"); + commands.push_back("BOGUS;"); + + cout << "Begin command package test" << endl; + + ByteStream bytestream; + + std::vector::const_iterator iter = commands.begin(); + + while (iter != commands.end()) { + std::string command = *iter; + cout << command << endl; - cout << "Begin insert package failure test" << endl; + VendorDMLStatement dml_command(command, 1); + CalpontDMLPackage* dmlCommandPkgPtr = CalpontDMLFactory::makeCalpontDMLPackage(dml_command); - ByteStream bytestream; - std::string dmlStatement = "insert into tpch.region(r_regionkey, r_name) values(1,2);"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_insert_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_insert_object( bytestream ); - - cout << "\nEnd insert package test" << endl; - - } - - void write_insert_object( ByteStream& bs, CalpontDMLPackage* pDMLPackage ) - { + if (dmlCommandPkgPtr) + { + cout << "CalpontDMLFactory::makeCalpontDMLPackage: success" << endl; cout << "Writing out package over bytestream" << endl; - pDMLPackage->write( bs ); + dmlCommandPkgPtr->write(bytestream); + delete dmlCommandPkgPtr; + + process_command_object(bytestream); + } + + ++iter; } - void processs_insert_object( ByteStream& bs ) + cout << "\nEnd commit package test" << endl; + } + + void process_command_object(ByteStream& bs) + { + cout << "Reading in package from bytestream" << endl; + + ByteStream::byte package_type; + bs >> package_type; + + CPPUNIT_ASSERT(DML_COMMAND == package_type); + + CommandDMLPackage* pObject = new CommandDMLPackage(); + + pObject->read(bs); + + cout << "Success reading package from bytestream" << endl; + + DMLPackageProcessor* pkgProcPtr = + DMLPackageProcessorFactory::makePackageProcessor(package_type, *pObject); + + CPPUNIT_ASSERT(0 != pkgProcPtr); + pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); + + DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage(*pObject); + + if (DMLPackageProcessor::NO_ERROR != result.result) { + cout << "Command process failed!" << endl; + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - cout << "Reading in package from bytestream" << endl; - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_INSERT == package_type ); - - InsertDMLPackage* pObject = new InsertDMLPackage(); - - pObject->read( bs ); - - cout << "Success reading package from bytestream" << endl; - - DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor( package_type, *pObject ); - - CPPUNIT_ASSERT( 0 != pkgProcPtr ); - - pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); - DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage( *pObject ); - - if ( DMLPackageProcessor::NO_ERROR != result.result ) - { - cout << "Insert failed!" << endl; - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - - ml.logDebugMessage( result.message ); - } - - delete pkgProcPtr; - delete pObject; - + ml.logDebugMessage(result.message); } - void test_delete_package() + delete pkgProcPtr; + delete pObject; + } + + void test_update_package_fail() + { + cout << "Begin update package failure test" << endl; + + ByteStream bytestream; + std::string dmlStatement = "update tpch.region set r_name = NORTH;"; + + cout << "\nDML statement:" << dmlStatement.c_str() << endl; + + VendorDMLStatement dmlStmt(dmlStatement, 1); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + CPPUNIT_ASSERT(0 != pDMLPackage); + + cout << "CalpontDMLFactory::makeCalpontDMLPackage:" + << "success" << endl; + + write_update_object(bytestream, pDMLPackage); + delete pDMLPackage; + + // process the package + processs_update_object(bytestream); + + cout << "\nEnd update package test" << endl; + } + + void write_update_object(ByteStream& bs, CalpontDMLPackage* pDMLPackage) + { + cout << "Writing out package over bytestream" << endl; + pDMLPackage->write(bs); + } + + void processs_update_object(ByteStream& bs) + { + cout << "Reading in package from bytestream" << endl; + + ByteStream::byte package_type; + bs >> package_type; + + CPPUNIT_ASSERT(DML_UPDATE == package_type); + + UpdateDMLPackage* pObject = new UpdateDMLPackage(); + + pObject->read(bs); + + cout << "Success reading package from bytestream" << endl; + + DMLPackageProcessor* pkgProcPtr = + DMLPackageProcessorFactory::makePackageProcessor(package_type, *pObject); + + CPPUNIT_ASSERT(0 != pkgProcPtr); + + pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); + + DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage(*pObject); + + if (DMLPackageProcessor::NO_ERROR != result.result) { + cout << "Update failed!" << endl; + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - cout << "Begin delete package test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "delete from tpch.part;"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_delete_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_delete_object( bytestream ); - - cout << "\nEnd delete package test" << endl; - - } - - void test_delete_package_fail() - { - - cout << "Begin delete package failure test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "delete from tpch.region;"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_delete_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_delete_object( bytestream ); - - cout << "\nEnd delete package test" << endl; - - } - void write_delete_object( ByteStream& bs, CalpontDMLPackage* pDMLPackage ) - { - cout << "Writing out package over bytestream" << endl; - pDMLPackage->write( bs ); - } - - void processs_delete_object( ByteStream& bs ) - { - - cout << "Reading in package from bytestream" << endl; - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_DELETE == package_type ); - - DeleteDMLPackage* pObject = new DeleteDMLPackage(); - - pObject->read( bs ); - - cout << "Success reading package from bytestream" << endl; - - DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor( package_type, *pObject ); - - CPPUNIT_ASSERT( 0 != pkgProcPtr ); - - pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); - DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage( *pObject ); - - if ( DMLPackageProcessor::NO_ERROR != result.result ) - { - cout << "Delete failed!" << endl; - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - - ml.logDebugMessage( result.message ); - } - - delete pkgProcPtr; - delete pObject; - - } - - void test_update_package() - { - - cout << "Begin update package test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "update tpch.part set p_mfgr = 'bigco';"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_update_object(bytestream, pDMLPackage); - delete pDMLPackage; - - // process the package - processs_update_object( bytestream ); - - cout << "\nEnd update package test" << endl; - - } - - void test_command_package() - { - std::vector commands; - commands.push_back("COMMIT;"); - commands.push_back("ROLLBACK;"); - commands.push_back("BOGUS;"); - - - cout << "Begin command package test" << endl; - - ByteStream bytestream; - - std::vector::const_iterator iter = commands.begin(); - - while (iter != commands.end()) - { - std::string command = *iter; - cout << command << endl; - - VendorDMLStatement dml_command(command, 1); - CalpontDMLPackage* dmlCommandPkgPtr = CalpontDMLFactory::makeCalpontDMLPackage(dml_command); - - if (dmlCommandPkgPtr) - { - cout << "CalpontDMLFactory::makeCalpontDMLPackage: success" << endl; - cout << "Writing out package over bytestream" << endl; - dmlCommandPkgPtr->write(bytestream); - delete dmlCommandPkgPtr; - - process_command_object(bytestream); - } - - ++iter; - } - - cout << "\nEnd commit package test" << endl; - - } - - void process_command_object( ByteStream& bs ) - { - - cout << "Reading in package from bytestream" << endl; - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_COMMAND == package_type ); - - CommandDMLPackage* pObject = new CommandDMLPackage(); - - pObject->read( bs ); - - cout << "Success reading package from bytestream" << endl; - - DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor( package_type, *pObject ); - - CPPUNIT_ASSERT( 0 != pkgProcPtr ); - pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); - - DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage( *pObject ); - - if ( DMLPackageProcessor::NO_ERROR != result.result ) - { - cout << "Command process failed!" << endl; - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - - ml.logDebugMessage( result.message ); - } - - delete pkgProcPtr; - delete pObject; - } - - void test_update_package_fail() - { - - cout << "Begin update package failure test" << endl; - - ByteStream bytestream; - std::string dmlStatement = "update tpch.region set r_name = NORTH;"; - - cout << "\nDML statement:" << dmlStatement.c_str() << endl; - - VendorDMLStatement dmlStmt(dmlStatement, 1); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - CPPUNIT_ASSERT( 0 != pDMLPackage ); - - cout << "CalpontDMLFactory::makeCalpontDMLPackage:" << "success" << endl; - - write_update_object(bytestream, pDMLPackage); - delete pDMLPackage; - - //process the package - processs_update_object( bytestream ); - - cout << "\nEnd update package test" << endl; - - } - - void write_update_object( ByteStream& bs, CalpontDMLPackage* pDMLPackage ) - { - cout << "Writing out package over bytestream" << endl; - pDMLPackage->write( bs ); - } - - void processs_update_object( ByteStream& bs ) - { - - cout << "Reading in package from bytestream" << endl; - - ByteStream::byte package_type; - bs >> package_type; - - CPPUNIT_ASSERT( DML_UPDATE == package_type ); - - UpdateDMLPackage* pObject = new UpdateDMLPackage(); - - pObject->read( bs ); - - cout << "Success reading package from bytestream" << endl; - - DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor( package_type, *pObject ); - - CPPUNIT_ASSERT( 0 != pkgProcPtr ); - - pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); - - DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage( *pObject ); - - if ( DMLPackageProcessor::NO_ERROR != result.result ) - { - cout << "Update failed!" << endl; - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - - ml.logDebugMessage( result.message ); - } - - delete pkgProcPtr; - delete pObject; - + ml.logDebugMessage(result.message); } + delete pkgProcPtr; + delete pObject; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( DMLPackageProcessorTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DMLPackageProcessorTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - // Uncomment before running tests - //setUp(); + // Uncomment before running tests + // setUp(); - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); - tearDown(); + tearDown(); - return (wasSuccessful ? 0 : 1); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/dmlpackageproc/updatepackageprocessor.cpp b/dbcon/dmlpackageproc/updatepackageprocessor.cpp index eeac1f0f9..344796ce6 100644 --- a/dbcon/dmlpackageproc/updatepackageprocessor.cpp +++ b/dbcon/dmlpackageproc/updatepackageprocessor.cpp @@ -60,274 +60,309 @@ using namespace oam; //#define PROFILE 1 namespace dmlpackageprocessor { - - -//StopWatch timer; -DMLPackageProcessor::DMLResult -UpdatePackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) +// StopWatch timer; +DMLPackageProcessor::DMLResult UpdatePackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) { - SUMMARY_INFO("UpdatePackageProcessor::processPackage"); + SUMMARY_INFO("UpdatePackageProcessor::processPackage"); - std::string err; - DMLResult result; - result.result = NO_ERROR; - result.rowCount = 0; - BRM::TxnID txnid; - // set-up the transaction - txnid.id = cpackage.get_TxnID(); - txnid.valid = true; - fSessionID = cpackage.get_SessionID(); - VERBOSE_INFO("Processing Update DML Package..."); - TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); - uint64_t uniqueId = 0; + std::string err; + DMLResult result; + result.result = NO_ERROR; + result.rowCount = 0; + BRM::TxnID txnid; + // set-up the transaction + txnid.id = cpackage.get_TxnID(); + txnid.valid = true; + fSessionID = cpackage.get_SessionID(); + VERBOSE_INFO("Processing Update DML Package..."); + TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID); + uint64_t uniqueId = 0; - //Bug 5070. Added exception handling - try + // Bug 5070. Added exception handling + try + { + uniqueId = fDbrm->getUnique64(); + } + catch (std::exception& ex) + { + logging::Message::Args args; + logging::Message message(9); + args.add(ex.what()); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + catch (...) + { + logging::Message::Args args; + logging::Message message(9); + args.add("Unknown error occured while getting unique number."); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + fSessionManager.rolledback(txnid); + return result; + } + + uint64_t tableLockId = 0; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + CalpontSystemCatalog::TableName tableName; + // get the table object from the package + DMLTable* tablePtr = cpackage.get_Table(); + tableName.schema = tablePtr->get_SchemaName(); + tableName.table = tablePtr->get_TableName(); + fWEClient->addQueue(uniqueId); + execplan::CalpontSystemCatalog::ROPair roPair; + + //#ifdef PROFILE + // StopWatch timer; + //#endif + try + { + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("Start SQL statement: "); + ostringstream oss; + oss << cpackage.get_SQLStatement() << "|" << tablePtr->get_SchemaName() << "|"; + args1.add(oss.str()); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + + VERBOSE_INFO("The table name is:"); + VERBOSE_INFO(tablePtr->get_TableName()); + + if (0 != tablePtr) { - uniqueId = fDbrm->getUnique64(); - } - catch (std::exception& ex) - { - logging::Message::Args args; - logging::Message message(9); - args.add(ex.what()); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); + // get the row(s) from the table + RowList rows = tablePtr->get_RowList(); + + if (rows.size() == 0) + { + SUMMARY_INFO("No row to update!"); + fWEClient->removeQueue(uniqueId); return result; - } - catch ( ... ) - { - logging::Message::Args args; - logging::Message message(9); - args.add("Unknown error occured while getting unique number."); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - fSessionManager.rolledback(txnid); - return result; - } + } - uint64_t tableLockId = 0; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); - CalpontSystemCatalog::TableName tableName; - // get the table object from the package - DMLTable* tablePtr = cpackage.get_Table(); - tableName.schema = tablePtr->get_SchemaName(); - tableName.table = tablePtr->get_TableName(); - fWEClient->addQueue(uniqueId); - execplan::CalpontSystemCatalog::ROPair roPair; + roPair = systemCatalogPtr->tableRID(tableName); + tableLockId = tablelockData->getTablelockId( + roPair.objnum); // check whether this table is locked already for this session -//#ifdef PROFILE -// StopWatch timer; -//#endif - try - { - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Start SQL statement: "); - ostringstream oss; - oss << cpackage.get_SQLStatement() << "|" << tablePtr->get_SchemaName() << "|"; - args1.add(oss.str()); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + if (tableLockId == 0) + { + // cout << "tablelock is not found in cache, getting from dbrm" << endl; + uint32_t processID = ::getpid(); + int32_t txnId = txnid.id; + int32_t sessionId = fSessionID; + std::string processName("DMLProc"); + int i = 0; + OamCache* oamcache = OamCache::makeOamCache(); + std::vector pmList = oamcache->getModuleIds(); + std::vector pms; - VERBOSE_INFO("The table name is:"); - VERBOSE_INFO(tablePtr->get_TableName()); - - if (0 != tablePtr) + for (unsigned i = 0; i < pmList.size(); i++) { - // get the row(s) from the table - RowList rows = tablePtr->get_RowList(); + pms.push_back((uint32_t)pmList[i]); + } - if (rows.size() == 0) - { - SUMMARY_INFO("No row to update!"); - fWEClient->removeQueue(uniqueId); - return result; - } + try + { + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, + BRM::LOADING); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } - roPair = systemCatalogPtr->tableRID(tableName); - tableLockId = tablelockData->getTablelockId(roPair.objnum); //check whether this table is locked already for this session + if (tableLockId == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + waitPeriod = Config::getWaitPeriod(); + numTries = waitPeriod * 10; + struct timespec rm_ts; - if (tableLockId == 0) - { - //cout << "tablelock is not found in cache, getting from dbrm" << endl; - uint32_t processID = ::getpid(); - int32_t txnId = txnid.id; - int32_t sessionId = fSessionID; - std::string processName("DMLProc"); - int i = 0; - OamCache* oamcache = OamCache::makeOamCache(); - std::vector pmList = oamcache->getModuleIds(); - std::vector pms; + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - for (unsigned i = 0; i < pmList.size(); i++) - { - pms.push_back((uint32_t)pmList[i]); - } - - try - { - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if ( tableLockId == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - waitPeriod = Config::getWaitPeriod(); - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { + for (; i < numTries; i++) + { #ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); + Sleep(rm_ts.tv_sec * 1000); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif - try - { - processID = ::getpid(); - txnId = txnid.id; - sessionId = fSessionID; - processName = "DMLProc"; - tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (tableLockId > 0) - break; - } - - if (i >= numTries) //error out - { - result.result = UPDATE_ERROR; - logging::Message::Args args; - string strOp("update"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add(sessionId); - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - } - } - } - - //cout << " tablelock is obtained with id " << tableLockId << endl; - tablelockData->setTablelock(roPair.objnum, tableLockId); - //@Bug 4491 start AI sequence for autoincrement column - const CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName); - CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); - CalpontSystemCatalog::ColType colType; - - while (rid_iterator != ridList.end()) + try { - // If user hit ctrl+c in the mysql console, this will be true. - if (fRollbackPending) - { - result.result = JOB_CANCELED; - break; - } - - CalpontSystemCatalog::ROPair roPair = *rid_iterator; - colType = systemCatalogPtr->colType(roPair.objnum); - - if (colType.autoincrement) - { - try - { - uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - fDbrm->startAISequence(roPair.objnum, nextVal, colType.colWidth, colType.colDataType); - break; //Only one autoincrement column per table - } - catch (std::exception& ex) - { - result.result = UPDATE_ERROR; - throw std::runtime_error(ex.what()); - } - } - - ++rid_iterator; + processID = ::getpid(); + txnId = txnid.id; + sessionId = fSessionID; + processName = "DMLProc"; + tableLockId = fDbrm->getTableLock(pms, roPair.objnum, &processName, &processID, &sessionId, + &txnId, BRM::LOADING); } - - uint64_t rowsProcessed = 0; - - if (!fRollbackPending) + catch (std::exception&) { - rowsProcessed = fixUpRows(cpackage, result, uniqueId, roPair.objnum); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); } - //@Bug 4994 Cancelled job is not error - if (result.result == JOB_CANCELED) - throw std::runtime_error("Query execution was interrupted"); + if (tableLockId > 0) + break; + } - if ((result.result != 0) && (result.result != DMLPackageProcessor::IDBRANGE_WARNING)) - throw std::runtime_error(result.message.msg()); - - result.rowCount = rowsProcessed; - - // Log the update statement. - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement(), cpackage.get_SchemaName()); - } - } - catch (std::exception& ex) - { - cerr << "UpdatePackageProcessor::processPackage:" << ex.what() << endl; - - if (result.result == 0) - { + if (i >= numTries) // error out + { result.result = UPDATE_ERROR; + logging::Message::Args args; + string strOp("update"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add(sessionId); + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + } + } + } + + // cout << " tablelock is obtained with id " << tableLockId << endl; + tablelockData->setTablelock(roPair.objnum, tableLockId); + //@Bug 4491 start AI sequence for autoincrement column + const CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName); + CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); + CalpontSystemCatalog::ColType colType; + + while (rid_iterator != ridList.end()) + { + // If user hit ctrl+c in the mysql console, this will be true. + if (fRollbackPending) + { + result.result = JOB_CANCELED; + break; } - result.message = Message(ex.what()); - result.rowCount = 0; - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with error"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + CalpontSystemCatalog::ROPair roPair = *rid_iterator; + colType = systemCatalogPtr->colType(roPair.objnum); + + if (colType.autoincrement) + { + try + { + uint64_t nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + fDbrm->startAISequence(roPair.objnum, nextVal, colType.colWidth, colType.colDataType); + break; // Only one autoincrement column per table + } + catch (std::exception& ex) + { + result.result = UPDATE_ERROR; + throw std::runtime_error(ex.what()); + } + } + + ++rid_iterator; + } + + uint64_t rowsProcessed = 0; + + if (!fRollbackPending) + { + rowsProcessed = fixUpRows(cpackage, result, uniqueId, roPair.objnum); + } + + //@Bug 4994 Cancelled job is not error + if (result.result == JOB_CANCELED) + throw std::runtime_error("Query execution was interrupted"); + + if ((result.result != 0) && (result.result != DMLPackageProcessor::IDBRANGE_WARNING)) + throw std::runtime_error(result.message.msg()); + + result.rowCount = rowsProcessed; + + // Log the update statement. + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(cpackage.get_SessionID(), txnid.id, cpackage.get_SQLStatement(), + cpackage.get_SchemaName()); } - catch (...) + } + catch (std::exception& ex) + { + cerr << "UpdatePackageProcessor::processPackage:" << ex.what() << endl; + + if (result.result == 0) { - cerr << "UpdatePackageProcessor::processPackage: caught unknown exception!" << endl; + result.result = UPDATE_ERROR; + } + + result.message = Message(ex.what()); + result.rowCount = 0; + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with error"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + } + catch (...) + { + cerr << "UpdatePackageProcessor::processPackage: caught unknown exception!" << endl; + logging::Message::Args args; + logging::Message message(7); + args.add("Update Failed: "); + args.add("encountered unkown exception"); + args.add(""); + args.add(""); + message.format(args); + + result.result = UPDATE_ERROR; + result.message = message; + result.rowCount = 0; + LoggingID logid(DMLLoggingId, fSessionID, txnid.id); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with error"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + } + + // timer.finish(); + //@Bug 1886,2870 Flush VM cache only once per statement. send to all PMs. + // WriteEngineWrapper writeEngine; + std::map oids; + int rc = 0; + + if (result.result == NO_ERROR || result.result == IDBRANGE_WARNING) + { + if ((rc = flushDataFiles(NO_ERROR, oids, uniqueId, txnid, roPair.objnum)) != NO_ERROR) + { + cerr << "UpdatePackageProcessor::processPackage: write data to disk failed" << endl; + + if (!fRollbackPending) + { logging::Message::Args args; logging::Message message(7); args.add("Update Failed: "); - args.add("encountered unkown exception"); + args.add("error when writing data to disk"); args.add(""); args.add(""); message.format(args); @@ -335,697 +370,662 @@ UpdatePackageProcessor::processPackage(dmlpackage::CalpontDMLPackage& cpackage) result.result = UPDATE_ERROR; result.message = message; result.rowCount = 0; - LoggingID logid( DMLLoggingId, fSessionID, txnid.id); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with error"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - } + } - // timer.finish(); - //@Bug 1886,2870 Flush VM cache only once per statement. send to all PMs. - //WriteEngineWrapper writeEngine; - std::map oids; - int rc = 0; - - if (result.result == NO_ERROR || result.result == IDBRANGE_WARNING) - { - if ((rc = flushDataFiles(NO_ERROR, oids, uniqueId, txnid, roPair.objnum)) != NO_ERROR) - { - cerr << "UpdatePackageProcessor::processPackage: write data to disk failed" << endl; - - if (!fRollbackPending) - { - logging::Message::Args args; - logging::Message message(7); - args.add("Update Failed: "); - args.add("error when writing data to disk"); - args.add(""); - args.add(""); - message.format(args); - - result.result = UPDATE_ERROR; - result.message = message; - result.rowCount = 0; - } - - rc = endTransaction(uniqueId, txnid, false); - } - else - { - if (fRollbackPending) - rc = endTransaction(uniqueId, txnid, false); - else - rc = endTransaction(uniqueId, txnid, true); - - if (( rc != NO_ERROR) && (!fRollbackPending)) - { - logging::Message::Args args; - logging::Message message(7); - args.add("Update Failed: "); - args.add("error when cleaning up data files"); - args.add(""); - args.add(""); - message.format(args); - - result.result = UPDATE_ERROR; - result.message = message; - result.rowCount = 0; - - } - } + rc = endTransaction(uniqueId, txnid, false); } else { - //@Bug 4563. Always flush. error is already set - rc = flushDataFiles(result.result, oids, uniqueId, txnid, roPair.objnum); + if (fRollbackPending) rc = endTransaction(uniqueId, txnid, false); + else + rc = endTransaction(uniqueId, txnid, true); + + if ((rc != NO_ERROR) && (!fRollbackPending)) + { + logging::Message::Args args; + logging::Message message(7); + args.add("Update Failed: "); + args.add("error when cleaning up data files"); + args.add(""); + args.add(""); + message.format(args); + + result.result = UPDATE_ERROR; + result.message = message; + result.rowCount = 0; + } } + } + else + { + //@Bug 4563. Always flush. error is already set + rc = flushDataFiles(result.result, oids, uniqueId, txnid, roPair.objnum); + rc = endTransaction(uniqueId, txnid, false); + } - //timer.finish(); + // timer.finish(); - /* if (result.result != IDBRANGE_WARNING) - flushDataFiles(result.result, oids, uniqueId, txnid); - else - flushDataFiles(0, oids, uniqueId, txnid); - */ - if (fRollbackPending) - { - result.result = JOB_CANCELED; - logging::Message::Args args1; - args1.add("Query execution was interrupted"); - result.message.format(args1); - } + /* if (result.result != IDBRANGE_WARNING) + flushDataFiles(result.result, oids, uniqueId, txnid); + else + flushDataFiles(0, oids, uniqueId, txnid); + */ + if (fRollbackPending) + { + result.result = JOB_CANCELED; + logging::Message::Args args1; + args1.add("Query execution was interrupted"); + result.message.format(args1); + } - fWEClient->removeQueue(uniqueId); - VERBOSE_INFO("Finished Processing Update DML Package"); - return result; + fWEClient->removeQueue(uniqueId); + VERBOSE_INFO("Finished Processing Update DML Package"); + return result; } uint64_t UpdatePackageProcessor::fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, - const uint64_t uniqueId, const uint32_t tableOid) + const uint64_t uniqueId, const uint32_t tableOid) { - ByteStream msg, msgBk, emsgBs; - RGData rgData; - uint32_t qb = 4; - msg << qb; - boost::scoped_ptr rowGroup; - uint64_t rowsProcessed = 0; - uint32_t dbroot = 1; - bool metaData = false; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - std::vector fPMs = oamCache->getModuleIds(); - std::map pmState; - string emsg; - string emsgStr; - bool err = false; + ByteStream msg, msgBk, emsgBs; + RGData rgData; + uint32_t qb = 4; + msg << qb; + boost::scoped_ptr rowGroup; + uint64_t rowsProcessed = 0; + uint32_t dbroot = 1; + bool metaData = false; + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + std::vector fPMs = oamCache->getModuleIds(); + std::map pmState; + string emsg; + string emsgStr; + bool err = false; - //boost::scoped_ptr fExeMgr; - //fExeMgr.reset( new messageqcpp::MessageQueueClient("ExeMgr1")); - try + // boost::scoped_ptr fExeMgr; + // fExeMgr.reset( new messageqcpp::MessageQueueClient("ExeMgr1")); + try + { + for (unsigned i = 0; i < fPMs.size(); i++) { - - for (unsigned i = 0; i < fPMs.size(); i++) - { - pmState[fPMs[i]] = true; - } - - //timer.start("ExeMgr"); - fExeMgr->write(msg); - fExeMgr->write(*(cpackage.get_ExecutionPlan())); - //cout << "sending to ExeMgr plan with length " << (cpackage.get_ExecutionPlan())->length() << endl; - msg.restart(); - emsgBs.restart(); - msg = fExeMgr->read(); //error handling - - if (msg.length() == 4) - { - msg >> qb; - - if (qb != 0) - err = true; - } - else - { - qb = 999; - err = true; - } - - if (err) - { - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: ExeMgr Error"); - args.add((int)qb); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - //timer.finish(); - return rowsProcessed; - } - - emsgBs = fExeMgr->read(); - - if (emsgBs.length() == 0) - { - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add("Lost connection to ExeMgr"); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - //timer.finish(); - return rowsProcessed; - } - - emsgBs >> emsgStr; - - while (true) - { - if (fRollbackPending) - { - break; - } - - msg.restart(); - msgBk.restart(); - msg = fExeMgr->read(); - msgBk = msg; - - if ( msg.length() == 0 ) - { - cerr << "UpdatePackageProcessor::processPackage::fixupRows" << endl; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add("Lost connection to ExeMgr"); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - //timer.finish(); - //return rowsProcessed; - break; - } - else - { - if (rowGroup.get() == NULL) - { - //This is mete data, need to send all PMs. - metaData = true; - //cout << "sending meta data" << endl; - //timer.start("Meta"); - err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot); - rowGroup.reset(new rowgroup::RowGroup()); - rowGroup->deserialize(msg); - qb = 100; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - metaData = false; - //timer.stop("Meta"); - continue; - } - - rgData.deserialize(msg, true); - rowGroup->setData(&rgData); - //rowGroup->setData(const_cast(msg.buf())); - err = (rowGroup->getStatus() != 0); - - if (err) - { - //msgBk.advance(rowGroup->getDataSize()); - string errorMsg; - msg >> errorMsg; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add(errorMsg); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - DMLResult tmpResult; - receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid); - /* qb = 100; - //@Bug 4358 get rid of broken pipe error. - msg.restart(); - msg << qb; - fExeMgr->write(msg); - */ //timer.finish(); - //return rowsProcessed; - //err = true; - break; - } - - if (rowGroup->getRGData() == NULL) - { - msg.restart(); - } - - if (rowGroup->getRowCount() == 0) //done fetching - { - //timer.finish(); - //need to receive all response - err = receiveAll( result, uniqueId, fPMs, pmState, tableOid); - //return rowsProcessed; - break; - } - - if (rowGroup->getBaseRid() == (uint64_t) (-1)) - { - continue; // @bug4247, not valid row ids, may from small side outer - } - - dbroot = rowGroup->getDBRoot(); - //cout << "dbroot in the rowgroup is " << dbroot << endl; - //timer.start("processRowgroup"); - err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot); - - //timer.stop("processRowgroup"); - if (err) - { - //timer.finish(); - LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID()); - logging::Message::Args args1; - logging::Message msg1(1); - args1.add("SQL statement erroring out, need to receive all messages from WES"); - msg1.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); - DMLResult tmpResult; - receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid); - logging::Message::Args args2; - logging::Message msg2(1); - args2.add("SQL statement erroring out, received all messages from WES"); - msg2.format( args2 ); - logger.logMessage(LOG_TYPE_DEBUG, msg2, logid); - //@Bug 4358 get rid of broken pipe error. - /* msg.restart(); - msg << qb; - fExeMgr->write(msg); - return rowsProcessed; - */ - //err = true; - break; - } - - rowsProcessed += rowGroup->getRowCount(); - } - } - - if (fRollbackPending) - { - err = true; - // Response to user - cerr << "UpdatePackageProcessor::processPackage::fixupRows Rollback Pending" << endl; - //@Bug 4994 Cancelled job is not error - result.result = JOB_CANCELED; - - // Log - LoggingID logid( DMLLoggingId, fSessionID, cpackage.get_TxnID()); - logging::Message::Args args1; - logging::Message msg1(1); - args1.add("SQL statement canceled by user"); - msg1.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); - - // Clean out the pipe; - DMLResult tmpResult; - receiveAll( tmpResult, uniqueId, fPMs, pmState, tableOid); - } - - // get stats from ExeMgr - if (!err) - { - qb = 3; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - msg = fExeMgr->read(); - msg >> result.queryStats; - msg >> result.extendedStats; - msg >> result.miniStats; - result.stats.unserialize(msg); - } - - //@Bug 4358 get rid of broken pipe error by sending a dummy bs. - if (err) - { - msg.restart(); - msg << qb; - fExeMgr->write(msg); - } - - return rowsProcessed; - //stats.insert(); + pmState[fPMs[i]] = true; } - catch (runtime_error& ex) + + // timer.start("ExeMgr"); + fExeMgr->write(msg); + fExeMgr->write(*(cpackage.get_ExecutionPlan())); + // cout << "sending to ExeMgr plan with length " << (cpackage.get_ExecutionPlan())->length() << endl; + msg.restart(); + emsgBs.restart(); + msg = fExeMgr->read(); // error handling + + if (msg.length() == 4) { - cerr << "UpdatePackageProcessor::processPackage::fixupRows" << ex.what() << endl; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add(ex.what()); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - qb = 0; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - return rowsProcessed; + msg >> qb; + + if (qb != 0) + err = true; } - catch ( ... ) + else { + qb = 999; + err = true; + } + + if (err) + { + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: ExeMgr Error"); + args.add((int)qb); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + // timer.finish(); + return rowsProcessed; + } + + emsgBs = fExeMgr->read(); + + if (emsgBs.length() == 0) + { + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("Lost connection to ExeMgr"); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + // timer.finish(); + return rowsProcessed; + } + + emsgBs >> emsgStr; + + while (true) + { + if (fRollbackPending) + { + break; + } + + msg.restart(); + msgBk.restart(); + msg = fExeMgr->read(); + msgBk = msg; + + if (msg.length() == 0) + { cerr << "UpdatePackageProcessor::processPackage::fixupRows" << endl; logging::Message::Args args; logging::Message message(2); args.add("Update Failed: "); - args.add("Unknown error caught when communicating with ExeMgr"); + args.add("Lost connection to ExeMgr"); message.format(args); result.result = UPDATE_ERROR; result.message = message; - qb = 0; - msg.restart(); - msg << qb; - fExeMgr->write(msg); - return rowsProcessed; + // timer.finish(); + // return rowsProcessed; + break; + } + else + { + if (rowGroup.get() == NULL) + { + // This is mete data, need to send all PMs. + metaData = true; + // cout << "sending meta data" << endl; + // timer.start("Meta"); + err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot); + rowGroup.reset(new rowgroup::RowGroup()); + rowGroup->deserialize(msg); + qb = 100; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + metaData = false; + // timer.stop("Meta"); + continue; + } + + rgData.deserialize(msg, true); + rowGroup->setData(&rgData); + // rowGroup->setData(const_cast(msg.buf())); + err = (rowGroup->getStatus() != 0); + + if (err) + { + // msgBk.advance(rowGroup->getDataSize()); + string errorMsg; + msg >> errorMsg; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add(errorMsg); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + DMLResult tmpResult; + receiveAll(tmpResult, uniqueId, fPMs, pmState, tableOid); + /* qb = 100; + //@Bug 4358 get rid of broken pipe error. + msg.restart(); + msg << qb; + fExeMgr->write(msg); + */ //timer.finish(); + // return rowsProcessed; + // err = true; + break; + } + + if (rowGroup->getRGData() == NULL) + { + msg.restart(); + } + + if (rowGroup->getRowCount() == 0) // done fetching + { + // timer.finish(); + // need to receive all response + err = receiveAll(result, uniqueId, fPMs, pmState, tableOid); + // return rowsProcessed; + break; + } + + if (rowGroup->getBaseRid() == (uint64_t)(-1)) + { + continue; // @bug4247, not valid row ids, may from small side outer + } + + dbroot = rowGroup->getDBRoot(); + // cout << "dbroot in the rowgroup is " << dbroot << endl; + // timer.start("processRowgroup"); + err = processRowgroup(msgBk, result, uniqueId, cpackage, pmState, metaData, dbroot); + + // timer.stop("processRowgroup"); + if (err) + { + // timer.finish(); + LoggingID logid(DMLLoggingId, fSessionID, cpackage.get_TxnID()); + logging::Message::Args args1; + logging::Message msg1(1); + args1.add("SQL statement erroring out, need to receive all messages from WES"); + msg1.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); + DMLResult tmpResult; + receiveAll(tmpResult, uniqueId, fPMs, pmState, tableOid); + logging::Message::Args args2; + logging::Message msg2(1); + args2.add("SQL statement erroring out, received all messages from WES"); + msg2.format(args2); + logger.logMessage(LOG_TYPE_DEBUG, msg2, logid); + //@Bug 4358 get rid of broken pipe error. + /* msg.restart(); + msg << qb; + fExeMgr->write(msg); + return rowsProcessed; + */ + // err = true; + break; + } + + rowsProcessed += rowGroup->getRowCount(); + } + } + + if (fRollbackPending) + { + err = true; + // Response to user + cerr << "UpdatePackageProcessor::processPackage::fixupRows Rollback Pending" << endl; + //@Bug 4994 Cancelled job is not error + result.result = JOB_CANCELED; + + // Log + LoggingID logid(DMLLoggingId, fSessionID, cpackage.get_TxnID()); + logging::Message::Args args1; + logging::Message msg1(1); + args1.add("SQL statement canceled by user"); + msg1.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg1, logid); + + // Clean out the pipe; + DMLResult tmpResult; + receiveAll(tmpResult, uniqueId, fPMs, pmState, tableOid); + } + + // get stats from ExeMgr + if (!err) + { + qb = 3; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + msg = fExeMgr->read(); + msg >> result.queryStats; + msg >> result.extendedStats; + msg >> result.miniStats; + result.stats.unserialize(msg); + } + + //@Bug 4358 get rid of broken pipe error by sending a dummy bs. + if (err) + { + msg.restart(); + msg << qb; + fExeMgr->write(msg); } - //timer.finish(); return rowsProcessed; + // stats.insert(); + } + catch (runtime_error& ex) + { + cerr << "UpdatePackageProcessor::processPackage::fixupRows" << ex.what() << endl; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add(ex.what()); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + qb = 0; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + return rowsProcessed; + } + catch (...) + { + cerr << "UpdatePackageProcessor::processPackage::fixupRows" << endl; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("Unknown error caught when communicating with ExeMgr"); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + qb = 0; + msg.restart(); + msg << qb; + fExeMgr->write(msg); + return rowsProcessed; + } + + // timer.finish(); + return rowsProcessed; } -bool UpdatePackageProcessor::processRowgroup(ByteStream& aRowGroup, DMLResult& result, const uint64_t uniqueId, - dmlpackage::CalpontDMLPackage& cpackage, std::map& pmState, bool isMeta, uint32_t dbroot) +bool UpdatePackageProcessor::processRowgroup(ByteStream& aRowGroup, DMLResult& result, + const uint64_t uniqueId, dmlpackage::CalpontDMLPackage& cpackage, + std::map& pmState, bool isMeta, uint32_t dbroot) { - bool rc = false; - //cout << "Get dbroot " << dbroot << endl; - uint32_t pmNum = (*fDbRootPMMap)[dbroot]; + bool rc = false; + // cout << "Get dbroot " << dbroot << endl; + uint32_t pmNum = (*fDbRootPMMap)[dbroot]; - ByteStream bytestream; - bytestream << (uint8_t)WE_SVR_UPDATE; - bytestream << uniqueId; - bytestream << pmNum; - bytestream << (uint32_t)cpackage.get_TxnID(); - bytestream += aRowGroup; - //cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; - uint32_t msgRecived = 0; - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - ByteStream::byte tmp8; - string errorMsg; - uint32_t tmp32; - uint64_t blocksChanged = 0; + ByteStream bytestream; + bytestream << (uint8_t)WE_SVR_UPDATE; + bytestream << uniqueId; + bytestream << pmNum; + bytestream << (uint32_t)cpackage.get_TxnID(); + bytestream += aRowGroup; + // cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; + uint32_t msgRecived = 0; + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + ByteStream::byte tmp8; + string errorMsg; + uint32_t tmp32; + uint64_t blocksChanged = 0; - if (isMeta) //send to all PMs + if (isMeta) // send to all PMs + { + cpackage.write(bytestream); + fWEClient->write_to_all(bytestream); + + while (1) { - cpackage.write(bytestream); - fWEClient->write_to_all(bytestream); + if (msgRecived == fWEClient->getPmCount()) + break; - while (1) + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = true; + break; + } + else + { + *bsIn >> tmp8; + + if (tmp8 > 0) { - if (msgRecived == fWEClient->getPmCount()) - break; - - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = true; - break; - } - else - { - *bsIn >> tmp8; - - if (tmp8 > 0) - { - *bsIn >> errorMsg; - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add(errorMsg); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - break; - } - else - msgRecived++; - } - } - - return rc; - } - - if (pmState[pmNum]) - { - try - { - //cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; - fWEClient->write(bytestream, (uint32_t)pmNum); - pmState[pmNum] = false; - } - catch (runtime_error& ex) //write error - { - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add(ex.what()); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - } - catch (...) - { - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add("Unknown error caught when communicating with WES"); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - } - } - else - { - while (1) - { - bsIn.reset(new ByteStream()); - - try - { - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = true; - errorMsg = "Lost connection to Write Engine Server while updating"; - throw std::runtime_error(errorMsg); - } - else - { - *bsIn >> tmp8; - *bsIn >> errorMsg; - - if (tmp8 == IDBRANGE_WARNING) - { - result.result = IDBRANGE_WARNING; - logging::Message::Args args; - logging::Message message(2); - args.add(errorMsg); - message.format(args); - result.message = message; - } - else if (tmp8 > 0) - { - result.stats.fErrorNo = tmp8; - rc = (tmp8 != 0); - } - - *bsIn >> tmp32; - //cout << "Received response from pm " << tmp32 << endl; - pmState[tmp32] = true; - *bsIn >> blocksChanged; - result.stats.fBlocksChanged += blocksChanged; - - if (rc != 0) - { - throw std::runtime_error(errorMsg); - } - - if (tmp32 == (uint32_t)pmNum) - { - //cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; - fWEClient->write(bytestream, (uint32_t)pmNum); - pmState[pmNum] = false; - break; - } - } - } - catch (runtime_error& ex) //write error - { - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add(ex.what()); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - break; - } - catch (...) - { - rc = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add("Unknown error caught when communicating with WES"); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - break; - } + *bsIn >> errorMsg; + rc = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add(errorMsg); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + break; } + else + msgRecived++; + } } return rc; + } + + if (pmState[pmNum]) + { + try + { + // cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; + fWEClient->write(bytestream, (uint32_t)pmNum); + pmState[pmNum] = false; + } + catch (runtime_error& ex) // write error + { + rc = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add(ex.what()); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + } + catch (...) + { + rc = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("Unknown error caught when communicating with WES"); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + } + } + else + { + while (1) + { + bsIn.reset(new ByteStream()); + + try + { + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = true; + errorMsg = "Lost connection to Write Engine Server while updating"; + throw std::runtime_error(errorMsg); + } + else + { + *bsIn >> tmp8; + *bsIn >> errorMsg; + + if (tmp8 == IDBRANGE_WARNING) + { + result.result = IDBRANGE_WARNING; + logging::Message::Args args; + logging::Message message(2); + args.add(errorMsg); + message.format(args); + result.message = message; + } + else if (tmp8 > 0) + { + result.stats.fErrorNo = tmp8; + rc = (tmp8 != 0); + } + + *bsIn >> tmp32; + // cout << "Received response from pm " << tmp32 << endl; + pmState[tmp32] = true; + *bsIn >> blocksChanged; + result.stats.fBlocksChanged += blocksChanged; + + if (rc != 0) + { + throw std::runtime_error(errorMsg); + } + + if (tmp32 == (uint32_t)pmNum) + { + // cout << "sending rows to pm " << pmNum << " with msg length " << bytestream.length() << endl; + fWEClient->write(bytestream, (uint32_t)pmNum); + pmState[pmNum] = false; + break; + } + } + } + catch (runtime_error& ex) // write error + { + rc = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add(ex.what()); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + break; + } + catch (...) + { + rc = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("Unknown error caught when communicating with WES"); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + break; + } + } + } + + return rc; } bool UpdatePackageProcessor::receiveAll(DMLResult& result, const uint64_t uniqueId, std::vector& fPMs, std::map& pmState, const uint32_t tableOid) { - //check how many message we need to receive - uint32_t messagesNotReceived = 0; - bool err = false; + // check how many message we need to receive + uint32_t messagesNotReceived = 0; + bool err = false; - for (unsigned i = 0; i < fPMs.size(); i++) + for (unsigned i = 0; i < fPMs.size(); i++) + { + if (!pmState[fPMs[i]]) + messagesNotReceived++; + } + + boost::shared_ptr bsIn; + ByteStream::byte tmp8; + string errorMsg; + uint32_t msgReceived = 0; + + if (messagesNotReceived > 0) + { + LoggingID logid(DMLLoggingId, fSessionID, fSessionID); + + if (messagesNotReceived > fWEClient->getPmCount()) { - if (!pmState[fPMs[i]]) - messagesNotReceived++; + logging::Message::Args args1; + logging::Message msg(1); + args1.add("Update outstanding messages exceed PM count , need to receive messages:PMcount = "); + ostringstream oss; + oss << messagesNotReceived << ":" << fWEClient->getPmCount(); + args1.add(oss.str()); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_ERROR, msg, logid); + err = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("One of WriteEngineServer went away."); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + return err; } - boost::shared_ptr bsIn; - ByteStream::byte tmp8; - string errorMsg; - uint32_t msgReceived = 0; + bsIn.reset(new ByteStream()); + ByteStream::quadbyte tmp32; + uint64_t blocksChanged = 0; - if (messagesNotReceived > 0) + while (1) { - LoggingID logid( DMLLoggingId, fSessionID, fSessionID); + if (msgReceived == messagesNotReceived) + break; - if ( messagesNotReceived > fWEClient->getPmCount()) + bsIn.reset(new ByteStream()); + + try + { + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Update outstanding messages exceed PM count , need to receive messages:PMcount = "); - ostringstream oss; - oss << messagesNotReceived << ":" << fWEClient->getPmCount(); - args1.add(oss.str()); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_ERROR, msg, logid); - err = true; + err = true; + errorMsg = "Lost connection to Write Engine Server while updating"; + throw std::runtime_error(errorMsg); + } + else + { + *bsIn >> tmp8; + *bsIn >> errorMsg; + + if (tmp8 == IDBRANGE_WARNING) + { + result.result = IDBRANGE_WARNING; logging::Message::Args args; logging::Message message(2); - args.add("Update Failed: "); - args.add("One of WriteEngineServer went away."); + args.add(errorMsg); message.format(args); - result.result = UPDATE_ERROR; result.message = message; - return err; - } - - bsIn.reset(new ByteStream()); - ByteStream::quadbyte tmp32; - uint64_t blocksChanged = 0; - - while (1) - { - if (msgReceived == messagesNotReceived) - break; - - bsIn.reset(new ByteStream()); - - try - { - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - err = true; - errorMsg = "Lost connection to Write Engine Server while updating"; - throw std::runtime_error(errorMsg); - } - else - { - *bsIn >> tmp8; - *bsIn >> errorMsg; - - if (tmp8 == IDBRANGE_WARNING) - { - result.result = IDBRANGE_WARNING; - logging::Message::Args args; - logging::Message message(2); - args.add(errorMsg); - message.format(args); - result.message = message; - } - else - { - result.stats.fErrorNo = tmp8; - err = (tmp8 != 0); - } - - *bsIn >> tmp32; - *bsIn >> blocksChanged; - //cout << "Received response from pm " << tmp32 << endl; - pmState[tmp32] = true; - - if (err) - { - throw std::runtime_error(errorMsg); - } - - msgReceived++; - result.stats.fBlocksChanged += blocksChanged; - } - } - catch (runtime_error& ex) //write error - { - err = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add(ex.what()); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - break; - } - catch (...) - { - err = true; - logging::Message::Args args; - logging::Message message(2); - args.add("Update Failed: "); - args.add("Unknown error caught when communicating with WES"); - message.format(args); - result.result = UPDATE_ERROR; - result.message = message; - break; - } + } + else + { + result.stats.fErrorNo = tmp8; + err = (tmp8 != 0); + } + + *bsIn >> tmp32; + *bsIn >> blocksChanged; + // cout << "Received response from pm " << tmp32 << endl; + pmState[tmp32] = true; + + if (err) + { + throw std::runtime_error(errorMsg); + } + + msgReceived++; + result.stats.fBlocksChanged += blocksChanged; } + } + catch (runtime_error& ex) // write error + { + err = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add(ex.what()); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + break; + } + catch (...) + { + err = true; + logging::Message::Args args; + logging::Message message(2); + args.add("Update Failed: "); + args.add("Unknown error caught when communicating with WES"); + message.format(args); + result.result = UPDATE_ERROR; + result.message = message; + break; + } } + } - return err; + return err; } -} // namespace dmlpackageprocessor +} // namespace dmlpackageprocessor diff --git a/dbcon/dmlpackageproc/updatepackageprocessor.h b/dbcon/dmlpackageproc/updatepackageprocessor.h index 178472173..50f27cd28 100644 --- a/dbcon/dmlpackageproc/updatepackageprocessor.h +++ b/dbcon/dmlpackageproc/updatepackageprocessor.h @@ -43,41 +43,42 @@ namespace dmlpackageprocessor */ class UpdatePackageProcessor : public DMLPackageProcessor { + public: + UpdatePackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) + { + } + /** @brief process an UpdateDMLPackage + * + * @param cpackage the UpdateDMLPackage to process + */ + EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); -public: - UpdatePackageProcessor(BRM::DBRM* aDbrm, uint32_t sid) : DMLPackageProcessor(aDbrm, sid) - { - } - /** @brief process an UpdateDMLPackage - * - * @param cpackage the UpdateDMLPackage to process - */ - EXPORT DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage); + protected: + private: + /** @brief send execution plan to ExeMgr and fetch rows + * + * @param cpackage the UpdateDMLPackage to process + * @param result the result of the operation + * @return rows processed + */ + uint64_t fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, const uint64_t uniqueId, + const uint32_t tableOid); -protected: - -private: - /** @brief send execution plan to ExeMgr and fetch rows - * - * @param cpackage the UpdateDMLPackage to process - * @param result the result of the operation - * @return rows processed - */ - uint64_t fixUpRows(dmlpackage::CalpontDMLPackage& cpackage, DMLResult& result, const uint64_t uniqueId, const uint32_t tableOid); - - - /** @brief send row group to the PM to process - * - * @param aRowGroup the row group to be sent - * @param result the result of the operation - * @return the error code - */ - bool processRowgroup(messageqcpp::ByteStream& aRowGroup, DMLResult& result, const uint64_t uniqueId, dmlpackage::CalpontDMLPackage& cpackage, std::map& pmState, bool isMeta = false, uint32_t dbroot = 1); - bool receiveAll(DMLResult& result, const uint64_t uniqueId, std::vector& fPMs, std::map& pmState, const uint32_t tableOid); + /** @brief send row group to the PM to process + * + * @param aRowGroup the row group to be sent + * @param result the result of the operation + * @return the error code + */ + bool processRowgroup(messageqcpp::ByteStream& aRowGroup, DMLResult& result, const uint64_t uniqueId, + dmlpackage::CalpontDMLPackage& cpackage, std::map& pmState, + bool isMeta = false, uint32_t dbroot = 1); + bool receiveAll(DMLResult& result, const uint64_t uniqueId, std::vector& fPMs, + std::map& pmState, const uint32_t tableOid); }; -} +} // namespace dmlpackageprocessor #undef EXPORT -#endif //UPDATEPACKAGEPROCESSOR_H +#endif // UPDATEPACKAGEPROCESSOR_H diff --git a/dbcon/doc/calpontsystemcatalog.h b/dbcon/doc/calpontsystemcatalog.h index a3243e7bf..fa821e9e9 100644 --- a/dbcon/doc/calpontsystemcatalog.h +++ b/dbcon/doc/calpontsystemcatalog.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontsystemcatalog.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: calpontsystemcatalog.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DBCON_CALPONTSYSTEMCATALOG_H @@ -27,32 +27,30 @@ namespace dbcon { - class CalpontSystemCatalog { -public: - /** looks up a column's TCN in the System Catalog - * - * For a unique table_name.column_name return the internal TCN - */ - TCN lookupTCN(const TableColName& tableColName) const; + public: + /** looks up a column's TCN in the System Catalog + * + * For a unique table_name.column_name return the internal TCN + */ + TCN lookupTCN(const TableColName& tableColName) const; - /** returns the colunm type attribute(s) for a column - * - * return the various colunm attributes for a given TCN: - * dictionary/native - * DDN - */ - ColType colType(const TCN& tcn) const; + /** returns the colunm type attribute(s) for a column + * + * return the various colunm attributes for a given TCN: + * dictionary/native + * DDN + */ + ColType colType(const TCN& tcn) const; - /** return the current SCN - * - * returns the current System Change Number (for versioning support) - */ - SCN scn(void) const; + /** return the current SCN + * + * returns the current System Change Number (for versioning support) + */ + SCN scn(void) const; }; -} //namespace dbcon - -#endif //DBCON_CALPONTSYSTEMCATALOG_H +} // namespace dbcon +#endif // DBCON_CALPONTSYSTEMCATALOG_H diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index 54e7299a2..c3919b893 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: aggregatecolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: aggregatecolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ #include #include using namespace std; @@ -46,84 +46,79 @@ using namespace joblist; namespace execplan { - void getAggCols(execplan::ParseTree* n, void* obj) { - vector* list = reinterpret_cast< vector*>(obj); - TreeNode* tn = n->data(); - AggregateColumn* sc = dynamic_cast(tn); - FunctionColumn* fc = dynamic_cast(tn); - ArithmeticColumn* ac = dynamic_cast(tn); - SimpleFilter* sf = dynamic_cast(tn); - ConstantFilter* cf = dynamic_cast(tn); + vector* list = reinterpret_cast*>(obj); + TreeNode* tn = n->data(); + AggregateColumn* sc = dynamic_cast(tn); + FunctionColumn* fc = dynamic_cast(tn); + ArithmeticColumn* ac = dynamic_cast(tn); + SimpleFilter* sf = dynamic_cast(tn); + ConstantFilter* cf = dynamic_cast(tn); - if (sc) - { - list->push_back(sc); - } - else if (fc) - { - fc->hasAggregate(); - list->insert(list->end(), fc->aggColumnList().begin(), fc->aggColumnList().end()); - } - else if (ac) - { - ac->hasAggregate(); - list->insert(list->end(), ac->aggColumnList().begin(), ac->aggColumnList().end()); - } - else if (sf) - { - sf->hasAggregate(); - list->insert(list->end(), sf->aggColumnList().begin(), sf->aggColumnList().end()); - } - else if (cf) - { - cf->hasAggregate(); - list->insert(list->end(), cf->aggColumnList().begin(), cf->aggColumnList().end()); - } + if (sc) + { + list->push_back(sc); + } + else if (fc) + { + fc->hasAggregate(); + list->insert(list->end(), fc->aggColumnList().begin(), fc->aggColumnList().end()); + } + else if (ac) + { + ac->hasAggregate(); + list->insert(list->end(), ac->aggColumnList().begin(), ac->aggColumnList().end()); + } + else if (sf) + { + sf->hasAggregate(); + list->insert(list->end(), sf->aggColumnList().begin(), sf->aggColumnList().end()); + } + else if (cf) + { + cf->hasAggregate(); + list->insert(list->end(), cf->aggColumnList().begin(), cf->aggColumnList().end()); + } } /** * Constructors/Destructors */ -AggregateColumn::AggregateColumn(): - fAggOp(NOOP), - fAsc(false) +AggregateColumn::AggregateColumn() : fAggOp(NOOP), fAsc(false) { } -AggregateColumn::AggregateColumn(const uint32_t sessionID): - ReturnedColumn(sessionID), - fAggOp(NOOP), - fAsc(false) +AggregateColumn::AggregateColumn(const uint32_t sessionID) + : ReturnedColumn(sessionID), fAggOp(NOOP), fAsc(false) { } // deprecated constructor. use function name as string -AggregateColumn::AggregateColumn(const string& functionName, const string& content, const uint32_t sessionID): - ReturnedColumn(sessionID), - fFunctionName(functionName), - fAggOp(NOOP), - fAsc(false), - fData(functionName + "(" + content + ")") +AggregateColumn::AggregateColumn(const string& functionName, const string& content, const uint32_t sessionID) + : ReturnedColumn(sessionID) + , fFunctionName(functionName) + , fAggOp(NOOP) + , fAsc(false) + , fData(functionName + "(" + content + ")") { - // TODO: need to handle distinct - SRCP srcp(new ArithmeticColumn(content)); - fAggParms.push_back(srcp); + // TODO: need to handle distinct + SRCP srcp(new ArithmeticColumn(content)); + fAggParms.push_back(srcp); } -AggregateColumn::AggregateColumn( const AggregateColumn& rhs, const uint32_t sessionID ): - ReturnedColumn(rhs, sessionID), - fFunctionName (rhs.fFunctionName), - fAggOp(rhs.fAggOp), - fTableAlias(rhs.tableAlias()), - fAsc(rhs.asc()), - fData(rhs.data()), - fConstCol(rhs.fConstCol), - fTimeZone(rhs.timeZone()) +AggregateColumn::AggregateColumn(const AggregateColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID) + , fFunctionName(rhs.fFunctionName) + , fAggOp(rhs.fAggOp) + , fTableAlias(rhs.tableAlias()) + , fAsc(rhs.asc()) + , fData(rhs.data()) + , fConstCol(rhs.fConstCol) + , fTimeZone(rhs.timeZone()) { - fAlias = rhs.alias(); - fAggParms = rhs.fAggParms; + fAlias = rhs.alias(); + fAggParms = rhs.fAggParms; } /** @@ -132,544 +127,531 @@ AggregateColumn::AggregateColumn( const AggregateColumn& rhs, const uint32_t ses const string AggregateColumn::toString() const { - ostringstream output; - output << "AggregateColumn " << data() << endl; - output << "func/distinct: " << (int)fAggOp << "/" << fDistinct << endl; - output << "expressionId=" << fExpressionId << endl; + ostringstream output; + output << "AggregateColumn " << data() << endl; + output << "func/distinct: " << (int)fAggOp << "/" << fDistinct << endl; + output << "expressionId=" << fExpressionId << endl; - if (fAlias.length() > 0) output << "/Alias: " << fAlias << endl; + if (fAlias.length() > 0) + output << "/Alias: " << fAlias << endl; - if (fAggParms.size() == 0) - output << "No arguments"; - else - for (uint32_t i = 0; i < fAggParms.size(); ++i) - { - output << *(fAggParms[i]) << " "; - } + if (fAggParms.size() == 0) + output << "No arguments"; + else + for (uint32_t i = 0; i < fAggParms.size(); ++i) + { + output << *(fAggParms[i]) << " "; + } - output << endl; + output << endl; - if (fConstCol) - output << *fConstCol; + if (fConstCol) + output << *fConstCol; - return output.str(); + return output.str(); } ostream& operator<<(ostream& output, const AggregateColumn& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void AggregateColumn::serialize(messageqcpp::ByteStream& b) const { - CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator rcit; - b << (uint8_t) ObjectReader::AGGREGATECOLUMN; - ReturnedColumn::serialize(b); - b << fFunctionName; - b << static_cast(fAggOp); + CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator rcit; + b << (uint8_t)ObjectReader::AGGREGATECOLUMN; + ReturnedColumn::serialize(b); + b << fFunctionName; + b << static_cast(fAggOp); - b << static_cast(fAggParms.size()); + b << static_cast(fAggParms.size()); - for (uint32_t i = 0; i < fAggParms.size(); ++i) - { - fAggParms[i]->serialize(b); - } + for (uint32_t i = 0; i < fAggParms.size(); ++i) + { + fAggParms[i]->serialize(b); + } - b << static_cast(fGroupByColList.size()); + b << static_cast(fGroupByColList.size()); - for (rcit = fGroupByColList.begin(); rcit != fGroupByColList.end(); ++rcit) - (*rcit)->serialize(b); + for (rcit = fGroupByColList.begin(); rcit != fGroupByColList.end(); ++rcit) + (*rcit)->serialize(b); - b << static_cast(fProjectColList.size()); + b << static_cast(fProjectColList.size()); - for (rcit = fProjectColList.begin(); rcit != fProjectColList.end(); ++rcit) - (*rcit)->serialize(b); + for (rcit = fProjectColList.begin(); rcit != fProjectColList.end(); ++rcit) + (*rcit)->serialize(b); - b << fData; - b << fTimeZone; - //b << fAlias; - b << fTableAlias; - b << static_cast(fAsc); + b << fData; + b << fTimeZone; + // b << fAlias; + b << fTableAlias; + b << static_cast(fAsc); - if (fConstCol.get() == 0) - b << (uint8_t) ObjectReader::NULL_CLASS; - else - fConstCol->serialize(b); + if (fConstCol.get() == 0) + b << (uint8_t)ObjectReader::NULL_CLASS; + else + fConstCol->serialize(b); } void AggregateColumn::unserialize(messageqcpp::ByteStream& b) { - messageqcpp::ByteStream::quadbyte size; - messageqcpp::ByteStream::quadbyte i; - ReturnedColumn* rc; + messageqcpp::ByteStream::quadbyte size; + messageqcpp::ByteStream::quadbyte i; + ReturnedColumn* rc; - ObjectReader::checkType(b, ObjectReader::AGGREGATECOLUMN); - fGroupByColList.erase(fGroupByColList.begin(), fGroupByColList.end()); - fProjectColList.erase(fProjectColList.begin(), fProjectColList.end()); - fAggParms.erase(fAggParms.begin(), fAggParms.end()); - ReturnedColumn::unserialize(b); - b >> fFunctionName; - b >> fAggOp; + ObjectReader::checkType(b, ObjectReader::AGGREGATECOLUMN); + fGroupByColList.erase(fGroupByColList.begin(), fGroupByColList.end()); + fProjectColList.erase(fProjectColList.begin(), fProjectColList.end()); + fAggParms.erase(fAggParms.begin(), fAggParms.end()); + ReturnedColumn::unserialize(b); + b >> fFunctionName; + b >> fAggOp; - b >> size; + b >> size; - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fAggParms.push_back(srcp); - } + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fAggParms.push_back(srcp); + } - b >> size; + b >> size; - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fGroupByColList.push_back(srcp); - } + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fGroupByColList.push_back(srcp); + } - b >> size; + b >> size; - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fProjectColList.push_back(srcp); - } + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fProjectColList.push_back(srcp); + } - b >> fData; - b >> fTimeZone; - //b >> fAlias; - b >> fTableAlias; - b >> reinterpret_cast< ByteStream::doublebyte&>(fAsc); - fConstCol.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + b >> fData; + b >> fTimeZone; + // b >> fAlias; + b >> fTableAlias; + b >> reinterpret_cast(fAsc); + fConstCol.reset(dynamic_cast(ObjectReader::createTreeNode(b))); } bool AggregateColumn::operator==(const AggregateColumn& t) const { - const ReturnedColumn* rc1, *rc2; - AggParms::const_iterator it, it2; + const ReturnedColumn *rc1, *rc2; + AggParms::const_iterator it, it2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (fFunctionName != t.fFunctionName) - return false; - - if (fAggOp == COUNT_ASTERISK && t.fAggOp == COUNT_ASTERISK) - return true; - - if (fAggOp != t.fAggOp) - return false; - - if (aggParms().size() != t.aggParms().size()) - { - return false; - } - - for (it = fAggParms.begin(), it2 = t.fAggParms.begin(); - it != fAggParms.end(); - ++it, ++it2) - { - if (**it != **it2) - return false; - } - - if (fTableAlias != t.fTableAlias) - return false; - - if (fData != t.fData) - return false; - - if (fAsc != t.fAsc) - return false; - - if ((fConstCol.get() != NULL && t.fConstCol.get() == NULL) || - (fConstCol.get() == NULL && t.fConstCol.get() != NULL) || - (fConstCol.get() != NULL && t.fConstCol.get() != NULL && - *(fConstCol.get()) != t.fConstCol.get())) - return false; - - if (fTimeZone != t.fTimeZone) - return false; + if (fFunctionName != t.fFunctionName) + return false; + if (fAggOp == COUNT_ASTERISK && t.fAggOp == COUNT_ASTERISK) return true; + + if (fAggOp != t.fAggOp) + return false; + + if (aggParms().size() != t.aggParms().size()) + { + return false; + } + + for (it = fAggParms.begin(), it2 = t.fAggParms.begin(); it != fAggParms.end(); ++it, ++it2) + { + if (**it != **it2) + return false; + } + + if (fTableAlias != t.fTableAlias) + return false; + + if (fData != t.fData) + return false; + + if (fAsc != t.fAsc) + return false; + + if ((fConstCol.get() != NULL && t.fConstCol.get() == NULL) || + (fConstCol.get() == NULL && t.fConstCol.get() != NULL) || + (fConstCol.get() != NULL && t.fConstCol.get() != NULL && *(fConstCol.get()) != t.fConstCol.get())) + return false; + + if (fTimeZone != t.fTimeZone) + return false; + + return true; } bool AggregateColumn::operator==(const TreeNode* t) const { - const AggregateColumn* ac; + const AggregateColumn* ac; - ac = dynamic_cast(t); + ac = dynamic_cast(t); - if (ac == NULL) - return false; + if (ac == NULL) + return false; - return *this == *ac; + return *this == *ac; } bool AggregateColumn::operator!=(const AggregateColumn& t) const { - return !(*this == t); + return !(*this == t); } bool AggregateColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } bool AggregateColumn::hasAggregate() { - fAggColumnList.push_back(this); - return true; + fAggColumnList.push_back(this); + return true; } void AggregateColumn::evaluate(Row& row, bool& isNull) { - switch (fResultType.colDataType) - { - case CalpontSystemCatalog::DATE: - if (row.equals<4>(DATENULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<4>(fInputIndex); + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::DATE: + if (row.equals<4>(DATENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<4>(fInputIndex); - break; + break; - case CalpontSystemCatalog::DATETIME: - if (row.equals<8>(DATETIMENULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<8>(fInputIndex); + case CalpontSystemCatalog::DATETIME: + if (row.equals<8>(DATETIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); - break; + break; - case CalpontSystemCatalog::TIMESTAMP: - if (row.equals<8>(TIMESTAMPNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<8>(fInputIndex); + case CalpontSystemCatalog::TIMESTAMP: + if (row.equals<8>(TIMESTAMPNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); - break; + break; - case CalpontSystemCatalog::TIME: - if (row.equals<8>(TIMENULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<8>(fInputIndex); + case CalpontSystemCatalog::TIME: + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); - break; + break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::STRINT: - case CalpontSystemCatalog::TEXT: - switch (row.getColumnWidth(fInputIndex)) - { - case 1: - if (row.equals<1>(CHAR1NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<1>(fInputIndex); - - break; - - case 2: - if (row.equals<2>(CHAR2NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<2>(fInputIndex); - - break; - - case 3: - case 4: - if (row.equals<4>(CHAR4NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<4>(fInputIndex); - - break; - - case 5: - case 6: - case 7: - case 8: - if (row.equals<8>(CHAR8NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<8>(fInputIndex); - - break; - - default: - { - auto const str = row.getConstString(fInputIndex); - if (str.eq(utils::ConstString(CPNULLSTRMARK))) - isNull = true; - else - fResult.strVal = str.toString(); - - // stringColVal is padded with '\0' to colWidth so can't use str.length() - if (strlen(fResult.strVal.c_str()) == 0) - isNull = true; - - break; - } - } - - if (fResultType.colDataType == CalpontSystemCatalog::STRINT) - fResult.intVal = uint64ToStr(fResult.origIntVal); - else - fResult.intVal = atoll((char*)&fResult.origIntVal); - - break; - - case CalpontSystemCatalog::BIGINT: - if (row.equals<8>(BIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<8>(fInputIndex); - - break; - - case CalpontSystemCatalog::UBIGINT: - if (row.equals<8>(UBIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<8>(fInputIndex); - - break; - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - if (row.equals<4>(INTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<4>(fInputIndex); - - break; - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - if (row.equals<4>(UINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<4>(fInputIndex); - - break; - - case CalpontSystemCatalog::SMALLINT: - if (row.equals<2>(SMALLINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<2>(fInputIndex); - - break; - - case CalpontSystemCatalog::USMALLINT: - if (row.equals<2>(USMALLINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<2>(fInputIndex); - - break; - - case CalpontSystemCatalog::TINYINT: - if (row.equals<1>(TINYINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<1>(fInputIndex); - - break; - - case CalpontSystemCatalog::UTINYINT: - if (row.equals<1>(UTINYINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<1>(fInputIndex); - - break; - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - if (row.equals<4>(FLOATNULL, fInputIndex)) - isNull = true; - else - fResult.floatVal = row.getFloatField(fInputIndex); - - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - if (row.equals<8>(DOUBLENULL, fInputIndex)) - isNull = true; - else - fResult.doubleVal = row.getDoubleField(fInputIndex); - - break; - - case CalpontSystemCatalog::LONGDOUBLE: - if (row.equals(LONGDOUBLENULL, fInputIndex)) - isNull = true; - else - fResult.longDoubleVal = row.getLongDoubleField(fInputIndex); - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - switch (fResultType.colWidth) - { - case 16: - { - datatypes::TSInt128 val = row.getTSInt128Field(fInputIndex); - - if (val.isNull()) - isNull = true; - else - fResult.decimalVal = IDB_Decimal(val, fResultType.scale, fResultType.precision); - - break; - } - - case 1: - if (row.equals<1>(TINYINTNULL, fInputIndex)) - isNull = true; - else - fResult.decimalVal = IDB_Decimal( - row.getIntField<1>(fInputIndex), - fResultType.scale, - fResultType.precision); - - break; - - case 2: - if (row.equals<2>(SMALLINTNULL, fInputIndex)) - isNull = true; - else - fResult.decimalVal = IDB_Decimal( - row.getIntField<2>(fInputIndex), - fResultType.scale, - fResultType.precision); - - break; - - case 4: - if (row.equals<4>(INTNULL, fInputIndex)) - isNull = true; - else - fResult.decimalVal = IDB_Decimal( - row.getIntField<4>(fInputIndex), - fResultType.scale, - fResultType.precision); - - break; - - default: - if (row.equals<8>(BIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.decimalVal = IDB_Decimal( - (int64_t) row.getUintField<8>(fInputIndex), - fResultType.scale, - fResultType.precision); - - break; - } - - break; - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::STRINT: + case CalpontSystemCatalog::TEXT: + switch (row.getColumnWidth(fInputIndex)) + { + case 1: + if (row.equals<1>(CHAR1NULL, fInputIndex)) isNull = true; - break; + else + fResult.origIntVal = row.getUintField<1>(fInputIndex); - default: // treat as int64 - if (row.equals<8>(BIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<8>(fInputIndex); + break; - break; - } + case 2: + if (row.equals<2>(CHAR2NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<2>(fInputIndex); + + break; + + case 3: + case 4: + if (row.equals<4>(CHAR4NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<4>(fInputIndex); + + break; + + case 5: + case 6: + case 7: + case 8: + if (row.equals<8>(CHAR8NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<8>(fInputIndex); + + break; + + default: + { + auto const str = row.getConstString(fInputIndex); + if (str.eq(utils::ConstString(CPNULLSTRMARK))) + isNull = true; + else + fResult.strVal = str.toString(); + + // stringColVal is padded with '\0' to colWidth so can't use str.length() + if (strlen(fResult.strVal.c_str()) == 0) + isNull = true; + + break; + } + } + + if (fResultType.colDataType == CalpontSystemCatalog::STRINT) + fResult.intVal = uint64ToStr(fResult.origIntVal); + else + fResult.intVal = atoll((char*)&fResult.origIntVal); + + break; + + case CalpontSystemCatalog::BIGINT: + if (row.equals<8>(BIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + + case CalpontSystemCatalog::UBIGINT: + if (row.equals<8>(UBIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<8>(fInputIndex); + + break; + + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + if (row.equals<4>(INTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<4>(fInputIndex); + + break; + + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + if (row.equals<4>(UINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<4>(fInputIndex); + + break; + + case CalpontSystemCatalog::SMALLINT: + if (row.equals<2>(SMALLINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<2>(fInputIndex); + + break; + + case CalpontSystemCatalog::USMALLINT: + if (row.equals<2>(USMALLINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<2>(fInputIndex); + + break; + + case CalpontSystemCatalog::TINYINT: + if (row.equals<1>(TINYINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<1>(fInputIndex); + + break; + + case CalpontSystemCatalog::UTINYINT: + if (row.equals<1>(UTINYINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<1>(fInputIndex); + + break; + + // In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + if (row.equals<4>(FLOATNULL, fInputIndex)) + isNull = true; + else + fResult.floatVal = row.getFloatField(fInputIndex); + + break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + if (row.equals<8>(DOUBLENULL, fInputIndex)) + isNull = true; + else + fResult.doubleVal = row.getDoubleField(fInputIndex); + + break; + + case CalpontSystemCatalog::LONGDOUBLE: + if (row.equals(LONGDOUBLENULL, fInputIndex)) + isNull = true; + else + fResult.longDoubleVal = row.getLongDoubleField(fInputIndex); + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + switch (fResultType.colWidth) + { + case 16: + { + datatypes::TSInt128 val = row.getTSInt128Field(fInputIndex); + + if (val.isNull()) + isNull = true; + else + fResult.decimalVal = IDB_Decimal(val, fResultType.scale, fResultType.precision); + + break; + } + + case 1: + if (row.equals<1>(TINYINTNULL, fInputIndex)) + isNull = true; + else + fResult.decimalVal = + IDB_Decimal(row.getIntField<1>(fInputIndex), fResultType.scale, fResultType.precision); + + break; + + case 2: + if (row.equals<2>(SMALLINTNULL, fInputIndex)) + isNull = true; + else + fResult.decimalVal = + IDB_Decimal(row.getIntField<2>(fInputIndex), fResultType.scale, fResultType.precision); + + break; + + case 4: + if (row.equals<4>(INTNULL, fInputIndex)) + isNull = true; + else + fResult.decimalVal = + IDB_Decimal(row.getIntField<4>(fInputIndex), fResultType.scale, fResultType.precision); + + break; + + default: + if (row.equals<8>(BIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.decimalVal = IDB_Decimal((int64_t)row.getUintField<8>(fInputIndex), fResultType.scale, + fResultType.precision); + + break; + } + + break; + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: isNull = true; break; + + default: // treat as int64 + if (row.equals<8>(BIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); + + break; + } } /*static*/ AggregateColumn::AggOp AggregateColumn::agname2num(const string& agname) { - /* - NOOP = 0, - COUNT_ASTERISK, - COUNT, - SUM, - AVG, - MIN, - MAX, - CONSTANT, - DISTINCT_COUNT, - DISTINCT_SUM, - DISTINCT_AVG, - STDDEV_POP, - STDDEV_SAMP, - VAR_POP, - VAR_SAMP, - BIT_AND, - BIT_OR, - BIT_XOR, - GROUP_CONCAT - */ - string lfn(agname); - algorithm::to_lower(lfn); + /* + NOOP = 0, + COUNT_ASTERISK, + COUNT, + SUM, + AVG, + MIN, + MAX, + CONSTANT, + DISTINCT_COUNT, + DISTINCT_SUM, + DISTINCT_AVG, + STDDEV_POP, + STDDEV_SAMP, + VAR_POP, + VAR_SAMP, + BIT_AND, + BIT_OR, + BIT_XOR, + GROUP_CONCAT + */ + string lfn(agname); + algorithm::to_lower(lfn); - if (lfn == "count(*)") - return COUNT_ASTERISK; + if (lfn == "count(*)") + return COUNT_ASTERISK; - if (lfn == "count") - return COUNT; + if (lfn == "count") + return COUNT; - if (lfn == "sum") - return SUM; + if (lfn == "sum") + return SUM; - if (lfn == "avg") - return AVG; + if (lfn == "avg") + return AVG; - if (lfn == "min") - return MIN; + if (lfn == "min") + return MIN; - if (lfn == "max") - return MAX; + if (lfn == "max") + return MAX; - if (lfn == "std") - return STDDEV_POP; + if (lfn == "std") + return STDDEV_POP; - if (lfn == "stddev_pop") - return STDDEV_POP; + if (lfn == "stddev_pop") + return STDDEV_POP; - if (lfn == "stddev_samp") - return STDDEV_SAMP; + if (lfn == "stddev_samp") + return STDDEV_SAMP; - if (lfn == "stddev") - return STDDEV_POP; + if (lfn == "stddev") + return STDDEV_POP; - if (lfn == "var_pop") - return VAR_POP; + if (lfn == "var_pop") + return VAR_POP; - if (lfn == "var_samp") - return VAR_SAMP; + if (lfn == "var_samp") + return VAR_SAMP; - if (lfn == "variance") - return VAR_POP; + if (lfn == "variance") + return VAR_POP; - return NOOP; + return NOOP; } -} // namespace execplan - +} // namespace execplan diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index cea23c259..b60fe6183 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: aggregatecolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: aggregatecolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef AGGREGATECOLUMN_H @@ -40,7 +40,6 @@ class ByteStream; */ namespace execplan { - typedef std::vector AggParms; /** @@ -51,419 +50,416 @@ typedef std::vector AggParms; */ class AggregateColumn : public ReturnedColumn { + public: + /** + * AggOp enum + */ + enum AggOp + { + NOOP = 0, + COUNT_ASTERISK, + COUNT, + SUM, + AVG, + MIN, + MAX, + CONSTANT, + DISTINCT_COUNT, + DISTINCT_SUM, + DISTINCT_AVG, + STDDEV_POP, + STDDEV_SAMP, + VAR_POP, + VAR_SAMP, + BIT_AND, + BIT_OR, + BIT_XOR, + GROUP_CONCAT, + UDAF, + MULTI_PARM + }; -public: - /** - * AggOp enum - */ - enum AggOp - { - NOOP = 0, - COUNT_ASTERISK, - COUNT, - SUM, - AVG, - MIN, - MAX, - CONSTANT, - DISTINCT_COUNT, - DISTINCT_SUM, - DISTINCT_AVG, - STDDEV_POP, - STDDEV_SAMP, - VAR_POP, - VAR_SAMP, - BIT_AND, - BIT_OR, - BIT_XOR, - GROUP_CONCAT, - UDAF, - MULTI_PARM - }; + /** + * typedef + */ + typedef std::vector ColumnList; + /* + * Constructors + */ + /** + * ctor + */ + AggregateColumn(); - /** - * typedef - */ - typedef std::vector ColumnList; - /* - * Constructors - */ - /** - * ctor - */ - AggregateColumn(); + /** + * ctor + */ + AggregateColumn(const uint32_t sessionID); - /** - * ctor - */ - AggregateColumn(const uint32_t sessionID); + /** + * ctor + */ + AggregateColumn(const std::string& functionName, const std::string& content, const uint32_t sessionID = 0); - /** - * ctor - */ - AggregateColumn(const std::string& functionName, const std::string& content, const uint32_t sessionID = 0); + /** + * ctor + */ + AggregateColumn(const AggregateColumn& rhs, const uint32_t sessionID = 0); - /** - * ctor - */ - AggregateColumn( const AggregateColumn& rhs, const uint32_t sessionID = 0 ); + /** + * Destructors + */ + virtual ~AggregateColumn() + { + } - /** - * Destructors - */ - virtual ~AggregateColumn() { } + /** + * Accessor Methods + */ + virtual const std::string functionName() const + { + return fFunctionName; + } - /** - * Accessor Methods - */ - virtual const std::string functionName() const - { - return fFunctionName; - } + /** + * accessor + */ + virtual void functionName(const std::string& functionName) + { + fFunctionName = functionName; + } - /** - * accessor - */ - virtual void functionName(const std::string& functionName) - { - fFunctionName = functionName; - } + /** + * accessor + */ + virtual uint8_t aggOp() const + { + return fAggOp; + } + /** + * accessor + */ + virtual void aggOp(const uint8_t aggOp) + { + fAggOp = aggOp; + } - /** - * accessor - */ - virtual uint8_t aggOp() const - { - return fAggOp; - } - /** - * accessor - */ - virtual void aggOp(const uint8_t aggOp) - { - fAggOp = aggOp; - } + /** get function parms + */ + virtual AggParms& aggParms() + { + return fAggParms; + } + virtual const AggParms& aggParms() const + { + return fAggParms; + } - /** get function parms - */ - virtual AggParms& aggParms() - { - return fAggParms; - } + /** set function parms + */ + virtual void aggParms(const AggParms& parms) + { + fAggParms = parms; + } - virtual const AggParms& aggParms() const - { - return fAggParms; - } + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual AggregateColumn* clone() const + { + return new AggregateColumn(*this); + } - /** set function parms - */ - virtual void aggParms(const AggParms& parms) - { - fAggParms = parms; - } + /** + * table alias name + */ + virtual const std::string tableAlias() const + { + return fTableAlias; + } + /** + * table alias name + */ + virtual void tableAlias(const std::string& tableAlias) + { + fTableAlias = tableAlias; + } + /** + * ASC flag + */ + inline virtual bool asc() const + { + return fAsc; + } + /** + * ASC flag + */ + inline virtual void asc(const bool asc) + { + fAsc = asc; + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual AggregateColumn* clone() const - { - return new AggregateColumn (*this); - } + /** + * fData: SQL representation of this object + */ + virtual const std::string data() const + { + return fData; + } + /** + * fData: SQL representation of this object + */ + using ReturnedColumn::data; + virtual void data(const std::string& data) + { + fData = data; + } - /** - * table alias name - */ - virtual const std::string tableAlias() const - { - return fTableAlias; - } - /** - * table alias name - */ - virtual void tableAlias (const std::string& tableAlias) - { - fTableAlias = tableAlias; - } + /** + * Overloaded stream operator + */ + virtual const std::string toString() const; - /** - * ASC flag - */ - inline virtual bool asc() const - { - return fAsc; - } - /** - * ASC flag - */ - inline virtual void asc(const bool asc) - { - fAsc = asc; - } + /** + * Serialize interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + /** + * Serialize interface + */ + virtual void unserialize(messageqcpp::ByteStream&); - /** - * fData: SQL representation of this object - */ - virtual const std::string data() const - { - return fData; - } - /** - * fData: SQL representation of this object - */ - using ReturnedColumn::data; - virtual void data(const std::string& data) - { - fData = data; - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** - * Overloaded stream operator - */ - virtual const std::string toString() const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + using ReturnedColumn::operator=; + virtual bool operator==(const AggregateColumn& t) const; - /** - * Serialize interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - /** - * Serialize interface - */ - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + using ReturnedColumn::operator!=; + virtual bool operator!=(const AggregateColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - using ReturnedColumn::operator=; - virtual bool operator==(const AggregateColumn& t) const; + /** @brief push back arg to group by column list*/ + virtual void addGroupByCol(SRCP ac) + { + fGroupByColList.push_back(ac); + } - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief push back arg to project by column list*/ + virtual void addProjectCol(SRCP ac) + { + fProjectColList.push_back(ac); + } - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - using ReturnedColumn::operator!=; - virtual bool operator!=(const AggregateColumn& t) const; + /** + * accessor + */ + virtual const ColumnList& groupByColList() const + { + return fGroupByColList; + } + /** + * accessor + */ + virtual const ColumnList& projectColList() const + { + return fProjectColList; + } - /** @brief push back arg to group by column list*/ - virtual void addGroupByCol(SRCP ac) - { - fGroupByColList.push_back(ac); - } + /** @brief constant argument for aggregate with constant */ + inline const SRCP constCol() const + { + return fConstCol; + } + /** + * accessor + */ + inline void constCol(const SRCP& constCol) + { + fConstCol = constCol; + } - /** @brief push back arg to project by column list*/ - virtual void addProjectCol(SRCP ac) - { - fProjectColList.push_back(ac); - } + /** + * convert an aggregate name to an AggOp enum + */ + static AggOp agname2num(const std::string&); - /** - * accessor - */ - virtual const ColumnList& groupByColList() const - { - return fGroupByColList; - } - /** - * accessor - */ - virtual const ColumnList& projectColList() const - { - return fProjectColList; - } + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate(); + virtual bool hasWindowFunc() + { + return false; + } - /** @brief constant argument for aggregate with constant */ - inline const SRCP constCol() const - { - return fConstCol; - } - /** - * accessor - */ - inline void constCol(const SRCP& constCol) - { - fConstCol = constCol; - } + inline const std::string timeZone() const + { + return fTimeZone; + } - /** - * convert an aggregate name to an AggOp enum - */ - static AggOp agname2num(const std::string&); + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate(); - virtual bool hasWindowFunc() - { - return false; - } + protected: + std::string fFunctionName; // deprecated field + uint8_t fAggOp; - inline const std::string timeZone () const - { - return fTimeZone; - } + /** + * ReturnedColumn objects that are the arguments to this + * function + */ + AggParms fAggParms; - inline void timeZone (const std::string& timeZone) - { - fTimeZone = timeZone; - } + /** table alias + * A string to represent table alias name which contains this column + */ + std::string fTableAlias; -protected: - std::string fFunctionName; // deprecated field - uint8_t fAggOp; + /** + * Flag to indicate asc or desc order for order by column + */ + bool fAsc; + std::string fData; + ColumnList fGroupByColList; + ColumnList fProjectColList; + SRCP fConstCol; + std::string fTimeZone; - /** - * ReturnedColumn objects that are the arguments to this - * function - */ - AggParms fAggParms; + public: + /*********************************************************** + * F&E framework * + ***********************************************************/ + /** + * F&E + */ + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getStrVal(fTimeZone); + } - /** table alias - * A string to represent table alias name which contains this column - */ - std:: string fTableAlias; + /** + * F&E + */ + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getIntVal(); + } - /** - * Flag to indicate asc or desc order for order by column - */ - bool fAsc; - std::string fData; - ColumnList fGroupByColList; - ColumnList fProjectColList; - SRCP fConstCol; - std::string fTimeZone; + /** + * F&E + */ + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getUintVal(); + } -public: - /*********************************************************** - * F&E framework * - ***********************************************************/ - /** - * F&E - */ - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getStrVal(fTimeZone); - } + /** + * F&E + */ + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getFloatVal(); + } - /** - * F&E - */ - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getIntVal(); - } + /** + * F&E + */ + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDoubleVal(); + } - /** - * F&E - */ - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getUintVal(); - } + /** + * F&E + */ + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getLongDoubleVal(); + } - /** - * F&E - */ - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getFloatVal(); - } + /** + * F&E + */ + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDecimalVal(); + } + /** + * F&E + */ + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDateIntVal(); + } + /** + * F&E + */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + /** + * F&E + */ + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDatetimeIntVal(); + } + /** + * F&E + */ + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimestampIntVal(); + } - /** - * F&E - */ - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDoubleVal(); - } - - /** - * F&E - */ - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getLongDoubleVal(); - } - - /** - * F&E - */ - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDecimalVal(); - } - /** - * F&E - */ - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDateIntVal(); - } - /** - * F&E - */ - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getTimeIntVal(); - } - /** - * F&E - */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); - } - /** - * F&E - */ - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getTimestampIntVal(); - } - - -private: - void evaluate(rowgroup::Row& row, bool& isNull); + private: + void evaluate(rowgroup::Row& row, bool& isNull); }; /** -* stream operator -*/ + * stream operator + */ std::ostream& operator<<(std::ostream& os, const AggregateColumn& rhs); /** * utility function to extract all aggregate columns from a parse tree */ void getAggCols(ParseTree* n, void* obj); -} //namespace execplan -#endif //AGGREGATECOLUMN_H - +} // namespace execplan +#endif // AGGREGATECOLUMN_H diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index e88adf899..70bf7be81 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: arithmeticcolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: arithmeticcolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ #include #include #include @@ -49,52 +49,48 @@ namespace */ void walkfn(const execplan::ParseTree* n, ostream& output) { - output << *(n->data()) << endl; + output << *(n->data()) << endl; } -} +} // namespace namespace execplan { - /** * Constructors/Destructors */ -ArithmeticColumn::ArithmeticColumn(): - ReturnedColumn(), - fExpression(0) -{} - -ArithmeticColumn::ArithmeticColumn(const string& sql, const uint32_t sessionID): - ReturnedColumn(sessionID), - fData(sql), - fExpression(0) +ArithmeticColumn::ArithmeticColumn() : ReturnedColumn(), fExpression(0) { - buildTree(); } -ArithmeticColumn::ArithmeticColumn(const ArithmeticColumn& rhs, const uint32_t sessionID): - ReturnedColumn(rhs, sessionID), - fTableAlias (rhs.fTableAlias), - fAsc (rhs.fAsc), - fData (rhs.fData), - fExpression (new ParseTree (*(rhs.expression()))) +ArithmeticColumn::ArithmeticColumn(const string& sql, const uint32_t sessionID) + : ReturnedColumn(sessionID), fData(sql), fExpression(0) { - fAlias = rhs.fAlias; - fSimpleColumnList.clear(); - fExpression->walk(getSimpleCols, &fSimpleColumnList); - fAggColumnList.clear(); - fExpression->walk(getAggCols, &fAggColumnList); - fWindowFunctionColumnList.clear(); - fExpression->walk(getWindowFunctionCols, &fWindowFunctionColumnList); + buildTree(); +} + +ArithmeticColumn::ArithmeticColumn(const ArithmeticColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID) + , fTableAlias(rhs.fTableAlias) + , fAsc(rhs.fAsc) + , fData(rhs.fData) + , fExpression(new ParseTree(*(rhs.expression()))) +{ + fAlias = rhs.fAlias; + fSimpleColumnList.clear(); + fExpression->walk(getSimpleCols, &fSimpleColumnList); + fAggColumnList.clear(); + fExpression->walk(getAggCols, &fAggColumnList); + fWindowFunctionColumnList.clear(); + fExpression->walk(getWindowFunctionCols, &fWindowFunctionColumnList); } ArithmeticColumn::~ArithmeticColumn() { - if (fExpression != NULL) - delete fExpression; + if (fExpression != NULL) + delete fExpression; - fExpression = NULL; + fExpression = NULL; } /** @@ -103,374 +99,366 @@ ArithmeticColumn::~ArithmeticColumn() void ArithmeticColumn::expression(ParseTree*& expression) { - if (fExpression != NULL) - delete fExpression; + if (fExpression != NULL) + delete fExpression; - fExpression = expression; - expression = 0; + fExpression = expression; + expression = 0; } void ArithmeticColumn::buildTree() { - CalpontSelectExecutionPlan::Parser parser; - vector tokens; - Token t; + CalpontSelectExecutionPlan::Parser parser; + vector tokens; + Token t; - string::size_type i = 0; + string::size_type i = 0; - //string fData = ReturnedColumn::data(); + // string fData = ReturnedColumn::data(); - try + try + { + while (fData[i]) { - while (fData[i]) + if (isdigit(fData[i]) || fData[i] == '.') + { + string num; + + while (isdigit(fData[i]) || fData[i] == '.') { - if (isdigit(fData[i]) || fData[i] == '.') - { - string num; - - while (isdigit(fData[i]) || fData[i] == '.') - { - num.push_back(fData[i++]); - } - - ConstantColumn* cc = new ConstantColumn(num, ConstantColumn::NUM); - t.value = cc; - - tokens.push_back(t); - continue; - } - else if (fData[i] == '+' || - fData[i] == '-' || - fData[i] == '*' || - fData[i] == '/' || - fData[i] == '^' || - fData[i] == '(' || - fData[i] == ')' ) - { - // t.is_operator now indicate the previous token type - // if prev token is operand, then this '(' is func_open - // otherwise, this '(' is open - if (fData[i] == '(' && fData[i + 1] != '-' && !t.is_operator()) - { - // open '(' - Operator* op1 = new Operator("("); - t.value = op1; - tokens.push_back(t); - - //This is not complete... we shouldn't be creating TreeNodes - string param = nextToken(++i, ')'); - - TreeNode* tn = new TreeNodeImpl(param); - t.value = tn; - - tokens.push_back(t); - - // close ')' - Operator* op2 = new Operator(")"); - t.value = op2; - tokens.push_back(t); - continue; - } - - string op; - - // Bug 319 fix. recover '^' to '||' - if (fData[i] == '^') - op = "||"; - else - op.push_back(fData[i]); - - Operator* oper = new Operator(op); - t.value = oper; - - tokens.push_back(t); - ++i; - - // t.is_operator now indicate the previous token type - // if prev token is operand, then this '(' is func_open - // otherwise, this '(' is open - // @bug 241 fix. check (-n_nationkey) case - if (fData[i] == '(' && fData[i] != '-' && !t.is_operator()) - { - //This is not complete... we shouldn't be creating TreeNodes - string param = nextToken(++i, ')'); - TreeNode* sc = new TreeNodeImpl(param); - t.value = sc; - - tokens.push_back(t); - - // close ')' - Operator* oper = new Operator(")"); - t.value = oper; - tokens.push_back(t); - } - - continue; - } - - else if (isalpha(fData[i]) || fData[i] == '_' ) - { - string identifier; - - while (isalnum(fData[i]) || - fData[i] == '_' || - fData[i] == '.' ) - { - identifier.push_back(fData[i++]); - } - - SimpleColumn* sc = new SimpleColumn(identifier, fSessionID ); - t.value = sc; - - tokens.push_back(t); - continue; - } - - else if (fData[i] == '\'') - { - string literal = nextToken(++i, '\''); - ConstantColumn* cc = new ConstantColumn (literal, ConstantColumn::LITERAL); - t.value = cc; - - tokens.push_back(t); - continue; - } - - ++i; + num.push_back(fData[i++]); } - fExpression = parser.parse(tokens.begin(), tokens.end()); - } - catch (const invalid_argument& e) - { - // clean up tokens - for (unsigned int i = 0; i < tokens.size(); i++) + ConstantColumn* cc = new ConstantColumn(num, ConstantColumn::NUM); + t.value = cc; + + tokens.push_back(t); + continue; + } + else if (fData[i] == '+' || fData[i] == '-' || fData[i] == '*' || fData[i] == '/' || fData[i] == '^' || + fData[i] == '(' || fData[i] == ')') + { + // t.is_operator now indicate the previous token type + // if prev token is operand, then this '(' is func_open + // otherwise, this '(' is open + if (fData[i] == '(' && fData[i + 1] != '-' && !t.is_operator()) { - delete tokens[i].value; - tokens[i].value = 0; + // open '(' + Operator* op1 = new Operator("("); + t.value = op1; + tokens.push_back(t); + + // This is not complete... we shouldn't be creating TreeNodes + string param = nextToken(++i, ')'); + + TreeNode* tn = new TreeNodeImpl(param); + t.value = tn; + + tokens.push_back(t); + + // close ')' + Operator* op2 = new Operator(")"); + t.value = op2; + tokens.push_back(t); + continue; } - throw runtime_error(e.what()); + string op; + + // Bug 319 fix. recover '^' to '||' + if (fData[i] == '^') + op = "||"; + else + op.push_back(fData[i]); + + Operator* oper = new Operator(op); + t.value = oper; + + tokens.push_back(t); + ++i; + + // t.is_operator now indicate the previous token type + // if prev token is operand, then this '(' is func_open + // otherwise, this '(' is open + // @bug 241 fix. check (-n_nationkey) case + if (fData[i] == '(' && fData[i] != '-' && !t.is_operator()) + { + // This is not complete... we shouldn't be creating TreeNodes + string param = nextToken(++i, ')'); + TreeNode* sc = new TreeNodeImpl(param); + t.value = sc; + + tokens.push_back(t); + + // close ')' + Operator* oper = new Operator(")"); + t.value = oper; + tokens.push_back(t); + } + + continue; + } + + else if (isalpha(fData[i]) || fData[i] == '_') + { + string identifier; + + while (isalnum(fData[i]) || fData[i] == '_' || fData[i] == '.') + { + identifier.push_back(fData[i++]); + } + + SimpleColumn* sc = new SimpleColumn(identifier, fSessionID); + t.value = sc; + + tokens.push_back(t); + continue; + } + + else if (fData[i] == '\'') + { + string literal = nextToken(++i, '\''); + ConstantColumn* cc = new ConstantColumn(literal, ConstantColumn::LITERAL); + t.value = cc; + + tokens.push_back(t); + continue; + } + + ++i; } + + fExpression = parser.parse(tokens.begin(), tokens.end()); + } + catch (const invalid_argument& e) + { + // clean up tokens + for (unsigned int i = 0; i < tokens.size(); i++) + { + delete tokens[i].value; + tokens[i].value = 0; + } + + throw runtime_error(e.what()); + } } const string ArithmeticColumn::nextToken(string::size_type& pos, char end) const { - string token; - // string fData = ReturnedColumn::data(); + string token; + // string fData = ReturnedColumn::data(); - // increment num when get '(' and decrement when get ')' - // to find the mathing ')' when num = 0 - int num = 1; + // increment num when get '(' and decrement when get ')' + // to find the mathing ')' when num = 0 + int num = 1; - for (; pos < fData.length(); ) + for (; pos < fData.length();) + { + if (end == ')') { - if (end == ')') - { - if (fData[pos] == '(') - num++; - else if (fData[pos] == ')') - num--; + if (fData[pos] == '(') + num++; + else if (fData[pos] == ')') + num--; - if (num == 0) - { - pos++; - return token; - } - } - else - { - if (fData[pos] == end) - { - pos++; - return token; - } - } - - token.push_back(fData[pos++]); + if (num == 0) + { + pos++; + return token; + } + } + else + { + if (fData[pos] == end) + { + pos++; + return token; + } } - string msg = "No "; - msg.append(1, end); - msg.append(" found in " + fData); - throw invalid_argument ( msg ); - return 0; + token.push_back(fData[pos++]); + } + + string msg = "No "; + msg.append(1, end); + msg.append(" found in " + fData); + throw invalid_argument(msg); + return 0; } ostream& operator<<(ostream& output, const ArithmeticColumn& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } const string ArithmeticColumn::toString() const { - ostringstream oss; - oss << "ArithmeticColumn: "; + ostringstream oss; + oss << "ArithmeticColumn: "; - if (fAlias.length() > 0) oss << "Alias: " << fAlias << endl; + if (fAlias.length() > 0) + oss << "Alias: " << fAlias << endl; - if (fExpression != 0) fExpression->walk(walkfn, oss); + if (fExpression != 0) + fExpression->walk(walkfn, oss); - oss << "expressionId=" << fExpressionId << endl; - oss << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; - oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; - return oss.str(); + oss << "expressionId=" << fExpressionId << endl; + oss << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; + oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; + return oss.str(); } void ArithmeticColumn::serialize(messageqcpp::ByteStream& b) const { - b << static_cast(ObjectReader::ARITHMETICCOLUMN); - ReturnedColumn::serialize(b); - ObjectReader::writeParseTree(fExpression, b); - b << fTableAlias; - b << fData; - const ByteStream::doublebyte tmp = fAsc; - b << tmp; + b << static_cast(ObjectReader::ARITHMETICCOLUMN); + ReturnedColumn::serialize(b); + ObjectReader::writeParseTree(fExpression, b); + b << fTableAlias; + b << fData; + const ByteStream::doublebyte tmp = fAsc; + b << tmp; } void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::ARITHMETICCOLUMN); - ReturnedColumn::unserialize(b); + ObjectReader::checkType(b, ObjectReader::ARITHMETICCOLUMN); + ReturnedColumn::unserialize(b); - if (fExpression != NULL) - delete fExpression; + if (fExpression != NULL) + delete fExpression; - fExpression = ObjectReader::createParseTree(b); - b >> fTableAlias; - b >> fData; - ByteStream::doublebyte tmp; - b >> tmp; - fAsc = (tmp); + fExpression = ObjectReader::createParseTree(b); + b >> fTableAlias; + b >> fData; + ByteStream::doublebyte tmp; + b >> tmp; + fAsc = (tmp); - fSimpleColumnList.clear(); - fExpression->walk(getSimpleCols, &fSimpleColumnList); - fAggColumnList.clear(); - fExpression->walk(getAggCols, &fAggColumnList); - fWindowFunctionColumnList.clear(); - fExpression->walk(getWindowFunctionCols, &fWindowFunctionColumnList); + fSimpleColumnList.clear(); + fExpression->walk(getSimpleCols, &fSimpleColumnList); + fAggColumnList.clear(); + fExpression->walk(getAggCols, &fAggColumnList); + fWindowFunctionColumnList.clear(); + fExpression->walk(getWindowFunctionCols, &fWindowFunctionColumnList); } bool ArithmeticColumn::operator==(const ArithmeticColumn& t) const { - const ReturnedColumn* rc1, *rc2; + const ReturnedColumn *rc1, *rc2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (fExpression != NULL && t.fExpression != NULL) - { - if (*fExpression != *t.fExpression) - return false; - } - else if (fExpression != NULL || t.fExpression != NULL) - return false; + if (fExpression != NULL && t.fExpression != NULL) + { + if (*fExpression != *t.fExpression) + return false; + } + else if (fExpression != NULL || t.fExpression != NULL) + return false; + if (fData != t.fData) + return false; - - if (fData != t.fData) - return false; - - return true; + return true; } bool ArithmeticColumn::operator==(const TreeNode* t) const { - const ArithmeticColumn* o; + const ArithmeticColumn* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool ArithmeticColumn::operator!=(const ArithmeticColumn& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool ArithmeticColumn::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } bool ArithmeticColumn::hasAggregate() { - if (fHasAggregate) return true; + if (fHasAggregate) + return true; - fAggColumnList.clear(); - fExpression->walk(getAggCols, &fAggColumnList); + fAggColumnList.clear(); + fExpression->walk(getAggCols, &fAggColumnList); - if (!fAggColumnList.empty()) - fHasAggregate = true; + if (!fAggColumnList.empty()) + fHasAggregate = true; - return fHasAggregate; + return fHasAggregate; } bool ArithmeticColumn::hasWindowFunc() { - fWindowFunctionColumnList.clear(); - fExpression->walk(getWindowFunctionCols, &fWindowFunctionColumnList); + fWindowFunctionColumnList.clear(); + fExpression->walk(getWindowFunctionCols, &fWindowFunctionColumnList); - if (fWindowFunctionColumnList.empty()) - return false; + if (fWindowFunctionColumnList.empty()) + return false; - return true; + return true; } void ArithmeticColumn::setDerivedTable() { - if (hasAggregate()) - { - fDerivedTable = ""; - return; - } + if (hasAggregate()) + { + fDerivedTable = ""; + return; + } - if (fExpression) - { - fExpression->setDerivedTable(); - fDerivedTable = fExpression->derivedTable(); - } + if (fExpression) + { + fExpression->setDerivedTable(); + fDerivedTable = fExpression->derivedTable(); + } } void ArithmeticColumn::replaceRealCol(std::vector& derivedColList) { - if (fExpression) - replaceRefCol(fExpression, derivedColList); + if (fExpression) + replaceRefCol(fExpression, derivedColList); } void ArithmeticColumn::setSimpleColumnList() { - fSimpleColumnList.clear(); - fExpression->walk(getSimpleCols, &fSimpleColumnList); + fSimpleColumnList.clear(); + fExpression->walk(getSimpleCols, &fSimpleColumnList); } bool ArithmeticColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) { - tan.clear(); - setSimpleColumnList(); + tan.clear(); + setSimpleColumnList(); - for (uint32_t i = 0; i < fSimpleColumnList.size(); i++) - { - CalpontSystemCatalog::TableAliasName stan(fSimpleColumnList[i]->schemaName(), - fSimpleColumnList[i]->tableName(), - fSimpleColumnList[i]->tableAlias(), - fSimpleColumnList[i]->viewName()); + for (uint32_t i = 0; i < fSimpleColumnList.size(); i++) + { + CalpontSystemCatalog::TableAliasName stan( + fSimpleColumnList[i]->schemaName(), fSimpleColumnList[i]->tableName(), + fSimpleColumnList[i]->tableAlias(), fSimpleColumnList[i]->viewName()); - if (tan.table.empty()) - tan = stan; - else if (stan != tan) - return false; - } + if (tan.table.empty()) + tan = stan; + else if (stan != tan) + return false; + } - return true; + return true; } -} // namespace - +} // namespace execplan diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index 5aeb6f435..25bd421e8 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: arithmeticcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: arithmeticcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef ARITHMETICCOLUMN_H @@ -48,240 +48,240 @@ class AggregateColumn; */ class ArithmeticColumn : public ReturnedColumn { -public: - ArithmeticColumn(); - ArithmeticColumn( const std::string& sql, const uint32_t sessionID = 0 ); - ArithmeticColumn( const ArithmeticColumn& rhs, const uint32_t sessionID = 0 ); - virtual ~ArithmeticColumn(); + public: + ArithmeticColumn(); + ArithmeticColumn(const std::string& sql, const uint32_t sessionID = 0); + ArithmeticColumn(const ArithmeticColumn& rhs, const uint32_t sessionID = 0); + virtual ~ArithmeticColumn(); - inline ParseTree* expression() const - { - return fExpression; - } + inline ParseTree* expression() const + { + return fExpression; + } - /** get table alias name - * - * get the table alias name for this arithmetic function - */ - inline const std::string& tableAlias () const - { - return fTableAlias; - } + /** get table alias name + * + * get the table alias name for this arithmetic function + */ + inline const std::string& tableAlias() const + { + return fTableAlias; + } - /** set table alias name - * - * set the table alias name for this arithmetic function - */ - inline void tableAlias (const std::string& tableAlias) - { - fTableAlias = tableAlias; - } + /** set table alias name + * + * set the table alias name for this arithmetic function + */ + inline void tableAlias(const std::string& tableAlias) + { + fTableAlias = tableAlias; + } - /** set the parse tree - * - * set the parse tree to expression. - * @note this object takes ownership of the tree. The supplied pointer is set to null. - */ - void expression (ParseTree*& expression); + /** set the parse tree + * + * set the parse tree to expression. + * @note this object takes ownership of the tree. The supplied pointer is set to null. + */ + void expression(ParseTree*& expression); - /** - * get asc flag - */ - inline bool asc() const - { - return fAsc; - } + /** + * get asc flag + */ + inline bool asc() const + { + return fAsc; + } - /** - * set asc flag - */ - inline void asc(const bool asc) - { - fAsc = asc; - } + /** + * set asc flag + */ + inline void asc(const bool asc) + { + fAsc = asc; + } - /** - * get SQL representation of this object - */ - virtual const std::string data() const - { - return fData; - } + /** + * get SQL representation of this object + */ + virtual const std::string data() const + { + return fData; + } - /** - * set SQL representation of this object - */ - virtual void data(const std::string data) - { - fData = data; - } + /** + * set SQL representation of this object + */ + virtual void data(const std::string data) + { + fData = data; + } - /** - * virtual stream method - */ - virtual const std::string toString() const; + /** + * virtual stream method + */ + virtual const std::string toString() const; - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual ArithmeticColumn* clone() const - { - return new ArithmeticColumn (*this); - } + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual ArithmeticColumn* clone() const + { + return new ArithmeticColumn(*this); + } - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const ArithmeticColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const ArithmeticColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const ArithmeticColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const ArithmeticColumn& t) const; - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate(); - virtual bool hasWindowFunc(); + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate(); + virtual bool hasWindowFunc(); - virtual void setDerivedTable(); - virtual void replaceRealCol(std::vector&); - virtual const std::vector& simpleColumnList() const - { - return fSimpleColumnList; - } - virtual void setSimpleColumnList(); + virtual void setDerivedTable(); + virtual void replaceRealCol(std::vector&); + virtual const std::vector& simpleColumnList() const + { + return fSimpleColumnList; + } + virtual void setSimpleColumnList(); - /** - * Return the table that the column arguments belong to. - * - * @return tablename, if all arguments belong to one table - * empty string "", if multiple tables are involved in this func - */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan); + /** + * Return the table that the column arguments belong to. + * + * @return tablename, if all arguments belong to one table + * empty string "", if multiple tables are involved in this func + */ + virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan); -private: - std::string fTableAlias; // table alias for this column - bool fAsc; // asc flag for order by column - std::string fData; + private: + std::string fTableAlias; // table alias for this column + bool fAsc; // asc flag for order by column + std::string fData; - /** build expression tree - * - * this function is called by the constructor. the incomming - * sql string is parsed and tokenized and built into a - * parse tree. - */ - void buildTree(); + /** build expression tree + * + * this function is called by the constructor. the incomming + * sql string is parsed and tokenized and built into a + * parse tree. + */ + void buildTree(); - /** get next token from expression - * - * this is a util function used by buildTree(). next token string - * is retrived from expression from curPos, until end char. - * return the retrived token. curPos reference is updated to the - * new position after end char - */ - const std::string nextToken(std::string::size_type& curPos, char end) const; + /** get next token from expression + * + * this is a util function used by buildTree(). next token string + * is retrived from expression from curPos, until end char. + * return the retrived token. curPos reference is updated to the + * new position after end char + */ + const std::string nextToken(std::string::size_type& curPos, char end) const; - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getStrVal(row, isNull); - } + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getStrVal(row, isNull); + } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getIntVal(row, isNull); - } + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getIntVal(row, isNull); + } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getUintVal(row, isNull); - } + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getUintVal(row, isNull); + } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getFloatVal(row, isNull); - } + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getFloatVal(row, isNull); + } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getDoubleVal(row, isNull); - } + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getDoubleVal(row, isNull); + } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getLongDoubleVal(row, isNull); - } + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getLongDoubleVal(row, isNull); + } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getDecimalVal(row, isNull); - } + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getDecimalVal(row, isNull); + } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getDateIntVal(row, isNull); - } + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getDateIntVal(row, isNull); + } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getDatetimeIntVal(row, isNull); - } + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getDatetimeIntVal(row, isNull); + } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getTimestampIntVal(row, isNull); - } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimestampIntVal(row, isNull); + } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getTimeIntVal(row, isNull); - } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimeIntVal(row, isNull); + } - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) - { - return fExpression->getBoolVal(row, isNull); - } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getBoolVal(row, isNull); + } - -private: - ParseTree* fExpression; - using TreeNode::evaluate; - void evaluate(rowgroup::Row& row) {} + private: + ParseTree* fExpression; + using TreeNode::evaluate; + void evaluate(rowgroup::Row& row) + { + } }; /** -* ostream operator -*/ + * ostream operator + */ std::ostream& operator<<(std::ostream& output, const ArithmeticColumn& rhs); -} -#endif //ARITHMETICCOLUMN_H - +} // namespace execplan +#endif // ARITHMETICCOLUMN_H diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index 26fc4e1fe..f3e235f14 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: arithmeticoperator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: arithmeticoperator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include "bytestream.h" @@ -30,28 +30,24 @@ using namespace std; namespace execplan { - /** * Constructors/Destructors */ -ArithmeticOperator::ArithmeticOperator() : Operator(), - fDecimalOverflowCheck(false) +ArithmeticOperator::ArithmeticOperator() : Operator(), fDecimalOverflowCheck(false) { } -ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(operatorName), - fDecimalOverflowCheck(false) +ArithmeticOperator::ArithmeticOperator(const string& operatorName) + : Operator(operatorName), fDecimalOverflowCheck(false) { } -ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs): - Operator(rhs), - fTimeZone(rhs.timeZone()), - fDecimalOverflowCheck(rhs.getOverflowCheck()) +ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs) + : Operator(rhs), fTimeZone(rhs.timeZone()), fDecimalOverflowCheck(rhs.getOverflowCheck()) { } -ArithmeticOperator:: ~ArithmeticOperator() +ArithmeticOperator::~ArithmeticOperator() { } @@ -64,10 +60,10 @@ ArithmeticOperator:: ~ArithmeticOperator() */ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) { - output << rhs.toString(); - output << "opType=" << rhs.operationType().colDataType << endl; - output << "decimalOverflowCheck=" << rhs.getOverflowCheck() << endl; - return output; + output << rhs.toString(); + output << "opType=" << rhs.operationType().colDataType << endl; + output << "decimalOverflowCheck=" << rhs.getOverflowCheck() << endl; + return output; } /** @@ -75,72 +71,71 @@ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) */ void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::ARITHMETICOPERATOR; - b << fTimeZone; - const messageqcpp::ByteStream::byte tmp = fDecimalOverflowCheck; - b << tmp; - Operator::serialize(b); + b << (ObjectReader::id_t)ObjectReader::ARITHMETICOPERATOR; + b << fTimeZone; + const messageqcpp::ByteStream::byte tmp = fDecimalOverflowCheck; + b << tmp; + Operator::serialize(b); } void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); - b >> fTimeZone; - messageqcpp::ByteStream::byte tmp; - b >> tmp; - fDecimalOverflowCheck = tmp; - Operator::unserialize(b); + ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); + b >> fTimeZone; + messageqcpp::ByteStream::byte tmp; + b >> tmp; + fDecimalOverflowCheck = tmp; + Operator::unserialize(b); } bool ArithmeticOperator::operator==(const ArithmeticOperator& t) const { - if (data() != t.data()) - return false; + if (data() != t.data()) + return false; - if (timeZone() != t.timeZone()) - return false; + if (timeZone() != t.timeZone()) + return false; - return true; + return true; } bool ArithmeticOperator::operator==(const TreeNode* t) const { - const ArithmeticOperator* o; + const ArithmeticOperator* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool ArithmeticOperator::operator!=(const ArithmeticOperator& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool ArithmeticOperator::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } void ArithmeticOperator::adjustResultType(const CalpontSystemCatalog::ColType& m) { - if (m.colDataType != CalpontSystemCatalog::DECIMAL && - m.colDataType != CalpontSystemCatalog::UDECIMAL) - { - fResultType = m; - } - else - { - CalpontSystemCatalog::ColType n; - n.colDataType = CalpontSystemCatalog::LONGDOUBLE; - n.scale = m.scale; // @bug5736, save the original decimal scale - n.precision = -1; // @bug5736, indicate this double is for decimal math - n.colWidth = sizeof(long double); - fResultType = n; - } + if (m.colDataType != CalpontSystemCatalog::DECIMAL && m.colDataType != CalpontSystemCatalog::UDECIMAL) + { + fResultType = m; + } + else + { + CalpontSystemCatalog::ColType n; + n.colDataType = CalpontSystemCatalog::LONGDOUBLE; + n.scale = m.scale; // @bug5736, save the original decimal scale + n.precision = -1; // @bug5736, indicate this double is for decimal math + n.colWidth = sizeof(long double); + fResultType = n; + } } -} // namespace +} // namespace execplan diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 163151f7a..899818ad6 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* -***********************************************************************/ + * $Id$ + * + * + ***********************************************************************/ /** @file */ #ifndef ARITHMETICOPERATOR_H @@ -40,422 +40,398 @@ class ByteStream; namespace execplan { - class ArithmeticOperator : public Operator { -using cscType = execplan::CalpontSystemCatalog::ColType; + using cscType = execplan::CalpontSystemCatalog::ColType; -public: - ArithmeticOperator(); - ArithmeticOperator(const std::string& operatorName); - ArithmeticOperator(const ArithmeticOperator& rhs); + public: + ArithmeticOperator(); + ArithmeticOperator(const std::string& operatorName); + ArithmeticOperator(const ArithmeticOperator& rhs); - virtual ~ArithmeticOperator(); + virtual ~ArithmeticOperator(); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual ArithmeticOperator* clone() const + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual ArithmeticOperator* clone() const + { + return new ArithmeticOperator(*this); + } + + inline const std::string& timeZone() const + { + return fTimeZone; + } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const ArithmeticOperator& t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const ArithmeticOperator& t) const; + + /*********************************************************** + * F&E framework * + ***********************************************************/ + using Operator::evaluate; + inline virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop); + + using Operator::getStrVal; + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getStrVal(fTimeZone); + } + using Operator::getIntVal; + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getIntVal(); + } + using Operator::getUintVal; + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getUintVal(); + } + using Operator::getFloatVal; + virtual float getFloatVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getFloatVal(); + } + using Operator::getDoubleVal; + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getDoubleVal(); + } + using Operator::getLongDoubleVal; + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getLongDoubleVal(); + } + using Operator::getDecimalVal; + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + + // @bug5736, double type with precision -1 indicates that this type is for decimal math, + // the original decimal scale is stored in scale field, which is no use for double. + if (fResultType.colDataType == CalpontSystemCatalog::DOUBLE && fResultType.precision == -1) { - return new ArithmeticOperator (*this); + IDB_Decimal rv; + rv.scale = fResultType.scale; + rv.precision = 15; + rv.value = (int64_t)(TreeNode::getDoubleVal() * IDB_pow[rv.scale]); + + return rv; } - inline const std::string& timeZone() const - { - return fTimeZone; - } - inline void timeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } + return TreeNode::getDecimalVal(); + } + using Operator::getDateIntVal; + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getDateIntVal(); + } + using Operator::getDatetimeIntVal; + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getDatetimeIntVal(); + } + using Operator::getTimestampIntVal; + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimestampIntVal(); + } + using Operator::getTimeIntVal; + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimeIntVal(); + } + using Operator::getBoolVal; + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getBoolVal(); + } + void adjustResultType(const CalpontSystemCatalog::ColType& m); + inline bool getOverflowCheck() const + { + return fDecimalOverflowCheck; + } + inline void setOverflowCheck(bool check) + { + fDecimalOverflowCheck = check; + } - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const ArithmeticOperator& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const ArithmeticOperator& t) const; - - /*********************************************************** - * F&E framework * - ***********************************************************/ - using Operator::evaluate; - inline virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop); - - using Operator::getStrVal; - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getStrVal(fTimeZone); - } - using Operator::getIntVal; - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getIntVal(); - } - using Operator::getUintVal; - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getUintVal(); - } - using Operator::getFloatVal; - virtual float getFloatVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getFloatVal(); - } - using Operator::getDoubleVal; - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getDoubleVal(); - } - using Operator::getLongDoubleVal; - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getLongDoubleVal(); - } - using Operator::getDecimalVal; - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - - // @bug5736, double type with precision -1 indicates that this type is for decimal math, - // the original decimal scale is stored in scale field, which is no use for double. - if (fResultType.colDataType == CalpontSystemCatalog::DOUBLE && fResultType.precision == -1) - { - IDB_Decimal rv; - rv.scale = fResultType.scale; - rv.precision = 15; - rv.value = (int64_t)(TreeNode::getDoubleVal() * IDB_pow[rv.scale]); - - return rv; - } - - return TreeNode::getDecimalVal(); - } - using Operator::getDateIntVal; - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getDateIntVal(); - } - using Operator::getDatetimeIntVal; - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getDatetimeIntVal(); - } - using Operator::getTimestampIntVal; - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getTimestampIntVal(); - } - using Operator::getTimeIntVal; - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getTimeIntVal(); - } - using Operator::getBoolVal; - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - evaluate(row, isNull, lop, rop); - return TreeNode::getBoolVal(); - } - void adjustResultType(const CalpontSystemCatalog::ColType& m); - inline bool getOverflowCheck() const - { - return fDecimalOverflowCheck; - } - inline void setOverflowCheck(bool check) - { - fDecimalOverflowCheck = check; - } - -private: - template - inline result_t execute(result_t op1, result_t op2, bool& isNull); - inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); - std::string fTimeZone; - bool fDecimalOverflowCheck; + private: + template + inline result_t execute(result_t op1, result_t op2, bool& isNull); + inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); + std::string fTimeZone; + bool fDecimalOverflowCheck; }; #include "parsetree.h" inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { - // fOpType should have already been set on the connector during parsing - switch (fOperationType.colDataType) + // fOpType should have already been set on the connector during parsing + switch (fOperationType.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::TINYINT: + fResult.intVal = execute(lop->getIntVal(row, isNull), rop->getIntVal(row, isNull), isNull); + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UTINYINT: + fResult.uintVal = execute(lop->getUintVal(row, isNull), rop->getUintVal(row, isNull), isNull); + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::UFLOAT: + fResult.doubleVal = execute(lop->getDoubleVal(row, isNull), rop->getDoubleVal(row, isNull), isNull); + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + fResult.longDoubleVal = + execute(lop->getLongDoubleVal(row, isNull), rop->getLongDoubleVal(row, isNull), isNull); + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); + break; + + default: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::TINYINT: - fResult.intVal = execute(lop->getIntVal(row, isNull), rop->getIntVal(row, isNull), isNull); - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UTINYINT: - fResult.uintVal = execute(lop->getUintVal(row, isNull), rop->getUintVal(row, isNull), isNull); - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::UFLOAT: - fResult.doubleVal = execute(lop->getDoubleVal(row, isNull), rop->getDoubleVal(row, isNull), isNull); - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - fResult.longDoubleVal = execute(lop->getLongDoubleVal(row, isNull), rop->getLongDoubleVal(row, isNull), isNull); - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); - break; - - default: - { - std::ostringstream oss; - oss << "invalid arithmetic operand type: " << fOperationType.colDataType; - throw logging::InvalidArgumentExcept(oss.str()); - } + std::ostringstream oss; + oss << "invalid arithmetic operand type: " << fOperationType.colDataType; + throw logging::InvalidArgumentExcept(oss.str()); } + } } template inline result_t ArithmeticOperator::execute(result_t op1, result_t op2, bool& isNull) { - switch (fOp) + switch (fOp) + { + case OP_ADD: return op1 + op2; + + case OP_SUB: return op1 - op2; + + case OP_MUL: return op1 * op2; + + case OP_DIV: + if (op2) + return op1 / op2; + else + isNull = true; + + return 0; + + default: { - case OP_ADD: - return op1 + op2; - - case OP_SUB: - return op1 - op2; - - case OP_MUL: - return op1 * op2; - - case OP_DIV: - if (op2) - return op1 / op2; - else - isNull = true; - - return 0; - - default: - { - std::ostringstream oss; - oss << "invalid arithmetic operation: " << fOp; - throw logging::InvalidOperationExcept(oss.str()); - } + std::ostringstream oss; + oss << "invalid arithmetic operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); } + } } inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull) { - switch (fOp) - { - case OP_ADD: - if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) - { - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::addition( - op1, op2, result); - } - else - { - datatypes::Decimal::addition( - op1, op2, result); - } - } - else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) - { - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::addition( - op1, op2, result); - } - else - { - datatypes::Decimal::addition( - op1, op2, result); - } - } - else - { - throw logging::InvalidArgumentExcept( - "Unexpected result width"); - } - break; - - case OP_SUB: - if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) - { - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::subtraction( - op1, op2, result); - } - else - { - datatypes::Decimal::subtraction( - op1, op2, result); - } - } - else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) - { - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::subtraction( - op1, op2, result); - } - else - { - datatypes::Decimal::subtraction( - op1, op2, result); - } - } - else - { - throw logging::InvalidArgumentExcept( - "Unexpected result width"); - } - break; - - case OP_MUL: - if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) - { - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::multiplication( - op1, op2, result); - } - else - { - datatypes::Decimal::multiplication( - op1, op2, result); - } - } - else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) - { - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::multiplication( - op1, op2, result); - } - else - { - datatypes::Decimal::multiplication( - op1, op2, result); - } - } - else - { - throw logging::InvalidArgumentExcept( - "Unexpected result width"); - } - break; - - case OP_DIV: - if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) - { - if ((datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.s128Value == 0) - || (!datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.value == 0)) - { - isNull = true; - break; - } - - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::division( - op1, op2, result); - } - else - { - datatypes::Decimal::division( - op1, op2, result); - } - } - else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) - { - if (op2.value == 0) - { - isNull = true; - break; - } - - if (LIKELY(!fDecimalOverflowCheck)) - { - datatypes::Decimal::division( - op1, op2, result); - } - else - { - datatypes::Decimal::division( - op1, op2, result); - } - } - else - { - throw logging::InvalidArgumentExcept( - "Unexpected result width"); - } - break; - - default: + switch (fOp) + { + case OP_ADD: + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) { - std::ostringstream oss; - oss << "invalid arithmetic operation: " << fOp; - throw logging::InvalidOperationExcept(oss.str()); + datatypes::Decimal::addition(op1, op2, result); } + else + { + datatypes::Decimal::addition(op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::addition(op1, op2, result); + } + else + { + datatypes::Decimal::addition(op1, op2, result); + } + } + else + { + throw logging::InvalidArgumentExcept("Unexpected result width"); + } + break; + + case OP_SUB: + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::subtraction(op1, op2, result); + } + else + { + datatypes::Decimal::subtraction(op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::subtraction(op1, op2, result); + } + else + { + datatypes::Decimal::subtraction(op1, op2, result); + } + } + else + { + throw logging::InvalidArgumentExcept("Unexpected result width"); + } + break; + + case OP_MUL: + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::multiplication(op1, op2, result); + } + else + { + datatypes::Decimal::multiplication(op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::multiplication(op1, op2, result); + } + else + { + datatypes::Decimal::multiplication(op1, op2, result); + } + } + else + { + throw logging::InvalidArgumentExcept("Unexpected result width"); + } + break; + + case OP_DIV: + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) + { + if ((datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.s128Value == 0) || + (!datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.value == 0)) + { + isNull = true; + break; + } + + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::division(op1, op2, result); + } + else + { + datatypes::Decimal::division(op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (op2.value == 0) + { + isNull = true; + break; + } + + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::division(op1, op2, result); + } + else + { + datatypes::Decimal::division(op1, op2, result); + } + } + else + { + throw logging::InvalidArgumentExcept("Unexpected result width"); + } + break; + + default: + { + std::ostringstream oss; + oss << "invalid arithmetic operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); } + } } std::ostream& operator<<(std::ostream& os, const ArithmeticOperator& rhs); -} +} // namespace execplan #endif - diff --git a/dbcon/execplan/blocksize.h b/dbcon/execplan/blocksize.h index f61cee5c4..cc2286479 100644 --- a/dbcon/execplan/blocksize.h +++ b/dbcon/execplan/blocksize.h @@ -29,4 +29,3 @@ const uint64_t BLOCK_SIZE = 8192; const uint64_t LOGICAL_BLOCK_RIDS = BLOCK_SIZE; #endif - diff --git a/dbcon/execplan/btdriver.cpp b/dbcon/execplan/btdriver.cpp index 21ac751cd..6ef9a0eeb 100644 --- a/dbcon/execplan/btdriver.cpp +++ b/dbcon/execplan/btdriver.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,191 +47,180 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_1); - CPPUNIT_TEST( selectExecutionPlan_1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_1() - { - cout << "SQL: select r_regionkey from region, nation where n_regionkey = r_regionkey and n_regionkey = 2;" << endl; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList colList; - ParseTree* filterList; + cout << endl; + } - // returned columns - SimpleColumn r_regionkey("tpch.region.r_regionkey"); - SimpleColumn n_regionkey("tpch.nation.n_regionkey"); - SimpleColumn p_partkey("tpch.part.p_partkey"); - SimpleColumn n_name("tpch.nation.n_name"); - SimpleColumn c_custkey("tpch.customer.c_custkey"); + void setUp() + { + } - colList.push_back(new SimpleColumn(r_regionkey)); + void tearDown() + { + } - // filters - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; + void selectExecutionPlan_1() + { + cout << "SQL: select r_regionkey from region, nation where n_regionkey = r_regionkey and n_regionkey = 2;" + << endl; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList colList; + ParseTree* filterList; - //tokens.push_back(Token(new Operator("("))); - //tokens.push_back(Token(new Operator(")"))); + // returned columns + SimpleColumn r_regionkey("tpch.region.r_regionkey"); + SimpleColumn n_regionkey("tpch.nation.n_regionkey"); + SimpleColumn p_partkey("tpch.part.p_partkey"); + SimpleColumn n_name("tpch.nation.n_name"); + SimpleColumn c_custkey("tpch.customer.c_custkey"); - tokens.push_back(Token(new SimpleFilter(new Operator("="), - new SimpleColumn(r_regionkey), - new SimpleColumn(n_regionkey)))); + colList.push_back(new SimpleColumn(r_regionkey)); - tokens.push_back(Token(new Operator("and"))); + // filters + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; - tokens.push_back(Token(new SimpleFilter(new Operator("="), - new SimpleColumn(r_regionkey), - new SimpleColumn(c_custkey)))); + // tokens.push_back(Token(new Operator("("))); + // tokens.push_back(Token(new Operator(")"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token( + new SimpleFilter(new Operator("="), new SimpleColumn(r_regionkey), new SimpleColumn(n_regionkey)))); - tokens.push_back(Token(new Operator("("))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new SimpleFilter(new Operator("="), - new SimpleColumn(n_regionkey), - new ConstantColumn("779")))); + tokens.push_back(Token( + new SimpleFilter(new Operator("="), new SimpleColumn(r_regionkey), new SimpleColumn(c_custkey)))); - tokens.push_back(Token(new Operator("or"))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new SimpleFilter(new Operator("!="), - new SimpleColumn(n_name), - new ConstantColumn("'ASIA'")))); + tokens.push_back(Token(new Operator("("))); - tokens.push_back(Token(new Operator(")"))); + tokens.push_back( + Token(new SimpleFilter(new Operator("="), new SimpleColumn(n_regionkey), new ConstantColumn("779")))); - tokens.push_back(Token(new Operator ("and"))); + tokens.push_back(Token(new Operator("or"))); - tokens.push_back(Token(new Operator("("))); + tokens.push_back( + Token(new SimpleFilter(new Operator("!="), new SimpleColumn(n_name), new ConstantColumn("'ASIA'")))); - tokens.push_back(Token(new SimpleFilter(new Operator("<"), - new SimpleColumn(n_regionkey), - new ConstantColumn("77")))); + tokens.push_back(Token(new Operator(")"))); - tokens.push_back(Token(new Operator("or"))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new SimpleFilter(new Operator(">"), - new SimpleColumn(p_partkey), - new ConstantColumn("7007")))); + tokens.push_back(Token(new Operator("("))); - tokens.push_back(Token(new Operator(")"))); + tokens.push_back( + Token(new SimpleFilter(new Operator("<"), new SimpleColumn(n_regionkey), new ConstantColumn("77")))); - filterList = parser.parse(tokens.begin(), tokens.end()); + tokens.push_back(Token(new Operator("or"))); - // draw filterList tree - filterList->drawTree("selectExecutionPlan_1.dot"); + tokens.push_back( + Token(new SimpleFilter(new Operator(">"), new SimpleColumn(p_partkey), new ConstantColumn("7007")))); - // calpont execution plan - csep.returnedCols (colList); - csep.filters (filterList); - cout << "\nCalpont Execution Plan:" << endl; - cout << csep << endl; - cout << " --- end of test 1 ---" << endl; + tokens.push_back(Token(new Operator(")"))); - filterList->walk(walkfnString); + filterList = parser.parse(tokens.begin(), tokens.end()); - } + // draw filterList tree + filterList->drawTree("selectExecutionPlan_1.dot"); + // calpont execution plan + csep.returnedCols(colList); + csep.filters(filterList); + cout << "\nCalpont Execution Plan:" << endl; + cout << csep << endl; + cout << " --- end of test 1 ---" << endl; + + filterList->walk(walkfnString); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/calpontexecutionplan.cpp b/dbcon/execplan/calpontexecutionplan.cpp index a76439917..c6bb614b1 100644 --- a/dbcon/execplan/calpontexecutionplan.cpp +++ b/dbcon/execplan/calpontexecutionplan.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: calpontexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "calpontexecutionplan.h" @@ -39,5 +39,4 @@ CalpontExecutionPlan::~CalpontExecutionPlan() /** * Methods */ -} - +} // namespace execplan diff --git a/dbcon/execplan/calpontexecutionplan.h b/dbcon/execplan/calpontexecutionplan.h index 213962ef4..49c86304f 100644 --- a/dbcon/execplan/calpontexecutionplan.h +++ b/dbcon/execplan/calpontexecutionplan.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: calpontexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef CALPONTEXECUTIONPLAN_H @@ -46,121 +46,119 @@ namespace execplan */ class CalpontExecutionPlan { - /** - * Public stuff - */ -public: + /** + * Public stuff + */ + public: + /** + * Flags that can be passed to caltraceon(). + */ + enum TRACE_FLAGS + { + TRACE_NONE = 0x0000, /*!< No tracing */ + TRACE_LOG = 0x0001, /*!< Full return of rows, extra debug about query in log */ + TRACE_NO_ROWS1 = 0x0002, /*!< Same as above, but rows not given to OCI layer */ + TRACE_NO_ROWS2 = 0x0004, /*!< Same as above, but rows not converted from stream */ + TRACE_NO_ROWS3 = 0x0008, /*!< Same as above, but rows not sent to DM from UM */ + TRACE_NO_ROWS4 = 0x0010, /*!< Same as above, but rows not sent to DeliveryStep */ + TRACE_LBIDS = 0x0020, /*!< Enable LBID tracing in PrimProc */ + TRACE_PLAN_ONLY = 0x0040, /*!< Only generate a serialized CSEP */ + PM_PROFILE = 0x0080, /*!< Enable PM profiling in PrimProc */ + IGNORE_CP = 0x0100, /*!< Ignore casual partitioning metadata */ + WRITE_TO_FILE = 0x0200, /*!< writes table rows out to a file from the Oracle connector */ + NOWRITE_TO_FILE = 0x0400, /*!< does not write table rows out to a file from the Oracle connector */ + TRACE_DISKIO_UM = 0x0800, /*!< Enable UM disk I/O logging */ + TRACE_RESRCMGR = 0x1000, /*!< Trace Resource Manager Usage */ + TRACE_TUPLE_AUTOSWITCH = 0x4000, /*!< Enable MySQL tuple-to-table auto switch */ + TRACE_TUPLE_OFF = 0x8000, /*!< Enable MySQL table interface */ + }; - /** - * Flags that can be passed to caltraceon(). - */ - enum TRACE_FLAGS - { - TRACE_NONE = 0x0000, /*!< No tracing */ - TRACE_LOG = 0x0001, /*!< Full return of rows, extra debug about query in log */ - TRACE_NO_ROWS1 = 0x0002, /*!< Same as above, but rows not given to OCI layer */ - TRACE_NO_ROWS2 = 0x0004, /*!< Same as above, but rows not converted from stream */ - TRACE_NO_ROWS3 = 0x0008, /*!< Same as above, but rows not sent to DM from UM */ - TRACE_NO_ROWS4 = 0x0010, /*!< Same as above, but rows not sent to DeliveryStep */ - TRACE_LBIDS = 0x0020, /*!< Enable LBID tracing in PrimProc */ - TRACE_PLAN_ONLY = 0x0040, /*!< Only generate a serialized CSEP */ - PM_PROFILE = 0x0080, /*!< Enable PM profiling in PrimProc */ - IGNORE_CP = 0x0100, /*!< Ignore casual partitioning metadata */ - WRITE_TO_FILE = 0x0200, /*!< writes table rows out to a file from the Oracle connector */ - NOWRITE_TO_FILE = 0x0400, /*!< does not write table rows out to a file from the Oracle connector */ - TRACE_DISKIO_UM = 0x0800, /*!< Enable UM disk I/O logging */ - TRACE_RESRCMGR = 0x1000, /*!< Trace Resource Manager Usage */ - TRACE_TUPLE_AUTOSWITCH = 0x4000, /*!< Enable MySQL tuple-to-table auto switch */ - TRACE_TUPLE_OFF = 0x8000, /*!< Enable MySQL table interface */ - }; + /** + * Constructors + */ + CalpontExecutionPlan(); + /** + * Destructors + */ + virtual ~CalpontExecutionPlan(); + /** + * Accessor Methods + */ + /** + * Operations + */ + /* + * The serialization interface + */ + /** @brief Convert *this to a stream of bytes + * + * Convert *this to a stream of bytes. + * @param b The ByteStream to write the bytes to. + */ + virtual void serialize(messageqcpp::ByteStream& b) const = 0; - /** - * Constructors - */ - CalpontExecutionPlan(); - /** - * Destructors - */ - virtual ~CalpontExecutionPlan(); - /** - * Accessor Methods - */ - /** - * Operations - */ - /* - * The serialization interface - */ - /** @brief Convert *this to a stream of bytes - * - * Convert *this to a stream of bytes. - * @param b The ByteStream to write the bytes to. - */ - virtual void serialize(messageqcpp::ByteStream& b) const = 0; + /** @brief Construct a CalpontExecutionPlan from a stream of bytes + * + * Construct a CalpontExecutionPlan from a stream of bytes. + * @param b The ByteStream to read from. + */ + virtual void unserialize(messageqcpp::ByteStream& b) = 0; - /** @brief Construct a CalpontExecutionPlan from a stream of bytes - * - * Construct a CalpontExecutionPlan from a stream of bytes. - * @param b The ByteStream to read from. - */ - virtual void unserialize(messageqcpp::ByteStream& b) = 0; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const CalpontExecutionPlan* t) const = 0; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const CalpontExecutionPlan* t) const = 0; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const CalpontExecutionPlan* t) const = 0; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const CalpontExecutionPlan* t) const = 0; - - /** - * Protected stuff - */ -protected: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Private stuff - */ -private: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ + /** + * Protected stuff + */ + protected: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Private stuff + */ + private: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ }; typedef boost::shared_ptr SCEP; -} -#endif //CALPONTEXECUTIONPLAN_H - +} // namespace execplan +#endif // CALPONTEXECUTIONPLAN_H diff --git a/dbcon/execplan/calpontexecutionplanfactory.cpp b/dbcon/execplan/calpontexecutionplanfactory.cpp index 800dfd5f3..06875582f 100644 --- a/dbcon/execplan/calpontexecutionplanfactory.cpp +++ b/dbcon/execplan/calpontexecutionplanfactory.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontexecutionplanfactory.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: calpontexecutionplanfactory.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "calpontexecutionplanfactory.h" #include "calpontexecutionplan.h" @@ -28,10 +28,10 @@ namespace execplan { - -//CalpontExecutionPlan* CalpontExecutionPlanFactory::makeCalpontExecutionPlan(const VendorExecutionPlan* vplan) +// CalpontExecutionPlan* CalpontExecutionPlanFactory::makeCalpontExecutionPlan(const VendorExecutionPlan* +// vplan) //{ // return new CalpontSelectExecutionPlan(); //} -} //namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/calpontexecutionplanfactory.h b/dbcon/execplan/calpontexecutionplanfactory.h index d255ca24b..f5e0161be 100644 --- a/dbcon/execplan/calpontexecutionplanfactory.h +++ b/dbcon/execplan/calpontexecutionplanfactory.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontexecutionplanfactory.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: calpontexecutionplanfactory.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef CALPONTEXECUTIONPLANFACTORY_H @@ -30,15 +30,11 @@ namespace execplan { - class CalpontExecutionPlanFactory { - -public: - //static CalpontExecutionPlan* makeCalpontExecutionPlan(const VendorExecutionPlan* vplan); - + public: + // static CalpontExecutionPlan* makeCalpontExecutionPlan(const VendorExecutionPlan* vplan); }; -} //namespace execplan -#endif //CALPONTEXECUTIONPLANFACTORY_H - +} // namespace execplan +#endif // CALPONTEXECUTIONPLANFACTORY_H diff --git a/dbcon/execplan/calpontselectexecutionplan.cpp b/dbcon/execplan/calpontselectexecutionplan.cpp index f8257c0b9..de4291c0e 100644 --- a/dbcon/execplan/calpontselectexecutionplan.cpp +++ b/dbcon/execplan/calpontselectexecutionplan.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontselectexecutionplan.cpp 9576 2013-05-29 21:02:11Z zzhu $ -* -* -***********************************************************************/ + * $Id: calpontselectexecutionplan.cpp 9576 2013-05-29 21:02:11Z zzhu $ + * + * + ***********************************************************************/ #include #include using namespace std; @@ -41,837 +41,813 @@ using namespace querytele; namespace { - -template struct deleter : public unary_function +template +struct deleter : public unary_function { - void operator()(T& x) - { - delete x; - x = 0; - } + void operator()(T& x) + { + delete x; + x = 0; + } }; -} +} // namespace namespace execplan { - /** Static */ CalpontSelectExecutionPlan::ColumnMap CalpontSelectExecutionPlan::fColMap; /** * Constructors/Destructors */ -CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(const int location): - fLocalQuery (GLOBAL_QUERY), - fFilters (0), - fHaving (0), - fLocation (location), - fDependent (false), - fTxnID(-1), - fTraceFlags(TRACE_NONE), - fStatementID(0), - fDistinct(false), - fOverrideLargeSideEstimate(false), - fDistinctUnionNum(0), - fSubType(MAIN_SELECT), - fLimitStart(0), - fLimitNum(-1), - fHasOrderBy(false), - fStringScanThreshold(ULONG_MAX), - fQueryType(SELECT), - fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL), - fStringTableThreshold(20), - fOrderByThreads(1), - fDJSSmallSideLimit(0), - fDJSLargeSideLimit(0), - fDJSPartitionSize(100 * 1024 * 1024), // 100MB mem usage for disk based join, - fUMMemLimit(numeric_limits::max()), - fIsDML(false) +CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(const int location) + : fLocalQuery(GLOBAL_QUERY) + , fFilters(0) + , fHaving(0) + , fLocation(location) + , fDependent(false) + , fTxnID(-1) + , fTraceFlags(TRACE_NONE) + , fStatementID(0) + , fDistinct(false) + , fOverrideLargeSideEstimate(false) + , fDistinctUnionNum(0) + , fSubType(MAIN_SELECT) + , fLimitStart(0) + , fLimitNum(-1) + , fHasOrderBy(false) + , fStringScanThreshold(ULONG_MAX) + , fQueryType(SELECT) + , fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL) + , fStringTableThreshold(20) + , fOrderByThreads(1) + , fDJSSmallSideLimit(0) + , fDJSLargeSideLimit(0) + , fDJSPartitionSize(100 * 1024 * 1024) + , // 100MB mem usage for disk based join, + fUMMemLimit(numeric_limits::max()) + , fIsDML(false) { - fUuid = QueryTeleClient::genUUID(); + fUuid = QueryTeleClient::genUUID(); } CalpontSelectExecutionPlan::CalpontSelectExecutionPlan( - const ReturnedColumnList& returnedCols, - ParseTree* filters, - const SelectList& subSelects, - const GroupByColumnList& groupByCols, - ParseTree* having, - const OrderByColumnList& orderByCols, - const string alias, - const int location, - const bool dependent) : - fLocalQuery (GLOBAL_QUERY), - fReturnedCols (returnedCols), - fFilters (filters), - fSubSelects (subSelects), - fGroupByCols (groupByCols), - fHaving (having), - fOrderByCols (orderByCols), - fTableAlias (alias), - fLocation (location), - fDependent (dependent), - fTxnID(-1), - fTraceFlags(TRACE_NONE), - fStatementID(0), - fDistinct(false), - fOverrideLargeSideEstimate(false), - fDistinctUnionNum(0), - fSubType(MAIN_SELECT), - fLimitStart(0), - fLimitNum(-1), - fHasOrderBy(false), - fStringScanThreshold(ULONG_MAX), - fQueryType(SELECT), - fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL), - fStringTableThreshold(20), - fOrderByThreads(1), - fDJSSmallSideLimit(0), - fDJSLargeSideLimit(0), - fDJSPartitionSize(100 * 1024 * 1024), // 100MB mem usage for disk based join - fUMMemLimit(numeric_limits::max()), - fIsDML(false) + const ReturnedColumnList& returnedCols, ParseTree* filters, const SelectList& subSelects, + const GroupByColumnList& groupByCols, ParseTree* having, const OrderByColumnList& orderByCols, + const string alias, const int location, const bool dependent) + : fLocalQuery(GLOBAL_QUERY) + , fReturnedCols(returnedCols) + , fFilters(filters) + , fSubSelects(subSelects) + , fGroupByCols(groupByCols) + , fHaving(having) + , fOrderByCols(orderByCols) + , fTableAlias(alias) + , fLocation(location) + , fDependent(dependent) + , fTxnID(-1) + , fTraceFlags(TRACE_NONE) + , fStatementID(0) + , fDistinct(false) + , fOverrideLargeSideEstimate(false) + , fDistinctUnionNum(0) + , fSubType(MAIN_SELECT) + , fLimitStart(0) + , fLimitNum(-1) + , fHasOrderBy(false) + , fStringScanThreshold(ULONG_MAX) + , fQueryType(SELECT) + , fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL) + , fStringTableThreshold(20) + , fOrderByThreads(1) + , fDJSSmallSideLimit(0) + , fDJSLargeSideLimit(0) + , fDJSPartitionSize(100 * 1024 * 1024) + , // 100MB mem usage for disk based join + fUMMemLimit(numeric_limits::max()) + , fIsDML(false) { - fUuid = QueryTeleClient::genUUID(); + fUuid = QueryTeleClient::genUUID(); } -CalpontSelectExecutionPlan::CalpontSelectExecutionPlan (string data) : - fLocalQuery (GLOBAL_QUERY), - fData(data), - fTxnID(-1), - fTraceFlags(TRACE_NONE), - fStatementID(0), - fDistinct(false), - fOverrideLargeSideEstimate(false), - fDistinctUnionNum(0), - fSubType(MAIN_SELECT), - fLimitStart(0), - fLimitNum(-1), - fHasOrderBy(false), - fStringScanThreshold(ULONG_MAX), - fQueryType(SELECT), - fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL), - fStringTableThreshold(20), - fOrderByThreads(1), - fDJSSmallSideLimit(0), - fDJSLargeSideLimit(0), - fDJSPartitionSize(100 * 1024 * 1024), // 100MB mem usage for disk based join - fUMMemLimit(numeric_limits::max()), - fIsDML(false) +CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(string data) + : fLocalQuery(GLOBAL_QUERY) + , fData(data) + , fTxnID(-1) + , fTraceFlags(TRACE_NONE) + , fStatementID(0) + , fDistinct(false) + , fOverrideLargeSideEstimate(false) + , fDistinctUnionNum(0) + , fSubType(MAIN_SELECT) + , fLimitStart(0) + , fLimitNum(-1) + , fHasOrderBy(false) + , fStringScanThreshold(ULONG_MAX) + , fQueryType(SELECT) + , fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL) + , fStringTableThreshold(20) + , fOrderByThreads(1) + , fDJSSmallSideLimit(0) + , fDJSLargeSideLimit(0) + , fDJSPartitionSize(100 * 1024 * 1024) + , // 100MB mem usage for disk based join + fUMMemLimit(numeric_limits::max()) + , fIsDML(false) { - fUuid = QueryTeleClient::genUUID(); + fUuid = QueryTeleClient::genUUID(); } CalpontSelectExecutionPlan::~CalpontSelectExecutionPlan() { - if (fFilters != NULL) - delete fFilters; + if (fFilters != NULL) + delete fFilters; - if (fHaving != NULL) - delete fHaving; + if (fHaving != NULL) + delete fHaving; - fFilters = NULL; - fHaving = NULL; + fFilters = NULL; + fHaving = NULL; - if (!fDynamicParseTreeVec.empty()) + if (!fDynamicParseTreeVec.empty()) + { + for (auto& parseTree : fDynamicParseTreeVec) { - for (auto& parseTree : fDynamicParseTreeVec) - { - if (parseTree) - { - // 'delete fFilters;' above has already deleted objects pointed - // to by parseTree->left()/right()/data(), so we set the - // pointers to NULL here before calling 'delete parseTree;' - parseTree->left((ParseTree*) (NULL)); - parseTree->right((ParseTree*) (NULL)); - parseTree->data((TreeNode*) (NULL)); - delete parseTree; - parseTree = NULL; - } - } - - fDynamicParseTreeVec.clear(); + if (parseTree) + { + // 'delete fFilters;' above has already deleted objects pointed + // to by parseTree->left()/right()/data(), so we set the + // pointers to NULL here before calling 'delete parseTree;' + parseTree->left((ParseTree*)(NULL)); + parseTree->right((ParseTree*)(NULL)); + parseTree->data((TreeNode*)(NULL)); + delete parseTree; + parseTree = NULL; + } } + + fDynamicParseTreeVec.clear(); + } } /** * Methods */ -void CalpontSelectExecutionPlan::filterTokenList( FilterTokenList& filterTokenList) +void CalpontSelectExecutionPlan::filterTokenList(FilterTokenList& filterTokenList) { - fFilterTokenList = filterTokenList; + fFilterTokenList = filterTokenList; - Parser parser; - std::vector tokens; - Token t; + Parser parser; + std::vector tokens; + Token t; - for (unsigned int i = 0; i < filterTokenList.size(); i++) - { - t.value = filterTokenList[i]; - tokens.push_back(t); - } + for (unsigned int i = 0; i < filterTokenList.size(); i++) + { + t.value = filterTokenList[i]; + tokens.push_back(t); + } - if (tokens.size() > 0) - filters(parser.parse(tokens.begin(), tokens.end())); + if (tokens.size() > 0) + filters(parser.parse(tokens.begin(), tokens.end())); } -void CalpontSelectExecutionPlan::havingTokenList( const FilterTokenList& havingTokenList) +void CalpontSelectExecutionPlan::havingTokenList(const FilterTokenList& havingTokenList) { - fHavingTokenList = havingTokenList; + fHavingTokenList = havingTokenList; - Parser parser; - std::vector tokens; - Token t; + Parser parser; + std::vector tokens; + Token t; - for (unsigned int i = 0; i < havingTokenList.size(); i++) - { - t.value = havingTokenList[i]; - tokens.push_back(t); - } + for (unsigned int i = 0; i < havingTokenList.size(); i++) + { + t.value = havingTokenList[i]; + tokens.push_back(t); + } - if (tokens.size() > 0) - having(parser.parse(tokens.begin(), tokens.end())); + if (tokens.size() > 0) + having(parser.parse(tokens.begin(), tokens.end())); } string CalpontSelectExecutionPlan::toString() const { - ostringstream output; + ostringstream output; - output << ">SELECT " ; + output << ">SELECT "; - if (distinct()) - output << "DISTINCT "; + if (distinct()) + output << "DISTINCT "; - output << "limit: " << limitStart() << " - " << limitNum() << endl; + output << "limit: " << limitStart() << " - " << limitNum() << endl; - switch (location()) + switch (location()) + { + case CalpontSelectExecutionPlan::MAIN: output << "MAIN" << endl; break; + + case CalpontSelectExecutionPlan::FROM: output << "FROM" << endl; break; + + case CalpontSelectExecutionPlan::WHERE: output << "WHERE" << endl; break; + + case CalpontSelectExecutionPlan::HAVING: output << "HAVING" << endl; break; + } + + // Returned Column + CalpontSelectExecutionPlan::ReturnedColumnList retCols = returnedCols(); + output << ">>Returned Columns" << endl; + uint32_t seq = 0; + + for (unsigned int i = 0; i < retCols.size(); i++) + { + output << *retCols[i] << endl; + + if (retCols[i]->colSource() & SELECT_SUB) { - case CalpontSelectExecutionPlan::MAIN: - output << "MAIN" << endl; - break; + output << "select sub -- " << endl; + CalpontSelectExecutionPlan* plan = + dynamic_cast(fSelectSubList[seq++].get()); - case CalpontSelectExecutionPlan::FROM: - output << "FROM" << endl; - break; - - case CalpontSelectExecutionPlan::WHERE: - output << "WHERE" << endl; - break; - - case CalpontSelectExecutionPlan::HAVING: - output << "HAVING" << endl; - break; + if (plan) + output << "{" << *plan << "}" << endl; } + } - // Returned Column - CalpontSelectExecutionPlan::ReturnedColumnList retCols = returnedCols(); - output << ">>Returned Columns" << endl; - uint32_t seq = 0; + // From Clause + CalpontSelectExecutionPlan::TableList tables = tableList(); + output << ">>From Tables" << endl; + seq = 0; - for (unsigned int i = 0; i < retCols.size(); i++) + for (unsigned int i = 0; i < tables.size(); i++) + { + // derived table + if (tables[i].schema.length() == 0 && tables[i].table.length() == 0) { - output << *retCols[i] << endl; + output << "derived table - " << tables[i].alias << endl; + CalpontSelectExecutionPlan* plan = + dynamic_cast(fDerivedTableList[seq++].get()); - if (retCols[i]->colSource() & SELECT_SUB) - { - output << "select sub -- " << endl; - CalpontSelectExecutionPlan* plan = dynamic_cast(fSelectSubList[seq++].get()); - - if (plan) - output << "{" << *plan << "}" << endl; - } + if (plan) + output << "{" << *plan << "}" << endl; } - - // From Clause - CalpontSelectExecutionPlan::TableList tables = tableList(); - output << ">>From Tables" << endl; - seq = 0; - - for (unsigned int i = 0; i < tables.size(); i++) - { - // derived table - if (tables[i].schema.length() == 0 && tables[i].table.length() == 0) - { - output << "derived table - " << tables[i].alias << endl; - CalpontSelectExecutionPlan* plan = dynamic_cast(fDerivedTableList[seq++].get()); - - if (plan) - output << "{" << *plan << "}" << endl; - } - else - { - output << tables[i] << endl; - } - } - - // Filters - output << ">>Filters" << endl; - - if (filters() != 0) - filters()->walk (ParseTree::print, output); else - output << "empty filter tree" << endl; - - // Group by columns - const CalpontSelectExecutionPlan::GroupByColumnList& gbc = groupByCols(); - - if (gbc.size() > 0) { - output << ">>Group By Columns" << endl; - - for (unsigned int i = 0; i < gbc.size(); i++) - output << *gbc[i] << endl; + output << tables[i] << endl; } + } - // Having - if (having() != 0) - { - output << ">>Having" << endl; - having()->walk (ParseTree::print, output); - } + // Filters + output << ">>Filters" << endl; - // Order by columns - const CalpontSelectExecutionPlan::OrderByColumnList& obc = orderByCols(); + if (filters() != 0) + filters()->walk(ParseTree::print, output); + else + output << "empty filter tree" << endl; - if (obc.size() > 0) - { - output << ">>Order By Columns" << endl; + // Group by columns + const CalpontSelectExecutionPlan::GroupByColumnList& gbc = groupByCols(); - for (unsigned int i = 0; i < obc.size(); i++) - output << *obc[i] << endl; - } + if (gbc.size() > 0) + { + output << ">>Group By Columns" << endl; - output << "SessionID: " << fSessionID << endl; - output << "TxnID: " << fTxnID << endl; - output << "VerID: " << fVerID << endl; - output << "TraceFlags: " << fTraceFlags << endl; - output << "StatementID: " << fStatementID << endl; - output << "DistUnionNum: " << (int)fDistinctUnionNum << endl; - output << "Limit: " << fLimitStart << " - " << fLimitNum << endl; - output << "String table threshold: " << fStringTableThreshold << endl; + for (unsigned int i = 0; i < gbc.size(); i++) + output << *gbc[i] << endl; + } - output << "--- Column Map ---" << endl; - CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + // Having + if (having() != 0) + { + output << ">>Having" << endl; + having()->walk(ParseTree::print, output); + } - for (iter = columnMap().begin(); iter != columnMap().end(); iter++) - output << (*iter).first << " : " << (*iter).second << endl; + // Order by columns + const CalpontSelectExecutionPlan::OrderByColumnList& obc = orderByCols(); - output << "UUID: " << fUuid << endl; - output << "QueryType: " << queryType() << endl; + if (obc.size() > 0) + { + output << ">>Order By Columns" << endl; - if (!unionVec().empty()) - output << "\n--- Union Unit ---" << endl; + for (unsigned int i = 0; i < obc.size(); i++) + output << *obc[i] << endl; + } - for (unsigned i = 0; i < unionVec().size(); i++) - { - CalpontSelectExecutionPlan* plan = - dynamic_cast(unionVec()[i].get()); + output << "SessionID: " << fSessionID << endl; + output << "TxnID: " << fTxnID << endl; + output << "VerID: " << fVerID << endl; + output << "TraceFlags: " << fTraceFlags << endl; + output << "StatementID: " << fStatementID << endl; + output << "DistUnionNum: " << (int)fDistinctUnionNum << endl; + output << "Limit: " << fLimitStart << " - " << fLimitNum << endl; + output << "String table threshold: " << fStringTableThreshold << endl; - if (plan) - output << "{" << *plan << "}\n" << endl; - } + output << "--- Column Map ---" << endl; + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; - return output.str(); + for (iter = columnMap().begin(); iter != columnMap().end(); iter++) + output << (*iter).first << " : " << (*iter).second << endl; + + output << "UUID: " << fUuid << endl; + output << "QueryType: " << queryType() << endl; + + if (!unionVec().empty()) + output << "\n--- Union Unit ---" << endl; + + for (unsigned i = 0; i < unionVec().size(); i++) + { + CalpontSelectExecutionPlan* plan = dynamic_cast(unionVec()[i].get()); + + if (plan) + output << "{" << *plan << "}\n" << endl; + } + + return output.str(); } string CalpontSelectExecutionPlan::queryTypeToString(const uint32_t queryType) { - switch (queryType) - { - case SELECT: - return "SELECT"; + switch (queryType) + { + case SELECT: return "SELECT"; - case UPDATE: - return "UPDATE"; + case UPDATE: return "UPDATE"; - case DELETE: - return "DELETE"; + case DELETE: return "DELETE"; - case INSERT_SELECT: - return "INSERT_SELECT"; + case INSERT_SELECT: return "INSERT_SELECT"; - case CREATE_TABLE: - return "CREATE_TABLE"; + case CREATE_TABLE: return "CREATE_TABLE"; - case DROP_TABLE: - return "DROP_TABLE"; + case DROP_TABLE: return "DROP_TABLE"; - case ALTER_TABLE: - return "ALTER_TABLE"; + case ALTER_TABLE: return "ALTER_TABLE"; - case INSERT: - return "INSERT"; + case INSERT: return "INSERT"; - case LOAD_DATA_INFILE: - return "LOAD_DATA_INFILE"; - } + case LOAD_DATA_INFILE: return "LOAD_DATA_INFILE"; + } - return "UNKNOWN"; + return "UNKNOWN"; } void CalpontSelectExecutionPlan::serialize(messageqcpp::ByteStream& b) const { - ReturnedColumnList::const_iterator rcit; - ColumnMap::const_iterator mapiter; - TableList::const_iterator tit; + ReturnedColumnList::const_iterator rcit; + ColumnMap::const_iterator mapiter; + TableList::const_iterator tit; - b << static_cast(ObjectReader::CALPONTSELECTEXECUTIONPLAN); + b << static_cast(ObjectReader::CALPONTSELECTEXECUTIONPLAN); - b << static_cast(fReturnedCols.size()); + b << static_cast(fReturnedCols.size()); - for (rcit = fReturnedCols.begin(); rcit != fReturnedCols.end(); ++rcit) - (*rcit)->serialize(b); + for (rcit = fReturnedCols.begin(); rcit != fReturnedCols.end(); ++rcit) + (*rcit)->serialize(b); - b << static_cast(fTableList.size()); + b << static_cast(fTableList.size()); - for (tit = fTableList.begin(); tit != fTableList.end(); ++tit) - { - (*tit).serialize(b); - } + for (tit = fTableList.begin(); tit != fTableList.end(); ++tit) + { + (*tit).serialize(b); + } - ObjectReader::writeParseTree(fFilters, b); + ObjectReader::writeParseTree(fFilters, b); - b << static_cast(fSubSelects.size()); + b << static_cast(fSubSelects.size()); - for (uint32_t i = 0; i < fSubSelects.size(); i++) - fSubSelects[i]->serialize(b); + for (uint32_t i = 0; i < fSubSelects.size(); i++) + fSubSelects[i]->serialize(b); - b << static_cast(fGroupByCols.size()); + b << static_cast(fGroupByCols.size()); - for (rcit = fGroupByCols.begin(); rcit != fGroupByCols.end(); ++rcit) - (*rcit)->serialize(b); + for (rcit = fGroupByCols.begin(); rcit != fGroupByCols.end(); ++rcit) + (*rcit)->serialize(b); - ObjectReader::writeParseTree(fHaving, b); + ObjectReader::writeParseTree(fHaving, b); - b << static_cast(fOrderByCols.size()); + b << static_cast(fOrderByCols.size()); - for (rcit = fOrderByCols.begin(); rcit != fOrderByCols.end(); ++rcit) - (*rcit)->serialize(b); + for (rcit = fOrderByCols.begin(); rcit != fOrderByCols.end(); ++rcit) + (*rcit)->serialize(b); - b << static_cast(fColumnMap.size()); + b << static_cast(fColumnMap.size()); - for (mapiter = fColumnMap.begin(); mapiter != fColumnMap.end(); ++mapiter) - { - b << (*mapiter).first; - (*mapiter).second->serialize(b); - } + for (mapiter = fColumnMap.begin(); mapiter != fColumnMap.end(); ++mapiter) + { + b << (*mapiter).first; + (*mapiter).second->serialize(b); + } - b << static_cast(frmParms.size()); + b << static_cast(frmParms.size()); - for (RMParmVec::const_iterator it = frmParms.begin(); it != frmParms.end(); ++it) - { - b << it->sessionId; - b << it->id; - b << it->value; - } + for (RMParmVec::const_iterator it = frmParms.begin(); it != frmParms.end(); ++it) + { + b << it->sessionId; + b << it->id; + b << it->value; + } - b << fTableAlias; - b << static_cast(fLocation); + b << fTableAlias; + b << static_cast(fLocation); - b << static_cast< ByteStream::byte>(fDependent); + b << static_cast(fDependent); - // ? not sure if this needs to be added - b << fData; - b << static_cast(fSessionID); - b << static_cast(fTxnID); - b << fVerID; - b << fTraceFlags; - b << fStatementID; - b << static_cast(fDistinct); - b << static_cast(fOverrideLargeSideEstimate); + // ? not sure if this needs to be added + b << fData; + b << static_cast(fSessionID); + b << static_cast(fTxnID); + b << fVerID; + b << fTraceFlags; + b << fStatementID; + b << static_cast(fDistinct); + b << static_cast(fOverrideLargeSideEstimate); - // for union - b << (uint8_t)fDistinctUnionNum; - b << (uint32_t)fUnionVec.size(); + // for union + b << (uint8_t)fDistinctUnionNum; + b << (uint32_t)fUnionVec.size(); - for (uint32_t i = 0; i < fUnionVec.size(); i++) - fUnionVec[i]->serialize(b); + for (uint32_t i = 0; i < fUnionVec.size(); i++) + fUnionVec[i]->serialize(b); - b << (uint64_t)fSubType; + b << (uint64_t)fSubType; - // for FROM subquery - b << static_cast(fDerivedTableList.size()); + // for FROM subquery + b << static_cast(fDerivedTableList.size()); - for (uint32_t i = 0; i < fDerivedTableList.size(); i++) - fDerivedTableList[i]->serialize(b); + for (uint32_t i = 0; i < fDerivedTableList.size(); i++) + fDerivedTableList[i]->serialize(b); - b << (uint64_t)fLimitStart; - b << (uint64_t)fLimitNum; - b << static_cast(fHasOrderBy); - b << static_cast(fSpecHandlerProcessed); - b << reinterpret_cast(fOrderByThreads); + b << (uint64_t)fLimitStart; + b << (uint64_t)fLimitNum; + b << static_cast(fHasOrderBy); + b << static_cast(fSpecHandlerProcessed); + b << reinterpret_cast(fOrderByThreads); - b << static_cast(fSelectSubList.size()); + b << static_cast(fSelectSubList.size()); - for (uint32_t i = 0; i < fSelectSubList.size(); i++) - fSelectSubList[i]->serialize(b); + for (uint32_t i = 0; i < fSelectSubList.size(); i++) + fSelectSubList[i]->serialize(b); - b << (uint64_t)fStringScanThreshold; - b << (uint32_t)fQueryType; - b << fPriority; - b << fStringTableThreshold; - b << fSchemaName; - b << fLocalQuery; - b << fUuid; - b << fDJSSmallSideLimit; - b << fDJSLargeSideLimit; - b << fDJSPartitionSize; - b << fUMMemLimit; - b << (uint8_t) fIsDML; - b << fTimeZone; + b << (uint64_t)fStringScanThreshold; + b << (uint32_t)fQueryType; + b << fPriority; + b << fStringTableThreshold; + b << fSchemaName; + b << fLocalQuery; + b << fUuid; + b << fDJSSmallSideLimit; + b << fDJSLargeSideLimit; + b << fDJSPartitionSize; + b << fUMMemLimit; + b << (uint8_t)fIsDML; + b << fTimeZone; } void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) { - ReturnedColumn* rc; - CalpontExecutionPlan* cep; - string colName; - uint8_t tmp8; + ReturnedColumn* rc; + CalpontExecutionPlan* cep; + string colName; + uint8_t tmp8; - ObjectReader::checkType(b, ObjectReader::CALPONTSELECTEXECUTIONPLAN); + ObjectReader::checkType(b, ObjectReader::CALPONTSELECTEXECUTIONPLAN); - // erase elements, otherwise vectors contain null pointers - fReturnedCols.clear(); - fSubSelects.clear(); - fGroupByCols.clear(); - fOrderByCols.clear(); - fTableList.clear(); - fColumnMap.clear(); - fUnionVec.clear(); - frmParms.clear(); - fDerivedTableList.clear(); - fSelectSubList.clear(); + // erase elements, otherwise vectors contain null pointers + fReturnedCols.clear(); + fSubSelects.clear(); + fGroupByCols.clear(); + fOrderByCols.clear(); + fTableList.clear(); + fColumnMap.clear(); + fUnionVec.clear(); + frmParms.clear(); + fDerivedTableList.clear(); + fSelectSubList.clear(); - if (fFilters != 0) + if (fFilters != 0) + { + delete fFilters; + fFilters = 0; + } + + if (fHaving != 0) + { + delete fHaving; + fHaving = 0; + } + + if (!fDynamicParseTreeVec.empty()) + { + for (auto& parseTree : fDynamicParseTreeVec) { - delete fFilters; - fFilters = 0; + if (parseTree) + { + // 'delete fFilters;' above has already deleted objects pointed + // to by parseTree->left()/right()/data(), so we set the + // pointers to NULL here before calling 'delete parseTree;' + parseTree->left((ParseTree*)(NULL)); + parseTree->right((ParseTree*)(NULL)); + parseTree->data((TreeNode*)(NULL)); + delete parseTree; + parseTree = NULL; + } } - if (fHaving != 0) - { - delete fHaving; - fHaving = 0; - } + fDynamicParseTreeVec.clear(); + } - if (!fDynamicParseTreeVec.empty()) - { - for (auto& parseTree : fDynamicParseTreeVec) - { - if (parseTree) - { - // 'delete fFilters;' above has already deleted objects pointed - // to by parseTree->left()/right()/data(), so we set the - // pointers to NULL here before calling 'delete parseTree;' - parseTree->left((ParseTree*) (NULL)); - parseTree->right((ParseTree*) (NULL)); - parseTree->data((TreeNode*) (NULL)); - delete parseTree; - parseTree = NULL; - } - } + messageqcpp::ByteStream::quadbyte size; + messageqcpp::ByteStream::quadbyte i; - fDynamicParseTreeVec.clear(); - } + b >> size; - messageqcpp::ByteStream::quadbyte size; - messageqcpp::ByteStream::quadbyte i; + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fReturnedCols.push_back(srcp); + } - b >> size; + b >> size; + CalpontSystemCatalog::TableAliasName tan; - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fReturnedCols.push_back(srcp); - } + for (i = 0; i < size; i++) + { + tan.unserialize(b); + fTableList.push_back(tan); + } - b >> size; - CalpontSystemCatalog::TableAliasName tan; + fFilters = ObjectReader::createParseTree(b); - for (i = 0; i < size; i++) - { - tan.unserialize(b); - fTableList.push_back(tan); - } + b >> size; - fFilters = ObjectReader::createParseTree(b); + for (i = 0; i < size; i++) + { + cep = ObjectReader::createExecutionPlan(b); + fSubSelects.push_back(SCEP(cep)); + } - b >> size; + b >> size; - for (i = 0; i < size; i++) - { - cep = ObjectReader::createExecutionPlan(b); - fSubSelects.push_back(SCEP(cep)); - } + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fGroupByCols.push_back(srcp); + } - b >> size; + fHaving = ObjectReader::createParseTree(b); - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fGroupByCols.push_back(srcp); - } + b >> size; - fHaving = ObjectReader::createParseTree(b); + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fOrderByCols.push_back(srcp); + } - b >> size; + b >> size; - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fOrderByCols.push_back(srcp); - } + for (i = 0; i < size; i++) + { + b >> colName; + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fColumnMap.insert(ColumnMap::value_type(colName, srcp)); + } - b >> size; + b >> size; + messageqcpp::ByteStream::doublebyte id; + messageqcpp::ByteStream::quadbyte sessionId; + messageqcpp::ByteStream::octbyte memory; - for (i = 0; i < size; i++) - { - b >> colName; - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fColumnMap.insert(ColumnMap::value_type(colName, srcp)); - } + for (i = 0; i < size; i++) + { + b >> sessionId; + b >> id; + b >> memory; + frmParms.push_back(RMParam(sessionId, id, memory)); + } - b >> size; - messageqcpp::ByteStream::doublebyte id; - messageqcpp::ByteStream::quadbyte sessionId; - messageqcpp::ByteStream::octbyte memory; + b >> fTableAlias; + b >> reinterpret_cast(fLocation); + b >> reinterpret_cast(fDependent); - for (i = 0; i < size; i++) - { - b >> sessionId; - b >> id; - b >> memory; - frmParms.push_back(RMParam(sessionId, id, memory)); - } + // ? not sure if this needs to be added + b >> fData; + b >> reinterpret_cast(fSessionID); + b >> reinterpret_cast(fTxnID); + b >> fVerID; + b >> fTraceFlags; + b >> fStatementID; + b >> reinterpret_cast(fDistinct); + uint8_t val; + b >> reinterpret_cast(val); + fOverrideLargeSideEstimate = (val != 0); - b >> fTableAlias; - b >> reinterpret_cast(fLocation); - b >> reinterpret_cast< ByteStream::byte&>(fDependent); + // for union + b >> (uint8_t&)(fDistinctUnionNum); + b >> size; - // ? not sure if this needs to be added - b >> fData; - b >> reinterpret_cast(fSessionID); - b >> reinterpret_cast(fTxnID); - b >> fVerID; - b >> fTraceFlags; - b >> fStatementID; - b >> reinterpret_cast< ByteStream::byte&>(fDistinct); - uint8_t val; - b >> reinterpret_cast(val); - fOverrideLargeSideEstimate = (val != 0); + for (i = 0; i < size; i++) + { + cep = ObjectReader::createExecutionPlan(b); + fUnionVec.push_back(SCEP(cep)); + } - // for union - b >> (uint8_t&)(fDistinctUnionNum); - b >> size; + b >> (uint64_t&)fSubType; - for (i = 0; i < size; i++) - { - cep = ObjectReader::createExecutionPlan(b); - fUnionVec.push_back(SCEP(cep)); - } + // for FROM subquery + b >> size; - b >> (uint64_t&)fSubType; + for (i = 0; i < size; i++) + { + cep = ObjectReader::createExecutionPlan(b); + fDerivedTableList.push_back(SCEP(cep)); + } - // for FROM subquery - b >> size; + b >> (uint64_t&)fLimitStart; + b >> (uint64_t&)fLimitNum; + b >> reinterpret_cast(fHasOrderBy); + b >> reinterpret_cast(fSpecHandlerProcessed); + b >> reinterpret_cast(fOrderByThreads); - for (i = 0; i < size; i++) - { - cep = ObjectReader::createExecutionPlan(b); - fDerivedTableList.push_back(SCEP(cep)); - } + // for SELECT subquery + b >> size; - b >> (uint64_t&)fLimitStart; - b >> (uint64_t&)fLimitNum; - b >> reinterpret_cast< ByteStream::byte&>(fHasOrderBy); - b >> reinterpret_cast< ByteStream::byte&>(fSpecHandlerProcessed); - b >> reinterpret_cast(fOrderByThreads); + for (i = 0; i < size; i++) + { + cep = ObjectReader::createExecutionPlan(b); + fSelectSubList.push_back(SCEP(cep)); + } - // for SELECT subquery - b >> size; - - for (i = 0; i < size; i++) - { - cep = ObjectReader::createExecutionPlan(b); - fSelectSubList.push_back(SCEP(cep)); - } - - b >> (uint64_t&)fStringScanThreshold; - b >> (uint32_t&)fQueryType; - b >> fPriority; - b >> fStringTableThreshold; - b >> fSchemaName; - b >> fLocalQuery; - b >> fUuid; - b >> fDJSSmallSideLimit; - b >> fDJSLargeSideLimit; - b >> fDJSPartitionSize; - b >> fUMMemLimit; - b >> tmp8; - fIsDML = tmp8; - b >> fTimeZone; + b >> (uint64_t&)fStringScanThreshold; + b >> (uint32_t&)fQueryType; + b >> fPriority; + b >> fStringTableThreshold; + b >> fSchemaName; + b >> fLocalQuery; + b >> fUuid; + b >> fDJSSmallSideLimit; + b >> fDJSLargeSideLimit; + b >> fDJSPartitionSize; + b >> fUMMemLimit; + b >> tmp8; + fIsDML = tmp8; + b >> fTimeZone; } bool CalpontSelectExecutionPlan::operator==(const CalpontSelectExecutionPlan& t) const { + // If we use this outside the serialization tests, we should + // reorder these comparisons to speed up the common case - // If we use this outside the serialization tests, we should - // reorder these comparisons to speed up the common case + ReturnedColumnList::const_iterator rcit; + ReturnedColumnList::const_iterator rcit2; + SelectList::const_iterator sit, sit2; + ColumnMap::const_iterator map_it, map_it2; - ReturnedColumnList::const_iterator rcit; - ReturnedColumnList::const_iterator rcit2; - SelectList::const_iterator sit, sit2; - ColumnMap::const_iterator map_it, map_it2; + // fReturnedCols + if (fReturnedCols.size() != t.fReturnedCols.size()) + return false; - //fReturnedCols - if (fReturnedCols.size() != t.fReturnedCols.size()) - return false; + for (rcit = fReturnedCols.begin(), rcit2 = t.fReturnedCols.begin(); rcit != fReturnedCols.end(); + ++rcit, ++rcit2) + if (**rcit != **rcit2) + return false; - for (rcit = fReturnedCols.begin(), rcit2 = t.fReturnedCols.begin(); - rcit != fReturnedCols.end(); ++rcit, ++rcit2) - if (**rcit != **rcit2) - return false; + // fFilters + if (fFilters != NULL && t.fFilters != NULL) + { + if (*fFilters != *t.fFilters) + return false; + } + else if (fFilters != NULL || t.fFilters != NULL) + return false; - //fFilters - if (fFilters != NULL && t.fFilters != NULL) - { - if (*fFilters != *t.fFilters) - return false; - } - else if (fFilters != NULL || t.fFilters != NULL) - return false; + // fSubSelects + if (fSubSelects.size() != t.fSubSelects.size()) + return false; - //fSubSelects - if (fSubSelects.size() != t.fSubSelects.size()) - return false; + for (sit = fSubSelects.begin(), sit2 = t.fSubSelects.begin(); sit != fSubSelects.end(); ++sit, ++sit2) + if (*((*sit).get()) != (*sit2).get()) + return false; - for (sit = fSubSelects.begin(), sit2 = t.fSubSelects.begin(); - sit != fSubSelects.end(); ++sit, ++sit2) - if (*((*sit).get()) != (*sit2).get()) - return false; + // fGroupByCols + if (fGroupByCols.size() != t.fGroupByCols.size()) + return false; - //fGroupByCols - if (fGroupByCols.size() != t.fGroupByCols.size()) - return false; + for (rcit = fGroupByCols.begin(), rcit2 = t.fGroupByCols.begin(); rcit != fGroupByCols.end(); + ++rcit, ++rcit2) + if (**rcit != **rcit2) + return false; - for (rcit = fGroupByCols.begin(), rcit2 = t.fGroupByCols.begin(); - rcit != fGroupByCols.end(); ++rcit, ++rcit2) - if (**rcit != **rcit2) - return false; + // fHaving + if (fHaving != NULL && t.fHaving != NULL) + { + if (*fHaving != *t.fHaving) + return false; + } + else if (fHaving != NULL || t.fHaving != NULL) + return false; - //fHaving - if (fHaving != NULL && t.fHaving != NULL) - { - if (*fHaving != *t.fHaving) - return false; - } - else if (fHaving != NULL || t.fHaving != NULL) - return false; + // fOrderByCols + if (fOrderByCols.size() != t.fOrderByCols.size()) + return false; - //fOrderByCols - if (fOrderByCols.size() != t.fOrderByCols.size()) - return false; + for (rcit = fOrderByCols.begin(), rcit2 = t.fOrderByCols.begin(); rcit != fOrderByCols.end(); + ++rcit, ++rcit2) + if (**rcit != **rcit2) + return false; - for (rcit = fOrderByCols.begin(), rcit2 = t.fOrderByCols.begin(); - rcit != fOrderByCols.end(); ++rcit, ++rcit2) - if (**rcit != **rcit2) - return false; + // fColumnMap + if (fColumnMap.size() != t.fColumnMap.size()) + return false; - //fColumnMap - if (fColumnMap.size() != t.fColumnMap.size()) - return false; + for (map_it = fColumnMap.begin(), map_it2 = t.fColumnMap.begin(); map_it != fColumnMap.end(); + ++map_it, ++map_it2) + if (*(map_it->second) != *(map_it2->second)) + return false; - for (map_it = fColumnMap.begin(), map_it2 = t.fColumnMap.begin(); - map_it != fColumnMap.end(); ++map_it, ++map_it2) - if (*(map_it->second) != *(map_it2->second)) - return false; + if (fTableAlias != t.fTableAlias) + return false; - if (fTableAlias != t.fTableAlias) - return false; + if (fLocation != t.fLocation) + return false; - if (fLocation != t.fLocation) - return false; + if (fDependent != t.fDependent) + return false; - if (fDependent != t.fDependent) - return false; + // Trace flags don't affect equivalency? + // if (fTraceFlags != t.fTraceFlags) return false; + if (fStatementID != t.fStatementID) + return false; - // Trace flags don't affect equivalency? - //if (fTraceFlags != t.fTraceFlags) return false; - if (fStatementID != t.fStatementID) - return false; + if (fSubType != t.fSubType) + return false; - if (fSubType != t.fSubType) - return false; + if (fPriority != t.fPriority) + return false; - if (fPriority != t.fPriority) - return false; + if (fStringTableThreshold != t.fStringTableThreshold) + return false; - if (fStringTableThreshold != t.fStringTableThreshold) - return false; + if (fDJSSmallSideLimit != t.fDJSSmallSideLimit) + return false; - if (fDJSSmallSideLimit != t.fDJSSmallSideLimit) - return false; + if (fDJSLargeSideLimit != t.fDJSLargeSideLimit) + return false; - if (fDJSLargeSideLimit != t.fDJSLargeSideLimit) - return false; + if (fDJSPartitionSize != t.fDJSPartitionSize) + return false; - if (fDJSPartitionSize != t.fDJSPartitionSize) - return false; + if (fUMMemLimit != t.fUMMemLimit) + return false; - if (fUMMemLimit != t.fUMMemLimit) - return false; - - return true; + return true; } bool CalpontSelectExecutionPlan::operator==(const CalpontExecutionPlan* t) const { - const CalpontSelectExecutionPlan* ac; + const CalpontSelectExecutionPlan* ac; - ac = dynamic_cast(t); + ac = dynamic_cast(t); - if (ac == NULL) - return false; + if (ac == NULL) + return false; - return *this == *ac; + return *this == *ac; } bool CalpontSelectExecutionPlan::operator!=(const CalpontSelectExecutionPlan& t) const { - return !(*this == t); + return !(*this == t); } bool CalpontSelectExecutionPlan::operator!=(const CalpontExecutionPlan* t) const { - return !(*this == t); + return !(*this == t); } -void CalpontSelectExecutionPlan::columnMap (const ColumnMap& columnMap) +void CalpontSelectExecutionPlan::columnMap(const ColumnMap& columnMap) { - ColumnMap::const_iterator map_it1, map_it2; - fColumnMap.erase(fColumnMap.begin(), fColumnMap.end()); + ColumnMap::const_iterator map_it1, map_it2; + fColumnMap.erase(fColumnMap.begin(), fColumnMap.end()); - SRCP srcp; + SRCP srcp; - for (map_it2 = columnMap.begin(); map_it2 != columnMap.end(); ++map_it2) - { - srcp.reset(map_it2->second->clone()); - fColumnMap.insert(ColumnMap::value_type(map_it2->first, srcp)); - } + for (map_it2 = columnMap.begin(); map_it2 != columnMap.end(); ++map_it2) + { + srcp.reset(map_it2->second->clone()); + fColumnMap.insert(ColumnMap::value_type(map_it2->first, srcp)); + } } -void CalpontSelectExecutionPlan::rmParms (const RMParmVec& parms) +void CalpontSelectExecutionPlan::rmParms(const RMParmVec& parms) { - - frmParms.clear(); - frmParms.assign(parms.begin(), parms.end()); + frmParms.clear(); + frmParms.assign(parms.begin(), parms.end()); } - -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index 403ec3d8f..026b61d7a 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: calpontselectexecutionplan.h 9576 2013-05-29 21:02:11Z zzhu $ -* -* -***********************************************************************/ + * $Id: calpontselectexecutionplan.h 9576 2013-05-29 21:02:11Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef CALPONTSELECTEXECUTIONPLAN_H @@ -40,9 +40,9 @@ #include #ifndef __GNUC__ -# ifndef __attribute__ -# define __attribute__(x) -# endif +#ifndef __attribute__ +#define __attribute__(x) +#endif #endif /** @@ -50,20 +50,23 @@ */ namespace execplan { - enum RM_PARMS { - PMSMALLSIDEMEMORY, - UMSMALLSIDEMEMORY, + PMSMALLSIDEMEMORY, + UMSMALLSIDEMEMORY, }; struct RMParam { - RMParam(uint32_t s, uint16_t i, uint64_t v): sessionId(s), id(i), value(v) {} - RMParam(): sessionId(0), id(0), value(0) {} - uint32_t sessionId; - uint16_t id; - uint64_t value; + RMParam(uint32_t s, uint16_t i, uint64_t v) : sessionId(s), id(i), value(v) + { + } + RMParam() : sessionId(0), id(0), value(0) + { + } + uint32_t sessionId; + uint16_t id; + uint64_t value; }; typedef boost::shared_ptr SCSEP; @@ -76,859 +79,854 @@ typedef boost::shared_ptr SCSEP; */ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { - - /** - * Public stuff - */ -public: - /** - * Types and constants - */ - typedef std::vector ReturnedColumnList; - typedef std::vector GroupByColumnList; - typedef std::vector OrderByColumnList; - typedef std::multimap ColumnMap; - typedef std::vector FilterTokenList; - - typedef expression::expression_parser Parser; - typedef std::vector TableList; - typedef std::vector SelectList; - - typedef std::vector RMParmVec; - - // query type of this select plan. -#undef DELETE //Windows defines this... - enum IDB_QUERYTYPE - { - SELECT, - UPDATE, - DELETE, - INSERT_SELECT, - CREATE_TABLE, - DROP_TABLE, - ALTER_TABLE, - INSERT, - LOAD_DATA_INFILE - }; - - enum SE_LOCATION - { - MAIN, - FROM, - WHERE, - HAVING - }; - - /** subselect type */ - enum SE_SubSelectType - { - MAIN_SELECT, - SINGLEROW_SUBS, - EXISTS_SUBS, - NOT_EXISTS_SUBS, - IN_SUBS, - NOT_IN_SUBS, - ALL_SUBS, - ANY_SUBS, - FROM_SUBS, - SELECT_SUBS, - UNKNOWN_SUBS - }; - - enum IDB_LOCAL_QUERY - { - GLOBAL_QUERY = 0, /*!< Standard processing */ - LOCAL_QUERY = 1, /*!< Use local ExeMgr and local PrimProc */ - LOCAL_UM = 2, /*!< Use local ExeMgr with global PrimProc*/ - }; - - /** - * Constructors - */ - CalpontSelectExecutionPlan(const int location = MAIN); - - CalpontSelectExecutionPlan(const ReturnedColumnList& returnedCols, - ParseTree* filters, - const SelectList& subSelects, - const GroupByColumnList& groupByCols, - ParseTree* having, - const OrderByColumnList& orderByCols, - const std::string alias, - const int location, - const bool dependent); - - CalpontSelectExecutionPlan (const std::string data); - - /** - * Destructors - */ - virtual ~CalpontSelectExecutionPlan(); - - /** - * Access and mutator methods - */ - - /** - * returned column list - */ - const ReturnedColumnList& returnedCols() const - { - return fReturnedCols; - } - ReturnedColumnList& returnedCols() - { - return fReturnedCols; - } - void returnedCols (const ReturnedColumnList& returnedCols) - { - fReturnedCols = returnedCols; - } - - /** - * Are we in local PM only query mode? - */ - uint32_t localQuery() const - { - return fLocalQuery; - } - void localQuery (const uint32_t localQuery) - { - fLocalQuery = localQuery; - } - - /** - * filters parse tree - */ - const ParseTree* filters() const - { - return fFilters; - } - ParseTree* filters() - { - return fFilters; - } - void filters (ParseTree* filters) - { - fFilters = filters; - } - - /** filter token list - * Set filter list field and build filters tree from the filter list - * This is an alternative way to build filter tree. Hide the parser - * from the user. filterTokenList will be destroyed after the parsing - */ - //const FilterTokenList& filterTokenList() const { return fFilterTokenList;} - void filterTokenList (FilterTokenList& filterTokenList); - - /** - * sub select list - */ - const SelectList& subSelects() const - { - return fSubSelects; - } - void subSelects (const SelectList& subSelects) - { - fSubSelects = subSelects; - } - - /** - * group by column list - */ - const GroupByColumnList& groupByCols() const - { - return fGroupByCols; - } - GroupByColumnList& groupByCols() - { - return fGroupByCols; - } - void groupByCols( const GroupByColumnList& groupByCols) - { - fGroupByCols = groupByCols; - } - - /** - * order by column list - */ - const OrderByColumnList& orderByCols() const - { - return fOrderByCols; - } - OrderByColumnList& orderByCols() - { - return fOrderByCols; - } - void orderByCols( const OrderByColumnList& orderByCols) - { - fOrderByCols = orderByCols; - } - - /** - * table alias - */ - const std::string& tableAlias() const - { - return fTableAlias; - } - void tableAlias (const std::string& tableAlias, int lower_case_table_names) - { - fTableAlias = tableAlias; - if (lower_case_table_names) - boost::algorithm::to_lower(fTableAlias); - } - - /** - * location of this select - */ - int location () const - { - return fLocation; - } - void location (const int location) - { - fLocation = location; - } - - /** - * dependence of this select - */ - bool dependent() const - { - return fDependent; - } - void dependent (const bool dependent) - { - fDependent = dependent; - } - - /** - * having filter parse tree - */ - inline const ParseTree* having() const - { - return fHaving; - } - inline ParseTree* having() - { - return fHaving; - } - inline void having (ParseTree* having) - { - fHaving = having; - } - - /** having filter token list - * Set filter list field and build filters tree from the filter list - * This is an alternative way to build filter tree. Hide the parser - * from the user - */ - const FilterTokenList& havingTokenList() const - { - return fHavingTokenList; - } - void havingTokenList (const FilterTokenList& havingTokenList); - - /** column map - * all the columns appeared on query - */ - const ColumnMap& columnMap() const - { - return fColumnMap; - } - - /** column map - * all the columns appeared on query - */ - ColumnMap& columnMap() - { - return fColumnMap; - } - - /** assign the static fColMap to non-static fColumnMap. map-wise copy */ - void columnMap (const ColumnMap& columnMap); - - /** assign a regular map object to non-static fColumnMap. pure assignment */ - // @note this is to fix memory leak in CSC, becasue no static map is needed there. - inline void columnMapNonStatic (const ColumnMap& columnMap) - { - fColumnMap = columnMap; - } - - /** sql representation of this select query - * - */ - const std::string data() const - { - return fData; - } - void data ( const std::string data ) - { - fData = data; - } - - /** session id - * - */ - uint32_t sessionID() const - { - return fSessionID; - } - void sessionID ( const uint32_t sessionID ) - { - fSessionID = sessionID; - } - - /** transaction id - * - */ - int txnID() const - { - return fTxnID; - } - void txnID ( const int txnID ) - { - fTxnID = txnID; - } - - /** version id - * - */ - const BRM::QueryContext verID() const - { - return fVerID; - } - void verID ( const BRM::QueryContext verID ) - { - fVerID = verID; - } - - inline static ColumnMap& colMap() - { - return fColMap; - } - - inline std::string& schemaName() - { - return fSchemaName; - } - inline void schemaName(const std::string& schemaName, int lower_case_table_names) - { - fSchemaName = schemaName; - if (lower_case_table_names) - boost::algorithm::to_lower(fSchemaName); - } - - inline std::string& tableName() - { - return fTableName; - } - inline void tableName(const std::string& tableName, int lower_case_table_names) - { - fTableName = tableName; - if (lower_case_table_names) - boost::algorithm::to_lower(fTableName); - } - - inline void traceOn(bool traceOn) __attribute__((deprecated)) - { - if (traceOn) fTraceFlags |= TRACE_LOG; - else fTraceFlags &= ~TRACE_LOG; - } - inline bool traceOn() const - { - return (traceFlags() & TRACE_LOG); - } - - inline uint32_t traceFlags() const - { - return fTraceFlags; - } - inline void traceFlags(uint32_t traceFlags) - { - fTraceFlags = traceFlags; - } - - uint32_t statementID() const - { - return fStatementID; - } - void statementID (const uint32_t statementID) - { - fStatementID = statementID; - } - - const RMParmVec& rmParms() - { - return frmParms; - } - void rmParms (const RMParmVec& parms); - - const TableList& tableList() const - { - return fTableList; - } - void tableList (const TableList& tableList) - { - fTableList = tableList; - } - - const SelectList& derivedTableList() const - { - return fDerivedTableList; - } - void derivedTableList(SelectList& derivedTableList) - { - fDerivedTableList = derivedTableList; - } - - bool distinct() const - { - return fDistinct; - } - void distinct(const bool distinct) - { - fDistinct = distinct; - } - - void overrideLargeSideEstimate (const bool over) - { - fOverrideLargeSideEstimate = over; - } - bool overrideLargeSideEstimate() const - { - return fOverrideLargeSideEstimate; - } - - void unionVec(const SelectList& unionVec) - { - fUnionVec = unionVec; - } - const SelectList& unionVec() const - { - return fUnionVec; - } - SelectList& unionVec() - { - return fUnionVec; - } - - void distinctUnionNum(const uint8_t distinctUnionNum) - { - fDistinctUnionNum = distinctUnionNum; - } - uint8_t distinctUnionNum() const - { - return fDistinctUnionNum; - } - - void subType(const uint64_t subType) - { - fSubType = subType; - } - uint64_t subType() const - { - return fSubType; - } - - void derivedTbAlias(const std::string derivedTbAlias, int lower_case_table_names=0) - { - fDerivedTbAlias = derivedTbAlias; - if (lower_case_table_names) - boost::algorithm::to_lower(fDerivedTbAlias); - } - const std::string derivedTbAlias() const - { - return fDerivedTbAlias; - } - - void derivedTbView(const std::string derivedTbView, int lower_case_table_names) - { - fDerivedTbView = derivedTbView; - if (lower_case_table_names) - boost::algorithm::to_lower(fDerivedTbView); - } - const std::string derivedTbView() const { return fDerivedTbView; } - - - void limitStart(const uint64_t limitStart) - { - fLimitStart = limitStart; - } - uint64_t limitStart() const - { - return fLimitStart; - } - - void limitNum(const uint64_t limitNum) - { - fLimitNum = limitNum; - } - uint64_t limitNum() const - { - return fLimitNum; - } - - void hasOrderBy(const bool hasOrderBy) - { - fHasOrderBy = hasOrderBy; - } - bool hasOrderBy() const - { - return fHasOrderBy; - } - - void specHandlerProcessed(const bool hand) - { - fSpecHandlerProcessed = hand; - } - bool specHandlerProcessed() const - { - return fSpecHandlerProcessed; - } - - void orderByThreads(const uint32_t number) - { - fOrderByThreads = number; - } - uint32_t orderByThreads() const - { - return fOrderByThreads; - } - - - void selectSubList(const SelectList& selectSubList) - { - fSelectSubList = selectSubList; - } - const SelectList& selectSubList() const - { - return fSelectSubList; - } - - void subSelectList(const std::vector& subSelectList) - { - fSubSelectList = subSelectList; - } - const std::vector& subSelectList() const - { - return fSubSelectList; - } - - void stringScanThreshold(uint64_t n) - { - fStringScanThreshold = n; - } - uint64_t stringScanThreshold() const - { - return fStringScanThreshold; - } - - // query type. return string for easy stats insert - void queryType(const uint32_t queryType) - { - fQueryType = queryType; - } - const std::string queryType() const - { - return queryTypeToString(fQueryType); - } - static std::string queryTypeToString(const uint32_t queryType); - - void priority(uint32_t p) - { - fPriority = p; - } - uint32_t priority() const - { - return fPriority; - } - - void stringTableThreshold(uint32_t t) - { - fStringTableThreshold = t; - } - uint32_t stringTableThreshold() const - { - return fStringTableThreshold; - } - - void uuid(const boost::uuids::uuid& uuid) - { - fUuid = uuid; - } - const boost::uuids::uuid& uuid() const - { - return fUuid; - } - - void djsSmallSideLimit(uint64_t l) - { - fDJSSmallSideLimit = l; - } - uint64_t djsSmallSideLimit() - { - return fDJSSmallSideLimit; - } - - void djsLargeSideLimit(uint64_t l) - { - fDJSLargeSideLimit = l; - } - uint64_t djsLargeSideLimit() - { - return fDJSLargeSideLimit; - } - - void djsPartitionSize(uint64_t l) - { - fDJSPartitionSize = l; - } - uint64_t djsPartitionSize() - { - return fDJSPartitionSize; - } - - void umMemLimit(uint64_t l) - { - fUMMemLimit = l; - } - int64_t umMemLimit() - { - return fUMMemLimit; - } - - void isDML(bool b) - { - fIsDML = b; - } - bool isDML() - { - return fIsDML; - } - - void timeZone(const std::string& timezone) - { - fTimeZone = timezone; - } - const std::string timeZone() const - { - return fTimeZone; - } - - /** - * The serialization interface - */ - /** - * @note Serialize() assumes that none of the vectors contain NULL pointers. - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const CalpontExecutionPlan* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const CalpontSelectExecutionPlan& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const CalpontExecutionPlan* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const CalpontSelectExecutionPlan& t) const; - - /** @brief Return a string rep of the CSEP - * - * Return a string rep of the CSEP - * @return a string - */ - virtual std::string toString() const; - - /** @brief Is this an internal query? - * - * Is this an internal query (a syscat query performed on behalf of another query) - * FIXME: add a setter and make this work for really big session ids - * @return true/false - */ - virtual bool isInternal() const - { - return ((fSessionID & 0x80000000) != 0); - } - - virtual void setDynamicParseTreeVec( - const std::vector& dynamicParseTreeVec) - { - fDynamicParseTreeVec = dynamicParseTreeVec; - } - - /** - * Protected stuff - */ -protected: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Private stuff - */ -private: - - /** - * If set, then the local PM only option is turned on - */ - uint32_t fLocalQuery; - - /** - * A list of ReturnedColumn objects - */ - ReturnedColumnList fReturnedCols; - /** - * A list of filter tokens including filters and operators - * This is a helper data structure, it holds the tokens of - * the filter parse tree and will be built into a parse tree - * in the mutator method. - * @note the elements in this list will be deleted when fFilters - * parse tree is deleted. So this list should not be deleted - * again in destructor. - */ - FilterTokenList fFilterTokenList; - FilterTokenList fHavingTokenList; - - /** - * A tree of Filter objects - */ - ParseTree* fFilters; - /** - * A list of CalpontExecutionPlan objects - */ - SelectList fSubSelects; - /** - * A list of group by columns - */ - GroupByColumnList fGroupByCols; - /** - * A tree of having clause condition associated with group by clause - */ - ParseTree* fHaving; - /** - * A list of order by columns - */ - OrderByColumnList fOrderByCols; - /** - * Table or alias name for subselect in FROM clause - */ - std::string fTableAlias; - /** - * An enum indicating the location of this select statement in the enclosing select statement - */ - int fLocation; - /** - * A flag indicating if this sub-select is dependent on the enclosing query or is constant - */ - bool fDependent; - - /** - * SQL representation of this execution plan - */ - std::string fData; - static ColumnMap fColMap; // for getplan to use. class-wise map - ColumnMap fColumnMap; // for ExeMgr to use. not shared between objects - - uint32_t fSessionID; - int fTxnID; // SQLEngine only needs the ID value - BRM::QueryContext fVerID; - // @bug5316. remove static - std::string fSchemaName; - std::string fTableName; - uint32_t fTraceFlags; - - /** - * One-up statementID number for this session (fSessionID) - */ - uint32_t fStatementID; - - RMParmVec frmParms; - TableList fTableList; - SelectList fDerivedTableList; - - bool fDistinct; - bool fOverrideLargeSideEstimate; - - // for union - SelectList fUnionVec; - uint8_t fDistinctUnionNum; - - // for subselect - uint64_t fSubType; - std::string fDerivedTbAlias; - std::string fDerivedTbView; - - // for limit - uint64_t fLimitStart; - uint64_t fLimitNum; - - // for parent select order by - bool fHasOrderBy; - - // for Select clause subquery - SelectList fSelectSubList; - - // @bug3321, for string scan blocks - uint64_t fStringScanThreshold; - - // query type - uint32_t fQueryType; - - uint32_t fPriority; - uint32_t fStringTableThreshold; - - // for specific handlers processing, e.g. GROUP BY - bool fSpecHandlerProcessed; - uint32_t fOrderByThreads; - - // Derived table involved in the query. For derived table optimization - std::vector fSubSelectList; - - boost::uuids::uuid fUuid; - - /* Disk-based join vars */ - uint64_t fDJSSmallSideLimit; - uint64_t fDJSLargeSideLimit; - uint64_t fDJSPartitionSize; - int64_t fUMMemLimit; - bool fIsDML; - - std::string fTimeZone; - - std::vector fDynamicParseTreeVec; + /** + * Public stuff + */ + public: + /** + * Types and constants + */ + typedef std::vector ReturnedColumnList; + typedef std::vector GroupByColumnList; + typedef std::vector OrderByColumnList; + typedef std::multimap ColumnMap; + typedef std::vector FilterTokenList; + + typedef expression::expression_parser Parser; + typedef std::vector TableList; + typedef std::vector SelectList; + + typedef std::vector RMParmVec; + + // query type of this select plan. +#undef DELETE // Windows defines this... + enum IDB_QUERYTYPE + { + SELECT, + UPDATE, + DELETE, + INSERT_SELECT, + CREATE_TABLE, + DROP_TABLE, + ALTER_TABLE, + INSERT, + LOAD_DATA_INFILE + }; + + enum SE_LOCATION + { + MAIN, + FROM, + WHERE, + HAVING + }; + + /** subselect type */ + enum SE_SubSelectType + { + MAIN_SELECT, + SINGLEROW_SUBS, + EXISTS_SUBS, + NOT_EXISTS_SUBS, + IN_SUBS, + NOT_IN_SUBS, + ALL_SUBS, + ANY_SUBS, + FROM_SUBS, + SELECT_SUBS, + UNKNOWN_SUBS + }; + + enum IDB_LOCAL_QUERY + { + GLOBAL_QUERY = 0, /*!< Standard processing */ + LOCAL_QUERY = 1, /*!< Use local ExeMgr and local PrimProc */ + LOCAL_UM = 2, /*!< Use local ExeMgr with global PrimProc*/ + }; + + /** + * Constructors + */ + CalpontSelectExecutionPlan(const int location = MAIN); + + CalpontSelectExecutionPlan(const ReturnedColumnList& returnedCols, ParseTree* filters, + const SelectList& subSelects, const GroupByColumnList& groupByCols, + ParseTree* having, const OrderByColumnList& orderByCols, const std::string alias, + const int location, const bool dependent); + + CalpontSelectExecutionPlan(const std::string data); + + /** + * Destructors + */ + virtual ~CalpontSelectExecutionPlan(); + + /** + * Access and mutator methods + */ + + /** + * returned column list + */ + const ReturnedColumnList& returnedCols() const + { + return fReturnedCols; + } + ReturnedColumnList& returnedCols() + { + return fReturnedCols; + } + void returnedCols(const ReturnedColumnList& returnedCols) + { + fReturnedCols = returnedCols; + } + + /** + * Are we in local PM only query mode? + */ + uint32_t localQuery() const + { + return fLocalQuery; + } + void localQuery(const uint32_t localQuery) + { + fLocalQuery = localQuery; + } + + /** + * filters parse tree + */ + const ParseTree* filters() const + { + return fFilters; + } + ParseTree* filters() + { + return fFilters; + } + void filters(ParseTree* filters) + { + fFilters = filters; + } + + /** filter token list + * Set filter list field and build filters tree from the filter list + * This is an alternative way to build filter tree. Hide the parser + * from the user. filterTokenList will be destroyed after the parsing + */ + // const FilterTokenList& filterTokenList() const { return fFilterTokenList;} + void filterTokenList(FilterTokenList& filterTokenList); + + /** + * sub select list + */ + const SelectList& subSelects() const + { + return fSubSelects; + } + void subSelects(const SelectList& subSelects) + { + fSubSelects = subSelects; + } + + /** + * group by column list + */ + const GroupByColumnList& groupByCols() const + { + return fGroupByCols; + } + GroupByColumnList& groupByCols() + { + return fGroupByCols; + } + void groupByCols(const GroupByColumnList& groupByCols) + { + fGroupByCols = groupByCols; + } + + /** + * order by column list + */ + const OrderByColumnList& orderByCols() const + { + return fOrderByCols; + } + OrderByColumnList& orderByCols() + { + return fOrderByCols; + } + void orderByCols(const OrderByColumnList& orderByCols) + { + fOrderByCols = orderByCols; + } + + /** + * table alias + */ + const std::string& tableAlias() const + { + return fTableAlias; + } + void tableAlias(const std::string& tableAlias, int lower_case_table_names) + { + fTableAlias = tableAlias; + if (lower_case_table_names) + boost::algorithm::to_lower(fTableAlias); + } + + /** + * location of this select + */ + int location() const + { + return fLocation; + } + void location(const int location) + { + fLocation = location; + } + + /** + * dependence of this select + */ + bool dependent() const + { + return fDependent; + } + void dependent(const bool dependent) + { + fDependent = dependent; + } + + /** + * having filter parse tree + */ + inline const ParseTree* having() const + { + return fHaving; + } + inline ParseTree* having() + { + return fHaving; + } + inline void having(ParseTree* having) + { + fHaving = having; + } + + /** having filter token list + * Set filter list field and build filters tree from the filter list + * This is an alternative way to build filter tree. Hide the parser + * from the user + */ + const FilterTokenList& havingTokenList() const + { + return fHavingTokenList; + } + void havingTokenList(const FilterTokenList& havingTokenList); + + /** column map + * all the columns appeared on query + */ + const ColumnMap& columnMap() const + { + return fColumnMap; + } + + /** column map + * all the columns appeared on query + */ + ColumnMap& columnMap() + { + return fColumnMap; + } + + /** assign the static fColMap to non-static fColumnMap. map-wise copy */ + void columnMap(const ColumnMap& columnMap); + + /** assign a regular map object to non-static fColumnMap. pure assignment */ + // @note this is to fix memory leak in CSC, becasue no static map is needed there. + inline void columnMapNonStatic(const ColumnMap& columnMap) + { + fColumnMap = columnMap; + } + + /** sql representation of this select query + * + */ + const std::string data() const + { + return fData; + } + void data(const std::string data) + { + fData = data; + } + + /** session id + * + */ + uint32_t sessionID() const + { + return fSessionID; + } + void sessionID(const uint32_t sessionID) + { + fSessionID = sessionID; + } + + /** transaction id + * + */ + int txnID() const + { + return fTxnID; + } + void txnID(const int txnID) + { + fTxnID = txnID; + } + + /** version id + * + */ + const BRM::QueryContext verID() const + { + return fVerID; + } + void verID(const BRM::QueryContext verID) + { + fVerID = verID; + } + + inline static ColumnMap& colMap() + { + return fColMap; + } + + inline std::string& schemaName() + { + return fSchemaName; + } + inline void schemaName(const std::string& schemaName, int lower_case_table_names) + { + fSchemaName = schemaName; + if (lower_case_table_names) + boost::algorithm::to_lower(fSchemaName); + } + + inline std::string& tableName() + { + return fTableName; + } + inline void tableName(const std::string& tableName, int lower_case_table_names) + { + fTableName = tableName; + if (lower_case_table_names) + boost::algorithm::to_lower(fTableName); + } + + inline void traceOn(bool traceOn) __attribute__((deprecated)) + { + if (traceOn) + fTraceFlags |= TRACE_LOG; + else + fTraceFlags &= ~TRACE_LOG; + } + inline bool traceOn() const + { + return (traceFlags() & TRACE_LOG); + } + + inline uint32_t traceFlags() const + { + return fTraceFlags; + } + inline void traceFlags(uint32_t traceFlags) + { + fTraceFlags = traceFlags; + } + + uint32_t statementID() const + { + return fStatementID; + } + void statementID(const uint32_t statementID) + { + fStatementID = statementID; + } + + const RMParmVec& rmParms() + { + return frmParms; + } + void rmParms(const RMParmVec& parms); + + const TableList& tableList() const + { + return fTableList; + } + void tableList(const TableList& tableList) + { + fTableList = tableList; + } + + const SelectList& derivedTableList() const + { + return fDerivedTableList; + } + void derivedTableList(SelectList& derivedTableList) + { + fDerivedTableList = derivedTableList; + } + + bool distinct() const + { + return fDistinct; + } + void distinct(const bool distinct) + { + fDistinct = distinct; + } + + void overrideLargeSideEstimate(const bool over) + { + fOverrideLargeSideEstimate = over; + } + bool overrideLargeSideEstimate() const + { + return fOverrideLargeSideEstimate; + } + + void unionVec(const SelectList& unionVec) + { + fUnionVec = unionVec; + } + const SelectList& unionVec() const + { + return fUnionVec; + } + SelectList& unionVec() + { + return fUnionVec; + } + + void distinctUnionNum(const uint8_t distinctUnionNum) + { + fDistinctUnionNum = distinctUnionNum; + } + uint8_t distinctUnionNum() const + { + return fDistinctUnionNum; + } + + void subType(const uint64_t subType) + { + fSubType = subType; + } + uint64_t subType() const + { + return fSubType; + } + + void derivedTbAlias(const std::string derivedTbAlias, int lower_case_table_names = 0) + { + fDerivedTbAlias = derivedTbAlias; + if (lower_case_table_names) + boost::algorithm::to_lower(fDerivedTbAlias); + } + const std::string derivedTbAlias() const + { + return fDerivedTbAlias; + } + + void derivedTbView(const std::string derivedTbView, int lower_case_table_names) + { + fDerivedTbView = derivedTbView; + if (lower_case_table_names) + boost::algorithm::to_lower(fDerivedTbView); + } + const std::string derivedTbView() const + { + return fDerivedTbView; + } + + void limitStart(const uint64_t limitStart) + { + fLimitStart = limitStart; + } + uint64_t limitStart() const + { + return fLimitStart; + } + + void limitNum(const uint64_t limitNum) + { + fLimitNum = limitNum; + } + uint64_t limitNum() const + { + return fLimitNum; + } + + void hasOrderBy(const bool hasOrderBy) + { + fHasOrderBy = hasOrderBy; + } + bool hasOrderBy() const + { + return fHasOrderBy; + } + + void specHandlerProcessed(const bool hand) + { + fSpecHandlerProcessed = hand; + } + bool specHandlerProcessed() const + { + return fSpecHandlerProcessed; + } + + void orderByThreads(const uint32_t number) + { + fOrderByThreads = number; + } + uint32_t orderByThreads() const + { + return fOrderByThreads; + } + + void selectSubList(const SelectList& selectSubList) + { + fSelectSubList = selectSubList; + } + const SelectList& selectSubList() const + { + return fSelectSubList; + } + + void subSelectList(const std::vector& subSelectList) + { + fSubSelectList = subSelectList; + } + const std::vector& subSelectList() const + { + return fSubSelectList; + } + + void stringScanThreshold(uint64_t n) + { + fStringScanThreshold = n; + } + uint64_t stringScanThreshold() const + { + return fStringScanThreshold; + } + + // query type. return string for easy stats insert + void queryType(const uint32_t queryType) + { + fQueryType = queryType; + } + const std::string queryType() const + { + return queryTypeToString(fQueryType); + } + static std::string queryTypeToString(const uint32_t queryType); + + void priority(uint32_t p) + { + fPriority = p; + } + uint32_t priority() const + { + return fPriority; + } + + void stringTableThreshold(uint32_t t) + { + fStringTableThreshold = t; + } + uint32_t stringTableThreshold() const + { + return fStringTableThreshold; + } + + void uuid(const boost::uuids::uuid& uuid) + { + fUuid = uuid; + } + const boost::uuids::uuid& uuid() const + { + return fUuid; + } + + void djsSmallSideLimit(uint64_t l) + { + fDJSSmallSideLimit = l; + } + uint64_t djsSmallSideLimit() + { + return fDJSSmallSideLimit; + } + + void djsLargeSideLimit(uint64_t l) + { + fDJSLargeSideLimit = l; + } + uint64_t djsLargeSideLimit() + { + return fDJSLargeSideLimit; + } + + void djsPartitionSize(uint64_t l) + { + fDJSPartitionSize = l; + } + uint64_t djsPartitionSize() + { + return fDJSPartitionSize; + } + + void umMemLimit(uint64_t l) + { + fUMMemLimit = l; + } + int64_t umMemLimit() + { + return fUMMemLimit; + } + + void isDML(bool b) + { + fIsDML = b; + } + bool isDML() + { + return fIsDML; + } + + void timeZone(const std::string& timezone) + { + fTimeZone = timezone; + } + const std::string timeZone() const + { + return fTimeZone; + } + + /** + * The serialization interface + */ + /** + * @note Serialize() assumes that none of the vectors contain NULL pointers. + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const CalpontExecutionPlan* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const CalpontSelectExecutionPlan& t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const CalpontExecutionPlan* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const CalpontSelectExecutionPlan& t) const; + + /** @brief Return a string rep of the CSEP + * + * Return a string rep of the CSEP + * @return a string + */ + virtual std::string toString() const; + + /** @brief Is this an internal query? + * + * Is this an internal query (a syscat query performed on behalf of another query) + * FIXME: add a setter and make this work for really big session ids + * @return true/false + */ + virtual bool isInternal() const + { + return ((fSessionID & 0x80000000) != 0); + } + + virtual void setDynamicParseTreeVec(const std::vector& dynamicParseTreeVec) + { + fDynamicParseTreeVec = dynamicParseTreeVec; + } + + /** + * Protected stuff + */ + protected: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Private stuff + */ + private: + /** + * If set, then the local PM only option is turned on + */ + uint32_t fLocalQuery; + + /** + * A list of ReturnedColumn objects + */ + ReturnedColumnList fReturnedCols; + /** + * A list of filter tokens including filters and operators + * This is a helper data structure, it holds the tokens of + * the filter parse tree and will be built into a parse tree + * in the mutator method. + * @note the elements in this list will be deleted when fFilters + * parse tree is deleted. So this list should not be deleted + * again in destructor. + */ + FilterTokenList fFilterTokenList; + FilterTokenList fHavingTokenList; + + /** + * A tree of Filter objects + */ + ParseTree* fFilters; + /** + * A list of CalpontExecutionPlan objects + */ + SelectList fSubSelects; + /** + * A list of group by columns + */ + GroupByColumnList fGroupByCols; + /** + * A tree of having clause condition associated with group by clause + */ + ParseTree* fHaving; + /** + * A list of order by columns + */ + OrderByColumnList fOrderByCols; + /** + * Table or alias name for subselect in FROM clause + */ + std::string fTableAlias; + /** + * An enum indicating the location of this select statement in the enclosing select statement + */ + int fLocation; + /** + * A flag indicating if this sub-select is dependent on the enclosing query or is constant + */ + bool fDependent; + + /** + * SQL representation of this execution plan + */ + std::string fData; + static ColumnMap fColMap; // for getplan to use. class-wise map + ColumnMap fColumnMap; // for ExeMgr to use. not shared between objects + + uint32_t fSessionID; + int fTxnID; // SQLEngine only needs the ID value + BRM::QueryContext fVerID; + // @bug5316. remove static + std::string fSchemaName; + std::string fTableName; + uint32_t fTraceFlags; + + /** + * One-up statementID number for this session (fSessionID) + */ + uint32_t fStatementID; + + RMParmVec frmParms; + TableList fTableList; + SelectList fDerivedTableList; + + bool fDistinct; + bool fOverrideLargeSideEstimate; + + // for union + SelectList fUnionVec; + uint8_t fDistinctUnionNum; + + // for subselect + uint64_t fSubType; + std::string fDerivedTbAlias; + std::string fDerivedTbView; + + // for limit + uint64_t fLimitStart; + uint64_t fLimitNum; + + // for parent select order by + bool fHasOrderBy; + + // for Select clause subquery + SelectList fSelectSubList; + + // @bug3321, for string scan blocks + uint64_t fStringScanThreshold; + + // query type + uint32_t fQueryType; + + uint32_t fPriority; + uint32_t fStringTableThreshold; + + // for specific handlers processing, e.g. GROUP BY + bool fSpecHandlerProcessed; + uint32_t fOrderByThreads; + + // Derived table involved in the query. For derived table optimization + std::vector fSubSelectList; + + boost::uuids::uuid fUuid; + + /* Disk-based join vars */ + uint64_t fDJSSmallSideLimit; + uint64_t fDJSLargeSideLimit; + uint64_t fDJSPartitionSize; + int64_t fUMMemLimit; + bool fIsDML; + + std::string fTimeZone; + + std::vector fDynamicParseTreeVec; }; /** @@ -937,11 +935,10 @@ private: inline std::ostream& operator<<(std::ostream& os, const CalpontSelectExecutionPlan& rhs) { - os << rhs.toString(); - return os; + os << rhs.toString(); + return os; } -} -#endif //CALPONTSELECTEXECUTIONPLAN_H +} // namespace execplan +#endif // CALPONTSELECTEXECUTIONPLAN_H // vim:ts=4 sw=4: - diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index f122fbae8..a2f5ea525 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -78,16 +78,15 @@ using namespace rowgroup; #include "idbregistry.h" #endif - #undef BAIL_IF_0 #if 1 -//We are unlikely to ever get anything more out of this connection, so bail out -#define BAIL_IF_0(m) \ -if ((m).length() == 0) \ -{ \ -fExeMgr->shutdown(); \ -throw runtime_error("CALPONT_INTERNAL_ERROR"); \ -} +// We are unlikely to ever get anything more out of this connection, so bail out +#define BAIL_IF_0(m) \ + if ((m).length() == 0) \ + { \ + fExeMgr->shutdown(); \ + throw runtime_error("CALPONT_INTERNAL_ERROR"); \ + } #else #define BAIL_IF_0(m) #endif @@ -97,330 +96,278 @@ throw runtime_error("CALPONT_INTERNAL_ERROR"); \ #if CSC_DEBUG namespace { - string tmpDir = startup::StartUp::tmpDir() + "/csc.log"; - std::ofstream csclog(tmpDir, std::ios::app); -} +string tmpDir = startup::StartUp::tmpDir() + "/csc.log"; +std::ofstream csclog(tmpDir, std::ios::app); +} // namespace #define DEBUG csclog #else -#define DEBUG if (false) cerr +#define DEBUG \ + if (false) \ + cerr #endif namespace execplan { - const SOP opeq(new Operator("=")); const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) { - switch (cdt) - { - case CalpontSystemCatalog::BIT: - return "bit"; - break; + switch (cdt) + { + case CalpontSystemCatalog::BIT: return "bit"; break; - case CalpontSystemCatalog::TINYINT: - return "tinyint"; - break; + case CalpontSystemCatalog::TINYINT: return "tinyint"; break; - case CalpontSystemCatalog::CHAR: - return "char"; - break; + case CalpontSystemCatalog::CHAR: return "char"; break; - case CalpontSystemCatalog::SMALLINT: - return "smallint"; - break; + case CalpontSystemCatalog::SMALLINT: return "smallint"; break; - case CalpontSystemCatalog::DECIMAL: - return "decimal"; - break; + case CalpontSystemCatalog::DECIMAL: return "decimal"; break; - case CalpontSystemCatalog::MEDINT: - return "medint"; - break; + case CalpontSystemCatalog::MEDINT: return "medint"; break; - case CalpontSystemCatalog::INT: - return "int"; - break; + case CalpontSystemCatalog::INT: return "int"; break; - case CalpontSystemCatalog::FLOAT: - return "float"; - break; + case CalpontSystemCatalog::FLOAT: return "float"; break; - case CalpontSystemCatalog::DATE: - return "date"; - break; + case CalpontSystemCatalog::DATE: return "date"; break; - case CalpontSystemCatalog::BIGINT: - return "bigint"; - break; + case CalpontSystemCatalog::BIGINT: return "bigint"; break; - case CalpontSystemCatalog::DOUBLE: - return "double"; - break; + case CalpontSystemCatalog::DOUBLE: return "double"; break; - case CalpontSystemCatalog::LONGDOUBLE: - return "long double"; - break; + case CalpontSystemCatalog::LONGDOUBLE: return "long double"; break; - case CalpontSystemCatalog::DATETIME: - return "datetime"; - break; + case CalpontSystemCatalog::DATETIME: return "datetime"; break; - case CalpontSystemCatalog::TIME: - return "time"; - break; + case CalpontSystemCatalog::TIME: return "time"; break; - case CalpontSystemCatalog::TIMESTAMP: - return "timestamp"; - break; + case CalpontSystemCatalog::TIMESTAMP: return "timestamp"; break; - case CalpontSystemCatalog::VARCHAR: - return "varchar"; - break; + case CalpontSystemCatalog::VARCHAR: return "varchar"; break; - case CalpontSystemCatalog::VARBINARY: - return "varbinary"; - break; + case CalpontSystemCatalog::VARBINARY: return "varbinary"; break; - case CalpontSystemCatalog::CLOB: - return "clob"; - break; + case CalpontSystemCatalog::CLOB: return "clob"; break; - case CalpontSystemCatalog::BLOB: - return "blob"; - break; + case CalpontSystemCatalog::BLOB: return "blob"; break; - case CalpontSystemCatalog::TEXT: - return "text"; - break; + case CalpontSystemCatalog::TEXT: return "text"; break; - case CalpontSystemCatalog::UTINYINT: - return "utinyint"; - break; + case CalpontSystemCatalog::UTINYINT: return "utinyint"; break; - case CalpontSystemCatalog::USMALLINT: - return "usmallint"; - break; + case CalpontSystemCatalog::USMALLINT: return "usmallint"; break; - case CalpontSystemCatalog::UDECIMAL: - return "udecimal"; - break; + case CalpontSystemCatalog::UDECIMAL: return "udecimal"; break; - case CalpontSystemCatalog::UMEDINT: - return "umedint"; - break; + case CalpontSystemCatalog::UMEDINT: return "umedint"; break; - case CalpontSystemCatalog::UINT: - return "uint32_t"; - break; + case CalpontSystemCatalog::UINT: return "uint32_t"; break; - case CalpontSystemCatalog::UFLOAT: - return "ufloat"; - break; + case CalpontSystemCatalog::UFLOAT: return "ufloat"; break; - case CalpontSystemCatalog::UBIGINT: - return "ubigint"; - break; + case CalpontSystemCatalog::UBIGINT: return "ubigint"; break; - case CalpontSystemCatalog::UDOUBLE: - return "udouble"; - break; + case CalpontSystemCatalog::UDOUBLE: return "udouble"; break; - default: - break; - } + default: break; + } - return "invalid!"; + return "invalid!"; } -} +} // namespace execplan namespace execplan { - typedef CalpontSelectExecutionPlan::ColumnMap::value_type CMVT_; boost::shared_ptr fSessionManager; CalpontSystemCatalog::NJLSysDataList::~NJLSysDataList() { - NJLSysDataVector::iterator it; + NJLSysDataVector::iterator it; - for (it = sysDataVec.begin(); it != sysDataVec.end(); it++) - delete *it; + for (it = sysDataVec.begin(); it != sysDataVec.end(); it++) + delete *it; } -const CalpontSystemCatalog::TableColName make_tcn(const string& s, const string& t, const string& c, int lower_case_table_names) +const CalpontSystemCatalog::TableColName make_tcn(const string& s, const string& t, const string& c, + int lower_case_table_names) { - CalpontSystemCatalog::TableColName tcns; + CalpontSystemCatalog::TableColName tcns; - tcns.schema = s; - tcns.table = t; - tcns.column = c; - if (lower_case_table_names) - { - boost::algorithm::to_lower(tcns.schema); - boost::algorithm::to_lower(tcns.table); - } - boost::algorithm::to_lower(tcns.column); - return tcns; + tcns.schema = s; + tcns.table = t; + tcns.column = c; + if (lower_case_table_names) + { + boost::algorithm::to_lower(tcns.schema); + boost::algorithm::to_lower(tcns.table); + } + boost::algorithm::to_lower(tcns.column); + return tcns; } const CalpontSystemCatalog::TableName make_table(const string& s, const string& t, int lower_case_table_names) { - CalpontSystemCatalog::TableName tn; - tn.schema = s; - tn.table = t; - if (lower_case_table_names) - { - boost::algorithm::to_lower(tn.schema); - boost::algorithm::to_lower(tn.table); - } - return tn; + CalpontSystemCatalog::TableName tn; + tn.schema = s; + tn.table = t; + if (lower_case_table_names) + { + boost::algorithm::to_lower(tn.schema); + boost::algorithm::to_lower(tn.table); + } + return tn; } -const CalpontSystemCatalog::TableAliasName make_aliastable(const string& s, const string& t, const string& a, const bool isColumnStore, int lower_case_table_names) +const CalpontSystemCatalog::TableAliasName make_aliastable(const string& s, const string& t, const string& a, + const bool isColumnStore, + int lower_case_table_names) { - CalpontSystemCatalog::TableAliasName tn; - tn.schema = s; - tn.table = t; - tn.alias = a; - tn.view = ""; - tn.fisColumnStore = isColumnStore; - if (lower_case_table_names) - { - boost::algorithm::to_lower(tn.schema); - boost::algorithm::to_lower(tn.table); - boost::algorithm::to_lower(tn.alias); - } - return tn; + CalpontSystemCatalog::TableAliasName tn; + tn.schema = s; + tn.table = t; + tn.alias = a; + tn.view = ""; + tn.fisColumnStore = isColumnStore; + if (lower_case_table_names) + { + boost::algorithm::to_lower(tn.schema); + boost::algorithm::to_lower(tn.table); + boost::algorithm::to_lower(tn.alias); + } + return tn; } -const CalpontSystemCatalog::TableAliasName make_aliasview(const string& s, const string& t, const string& a, const string& v, const bool isColumnStore, int lower_case_table_names) +const CalpontSystemCatalog::TableAliasName make_aliasview(const string& s, const string& t, const string& a, + const string& v, const bool isColumnStore, + int lower_case_table_names) { - CalpontSystemCatalog::TableAliasName tn; - tn.schema = s; - tn.table = t; - tn.alias = a; - tn.view = v; - tn.fisColumnStore = isColumnStore; - if (lower_case_table_names) - { - boost::algorithm::to_lower(tn.schema); - boost::algorithm::to_lower(tn.table); - boost::algorithm::to_lower(tn.alias); - boost::algorithm::to_lower(tn.view); - } - return tn; + CalpontSystemCatalog::TableAliasName tn; + tn.schema = s; + tn.table = t; + tn.alias = a; + tn.view = v; + tn.fisColumnStore = isColumnStore; + if (lower_case_table_names) + { + boost::algorithm::to_lower(tn.schema); + boost::algorithm::to_lower(tn.table); + boost::algorithm::to_lower(tn.alias); + boost::algorithm::to_lower(tn.view); + } + return tn; } bool CalpontSystemCatalog::TableColName::operator<(const TableColName& rhs) const { - if (schema < rhs.schema) + if (schema < rhs.schema) + { + return true; + } + else if (schema == rhs.schema) + { + if (table < rhs.table) { + return true; + } + else if (table == rhs.table) + { + if (column < rhs.column) + { return true; + } } - else if (schema == rhs.schema) - { - if (table < rhs.table) - { - return true; - } - else if (table == rhs.table) - { - if (column < rhs.column) - { - return true; - } - } - } + } - return false; + return false; } const string CalpontSystemCatalog::TableColName::toString() const { - string os; - os = schema + '.' + table + '.' + column; - return os; + string os; + os = schema + '.' + table + '.' + column; + return os; } bool CalpontSystemCatalog::TableName::operator<(const TableName& rhs) const { - if (schema < rhs.schema) + if (schema < rhs.schema) + { + return true; + } + else if (schema == rhs.schema) + { + if (table < rhs.table) { - return true; - } - else if (schema == rhs.schema) - { - if (table < rhs.table) - { - return true; - } + return true; } + } - return false; + return false; } void CalpontSystemCatalog::TableAliasName::clear() { - schema.clear(); - table.clear(); - alias.clear(); - view.clear(); + schema.clear(); + table.clear(); + alias.clear(); + view.clear(); } bool CalpontSystemCatalog::TableAliasName::operator<(const TableAliasName& rhs) const { - if (schema < rhs.schema) + if (schema < rhs.schema) + { + return true; + } + else if (schema == rhs.schema) + { + if (table < rhs.table) { - return true; + return true; } - else if (schema == rhs.schema) + else if (table == rhs.table) { - if (table < rhs.table) + if (alias < rhs.alias) + { + return true; + } + else if (alias == rhs.alias) + { + if (view < rhs.view) { + return true; + } + else if (view == rhs.view) + { + if (fisColumnStore < rhs.fisColumnStore) return true; } - else if (table == rhs.table) - { - if (alias < rhs.alias) - { - return true; - } - else if (alias == rhs.alias) - { - if (view < rhs.view) - { - return true; - } - else if (view == rhs.view) - { - if (fisColumnStore < rhs.fisColumnStore) - return true; - } - } - } + } } + } - return false; + return false; } void CalpontSystemCatalog::TableAliasName::serialize(messageqcpp::ByteStream& b) const { - b << schema; - b << table; - b << alias; - b << view; - b << static_cast(fisColumnStore); + b << schema; + b << table; + b << alias; + b << view; + b << static_cast(fisColumnStore); } void CalpontSystemCatalog::TableAliasName::unserialize(messageqcpp::ByteStream& b) { - b >> schema; - b >> table; - b >> alias; - b >> view; - b >> reinterpret_cast< ByteStream::doublebyte&>(fisColumnStore); + b >> schema; + b >> table; + b >> alias; + b >> view; + b >> reinterpret_cast(fisColumnStore); } /*static*/ @@ -430,1472 +377,1478 @@ CalpontSystemCatalog::CatalogMap CalpontSystemCatalog::fCatalogMap; /*static*/ uint32_t CalpontSystemCatalog::fModuleID = numeric_limits::max(); -CalpontSystemCatalog::OID CalpontSystemCatalog::lookupTableOID(const TableName& tablename, int lower_case_table_names) +CalpontSystemCatalog::OID CalpontSystemCatalog::lookupTableOID(const TableName& tablename, + int lower_case_table_names) { - TableName aTableName; - aTableName.schema = tablename.schema; - aTableName.table = tablename.table; - if (lower_case_table_names) + TableName aTableName; + aTableName.schema = tablename.schema; + aTableName.table = tablename.table; + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } + + if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) + DEBUG << "Enter tableInfo: " << tablename.schema << "|" << tablename.table << endl; + + // select objectid from systable where schema = tableName.schema and tablename = tableName.table; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); + SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c3 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); + + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + OID oid = c1->oid(); + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c2->clone(), new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, c3->clone(), new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select objectid from systable where schema='" << aTableName.schema << "' and tablename='" + << aTableName.table << "' --tableRID/"; + + csep.data(oss.str()); //@bug 6078. Log the statement + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + NJLSysDataList sysDataList; + + try + { + getSysData(csep, sysDataList, SYSTABLE_TABLE); + } + catch (IDBExcept&) + { + throw; + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } + + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->dataCount() == 0) { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); + return (OID)0; } - if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) - DEBUG << "Enter tableInfo: " << tablename.schema << "|" << tablename.table << endl; + if ((*it)->ColumnOID() == oid) + { + if (fIdentity == EC) + return (*it)->GetRid(0); + else + return (OID)((*it)->GetData(0)); + } + } - // select objectid from systable where schema = tableName.schema and tablename = tableName.table; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; + return (OID)0; +} - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); +CalpontSystemCatalog::OID CalpontSystemCatalog::lookupOID(const TableColName& tableColName, + int lower_case_table_names) +{ + if (tableColName.schema.length() == 0 || tableColName.table.length() == 0 || + tableColName.column.length() == 0) + return -1; - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); - csep.columnMapNonStatic(colMap); + TableColName aTableColName; + aTableColName.schema = tableColName.schema; + aTableColName.table = tableColName.table; + aTableColName.column = tableColName.column; + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableColName.schema); + boost::algorithm::to_lower(aTableColName.table); + } + boost::algorithm::to_lower(aTableColName.column); - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - OID oid = c1->oid(); + if (aTableColName.schema.compare(CALPONT_SCHEMA) != 0) + DEBUG << "Enter lookupOID: " << tableColName.schema << "|" << tableColName.table << "|" + << tableColName.column << endl; - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c2->clone(), - new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); + // Check whether cache needs to be flushed + if (aTableColName.schema.compare(CALPONT_SCHEMA) != 0) + { + checkSysCatVer(); + } - SimpleFilter* f2 = new SimpleFilter (opeq, - c3->clone(), - new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - csep.filterTokenList(filterTokenList); + boost::mutex::scoped_lock lk2(fOIDmapLock); - ostringstream oss; - oss << "select objectid from systable where schema='" << aTableName.schema << "' and tablename='" << - aTableName.table << "' --tableRID/"; + if (fOIDmap.size() > 0) + { + OIDmap::const_iterator iter = fOIDmap.find(aTableColName); - csep.data(oss.str()); //@bug 6078. Log the statement + // @bug 1358. double check fColRIDMap in case it's not filled with valid rid. + if (iter != fOIDmap.end()) + { + if (fIdentity == EC && aTableColName.schema.compare(CALPONT_SCHEMA) != 0) + { + ColRIDmap::const_iterator iter1 = fColRIDmap.find(aTableColName); - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; + if (iter1 != fColRIDmap.end()) + { + return iter->second; + } + } + else + { + return iter->second; + } + } + } - NJLSysDataList sysDataList; + lk2.unlock(); + // select objectid,columnlength,datatype,dictobjectid,listobjectid,treeobjectid,columnposition,scale,prec, + // defaultvalue from syscolumn where schema=schema and tablename=table and columnname=column; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; + string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; + string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; + string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; + string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; + string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; + string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; + string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; + // the following columns will be save in cache although it's not needed for now + string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; + string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; + string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; + string compressiontype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; + string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; + string nextVal = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; + string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; + + SimpleColumn* col[17]; + col[0] = new SimpleColumn(columnlength, fSessionID); + col[1] = new SimpleColumn(objectid, fSessionID); + col[2] = new SimpleColumn(datatype, fSessionID); + col[3] = new SimpleColumn(dictobjectid, fSessionID); + col[4] = new SimpleColumn(listobjectid, fSessionID); + col[5] = new SimpleColumn(treeobjectid, fSessionID); + col[6] = new SimpleColumn(columnposition, fSessionID); + col[7] = new SimpleColumn(scale, fSessionID); + col[8] = new SimpleColumn(precision, fSessionID); + col[9] = new SimpleColumn(defaultvalue, fSessionID); + col[10] = new SimpleColumn(schemaname, fSessionID); + col[11] = new SimpleColumn(tablename, fSessionID); + col[12] = new SimpleColumn(columnname, fSessionID); + col[13] = new SimpleColumn(compressiontype, fSessionID); + col[14] = new SimpleColumn(autoincrement, fSessionID); + col[15] = new SimpleColumn(nextVal, fSessionID); + col[16] = new SimpleColumn(nullable, fSessionID); + + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(columnlength, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(objectid, srcp)); + srcp.reset(col[2]); + colMap.insert(CMVT_(datatype, srcp)); + srcp.reset(col[3]); + colMap.insert(CMVT_(dictobjectid, srcp)); + srcp.reset(col[4]); + colMap.insert(CMVT_(listobjectid, srcp)); + srcp.reset(col[5]); + colMap.insert(CMVT_(treeobjectid, srcp)); + srcp.reset(col[6]); + colMap.insert(CMVT_(columnposition, srcp)); + srcp.reset(col[7]); + colMap.insert(CMVT_(scale, srcp)); + srcp.reset(col[8]); + colMap.insert(CMVT_(precision, srcp)); + // TODO: NULL value handling + srcp.reset(col[9]); + colMap.insert(CMVT_(defaultvalue, srcp)); + srcp.reset(col[10]); + colMap.insert(CMVT_(schemaname, srcp)); + srcp.reset(col[11]); + colMap.insert(CMVT_(tablename, srcp)); + srcp.reset(col[12]); + colMap.insert(CMVT_(columnname, srcp)); + srcp.reset(col[13]); + colMap.insert(CMVT_(compressiontype, srcp)); + srcp.reset(col[14]); + colMap.insert(CMVT_(autoincrement, srcp)); + srcp.reset(col[15]); + colMap.insert(CMVT_(nextVal, srcp)); + srcp.reset(col[16]); + colMap.insert(CMVT_(nullable, srcp)); + csep.columnMapNonStatic(colMap); + + // ignore returnedcolumn, because it's not read by Joblist for now + csep.returnedCols(returnedColumnList); + OID oid[17]; + + for (int i = 0; i < 17; i++) + oid[i] = col[i]->oid(); + + // Filters + SimpleFilter* f1 = new SimpleFilter(opeq, col[10]->clone(), + new ConstantColumn(aTableColName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = new SimpleFilter(opeq, col[11]->clone(), + new ConstantColumn(aTableColName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = new SimpleFilter(opeq, col[12]->clone(), + new ConstantColumn(aTableColName.column, ConstantColumn::LITERAL)); + filterTokenList.push_back(f3); + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select objectid,columnlength,datatype,dictobjectid,listobjectid,treeobjectid,columnposition,scale," + "prec,defaultvalue from syscolumn where schema='" + << aTableColName.schema << "' and tablename='" << aTableColName.table << "' and columnname='" + << aTableColName.column << "' --lookupOID/"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + TableColName tcn; + ColType ct; + OID coloid = -1; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[1]) + { + // populate cache + coloid = (OID)((*it)->GetData(0)); + lk2.lock(); + fOIDmap[aTableColName] = coloid; + + if (fIdentity == EC) + fColRIDmap[aTableColName] = (*it)->GetRid(0); + + // @bug 1358. do not insert this entry to map + // else + // fColRIDmap[aTableColName] = 0; + lk2.unlock(); + } + + if ((*it)->ColumnOID() == oid[0]) + ct.colWidth = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[2]) + ct.colDataType = (ColDataType)((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[3]) + ct.ddn.dictOID = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[4]) + ct.ddn.listOID = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[5]) + ct.ddn.treeOID = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[6]) + ct.colPosition = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[7]) + ct.scale = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[8]) + ct.precision = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[13]) + ct.compressionType = ct.ddn.compressionType = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[14]) + { + ostringstream os; + os << (char)(*it)->GetData(0); + + if (os.str().compare("y") == 0) + ct.autoincrement = true; + else + ct.autoincrement = false; + } + else if ((*it)->ColumnOID() == oid[15]) + ct.nextvalue = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[16]) + { + if (static_cast((*it)->GetData(0)) == 0) + { + ct.constraintType = NOTNULL_CONSTRAINT; + } + } + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) + { + ct.defaultValue = ((*it)->GetStringData(0)); + + if ((!ct.defaultValue.empty()) || (ct.defaultValue.length() > 0)) + { + if (ct.constraintType != NOTNULL_CONSTRAINT) + ct.constraintType = DEFAULT_CONSTRAINT; + } + } + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_SCHEMA) + tcn.schema = ((*it)->GetStringData(0)); + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_TABLENAME) + tcn.table = ((*it)->GetStringData(0)); + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_COLNAME) + tcn.column = ((*it)->GetStringData(0)); + } + + // temporialy memory leak fix until defaultvalue is added. + // delete col[9]; + ct.columnOID = coloid; + // populate colinfomap cache and oidbitmap + boost::mutex::scoped_lock lk3(fColinfomapLock); + fColinfomap[coloid] = ct; + return coloid; +} + +void CalpontSystemCatalog::getSysData(CalpontSelectExecutionPlan& csep, NJLSysDataList& sysDataList, + const string& sysTableName) +{ + // start up new transaction + + BRM::TxnID txnID; + int oldTxnID; + txnID = fSessionManager->getTxnID(fSessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + BRM::QueryContext verID, oldVerID; + verID = fSessionManager->verID(); + oldTxnID = csep.txnID(); + csep.txnID(txnID.id); + oldVerID = csep.verID(); + csep.verID(verID); + // We need to use a session ID that's separate from the actual query SID, because the dbcon runs queries + // in the middle of receiving data bands for the real query. + // TODO: we really need a flag or something to identify this as a syscat query: there are assumptions made + // in joblist that a high-bit-set session id is always a syscat query. This will be okay for a long time, + // but not forever... + + csep.sessionID(fSessionID | 0x80000000); + int tryCnt = 0; + + // add the tableList to csep for tuple joblist to use + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable("calpontsys", sysTableName, "")); + csep.tableList(tablelist); + + // populate the returned column list as column map + csep.returnedCols().clear(); + CalpontSelectExecutionPlan::ColumnMap::const_iterator it; + + for (it = csep.columnMap().begin(); it != csep.columnMap().end(); ++it) + { + csep.returnedCols().push_back(it->second); + } + + if (fIdentity == EC) + { try { - getSysData (csep, sysDataList, SYSTABLE_TABLE); + getSysData_EC(csep, sysDataList, sysTableName); } - catch ( IDBExcept& ) + catch (IDBExcept&) { + throw; + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } + } + else + { + while (tryCnt < 5) + { + tryCnt++; + + try + { + getSysData_FE(csep, sysDataList, sysTableName); + break; + } + catch (IDBExcept&) // error already occured. this is not a broken pipe + { throw; - } - catch ( runtime_error& e ) - { - throw runtime_error ( e.what() ); - } + } + catch (...) + { + // may be a broken pipe. re-establish exeMgr and send the message + delete fExeMgr; + fExeMgr = new ClientRotator(0, "ExeMgr"); - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->dataCount() == 0) - { - return (OID)0; - } - - if ((*it)->ColumnOID() == oid) - { - if (fIdentity == EC) - return (*it)->GetRid(0); - else - return (OID)((*it)->GetData(0)); - } - } - - return (OID)0; -} - -CalpontSystemCatalog::OID CalpontSystemCatalog::lookupOID(const TableColName& tableColName, int lower_case_table_names) -{ - if (tableColName.schema.length() == 0 || tableColName.table.length() == 0 || tableColName.column.length() == 0) - return -1; - - TableColName aTableColName; - aTableColName.schema = tableColName.schema; - aTableColName.table = tableColName.table; - aTableColName.column = tableColName.column; - if (lower_case_table_names) - { - boost::algorithm::to_lower(aTableColName.schema); - boost::algorithm::to_lower(aTableColName.table); - } - boost::algorithm::to_lower(aTableColName.column); - - if (aTableColName.schema.compare(CALPONT_SCHEMA) != 0) - DEBUG << "Enter lookupOID: " << tableColName.schema << "|" << tableColName.table - << "|" << tableColName.column << endl; - - //Check whether cache needs to be flushed - if ( aTableColName.schema.compare(CALPONT_SCHEMA) != 0) - { - checkSysCatVer(); - } - - boost::mutex::scoped_lock lk2(fOIDmapLock); - - if ( fOIDmap.size() > 0 ) - { - OIDmap::const_iterator iter = fOIDmap.find(aTableColName); - - // @bug 1358. double check fColRIDMap in case it's not filled with valid rid. - if (iter != fOIDmap.end()) - { - if (fIdentity == EC && aTableColName.schema.compare(CALPONT_SCHEMA) != 0) - { - ColRIDmap::const_iterator iter1 = fColRIDmap.find(aTableColName); - - if ( iter1 != fColRIDmap.end() ) - { - return iter->second; - } - } - else - { - return iter->second; - } - } - } - - lk2.unlock(); - - // select objectid,columnlength,datatype,dictobjectid,listobjectid,treeobjectid,columnposition,scale,prec, - // defaultvalue from syscolumn where schema=schema and tablename=table and columnname=column; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; - string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; - string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; - string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; - string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; - string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; - string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; - string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; - // the following columns will be save in cache although it's not needed for now - string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; - string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; - string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; - string compressiontype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; - string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; - string nextVal = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; - string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; - - SimpleColumn* col[17]; - col[0] = new SimpleColumn(columnlength, fSessionID); - col[1] = new SimpleColumn(objectid, fSessionID); - col[2] = new SimpleColumn(datatype, fSessionID); - col[3] = new SimpleColumn(dictobjectid, fSessionID); - col[4] = new SimpleColumn(listobjectid, fSessionID); - col[5] = new SimpleColumn(treeobjectid, fSessionID); - col[6] = new SimpleColumn(columnposition, fSessionID); - col[7] = new SimpleColumn(scale, fSessionID); - col[8] = new SimpleColumn(precision, fSessionID); - col[9] = new SimpleColumn(defaultvalue, fSessionID); - col[10] = new SimpleColumn(schemaname, fSessionID); - col[11] = new SimpleColumn(tablename, fSessionID); - col[12] = new SimpleColumn(columnname, fSessionID); - col[13] = new SimpleColumn(compressiontype, fSessionID); - col[14] = new SimpleColumn(autoincrement, fSessionID); - col[15] = new SimpleColumn(nextVal, fSessionID); - col[16] = new SimpleColumn(nullable, fSessionID); - - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(columnlength, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(objectid, srcp)); - srcp.reset(col[2]); - colMap.insert(CMVT_(datatype, srcp)); - srcp.reset(col[3]); - colMap.insert(CMVT_(dictobjectid, srcp)); - srcp.reset(col[4]); - colMap.insert(CMVT_(listobjectid, srcp)); - srcp.reset(col[5]); - colMap.insert(CMVT_(treeobjectid, srcp)); - srcp.reset(col[6]); - colMap.insert(CMVT_(columnposition, srcp)); - srcp.reset(col[7]); - colMap.insert(CMVT_(scale, srcp)); - srcp.reset(col[8]); - colMap.insert(CMVT_(precision, srcp)); - // TODO: NULL value handling - srcp.reset(col[9]); - colMap.insert(CMVT_(defaultvalue, srcp)); - srcp.reset(col[10]); - colMap.insert(CMVT_(schemaname, srcp)); - srcp.reset(col[11]); - colMap.insert(CMVT_(tablename, srcp)); - srcp.reset(col[12]); - colMap.insert(CMVT_(columnname, srcp)); - srcp.reset(col[13]); - colMap.insert(CMVT_(compressiontype, srcp)); - srcp.reset(col[14]); - colMap.insert(CMVT_(autoincrement, srcp)); - srcp.reset(col[15]); - colMap.insert(CMVT_(nextVal, srcp)); - srcp.reset(col[16]); - colMap.insert(CMVT_(nullable, srcp)); - csep.columnMapNonStatic(colMap); - - // ignore returnedcolumn, because it's not read by Joblist for now - csep.returnedCols(returnedColumnList); - OID oid[17]; - - for (int i = 0; i < 17; i++) - oid[i] = col[i]->oid(); - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[10]->clone(), - new ConstantColumn(aTableColName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - col[11]->clone(), - new ConstantColumn(aTableColName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f3 = new SimpleFilter (opeq, - col[12]->clone(), - new ConstantColumn(aTableColName.column, ConstantColumn::LITERAL)); - filterTokenList.push_back(f3); - csep.filterTokenList(filterTokenList); - - ostringstream oss; - oss << "select objectid,columnlength,datatype,dictobjectid,listobjectid,treeobjectid,columnposition,scale," - "prec,defaultvalue from syscolumn where schema='" << aTableColName.schema << "' and tablename='" << - aTableColName.table << "' and columnname='" << aTableColName.column << "' --lookupOID/"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - TableColName tcn; - ColType ct; - OID coloid = -1; - getSysData (csep, sysDataList, SYSCOLUMN_TABLE); - - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid[1]) - { - // populate cache - coloid = (OID)((*it)->GetData(0)); - lk2.lock(); - fOIDmap[aTableColName] = coloid; - - if (fIdentity == EC) - fColRIDmap[aTableColName] = (*it)->GetRid(0); - - // @bug 1358. do not insert this entry to map - //else - // fColRIDmap[aTableColName] = 0; - lk2.unlock(); - } - - if ((*it)->ColumnOID() == oid[0]) - ct.colWidth = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[2]) - ct.colDataType = (ColDataType)((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[3]) - ct.ddn.dictOID = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[4]) - ct.ddn.listOID = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[5]) - ct.ddn.treeOID = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[6]) - ct.colPosition = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[7]) - ct.scale = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[8]) - ct.precision = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[13]) - ct.compressionType = ct.ddn.compressionType = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[14]) - { - ostringstream os; - os << (char) (*it)->GetData(0); - - if (os.str().compare("y") == 0) - ct.autoincrement = true; - else - ct.autoincrement = false; - } - else if ((*it)->ColumnOID() == oid[15]) - ct.nextvalue = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[16]) - { - if (static_cast ((*it)->GetData(0)) == 0) - { - ct.constraintType = NOTNULL_CONSTRAINT; - } - } - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) - { - ct.defaultValue = ((*it)->GetStringData(0)); - - if ((!ct.defaultValue.empty()) || (ct.defaultValue.length() > 0)) - { - if (ct.constraintType != NOTNULL_CONSTRAINT) - ct.constraintType = DEFAULT_CONSTRAINT; - } - } - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_SCHEMA) - tcn.schema = ((*it)->GetStringData(0)); - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_TABLENAME) - tcn.table = ((*it)->GetStringData(0)); - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_COLNAME) - tcn.column = ((*it)->GetStringData(0)); - } - - // temporialy memory leak fix until defaultvalue is added. - // delete col[9]; - ct.columnOID = coloid; - // populate colinfomap cache and oidbitmap - boost::mutex::scoped_lock lk3(fColinfomapLock); - fColinfomap[coloid] = ct; - return coloid; - -} - -void CalpontSystemCatalog::getSysData (CalpontSelectExecutionPlan& csep, - NJLSysDataList& sysDataList, - const string& sysTableName) -{ - // start up new transaction - - BRM::TxnID txnID; - int oldTxnID; - txnID = fSessionManager->getTxnID(fSessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - BRM::QueryContext verID, oldVerID; - verID = fSessionManager->verID(); - oldTxnID = csep.txnID(); - csep.txnID(txnID.id); - oldVerID = csep.verID(); - csep.verID(verID); - //We need to use a session ID that's separate from the actual query SID, because the dbcon runs queries - // in the middle of receiving data bands for the real query. - //TODO: we really need a flag or something to identify this as a syscat query: there are assumptions made - // in joblist that a high-bit-set session id is always a syscat query. This will be okay for a long time, - // but not forever... - - csep.sessionID(fSessionID | 0x80000000); - int tryCnt = 0; - - // add the tableList to csep for tuple joblist to use - CalpontSelectExecutionPlan::TableList tablelist; - tablelist.push_back(make_aliastable("calpontsys", sysTableName, "")); - csep.tableList(tablelist); - - // populate the returned column list as column map - csep.returnedCols().clear(); - CalpontSelectExecutionPlan::ColumnMap::const_iterator it; - - for (it = csep.columnMap().begin(); it != csep.columnMap().end(); ++it) - { - csep.returnedCols().push_back(it->second); - } - - if (fIdentity == EC) - { try { - getSysData_EC(csep, sysDataList, sysTableName); + fExeMgr->connect(5); } - catch ( IDBExcept& ) + catch (...) { - throw; + throw IDBExcept(ERR_LOST_CONN_EXEMGR); } - catch ( runtime_error& e ) - { - throw runtime_error ( e.what() ); - } - } - else - { - while (tryCnt < 5) - { - tryCnt++; - - try - { - getSysData_FE(csep, sysDataList, sysTableName); - break; - } - catch (IDBExcept&) // error already occured. this is not a broken pipe - { - throw; - } - catch (...) - { - // may be a broken pipe. re-establish exeMgr and send the message - delete fExeMgr; - fExeMgr = new ClientRotator(0, "ExeMgr"); - - try - { - fExeMgr->connect(5); - } - catch (...) - { - throw IDBExcept(ERR_LOST_CONN_EXEMGR); - } - } - } - - if (tryCnt >= 5) - //throw runtime_error("Error occured when calling system catalog. ExeMgr is not functioning."); - throw IDBExcept(ERR_SYSTEM_CATALOG); + } } - csep.sessionID(fSessionID); - csep.txnID(oldTxnID); - csep.verID(oldVerID); + if (tryCnt >= 5) + // throw runtime_error("Error occured when calling system catalog. ExeMgr is not functioning."); + throw IDBExcept(ERR_SYSTEM_CATALOG); + } + + csep.sessionID(fSessionID); + csep.txnID(oldTxnID); + csep.verID(oldVerID); } -void CalpontSystemCatalog::getSysData_EC(CalpontSelectExecutionPlan& csep, - NJLSysDataList& sysDataList, - const string& sysTableName) +void CalpontSystemCatalog::getSysData_EC(CalpontSelectExecutionPlan& csep, NJLSysDataList& sysDataList, + const string& sysTableName) { - DEBUG << "Enter getSysData_EC " << fSessionID << endl; + DEBUG << "Enter getSysData_EC " << fSessionID << endl; - uint32_t tableOID = IDB_VTABLE_ID; - ByteStream bs; - uint32_t status; + uint32_t tableOID = IDB_VTABLE_ID; + ByteStream bs; + uint32_t status; - ResourceManager* rm = ResourceManager::instance(true); - DistributedEngineComm* fEc = DistributedEngineComm::instance(rm); - SJLP jl = JobListFactory::makeJobList(&csep, rm, true); - //@bug 2221. Work around to prevent DMLProc crash. - int retryNum = 0; + ResourceManager* rm = ResourceManager::instance(true); + DistributedEngineComm* fEc = DistributedEngineComm::instance(rm); + SJLP jl = JobListFactory::makeJobList(&csep, rm, true); + //@bug 2221. Work around to prevent DMLProc crash. + int retryNum = 0; - while ( jl->status() != 0 ) - { - if ( retryNum >= 6 ) - throw runtime_error("Error occured when calling makeJobList"); + while (jl->status() != 0) + { + if (retryNum >= 6) + throw runtime_error("Error occured when calling makeJobList"); #ifdef _MSC_VER - Sleep(1 * 1000); + Sleep(1 * 1000); #else - sleep(1); + sleep(1); #endif - jl = JobListFactory::makeJobList(&csep, rm, true); - retryNum++; - } + jl = JobListFactory::makeJobList(&csep, rm, true); + retryNum++; + } - if (jl->status() != 0 || jl->putEngineComm(fEc) != 0) + if (jl->status() != 0 || jl->putEngineComm(fEc) != 0) + { + string emsg = jl->errMsg(); + throw runtime_error("Error occured when calling system catalog (1). " + emsg); + } + + if (jl->doQuery() != 0) + { + throw runtime_error( + "Error occured when calling system catalog (2). Make sure all processes are running."); + } + + TupleJobList* tjlp = dynamic_cast(jl.get()); + idbassert(tjlp); + RowGroup rowGroup = tjlp->getOutputRowGroup(); + RGData rgData; + + while (true) + { + bs.restart(); + uint32_t rowCount = jl->projectTable(tableOID, bs); + + // XXXST: take out the 'true' when all jobsteps have been made st-compatible. + rgData.deserialize(bs, true); + rowGroup.setData(&rgData); + + // rowGroup.setData(const_cast(bs.buf())); + if ((status = rowGroup.getStatus()) != 0) { - string emsg = jl->errMsg(); - throw runtime_error("Error occured when calling system catalog (1). " + emsg); + if (status >= 1000) // new error system + throw IDBExcept(status); + else + throw IDBExcept(ERR_SYSTEM_CATALOG); } - if ( jl->doQuery() != 0) - { - throw runtime_error("Error occured when calling system catalog (2). Make sure all processes are running."); - } - - TupleJobList* tjlp = dynamic_cast(jl.get()); - idbassert(tjlp); - RowGroup rowGroup = tjlp->getOutputRowGroup(); - RGData rgData; - - while (true) - { - bs.restart(); - uint32_t rowCount = jl->projectTable(tableOID, bs); - - // XXXST: take out the 'true' when all jobsteps have been made st-compatible. - rgData.deserialize(bs, true); - rowGroup.setData(&rgData); - - //rowGroup.setData(const_cast(bs.buf())); - if ((status = rowGroup.getStatus()) != 0) - { - if (status >= 1000) // new error system - throw IDBExcept(status); - else - throw IDBExcept(ERR_SYSTEM_CATALOG); - } - - if (rowCount > 0) - rowGroup.addToSysDataList(sysDataList); - else - break; - } + if (rowCount > 0) + rowGroup.addToSysDataList(sysDataList); + else + break; + } } -void CalpontSystemCatalog::getSysData_FE(const CalpontSelectExecutionPlan& csep, - NJLSysDataList& sysDataList, - const string& sysTableName) +void CalpontSystemCatalog::getSysData_FE(const CalpontSelectExecutionPlan& csep, NJLSysDataList& sysDataList, + const string& sysTableName) { - DEBUG << "Enter getSysData_FE " << fSessionID << endl; + DEBUG << "Enter getSysData_FE " << fSessionID << endl; - ByteStream msg; + ByteStream msg; - // send code to indicat tuple - ByteStream::quadbyte qb = 4; - msg << qb; - fExeMgr->write(msg); - msg.restart(); + // send code to indicat tuple + ByteStream::quadbyte qb = 4; + msg << qb; + fExeMgr->write(msg); + msg.restart(); - // Send the CalpontSelectExecutionPlan to ExeMgr. - csep.serialize(msg); - fExeMgr->write(msg); + // Send the CalpontSelectExecutionPlan to ExeMgr. + csep.serialize(msg); + fExeMgr->write(msg); - // Get the table oid for the system table being queried. - TableName tableName; - tableName.schema = CALPONT_SCHEMA; - tableName.table = sysTableName; - uint32_t tableOID = IDB_VTABLE_ID; - uint16_t status = 0; + // Get the table oid for the system table being queried. + TableName tableName; + tableName.schema = CALPONT_SCHEMA; + tableName.table = sysTableName; + uint32_t tableOID = IDB_VTABLE_ID; + uint16_t status = 0; - // Send the request for the table. - qb = static_cast(tableOID); - ByteStream bs; - bs << qb; - fExeMgr->write(bs); - boost::scoped_ptr rowGroup; - RGData rgData; + // Send the request for the table. + qb = static_cast(tableOID); + ByteStream bs; + bs << qb; + fExeMgr->write(bs); + boost::scoped_ptr rowGroup; + RGData rgData; - msg.restart(); + msg.restart(); + bs.restart(); + msg = fExeMgr->read(); + bs = fExeMgr->read(); + + if (bs.length() == 0) + { + throw IDBExcept(ERR_LOST_CONN_EXEMGR); + } + + string emsgStr; + bs >> emsgStr; + bool err = false; + + if (msg.length() == 4) + { + msg >> qb; + + if (qb != 0) + err = true; + } + else + { + err = true; + } + + if (err) + { + throw runtime_error(emsgStr); + } + + while (true) + { bs.restart(); - msg = fExeMgr->read(); bs = fExeMgr->read(); + // @bug 1782. check ExeMgr connection lost if (bs.length() == 0) + throw IDBExcept(ERR_LOST_CONN_EXEMGR); + + if (!rowGroup) { - throw IDBExcept(ERR_LOST_CONN_EXEMGR); - } - - string emsgStr; - bs >> emsgStr; - bool err = false; - - if (msg.length() == 4) - { - msg >> qb; - - if (qb != 0) - err = true; + rowGroup.reset(new RowGroup()); + rowGroup->deserialize(bs); + continue; } else { - err = true; + // XXXST + rgData.deserialize(bs, true); + rowGroup->setData(&rgData); + // rowGroup->setData(const_cast(bs.buf())); } - if (err) + if ((status = rowGroup->getStatus()) != 0) { - throw runtime_error(emsgStr); + if (status >= 1000) // new error system + { + // bs.advance(rowGroup->getDataSize()); + bs >> emsgStr; + throw IDBExcept(emsgStr, rowGroup->getStatus()); + } + else + { + throw IDBExcept(ERR_SYSTEM_CATALOG); + } } - while (true) - { - bs.restart(); - bs = fExeMgr->read(); + if (rowGroup->getRowCount() > 0) + rowGroup->addToSysDataList(sysDataList); + else + break; + } - // @bug 1782. check ExeMgr connection lost - if (bs.length() == 0) - throw IDBExcept(ERR_LOST_CONN_EXEMGR); - - if (!rowGroup) - { - rowGroup.reset(new RowGroup()); - rowGroup->deserialize(bs); - continue; - } - else - { - // XXXST - rgData.deserialize(bs, true); - rowGroup->setData(&rgData); - //rowGroup->setData(const_cast(bs.buf())); - } - - if ((status = rowGroup->getStatus()) != 0) - { - if (status >= 1000) // new error system - { - //bs.advance(rowGroup->getDataSize()); - bs >> emsgStr; - throw IDBExcept(emsgStr, rowGroup->getStatus()); - } - else - { - throw IDBExcept(ERR_SYSTEM_CATALOG); - } - } - - if (rowGroup->getRowCount() > 0) - rowGroup->addToSysDataList(sysDataList); - else - break; - } - - bs.reset(); - qb = 0; - bs << qb; - fExeMgr->write(bs); + bs.reset(); + qb = 0; + bs << qb; + fExeMgr->write(bs); } const CalpontSystemCatalog::ColType CalpontSystemCatalog::colType(const OID& Oid) { - if ( Oid >= 3000) - DEBUG << "Enter colType: " << Oid << endl; + if (Oid >= 3000) + DEBUG << "Enter colType: " << Oid << endl; - ColType ct; - - // invalid oid - if (Oid < 1000) - return ct; - - //Check whether cache needs to be flushed - if ( Oid >= 3000) - { - checkSysCatVer(); - } - - // check colinfomap first for system table column or cached column type - boost::mutex::scoped_lock lk3(fColinfomapLock); - - if ( fColinfomap.size() > 0 ) - { - Colinfomap::const_iterator iter = fColinfomap.find(Oid); - - if (iter != fColinfomap.end()) - { - return iter->second; - } - } - - lk3.unlock(); - - /* SQL statement: select columnlength, datatype, dictobjectid,listobjectid,treeobjectid, - * columnposition, scale, prec, defaultvalue, schema, tablename, columnname - * from syscolumn where objectid = Oid; - */ - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; - string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; - string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; - string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; - string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; - string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; - string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; - string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; - // the following columns will be save in cache although it's not needed for now - string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; - string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; - string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; - string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; - string compressionType = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; - string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; - string nextvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; - - SimpleColumn* col[17]; - col[0] = new SimpleColumn(columnlength, fSessionID); - col[1] = new SimpleColumn(objectid, fSessionID); - col[2] = new SimpleColumn(datatype, fSessionID); - col[3] = new SimpleColumn(dictobjectid, fSessionID); - col[4] = new SimpleColumn(listobjectid, fSessionID); - col[5] = new SimpleColumn(treeobjectid, fSessionID); - col[6] = new SimpleColumn(columnposition, fSessionID); - col[7] = new SimpleColumn(scale, fSessionID); - col[8] = new SimpleColumn(precision, fSessionID); - col[9] = new SimpleColumn(defaultvalue, fSessionID); - col[10] = new SimpleColumn(schemaname, fSessionID); - col[11] = new SimpleColumn(tablename, fSessionID); - col[12] = new SimpleColumn(columnname, fSessionID); - col[13] = new SimpleColumn(nullable, fSessionID); - col[14] = new SimpleColumn(compressionType, fSessionID); - col[15] = new SimpleColumn(autoincrement, fSessionID); - col[16] = new SimpleColumn(nextvalue, fSessionID); - - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(columnlength, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(objectid, srcp)); - srcp.reset(col[2]); - colMap.insert(CMVT_(datatype, srcp)); - srcp.reset(col[3]); - colMap.insert(CMVT_(dictobjectid, srcp)); - srcp.reset(col[4]); - colMap.insert(CMVT_(listobjectid, srcp)); - srcp.reset(col[5]); - colMap.insert(CMVT_(treeobjectid, srcp)); - srcp.reset(col[6]); - colMap.insert(CMVT_(columnposition, srcp)); - srcp.reset(col[7]); - colMap.insert(CMVT_(scale, srcp)); - srcp.reset(col[8]); - colMap.insert(CMVT_(precision, srcp)); - // TODO: NULL value handling & convert to static_any::any - // delete this manually at fcn exit - srcp.reset(col[9]); - colMap.insert(CMVT_(defaultvalue, srcp)); - srcp.reset(col[10]); - colMap.insert(CMVT_(schemaname, srcp)); - srcp.reset(col[11]); - colMap.insert(CMVT_(tablename, srcp)); - srcp.reset(col[12]); - colMap.insert(CMVT_(columnname, srcp)); - srcp.reset(col[13]); - colMap.insert(CMVT_(nullable, srcp)); - srcp.reset(col[14]); - colMap.insert(CMVT_(compressionType, srcp)); - srcp.reset(col[15]); - colMap.insert(CMVT_(autoincrement, srcp)); - srcp.reset(col[16]); - colMap.insert(CMVT_(nextvalue, srcp)); - - csep.columnMapNonStatic(colMap); - - // ignore returnedcolumn, because it's not read by Joblist for now - csep.returnedCols(returnedColumnList); - OID oid[17]; - - for (int i = 0; i < 17; i++) - oid[i] = col[i]->oid(); - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[1]->clone(), - new ConstantColumn((int64_t)Oid, ConstantColumn::NUM)); - filterTokenList.push_back(f1); - - csep.filterTokenList(filterTokenList); - ostringstream oss; - oss << "select columnlength,datatype,dictobjectid,listobjectid,treeobjectid,columnposition,scale,compressiontype" - "prec,defaultvalue,schema,tablename,columnname from syscolumn where objectid=" << Oid << - " --colType/"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - getSysData(csep, sysDataList, SYSCOLUMN_TABLE); - - TableColName tcn; - vector::const_iterator it; - RID rid = std::numeric_limits::max(); - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - - if ((*it)->ColumnOID() == oid[0]) - { - ct.colWidth = ((*it)->GetData(0)); - - if (fIdentity == EC) - rid = (*it)->GetRid(0); - } - else if ((*it)->ColumnOID() == oid[2]) - ct.colDataType = (ColDataType)((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[3]) - ct.ddn.dictOID = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[4]) - ct.ddn.listOID = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[5]) - ct.ddn.treeOID = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[6]) - ct.colPosition = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[7]) - ct.scale = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[8]) - ct.precision = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) - { - ct.defaultValue = ((*it)->GetStringData(0)); - - if ((!ct.defaultValue.empty()) || (ct.defaultValue.length() > 0)) - { - if (ct.constraintType != NOTNULL_CONSTRAINT) - ct.constraintType = DEFAULT_CONSTRAINT; - } - } - // NJL fix. The schema, table, and column now return the oids for the dictionary columns - // on schema, table, and column. - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_SCHEMA) - tcn.schema = ((*it)->GetStringData(0)); - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_TABLENAME) - tcn.table = ((*it)->GetStringData(0)); - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_COLNAME) - tcn.column = ((*it)->GetStringData(0)); - else if ((*it)->ColumnOID() == oid[13]) - { - if (static_cast ((*it)->GetData(0)) == 0) - { - ct.constraintType = NOTNULL_CONSTRAINT; - } - } - else if ((*it)->ColumnOID() == oid[14]) - ct.compressionType = ct.ddn.compressionType = ((*it)->GetData(0)); - else if ((*it)->ColumnOID() == oid[15]) - { - ostringstream os; - os << (char) (*it)->GetData(0); - - if (os.str().compare("y") == 0) - ct.autoincrement = true; - else - ct.autoincrement = false; - } - else if ((*it)->ColumnOID() == oid[16]) - ct.nextvalue = ((*it)->GetData(0)); - - ct.columnOID = Oid; - } - - // populate colinfomap cache and oidbitmap - lk3.lock(); - boost::mutex::scoped_lock lk2(fOIDmapLock); - fColinfomap[Oid] = ct; - fOIDmap[tcn] = Oid; - - if (fIdentity == EC) - fColRIDmap[tcn] = rid; - - //Prevent mem leak - //delete col[9]; + ColType ct; + // invalid oid + if (Oid < 1000) return ct; + + // Check whether cache needs to be flushed + if (Oid >= 3000) + { + checkSysCatVer(); + } + + // check colinfomap first for system table column or cached column type + boost::mutex::scoped_lock lk3(fColinfomapLock); + + if (fColinfomap.size() > 0) + { + Colinfomap::const_iterator iter = fColinfomap.find(Oid); + + if (iter != fColinfomap.end()) + { + return iter->second; + } + } + + lk3.unlock(); + + /* SQL statement: select columnlength, datatype, dictobjectid,listobjectid,treeobjectid, + * columnposition, scale, prec, defaultvalue, schema, tablename, columnname + * from syscolumn where objectid = Oid; + */ + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; + string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; + string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; + string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; + string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; + string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; + string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; + string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; + // the following columns will be save in cache although it's not needed for now + string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; + string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; + string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; + string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; + string compressionType = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; + string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; + string nextvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; + + SimpleColumn* col[17]; + col[0] = new SimpleColumn(columnlength, fSessionID); + col[1] = new SimpleColumn(objectid, fSessionID); + col[2] = new SimpleColumn(datatype, fSessionID); + col[3] = new SimpleColumn(dictobjectid, fSessionID); + col[4] = new SimpleColumn(listobjectid, fSessionID); + col[5] = new SimpleColumn(treeobjectid, fSessionID); + col[6] = new SimpleColumn(columnposition, fSessionID); + col[7] = new SimpleColumn(scale, fSessionID); + col[8] = new SimpleColumn(precision, fSessionID); + col[9] = new SimpleColumn(defaultvalue, fSessionID); + col[10] = new SimpleColumn(schemaname, fSessionID); + col[11] = new SimpleColumn(tablename, fSessionID); + col[12] = new SimpleColumn(columnname, fSessionID); + col[13] = new SimpleColumn(nullable, fSessionID); + col[14] = new SimpleColumn(compressionType, fSessionID); + col[15] = new SimpleColumn(autoincrement, fSessionID); + col[16] = new SimpleColumn(nextvalue, fSessionID); + + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(columnlength, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(objectid, srcp)); + srcp.reset(col[2]); + colMap.insert(CMVT_(datatype, srcp)); + srcp.reset(col[3]); + colMap.insert(CMVT_(dictobjectid, srcp)); + srcp.reset(col[4]); + colMap.insert(CMVT_(listobjectid, srcp)); + srcp.reset(col[5]); + colMap.insert(CMVT_(treeobjectid, srcp)); + srcp.reset(col[6]); + colMap.insert(CMVT_(columnposition, srcp)); + srcp.reset(col[7]); + colMap.insert(CMVT_(scale, srcp)); + srcp.reset(col[8]); + colMap.insert(CMVT_(precision, srcp)); + // TODO: NULL value handling & convert to static_any::any + // delete this manually at fcn exit + srcp.reset(col[9]); + colMap.insert(CMVT_(defaultvalue, srcp)); + srcp.reset(col[10]); + colMap.insert(CMVT_(schemaname, srcp)); + srcp.reset(col[11]); + colMap.insert(CMVT_(tablename, srcp)); + srcp.reset(col[12]); + colMap.insert(CMVT_(columnname, srcp)); + srcp.reset(col[13]); + colMap.insert(CMVT_(nullable, srcp)); + srcp.reset(col[14]); + colMap.insert(CMVT_(compressionType, srcp)); + srcp.reset(col[15]); + colMap.insert(CMVT_(autoincrement, srcp)); + srcp.reset(col[16]); + colMap.insert(CMVT_(nextvalue, srcp)); + + csep.columnMapNonStatic(colMap); + + // ignore returnedcolumn, because it's not read by Joblist for now + csep.returnedCols(returnedColumnList); + OID oid[17]; + + for (int i = 0; i < 17; i++) + oid[i] = col[i]->oid(); + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, col[1]->clone(), new ConstantColumn((int64_t)Oid, ConstantColumn::NUM)); + filterTokenList.push_back(f1); + + csep.filterTokenList(filterTokenList); + ostringstream oss; + oss << "select " + "columnlength,datatype,dictobjectid,listobjectid,treeobjectid,columnposition,scale,compressiontype" + "prec,defaultvalue,schema,tablename,columnname from syscolumn where objectid=" + << Oid << " --colType/"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + + TableColName tcn; + vector::const_iterator it; + RID rid = std::numeric_limits::max(); + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[0]) + { + ct.colWidth = ((*it)->GetData(0)); + + if (fIdentity == EC) + rid = (*it)->GetRid(0); + } + else if ((*it)->ColumnOID() == oid[2]) + ct.colDataType = (ColDataType)((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[3]) + ct.ddn.dictOID = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[4]) + ct.ddn.listOID = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[5]) + ct.ddn.treeOID = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[6]) + ct.colPosition = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[7]) + ct.scale = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[8]) + ct.precision = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) + { + ct.defaultValue = ((*it)->GetStringData(0)); + + if ((!ct.defaultValue.empty()) || (ct.defaultValue.length() > 0)) + { + if (ct.constraintType != NOTNULL_CONSTRAINT) + ct.constraintType = DEFAULT_CONSTRAINT; + } + } + // NJL fix. The schema, table, and column now return the oids for the dictionary columns + // on schema, table, and column. + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_SCHEMA) + tcn.schema = ((*it)->GetStringData(0)); + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_TABLENAME) + tcn.table = ((*it)->GetStringData(0)); + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_COLNAME) + tcn.column = ((*it)->GetStringData(0)); + else if ((*it)->ColumnOID() == oid[13]) + { + if (static_cast((*it)->GetData(0)) == 0) + { + ct.constraintType = NOTNULL_CONSTRAINT; + } + } + else if ((*it)->ColumnOID() == oid[14]) + ct.compressionType = ct.ddn.compressionType = ((*it)->GetData(0)); + else if ((*it)->ColumnOID() == oid[15]) + { + ostringstream os; + os << (char)(*it)->GetData(0); + + if (os.str().compare("y") == 0) + ct.autoincrement = true; + else + ct.autoincrement = false; + } + else if ((*it)->ColumnOID() == oid[16]) + ct.nextvalue = ((*it)->GetData(0)); + + ct.columnOID = Oid; + } + + // populate colinfomap cache and oidbitmap + lk3.lock(); + boost::mutex::scoped_lock lk2(fOIDmapLock); + fColinfomap[Oid] = ct; + fOIDmap[tcn] = Oid; + + if (fIdentity == EC) + fColRIDmap[tcn] = rid; + + // Prevent mem leak + // delete col[9]; + + return ct; } const CalpontSystemCatalog::ColType CalpontSystemCatalog::colTypeDct(const OID& dictOid) { - if ( dictOid >= 3000) - DEBUG << "Enter colType: " << dictOid << endl; + if (dictOid >= 3000) + DEBUG << "Enter colType: " << dictOid << endl; - ColType ct; + ColType ct; - // invalid oid - if (dictOid < 1000) - return ct; + // invalid oid + if (dictOid < 1000) + return ct; - //Check whether cache needs to be flushed - if ( dictOid >= 3000) - { - checkSysCatVer(); - } + // Check whether cache needs to be flushed + if (dictOid >= 3000) + { + checkSysCatVer(); + } - // check map first cached column type - boost::recursive_mutex::scoped_lock lk3(fDctTokenMapLock); - DctTokenMap::const_iterator iter = fDctTokenMap.find(dictOid); + // check map first cached column type + boost::recursive_mutex::scoped_lock lk3(fDctTokenMapLock); + DctTokenMap::const_iterator iter = fDctTokenMap.find(dictOid); - if (iter != fDctTokenMap.end()) - return colType(iter->second); + if (iter != fDctTokenMap.end()) + return colType(iter->second); - lk3.unlock(); + lk3.unlock(); - /* SQL statement: select objectid from syscolumn where dictobjectid = dictOid; - */ - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; + /* SQL statement: select objectid from syscolumn where dictobjectid = dictOid; + */ + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; - SimpleColumn* col[2]; - col[0] = new SimpleColumn(objectid, fSessionID); - col[1] = new SimpleColumn(dictobjectid, fSessionID); + SimpleColumn* col[2]; + col[0] = new SimpleColumn(objectid, fSessionID); + col[1] = new SimpleColumn(dictobjectid, fSessionID); - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(objectid, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(dictobjectid, srcp)); + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(objectid, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(dictobjectid, srcp)); - csep.columnMapNonStatic(colMap); + csep.columnMapNonStatic(colMap); - // ignore returnedcolumn, because it's not read by Joblist for now - csep.returnedCols(returnedColumnList); - OID oid[2]; + // ignore returnedcolumn, because it's not read by Joblist for now + csep.returnedCols(returnedColumnList); + OID oid[2]; - for (int i = 0; i < 2; i++) - oid[i] = col[i]->oid(); + for (int i = 0; i < 2; i++) + oid[i] = col[i]->oid(); - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[1]->clone(), - new ConstantColumn((int64_t)dictOid, ConstantColumn::NUM)); - filterTokenList.push_back(f1); + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, col[1]->clone(), new ConstantColumn((int64_t)dictOid, ConstantColumn::NUM)); + filterTokenList.push_back(f1); - csep.filterTokenList(filterTokenList); - ostringstream oss; - oss << "select objectid from syscolumn where dictobjectid=" << dictOid << " --colTypeDct/"; + csep.filterTokenList(filterTokenList); + ostringstream oss; + oss << "select objectid from syscolumn where dictobjectid=" << dictOid << " --colTypeDct/"; - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; - csep.data(oss.str()); - NJLSysDataList sysDataList; - getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + csep.data(oss.str()); + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); - vector::const_iterator it; + vector::const_iterator it; - OID tokenOID = 0; + OID tokenOID = 0; - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid[0]) - tokenOID = ((*it)->GetData(0)); - } + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[0]) + tokenOID = ((*it)->GetData(0)); + } - // populate cache - lk3.lock(); - fDctTokenMap[dictOid] = tokenOID; + // populate cache + lk3.lock(); + fDctTokenMap[dictOid] = tokenOID; - return colType(tokenOID); + return colType(tokenOID); } const CalpontSystemCatalog::TableColName CalpontSystemCatalog::colName(const OID& oid) { - if (oid >= 3000) - DEBUG << "Enter colName: " << oid; + if (oid >= 3000) + DEBUG << "Enter colName: " << oid; - TableColName tableColName; + TableColName tableColName; - // invalid oid - if (oid < 1000) - return tableColName; + // invalid oid + if (oid < 1000) + return tableColName; - //Check whether cache needs to be flushed - if ( oid >= 3000) + // Check whether cache needs to be flushed + if (oid >= 3000) + { + checkSysCatVer(); + } + + // check oidmap for system table columns and cached columns + boost::mutex::scoped_lock lk2(fOIDmapLock); + OIDmap::const_iterator iter = fOIDmap.begin(); + + while (iter != fOIDmap.end()) + { + if (oid == (*iter).second) { - checkSysCatVer(); + tableColName = (*iter).first; + DEBUG << "|in cache|" << tableColName.schema << "|" << tableColName.table << "|" << tableColName.column + << endl; + return tableColName; } - // check oidmap for system table columns and cached columns - boost::mutex::scoped_lock lk2(fOIDmapLock); - OIDmap::const_iterator iter = fOIDmap.begin(); + ++iter; + } - while ( iter != fOIDmap.end() ) - { - if (oid == (*iter).second ) - { - tableColName = (*iter).first; - DEBUG << "|in cache|" << tableColName.schema << "|" << tableColName.table << "|" << tableColName.column << endl; - return tableColName; - } + lk2.unlock(); - ++iter; - } + // SQL statement: select schema, tablename, columnname from syscolumn where objectid = oid; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; - lk2.unlock(); + SimpleColumn* c1 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL, fSessionID); + SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c3 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, fSessionID); + SimpleColumn* c4 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, fSessionID); - // SQL statement: select schema, tablename, columnname from syscolumn where objectid = oid; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, srcp)); + csep.columnMapNonStatic(colMap); - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, fSessionID); - SimpleColumn* c4 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, fSessionID); + srcp.reset(c2->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c3->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c4->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, srcp)); - srcp.reset(c4); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, srcp)); - csep.columnMapNonStatic(colMap); + // NJL fix. The dictionary column OID is now returned from getSysData. + // OID oid2 = c2->oid(); + // OID oid3 = c3->oid(); + // OID oid4 = c4->oid(); + OID oid2 = DICTOID_SYSCOLUMN_SCHEMA; + OID oid3 = DICTOID_SYSCOLUMN_TABLENAME; + OID oid4 = DICTOID_SYSCOLUMN_COLNAME; - srcp.reset(c2->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c3->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c4->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c1->clone(), new ConstantColumn((int64_t)oid, ConstantColumn::NUM)); + filterTokenList.push_back(f1); - // NJL fix. The dictionary column OID is now returned from getSysData. - // OID oid2 = c2->oid(); - // OID oid3 = c3->oid(); - // OID oid4 = c4->oid(); - OID oid2 = DICTOID_SYSCOLUMN_SCHEMA; - OID oid3 = DICTOID_SYSCOLUMN_TABLENAME; - OID oid4 = DICTOID_SYSCOLUMN_COLNAME; + csep.filterTokenList(filterTokenList); - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c1->clone(), - new ConstantColumn((int64_t)oid, ConstantColumn::NUM)); - filterTokenList.push_back(f1); + ostringstream oss; + oss << "select schema,tablename,columnname from syscolumn where objectid=" << oid << " --colName/"; - csep.filterTokenList(filterTokenList); + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; - ostringstream oss; - oss << "select schema,tablename,columnname from syscolumn where objectid=" << oid << - " --colName/"; + csep.data(oss.str()); + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; + vector::const_iterator it; - csep.data(oss.str()); - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSCOLUMN_TABLE); + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid2) + tableColName.schema = (*it)->GetStringData(0); + else if ((*it)->ColumnOID() == oid3) + tableColName.table = (*it)->GetStringData(0); + else if ((*it)->ColumnOID() == oid4) + tableColName.column = (*it)->GetStringData(0); + } - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid2) - tableColName.schema = (*it)->GetStringData(0); - else if ((*it)->ColumnOID() == oid3) - tableColName.table = (*it)->GetStringData(0); - else if ((*it)->ColumnOID() == oid4) - tableColName.column = (*it)->GetStringData(0); - } - - if (oid > 3000) - DEBUG << "|" << tableColName.schema << "|" << tableColName.table << "|" << tableColName.column << endl; + if (oid > 3000) + DEBUG << "|" << tableColName.schema << "|" << tableColName.table << "|" << tableColName.column << endl; #if BOOST_VERSION < 103800 - if (!lk2.locked()) lk2.lock(); + if (!lk2.locked()) + lk2.lock(); #else - if (!lk2.owns_lock()) lk2.lock(); + if (!lk2.owns_lock()) + lk2.lock(); #endif - fOIDmap[tableColName] = oid; + fOIDmap[tableColName] = oid; - return tableColName; + return tableColName; } const CalpontSystemCatalog::TableColName CalpontSystemCatalog::dictColName(const OID& oid) { - if (oid >= 3000) - DEBUG << "Enter dictColName: " << oid; + if (oid >= 3000) + DEBUG << "Enter dictColName: " << oid; - TableColName tableColName; - - // invalid oid - if (oid < 1000) - return tableColName; - - //Check whether cache needs to be flushed - if ( oid >= 3000) - { - checkSysCatVer(); - } - - // SQL statement: select schema, tablename, columnname from syscolumn where dictobjectid = oid; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, fSessionID); - SimpleColumn* c4 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, fSessionID); - - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, srcp)); - srcp.reset(c4); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(c2->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c3->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c4->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - OID oid2 = DICTOID_SYSCOLUMN_SCHEMA; - OID oid3 = DICTOID_SYSCOLUMN_TABLENAME; - OID oid4 = DICTOID_SYSCOLUMN_COLNAME; - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c1->clone(), - new ConstantColumn((int64_t)oid, ConstantColumn::NUM)); - filterTokenList.push_back(f1); - - csep.filterTokenList(filterTokenList); - - ostringstream oss; - oss << "select schema,tablename,columnname from syscolumn where dictobjectid=" << oid << - " --colName/"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSCOLUMN_TABLE); - - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid2) - tableColName.schema = (*it)->GetStringData(0); - else if ((*it)->ColumnOID() == oid3) - tableColName.table = (*it)->GetStringData(0); - else if ((*it)->ColumnOID() == oid4) - tableColName.column = (*it)->GetStringData(0); - } - - if (oid > 3000) - DEBUG << "|" << tableColName.schema << "|" << tableColName.table << "|" << tableColName.column << endl; + TableColName tableColName; + // invalid oid + if (oid < 1000) return tableColName; + + // Check whether cache needs to be flushed + if (oid >= 3000) + { + checkSysCatVer(); + } + + // SQL statement: select schema, tablename, columnname from syscolumn where dictobjectid = oid; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, fSessionID); + SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c3 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, fSessionID); + SimpleColumn* c4 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, fSessionID); + + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(c2->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c3->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c4->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + OID oid2 = DICTOID_SYSCOLUMN_SCHEMA; + OID oid3 = DICTOID_SYSCOLUMN_TABLENAME; + OID oid4 = DICTOID_SYSCOLUMN_COLNAME; + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c1->clone(), new ConstantColumn((int64_t)oid, ConstantColumn::NUM)); + filterTokenList.push_back(f1); + + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select schema,tablename,columnname from syscolumn where dictobjectid=" << oid << " --colName/"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid2) + tableColName.schema = (*it)->GetStringData(0); + else if ((*it)->ColumnOID() == oid3) + tableColName.table = (*it)->GetStringData(0); + else if ((*it)->ColumnOID() == oid4) + tableColName.column = (*it)->GetStringData(0); + } + + if (oid > 3000) + DEBUG << "|" << tableColName.schema << "|" << tableColName.table << "|" << tableColName.column << endl; + + return tableColName; } -uint64_t CalpontSystemCatalog::nextAutoIncrValue (TableName aTableName, int lower_case_table_names) +uint64_t CalpontSystemCatalog::nextAutoIncrValue(TableName aTableName, int lower_case_table_names) { - TableInfo tbInfo; + TableInfo tbInfo; - if (lower_case_table_names) + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } + + try + { + tbInfo = tableInfo(aTableName); + } + catch (runtime_error& /*ex*/) + { + throw; + } + + if (tbInfo.tablewithautoincr == NO_AUTOINCRCOL) + return AUTOINCR_SATURATED; + + // Build a plan to get current nextvalue: select nextvalue from syscolumn where schema = tableName.schema + // and tablename = tableName.table and autoincrement='y'; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; + string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; + string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; + string nextvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; + + SimpleColumn* col[5]; + col[0] = new SimpleColumn(tablename, fSessionID); + col[1] = new SimpleColumn(schemaname, fSessionID); + col[2] = new SimpleColumn(autoincrement, fSessionID); + col[3] = new SimpleColumn(nextvalue, fSessionID); + + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(tablename, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(schemaname, srcp)); + srcp.reset(col[2]); + colMap.insert(CMVT_(autoincrement, srcp)); + srcp.reset(col[3]); + colMap.insert(CMVT_(nextvalue, srcp)); + + csep.columnMapNonStatic(colMap); + + csep.returnedCols(returnedColumnList); + + OID oid[4]; + + for (int i = 0; i < 4; i++) + oid[i] = col[i]->oid(); + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, col[1]->clone(), new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, col[0]->clone(), new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = + new SimpleFilter(opeq, col[2]->clone(), new ConstantColumn("y", ConstantColumn::LITERAL)); + filterTokenList.push_back(f3); + csep.filterTokenList(filterTokenList); + ostringstream oss; + oss << "select nextvalue from syscolumn where schema = aTableName.schema and tablename = aTableName.table " + "and autoincrement='y'"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + + NJLSysDataList sysDataList; + + try + { + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } + + uint64_t nextVal = AUTOINCR_SATURATED; + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[3]) { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); + nextVal = static_cast(((*it)->GetData(0))); } + } - try - { - tbInfo = tableInfo(aTableName); - } - catch (runtime_error& /*ex*/) - { - throw; - } - - if (tbInfo.tablewithautoincr == NO_AUTOINCRCOL) - return AUTOINCR_SATURATED; - - //Build a plan to get current nextvalue: select nextvalue from syscolumn where schema = tableName.schema and tablename = tableName.table and autoincrement='y'; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; - string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; - string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; - string nextvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; - - SimpleColumn* col[5]; - col[0] = new SimpleColumn(tablename, fSessionID); - col[1] = new SimpleColumn(schemaname, fSessionID); - col[2] = new SimpleColumn(autoincrement, fSessionID); - col[3] = new SimpleColumn(nextvalue, fSessionID); - - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(tablename, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(schemaname, srcp)); - srcp.reset(col[2]); - colMap.insert(CMVT_(autoincrement, srcp)); - srcp.reset(col[3]); - colMap.insert(CMVT_(nextvalue, srcp)); - - csep.columnMapNonStatic(colMap); - - csep.returnedCols(returnedColumnList); - - OID oid[4]; - - for (int i = 0; i < 4; i++) - oid[i] = col[i]->oid(); - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[1]->clone(), - new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - col[0]->clone(), - new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f3 = new SimpleFilter (opeq, - col[2]->clone(), - new ConstantColumn("y", ConstantColumn::LITERAL)); - filterTokenList.push_back(f3); - csep.filterTokenList(filterTokenList); - ostringstream oss; - oss << "select nextvalue from syscolumn where schema = aTableName.schema and tablename = aTableName.table and autoincrement='y'"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - - NJLSysDataList sysDataList; - - try - { - getSysData(csep, sysDataList, SYSCOLUMN_TABLE); - } - catch (runtime_error& e) - { - throw runtime_error ( e.what() ); - } - - uint64_t nextVal = AUTOINCR_SATURATED; - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - - if ((*it)->ColumnOID() == oid[3]) - { - nextVal = static_cast(((*it)->GetData(0))); - } - } - - return (nextVal); - + return (nextVal); } -int32_t CalpontSystemCatalog::autoColumOid (TableName aTableName, int lower_case_table_names) +int32_t CalpontSystemCatalog::autoColumOid(TableName aTableName, int lower_case_table_names) { - TableInfo tbInfo; + TableInfo tbInfo; - if (lower_case_table_names) + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } + + try + { + tbInfo = tableInfo(aTableName); + } + catch (runtime_error& /*ex*/) + { + return -2; + } + + if (tbInfo.tablewithautoincr == NO_AUTOINCRCOL) + return 0; + + // Build a plan to get column oid: select objectid from syscolumn where schema = tableName.schema and + // tablename = tableName.table and autoincrement='y'; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; + string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; + string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + + SimpleColumn* col[5]; + col[0] = new SimpleColumn(tablename, fSessionID); + col[1] = new SimpleColumn(schemaname, fSessionID); + col[2] = new SimpleColumn(autoincrement, fSessionID); + col[3] = new SimpleColumn(objectid, fSessionID); + + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(tablename, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(schemaname, srcp)); + srcp.reset(col[2]); + colMap.insert(CMVT_(autoincrement, srcp)); + srcp.reset(col[3]); + colMap.insert(CMVT_(objectid, srcp)); + + csep.columnMapNonStatic(colMap); + + csep.returnedCols(returnedColumnList); + + OID oid[4]; + + for (int i = 0; i < 4; i++) + oid[i] = col[i]->oid(); + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, col[1]->clone(), new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, col[0]->clone(), new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = + new SimpleFilter(opeq, col[2]->clone(), new ConstantColumn("y", ConstantColumn::LITERAL)); + filterTokenList.push_back(f3); + csep.filterTokenList(filterTokenList); + ostringstream oss; + oss << "select nextvalue from syscolumn where schema = aTableName.schema and tablename = aTableName.table " + "and autoincrement='y'"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + + try + { + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } + + int32_t columnOid = 0; + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[3]) { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); + columnOid = ((*it)->GetData(0)); } + } - try - { - tbInfo = tableInfo (aTableName); - } - catch (runtime_error& /*ex*/) - { - return -2; - } - - if (tbInfo.tablewithautoincr == NO_AUTOINCRCOL) - return 0; - - //Build a plan to get column oid: select objectid from syscolumn where schema = tableName.schema and tablename = tableName.table and autoincrement='y'; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; - string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; - string autoincrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - - SimpleColumn* col[5]; - col[0] = new SimpleColumn(tablename, fSessionID); - col[1] = new SimpleColumn(schemaname, fSessionID); - col[2] = new SimpleColumn(autoincrement, fSessionID); - col[3] = new SimpleColumn(objectid, fSessionID); - - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(tablename, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(schemaname, srcp)); - srcp.reset(col[2]); - colMap.insert(CMVT_(autoincrement, srcp)); - srcp.reset(col[3]); - colMap.insert(CMVT_(objectid, srcp)); - - csep.columnMapNonStatic(colMap); - - csep.returnedCols(returnedColumnList); - - OID oid[4]; - - for (int i = 0; i < 4; i++) - oid[i] = col[i]->oid(); - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[1]->clone(), - new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - col[0]->clone(), - new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f3 = new SimpleFilter (opeq, - col[2]->clone(), - new ConstantColumn("y", ConstantColumn::LITERAL)); - filterTokenList.push_back(f3); - csep.filterTokenList(filterTokenList); - ostringstream oss; - oss << "select nextvalue from syscolumn where schema = aTableName.schema and tablename = aTableName.table and autoincrement='y'"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - - try - { - getSysData(csep, sysDataList, SYSCOLUMN_TABLE); - } - catch (runtime_error& e) - { - throw runtime_error ( e.what() ); - } - - int32_t columnOid = 0; - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - - if ((*it)->ColumnOID() == oid[3]) - { - columnOid = ((*it)->GetData(0)); - } - } - - return (columnOid); - + return (columnOid); } -const CalpontSystemCatalog::ROPair CalpontSystemCatalog::nextAutoIncrRid ( const OID& columnoid) +const CalpontSystemCatalog::ROPair CalpontSystemCatalog::nextAutoIncrRid(const OID& columnoid) { + // Build a plan to get rid of nextvalue: select nextvalue from syscolumn where objectid = columnoid; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; - //Build a plan to get rid of nextvalue: select nextvalue from syscolumn where objectid = columnoid; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + string nextvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - string nextvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; + SimpleColumn* col[2]; + col[0] = new SimpleColumn(objectid, fSessionID); + col[1] = new SimpleColumn(nextvalue, fSessionID); - SimpleColumn* col[2]; - col[0] = new SimpleColumn(objectid, fSessionID); - col[1] = new SimpleColumn(nextvalue, fSessionID); + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(objectid, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(nextvalue, srcp)); - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(objectid, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(nextvalue, srcp)); + csep.columnMapNonStatic(colMap); - csep.columnMapNonStatic(colMap); + csep.returnedCols(returnedColumnList); - csep.returnedCols(returnedColumnList); + OID oid[2]; - OID oid[2]; + for (int i = 0; i < 2; i++) + oid[i] = col[i]->oid(); - for (int i = 0; i < 2; i++) - oid[i] = col[i]->oid(); + // Filters + SimpleFilter* f1 = new SimpleFilter(opeq, col[0]->clone(), + new ConstantColumn((int64_t)columnoid, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[0]->clone(), - new ConstantColumn((int64_t)columnoid, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); + csep.filterTokenList(filterTokenList); + ostringstream oss; + oss << "select nextvalue from syscolumn objectid = columnoid"; - csep.filterTokenList(filterTokenList); - ostringstream oss; - oss << "select nextvalue from syscolumn objectid = columnoid"; + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; + csep.data(oss.str()); + NJLSysDataList sysDataList; - csep.data(oss.str()); - NJLSysDataList sysDataList; + try + { + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } - try + vector::const_iterator it; + ROPair roPair; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[1]) { - getSysData(csep, sysDataList, SYSCOLUMN_TABLE); - } - catch (runtime_error& e) - { - throw runtime_error ( e.what() ); + roPair.rid = ((*it)->GetRid(0)); + roPair.objnum = oid[1]; + return roPair; } + } - vector::const_iterator it; - ROPair roPair; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - - if ((*it)->ColumnOID() == oid[1]) - { - roPair.rid = ((*it)->GetRid(0)); - roPair.objnum = oid[1]; - return roPair; - } - } - - return roPair; + return roPair; } -#if 0 //Not implemented +#if 0 // Not implemented const CalpontSystemCatalog::OID CalpontSystemCatalog::colBitmap(const OID& oid) const { return oid; @@ -1914,18 +1867,18 @@ const CalpontSystemCatalog::SCN CalpontSystemCatalog::scn(void) const /* static */ boost::shared_ptr CalpontSystemCatalog::makeCalpontSystemCatalog(uint32_t sessionID) { - boost::mutex::scoped_lock lock(map_mutex); - boost::shared_ptr instance; - CatalogMap::const_iterator it = fCatalogMap.find(sessionID); + boost::mutex::scoped_lock lock(map_mutex); + boost::shared_ptr instance; + CatalogMap::const_iterator it = fCatalogMap.find(sessionID); - if (sessionID == 0) + if (sessionID == 0) + { + if (it == fCatalogMap.end()) { - if (it == fCatalogMap.end()) - { - instance.reset(new CalpontSystemCatalog()); - fCatalogMap[0] = instance; - return instance; - } + instance.reset(new CalpontSystemCatalog()); + fCatalogMap[0] = instance; + return instance; + } #if 0 @@ -1941,97 +1894,95 @@ boost::shared_ptr CalpontSystemCatalog::makeCalpontSystemC #endif - return it->second; - } - - if (it == fCatalogMap.end()) - { - instance.reset(new CalpontSystemCatalog()); - instance->sessionID(sessionID); - instance->fExeMgr->setSessionId(sessionID); - fCatalogMap[sessionID] = instance; - return instance; - } - return it->second; + } + + if (it == fCatalogMap.end()) + { + instance.reset(new CalpontSystemCatalog()); + instance->sessionID(sessionID); + instance->fExeMgr->setSessionId(sessionID); + fCatalogMap[sessionID] = instance; + return instance; + } + + return it->second; } /* static */ void CalpontSystemCatalog::removeCalpontSystemCatalog(uint32_t sessionID) { - boost::mutex::scoped_lock lock(map_mutex); - DEBUG << "remove calpont system catalog for session " << sessionID << endl; - fCatalogMap.erase(sessionID); - /* - CatalogMap::iterator it = fCatalogMap.find(sessionID); - if (it != fCatalogMap.end()) - { - delete (*it).second; - fCatalogMap.erase(it); - } - */ + boost::mutex::scoped_lock lock(map_mutex); + DEBUG << "remove calpont system catalog for session " << sessionID << endl; + fCatalogMap.erase(sessionID); + /* + CatalogMap::iterator it = fCatalogMap.find(sessionID); + if (it != fCatalogMap.end()) + { + delete (*it).second; + fCatalogMap.erase(it); + } + */ } -CalpontSystemCatalog::CalpontSystemCatalog(): - fExeMgr (new ClientRotator(0, "ExeMgr")), - fSessionID (0) +CalpontSystemCatalog::CalpontSystemCatalog() : fExeMgr(new ClientRotator(0, "ExeMgr")), fSessionID(0) { - // Set fIdentity based on the module on which we are running. - fIdentity = EC; + // Set fIdentity based on the module on which we are running. + fIdentity = EC; - if ( fSessionManager.get() == 0 ) - fSessionManager.reset(new SessionManager()); + if (fSessionManager.get() == 0) + fSessionManager.reset(new SessionManager()); - try - { - string localModuleType; - const char* p = 0; - //see if env is set to override identity lookup + try + { + string localModuleType; + const char* p = 0; + // see if env is set to override identity lookup #ifdef _MSC_VER - p = "EC"; - string cfStr = IDBreadRegistry("SyscatIdent"); + p = "EC"; + string cfStr = IDBreadRegistry("SyscatIdent"); - if (!cfStr.empty()) - p = cfStr.c_str(); + if (!cfStr.empty()) + p = cfStr.c_str(); #else - p = getenv("CALPONT_CSC_IDENT"); + p = getenv("CALPONT_CSC_IDENT"); #endif - if (p && *p) - { - localModuleType = p; - } - else - { - oam::Oam oam; - oam::oamModuleInfo_t t = oam.getModuleInfo(); - localModuleType = boost::get<1>(t); - } - - // If dm (director module), set the identity to FE (Front End). - // @bug 1029. set "FE" for beetlejuice (xm) - if (localModuleType == "dm" || localModuleType == "xm") - { - fIdentity = FE; - } - } - catch (exception&) + if (p && *p) { - // If not in an environment with OAM set up, default to Front End. - fIdentity = FE; + localModuleType = p; + } + else + { + oam::Oam oam; + oam::oamModuleInfo_t t = oam.getModuleInfo(); + localModuleType = boost::get<1>(t); } - buildSysColinfomap(); - buildSysOIDmap(); - buildSysTablemap(); - buildSysDctmap(); - fSyscatSCN = fSessionManager->sysCatVerID().currentScn; + // If dm (director module), set the identity to FE (Front End). + // @bug 1029. set "FE" for beetlejuice (xm) + if (localModuleType == "dm" || localModuleType == "xm") + { + fIdentity = FE; + } + } + catch (exception&) + { + // If not in an environment with OAM set up, default to Front End. + fIdentity = FE; + } + + buildSysColinfomap(); + buildSysOIDmap(); + buildSysTablemap(); + buildSysDctmap(); + fSyscatSCN = fSessionManager->sysCatVerID().currentScn; } CalpontSystemCatalog::~CalpontSystemCatalog() { - delete fExeMgr; + delete fExeMgr; } #if 0 @@ -2789,867 +2740,869 @@ const CalpontSystemCatalog::RID CalpontSystemCatalog::constraintColRID(const Tab } #endif -const vector< pair > CalpontSystemCatalog::getTables(const std::string schema, int lower_case_table_names) +const vector > +CalpontSystemCatalog::getTables(const std::string schema, int lower_case_table_names) { - string schemaname = schema; - if (lower_case_table_names) - { - boost::algorithm::to_lower(schemaname); - } - vector < pair > tables; - - if (schemaname == CALPONT_SCHEMA) - { - // systables - tables.push_back( make_pair(SYSTABLE_BASE, make_table(CALPONT_SCHEMA, SYSTABLE_TABLE))); - tables.push_back( make_pair(SYSCOLUMN_BASE, make_table(CALPONT_SCHEMA, SYSCOLUMN_TABLE))); - return tables; - } - - DEBUG << "Enter getTables" << endl; - // SQL statement: select tablename from systable where schemaname = schema; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); - SimpleColumn* c4 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + CREATEDATE_COL, fSessionID); - - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); - srcp.reset(c4); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + CREATEDATE_COL, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c2->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - OID oid1 = DICTOID_SYSTABLE_TABLENAME; - OID oid2 = DICTOID_SYSTABLE_SCHEMA; - - if (!schema.empty()) - { - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c2->clone(), - new ConstantColumn(schemaname, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - csep.filterTokenList(filterTokenList); - } - - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSTABLE_TABLE); - - vector::const_iterator it; - vector tnl; - ROPair rp; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid1) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - tables.push_back( make_pair(0, make_table("", (*it)->GetStringData(i)))); - tnl.push_back((*it)->GetStringData(i)); - } - } - } - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid2) - { - for (int i = 0; i < (*it)->dataCount(); i++) - tables[i].second.schema = (*it)->GetStringData(i); - } - } - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == c4->oid()) - { - for (int i = 0; i < (*it)->dataCount(); i++) - tables[i].second.create_date = (*it)->GetData(i); - } - } - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == c3->oid()) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - rp.objnum = (OID)((*it)->GetData(i)); - - if (fIdentity == EC) - rp.rid = (*it)->GetRid(i); - - fTablemap[tables[i].second] = rp.objnum; - fTableRIDmap[tables[i].second] = rp.rid; - tables[i].first = rp.objnum; - } - } - } + string schemaname = schema; + if (lower_case_table_names) + { + boost::algorithm::to_lower(schemaname); + } + vector > tables; + if (schemaname == CALPONT_SCHEMA) + { + // systables + tables.push_back(make_pair(SYSTABLE_BASE, make_table(CALPONT_SCHEMA, SYSTABLE_TABLE))); + tables.push_back(make_pair(SYSCOLUMN_BASE, make_table(CALPONT_SCHEMA, SYSCOLUMN_TABLE))); return tables; + } + + DEBUG << "Enter getTables" << endl; + // SQL statement: select tablename from systable where schemaname = schema; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SimpleColumn* c1 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); + SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); + SimpleColumn* c4 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + CREATEDATE_COL, fSessionID); + + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + CREATEDATE_COL, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c2->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + OID oid1 = DICTOID_SYSTABLE_TABLENAME; + OID oid2 = DICTOID_SYSTABLE_SCHEMA; + + if (!schema.empty()) + { + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c2->clone(), new ConstantColumn(schemaname, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + csep.filterTokenList(filterTokenList); + } + + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSTABLE_TABLE); + + vector::const_iterator it; + vector tnl; + ROPair rp; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid1) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + tables.push_back(make_pair(0, make_table("", (*it)->GetStringData(i)))); + tnl.push_back((*it)->GetStringData(i)); + } + } + } + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid2) + { + for (int i = 0; i < (*it)->dataCount(); i++) + tables[i].second.schema = (*it)->GetStringData(i); + } + } + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == c4->oid()) + { + for (int i = 0; i < (*it)->dataCount(); i++) + tables[i].second.create_date = (*it)->GetData(i); + } + } + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == c3->oid()) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + rp.objnum = (OID)((*it)->GetData(i)); + + if (fIdentity == EC) + rp.rid = (*it)->GetRid(i); + + fTablemap[tables[i].second] = rp.objnum; + fTableRIDmap[tables[i].second] = rp.rid; + tables[i].first = rp.objnum; + } + } + } + + return tables; } /* SQL statement: select objectid from systable */ -int CalpontSystemCatalog::getTableCount () +int CalpontSystemCatalog::getTableCount() { - int tableCnt = 0; + int tableCnt = 0; - DEBUG << "Enter getTableCount" << endl; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::ColumnMap colMap; + DEBUG << "Enter getTableCount" << endl; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::ColumnMap colMap; - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); + SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); - csep.columnMapNonStatic(colMap); + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); + csep.columnMapNonStatic(colMap); - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - OID oid1 = OID_SYSTABLE_OBJECTID; + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + OID oid1 = OID_SYSTABLE_OBJECTID; + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSTABLE_TABLE); - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSTABLE_TABLE); + vector::const_iterator it; - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid1) { - if ((*it)->ColumnOID() == oid1) - { - tableCnt = (*it)->dataCount(); - } + tableCnt = (*it)->dataCount(); } + } - return tableCnt; + return tableCnt; } /* SQL statement: select objectid from syscolumn where schema=tableColName.schema and * tablename=tableColName.table and columnname=tableColName.column;*/ -const CalpontSystemCatalog::ROPair CalpontSystemCatalog::columnRID(const TableColName& tableColName, int lower_case_table_names) +const CalpontSystemCatalog::ROPair CalpontSystemCatalog::columnRID(const TableColName& tableColName, + int lower_case_table_names) { - ROPair rp; - TableColName aTableColName; - aTableColName.schema = tableColName.schema; - aTableColName.table = tableColName.table; - aTableColName.column = tableColName.column; - if (lower_case_table_names) - { - boost::algorithm::to_lower(aTableColName.schema); - boost::algorithm::to_lower(aTableColName.table); - } - boost::algorithm::to_lower(aTableColName.column); + ROPair rp; + TableColName aTableColName; + aTableColName.schema = tableColName.schema; + aTableColName.table = tableColName.table; + aTableColName.column = tableColName.column; + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableColName.schema); + boost::algorithm::to_lower(aTableColName.table); + } + boost::algorithm::to_lower(aTableColName.column); - if (aTableColName.schema.compare(CALPONT_SCHEMA) != 0) - DEBUG << "Enter columnRID: " << tableColName.schema << "|" << tableColName.table - << "|" << tableColName.column << endl; + if (aTableColName.schema.compare(CALPONT_SCHEMA) != 0) + DEBUG << "Enter columnRID: " << tableColName.schema << "|" << tableColName.table << "|" + << tableColName.column << endl; - //Check whether cache needs to be flushed - if ( aTableColName.schema.compare(CALPONT_SCHEMA) ) - { - checkSysCatVer(); - } + // Check whether cache needs to be flushed + if (aTableColName.schema.compare(CALPONT_SCHEMA)) + { + checkSysCatVer(); + } - /* SQL statement: select objectid from syscolumn where schema=tableColName.schema and - tablename=tableColName.table and columnname=tableColName.column;*/ - // this function is duplicate to lookupOID() and will be deprecated soon - rp.objnum = lookupOID(tableColName); - boost::mutex::scoped_lock lk2(fOIDmapLock); + /* SQL statement: select objectid from syscolumn where schema=tableColName.schema and + tablename=tableColName.table and columnname=tableColName.column;*/ + // this function is duplicate to lookupOID() and will be deprecated soon + rp.objnum = lookupOID(tableColName); + boost::mutex::scoped_lock lk2(fOIDmapLock); - ColRIDmap::const_iterator iter = fColRIDmap.find(aTableColName); + ColRIDmap::const_iterator iter = fColRIDmap.find(aTableColName); - if (iter != fColRIDmap.end()) - rp.rid = (*iter).second; - - return rp; + if (iter != fColRIDmap.end()) + rp.rid = (*iter).second; + return rp; } -const CalpontSystemCatalog::RIDList CalpontSystemCatalog::columnRIDs(const TableName& tableName, bool useCache, int lower_case_table_names) +const CalpontSystemCatalog::RIDList CalpontSystemCatalog::columnRIDs(const TableName& tableName, + bool useCache, + int lower_case_table_names) { - TableName aTableName(tableName); + TableName aTableName(tableName); - if (lower_case_table_names) - { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); - } + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } - if (aTableName.schema.empty() || aTableName.table.empty()) - throw runtime_error("ColumnRIDs: Invalid table name"); + if (aTableName.schema.empty() || aTableName.table.empty()) + throw runtime_error("ColumnRIDs: Invalid table name"); - if (aTableName.schema != CALPONT_SCHEMA) - DEBUG << "Enter columnRIDs: " << tableName.schema << "|" << tableName.table << endl; + if (aTableName.schema != CALPONT_SCHEMA) + DEBUG << "Enter columnRIDs: " << tableName.schema << "|" << tableName.table << endl; - RIDList rl; + RIDList rl; - //Check whether cache needs to be flushed - if ( aTableName.schema != CALPONT_SCHEMA) - { - checkSysCatVer(); - } + // Check whether cache needs to be flushed + if (aTableName.schema != CALPONT_SCHEMA) + { + checkSysCatVer(); + } - boost::mutex::scoped_lock lk1(fTableInfoMapLock); - TableInfoMap::const_iterator ti_iter = fTableInfoMap.find(aTableName); + boost::mutex::scoped_lock lk1(fTableInfoMapLock); + TableInfoMap::const_iterator ti_iter = fTableInfoMap.find(aTableName); - // search fOIDmap for system catalog tables - // or if fTableInfoMap has entry for this table, column oids are cached. - // because columnRIDs(), colType() and tableInfo() are actually binded. + // search fOIDmap for system catalog tables + // or if fTableInfoMap has entry for this table, column oids are cached. + // because columnRIDs(), colType() and tableInfo() are actually binded. #if BOOST_VERSION < 103800 - boost::mutex::scoped_lock lk2(fOIDmapLock, false); + boost::mutex::scoped_lock lk2(fOIDmapLock, false); #else - boost::mutex::scoped_lock lk2(fOIDmapLock, boost::defer_lock); + boost::mutex::scoped_lock lk2(fOIDmapLock, boost::defer_lock); #endif - boost::mutex::scoped_lock lk3(fColinfomapLock); + boost::mutex::scoped_lock lk3(fColinfomapLock); - if (aTableName.schema == CALPONT_SCHEMA || (useCache && ti_iter != fTableInfoMap.end())) - { - if (aTableName.schema == CALPONT_SCHEMA) - lk3.unlock(); - else - rl.resize(ti_iter->second.numOfCols); + if (aTableName.schema == CALPONT_SCHEMA || (useCache && ti_iter != fTableInfoMap.end())) + { + if (aTableName.schema == CALPONT_SCHEMA) + lk3.unlock(); + else + rl.resize(ti_iter->second.numOfCols); - lk2.lock(); - - if (aTableName.schema != CALPONT_SCHEMA) - DEBUG << "for " << aTableName << ", searching " << fOIDmap.size() << " oids" << endl; - - OIDmap::const_iterator iter = fOIDmap.begin(); - - while ( iter != fOIDmap.end() ) - { - TableColName tableColName = (*iter).first; - - if ( tableColName.schema == aTableName.schema - && tableColName.table == aTableName.table ) - { - ROPair rp; - rp.objnum = (*iter).second; - ColRIDmap::const_iterator rid_iter = fColRIDmap.find(tableColName); - - if (rid_iter != fColRIDmap.end()) - rp.rid = (*rid_iter).second; - - // @bug 1584. make sure the columns are in position order - if (aTableName.schema == CALPONT_SCHEMA) - { - rl.push_back(rp); - } - else - { - Colinfomap::const_iterator ct_iter = fColinfomap.find(rp.objnum); - rl[ct_iter->second.colPosition] = rp; - } - } - - ++iter; - } - - if (aTableName.schema != CALPONT_SCHEMA) - DEBUG << aTableName << " was cached: " << rl.size() << " rows" << endl; - - return rl; - } - - lk1.unlock(); - lk3.unlock(); + lk2.lock(); if (aTableName.schema != CALPONT_SCHEMA) - DEBUG << aTableName << " was not cached, fetching..." << endl; + DEBUG << "for " << aTableName << ", searching " << fOIDmap.size() << " oids" << endl; - // get real data from system catalog for all user tables. don't check cache - // because cache may not have complete columns for this table - // SQL statement: select objectid,columnname from syscolumn where schema=tableName.schema and - // tablename=tableName.table; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; + OIDmap::const_iterator iter = fOIDmap.begin(); - string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; - string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; - string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; - string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; - string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; - string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; - string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; - string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; - // the following columns will be save in cache although it's not needed for now - string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; - string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; - string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; - string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; - string compressiontype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; - string autoIncrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; - string nextVal = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; - - SimpleColumn* col[17]; - col[0] = new SimpleColumn(columnlength, fSessionID); - col[1] = new SimpleColumn(objectid, fSessionID); - col[2] = new SimpleColumn(datatype, fSessionID); - col[3] = new SimpleColumn(dictobjectid, fSessionID); - col[4] = new SimpleColumn(listobjectid, fSessionID); - col[5] = new SimpleColumn(treeobjectid, fSessionID); - col[6] = new SimpleColumn(columnposition, fSessionID); - col[7] = new SimpleColumn(scale, fSessionID); - col[8] = new SimpleColumn(precision, fSessionID); - col[9] = new SimpleColumn(defaultvalue, fSessionID); - col[10] = new SimpleColumn(schemaname, fSessionID); - col[11] = new SimpleColumn(tablename, fSessionID); - col[12] = new SimpleColumn(columnname, fSessionID); - col[13] = new SimpleColumn(nullable, fSessionID); - col[14] = new SimpleColumn(compressiontype, fSessionID); - col[15] = new SimpleColumn(autoIncrement, fSessionID); - col[16] = new SimpleColumn(nextVal, fSessionID); - - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(columnlength, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(objectid, srcp)); - srcp.reset(col[2]); - colMap.insert(CMVT_(datatype, srcp)); - srcp.reset(col[3]); - colMap.insert(CMVT_(dictobjectid, srcp)); - srcp.reset(col[4]); - colMap.insert(CMVT_(listobjectid, srcp)); - srcp.reset(col[5]); - colMap.insert(CMVT_(treeobjectid, srcp)); - srcp.reset(col[6]); - colMap.insert(CMVT_(columnposition, srcp)); - srcp.reset(col[7]); - colMap.insert(CMVT_(scale, srcp)); - srcp.reset(col[8]); - colMap.insert(CMVT_(precision, srcp)); - - srcp.reset(col[9]); - colMap.insert(CMVT_(defaultvalue, srcp)); - srcp.reset(col[10]); - colMap.insert(CMVT_(schemaname, srcp)); - srcp.reset(col[11]); - colMap.insert(CMVT_(tablename, srcp)); - srcp.reset(col[12]); - colMap.insert(CMVT_(columnname, srcp)); - srcp.reset(col[13]); - colMap.insert(CMVT_(nullable, srcp)); - srcp.reset(col[14]); - colMap.insert(CMVT_(compressiontype, srcp)); - srcp.reset(col[15]); - colMap.insert(CMVT_(autoIncrement, srcp)); - srcp.reset(col[16]); - colMap.insert(CMVT_(nextVal, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(col[1]->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - OID oid[17]; - - for (int i = 0; i < 17; i++) - oid[i] = col[i]->oid(); - - oid[12] = DICTOID_SYSCOLUMN_COLNAME; - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[10]->clone(), - new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - col[11]->clone(), - new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - csep.filterTokenList(filterTokenList); - - ostringstream oss; - oss << "select objectid,columnname from syscolumn where schema='" << aTableName.schema << "' and tablename='" << - aTableName.table << "' --columnRIDs/"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSCOLUMN_TABLE); - - vector::const_iterator it; - ColType ct; - ColType* ctList = NULL; - TableInfo ti; - ti.tablewithautoincr = NO_AUTOINCRCOL; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) + while (iter != fOIDmap.end()) { - if ((*it)->ColumnOID() == oid[1]) // objectid + TableColName tableColName = (*iter).first; + + if (tableColName.schema == aTableName.schema && tableColName.table == aTableName.table) + { + ROPair rp; + rp.objnum = (*iter).second; + ColRIDmap::const_iterator rid_iter = fColRIDmap.find(tableColName); + + if (rid_iter != fColRIDmap.end()) + rp.rid = (*rid_iter).second; + + // @bug 1584. make sure the columns are in position order + if (aTableName.schema == CALPONT_SCHEMA) { - DEBUG << "column count: " << (*it)->dataCount() << endl; - // populate tableinfo cache for numOfCols - ti.numOfCols = (*it)->dataCount(); -// ti.tablewithautoincr = NO_AUTOINCRCOL; - - ctList = new ColType[ti.numOfCols]; - - for (int i = 0 ; i < (*it)->dataCount(); i++) - { - ROPair rp; -// rp.rid = -1; - rp.objnum = (*it)->GetData(i); - - if (fIdentity == EC) - rp.rid = (*it)->GetRid(i); - - DEBUG << rp.rid << " "; - rl.push_back(rp); - ColType ct; - ct.columnOID = rp.objnum; - ctList[i] = ct; - } - - DEBUG << endl; + rl.push_back(rp); } - else if ((*it)->ColumnOID() == oid[15]) //autoincrement + else { - - for (int i = 0 ; i < (*it)->dataCount(); i++) - { - ostringstream os; - os << (char) (*it)->GetData(i); - - if (os.str().compare("y") == 0) - { - ti.tablewithautoincr = AUTOINCRCOL; - break; - } - } + Colinfomap::const_iterator ct_iter = fColinfomap.find(rp.objnum); + rl[ct_iter->second.colPosition] = rp; } + } - lk1.lock(); - fTableInfoMap[aTableName] = ti; - lk1.unlock(); + ++iter; } - // loop 2nd time to make sure rl has been populated. - for (it = sysDataList.begin(); it != sysDataList.end(); it++) + if (aTableName.schema != CALPONT_SCHEMA) + DEBUG << aTableName << " was cached: " << rl.size() << " rows" << endl; + + return rl; + } + + lk1.unlock(); + lk3.unlock(); + + if (aTableName.schema != CALPONT_SCHEMA) + DEBUG << aTableName << " was not cached, fetching..." << endl; + + // get real data from system catalog for all user tables. don't check cache + // because cache may not have complete columns for this table + // SQL statement: select objectid,columnname from syscolumn where schema=tableName.schema and + // tablename=tableName.table; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; + string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; + string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; + string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; + string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; + string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; + string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; + string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; + // the following columns will be save in cache although it's not needed for now + string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; + string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; + string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; + string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; + string compressiontype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; + string autoIncrement = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; + string nextVal = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; + + SimpleColumn* col[17]; + col[0] = new SimpleColumn(columnlength, fSessionID); + col[1] = new SimpleColumn(objectid, fSessionID); + col[2] = new SimpleColumn(datatype, fSessionID); + col[3] = new SimpleColumn(dictobjectid, fSessionID); + col[4] = new SimpleColumn(listobjectid, fSessionID); + col[5] = new SimpleColumn(treeobjectid, fSessionID); + col[6] = new SimpleColumn(columnposition, fSessionID); + col[7] = new SimpleColumn(scale, fSessionID); + col[8] = new SimpleColumn(precision, fSessionID); + col[9] = new SimpleColumn(defaultvalue, fSessionID); + col[10] = new SimpleColumn(schemaname, fSessionID); + col[11] = new SimpleColumn(tablename, fSessionID); + col[12] = new SimpleColumn(columnname, fSessionID); + col[13] = new SimpleColumn(nullable, fSessionID); + col[14] = new SimpleColumn(compressiontype, fSessionID); + col[15] = new SimpleColumn(autoIncrement, fSessionID); + col[16] = new SimpleColumn(nextVal, fSessionID); + + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(columnlength, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(objectid, srcp)); + srcp.reset(col[2]); + colMap.insert(CMVT_(datatype, srcp)); + srcp.reset(col[3]); + colMap.insert(CMVT_(dictobjectid, srcp)); + srcp.reset(col[4]); + colMap.insert(CMVT_(listobjectid, srcp)); + srcp.reset(col[5]); + colMap.insert(CMVT_(treeobjectid, srcp)); + srcp.reset(col[6]); + colMap.insert(CMVT_(columnposition, srcp)); + srcp.reset(col[7]); + colMap.insert(CMVT_(scale, srcp)); + srcp.reset(col[8]); + colMap.insert(CMVT_(precision, srcp)); + + srcp.reset(col[9]); + colMap.insert(CMVT_(defaultvalue, srcp)); + srcp.reset(col[10]); + colMap.insert(CMVT_(schemaname, srcp)); + srcp.reset(col[11]); + colMap.insert(CMVT_(tablename, srcp)); + srcp.reset(col[12]); + colMap.insert(CMVT_(columnname, srcp)); + srcp.reset(col[13]); + colMap.insert(CMVT_(nullable, srcp)); + srcp.reset(col[14]); + colMap.insert(CMVT_(compressiontype, srcp)); + srcp.reset(col[15]); + colMap.insert(CMVT_(autoIncrement, srcp)); + srcp.reset(col[16]); + colMap.insert(CMVT_(nextVal, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(col[1]->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + OID oid[17]; + + for (int i = 0; i < 17; i++) + oid[i] = col[i]->oid(); + + oid[12] = DICTOID_SYSCOLUMN_COLNAME; + // Filters + SimpleFilter* f1 = new SimpleFilter(opeq, col[10]->clone(), + new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, col[11]->clone(), new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select objectid,columnname from syscolumn where schema='" << aTableName.schema + << "' and tablename='" << aTableName.table << "' --columnRIDs/"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + + vector::const_iterator it; + ColType ct; + ColType* ctList = NULL; + TableInfo ti; + ti.tablewithautoincr = NO_AUTOINCRCOL; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[1]) // objectid { - if ((*it)->ColumnOID() == oid[12]) - { - lk2.lock(); + DEBUG << "column count: " << (*it)->dataCount() << endl; + // populate tableinfo cache for numOfCols + ti.numOfCols = (*it)->dataCount(); + // ti.tablewithautoincr = NO_AUTOINCRCOL; - for (int i = 0; i < (*it)->dataCount(); i++) - { - TableColName tcn = make_tcn(aTableName.schema, aTableName.table, (*it)->GetStringData(i)); - fOIDmap[tcn] = rl[i].objnum; + ctList = new ColType[ti.numOfCols]; - if (fIdentity == EC) - fColRIDmap[tcn] = rl[i].rid; - } + for (int i = 0; i < (*it)->dataCount(); i++) + { + ROPair rp; + // rp.rid = -1; + rp.objnum = (*it)->GetData(i); - lk2.unlock(); - } - else if ((*it)->ColumnOID() == oid[0]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].colWidth = (*it)->GetData(i); - } - else if ((*it)->ColumnOID() == oid[2]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].colDataType = (ColDataType)((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[3]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].ddn.dictOID = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[4]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].ddn.listOID = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[5]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].ddn.treeOID = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[6]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].colPosition = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[7]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].scale = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[8]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].precision = ((*it)->GetData(i)); - } - // TODO: check datatype to call GetData() or GetStringData() - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - ctList[i].defaultValue = ((*it)->GetStringData(i)); + if (fIdentity == EC) + rp.rid = (*it)->GetRid(i); - if ((!ctList[i].defaultValue.empty()) || (ctList[i].defaultValue.length() > 0)) - { - if (ctList[i].constraintType != NOTNULL_CONSTRAINT) - ctList[i].constraintType = DEFAULT_CONSTRAINT; - } - } - } - else if ((*it)->ColumnOID() == oid[13]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - if ((*it)->GetData(i) == 0) - ctList[i].constraintType = NOTNULL_CONSTRAINT; - } - else if ((*it)->ColumnOID() == oid[14]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].compressionType = ctList[i].ddn.compressionType = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[15]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - ostringstream os; - os << (char) (*it)->GetData(i); + DEBUG << rp.rid << " "; + rl.push_back(rp); + ColType ct; + ct.columnOID = rp.objnum; + ctList[i] = ct; + } - if (os.str().compare("y") == 0) - ctList[i].autoincrement = true; - else - ctList[i].autoincrement = false; - } - } - else if ((*it)->ColumnOID() == oid[16]) + DEBUG << endl; + } + else if ((*it)->ColumnOID() == oid[15]) // autoincrement + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + ostringstream os; + os << (char)(*it)->GetData(i); + + if (os.str().compare("y") == 0) { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].nextvalue = ((*it)->GetData(i)); + ti.tablewithautoincr = AUTOINCRCOL; + break; } + } } - // MCOL-895 sort ctList, we can't specify an ORDER BY to do this yet - std::sort(ctList, ctList + ti.numOfCols, ctListSort); + lk1.lock(); + fTableInfoMap[aTableName] = ti; + lk1.unlock(); + } - // populate colinfo cache - lk3.lock(); - - for (int i = 0; i < ti.numOfCols; i++) - fColinfomap[ctList[i].columnOID] = ctList[i]; - - lk3.unlock(); - - // Re-sort the output based on the sorted ctList - // Don't need to do this for the cached list as this will be already sorted - RIDList rlOut; - - for (int i = 0; i < ti.numOfCols; i++) + // loop 2nd time to make sure rl has been populated. + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[12]) { - OID objid = ctList[i].columnOID; + lk2.lock(); - for (size_t j = 0; j < rl.size(); j++) + for (int i = 0; i < (*it)->dataCount(); i++) + { + TableColName tcn = make_tcn(aTableName.schema, aTableName.table, (*it)->GetStringData(i)); + fOIDmap[tcn] = rl[i].objnum; + + if (fIdentity == EC) + fColRIDmap[tcn] = rl[i].rid; + } + + lk2.unlock(); + } + else if ((*it)->ColumnOID() == oid[0]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].colWidth = (*it)->GetData(i); + } + else if ((*it)->ColumnOID() == oid[2]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].colDataType = (ColDataType)((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[3]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].ddn.dictOID = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[4]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].ddn.listOID = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[5]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].ddn.treeOID = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[6]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].colPosition = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[7]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].scale = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[8]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].precision = ((*it)->GetData(i)); + } + // TODO: check datatype to call GetData() or GetStringData() + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + ctList[i].defaultValue = ((*it)->GetStringData(i)); + + if ((!ctList[i].defaultValue.empty()) || (ctList[i].defaultValue.length() > 0)) { - if (rl[j].objnum == objid) - { - rlOut.push_back(rl[j]); - } + if (ctList[i].constraintType != NOTNULL_CONSTRAINT) + ctList[i].constraintType = DEFAULT_CONSTRAINT; } + } } - - delete [] ctList; - - // delete col[9]; - if (rlOut.size() != 0) + else if ((*it)->ColumnOID() == oid[13]) { - return rlOut; + for (int i = 0; i < (*it)->dataCount(); i++) + if ((*it)->GetData(i) == 0) + ctList[i].constraintType = NOTNULL_CONSTRAINT; } + else if ((*it)->ColumnOID() == oid[14]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].compressionType = ctList[i].ddn.compressionType = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[15]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + ostringstream os; + os << (char)(*it)->GetData(i); - Message::Args args; - args.add("'" + tableName.schema + "." + tableName.table + "'"); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); + if (os.str().compare("y") == 0) + ctList[i].autoincrement = true; + else + ctList[i].autoincrement = false; + } + } + else if ((*it)->ColumnOID() == oid[16]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].nextvalue = ((*it)->GetData(i)); + } + } + + // MCOL-895 sort ctList, we can't specify an ORDER BY to do this yet + std::sort(ctList, ctList + ti.numOfCols, ctListSort); + + // populate colinfo cache + lk3.lock(); + + for (int i = 0; i < ti.numOfCols; i++) + fColinfomap[ctList[i].columnOID] = ctList[i]; + + lk3.unlock(); + + // Re-sort the output based on the sorted ctList + // Don't need to do this for the cached list as this will be already sorted + RIDList rlOut; + + for (int i = 0; i < ti.numOfCols; i++) + { + OID objid = ctList[i].columnOID; + + for (size_t j = 0; j < rl.size(); j++) + { + if (rl[j].objnum == objid) + { + rlOut.push_back(rl[j]); + } + } + } + + delete[] ctList; + + // delete col[9]; + if (rlOut.size() != 0) + { + return rlOut; + } + + Message::Args args; + args.add("'" + tableName.schema + "." + tableName.table + "'"); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); } const CalpontSystemCatalog::TableName CalpontSystemCatalog::tableName(const OID& tableoid) { - //Check whether cache needs to be flushed - if ( tableoid >= 3000) + // Check whether cache needs to be flushed + if (tableoid >= 3000) + { + checkSysCatVer(); + } + + // check cache + boost::mutex::scoped_lock lk(fTableNameMapLock); + + if (fTableNameMap.size() > 0) + { + TableNameMap::const_iterator iter = fTableNameMap.find(tableoid); + + if (iter != fTableNameMap.end()) + return iter->second; + } + + lk.unlock(); + + // select schema, tablename from systable where objectid = tableoid + TableName tableName; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); + SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c3 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); + + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(c2->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c3->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + OID oid2 = DICTOID_SYSTABLE_SCHEMA; + OID oid3 = DICTOID_SYSTABLE_TABLENAME; + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c1->clone(), new ConstantColumn((int64_t)tableoid, ConstantColumn::NUM)); + filterTokenList.push_back(f1); + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select schema,tablename,columnname from syscolumn where objectid=" << tableoid << " --colName/"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + + try + { + getSysData(csep, sysDataList, SYSTABLE_TABLE); + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } + + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->dataCount() == 0) { - checkSysCatVer(); + Message::Args args; + oss << tableoid; + args.add("'" + oss.str() + "'"); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); } - // check cache - boost::mutex::scoped_lock lk(fTableNameMapLock); + if ((*it)->ColumnOID() == oid2) + tableName.schema = (*it)->GetStringData(0); + else if ((*it)->ColumnOID() == oid3) + tableName.table = (*it)->GetStringData(0); + } - if ( fTableNameMap.size() > 0 ) - { - TableNameMap::const_iterator iter = fTableNameMap.find(tableoid); + //@Bug 2682. datacount 0 sometimes does not mean the table is not found. + if ((tableName.schema.length() == 0) || (tableName.table.length() == 0)) + { + Message::Args args; + oss << tableoid; + args.add("'" + oss.str() + "'"); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); + } - if (iter != fTableNameMap.end()) - return iter->second; - } - - lk.unlock(); - - //select schema, tablename from systable where objectid = tableoid - TableName tableName; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); - - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(c2->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c3->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - - OID oid2 = DICTOID_SYSTABLE_SCHEMA; - OID oid3 = DICTOID_SYSTABLE_TABLENAME; - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c1->clone(), - new ConstantColumn((int64_t)tableoid, ConstantColumn::NUM)); - filterTokenList.push_back(f1); - csep.filterTokenList(filterTokenList); - - ostringstream oss; - oss << "select schema,tablename,columnname from syscolumn where objectid=" << tableoid << - " --colName/"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - - try - { - getSysData (csep, sysDataList, SYSTABLE_TABLE); - } - catch (runtime_error& e) - { - throw runtime_error ( e.what() ); - } - - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->dataCount() == 0) - { - Message::Args args; - oss << tableoid; - args.add("'" + oss.str() + "'"); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); - } - - if ((*it)->ColumnOID() == oid2) - tableName.schema = (*it)->GetStringData(0); - else if ((*it)->ColumnOID() == oid3) - tableName.table = (*it)->GetStringData(0); - } - - //@Bug 2682. datacount 0 sometimes does not mean the table is not found. - if ( (tableName.schema.length() == 0) || (tableName.table.length() == 0) ) - { - Message::Args args; - oss << tableoid; - args.add("'" + oss.str() + "'"); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); - } - - // populate cache - lk.lock(); - fTableNameMap[tableoid] = tableName; - lk.unlock(); - return tableName; + // populate cache + lk.lock(); + fTableNameMap[tableoid] = tableName; + lk.unlock(); + return tableName; } - -const CalpontSystemCatalog::ROPair CalpontSystemCatalog::tableRID(const TableName& tableName, int lower_case_table_names) +const CalpontSystemCatalog::ROPair CalpontSystemCatalog::tableRID(const TableName& tableName, + int lower_case_table_names) { - TableName aTableName; - aTableName.schema = tableName.schema; - aTableName.table = tableName.table; - if (lower_case_table_names) + TableName aTableName; + aTableName.schema = tableName.schema; + aTableName.table = tableName.table; + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } + + if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) + DEBUG << "Enter tableRID: " << tableName.schema << "|" << tableName.table << endl; + + // look up cache first for system table and cached table + ROPair rp; + // rp.rid = -1; @bug1866 use default + + // calpontsys only needs oid + boost::mutex::scoped_lock lk1(fTableInfoMapLock); + Tablemap::const_iterator iter = fTablemap.find(aTableName); + + if (aTableName.schema.compare("calpontsys") == 0 && iter != fTablemap.end()) + { + rp.objnum = (*iter).second; + return rp; + } + + lk1.unlock(); + + checkSysCatVer(); + + lk1.lock(); + iter = fTablemap.find(aTableName); + TableRIDmap::const_iterator rid_iter = fTableRIDmap.find(aTableName); + + if (iter != fTablemap.end() && rid_iter != fTableRIDmap.end()) + { + rp.objnum = (*iter).second; + rp.rid = (*rid_iter).second; + return rp; + } + + lk1.unlock(); + + // select objectid from systable where schema = tableName.schema and tablename = tableName.table; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); + SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c3 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); + + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + OID oid = c1->oid(); + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c2->clone(), new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, c3->clone(), new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select objectid from systable where schema='" << aTableName.schema << "' and tablename='" + << aTableName.table << "' --tableRID/"; + + csep.data(oss.str()); //@bug 6078. Log the statement + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + NJLSysDataList sysDataList; + + try + { + getSysData(csep, sysDataList, SYSTABLE_TABLE); + } + catch (IDBExcept&) + { + throw; + } + catch (runtime_error& e) + { + throw runtime_error(e.what()); + } + + vector::const_iterator it; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->dataCount() == 0) { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); + Message::Args args; + args.add("'" + tableName.schema + "." + tableName.table + "'"); + // throw logging::NoTableExcept(msg); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); } - if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) - DEBUG << "Enter tableRID: " << tableName.schema << "|" << tableName.table << endl; - - // look up cache first for system table and cached table - ROPair rp; -// rp.rid = -1; @bug1866 use default - - // calpontsys only needs oid - boost::mutex::scoped_lock lk1(fTableInfoMapLock); - Tablemap::const_iterator iter = fTablemap.find(aTableName); - - if (aTableName.schema.compare("calpontsys") == 0 && iter != fTablemap.end() ) + if ((*it)->ColumnOID() == oid) { - rp.objnum = (*iter).second; - return rp; + rp.objnum = (OID)((*it)->GetData(0)); + + if (fIdentity == EC) + { + rp.rid = (*it)->GetRid(0); + } + + // populate cache + lk1.lock(); + fTablemap[aTableName] = rp.objnum; + fTableRIDmap[aTableName] = rp.rid; + return rp; } + } - lk1.unlock(); - - checkSysCatVer(); - - lk1.lock(); - iter = fTablemap.find(aTableName); - TableRIDmap::const_iterator rid_iter = fTableRIDmap.find(aTableName); - - if (iter != fTablemap.end() && rid_iter != fTableRIDmap.end()) - { - rp.objnum = (*iter).second; - rp.rid = (*rid_iter).second; - return rp; - } - - lk1.unlock(); - - // select objectid from systable where schema = tableName.schema and tablename = tableName.table; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); - - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - OID oid = c1->oid(); - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c2->clone(), - new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - c3->clone(), - new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - csep.filterTokenList(filterTokenList); - - ostringstream oss; - oss << "select objectid from systable where schema='" << aTableName.schema << "' and tablename='" << - aTableName.table << "' --tableRID/"; - - csep.data(oss.str()); //@bug 6078. Log the statement - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - NJLSysDataList sysDataList; - - try - { - getSysData (csep, sysDataList, SYSTABLE_TABLE); - } - catch ( IDBExcept& ) - { - throw; - } - catch ( runtime_error& e ) - { - throw runtime_error ( e.what() ); - } - - vector::const_iterator it; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->dataCount() == 0) - { - Message::Args args; - args.add("'" + tableName.schema + "." + tableName.table + "'"); - //throw logging::NoTableExcept(msg); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); - } - - if ((*it)->ColumnOID() == oid) - { - rp.objnum = (OID)((*it)->GetData(0)); - - if (fIdentity == EC) - { - rp.rid = (*it)->GetRid(0); - } - - // populate cache - lk1.lock(); - fTablemap[aTableName] = rp.objnum; - fTableRIDmap[aTableName] = rp.rid; - return rp; - } - } - - //string msg("CalpontSystemCatalog::tableRID: no OID found for "); - //msg += tableName.schema; - //msg += "."; - //msg += tableName.table; - Message::Args args; - args.add("'" + tableName.schema + "." + tableName.table + "'"); - //throw logging::NoTableExcept(msg); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); + // string msg("CalpontSystemCatalog::tableRID: no OID found for "); + // msg += tableName.schema; + // msg += "."; + // msg += tableName.table; + Message::Args args; + args.add("'" + tableName.schema + "." + tableName.table + "'"); + // throw logging::NoTableExcept(msg); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); } #if 0 @@ -4035,11 +3988,11 @@ const CalpontSystemCatalog::ROPair CalpontSystemCatalog::indexRID(const IndexNam int CalpontSystemCatalog::colNumbers(const TableName& tableName, int lower_case_table_names) { - DEBUG << "Enter colNumbers: " << tableName.schema << "|" << tableName.table << endl; + DEBUG << "Enter colNumbers: " << tableName.schema << "|" << tableName.table << endl; - TableInfo ti = tableInfo(tableName, lower_case_table_names); + TableInfo ti = tableInfo(tableName, lower_case_table_names); - return ti.numOfCols; + return ti.numOfCols; } #if 0 @@ -4754,126 +4707,128 @@ const CalpontSystemCatalog::IndexOIDList CalpontSystemCatalog::indexOIDs( const } #endif -const CalpontSystemCatalog::DictOIDList CalpontSystemCatalog::dictOIDs(const TableName& tableName, int lower_case_table_names) +const CalpontSystemCatalog::DictOIDList CalpontSystemCatalog::dictOIDs(const TableName& tableName, + int lower_case_table_names) { - /* SQL statement: select dictobjectid, listobjectid, treeobjectid from syscolumn where - * schema=tableName.schema and table=tableName.table;*/ - DictOIDList dictOIDList; - TableColName aTableName; - CalpontSystemCatalog::DictOID dictoid; + /* SQL statement: select dictobjectid, listobjectid, treeobjectid from syscolumn where + * schema=tableName.schema and table=tableName.table;*/ + DictOIDList dictOIDList; + TableColName aTableName; + CalpontSystemCatalog::DictOID dictoid; - aTableName.schema = tableName.schema; - aTableName.table = tableName.table; - if (lower_case_table_names) - { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); - } + aTableName.schema = tableName.schema; + aTableName.table = tableName.table; + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } - if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) - DEBUG << "Enter dictOIDs: " << tableName.schema << "|" << tableName.table << endl; - - // return pre-defined indexoid for system catalog index. currently no index - // created for system catalog, return invalid(default) indexoid. - if (aTableName.schema.compare(CALPONT_SCHEMA) == 0) - return dictOIDList; - - // select objectid from syscolumn where schema = tableColName.schema and tablename = tableColName.table and columnname = tableColName.column; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SimpleColumn* c1 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL, fSessionID); - SimpleColumn* c2 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL, fSessionID); - SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, fSessionID); - SimpleColumn* c4 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, fSessionID); - SimpleColumn* c5 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, fSessionID); - - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, srcp)); - srcp.reset(c4); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, srcp)); - srcp.reset(c5); - colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c2->clone()); - returnedColumnList.push_back(srcp); - srcp.reset(c5->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - OID oid1 = c1->oid(); - OID oid2 = c2->oid(); - OID oid3 = c5->oid(); - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - c3->clone(), - new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - c4->clone(), - new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - SOP opisnotnull(new Operator("isnotnull")); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter (opisnotnull, - c5->clone(), - new ConstantColumn("", ConstantColumn::NULLDATA)); - filterTokenList.push_back(f3); - csep.filterTokenList(filterTokenList); - - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSCOLUMN_TABLE); - - vector::const_iterator it; - - // loop for oid1 first to make sure dictOIDList is populated - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->dataCount() == 0) - return dictOIDList; - - if ((*it)->ColumnOID() == oid1) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - dictoid.listOID = (*it)->GetData(i); - dictOIDList.push_back(dictoid); - } - - break; - } - } - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) - { - if ((*it)->ColumnOID() == oid2) - { - for (int i = 0; i < (*it)->dataCount(); i++) - dictOIDList[i].treeOID = (*it)->GetData(i); - } - - else if ((*it)->ColumnOID() == oid3) - { - for (int i = 0; i < (*it)->dataCount(); i++) - dictOIDList[i].dictOID = (*it)->GetData(i); - } - } + if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) + DEBUG << "Enter dictOIDs: " << tableName.schema << "|" << tableName.table << endl; + // return pre-defined indexoid for system catalog index. currently no index + // created for system catalog, return invalid(default) indexoid. + if (aTableName.schema.compare(CALPONT_SCHEMA) == 0) return dictOIDList; + + // select objectid from syscolumn where schema = tableColName.schema and tablename = tableColName.table and + // columnname = tableColName.column; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SimpleColumn* c1 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL, fSessionID); + SimpleColumn* c2 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL, fSessionID); + SimpleColumn* c3 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, fSessionID); + SimpleColumn* c4 = + new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, fSessionID); + SimpleColumn* c5 = new SimpleColumn(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, fSessionID); + + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL, srcp)); + srcp.reset(c5); + colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c2->clone()); + returnedColumnList.push_back(srcp); + srcp.reset(c5->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + OID oid1 = c1->oid(); + OID oid2 = c2->oid(); + OID oid3 = c5->oid(); + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, c3->clone(), new ConstantColumn(aTableName.schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, c4->clone(), new ConstantColumn(aTableName.table, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + SOP opisnotnull(new Operator("isnotnull")); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f3 = + new SimpleFilter(opisnotnull, c5->clone(), new ConstantColumn("", ConstantColumn::NULLDATA)); + filterTokenList.push_back(f3); + csep.filterTokenList(filterTokenList); + + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + + vector::const_iterator it; + + // loop for oid1 first to make sure dictOIDList is populated + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->dataCount() == 0) + return dictOIDList; + + if ((*it)->ColumnOID() == oid1) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + dictoid.listOID = (*it)->GetData(i); + dictOIDList.push_back(dictoid); + } + + break; + } + } + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid2) + { + for (int i = 0; i < (*it)->dataCount(); i++) + dictOIDList[i].treeOID = (*it)->GetData(i); + } + + else if ((*it)->ColumnOID() == oid3) + { + for (int i = 0; i < (*it)->dataCount(); i++) + dictOIDList[i].dictOID = (*it)->GetData(i); + } + } + + return dictOIDList; } -#if 0 //Not implemented +#if 0 // Not implemented void CalpontSystemCatalog::storeColOID(void) { } @@ -4891,60 +4846,61 @@ void CalpontSystemCatalog::updateColInfo(void) } #endif -int CalpontSystemCatalog::colPosition (const OID& oid) +int CalpontSystemCatalog::colPosition(const OID& oid) { - DEBUG << "Enter colPosition: " << oid << endl; - ColType col = colType (oid); - return col.colPosition; + DEBUG << "Enter colPosition: " << oid << endl; + ColType col = colType(oid); + return col.colPosition; } -const CalpontSystemCatalog::TableInfo CalpontSystemCatalog::tableInfo(const TableName& tb, int lower_case_table_names) +const CalpontSystemCatalog::TableInfo CalpontSystemCatalog::tableInfo(const TableName& tb, + int lower_case_table_names) { - TableName aTableName; - aTableName.schema = tb.schema; - aTableName.table = tb.table; - if (lower_case_table_names) - { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); - } + TableName aTableName; + aTableName.schema = tb.schema; + aTableName.table = tb.table; + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } - if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) - DEBUG << "Enter tableInfo: " << tb.schema << "|" << tb.table << endl; + if (aTableName.schema.compare(CALPONT_SCHEMA) != 0) + DEBUG << "Enter tableInfo: " << tb.schema << "|" << tb.table << endl; - // look up cache first - TableInfo ti; - RIDList ridlist ; + // look up cache first + TableInfo ti; + RIDList ridlist; - // select count(objectid) from syscolumn where schema=tableName.schema and tablename=tableName.table; - try - { - ridlist = columnRIDs(aTableName); - } - catch (logging::IDBExcept& noTable) - { - throw runtime_error (noTable.what()); - } + // select count(objectid) from syscolumn where schema=tableName.schema and tablename=tableName.table; + try + { + ridlist = columnRIDs(aTableName); + } + catch (logging::IDBExcept& noTable) + { + throw runtime_error(noTable.what()); + } - if (ridlist.size() == 0) - throw runtime_error ("No table info found for" + tb.schema + "." + tb.table); + if (ridlist.size() == 0) + throw runtime_error("No table info found for" + tb.schema + "." + tb.table); - if ( aTableName.schema.compare(CALPONT_SCHEMA) == 0) - { - ti.numOfCols = ridlist.size(); - ti.tablewithautoincr = 0; - return ti; - } + if (aTableName.schema.compare(CALPONT_SCHEMA) == 0) + { + ti.numOfCols = ridlist.size(); + ti.tablewithautoincr = 0; + return ti; + } - boost::mutex::scoped_lock lk1(fTableInfoMapLock); - TableInfoMap::const_iterator ti_iter = fTableInfoMap.find(aTableName); + boost::mutex::scoped_lock lk1(fTableInfoMapLock); + TableInfoMap::const_iterator ti_iter = fTableInfoMap.find(aTableName); - if (ti_iter != fTableInfoMap.end()) - { - return (*ti_iter).second; - } - else - throw runtime_error ("No table info found for" + tb.schema + "." + tb.table); + if (ti_iter != fTableInfoMap.end()) + { + return (*ti_iter).second; + } + else + throw runtime_error("No table info found for" + tb.schema + "." + tb.table); } #if 0 @@ -5300,340 +5256,342 @@ const string CalpontSystemCatalog::primaryKeyName (const TableName& tableName ) void CalpontSystemCatalog::getSchemaInfo(const string& in_schema, int lower_case_table_names) { - string schema = in_schema; - if (lower_case_table_names) + string schema = in_schema; + if (lower_case_table_names) + { + boost::algorithm::to_lower(schema); + } + if (schema == CALPONT_SCHEMA) + return; + else + DEBUG << "Enter getSchemaInfo: " << schema << endl; + + // Check whether cache needs to be flushed + checkSysCatVer(); + + boost::mutex::scoped_lock lk(fSchemaCacheLock); + set::iterator setIt = fSchemaCache.find(schema); + + if (setIt != fSchemaCache.end()) + { + DEBUG << "getSchemaInfo Cached" << endl; + return; + } + + lk.unlock(); + + // get table info first + getTables(schema); + + // get column info now + RIDList rl; + + // get real data from system catalog for all user tables under the schema + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; + string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; + string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; + string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; + string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; + string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; + string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; + string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; + string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; + string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; + // the following columns will be save in cache although it's not needed for now + string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; + string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; + string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; + string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; + string compressiontype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; + string autoinc = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; + string nextval = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; + + SimpleColumn* col[17]; + col[0] = new SimpleColumn(columnlength, fSessionID); + col[1] = new SimpleColumn(objectid, fSessionID); + col[2] = new SimpleColumn(datatype, fSessionID); + col[3] = new SimpleColumn(dictobjectid, fSessionID); + col[4] = new SimpleColumn(listobjectid, fSessionID); + col[5] = new SimpleColumn(treeobjectid, fSessionID); + col[6] = new SimpleColumn(columnposition, fSessionID); + col[7] = new SimpleColumn(scale, fSessionID); + col[8] = new SimpleColumn(precision, fSessionID); + col[9] = new SimpleColumn(defaultvalue, fSessionID); + col[10] = new SimpleColumn(schemaname, fSessionID); + col[11] = new SimpleColumn(tablename, fSessionID); + col[12] = new SimpleColumn(columnname, fSessionID); + col[13] = new SimpleColumn(nullable, fSessionID); + col[14] = new SimpleColumn(compressiontype, fSessionID); + col[15] = new SimpleColumn(autoinc, fSessionID); + col[16] = new SimpleColumn(nextval, fSessionID); + + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(columnlength, srcp)); + srcp.reset(col[1]); + colMap.insert(CMVT_(objectid, srcp)); + srcp.reset(col[2]); + colMap.insert(CMVT_(datatype, srcp)); + srcp.reset(col[3]); + colMap.insert(CMVT_(dictobjectid, srcp)); + srcp.reset(col[4]); + colMap.insert(CMVT_(listobjectid, srcp)); + srcp.reset(col[5]); + colMap.insert(CMVT_(treeobjectid, srcp)); + srcp.reset(col[6]); + colMap.insert(CMVT_(columnposition, srcp)); + srcp.reset(col[7]); + colMap.insert(CMVT_(scale, srcp)); + srcp.reset(col[8]); + colMap.insert(CMVT_(precision, srcp)); + // TODO: NULL value handling & convert to static_any::any + // delete this manually at fcn exit + srcp.reset(col[9]); + colMap.insert(CMVT_(defaultvalue, srcp)); + srcp.reset(col[10]); + colMap.insert(CMVT_(schemaname, srcp)); + srcp.reset(col[11]); + colMap.insert(CMVT_(tablename, srcp)); + srcp.reset(col[12]); + colMap.insert(CMVT_(columnname, srcp)); + srcp.reset(col[13]); + colMap.insert(CMVT_(nullable, srcp)); + srcp.reset(col[14]); + colMap.insert(CMVT_(compressiontype, srcp)); + srcp.reset(col[15]); + colMap.insert(CMVT_(autoinc, srcp)); + srcp.reset(col[16]); + colMap.insert(CMVT_(nextval, srcp)); + csep.columnMapNonStatic(colMap); + + srcp.reset(col[1]->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + OID oid[17]; + + for (int i = 0; i < 17; i++) + oid[i] = col[i]->oid(); + + oid[12] = DICTOID_SYSCOLUMN_COLNAME; + oid[11] = DICTOID_SYSCOLUMN_TABLENAME; + + // Filters + SimpleFilter* f1 = + new SimpleFilter(opeq, col[10]->clone(), new ConstantColumn(schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + + csep.filterTokenList(filterTokenList); + + ostringstream oss; + oss << "select objectid,columnname from syscolumn where schema='" << schema << "' --getSchemaInfo/"; + + if (fIdentity == EC) + oss << "EC"; + else + oss << "FE"; + + csep.data(oss.str()); + NJLSysDataList sysDataList; + getSysData(csep, sysDataList, SYSCOLUMN_TABLE); + + vector::const_iterator it; + ColType ct; + // ColType *ctList = NULL; + vector ctList; + vector tableNames; + TableInfo ti; + map tbInfo; + map::iterator tbIter; + + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[1]) // objectid { - boost::algorithm::to_lower(schema); + for (int i = 0; i < (*it)->dataCount(); i++) + { + ROPair rp; + rp.objnum = (*it)->GetData(i); + + if (fIdentity == EC) + rp.rid = (*it)->GetRid(i); + + // DEBUG << rp.rid << " "; + rl.push_back(rp); + ColType ct; + ct.columnOID = rp.objnum; + ctList.push_back(ct); + } + + DEBUG << endl; } - if (schema == CALPONT_SCHEMA) - return; - else - DEBUG << "Enter getSchemaInfo: " << schema << endl; - - //Check whether cache needs to be flushed - checkSysCatVer(); - - boost::mutex::scoped_lock lk(fSchemaCacheLock); - set::iterator setIt = fSchemaCache.find(schema); - - if (setIt != fSchemaCache.end()) + // table name + else if ((*it)->ColumnOID() == oid[11]) { - DEBUG << "getSchemaInfo Cached" << endl; - return; + for (int i = 0; i < (*it)->dataCount(); i++) + { + tableNames.push_back((*it)->GetStringData(i)); + tbIter = tbInfo.find(tableNames[i]); + + if (tbIter == tbInfo.end()) + { + tbInfo[tableNames[i]].numOfCols = 1; + } + else + tbInfo[tableNames[i]].numOfCols += 1; + } } + } - lk.unlock(); - - // get table info first - getTables(schema); - - // get column info now - RIDList rl; - - // get real data from system catalog for all user tables under the schema - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - string columnlength = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNLEN_COL; - string objectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + OBJECTID_COL; - string datatype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DATATYPE_COL; - string dictobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DICTOID_COL; - string listobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + LISTOBJID_COL; - string treeobjectid = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TREEOBJID_COL; - string columnposition = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLUMNPOS_COL; - string scale = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCALE_COL; - string precision = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + PRECISION_COL; - string defaultvalue = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + DEFAULTVAL_COL; - // the following columns will be save in cache although it's not needed for now - string columnname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COLNAME_COL; - string tablename = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + TABLENAME_COL; - string schemaname = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + SCHEMA_COL; - string nullable = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NULLABLE_COL; - string compressiontype = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + COMPRESSIONTYPE_COL; - string autoinc = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + AUTOINC_COL; - string nextval = CALPONT_SCHEMA + "." + SYSCOLUMN_TABLE + "." + NEXTVALUE_COL; - - SimpleColumn* col[17]; - col[0] = new SimpleColumn(columnlength, fSessionID); - col[1] = new SimpleColumn(objectid, fSessionID); - col[2] = new SimpleColumn(datatype, fSessionID); - col[3] = new SimpleColumn(dictobjectid, fSessionID); - col[4] = new SimpleColumn(listobjectid, fSessionID); - col[5] = new SimpleColumn(treeobjectid, fSessionID); - col[6] = new SimpleColumn(columnposition, fSessionID); - col[7] = new SimpleColumn(scale, fSessionID); - col[8] = new SimpleColumn(precision, fSessionID); - col[9] = new SimpleColumn(defaultvalue, fSessionID); - col[10] = new SimpleColumn(schemaname, fSessionID); - col[11] = new SimpleColumn(tablename, fSessionID); - col[12] = new SimpleColumn(columnname, fSessionID); - col[13] = new SimpleColumn(nullable, fSessionID); - col[14] = new SimpleColumn(compressiontype, fSessionID); - col[15] = new SimpleColumn(autoinc, fSessionID); - col[16] = new SimpleColumn(nextval, fSessionID); - - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(columnlength, srcp)); - srcp.reset(col[1]); - colMap.insert(CMVT_(objectid, srcp)); - srcp.reset(col[2]); - colMap.insert(CMVT_(datatype, srcp)); - srcp.reset(col[3]); - colMap.insert(CMVT_(dictobjectid, srcp)); - srcp.reset(col[4]); - colMap.insert(CMVT_(listobjectid, srcp)); - srcp.reset(col[5]); - colMap.insert(CMVT_(treeobjectid, srcp)); - srcp.reset(col[6]); - colMap.insert(CMVT_(columnposition, srcp)); - srcp.reset(col[7]); - colMap.insert(CMVT_(scale, srcp)); - srcp.reset(col[8]); - colMap.insert(CMVT_(precision, srcp)); - // TODO: NULL value handling & convert to static_any::any - // delete this manually at fcn exit - srcp.reset(col[9]); - colMap.insert(CMVT_(defaultvalue, srcp)); - srcp.reset(col[10]); - colMap.insert(CMVT_(schemaname, srcp)); - srcp.reset(col[11]); - colMap.insert(CMVT_(tablename, srcp)); - srcp.reset(col[12]); - colMap.insert(CMVT_(columnname, srcp)); - srcp.reset(col[13]); - colMap.insert(CMVT_(nullable, srcp)); - srcp.reset(col[14]); - colMap.insert(CMVT_(compressiontype, srcp)); - srcp.reset(col[15]); - colMap.insert(CMVT_(autoinc, srcp)); - srcp.reset(col[16]); - colMap.insert(CMVT_(nextval, srcp)); - csep.columnMapNonStatic(colMap); - - srcp.reset(col[1]->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - OID oid[17]; - - for (int i = 0; i < 17; i++) - oid[i] = col[i]->oid(); - - oid[12] = DICTOID_SYSCOLUMN_COLNAME; - oid[11] = DICTOID_SYSCOLUMN_TABLENAME; - - // Filters - SimpleFilter* f1 = new SimpleFilter (opeq, - col[10]->clone(), - new ConstantColumn(schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - - csep.filterTokenList(filterTokenList); - - ostringstream oss; - oss << "select objectid,columnname from syscolumn where schema='" << schema << "' --getSchemaInfo/"; - - if (fIdentity == EC) oss << "EC"; - else oss << "FE"; - - csep.data(oss.str()); - NJLSysDataList sysDataList; - getSysData (csep, sysDataList, SYSCOLUMN_TABLE); - - vector::const_iterator it; - ColType ct; - //ColType *ctList = NULL; - vector ctList; - vector tableNames; - TableInfo ti; - map tbInfo; - map::iterator tbIter; - - for (it = sysDataList.begin(); it != sysDataList.end(); it++) + // loop 3rd time to populate col cache + for (it = sysDataList.begin(); it != sysDataList.end(); it++) + { + if ((*it)->ColumnOID() == oid[15]) { - if ((*it)->ColumnOID() == oid[1]) // objectid + for (int i = 0; i < (*it)->dataCount(); i++) + { + ostringstream os; + os << (char)(*it)->GetData(i); + tbIter = tbInfo.find(tableNames[i]); + + if (tbIter == tbInfo.end()) { - for (int i = 0 ; i < (*it)->dataCount(); i++) - { - ROPair rp; - rp.objnum = (*it)->GetData(i); - - if (fIdentity == EC) - rp.rid = (*it)->GetRid(i); - - //DEBUG << rp.rid << " "; - rl.push_back(rp); - ColType ct; - ct.columnOID = rp.objnum; - ctList.push_back(ct); - } - - DEBUG << endl; - } - // table name - else if ((*it)->ColumnOID() == oid[11]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - tableNames.push_back((*it)->GetStringData(i)); - tbIter = tbInfo.find(tableNames[i]); - - if (tbIter == tbInfo.end()) - { - tbInfo[tableNames[i]].numOfCols = 1; - } - else - tbInfo[tableNames[i]].numOfCols += 1; - } + if (os.str().compare("y") == 0) + { + tbInfo[tableNames[i]].tablewithautoincr = AUTOINCRCOL; + } + else + { + tbInfo[tableNames[i]].tablewithautoincr = NO_AUTOINCRCOL; + } } + } } - - // loop 3rd time to populate col cache - for (it = sysDataList.begin(); it != sysDataList.end(); it++) + // column name + else if ((*it)->ColumnOID() == oid[12]) { - if ((*it)->ColumnOID() == oid[15]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - ostringstream os; - os << (char) (*it)->GetData(i); - tbIter = tbInfo.find(tableNames[i]); + // lk2.lock(); + for (int i = 0; i < (*it)->dataCount(); i++) + { + TableColName tcn = make_tcn(schema, tableNames[i], (*it)->GetStringData(i)); + fOIDmap[tcn] = rl[i].objnum; - if (tbIter == tbInfo.end()) - { - if (os.str().compare("y") == 0) - { - tbInfo[tableNames[i]].tablewithautoincr = AUTOINCRCOL; - } - else - { - tbInfo[tableNames[i]].tablewithautoincr = NO_AUTOINCRCOL; - } - } - } - } - // column name - else if ((*it)->ColumnOID() == oid[12]) - { - //lk2.lock(); - for (int i = 0; i < (*it)->dataCount(); i++) - { - TableColName tcn = make_tcn(schema, tableNames[i], (*it)->GetStringData(i)); - fOIDmap[tcn] = rl[i].objnum; + if (fIdentity == EC) + fColRIDmap[tcn] = rl[i].rid; + } - if (fIdentity == EC) - fColRIDmap[tcn] = rl[i].rid; - } - - //lk2.unlock(); - } - else if ((*it)->ColumnOID() == oid[0]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].colWidth = (*it)->GetData(i); - } - else if ((*it)->ColumnOID() == oid[2]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].colDataType = (ColDataType)((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[3]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].ddn.dictOID = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[4]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].ddn.listOID = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[5]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].ddn.treeOID = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[6]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].colPosition = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[7]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].scale = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[8]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].precision = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - ctList[i].defaultValue = ((*it)->GetStringData(i)); - - if ((!ctList[i].defaultValue.empty()) || (ctList[i].defaultValue.length() > 0)) - { - if (ctList[i].constraintType != NOTNULL_CONSTRAINT) - ctList[i].constraintType = DEFAULT_CONSTRAINT; - } - } - } - else if ((*it)->ColumnOID() == oid[13]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - if ((*it)->GetData(i) == 0) - ctList[i].constraintType = NOTNULL_CONSTRAINT; - } - else if ((*it)->ColumnOID() == oid[14]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].compressionType = ctList[i].ddn.compressionType = ((*it)->GetData(i)); - } - else if ((*it)->ColumnOID() == oid[15]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - { - ostringstream os; - os << (char) (*it)->GetData(i); - - if (os.str().compare("y") == 0) - ctList[i].autoincrement = true; - else - ctList[i].autoincrement = false; - } - } - else if ((*it)->ColumnOID() == oid[16]) - { - for (int i = 0; i < (*it)->dataCount(); i++) - ctList[i].nextvalue = ((*it)->GetData(i)); - } + // lk2.unlock(); } - - // populate colinfo cache - //boost::mutex::scoped_lock lk3(fColinfomapLock); - for (uint32_t i = 0; i < ctList.size(); i++) - fColinfomap[ctList[i].columnOID] = ctList[i]; - - //lk3.unlock(); - // populate tbinfomap - for (tbIter = tbInfo.begin(); tbIter != tbInfo.end(); ++tbIter) + else if ((*it)->ColumnOID() == oid[0]) { - TableName tn(schema, tbIter->first); - //ti.numOfCols = (tbIter->second).numOfCols; - //ti.tablewithautoincr = (tbIter->second).withAutoInc; - fTableInfoMap[tn] = tbIter->second; - DEBUG << tbIter->first << " " << tbIter->second.numOfCols << " " << (tbIter->second).tablewithautoincr << endl; + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].colWidth = (*it)->GetData(i); } + else if ((*it)->ColumnOID() == oid[2]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].colDataType = (ColDataType)((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[3]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].ddn.dictOID = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[4]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].ddn.listOID = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[5]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].ddn.treeOID = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[6]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].colPosition = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[7]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].scale = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[8]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].precision = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == DICTOID_SYSCOLUMN_DEFAULTVAL) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + ctList[i].defaultValue = ((*it)->GetStringData(i)); - //delete col[9]; - lk.lock(); - fSchemaCache.insert(schema); - lk.unlock(); + if ((!ctList[i].defaultValue.empty()) || (ctList[i].defaultValue.length() > 0)) + { + if (ctList[i].constraintType != NOTNULL_CONSTRAINT) + ctList[i].constraintType = DEFAULT_CONSTRAINT; + } + } + } + else if ((*it)->ColumnOID() == oid[13]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + if ((*it)->GetData(i) == 0) + ctList[i].constraintType = NOTNULL_CONSTRAINT; + } + else if ((*it)->ColumnOID() == oid[14]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].compressionType = ctList[i].ddn.compressionType = ((*it)->GetData(i)); + } + else if ((*it)->ColumnOID() == oid[15]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + { + ostringstream os; + os << (char)(*it)->GetData(i); + + if (os.str().compare("y") == 0) + ctList[i].autoincrement = true; + else + ctList[i].autoincrement = false; + } + } + else if ((*it)->ColumnOID() == oid[16]) + { + for (int i = 0; i < (*it)->dataCount(); i++) + ctList[i].nextvalue = ((*it)->GetData(i)); + } + } + + // populate colinfo cache + // boost::mutex::scoped_lock lk3(fColinfomapLock); + for (uint32_t i = 0; i < ctList.size(); i++) + fColinfomap[ctList[i].columnOID] = ctList[i]; + + // lk3.unlock(); + // populate tbinfomap + for (tbIter = tbInfo.begin(); tbIter != tbInfo.end(); ++tbIter) + { + TableName tn(schema, tbIter->first); + // ti.numOfCols = (tbIter->second).numOfCols; + // ti.tablewithautoincr = (tbIter->second).withAutoInc; + fTableInfoMap[tn] = tbIter->second; + DEBUG << tbIter->first << " " << tbIter->second.numOfCols << " " << (tbIter->second).tablewithautoincr + << endl; + } + + // delete col[9]; + lk.lock(); + fSchemaCache.insert(schema); + lk.unlock(); } #if 0 @@ -5645,617 +5603,599 @@ ostream& operator<<(ostream& os, const CalpontSystemCatalog::TableName& rhs) #endif const string CalpontSystemCatalog::TableName::toString() const { - string str = schema + "." + table; - return str; + string str = schema + "." + table; + return str; } ostream& operator<<(ostream& os, const CalpontSystemCatalog::TableAliasName& rhs) { - os << rhs.schema << '.' << rhs.table << "(" << rhs.alias << "/" << rhs.view - << ") engineType=" << (rhs.fisColumnStore ? "ColumnStore" : "ForeignEngine"); - return os; + os << rhs.schema << '.' << rhs.table << "(" << rhs.alias << "/" << rhs.view + << ") engineType=" << (rhs.fisColumnStore ? "ColumnStore" : "ForeignEngine"); + return os; } ostream& operator<<(ostream& os, const CalpontSystemCatalog::TableColName& rhs) { - os << rhs.toString(); - return os; + os << rhs.toString(); + return os; } void CalpontSystemCatalog::flushCache() { - boost::mutex::scoped_lock lk1(fOIDmapLock); - fOIDmap.clear(); - buildSysOIDmap(); - lk1.unlock(); + boost::mutex::scoped_lock lk1(fOIDmapLock); + fOIDmap.clear(); + buildSysOIDmap(); + lk1.unlock(); - boost::mutex::scoped_lock lk2(fColinfomapLock); - fColinfomap.clear(); - buildSysColinfomap(); - lk2.unlock(); + boost::mutex::scoped_lock lk2(fColinfomapLock); + fColinfomap.clear(); + buildSysColinfomap(); + lk2.unlock(); - boost::mutex::scoped_lock lk3(fTableInfoMapLock); - fTableInfoMap.clear(); - fTablemap.clear(); - fTableRIDmap.clear(); - buildSysTablemap(); - lk3.unlock(); + boost::mutex::scoped_lock lk3(fTableInfoMapLock); + fTableInfoMap.clear(); + fTablemap.clear(); + fTableRIDmap.clear(); + buildSysTablemap(); + lk3.unlock(); - boost::recursive_mutex::scoped_lock lk4(fDctTokenMapLock); - fDctTokenMap.clear(); - buildSysDctmap(); - lk4.unlock(); + boost::recursive_mutex::scoped_lock lk4(fDctTokenMapLock); + fDctTokenMap.clear(); + buildSysDctmap(); + lk4.unlock(); - fSyscatSCN = fSessionManager->sysCatVerID().currentScn; - //cout << "Cache flushed and current sysCatVerID is " << newScn << endl; + fSyscatSCN = fSessionManager->sysCatVerID().currentScn; + // cout << "Cache flushed and current sysCatVerID is " << newScn << endl; } void CalpontSystemCatalog::updateColinfoCache(CalpontSystemCatalog::OIDNextvalMap& oidNextvalMap) { - boost::mutex::scoped_lock lk(fColinfomapLock); - CalpontSystemCatalog::OIDNextvalMap::const_iterator iter = oidNextvalMap.begin(); - OID oid = 0; - long long nextVal = 0; - - while (iter != oidNextvalMap.end()) - { - oid = (*iter).first; - nextVal = (*iter).second; - fColinfomap[oid].nextvalue = nextVal; - iter++; - } + boost::mutex::scoped_lock lk(fColinfomapLock); + CalpontSystemCatalog::OIDNextvalMap::const_iterator iter = oidNextvalMap.begin(); + OID oid = 0; + long long nextVal = 0; + while (iter != oidNextvalMap.end()) + { + oid = (*iter).first; + nextVal = (*iter).second; + fColinfomap[oid].nextvalue = nextVal; + iter++; + } } void CalpontSystemCatalog::buildSysColinfomap() { - ColType aCol; - // aCol.defaultValue = ""; - aCol.scale = 0; - aCol.precision = 10; - aCol.compressionType = 0; + ColType aCol; + // aCol.defaultValue = ""; + aCol.scale = 0; + aCol.precision = 10; + aCol.compressionType = 0; - ResourceManager* rm = ResourceManager::instance(); + ResourceManager* rm = ResourceManager::instance(); - if ( rm->useHdfs() ) - aCol.compressionType = 2; + if (rm->useHdfs()) + aCol.compressionType = 2; - DictOID notDict; + DictOID notDict; - // @bug 4433 - Increase object width from 64 to 128 for schema names, table names, and column names. - aCol.colWidth = 129; // @bug 4433 - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSTABLE_TABLENAME; - aCol.ddn.listOID = LISTOID_SYSTABLE_TABLENAME; - aCol.ddn.treeOID = TREEOID_SYSTABLE_TABLENAME; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition = 0; - aCol.columnOID = OID_SYSTABLE_TABLENAME; - fColinfomap[aCol.columnOID] = aCol; + // @bug 4433 - Increase object width from 64 to 128 for schema names, table names, and column names. + aCol.colWidth = 129; // @bug 4433 + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSTABLE_TABLENAME; + aCol.ddn.listOID = LISTOID_SYSTABLE_TABLENAME; + aCol.ddn.treeOID = TREEOID_SYSTABLE_TABLENAME; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition = 0; + aCol.columnOID = OID_SYSTABLE_TABLENAME; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 129; // @bug 4433 - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSTABLE_SCHEMA; - aCol.ddn.listOID = LISTOID_SYSTABLE_SCHEMA; - aCol.ddn.treeOID = TREEOID_SYSTABLE_SCHEMA; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_SCHEMA; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 129; // @bug 4433 + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSTABLE_SCHEMA; + aCol.ddn.listOID = LISTOID_SYSTABLE_SCHEMA; + aCol.ddn.treeOID = TREEOID_SYSTABLE_SCHEMA; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_SCHEMA; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_OBJECTID; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_OBJECTID; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = DATE; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_CREATEDATE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = DATE; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_CREATEDATE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = DATE; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_LASTUPDATE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = DATE; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_LASTUPDATE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_INIT; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_INIT; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_NEXT; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_NEXT; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_NUMOFROWS; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_NUMOFROWS; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_AVGROWLEN; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_AVGROWLEN; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_NUMOFBLOCKS; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_NUMOFBLOCKS; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSTABLE_AUTOINCREMENT; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSTABLE_AUTOINCREMENT; + fColinfomap[aCol.columnOID] = aCol; - fTablemap[make_table(CALPONT_SCHEMA, SYSCOLUMN_TABLE )] = SYSCOLUMN_BASE; + fTablemap[make_table(CALPONT_SCHEMA, SYSCOLUMN_TABLE)] = SYSCOLUMN_BASE; - aCol.colWidth = 129; // @bug 4433 - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSCOLUMN_SCHEMA; - aCol.ddn.listOID = LISTOID_SYSCOLUMN_SCHEMA; - aCol.ddn.treeOID = TREEOID_SYSCOLUMN_SCHEMA; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition = 0; - aCol.columnOID = OID_SYSCOLUMN_SCHEMA; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 129; // @bug 4433 + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSCOLUMN_SCHEMA; + aCol.ddn.listOID = LISTOID_SYSCOLUMN_SCHEMA; + aCol.ddn.treeOID = TREEOID_SYSCOLUMN_SCHEMA; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition = 0; + aCol.columnOID = OID_SYSCOLUMN_SCHEMA; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 129; // @bug 4433 - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSCOLUMN_TABLENAME; - aCol.ddn.listOID = LISTOID_SYSCOLUMN_TABLENAME; - aCol.ddn.treeOID = TREEOID_SYSCOLUMN_TABLENAME; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_TABLENAME; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 129; // @bug 4433 + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSCOLUMN_TABLENAME; + aCol.ddn.listOID = LISTOID_SYSCOLUMN_TABLENAME; + aCol.ddn.treeOID = TREEOID_SYSCOLUMN_TABLENAME; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_TABLENAME; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 129; // @bug 4433 - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSCOLUMN_COLNAME; - aCol.ddn.listOID = LISTOID_SYSCOLUMN_COLNAME; - aCol.ddn.treeOID = TREEOID_SYSCOLUMN_COLNAME; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_COLNAME; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 129; // @bug 4433 + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSCOLUMN_COLNAME; + aCol.ddn.listOID = LISTOID_SYSCOLUMN_COLNAME; + aCol.ddn.treeOID = TREEOID_SYSCOLUMN_COLNAME; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_COLNAME; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_OBJECTID; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_OBJECTID; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_DICTOID; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_DICTOID; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_LISTOBJID; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_LISTOBJID; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_TREEOBJID; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_TREEOBJID; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_DATATYPE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_DATATYPE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_COLUMNLEN; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_COLUMNLEN; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_COLUMNPOS; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_COLUMNPOS; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = DATE; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_LASTUPDATE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = DATE; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_LASTUPDATE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 64; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSCOLUMN_DEFAULTVAL; - aCol.ddn.listOID = LISTOID_SYSCOLUMN_DEFAULTVAL; - aCol.ddn.treeOID = TREEOID_SYSCOLUMN_DEFAULTVAL; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_DEFAULTVAL; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 64; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSCOLUMN_DEFAULTVAL; + aCol.ddn.listOID = LISTOID_SYSCOLUMN_DEFAULTVAL; + aCol.ddn.treeOID = TREEOID_SYSCOLUMN_DEFAULTVAL; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_DEFAULTVAL; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_NULLABLE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_NULLABLE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_SCALE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_SCALE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_PRECISION; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_PRECISION; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 1; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = CHAR; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_AUTOINC; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 1; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = CHAR; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_AUTOINC; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_DISTCOUNT; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_DISTCOUNT; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_NULLCOUNT; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_NULLCOUNT; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 65; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSCOLUMN_MINVALUE; - aCol.ddn.listOID = LISTOID_SYSCOLUMN_MINVALUE; - aCol.ddn.treeOID = TREEOID_SYSCOLUMN_MINVALUE; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_MINVALUE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 65; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSCOLUMN_MINVALUE; + aCol.ddn.listOID = LISTOID_SYSCOLUMN_MINVALUE; + aCol.ddn.treeOID = TREEOID_SYSCOLUMN_MINVALUE; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_MINVALUE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 65; - aCol.constraintType = NO_CONSTRAINT; - aCol.colDataType = VARCHAR; - aCol.ddn.dictOID = DICTOID_SYSCOLUMN_MAXVALUE; - aCol.ddn.listOID = LISTOID_SYSCOLUMN_MAXVALUE; - aCol.ddn.treeOID = TREEOID_SYSCOLUMN_MAXVALUE; - aCol.ddn.compressionType = aCol.compressionType; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_MAXVALUE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 65; + aCol.constraintType = NO_CONSTRAINT; + aCol.colDataType = VARCHAR; + aCol.ddn.dictOID = DICTOID_SYSCOLUMN_MAXVALUE; + aCol.ddn.listOID = LISTOID_SYSCOLUMN_MAXVALUE; + aCol.ddn.treeOID = TREEOID_SYSCOLUMN_MAXVALUE; + aCol.ddn.compressionType = aCol.compressionType; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_MAXVALUE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 4; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = INT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_COMPRESSIONTYPE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 4; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = INT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_COMPRESSIONTYPE; + fColinfomap[aCol.columnOID] = aCol; - aCol.colWidth = 8; - aCol.constraintType = NOTNULL_CONSTRAINT; - aCol.colDataType = UBIGINT; - aCol.ddn = notDict; - aCol.colPosition++; - aCol.columnOID = OID_SYSCOLUMN_NEXTVALUE; - fColinfomap[aCol.columnOID] = aCol; + aCol.colWidth = 8; + aCol.constraintType = NOTNULL_CONSTRAINT; + aCol.colDataType = UBIGINT; + aCol.ddn = notDict; + aCol.colPosition++; + aCol.columnOID = OID_SYSCOLUMN_NEXTVALUE; + fColinfomap[aCol.columnOID] = aCol; } void CalpontSystemCatalog::buildSysOIDmap() { - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, TABLENAME_COL)] = OID_SYSTABLE_TABLENAME; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, SCHEMA_COL)] = OID_SYSTABLE_SCHEMA; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, OBJECTID_COL)] = OID_SYSTABLE_OBJECTID; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, CREATEDATE_COL)] = OID_SYSTABLE_CREATEDATE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, LASTUPDATE_COL)] = OID_SYSTABLE_LASTUPDATE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, INIT_COL)] = OID_SYSTABLE_INIT; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, NEXT_COL)] = OID_SYSTABLE_NEXT; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, NUMOFROWS_COL)] = OID_SYSTABLE_NUMOFROWS; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, AVGROWLEN_COL)] = OID_SYSTABLE_AVGROWLEN; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, NUMOFBLOCKS_COL)] = OID_SYSTABLE_NUMOFBLOCKS; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, AUTOINC_COL)] = OID_SYSTABLE_AUTOINCREMENT; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, SCHEMA_COL)] = OID_SYSCOLUMN_SCHEMA; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, TABLENAME_COL)] = OID_SYSCOLUMN_TABLENAME; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COLNAME_COL)] = OID_SYSCOLUMN_COLNAME; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, OBJECTID_COL)] = OID_SYSCOLUMN_OBJECTID; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DICTOID_COL)] = OID_SYSCOLUMN_DICTOID; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, LISTOBJID_COL)] = OID_SYSCOLUMN_LISTOBJID; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, TREEOBJID_COL)] = OID_SYSCOLUMN_TREEOBJID; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DATATYPE_COL)] = OID_SYSCOLUMN_DATATYPE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COLUMNLEN_COL)] = OID_SYSCOLUMN_COLUMNLEN; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COLUMNPOS_COL )] = OID_SYSCOLUMN_COLUMNPOS; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, LASTUPDATE_COL)] = OID_SYSCOLUMN_LASTUPDATE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DEFAULTVAL_COL)] = OID_SYSCOLUMN_DEFAULTVAL; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, NULLABLE_COL)] = OID_SYSCOLUMN_NULLABLE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, SCALE_COL)] = OID_SYSCOLUMN_SCALE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, PRECISION_COL)] = OID_SYSCOLUMN_PRECISION; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, AUTOINC_COL)] = OID_SYSCOLUMN_AUTOINC; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DISTCOUNT_COL)] = OID_SYSCOLUMN_DISTCOUNT; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, NULLCOUNT_COL)] = OID_SYSCOLUMN_NULLCOUNT; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, MINVALUE_COL)] = OID_SYSCOLUMN_MINVALUE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, MAXVALUE_COL)] = OID_SYSCOLUMN_MAXVALUE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COMPRESSIONTYPE_COL)] = OID_SYSCOLUMN_COMPRESSIONTYPE; - fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, NEXTVALUE_COL)] = OID_SYSCOLUMN_NEXTVALUE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, TABLENAME_COL)] = OID_SYSTABLE_TABLENAME; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, SCHEMA_COL)] = OID_SYSTABLE_SCHEMA; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, OBJECTID_COL)] = OID_SYSTABLE_OBJECTID; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, CREATEDATE_COL)] = OID_SYSTABLE_CREATEDATE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, LASTUPDATE_COL)] = OID_SYSTABLE_LASTUPDATE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, INIT_COL)] = OID_SYSTABLE_INIT; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, NEXT_COL)] = OID_SYSTABLE_NEXT; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, NUMOFROWS_COL)] = OID_SYSTABLE_NUMOFROWS; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, AVGROWLEN_COL)] = OID_SYSTABLE_AVGROWLEN; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, NUMOFBLOCKS_COL)] = OID_SYSTABLE_NUMOFBLOCKS; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSTABLE_TABLE, AUTOINC_COL)] = OID_SYSTABLE_AUTOINCREMENT; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, SCHEMA_COL)] = OID_SYSCOLUMN_SCHEMA; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, TABLENAME_COL)] = OID_SYSCOLUMN_TABLENAME; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COLNAME_COL)] = OID_SYSCOLUMN_COLNAME; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, OBJECTID_COL)] = OID_SYSCOLUMN_OBJECTID; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DICTOID_COL)] = OID_SYSCOLUMN_DICTOID; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, LISTOBJID_COL)] = OID_SYSCOLUMN_LISTOBJID; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, TREEOBJID_COL)] = OID_SYSCOLUMN_TREEOBJID; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DATATYPE_COL)] = OID_SYSCOLUMN_DATATYPE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COLUMNLEN_COL)] = OID_SYSCOLUMN_COLUMNLEN; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COLUMNPOS_COL)] = OID_SYSCOLUMN_COLUMNPOS; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, LASTUPDATE_COL)] = OID_SYSCOLUMN_LASTUPDATE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DEFAULTVAL_COL)] = OID_SYSCOLUMN_DEFAULTVAL; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, NULLABLE_COL)] = OID_SYSCOLUMN_NULLABLE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, SCALE_COL)] = OID_SYSCOLUMN_SCALE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, PRECISION_COL)] = OID_SYSCOLUMN_PRECISION; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, AUTOINC_COL)] = OID_SYSCOLUMN_AUTOINC; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, DISTCOUNT_COL)] = OID_SYSCOLUMN_DISTCOUNT; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, NULLCOUNT_COL)] = OID_SYSCOLUMN_NULLCOUNT; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, MINVALUE_COL)] = OID_SYSCOLUMN_MINVALUE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, MAXVALUE_COL)] = OID_SYSCOLUMN_MAXVALUE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, COMPRESSIONTYPE_COL)] = OID_SYSCOLUMN_COMPRESSIONTYPE; + fOIDmap[make_tcn(CALPONT_SCHEMA, SYSCOLUMN_TABLE, NEXTVALUE_COL)] = OID_SYSCOLUMN_NEXTVALUE; } void CalpontSystemCatalog::buildSysTablemap() { - fTablemap[make_table(CALPONT_SCHEMA, SYSTABLE_TABLE)] = SYSTABLE_BASE; - fTablemap[make_table(CALPONT_SCHEMA, SYSCOLUMN_TABLE )] = SYSCOLUMN_BASE; + fTablemap[make_table(CALPONT_SCHEMA, SYSTABLE_TABLE)] = SYSTABLE_BASE; + fTablemap[make_table(CALPONT_SCHEMA, SYSCOLUMN_TABLE)] = SYSCOLUMN_BASE; } void CalpontSystemCatalog::buildSysDctmap() { - fDctTokenMap[DICTOID_SYSTABLE_TABLENAME] = OID_SYSTABLE_TABLENAME; - fDctTokenMap[DICTOID_SYSTABLE_SCHEMA] = OID_SYSTABLE_SCHEMA; + fDctTokenMap[DICTOID_SYSTABLE_TABLENAME] = OID_SYSTABLE_TABLENAME; + fDctTokenMap[DICTOID_SYSTABLE_SCHEMA] = OID_SYSTABLE_SCHEMA; - fDctTokenMap[DICTOID_SYSCOLUMN_SCHEMA] = OID_SYSCOLUMN_SCHEMA; - fDctTokenMap[DICTOID_SYSCOLUMN_TABLENAME] = OID_SYSCOLUMN_TABLENAME; - fDctTokenMap[DICTOID_SYSCOLUMN_COLNAME] = OID_SYSCOLUMN_COLNAME; - fDctTokenMap[DICTOID_SYSCOLUMN_DEFAULTVAL] = OID_SYSCOLUMN_DEFAULTVAL; - fDctTokenMap[DICTOID_SYSCOLUMN_MINVALUE] = OID_SYSCOLUMN_MINVALUE; - fDctTokenMap[DICTOID_SYSCOLUMN_MAXVALUE] = OID_SYSCOLUMN_MAXVALUE; + fDctTokenMap[DICTOID_SYSCOLUMN_SCHEMA] = OID_SYSCOLUMN_SCHEMA; + fDctTokenMap[DICTOID_SYSCOLUMN_TABLENAME] = OID_SYSCOLUMN_TABLENAME; + fDctTokenMap[DICTOID_SYSCOLUMN_COLNAME] = OID_SYSCOLUMN_COLNAME; + fDctTokenMap[DICTOID_SYSCOLUMN_DEFAULTVAL] = OID_SYSCOLUMN_DEFAULTVAL; + fDctTokenMap[DICTOID_SYSCOLUMN_MINVALUE] = OID_SYSCOLUMN_MINVALUE; + fDctTokenMap[DICTOID_SYSCOLUMN_MAXVALUE] = OID_SYSCOLUMN_MAXVALUE; } void CalpontSystemCatalog::checkSysCatVer() { - SCN newScn = fSessionManager->sysCatVerID().currentScn; + SCN newScn = fSessionManager->sysCatVerID().currentScn; - if (newScn < 0) - { - fSessionManager.reset(new SessionManager()); - newScn = fSessionManager->sysCatVerID().currentScn; - } + if (newScn < 0) + { + fSessionManager.reset(new SessionManager()); + newScn = fSessionManager->sysCatVerID().currentScn; + } - boost::mutex::scoped_lock sysCatLk(fSyscatSCNLock); + boost::mutex::scoped_lock sysCatLk(fSyscatSCNLock); - if ( fSyscatSCN != newScn ) - { - flushCache(); - } + if (fSyscatSCN != newScn) + { + flushCache(); + } } -CalpontSystemCatalog::ColType::ColType() : - constraintType(NO_CONSTRAINT), - defaultValue(""), - colPosition(-1), - compressionType(NO_COMPRESSION), - columnOID(0), - autoincrement(0), - nextvalue(0), - cs(NULL) +CalpontSystemCatalog::ColType::ColType() + : constraintType(NO_CONSTRAINT) + , defaultValue("") + , colPosition(-1) + , compressionType(NO_COMPRESSION) + , columnOID(0) + , autoincrement(0) + , nextvalue(0) + , cs(NULL) { - charsetNumber = default_charset_info->number; + charsetNumber = default_charset_info->number; } -CalpontSystemCatalog::ColType::ColType(const ColType& rhs) - :TypeHolderStd(rhs) +CalpontSystemCatalog::ColType::ColType(const ColType& rhs) : TypeHolderStd(rhs) { - constraintType = rhs.constraintType; - ddn = rhs.ddn; - defaultValue = rhs.defaultValue; - colPosition = rhs.colPosition; - compressionType = rhs.compressionType; - columnOID = rhs.columnOID; - autoincrement = rhs.autoincrement; - nextvalue = rhs.nextvalue; - charsetNumber = rhs.charsetNumber; - cs = rhs.cs; + constraintType = rhs.constraintType; + ddn = rhs.ddn; + defaultValue = rhs.defaultValue; + colPosition = rhs.colPosition; + compressionType = rhs.compressionType; + columnOID = rhs.columnOID; + autoincrement = rhs.autoincrement; + nextvalue = rhs.nextvalue; + charsetNumber = rhs.charsetNumber; + cs = rhs.cs; } CalpontSystemCatalog::ColType& CalpontSystemCatalog::ColType::operator=(const ColType& rhs) { - TypeHolderStd::operator=(rhs); - constraintType = rhs.constraintType; - ddn = rhs.ddn; - defaultValue = rhs.defaultValue; - colPosition = rhs.colPosition; - compressionType = rhs.compressionType; - columnOID = rhs.columnOID; - autoincrement = rhs.autoincrement; - nextvalue = rhs.nextvalue; - charsetNumber = rhs.charsetNumber; - cs = rhs.cs; + TypeHolderStd::operator=(rhs); + constraintType = rhs.constraintType; + ddn = rhs.ddn; + defaultValue = rhs.defaultValue; + colPosition = rhs.colPosition; + compressionType = rhs.compressionType; + columnOID = rhs.columnOID; + autoincrement = rhs.autoincrement; + nextvalue = rhs.nextvalue; + charsetNumber = rhs.charsetNumber; + cs = rhs.cs; - return *this; + return *this; } - - - CHARSET_INFO* CalpontSystemCatalog::ColType::getCharset() { - if (!cs) - cs= & datatypes::Charset(charsetNumber).getCharset(); - return cs; + if (!cs) + cs = &datatypes::Charset(charsetNumber).getCharset(); + return cs; } const string CalpontSystemCatalog::ColType::toString() const { - ostringstream output; - output << "cw: " << colWidth - << " dt: " << colDataTypeToString(colDataType) - << " do: " << ddn.dictOID - << " lo: " << ddn.listOID - << " to: " << ddn.treeOID - << " cp: " << colPosition - << " sc: " << scale - << " pr: " << precision - << " od: " << columnOID - << " ct: " << compressionType - << " ai: " << autoincrement - << " nv: " << nextvalue; - return output.str(); + ostringstream output; + output << "cw: " << colWidth << " dt: " << colDataTypeToString(colDataType) << " do: " << ddn.dictOID + << " lo: " << ddn.listOID << " to: " << ddn.treeOID << " cp: " << colPosition << " sc: " << scale + << " pr: " << precision << " od: " << columnOID << " ct: " << compressionType + << " ai: " << autoincrement << " nv: " << nextvalue; + return output.str(); } - -boost::any -CalpontSystemCatalog::ColType::convertColumnData(const std::string& data, - bool& pushWarning, - const std::string& timeZone, - bool nulFlag, - bool noRoundup, - bool isUpdate) const +boost::any CalpontSystemCatalog::ColType::convertColumnData(const std::string& data, bool& pushWarning, + const std::string& timeZone, bool nulFlag, + bool noRoundup, bool isUpdate) const { - pushWarning = false; - const datatypes::TypeHandler *h= typeHandler(); - if (!h) - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); + pushWarning = false; + const datatypes::TypeHandler* h = typeHandler(); + if (!h) + throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - if (nulFlag) - return h->getNullValueForType(*this); + if (nulFlag) + return h->getNullValueForType(*this); - const datatypes::ConvertFromStringParam prm(timeZone, noRoundup, isUpdate); - return h->convertFromString(*this, prm, data, pushWarning); + const datatypes::ConvertFromStringParam prm(timeZone, noRoundup, isUpdate); + return h->convertFromString(*this, prm, data, pushWarning); } - -CalpontSystemCatalog::ColType CalpontSystemCatalog::ColType::convertUnionColType(vector& types) +CalpontSystemCatalog::ColType CalpontSystemCatalog::ColType::convertUnionColType( + vector& types) { - idbassert(types.size()); - CalpontSystemCatalog::ColType unionedType = types[0]; - for (uint64_t i = 1; i < types.size(); i++) - dataconvert::DataConvert::joinColTypeForUnion(unionedType, types[i]); - return unionedType; + idbassert(types.size()); + CalpontSystemCatalog::ColType unionedType = types[0]; + for (uint64_t i = 1; i < types.size(); i++) + dataconvert::DataConvert::joinColTypeForUnion(unionedType, types[i]); + return unionedType; } - -//format a session id that includes the module id -//we want the top bit clear to use as a syscat flag, then we want 7 bits of module id, then 24 bits of thread id +// format a session id that includes the module id +// we want the top bit clear to use as a syscat flag, then we want 7 bits of module id, then 24 bits of thread +// id /*static*/ uint32_t CalpontSystemCatalog::idb_tid2sid(const uint32_t tid) { - //don't care about locking here... - if (fModuleID == numeric_limits::max()) + // don't care about locking here... + if (fModuleID == numeric_limits::max()) + { + uint32_t tmid = 1; + oam::Oam oam; + oam::oamModuleInfo_t minfo; + + try { - uint32_t tmid = 1; - oam::Oam oam; - oam::oamModuleInfo_t minfo; + minfo = oam.getModuleInfo(); + tmid = static_cast(boost::get<2>(minfo)); - try - { - minfo = oam.getModuleInfo(); - tmid = static_cast(boost::get<2>(minfo)); - - if (tmid == 0) - tmid = 1; - } - catch (...) - { - tmid = 1; - } - - fModuleID = tmid; + if (tmid == 0) + tmid = 1; + } + catch (...) + { + tmid = 1; } - uint32_t mid = fModuleID; - mid--; //make module id zero-based - mid &= 0x0000007f; - uint32_t sid = (mid << 24) | (tid & 0x00ffffff); - return sid; + fModuleID = tmid; + } + + uint32_t mid = fModuleID; + mid--; // make module id zero-based + mid &= 0x0000007f; + uint32_t sid = (mid << 24) | (tid & 0x00ffffff); + return sid; } ostream& operator<<(ostream& output, const CalpontSystemCatalog::ColType& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } vector getAllSysCatOIDs() { - vector ret; - CalpontSystemCatalog::OID oid; + vector ret; + CalpontSystemCatalog::OID oid; - for (oid = SYSTABLE_BASE + 1; oid < SYSTABLE_MAX; oid++) - ret.push_back(oid); + for (oid = SYSTABLE_BASE + 1; oid < SYSTABLE_MAX; oid++) + ret.push_back(oid); - for (oid = SYSCOLUMN_BASE + 1; oid < SYSCOLUMN_MAX; oid++) - ret.push_back(oid); + for (oid = SYSCOLUMN_BASE + 1; oid < SYSCOLUMN_MAX; oid++) + ret.push_back(oid); - for (oid = SYSTABLE_DICT_BASE + 1; oid < SYSTABLE_DICT_MAX; oid++) - ret.push_back(oid); + for (oid = SYSTABLE_DICT_BASE + 1; oid < SYSTABLE_DICT_MAX; oid++) + ret.push_back(oid); - for (oid = SYSCOLUMN_DICT_BASE + 1; oid < SYSCOLUMN_DICT_MAX; oid++) - ret.push_back(oid); + for (oid = SYSCOLUMN_DICT_BASE + 1; oid < SYSCOLUMN_DICT_MAX; oid++) + ret.push_back(oid); - return ret; + return ret; } bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatalog::ColType& b) { - return a.colPosition < b.colPosition; + return a.colPosition < b.colPosition; } -} // namespace execplan +} // namespace execplan // vim:sw=4 ts=4: diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 8f64b90cb..b102a666d 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -50,8 +50,7 @@ #undef max #include "mcs_datatype.h" -#include "collation.h" // CHARSET_INFO, class Charset - +#include "collation.h" // CHARSET_INFO, class Charset class ExecPlanTest; namespace messageqcpp @@ -59,10 +58,9 @@ namespace messageqcpp class MessageQueueClient; } - // This is now set in the Columnstore.xml file // Use, e.g., 0x500 for uid 500 so it is easy to spot in ipcs list -//const int32_t BRM_UID = 0x0; +// const int32_t BRM_UID = 0x0; namespace execplan { @@ -78,1009 +76,1051 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID; * * This object encapsulates the system catalog stored in the engine */ -class CalpontSystemCatalog: public datatypes::SystemCatalog +class CalpontSystemCatalog : public datatypes::SystemCatalog { -public: + public: + /** static calpontsystemcatalog instance map. one instance per session + * TODO: should be one per transaction + */ + typedef std::map > CatalogMap; - /** static calpontsystemcatalog instance map. one instance per session - * TODO: should be one per transaction - */ - typedef std::map > CatalogMap; + /** Server Identity + * + */ + enum Identity + { + EC = 0, + FE + }; - /** Server Identity - * - */ - enum Identity { EC = 0, FE }; + /** the set of column constraint types + * + */ + enum ConstraintType + { + NO_CONSTRAINT, + UNIQUE_CONSTRAINT, + CHECK_CONSTRAINT, + NOTNULL_CONSTRAINT, + PRIMARYKEY_CONSTRAINT, + REFERENCE_CONSTRAINT, + DEFAULT_CONSTRAINT + }; - /** the set of column constraint types - * - */ - enum ConstraintType + enum CompressionType + { + NO_COMPRESSION, + COMPRESSION1, + COMPRESSION2 + }; + + enum AutoincrColumn + { + NO_AUTOINCRCOL, + AUTOINCRCOL + }; + + /** the type of an object number + * + * @todo TODO: OIDs aren't really signed. This should be fixed. + */ + typedef int32_t OID; + + /** @brief a structure to hold a dictionary's + * object ids + */ + struct DictOID + { + DictOID() : dictOID(0), listOID(0), treeOID(0), compressionType(0) { - NO_CONSTRAINT, - UNIQUE_CONSTRAINT, - CHECK_CONSTRAINT, - NOTNULL_CONSTRAINT, - PRIMARYKEY_CONSTRAINT, - REFERENCE_CONSTRAINT, - DEFAULT_CONSTRAINT - }; - - enum CompressionType + } + OID dictOID; + OID listOID; + OID treeOID; + int compressionType; + bool operator==(const DictOID& t) const { - NO_COMPRESSION, - COMPRESSION1, - COMPRESSION2 - }; + if (dictOID != t.dictOID) + return false; - enum AutoincrColumn + if (listOID != t.listOID) + return false; + + if (treeOID != t.treeOID) + return false; + + if (compressionType != t.compressionType) + return false; + + return true; + } + bool operator!=(const DictOID& t) const { - NO_AUTOINCRCOL, - AUTOINCRCOL - }; + return !(*this == t); + } + }; - /** the type of an object number - * - * @todo TODO: OIDs aren't really signed. This should be fixed. - */ - typedef int32_t OID; - - /** @brief a structure to hold a dictionary's - * object ids - */ - struct DictOID + /** the type of a list of ColumnResult as returned from getSysData + */ + typedef std::vector NJLSysDataVector; + struct NJLSysDataList + { + // If we used an unorderedmap, we might improve performance. + // Maybe. + NJLSysDataVector sysDataVec; + NJLSysDataList(){}; + ~NJLSysDataList(); + NJLSysDataVector::const_iterator begin() const { - DictOID() : dictOID(0), listOID(0), treeOID(0), compressionType(0) { } - OID dictOID; - OID listOID; - OID treeOID; - int compressionType; - bool operator==(const DictOID& t) const - { - if (dictOID != t.dictOID) - return false; - - if (listOID != t.listOID) - return false; - - if (treeOID != t.treeOID) - return false; - - if (compressionType != t.compressionType) - return false; - - return true; - } - bool operator!=(const DictOID& t) const - { - return !(*this == t); - } - }; - - /** the type of a list of ColumnResult as returned from getSysData - */ - typedef std::vector NJLSysDataVector; - struct NJLSysDataList + return sysDataVec.begin(); + } + NJLSysDataVector::const_iterator end() const { - // If we used an unorderedmap, we might improve performance. - // Maybe. - NJLSysDataVector sysDataVec; - NJLSysDataList() {}; - ~NJLSysDataList(); - NJLSysDataVector::const_iterator begin() const - { - return sysDataVec.begin(); - } - NJLSysDataVector::const_iterator end() const - { - return sysDataVec.end(); - } - void push_back(ColumnResult* cr) - { - sysDataVec.push_back(cr); - } - NJLSysDataVector::size_type size() const - { - return sysDataVec.size(); - } - NJLSysDataVector::size_type findColumn(const OID& columnOID) const - { - for (NJLSysDataVector::size_type i = 0; i < sysDataVec.size(); i++) - if (sysDataVec[i]->ColumnOID() == columnOID) - return i; - - return -1; - } - }; - - /** the type of a list of dictionary OIDs - */ - typedef std::vector DictOIDList; - - - /** the structure returned by colType - * - * defaultValue is only meaningful when constraintType == DEFAULT_CONSTRAINT - */ - struct ColType: public datatypes::SystemCatalog::TypeHolderStd + return sysDataVec.end(); + } + void push_back(ColumnResult* cr) { - ColType(); - ConstraintType constraintType; - DictOID ddn; - std::string defaultValue; - int32_t colPosition; // temporally put here. may need to have ColInfo struct later - int32_t compressionType; - OID columnOID; - bool autoincrement; //set to true if SYSCOLUMN autoincrement is �y� - uint64_t nextvalue; //next autoincrement value - uint32_t charsetNumber; - const CHARSET_INFO* cs; - - ColType(const ColType& rhs); - ColType& operator=(const ColType& rhs); - - CHARSET_INFO* getCharset(); - // for F&E use. only serialize necessary info for now - void serialize (messageqcpp::ByteStream& b) const - { - b << (uint32_t)colDataType; - b << (uint32_t)colWidth; - b << (uint32_t)scale; - b << (uint32_t)precision; - b << (uint32_t)compressionType; - b << charsetNumber; - } - - void unserialize (messageqcpp::ByteStream& b) - { - uint32_t val; - b >> (uint32_t&)val; - colDataType = (ColDataType)val; - b >> (uint32_t&)colWidth; - b >> (uint32_t&)scale; - b >> (uint32_t&)precision; - b >> (uint32_t&)compressionType; - b >> charsetNumber; - } - - /** - * @brief convert a columns data, represnted as a string, - * to its native format - * @param data - the string representation - * @param [OUT] bSaturate - the value was truncated/adjusted - * @param timeZone - the time zone name, for TIMESTAMP conversion - * @param nullFlag - SQL NULL flag - * @param nRoundtrip - * @param isUpdate - */ - boost::any convertColumnData(const std::string& data, - bool& bSaturate, - const std::string& timeZone, - bool nulFlag = false, - bool noRoundup = false, - bool isUpdate = false) const; - - const std::string toString() const; - - //Put these here so udf doesn't need to link libexecplan - bool operator==(const ColType& t) const - { - //order these with the most likely first - if (columnOID != t.columnOID) - return false; - - if (colPosition != t.colPosition) - return false; - - if (ddn != t.ddn) - return false; - - if (colWidth != t.colWidth) - return false; - - if (scale != t.scale) - return false; - - if (precision != t.precision) - return false; - - if (constraintType != t.constraintType) - return false; - - return true; - } - - bool operator!=(const ColType& t) const - { - return !(*this == t); - } - - static ColType convertUnionColType(std::vector&); - - }; - - /** the structure of a table infomation - * - * this structure holds table related info. More in the future - */ - struct TableInfo + sysDataVec.push_back(cr); + } + NJLSysDataVector::size_type size() const { - TableInfo() : numOfCols(0), tablewithautoincr(NO_AUTOINCRCOL) { } - int numOfCols; - int tablewithautoincr; - }; + return sysDataVec.size(); + } + NJLSysDataVector::size_type findColumn(const OID& columnOID) const + { + for (NJLSysDataVector::size_type i = 0; i < sysDataVec.size(); i++) + if (sysDataVec[i]->ColumnOID() == columnOID) + return i; - /** the type of a Row ID - * - * @todo TODO: RIDs aren't really signed. This should be fixed. - */ - //@bug 1866 match RID to writeengine, uint64_t - typedef WriteEngine::RID RID; + return -1; + } + }; + + /** the type of a list of dictionary OIDs + */ + typedef std::vector DictOIDList; + + /** the structure returned by colType + * + * defaultValue is only meaningful when constraintType == DEFAULT_CONSTRAINT + */ + struct ColType : public datatypes::SystemCatalog::TypeHolderStd + { + ColType(); + ConstraintType constraintType; + DictOID ddn; + std::string defaultValue; + int32_t colPosition; // temporally put here. may need to have ColInfo struct later + int32_t compressionType; + OID columnOID; + bool autoincrement; // set to true if SYSCOLUMN autoincrement is �y� + uint64_t nextvalue; // next autoincrement value + uint32_t charsetNumber; + const CHARSET_INFO* cs; + + ColType(const ColType& rhs); + ColType& operator=(const ColType& rhs); + + CHARSET_INFO* getCharset(); + // for F&E use. only serialize necessary info for now + void serialize(messageqcpp::ByteStream& b) const + { + b << (uint32_t)colDataType; + b << (uint32_t)colWidth; + b << (uint32_t)scale; + b << (uint32_t)precision; + b << (uint32_t)compressionType; + b << charsetNumber; + } + + void unserialize(messageqcpp::ByteStream& b) + { + uint32_t val; + b >> (uint32_t&)val; + colDataType = (ColDataType)val; + b >> (uint32_t&)colWidth; + b >> (uint32_t&)scale; + b >> (uint32_t&)precision; + b >> (uint32_t&)compressionType; + b >> charsetNumber; + } /** - * + * @brief convert a columns data, represnted as a string, + * to its native format + * @param data - the string representation + * @param [OUT] bSaturate - the value was truncated/adjusted + * @param timeZone - the time zone name, for TIMESTAMP conversion + * @param nullFlag - SQL NULL flag + * @param nRoundtrip + * @param isUpdate */ + boost::any convertColumnData(const std::string& data, bool& bSaturate, const std::string& timeZone, + bool nulFlag = false, bool noRoundup = false, bool isUpdate = false) const; - /** the type of a pair - * - * @todo TODO: the rid field in this structure does not make sense. the structure is equalivent - * to OID and therefore will be deprecated soon. - */ - //@bug 1866 changed rid default to 0; RID matches writeengine rid, uint64_t - struct ROPair + const std::string toString() const; + + // Put these here so udf doesn't need to link libexecplan + bool operator==(const ColType& t) const { - ROPair() : rid(std::numeric_limits::max()), objnum(0) { } - RID rid; - OID objnum; - }; + // order these with the most likely first + if (columnOID != t.columnOID) + return false; - /** the type of a list of Object ID's - * - * @todo TODO: the rid field in this structure does not make sense. the structure is equalivent - * to OID and therefore will be deprecated soon. - */ - typedef std::vector RIDList; + if (colPosition != t.colPosition) + return false; - /** the type of a pair - * - */ - struct IndexOID - { - OID objnum; - OID listOID; - bool multiColFlag; - }; + if (ddn != t.ddn) + return false; - /** the type of a list of Index OIDs - */ - typedef std::vector IndexOIDList; + if (colWidth != t.colWidth) + return false; - /** the structure for libcalmysql to use. Alias is taken account to the identity of tables. - * - */ - struct TableAliasName - { - TableAliasName (): fisColumnStore (true) {} - TableAliasName (const std::string &sch, const std::string &tb, const std::string &al) : - schema (sch), table (tb), alias (al), fisColumnStore(true) {} - TableAliasName (const std::string &sch, const std::string &tb, const std::string &al, const std::string &v) : - schema (sch), table (tb), alias (al), view(v), fisColumnStore(true) {} - std::string schema; - std::string table; - std::string alias; - std::string view; - bool fisColumnStore; - void clear(); - bool operator<(const TableAliasName& rhs) const; - bool operator>=(const TableAliasName& rhs) const - { - return !(*this < rhs); - } - bool operator==(const TableAliasName& rhs) const - { - return (schema == rhs.schema && - table == rhs.table && - alias == rhs.alias && - view == rhs.view && - fisColumnStore == rhs.fisColumnStore); - } - bool operator!=(const TableAliasName& rhs) const - { - return !(*this == rhs); - } - void serialize(messageqcpp::ByteStream&) const; - void unserialize(messageqcpp::ByteStream&); - friend std::ostream& operator<<(std::ostream& os, const TableAliasName& rhs); - }; + if (scale != t.scale) + return false; - /** the structure passed into various RID methods - * - */ - struct TableName - { - TableName() {} - TableName(std::string sch, std::string tb) : - schema(sch), table(tb) {} - TableName(const TableAliasName& tan): schema(tan.schema), table(tan.table) {} - std::string schema; - std::string table; - int64_t create_date; - bool operator<(const TableName& rhs) const; - bool operator>=(const TableName& rhs) const - { - return !(*this < rhs); - } - bool operator==(const TableName& rhs) const - { - return (schema == rhs.schema && table == rhs.table); - } - bool operator!=(const TableName& rhs) const - { - return !(*this == rhs); - } - const std::string toString() const; - friend std::ostream& operator<<(std::ostream& os, const TableName& rhs) - { - os << rhs.toString(); - return os; - } - }; + if (precision != t.precision) + return false; - /** the structure passed into get RID for Index values - * - */ - struct IndexName - { - std::string schema; - std::string table; - std::string index; - bool multiColFlag; - bool operator<(const IndexName& rhs) const; - }; + if (constraintType != t.constraintType) + return false; - /** the structure of a constraint infomation */ - struct ConstraintInfo - { - ConstraintInfo(): constraintType(NO_CONSTRAINT) {} - int constraintType; - IndexName constraintName; - std::string constraintText; - std::string referenceSchema; // for foreign key constraint - std::string referenceTable; // for foreign key constraint - std::string referencePKName; // for foreign key constraint - std::string constraintStatus; - }; - - /** the structure passed into lookupOID - * - */ - struct TableColName - { - std::string schema; - std::string table; - std::string column; - bool operator<(const TableColName& rhs) const; - const std::string toString() const; - friend std::ostream& operator<<(std::ostream& os, const TableColName& rhs); - }; - - typedef std::vector IndexNameList; - typedef std::vector TableColNameList; - - /** the type of a System Change Number - * - */ - typedef int SCN; - - /** the type of an index number - * - */ - typedef int INDNUM; - - /** the type returned by makeCalpontSystemCatalog() - * - */ - typedef boost::shared_ptr SPCSC; - - /** looks up a table's OID in the System Catalog - * - * For a unique table_name return the internal OID - */ - OID lookupTableOID(const TableName& tableName, int lower_case_table_names=0); - - /** looks up a column's OID in the System Catalog - * - * For a unique table_name.column_name return the internal OID - */ - OID lookupOID(const TableColName& tableColName, int lower_case_table_names=0); - - /** returns the column type attribute(s) for a column - * - * return the various column attributes for a given OID - */ - const ColType colType(const OID& oid); - - /** returns the column type attribute(s) for a column gived a dictionary OID - * - * return the same thing as colType does for the corresponding token OID - */ - const ColType colTypeDct(const OID& dictOid); - - /** returns the table column name for a column - * - * return the table column name for a given OID: - */ - const TableColName colName(const OID& oid); - /** returns the table column name for a dictionary column - * - * return the table column name for a given OID: - */ - const TableColName dictColName(const OID& oid); - - /** returns the next value of autoincrement for the table - * - * return the next value of autoincrement for a given table: - * AUTOINCR_SATURATED: limit exceeded - * 0: Autoincrement does not exist for this table - * Throws runtime_error if no such table found - */ - uint64_t nextAutoIncrValue(TableName tableName, int lower_case_table_names=0); - - /** returns the rid of next autoincrement value for the table oid - * - * return the rid of next value of autoincrement for a given table: - */ - const ROPair nextAutoIncrRid(const OID& oid); - - /** returns the oid of autoincrement column for the table - * - * return the oid of autoincrement column for a given table: - */ - int32_t autoColumOid(TableName tableName, int lower_case_table_names=0); - - /** returns the columns bitmap file object number - * - * return the bitmap file object number for a given OID: - */ - OID colBitmap(const OID& oid) const; - - /** return the current SCN - * - * returns the current System Change Number (for versioning support) - */ - SCN scn(void) const; - - /** return the RID's of the indexes for a table - * - * returns a list of the RID's of the indexes for a table - */ - const RIDList indexRIDs(const TableName& tableName, int lower_case_table_names=0); - - /** return the total number of columns for a table - * - * returns the total number of columns for a table - */ - int colNumbers(const TableName& tableName, int lower_case_table_names=0); - - /** return the RID's of the colindexes for a table - * - * returns a list of the RID's of the colindexes for a table - */ - const RIDList indexColRIDs(const TableName& tableName, int lower_case_table_names=0); - - /** return the RID's of the index columns (SYSINDEXCOL) for a index - * - * returns a list of the RID's of the colindexes for a index - */ - const RIDList indexColRIDs(const IndexName& indexName, int lower_case_table_names=0); - - /** return the RID's of the constraints for a table - * - * returns a list of the RID's of the constraints for a table - */ - const RIDList constraintRIDs(const TableName& tableName, int lower_case_table_names=0); - - /** return the RID of the constraint for a ConstrainName - * - * returns a RID of the constraint for a ConstrainName fron table SYSCONSTRAINT - */ - RID constraintRID(const std::string constraintName); - - /** return the list of IndexName for a given TableColName - * - * return the list of IndexName for a given TableColName from table SYSCONSTRAINTCOL - */ - const IndexNameList colValueSysconstraint (const TableColName& colName, bool useCache = false); - - /** return the RID's of the colconstraints for a table - * - * returns a list of the RID's of the colconstraints for a table - */ - const RIDList constraintColRIDs(const TableName& tableName, int lower_case_table_names=0); - - /** return the RID of the colconstraint for a column - * - * returns the RID of the colconstraints for a column - */ - RID constraintColRID(const TableColName& tableColName, int lower_case_table_names=0); - - /** return the value for the given RID and column name from table SYSCONSTRAINTCOL - * - * returns the column value for the given RID a column name fron table SYSCONSTRAINTCOL - */ - const std::string colValueSysconstraintCol (const TableColName& colName, int lower_case_table_names=0); - - /** return the RID of the constraint for a ConstrainName - * - * returns a RID of the constraint for a ConstrainName fron table SYSCONSTRAINTCOL - */ - const RIDList constraintColRID(const std::string constraintName); - - /** return the ROPair of the column - * - * @note the RID field in ROPair does not make sense. the purpose of this function is to - * return OID. Therefore it's duplicate to lookupOID function. This function is to be - * deprecated. - */ - const ROPair columnRID(const TableColName& tableColName, int lower_case_table_names=0); - - /** return the RID's of the columns for a table - * - * returns a list of the RID's of the columns for a table - */ - const RIDList columnRIDs(const TableName& tableName, bool useCache = false, int lower_case_table_names=0); - - /** return the RID of the table - * - * returns the RID of the table - */ - const ROPair tableRID(const TableName& tableName, int lower_case_table_names=0); - - /** return the RID of the index for a table - * - * returns the RID of the indexes for a table - */ - const ROPair indexRID(const IndexName& indexName); - /** return the INDEX NAME list for a table - * - * returns the index name list for a table - */ - const IndexNameList indexNames(const TableName& tableName, int lower_case_table_names=0); - /** return the column names for a index - * - * returns the column name list for a index - */ - const TableColNameList indexColNames ( const IndexName& indexName); - - /** return the column names for a index - * - * returns the column name list for a index - */ - const TableColNameList constraintColNames ( const std::string constraintName); - - /** return the value for the given RID and column name from table SYSINDEX - * - * returns the column value for the given RID a column name fron table SYSINDEX - */ - const std::string colValueSysindex (const TableColName& colName, int lower_case_table_names=0); - - /** return the RID of the colindexe for a table - * - * returns the RID's of the colindexe for a table - */ - const RIDList indexColRID(const IndexName& indexName); - - /** return the RID of the colindexe for a table - * - * returns the RID's of the colindexe for a table - */ - const ROPair indexColRID(const TableColName& tableColName, int lower_case_table_names=0); - - /** return the value for the given RID and column name from table SYSINDEX - * - * returns the column value for the given RID a column name fron table SYSINDEX - */ - const IndexNameList colValueSysindexCol (const TableColName& colName, int lower_case_table_names=0); - - /** looks up a Table Name for a Index in the System Catalog - * - * For a unique Index return the Table Name Structure - */ - const TableName lookupTableForIndex(const std::string indexName, const std::string schema); - - /** looks up a index oid for a Index in the System Catalog - * - * For an index name return index OID - */ - const IndexOID lookupIndexNbr(const IndexName& indexName); - - /** look up an Index Number for the given table column name - * - * If an index exists for the table and column return its number - * @note if one column belongs to multiple index, this function will get confused. - * for now, assume one column belongs to just one index. In the future getPlan - * should give index name therefore this function will be deprecated. - */ - const IndexOID lookupIndexNbr(const TableColName& tableColName, int lower_case_table_names=0); - - /** return the list of Index OIDs for the given table - * - * returns the list of Index OIDs for a table - */ - const IndexOIDList indexOIDs(const TableName& tableName, int lower_case_table_names=0); - - /** return the list of Dictionary OIDs for the given table - * - * returns the list of Dictionary OIDs for a table - */ - const DictOIDList dictOIDs(const TableName& tableName, int lower_case_table_names=0); - - /** Update column OID. This is for testing DDL and DML only - * and will go away once READ works - */ - void storeColOID (void) ; - - /** Update dictionary OIDs. This is for testing DDL and DML only - * and will go away once READ works - */ - void storeDictOID (void) ; - - /** Update index OIDs. This is for testing DDL and DML only - * and will go away once READ works - */ - void storeIndexOID (void) ; - - /** Reload dictionary OIDs, index OIDs, column OID. This is for testing DDL and DML only - * and will go away once READ works - */ - void updateColInfo (void) ; - - /** returns a pointer to the System Catalog singleton per session - * TODO: may need to change to one instance per transaction - * @parm sessionID to map the key of catalog map - */ - static SPCSC makeCalpontSystemCatalog(uint32_t sessionID = 0); - - /** remove and delete the instance map to the sessionid - * @param sessionID - */ - static void removeCalpontSystemCatalog(uint32_t sessionID = 0); - /** sessionid access and mutator methods - * - */ - uint32_t sessionID() const - { - return fSessionID; - } - void sessionID (uint32_t sessionID) - { - fSessionID = sessionID; - } - /** identity access and mutator methods - * - */ - int identity() const - { - return fIdentity; - } - void identity (int identity) - { - fIdentity = identity; + return true; } - /** return the column position - * - * return the column position for a given OID - */ - int colPosition (const OID& oid); - /** return primary key name for the given table */ - const std::string primaryKeyName (const TableName& tableName, int lower_case_table_names=0); - /** return the table info - * - * return the table info for a given TableName - */ - const TableInfo tableInfo (const TableName& tb, int lower_case_table_names=0); - /** return the table name for a give table oid */ - const TableName tableName (const OID& oid); - /** return the list of tables for a given schema */ - const std::vector< std::pair > getTables(const std::string schema = "", int lower_case_table_names=0); - /** return the number of tables in the whole database */ - int getTableCount (); - /** return the constraint info for a given constraint */ - const ConstraintInfo constraintInfo (const IndexName& constraintName); - /** return the constraintName list for a given referencePKName */ - const IndexNameList referenceConstraints (const IndexName& referencePKName); + bool operator!=(const ColType& t) const + { + return !(*this == t); + } - // @bug 682 - void getSchemaInfo(const std::string& schema, int lower_case_table_names=0); + static ColType convertUnionColType(std::vector&); + }; - typedef std::map OIDNextvalMap; - void updateColinfoCache(OIDNextvalMap& oidNextvalMap); + /** the structure of a table infomation + * + * this structure holds table related info. More in the future + */ + struct TableInfo + { + TableInfo() : numOfCols(0), tablewithautoincr(NO_AUTOINCRCOL) + { + } + int numOfCols; + int tablewithautoincr; + }; - void flushCache(); + /** the type of a Row ID + * + * @todo TODO: RIDs aren't really signed. This should be fixed. + */ + //@bug 1866 match RID to writeengine, uint64_t + typedef WriteEngine::RID RID; - /** Convert a MySQL thread id to an InfiniDB session id */ - static uint32_t idb_tid2sid(const uint32_t tid); + /** + * + */ - friend class ::ExecPlanTest; + /** the type of a pair + * + * @todo TODO: the rid field in this structure does not make sense. the structure is equalivent + * to OID and therefore will be deprecated soon. + */ + //@bug 1866 changed rid default to 0; RID matches writeengine rid, uint64_t + struct ROPair + { + ROPair() : rid(std::numeric_limits::max()), objnum(0) + { + } + RID rid; + OID objnum; + }; - /** Destructor */ - ~CalpontSystemCatalog(); + /** the type of a list of Object ID's + * + * @todo TODO: the rid field in this structure does not make sense. the structure is equalivent + * to OID and therefore will be deprecated soon. + */ + typedef std::vector RIDList; -private: - /** Constuctors */ - explicit CalpontSystemCatalog(); - explicit CalpontSystemCatalog(const CalpontSystemCatalog& rhs); + /** the type of a pair + * + */ + struct IndexOID + { + OID objnum; + OID listOID; + bool multiColFlag; + }; + /** the type of a list of Index OIDs + */ + typedef std::vector IndexOIDList; - CalpontSystemCatalog& operator=(const CalpontSystemCatalog& rhs); + /** the structure for libcalmysql to use. Alias is taken account to the identity of tables. + * + */ + struct TableAliasName + { + TableAliasName() : fisColumnStore(true) + { + } + TableAliasName(const std::string& sch, const std::string& tb, const std::string& al) + : schema(sch), table(tb), alias(al), fisColumnStore(true) + { + } + TableAliasName(const std::string& sch, const std::string& tb, const std::string& al, const std::string& v) + : schema(sch), table(tb), alias(al), view(v), fisColumnStore(true) + { + } + std::string schema; + std::string table; + std::string alias; + std::string view; + bool fisColumnStore; + void clear(); + bool operator<(const TableAliasName& rhs) const; + bool operator>=(const TableAliasName& rhs) const + { + return !(*this < rhs); + } + bool operator==(const TableAliasName& rhs) const + { + return (schema == rhs.schema && table == rhs.table && alias == rhs.alias && view == rhs.view && + fisColumnStore == rhs.fisColumnStore); + } + bool operator!=(const TableAliasName& rhs) const + { + return !(*this == rhs); + } + void serialize(messageqcpp::ByteStream&) const; + void unserialize(messageqcpp::ByteStream&); + friend std::ostream& operator<<(std::ostream& os, const TableAliasName& rhs); + }; - /** get system data */ - void getSysData (execplan::CalpontSelectExecutionPlan&, NJLSysDataList&, const std::string& sysTableName); - /** get system data for Front End */ - void getSysData_FE(const execplan::CalpontSelectExecutionPlan&, NJLSysDataList&, const std::string& sysTableName); - /** get system data for Engine Controller */ - void getSysData_EC(execplan::CalpontSelectExecutionPlan&, NJLSysDataList&, const std::string& sysTableName); + /** the structure passed into various RID methods + * + */ + struct TableName + { + TableName() + { + } + TableName(std::string sch, std::string tb) : schema(sch), table(tb) + { + } + TableName(const TableAliasName& tan) : schema(tan.schema), table(tan.table) + { + } + std::string schema; + std::string table; + int64_t create_date; + bool operator<(const TableName& rhs) const; + bool operator>=(const TableName& rhs) const + { + return !(*this < rhs); + } + bool operator==(const TableName& rhs) const + { + return (schema == rhs.schema && table == rhs.table); + } + bool operator!=(const TableName& rhs) const + { + return !(*this == rhs); + } + const std::string toString() const; + friend std::ostream& operator<<(std::ostream& os, const TableName& rhs) + { + os << rhs.toString(); + return os; + } + }; - void buildSysColinfomap(); - void buildSysOIDmap(); - void buildSysTablemap(); - void buildSysDctmap(); + /** the structure passed into get RID for Index values + * + */ + struct IndexName + { + std::string schema; + std::string table; + std::string index; + bool multiColFlag; + bool operator<(const IndexName& rhs) const; + }; - void checkSysCatVer(); + /** the structure of a constraint infomation */ + struct ConstraintInfo + { + ConstraintInfo() : constraintType(NO_CONSTRAINT) + { + } + int constraintType; + IndexName constraintName; + std::string constraintText; + std::string referenceSchema; // for foreign key constraint + std::string referenceTable; // for foreign key constraint + std::string referencePKName; // for foreign key constraint + std::string constraintStatus; + }; - static boost::mutex map_mutex; - static CatalogMap fCatalogMap; + /** the structure passed into lookupOID + * + */ + struct TableColName + { + std::string schema; + std::string table; + std::string column; + bool operator<(const TableColName& rhs) const; + const std::string toString() const; + friend std::ostream& operator<<(std::ostream& os, const TableColName& rhs); + }; - typedef std::map OIDmap; - OIDmap fOIDmap; - boost::mutex fOIDmapLock; //Also locks fColRIDmap + typedef std::vector IndexNameList; + typedef std::vector TableColNameList; - typedef std::map Tablemap; - Tablemap fTablemap; + /** the type of a System Change Number + * + */ + typedef int SCN; - typedef std::map Colinfomap; - Colinfomap fColinfomap; - boost::mutex fColinfomapLock; + /** the type of an index number + * + */ + typedef int INDNUM; - /** this structure is used by ddl only. it cache the rid for rows in syscolumn - that match the tableColName */ - typedef std::map ColRIDmap; - ColRIDmap fColRIDmap; + /** the type returned by makeCalpontSystemCatalog() + * + */ + typedef boost::shared_ptr SPCSC; - /** this structure is used by ddl only. it cache the rid for rows in systable - that match the tableName */ - typedef std::map TableRIDmap; - TableRIDmap fTableRIDmap; + /** looks up a table's OID in the System Catalog + * + * For a unique table_name return the internal OID + */ + OID lookupTableOID(const TableName& tableName, int lower_case_table_names = 0); + /** looks up a column's OID in the System Catalog + * + * For a unique table_name.column_name return the internal OID + */ + OID lookupOID(const TableColName& tableColName, int lower_case_table_names = 0); - // this structure may combine with Tablemap, where RID is added to TalbeInfo struct - typedef std::map TableInfoMap; - TableInfoMap fTableInfoMap; - boost::mutex fTableInfoMapLock; + /** returns the column type attribute(s) for a column + * + * return the various column attributes for a given OID + */ + const ColType colType(const OID& oid); - typedef std::map ColIndexListmap; - ColIndexListmap fColIndexListmap; - boost::mutex fColIndexListmapLock; + /** returns the column type attribute(s) for a column gived a dictionary OID + * + * return the same thing as colType does for the corresponding token OID + */ + const ColType colTypeDct(const OID& dictOid); - typedef std::map DctTokenMap; - DctTokenMap fDctTokenMap; - // MCOL-859: this can lock when already locked in the same thread - boost::recursive_mutex fDctTokenMapLock; + /** returns the table column name for a column + * + * return the table column name for a given OID: + */ + const TableColName colName(const OID& oid); + /** returns the table column name for a dictionary column + * + * return the table column name for a given OID: + */ + const TableColName dictColName(const OID& oid); - typedef std::map TableNameMap; - TableNameMap fTableNameMap; - boost::mutex fTableNameMapLock; + /** returns the next value of autoincrement for the table + * + * return the next value of autoincrement for a given table: + * AUTOINCR_SATURATED: limit exceeded + * 0: Autoincrement does not exist for this table + * Throws runtime_error if no such table found + */ + uint64_t nextAutoIncrValue(TableName tableName, int lower_case_table_names = 0); - ClientRotator* fExeMgr; - uint32_t fSessionID; - uint32_t fTxn; - int fIdentity; - std::set fSchemaCache; - boost::mutex fSchemaCacheLock; - //Cache flush - boost::mutex fSyscatSCNLock; - SCN fSyscatSCN; + /** returns the rid of next autoincrement value for the table oid + * + * return the rid of next value of autoincrement for a given table: + */ + const ROPair nextAutoIncrRid(const OID& oid); - static uint32_t fModuleID; + /** returns the oid of autoincrement column for the table + * + * return the oid of autoincrement column for a given table: + */ + int32_t autoColumOid(TableName tableName, int lower_case_table_names = 0); + + /** returns the columns bitmap file object number + * + * return the bitmap file object number for a given OID: + */ + OID colBitmap(const OID& oid) const; + + /** return the current SCN + * + * returns the current System Change Number (for versioning support) + */ + SCN scn(void) const; + + /** return the RID's of the indexes for a table + * + * returns a list of the RID's of the indexes for a table + */ + const RIDList indexRIDs(const TableName& tableName, int lower_case_table_names = 0); + + /** return the total number of columns for a table + * + * returns the total number of columns for a table + */ + int colNumbers(const TableName& tableName, int lower_case_table_names = 0); + + /** return the RID's of the colindexes for a table + * + * returns a list of the RID's of the colindexes for a table + */ + const RIDList indexColRIDs(const TableName& tableName, int lower_case_table_names = 0); + + /** return the RID's of the index columns (SYSINDEXCOL) for a index + * + * returns a list of the RID's of the colindexes for a index + */ + const RIDList indexColRIDs(const IndexName& indexName, int lower_case_table_names = 0); + + /** return the RID's of the constraints for a table + * + * returns a list of the RID's of the constraints for a table + */ + const RIDList constraintRIDs(const TableName& tableName, int lower_case_table_names = 0); + + /** return the RID of the constraint for a ConstrainName + * + * returns a RID of the constraint for a ConstrainName fron table SYSCONSTRAINT + */ + RID constraintRID(const std::string constraintName); + + /** return the list of IndexName for a given TableColName + * + * return the list of IndexName for a given TableColName from table SYSCONSTRAINTCOL + */ + const IndexNameList colValueSysconstraint(const TableColName& colName, bool useCache = false); + + /** return the RID's of the colconstraints for a table + * + * returns a list of the RID's of the colconstraints for a table + */ + const RIDList constraintColRIDs(const TableName& tableName, int lower_case_table_names = 0); + + /** return the RID of the colconstraint for a column + * + * returns the RID of the colconstraints for a column + */ + RID constraintColRID(const TableColName& tableColName, int lower_case_table_names = 0); + + /** return the value for the given RID and column name from table SYSCONSTRAINTCOL + * + * returns the column value for the given RID a column name fron table SYSCONSTRAINTCOL + */ + const std::string colValueSysconstraintCol(const TableColName& colName, int lower_case_table_names = 0); + + /** return the RID of the constraint for a ConstrainName + * + * returns a RID of the constraint for a ConstrainName fron table SYSCONSTRAINTCOL + */ + const RIDList constraintColRID(const std::string constraintName); + + /** return the ROPair of the column + * + * @note the RID field in ROPair does not make sense. the purpose of this function is to + * return OID. Therefore it's duplicate to lookupOID function. This function is to be + * deprecated. + */ + const ROPair columnRID(const TableColName& tableColName, int lower_case_table_names = 0); + + /** return the RID's of the columns for a table + * + * returns a list of the RID's of the columns for a table + */ + const RIDList columnRIDs(const TableName& tableName, bool useCache = false, int lower_case_table_names = 0); + + /** return the RID of the table + * + * returns the RID of the table + */ + const ROPair tableRID(const TableName& tableName, int lower_case_table_names = 0); + + /** return the RID of the index for a table + * + * returns the RID of the indexes for a table + */ + const ROPair indexRID(const IndexName& indexName); + /** return the INDEX NAME list for a table + * + * returns the index name list for a table + */ + const IndexNameList indexNames(const TableName& tableName, int lower_case_table_names = 0); + /** return the column names for a index + * + * returns the column name list for a index + */ + const TableColNameList indexColNames(const IndexName& indexName); + + /** return the column names for a index + * + * returns the column name list for a index + */ + const TableColNameList constraintColNames(const std::string constraintName); + + /** return the value for the given RID and column name from table SYSINDEX + * + * returns the column value for the given RID a column name fron table SYSINDEX + */ + const std::string colValueSysindex(const TableColName& colName, int lower_case_table_names = 0); + + /** return the RID of the colindexe for a table + * + * returns the RID's of the colindexe for a table + */ + const RIDList indexColRID(const IndexName& indexName); + + /** return the RID of the colindexe for a table + * + * returns the RID's of the colindexe for a table + */ + const ROPair indexColRID(const TableColName& tableColName, int lower_case_table_names = 0); + + /** return the value for the given RID and column name from table SYSINDEX + * + * returns the column value for the given RID a column name fron table SYSINDEX + */ + const IndexNameList colValueSysindexCol(const TableColName& colName, int lower_case_table_names = 0); + + /** looks up a Table Name for a Index in the System Catalog + * + * For a unique Index return the Table Name Structure + */ + const TableName lookupTableForIndex(const std::string indexName, const std::string schema); + + /** looks up a index oid for a Index in the System Catalog + * + * For an index name return index OID + */ + const IndexOID lookupIndexNbr(const IndexName& indexName); + + /** look up an Index Number for the given table column name + * + * If an index exists for the table and column return its number + * @note if one column belongs to multiple index, this function will get confused. + * for now, assume one column belongs to just one index. In the future getPlan + * should give index name therefore this function will be deprecated. + */ + const IndexOID lookupIndexNbr(const TableColName& tableColName, int lower_case_table_names = 0); + + /** return the list of Index OIDs for the given table + * + * returns the list of Index OIDs for a table + */ + const IndexOIDList indexOIDs(const TableName& tableName, int lower_case_table_names = 0); + + /** return the list of Dictionary OIDs for the given table + * + * returns the list of Dictionary OIDs for a table + */ + const DictOIDList dictOIDs(const TableName& tableName, int lower_case_table_names = 0); + + /** Update column OID. This is for testing DDL and DML only + * and will go away once READ works + */ + void storeColOID(void); + + /** Update dictionary OIDs. This is for testing DDL and DML only + * and will go away once READ works + */ + void storeDictOID(void); + + /** Update index OIDs. This is for testing DDL and DML only + * and will go away once READ works + */ + void storeIndexOID(void); + + /** Reload dictionary OIDs, index OIDs, column OID. This is for testing DDL and DML only + * and will go away once READ works + */ + void updateColInfo(void); + + /** returns a pointer to the System Catalog singleton per session + * TODO: may need to change to one instance per transaction + * @parm sessionID to map the key of catalog map + */ + static SPCSC makeCalpontSystemCatalog(uint32_t sessionID = 0); + + /** remove and delete the instance map to the sessionid + * @param sessionID + */ + static void removeCalpontSystemCatalog(uint32_t sessionID = 0); + /** sessionid access and mutator methods + * + */ + uint32_t sessionID() const + { + return fSessionID; + } + void sessionID(uint32_t sessionID) + { + fSessionID = sessionID; + } + /** identity access and mutator methods + * + */ + int identity() const + { + return fIdentity; + } + void identity(int identity) + { + fIdentity = identity; + } + + /** return the column position + * + * return the column position for a given OID + */ + int colPosition(const OID& oid); + /** return primary key name for the given table */ + const std::string primaryKeyName(const TableName& tableName, int lower_case_table_names = 0); + /** return the table info + * + * return the table info for a given TableName + */ + const TableInfo tableInfo(const TableName& tb, int lower_case_table_names = 0); + /** return the table name for a give table oid */ + const TableName tableName(const OID& oid); + /** return the list of tables for a given schema */ + const std::vector > getTables(const std::string schema = "", + int lower_case_table_names = 0); + /** return the number of tables in the whole database */ + int getTableCount(); + /** return the constraint info for a given constraint */ + const ConstraintInfo constraintInfo(const IndexName& constraintName); + /** return the constraintName list for a given referencePKName */ + const IndexNameList referenceConstraints(const IndexName& referencePKName); + + // @bug 682 + void getSchemaInfo(const std::string& schema, int lower_case_table_names = 0); + + typedef std::map OIDNextvalMap; + void updateColinfoCache(OIDNextvalMap& oidNextvalMap); + + void flushCache(); + + /** Convert a MySQL thread id to an InfiniDB session id */ + static uint32_t idb_tid2sid(const uint32_t tid); + + friend class ::ExecPlanTest; + + /** Destructor */ + ~CalpontSystemCatalog(); + + private: + /** Constuctors */ + explicit CalpontSystemCatalog(); + explicit CalpontSystemCatalog(const CalpontSystemCatalog& rhs); + + CalpontSystemCatalog& operator=(const CalpontSystemCatalog& rhs); + + /** get system data */ + void getSysData(execplan::CalpontSelectExecutionPlan&, NJLSysDataList&, const std::string& sysTableName); + /** get system data for Front End */ + void getSysData_FE(const execplan::CalpontSelectExecutionPlan&, NJLSysDataList&, + const std::string& sysTableName); + /** get system data for Engine Controller */ + void getSysData_EC(execplan::CalpontSelectExecutionPlan&, NJLSysDataList&, const std::string& sysTableName); + + void buildSysColinfomap(); + void buildSysOIDmap(); + void buildSysTablemap(); + void buildSysDctmap(); + + void checkSysCatVer(); + + static boost::mutex map_mutex; + static CatalogMap fCatalogMap; + + typedef std::map OIDmap; + OIDmap fOIDmap; + boost::mutex fOIDmapLock; // Also locks fColRIDmap + + typedef std::map Tablemap; + Tablemap fTablemap; + + typedef std::map Colinfomap; + Colinfomap fColinfomap; + boost::mutex fColinfomapLock; + + /** this structure is used by ddl only. it cache the rid for rows in syscolumn + that match the tableColName */ + typedef std::map ColRIDmap; + ColRIDmap fColRIDmap; + + /** this structure is used by ddl only. it cache the rid for rows in systable + that match the tableName */ + typedef std::map TableRIDmap; + TableRIDmap fTableRIDmap; + + // this structure may combine with Tablemap, where RID is added to TalbeInfo struct + typedef std::map TableInfoMap; + TableInfoMap fTableInfoMap; + boost::mutex fTableInfoMapLock; + + typedef std::map ColIndexListmap; + ColIndexListmap fColIndexListmap; + boost::mutex fColIndexListmapLock; + + typedef std::map DctTokenMap; + DctTokenMap fDctTokenMap; + // MCOL-859: this can lock when already locked in the same thread + boost::recursive_mutex fDctTokenMapLock; + + typedef std::map TableNameMap; + TableNameMap fTableNameMap; + boost::mutex fTableNameMapLock; + + ClientRotator* fExeMgr; + uint32_t fSessionID; + uint32_t fTxn; + int fIdentity; + std::set fSchemaCache; + boost::mutex fSchemaCacheLock; + // Cache flush + boost::mutex fSyscatSCNLock; + SCN fSyscatSCN; + + static uint32_t fModuleID; }; /** convenience function to make a TableColName from 3 strings */ -const CalpontSystemCatalog::TableColName make_tcn(const std::string& s, const std::string& t, const std::string& c, int lower_case_table_names=0); +const CalpontSystemCatalog::TableColName make_tcn(const std::string& s, const std::string& t, + const std::string& c, int lower_case_table_names = 0); /** convenience function to make a TableName from 2 strings */ -const CalpontSystemCatalog::TableName make_table(const std::string& s, const std::string& t, int lower_case_table_names=0); -const CalpontSystemCatalog::TableAliasName make_aliastable(const std::string& s, const std::string& t, const std::string& a, - const bool fisColumnStore = true, int lower_case_table_names=0); -const CalpontSystemCatalog::TableAliasName make_aliasview(const std::string& s, const std::string& t, const std::string& a, const std::string& v, - const bool fisColumnStore = true, int lower_case_table_names=0); +const CalpontSystemCatalog::TableName make_table(const std::string& s, const std::string& t, + int lower_case_table_names = 0); +const CalpontSystemCatalog::TableAliasName make_aliastable(const std::string& s, const std::string& t, + const std::string& a, + const bool fisColumnStore = true, + int lower_case_table_names = 0); +const CalpontSystemCatalog::TableAliasName make_aliasview(const std::string& s, const std::string& t, + const std::string& a, const std::string& v, + const bool fisColumnStore = true, + int lower_case_table_names = 0); inline bool isNull(int128_t val, const execplan::CalpontSystemCatalog::ColType& ct) { - return datatypes::Decimal::isWideDecimalNullValue(val); + return datatypes::Decimal::isWideDecimalNullValue(val); } inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& ct) { - bool ret = false; + bool ret = false; - switch (ct.colDataType) + switch (ct.colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: { - case execplan::CalpontSystemCatalog::TINYINT: - { - if ((int8_t) joblist::TINYINTNULL == val) ret = true; + if ((int8_t)joblist::TINYINTNULL == val) + ret = true; - break; + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + { + int colWidth = ct.colWidth; + + if (colWidth <= 8) + { + if ((colWidth == 1) && ((int8_t)joblist::CHAR1NULL == val)) + ret = true; + else if ((colWidth == 2) && ((int16_t)joblist::CHAR2NULL == val)) + ret = true; + else if ((colWidth < 5) && ((int32_t)joblist::CHAR4NULL == val)) + ret = true; + else if ((int64_t)joblist::CHAR8NULL == val) + ret = true; + } + else + { + throw std::logic_error("Not a int column."); + } + + break; + } + + case execplan::CalpontSystemCatalog::SMALLINT: + { + if ((int16_t)joblist::SMALLINTNULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + switch (ct.colWidth) + { + case 1: + { + if ((int8_t)joblist::TINYINTNULL == val) + ret = true; + + break; } - case execplan::CalpontSystemCatalog::CHAR: + case 2: { - int colWidth = ct.colWidth; + if ((int16_t)joblist::SMALLINTNULL == val) + ret = true; - if (colWidth <= 8) - { - if ((colWidth == 1) && ((int8_t) joblist::CHAR1NULL == val)) ret = true ; - else if ((colWidth == 2) && ((int16_t) joblist::CHAR2NULL == val)) ret = true; - else if ((colWidth < 5) && ((int32_t) joblist::CHAR4NULL == val)) ret = true; - else if ((int64_t) joblist::CHAR8NULL == val) ret = true; - } - else - { - throw std::logic_error("Not a int column."); - } - - break; + break; } - case execplan::CalpontSystemCatalog::SMALLINT: + case 4: { - if ((int16_t) joblist::SMALLINTNULL == val) ret = true; + if ((int32_t)joblist::INTNULL == val) + ret = true; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - switch (ct.colWidth) - { - case 1: - { - if ((int8_t)joblist::TINYINTNULL == val) ret = true; - - break; - } - - case 2: - { - if ((int16_t)joblist::SMALLINTNULL == val) ret = true; - - break; - } - - case 4: - { - if ((int32_t)joblist::INTNULL == val) ret = true; - - break; - } - - default: - { - if ((int64_t)joblist::BIGINTNULL == val) ret = true; - - break; - } - } - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - if ((int64_t)joblist::DOUBLENULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - { - if ((int32_t)joblist::INTNULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - if ((int32_t)joblist::FLOATNULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - if ((int32_t) joblist::DATENULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::BIGINT: - { - if ((int64_t)joblist::BIGINTNULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - if ((int64_t)joblist::DATETIMENULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - if ((int64_t)joblist::TIMESTAMPNULL == val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - { - int colWidth = ct.colWidth; - - if (colWidth <= 8) - { - if ((colWidth < 3) && ((int16_t) joblist::CHAR2NULL == val)) ret = true; - else if ((colWidth < 5) && ((int32_t) joblist::CHAR4NULL == val)) ret = true; - else if ((int64_t)joblist::CHAR8NULL == val) ret = true; - } - else - { - throw std::logic_error("Not a int column."); - } - - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - { - if (joblist::UTINYINTNULL == (uint8_t)val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::USMALLINT: - { - if (joblist::USMALLINTNULL == (uint16_t)val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - if (joblist::UINTNULL == (uint32_t)val) ret = true; - - break; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - { - if (joblist::UBIGINTNULL == (uint64_t)val) ret = true; - - break; + break; } default: - break; + { + if ((int64_t)joblist::BIGINTNULL == val) + ret = true; + + break; + } + } + break; } - return ret; + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + if ((int64_t)joblist::DOUBLENULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + { + if ((int32_t)joblist::INTNULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + if ((int32_t)joblist::FLOATNULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + if ((int32_t)joblist::DATENULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::BIGINT: + { + if ((int64_t)joblist::BIGINTNULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + if ((int64_t)joblist::DATETIMENULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if ((int64_t)joblist::TIMESTAMPNULL == val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + { + int colWidth = ct.colWidth; + + if (colWidth <= 8) + { + if ((colWidth < 3) && ((int16_t)joblist::CHAR2NULL == val)) + ret = true; + else if ((colWidth < 5) && ((int32_t)joblist::CHAR4NULL == val)) + ret = true; + else if ((int64_t)joblist::CHAR8NULL == val) + ret = true; + } + else + { + throw std::logic_error("Not a int column."); + } + + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: + { + if (joblist::UTINYINTNULL == (uint8_t)val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::USMALLINT: + { + if (joblist::USMALLINTNULL == (uint16_t)val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + { + if (joblist::UINTNULL == (uint32_t)val) + ret = true; + + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: + { + if (joblist::UBIGINTNULL == (uint64_t)val) + ret = true; + + break; + } + + default: break; + } + + return ret; } /** constants for system table names @@ -1130,103 +1170,102 @@ const std::string COMPRESSIONTYPE_COL = "compressiontype"; const std::string NEXTVALUE_COL = "nextvalue"; /***************************************************** -* System tables OID definition -******************************************************/ -const int SYSTABLE_BASE = 1000; /** @brief SYSTABLE table base */ -const int SYSTABLE_DICT_BASE = 2000; /** @brief SYSTABLE table dictionary files base */ -const int SYSCOLUMN_BASE = 1020; /** @brief SYSCOLUMN table base */ -const int SYSCOLUMN_DICT_BASE = 2060; /** @brief SYSCOLUMN table dictionary files base */ + * System tables OID definition + ******************************************************/ +const int SYSTABLE_BASE = 1000; /** @brief SYSTABLE table base */ +const int SYSTABLE_DICT_BASE = 2000; /** @brief SYSTABLE table dictionary files base */ +const int SYSCOLUMN_BASE = 1020; /** @brief SYSCOLUMN table base */ +const int SYSCOLUMN_DICT_BASE = 2060; /** @brief SYSCOLUMN table dictionary files base */ /***************************************************** -* SYSTABLE columns OID definition -******************************************************/ -const int OID_SYSTABLE_TABLENAME = SYSTABLE_BASE + 1; /** @brief TABLENAME column */ -const int OID_SYSTABLE_SCHEMA = SYSTABLE_BASE + 2; /** @brief SCHEMA column */ -const int OID_SYSTABLE_OBJECTID = SYSTABLE_BASE + 3; /** @brief OBJECTID column */ -const int OID_SYSTABLE_CREATEDATE = SYSTABLE_BASE + 4; /** @brief CREATEDATE column */ -const int OID_SYSTABLE_LASTUPDATE = SYSTABLE_BASE + 5; /** @brief LASTUPDATE column */ -const int OID_SYSTABLE_INIT = SYSTABLE_BASE + 6; /** @brief INIT column */ -const int OID_SYSTABLE_NEXT = SYSTABLE_BASE + 7; /** @brief NEXT column */ -const int OID_SYSTABLE_NUMOFROWS = SYSTABLE_BASE + 8; /** @brief total num of rows column */ -const int OID_SYSTABLE_AVGROWLEN = SYSTABLE_BASE + 9; /** @brief avg. row length column */ -const int OID_SYSTABLE_NUMOFBLOCKS = SYSTABLE_BASE + 10; /** @brief num. of blocks column */ -const int OID_SYSTABLE_AUTOINCREMENT = SYSTABLE_BASE + 11; /** @brief AUTOINCREMENT column */ -const int SYSTABLE_MAX = SYSTABLE_BASE + 12; // be sure this is one more than the highest # + * SYSTABLE columns OID definition + ******************************************************/ +const int OID_SYSTABLE_TABLENAME = SYSTABLE_BASE + 1; /** @brief TABLENAME column */ +const int OID_SYSTABLE_SCHEMA = SYSTABLE_BASE + 2; /** @brief SCHEMA column */ +const int OID_SYSTABLE_OBJECTID = SYSTABLE_BASE + 3; /** @brief OBJECTID column */ +const int OID_SYSTABLE_CREATEDATE = SYSTABLE_BASE + 4; /** @brief CREATEDATE column */ +const int OID_SYSTABLE_LASTUPDATE = SYSTABLE_BASE + 5; /** @brief LASTUPDATE column */ +const int OID_SYSTABLE_INIT = SYSTABLE_BASE + 6; /** @brief INIT column */ +const int OID_SYSTABLE_NEXT = SYSTABLE_BASE + 7; /** @brief NEXT column */ +const int OID_SYSTABLE_NUMOFROWS = SYSTABLE_BASE + 8; /** @brief total num of rows column */ +const int OID_SYSTABLE_AVGROWLEN = SYSTABLE_BASE + 9; /** @brief avg. row length column */ +const int OID_SYSTABLE_NUMOFBLOCKS = SYSTABLE_BASE + 10; /** @brief num. of blocks column */ +const int OID_SYSTABLE_AUTOINCREMENT = SYSTABLE_BASE + 11; /** @brief AUTOINCREMENT column */ +const int SYSTABLE_MAX = SYSTABLE_BASE + 12; // be sure this is one more than the highest # /***************************************************** -* SYSCOLUMN columns OID definition -******************************************************/ -const int OID_SYSCOLUMN_SCHEMA = SYSCOLUMN_BASE + 1; /** @brief SCHEMA column */ -const int OID_SYSCOLUMN_TABLENAME = SYSCOLUMN_BASE + 2; /** @brief TABLENAME column */ -const int OID_SYSCOLUMN_COLNAME = SYSCOLUMN_BASE + 3; /** @brief COLNAME column */ -const int OID_SYSCOLUMN_OBJECTID = SYSCOLUMN_BASE + 4; /** @brief OBJECTID column */ -const int OID_SYSCOLUMN_DICTOID = SYSCOLUMN_BASE + 5; /** @brief DICTOID column */ -const int OID_SYSCOLUMN_LISTOBJID = SYSCOLUMN_BASE + 6; /** @brief LISTOBJID column */ -const int OID_SYSCOLUMN_TREEOBJID = SYSCOLUMN_BASE + 7; /** @brief TREEOBJID column */ -const int OID_SYSCOLUMN_DATATYPE = SYSCOLUMN_BASE + 8; /** @brief DATATYPE column */ -const int OID_SYSCOLUMN_COLUMNLEN = SYSCOLUMN_BASE + 9; /** @brief COLUMNLEN column */ -const int OID_SYSCOLUMN_COLUMNPOS = SYSCOLUMN_BASE + 10; /** @brief COLUMNPOS column */ -const int OID_SYSCOLUMN_LASTUPDATE = SYSCOLUMN_BASE + 11; /** @brief LASTUPDATE column */ -const int OID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_BASE + 12; /** @brief DEFAULTVAL column */ -const int OID_SYSCOLUMN_NULLABLE = SYSCOLUMN_BASE + 13; /** @brief NULLABLE column */ -const int OID_SYSCOLUMN_SCALE = SYSCOLUMN_BASE + 14; /** @brief SCALE column */ -const int OID_SYSCOLUMN_PRECISION = SYSCOLUMN_BASE + 15; /** @brief PRECISION column */ -const int OID_SYSCOLUMN_AUTOINC = SYSCOLUMN_BASE + 16; /** @brief AUTOINC column */ -const int OID_SYSCOLUMN_DISTCOUNT = SYSCOLUMN_BASE + 17; /** @brief Num of distinct value col */ -const int OID_SYSCOLUMN_NULLCOUNT = SYSCOLUMN_BASE + 18; /** @brief Num of null value col */ -const int OID_SYSCOLUMN_MINVALUE = SYSCOLUMN_BASE + 19; /** @brief min value col */ -const int OID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_BASE + 20; /** @brief max value col */ -const int OID_SYSCOLUMN_COMPRESSIONTYPE = SYSCOLUMN_BASE + 21; /** @brief compression type */ -const int OID_SYSCOLUMN_NEXTVALUE = SYSCOLUMN_BASE + 22; /** @brief next value */ -const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 23; // be sure this is one more than the highest # + * SYSCOLUMN columns OID definition + ******************************************************/ +const int OID_SYSCOLUMN_SCHEMA = SYSCOLUMN_BASE + 1; /** @brief SCHEMA column */ +const int OID_SYSCOLUMN_TABLENAME = SYSCOLUMN_BASE + 2; /** @brief TABLENAME column */ +const int OID_SYSCOLUMN_COLNAME = SYSCOLUMN_BASE + 3; /** @brief COLNAME column */ +const int OID_SYSCOLUMN_OBJECTID = SYSCOLUMN_BASE + 4; /** @brief OBJECTID column */ +const int OID_SYSCOLUMN_DICTOID = SYSCOLUMN_BASE + 5; /** @brief DICTOID column */ +const int OID_SYSCOLUMN_LISTOBJID = SYSCOLUMN_BASE + 6; /** @brief LISTOBJID column */ +const int OID_SYSCOLUMN_TREEOBJID = SYSCOLUMN_BASE + 7; /** @brief TREEOBJID column */ +const int OID_SYSCOLUMN_DATATYPE = SYSCOLUMN_BASE + 8; /** @brief DATATYPE column */ +const int OID_SYSCOLUMN_COLUMNLEN = SYSCOLUMN_BASE + 9; /** @brief COLUMNLEN column */ +const int OID_SYSCOLUMN_COLUMNPOS = SYSCOLUMN_BASE + 10; /** @brief COLUMNPOS column */ +const int OID_SYSCOLUMN_LASTUPDATE = SYSCOLUMN_BASE + 11; /** @brief LASTUPDATE column */ +const int OID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_BASE + 12; /** @brief DEFAULTVAL column */ +const int OID_SYSCOLUMN_NULLABLE = SYSCOLUMN_BASE + 13; /** @brief NULLABLE column */ +const int OID_SYSCOLUMN_SCALE = SYSCOLUMN_BASE + 14; /** @brief SCALE column */ +const int OID_SYSCOLUMN_PRECISION = SYSCOLUMN_BASE + 15; /** @brief PRECISION column */ +const int OID_SYSCOLUMN_AUTOINC = SYSCOLUMN_BASE + 16; /** @brief AUTOINC column */ +const int OID_SYSCOLUMN_DISTCOUNT = SYSCOLUMN_BASE + 17; /** @brief Num of distinct value col */ +const int OID_SYSCOLUMN_NULLCOUNT = SYSCOLUMN_BASE + 18; /** @brief Num of null value col */ +const int OID_SYSCOLUMN_MINVALUE = SYSCOLUMN_BASE + 19; /** @brief min value col */ +const int OID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_BASE + 20; /** @brief max value col */ +const int OID_SYSCOLUMN_COMPRESSIONTYPE = SYSCOLUMN_BASE + 21; /** @brief compression type */ +const int OID_SYSCOLUMN_NEXTVALUE = SYSCOLUMN_BASE + 22; /** @brief next value */ +const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 23; // be sure this is one more than the highest # /***************************************************** -* SYSTABLE columns dictionary OID definition -******************************************************/ -const int DICTOID_SYSTABLE_TABLENAME = SYSTABLE_DICT_BASE + 1; /** @brief TABLENAME column DICTOID*/ -const int LISTOID_SYSTABLE_TABLENAME = SYSTABLE_DICT_BASE + 2; /** @brief TABLENAME column LISTOID*/ -const int TREEOID_SYSTABLE_TABLENAME = SYSTABLE_DICT_BASE + 3; /** @brief TABLENAME column TREEOID*/ -const int DICTOID_SYSTABLE_SCHEMA = SYSTABLE_DICT_BASE + 4; /** @brief SCHEMA column DICTOID*/ -const int LISTOID_SYSTABLE_SCHEMA = SYSTABLE_DICT_BASE + 5; /** @brief SCHEMA column LISTOID*/ -const int TREEOID_SYSTABLE_SCHEMA = SYSTABLE_DICT_BASE + 6; /** @brief SCHEMA column TREEOID*/ -const int SYSTABLE_DICT_MAX = SYSTABLE_DICT_BASE + 7; // be sure this is one more than the highest # + * SYSTABLE columns dictionary OID definition + ******************************************************/ +const int DICTOID_SYSTABLE_TABLENAME = SYSTABLE_DICT_BASE + 1; /** @brief TABLENAME column DICTOID*/ +const int LISTOID_SYSTABLE_TABLENAME = SYSTABLE_DICT_BASE + 2; /** @brief TABLENAME column LISTOID*/ +const int TREEOID_SYSTABLE_TABLENAME = SYSTABLE_DICT_BASE + 3; /** @brief TABLENAME column TREEOID*/ +const int DICTOID_SYSTABLE_SCHEMA = SYSTABLE_DICT_BASE + 4; /** @brief SCHEMA column DICTOID*/ +const int LISTOID_SYSTABLE_SCHEMA = SYSTABLE_DICT_BASE + 5; /** @brief SCHEMA column LISTOID*/ +const int TREEOID_SYSTABLE_SCHEMA = SYSTABLE_DICT_BASE + 6; /** @brief SCHEMA column TREEOID*/ +const int SYSTABLE_DICT_MAX = SYSTABLE_DICT_BASE + 7; // be sure this is one more than the highest # /***************************************************** -* SYSCOLUMN columns dictionary OID definition -******************************************************/ -const int DICTOID_SYSCOLUMN_SCHEMA = SYSCOLUMN_DICT_BASE + 1; /** @brief SCHEMA column DICTOID*/ -const int LISTOID_SYSCOLUMN_SCHEMA = SYSCOLUMN_DICT_BASE + 2; /** @brief SCHEMA column LISTOID*/ -const int TREEOID_SYSCOLUMN_SCHEMA = SYSCOLUMN_DICT_BASE + 3; /** @brief SCHEMA column TREEOID*/ -const int DICTOID_SYSCOLUMN_TABLENAME = SYSCOLUMN_DICT_BASE + 4; /** @brief TABLENAME column DICTOID*/ -const int LISTOID_SYSCOLUMN_TABLENAME = SYSCOLUMN_DICT_BASE + 5; /** @brief TABLENAME column LISTOID*/ -const int TREEOID_SYSCOLUMN_TABLENAME = SYSCOLUMN_DICT_BASE + 6; /** @brief TABLENAME column TREEOID*/ -const int DICTOID_SYSCOLUMN_COLNAME = SYSCOLUMN_DICT_BASE + 7; /** @brief COLNAME column DICTOID*/ -const int LISTOID_SYSCOLUMN_COLNAME = SYSCOLUMN_DICT_BASE + 8; /** @brief COLNAME column LISTOID*/ -const int TREEOID_SYSCOLUMN_COLNAME = SYSCOLUMN_DICT_BASE + 9; /** @brief COLNAME column TREEOID*/ -const int DICTOID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_DICT_BASE + 10; /** @brief DEFAULTVAL column DICTOID*/ -const int LISTOID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_DICT_BASE + 11; /** @brief DEFAULTVAL column LISTOID*/ -const int TREEOID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_DICT_BASE + 12; /** @brief DEFAULTVAL column TREEOID*/ -const int DICTOID_SYSCOLUMN_MINVALUE = SYSCOLUMN_DICT_BASE + 13; /** @brief MINVAL column DICTOID*/ -const int LISTOID_SYSCOLUMN_MINVALUE = SYSCOLUMN_DICT_BASE + 14; /** @brief MINVAL column LISTOID*/ -const int TREEOID_SYSCOLUMN_MINVALUE = SYSCOLUMN_DICT_BASE + 15; /** @brief MINVAL column TREEOID*/ -const int DICTOID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_DICT_BASE + 16; /** @brief MAXVAL column DICTOID*/ -const int LISTOID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_DICT_BASE + 17; /** @brief MAXVAL column LISTOID*/ -const int TREEOID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_DICT_BASE + 18; /** @brief MAXVAL column TREEOID*/ -const int SYSCOLUMN_DICT_MAX = SYSCOLUMN_DICT_BASE + 19; // be sure this is one more than the highest # + * SYSCOLUMN columns dictionary OID definition + ******************************************************/ +const int DICTOID_SYSCOLUMN_SCHEMA = SYSCOLUMN_DICT_BASE + 1; /** @brief SCHEMA column DICTOID*/ +const int LISTOID_SYSCOLUMN_SCHEMA = SYSCOLUMN_DICT_BASE + 2; /** @brief SCHEMA column LISTOID*/ +const int TREEOID_SYSCOLUMN_SCHEMA = SYSCOLUMN_DICT_BASE + 3; /** @brief SCHEMA column TREEOID*/ +const int DICTOID_SYSCOLUMN_TABLENAME = SYSCOLUMN_DICT_BASE + 4; /** @brief TABLENAME column DICTOID*/ +const int LISTOID_SYSCOLUMN_TABLENAME = SYSCOLUMN_DICT_BASE + 5; /** @brief TABLENAME column LISTOID*/ +const int TREEOID_SYSCOLUMN_TABLENAME = SYSCOLUMN_DICT_BASE + 6; /** @brief TABLENAME column TREEOID*/ +const int DICTOID_SYSCOLUMN_COLNAME = SYSCOLUMN_DICT_BASE + 7; /** @brief COLNAME column DICTOID*/ +const int LISTOID_SYSCOLUMN_COLNAME = SYSCOLUMN_DICT_BASE + 8; /** @brief COLNAME column LISTOID*/ +const int TREEOID_SYSCOLUMN_COLNAME = SYSCOLUMN_DICT_BASE + 9; /** @brief COLNAME column TREEOID*/ +const int DICTOID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_DICT_BASE + 10; /** @brief DEFAULTVAL column DICTOID*/ +const int LISTOID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_DICT_BASE + 11; /** @brief DEFAULTVAL column LISTOID*/ +const int TREEOID_SYSCOLUMN_DEFAULTVAL = SYSCOLUMN_DICT_BASE + 12; /** @brief DEFAULTVAL column TREEOID*/ +const int DICTOID_SYSCOLUMN_MINVALUE = SYSCOLUMN_DICT_BASE + 13; /** @brief MINVAL column DICTOID*/ +const int LISTOID_SYSCOLUMN_MINVALUE = SYSCOLUMN_DICT_BASE + 14; /** @brief MINVAL column LISTOID*/ +const int TREEOID_SYSCOLUMN_MINVALUE = SYSCOLUMN_DICT_BASE + 15; /** @brief MINVAL column TREEOID*/ +const int DICTOID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_DICT_BASE + 16; /** @brief MAXVAL column DICTOID*/ +const int LISTOID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_DICT_BASE + 17; /** @brief MAXVAL column LISTOID*/ +const int TREEOID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_DICT_BASE + 18; /** @brief MAXVAL column TREEOID*/ +const int SYSCOLUMN_DICT_MAX = SYSCOLUMN_DICT_BASE + 19; // be sure this is one more than the highest # std::vector getAllSysCatOIDs(); /** -* stream operator -*/ + * stream operator + */ std::ostream& operator<<(std::ostream& os, const CalpontSystemCatalog::ColType& rhs); const std::string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt); bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatalog::ColType& b); -} //namespace execplan +} // namespace execplan -#endif //EXECPLAN_CALPONTSYSTEMCATALOG_H +#endif // EXECPLAN_CALPONTSYSTEMCATALOG_H // vim:ts=4 sw=4: - diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index d960f0e26..7616f90ed 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /********************************************************************* -* $Id: clientrotator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: clientrotator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -51,195 +51,203 @@ using namespace logging; /** Debug macro */ #ifdef INFINIDB_DEBUG -#define IDEBUG(x) {x;} +#define IDEBUG(x) \ + { \ + x; \ + } #else -#define IDEBUG(x) {} +#define IDEBUG(x) \ + { \ + } #endif #define LOG_TO_CERR namespace execplan { - const string LOCAL_EXEMGR_IP = "127.0.0.1"; const uint64_t LOCAL_EXEMGR_PORT = 8601; string ClientRotator::getModule() { - //Log to debug.log - LoggingID logid( 24, 0, 0); + // Log to debug.log + LoggingID logid(24, 0, 0); - string fileName = "/var/lib/columnstore/local/module"; + string fileName = "/var/lib/columnstore/local/module"; - string module; - ifstream moduleFile (fileName.c_str()); + string module; + ifstream moduleFile(fileName.c_str()); - if (moduleFile.is_open()) - { - getline (moduleFile, module); - } - else - { - { - logging::Message::Args args1; - logging::Message msg(1); - std::ostringstream oss; - oss << "ClientRotator::getModule open status2 =" << strerror(errno); - args1.add(oss.str()); - args1.add(fileName); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - } - } + if (moduleFile.is_open()) + { + getline(moduleFile, module); + } + else + { + { + logging::Message::Args args1; + logging::Message msg(1); + std::ostringstream oss; + oss << "ClientRotator::getModule open status2 =" << strerror(errno); + args1.add(oss.str()); + args1.add(fileName); + msg.format(args1); + Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + } + } - moduleFile.close(); + moduleFile.close(); - return module; + return module; } ostream& operator<<(ostream& output, const ClientRotator& rhs) { - output << __FILE__ << rhs.fName << "\t" << rhs.fSessionId << endl; - return output; + output << __FILE__ << rhs.fName << "\t" << rhs.fSessionId << endl; + return output; } -ClientRotator::ClientRotator(uint32_t sid, const std::string& name, bool localQuery) : - fName(name), fSessionId(sid), fClient(0), fClients(), fCf(Config::makeConfig()), - fDebug(0), fLocalQuery(localQuery) +ClientRotator::ClientRotator(uint32_t sid, const std::string& name, bool localQuery) + : fName(name) + , fSessionId(sid) + , fClient(0) + , fClients() + , fCf(Config::makeConfig()) + , fDebug(0) + , fLocalQuery(localQuery) { - if (! fCf) - throw runtime_error((string)__FILE__ + ": No configuration file"); + if (!fCf) + throw runtime_error((string)__FILE__ + ": No configuration file"); - fDebug = static_cast(Config::fromText(fCf->getConfig("CalpontConnector", "DebugLevel"))); + fDebug = static_cast(Config::fromText(fCf->getConfig("CalpontConnector", "DebugLevel"))); } void ClientRotator::loadClients() { - //This object is statically allocated somewhere. We need to reload the config file here - // to search the extproc env for changes made after libcalora.so is loaded. - fCf = Config::makeConfig(); + // This object is statically allocated somewhere. We need to reload the config file here + // to search the extproc env for changes made after libcalora.so is loaded. + fCf = Config::makeConfig(); - string pmWithUMStr = fCf->getConfig("Installation", "PMwithUM"); - bool pmWithUM = (pmWithUMStr == "y" || pmWithUMStr == "Y"); + string pmWithUMStr = fCf->getConfig("Installation", "PMwithUM"); + bool pmWithUM = (pmWithUMStr == "y" || pmWithUMStr == "Y"); - // check current module type - if (!fLocalQuery && pmWithUM) + // check current module type + if (!fLocalQuery && pmWithUM) + { + string module = getModule(); + + if (!module.empty() && (module[0] == 'P' || module[0] == 'p')) + fLocalQuery = true; + } + + // connect to loopback ExeMgr for local query set up + if (fLocalQuery) + { + fClient = new MessageQueueClient(LOCAL_EXEMGR_IP, LOCAL_EXEMGR_PORT); + return; + } + + stringstream ss(fName); + size_t pos = fName.length(); + string str; + int i = 1; + + do + { + ss.seekp(pos); + ss << i++; + str = fCf->getConfig(ss.str(), "Port"); + + if (str.length()) { - string module = getModule(); + string moduleStr = fCf->getConfig(ss.str(), "Module"); - if (!module.empty() && (module[0] == 'P' || module[0] == 'p')) - fLocalQuery = true; + // "if the system is not running in a 'PM with UM' config, the module type is unspecified, or the + // module is specified as a UM, use it" + if (!pmWithUM || moduleStr.empty() || moduleStr[0] == 'u' || moduleStr[0] == 'U') + fClients.push_back(ss.str()); } + } while (str.length()); - // connect to loopback ExeMgr for local query set up - if (fLocalQuery) - { - fClient = new MessageQueueClient(LOCAL_EXEMGR_IP, LOCAL_EXEMGR_PORT); - return; - } - - stringstream ss(fName); - size_t pos = fName.length(); - string str; - int i = 1; - - do - { - ss.seekp(pos); - ss << i++; - str = fCf->getConfig(ss.str(), "Port"); - - if (str.length() ) - { - string moduleStr = fCf->getConfig(ss.str(), "Module"); - - // "if the system is not running in a 'PM with UM' config, the module type is unspecified, or the - // module is specified as a UM, use it" - if (!pmWithUM || moduleStr.empty() || moduleStr[0] == 'u' || moduleStr[0] == 'U') - fClients.push_back(ss.str()); - } - } - while ( str.length() ); - - if (fClients.empty()) - throw runtime_error((string)__FILE__ + ": No configuration tags for " + fName + "\n"); + if (fClients.empty()) + throw runtime_error((string)__FILE__ + ": No configuration tags for " + fName + "\n"); } void ClientRotator::resetClient() { - try //one more time... - { - delete fClient; - fClient = 0; - connectList(); - //fClient->write(msg); - } - catch (std::exception& e) - { - /* Can't fail silently */ - writeToLog(__LINE__, e.what(), true); + try // one more time... + { + delete fClient; + fClient = 0; + connectList(); + // fClient->write(msg); + } + catch (std::exception& e) + { + /* Can't fail silently */ + writeToLog(__LINE__, e.what(), true); #ifdef LOG_TO_CERR - cerr << "ClientRotator::write() failed: " << e.what() << endl; + cerr << "ClientRotator::write() failed: " << e.what() << endl; #endif - throw; - } + throw; + } } void ClientRotator::write(const ByteStream& msg) { - if (!fClient) - connect(); + if (!fClient) + connect(); - try - { - fClient->write(msg); - return; - } - catch (std::exception& e) - { - resetClient(); - string errmsg = "ClientRotator caught exception: " + string(e.what()); - cout << errmsg << endl; - throw runtime_error(errmsg); - } - catch (...) - { - resetClient(); - string errmsg = "ClientRotator caught unknown exception"; - cout << errmsg << endl; - throw runtime_error(errmsg); - } + try + { + fClient->write(msg); + return; + } + catch (std::exception& e) + { + resetClient(); + string errmsg = "ClientRotator caught exception: " + string(e.what()); + cout << errmsg << endl; + throw runtime_error(errmsg); + } + catch (...) + { + resetClient(); + string errmsg = "ClientRotator caught unknown exception"; + cout << errmsg << endl; + throw runtime_error(errmsg); + } } ByteStream ClientRotator::read() { - boost::mutex::scoped_lock lk(fClientLock); + boost::mutex::scoped_lock lk(fClientLock); - ByteStream bs; + ByteStream bs; - if (!fClient) - connect(); + if (!fClient) + connect(); - try - { - bs = fClient->read(); - return bs; - } - catch (std::exception& e) - { - resetClient(); - string errmsg = "ClientRotator caught exception: " + string(e.what()); - cout << errmsg << endl; - throw runtime_error(errmsg); - } - catch (...) - { - resetClient(); - string errmsg = "ClientRotator caught unknown exception"; - cout << errmsg << endl; - throw runtime_error(errmsg); - } + try + { + bs = fClient->read(); + return bs; + } + catch (std::exception& e) + { + resetClient(); + string errmsg = "ClientRotator caught exception: " + string(e.what()); + cout << errmsg << endl; + throw runtime_error(errmsg); + } + catch (...) + { + resetClient(); + string errmsg = "ClientRotator caught unknown exception"; + cout << errmsg << endl; + throw runtime_error(errmsg); + } #if 0 @@ -262,131 +270,133 @@ ByteStream ClientRotator::read() } #endif - return bs; + return bs; } void ClientRotator::connect(double timeout) { - if (fClient) return; + if (fClient) + return; - if (fClients.empty()) - loadClients(); + if (fClients.empty()) + loadClients(); - if (fClient) return; + if (fClient) + return; - size_t idx = fSessionId % fClients.size(); - bool connected = false; + size_t idx = fSessionId % fClients.size(); + bool connected = false; - try - { - connected = exeConnect(fClients.at(idx)); - } - catch (... ) - { - } - - if (!connected) - { - if (fLocalQuery) - loadClients(); - else - connectList(timeout); - } + try + { + connected = exeConnect(fClients.at(idx)); + } + catch (...) + { + } + if (!connected) + { + if (fLocalQuery) + loadClients(); + else + connectList(timeout); + } } -bool ClientRotator::exeConnect( const string& clientName ) +bool ClientRotator::exeConnect(const string& clientName) { - fClient = new messageqcpp::MessageQueueClient( clientName, fCf); + fClient = new messageqcpp::MessageQueueClient(clientName, fCf); - if (fDebug > 12) - { - stringstream ss; - ss << fSessionId; + if (fDebug > 12) + { + stringstream ss; + ss << fSessionId; #ifdef LOG_TO_CERR - cerr << "Connecting to " << clientName << " with sessionId " << ss.str() << endl; + cerr << "Connecting to " << clientName << " with sessionId " << ss.str() << endl; #endif - writeToLog( __LINE__, "Connecting to " + clientName + " with sessionId " + ss.str(), 0); - } + writeToLog(__LINE__, "Connecting to " + clientName + " with sessionId " + ss.str(), 0); + } - try + try + { + if (!fClient->connect()) { - if (!fClient->connect()) - { - delete fClient; - fClient = 0; - return false; - } - } - catch (...) - { - delete fClient; - fClient = 0; - return false; + delete fClient; + fClient = 0; + return false; } + } + catch (...) + { + delete fClient; + fClient = 0; + return false; + } - return true; + return true; } void ClientRotator::connectList(double timeout) { - if (fClient) return; + if (fClient) + return; - if (fLocalQuery || fClients.empty()) - loadClients(); + if (fLocalQuery || fClients.empty()) + loadClients(); - if (fLocalQuery) return; + if (fLocalQuery) + return; - idbassert(!fClients.empty()); - uint16_t idx = fSessionId % fClients.size(); + idbassert(!fClients.empty()); + uint16_t idx = fSessionId % fClients.size(); - if (++idx >= fClients.size() ) - idx = 0; + if (++idx >= fClients.size()) + idx = 0; - typedef std::chrono::steady_clock clock; - auto start = clock::now(); + typedef std::chrono::steady_clock clock; + auto start = clock::now(); - typedef std::chrono::duration double_secs; - while (std::chrono::duration_cast(clock::now() - start).count() < timeout) + typedef std::chrono::duration double_secs; + while (std::chrono::duration_cast(clock::now() - start).count() < timeout) + { + try { - try - { - if (exeConnect(fClients.at(idx++))) - return; + if (exeConnect(fClients.at(idx++))) + return; - if (fClients.size() == idx) - idx = 0; - } - catch (...) - { - } + if (fClients.size() == idx) + idx = 0; } + catch (...) + { + } + } #ifdef LOG_TO_CERR - cerr << "Could not get a " << fName << " connection.\n"; + cerr << "Could not get a " << fName << " connection.\n"; #endif - writeToLog(__LINE__, "Could not get a " + fName + " connection.", 1); - throw runtime_error((string)__FILE__ + ": Could not get a connection to a " + fName); + writeToLog(__LINE__, "Could not get a " + fName + " connection.", 1); + throw runtime_error((string)__FILE__ + ": Could not get a connection to a " + fName); } void ClientRotator::writeToLog(int line, const string& msg, bool critical) const { - LoggingID lid(05); - MessageLog ml(lid); - Message::Args args; - Message m(0); - args.add(__FILE__); - args.add("@"); - args.add(line); - args.add(msg); - m.format(args); + LoggingID lid(05); + MessageLog ml(lid); + Message::Args args; + Message m(0); + args.add(__FILE__); + args.add("@"); + args.add(line); + args.add(msg); + m.format(args); - if (critical) - ml.logCriticalMessage(m); - else if (fDebug) - ml.logDebugMessage(m); + if (critical) + ml.logCriticalMessage(m); + else if (fDebug) + ml.logDebugMessage(m); } -} +} // namespace execplan // vim:ts=4 sw=4: - diff --git a/dbcon/execplan/clientrotator.h b/dbcon/execplan/clientrotator.h index 051605bb1..f0be4ff78 100644 --- a/dbcon/execplan/clientrotator.h +++ b/dbcon/execplan/clientrotator.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /********************************************************************* -* $Id: clientrotator.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: clientrotator.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ @@ -38,138 +38,134 @@ namespace execplan { - /** @brief connection handle structure */ class ClientRotator { -public: - /** @brief ctor - */ - ClientRotator(uint32_t sid, const std::string& name, bool localQuery = false); + public: + /** @brief ctor + */ + ClientRotator(uint32_t sid, const std::string& name, bool localQuery = false); - /** @brief dtor - */ - ~ClientRotator() + /** @brief dtor + */ + ~ClientRotator() + { + if (fClient) { - if (fClient) - { - fClient->shutdown(); - delete fClient; - } + fClient->shutdown(); + delete fClient; } + } - /** @brief connnect - * - * Try connecting to client based on session id. If no connection, - * try connectList. - * @param timeout in seconds. - */ - void connect(double timeout = 50); + /** @brief connnect + * + * Try connecting to client based on session id. If no connection, + * try connectList. + * @param timeout in seconds. + */ + void connect(double timeout = 50); - /** @brief write - * - * Write msg to fClient. If unsuccessful, get new connection with - * connectList and write. - */ - void write(const messageqcpp::ByteStream& msg); + /** @brief write + * + * Write msg to fClient. If unsuccessful, get new connection with + * connectList and write. + */ + void write(const messageqcpp::ByteStream& msg); - /** @brief shutdown - */ - void shutdown() + /** @brief shutdown + */ + void shutdown() + { + if (fClient) { - if (fClient) - { - fClient->shutdown(); - delete fClient; - fClient = 0; - } + fClient->shutdown(); + delete fClient; + fClient = 0; } + } - /** @brief read - */ - messageqcpp::ByteStream read(); + /** @brief read + */ + messageqcpp::ByteStream read(); - /** @brief getClient - */ - messageqcpp::MessageQueueClient* getClient() const - { - return fClient; - } + /** @brief getClient + */ + messageqcpp::MessageQueueClient* getClient() const + { + return fClient; + } - /** @brief getSessionId - */ - uint32_t getSessionId() const - { - return fSessionId; - } + /** @brief getSessionId + */ + uint32_t getSessionId() const + { + return fSessionId; + } - /** @brief setSessionId - */ - void setSessionId(uint32_t sid) - { - fSessionId = sid; - } + /** @brief setSessionId + */ + void setSessionId(uint32_t sid) + { + fSessionId = sid; + } - friend std::ostream& operator<<(std::ostream& output, const ClientRotator& rhs); + friend std::ostream& operator<<(std::ostream& output, const ClientRotator& rhs); - /** @brief reset fClient */ - void resetClient(); + /** @brief reset fClient */ + void resetClient(); - bool localQuery() - { - return fLocalQuery; - } - void localQuery(bool localQuery) - { - fLocalQuery = localQuery; - } - static std::string getModule(); + bool localQuery() + { + return fLocalQuery; + } + void localQuery(bool localQuery) + { + fLocalQuery = localQuery; + } + static std::string getModule(); -private: + private: + // Not copyable + ClientRotator(const ClientRotator&); + ClientRotator& operator=(const ClientRotator&); - //Not copyable - ClientRotator(const ClientRotator& ); - ClientRotator& operator=(const ClientRotator& ); + /** @brief load Clients + * + * Put all entries for client name tag from config file into client list + */ + void loadClients(); - /** @brief load Clients - * - * Put all entries for client name tag from config file into client list - */ - void loadClients(); + /** @brief execute connect + * + * Make connection and return success. + */ + bool exeConnect(const std::string& clientName); - /** @brief execute connect - * - * Make connection and return success. - */ - bool exeConnect(const std::string& clientName ); + /** @brief connnect to list + * + * Try connecting to next client on list + * until timeout lapses. Then throw exception. + */ + void connectList(double timeout = 0.005); - /** @brief connnect to list - * - * Try connecting to next client on list - * until timeout lapses. Then throw exception. - */ - void connectList(double timeout = 0.005); + /** @brief write to message log + * + * writes message with file name to debug or + * critical log. + */ + void writeToLog(int line, const std::string& msg, bool critical) const; - /** @brief write to message log - * - * writes message with file name to debug or - * critical log. - */ - void writeToLog(int line, const std::string& msg, bool critical) const; - - const std::string fName; - uint32_t fSessionId; - messageqcpp::MessageQueueClient* fClient; - typedef std::vector ClientList; - ClientList fClients; - config::Config* fCf; - int fDebug; - boost::mutex fClientLock; - bool fLocalQuery; + const std::string fName; + uint32_t fSessionId; + messageqcpp::MessageQueueClient* fClient; + typedef std::vector ClientList; + ClientList fClients; + config::Config* fCf; + int fDebug; + boost::mutex fClientLock; + bool fLocalQuery; }; - -} // namespace +} // namespace execplan #endif // vim:ts=4 sw=4: - diff --git a/dbcon/execplan/columnresult.h b/dbcon/execplan/columnresult.h index 9fa105836..401e71da7 100644 --- a/dbcon/execplan/columnresult.h +++ b/dbcon/execplan/columnresult.h @@ -31,87 +31,88 @@ namespace execplan { - /** @file - This serves as a bridge between the new and old joblist code. - ColumnResults are column data created from the TableBands - returned by the NJL. In getSysData, we convert TableBands - to NJLCR's to avoid having to rewrite all of the systemcatalog - code that used the old ColumnResult class. It only implements - the part of the ColumnResult API used by the system catalog. - Hack as necessary. + This serves as a bridge between the new and old joblist code. + ColumnResults are column data created from the TableBands + returned by the NJL. In getSysData, we convert TableBands + to NJLCR's to avoid having to rewrite all of the systemcatalog + code that used the old ColumnResult class. It only implements + the part of the ColumnResult API used by the system catalog. + Hack as necessary. */ class ColumnResult { -public: - ColumnResult() : oid(0), dcount(0) { } - // the other defaults are OK. + public: + ColumnResult() : oid(0), dcount(0) + { + } + // the other defaults are OK. - int64_t GetData(uint32_t index) const - { - return intData[index]; - } + int64_t GetData(uint32_t index) const + { + return intData[index]; + } - void PutData(int64_t d) - { - intData.push_back(d); - dcount++; - } + void PutData(int64_t d) + { + intData.push_back(d); + dcount++; + } - const std::string& GetStringData(uint32_t index) const - { - return stringData[index]; - } + const std::string& GetStringData(uint32_t index) const + { + return stringData[index]; + } - void PutStringData(const std::string& s) - { - stringData.push_back(s); - dcount++; - } + void PutStringData(const std::string& s) + { + stringData.push_back(s); + dcount++; + } - int ColumnOID() const - { - return oid; - } + int ColumnOID() const + { + return oid; + } - void SetColumnOID(int o) - { - oid = o; - } + void SetColumnOID(int o) + { + oid = o; + } - uint64_t GetRid(uint32_t index) const - { - return rids[index]; - } + uint64_t GetRid(uint32_t index) const + { + return rids[index]; + } - void PutRid(uint64_t rid) - { - rids.push_back(rid); - } + void PutRid(uint64_t rid) + { + rids.push_back(rid); + } - void PutRidOnly(uint64_t rid) - { - rids.push_back(rid); - dcount++; - } + void PutRidOnly(uint64_t rid) + { + rids.push_back(rid); + dcount++; + } - int dataCount() const - { - return dcount; - } + int dataCount() const + { + return dcount; + } -private: - //defaults okay - //ColumnResult(const ColumnResult& rhs); - //ColumnResult& operator=(const ColumnResult& rhs); + private: + // defaults okay + // ColumnResult(const ColumnResult& rhs); + // ColumnResult& operator=(const ColumnResult& rhs); - std::vector intData; - std::vector stringData; - std::vector rids; - int oid; - int dcount; // data, string, and row counters + std::vector intData; + std::vector stringData; + std::vector rids; + int oid; + int dcount; // data, string, and row counters }; -} +} // namespace execplan #endif diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index f40ccc344..65406b108 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: constantcolumn.cpp 9474 2013-05-02 15:28:09Z rdempsey $ -* -* -***********************************************************************/ + * $Id: constantcolumn.cpp 9474 2013-05-02 15:28:09Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -39,218 +39,173 @@ namespace execplan /** * Constructors/Destructors */ -ConstantColumn::ConstantColumn() : - ReturnedColumn(), - fType(0) +ConstantColumn::ConstantColumn() : ReturnedColumn(), fType(0) { } -ConstantColumn::ConstantColumn(const string& sql, TYPE type) : - ReturnedColumn(), - fConstval(sql), - fType(type), - fData(sql) +ConstantColumn::ConstantColumn(const string& sql, TYPE type) + : ReturnedColumn(), fConstval(sql), fType(type), fData(sql) { - fResult.strVal = sql; - - if (type == LITERAL && sql.length() < 9) - { - memcpy(tmp, sql.c_str(), sql.length()); - memset(tmp + sql.length(), 0, 8); - fResult.uintVal = uint64ToStr(*((uint64_t*) tmp)); - fResult.intVal = (int64_t)fResult.uintVal; - } - else - { - fResult.intVal = atoll(sql.c_str()); - fResult.uintVal = strtoull(sql.c_str(), NULL, 0); - } + fResult.strVal = sql; - fResult.floatVal = atof(sql.c_str()); - fResult.doubleVal = atof(sql.c_str()); - fResult.longDoubleVal = strtold(sql.c_str(), NULL); - - // decimal for constant should be constructed by the caller and call the decimal constructor - fResult.decimalVal = datatypes::Decimal(fResult.intVal, - 0, - 18, - (int128_t) fResult.intVal); - // @bug 3381, default null item to integer type. - if (fType == ConstantColumn::NULLDATA) - { - if (fResult.uintVal > (uint64_t)MAX_BIGINT) - { - fResultType.colDataType = CalpontSystemCatalog::UBIGINT; - } - else - { - fResultType.colDataType = CalpontSystemCatalog::BIGINT; - } - - fResultType.colWidth = 8; - } - else - { - fResultType.colDataType = CalpontSystemCatalog::VARCHAR; - fResultType.colWidth = sql.length(); - } -} - -ConstantColumn::ConstantColumn(const string& sql, const double val) : - ReturnedColumn(), - fConstval(sql), - fType(NUM), - fData(sql) -{ - fResult.strVal = sql; - fResult.doubleVal = val; - fResult.intVal = (int64_t)val; - fResult.uintVal = (uint64_t)val; - fResult.floatVal = (float)val; - fResult.longDoubleVal = val; - // decimal for constant should be constructed by the caller and call the decimal constructor - fResult.decimalVal = datatypes::Decimal(fResult.intVal, - 0, - 18, - (int128_t) fResult.intVal); - fResultType.colDataType = CalpontSystemCatalog::DOUBLE; - fResultType.colWidth = 8; -} - -ConstantColumn::ConstantColumn(const string& sql, const long double val) : - ReturnedColumn(), - fConstval(sql), - fType(NUM), - fData(sql) -{ - fResult.strVal = sql; - fResult.doubleVal = (double)val; - fResult.intVal = (int64_t)val; - fResult.uintVal = (uint64_t)val; - fResult.floatVal = (float)val; - fResult.longDoubleVal = val; - // decimal for constant should be constructed by the caller and call the decimal constructor - fResult.decimalVal = datatypes::Decimal(fResult.intVal, - 0, - 18, - (int128_t) fResult.intVal); - fResultType.colDataType = CalpontSystemCatalog::DOUBLE; - fResultType.colWidth = 8; -} - -ConstantColumn::ConstantColumn(const string& sql, const int64_t val, TYPE type) : - ReturnedColumn(), - fConstval(sql), - fType(type), - fData(sql) -{ - fResult.strVal = sql; - fResult.intVal = val; - fResult.uintVal = (uint64_t)fResult.intVal; - fResult.floatVal = (float)fResult.intVal; - fResult.doubleVal = (double)fResult.intVal; - fResult.longDoubleVal = (long double)fResult.intVal; - fResult.decimalVal = datatypes::Decimal(fResult.intVal, - 0, - 0, - (int128_t) fResult.intVal); - fResultType.colDataType = CalpontSystemCatalog::BIGINT; - fResultType.colWidth = 8; -} - -ConstantColumn::ConstantColumn(const string& sql, const uint64_t val, TYPE type) : - ReturnedColumn(), - fConstval(sql), - fType(type), - fData(sql) -{ - fResult.strVal = sql; - fResult.uintVal = val; + if (type == LITERAL && sql.length() < 9) + { + memcpy(tmp, sql.c_str(), sql.length()); + memset(tmp + sql.length(), 0, 8); + fResult.uintVal = uint64ToStr(*((uint64_t*)tmp)); fResult.intVal = (int64_t)fResult.uintVal; - fResult.floatVal = (float)fResult.uintVal; - fResult.doubleVal = (double)fResult.uintVal; - fResult.longDoubleVal = (long double)fResult.uintVal; - fResult.decimalVal = datatypes::Decimal(fResult.uintVal, - 0, - 0, - (int128_t) fResult.uintVal); - fResultType.colDataType = CalpontSystemCatalog::UBIGINT; - fResultType.colWidth = 8; -} - -ConstantColumn::ConstantColumn(const string& sql, const IDB_Decimal& val) : - ReturnedColumn(), - fConstval(sql), - fType(NUM), - fData(sql) -{ - fResult.strVal = sql; - fResult.intVal = (int64_t)atoll(sql.c_str()); + } + else + { + fResult.intVal = atoll(sql.c_str()); fResult.uintVal = strtoull(sql.c_str(), NULL, 0); - fResult.floatVal = atof(sql.c_str()); - fResult.doubleVal = atof(sql.c_str()); - fResult.longDoubleVal = strtold(sql.c_str(), NULL); - fResult.decimalVal = val; - fResultType.colDataType = CalpontSystemCatalog::DECIMAL; - fResultType.setDecimalScalePrecision(val.precision, val.scale); -} + } -ConstantColumn::ConstantColumn( const ConstantColumn& rhs): - ReturnedColumn(rhs), - fConstval (rhs.constval()), - fType (rhs.type()), - fData (rhs.data()), - fTimeZone (rhs.timeZone()) -{ - sequence(rhs.sequence()); - fAlias = rhs.alias(); - fResult = rhs.fResult; - fResultType = rhs.fResultType; -} + fResult.floatVal = atof(sql.c_str()); + fResult.doubleVal = atof(sql.c_str()); + fResult.longDoubleVal = strtold(sql.c_str(), NULL); + + // decimal for constant should be constructed by the caller and call the decimal constructor + fResult.decimalVal = datatypes::Decimal(fResult.intVal, 0, 18, (int128_t)fResult.intVal); + // @bug 3381, default null item to integer type. + if (fType == ConstantColumn::NULLDATA) + { + if (fResult.uintVal > (uint64_t)MAX_BIGINT) + { + fResultType.colDataType = CalpontSystemCatalog::UBIGINT; + } + else + { + fResultType.colDataType = CalpontSystemCatalog::BIGINT; + } -ConstantColumn::ConstantColumn(const int64_t val, TYPE type) : - ReturnedColumn(), - fType(type) -{ - ostringstream oss; - oss << val; - fConstval = oss.str(); - fData = oss.str(); - fResult.strVal = fData; - fResult.intVal = val; - fResult.uintVal = (uint64_t)fResult.intVal; - fResult.floatVal = (float)fResult.intVal; - fResult.doubleVal = (double)fResult.intVal; - fResult.longDoubleVal = (long double)fResult.intVal; - fResult.decimalVal = datatypes::Decimal(fResult.intVal, - 0, - 0, - (int128_t) fResult.intVal); - fResultType.colDataType = CalpontSystemCatalog::BIGINT; fResultType.colWidth = 8; + } + else + { + fResultType.colDataType = CalpontSystemCatalog::VARCHAR; + fResultType.colWidth = sql.length(); + } } -ConstantColumn::ConstantColumn(const uint64_t val, TYPE type, - int8_t scale, uint8_t precision) : - ReturnedColumn(), - fType(type) +ConstantColumn::ConstantColumn(const string& sql, const double val) + : ReturnedColumn(), fConstval(sql), fType(NUM), fData(sql) { - ostringstream oss; - oss << val; - fConstval = oss.str(); - fData = oss.str(); - fResult.strVal = fData; - fResult.intVal = (int64_t)val; - fResult.uintVal = val; - fResult.floatVal = (float)fResult.uintVal; - fResult.doubleVal = (double)fResult.uintVal; - fResult.longDoubleVal = (long double)fResult.uintVal; - fResult.decimalVal = datatypes::Decimal(fResult.uintVal, - scale, - precision, - (int128_t) fResult.uintVal); - fResultType.colDataType = CalpontSystemCatalog::UBIGINT; - fResultType.colWidth = 8; + fResult.strVal = sql; + fResult.doubleVal = val; + fResult.intVal = (int64_t)val; + fResult.uintVal = (uint64_t)val; + fResult.floatVal = (float)val; + fResult.longDoubleVal = val; + // decimal for constant should be constructed by the caller and call the decimal constructor + fResult.decimalVal = datatypes::Decimal(fResult.intVal, 0, 18, (int128_t)fResult.intVal); + fResultType.colDataType = CalpontSystemCatalog::DOUBLE; + fResultType.colWidth = 8; +} + +ConstantColumn::ConstantColumn(const string& sql, const long double val) + : ReturnedColumn(), fConstval(sql), fType(NUM), fData(sql) +{ + fResult.strVal = sql; + fResult.doubleVal = (double)val; + fResult.intVal = (int64_t)val; + fResult.uintVal = (uint64_t)val; + fResult.floatVal = (float)val; + fResult.longDoubleVal = val; + // decimal for constant should be constructed by the caller and call the decimal constructor + fResult.decimalVal = datatypes::Decimal(fResult.intVal, 0, 18, (int128_t)fResult.intVal); + fResultType.colDataType = CalpontSystemCatalog::DOUBLE; + fResultType.colWidth = 8; +} + +ConstantColumn::ConstantColumn(const string& sql, const int64_t val, TYPE type) + : ReturnedColumn(), fConstval(sql), fType(type), fData(sql) +{ + fResult.strVal = sql; + fResult.intVal = val; + fResult.uintVal = (uint64_t)fResult.intVal; + fResult.floatVal = (float)fResult.intVal; + fResult.doubleVal = (double)fResult.intVal; + fResult.longDoubleVal = (long double)fResult.intVal; + fResult.decimalVal = datatypes::Decimal(fResult.intVal, 0, 0, (int128_t)fResult.intVal); + fResultType.colDataType = CalpontSystemCatalog::BIGINT; + fResultType.colWidth = 8; +} + +ConstantColumn::ConstantColumn(const string& sql, const uint64_t val, TYPE type) + : ReturnedColumn(), fConstval(sql), fType(type), fData(sql) +{ + fResult.strVal = sql; + fResult.uintVal = val; + fResult.intVal = (int64_t)fResult.uintVal; + fResult.floatVal = (float)fResult.uintVal; + fResult.doubleVal = (double)fResult.uintVal; + fResult.longDoubleVal = (long double)fResult.uintVal; + fResult.decimalVal = datatypes::Decimal(fResult.uintVal, 0, 0, (int128_t)fResult.uintVal); + fResultType.colDataType = CalpontSystemCatalog::UBIGINT; + fResultType.colWidth = 8; +} + +ConstantColumn::ConstantColumn(const string& sql, const IDB_Decimal& val) + : ReturnedColumn(), fConstval(sql), fType(NUM), fData(sql) +{ + fResult.strVal = sql; + fResult.intVal = (int64_t)atoll(sql.c_str()); + fResult.uintVal = strtoull(sql.c_str(), NULL, 0); + fResult.floatVal = atof(sql.c_str()); + fResult.doubleVal = atof(sql.c_str()); + fResult.longDoubleVal = strtold(sql.c_str(), NULL); + fResult.decimalVal = val; + fResultType.colDataType = CalpontSystemCatalog::DECIMAL; + fResultType.setDecimalScalePrecision(val.precision, val.scale); +} + +ConstantColumn::ConstantColumn(const ConstantColumn& rhs) + : ReturnedColumn(rhs) + , fConstval(rhs.constval()) + , fType(rhs.type()) + , fData(rhs.data()) + , fTimeZone(rhs.timeZone()) +{ + sequence(rhs.sequence()); + fAlias = rhs.alias(); + fResult = rhs.fResult; + fResultType = rhs.fResultType; +} + +ConstantColumn::ConstantColumn(const int64_t val, TYPE type) : ReturnedColumn(), fType(type) +{ + ostringstream oss; + oss << val; + fConstval = oss.str(); + fData = oss.str(); + fResult.strVal = fData; + fResult.intVal = val; + fResult.uintVal = (uint64_t)fResult.intVal; + fResult.floatVal = (float)fResult.intVal; + fResult.doubleVal = (double)fResult.intVal; + fResult.longDoubleVal = (long double)fResult.intVal; + fResult.decimalVal = datatypes::Decimal(fResult.intVal, 0, 0, (int128_t)fResult.intVal); + fResultType.colDataType = CalpontSystemCatalog::BIGINT; + fResultType.colWidth = 8; +} + +ConstantColumn::ConstantColumn(const uint64_t val, TYPE type, int8_t scale, uint8_t precision) + : ReturnedColumn(), fType(type) +{ + ostringstream oss; + oss << val; + fConstval = oss.str(); + fData = oss.str(); + fResult.strVal = fData; + fResult.intVal = (int64_t)val; + fResult.uintVal = val; + fResult.floatVal = (float)fResult.uintVal; + fResult.doubleVal = (double)fResult.uintVal; + fResult.longDoubleVal = (long double)fResult.uintVal; + fResult.decimalVal = datatypes::Decimal(fResult.uintVal, scale, precision, (int128_t)fResult.uintVal); + fResultType.colDataType = CalpontSystemCatalog::UBIGINT; + fResultType.colWidth = 8; } ConstantColumn::~ConstantColumn() @@ -259,136 +214,134 @@ ConstantColumn::~ConstantColumn() const string ConstantColumn::toString() const { - ostringstream oss; - oss << "ConstantColumn: " << fConstval << " intVal=" << fResult.intVal << " uintVal=" << fResult.uintVal; - oss << '('; + ostringstream oss; + oss << "ConstantColumn: " << fConstval << " intVal=" << fResult.intVal << " uintVal=" << fResult.uintVal; + oss << '('; - if (fType == LITERAL) - oss << 'l'; - else if (fType == NUM) - oss << 'n'; - else - oss << "null"; + if (fType == LITERAL) + oss << 'l'; + else if (fType == NUM) + oss << 'n'; + else + oss << "null"; - oss << ')'; - oss << " resultType=" << colDataTypeToString(fResultType.colDataType); + oss << ')'; + oss << " resultType=" << colDataTypeToString(fResultType.colDataType); - if (fAlias.length() > 0) oss << "/Alias: " << fAlias; + if (fAlias.length() > 0) + oss << "/Alias: " << fAlias; - return oss.str(); + return oss.str(); } const string ConstantColumn::data() const { - return fData; + return fData; } ostream& operator<<(ostream& output, const ConstantColumn& rhs) { - output << rhs.toString(); + output << rhs.toString(); - return output; + return output; } void ConstantColumn::serialize(messageqcpp::ByteStream& b) const { - - b << (ObjectReader::id_t) ObjectReader::CONSTANTCOLUMN; - ReturnedColumn::serialize(b); - b << fConstval; - b << (uint32_t) fType; - //b << fAlias; - b << fData; - b << fTimeZone; - b << static_cast(fReturnAll); - b << (uint64_t)fResult.intVal; - b << fResult.uintVal; - b << fResult.doubleVal; - b << fResult.longDoubleVal; - b << fResult.floatVal; - b << (uint8_t)fResult.boolVal; - b << fResult.strVal; - b << (uint64_t)fResult.decimalVal.value; - b << fResult.decimalVal.s128Value; - b << (uint8_t)fResult.decimalVal.scale; - b << (uint8_t)fResult.decimalVal.precision; + b << (ObjectReader::id_t)ObjectReader::CONSTANTCOLUMN; + ReturnedColumn::serialize(b); + b << fConstval; + b << (uint32_t)fType; + // b << fAlias; + b << fData; + b << fTimeZone; + b << static_cast(fReturnAll); + b << (uint64_t)fResult.intVal; + b << fResult.uintVal; + b << fResult.doubleVal; + b << fResult.longDoubleVal; + b << fResult.floatVal; + b << (uint8_t)fResult.boolVal; + b << fResult.strVal; + b << (uint64_t)fResult.decimalVal.value; + b << fResult.decimalVal.s128Value; + b << (uint8_t)fResult.decimalVal.scale; + b << (uint8_t)fResult.decimalVal.precision; } void ConstantColumn::unserialize(messageqcpp::ByteStream& b) { + ObjectReader::checkType(b, ObjectReader::CONSTANTCOLUMN); + ReturnedColumn::unserialize(b); + // uint64_t val; - ObjectReader::checkType(b, ObjectReader::CONSTANTCOLUMN); - ReturnedColumn::unserialize(b); - //uint64_t val; - - b >> fConstval; - b >> (uint32_t&) fType; - b >> fData; - b >> fTimeZone; - b >> reinterpret_cast< ByteStream::doublebyte&>(fReturnAll); - b >> (uint64_t&)fResult.intVal; - b >> fResult.uintVal; - b >> fResult.doubleVal; - b >> fResult.longDoubleVal; - b >> fResult.floatVal; - b >> (uint8_t&)fResult.boolVal; - b >> fResult.strVal; - b >> (uint64_t&)fResult.decimalVal.value; - b >> fResult.decimalVal.s128Value; - b >> (uint8_t&)fResult.decimalVal.scale; - b >> (uint8_t&)fResult.decimalVal.precision; + b >> fConstval; + b >> (uint32_t&)fType; + b >> fData; + b >> fTimeZone; + b >> reinterpret_cast(fReturnAll); + b >> (uint64_t&)fResult.intVal; + b >> fResult.uintVal; + b >> fResult.doubleVal; + b >> fResult.longDoubleVal; + b >> fResult.floatVal; + b >> (uint8_t&)fResult.boolVal; + b >> fResult.strVal; + b >> (uint64_t&)fResult.decimalVal.value; + b >> fResult.decimalVal.s128Value; + b >> (uint8_t&)fResult.decimalVal.scale; + b >> (uint8_t&)fResult.decimalVal.precision; } bool ConstantColumn::operator==(const ConstantColumn& t) const { - const ReturnedColumn* rc1, *rc2; + const ReturnedColumn *rc1, *rc2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (fConstval != t.fConstval) - return false; + if (fConstval != t.fConstval) + return false; - if (fType != t.fType) - return false; + if (fType != t.fType) + return false; + if (fData != t.fData) + return false; - if (fData != t.fData) - return false; + if (fReturnAll != t.fReturnAll) + return false; - if (fReturnAll != t.fReturnAll) - return false; + if (fTimeZone != t.fTimeZone) + return false; - if (fTimeZone != t.fTimeZone) - return false; - - return true; + return true; } bool ConstantColumn::operator==(const TreeNode* t) const { - const ConstantColumn* o; + const ConstantColumn* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool ConstantColumn::operator!=(const ConstantColumn& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool ConstantColumn::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } -} +} // namespace execplan // vim:ts=4 sw=4: diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index 33a02fdd2..badfa4789 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: constantcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: constantcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef CONSTANTCOLUMN_H @@ -49,414 +49,400 @@ class ConstantColumn; */ class ConstantColumn : public ReturnedColumn { -public: + public: + enum TYPE + { + NUM, + LITERAL, + NULLDATA + }; - enum TYPE - { - NUM, - LITERAL, - NULLDATA - }; + /** + * ctor + */ + ConstantColumn(); + /** + * ctor + */ + ConstantColumn(const std::string& sql, TYPE type = LITERAL); + /** + * ctor + */ + ConstantColumn(const int64_t val, TYPE type = NUM); // deprecate + /** + * ctor + */ + ConstantColumn(const uint64_t val, TYPE type = NUM, int8_t scale = 0, uint8_t precision = 0); // deprecate + // There are more ctors below... - /** - * ctor - */ - ConstantColumn(); - /** - * ctor - */ - ConstantColumn(const std::string& sql, TYPE type = LITERAL); - /** - * ctor - */ - ConstantColumn(const int64_t val, TYPE type = NUM); //deprecate - /** - * ctor - */ - ConstantColumn(const uint64_t val, TYPE type = NUM, - int8_t scale = 0, uint8_t precision = 0); // deprecate - //There are more ctors below... + /** + * dtor + */ + virtual ~ConstantColumn(); - /** - * dtor - */ - virtual ~ConstantColumn(); + /* + * Accessor Methods + */ + /** + * accessor + */ + inline unsigned int type() const + { + return fType; + } + /** + * accessor + */ + inline void type(unsigned int type) + { + fType = type; + } + /** + * accessor + */ + inline const std::string& constval() const + { + return fConstval; + } + /** + * accessor + */ + inline void constval(const std::string& constval) + { + fConstval = constval; + } + /** + * accessor + */ + inline const std::string& timeZone() const + { + return fTimeZone; + } + /** + * mutator + */ + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + /** + * accessor + */ + virtual const std::string data() const; + /** + * accessor + */ + virtual void data(const std::string data) + { + fData = data; + } + /** + * accessor + */ + virtual const std::string toString() const; - /* - * Accessor Methods - */ - /** - * accessor - */ - inline unsigned int type() const - { - return fType; - } - /** - * accessor - */ - inline void type (unsigned int type) - { - fType = type; - } - /** - * accessor - */ - inline const std::string& constval() const - { - return fConstval; - } - /** - * accessor - */ - inline void constval(const std::string& constval) - { - fConstval = constval; - } - /** - * accessor - */ - inline const std::string& timeZone() const - { - return fTimeZone; - } - /** - * mutator - */ - inline void timeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } - /** - * accessor - */ - virtual const std::string data() const; - /** - * accessor - */ - virtual void data(const std::string data) - { - fData = data; - } - /** - * accessor - */ - virtual const std::string toString() const; + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual ConstantColumn* clone() const + { + return new ConstantColumn(*this); + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual ConstantColumn* clone() const + /* + * The serialization interface + */ + /** + * serialize + */ + virtual void serialize(messageqcpp::ByteStream&) const; + /** + * unserialize + */ + virtual void unserialize(messageqcpp::ByteStream&); + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const ConstantColumn& t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const ConstantColumn& t) const; + + virtual bool hasWindowFunc() + { + return false; + } + + /** Constant column on the filte can always be moved into derived table */ + virtual void setDerivedTable() + { + fDerivedTable = "*"; + } + + private: + std::string fConstval; + int fType; + std::string fData; + std::string fTimeZone; + + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + /** + * ctor + */ + ConstantColumn(const std::string& sql, const double val); + /** + * ctor + */ + ConstantColumn(const std::string& sql, const long double val); + /** + * ctor + */ + ConstantColumn(const std::string& sql, const int64_t val, TYPE type = NUM); + /** + * ctor + */ + ConstantColumn(const std::string& sql, const uint64_t val, TYPE type = NUM); + /** + * ctor + */ + ConstantColumn(const std::string& sql, const IDB_Decimal& val); + /** + * copy ctor + */ + ConstantColumn(const ConstantColumn& rhs); + /** + * F&E + */ + using ReturnedColumn::evaluate; + virtual void evaluate(rowgroup::Row& row) + { + } + /** + * F&E + */ + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return TreeNode::getBoolVal(); + } + /** + * F&E + */ + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return fResult.strVal; + } + /** + * F&E + */ + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return fResult.intVal; + } + /** + * F&E + */ + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return fResult.uintVal; + } + /** + * F&E + */ + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return fResult.floatVal; + } + /** + * F&E + */ + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return fResult.doubleVal; + } + /** + * F&E + */ + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + return fResult.decimalVal; + } + /** + * F&E + */ + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) { - return new ConstantColumn (*this); + fResult.intVal = dataconvert::DataConvert::stringToDate(fResult.strVal); + fResult.valueConverted = true; } - /* - * The serialization interface - */ - /** - * serialize - */ - virtual void serialize(messageqcpp::ByteStream&) const; - /** - * unserialize - */ - virtual void unserialize(messageqcpp::ByteStream&); + return fResult.intVal; + } + /** + * F&E + */ + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const ConstantColumn& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const ConstantColumn& t) const; - - virtual bool hasWindowFunc() + if (!fResult.valueConverted) { - return false; + fResult.intVal = dataconvert::DataConvert::stringToDatetime(fResult.strVal); + fResult.valueConverted = true; } - /** Constant column on the filte can always be moved into derived table */ - virtual void setDerivedTable() + return fResult.intVal; + } + /** + * F&E + */ + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) { - fDerivedTable = "*"; + fResult.intVal = dataconvert::DataConvert::stringToTimestamp(fResult.strVal, fTimeZone); + fResult.valueConverted = true; } -private: - std::string fConstval; - int fType; - std::string fData; - std::string fTimeZone; + return fResult.intVal; + } + /** + * F&E + */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - /** - * ctor - */ - ConstantColumn(const std::string& sql, const double val); - /** - * ctor - */ - ConstantColumn(const std::string& sql, const long double val); - /** - * ctor - */ - ConstantColumn(const std::string& sql, const int64_t val, TYPE type = NUM); - /** - * ctor - */ - ConstantColumn(const std::string& sql, const uint64_t val, TYPE type = NUM); - /** - * ctor - */ - ConstantColumn(const std::string& sql, const IDB_Decimal& val); - /** - * copy ctor - */ - ConstantColumn(const ConstantColumn& rhs); - /** - * F&E - */ - using ReturnedColumn::evaluate; - virtual void evaluate(rowgroup::Row& row) {} - /** - * F&E - */ - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + if (!fResult.valueConverted) { - isNull = isNull || (fType == NULLDATA); - return TreeNode::getBoolVal(); - } - /** - * F&E - */ - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - return fResult.strVal; - } - /** - * F&E - */ - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - return fResult.intVal; - } - /** - * F&E - */ - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - return fResult.uintVal; - } - /** - * F&E - */ - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - return fResult.floatVal; - } - /** - * F&E - */ - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - return fResult.doubleVal; - } - /** - * F&E - */ - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - return fResult.decimalVal; - } - /** - * F&E - */ - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - - if (!fResult.valueConverted) - { - fResult.intVal = dataconvert::DataConvert::stringToDate(fResult.strVal); - fResult.valueConverted = true; - } - - return fResult.intVal; - } - /** - * F&E - */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - - if (!fResult.valueConverted) - { - fResult.intVal = dataconvert::DataConvert::stringToDatetime(fResult.strVal); - fResult.valueConverted = true; - } - - return fResult.intVal; - } - /** - * F&E - */ - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - - if (!fResult.valueConverted) - { - fResult.intVal = dataconvert::DataConvert::stringToTimestamp(fResult.strVal, fTimeZone); - fResult.valueConverted = true; - } - - return fResult.intVal; - } - /** - * F&E - */ - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - isNull = isNull || (fType == NULLDATA); - - if (!fResult.valueConverted) - { - fResult.intVal = dataconvert::DataConvert::stringToTime(fResult.strVal); - fResult.valueConverted = true; - } - - return fResult.intVal; - } - /** - * F&E - */ - inline float getFloatVal() const - { - return fResult.floatVal; - } - /** - * F&E - */ - inline double getDoubleVal() const - { - return fResult.doubleVal; + fResult.intVal = dataconvert::DataConvert::stringToTime(fResult.strVal); + fResult.valueConverted = true; } + return fResult.intVal; + } + /** + * F&E + */ + inline float getFloatVal() const + { + return fResult.floatVal; + } + /** + * F&E + */ + inline double getDoubleVal() const + { + return fResult.doubleVal; + } }; - -class ConstantColumnNull: public ConstantColumn +class ConstantColumnNull : public ConstantColumn { -public: - ConstantColumnNull() - :ConstantColumn("", ConstantColumn::NULLDATA) - { } + public: + ConstantColumnNull() : ConstantColumn("", ConstantColumn::NULLDATA) + { + } }; - -class ConstantColumnString: public ConstantColumn +class ConstantColumnString : public ConstantColumn { -public: - ConstantColumnString(const std::string &str) - :ConstantColumn(str, ConstantColumn::LITERAL) - { } + public: + ConstantColumnString(const std::string& str) : ConstantColumn(str, ConstantColumn::LITERAL) + { + } }; - -class ConstantColumnNum: public ConstantColumn +class ConstantColumnNum : public ConstantColumn { -public: - ConstantColumnNum(const CalpontSystemCatalog::ColType &type, - const std::string &str) - :ConstantColumn(str, ConstantColumn::NUM) - { - resultType(type); - } + public: + ConstantColumnNum(const CalpontSystemCatalog::ColType& type, const std::string& str) + : ConstantColumn(str, ConstantColumn::NUM) + { + resultType(type); + } }; - -class ConstantColumnUInt: public ConstantColumn +class ConstantColumnUInt : public ConstantColumn { -public: - ConstantColumnUInt(uint64_t val, int8_t scale, uint8_t precision) - :ConstantColumn(val, ConstantColumn::NUM, scale, precision) - { } + public: + ConstantColumnUInt(uint64_t val, int8_t scale, uint8_t precision) + : ConstantColumn(val, ConstantColumn::NUM, scale, precision) + { + } }; - -class ConstantColumnSInt: public ConstantColumn +class ConstantColumnSInt : public ConstantColumn { -public: - ConstantColumnSInt(const CalpontSystemCatalog::ColType &type, - const std::string &str, int64_t val) - :ConstantColumn(str, val, ConstantColumn::NUM) - { - resultType(type); - } + public: + ConstantColumnSInt(const CalpontSystemCatalog::ColType& type, const std::string& str, int64_t val) + : ConstantColumn(str, val, ConstantColumn::NUM) + { + resultType(type); + } }; - -class ConstantColumnReal: public ConstantColumn +class ConstantColumnReal : public ConstantColumn { -public: - ConstantColumnReal(const CalpontSystemCatalog::ColType &type, - const std::string &str, - double val) - :ConstantColumn(str, val) - { - resultType(type); - } + public: + ConstantColumnReal(const CalpontSystemCatalog::ColType& type, const std::string& str, double val) + : ConstantColumn(str, val) + { + resultType(type); + } }; - -class ConstantColumnTemporal: public ConstantColumn +class ConstantColumnTemporal : public ConstantColumn { -public: - ConstantColumnTemporal(const CalpontSystemCatalog::ColType &type, - const std::string &str) - :ConstantColumn(str) - { - resultType(type); - } + public: + ConstantColumnTemporal(const CalpontSystemCatalog::ColType& type, const std::string& str) + : ConstantColumn(str) + { + resultType(type); + } }; - /** * ostream operator */ std::ostream& operator<<(std::ostream& output, const ConstantColumn& rhs); -} -#endif //CONSTANTCOLUMN_H - +} // namespace execplan +#endif // CONSTANTCOLUMN_H diff --git a/dbcon/execplan/constantfilter.cpp b/dbcon/execplan/constantfilter.cpp index 179e9ad34..4f837d0ad 100644 --- a/dbcon/execplan/constantfilter.cpp +++ b/dbcon/execplan/constantfilter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: constantfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: constantfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -39,64 +39,59 @@ using namespace std; namespace { -template struct deleter : public unary_function +template +struct deleter : public unary_function { - void operator()(T& x) - { - delete x; - } + void operator()(T& x) + { + delete x; + } }; -} +} // namespace namespace execplan { - /** * Constructors/Destructors */ ConstantFilter::ConstantFilter() -{} +{ +} ConstantFilter::ConstantFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs) { - SSFP ssfp(new SimpleFilter(op, lhs, rhs)); - fFilterList.push_back(ssfp); - SimpleColumn* sc = dynamic_cast(lhs); - fCol.reset(sc->clone()); + SSFP ssfp(new SimpleFilter(op, lhs, rhs)); + fFilterList.push_back(ssfp); + SimpleColumn* sc = dynamic_cast(lhs); + fCol.reset(sc->clone()); } ConstantFilter::ConstantFilter(SimpleFilter* sf) { - SSFP ssfp(sf); - fFilterList.push_back(ssfp); - const SimpleColumn* sc = dynamic_cast(sf->lhs()); - fCol.reset(sc->clone()); + SSFP ssfp(sf); + fFilterList.push_back(ssfp); + const SimpleColumn* sc = dynamic_cast(sf->lhs()); + fCol.reset(sc->clone()); } -ConstantFilter::ConstantFilter(const ConstantFilter& rhs): - Filter(rhs), - fOp(rhs.fOp), - fCol(rhs.fCol) +ConstantFilter::ConstantFilter(const ConstantFilter& rhs) : Filter(rhs), fOp(rhs.fOp), fCol(rhs.fCol) { - fFilterList.clear(); - fSimpleColumnList.clear(); - fWindowFunctionColumnList.clear(); - SSFP ssfp; + fFilterList.clear(); + fSimpleColumnList.clear(); + fWindowFunctionColumnList.clear(); + SSFP ssfp; - for (uint32_t i = 0; i < rhs.fFilterList.size(); i++) - { - ssfp.reset(rhs.fFilterList[i]->clone()); - fFilterList.push_back(ssfp); - fSimpleColumnList.insert(fSimpleColumnList.end(), - ssfp->simpleColumnList().begin(), - ssfp->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), - ssfp->aggColumnList().begin(), - ssfp->aggColumnList().end()); - fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), - ssfp->windowfunctionColumnList().begin(), - ssfp->windowfunctionColumnList().end()); - } + for (uint32_t i = 0; i < rhs.fFilterList.size(); i++) + { + ssfp.reset(rhs.fFilterList[i]->clone()); + fFilterList.push_back(ssfp); + fSimpleColumnList.insert(fSimpleColumnList.end(), ssfp->simpleColumnList().begin(), + ssfp->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), ssfp->aggColumnList().begin(), ssfp->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), + ssfp->windowfunctionColumnList().begin(), + ssfp->windowfunctionColumnList().end()); + } } ConstantFilter::~ConstantFilter() @@ -109,212 +104,208 @@ ConstantFilter::~ConstantFilter() const string ConstantFilter::toString() const { - ostringstream output; - output << "ConstantFilter" << endl; + ostringstream output; + output << "ConstantFilter" << endl; - if (fOp) output << " " << *fOp << endl; + if (fOp) + output << " " << *fOp << endl; - if (!fFunctionName.empty()) output << " Func: " << fFunctionName << endl; + if (!fFunctionName.empty()) + output << " Func: " << fFunctionName << endl; - if (fCol) output << " " << *fCol << endl; + if (fCol) + output << " " << *fCol << endl; - for (unsigned int i = 0; i < fFilterList.size(); i++) - output << " " << *fFilterList[i] << endl; + for (unsigned int i = 0; i < fFilterList.size(); i++) + output << " " << *fFilterList[i] << endl; - return output.str(); + return output.str(); } ostream& operator<<(ostream& output, const ConstantFilter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } bool ConstantFilter::hasAggregate() { - if (fAggColumnList.empty()) + if (fAggColumnList.empty()) + { + for (uint32_t i = 0; i < fFilterList.size(); i++) { - for (uint32_t i = 0; i < fFilterList.size(); i++) - { - if (fFilterList[i]->hasAggregate()) - { - fAggColumnList.insert(fAggColumnList.end(), - fFilterList[i]->aggColumnList().begin(), - fFilterList[i]->aggColumnList().end()); - } - } + if (fFilterList[i]->hasAggregate()) + { + fAggColumnList.insert(fAggColumnList.end(), fFilterList[i]->aggColumnList().begin(), + fFilterList[i]->aggColumnList().end()); + } } + } - if (!fAggColumnList.empty()) - { - return true; - } + if (!fAggColumnList.empty()) + { + return true; + } - return false; + return false; } void ConstantFilter::serialize(messageqcpp::ByteStream& b) const { - FilterList::const_iterator it; - b << static_cast(ObjectReader::CONSTANTFILTER); - Filter::serialize(b); + FilterList::const_iterator it; + b << static_cast(ObjectReader::CONSTANTFILTER); + Filter::serialize(b); - if (fOp != NULL) - fOp->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fOp != NULL) + fOp->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - if (fCol != NULL) - fCol->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fCol != NULL) + fCol->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - b << static_cast(fFilterList.size()); + b << static_cast(fFilterList.size()); - for (it = fFilterList.begin(); it != fFilterList.end(); it++) - (*it)->serialize(b); + for (it = fFilterList.begin(); it != fFilterList.end(); it++) + (*it)->serialize(b); - b << fFunctionName; + b << fFunctionName; } void ConstantFilter::unserialize(messageqcpp::ByteStream& b) { - uint32_t size, i; - ObjectReader::checkType(b, ObjectReader::CONSTANTFILTER); - SimpleFilter* sf; + uint32_t size, i; + ObjectReader::checkType(b, ObjectReader::CONSTANTFILTER); + SimpleFilter* sf; - Filter::unserialize(b); - fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fCol.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - b >> size; - fFilterList.clear(); - fSimpleColumnList.clear(); - fAggColumnList.clear(); - fWindowFunctionColumnList.clear(); + Filter::unserialize(b); + fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fCol.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + b >> size; + fFilterList.clear(); + fSimpleColumnList.clear(); + fAggColumnList.clear(); + fWindowFunctionColumnList.clear(); - for (i = 0; i < size; i++) - { - sf = dynamic_cast(ObjectReader::createTreeNode(b)); - SSFP ssfp(sf); - fFilterList.push_back(ssfp); - fSimpleColumnList.insert(fSimpleColumnList.end(), - ssfp->simpleColumnList().begin(), - ssfp->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), - ssfp->aggColumnList().begin(), - ssfp->aggColumnList().end()); - fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), - ssfp->windowfunctionColumnList().begin(), - ssfp->windowfunctionColumnList().end()); - } + for (i = 0; i < size; i++) + { + sf = dynamic_cast(ObjectReader::createTreeNode(b)); + SSFP ssfp(sf); + fFilterList.push_back(ssfp); + fSimpleColumnList.insert(fSimpleColumnList.end(), ssfp->simpleColumnList().begin(), + ssfp->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), ssfp->aggColumnList().begin(), ssfp->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), + ssfp->windowfunctionColumnList().begin(), + ssfp->windowfunctionColumnList().end()); + } - b >> fFunctionName; + b >> fFunctionName; } bool ConstantFilter::operator==(const ConstantFilter& t) const { - const Filter* f1, *f2; - FilterList::const_iterator it, it2; + const Filter *f1, *f2; + FilterList::const_iterator it, it2; - f1 = static_cast(this); - f2 = static_cast(&t); + f1 = static_cast(this); + f2 = static_cast(&t); - if (*f1 != *f2) - return false; + if (*f1 != *f2) + return false; - if (fOp != NULL) - { - if (*fOp != *t.fOp) - return false; - } - else if (t.fOp != NULL) - return false; + if (fOp != NULL) + { + if (*fOp != *t.fOp) + return false; + } + else if (t.fOp != NULL) + return false; - //fFilterList - if (fFilterList.size() != t.fFilterList.size()) - return false; + // fFilterList + if (fFilterList.size() != t.fFilterList.size()) + return false; - for (it = fFilterList.begin(), it2 = t.fFilterList.begin(); - it != fFilterList.end(); ++it, ++it2) - { - if (**it != **it2) - return false; - } + for (it = fFilterList.begin(), it2 = t.fFilterList.begin(); it != fFilterList.end(); ++it, ++it2) + { + if (**it != **it2) + return false; + } - return true; + return true; } bool ConstantFilter::operator==(const TreeNode* t) const { - const ConstantFilter* o; + const ConstantFilter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool ConstantFilter::operator!=(const ConstantFilter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool ConstantFilter::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } void ConstantFilter::setDerivedTable() { - if (fCol->hasAggregate()) - { - fDerivedTable = ""; - return; - } - for (unsigned i = 0; i < fFilterList.size(); i++) - { - fFilterList[i]->setDerivedTable(); - } + if (fCol->hasAggregate()) + { + fDerivedTable = ""; + return; + } + for (unsigned i = 0; i < fFilterList.size(); i++) + { + fFilterList[i]->setDerivedTable(); + } - if (!fFilterList.empty()) - { - fDerivedTable = fFilterList[0]->derivedTable(); - } - else - { - fDerivedTable = ""; - } + if (!fFilterList.empty()) + { + fDerivedTable = fFilterList[0]->derivedTable(); + } + else + { + fDerivedTable = ""; + } } void ConstantFilter::replaceRealCol(std::vector& derivedColList) { - ReturnedColumn* tmp = derivedColList[fCol->colPosition()]->clone(); - fCol.reset(tmp); + ReturnedColumn* tmp = derivedColList[fCol->colPosition()]->clone(); + fCol.reset(tmp); - for (unsigned i = 0; i < fFilterList.size(); i++) - fFilterList[i]->replaceRealCol(derivedColList); + for (unsigned i = 0; i < fFilterList.size(); i++) + fFilterList[i]->replaceRealCol(derivedColList); } const std::vector& ConstantFilter::simpleColumnList() { - return fSimpleColumnList; + return fSimpleColumnList; } void ConstantFilter::setSimpleColumnList() { - fSimpleColumnList.clear(); + fSimpleColumnList.clear(); - for (uint32_t i = 0; i < fFilterList.size(); i++) - { - fFilterList[i]->setSimpleColumnList(); - fSimpleColumnList.insert(fSimpleColumnList.end(), - fFilterList[i]->simpleColumnList().begin(), - fFilterList[i]->simpleColumnList().end()); - } + for (uint32_t i = 0; i < fFilterList.size(); i++) + { + fFilterList[i]->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), fFilterList[i]->simpleColumnList().begin(), + fFilterList[i]->simpleColumnList().end()); + } } - -} // namespace execplan +} // namespace execplan // vim:ts=4 sw=4: diff --git a/dbcon/execplan/constantfilter.h b/dbcon/execplan/constantfilter.h index b0cdaf4cb..45e012c71 100644 --- a/dbcon/execplan/constantfilter.h +++ b/dbcon/execplan/constantfilter.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: constantfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: constantfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef CONSTANTFILTER_H @@ -38,7 +38,6 @@ */ namespace execplan { - class ReturnedColumn; class AggregateColumn; class WindowFunctionColumn; @@ -55,199 +54,197 @@ class WindowFunctionColumn; */ class ConstantFilter : public Filter { + public: + /** + * Types and constants + */ + typedef std::vector FilterList; -public: - /** - * Types and constants - */ - typedef std::vector FilterList; + /** + * Constructors + */ + ConstantFilter(); + ConstantFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs); + ConstantFilter(SimpleFilter* sf); - /** - * Constructors - */ - ConstantFilter(); - ConstantFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs); - ConstantFilter(SimpleFilter* sf); + // not needed yet + // ConstantFilter(const ConstantFilter& rhs); - //not needed yet - //ConstantFilter(const ConstantFilter& rhs); + /** + * Destructors + */ + virtual ~ConstantFilter(); - /** - * Destructors - */ - virtual ~ConstantFilter(); + /** + * Accessor Methods + */ + const FilterList& filterList() const + { + return fFilterList; + } + void filterList(const FilterList& filterList) + { + fFilterList = filterList; + } + const SOP& op() const + { + return fOp; + } + void op(const SOP& op) + { + fOp = op; + } + const SRCP& col() const + { + return fCol; + } + void col(const SRCP& col) + { + fCol = col; + } - /** - * Accessor Methods - */ - const FilterList& filterList() const - { - return fFilterList; - } - void filterList( const FilterList& filterList) - { - fFilterList = filterList; - } - const SOP& op() const - { - return fOp; - } - void op(const SOP& op) - { - fOp = op; - } - const SRCP& col() const - { - return fCol; - } - void col(const SRCP& col) - { - fCol = col; - } + /** + * Operations + */ + void pushFilter(SimpleFilter* sf) + { + SSFP ssfp(sf); + fFilterList.push_back(ssfp); + } - /** - * Operations - */ - void pushFilter (SimpleFilter* sf) - { - SSFP ssfp(sf); - fFilterList.push_back(ssfp); - } + // virtual const std::string data() const; + virtual const std::string toString() const; - //virtual const std::string data() const; - virtual const std::string toString() const; + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const ConstantFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const ConstantFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const ConstantFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const ConstantFilter& t) const; + /** @brief test if this filter can be combined with the argument filter + * This is for operation combine optimization + * @param f the filter that this fiter tries to combine with + * @param op the operator that connects the two filters. if one or both of the + * two filters is constantFilter, need to make sure operator is consistant. + * @return a filter(constantfilter) if successfully combined. otherwise + * return NULL + */ + // Used by Oracle frontend, deprecated now. + // virtual Filter* combinable(Filter* f, Operator* op); - /** @brief test if this filter can be combined with the argument filter - * This is for operation combine optimization - * @param f the filter that this fiter tries to combine with - * @param op the operator that connects the two filters. if one or both of the - * two filters is constantFilter, need to make sure operator is consistant. - * @return a filter(constantfilter) if successfully combined. otherwise - * return NULL - */ - // Used by Oracle frontend, deprecated now. - //virtual Filter* combinable(Filter* f, Operator* op); + /** get function name + * + * get the function name for this function column + */ + std::string functionName() const + { + return fFunctionName; + } - /** get function name - * - * get the function name for this function column - */ - std::string functionName() const - { - return fFunctionName; - } + /** set function name + * + * set the function name for this function column + */ + void functionName(const std::string& functionName) + { + fFunctionName = functionName; + } - /** set function name - * - * set the function name for this function column - */ - void functionName(const std::string& functionName) - { - fFunctionName = functionName; - } + void setDerivedTable(); + virtual void replaceRealCol(std::vector&); + virtual bool hasAggregate(); - void setDerivedTable(); - virtual void replaceRealCol(std::vector&); - virtual bool hasAggregate(); + private: + SOP fOp; /// connect operator (and or) + FilterList fFilterList; /// vector of simple filters + SRCP fCol; /// the common column + std::string fFunctionName; /// function name -private: - SOP fOp; /// connect operator (and or) - FilterList fFilterList; /// vector of simple filters - SRCP fCol; /// the common column - std::string fFunctionName; /// function name + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + ConstantFilter(const ConstantFilter& rhs); + inline virtual ConstantFilter* clone() const + { + return new ConstantFilter(*this); + } - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - ConstantFilter(const ConstantFilter& rhs); - inline virtual ConstantFilter* clone() const - { - return new ConstantFilter (*this); - } + inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull); - inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull); + // get all simple columns involved in this column + const std::vector& simpleColumnList(); + // walk through the constant filter operands to re-populate fSimpleColumnList + void setSimpleColumnList(); - // get all simple columns involved in this column - const std::vector& simpleColumnList(); - // walk through the constant filter operands to re-populate fSimpleColumnList - void setSimpleColumnList(); + // get all aggregate columns involved in this column + const std::vector& aggColumnList() const + { + return fAggColumnList; + } - // get all aggregate columns involved in this column - const std::vector& aggColumnList() const - { - return fAggColumnList; - } - -private: - std::vector fSimpleColumnList; - std::vector fAggColumnList; - std::vector fWindowFunctionColumnList; + private: + std::vector fSimpleColumnList; + std::vector fAggColumnList; + std::vector fWindowFunctionColumnList; }; inline bool ConstantFilter::getBoolVal(rowgroup::Row& row, bool& isNull) { - switch (fOp->op()) + switch (fOp->op()) + { + case OP_AND: + for (uint32_t i = 0; i < fFilterList.size(); i++) + if (!fFilterList[i]->getBoolVal(row, isNull)) + return false; + + return true; + + case OP_OR: + for (uint32_t i = 0; i < fFilterList.size(); i++) + if (fFilterList[i]->getBoolVal(row, isNull)) + return true; + + return false; + + default: { - case OP_AND: - for (uint32_t i = 0; i < fFilterList.size(); i++) - if (!fFilterList[i]->getBoolVal(row, isNull)) - return false; - - return true; - - case OP_OR: - for (uint32_t i = 0; i < fFilterList.size(); i++) - if (fFilterList[i]->getBoolVal(row, isNull)) - return true; - - return false; - - default: - { - std::ostringstream oss; - oss << "ConstantFilter:: Non support logic operation: " << fOp->op(); - throw logging::InvalidOperationExcept(oss.str()); - } + std::ostringstream oss; + oss << "ConstantFilter:: Non support logic operation: " << fOp->op(); + throw logging::InvalidOperationExcept(oss.str()); } + } } std::ostream& operator<<(std::ostream& output, const ConstantFilter& rhs); -} -#endif //CONSTANTFILTER_H - +} // namespace execplan +#endif // CONSTANTFILTER_H diff --git a/dbcon/execplan/existsfilter.cpp b/dbcon/execplan/existsfilter.cpp index 137e75b98..faf1eaad7 100644 --- a/dbcon/execplan/existsfilter.cpp +++ b/dbcon/execplan/existsfilter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: existsfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: existsfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include using namespace std; @@ -34,112 +34,104 @@ namespace execplan /** * Constructors/Destructors */ -ExistsFilter::ExistsFilter(): - fNotExists (false), - fCorrelated (false), - fData("Exists Filter") -{} +ExistsFilter::ExistsFilter() : fNotExists(false), fCorrelated(false), fData("Exists Filter") +{ +} -ExistsFilter::ExistsFilter( const SCSEP& sub, - const bool existFlag, - const bool correlated): - fSub (sub), - fNotExists (existFlag), - fCorrelated (correlated), - fData ("Exists Filter") -{} +ExistsFilter::ExistsFilter(const SCSEP& sub, const bool existFlag, const bool correlated) + : fSub(sub), fNotExists(existFlag), fCorrelated(correlated), fData("Exists Filter") +{ +} -ExistsFilter::ExistsFilter(const ExistsFilter& rhs): - Filter(rhs), - fSub (rhs.fSub), - fNotExists (rhs.fNotExists), - fCorrelated (rhs.fCorrelated), - fData (rhs.fData) -{} +ExistsFilter::ExistsFilter(const ExistsFilter& rhs) + : Filter(rhs), fSub(rhs.fSub), fNotExists(rhs.fNotExists), fCorrelated(rhs.fCorrelated), fData(rhs.fData) +{ +} ExistsFilter::~ExistsFilter() -{} +{ +} const string ExistsFilter::toString() const { - ostringstream oss; - oss << "ExistsFilter " << "correlated=" << fCorrelated << " notExists=" << fNotExists << endl; - oss << *(fSub.get()); - return oss.str(); + ostringstream oss; + oss << "ExistsFilter " + << "correlated=" << fCorrelated << " notExists=" << fNotExists << endl; + oss << *(fSub.get()); + return oss.str(); } ostream& operator<<(ostream& output, const ExistsFilter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void ExistsFilter::serialize(messageqcpp::ByteStream& b) const { - b << static_cast(ObjectReader::EXISTSFILTER); - Filter::serialize(b); + b << static_cast(ObjectReader::EXISTSFILTER); + Filter::serialize(b); - if (fSub.get() != NULL) - fSub->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fSub.get() != NULL) + fSub->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - b << static_cast(fNotExists); - b << static_cast(fCorrelated); + b << static_cast(fNotExists); + b << static_cast(fCorrelated); } void ExistsFilter::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::EXISTSFILTER); - Filter::unserialize(b); - fSub.reset(dynamic_cast(ObjectReader::createExecutionPlan(b))); - b >> reinterpret_cast< ByteStream::doublebyte&>(fNotExists); - b >> reinterpret_cast< ByteStream::doublebyte&>(fCorrelated); + ObjectReader::checkType(b, ObjectReader::EXISTSFILTER); + Filter::unserialize(b); + fSub.reset(dynamic_cast(ObjectReader::createExecutionPlan(b))); + b >> reinterpret_cast(fNotExists); + b >> reinterpret_cast(fCorrelated); } bool ExistsFilter::operator==(const ExistsFilter& t) const { - const Filter* f1, *f2; + const Filter *f1, *f2; - f1 = static_cast(this); - f2 = static_cast(&t); + f1 = static_cast(this); + f2 = static_cast(&t); - if (*f1 != *f2) - return false; + if (*f1 != *f2) + return false; - if (*(fSub.get()) != t.fSub.get()) - return false; + if (*(fSub.get()) != t.fSub.get()) + return false; - if (fNotExists != t.fNotExists) - return false; + if (fNotExists != t.fNotExists) + return false; - if (fCorrelated != t.fCorrelated) - return false; + if (fCorrelated != t.fCorrelated) + return false; - return true; + return true; } bool ExistsFilter::operator==(const TreeNode* t) const { - const ExistsFilter* o; + const ExistsFilter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool ExistsFilter::operator!=(const ExistsFilter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool ExistsFilter::operator!=(const TreeNode* t) const { - return (!(*this == t)); -} - + return (!(*this == t)); } +} // namespace execplan diff --git a/dbcon/execplan/existsfilter.h b/dbcon/execplan/existsfilter.h index 1a907a32b..f3150e648 100644 --- a/dbcon/execplan/existsfilter.h +++ b/dbcon/execplan/existsfilter.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: existsfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: existsfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef EXISTSFILTER_H @@ -42,109 +42,108 @@ namespace execplan */ class ExistsFilter : public Filter { - /** - * Public stuff - */ -public: - /** - * Constructors - */ - ExistsFilter(); - ExistsFilter( const SCSEP& sub, const bool existsFlag = false, const bool correlated = false); - ExistsFilter(const ExistsFilter& rhs); - virtual ~ExistsFilter(); + /** + * Public stuff + */ + public: + /** + * Constructors + */ + ExistsFilter(); + ExistsFilter(const SCSEP& sub, const bool existsFlag = false, const bool correlated = false); + ExistsFilter(const ExistsFilter& rhs); + virtual ~ExistsFilter(); - /** - * Accessor Methods - */ - const SCSEP& sub() const - { - return fSub; - } - void sub (SCSEP& sub) - { - fSub = sub; - } + /** + * Accessor Methods + */ + const SCSEP& sub() const + { + return fSub; + } + void sub(SCSEP& sub) + { + fSub = sub; + } - bool notExists() const - { - return fNotExists; - } - void notExists (const bool notExists) - { - fNotExists = notExists; - } + bool notExists() const + { + return fNotExists; + } + void notExists(const bool notExists) + { + fNotExists = notExists; + } - bool correlated() const - { - return fCorrelated; - } - void correlated(const bool correlated) - { - fCorrelated = correlated; - } + bool correlated() const + { + return fCorrelated; + } + void correlated(const bool correlated) + { + fCorrelated = correlated; + } - /** - * Overloaded stream operator - */ - //virtual std::ostream& operator<< (std::ostream& output); - virtual const std::string toString() const; + /** + * Overloaded stream operator + */ + // virtual std::ostream& operator<< (std::ostream& output); + virtual const std::string toString() const; - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual ExistsFilter* clone() const - { - return new ExistsFilter (*this); - } + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual ExistsFilter* clone() const + { + return new ExistsFilter(*this); + } - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const ExistsFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const ExistsFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const ExistsFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const ExistsFilter& t) const; -private: - //default okay? - //ExistsFilter& operator=(const ExistsFilter& rhs); + private: + // default okay? + // ExistsFilter& operator=(const ExistsFilter& rhs); - SCSEP fSub; - bool fNotExists; /// if this is not exist? - bool fCorrelated; - std::string fData; + SCSEP fSub; + bool fNotExists; /// if this is not exist? + bool fCorrelated; + std::string fData; }; std::ostream& operator<<(std::ostream& output, const ExistsFilter& rhs); -} -#endif //EXISTSFILTER_H - +} // namespace execplan +#endif // EXISTSFILTER_H diff --git a/dbcon/execplan/exp_templates.h b/dbcon/execplan/exp_templates.h index ebc935146..e9bfd7fbd 100644 --- a/dbcon/execplan/exp_templates.h +++ b/dbcon/execplan/exp_templates.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: exp_templates.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: exp_templates.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef EXP_TEMPLATES @@ -36,641 +36,630 @@ namespace expression { -enum precedence { none, lower, equal, higher }; +enum precedence +{ + none, + lower, + equal, + higher +}; -struct not_an_acceptor_tag {}; -struct acceptor_tag {}; -struct accumulating_acceptor_tag {}; +struct not_an_acceptor_tag +{ +}; +struct acceptor_tag +{ +}; +struct accumulating_acceptor_tag +{ +}; -template +template struct acceptor_traits { - typedef typename Acceptor::acceptor_category acceptor_category; - typedef typename Acceptor::input_symbol_type input_symbol_type; - typedef typename Acceptor::result_type result_type; + typedef typename Acceptor::acceptor_category acceptor_category; + typedef typename Acceptor::input_symbol_type input_symbol_type; + typedef typename Acceptor::result_type result_type; }; // Constants indicating possible operator placements. An incoming operator // may be ambiguous so its position will be the bitwise or of several of // these positions. -const int prefix = 0x01; -const int postfix = 0x02; -const int infix = 0x04; -const int open = 0x08; -const int close = 0x10; -const int function_open = 0x20; +const int prefix = 0x01; +const int postfix = 0x02; +const int infix = 0x04; +const int open = 0x08; +const int close = 0x10; +const int function_open = 0x20; const int function_close = close; -enum associativity { non_associative, left_associative, right_associative }; +enum associativity +{ + non_associative, + left_associative, + right_associative +}; struct default_expression_parser_error_policy { - template - void invalid_operand_position(Operand) - { - std::cerr << "Invalid operand position\n"; - } + template + void invalid_operand_position(Operand) + { + std::cerr << "Invalid operand position\n"; + } - template - void invalid_operator_position(Operator) - { - std::cerr << "Invalid operator position\n"; - } + template + void invalid_operator_position(Operator) + { + std::cerr << "Invalid operator position\n"; + } - template - void requires_precedence_relation(Operator, Operator) - { - std::cerr << "Requires precedence relation\n"; - } + template + void requires_precedence_relation(Operator, Operator) + { + std::cerr << "Requires precedence relation\n"; + } - template - void requires_associativity_relation(Operator, Operator) - { - std::cerr << "Requires associativity relation\n"; - } + template + void requires_associativity_relation(Operator, Operator) + { + std::cerr << "Requires associativity relation\n"; + } - template - void ambiguity(Operator, Operator) - { - std::cerr << "Unresolvable ambiguity\n"; - } + template + void ambiguity(Operator, Operator) + { + std::cerr << "Unresolvable ambiguity\n"; + } - template - void unbalanced_confix(Operator) - { - std::cerr << "Unbalanced confix operator\n"; - } + template + void unbalanced_confix(Operator) + { + std::cerr << "Unbalanced confix operator\n"; + } - template - void missing_operand(Operator) - { - std::cerr << "Missing operand\n"; - } + template + void missing_operand(Operator) + { + std::cerr << "Missing operand\n"; + } }; -#define EXP_TEMPLATE \ - template +#define EXP_TEMPLATE \ + template -#define EXP_ACCEPTOR \ - expression_acceptor +#define EXP_ACCEPTOR expression_acceptor namespace detail { -enum action {shift, reduce, prec, prec_assoc, invalid}; - -// Current larser state -enum state { accepting, rejected, pre_operand, post_operand, lookahead }; - -// The operator placements that are allowed for each state -const int pre_positions = prefix | open; -const int post_positions = postfix | close | infix | function_open; - -const action parse_action[6][6] = +enum action { - /* prefix postfix infix open close func open */ - /* prefix */ { shift, prec, prec, shift, reduce, prec }, - /* postfix */ { invalid, reduce, reduce, reduce, reduce, reduce }, - /* infix */ { shift, prec, prec_assoc, shift, reduce, prec }, - /* open */ { shift, shift, shift, shift, shift, shift }, - /* close */ { reduce, reduce, reduce, reduce, reduce, reduce }, - /* func open */ { shift, shift, shift, shift, shift, shift } + shift, + reduce, + prec, + prec_assoc, + invalid }; -template < - typename Token, - typename Operand, - typename Operator, - typename Policy, - typename OperandStack = std::stack, - typename OperatorStack = std::stack - > +// Current larser state +enum state +{ + accepting, + rejected, + pre_operand, + post_operand, + lookahead +}; + +// The operator placements that are allowed for each state +const int pre_positions = prefix | open; +const int post_positions = postfix | close | infix | function_open; + +const action parse_action[6][6] = { + /* prefix postfix infix open close func open */ + /* prefix */ {shift, prec, prec, shift, reduce, prec}, + /* postfix */ {invalid, reduce, reduce, reduce, reduce, reduce}, + /* infix */ {shift, prec, prec_assoc, shift, reduce, prec}, + /* open */ {shift, shift, shift, shift, shift, shift}, + /* close */ {reduce, reduce, reduce, reduce, reduce, reduce}, + /* func open */ {shift, shift, shift, shift, shift, shift}}; + +template , typename OperatorStack = std::stack > class expression_acceptor { -private: - class assignment_proxy - { - public: - explicit assignment_proxy(expression_acceptor* acceptor) : - m_expression_acceptor(acceptor) {} - - assignment_proxy& operator=(const Token& t) - { - m_expression_acceptor->disambiguate_and_parse(t); - return *this; - } - - private: - expression_acceptor* m_expression_acceptor; - }; - - friend class assignment_proxy; - -public: - typedef accumulating_acceptor_tag acceptor_category; - typedef Token input_symbol_type; - typedef Operand result_type; - typedef std::output_iterator_tag iterator_category; - typedef Token value_type; - typedef Token& reference; - typedef Token* pointer; - typedef int difference_type; - - explicit expression_acceptor(const Policy& policy, - OperandStack& operandStack, - OperatorStack& operatorStack) : - m_policy(policy), - m_operand_stack (operandStack), - m_operator_stack (operatorStack), - m_state(detail::pre_operand) + private: + class assignment_proxy + { + public: + explicit assignment_proxy(expression_acceptor* acceptor) : m_expression_acceptor(acceptor) { } - assignment_proxy operator*() + assignment_proxy& operator=(const Token& t) { - return assignment_proxy(this); - } - expression_acceptor& operator++() - { - return *this; - } - expression_acceptor& operator++(int) - { - return *this; + m_expression_acceptor->disambiguate_and_parse(t); + return *this; } - bool accepted(); - bool trapped() - { - return m_state == rejected; - } + private: + expression_acceptor* m_expression_acceptor; + }; - operator Operand() - { - return this->accepted() ? m_operand_stack.top() : Operand(); - } + friend class assignment_proxy; -private: - void disambiguate_and_parse(const Token&); - int operator_type_index(int); - void parse_operator(const Operator&); - void do_reduce(); + public: + typedef accumulating_acceptor_tag acceptor_category; + typedef Token input_symbol_type; + typedef Operand result_type; + typedef std::output_iterator_tag iterator_category; + typedef Token value_type; + typedef Token& reference; + typedef Token* pointer; + typedef int difference_type; -private: - Policy m_policy; - OperandStack& m_operand_stack; // changed to reference by Zhixuan Zhu - OperatorStack& m_operator_stack; // changed to reference by Zhixuan Zhu - state m_state; - Token m_ambiguous_operator; + explicit expression_acceptor(const Policy& policy, OperandStack& operandStack, OperatorStack& operatorStack) + : m_policy(policy) + , m_operand_stack(operandStack) + , m_operator_stack(operatorStack) + , m_state(detail::pre_operand) + { + } + + assignment_proxy operator*() + { + return assignment_proxy(this); + } + expression_acceptor& operator++() + { + return *this; + } + expression_acceptor& operator++(int) + { + return *this; + } + + bool accepted(); + bool trapped() + { + return m_state == rejected; + } + + operator Operand() + { + return this->accepted() ? m_operand_stack.top() : Operand(); + } + + private: + void disambiguate_and_parse(const Token&); + int operator_type_index(int); + void parse_operator(const Operator&); + void do_reduce(); + + private: + Policy m_policy; + OperandStack& m_operand_stack; // changed to reference by Zhixuan Zhu + OperatorStack& m_operator_stack; // changed to reference by Zhixuan Zhu + state m_state; + Token m_ambiguous_operator; }; EXP_TEMPLATE bool EXP_ACCEPTOR::accepted() { - if (m_state == accepting) - { - return true; - } - else if (m_state == rejected) - { - return false; - } - else if (m_state == lookahead) - { - m_state = post_operand; - int position = m_policy.positions(m_ambiguous_operator); - parse_operator(m_policy.as_operator(m_ambiguous_operator, - position & (postfix | close))); - } + if (m_state == accepting) + { + return true; + } + else if (m_state == rejected) + { + return false; + } + else if (m_state == lookahead) + { + m_state = post_operand; + int position = m_policy.positions(m_ambiguous_operator); + parse_operator(m_policy.as_operator(m_ambiguous_operator, position & (postfix | close))); + } - while (!m_operator_stack.empty() && m_state != rejected) - { - do_reduce(); - } + while (!m_operator_stack.empty() && m_state != rejected) + { + do_reduce(); + } - if (m_state == rejected) - { - return false; - } - else if (m_operand_stack.size() > 1 || !m_operator_stack.empty()) - { - m_state = rejected; - return false; - } - else - { - m_state = accepting; - return true; - } + if (m_state == rejected) + { + return false; + } + else if (m_operand_stack.size() > 1 || !m_operator_stack.empty()) + { + m_state = rejected; + return false; + } + else + { + m_state = accepting; + return true; + } } EXP_TEMPLATE void EXP_ACCEPTOR::disambiguate_and_parse(const Token& t) { - assert(m_state != accepting); + assert(m_state != accepting); - if (m_state == rejected) - return; + if (m_state == rejected) + return; - if (m_policy.is_operator(t)) + if (m_policy.is_operator(t)) + { + int operator_positions = m_policy.positions(t); + + if (m_state == lookahead) { - int operator_positions = m_policy.positions(t); + bool could_be_pre = ((operator_positions & pre_positions) != 0); + bool could_be_post = ((operator_positions & post_positions) != 0); - if (m_state == lookahead) - { - bool could_be_pre = ((operator_positions & pre_positions) != 0); - bool could_be_post = ((operator_positions & post_positions) != 0); + int ambiguous_position = m_policy.positions(m_ambiguous_operator); - int ambiguous_position = m_policy.positions(m_ambiguous_operator); + if ((could_be_pre && could_be_post) || (!could_be_pre && !could_be_post)) + { + m_policy.ambiguity(m_ambiguous_operator, t); + m_state = rejected; + return; + } + else if (could_be_pre) + { + ambiguous_position &= (infix | function_open); + parse_operator(m_policy.as_operator(m_ambiguous_operator, ambiguous_position)); + m_state = pre_operand; + } + else + { + ambiguous_position &= (postfix | close); + parse_operator(m_policy.as_operator(m_ambiguous_operator, ambiguous_position)); + m_state = post_operand; + } + } - if ( (could_be_pre && could_be_post) || - (!could_be_pre && !could_be_post) ) - { - m_policy.ambiguity(m_ambiguous_operator, t); - m_state = rejected; - return; - } - else if (could_be_pre) - { - ambiguous_position &= (infix | function_open); - parse_operator(m_policy.as_operator(m_ambiguous_operator, - ambiguous_position)); - m_state = pre_operand; - } - else - { - ambiguous_position &= (postfix | close); - parse_operator(m_policy.as_operator(m_ambiguous_operator, - ambiguous_position)); - m_state = post_operand; - } - } + int valid_positions = (m_state == pre_operand) ? pre_positions : post_positions; - int valid_positions = - (m_state == pre_operand) ? pre_positions : post_positions; + int positions = operator_positions & valid_positions; - int positions = operator_positions & valid_positions; + switch (positions) + { + case infix: + case function_open: m_state = pre_operand; - switch (positions) - { - case infix: - case function_open: - m_state = pre_operand; + // Fall through + case prefix: + case postfix: + case open: + case close: parse_operator(m_policy.as_operator(t, positions)); break; - // Fall through - case prefix: - case postfix: - case open: - case close: - parse_operator(m_policy.as_operator(t, positions)); - break; + case postfix | infix: + case close | infix: + case postfix | function_open: + case function_open | function_close: + m_state = lookahead; + m_ambiguous_operator = t; + break; - case postfix | infix: - case close | infix: - case postfix | function_open: - case function_open | function_close: - m_state = lookahead; - m_ambiguous_operator = t; - break; - - default: - m_policy.invalid_operator_position(t); - m_state = rejected; - return; - } + default: + m_policy.invalid_operator_position(t); + m_state = rejected; + return; + } + } + else + { + if (m_state == lookahead) + { + int positions = m_policy.positions(m_ambiguous_operator); + positions &= (infix | function_open); + parse_operator(m_policy.as_operator(m_ambiguous_operator, positions)); + m_state = post_operand; + } + else if (m_state == post_operand) + { + m_policy.invalid_operand_position(m_policy.as_operand(t)); + m_state = rejected; + return; } else { - if (m_state == lookahead) - { - int positions = m_policy.positions(m_ambiguous_operator); - positions &= (infix | function_open); - parse_operator(m_policy.as_operator(m_ambiguous_operator, - positions)); - m_state = post_operand; - } - else if (m_state == post_operand) - { - m_policy.invalid_operand_position(m_policy.as_operand(t)); - m_state = rejected; - return; - } - else - { - m_state = post_operand; - } - - m_operand_stack.push(m_policy.as_operand(t)); + m_state = post_operand; } + + m_operand_stack.push(m_policy.as_operand(t)); + } } EXP_TEMPLATE inline int EXP_ACCEPTOR::operator_type_index(int position) { - switch (position) - { - case prefix: - return 0; + switch (position) + { + case prefix: return 0; - case postfix: - return 1; + case postfix: return 1; - case infix: - return 2; + case infix: return 2; - case open: - return 3; + case open: return 3; - case close: - return 4; + case close: return 4; - case function_open: - return 5; - } + case function_open: return 5; + } - assert(0); - return 0; + assert(0); + return 0; } EXP_TEMPLATE void EXP_ACCEPTOR::parse_operator(const Operator& cur_operator) { - if (m_operator_stack.empty()) - { - m_operator_stack.push(cur_operator); - } - else - { - int cur_index = operator_type_index(m_policy.position(cur_operator)); + if (m_operator_stack.empty()) + { + m_operator_stack.push(cur_operator); + } + else + { + int cur_index = operator_type_index(m_policy.position(cur_operator)); - Operator& prev_operator = m_operator_stack.top(); - int prev_index = operator_type_index(m_policy.position(prev_operator)); + Operator& prev_operator = m_operator_stack.top(); + int prev_index = operator_type_index(m_policy.position(prev_operator)); - switch (parse_action[prev_index][cur_index]) + switch (parse_action[prev_index][cur_index]) + { + case shift: m_operator_stack.push(cur_operator); break; + + case reduce: + do_reduce(); + parse_operator(cur_operator); + break; + + case prec: + switch (m_policy.precedence(prev_operator, cur_operator)) { - case shift: - m_operator_stack.push(cur_operator); - break; + case lower: m_operator_stack.push(cur_operator); break; - case reduce: + case higher: + do_reduce(); + parse_operator(cur_operator); + break; + + default: + m_policy.requires_precedence_relation(prev_operator, cur_operator); + m_state = rejected; + return; + } + + break; + + case prec_assoc: + switch (m_policy.precedence(prev_operator, cur_operator)) + { + case lower: m_operator_stack.push(cur_operator); break; + + case higher: + do_reduce(); + parse_operator(cur_operator); + break; + + case equal: + switch (m_policy.associativity(prev_operator, cur_operator)) + { + case left_associative: do_reduce(); parse_operator(cur_operator); break; - case prec: - switch (m_policy.precedence(prev_operator, cur_operator)) - { - case lower: - m_operator_stack.push(cur_operator); - break; + case right_associative: m_operator_stack.push(cur_operator); break; - case higher: - do_reduce(); - parse_operator(cur_operator); - break; - - default: - m_policy.requires_precedence_relation(prev_operator, cur_operator); - m_state = rejected; - return; - } - - break; - - case prec_assoc: - switch (m_policy.precedence(prev_operator, cur_operator)) - { - case lower: - m_operator_stack.push(cur_operator); - break; - - case higher: - do_reduce(); - parse_operator(cur_operator); - break; - - case equal: - switch (m_policy.associativity(prev_operator, cur_operator)) - { - case left_associative: - do_reduce(); - parse_operator(cur_operator); - break; - - case right_associative: - m_operator_stack.push(cur_operator); - break; - - default: - m_policy.requires_associativity_relation(prev_operator, - cur_operator); - m_state = rejected; - return; - } - - break; - - default: - m_policy.requires_precedence_relation(prev_operator, cur_operator); - m_state = rejected; - return; - } - - break; - - case invalid: - m_policy.invalid_operator_position(cur_operator); + default: + m_policy.requires_associativity_relation(prev_operator, cur_operator); m_state = rejected; return; + } + + break; + + default: + m_policy.requires_precedence_relation(prev_operator, cur_operator); + m_state = rejected; + return; } + + break; + + case invalid: + m_policy.invalid_operator_position(cur_operator); + m_state = rejected; + return; } + } } EXP_TEMPLATE void EXP_ACCEPTOR::do_reduce() { - Operator op = m_operator_stack.top(); - m_operator_stack.pop(); + Operator op = m_operator_stack.top(); + m_operator_stack.pop(); - switch (m_policy.position(op)) + switch (m_policy.position(op)) + { + case prefix: + case postfix: { - case prefix: - case postfix: + if (m_operand_stack.empty()) + { + m_policy.missing_operand(op); + m_state = rejected; + return; + } + + Operand operand = m_operand_stack.top(); + m_operand_stack.pop(); + + m_operand_stack.push(m_policy.reduce(op, operand)); + } + break; + + case infix: + { + if (m_operand_stack.size() < 2) + { + m_policy.missing_operand(op); + m_state = rejected; + return; + } + + Operand rhs = m_operand_stack.top(); + m_operand_stack.pop(); + + Operand lhs = m_operand_stack.top(); + m_operand_stack.pop(); + + m_operand_stack.push(m_policy.reduce(op, lhs, rhs)); + } + break; + + case open: + m_policy.unbalanced_confix(op); + m_state = rejected; + return; + + case close: + if (m_operator_stack.empty()) + { + m_policy.unbalanced_confix(op); + m_state = rejected; + return; + } + + if (m_operand_stack.empty()) + { + m_policy.missing_operand(op); + m_state = rejected; + return; + } + + { + Operator open = m_operator_stack.top(); + m_operator_stack.pop(); + + Operand operand = m_operand_stack.top(); + m_operand_stack.pop(); + + if (m_policy.position(open) == function_open) { - if (m_operand_stack.empty()) - { - m_policy.missing_operand(op); - m_state = rejected; - return; - } - - Operand operand = m_operand_stack.top(); - m_operand_stack.pop(); - - m_operand_stack.push(m_policy.reduce(op, operand)); - } - break; - - case infix: - { - if (m_operand_stack.size() < 2) - { - m_policy.missing_operand(op); - m_state = rejected; - return; - } - - Operand rhs = m_operand_stack.top(); - m_operand_stack.pop(); - - Operand lhs = m_operand_stack.top(); - m_operand_stack.pop(); - - m_operand_stack.push(m_policy.reduce(op, lhs, rhs)); - } - break; - - case open: - m_policy.unbalanced_confix(op); + if (m_operand_stack.empty()) + { + m_policy.missing_operand(open); m_state = rejected; return; + } - case close: - if (m_operator_stack.empty()) - { - m_policy.unbalanced_confix(op); - m_state = rejected; - return; - } + Operand function = m_operand_stack.top(); + m_operand_stack.pop(); - if (m_operand_stack.empty()) - { - m_policy.missing_operand(op); - m_state = rejected; - return; - } + m_operand_stack.push(m_policy.reduce(function, open, operand, op)); + } + else + { + m_operand_stack.push(m_policy.reduce(open, op, operand)); + } + } - { - Operator open = m_operator_stack.top(); - m_operator_stack.pop(); + return; - Operand operand = m_operand_stack.top(); - m_operand_stack.pop(); - - if (m_policy.position(open) == function_open) - { - if (m_operand_stack.empty()) - { - m_policy.missing_operand(open); - m_state = rejected; - return; - } - - Operand function = m_operand_stack.top(); - m_operand_stack.pop(); - - m_operand_stack.push( - m_policy.reduce(function, open, operand, op) - ); - } - else - { - m_operand_stack.push(m_policy.reduce(open, op, operand)); - } - } - - return; - - default: - assert(0); - } -} + default: assert(0); + } } +} // namespace detail EXP_TEMPLATE inline bool accepted(const detail::EXP_ACCEPTOR& acceptor) { - return acceptor.accepted(); + return acceptor.accepted(); } EXP_TEMPLATE inline bool trapped(const detail::EXP_ACCEPTOR& acceptor) { - return acceptor.trapped(); + return acceptor.trapped(); } -template < - typename Token, - typename Operand, - typename Operator, - typename Policy, - typename OperandStack = std::stack, - typename OperatorStack = std::stack - > +template , typename OperatorStack = std::stack > class expression_parser { -public: - typedef detail::EXP_ACCEPTOR acceptor; - typedef Token token_type; - typedef Operand operand_type; - typedef Operator operator_type; - typedef Policy policy_type; - typedef Token input_symbol_type; - typedef Operand result_type; + public: + typedef detail::EXP_ACCEPTOR acceptor; + typedef Token token_type; + typedef Operand operand_type; + typedef Operator operator_type; + typedef Policy policy_type; + typedef Token input_symbol_type; + typedef Operand result_type; - expression_parser() : m_policy() {} - explicit expression_parser(const Policy& policy) : m_policy(policy) {} + expression_parser() : m_policy() + { + } + explicit expression_parser(const Policy& policy) : m_policy(policy) + { + } - template - Operand parse(InputIterator first, InputIterator last) + template + Operand parse(InputIterator first, InputIterator last) + { + try { - try - { #if _MSC_VER > 1600 - return std::_Copy_impl(first, last, start()); - } + return std::_Copy_impl(first, last, start()); + } #else - return std::copy(first, last, start()); - } + return std::copy(first, last, start()); + } #endif - catch (const std::runtime_error&) - { - m_policy.cleanup(operandStack, operatorStack); - throw; - } - } - - acceptor start() + catch (const std::runtime_error&) { - // pass in operator and operand stacks to acceptor constructor. - // this is to chean up the pointers in the stack when exception - // is thrown. the exception is then passed to the upper level. - // changed by Zhixuan Zhu 07/03/06 - try - { - return acceptor(m_policy, operandStack, operatorStack); - - } - catch (const std::runtime_error&) - { - m_policy.cleanup(operandStack, operatorStack); - throw; - } + m_policy.cleanup(operandStack, operatorStack); + throw; } + } -private: - Policy m_policy; + acceptor start() + { + // pass in operator and operand stacks to acceptor constructor. + // this is to chean up the pointers in the stack when exception + // is thrown. the exception is then passed to the upper level. + // changed by Zhixuan Zhu 07/03/06 + try + { + return acceptor(m_policy, operandStack, operatorStack); + } + catch (const std::runtime_error&) + { + m_policy.cleanup(operandStack, operatorStack); + throw; + } + } - // added operand and operator stacks by Zhixuan Zhu - OperandStack operandStack; - OperatorStack operatorStack; + private: + Policy m_policy; + + // added operand and operator stacks by Zhixuan Zhu + OperandStack operandStack; + OperatorStack operatorStack; }; #undef EXP_ACCEPTOR #undef EXP_TEMPLATE -} +} // namespace expression #endif - diff --git a/dbcon/execplan/expressionparser.cpp b/dbcon/execplan/expressionparser.cpp index 4201b90e4..50ac3f35f 100644 --- a/dbcon/execplan/expressionparser.cpp +++ b/dbcon/execplan/expressionparser.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: expressionparser.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: expressionparser.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -35,405 +35,382 @@ using namespace std; namespace execplan { - /** * Constructor / Destructor */ ExpressionParser::ExpressionParser() -{} +{ +} ExpressionParser::~ExpressionParser() -{} +{ +} void ExpressionParser::invalid_operator_position(TreeNode* oper) { - std::string str = oper->data(); - delete oper; - throw std::runtime_error ("Invalid operator position: " + str + "\n"); + std::string str = oper->data(); + delete oper; + throw std::runtime_error("Invalid operator position: " + str + "\n"); } void ExpressionParser::invalid_operator_position(const Token& oper) { - throw std::runtime_error ("Invalid operator position: " + oper.value->data() + "\n"); + throw std::runtime_error("Invalid operator position: " + oper.value->data() + "\n"); } void ExpressionParser::invalid_operand_position(ParseTree* operand) { - delete operand; - throw std::runtime_error ("Invalid operand position\n"); + delete operand; + throw std::runtime_error("Invalid operand position\n"); } void ExpressionParser::unbalanced_confix(TreeNode* oper) { - delete oper; - throw std::runtime_error ("Unbalanced confix operator\n"); + delete oper; + throw std::runtime_error("Unbalanced confix operator\n"); } -void ExpressionParser::missing_operand (const Token& oper) +void ExpressionParser::missing_operand(const Token& oper) { - delete oper.value; - throw std::runtime_error ("Imissing operand\n"); + delete oper.value; + throw std::runtime_error("Imissing operand\n"); } int ExpressionParser::positions(Token t) { - string oper = t.value->data(); - char c = oper.at(0); + string oper = t.value->data(); + char c = oper.at(0); - switch (c) - { - case '+': - case '-': - return expression::infix | expression::prefix; + switch (c) + { + case '+': + case '-': return expression::infix | expression::prefix; - case '*': - case '/': - case '^': - case '|': // concat || - return expression::infix; + case '*': + case '/': + case '^': + case '|': // concat || + return expression::infix; - case '(': - return expression::open | expression::function_open; + case '(': return expression::open | expression::function_open; - case ')': - return expression::close | expression::function_close; + case ')': return expression::close | expression::function_close; - default: - boost::algorithm::to_lower(oper); + default: + boost::algorithm::to_lower(oper); - if (oper.compare ("and") == 0 || oper.compare ("or") == 0 ) - return expression::infix; - } + if (oper.compare("and") == 0 || oper.compare("or") == 0) + return expression::infix; + } - ostringstream oss; - oss << "ExpressionParser::positions(Token): invalid input token: >" << oper << '<'; - throw std::runtime_error(oss.str()); - return 0; + ostringstream oss; + oss << "ExpressionParser::positions(Token): invalid input token: >" << oper << '<'; + throw std::runtime_error(oss.str()); + return 0; } int ExpressionParser::position(TreeNode* op) { - string oper = op->data(); - char c = oper.at(0); + string oper = op->data(); + char c = oper.at(0); - switch (c) - { - case '+': - case '-': - case '*': - case '/': - case '|': // concat || - return expression::infix; + switch (c) + { + case '+': + case '-': + case '*': + case '/': + case '|': // concat || + return expression::infix; - case 'M': - case 'm': - case 'I': - case 'i': - return expression::prefix; + case 'M': + case 'm': + case 'I': + case 'i': return expression::prefix; - case '(': - return expression::open; + case '(': return expression::open; - case '[': - return expression::function_open; + case '[': return expression::function_open; - case ')': - return expression::close; + case ')': return expression::close; - case ']': - return expression::function_close; + case ']': return expression::function_close; - default: - boost::algorithm::to_lower(oper); + default: + boost::algorithm::to_lower(oper); - if (oper.compare ("and") == 0 || oper.compare ("or") == 0 ) - return expression::infix; - } + if (oper.compare("and") == 0 || oper.compare("or") == 0) + return expression::infix; + } - ostringstream oss; - oss << "ExpressionParser::position(TreeNode*): invalid input token: >" << oper << '<'; - throw std::runtime_error(oss.str()); + ostringstream oss; + oss << "ExpressionParser::position(TreeNode*): invalid input token: >" << oper << '<'; + throw std::runtime_error(oss.str()); } ParseTree* ExpressionParser::as_operand(Token t) { - return new ParseTree(t.value); + return new ParseTree(t.value); } TreeNode* ExpressionParser::as_operator(Token t, int pos) { - string oper = t.value->data(); - char c = oper.at(0); + string oper = t.value->data(); + char c = oper.at(0); - switch (c) - { - case '+': - if (pos == expression::infix) - return t.value; - else - { - delete t.value; - return new Operator("I"); - } + switch (c) + { + case '+': + if (pos == expression::infix) + return t.value; + else + { + delete t.value; + return new Operator("I"); + } - case '-': - if (pos == expression::infix) - return t.value; - else - { - delete t.value; - return new Operator("M"); - } + case '-': + if (pos == expression::infix) + return t.value; + else + { + delete t.value; + return new Operator("M"); + } - case '*': - case '/': - case ')': - case '|': - return t.value; + case '*': + case '/': + case ')': + case '|': return t.value; - case '(': - if (pos == expression::open) - return t.value; - else - { - delete t.value; - return new Operator("["); - } + case '(': + if (pos == expression::open) + return t.value; + else + { + delete t.value; + return new Operator("["); + } - default: - boost::algorithm::to_lower(oper); + default: + boost::algorithm::to_lower(oper); - if (oper.compare ("and") == 0 || oper.compare ("or") == 0 ) - return t.value; - } + if (oper.compare("and") == 0 || oper.compare("or") == 0) + return t.value; + } - ostringstream oss; - oss << "ExpressionParser::as_operator(Token,int): invalid input token: >" << oper << '<'; - throw std::runtime_error(oss.str()); + ostringstream oss; + oss << "ExpressionParser::as_operator(Token,int): invalid input token: >" << oper << '<'; + throw std::runtime_error(oss.str()); } -expression::precedence ExpressionParser::precedence (TreeNode* op1, TreeNode* op2) +expression::precedence ExpressionParser::precedence(TreeNode* op1, TreeNode* op2) { - int p1 = precnum(op1); - int p2 = precnum(op2); + int p1 = precnum(op1); + int p2 = precnum(op2); - if (p1 < p2) - return expression::lower; - else if (p1 > p2) - return expression::higher; - else if (p1 == p2) - return expression::equal; - else - return expression::none; + if (p1 < p2) + return expression::lower; + else if (p1 > p2) + return expression::higher; + else if (p1 == p2) + return expression::equal; + else + return expression::none; } expression::associativity ExpressionParser::associativity(TreeNode* op1, TreeNode* op2) { - expression::associativity assoc1 = assoc(op1); - expression::associativity assoc2 = assoc(op2); + expression::associativity assoc1 = assoc(op1); + expression::associativity assoc2 = assoc(op2); - if (assoc1 == assoc2) - return assoc1; - else - return expression::non_associative; + if (assoc1 == assoc2) + return assoc1; + else + return expression::non_associative; } /** -* Build an expression tree with the tokens -*/ + * Build an expression tree with the tokens + */ ParseTree* ExpressionParser::reduce(TreeNode* op, ParseTree* value) { - char c = op->data().at(0); + char c = op->data().at(0); - switch (c) + switch (c) + { + case 'M': + case 'm': { - case 'M': - case 'm': - { - ParseTree* root = new ParseTree(op); - ParseTree* lhs = new ParseTree(new ConstantColumn("0", ConstantColumn::NUM)); - root->left(lhs); - root->right(value); - return root; - } - - case 'I': - case 'i': - delete op; - return value; - - default: - idbassert(0); + ParseTree* root = new ParseTree(op); + ParseTree* lhs = new ParseTree(new ConstantColumn("0", ConstantColumn::NUM)); + root->left(lhs); + root->right(value); + return root; } - ostringstream oss; - oss << "ExpressionParser::reduce(TreeNode*,ParseTree*): invalid input token: >" << op->data() << '<'; - throw std::runtime_error(oss.str()); - return 0; + case 'I': + case 'i': delete op; return value; + + default: idbassert(0); + } + + ostringstream oss; + oss << "ExpressionParser::reduce(TreeNode*,ParseTree*): invalid input token: >" << op->data() << '<'; + throw std::runtime_error(oss.str()); + return 0; } ParseTree* ExpressionParser::reduce(TreeNode* op, ParseTree* lhs, ParseTree* rhs) { - ParseTree* root = new ParseTree(op); - root->left(lhs); - root->right(rhs); - return root; + ParseTree* root = new ParseTree(op); + root->left(lhs); + root->right(rhs); + return root; } // parenthesis ParseTree* ExpressionParser::reduce(TreeNode* a, TreeNode* b, ParseTree* value) { - delete a; - delete b; - return value; + delete a; + delete b; + return value; } // function call: handle aggregation functions and other functions -ParseTree* ExpressionParser::reduce(ParseTree* a, TreeNode* b, - ParseTree* value, TreeNode* d) +ParseTree* ExpressionParser::reduce(ParseTree* a, TreeNode* b, ParseTree* value, TreeNode* d) { - string functionName = a->data()->data(); - string content = value->data()->data(); - ParseTree* root; + string functionName = a->data()->data(); + string content = value->data()->data(); + ParseTree* root; - boost::algorithm::to_lower(functionName); + boost::algorithm::to_lower(functionName); - if (functionName.compare("sum") == 0 || - functionName.compare("avg") == 0 || - functionName.compare("count") == 0 || - functionName.compare("min") == 0 || - functionName.compare("max") == 0) - { - AggregateColumn* ac = new AggregateColumn(functionName, content); - root = new ParseTree(ac); - } - else - { - FunctionColumn* fc = new FunctionColumn(functionName, content); - root = new ParseTree(fc); - } + if (functionName.compare("sum") == 0 || functionName.compare("avg") == 0 || + functionName.compare("count") == 0 || functionName.compare("min") == 0 || + functionName.compare("max") == 0) + { + AggregateColumn* ac = new AggregateColumn(functionName, content); + root = new ParseTree(ac); + } + else + { + FunctionColumn* fc = new FunctionColumn(functionName, content); + root = new ParseTree(fc); + } - delete a; - delete value; - delete b; - delete d; - return root; + delete a; + delete value; + delete b; + delete d; + return root; } int ExpressionParser::precnum(TreeNode* op) { - string oper = op->data(); - char c = oper.at(0); + string oper = op->data(); + char c = oper.at(0); - switch (c) - { - case '+': - case '-': - case '|': - return 3; + switch (c) + { + case '+': + case '-': + case '|': return 3; - case '*': - case '/': - return 4; + case '*': + case '/': return 4; - case 'M': - case 'I': - return 5; + case 'M': + case 'I': return 5; - case '(': - return 6; + case '(': return 6; - case '[': - return 7; + case '[': return 7; - default: - boost::algorithm::to_lower(oper); + default: + boost::algorithm::to_lower(oper); - if (oper.compare("or") == 0) - return 1; - else if (oper.compare("and") == 0) - return 2; - } + if (oper.compare("or") == 0) + return 1; + else if (oper.compare("and") == 0) + return 2; + } - return 0; + return 0; } expression::associativity ExpressionParser::assoc(TreeNode* op) { - string oper = op->data(); - char c = oper.at(0); + string oper = op->data(); + char c = oper.at(0); - switch (c) - { - case '+': - case '-': - case '*': - case '/': - case '|': - return expression::left_associative; + switch (c) + { + case '+': + case '-': + case '*': + case '/': + case '|': return expression::left_associative; - default: - boost::algorithm::to_lower(oper); + default: + boost::algorithm::to_lower(oper); - if (oper.compare("or") == 0 || oper.compare("and") == 0) - return expression::left_associative; + if (oper.compare("or") == 0 || oper.compare("and") == 0) + return expression::left_associative; - return expression::non_associative; - } + return expression::non_associative; + } } -void ExpressionParser::cleanup(stack operandStack, - stack operatorStack) +void ExpressionParser::cleanup(stack operandStack, stack operatorStack) { - while ( !operandStack.empty() ) - { - ParseTree* a = operandStack.top(); - operandStack.pop(); - delete a; - } + while (!operandStack.empty()) + { + ParseTree* a = operandStack.top(); + operandStack.pop(); + delete a; + } - while ( !operatorStack.empty() ) - { - TreeNode* a = operatorStack.top(); - operatorStack.pop(); - delete a; - } + while (!operatorStack.empty()) + { + TreeNode* a = operatorStack.top(); + operatorStack.pop(); + delete a; + } } -bool operator==(const ParseTree& t1, - const ParseTree& t2) +bool operator==(const ParseTree& t1, const ParseTree& t2) { - if (t1.data() != NULL && t2.data() != NULL) - { - if (*t1.data() != t2.data()) - return false; - } - else if (t1.data() != NULL || t2.data() != NULL) - return false; + if (t1.data() != NULL && t2.data() != NULL) + { + if (*t1.data() != t2.data()) + return false; + } + else if (t1.data() != NULL || t2.data() != NULL) + return false; - if (t1.left() != NULL && t2.left() != NULL) - { - if (*t1.left() != *t2.left()) - return false; - } - else if (t1.left() != NULL || t2.left() != NULL) - return false; + if (t1.left() != NULL && t2.left() != NULL) + { + if (*t1.left() != *t2.left()) + return false; + } + else if (t1.left() != NULL || t2.left() != NULL) + return false; - if (t1.right() != NULL && t2.right() != NULL) - { - if (*t1.right() != *t2.right()) - return false; - } - else if (t1.right() != NULL || t2.right() != NULL) - return false; + if (t1.right() != NULL && t2.right() != NULL) + { + if (*t1.right() != *t2.right()) + return false; + } + else if (t1.right() != NULL || t2.right() != NULL) + return false; - return true; + return true; } -bool operator!=(const ParseTree& t1, - const ParseTree& t2) +bool operator!=(const ParseTree& t1, const ParseTree& t2) { - return !(t1 == t2); + return !(t1 == t2); } -} +} // namespace execplan diff --git a/dbcon/execplan/expressionparser.h b/dbcon/execplan/expressionparser.h index cc7f34f7b..23697d2ce 100644 --- a/dbcon/execplan/expressionparser.h +++ b/dbcon/execplan/expressionparser.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: expressionparser.h 9633 2013-06-19 13:36:01Z rdempsey $ -* -* -***********************************************************************/ + * $Id: expressionparser.h 9633 2013-06-19 13:36:01Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef EXPRESSION_PARSER #define EXPRESSION_PARSER @@ -39,7 +39,6 @@ namespace execplan { - /** * type define */ @@ -52,24 +51,29 @@ typedef std::stack OperatorStack; */ struct Token { - TreeNode* value; - bool is_operator() const - { - if (value == 0) return false; + TreeNode* value; + bool is_operator() const + { + if (value == 0) + return false; - return (typeid(*value) == typeid(Operator)); - } - Token() : value(0) {} - Token(TreeNode* v) : value(v) {} -private: - // technically, these should be defined since Token is used in a - // std::vector, but it appears to work okay...ownership of the - // value ptr is dubious - //Token(const Token& rhs); - //Token& operator=(const Token& rhs); + return (typeid(*value) == typeid(Operator)); + } + Token() : value(0) + { + } + Token(TreeNode* v) : value(v) + { + } + + private: + // technically, these should be defined since Token is used in a + // std::vector, but it appears to work okay...ownership of the + // value ptr is dubious + // Token(const Token& rhs); + // Token& operator=(const Token& rhs); }; - /** * @brief this class builds an expression tree * @@ -80,63 +84,62 @@ private: */ class ExpressionParser : public expression::default_expression_parser_error_policy { -public: - /** - * Constructors/Destructors - */ - ExpressionParser(); - virtual ~ExpressionParser(); + public: + /** + * Constructors/Destructors + */ + ExpressionParser(); + virtual ~ExpressionParser(); - /** - * Operations - */ + /** + * Operations + */ - /* - * Error handling - */ - static void invalid_operator_position(TreeNode* oper); - static void invalid_operator_position(const Token& oper); - static void invalid_operand_position(ParseTree* operand); - static void unbalanced_confix(TreeNode* oper); - static void missing_operand(const Token& oper); + /* + * Error handling + */ + static void invalid_operator_position(TreeNode* oper); + static void invalid_operator_position(const Token& oper); + static void invalid_operand_position(ParseTree* operand); + static void unbalanced_confix(TreeNode* oper); + static void missing_operand(const Token& oper); - /** - * Syntax and precedence rules - */ - static int positions(Token t); - static int position(TreeNode* op); - inline static bool is_operator(const Token& t) - { - return t.is_operator(); - } - static ParseTree* as_operand(Token t); - static TreeNode* as_operator(Token t, int pos); - static expression::precedence precedence (TreeNode* op1, TreeNode* op2); - static expression::associativity associativity(TreeNode* op1, TreeNode* op2); + /** + * Syntax and precedence rules + */ + static int positions(Token t); + static int position(TreeNode* op); + inline static bool is_operator(const Token& t) + { + return t.is_operator(); + } + static ParseTree* as_operand(Token t); + static TreeNode* as_operator(Token t, int pos); + static expression::precedence precedence(TreeNode* op1, TreeNode* op2); + static expression::associativity associativity(TreeNode* op1, TreeNode* op2); - /** - * Build an expression tree with the tokens - */ - static ParseTree* reduce(TreeNode* op, ParseTree* value); - static ParseTree* reduce(TreeNode* op, ParseTree* lhs, ParseTree* rhs); + /** + * Build an expression tree with the tokens + */ + static ParseTree* reduce(TreeNode* op, ParseTree* value); + static ParseTree* reduce(TreeNode* op, ParseTree* lhs, ParseTree* rhs); - // parenthesis - static ParseTree* reduce(TreeNode* a, TreeNode* b, ParseTree* value); + // parenthesis + static ParseTree* reduce(TreeNode* a, TreeNode* b, ParseTree* value); - // function call - static ParseTree* reduce(ParseTree* a, TreeNode* b, ParseTree* value, TreeNode* d); + // function call + static ParseTree* reduce(ParseTree* a, TreeNode* b, ParseTree* value, TreeNode* d); + /** + * to clean up operator and operand stack when exception throws. + * this is to be used by Calpont execplan project, where the operator + * and operand are pointers. added by Zhixuan Zhu 06/30/06 + */ + static void cleanup(std::stack operandStack, std::stack operatorStack); - /** - * to clean up operator and operand stack when exception throws. - * this is to be used by Calpont execplan project, where the operator - * and operand are pointers. added by Zhixuan Zhu 06/30/06 - */ - static void cleanup(std::stack operandStack, std::stack operatorStack); - -private: - static int precnum(TreeNode* op); - static expression::associativity assoc(TreeNode* op); + private: + static int precnum(TreeNode* op); + static expression::associativity assoc(TreeNode* op); }; /** @brief Do a deep, strict (as opposed to semantic) equivalence test @@ -144,18 +147,15 @@ private: * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t1 is a duplicate copy of every member of t2; false otherwise */ -bool operator==(const ParseTree& t1, - const ParseTree& t2); +bool operator==(const ParseTree& t1, const ParseTree& t2); /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t1 is a duplicate copy of every member of t2; true otherwise */ -bool operator!=(const ParseTree& t1, - const ParseTree& t2); +bool operator!=(const ParseTree& t1, const ParseTree& t2); - -} // namespace execplan +} // namespace execplan #endif diff --git a/dbcon/execplan/filter.cpp b/dbcon/execplan/filter.cpp index b76755872..a1b56336f 100644 --- a/dbcon/execplan/filter.cpp +++ b/dbcon/execplan/filter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: filter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: filter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include using namespace std; @@ -36,69 +36,68 @@ namespace execplan */ Filter::Filter() { - fCardinality = 0; + fCardinality = 0; } -Filter::Filter(const string& sql) : - fData(sql) +Filter::Filter(const string& sql) : fData(sql) { - fCardinality = 0; + fCardinality = 0; } Filter::~Filter() -{} - +{ +} /** * The serialization interface */ void Filter::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::FILTER; - b << fData; - b << (uint64_t)fCardinality; + b << (ObjectReader::id_t)ObjectReader::FILTER; + b << fData; + b << (uint64_t)fCardinality; } void Filter::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::FILTER); - b >> fData; - b >> (uint64_t&)fCardinality; + ObjectReader::checkType(b, ObjectReader::FILTER); + b >> fData; + b >> (uint64_t&)fCardinality; } const string Filter::toString() const { - return string(">Filter<"); + return string(">Filter<"); } bool Filter::operator==(const Filter& t) const { - if (data() == t.data()) - return true; + if (data() == t.data()) + return true; - return false; + return false; } bool Filter::operator==(const TreeNode* t) const { - const Filter* o; + const Filter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool Filter::operator!=(const Filter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool Filter::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } /** @@ -106,8 +105,8 @@ bool Filter::operator!=(const TreeNode* t) const */ ostream& operator<<(ostream& output, const Filter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/filter.h b/dbcon/execplan/filter.h index 4e376f2ba..9a3d8eae0 100644 --- a/dbcon/execplan/filter.h +++ b/dbcon/execplan/filter.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: filter.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: filter.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef EXECPLAN_FILTER_H @@ -39,7 +39,6 @@ class ByteStream; */ namespace execplan { - class Operator; /** @brief A class to represent a generic filter predicate @@ -52,118 +51,115 @@ class Operator; */ class Filter : public TreeNode { -//friend std::ostream &operator<< (std::ostream &, Filter &); + // friend std::ostream &operator<< (std::ostream &, Filter &); - /** - * Public stuff - */ -public: + /** + * Public stuff + */ + public: + /** + * Constructors + */ + Filter(); + Filter(const std::string& sql); + // not needed yet + // Filter(const Filter& rhs); - /** - * Constructors - */ - Filter(); - Filter(const std::string& sql); - // not needed yet - //Filter(const Filter& rhs); + /** + * Destructors + */ + virtual ~Filter(); + /** + * Accessor Methods + */ - /** - * Destructors - */ - virtual ~Filter(); - /** - * Accessor Methods - */ + /** + * Operations + */ + virtual const std::string toString() const; - /** - * Operations - */ - virtual const std::string toString() const; + virtual const std::string data() const + { + return fData; + } + virtual void data(const std::string data) + { + fData = data; + } - virtual const std::string data() const - { - return fData; - } - virtual void data(const std::string data) - { - fData = data; - } + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual Filter* clone() const + { + return new Filter(*this); + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual Filter* clone() const - { - return new Filter (*this); - } + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const Filter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const Filter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const Filter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const Filter& t) const; + /** @brief test if this filter can be combined with the argument filter + * This is for operation combine optimization + * @param f the filter that this fiter tries to combine with + * @param op the operator that connects the two filters. if one or both of the + * two filters is constantFilter, need to make sure operator is consistant. + * @return a filter(constantfilter) if successfully combined. otherwise + * return NULL + * For Oracle front end. Deprecated now. + */ + // virtual Filter* combinable(Filter* f, Operator* op); + virtual uint64_t cardinality() const + { + return fCardinality; + } + virtual void cardinality(const uint64_t cardinality) + { + fCardinality = cardinality; + } - /** @brief test if this filter can be combined with the argument filter - * This is for operation combine optimization - * @param f the filter that this fiter tries to combine with - * @param op the operator that connects the two filters. if one or both of the - * two filters is constantFilter, need to make sure operator is consistant. - * @return a filter(constantfilter) if successfully combined. otherwise - * return NULL - * For Oracle front end. Deprecated now. - */ - //virtual Filter* combinable(Filter* f, Operator* op); - virtual uint64_t cardinality () const - { - return fCardinality; - } - virtual void cardinality (const uint64_t cardinality) - { - fCardinality = cardinality; - } + protected: + uint64_t fCardinality; -protected: - uint64_t fCardinality; - -private: - //default okay - //Filter& operator=(const Filter& rhs); - - std::string fData; + private: + // default okay + // Filter& operator=(const Filter& rhs); + std::string fData; }; std::ostream& operator<<(std::ostream& os, const Filter& rhs); -} -#endif //EXECPLAN_FILTER_H - +} // namespace execplan +#endif // EXECPLAN_FILTER_H diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 18833afa3..6aa153528 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: functioncolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: functioncolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ #include #include #include @@ -55,52 +55,54 @@ namespace execplan * Constructors/Destructors */ FunctionColumn::FunctionColumn() -{} - -FunctionColumn::FunctionColumn(string& funcName): - fFunctionName(funcName) -{} - -FunctionColumn::FunctionColumn(const string& functionName, const string& funcParmsInString, const uint32_t sessionID): - ReturnedColumn(sessionID), - fFunctionName(functionName), - fData (functionName + "(" + funcParmsInString + ")"), - fFunctor(0) { - funcParms (funcParmsInString); } -FunctionColumn::FunctionColumn( const FunctionColumn& rhs, const uint32_t sessionID): - ReturnedColumn(rhs, sessionID), - fFunctionName(rhs.functionName()), - fTableAlias (rhs.tableAlias()), - fData (rhs.data()), - fTimeZone(rhs.timeZone()), - fFunctor(rhs.fFunctor) +FunctionColumn::FunctionColumn(string& funcName) : fFunctionName(funcName) { - fFunctionParms.clear(); - fSimpleColumnList.clear(); - fAggColumnList.clear(); - fWindowFunctionColumnList.clear(); +} - SPTP pt; +FunctionColumn::FunctionColumn(const string& functionName, const string& funcParmsInString, + const uint32_t sessionID) + : ReturnedColumn(sessionID) + , fFunctionName(functionName) + , fData(functionName + "(" + funcParmsInString + ")") + , fFunctor(0) +{ + funcParms(funcParmsInString); +} - for (uint32_t i = 0; i < rhs.fFunctionParms.size(); i++) - { - pt.reset(new ParseTree (*(rhs.fFunctionParms[i]))); - fFunctionParms.push_back(pt); - pt->walk(getSimpleCols, &fSimpleColumnList); - pt->walk(getAggCols, &fAggColumnList); - pt->walk(getWindowFunctionCols, &fWindowFunctionColumnList); - } +FunctionColumn::FunctionColumn(const FunctionColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID) + , fFunctionName(rhs.functionName()) + , fTableAlias(rhs.tableAlias()) + , fData(rhs.data()) + , fTimeZone(rhs.timeZone()) + , fFunctor(rhs.fFunctor) +{ + fFunctionParms.clear(); + fSimpleColumnList.clear(); + fAggColumnList.clear(); + fWindowFunctionColumnList.clear(); - fAlias = rhs.alias(); + SPTP pt; + + for (uint32_t i = 0; i < rhs.fFunctionParms.size(); i++) + { + pt.reset(new ParseTree(*(rhs.fFunctionParms[i]))); + fFunctionParms.push_back(pt); + pt->walk(getSimpleCols, &fSimpleColumnList); + pt->walk(getAggCols, &fAggColumnList); + pt->walk(getWindowFunctionCols, &fWindowFunctionColumnList); + } + + fAlias = rhs.alias(); } FunctionColumn::~FunctionColumn() { - if (fDynamicFunctor) - delete fDynamicFunctor; + if (fDynamicFunctor) + delete fDynamicFunctor; } /** @@ -108,386 +110,383 @@ FunctionColumn::~FunctionColumn() */ ostream& operator<<(ostream& output, const FunctionColumn& rhs) { - output << rhs.toString(); + output << rhs.toString(); - return output; + return output; } const string FunctionColumn::toString() const { - ostringstream output; - output << std::endl << "FunctionColumn: " << fFunctionName << endl; + ostringstream output; + output << std::endl << "FunctionColumn: " << fFunctionName << endl; - if (fAlias.length() > 0) output << "/Alias: " << fAlias; + if (fAlias.length() > 0) + output << "/Alias: " << fAlias; - output << "expressionId=" << fExpressionId << endl; - output << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; - output << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; - output << "operationType=" << colDataTypeToString(fOperationType.colDataType) << endl; - output << "function parm: " << endl; + output << "expressionId=" << fExpressionId << endl; + output << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; + output << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth + << endl; + output << "operationType=" << colDataTypeToString(fOperationType.colDataType) << endl; + output << "function parm: " << endl; - for (uint32_t i = 0; i < fFunctionParms.size(); i++) - output << fFunctionParms[i]->data()->toString() << endl; + for (uint32_t i = 0; i < fFunctionParms.size(); i++) + output << fFunctionParms[i]->data()->toString() << endl; - return output.str(); + return output.str(); } const string FunctionColumn::data() const { - return fData; + return fData; } -void FunctionColumn::funcParms (const string& funcParmsInString) +void FunctionColumn::funcParms(const string& funcParmsInString) { - // special process to interval function. make the passed in string one literal constant - if (fFunctionName.compare("interval") == 0) + // special process to interval function. make the passed in string one literal constant + if (fFunctionName.compare("interval") == 0) + { + SPTP sptp(new ParseTree(new ConstantColumn(funcParmsInString, ConstantColumn::LITERAL))); + fFunctionParms.push_back(sptp); + return; + } + + // try to replace the comma and space in '' to avoid string conflict + // @bug #396 for space delimited function arguments. + string funcParam = funcParmsInString; + string::size_type pos1 = 0; + string::size_type pos2 = 0; + string::size_type pos3 = 0; + + while (pos2 < funcParam.length() && pos1 != string::npos) + { + pos1 = funcParam.find_first_of("'", pos2); + pos2 = funcParam.find_first_of("'", pos1 + 1); + pos3 = funcParam.find_first_of(", ", pos1); + + if (pos3 < pos2) { - SPTP sptp(new ParseTree(new ConstantColumn(funcParmsInString, ConstantColumn::LITERAL))); - fFunctionParms.push_back(sptp); - return; + if (funcParam[pos3] == ',') + funcParam.replace(pos3, 1, "^"); + else if (funcParam[pos3] == ' ') + funcParam[pos3] = 0x1f; // special char to replace space in quotes } - // try to replace the comma and space in '' to avoid string conflict - // @bug #396 for space delimited function arguments. - string funcParam = funcParmsInString; - string::size_type pos1 = 0; - string::size_type pos2 = 0; - string::size_type pos3 = 0; + pos2++; + } - while (pos2 < funcParam.length() && pos1 != string::npos) + // also replace the comma and space in () to avoid function conflict + unsigned int par = 0; + + for (unsigned int i = 0; i < funcParam.length(); i++) + { + if (funcParam[i] == '(') + par++; + + if (funcParam[i] == ')') + par--; + + if (funcParam[i] == ',' && par != 0) + funcParam.replace(i, 1, "^"); + + if (funcParam[i] == ' ' && par != 0) + funcParam[i] = 0x1f; + } + + typedef boost::tokenizer > tokenizer; + boost::char_separator sep(", "); + tokenizer tokens(funcParam, sep); + + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + string tok = (*tok_iter); + // remove the to_upper line. don't know why have it in the first place. for bug #320 + // to_upper(tok); + + // take off the trailing and ending spaces + std::string::size_type first = tok.find_first_not_of(" "); + tok = tok.substr(first, tok.find_last_not_of(" ") - first + 1); + + // recover '^' to ',' and recover 0x1f to ' ' + for (unsigned int i = 0; i < tok.length(); i++) { - pos1 = funcParam.find_first_of("'", pos2); - pos2 = funcParam.find_first_of("'", pos1 + 1); - pos3 = funcParam.find_first_of(", ", pos1); + if (tok[i] == '^') + tok[i] = ','; - if (pos3 < pos2) - { - if (funcParam[pos3] == ',') - funcParam.replace(pos3, 1, "^"); - else if (funcParam[pos3] == ' ') - funcParam[pos3] = 0x1f; // special char to replace space in quotes - } - - pos2++; + if (tok[i] == 0x1f) + tok[i] = ' '; } - // also replace the comma and space in () to avoid function conflict - unsigned int par = 0; - - for (unsigned int i = 0; i < funcParam.length(); i++) + // parse argument + if (tok[0] == '\'' && tok[tok.length() - 1] == '\'') { - if (funcParam[i] == '(') - par++; - - if (funcParam[i] == ')') - par--; - - if (funcParam[i] == ',' && par != 0) - funcParam.replace(i, 1, "^"); - - if (funcParam[i] == ' ' && par != 0) - funcParam[i] = 0x1f; + // take off quotes + first = tok.find_first_not_of("'"); + tok = tok.substr(first, tok.find_last_not_of("'") - first + 1); + SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::LITERAL))); + fFunctionParms.push_back(cc); } - - typedef boost::tokenizer > tokenizer; - boost::char_separator sep(", "); - tokenizer tokens(funcParam, sep); - - for (tokenizer::iterator tok_iter = tokens.begin(); - tok_iter != tokens.end(); ++tok_iter) + else if (tok.find("+", 0) != string::npos || tok.find("-", 0) != string::npos || + tok.find("*", 0) != string::npos || tok.find("/", 0) != string::npos || + tok.find("^", 0) != string::npos || tok.find("(", 0) != string::npos || + tok.find(")", 0) != string::npos) { - string tok = (*tok_iter); - // remove the to_upper line. don't know why have it in the first place. for bug #320 - //to_upper(tok); - - // take off the trailing and ending spaces - std::string::size_type first = tok.find_first_not_of(" "); - tok = tok.substr(first, tok.find_last_not_of(" ") - first + 1); - - // recover '^' to ',' and recover 0x1f to ' ' - for (unsigned int i = 0; i < tok.length(); i++) - { - if (tok[i] == '^') tok[i] = ','; - - if (tok[i] == 0x1f) tok[i] = ' '; - } - - // parse argument - if (tok[0] == '\'' && tok[tok.length() - 1] == '\'') - { - // take off quotes - first = tok.find_first_not_of("'"); - tok = tok.substr(first, tok.find_last_not_of("'") - first + 1); - SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::LITERAL))); - fFunctionParms.push_back(cc); - } - else if (tok.find("+", 0) != string::npos || - tok.find("-", 0) != string::npos || - tok.find("*", 0) != string::npos || - tok.find("/", 0) != string::npos || - tok.find("^", 0) != string::npos || - tok.find("(", 0) != string::npos || - tok.find(")", 0) != string::npos) - { - SPTP ac(new ParseTree(new ArithmeticColumn(tok))); - fFunctionParms.push_back(ac); - } - else if (isdigit(tok[0]) || tok[0] == '.') - { - SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::NUM))); - fFunctionParms.push_back(cc); - } - else if (strcasecmp(tok.c_str(), "NULL") == 0) - { - SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::NULLDATA))); - fFunctionParms.push_back(cc); - } - // map keyword to constant column - else if (strcasecmp(tok.c_str(), "FROM") == 0 || - strcasecmp(tok.c_str(), "TO") == 0) - { - SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::LITERAL))); - fFunctionParms.push_back(cc); - } - // simplecolumn - else - { - SPTP sc(new ParseTree(new SimpleColumn(tok, fSessionID))); - fFunctionParms.push_back(sc); - } + SPTP ac(new ParseTree(new ArithmeticColumn(tok))); + fFunctionParms.push_back(ac); } + else if (isdigit(tok[0]) || tok[0] == '.') + { + SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::NUM))); + fFunctionParms.push_back(cc); + } + else if (strcasecmp(tok.c_str(), "NULL") == 0) + { + SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::NULLDATA))); + fFunctionParms.push_back(cc); + } + // map keyword to constant column + else if (strcasecmp(tok.c_str(), "FROM") == 0 || strcasecmp(tok.c_str(), "TO") == 0) + { + SPTP cc(new ParseTree(new ConstantColumn(tok, ConstantColumn::LITERAL))); + fFunctionParms.push_back(cc); + } + // simplecolumn + else + { + SPTP sc(new ParseTree(new SimpleColumn(tok, fSessionID))); + fFunctionParms.push_back(sc); + } + } } void FunctionColumn::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::FUNCTIONCOLUMN; - ReturnedColumn::serialize(b); - b << fFunctionName; + b << (ObjectReader::id_t)ObjectReader::FUNCTIONCOLUMN; + ReturnedColumn::serialize(b); + b << fFunctionName; - b << static_cast(fFunctionParms.size()); + b << static_cast(fFunctionParms.size()); - for (uint32_t i = 0; i < fFunctionParms.size(); i++) - ObjectReader::writeParseTree(fFunctionParms[i].get(), b); + for (uint32_t i = 0; i < fFunctionParms.size(); i++) + ObjectReader::writeParseTree(fFunctionParms[i].get(), b); - b << fTableAlias; - b << fData; - b << fTimeZone; + b << fTableAlias; + b << fData; + b << fTimeZone; } void FunctionColumn::unserialize(messageqcpp::ByteStream& b) { - uint32_t size, i; - //SRCP rc; - SPTP pt; + uint32_t size, i; + // SRCP rc; + SPTP pt; - fFunctionParms.erase(fFunctionParms.begin(), fFunctionParms.end()); - fSimpleColumnList.clear(); - fAggColumnList.clear(); - fWindowFunctionColumnList.clear(); + fFunctionParms.erase(fFunctionParms.begin(), fFunctionParms.end()); + fSimpleColumnList.clear(); + fAggColumnList.clear(); + fWindowFunctionColumnList.clear(); - ObjectReader::checkType(b, ObjectReader::FUNCTIONCOLUMN); - ReturnedColumn::unserialize(b); - b >> fFunctionName; + ObjectReader::checkType(b, ObjectReader::FUNCTIONCOLUMN); + ReturnedColumn::unserialize(b); + b >> fFunctionName; - b >> size; + b >> size; - for (i = 0; i < size; i++) - { - pt.reset(ObjectReader::createParseTree(b)); - fFunctionParms.push_back(pt); - pt->walk(getSimpleCols, &fSimpleColumnList); - pt->walk(getAggCols, &fAggColumnList); - pt->walk(getWindowFunctionCols, &fWindowFunctionColumnList); - } + for (i = 0; i < size; i++) + { + pt.reset(ObjectReader::createParseTree(b)); + fFunctionParms.push_back(pt); + pt->walk(getSimpleCols, &fSimpleColumnList); + pt->walk(getAggCols, &fAggColumnList); + pt->walk(getWindowFunctionCols, &fWindowFunctionColumnList); + } - b >> fTableAlias; - b >> fData; - b >> fTimeZone; - FuncExp* funcExp = FuncExp::instance(); - fFunctor = funcExp->getFunctor(fFunctionName); - fFunctor->timeZone(fTimeZone); - fFunctor->fix(*this); + b >> fTableAlias; + b >> fData; + b >> fTimeZone; + FuncExp* funcExp = FuncExp::instance(); + fFunctor = funcExp->getFunctor(fFunctionName); + fFunctor->timeZone(fTimeZone); + fFunctor->fix(*this); - // @bug 3506. Special treatment for rand() function. reset the seed - Func_rand* rand = dynamic_cast(fFunctor); - if (rand) - fFunctor = fDynamicFunctor = new Func_rand(); + // @bug 3506. Special treatment for rand() function. reset the seed + Func_rand* rand = dynamic_cast(fFunctor); + if (rand) + fFunctor = fDynamicFunctor = new Func_rand(); - Func_encode* encode = dynamic_cast(fFunctor); - if (encode) - fFunctor = fDynamicFunctor = new Func_encode(); + Func_encode* encode = dynamic_cast(fFunctor); + if (encode) + fFunctor = fDynamicFunctor = new Func_encode(); - Func_decode* decode = dynamic_cast(fFunctor); - if (decode) - fFunctor = fDynamicFunctor = new Func_decode(); + Func_decode* decode = dynamic_cast(fFunctor); + if (decode) + fFunctor = fDynamicFunctor = new Func_decode(); } bool FunctionColumn::operator==(const FunctionColumn& t) const { - // this is not being used. compilation error. + // this is not being used. compilation error. - const ReturnedColumn* rc1, *rc2; - FunctionParm::const_iterator it, it2; + const ReturnedColumn *rc1, *rc2; + FunctionParm::const_iterator it, it2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (fFunctionName != t.fFunctionName) - return false; + if (fFunctionName != t.fFunctionName) + return false; - if (fFunctionParms.size() != t.fFunctionParms.size()) - return false; + if (fFunctionParms.size() != t.fFunctionParms.size()) + return false; - for (it = fFunctionParms.begin(), it2 = t.fFunctionParms.begin(); - it != fFunctionParms.end(); - ++it, ++it2) - if (**it != **it2) - return false; + for (it = fFunctionParms.begin(), it2 = t.fFunctionParms.begin(); it != fFunctionParms.end(); ++it, ++it2) + if (**it != **it2) + return false; - if (fTableAlias != t.fTableAlias) - return false; + if (fTableAlias != t.fTableAlias) + return false; - if (fData != t.fData) - return false; + if (fData != t.fData) + return false; - if (fTimeZone != t.fTimeZone) - return false; + if (fTimeZone != t.fTimeZone) + return false; - return true; + return true; } bool FunctionColumn::operator==(const TreeNode* t) const { - const FunctionColumn* o; + const FunctionColumn* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool FunctionColumn::operator!=(const FunctionColumn& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool FunctionColumn::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } bool FunctionColumn::hasAggregate() { - if (fHasAggregate) return true; + if (fHasAggregate) + return true; - fAggColumnList.clear(); + fAggColumnList.clear(); - for (uint32_t i = 0; i < fFunctionParms.size(); i++) - fFunctionParms[i]->walk(getAggCols, &fAggColumnList); + for (uint32_t i = 0; i < fFunctionParms.size(); i++) + fFunctionParms[i]->walk(getAggCols, &fAggColumnList); - if (!fAggColumnList.empty()) - fHasAggregate = true; + if (!fAggColumnList.empty()) + fHasAggregate = true; - return fHasAggregate; + return fHasAggregate; } bool FunctionColumn::hasWindowFunc() { - fWindowFunctionColumnList.clear(); + fWindowFunctionColumnList.clear(); - for (uint32_t i = 0; i < fFunctionParms.size(); i++) - fFunctionParms[i]->walk(getWindowFunctionCols, &fWindowFunctionColumnList); + for (uint32_t i = 0; i < fFunctionParms.size(); i++) + fFunctionParms[i]->walk(getWindowFunctionCols, &fWindowFunctionColumnList); - if (fWindowFunctionColumnList.empty()) - return false; + if (fWindowFunctionColumnList.empty()) + return false; - return true; + return true; } void FunctionColumn::setDerivedTable() { - if (hasAggregate()) + if (hasAggregate()) + { + fDerivedTable = ""; + return; + } + + setSimpleColumnList(); + string derivedTableAlias = ""; + + for (uint32_t i = 0; i < fSimpleColumnList.size(); i++) + { + SimpleColumn* sc = fSimpleColumnList[i]; + sc->setDerivedTable(); + + if (sc->derivedTable() != derivedTableAlias) { - fDerivedTable = ""; - return; + if (derivedTableAlias == "") + { + derivedTableAlias = sc->tableName(); + } + else + { + derivedTableAlias = ""; + break; + } } - - setSimpleColumnList(); - string derivedTableAlias = ""; - - for (uint32_t i = 0; i < fSimpleColumnList.size(); i++) + // MCOL-3239 Block for func column with both + // derived table column and normal table column. + else if (derivedTableAlias == "") { - SimpleColumn* sc = fSimpleColumnList[i]; - sc->setDerivedTable(); - - if (sc->derivedTable() != derivedTableAlias) - { - if (derivedTableAlias == "") - { - derivedTableAlias = sc->tableName(); - } - else - { - derivedTableAlias = ""; - break; - } - } - // MCOL-3239 Block for func column with both - // derived table column and normal table column. - else if (derivedTableAlias == "") - { - if (sc->tableAlias().length()) - { - derivedTableAlias = ""; - break; - } - } + if (sc->tableAlias().length()) + { + derivedTableAlias = ""; + break; + } } + } - fDerivedTable = derivedTableAlias; + fDerivedTable = derivedTableAlias; } void FunctionColumn::replaceRealCol(CalpontSelectExecutionPlan::ReturnedColumnList& derivedColList) { - for (uint i = 0; i < fFunctionParms.size(); i++) - { - ParseTree* pt = fFunctionParms[i].get(); - replaceRefCol(pt, derivedColList); - } + for (uint i = 0; i < fFunctionParms.size(); i++) + { + ParseTree* pt = fFunctionParms[i].get(); + replaceRefCol(pt, derivedColList); + } } void FunctionColumn::setSimpleColumnList() { - fSimpleColumnList.clear(); + fSimpleColumnList.clear(); - for (uint i = 0; i < fFunctionParms.size(); i++) - fFunctionParms[i]->walk(getSimpleCols, &fSimpleColumnList); + for (uint i = 0; i < fFunctionParms.size(); i++) + fFunctionParms[i]->walk(getSimpleCols, &fSimpleColumnList); } bool FunctionColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) { - tan.clear(); - setSimpleColumnList(); + tan.clear(); + setSimpleColumnList(); - for (uint i = 0; i < fSimpleColumnList.size(); i++) - { - CalpontSystemCatalog::TableAliasName stan(fSimpleColumnList[i]->schemaName(), - fSimpleColumnList[i]->tableName(), - fSimpleColumnList[i]->tableAlias(), - fSimpleColumnList[i]->viewName()); + for (uint i = 0; i < fSimpleColumnList.size(); i++) + { + CalpontSystemCatalog::TableAliasName stan( + fSimpleColumnList[i]->schemaName(), fSimpleColumnList[i]->tableName(), + fSimpleColumnList[i]->tableAlias(), fSimpleColumnList[i]->viewName()); - if (tan.table.empty()) - tan = stan; - else if (stan != tan) - return false; - } + if (tan.table.empty()) + tan = stan; + else if (stan != tan) + return false; + } - return true; + return true; } -} //namespace +} // namespace execplan diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index d5f5b65bb..da37fe1e1 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: functioncolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: functioncolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef FUNCTIONCOLUMN_H @@ -42,7 +42,6 @@ class ByteStream; */ namespace execplan { - /** * @brief A class to represent a functional column * @@ -51,280 +50,280 @@ namespace execplan */ class FunctionColumn : public ReturnedColumn { + public: + FunctionColumn(); + FunctionColumn(std::string& funcName); + FunctionColumn(const std::string& functionName, const std::string& funcParmsInString, + const uint32_t sessionID = 0); + FunctionColumn(const FunctionColumn& rhs, const uint32_t sessionID = 0); + virtual ~FunctionColumn(); -public: - FunctionColumn(); - FunctionColumn(std::string& funcName); - FunctionColumn(const std::string& functionName, const std::string& funcParmsInString, const uint32_t sessionID = 0); - FunctionColumn( const FunctionColumn& rhs, const uint32_t sessionID = 0); - virtual ~FunctionColumn(); + /** get function name + * + * get the function name for this function column + */ + inline const std::string& functionName() const + { + return fFunctionName; + } - /** get function name - * - * get the function name for this function column - */ - inline const std::string& functionName() const + /** set function name + * + * set the function name for this function column + */ + inline void functionName(const std::string functionName) + { + fFunctionName = functionName; + } + + /** get function parameters + * + * get the function parameters for this function column. + * @return a vector of paramenters in string format + */ + inline const funcexp::FunctionParm& functionParms() const + { + return fFunctionParms; + } + + /** set function parameters + * + * set the function parameters for this function column. + */ + void functionParms(const funcexp::FunctionParm& functionParms) + { + fFunctionParms = functionParms; + } + + /** set function parameters + * + * set the function parameters for this function column. + * pass in the functionParms with parenthesis as one string. + * tokenize the string with ' ' or ',' and form a vector of + * parameters in string. + */ + void funcParms(const std::string& funcParmsInString); + + /** get table alias name + * + * get the table alias name for this function + */ + inline const std::string& tableAlias() const + { + return fTableAlias; + } + + /** set table alias name + * + * set the table alias name for this function + */ + inline void tableAlias(const std::string& tableAlias) + { + fTableAlias = tableAlias; + } + + inline const std::string timeZone() const + { + return fTimeZone; + } + + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + + virtual const std::string data() const; + virtual void data(const std::string data) + { + fData = data; + } + + virtual const std::string toString() const; + + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual FunctionColumn* clone() const + { + return new FunctionColumn(*this); + } + + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate(); + virtual bool hasWindowFunc(); + virtual void setDerivedTable(); + virtual void replaceRealCol(std::vector&); + virtual const std::vector& simpleColumnList() const + { + return fSimpleColumnList; + } + + virtual void setSimpleColumnList(); + /** + * Return the tableAlias name of the table that the column arguments belong to. + * + * @param TableAliasName that will be set in the function + * @return true, if all arguments belong to one table + * false, if multiple tables are involved in the function + */ + virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan); + + private: + /** + * Fields + */ + std::string fFunctionName; /// function name + std::string fTableAlias; /// table alias which has the column + std::string fData; /// SQL representation + std::string fTimeZone; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const FunctionColumn& t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const FunctionColumn& t) const; + + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + fResult.strVal = fFunctor->getStrVal(row, fFunctionParms, isNull, fOperationType); + return fResult.strVal; + } + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getIntVal(row, fFunctionParms, isNull, fOperationType); + } + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getUintVal(row, fFunctionParms, isNull, fOperationType); + } + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getFloatVal(row, fFunctionParms, isNull, fOperationType); + } + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getDoubleVal(row, fFunctionParms, isNull, fOperationType); + } + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getLongDoubleVal(row, fFunctionParms, isNull, fOperationType); + } + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); + + if (UNLIKELY(fResultType.colWidth == utils::MAXLEGACYWIDTH && fResultType.scale == decimal.scale)) + return decimal; + + if (LIKELY(fResultType.isWideDecimalType())) { - return fFunctionName; - } + decimal.s128Value = (datatypes::Decimal::isWideDecimalTypeByPrecision(decimal.precision)) + ? decimal.s128Value + : decimal.value; - /** set function name - * - * set the function name for this function column - */ - inline void functionName(const std::string functionName) - { - fFunctionName = functionName; - } + int128_t scaleMultiplier; + int32_t scaleDiff = fResultType.scale - decimal.scale; + datatypes::getScaleDivisor(scaleMultiplier, abs(scaleDiff)); - /** get function parameters - * - * get the function parameters for this function column. - * @return a vector of paramenters in string format - */ - inline const funcexp::FunctionParm& functionParms() const - { - return fFunctionParms; - } - - /** set function parameters - * - * set the function parameters for this function column. - */ - void functionParms(const funcexp::FunctionParm& functionParms) - { - fFunctionParms = functionParms; - } - - /** set function parameters - * - * set the function parameters for this function column. - * pass in the functionParms with parenthesis as one string. - * tokenize the string with ' ' or ',' and form a vector of - * parameters in string. - */ - void funcParms(const std::string& funcParmsInString); - - /** get table alias name - * - * get the table alias name for this function - */ - inline const std::string& tableAlias () const - { - return fTableAlias; - } - - /** set table alias name - * - * set the table alias name for this function - */ - inline void tableAlias (const std::string& tableAlias) - { - fTableAlias = tableAlias; - } - - inline const std::string timeZone () const - { - return fTimeZone; - } - - inline void timeZone (const std::string& timeZone) - { - fTimeZone = timeZone; - } - - virtual const std::string data() const; - virtual void data(const std::string data) - { - fData = data; - } - - virtual const std::string toString() const; - - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual FunctionColumn* clone() const - { - return new FunctionColumn (*this); - } - - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate(); - virtual bool hasWindowFunc(); - virtual void setDerivedTable(); - virtual void replaceRealCol(std::vector&); - virtual const std::vector& simpleColumnList() const - { - return fSimpleColumnList; - } - - virtual void setSimpleColumnList(); - /** - * Return the tableAlias name of the table that the column arguments belong to. - * - * @param TableAliasName that will be set in the function - * @return true, if all arguments belong to one table - * false, if multiple tables are involved in the function - */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan); - -private: - /** - * Fields - */ - std::string fFunctionName; /// function name - std::string fTableAlias; /// table alias which has the column - std::string fData; /// SQL representation - std::string fTimeZone; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const FunctionColumn& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const FunctionColumn& t) const; - - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - fResult.strVal = fFunctor->getStrVal(row, fFunctionParms, isNull, fOperationType); - return fResult.strVal; - } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getIntVal(row, fFunctionParms, isNull, fOperationType); - } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getUintVal(row, fFunctionParms, isNull, fOperationType); - } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getFloatVal(row, fFunctionParms, isNull, fOperationType); - } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getDoubleVal(row, fFunctionParms, isNull, fOperationType); - } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getLongDoubleVal(row, fFunctionParms, isNull, fOperationType); - } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); - - if (UNLIKELY(fResultType.colWidth == utils::MAXLEGACYWIDTH - && fResultType.scale == decimal.scale)) - return decimal; - - if (LIKELY(fResultType.isWideDecimalType())) + if (scaleMultiplier > 1) + { + if (scaleDiff > 0) { - decimal.s128Value = - (datatypes::Decimal::isWideDecimalTypeByPrecision(decimal.precision)) ? - decimal.s128Value : decimal.value; - - int128_t scaleMultiplier; - int32_t scaleDiff = fResultType.scale - decimal.scale; - datatypes::getScaleDivisor(scaleMultiplier, abs(scaleDiff)); - - if (scaleMultiplier > 1) - { - if (scaleDiff > 0) - { - // TODO MCOL-641 Unconditional overflow check - datatypes::MultiplicationNoOverflowCheck mul; - mul(decimal.s128Value, scaleMultiplier, decimal.s128Value); - } - else - { - decimal = decimal.integralWideRound(); - } - } + // TODO MCOL-641 Unconditional overflow check + datatypes::MultiplicationNoOverflowCheck mul; + mul(decimal.s128Value, scaleMultiplier, decimal.s128Value); } - else if (fResultType.colWidth == utils::MAXLEGACYWIDTH) + else { - if (fResultType.scale > decimal.scale) - decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; - else - decimal.value = (int64_t)(decimal.value > 0 ? - (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 : - (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + decimal = decimal.integralWideRound(); } - - decimal.scale = fResultType.scale; - decimal.precision = std::max(fResultType.precision, static_cast(decimal.precision)); - return decimal; + } } - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + else if (fResultType.colWidth == utils::MAXLEGACYWIDTH) { - return fFunctor->getBoolVal(row, fFunctionParms, isNull, fOperationType); - } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getDateIntVal(row, fFunctionParms, isNull, fOperationType); - } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); - } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getTimestampIntVal(row, fFunctionParms, isNull, fOperationType); - } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); + if (fResultType.scale > decimal.scale) + decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; + else + decimal.value = + (int64_t)(decimal.value > 0 + ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 + : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); } - void setFunctor(funcexp::Func* functor) - { - fFunctor = functor; - } -private: - funcexp::FunctionParm fFunctionParms; - funcexp::Func* fFunctor; /// functor to execute this function - funcexp::Func* fDynamicFunctor = NULL; // for rand encode decode - bool fFixed = false; + decimal.scale = fResultType.scale; + decimal.precision = std::max(fResultType.precision, static_cast(decimal.precision)); + return decimal; + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getBoolVal(row, fFunctionParms, isNull, fOperationType); + } + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getDateIntVal(row, fFunctionParms, isNull, fOperationType); + } + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); + } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimestampIntVal(row, fFunctionParms, isNull, fOperationType); + } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); + } + + void setFunctor(funcexp::Func* functor) + { + fFunctor = functor; + } + + private: + funcexp::FunctionParm fFunctionParms; + funcexp::Func* fFunctor; /// functor to execute this function + funcexp::Func* fDynamicFunctor = NULL; // for rand encode decode + bool fFixed = false; }; /** -* ostream operator -*/ + * ostream operator + */ std::ostream& operator<<(std::ostream& output, const FunctionColumn& rhs); -} -#endif //FUNCTIONCOLUMN_H - +} // namespace execplan +#endif // FUNCTIONCOLUMN_H diff --git a/dbcon/execplan/groupconcatcolumn.cpp b/dbcon/execplan/groupconcatcolumn.cpp index b5909d6c5..535733c38 100644 --- a/dbcon/execplan/groupconcatcolumn.cpp +++ b/dbcon/execplan/groupconcatcolumn.cpp @@ -38,24 +38,21 @@ using namespace joblist; namespace execplan { - /** * Constructors/Destructors */ -GroupConcatColumn::GroupConcatColumn(): - AggregateColumn() +GroupConcatColumn::GroupConcatColumn() : AggregateColumn() { } -GroupConcatColumn::GroupConcatColumn(const uint32_t sessionID): - AggregateColumn(sessionID) +GroupConcatColumn::GroupConcatColumn(const uint32_t sessionID) : AggregateColumn(sessionID) { } -GroupConcatColumn::GroupConcatColumn(const GroupConcatColumn& rhs, const uint32_t sessionID): - AggregateColumn(dynamic_cast(rhs)), - fOrderCols(rhs.fOrderCols), - fSeparator(rhs.fSeparator) +GroupConcatColumn::GroupConcatColumn(const GroupConcatColumn& rhs, const uint32_t sessionID) + : AggregateColumn(dynamic_cast(rhs)) + , fOrderCols(rhs.fOrderCols) + , fSeparator(rhs.fSeparator) { } @@ -69,110 +66,110 @@ GroupConcatColumn::~GroupConcatColumn() const string GroupConcatColumn::toString() const { - ostringstream output; - output << "GroupConcatColumn " << data() << endl; - output << AggregateColumn::toString() << endl; - output << "Group Concat Order Columns: " << endl; + ostringstream output; + output << "GroupConcatColumn " << data() << endl; + output << AggregateColumn::toString() << endl; + output << "Group Concat Order Columns: " << endl; - for (uint32_t i = 0; i < fOrderCols.size(); i++) - { - output << *fOrderCols[i]; - } + for (uint32_t i = 0; i < fOrderCols.size(); i++) + { + output << *fOrderCols[i]; + } - output << "\nSeparator: " << fSeparator << endl; - return output.str(); + output << "\nSeparator: " << fSeparator << endl; + return output.str(); } ostream& operator<<(ostream& output, const GroupConcatColumn& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void GroupConcatColumn::serialize(messageqcpp::ByteStream& b) const { - b << (uint8_t) ObjectReader::GROUPCONCATCOLUMN; - AggregateColumn::serialize(b); + b << (uint8_t)ObjectReader::GROUPCONCATCOLUMN; + AggregateColumn::serialize(b); - CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator rcit; - b << static_cast(fOrderCols.size()); + CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator rcit; + b << static_cast(fOrderCols.size()); - for (rcit = fOrderCols.begin(); rcit != fOrderCols.end(); ++rcit) - (*rcit)->serialize(b); + for (rcit = fOrderCols.begin(); rcit != fOrderCols.end(); ++rcit) + (*rcit)->serialize(b); - b << fSeparator; + b << fSeparator; } void GroupConcatColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::GROUPCONCATCOLUMN); - AggregateColumn::unserialize(b); - fOrderCols.erase(fOrderCols.begin(), fOrderCols.end()); + ObjectReader::checkType(b, ObjectReader::GROUPCONCATCOLUMN); + AggregateColumn::unserialize(b); + fOrderCols.erase(fOrderCols.begin(), fOrderCols.end()); - uint32_t size, i; - ReturnedColumn* rc; - b >> size; + uint32_t size, i; + ReturnedColumn* rc; + b >> size; - for (i = 0; i < size; i++) - { - rc = dynamic_cast(ObjectReader::createTreeNode(b)); - SRCP srcp(rc); - fOrderCols.push_back(srcp); - } + for (i = 0; i < size; i++) + { + rc = dynamic_cast(ObjectReader::createTreeNode(b)); + SRCP srcp(rc); + fOrderCols.push_back(srcp); + } - b >> fSeparator; + b >> fSeparator; } bool GroupConcatColumn::operator==(const GroupConcatColumn& t) const { - const AggregateColumn* rc1, *rc2; + const AggregateColumn *rc1, *rc2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - for (uint32_t i = 0; i < fOrderCols.size(); i++) + for (uint32_t i = 0; i < fOrderCols.size(); i++) + { + if (fOrderCols[i].get() != NULL) { - if (fOrderCols[i].get() != NULL) - { - if (t.fOrderCols[i] == NULL) - return false; - - if (*(fOrderCols[i].get()) != t.fOrderCols[i].get()) - return false; - } - else if (t.fOrderCols[i].get() != NULL) - return false; - } - - if (fSeparator != t.fSeparator) + if (t.fOrderCols[i] == NULL) return false; - return true; + if (*(fOrderCols[i].get()) != t.fOrderCols[i].get()) + return false; + } + else if (t.fOrderCols[i].get() != NULL) + return false; + } + + if (fSeparator != t.fSeparator) + return false; + + return true; } bool GroupConcatColumn::operator==(const TreeNode* t) const { - const GroupConcatColumn* ac; + const GroupConcatColumn* ac; - ac = dynamic_cast(t); + ac = dynamic_cast(t); - if (ac == NULL) - return false; + if (ac == NULL) + return false; - return *this == *ac; + return *this == *ac; } bool GroupConcatColumn::operator!=(const GroupConcatColumn& t) const { - return !(*this == t); + return !(*this == t); } bool GroupConcatColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/groupconcatcolumn.h b/dbcon/execplan/groupconcatcolumn.h index 8a1982e74..251575650 100644 --- a/dbcon/execplan/groupconcatcolumn.h +++ b/dbcon/execplan/groupconcatcolumn.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: groupconcatcolumn.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: groupconcatcolumn.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef GROUPCONCATCOLUMN_H @@ -47,107 +47,104 @@ namespace execplan */ class GroupConcatColumn : public AggregateColumn { + public: + /** + * Constructors + */ + GroupConcatColumn(); -public: + GroupConcatColumn(const uint32_t sessionID); - /** - * Constructors - */ - GroupConcatColumn(); + GroupConcatColumn(const GroupConcatColumn& rhs, const uint32_t sessionID = 0); - GroupConcatColumn(const uint32_t sessionID); + /** + * Destructors + */ + virtual ~GroupConcatColumn(); - GroupConcatColumn(const GroupConcatColumn& rhs, const uint32_t sessionID = 0); + /** + * Overloaded stream operator + */ + virtual const std::string toString() const; - /** - * Destructors - */ - virtual ~GroupConcatColumn(); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + virtual GroupConcatColumn* clone() const + { + return new GroupConcatColumn(*this); + } - /** - * Overloaded stream operator - */ - virtual const std::string toString() const; + /** + * Accessors and Mutators + */ + void orderCols(const std::vector& orderCols) + { + fOrderCols = orderCols; + } + std::vector& orderCols() + { + return fOrderCols; + } + void separator(const std::string& separator) + { + fSeparator = separator; + } + std::string& separator() + { + return fSeparator; + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - virtual GroupConcatColumn* clone() const - { - return new GroupConcatColumn(*this); - } + /** + * Serialize interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * Accessors and Mutators - */ - void orderCols(const std::vector& orderCols) - { - fOrderCols = orderCols; - } - std::vector& orderCols() - { - return fOrderCols; - } - void separator(const std::string& separator) - { - fSeparator = separator; - } - std::string& separator() - { - return fSeparator; - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; + * false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** - * Serialize interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; + * false otherwise + */ + using AggregateColumn::operator==; + virtual bool operator==(const GroupConcatColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; - * false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; + * true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; - * false otherwise - */ - using AggregateColumn::operator==; - virtual bool operator==(const GroupConcatColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; + * true otherwise + */ + using AggregateColumn::operator!=; + virtual bool operator!=(const GroupConcatColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; - * true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; - * true otherwise - */ - using AggregateColumn::operator!=; - virtual bool operator!=(const GroupConcatColumn& t) const; - -private: - std::vector fOrderCols; - std::string fSeparator; + private: + std::vector fOrderCols; + std::string fSeparator; }; /** -* stream operator -*/ + * stream operator + */ std::ostream& operator<<(std::ostream& os, const GroupConcatColumn& rhs); -} -#endif //GROUPCONCATCOLUMN_H - +} // namespace execplan +#endif // GROUPCONCATCOLUMN_H diff --git a/dbcon/execplan/intervalcolumn.cpp b/dbcon/execplan/intervalcolumn.cpp index 838f32236..c3fcec936 100644 --- a/dbcon/execplan/intervalcolumn.cpp +++ b/dbcon/execplan/intervalcolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: intervalcolumn.cpp 9414 2013-04-22 22:18:30Z xlou $ -* -* -***********************************************************************/ + * $Id: intervalcolumn.cpp 9414 2013-04-22 22:18:30Z xlou $ + * + * + ***********************************************************************/ #include #include @@ -39,44 +39,42 @@ using namespace funcexp; namespace execplan { - /** * Constructors/Destructors */ IntervalColumn::IntervalColumn() -{} - -IntervalColumn::IntervalColumn(SRCP& val, int intervalType): - fVal(val->clone()), fIntervalType(intervalType) { -// cout << "intervalType=" << fIntervalType << endl; } -IntervalColumn::IntervalColumn( const IntervalColumn& rhs, const uint32_t sessionID): - ReturnedColumn(rhs, sessionID), - fVal(rhs.val()), - fIntervalType(rhs.intervalType()) -{} +IntervalColumn::IntervalColumn(SRCP& val, int intervalType) : fVal(val->clone()), fIntervalType(intervalType) +{ + // cout << "intervalType=" << fIntervalType << endl; +} + +IntervalColumn::IntervalColumn(const IntervalColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID), fVal(rhs.val()), fIntervalType(rhs.intervalType()) +{ +} /** * Methods */ const string IntervalColumn::toString() const { - ostringstream output; - output << "INTERVAL" << endl; + ostringstream output; + output << "INTERVAL" << endl; - if (fVal) - output << fVal->toString(); + if (fVal) + output << fVal->toString(); - output << " IntervalType=" << fIntervalType << endl; - return output.str(); + output << " IntervalType=" << fIntervalType << endl; + return output.str(); } ostream& operator<<(ostream& output, const IntervalColumn& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } -} //namespace +} // namespace execplan diff --git a/dbcon/execplan/intervalcolumn.h b/dbcon/execplan/intervalcolumn.h index 6c6765422..63e36d61d 100644 --- a/dbcon/execplan/intervalcolumn.h +++ b/dbcon/execplan/intervalcolumn.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: intervalcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: intervalcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ @@ -50,80 +50,95 @@ namespace execplan */ class IntervalColumn : public ReturnedColumn { + public: + /* from my_time.h */ + enum interval_type + { + INTERVAL_YEAR, + INTERVAL_QUARTER, + INTERVAL_MONTH, + INTERVAL_WEEK, + INTERVAL_DAY, + INTERVAL_HOUR, + INTERVAL_MINUTE, + INTERVAL_SECOND, + INTERVAL_MICROSECOND, + INTERVAL_YEAR_MONTH, + INTERVAL_DAY_HOUR, + INTERVAL_DAY_MINUTE, + INTERVAL_DAY_SECOND, + INTERVAL_HOUR_MINUTE, + INTERVAL_HOUR_SECOND, + INTERVAL_MINUTE_SECOND, + INTERVAL_DAY_MICROSECOND, + INTERVAL_HOUR_MICROSECOND, + INTERVAL_MINUTE_MICROSECOND, + INTERVAL_SECOND_MICROSECOND, + INTERVAL_LAST + }; -public: + IntervalColumn(); + IntervalColumn(SRCP&, int); + IntervalColumn(const IntervalColumn& rhs, const uint32_t sessionID = 0); + virtual ~IntervalColumn() + { + } + const SRCP& val() const + { + return fVal; + } + void val(const SRCP& val) + { + fVal = val; + } + int intervalType() const + { + return fIntervalType; + } + void intervalType(int intervalType) + { + fIntervalType = intervalType; + } + const std::string toString() const; + inline virtual IntervalColumn* clone() const + { + return new IntervalColumn(*this); + } - /* from my_time.h */ - enum interval_type - { - INTERVAL_YEAR, INTERVAL_QUARTER, INTERVAL_MONTH, INTERVAL_WEEK, INTERVAL_DAY, - INTERVAL_HOUR, INTERVAL_MINUTE, INTERVAL_SECOND, INTERVAL_MICROSECOND, - INTERVAL_YEAR_MONTH, INTERVAL_DAY_HOUR, INTERVAL_DAY_MINUTE, - INTERVAL_DAY_SECOND, INTERVAL_HOUR_MINUTE, INTERVAL_HOUR_SECOND, - INTERVAL_MINUTE_SECOND, INTERVAL_DAY_MICROSECOND, INTERVAL_HOUR_MICROSECOND, - INTERVAL_MINUTE_MICROSECOND, INTERVAL_SECOND_MICROSECOND, INTERVAL_LAST - }; + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate() + { + return false; + } + virtual bool hasWindowFunc() + { + return false; + } - IntervalColumn(); - IntervalColumn(SRCP&, int); - IntervalColumn(const IntervalColumn& rhs, const uint32_t sessionID = 0); - virtual ~IntervalColumn() {} - const SRCP& val() const - { - return fVal; - } - void val(const SRCP& val) - { - fVal = val; - } - int intervalType() const - { - return fIntervalType; - } - void intervalType(int intervalType) - { - fIntervalType = intervalType; - } - const std::string toString() const; - inline virtual IntervalColumn* clone() const - { - return new IntervalColumn (*this); - } + private: + /** + * Fields + */ + SRCP fVal; + int fIntervalType; - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() - { - return false; - } - virtual bool hasWindowFunc() - { - return false; - } - -private: - /** - * Fields - */ - SRCP fVal; - int fIntervalType; - - // okay to be private for now. - virtual bool operator==(const TreeNode* t) const - { - return false; - } - bool operator==(const IntervalColumn& t) const; - virtual bool operator!=(const TreeNode* t) const - { - return false; - } - bool operator!=(const IntervalColumn& t) const; + // okay to be private for now. + virtual bool operator==(const TreeNode* t) const + { + return false; + } + bool operator==(const IntervalColumn& t) const; + virtual bool operator!=(const TreeNode* t) const + { + return false; + } + bool operator!=(const IntervalColumn& t) const; }; /** -* ostream operator -*/ + * ostream operator + */ std::ostream& operator<<(std::ostream& output, const IntervalColumn& rhs); -} -#endif //INTERVALCOLUMN_H +} // namespace execplan +#endif // INTERVALCOLUMN_H diff --git a/dbcon/execplan/logicoperator.cpp b/dbcon/execplan/logicoperator.cpp index 662653195..6a5e5907f 100644 --- a/dbcon/execplan/logicoperator.cpp +++ b/dbcon/execplan/logicoperator.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: logicoperator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: logicoperator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include "bytestream.h" @@ -30,7 +30,6 @@ using namespace std; namespace execplan { - /** * Constructors/Destructors */ @@ -40,15 +39,15 @@ LogicOperator::LogicOperator() LogicOperator::LogicOperator(const string& operatorName) { - data(operatorName); + data(operatorName); } LogicOperator::LogicOperator(const LogicOperator& rhs) : Operator(rhs) { - data(rhs.fData); + data(rhs.fData); } -LogicOperator:: ~LogicOperator() +LogicOperator::~LogicOperator() { } @@ -56,14 +55,13 @@ LogicOperator:: ~LogicOperator() * Operations */ - /** * friend function */ ostream& operator<<(ostream& output, const LogicOperator& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } /** @@ -71,46 +69,46 @@ ostream& operator<<(ostream& output, const LogicOperator& rhs) */ void LogicOperator::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::LOGICOPERATOR; - //b << fData; - Operator::serialize(b); + b << (ObjectReader::id_t)ObjectReader::LOGICOPERATOR; + // b << fData; + Operator::serialize(b); } void LogicOperator::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::LOGICOPERATOR); - //b >> fData; - Operator::unserialize(b); + ObjectReader::checkType(b, ObjectReader::LOGICOPERATOR); + // b >> fData; + Operator::unserialize(b); } bool LogicOperator::operator==(const LogicOperator& t) const { - if (data() == t.data()) - return true; + if (data() == t.data()) + return true; - return false; + return false; } bool LogicOperator::operator==(const TreeNode* t) const { - const LogicOperator* o; + const LogicOperator* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool LogicOperator::operator!=(const LogicOperator& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool LogicOperator::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } -} // namespace +} // namespace execplan diff --git a/dbcon/execplan/logicoperator.h b/dbcon/execplan/logicoperator.h index 74cbc0cf8..31bc715a3 100644 --- a/dbcon/execplan/logicoperator.h +++ b/dbcon/execplan/logicoperator.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: logicoperator.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: logicoperator.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef LOGICOPERATOR_H @@ -57,134 +57,129 @@ class ParseTree; class LogicOperator : public Operator { + /** + * Public stuff + */ + public: + /** + * Constructors + */ + LogicOperator(); + LogicOperator(const std::string& operatorName); + LogicOperator(const LogicOperator& rhs); - /** - * Public stuff - */ -public: - /** - * Constructors - */ - LogicOperator(); - LogicOperator(const std::string& operatorName); - LogicOperator(const LogicOperator& rhs); + /** + * Destructors + */ + virtual ~LogicOperator(); - /** - * Destructors - */ - virtual ~LogicOperator(); + /** + * Accessor Methods + */ - /** - * Accessor Methods - */ + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual LogicOperator* clone() const + { + return new LogicOperator(*this); + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual LogicOperator* clone() const + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const LogicOperator& t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const LogicOperator& t) const; + // template + // result_t evaluate(result_t op1, result_t op2); + + // F&E framework + using Operator::getBoolVal; + inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + switch (fOp) { - return new LogicOperator (*this); + case OP_AND: return (lop->getBoolVal(row, isNull) && rop->getBoolVal(row, isNull)); + + case OP_OR: + { + if (lop->getBoolVal(row, isNull)) + return true; + + isNull = false; + // return (lop->getBoolVal(row, isNull) || rop->getBoolVal(row, isNull)); + return rop->getBoolVal(row, isNull); + } + + case OP_XOR: + { + // Logical XOR. Returns NULL if either operand is NULL. + // For non-NULL operands, evaluates to 1 if an odd number of operands is nonzero, + // otherwise 0 is returned. + bool lopv = lop->getBoolVal(row, isNull); + + if (isNull) + return false; + + bool ropv = rop->getBoolVal(row, isNull); + + if (isNull) + return false; + + if ((lopv && !ropv) || (ropv && !lopv)) + return true; + else + return false; + } + + default: throw std::runtime_error("invalid logical operation"); } + } - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const LogicOperator& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const LogicOperator& t) const; - //template - //result_t evaluate(result_t op1, result_t op2); - - // F&E framework - using Operator::getBoolVal; - inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - switch (fOp) - { - case OP_AND: - return (lop->getBoolVal(row, isNull) && rop->getBoolVal(row, isNull)); - - case OP_OR: - { - if (lop->getBoolVal(row, isNull)) - return true; - - isNull = false; - //return (lop->getBoolVal(row, isNull) || rop->getBoolVal(row, isNull)); - return rop->getBoolVal(row, isNull); - } - - case OP_XOR: - { - // Logical XOR. Returns NULL if either operand is NULL. - // For non-NULL operands, evaluates to 1 if an odd number of operands is nonzero, - // otherwise 0 is returned. - bool lopv = lop->getBoolVal(row, isNull); - - if (isNull) - return false; - - bool ropv = rop->getBoolVal(row, isNull); - - if (isNull) - return false; - - if ((lopv && !ropv) || (ropv && !lopv)) - return true; - else - return false; - } - - default: - throw std::runtime_error("invalid logical operation"); - } - } - - using TreeNode::evaluate; - inline virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - fResult.boolVal = getBoolVal(row, isNull, lop, rop); - } - -private: - // default okay - //Operator& operator=(const Operator& rhs); - //std::string fData; + using TreeNode::evaluate; + inline virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + fResult.boolVal = getBoolVal(row, isNull, lop, rop); + } + private: + // default okay + // Operator& operator=(const Operator& rhs); + // std::string fData; }; -//typedef boost::shared_ptr SOP; +// typedef boost::shared_ptr SOP; std::ostream& operator<<(std::ostream& os, const LogicOperator& rhs); -} +} // namespace execplan #endif - diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.cpp b/dbcon/execplan/mcsanalyzetableexecutionplan.cpp index a383dc88f..982135ed9 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.cpp +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.cpp @@ -36,121 +36,120 @@ using namespace querytele; namespace execplan { - std::string MCSAnalyzeTableExecutionPlan::toString() const { - std::ostringstream output; - output << ">ANALYZE TABLE " << std::endl; - output << "Shema: " << fSchemaName << std::endl; - output << "Table: " << fTableName << std::endl; + std::ostringstream output; + output << ">ANALYZE TABLE " << std::endl; + output << "Shema: " << fSchemaName << std::endl; + output << "Table: " << fTableName << std::endl; - output << ">>Returned Columns" << std::endl; + output << ">>Returned Columns" << std::endl; - for (auto& rColumn : fReturnedCols) - output << *rColumn << std::endl; + for (auto& rColumn : fReturnedCols) + output << *rColumn << std::endl; - output << "--- Column Map ---" << std::endl; - CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + output << "--- Column Map ---" << std::endl; + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; - for (iter = columnMap().begin(); iter != columnMap().end(); iter++) - output << (*iter).first << " : " << (*iter).second << endl; + for (iter = columnMap().begin(); iter != columnMap().end(); iter++) + output << (*iter).first << " : " << (*iter).second << endl; - output << "SessionID: " << fSessionID << std::endl; - output << "TxnID: " << fTxnID << std::endl; - output << "VerID: " << fVerID << std::endl; + output << "SessionID: " << fSessionID << std::endl; + output << "TxnID: " << fTxnID << std::endl; + output << "VerID: " << fVerID << std::endl; - return output.str(); + return output.str(); } void MCSAnalyzeTableExecutionPlan::serialize(messageqcpp::ByteStream& bs) const { - bs << static_cast(ObjectReader::MCSANALYZETBLEXECUTIONPLAN); + bs << static_cast(ObjectReader::MCSANALYZETBLEXECUTIONPLAN); - // Returned columns. - bs << static_cast(fReturnedCols.size()); - for (auto& rColumn : fReturnedCols) - rColumn->serialize(bs); + // Returned columns. + bs << static_cast(fReturnedCols.size()); + for (auto& rColumn : fReturnedCols) + rColumn->serialize(bs); - // Column map. - bs << static_cast(fColumnMap.size()); - for (auto& column : fColumnMap) - { - bs << column.first; - column.second->serialize(bs); - } + // Column map. + bs << static_cast(fColumnMap.size()); + for (auto& column : fColumnMap) + { + bs << column.first; + column.second->serialize(bs); + } - bs << static_cast(frmParms.size()); + bs << static_cast(frmParms.size()); - for (RMParmVec::const_iterator it = frmParms.begin(); it != frmParms.end(); ++it) - { - bs << it->sessionId; - bs << it->id; - bs << it->value; - } + for (RMParmVec::const_iterator it = frmParms.begin(); it != frmParms.end(); ++it) + { + bs << it->sessionId; + bs << it->id; + bs << it->value; + } - bs << fData; - bs << static_cast(fSessionID); - bs << static_cast(fTxnID); - bs << fVerID; - bs << fStatementID; - bs << static_cast(fStringScanThreshold); - bs << fPriority; - bs << fSchemaName; - bs << fTableName; - bs << fLocalQuery; - bs << fTimeZone; - bs << fTraceFlags; + bs << fData; + bs << static_cast(fSessionID); + bs << static_cast(fTxnID); + bs << fVerID; + bs << fStatementID; + bs << static_cast(fStringScanThreshold); + bs << fPriority; + bs << fSchemaName; + bs << fTableName; + bs << fLocalQuery; + bs << fTimeZone; + bs << fTraceFlags; } void MCSAnalyzeTableExecutionPlan::unserialize(messageqcpp::ByteStream& bs) { - ObjectReader::checkType(bs, ObjectReader::MCSANALYZETBLEXECUTIONPLAN); - fReturnedCols.clear(); - fColumnMap.clear(); - uint32_t size; + ObjectReader::checkType(bs, ObjectReader::MCSANALYZETBLEXECUTIONPLAN); + fReturnedCols.clear(); + fColumnMap.clear(); + uint32_t size; - bs >> size; - for (uint32_t i = 0; i < size; ++i) - { - auto* returnedColumn = dynamic_cast(ObjectReader::createTreeNode(bs)); - SRCP srcp(returnedColumn); - fReturnedCols.push_back(srcp); - } + bs >> size; + for (uint32_t i = 0; i < size; ++i) + { + auto* returnedColumn = dynamic_cast(ObjectReader::createTreeNode(bs)); + SRCP srcp(returnedColumn); + fReturnedCols.push_back(srcp); + } - bs >> size; - for (uint32_t i = 0; i < size; ++i) - { - std::string colName; - bs >> colName; - auto* returnedColumn = dynamic_cast(ObjectReader::createTreeNode(bs)); - SRCP srcp(returnedColumn); - fColumnMap.insert(ColumnMap::value_type(colName, srcp)); - } + bs >> size; + for (uint32_t i = 0; i < size; ++i) + { + std::string colName; + bs >> colName; + auto* returnedColumn = dynamic_cast(ObjectReader::createTreeNode(bs)); + SRCP srcp(returnedColumn); + fColumnMap.insert(ColumnMap::value_type(colName, srcp)); + } - bs >> size; - messageqcpp::ByteStream::doublebyte id; - messageqcpp::ByteStream::quadbyte sessionId; - messageqcpp::ByteStream::octbyte memory; + bs >> size; + messageqcpp::ByteStream::doublebyte id; + messageqcpp::ByteStream::quadbyte sessionId; + messageqcpp::ByteStream::octbyte memory; - for (uint32_t i = 0; i < size; i++) - { - bs >> sessionId; - bs >> id; - bs >> memory; - frmParms.push_back(RMParam(sessionId, id, memory)); - } + for (uint32_t i = 0; i < size; i++) + { + bs >> sessionId; + bs >> id; + bs >> memory; + frmParms.push_back(RMParam(sessionId, id, memory)); + } - bs >> fData; - bs >> reinterpret_cast(fSessionID); - bs >> reinterpret_cast(fTxnID); - bs >> fVerID; - bs >> fStatementID; - bs >> reinterpret_cast(fStringScanThreshold); - bs >> fPriority; - bs >> fSchemaName; - bs >> fTableName; - bs >> fLocalQuery; - bs >> fTimeZone; - bs >> fTraceFlags; + bs >> fData; + bs >> reinterpret_cast(fSessionID); + bs >> reinterpret_cast(fTxnID); + bs >> fVerID; + bs >> fStatementID; + bs >> reinterpret_cast(fStringScanThreshold); + bs >> fPriority; + bs >> fSchemaName; + bs >> fTableName; + bs >> fLocalQuery; + bs >> fTimeZone; + bs >> fTraceFlags; } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.h b/dbcon/execplan/mcsanalyzetableexecutionplan.h index 4090174be..28aad9cdd 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.h +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.h @@ -39,132 +39,231 @@ */ namespace execplan { - class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan { - public: - typedef std::vector ReturnedColumnList; - typedef std::multimap ColumnMap; - typedef std::vector RMParmVec; + public: + typedef std::vector ReturnedColumnList; + typedef std::multimap ColumnMap; + typedef std::vector RMParmVec; - MCSAnalyzeTableExecutionPlan() : fTraceFlags(TRACE_NONE) {} + MCSAnalyzeTableExecutionPlan() : fTraceFlags(TRACE_NONE) + { + } - MCSAnalyzeTableExecutionPlan(const ReturnedColumnList& returnedCols, - const ColumnMap& columnMap) - : fReturnedCols(returnedCols), fColumnMap(columnMap), fTraceFlags(TRACE_NONE) - { - } + MCSAnalyzeTableExecutionPlan(const ReturnedColumnList& returnedCols, const ColumnMap& columnMap) + : fReturnedCols(returnedCols), fColumnMap(columnMap), fTraceFlags(TRACE_NONE) + { + } - virtual ~MCSAnalyzeTableExecutionPlan() = default; + virtual ~MCSAnalyzeTableExecutionPlan() = default; - const ReturnedColumnList& returnedCols() const { return fReturnedCols; } - ReturnedColumnList& returnedCols() { return fReturnedCols; } + const ReturnedColumnList& returnedCols() const + { + return fReturnedCols; + } + ReturnedColumnList& returnedCols() + { + return fReturnedCols; + } - void returnedCols(const ReturnedColumnList& returnedCols) { fReturnedCols = returnedCols; } + void returnedCols(const ReturnedColumnList& returnedCols) + { + fReturnedCols = returnedCols; + } - const ColumnMap& columnMap() const { return fColumnMap; } + const ColumnMap& columnMap() const + { + return fColumnMap; + } - ColumnMap& columnMap() { return fColumnMap; } + ColumnMap& columnMap() + { + return fColumnMap; + } - void columnMap(const ColumnMap& columnMap) { fColumnMap = columnMap; } + void columnMap(const ColumnMap& columnMap) + { + fColumnMap = columnMap; + } - const std::string data() const { return fData; } + const std::string data() const + { + return fData; + } - void data(const std::string data) { fData = data; } + void data(const std::string data) + { + fData = data; + } - uint32_t sessionID() const { return fSessionID; } + uint32_t sessionID() const + { + return fSessionID; + } - void sessionID(const uint32_t sessionID) { fSessionID = sessionID; } + void sessionID(const uint32_t sessionID) + { + fSessionID = sessionID; + } - int txnID() const { return fTxnID; } + int txnID() const + { + return fTxnID; + } - void txnID(const int txnID) { fTxnID = txnID; } + void txnID(const int txnID) + { + fTxnID = txnID; + } - const BRM::QueryContext verID() const { return fVerID; } + const BRM::QueryContext verID() const + { + return fVerID; + } - void verID(const BRM::QueryContext verID) { fVerID = verID; } + void verID(const BRM::QueryContext verID) + { + fVerID = verID; + } - inline std::string& schemaName() { return fSchemaName; } + inline std::string& schemaName() + { + return fSchemaName; + } - inline void schemaName(const std::string& schemaName, int lower_case_table_names) - { - fSchemaName = schemaName; - if (lower_case_table_names) - boost::algorithm::to_lower(fSchemaName); - } + inline void schemaName(const std::string& schemaName, int lower_case_table_names) + { + fSchemaName = schemaName; + if (lower_case_table_names) + boost::algorithm::to_lower(fSchemaName); + } - inline std::string& tableName() { return fTableName; } + inline std::string& tableName() + { + return fTableName; + } - inline void tableName(const std::string& tableName, int lower_case_table_names) - { - fTableName = tableName; - if (lower_case_table_names) - boost::algorithm::to_lower(fTableName); - } + inline void tableName(const std::string& tableName, int lower_case_table_names) + { + fTableName = tableName; + if (lower_case_table_names) + boost::algorithm::to_lower(fTableName); + } - uint32_t statementID() const { return fStatementID; } + uint32_t statementID() const + { + return fStatementID; + } - void statementID(const uint32_t statementID) { fStatementID = statementID; } + void statementID(const uint32_t statementID) + { + fStatementID = statementID; + } - void uuid(const boost::uuids::uuid& uuid) { fUuid = uuid; } + void uuid(const boost::uuids::uuid& uuid) + { + fUuid = uuid; + } - const boost::uuids::uuid& uuid() const { return fUuid; } + const boost::uuids::uuid& uuid() const + { + return fUuid; + } - void timeZone(const std::string& timezone) { fTimeZone = timezone; } + void timeZone(const std::string& timezone) + { + fTimeZone = timezone; + } - const std::string timeZone() const { return fTimeZone; } + const std::string timeZone() const + { + return fTimeZone; + } - void priority(uint32_t p) { fPriority = p; } + void priority(uint32_t p) + { + fPriority = p; + } - uint32_t priority() const { return fPriority; } + uint32_t priority() const + { + return fPriority; + } - const RMParmVec& rmParms() { return frmParms; } + const RMParmVec& rmParms() + { + return frmParms; + } - void rmParms(const RMParmVec& parms) - { - frmParms.clear(); - frmParms.assign(parms.begin(), parms.end()); - } + void rmParms(const RMParmVec& parms) + { + frmParms.clear(); + frmParms.assign(parms.begin(), parms.end()); + } - uint32_t localQuery() const { return fLocalQuery; } + uint32_t localQuery() const + { + return fLocalQuery; + } - void localQuery(const uint32_t localQuery) { fLocalQuery = localQuery; } + void localQuery(const uint32_t localQuery) + { + fLocalQuery = localQuery; + } - inline bool traceOn() const { return (traceFlags() & TRACE_LOG); } + inline bool traceOn() const + { + return (traceFlags() & TRACE_LOG); + } - inline uint32_t traceFlags() const { return fTraceFlags; } + inline uint32_t traceFlags() const + { + return fTraceFlags; + } - inline void traceFlags(uint32_t traceFlags) { fTraceFlags = traceFlags; } + inline void traceFlags(uint32_t traceFlags) + { + fTraceFlags = traceFlags; + } - virtual std::string toString() const; + virtual std::string toString() const; - virtual bool isInternal() const { return ((fSessionID & 0x80000000) != 0); } + virtual bool isInternal() const + { + return ((fSessionID & 0x80000000) != 0); + } - virtual void serialize(messageqcpp::ByteStream& bs) const; + virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + virtual void unserialize(messageqcpp::ByteStream& bs); - // TODO: Why do we need this? - virtual bool operator==(const CalpontExecutionPlan* t) const { return false; } - virtual bool operator!=(const CalpontExecutionPlan* t) const { return false; } + // TODO: Why do we need this? + virtual bool operator==(const CalpontExecutionPlan* t) const + { + return false; + } + virtual bool operator!=(const CalpontExecutionPlan* t) const + { + return false; + } - private: - ReturnedColumnList fReturnedCols; - ColumnMap fColumnMap; - uint32_t fSessionID; - int fTxnID; - BRM::QueryContext fVerID; - std::string fSchemaName; - std::string fTableName; - uint32_t fTraceFlags; - boost::uuids::uuid fUuid; - std::string fTimeZone; - uint32_t fStatementID; - uint64_t fStringScanThreshold; - std::string fData; - RMParmVec frmParms; - uint32_t fPriority; - uint32_t fLocalQuery; + private: + ReturnedColumnList fReturnedCols; + ColumnMap fColumnMap; + uint32_t fSessionID; + int fTxnID; + BRM::QueryContext fVerID; + std::string fSchemaName; + std::string fTableName; + uint32_t fTraceFlags; + boost::uuids::uuid fUuid; + std::string fTimeZone; + uint32_t fStatementID; + uint64_t fStringScanThreshold; + std::string fData; + RMParmVec frmParms; + uint32_t fPriority; + uint32_t fLocalQuery; }; -} // namespace execplan +} // namespace execplan #endif diff --git a/dbcon/execplan/mysqlexecutionplan.cpp b/dbcon/execplan/mysqlexecutionplan.cpp index 4560ef358..1c6fe5bd1 100644 --- a/dbcon/execplan/mysqlexecutionplan.cpp +++ b/dbcon/execplan/mysqlexecutionplan.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: mysqlexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: mysqlexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "mysqlexecutionplan.h" @@ -29,4 +29,3 @@ /** * Methods */ - diff --git a/dbcon/execplan/mysqlexecutionplan.h b/dbcon/execplan/mysqlexecutionplan.h index 3ed8d5c92..b9999507b 100644 --- a/dbcon/execplan/mysqlexecutionplan.h +++ b/dbcon/execplan/mysqlexecutionplan.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: mysqlexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: mysqlexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef MYSQLEXECUTIONPLAN_H @@ -38,64 +38,63 @@ namespace execplan */ class MySQLExecutionPlan : public VendorExecutionPlan { - /** - * Public stuff - */ -public: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Protected stuff - */ -protected: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Private stuff - */ -private: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ + /** + * Public stuff + */ + public: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Protected stuff + */ + protected: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Private stuff + */ + private: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ }; -} -#endif //MYSQLEXECUTIONPLAN_H - +} // namespace execplan +#endif // MYSQLEXECUTIONPLAN_H diff --git a/dbcon/execplan/njlcolumnresult.h b/dbcon/execplan/njlcolumnresult.h index d75013675..64b2464d7 100644 --- a/dbcon/execplan/njlcolumnresult.h +++ b/dbcon/execplan/njlcolumnresult.h @@ -28,7 +28,6 @@ namespace execplan { - typedef ColumnResult NJLColumnResult; } diff --git a/dbcon/execplan/objectidmanager.cpp b/dbcon/execplan/objectidmanager.cpp index ea731b7c6..390e504d8 100644 --- a/dbcon/execplan/objectidmanager.cpp +++ b/dbcon/execplan/objectidmanager.cpp @@ -74,16 +74,16 @@ using namespace std; #include "dbrm.h" #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_DIRECT -# define O_DIRECT 0 +#define O_DIRECT 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif namespace @@ -93,27 +93,26 @@ boost::mutex CtorMutex; namespace execplan { - ObjectIDManager::ObjectIDManager() { - boost::mutex::scoped_lock lk(CtorMutex); + boost::mutex::scoped_lock lk(CtorMutex); - config::Config* conf; - string tmp; + config::Config* conf; + string tmp; - conf = config::Config::makeConfig(); + conf = config::Config::makeConfig(); - try - { - fFilename = conf->getConfig("OIDManager", "OIDBitmapFile"); - } - catch (exception&) - { - fFilename = "/mnt/OAM/dbrm/oidbitmap"; - } + try + { + fFilename = conf->getConfig("OIDManager", "OIDBitmapFile"); + } + catch (exception&) + { + fFilename = "/mnt/OAM/dbrm/oidbitmap"; + } - if (fFilename.empty()) - fFilename = "/mnt/OAM/dbrm/oidbitmap"; + if (fFilename.empty()) + fFilename = "/mnt/OAM/dbrm/oidbitmap"; } ObjectIDManager::~ObjectIDManager() @@ -122,49 +121,48 @@ ObjectIDManager::~ObjectIDManager() int ObjectIDManager::allocOID() { - return allocOIDs(1); + return allocOIDs(1); } int ObjectIDManager::allocVBOID(uint32_t dbroot) { - return dbrm.allocVBOID(dbroot); + return dbrm.allocVBOID(dbroot); } int ObjectIDManager::getDBRootOfVBOID(uint32_t vboid) { - return dbrm.getDBRootOfVBOID(vboid); + return dbrm.getDBRootOfVBOID(vboid); } int ObjectIDManager::allocOIDs(int num) { - return dbrm.allocOIDs(num); + return dbrm.allocOIDs(num); } void ObjectIDManager::returnOID(int oid) { - returnOIDs(oid, oid); + returnOIDs(oid, oid); } void ObjectIDManager::returnOIDs(int start, int end) { - //@Bug 1412. Do not reuse oids for now. - return; + //@Bug 1412. Do not reuse oids for now. + return; } const string ObjectIDManager::getFilename() const { - return fFilename; + return fFilename; } int ObjectIDManager::size() { - return dbrm.oidm_size(); + return dbrm.oidm_size(); } vector ObjectIDManager::getVBOIDToDBRootMap() { - return dbrm.getVBOIDToDBRootMap(); + return dbrm.getVBOIDToDBRootMap(); } - -} // namespace +} // namespace execplan diff --git a/dbcon/execplan/objectidmanager.h b/dbcon/execplan/objectidmanager.h index a0427f3ce..0a389db6a 100644 --- a/dbcon/execplan/objectidmanager.h +++ b/dbcon/execplan/objectidmanager.h @@ -33,7 +33,6 @@ namespace execplan { - /** @brief The class that manages the Object ID space * * The class that manages the allocation and deallocation of IDs @@ -41,71 +40,70 @@ namespace execplan */ class ObjectIDManager { + public: + /** @brief Default constructor + * + * @note Throws ios::failure on a file IO error + */ + ObjectIDManager(); + virtual ~ObjectIDManager(); -public: - /** @brief Default constructor - * - * @note Throws ios::failure on a file IO error - */ - ObjectIDManager(); - virtual ~ObjectIDManager(); + /** @brief Allocate one Object ID + * + * @return The allocated OID (>= 0) on success, -1 on error. + */ + int allocOID(); - /** @brief Allocate one Object ID - * - * @return The allocated OID (>= 0) on success, -1 on error. - */ - int allocOID(); + /** @brief Allocate a contiguous range of Object IDs + * + * @param num The number of contiguous Object IDs to allocate + * @return The first OID allocated on success, -1 on failure + */ + int allocOIDs(int num); - /** @brief Allocate a contiguous range of Object IDs - * - * @param num The number of contiguous Object IDs to allocate - * @return The first OID allocated on success, -1 on failure - */ - int allocOIDs(int num); + /** @brief Allocates a new oid for a version buffer file, associates it + * with dbroot. + * + * @return Returns -1 on all errors. + */ + int allocVBOID(uint32_t dbroot); - /** @brief Allocates a new oid for a version buffer file, associates it - * with dbroot. - * - * @return Returns -1 on all errors. - */ - int allocVBOID(uint32_t dbroot); + /** @brief Returns the DBRoot of the given version buffer OID, or -1 on error. */ + int getDBRootOfVBOID(uint32_t vboid); - /** @brief Returns the DBRoot of the given version buffer OID, or -1 on error. */ - int getDBRootOfVBOID(uint32_t vboid); + /** @brief Returns the VB OID -> DB Root mapping. ret[0] = dbroot of VB OID 0 + * + * @return vector where index n = dbroot of VBOID n. + * @throw runtime_exception on error + */ + std::vector getVBOIDToDBRootMap(); + /** @brief Return an OID to the pool + * + * @param oid The OID to return + */ + void returnOID(int oid); - /** @brief Returns the VB OID -> DB Root mapping. ret[0] = dbroot of VB OID 0 - * - * @return vector where index n = dbroot of VBOID n. - * @throw runtime_exception on error - */ - std::vector getVBOIDToDBRootMap(); - /** @brief Return an OID to the pool - * - * @param oid The OID to return - */ - void returnOID(int oid); + /** @brief Return a list of OIDs to the pool + * + * @param start The first OID to return + * @param end The last OID to return + */ + void returnOIDs(int start, int end); - /** @brief Return a list of OIDs to the pool - * - * @param start The first OID to return - * @param end The last OID to return - */ - void returnOIDs(int start, int end); + /** @brief Counts the number of allocated OIDs + * + * @note This currently assumes the bitmap length is a multiple of 4096 + * @return The number of allocated OIDs + */ + int size(); - /** @brief Counts the number of allocated OIDs - * - * @note This currently assumes the bitmap length is a multiple of 4096 - * @return The number of allocated OIDs - */ - int size(); + /** @brief Get the OID bitmap filename + */ + const std::string getFilename() const; - /** @brief Get the OID bitmap filename - */ - const std::string getFilename() const; - -private: - std::string fFilename; - BRM::DBRM dbrm; + private: + std::string fFilename; + BRM::DBRM dbrm; #if 0 @@ -189,10 +187,8 @@ private: */ void patchFreelist(struct FEntry* freelist, int start, int num) const; #endif - - }; -} // namespace +} // namespace execplan #endif diff --git a/dbcon/execplan/objectreader.cpp b/dbcon/execplan/objectreader.cpp index b7daf6af9..071184b15 100644 --- a/dbcon/execplan/objectreader.cpp +++ b/dbcon/execplan/objectreader.cpp @@ -67,319 +67,222 @@ using namespace std; namespace execplan { - TreeNode* ObjectReader::createTreeNode(messageqcpp::ByteStream& b) { + CLASSID id = ZERO; + TreeNode* ret; - CLASSID id = ZERO; - TreeNode* ret; + b.peek(reinterpret_cast(id)); - b.peek(reinterpret_cast(id)); + switch (id) + { + case TREENODEIMPL: ret = new TreeNodeImpl(); break; - switch (id) + case SIMPLECOLUMN: ret = new SimpleColumn(); break; + + case SIMPLECOLUMN_INT2: ret = new SimpleColumn_INT<2>(); break; + + case SIMPLECOLUMN_INT4: ret = new SimpleColumn_INT<4>(); break; + + case SIMPLECOLUMN_INT8: ret = new SimpleColumn_INT<8>(); break; + + case SIMPLECOLUMN_INT1: ret = new SimpleColumn_INT<1>(); break; + + case SIMPLECOLUMN_UINT2: ret = new SimpleColumn_UINT<2>(); break; + + case SIMPLECOLUMN_UINT4: ret = new SimpleColumn_UINT<4>(); break; + + case SIMPLECOLUMN_UINT8: ret = new SimpleColumn_UINT<8>(); break; + + case SIMPLECOLUMN_UINT1: ret = new SimpleColumn_UINT<1>(); break; + + case SIMPLECOLUMN_DECIMAL2: ret = new SimpleColumn_Decimal<2>(); break; + + case SIMPLECOLUMN_DECIMAL4: ret = new SimpleColumn_Decimal<4>(); break; + + case SIMPLECOLUMN_DECIMAL8: ret = new SimpleColumn_Decimal<8>(); break; + + case SIMPLECOLUMN_DECIMAL1: ret = new SimpleColumn_Decimal<1>(); break; + + case AGGREGATECOLUMN: ret = new AggregateColumn(); break; + + case GROUPCONCATCOLUMN: ret = new GroupConcatColumn(); break; + + case UDAFCOLUMN: ret = new UDAFColumn(); break; + + case ARITHMETICCOLUMN: ret = new ArithmeticColumn(); break; + + case CONSTANTCOLUMN: ret = new ConstantColumn(); break; + + case FUNCTIONCOLUMN: ret = new FunctionColumn(); break; + + case ROWCOLUMN: ret = new RowColumn(); break; + + case WINDOWFUNCTIONCOLUMN: ret = new WindowFunctionColumn(); break; + + case PSEUDOCOLUMN: ret = new PseudoColumn(); break; + + case FILTER: ret = new Filter(); break; + + case EXISTSFILTER: ret = new ExistsFilter(); break; + + case SELECTFILTER: ret = new SelectFilter(); break; + + case SIMPLEFILTER: ret = new SimpleFilter(); break; + + case CONSTANTFILTER: ret = new ConstantFilter(); break; + + case SIMPLESCALARFILTER: ret = new SimpleScalarFilter(); break; + + case OUTERJOINONFILTER: ret = new OuterJoinOnFilter(); break; + + case OPERATOR: ret = new Operator(); break; + + case ARITHMETICOPERATOR: ret = new ArithmeticOperator(); break; + + case LOGICOPERATOR: ret = new LogicOperator(); break; + + case PREDICATEOPERATOR: ret = new PredicateOperator(); break; + + case NULL_CLASS: + b >> (id_t&)id; // eat the ID + return NULL; + + default: { - case TREENODEIMPL: - ret = new TreeNodeImpl(); - break; - - case SIMPLECOLUMN: - ret = new SimpleColumn(); - break; - - case SIMPLECOLUMN_INT2: - ret = new SimpleColumn_INT<2>(); - break; - - case SIMPLECOLUMN_INT4: - ret = new SimpleColumn_INT<4>(); - break; - - case SIMPLECOLUMN_INT8: - ret = new SimpleColumn_INT<8>(); - break; - - case SIMPLECOLUMN_INT1: - ret = new SimpleColumn_INT<1>(); - break; - - case SIMPLECOLUMN_UINT2: - ret = new SimpleColumn_UINT<2>(); - break; - - case SIMPLECOLUMN_UINT4: - ret = new SimpleColumn_UINT<4>(); - break; - - case SIMPLECOLUMN_UINT8: - ret = new SimpleColumn_UINT<8>(); - break; - - case SIMPLECOLUMN_UINT1: - ret = new SimpleColumn_UINT<1>(); - break; - - case SIMPLECOLUMN_DECIMAL2: - ret = new SimpleColumn_Decimal<2>(); - break; - - case SIMPLECOLUMN_DECIMAL4: - ret = new SimpleColumn_Decimal<4>(); - break; - - case SIMPLECOLUMN_DECIMAL8: - ret = new SimpleColumn_Decimal<8>(); - break; - - case SIMPLECOLUMN_DECIMAL1: - ret = new SimpleColumn_Decimal<1>(); - break; - - case AGGREGATECOLUMN: - ret = new AggregateColumn(); - break; - - case GROUPCONCATCOLUMN: - ret = new GroupConcatColumn(); - break; - - case UDAFCOLUMN: - ret = new UDAFColumn(); - break; - - case ARITHMETICCOLUMN: - ret = new ArithmeticColumn(); - break; - - case CONSTANTCOLUMN: - ret = new ConstantColumn(); - break; - - case FUNCTIONCOLUMN: - ret = new FunctionColumn(); - break; - - case ROWCOLUMN: - ret = new RowColumn(); - break; - - case WINDOWFUNCTIONCOLUMN: - ret = new WindowFunctionColumn(); - break; - - case PSEUDOCOLUMN: - ret = new PseudoColumn(); - break; - - case FILTER: - ret = new Filter(); - break; - - case EXISTSFILTER: - ret = new ExistsFilter(); - break; - - case SELECTFILTER: - ret = new SelectFilter(); - break; - - case SIMPLEFILTER: - ret = new SimpleFilter(); - break; - - case CONSTANTFILTER: - ret = new ConstantFilter(); - break; - - case SIMPLESCALARFILTER: - ret = new SimpleScalarFilter(); - break; - - case OUTERJOINONFILTER: - ret = new OuterJoinOnFilter(); - break; - - case OPERATOR: - ret = new Operator(); - break; - - case ARITHMETICOPERATOR: - ret = new ArithmeticOperator(); - break; - - case LOGICOPERATOR: - ret = new LogicOperator(); - break; - - case PREDICATEOPERATOR: - ret = new PredicateOperator(); - break; - - case NULL_CLASS: - b >> (id_t&) id; //eat the ID - return NULL; - - default: - { - ostringstream oss; - oss << "Bad type: " << (int)id << ". Stream out of sync? (1)"; - throw UnserializeException(oss.str()); - break; - } + ostringstream oss; + oss << "Bad type: " << (int)id << ". Stream out of sync? (1)"; + throw UnserializeException(oss.str()); + break; } + } - ret->unserialize(b); - return ret; + ret->unserialize(b); + return ret; } CalpontExecutionPlan* ObjectReader::createExecutionPlan(messageqcpp::ByteStream& b) { - CLASSID id = ZERO; - CalpontExecutionPlan* ret; + CLASSID id = ZERO; + CalpontExecutionPlan* ret; - b.peek(reinterpret_cast(id)); + b.peek(reinterpret_cast(id)); - switch (id) + switch (id) + { + case CALPONTSELECTEXECUTIONPLAN: ret = new CalpontSelectExecutionPlan(); break; + + case NULL_CLASS: b >> reinterpret_cast(id); return NULL; + + default: { - case CALPONTSELECTEXECUTIONPLAN: - ret = new CalpontSelectExecutionPlan(); - break; - - case NULL_CLASS: - b >> reinterpret_cast(id); - return NULL; - - default: - { - ostringstream oss; - oss << "Bad type: " << (int)id << ". Stream out of sync? (2)"; - throw UnserializeException(oss.str()); - break; - } + ostringstream oss; + oss << "Bad type: " << (int)id << ". Stream out of sync? (2)"; + throw UnserializeException(oss.str()); + break; } + } - ret->unserialize(b); - return ret; + ret->unserialize(b); + return ret; } void ObjectReader::writeParseTree(const ParseTree* tree, messageqcpp::ByteStream& b) { - if (tree == NULL) - { - b << (id_t) NULL_CLASS; - return; - } + if (tree == NULL) + { + b << (id_t)NULL_CLASS; + return; + } - b << (id_t) PARSETREE; - writeParseTree(tree->left(), b); - writeParseTree(tree->right(), b); + b << (id_t)PARSETREE; + writeParseTree(tree->left(), b); + writeParseTree(tree->right(), b); - if (tree->data() == NULL) - b << (id_t) NULL_CLASS; - else - tree->data()->serialize(b); + if (tree->data() == NULL) + b << (id_t)NULL_CLASS; + else + tree->data()->serialize(b); } ParseTree* ObjectReader::createParseTree(messageqcpp::ByteStream& b) { - CLASSID id = ZERO; - ParseTree* ret; + CLASSID id = ZERO; + ParseTree* ret; - b >> (id_t&) id; + b >> (id_t&)id; - if (id == NULL_CLASS) - return NULL; + if (id == NULL_CLASS) + return NULL; - if (id != PARSETREE) - throw UnserializeException("Not a ParseTree"); + if (id != PARSETREE) + throw UnserializeException("Not a ParseTree"); - ret = new ParseTree(); - ret->left(createParseTree(b)); - ret->right(createParseTree(b)); - ret->data(createTreeNode(b)); - return ret; + ret = new ParseTree(); + ret->left(createParseTree(b)); + ret->right(createParseTree(b)); + ret->data(createTreeNode(b)); + return ret; } void ObjectReader::checkType(messageqcpp::ByteStream& b, const CLASSID id) { - CLASSID readId = ZERO; + CLASSID readId = ZERO; - b >> (id_t&) readId; + b >> (id_t&)readId; - if (readId != id) - switch (id) - { - case TREENODEIMPL: - throw UnserializeException("Not a TreeNodeImpl"); + if (readId != id) + switch (id) + { + case TREENODEIMPL: throw UnserializeException("Not a TreeNodeImpl"); - case RETURNEDCOLUMN: - throw UnserializeException("Not a ReturnedColumn"); + case RETURNEDCOLUMN: throw UnserializeException("Not a ReturnedColumn"); - case SIMPLECOLUMN: - throw UnserializeException("Not a SimpleColumn"); + case SIMPLECOLUMN: throw UnserializeException("Not a SimpleColumn"); - case AGGREGATECOLUMN: - throw UnserializeException("Not an AggregateColumn"); + case AGGREGATECOLUMN: throw UnserializeException("Not an AggregateColumn"); - case ARITHMETICCOLUMN: - throw UnserializeException("Not an ArithmeticColumn"); + case ARITHMETICCOLUMN: throw UnserializeException("Not an ArithmeticColumn"); - case CONSTANTCOLUMN: - throw UnserializeException("Not a ConstantColumn"); + case CONSTANTCOLUMN: throw UnserializeException("Not a ConstantColumn"); - case FUNCTIONCOLUMN: - throw UnserializeException("Not a FunctionColumn"); + case FUNCTIONCOLUMN: throw UnserializeException("Not a FunctionColumn"); - case ROWCOLUMN: - throw UnserializeException("Not a RowColumn"); + case ROWCOLUMN: throw UnserializeException("Not a RowColumn"); - case WINDOWFUNCTIONCOLUMN: - throw UnserializeException("Not a WindowFunctionColumn"); + case WINDOWFUNCTIONCOLUMN: throw UnserializeException("Not a WindowFunctionColumn"); - case PSEUDOCOLUMN: - throw UnserializeException("Not a PseudoColumn"); + case PSEUDOCOLUMN: throw UnserializeException("Not a PseudoColumn"); - case FILTER: - throw UnserializeException("Not a Filter"); + case FILTER: throw UnserializeException("Not a Filter"); - case CONDITIONFILTER: - throw UnserializeException("Not a ConditionFilter"); + case CONDITIONFILTER: throw UnserializeException("Not a ConditionFilter"); - case EXISTSFILTER: - throw UnserializeException("Not an ExistsFilter"); + case EXISTSFILTER: throw UnserializeException("Not an ExistsFilter"); - case SELECTFILTER: - throw UnserializeException("Not a SelectFilter"); + case SELECTFILTER: throw UnserializeException("Not a SelectFilter"); - case SIMPLEFILTER: - throw UnserializeException("Not a SimpleFilter"); + case SIMPLEFILTER: throw UnserializeException("Not a SimpleFilter"); - case CONSTANTFILTER: - throw UnserializeException("Not a ConstantFilter"); + case CONSTANTFILTER: throw UnserializeException("Not a ConstantFilter"); - case OPERATOR: - throw UnserializeException("Not an Operator"); + case OPERATOR: throw UnserializeException("Not an Operator"); - case PARSETREE: - throw UnserializeException("Not an ParseTree"); + case PARSETREE: throw UnserializeException("Not an ParseTree"); - case CALPONTSELECTEXECUTIONPLAN: - throw UnserializeException("Not a CalpontSelectExecutionPlan"); + case CALPONTSELECTEXECUTIONPLAN: throw UnserializeException("Not a CalpontSelectExecutionPlan"); - case NULL_CLASS: - throw UnserializeException("Not NULL"); // ?? + case NULL_CLASS: throw UnserializeException("Not NULL"); // ?? - case MCSV1_CONTEXT: - throw UnserializeException("Not a MCSV1_CONTEXT"); + case MCSV1_CONTEXT: throw UnserializeException("Not a MCSV1_CONTEXT"); - default: - throw UnserializeException("Bad id"); - } + default: throw UnserializeException("Bad id"); + } - return; + return; } -ObjectReader::UnserializeException::UnserializeException(string msg) -throw() : fWhat(msg) +ObjectReader::UnserializeException::UnserializeException(string msg) throw() : fWhat(msg) { } @@ -389,8 +292,7 @@ ObjectReader::UnserializeException::~UnserializeException() throw() const char* ObjectReader::UnserializeException::what() const throw() { - return fWhat.c_str(); + return fWhat.c_str(); } - -} /* namespace */ +} // namespace execplan diff --git a/dbcon/execplan/objectreader.h b/dbcon/execplan/objectreader.h index a3cba13c5..eb65a8cf2 100644 --- a/dbcon/execplan/objectreader.h +++ b/dbcon/execplan/objectreader.h @@ -38,7 +38,6 @@ class ByteStream; namespace execplan { - class TreeNode; class ParseTree; class CalpontExecutionPlan; @@ -51,124 +50,121 @@ class CalpontExecutionPlan; class ObjectReader { + public: + class UnserializeException : public std::exception + { + public: + UnserializeException(std::string) throw(); + virtual ~UnserializeException() throw(); + virtual const char* what() const throw(); -public: + private: + std::string fWhat; + }; - class UnserializeException : public std::exception - { - public: - UnserializeException(std::string) throw(); - virtual ~UnserializeException() throw(); - virtual const char* what() const throw(); - private: - std::string fWhat; - }; + /** @brief Enumerates classes supporting serialization + * + * This defines one constant for each class that supports + * serialization. + */ + enum CLASSID + { + ZERO, // an appropriate initializer + NULL_CLASS, // to denote that some member is NULL - /** @brief Enumerates classes supporting serialization - * - * This defines one constant for each class that supports - * serialization. - */ - enum CLASSID - { - ZERO, // an appropriate initializer - NULL_CLASS, // to denote that some member is NULL + /**** TreeNodes */ + TREENODE, + TREENODEIMPL, + RETURNEDCOLUMN, + AGGREGATECOLUMN, + GROUPCONCATCOLUMN, + ARITHMETICCOLUMN, + CONSTANTCOLUMN, + FUNCTIONCOLUMN, + ROWCOLUMN, + WINDOWFUNCTIONCOLUMN, + PSEUDOCOLUMN, - /**** TreeNodes */ - TREENODE, - TREENODEIMPL, - RETURNEDCOLUMN, - AGGREGATECOLUMN, - GROUPCONCATCOLUMN, - ARITHMETICCOLUMN, - CONSTANTCOLUMN, - FUNCTIONCOLUMN, - ROWCOLUMN, - WINDOWFUNCTIONCOLUMN, - PSEUDOCOLUMN, + SIMPLECOLUMN, + SIMPLECOLUMN_INT1, + SIMPLECOLUMN_INT2, + SIMPLECOLUMN_INT4, + SIMPLECOLUMN_INT8, + SIMPLECOLUMN_UINT1, + SIMPLECOLUMN_UINT2, + SIMPLECOLUMN_UINT4, + SIMPLECOLUMN_UINT8, + SIMPLECOLUMN_DECIMAL1, + SIMPLECOLUMN_DECIMAL2, + SIMPLECOLUMN_DECIMAL4, + SIMPLECOLUMN_DECIMAL8, - SIMPLECOLUMN, - SIMPLECOLUMN_INT1, - SIMPLECOLUMN_INT2, - SIMPLECOLUMN_INT4, - SIMPLECOLUMN_INT8, - SIMPLECOLUMN_UINT1, - SIMPLECOLUMN_UINT2, - SIMPLECOLUMN_UINT4, - SIMPLECOLUMN_UINT8, - SIMPLECOLUMN_DECIMAL1, - SIMPLECOLUMN_DECIMAL2, - SIMPLECOLUMN_DECIMAL4, - SIMPLECOLUMN_DECIMAL8, + FILTER, + CONDITIONFILTER, + EXISTSFILTER, + SELECTFILTER, + SIMPLEFILTER, + SIMPLESCALARFILTER, - FILTER, - CONDITIONFILTER, - EXISTSFILTER, - SELECTFILTER, - SIMPLEFILTER, - SIMPLESCALARFILTER, + OPERATOR, + ARITHMETICOPERATOR, + PREDICATEOPERATOR, + LOGICOPERATOR, - OPERATOR, - ARITHMETICOPERATOR, - PREDICATEOPERATOR, - LOGICOPERATOR, + /**** /TreeNodes */ - /**** /TreeNodes */ + PARSETREE, + CALPONTSELECTEXECUTIONPLAN, + CONSTANTFILTER, + OUTERJOINONFILTER, - PARSETREE, - CALPONTSELECTEXECUTIONPLAN, - CONSTANTFILTER, - OUTERJOINONFILTER, + /** UDAF SDK */ + MCSV1_CONTEXT, + UDAFCOLUMN, - /** UDAF SDK */ - MCSV1_CONTEXT, - UDAFCOLUMN, + /** ANALYZE TABLE */ + MCSANALYZETBLEXECUTIONPLAN, + }; - /** ANALYZE TABLE */ - MCSANALYZETBLEXECUTIONPLAN, - }; + typedef uint8_t id_t; // expand as necessary - typedef uint8_t id_t; //expand as necessary + /** @brief Creates a new TreeNode object from the ByteStream + * + * @param b The ByteStream to create it from + * @return A newly allocated TreeNode + */ + static TreeNode* createTreeNode(messageqcpp::ByteStream& b); - /** @brief Creates a new TreeNode object from the ByteStream - * - * @param b The ByteStream to create it from - * @return A newly allocated TreeNode - */ - static TreeNode* createTreeNode(messageqcpp::ByteStream& b); + /** @brief Creates a new ParseTree from the ByteStream + * + * @param b The ByteStream to create it from + * @return A newly allocated ParseTree + */ + static ParseTree* createParseTree(messageqcpp::ByteStream& b); - /** @brief Creates a new ParseTree from the ByteStream - * - * @param b The ByteStream to create it from - * @return A newly allocated ParseTree - */ - static ParseTree* createParseTree(messageqcpp::ByteStream& b); + /** @brief Creates a new CalpontExecutionPlan from the ByteStream + * + * @param b The ByteStream to create it from + * @return A newly allocated CalpontExecutionPlan + */ + static CalpontExecutionPlan* createExecutionPlan(messageqcpp::ByteStream& b); - /** @brief Creates a new CalpontExecutionPlan from the ByteStream - * - * @param b The ByteStream to create it from - * @return A newly allocated CalpontExecutionPlan - */ - static CalpontExecutionPlan* createExecutionPlan(messageqcpp::ByteStream& b); + /** @brief Serialize() for ParseTrees + * + * This function effectively serializes a ParseTree. + * @param tree The ParseTree to write out + * @param b The ByteStream to write tree to + */ + static void writeParseTree(const ParseTree* tree, messageqcpp::ByteStream& b); - /** @brief Serialize() for ParseTrees - * - * This function effectively serializes a ParseTree. - * @param tree The ParseTree to write out - * @param b The ByteStream to write tree to - */ - static void writeParseTree(const ParseTree* tree, - messageqcpp::ByteStream& b); - - /** @brief Verify the type of the next object in the ByteStream - * - * @param b The ByteStream to read from - * @param type The type it should be - * @throw UnserializeException if the type does not match; this is a fatal error. - */ - static void checkType(messageqcpp::ByteStream& b, const CLASSID type); + /** @brief Verify the type of the next object in the ByteStream + * + * @param b The ByteStream to read from + * @param type The type it should be + * @throw UnserializeException if the type does not match; this is a fatal error. + */ + static void checkType(messageqcpp::ByteStream& b, const CLASSID type); }; -} -#endif // EXECPLAN_OBJECTREADER_H - +} // namespace execplan +#endif // EXECPLAN_OBJECTREADER_H diff --git a/dbcon/execplan/operator.cpp b/dbcon/execplan/operator.cpp index 60ec850f7..de5101c4d 100644 --- a/dbcon/execplan/operator.cpp +++ b/dbcon/execplan/operator.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: operator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: operator.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -31,37 +31,37 @@ using namespace std; namespace { - /**@brief util struct for converting string to lower case */ struct to_lower { - char operator() (char c) const - { - return tolower(c); - } + char operator()(char c) const + { + return tolower(c); + } }; -//Trim any leading/trailing ws +// Trim any leading/trailing ws const string lrtrim(const string& in) { - string::size_type p1; - p1 = in.find_first_not_of(" \t\n"); + string::size_type p1; + p1 = in.find_first_not_of(" \t\n"); - if (p1 == string::npos) p1 = 0; + if (p1 == string::npos) + p1 = 0; - string::size_type p2; - p2 = in.find_last_not_of(" \t\n"); + string::size_type p2; + p2 = in.find_last_not_of(" \t\n"); - if (p2 == string::npos) p2 = in.size() - 1; + if (p2 == string::npos) + p2 = in.size() - 1; - return string(in, p1, (p2 - p1 + 1)); + return string(in, p1, (p2 - p1 + 1)); } -} +} // namespace namespace execplan { - /** * Constructors/Destructors */ @@ -71,17 +71,15 @@ Operator::Operator() Operator::Operator(const string& operatorName) { - data(operatorName); + data(operatorName); } -Operator::Operator(const Operator& rhs) : - TreeNode(rhs), - fOp(rhs.op()) +Operator::Operator(const Operator& rhs) : TreeNode(rhs), fOp(rhs.op()) { - data(rhs.fData); + data(rhs.fData); } -Operator:: ~Operator() +Operator::~Operator() { } @@ -90,109 +88,110 @@ Operator:: ~Operator() */ void Operator::data(const string data) { - fData = lrtrim(data); - transform (fData.begin(), fData.end(), fData.begin(), to_lower()); + fData = lrtrim(data); + transform(fData.begin(), fData.end(), fData.begin(), to_lower()); - if (fData == "+") - { - fOp = OP_ADD; - } - else if (fData == "-") - { - fOp = OP_SUB; - } - else if (fData == "*") - { - fOp = OP_MUL; - } - else if (fData == "/") - { - fOp = OP_DIV; - } - else if (fData == "=") - { - fOp = OP_EQ; - } - else if (fData == "!=" || fData == "<>") - { - fOp = OP_NE; - } - else if (fData == ">") - { - fOp = OP_GT; - } - else if (fData == ">=") - { - fOp = OP_GE; - } - else if (fData == "<") - { - fOp = OP_LT; - } - else if (fData == "<=") - { - fOp = OP_LE; - } - else if (fData == "and") - { - fOp = OP_AND; - } - else if (fData == "or") - { - fOp = OP_OR; - } - else if (fData == "like") - { - fOp = OP_LIKE; - } - else if (fData == "not like") - { - fOp = OP_NOTLIKE; - fData = "not like"; - } - else if (fData == "isnull") - { - fOp = OP_ISNULL; - fData = "is null"; - } - else if (fData == "isnotnull") - { - fOp = OP_ISNOTNULL; - fData = "is not null"; - } - else if (fData == "xor") - { - fOp = OP_XOR; - } - else - { - fOp = OP_UNKNOWN; - } + if (fData == "+") + { + fOp = OP_ADD; + } + else if (fData == "-") + { + fOp = OP_SUB; + } + else if (fData == "*") + { + fOp = OP_MUL; + } + else if (fData == "/") + { + fOp = OP_DIV; + } + else if (fData == "=") + { + fOp = OP_EQ; + } + else if (fData == "!=" || fData == "<>") + { + fOp = OP_NE; + } + else if (fData == ">") + { + fOp = OP_GT; + } + else if (fData == ">=") + { + fOp = OP_GE; + } + else if (fData == "<") + { + fOp = OP_LT; + } + else if (fData == "<=") + { + fOp = OP_LE; + } + else if (fData == "and") + { + fOp = OP_AND; + } + else if (fData == "or") + { + fOp = OP_OR; + } + else if (fData == "like") + { + fOp = OP_LIKE; + } + else if (fData == "not like") + { + fOp = OP_NOTLIKE; + fData = "not like"; + } + else if (fData == "isnull") + { + fOp = OP_ISNULL; + fData = "is null"; + } + else if (fData == "isnotnull") + { + fOp = OP_ISNOTNULL; + fData = "is not null"; + } + else if (fData == "xor") + { + fOp = OP_XOR; + } + else + { + fOp = OP_UNKNOWN; + } } const string Operator::toString() const { - ostringstream oss; - oss << string("Operator: " + fData) << " fOp=" << fOp; - oss << " " << "opType=" << fOperationType.colDataType; - return oss.str(); + ostringstream oss; + oss << string("Operator: " + fData) << " fOp=" << fOp; + oss << " " + << "opType=" << fOperationType.colDataType; + return oss.str(); } Operator* Operator::opposite() const { - if (fData.compare(">") == 0) - return new Operator("<"); + if (fData.compare(">") == 0) + return new Operator("<"); - if (fData.compare("<") == 0) - return new Operator(">"); + if (fData.compare("<") == 0) + return new Operator(">"); - if (fData.compare(">=") == 0) - return new Operator("<="); + if (fData.compare(">=") == 0) + return new Operator("<="); - if (fData.compare("<=") == 0) - return new Operator(">="); + if (fData.compare("<=") == 0) + return new Operator(">="); - return this->clone(); + return this->clone(); } /** @@ -200,8 +199,8 @@ Operator* Operator::opposite() const */ ostream& operator<<(ostream& output, const Operator& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } /** @@ -209,137 +208,137 @@ ostream& operator<<(ostream& output, const Operator& rhs) */ void Operator::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::OPERATOR; - b << fData; - fResultType.serialize(b); - fOperationType.serialize(b); - b << (uint32_t)fOp; + b << (ObjectReader::id_t)ObjectReader::OPERATOR; + b << fData; + fResultType.serialize(b); + fOperationType.serialize(b); + b << (uint32_t)fOp; } void Operator::unserialize(messageqcpp::ByteStream& b) { - uint32_t val; - ObjectReader::checkType(b, ObjectReader::OPERATOR); - b >> fData; - fResultType.unserialize(b); - fOperationType.unserialize(b); + uint32_t val; + ObjectReader::checkType(b, ObjectReader::OPERATOR); + b >> fData; + fResultType.unserialize(b); + fOperationType.unserialize(b); - b >> (uint32_t&)val; - fOp = (OpType)val; + b >> (uint32_t&)val; + fOp = (OpType)val; - fResult.decimalVal.scale = fResultType.scale; - fResult.decimalVal.precision = fResultType.precision; + fResult.decimalVal.scale = fResultType.scale; + fResult.decimalVal.precision = fResultType.precision; } bool Operator::operator==(const Operator& t) const { - if (fOp == t.fOp) - return true; + if (fOp == t.fOp) + return true; - return false; + return false; } bool Operator::operator==(const TreeNode* t) const { - const Operator* o; + const Operator* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool Operator::operator!=(const Operator& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool Operator::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } void Operator::reverseOp() { - switch (fOp) - { - case OP_EQ: - fOp = OP_NE; - fData = "!="; - break; + switch (fOp) + { + case OP_EQ: + fOp = OP_NE; + fData = "!="; + break; - case OP_NE: - fOp = OP_EQ; - fData = "="; - break; + case OP_NE: + fOp = OP_EQ; + fData = "="; + break; - case OP_GT: - fOp = OP_LT; - fData = "<"; - break; + case OP_GT: + fOp = OP_LT; + fData = "<"; + break; - case OP_GE: - fOp = OP_LE; - fData = "<="; - break; + case OP_GE: + fOp = OP_LE; + fData = "<="; + break; - case OP_LT: - fOp = OP_GT; - fData = ">"; - break; + case OP_LT: + fOp = OP_GT; + fData = ">"; + break; - case OP_LE: - fOp = OP_GE; - fData = ">="; - break; + case OP_LE: + fOp = OP_GE; + fData = ">="; + break; - case OP_LIKE: - fOp = OP_NOTLIKE; - fData = "not like"; - break; + case OP_LIKE: + fOp = OP_NOTLIKE; + fData = "not like"; + break; - case OP_NOTLIKE: - fOp = OP_LIKE; - fData = "like"; - break; + case OP_NOTLIKE: + fOp = OP_LIKE; + fData = "like"; + break; - case OP_ISNULL: - fOp = OP_ISNOTNULL; - fData = "isnotnull"; - break; + case OP_ISNULL: + fOp = OP_ISNOTNULL; + fData = "isnotnull"; + break; - case OP_ISNOTNULL: - fOp = OP_ISNULL; - fData = "isnull"; - break; + case OP_ISNOTNULL: + fOp = OP_ISNULL; + fData = "isnull"; + break; - case OP_BETWEEN: - fOp = OP_NOTBETWEEN; - fData = "not between"; - break; + case OP_BETWEEN: + fOp = OP_NOTBETWEEN; + fData = "not between"; + break; - case OP_NOTBETWEEN: - fOp = OP_BETWEEN; - fData = "between"; - break; + case OP_NOTBETWEEN: + fOp = OP_BETWEEN; + fData = "between"; + break; - case OP_IN: - fOp = OP_NOTIN; - fData = "not in"; - break; + case OP_IN: + fOp = OP_NOTIN; + fData = "not in"; + break; - case OP_NOTIN: - fOp = OP_IN; - fData = "in"; - break; + case OP_NOTIN: + fOp = OP_IN; + fData = "in"; + break; - default: - fOp = OP_UNKNOWN; - fData = "unknown"; - break; - } + default: + fOp = OP_UNKNOWN; + fData = "unknown"; + break; + } } -} // namespace +} // namespace execplan diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index 660f9f8c6..b79a35505 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: operator.h 9635 2013-06-19 21:42:30Z bwilkinson $ -* -* -***********************************************************************/ + * $Id: operator.h 9635 2013-06-19 21:42:30Z bwilkinson $ + * + * + ***********************************************************************/ /** @file */ #ifndef OPERATOR_H @@ -43,207 +43,208 @@ class ReturnedColumn; enum OpType { - OP_ADD = 0, - OP_SUB, - OP_MUL, - OP_DIV, - OP_EQ, - OP_NE, - OP_GT, - OP_GE, - OP_LT, - OP_LE, - OP_LIKE, - OP_NOTLIKE, - OP_AND, - OP_OR, - OP_ISNULL, - OP_ISNOTNULL, - OP_BETWEEN, - OP_NOTBETWEEN, - OP_IN, - OP_NOTIN, - OP_XOR, - OP_UNKNOWN, + OP_ADD = 0, + OP_SUB, + OP_MUL, + OP_DIV, + OP_EQ, + OP_NE, + OP_GT, + OP_GE, + OP_LT, + OP_LE, + OP_LIKE, + OP_NOTLIKE, + OP_AND, + OP_OR, + OP_ISNULL, + OP_ISNOTNULL, + OP_BETWEEN, + OP_NOTBETWEEN, + OP_IN, + OP_NOTIN, + OP_XOR, + OP_UNKNOWN, }; class Operator : public TreeNode { + public: + Operator(); + Operator(const std::string& operatorName); + Operator(const Operator& rhs); -public: - Operator(); - Operator(const std::string& operatorName); - Operator(const Operator& rhs); + virtual ~Operator(); - virtual ~Operator(); + virtual const std::string toString() const; + virtual const std::string data() const + { + return fData; + } + virtual void data(const std::string data); - virtual const std::string toString() const; - virtual const std::string data() const - { - return fData; - } - virtual void data(const std::string data); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual Operator* clone() const + { + return new Operator(*this); + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual Operator* clone() const - { - return new Operator (*this); - } + /** return the opposite operator + * @note this is not a compliment of operator. this is for the case that + * the two operands of operator get switched, and the operator needs to + * be reversed. e.g., where C1 > C2 ==> where C2 < C1 + */ + virtual Operator* opposite() const; - /** return the opposite operator - * @note this is not a compliment of operator. this is for the case that - * the two operands of operator get switched, and the operator needs to - * be reversed. e.g., where C1 > C2 ==> where C2 < C1 - */ - virtual Operator* opposite() const; + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const Operator& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const Operator& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const Operator& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const Operator& t) const; + /** @breif reverse the operator + * + * Literally reverse the operator, as left operand and right operand are swapped, or a NOT + * is applied before the operator + */ + virtual void reverseOp(); - /** @breif reverse the operator - * - * Literally reverse the operator, as left operand and right operand are swapped, or a NOT - * is applied before the operator - */ - virtual void reverseOp(); + protected: + std::string fData; -protected: - std::string fData; + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + virtual OpType op() const + { + return fOp; + } + virtual void op(const OpType op) + { + fOp = op; + } + using TreeNode::evaluate; + virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + } - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - virtual OpType op() const - { - return fOp; - } - virtual void op(const OpType op) - { - fOp = op; - } - using TreeNode::evaluate; - virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) {} + // The following methods should be pure virtual. Currently too many instanslization exists. + using TreeNode::getStrVal; + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.strVal; + } + using TreeNode::getIntVal; + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } + using TreeNode::getUintVal; + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.uintVal; + } + using TreeNode::getFloatVal; + virtual float getFloatVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.floatVal; + } + using TreeNode::getDoubleVal; + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.doubleVal; + } + using TreeNode::getLongDoubleVal; + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.longDoubleVal; + } + using TreeNode::getDecimalVal; + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.decimalVal; + } + using TreeNode::getDateIntVal; + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } + using TreeNode::getDatetimeIntVal; + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } + using TreeNode::getTimestampIntVal; + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } + using TreeNode::getTimeIntVal; + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } + using TreeNode::getBoolVal; + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.boolVal; + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) + { + return fResult.boolVal; + } - // The following methods should be pure virtual. Currently too many instanslization exists. - using TreeNode::getStrVal; - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.strVal; - } - using TreeNode::getIntVal; - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.intVal; - } - using TreeNode::getUintVal; - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.uintVal; - } - using TreeNode::getFloatVal; - virtual float getFloatVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.floatVal; - } - using TreeNode::getDoubleVal; - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.doubleVal; - } - using TreeNode::getLongDoubleVal; - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.longDoubleVal; - } - using TreeNode::getDecimalVal; - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.decimalVal; - } - using TreeNode::getDateIntVal; - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.intVal; - } - using TreeNode::getDatetimeIntVal; - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.intVal; - } - using TreeNode::getTimestampIntVal; - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.intVal; - } - using TreeNode::getTimeIntVal; - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.intVal; - } - using TreeNode::getBoolVal; - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) - { - return fResult.boolVal; - } - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) - { - return fResult.boolVal; - } + virtual void setOpType(Type& l, Type& r) + { + } + virtual void operationType(const Type& ot) + { + fOperationType = ot; + } + virtual const Type& operationType() const + { + return fOperationType; + } - virtual void setOpType(Type& l, Type& r) {} - virtual void operationType(const Type& ot) - { - fOperationType = ot; - } - virtual const Type& operationType() const - { - return fOperationType; - } - -protected: - OpType fOp; + protected: + OpType fOp; }; typedef boost::shared_ptr SOP; std::ostream& operator<<(std::ostream& os, const Operator& rhs); -} - - -#endif //OPERATOR_H +} // namespace execplan +#endif // OPERATOR_H diff --git a/dbcon/execplan/oracleexecutionplan.cpp b/dbcon/execplan/oracleexecutionplan.cpp index 63a06af57..bc8ce633c 100644 --- a/dbcon/execplan/oracleexecutionplan.cpp +++ b/dbcon/execplan/oracleexecutionplan.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: oracleexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: oracleexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "oracleexecutionplan.h" @@ -29,4 +29,3 @@ /** * Methods */ - diff --git a/dbcon/execplan/oracleexecutionplan.h b/dbcon/execplan/oracleexecutionplan.h index c121e329e..9ba3bbc52 100644 --- a/dbcon/execplan/oracleexecutionplan.h +++ b/dbcon/execplan/oracleexecutionplan.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: oracleexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: oracleexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef ORACLEEXECUTIONPLAN_H @@ -38,64 +38,63 @@ namespace execplan */ class OracleExecutionPlan : public VendorExecutionPlan { - /** - * Public stuff - */ -public: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Protected stuff - */ -protected: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Private stuff - */ -private: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ + /** + * Public stuff + */ + public: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Protected stuff + */ + protected: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Private stuff + */ + private: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ }; -} -#endif //ORACLEEXECUTIONPLAN_H - +} // namespace execplan +#endif // ORACLEEXECUTIONPLAN_H diff --git a/dbcon/execplan/outerjoinonfilter.cpp b/dbcon/execplan/outerjoinonfilter.cpp index 1ee46f95c..5a3145a01 100644 --- a/dbcon/execplan/outerjoinonfilter.cpp +++ b/dbcon/execplan/outerjoinonfilter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: outerjoinonfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: outerjoinonfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include using namespace std; @@ -34,100 +34,98 @@ namespace execplan /** * Constructors/Destructors */ -OuterJoinOnFilter::OuterJoinOnFilter(): - fData("Outer Join On Filter") -{} +OuterJoinOnFilter::OuterJoinOnFilter() : fData("Outer Join On Filter") +{ +} -OuterJoinOnFilter::OuterJoinOnFilter(const SPTP& pt): - Filter(), - fPt(new ParseTree (*(pt.get()))), - fData("Outer Join On Filter") -{} +OuterJoinOnFilter::OuterJoinOnFilter(const SPTP& pt) + : Filter(), fPt(new ParseTree(*(pt.get()))), fData("Outer Join On Filter") +{ +} -OuterJoinOnFilter::OuterJoinOnFilter(const OuterJoinOnFilter& rhs): - Filter(rhs), - fPt (rhs.fPt), - fData (rhs.fData) -{} +OuterJoinOnFilter::OuterJoinOnFilter(const OuterJoinOnFilter& rhs) + : Filter(rhs), fPt(rhs.fPt), fData(rhs.fData) +{ +} OuterJoinOnFilter::~OuterJoinOnFilter() -{} +{ +} const string OuterJoinOnFilter::toString() const { - ostringstream oss; - oss << "OuterJoinOnFilter" << endl; + ostringstream oss; + oss << "OuterJoinOnFilter" << endl; - if (fPt.get()) - oss << fPt->toString(); - else - oss << "Empty Tree." << endl; + if (fPt.get()) + oss << fPt->toString(); + else + oss << "Empty Tree." << endl; - oss << "End OuterJoinOnFilter" << endl; - return oss.str(); + oss << "End OuterJoinOnFilter" << endl; + return oss.str(); } ostream& operator<<(ostream& output, const OuterJoinOnFilter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void OuterJoinOnFilter::serialize(messageqcpp::ByteStream& b) const { - b << static_cast(ObjectReader::OUTERJOINONFILTER); - Filter::serialize(b); + b << static_cast(ObjectReader::OUTERJOINONFILTER); + Filter::serialize(b); - if (fPt.get() != NULL) - ObjectReader::writeParseTree(fPt.get(), b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fPt.get() != NULL) + ObjectReader::writeParseTree(fPt.get(), b); + else + b << static_cast(ObjectReader::NULL_CLASS); } void OuterJoinOnFilter::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::OUTERJOINONFILTER); - Filter::unserialize(b); - fPt.reset(ObjectReader::createParseTree(b)); + ObjectReader::checkType(b, ObjectReader::OUTERJOINONFILTER); + Filter::unserialize(b); + fPt.reset(ObjectReader::createParseTree(b)); } bool OuterJoinOnFilter::operator==(const OuterJoinOnFilter& t) const { - const Filter* f1, *f2; + const Filter *f1, *f2; - f1 = static_cast(this); - f2 = static_cast(&t); + f1 = static_cast(this); + f2 = static_cast(&t); - if (*f1 != *f2) - return false; + if (*f1 != *f2) + return false; - if (*(fPt.get()) != *(t.fPt.get())) - return false; + if (*(fPt.get()) != *(t.fPt.get())) + return false; - return true; + return true; } bool OuterJoinOnFilter::operator==(const TreeNode* t) const { - const OuterJoinOnFilter* o; + const OuterJoinOnFilter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool OuterJoinOnFilter::operator!=(const OuterJoinOnFilter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool OuterJoinOnFilter::operator!=(const TreeNode* t) const { - return (!(*this == t)); -} - + return (!(*this == t)); } +} // namespace execplan diff --git a/dbcon/execplan/outerjoinonfilter.h b/dbcon/execplan/outerjoinonfilter.h index 7d28aa21a..db6740683 100644 --- a/dbcon/execplan/outerjoinonfilter.h +++ b/dbcon/execplan/outerjoinonfilter.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: outerjoinonfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: outerjoinonfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef OUTERJOINONFILTER_H @@ -42,88 +42,87 @@ namespace execplan */ class OuterJoinOnFilter : public Filter { - /** - * Public stuff - */ -public: - /** - * Constructors - */ - OuterJoinOnFilter(); - OuterJoinOnFilter(const SPTP& pt); - OuterJoinOnFilter(const OuterJoinOnFilter& rhs); - virtual ~OuterJoinOnFilter(); + /** + * Public stuff + */ + public: + /** + * Constructors + */ + OuterJoinOnFilter(); + OuterJoinOnFilter(const SPTP& pt); + OuterJoinOnFilter(const OuterJoinOnFilter& rhs); + virtual ~OuterJoinOnFilter(); - /** - * Accessor Methods - */ - const SPTP& pt() const - { - return fPt; - } - void pt (SPTP& pt) - { - fPt = pt; - } + /** + * Accessor Methods + */ + const SPTP& pt() const + { + return fPt; + } + void pt(SPTP& pt) + { + fPt = pt; + } - /** - * Overloaded stream operator - */ - //virtual std::ostream& operator<< (std::ostream& output); - virtual const std::string toString() const; + /** + * Overloaded stream operator + */ + // virtual std::ostream& operator<< (std::ostream& output); + virtual const std::string toString() const; - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual OuterJoinOnFilter* clone() const - { - return new OuterJoinOnFilter (*this); - } + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual OuterJoinOnFilter* clone() const + { + return new OuterJoinOnFilter(*this); + } - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const OuterJoinOnFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const OuterJoinOnFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const OuterJoinOnFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const OuterJoinOnFilter& t) const; -private: - //default okay? - //OuterJoinOnFilter& operator=(const OuterJoinOnFilter& rhs); - SPTP fPt; - std::string fData; + private: + // default okay? + // OuterJoinOnFilter& operator=(const OuterJoinOnFilter& rhs); + SPTP fPt; + std::string fData; }; std::ostream& operator<<(std::ostream& output, const OuterJoinOnFilter& rhs); -} -#endif //OUTERJOINONFILTER_H - +} // namespace execplan +#endif // OUTERJOINONFILTER_H diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index 34131615d..77d777098 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: parsetree.h 9635 2013-06-19 21:42:30Z bwilkinson $ -* -* -***********************************************************************/ + * $Id: parsetree.h 9635 2013-06-19 21:42:30Z bwilkinson $ + * + * + ***********************************************************************/ /** @file */ #ifndef PARSETREE_H @@ -39,7 +39,7 @@ class Row; namespace execplan { -//class Operator; +// class Operator; /** * @brief A template class template to represent an expression tree * @@ -51,276 +51,274 @@ namespace execplan class ParseTree { + public: + /** + * Constructor / Destructor + */ + inline ParseTree(); + inline ParseTree(TreeNode* data); + inline ParseTree(const ParseTree& rhs); + inline virtual ~ParseTree(); -public: - /** - * Constructor / Destructor - */ - inline ParseTree(); - inline ParseTree(TreeNode* data); - inline ParseTree(const ParseTree& rhs); - inline virtual ~ParseTree(); + /** + * Access methods + */ + inline void left(ParseTree* expressionTree) + { + fLeft = expressionTree; + } - /** - * Access methods - */ - inline void left(ParseTree* expressionTree) - { - fLeft = expressionTree; - } + inline void left(TreeNode* node) + { + ParseTree* pt = new ParseTree(node); + left(pt); + } - inline void left(TreeNode* node) - { - ParseTree* pt = new ParseTree(node); - left(pt); - } + inline ParseTree* left() const + { + return fLeft; + } - inline ParseTree* left() const - { - return fLeft; - } + inline void right(ParseTree* expressionTree) + { + fRight = expressionTree; + } - inline void right(ParseTree* expressionTree) - { - fRight = expressionTree; - } + inline void right(TreeNode* node) + { + ParseTree* pt = new ParseTree(node); + right(pt); + } - inline void right(TreeNode* node) - { - ParseTree* pt = new ParseTree(node); - right(pt); - } + inline ParseTree* right() const + { + return fRight; + } - inline ParseTree* right() const - { - return fRight; - } + inline void data(TreeNode* data) + { + fData = data; + } - inline void data(TreeNode* data) - { - fData = data; - } + inline TreeNode* data() const + { + return fData; + } - inline TreeNode* data() const - { - return fData; - } + /** walk the tree + * + * postfix walking of a const tree + */ + inline void walk(void (*fn)(ParseTree* n)) const; - /** walk the tree - * - * postfix walking of a const tree - */ - inline void walk(void (*fn)( ParseTree* n)) const; + /** walk the tree + * + * postfix walking of a non-const tree. This is for deleting the tree + */ + inline void walk(void (*fn)(const ParseTree* n)) const; - /** walk the tree - * - * postfix walking of a non-const tree. This is for deleting the tree - */ - inline void walk(void (*fn)( const ParseTree* n)) const; + /** output the tree + * + * take ostream argument to walk and output the tree + */ + inline void walk(void (*fn)(const ParseTree* n, std::ostream& output), std::ostream& output) const; - /** output the tree - * - * take ostream argument to walk and output the tree - */ - inline void walk(void (*fn)(const ParseTree* n, std::ostream& output), std::ostream& output) const; + /** output the tree + * + * take user argument to walk and output the tree + */ + inline void walk(void (*fn)(const ParseTree* n, void* obj), void* object) const; - /** output the tree - * - * take user argument to walk and output the tree - */ - inline void walk(void (*fn)(const ParseTree* n, void* obj), void* object) const; + /** output the tree + * + * take user argument to walk and output the tree + */ + inline void walk(void (*fn)(ParseTree* n, void* obj), void* object) const; - /** output the tree - * - * take user argument to walk and output the tree - */ - inline void walk(void (*fn)(ParseTree* n, void* obj), void* object) const; + /** output the tree to string + * for debug purpose + */ + inline std::string toString() const; - /** output the tree to string - * for debug purpose - */ - inline std::string toString() const; + /** assignment operator + * + */ + inline ParseTree& operator=(const ParseTree& rhs); - /** assignment operator - * - */ - inline ParseTree& operator=(const ParseTree& rhs); + /** deep copy of a tree + * + * copy src tree on top of dest tree. Dest tree is destroyed before copy + * takes place. Src tree still owns the tree data, though. Assume tree node + * are pointers. + */ + inline void copyTree(const ParseTree& src); - /** deep copy of a tree - * - * copy src tree on top of dest tree. Dest tree is destroyed before copy - * takes place. Src tree still owns the tree data, though. Assume tree node - * are pointers. - */ - inline void copyTree (const ParseTree& src); + /** destroy a tree + * + * destroy a tree where tree nodes are values + */ + inline static void destroyTree(ParseTree* root); - /** destroy a tree - * - * destroy a tree where tree nodes are values - */ - inline static void destroyTree (ParseTree* root); + /** draw the tree using dot + * + * this function is mostly for debug purpose, where T represent a TreeNode + * pointer + */ + inline void drawTree(std::string filename); - /** draw the tree using dot - * - * this function is mostly for debug purpose, where T represent a TreeNode - * pointer - */ - inline void drawTree(std::string filename); + /** print the tree + * + * this function is mostly for debug purpose, where T represent a TreeNode + * pointer + */ + inline static void print(const ParseTree* n, std::ostream& output) + { + output << *n->data() << std::endl; + } - /** print the tree - * - * this function is mostly for debug purpose, where T represent a TreeNode - * pointer - */ - inline static void print(const ParseTree* n, std::ostream& output) - { - output << *n->data() << std::endl; - } + /** delete treenode + * + * delete fData of a tree node, and then delete the treenode + */ + inline static void deleter(ParseTree*& n) + { + delete n->fData; + n->fData = 0; + delete n; + n = 0; + } - /** delete treenode - * - * delete fData of a tree node, and then delete the treenode - */ - inline static void deleter( ParseTree*& n) - { - delete n->fData; - n->fData = 0; - delete n; - n = 0; - } + inline void derivedTable(const std::string& derivedTable) + { + fDerivedTable = derivedTable; + } - inline void derivedTable (const std::string& derivedTable) - { - fDerivedTable = derivedTable; - } + inline std::string& derivedTable() + { + return fDerivedTable; + } - inline std::string& derivedTable() - { - return fDerivedTable; - } + inline void setDerivedTable(); - inline void setDerivedTable(); + private: + TreeNode* fData; + ParseTree* fLeft; + ParseTree* fRight; + std::string fDerivedTable; -private: - TreeNode* fData; - ParseTree* fLeft; - ParseTree* fRight; - std::string fDerivedTable; + /********************************************************************** + * F&E Framework * + **********************************************************************/ - /********************************************************************** - * F&E Framework * - **********************************************************************/ + public: + inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getStrVal(row, isNull, fLeft, fRight); + else + return fData->getStrVal(row, isNull); + } -public: - inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getStrVal(row, isNull, fLeft, fRight); - else - return fData->getStrVal(row, isNull); - } + inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getIntVal(row, isNull, fLeft, fRight); + else + return fData->getIntVal(row, isNull); + } - inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getIntVal(row, isNull, fLeft, fRight); - else - return fData->getIntVal(row, isNull); - } + inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getUintVal(row, isNull, fLeft, fRight); + else + return fData->getUintVal(row, isNull); + } - inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getUintVal(row, isNull, fLeft, fRight); - else - return fData->getUintVal(row, isNull); - } + inline float getFloatVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getFloatVal(row, isNull, fLeft, fRight); + else + return fData->getFloatVal(row, isNull); + } - inline float getFloatVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getFloatVal(row, isNull, fLeft, fRight); - else - return fData->getFloatVal(row, isNull); - } + inline double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getDoubleVal(row, isNull, fLeft, fRight); + else + return fData->getDoubleVal(row, isNull); + } - inline double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getDoubleVal(row, isNull, fLeft, fRight); - else - return fData->getDoubleVal(row, isNull); - } + inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getLongDoubleVal(row, isNull, fLeft, fRight); + else + return fData->getLongDoubleVal(row, isNull); + } - inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getLongDoubleVal(row, isNull, fLeft, fRight); - else - return fData->getLongDoubleVal(row, isNull); - } + inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getDecimalVal(row, isNull, fLeft, fRight); + else + return fData->getDecimalVal(row, isNull); + } - inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getDecimalVal(row, isNull, fLeft, fRight); - else - return fData->getDecimalVal(row, isNull); - } + inline bool getBoolVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getBoolVal(row, isNull, fLeft, fRight); + else + return fData->getBoolVal(row, isNull); + } - inline bool getBoolVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getBoolVal(row, isNull, fLeft, fRight); - else - return fData->getBoolVal(row, isNull); - } + inline int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getDateIntVal(row, isNull, fLeft, fRight); + else + return fData->getDateIntVal(row, isNull); + } - inline int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getDateIntVal(row, isNull, fLeft, fRight); - else - return fData->getDateIntVal(row, isNull); - } + inline int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getDatetimeIntVal(row, isNull, fLeft, fRight); + else + return fData->getDatetimeIntVal(row, isNull); + } - inline int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getDatetimeIntVal(row, isNull, fLeft, fRight); - else - return fData->getDatetimeIntVal(row, isNull); - } + inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimestampIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimestampIntVal(row, isNull); + } - inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getTimestampIntVal(row, isNull, fLeft, fRight); - else - return fData->getTimestampIntVal(row, isNull); - } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimeIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimeIntVal(row, isNull); + } - inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - if (fLeft && fRight) - return (reinterpret_cast(fData))->getTimeIntVal(row, isNull, fLeft, fRight); - else - return fData->getTimeIntVal(row, isNull); - } - -private: - /** draw the tree - * - * this function is used by draw tree to print out dot file - */ - static void draw(const ParseTree* n, std::ostream& dotFile); - - // F&E framework - void evaluate (rowgroup::Row& row, bool& isNull); + private: + /** draw the tree + * + * this function is used by draw tree to print out dot file + */ + static void draw(const ParseTree* n, std::ostream& dotFile); + // F&E framework + void evaluate(rowgroup::Row& row, bool& isNull); }; -} +} // namespace execplan #include "operator.h" @@ -329,259 +327,263 @@ namespace execplan /** * Class Definition */ -inline ParseTree::ParseTree() : - fData(0), fLeft(0), fRight(0), fDerivedTable("") +inline ParseTree::ParseTree() : fData(0), fLeft(0), fRight(0), fDerivedTable("") { } -inline ParseTree::ParseTree(TreeNode* data) : - fData(data), - fLeft(0), - fRight(0) +inline ParseTree::ParseTree(TreeNode* data) : fData(data), fLeft(0), fRight(0) { - // bug5984. Need to validate data to be not null - if (data) - fDerivedTable = data->derivedTable(); + // bug5984. Need to validate data to be not null + if (data) + fDerivedTable = data->derivedTable(); } -inline ParseTree::ParseTree(const ParseTree& rhs): - fData(0), - fLeft(0), - fRight(0), - fDerivedTable(rhs.fDerivedTable) +inline ParseTree::ParseTree(const ParseTree& rhs) + : fData(0), fLeft(0), fRight(0), fDerivedTable(rhs.fDerivedTable) { - copyTree(rhs); + copyTree(rhs); } inline ParseTree::~ParseTree() { - if (fLeft != NULL) - delete fLeft; + if (fLeft != NULL) + delete fLeft; - if (fRight != NULL) - delete fRight; + if (fRight != NULL) + delete fRight; - if (fData != NULL) - delete fData; + if (fData != NULL) + delete fData; - fLeft = NULL; - fRight = NULL; - fData = NULL; + fLeft = NULL; + fRight = NULL; + fData = NULL; } -inline void ParseTree::walk(void (*fn)( ParseTree* n)) const +inline void ParseTree::walk(void (*fn)(ParseTree* n)) const { - if (fLeft != 0) fLeft->walk(fn); + if (fLeft != 0) + fLeft->walk(fn); - if (fRight != 0) fRight->walk(fn); + if (fRight != 0) + fRight->walk(fn); - ParseTree* temp = const_cast(this); - fn (temp); + ParseTree* temp = const_cast(this); + fn(temp); } -inline void ParseTree::walk(void (*fn)( const ParseTree* n)) const +inline void ParseTree::walk(void (*fn)(const ParseTree* n)) const { - if (fLeft != 0) fLeft->walk(fn); + if (fLeft != 0) + fLeft->walk(fn); - if (fRight != 0) fRight->walk(fn); + if (fRight != 0) + fRight->walk(fn); - fn (this); + fn(this); } inline void ParseTree::walk(void (*fn)(const ParseTree* n, std::ostream& output), std::ostream& output) const { - if (fLeft != 0) fLeft->walk(fn, output); + if (fLeft != 0) + fLeft->walk(fn, output); - if (fRight != 0) fRight->walk(fn, output); + if (fRight != 0) + fRight->walk(fn, output); - fn(this, output); + fn(this, output); } inline void ParseTree::walk(void (*fn)(const ParseTree* n, void* obj), void* obj) const { - if (fLeft != 0) fLeft->walk(fn, obj); + if (fLeft != 0) + fLeft->walk(fn, obj); - if (fRight != 0) fRight->walk(fn, obj); + if (fRight != 0) + fRight->walk(fn, obj); - fn(this, obj); + fn(this, obj); } inline std::string ParseTree::toString() const { - std::ostringstream oss; - walk (ParseTree::print, oss); - return oss.str(); + std::ostringstream oss; + walk(ParseTree::print, oss); + return oss.str(); } inline void ParseTree::walk(void (*fn)(ParseTree* n, void* obj), void* obj) const { - if (fLeft != 0) fLeft->walk(fn, obj); + if (fLeft != 0) + fLeft->walk(fn, obj); - if (fRight != 0) fRight->walk(fn, obj); + if (fRight != 0) + fRight->walk(fn, obj); - fn(const_cast(this), obj); + fn(const_cast(this), obj); } inline ParseTree& ParseTree::operator=(const ParseTree& rhs) { - if (this != &rhs) - { - //copyTree(*this, rhs); - copyTree(rhs); - } + if (this != &rhs) + { + // copyTree(*this, rhs); + copyTree(rhs); + } - return *this; + return *this; } inline void ParseTree::copyTree(const ParseTree& src) { - if (fLeft != NULL) - delete fLeft; + if (fLeft != NULL) + delete fLeft; - if (fRight != NULL) - delete fRight; + if (fRight != NULL) + delete fRight; - fLeft = NULL; - fRight = NULL; + fLeft = NULL; + fRight = NULL; - if (src.left() != NULL) - { - fLeft = new ParseTree(); - fLeft->copyTree(*(src.left())); - } + if (src.left() != NULL) + { + fLeft = new ParseTree(); + fLeft->copyTree(*(src.left())); + } - if (src.right() != NULL) - { - fRight = new ParseTree(); - fRight->copyTree(*(src.right())); - } + if (src.right() != NULL) + { + fRight = new ParseTree(); + fRight->copyTree(*(src.right())); + } - delete fData; + delete fData; - if (src.data() == NULL) - fData = NULL; - else - fData = src.data()->clone(); + if (src.data() == NULL) + fData = NULL; + else + fData = src.data()->clone(); } inline void ParseTree::destroyTree(ParseTree* root) { - if (root == NULL) - return; + if (root == NULL) + return; - if (root->left() != NULL) - { - destroyTree (root->fLeft); - } + if (root->left() != NULL) + { + destroyTree(root->fLeft); + } - if (root->right() != NULL) - { - destroyTree (root->fRight); - } + if (root->right() != NULL) + { + destroyTree(root->fRight); + } - delete root; - root = 0; + delete root; + root = 0; } inline void ParseTree::draw(const ParseTree* n, std::ostream& dotFile) { - const ParseTree* r; - const ParseTree* l; - l = n->left(); - r = n->right(); + const ParseTree* r; + const ParseTree* l; + l = n->left(); + r = n->right(); - if (l != 0) - dotFile << "n" << (void*)n << " -> " << "n" << (void*)l << std::endl; + if (l != 0) + dotFile << "n" << (void*)n << " -> " + << "n" << (void*)l << std::endl; - if (r != 0) - dotFile << "n" << (void*)n << " -> " << "n" << (void*)r << std::endl; + if (r != 0) + dotFile << "n" << (void*)n << " -> " + << "n" << (void*)r << std::endl; - dotFile << "n" << (void*)n << " [label=\"" - << n->data()->data() << "\"]" << std::endl; + dotFile << "n" << (void*)n << " [label=\"" << n->data()->data() << "\"]" << std::endl; } inline void ParseTree::drawTree(std::string filename) { - std::ofstream dotFile (filename.c_str(), std::ios::out); + std::ofstream dotFile(filename.c_str(), std::ios::out); - dotFile << "digraph G {" << std::endl; - walk (draw, dotFile); - dotFile << "}" << std::endl; + dotFile << "digraph G {" << std::endl; + walk(draw, dotFile); + dotFile << "}" << std::endl; - dotFile.close(); + dotFile.close(); } inline void ParseTree::evaluate(rowgroup::Row& row, bool& isNull) { - // Non-leaf node is operator. leaf node is SimpleFilter for logical expression, - // or ReturnedColumn for arithmetical expression. - if (fLeft && fRight) - { - Operator* op = reinterpret_cast(fData); - op->evaluate(row, isNull, fLeft, fRight); - } - else - { - fData->evaluate(row, isNull); - } + // Non-leaf node is operator. leaf node is SimpleFilter for logical expression, + // or ReturnedColumn for arithmetical expression. + if (fLeft && fRight) + { + Operator* op = reinterpret_cast(fData); + op->evaluate(row, isNull, fLeft, fRight); + } + else + { + fData->evaluate(row, isNull); + } } inline void ParseTree::setDerivedTable() { - std::string lDerivedTable = ""; - std::string rDerivedTable = ""; + std::string lDerivedTable = ""; + std::string rDerivedTable = ""; - if (fLeft) + if (fLeft) + { + fLeft->setDerivedTable(); + lDerivedTable = fLeft->derivedTable(); + } + else + { + lDerivedTable = "*"; + } + + if (fRight) + { + fRight->setDerivedTable(); + rDerivedTable = fRight->derivedTable(); + } + else + { + rDerivedTable = "*"; + } + + Operator* op = dynamic_cast(fData); + + if (op) + { + if (lDerivedTable == "*") { - fLeft->setDerivedTable(); - lDerivedTable = fLeft->derivedTable(); + fDerivedTable = rDerivedTable; + } + else if (rDerivedTable == "*") + { + fDerivedTable = lDerivedTable; + } + else if (lDerivedTable == rDerivedTable) + { + fDerivedTable = lDerivedTable; // should be the same as rhs } else { - lDerivedTable = "*"; - } - - if (fRight) - { - fRight->setDerivedTable(); - rDerivedTable = fRight->derivedTable(); - } - else - { - rDerivedTable = "*"; - } - - Operator* op = dynamic_cast(fData); - - if (op) - { - if (lDerivedTable == "*") - { - fDerivedTable = rDerivedTable; - } - else if (rDerivedTable == "*") - { - fDerivedTable = lDerivedTable; - } - else if (lDerivedTable == rDerivedTable) - { - fDerivedTable = lDerivedTable; // should be the same as rhs - } - else - { - fDerivedTable = ""; - } - } - else - { - fData->setDerivedTable(); - fDerivedTable = fData->derivedTable(); - fDerivedTable = fData->derivedTable(); + fDerivedTable = ""; } + } + else + { + fData->setDerivedTable(); + fDerivedTable = fData->derivedTable(); + fDerivedTable = fData->derivedTable(); + } } typedef boost::shared_ptr SPTP; -} // namespace +} // namespace execplan #endif diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index d5bbd3b24..465f32af2 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: predicateoperator.cpp 9306 2013-03-12 15:49:11Z rdempsey $ -* -* -***********************************************************************/ + * $Id: predicateoperator.cpp 9306 2013-03-12 15:49:11Z rdempsey $ + * + * + ***********************************************************************/ #include @@ -30,35 +30,31 @@ #include "liboamcpp.h" - using namespace oam; using namespace std; namespace execplan { - /** * Constructors/Destructors */ -PredicateOperator::PredicateOperator() : - cs(NULL) +PredicateOperator::PredicateOperator() : cs(NULL) { } -PredicateOperator::PredicateOperator(const string& operatorName) : - cs(NULL) +PredicateOperator::PredicateOperator(const string& operatorName) : cs(NULL) { - data(operatorName); + data(operatorName); } PredicateOperator::PredicateOperator(const PredicateOperator& rhs) : Operator(rhs) { - data(rhs.data()); - cs = rhs.getCharset(); + data(rhs.data()); + cs = rhs.getCharset(); } -PredicateOperator:: ~PredicateOperator() +PredicateOperator::~PredicateOperator() { } @@ -71,9 +67,9 @@ PredicateOperator:: ~PredicateOperator() */ ostream& operator<<(ostream& output, const PredicateOperator& rhs) { - output << rhs.toString() << endl; - output << "OpType=" << rhs.operationType().colDataType << endl; - return output; + output << rhs.toString() << endl; + output << "OpType=" << rhs.operationType().colDataType << endl; + return output; } /** @@ -81,222 +77,214 @@ ostream& operator<<(ostream& output, const PredicateOperator& rhs) */ void PredicateOperator::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::PREDICATEOPERATOR; - //b << fData; - Operator::serialize(b); + b << (ObjectReader::id_t)ObjectReader::PREDICATEOPERATOR; + // b << fData; + Operator::serialize(b); } void PredicateOperator::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::PREDICATEOPERATOR); - //b >> fData; - Operator::unserialize(b); - cs = & datatypes::Charset(fOperationType.charsetNumber).getCharset(); + ObjectReader::checkType(b, ObjectReader::PREDICATEOPERATOR); + // b >> fData; + Operator::unserialize(b); + cs = &datatypes::Charset(fOperationType.charsetNumber).getCharset(); } bool PredicateOperator::operator==(const PredicateOperator& t) const { - if (data() == t.data()) - return true; + if (data() == t.data()) + return true; - return false; + return false; } bool PredicateOperator::operator==(const TreeNode* t) const { - const PredicateOperator* o; + const PredicateOperator* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool PredicateOperator::operator!=(const PredicateOperator& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool PredicateOperator::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } void PredicateOperator::setOpType(Type& l, Type& r) { - fOperationType = l; // Default to left side. Modify as needed. - if ( l.colDataType == execplan::CalpontSystemCatalog::DATETIME || - l.colDataType == execplan::CalpontSystemCatalog::TIME || - l.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP || - l.colDataType == execplan::CalpontSystemCatalog::DATE ) + fOperationType = l; // Default to left side. Modify as needed. + if (l.colDataType == execplan::CalpontSystemCatalog::DATETIME || + l.colDataType == execplan::CalpontSystemCatalog::TIME || + l.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP || + l.colDataType == execplan::CalpontSystemCatalog::DATE) + { + switch (r.colDataType) { - switch (r.colDataType) - { - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - fOperationType.charsetNumber = r.charsetNumber; - break; + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: fOperationType.charsetNumber = r.charsetNumber; break; - case execplan::CalpontSystemCatalog::DATETIME: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DATETIME; - fOperationType.colWidth = 8; - break; + case execplan::CalpontSystemCatalog::DATETIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DATETIME; + fOperationType.colWidth = 8; + break; - case execplan::CalpontSystemCatalog::TIMESTAMP: - fOperationType.colDataType = execplan::CalpontSystemCatalog::TIMESTAMP; - fOperationType.colWidth = 8; - break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIMESTAMP; + fOperationType.colWidth = 8; + break; - case execplan::CalpontSystemCatalog::TIME: - fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; - fOperationType.colWidth = 8; - break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; - case execplan::CalpontSystemCatalog::DATE: - fOperationType = l; - break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = l; break; - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - fOperationType.colWidth = 8; - break; - } + default: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; + fOperationType.colWidth = 8; + break; } - else if ( r.colDataType == execplan::CalpontSystemCatalog::DATETIME || - r.colDataType == execplan::CalpontSystemCatalog::TIME || - r.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP || - r.colDataType == execplan::CalpontSystemCatalog::DATE ) + } + else if (r.colDataType == execplan::CalpontSystemCatalog::DATETIME || + r.colDataType == execplan::CalpontSystemCatalog::TIME || + r.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP || + r.colDataType == execplan::CalpontSystemCatalog::DATE) + { + switch (l.colDataType) { - switch (l.colDataType) - { - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - fOperationType.colDataType = execplan::CalpontSystemCatalog::VARCHAR; - fOperationType.colWidth = 255; - break; + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + fOperationType.colDataType = execplan::CalpontSystemCatalog::VARCHAR; + fOperationType.colWidth = 255; + break; - case execplan::CalpontSystemCatalog::DATETIME: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DATETIME; - fOperationType.colWidth = 8; - break; + case execplan::CalpontSystemCatalog::DATETIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DATETIME; + fOperationType.colWidth = 8; + break; - case execplan::CalpontSystemCatalog::TIMESTAMP: - fOperationType.colDataType = execplan::CalpontSystemCatalog::TIMESTAMP; - fOperationType.colWidth = 8; - break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIMESTAMP; + fOperationType.colWidth = 8; + break; - case execplan::CalpontSystemCatalog::TIME: - fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; - fOperationType.colWidth = 8; - break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; - case execplan::CalpontSystemCatalog::DATE: - fOperationType = r; - break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = r; break; - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - fOperationType.colWidth = 8; - break; - } + default: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; + fOperationType.colWidth = 8; + break; } - else if (l.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - l.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + } + else if (l.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + l.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + switch (r.colDataType) { - switch (r.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - // should following the result type that MySQL gives - fOperationType = l; - fOperationType.scale = std::max(l.scale, r.scale); - fOperationType.precision = std::max(l.precision, r.precision); - fOperationType.colWidth = std::max(l.colWidth, r.colWidth); - break; - } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + // should following the result type that MySQL gives + fOperationType = l; + fOperationType.scale = std::max(l.scale, r.scale); + fOperationType.precision = std::max(l.precision, r.precision); + fOperationType.colWidth = std::max(l.colWidth, r.colWidth); + break; + } - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::UBIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = l.scale; - fOperationType.precision = l.precision; - fOperationType.colWidth = (l.colWidth == datatypes::MAXDECIMALWIDTH) ? - l.colWidth : 8; - break; + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::UBIGINT: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + fOperationType.scale = l.scale; + fOperationType.precision = l.precision; + fOperationType.colWidth = (l.colWidth == datatypes::MAXDECIMALWIDTH) ? l.colWidth : 8; + break; - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - fOperationType.colWidth = 8; - } - } - else if (r.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - r.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) - { - switch (l.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - // should following the result type that MySQL gives based on the following logic? - // @NOTE is this trustable? - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::UBIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = r.scale; - fOperationType.precision = r.precision; - fOperationType.colWidth = (r.colWidth == datatypes::MAXDECIMALWIDTH) ? - r.colWidth : 8; - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - fOperationType.colDataType = execplan::CalpontSystemCatalog::LONGDOUBLE; - fOperationType.colWidth = sizeof(long double); - break; - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - fOperationType.colWidth = 8; - } - } - // If both sides are unsigned, use UBIGINT as result type, otherwise - // "promote" to BIGINT. - else if (isUnsigned(l.colDataType) && isUnsigned(r.colDataType)) - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::UBIGINT; + default: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; fOperationType.colWidth = 8; } - else if ((isSignedInteger(l.colDataType) && isUnsigned(r.colDataType)) || - (isUnsigned(l.colDataType) && isSignedInteger(r.colDataType)) || - (isSignedInteger(l.colDataType) && isSignedInteger(r.colDataType))) + } + else if (r.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + r.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + switch (l.colDataType) { - fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + // should following the result type that MySQL gives based on the following logic? + // @NOTE is this trustable? + fOperationType = fResultType; + break; + } + + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::UBIGINT: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + fOperationType.scale = r.scale; + fOperationType.precision = r.precision; + fOperationType.colWidth = (r.colWidth == datatypes::MAXDECIMALWIDTH) ? r.colWidth : 8; + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + fOperationType.colDataType = execplan::CalpontSystemCatalog::LONGDOUBLE; + fOperationType.colWidth = sizeof(long double); + break; + default: + fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; fOperationType.colWidth = 8; } - else if ((l.colDataType == execplan::CalpontSystemCatalog::CHAR || - l.colDataType == execplan::CalpontSystemCatalog::VARCHAR || - l.colDataType == execplan::CalpontSystemCatalog::TEXT) && - (r.colDataType == execplan::CalpontSystemCatalog::CHAR || - r.colDataType == execplan::CalpontSystemCatalog::VARCHAR || - r.colDataType == execplan::CalpontSystemCatalog::TEXT)) - { + } + // If both sides are unsigned, use UBIGINT as result type, otherwise + // "promote" to BIGINT. + else if (isUnsigned(l.colDataType) && isUnsigned(r.colDataType)) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::UBIGINT; + fOperationType.colWidth = 8; + } + else if ((isSignedInteger(l.colDataType) && isUnsigned(r.colDataType)) || + (isUnsigned(l.colDataType) && isSignedInteger(r.colDataType)) || + (isSignedInteger(l.colDataType) && isSignedInteger(r.colDataType))) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; + fOperationType.colWidth = 8; + } + else if ((l.colDataType == execplan::CalpontSystemCatalog::CHAR || + l.colDataType == execplan::CalpontSystemCatalog::VARCHAR || + l.colDataType == execplan::CalpontSystemCatalog::TEXT) && + (r.colDataType == execplan::CalpontSystemCatalog::CHAR || + r.colDataType == execplan::CalpontSystemCatalog::VARCHAR || + r.colDataType == execplan::CalpontSystemCatalog::TEXT)) + { #if 0 // Currently, STRINT isn't properly implemented everywhere // For short strings, we can get a faster execution for charset that fit in one byte. @@ -328,427 +316,416 @@ void PredicateOperator::setOpType(Type& l, Type& r) } else #endif - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::VARCHAR; - fOperationType.colWidth = 255; - } - } - else if (l.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE || - r.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE) { - fOperationType.colDataType = execplan::CalpontSystemCatalog::LONGDOUBLE; - fOperationType.colWidth = sizeof(long double); - } - else - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - fOperationType.colWidth = 8; + fOperationType.colDataType = execplan::CalpontSystemCatalog::VARCHAR; + fOperationType.colWidth = 255; } + } + else if (l.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE || + r.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::LONGDOUBLE; + fOperationType.colWidth = sizeof(long double); + } + else + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; + fOperationType.colWidth = 8; + } - cs = & datatypes::Charset(fOperationType.charsetNumber).getCharset(); + cs = &datatypes::Charset(fOperationType.charsetNumber).getCharset(); } inline bool PredicateOperator::strTrimCompare(const std::string& op1, const std::string& op2) { - int r1 = cs->strnncollsp(op1.c_str(), op1.length(), op2.c_str(), op2.length()); - switch (fOp) + int r1 = cs->strnncollsp(op1.c_str(), op1.length(), op2.c_str(), op2.length()); + switch (fOp) + { + case OP_EQ: return r1 == 0; + + case OP_NE: return r1 != 0; + + case OP_GT: return r1 > 0; + + case OP_GE: return r1 >= 0; + + case OP_LT: return r1 < 0; + + case OP_LE: return r1 <= 0; + + default: { - case OP_EQ: - return r1 == 0; - - case OP_NE: - return r1 != 0; - - case OP_GT: - return r1 > 0; - - case OP_GE: - return r1 >= 0; - - case OP_LT: - return r1 < 0; - - case OP_LE: - return r1 <= 0; - - default: - { - std::ostringstream oss; - oss << "Unsupported predicate operation: " << fOp; - throw logging::InvalidOperationExcept(oss.str()); - } + std::ostringstream oss; + oss << "Unsupported predicate operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); } + } } bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) { - // like operator. both sides are string. - if (fOp == OP_LIKE || fOp == OP_NOTLIKE) + // like operator. both sides are string. + if (fOp == OP_LIKE || fOp == OP_NOTLIKE) + { + const std::string& subject = lop->getStrVal(row, isNull); + if (isNull) + return false; + const std::string& pattern = rop->getStrVal(row, isNull); + if (isNull) + return false; + return datatypes::Charset(cs).like(fOp == OP_NOTLIKE, utils::ConstString(subject), + utils::ConstString(pattern)); + } + + // fOpType should have already been set on the connector during parsing + switch (fOperationType.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - const std::string & subject = lop->getStrVal(row, isNull); - if (isNull) - return false; - const std::string & pattern = rop->getStrVal(row, isNull); - if (isNull) - return false; - return datatypes::Charset(cs).like(fOp == OP_NOTLIKE, - utils::ConstString(subject), - utils::ConstString(pattern)); + if (fOp == OP_ISNULL) + { + lop->getIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getIntVal(row, isNull)) && !isNull; } - // fOpType should have already been set on the connector during parsing - switch (fOperationType.colDataType) + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - if (fOp == OP_ISNULL) - { - lop->getIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } + if (fOp == OP_ISNULL) + { + lop->getUintVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } - if (fOp == OP_ISNOTNULL) - { - lop->getIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } + if (fOp == OP_ISNOTNULL) + { + lop->getUintVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } - if (isNull) - return false; + if (isNull) + return false; - int64_t val1 = lop->getIntVal(row, isNull); + uint64_t val1 = lop->getUintVal(row, isNull); - if (isNull) - return false; + if (isNull) + return false; - return numericCompare(val1, rop->getIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - if (fOp == OP_ISNULL) - { - lop->getUintVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getUintVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - uint64_t val1 = lop->getUintVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getUintVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - if (fOp == OP_ISNULL) - { - lop->getDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - double val1 = lop->getDoubleVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getDoubleVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - if (fOp == OP_ISNULL) - { - lop->getLongDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getLongDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - long double val1 = lop->getLongDoubleVal(row, isNull); - if (isNull) - return false; - - long double val2 = rop->getLongDoubleVal(row, isNull); - if (isNull) - return false; - - // In many case, rounding error will prevent an eq compare to work - // In these cases, use the largest scale of the two items. - if (fOp == execplan::OP_EQ) - { - // In case a val is a representation of a very large integer, - // we won't want to just multiply by scale, as it may move - // significant digits out of scope. So we break them apart - // and compare each separately - int64_t scale = std::max(lop->resultType().scale, rop->resultType().scale); - if (scale) - { - long double intpart1; - long double fract1 = modfl(val1, &intpart1); - long double intpart2; - long double fract2 = modfl(val2, &intpart2); - if (numericCompare(intpart1, intpart2)) - { - double factor = pow(10.0, (double)scale); - fract1 = roundl(fract1 * factor); - fract2 = roundl(fract2 * factor); - return numericCompare(fract1, fract2); - } - else - { - return false; - } - } - } - return numericCompare(val1, val2); - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (fOp == OP_ISNULL) - { - lop->getDecimalVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDecimalVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - IDB_Decimal val1 = lop->getDecimalVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getDecimalVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::DATE: - { - if (fOp == OP_ISNULL) - { - lop->getDateIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDateIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getDateIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, (int64_t)rop->getDateIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - if (fOp == OP_ISNULL) - { - lop->getDatetimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDatetimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getDatetimeIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - if (fOp == OP_ISNULL) - { - lop->getTimestampIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getTimestampIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getTimestampIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getTimestampIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::TIME: - { - if (fOp == OP_ISNULL) - { - lop->getTimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getTimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getTimeIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; - } - - - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - if (fOp == OP_ISNULL) - { - lop->getStrVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getStrVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - const std::string& val1 = lop->getStrVal(row, isNull); - if (isNull) - return false; - - return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::BLOB: - return false; - break; - - default: - { - std::ostringstream oss; - oss << "invalid predicate operation type: " << fOperationType.colDataType; - throw logging::InvalidOperationExcept(oss.str()); - } + return numericCompare(val1, rop->getUintVal(row, isNull)) && !isNull; } - return false; + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + if (fOp == OP_ISNULL) + { + lop->getDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + double val1 = lop->getDoubleVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getDoubleVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + if (fOp == OP_ISNULL) + { + lop->getLongDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getLongDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + long double val1 = lop->getLongDoubleVal(row, isNull); + if (isNull) + return false; + + long double val2 = rop->getLongDoubleVal(row, isNull); + if (isNull) + return false; + + // In many case, rounding error will prevent an eq compare to work + // In these cases, use the largest scale of the two items. + if (fOp == execplan::OP_EQ) + { + // In case a val is a representation of a very large integer, + // we won't want to just multiply by scale, as it may move + // significant digits out of scope. So we break them apart + // and compare each separately + int64_t scale = std::max(lop->resultType().scale, rop->resultType().scale); + if (scale) + { + long double intpart1; + long double fract1 = modfl(val1, &intpart1); + long double intpart2; + long double fract2 = modfl(val2, &intpart2); + if (numericCompare(intpart1, intpart2)) + { + double factor = pow(10.0, (double)scale); + fract1 = roundl(fract1 * factor); + fract2 = roundl(fract2 * factor); + return numericCompare(fract1, fract2); + } + else + { + return false; + } + } + } + return numericCompare(val1, val2); + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (fOp == OP_ISNULL) + { + lop->getDecimalVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDecimalVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + IDB_Decimal val1 = lop->getDecimalVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getDecimalVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::DATE: + { + if (fOp == OP_ISNULL) + { + lop->getDateIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDateIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getDateIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, (int64_t)rop->getDateIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + if (fOp == OP_ISNULL) + { + lop->getDatetimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDatetimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getDatetimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if (fOp == OP_ISNULL) + { + lop->getTimestampIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimestampIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimestampIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimestampIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::TIME: + { + if (fOp == OP_ISNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + if (fOp == OP_ISNULL) + { + lop->getStrVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getStrVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + const std::string& val1 = lop->getStrVal(row, isNull); + if (isNull) + return false; + + return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: return false; break; + + default: + { + std::ostringstream oss; + oss << "invalid predicate operation type: " << fOperationType.colDataType; + throw logging::InvalidOperationExcept(oss.str()); + } + } + + return false; } -} // namespace +} // namespace execplan diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 154771112..faafc67e6 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: predicateoperator.h 9667 2013-07-08 16:37:10Z bpaul $ -* -* -***********************************************************************/ + * $Id: predicateoperator.h 9667 2013-07-08 16:37:10Z bpaul $ + * + * + ***********************************************************************/ /** @file */ #ifndef PREDICATEOPERATOR_H @@ -43,7 +43,7 @@ #include "returnedcolumn.h" #include "dataconvert.h" -#include "collation.h" // CHARSET_INFO +#include "collation.h" // CHARSET_INFO namespace messageqcpp { @@ -52,145 +52,129 @@ class ByteStream; namespace execplan { - class PredicateOperator : public Operator { + public: + PredicateOperator(); + PredicateOperator(const std::string& operatorName); + PredicateOperator(const PredicateOperator& rhs); + virtual ~PredicateOperator(); -public: - PredicateOperator(); - PredicateOperator(const std::string& operatorName); - PredicateOperator(const PredicateOperator& rhs); - virtual ~PredicateOperator(); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual PredicateOperator* clone() const + { + return new PredicateOperator(*this); + } + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual PredicateOperator* clone() const - { - return new PredicateOperator (*this); - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const PredicateOperator& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const PredicateOperator& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const PredicateOperator& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + const CHARSET_INFO* getCharset() const + { + return cs; + } + /*********************************************************** + * F&E framework * + ***********************************************************/ + using Operator::getBoolVal; + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop); + void setOpType(Type& l, Type& r); - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const PredicateOperator& t) const; + private: + inline bool numericCompare(const IDB_Decimal& op1, const IDB_Decimal& op2); + template + inline bool numericCompare(const result_t op1, const result_t op2); + inline bool strTrimCompare(const std::string& op1, const std::string& op2); - const CHARSET_INFO* getCharset() const - { - return cs; - } - /*********************************************************** - * F&E framework * - ***********************************************************/ - using Operator::getBoolVal; - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop); - void setOpType(Type& l, Type& r); - -private: - inline bool numericCompare(const IDB_Decimal& op1, const IDB_Decimal& op2); - template - inline bool numericCompare(const result_t op1, const result_t op2); - inline bool strTrimCompare(const std::string& op1, const std::string& op2); - - const CHARSET_INFO* cs; + const CHARSET_INFO* cs; }; inline bool PredicateOperator::numericCompare(const IDB_Decimal& op1, const IDB_Decimal& op2) { - switch (fOp) + switch (fOp) + { + case OP_EQ: return op1 == op2; + + case OP_NE: return op1 != op2; + + case OP_GT: return op1 > op2; + + case OP_GE: return op1 >= op2; + + case OP_LT: return op1 < op2; + + case OP_LE: return op1 <= op2; + + default: { - case OP_EQ: - return op1 == op2; - - case OP_NE: - return op1 != op2; - - case OP_GT: - return op1 > op2; - - case OP_GE: - return op1 >= op2; - - case OP_LT: - return op1 < op2; - - case OP_LE: - return op1 <= op2; - - default: - { - std::ostringstream oss; - oss << "invalid predicate operation: " << fOp; - throw logging::InvalidOperationExcept(oss.str()); - } + std::ostringstream oss; + oss << "invalid predicate operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); } + } } template inline bool PredicateOperator::numericCompare(const result_t op1, const result_t op2) { - switch (fOp) + switch (fOp) + { + case OP_EQ: return op1 == op2; + + case OP_NE: return op1 != op2; + + case OP_GT: return op1 > op2; + + case OP_GE: return op1 >= op2; + + case OP_LT: return op1 < op2; + + case OP_LE: return op1 <= op2; + + default: { - case OP_EQ: - return op1 == op2; - - case OP_NE: - return op1 != op2; - - case OP_GT: - return op1 > op2; - - case OP_GE: - return op1 >= op2; - - case OP_LT: - return op1 < op2; - - case OP_LE: - return op1 <= op2; - - default: - { - std::ostringstream oss; - oss << "invalid predicate operation: " << fOp; - throw logging::InvalidOperationExcept(oss.str()); - } + std::ostringstream oss; + oss << "invalid predicate operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); } + } } std::ostream& operator<<(std::ostream& os, const PredicateOperator& rhs); -} - -#endif //PREDICATEOPERATOR_H +} // namespace execplan +#endif // PREDICATEOPERATOR_H diff --git a/dbcon/execplan/pseudocolumn.cpp b/dbcon/execplan/pseudocolumn.cpp index dcf35596b..35bd677bb 100644 --- a/dbcon/execplan/pseudocolumn.cpp +++ b/dbcon/execplan/pseudocolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: pseudocolumn.cpp 9576 2013-05-29 21:02:11Z zzhu $ -* -* -***********************************************************************/ + * $Id: pseudocolumn.cpp 9576 2013-05-29 21:02:11Z zzhu $ + * + * + ***********************************************************************/ #include #include @@ -36,71 +36,52 @@ using namespace messageqcpp; namespace execplan { - /** * Constructors/Destructors */ -PseudoColumn::PseudoColumn(): - SimpleColumn(), - fPseudoType (PSEUDO_UNKNOWN) -{} - -PseudoColumn::PseudoColumn(const uint32_t pseudoType): - SimpleColumn(), - fPseudoType(pseudoType) -{} - -PseudoColumn::PseudoColumn(const string& token, - const uint32_t pseudoType, - const uint32_t sessionID): - SimpleColumn(token, sessionID), - fPseudoType(pseudoType) +PseudoColumn::PseudoColumn() : SimpleColumn(), fPseudoType(PSEUDO_UNKNOWN) { - adjustResultType(); } -PseudoColumn::PseudoColumn(const string& schemaName, - const string& tableName, - const string& columnName, - const uint32_t pseudoType, - const uint32_t sessionID): - SimpleColumn(schemaName, tableName, columnName, sessionID), - fPseudoType(pseudoType) - -{} - -PseudoColumn::PseudoColumn(const string& schemaName, - const string& tableName, - const string& columnName, - const bool isColumnStore, - const uint32_t pseudoType, - const uint32_t sessionID): - SimpleColumn(schemaName, tableName, columnName, isColumnStore, sessionID), - fPseudoType(pseudoType) +PseudoColumn::PseudoColumn(const uint32_t pseudoType) : SimpleColumn(), fPseudoType(pseudoType) { - adjustResultType(); } -PseudoColumn::PseudoColumn (const SimpleColumn& rhs, - const uint32_t pseudoType, - const uint32_t sessionID): - SimpleColumn(rhs, sessionID), - fPseudoType(pseudoType) +PseudoColumn::PseudoColumn(const string& token, const uint32_t pseudoType, const uint32_t sessionID) + : SimpleColumn(token, sessionID), fPseudoType(pseudoType) { - adjustResultType(); + adjustResultType(); } -PseudoColumn::PseudoColumn(const PseudoColumn& rhs, const uint32_t sessionID): - SimpleColumn(rhs, sessionID), - fPseudoType (rhs.pseudoType()) +PseudoColumn::PseudoColumn(const string& schemaName, const string& tableName, const string& columnName, + const uint32_t pseudoType, const uint32_t sessionID) + : SimpleColumn(schemaName, tableName, columnName, sessionID), fPseudoType(pseudoType) + { - adjustResultType(); } +PseudoColumn::PseudoColumn(const string& schemaName, const string& tableName, const string& columnName, + const bool isColumnStore, const uint32_t pseudoType, const uint32_t sessionID) + : SimpleColumn(schemaName, tableName, columnName, isColumnStore, sessionID), fPseudoType(pseudoType) +{ + adjustResultType(); +} +PseudoColumn::PseudoColumn(const SimpleColumn& rhs, const uint32_t pseudoType, const uint32_t sessionID) + : SimpleColumn(rhs, sessionID), fPseudoType(pseudoType) +{ + adjustResultType(); +} + +PseudoColumn::PseudoColumn(const PseudoColumn& rhs, const uint32_t sessionID) + : SimpleColumn(rhs, sessionID), fPseudoType(rhs.pseudoType()) +{ + adjustResultType(); +} PseudoColumn::~PseudoColumn() -{} +{ +} /** * Methods @@ -108,179 +89,169 @@ PseudoColumn::~PseudoColumn() PseudoColumn& PseudoColumn::operator=(const PseudoColumn& rhs) { - if (this != &rhs) - { - fTableName = rhs.tableName(); - fColumnName = rhs.columnName(); - fOid = rhs.oid(); - fSchemaName = rhs.schemaName(); - fAlias = rhs.alias(); - fTableAlias = rhs.tableAlias(); - fAsc = rhs.asc(); - fIndexName = rhs.indexName(); - fViewName = rhs.viewName(); - fData = rhs.data(); - fSequence = rhs.sequence(); - fDistinct = rhs.distinct(); - fisColumnStore = rhs.isColumnStore(); - fPseudoType = rhs.pseudoType(); - } + if (this != &rhs) + { + fTableName = rhs.tableName(); + fColumnName = rhs.columnName(); + fOid = rhs.oid(); + fSchemaName = rhs.schemaName(); + fAlias = rhs.alias(); + fTableAlias = rhs.tableAlias(); + fAsc = rhs.asc(); + fIndexName = rhs.indexName(); + fViewName = rhs.viewName(); + fData = rhs.data(); + fSequence = rhs.sequence(); + fDistinct = rhs.distinct(); + fisColumnStore = rhs.isColumnStore(); + fPseudoType = rhs.pseudoType(); + } - return *this; + return *this; } ostream& operator<<(ostream& output, const PseudoColumn& rhs) { - output << rhs.toString(); + output << rhs.toString(); - return output; + return output; } const string PseudoColumn::toString() const { - ostringstream output; - output << "PseudoColumn " << data() << endl; + ostringstream output; + output << "PseudoColumn " << data() << endl; - output << "SimpleColumn " << data() << endl; - output << " s/t/c/v/o/ct/TA/CA/RA/#/card/join/source/engine: " << schemaName() << '/' - << tableName() << '/' - << columnName() << '/' - << viewName() << '/' - << oid() << '/' - << colDataTypeToString(fResultType.colDataType) << '/' - << tableAlias() << '/' - << alias() << '/' - << returnAll() << '/' - << sequence() << '/' - << cardinality() << '/' - << joinInfo() << '/' - << colSource() << '/' - << (isColumnStore() ? "ColumnStore" : "ForeignEngine") << endl; + output << "SimpleColumn " << data() << endl; + output << " s/t/c/v/o/ct/TA/CA/RA/#/card/join/source/engine: " << schemaName() << '/' << tableName() << '/' + << columnName() << '/' << viewName() << '/' << oid() << '/' + << colDataTypeToString(fResultType.colDataType) << '/' << tableAlias() << '/' << alias() << '/' + << returnAll() << '/' << sequence() << '/' << cardinality() << '/' << joinInfo() << '/' + << colSource() << '/' << (isColumnStore() ? "ColumnStore" : "ForeignEngine") << endl; - output << "Pseudotype=" << fPseudoType << endl; - return output.str(); + output << "Pseudotype=" << fPseudoType << endl; + return output.str(); } void PseudoColumn::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::PSEUDOCOLUMN; - SimpleColumn::serialize(b); - b << static_cast(fPseudoType); + b << (ObjectReader::id_t)ObjectReader::PSEUDOCOLUMN; + SimpleColumn::serialize(b); + b << static_cast(fPseudoType); } void PseudoColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::PSEUDOCOLUMN); - SimpleColumn::unserialize(b); - b >> reinterpret_cast< ByteStream::quadbyte&>(fPseudoType); + ObjectReader::checkType(b, ObjectReader::PSEUDOCOLUMN); + SimpleColumn::unserialize(b); + b >> reinterpret_cast(fPseudoType); } bool PseudoColumn::operator==(const PseudoColumn& t) const { - const SimpleColumn* rc1, *rc2; + const SimpleColumn *rc1, *rc2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (fPseudoType != t.fPseudoType) - return false; + if (fPseudoType != t.fPseudoType) + return false; - return true; + return true; } bool PseudoColumn::operator==(const TreeNode* t) const { - const PseudoColumn* pc; + const PseudoColumn* pc; - pc = dynamic_cast(t); + pc = dynamic_cast(t); - if (pc == NULL) - return false; + if (pc == NULL) + return false; - return *this == *pc; + return *this == *pc; } bool PseudoColumn::operator!=(const PseudoColumn& t) const { - return !(*this == t); + return !(*this == t); } bool PseudoColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } -uint32_t PseudoColumn::pseudoNameToType (string& name) +uint32_t PseudoColumn::pseudoNameToType(string& name) { - if (boost::iequals(name, "idbpm")) - return PSEUDO_PM; + if (boost::iequals(name, "idbpm")) + return PSEUDO_PM; - if (boost::iequals(name, "idbdbroot")) - return PSEUDO_DBROOT; + if (boost::iequals(name, "idbdbroot")) + return PSEUDO_DBROOT; - if (boost::iequals(name, "idbextentrelativerid")) - return PSEUDO_EXTENTRELATIVERID; + if (boost::iequals(name, "idbextentrelativerid")) + return PSEUDO_EXTENTRELATIVERID; - if (boost::iequals(name, "idbsegment")) - return PSEUDO_SEGMENT; + if (boost::iequals(name, "idbsegment")) + return PSEUDO_SEGMENT; - if (boost::iequals(name, "idbsegmentdir")) - return PSEUDO_SEGMENTDIR; + if (boost::iequals(name, "idbsegmentdir")) + return PSEUDO_SEGMENTDIR; - if (boost::iequals(name, "idbextentmin")) - return PSEUDO_EXTENTMIN; + if (boost::iequals(name, "idbextentmin")) + return PSEUDO_EXTENTMIN; - if (boost::iequals(name, "idbextentmax")) - return PSEUDO_EXTENTMAX; + if (boost::iequals(name, "idbextentmax")) + return PSEUDO_EXTENTMAX; - if (boost::iequals(name, "idbblockid")) - return PSEUDO_BLOCKID; + if (boost::iequals(name, "idbblockid")) + return PSEUDO_BLOCKID; - if (boost::iequals(name, "idbextentid")) - return PSEUDO_EXTENTID; + if (boost::iequals(name, "idbextentid")) + return PSEUDO_EXTENTID; - if (boost::iequals(name, "idbpartition")) - return PSEUDO_PARTITION; + if (boost::iequals(name, "idbpartition")) + return PSEUDO_PARTITION; - if (boost::iequals(name, "idblocalpm")) - return PSEUDO_LOCALPM; + if (boost::iequals(name, "idblocalpm")) + return PSEUDO_LOCALPM; - return PSEUDO_UNKNOWN; + return PSEUDO_UNKNOWN; } void PseudoColumn::adjustResultType() { - switch (fPseudoType) + switch (fPseudoType) + { + case PSEUDO_EXTENTRELATIVERID: + case PSEUDO_DBROOT: + case PSEUDO_PM: + case PSEUDO_SEGMENT: + case PSEUDO_SEGMENTDIR: + case PSEUDO_BLOCKID: + case PSEUDO_EXTENTID: { - case PSEUDO_EXTENTRELATIVERID: - case PSEUDO_DBROOT: - case PSEUDO_PM: - case PSEUDO_SEGMENT: - case PSEUDO_SEGMENTDIR: - case PSEUDO_BLOCKID: - case PSEUDO_EXTENTID: - { - fResultType.colDataType = CalpontSystemCatalog::BIGINT; - fResultType.colWidth = 8; - break; - } - - case PSEUDO_PARTITION: - { - fResultType.colDataType = CalpontSystemCatalog::VARCHAR; - fResultType.colWidth = 256; - break; - } - - default: - { - // same type of pseudocolumn arg - } + fResultType.colDataType = CalpontSystemCatalog::BIGINT; + fResultType.colWidth = 8; + break; } + case PSEUDO_PARTITION: + { + fResultType.colDataType = CalpontSystemCatalog::VARCHAR; + fResultType.colWidth = 256; + break; + } + + default: + { + // same type of pseudocolumn arg + } + } } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/pseudocolumn.h b/dbcon/execplan/pseudocolumn.h index 51ad55dcb..52290f809 100644 --- a/dbcon/execplan/pseudocolumn.h +++ b/dbcon/execplan/pseudocolumn.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplecolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplecolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef PSEUDOCOLUMN_H @@ -37,7 +37,6 @@ class ByteStream; */ namespace execplan { - const uint32_t PSEUDO_UNKNOWN = 0; const uint32_t PSEUDO_EXTENTRELATIVERID = 1; const uint32_t PSEUDO_DBROOT = 2; @@ -59,111 +58,98 @@ const uint32_t PSEUDO_LOCALPM = 11; */ class PseudoColumn : public SimpleColumn { + public: + /** + * Constructors + */ + PseudoColumn(); + PseudoColumn(const uint32_t pseudoType); + PseudoColumn(const std::string& token, const uint32_t pseudoType, const uint32_t sessionID = 0); + PseudoColumn(const std::string& schema, const std::string& table, const std::string& col, + const uint32_t pseudoType, const uint32_t sessionID = 0); + PseudoColumn(const std::string& schema, const std::string& table, const std::string& col, + const bool isColumnStore, const uint32_t pseudoType, const uint32_t sessionID = 0); + PseudoColumn(const SimpleColumn& rhs, const uint32_t pseudoType, const uint32_t sessionID = 0); + PseudoColumn(const PseudoColumn& rhs, const uint32_t sessionID = 0); -public: + /** + * Destructor + */ + virtual ~PseudoColumn(); - /** - * Constructors - */ - PseudoColumn(); - PseudoColumn(const uint32_t pseudoType); - PseudoColumn(const std::string& token, - const uint32_t pseudoType, - const uint32_t sessionID = 0); - PseudoColumn(const std::string& schema, - const std::string& table, - const std::string& col, - const uint32_t pseudoType, - const uint32_t sessionID = 0); - PseudoColumn(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t pseudoType, - const uint32_t sessionID = 0); - PseudoColumn(const SimpleColumn& rhs, const uint32_t pseudoType, const uint32_t sessionID = 0); - PseudoColumn(const PseudoColumn& rhs, const uint32_t sessionID = 0); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual PseudoColumn* clone() const + { + return new PseudoColumn(*this); + } - /** - * Destructor - */ - virtual ~PseudoColumn(); + /** + * Overloaded assignment operator + */ + PseudoColumn& operator=(const PseudoColumn& rhs); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual PseudoColumn* clone() const - { - return new PseudoColumn (*this); - } + /** + * Accessor and mutator + */ + uint32_t pseudoType() const + { + return fPseudoType; + } - /** - * Overloaded assignment operator - */ - PseudoColumn& operator=(const PseudoColumn& rhs); + void pseudoType(const uint32_t pseudoType) + { + fPseudoType = pseudoType; + } - /** - * Accessor and mutator - */ - uint32_t pseudoType() const - { - return fPseudoType; - } + /** + * The serialize interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - void pseudoType (const uint32_t pseudoType) - { - fPseudoType = pseudoType; - } + virtual const std::string toString() const; - /** - * The serialize interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - virtual const std::string toString() const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const PseudoColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const PseudoColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const PseudoColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + static uint32_t pseudoNameToType(std::string& name); - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const PseudoColumn& t) const; - - static uint32_t pseudoNameToType(std::string& name); - - -private: - /** - * Fields - */ - uint32_t fPseudoType; - void adjustResultType(); + private: + /** + * Fields + */ + uint32_t fPseudoType; + void adjustResultType(); }; -} // namespace execplan - -#endif //PSEUDOCOLUMN_H +} // namespace execplan +#endif // PSEUDOCOLUMN_H diff --git a/dbcon/execplan/range.cpp b/dbcon/execplan/range.cpp index 859de25ec..ebb8294e5 100644 --- a/dbcon/execplan/range.cpp +++ b/dbcon/execplan/range.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: range.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: range.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "range.h" @@ -29,4 +29,3 @@ /** * Methods */ - diff --git a/dbcon/execplan/range.h b/dbcon/execplan/range.h index 814c44bac..5fc3549d6 100644 --- a/dbcon/execplan/range.h +++ b/dbcon/execplan/range.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: range.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: range.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef RANGE_H @@ -37,64 +37,63 @@ namespace execplan */ class Range { - /** - * Public stuff - */ -public: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Protected stuff - */ -protected: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ - /** - * Private stuff - */ -private: - /** - * Fields - */ - /** - * - */ - /** - * Constructors - */ - /** - * Accessor Methods - */ - /** - * Operations - */ + /** + * Public stuff + */ + public: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Protected stuff + */ + protected: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ + /** + * Private stuff + */ + private: + /** + * Fields + */ + /** + * + */ + /** + * Constructors + */ + /** + * Accessor Methods + */ + /** + * Operations + */ }; -} -#endif //RANGE_H - +} // namespace execplan +#endif // RANGE_H diff --git a/dbcon/execplan/returnedcolumn.cpp b/dbcon/execplan/returnedcolumn.cpp index 5900f077f..fffc5a26b 100644 --- a/dbcon/execplan/returnedcolumn.cpp +++ b/dbcon/execplan/returnedcolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: returnedcolumn.cpp 9413 2013-04-22 22:03:42Z zzhu $ -* -* -***********************************************************************/ + * $Id: returnedcolumn.cpp 9413 2013-04-22 22:03:42Z zzhu $ + * + * + ***********************************************************************/ #include #include @@ -36,85 +36,86 @@ namespace execplan /** * Constructors/Destructors */ -ReturnedColumn::ReturnedColumn(): fReturnAll (false), - fSessionID(0), - fSequence(-1), - fCardinality(0), - fDistinct(false), - fJoinInfo(0), - fAsc(true), - fNullsFirst(true), - fOrderPos((uint64_t) - 1), - fColSource(0), - fColPosition(-1), - fHasAggregate(false), - fInputIndex(-1), - fInputOffset(-1), - fOutputIndex(-1), - fExpressionId ((uint32_t) - 1) +ReturnedColumn::ReturnedColumn() + : fReturnAll(false) + , fSessionID(0) + , fSequence(-1) + , fCardinality(0) + , fDistinct(false) + , fJoinInfo(0) + , fAsc(true) + , fNullsFirst(true) + , fOrderPos((uint64_t)-1) + , fColSource(0) + , fColPosition(-1) + , fHasAggregate(false) + , fInputIndex(-1) + , fInputOffset(-1) + , fOutputIndex(-1) + , fExpressionId((uint32_t)-1) { } -ReturnedColumn::ReturnedColumn(const string& sql): - fReturnAll (false), - fSessionID(0), - fSequence(-1), - fCardinality(0), - fDistinct(false), - fJoinInfo(0), - fAsc(true), - fNullsFirst(true), - fOrderPos((uint64_t) - 1), - fColSource(0), - fColPosition(-1), - fHasAggregate(false), - fData(sql), - fInputIndex(-1), - fInputOffset(-1), - fOutputIndex(-1), - fExpressionId ((uint32_t) - 1) +ReturnedColumn::ReturnedColumn(const string& sql) + : fReturnAll(false) + , fSessionID(0) + , fSequence(-1) + , fCardinality(0) + , fDistinct(false) + , fJoinInfo(0) + , fAsc(true) + , fNullsFirst(true) + , fOrderPos((uint64_t)-1) + , fColSource(0) + , fColPosition(-1) + , fHasAggregate(false) + , fData(sql) + , fInputIndex(-1) + , fInputOffset(-1) + , fOutputIndex(-1) + , fExpressionId((uint32_t)-1) { } -ReturnedColumn::ReturnedColumn(const uint32_t sessionID, const bool returnAll): - fReturnAll(returnAll), - fSessionID(sessionID), - fSequence(-1), - fCardinality(0), - fDistinct(false), - fJoinInfo(0), - fAsc(true), - fNullsFirst(true), - fOrderPos((uint64_t) - 1), - fColSource(0), - fColPosition(-1), - fHasAggregate(false), - fInputIndex(-1), - fInputOffset(-1), - fOutputIndex(-1), - fExpressionId ((uint32_t) - 1) +ReturnedColumn::ReturnedColumn(const uint32_t sessionID, const bool returnAll) + : fReturnAll(returnAll) + , fSessionID(sessionID) + , fSequence(-1) + , fCardinality(0) + , fDistinct(false) + , fJoinInfo(0) + , fAsc(true) + , fNullsFirst(true) + , fOrderPos((uint64_t)-1) + , fColSource(0) + , fColPosition(-1) + , fHasAggregate(false) + , fInputIndex(-1) + , fInputOffset(-1) + , fOutputIndex(-1) + , fExpressionId((uint32_t)-1) { } -ReturnedColumn::ReturnedColumn(const ReturnedColumn& rhs, const uint32_t sessionID): - TreeNode(rhs), - fReturnAll(rhs.fReturnAll), - fSessionID(sessionID), - fSequence(rhs.fSequence), - fCardinality(rhs.fCardinality), - fDistinct(rhs.fDistinct), - fJoinInfo(rhs.fJoinInfo), - fAsc(rhs.fAsc), - fNullsFirst(rhs.fNullsFirst), - fOrderPos(rhs.fOrderPos), - fColSource(rhs.fColSource), - fColPosition(rhs.fColPosition), - fHasAggregate(rhs.fHasAggregate), - fData(rhs.fData), - fInputIndex(rhs.fInputIndex), - fInputOffset(rhs.fInputOffset), - fOutputIndex(rhs.fOutputIndex), - fExpressionId (rhs.fExpressionId) +ReturnedColumn::ReturnedColumn(const ReturnedColumn& rhs, const uint32_t sessionID) + : TreeNode(rhs) + , fReturnAll(rhs.fReturnAll) + , fSessionID(sessionID) + , fSequence(rhs.fSequence) + , fCardinality(rhs.fCardinality) + , fDistinct(rhs.fDistinct) + , fJoinInfo(rhs.fJoinInfo) + , fAsc(rhs.fAsc) + , fNullsFirst(rhs.fNullsFirst) + , fOrderPos(rhs.fOrderPos) + , fColSource(rhs.fColSource) + , fColPosition(rhs.fColPosition) + , fHasAggregate(rhs.fHasAggregate) + , fData(rhs.fData) + , fInputIndex(rhs.fInputIndex) + , fInputOffset(rhs.fInputOffset) + , fOutputIndex(rhs.fOutputIndex) + , fExpressionId(rhs.fExpressionId) { } @@ -127,124 +128,124 @@ ReturnedColumn::~ReturnedColumn() */ ostream& operator<<(ostream& output, const ReturnedColumn& retCol) { - output << retCol.toString() << endl; - return output; + output << retCol.toString() << endl; + return output; } void ReturnedColumn::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::RETURNEDCOLUMN; - b << fData; - b << (uint64_t) fCardinality; - b << fAlias; - b << (uint8_t)fDistinct; - b << (uint64_t)fJoinInfo; - b << (uint8_t)fAsc; - b << (uint8_t)fNullsFirst; - b << (uint64_t)fOrderPos; - b << (uint64_t)fColSource; - b << (int64_t)fColPosition; - b << (uint32_t)fInputIndex; - b << (uint32_t)fInputOffset; - b << (uint32_t)fOutputIndex; - b << (int32_t)fSequence; - b << (uint8_t)fReturnAll; - fResultType.serialize(b); - fOperationType.serialize(b); - b << (uint32_t)fExpressionId; + b << (ObjectReader::id_t)ObjectReader::RETURNEDCOLUMN; + b << fData; + b << (uint64_t)fCardinality; + b << fAlias; + b << (uint8_t)fDistinct; + b << (uint64_t)fJoinInfo; + b << (uint8_t)fAsc; + b << (uint8_t)fNullsFirst; + b << (uint64_t)fOrderPos; + b << (uint64_t)fColSource; + b << (int64_t)fColPosition; + b << (uint32_t)fInputIndex; + b << (uint32_t)fInputOffset; + b << (uint32_t)fOutputIndex; + b << (int32_t)fSequence; + b << (uint8_t)fReturnAll; + fResultType.serialize(b); + fOperationType.serialize(b); + b << (uint32_t)fExpressionId; } void ReturnedColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::RETURNEDCOLUMN); - b >> fData; - b >> (uint64_t&)fCardinality; - b >> fAlias; - b >> (uint8_t&)fDistinct; - b >> (uint64_t&)fJoinInfo; - b >> (uint8_t&)fAsc; - b >> (uint8_t&)fNullsFirst; - b >> (uint64_t&)fOrderPos; - b >> (uint64_t&)fColSource; - b >> (int64_t&)fColPosition; - b >> (uint32_t&)fInputIndex; - b >> (uint32_t&)fInputOffset; - b >> (uint32_t&)fOutputIndex; - b >> (int32_t&)fSequence; - b >> (uint8_t&)fReturnAll; - fResultType.unserialize(b); - fOperationType.unserialize(b); - b >> (uint32_t&)fExpressionId; + ObjectReader::checkType(b, ObjectReader::RETURNEDCOLUMN); + b >> fData; + b >> (uint64_t&)fCardinality; + b >> fAlias; + b >> (uint8_t&)fDistinct; + b >> (uint64_t&)fJoinInfo; + b >> (uint8_t&)fAsc; + b >> (uint8_t&)fNullsFirst; + b >> (uint64_t&)fOrderPos; + b >> (uint64_t&)fColSource; + b >> (int64_t&)fColPosition; + b >> (uint32_t&)fInputIndex; + b >> (uint32_t&)fInputOffset; + b >> (uint32_t&)fOutputIndex; + b >> (int32_t&)fSequence; + b >> (uint8_t&)fReturnAll; + fResultType.unserialize(b); + fOperationType.unserialize(b); + b >> (uint32_t&)fExpressionId; } bool ReturnedColumn::operator==(const ReturnedColumn& t) const { - // Not all fields are considered for a positive equality. - if (fData != t.fData) - return false; + // Not all fields are considered for a positive equality. + if (fData != t.fData) + return false; - if (fCardinality != t.fCardinality) - return false; + if (fCardinality != t.fCardinality) + return false; - if (fDistinct != t.fDistinct) - return false; + if (fDistinct != t.fDistinct) + return false; - if (fJoinInfo != t.fJoinInfo) - return false; + if (fJoinInfo != t.fJoinInfo) + return false; - if (fAsc != t.fAsc) - return false; + if (fAsc != t.fAsc) + return false; - if (fNullsFirst != t.fNullsFirst) - return false; + if (fNullsFirst != t.fNullsFirst) + return false; - if (fInputIndex != t.fInputIndex) - return false; + if (fInputIndex != t.fInputIndex) + return false; - if (fOutputIndex != t.fOutputIndex) - return false; + if (fOutputIndex != t.fOutputIndex) + return false; - if (fResultType != t.fResultType) - return false; + if (fResultType != t.fResultType) + return false; - if (fOperationType != t.fOperationType) - return false; + if (fOperationType != t.fOperationType) + return false; - return true; + return true; } bool ReturnedColumn::operator==(const TreeNode* t) const { - const ReturnedColumn* rc; + const ReturnedColumn* rc; - rc = dynamic_cast(t); + rc = dynamic_cast(t); - if (rc == NULL) - return false; + if (rc == NULL) + return false; - return *this == *rc; + return *this == *rc; } bool ReturnedColumn::operator!=(const ReturnedColumn& t) const { - return !(*this == t); + return !(*this == t); } bool ReturnedColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } const string ReturnedColumn::data() const { - return fData; + return fData; } const string ReturnedColumn::toString() const { - ostringstream oss; - oss << ">ReturnedColumn " << fJoinInfo << "<" << endl; - return oss.str(); + ostringstream oss; + oss << ">ReturnedColumn " << fJoinInfo << "<" << endl; + return oss.str(); } // All columns that may have simple column added to the list need to implement @@ -252,8 +253,7 @@ const string ReturnedColumn::toString() const // fSimpleColumnList will be cleared. void ReturnedColumn::setSimpleColumnList() { - fSimpleColumnList.clear(); + fSimpleColumnList.clear(); } - -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/returnedcolumn.h b/dbcon/execplan/returnedcolumn.h index 6c07f5694..c04413176 100644 --- a/dbcon/execplan/returnedcolumn.h +++ b/dbcon/execplan/returnedcolumn.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: returnedcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: returnedcolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef RETURNEDCOLUMN_H @@ -48,9 +48,8 @@ class Row; */ namespace execplan { - // Join info bit mask -//const uint64_t JOIN_OUTER = 0x0001; +// const uint64_t JOIN_OUTER = 0x0001; const uint64_t JOIN_SEMI = 0x0002; const uint64_t JOIN_ANTI = 0x0004; const uint64_t JOIN_SCALAR = 0x0008; @@ -80,307 +79,306 @@ typedef boost::shared_ptr SRCP; */ class ReturnedColumn : public TreeNode { + public: + /** + * Constructors + */ + ReturnedColumn(); + ReturnedColumn(const std::string& sql); + ReturnedColumn(const uint32_t sessionID, const bool returnAll = false); + ReturnedColumn(const ReturnedColumn& rhs, const uint32_t sessionID = 0); -public: + /** + * Destructors + */ + virtual ~ReturnedColumn(); - /** - * Constructors - */ - ReturnedColumn(); - ReturnedColumn(const std::string& sql); - ReturnedColumn(const uint32_t sessionID, const bool returnAll = false); - ReturnedColumn(const ReturnedColumn& rhs, const uint32_t sessionID = 0); + /** + * Accessor Methods + */ + virtual const std::string data() const; + virtual void data(const std::string data) + { + fData = data; + } - /** - * Destructors - */ - virtual ~ReturnedColumn(); + virtual bool returnAll() const + { + return fReturnAll; + } + virtual void returnAll(const bool returnAll) + { + fReturnAll = returnAll; + } - /** - * Accessor Methods - */ - virtual const std::string data() const; - virtual void data(const std::string data) - { - fData = data; - } + uint32_t sessionID() const + { + return fSessionID; + } + void sessionID(const uint32_t sessionID) + { + fSessionID = sessionID; + } - virtual bool returnAll() const - { - return fReturnAll; - } - virtual void returnAll(const bool returnAll) - { - fReturnAll = returnAll; - } + inline int32_t sequence() const + { + return fSequence; + } + inline void sequence(const int32_t sequence) + { + fSequence = sequence; + } - uint32_t sessionID() const - { - return fSessionID; - } - void sessionID(const uint32_t sessionID) - { - fSessionID = sessionID; - } + inline const std::string& alias() const + { + return fAlias; + } + inline void alias(const std::string& alias) + { + fAlias = alias; + } - inline int32_t sequence() const - { - return fSequence; - } - inline void sequence(const int32_t sequence) - { - fSequence = sequence; - } + virtual uint64_t cardinality() const + { + return fCardinality; + } + virtual void cardinality(const uint64_t cardinality) + { + fCardinality = cardinality; + } - inline const std::string& alias() const - { - return fAlias; - } - inline void alias(const std::string& alias) - { - fAlias = alias; - } + virtual bool distinct() const + { + return fDistinct; + } + virtual void distinct(const bool distinct) + { + fDistinct = distinct; + } - virtual uint64_t cardinality() const - { - return fCardinality; - } - virtual void cardinality( const uint64_t cardinality) - { - fCardinality = cardinality; - } + uint32_t expressionId() const + { + return fExpressionId; + } + void expressionId(const uint32_t expressionId) + { + fExpressionId = expressionId; + } - virtual bool distinct() const - { - return fDistinct; - } - virtual void distinct(const bool distinct) - { - fDistinct = distinct; - } + virtual uint64_t joinInfo() const + { + return fJoinInfo; + } + virtual void joinInfo(const uint64_t joinInfo) + { + fJoinInfo = joinInfo; + } - uint32_t expressionId() const - { - return fExpressionId; - } - void expressionId(const uint32_t expressionId) - { - fExpressionId = expressionId; - } + virtual bool asc() const + { + return fAsc; + } + virtual void asc(const bool asc) + { + fAsc = asc; + } - virtual uint64_t joinInfo() const - { - return fJoinInfo; - } - virtual void joinInfo(const uint64_t joinInfo) - { - fJoinInfo = joinInfo; - } + virtual bool nullsFirst() const + { + return fNullsFirst; + } + virtual void nullsFirst(const bool nullsFirst) + { + fNullsFirst = nullsFirst; + } - virtual bool asc() const - { - return fAsc; - } - virtual void asc(const bool asc) - { - fAsc = asc; - } + virtual uint64_t orderPos() const + { + return fOrderPos; + } + virtual void orderPos(const uint64_t orderPos) + { + fOrderPos = orderPos; + } - virtual bool nullsFirst() const - { - return fNullsFirst; - } - virtual void nullsFirst(const bool nullsFirst) - { - fNullsFirst = nullsFirst; - } + virtual uint64_t colSource() const + { + return fColSource; + } + virtual void colSource(const uint64_t colSource) + { + fColSource = colSource; + } - virtual uint64_t orderPos() const - { - return fOrderPos; - } - virtual void orderPos(const uint64_t orderPos) - { - fOrderPos = orderPos; - } + virtual int64_t colPosition() const + { + return fColPosition; + } + virtual void colPosition(const int64_t colPosition) + { + fColPosition = colPosition; + } - virtual uint64_t colSource() const - { - return fColSource; - } - virtual void colSource(const uint64_t colSource) - { - fColSource = colSource; - } + // Columns that may have aggregate column involved should implement this interface. + virtual bool hasAggregate() + { + return fHasAggregate; + } + virtual void hasAggregate(bool hasAgg) + { + fHasAggregate = hasAgg; + } - virtual int64_t colPosition() const - { - return fColPosition; - } - virtual void colPosition(const int64_t colPosition) - { - fColPosition = colPosition; - } + /** + * Operations + */ + virtual ReturnedColumn* clone() const = 0; - // Columns that may have aggregate column involved should implement this interface. - virtual bool hasAggregate() - { - return fHasAggregate; - } - virtual void hasAggregate(bool hasAgg) - { - fHasAggregate = hasAgg; - } + /** + * The serialize interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * Operations - */ - virtual ReturnedColumn* clone() const = 0; + virtual const std::string toString() const; - /** - * The serialize interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - virtual const std::string toString() const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + * @warning this member function is NOT virtual (why?) + */ + bool operator==(const ReturnedColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - * @warning this member function is NOT virtual (why?) - */ - bool operator==(const ReturnedColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + * @warning this member function is NOT virtual (why?) + */ + bool operator!=(const ReturnedColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief check if this column is the same as the argument + * + * @note Why can't operator==() be used? + */ + virtual bool sameColumn(const ReturnedColumn* rc) const + { + return (fData.compare(rc->data()) == 0); + } - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - * @warning this member function is NOT virtual (why?) - */ - bool operator!=(const ReturnedColumn& t) const; + virtual const std::vector& simpleColumnList() const + { + return fSimpleColumnList; + } - /** @brief check if this column is the same as the argument - * - * @note Why can't operator==() be used? - */ - virtual bool sameColumn(const ReturnedColumn* rc) const - { - return (fData.compare(rc->data()) == 0); - } + /* @brief traverse this ReturnedColumn and re-populate fSimpleColumnList. + * + * @note all ReturnedColumns that may have simple column arguments added + * to the list need to implement thhis function. + */ + virtual void setSimpleColumnList(); - virtual const std::vector& simpleColumnList() const - { - return fSimpleColumnList; - } + // get all aggregate column list in this expression + const std::vector& aggColumnList() const + { + return fAggColumnList; + } - /* @brief traverse this ReturnedColumn and re-populate fSimpleColumnList. - * - * @note all ReturnedColumns that may have simple column arguments added - * to the list need to implement thhis function. - */ - virtual void setSimpleColumnList(); + // get all window function column list in this expression + const std::vector& windowfunctionColumnList() const + { + return fWindowFunctionColumnList; + } - // get all aggregate column list in this expression - const std::vector& aggColumnList() const - { - return fAggColumnList; - } + /* @brief if this column is or contains window function column + * + * @note after this function call fWindowFunctionColumnList is populated + */ + virtual bool hasWindowFunc() = 0; - // get all window function column list in this expression - const std::vector& windowfunctionColumnList() const - { - return fWindowFunctionColumnList; - } + virtual void replaceRealCol(std::vector&) + { + } - /* @brief if this column is or contains window function column - * - * @note after this function call fWindowFunctionColumnList is populated - */ - virtual bool hasWindowFunc() = 0; + /** + * Return the tableAlias name of the table that the column arguments belong to. + * + * @param TableAliasName that will be set in the function + * @return true, if all arguments belong to one table + * false, if multiple tables are involved in the function + */ + virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan) + { + return false; + } - virtual void replaceRealCol(std::vector&) {} + protected: + // return all flag set if the other column is outer join column (+) + bool fReturnAll; + uint32_t fSessionID; + int32_t fSequence; /// column sequence on the SELECT mapped to the correlated joins + uint64_t fCardinality; + std::string fAlias; /// column alias + bool fDistinct; + uint64_t fJoinInfo; + bool fAsc; /// for order by column + bool fNullsFirst; /// for window function + uint64_t fOrderPos; /// for order by and group by column + uint64_t fColSource; /// from which subquery + int64_t fColPosition; /// the column position in the source subquery + std::vector fSimpleColumnList; + std::vector fAggColumnList; + std::vector fWindowFunctionColumnList; + bool fHasAggregate; /// connector internal use. no need to serialize - /** - * Return the tableAlias name of the table that the column arguments belong to. - * - * @param TableAliasName that will be set in the function - * @return true, if all arguments belong to one table - * false, if multiple tables are involved in the function - */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan) - { - return false; - } + private: + std::string fData; -protected: - // return all flag set if the other column is outer join column (+) - bool fReturnAll; - uint32_t fSessionID; - int32_t fSequence; /// column sequence on the SELECT mapped to the correlated joins - uint64_t fCardinality; - std::string fAlias; /// column alias - bool fDistinct; - uint64_t fJoinInfo; - bool fAsc; /// for order by column - bool fNullsFirst; /// for window function - uint64_t fOrderPos; /// for order by and group by column - uint64_t fColSource; /// from which subquery - int64_t fColPosition; /// the column position in the source subquery - std::vector fSimpleColumnList; - std::vector fAggColumnList; - std::vector fWindowFunctionColumnList; - bool fHasAggregate; /// connector internal use. no need to serialize + /****************************************************************** + * F&E framework * + ******************************************************************/ + public: + uint32_t inputIndex() const + { + return fInputIndex; + } + void inputIndex(const uint32_t inputIndex) + { + fInputIndex = inputIndex; + } + uint32_t outputIndex() const + { + return fOutputIndex; + } + void outputIndex(const uint32_t outputIndex) + { + fOutputIndex = outputIndex; + } -private: - std::string fData; - - /****************************************************************** - * F&E framework * - ******************************************************************/ -public: - uint32_t inputIndex() const - { - return fInputIndex; - } - void inputIndex ( const uint32_t inputIndex ) - { - fInputIndex = inputIndex; - } - uint32_t outputIndex() const - { - return fOutputIndex; - } - void outputIndex ( const uint32_t outputIndex ) - { - fOutputIndex = outputIndex; - } - -protected: - std::string fErrMsg; /// error occured in evaluation - uint32_t fInputIndex; /// index to the input rowgroup - uint32_t fInputOffset; /// index to the input rowgroup - uint32_t fOutputIndex; /// index to the output rowgroup - uint32_t fExpressionId; /// unique id for this expression + protected: + std::string fErrMsg; /// error occured in evaluation + uint32_t fInputIndex; /// index to the input rowgroup + uint32_t fInputOffset; /// index to the input rowgroup + uint32_t fOutputIndex; /// index to the output rowgroup + uint32_t fExpressionId; /// unique id for this expression }; std::ostream& operator<<(std::ostream& os, const ReturnedColumn& rhs); -} -#endif //RETURNEDCOLUMN_H - +} // namespace execplan +#endif // RETURNEDCOLUMN_H diff --git a/dbcon/execplan/rowcolumn.cpp b/dbcon/execplan/rowcolumn.cpp index 6e51280c1..ad8e1d0b9 100644 --- a/dbcon/execplan/rowcolumn.cpp +++ b/dbcon/execplan/rowcolumn.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: rowcolumn.cpp 6309 2010-03-04 19:33:12Z zzhu $ -* -* -***********************************************************************/ + * $Id: rowcolumn.cpp 6309 2010-03-04 19:33:12Z zzhu $ + * + * + ***********************************************************************/ #include #include @@ -52,30 +52,29 @@ using namespace joblist; namespace execplan { - /** * Constructors/Destructors */ -RowColumn::RowColumn(const uint32_t sessionID): - ReturnedColumn(sessionID) -{} - -RowColumn::RowColumn (const RowColumn& rhs, const uint32_t sessionID): - ReturnedColumn(rhs, sessionID) +RowColumn::RowColumn(const uint32_t sessionID) : ReturnedColumn(sessionID) { - fColumnVec.clear(); - //fColumnVec = rhs.fColumnVec; - SRCP srcp; +} - for (uint32_t i = 0; i < rhs.fColumnVec.size(); i++) - { - srcp.reset(rhs.fColumnVec[i]->clone()); - fColumnVec.push_back(srcp); - } +RowColumn::RowColumn(const RowColumn& rhs, const uint32_t sessionID) : ReturnedColumn(rhs, sessionID) +{ + fColumnVec.clear(); + // fColumnVec = rhs.fColumnVec; + SRCP srcp; + + for (uint32_t i = 0; i < rhs.fColumnVec.size(); i++) + { + srcp.reset(rhs.fColumnVec[i]->clone()); + fColumnVec.push_back(srcp); + } } RowColumn::~RowColumn() -{} +{ +} /** * Methods @@ -83,112 +82,112 @@ RowColumn::~RowColumn() RowColumn& RowColumn::operator=(const RowColumn& rhs) { - if (this != &rhs) - { - fColumnVec.clear(); - fColumnVec = rhs.fColumnVec; - } + if (this != &rhs) + { + fColumnVec.clear(); + fColumnVec = rhs.fColumnVec; + } - return *this; + return *this; } ostream& operator<<(ostream& output, const RowColumn& rhs) { - output << rhs.toString(); + output << rhs.toString(); - return output; + return output; } const string RowColumn::toString() const { - ostringstream output; - output << "RowColumn" << endl; + ostringstream output; + output << "RowColumn" << endl; - for (uint32_t i = 0; i < fColumnVec.size(); i++) - output << fColumnVec[i]->toString(); + for (uint32_t i = 0; i < fColumnVec.size(); i++) + output << fColumnVec[i]->toString(); - return output.str(); + return output.str(); } void RowColumn::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::ROWCOLUMN; - ReturnedColumn::serialize(b); - b << (uint32_t)fColumnVec.size(); + b << (ObjectReader::id_t)ObjectReader::ROWCOLUMN; + ReturnedColumn::serialize(b); + b << (uint32_t)fColumnVec.size(); - for (uint32_t i = 0; i < fColumnVec.size(); i++) - fColumnVec[i]->serialize(b); + for (uint32_t i = 0; i < fColumnVec.size(); i++) + fColumnVec[i]->serialize(b); } void RowColumn::unserialize(messageqcpp::ByteStream& b) { - fColumnVec.clear(); - ObjectReader::checkType(b, ObjectReader::ROWCOLUMN); - ReturnedColumn::unserialize(b); - uint32_t size; - SRCP srcp; - b >> (uint32_t&)size; + fColumnVec.clear(); + ObjectReader::checkType(b, ObjectReader::ROWCOLUMN); + ReturnedColumn::unserialize(b); + uint32_t size; + SRCP srcp; + b >> (uint32_t&)size; - for (uint32_t i = 0; i < size; i++) - { - srcp.reset(dynamic_cast((ObjectReader::createTreeNode(b)))); - fColumnVec.push_back(srcp); - } + for (uint32_t i = 0; i < size; i++) + { + srcp.reset(dynamic_cast((ObjectReader::createTreeNode(b)))); + fColumnVec.push_back(srcp); + } } bool RowColumn::operator==(const RowColumn& t) const { - if (fColumnVec.size() != t.columnVec().size()) + if (fColumnVec.size() != t.columnVec().size()) + return false; + + for (uint32_t i = 0; i < fColumnVec.size(); i++) + { + if (fColumnVec[i].get() != NULL) + { + if (t.columnVec()[i].get() == NULL) return false; - for (uint32_t i = 0; i < fColumnVec.size(); i++) - { - if (fColumnVec[i].get() != NULL) - { - if (t.columnVec()[i].get() == NULL) - return false; - - if (*(fColumnVec[i].get()) != t.columnVec()[i].get()) - return false; - } - else if (t.columnVec()[i].get() != NULL) - return false; + if (*(fColumnVec[i].get()) != t.columnVec()[i].get()) + return false; } + else if (t.columnVec()[i].get() != NULL) + return false; + } - return true; + return true; } bool RowColumn::operator==(const TreeNode* t) const { - const RowColumn* rc; + const RowColumn* rc; - rc = dynamic_cast(t); + rc = dynamic_cast(t); - if (rc == NULL) - return false; + if (rc == NULL) + return false; - return *this == *rc; + return *this == *rc; } bool RowColumn::operator!=(const RowColumn& t) const { - return !(*this == t); + return !(*this == t); } bool RowColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } ostream& operator<<(ostream& output, const SubSelect& ss) { - output << ss.toString() << endl; - return output; + output << ss.toString() << endl; + return output; } const string SubSelect::toString() const { - return string(">SubSelect<"); + return string(">SubSelect<"); } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/rowcolumn.h b/dbcon/execplan/rowcolumn.h index 7a227f974..560d04c9e 100644 --- a/dbcon/execplan/rowcolumn.h +++ b/dbcon/execplan/rowcolumn.h @@ -15,10 +15,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ /*********************************************************************** -* $Id: rowcolumn.h 6309 2010-03-04 19:33:12Z zzhu $ -* -* -***********************************************************************/ + * $Id: rowcolumn.h 6309 2010-03-04 19:33:12Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef ROWCOLUMN_H @@ -42,7 +42,6 @@ class ByteStream; */ namespace execplan { - class ParseTree; /** * @brief A class to represent a simple returned column @@ -54,133 +53,131 @@ class ParseTree; */ class RowColumn : public ReturnedColumn { + public: + /** + * Constructors + */ + RowColumn(const uint32_t sessionID = 0); + RowColumn(const RowColumn& rhs, const uint32_t sessionID = 0); -public: + /** + * Destructor + */ + virtual ~RowColumn(); - /** - * Constructors - */ - RowColumn(const uint32_t sessionID = 0); - RowColumn(const RowColumn& rhs, const uint32_t sessionID = 0); + /** + * Accessor Methods + */ + const std::vector& columnVec() const + { + return fColumnVec; + } + void columnVec(const std::vector& columnVec) + { + fColumnVec = columnVec; + } - /** - * Destructor - */ - virtual ~RowColumn(); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual RowColumn* clone() const + { + return new RowColumn(*this); + } + /** + * Overloaded assignment operator + */ + RowColumn& operator=(const RowColumn& rhs); - /** - * Accessor Methods - */ - const std::vector& columnVec() const - { - return fColumnVec; - } - void columnVec( const std::vector& columnVec ) - { - fColumnVec = columnVec; - } + /** + * The serialize interface + */ + // virtual void serialize(messageqcpp::ByteStream&) const; + // virtual void unserialize(messageqcpp::ByteStream&); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual RowColumn* clone() const - { - return new RowColumn (*this); - } - /** - * Overloaded assignment operator - */ - RowColumn& operator=(const RowColumn& rhs); + virtual const std::string toString() const; - /** - * The serialize interface - */ - //virtual void serialize(messageqcpp::ByteStream&) const; - //virtual void unserialize(messageqcpp::ByteStream&); + /** + * Serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - virtual const std::string toString() const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** - * Serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const RowColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const RowColumn& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const RowColumn& t) const; - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() - { - return false; - } - virtual bool hasWindowFunc() - { - return false; - } - -private: - /** - * Fields - */ - std::vector fColumnVec; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const RowColumn& t) const; + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate() + { + return false; + } + virtual bool hasWindowFunc() + { + return false; + } + private: + /** + * Fields + */ + std::vector fColumnVec; }; /** dummy class. For the connector to use in gp_walk*/ class SubSelect : public ReturnedColumn { -public: - SubSelect(): ReturnedColumn() {} - ~SubSelect() {} - SubSelect* clone() const - { - return new SubSelect(); - } - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() - { - return false; - } - virtual bool hasWindowFunc() - { - return false; - } - virtual const std::string toString() const; + public: + SubSelect() : ReturnedColumn() + { + } + ~SubSelect() + { + } + SubSelect* clone() const + { + return new SubSelect(); + } + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate() + { + return false; + } + virtual bool hasWindowFunc() + { + return false; + } + virtual const std::string toString() const; }; - /** * ostream operator */ std::ostream& operator<<(std::ostream& output, const RowColumn& rhs); -} -#endif //SIMPLECOLUMN_H - +} // namespace execplan +#endif // SIMPLECOLUMN_H diff --git a/dbcon/execplan/selectfilter.cpp b/dbcon/execplan/selectfilter.cpp index bca5fddda..8eccaa6b1 100644 --- a/dbcon/execplan/selectfilter.cpp +++ b/dbcon/execplan/selectfilter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: selectfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: selectfilter.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -35,170 +35,166 @@ namespace execplan * Constructors/Destructors */ SelectFilter::SelectFilter() -{} +{ +} -SelectFilter::SelectFilter(const vector& cols, - const SOP& op, - SCSEP& sub, - bool correlated) : - fCols(cols), - fOp(op), - fSub(sub), - fCorrelated(correlated), - fData("subselect"), - fReturnedColPos(0) -{} - -SelectFilter::SelectFilter(const SelectFilter& rhs): - Filter(rhs), - fCols(rhs.fCols), - fOp(rhs.fOp), - fSub(rhs.fSub), - fCorrelated(rhs.fCorrelated), - fData(rhs.fData), - fReturnedColPos(0) -{} +SelectFilter::SelectFilter(const vector& cols, const SOP& op, SCSEP& sub, bool correlated) + : fCols(cols), fOp(op), fSub(sub), fCorrelated(correlated), fData("subselect"), fReturnedColPos(0) +{ +} +SelectFilter::SelectFilter(const SelectFilter& rhs) + : Filter(rhs) + , fCols(rhs.fCols) + , fOp(rhs.fOp) + , fSub(rhs.fSub) + , fCorrelated(rhs.fCorrelated) + , fData(rhs.fData) + , fReturnedColPos(0) +{ +} SelectFilter::~SelectFilter() -{} +{ +} const string SelectFilter::toString() const { - ostringstream oss; - oss << "SelectFilter " << "returnedColPos=" << fReturnedColPos << endl; + ostringstream oss; + oss << "SelectFilter " + << "returnedColPos=" << fReturnedColPos << endl; - for (uint32_t i = 0; i < fCols.size(); i++) - oss << fCols[i]->toString(); + for (uint32_t i = 0; i < fCols.size(); i++) + oss << fCols[i]->toString(); - oss << fOp->toString() << endl; - oss << *(fSub.get()); - return oss.str(); + oss << fOp->toString() << endl; + oss << *(fSub.get()); + return oss.str(); } ostream& operator<<(ostream& output, const SelectFilter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void SelectFilter::serialize(messageqcpp::ByteStream& b) const { - b << static_cast(ObjectReader::SELECTFILTER); - Filter::serialize(b); - b << static_cast(fCols.size()); + b << static_cast(ObjectReader::SELECTFILTER); + Filter::serialize(b); + b << static_cast(fCols.size()); - for (uint32_t i = 0; i < fCols.size(); i++) - { - if (fCols[i] != NULL) - fCols[i]->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); - } - - if (fOp != NULL) - fOp->serialize(b); + for (uint32_t i = 0; i < fCols.size(); i++) + { + if (fCols[i] != NULL) + fCols[i]->serialize(b); else - b << static_cast(ObjectReader::NULL_CLASS); + b << static_cast(ObjectReader::NULL_CLASS); + } - if (fSub.get() != NULL) - fSub->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fOp != NULL) + fOp->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - b << static_cast(fReturnedColPos); + if (fSub.get() != NULL) + fSub->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); + + b << static_cast(fReturnedColPos); } void SelectFilter::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::SELECTFILTER); + ObjectReader::checkType(b, ObjectReader::SELECTFILTER); - Filter::unserialize(b); - fCols.clear(); - uint32_t size; - SRCP srcp; - b >> size; + Filter::unserialize(b); + fCols.clear(); + uint32_t size; + SRCP srcp; + b >> size; - for (uint32_t i = 0; i < size; i++) - { - srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fCols.push_back(srcp); - } + for (uint32_t i = 0; i < size; i++) + { + srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fCols.push_back(srcp); + } - fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fSub.reset(dynamic_cast(ObjectReader::createExecutionPlan(b))); - b >> static_cast(fReturnedColPos); + fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fSub.reset(dynamic_cast(ObjectReader::createExecutionPlan(b))); + b >> static_cast(fReturnedColPos); } bool SelectFilter::operator==(const SelectFilter& t) const { - const Filter* f1, *f2; + const Filter *f1, *f2; - f1 = static_cast(this); - f2 = static_cast(&t); + f1 = static_cast(this); + f2 = static_cast(&t); - if (*f1 != *f2) - return false; + if (*f1 != *f2) + return false; - if (fCols.size() != t.fCols.size()) - return false; + if (fCols.size() != t.fCols.size()) + return false; - for (uint32_t i = 0; i < fCols.size(); i++) + for (uint32_t i = 0; i < fCols.size(); i++) + { + if (fCols[i] != NULL) { - if (fCols[i] != NULL) - { - if (*(fCols[i]) != *(t.fCols[i])) - return false; - } - else if (t.fCols[i].get() != NULL) - return false; + if (*(fCols[i]) != *(t.fCols[i])) + return false; } + else if (t.fCols[i].get() != NULL) + return false; + } - if (fOp != NULL) - { - if (*fOp != *t.fOp) - return false; - } - else if (t.fOp != NULL) - return false; + if (fOp != NULL) + { + if (*fOp != *t.fOp) + return false; + } + else if (t.fOp != NULL) + return false; - if (fSub != NULL) - { - if (*fSub != t.fSub.get()) - return false; - } - else if (t.fSub != NULL) - return false; + if (fSub != NULL) + { + if (*fSub != t.fSub.get()) + return false; + } + else if (t.fSub != NULL) + return false; - if (fData != t.fData) - return false; + if (fData != t.fData) + return false; - if (fReturnedColPos != t.fReturnedColPos) - return false; + if (fReturnedColPos != t.fReturnedColPos) + return false; - return true; + return true; } bool SelectFilter::operator==(const TreeNode* t) const { - const SelectFilter* o; + const SelectFilter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool SelectFilter::operator!=(const SelectFilter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool SelectFilter::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } -} +} // namespace execplan diff --git a/dbcon/execplan/selectfilter.h b/dbcon/execplan/selectfilter.h index b6deca824..379cd3133 100644 --- a/dbcon/execplan/selectfilter.h +++ b/dbcon/execplan/selectfilter.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: selectfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: selectfilter.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef SELECTFILTER_H @@ -44,148 +44,146 @@ namespace execplan */ class SelectFilter : public Filter { - /** - * Public stuff - */ -public: + /** + * Public stuff + */ + public: + /** + * Constructors / copy constructor + */ + SelectFilter(); + SelectFilter(const SelectFilter& rhs); - /** - * Constructors / copy constructor - */ - SelectFilter(); - SelectFilter(const SelectFilter& rhs); + /** Constructors + * + * pass all parts in ctor + * @note SimpleFilter takes ownership of all these pointers + */ + SelectFilter(const std::vector& cols, const SOP& op, SCSEP& sub, bool correlated = false); - /** Constructors - * - * pass all parts in ctor - * @note SimpleFilter takes ownership of all these pointers - */ - SelectFilter(const std::vector& cols, const SOP& op, SCSEP& sub, bool correlated = false); + /** + * Destructors + */ + virtual ~SelectFilter(); - /** - * Destructors - */ - virtual ~SelectFilter(); + /** + * Accessor Methods + */ + inline const std::vector& cols() const + { + return fCols; + } - /** - * Accessor Methods - */ - inline const std::vector& cols() const - { - return fCols; - } + void cols(const std::vector& cols) + { + fCols = cols; + } - void cols(const std::vector& cols) - { - fCols = cols; - } + inline const SOP& op() const + { + return fOp; + } - inline const SOP& op() const - { - return fOp; - } + inline void op(const SOP& op) + { + fOp = op; + } - inline void op (const SOP& op) - { - fOp = op; - } + inline const SCSEP& sub() const + { + return fSub; + } - inline const SCSEP& sub() const - { - return fSub; - } + inline void sub(SCSEP& sub) + { + fSub = sub; + } - inline void sub(SCSEP& sub) - { - fSub = sub; - } + bool correlated() const + { + return fCorrelated; + } + void correlated(const bool correlated) + { + fCorrelated = correlated; + } - bool correlated() const - { - return fCorrelated; - } - void correlated(const bool correlated) - { - fCorrelated = correlated; - } + virtual const std::string toString() const; - virtual const std::string toString() const; + virtual inline const std::string data() const + { + return fData; + } + virtual inline void data(const std::string data) + { + fData = data; + } - virtual inline const std::string data() const - { - return fData; - } - virtual inline void data( const std::string data ) - { - fData = data; - } + uint64_t returnedColPos() const + { + return fReturnedColPos; + } + void returnedColPos(const uint64_t returnedColPos) + { + fReturnedColPos = returnedColPos; + } - uint64_t returnedColPos() const - { - return fReturnedColPos; - } - void returnedColPos(const uint64_t returnedColPos) - { - fReturnedColPos = returnedColPos; - } + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual SelectFilter* clone() const + { + return new SelectFilter(*this); + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual SelectFilter* clone() const - { - return new SelectFilter (*this); - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const SelectFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const SelectFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const SelectFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const SelectFilter& t) const; + private: + // default okay? + // SelectFilter& operator=(const SelectFilter& rhs); -private: - //default okay? - //SelectFilter& operator=(const SelectFilter& rhs); - - std::vector fCols; - SOP fOp; - SCSEP fSub; - bool fCorrelated; - std::string fData; - uint64_t fReturnedColPos; // offset in fSub->returnedColList to indicate the start of projection + std::vector fCols; + SOP fOp; + SCSEP fSub; + bool fCorrelated; + std::string fData; + uint64_t fReturnedColPos; // offset in fSub->returnedColList to indicate the start of projection }; std::ostream& operator<<(std::ostream& output, const SelectFilter& rhs); -} -#endif //SELECTFILTER_H - +} // namespace execplan +#endif // SELECTFILTER_H diff --git a/dbcon/execplan/sessionmanager.cpp b/dbcon/execplan/sessionmanager.cpp index 80041d576..526593252 100644 --- a/dbcon/execplan/sessionmanager.cpp +++ b/dbcon/execplan/sessionmanager.cpp @@ -41,29 +41,26 @@ using namespace BRM; namespace execplan { - - - SessionManager::SessionManager() { - config::Config* conf; + config::Config* conf; - conf = config::Config::makeConfig(); - txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); + conf = config::Config::makeConfig(); + txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); } SessionManager::SessionManager(bool nolock) { - config::Config* conf; - string stmp; + config::Config* conf; + string stmp; - conf = config::Config::makeConfig(); - txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); + conf = config::Config::makeConfig(); + txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); } SessionManager::SessionManager(const SessionManager& sm) { - txnidFilename = sm.txnidFilename; + txnidFilename = sm.txnidFilename; } SessionManager::~SessionManager() @@ -72,136 +69,136 @@ SessionManager::~SessionManager() const QueryContext SessionManager::verID() { - return dbrm.verID(); + return dbrm.verID(); } const QueryContext SessionManager::sysCatVerID() { - return dbrm.sysCatVerID(); + return dbrm.sysCatVerID(); } const TxnID SessionManager::newTxnID(const SID session, bool block, bool isDDL) { - TxnID ret; + TxnID ret; - ret = dbrm.newTxnID(session, block, isDDL); - return ret; + ret = dbrm.newTxnID(session, block, isDDL); + return ret; } void SessionManager::committed(TxnID& txn) { - TxnID tmp; + TxnID tmp; - tmp.id = txn.id; - tmp.valid = txn.valid; - dbrm.committed(tmp); - txn.id = tmp.id; - txn.valid = tmp.valid; + tmp.id = txn.id; + tmp.valid = txn.valid; + dbrm.committed(tmp); + txn.id = tmp.id; + txn.valid = tmp.valid; } void SessionManager::rolledback(TxnID& txn) { - TxnID tmp; + TxnID tmp; - tmp.id = txn.id; - tmp.valid = txn.valid; - dbrm.rolledback(tmp); - txn.id = tmp.id; - txn.valid = tmp.valid; + tmp.id = txn.id; + tmp.valid = txn.valid; + dbrm.rolledback(tmp); + txn.id = tmp.id; + txn.valid = tmp.valid; } const TxnID SessionManager::getTxnID(const SID session) { - TxnID tmp; - TxnID ret; + TxnID tmp; + TxnID ret; - tmp = dbrm.getTxnID(session); - ret.id = tmp.id; - ret.valid = tmp.valid; + tmp = dbrm.getTxnID(session); + ret.id = tmp.id; + ret.valid = tmp.valid; - return ret; + return ret; } boost::shared_array SessionManager::SIDTIDMap(int& len) { - // is this cast valid? - return dbrm.SIDTIDMap(len); + // is this cast valid? + return dbrm.SIDTIDMap(len); } string SessionManager::getTxnIDFilename() const { - return txnidFilename; + return txnidFilename; } int SessionManager::verifySize() { - return 1; + return 1; } void SessionManager::reset() { - dbrm.sessionmanager_reset(); + dbrm.sessionmanager_reset(); } uint32_t SessionManager::getUnique32() { - return dbrm.getUnique32(); + return dbrm.getUnique32(); } -bool SessionManager::checkActiveTransaction( const SID sessionId, bool& bIsDbrmUp, SIDTIDEntry& blocker ) +bool SessionManager::checkActiveTransaction(const SID sessionId, bool& bIsDbrmUp, SIDTIDEntry& blocker) { - bIsDbrmUp = true; - int arrayLenth = 0; - bool ret = false; - boost::shared_array sIDTIDMap; + bIsDbrmUp = true; + int arrayLenth = 0; + bool ret = false; + boost::shared_array sIDTIDMap; - sIDTIDMap = SIDTIDMap( arrayLenth ); + sIDTIDMap = SIDTIDMap(arrayLenth); - if (sIDTIDMap) + if (sIDTIDMap) + { + for (int i = 0; i < arrayLenth; i++) { - for ( int i = 0; i < arrayLenth; i++ ) - { - if ( sIDTIDMap[i].txnid.valid && ( sIDTIDMap[i].sessionid != sessionId || sessionId == 0 ) ) - { - blocker = sIDTIDMap[i]; - ret = true; - } - } - } - else - { - bIsDbrmUp = false; + if (sIDTIDMap[i].txnid.valid && (sIDTIDMap[i].sessionid != sessionId || sessionId == 0)) + { + blocker = sIDTIDMap[i]; + ret = true; + } } + } + else + { + bIsDbrmUp = false; + } - return ret; + return ret; } bool SessionManager::isTransactionActive(const SID sessionId, bool& bIsDbrmUp) { - bIsDbrmUp = true; - int arrayLenth = 0; - bool ret = false; - boost::shared_array sIDTIDMap; + bIsDbrmUp = true; + int arrayLenth = 0; + bool ret = false; + boost::shared_array sIDTIDMap; - sIDTIDMap = SIDTIDMap(arrayLenth); + sIDTIDMap = SIDTIDMap(arrayLenth); - if (sIDTIDMap) + if (sIDTIDMap) + { + for (int i = 0; i < arrayLenth; i++) { - for ( int i = 0; i < arrayLenth; i++ ) - { - if (sIDTIDMap[i].txnid.valid && (sIDTIDMap[i].sessionid == sessionId)) - { - ret = true; - break; - } - } - } - else - { - bIsDbrmUp = false; + if (sIDTIDMap[i].txnid.valid && (sIDTIDMap[i].sessionid == sessionId)) + { + ret = true; + break; + } } + } + else + { + bIsDbrmUp = false; + } - return ret; + return ret; } -} //namespace +} // namespace execplan diff --git a/dbcon/execplan/sessionmanager.h b/dbcon/execplan/sessionmanager.h index e7dab7874..71076c263 100644 --- a/dbcon/execplan/sessionmanager.h +++ b/dbcon/execplan/sessionmanager.h @@ -33,11 +33,10 @@ namespace execplan { - /** @brief Issues transaction IDs and keeps track of the current system-wide version ID. * - * This class's primary purpose is to keep track of the current system-wide version ID and issue transaction IDs. - * It can be used simultaneously by multiple threads and processes. + * This class's primary purpose is to keep track of the current system-wide version ID and issue transaction + * IDs. It can be used simultaneously by multiple threads and processes. * * It uses system-wide semaphores and shared memory segments for IPC. * It allocates both if they don't already exist, but it only deallocates @@ -74,148 +73,146 @@ namespace execplan class SessionManager { -public: + public: + /** @brief SID = Session ID */ + typedef uint32_t SID; - /** @brief SID = Session ID */ - typedef uint32_t SID; + /** @brief Constructor + * + * This sets up the shared memory segment & semaphores used by this + * instance. No additional set-up calls are necessary. + * @note throws ios_base::failure on file IO error, runtime_error for + * other types of errors. It might be worthwhile to define additional + * exceptions for all the different types of failures that can happen + * here. + */ + SessionManager(); + SessionManager(const SessionManager&); - /** @brief Constructor - * - * This sets up the shared memory segment & semaphores used by this - * instance. No additional set-up calls are necessary. - * @note throws ios_base::failure on file IO error, runtime_error for - * other types of errors. It might be worthwhile to define additional - * exceptions for all the different types of failures that can happen - * here. - */ - SessionManager(); - SessionManager(const SessionManager&); + /** @brief Constructor for use during debugging only + * + * This constructor is only used to grab the semaphores & reset them + * if they exist. It does not attach the shared memory segment, + * and no operation other than reset() should be performed on a + * SessionManager instantiated with this. + */ + SessionManager(bool nolock); - /** @brief Constructor for use during debugging only - * - * This constructor is only used to grab the semaphores & reset them - * if they exist. It does not attach the shared memory segment, - * and no operation other than reset() should be performed on a - * SessionManager instantiated with this. - */ - SessionManager(bool nolock); + /** @brief Destructor + * + * This detaches the shared memory segment. If DESTROYSHMSEG is defined and this + * is the last reference to it, it will be saved to disk and destroyed. + * It does not destroy the semaphores. Those persist until the system + * is shut down. + */ + virtual ~SessionManager(); - /** @brief Destructor - * - * This detaches the shared memory segment. If DESTROYSHMSEG is defined and this - * is the last reference to it, it will be saved to disk and destroyed. - * It does not destroy the semaphores. Those persist until the system - * is shut down. - */ - virtual ~SessionManager(); + /** @brief Gets the current version ID + * + * Gets the current version ID. + */ + const BRM::QueryContext verID(); - /** @brief Gets the current version ID - * - * Gets the current version ID. - */ - const BRM::QueryContext verID(); + /** @brief Gets the current systemcatalog version ID + * + * Gets the current systemcatalog version ID. + */ + const BRM::QueryContext sysCatVerID(); - /** @brief Gets the current systemcatalog version ID - * - * Gets the current systemcatalog version ID. - */ - const BRM::QueryContext sysCatVerID(); + /** @brief Gets a new Transaction ID + * + * Makes a new Transaction ID, associates it with the given session + * and increments the system-wide version ID + * @note This blocks until (# active transactions) \< MaxTxns unless block == false + * @note Throws runtime_error on semaphore-related error + * @note This will always return a valid TxnID unless block == false, in which case + * it will return a valid TxnID iff it doesn't have to block. + * @param session The session ID to associate with the transaction ID + * @param block If true, this will block until there is an available slot, otherwise + * it will return a TxnID marked invalid, signifying that it could not start a new transaction + * @return The new transaction ID. + */ + const BRM::TxnID newTxnID(const SID session, bool block = true, bool isDDL = false); - /** @brief Gets a new Transaction ID - * - * Makes a new Transaction ID, associates it with the given session - * and increments the system-wide version ID - * @note This blocks until (# active transactions) \< MaxTxns unless block == false - * @note Throws runtime_error on semaphore-related error - * @note This will always return a valid TxnID unless block == false, in which case - * it will return a valid TxnID iff it doesn't have to block. - * @param session The session ID to associate with the transaction ID - * @param block If true, this will block until there is an available slot, otherwise - * it will return a TxnID marked invalid, signifying that it could not start a new transaction - * @return The new transaction ID. - */ - const BRM::TxnID newTxnID(const SID session, bool block = true, bool isDDL = false); + /** @brief Record that a transaction has been committed + * + * Record that a transaction has been committed. + * @note Throws runtime_error on a semaphore-related error, invalid_argument + * when txnid can't be found + * @param txnid The committed transaction ID. This is marked invalid + * on return. + */ + void committed(BRM::TxnID& txnid); - /** @brief Record that a transaction has been committed - * - * Record that a transaction has been committed. - * @note Throws runtime_error on a semaphore-related error, invalid_argument - * when txnid can't be found - * @param txnid The committed transaction ID. This is marked invalid - * on return. - */ - void committed(BRM::TxnID& txnid); + /** @brief Record that a transaction has been rolled back + * + * Record that a transaction has been rolled back. + * @note Throws runtime_error on a semaphore-related error, invalid_argument + * when txnid can't be found + * @param txnid The rolled back transaction ID. This is marked invalid + * on return. + */ + void rolledback(BRM::TxnID& txnid); - /** @brief Record that a transaction has been rolled back - * - * Record that a transaction has been rolled back. - * @note Throws runtime_error on a semaphore-related error, invalid_argument - * when txnid can't be found - * @param txnid The rolled back transaction ID. This is marked invalid - * on return. - */ - void rolledback(BRM::TxnID& txnid); + /** @brief Gets the transaction ID associated with a given session ID + * + * Gets the transaction ID associated with a given session ID. + * @note Throws runtime_error on a semaphore-related error + * @param session The session ID + * @return A valid transaction ID if there's an association; an invalid + * one if there isn't. + */ + const BRM::TxnID getTxnID(const SID session); - /** @brief Gets the transaction ID associated with a given session ID - * - * Gets the transaction ID associated with a given session ID. - * @note Throws runtime_error on a semaphore-related error - * @param session The session ID - * @return A valid transaction ID if there's an association; an invalid - * one if there isn't. - */ - const BRM::TxnID getTxnID(const SID session); + /** @brief Gets an array containing all active SID-TID associations + * + * Gets an array containing the SID-TID associations of all active + * transactions. This is intended to be used by another object + * that can determine which sessions are still live and which + * transactions need to go away. + * @note Throws runtime_error on a semaphore-related error + * @param len (out) the length of the array + * @return A pointer to the array. Note: The caller is responsible for + * deallocating it. Use delete[]. + */ + boost::shared_array SIDTIDMap(int& len); - /** @brief Gets an array containing all active SID-TID associations - * - * Gets an array containing the SID-TID associations of all active - * transactions. This is intended to be used by another object - * that can determine which sessions are still live and which - * transactions need to go away. - * @note Throws runtime_error on a semaphore-related error - * @param len (out) the length of the array - * @return A pointer to the array. Note: The caller is responsible for - * deallocating it. Use delete[]. - */ - boost::shared_array SIDTIDMap(int& len); + /** @brief Returns a unique uint32_t. It eventually wraps around, but who cares. + * + * This returns a "unique" uint32_t for when we need such a thing. The current + * usage is to get a unique identifier for each BPPJL and its associated BPP + * object on the PM. + * @return A "unique" uint32_t. + */ + uint32_t getUnique32(); - /** @brief Returns a unique uint32_t. It eventually wraps around, but who cares. - * - * This returns a "unique" uint32_t for when we need such a thing. The current - * usage is to get a unique identifier for each BPPJL and its associated BPP - * object on the PM. - * @return A "unique" uint32_t. - */ - uint32_t getUnique32(); + /** @brief Returns the number of active transactions. Only useful in testing. + * + * This returns the number of active transactions and verifies it against + * the transaction semaphore's value and the value reported by shared->txnCount. + * @note Throws logic_error if there's a mismatch, runtime_error on a + * semaphore operation error. + * @return The number of active transactions. + */ + int verifySize(); - /** @brief Returns the number of active transactions. Only useful in testing. - * - * This returns the number of active transactions and verifies it against - * the transaction semaphore's value and the value reported by shared->txnCount. - * @note Throws logic_error if there's a mismatch, runtime_error on a - * semaphore operation error. - * @return The number of active transactions. - */ - int verifySize(); + /** @brief Resets the semaphores to their original state. For testing only. + * + * Resets the semaphores to their original state. For testing only. + */ + void reset(); - /** @brief Resets the semaphores to their original state. For testing only. - * - * Resets the semaphores to their original state. For testing only. - */ - void reset(); + std::string getTxnIDFilename() const; - std::string getTxnIDFilename() const; + bool checkActiveTransaction(const SID sessionId, bool& bIsDbrmUp, BRM::SIDTIDEntry& blocker); + bool isTransactionActive(const SID sessionId, bool& bIsDbrmUp); - bool checkActiveTransaction(const SID sessionId, bool& bIsDbrmUp, BRM::SIDTIDEntry& blocker); - bool isTransactionActive(const SID sessionId, bool& bIsDbrmUp); - -private: - BRM::DBRM dbrm; - std::string txnidFilename; + private: + BRM::DBRM dbrm; + std::string txnidFilename; }; -} //namespace +} // namespace execplan #endif // vim:ts=4 sw=4: - diff --git a/dbcon/execplan/sessionmonitor.cpp b/dbcon/execplan/sessionmonitor.cpp index 651c81575..8a1772993 100644 --- a/dbcon/execplan/sessionmonitor.cpp +++ b/dbcon/execplan/sessionmonitor.cpp @@ -56,101 +56,99 @@ using namespace BRM; #include "installdir.h" - namespace execplan { - SessionMonitor::SessionMonitor() { - config::Config* conf; - int madeSems; - string stmp; + config::Config* conf; + int madeSems; + string stmp; - fuid = getuid(); - conf = config::Config::makeConfig(); + fuid = getuid(); + conf = config::Config::makeConfig(); - try - { - stmp = conf->getConfig("SessionManager", "MaxConcurrentTransactions"); - } - catch (exception& e) - { - cout << e.what() << endl; - stmp.empty(); - } + try + { + stmp = conf->getConfig("SessionManager", "MaxConcurrentTransactions"); + } + catch (exception& e) + { + cout << e.what() << endl; + stmp.empty(); + } - int tmp = static_cast(config::Config::fromText(stmp)); + int tmp = static_cast(config::Config::fromText(stmp)); - if (tmp <= 0) - fMaxTxns = 1000; - else - fMaxTxns = tmp; + if (tmp <= 0) + fMaxTxns = 1000; + else + fMaxTxns = tmp; - stmp.clear(); + stmp.clear(); - try - { - stmp = conf->getConfig("SessionMonitor", "SharedMemoryTmpFile"); - } - catch (exception& e) - { - cout << e.what() << endl; - stmp.empty(); - } + try + { + stmp = conf->getConfig("SessionMonitor", "SharedMemoryTmpFile"); + } + catch (exception& e) + { + cout << e.what() << endl; + stmp.empty(); + } - // Instantiate/delete a SessionManager to make sure it's shared memory segments are present. - SessionManager* mgr = new SessionManager(); - delete mgr; + // Instantiate/delete a SessionManager to make sure it's shared memory segments are present. + SessionManager* mgr = new SessionManager(); + delete mgr; - if (stmp != "") - fSegmentFilename = strdup(stmp.c_str()); - else - { - string tmpdir = "/var/lib/columnstore/CalpontSessionMonitorShm"; - - fSegmentFilename = strdup(tmpdir); - } - - try - { - madeSems = getSems(); - fHaveSemaphores = true; - } - catch (...) - { - fHaveSemaphores = false; - } + if (stmp != "") + fSegmentFilename = strdup(stmp.c_str()); + else + { + string tmpdir = "/var/lib/columnstore/CalpontSessionMonitorShm"; - stmp.clear(); + fSegmentFilename = strdup(tmpdir); + } - try - { - stmp = conf->getConfig("SessionMonitor", "TransactionAgeLimit"); - } - catch (exception& e) - { - cerr << e.what() << endl; - stmp.empty(); - } + try + { + madeSems = getSems(); + fHaveSemaphores = true; + } + catch (...) + { + fHaveSemaphores = false; + } - tmp = static_cast(config::Config::fromText(stmp)); + stmp.clear(); - if (tmp <= 0) - fAgeLimit = fDefaultAgeLimit; - else - fAgeLimit = tmp; + try + { + stmp = conf->getConfig("SessionMonitor", "TransactionAgeLimit"); + } + catch (exception& e) + { + cerr << e.what() << endl; + stmp.empty(); + } - fIsAttached = false; - getSharedData(); - unlock(); - fCurrentSegment = NULL; - fPrevSegment.activeTxns = NULL; + tmp = static_cast(config::Config::fromText(stmp)); - if (haveSharedMemory()) - copyCurrentSegment(); + if (tmp <= 0) + fAgeLimit = fDefaultAgeLimit; + else + fAgeLimit = tmp; - if (haveSemaphores()) - copyPreviousSegment(); + fIsAttached = false; + getSharedData(); + unlock(); + fCurrentSegment = NULL; + fPrevSegment.activeTxns = NULL; + + if (haveSharedMemory()) + copyCurrentSegment(); + + if (haveSemaphores()) + copyPreviousSegment(); } SessionMonitor::SessionMonitor(const SessionMonitor& sm) @@ -159,31 +157,31 @@ SessionMonitor::SessionMonitor(const SessionMonitor& sm) SessionMonitor::~SessionMonitor() { - saveAsMonitorData(segmentFilename()); + saveAsMonitorData(segmentFilename()); - if (fSessionManagerData != NULL) - { - lock(); - detachSegment(); - unlock(); - } + if (fSessionManagerData != NULL) + { + lock(); + detachSegment(); + unlock(); + } - //delete [] fCurrentSegment; - //delete [] fSessionMonitorData.activeTxns; - //delete [] fPrevSegment.activeTxns; + // delete [] fCurrentSegment; + // delete [] fSessionMonitorData.activeTxns; + // delete [] fPrevSegment.activeTxns; } void SessionMonitor::detachSegment() { - //delete [] fSessionManagerData; + // delete [] fSessionManagerData; - fIsAttached = false; + fIsAttached = false; } -//returns 1 if it attached to pre-existing semaphores, else it throws exception. +// returns 1 if it attached to pre-existing semaphores, else it throws exception. int SessionMonitor::getSems() { - return 1; + return 1; } void SessionMonitor::lock() @@ -196,326 +194,321 @@ void SessionMonitor::unlock() void SessionMonitor::printTxns(const MonSIDTIDEntry& txn) const { - cout << "sessionid " << txn.sessionid - << " txnid " << txn.txnid.id - << " valid " << (txn.txnid.valid == true ? "TRUE" : "FALSE") - << " time_t " << txn.txnid.firstrecord - << " tdiff " << time(NULL) - txn.txnid.firstrecord - << " ctime " << ctime(&txn.txnid.firstrecord); + cout << "sessionid " << txn.sessionid << " txnid " << txn.txnid.id << " valid " + << (txn.txnid.valid == true ? "TRUE" : "FALSE") << " time_t " << txn.txnid.firstrecord << " tdiff " + << time(NULL) - txn.txnid.firstrecord << " ctime " << ctime(&txn.txnid.firstrecord); } #ifdef SM_DEBUG void SessionMonitor::printTxns(const SessionManager::SIDTIDEntry& txn) const { - cout << "sessionid " << txn.sessionid - << " txnid " << txn.txnid.id - << " valid " << (txn.txnid.valid == true ? "TRUE" : "FALSE") - << endl; + cout << "sessionid " << txn.sessionid << " txnid " << txn.txnid.id << " valid " + << (txn.txnid.valid == true ? "TRUE" : "FALSE") << endl; } void SessionMonitor::printMonitorData(const int len) const { - MonSIDTIDEntry* txns = fPrevSegment.activeTxns; + MonSIDTIDEntry* txns = fPrevSegment.activeTxns; - cout << "Monitor txnCount " << fPrevSegment.txnCount << " verID " << fPrevSegment.verID << endl; + cout << "Monitor txnCount " << fPrevSegment.txnCount << " verID " << fPrevSegment.verID << endl; - for (int idx = 0; txns && idx < len; idx++) - { - printTxns(txns[idx]); - } + for (int idx = 0; txns && idx < len; idx++) + { + printTxns(txns[idx]); + } - cout << "==" << endl; + cout << "==" << endl; } void SessionMonitor::printSegment(const SessionManagerData_t* seg, const int len) const { - if (seg == NULL) - { - cerr << "No SessionManagerData" << endl; - return; - } + if (seg == NULL) + { + cerr << "No SessionManagerData" << endl; + return; + } - cout << "Manager txnCount " << seg->txnCount << " verID " << seg->verID << endl; + cout << "Manager txnCount " << seg->txnCount << " verID " << seg->verID << endl; - for (int idx = 0; idx < len; idx++) - { - printTxns(seg->activeTxns[idx]); - } + for (int idx = 0; idx < len; idx++) + { + printTxns(seg->activeTxns[idx]); + } } #endif void SessionMonitor::getSharedData() { - int len; + int len; - fSessionManagerData = reinterpret_cast(sm.getShmContents(len)); + fSessionManagerData = reinterpret_cast(sm.getShmContents(len)); - if (fSessionManagerData == NULL) - { - cerr << "SessionMonitor::getSharedData(): getShmContents() failed" << endl; - throw runtime_error("SessionMonitor::getSharedData(): getShmContents() failed. Check the error log."); - } + if (fSessionManagerData == NULL) + { + cerr << "SessionMonitor::getSharedData(): getShmContents() failed" << endl; + throw runtime_error("SessionMonitor::getSharedData(): getShmContents() failed. Check the error log."); + } - fIsAttached = true; + fIsAttached = true; } void SessionMonitor::initSegment(SessionMonitorData_t* seg) { - if (!seg) - return; + if (!seg) + return; - seg->txnCount = 0; - seg->verID = 0; - int size = maxTxns() * sizeof(MonSIDTIDEntry_t); + seg->txnCount = 0; + seg->verID = 0; + int size = maxTxns() * sizeof(MonSIDTIDEntry_t); - if (seg->activeTxns) - memset(seg->activeTxns, 0, size); + if (seg->activeTxns) + memset(seg->activeTxns, 0, size); } void SessionMonitor::initSegment(SessionManagerData_t* seg) { - if (!seg) - return; + if (!seg) + return; - int size = maxTxns() * sizeof(SIDTIDEntry_t); - seg->txnCount = 0; - seg->verID = 0; - memset(seg->activeTxns, 0, size); + int size = maxTxns() * sizeof(SIDTIDEntry_t); + seg->txnCount = 0; + seg->verID = 0; + memset(seg->activeTxns, 0, size); } void SessionMonitor::copyPreviousSegment() { - try - { - bool loadSuccessful = readMonitorDataFromFile(segmentFilename()); + try + { + bool loadSuccessful = readMonitorDataFromFile(segmentFilename()); - if (!loadSuccessful) - { - saveAsMonitorData(segmentFilename()); - readMonitorDataFromFile(segmentFilename()); - } - } - catch (...) + if (!loadSuccessful) { - saveAsMonitorData(segmentFilename()); - readMonitorDataFromFile(segmentFilename()); + saveAsMonitorData(segmentFilename()); + readMonitorDataFromFile(segmentFilename()); } + } + catch (...) + { + saveAsMonitorData(segmentFilename()); + readMonitorDataFromFile(segmentFilename()); + } } bool SessionMonitor::readMonitorDataFromFile(const std::string filename) { - int err = 0; - uint32_t headerSize = 2 * sizeof(int); - char* data = reinterpret_cast(&fPrevSegment); - int fd = open(filename.c_str(), O_RDONLY); + int err = 0; + uint32_t headerSize = 2 * sizeof(int); + char* data = reinterpret_cast(&fPrevSegment); + int fd = open(filename.c_str(), O_RDONLY); - if (fd < 0) + if (fd < 0) + { + perror("SessionMonitor::readMonitorDataFromFile(): open"); + return false; + } + + err = read(fd, data, headerSize); + + if (err < 0) + { + if (errno != EINTR) { - perror("SessionMonitor::readMonitorDataFromFile(): open"); - return false; + perror("SessionMonitor::readMonitorDataFromFile(): read"); } - - err = read(fd, data, headerSize); - - if (err < 0) - { - if (errno != EINTR) - { - perror("SessionMonitor::readMonitorDataFromFile(): read"); - } - } - else if (err == 0) - { - close(fd); - return false; - } - - int dataSize = maxTxns() * sizeof(MonSIDTIDEntry_t); - delete [] fPrevSegment.activeTxns; - - fPrevSegment.activeTxns = new MonSIDTIDEntry[maxTxns()]; - data = reinterpret_cast(fPrevSegment.activeTxns); - memset(data, 0, sizeof(MonSIDTIDEntry)*maxTxns()); - err = read(fd, data, dataSize); - - if (err < 0) - { - if (errno != EINTR) - { - perror("SessionMonitor::readMonitorDataFromFile(): read"); - } - } - else if (err == 0) - { - close(fd); - perror("SessionMonitor::readMonitorDataFromFile(): read 0"); - return false; - } - + } + else if (err == 0) + { close(fd); + return false; + } - return true; + int dataSize = maxTxns() * sizeof(MonSIDTIDEntry_t); + delete[] fPrevSegment.activeTxns; + + fPrevSegment.activeTxns = new MonSIDTIDEntry[maxTxns()]; + data = reinterpret_cast(fPrevSegment.activeTxns); + memset(data, 0, sizeof(MonSIDTIDEntry) * maxTxns()); + err = read(fd, data, dataSize); + + if (err < 0) + { + if (errno != EINTR) + { + perror("SessionMonitor::readMonitorDataFromFile(): read"); + } + } + else if (err == 0) + { + close(fd); + perror("SessionMonitor::readMonitorDataFromFile(): read 0"); + return false; + } + + close(fd); + + return true; } void SessionMonitor::saveAsMonitorData(const std::string) { - int fd; - int err = 0; - char* data = reinterpret_cast(&fSessionMonitorData); + int fd; + int err = 0; + char* data = reinterpret_cast(&fSessionMonitorData); - if (!fSessionMonitorData.activeTxns) + if (!fSessionMonitorData.activeTxns) + { + fSessionMonitorData.activeTxns = new MonSIDTIDEntry[maxTxns()]; + } + + initSegment(&fSessionMonitorData); + + // get the most recent SessionManagerData + copyCurrentSegment(); + fSessionMonitorData.txnCount = fCurrentSegment->txnCount; + fSessionMonitorData.verID = fCurrentSegment->verID; + + for (int idx = 0; idx < maxTxns(); idx++) + { + // is this a new txns or previously existing + MonSIDTIDEntry* monitor = &fSessionMonitorData.activeTxns[idx]; + MonSIDTIDEntry* prevMonitor = (fPrevSegment.activeTxns ? &fPrevSegment.activeTxns[idx] : NULL); + SIDTIDEntry* manager = &fCurrentSegment->activeTxns[idx]; + + if (prevMonitor) { - fSessionMonitorData.activeTxns = new MonSIDTIDEntry[maxTxns()]; - } - - initSegment(&fSessionMonitorData); - - // get the most recent SessionManagerData - copyCurrentSegment(); - fSessionMonitorData.txnCount = fCurrentSegment->txnCount; - fSessionMonitorData.verID = fCurrentSegment->verID; - - for (int idx = 0; idx < maxTxns(); idx++) - { - // is this a new txns or previously existing - MonSIDTIDEntry* monitor = &fSessionMonitorData.activeTxns[idx]; - MonSIDTIDEntry* prevMonitor = (fPrevSegment.activeTxns ? &fPrevSegment.activeTxns[idx] : NULL); - SIDTIDEntry* manager = &fCurrentSegment->activeTxns[idx]; - - if (prevMonitor) - { - if (!isEqualSIDTID(*manager, *prevMonitor)) - { - monitor->txnid.firstrecord = time(NULL); - } - else if (isEqualSIDTID(*manager, *prevMonitor) && isUsed(*prevMonitor)) - { - if (prevMonitor && prevMonitor->txnid.firstrecord == 0) - monitor->txnid.firstrecord = time(NULL); - else - monitor->txnid.firstrecord = prevMonitor->txnid.firstrecord; - } - else if (manager->txnid.valid == false && monitor->txnid.id == manager->txnid.id) - monitor->txnid.firstrecord = 0; - } + if (!isEqualSIDTID(*manager, *prevMonitor)) + { + monitor->txnid.firstrecord = time(NULL); + } + else if (isEqualSIDTID(*manager, *prevMonitor) && isUsed(*prevMonitor)) + { + if (prevMonitor && prevMonitor->txnid.firstrecord == 0) + monitor->txnid.firstrecord = time(NULL); else - { - if (manager->txnid.valid && manager->txnid.id) - { - monitor->txnid.firstrecord = time(NULL); - } - else - monitor->txnid.firstrecord = 0; - } - - monitor->sessionid = manager->sessionid; - monitor->txnid.id = manager->txnid.id; - monitor->txnid.valid = manager->txnid.valid; + monitor->txnid.firstrecord = prevMonitor->txnid.firstrecord; + } + else if (manager->txnid.valid == false && monitor->txnid.id == manager->txnid.id) + monitor->txnid.firstrecord = 0; } - - // Always write to a new empty file - fd = open(segmentFilename(), O_WRONLY | O_CREAT | O_TRUNC, 0666); - - if (fd < 0) + else { - perror("SessionMonitor::saveAsMonitorData(): open"); - throw ios_base::failure("SessionMonitor::saveAsMonitorData(): open failed. Check the error log."); + if (manager->txnid.valid && manager->txnid.id) + { + monitor->txnid.firstrecord = time(NULL); + } + else + monitor->txnid.firstrecord = 0; } - int headerSize = 2 * (sizeof(int)); - err = write(fd, data, headerSize); + monitor->sessionid = manager->sessionid; + monitor->txnid.id = manager->txnid.id; + monitor->txnid.valid = manager->txnid.valid; + } - if (err < 0) - { - if (errno != EINTR) - perror("SessionMonitor::saveAsMonitorData(): write"); - } + // Always write to a new empty file + fd = open(segmentFilename(), O_WRONLY | O_CREAT | O_TRUNC, 0666); - int dataSize = maxTxns() * sizeof(MonSIDTIDEntry); - data = reinterpret_cast(fSessionMonitorData.activeTxns); - err = write(fd, data, dataSize); + if (fd < 0) + { + perror("SessionMonitor::saveAsMonitorData(): open"); + throw ios_base::failure("SessionMonitor::saveAsMonitorData(): open failed. Check the error log."); + } - if (err < 0) - { - if (errno != EINTR) - perror("SessionMonitor::saveAsMonitorData(): write"); - } + int headerSize = 2 * (sizeof(int)); + err = write(fd, data, headerSize); - close(fd); + if (err < 0) + { + if (errno != EINTR) + perror("SessionMonitor::saveAsMonitorData(): write"); + } + + int dataSize = maxTxns() * sizeof(MonSIDTIDEntry); + data = reinterpret_cast(fSessionMonitorData.activeTxns); + err = write(fd, data, dataSize); + + if (err < 0) + { + if (errno != EINTR) + perror("SessionMonitor::saveAsMonitorData(): write"); + } + + close(fd); } void SessionMonitor::copyCurrentSegment() { - const int cpsize = 4 * sizeof(int) + 2 * sizeof(boost::interprocess::interprocess_semaphore) + maxTxns() * sizeof(SIDTIDEntry); + const int cpsize = 4 * sizeof(int) + 2 * sizeof(boost::interprocess::interprocess_semaphore) + + maxTxns() * sizeof(SIDTIDEntry); - delete [] reinterpret_cast(fCurrentSegment); + delete[] reinterpret_cast(fCurrentSegment); - fCurrentSegment = reinterpret_cast(new char[cpsize]); - lock(); - memcpy(fCurrentSegment, fSessionManagerData, cpsize); - unlock(); + fCurrentSegment = reinterpret_cast(new char[cpsize]); + lock(); + memcpy(fCurrentSegment, fSessionManagerData, cpsize); + unlock(); } -bool SessionMonitor::isUsed( const MonSIDTIDEntry& e) const +bool SessionMonitor::isUsed(const MonSIDTIDEntry& e) const { - if (e.sessionid == 0 && e.txnid.id == 0 && e.txnid.valid == false && e.txnid.firstrecord == 0) - return false; - - return true; -} - -bool SessionMonitor::isUsedSIDTID( const SIDTIDEntry& e) const -{ - if (e.sessionid == 0 && e.txnid.id == 0 && e.txnid.valid == false) - return false; - - return true; -} - -bool SessionMonitor::isStaleSIDTID( const SIDTIDEntry& a, const MonSIDTIDEntry& b) const -{ - if (b.txnid.valid && isEqualSIDTID(a, b) && b.txnid.firstrecord && (time(NULL) - b.txnid.firstrecord) > fAgeLimit) - return true; - + if (e.sessionid == 0 && e.txnid.id == 0 && e.txnid.valid == false && e.txnid.firstrecord == 0) return false; + + return true; } -bool SessionMonitor::isEqualSIDTID( const SIDTIDEntry& a, const MonSIDTIDEntry& b) const +bool SessionMonitor::isUsedSIDTID(const SIDTIDEntry& e) const { - if (a.sessionid == b.sessionid && - a.txnid.id == b.txnid.id && - a.txnid.valid == b.txnid.valid) - return true; - + if (e.sessionid == 0 && e.txnid.id == 0 && e.txnid.valid == false) return false; + + return true; +} + +bool SessionMonitor::isStaleSIDTID(const SIDTIDEntry& a, const MonSIDTIDEntry& b) const +{ + if (b.txnid.valid && isEqualSIDTID(a, b) && b.txnid.firstrecord && + (time(NULL) - b.txnid.firstrecord) > fAgeLimit) + return true; + + return false; +} + +bool SessionMonitor::isEqualSIDTID(const SIDTIDEntry& a, const MonSIDTIDEntry& b) const +{ + if (a.sessionid == b.sessionid && a.txnid.id == b.txnid.id && a.txnid.valid == b.txnid.valid) + return true; + + return false; } vector SessionMonitor::timedOutTxns() { - vector txnsVec; + vector txnsVec; - copyCurrentSegment(); + copyCurrentSegment(); - if (fCurrentSegment && fPrevSegment.activeTxns != NULL) + if (fCurrentSegment && fPrevSegment.activeTxns != NULL) + { + for (int idx = 0; fCurrentSegment->activeTxns && fPrevSegment.activeTxns && idx < maxTxns(); idx++) { - for (int idx = 0; fCurrentSegment->activeTxns && fPrevSegment.activeTxns && idx < maxTxns(); idx++) - { - if (isUsedSIDTID(fCurrentSegment->activeTxns[idx]) && - isStaleSIDTID(fCurrentSegment->activeTxns[idx], fPrevSegment.activeTxns[idx])) - { - txnsVec.push_back(&fPrevSegment.activeTxns[idx]); - } - } - - sort(txnsVec.begin(), txnsVec.end(), lessMonSIDTIDEntry()); + if (isUsedSIDTID(fCurrentSegment->activeTxns[idx]) && + isStaleSIDTID(fCurrentSegment->activeTxns[idx], fPrevSegment.activeTxns[idx])) + { + txnsVec.push_back(&fPrevSegment.activeTxns[idx]); + } } - return txnsVec; + sort(txnsVec.begin(), txnsVec.end(), lessMonSIDTIDEntry()); + } + + return txnsVec; } const int SessionMonitor::txnCount() const { - return fSessionMonitorData.txnCount; + return fSessionMonitorData.txnCount; } -} //namespace +} // namespace execplan diff --git a/dbcon/execplan/sessionmonitor.h b/dbcon/execplan/sessionmonitor.h index cf31b63b3..fb65dbc93 100644 --- a/dbcon/execplan/sessionmonitor.h +++ b/dbcon/execplan/sessionmonitor.h @@ -41,7 +41,6 @@ namespace execplan { - /** @brief Identifies stale and orphaned transaction IDs. * * @@ -68,277 +67,275 @@ namespace execplan class SessionMonitor { -public: - - /** @brief A type describing a single transaction ID */ - struct MonTxnID + public: + /** @brief A type describing a single transaction ID */ + struct MonTxnID + { + /// The TransactionID number + CalpontSystemCatalog::SCN id; + /// True if the id is valid. + bool valid; + /// timestamp of firstrecord of this TID + time_t firstrecord; + MonTxnID() { - /// The TransactionID number - CalpontSystemCatalog::SCN id; - /// True if the id is valid. - bool valid; - /// timestamp of firstrecord of this TID - time_t firstrecord; - MonTxnID() - { - id = 0; - valid = false; - firstrecord = 0; - }; + id = 0; + valid = false; + firstrecord = 0; }; + }; - /** @brief A type describing a single transaction ID */ - typedef struct MonTxnID MonTxnID; + /** @brief A type describing a single transaction ID */ + typedef struct MonTxnID MonTxnID; - /** @brief A type associating a session with a transaction */ - struct MonSIDTIDEntry + /** @brief A type associating a session with a transaction */ + struct MonSIDTIDEntry + { + /// The Transaction ID. txnid.valid determines whether or not this SIDTIDEntry is valid + MonTxnID txnid; + /// The session doing the transaction + SessionManager::SID sessionid; + MonSIDTIDEntry() { - /// The Transaction ID. txnid.valid determines whether or not this SIDTIDEntry is valid - MonTxnID txnid; - /// The session doing the transaction - SessionManager::SID sessionid; - MonSIDTIDEntry() - { - txnid = MonTxnID(); - sessionid = 0; - }; + txnid = MonTxnID(); + sessionid = 0; }; + }; - /** @brief A type associating a session with a transaction */ - typedef struct MonSIDTIDEntry MonSIDTIDEntry_t; + /** @brief A type associating a session with a transaction */ + typedef struct MonSIDTIDEntry MonSIDTIDEntry_t; - /** @brief Vector of MonSIDTIDEntry structures */ - typedef std::vector MonSIDTIDEntryVector_t; + /** @brief Vector of MonSIDTIDEntry structures */ + typedef std::vector MonSIDTIDEntryVector_t; - /** @brief needed to sort txns list in find timedOutTnxs()*/ - struct lessMonSIDTIDEntry + /** @brief needed to sort txns list in find timedOutTnxs()*/ + struct lessMonSIDTIDEntry + { + bool operator()(MonSIDTIDEntry const* p1, MonSIDTIDEntry const* p2) { - bool operator()(MonSIDTIDEntry const* p1, MonSIDTIDEntry const* p2) - { - if (!p1) - return true; + if (!p1) + return true; - if (!p2) - return false; + if (!p2) + return false; - return p1->txnid.firstrecord < p2->txnid.firstrecord; - } - }; - - /** @brief monitor version of the SessionManagerData */ - struct SessionMonitorData_struct - { - int txnCount; - CalpontSystemCatalog::SCN verID; - MonSIDTIDEntry_t* activeTxns; - SessionMonitorData_struct() - { - txnCount = 0; - verID = 0; - activeTxns = NULL; - }; - }; - typedef struct SessionMonitorData_struct SessionMonitorData_t; - - /** @brief typedef if SessionManager struct SIDTIDEntry. For convenience.*/ - typedef BRM::SIDTIDEntry SIDTIDEntry_t; - /** @brief This struct describes the layout of the shared memory segment copied from SessionManager.h */ - struct SessionManagerData_struct - { - int txnCount; - CalpontSystemCatalog::SCN verID; - CalpontSystemCatalog::SCN syscatVerID; - int systemstate; - boost::interprocess::interprocess_semaphore sems[2]; - SIDTIDEntry_t activeTxns[]; - }; - typedef SessionManagerData_struct SessionManagerData_t; - - /** @brief Constructor - * - * This attaches to existing shared memory segments. - * @note throws ios_base::failure on file IO error, runtime_error for - * other types of errors. It might be worthwhile to define additional - * exceptions for all the different types of failures that can happen - * here. - */ - SessionMonitor(); - - /** @brief Destructor - * - * This detaches the shared memory segment then saves it - * to disk and destroyed. It does not destroy the semaphores - * or the shared memory segment. Deletes the local copies - * of the SessionManager data and SessionMonitor data. - */ - virtual ~SessionMonitor(); - - const int maxTxns() const - { - return fMaxTxns; + return p1->txnid.firstrecord < p2->txnid.firstrecord; } - const int txnCount() const; + }; - /** - * @brief identifies timed out SessionManager structures - * @param - * @return - */ - MonSIDTIDEntryVector_t timedOutTxns(); + /** @brief monitor version of the SessionManagerData */ + struct SessionMonitorData_struct + { + int txnCount; + CalpontSystemCatalog::SCN verID; + MonSIDTIDEntry_t* activeTxns; + SessionMonitorData_struct() + { + txnCount = 0; + verID = 0; + activeTxns = NULL; + }; + }; + typedef struct SessionMonitorData_struct SessionMonitorData_t; - const SessionMonitorData_t* previousManagerData() - { - return &fPrevSegment; - } - SessionManagerData_t* currentManagerData() - { - return fCurrentSegment; - } - const SessionManagerData_t* sessionManagerData() - { - return fSessionManagerData; - } + /** @brief typedef if SessionManager struct SIDTIDEntry. For convenience.*/ + typedef BRM::SIDTIDEntry SIDTIDEntry_t; + /** @brief This struct describes the layout of the shared memory segment copied from SessionManager.h */ + struct SessionManagerData_struct + { + int txnCount; + CalpontSystemCatalog::SCN verID; + CalpontSystemCatalog::SCN syscatVerID; + int systemstate; + boost::interprocess::interprocess_semaphore sems[2]; + SIDTIDEntry_t activeTxns[]; + }; + typedef SessionManagerData_struct SessionManagerData_t; - void printTxns(const MonSIDTIDEntry_t& txn) const; + /** @brief Constructor + * + * This attaches to existing shared memory segments. + * @note throws ios_base::failure on file IO error, runtime_error for + * other types of errors. It might be worthwhile to define additional + * exceptions for all the different types of failures that can happen + * here. + */ + SessionMonitor(); + + /** @brief Destructor + * + * This detaches the shared memory segment then saves it + * to disk and destroyed. It does not destroy the semaphores + * or the shared memory segment. Deletes the local copies + * of the SessionManager data and SessionMonitor data. + */ + virtual ~SessionMonitor(); + + const int maxTxns() const + { + return fMaxTxns; + } + const int txnCount() const; + + /** + * @brief identifies timed out SessionManager structures + * @param + * @return + */ + MonSIDTIDEntryVector_t timedOutTxns(); + + const SessionMonitorData_t* previousManagerData() + { + return &fPrevSegment; + } + SessionManagerData_t* currentManagerData() + { + return fCurrentSegment; + } + const SessionManagerData_t* sessionManagerData() + { + return fSessionManagerData; + } + + void printTxns(const MonSIDTIDEntry_t& txn) const; #ifdef SM_DEBUG - void printSegment(const SessionManagerData_t* seg, const int l = 1000) const; - void printMonitorData(const int l = 1000) const; - void printTxns(const SIDTIDEntry_t& txn) const; + void printSegment(const SessionManagerData_t* seg, const int l = 1000) const; + void printMonitorData(const int l = 1000) const; + void printTxns(const SIDTIDEntry_t& txn) const; #endif - const int AgeLimit() const - { - return fAgeLimit; // to speed up testing - } - void AgeLimit(const int& age) - { - fAgeLimit = age; - } - const char* segmentFilename() const - { - return fSegmentFilename.c_str(); - } - bool haveSemaphores() const - { - return fHaveSemaphores; - } - bool haveSharedMemory() const - { - return fIsAttached; - } + const int AgeLimit() const + { + return fAgeLimit; // to speed up testing + } + void AgeLimit(const int& age) + { + fAgeLimit = age; + } + const char* segmentFilename() const + { + return fSegmentFilename.c_str(); + } + bool haveSemaphores() const + { + return fHaveSemaphores; + } + bool haveSharedMemory() const + { + return fIsAttached; + } -private: + private: + int fMaxTxns; // the maximum number of concurrent transactions + static const int fMaxRetries = 10; // the max number of retries on file IO + std::string fSegmentFilename; // filename used to store the image of the SessionManager data + time_t fDefaultAgeLimit; + time_t fAgeLimit; // age limit in seconds used to timeout/orpan a transaction - int fMaxTxns; // the maximum number of concurrent transactions - static const int fMaxRetries = 10; // the max number of retries on file IO - std::string fSegmentFilename; // filename used to store the image of the SessionManager data - time_t fDefaultAgeLimit; - time_t fAgeLimit; // age limit in seconds used to timeout/orpan a transaction + SessionMonitorData_t fSessionMonitorData; // pointer to in memory copy of the Monitor data + SessionMonitorData_t fPrevSegment; // in memory copy of SessionManager data from backup file + SessionManagerData_t* fCurrentSegment; // in memory of SessionManager data + SessionManagerData_t* fSessionManagerData; // current shared memory SessionManager data - SessionMonitorData_t fSessionMonitorData; // pointer to in memory copy of the Monitor data - SessionMonitorData_t fPrevSegment; // in memory copy of SessionManager data from backup file - SessionManagerData_t* fCurrentSegment; // in memory of SessionManager data - SessionManagerData_t* fSessionManagerData; // current shared memory SessionManager data + // refers to 2 semaphores; the first is a mutex that guards the shmseg + // the second is used to block processes when there are too many concurrent txns + int fsems; + int fshmid; // shared memory segment id + uint32_t fuid; // user id used to create the shared memory key - // refers to 2 semaphores; the first is a mutex that guards the shmseg - // the second is used to block processes when there are too many concurrent txns - int fsems; - int fshmid; // shared memory segment id - uint32_t fuid; // user id used to create the shared memory key + SessionManager sm; - SessionManager sm; + bool isStaleSIDTID(const SIDTIDEntry_t& src, const MonSIDTIDEntry_t& dest) const; + bool isEqualSIDTID(const SIDTIDEntry_t& src, const MonSIDTIDEntry_t& dest) const; + bool isUsedSIDTID(const SIDTIDEntry_t& e) const; + bool isUsed(const MonSIDTIDEntry_t& e) const; - bool isStaleSIDTID(const SIDTIDEntry_t& src, const MonSIDTIDEntry_t& dest) const; - bool isEqualSIDTID(const SIDTIDEntry_t& src, const MonSIDTIDEntry_t& dest) const; - bool isUsedSIDTID(const SIDTIDEntry_t& e) const; - bool isUsed(const MonSIDTIDEntry_t& e) const; + /** + * @brief attached to SessionManagerData shared memory + * @param + * @return void + */ + void getSharedData(void); - /** - * @brief attached to SessionManagerData shared memory - * @param - * @return void - */ - void getSharedData(void); + bool fIsAttached; - bool fIsAttached; + /** + * @brief unattached from SessionManagerData shared memory + * @param + * @return void + */ + void detachSegment(void); - /** - * @brief unattached from SessionManagerData shared memory - * @param - * @return void - */ - void detachSegment(void); + /** + * @brief initialize SessionMonitorData + * @param + * @return void + */ + void initSegment(SessionMonitorData_t* seg); - /** - * @brief initialize SessionMonitorData - * @param - * @return void - */ - void initSegment(SessionMonitorData_t* seg); + /** + * @brief initialize SessionManagerData + * @param + * @return void + */ + void initSegment(SessionManagerData_t* seg); - /** - * @brief initialize SessionManagerData - * @param - * @return void - */ - void initSegment(SessionManagerData_t* seg); + const key_t IPCKey() const + { + return fShmKeys.SESSIONMANAGER_SYSVKEY; + }; - const key_t IPCKey() const - { - return fShmKeys.SESSIONMANAGER_SYSVKEY; - }; + /** + * @brief copies the SessionMonitor data from a file into memory + * @param + * @return void + */ + void copyPreviousSegment(); - /** - * @brief copies the SessionMonitor data from a file into memory - * @param - * @return void - */ - void copyPreviousSegment(); + /** + * @brief copies the SessionManager data from shared memory into an SessionMonitor memory + * @param + * @return void + */ + void copyCurrentSegment(); - /** - * @brief copies the SessionManager data from shared memory into an SessionMonitor memory - * @param - * @return void - */ - void copyCurrentSegment(); + /** + * @brief Reads the SM data from file into memory. Used by copyPreviousSegment(). + * @param + * @return bool + */ + bool readMonitorDataFromFile(const std::string); - /** - * @brief Reads the SM data from file into memory. Used by copyPreviousSegment(). - * @param - * @return bool - */ - bool readMonitorDataFromFile(const std::string); + /** + * @brief write the current SessionManagerData as SessionMonitorData to a file. + * @param + * @return void + */ + void saveAsMonitorData(const std::string); - /** - * @brief write the current SessionManagerData as SessionMonitorData to a file. - * @param - * @return void - */ - void saveAsMonitorData(const std::string); + bool fHaveSemaphores; - bool fHaveSemaphores; + /** + * @brief get the SessionManager semaphores + * @param + * @return void + */ + int getSems(void); - /** - * @brief get the SessionManager semaphores - * @param - * @return void - */ - int getSems(void); + void lock(void); - void lock(void); + void unlock(void); - void unlock(void); + /** + * @brief do not use the copy constructor. + * @param + * @return + */ + SessionMonitor(const SessionMonitor&); - /** - * @brief do not use the copy constructor. - * @param - * @return - */ - SessionMonitor(const SessionMonitor&); - -private: - BRM::ShmKeys fShmKeys; + private: + BRM::ShmKeys fShmKeys; }; -} //namespace +} // namespace execplan #endif diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 8b77426ea..d08144d75 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplecolumn.cpp 9576 2013-05-29 21:02:11Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplecolumn.cpp 9576 2013-05-29 21:02:11Z zzhu $ + * + * + ***********************************************************************/ #include #include @@ -55,672 +55,644 @@ using namespace joblist; namespace execplan { - void getSimpleCols(execplan::ParseTree* n, void* obj) { - vector* list = reinterpret_cast< vector*>(obj); - TreeNode* tn = n->data(); - SimpleColumn* sc = dynamic_cast(tn); - FunctionColumn* fc = dynamic_cast(tn); - ArithmeticColumn* ac = dynamic_cast(tn); - SimpleFilter* sf = dynamic_cast(tn); - ConstantFilter* cf = dynamic_cast(tn); + vector* list = reinterpret_cast*>(obj); + TreeNode* tn = n->data(); + SimpleColumn* sc = dynamic_cast(tn); + FunctionColumn* fc = dynamic_cast(tn); + ArithmeticColumn* ac = dynamic_cast(tn); + SimpleFilter* sf = dynamic_cast(tn); + ConstantFilter* cf = dynamic_cast(tn); - if (sc) - { - list->push_back(sc); - } - else if (fc) - { - fc->setSimpleColumnList(); - list->insert(list->end(), fc->simpleColumnList().begin(), fc->simpleColumnList().end()); - } - else if (ac) - { - ac->setSimpleColumnList(); - list->insert(list->end(), ac->simpleColumnList().begin(), ac->simpleColumnList().end()); - } - else if (sf) - { - sf->setSimpleColumnList(); - list->insert(list->end(), sf->simpleColumnList().begin(), sf->simpleColumnList().end()); - } - else if (cf) - { - cf->setSimpleColumnList(); - list->insert(list->end(), cf->simpleColumnList().begin(), cf->simpleColumnList().end()); - } + if (sc) + { + list->push_back(sc); + } + else if (fc) + { + fc->setSimpleColumnList(); + list->insert(list->end(), fc->simpleColumnList().begin(), fc->simpleColumnList().end()); + } + else if (ac) + { + ac->setSimpleColumnList(); + list->insert(list->end(), ac->simpleColumnList().begin(), ac->simpleColumnList().end()); + } + else if (sf) + { + sf->setSimpleColumnList(); + list->insert(list->end(), sf->simpleColumnList().begin(), sf->simpleColumnList().end()); + } + else if (cf) + { + cf->setSimpleColumnList(); + list->insert(list->end(), cf->simpleColumnList().begin(), cf->simpleColumnList().end()); + } } ParseTree* replaceRefCol(ParseTree*& n, CalpontSelectExecutionPlan::ReturnedColumnList& derivedColList) { - ParseTree* lhs = n->left(); - ParseTree* rhs = n->right(); + ParseTree* lhs = n->left(); + ParseTree* rhs = n->right(); - if (lhs) - n->left(replaceRefCol(lhs, derivedColList)); + if (lhs) + n->left(replaceRefCol(lhs, derivedColList)); - if (rhs) - n->right(replaceRefCol(rhs, derivedColList)); + if (rhs) + n->right(replaceRefCol(rhs, derivedColList)); - SimpleFilter* sf = dynamic_cast(n->data()); - ConstantFilter* cf = dynamic_cast(n->data()); - ReturnedColumn* rc = dynamic_cast(n->data()); + SimpleFilter* sf = dynamic_cast(n->data()); + ConstantFilter* cf = dynamic_cast(n->data()); + ReturnedColumn* rc = dynamic_cast(n->data()); - if (sf) + if (sf) + { + sf->replaceRealCol(derivedColList); + } + else if (cf) + { + cf->replaceRealCol(derivedColList); + } + else if (rc) + { + SimpleColumn* sc = dynamic_cast(rc); + + if (sc && (sc->colPosition() > -1)) { - sf->replaceRealCol(derivedColList); + ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone(); + delete sc; + n->data(tmp); } - else if (cf) + else { - cf->replaceRealCol(derivedColList); + rc->replaceRealCol(derivedColList); } - else if (rc) - { - SimpleColumn* sc = dynamic_cast(rc); + } - if (sc && (sc->colPosition() > -1)) - { - ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone(); - delete sc; - n->data(tmp); - } - else - { - rc->replaceRealCol(derivedColList); - } - } - - return n; + return n; } /** * Constructors/Destructors */ -SimpleColumn::SimpleColumn(): - ReturnedColumn(), - fOid (0), - fisColumnStore (true) +SimpleColumn::SimpleColumn() : ReturnedColumn(), fOid(0), fisColumnStore(true) { - fDistinct = false; + fDistinct = false; } -SimpleColumn::SimpleColumn(const string& token, const uint32_t sessionID): - ReturnedColumn(sessionID), - fOid (0), - fData(token), - fisColumnStore (true) +SimpleColumn::SimpleColumn(const string& token, const uint32_t sessionID) + : ReturnedColumn(sessionID), fOid(0), fData(token), fisColumnStore(true) { - parse (token); + parse(token); + setOID(); + fDistinct = false; +} + +SimpleColumn::SimpleColumn(const string& schemaName, const string& tableName, const string& columnName, + const uint32_t sessionID, const int lower_case_table_names) + : ReturnedColumn(sessionID) + , fSchemaName(schemaName) + , fTableName(tableName) + , fColumnName(columnName) + , fisColumnStore(true) +{ + setOID(); + fDistinct = false; + if (lower_case_table_names) + { + boost::algorithm::to_lower(fSchemaName); + boost::algorithm::to_lower(fTableName); + } + boost::algorithm::to_lower(fColumnName); +} + +SimpleColumn::SimpleColumn(const string& schemaName, const string& tableName, const string& columnName, + const bool isColumnStore, const uint32_t sessionID, + const int lower_case_table_names) + : ReturnedColumn(sessionID) + , fSchemaName(schemaName) + , fTableName(tableName) + , fColumnName(columnName) + , fisColumnStore(isColumnStore) +{ + if (isColumnStore) setOID(); - fDistinct = false; + fDistinct = false; + if (lower_case_table_names) + { + boost::algorithm::to_lower(fSchemaName); + boost::algorithm::to_lower(fTableName); + } + boost::algorithm::to_lower(fColumnName); } -SimpleColumn::SimpleColumn(const string& schemaName, - const string& tableName, - const string& columnName, - const uint32_t sessionID, - const int lower_case_table_names) : - ReturnedColumn(sessionID), - fSchemaName (schemaName), - fTableName (tableName), - fColumnName (columnName), - fisColumnStore (true) -{ - setOID(); - fDistinct = false; - if (lower_case_table_names) - { - boost::algorithm::to_lower(fSchemaName); - boost::algorithm::to_lower(fTableName); - } - boost::algorithm::to_lower(fColumnName); -} - -SimpleColumn::SimpleColumn(const string& schemaName, - const string& tableName, - const string& columnName, - const bool isColumnStore, - const uint32_t sessionID, - const int lower_case_table_names) : - ReturnedColumn(sessionID), - fSchemaName (schemaName), - fTableName (tableName), - fColumnName (columnName), - fisColumnStore (isColumnStore) -{ - if (isColumnStore) - setOID(); - fDistinct = false; - if (lower_case_table_names) - { - boost::algorithm::to_lower(fSchemaName); - boost::algorithm::to_lower(fTableName); - } - boost::algorithm::to_lower(fColumnName); -} - -SimpleColumn::SimpleColumn (const SimpleColumn& rhs, const uint32_t sessionID): - ReturnedColumn(rhs, sessionID), - fSchemaName (rhs.schemaName()), - fTableName (rhs.tableName()), - fColumnName (rhs.columnName()), - fOid (rhs.oid()), - fTableAlias (rhs.tableAlias()), - fData (rhs.data()), - fIndexName (rhs.indexName()), - fViewName (rhs.viewName()), - fTimeZone (rhs.timeZone()), - fisColumnStore (rhs.isColumnStore()) +SimpleColumn::SimpleColumn(const SimpleColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID) + , fSchemaName(rhs.schemaName()) + , fTableName(rhs.tableName()) + , fColumnName(rhs.columnName()) + , fOid(rhs.oid()) + , fTableAlias(rhs.tableAlias()) + , fData(rhs.data()) + , fIndexName(rhs.indexName()) + , fViewName(rhs.viewName()) + , fTimeZone(rhs.timeZone()) + , fisColumnStore(rhs.isColumnStore()) { } SimpleColumn::~SimpleColumn() -{} +{ +} /** * Methods */ const string SimpleColumn::data() const { - if (!fData.empty()) - return fData; - else if (!fTableAlias.empty()) - return string("`" + fSchemaName + "`.`" + fTableAlias + "`.`" + fColumnName + "`"); + if (!fData.empty()) + return fData; + else if (!fTableAlias.empty()) + return string("`" + fSchemaName + "`.`" + fTableAlias + "`.`" + fColumnName + "`"); - return string("`" + fSchemaName + "`.`" + fTableName + "`.`" + fColumnName + "`"); + return string("`" + fSchemaName + "`.`" + fTableName + "`.`" + fColumnName + "`"); } SimpleColumn& SimpleColumn::operator=(const SimpleColumn& rhs) { - if (this != &rhs) - { - fTableName = rhs.tableName(); - fColumnName = rhs.columnName(); - fOid = rhs.oid(); - fSchemaName = rhs.schemaName(); - fAlias = rhs.alias(); - fTableAlias = rhs.tableAlias(); - fAsc = rhs.asc(); - fIndexName = rhs.indexName(); - fViewName = rhs.viewName(); - fTimeZone = rhs.timeZone(); - fData = rhs.data(); - fSequence = rhs.sequence(); - fDistinct = rhs.distinct(); - fisColumnStore = rhs.isColumnStore(); - } + if (this != &rhs) + { + fTableName = rhs.tableName(); + fColumnName = rhs.columnName(); + fOid = rhs.oid(); + fSchemaName = rhs.schemaName(); + fAlias = rhs.alias(); + fTableAlias = rhs.tableAlias(); + fAsc = rhs.asc(); + fIndexName = rhs.indexName(); + fViewName = rhs.viewName(); + fTimeZone = rhs.timeZone(); + fData = rhs.data(); + fSequence = rhs.sequence(); + fDistinct = rhs.distinct(); + fisColumnStore = rhs.isColumnStore(); + } - return *this; + return *this; } ostream& operator<<(ostream& output, const SimpleColumn& rhs) { - output << rhs.toString(); + output << rhs.toString(); - return output; + return output; } const string SimpleColumn::toString() const { - static const char delim = '/'; - ostringstream output; + static const char delim = '/'; + ostringstream output; - output << "SimpleColumn " << data() << endl; - // collations in both result and operations type are the same and - // set in the plugin code. - datatypes::Charset cs(fResultType.charsetNumber); - output << " s/t/c/v/o/ct/TA/CA/RA/#/card/join/source/engine/colPos/cs/coll: " - << schemaName() << delim - << tableName() << delim - << columnName() << delim - << viewName() << delim - << oid() << delim - << colDataTypeToString(fResultType.colDataType) << delim - << tableAlias() << delim - << alias() << delim - << returnAll() << delim - << sequence() << delim - << cardinality() << delim - << joinInfo() << delim - << colSource() << delim - << (isColumnStore() ? "ColumnStore" : "ForeignEngine") << delim - << colPosition() << delim - << cs.getCharset().cs_name.str << delim - << cs.getCharset().coll_name.str << delim - << endl; + output << "SimpleColumn " << data() << endl; + // collations in both result and operations type are the same and + // set in the plugin code. + datatypes::Charset cs(fResultType.charsetNumber); + output << " s/t/c/v/o/ct/TA/CA/RA/#/card/join/source/engine/colPos/cs/coll: " << schemaName() << delim + << tableName() << delim << columnName() << delim << viewName() << delim << oid() << delim + << colDataTypeToString(fResultType.colDataType) << delim << tableAlias() << delim << alias() << delim + << returnAll() << delim << sequence() << delim << cardinality() << delim << joinInfo() << delim + << colSource() << delim << (isColumnStore() ? "ColumnStore" : "ForeignEngine") << delim + << colPosition() << delim << cs.getCharset().cs_name.str << delim << cs.getCharset().coll_name.str + << delim << endl; - return output.str(); + return output.str(); } void SimpleColumn::parse(const string& token) { - // get schema name, table name and column name for token. - string::size_type pos = token.find_first_of(".", 0); + // get schema name, table name and column name for token. + string::size_type pos = token.find_first_of(".", 0); - // if no '.' in column name, consider it a function name; - if (pos == string::npos) - { - fData = token; - fColumnName = token; - return; - } + // if no '.' in column name, consider it a function name; + if (pos == string::npos) + { + fData = token; + fColumnName = token; + return; + } - fSchemaName = token.substr(0, pos); + fSchemaName = token.substr(0, pos); - string::size_type newPos = token.find_first_of(".", pos + 1); + string::size_type newPos = token.find_first_of(".", pos + 1); - if (newPos == string::npos) - { - // only one '.' in column name, consider table.col pattern - fTableName = fSchemaName; - fColumnName = token.substr(pos + 1, token.length()); - return; - } + if (newPos == string::npos) + { + // only one '.' in column name, consider table.col pattern + fTableName = fSchemaName; + fColumnName = token.substr(pos + 1, token.length()); + return; + } - fTableName = token.substr(pos + 1, newPos - pos - 1); - fColumnName = token.substr(newPos + 1, token.length()); + fTableName = token.substr(pos + 1, newPos - pos - 1); + fColumnName = token.substr(newPos + 1, token.length()); } void SimpleColumn::setOID() { - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); - CalpontSystemCatalog::TableColName tcn; - tcn = make_tcn(fSchemaName, fTableName, fColumnName); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionID); + CalpontSystemCatalog::TableColName tcn; + tcn = make_tcn(fSchemaName, fTableName, fColumnName); - fOid = csc->lookupOID (tcn); + fOid = csc->lookupOID(tcn); - if (fOid == -1) + if (fOid == -1) + { + // get colMap from CalpontSelectExecutionPlan + // and try to map the schema and table name + CalpontSelectExecutionPlan::ColumnMap::iterator iter; + CalpontSelectExecutionPlan::ColumnMap colMap = CalpontSelectExecutionPlan::colMap(); + + // if this is the only column name exist in the map, return it ?? + for (iter = colMap.find(fColumnName); iter != colMap.end(); ++iter) { - // get colMap from CalpontSelectExecutionPlan - // and try to map the schema and table name - CalpontSelectExecutionPlan::ColumnMap::iterator iter; - CalpontSelectExecutionPlan::ColumnMap colMap = CalpontSelectExecutionPlan::colMap(); + SRCP srcp = iter->second; + SimpleColumn* scp = dynamic_cast(srcp.get()); - // if this is the only column name exist in the map, return it ?? - for (iter = colMap.find(fColumnName); iter != colMap.end(); ++iter) - { - SRCP srcp = iter->second; - SimpleColumn* scp = dynamic_cast(srcp.get()); - - if (colMap.count(fColumnName) == 1 || - scp->tableName().compare(fTableName) == 0) - { - fOid = scp->oid(); - //@bug 221 fix - fTableName = scp->tableName(); - fSchemaName = scp->schemaName(); - //@info assign tableAlias also. watch for other conflict - fTableAlias = scp->tableAlias(); - fResultType = scp->resultType(); - return; - } - } + if (colMap.count(fColumnName) == 1 || scp->tableName().compare(fTableName) == 0) + { + fOid = scp->oid(); + //@bug 221 fix + fTableName = scp->tableName(); + fSchemaName = scp->schemaName(); + //@info assign tableAlias also. watch for other conflict + fTableAlias = scp->tableAlias(); + fResultType = scp->resultType(); + return; + } } + } - fResultType = csc->colType(fOid); + fResultType = csc->colType(fOid); } void SimpleColumn::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN; - ReturnedColumn::serialize(b); - b << fSchemaName; - b << fTableName; - b << fColumnName; - b << fIndexName; - b << fViewName; - b << fTimeZone; - b << (uint32_t) fOid; - b << fData; - b << fTableAlias; - b << (uint32_t) fSequence; - b << static_cast(fisColumnStore); + b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN; + ReturnedColumn::serialize(b); + b << fSchemaName; + b << fTableName; + b << fColumnName; + b << fIndexName; + b << fViewName; + b << fTimeZone; + b << (uint32_t)fOid; + b << fData; + b << fTableAlias; + b << (uint32_t)fSequence; + b << static_cast(fisColumnStore); } void SimpleColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN); - ReturnedColumn::unserialize(b); - b >> fSchemaName; - b >> fTableName; - b >> fColumnName; - b >> fIndexName; - b >> fViewName; - b >> fTimeZone; - b >> (uint32_t&) fOid; - b >> fData; - b >> fTableAlias; - b >> (uint32_t&) fSequence; - b >> reinterpret_cast< ByteStream::doublebyte&>(fisColumnStore); + ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN); + ReturnedColumn::unserialize(b); + b >> fSchemaName; + b >> fTableName; + b >> fColumnName; + b >> fIndexName; + b >> fViewName; + b >> fTimeZone; + b >> (uint32_t&)fOid; + b >> fData; + b >> fTableAlias; + b >> (uint32_t&)fSequence; + b >> reinterpret_cast(fisColumnStore); } bool SimpleColumn::operator==(const SimpleColumn& t) const { - const ReturnedColumn* rc1, *rc2; + const ReturnedColumn *rc1, *rc2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (fSchemaName != t.fSchemaName) - return false; + if (fSchemaName != t.fSchemaName) + return false; - if (fTableName != t.fTableName) - return false; + if (fTableName != t.fTableName) + return false; - if (fColumnName != t.fColumnName) - return false; + if (fColumnName != t.fColumnName) + return false; - if (fViewName != t.fViewName) - return false; + if (fViewName != t.fViewName) + return false; - if (fTimeZone != t.fTimeZone) - return false; + if (fTimeZone != t.fTimeZone) + return false; - if (fOid != t.fOid) - return false; + if (fOid != t.fOid) + return false; - if (data() != t.data()) - return false; + if (data() != t.data()) + return false; - if (fTableAlias != t.fTableAlias) - return false; + if (fTableAlias != t.fTableAlias) + return false; - if (fAsc != t.fAsc) - return false; + if (fAsc != t.fAsc) + return false; - if (fReturnAll != t.fReturnAll) - return false; + if (fReturnAll != t.fReturnAll) + return false; - if (fisColumnStore != t.fisColumnStore) - return false; + if (fisColumnStore != t.fisColumnStore) + return false; - return true; + return true; } bool SimpleColumn::operator==(const TreeNode* t) const { - const SimpleColumn* sc; + const SimpleColumn* sc; - sc = dynamic_cast(t); + sc = dynamic_cast(t); - if (sc == NULL) - return false; + if (sc == NULL) + return false; - return *this == *sc; + return *this == *sc; } bool SimpleColumn::operator!=(const SimpleColumn& t) const { - return !(*this == t); + return !(*this == t); } bool SimpleColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } bool SimpleColumn::sameColumn(const ReturnedColumn* rc) const { - /** NOTE: Operations can still be merged on different table alias */ - const SimpleColumn* sc = dynamic_cast(rc); + /** NOTE: Operations can still be merged on different table alias */ + const SimpleColumn* sc = dynamic_cast(rc); - if (!sc) return false; + if (!sc) + return false; - return (fSchemaName.compare(sc->schemaName()) == 0 && - fTableName.compare(sc->tableName()) == 0 && - fColumnName.compare(sc->columnName()) == 0 && - fTableAlias.compare(sc->tableAlias()) == 0 && - fViewName.compare(sc->viewName()) == 0 && - fisColumnStore == sc->isColumnStore()); + return (fSchemaName.compare(sc->schemaName()) == 0 && fTableName.compare(sc->tableName()) == 0 && + fColumnName.compare(sc->columnName()) == 0 && fTableAlias.compare(sc->tableAlias()) == 0 && + fViewName.compare(sc->viewName()) == 0 && fisColumnStore == sc->isColumnStore()); } void SimpleColumn::setDerivedTable() { - if (hasAggregate() || hasWindowFunc()) + if (hasAggregate() || hasWindowFunc()) + { + fDerivedTable = ""; + return; + } + ReturnedColumn* rc = dynamic_cast(fDerivedRefCol); + // @todo make aggregate filter to having clause. not optimize it for now + if (rc) + { + if (rc->hasAggregate() || rc->hasWindowFunc()) { - fDerivedTable = ""; - return; - } - ReturnedColumn* rc = dynamic_cast(fDerivedRefCol); - // @todo make aggregate filter to having clause. not optimize it for now - if (rc) - { - if (rc->hasAggregate() || rc->hasWindowFunc()) - { - fDerivedTable = ""; - return; - } + fDerivedTable = ""; + return; } + } - // fDerivedTable is set at the parsing phase - if (!fSchemaName.empty()) - fDerivedTable = ""; + // fDerivedTable is set at the parsing phase + if (!fSchemaName.empty()) + fDerivedTable = ""; } bool SimpleColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) { - tan.table = fTableName; - tan.schema = fSchemaName; - tan.view = fViewName; - tan.alias = fTableAlias; - return true; + tan.table = fTableName; + tan.schema = fSchemaName; + tan.view = fViewName; + tan.alias = fTableAlias; + return true; } - // @todo move to inline void SimpleColumn::evaluate(Row& row, bool& isNull) { - // TODO Move this block into an appropriate place - if (UNLIKELY((int)(fInputOffset == (uint32_t)-1))) + // TODO Move this block into an appropriate place + if (UNLIKELY((int)(fInputOffset == (uint32_t)-1))) + { + fInputOffset = row.getOffset(fInputIndex); + } + + bool isNull2 = row.isNullValue(fInputIndex); + + if (isNull2) + { + isNull = true; + return; + } + + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::DATE: { - fInputOffset = row.getOffset(fInputIndex); + fResult.intVal = row.getUintField<4>(fInputIndex); + break; } - bool isNull2 = row.isNullValue(fInputIndex); - - if (isNull2) + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: { - isNull = true; - return; + fResult.intVal = row.getUintField<8>(fInputIndex); + break; } - switch (fResultType.colDataType) + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::STRINT: { - case CalpontSystemCatalog::DATE: + switch (row.getColumnWidth(fInputIndex)) + { + case 1: { - fResult.intVal = row.getUintField<4>(fInputIndex); - break; + fResult.origIntVal = row.getUintField<1>(fInputIndex); + break; } - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: + case 2: { - fResult.intVal = row.getUintField<8>(fInputIndex); - break; + fResult.origIntVal = row.getUintField<2>(fInputIndex); + break; } - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::STRINT: + case 3: + case 4: { - switch (row.getColumnWidth(fInputIndex)) - { - case 1: - { - fResult.origIntVal = row.getUintField<1>(fInputIndex); - break; - } - - case 2: - { - fResult.origIntVal = row.getUintField<2>(fInputIndex); - break; - } - - case 3: - case 4: - { - fResult.origIntVal = row.getUintField<4>(fInputIndex); - break; - } - - case 5: - case 6: - case 7: - case 8: - { - fResult.origIntVal = row.getUintField<8>(fInputIndex); - break; - } - - default: - { - fResult.strVal = row.getStringField(fInputIndex); - break; - } - } - - if (fResultType.colDataType == CalpontSystemCatalog::STRINT) - fResult.intVal = uint64ToStr(fResult.origIntVal); - else - fResult.intVal = atoll((char*)&fResult.origIntVal); - - break; + fResult.origIntVal = row.getUintField<4>(fInputIndex); + break; } - case CalpontSystemCatalog::BIGINT: + case 5: + case 6: + case 7: + case 8: { - fResult.intVal = row.getIntField<8>(fInputIndex); - break; + fResult.origIntVal = row.getUintField<8>(fInputIndex); + break; } - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: + default: { - fResult.intVal = row.getIntField<4>(fInputIndex); - break; + fResult.strVal = row.getStringField(fInputIndex); + break; } + } - case CalpontSystemCatalog::SMALLINT: - { - fResult.intVal = row.getIntField<2>(fInputIndex); - break; - } + if (fResultType.colDataType == CalpontSystemCatalog::STRINT) + fResult.intVal = uint64ToStr(fResult.origIntVal); + else + fResult.intVal = atoll((char*)&fResult.origIntVal); - case CalpontSystemCatalog::TINYINT: - { - fResult.intVal = row.getIntField<1>(fInputIndex); - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - fResult.floatVal = row.getFloatField(fInputIndex); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - fResult.doubleVal = row.getDoubleField(fInputIndex); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - fResult.longDoubleVal = row.getLongDoubleField(fInputIndex); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - switch (fResultType.colWidth) - { - case 16: - { - datatypes::TSInt128::assignPtrPtr(&fResult.decimalVal.s128Value, - row.getBinaryField_offset(fInputOffset)); - break; - } - case 1: - { - fResult.decimalVal.value = row.getIntField<1>(fInputIndex); - break; - } - - case 2: - { - fResult.decimalVal.value = row.getIntField<2>(fInputIndex); - break; - } - - case 4: - { - fResult.decimalVal.value = row.getIntField<4>(fInputIndex); - break; - } - - default: - { - fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); - break; - } - } - - fResult.decimalVal.scale = (unsigned)fResultType.scale; - fResult.decimalVal.precision = (unsigned)fResultType.precision; - - break; - } - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - fResult.strVal = row.getVarBinaryStringField(fInputIndex); - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - fResult.uintVal = row.getUintField<8>(fInputIndex); - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - fResult.uintVal = row.getUintField<4>(fInputIndex); - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - fResult.uintVal = row.getUintField<2>(fInputIndex); - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - fResult.uintVal = row.getUintField<1>(fInputIndex); - break; - } - - default: // treat as int64 - { - fResult.intVal = row.getUintField<8>(fInputIndex); - break; - } + break; } + + case CalpontSystemCatalog::BIGINT: + { + fResult.intVal = row.getIntField<8>(fInputIndex); + break; + } + + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + { + fResult.intVal = row.getIntField<4>(fInputIndex); + break; + } + + case CalpontSystemCatalog::SMALLINT: + { + fResult.intVal = row.getIntField<2>(fInputIndex); + break; + } + + case CalpontSystemCatalog::TINYINT: + { + fResult.intVal = row.getIntField<1>(fInputIndex); + break; + } + + // In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + fResult.floatVal = row.getFloatField(fInputIndex); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + fResult.doubleVal = row.getDoubleField(fInputIndex); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + fResult.longDoubleVal = row.getLongDoubleField(fInputIndex); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + switch (fResultType.colWidth) + { + case 16: + { + datatypes::TSInt128::assignPtrPtr(&fResult.decimalVal.s128Value, + row.getBinaryField_offset(fInputOffset)); + break; + } + case 1: + { + fResult.decimalVal.value = row.getIntField<1>(fInputIndex); + break; + } + + case 2: + { + fResult.decimalVal.value = row.getIntField<2>(fInputIndex); + break; + } + + case 4: + { + fResult.decimalVal.value = row.getIntField<4>(fInputIndex); + break; + } + + default: + { + fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); + break; + } + } + + fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal.precision = (unsigned)fResultType.precision; + + break; + } + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + fResult.strVal = row.getVarBinaryStringField(fInputIndex); + break; + } + + case CalpontSystemCatalog::UBIGINT: + { + fResult.uintVal = row.getUintField<8>(fInputIndex); + break; + } + + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + { + fResult.uintVal = row.getUintField<4>(fInputIndex); + break; + } + + case CalpontSystemCatalog::USMALLINT: + { + fResult.uintVal = row.getUintField<2>(fInputIndex); + break; + } + + case CalpontSystemCatalog::UTINYINT: + { + fResult.uintVal = row.getUintField<1>(fInputIndex); + break; + } + + default: // treat as int64 + { + fResult.intVal = row.getUintField<8>(fInputIndex); + break; + } + } } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index ea29444a7..bf32c6490 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplecolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplecolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef SIMPLECOLUMN_H @@ -45,7 +45,6 @@ class ByteStream; */ namespace execplan { - class ParseTree; /** * @brief A class to represent a simple returned column @@ -55,329 +54,316 @@ class ParseTree; */ class SimpleColumn : public ReturnedColumn { + public: + /** + * Constructors + */ + SimpleColumn(); + SimpleColumn(const std::string& token, const uint32_t sessionID = 0); + SimpleColumn(const std::string& schema, const std::string& table, const std::string& col, + const uint32_t sessionID = 0, const int lower_case_table_names = 0); + SimpleColumn(const std::string& schema, const std::string& table, const std::string& col, + const bool isColumnStore, const uint32_t sessionID = 0, const int lower_case_table_names = 0); + SimpleColumn(const SimpleColumn& rhs, const uint32_t sessionID = 0); -public: + /** + * Destructor + */ + virtual ~SimpleColumn(); - /** - * Constructors - */ - SimpleColumn(); - SimpleColumn(const std::string& token, - const uint32_t sessionID = 0); - SimpleColumn(const std::string& schema, - const std::string& table, - const std::string& col, - const uint32_t sessionID = 0, - const int lower_case_table_names = 0); - SimpleColumn(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID = 0, - const int lower_case_table_names = 0); - SimpleColumn(const SimpleColumn& rhs, const uint32_t sessionID = 0); + /** + * Accessor Methods + */ + inline const std::string& schemaName() const + { + return fSchemaName; + } - /** - * Destructor - */ - virtual ~SimpleColumn(); + inline void schemaName(const std::string& schemaName, int lower_case_table_names = 0) + { + fSchemaName = schemaName; + if (lower_case_table_names) + boost::algorithm::to_lower(fSchemaName); + } - /** - * Accessor Methods - */ - inline const std::string& schemaName() const + inline const std::string& tableName() const + { + return fTableName; + } + + inline void tableName(const std::string& tableName, int lower_case_table_names = 0) + { + fTableName = tableName; + if (lower_case_table_names) + boost::algorithm::to_lower(fTableName); + } + + inline const std::string& columnName() const + { + return fColumnName; + } + + inline void columnName(const std::string& columnName) + { + fColumnName = columnName; + } + + inline const CalpontSystemCatalog::OID& oid() const + { + return fOid; + } + + inline void oid(const CalpontSystemCatalog::OID& oid) + { + fOid = oid; + } + + virtual const std::string data() const; + virtual void data(const std::string data) + { + fData = data; + } + + inline const std::string& tableAlias() const + { + return fTableAlias; + } + inline void tableAlias(const std::string& tableAlias, int lower_case_table_names = 0) + { + fTableAlias = tableAlias; + if (lower_case_table_names) + boost::algorithm::to_lower(fTableAlias); + } + inline const std::string& indexName() const + { + return fIndexName; + } + inline void indexName(const std::string& indexName) + { + fIndexName = indexName; + } + inline const std::string& viewName() const + { + return fViewName; + } + inline void viewName(const std::string& viewName, int lower_case_table_names = 0) + { + fViewName = viewName; + if (lower_case_table_names) + boost::algorithm::to_lower(fViewName); + } + inline const std::string& timeZone() const + { + return fTimeZone; + } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + inline bool isColumnStore() const + { + return fisColumnStore; + } + inline void isColumnStore(const bool isColumnStore) + { + fisColumnStore = isColumnStore; + } + + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual SimpleColumn* clone() const + { + return new SimpleColumn(*this); + } + /** + * Overloaded assignment operator + */ + SimpleColumn& operator=(const SimpleColumn& rhs); + + /** + * The serialize interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + + virtual const std::string toString() const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const SimpleColumn& t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; + + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const SimpleColumn& t) const; + + /** @brief check if this column is the same as the argument */ + virtual bool sameColumn(const ReturnedColumn* rc) const; + + /** @brief return column type of this column (could be of any engine type) */ + const CalpontSystemCatalog::ColType& colType() const + { + return fResultType; + } + + /** @brief set the column's OID from the syscat */ + void setOID(); + + virtual bool hasWindowFunc() + { + return false; + } + + void setDerivedTable(); + + /** + * Return the tableAlias name of the table that the column arguments belong to. + * + * @param TableAliasName that will be set in the function + * @return true, if all arguments belong to one table + * false, if multiple tables are involved in the function + */ + virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan); + + protected: + /** + * Fields + */ + std::string fSchemaName; /// schema name + std::string fTableName; /// table name + std::string fColumnName; /// column name + CalpontSystemCatalog::OID fOid; /// column TCN number + std::string fTableAlias; /// table alias + std::string fData; + /// index name. for oracle use. deprecated + std::string fIndexName; + // if belong to view, view name is non-empty + std::string fViewName; + std::string fTimeZone; + bool fisColumnStore; + + /** @brief parse SimpleColumn text + * + * parse the incomming sql text to construct a SimpleColumn object. + * used by the constructor. + */ + void parse(const std::string& sql); + + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + virtual void evaluate(rowgroup::Row& row, bool& isNull); + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getBoolVal(); + } + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getStrVal(fTimeZone); + } + + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getIntVal(); + } + + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getUintVal(); + } + + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getFloatVal(); + } + + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDoubleVal(); + } + + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getLongDoubleVal(); + } + + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + + // @bug5736, double type with precision -1 indicates that this type is for decimal math, + // the original decimal scale is stored in scale field, which is no use for double. + if (fResultType.precision == -1) { - return fSchemaName; + if (fResultType.colDataType == CalpontSystemCatalog::DOUBLE) + { + IDB_Decimal rv((int64_t)(TreeNode::getDoubleVal() * IDB_pow[fResultType.scale]), fResultType.scale, + 15); + return rv; + } + else if (fResultType.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + IDB_Decimal rv((int64_t)(TreeNode::getLongDoubleVal() * IDB_pow[fResultType.scale]), + fResultType.scale, fResultType.precision); + return rv; + } } - inline void schemaName(const std::string& schemaName, int lower_case_table_names = 0) - { - fSchemaName = schemaName; - if (lower_case_table_names) - boost::algorithm::to_lower(fSchemaName); - } + return TreeNode::getDecimalVal(); + } - inline const std::string& tableName() const - { - return fTableName; - } + inline int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDateIntVal(); + } - inline void tableName(const std::string& tableName, int lower_case_table_names = 0) - { - fTableName = tableName; - if (lower_case_table_names) - boost::algorithm::to_lower(fTableName); - } + inline int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDatetimeIntVal(); + } - inline const std::string& columnName() const - { - return fColumnName; - } - - inline void columnName(const std::string& columnName) - { - fColumnName = columnName; - } - - inline const CalpontSystemCatalog::OID& oid() const - { - return fOid; - } - - inline void oid (const CalpontSystemCatalog::OID& oid) - { - fOid = oid; - } - - virtual const std::string data() const; - virtual void data(const std::string data) - { - fData = data; - } - - inline const std::string& tableAlias() const - { - return fTableAlias; - } - inline void tableAlias(const std::string& tableAlias, int lower_case_table_names = 0) - { - fTableAlias = tableAlias; - if (lower_case_table_names) - boost::algorithm::to_lower(fTableAlias); - } - inline const std::string& indexName() const - { - return fIndexName; - } - inline void indexName(const std::string& indexName) - { - fIndexName = indexName; - } - inline const std::string& viewName() const - { - return fViewName; - } - inline void viewName(const std::string& viewName, int lower_case_table_names = 0) - { - fViewName = viewName; - if (lower_case_table_names) - boost::algorithm::to_lower(fViewName); - } - inline const std::string& timeZone() const - { - return fTimeZone; - } - inline void timeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } - inline bool isColumnStore() const - { - return fisColumnStore; - } - inline void isColumnStore(const bool isColumnStore) - { - fisColumnStore = isColumnStore; - } - - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual SimpleColumn* clone() const - { - return new SimpleColumn (*this); - } - /** - * Overloaded assignment operator - */ - SimpleColumn& operator=(const SimpleColumn& rhs); - - /** - * The serialize interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - - virtual const std::string toString() const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const SimpleColumn& t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const SimpleColumn& t) const; - - /** @brief check if this column is the same as the argument */ - virtual bool sameColumn(const ReturnedColumn* rc) const; - - /** @brief return column type of this column (could be of any engine type) */ - const CalpontSystemCatalog::ColType& colType() const - { - return fResultType; - } - - /** @brief set the column's OID from the syscat */ - void setOID(); - - virtual bool hasWindowFunc() - { - return false; - } - - void setDerivedTable(); - - /** - * Return the tableAlias name of the table that the column arguments belong to. - * - * @param TableAliasName that will be set in the function - * @return true, if all arguments belong to one table - * false, if multiple tables are involved in the function - */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan); - -protected: - /** - * Fields - */ - std::string fSchemaName; /// schema name - std::string fTableName; /// table name - std::string fColumnName; /// column name - CalpontSystemCatalog::OID fOid; /// column TCN number - std::string fTableAlias; /// table alias - std::string fData; - /// index name. for oracle use. deprecated - std::string fIndexName; - // if belong to view, view name is non-empty - std::string fViewName; - std::string fTimeZone; - bool fisColumnStore; - - /** @brief parse SimpleColumn text - * - * parse the incomming sql text to construct a SimpleColumn object. - * used by the constructor. - */ - void parse(const std::string& sql); - - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - virtual void evaluate(rowgroup::Row& row, bool& isNull); - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getBoolVal(); - } - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getStrVal(fTimeZone); - } - - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getIntVal(); - } - - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getUintVal(); - } - - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getFloatVal(); - } - - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDoubleVal(); - } - - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getLongDoubleVal(); - } - - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - - // @bug5736, double type with precision -1 indicates that this type is for decimal math, - // the original decimal scale is stored in scale field, which is no use for double. - if (fResultType.precision == -1) - { - if (fResultType.colDataType == CalpontSystemCatalog::DOUBLE) - { - IDB_Decimal rv( - (int64_t)(TreeNode::getDoubleVal() * IDB_pow[fResultType.scale]), - fResultType.scale, 15); - return rv; - } - else if (fResultType.colDataType == CalpontSystemCatalog::LONGDOUBLE) - { - IDB_Decimal rv ( - (int64_t)(TreeNode::getLongDoubleVal() * IDB_pow[fResultType.scale]), - fResultType.scale, fResultType.precision); - return rv; - } - } - - return TreeNode::getDecimalVal(); - } - - inline int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDateIntVal(); - } - - inline int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); - } - - inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getTimestampIntVal(); - } - - inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getTimeIntVal(); - } + inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimestampIntVal(); + } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } }; typedef boost::shared_ptr SSC; @@ -393,6 +379,5 @@ std::ostream& operator<<(std::ostream& output, const SimpleColumn& rhs); void getSimpleCols(ParseTree* n, void* obj); ParseTree* replaceRefCol(ParseTree*& n, CalpontSelectExecutionPlan::ReturnedColumnList&); -} -#endif //SIMPLECOLUMN_H - +} // namespace execplan +#endif // SIMPLECOLUMN_H diff --git a/dbcon/execplan/simplecolumn_decimal.h b/dbcon/execplan/simplecolumn_decimal.h index 4c806c65f..0fb70db00 100644 --- a/dbcon/execplan/simplecolumn_decimal.h +++ b/dbcon/execplan/simplecolumn_decimal.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplecolumn_decimal.h 9635 2013-06-19 21:42:30Z bwilkinson $ -* -* -***********************************************************************/ + * $Id: simplecolumn_decimal.h 9635 2013-06-19 21:42:30Z bwilkinson $ + * + * + ***********************************************************************/ /** @file */ #ifndef SIMPLECOLUMNDECIMAL_H @@ -49,224 +49,191 @@ namespace execplan template class SimpleColumn_Decimal : public SimpleColumn { + /** + * Public stuff + */ + public: + /** Constructors */ + SimpleColumn_Decimal(); + SimpleColumn_Decimal(const std::string& token, const uint32_t sessionID = 0); + SimpleColumn_Decimal(const std::string& schema, const std::string& table, const std::string& col, + const bool isColumnStore, const uint32_t sessionID = 0); + SimpleColumn_Decimal(const SimpleColumn& rhs, const uint32_t sessionID = 0); - /** - * Public stuff - */ -public: + /** Destructor */ + virtual ~SimpleColumn_Decimal() + { + } - /** Constructors */ - SimpleColumn_Decimal(); - SimpleColumn_Decimal(const std::string& token, const uint32_t sessionID = 0); - SimpleColumn_Decimal(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID = 0); - SimpleColumn_Decimal(const SimpleColumn& rhs, const uint32_t sessionID = 0); + inline virtual SimpleColumn_Decimal* clone() const + { + return new SimpleColumn_Decimal(*this); + } - /** Destructor */ - virtual ~SimpleColumn_Decimal() {} + /** Evaluate methods */ + virtual inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull); + virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull); + virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull); + virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull); + virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); + virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull); - inline virtual SimpleColumn_Decimal* clone() const - { - return new SimpleColumn_Decimal (*this); - } - - /** Evaluate methods */ - virtual inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull); - virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull); - virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull); - virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull); - virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); - virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull); - - /** The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - uint64_t fNullVal; - -private: - void setNullVal(); + /** The serialize interface */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + uint64_t fNullVal; + private: + void setNullVal(); }; -template -SimpleColumn_Decimal::SimpleColumn_Decimal(): SimpleColumn() +template +SimpleColumn_Decimal::SimpleColumn_Decimal() : SimpleColumn() { - setNullVal(); + setNullVal(); } -template -SimpleColumn_Decimal::SimpleColumn_Decimal(const std::string& token, const uint32_t sessionID): - SimpleColumn(token, sessionID) +template +SimpleColumn_Decimal::SimpleColumn_Decimal(const std::string& token, const uint32_t sessionID) + : SimpleColumn(token, sessionID) { - setNullVal(); + setNullVal(); } -template -SimpleColumn_Decimal::SimpleColumn_Decimal(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID) : - SimpleColumn(schema, table, col, isColumnStore, sessionID) +template +SimpleColumn_Decimal::SimpleColumn_Decimal(const std::string& schema, const std::string& table, + const std::string& col, const bool isColumnStore, + const uint32_t sessionID) + : SimpleColumn(schema, table, col, isColumnStore, sessionID) { - setNullVal(); + setNullVal(); } -template -SimpleColumn_Decimal::SimpleColumn_Decimal(const SimpleColumn& rhs, const uint32_t sessionID): - SimpleColumn(rhs, sessionID) +template +SimpleColumn_Decimal::SimpleColumn_Decimal(const SimpleColumn& rhs, const uint32_t sessionID) + : SimpleColumn(rhs, sessionID) { - setNullVal(); + setNullVal(); } -template +template void SimpleColumn_Decimal::setNullVal() { - switch (len) - { - case 8: - fNullVal = joblist::BIGINTNULL; - break; + switch (len) + { + case 8: fNullVal = joblist::BIGINTNULL; break; - case 4: - fNullVal = joblist::INTNULL; - break; + case 4: fNullVal = joblist::INTNULL; break; - case 2: - fNullVal = joblist::SMALLINTNULL; - break; + case 2: fNullVal = joblist::SMALLINTNULL; break; - case 1: - fNullVal = joblist::TINYINTNULL; - break; - case 16: - // TODO MCOL-641 - //fallthrough - default: - fNullVal = joblist::BIGINTNULL; - } + case 1: fNullVal = joblist::TINYINTNULL; break; + case 16: + // TODO MCOL-641 + // fallthrough + default: fNullVal = joblist::BIGINTNULL; + } } -template -inline const std::string& SimpleColumn_Decimal:: getStrVal(rowgroup::Row& row, bool& isNull) +template +inline const std::string& SimpleColumn_Decimal::getStrVal(rowgroup::Row& row, bool& isNull) { - datatypes::Decimal dec((int64_t)row.getIntField(fInputIndex), - fResultType.scale, - fResultType.precision); - fResult.strVal = dec.toString(); - return fResult.strVal; + datatypes::Decimal dec((int64_t)row.getIntField(fInputIndex), fResultType.scale, + fResultType.precision); + fResult.strVal = dec.toString(); + return fResult.strVal; } -template -inline int64_t SimpleColumn_Decimal:: getIntVal(rowgroup::Row& row, bool& isNull) +template +inline int64_t SimpleColumn_Decimal::getIntVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - // TODO: fix double division to integer division - return (int64_t)(row.getIntField(fInputIndex) / - datatypes::scaleDivisor(fResultType.scale)); + // TODO: fix double division to integer division + return (int64_t)(row.getIntField(fInputIndex) / datatypes::scaleDivisor(fResultType.scale)); } -template +template inline float SimpleColumn_Decimal::getFloatVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (row.getIntField(fInputIndex) / - datatypes::scaleDivisor(fResultType.scale)); + return (row.getIntField(fInputIndex) / datatypes::scaleDivisor(fResultType.scale)); } -template +template inline double SimpleColumn_Decimal::getDoubleVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (row.getIntField(fInputIndex) / - datatypes::scaleDivisor(fResultType.scale)); + return (row.getIntField(fInputIndex) / datatypes::scaleDivisor(fResultType.scale)); } -template +template inline long double SimpleColumn_Decimal::getLongDoubleVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (row.getIntField(fInputIndex) / - datatypes::scaleDivisor(fResultType.scale)); + return (row.getIntField(fInputIndex) / datatypes::scaleDivisor(fResultType.scale)); } -template +template inline IDB_Decimal SimpleColumn_Decimal::getDecimalVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - fResult.decimalVal.value = (int64_t)row.getIntField(fInputIndex); - fResult.decimalVal.precision = fResultType.precision; - fResult.decimalVal.scale = fResultType.scale; - return fResult.decimalVal; + fResult.decimalVal.value = (int64_t)row.getIntField(fInputIndex); + fResult.decimalVal.precision = fResultType.precision; + fResult.decimalVal.scale = fResultType.scale; + return fResult.decimalVal; } -template +template void SimpleColumn_Decimal::serialize(messageqcpp::ByteStream& b) const { - switch (len) - { - case 1: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_DECIMAL1; - break; + switch (len) + { + case 1: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_DECIMAL1; break; - case 2: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_DECIMAL2; - break; + case 2: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_DECIMAL2; break; - case 4: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_DECIMAL4; - break; + case 4: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_DECIMAL4; break; - case 8: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_DECIMAL8; - break; - // TODO MCOL-641 - // case 16: - } + case 8: + b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_DECIMAL8; + break; + // TODO MCOL-641 + // case 16: + } - SimpleColumn::serialize(b); + SimpleColumn::serialize(b); } -template +template void SimpleColumn_Decimal::unserialize(messageqcpp::ByteStream& b) { - switch (len) - { - case 1: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL1); - break; + switch (len) + { + case 1: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL1); break; - case 2: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL2); - break; + case 2: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL2); break; - case 4: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL4); - break; + case 4: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL4); break; - case 8: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL8); - break; - // TODO MCOL-641 - // case 16: - } + case 8: + ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL8); + break; + // TODO MCOL-641 + // case 16: + } - SimpleColumn::unserialize(b); + SimpleColumn::unserialize(b); } -} -#endif //SIMPLECOLUMN_INT_H - +} // namespace execplan +#endif // SIMPLECOLUMN_INT_H diff --git a/dbcon/execplan/simplecolumn_int.h b/dbcon/execplan/simplecolumn_int.h index 7e00207ca..d8a4818ad 100644 --- a/dbcon/execplan/simplecolumn_int.h +++ b/dbcon/execplan/simplecolumn_int.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplecolumn_int.h 9635 2013-06-19 21:42:30Z bwilkinson $ -* -* -***********************************************************************/ + * $Id: simplecolumn_int.h 9635 2013-06-19 21:42:30Z bwilkinson $ + * + * + ***********************************************************************/ /** @file */ #ifndef SIMPLECOLUMNINT_H @@ -48,231 +48,199 @@ namespace execplan template class SimpleColumn_INT : public SimpleColumn { + /** + * Public stuff + */ + public: + /** Constructors */ + SimpleColumn_INT(); + SimpleColumn_INT(const std::string& token, const uint32_t sessionID = 0); + SimpleColumn_INT(const std::string& schema, const std::string& table, const std::string& col, + const bool isColumnStore, const uint32_t sessionID = 0); + SimpleColumn_INT(const SimpleColumn& rhs, const uint32_t sessionID = 0); - /** - * Public stuff - */ -public: + /** Destructor */ + virtual ~SimpleColumn_INT() + { + } - /** Constructors */ - SimpleColumn_INT(); - SimpleColumn_INT(const std::string& token, const uint32_t sessionID = 0); - SimpleColumn_INT(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID = 0); - SimpleColumn_INT(const SimpleColumn& rhs, const uint32_t sessionID = 0); + inline virtual SimpleColumn_INT* clone() const + { + return new SimpleColumn_INT(*this); + } - /** Destructor */ - virtual ~SimpleColumn_INT() {} + /** Evaluate methods */ + virtual inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull); + virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull); + virtual inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull); + virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull); + virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull); + virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); + virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull); - inline virtual SimpleColumn_INT* clone() const - { - return new SimpleColumn_INT (*this); - } - - /** Evaluate methods */ - virtual inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull); - virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull); - virtual inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull); - virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull); - virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull); - virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); - virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull); - - /** The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - uint64_t fNullVal; - -private: - void setNullVal(); + /** The serialize interface */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + uint64_t fNullVal; + private: + void setNullVal(); }; -template -SimpleColumn_INT::SimpleColumn_INT(): SimpleColumn() +template +SimpleColumn_INT::SimpleColumn_INT() : SimpleColumn() { - setNullVal(); + setNullVal(); } -template -SimpleColumn_INT::SimpleColumn_INT(const std::string& token, const uint32_t sessionID): - SimpleColumn(token, sessionID) +template +SimpleColumn_INT::SimpleColumn_INT(const std::string& token, const uint32_t sessionID) + : SimpleColumn(token, sessionID) { - setNullVal(); + setNullVal(); } -template -SimpleColumn_INT::SimpleColumn_INT(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID) : - SimpleColumn(schema, table, col, isColumnStore, sessionID) +template +SimpleColumn_INT::SimpleColumn_INT(const std::string& schema, const std::string& table, + const std::string& col, const bool isColumnStore, + const uint32_t sessionID) + : SimpleColumn(schema, table, col, isColumnStore, sessionID) { - setNullVal(); + setNullVal(); } -template -SimpleColumn_INT::SimpleColumn_INT(const SimpleColumn& rhs, const uint32_t sessionID): - SimpleColumn(rhs, sessionID) +template +SimpleColumn_INT::SimpleColumn_INT(const SimpleColumn& rhs, const uint32_t sessionID) + : SimpleColumn(rhs, sessionID) { - setNullVal(); + setNullVal(); } -template +template void SimpleColumn_INT::setNullVal() { - switch (len) - { - case 8: - fNullVal = joblist::BIGINTNULL; - break; + switch (len) + { + case 8: fNullVal = joblist::BIGINTNULL; break; - case 4: - fNullVal = joblist::INTNULL; - break; + case 4: fNullVal = joblist::INTNULL; break; - case 2: - fNullVal = joblist::SMALLINTNULL; - break; + case 2: fNullVal = joblist::SMALLINTNULL; break; - case 1: - fNullVal = joblist::TINYINTNULL; - break; + case 1: fNullVal = joblist::TINYINTNULL; break; - default: - fNullVal = joblist::BIGINTNULL; - } + default: fNullVal = joblist::BIGINTNULL; + } } -template -inline const std::string& SimpleColumn_INT:: getStrVal(rowgroup::Row& row, bool& isNull) +template +inline const std::string& SimpleColumn_INT::getStrVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; - else - { + if (row.equals(fNullVal, fInputIndex)) + isNull = true; + else + { #ifndef __LP64__ - snprintf(tmp, 20, "%lld", (int64_t)row.getIntField(fInputIndex)); + snprintf(tmp, 20, "%lld", (int64_t)row.getIntField(fInputIndex)); #else - snprintf(tmp, 20, "%ld", (int64_t)row.getIntField(fInputIndex)); + snprintf(tmp, 20, "%ld", (int64_t)row.getIntField(fInputIndex)); #endif - } + } - fResult.strVal = std::string(tmp); - return fResult.strVal; + fResult.strVal = std::string(tmp); + return fResult.strVal; } -template -inline int64_t SimpleColumn_INT:: getIntVal(rowgroup::Row& row, bool& isNull) +template +inline int64_t SimpleColumn_INT::getIntVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (int64_t)row.getIntField(fInputIndex); + return (int64_t)row.getIntField(fInputIndex); } -template -inline uint64_t SimpleColumn_INT:: getUintVal(rowgroup::Row& row, bool& isNull) +template +inline uint64_t SimpleColumn_INT::getUintVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (uint64_t)row.getIntField(fInputIndex); + return (uint64_t)row.getIntField(fInputIndex); } -template +template inline float SimpleColumn_INT::getFloatVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (float)row.getIntField(fInputIndex); + return (float)row.getIntField(fInputIndex); } -template +template inline double SimpleColumn_INT::getDoubleVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (double)row.getIntField(fInputIndex); + return (double)row.getIntField(fInputIndex); } -template +template inline long double SimpleColumn_INT::getLongDoubleVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (long double)row.getIntField(fInputIndex); + return (long double)row.getIntField(fInputIndex); } -template +template inline IDB_Decimal SimpleColumn_INT::getDecimalVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - fResult.decimalVal.value = (int64_t)row.getIntField(fInputIndex); - fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; - fResult.decimalVal.scale = 0; - return fResult.decimalVal; + fResult.decimalVal.value = (int64_t)row.getIntField(fInputIndex); + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; + fResult.decimalVal.scale = 0; + return fResult.decimalVal; } -template +template void SimpleColumn_INT::serialize(messageqcpp::ByteStream& b) const { - switch (len) - { - case 1: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_INT1; - break; + switch (len) + { + case 1: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_INT1; break; - case 2: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_INT2; - break; + case 2: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_INT2; break; - case 4: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_INT4; - break; + case 4: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_INT4; break; - case 8: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_INT8; - break; - } + case 8: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_INT8; break; + } - SimpleColumn::serialize(b); + SimpleColumn::serialize(b); } -template +template void SimpleColumn_INT::unserialize(messageqcpp::ByteStream& b) { - switch (len) - { - case 1: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT1); - break; + switch (len) + { + case 1: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT1); break; - case 2: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT2); - break; + case 2: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT2); break; - case 4: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT4); - break; + case 4: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT4); break; - case 8: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT8); - break; - } + case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT8); break; + } - SimpleColumn::unserialize(b); + SimpleColumn::unserialize(b); } -} -#endif //SIMPLECOLUMN_INT_H - +} // namespace execplan +#endif // SIMPLECOLUMN_INT_H diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index 14144e847..a42b9c2ae 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplecolumn_uint.h 8536 2012-05-21 21:27:17Z dhall $ -* -* -***********************************************************************/ + * $Id: simplecolumn_uint.h 8536 2012-05-21 21:27:17Z dhall $ + * + * + ***********************************************************************/ /** @file */ #ifndef SIMPLECOLUMNUINT_H @@ -48,239 +48,211 @@ namespace execplan template class SimpleColumn_UINT : public SimpleColumn { + /** + * Public stuff + */ + public: + /** Constructors */ + SimpleColumn_UINT(); + SimpleColumn_UINT(const std::string& token, const uint32_t sessionID = 0); + SimpleColumn_UINT(const std::string& schema, const std::string& table, const std::string& col, + const bool isColumnStore, const uint32_t sessionID = 0); + SimpleColumn_UINT(const SimpleColumn& rhs, const uint32_t sessionID = 0); - /** - * Public stuff - */ -public: + /** Destructor */ + virtual ~SimpleColumn_UINT() + { + } - /** Constructors */ - SimpleColumn_UINT(); - SimpleColumn_UINT(const std::string& token, const uint32_t sessionID = 0); - SimpleColumn_UINT(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID = 0); - SimpleColumn_UINT(const SimpleColumn& rhs, const uint32_t sessionID = 0); + inline virtual SimpleColumn_UINT* clone() const + { + return new SimpleColumn_UINT(*this); + } - /** Destructor */ - virtual ~SimpleColumn_UINT() {} + /** Evaluate methods */ + virtual inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull); + virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull); + virtual inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull); + virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull); + virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull); + virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); + virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull); - inline virtual SimpleColumn_UINT* clone() const - { - return new SimpleColumn_UINT (*this); - } - - /** Evaluate methods */ - virtual inline const std::string& getStrVal(rowgroup::Row& row, bool& isNull); - virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull); - virtual inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull); - virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull); - virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull); - virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); - virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull); - - /** The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); - uint64_t fNullVal; - -private: - void setNullVal(); + /** The serialize interface */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); + uint64_t fNullVal; + private: + void setNullVal(); }; -template -SimpleColumn_UINT::SimpleColumn_UINT(): SimpleColumn() +template +SimpleColumn_UINT::SimpleColumn_UINT() : SimpleColumn() { - setNullVal(); + setNullVal(); } -template -SimpleColumn_UINT::SimpleColumn_UINT(const std::string& token, const uint32_t sessionID): - SimpleColumn(token, sessionID) +template +SimpleColumn_UINT::SimpleColumn_UINT(const std::string& token, const uint32_t sessionID) + : SimpleColumn(token, sessionID) { - setNullVal(); + setNullVal(); } -template -SimpleColumn_UINT::SimpleColumn_UINT(const std::string& schema, - const std::string& table, - const std::string& col, - const bool isColumnStore, - const uint32_t sessionID) : - SimpleColumn(schema, table, col, isColumnStore, sessionID) +template +SimpleColumn_UINT::SimpleColumn_UINT(const std::string& schema, const std::string& table, + const std::string& col, const bool isColumnStore, + const uint32_t sessionID) + : SimpleColumn(schema, table, col, isColumnStore, sessionID) { - setNullVal(); + setNullVal(); } -template -SimpleColumn_UINT::SimpleColumn_UINT(const SimpleColumn& rhs, const uint32_t sessionID): - SimpleColumn(rhs, sessionID) +template +SimpleColumn_UINT::SimpleColumn_UINT(const SimpleColumn& rhs, const uint32_t sessionID) + : SimpleColumn(rhs, sessionID) { - setNullVal(); + setNullVal(); } -template +template void SimpleColumn_UINT::setNullVal() { - switch (len) - { - case 8: - fNullVal = joblist::UBIGINTNULL; - break; + switch (len) + { + case 8: fNullVal = joblist::UBIGINTNULL; break; - case 4: - fNullVal = joblist::UINTNULL; - break; + case 4: fNullVal = joblist::UINTNULL; break; - case 2: - fNullVal = joblist::USMALLINTNULL; - break; + case 2: fNullVal = joblist::USMALLINTNULL; break; - case 1: - fNullVal = joblist::UTINYINTNULL; - break; - // TODO MCOL-641 - case 16: - //fallthrough - default: - fNullVal = joblist::UBIGINTNULL; - } + case 1: fNullVal = joblist::UTINYINTNULL; break; + // TODO MCOL-641 + case 16: + // fallthrough + default: fNullVal = joblist::UBIGINTNULL; + } } -template -inline const std::string& SimpleColumn_UINT:: getStrVal(rowgroup::Row& row, bool& isNull) +template +inline const std::string& SimpleColumn_UINT::getStrVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; - else - { + if (row.equals(fNullVal, fInputIndex)) + isNull = true; + else + { #ifndef __LP64__ - snprintf(tmp, 21, "%llu", row.getUintField(fInputIndex)); + snprintf(tmp, 21, "%llu", row.getUintField(fInputIndex)); #else - snprintf(tmp, 21, "%lu", row.getUintField(fInputIndex)); + snprintf(tmp, 21, "%lu", row.getUintField(fInputIndex)); #endif - } + } - fResult.strVal = std::string(tmp); - return fResult.strVal; + fResult.strVal = std::string(tmp); + return fResult.strVal; } -template -inline int64_t SimpleColumn_UINT:: getIntVal(rowgroup::Row& row, bool& isNull) +template +inline int64_t SimpleColumn_UINT::getIntVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (int64_t)row.getUintField(fInputIndex); + return (int64_t)row.getUintField(fInputIndex); } -template -inline uint64_t SimpleColumn_UINT:: getUintVal(rowgroup::Row& row, bool& isNull) +template +inline uint64_t SimpleColumn_UINT::getUintVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (uint64_t)row.getUintField(fInputIndex); + return (uint64_t)row.getUintField(fInputIndex); } -template +template inline float SimpleColumn_UINT::getFloatVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (float)row.getUintField(fInputIndex); + return (float)row.getUintField(fInputIndex); } -template +template inline double SimpleColumn_UINT::getDoubleVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (double)row.getUintField(fInputIndex); + return (double)row.getUintField(fInputIndex); } -template +template inline long double SimpleColumn_UINT::getLongDoubleVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - return (long double)row.getUintField(fInputIndex); + return (long double)row.getUintField(fInputIndex); } -template +template inline IDB_Decimal SimpleColumn_UINT::getDecimalVal(rowgroup::Row& row, bool& isNull) { - if (row.equals(fNullVal, fInputIndex)) - isNull = true; + if (row.equals(fNullVal, fInputIndex)) + isNull = true; - fResult.decimalVal.value = (uint64_t)row.getUintField(fInputIndex); - fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; - fResult.decimalVal.scale = 0; - return fResult.decimalVal; + fResult.decimalVal.value = (uint64_t)row.getUintField(fInputIndex); + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; + fResult.decimalVal.scale = 0; + return fResult.decimalVal; } -template +template void SimpleColumn_UINT::serialize(messageqcpp::ByteStream& b) const { - switch (len) - { - case 1: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_UINT1; - break; + switch (len) + { + case 1: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_UINT1; break; - case 2: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_UINT2; - break; + case 2: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_UINT2; break; - case 4: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_UINT4; - break; + case 4: b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_UINT4; break; - case 8: - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_UINT8; - break; + case 8: + b << (ObjectReader::id_t)ObjectReader::SIMPLECOLUMN_UINT8; + break; - // TODO MCOL-641 - // case 16: - } + // TODO MCOL-641 + // case 16: + } - SimpleColumn::serialize(b); + SimpleColumn::serialize(b); } -template +template void SimpleColumn_UINT::unserialize(messageqcpp::ByteStream& b) { - switch (len) - { - case 1: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT1); - break; + switch (len) + { + case 1: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT1); break; - case 2: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT2); - break; + case 2: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT2); break; - case 4: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT4); - break; + case 4: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT4); break; - case 8: - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT8); - break; + case 8: + ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT8); + break; - // TODO MCOL-641 - // case 16: - } + // TODO MCOL-641 + // case 16: + } - SimpleColumn::unserialize(b); + SimpleColumn::unserialize(b); } -} -#endif //SIMPLECOLUMN_INT_H - +} // namespace execplan +#endif // SIMPLECOLUMN_INT_H diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 37ebc3f38..ed1459b25 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplefilter.cpp 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplefilter.cpp 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ #include #include #include @@ -42,123 +42,120 @@ using namespace std; namespace execplan { - /** * Constructors/Destructors */ -SimpleFilter::SimpleFilter(): - fLhs(0), - fRhs(0), - fIndexFlag(NOINDEX), - fJoinFlag (EQUA) -{} +SimpleFilter::SimpleFilter() : fLhs(0), fRhs(0), fIndexFlag(NOINDEX), fJoinFlag(EQUA) +{ +} // TODO: only handled simplecolumn operands for now -SimpleFilter::SimpleFilter(const string& sql): - Filter(sql) +SimpleFilter::SimpleFilter(const string& sql) : Filter(sql) { - parse(sql); + parse(sql); } -SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const string& timeZone) : - fOp(op), fLhs(lhs), fRhs(rhs), fIndexFlag(NOINDEX), fJoinFlag(EQUA), fTimeZone(timeZone) +SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const string& timeZone) + : fOp(op), fLhs(lhs), fRhs(rhs), fIndexFlag(NOINDEX), fJoinFlag(EQUA), fTimeZone(timeZone) { - convertConstant(); + convertConstant(); } -SimpleFilter::SimpleFilter(const SimpleFilter& rhs) : - Filter(rhs), - fOp(rhs.op()), - fIndexFlag(rhs.indexFlag()), - fJoinFlag(rhs.joinFlag()), - fTimeZone(rhs.timeZone()) +SimpleFilter::SimpleFilter(const SimpleFilter& rhs) + : Filter(rhs) + , fOp(rhs.op()) + , fIndexFlag(rhs.indexFlag()) + , fJoinFlag(rhs.joinFlag()) + , fTimeZone(rhs.timeZone()) { - fLhs = rhs.lhs()->clone(); - fRhs = rhs.rhs()->clone(); + fLhs = rhs.lhs()->clone(); + fRhs = rhs.rhs()->clone(); - fSimpleColumnList.clear(); - fAggColumnList.clear(); - fWindowFunctionColumnList.clear(); + fSimpleColumnList.clear(); + fAggColumnList.clear(); + fWindowFunctionColumnList.clear(); - SimpleColumn* lsc = dynamic_cast(fLhs); - FunctionColumn* lfc = dynamic_cast(fLhs); - ArithmeticColumn* lac = dynamic_cast(fLhs); - WindowFunctionColumn* laf = dynamic_cast(fLhs); - AggregateColumn* lagc = dynamic_cast(fLhs); - SimpleColumn* rsc = dynamic_cast(fRhs); - FunctionColumn* rfc = dynamic_cast(fRhs); - ArithmeticColumn* rac = dynamic_cast(fRhs); - AggregateColumn* ragc = dynamic_cast(fRhs); - WindowFunctionColumn* raf = dynamic_cast(fRhs); + SimpleColumn* lsc = dynamic_cast(fLhs); + FunctionColumn* lfc = dynamic_cast(fLhs); + ArithmeticColumn* lac = dynamic_cast(fLhs); + WindowFunctionColumn* laf = dynamic_cast(fLhs); + AggregateColumn* lagc = dynamic_cast(fLhs); + SimpleColumn* rsc = dynamic_cast(fRhs); + FunctionColumn* rfc = dynamic_cast(fRhs); + ArithmeticColumn* rac = dynamic_cast(fRhs); + AggregateColumn* ragc = dynamic_cast(fRhs); + WindowFunctionColumn* raf = dynamic_cast(fRhs); - if (lsc) - { - fSimpleColumnList.push_back(lsc); - } - else if (lagc) - { - fAggColumnList.push_back(lagc); - } - else if (lfc) - { - fSimpleColumnList.insert(fSimpleColumnList.end(), lfc->simpleColumnList().begin(), lfc->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), lfc->aggColumnList().begin(), lfc->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), lfc->windowfunctionColumnList().begin(), lfc->windowfunctionColumnList().end()); - } - else if (lac) - { - fSimpleColumnList.insert(fSimpleColumnList.end(), lac->simpleColumnList().begin(), lac->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), lac->aggColumnList().begin(), lac->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), lac->windowfunctionColumnList().begin(), lac->windowfunctionColumnList().end()); - } - else if (laf) - { - fWindowFunctionColumnList.push_back(laf); - } + if (lsc) + { + fSimpleColumnList.push_back(lsc); + } + else if (lagc) + { + fAggColumnList.push_back(lagc); + } + else if (lfc) + { + fSimpleColumnList.insert(fSimpleColumnList.end(), lfc->simpleColumnList().begin(), + lfc->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), lfc->aggColumnList().begin(), lfc->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), lfc->windowfunctionColumnList().begin(), + lfc->windowfunctionColumnList().end()); + } + else if (lac) + { + fSimpleColumnList.insert(fSimpleColumnList.end(), lac->simpleColumnList().begin(), + lac->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), lac->aggColumnList().begin(), lac->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), lac->windowfunctionColumnList().begin(), + lac->windowfunctionColumnList().end()); + } + else if (laf) + { + fWindowFunctionColumnList.push_back(laf); + } - if (rsc) - { - fSimpleColumnList.push_back(rsc); - } - else if (ragc) - { - fAggColumnList.push_back(ragc); - } - else if (rfc) - { - fSimpleColumnList.insert - (fSimpleColumnList.end(), rfc->simpleColumnList().begin(), rfc->simpleColumnList().end()); - fAggColumnList.insert - (fAggColumnList.end(), rfc->aggColumnList().begin(), rfc->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), rfc->windowfunctionColumnList().begin(), rfc->windowfunctionColumnList().end()); - } - else if (rac) - { - fSimpleColumnList.insert(fSimpleColumnList.end(), rac->simpleColumnList().begin(), rac->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), rac->aggColumnList().begin(), rac->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), rac->windowfunctionColumnList().begin(), rac->windowfunctionColumnList().end()); - } - else if (raf) - { - fWindowFunctionColumnList.push_back(raf); - } + if (rsc) + { + fSimpleColumnList.push_back(rsc); + } + else if (ragc) + { + fAggColumnList.push_back(ragc); + } + else if (rfc) + { + fSimpleColumnList.insert(fSimpleColumnList.end(), rfc->simpleColumnList().begin(), + rfc->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), rfc->aggColumnList().begin(), rfc->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), rfc->windowfunctionColumnList().begin(), + rfc->windowfunctionColumnList().end()); + } + else if (rac) + { + fSimpleColumnList.insert(fSimpleColumnList.end(), rac->simpleColumnList().begin(), + rac->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), rac->aggColumnList().begin(), rac->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), rac->windowfunctionColumnList().begin(), + rac->windowfunctionColumnList().end()); + } + else if (raf) + { + fWindowFunctionColumnList.push_back(raf); + } } SimpleFilter::~SimpleFilter() { - //delete fOp; - if (fLhs != NULL) - delete fLhs; + // delete fOp; + if (fLhs != NULL) + delete fLhs; - if (fRhs != NULL) - delete fRhs; + if (fRhs != NULL) + delete fRhs; - fLhs = NULL; - fRhs = NULL; + fLhs = NULL; + fRhs = NULL; } /** @@ -167,658 +164,650 @@ SimpleFilter::~SimpleFilter() void SimpleFilter::lhs(ReturnedColumn* lhs) { - fLhs = lhs; + fLhs = lhs; - if (fLhs && fRhs) - convertConstant(); + if (fLhs && fRhs) + convertConstant(); } void SimpleFilter::rhs(ReturnedColumn* rhs) { - fRhs = rhs; + fRhs = rhs; - if (fLhs && fRhs) - convertConstant(); + if (fLhs && fRhs) + convertConstant(); } std::string SimpleFilter::escapeString(const std::string& input) { - std::ostringstream ss; + std::ostringstream ss; - for (std::string::const_iterator iter = input.begin(); iter != input.end(); iter++) + for (std::string::const_iterator iter = input.begin(); iter != input.end(); iter++) + { + switch (*iter) { - switch (*iter) - { - case '\\': - ss << "\\\\"; - break; + case '\\': ss << "\\\\"; break; - case '\'': - ss << "\\'"; - break; + case '\'': ss << "\\'"; break; - default: - ss << *iter; - break; - } + default: ss << *iter; break; } + } - return ss.str(); + return ss.str(); } const string SimpleFilter::data() const { - string rhs, lhs; + string rhs, lhs; - if (dynamic_cast(fRhs) && - (fRhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR || - fRhs->resultType().colDataType == CalpontSystemCatalog::CHAR || - fRhs->resultType().colDataType == CalpontSystemCatalog::BLOB || - fRhs->resultType().colDataType == CalpontSystemCatalog::TEXT || - fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || - fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || - fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) - rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; - else - rhs = fRhs->data(); + if (dynamic_cast(fRhs) && + (fRhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR || + fRhs->resultType().colDataType == CalpontSystemCatalog::CHAR || + fRhs->resultType().colDataType == CalpontSystemCatalog::BLOB || + fRhs->resultType().colDataType == CalpontSystemCatalog::TEXT || + fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || + fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) + rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; + else + rhs = fRhs->data(); - if (dynamic_cast(fLhs) && - (fLhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR || - fLhs->resultType().colDataType == CalpontSystemCatalog::CHAR || - fLhs->resultType().colDataType == CalpontSystemCatalog::BLOB || - fLhs->resultType().colDataType == CalpontSystemCatalog::TEXT || - fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || - fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || - fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || - fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) - lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; - else - lhs = fLhs->data(); + if (dynamic_cast(fLhs) && + (fLhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR || + fLhs->resultType().colDataType == CalpontSystemCatalog::CHAR || + fLhs->resultType().colDataType == CalpontSystemCatalog::BLOB || + fLhs->resultType().colDataType == CalpontSystemCatalog::TEXT || + fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || + fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) + lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; + else + lhs = fLhs->data(); - return lhs + " " + fOp->data() + " " + rhs; + return lhs + " " + fOp->data() + " " + rhs; } const string SimpleFilter::toString() const { - ostringstream output; - output << "SimpleFilter(indexflag=" << fIndexFlag; - output << " joinFlag= " << fJoinFlag; - output << " card= " << fCardinality << ")" << endl; - output << " " << *fLhs; - output << " " << *fOp; - output << " " << *fRhs; - return output.str(); + ostringstream output; + output << "SimpleFilter(indexflag=" << fIndexFlag; + output << " joinFlag= " << fJoinFlag; + output << " card= " << fCardinality << ")" << endl; + output << " " << *fLhs; + output << " " << *fOp; + output << " " << *fRhs; + return output.str(); } void SimpleFilter::parse(string sql) { - fLhs = 0; - fRhs = 0; - string delimiter[7] = {">=", "<=", "<>", "!=", "=", "<", ">"}; - string::size_type pos; + fLhs = 0; + fRhs = 0; + string delimiter[7] = {">=", "<=", "<>", "!=", "=", "<", ">"}; + string::size_type pos; - for (int i = 0; i < 7; i++) - { - pos = sql.find(delimiter[i], 0); + for (int i = 0; i < 7; i++) + { + pos = sql.find(delimiter[i], 0); - if (pos == string::npos) - continue; + if (pos == string::npos) + continue; - fOp.reset(new Operator (delimiter[i])); - string lhs = sql.substr(0, pos); + fOp.reset(new Operator(delimiter[i])); + string lhs = sql.substr(0, pos); - if (lhs.at(0) == ' ') - lhs = lhs.substr(1, pos); + if (lhs.at(0) == ' ') + lhs = lhs.substr(1, pos); - if (lhs.at(lhs.length() - 1) == ' ') - lhs = lhs.substr(0, pos - 1); + if (lhs.at(lhs.length() - 1) == ' ') + lhs = lhs.substr(0, pos - 1); - fLhs = new SimpleColumn(lhs); + fLhs = new SimpleColumn(lhs); - pos = pos + delimiter[i].length(); - string rhs = sql.substr(pos, sql.length()); + pos = pos + delimiter[i].length(); + string rhs = sql.substr(pos, sql.length()); - if (rhs.at(0) == ' ') - rhs = rhs.substr(1, pos); + if (rhs.at(0) == ' ') + rhs = rhs.substr(1, pos); - if (rhs.at(rhs.length() - 1) == ' ') - rhs = rhs.substr(0, pos - 1); + if (rhs.at(rhs.length() - 1) == ' ') + rhs = rhs.substr(0, pos - 1); - fRhs = new SimpleColumn (rhs); - break; - } + fRhs = new SimpleColumn(rhs); + break; + } - if (fLhs == NULL || fRhs == NULL) - throw runtime_error ("invalid sql for simple filter\n" ); + if (fLhs == NULL || fRhs == NULL) + throw runtime_error("invalid sql for simple filter\n"); } ostream& operator<<(ostream& output, const SimpleFilter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void SimpleFilter::serialize(messageqcpp::ByteStream& b) const { - b << static_cast(ObjectReader::SIMPLEFILTER); - Filter::serialize(b); + b << static_cast(ObjectReader::SIMPLEFILTER); + Filter::serialize(b); - if (fOp != NULL) - fOp->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fOp != NULL) + fOp->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - if (fLhs != NULL) - fLhs->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fLhs != NULL) + fLhs->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - if (fRhs != NULL) - fRhs->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fRhs != NULL) + fRhs->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); - b << static_cast(fIndexFlag); - b << static_cast(fJoinFlag); - b << fTimeZone; + b << static_cast(fIndexFlag); + b << static_cast(fJoinFlag); + b << fTimeZone; } void SimpleFilter::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::SIMPLEFILTER); + ObjectReader::checkType(b, ObjectReader::SIMPLEFILTER); - //delete fOp; - delete fLhs; - delete fRhs; - Filter::unserialize(b); - fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fLhs = dynamic_cast(ObjectReader::createTreeNode(b)); - fRhs = dynamic_cast(ObjectReader::createTreeNode(b)); - b >> reinterpret_cast(fIndexFlag); - b >> reinterpret_cast(fJoinFlag); - b >> fTimeZone; + // delete fOp; + delete fLhs; + delete fRhs; + Filter::unserialize(b); + fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fLhs = dynamic_cast(ObjectReader::createTreeNode(b)); + fRhs = dynamic_cast(ObjectReader::createTreeNode(b)); + b >> reinterpret_cast(fIndexFlag); + b >> reinterpret_cast(fJoinFlag); + b >> fTimeZone; - fSimpleColumnList.clear(); - fAggColumnList.clear(); - fWindowFunctionColumnList.clear(); + fSimpleColumnList.clear(); + fAggColumnList.clear(); + fWindowFunctionColumnList.clear(); - SimpleColumn* lsc = dynamic_cast(fLhs); - FunctionColumn* lfc = dynamic_cast(fLhs); - ArithmeticColumn* lac = dynamic_cast(fLhs); - WindowFunctionColumn* laf = dynamic_cast(fLhs); - AggregateColumn* lagc = dynamic_cast(fLhs); - SimpleColumn* rsc = dynamic_cast(fRhs); - FunctionColumn* rfc = dynamic_cast(fRhs); - ArithmeticColumn* rac = dynamic_cast(fRhs); - AggregateColumn* ragc = dynamic_cast(fRhs); - WindowFunctionColumn* raf = dynamic_cast(fRhs); + SimpleColumn* lsc = dynamic_cast(fLhs); + FunctionColumn* lfc = dynamic_cast(fLhs); + ArithmeticColumn* lac = dynamic_cast(fLhs); + WindowFunctionColumn* laf = dynamic_cast(fLhs); + AggregateColumn* lagc = dynamic_cast(fLhs); + SimpleColumn* rsc = dynamic_cast(fRhs); + FunctionColumn* rfc = dynamic_cast(fRhs); + ArithmeticColumn* rac = dynamic_cast(fRhs); + AggregateColumn* ragc = dynamic_cast(fRhs); + WindowFunctionColumn* raf = dynamic_cast(fRhs); - if (lsc) - { - fSimpleColumnList.push_back(lsc); - } - else if (lagc) - { - fAggColumnList.push_back(lagc); - } - else if (lfc) - { - lfc->setSimpleColumnList(); - fSimpleColumnList.insert(fSimpleColumnList.end(), lfc->simpleColumnList().begin(), lfc->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), lfc->aggColumnList().begin(), lfc->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), lfc->windowfunctionColumnList().begin(), lfc->windowfunctionColumnList().end()); - } - else if (lac) - { - lac->setSimpleColumnList(); - fSimpleColumnList.insert(fSimpleColumnList.end(), lac->simpleColumnList().begin(), lac->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), lac->aggColumnList().begin(), lac->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), lac->windowfunctionColumnList().begin(), lac->windowfunctionColumnList().end()); - } - else if (laf) - { - fWindowFunctionColumnList.push_back(laf); - } - - if (rsc) - { - fSimpleColumnList.push_back(rsc); - } - else if (ragc) - { - fAggColumnList.push_back(ragc); - } - else if (rfc) - { - rfc->setSimpleColumnList(); - fSimpleColumnList.insert - (fSimpleColumnList.end(), rfc->simpleColumnList().begin(), rfc->simpleColumnList().end()); - fAggColumnList.insert - (fAggColumnList.end(), rfc->aggColumnList().begin(), rfc->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), rfc->windowfunctionColumnList().begin(), rfc->windowfunctionColumnList().end()); - } - else if (rac) - { - rac->setSimpleColumnList(); - fSimpleColumnList.insert(fSimpleColumnList.end(), rac->simpleColumnList().begin(), rac->simpleColumnList().end()); - fAggColumnList.insert(fAggColumnList.end(), rac->aggColumnList().begin(), rac->aggColumnList().end()); - fWindowFunctionColumnList.insert - (fWindowFunctionColumnList.end(), rac->windowfunctionColumnList().begin(), rac->windowfunctionColumnList().end()); - } - else if (raf) - { - fWindowFunctionColumnList.push_back(raf); - } + if (lsc) + { + fSimpleColumnList.push_back(lsc); + } + else if (lagc) + { + fAggColumnList.push_back(lagc); + } + else if (lfc) + { + lfc->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), lfc->simpleColumnList().begin(), + lfc->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), lfc->aggColumnList().begin(), lfc->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), lfc->windowfunctionColumnList().begin(), + lfc->windowfunctionColumnList().end()); + } + else if (lac) + { + lac->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), lac->simpleColumnList().begin(), + lac->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), lac->aggColumnList().begin(), lac->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), lac->windowfunctionColumnList().begin(), + lac->windowfunctionColumnList().end()); + } + else if (laf) + { + fWindowFunctionColumnList.push_back(laf); + } + if (rsc) + { + fSimpleColumnList.push_back(rsc); + } + else if (ragc) + { + fAggColumnList.push_back(ragc); + } + else if (rfc) + { + rfc->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), rfc->simpleColumnList().begin(), + rfc->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), rfc->aggColumnList().begin(), rfc->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), rfc->windowfunctionColumnList().begin(), + rfc->windowfunctionColumnList().end()); + } + else if (rac) + { + rac->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), rac->simpleColumnList().begin(), + rac->simpleColumnList().end()); + fAggColumnList.insert(fAggColumnList.end(), rac->aggColumnList().begin(), rac->aggColumnList().end()); + fWindowFunctionColumnList.insert(fWindowFunctionColumnList.end(), rac->windowfunctionColumnList().begin(), + rac->windowfunctionColumnList().end()); + } + else if (raf) + { + fWindowFunctionColumnList.push_back(raf); + } } bool SimpleFilter::operator==(const SimpleFilter& t) const { - const Filter* f1, *f2; + const Filter *f1, *f2; - f1 = static_cast(this); - f2 = static_cast(&t); + f1 = static_cast(this); + f2 = static_cast(&t); - if (*f1 != *f2) - return false; + if (*f1 != *f2) + return false; - if (fOp != NULL) - { - if (*fOp != *t.fOp) - return false; - } - else if (t.fOp != NULL) - return false; + if (fOp != NULL) + { + if (*fOp != *t.fOp) + return false; + } + else if (t.fOp != NULL) + return false; - if (fLhs != NULL) - { - if (*fLhs != t.fLhs) - return false; - } - else if (t.fLhs != NULL) - return false; + if (fLhs != NULL) + { + if (*fLhs != t.fLhs) + return false; + } + else if (t.fLhs != NULL) + return false; - if (fRhs != NULL) - { - if (*fRhs != t.fRhs) - return false; - } - else if (t.fRhs != NULL) - return false; + if (fRhs != NULL) + { + if (*fRhs != t.fRhs) + return false; + } + else if (t.fRhs != NULL) + return false; - else if (fIndexFlag != t.fIndexFlag) - return false; + else if (fIndexFlag != t.fIndexFlag) + return false; - else if (fJoinFlag != t.fJoinFlag) - return false; + else if (fJoinFlag != t.fJoinFlag) + return false; - else if (fTimeZone != t.fTimeZone) - return false; + else if (fTimeZone != t.fTimeZone) + return false; - return true; + return true; } bool SimpleFilter::semanticEq(const SimpleFilter& t) const { - if (fOp != NULL) - { - if (*fOp != *t.fOp) - return false; - } - if (fLhs != NULL) - { - if (*fLhs != t.fLhs && *fLhs != *t.fRhs) - return false; - } - if (fRhs != NULL) - { - if (*fRhs != t.fRhs && *fRhs != *t.fLhs) - return false; - } - - return true; + if (fOp != NULL) + { + if (*fOp != *t.fOp) + return false; + } + if (fLhs != NULL) + { + if (*fLhs != t.fLhs && *fLhs != *t.fRhs) + return false; + } + if (fRhs != NULL) + { + if (*fRhs != t.fRhs && *fRhs != *t.fLhs) + return false; + } + + return true; } bool SimpleFilter::operator==(const TreeNode* t) const { - const SimpleFilter* o; + const SimpleFilter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool SimpleFilter::operator!=(const SimpleFilter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool SimpleFilter::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } bool SimpleFilter::pureFilter() { - if (typeid (*fLhs) == typeid(ConstantColumn) && - typeid (*fRhs) != typeid(ConstantColumn)) - { - // make sure constantCol sit on right hand side - ReturnedColumn* temp = fLhs; - fLhs = fRhs; - fRhs = temp; + if (typeid(*fLhs) == typeid(ConstantColumn) && typeid(*fRhs) != typeid(ConstantColumn)) + { + // make sure constantCol sit on right hand side + ReturnedColumn* temp = fLhs; + fLhs = fRhs; + fRhs = temp; - // also switch indexFlag - if (fIndexFlag == SimpleFilter::LEFT) fIndexFlag = SimpleFilter::RIGHT; - else if (fIndexFlag == SimpleFilter::RIGHT) fIndexFlag = SimpleFilter::LEFT; + // also switch indexFlag + if (fIndexFlag == SimpleFilter::LEFT) + fIndexFlag = SimpleFilter::RIGHT; + else if (fIndexFlag == SimpleFilter::RIGHT) + fIndexFlag = SimpleFilter::LEFT; - return true; - } + return true; + } - if (typeid (*fRhs) == typeid(ConstantColumn) && - typeid (*fLhs) != typeid(ConstantColumn)) - return true; + if (typeid(*fRhs) == typeid(ConstantColumn) && typeid(*fLhs) != typeid(ConstantColumn)) + return true; - return false; + return false; } void SimpleFilter::convertConstant() { - if (fOp->op() == OP_ISNULL || fOp->op() == OP_ISNOTNULL) - return; + if (fOp->op() == OP_ISNULL || fOp->op() == OP_ISNOTNULL) + return; - ConstantColumn* lcc = dynamic_cast(fLhs); - ConstantColumn* rcc = dynamic_cast(fRhs); + ConstantColumn* lcc = dynamic_cast(fLhs); + ConstantColumn* rcc = dynamic_cast(fRhs); - if (lcc) + if (lcc) + { + Result result = lcc->result(); + + if (fRhs->resultType().colDataType == CalpontSystemCatalog::DATE) { - Result result = lcc->result(); - - if (fRhs->resultType().colDataType == CalpontSystemCatalog::DATE) - { - if (lcc->constval().empty()) - { - lcc->constval("0000-00-00"); - result.intVal = 0; - result.strVal = lcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::dateToInt(result.strVal); - } - } - else if (fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME) - { - if (lcc->constval().empty()) - { - lcc->constval("0000-00-00 00:00:00"); - result.intVal = 0; - result.strVal = lcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); - } - } - else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) - { - if (lcc->constval().empty()) - { - lcc->constval("0000-00-00 00:00:00"); - result.intVal = 0; - result.strVal = lcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); - } - } - else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) - { - if (lcc->constval().empty()) - { - lcc->constval("00:00:00"); - result.intVal = 0; - result.strVal = lcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); - } - } - - lcc->result(result); + if (lcc->constval().empty()) + { + lcc->constval("0000-00-00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::dateToInt(result.strVal); + } + } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME) + { + if (lcc->constval().empty()) + { + lcc->constval("0000-00-00 00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); + } + } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) + { + if (lcc->constval().empty()) + { + lcc->constval("0000-00-00 00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + } + } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (lcc->constval().empty()) + { + lcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } } - if (rcc) + lcc->result(result); + } + + if (rcc) + { + Result result = rcc->result(); + + if (fLhs->resultType().colDataType == CalpontSystemCatalog::DATE) { - Result result = rcc->result(); - - if (fLhs->resultType().colDataType == CalpontSystemCatalog::DATE) - { - if (rcc->constval().empty()) - { - rcc->constval("0000-00-00"); - result.intVal = 0; - result.strVal = rcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::dateToInt(result.strVal); - } - } - else if (fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME) - { - if (rcc->constval().empty()) - { - rcc->constval("0000-00-00 00:00:00"); - result.intVal = 0; - result.strVal = rcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); - } - } - else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) - { - if (rcc->constval().empty()) - { - rcc->constval("0000-00-00 00:00:00"); - result.intVal = 0; - result.strVal = rcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); - } - } - else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) - { - if (rcc->constval().empty()) - { - rcc->constval("00:00:00"); - result.intVal = 0; - result.strVal = rcc->constval(); - } - else - { - result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); - } - } - - rcc->result(result); + if (rcc->constval().empty()) + { + rcc->constval("0000-00-00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::dateToInt(result.strVal); + } } + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME) + { + if (rcc->constval().empty()) + { + rcc->constval("0000-00-00 00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); + } + } + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) + { + if (rcc->constval().empty()) + { + rcc->constval("0000-00-00 00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + } + } + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (rcc->constval().empty()) + { + rcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } + + rcc->result(result); + } } void SimpleFilter::setDerivedTable() { - string lDerivedTable = ""; - string rDerivedTable = ""; + string lDerivedTable = ""; + string rDerivedTable = ""; - if (hasAggregate()) - return; + if (hasAggregate()) + return; - if (fLhs) - { - fLhs->setDerivedTable(); - lDerivedTable = fLhs->derivedTable(); - } - else - { - lDerivedTable = "*"; - } + if (fLhs) + { + fLhs->setDerivedTable(); + lDerivedTable = fLhs->derivedTable(); + } + else + { + lDerivedTable = "*"; + } - if (fRhs) - { - fRhs->setDerivedTable(); - rDerivedTable = fRhs->derivedTable(); - } - else - { - rDerivedTable = "*"; - } + if (fRhs) + { + fRhs->setDerivedTable(); + rDerivedTable = fRhs->derivedTable(); + } + else + { + rDerivedTable = "*"; + } - if (lDerivedTable == "*") - { - fDerivedTable = rDerivedTable; - } - else if (rDerivedTable == "*") - { - fDerivedTable = lDerivedTable; - } - else if (lDerivedTable == rDerivedTable) - { - fDerivedTable = lDerivedTable; // should be the same as rhs - } - else - { - fDerivedTable = ""; - } + if (lDerivedTable == "*") + { + fDerivedTable = rDerivedTable; + } + else if (rDerivedTable == "*") + { + fDerivedTable = lDerivedTable; + } + else if (lDerivedTable == rDerivedTable) + { + fDerivedTable = lDerivedTable; // should be the same as rhs + } + else + { + fDerivedTable = ""; + } } void SimpleFilter::replaceRealCol(CalpontSelectExecutionPlan::ReturnedColumnList& derivedColList) { - SimpleColumn* sc = NULL; + SimpleColumn* sc = NULL; - if (fLhs) + if (fLhs) + { + sc = dynamic_cast(fLhs); + + if (sc) { - sc = dynamic_cast(fLhs); - - if (sc) - { - ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone(); - delete fLhs; - fLhs = tmp; - } - else - { - fLhs->replaceRealCol(derivedColList); - } + ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone(); + delete fLhs; + fLhs = tmp; } - - if (fRhs) + else { - sc = dynamic_cast(fRhs); - - if (sc) - { - ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone(); - delete fRhs; - fRhs = tmp; - } - else - { - fRhs->replaceRealCol(derivedColList); - } + fLhs->replaceRealCol(derivedColList); } + } + + if (fRhs) + { + sc = dynamic_cast(fRhs); + + if (sc) + { + ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone(); + delete fRhs; + fRhs = tmp; + } + else + { + fRhs->replaceRealCol(derivedColList); + } + } } const std::vector& SimpleFilter::simpleColumnList() { - return fSimpleColumnList; + return fSimpleColumnList; } void SimpleFilter::setSimpleColumnList() { - SimpleColumn* lsc = dynamic_cast(fLhs); - SimpleColumn* rsc = dynamic_cast(fRhs); - fSimpleColumnList.clear(); + SimpleColumn* lsc = dynamic_cast(fLhs); + SimpleColumn* rsc = dynamic_cast(fRhs); + fSimpleColumnList.clear(); - if (lsc) - { - fSimpleColumnList.push_back(lsc); - } - else if (fLhs) - { - fLhs->setSimpleColumnList(); - fSimpleColumnList.insert - (fSimpleColumnList.end(), fLhs->simpleColumnList().begin(), fLhs->simpleColumnList().end()); - } + if (lsc) + { + fSimpleColumnList.push_back(lsc); + } + else if (fLhs) + { + fLhs->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), fLhs->simpleColumnList().begin(), + fLhs->simpleColumnList().end()); + } - if (rsc) - { - fSimpleColumnList.push_back(rsc); - } - else if (fRhs) - { - fRhs->setSimpleColumnList(); - fSimpleColumnList.insert - (fSimpleColumnList.end(), fRhs->simpleColumnList().begin(), fRhs->simpleColumnList().end()); - } + if (rsc) + { + fSimpleColumnList.push_back(rsc); + } + else if (fRhs) + { + fRhs->setSimpleColumnList(); + fSimpleColumnList.insert(fSimpleColumnList.end(), fRhs->simpleColumnList().begin(), + fRhs->simpleColumnList().end()); + } } bool SimpleFilter::hasAggregate() { - if (fAggColumnList.empty()) + if (fAggColumnList.empty()) + { + AggregateColumn* lac = dynamic_cast(fLhs); + AggregateColumn* rac = dynamic_cast(fRhs); + fAggColumnList.clear(); + + if (lac) { - AggregateColumn* lac = dynamic_cast(fLhs); - AggregateColumn* rac = dynamic_cast(fRhs); - fAggColumnList.clear(); - - if (lac) - { - fAggColumnList.insert(fAggColumnList.end(), lac); - } - - if (rac) - { - fAggColumnList.insert(fAggColumnList.end(), rac); - } - - if (fLhs) - { - if (fLhs->hasAggregate()) - fAggColumnList.insert(fAggColumnList.end(), - fLhs->aggColumnList().begin(), - fLhs->aggColumnList().end()); - } - - if (fRhs) - { - if (fRhs->hasAggregate()) - fAggColumnList.insert(fAggColumnList.end(), - fRhs->aggColumnList().begin(), - fRhs->aggColumnList().end()); - } + fAggColumnList.insert(fAggColumnList.end(), lac); } - if (!fAggColumnList.empty()) + if (rac) { - return true; + fAggColumnList.insert(fAggColumnList.end(), rac); } - return false; + if (fLhs) + { + if (fLhs->hasAggregate()) + fAggColumnList.insert(fAggColumnList.end(), fLhs->aggColumnList().begin(), + fLhs->aggColumnList().end()); + } + + if (fRhs) + { + if (fRhs->hasAggregate()) + fAggColumnList.insert(fAggColumnList.end(), fRhs->aggColumnList().begin(), + fRhs->aggColumnList().end()); + } + } + + if (!fAggColumnList.empty()) + { + return true; + } + + return false; } - -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/simplefilter.h b/dbcon/execplan/simplefilter.h index 4b3679777..f14019105 100644 --- a/dbcon/execplan/simplefilter.h +++ b/dbcon/execplan/simplefilter.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplefilter.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplefilter.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef SIMPLEFILTER_H @@ -49,241 +49,239 @@ class WindowFunctionColumn; */ class SimpleFilter : public Filter { + public: + /** index flag */ + enum IndexFlag + { + NOINDEX = 0, + LEFT, + RIGHT, + BOTH + }; -public: - /** index flag */ - enum IndexFlag - { - NOINDEX = 0, - LEFT, - RIGHT, - BOTH - }; + enum JoinFlag + { + EQUA = 0, + ANTI, + SEMI + }; - enum JoinFlag - { - EQUA = 0, - ANTI, - SEMI - }; + SimpleFilter(); + SimpleFilter(const std::string& sql); + SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const std::string& timeZone = ""); + SimpleFilter(const SimpleFilter& rhs); - SimpleFilter(); - SimpleFilter(const std::string& sql); - SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const std::string& timeZone = ""); - SimpleFilter(const SimpleFilter& rhs); + virtual ~SimpleFilter(); - virtual ~SimpleFilter(); + inline virtual SimpleFilter* clone() const + { + return new SimpleFilter(*this); + } - inline virtual SimpleFilter* clone() const - { - return new SimpleFilter (*this); - } + const SOP& op() const + { + return fOp; + } - const SOP& op() const - { - return fOp; - } + void op(const SOP& op) + { + fOp = op; + } - void op(const SOP& op) - { - fOp = op; - } + inline ReturnedColumn* lhs() const + { + return fLhs; + } - inline ReturnedColumn* lhs() const - { - return fLhs; - } + inline const std::string& timeZone() const + { + return fTimeZone; + } - inline const std::string& timeZone() const - { - return fTimeZone; - } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } - inline void timeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } + using Filter::data; + virtual const std::string data() const; - using Filter::data; - virtual const std::string data() const; + /** assign fLhs + * + * this call takes over the ownership of the input pointer. Caller needs + * to put the input pointer to 0 after the call; or to remember not to + * delete the pointer accidentally. + */ + void lhs(ReturnedColumn* lhs); - /** assign fLhs - * - * this call takes over the ownership of the input pointer. Caller needs - * to put the input pointer to 0 after the call; or to remember not to - * delete the pointer accidentally. - */ - void lhs(ReturnedColumn* lhs); + inline ReturnedColumn* rhs() const + { + return fRhs; + } - inline ReturnedColumn* rhs() const - { - return fRhs; - } + /** assign fRhs + * + * this call takes over the ownership of the pointer rhs. Called need + * to put the input pointer to 0 after the all; or remember not to + * delete the pointer accidentally. + */ + void rhs(ReturnedColumn* rhs); - /** assign fRhs - * - * this call takes over the ownership of the pointer rhs. Called need - * to put the input pointer to 0 after the all; or remember not to - * delete the pointer accidentally. - */ - void rhs(ReturnedColumn* rhs); + virtual const std::string toString() const; - virtual const std::string toString() const; + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true if every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true if every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true if every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const SimpleFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true if every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const SimpleFilter& t) const; + /** @brief Do a semantic equivalence test + * + * Do a semantic equivalence test. + * @return true if filter operation are the same and + * the sets of arguments are the same; false otherwise + */ + bool semanticEq(const SimpleFilter& t) const; - /** @brief Do a semantic equivalence test - * - * Do a semantic equivalence test. - * @return true if filter operation are the same and - * the sets of arguments are the same; false otherwise - */ - bool semanticEq(const SimpleFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false if every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false if every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false if every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const SimpleFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false if every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const SimpleFilter& t) const; + /** @brief determin if this simple filter is a pure filter (col/const) + */ + bool pureFilter(); - /** @brief determin if this simple filter is a pure filter (col/const) - */ - bool pureFilter(); + /** @brief test if this filter can be combined with the argument filter + * This is for operation combine optimization + * @param f the filter that this fiter tries to combine with + * @param op the operator that connects the two filters. if one or both of the + * two filters is constantFilter, need to make sure operator is consistant. + * @return a filter(constantfilter) if successfully combined. otherwise + * return NULL + * For Oracle Frontend use. Deprecated now. + */ + // virtual Filter* combinable(Filter* f, Operator* op); - /** @brief test if this filter can be combined with the argument filter - * This is for operation combine optimization - * @param f the filter that this fiter tries to combine with - * @param op the operator that connects the two filters. if one or both of the - * two filters is constantFilter, need to make sure operator is consistant. - * @return a filter(constantfilter) if successfully combined. otherwise - * return NULL - * For Oracle Frontend use. Deprecated now. - */ - //virtual Filter* combinable(Filter* f, Operator* op); + /** @brief assign indexflag to indicate which side col is index */ + void indexFlag(const int indexFlag) + { + fIndexFlag = indexFlag; + } + int indexFlag() const + { + return fIndexFlag; + } - /** @brief assign indexflag to indicate which side col is index */ - void indexFlag (const int indexFlag) - { - fIndexFlag = indexFlag; - } - int indexFlag() const - { - return fIndexFlag; - } + /** @brief assign joinflag to indicate which join type */ + void joinFlag(const int joinFlag) + { + fJoinFlag = joinFlag; + } + int joinFlag() const + { + return fJoinFlag; + } - /** @brief assign joinflag to indicate which join type */ - void joinFlag (const int joinFlag) - { - fJoinFlag = joinFlag; - } - int joinFlag() const - { - return fJoinFlag; - } + /** @brief this function is called by the connector to set constant values according to the compare type */ + void convertConstant(); - /** @brief this function is called by the connector to set constant values according to the compare type */ - void convertConstant(); + void setDerivedTable(); - void setDerivedTable(); + virtual void replaceRealCol(std::vector&); - virtual void replaceRealCol(std::vector&); + static std::string escapeString(const std::string& input); - static std::string escapeString(const std::string& input); + private: + SOP fOp; /// operator + ReturnedColumn* fLhs; /// left operand + ReturnedColumn* fRhs; /// right operand + int fIndexFlag; /// which side col is index + int fJoinFlag; /// hash join type + std::string fTimeZone; -private: - SOP fOp; /// operator - ReturnedColumn* fLhs; /// left operand - ReturnedColumn* fRhs; /// right operand - int fIndexFlag; /// which side col is index - int fJoinFlag; /// hash join type - std::string fTimeZone; + void parse(std::string); - void parse (std::string); + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull); + inline virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull); + inline virtual double getDoubleVal(rowgroup::Row& row, bool& isNull); + inline virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull); - inline virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull); - inline virtual double getDoubleVal(rowgroup::Row& row, bool& isNull); - inline virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull); + // get all simple columns involved in this column + const std::vector& simpleColumnList(); + // walk through the simple filter operands to re-populate fSimpleColumnList + void setSimpleColumnList(); + // walk through the simple filter operands to check existence of aggregate + bool hasAggregate(); - // get all simple columns involved in this column - const std::vector& simpleColumnList(); - // walk through the simple filter operands to re-populate fSimpleColumnList - void setSimpleColumnList(); - // walk through the simple filter operands to check existence of aggregate - bool hasAggregate(); + // get all aggregate columns involved in this column + const std::vector& aggColumnList() const + { + return fAggColumnList; + } - // get all aggregate columns involved in this column - const std::vector& aggColumnList() const - { - return fAggColumnList; - } + // get all window function columns involved in this column + const std::vector& windowfunctionColumnList() const + { + return fWindowFunctionColumnList; + } - // get all window function columns involved in this column - const std::vector& windowfunctionColumnList() const - { - return fWindowFunctionColumnList; - } - -private: - std::vector fSimpleColumnList; - std::vector fAggColumnList; - std::vector fWindowFunctionColumnList; + private: + std::vector fSimpleColumnList; + std::vector fAggColumnList; + std::vector fWindowFunctionColumnList; }; inline bool SimpleFilter::getBoolVal(rowgroup::Row& row, bool& isNull) { - return (reinterpret_cast(fOp.get())->getBoolVal(row, isNull, fLhs, fRhs)); + return (reinterpret_cast(fOp.get())->getBoolVal(row, isNull, fLhs, fRhs)); } inline int64_t SimpleFilter::getIntVal(rowgroup::Row& row, bool& isNull) { - return (reinterpret_cast(fOp.get())->getBoolVal(row, isNull, fLhs, fRhs) ? 1 : 0); + return (reinterpret_cast(fOp.get())->getBoolVal(row, isNull, fLhs, fRhs) ? 1 : 0); } inline double SimpleFilter::getDoubleVal(rowgroup::Row& row, bool& isNull) { - return getIntVal(row, isNull); + return getIntVal(row, isNull); } inline long double SimpleFilter::getLongDoubleVal(rowgroup::Row& row, bool& isNull) { - return getIntVal(row, isNull); + return getIntVal(row, isNull); } typedef boost::shared_ptr SSFP; std::ostream& operator<<(std::ostream& output, const SimpleFilter& rhs); -} -#endif //SIMPLEFILTER_H - +} // namespace execplan +#endif // SIMPLEFILTER_H diff --git a/dbcon/execplan/simplescalarfilter.cpp b/dbcon/execplan/simplescalarfilter.cpp index 584222428..f76e614ab 100644 --- a/dbcon/execplan/simplescalarfilter.cpp +++ b/dbcon/execplan/simplescalarfilter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplescalarfilter.cpp 6310 2010-03-04 19:46:18Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplescalarfilter.cpp 6310 2010-03-04 19:46:18Z zzhu $ + * + * + ***********************************************************************/ #include #include #include @@ -35,158 +35,153 @@ namespace execplan * Constructors/Destructors */ SimpleScalarFilter::SimpleScalarFilter() -{} +{ +} -SimpleScalarFilter::SimpleScalarFilter(const vector& cols, - const SOP& op, - SCSEP& sub) : - fCols(cols), - fOp(op), - fSub(sub), - fData("simple scalar") -{} +SimpleScalarFilter::SimpleScalarFilter(const vector& cols, const SOP& op, SCSEP& sub) + : fCols(cols), fOp(op), fSub(sub), fData("simple scalar") +{ +} -SimpleScalarFilter::SimpleScalarFilter(const SimpleScalarFilter& rhs): - Filter(rhs), - fCols (rhs.fCols), - fOp (rhs.fOp), - fSub (rhs.fSub), - fData (rhs.fData) -{} +SimpleScalarFilter::SimpleScalarFilter(const SimpleScalarFilter& rhs) + : Filter(rhs), fCols(rhs.fCols), fOp(rhs.fOp), fSub(rhs.fSub), fData(rhs.fData) +{ +} SimpleScalarFilter::~SimpleScalarFilter() -{} +{ +} const string SimpleScalarFilter::toString() const { - ostringstream oss; - oss << "SimpleScalarFilter" << endl; + ostringstream oss; + oss << "SimpleScalarFilter" << endl; - for (uint32_t i = 0; i < fCols.size(); i++) - oss << fCols[i]->toString(); + for (uint32_t i = 0; i < fCols.size(); i++) + oss << fCols[i]->toString(); - oss << fOp->toString() << endl; - oss << *(fSub.get()); - return oss.str(); + oss << fOp->toString() << endl; + oss << *(fSub.get()); + return oss.str(); } ostream& operator<<(ostream& output, const SimpleScalarFilter& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void SimpleScalarFilter::serialize(messageqcpp::ByteStream& b) const { - b << static_cast(ObjectReader::SIMPLESCALARFILTER); - Filter::serialize(b); - b << static_cast(fCols.size()); + b << static_cast(ObjectReader::SIMPLESCALARFILTER); + Filter::serialize(b); + b << static_cast(fCols.size()); - for (uint32_t i = 0; i < fCols.size(); i++) - { - if (fCols[i] != NULL) - fCols[i]->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); - } - - if (fOp != NULL) - fOp->serialize(b); + for (uint32_t i = 0; i < fCols.size(); i++) + { + if (fCols[i] != NULL) + fCols[i]->serialize(b); else - b << static_cast(ObjectReader::NULL_CLASS); + b << static_cast(ObjectReader::NULL_CLASS); + } - if (fSub.get() != NULL) - fSub->serialize(b); - else - b << static_cast(ObjectReader::NULL_CLASS); + if (fOp != NULL) + fOp->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); + + if (fSub.get() != NULL) + fSub->serialize(b); + else + b << static_cast(ObjectReader::NULL_CLASS); } void SimpleScalarFilter::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::SIMPLESCALARFILTER); + ObjectReader::checkType(b, ObjectReader::SIMPLESCALARFILTER); - Filter::unserialize(b); - uint32_t size; - b >> size; - fCols.clear(); - SRCP srcp; + Filter::unserialize(b); + uint32_t size; + b >> size; + fCols.clear(); + SRCP srcp; - for (uint32_t i = 0; i < size; i++) - { - srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fCols.push_back(srcp); - } + for (uint32_t i = 0; i < size; i++) + { + srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fCols.push_back(srcp); + } - fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fSub.reset(dynamic_cast(ObjectReader::createExecutionPlan(b))); + fOp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fSub.reset(dynamic_cast(ObjectReader::createExecutionPlan(b))); } bool SimpleScalarFilter::operator==(const SimpleScalarFilter& t) const { - const Filter* f1, *f2; + const Filter *f1, *f2; - f1 = static_cast(this); - f2 = static_cast(&t); + f1 = static_cast(this); + f2 = static_cast(&t); - if (*f1 != *f2) - return false; + if (*f1 != *f2) + return false; - if (fCols.size() != t.fCols.size()) - return false; + if (fCols.size() != t.fCols.size()) + return false; - for (uint32_t i = 0; i < fCols.size(); i++) + for (uint32_t i = 0; i < fCols.size(); i++) + { + if (fCols[i].get() != NULL) { - if (fCols[i].get() != NULL) - { - if (*(fCols[i].get()) != *(t.fCols[i]).get()) - return false; - } - else if (t.fCols[i].get() != NULL) - return false; - } - - if (fOp != NULL) - { - if (*fOp != *t.fOp) - return false; - } - else if (t.fOp != NULL) + if (*(fCols[i].get()) != *(t.fCols[i]).get()) return false; - - if (fSub != NULL) - { - if (*fSub != t.fSub.get()) - return false; } - else if (t.fSub != NULL) - return false; + else if (t.fCols[i].get() != NULL) + return false; + } - if (fData != t.fData) - return false; + if (fOp != NULL) + { + if (*fOp != *t.fOp) + return false; + } + else if (t.fOp != NULL) + return false; - return true; + if (fSub != NULL) + { + if (*fSub != t.fSub.get()) + return false; + } + else if (t.fSub != NULL) + return false; + + if (fData != t.fData) + return false; + + return true; } bool SimpleScalarFilter::operator==(const TreeNode* t) const { - const SimpleScalarFilter* o; + const SimpleScalarFilter* o; - o = dynamic_cast(t); + o = dynamic_cast(t); - if (o == NULL) - return false; + if (o == NULL) + return false; - return *this == *o; + return *this == *o; } bool SimpleScalarFilter::operator!=(const SimpleScalarFilter& t) const { - return (!(*this == t)); + return (!(*this == t)); } bool SimpleScalarFilter::operator!=(const TreeNode* t) const { - return (!(*this == t)); + return (!(*this == t)); } -} +} // namespace execplan diff --git a/dbcon/execplan/simplescalarfilter.h b/dbcon/execplan/simplescalarfilter.h index 8d1a7b5cb..75578c40e 100644 --- a/dbcon/execplan/simplescalarfilter.h +++ b/dbcon/execplan/simplescalarfilter.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: simplescalarfilter.h 6366 2010-03-15 15:07:01Z zzhu $ -* -* -***********************************************************************/ + * $Id: simplescalarfilter.h 6366 2010-03-15 15:07:01Z zzhu $ + * + * + ***********************************************************************/ /** @file */ #ifndef SIMPLESCALARFILTER_H @@ -46,128 +46,126 @@ namespace execplan */ class SimpleScalarFilter : public Filter { - /** - * Public stuff - */ -public: + /** + * Public stuff + */ + public: + /** + * Constructors + */ + SimpleScalarFilter(); + SimpleScalarFilter(const SimpleScalarFilter& rhs); - /** - * Constructors - */ - SimpleScalarFilter(); - SimpleScalarFilter(const SimpleScalarFilter& rhs); + /** Constructors + * + * pass all parts in ctor + * @note SimpleFilter takes ownership of all these pointers + */ + SimpleScalarFilter(const std::vector& cols, const SOP& op, SCSEP& sub); - /** Constructors - * - * pass all parts in ctor - * @note SimpleFilter takes ownership of all these pointers - */ - SimpleScalarFilter(const std::vector& cols, const SOP& op, SCSEP& sub); + /** + * Destructors + */ + virtual ~SimpleScalarFilter(); - /** - * Destructors - */ - virtual ~SimpleScalarFilter(); + /** + * Accessor Methods + */ + inline const std::vector& cols() const + { + return fCols; + } - /** - * Accessor Methods - */ - inline const std::vector& cols() const - { - return fCols; - } + void cols(const std::vector& cols) + { + fCols = cols; + } - void cols(const std::vector& cols) - { - fCols = cols; - } + inline const SOP& op() const + { + return fOp; + } - inline const SOP& op() const - { - return fOp; - } + inline void op(const SOP& op) + { + fOp = op; + } - inline void op (const SOP& op) - { - fOp = op; - } + inline const SCSEP& sub() const + { + return fSub; + } - inline const SCSEP& sub() const - { - return fSub; - } + inline void sub(SCSEP& sub) + { + fSub = sub; + } - inline void sub(SCSEP& sub) - { - fSub = sub; - } + virtual const std::string toString() const; - virtual const std::string toString() const; + virtual inline const std::string data() const + { + return fData; + } + virtual inline void data(const std::string data) + { + fData = data; + } - virtual inline const std::string data() const - { - return fData; - } - virtual inline void data( const std::string data ) - { - fData = data; - } + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual SimpleScalarFilter* clone() const + { + return new SimpleScalarFilter(*this); + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual SimpleScalarFilter* clone() const - { - return new SimpleScalarFilter (*this); - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const SimpleScalarFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const SimpleScalarFilter& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const SimpleScalarFilter& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const SimpleScalarFilter& t) const; + private: + // default okay? + // SelectFilter& operator=(const SelectFilter& rhs); -private: - //default okay? - //SelectFilter& operator=(const SelectFilter& rhs); - - std::vector fCols; - SOP fOp; - SCSEP fSub; - std::string fData; + std::vector fCols; + SOP fOp; + SCSEP fSub; + std::string fData; }; std::ostream& operator<<(std::ostream& output, const SimpleScalarFilter& rhs); -} -#endif //SELECTFILTER_H - +} // namespace execplan +#endif // SELECTFILTER_H diff --git a/dbcon/execplan/tdriver-oid.cpp b/dbcon/execplan/tdriver-oid.cpp index 80f39dd0f..f4ba2e2fe 100644 --- a/dbcon/execplan/tdriver-oid.cpp +++ b/dbcon/execplan/tdriver-oid.cpp @@ -39,7 +39,6 @@ using namespace std; - void* OIDManClient(void* arg); int timer; pthread_mutex_t lock; @@ -48,85 +47,78 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); + unlink("/tmp/oidbitmap"); + CPPUNIT_TEST(objectIDManager_2); + CPPUNIT_TEST(objectIDManager_3); + unlink("/tmp/oidbitmap"); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE( ExecPlanTest ); - unlink("/tmp/oidbitmap"); - CPPUNIT_TEST( objectIDManager_2 ); - CPPUNIT_TEST( objectIDManager_3 ); - unlink("/tmp/oidbitmap"); - CPPUNIT_TEST_SUITE_END(); + public: + void objectIDManager_2() + { + ObjectIDManager o; + int i, j, allocsize, loopCount, firstOID; + const int chunkcap = 10000; + const int OIDSpaceSize = 16777216; -public: + cout << endl << "Long OID Manager test" << endl; + cout << "using bitmap file " << o.getFilename() << endl; - void objectIDManager_2() + try { - ObjectIDManager o; - int i, j, allocsize, loopCount, firstOID; - const int chunkcap = 10000; - const int OIDSpaceSize = 16777216; + // make sure we can fill the entire OID space and clear it out a + // few times. This should stress test the implementation pretty + // well. - cout << endl << "Long OID Manager test" << endl; - cout << "using bitmap file " << o.getFilename() << endl; + firstOID = o.allocOID(); + cerr << "first OID is " << firstOID << endl; + o.returnOIDs(0, firstOID); - try + srand(time(NULL)); + + // fill the entire space then empty it out a few times. + for (i = 0; i < 10; i++) + { + cout << "fill & empty test " << i + 1 << "/10" << endl; + + for (j = 0, loopCount = 1; j < OIDSpaceSize; j += allocsize, loopCount++) { - // make sure we can fill the entire OID space and clear it out a - // few times. This should stress test the implementation pretty - // well. + allocsize = rand() % chunkcap; + allocsize = (allocsize > OIDSpaceSize - j ? OIDSpaceSize - j : allocsize); + CPPUNIT_ASSERT(o.allocOIDs(allocsize) > -1); + } - firstOID = o.allocOID(); - cerr << "first OID is " << firstOID << endl; - o.returnOIDs(0, firstOID); + CPPUNIT_ASSERT(o.allocOID() == -1); + CPPUNIT_ASSERT(o.size() == OIDSpaceSize); + o.returnOIDs(0, OIDSpaceSize - 1); // (gets rid of fragmentation in the freelist) + } - srand(time(NULL)); + // fill the space again, then deallocate randomly to fragment + // the freelist + for (j = 0, loopCount = 1; j < OIDSpaceSize; j += allocsize, loopCount++) + { + allocsize = rand() % chunkcap; + allocsize = (allocsize > OIDSpaceSize - j ? OIDSpaceSize - j : allocsize); + CPPUNIT_ASSERT(o.allocOIDs(allocsize) > -1); + } - // fill the entire space then empty it out a few times. - for (i = 0; i < 10; i++) - { - cout << "fill & empty test " << i + 1 << "/10" << endl; + CPPUNIT_ASSERT(o.allocOID() == -1); + CPPUNIT_ASSERT(o.size() == OIDSpaceSize); - for (j = 0, loopCount = 1; j < OIDSpaceSize ; j += allocsize, loopCount++) - { - allocsize = rand() % chunkcap; - allocsize = (allocsize > OIDSpaceSize - j ? - OIDSpaceSize - j : allocsize); - CPPUNIT_ASSERT(o.allocOIDs(allocsize) > -1); - } + for (j = 0, loopCount = 1; j < OIDSpaceSize; j += allocsize, loopCount++) + { + allocsize = rand() % chunkcap; + allocsize = (allocsize > OIDSpaceSize - j ? OIDSpaceSize - j : allocsize); + o.returnOIDs(j, j + allocsize - 1); + } - CPPUNIT_ASSERT(o.allocOID() == -1); - CPPUNIT_ASSERT(o.size() == OIDSpaceSize); - o.returnOIDs(0, OIDSpaceSize - 1); // (gets rid of fragmentation in the freelist) - } + CPPUNIT_ASSERT(o.size() == 0); + CPPUNIT_ASSERT(o.allocOIDs(OIDSpaceSize) == 0); - // fill the space again, then deallocate randomly to fragment - // the freelist - for (j = 0, loopCount = 1; j < OIDSpaceSize ; j += allocsize, loopCount++) - { - allocsize = rand() % chunkcap; - allocsize = (allocsize > OIDSpaceSize - j ? - OIDSpaceSize - j : allocsize); - CPPUNIT_ASSERT(o.allocOIDs(allocsize) > -1); - } - - CPPUNIT_ASSERT(o.allocOID() == -1); - CPPUNIT_ASSERT(o.size() == OIDSpaceSize); - - for (j = 0, loopCount = 1; j < OIDSpaceSize; j += allocsize, loopCount++) - { - allocsize = rand() % chunkcap; - allocsize = (allocsize > OIDSpaceSize - j ? - OIDSpaceSize - j : allocsize); - o.returnOIDs(j, j + allocsize - 1); - } - - CPPUNIT_ASSERT(o.size() == 0); - CPPUNIT_ASSERT(o.allocOIDs(OIDSpaceSize) == 0); - - - - // create small random holes & fill them. This test takes > 15 - // minutes, so it should be commented out unless there's a - // specific need for it + // create small random holes & fill them. This test takes > 15 + // minutes, so it should be commented out unless there's a + // specific need for it #if 0 const int smallcap = 20; @@ -174,127 +166,126 @@ public: CPPUNIT_ASSERT(o.allocOID() == -1); cout << "done." << endl; #endif - } - catch (...) - { - unlink(o.getFilename().c_str()); - throw; - } - - unlink(o.getFilename().c_str()); } - - /* This test attempts to simulate real transactions: many small - * blocks of allocs and returns in a multithreaded/multiprocess - * way. This test only uses threads, but that should be good enough. - */ - void objectIDManager_3() + catch (...) { - const int threadCount = 4; - int i; - pthread_t threads[threadCount]; - - cout << endl << "Multithreaded OIDManager test. " - "This runs for 2 minutes." << endl; - - timer = 0; - - for (i = 0; i < threadCount; i++) - if (pthread_create(&threads[i], NULL, OIDManClient, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the OID Manager test"); - - sleep(120); - timer = 1; - - for (i = 0; i < threadCount; i++) - pthread_join(threads[i], NULL); + unlink(o.getFilename().c_str()); + throw; } + unlink(o.getFilename().c_str()); + } + + /* This test attempts to simulate real transactions: many small + * blocks of allocs and returns in a multithreaded/multiprocess + * way. This test only uses threads, but that should be good enough. + */ + void objectIDManager_3() + { + const int threadCount = 4; + int i; + pthread_t threads[threadCount]; + + cout << endl + << "Multithreaded OIDManager test. " + "This runs for 2 minutes." + << endl; + + timer = 0; + + for (i = 0; i < threadCount; i++) + if (pthread_create(&threads[i], NULL, OIDManClient, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the OID Manager test"); + + sleep(120); + timer = 1; + + for (i = 0; i < threadCount; i++) + pthread_join(threads[i], NULL); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); void* OIDManClient(void* arg) { - ObjectIDManager o; - const int sizecap = 5; - int op, size, tmp; - uint32_t seed; - int threadnum = reinterpret_cast(arg); + ObjectIDManager o; + const int sizecap = 5; + int op, size, tmp; + uint32_t seed; + int threadnum = reinterpret_cast(arg); - struct entry + struct entry + { + int begin; + int end; + struct entry* next; + }; + + struct entry* e; + struct entry* listHead = NULL; + struct entry* listTail = NULL; + + cout << " Thread " << threadnum << " started." << endl; + + seed = time(NULL) % MAXINT; + + while (timer == 0) + { + op = rand_r(&seed) % 2; + size = (rand_r(&seed) % sizecap) + 1; + + if (op == 1) // allocate an OID block { - int begin; - int end; - struct entry* next; - }; + e = new struct entry; + tmp = o.allocOIDs(size); + CPPUNIT_ASSERT(tmp != -1); + e->begin = tmp; + e->end = tmp + size - 1; + e->next = NULL; - struct entry* e; - struct entry* listHead = NULL; - struct entry* listTail = NULL; + if (listTail != NULL) + listTail->next = e; + else + listHead = e; - cout << " Thread " << threadnum << " started." << endl; - - seed = time(NULL) % MAXINT; - - while (timer == 0) - { - op = rand_r(&seed) % 2; - size = (rand_r(&seed) % sizecap) + 1; - - if (op == 1) // allocate an OID block - { - e = new struct entry; - tmp = o.allocOIDs(size); - CPPUNIT_ASSERT(tmp != -1); - e->begin = tmp; - e->end = tmp + size - 1; - e->next = NULL; - - if (listTail != NULL) - listTail->next = e; - else - listHead = e; - - listTail = e; - } - else //deallocate an OID block - { - if (listHead == NULL) - continue; - - o.returnOIDs(listHead->begin, listHead->end); - e = listHead; - listHead = listHead->next; - - if (listHead == NULL) - listTail = NULL; - - delete e; - } + listTail = e; } - - while (listHead != NULL) + else // deallocate an OID block { - e = listHead; - listHead = listHead->next; - delete e; - } + if (listHead == NULL) + continue; - cout << " Thread " << threadnum << " exiting." << endl; - pthread_exit(0); + o.returnOIDs(listHead->begin, listHead->end); + e = listHead; + listHead = listHead->next; + + if (listHead == NULL) + listTail = NULL; + + delete e; + } + } + + while (listHead != NULL) + { + e = listHead; + listHead = listHead->next; + delete e; + } + + cout << " Thread " << threadnum << " exiting." << endl; + pthread_exit(0); } - #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/execplan/tdriver-sm.cpp b/dbcon/execplan/tdriver-sm.cpp index 853909eed..112b2b10d 100644 --- a/dbcon/execplan/tdriver-sm.cpp +++ b/dbcon/execplan/tdriver-sm.cpp @@ -51,218 +51,213 @@ int threadStop, threadNum; static void* SMRunner(void* arg) { - SessionManager* sm; - int op; - uint32_t seed, sessionnum; - SessionManager::TxnID tmp; - int tNum = reinterpret_cast(arg); + SessionManager* sm; + int op; + uint32_t seed, sessionnum; + SessionManager::TxnID tmp; + int tNum = reinterpret_cast(arg); - struct entry + struct entry + { + SessionManager::TxnID tid; + uint32_t sessionnum; + struct entry* next; + }; + + struct entry* e; + struct entry* listHead = NULL; + struct entry* listTail = NULL; + + cerr << "Thread " << tNum << " started." << endl; + + seed = time(NULL) % MAXINT; + + sm = new SessionManager(); + + while (!threadStop) + { + sm->verifySize(); + op = rand_r(&seed) % 4; // 0 = newTxnID, 1 = committed, 2 = getTxnID, 3 = delete sm; new sm + sessionnum = rand_r(&seed); + + switch (op) { - SessionManager::TxnID tid; - uint32_t sessionnum; - struct entry* next; - }; + case 0: + e = new struct entry; + e->tid = sm->newTxnID(sessionnum, false); - struct entry* e; - struct entry* listHead = NULL; - struct entry* listTail = NULL; - - cerr << "Thread " << tNum << " started." << endl; - - seed = time(NULL) % MAXINT; - - sm = new SessionManager(); - - while (!threadStop) - { - sm->verifySize(); - op = rand_r(&seed) % 4; // 0 = newTxnID, 1 = committed, 2 = getTxnID, 3 = delete sm; new sm - sessionnum = rand_r(&seed); - - switch (op) + if (e->tid.valid == false) // SM is full { - case 0: - e = new struct entry; - e->tid = sm->newTxnID(sessionnum, false); + delete e; + break; + } - if (e->tid.valid == false) // SM is full - { - delete e; - break; - } + e->sessionnum = sessionnum; + e->next = NULL; - e->sessionnum = sessionnum; - e->next = NULL; + if (listTail != NULL) + listTail->next = e; + else + listHead = e; - if (listTail != NULL) - listTail->next = e; - else - listHead = e; + listTail = e; + break; - listTail = e; - break; + case 1: + if (listHead == NULL) + continue; - case 1: - if (listHead == NULL) - continue; - - sm->committed(listHead->tid); - e = listHead; - listHead = listHead->next; - - if (listHead == NULL) - listTail = NULL; - - delete e; - break; - - case 2: - if (listHead == NULL) - continue; - - tmp = sm->getTxnID(listHead->sessionnum); - CPPUNIT_ASSERT(tmp.valid == listHead->tid.valid == true); - // there's some risk of collision here if 2 threads happen to choose the - // same session number - //CPPUNIT_ASSERT(tmp.id == listHead->tid.id); - break; - - case 3: - delete sm; - sm = new SessionManager(); - break; - - default: - cerr << "SMRunner: ??" << endl; - }; - } - - while (listHead != NULL) - { + sm->committed(listHead->tid); e = listHead; listHead = listHead->next; - delete e; - } - delete sm; - cerr << "Thread " << tNum << " exiting." << endl; - pthread_exit(0); + if (listHead == NULL) + listTail = NULL; + + delete e; + break; + + case 2: + if (listHead == NULL) + continue; + + tmp = sm->getTxnID(listHead->sessionnum); + CPPUNIT_ASSERT(tmp.valid == listHead->tid.valid == true); + // there's some risk of collision here if 2 threads happen to choose the + // same session number + // CPPUNIT_ASSERT(tmp.id == listHead->tid.id); + break; + + case 3: + delete sm; + sm = new SessionManager(); + break; + + default: cerr << "SMRunner: ??" << endl; + }; + } + + while (listHead != NULL) + { + e = listHead; + listHead = listHead->next; + delete e; + } + + delete sm; + cerr << "Thread " << tNum << " exiting." << endl; + pthread_exit(0); } class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); + CPPUNIT_TEST(sessionManager_3); + unlink("/tmp/CalpontShm"); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE( ExecPlanTest ); - CPPUNIT_TEST(sessionManager_3); + public: + /* + * destroySemaphores() and destroyShmseg() will print error messages + * if there are no objects to destroy. That's OK. + */ + void destroySemaphores() + { + key_t semkey; + char* semseed = "/usr/local/mariadb/columnstore/etc/Columnstore.xml"; + int sems, err; + + // semkey = ftok(semseed, 0x2149bdd2); // these things must match in the SM constructor + semkey = 0x2149bdd2; + + if (semkey == -1) + perror("tdriver: ftok"); + + sems = semget(semkey, 2, 0666); + + if (sems != -1) + { + err = semctl(sems, 0, IPC_RMID); + + if (err == -1) + perror("tdriver: semctl"); + } + } + + void destroyShmseg() + { + key_t shmkey; + char* shmseed = "/usr/local/mariadb/columnstore/etc/Columnstore.xml"; + int shms, err; + + // shmkey = ftok(shmseed, 0x2149bdd2); // these things much match in the SM constructor + shmkey = 0x2149bdd2; + + if (shmkey == -1) + perror("tdriver: ftok"); + + shms = shmget(shmkey, 0, 0666); + + if (shms != -1) + { + err = shmctl(shms, IPC_RMID, NULL); + + if (err == -1 && errno != EINVAL) + { + perror("tdriver: shmctl"); + return; + } + } + } + + /** This launches several threads to stress test the Session Manager + */ + void sessionManager_3() + { + const int threadCount = 4; + int i; + pthread_t threads[threadCount]; + + cerr << endl + << "Multithreaded SessionManager test. " + "This runs for 2 minutes." + << endl; + + destroySemaphores(); + destroyShmseg(); unlink("/tmp/CalpontShm"); - CPPUNIT_TEST_SUITE_END(); + threadStop = 0; -public: - - /* - * destroySemaphores() and destroyShmseg() will print error messages - * if there are no objects to destroy. That's OK. - */ - void destroySemaphores() + for (i = 0; i < threadCount; i++) { - key_t semkey; - char* semseed = "/usr/local/mariadb/columnstore/etc/Columnstore.xml"; - int sems, err; + if (pthread_create(&threads[i], NULL, SMRunner, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the Session Manager test"); -// semkey = ftok(semseed, 0x2149bdd2); // these things must match in the SM constructor - semkey = 0x2149bdd2; - - if (semkey == -1) - perror("tdriver: ftok"); - - sems = semget(semkey, 2, 0666); - - if (sems != -1) - { - err = semctl(sems, 0, IPC_RMID); - - if (err == -1) - perror("tdriver: semctl"); - } + usleep(1000); } - void destroyShmseg() - { - key_t shmkey; - char* shmseed = "/usr/local/mariadb/columnstore/etc/Columnstore.xml"; - int shms, err; - -// shmkey = ftok(shmseed, 0x2149bdd2); // these things much match in the SM constructor - shmkey = 0x2149bdd2; - - if (shmkey == -1) - perror("tdriver: ftok"); - - shms = shmget(shmkey, 0, 0666); - - if (shms != -1) - { - err = shmctl(shms, IPC_RMID, NULL); - - if (err == -1 && errno != EINVAL) - { - perror("tdriver: shmctl"); - return; - } - } - } - - /** This launches several threads to stress test the Session Manager - */ - void sessionManager_3() - { - const int threadCount = 4; - int i; - pthread_t threads[threadCount]; - - cerr << endl << "Multithreaded SessionManager test. " - "This runs for 2 minutes." << endl; - - destroySemaphores(); - destroyShmseg(); - unlink("/tmp/CalpontShm"); - - threadStop = 0; - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, SMRunner, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the Session Manager test"); - - usleep(1000); - } - - sleep(120); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - pthread_join(threads[i], NULL); - - destroySemaphores(); - destroyShmseg(); - } + sleep(120); + threadStop = 1; + for (i = 0; i < threadCount; i++) + pthread_join(threads[i], NULL); + destroySemaphores(); + destroyShmseg(); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/execplan/tdriver.cpp b/dbcon/execplan/tdriver.cpp index 080c54ee7..0a0210806 100644 --- a/dbcon/execplan/tdriver.cpp +++ b/dbcon/execplan/tdriver.cpp @@ -23,9 +23,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include #include @@ -69,1476 +69,1456 @@ int maxNewTxns = 1000; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_1); + CPPUNIT_TEST(selectExecutionPlan_2); + CPPUNIT_TEST(expressionParser_1); + CPPUNIT_TEST(expressionParser_2); + CPPUNIT_TEST(aggregatecolumn_1); + CPPUNIT_TEST(arithmeticExpression_1); + CPPUNIT_TEST(arithmeticExpression_2); + CPPUNIT_TEST(arithmeticExpression_3); + CPPUNIT_TEST(copyTree); + CPPUNIT_TEST(treeNodeImpl); + CPPUNIT_TEST(serializeSimpleColumn); + CPPUNIT_TEST(serializeAggregateColumn); + CPPUNIT_TEST(serializeOperator); + CPPUNIT_TEST(serializeFilter); + CPPUNIT_TEST(serializeFunctionColumn); + CPPUNIT_TEST(serializeConstantColumn); + CPPUNIT_TEST(serializeParseTree); + CPPUNIT_TEST(serializeArithmeticColumn); + CPPUNIT_TEST(serializeSimpleFilter); + CPPUNIT_TEST(serializeCSEP); + CPPUNIT_TEST(serializeSelectFilter); + CPPUNIT_TEST(serializeExistsFilter); - CPPUNIT_TEST( selectExecutionPlan_1 ); - CPPUNIT_TEST( selectExecutionPlan_2 ); - CPPUNIT_TEST( expressionParser_1 ); - CPPUNIT_TEST( expressionParser_2 ); - CPPUNIT_TEST( aggregatecolumn_1 ); - CPPUNIT_TEST( arithmeticExpression_1 ); - CPPUNIT_TEST( arithmeticExpression_2 ); - CPPUNIT_TEST( arithmeticExpression_3 ); - CPPUNIT_TEST( copyTree ); - CPPUNIT_TEST( treeNodeImpl ); - CPPUNIT_TEST( serializeSimpleColumn ); - CPPUNIT_TEST( serializeAggregateColumn ); - CPPUNIT_TEST( serializeOperator ); - CPPUNIT_TEST( serializeFilter ); - CPPUNIT_TEST( serializeFunctionColumn ); - CPPUNIT_TEST( serializeConstantColumn ); - CPPUNIT_TEST( serializeParseTree ); - CPPUNIT_TEST( serializeArithmeticColumn ); - CPPUNIT_TEST( serializeSimpleFilter ); - CPPUNIT_TEST( serializeCSEP ); - CPPUNIT_TEST( serializeSelectFilter ); - CPPUNIT_TEST( serializeExistsFilter ); + CPPUNIT_TEST(sessionManager_1); + CPPUNIT_TEST(sessionManager_2); + CPPUNIT_TEST(sessionManager_3); + CPPUNIT_TEST(sessionManager_4); - CPPUNIT_TEST( sessionManager_1 ); - CPPUNIT_TEST( sessionManager_2 ); - CPPUNIT_TEST( sessionManager_3 ); - CPPUNIT_TEST( sessionManager_4 ); + unlink("/tmp/CalpontSessionMonitorShm"); + // CPPUNIT_TEST( MonitorTestPlan_1 ); + // CPPUNIT_TEST( MonitorTestPlan_1 ); + unlink("/tmp/CalpontSessionMonitorShm"); + CPPUNIT_TEST(objectIDManager_1); - unlink("/tmp/CalpontSessionMonitorShm"); -//CPPUNIT_TEST( MonitorTestPlan_1 ); -//CPPUNIT_TEST( MonitorTestPlan_1 ); - unlink("/tmp/CalpontSessionMonitorShm"); - CPPUNIT_TEST( objectIDManager_1 ); + if (OIDBitmapFilename != NULL) + unlink(OIDBitmapFilename); - if (OIDBitmapFilename != NULL) - unlink(OIDBitmapFilename); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + boost::shared_ptr csc; -private: - boost::shared_ptr csc; -public: + public: + static void walkfnString(const ParseTree* n) + { + cout << *(n->data()) << endl; + } - static void walkfnString(const ParseTree* n) + static void walkfnInt(const ExpressionTree* n) + { + cout << n->data() << endl; + } + + void setUp() + { + csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); + csc->identity(CalpontSystemCatalog::FE); + } + + void tearDown() + { + } + + void selectExecutionPlan_1() + { + cout << "SQL: select region.r_regionkey from region, nation where nation.n_regionkey = " + "region.r_regionkey and nation.n_regionkey != 3;" + << endl; + + CalpontSelectExecutionPlan csep; + CPPUNIT_ASSERT(csep.location() == CalpontSelectExecutionPlan::MAIN); + CPPUNIT_ASSERT(csep.dependent() == false); + CPPUNIT_ASSERT(csep.subSelects().size() == 0); + + // returned columns + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* sc = new SimpleColumn("tpch.region.r_regionkey", 0); + colList.push_back(sc); + ArithmeticColumn* ac = new ArithmeticColumn("a+sum(r_regionkey)", 0); + colList.push_back(ac); + csep.returnedCols(colList); + CPPUNIT_ASSERT(csep.returnedCols().size() == 2); + + // filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + + SimpleFilter* sf = new SimpleFilter(); + SimpleColumn* lhs = new SimpleColumn(); + *lhs = *sc; + SimpleColumn* rhs = new SimpleColumn("tpch.nation.n_regionkey", 0); + CPPUNIT_ASSERT(*lhs == *sc); + CPPUNIT_ASSERT(*rhs != *lhs); + Operator* op = new Operator("="); + + sf->op(op); + sf->lhs(lhs); + sf->rhs(rhs); + filterTokenList.push_back(sf); + + filterTokenList.push_back(new Operator("And")); + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), sc->clone(), ac->clone()); + + filterTokenList.push_back(sf1); + + csep.filterTokenList(filterTokenList); + ParseTree* filterList = const_cast(csep.filters()); + + // draw filterList tree + filterList->drawTree("selectExecutionPlan_1.dot"); + csep.filters(filterList); + + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + groupByList.push_back(sc->clone()); + csep.groupByCols(groupByList); + CPPUNIT_ASSERT(csep.groupByCols().size() == 1); + + // Having + CalpontSelectExecutionPlan::FilterTokenList havingTokenList; + SimpleFilter* having = + new SimpleFilter(new Operator("="), new ArithmeticColumn("sum(volumn)", 0), new ConstantColumn(8)); + havingTokenList.push_back(having); + csep.havingTokenList(havingTokenList); + CPPUNIT_ASSERT(*sf1 != *having); + CPPUNIT_ASSERT(csep.havingTokenList().size() == 1); + + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + ArithmeticColumn* o1 = new ArithmeticColumn(*ac); + o1->asc(false); + orderByList.push_back(o1); + csep.orderByCols(orderByList); + CPPUNIT_ASSERT(csep.orderByCols().size() == 1); + + // another csep + CalpontSelectExecutionPlan* newcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); + CalpontSelectExecutionPlan::ReturnedColumnList ncolList; + SimpleColumn* newsc = new SimpleColumn("tpch.region.r_regionkey", 0); + ncolList.push_back(newsc); + newcsep->returnedCols(ncolList); + CalpontSelectExecutionPlan::FilterTokenList nfilterTokenList; + SimpleFilter* newsf = new SimpleFilter(new Operator(">"), sc->clone(), newsc->clone()); + nfilterTokenList.push_back(newsf); + newcsep->filterTokenList(nfilterTokenList); + CalpontSelectExecutionPlan::FilterTokenList nhavingTokenList; + SimpleFilter* newhaving = new SimpleFilter(new Operator(">"), sc->clone(), newsc->clone()); + CPPUNIT_ASSERT(*newsf == *newhaving); + nhavingTokenList.push_back(newhaving); + newcsep->havingTokenList(nhavingTokenList); + CPPUNIT_ASSERT(*newcsep != csep); + CPPUNIT_ASSERT(*newcsep->filters() == *newcsep->having()); + ByteStream b; + csep.serialize(b); + newcsep->unserialize(b); + CPPUNIT_ASSERT(csep == *newcsep); + CalpontSelectExecutionPlan::SelectList selectList; + selectList.push_back(newcsep); + csep.subSelects(selectList); + cout << "\nCalpont Execution Plan:" << endl; + cout << csep; + cout << " --- end of test 1 ---" << endl; + } + + void selectExecutionPlan_2() + { + CalpontSelectExecutionPlan cep; + + // select filter + CalpontSelectExecutionPlan* subselect = new CalpontSelectExecutionPlan; + subselect->location(CalpontSelectExecutionPlan::WHERE); + subselect->dependent(false); + CPPUNIT_ASSERT(subselect->location() == CalpontSelectExecutionPlan::WHERE); + CPPUNIT_ASSERT(subselect->dependent() == false); + ByteStream selb; + subselect->serialize(selb); + CalpontSelectExecutionPlan* subselect1 = new CalpontSelectExecutionPlan; + subselect->location(CalpontSelectExecutionPlan::WHERE); + subselect1->unserialize(selb); + + SelectFilter* sef = + new SelectFilter(new SimpleColumn("tpch.region.r_regionkey", 0), new Operator(">="), subselect); + cout << *sef; + ByteStream b; + sef->serialize(b); + SelectFilter* newsef = + new SelectFilter(new SimpleColumn("tpch.nation.n_regionke", 0), new Operator("<="), subselect1); + newsef->unserialize(b); + CPPUNIT_ASSERT(*sef == *newsef); + delete sef; + delete newsef; + + // simple filter + Filter* sf = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_regionke", 0), + new SimpleColumn("tpch.region.r_regionkey", 0)); + cout << *sf; + + ByteStream sfb; + SimpleFilter* sf2 = new SimpleFilter(); + sf2->serialize(sfb); + sf->unserialize(sfb); + CPPUNIT_ASSERT(*sf == *sf2); + delete sf2; + delete sf; + + // exist filter + CalpontSelectExecutionPlan* cep1 = new CalpontSelectExecutionPlan(); + ExistsFilter* filter = new ExistsFilter(); + delete filter; + filter = new ExistsFilter(cep1); + filter->exists(cep1); + const CalpontSelectExecutionPlan* cep2 = filter->exists(); + cout << *filter; + CPPUNIT_ASSERT(*cep1 == *cep2); + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; + Token t; + t.value = filter; + tokens.push_back(t); + cep.filters(parser.parse(tokens.begin(), tokens.end())); + cout << cep; + + cout << " --- end of test 2 ---" << endl; + } + + void expressionParser_1() + { + cout << "\nPost order of int expression tree(pointer):" << endl; + ExpressionTree* et = new ExpressionTree(3); + ExpressionTree* et1 = new ExpressionTree(5); + ExpressionTree* et2 = new ExpressionTree(6); + et->left(et1); + et->right(et2); + et->walk(walkfnInt); + et->destroyTree(et); + cout << " --- end of test 3 ---" << endl; + } + + void expressionParser_2() + { + cout << "\nPost order of int expression tree (reference):" << endl; + ExpressionTree et(3); + ExpressionTree et1(5); + ExpressionTree et2(6); + et.left(&et1); + et.right(&et2); + et.walk(walkfnInt); + cout << " --- end of test 4 ---" << endl; + } + + void aggregatecolumn_1() + { + cout << "\naggregate column: " << endl; + AggregateColumn a; + cout << a; + cout << "\nsum(p_type)" << endl; + AggregateColumn d("sum", "p_type"); + cout << d; + a.functionName("avg"); + ArithmeticColumn* b = new ArithmeticColumn("a-(b-c)"); + a.functionParms(b); + CPPUNIT_ASSERT(a.functionName() == "avg"); + cout << *const_cast(a.functionParms()); + cout << " --- end of test 5 ---" << endl; + } + + void arithmeticExpression_1() + { + cout << "\narithmetic expression: " << endl; + string exp( + "substr(a)+ 100.00 * sum(tpch.part.p_type) / sum(tpch.lineitem.l_extendedprice " + "*(1-tpch.lineitem.l_discount))"); + cout << exp << endl; + ArithmeticColumn a(exp, 0); + ParseTree* pt = const_cast(a.expression()); + + if (pt != NULL) { - cout << *(n->data()) << endl; + pt->walk(walkfnString); + pt->drawTree("arithmeticExpression_1.dot"); } - static void walkfnInt(const ExpressionTree* n) + cout << " --- end of test 6 ---" << endl; + } + + void copyTree() + { + // cout << "\narithmetic expression: " << endl; + string exp( + "substr(a)+ 100.00 * sum(tpch.part.p_type) / sum(tpch.lineitem.l_extendedprice " + "*(1-tpch.lineitem.l_discount))"); + ArithmeticColumn a(exp); + ParseTree* pt = const_cast(a.expression()); + + ParseTree* newTree = new ParseTree(); + + // copy 1st time + newTree->copyTree(*pt); + + // copy 2nd time, see if the existing tree is deleted. + newTree->copyTree(*pt); + + // explicitly delete the 2nd copied tree + delete newTree; + } + + void arithmeticExpression_2() + { + cout << "\nbuild arithmetic column by using accessing methods:" << endl; + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; + Token t; + + ArithmeticColumn b; + cout << b; + + ConstantColumn* c1 = new ConstantColumn(); + c1->constval("'ASIA'"); + c1->type(ConstantColumn::LITERAL); + // CPPUNIT_ASSERT (c1->data() == "'ASIA'(l)"); + + t.value = c1; + tokens.push_back(t); + + t.value = new Operator("/"); + tokens.push_back(t); + + ConstantColumn* c2 = new ConstantColumn(5); + CPPUNIT_ASSERT(c2->type() == ConstantColumn::NUM); + t.value = c2; + tokens.push_back(t); + + ParseTree* tree = parser.parse(tokens.begin(), tokens.end()); + b.expression(tree); + + cout << b; + cout << " --- end of test 7 ---" << endl; + } + + void arithmeticExpression_3() + { + // invalid expression test + try { - cout << n->data() << endl; + ArithmeticColumn a("-a+b", 0); + ArithmeticColumn d("a* b-", 0); + } + catch (const runtime_error& e) + { + cerr << e.what() << endl; } - void setUp() + try { - csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); - csc->identity(CalpontSystemCatalog::FE); + ArithmeticColumn e("a+substr (c from 1 4", 0); + } + catch (const runtime_error& e) + { + cerr << e.what() << endl; } - void tearDown() + try { + ArithmeticColumn f("a + b c", 0); + } + catch (const runtime_error& e) + { + cerr << e.what() << endl; } - void selectExecutionPlan_1() + try { - cout << "SQL: select region.r_regionkey from region, nation where nation.n_regionkey = region.r_regionkey and nation.n_regionkey != 3;" << endl; - - CalpontSelectExecutionPlan csep; - CPPUNIT_ASSERT (csep.location() == CalpontSelectExecutionPlan::MAIN); - CPPUNIT_ASSERT (csep.dependent() == false); - CPPUNIT_ASSERT (csep.subSelects().size() == 0); - - // returned columns - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* sc = new SimpleColumn("tpch.region.r_regionkey", 0); - colList.push_back(sc); - ArithmeticColumn* ac = new ArithmeticColumn("a+sum(r_regionkey)", 0); - colList.push_back(ac); - csep.returnedCols (colList); - CPPUNIT_ASSERT(csep.returnedCols().size() == 2); - - // filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - - SimpleFilter* sf = new SimpleFilter(); - SimpleColumn* lhs = new SimpleColumn(); - *lhs = *sc; - SimpleColumn* rhs = new SimpleColumn("tpch.nation.n_regionkey", 0); - CPPUNIT_ASSERT (*lhs == *sc); - CPPUNIT_ASSERT (*rhs != *lhs); - Operator* op = new Operator("="); - - sf->op(op); - sf->lhs(lhs); - sf->rhs(rhs); - filterTokenList.push_back (sf); - - filterTokenList.push_back( new Operator ("And") ); - SimpleFilter* sf1 = new SimpleFilter (new Operator("="), sc->clone(), ac->clone()); - - filterTokenList.push_back (sf1); - - csep.filterTokenList (filterTokenList); - ParseTree* filterList = const_cast (csep.filters()); - - // draw filterList tree - filterList->drawTree("selectExecutionPlan_1.dot"); - csep.filters (filterList); - - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - groupByList.push_back(sc->clone()); - csep.groupByCols (groupByList); - CPPUNIT_ASSERT(csep.groupByCols().size() == 1); - - // Having - CalpontSelectExecutionPlan::FilterTokenList havingTokenList; - SimpleFilter* having = new SimpleFilter( new Operator("="), - new ArithmeticColumn("sum(volumn)", 0), - new ConstantColumn(8)); - havingTokenList.push_back (having); - csep.havingTokenList (havingTokenList); - CPPUNIT_ASSERT (*sf1 != *having); - CPPUNIT_ASSERT (csep.havingTokenList().size() == 1); - - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - ArithmeticColumn* o1 = new ArithmeticColumn(*ac); - o1->asc(false); - orderByList.push_back(o1); - csep.orderByCols(orderByList); - CPPUNIT_ASSERT(csep.orderByCols().size() == 1); - - // another csep - CalpontSelectExecutionPlan* newcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); - CalpontSelectExecutionPlan::ReturnedColumnList ncolList; - SimpleColumn* newsc = new SimpleColumn("tpch.region.r_regionkey", 0); - ncolList.push_back(newsc); - newcsep->returnedCols (ncolList); - CalpontSelectExecutionPlan::FilterTokenList nfilterTokenList; - SimpleFilter* newsf = new SimpleFilter ( new Operator (">"), - sc->clone(), - newsc->clone()); - nfilterTokenList.push_back(newsf); - newcsep->filterTokenList (nfilterTokenList); - CalpontSelectExecutionPlan::FilterTokenList nhavingTokenList; - SimpleFilter* newhaving = new SimpleFilter ( new Operator (">"), - sc->clone(), - newsc->clone()); - CPPUNIT_ASSERT (*newsf == *newhaving); - nhavingTokenList.push_back(newhaving); - newcsep->havingTokenList (nhavingTokenList); - CPPUNIT_ASSERT (*newcsep != csep); - CPPUNIT_ASSERT (*newcsep->filters() == *newcsep->having()); - ByteStream b; - csep.serialize (b); - newcsep->unserialize (b); - CPPUNIT_ASSERT (csep == *newcsep); - CalpontSelectExecutionPlan::SelectList selectList; - selectList.push_back(newcsep); - csep.subSelects(selectList); - cout << "\nCalpont Execution Plan:" << endl; - cout << csep; - cout << " --- end of test 1 ---" << endl; + ArithmeticColumn b("a + ((b+ c -e)", 0); + } + catch (const runtime_error& e) + { + cerr << e.what() << endl; } - void selectExecutionPlan_2() + try { - CalpontSelectExecutionPlan cep; - - // select filter - CalpontSelectExecutionPlan* subselect = new CalpontSelectExecutionPlan; - subselect->location(CalpontSelectExecutionPlan::WHERE); - subselect->dependent (false); - CPPUNIT_ASSERT (subselect->location() == CalpontSelectExecutionPlan::WHERE); - CPPUNIT_ASSERT (subselect->dependent() == false); - ByteStream selb; - subselect->serialize(selb); - CalpontSelectExecutionPlan* subselect1 = new CalpontSelectExecutionPlan; - subselect->location(CalpontSelectExecutionPlan::WHERE); - subselect1->unserialize(selb); - - SelectFilter* sef = new SelectFilter ( new SimpleColumn ("tpch.region.r_regionkey", 0), - new Operator (">="), - subselect); - cout << *sef; - ByteStream b; - sef->serialize(b); - SelectFilter* newsef = new SelectFilter ( new SimpleColumn ("tpch.nation.n_regionke", 0), - new Operator ("<="), - subselect1); - newsef->unserialize(b); - CPPUNIT_ASSERT(*sef == *newsef); - delete sef; - delete newsef; - - // simple filter - Filter* sf = new SimpleFilter(new Operator("="), - new SimpleColumn ("tpch.nation.n_regionke", 0), - new SimpleColumn ("tpch.region.r_regionkey", 0)); - cout << *sf; - - ByteStream sfb; - SimpleFilter* sf2 = new SimpleFilter(); - sf2->serialize (sfb); - sf->unserialize (sfb); - CPPUNIT_ASSERT (*sf == *sf2); - delete sf2; - delete sf; - - // exist filter - CalpontSelectExecutionPlan* cep1 = new CalpontSelectExecutionPlan(); - ExistsFilter* filter = new ExistsFilter(); - delete filter; - filter = new ExistsFilter(cep1); - filter->exists(cep1); - const CalpontSelectExecutionPlan* cep2 = filter->exists(); - cout << *filter; - CPPUNIT_ASSERT (*cep1 == *cep2); - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; - Token t; - t.value = filter; - tokens.push_back(t); - cep.filters(parser.parse(tokens.begin(), tokens.end())); - cout << cep; - - cout << " --- end of test 2 ---" << endl; + ArithmeticColumn g("a ++ b", 0); // valid } - - - void expressionParser_1() + catch (const runtime_error& e) { - cout << "\nPost order of int expression tree(pointer):" << endl; - ExpressionTree* et = new ExpressionTree(3); - ExpressionTree* et1 = new ExpressionTree(5); - ExpressionTree* et2 = new ExpressionTree(6); - et->left(et1); - et->right(et2); - et->walk(walkfnInt); - et->destroyTree(et); - cout << " --- end of test 3 ---" << endl; - } - - void expressionParser_2() - { - cout << "\nPost order of int expression tree (reference):" << endl; - ExpressionTree et(3); - ExpressionTree et1(5); - ExpressionTree et2(6); - et.left(&et1); - et.right(&et2); - et.walk(walkfnInt); - cout << " --- end of test 4 ---" << endl; - } - - void aggregatecolumn_1() - { - cout << "\naggregate column: " << endl; - AggregateColumn a; - cout << a; - cout << "\nsum(p_type)" << endl; - AggregateColumn d("sum", "p_type"); - cout << d; - a.functionName("avg"); - ArithmeticColumn* b = new ArithmeticColumn("a-(b-c)"); - a.functionParms(b); - CPPUNIT_ASSERT(a.functionName() == "avg"); - cout << *const_cast(a.functionParms()); - cout << " --- end of test 5 ---" << endl; - } - - void arithmeticExpression_1() - { - cout << "\narithmetic expression: " << endl; - string exp("substr(a)+ 100.00 * sum(tpch.part.p_type) / sum(tpch.lineitem.l_extendedprice *(1-tpch.lineitem.l_discount))"); - cout << exp << endl; - ArithmeticColumn a(exp, 0); - ParseTree* pt = const_cast(a.expression()); - - if (pt != NULL) - { - pt->walk(walkfnString); - pt->drawTree("arithmeticExpression_1.dot"); - } - - cout << " --- end of test 6 ---" << endl; - } - - void copyTree() - { - //cout << "\narithmetic expression: " << endl; - string exp("substr(a)+ 100.00 * sum(tpch.part.p_type) / sum(tpch.lineitem.l_extendedprice *(1-tpch.lineitem.l_discount))"); - ArithmeticColumn a(exp); - ParseTree* pt = const_cast(a.expression()); - - ParseTree* newTree = new ParseTree(); - - // copy 1st time - newTree->copyTree(*pt); - - // copy 2nd time, see if the existing tree is deleted. - newTree->copyTree (*pt); - - // explicitly delete the 2nd copied tree - delete newTree; - } - - void arithmeticExpression_2() - { - cout << "\nbuild arithmetic column by using accessing methods:" << endl; - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; - Token t; - - ArithmeticColumn b; - cout << b; - - ConstantColumn* c1 = new ConstantColumn(); - c1->constval("'ASIA'"); - c1->type(ConstantColumn::LITERAL); - //CPPUNIT_ASSERT (c1->data() == "'ASIA'(l)"); - - t.value = c1; - tokens.push_back(t); - - t.value = new Operator ("/"); - tokens.push_back(t); - - ConstantColumn* c2 = new ConstantColumn(5); - CPPUNIT_ASSERT (c2->type() == ConstantColumn::NUM); - t.value = c2; - tokens.push_back(t); - - ParseTree* tree = parser.parse(tokens.begin(), tokens.end()); - b.expression (tree); - - cout << b; - cout << " --- end of test 7 ---" << endl; - } - - void arithmeticExpression_3() - { - // invalid expression test - try - { - ArithmeticColumn a ("-a+b", 0); - ArithmeticColumn d("a* b-", 0); - } - catch (const runtime_error& e) - { - cerr << e.what() << endl; - } - - try - { - ArithmeticColumn e("a+substr (c from 1 4", 0); - } - catch (const runtime_error& e) - { - cerr << e.what() << endl; - } - - try - { - ArithmeticColumn f("a + b c", 0); - } - catch (const runtime_error& e) - { - cerr << e.what() << endl; - } - - try - { - ArithmeticColumn b("a + ((b+ c -e)", 0); - } - catch (const runtime_error& e) - { - cerr << e.what() << endl; - } - - try - { - ArithmeticColumn g("a ++ b", 0); // valid - } - catch (const runtime_error& e) - { - cerr << e.what() << endl; - } - } - - void treeNodeImpl () - { - TreeNodeImpl* node1 = new TreeNodeImpl(); - TreeNodeImpl* node2 = new TreeNodeImpl( "node2" ); - CPPUNIT_ASSERT (node2->data() == "node2"); - CPPUNIT_ASSERT (*node1 != *node2); - - ByteStream b; - node2->serialize (b); - node1->unserialize (b); - CPPUNIT_ASSERT (*node1 == *node2); - - node2->data ("node3"); - cout << *node2; - - TreeNodeImpl* node4 = node2->clone(); - CPPUNIT_ASSERT (*node2 == node4); - - delete node1; - delete node2; - delete node4; - } - - void serializeSimpleColumn() - { - SimpleColumn s1, s2; - TreeNode* t; - ByteStream b; - - t = &s2; - - CPPUNIT_ASSERT(s1 == s2); - CPPUNIT_ASSERT(!(s1 != s2)); - CPPUNIT_ASSERT(s1 == t); - CPPUNIT_ASSERT(!(s1 != t)); - - s1.schemaName("Schema Name 1"); - s1.tableName("Table Name 1"); - s1.columnName("Column Name 1"); - //s1.tcn(5); - s1.data("sc1"); - - s1.serialize(b); - - CPPUNIT_ASSERT(s2.schemaName() == ""); - CPPUNIT_ASSERT(s2.tableName() == ""); - CPPUNIT_ASSERT(s2.columnName() == ""); - //CPPUNIT_ASSERT(s2.tcn() == 0); - - CPPUNIT_ASSERT(s1 != s2); - CPPUNIT_ASSERT(s1 == s1); - CPPUNIT_ASSERT(s1 != t); - CPPUNIT_ASSERT(!(s1 == t)); - - s2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - - CPPUNIT_ASSERT(s2.schemaName() == "Schema Name 1"); - CPPUNIT_ASSERT(s2.tableName() == "Table Name 1"); - CPPUNIT_ASSERT(s2.columnName() == "Column Name 1"); - //CPPUNIT_ASSERT(s2.tcn() == 5); - CPPUNIT_ASSERT(s2.data() == "sc1"); - - CPPUNIT_ASSERT(s1 == s2); - CPPUNIT_ASSERT(!(s1 != s2)); - CPPUNIT_ASSERT(s1 == t); - CPPUNIT_ASSERT(!(s1 != t)); - } - - void serializeAggregateColumn() - { - AggregateColumn a1, a2; - SimpleColumn* s1; - TreeNode* t; - ByteStream b; - - t = &a2; - - s1 = new SimpleColumn(); - - CPPUNIT_ASSERT(a1 == a2); - CPPUNIT_ASSERT(!(a1 != a2)); - CPPUNIT_ASSERT(a1 == t); - CPPUNIT_ASSERT(!(a1 != t)); - - a1.functionName("AggregateColumn test"); - a1.data("agg1"); - a1.functionParms(s1); - - a1.serialize(b); - - CPPUNIT_ASSERT(a2.functionName() == ""); - CPPUNIT_ASSERT(a2.data() == ""); - CPPUNIT_ASSERT(a2.functionParms() == NULL); - - CPPUNIT_ASSERT(a1 != a2); - CPPUNIT_ASSERT(!(a1 == a2)); - CPPUNIT_ASSERT(a1 != t); - CPPUNIT_ASSERT(!(a1 == t)); - - a2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - - CPPUNIT_ASSERT(a2.functionName() == "AggregateColumn test"); - CPPUNIT_ASSERT(a2.data() == "agg1"); - CPPUNIT_ASSERT(a2.functionParms() != NULL); - CPPUNIT_ASSERT(*(a2.functionParms()) == s1); - - CPPUNIT_ASSERT(a1 == a2); - CPPUNIT_ASSERT(!(a1 != a2)); - CPPUNIT_ASSERT(a1 == t); - CPPUNIT_ASSERT(!(a1 != t)); - } - - void serializeOperator() - { - Operator o1, o2; - TreeNode* t; - ByteStream b; - - t = &o2; - - CPPUNIT_ASSERT(o1 == o2); - CPPUNIT_ASSERT(!(o1 != o2)); - CPPUNIT_ASSERT(o1 == t); - CPPUNIT_ASSERT(!(o1 != t)); - - o1.data("="); - - CPPUNIT_ASSERT(o1 != o2); - CPPUNIT_ASSERT(!(o1 == o2)); - CPPUNIT_ASSERT(o1 != t); - CPPUNIT_ASSERT(!(o1 == t)); - - o1.serialize(b); - - CPPUNIT_ASSERT(o2.data() == ""); - o2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(o2.data() == "="); - - CPPUNIT_ASSERT(o1 == o2); - CPPUNIT_ASSERT(!(o1 != o2)); - CPPUNIT_ASSERT(o1 == t); - CPPUNIT_ASSERT(!(o1 != t)); - } - - void serializeFilter() - { - Filter f1, f2; - TreeNode* t; - ByteStream b; - - t = &f2; - - CPPUNIT_ASSERT(f1 == f2); - CPPUNIT_ASSERT(!(f1 != f2)); - CPPUNIT_ASSERT(f1 == t); - CPPUNIT_ASSERT(!(f1 != t)); - - f1.data("Filter test"); - - CPPUNIT_ASSERT(f1 != f2); - CPPUNIT_ASSERT(!(f1 == f2)); - CPPUNIT_ASSERT(f1 != t); - CPPUNIT_ASSERT(!(f1 == t)); - - f1.serialize(b); - - CPPUNIT_ASSERT(f2.data() == ""); - f2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(f2.data() == "Filter test"); - - CPPUNIT_ASSERT(f1 == f2); - CPPUNIT_ASSERT(!(f1 != f2)); - CPPUNIT_ASSERT(f1 == t); - CPPUNIT_ASSERT(!(f1 != t)); - } - - void serializeFunctionColumn() - { - FunctionColumn fc1, fc2; - TreeNode* t; - ByteStream b; - - t = &fc2; - - CPPUNIT_ASSERT(fc1 == fc2); - CPPUNIT_ASSERT(!(fc1 != fc2)); - CPPUNIT_ASSERT(fc1 == t); - CPPUNIT_ASSERT(!(fc1 != t)); - - /* FunctionColumn */ - fc1.sessionID(0); - fc1.functionName("FunctionColumn test"); - fc1.functionParms("tpch.region.r_regionkey, tpch.nation.n_nationkey"); - fc1.data("fc1"); - - CPPUNIT_ASSERT(fc1 != fc2); - CPPUNIT_ASSERT(!(fc1 == fc2)); - CPPUNIT_ASSERT(fc1 != t); - CPPUNIT_ASSERT(!(fc1 == t)); - - FunctionColumn::FunctionParm functionParms; - functionParms = fc1.functionParms(); - CPPUNIT_ASSERT(functionParms.size() == 2); - - fc1.serialize(b); - - CPPUNIT_ASSERT(fc2.functionName() == ""); - CPPUNIT_ASSERT(fc2.functionParms().size() == 0); - - fc2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(fc2.functionName() == "FunctionColumn test"); - CPPUNIT_ASSERT(fc2.functionParms().size() == 2); - functionParms = fc2.functionParms(); - CPPUNIT_ASSERT(functionParms.size() == 2); - - CPPUNIT_ASSERT(fc1 == fc2); - CPPUNIT_ASSERT(!(fc1 != fc2)); - CPPUNIT_ASSERT(fc1 == t); - CPPUNIT_ASSERT(!(fc1 != t)); - } - - void serializeConstantColumn() - { - ConstantColumn c1, c2; - TreeNode* t; - ByteStream b; - - t = &c2; - - CPPUNIT_ASSERT(c1 == c2); - CPPUNIT_ASSERT(!(c1 != c2)); - CPPUNIT_ASSERT(c1 == t); - CPPUNIT_ASSERT(!(c1 != t)); - - c1.type(5); - c1.constval("ConstantColumn test"); - c1.data("c1"); - - CPPUNIT_ASSERT(c1 != c2); - CPPUNIT_ASSERT(!(c1 == c2)); - CPPUNIT_ASSERT(c1 != t); - CPPUNIT_ASSERT(!(c1 == t)); - - c1.serialize(b); - CPPUNIT_ASSERT(c2.constval() == ""); - c2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(c2.type() == 5); - CPPUNIT_ASSERT(c2.constval() == "ConstantColumn test"); - - CPPUNIT_ASSERT(c1 == c2); - CPPUNIT_ASSERT(!(c1 != c2)); - CPPUNIT_ASSERT(c1 == t); - CPPUNIT_ASSERT(!(c1 != t)); - } - - ParseTree* makeParseTree() - { - ParseTree* t[5]; - SimpleColumn* s[5]; - int i; - - /* ParseTree (ExpressionTree) */ - - /* - t0(s0) - / \ - s1 s2 - / \ - s3 s4 - */ - - for (i = 0; i < 5; i++) - { - t[i] = new ParseTree(NULL); - s[i] = new SimpleColumn(); - t[i]->data(s[i]); - } - - s[0]->schemaName("Schema Name 0"); - s[1]->schemaName("Schema Name 1"); - s[2]->schemaName("Schema Name 2"); - s[3]->schemaName("Schema Name 3"); - s[4]->schemaName("Schema Name 4"); - - t[0]->left(t[1]); - t[0]->right(t[2]); - t[1]->left(t[3]); - t[2]->right(t[4]); - - return t[0]; - } - - void verifyParseTree(const ParseTree* t) - { - const ParseTree* ct, *ct2; - SimpleColumn* s; - - CPPUNIT_ASSERT(t != NULL); - s = dynamic_cast(t->data()); - CPPUNIT_ASSERT(s != NULL); - CPPUNIT_ASSERT(s->schemaName() == "Schema Name 0"); - ct = t->left(); - CPPUNIT_ASSERT(ct != NULL); - s = dynamic_cast(ct->data()); - CPPUNIT_ASSERT(s != NULL); - CPPUNIT_ASSERT(s->schemaName() == "Schema Name 1"); - ct2 = ct->left(); - CPPUNIT_ASSERT(ct2 != NULL); - s = dynamic_cast(ct2->data()); - CPPUNIT_ASSERT(s != NULL); - CPPUNIT_ASSERT(s->schemaName() == "Schema Name 3"); - CPPUNIT_ASSERT(ct->right() == NULL); - CPPUNIT_ASSERT(ct2->left() == NULL); - CPPUNIT_ASSERT(ct2->right() == NULL); - ct = t->right(); - CPPUNIT_ASSERT(ct != NULL); - s = dynamic_cast(ct->data()); - CPPUNIT_ASSERT(s != NULL); - CPPUNIT_ASSERT(s->schemaName() == "Schema Name 2"); - ct2 = ct->right(); - CPPUNIT_ASSERT(ct2 != NULL); - s = dynamic_cast(ct2->data()); - CPPUNIT_ASSERT(s != NULL); - CPPUNIT_ASSERT(s->schemaName() == "Schema Name 4"); - CPPUNIT_ASSERT(ct->left() == NULL); - CPPUNIT_ASSERT(ct2->left() == NULL); - CPPUNIT_ASSERT(ct2->right() == NULL); - } - - void serializeParseTree() - { - ByteStream b; - ParseTree* t, *tmodel; - - t = makeParseTree(); - tmodel = makeParseTree(); - verifyParseTree(t); //sanity check on the test itself - - CPPUNIT_ASSERT(*t == *tmodel); - CPPUNIT_ASSERT(!(*t != *tmodel)); - - ObjectReader::writeParseTree(t, b); - - delete t; - - t = ObjectReader::createParseTree(b); - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(t != NULL); - verifyParseTree(t); - - CPPUNIT_ASSERT(*t == *tmodel); - CPPUNIT_ASSERT(!(*t != *tmodel)); - - delete t; - delete tmodel; - } - - ArithmeticColumn* makeArithmeticColumn() - { - ArithmeticColumn* ret; - ParseTree* t; - - t = makeParseTree(); - ret = new ArithmeticColumn(); - ret->expression(t); - ret->alias("ArithmeticColumn"); - ret->data("AD"); - return ret; - } - - void serializeArithmeticColumn() - { - ParseTree* t; - const ParseTree* ct; - ArithmeticColumn ac, ac2; - TreeNode* tn; - ByteStream b; - - tn = &ac2; - - CPPUNIT_ASSERT(ac == ac2); - CPPUNIT_ASSERT(!(ac != ac2)); - CPPUNIT_ASSERT(ac == tn); - CPPUNIT_ASSERT(!(ac != tn)); - - t = makeParseTree(); - - ac.expression(t); - ac.alias("ArithmeticColumn"); - ac.data("AD"); - - CPPUNIT_ASSERT(ac != ac2); - CPPUNIT_ASSERT(!(ac == ac2)); - CPPUNIT_ASSERT(ac != tn); - CPPUNIT_ASSERT(!(ac == tn)); - - ac.serialize(b); - ac2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - ct = ac2.expression(); - verifyParseTree(ct); - CPPUNIT_ASSERT(ac2.alias() == "ArithmeticColumn"); - CPPUNIT_ASSERT(ac2.data() == "AD"); - - CPPUNIT_ASSERT(ac == ac2); - CPPUNIT_ASSERT(!(ac != ac2)); - CPPUNIT_ASSERT(ac == tn); - CPPUNIT_ASSERT(!(ac != tn)); - } - - void serializeSimpleFilter() - { - ArithmeticColumn* pac1, *pac2; - const ArithmeticColumn* cpac1, *cpac2; - SimpleFilter sf1, sf2; - Operator* o1; - const Operator* co2; - TreeNode* t; - ByteStream b; - - t = &sf2; - - CPPUNIT_ASSERT(sf1 == sf2); - CPPUNIT_ASSERT(!(sf1 != sf2)); - CPPUNIT_ASSERT(sf1 == t); - CPPUNIT_ASSERT(!(sf1 != t)); - - pac1 = makeArithmeticColumn(); - pac2 = makeArithmeticColumn(); - - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(pac1 != NULL); - CPPUNIT_ASSERT(pac2 != NULL); - - o1 = new Operator("="); - sf1.lhs(pac1); - sf1.rhs(pac2); - sf1.op(o1); - - sf1.serialize(b); - - CPPUNIT_ASSERT(sf1 != sf2); - CPPUNIT_ASSERT(!(sf1 == sf2)); - CPPUNIT_ASSERT(sf1 != t); - CPPUNIT_ASSERT(!(sf1 == t)); - - sf2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - - cpac1 = dynamic_cast(sf2.lhs()); - CPPUNIT_ASSERT(cpac1 != NULL); - verifyParseTree(cpac1->expression()); - cpac2 = dynamic_cast(sf2.rhs()); - CPPUNIT_ASSERT(cpac2 != NULL); - verifyParseTree(cpac2->expression()); - co2 = sf2.op(); - CPPUNIT_ASSERT(co2 != NULL); - CPPUNIT_ASSERT(co2->data() == o1->data()); - - CPPUNIT_ASSERT(sf1 == sf2); - CPPUNIT_ASSERT(!(sf1 != sf2)); - CPPUNIT_ASSERT(sf1 == t); - CPPUNIT_ASSERT(!(sf1 != t)); - - } - - void serializeCSEP() - { - /* - * CalpontSelectExecutionPlan - * This is a large class; it makes more sense to write == operators - * for everything than to write a giant equivalance test here. - * For now this is mostly a regression test. - */ - - CalpontSelectExecutionPlan csep1, csep2; - CalpontSelectExecutionPlan::ReturnedColumnList colList; - ParseTree* filterList; - CalpontExecutionPlan* cep; - ByteStream b; - - cep = &csep2; - - CPPUNIT_ASSERT(csep1 == csep2); - CPPUNIT_ASSERT(!(csep1 != csep2)); - CPPUNIT_ASSERT(csep1 == cep); - CPPUNIT_ASSERT(!(csep1 != cep)); - - // returned columns - SimpleColumn* sc = new SimpleColumn("tpch.region.r_regionkey"); - colList.push_back(sc); - - // filters - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; - Token t; - - SimpleFilter* sf = new SimpleFilter(); - SimpleColumn* lhs = new SimpleColumn(*sc); - SimpleColumn* rhs = new SimpleColumn("tpch.nation.n_regionkey"); - Operator* op = new Operator("="); - - sf->op(op); - sf->lhs(lhs); - sf->rhs(rhs); - - t.value = sf; - tokens.push_back(t); - - Operator* op1 = new Operator ("and"); - t.value = op1; - tokens.push_back(t); - - SimpleFilter* sf1 = new SimpleFilter(); - SimpleColumn* lhs1 = new SimpleColumn (*rhs); - ConstantColumn* constCol = new ConstantColumn("3", ConstantColumn::NUM); - Operator* op2 = new Operator("!="); - - sf1->op(op2); - sf1->lhs(lhs1); - sf1->rhs(constCol); - - t.value = sf1; - tokens.push_back(t); - - filterList = parser.parse(tokens.begin(), tokens.end()); - - // draw filterList tree - filterList->drawTree("selectExecutionPlan_1.dot"); - - // calpont execution plan - csep1.returnedCols (colList); - csep1.filters (filterList); - - CPPUNIT_ASSERT(csep1 != csep2); - CPPUNIT_ASSERT(!(csep1 == csep2)); - CPPUNIT_ASSERT(csep1 != cep); - CPPUNIT_ASSERT(!(csep1 == cep)); - - csep1.serialize(b); - csep2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - - CPPUNIT_ASSERT(csep1 == csep2); - CPPUNIT_ASSERT(!(csep1 != csep2)); - CPPUNIT_ASSERT(csep1 == cep); - CPPUNIT_ASSERT(!(csep1 != cep)); - - CalpontSelectExecutionPlan csep3, csep4; - - // subselect - CalpontSelectExecutionPlan* subselect = new CalpontSelectExecutionPlan; - subselect->location(CalpontSelectExecutionPlan::WHERE); - subselect->dependent (false); - CPPUNIT_ASSERT (subselect->location() == CalpontSelectExecutionPlan::WHERE); - CPPUNIT_ASSERT (subselect->dependent() == false); - CalpontSelectExecutionPlan::SelectList selectList; - selectList.push_back(subselect); - csep3.subSelects(selectList); - - // exist filter - CalpontSelectExecutionPlan* cep1 = new CalpontSelectExecutionPlan(); - ExistsFilter* filter = new ExistsFilter(); - delete filter; - filter = new ExistsFilter(cep1); - filter->exists(cep1); - //CalpontSelectExecutionPlan* cep2 = const_cast(filter->exists()); - - CalpontSelectExecutionPlan::Parser parser1; - std::vector tokens1; - Token t1; - t1.value = filter; - tokens1.push_back(t1); - csep3.filters(parser1.parse(tokens1.begin(), tokens1.end())); - - csep3.serialize(b); - csep4.unserialize(b); - - CPPUNIT_ASSERT(csep3 == csep4); - CPPUNIT_ASSERT(!(csep3 != csep4)); - - - } - - void serializeSelectFilter() - { - ByteStream b; - ArithmeticColumn* pac1; - Operator* o1; - const Operator* co2; - CalpontSelectExecutionPlan csep1; - SelectFilter sel1, sel2; - const ArithmeticColumn* cpac1; - const ParseTree* ct; - TreeNode* t; - - t = &sel2; - - CPPUNIT_ASSERT(sel1 == sel2); - CPPUNIT_ASSERT(!(sel1 != sel2)); - CPPUNIT_ASSERT(sel1 == t); - CPPUNIT_ASSERT(!(sel1 != t)); - - pac1 = makeArithmeticColumn(); - o1 = new Operator("="); - sel1.lhs(pac1); - sel1.op(o1); - - CPPUNIT_ASSERT(sel1 != sel2); - CPPUNIT_ASSERT(!(sel1 == sel2)); - CPPUNIT_ASSERT(sel1 != t); - CPPUNIT_ASSERT(!(sel1 == t)); - - sel1.serialize(b); - sel2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - - CPPUNIT_ASSERT(sel1 == sel2); - CPPUNIT_ASSERT(!(sel1 != sel2)); - CPPUNIT_ASSERT(sel1 == t); - CPPUNIT_ASSERT(!(sel1 != t)); - - cpac1 = dynamic_cast(sel2.lhs()); - CPPUNIT_ASSERT(cpac1 != NULL); - ct = cpac1->expression(); - verifyParseTree(ct); - co2 = sel2.op(); - CPPUNIT_ASSERT(co2 != NULL); - CPPUNIT_ASSERT(co2->data() == o1->data()); - } - - /* ExistFilters get tested as part of the CSEP test at the moment. */ - void serializeExistsFilter() - { - ExistsFilter ef1, ef2; - ByteStream b; - - ef2.data("ExistsFilter test"); - ef1.serialize(b); - ef2.unserialize(b); - CPPUNIT_ASSERT(b.length() == 0); - CPPUNIT_ASSERT(ef2.data() == ""); - } - - void objectIDManager_1() - { - int oid, oidBase; - - // fake out the objmgr... - setenv("CALPONT_CONFIG_FILE", "/usr/local/mariadb/columnstore/etc/Columnstore.xml", 1); - Config* cf = Config::makeConfig(); - cf->setConfig("OIDManager", "OIDBitmapFile", "./oidbitmap"); - - try - { - ObjectIDManager o; - - OIDBitmapFilename = strdup(o.getFilename().c_str()); - oidBase = o.allocOID(); - oid = o.allocOID(); - CPPUNIT_ASSERT(oid == oidBase + 1); - oid = o.allocOIDs(20); - CPPUNIT_ASSERT(oid == oidBase + 2); - oid = o.allocOIDs(20); - CPPUNIT_ASSERT(oid == oidBase + 22); - o.returnOID(oidBase + 5); - oid = o.allocOID(); - CPPUNIT_ASSERT(oid == oidBase + 5); - o.returnOID(oidBase + 5); - oid = o.allocOIDs(20); - CPPUNIT_ASSERT(oid == oidBase + 42); - oid = o.allocOID(); - CPPUNIT_ASSERT(oid == oidBase + 5); - o.returnOIDs(oidBase, oidBase + 61); - oid = o.allocOID(); - CPPUNIT_ASSERT(oid == oidBase); - o.returnOID(0); - } - catch (...) - { - if (OIDBitmapFilename != NULL) - unlink(OIDBitmapFilename); // XXXPAT: fix this when libstdc++ regains its sanity - - throw; - } - - unlink(OIDBitmapFilename); + cerr << e.what() << endl; } + } + + void treeNodeImpl() + { + TreeNodeImpl* node1 = new TreeNodeImpl(); + TreeNodeImpl* node2 = new TreeNodeImpl("node2"); + CPPUNIT_ASSERT(node2->data() == "node2"); + CPPUNIT_ASSERT(*node1 != *node2); + + ByteStream b; + node2->serialize(b); + node1->unserialize(b); + CPPUNIT_ASSERT(*node1 == *node2); + + node2->data("node3"); + cout << *node2; + + TreeNodeImpl* node4 = node2->clone(); + CPPUNIT_ASSERT(*node2 == node4); + + delete node1; + delete node2; + delete node4; + } + + void serializeSimpleColumn() + { + SimpleColumn s1, s2; + TreeNode* t; + ByteStream b; + + t = &s2; + + CPPUNIT_ASSERT(s1 == s2); + CPPUNIT_ASSERT(!(s1 != s2)); + CPPUNIT_ASSERT(s1 == t); + CPPUNIT_ASSERT(!(s1 != t)); + + s1.schemaName("Schema Name 1"); + s1.tableName("Table Name 1"); + s1.columnName("Column Name 1"); + // s1.tcn(5); + s1.data("sc1"); + + s1.serialize(b); + + CPPUNIT_ASSERT(s2.schemaName() == ""); + CPPUNIT_ASSERT(s2.tableName() == ""); + CPPUNIT_ASSERT(s2.columnName() == ""); + // CPPUNIT_ASSERT(s2.tcn() == 0); + + CPPUNIT_ASSERT(s1 != s2); + CPPUNIT_ASSERT(s1 == s1); + CPPUNIT_ASSERT(s1 != t); + CPPUNIT_ASSERT(!(s1 == t)); + + s2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + + CPPUNIT_ASSERT(s2.schemaName() == "Schema Name 1"); + CPPUNIT_ASSERT(s2.tableName() == "Table Name 1"); + CPPUNIT_ASSERT(s2.columnName() == "Column Name 1"); + // CPPUNIT_ASSERT(s2.tcn() == 5); + CPPUNIT_ASSERT(s2.data() == "sc1"); + + CPPUNIT_ASSERT(s1 == s2); + CPPUNIT_ASSERT(!(s1 != s2)); + CPPUNIT_ASSERT(s1 == t); + CPPUNIT_ASSERT(!(s1 != t)); + } + + void serializeAggregateColumn() + { + AggregateColumn a1, a2; + SimpleColumn* s1; + TreeNode* t; + ByteStream b; + + t = &a2; + + s1 = new SimpleColumn(); + + CPPUNIT_ASSERT(a1 == a2); + CPPUNIT_ASSERT(!(a1 != a2)); + CPPUNIT_ASSERT(a1 == t); + CPPUNIT_ASSERT(!(a1 != t)); + + a1.functionName("AggregateColumn test"); + a1.data("agg1"); + a1.functionParms(s1); + + a1.serialize(b); + + CPPUNIT_ASSERT(a2.functionName() == ""); + CPPUNIT_ASSERT(a2.data() == ""); + CPPUNIT_ASSERT(a2.functionParms() == NULL); + + CPPUNIT_ASSERT(a1 != a2); + CPPUNIT_ASSERT(!(a1 == a2)); + CPPUNIT_ASSERT(a1 != t); + CPPUNIT_ASSERT(!(a1 == t)); + + a2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + + CPPUNIT_ASSERT(a2.functionName() == "AggregateColumn test"); + CPPUNIT_ASSERT(a2.data() == "agg1"); + CPPUNIT_ASSERT(a2.functionParms() != NULL); + CPPUNIT_ASSERT(*(a2.functionParms()) == s1); + + CPPUNIT_ASSERT(a1 == a2); + CPPUNIT_ASSERT(!(a1 != a2)); + CPPUNIT_ASSERT(a1 == t); + CPPUNIT_ASSERT(!(a1 != t)); + } + + void serializeOperator() + { + Operator o1, o2; + TreeNode* t; + ByteStream b; + + t = &o2; + + CPPUNIT_ASSERT(o1 == o2); + CPPUNIT_ASSERT(!(o1 != o2)); + CPPUNIT_ASSERT(o1 == t); + CPPUNIT_ASSERT(!(o1 != t)); + + o1.data("="); + + CPPUNIT_ASSERT(o1 != o2); + CPPUNIT_ASSERT(!(o1 == o2)); + CPPUNIT_ASSERT(o1 != t); + CPPUNIT_ASSERT(!(o1 == t)); + + o1.serialize(b); + + CPPUNIT_ASSERT(o2.data() == ""); + o2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(o2.data() == "="); + + CPPUNIT_ASSERT(o1 == o2); + CPPUNIT_ASSERT(!(o1 != o2)); + CPPUNIT_ASSERT(o1 == t); + CPPUNIT_ASSERT(!(o1 != t)); + } + + void serializeFilter() + { + Filter f1, f2; + TreeNode* t; + ByteStream b; + + t = &f2; + + CPPUNIT_ASSERT(f1 == f2); + CPPUNIT_ASSERT(!(f1 != f2)); + CPPUNIT_ASSERT(f1 == t); + CPPUNIT_ASSERT(!(f1 != t)); + + f1.data("Filter test"); + + CPPUNIT_ASSERT(f1 != f2); + CPPUNIT_ASSERT(!(f1 == f2)); + CPPUNIT_ASSERT(f1 != t); + CPPUNIT_ASSERT(!(f1 == t)); + + f1.serialize(b); + + CPPUNIT_ASSERT(f2.data() == ""); + f2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(f2.data() == "Filter test"); + + CPPUNIT_ASSERT(f1 == f2); + CPPUNIT_ASSERT(!(f1 != f2)); + CPPUNIT_ASSERT(f1 == t); + CPPUNIT_ASSERT(!(f1 != t)); + } + + void serializeFunctionColumn() + { + FunctionColumn fc1, fc2; + TreeNode* t; + ByteStream b; + + t = &fc2; + + CPPUNIT_ASSERT(fc1 == fc2); + CPPUNIT_ASSERT(!(fc1 != fc2)); + CPPUNIT_ASSERT(fc1 == t); + CPPUNIT_ASSERT(!(fc1 != t)); + + /* FunctionColumn */ + fc1.sessionID(0); + fc1.functionName("FunctionColumn test"); + fc1.functionParms("tpch.region.r_regionkey, tpch.nation.n_nationkey"); + fc1.data("fc1"); + + CPPUNIT_ASSERT(fc1 != fc2); + CPPUNIT_ASSERT(!(fc1 == fc2)); + CPPUNIT_ASSERT(fc1 != t); + CPPUNIT_ASSERT(!(fc1 == t)); + + FunctionColumn::FunctionParm functionParms; + functionParms = fc1.functionParms(); + CPPUNIT_ASSERT(functionParms.size() == 2); + + fc1.serialize(b); + + CPPUNIT_ASSERT(fc2.functionName() == ""); + CPPUNIT_ASSERT(fc2.functionParms().size() == 0); + + fc2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(fc2.functionName() == "FunctionColumn test"); + CPPUNIT_ASSERT(fc2.functionParms().size() == 2); + functionParms = fc2.functionParms(); + CPPUNIT_ASSERT(functionParms.size() == 2); + + CPPUNIT_ASSERT(fc1 == fc2); + CPPUNIT_ASSERT(!(fc1 != fc2)); + CPPUNIT_ASSERT(fc1 == t); + CPPUNIT_ASSERT(!(fc1 != t)); + } + + void serializeConstantColumn() + { + ConstantColumn c1, c2; + TreeNode* t; + ByteStream b; + + t = &c2; + + CPPUNIT_ASSERT(c1 == c2); + CPPUNIT_ASSERT(!(c1 != c2)); + CPPUNIT_ASSERT(c1 == t); + CPPUNIT_ASSERT(!(c1 != t)); + + c1.type(5); + c1.constval("ConstantColumn test"); + c1.data("c1"); + + CPPUNIT_ASSERT(c1 != c2); + CPPUNIT_ASSERT(!(c1 == c2)); + CPPUNIT_ASSERT(c1 != t); + CPPUNIT_ASSERT(!(c1 == t)); + + c1.serialize(b); + CPPUNIT_ASSERT(c2.constval() == ""); + c2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(c2.type() == 5); + CPPUNIT_ASSERT(c2.constval() == "ConstantColumn test"); + + CPPUNIT_ASSERT(c1 == c2); + CPPUNIT_ASSERT(!(c1 != c2)); + CPPUNIT_ASSERT(c1 == t); + CPPUNIT_ASSERT(!(c1 != t)); + } + + ParseTree* makeParseTree() + { + ParseTree* t[5]; + SimpleColumn* s[5]; + int i; + + /* ParseTree (ExpressionTree) */ /* - * destroySemaphores() and destroyShmseg() will print error messages - * if there are no objects to destroy. That's OK. + t0(s0) + / \ + s1 s2 + / \ + s3 s4 + */ + + for (i = 0; i < 5; i++) + { + t[i] = new ParseTree(NULL); + s[i] = new SimpleColumn(); + t[i]->data(s[i]); + } + + s[0]->schemaName("Schema Name 0"); + s[1]->schemaName("Schema Name 1"); + s[2]->schemaName("Schema Name 2"); + s[3]->schemaName("Schema Name 3"); + s[4]->schemaName("Schema Name 4"); + + t[0]->left(t[1]); + t[0]->right(t[2]); + t[1]->left(t[3]); + t[2]->right(t[4]); + + return t[0]; + } + + void verifyParseTree(const ParseTree* t) + { + const ParseTree *ct, *ct2; + SimpleColumn* s; + + CPPUNIT_ASSERT(t != NULL); + s = dynamic_cast(t->data()); + CPPUNIT_ASSERT(s != NULL); + CPPUNIT_ASSERT(s->schemaName() == "Schema Name 0"); + ct = t->left(); + CPPUNIT_ASSERT(ct != NULL); + s = dynamic_cast(ct->data()); + CPPUNIT_ASSERT(s != NULL); + CPPUNIT_ASSERT(s->schemaName() == "Schema Name 1"); + ct2 = ct->left(); + CPPUNIT_ASSERT(ct2 != NULL); + s = dynamic_cast(ct2->data()); + CPPUNIT_ASSERT(s != NULL); + CPPUNIT_ASSERT(s->schemaName() == "Schema Name 3"); + CPPUNIT_ASSERT(ct->right() == NULL); + CPPUNIT_ASSERT(ct2->left() == NULL); + CPPUNIT_ASSERT(ct2->right() == NULL); + ct = t->right(); + CPPUNIT_ASSERT(ct != NULL); + s = dynamic_cast(ct->data()); + CPPUNIT_ASSERT(s != NULL); + CPPUNIT_ASSERT(s->schemaName() == "Schema Name 2"); + ct2 = ct->right(); + CPPUNIT_ASSERT(ct2 != NULL); + s = dynamic_cast(ct2->data()); + CPPUNIT_ASSERT(s != NULL); + CPPUNIT_ASSERT(s->schemaName() == "Schema Name 4"); + CPPUNIT_ASSERT(ct->left() == NULL); + CPPUNIT_ASSERT(ct2->left() == NULL); + CPPUNIT_ASSERT(ct2->right() == NULL); + } + + void serializeParseTree() + { + ByteStream b; + ParseTree *t, *tmodel; + + t = makeParseTree(); + tmodel = makeParseTree(); + verifyParseTree(t); // sanity check on the test itself + + CPPUNIT_ASSERT(*t == *tmodel); + CPPUNIT_ASSERT(!(*t != *tmodel)); + + ObjectReader::writeParseTree(t, b); + + delete t; + + t = ObjectReader::createParseTree(b); + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(t != NULL); + verifyParseTree(t); + + CPPUNIT_ASSERT(*t == *tmodel); + CPPUNIT_ASSERT(!(*t != *tmodel)); + + delete t; + delete tmodel; + } + + ArithmeticColumn* makeArithmeticColumn() + { + ArithmeticColumn* ret; + ParseTree* t; + + t = makeParseTree(); + ret = new ArithmeticColumn(); + ret->expression(t); + ret->alias("ArithmeticColumn"); + ret->data("AD"); + return ret; + } + + void serializeArithmeticColumn() + { + ParseTree* t; + const ParseTree* ct; + ArithmeticColumn ac, ac2; + TreeNode* tn; + ByteStream b; + + tn = &ac2; + + CPPUNIT_ASSERT(ac == ac2); + CPPUNIT_ASSERT(!(ac != ac2)); + CPPUNIT_ASSERT(ac == tn); + CPPUNIT_ASSERT(!(ac != tn)); + + t = makeParseTree(); + + ac.expression(t); + ac.alias("ArithmeticColumn"); + ac.data("AD"); + + CPPUNIT_ASSERT(ac != ac2); + CPPUNIT_ASSERT(!(ac == ac2)); + CPPUNIT_ASSERT(ac != tn); + CPPUNIT_ASSERT(!(ac == tn)); + + ac.serialize(b); + ac2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + ct = ac2.expression(); + verifyParseTree(ct); + CPPUNIT_ASSERT(ac2.alias() == "ArithmeticColumn"); + CPPUNIT_ASSERT(ac2.data() == "AD"); + + CPPUNIT_ASSERT(ac == ac2); + CPPUNIT_ASSERT(!(ac != ac2)); + CPPUNIT_ASSERT(ac == tn); + CPPUNIT_ASSERT(!(ac != tn)); + } + + void serializeSimpleFilter() + { + ArithmeticColumn *pac1, *pac2; + const ArithmeticColumn *cpac1, *cpac2; + SimpleFilter sf1, sf2; + Operator* o1; + const Operator* co2; + TreeNode* t; + ByteStream b; + + t = &sf2; + + CPPUNIT_ASSERT(sf1 == sf2); + CPPUNIT_ASSERT(!(sf1 != sf2)); + CPPUNIT_ASSERT(sf1 == t); + CPPUNIT_ASSERT(!(sf1 != t)); + + pac1 = makeArithmeticColumn(); + pac2 = makeArithmeticColumn(); + + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(pac1 != NULL); + CPPUNIT_ASSERT(pac2 != NULL); + + o1 = new Operator("="); + sf1.lhs(pac1); + sf1.rhs(pac2); + sf1.op(o1); + + sf1.serialize(b); + + CPPUNIT_ASSERT(sf1 != sf2); + CPPUNIT_ASSERT(!(sf1 == sf2)); + CPPUNIT_ASSERT(sf1 != t); + CPPUNIT_ASSERT(!(sf1 == t)); + + sf2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + + cpac1 = dynamic_cast(sf2.lhs()); + CPPUNIT_ASSERT(cpac1 != NULL); + verifyParseTree(cpac1->expression()); + cpac2 = dynamic_cast(sf2.rhs()); + CPPUNIT_ASSERT(cpac2 != NULL); + verifyParseTree(cpac2->expression()); + co2 = sf2.op(); + CPPUNIT_ASSERT(co2 != NULL); + CPPUNIT_ASSERT(co2->data() == o1->data()); + + CPPUNIT_ASSERT(sf1 == sf2); + CPPUNIT_ASSERT(!(sf1 != sf2)); + CPPUNIT_ASSERT(sf1 == t); + CPPUNIT_ASSERT(!(sf1 != t)); + } + + void serializeCSEP() + { + /* + * CalpontSelectExecutionPlan + * This is a large class; it makes more sense to write == operators + * for everything than to write a giant equivalance test here. + * For now this is mostly a regression test. */ - void destroySemaphores() + + CalpontSelectExecutionPlan csep1, csep2; + CalpontSelectExecutionPlan::ReturnedColumnList colList; + ParseTree* filterList; + CalpontExecutionPlan* cep; + ByteStream b; + + cep = &csep2; + + CPPUNIT_ASSERT(csep1 == csep2); + CPPUNIT_ASSERT(!(csep1 != csep2)); + CPPUNIT_ASSERT(csep1 == cep); + CPPUNIT_ASSERT(!(csep1 != cep)); + + // returned columns + SimpleColumn* sc = new SimpleColumn("tpch.region.r_regionkey"); + colList.push_back(sc); + + // filters + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; + Token t; + + SimpleFilter* sf = new SimpleFilter(); + SimpleColumn* lhs = new SimpleColumn(*sc); + SimpleColumn* rhs = new SimpleColumn("tpch.nation.n_regionkey"); + Operator* op = new Operator("="); + + sf->op(op); + sf->lhs(lhs); + sf->rhs(rhs); + + t.value = sf; + tokens.push_back(t); + + Operator* op1 = new Operator("and"); + t.value = op1; + tokens.push_back(t); + + SimpleFilter* sf1 = new SimpleFilter(); + SimpleColumn* lhs1 = new SimpleColumn(*rhs); + ConstantColumn* constCol = new ConstantColumn("3", ConstantColumn::NUM); + Operator* op2 = new Operator("!="); + + sf1->op(op2); + sf1->lhs(lhs1); + sf1->rhs(constCol); + + t.value = sf1; + tokens.push_back(t); + + filterList = parser.parse(tokens.begin(), tokens.end()); + + // draw filterList tree + filterList->drawTree("selectExecutionPlan_1.dot"); + + // calpont execution plan + csep1.returnedCols(colList); + csep1.filters(filterList); + + CPPUNIT_ASSERT(csep1 != csep2); + CPPUNIT_ASSERT(!(csep1 == csep2)); + CPPUNIT_ASSERT(csep1 != cep); + CPPUNIT_ASSERT(!(csep1 == cep)); + + csep1.serialize(b); + csep2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + + CPPUNIT_ASSERT(csep1 == csep2); + CPPUNIT_ASSERT(!(csep1 != csep2)); + CPPUNIT_ASSERT(csep1 == cep); + CPPUNIT_ASSERT(!(csep1 != cep)); + + CalpontSelectExecutionPlan csep3, csep4; + + // subselect + CalpontSelectExecutionPlan* subselect = new CalpontSelectExecutionPlan; + subselect->location(CalpontSelectExecutionPlan::WHERE); + subselect->dependent(false); + CPPUNIT_ASSERT(subselect->location() == CalpontSelectExecutionPlan::WHERE); + CPPUNIT_ASSERT(subselect->dependent() == false); + CalpontSelectExecutionPlan::SelectList selectList; + selectList.push_back(subselect); + csep3.subSelects(selectList); + + // exist filter + CalpontSelectExecutionPlan* cep1 = new CalpontSelectExecutionPlan(); + ExistsFilter* filter = new ExistsFilter(); + delete filter; + filter = new ExistsFilter(cep1); + filter->exists(cep1); + // CalpontSelectExecutionPlan* cep2 = const_cast(filter->exists()); + + CalpontSelectExecutionPlan::Parser parser1; + std::vector tokens1; + Token t1; + t1.value = filter; + tokens1.push_back(t1); + csep3.filters(parser1.parse(tokens1.begin(), tokens1.end())); + + csep3.serialize(b); + csep4.unserialize(b); + + CPPUNIT_ASSERT(csep3 == csep4); + CPPUNIT_ASSERT(!(csep3 != csep4)); + } + + void serializeSelectFilter() + { + ByteStream b; + ArithmeticColumn* pac1; + Operator* o1; + const Operator* co2; + CalpontSelectExecutionPlan csep1; + SelectFilter sel1, sel2; + const ArithmeticColumn* cpac1; + const ParseTree* ct; + TreeNode* t; + + t = &sel2; + + CPPUNIT_ASSERT(sel1 == sel2); + CPPUNIT_ASSERT(!(sel1 != sel2)); + CPPUNIT_ASSERT(sel1 == t); + CPPUNIT_ASSERT(!(sel1 != t)); + + pac1 = makeArithmeticColumn(); + o1 = new Operator("="); + sel1.lhs(pac1); + sel1.op(o1); + + CPPUNIT_ASSERT(sel1 != sel2); + CPPUNIT_ASSERT(!(sel1 == sel2)); + CPPUNIT_ASSERT(sel1 != t); + CPPUNIT_ASSERT(!(sel1 == t)); + + sel1.serialize(b); + sel2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + + CPPUNIT_ASSERT(sel1 == sel2); + CPPUNIT_ASSERT(!(sel1 != sel2)); + CPPUNIT_ASSERT(sel1 == t); + CPPUNIT_ASSERT(!(sel1 != t)); + + cpac1 = dynamic_cast(sel2.lhs()); + CPPUNIT_ASSERT(cpac1 != NULL); + ct = cpac1->expression(); + verifyParseTree(ct); + co2 = sel2.op(); + CPPUNIT_ASSERT(co2 != NULL); + CPPUNIT_ASSERT(co2->data() == o1->data()); + } + + /* ExistFilters get tested as part of the CSEP test at the moment. */ + void serializeExistsFilter() + { + ExistsFilter ef1, ef2; + ByteStream b; + + ef2.data("ExistsFilter test"); + ef1.serialize(b); + ef2.unserialize(b); + CPPUNIT_ASSERT(b.length() == 0); + CPPUNIT_ASSERT(ef2.data() == ""); + } + + void objectIDManager_1() + { + int oid, oidBase; + + // fake out the objmgr... + setenv("CALPONT_CONFIG_FILE", "/usr/local/mariadb/columnstore/etc/Columnstore.xml", 1); + Config* cf = Config::makeConfig(); + cf->setConfig("OIDManager", "OIDBitmapFile", "./oidbitmap"); + + try { - key_t semkey; - int sems, err; + ObjectIDManager o; - semkey = SESSIONMANAGER_SYSVKEY; - sems = semget(semkey, 2, 0666); + OIDBitmapFilename = strdup(o.getFilename().c_str()); + oidBase = o.allocOID(); + oid = o.allocOID(); + CPPUNIT_ASSERT(oid == oidBase + 1); + oid = o.allocOIDs(20); + CPPUNIT_ASSERT(oid == oidBase + 2); + oid = o.allocOIDs(20); + CPPUNIT_ASSERT(oid == oidBase + 22); + o.returnOID(oidBase + 5); + oid = o.allocOID(); + CPPUNIT_ASSERT(oid == oidBase + 5); + o.returnOID(oidBase + 5); + oid = o.allocOIDs(20); + CPPUNIT_ASSERT(oid == oidBase + 42); + oid = o.allocOID(); + CPPUNIT_ASSERT(oid == oidBase + 5); + o.returnOIDs(oidBase, oidBase + 61); + oid = o.allocOID(); + CPPUNIT_ASSERT(oid == oidBase); + o.returnOID(0); + } + catch (...) + { + if (OIDBitmapFilename != NULL) + unlink(OIDBitmapFilename); // XXXPAT: fix this when libstdc++ regains its sanity - if (sems != -1) - { - err = semctl(sems, 0, IPC_RMID); - - if (err == -1) - perror("tdriver: semctl"); - } + throw; } - void destroyShmseg() + unlink(OIDBitmapFilename); + } + + /* + * destroySemaphores() and destroyShmseg() will print error messages + * if there are no objects to destroy. That's OK. + */ + void destroySemaphores() + { + key_t semkey; + int sems, err; + + semkey = SESSIONMANAGER_SYSVKEY; + sems = semget(semkey, 2, 0666); + + if (sems != -1) { - key_t shmkey; - int shms, err; + err = semctl(sems, 0, IPC_RMID); - shmkey = SESSIONMANAGER_SYSVKEY; - shms = shmget(shmkey, 0, 0666); - - if (shms != -1) - { - err = shmctl(shms, IPC_RMID, NULL); - - if (err == -1 && errno != EINVAL) - { - perror("tdriver: shmctl"); - return; - } - } + if (err == -1) + perror("tdriver: semctl"); } + } - void sessionManager_1() + void destroyShmseg() + { + key_t shmkey; + int shms, err; + + shmkey = SESSIONMANAGER_SYSVKEY; + shms = shmget(shmkey, 0, 0666); + + if (shms != -1) { - SessionManager* sm = NULL; - SessionManager::TxnID txn; - const SessionManager::SIDTIDEntry* activeTxns; - int len; - string filename; + err = shmctl(shms, IPC_RMID, NULL); -// destroySemaphores(); -// destroyShmseg(); + if (err == -1 && errno != EINVAL) + { + perror("tdriver: shmctl"); + return; + } + } + } - try - { - sm = new SessionManager(); - //CPPUNIT_ASSERT(sm->verID() == 0); - filename = sm->getTxnIDFilename(); - delete sm; - sm = new SessionManager(); - txn = sm->newTxnID(0); - CPPUNIT_ASSERT(txn.valid == true); -// CPPUNIT_ASSERT(txn.id == 1); -// CPPUNIT_ASSERT(sm->verID() == 1); - txn = sm->newTxnID(1); - CPPUNIT_ASSERT(txn.valid == true); -// CPPUNIT_ASSERT(txn.id == 2); -// CPPUNIT_ASSERT(sm->verID() == 2); - activeTxns = sm->SIDTIDMap(len); - CPPUNIT_ASSERT(activeTxns != NULL); - CPPUNIT_ASSERT(len == 2); - txn = sm->getTxnID(0); - CPPUNIT_ASSERT(txn.valid == true); -// CPPUNIT_ASSERT(txn.id == 1); - CPPUNIT_ASSERT(txn.valid == activeTxns[0].txnid.valid); -// CPPUNIT_ASSERT(txn.id == activeTxns[0].txnid.id); - txn = sm->getTxnID(1); - CPPUNIT_ASSERT(txn.valid == true); -// CPPUNIT_ASSERT(txn.id == 2); - CPPUNIT_ASSERT(txn.valid == activeTxns[1].txnid.valid); -// CPPUNIT_ASSERT(txn.id == activeTxns[1].txnid.id); - delete [] activeTxns; + void sessionManager_1() + { + SessionManager* sm = NULL; + SessionManager::TxnID txn; + const SessionManager::SIDTIDEntry* activeTxns; + int len; + string filename; - //make sure it's consistent across invocations - delete sm; - sm = new SessionManager(); - activeTxns = sm->SIDTIDMap(len); - CPPUNIT_ASSERT(activeTxns != NULL); - CPPUNIT_ASSERT(len == 2); - txn = sm->getTxnID(0); - CPPUNIT_ASSERT(txn.valid == true); -// CPPUNIT_ASSERT(txn.id == 1); - CPPUNIT_ASSERT(txn.valid == activeTxns[0].txnid.valid); -// CPPUNIT_ASSERT(txn.id == activeTxns[0].txnid.id); - txn = sm->getTxnID(1); - CPPUNIT_ASSERT(txn.valid == true); -// CPPUNIT_ASSERT(txn.id == 2); - CPPUNIT_ASSERT(txn.valid == activeTxns[1].txnid.valid); -// CPPUNIT_ASSERT(txn.id == activeTxns[1].txnid.id); - sm->rolledback(txn); - CPPUNIT_ASSERT(txn.valid == false); - txn = sm->getTxnID(0); - sm->committed(txn); - CPPUNIT_ASSERT(txn.valid == false); - delete [] activeTxns; - activeTxns = sm->SIDTIDMap(len); - CPPUNIT_ASSERT(len == 0); - delete [] activeTxns; - } - catch (runtime_error& e) - { - cout << "caught runtime_error (why doesn't cppunit notice these?): " << e.what() << endl; + // destroySemaphores(); + // destroyShmseg(); - if (sm != NULL) - delete sm; + try + { + sm = new SessionManager(); + // CPPUNIT_ASSERT(sm->verID() == 0); + filename = sm->getTxnIDFilename(); + delete sm; + sm = new SessionManager(); + txn = sm->newTxnID(0); + CPPUNIT_ASSERT(txn.valid == true); + // CPPUNIT_ASSERT(txn.id == 1); + // CPPUNIT_ASSERT(sm->verID() == 1); + txn = sm->newTxnID(1); + CPPUNIT_ASSERT(txn.valid == true); + // CPPUNIT_ASSERT(txn.id == 2); + // CPPUNIT_ASSERT(sm->verID() == 2); + activeTxns = sm->SIDTIDMap(len); + CPPUNIT_ASSERT(activeTxns != NULL); + CPPUNIT_ASSERT(len == 2); + txn = sm->getTxnID(0); + CPPUNIT_ASSERT(txn.valid == true); + // CPPUNIT_ASSERT(txn.id == 1); + CPPUNIT_ASSERT(txn.valid == activeTxns[0].txnid.valid); + // CPPUNIT_ASSERT(txn.id == activeTxns[0].txnid.id); + txn = sm->getTxnID(1); + CPPUNIT_ASSERT(txn.valid == true); + // CPPUNIT_ASSERT(txn.id == 2); + CPPUNIT_ASSERT(txn.valid == activeTxns[1].txnid.valid); + // CPPUNIT_ASSERT(txn.id == activeTxns[1].txnid.id); + delete[] activeTxns; -// destroySemaphores(); -// destroyShmseg(); - throw logic_error("Hey! Stop!"); - } - catch (exception& e) - { - cout << "caught exception: " << e.what() << endl; - - if (sm != NULL) - delete sm; - -// destroySemaphores(); -// destroyShmseg(); - throw; - } + // make sure it's consistent across invocations + delete sm; + sm = new SessionManager(); + activeTxns = sm->SIDTIDMap(len); + CPPUNIT_ASSERT(activeTxns != NULL); + CPPUNIT_ASSERT(len == 2); + txn = sm->getTxnID(0); + CPPUNIT_ASSERT(txn.valid == true); + // CPPUNIT_ASSERT(txn.id == 1); + CPPUNIT_ASSERT(txn.valid == activeTxns[0].txnid.valid); + // CPPUNIT_ASSERT(txn.id == activeTxns[0].txnid.id); + txn = sm->getTxnID(1); + CPPUNIT_ASSERT(txn.valid == true); + // CPPUNIT_ASSERT(txn.id == 2); + CPPUNIT_ASSERT(txn.valid == activeTxns[1].txnid.valid); + // CPPUNIT_ASSERT(txn.id == activeTxns[1].txnid.id); + sm->rolledback(txn); + CPPUNIT_ASSERT(txn.valid == false); + txn = sm->getTxnID(0); + sm->committed(txn); + CPPUNIT_ASSERT(txn.valid == false); + delete[] activeTxns; + activeTxns = sm->SIDTIDMap(len); + CPPUNIT_ASSERT(len == 0); + delete[] activeTxns; + } + catch (runtime_error& e) + { + cout << "caught runtime_error (why doesn't cppunit notice these?): " << e.what() << endl; + if (sm != NULL) delete sm; -// destroySemaphores(); -// destroyShmseg(); + + // destroySemaphores(); + // destroyShmseg(); + throw logic_error("Hey! Stop!"); + } + catch (exception& e) + { + cout << "caught exception: " << e.what() << endl; + + if (sm != NULL) + delete sm; + + // destroySemaphores(); + // destroyShmseg(); + throw; } - /** Verifies that we can only have MaxTxns (1000 right now) active transactions at - any given time */ - void sessionManager_2() + delete sm; + // destroySemaphores(); + // destroyShmseg(); + } + + /** Verifies that we can only have MaxTxns (1000 right now) active transactions at + any given time */ + void sessionManager_2() + { + int i; + SessionManager* sm; + SessionManager::TxnID txns[1001]; + string filename; + + // destroySemaphores(); + // destroyShmseg(); + + sm = new SessionManager(); + filename = sm->getTxnIDFilename(); + delete sm; + sm = new SessionManager(); + + for (i = 0; i < 1000; i++) { - int i; - SessionManager* sm; - SessionManager::TxnID txns[1001]; - string filename; - -// destroySemaphores(); -// destroyShmseg(); - - sm = new SessionManager(); - filename = sm->getTxnIDFilename(); - delete sm; - sm = new SessionManager(); - - for (i = 0; i < 1000; i++) - { - txns[i] = sm->newTxnID(i, false); - CPPUNIT_ASSERT(txns[i].valid == true); - //CPPUNIT_ASSERT(sm->verID() == txns[i].id); - } - - txns[1000] = sm->newTxnID(i, false); - CPPUNIT_ASSERT(txns[1000].valid == false); - - for (i = 999; i >= 0; i--) - { - SessionManager::TxnID tmp = sm->getTxnID(i); - CPPUNIT_ASSERT(tmp.valid == txns[i].valid == true); - CPPUNIT_ASSERT(tmp.id == txns[i].id); - sm->committed(txns[i]); - tmp = sm->getTxnID(i); - CPPUNIT_ASSERT(tmp.valid == false); - CPPUNIT_ASSERT(txns[i].valid == false); - } - - try - { - sm->committed(txns[1000]); - } - // expected exception - catch (invalid_argument& e) - { } - - delete sm; -// destroySemaphores(); -// destroyShmseg(); + txns[i] = sm->newTxnID(i, false); + CPPUNIT_ASSERT(txns[i].valid == true); + // CPPUNIT_ASSERT(sm->verID() == txns[i].id); } - /** Verifies that transaction IDs get saved and restored correctly across "reboots" */ - void sessionManager_3() + txns[1000] = sm->newTxnID(i, false); + CPPUNIT_ASSERT(txns[1000].valid == false); + + for (i = 999; i >= 0; i--) { - - SessionManager* sm; - string filename; - SessionManager::TxnID txnid; - - // scrub env - sm = new SessionManager(); - filename = sm->getTxnIDFilename(); - delete sm; -// destroyShmseg(); -// destroySemaphores(); - - sm = new SessionManager(); - - txnid = sm->newTxnID(0); -// CPPUNIT_ASSERT(txnid.id == 1); - - delete sm; -// destroyShmseg(); -// destroySemaphores(); - - sm = new SessionManager(); - sm->committed(txnid); - txnid = sm->newTxnID(1); -// CPPUNIT_ASSERT(txnid.id == 2); - - delete sm; - sm = new SessionManager(); - sm->committed(txnid); - -// destroyShmseg(); -// destroySemaphores(); - + SessionManager::TxnID tmp = sm->getTxnID(i); + CPPUNIT_ASSERT(tmp.valid == txns[i].valid == true); + CPPUNIT_ASSERT(tmp.id == txns[i].id); + sm->committed(txns[i]); + tmp = sm->getTxnID(i); + CPPUNIT_ASSERT(tmp.valid == false); + CPPUNIT_ASSERT(txns[i].valid == false); } - void sessionManager_4() + try + { + sm->committed(txns[1000]); + } + // expected exception + catch (invalid_argument& e) { - char* buf; - int len; - SessionManager sm; - - buf = sm.getShmContents(len); - CPPUNIT_ASSERT(len > 0); - CPPUNIT_ASSERT(buf != NULL); - delete [] buf; } + delete sm; + // destroySemaphores(); + // destroyShmseg(); + } - SessionManager* manager; - SessionManager::TxnID managerTxns[1000]; + /** Verifies that transaction IDs get saved and restored correctly across "reboots" */ + void sessionManager_3() + { + SessionManager* sm; + string filename; + SessionManager::TxnID txnid; - int createTxns(const int& start, const int& end) + // scrub env + sm = new SessionManager(); + filename = sm->getTxnIDFilename(); + delete sm; + // destroyShmseg(); + // destroySemaphores(); + + sm = new SessionManager(); + + txnid = sm->newTxnID(0); + // CPPUNIT_ASSERT(txnid.id == 1); + + delete sm; + // destroyShmseg(); + // destroySemaphores(); + + sm = new SessionManager(); + sm->committed(txnid); + txnid = sm->newTxnID(1); + // CPPUNIT_ASSERT(txnid.id == 2); + + delete sm; + sm = new SessionManager(); + sm->committed(txnid); + + // destroyShmseg(); + // destroySemaphores(); + } + + void sessionManager_4() + { + char* buf; + int len; + SessionManager sm; + + buf = sm.getShmContents(len); + CPPUNIT_ASSERT(len > 0); + CPPUNIT_ASSERT(buf != NULL); + delete[] buf; + } + + SessionManager* manager; + SessionManager::TxnID managerTxns[1000]; + + int createTxns(const int& start, const int& end) + { + const int first = start; + const int last = end; + int newTxns = 0; + int verifyLen = 0; + + verifyLen = manager->verifySize(); + + for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++) { - - const int first = start; - const int last = end; - int newTxns = 0; - int verifyLen = 0; - - verifyLen = manager->verifySize(); - - for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++) - { - managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000); - CPPUNIT_ASSERT(managerTxns[idx].id > 0); - CPPUNIT_ASSERT(managerTxns[idx].valid == true); - verifyLen = manager->verifySize(); - newTxns++; - } - - CPPUNIT_ASSERT(newTxns == last - first); - return newTxns; + managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000); + CPPUNIT_ASSERT(managerTxns[idx].id > 0); + CPPUNIT_ASSERT(managerTxns[idx].valid == true); + verifyLen = manager->verifySize(); + newTxns++; } - int closeTxns(const int& start, const int& end) + CPPUNIT_ASSERT(newTxns == last - first); + return newTxns; + } + + int closeTxns(const int& start, const int& end) + { + int first = start; + int last = end; + int totalClosed = 0; + + for (int idx = first; idx < last; idx++) { + try + { + SessionManager::TxnID tmp = manager->getTxnID(idx + 1000); - int first = start; - int last = end; - int totalClosed = 0; - - for (int idx = first; idx < last ; idx++) + if (tmp.valid == true) { - try - { - SessionManager::TxnID tmp = manager->getTxnID(idx + 1000); - - if (tmp.valid == true) - { - manager->committed(tmp); - CPPUNIT_ASSERT(tmp.valid == false); - totalClosed++; - } - - } - catch (exception& e) - { - cerr << e.what() << endl; - continue; - } + manager->committed(tmp); + CPPUNIT_ASSERT(tmp.valid == false); + totalClosed++; } + } + catch (exception& e) + { + cerr << e.what() << endl; + continue; + } + } - return totalClosed; + return totalClosed; - } //closeTxns + } // closeTxns - void MonitorTestPlan_1() + void MonitorTestPlan_1() + { + int currStartTxn = 0; + int currEndTxn = 5; + int txnCntIncr = 5; + const int sleepTime = 11; + const int iterMax = 1; + vector toTxns; + + destroySemaphores(); + destroyShmseg(); + + manager = new SessionManager(); + manager->reset(); + CPPUNIT_ASSERT(manager->verifySize() == 0); + + SessionMonitor* monitor = NULL; + + for (int jdx = 0; jdx < iterMax; jdx++) { + // store the current state of the SessionManager + monitor = new SessionMonitor(); + delete monitor; + int idx = 0; + int grpStart = currStartTxn; - int currStartTxn = 0; - int currEndTxn = 5; - int txnCntIncr = 5; - const int sleepTime = 11; - const int iterMax = 1; - vector toTxns; + for (idx = 0; idx < 3; idx++) + { + createTxns(currStartTxn, currEndTxn); + CPPUNIT_ASSERT(manager->verifySize() == (idx + 1) * txnCntIncr); - destroySemaphores(); - destroyShmseg(); - - manager = new SessionManager(); - manager->reset(); - CPPUNIT_ASSERT(manager->verifySize() == 0); - - SessionMonitor* monitor = NULL; - - for (int jdx = 0; jdx < iterMax; jdx++) - { - - // store the current state of the SessionManager - monitor = new SessionMonitor(); - delete monitor; - int idx = 0; - int grpStart = currStartTxn; - - for (idx = 0; idx < 3; idx++ ) - { - - createTxns(currStartTxn, currEndTxn); - CPPUNIT_ASSERT(manager->verifySize() == (idx + 1)*txnCntIncr); - - currStartTxn += txnCntIncr; - currEndTxn += txnCntIncr; - sleep(sleepTime); //make sessions time out - - monitor = new SessionMonitor(); // read Monitor data - toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx); - - delete monitor; - } - - int grpEnd = currEndTxn; - monitor = new SessionMonitor(); - closeTxns(grpStart, grpEnd); // close this iteration of txns - CPPUNIT_ASSERT(manager->verifySize() == 0); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == 0); - - delete monitor; - - } - - monitor = new SessionMonitor(); // readload Monitor data + currStartTxn += txnCntIncr; + currEndTxn += txnCntIncr; + sleep(sleepTime); // make sessions time out + monitor = new SessionMonitor(); // read Monitor data toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == 0); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx); + delete monitor; + } - CPPUNIT_ASSERT(manager->verifySize() == 0); - - if (manager) - delete manager; + int grpEnd = currEndTxn; + monitor = new SessionMonitor(); + closeTxns(grpStart, grpEnd); // close this iteration of txns + CPPUNIT_ASSERT(manager->verifySize() == 0); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == 0); + delete monitor; } + monitor = new SessionMonitor(); // readload Monitor data + + toTxns.clear(); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == 0); + delete monitor; + + CPPUNIT_ASSERT(manager->verifySize() == 0); + + if (manager) + delete manager; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver1.cpp b/dbcon/execplan/tdriver1.cpp index 170028f48..0cabed35e 100644 --- a/dbcon/execplan/tdriver1.cpp +++ b/dbcon/execplan/tdriver1.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -46,27 +46,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ l_returnflag,\ l_linestatus,\ @@ -89,95 +88,90 @@ public: l_returnflag,\ l_linestatus;"; + CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan csep; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + SimpleColumn* c1 = new SimpleColumn("tpch.lineitem.l_returnflag"); + returnedColumnList.push_back(c1); - SimpleColumn* c1 = new SimpleColumn("tpch.lineitem.l_returnflag"); - returnedColumnList.push_back(c1); + SimpleColumn* c2 = new SimpleColumn("tpch.lineitem.l_linestatus"); + returnedColumnList.push_back(c2); - SimpleColumn* c2 = new SimpleColumn("tpch.lineitem.l_linestatus"); - returnedColumnList.push_back(c2); + ArithmeticColumn* c3 = new ArithmeticColumn("sum(tpch.lineitem.l_quantity)"); + c3->alias("sum_qty"); + returnedColumnList.push_back(c3); - ArithmeticColumn* c3 = new ArithmeticColumn("sum(tpch.lineitem.l_quantity)"); - c3->alias("sum_qty"); - returnedColumnList.push_back(c3); + ArithmeticColumn* c4 = new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice)"); + c4->alias("sum_base_price"); + returnedColumnList.push_back(c4); - ArithmeticColumn* c4 = new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice)"); - c4->alias("sum_base_price"); - returnedColumnList.push_back(c4); + ArithmeticColumn* c5 = + new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * (1 - tpch.lineitem.l_discount))"); + c5->alias("sum_disc_price"); + returnedColumnList.push_back(c5); - ArithmeticColumn* c5 = new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * (1 - tpch.lineitem.l_discount))"); - c5->alias("sum_disc_price"); - returnedColumnList.push_back(c5); + ArithmeticColumn* c6 = new ArithmeticColumn("avg(tpch.lineitem.l_quantity)"); + c6->alias("avg_qty"); + returnedColumnList.push_back(c6); - ArithmeticColumn* c6 = new ArithmeticColumn("avg(tpch.lineitem.l_quantity)"); - c6->alias("avg_qty"); - returnedColumnList.push_back(c6); + ArithmeticColumn* c7 = new ArithmeticColumn("avg(tpch.lineitem.l_extendedprice)"); + c6->alias("avg_price"); + returnedColumnList.push_back(c7); - ArithmeticColumn* c7 = new ArithmeticColumn("avg(tpch.lineitem.l_extendedprice)"); - c6->alias("avg_price"); - returnedColumnList.push_back(c7); + ArithmeticColumn* c8 = new ArithmeticColumn("avg(tpch.lineitem.l_discount)"); + c8->alias("avg_disc"); + returnedColumnList.push_back(c8); - ArithmeticColumn* c8 = new ArithmeticColumn("avg(tpch.lineitem.l_discount)"); - c8->alias("avg_disc"); - returnedColumnList.push_back(c8); + // count(*) -> count(ALL) + ArithmeticColumn* c9 = new ArithmeticColumn("count(ALL)"); + c9->alias("count_order"); + returnedColumnList.push_back(c9); - // count(*) -> count(ALL) - ArithmeticColumn* c9 = new ArithmeticColumn("count(ALL)"); - c9->alias("count_order"); - returnedColumnList.push_back(c9); + csep.returnedCols(returnedColumnList); - csep.returnedCols(returnedColumnList); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ArithmeticColumn("date('1998-12-01')")); + filterTokenList.push_back(f1); + csep.filterTokenList(filterTokenList); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator("<="), - new SimpleColumn("tpch.lineitem.l_shipdate"), - new ArithmeticColumn("date('1998-12-01')")); - filterTokenList.push_back(f1); - csep.filterTokenList(filterTokenList); + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + SimpleColumn* g1 = new SimpleColumn(*c1); + groupByList.push_back(g1); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn(*c1); - groupByList.push_back(g1); + SimpleColumn* g2 = new SimpleColumn(*c2); + groupByList.push_back(g2); - SimpleColumn* g2 = new SimpleColumn(*c2); - groupByList.push_back(g2); + csep.groupByCols(groupByList); - csep.groupByCols (groupByList); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + SimpleColumn* o1 = new SimpleColumn(*g1); + orderByList.push_back(o1); - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - SimpleColumn* o1 = new SimpleColumn(*g1); - orderByList.push_back(o1); - - SimpleColumn* o2 = new SimpleColumn(*g2); - orderByList.push_back(o2); - - csep.orderByCols(orderByList); - - cout << csep; - } + SimpleColumn* o2 = new SimpleColumn(*g2); + orderByList.push_back(o2); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver10.cpp b/dbcon/execplan/tdriver10.cpp index db0a2d077..e9847ab91 100644 --- a/dbcon/execplan/tdriver10.cpp +++ b/dbcon/execplan/tdriver10.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -46,27 +46,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ c_custkey,\ c_name,\ @@ -99,117 +98,107 @@ public: order by\ revenue desc;"; + CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan csep; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + SimpleColumn* c1 = new SimpleColumn("tpch.customer.l_returnflag"); + returnedColumnList.push_back(c1); - SimpleColumn* c1 = new SimpleColumn("tpch.customer.l_returnflag"); - returnedColumnList.push_back(c1); + SimpleColumn* c2 = new SimpleColumn("tpch.customer.c_name"); + returnedColumnList.push_back(c2); - SimpleColumn* c2 = new SimpleColumn("tpch.customer.c_name"); - returnedColumnList.push_back(c2); + ArithmeticColumn* c3 = + new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice*(1-tpch.lineitem.l_discount))"); + c3->alias("revenue"); + returnedColumnList.push_back(c3); - ArithmeticColumn* c3 = new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice*(1-tpch.lineitem.l_discount))"); - c3->alias("revenue"); - returnedColumnList.push_back(c3); + SimpleColumn* c4 = new SimpleColumn("tpch.customer.c_acctbal"); + returnedColumnList.push_back(c4); - SimpleColumn* c4 = new SimpleColumn("tpch.customer.c_acctbal"); - returnedColumnList.push_back(c4); + SimpleColumn* c5 = new SimpleColumn("tpch.nation.n_name"); + returnedColumnList.push_back(c5); - SimpleColumn* c5 = new SimpleColumn("tpch.nation.n_name"); - returnedColumnList.push_back(c5); + SimpleColumn* c6 = new SimpleColumn("tpch.customer.c_address"); + returnedColumnList.push_back(c6); - SimpleColumn* c6 = new SimpleColumn("tpch.customer.c_address"); - returnedColumnList.push_back(c6); + SimpleColumn* c7 = new SimpleColumn("tpch.customer.c_phone"); + returnedColumnList.push_back(c7); - SimpleColumn* c7 = new SimpleColumn("tpch.customer.c_phone"); - returnedColumnList.push_back(c7); + SimpleColumn* c8 = new SimpleColumn("tpch.costomer.c_comment"); + returnedColumnList.push_back(c8); - SimpleColumn* c8 = new SimpleColumn("tpch.costomer.c_comment"); - returnedColumnList.push_back(c8); + csep.returnedCols(returnedColumnList); - csep.returnedCols(returnedColumnList); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_custkey"), + new SimpleColumn("tpch.orders.o_custkey")); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_custkey"), - new SimpleColumn("tpch.orders.o_custkey")); - filterTokenList.push_back(f1); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_orderkey"), + new SimpleColumn("tpch.orders.o_orderkey")); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.lineitem.l_orderkey"), - new SimpleColumn("tpch.orders.o_orderkey")); - filterTokenList.push_back(f2); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f3 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.orders.o_orderdate"), + new ArithmeticColumn("date(':1')")); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter (new Operator(">="), - new SimpleColumn("tpch.orders.o_orderdate"), - new ArithmeticColumn("date(':1')")); - filterTokenList.push_back(f3); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f4 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.orders.o_orderdate"), + new ArithmeticColumn("date(':1') + interval ('3', month)")); + filterTokenList.push_back(f4); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f4 = new SimpleFilter (new Operator("<"), - new SimpleColumn("tpch.orders.o_orderdate"), - new ArithmeticColumn("date(':1') + interval ('3', month)")); - filterTokenList.push_back(f4); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f5 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_returnflag"), + new ConstantColumn("R")); + filterTokenList.push_back(f5); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.lineitem.l_returnflag"), - new ConstantColumn("R")); - filterTokenList.push_back(f5); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f6 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + filterTokenList.push_back(f6); - SimpleFilter* f6 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_nationkey"), - new SimpleColumn("tpch.nation.n_nationkey")); - filterTokenList.push_back(f6); + csep.filterTokenList(filterTokenList); - csep.filterTokenList(filterTokenList); + ParseTree* pt = const_cast(csep.filters()); + pt->drawTree("q10.dot"); - ParseTree* pt = const_cast(csep.filters()); - pt->drawTree ("q10.dot"); - - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - groupByList.push_back(c1->clone()); - groupByList.push_back(c2->clone()); - groupByList.push_back(c4->clone()); - groupByList.push_back(c7->clone()); - groupByList.push_back(c5->clone()); - groupByList.push_back(c6->clone()); - groupByList.push_back(c8->clone()); - csep.groupByCols (groupByList); - - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - ArithmeticColumn* o1 = new ArithmeticColumn(*c3); - o1->asc(false); - orderByList.push_back(o1); - csep.orderByCols(orderByList); - - cout << csep; - } + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + groupByList.push_back(c1->clone()); + groupByList.push_back(c2->clone()); + groupByList.push_back(c4->clone()); + groupByList.push_back(c7->clone()); + groupByList.push_back(c5->clone()); + groupByList.push_back(c6->clone()); + groupByList.push_back(c8->clone()); + csep.groupByCols(groupByList); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + ArithmeticColumn* o1 = new ArithmeticColumn(*c3); + o1->asc(false); + orderByList.push_back(o1); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver11.cpp b/dbcon/execplan/tdriver11.cpp index 49be9f64b..e6a0eb60f 100644 --- a/dbcon/execplan/tdriver11.cpp +++ b/dbcon/execplan/tdriver11.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ ps_partkey,\ sum(ps_supplycost * ps_availqty) as value\ @@ -96,116 +95,106 @@ public: order by\ value desc;"; - CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan csep; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - SimpleColumn* c1 = new SimpleColumn("tpch.partsupp.ps_partkey"); - returnedColumnList.push_back(c1); + SimpleColumn* c1 = new SimpleColumn("tpch.partsupp.ps_partkey"); + returnedColumnList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn("sum(tpch.partsupp.ps_supplycost*tpch.partsupp.ps_availqty)"); - c2->alias ("value"); - returnedColumnList.push_back(c2); + ArithmeticColumn* c2 = new ArithmeticColumn("sum(tpch.partsupp.ps_supplycost*tpch.partsupp.ps_availqty)"); + c2->alias("value"); + returnedColumnList.push_back(c2); - csep.returnedCols(returnedColumnList); + csep.returnedCols(returnedColumnList); - // Where filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* sf1 = new SimpleFilter( new Operator("="), - new SimpleColumn("tpch.partsupp.ps_suppkey"), - new SimpleColumn("tpch.supplier.s_suppkey") ); - filterTokenList.push_back(sf1); - filterTokenList.push_back( new Operator ("and")); + // Where filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.partsupp.ps_suppkey"), + new SimpleColumn("tpch.supplier.s_suppkey")); + filterTokenList.push_back(sf1); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* sf2 = new SimpleFilter( new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - new SimpleColumn("tpch.nation.n_nationkey") ); - filterTokenList.push_back(sf2); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* sf2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + filterTokenList.push_back(sf2); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* sf3 = new SimpleFilter( new Operator("="), - new SimpleColumn ("tpch.nation.n_name"), - new ConstantColumn (":1")); - filterTokenList.push_back(sf3); - csep.filterTokenList(filterTokenList); + SimpleFilter* sf3 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_name"), new ConstantColumn(":1")); + filterTokenList.push_back(sf3); + csep.filterTokenList(filterTokenList); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn ("tpch.partsupp.ps_partkey"); - groupByList.push_back(g1); + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + SimpleColumn* g1 = new SimpleColumn("tpch.partsupp.ps_partkey"); + groupByList.push_back(g1); - // Having - CalpontSelectExecutionPlan::FilterTokenList havingTokenList; + // Having + CalpontSelectExecutionPlan::FilterTokenList havingTokenList; - ArithmeticColumn* hc = new ArithmeticColumn ("sum(tpch.partsupp.ps_supplycost*tpch.partsupp.ps_availqty)"); + ArithmeticColumn* hc = new ArithmeticColumn("sum(tpch.partsupp.ps_supplycost*tpch.partsupp.ps_availqty)"); - // sub select in having - CalpontSelectExecutionPlan* subsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::HAVING); + // sub select in having + CalpontSelectExecutionPlan* subsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::HAVING); - // subselect returned columns - CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; - ArithmeticColumn* sc1 = new ArithmeticColumn("sum(tpch.partsupp.ps_supplycost*tpch.partsupp.ps_availqty)*':2'"); - subReturnedColList.push_back(sc1); + // subselect returned columns + CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; + ArithmeticColumn* sc1 = + new ArithmeticColumn("sum(tpch.partsupp.ps_supplycost*tpch.partsupp.ps_availqty)*':2'"); + subReturnedColList.push_back(sc1); - subsep->returnedCols(subReturnedColList); + subsep->returnedCols(subReturnedColList); - // subselect filters - CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + // subselect filters + CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SimpleFilter* subsf1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.partsupp.ps_suppkey"), - new SimpleColumn("tpch.supplier.s_suppkey")); - subFilterTokenList.push_back(subsf1); + SimpleFilter* subsf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.partsupp.ps_suppkey"), + new SimpleColumn("tpch.supplier.s_suppkey")); + subFilterTokenList.push_back(subsf1); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* subsf2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - new SimpleColumn("tpch.nation.n_nationkey")); - subFilterTokenList.push_back(subsf2); - subFilterTokenList.push_back(new Operator("and")); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* subsf2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + subFilterTokenList.push_back(subsf2); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* subsf3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.nation.n_name"), - new ConstantColumn(":1")); - subFilterTokenList.push_back(subsf3); + SimpleFilter* subsf3 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_name"), new ConstantColumn(":1")); + subFilterTokenList.push_back(subsf3); - subsep->filterTokenList(subFilterTokenList); + subsep->filterTokenList(subFilterTokenList); - SelectFilter* sef = new SelectFilter (hc, new Operator (">"), subsep); - havingTokenList.push_back (sef); + SelectFilter* sef = new SelectFilter(hc, new Operator(">"), subsep); + havingTokenList.push_back(sef); - csep.havingTokenList (havingTokenList); + csep.havingTokenList(havingTokenList); - //ParseTree* pt = const_cast(subsep->filters()); - //pt->drawTree("q7.dot"); + // ParseTree* pt = const_cast(subsep->filters()); + // pt->drawTree("q7.dot"); - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - ArithmeticColumn* o1 = new ArithmeticColumn(*c2); - orderByList.push_back(o1); - - csep.orderByCols(orderByList); - - cout << csep; - } + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + ArithmeticColumn* o1 = new ArithmeticColumn(*c2); + orderByList.push_back(o1); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver12.cpp b/dbcon/execplan/tdriver12.cpp index dd9dc623a..a433f1eab 100644 --- a/dbcon/execplan/tdriver12.cpp +++ b/dbcon/execplan/tdriver12.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -48,102 +48,99 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_12); - CPPUNIT_TEST( selectExecutionPlan_12 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_12() - { + cout << endl; + } -// cout << "SQL: select r_regionkey from region, nation where n_regionkey = r_regionkey and n_regionkey = 2;" << endl; - cout << - "SQL: select \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_12() + { + // cout << "SQL: select r_regionkey from region, nation where n_regionkey = r_regionkey and + // n_regionkey = 2;" << endl; + cout << "SQL: select \ l_shipmode, \ sum(case \ when o_orderpriority = '1-URGENT' \ @@ -170,121 +167,111 @@ where \ group by \ l_shipmode \ order by \ - l_shipmode;" << endl; + l_shipmode;" + << endl; + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList colList; + CalpontSelectExecutionPlan::GroupByColumnList groupbyList; + CalpontSelectExecutionPlan::OrderByColumnList orderbyList; + ParseTree* filterList; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList colList; - CalpontSelectExecutionPlan::GroupByColumnList groupbyList; - CalpontSelectExecutionPlan::OrderByColumnList orderbyList; - ParseTree* filterList; + // returned columns + SimpleColumn l_shipmode("tpch.lineitem.l_shipmode"); + AggregateColumn* high_line_count = new AggregateColumn(); + high_line_count->functionName("sum"); + ArithmeticColumn* a1 = new ArithmeticColumn( + "( CASE WHEN o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' THEN 1 ELSE 0 END )"); + high_line_count->functionParms(a1); - // returned columns - SimpleColumn l_shipmode("tpch.lineitem.l_shipmode"); - AggregateColumn* high_line_count = new AggregateColumn(); - high_line_count->functionName("sum"); - ArithmeticColumn* a1 = new ArithmeticColumn ("( CASE WHEN o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' THEN 1 ELSE 0 END )"); - high_line_count->functionParms(a1); + AggregateColumn* low_line_count = new AggregateColumn(); + low_line_count->functionName("sum"); + ArithmeticColumn* a2 = new ArithmeticColumn( + "( CASE WHEN o_orderpriority <> '1-URGENT' and o_orderpriority <> '2-HIGH' THEN 1 ELSE 0 END )"); + low_line_count->functionParms(a2); - AggregateColumn* low_line_count = new AggregateColumn(); - low_line_count->functionName("sum"); - ArithmeticColumn* a2 = new ArithmeticColumn ("( CASE WHEN o_orderpriority <> '1-URGENT' and o_orderpriority <> '2-HIGH' THEN 1 ELSE 0 END )"); - low_line_count->functionParms(a2); + // Create the Projection + colList.push_back(high_line_count); + colList.push_back(low_line_count); + // Filter columns + SimpleColumn l_shipmode0("tpch.lineitem.l_shipmode"); + SimpleColumn l_shipmode1("tpch.lineitem.l_shipmode"); + SimpleColumn l_commitdate("tpch.lineitem.l_commitdate"); + SimpleColumn l_receiptdate("tpch.lineitem.l_receiptdate"); + SimpleColumn l_receiptdate1("tpch.lineitem.l_receiptdate"); + SimpleColumn l_shipdate("tpch.lineitem.l_shipdate"); - // Create the Projection - colList.push_back(high_line_count); - colList.push_back(low_line_count); + // filters + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; - // Filter columns - SimpleColumn l_shipmode0("tpch.lineitem.l_shipmode"); - SimpleColumn l_shipmode1("tpch.lineitem.l_shipmode"); - SimpleColumn l_commitdate("tpch.lineitem.l_commitdate"); - SimpleColumn l_receiptdate("tpch.lineitem.l_receiptdate"); - SimpleColumn l_receiptdate1("tpch.lineitem.l_receiptdate"); - SimpleColumn l_shipdate("tpch.lineitem.l_shipdate"); + // tokens.push_back(Token(new Operator("("))); + // tokens.push_back(Token(new Operator(")"))); + tokens.push_back( + Token(new SimpleFilter(new Operator("="), new SimpleColumn(l_shipmode), new ConstantColumn("1")))); - // filters - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; + tokens.push_back(Token(new Operator("and"))); - //tokens.push_back(Token(new Operator("("))); - //tokens.push_back(Token(new Operator(")"))); + tokens.push_back( + Token(new SimpleFilter(new Operator("="), new SimpleColumn(l_shipmode), new ConstantColumn("2")))); - tokens.push_back(Token(new SimpleFilter(new Operator("="), - new SimpleColumn(l_shipmode), - new ConstantColumn("1")))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token(new SimpleFilter(new Operator("<"), new SimpleColumn(l_commitdate), + new SimpleColumn(l_receiptdate)))); - tokens.push_back(Token(new SimpleFilter(new Operator("="), - new SimpleColumn(l_shipmode), - new ConstantColumn("2")))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token( + new SimpleFilter(new Operator("<"), new SimpleColumn(l_shipdate), new SimpleColumn(l_commitdate)))); - tokens.push_back(Token(new SimpleFilter(new Operator("<"), - new SimpleColumn(l_commitdate), - new SimpleColumn(l_receiptdate)))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token(new SimpleFilter(new Operator(">="), new SimpleColumn(l_receiptdate), + new ConstantColumn("06/01/2005")))); - tokens.push_back(Token(new SimpleFilter(new Operator("<"), - new SimpleColumn(l_shipdate), - new SimpleColumn(l_commitdate)))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token(new SimpleFilter(new Operator("<"), new SimpleColumn(l_receiptdate), + new ConstantColumn("06/01/2006")))); - tokens.push_back(Token(new SimpleFilter(new Operator(">="), - new SimpleColumn(l_receiptdate), - new ConstantColumn("06/01/2005")))); + // old below + filterList = parser.parse(tokens.begin(), tokens.end()); - tokens.push_back(Token(new Operator("and"))); + // draw filterList tree + filterList->drawTree("selectExecutionPlan_12.dot"); - tokens.push_back(Token(new SimpleFilter(new Operator("<"), - new SimpleColumn(l_receiptdate), - new ConstantColumn("06/01/2006")))); + // Build Group By List + SimpleColumn* l_shipmode2 = new SimpleColumn("tpch.lineitem.l_shipmode"); + groupbyList.push_back(l_shipmode2); - // old below - filterList = parser.parse(tokens.begin(), tokens.end()); + // Build Order By List + // SimpleColumn *l_shipmode2 = new SimpleColumn("tpch.lineitem.l_shipmode"); + orderbyList.push_back(l_shipmode2); - // draw filterList tree - filterList->drawTree("selectExecutionPlan_12.dot"); - - // Build Group By List - SimpleColumn* l_shipmode2 = new SimpleColumn("tpch.lineitem.l_shipmode"); - groupbyList.push_back(l_shipmode2); - - // Build Order By List - // SimpleColumn *l_shipmode2 = new SimpleColumn("tpch.lineitem.l_shipmode"); - orderbyList.push_back(l_shipmode2); - - // calpont execution plan - csep.returnedCols (colList); - csep.filters (filterList); - cout << "\nCalpont Execution Plan:" << endl; - cout << csep << endl; - cout << " --- end of test 12 ---" << endl; - - filterList->walk(walkfnString); - - } + // calpont execution plan + csep.returnedCols(colList); + csep.filters(filterList); + cout << "\nCalpont Execution Plan:" << endl; + cout << csep << endl; + cout << " --- end of test 12 ---" << endl; + filterList->walk(walkfnString); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver13.cpp b/dbcon/execplan/tdriver13.cpp index 9ee48eedc..eceb8d730 100644 --- a/dbcon/execplan/tdriver13.cpp +++ b/dbcon/execplan/tdriver13.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -51,101 +51,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_13); - CPPUNIT_TEST( selectExecutionPlan_13 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_13() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_13() + { + cout << "SQL: \ select \ c_count, \ count(*) as custdist \ @@ -166,123 +162,117 @@ group by \ order by \ custdist desc,\ c_count desc;" - << endl; + << endl; -//x ------------The create view statement is being re-written as an inline view ----- x/ + // x ------------The create view statement is being re-written as an inline view ----- x/ - //This is the main query body for query 16 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 16 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.c_count"); - c1->tableAlias("c_orders"); - colList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn("count(ALL)"); - c2->alias("custdist"); - colList.push_back(c2); + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.c_count"); + c1->tableAlias("c_orders"); + colList.push_back(c1); + ArithmeticColumn* c2 = new ArithmeticColumn("count(ALL)"); + c2->alias("custdist"); + colList.push_back(c2); - csep->returnedCols(colList); // set Projection columns + csep->returnedCols(colList); // set Projection columns - // Filter columns (Not needed for outer query) - //CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns (Not needed for outer query) + // CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - /* --------------- sub select begins here ---------------- */ - //Sub select filter - CalpontSelectExecutionPlan* subcsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); + /* --------------- sub select begins here ---------------- */ + // Sub select filter + CalpontSelectExecutionPlan* subcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList subColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList subColList; - SimpleColumn* sc1 = new SimpleColumn ("tpch.customer.c_custkey"); - subColList.push_back(sc1); - ArithmeticColumn* sc2 = new ArithmeticColumn ("count(tpch.orders.o_orderkey)"); - sc2->alias("c_count"); - subColList.push_back(sc2); - // Append returned columns to subselect - subcsep->returnedCols(subColList); + SimpleColumn* sc1 = new SimpleColumn("tpch.customer.c_custkey"); + subColList.push_back(sc1); + ArithmeticColumn* sc2 = new ArithmeticColumn("count(tpch.orders.o_orderkey)"); + sc2->alias("c_count"); + subColList.push_back(sc2); + // Append returned columns to subselect + subcsep->returnedCols(subColList); - //subselect Filters - CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; - SimpleFilter* sf1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.customer.c_custkey"), - new SimpleColumn ("tpch.orders.o_custkey")); - subcsep_filterlist.push_back(sf1); + // subselect Filters + CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_custkey"), + new SimpleColumn("tpch.orders.o_custkey")); + subcsep_filterlist.push_back(sf1); - subcsep_filterlist.push_back(new Operator("and")); + subcsep_filterlist.push_back(new Operator("and")); - SimpleFilter* sf2 = new SimpleFilter ( new Operator("NOT LIKE"), - new SimpleColumn ("tpch.orders.o_comment"), - new ConstantColumn("%:1%:2%")); - subcsep_filterlist.push_back(sf2); + SimpleFilter* sf2 = new SimpleFilter(new Operator("NOT LIKE"), new SimpleColumn("tpch.orders.o_comment"), + new ConstantColumn("%:1%:2%")); + subcsep_filterlist.push_back(sf2); - subcsep->filterTokenList(subcsep_filterlist); //Set Filter Columns + subcsep->filterTokenList(subcsep_filterlist); // Set Filter Columns - // sub Group By List - CalpontSelectExecutionPlan::GroupByColumnList subcsep_groupbyList; - SimpleColumn* sg1 = new SimpleColumn("tpch.customer.c_custkey"); - subcsep_groupbyList.push_back(sg1); - subcsep->groupByCols(subcsep_groupbyList); //Set GroupBy columns - subcsep->tableAlias("c_orders"); + // sub Group By List + CalpontSelectExecutionPlan::GroupByColumnList subcsep_groupbyList; + SimpleColumn* sg1 = new SimpleColumn("tpch.customer.c_custkey"); + subcsep_groupbyList.push_back(sg1); + subcsep->groupByCols(subcsep_groupbyList); // Set GroupBy columns + subcsep->tableAlias("c_orders"); - // Draw the subselect tree. - ParseTree* pt = const_cast(subcsep->filters()); - pt->drawTree("selectExecutionPlan_13_subquery.dot"); + // Draw the subselect tree. + ParseTree* pt = const_cast(subcsep->filters()); + pt->drawTree("selectExecutionPlan_13_subquery.dot"); - // x --------------- sub select begins here ---------------- x/ + // x --------------- sub select begins here ---------------- x/ - CalpontSelectExecutionPlan::SelectList subselectlist; - subselectlist.push_back(subcsep); - csep->subSelects(subselectlist); + CalpontSelectExecutionPlan::SelectList subselectlist; + subselectlist.push_back(subcsep); + csep->subSelects(subselectlist); + // Build Group By List + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + // SimpleColumn *g1 = new SimpleColumn("calpont.FROMTABLE.c_count"); + SimpleColumn* g1 = new SimpleColumn(*c1); + g1->alias("c_count"); + cout << "\n I am testing \n\n" + << "g1=" << *g1 << "\n end of data\n" + << endl; + csep_groupbyList.push_back(g1); + csep->groupByCols(csep_groupbyList); // Set GroupBy columns - // Build Group By List - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - //SimpleColumn *g1 = new SimpleColumn("calpont.FROMTABLE.c_count"); - SimpleColumn* g1 = new SimpleColumn(*c1); - g1->alias("c_count"); - cout << "\n I am testing \n\n" << "g1=" << *g1 << "\n end of data\n" << endl; - csep_groupbyList.push_back(g1); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns + // Order By List + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + // ArithmeticColumn *o1 = new ArithmeticColumn("count(ALL)"); + ArithmeticColumn* o1 = new ArithmeticColumn(*c2); + o1->alias("custdist"); + o1->asc(false); + csep_orderbyList.push_back(o1); + SimpleColumn* o2 = new SimpleColumn(*c1); + o2->alias("c_count"); + o2->asc(false); + csep_orderbyList.push_back(o2); + csep->orderByCols(csep_orderbyList); // Set OrderBy columns - // Order By List - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - //ArithmeticColumn *o1 = new ArithmeticColumn("count(ALL)"); - ArithmeticColumn* o1 = new ArithmeticColumn(*c2); - o1->alias("custdist"); - o1->asc(false); - csep_orderbyList.push_back(o1); - SimpleColumn* o2 = new SimpleColumn(*c1); - o2->alias("c_count"); - o2->asc(false); - csep_orderbyList.push_back(o2); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns + // SimpleColumn *abc = new SimpleColumn(*c1); - //SimpleColumn *abc = new SimpleColumn(*c1); - - //filterList->walk(walkfnString); ?? - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 13 ---" << endl; - - } + // filterList->walk(walkfnString); ?? + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 13 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver14.cpp b/dbcon/execplan/tdriver14.cpp index b2f171d7c..4f5924281 100644 --- a/dbcon/execplan/tdriver14.cpp +++ b/dbcon/execplan/tdriver14.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -48,101 +48,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan14); - CPPUNIT_TEST( selectExecutionPlan14 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan14() - { + cout << endl; + } - cout << - "SQL: select \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan14() + { + cout << "SQL: select \ 100.00 * sum(decode(substring(p_type from 1 for 5),'%PROMO', \ l_extendedprice * ( 1 - l_discount), 0)) / \ sum(l_extendedprice * (1 - l_discount)) as promo_revenue \ @@ -153,101 +149,94 @@ where \ ;_partkey = p_partkey \ and l_shipdate >= '[DATE]' \ and l_receiptdate < '[DATE]' + interval '1' month;" - << endl; - /* ---- This was the orginal query. The Program is using the alternative tpch Q14 code. - cout << - "SQL: select \ - 100.00 * \ - sum(case \ - when p_type like '%PROMO%' \ - then l_extendedprice * ( 1 - l_discount) \ - else 0 \ - end ) / sum (l_extendedprice * (1 - l_discount)) as promo_revenue \ - from \ - lineitem \ - part \ - where \ - ;_partkey = p_partkey \ - and l_shipdate >= '1995-09-01' \ - and l_receiptdate < '1995-0901' + interval '1' month;" - << endl; - ------- end of comment line --------- */ + << endl; + /* ---- This was the orginal query. The Program is using the alternative tpch Q14 code. + cout << + "SQL: select \ + 100.00 * \ + sum(case \ + when p_type like '%PROMO%' \ + then l_extendedprice * ( 1 - l_discount) \ + else 0 \ + end ) / sum (l_extendedprice * (1 - l_discount)) as promo_revenue \ + from \ + lineitem \ + part \ + where \ + ;_partkey = p_partkey \ + and l_shipdate >= '1995-09-01' \ + and l_receiptdate < '1995-0901' + interval '1' month;" + << endl; + ------- end of comment line --------- */ + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList colList; + CalpontSelectExecutionPlan::GroupByColumnList groupbyList; + CalpontSelectExecutionPlan::OrderByColumnList orderbyList; + ParseTree* filterList; - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList colList; - CalpontSelectExecutionPlan::GroupByColumnList groupbyList; - CalpontSelectExecutionPlan::OrderByColumnList orderbyList; - ParseTree* filterList; - - // returned columns - //SimpleColumn l_shipmode("tpch.lineitem.l_shipmode"); - AggregateColumn* promo_revenue = new AggregateColumn(); - promo_revenue->functionName("sum"); - ArithmeticColumn* a3 = new ArithmeticColumn ("100.00 * sum(decode(substring(p_type from 1 for 5),'%PROMO', \ + // returned columns + // SimpleColumn l_shipmode("tpch.lineitem.l_shipmode"); + AggregateColumn* promo_revenue = new AggregateColumn(); + promo_revenue->functionName("sum"); + ArithmeticColumn* a3 = new ArithmeticColumn( + "100.00 * sum(decode(substring(p_type from 1 for 5),'%PROMO', \ l_extendedprice * ( 1 - l_discount), 0)) / sum(l_extendedprice * (1 - l_discount))"); - promo_revenue->functionParms(a3); + promo_revenue->functionParms(a3); - // Create the Projection - colList.push_back(promo_revenue); + // Create the Projection + colList.push_back(promo_revenue); - // Filter columns - SimpleColumn l_partkey("tpch.lineitem.l_shipmode"); - SimpleColumn l_shipdate("tpch.lineitem.l_shipdate"); - //SimpleColumn l_shipdate("tpch.lineitem.l_shipmode"); - //SimpleColumn l_commitdate("tpch.lineitem.l_commitdate"); - //SimpleColumn l_receiptdate("tpch.lineitem.l_receiptdate"); - //SimpleColumn l_receiptdate1("tpch.lineitem.l_receiptdate"); + // Filter columns + SimpleColumn l_partkey("tpch.lineitem.l_shipmode"); + SimpleColumn l_shipdate("tpch.lineitem.l_shipdate"); + // SimpleColumn l_shipdate("tpch.lineitem.l_shipmode"); + // SimpleColumn l_commitdate("tpch.lineitem.l_commitdate"); + // SimpleColumn l_receiptdate("tpch.lineitem.l_receiptdate"); + // SimpleColumn l_receiptdate1("tpch.lineitem.l_receiptdate"); + // filters + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; - // filters - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; + // tokens.push_back(Token(new Operator("("))); + // tokens.push_back(Token(new Operator(")"))); - //tokens.push_back(Token(new Operator("("))); - //tokens.push_back(Token(new Operator(")"))); + tokens.push_back(Token(new SimpleFilter(new Operator(">="), new SimpleColumn(l_shipdate), + new ConstantColumn("1995-09-01")))); - tokens.push_back(Token(new SimpleFilter(new Operator(">="), - new SimpleColumn(l_shipdate), - new ConstantColumn("1995-09-01")))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token( + new SimpleFilter(new Operator("<"), new SimpleColumn(l_shipdate), new ConstantColumn("1995-10-01")))); - tokens.push_back(Token(new SimpleFilter(new Operator("<"), - new SimpleColumn(l_shipdate), - new ConstantColumn("1995-10-01")))); + filterList = parser.parse(tokens.begin(), tokens.end()); - filterList = parser.parse(tokens.begin(), tokens.end()); + // draw filterList tree + filterList->drawTree("selectExecutionPlan14.dot"); - // draw filterList tree - filterList->drawTree("selectExecutionPlan14.dot"); - - // calpont execution plan - csep.returnedCols (colList); - csep.filters (filterList); - cout << "\nCalpont Execution Plan:" << endl; - cout << csep << endl; - cout << " --- end of test 14 ---" << endl; - - filterList->walk(walkfnString); - - } + // calpont execution plan + csep.returnedCols(colList); + csep.filters(filterList); + cout << "\nCalpont Execution Plan:" << endl; + cout << csep << endl; + cout << " --- end of test 14 ---" << endl; + filterList->walk(walkfnString); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver15.cpp b/dbcon/execplan/tdriver15.cpp index 3c09d5537..aa6727c6c 100644 --- a/dbcon/execplan/tdriver15.cpp +++ b/dbcon/execplan/tdriver15.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -49,101 +49,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_15); - CPPUNIT_TEST( selectExecutionPlan_15 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_15() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_15() + { + cout << "SQL: \ create view revenue:s (supplier_no, total_revenue) as \ select \ l_suppkey, \ @@ -179,218 +175,204 @@ order by \ drop view revenue:s; \ " << endl; - /* ------------The create view statement is being re-written as an inline view ----- */ - /* ------------ (dynamic table) to be used in a from cluase. ----- */ + /* ------------The create view statement is being re-written as an inline view ----- */ + /* ------------ (dynamic table) to be used in a from cluase. ----- */ - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - CalpontSelectExecutionPlan::ReturnedColumnList colList; - CalpontSelectExecutionPlan::GroupByColumnList groupbyList; - CalpontSelectExecutionPlan::OrderByColumnList orderbyList; - ParseTree* filterList; + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + CalpontSelectExecutionPlan::ReturnedColumnList colList; + CalpontSelectExecutionPlan::GroupByColumnList groupbyList; + CalpontSelectExecutionPlan::OrderByColumnList orderbyList; + ParseTree* filterList; - // returned columns - //SimpleColumn l_suppkey("tpch.lineitem.l_suppkey"); - SimpleColumn* l_suppkey = new SimpleColumn("tpch.lineitem.l_suppkey"); - AggregateColumn* total_revenue = new AggregateColumn(); - total_revenue->functionName("sum"); - total_revenue->alias("total_revenue"); - ArithmeticColumn* a1 = new ArithmeticColumn ("l_extenedprice * (1 - l_discount)"); - total_revenue->functionParms(a1); + // returned columns + // SimpleColumn l_suppkey("tpch.lineitem.l_suppkey"); + SimpleColumn* l_suppkey = new SimpleColumn("tpch.lineitem.l_suppkey"); + AggregateColumn* total_revenue = new AggregateColumn(); + total_revenue->functionName("sum"); + total_revenue->alias("total_revenue"); + ArithmeticColumn* a1 = new ArithmeticColumn("l_extenedprice * (1 - l_discount)"); + total_revenue->functionParms(a1); - // Create the Projection - colList.push_back(l_suppkey); - colList.push_back(total_revenue); + // Create the Projection + colList.push_back(l_suppkey); + colList.push_back(total_revenue); - // Filter columns - SimpleColumn l_shipdate("tpch.lineitem.l_shipdate"); - //SimpleColumn l_shipmode0("tpch.lineitem.l_shipmode"); - //SimpleColumn l_shipmode1("tpch.lineitem.l_shipmode"); - //SimpleColumn l_commitdate("tpch.lineitem.l_commitdate"); - //SimpleColumn l_receiptdate("tpch.lineitem.l_receiptdate"); - //SimpleColumn l_receiptdate1("tpch.lineitem.l_receiptdate"); + // Filter columns + SimpleColumn l_shipdate("tpch.lineitem.l_shipdate"); + // SimpleColumn l_shipmode0("tpch.lineitem.l_shipmode"); + // SimpleColumn l_shipmode1("tpch.lineitem.l_shipmode"); + // SimpleColumn l_commitdate("tpch.lineitem.l_commitdate"); + // SimpleColumn l_receiptdate("tpch.lineitem.l_receiptdate"); + // SimpleColumn l_receiptdate1("tpch.lineitem.l_receiptdate"); + // filters + CalpontSelectExecutionPlan::Parser parser; + std::vector tokens; - // filters - CalpontSelectExecutionPlan::Parser parser; - std::vector tokens; + // tokens.push_back(Token(new Operator("("))); + // tokens.push_back(Token(new Operator(")"))); - //tokens.push_back(Token(new Operator("("))); - //tokens.push_back(Token(new Operator(")"))); + tokens.push_back(Token(new SimpleFilter(new Operator(">="), new SimpleColumn(l_shipdate), + new ConstantColumn("06/01/2005")))); - tokens.push_back(Token(new SimpleFilter(new Operator(">="), - new SimpleColumn(l_shipdate), - new ConstantColumn("06/01/2005")))); + tokens.push_back(Token(new Operator("and"))); - tokens.push_back(Token(new Operator("and"))); + tokens.push_back(Token( + new SimpleFilter(new Operator("<"), new SimpleColumn(l_shipdate), new ConstantColumn("07/01/2005")))); - tokens.push_back(Token(new SimpleFilter(new Operator("<"), - new SimpleColumn(l_shipdate), - new ConstantColumn("07/01/2005")))); + // old below + filterList = parser.parse(tokens.begin(), tokens.end()); - // old below - filterList = parser.parse(tokens.begin(), tokens.end()); + // draw filterList tree + filterList->drawTree("selectExecutionPlan_15sub.dot"); - // draw filterList tree - filterList->drawTree("selectExecutionPlan_15sub.dot"); + // Build Group By List + SimpleColumn* l_suppkey2 = new SimpleColumn("tpch.lineitem.l_suppkey"); + groupbyList.push_back(l_suppkey2); - // Build Group By List - SimpleColumn* l_suppkey2 = new SimpleColumn("tpch.lineitem.l_suppkey"); - groupbyList.push_back(l_suppkey2); + // calpont execution plan + csep->returnedCols(colList); + csep->filters(filterList); - // calpont execution plan - csep->returnedCols (colList); - csep->filters (filterList); + csep->location(CalpontSelectExecutionPlan::FROM); // Use scope resolution to use enum. - csep->location (CalpontSelectExecutionPlan::FROM); // Use scope resolution to use enum. + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 15 ---" << endl; - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 15 ---" << endl; + /* ------ This is the begining of the outer query ---------------------- */ + /* ------ This is the begining of the outer query ---------------------- */ - /* ------ This is the begining of the outer query ---------------------- */ - /* ------ This is the begining of the outer query ---------------------- */ + CalpontSelectExecutionPlan* csep_parent = new CalpontSelectExecutionPlan(); + cout << "********* I'm ok to here 01 ***************" << endl; + CalpontSelectExecutionPlan::ReturnedColumnList colList2; + CalpontSelectExecutionPlan::GroupByColumnList groupbyList2; + cout << "********* I'm ok to here 02 ***************" << endl; + CalpontSelectExecutionPlan::OrderByColumnList orderbyList2; + CalpontSelectExecutionPlan::SelectList subSelectList2; + //`ParseTree* filterList2; + cout << "********* I'm ok to here 1 ***************" << endl; + // returned columns + SimpleColumn* s_suppkey = new SimpleColumn("tpch.supplier.s_suppkey"); + SimpleColumn* s_name = new SimpleColumn("tpch.supplier.s_name"); + SimpleColumn* s_address = new SimpleColumn("tpch.supplier.s_address"); + SimpleColumn* s_phone = new SimpleColumn("tpch.supplier.s_phone"); + cout << "********* I'm ok to here 1a ***************" << endl; + AggregateColumn* s_total_revenue = new AggregateColumn(); + cout << "********* I'm ok to here 1b ***************" << endl; + s_total_revenue->functionName("sum"); + ArithmeticColumn* a5 = new ArithmeticColumn("o_orderpriority = '2-HIGH' THEN 1 ELSE 0 END )"); + cout << "********* I'm ok to here 1c ***************" << endl; + s_total_revenue->functionParms(a5); - CalpontSelectExecutionPlan* csep_parent = new CalpontSelectExecutionPlan(); - cout << "********* I'm ok to here 01 ***************" << endl; - CalpontSelectExecutionPlan::ReturnedColumnList colList2; - CalpontSelectExecutionPlan::GroupByColumnList groupbyList2; - cout << "********* I'm ok to here 02 ***************" << endl; - CalpontSelectExecutionPlan::OrderByColumnList orderbyList2; - CalpontSelectExecutionPlan::SelectList subSelectList2; - //`ParseTree* filterList2; + // Create the Projection + cout << "********* I'm ok to here 1d ***************" << endl; + colList2.push_back(s_suppkey); + colList2.push_back(s_name); + colList2.push_back(s_address); + colList2.push_back(s_phone); + colList2.push_back(s_total_revenue); + cout << "********* I'm ok to here 1e ***************" << endl; - cout << "********* I'm ok to here 1 ***************" << endl; - // returned columns - SimpleColumn* s_suppkey = new SimpleColumn ("tpch.supplier.s_suppkey"); - SimpleColumn* s_name = new SimpleColumn("tpch.supplier.s_name"); - SimpleColumn* s_address = new SimpleColumn("tpch.supplier.s_address"); - SimpleColumn* s_phone = new SimpleColumn("tpch.supplier.s_phone"); - cout << "********* I'm ok to here 1a ***************" << endl; - AggregateColumn* s_total_revenue = new AggregateColumn(); - cout << "********* I'm ok to here 1b ***************" << endl; - s_total_revenue->functionName("sum"); - ArithmeticColumn* a5 = new ArithmeticColumn ("o_orderpriority = '2-HIGH' THEN 1 ELSE 0 END )"); - cout << "********* I'm ok to here 1c ***************" << endl; - s_total_revenue->functionParms(a5); + /* ------ This is the begining of the query filter ---------------------- */ + /* ------ This is the begining of the query filter ---------------------- */ - // Create the Projection - cout << "********* I'm ok to here 1d ***************" << endl; - colList2.push_back(s_suppkey); - colList2.push_back(s_name); - colList2.push_back(s_address); - colList2.push_back(s_phone); - colList2.push_back(s_total_revenue); - cout << "********* I'm ok to here 1e ***************" << endl; + // std::vector tokens_n; + CalpontSelectExecutionPlan::FilterTokenList regFilterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_suppkey"), + new SimpleColumn("tpch.supplier.s_supplier_no")); - /* ------ This is the begining of the query filter ---------------------- */ - /* ------ This is the begining of the query filter ---------------------- */ + regFilterTokenList.push_back(f1); + regFilterTokenList.push_back(new Operator("and")); - //std::vector tokens_n; - CalpontSelectExecutionPlan::FilterTokenList regFilterTokenList; - SimpleFilter* f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.supplier.s_suppkey"), - new SimpleColumn ("tpch.supplier.s_supplier_no")); + cout << "********* I'm ok to here 1 ***************" << endl; + // Sub select filter + CalpontSelectExecutionPlan* subcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - regFilterTokenList.push_back(f1); - regFilterTokenList.push_back(new Operator("and")); + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList subColList; + ArithmeticColumn* sc1 = new ArithmeticColumn("max(total_revenue)"); + // CalpontSelectExecutionPlan::SelectList subSelectList; - cout << "********* I'm ok to here 1 ***************" << endl; - //Sub select filter - CalpontSelectExecutionPlan* subcsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + subColList.push_back(sc1); + subcsep->returnedCols(subColList); + cout << "********* I'm ok to here 2 ***************" << endl; - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList subColList; + // subselect Filters + // CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + SelectFilter* f2 = + new SelectFilter(new SimpleColumn("tpch.revenue.total_revenue"), new Operator("="), subcsep); - ArithmeticColumn* sc1 = new ArithmeticColumn ("max(total_revenue)"); - //CalpontSelectExecutionPlan::SelectList subSelectList; + regFilterTokenList.push_back(f2); - subColList.push_back(sc1); - subcsep->returnedCols(subColList); - cout << "********* I'm ok to here 2 ***************" << endl; + cout << "********* I'm ok to here 3 ***************" << endl; + // Filter columns + // SimpleColumn l_shipmode0("tpch.lineitem.l_shipmode"); + // *** Already creted via regFilterTokenList object **** - //subselect Filters - //CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SelectFilter* f2 = new SelectFilter (new SimpleColumn ("tpch.revenue.total_revenue"), - new Operator("="), - subcsep); + // filters + // CalpontSelectExecutionPlan::Parser parser; + // std::vector tokens; + // *** Already creted via regFilterTokenList object **** - regFilterTokenList.push_back(f2); + // tokens.push_back(Token(new Operator("("))); + // tokens.push_back(Token(new Operator(")"))); + // tokens.push_back(Token(new SimpleFilter(new Operator("="), + // new SimpleColumn(l_shipmode), + // new ConstantColumn("1")))); - cout << "********* I'm ok to here 3 ***************" << endl; - // Filter columns - //SimpleColumn l_shipmode0("tpch.lineitem.l_shipmode"); - // *** Already creted via regFilterTokenList object **** + // tokens.push_back(Token(new Operator("and"))); - // filters - //CalpontSelectExecutionPlan::Parser parser; - //std::vector tokens; - // *** Already creted via regFilterTokenList object **** + // tokens.push_back(Token(new SimpleFilter(new Operator("="), + // new SimpleColumn(l_shipmode), + // new ConstantColumn("2")))); - //tokens.push_back(Token(new Operator("("))); - //tokens.push_back(Token(new Operator(")"))); + // old below + // filterList = parser.parse(tokens.begin(), tokens.end()); - //tokens.push_back(Token(new SimpleFilter(new Operator("="), - // new SimpleColumn(l_shipmode), - // new ConstantColumn("1")))); + // draw filterList tree + // filterList->drawTree("selectExecutionPlan_15n.dot"); - //tokens.push_back(Token(new Operator("and"))); + // Build Group By List + // SimpleColumn *l_shipmode2 = new SimpleColumn("tpch.lineitem.l_shipmode"); + // groupbyList.push_back(l_shipmode2); - //tokens.push_back(Token(new SimpleFilter(new Operator("="), - // new SimpleColumn(l_shipmode), - // new ConstantColumn("2")))); + // Build Order By List + SimpleColumn* s_suppkey3 = new SimpleColumn("tpch.supplier.s_suppkey"); + orderbyList.push_back(s_suppkey3); + // calpont execution plan + csep_parent->returnedCols(colList2); + csep_parent->filterTokenList(regFilterTokenList); + csep_parent->orderByCols(orderbyList); - // old below - // filterList = parser.parse(tokens.begin(), tokens.end()); + cout << "********* I'm ok to here 4 ***************" << endl; + // draw filterList tree + // csep_parent->filterTokenList->drawTree("selectExecutionPlan_15.dot"); + ParseTree* pt = const_cast(csep_parent->filters()); + pt->drawTree("selectExecutionPlan_15.dot"); - // draw filterList tree - //filterList->drawTree("selectExecutionPlan_15n.dot"); - - // Build Group By List - //SimpleColumn *l_shipmode2 = new SimpleColumn("tpch.lineitem.l_shipmode"); - //groupbyList.push_back(l_shipmode2); - - // Build Order By List - SimpleColumn* s_suppkey3 = new SimpleColumn("tpch.supplier.s_suppkey"); - orderbyList.push_back(s_suppkey3); - - // calpont execution plan - csep_parent->returnedCols (colList2); - csep_parent->filterTokenList(regFilterTokenList); - csep_parent->orderByCols(orderbyList); - - cout << "********* I'm ok to here 4 ***************" << endl; - // draw filterList tree - //csep_parent->filterTokenList->drawTree("selectExecutionPlan_15.dot"); - ParseTree* pt = const_cast(csep_parent->filters()); - pt->drawTree("selectExecutionPlan_15.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << csep << endl; - cout << " --- end of test 15 ---" << endl; - - filterList->walk(walkfnString); - - } + cout << "\nCalpont Execution Plan:" << endl; + cout << csep << endl; + cout << " --- end of test 15 ---" << endl; + filterList->walk(walkfnString); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver16.cpp b/dbcon/execplan/tdriver16.cpp index 8a8ea5679..d86015351 100644 --- a/dbcon/execplan/tdriver16.cpp +++ b/dbcon/execplan/tdriver16.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -51,101 +51,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_16); - CPPUNIT_TEST( selectExecutionPlan_16 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_16() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_16() + { + cout << "SQL: \ select \ p_brand, \ p_type, \ @@ -176,214 +172,195 @@ order by \ p_brand, \ p_type, \ p_size;" - << endl; + << endl; - /* ------------The create view statement is being re-written as an inline view ----- */ - /* ------------ (dynamic table) to be used in a from cluase. ----- */ + /* ------------The create view statement is being re-written as an inline view ----- */ + /* ------------ (dynamic table) to be used in a from cluase. ----- */ - //This is the main query body for query 16 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 16 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* c1 = new SimpleColumn("tpch.part.p_brand"); - colList.push_back(c1); - SimpleColumn* c2 = new SimpleColumn("tpch.part.p_type"); - colList.push_back(c2); - SimpleColumn* c3 = new SimpleColumn("tpch.part.p_size"); - colList.push_back(c3); - AggregateColumn* c4 = new AggregateColumn(); - c4->functionName("count"); - c4->alias("supplier_cnt"); - ArithmeticColumn* a1 = new ArithmeticColumn("(distinct tpch.part.ps_suppkey)"); - c4->functionParms(a1); - colList.push_back(c4); + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* c1 = new SimpleColumn("tpch.part.p_brand"); + colList.push_back(c1); + SimpleColumn* c2 = new SimpleColumn("tpch.part.p_type"); + colList.push_back(c2); + SimpleColumn* c3 = new SimpleColumn("tpch.part.p_size"); + colList.push_back(c3); + AggregateColumn* c4 = new AggregateColumn(); + c4->functionName("count"); + c4->alias("supplier_cnt"); + ArithmeticColumn* a1 = new ArithmeticColumn("(distinct tpch.part.ps_suppkey)"); + c4->functionParms(a1); + colList.push_back(c4); - csep->returnedCols(colList); // set Projection columns + csep->returnedCols(colList); // set Projection columns - // Filter columns - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - SimpleFilter* f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_partkey"), - new SimpleColumn ("tpch.partsupp.ps_partkey")); - csep_filterlist.push_back(f1); + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.partsupp.ps_partkey")); + csep_filterlist.push_back(f1); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_brand"), - new ConstantColumn(":1")); - csep_filterlist.push_back(f2); + SimpleFilter* f2 = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_brand"), new ConstantColumn(":1")); + csep_filterlist.push_back(f2); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - //These next condition blocks where orginally written with ConditionFilter - SimpleFilter* f3 = new SimpleFilter ( new Operator("not like"), - new SimpleColumn ("tpch.part.p_type"), - new ConstantColumn(":2")); - csep_filterlist.push_back(f3); + // These next condition blocks where orginally written with ConditionFilter + SimpleFilter* f3 = new SimpleFilter(new Operator("not like"), new SimpleColumn("tpch.part.p_type"), + new ConstantColumn(":2")); + csep_filterlist.push_back(f3); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f4a = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":3")); - csep_filterlist.push_back(f4a); + SimpleFilter* f4a = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":3")); + csep_filterlist.push_back(f4a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4b = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":4")); - csep_filterlist.push_back(f4b); + SimpleFilter* f4b = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":4")); + csep_filterlist.push_back(f4b); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4c = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":5")); - csep_filterlist.push_back(f4c); + SimpleFilter* f4c = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":5")); + csep_filterlist.push_back(f4c); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4d = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":6")); - csep_filterlist.push_back(f4d); + SimpleFilter* f4d = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":6")); + csep_filterlist.push_back(f4d); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4e = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":7")); - csep_filterlist.push_back(f4e); + SimpleFilter* f4e = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":7")); + csep_filterlist.push_back(f4e); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4f = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":8")); - csep_filterlist.push_back(f4f); + SimpleFilter* f4f = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":8")); + csep_filterlist.push_back(f4f); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4g = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":9")); - csep_filterlist.push_back(f4g); + SimpleFilter* f4g = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":9")); + csep_filterlist.push_back(f4g); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f4h = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn(":10")); - csep_filterlist.push_back(f4h); + SimpleFilter* f4h = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(":10")); + csep_filterlist.push_back(f4h); - csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - /* --------------- sub select begins here ---------------- */ - //Sub select filter - CalpontSelectExecutionPlan* subcsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + /* --------------- sub select begins here ---------------- */ + // Sub select filter + CalpontSelectExecutionPlan* subcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList subColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList subColList; - SimpleColumn* sc1 = new SimpleColumn ("tpch.supplier.s_suppkey"); - subColList.push_back(sc1); - // Append returned columns to subselect - subcsep->returnedCols(subColList); + SimpleColumn* sc1 = new SimpleColumn("tpch.supplier.s_suppkey"); + subColList.push_back(sc1); + // Append returned columns to subselect + subcsep->returnedCols(subColList); - //subselect Filters - CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; - SimpleFilter* sf1 = new SimpleFilter ( new Operator("LIKE"), - new SimpleColumn ("tpch.supplier.s_comment"), - new ConstantColumn("%Customer%Complaints%")); - subcsep_filterlist.push_back(sf1); + // subselect Filters + CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; + SimpleFilter* sf1 = new SimpleFilter(new Operator("LIKE"), new SimpleColumn("tpch.supplier.s_comment"), + new ConstantColumn("%Customer%Complaints%")); + subcsep_filterlist.push_back(sf1); - /* --------------- sub select begins here ---------------- */ + /* --------------- sub select begins here ---------------- */ - //CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SelectFilter* f5 = new SelectFilter (new SimpleColumn ("tpch.partsupp.ps_suppkey"), - new Operator("not in"), - subcsep); + // CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + SelectFilter* f5 = + new SelectFilter(new SimpleColumn("tpch.partsupp.ps_suppkey"), new Operator("not in"), subcsep); - csep_filterlist.push_back(f5); - /* + csep_filterlist.push_back(f5); + /* - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - //ConditionFilter *f3 = new ConditionFilter ("p_brand not like ':2%'"); - ConditionFilter *f3 = new ConditionFilter (); - f3->expression ("p_brand not like ':2%'"); - csep_filterlist.push_back(f3); + //ConditionFilter *f3 = new ConditionFilter ("p_brand not like ':2%'"); + ConditionFilter *f3 = new ConditionFilter (); + f3->expression ("p_brand not like ':2%'"); + csep_filterlist.push_back(f3); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - //ConditionFilter *f4 = new ConditionFilter ("p_brand not in (:3, :4, :5, :6, :7, :8, :9, :10"); - ConditionFilter *f4 = new ConditionFilter (); - f4->expression ("p_brand not in (:3, :4, :5, :6, :7, :8, :9, :10"); - csep_filterlist.push_back(f4); + //ConditionFilter *f4 = new ConditionFilter ("p_brand not in (:3, :4, :5, :6, :7, :8, :9, :10"); + ConditionFilter *f4 = new ConditionFilter (); + f4->expression ("p_brand not in (:3, :4, :5, :6, :7, :8, :9, :10"); + csep_filterlist.push_back(f4); - */ - csep->filterTokenList(csep_filterlist); //Set Filter Columns + */ + csep->filterTokenList(csep_filterlist); // Set Filter Columns + // Build Group By List + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + SimpleColumn* g1 = new SimpleColumn("tpch.part.p_brand"); + csep_groupbyList.push_back(g1); + SimpleColumn* g2 = new SimpleColumn("tpch.part.p_type"); + csep_groupbyList.push_back(g2); + SimpleColumn* g3 = new SimpleColumn("tpch.part.p_size"); + csep_groupbyList.push_back(g3); + csep->groupByCols(csep_groupbyList); // Set GroupBy columns - // Build Group By List - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - SimpleColumn* g1 = new SimpleColumn("tpch.part.p_brand"); - csep_groupbyList.push_back(g1); - SimpleColumn* g2 = new SimpleColumn("tpch.part.p_type"); - csep_groupbyList.push_back(g2); - SimpleColumn* g3 = new SimpleColumn("tpch.part.p_size"); - csep_groupbyList.push_back(g3); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns + // Order By List + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + SimpleColumn* o0 = new SimpleColumn("supplier_cnt"); // AggregateColumn has no asc/desc order + o0->alias("supplier_cnt"); + o0->asc(false); + csep_groupbyList.push_back(o0); + SimpleColumn* o1 = new SimpleColumn("tpch.part.p_brand"); + csep_orderbyList.push_back(o1); + SimpleColumn* o2 = new SimpleColumn("tpch.part.p_type"); + csep_orderbyList.push_back(o2); + SimpleColumn* o3 = new SimpleColumn("tpch.part.p_size"); + csep_orderbyList.push_back(o3); + csep->orderByCols(csep_orderbyList); // Set OrderBy columns - // Order By List - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - SimpleColumn* o0 = new SimpleColumn("supplier_cnt"); // AggregateColumn has no asc/desc order - o0->alias("supplier_cnt"); - o0->asc(false); - csep_groupbyList.push_back(o0); - SimpleColumn* o1 = new SimpleColumn("tpch.part.p_brand"); - csep_orderbyList.push_back(o1); - SimpleColumn* o2 = new SimpleColumn("tpch.part.p_type"); - csep_orderbyList.push_back(o2); - SimpleColumn* o3 = new SimpleColumn("tpch.part.p_size"); - csep_orderbyList.push_back(o3); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns + // filterList->walk(walkfnString); ?? - //filterList->walk(walkfnString); ?? - - // Print the parse tree - ParseTree* pt = const_cast(csep->filters()); - pt->drawTree("selectExecutionPlan_16.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 16 ---" << endl; - - } + // Print the parse tree + ParseTree* pt = const_cast(csep->filters()); + pt->drawTree("selectExecutionPlan_16.dot"); + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 16 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver17.cpp b/dbcon/execplan/tdriver17.cpp index 690e4cc9f..e3c8ab7ac 100644 --- a/dbcon/execplan/tdriver17.cpp +++ b/dbcon/execplan/tdriver17.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -51,101 +51,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_17); - CPPUNIT_TEST( selectExecutionPlan_17 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_17() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_17() + { + cout << "SQL: \ select \ sum(l_extendedprice) / 7.0 as avg_yearly \ from \ @@ -163,118 +159,108 @@ where \ where \ l_partkey = p_partkey \ );" - << endl; + << endl; - /* ------------ Start Query Execution --------- */ + /* ------------ Start Query Execution --------- */ - //This is the main query body for query 17 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 17 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - AggregateColumn* c1 = new AggregateColumn(); - c1->functionName("sum"); - c1->alias("avg_yearly"); - ArithmeticColumn* a1 = new ArithmeticColumn("(l_extendedprice ) / 7.0 "); - c1->functionParms(a1); - colList.push_back(c1); - csep->returnedCols(colList); // set Projection columns + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + AggregateColumn* c1 = new AggregateColumn(); + c1->functionName("sum"); + c1->alias("avg_yearly"); + ArithmeticColumn* a1 = new ArithmeticColumn("(l_extendedprice ) / 7.0 "); + c1->functionParms(a1); + colList.push_back(c1); + csep->returnedCols(colList); // set Projection columns - // Filter columns - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - SimpleFilter* f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_partkey"), - new SimpleColumn ("tpch.lineitem.l_partkey")); - csep_filterlist.push_back(f1); + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + csep_filterlist.push_back(f1); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.part.p_brand"), - new ConstantColumn(":1")); - csep_filterlist.push_back(f2); + SimpleFilter* f2 = + new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.part.p_brand"), new ConstantColumn(":1")); + csep_filterlist.push_back(f2); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter ( new Operator("not like"), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn(":2")); - csep_filterlist.push_back(f3); + SimpleFilter* f3 = new SimpleFilter(new Operator("not like"), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn(":2")); + csep_filterlist.push_back(f3); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - /* --------------- sub select begins here ---------------- */ - //Sub select filter - CalpontSelectExecutionPlan* subcsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + /* --------------- sub select begins here ---------------- */ + // Sub select filter + CalpontSelectExecutionPlan* subcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList subColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList subColList; - ArithmeticColumn* sc1 = new ArithmeticColumn ("0.2 * avg(l_quantity)"); - subColList.push_back(sc1); - // Append returned columns to subselect - subcsep->returnedCols(subColList); + ArithmeticColumn* sc1 = new ArithmeticColumn("0.2 * avg(l_quantity)"); + subColList.push_back(sc1); + // Append returned columns to subselect + subcsep->returnedCols(subColList); - //subselect Filters - CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; - SimpleFilter* sf1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_suppkey"), - new SimpleColumn ("tpch.part.p_partkey")); - subcsep_filterlist.push_back(sf1); + // subselect Filters + CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_suppkey"), + new SimpleColumn("tpch.part.p_partkey")); + subcsep_filterlist.push_back(sf1); - /* --------------- sub select ends here ---------------- */ + /* --------------- sub select ends here ---------------- */ - //CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SelectFilter* f4 = new SelectFilter (new SimpleColumn ("tpch.lineitem.l_quantity"), - new Operator("<"), - subcsep); + // CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + SelectFilter* f4 = + new SelectFilter(new SimpleColumn("tpch.lineitem.l_quantity"), new Operator("<"), subcsep); - csep_filterlist.push_back(f4); + csep_filterlist.push_back(f4); - csep->filterTokenList(csep_filterlist); //Set Filter Columns + csep->filterTokenList(csep_filterlist); // Set Filter Columns - /* *** not used here ** - // Build Group By List - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - SimpleColumn *g1 = new SimpleColumn("tpch.part.p_brand"); - csep_groupbyList.push_back(g1); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns + /* *** not used here ** + // Build Group By List + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + SimpleColumn *g1 = new SimpleColumn("tpch.part.p_brand"); + csep_groupbyList.push_back(g1); + csep->groupByCols(csep_groupbyList); //Set GroupBy columns - // Order By List - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - SimpleColumn *o0 = new SimpleColumn("supplier_cnt"); // AggregateColumn has no asc/desc order - o0->alias("supplier_cnt"); - o0->asc(false); - csep_groupbyList.push_back(o0); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns - */ + // Order By List + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + SimpleColumn *o0 = new SimpleColumn("supplier_cnt"); // AggregateColumn has no asc/desc order + o0->alias("supplier_cnt"); + o0->asc(false); + csep_groupbyList.push_back(o0); + csep->orderByCols(csep_orderbyList); //Set OrderBy columns + */ - // Print the parse tree - ParseTree* pt = const_cast(csep->filters()); - pt->drawTree("selectExecutionPlan_17.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 17 ---" << endl; - - } + // Print the parse tree + ParseTree* pt = const_cast(csep->filters()); + pt->drawTree("selectExecutionPlan_17.dot"); + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 17 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver18.cpp b/dbcon/execplan/tdriver18.cpp index fe59c3141..c0a4be414 100644 --- a/dbcon/execplan/tdriver18.cpp +++ b/dbcon/execplan/tdriver18.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -50,101 +50,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_18); - CPPUNIT_TEST( selectExecutionPlan_18 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_18() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_18() + { + cout << "SQL: \ select \ c_name, \ c_custkey, \ @@ -177,145 +173,134 @@ group by \ order by \ o_totalprice desc, \ o_orderdate;" - << endl; + << endl; -// **** -// **** THIS QUERY IS MISSING THE HAVING CLAUSE ON THE GROUP BY **** -// **** + // **** + // **** THIS QUERY IS MISSING THE HAVING CLAUSE ON THE GROUP BY **** + // **** - //This is the main query body for query 18 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 18 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* c1 = new SimpleColumn("tpch.customer.c_name"); - colList.push_back(c1); - SimpleColumn* c2 = new SimpleColumn("tpch.customer.c_custkey"); - colList.push_back(c2); - SimpleColumn* c3 = new SimpleColumn("tpch.orders.o_orderkey"); - colList.push_back(c3); - SimpleColumn* c4 = new SimpleColumn("tpch.orders.o_orderdate"); - colList.push_back(c4); - SimpleColumn* c5 = new SimpleColumn("tpch.orders.o_totalprice"); - colList.push_back(c5); - ArithmeticColumn* c6 = new ArithmeticColumn("sum(l_quantity)"); - colList.push_back(c6); + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* c1 = new SimpleColumn("tpch.customer.c_name"); + colList.push_back(c1); + SimpleColumn* c2 = new SimpleColumn("tpch.customer.c_custkey"); + colList.push_back(c2); + SimpleColumn* c3 = new SimpleColumn("tpch.orders.o_orderkey"); + colList.push_back(c3); + SimpleColumn* c4 = new SimpleColumn("tpch.orders.o_orderdate"); + colList.push_back(c4); + SimpleColumn* c5 = new SimpleColumn("tpch.orders.o_totalprice"); + colList.push_back(c5); + ArithmeticColumn* c6 = new ArithmeticColumn("sum(l_quantity)"); + colList.push_back(c6); - csep->returnedCols(colList); // set Projection columns + csep->returnedCols(colList); // set Projection columns - // Filter columns - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - //Sub select filter - CalpontSelectExecutionPlan* subcsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // Sub select filter + CalpontSelectExecutionPlan* subcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList subColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList subColList; - SimpleColumn* sc1 = new SimpleColumn ("tpch.lineitem.l_orderkey"); - subColList.push_back(sc1); - // Append returned columns to subselect - subcsep->returnedCols(subColList); + SimpleColumn* sc1 = new SimpleColumn("tpch.lineitem.l_orderkey"); + subColList.push_back(sc1); + // Append returned columns to subselect + subcsep->returnedCols(subColList); - //subselect Filters - CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; - SimpleFilter* sf1 = new SimpleFilter ( new Operator("IN"), - new SimpleColumn ("tpch.lineitem.??"), - new ConstantColumn("%Customer%Complaints%")); - subcsep_filterlist.push_back(sf1); + // subselect Filters + CalpontSelectExecutionPlan::FilterTokenList subcsep_filterlist; + SimpleFilter* sf1 = new SimpleFilter(new Operator("IN"), new SimpleColumn("tpch.lineitem.??"), + new ConstantColumn("%Customer%Complaints%")); + subcsep_filterlist.push_back(sf1); - // Build subGroup By List - CalpontSelectExecutionPlan::GroupByColumnList subcsep_groupbyList; - SimpleColumn* sg1 = new SimpleColumn ("tpch.lineitem.l_orderkey"); - subcsep_groupbyList.push_back(sg1); - subcsep->groupByCols(subcsep_groupbyList); //Set GroupBy columns + // Build subGroup By List + CalpontSelectExecutionPlan::GroupByColumnList subcsep_groupbyList; + SimpleColumn* sg1 = new SimpleColumn("tpch.lineitem.l_orderkey"); + subcsep_groupbyList.push_back(sg1); + subcsep->groupByCols(subcsep_groupbyList); // Set GroupBy columns - //subHaving Filters - CalpontSelectExecutionPlan::FilterTokenList subcsep_havingList; + // subHaving Filters + CalpontSelectExecutionPlan::FilterTokenList subcsep_havingList; - SimpleFilter* sh1 = new SimpleFilter ( new Operator(">"), - new ArithmeticColumn("sum(l_quantity)"), - new ConstantColumn(":1")); - subcsep_havingList.push_back(sh1); - subcsep->havingTokenList(subcsep_havingList); //Set GroupBy columns + SimpleFilter* sh1 = new SimpleFilter(new Operator(">"), new ArithmeticColumn("sum(l_quantity)"), + new ConstantColumn(":1")); + subcsep_havingList.push_back(sh1); + subcsep->havingTokenList(subcsep_havingList); // Set GroupBy columns - //subcsep->filterTokenList(subcsep_havingList); //Set GroupBy columns - /* --------------- sub select ends here ---------------- */ + // subcsep->filterTokenList(subcsep_havingList); //Set GroupBy columns + /* --------------- sub select ends here ---------------- */ - //CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SelectFilter* f1 = new SelectFilter (new SimpleColumn ("tpch.orders.o_orderkey"), - new Operator("IN"), - subcsep); + // CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + SelectFilter* f1 = + new SelectFilter(new SimpleColumn("tpch.orders.o_orderkey"), new Operator("IN"), subcsep); - csep_filterlist.push_back(f1); + csep_filterlist.push_back(f1); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.customer.c_custkey"), - new SimpleColumn ("tpch.orders.o_custkey")); - csep_filterlist.push_back(f2); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_custkey"), + new SimpleColumn("tpch.orders.o_custkey")); + csep_filterlist.push_back(f2); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.orders.o_orderkey"), - new SimpleColumn ("tpch.lineitem.l_orderkey")); - csep_filterlist.push_back(f3); + SimpleFilter* f3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.orders.o_orderkey"), + new SimpleColumn("tpch.lineitem.l_orderkey")); + csep_filterlist.push_back(f3); - csep->filterTokenList(csep_filterlist); //Set Filter Columns + csep->filterTokenList(csep_filterlist); // Set Filter Columns + // Build Group By List + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + SimpleColumn* g1 = new SimpleColumn("tpch.customer.c_name"); + csep_groupbyList.push_back(g1); + SimpleColumn* g2 = new SimpleColumn("tpch.customer.c_custkey"); + csep_groupbyList.push_back(g2); + SimpleColumn* g3 = new SimpleColumn("tpch.orders.o_orderkey"); + csep_groupbyList.push_back(g3); + SimpleColumn* g4 = new SimpleColumn("tpch.orders.o_orderdate"); + csep_groupbyList.push_back(g4); + SimpleColumn* g5 = new SimpleColumn("tpch.orders.o_totalprice"); + csep_groupbyList.push_back(g5); + csep->groupByCols(csep_groupbyList); // Set GroupBy columns - // Build Group By List - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - SimpleColumn* g1 = new SimpleColumn("tpch.customer.c_name"); - csep_groupbyList.push_back(g1); - SimpleColumn* g2 = new SimpleColumn("tpch.customer.c_custkey"); - csep_groupbyList.push_back(g2); - SimpleColumn* g3 = new SimpleColumn("tpch.orders.o_orderkey"); - csep_groupbyList.push_back(g3); - SimpleColumn* g4 = new SimpleColumn("tpch.orders.o_orderdate"); - csep_groupbyList.push_back(g4); - SimpleColumn* g5 = new SimpleColumn("tpch.orders.o_totalprice"); - csep_groupbyList.push_back(g5); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns + // Order By List + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + SimpleColumn* o1 = new SimpleColumn("tpch.orders.o_totalprice"); + o1->asc(false); + csep_orderbyList.push_back(o1); + SimpleColumn* o2 = new SimpleColumn("tpch.orders.o_orderdate"); + csep_orderbyList.push_back(o2); + csep->orderByCols(csep_orderbyList); // Set OrderBy columns - // Order By List - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - SimpleColumn* o1 = new SimpleColumn("tpch.orders.o_totalprice"); - o1->asc(false); - csep_orderbyList.push_back(o1); - SimpleColumn* o2 = new SimpleColumn("tpch.orders.o_orderdate"); - csep_orderbyList.push_back(o2); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns + // filterList->walk(walkfnString); ?? - //filterList->walk(walkfnString); ?? - - // Print the parse tree - ParseTree* pt = const_cast(csep->filters()); - pt->drawTree("selectExecutionPlan_18.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 18 ---" << endl; - - } + // Print the parse tree + ParseTree* pt = const_cast(csep->filters()); + pt->drawTree("selectExecutionPlan_18.dot"); + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 18 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver19.cpp b/dbcon/execplan/tdriver19.cpp index 9d935448b..a219b68dc 100644 --- a/dbcon/execplan/tdriver19.cpp +++ b/dbcon/execplan/tdriver19.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -49,101 +49,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_19); - CPPUNIT_TEST( selectExecutionPlan_19 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_19() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_19() + { + cout << "SQL: \ select \ sum(l_extendedprice* (1 - l_discount)) as revenue \ from \ @@ -179,370 +175,326 @@ where \ and l_shipmode in ('AIR', 'AIR REG') \ and l_shipinstruct = 'DELIVER IN PERSON' \ ); " - << endl; + << endl; - /* ------------The create view statement is being re-written as an inline view ----- */ - /* ------------ (dynamic table) to be used in a from cluase. ----- */ + /* ------------The create view statement is being re-written as an inline view ----- */ + /* ------------ (dynamic table) to be used in a from cluase. ----- */ - //This is the main query body for query 16 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 16 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - ArithmeticColumn* c1 = new ArithmeticColumn("sum(l_quantity * (1 - l_discount))"); - c1->alias("revenue"); - colList.push_back(c1); - csep->returnedCols(colList); // set Projection columns + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + ArithmeticColumn* c1 = new ArithmeticColumn("sum(l_quantity * (1 - l_discount))"); + c1->alias("revenue"); + colList.push_back(c1); + csep->returnedCols(colList); // set Projection columns - // Filter columns - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - /* -- Where: Part 1 -- */ - csep_filterlist.push_back(new Operator("(")); + /* -- Where: Part 1 -- */ + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_partkey"), - new SimpleColumn ("tpch.lineitem.l_partkey")); - csep_filterlist.push_back(f1); + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + csep_filterlist.push_back(f1); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_brand"), - new ConstantColumn (":1")); - csep_filterlist.push_back(f2); + SimpleFilter* f2 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_brand"), new ConstantColumn(":1")); + csep_filterlist.push_back(f2); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f3a = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("SM CASE")); - csep_filterlist.push_back(f3a); + SimpleFilter* f3a = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("SM CASE")); + csep_filterlist.push_back(f3a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f3b = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("SM BOX")); - csep_filterlist.push_back(f3b); + SimpleFilter* f3b = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("SM BOX")); + csep_filterlist.push_back(f3b); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f3c = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("SM PACK")); - csep_filterlist.push_back(f3c); + SimpleFilter* f3c = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("SM PACK")); + csep_filterlist.push_back(f3c); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f3d = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("SM PKG")); - csep_filterlist.push_back(f3d); + SimpleFilter* f3d = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("SM PKG")); + csep_filterlist.push_back(f3d); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f4a = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.lineitem.l_quantity"), - new ConstantColumn (":4")); - csep_filterlist.push_back(f4a); + SimpleFilter* f4a = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_quantity"), + new ConstantColumn(":4")); + csep_filterlist.push_back(f4a); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f4b = new SimpleFilter ( new Operator("<="), - new SimpleColumn ("tpch.lineitem.l_quantity"), - new ArithmeticColumn ("(l_quantity :4 + 10)")); - csep_filterlist.push_back(f4b); + SimpleFilter* f4b = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.lineitem.l_quantity"), + new ArithmeticColumn("(l_quantity :4 + 10)")); + csep_filterlist.push_back(f4b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f5a = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn ("1")); - csep_filterlist.push_back(f5a); + SimpleFilter* f5a = + new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn("1")); + csep_filterlist.push_back(f5a); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f5b = new SimpleFilter ( new Operator("<="), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn ("5")); - csep_filterlist.push_back(f5b); + SimpleFilter* f5b = + new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn("5")); + csep_filterlist.push_back(f5b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f6a = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipmode"), - new ConstantColumn ("AIR")); - csep_filterlist.push_back(f6a); + SimpleFilter* f6a = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipmode"), + new ConstantColumn("AIR")); + csep_filterlist.push_back(f6a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f6b = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipmode"), - new ConstantColumn ("AIR REG")); - csep_filterlist.push_back(f6b); + SimpleFilter* f6b = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipmode"), + new ConstantColumn("AIR REG")); + csep_filterlist.push_back(f6b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f7 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.o_orderkey"), - new ConstantColumn ("DELIVER IN PERSON")); - csep_filterlist.push_back(f7); + SimpleFilter* f7 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.o_orderkey"), + new ConstantColumn("DELIVER IN PERSON")); + csep_filterlist.push_back(f7); - csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator(")")); - /* -- Where: Part 2 -- */ - csep_filterlist.push_back(new Operator("or")); - csep_filterlist.push_back(new Operator("(")); + /* -- Where: Part 2 -- */ + csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f8 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_partkey"), - new SimpleColumn ("tpch.lineitem.l_partkey")); - csep_filterlist.push_back(f8); + SimpleFilter* f8 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + csep_filterlist.push_back(f8); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f9 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_brand"), - new ConstantColumn (":2")); - csep_filterlist.push_back(f9); + SimpleFilter* f9 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_brand"), new ConstantColumn(":2")); + csep_filterlist.push_back(f9); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f10a = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("MED BAG")); - csep_filterlist.push_back(f10a); + SimpleFilter* f10a = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("MED BAG")); + csep_filterlist.push_back(f10a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f10b = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("MED BOX")); - csep_filterlist.push_back(f10b); + SimpleFilter* f10b = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("MED BOX")); + csep_filterlist.push_back(f10b); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f10c = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("MED PKG")); - csep_filterlist.push_back(f10c); + SimpleFilter* f10c = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("MED PKG")); + csep_filterlist.push_back(f10c); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f10d = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("MED PACK")); - csep_filterlist.push_back(f10d); + SimpleFilter* f10d = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("MED PACK")); + csep_filterlist.push_back(f10d); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f11a = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.lineitem.l_quantity"), - new ConstantColumn (":5")); - csep_filterlist.push_back(f11a); + SimpleFilter* f11a = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_quantity"), + new ConstantColumn(":5")); + csep_filterlist.push_back(f11a); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f11b = new SimpleFilter ( new Operator("<="), - new SimpleColumn ("tpch.lineitem.l_quantity"), - new ArithmeticColumn ("(l_quantity :5 + 10)")); - csep_filterlist.push_back(f11b); + SimpleFilter* f11b = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.lineitem.l_quantity"), + new ArithmeticColumn("(l_quantity :5 + 10)")); + csep_filterlist.push_back(f11b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f12a = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn ("1")); - csep_filterlist.push_back(f12a); + SimpleFilter* f12a = + new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn("1")); + csep_filterlist.push_back(f12a); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f12b = new SimpleFilter ( new Operator("<="), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn ("10")); - csep_filterlist.push_back(f12b); + SimpleFilter* f12b = + new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn("10")); + csep_filterlist.push_back(f12b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f13a = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipmode"), - new ConstantColumn ("AIR")); - csep_filterlist.push_back(f13a); + SimpleFilter* f13a = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipmode"), + new ConstantColumn("AIR")); + csep_filterlist.push_back(f13a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f13b = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipmode"), - new ConstantColumn ("AIR REG")); - csep_filterlist.push_back(f13b); + SimpleFilter* f13b = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipmode"), + new ConstantColumn("AIR REG")); + csep_filterlist.push_back(f13b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f14 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.o_orderkey"), - new ConstantColumn ("DELIVER IN PERSON")); - csep_filterlist.push_back(f14); + SimpleFilter* f14 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.o_orderkey"), + new ConstantColumn("DELIVER IN PERSON")); + csep_filterlist.push_back(f14); - csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator(")")); - /* -- Where: Part 3 -- */ - csep_filterlist.push_back(new Operator("or")); - csep_filterlist.push_back(new Operator("(")); + /* -- Where: Part 3 -- */ + csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f15 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_partkey"), - new SimpleColumn ("tpch.lineitem.l_partkey")); - csep_filterlist.push_back(f15); + SimpleFilter* f15 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + csep_filterlist.push_back(f15); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f16 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_brand"), - new ConstantColumn (":3")); - csep_filterlist.push_back(f16); + SimpleFilter* f16 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_brand"), new ConstantColumn(":3")); + csep_filterlist.push_back(f16); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f17a = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("LG CASE")); - csep_filterlist.push_back(f17a); + SimpleFilter* f17a = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("LG CASE")); + csep_filterlist.push_back(f17a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f17b = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("LG BOX")); - csep_filterlist.push_back(f17b); + SimpleFilter* f17b = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("LG BOX")); + csep_filterlist.push_back(f17b); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f17c = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("LG PACK")); - csep_filterlist.push_back(f17c); + SimpleFilter* f17c = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("LG PACK")); + csep_filterlist.push_back(f17c); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f17d = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.part.p_container"), - new ConstantColumn ("LG PKG")); - csep_filterlist.push_back(f17d); + SimpleFilter* f17d = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_container"), + new ConstantColumn("LG PKG")); + csep_filterlist.push_back(f17d); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f18a = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.lineitem.l_quantity"), - new ConstantColumn (":6")); - csep_filterlist.push_back(f18a); + SimpleFilter* f18a = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_quantity"), + new ConstantColumn(":6")); + csep_filterlist.push_back(f18a); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f18b = new SimpleFilter ( new Operator("<="), - new SimpleColumn ("tpch.lineitem.l_quantity"), - new ArithmeticColumn ("(l_quantity :6 + 10)")); - csep_filterlist.push_back(f18b); + SimpleFilter* f18b = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.lineitem.l_quantity"), + new ArithmeticColumn("(l_quantity :6 + 10)")); + csep_filterlist.push_back(f18b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f19a = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn ("1")); - csep_filterlist.push_back(f19a); + SimpleFilter* f19a = + new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn("1")); + csep_filterlist.push_back(f19a); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f19b = new SimpleFilter ( new Operator("<="), - new SimpleColumn ("tpch.part.p_size"), - new ConstantColumn ("15")); - csep_filterlist.push_back(f19b); + SimpleFilter* f19b = + new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn("15")); + csep_filterlist.push_back(f19b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("(")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("(")); - SimpleFilter* f20a = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipmode"), - new ConstantColumn ("AIR")); - csep_filterlist.push_back(f20a); + SimpleFilter* f20a = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipmode"), + new ConstantColumn("AIR")); + csep_filterlist.push_back(f20a); - csep_filterlist.push_back(new Operator("or")); + csep_filterlist.push_back(new Operator("or")); - SimpleFilter* f20b = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipmode"), - new ConstantColumn ("AIR REG")); - csep_filterlist.push_back(f20b); + SimpleFilter* f20b = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipmode"), + new ConstantColumn("AIR REG")); + csep_filterlist.push_back(f20b); - csep_filterlist.push_back(new Operator(")")); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f21 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.o_orderkey"), - new ConstantColumn ("DELIVER IN PERSON")); - csep_filterlist.push_back(f21); + SimpleFilter* f21 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.o_orderkey"), + new ConstantColumn("DELIVER IN PERSON")); + csep_filterlist.push_back(f21); - csep_filterlist.push_back(new Operator(")")); + csep_filterlist.push_back(new Operator(")")); + /*----- THIS IS WHERE IT ALL COMES TOGETHER ----- */ + csep->filterTokenList(csep_filterlist); // Set Filter Columns - /*----- THIS IS WHERE IT ALL COMES TOGETHER ----- */ - csep->filterTokenList(csep_filterlist); //Set Filter Columns + // Build Group By List + // CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + // csep->groupByCols(csep_groupbyList); //Set GroupBy columns - // Build Group By List - //CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - //csep->groupByCols(csep_groupbyList); //Set GroupBy columns + // Order By List + // CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + // csep->orderByCols(csep_orderbyList); //Set OrderBy columns - // Order By List - //CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - //csep->orderByCols(csep_orderbyList); //Set OrderBy columns + // filterList->walk(walkfnString); ?? - //filterList->walk(walkfnString); ?? - - // Print the parse tree - ParseTree* pt = const_cast(csep->filters()); - pt->drawTree("selectExecutionPlan_19.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 19 ---" << endl; - - } + // Print the parse tree + ParseTree* pt = const_cast(csep->filters()); + pt->drawTree("selectExecutionPlan_19.dot"); + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 19 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver2.cpp b/dbcon/execplan/tdriver2.cpp index cf1088bd8..71d88593b 100644 --- a/dbcon/execplan/tdriver2.cpp +++ b/dbcon/execplan/tdriver2.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ s_acctbal,\ s_name,\ @@ -112,160 +111,142 @@ public: s_name,\ p_partkey;"; - CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan csep; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - SimpleColumn* c1 = new SimpleColumn("tpch.supplier.s_acctbal"); - returnedColumnList.push_back(c1); + SimpleColumn* c1 = new SimpleColumn("tpch.supplier.s_acctbal"); + returnedColumnList.push_back(c1); - SimpleColumn* c2 = new SimpleColumn("tpch.supplier.s_name"); - returnedColumnList.push_back(c2); + SimpleColumn* c2 = new SimpleColumn("tpch.supplier.s_name"); + returnedColumnList.push_back(c2); - SimpleColumn* c3 = new SimpleColumn("tpch.nation.n_name"); - returnedColumnList.push_back(c3); + SimpleColumn* c3 = new SimpleColumn("tpch.nation.n_name"); + returnedColumnList.push_back(c3); - SimpleColumn* c4 = new SimpleColumn("tpch.part.p_partkey"); - returnedColumnList.push_back(c4); + SimpleColumn* c4 = new SimpleColumn("tpch.part.p_partkey"); + returnedColumnList.push_back(c4); - SimpleColumn* c5 = new SimpleColumn("tpch.part.p_mfgr"); - returnedColumnList.push_back(c5); + SimpleColumn* c5 = new SimpleColumn("tpch.part.p_mfgr"); + returnedColumnList.push_back(c5); - SimpleColumn* c6 = new SimpleColumn("tpch.supplier.s_address"); - returnedColumnList.push_back(c6); + SimpleColumn* c6 = new SimpleColumn("tpch.supplier.s_address"); + returnedColumnList.push_back(c6); - SimpleColumn* c7 = new SimpleColumn("tpch.supplier.s_phone"); - returnedColumnList.push_back(c7); + SimpleColumn* c7 = new SimpleColumn("tpch.supplier.s_phone"); + returnedColumnList.push_back(c7); - SimpleColumn* c8 = new SimpleColumn("tpch.supplier.s_comment"); - returnedColumnList.push_back(c8); + SimpleColumn* c8 = new SimpleColumn("tpch.supplier.s_comment"); + returnedColumnList.push_back(c8); - csep.returnedCols(returnedColumnList); + csep.returnedCols(returnedColumnList); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.part.p_partkey"), - new SimpleColumn("tpch.partsupp.ps_partkey")); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_suppkey"), - new SimpleColumn("tpch.partsupp.ps_suppkey")); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.part.p_size"), - new ConstantColumn(1)); - filterTokenList.push_back(f3); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f4 = new SimpleFilter (new Operator("like"), - new SimpleColumn("tpch.part.p_type"), - new ConstantColumn(":2")); - filterTokenList.push_back(f4); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - new SimpleColumn("tpch.nation.n_nationkey")); - filterTokenList.push_back(f5); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f6 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.nation.n_regionkey"), - new SimpleColumn("tpch.region.r_regionkey")); - filterTokenList.push_back(f6); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f7 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.region.r_name"), - new ConstantColumn(":3")); - filterTokenList.push_back(f7); - filterTokenList.push_back(new Operator("and")); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.partsupp.ps_partkey")); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_suppkey"), + new SimpleColumn("tpch.partsupp.ps_suppkey")); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f3 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_size"), new ConstantColumn(1)); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f4 = new SimpleFilter(new Operator("like"), new SimpleColumn("tpch.part.p_type"), + new ConstantColumn(":2")); + filterTokenList.push_back(f4); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f5 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + filterTokenList.push_back(f5); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f6 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_regionkey"), + new SimpleColumn("tpch.region.r_regionkey")); + filterTokenList.push_back(f6); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f7 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.region.r_name"), new ConstantColumn(":3")); + filterTokenList.push_back(f7); + filterTokenList.push_back(new Operator("and")); - // sub select execution plan for select filter - CalpontSelectExecutionPlan* subsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // sub select execution plan for select filter + CalpontSelectExecutionPlan* subsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - // subselect returned columns - CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; - ArithmeticColumn* sc1 = new ArithmeticColumn("min(tpch.partsupp.ps_supplycost)"); - subReturnedColList.push_back(sc1); - subsep->returnedCols(subReturnedColList); + // subselect returned columns + CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; + ArithmeticColumn* sc1 = new ArithmeticColumn("min(tpch.partsupp.ps_supplycost)"); + subReturnedColList.push_back(sc1); + subsep->returnedCols(subReturnedColList); - // subselect filters - CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + // subselect filters + CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SimpleFilter* sf1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.part.p_partkey"), - new SimpleColumn("tpch.partsupp.ps_partkey")); - subFilterTokenList.push_back(sf1); + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.partsupp.ps_partkey")); + subFilterTokenList.push_back(sf1); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.part.p_partkey"), - new SimpleColumn("tpch.partsupp.ps_partkey")); - subFilterTokenList.push_back(sf2); - subFilterTokenList.push_back(new Operator("and")); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.partsupp.ps_partkey")); + subFilterTokenList.push_back(sf2); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - new SimpleColumn("tpch.nation.n_nationkey")); - subFilterTokenList.push_back(sf3); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf4 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.nation.n_regionkey"), - new SimpleColumn("tpch.region.r_regionkey")); - subFilterTokenList.push_back(sf4); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.region.r_name"), - new ConstantColumn(":3")); - subFilterTokenList.push_back(sf5); + SimpleFilter* sf3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + subFilterTokenList.push_back(sf3); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf4 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_regionkey"), + new SimpleColumn("tpch.region.r_regionkey")); + subFilterTokenList.push_back(sf4); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf5 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.region.r_name"), new ConstantColumn(":3")); + subFilterTokenList.push_back(sf5); - subsep->filterTokenList(subFilterTokenList); + subsep->filterTokenList(subFilterTokenList); - // end of sub select - SelectFilter* f8 = new SelectFilter (new SimpleColumn("tpch.partsupp.ps_supplycost"), - new Operator("="), - subsep); + // end of sub select + SelectFilter* f8 = + new SelectFilter(new SimpleColumn("tpch.partsupp.ps_supplycost"), new Operator("="), subsep); - filterTokenList.push_back(f8); - csep.filterTokenList(filterTokenList); + filterTokenList.push_back(f8); + csep.filterTokenList(filterTokenList); - //ParseTree* pt = const_cast(csep.filters()); - //pt->drawTree("q2.dot"); + // ParseTree* pt = const_cast(csep.filters()); + // pt->drawTree("q2.dot"); - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - SimpleColumn* o1 = new SimpleColumn(*c1); - o1->asc(false); - orderByList.push_back(o1); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + SimpleColumn* o1 = new SimpleColumn(*c1); + o1->asc(false); + orderByList.push_back(o1); - SimpleColumn* o2 = new SimpleColumn(*c3); - orderByList.push_back(o2); + SimpleColumn* o2 = new SimpleColumn(*c3); + orderByList.push_back(o2); - SimpleColumn* o3 = new SimpleColumn(*c2); - orderByList.push_back(o3); - - csep.orderByCols(orderByList); - - cout << csep; - } + SimpleColumn* o3 = new SimpleColumn(*c2); + orderByList.push_back(o3); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver20.cpp b/dbcon/execplan/tdriver20.cpp index 6981a7980..2d6c0ecca 100644 --- a/dbcon/execplan/tdriver20.cpp +++ b/dbcon/execplan/tdriver20.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -51,101 +51,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_20); - CPPUNIT_TEST( selectExecutionPlan_20 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_20() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_20() + { + cout << "SQL: \ select \ s_name, \ s_address \ @@ -183,198 +179,182 @@ where \ and n_name = ':3' \ order by \ s_name;" - << endl; + << endl; - /* ------------The create view statement is being re-written as an inline view ----- */ - /* ------------ (dynamic table) to be used in a from cluase. ----- */ + /* ------------The create view statement is being re-written as an inline view ----- */ + /* ------------ (dynamic table) to be used in a from cluase. ----- */ - //This is the main query body for query 20 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 20 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* c1 = new SimpleColumn("tpch.supplier.s_name"); - colList.push_back(c1); - SimpleColumn* c2 = new SimpleColumn("tpch.supplier.s_address"); - colList.push_back(c2); - csep->returnedCols(colList); // set Projection columns + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* c1 = new SimpleColumn("tpch.supplier.s_name"); + colList.push_back(c1); + SimpleColumn* c2 = new SimpleColumn("tpch.supplier.s_address"); + colList.push_back(c2); + csep->returnedCols(colList); // set Projection columns - cout << "I am here 1" << endl; - // Filter columns (csep) - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + cout << "I am here 1" << endl; + // Filter columns (csep) + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - // --------------- first sub select begins here ---------------- x - //Sub select filter - CalpontSelectExecutionPlan* sub01csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // --------------- first sub select begins here ---------------- x + // Sub select filter + CalpontSelectExecutionPlan* sub01csep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub01ColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub01ColList; - SimpleColumn* s01c1 = new SimpleColumn ("tpch.partsupp.ps_suppkey"); - sub01ColList.push_back(s01c1); - // Append returned columns to subselect - sub01csep->returnedCols(sub01ColList); + SimpleColumn* s01c1 = new SimpleColumn("tpch.partsupp.ps_suppkey"); + sub01ColList.push_back(s01c1); + // Append returned columns to subselect + sub01csep->returnedCols(sub01ColList); - cout << "I am here 2" << endl; - //subselect Filters (sub011csep) - CalpontSelectExecutionPlan::FilterTokenList sub01csep_filterlist; + cout << "I am here 2" << endl; + // subselect Filters (sub011csep) + CalpontSelectExecutionPlan::FilterTokenList sub01csep_filterlist; - // --------------- inner sub select begins here ---------------- x - //Sub select filter - CalpontSelectExecutionPlan* sub011csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // --------------- inner sub select begins here ---------------- x + // Sub select filter + CalpontSelectExecutionPlan* sub011csep = + new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub011ColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub011ColList; - SimpleColumn* s011c1 = new SimpleColumn ("tpch.part.p_partkey"); - sub011ColList.push_back(s011c1); - // Append returned columns to subselect - sub011csep->returnedCols(sub011ColList); + SimpleColumn* s011c1 = new SimpleColumn("tpch.part.p_partkey"); + sub011ColList.push_back(s011c1); + // Append returned columns to subselect + sub011csep->returnedCols(sub011ColList); - cout << "I am here 3" << endl; - //subselect Filters (sub011csep) - CalpontSelectExecutionPlan::FilterTokenList sub011csep_filterlist; - SimpleFilter* s011f1 = new SimpleFilter ( new Operator("LIKE"), - new SimpleColumn ("tpch.part.p_name"), - new ConstantColumn("%:1")); - sub011csep_filterlist.push_back(s011f1); - sub011csep->filterTokenList(sub011csep_filterlist); + cout << "I am here 3" << endl; + // subselect Filters (sub011csep) + CalpontSelectExecutionPlan::FilterTokenList sub011csep_filterlist; + SimpleFilter* s011f1 = new SimpleFilter(new Operator("LIKE"), new SimpleColumn("tpch.part.p_name"), + new ConstantColumn("%:1")); + sub011csep_filterlist.push_back(s011f1); + sub011csep->filterTokenList(sub011csep_filterlist); - // --------------- inner sub ends here ---------------- x + // --------------- inner sub ends here ---------------- x - // --------------- second inner sub select begins here ---------------- x - //Sub select filter - CalpontSelectExecutionPlan* sub012csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // --------------- second inner sub select begins here ---------------- x + // Sub select filter + CalpontSelectExecutionPlan* sub012csep = + new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub012ColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub012ColList; - ArithmeticColumn* s012c1 = new ArithmeticColumn ("0.5 * sum(l_quantity)"); - sub012ColList.push_back(s012c1); - // Append returned columns to subselect - sub012csep->returnedCols(sub012ColList); + ArithmeticColumn* s012c1 = new ArithmeticColumn("0.5 * sum(l_quantity)"); + sub012ColList.push_back(s012c1); + // Append returned columns to subselect + sub012csep->returnedCols(sub012ColList); - cout << "I am here 31" << endl; - //subselect Filters (sub012csep) - CalpontSelectExecutionPlan::FilterTokenList sub012csep_filterlist; - SimpleFilter* s012f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_partkey"), - new SimpleColumn ("tpch.partsupp.ps_partkey")); - sub012csep_filterlist.push_back(s012f1); + cout << "I am here 31" << endl; + // subselect Filters (sub012csep) + CalpontSelectExecutionPlan::FilterTokenList sub012csep_filterlist; + SimpleFilter* s012f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_partkey"), + new SimpleColumn("tpch.partsupp.ps_partkey")); + sub012csep_filterlist.push_back(s012f1); - sub012csep_filterlist.push_back(new Operator("and")); + sub012csep_filterlist.push_back(new Operator("and")); - SimpleFilter* s012f2 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_suppkey"), - new SimpleColumn ("tpch.partsupp.ps_suppkey")); - sub012csep_filterlist.push_back(s012f2); + SimpleFilter* s012f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_suppkey"), + new SimpleColumn("tpch.partsupp.ps_suppkey")); + sub012csep_filterlist.push_back(s012f2); - sub012csep_filterlist.push_back(new Operator("and")); + sub012csep_filterlist.push_back(new Operator("and")); - SimpleFilter* s012f3 = new SimpleFilter ( new Operator(">="), - new SimpleColumn ("tpch.lineitem.l_shipdate"), - new ConstantColumn ("date ':2'")); - sub012csep_filterlist.push_back(s012f3); + SimpleFilter* s012f3 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ConstantColumn("date ':2'")); + sub012csep_filterlist.push_back(s012f3); - sub012csep_filterlist.push_back(new Operator("and")); + sub012csep_filterlist.push_back(new Operator("and")); - SimpleFilter* s012f4 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_shipdate"), - new ConstantColumn ("date ':2' + interval '1' year")); - sub012csep_filterlist.push_back(s012f4); + SimpleFilter* s012f4 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ConstantColumn("date ':2' + interval '1' year")); + sub012csep_filterlist.push_back(s012f4); - sub012csep->filterTokenList(sub012csep_filterlist); + sub012csep->filterTokenList(sub012csep_filterlist); + // --------------- second inner sub ends here ---------------- x - // --------------- second inner sub ends here ---------------- x + // Now resolve the first inner sub_query + SelectFilter* s01f1 = + new SelectFilter(new SimpleColumn("tpch.partsupp.ps_suppkey"), new Operator("IN"), sub011csep); - // Now resolve the first inner sub_query - SelectFilter* s01f1 = new SelectFilter (new SimpleColumn ("tpch.partsupp.ps_suppkey"), - new Operator("IN"), - sub011csep); + // Now resolve the second inner sub_query + SelectFilter* s01f2 = + new SelectFilter(new SimpleColumn("tpch.partsupp.ps_availqty"), new Operator(">"), sub012csep); - // Now resolve the second inner sub_query - SelectFilter* s01f2 = new SelectFilter (new SimpleColumn ("tpch.partsupp.ps_availqty"), - new Operator(">"), - sub012csep); + sub01csep_filterlist.push_back(s01f1); + sub01csep_filterlist.push_back(new Operator("and")); + sub01csep_filterlist.push_back(s01f2); + sub01csep->filterTokenList(sub01csep_filterlist); - sub01csep_filterlist.push_back(s01f1); - sub01csep_filterlist.push_back(new Operator("and")); - sub01csep_filterlist.push_back(s01f2); - sub01csep->filterTokenList(sub01csep_filterlist); + cout << "I am here 4" << endl; + // --------------- first sub ends here ---------------- x + // Now resolve the outer query + SelectFilter* f1 = + new SelectFilter(new SimpleColumn("tpch.supplier.s_suppkey"), new Operator("IN"), sub01csep); + csep_filterlist.push_back(f1); + cout << "I am here 5" << endl; - cout << "I am here 4" << endl; - // --------------- first sub ends here ---------------- x - // Now resolve the outer query - SelectFilter* f1 = new SelectFilter (new SimpleColumn ("tpch.supplier.s_suppkey"), - new Operator("IN"), - sub01csep); - csep_filterlist.push_back(f1); - cout << "I am here 5" << endl; + csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + csep_filterlist.push_back(f2); - SimpleFilter* f2 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.supplier.s_nationkey"), - new SimpleColumn ("tpch.nation.n_nationkey")); - csep_filterlist.push_back(f2); + csep_filterlist.push_back(new Operator("and")); - csep_filterlist.push_back(new Operator("and")); + SimpleFilter* f3 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_name"), new ConstantColumn(":3")); + csep_filterlist.push_back(f3); - SimpleFilter* f3 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.nation.n_name"), - new ConstantColumn(":3")); - csep_filterlist.push_back(f3); + cout << "I am here 6" << endl; + csep->filterTokenList(csep_filterlist); - cout << "I am here 6" << endl; - csep->filterTokenList(csep_filterlist); + // Build Group By List + /* + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + SimpleColumn *g1 = new SimpleColumn("tpch.part.p_brand"); + csep_groupbyList.push_back(g1); + csep->groupByCols(csep_groupbyList); //Set GroupBy columns + */ - // Build Group By List - /* - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - SimpleColumn *g1 = new SimpleColumn("tpch.part.p_brand"); - csep_groupbyList.push_back(g1); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns - */ + // Order By List + cout << "I am here 7" << endl; + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + SimpleColumn* o1 = new SimpleColumn("tpch.supplier.s_name"); + csep_orderbyList.push_back(o1); + csep->orderByCols(csep_orderbyList); // Set OrderBy columns + cout << "I am here 8" << endl; - // Order By List - cout << "I am here 7" << endl; - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - SimpleColumn* o1 = new SimpleColumn("tpch.supplier.s_name"); - csep_orderbyList.push_back(o1); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns - cout << "I am here 8" << endl; + // filterList->walk(walkfnString); ?? - //filterList->walk(walkfnString); ?? - - // Print the parse tree - ParseTree* pt = const_cast(csep->filters()); - cout << "I am here 9" << endl; - pt->drawTree("selectExecutionPlan_20.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 20 ---" << endl; - - } + // Print the parse tree + ParseTree* pt = const_cast(csep->filters()); + cout << "I am here 9" << endl; + pt->drawTree("selectExecutionPlan_20.dot"); + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 20 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver21.cpp b/dbcon/execplan/tdriver21.cpp index 8af08ce2f..52d9f4edc 100644 --- a/dbcon/execplan/tdriver21.cpp +++ b/dbcon/execplan/tdriver21.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -51,101 +51,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_21); - CPPUNIT_TEST( selectExecutionPlan_21 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_21() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_21() + { + cout << "SQL: \ select \ s_name, \ s_address \ @@ -183,268 +179,251 @@ where \ and n_name = ':3' \ order by \ s_name;" - << endl; + << endl; -//NOTE(7/19): Add "NOT" to second Exists stanza sub02csep, filter f6. + // NOTE(7/19): Add "NOT" to second Exists stanza sub02csep, filter f6. - //This is the main query body for query 21 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 21 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* c1 = new SimpleColumn("tpch.supplier.s_name"); - colList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn("count(all)"); - c2->alias("numwait"); - colList.push_back(c2); - csep->returnedCols(colList); // set Projection columns + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* c1 = new SimpleColumn("tpch.supplier.s_name"); + colList.push_back(c1); + ArithmeticColumn* c2 = new ArithmeticColumn("count(all)"); + c2->alias("numwait"); + colList.push_back(c2); + csep->returnedCols(colList); // set Projection columns - // Filter columns (csep) - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns (csep) + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - SimpleFilter* f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.supplier.s_suppkey"), - new SimpleColumn ("tpch.lineitem.l_suppkey")); - csep_filterlist.push_back(f1); + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + csep_filterlist.push_back(f1); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.orders.o_orderkey"), - new SimpleColumn ("tpch.lineitem.l_orderkey")); - csep_filterlist.push_back(f2); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.orders.o_orderkey"), + new SimpleColumn("tpch.lineitem.l_orderkey")); + csep_filterlist.push_back(f2); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.orders.o_orderstatus"), - new ConstantColumn ("F")); - csep_filterlist.push_back(f3); + SimpleFilter* f3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.orders.o_orderstatus"), + new ConstantColumn("F")); + csep_filterlist.push_back(f3); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f4 = new SimpleFilter ( new Operator(">"), - new SimpleColumn ("tpch.lineitem.l_receiptdate"), - new SimpleColumn ("tpch.lineitem.l_commitdate")); - csep_filterlist.push_back(f4); + SimpleFilter* f4 = new SimpleFilter(new Operator(">"), new SimpleColumn("tpch.lineitem.l_receiptdate"), + new SimpleColumn("tpch.lineitem.l_commitdate")); + csep_filterlist.push_back(f4); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - // --------------- first sub select begins here ---------------- x - //Sub select filter - // Note: A select "*" is translated to select - // each fully qualified column in an Oracle execution plan. In these - // examples the shorthand notation for "*" will be "ALL" and the query - // will be written as it would be represented in an exeution pland. + // --------------- first sub select begins here ---------------- x + // Sub select filter + // Note: A select "*" is translated to select + // each fully qualified column in an Oracle execution plan. In these + // examples the shorthand notation for "*" will be "ALL" and the query + // will be written as it would be represented in an exeution pland. - CalpontSelectExecutionPlan* sub01csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + CalpontSelectExecutionPlan* sub01csep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub01ColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub01ColList; - SimpleColumn* s01c1 = new SimpleColumn ("tpch.lineitem.l_orderkey"); - sub01ColList.push_back(s01c1); - SimpleColumn* s01c2 = new SimpleColumn ("tpch.lineitem.l_partkey"); - sub01ColList.push_back(s01c2); - SimpleColumn* s01c3 = new SimpleColumn ("tpch.lineitem.l_suppkey"); - sub01ColList.push_back(s01c3); - SimpleColumn* s01c4 = new SimpleColumn ("tpch.lineitem.l_linenumber"); - sub01ColList.push_back(s01c4); - SimpleColumn* s01c5 = new SimpleColumn ("tpch.lineitem.l_quantity"); - sub01ColList.push_back(s01c5); - SimpleColumn* s01c6 = new SimpleColumn ("tpch.lineitem.l_extenedprice"); - sub01ColList.push_back(s01c6); - SimpleColumn* s01c7 = new SimpleColumn ("tpch.lineitem.l_discount"); - sub01ColList.push_back(s01c7); - SimpleColumn* s01c8 = new SimpleColumn ("tpch.lineitem.l_tax"); - sub01ColList.push_back(s01c8); - SimpleColumn* s01c9 = new SimpleColumn ("tpch.lineitem.l_returnflag"); - sub01ColList.push_back(s01c9); - SimpleColumn* s01c10 = new SimpleColumn ("tpch.lineitem.l_linestatus"); - sub01ColList.push_back(s01c10); - SimpleColumn* s01c11 = new SimpleColumn ("tpch.lineitem.l_shipdate"); - sub01ColList.push_back(s01c11); - SimpleColumn* s01c12 = new SimpleColumn ("tpch.lineitem.l_commitdate"); - sub01ColList.push_back(s01c12); - SimpleColumn* s01c13 = new SimpleColumn ("tpch.lineitem.l_receiptdate"); - sub01ColList.push_back(s01c13); - SimpleColumn* s01c14 = new SimpleColumn ("tpch.lineitem.l_shipinstruct"); - sub01ColList.push_back(s01c14); - SimpleColumn* s01c15 = new SimpleColumn ("tpch.lineitem.l_shipmode"); - sub01ColList.push_back(s01c15); - SimpleColumn* s01c16 = new SimpleColumn ("tpch.lineitem.l_comment"); - sub01ColList.push_back(s01c16); - // Append returned columns to subselect - sub01csep->returnedCols(sub01ColList); + SimpleColumn* s01c1 = new SimpleColumn("tpch.lineitem.l_orderkey"); + sub01ColList.push_back(s01c1); + SimpleColumn* s01c2 = new SimpleColumn("tpch.lineitem.l_partkey"); + sub01ColList.push_back(s01c2); + SimpleColumn* s01c3 = new SimpleColumn("tpch.lineitem.l_suppkey"); + sub01ColList.push_back(s01c3); + SimpleColumn* s01c4 = new SimpleColumn("tpch.lineitem.l_linenumber"); + sub01ColList.push_back(s01c4); + SimpleColumn* s01c5 = new SimpleColumn("tpch.lineitem.l_quantity"); + sub01ColList.push_back(s01c5); + SimpleColumn* s01c6 = new SimpleColumn("tpch.lineitem.l_extenedprice"); + sub01ColList.push_back(s01c6); + SimpleColumn* s01c7 = new SimpleColumn("tpch.lineitem.l_discount"); + sub01ColList.push_back(s01c7); + SimpleColumn* s01c8 = new SimpleColumn("tpch.lineitem.l_tax"); + sub01ColList.push_back(s01c8); + SimpleColumn* s01c9 = new SimpleColumn("tpch.lineitem.l_returnflag"); + sub01ColList.push_back(s01c9); + SimpleColumn* s01c10 = new SimpleColumn("tpch.lineitem.l_linestatus"); + sub01ColList.push_back(s01c10); + SimpleColumn* s01c11 = new SimpleColumn("tpch.lineitem.l_shipdate"); + sub01ColList.push_back(s01c11); + SimpleColumn* s01c12 = new SimpleColumn("tpch.lineitem.l_commitdate"); + sub01ColList.push_back(s01c12); + SimpleColumn* s01c13 = new SimpleColumn("tpch.lineitem.l_receiptdate"); + sub01ColList.push_back(s01c13); + SimpleColumn* s01c14 = new SimpleColumn("tpch.lineitem.l_shipinstruct"); + sub01ColList.push_back(s01c14); + SimpleColumn* s01c15 = new SimpleColumn("tpch.lineitem.l_shipmode"); + sub01ColList.push_back(s01c15); + SimpleColumn* s01c16 = new SimpleColumn("tpch.lineitem.l_comment"); + sub01ColList.push_back(s01c16); + // Append returned columns to subselect + sub01csep->returnedCols(sub01ColList); - cout << "I am here 2" << endl; - //subselect Filters (sub01csep) - CalpontSelectExecutionPlan::FilterTokenList sub01csep_filterlist; + cout << "I am here 2" << endl; + // subselect Filters (sub01csep) + CalpontSelectExecutionPlan::FilterTokenList sub01csep_filterlist; - SimpleFilter* s01f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_orderkey"), - new SimpleColumn ("tpch.lineitem.l_orderkey")); - sub01csep_filterlist.push_back(s01f1); + SimpleFilter* s01f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_orderkey"), + new SimpleColumn("tpch.lineitem.l_orderkey")); + sub01csep_filterlist.push_back(s01f1); - sub01csep_filterlist.push_back(new Operator("and")); + sub01csep_filterlist.push_back(new Operator("and")); - SimpleFilter* s01f2 = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.lineitem.l_suppkey"), - new SimpleColumn ("tpch.lineitem.l_suppkey")); - sub01csep_filterlist.push_back(s01f2); + SimpleFilter* s01f2 = new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.lineitem.l_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + sub01csep_filterlist.push_back(s01f2); - sub01csep->filterTokenList(sub01csep_filterlist); + sub01csep->filterTokenList(sub01csep_filterlist); - cout << "I am here 4" << endl; - // --------------- first sub ends here ---------------- x - // Now resolve the outer query - ExistsFilter* f5 = new ExistsFilter (sub01csep); - csep_filterlist.push_back(f5); - cout << "I am here 5" << endl; + cout << "I am here 4" << endl; + // --------------- first sub ends here ---------------- x + // Now resolve the outer query + ExistsFilter* f5 = new ExistsFilter(sub01csep); + csep_filterlist.push_back(f5); + cout << "I am here 5" << endl; - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - // --------------- second sub select begins here ---------------- x - //Sub select filter - // Note: A select "*" is translated to select - // each fully qualified column in an Oracle execution plan. In these - // examples the shorthand notation for "*" will be "ALL" and the query - // will be written as it would be represented in an exeution pland. + // --------------- second sub select begins here ---------------- x + // Sub select filter + // Note: A select "*" is translated to select + // each fully qualified column in an Oracle execution plan. In these + // examples the shorthand notation for "*" will be "ALL" and the query + // will be written as it would be represented in an exeution pland. - CalpontSelectExecutionPlan* sub02csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + CalpontSelectExecutionPlan* sub02csep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub02ColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub02ColList; - SimpleColumn* s02c1 = new SimpleColumn ("tpch.lineitem.l_orderkey"); - sub01ColList.push_back(s02c1); - SimpleColumn* s02c2 = new SimpleColumn ("tpch.lineitem.l_partkey"); - sub01ColList.push_back(s02c2); - SimpleColumn* s02c3 = new SimpleColumn ("tpch.lineitem.l_suppkey"); - sub01ColList.push_back(s02c3); - SimpleColumn* s02c4 = new SimpleColumn ("tpch.lineitem.l_linenumber"); - sub01ColList.push_back(s02c4); - SimpleColumn* s02c5 = new SimpleColumn ("tpch.lineitem.l_quantity"); - sub01ColList.push_back(s02c5); - SimpleColumn* s02c6 = new SimpleColumn ("tpch.lineitem.l_extenedprice"); - sub01ColList.push_back(s02c6); - SimpleColumn* s02c7 = new SimpleColumn ("tpch.lineitem.l_discount"); - sub01ColList.push_back(s02c7); - SimpleColumn* s02c8 = new SimpleColumn ("tpch.lineitem.l_tax"); - sub01ColList.push_back(s02c8); - SimpleColumn* s02c9 = new SimpleColumn ("tpch.lineitem.l_returnflag"); - sub01ColList.push_back(s02c9); - SimpleColumn* s02c10 = new SimpleColumn ("tpch.lineitem.l_linestatus"); - sub01ColList.push_back(s02c10); - SimpleColumn* s02c11 = new SimpleColumn ("tpch.lineitem.l_shipdate"); - sub01ColList.push_back(s02c11); - SimpleColumn* s02c12 = new SimpleColumn ("tpch.lineitem.l_commitdate"); - sub01ColList.push_back(s02c12); - SimpleColumn* s02c13 = new SimpleColumn ("tpch.lineitem.l_receiptdate"); - sub01ColList.push_back(s02c13); - SimpleColumn* s02c14 = new SimpleColumn ("tpch.lineitem.l_shipinstruct"); - sub01ColList.push_back(s02c14); - SimpleColumn* s02c15 = new SimpleColumn ("tpch.lineitem.l_shipmode"); - sub01ColList.push_back(s02c15); - SimpleColumn* s02c16 = new SimpleColumn ("tpch.lineitem.l_comment"); - sub01ColList.push_back(s02c16); - // Append returned columns to subselect - sub02csep->returnedCols(sub02ColList); + SimpleColumn* s02c1 = new SimpleColumn("tpch.lineitem.l_orderkey"); + sub01ColList.push_back(s02c1); + SimpleColumn* s02c2 = new SimpleColumn("tpch.lineitem.l_partkey"); + sub01ColList.push_back(s02c2); + SimpleColumn* s02c3 = new SimpleColumn("tpch.lineitem.l_suppkey"); + sub01ColList.push_back(s02c3); + SimpleColumn* s02c4 = new SimpleColumn("tpch.lineitem.l_linenumber"); + sub01ColList.push_back(s02c4); + SimpleColumn* s02c5 = new SimpleColumn("tpch.lineitem.l_quantity"); + sub01ColList.push_back(s02c5); + SimpleColumn* s02c6 = new SimpleColumn("tpch.lineitem.l_extenedprice"); + sub01ColList.push_back(s02c6); + SimpleColumn* s02c7 = new SimpleColumn("tpch.lineitem.l_discount"); + sub01ColList.push_back(s02c7); + SimpleColumn* s02c8 = new SimpleColumn("tpch.lineitem.l_tax"); + sub01ColList.push_back(s02c8); + SimpleColumn* s02c9 = new SimpleColumn("tpch.lineitem.l_returnflag"); + sub01ColList.push_back(s02c9); + SimpleColumn* s02c10 = new SimpleColumn("tpch.lineitem.l_linestatus"); + sub01ColList.push_back(s02c10); + SimpleColumn* s02c11 = new SimpleColumn("tpch.lineitem.l_shipdate"); + sub01ColList.push_back(s02c11); + SimpleColumn* s02c12 = new SimpleColumn("tpch.lineitem.l_commitdate"); + sub01ColList.push_back(s02c12); + SimpleColumn* s02c13 = new SimpleColumn("tpch.lineitem.l_receiptdate"); + sub01ColList.push_back(s02c13); + SimpleColumn* s02c14 = new SimpleColumn("tpch.lineitem.l_shipinstruct"); + sub01ColList.push_back(s02c14); + SimpleColumn* s02c15 = new SimpleColumn("tpch.lineitem.l_shipmode"); + sub01ColList.push_back(s02c15); + SimpleColumn* s02c16 = new SimpleColumn("tpch.lineitem.l_comment"); + sub01ColList.push_back(s02c16); + // Append returned columns to subselect + sub02csep->returnedCols(sub02ColList); - cout << "I am here 3" << endl; - //subselect Filters (sub01csep) - CalpontSelectExecutionPlan::FilterTokenList sub02csep_filterlist; + cout << "I am here 3" << endl; + // subselect Filters (sub01csep) + CalpontSelectExecutionPlan::FilterTokenList sub02csep_filterlist; - SimpleFilter* s02f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.lineitem.l_orderkey"), - new SimpleColumn ("tpch.lineitem.l_orderkey")); - sub02csep_filterlist.push_back(s02f1); + SimpleFilter* s02f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_orderkey"), + new SimpleColumn("tpch.lineitem.l_orderkey")); + sub02csep_filterlist.push_back(s02f1); - sub02csep_filterlist.push_back(new Operator("and")); + sub02csep_filterlist.push_back(new Operator("and")); - SimpleFilter* s02f2 = new SimpleFilter ( new Operator("<>"), - new SimpleColumn ("tpch.lineitem.l_suppkey"), - new SimpleColumn ("tpch.lineitem.l_suppkey")); - sub02csep_filterlist.push_back(s02f2); + SimpleFilter* s02f2 = new SimpleFilter(new Operator("<>"), new SimpleColumn("tpch.lineitem.l_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + sub02csep_filterlist.push_back(s02f2); - sub02csep_filterlist.push_back(new Operator("and")); + sub02csep_filterlist.push_back(new Operator("and")); - SimpleFilter* s02f3 = new SimpleFilter ( new Operator(">"), - new SimpleColumn ("tpch.lineitem.l_receiptdate"), - new SimpleColumn ("tpch.lineitem.l_commitdate")); - sub02csep_filterlist.push_back(s02f3); - sub02csep->filterTokenList(sub02csep_filterlist); + SimpleFilter* s02f3 = new SimpleFilter(new Operator(">"), new SimpleColumn("tpch.lineitem.l_receiptdate"), + new SimpleColumn("tpch.lineitem.l_commitdate")); + sub02csep_filterlist.push_back(s02f3); + sub02csep->filterTokenList(sub02csep_filterlist); - cout << "I am here 4" << endl; - // --------------- first sub ends here ---------------- x - // Now resolve the outer query - ExistsFilter* f6 = new ExistsFilter (sub02csep, true); - csep_filterlist.push_back(f6); - cout << "I am here 5" << endl; + cout << "I am here 4" << endl; + // --------------- first sub ends here ---------------- x + // Now resolve the outer query + ExistsFilter* f6 = new ExistsFilter(sub02csep, true); + csep_filterlist.push_back(f6); + cout << "I am here 5" << endl; - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f7 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.supplier.s_nationkey"), - new SimpleColumn ("tpch.nation.n_nationkey")); - csep_filterlist.push_back(f7); + SimpleFilter* f7 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + csep_filterlist.push_back(f7); - csep_filterlist.push_back(new Operator("and")); + csep_filterlist.push_back(new Operator("and")); - SimpleFilter* f8 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.nation.n_name"), - new ConstantColumn(":1")); - csep_filterlist.push_back(f8); + SimpleFilter* f8 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.nation.n_name"), new ConstantColumn(":1")); + csep_filterlist.push_back(f8); - cout << "I am here 6" << endl; - csep->filterTokenList(csep_filterlist); + cout << "I am here 6" << endl; + csep->filterTokenList(csep_filterlist); - // Build Group By List - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - SimpleColumn* g1 = new SimpleColumn("tpch.supplier.s_name"); - csep_groupbyList.push_back(g1); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns + // Build Group By List + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + SimpleColumn* g1 = new SimpleColumn("tpch.supplier.s_name"); + csep_groupbyList.push_back(g1); + csep->groupByCols(csep_groupbyList); // Set GroupBy columns - // Order By List - cout << "I am here 7" << endl; - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - SimpleColumn* o1 = new SimpleColumn("tpch.temp.numwait"); - o1->asc(false); - csep_orderbyList.push_back(o1); - SimpleColumn* o2 = new SimpleColumn("tpch.supplier.s_name"); - csep_orderbyList.push_back(o2); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns - cout << "I am here 8" << endl; + // Order By List + cout << "I am here 7" << endl; + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + SimpleColumn* o1 = new SimpleColumn("tpch.temp.numwait"); + o1->asc(false); + csep_orderbyList.push_back(o1); + SimpleColumn* o2 = new SimpleColumn("tpch.supplier.s_name"); + csep_orderbyList.push_back(o2); + csep->orderByCols(csep_orderbyList); // Set OrderBy columns + cout << "I am here 8" << endl; - //filterList->walk(walkfnString); ?? + // filterList->walk(walkfnString); ?? - // Print the parse tree - ParseTree* pt = const_cast(csep->filters()); - cout << "I am here 9" << endl; - pt->drawTree("selectExecutionPlan_21.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 21 ---" << endl; - - } + // Print the parse tree + ParseTree* pt = const_cast(csep->filters()); + cout << "I am here 9" << endl; + pt->drawTree("selectExecutionPlan_21.dot"); + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 21 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver22.cpp b/dbcon/execplan/tdriver22.cpp index b5dd37596..450b42814 100644 --- a/dbcon/execplan/tdriver22.cpp +++ b/dbcon/execplan/tdriver22.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -51,101 +51,97 @@ using namespace execplan; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(selectExecutionPlan_22); - CPPUNIT_TEST( selectExecutionPlan_22 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + static void walkfnString(const ParseTree* n) + { + char* r; + static bool is_init = false; + const char* mname = typeid(*(n->data())).name(); -private: -public: - - static void walkfnString(const ParseTree* n) + if (!is_init) { - char* r; - static bool is_init = false; - const char* mname = typeid(*(n->data())).name(); - - if (!is_init) - { - ::init_demangler(0, 0, 0); - is_init = true; - } - - r = ::cplus_demangle_with_style(mname, 7, 27); - - if (r != 0) - { - //cout << "mangle: " << mname << " demangle: " << r << endl; - ::free(r); - } - - if (typeid(*(n->data())) == typeid(SimpleFilter)) - { - cout << "SimpleFilter: " << endl; - const SimpleFilter* sf = dynamic_cast(n->data()); - const ReturnedColumn* lhs = sf->lhs(); - const ReturnedColumn* rhs = sf->rhs(); - const Operator* op = sf->op(); - cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); - cout << endl << "\t\t"; - - if (typeid(*lhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << lhs->data() << " / "; - } - else if (typeid(*lhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << lhs->data() << " / "; - } - else - { - cout << "UNK: " << lhs->data() << " / "; - } - - cout << "Operator: " << op->data() << " / "; - - if (typeid(*rhs) == typeid(SimpleColumn)) - { - cout << "SimpleColumn: " << rhs->data(); - } - else if (typeid(*rhs) == typeid(ConstantColumn)) - { - cout << "ConstantColumn: " << rhs->data(); - } - else - { - cout << "UNK: " << rhs->data(); - } - } - else if (typeid(*(n->data())) == typeid(Operator)) - { - cout << "Operator: "; - const Operator* op = dynamic_cast(n->data()); - cout << '\t' << op->data(); - } - else - { - cout << mname << " -x-: "; - } - - cout << endl; + ::init_demangler(0, 0, 0); + is_init = true; } - void setUp() + r = ::cplus_demangle_with_style(mname, 7, 27); + + if (r != 0) { + // cout << "mangle: " << mname << " demangle: " << r << endl; + ::free(r); } - void tearDown() + if (typeid(*(n->data())) == typeid(SimpleFilter)) { + cout << "SimpleFilter: " << endl; + const SimpleFilter* sf = dynamic_cast(n->data()); + const ReturnedColumn* lhs = sf->lhs(); + const ReturnedColumn* rhs = sf->rhs(); + const Operator* op = sf->op(); + cout << '\t' << lhs->data() << ' ' << op->data() << ' ' << rhs->data(); + cout << endl << "\t\t"; + + if (typeid(*lhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << lhs->data() << " / "; + } + else if (typeid(*lhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << lhs->data() << " / "; + } + else + { + cout << "UNK: " << lhs->data() << " / "; + } + + cout << "Operator: " << op->data() << " / "; + + if (typeid(*rhs) == typeid(SimpleColumn)) + { + cout << "SimpleColumn: " << rhs->data(); + } + else if (typeid(*rhs) == typeid(ConstantColumn)) + { + cout << "ConstantColumn: " << rhs->data(); + } + else + { + cout << "UNK: " << rhs->data(); + } + } + else if (typeid(*(n->data())) == typeid(Operator)) + { + cout << "Operator: "; + const Operator* op = dynamic_cast(n->data()); + cout << '\t' << op->data(); + } + else + { + cout << mname << " -x-: "; } - void selectExecutionPlan_22() - { + cout << endl; + } - cout << - "SQL: \ + void setUp() + { + } + + void tearDown() + { + } + + void selectExecutionPlan_22() + { + cout << "SQL: \ select \ cntrycode, \ count(*) as numcust, \ @@ -183,309 +179,294 @@ group by \ cntrycode \ order by \ cntrycode; " - << endl; + << endl; - /* ------------ Starting Query 22 from Here ----- */ + /* ------------ Starting Query 22 from Here ----- */ - //This is the main query body for query 22 - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + // This is the main query body for query 22 + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - // Create the Projection (returned columns) - CalpontSelectExecutionPlan::ReturnedColumnList colList; - SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.cntrycode"); - //c1->tableAlias("custsale"); - colList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn("count(ALL)"); - c2->alias("numcust"); - //c2->tableAlias("custsale"); - colList.push_back(c2); - ArithmeticColumn* c3 = new ArithmeticColumn("sum(c_acctbal)"); - c3->alias("totacctbal"); - //c3->tableAlias("custsale"); - colList.push_back(c3); - csep->returnedCols(colList); // set Projection columns + // Create the Projection (returned columns) + CalpontSelectExecutionPlan::ReturnedColumnList colList; + SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.cntrycode"); + // c1->tableAlias("custsale"); + colList.push_back(c1); + ArithmeticColumn* c2 = new ArithmeticColumn("count(ALL)"); + c2->alias("numcust"); + // c2->tableAlias("custsale"); + colList.push_back(c2); + ArithmeticColumn* c3 = new ArithmeticColumn("sum(c_acctbal)"); + c3->alias("totacctbal"); + // c3->tableAlias("custsale"); + colList.push_back(c3); + csep->returnedCols(colList); // set Projection columns - // Filter columns (csep) - CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; + // Filter columns (csep) + CalpontSelectExecutionPlan::FilterTokenList csep_filterlist; - // --------------- first sub select begins here ---------------- x - //Sub select filter - CalpontSelectExecutionPlan* sub01csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); + // --------------- first sub select begins here ---------------- x + // Sub select filter + CalpontSelectExecutionPlan* sub01csep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); - sub01csep->tableAlias("custsale"); + sub01csep->tableAlias("custsale"); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub01ColList; + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub01ColList; - ArithmeticColumn* s01c1 = new ArithmeticColumn ("substring(c_phone from 1 for 2 )"); - sub01ColList.push_back(s01c1); - s01c1->alias("cntrycode"); - SimpleColumn* s01c2 = new SimpleColumn ("tpch.customer.c_acctbal"); - sub01ColList.push_back(s01c2); - // Append returned columns to subselect - sub01csep->returnedCols(sub01ColList); + ArithmeticColumn* s01c1 = new ArithmeticColumn("substring(c_phone from 1 for 2 )"); + sub01ColList.push_back(s01c1); + s01c1->alias("cntrycode"); + SimpleColumn* s01c2 = new SimpleColumn("tpch.customer.c_acctbal"); + sub01ColList.push_back(s01c2); + // Append returned columns to subselect + sub01csep->returnedCols(sub01ColList); - //subselect Filters (sub01csep) - CalpontSelectExecutionPlan::FilterTokenList sub01csep_filterlist; + // subselect Filters (sub01csep) + CalpontSelectExecutionPlan::FilterTokenList sub01csep_filterlist; - sub01csep_filterlist.push_back(new Operator("(")); + sub01csep_filterlist.push_back(new Operator("(")); - SimpleFilter* s01f1 = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:1")); - sub01csep_filterlist.push_back(s01f1); + SimpleFilter* s01f1 = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:1")); + sub01csep_filterlist.push_back(s01f1); - sub01csep_filterlist.push_back(new Operator("or")); + sub01csep_filterlist.push_back(new Operator("or")); - SimpleFilter* s01f1a = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:2")); - sub01csep_filterlist.push_back(s01f1a); + SimpleFilter* s01f1a = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:2")); + sub01csep_filterlist.push_back(s01f1a); - sub01csep_filterlist.push_back(new Operator("or")); + sub01csep_filterlist.push_back(new Operator("or")); - SimpleFilter* s01f1b = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:3")); - sub01csep_filterlist.push_back(s01f1b); + SimpleFilter* s01f1b = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:3")); + sub01csep_filterlist.push_back(s01f1b); - sub01csep_filterlist.push_back(new Operator("or")); + sub01csep_filterlist.push_back(new Operator("or")); - SimpleFilter* s01f1c = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:4")); - sub01csep_filterlist.push_back(s01f1c); + SimpleFilter* s01f1c = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:4")); + sub01csep_filterlist.push_back(s01f1c); - sub01csep_filterlist.push_back(new Operator("or")); + sub01csep_filterlist.push_back(new Operator("or")); - SimpleFilter* s01f1d = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:5")); - sub01csep_filterlist.push_back(s01f1d); + SimpleFilter* s01f1d = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:5")); + sub01csep_filterlist.push_back(s01f1d); - sub01csep_filterlist.push_back(new Operator("or")); + sub01csep_filterlist.push_back(new Operator("or")); - SimpleFilter* s01f1e = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:6")); - sub01csep_filterlist.push_back(s01f1e); + SimpleFilter* s01f1e = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:6")); + sub01csep_filterlist.push_back(s01f1e); - sub01csep_filterlist.push_back(new Operator("or")); + sub01csep_filterlist.push_back(new Operator("or")); - SimpleFilter* s01f1f = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:7")); - sub01csep_filterlist.push_back(s01f1f); + SimpleFilter* s01f1f = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:7")); + sub01csep_filterlist.push_back(s01f1f); - sub01csep_filterlist.push_back(new Operator(")")); + sub01csep_filterlist.push_back(new Operator(")")); + sub01csep_filterlist.push_back(new Operator("and")); - sub01csep_filterlist.push_back(new Operator("and")); + // --------------- inner sub select begins here ---------------- x + // Sub select filter + CalpontSelectExecutionPlan* sub011csep = + new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub011ColList; + + ArithmeticColumn* s011c1 = new ArithmeticColumn("avg(c_acctbal)"); + sub011ColList.push_back(s011c1); + // Append returned columns to subselect + sub011csep->returnedCols(sub011ColList); + + // subselect Filters (sub011csep) + CalpontSelectExecutionPlan::FilterTokenList sub011csep_filterlist; + SimpleFilter* s011f1 = new SimpleFilter(new Operator(">"), new SimpleColumn("tpch.customer.c_acctbal"), + new ConstantColumn("0.00")); + sub011csep_filterlist.push_back(s011f1); - // --------------- inner sub select begins here ---------------- x - //Sub select filter - CalpontSelectExecutionPlan* sub011csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + sub011csep_filterlist.push_back(new Operator("and")); - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub011ColList; - - ArithmeticColumn* s011c1 = new ArithmeticColumn ("avg(c_acctbal)"); - sub011ColList.push_back(s011c1); - // Append returned columns to subselect - sub011csep->returnedCols(sub011ColList); + sub011csep_filterlist.push_back(new Operator("(")); - //subselect Filters (sub011csep) - CalpontSelectExecutionPlan::FilterTokenList sub011csep_filterlist; - SimpleFilter* s011f1 = new SimpleFilter ( new Operator(">"), - new SimpleColumn ("tpch.customer.c_acctbal"), - new ConstantColumn("0.00")); - sub011csep_filterlist.push_back(s011f1); + SimpleFilter* s011f2 = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:1")); + sub011csep_filterlist.push_back(s011f2); - sub011csep_filterlist.push_back(new Operator("and")); + sub011csep_filterlist.push_back(new Operator("or")); - sub011csep_filterlist.push_back(new Operator("(")); + SimpleFilter* s011f2a = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:2")); + sub011csep_filterlist.push_back(s011f2a); - SimpleFilter* s011f2 = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:1")); - sub011csep_filterlist.push_back(s011f2); + sub011csep_filterlist.push_back(new Operator("or")); - sub011csep_filterlist.push_back(new Operator("or")); + SimpleFilter* s011f2b = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:3")); + sub011csep_filterlist.push_back(s011f2b); - SimpleFilter* s011f2a = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:2")); - sub011csep_filterlist.push_back(s011f2a); + sub011csep_filterlist.push_back(new Operator("or")); - sub011csep_filterlist.push_back(new Operator("or")); + SimpleFilter* s011f2c = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:4")); + sub011csep_filterlist.push_back(s011f2c); - SimpleFilter* s011f2b = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:3")); - sub011csep_filterlist.push_back(s011f2b); + sub011csep_filterlist.push_back(new Operator("or")); - sub011csep_filterlist.push_back(new Operator("or")); + SimpleFilter* s011f2d = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:5")); + sub011csep_filterlist.push_back(s011f2d); + + sub011csep_filterlist.push_back(new Operator("or")); + + SimpleFilter* s011f2e = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:6")); + sub011csep_filterlist.push_back(s011f2e); - SimpleFilter* s011f2c = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:4")); - sub011csep_filterlist.push_back(s011f2c); + sub011csep_filterlist.push_back(new Operator("or")); - sub011csep_filterlist.push_back(new Operator("or")); + SimpleFilter* s011f2f = + new SimpleFilter(new Operator("="), new ArithmeticColumn("substring(c_phone from 1 for 2 )"), + new ConstantColumn("%:7")); + sub011csep_filterlist.push_back(s011f2f); - SimpleFilter* s011f2d = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:5")); - sub011csep_filterlist.push_back(s011f2d); + sub011csep_filterlist.push_back(new Operator(")")); - sub011csep_filterlist.push_back(new Operator("or")); + sub011csep->filterTokenList(sub011csep_filterlist); + // --------------- inner sub ends here ---------------- x - SimpleFilter* s011f2e = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:6")); - sub011csep_filterlist.push_back(s011f2e); + // Now resolve the first inner sub_query + SelectFilter* s01f2 = + new SelectFilter(new SimpleColumn("tpch.customer.c_acctbal"), new Operator(">"), sub011csep); + sub01csep_filterlist.push_back(s01f2); - sub011csep_filterlist.push_back(new Operator("or")); - - SimpleFilter* s011f2f = new SimpleFilter ( new Operator("="), - new ArithmeticColumn ("substring(c_phone from 1 for 2 )"), - new ConstantColumn("%:7")); - sub011csep_filterlist.push_back(s011f2f); - - sub011csep_filterlist.push_back(new Operator(")")); - - sub011csep->filterTokenList(sub011csep_filterlist); - // --------------- inner sub ends here ---------------- x - - // Now resolve the first inner sub_query - SelectFilter* s01f2 = new SelectFilter (new SimpleColumn ("tpch.customer.c_acctbal"), - new Operator(">"), - sub011csep); - sub01csep_filterlist.push_back(s01f2); - - - // --------------- second inner sub select begins here ---------------- x - //Sub select filter - CalpontSelectExecutionPlan* sub02csep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - - //subselect return column(s) - CalpontSelectExecutionPlan::ReturnedColumnList sub02colList; - - SimpleColumn* c21 = new SimpleColumn("tpch.orders.o_orderkey"); - sub02colList.push_back(c21); - SimpleColumn* c22 = new SimpleColumn("tpch.orders.o_custkey"); - sub02colList.push_back(c22); - SimpleColumn* c23 = new SimpleColumn("tpch.orders.o_orderstatus"); - sub02colList.push_back(c23); - SimpleColumn* c24 = new SimpleColumn("tpch.orders.o_totalprice"); - sub02colList.push_back(c24); - SimpleColumn* c25 = new SimpleColumn("tpch.orders.o_orderdate"); - sub02colList.push_back(c25); - SimpleColumn* c26 = new SimpleColumn("tpch.orders.o_orderpriority"); - sub02colList.push_back(c26); - SimpleColumn* c27 = new SimpleColumn("tpch.orders.o_clerk"); - sub02colList.push_back(c27); - SimpleColumn* c28 = new SimpleColumn("tpch.orders.o_shippriority"); - sub02colList.push_back(c28); - SimpleColumn* c29 = new SimpleColumn("tpch.orders.o_comment"); - sub02colList.push_back(c29); - // Append returned columns to subselect - sub02csep->returnedCols(sub02colList); - - //subselect Filters (sub02csep) - CalpontSelectExecutionPlan::FilterTokenList sub02csep_filterlist; - SimpleFilter* s02f1 = new SimpleFilter ( new Operator("="), - new SimpleColumn ("tpch.orders.o_custkey"), - new SimpleColumn ("tpch.customer.c_custkey")); - sub02csep_filterlist.push_back(s02f1); - - sub02csep->filterTokenList(sub02csep_filterlist); - - // --------------- second inner sub ends here ---------------- x - - ExistsFilter* s02f2 = new ExistsFilter (sub02csep, "FALSE"); - -// sub01csep_filterlist.push_back(s01f1); //Previous added. - sub01csep_filterlist.push_back(new Operator("and")); - sub01csep_filterlist.push_back(s02f2); - - // --------------- first sub ends here ---------------- x - // Now resolve the outer query - - sub01csep->filterTokenList(sub01csep_filterlist); - /* - SelectFilter *f1 = new SelectFilter (new SimpleColumn ("tpch.supplier.s_suppkey"), - new Operator("IN"), - sub01csep); - csep_filterlist.push_back(f1); - csep->filterTokenList(csep_filterlist); - */ - - // -- New: Implemented with subselect versus selectFilter -- - CalpontSelectExecutionPlan::SelectList subselectlist; - subselectlist.push_back(sub01csep); - csep->subSelects(subselectlist); - - - // Build Group By List - - - CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; - //SimpleColumn *g1 = new SimpleColumn("tpch.custsale.cntrycode"); - SimpleColumn* g1 = new SimpleColumn(*c1); - g1->alias("cntrycode"); - g1->tableAlias("custtable"); - csep_groupbyList.push_back(g1); - csep->groupByCols(csep_groupbyList); //Set GroupBy columns - - // Order By List - CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; - SimpleColumn* o1 = new SimpleColumn("calpont.FROMTABLE.cntrycode"); - o1->tableAlias("custsale"); - csep_orderbyList.push_back(o1); - csep->orderByCols(csep_orderbyList); //Set OrderBy columns - - - //filterList->walk(walkfnString); ?? - - - // Print the parse tree - - ParseTree* sub011pt = const_cast(sub011csep->filters()); - sub011pt->drawTree("selectExecutionPlan_sub011_22.dot"); - - ParseTree* sub01pt = const_cast(sub01csep->filters()); - sub01pt->drawTree("selectExecutionPlan_sub01_22.dot"); - - ParseTree* sub02pt = const_cast(sub02csep->filters()); - sub02pt->drawTree("selectExecutionPlan_sub02_22.dot"); - - //ParseTree *pt = const_cast(csep->filters()); - //pt->drawTree("selectExecutionPlan_22.dot"); - - cout << "\nCalpont Execution Plan:" << endl; - cout << *csep << endl; - cout << " --- end of test 22 ---" << endl; - - } + // --------------- second inner sub select begins here ---------------- x + // Sub select filter + CalpontSelectExecutionPlan* sub02csep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // subselect return column(s) + CalpontSelectExecutionPlan::ReturnedColumnList sub02colList; + + SimpleColumn* c21 = new SimpleColumn("tpch.orders.o_orderkey"); + sub02colList.push_back(c21); + SimpleColumn* c22 = new SimpleColumn("tpch.orders.o_custkey"); + sub02colList.push_back(c22); + SimpleColumn* c23 = new SimpleColumn("tpch.orders.o_orderstatus"); + sub02colList.push_back(c23); + SimpleColumn* c24 = new SimpleColumn("tpch.orders.o_totalprice"); + sub02colList.push_back(c24); + SimpleColumn* c25 = new SimpleColumn("tpch.orders.o_orderdate"); + sub02colList.push_back(c25); + SimpleColumn* c26 = new SimpleColumn("tpch.orders.o_orderpriority"); + sub02colList.push_back(c26); + SimpleColumn* c27 = new SimpleColumn("tpch.orders.o_clerk"); + sub02colList.push_back(c27); + SimpleColumn* c28 = new SimpleColumn("tpch.orders.o_shippriority"); + sub02colList.push_back(c28); + SimpleColumn* c29 = new SimpleColumn("tpch.orders.o_comment"); + sub02colList.push_back(c29); + // Append returned columns to subselect + sub02csep->returnedCols(sub02colList); + + // subselect Filters (sub02csep) + CalpontSelectExecutionPlan::FilterTokenList sub02csep_filterlist; + SimpleFilter* s02f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.orders.o_custkey"), + new SimpleColumn("tpch.customer.c_custkey")); + sub02csep_filterlist.push_back(s02f1); + + sub02csep->filterTokenList(sub02csep_filterlist); + + // --------------- second inner sub ends here ---------------- x + + ExistsFilter* s02f2 = new ExistsFilter(sub02csep, "FALSE"); + + // sub01csep_filterlist.push_back(s01f1); //Previous added. + sub01csep_filterlist.push_back(new Operator("and")); + sub01csep_filterlist.push_back(s02f2); + + // --------------- first sub ends here ---------------- x + // Now resolve the outer query + + sub01csep->filterTokenList(sub01csep_filterlist); + /* + SelectFilter *f1 = new SelectFilter (new SimpleColumn ("tpch.supplier.s_suppkey"), + new Operator("IN"), + sub01csep); + csep_filterlist.push_back(f1); + csep->filterTokenList(csep_filterlist); + */ + + // -- New: Implemented with subselect versus selectFilter -- + CalpontSelectExecutionPlan::SelectList subselectlist; + subselectlist.push_back(sub01csep); + csep->subSelects(subselectlist); + + // Build Group By List + + CalpontSelectExecutionPlan::GroupByColumnList csep_groupbyList; + // SimpleColumn *g1 = new SimpleColumn("tpch.custsale.cntrycode"); + SimpleColumn* g1 = new SimpleColumn(*c1); + g1->alias("cntrycode"); + g1->tableAlias("custtable"); + csep_groupbyList.push_back(g1); + csep->groupByCols(csep_groupbyList); // Set GroupBy columns + + // Order By List + CalpontSelectExecutionPlan::OrderByColumnList csep_orderbyList; + SimpleColumn* o1 = new SimpleColumn("calpont.FROMTABLE.cntrycode"); + o1->tableAlias("custsale"); + csep_orderbyList.push_back(o1); + csep->orderByCols(csep_orderbyList); // Set OrderBy columns + + // filterList->walk(walkfnString); ?? + + // Print the parse tree + + ParseTree* sub011pt = const_cast(sub011csep->filters()); + sub011pt->drawTree("selectExecutionPlan_sub011_22.dot"); + + ParseTree* sub01pt = const_cast(sub01csep->filters()); + sub01pt->drawTree("selectExecutionPlan_sub01_22.dot"); + + ParseTree* sub02pt = const_cast(sub02csep->filters()); + sub02pt->drawTree("selectExecutionPlan_sub02_22.dot"); + + // ParseTree *pt = const_cast(csep->filters()); + // pt->drawTree("selectExecutionPlan_22.dot"); + + cout << "\nCalpont Execution Plan:" << endl; + cout << *csep << endl; + cout << " --- end of test 22 ---" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver23.cpp b/dbcon/execplan/tdriver23.cpp index 80c5e27c9..72a8df260 100644 --- a/dbcon/execplan/tdriver23.cpp +++ b/dbcon/execplan/tdriver23.cpp @@ -32,120 +32,113 @@ using namespace execplan; class TDriver23 : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TDriver23); - CPPUNIT_TEST_SUITE( TDriver23 ); + CPPUNIT_TEST(test1); + CPPUNIT_TEST(test2); - CPPUNIT_TEST( test1 ); - CPPUNIT_TEST( test2 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + boost::shared_ptr syscat; + CalpontSystemCatalog::OID o; + CalpontSystemCatalog::ColType ct; -private: + void setUp() + { + syscat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + } -public: + void tearDown() + { + CalpontSystemCatalog::removeCalpontSystemCatalog(); + } - boost::shared_ptr syscat; - CalpontSystemCatalog::OID o; - CalpontSystemCatalog::ColType ct; + void test1() + { + o = 1009; + ct = syscat->colType(o); + /* + cw: 4 dt: int do: 0 lo: 0 to: 0 cp: 8 sc: 0 pr: 10 od: 1009 ct: 0 + */ - void setUp() - { - syscat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - } + CPPUNIT_ASSERT(ct.columnOID == 1009); + CPPUNIT_ASSERT(ct.colWidth == 4); + CPPUNIT_ASSERT(ct.colPosition == 8); + CPPUNIT_ASSERT(ct.compressionType == 0); - void tearDown() - { - CalpontSystemCatalog::removeCalpontSystemCatalog(); - } + o = 2001; + ct = syscat->colType(o); + CPPUNIT_ASSERT(ct.columnOID == 0); - void test1() - { - o = 1009; - ct = syscat->colType(o); - /* - cw: 4 dt: int do: 0 lo: 0 to: 0 cp: 8 sc: 0 pr: 10 od: 1009 ct: 0 - */ + ct = syscat->colTypeDct(o); + /* + cw: 65 dt: varchar do: 2001 lo: 2002 to: 2003 cp: 0 sc: 0 pr: 10 od: 1001 ct: 0 + */ + CPPUNIT_ASSERT(ct.columnOID == 1001); + CPPUNIT_ASSERT(ct.colWidth == 65); + CPPUNIT_ASSERT(ct.ddn.dictOID == 2001); + CPPUNIT_ASSERT(ct.colPosition == 0); + CPPUNIT_ASSERT(ct.compressionType == 0); - CPPUNIT_ASSERT(ct.columnOID == 1009); - CPPUNIT_ASSERT(ct.colWidth == 4); - CPPUNIT_ASSERT(ct.colPosition == 8); - CPPUNIT_ASSERT(ct.compressionType == 0); + o = 3024; + ct = syscat->colType(o); + /* + int32_t colWidth; + ConstraintType constraintType; + ColDataType colDataType; + DictOID ddn; + boost::any defaultValue; + int32_t colPosition; // temporally put here. may need to have ColInfo struct later + int32_t scale; //number after decimal points + int32_t precision; + int32_t compressionType; + OID columnOID; + cw: 25 dt: char do: 3027 lo: -2147483648 to: -2147483648 cp: 1 sc: 0 pr: 10 od: 3024 comtype: 0 + */ + CPPUNIT_ASSERT(ct.columnOID == 3024); + CPPUNIT_ASSERT(ct.colWidth == 25); + CPPUNIT_ASSERT(ct.ddn.dictOID == 3027); + CPPUNIT_ASSERT(ct.colPosition == 1); + CPPUNIT_ASSERT(ct.scale == 0); + CPPUNIT_ASSERT(ct.precision == 10); + CPPUNIT_ASSERT(ct.compressionType == 0); - o = 2001; - ct = syscat->colType(o); - CPPUNIT_ASSERT(ct.columnOID == 0); + o = 3027; + ct = syscat->colType(o); + /* + cw: 0 dt: medint do: 0 lo: 0 to: 0 cp: -1 sc: 0 pr: -1 od: 0 comtype: 0 + */ + CPPUNIT_ASSERT(ct.columnOID == 0); - ct = syscat->colTypeDct(o); - /* - cw: 65 dt: varchar do: 2001 lo: 2002 to: 2003 cp: 0 sc: 0 pr: 10 od: 1001 ct: 0 - */ - CPPUNIT_ASSERT(ct.columnOID == 1001); - CPPUNIT_ASSERT(ct.colWidth == 65); - CPPUNIT_ASSERT(ct.ddn.dictOID == 2001); - CPPUNIT_ASSERT(ct.colPosition == 0); - CPPUNIT_ASSERT(ct.compressionType == 0); - - o = 3024; - ct = syscat->colType(o); - /* - int32_t colWidth; - ConstraintType constraintType; - ColDataType colDataType; - DictOID ddn; - boost::any defaultValue; - int32_t colPosition; // temporally put here. may need to have ColInfo struct later - int32_t scale; //number after decimal points - int32_t precision; - int32_t compressionType; - OID columnOID; - cw: 25 dt: char do: 3027 lo: -2147483648 to: -2147483648 cp: 1 sc: 0 pr: 10 od: 3024 comtype: 0 - */ - CPPUNIT_ASSERT(ct.columnOID == 3024); - CPPUNIT_ASSERT(ct.colWidth == 25); - CPPUNIT_ASSERT(ct.ddn.dictOID == 3027); - CPPUNIT_ASSERT(ct.colPosition == 1); - CPPUNIT_ASSERT(ct.scale == 0); - CPPUNIT_ASSERT(ct.precision == 10); - CPPUNIT_ASSERT(ct.compressionType == 0); - - o = 3027; - ct = syscat->colType(o); - /* - cw: 0 dt: medint do: 0 lo: 0 to: 0 cp: -1 sc: 0 pr: -1 od: 0 comtype: 0 - */ - CPPUNIT_ASSERT(ct.columnOID == 0); - - ct = syscat->colTypeDct(o); - CPPUNIT_ASSERT(ct.columnOID == 3024); - CPPUNIT_ASSERT(ct.colWidth == 25); - CPPUNIT_ASSERT(ct.ddn.dictOID == 3027); - CPPUNIT_ASSERT(ct.colPosition == 1); - CPPUNIT_ASSERT(ct.scale == 0); - CPPUNIT_ASSERT(ct.precision == 10); - CPPUNIT_ASSERT(ct.compressionType == 0); - - } - - void test2() - { - syscat->identity(CalpontSystemCatalog::EC); - test1(); - } + ct = syscat->colTypeDct(o); + CPPUNIT_ASSERT(ct.columnOID == 3024); + CPPUNIT_ASSERT(ct.colWidth == 25); + CPPUNIT_ASSERT(ct.ddn.dictOID == 3027); + CPPUNIT_ASSERT(ct.colPosition == 1); + CPPUNIT_ASSERT(ct.scale == 0); + CPPUNIT_ASSERT(ct.precision == 10); + CPPUNIT_ASSERT(ct.compressionType == 0); + } + void test2() + { + syscat->identity(CalpontSystemCatalog::EC); + test1(); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TDriver23 ); +CPPUNIT_TEST_SUITE_REGISTRATION(TDriver23); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver3.cpp b/dbcon/execplan/tdriver3.cpp index e6ddb6021..466aa71cb 100644 --- a/dbcon/execplan/tdriver3.cpp +++ b/dbcon/execplan/tdriver3.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ l_orderkey,\ sum(l_extendedprice * (1 - l_discount)) as revenue,\ @@ -91,101 +90,91 @@ public: revenue desc,\ o_orderdate;"; - CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan csep; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - SimpleColumn* c1 = new SimpleColumn("tpch.lineitem.l_orderkey"); - returnedColumnList.push_back(c1); + SimpleColumn* c1 = new SimpleColumn("tpch.lineitem.l_orderkey"); + returnedColumnList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn - ("sum(tpch.lineitem.l_extendedprice * (1 - tpch.lineitem.l_discount))"); - c2->alias("revenue"); - returnedColumnList.push_back(c2); + ArithmeticColumn* c2 = + new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * (1 - tpch.lineitem.l_discount))"); + c2->alias("revenue"); + returnedColumnList.push_back(c2); - SimpleColumn* c3 = new SimpleColumn("tpch.orders.o_orderdate"); - returnedColumnList.push_back(c3); + SimpleColumn* c3 = new SimpleColumn("tpch.orders.o_orderdate"); + returnedColumnList.push_back(c3); - SimpleColumn* c4 = new SimpleColumn("tpch.orders.o_shippriority"); - returnedColumnList.push_back(c4); + SimpleColumn* c4 = new SimpleColumn("tpch.orders.o_shippriority"); + returnedColumnList.push_back(c4); - csep.returnedCols(returnedColumnList); + csep.returnedCols(returnedColumnList); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_mktsegment"), - new ConstantColumn(":1")); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_custkey"), - new SimpleColumn("tpch.orders.o_custkey")); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.lineitem.l_orderkey"), - new SimpleColumn("tpch.orders.o_orderkey")); - filterTokenList.push_back(f3); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f4 = new SimpleFilter (new Operator("<"), - new SimpleColumn("tpch.orders.o_orderdate"), - new ArithmeticColumn("date(':2')")); - filterTokenList.push_back(f4); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f5 = new SimpleFilter (new Operator(">"), - new SimpleColumn("tpch.lineitem.l_shipdate"), - new ArithmeticColumn("date(':2')")); - filterTokenList.push_back(f5); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_mktsegment"), + new ConstantColumn(":1")); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_custkey"), + new SimpleColumn("tpch.orders.o_custkey")); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_orderkey"), + new SimpleColumn("tpch.orders.o_orderkey")); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f4 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.orders.o_orderdate"), + new ArithmeticColumn("date(':2')")); + filterTokenList.push_back(f4); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f5 = new SimpleFilter(new Operator(">"), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ArithmeticColumn("date(':2')")); + filterTokenList.push_back(f5); - csep.filterTokenList(filterTokenList); + csep.filterTokenList(filterTokenList); - ParseTree* pt = const_cast(csep.filters()); - pt->drawTree("q3.dot"); + ParseTree* pt = const_cast(csep.filters()); + pt->drawTree("q3.dot"); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn(*c1); - groupByList.push_back(g1); - SimpleColumn* g2 = new SimpleColumn(*c3); - groupByList.push_back(g2); - SimpleColumn* g3 = new SimpleColumn(*c4); - groupByList.push_back(g3); - csep.groupByCols(groupByList); + SimpleColumn* g1 = new SimpleColumn(*c1); + groupByList.push_back(g1); + SimpleColumn* g2 = new SimpleColumn(*c3); + groupByList.push_back(g2); + SimpleColumn* g3 = new SimpleColumn(*c4); + groupByList.push_back(g3); + csep.groupByCols(groupByList); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + // ArithmeticColumn *test = new ArithmeticColumn("a+b"); + ArithmeticColumn* o1 = new ArithmeticColumn(*c2); + o1->asc(false); + orderByList.push_back(o1); - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - //ArithmeticColumn *test = new ArithmeticColumn("a+b"); - ArithmeticColumn* o1 = new ArithmeticColumn(*c2); - o1->asc(false); - orderByList.push_back(o1); - - SimpleColumn* o2 = new SimpleColumn(*c3); - orderByList.push_back(o2); - - csep.orderByCols(orderByList); - - cout << csep; - } + SimpleColumn* o2 = new SimpleColumn(*c3); + orderByList.push_back(o2); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver4.cpp b/dbcon/execplan/tdriver4.cpp index 778802a7f..7873b82a5 100644 --- a/dbcon/execplan/tdriver4.cpp +++ b/dbcon/execplan/tdriver4.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ o_orderpriority,\ count(*) as order_count\ @@ -90,122 +89,110 @@ public: order by\ o_orderpriority;"; + CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan csep; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + SimpleColumn* c1 = new SimpleColumn("tpch.orders.o_orderpriority"); + returnedColumnList.push_back(c1); - SimpleColumn* c1 = new SimpleColumn("tpch.orders.o_orderpriority"); - returnedColumnList.push_back(c1); + ArithmeticColumn* c2 = new ArithmeticColumn("count(ALL)"); + c2->alias("order_count"); + returnedColumnList.push_back(c2); - ArithmeticColumn* c2 = new ArithmeticColumn ("count(ALL)"); - c2->alias("order_count"); - returnedColumnList.push_back(c2); + csep.returnedCols(returnedColumnList); - csep.returnedCols(returnedColumnList); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.orders.o_orderdate"), + new ConstantColumn("1998-05-01")); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.orders.o_orderdate"), + new ConstantColumn("1999-05-01")); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator(">="), - new SimpleColumn("tpch.orders.o_orderdate"), - new ConstantColumn("1998-05-01")); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter (new Operator("<"), - new SimpleColumn("tpch.orders.o_orderdate"), - new ConstantColumn("1999-05-01")); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); + // subselect for exists + CalpontSelectExecutionPlan* subcsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); + // select * (lineitem) + CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; + SimpleColumn* sc1 = new SimpleColumn("tpch.lineitem.l_orderkey"); + SimpleColumn* sc2 = new SimpleColumn("tpch.lineitem.l_partkey"); + SimpleColumn* sc3 = new SimpleColumn("tpch.lineitem.l_suppkey"); + SimpleColumn* sc4 = new SimpleColumn("tpch.lineitem.l_linenumber"); + SimpleColumn* sc5 = new SimpleColumn("tpch.lineitem.l_extendedprice"); + SimpleColumn* sc6 = new SimpleColumn("tpch.lineitem.l_tax"); + SimpleColumn* sc7 = new SimpleColumn("tpch.lineitem.l_returnflag"); + SimpleColumn* sc8 = new SimpleColumn("tpch.lineitem.l_linestatus"); + SimpleColumn* sc9 = new SimpleColumn("tpch.lineitem.l_commitdate"); + SimpleColumn* sc10 = new SimpleColumn("tpch.lineitem.l_receiptdate"); + SimpleColumn* sc11 = new SimpleColumn("tpch.lineitem.l_shipinstruct"); + SimpleColumn* sc12 = new SimpleColumn("tpch.lineitem.l_shipmode"); + SimpleColumn* sc13 = new SimpleColumn("tpch.lineitem.l_comment"); - // subselect for exists - CalpontSelectExecutionPlan* subcsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::WHERE); - // select * (lineitem) - CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; - SimpleColumn* sc1 = new SimpleColumn ("tpch.lineitem.l_orderkey"); - SimpleColumn* sc2 = new SimpleColumn ("tpch.lineitem.l_partkey"); - SimpleColumn* sc3 = new SimpleColumn ("tpch.lineitem.l_suppkey"); - SimpleColumn* sc4 = new SimpleColumn ("tpch.lineitem.l_linenumber"); - SimpleColumn* sc5 = new SimpleColumn ("tpch.lineitem.l_extendedprice"); - SimpleColumn* sc6 = new SimpleColumn ("tpch.lineitem.l_tax"); - SimpleColumn* sc7 = new SimpleColumn ("tpch.lineitem.l_returnflag"); - SimpleColumn* sc8 = new SimpleColumn ("tpch.lineitem.l_linestatus"); - SimpleColumn* sc9 = new SimpleColumn ("tpch.lineitem.l_commitdate"); - SimpleColumn* sc10 = new SimpleColumn ("tpch.lineitem.l_receiptdate"); - SimpleColumn* sc11 = new SimpleColumn ("tpch.lineitem.l_shipinstruct"); - SimpleColumn* sc12 = new SimpleColumn ("tpch.lineitem.l_shipmode"); - SimpleColumn* sc13 = new SimpleColumn ("tpch.lineitem.l_comment"); + subReturnedColList.push_back(sc1); + subReturnedColList.push_back(sc2); + subReturnedColList.push_back(sc3); + subReturnedColList.push_back(sc4); + subReturnedColList.push_back(sc5); + subReturnedColList.push_back(sc6); + subReturnedColList.push_back(sc7); + subReturnedColList.push_back(sc8); + subReturnedColList.push_back(sc9); + subReturnedColList.push_back(sc10); + subReturnedColList.push_back(sc11); + subReturnedColList.push_back(sc12); + subReturnedColList.push_back(sc13); - subReturnedColList.push_back(sc1); - subReturnedColList.push_back(sc2); - subReturnedColList.push_back(sc3); - subReturnedColList.push_back(sc4); - subReturnedColList.push_back(sc5); - subReturnedColList.push_back(sc6); - subReturnedColList.push_back(sc7); - subReturnedColList.push_back(sc8); - subReturnedColList.push_back(sc9); - subReturnedColList.push_back(sc10); - subReturnedColList.push_back(sc11); - subReturnedColList.push_back(sc12); - subReturnedColList.push_back(sc13); + subcsep->returnedCols(subReturnedColList); - subcsep->returnedCols(subReturnedColList); + // filters of subselect + CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + SimpleFilter* ssf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_orderkey"), + new SimpleColumn("tpch.orders.o_orderkey")); + subFilterTokenList.push_back(ssf1); + subFilterTokenList.push_back(new Operator("and")); - // filters of subselect - CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SimpleFilter* ssf1 = new SimpleFilter ( - new Operator("="), - new SimpleColumn("tpch.lineitem.l_orderkey"), - new SimpleColumn("tpch.orders.o_orderkey")); - subFilterTokenList.push_back (ssf1); - subFilterTokenList.push_back (new Operator("and")); + SimpleFilter* ssf2 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.lineitem.l_commitdate"), + new SimpleColumn("tpch.lineitem.l_receiptdate")); + subFilterTokenList.push_back(ssf2); + subcsep->filterTokenList(subFilterTokenList); - SimpleFilter* ssf2 = new SimpleFilter ( - new Operator("<"), - new SimpleColumn("tpch.lineitem.l_commitdate"), - new SimpleColumn("tpch.lineitem.l_receiptdate")); - subFilterTokenList.push_back (ssf2); - subcsep->filterTokenList(subFilterTokenList); + // back to parent select exist filter + ExistsFilter* exists = new ExistsFilter(subcsep); + filterTokenList.push_back(exists); + csep.filterTokenList(filterTokenList); - // back to parent select exist filter - ExistsFilter* exists = new ExistsFilter (subcsep); - filterTokenList.push_back (exists); - csep.filterTokenList(filterTokenList); + ParseTree* pt = const_cast(csep.filters()); + pt->drawTree("q4.dot"); - ParseTree* pt = const_cast(csep.filters()); - pt->drawTree("q4.dot"); - - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn(*c1); - groupByList.push_back(g1); - - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - SimpleColumn* o1 = new SimpleColumn(*c1); - orderByList.push_back(o1); - csep.orderByCols(orderByList); - - cout << csep; - } + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + SimpleColumn* g1 = new SimpleColumn(*c1); + groupByList.push_back(g1); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + SimpleColumn* o1 = new SimpleColumn(*c1); + orderByList.push_back(o1); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver5.cpp b/dbcon/execplan/tdriver5.cpp index ae62158a5..eac7804dd 100644 --- a/dbcon/execplan/tdriver5.cpp +++ b/dbcon/execplan/tdriver5.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -46,27 +46,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ n_name,\ sum(l_extendedprice * (1 - l_discount)) as revenue\ @@ -92,105 +91,93 @@ public: order by\ revenue desc;"; + CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan csep; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + SimpleColumn* c1 = new SimpleColumn("tpch.nation.n_name"); + returnedColumnList.push_back(c1); - SimpleColumn* c1 = new SimpleColumn("tpch.nation.n_name"); - returnedColumnList.push_back(c1); + ArithmeticColumn* c2 = + new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * (1 - tpch.lineitem.l_discount))"); + c2->alias("revenue"); + returnedColumnList.push_back(c2); - ArithmeticColumn* c2 = new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * (1 - tpch.lineitem.l_discount))"); - c2->alias("revenue"); - returnedColumnList.push_back(c2); + csep.returnedCols(returnedColumnList); - csep.returnedCols(returnedColumnList); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_custkey"), + new SimpleColumn("tpch.orders.o_custkey")); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_custkey"), - new SimpleColumn("tpch.orders.o_custkey")); - filterTokenList.push_back(f1); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_orderkey"), + new SimpleColumn("tpch.orders.o_orderkey")); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.lineitem.l_orderkey"), - new SimpleColumn("tpch.orders.o_orderkey")); - filterTokenList.push_back(f2); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_suppkey"), + new SimpleColumn("tpch.supplier.s_suppkey")); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.lineitem.l_suppkey"), - new SimpleColumn("tpch.supplier.s_suppkey")); - filterTokenList.push_back(f3); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f4 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_nationkey"), + new SimpleColumn("tpch.supplier.s_nationkey")); + filterTokenList.push_back(f4); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f4 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_nationkey"), - new SimpleColumn("tpch.supplier.s_nationkey")); - filterTokenList.push_back(f4); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f5 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nstionkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + filterTokenList.push_back(f5); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_nstionkey"), - new SimpleColumn("tpch.nation.n_nationkey")); - filterTokenList.push_back(f5); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f6 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.region.r_name"), new ConstantColumn(":1")); + filterTokenList.push_back(f6); + filterTokenList.push_back(new Operator("and")); + SimpleFilter* f7 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.orders.o_orderdate"), + new ArithmeticColumn("date(':2')")); + filterTokenList.push_back(f7); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f6 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.region.r_name"), - new ConstantColumn(":1")); - filterTokenList.push_back(f6); - filterTokenList.push_back( new Operator ("and")); - SimpleFilter* f7 = new SimpleFilter (new Operator(">="), - new SimpleColumn("tpch.orders.o_orderdate"), - new ArithmeticColumn("date(':2')")); - filterTokenList.push_back(f7); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f8 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.orders.o_orderdate"), + new ArithmeticColumn("date(':2') + interval ('1', year)")); + filterTokenList.push_back(f8); - SimpleFilter* f8 = new SimpleFilter (new Operator("<"), - new SimpleColumn("tpch.orders.o_orderdate"), - new ArithmeticColumn("date(':2') + interval ('1', year)")); - filterTokenList.push_back(f8); + csep.filterTokenList(filterTokenList); - csep.filterTokenList(filterTokenList); + ParseTree* pt = const_cast(csep.filters()); + pt->drawTree("q5.dot"); - ParseTree* pt = const_cast(csep.filters()); - pt->drawTree ("q5.dot"); + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + groupByList.push_back(c1->clone()); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - groupByList.push_back(c1->clone()); - - csep.groupByCols (groupByList); - - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - ArithmeticColumn* o1 = new ArithmeticColumn(*c2); - o1->asc(false); - orderByList.push_back(o1); - csep.orderByCols(orderByList); - - cout << csep; - } + csep.groupByCols(groupByList); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + ArithmeticColumn* o1 = new ArithmeticColumn(*c2); + o1->asc(false); + orderByList.push_back(o1); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver6.cpp b/dbcon/execplan/tdriver6.cpp index f078c9efb..f4dec6ae7 100644 --- a/dbcon/execplan/tdriver6.cpp +++ b/dbcon/execplan/tdriver6.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -46,27 +46,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ sum(l_extendedprice * l_discount) as revenue\ from\ @@ -77,72 +76,63 @@ public: and l_discount between :2 - 0.01 and :2 + 0.01\ and l_quantity < :3;"; + CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan csep; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + ArithmeticColumn* c1 = + new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * tpch.lineitem.l_discount)"); + c1->alias("revenue"); + returnedColumnList.push_back(c1); - ArithmeticColumn* c1 = new ArithmeticColumn("sum(tpch.lineitem.l_extendedprice * tpch.lineitem.l_discount)"); - c1->alias("revenue"); - returnedColumnList.push_back(c1); + csep.returnedCols(returnedColumnList); - csep.returnedCols(returnedColumnList); + // Filters + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + SimpleFilter* f1 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ArithmeticColumn("date (':1')")); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); - // Filters - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - SimpleFilter* f1 = new SimpleFilter (new Operator(">="), - new SimpleColumn("tpch.lineitem.l_shipdate"), - new ArithmeticColumn("date (':1')")); - filterTokenList.push_back(f1); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f2 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ArithmeticColumn("date (':1') + interval('1', year)")); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f2 = new SimpleFilter (new Operator("<"), - new SimpleColumn("tpch.lineitem.l_shipdate"), - new ArithmeticColumn("date (':1') + interval('1', year)")); - filterTokenList.push_back(f2); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f3 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_discount"), + new ArithmeticColumn("':2' - 0.01")); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f3 = new SimpleFilter (new Operator(">="), - new SimpleColumn("tpch.lineitem.l_discount"), - new ArithmeticColumn("':2' - 0.01")); - filterTokenList.push_back(f3); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f4 = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.lineitem.l_discount"), + new ArithmeticColumn("':2' + 0.01")); + filterTokenList.push_back(f4); + filterTokenList.push_back(new Operator("and")); - SimpleFilter* f4 = new SimpleFilter (new Operator("<="), - new SimpleColumn("tpch.lineitem.l_discount"), - new ArithmeticColumn("':2' + 0.01")); - filterTokenList.push_back(f4); - filterTokenList.push_back( new Operator ("and")); + SimpleFilter* f5 = new SimpleFilter(new Operator("<"), new SimpleColumn("tpch.lineitem.l_quantity"), + new ConstantColumn(":3")); + filterTokenList.push_back(f5); - SimpleFilter* f5 = new SimpleFilter (new Operator("<"), - new SimpleColumn("tpch.lineitem.l_quantity"), - new ConstantColumn(":3")); - filterTokenList.push_back(f5); - - csep.filterTokenList(filterTokenList); - - ParseTree* pt = const_cast(csep.filters()); - pt->drawTree ("q6.dot"); - - cout << csep; - } + csep.filterTokenList(filterTokenList); + ParseTree* pt = const_cast(csep.filters()); + pt->drawTree("q6.dot"); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver7.cpp b/dbcon/execplan/tdriver7.cpp index c7ae21118..65bccf29b 100644 --- a/dbcon/execplan/tdriver7.cpp +++ b/dbcon/execplan/tdriver7.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ supp_nation,\ cust_nation,\ @@ -108,200 +107,182 @@ public: cust_nation,\ l_year;"; - CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan csep; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // these columns are from the temp table of FROM clause. - // I hereby give schema name "calpont", table name "FROMTABLE", - // and alias name "shipping" - SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.supp_nation"); - c1->tableAlias ("shipping"); - returnedColumnList.push_back(c1); + // these columns are from the temp table of FROM clause. + // I hereby give schema name "calpont", table name "FROMTABLE", + // and alias name "shipping" + SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.supp_nation"); + c1->tableAlias("shipping"); + returnedColumnList.push_back(c1); - SimpleColumn* c2 = new SimpleColumn("calpont.FROMTABLE.cust_nation"); - c2->tableAlias ("shipping"); - returnedColumnList.push_back(c2); + SimpleColumn* c2 = new SimpleColumn("calpont.FROMTABLE.cust_nation"); + c2->tableAlias("shipping"); + returnedColumnList.push_back(c2); - SimpleColumn* c3 = new SimpleColumn("tpch.lineitem.l_year"); - returnedColumnList.push_back(c3); + SimpleColumn* c3 = new SimpleColumn("tpch.lineitem.l_year"); + returnedColumnList.push_back(c3); - ArithmeticColumn* c4 = new ArithmeticColumn("sum(volumn)"); - c4->alias("revenue"); - returnedColumnList.push_back(c4); + ArithmeticColumn* c4 = new ArithmeticColumn("sum(volumn)"); + c4->alias("revenue"); + returnedColumnList.push_back(c4); - csep.returnedCols(returnedColumnList); + csep.returnedCols(returnedColumnList); - // from subselect - CalpontSelectExecutionPlan* subsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); + // from subselect + CalpontSelectExecutionPlan* subsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); - // subselect returned columns - CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; - SimpleColumn* sc1 = new SimpleColumn("tpch.nation.n_name"); - sc1->alias ("supp_nation"); - sc1->tableAlias ("n1"); - subReturnedColList.push_back(sc1); + // subselect returned columns + CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; + SimpleColumn* sc1 = new SimpleColumn("tpch.nation.n_name"); + sc1->alias("supp_nation"); + sc1->tableAlias("n1"); + subReturnedColList.push_back(sc1); - SimpleColumn* sc2 = new SimpleColumn("tpch.nation.n_name"); - sc1->alias ("cust_nation"); - sc1->tableAlias ("n2"); - subReturnedColList.push_back(sc2); + SimpleColumn* sc2 = new SimpleColumn("tpch.nation.n_name"); + sc1->alias("cust_nation"); + sc1->tableAlias("n2"); + subReturnedColList.push_back(sc2); - ArithmeticColumn* sc3 = new ArithmeticColumn("extract(year from tpch.lineitem.l_shipdate)"); - sc3->alias("l_year"); - subReturnedColList.push_back(sc3); + ArithmeticColumn* sc3 = new ArithmeticColumn("extract(year from tpch.lineitem.l_shipdate)"); + sc3->alias("l_year"); + subReturnedColList.push_back(sc3); - ArithmeticColumn* sc4 = new ArithmeticColumn("tpch.lineitem.l_extendeprice * (1-tpch.lineitem.l_discount)"); - sc3->alias("volume"); - subReturnedColList.push_back(sc4); + ArithmeticColumn* sc4 = + new ArithmeticColumn("tpch.lineitem.l_extendeprice * (1-tpch.lineitem.l_discount)"); + sc3->alias("volume"); + subReturnedColList.push_back(sc4); - subsep->returnedCols(subReturnedColList); + subsep->returnedCols(subReturnedColList); - // subselect filters - CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + // subselect filters + CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SimpleFilter* sf1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_suppkey"), - new SimpleColumn("tpch.lineitem.l_suppkey")); - subFilterTokenList.push_back(sf1); + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + subFilterTokenList.push_back(sf1); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.orders.o_orderkey"), - new SimpleColumn("tpch.lineitem.l_orderkey")); - subFilterTokenList.push_back(sf2); - subFilterTokenList.push_back(new Operator("and")); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.orders.o_orderkey"), + new SimpleColumn("tpch.lineitem.l_orderkey")); + subFilterTokenList.push_back(sf2); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_custkey"), - new SimpleColumn("tpch.orders.o_custkey")); - subFilterTokenList.push_back(sf3); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_custkey"), + new SimpleColumn("tpch.orders.o_custkey")); + subFilterTokenList.push_back(sf3); + subFilterTokenList.push_back(new Operator("and")); - SimpleColumn* n1 = new SimpleColumn ("tpch.nation.n_nationkey"); - n1->tableAlias("n1"); - SimpleFilter* sf4 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - n1); - subFilterTokenList.push_back(sf4); - subFilterTokenList.push_back(new Operator("and")); + SimpleColumn* n1 = new SimpleColumn("tpch.nation.n_nationkey"); + n1->tableAlias("n1"); + SimpleFilter* sf4 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), n1); + subFilterTokenList.push_back(sf4); + subFilterTokenList.push_back(new Operator("and")); - SimpleColumn* n2 = new SimpleColumn ("tpch.nation.n_nationkey"); - n2->tableAlias("n2"); - SimpleFilter* sf5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_nationkey"), - n2); - subFilterTokenList.push_back(sf5); - subFilterTokenList.push_back(new Operator("and")); + SimpleColumn* n2 = new SimpleColumn("tpch.nation.n_nationkey"); + n2->tableAlias("n2"); + SimpleFilter* sf5 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_nationkey"), n2); + subFilterTokenList.push_back(sf5); + subFilterTokenList.push_back(new Operator("and")); - // ((n1.n_name = ':1' and n2.n_name = ':2') - // or (n1.n_name = ':2' and n2.n_name = ':1')) - subFilterTokenList.push_back(new Operator("(")); - subFilterTokenList.push_back(new Operator("(")); + // ((n1.n_name = ':1' and n2.n_name = ':2') + // or (n1.n_name = ':2' and n2.n_name = ':1')) + subFilterTokenList.push_back(new Operator("(")); + subFilterTokenList.push_back(new Operator("(")); - SimpleColumn* n1_name = new SimpleColumn("tpch.nation.n_name"); - n1_name->tableAlias ("n1"); - SimpleFilter* sf6 = new SimpleFilter ( new Operator("="), - n1_name, - new ConstantColumn (":1")); - subFilterTokenList.push_back(sf6); - subFilterTokenList.push_back(new Operator("and")); + SimpleColumn* n1_name = new SimpleColumn("tpch.nation.n_name"); + n1_name->tableAlias("n1"); + SimpleFilter* sf6 = new SimpleFilter(new Operator("="), n1_name, new ConstantColumn(":1")); + subFilterTokenList.push_back(sf6); + subFilterTokenList.push_back(new Operator("and")); - SimpleColumn* n2_name = new SimpleColumn("tpch.nation.n_name"); - n1_name->tableAlias ("n2"); - SimpleFilter* sf7 = new SimpleFilter ( new Operator("="), - n2_name, - new ConstantColumn (":2")); - subFilterTokenList.push_back(sf7); - subFilterTokenList.push_back(new Operator (")")); - subFilterTokenList.push_back(new Operator("or")); - subFilterTokenList.push_back(new Operator("(")); + SimpleColumn* n2_name = new SimpleColumn("tpch.nation.n_name"); + n1_name->tableAlias("n2"); + SimpleFilter* sf7 = new SimpleFilter(new Operator("="), n2_name, new ConstantColumn(":2")); + subFilterTokenList.push_back(sf7); + subFilterTokenList.push_back(new Operator(")")); + subFilterTokenList.push_back(new Operator("or")); + subFilterTokenList.push_back(new Operator("(")); - SimpleFilter* sf8 = new SimpleFilter ( new Operator("="), - new SimpleColumn(*n1_name), - new ConstantColumn (":2")); - subFilterTokenList.push_back(sf8); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf9 = new SimpleFilter ( new Operator("="), - new SimpleColumn(*n2_name), - new ConstantColumn (":1")); - subFilterTokenList.push_back(sf9); - subFilterTokenList.push_back(new Operator (")")); - subFilterTokenList.push_back(new Operator (")")); + SimpleFilter* sf8 = + new SimpleFilter(new Operator("="), new SimpleColumn(*n1_name), new ConstantColumn(":2")); + subFilterTokenList.push_back(sf8); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf9 = + new SimpleFilter(new Operator("="), new SimpleColumn(*n2_name), new ConstantColumn(":1")); + subFilterTokenList.push_back(sf9); + subFilterTokenList.push_back(new Operator(")")); + subFilterTokenList.push_back(new Operator(")")); + subFilterTokenList.push_back(new Operator("and")); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf10 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ConstantColumn("1995-01-01")); + subFilterTokenList.push_back(sf10); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf11 = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.lineitem.l_shipdate"), + new ConstantColumn("1995-01-06")); + subFilterTokenList.push_back(sf11); - SimpleFilter* sf10 = new SimpleFilter (new Operator (">="), - new SimpleColumn ("tpch.lineitem.l_shipdate"), - new ConstantColumn ("1995-01-01")); - subFilterTokenList.push_back(sf10); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf11 = new SimpleFilter (new Operator ("<="), - new SimpleColumn ("tpch.lineitem.l_shipdate"), - new ConstantColumn ("1995-01-06")); - subFilterTokenList.push_back(sf11); + subsep->filterTokenList(subFilterTokenList); - subsep->filterTokenList(subFilterTokenList); + // end of subselect in FROM. push FROM subselect to selectList + // NOTE: only FROM subselect needs to be pushed into selectList. + // Subselects in WHERE or HAVING clause are in where or having + // filter parse tree. It may make more sense to change the member + // fSelectList of CSEP class to fFromSubSelect (type CSEP*) + CalpontSelectExecutionPlan::SelectList fromSubSelectList; + fromSubSelectList.push_back(subsep); + csep.subSelects(fromSubSelectList); - // end of subselect in FROM. push FROM subselect to selectList - // NOTE: only FROM subselect needs to be pushed into selectList. - // Subselects in WHERE or HAVING clause are in where or having - // filter parse tree. It may make more sense to change the member - // fSelectList of CSEP class to fFromSubSelect (type CSEP*) - CalpontSelectExecutionPlan::SelectList fromSubSelectList; - fromSubSelectList.push_back(subsep); - csep.subSelects(fromSubSelectList); + ParseTree* pt = const_cast(subsep->filters()); + pt->drawTree("q7.dot"); - ParseTree* pt = const_cast(subsep->filters()); - pt->drawTree("q7.dot"); + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + SimpleColumn* g1 = new SimpleColumn(*c1); + groupByList.push_back(g1); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn (*c1); - groupByList.push_back (g1); + SimpleColumn* g2 = new SimpleColumn(*c2); + groupByList.push_back(g2); - SimpleColumn* g2 = new SimpleColumn (*c2); - groupByList.push_back (g2); + SimpleColumn* g3 = new SimpleColumn(*c3); + groupByList.push_back(g3); - SimpleColumn* g3 = new SimpleColumn (*c3); - groupByList.push_back (g3); + csep.groupByCols(groupByList); - csep.groupByCols(groupByList); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + SimpleColumn* o1 = new SimpleColumn(*c1); + orderByList.push_back(o1); - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - SimpleColumn* o1 = new SimpleColumn(*c1); - orderByList.push_back(o1); + SimpleColumn* o2 = new SimpleColumn(*c2); + orderByList.push_back(o2); - SimpleColumn* o2 = new SimpleColumn(*c2); - orderByList.push_back(o2); - - SimpleColumn* o3 = new SimpleColumn(*c3); - orderByList.push_back(o3); - - csep.orderByCols(orderByList); - - cout << csep; - } + SimpleColumn* o3 = new SimpleColumn(*c3); + orderByList.push_back(o3); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver8.cpp b/dbcon/execplan/tdriver8.cpp index 083d0a69f..2f96a4948 100644 --- a/dbcon/execplan/tdriver8.cpp +++ b/dbcon/execplan/tdriver8.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ o_year,\ sum(decode(nation, ':1', volumn, 0)) / sum(volume) as mkt_share\ @@ -103,161 +102,148 @@ group by\ order by\ o_year;"; - CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan csep; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // these columns are from the temp table of FROM clause. - // I hereby give schema name "calpont", table name "FROMTABLE", - SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.o_year"); - returnedColumnList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn("sum(decode(calpont.FROMTABLE.nation, ':1', calpont.FROMTABLE.volumn, 0))/sum(tpch.FROMTABLE.volumn)"); - returnedColumnList.push_back(c2); + // these columns are from the temp table of FROM clause. + // I hereby give schema name "calpont", table name "FROMTABLE", + SimpleColumn* c1 = new SimpleColumn("calpont.FROMTABLE.o_year"); + returnedColumnList.push_back(c1); + ArithmeticColumn* c2 = new ArithmeticColumn( + "sum(decode(calpont.FROMTABLE.nation, ':1', calpont.FROMTABLE.volumn, " + "0))/sum(tpch.FROMTABLE.volumn)"); + returnedColumnList.push_back(c2); - csep.returnedCols(returnedColumnList); + csep.returnedCols(returnedColumnList); - // subselect in FROM clause - CalpontSelectExecutionPlan* subsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); + // subselect in FROM clause + CalpontSelectExecutionPlan* subsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); - // subselect returned columns - CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; - ArithmeticColumn* sc1 = new ArithmeticColumn("extract(year from tpch.orders.o_orderdate)"); - sc1->alias ("o_year"); - subReturnedColList.push_back(sc1); + // subselect returned columns + CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; + ArithmeticColumn* sc1 = new ArithmeticColumn("extract(year from tpch.orders.o_orderdate)"); + sc1->alias("o_year"); + subReturnedColList.push_back(sc1); - ArithmeticColumn* sc2 = new ArithmeticColumn("tpch.lineitem.l_extendeprice * (1-tpch.lineitem.l_discount)"); - sc2->alias("volume"); - subReturnedColList.push_back(sc2); + ArithmeticColumn* sc2 = + new ArithmeticColumn("tpch.lineitem.l_extendeprice * (1-tpch.lineitem.l_discount)"); + sc2->alias("volume"); + subReturnedColList.push_back(sc2); - SimpleColumn* sc3 = new SimpleColumn("tpch.nation.n_name"); - sc3->tableAlias("n2"); - subReturnedColList.push_back(sc3); + SimpleColumn* sc3 = new SimpleColumn("tpch.nation.n_name"); + sc3->tableAlias("n2"); + subReturnedColList.push_back(sc3); - subsep->returnedCols(subReturnedColList); + subsep->returnedCols(subReturnedColList); - // subselect filters - CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + // subselect filters + CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SimpleFilter* sf1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.part.p_partkey"), - new SimpleColumn("tpch.lineitem.l_partkey")); - subFilterTokenList.push_back(sf1); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + subFilterTokenList.push_back(sf1); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_suppkey"), - new SimpleColumn("tpch.lineitem.l_suppkey")); - subFilterTokenList.push_back(sf2); + SimpleFilter* sf2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + subFilterTokenList.push_back(sf2); - subFilterTokenList.push_back(new Operator("and")); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.lineitem.l_ordertkey"), - new SimpleColumn("tpch.orders.o_orderkey")); - subFilterTokenList.push_back(sf3); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.lineitem.l_ordertkey"), + new SimpleColumn("tpch.orders.o_orderkey")); + subFilterTokenList.push_back(sf3); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf4 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.orders.o_custkey"), - new SimpleColumn("tpch.customer.c_custkey")); - subFilterTokenList.push_back(sf4); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf4 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.orders.o_custkey"), + new SimpleColumn("tpch.customer.c_custkey")); + subFilterTokenList.push_back(sf4); + subFilterTokenList.push_back(new Operator("and")); - SimpleColumn* n1_nationkey = new SimpleColumn("tpch.nation.n_nationkey"); - n1_nationkey->tableAlias("n1"); - SimpleFilter* sf5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.customer.c_nationkey"), - n1_nationkey); - subFilterTokenList.push_back(sf5); - subFilterTokenList.push_back(new Operator("and")); + SimpleColumn* n1_nationkey = new SimpleColumn("tpch.nation.n_nationkey"); + n1_nationkey->tableAlias("n1"); + SimpleFilter* sf5 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.customer.c_nationkey"), n1_nationkey); + subFilterTokenList.push_back(sf5); + subFilterTokenList.push_back(new Operator("and")); - SimpleColumn* n1_regionkey = new SimpleColumn("tpch.nation.n_regionkey"); - n1_regionkey->tableAlias ("n1"); - SimpleFilter* sf6 = new SimpleFilter ( new Operator("="), - n1_regionkey, - new SimpleColumn("tpch.region.r_regionkey")); - subFilterTokenList.push_back(sf6); - subFilterTokenList.push_back(new Operator("and")); + SimpleColumn* n1_regionkey = new SimpleColumn("tpch.nation.n_regionkey"); + n1_regionkey->tableAlias("n1"); + SimpleFilter* sf6 = + new SimpleFilter(new Operator("="), n1_regionkey, new SimpleColumn("tpch.region.r_regionkey")); + subFilterTokenList.push_back(sf6); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf7 = new SimpleFilter ( new Operator("="), - new SimpleColumn("tpch.region.r_name"), - new ConstantColumn (":2")); - subFilterTokenList.push_back(sf7); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf7 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.region.r_name"), new ConstantColumn(":2")); + subFilterTokenList.push_back(sf7); + subFilterTokenList.push_back(new Operator("and")); - SimpleColumn* n2_nationkey = new SimpleColumn("tpch.nation.n_nationkey"); - n2_nationkey->tableAlias("n2"); - SimpleFilter* sf9 = new SimpleFilter ( new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - n2_nationkey); - subFilterTokenList.push_back(sf9); + SimpleColumn* n2_nationkey = new SimpleColumn("tpch.nation.n_nationkey"); + n2_nationkey->tableAlias("n2"); + SimpleFilter* sf9 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), n2_nationkey); + subFilterTokenList.push_back(sf9); - subFilterTokenList.push_back(new Operator("and")); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf10 = new SimpleFilter (new Operator (">="), - new SimpleColumn ("tpch.orders.o_orderdate"), - new ConstantColumn ("1995-01-01")); - subFilterTokenList.push_back(sf10); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf11 = new SimpleFilter (new Operator ("<="), - new SimpleColumn ("tpch.orders.o_orderdate"), - new ConstantColumn ("1995-01-06")); - subFilterTokenList.push_back(sf11); - subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf12 = new SimpleFilter (new Operator ("="), - new SimpleColumn ("tpch.part.p_type"), - new ConstantColumn ("3")); - subFilterTokenList.push_back(sf12); + SimpleFilter* sf10 = new SimpleFilter(new Operator(">="), new SimpleColumn("tpch.orders.o_orderdate"), + new ConstantColumn("1995-01-01")); + subFilterTokenList.push_back(sf10); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf11 = new SimpleFilter(new Operator("<="), new SimpleColumn("tpch.orders.o_orderdate"), + new ConstantColumn("1995-01-06")); + subFilterTokenList.push_back(sf11); + subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf12 = + new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_type"), new ConstantColumn("3")); + subFilterTokenList.push_back(sf12); - subsep->filterTokenList(subFilterTokenList); - subsep->tableAlias("all_nations"); + subsep->filterTokenList(subFilterTokenList); + subsep->tableAlias("all_nations"); - // end of subselect in FROM. push FROM subselect to selectList - // NOTE: only FROM subselect needs to be pushed into selectList. - // Subselects in WHERE or HAVING clause are in where or having - // filter parse tree. It may make more sense to change the member - // fSelectList of CSEP class to fFromSubSelect (type CSEP*) - CalpontSelectExecutionPlan::SelectList fromSubSelectList; - fromSubSelectList.push_back(subsep); - csep.subSelects(fromSubSelectList); + // end of subselect in FROM. push FROM subselect to selectList + // NOTE: only FROM subselect needs to be pushed into selectList. + // Subselects in WHERE or HAVING clause are in where or having + // filter parse tree. It may make more sense to change the member + // fSelectList of CSEP class to fFromSubSelect (type CSEP*) + CalpontSelectExecutionPlan::SelectList fromSubSelectList; + fromSubSelectList.push_back(subsep); + csep.subSelects(fromSubSelectList); - ParseTree* pt = const_cast(subsep->filters()); - pt->drawTree("q8.dot"); + ParseTree* pt = const_cast(subsep->filters()); + pt->drawTree("q8.dot"); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn (*c1); - groupByList.push_back (g1); + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + SimpleColumn* g1 = new SimpleColumn(*c1); + groupByList.push_back(g1); - csep.groupByCols(groupByList); + csep.groupByCols(groupByList); - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - SimpleColumn* o1 = new SimpleColumn(*c1); - orderByList.push_back(o1); - - csep.orderByCols(orderByList); - - cout << csep; - } + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + SimpleColumn* o1 = new SimpleColumn(*c1); + orderByList.push_back(o1); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/tdriver9.cpp b/dbcon/execplan/tdriver9.cpp index b56390c0f..ca4e9c55a 100644 --- a/dbcon/execplan/tdriver9.cpp +++ b/dbcon/execplan/tdriver9.cpp @@ -22,9 +22,9 @@ using namespace std; #include -#include -#include -#include +#include +#include +#include #include #include "messagequeue.h" @@ -47,27 +47,26 @@ using namespace execplan; class TPCH_EXECPLAN : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TPCH_EXECPLAN); - CPPUNIT_TEST_SUITE( TPCH_EXECPLAN ); + CPPUNIT_TEST(Q1); - CPPUNIT_TEST( Q1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() - { - } - - void tearDown() - { - } - - void Q1() - { - string sql = "\ + void Q1() + { + string sql = + "\ select\ nation,\ o_year,\ @@ -101,141 +100,130 @@ order by\ nation,\ o_year desc;"; - CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan csep; - // Returned columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + // Returned columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - // these columns are from the temp table of FROM clause. - // I hereby give schema name "calpont", table name "FROMTABLE", - SimpleColumn* c1 = new SimpleColumn("tpch.nation.n_name"); - c1->tableAlias("profit"); - c1->alias("nation"); - returnedColumnList.push_back(c1); - ArithmeticColumn* c2 = new ArithmeticColumn("extract(year from tpch.orders.o_orderdate)"); - c2->tableAlias("profit"); - c2->tableAlias("o_year"); - returnedColumnList.push_back(c2); - ArithmeticColumn* c3 = new ArithmeticColumn("sum(amount)"); - c3->alias("sum_profit"); - returnedColumnList.push_back(c3); + // these columns are from the temp table of FROM clause. + // I hereby give schema name "calpont", table name "FROMTABLE", + SimpleColumn* c1 = new SimpleColumn("tpch.nation.n_name"); + c1->tableAlias("profit"); + c1->alias("nation"); + returnedColumnList.push_back(c1); + ArithmeticColumn* c2 = new ArithmeticColumn("extract(year from tpch.orders.o_orderdate)"); + c2->tableAlias("profit"); + c2->tableAlias("o_year"); + returnedColumnList.push_back(c2); + ArithmeticColumn* c3 = new ArithmeticColumn("sum(amount)"); + c3->alias("sum_profit"); + returnedColumnList.push_back(c3); - csep.returnedCols(returnedColumnList); + csep.returnedCols(returnedColumnList); - // subselect in FROM clause - CalpontSelectExecutionPlan* subsep = - new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); + // subselect in FROM clause + CalpontSelectExecutionPlan* subsep = new CalpontSelectExecutionPlan(CalpontSelectExecutionPlan::FROM); - // subselect returned columns - CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; - SimpleColumn* sc1 = new SimpleColumn("tpch.nation.n_name"); - sc1->alias("nation"); - sc1->tableAlias("profit"); - subReturnedColList.push_back(sc1); + // subselect returned columns + CalpontSelectExecutionPlan::ReturnedColumnList subReturnedColList; + SimpleColumn* sc1 = new SimpleColumn("tpch.nation.n_name"); + sc1->alias("nation"); + sc1->tableAlias("profit"); + subReturnedColList.push_back(sc1); - ArithmeticColumn* sc2 = new ArithmeticColumn("extract(year from tpch.orders.o_orderdate)"); - sc2->alias ("o_year"); - sc2->tableAlias("profit"); - subReturnedColList.push_back(sc2); + ArithmeticColumn* sc2 = new ArithmeticColumn("extract(year from tpch.orders.o_orderdate)"); + sc2->alias("o_year"); + sc2->tableAlias("profit"); + subReturnedColList.push_back(sc2); - ArithmeticColumn* sc3 = new ArithmeticColumn( - "tpch.lineitem.l_extendeprice * (1-tpch.lineitem.l_discount)- tpch.partsupp.ps_supplycost * tpch.lineitem.l_quantity"); - sc3->alias("amount"); - sc3->tableAlias("shipping"); - subReturnedColList.push_back(sc3); + ArithmeticColumn* sc3 = new ArithmeticColumn( + "tpch.lineitem.l_extendeprice * (1-tpch.lineitem.l_discount)- tpch.partsupp.ps_supplycost * " + "tpch.lineitem.l_quantity"); + sc3->alias("amount"); + sc3->tableAlias("shipping"); + subReturnedColList.push_back(sc3); - subsep->returnedCols(subReturnedColList); + subsep->returnedCols(subReturnedColList); - // subselect filters - CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; + // subselect filters + CalpontSelectExecutionPlan::FilterTokenList subFilterTokenList; - SimpleFilter* sf1 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_suppkey"), - new SimpleColumn("tpch.lineitem.l_suppkey")); - subFilterTokenList.push_back(sf1); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf1 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + subFilterTokenList.push_back(sf1); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf2 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.partsupp.ps_suppkey"), - new SimpleColumn("tpch.lineitem.l_suppkey")); - subFilterTokenList.push_back(sf2); + SimpleFilter* sf2 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.partsupp.ps_suppkey"), + new SimpleColumn("tpch.lineitem.l_suppkey")); + subFilterTokenList.push_back(sf2); - subFilterTokenList.push_back(new Operator("and")); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf3 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.partsupp.ps_partkey"), - new SimpleColumn("tpch.lineitem.l_partkey")); - subFilterTokenList.push_back(sf3); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf3 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.partsupp.ps_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + subFilterTokenList.push_back(sf3); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf4 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.part.p_partkey"), - new SimpleColumn("tpch.lineitem.l_partkey")); - subFilterTokenList.push_back(sf4); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf4 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.part.p_partkey"), + new SimpleColumn("tpch.lineitem.l_partkey")); + subFilterTokenList.push_back(sf4); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf5 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.order.o_orderkey"), - new SimpleColumn("tpch.lineitem.l_orderkey")); - subFilterTokenList.push_back(sf5); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf5 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.order.o_orderkey"), + new SimpleColumn("tpch.lineitem.l_orderkey")); + subFilterTokenList.push_back(sf5); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf6 = new SimpleFilter (new Operator("="), - new SimpleColumn("tpch.supplier.s_nationkey"), - new SimpleColumn("tpch.nation.n_nationkey")); - subFilterTokenList.push_back(sf6); - subFilterTokenList.push_back(new Operator("and")); + SimpleFilter* sf6 = new SimpleFilter(new Operator("="), new SimpleColumn("tpch.supplier.s_nationkey"), + new SimpleColumn("tpch.nation.n_nationkey")); + subFilterTokenList.push_back(sf6); + subFilterTokenList.push_back(new Operator("and")); - SimpleFilter* sf7 = new SimpleFilter ( new Operator("like"), - new SimpleColumn("tpch.part.p_name"), - new ConstantColumn ("%:1%")); - subFilterTokenList.push_back(sf7); + SimpleFilter* sf7 = new SimpleFilter(new Operator("like"), new SimpleColumn("tpch.part.p_name"), + new ConstantColumn("%:1%")); + subFilterTokenList.push_back(sf7); - subsep->filterTokenList(subFilterTokenList); - subsep->tableAlias("profit"); + subsep->filterTokenList(subFilterTokenList); + subsep->tableAlias("profit"); - CalpontSelectExecutionPlan::SelectList fromSubSelectList; - fromSubSelectList.push_back(subsep); - csep.subSelects(fromSubSelectList); + CalpontSelectExecutionPlan::SelectList fromSubSelectList; + fromSubSelectList.push_back(subsep); + csep.subSelects(fromSubSelectList); - ParseTree* pt = const_cast(subsep->filters()); - pt->drawTree("q9.dot"); + ParseTree* pt = const_cast(subsep->filters()); + pt->drawTree("q9.dot"); - // Group by - CalpontSelectExecutionPlan::GroupByColumnList groupByList; - SimpleColumn* g1 = new SimpleColumn (*c1); - groupByList.push_back (g1); - //ArithmeticColumn *g2 = new ArithmeticColumn (*c2); - groupByList.push_back (c2->clone()); - csep.groupByCols(groupByList); - - // Order by - CalpontSelectExecutionPlan::OrderByColumnList orderByList; - SimpleColumn* o1 = new SimpleColumn(*c1); - orderByList.push_back(o1); - ArithmeticColumn* o2 = c2->clone(); - o2->asc(false); - orderByList.push_back(o2); - csep.orderByCols(orderByList); - - cout << csep; - } + // Group by + CalpontSelectExecutionPlan::GroupByColumnList groupByList; + SimpleColumn* g1 = new SimpleColumn(*c1); + groupByList.push_back(g1); + // ArithmeticColumn *g2 = new ArithmeticColumn (*c2); + groupByList.push_back(c2->clone()); + csep.groupByCols(groupByList); + // Order by + CalpontSelectExecutionPlan::OrderByColumnList orderByList; + SimpleColumn* o1 = new SimpleColumn(*c1); + orderByList.push_back(o1); + ArithmeticColumn* o2 = c2->clone(); + o2->asc(false); + orderByList.push_back(o2); + csep.orderByCols(orderByList); + cout << csep; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( TPCH_EXECPLAN ); +CPPUNIT_TEST_SUITE_REGISTRATION(TPCH_EXECPLAN); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/execplan/treenode.cpp b/dbcon/execplan/treenode.cpp index d25343ca4..51625eeb0 100644 --- a/dbcon/execplan/treenode.cpp +++ b/dbcon/execplan/treenode.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: treenode.cpp 9317 2013-03-19 21:37:22Z dhall $ -* -* -***********************************************************************/ + * $Id: treenode.cpp 9317 2013-03-19 21:37:22Z dhall $ + * + * + ***********************************************************************/ /** @file */ #include @@ -35,41 +35,40 @@ using namespace std; namespace execplan { - /** * Constructors/Destructors */ -TreeNode::TreeNode(): fDerivedTable(""), - fRefCount(0), - fDerivedRefCol(NULL) +TreeNode::TreeNode() : fDerivedTable(""), fRefCount(0), fDerivedRefCol(NULL) { - memset(tmp, 0, 312); + memset(tmp, 0, 312); } -TreeNode::TreeNode(const TreeNode& rhs): - fResult(rhs.fResult), - fResultType(rhs.resultType()), - fOperationType(rhs.operationType()), - fDerivedTable (rhs.derivedTable()), - fRefCount(rhs.refCount()), - fDerivedRefCol(rhs.derivedRefCol()) +TreeNode::TreeNode(const TreeNode& rhs) + : fResult(rhs.fResult) + , fResultType(rhs.resultType()) + , fOperationType(rhs.operationType()) + , fDerivedTable(rhs.derivedTable()) + , fRefCount(rhs.refCount()) + , fDerivedRefCol(rhs.derivedRefCol()) { - memcpy(tmp, rhs.tmp, 312); + memcpy(tmp, rhs.tmp, 312); } -TreeNode::~TreeNode() {} - -void TreeNode::resultType ( const execplan::CalpontSystemCatalog::ColType& resultType) +TreeNode::~TreeNode() { - fResultType = resultType; +} - // set scale/precision for the result - if (fResultType.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - fResultType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) - { - fResult.decimalVal.scale = fResultType.scale; - fResult.decimalVal.precision = fResultType.precision; - } +void TreeNode::resultType(const execplan::CalpontSystemCatalog::ColType& resultType) +{ + fResultType = resultType; + + // set scale/precision for the result + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + fResultType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + fResult.decimalVal.scale = fResultType.scale; + fResult.decimalVal.precision = fResultType.precision; + } } /** @@ -77,8 +76,8 @@ void TreeNode::resultType ( const execplan::CalpontSystemCatalog::ColType& resul */ ostream& operator<<(ostream& output, const TreeNode& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } -} /* namespace */ +} // namespace execplan diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index e9dbe5903..1dfe1e4e2 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -18,7 +18,6 @@ // $Id: treenode.h 9635 2013-06-19 21:42:30Z bwilkinson $ - /** @file */ #ifndef CALPONT_TREENODE_H @@ -54,7 +53,6 @@ class Row; */ namespace execplan { - typedef execplan::CalpontSystemCatalog::ColType Type; typedef datatypes::Decimal IDB_Decimal; @@ -84,48 +82,48 @@ typedef SP_IDB_Regex SP_CNX_Regex; */ inline std::string removeTrailing0(char* val, uint32_t length) { - char* ptr = val; - int64_t i = 0; - bool decimal_point = false; + char* ptr = val; + int64_t i = 0; + bool decimal_point = false; - for (; i < length; i++, ptr++) + for (; i < length; i++, ptr++) + { + if (*ptr == '+' || *ptr == '-' || (*ptr >= '0' && *ptr <= '9')) { - if (*ptr == '+' || *ptr == '-' || (*ptr >= '0' && *ptr <= '9')) - { - continue; - } + continue; + } - if (*ptr == '.') - { - decimal_point = true; - continue; - } + if (*ptr == '.') + { + decimal_point = true; + continue; + } - *ptr = 0; + *ptr = 0; + break; + } + + if (decimal_point) + { + for (i = i - 1; i >= 0; i--) + { + if (val[i] == '0') + { + val[i] = 0; + } + else if (val[i] == '.') + { + val[i] = 0; break; + } + else + { + break; + } } + } - if (decimal_point) - { - for (i = i - 1; i >= 0; i--) - { - if (val[i] == '0') - { - val[i] = 0; - } - else if (val[i] == '.') - { - val[i] = 0; - break; - } - else - { - break; - } - } - } - - return std::string(val); + return std::string(val); } /** @@ -134,23 +132,33 @@ inline std::string removeTrailing0(char* val, uint32_t length) */ struct Result { - Result(): intVal(0), uintVal(0), origIntVal(0), dummy(0), - doubleVal(0), longDoubleVal(0), floatVal(0), boolVal(false), - strVal(""), decimalVal(IDB_Decimal()), - valueConverted(false) {} - int64_t intVal; - uint64_t uintVal; - uint64_t origIntVal; - // clear up the memory following origIntVal to make sure null terminated string - // when converting origIntVal - uint64_t dummy; - double doubleVal; - long double longDoubleVal; - float floatVal; - bool boolVal; - std::string strVal; - IDB_Decimal decimalVal; - bool valueConverted; + Result() + : intVal(0) + , uintVal(0) + , origIntVal(0) + , dummy(0) + , doubleVal(0) + , longDoubleVal(0) + , floatVal(0) + , boolVal(false) + , strVal("") + , decimalVal(IDB_Decimal()) + , valueConverted(false) + { + } + int64_t intVal; + uint64_t uintVal; + uint64_t origIntVal; + // clear up the memory following origIntVal to make sure null terminated string + // when converting origIntVal + uint64_t dummy; + double doubleVal; + long double longDoubleVal; + float floatVal; + bool boolVal; + std::string strVal; + IDB_Decimal decimalVal; + bool valueConverted; }; /** @@ -159,1020 +167,969 @@ struct Result */ class TreeNode { -public: - TreeNode(); - TreeNode(const TreeNode& rhs); - virtual ~TreeNode(); - virtual const std::string data() const = 0; - virtual void data(const std::string data) = 0; - virtual const std::string toString() const = 0; - virtual TreeNode* clone() const = 0; + public: + TreeNode(); + TreeNode(const TreeNode& rhs); + virtual ~TreeNode(); + virtual const std::string data() const = 0; + virtual void data(const std::string data) = 0; + virtual const std::string toString() const = 0; + virtual TreeNode* clone() const = 0; - /** - * Interface for serialization - */ + /** + * Interface for serialization + */ - /** @brief Convert *this to a stream of bytes - * - * Convert *this to a stream of bytes. - * @param b The ByteStream to add *this to. - */ - virtual void serialize(messageqcpp::ByteStream& b) const = 0; + /** @brief Convert *this to a stream of bytes + * + * Convert *this to a stream of bytes. + * @param b The ByteStream to add *this to. + */ + virtual void serialize(messageqcpp::ByteStream& b) const = 0; - /** @brief Construct a TreeNode from a stream of bytes - * - * Construct a TreeNode from a stream of bytes. - * @param b The ByteStream to parse - * @return The newly allocated TreeNode - */ - virtual void unserialize(messageqcpp::ByteStream& b) = 0; + /** @brief Construct a TreeNode from a stream of bytes + * + * Construct a TreeNode from a stream of bytes. + * @param b The ByteStream to parse + * @return The newly allocated TreeNode + */ + virtual void unserialize(messageqcpp::ByteStream& b) = 0; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const = 0; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const = 0; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const = 0; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const = 0; - // derivedTable mutator and accessor - virtual const std::string& derivedTable() const - { - return fDerivedTable; - } + // derivedTable mutator and accessor + virtual const std::string& derivedTable() const + { + return fDerivedTable; + } - virtual void derivedTable(const std::string& derivedTable) - { - fDerivedTable = derivedTable; - } + virtual void derivedTable(const std::string& derivedTable) + { + fDerivedTable = derivedTable; + } - // must to be implented by treenode that could potentially belong to - // one single derived table - virtual void setDerivedTable() - { - fDerivedTable = ""; - } + // must to be implented by treenode that could potentially belong to + // one single derived table + virtual void setDerivedTable() + { + fDerivedTable = ""; + } - virtual TreeNode* derivedRefCol() const - { - return fDerivedRefCol; - } + virtual TreeNode* derivedRefCol() const + { + return fDerivedRefCol; + } - virtual void derivedRefCol(TreeNode* derivedRefCol) - { - fDerivedRefCol = derivedRefCol; - } + virtual void derivedRefCol(TreeNode* derivedRefCol) + { + fDerivedRefCol = derivedRefCol; + } - virtual uint64_t refCount() const - { - return fRefCount; - } + virtual uint64_t refCount() const + { + return fRefCount; + } - virtual void refCount (const uint64_t refCount) - { - fRefCount = refCount; - } + virtual void refCount(const uint64_t refCount) + { + fRefCount = refCount; + } - // the inc and dec functions areparm[n]->data() used by connector single thread. - virtual void decRefCount() - { - fRefCount--; - } + // the inc and dec functions areparm[n]->data() used by connector single thread. + virtual void decRefCount() + { + fRefCount--; + } - virtual void incRefCount() - { - fRefCount++; - } + virtual void incRefCount() + { + fRefCount++; + } - /*********************************************************************** - * F&E framework * - ***********************************************************************/ - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - return fResult.strVal; - } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - return fResult.intVal; - } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - return fResult.uintVal; - } - datatypes::TUInt64Null toTUInt64Null(rowgroup::Row& row) - { - bool isNull = false; - uint64_t val = getUintVal(row, isNull); - return datatypes::TUInt64Null(val, isNull); - } - datatypes::TSInt64Null toTSInt64Null(rowgroup::Row& row) - { - bool isNull = false; - int64_t val = getIntVal(row, isNull); - return datatypes::TSInt64Null(val, isNull); - } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - return fResult.floatVal; - } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - return fResult.doubleVal; - } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - return fResult.longDoubleVal; - } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - return fResult.decimalVal; - } - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) - { - return fResult.boolVal; - } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - return fResult.intVal; - } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - return fResult.intVal; - } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - return fResult.intVal; - } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - return fResult.intVal; - } - virtual void evaluate(rowgroup::Row& row, bool& isNull) {} + /*********************************************************************** + * F&E framework * + ***********************************************************************/ + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + return fResult.strVal; + } + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + return fResult.uintVal; + } + datatypes::TUInt64Null toTUInt64Null(rowgroup::Row& row) + { + bool isNull = false; + uint64_t val = getUintVal(row, isNull); + return datatypes::TUInt64Null(val, isNull); + } + datatypes::TSInt64Null toTSInt64Null(rowgroup::Row& row) + { + bool isNull = false; + int64_t val = getIntVal(row, isNull); + return datatypes::TSInt64Null(val, isNull); + } + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + return fResult.floatVal; + } + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + return fResult.doubleVal; + } + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + return fResult.longDoubleVal; + } + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + return fResult.decimalVal; + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) + { + return fResult.boolVal; + } + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } + virtual void evaluate(rowgroup::Row& row, bool& isNull) + { + } - inline bool getBoolVal(); - inline const std::string& getStrVal(const std::string& timeZone); - inline int64_t getIntVal(); - inline uint64_t getUintVal(); - inline float getFloatVal(); - inline double getDoubleVal(); - inline long double getLongDoubleVal(); - inline IDB_Decimal getDecimalVal(); - inline int32_t getDateIntVal(); - inline int64_t getDatetimeIntVal(); - inline int64_t getTimestampIntVal(); - inline int64_t getTimeIntVal(); + inline bool getBoolVal(); + inline const std::string& getStrVal(const std::string& timeZone); + inline int64_t getIntVal(); + inline uint64_t getUintVal(); + inline float getFloatVal(); + inline double getDoubleVal(); + inline long double getLongDoubleVal(); + inline IDB_Decimal getDecimalVal(); + inline int32_t getDateIntVal(); + inline int64_t getDatetimeIntVal(); + inline int64_t getTimestampIntVal(); + inline int64_t getTimeIntVal(); - virtual const execplan::CalpontSystemCatalog::ColType& resultType() const - { - return fResultType; - } - virtual execplan::CalpontSystemCatalog::ColType& resultType() - { - return fResultType; - } - virtual void resultType ( const execplan::CalpontSystemCatalog::ColType& resultType ) ; - virtual void operationType(const Type& type) - { - fOperationType = type; - } - virtual const execplan::CalpontSystemCatalog::ColType& operationType() const - { - return fOperationType; - } + virtual const execplan::CalpontSystemCatalog::ColType& resultType() const + { + return fResultType; + } + virtual execplan::CalpontSystemCatalog::ColType& resultType() + { + return fResultType; + } + virtual void resultType(const execplan::CalpontSystemCatalog::ColType& resultType); + virtual void operationType(const Type& type) + { + fOperationType = type; + } + virtual const execplan::CalpontSystemCatalog::ColType& operationType() const + { + return fOperationType; + } - // result mutator and accessor. for speical functor to set for optimization. - virtual void result(const Result& result) - { - fResult = result; - } - virtual const Result& result() const - { - return fResult; - } + // result mutator and accessor. for speical functor to set for optimization. + virtual void result(const Result& result) + { + fResult = result; + } + virtual const Result& result() const + { + return fResult; + } - uint32_t charsetNumber() const - { - return fResultType.charsetNumber; - } - void charsetNumber(uint32_t cnum) - { - fResultType.charsetNumber = cnum; - fOperationType.charsetNumber = cnum; - } + uint32_t charsetNumber() const + { + return fResultType.charsetNumber; + } + void charsetNumber(uint32_t cnum) + { + fResultType.charsetNumber = cnum; + fOperationType.charsetNumber = cnum; + } -protected: - Result fResult; - execplan::CalpontSystemCatalog::ColType fResultType; // mapped from mysql data type - execplan::CalpontSystemCatalog::ColType fOperationType; // operator type, could be different from the result type + protected: + Result fResult; + execplan::CalpontSystemCatalog::ColType fResultType; // mapped from mysql data type + execplan::CalpontSystemCatalog::ColType + fOperationType; // operator type, could be different from the result type - // double's range is +/-1.7E308 with at least 15 digits of precision - char tmp[312]; // for conversion use + // double's range is +/-1.7E308 with at least 15 digits of precision + char tmp[312]; // for conversion use - // @bug5635 If any item involved in this filter belongs to a derived table, - // the derived table alias is added to the reference vector. - std::string fDerivedTable; - uint64_t fRefCount; - TreeNode* fDerivedRefCol; + // @bug5635 If any item involved in this filter belongs to a derived table, + // the derived table alias is added to the reference vector. + std::string fDerivedTable; + uint64_t fRefCount; + TreeNode* fDerivedRefCol; -private: - //default okay - //TreeNode& operator=(const TreeNode& rhs); + private: + // default okay + // TreeNode& operator=(const TreeNode& rhs); }; inline bool TreeNode::getBoolVal() { - switch (fResultType.colDataType) - { - case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return (atoi((char*)(&fResult.origIntVal)) != 0); + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + if (fResultType.colWidth <= 8) + return (atoi((char*)(&fResult.origIntVal)) != 0); - return (atoi(fResult.strVal.c_str()) != 0); + return (atoi(fResult.strVal.c_str()) != 0); - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return (atoi((char*)(&fResult.origIntVal)) != 0); + case CalpontSystemCatalog::VARCHAR: + if (fResultType.colWidth <= 7) + return (atoi((char*)(&fResult.origIntVal)) != 0); - return (atoi(fResult.strVal.c_str()) != 0); + return (atoi(fResult.strVal.c_str()) != 0); - //FIXME: Huh??? - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return (atoi((char*)(&fResult.origIntVal)) != 0); + // FIXME: Huh??? + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + if (fResultType.colWidth <= 7) + return (atoi((char*)(&fResult.origIntVal)) != 0); - return (atoi(fResult.strVal.c_str()) != 0); + return (atoi(fResult.strVal.c_str()) != 0); - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return (fResult.intVal != 0); + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return (fResult.intVal != 0); - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::UINT: - return (fResult.uintVal != 0); + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::UINT: return (fResult.uintVal != 0); - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (fResult.floatVal != 0); + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return (fResult.floatVal != 0); - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return (fResult.doubleVal != 0); + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return (fResult.doubleVal != 0); - case CalpontSystemCatalog::LONGDOUBLE: - return (fResult.longDoubleVal != 0); + case CalpontSystemCatalog::LONGDOUBLE: return (fResult.longDoubleVal != 0); - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - return (fResult.decimalVal.s128Value != 0); - else - return (fResult.decimalVal.value != 0); + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + return (fResult.decimalVal.s128Value != 0); + else + return (fResult.decimalVal.value != 0); - default: - throw logging::InvalidConversionExcept("TreeNode::getBoolVal: Invalid conversion."); - } + default: throw logging::InvalidConversionExcept("TreeNode::getBoolVal: Invalid conversion."); + } - return fResult.boolVal; + return fResult.boolVal; } inline const std::string& TreeNode::getStrVal(const std::string& timeZone) { - switch (fResultType.colDataType) + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + if (fResultType.colWidth <= 8) + fResult.strVal = (char*)(&fResult.origIntVal); + + break; + + case CalpontSystemCatalog::VARCHAR: + if (fResultType.colWidth <= 7) + fResult.strVal = (char*)(&fResult.origIntVal); + + break; + + // FIXME: ??? + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + if (fResultType.colWidth <= 7) + fResult.strVal = (char*)(&fResult.origIntVal); + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: { - case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - fResult.strVal = (char*)(&fResult.origIntVal); - - break; - - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - fResult.strVal = (char*)(&fResult.origIntVal); - - break; - - //FIXME: ??? - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - fResult.strVal = (char*)(&fResult.origIntVal); - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - { #ifndef __LP64__ - snprintf(tmp, 20, "%lld", fResult.intVal); + snprintf(tmp, 20, "%lld", fResult.intVal); #else - snprintf(tmp, 20, "%ld", fResult.intVal); + snprintf(tmp, 20, "%ld", fResult.intVal); #endif - fResult.strVal = std::string(tmp); - break; - } - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::UINT: - { -#ifndef __LP64__ - snprintf(tmp, 20, "%llu", fResult.uintVal); -#else - snprintf(tmp, 20, "%lu", fResult.uintVal); -#endif - fResult.strVal = std::string(tmp); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - if ((fabs(fResult.floatVal) > (1.0 / IDB_pow[4])) && - (fabs(fResult.floatVal) < (float) IDB_pow[6])) - { - snprintf(tmp, 312, "%f", fResult.floatVal); - fResult.strVal = removeTrailing0(tmp, 312); - } - else - { - // MCOL-299 Print scientific with 5 mantissa and no + sign for exponent - int exponent = (int)floor(log10( fabs(fResult.floatVal))); // This will round down the exponent - double base = fResult.floatVal * pow(10, -1.0 * exponent); - - if (std::isnan(exponent) || std::isnan(base)) - { - snprintf(tmp, 312, "%f", fResult.floatVal); - fResult.strVal = removeTrailing0(tmp, 312); - } - else - { - snprintf(tmp, 312, "%.5f", base); - fResult.strVal = removeTrailing0(tmp, 312); - snprintf(tmp, 312, "e%02d", exponent); - fResult.strVal += tmp; - } - -// snprintf(tmp, 312, "%e.5", fResult.floatVal); -// fResult.strVal = tmp; - } - - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - if ((fabs(fResult.doubleVal) > (1.0 / IDB_pow[13])) && - (fabs(fResult.doubleVal) < (float) IDB_pow[15])) - { - snprintf(tmp, 312, "%f", fResult.doubleVal); - fResult.strVal = removeTrailing0(tmp, 312); - } - else - { - // MCOL-299 Print scientific with 9 mantissa and no + sign for exponent - int exponent = (int)floor(log10( fabs(fResult.doubleVal))); // This will round down the exponent - double base = fResult.doubleVal * pow(10, -1.0 * exponent); - - if (std::isnan(exponent) || std::isnan(base)) - { - snprintf(tmp, 312, "%f", fResult.doubleVal); - fResult.strVal = removeTrailing0(tmp, 312); - } - else - { - snprintf(tmp, 312, "%.9f", base); - fResult.strVal = removeTrailing0(tmp, 312); - snprintf(tmp, 312, "e%02d", exponent); - fResult.strVal += tmp; - } - -// snprintf(tmp, 312, "%e", fResult.doubleVal); -// fResult.strVal = tmp; - } - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - if ((fabsl(fResult.longDoubleVal) > (1.0 / IDB_pow[13])) && - (fabsl(fResult.longDoubleVal) < (float) IDB_pow[15])) - { - snprintf(tmp, 312, "%Lf", fResult.longDoubleVal); - fResult.strVal = removeTrailing0(tmp, 312); - } - else - { - // MCOL-299 Print scientific with 9 mantissa and no + sign for exponent - int exponent = (int)floorl(log10( fabsl(fResult.longDoubleVal))); // This will round down the exponent - long double base = fResult.longDoubleVal * pow(10, -1.0 * exponent); - - if (std::isnan(exponent) || std::isnan(base)) - { - snprintf(tmp, 312, "%Lf", fResult.longDoubleVal); - fResult.strVal = removeTrailing0(tmp, 312); - } - else - { - snprintf(tmp, 312, "%.14Lf", base); - fResult.strVal = removeTrailing0(tmp, 312); - snprintf(tmp, 312, "e%02d", exponent); - fResult.strVal += tmp; - } - -// snprintf(tmp, 312, "%e", fResult.doubleVal); -// fResult.strVal = tmp; - } - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - // Explicit path for TSInt128 decimals with low precision - fResult.strVal = fResult.decimalVal.toString(true); - else - fResult.strVal = fResult.decimalVal.toString(); - break; - } - - case CalpontSystemCatalog::DATE: - { - dataconvert::DataConvert::dateToString(fResult.intVal, tmp, 255); - fResult.strVal = std::string(tmp); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255, fResultType.precision); - fResult.strVal = std::string(tmp); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::DataConvert::timestampToString(fResult.intVal, tmp, 255, timeZone, fResultType.precision); - fResult.strVal = std::string(tmp); - break; - } - - case CalpontSystemCatalog::TIME: - { - dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); - fResult.strVal = std::string(tmp); - break; - } - - default: - throw logging::InvalidConversionExcept("TreeNode::getStrVal: Invalid conversion."); + fResult.strVal = std::string(tmp); + break; } - return fResult.strVal; + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::UINT: + { +#ifndef __LP64__ + snprintf(tmp, 20, "%llu", fResult.uintVal); +#else + snprintf(tmp, 20, "%lu", fResult.uintVal); +#endif + fResult.strVal = std::string(tmp); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + if ((fabs(fResult.floatVal) > (1.0 / IDB_pow[4])) && (fabs(fResult.floatVal) < (float)IDB_pow[6])) + { + snprintf(tmp, 312, "%f", fResult.floatVal); + fResult.strVal = removeTrailing0(tmp, 312); + } + else + { + // MCOL-299 Print scientific with 5 mantissa and no + sign for exponent + int exponent = (int)floor(log10(fabs(fResult.floatVal))); // This will round down the exponent + double base = fResult.floatVal * pow(10, -1.0 * exponent); + + if (std::isnan(exponent) || std::isnan(base)) + { + snprintf(tmp, 312, "%f", fResult.floatVal); + fResult.strVal = removeTrailing0(tmp, 312); + } + else + { + snprintf(tmp, 312, "%.5f", base); + fResult.strVal = removeTrailing0(tmp, 312); + snprintf(tmp, 312, "e%02d", exponent); + fResult.strVal += tmp; + } + + // snprintf(tmp, 312, "%e.5", fResult.floatVal); + // fResult.strVal = tmp; + } + + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + if ((fabs(fResult.doubleVal) > (1.0 / IDB_pow[13])) && (fabs(fResult.doubleVal) < (float)IDB_pow[15])) + { + snprintf(tmp, 312, "%f", fResult.doubleVal); + fResult.strVal = removeTrailing0(tmp, 312); + } + else + { + // MCOL-299 Print scientific with 9 mantissa and no + sign for exponent + int exponent = (int)floor(log10(fabs(fResult.doubleVal))); // This will round down the exponent + double base = fResult.doubleVal * pow(10, -1.0 * exponent); + + if (std::isnan(exponent) || std::isnan(base)) + { + snprintf(tmp, 312, "%f", fResult.doubleVal); + fResult.strVal = removeTrailing0(tmp, 312); + } + else + { + snprintf(tmp, 312, "%.9f", base); + fResult.strVal = removeTrailing0(tmp, 312); + snprintf(tmp, 312, "e%02d", exponent); + fResult.strVal += tmp; + } + + // snprintf(tmp, 312, "%e", fResult.doubleVal); + // fResult.strVal = tmp; + } + + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + if ((fabsl(fResult.longDoubleVal) > (1.0 / IDB_pow[13])) && + (fabsl(fResult.longDoubleVal) < (float)IDB_pow[15])) + { + snprintf(tmp, 312, "%Lf", fResult.longDoubleVal); + fResult.strVal = removeTrailing0(tmp, 312); + } + else + { + // MCOL-299 Print scientific with 9 mantissa and no + sign for exponent + int exponent = (int)floorl(log10(fabsl(fResult.longDoubleVal))); // This will round down the exponent + long double base = fResult.longDoubleVal * pow(10, -1.0 * exponent); + + if (std::isnan(exponent) || std::isnan(base)) + { + snprintf(tmp, 312, "%Lf", fResult.longDoubleVal); + fResult.strVal = removeTrailing0(tmp, 312); + } + else + { + snprintf(tmp, 312, "%.14Lf", base); + fResult.strVal = removeTrailing0(tmp, 312); + snprintf(tmp, 312, "e%02d", exponent); + fResult.strVal += tmp; + } + + // snprintf(tmp, 312, "%e", fResult.doubleVal); + // fResult.strVal = tmp; + } + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + // Explicit path for TSInt128 decimals with low precision + fResult.strVal = fResult.decimalVal.toString(true); + else + fResult.strVal = fResult.decimalVal.toString(); + break; + } + + case CalpontSystemCatalog::DATE: + { + dataconvert::DataConvert::dateToString(fResult.intVal, tmp, 255); + fResult.strVal = std::string(tmp); + break; + } + + case CalpontSystemCatalog::DATETIME: + { + dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255, fResultType.precision); + fResult.strVal = std::string(tmp); + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::DataConvert::timestampToString(fResult.intVal, tmp, 255, timeZone, fResultType.precision); + fResult.strVal = std::string(tmp); + break; + } + + case CalpontSystemCatalog::TIME: + { + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); + fResult.strVal = std::string(tmp); + break; + } + + default: throw logging::InvalidConversionExcept("TreeNode::getStrVal: Invalid conversion."); + } + + return fResult.strVal; } inline int64_t TreeNode::getIntVal() { - switch (fResultType.colDataType) - { - case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return fResult.intVal; + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + if (fResultType.colWidth <= 8) + return fResult.intVal; - return atoll(fResult.strVal.c_str()); + return atoll(fResult.strVal.c_str()); - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return fResult.intVal; + case CalpontSystemCatalog::VARCHAR: + if (fResultType.colWidth <= 7) + return fResult.intVal; - return atoll(fResult.strVal.c_str()); + return atoll(fResult.strVal.c_str()); - //FIXME: ??? - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return fResult.intVal; + // FIXME: ??? + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + if (fResultType.colWidth <= 7) + return fResult.intVal; - return atoll(fResult.strVal.c_str()); + return atoll(fResult.strVal.c_str()); - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return fResult.intVal; + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return fResult.intVal; - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return fResult.uintVal; + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return fResult.uintVal; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (int64_t)fResult.floatVal; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return (int64_t)fResult.floatVal; - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return (int64_t)fResult.doubleVal; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return (int64_t)fResult.doubleVal; - case CalpontSystemCatalog::LONGDOUBLE: - return (int64_t)fResult.longDoubleVal; + case CalpontSystemCatalog::LONGDOUBLE: return (int64_t)fResult.longDoubleVal; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - return fResult.decimalVal.toSInt64Round(); + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: return fResult.decimalVal.toSInt64Round(); - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return fResult.intVal; + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return fResult.intVal; - default: - throw logging::InvalidConversionExcept("TreeNode::getIntVal: Invalid conversion."); - } + default: throw logging::InvalidConversionExcept("TreeNode::getIntVal: Invalid conversion."); + } - return fResult.intVal; + return fResult.intVal; } inline uint64_t TreeNode::getUintVal() { - switch (fResultType.colDataType) - { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return fResult.intVal; + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return fResult.intVal; - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return fResult.uintVal; + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return fResult.uintVal; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (uint64_t)fResult.floatVal; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return (uint64_t)fResult.floatVal; - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return (uint64_t)fResult.doubleVal; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return (uint64_t)fResult.doubleVal; - case CalpontSystemCatalog::LONGDOUBLE: - return (uint64_t)fResult.longDoubleVal; + case CalpontSystemCatalog::LONGDOUBLE: return (uint64_t)fResult.longDoubleVal; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - return fResult.decimalVal.toUInt64Round(); + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: return fResult.decimalVal.toUInt64Round(); - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return fResult.intVal; + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return fResult.intVal; - default: - throw logging::InvalidConversionExcept("TreeNode::getIntVal: Invalid conversion."); - } + default: throw logging::InvalidConversionExcept("TreeNode::getIntVal: Invalid conversion."); + } - return fResult.intVal; + return fResult.intVal; } inline float TreeNode::getFloatVal() { - switch (fResultType.colDataType) + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + if (fResultType.colWidth <= 8) + return atof((char*)(&fResult.origIntVal)); + + return atof(fResult.strVal.c_str()); + + case CalpontSystemCatalog::VARCHAR: + if (fResultType.colWidth <= 7) + return atof((char*)(&fResult.origIntVal)); + + return atof(fResult.strVal.c_str()); + + // FIXME: ??? + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + if (fResultType.colWidth <= 7) + return atof((char*)(&fResult.origIntVal)); + + return atof(fResult.strVal.c_str()); + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return (float)fResult.intVal; + + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return (float)fResult.uintVal; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return fResult.floatVal; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return (float)fResult.doubleVal; + + case CalpontSystemCatalog::LONGDOUBLE: return (float)fResult.doubleVal; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return atof((char*)(&fResult.origIntVal)); - - return atof(fResult.strVal.c_str()); - - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return atof((char*)(&fResult.origIntVal)); - - return atof(fResult.strVal.c_str()); - - //FIXME: ??? - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return atof((char*)(&fResult.origIntVal)); - - return atof(fResult.strVal.c_str()); - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return (float)fResult.intVal; - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return (float)fResult.uintVal; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return fResult.floatVal; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return (float)fResult.doubleVal; - - case CalpontSystemCatalog::LONGDOUBLE: - return (float)fResult.doubleVal; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - { - return static_cast(fResult.decimalVal); - } - else - { - return (float) fResult.decimalVal.decimal64ToXFloat(); - } - } - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return (float)fResult.intVal; - - default: - throw logging::InvalidConversionExcept("TreeNode::getFloatVal: Invalid conversion."); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + return static_cast(fResult.decimalVal); + } + else + { + return (float)fResult.decimalVal.decimal64ToXFloat(); + } } - return fResult.floatVal; + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return (float)fResult.intVal; + + default: throw logging::InvalidConversionExcept("TreeNode::getFloatVal: Invalid conversion."); + } + + return fResult.floatVal; } inline double TreeNode::getDoubleVal() { - switch (fResultType.colDataType) + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + if (fResultType.colWidth <= 8) + return strtod((char*)(&fResult.origIntVal), NULL); + + return strtod(fResult.strVal.c_str(), NULL); + + case CalpontSystemCatalog::VARCHAR: + if (fResultType.colWidth <= 7) + return strtod((char*)(&fResult.origIntVal), NULL); + + return strtod(fResult.strVal.c_str(), NULL); + + // FIXME: ??? + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + if (fResultType.colWidth <= 7) + return strtod((char*)(&fResult.origIntVal), NULL); + + return strtod(fResult.strVal.c_str(), NULL); + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return (double)fResult.intVal; + + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return (double)fResult.uintVal; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return (double)fResult.floatVal; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return fResult.doubleVal; + + case CalpontSystemCatalog::LONGDOUBLE: return (double)fResult.longDoubleVal; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return strtod((char*)(&fResult.origIntVal), NULL); - - return strtod(fResult.strVal.c_str(), NULL); - - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return strtod((char*)(&fResult.origIntVal), NULL); - - return strtod(fResult.strVal.c_str(), NULL); - - //FIXME: ??? - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return strtod((char*)(&fResult.origIntVal), NULL); - - return strtod(fResult.strVal.c_str(), NULL); - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return (double)fResult.intVal; - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return (double)fResult.uintVal; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (double)fResult.floatVal; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return fResult.doubleVal; - - case CalpontSystemCatalog::LONGDOUBLE: - return (double)fResult.longDoubleVal; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - { - return static_cast(fResult.decimalVal); - } - else - { - return fResult.decimalVal.decimal64ToXFloat(); - } - } - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return (double)fResult.intVal; - - default: - throw logging::InvalidConversionExcept("TreeNode::getDoubleVal: Invalid conversion."); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + return static_cast(fResult.decimalVal); + } + else + { + return fResult.decimalVal.decimal64ToXFloat(); + } } - return fResult.doubleVal; + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return (double)fResult.intVal; + + default: throw logging::InvalidConversionExcept("TreeNode::getDoubleVal: Invalid conversion."); + } + + return fResult.doubleVal; } inline long double TreeNode::getLongDoubleVal() { - switch (fResultType.colDataType) + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + if (fResultType.colWidth <= 8) + return strtold((char*)(&fResult.origIntVal), NULL); + + return strtold(fResult.strVal.c_str(), NULL); + + case CalpontSystemCatalog::VARCHAR: + if (fResultType.colWidth <= 7) + return strtold((char*)(&fResult.origIntVal), NULL); + + return strtold(fResult.strVal.c_str(), NULL); + + // FIXME: ??? + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + if (fResultType.colWidth <= 7) + return strtold((char*)(&fResult.origIntVal), NULL); + + return strtold(fResult.strVal.c_str(), NULL); + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return (long double)fResult.intVal; + + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return (long double)fResult.uintVal; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return (long double)fResult.floatVal; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return (long double)fResult.doubleVal; + + case CalpontSystemCatalog::LONGDOUBLE: return (long double)fResult.longDoubleVal; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return strtold((char*)(&fResult.origIntVal), NULL); - - return strtold(fResult.strVal.c_str(), NULL); - - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return strtold((char*)(&fResult.origIntVal), NULL); - - return strtold(fResult.strVal.c_str(), NULL); - - //FIXME: ??? - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return strtold((char*)(&fResult.origIntVal), NULL); - - return strtold(fResult.strVal.c_str(), NULL); - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return (long double)fResult.intVal; - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return (long double)fResult.uintVal; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (long double)fResult.floatVal; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return (long double)fResult.doubleVal; - - case CalpontSystemCatalog::LONGDOUBLE: - return (long double)fResult.longDoubleVal; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - { - return static_cast(fResult.decimalVal); - } - else - { - return fResult.decimalVal.decimal64ToXFloat(); - } - } - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - return (long double)fResult.intVal; - - default: - throw logging::InvalidConversionExcept("TreeNode::getDoubleVal: Invalid conversion."); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + return static_cast(fResult.decimalVal); + } + else + { + return fResult.decimalVal.decimal64ToXFloat(); + } } - return fResult.doubleVal; + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (long double)fResult.intVal; + + default: throw logging::InvalidConversionExcept("TreeNode::getDoubleVal: Invalid conversion."); + } + + return fResult.doubleVal; } inline IDB_Decimal TreeNode::getDecimalVal() { - switch (fResultType.colDataType) + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from string"); + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + throw logging::InvalidConversionExcept( + "TreeNode::getDecimalVal: non-support conversion from binary string"); + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from string"); - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from binary string"); - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - { - idbassert(fResultType.scale == 0); - fResult.decimalVal = IDB_Decimal(fResult.intVal, - 0, - fResultType.precision, - fResult.intVal); - break; - } - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UTINYINT: - { - idbassert(fResultType.scale == 0); - fResult.decimalVal = IDB_Decimal((int64_t)fResult.uintVal, - 0, - fResultType.precision, - fResult.uintVal); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dlScaled = fResult.longDoubleVal; - - if (fResultType.scale > 0) - { - dlScaled = fResult.longDoubleVal * pow((double)10.0, fResultType.scale); - } - - if ((dlScaled > (long double)INT64_MAX) || (dlScaled < (long double)(INT64_MIN))) - { - datatypes::TFloat128 temp((float128_t)dlScaled); - fResult.decimalVal = IDB_Decimal(0, fResultType.scale, - fResultType.precision, static_cast(temp)); - } - else - { - int64_t val = (int64_t)lroundl(dlScaled); - fResult.decimalVal = IDB_Decimal(val, - fResultType.scale, - fResultType.precision, - val); - } - - break; - } - - case CalpontSystemCatalog::DATE: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from date."); - - case CalpontSystemCatalog::DATETIME: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); - - case CalpontSystemCatalog::TIMESTAMP: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from timestamp."); - - case CalpontSystemCatalog::TIME: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); - - case CalpontSystemCatalog::FLOAT: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float"); - - case CalpontSystemCatalog::UFLOAT: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float unsigned"); - - case CalpontSystemCatalog::DOUBLE: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from double"); - - case CalpontSystemCatalog::UDOUBLE: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from double unsigned"); - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - return fResult.decimalVal; - - default: - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion."); + idbassert(fResultType.scale == 0); + fResult.decimalVal = IDB_Decimal(fResult.intVal, 0, fResultType.precision, fResult.intVal); + break; } - return fResult.decimalVal; + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UTINYINT: + { + idbassert(fResultType.scale == 0); + fResult.decimalVal = IDB_Decimal((int64_t)fResult.uintVal, 0, fResultType.precision, fResult.uintVal); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double dlScaled = fResult.longDoubleVal; + + if (fResultType.scale > 0) + { + dlScaled = fResult.longDoubleVal * pow((double)10.0, fResultType.scale); + } + + if ((dlScaled > (long double)INT64_MAX) || (dlScaled < (long double)(INT64_MIN))) + { + datatypes::TFloat128 temp((float128_t)dlScaled); + fResult.decimalVal = + IDB_Decimal(0, fResultType.scale, fResultType.precision, static_cast(temp)); + } + else + { + int64_t val = (int64_t)lroundl(dlScaled); + fResult.decimalVal = IDB_Decimal(val, fResultType.scale, fResultType.precision, val); + } + + break; + } + + case CalpontSystemCatalog::DATE: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from date."); + + case CalpontSystemCatalog::DATETIME: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); + + case CalpontSystemCatalog::TIMESTAMP: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from timestamp."); + + case CalpontSystemCatalog::TIME: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); + + case CalpontSystemCatalog::FLOAT: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float"); + + case CalpontSystemCatalog::UFLOAT: + throw logging::InvalidConversionExcept( + "TreeNode::getDecimalVal: non-support conversion from float unsigned"); + + case CalpontSystemCatalog::DOUBLE: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from double"); + + case CalpontSystemCatalog::UDOUBLE: + throw logging::InvalidConversionExcept( + "TreeNode::getDecimalVal: non-support conversion from double unsigned"); + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: return fResult.decimalVal; + + default: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion."); + } + + return fResult.decimalVal; } inline int64_t TreeNode::getDatetimeIntVal() { - if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) - return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; - else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) + return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + { + dataconvert::Time tt; + int day = 0; + + void* ttp = static_cast(&tt); + + memcpy(ttp, &fResult.intVal, 8); + + // Note, this should probably be current date +/- time + if ((tt.hour > 23) && (!tt.is_neg)) { - dataconvert::Time tt; - int day = 0; - - void *ttp = static_cast(&tt); - - memcpy(ttp, &fResult.intVal, 8); - - // Note, this should probably be current date +/- time - if ((tt.hour > 23) && (!tt.is_neg)) - { - day = tt.hour / 24; - tt.hour = tt.hour % 24; - } - else if ((tt.hour < 0) || (tt.is_neg)) - { - tt.hour = 0; - } - - dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); - memcpy(&fResult.intVal, &dt, 8); - return fResult.intVal; + day = tt.hour / 24; + tt.hour = tt.hour % 24; } - else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) - //return (fResult.intVal & 0xFFFFFFFFFFF00000LL); - return (fResult.intVal); - else - return getIntVal(); + else if ((tt.hour < 0) || (tt.is_neg)) + { + tt.hour = 0; + } + + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + // return (fResult.intVal & 0xFFFFFFFFFFF00000LL); + return (fResult.intVal); + else + return getIntVal(); } inline int64_t TreeNode::getTimestampIntVal() { - if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - return fResult.intVal; - else - return getIntVal(); + if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + return fResult.intVal; + else + return getIntVal(); } inline int64_t TreeNode::getTimeIntVal() { - if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) - { - dataconvert::DateTime dt; + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + { + dataconvert::DateTime dt; - memcpy((int64_t*)(&dt), &fResult.intVal, 8); - dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond, false); - memcpy(&fResult.intVal, &tt, 8); - return fResult.intVal; - } - else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) - return (fResult.intVal); - else - return getIntVal(); + memcpy((int64_t*)(&dt), &fResult.intVal, 8); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond, false); + memcpy(&fResult.intVal, &tt, 8); + return fResult.intVal; + } + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + return (fResult.intVal); + else + return getIntVal(); } inline int32_t TreeNode::getDateIntVal() { - if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) - return (((int32_t)(fResult.intVal >> 32) & 0xFFFFFFC0) | 0x3E); - else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) - return ((fResult.intVal & 0xFFFFFFC0) | 0x3E); - else - return getIntVal(); + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + return (((int32_t)(fResult.intVal >> 32) & 0xFFFFFFC0) | 0x3E); + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) + return ((fResult.intVal & 0xFFFFFFC0) | 0x3E); + else + return getIntVal(); } typedef boost::shared_ptr STNP; /** -* Operations -*/ + * Operations + */ std::ostream& operator<<(std::ostream& output, const TreeNode& rhs); -} +} // namespace execplan #endif diff --git a/dbcon/execplan/treenodeimpl.cpp b/dbcon/execplan/treenodeimpl.cpp index 1b7418d3d..d91751704 100644 --- a/dbcon/execplan/treenodeimpl.cpp +++ b/dbcon/execplan/treenodeimpl.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: treenodeimpl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: treenodeimpl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include using namespace std; @@ -33,63 +33,65 @@ namespace execplan * Constructors/Destructors */ TreeNodeImpl::TreeNodeImpl() -{} +{ +} -TreeNodeImpl::TreeNodeImpl(const string& sql) : - fData(sql) -{} +TreeNodeImpl::TreeNodeImpl(const string& sql) : fData(sql) +{ +} TreeNodeImpl::~TreeNodeImpl() -{} +{ +} /** * The serialization interface */ void TreeNodeImpl::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::TREENODEIMPL; - b << fData; + b << (ObjectReader::id_t)ObjectReader::TREENODEIMPL; + b << fData; } void TreeNodeImpl::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::TREENODEIMPL); - b >> fData; + ObjectReader::checkType(b, ObjectReader::TREENODEIMPL); + b >> fData; } const string TreeNodeImpl::toString() const { - return string(">TreeNodeImpl<"); + return string(">TreeNodeImpl<"); } bool TreeNodeImpl::operator==(const TreeNodeImpl& t) const { - if (data() == t.data()) - return true; + if (data() == t.data()) + return true; - return false; + return false; } bool TreeNodeImpl::operator==(const TreeNode* t) const { - const TreeNodeImpl* tni; + const TreeNodeImpl* tni; - tni = dynamic_cast(t); + tni = dynamic_cast(t); - if (tni == NULL) - return false; + if (tni == NULL) + return false; - return *this == *tni; + return *this == *tni; } bool TreeNodeImpl::operator!=(const TreeNodeImpl& t) const { - return !(*this == t); + return !(*this == t); } bool TreeNodeImpl::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } /** @@ -97,8 +99,8 @@ bool TreeNodeImpl::operator!=(const TreeNode* t) const */ ostream& operator<<(ostream& output, const TreeNodeImpl& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/treenodeimpl.h b/dbcon/execplan/treenodeimpl.h index 3a5b9f048..dca8c4a28 100644 --- a/dbcon/execplan/treenodeimpl.h +++ b/dbcon/execplan/treenodeimpl.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: treenodeimpl.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: treenodeimpl.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef EXECPLAN_TREENODEIMPL_H @@ -46,92 +46,88 @@ namespace execplan */ class TreeNodeImpl : public TreeNode { + public: + /** + * Constructors + */ + TreeNodeImpl(); + TreeNodeImpl(const std::string& sql); + // not needed yet + // TreeNodeImpl(const TreeNodeImpl& rhs); -public: + /** + * Destructors + */ + virtual ~TreeNodeImpl(); + /** + * Accessor Methods + */ - /** - * Constructors - */ - TreeNodeImpl(); - TreeNodeImpl(const std::string& sql); - // not needed yet - //TreeNodeImpl(const TreeNodeImpl& rhs); + /** + * Operations + */ + virtual const std::string toString() const; - /** - * Destructors - */ - virtual ~TreeNodeImpl(); - /** - * Accessor Methods - */ + virtual const std::string data() const + { + return fData; + } + virtual void data(const std::string data) + { + fData = data; + } - /** - * Operations - */ - virtual const std::string toString() const; + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + inline virtual TreeNodeImpl* clone() const + { + return new TreeNodeImpl(*this); + } - virtual const std::string data() const - { - return fData; - } - virtual void data(const std::string data) - { - fData = data; - } + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - inline virtual TreeNodeImpl* clone() const - { - return new TreeNodeImpl (*this); - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; false otherwise + */ + bool operator==(const TreeNodeImpl& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; false otherwise - */ - bool operator==(const TreeNodeImpl& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; true otherwise + */ + bool operator!=(const TreeNodeImpl& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; true otherwise - */ - bool operator!=(const TreeNodeImpl& t) const; - -private: - //default okay - //TreeNodeImpl& operator=(const TreeNodeImpl& rhs); - - std::string fData; + private: + // default okay + // TreeNodeImpl& operator=(const TreeNodeImpl& rhs); + std::string fData; }; std::ostream& operator<<(std::ostream& os, const TreeNodeImpl& rhs); -} -#endif //EXECPLAN_TREENODEIMPL_H - +} // namespace execplan +#endif // EXECPLAN_TREENODEIMPL_H diff --git a/dbcon/execplan/udafcolumn.cpp b/dbcon/execplan/udafcolumn.cpp index 7aba4b3bb..925663d4c 100644 --- a/dbcon/execplan/udafcolumn.cpp +++ b/dbcon/execplan/udafcolumn.cpp @@ -41,22 +41,19 @@ using namespace joblist; namespace execplan { - /** * Constructors/Destructors */ -UDAFColumn::UDAFColumn(): - AggregateColumn() +UDAFColumn::UDAFColumn() : AggregateColumn() { } -UDAFColumn::UDAFColumn(const uint32_t sessionID): - AggregateColumn(sessionID) +UDAFColumn::UDAFColumn(const uint32_t sessionID) : AggregateColumn(sessionID) { } -UDAFColumn::UDAFColumn(const UDAFColumn& rhs, const uint32_t sessionID): - AggregateColumn(dynamic_cast(rhs), sessionID), context(rhs.context) +UDAFColumn::UDAFColumn(const UDAFColumn& rhs, const uint32_t sessionID) + : AggregateColumn(dynamic_cast(rhs), sessionID), context(rhs.context) { } @@ -70,69 +67,69 @@ UDAFColumn::~UDAFColumn() const string UDAFColumn::toString() const { - ostringstream output; - output << "UDAFColumn " << endl; - output << AggregateColumn::toString() << endl; - output << context.toString() << endl; - return output.str(); + ostringstream output; + output << "UDAFColumn " << endl; + output << AggregateColumn::toString() << endl; + output << context.toString() << endl; + return output.str(); } ostream& operator<<(ostream& output, const UDAFColumn& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } void UDAFColumn::serialize(messageqcpp::ByteStream& b) const { - b << (uint8_t) ObjectReader::UDAFCOLUMN; - AggregateColumn::serialize(b); - context.serialize(b); + b << (uint8_t)ObjectReader::UDAFCOLUMN; + AggregateColumn::serialize(b); + context.serialize(b); } void UDAFColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::UDAFCOLUMN); - AggregateColumn::unserialize(b); - context.unserialize(b); + ObjectReader::checkType(b, ObjectReader::UDAFCOLUMN); + AggregateColumn::unserialize(b); + context.unserialize(b); } bool UDAFColumn::operator==(const UDAFColumn& t) const { - const AggregateColumn* rc1, *rc2; + const AggregateColumn *rc1, *rc2; - rc1 = static_cast(this); - rc2 = static_cast(&t); + rc1 = static_cast(this); + rc2 = static_cast(&t); - if (*rc1 != *rc2) - return false; + if (*rc1 != *rc2) + return false; - if (context != t.context) - return false; + if (context != t.context) + return false; - return true; + return true; } bool UDAFColumn::operator==(const TreeNode* t) const { - const UDAFColumn* ac; + const UDAFColumn* ac; - ac = dynamic_cast(t); + ac = dynamic_cast(t); - if (ac == NULL) - return false; + if (ac == NULL) + return false; - return *this == *ac; + return *this == *ac; } bool UDAFColumn::operator!=(const UDAFColumn& t) const { - return !(*this == t); + return !(*this == t); } bool UDAFColumn::operator!=(const TreeNode* t) const { - return !(*this == t); + return !(*this == t); } -} // namespace execplan +} // namespace execplan diff --git a/dbcon/execplan/udafcolumn.h b/dbcon/execplan/udafcolumn.h index da9f7fff3..c9ac379f7 100644 --- a/dbcon/execplan/udafcolumn.h +++ b/dbcon/execplan/udafcolumn.h @@ -43,94 +43,91 @@ namespace execplan */ class UDAFColumn : public AggregateColumn { + public: + /** + * Constructors + */ + UDAFColumn(); -public: + UDAFColumn(const uint32_t sessionID); - /** - * Constructors - */ - UDAFColumn(); + UDAFColumn(const UDAFColumn& rhs, const uint32_t sessionID = 0); - UDAFColumn(const uint32_t sessionID); + /** + * Destructors + */ + virtual ~UDAFColumn(); - UDAFColumn(const UDAFColumn& rhs, const uint32_t sessionID = 0); + /** + * Overloaded stream operator + */ + virtual const std::string toString() const; - /** - * Destructors - */ - virtual ~UDAFColumn(); + /** return a copy of this pointer + * + * deep copy of this pointer and return the copy + */ + virtual UDAFColumn* clone() const + { + return new UDAFColumn(*this); + } - /** - * Overloaded stream operator - */ - virtual const std::string toString() const; + /** + * Accessors and Mutators + */ + mcsv1sdk::mcsv1Context& getContext() + { + return context; + } - /** return a copy of this pointer - * - * deep copy of this pointer and return the copy - */ - virtual UDAFColumn* clone() const - { - return new UDAFColumn(*this); - } + /** + * Serialize interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * Accessors and Mutators - */ - mcsv1sdk::mcsv1Context& getContext() - { - return context; - } + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; + * false otherwise + */ + virtual bool operator==(const TreeNode* t) const; - /** - * Serialize interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return true iff every member of t is a duplicate copy of every member of this; + * false otherwise + */ + using AggregateColumn::operator==; + virtual bool operator==(const UDAFColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; - * false otherwise - */ - virtual bool operator==(const TreeNode* t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; + * true otherwise + */ + virtual bool operator!=(const TreeNode* t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return true iff every member of t is a duplicate copy of every member of this; - * false otherwise - */ - using AggregateColumn::operator==; - virtual bool operator==(const UDAFColumn& t) const; + /** @brief Do a deep, strict (as opposed to semantic) equivalence test + * + * Do a deep, strict (as opposed to semantic) equivalence test. + * @return false iff every member of t is a duplicate copy of every member of this; + * true otherwise + */ + using AggregateColumn::operator!=; + virtual bool operator!=(const UDAFColumn& t) const; - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; - * true otherwise - */ - virtual bool operator!=(const TreeNode* t) const; - - /** @brief Do a deep, strict (as opposed to semantic) equivalence test - * - * Do a deep, strict (as opposed to semantic) equivalence test. - * @return false iff every member of t is a duplicate copy of every member of this; - * true otherwise - */ - using AggregateColumn::operator!=; - virtual bool operator!=(const UDAFColumn& t) const; - -private: - mcsv1sdk::mcsv1Context context; + private: + mcsv1sdk::mcsv1Context context; }; /** -* stream operator -*/ + * stream operator + */ std::ostream& operator<<(std::ostream& os, const UDAFColumn& rhs); -} -#endif //UDAFCOLUMN_H - +} // namespace execplan +#endif // UDAFCOLUMN_H diff --git a/dbcon/execplan/vendorexecutionplan.cpp b/dbcon/execplan/vendorexecutionplan.cpp index 1f9edb3bc..555ad12f0 100644 --- a/dbcon/execplan/vendorexecutionplan.cpp +++ b/dbcon/execplan/vendorexecutionplan.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: vendorexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: vendorexecutionplan.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "vendorexecutionplan.h" @@ -29,4 +29,3 @@ /** * Methods */ - diff --git a/dbcon/execplan/vendorexecutionplan.h b/dbcon/execplan/vendorexecutionplan.h index 662bd4a5a..9688696d3 100644 --- a/dbcon/execplan/vendorexecutionplan.h +++ b/dbcon/execplan/vendorexecutionplan.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: vendorexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: vendorexecutionplan.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef EXECPLAN_VENDOREXECUTIONPLAN_H @@ -35,14 +35,10 @@ namespace execplan */ class VendorExecutionPlan { - -public: - -private: - + public: + private: }; -} - -#endif //VENDOREXECUTIONPLAN_H +} // namespace execplan +#endif // VENDOREXECUTIONPLAN_H diff --git a/dbcon/execplan/wf_frame.h b/dbcon/execplan/wf_frame.h index 2472cbbe3..277a553f7 100644 --- a/dbcon/execplan/wf_frame.h +++ b/dbcon/execplan/wf_frame.h @@ -31,45 +31,52 @@ */ namespace execplan { - // This enum is made consistant with mysql Item_window_func enum WF_FRAME { - WF_PRECEDING = 0, - WF_FOLLOWING, - WF_UNBOUNDED_PRECEDING, - WF_UNBOUNDED_FOLLOWING, - WF_CURRENT_ROW, - WF_UNKNOWN + WF_PRECEDING = 0, + WF_FOLLOWING, + WF_UNBOUNDED_PRECEDING, + WF_UNBOUNDED_FOLLOWING, + WF_CURRENT_ROW, + WF_UNKNOWN }; struct WF_Boundary { - WF_Boundary() {} - WF_Boundary(WF_FRAME frame): fFrame(frame) {} - ~WF_Boundary() {} - const std::string toString() const; - void serialize(messageqcpp::ByteStream&) const; - void unserialize(messageqcpp::ByteStream&); - SRCP fVal; /// has to evaluate to unsigned value - SRCP fBound; /// order by col +, -, date_add or date_sub for RANGE window - enum WF_FRAME fFrame; + WF_Boundary() + { + } + WF_Boundary(WF_FRAME frame) : fFrame(frame) + { + } + ~WF_Boundary() + { + } + const std::string toString() const; + void serialize(messageqcpp::ByteStream&) const; + void unserialize(messageqcpp::ByteStream&); + SRCP fVal; /// has to evaluate to unsigned value + SRCP fBound; /// order by col +, -, date_add or date_sub for RANGE window + enum WF_FRAME fFrame; }; struct WF_Frame { - WF_Frame(): fIsRange(true) - { - fStart.fFrame = WF_UNBOUNDED_PRECEDING; - fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - } - ~WF_Frame() {} - const std::string toString() const; - void serialize(messageqcpp::ByteStream&) const; - void unserialize(messageqcpp::ByteStream&); - WF_Boundary fStart; - WF_Boundary fEnd; - bool fIsRange; /// RANGE or ROWS + WF_Frame() : fIsRange(true) + { + fStart.fFrame = WF_UNBOUNDED_PRECEDING; + fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + } + ~WF_Frame() + { + } + const std::string toString() const; + void serialize(messageqcpp::ByteStream&) const; + void unserialize(messageqcpp::ByteStream&); + WF_Boundary fStart; + WF_Boundary fEnd; + bool fIsRange; /// RANGE or ROWS }; /** @@ -77,15 +84,19 @@ struct WF_Frame */ struct WF_OrderBy { - WF_OrderBy() {} - WF_OrderBy(std::vector orders): fOrders(orders) {} - ~WF_OrderBy() {}; - const std::string toString() const; - void serialize(messageqcpp::ByteStream&) const; - void unserialize(messageqcpp::ByteStream&); - std::vector fOrders; - WF_Frame fFrame; + WF_OrderBy() + { + } + WF_OrderBy(std::vector orders) : fOrders(orders) + { + } + ~WF_OrderBy(){}; + const std::string toString() const; + void serialize(messageqcpp::ByteStream&) const; + void unserialize(messageqcpp::ByteStream&); + std::vector fOrders; + WF_Frame fFrame; }; -} +} // namespace execplan #endif diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index d370c1a93..00b2e9ac5 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: windowfunctioncolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: windowfunctioncolumn.cpp 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ #include #include @@ -58,32 +58,31 @@ using namespace joblist; namespace execplan { - void getWindowFunctionCols(execplan::ParseTree* n, void* obj) { - vector* list = reinterpret_cast< vector*>(obj); - TreeNode* tn = n->data(); - WindowFunctionColumn* afc = dynamic_cast(tn); - ArithmeticColumn* ac = dynamic_cast(tn); - FunctionColumn* fc = dynamic_cast(tn); - SimpleFilter* sf = dynamic_cast(tn); + vector* list = reinterpret_cast*>(obj); + TreeNode* tn = n->data(); + WindowFunctionColumn* afc = dynamic_cast(tn); + ArithmeticColumn* ac = dynamic_cast(tn); + FunctionColumn* fc = dynamic_cast(tn); + SimpleFilter* sf = dynamic_cast(tn); - if (afc) - list->push_back(afc); - else if (ac) - { - if (ac->hasWindowFunc()) // adds window functions to list - list->insert(list->end(), ac->windowfunctionColumnList().begin(), ac->windowfunctionColumnList().end()); - } - else if (fc) - { - if (fc->hasWindowFunc()) - list->insert(list->end(), fc->windowfunctionColumnList().begin(), fc->windowfunctionColumnList().end()); - } - else if (sf) - { - list->insert(list->end(), sf->windowfunctionColumnList().begin(), sf->windowfunctionColumnList().end()); - } + if (afc) + list->push_back(afc); + else if (ac) + { + if (ac->hasWindowFunc()) // adds window functions to list + list->insert(list->end(), ac->windowfunctionColumnList().begin(), ac->windowfunctionColumnList().end()); + } + else if (fc) + { + if (fc->hasWindowFunc()) + list->insert(list->end(), fc->windowfunctionColumnList().begin(), fc->windowfunctionColumnList().end()); + } + else if (sf) + { + list->insert(list->end(), sf->windowfunctionColumnList().begin(), sf->windowfunctionColumnList().end()); + } } /** @@ -91,69 +90,58 @@ void getWindowFunctionCols(execplan::ParseTree* n, void* obj) */ const std::string WF_Boundary::toString() const { - ostringstream output; + ostringstream output; - if (fVal) - { - output << "val: "; - output << fVal->toString() << endl; - } + if (fVal) + { + output << "val: "; + output << fVal->toString() << endl; + } - if (fBound) - { - output << "bound exp: "; - output << fBound->toString() << endl; - } + if (fBound) + { + output << "bound exp: "; + output << fBound->toString() << endl; + } - switch (fFrame) - { - case WF_PRECEDING: - output << "PRECEDING"; - break; + switch (fFrame) + { + case WF_PRECEDING: output << "PRECEDING"; break; - case WF_FOLLOWING: - output << "FOLLOWING"; - break; + case WF_FOLLOWING: output << "FOLLOWING"; break; - case WF_UNBOUNDED_PRECEDING: - output << "UNBOUNDED PRECEDING"; - break; + case WF_UNBOUNDED_PRECEDING: output << "UNBOUNDED PRECEDING"; break; - case WF_UNBOUNDED_FOLLOWING: - output << "UNBOUNDED FOLLOWING"; - break; + case WF_UNBOUNDED_FOLLOWING: output << "UNBOUNDED FOLLOWING"; break; - case WF_CURRENT_ROW: - output << "CURRENT ROW"; - break; + case WF_CURRENT_ROW: output << "CURRENT ROW"; break; - default: - output << "UNKNOWN"; - } + default: output << "UNKNOWN"; + } - return output.str(); + return output.str(); } void WF_Boundary::serialize(messageqcpp::ByteStream& b) const { - b << (uint8_t)fFrame; + b << (uint8_t)fFrame; - if (fVal) - fVal->serialize(b); - else - b << (uint8_t) ObjectReader::NULL_CLASS; + if (fVal) + fVal->serialize(b); + else + b << (uint8_t)ObjectReader::NULL_CLASS; - if (fBound) - fBound->serialize(b); - else - b << (uint8_t) ObjectReader::NULL_CLASS; + if (fBound) + fBound->serialize(b); + else + b << (uint8_t)ObjectReader::NULL_CLASS; } void WF_Boundary::unserialize(messageqcpp::ByteStream& b) { - b >> (uint8_t&)fFrame; - fVal.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fBound.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + b >> (uint8_t&)fFrame; + fVal.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fBound.reset(dynamic_cast(ObjectReader::createTreeNode(b))); } /** @@ -161,27 +149,27 @@ void WF_Boundary::unserialize(messageqcpp::ByteStream& b) */ const string WF_Frame::toString() const { - ostringstream output; - output << "WindowFrame:" << endl; - output << "Start:" << endl; - output << fStart.toString() << endl; - output << "End:" << endl; - output << fEnd.toString() << endl; - return output.str(); + ostringstream output; + output << "WindowFrame:" << endl; + output << "Start:" << endl; + output << fStart.toString() << endl; + output << "End:" << endl; + output << fEnd.toString() << endl; + return output.str(); } void WF_Frame::serialize(messageqcpp::ByteStream& b) const { - fStart.serialize(b); - fEnd.serialize(b); - b << (uint8_t)fIsRange; + fStart.serialize(b); + fEnd.serialize(b); + b << (uint8_t)fIsRange; } void WF_Frame::unserialize(messageqcpp::ByteStream& b) { - fStart.unserialize(b); - fEnd.unserialize(b); - b >> (uint8_t&)fIsRange; + fStart.unserialize(b); + fEnd.unserialize(b); + b >> (uint8_t&)fIsRange; } /** @@ -190,568 +178,547 @@ void WF_Frame::unserialize(messageqcpp::ByteStream& b) const string WF_OrderBy::toString() const { - ostringstream output; - output << "order by: " << endl; + ostringstream output; + output << "order by: " << endl; - for (uint32_t i = 0; i < fOrders.size(); i++) - output << fOrders[i]->toString() << endl; + for (uint32_t i = 0; i < fOrders.size(); i++) + output << fOrders[i]->toString() << endl; - output << fFrame.toString(); - return output.str(); + output << fFrame.toString(); + return output.str(); } void WF_OrderBy::serialize(messageqcpp::ByteStream& b) const { - b << (uint32_t)fOrders.size(); + b << (uint32_t)fOrders.size(); - for (uint32_t i = 0; i < fOrders.size(); i++) - fOrders[i]->serialize(b); + for (uint32_t i = 0; i < fOrders.size(); i++) + fOrders[i]->serialize(b); - fFrame.serialize(b); + fFrame.serialize(b); } void WF_OrderBy::unserialize(messageqcpp::ByteStream& b) { - uint32_t size; - b >> (uint32_t&)size; - SRCP srcp; + uint32_t size; + b >> (uint32_t&)size; + SRCP srcp; - for (uint32_t i = 0; i < size; i++) - { - srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fOrders.push_back(srcp); - } + for (uint32_t i = 0; i < size; i++) + { + srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fOrders.push_back(srcp); + } - fFrame.unserialize(b); + fFrame.unserialize(b); } /** -* WindowFunctionColumn class definition -*/ + * WindowFunctionColumn class definition + */ WindowFunctionColumn::WindowFunctionColumn() -{} +{ +} -WindowFunctionColumn::WindowFunctionColumn(const string& functionName, const uint32_t sessionID): - ReturnedColumn(sessionID), - fFunctionName(functionName) -{} +WindowFunctionColumn::WindowFunctionColumn(const string& functionName, const uint32_t sessionID) + : ReturnedColumn(sessionID), fFunctionName(functionName) +{ +} -WindowFunctionColumn::WindowFunctionColumn( const WindowFunctionColumn& rhs, const uint32_t sessionID): - ReturnedColumn(rhs, sessionID), - fFunctionName(rhs.functionName()), - fFunctionParms(rhs.functionParms()), - fPartitions (rhs.partitions()), - fOrderBy (rhs.orderBy()), - udafContext(rhs.getUDAFContext()), - fTimeZone(rhs.timeZone()) -{} +WindowFunctionColumn::WindowFunctionColumn(const WindowFunctionColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID) + , fFunctionName(rhs.functionName()) + , fFunctionParms(rhs.functionParms()) + , fPartitions(rhs.partitions()) + , fOrderBy(rhs.orderBy()) + , udafContext(rhs.getUDAFContext()) + , fTimeZone(rhs.timeZone()) +{ +} const string WindowFunctionColumn::toString() const { - ostringstream output; - output << "WindowFunctionColumn: " << fFunctionName; + ostringstream output; + output << "WindowFunctionColumn: " << fFunctionName; - if (distinct()) - output << " DISTINCT"; + if (distinct()) + output << " DISTINCT"; - output << endl; - output << "expressionId=" << fExpressionId << endl; - output << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; - output << "operationType=" << colDataTypeToString(fOperationType.colDataType) << endl; - output << "function parm: " << endl; + output << endl; + output << "expressionId=" << fExpressionId << endl; + output << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth + << endl; + output << "operationType=" << colDataTypeToString(fOperationType.colDataType) << endl; + output << "function parm: " << endl; - for (uint32_t i = 0; i < fFunctionParms.size(); i++) - output << fFunctionParms[i]->toString() << endl; + for (uint32_t i = 0; i < fFunctionParms.size(); i++) + output << fFunctionParms[i]->toString() << endl; - output << "partition by: " << endl; + output << "partition by: " << endl; - for (uint32_t i = 0; i < fPartitions.size(); i++) - output << fPartitions[i]->toString() << endl; + for (uint32_t i = 0; i < fPartitions.size(); i++) + output << fPartitions[i]->toString() << endl; - output << fOrderBy.toString() << endl; - output << "getColumnList():" << endl; - vector colList = getColumnList(); + output << fOrderBy.toString() << endl; + output << "getColumnList():" << endl; + vector colList = getColumnList(); - for (uint32_t i = 0; i < colList.size(); i++) - output << colList[i]->toString() << endl; + for (uint32_t i = 0; i < colList.size(); i++) + output << colList[i]->toString() << endl; - return output.str(); + return output.str(); } void WindowFunctionColumn::serialize(messageqcpp::ByteStream& b) const { - b << (ObjectReader::id_t) ObjectReader::WINDOWFUNCTIONCOLUMN; - ReturnedColumn::serialize(b); - b << fFunctionName; + b << (ObjectReader::id_t)ObjectReader::WINDOWFUNCTIONCOLUMN; + ReturnedColumn::serialize(b); + b << fFunctionName; - b << (uint32_t)fFunctionParms.size(); + b << (uint32_t)fFunctionParms.size(); - for (uint32_t i = 0; i < fFunctionParms.size(); i++) - fFunctionParms[i]->serialize(b); + for (uint32_t i = 0; i < fFunctionParms.size(); i++) + fFunctionParms[i]->serialize(b); - b << (uint32_t)fPartitions.size(); + b << (uint32_t)fPartitions.size(); - for (uint32_t i = 0; i < fPartitions.size(); i++) - fPartitions[i]->serialize(b); + for (uint32_t i = 0; i < fPartitions.size(); i++) + fPartitions[i]->serialize(b); - fOrderBy.serialize(b); - udafContext.serialize(b); - b << fTimeZone; + fOrderBy.serialize(b); + udafContext.serialize(b); + b << fTimeZone; } void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) { - ObjectReader::checkType(b, ObjectReader::WINDOWFUNCTIONCOLUMN); - ReturnedColumn::unserialize(b); - uint32_t size; - SRCP srcp; + ObjectReader::checkType(b, ObjectReader::WINDOWFUNCTIONCOLUMN); + ReturnedColumn::unserialize(b); + uint32_t size; + SRCP srcp; - fFunctionParms.clear(); - fPartitions.clear(); + fFunctionParms.clear(); + fPartitions.clear(); - b >> fFunctionName; - b >> (uint32_t&)size; + b >> fFunctionName; + b >> (uint32_t&)size; - for (uint32_t i = 0; i < size; i++) - { - srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fFunctionParms.push_back(srcp); - } + for (uint32_t i = 0; i < size; i++) + { + srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fFunctionParms.push_back(srcp); + } - b >> (uint32_t&)size; + b >> (uint32_t&)size; - for (uint32_t i = 0; i < size; i++) - { - srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); - fPartitions.push_back(srcp); - } + for (uint32_t i = 0; i < size; i++) + { + srcp.reset(dynamic_cast(ObjectReader::createTreeNode(b))); + fPartitions.push_back(srcp); + } - fOrderBy.unserialize(b); - udafContext.unserialize(b); - b >> fTimeZone; + fOrderBy.unserialize(b); + udafContext.unserialize(b); + b >> fTimeZone; } void WindowFunctionColumn::addToPartition(vector& groupByList) { - fPartitions.insert(fPartitions.end(), groupByList.begin(), groupByList.end()); + fPartitions.insert(fPartitions.end(), groupByList.begin(), groupByList.end()); } vector WindowFunctionColumn::getColumnList() const { - vector columnList; - columnList.insert(columnList.end(), fFunctionParms.begin(), fFunctionParms.end()); - columnList.insert(columnList.end(), fPartitions.begin(), fPartitions.end()); - columnList.insert(columnList.end(), fOrderBy.fOrders.begin(), fOrderBy.fOrders.end()); + vector columnList; + columnList.insert(columnList.end(), fFunctionParms.begin(), fFunctionParms.end()); + columnList.insert(columnList.end(), fPartitions.begin(), fPartitions.end()); + columnList.insert(columnList.end(), fOrderBy.fOrders.begin(), fOrderBy.fOrders.end()); - if (fOrderBy.fFrame.fStart.fVal) - columnList.push_back(fOrderBy.fFrame.fStart.fVal); + if (fOrderBy.fFrame.fStart.fVal) + columnList.push_back(fOrderBy.fFrame.fStart.fVal); - if (fOrderBy.fFrame.fStart.fBound) - columnList.push_back(fOrderBy.fFrame.fStart.fBound); + if (fOrderBy.fFrame.fStart.fBound) + columnList.push_back(fOrderBy.fFrame.fStart.fBound); - if (fOrderBy.fFrame.fEnd.fVal) - columnList.push_back(fOrderBy.fFrame.fEnd.fVal); + if (fOrderBy.fFrame.fEnd.fVal) + columnList.push_back(fOrderBy.fFrame.fEnd.fVal); - if (fOrderBy.fFrame.fEnd.fBound) - columnList.push_back(fOrderBy.fFrame.fEnd.fBound); + if (fOrderBy.fFrame.fEnd.fBound) + columnList.push_back(fOrderBy.fFrame.fEnd.fBound); - return columnList; + return columnList; } bool WindowFunctionColumn::hasWindowFunc() { - fWindowFunctionColumnList.push_back(this); - return true; + fWindowFunctionColumnList.push_back(this); + return true; } void WindowFunctionColumn::adjustResultType() { - if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL || - fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) - && - !boost::iequals(fFunctionName, "COUNT") && - !boost::iequals(fFunctionName, "COUNT(*)") && - !boost::iequals(fFunctionName, "ROW_NUMBER") && - !boost::iequals(fFunctionName, "RANK") && - !boost::iequals(fFunctionName, "PERCENT_RANK") && - !boost::iequals(fFunctionName, "DENSE_RANK") && - !boost::iequals(fFunctionName, "CUME_DIST") && - !boost::iequals(fFunctionName, "NTILE") && - !boost::iequals(fFunctionName, "PERCENTILE") && - !fFunctionParms.empty() && - fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE) - fResultType = fFunctionParms[0]->resultType(); + if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL || + fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) && + !boost::iequals(fFunctionName, "COUNT") && !boost::iequals(fFunctionName, "COUNT(*)") && + !boost::iequals(fFunctionName, "ROW_NUMBER") && !boost::iequals(fFunctionName, "RANK") && + !boost::iequals(fFunctionName, "PERCENT_RANK") && !boost::iequals(fFunctionName, "DENSE_RANK") && + !boost::iequals(fFunctionName, "CUME_DIST") && !boost::iequals(fFunctionName, "NTILE") && + !boost::iequals(fFunctionName, "PERCENTILE") && !fFunctionParms.empty() && + fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE) + fResultType = fFunctionParms[0]->resultType(); - if ((boost::iequals(fFunctionName, "LEAD") || - boost::iequals(fFunctionName, "LAG") || - boost::iequals(fFunctionName, "MIN") || - boost::iequals(fFunctionName, "MAX") || - boost::iequals(fFunctionName, "FIRST_VALUE") || - boost::iequals(fFunctionName, "LAST_VALUE") || - boost::iequals(fFunctionName, "NTH_VALUE")) && - !fFunctionParms.empty()) - fResultType = fFunctionParms[0]->resultType(); + if ((boost::iequals(fFunctionName, "LEAD") || boost::iequals(fFunctionName, "LAG") || + boost::iequals(fFunctionName, "MIN") || boost::iequals(fFunctionName, "MAX") || + boost::iequals(fFunctionName, "FIRST_VALUE") || boost::iequals(fFunctionName, "LAST_VALUE") || + boost::iequals(fFunctionName, "NTH_VALUE")) && + !fFunctionParms.empty()) + fResultType = fFunctionParms[0]->resultType(); - if (boost::iequals(fFunctionName, "SUM") || - boost::iequals(fFunctionName, "AVG") || - boost::iequals(fFunctionName, "AVG_DISTINCT") || - boost::iequals(fFunctionName, "PERCENTILE")) + if (boost::iequals(fFunctionName, "SUM") || boost::iequals(fFunctionName, "AVG") || + boost::iequals(fFunctionName, "AVG_DISTINCT") || boost::iequals(fFunctionName, "PERCENTILE")) + { + if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL || + fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) { - if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL || - fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) - { - fResultType.colWidth = datatypes::MAXDECIMALWIDTH; - } - else - { - fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; - fResultType.colWidth = sizeof(long double); - fResultType.precision = -1; - } + fResultType.colWidth = datatypes::MAXDECIMALWIDTH; } + else + { + fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; + fResultType.colWidth = sizeof(long double); + fResultType.precision = -1; + } + } } void WindowFunctionColumn::evaluate(Row& row, bool& isNull) { - switch (fResultType.colDataType) + switch (fResultType.colDataType) + { + case CalpontSystemCatalog::DATE: { - case CalpontSystemCatalog::DATE: - { - if (row.equals<4>(DATENULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<4>(fInputIndex); + if (row.equals<4>(DATENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<4>(fInputIndex); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if (row.equals<8>(DATETIMENULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<8>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - if (row.equals<8>(TIMESTAMPNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<8>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::TIME: - { - if (row.equals<8>(TIMENULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<8>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::STRINT: - { - switch (row.getColumnWidth(fInputIndex)) - { - case 1: - if (row.equals<1>(CHAR1NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<1>(fInputIndex); - - break; - - case 2: - if (row.equals<2>(CHAR2NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<2>(fInputIndex); - - break; - - case 3: - case 4: - if (row.equals<4>(CHAR4NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<4>(fInputIndex); - - break; - - case 5: - case 6: - case 7: - case 8: - if (row.equals<8>(CHAR8NULL, fInputIndex)) - isNull = true; - else - fResult.origIntVal = row.getUintField<8>(fInputIndex); - - break; - - // TODO MCOL-641 - case 16: - //fallthrough - default: - { - const auto str = row.getConstString(fInputIndex); - if (str.eq(utils::ConstString(CPNULLSTRMARK))) - isNull = true; - else - fResult.strVal = str.toString(); - - // stringColVal is padded with '\0' to colWidth so can't use str.length() - if (strlen(fResult.strVal.c_str()) == 0) - isNull = true; - - break; - } - } - - if (fResultType.colDataType == CalpontSystemCatalog::STRINT) - fResult.intVal = uint64ToStr(fResult.origIntVal); - else - fResult.intVal = atoll((char*)&fResult.origIntVal); - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - if (row.equals<8>(BIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<8>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - if (row.equals<8>(UBIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<8>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - { - if (row.equals<4>(INTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<4>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - if (row.equals<4>(UINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<4>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - if (row.equals<2>(SMALLINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<2>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - if (row.equals<2>(USMALLINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<2>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::TINYINT: - { - if (row.equals<1>(TINYINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getIntField<1>(fInputIndex); - - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - if (row.equals<1>(UTINYINTNULL, fInputIndex)) - isNull = true; - else - fResult.uintVal = row.getUintField<1>(fInputIndex); - - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - if (row.equals<4>(FLOATNULL, fInputIndex)) - isNull = true; - else - fResult.floatVal = row.getFloatField(fInputIndex); - - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - if (row.equals<8>(DOUBLENULL, fInputIndex)) - isNull = true; - else - fResult.doubleVal = row.getDoubleField(fInputIndex); - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - if (row.equals(LONGDOUBLENULL, fInputIndex)) - isNull = true; - else - fResult.longDoubleVal = row.getLongDoubleField(fInputIndex); - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - switch (fResultType.colWidth) - { - case 1: - { - if (row.equals<1>(TINYINTNULL, fInputIndex)) - isNull = true; - else - { - fResult.decimalVal = IDB_Decimal( - row.getIntField<1>(fInputIndex), - fResultType.scale, - fResultType.precision); - } - - break; - } - - case 2: - { - if (row.equals<2>(SMALLINTNULL, fInputIndex)) - isNull = true; - else - { - fResult.decimalVal = IDB_Decimal( - row.getIntField<2>(fInputIndex), - fResultType.scale, - fResultType.precision); - } - - break; - } - - case 4: - { - if (row.equals<4>(INTNULL, fInputIndex)) - isNull = true; - else - { - fResult.decimalVal = IDB_Decimal( - row.getIntField<4>(fInputIndex), - fResultType.scale, - fResultType.precision); - } - - break; - } - - case 8: - { - if (row.equals<8>(BIGINTNULL, fInputIndex)) - isNull = true; - else - { - fResult.decimalVal = IDB_Decimal( - row.getIntField<8>(fInputIndex), - fResultType.scale, - fResultType.precision); - } - - break; - } - - case 16: - { - datatypes::TSInt128 val = row.getTSInt128Field(fInputIndex); - - if (val.isNull()) - { - isNull = true; - } - else - { - fResult.decimalVal = IDB_Decimal(val, - fResultType.scale, - fResultType.precision); - } - - break; - } - default: - // Should log error - break; - } - - break; - } - - case CalpontSystemCatalog::VARBINARY: - isNull = true; - break; - - default: // treat as int64 - { - if (row.equals<8>(BIGINTNULL, fInputIndex)) - isNull = true; - else - fResult.intVal = row.getUintField<8>(fInputIndex); - - break; - } + break; } -} + case CalpontSystemCatalog::DATETIME: + { + if (row.equals<8>(DATETIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + if (row.equals<8>(TIMESTAMPNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::TIME: + { + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::STRINT: + { + switch (row.getColumnWidth(fInputIndex)) + { + case 1: + if (row.equals<1>(CHAR1NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<1>(fInputIndex); + + break; + + case 2: + if (row.equals<2>(CHAR2NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<2>(fInputIndex); + + break; + + case 3: + case 4: + if (row.equals<4>(CHAR4NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<4>(fInputIndex); + + break; + + case 5: + case 6: + case 7: + case 8: + if (row.equals<8>(CHAR8NULL, fInputIndex)) + isNull = true; + else + fResult.origIntVal = row.getUintField<8>(fInputIndex); + + break; + + // TODO MCOL-641 + case 16: + // fallthrough + default: + { + const auto str = row.getConstString(fInputIndex); + if (str.eq(utils::ConstString(CPNULLSTRMARK))) + isNull = true; + else + fResult.strVal = str.toString(); + + // stringColVal is padded with '\0' to colWidth so can't use str.length() + if (strlen(fResult.strVal.c_str()) == 0) + isNull = true; + + break; + } + } + + if (fResultType.colDataType == CalpontSystemCatalog::STRINT) + fResult.intVal = uint64ToStr(fResult.origIntVal); + else + fResult.intVal = atoll((char*)&fResult.origIntVal); + + break; + } + + case CalpontSystemCatalog::BIGINT: + { + if (row.equals<8>(BIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::UBIGINT: + { + if (row.equals<8>(UBIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<8>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + { + if (row.equals<4>(INTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<4>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + { + if (row.equals<4>(UINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<4>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::SMALLINT: + { + if (row.equals<2>(SMALLINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<2>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::USMALLINT: + { + if (row.equals<2>(USMALLINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<2>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::TINYINT: + { + if (row.equals<1>(TINYINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<1>(fInputIndex); + + break; + } + + case CalpontSystemCatalog::UTINYINT: + { + if (row.equals<1>(UTINYINTNULL, fInputIndex)) + isNull = true; + else + fResult.uintVal = row.getUintField<1>(fInputIndex); + + break; + } + + // In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + if (row.equals<4>(FLOATNULL, fInputIndex)) + isNull = true; + else + fResult.floatVal = row.getFloatField(fInputIndex); + + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + if (row.equals<8>(DOUBLENULL, fInputIndex)) + isNull = true; + else + fResult.doubleVal = row.getDoubleField(fInputIndex); + + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + if (row.equals(LONGDOUBLENULL, fInputIndex)) + isNull = true; + else + fResult.longDoubleVal = row.getLongDoubleField(fInputIndex); + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + switch (fResultType.colWidth) + { + case 1: + { + if (row.equals<1>(TINYINTNULL, fInputIndex)) + isNull = true; + else + { + fResult.decimalVal = + IDB_Decimal(row.getIntField<1>(fInputIndex), fResultType.scale, fResultType.precision); + } + + break; + } + + case 2: + { + if (row.equals<2>(SMALLINTNULL, fInputIndex)) + isNull = true; + else + { + fResult.decimalVal = + IDB_Decimal(row.getIntField<2>(fInputIndex), fResultType.scale, fResultType.precision); + } + + break; + } + + case 4: + { + if (row.equals<4>(INTNULL, fInputIndex)) + isNull = true; + else + { + fResult.decimalVal = + IDB_Decimal(row.getIntField<4>(fInputIndex), fResultType.scale, fResultType.precision); + } + + break; + } + + case 8: + { + if (row.equals<8>(BIGINTNULL, fInputIndex)) + isNull = true; + else + { + fResult.decimalVal = + IDB_Decimal(row.getIntField<8>(fInputIndex), fResultType.scale, fResultType.precision); + } + + break; + } + + case 16: + { + datatypes::TSInt128 val = row.getTSInt128Field(fInputIndex); + + if (val.isNull()) + { + isNull = true; + } + else + { + fResult.decimalVal = IDB_Decimal(val, fResultType.scale, fResultType.precision); + } + + break; + } + default: + // Should log error + break; + } + + break; + } + + case CalpontSystemCatalog::VARBINARY: isNull = true; break; + + default: // treat as int64 + { + if (row.equals<8>(BIGINTNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); + + break; + } + } +} ostream& operator<<(ostream& output, const WindowFunctionColumn& rhs) { - output << rhs.toString(); - return output; + output << rhs.toString(); + return output; } -} //namespace +} // namespace execplan diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 36ef2099b..f1a40db09 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: windowfunctioncolumn.h 9679 2013-07-11 22:32:03Z zzhu $ -* -* -***********************************************************************/ + * $Id: windowfunctioncolumn.h 9679 2013-07-11 22:32:03Z zzhu $ + * + * + ***********************************************************************/ /** @file */ @@ -45,7 +45,6 @@ class ByteStream; */ namespace execplan { - /** * @brief A class to represent a functional column * @@ -54,206 +53,208 @@ namespace execplan */ class WindowFunctionColumn : public ReturnedColumn { + public: + WindowFunctionColumn(); + WindowFunctionColumn(const std::string& functionName, const uint32_t sessionID = 0); + WindowFunctionColumn(const WindowFunctionColumn& rhs, const uint32_t sessionID = 0); + virtual ~WindowFunctionColumn() + { + } -public: - WindowFunctionColumn(); - WindowFunctionColumn(const std::string& functionName, const uint32_t sessionID = 0); - WindowFunctionColumn(const WindowFunctionColumn& rhs, const uint32_t sessionID = 0); - virtual ~WindowFunctionColumn() {} + /** get function name */ + inline const std::string& functionName() const + { + return fFunctionName; + } - /** get function name */ - inline const std::string& functionName() const - { - return fFunctionName; - } + /** set function name */ + inline void functionName(const std::string functionName) + { + fFunctionName = functionName; + } - /** set function name */ - inline void functionName(const std::string functionName) - { - fFunctionName = functionName; - } + /** get function parameters */ + inline const std::vector& functionParms() const + { + return fFunctionParms; + } - /** get function parameters */ - inline const std::vector& functionParms() const - { - return fFunctionParms; - } + /** set function parameters*/ + inline void functionParms(const std::vector& functionParms) + { + fFunctionParms = functionParms; + } - /** set function parameters*/ - inline void functionParms(const std::vector& functionParms) - { - fFunctionParms = functionParms; - } + /** get partition columns */ + inline const std::vector& partitions() const + { + return fPartitions; + } - /** get partition columns */ - inline const std::vector& partitions() const - { - return fPartitions; - } + /** set partition columns */ + inline void partitions(const std::vector& partitions) + { + fPartitions = partitions; + } - /** set partition columns */ - inline void partitions(const std::vector& partitions) - { - fPartitions = partitions; - } + /** get order by clause */ + inline const WF_OrderBy& orderBy() const + { + return fOrderBy; + } - /** get order by clause */ - inline const WF_OrderBy& orderBy() const - { - return fOrderBy; - } + /** set order by clause */ + inline void orderBy(const WF_OrderBy& orderBy) + { + fOrderBy = orderBy; + } - /** set order by clause */ - inline void orderBy(const WF_OrderBy& orderBy) - { - fOrderBy = orderBy; - } + /** make a clone of this window function */ + inline virtual WindowFunctionColumn* clone() const + { + return new WindowFunctionColumn(*this); + } - /** make a clone of this window function */ - inline virtual WindowFunctionColumn* clone() const - { - return new WindowFunctionColumn (*this); - } + std::vector getColumnList() const; - std::vector getColumnList() const; + /** output the function for debug purpose */ + const std::string toString() const; - /** output the function for debug purpose */ - const std::string toString() const; + /** + * The serialization interface + */ + virtual void serialize(messageqcpp::ByteStream&) const; + virtual void unserialize(messageqcpp::ByteStream&); - /** - * The serialization interface - */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + // util function for connector to use. + void addToPartition(std::vector& groupByList); - // util function for connector to use. - void addToPartition(std::vector& groupByList); + using ReturnedColumn::hasAggregate; + virtual bool hasAggregate() + { + return false; + } + virtual bool hasWindowFunc(); + void adjustResultType(); - using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() - { - return false; - } - virtual bool hasWindowFunc(); - void adjustResultType(); + // UDAnF support + mcsv1sdk::mcsv1Context& getUDAFContext() + { + return udafContext; + } + const mcsv1sdk::mcsv1Context& getUDAFContext() const + { + return udafContext; + } - // UDAnF support - mcsv1sdk::mcsv1Context& getUDAFContext() - { - return udafContext; - } - const mcsv1sdk::mcsv1Context& getUDAFContext() const - { - return udafContext; - } + inline const std::string timeZone() const + { + return fTimeZone; + } - inline const std::string timeZone () const - { - return fTimeZone; - } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } - inline void timeZone (const std::string& timeZone) - { - fTimeZone = timeZone; - } + private: + /** + * Fields + */ + std::string fFunctionName; /// function name + std::vector fFunctionParms; /// function arguments + std::vector fPartitions; /// partition by clause + WF_OrderBy fOrderBy; /// order by clause -private: - /** - * Fields - */ - std::string fFunctionName; /// function name - std::vector fFunctionParms; /// function arguments - std::vector fPartitions; /// partition by clause - WF_OrderBy fOrderBy; /// order by clause + // not support for window functions for now. + virtual bool operator==(const TreeNode* t) const + { + return false; + } + bool operator==(const WindowFunctionColumn& t) const; + virtual bool operator!=(const TreeNode* t) const + { + return false; + } + bool operator!=(const WindowFunctionColumn& t) const; - // not support for window functions for now. - virtual bool operator==(const TreeNode* t) const - { - return false; - } - bool operator==(const WindowFunctionColumn& t) const; - virtual bool operator!=(const TreeNode* t) const - { - return false; - } - bool operator!=(const WindowFunctionColumn& t) const; + // UDAnF support + mcsv1sdk::mcsv1Context udafContext; - // UDAnF support - mcsv1sdk::mcsv1Context udafContext; + std::string fTimeZone; + /*********************************************************** + * F&E framework * + ***********************************************************/ + public: + virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getStrVal(fTimeZone); + } - std::string fTimeZone; - /*********************************************************** - * F&E framework * - ***********************************************************/ -public: - virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getStrVal(fTimeZone); - } + virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getIntVal(); + } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getIntVal(); - } + virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getUintVal(); + } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getUintVal(); - } + virtual float getFloatVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getFloatVal(); + } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getFloatVal(); - } + virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDoubleVal(); + } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDoubleVal(); - } + virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getLongDoubleVal(); + } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getLongDoubleVal(); - } + virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDecimalVal(); + } + virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDateIntVal(); + } + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDatetimeIntVal(); + } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimestampIntVal(); + } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDecimalVal(); - } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDateIntVal(); - } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); - } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getTimestampIntVal(); - } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) - { - evaluate(row, isNull); - return TreeNode::getTimeIntVal(); - } -private: - void evaluate(rowgroup::Row& row, bool& isNull); + private: + void evaluate(rowgroup::Row& row, bool& isNull); }; /** -* ostream operator -*/ + * ostream operator + */ std::ostream& operator<<(std::ostream& output, const WindowFunctionColumn& rhs); /** @@ -261,5 +262,5 @@ std::ostream& operator<<(std::ostream& output, const WindowFunctionColumn& rhs); */ void getWindowFunctionCols(ParseTree* n, void* obj); -} -#endif //WINDOW_FUNCTION_COLUMN_H +} // namespace execplan +#endif // WINDOW_FUNCTION_COLUMN_H diff --git a/dbcon/joblist/anydatalist.cpp b/dbcon/joblist/anydatalist.cpp index 944f9009a..4e5df4713 100644 --- a/dbcon/joblist/anydatalist.cpp +++ b/dbcon/joblist/anydatalist.cpp @@ -30,38 +30,43 @@ static const int showOidInDataList_Index = std::ios_base::xalloc(); /*static*/ AnyDataList::DataListTypes AnyDataList::dlType(const DataList_t* dl) { - if (dl == 0) return UNKNOWN_DATALIST; - -// if (typeid(*dl) == typeid(BandedDataList)) return BANDED_DATALIST; -// if (typeid(*dl) == typeid(WorkingSetDataList)) return WORKING_SET_DATALIST; - if (typeid(*dl) == typeid(FifoDataList)) return FIFO_DATALIST; - -// if (typeid(*dl) == typeid(BucketDataList)) return BUCKET_DATALIST; -// if (typeid(*dl) == typeid(ConstantDataList_t)) return CONSTANT_DATALIST; -// if (typeid(*dl) == typeid(SortedWSDL)) return SORTED_WORKING_SET_DATALIST; -// if (typeid(*dl) == typeid(ZonedDL)) return ZONED_DATALIST; -// if (typeid(*dl) == typeid(DeliveryWSDL)) return DELIVERYWSDL; - if (typeid(*dl) == typeid(RowGroupDL)) return ROWGROUP_DATALIST; - + if (dl == 0) return UNKNOWN_DATALIST; + + // if (typeid(*dl) == typeid(BandedDataList)) return BANDED_DATALIST; + // if (typeid(*dl) == typeid(WorkingSetDataList)) return WORKING_SET_DATALIST; + if (typeid(*dl) == typeid(FifoDataList)) + return FIFO_DATALIST; + + // if (typeid(*dl) == typeid(BucketDataList)) return BUCKET_DATALIST; + // if (typeid(*dl) == typeid(ConstantDataList_t)) return CONSTANT_DATALIST; + // if (typeid(*dl) == typeid(SortedWSDL)) return SORTED_WORKING_SET_DATALIST; + // if (typeid(*dl) == typeid(ZonedDL)) return ZONED_DATALIST; + // if (typeid(*dl) == typeid(DeliveryWSDL)) return DELIVERYWSDL; + if (typeid(*dl) == typeid(RowGroupDL)) + return ROWGROUP_DATALIST; + + return UNKNOWN_DATALIST; } AnyDataList::DataListTypes AnyDataList::strDlType(const StrDataList* dl) { - if (dl == 0) return UNKNOWN_DATALIST; - -// if (typeid(*dl) == typeid(StringDataList)) return STRINGBANDED_DATALIST; -// if (typeid(*dl) == typeid(StringFifoDataList)) return STRINGFIFO_DATALIST; -// if (typeid(*dl) == typeid(StringBucketDataList)) return STRINGBUCKET_DATALIST; - if (typeid(*dl) == typeid(StrDataList)) return STRING_DATALIST; - -// if (typeid(*dl) == typeid(StringConstantDataList_t)) return STRINGCONSTANT_DATALIST; -// if (typeid(*dl) == typeid(StringSortedWSDL)) return STRINGSORTED_WORKING_SET_DATALIST; -// if (typeid(*dl) == typeid(StringZonedDL)) return STRINGZONED_DATALIST; + if (dl == 0) return UNKNOWN_DATALIST; + + // if (typeid(*dl) == typeid(StringDataList)) return STRINGBANDED_DATALIST; + // if (typeid(*dl) == typeid(StringFifoDataList)) return STRINGFIFO_DATALIST; + // if (typeid(*dl) == typeid(StringBucketDataList)) return STRINGBUCKET_DATALIST; + if (typeid(*dl) == typeid(StrDataList)) + return STRING_DATALIST; + + // if (typeid(*dl) == typeid(StringConstantDataList_t)) return STRINGCONSTANT_DATALIST; + // if (typeid(*dl) == typeid(StringSortedWSDL)) return STRINGSORTED_WORKING_SET_DATALIST; + // if (typeid(*dl) == typeid(StringZonedDL)) return STRINGZONED_DATALIST; + return UNKNOWN_DATALIST; } -//AnyDataList::DataListTypes AnyDataList::tupleDlType(const TupleDataList* dl) +// AnyDataList::DataListTypes AnyDataList::tupleDlType(const TupleDataList* dl) //{ // if (dl == 0) return UNKNOWN_DATALIST; // if (typeid(*dl) == typeid(TupleBucketDataList)) return TUPLEBUCKET_DATALIST; @@ -70,79 +75,76 @@ AnyDataList::DataListTypes AnyDataList::strDlType(const StrDataList* dl) std::ostream& operator<<(std::ostream& oss, const AnyDataListSPtr& dl) { - DataList_t* dle = NULL; - StrDataList* dls = NULL; -// DoubleDataList * dld = NULL; -// TupleBucketDataList * dlt = NULL; - bool withOid = (oss.iword(showOidInDataList_Index) != 0); + DataList_t* dle = NULL; + StrDataList* dls = NULL; + // DoubleDataList * dld = NULL; + // TupleBucketDataList * dlt = NULL; + bool withOid = (oss.iword(showOidInDataList_Index) != 0); - if ((dle = dl->dataList()) != NULL) + if ((dle = dl->dataList()) != NULL) + { + if (withOid) + oss << dle->OID() << " "; + + //...If this datalist is saved to disk, then include the saved + //...element size in the printed information. + std::ostringstream elemSizeStr; + + if (dle->useDisk()) { - if (withOid) - oss << dle->OID() << " "; - - //...If this datalist is saved to disk, then include the saved - //...element size in the printed information. - std::ostringstream elemSizeStr; - - if ( dle->useDisk() ) - { - elemSizeStr << "(" << dle->getDiskElemSize1st() << "," << - dle->getDiskElemSize2nd() << ")"; - } - - oss << "(0x" - << std::hex << (ptrdiff_t)dle << std::dec << "[" << - AnyDataList::dlType(dle) << "]" << elemSizeStr.str() << ")"; - } - else if ((dls = dl->stringDataList()) != NULL) - { - if (withOid) - oss << dls->OID() << " "; - - //...If this datalist is saved to disk, then include the saved - //...element size in the printed information. - std::ostringstream elemSizeStr; - - if ( dls->useDisk() ) - { - elemSizeStr << "(" << dls->getDiskElemSize1st() << "," << - dls->getDiskElemSize2nd() << ")"; - } - - oss << "(0x" - << std::hex << (ptrdiff_t)dls << std::dec << "[" << - AnyDataList::strDlType(dls) << "]" << elemSizeStr.str() << ")"; - } -// else if ((dld = dl->doubleDL()) != NULL) -// { -// if (withOid) -// oss << dld->OID() << " "; -// -// //...If this datalist is saved to disk, then include the saved -// //...element size in the printed information. -// std::ostringstream elemSizeStr; -// if ( dld->useDisk() ) -// { -// elemSizeStr << "(" << dld->getDiskElemSize1st() << "," << -// dld->getDiskElemSize2nd() << ")"; -// } -// -// oss << "(0x" -// << std::hex << (ptrdiff_t)dld << std::dec << "[" << -// AnyDataList::DOUBLE_DATALIST << "])"; -// } -// else if ((dlt = dl->tupleBucketDL()) != NULL) -// { -// oss << dlt->OID() << " (0x"; -// oss << std::hex << (ptrdiff_t)dlt << std::dec << "[" << AnyDataList::TUPLEBUCKET_DATALIST << "]), "; -// } - else - { - oss << "0 (0x0000 [0])"; + elemSizeStr << "(" << dle->getDiskElemSize1st() << "," << dle->getDiskElemSize2nd() << ")"; } - return oss; + oss << "(0x" << std::hex << (ptrdiff_t)dle << std::dec << "[" << AnyDataList::dlType(dle) << "]" + << elemSizeStr.str() << ")"; + } + else if ((dls = dl->stringDataList()) != NULL) + { + if (withOid) + oss << dls->OID() << " "; + + //...If this datalist is saved to disk, then include the saved + //...element size in the printed information. + std::ostringstream elemSizeStr; + + if (dls->useDisk()) + { + elemSizeStr << "(" << dls->getDiskElemSize1st() << "," << dls->getDiskElemSize2nd() << ")"; + } + + oss << "(0x" << std::hex << (ptrdiff_t)dls << std::dec << "[" << AnyDataList::strDlType(dls) << "]" + << elemSizeStr.str() << ")"; + } + // else if ((dld = dl->doubleDL()) != NULL) + // { + // if (withOid) + // oss << dld->OID() << " "; + // + // //...If this datalist is saved to disk, then include the saved + // //...element size in the printed information. + // std::ostringstream elemSizeStr; + // if ( dld->useDisk() ) + // { + // elemSizeStr << "(" << dld->getDiskElemSize1st() << "," << + // dld->getDiskElemSize2nd() << ")"; + // } + // + // oss << "(0x" + // << std::hex << (ptrdiff_t)dld << std::dec << "[" << + // AnyDataList::DOUBLE_DATALIST << "])"; + // } + // else if ((dlt = dl->tupleBucketDL()) != NULL) + // { + // oss << dlt->OID() << " (0x"; + // oss << std::hex << (ptrdiff_t)dlt << std::dec << "[" << AnyDataList::TUPLEBUCKET_DATALIST << "]), + //"; + // } + else + { + oss << "0 (0x0000 [0])"; + } + + return oss; } // @@ -152,8 +154,8 @@ std::ostream& operator<<(std::ostream& oss, const AnyDataListSPtr& dl) // std::ostream& showOidInDL(std::ostream& strm) { - strm.iword(showOidInDataList_Index) = true; - return strm; + strm.iword(showOidInDataList_Index) = true; + return strm; } // @@ -163,11 +165,9 @@ std::ostream& showOidInDL(std::ostream& strm) // std::ostream& omitOidInDL(std::ostream& strm) { - strm.iword(showOidInDataList_Index) = false; - return strm; + strm.iword(showOidInDataList_Index) = false; + return strm; } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/bandeddl.h b/dbcon/joblist/bandeddl.h index 050544d02..d418a4cfe 100644 --- a/dbcon/joblist/bandeddl.h +++ b/dbcon/joblist/bandeddl.h @@ -38,53 +38,52 @@ namespace joblist { - /** @brief class BandedDL * */ -template +template class BandedDL : public LargeDataList, element_t> { - typedef LargeDataList, element_t> base; + typedef LargeDataList, element_t> base; -public: - BandedDL(uint32_t numConsumers, const ResourceManager& rm); -// BandedDL(BucketDL &, uint32_t numConsumers, const ResourceManager& rm); - virtual ~BandedDL(); + public: + BandedDL(uint32_t numConsumers, const ResourceManager& rm); + // BandedDL(BucketDL &, uint32_t numConsumers, const ResourceManager& rm); + virtual ~BandedDL(); - int64_t saveBand(); - void loadBand(uint64_t); - int64_t bandCount(); + int64_t saveBand(); + void loadBand(uint64_t); + int64_t bandCount(); - /// loads the first band, next() will return the first element - void restart(); + /// loads the first band, next() will return the first element + void restart(); - void insert(const element_t&); - void insert(const std::vector&); - uint64_t getIterator(); - bool next(uint64_t it, element_t* e); - void endOfInput(); - using DataListImpl, element_t>::shrink; - uint64_t totalSize(); - bool next(uint64_t it, element_t* e, bool* endOfBand); + void insert(const element_t&); + void insert(const std::vector&); + uint64_t getIterator(); + bool next(uint64_t it, element_t* e); + void endOfInput(); + using DataListImpl, element_t>::shrink; + uint64_t totalSize(); + bool next(uint64_t it, element_t* e, bool* endOfBand); -protected: + protected: + private: + explicit BandedDL(){}; + explicit BandedDL(const BandedDL&){}; + BandedDL& operator=(const BandedDL&){}; -private: - explicit BandedDL() { }; - explicit BandedDL(const BandedDL&) { }; - BandedDL& operator=(const BandedDL&) { }; - - // vars to support the WSDL-like next() fcn - boost::condition nextSetLoaded; - uint64_t waitingConsumers; + // vars to support the WSDL-like next() fcn + boost::condition nextSetLoaded; + uint64_t waitingConsumers; }; -template -BandedDL::BandedDL(uint32_t nc, const ResourceManager& rm) : base(nc, sizeof(uint64_t), sizeof(uint64_t), rm) +template +BandedDL::BandedDL(uint32_t nc, const ResourceManager& rm) + : base(nc, sizeof(uint64_t), sizeof(uint64_t), rm) { - //pthread_cond_init(&nextSetLoaded, NULL); - waitingConsumers = 0; + // pthread_cond_init(&nextSetLoaded, NULL); + waitingConsumers = 0; } #if 0 @@ -116,282 +115,272 @@ BandedDL::BandedDL(BucketDL& b, uint32_t nc, const Resourc } #endif -template +template BandedDL::~BandedDL() { - //pthread_cond_destroy(&nextSetLoaded); + // pthread_cond_destroy(&nextSetLoaded); } -template +template int64_t BandedDL::saveBand() { - int64_t ret; + int64_t ret; - if (base::multipleProducers) - base::lock(); + if (base::multipleProducers) + base::lock(); - sort(base::c->begin(), base::c->end()); + sort(base::c->begin(), base::c->end()); - if (typeid(element_t) == typeid(ElementType) || - typeid(element_t) == typeid(DoubleElementType)) - ret = base::save_contiguous(); - else - ret = base::save(); + if (typeid(element_t) == typeid(ElementType) || typeid(element_t) == typeid(DoubleElementType)) + ret = base::save_contiguous(); + else + ret = base::save(); - base::registerNewSet(); + base::registerNewSet(); - if (base::multipleProducers) - base::unlock(); + if (base::multipleProducers) + base::unlock(); - return ret; + return ret; } -template +template void BandedDL::loadBand(uint64_t band) { + base::lock(); - base::lock(); + if (typeid(element_t) == typeid(ElementType) || typeid(element_t) == typeid(DoubleElementType)) + base::load_contiguous(band); + else + base::load(band); - if (typeid(element_t) == typeid(ElementType) || - typeid(element_t) == typeid(DoubleElementType)) - base::load_contiguous(band); - else - base::load(band); + if (waitingConsumers > 0) + nextSetLoaded.notify_all(); // pthread_cond_broadcast(&nextSetLoaded); - if (waitingConsumers > 0) - nextSetLoaded.notify_all(); //pthread_cond_broadcast(&nextSetLoaded); - - base::unlock(); + base::unlock(); } -template +template int64_t BandedDL::bandCount() { - int64_t ret; + int64_t ret; - base::lock(); - ret = base::setCount(); - base::unlock(); - return ret; + base::lock(); + ret = base::setCount(); + base::unlock(); + return ret; } -template +template uint64_t BandedDL::getIterator() { - uint64_t ret; + uint64_t ret; - base::lock(); - ret = base::getIterator(); - base::unlock(); - return ret; + base::lock(); + ret = base::getIterator(); + base::unlock(); + return ret; } -template +template void BandedDL::endOfInput() { - base::lock(); - sort(base::c->begin(), base::c->end()); - base::endOfInput(); + base::lock(); + sort(base::c->begin(), base::c->end()); + base::endOfInput(); - if (base::setCount > 1) + if (base::setCount > 1) + { + if (typeid(element_t) == typeid(ElementType) || typeid(element_t) == typeid(DoubleElementType)) { - if (typeid(element_t) == typeid(ElementType) || - typeid(element_t) == typeid(DoubleElementType)) - { - base::save_contiguous(); - base::load_contiguous(0); - } - else - { - base::save(); - base::load(0); - } + base::save_contiguous(); + base::load_contiguous(0); } else - base::resetIterators(); + { + base::save(); + base::load(0); + } + } + else + base::resetIterators(); + base::unlock(); +} + +template +bool BandedDL::next(uint64_t it, element_t* e) +{ + /* Note: this is the code for WSDL::next(). The more I think about it, + the more I think they're the same thing. Not entirely sure yet though. */ + + bool ret, locked = false; + uint64_t nextSet; + + if (base::numConsumers > 1 || base::phase == 0) + { + locked = true; + base::lock(); + } + + ret = base::next(it, e); + + /* XXXPAT: insignificant race condition here. Technically, there's no + guarantee the caller will be wakened when the next set is loaded. It could + get skipped. It won't happen realistically, but it exists... */ + + // signifies the caller is at the end of the loaded set, + // but there are more sets + if (ret == false && (base::loadedSet < base::setCount - 1)) + { + nextSet = base::loadedSet + 1; + waitingConsumers++; + + if (waitingConsumers < base::numConsumers) + while (nextSet != base::loadedSet) + { + // std::cout << "waiting on nextSetLoaded" << std::endl; + nextSetLoaded.wait(this->mutex); // pthread_cond_wait(&nextSetLoaded, &(this->mutex)); + } + else + { + // std::cout << "loading set " << nextSet << std::endl; + if (typeid(element_t) == typeid(ElementType) || typeid(element_t) == typeid(DoubleElementType)) + base::load_contiguous(nextSet); + else + base::load(nextSet); + + nextSetLoaded.notify_all(); // pthread_cond_broadcast(&nextSetLoaded); + } + + waitingConsumers--; + ret = base::next(it, e); + } + + if (ret == false && ++base::consumersFinished == base::numConsumers) + base::shrink(); + + if (locked) + base::unlock(); + + return ret; +} + +template +void BandedDL::insert(const element_t& e) +{ + if (base::multipleProducers) + base::lock(); + + base::insert(e); + + if (base::multipleProducers) base::unlock(); } -template -bool BandedDL::next(uint64_t it, element_t* e) -{ - - /* Note: this is the code for WSDL::next(). The more I think about it, - the more I think they're the same thing. Not entirely sure yet though. */ - - bool ret, locked = false; - uint64_t nextSet; - - if (base::numConsumers > 1 || base::phase == 0) - { - locked = true; - base::lock(); - } - - ret = base::next(it, e); - - /* XXXPAT: insignificant race condition here. Technically, there's no - guarantee the caller will be wakened when the next set is loaded. It could - get skipped. It won't happen realistically, but it exists... */ - - // signifies the caller is at the end of the loaded set, - // but there are more sets - if (ret == false && (base::loadedSet < base::setCount - 1)) - { - - nextSet = base::loadedSet + 1; - waitingConsumers++; - - if (waitingConsumers < base::numConsumers) - while (nextSet != base::loadedSet) - { -// std::cout << "waiting on nextSetLoaded" << std::endl; - nextSetLoaded.wait(this->mutex); //pthread_cond_wait(&nextSetLoaded, &(this->mutex)); - } - else - { -// std::cout << "loading set " << nextSet << std::endl; - if (typeid(element_t) == typeid(ElementType) || - typeid(element_t) == typeid(DoubleElementType)) - base::load_contiguous(nextSet); - else - base::load(nextSet); - - nextSetLoaded.notify_all(); //pthread_cond_broadcast(&nextSetLoaded); - } - - waitingConsumers--; - ret = base::next(it, e); - } - - if (ret == false && ++base::consumersFinished == base::numConsumers) - base::shrink(); - - if (locked) - base::unlock(); - - return ret; - -} - -template -void BandedDL::insert(const element_t& e) -{ - if (base::multipleProducers) - base::lock(); - - base::insert(e); - - if (base::multipleProducers) - base::unlock(); -} - -template +template void BandedDL::insert(const std::vector& e) { - throw std::logic_error("BandedDL::insert(vector) isn't implemented yet"); + throw std::logic_error("BandedDL::insert(vector) isn't implemented yet"); } /* template bool BandedDL::get(const element_t &key, element_t *out) { - typename std::set::iterator it; - bool ret, locked = false; + typename std::set::iterator it; + bool ret, locked = false; - if (base::numConsumers > 1 || base::phase == 0) { - locked = true; - base::lock(); - } + if (base::numConsumers > 1 || base::phase == 0) { + locked = true; + base::lock(); + } - it = base::c->find(key); - if (it != base::c->end()) { - *out = *it; - ret = true; - } - else - ret = false; + it = base::c->find(key); + if (it != base::c->end()) { + *out = *it; + ret = true; + } + else + ret = false; - if (locked) - base::unlock(); + if (locked) + base::unlock(); - return ret; + return ret; } */ -template +template void BandedDL::restart() { - base::lock(); -// base::waitForConsumePhase(); + base::lock(); + // base::waitForConsumePhase(); - // hack! has it been shrunk already? - if (base::c == NULL) - base::c = new std::vector(); + // hack! has it been shrunk already? + if (base::c == NULL) + base::c = new std::vector(); - if (base::setCount > 1) - { - if (typeid(element_t) == typeid(ElementType) || - typeid(element_t) == typeid(DoubleElementType)) - base::load_contiguous(0); - else - base::load(0); - } + if (base::setCount > 1) + { + if (typeid(element_t) == typeid(ElementType) || typeid(element_t) == typeid(DoubleElementType)) + base::load_contiguous(0); else - base::resetIterators(); + base::load(0); + } + else + base::resetIterators(); - base::unlock(); + base::unlock(); } -template +template bool BandedDL::next(uint64_t it, element_t* e, bool* endOfBand) { - bool ret, locked = false; - - if (base::numConsumers > 1 || base::phase == 0) - { - locked = true; - base::lock(); - } - - base::waitForConsumePhase(); - ret = base::next(it, e); - - if (ret) - { - if (locked) - base::unlock(); - - *endOfBand = false; - return ret; - } - else - { - *endOfBand = true; - ret = base::loadedSet < (base::setCount() - 1); - - if (locked) - base::unlock(); - - return ret; - } -} - -template -uint64_t BandedDL::totalSize() -{ -//std::cout << "BandedDL: c.size() = " << base::c.size() << std::endl; return base::c.size(); - uint64_t ret; + bool ret, locked = false; + if (base::numConsumers > 1 || base::phase == 0) + { + locked = true; base::lock(); - ret = base::totalSize(); - base::unlock(); + } + + base::waitForConsumePhase(); + ret = base::next(it, e); + + if (ret) + { + if (locked) + base::unlock(); + + *endOfBand = false; + return ret; + } + else + { + *endOfBand = true; + ret = base::loadedSet < (base::setCount() - 1); + + if (locked) + base::unlock(); return ret; + } } -} // namespace +template +uint64_t BandedDL::totalSize() +{ + // std::cout << "BandedDL: c.size() = " << base::c.size() << std::endl; return base::c.size(); + uint64_t ret; + + base::lock(); + ret = base::totalSize(); + base::unlock(); + + return ret; +} + +} // namespace joblist #endif - diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index fc316bc5d..702c4c091 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -52,37 +52,36 @@ using namespace joiner; namespace joblist { - -BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) : - ot(BPS_ELEMENT_TYPE), - needToSetLBID(true), - count(1), - baseRid(0), - ridCount(0), - needStrValues(false), - wideColumnsWidths(0), - filterCount(0), - projectCount(0), - needRidsAtDelivery(false), - ridMap(0), - sendValues(false), - sendAbsRids(false), - _hasScan(false), - LBIDTrace(false), - tupleLength(0), - status(0), - sendRowGroups(false), - valueColumn(0), - sendTupleJoinRowGroupData(false), - bop(BOP_AND), - forHJ(false), - threadCount(1), - fJoinerChunkSize(rm->getJlJoinerChunkSize()), - hasSmallOuterJoin(false), - _priority(1) +BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) + : ot(BPS_ELEMENT_TYPE) + , needToSetLBID(true) + , count(1) + , baseRid(0) + , ridCount(0) + , needStrValues(false) + , wideColumnsWidths(0) + , filterCount(0) + , projectCount(0) + , needRidsAtDelivery(false) + , ridMap(0) + , sendValues(false) + , sendAbsRids(false) + , _hasScan(false) + , LBIDTrace(false) + , tupleLength(0) + , status(0) + , sendRowGroups(false) + , valueColumn(0) + , sendTupleJoinRowGroupData(false) + , bop(BOP_AND) + , forHJ(false) + , threadCount(1) + , fJoinerChunkSize(rm->getJlJoinerChunkSize()) + , hasSmallOuterJoin(false) + , _priority(1) { - PMJoinerCount = 0; - uuid = bu::nil_generator()(); + PMJoinerCount = 0; + uuid = bu::nil_generator()(); } BatchPrimitiveProcessorJL::~BatchPrimitiveProcessorJL() @@ -91,190 +90,187 @@ BatchPrimitiveProcessorJL::~BatchPrimitiveProcessorJL() void BatchPrimitiveProcessorJL::addFilterStep(const pColScanStep& scan, vector lastScannedLBID) { - SCommand cc; + SCommand cc; - tableOID = scan.tableOid(); - cc.reset(new ColumnCommandJL(scan, lastScannedLBID)); - cc->setBatchPrimitiveProcessor(this); - cc->setQueryUuid(scan.queryUuid()); - cc->setStepUuid(uuid); - filterSteps.push_back(cc); - filterCount++; - _hasScan = true; - if (utils::isWide(cc->getWidth())) - wideColumnsWidths |= cc->getWidth(); - idbassert(sessionID == scan.sessionId()); + tableOID = scan.tableOid(); + cc.reset(new ColumnCommandJL(scan, lastScannedLBID)); + cc->setBatchPrimitiveProcessor(this); + cc->setQueryUuid(scan.queryUuid()); + cc->setStepUuid(uuid); + filterSteps.push_back(cc); + filterCount++; + _hasScan = true; + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); + idbassert(sessionID == scan.sessionId()); } void BatchPrimitiveProcessorJL::addFilterStep(const PseudoColStep& pcs) { - SCommand cc; + SCommand cc; - tableOID = pcs.tableOid(); - cc.reset(new PseudoCCJL(pcs)); - cc->setBatchPrimitiveProcessor(this); - cc->setQueryUuid(pcs.queryUuid()); - cc->setStepUuid(uuid); - filterSteps.push_back(cc); - filterCount++; - idbassert(sessionID == pcs.sessionId()); + tableOID = pcs.tableOid(); + cc.reset(new PseudoCCJL(pcs)); + cc->setBatchPrimitiveProcessor(this); + cc->setQueryUuid(pcs.queryUuid()); + cc->setStepUuid(uuid); + filterSteps.push_back(cc); + filterCount++; + idbassert(sessionID == pcs.sessionId()); } void BatchPrimitiveProcessorJL::addFilterStep(const pColStep& step) { - SCommand cc; + SCommand cc; - tableOID = step.tableOid(); - cc.reset(new ColumnCommandJL(step)); - cc->setBatchPrimitiveProcessor(this); - cc->setQueryUuid(step.queryUuid()); - cc->setStepUuid(uuid); - filterSteps.push_back(cc); - filterCount++; - if (utils::isWide(cc->getWidth())) - wideColumnsWidths |= cc->getWidth(); - idbassert(sessionID == step.sessionId()); + tableOID = step.tableOid(); + cc.reset(new ColumnCommandJL(step)); + cc->setBatchPrimitiveProcessor(this); + cc->setQueryUuid(step.queryUuid()); + cc->setStepUuid(uuid); + filterSteps.push_back(cc); + filterCount++; + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); + idbassert(sessionID == step.sessionId()); } void BatchPrimitiveProcessorJL::addFilterStep(const pDictionaryStep& step) { - SCommand cc; + SCommand cc; - tableOID = step.tableOid(); + tableOID = step.tableOid(); - if (filterCount == 0 && !sendRowGroups) - { - sendAbsRids = true; - sendValues = true; - absRids.reset(new uint64_t[8192]); - } + if (filterCount == 0 && !sendRowGroups) + { + sendAbsRids = true; + sendValues = true; + absRids.reset(new uint64_t[8192]); + } - cc.reset(new DictStepJL(step)); - cc->setBatchPrimitiveProcessor(this); - cc->setQueryUuid(step.queryUuid()); - cc->setStepUuid(uuid); - filterSteps.push_back(cc); - filterCount++; - needStrValues = true; - idbassert(sessionID == step.sessionId()); + cc.reset(new DictStepJL(step)); + cc->setBatchPrimitiveProcessor(this); + cc->setQueryUuid(step.queryUuid()); + cc->setStepUuid(uuid); + filterSteps.push_back(cc); + filterCount++; + needStrValues = true; + idbassert(sessionID == step.sessionId()); } - void BatchPrimitiveProcessorJL::addFilterStep(const FilterStep& step) { - SCommand cc; + SCommand cc; - tableOID = step.tableOid(); - cc.reset(new FilterCommandJL(step)); - cc->setBatchPrimitiveProcessor(this); - cc->setQueryUuid(step.queryUuid()); - cc->setStepUuid(uuid); - filterSteps.push_back(cc); - filterCount++; - idbassert(sessionID == step.sessionId()); + tableOID = step.tableOid(); + cc.reset(new FilterCommandJL(step)); + cc->setBatchPrimitiveProcessor(this); + cc->setQueryUuid(step.queryUuid()); + cc->setStepUuid(uuid); + filterSteps.push_back(cc); + filterCount++; + idbassert(sessionID == step.sessionId()); } void BatchPrimitiveProcessorJL::addProjectStep(const PseudoColStep& step) { - SCommand cc; + SCommand cc; - cc.reset(new PseudoCCJL(step)); - cc->setBatchPrimitiveProcessor(this); - cc->setTupleKey(step.tupleId()); - cc->setQueryUuid(step.queryUuid()); - cc->setStepUuid(uuid); - projectSteps.push_back(cc); - colWidths.push_back(cc->getWidth()); - tupleLength += cc->getWidth(); - projectCount++; - idbassert(sessionID == step.sessionId()); + cc.reset(new PseudoCCJL(step)); + cc->setBatchPrimitiveProcessor(this); + cc->setTupleKey(step.tupleId()); + cc->setQueryUuid(step.queryUuid()); + cc->setStepUuid(uuid); + projectSteps.push_back(cc); + colWidths.push_back(cc->getWidth()); + tupleLength += cc->getWidth(); + projectCount++; + idbassert(sessionID == step.sessionId()); } void BatchPrimitiveProcessorJL::addProjectStep(const pColStep& step) { - SCommand cc; + SCommand cc; - cc.reset(new ColumnCommandJL(step)); - cc->setBatchPrimitiveProcessor(this); - cc->setTupleKey(step.tupleId()); - cc->setQueryUuid(step.queryUuid()); - cc->setStepUuid(uuid); - projectSteps.push_back(cc); - colWidths.push_back(cc->getWidth()); - tupleLength += cc->getWidth(); - projectCount++; - if (utils::isWide(cc->getWidth())) - wideColumnsWidths |= cc->getWidth(); - idbassert(sessionID == step.sessionId()); + cc.reset(new ColumnCommandJL(step)); + cc->setBatchPrimitiveProcessor(this); + cc->setTupleKey(step.tupleId()); + cc->setQueryUuid(step.queryUuid()); + cc->setStepUuid(uuid); + projectSteps.push_back(cc); + colWidths.push_back(cc->getWidth()); + tupleLength += cc->getWidth(); + projectCount++; + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); + idbassert(sessionID == step.sessionId()); } void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& step) { - SCommand cc; + SCommand cc; - cc.reset(new PassThruCommandJL(step)); - cc->setBatchPrimitiveProcessor(this); - cc->setTupleKey(step.tupleId()); - cc->setQueryUuid(step.queryUuid()); - cc->setStepUuid(uuid); - projectSteps.push_back(cc); - colWidths.push_back(cc->getWidth()); - tupleLength += cc->getWidth(); - projectCount++; + cc.reset(new PassThruCommandJL(step)); + cc->setBatchPrimitiveProcessor(this); + cc->setTupleKey(step.tupleId()); + cc->setQueryUuid(step.queryUuid()); + cc->setStepUuid(uuid); + projectSteps.push_back(cc); + colWidths.push_back(cc->getWidth()); + tupleLength += cc->getWidth(); + projectCount++; - if (utils::isWide(cc->getWidth())) - wideColumnsWidths |= cc->getWidth(); + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); - if (filterCount == 0 && !sendRowGroups) - sendValues = true; + if (filterCount == 0 && !sendRowGroups) + sendValues = true; - idbassert(sessionID == step.sessionId()); + idbassert(sessionID == step.sessionId()); } -void BatchPrimitiveProcessorJL::addProjectStep(const pColStep& col, - const pDictionaryStep& dict) +void BatchPrimitiveProcessorJL::addProjectStep(const pColStep& col, const pDictionaryStep& dict) { - SCommand cc; + SCommand cc; - cc.reset(new RTSCommandJL(col, dict)); - cc->setBatchPrimitiveProcessor(this); - cc->setTupleKey(dict.tupleId()); - cc->setQueryUuid(col.queryUuid()); - cc->setStepUuid(uuid); - projectSteps.push_back(cc); - colWidths.push_back(cc->getWidth()); - tupleLength += cc->getWidth(); - projectCount++; - needStrValues = true; - idbassert(sessionID == col.sessionId()); - idbassert(sessionID == dict.sessionId()); + cc.reset(new RTSCommandJL(col, dict)); + cc->setBatchPrimitiveProcessor(this); + cc->setTupleKey(dict.tupleId()); + cc->setQueryUuid(col.queryUuid()); + cc->setStepUuid(uuid); + projectSteps.push_back(cc); + colWidths.push_back(cc->getWidth()); + tupleLength += cc->getWidth(); + projectCount++; + needStrValues = true; + idbassert(sessionID == col.sessionId()); + idbassert(sessionID == dict.sessionId()); } -void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& p, - const pDictionaryStep& dict) +void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& p, const pDictionaryStep& dict) { - SCommand cc; + SCommand cc; - cc.reset(new RTSCommandJL(p, dict)); - cc->setBatchPrimitiveProcessor(this); - cc->setTupleKey(dict.tupleId()); - cc->setQueryUuid(p.queryUuid()); - cc->setStepUuid(uuid); - projectSteps.push_back(cc); - colWidths.push_back(cc->getWidth()); - tupleLength += cc->getWidth(); - projectCount++; - needStrValues = true; + cc.reset(new RTSCommandJL(p, dict)); + cc->setBatchPrimitiveProcessor(this); + cc->setTupleKey(dict.tupleId()); + cc->setQueryUuid(p.queryUuid()); + cc->setStepUuid(uuid); + projectSteps.push_back(cc); + colWidths.push_back(cc->getWidth()); + tupleLength += cc->getWidth(); + projectCount++; + needStrValues = true; - if (filterCount == 0 && !sendRowGroups) - { - sendValues = true; - sendAbsRids = true; - absRids.reset(new uint64_t[8192]); - } + if (filterCount == 0 && !sendRowGroups) + { + sendValues = true; + sendAbsRids = true; + absRids.reset(new uint64_t[8192]); + } - idbassert(sessionID == p.sessionId()); - idbassert(sessionID == dict.sessionId()); + idbassert(sessionID == p.sessionId()); + idbassert(sessionID == dict.sessionId()); } #if 0 @@ -330,39 +326,39 @@ void BatchPrimitiveProcessorJL::addDeliveryStep(const DeliveryStep& ds) void BatchPrimitiveProcessorJL::addElementType(const ElementType& et, uint32_t dbroot) { - uint32_t i; -// rowCounter++; + uint32_t i; + // rowCounter++; - if (needToSetLBID) - { - needToSetLBID = false; + if (needToSetLBID) + { + needToSetLBID = false; - for (i = 0; i < filterCount; ++i) - filterSteps[i]->setLBID(et.first, dbroot); + for (i = 0; i < filterCount; ++i) + filterSteps[i]->setLBID(et.first, dbroot); - for (i = 0; i < projectCount; ++i) - projectSteps[i]->setLBID(et.first, dbroot); + for (i = 0; i < projectCount; ++i) + projectSteps[i]->setLBID(et.first, dbroot); - baseRid = et.first & 0xffffffffffffe000ULL; - } + baseRid = et.first & 0xffffffffffffe000ULL; + } - // TODO: get rid of magics - if (sendAbsRids) - absRids[ridCount] = et.first; - else - { - relRids[ridCount] = et.first & 0x1fff; // 8192 rows per logical block - ridMap |= 1 << (relRids[ridCount] >> 9); // LSB -> 0-511, MSB -> 7680-8191 - } + // TODO: get rid of magics + if (sendAbsRids) + absRids[ridCount] = et.first; + else + { + relRids[ridCount] = et.first & 0x1fff; // 8192 rows per logical block + ridMap |= 1 << (relRids[ridCount] >> 9); // LSB -> 0-511, MSB -> 7680-8191 + } - if (sendValues) - { -// cout << "adding value " << et.second << endl; - values[ridCount] = et.second; - } + if (sendValues) + { + // cout << "adding value " << et.second << endl; + values[ridCount] = et.second; + } - ridCount++; - idbassert(ridCount <= 8192); + ridCount++; + idbassert(ridCount <= 8192); } #if 0 @@ -393,13 +389,12 @@ void BatchPrimitiveProcessorJL::setRowGroupData(const rowgroup::RowGroup& rg) void BatchPrimitiveProcessorJL::addElementType(const StringElementType& et, uint32_t dbroot) { - if (filterCount == 0) - throw logic_error("BPPJL::addElementType(StringElementType): doesn't work without filter steps yet"); + if (filterCount == 0) + throw logic_error("BPPJL::addElementType(StringElementType): doesn't work without filter steps yet"); - addElementType(ElementType(et.first, et.first), dbroot); + addElementType(ElementType(et.first, et.first), dbroot); } - /** * When the output type is ElementType, the messages have the following format: * @@ -422,68 +417,68 @@ void BatchPrimitiveProcessorJL::addElementType(const StringElementType& et, uint */ // TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() -void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, - vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, - int64_t* max, uint32_t* cachedIO, uint32_t* physIO, - uint32_t* touchedBlocks) const +void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, vector* out, bool* validCPData, + uint64_t* lbid, int64_t* min, int64_t* max, + uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks) const { - uint32_t i; - uint16_t l_count; - uint64_t l_baseRid; - uint16_t* rids; - uint64_t* vals; - uint8_t* buf; - uint64_t tmp64; - uint8_t tmp8; + uint32_t i; + uint16_t l_count; + uint64_t l_baseRid; + uint16_t* rids; + uint64_t* vals; + uint8_t* buf; + uint64_t tmp64; + uint8_t tmp8; - /* skip the header */ - idbassert(in.length() > sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + /* skip the header */ + idbassert(in.length() > sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - if (_hasScan) + if (_hasScan) + { + in >> tmp8; + *validCPData = (tmp8 != 0); + + if (*validCPData) { - in >> tmp8; - *validCPData = (tmp8 != 0); - - if (*validCPData) - { - in >> *lbid; - in >> tmp64; - *min = (int64_t) tmp64; - in >> tmp64; - *max = (int64_t) tmp64; - } - else - in >> *lbid; + in >> *lbid; + in >> tmp64; + *min = (int64_t)tmp64; + in >> tmp64; + *max = (int64_t)tmp64; } + else + in >> *lbid; + } - in >> l_baseRid; - in >> l_count; -// rowsProcessed += l_count; - idbassert(l_count <= 8192); - out->resize(l_count); + in >> l_baseRid; + in >> l_count; + // rowsProcessed += l_count; + idbassert(l_count <= 8192); + out->resize(l_count); - buf = (uint8_t*) in.buf(); + buf = (uint8_t*)in.buf(); - rids = (uint16_t*) buf; - vals = (uint64_t*) (buf + (l_count << 1)); - idbassert(in.length() > (uint32_t) ((l_count << 1) + (l_count << 3)) ); - in.advance((l_count << 1) + (l_count << 3)); + rids = (uint16_t*)buf; + vals = (uint64_t*)(buf + (l_count << 1)); + idbassert(in.length() > (uint32_t)((l_count << 1) + (l_count << 3))); + in.advance((l_count << 1) + (l_count << 3)); - for (i = 0; i < l_count; ++i) - { - (*out)[i].first = rids[i] + l_baseRid; -// if (tableOID >= 3000) -// idbassert((*out)[i].first > 1023); - (*out)[i].second = vals[i]; - } + for (i = 0; i < l_count; ++i) + { + (*out)[i].first = rids[i] + l_baseRid; + // if (tableOID >= 3000) + // idbassert((*out)[i].first > 1023); + (*out)[i].second = vals[i]; + } - in >> *cachedIO; - in >> *physIO; - in >> *touchedBlocks; -// cout << "ET: got physIO=" << (int) *physIO << " cachedIO=" << -// (int) *cachedIO << " touchedBlocks=" << (int) *touchedBlocks << endl; - idbassert(in.length() == 0); + in >> *cachedIO; + in >> *physIO; + in >> *touchedBlocks; + // cout << "ET: got physIO=" << (int) *physIO << " cachedIO=" << + // (int) *cachedIO << " touchedBlocks=" << (int) *touchedBlocks << endl; + idbassert(in.length() == 0); } /** @@ -503,55 +498,56 @@ void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, */ // TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() -void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, - vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, - int64_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const +void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, vector* out, + bool* validCPData, uint64_t* lbid, int64_t* min, + int64_t* max, uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks) const { - uint32_t i; - uint16_t l_count; - uint64_t* l_absRids; - uint64_t tmp64; - uint8_t tmp8; + uint32_t i; + uint16_t l_count; + uint64_t* l_absRids; + uint64_t tmp64; + uint8_t tmp8; -// cout << "get String ETs uniqueID\n"; - /* skip the header */ - in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + // cout << "get String ETs uniqueID\n"; + /* skip the header */ + in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - if (_hasScan) + if (_hasScan) + { + in >> tmp8; + *validCPData = (tmp8 != 0); + + if (*validCPData) { - in >> tmp8; - *validCPData = (tmp8 != 0); - - if (*validCPData) - { - in >> *lbid; - in >> tmp64; - *min = (int64_t) tmp64; - in >> tmp64; - *max = (int64_t) tmp64; - } - else - in >> *lbid; + in >> *lbid; + in >> tmp64; + *min = (int64_t)tmp64; + in >> tmp64; + *max = (int64_t)tmp64; } + else + in >> *lbid; + } - in >> l_count; -// cout << "parsing " << l_count << " strings\n"; - l_absRids = (uint64_t*) in.buf(); - out->resize(l_count); - in.advance(l_count << 3); + in >> l_count; + // cout << "parsing " << l_count << " strings\n"; + l_absRids = (uint64_t*)in.buf(); + out->resize(l_count); + in.advance(l_count << 3); - for (i = 0; i < l_count; ++i) - { - (*out)[i].first = l_absRids[i]; - in >> (*out)[i].second; - } + for (i = 0; i < l_count; ++i) + { + (*out)[i].first = l_absRids[i]; + in >> (*out)[i].second; + } - in >> *cachedIO; - in >> *physIO; - in >> *touchedBlocks; -// cout << "SET: got physIO=" << (int) *physIO << " cachedIO=" << -// (int) *cachedIO << " touchedBlocks=" << (int) *touchedBlocks << endl; - idbassert(in.length() == 0); + in >> *cachedIO; + in >> *physIO; + in >> *touchedBlocks; + // cout << "SET: got physIO=" << (int) *physIO << " cachedIO=" << + // (int) *cachedIO << " touchedBlocks=" << (int) *touchedBlocks << endl; + idbassert(in.length() == 0); } /** @@ -560,422 +556,421 @@ void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, */ // TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() -void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, - std::vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, - int64_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const +void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, std::vector* out, + bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, + uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const { - uint32_t i, j, pos, len; - uint16_t l_rowCount; - uint64_t l_baseRid; - uint16_t* l_relRids; - uint64_t absRids[8192]; - //const uint8_t* columnData[projectCount]; - const uint8_t** columnData = (const uint8_t**)alloca(projectCount * sizeof(uint8_t*)); - memset(columnData, 0, projectCount * sizeof(uint8_t*)); - const uint8_t* buf; - //uint32_t colLengths[projectCount]; - uint32_t* colLengths = (uint32_t*)alloca(projectCount * sizeof(uint32_t)); - uint64_t tmp64; - uint8_t tmp8; + uint32_t i, j, pos, len; + uint16_t l_rowCount; + uint64_t l_baseRid; + uint16_t* l_relRids; + uint64_t absRids[8192]; + // const uint8_t* columnData[projectCount]; + const uint8_t** columnData = (const uint8_t**)alloca(projectCount * sizeof(uint8_t*)); + memset(columnData, 0, projectCount * sizeof(uint8_t*)); + const uint8_t* buf; + // uint32_t colLengths[projectCount]; + uint32_t* colLengths = (uint32_t*)alloca(projectCount * sizeof(uint32_t)); + uint64_t tmp64; + uint8_t tmp8; -// cout << "getTuples msg is " << in.length() << " bytes\n"; - /* skip the header */ - in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + // cout << "getTuples msg is " << in.length() << " bytes\n"; + /* skip the header */ + in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - if (_hasScan) + if (_hasScan) + { + in >> tmp8; + *validCPData = (tmp8 != 0); + + if (*validCPData) { - in >> tmp8; - *validCPData = (tmp8 != 0); - - if (*validCPData) - { - in >> *lbid; - in >> tmp64; - *min = (int64_t) tmp64; - in >> tmp64; - *max = (int64_t) tmp64; - } - else - in >> *lbid; + in >> *lbid; + in >> tmp64; + *min = (int64_t)tmp64; + in >> tmp64; + *max = (int64_t)tmp64; } + else + in >> *lbid; + } - in >> l_rowCount; + in >> l_rowCount; -// cout << "read " << l_rowCount << " rows\n"; + // cout << "read " << l_rowCount << " rows\n"; - if (needRidsAtDelivery) - { - in >> l_baseRid; - l_relRids = (uint16_t*) in.buf(); - - for (i = 0; i < l_rowCount; i++) - absRids[i] = l_relRids[i] + l_baseRid; - - in.advance(l_rowCount << 1); - } - - /* Set up pointers to the column data */ - pos = 0; - buf = in.buf(); - - for (i = 0; i < projectCount; i++) - { - colLengths[i] = *((uint32_t*) &buf[pos]); - pos += 4; - columnData[i] = &buf[pos]; -// cout << "column " << i << " is " << colLengths[i] << " long at " << pos << endl; - pos += colLengths[i]; - idbassert(pos < in.length()); - } - - in.advance(pos); - - out->resize(l_rowCount); + if (needRidsAtDelivery) + { + in >> l_baseRid; + l_relRids = (uint16_t*)in.buf(); for (i = 0; i < l_rowCount; i++) + absRids[i] = l_relRids[i] + l_baseRid; + + in.advance(l_rowCount << 1); + } + + /* Set up pointers to the column data */ + pos = 0; + buf = in.buf(); + + for (i = 0; i < projectCount; i++) + { + colLengths[i] = *((uint32_t*)&buf[pos]); + pos += 4; + columnData[i] = &buf[pos]; + // cout << "column " << i << " is " << colLengths[i] << " long at " << pos << endl; + pos += colLengths[i]; + idbassert(pos < in.length()); + } + + in.advance(pos); + + out->resize(l_rowCount); + + for (i = 0; i < l_rowCount; i++) + { + (*out)[i].first = absRids[i]; + (*out)[i].second = new char[tupleLength]; + + for (j = 0, pos = 0; j < projectCount; j++) { - (*out)[i].first = absRids[i]; - (*out)[i].second = new char[tupleLength]; + idbassert(pos + colWidths[j] <= tupleLength); - for (j = 0, pos = 0; j < projectCount; j++) + if (projectSteps[j]->getCommandType() == CommandJL::RID_TO_STRING) + { + len = *((uint32_t*)columnData[j]); + columnData[j] += 4; + memcpy(&(*out)[i].second[pos], columnData[j], len); + pos += len; + columnData[j] += len; + + // insert padding... + memset(&(*out)[i].second[pos], 0, colWidths[j] - len); + pos += colWidths[j] - len; + } + else + { + switch (colWidths[j]) { - idbassert(pos + colWidths[j] <= tupleLength); + case 8: + *((uint64_t*)&(*out)[i].second[pos]) = *((uint64_t*)columnData[j]); + columnData[j] += 8; + pos += 8; + break; - if (projectSteps[j]->getCommandType() == CommandJL::RID_TO_STRING) - { - len = *((uint32_t*) columnData[j]); - columnData[j] += 4; - memcpy(&(*out)[i].second[pos], columnData[j], len); - pos += len; - columnData[j] += len; + case 4: + *((uint32_t*)&(*out)[i].second[pos]) = *((uint32_t*)columnData[j]); + columnData[j] += 4; + pos += 4; + break; - // insert padding... - memset(&(*out)[i].second[pos], 0, colWidths[j] - len); - pos += colWidths[j] - len; - } - else - { - switch (colWidths[j]) - { - case 8: - *((uint64_t*) & (*out)[i].second[pos]) = *((uint64_t*) columnData[j]); - columnData[j] += 8; - pos += 8; - break; + case 2: + *((uint16_t*)&(*out)[i].second[pos]) = *((uint16_t*)columnData[j]); + columnData[j] += 2; + pos += 4; + break; - case 4: - *((uint32_t*) & (*out)[i].second[pos]) = *((uint32_t*) columnData[j]); - columnData[j] += 4; - pos += 4; - break; + case 1: + (*out)[i].second[pos] = (char)*columnData[j]; + columnData[j]++; + pos++; + break; - case 2: - *((uint16_t*) & (*out)[i].second[pos]) = *((uint16_t*) columnData[j]); - columnData[j] += 2; - pos += 4; - break; - - case 1: - (*out)[i].second[pos] = (char) * columnData[j]; - columnData[j]++; - pos++; - break; - - // TODO MCOL-641 - case 16: - //fallthrough - default: - cout << "BPP::getTuples(): bad column width of " << colWidths[j] - << endl; - throw logic_error("BPP::getTuples(): bad column width"); - } - } + // TODO MCOL-641 + case 16: + // fallthrough + default: + cout << "BPP::getTuples(): bad column width of " << colWidths[j] << endl; + throw logic_error("BPP::getTuples(): bad column width"); } + } } + } - in >> *cachedIO; - in >> *physIO; - in >> *touchedBlocks; - idbassert(in.length() == 0); + in >> *cachedIO; + in >> *physIO; + in >> *touchedBlocks; + idbassert(in.length() == 0); } bool BatchPrimitiveProcessorJL::countThisMsg(messageqcpp::ByteStream& in) const { - const uint8_t* data = in.buf(); - uint32_t offset = sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader); // skip the headers + const uint8_t* data = in.buf(); + uint32_t offset = sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader); // skip the headers - ISMPacketHeader* hdr = (ISMPacketHeader*)(data); + ISMPacketHeader* hdr = (ISMPacketHeader*)(data); - if (_hasScan && in.length() > offset) + if (_hasScan && in.length() > offset) + { + // This is a legitimate error message sent by PrimProc + // so we need to return to allow upper layer to throw an error + // if needed. + if (hdr->Status > 0) { - // This is a legitimate error message sent by PrimProc - // so we need to return to allow upper layer to throw an error - // if needed. - if (hdr->Status > 0) - { - return true; - } - - if (data[offset] != 0) - offset += (data[offset + CP_FLAG_AND_LBID] * 2) + CP_FLAG_AND_LBID + 1; // skip the CP data with wide min/max values (16/32 bytes each) - else - offset += CP_FLAG_AND_LBID; // skip only the "valid CP data" & LBID bytes + return true; } - // Throw b/c PP throws and sends here error msg. - // See BatchPrimitiveProcessor::writeErrorMsg() for details. - // The inversion of the assert used here previously. - if (in.length() <= offset) + if (data[offset] != 0) + offset += (data[offset + CP_FLAG_AND_LBID] * 2) + CP_FLAG_AND_LBID + + 1; // skip the CP data with wide min/max values (16/32 bytes each) + else + offset += CP_FLAG_AND_LBID; // skip only the "valid CP data" & LBID bytes + } + + // Throw b/c PP throws and sends here error msg. + // See BatchPrimitiveProcessor::writeErrorMsg() for details. + // The inversion of the assert used here previously. + if (in.length() <= offset) + { + if (hdr->Status > 0) { - if (hdr->Status > 0) - { - throw std::runtime_error(" an exception originally thrown by PrimProc: "); - } - throw std::runtime_error(" an exception because there is not enough \ + throw std::runtime_error(" an exception originally thrown by PrimProc: "); + } + throw std::runtime_error( + " an exception because there is not enough \ data in the Primitive message from PrimProc."); - } + } - - return (data[offset] != 0); + return (data[offset] != 0); } -void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, - vector* out) const +void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, vector* out) const { - RGData rgData; - uint32_t count, i; + RGData rgData; + uint32_t count, i; - *in >> count; + *in >> count; - for (i = 0; i < count; i++) - { - rgData.deserialize(*in, true); - out->push_back(rgData); - } + for (i = 0; i < count; i++) + { + rgData.deserialize(*in, true); + out->push_back(rgData); + } } -void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, - bool* validCPData, uint64_t* lbid, int128_t* min, int128_t* max, - uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID, bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const +void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, bool* validCPData, + uint64_t* lbid, int128_t* min, int128_t* max, + uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, + bool* countThis, uint32_t threadID, bool* hasWideColumn, + const execplan::CalpontSystemCatalog::ColType& colType) const { - uint64_t tmp64; - int128_t tmp128; - uint8_t tmp8; - RGData rgData; - uint32_t rowCount; - RowGroup& org = primprocRG[threadID]; + uint64_t tmp64; + int128_t tmp128; + uint8_t tmp8; + RGData rgData; + uint32_t rowCount; + RowGroup& org = primprocRG[threadID]; - out->clear(); + out->clear(); - if (in.length() == 0) + if (in.length() == 0) + { + // done, return an empty RG + rgData = RGData(org, 0); + org.setData(&rgData); + org.resetRowGroup(0); + out->push_back(rgData); + *cachedIO = 0; + *physIO = 0; + *touchedBlocks = 0; + return; + } + + /* skip the header */ + in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + + if (_hasScan) + { + in >> tmp8; + *validCPData = (tmp8 != 0); + + if (*validCPData) { - // done, return an empty RG - rgData = RGData(org, 0); - org.setData(&rgData); - org.resetRowGroup(0); - out->push_back(rgData); - *cachedIO = 0; - *physIO = 0; - *touchedBlocks = 0; - return; - } - - /* skip the header */ - in.advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - - if (_hasScan) - { - in >> tmp8; - *validCPData = (tmp8 != 0); - - if (*validCPData) + in >> *lbid; + in >> tmp8; + *hasWideColumn = (tmp8 > utils::MAXLEGACYWIDTH); + if (UNLIKELY(*hasWideColumn)) + { + idbassert(colType.colWidth > utils::MAXLEGACYWIDTH); + if (LIKELY(colType.isWideDecimalType())) { - in >> *lbid; - in >> tmp8; - *hasWideColumn = (tmp8 > utils::MAXLEGACYWIDTH); - if (UNLIKELY(*hasWideColumn)) - { - idbassert(colType.colWidth > utils::MAXLEGACYWIDTH); - if (LIKELY(colType.isWideDecimalType())) - { - in >> tmp128; - *min = tmp128; - in >> tmp128; - *max = tmp128; - } - else - { - std::ostringstream oss; - oss << __func__ << " WARNING!!! Not implemented for the data type "; - oss << colType.colDataType << std::endl; - std::cout << oss.str(); - idbassert(false); - } - } - else - { - in >> tmp64; - *min = static_cast(tmp64); - in >> tmp64; - *max = static_cast(tmp64); - } + in >> tmp128; + *min = tmp128; + in >> tmp128; + *max = tmp128; } else - in >> *lbid; - } - - in >> tmp8; - *countThis = (tmp8 != 0); - - /* Would be cleaner to make the PM BPP's send a msg count to unify the msg formats. - * For later... */ - if (aggregatorPM) - { - deserializeAggregateResult(&in, out); - //for (uint32_t z = 0; z < out->size(); z++) { - // org.setData(&(*out)[z]); - // cout << "BPPJL: " << org.toString() << endl; - //} - } - else - { - rgData.deserialize(in, true); - out->push_back(rgData); - org.setData(&rgData); - - rowCount = org.getRowCount(); - - bool pmSendsMatchesAnyway = (hasSmallOuterJoin && *countThis && PMJoinerCount > 0 && - (fe2 || aggregatorPM)); - - if (!pmSendsFinalResult() || pmSendsMatchesAnyway) { - boost::shared_array > joinResults; - uint32_t i, j; - - if (pmSendsMatchesAnyway) - { - uint16_t joinRowCount; - in >> joinRowCount; - rowCount = joinRowCount; - } - - for (j = 0; j < PMJoinerCount; j++) - { - /* Reuse the result space if possible */ - joinResults = tJoiners[j]->getPMJoinArrays(threadID); - - if (joinResults.get() == NULL) - { - joinResults.reset(new vector[8192]); - tJoiners[j]->setPMJoinResults(joinResults, threadID); - } - - for (i = 0; i < rowCount; i++) - deserializeInlineVector(in, joinResults[i]); - - if (tJoiners[j]->smallOuterJoin()) - tJoiners[j]->markMatches(threadID, rowCount); - } + std::ostringstream oss; + oss << __func__ << " WARNING!!! Not implemented for the data type "; + oss << colType.colDataType << std::endl; + std::cout << oss.str(); + idbassert(false); } - } - - if (*countThis) - { -// cout << "grabbing io stats\n"; - in >> *cachedIO; - in >> *physIO; - in >> *touchedBlocks; + } + else + { + in >> tmp64; + *min = static_cast(tmp64); + in >> tmp64; + *max = static_cast(tmp64); + } } else - { - *cachedIO = 0; - *physIO = 0; - *touchedBlocks = 0; - } + in >> *lbid; + } - idbassert(in.length() == 0); + in >> tmp8; + *countThis = (tmp8 != 0); + + /* Would be cleaner to make the PM BPP's send a msg count to unify the msg formats. + * For later... */ + if (aggregatorPM) + { + deserializeAggregateResult(&in, out); + // for (uint32_t z = 0; z < out->size(); z++) { + // org.setData(&(*out)[z]); + // cout << "BPPJL: " << org.toString() << endl; + //} + } + else + { + rgData.deserialize(in, true); + out->push_back(rgData); + org.setData(&rgData); + + rowCount = org.getRowCount(); + + bool pmSendsMatchesAnyway = + (hasSmallOuterJoin && *countThis && PMJoinerCount > 0 && (fe2 || aggregatorPM)); + + if (!pmSendsFinalResult() || pmSendsMatchesAnyway) + { + boost::shared_array > joinResults; + uint32_t i, j; + + if (pmSendsMatchesAnyway) + { + uint16_t joinRowCount; + in >> joinRowCount; + rowCount = joinRowCount; + } + + for (j = 0; j < PMJoinerCount; j++) + { + /* Reuse the result space if possible */ + joinResults = tJoiners[j]->getPMJoinArrays(threadID); + + if (joinResults.get() == NULL) + { + joinResults.reset(new vector[8192]); + tJoiners[j]->setPMJoinResults(joinResults, threadID); + } + + for (i = 0; i < rowCount; i++) + deserializeInlineVector(in, joinResults[i]); + + if (tJoiners[j]->smallOuterJoin()) + tJoiners[j]->markMatches(threadID, rowCount); + } + } + } + + if (*countThis) + { + // cout << "grabbing io stats\n"; + in >> *cachedIO; + in >> *physIO; + in >> *touchedBlocks; + } + else + { + *cachedIO = 0; + *physIO = 0; + *touchedBlocks = 0; + } + + idbassert(in.length() == 0); } -//boost::shared_array -RGData -BatchPrimitiveProcessorJL::getErrorRowGroupData(uint16_t error) const +// boost::shared_array +RGData BatchPrimitiveProcessorJL::getErrorRowGroupData(uint16_t error) const { - RGData ret; - rowgroup::RowGroup rg(projectionRG); + RGData ret; + rowgroup::RowGroup rg(projectionRG); - ret = RGData(rg, 0); - rg.setData(&ret); - //rg.convertToInlineDataInPlace(); - rg.resetRowGroup(0); - rg.setStatus(error); - return ret; - //return ret->rowData; + ret = RGData(rg, 0); + rg.setData(&ret); + // rg.convertToInlineDataInPlace(); + rg.resetRowGroup(0); + rg.setStatus(error); + return ret; + // return ret->rowData; } void BatchPrimitiveProcessorJL::reset() { - ridCount = 0; - ridMap = 0; - needToSetLBID = true; + ridCount = 0; + ridMap = 0; + needToSetLBID = true; } void BatchPrimitiveProcessorJL::setLBID(uint64_t l, const BRM::EMEntry& scannedExtent) { - uint32_t i; + uint32_t i; - dbRoot = scannedExtent.dbRoot; - baseRid = rowgroup::convertToRid(scannedExtent.partitionNum, - scannedExtent.segmentNum, - scannedExtent.blockOffset / (scannedExtent.range.size * 1024), // the extent # - (l - scannedExtent.range.start) / scannedExtent.range.size); // the logical block # + dbRoot = scannedExtent.dbRoot; + baseRid = rowgroup::convertToRid( + scannedExtent.partitionNum, scannedExtent.segmentNum, + scannedExtent.blockOffset / (scannedExtent.range.size * 1024), // the extent # + (l - scannedExtent.range.start) / scannedExtent.range.size); // the logical block # - /* - cout << "got baserid=" << baseRid << " from partnum=" << scannedExtent.partitionNum - << " segnum=" << scannedExtent.segmentNum << " extentnum=" << - scannedExtent.blockOffset/(scannedExtent.range.size * 1024) << - " blocknum=" << (l - scannedExtent.range.start)/scannedExtent.range.size << endl; - */ + /* + cout << "got baserid=" << baseRid << " from partnum=" << scannedExtent.partitionNum + << " segnum=" << scannedExtent.segmentNum << " extentnum=" << + scannedExtent.blockOffset/(scannedExtent.range.size * 1024) << + " blocknum=" << (l - scannedExtent.range.start)/scannedExtent.range.size << endl; + */ - for (i = 0; i < filterCount; ++i) - filterSteps[i]->setLBID(baseRid, dbRoot); + for (i = 0; i < filterCount; ++i) + filterSteps[i]->setLBID(baseRid, dbRoot); - for (i = 0; i < projectCount; ++i) - projectSteps[i]->setLBID(baseRid, dbRoot); + for (i = 0; i < projectCount; ++i) + projectSteps[i]->setLBID(baseRid, dbRoot); } string BatchPrimitiveProcessorJL::toString() const { - ostringstream ret; - uint32_t i; + ostringstream ret; + uint32_t i; - ret << "BatchPrimitiveProcessorJL:" << endl; + ret << "BatchPrimitiveProcessorJL:" << endl; - if (!_hasScan) - { - if (sendValues) - ret << " -- serializing values" << endl; + if (!_hasScan) + { + if (sendValues) + ret << " -- serializing values" << endl; - if (sendAbsRids) - ret << " -- serializing absolute rids" << endl; - else - ret << " -- serializing relative rids" << endl; - } + if (sendAbsRids) + ret << " -- serializing absolute rids" << endl; else - ret << " -- scan driven" << endl; + ret << " -- serializing relative rids" << endl; + } + else + ret << " -- scan driven" << endl; - ret << " " << (int) filterCount << " filter steps:\n"; + ret << " " << (int)filterCount << " filter steps:\n"; - for (i = 0; i < filterCount; i++) - ret << " " << filterSteps[i]->toString() << endl; + for (i = 0; i < filterCount; i++) + ret << " " << filterSteps[i]->toString() << endl; - ret << " " << (int) projectCount << " projection steps:\n"; + ret << " " << (int)projectCount << " projection steps:\n"; - for (i = 0; i < projectCount; i++) - ret << " " << projectSteps[i]->toString() << endl; + for (i = 0; i < projectCount; i++) + ret << " " << projectSteps[i]->toString() << endl; - return ret.str(); + return ret.str(); } /** @@ -1001,205 +996,204 @@ string BatchPrimitiveProcessorJL::toString() const void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const { - ISMPacketHeader ism; - uint32_t i; - uint16_t flags = 0; + ISMPacketHeader ism; + uint32_t i; + uint16_t flags = 0; - ism.Command = BATCH_PRIMITIVE_CREATE; + ism.Command = BATCH_PRIMITIVE_CREATE; - bs.load((uint8_t*) &ism, sizeof(ism)); - bs << (uint8_t) ot; - bs << (messageqcpp::ByteStream::quadbyte)txnID; - bs << (messageqcpp::ByteStream::quadbyte)sessionID; - bs << (messageqcpp::ByteStream::quadbyte)stepID; - bs << uniqueID; - bs << versionInfo; + bs.load((uint8_t*)&ism, sizeof(ism)); + bs << (uint8_t)ot; + bs << (messageqcpp::ByteStream::quadbyte)txnID; + bs << (messageqcpp::ByteStream::quadbyte)sessionID; + bs << (messageqcpp::ByteStream::quadbyte)stepID; + bs << uniqueID; + bs << versionInfo; - if (needStrValues) - flags |= NEED_STR_VALUES; + if (needStrValues) + flags |= NEED_STR_VALUES; - if (sendAbsRids) - flags |= GOT_ABS_RIDS; + if (sendAbsRids) + flags |= GOT_ABS_RIDS; - if (sendValues) - flags |= GOT_VALUES; + if (sendValues) + flags |= GOT_VALUES; - if (LBIDTrace) - flags |= LBID_TRACE; + if (LBIDTrace) + flags |= LBID_TRACE; - if (needRidsAtDelivery) - flags |= SEND_RIDS_AT_DELIVERY; + if (needRidsAtDelivery) + flags |= SEND_RIDS_AT_DELIVERY; - if (tJoiners.size() > 0) - flags |= HAS_JOINER; + if (tJoiners.size() > 0) + flags |= HAS_JOINER; - if (sendRowGroups) - flags |= HAS_ROWGROUP; + if (sendRowGroups) + flags |= HAS_ROWGROUP; - if (sendTupleJoinRowGroupData) - flags |= JOIN_ROWGROUP_DATA; + if (sendTupleJoinRowGroupData) + flags |= JOIN_ROWGROUP_DATA; - if (wideColumnsWidths) - flags |= HAS_WIDE_COLUMNS; + if (wideColumnsWidths) + flags |= HAS_WIDE_COLUMNS; - bs << flags; + bs << flags; - if (wideColumnsWidths) - bs << wideColumnsWidths; + if (wideColumnsWidths) + bs << wideColumnsWidths; - bs << bop; - bs << (uint8_t) (forHJ ? 1 : 0); + bs << bop; + bs << (uint8_t)(forHJ ? 1 : 0); - if (sendRowGroups) + if (sendRowGroups) + { + bs << valueColumn; + bs << inputRG; + } + + if (ot == ROW_GROUP) + { + bs << projectionRG; + // cout << "BPPJL: projectionRG is:\n" << projectionRG.toString() << endl; + + /* F&E serialization */ + if (fe1) { - bs << valueColumn; - bs << inputRG; + bs << (uint8_t)1; + bs << *fe1; + bs << fe1Input; } + else + bs << (uint8_t)0; + if (fe2) + { + bs << (uint8_t)1; + bs << *fe2; + bs << fe2Output; + } + else + bs << (uint8_t)0; + } + + /* if HAS_JOINER, send the init params */ + if (flags & HAS_JOINER) + { if (ot == ROW_GROUP) { - bs << projectionRG; -// cout << "BPPJL: projectionRG is:\n" << projectionRG.toString() << endl; + idbassert(tJoiners.size() > 0); + bs << (messageqcpp::ByteStream::quadbyte)PMJoinerCount; - /* F&E serialization */ - if (fe1) - { - bs << (uint8_t) 1; - bs << *fe1; - bs << fe1Input; - } - else - bs << (uint8_t) 0; - - if (fe2) - { - bs << (uint8_t) 1; - bs << *fe2; - bs << fe2Output; - } - else - bs << (uint8_t) 0; - } - - /* if HAS_JOINER, send the init params */ - if (flags & HAS_JOINER) - { - if (ot == ROW_GROUP) - { - idbassert(tJoiners.size() > 0); - bs << (messageqcpp::ByteStream::quadbyte)PMJoinerCount; - - bool atLeastOneFE = false; + bool atLeastOneFE = false; #ifdef JLF_DEBUG - cout << "PMJoinerCount = " << PMJoinerCount << endl; + cout << "PMJoinerCount = " << PMJoinerCount << endl; #endif - bool smallSideRGSent = false; - for (i = 0; i < PMJoinerCount; i++) - { - bs << (uint32_t) tJoiners[i]->size(); - bs << tJoiners[i]->getJoinType(); + bool smallSideRGSent = false; + for (i = 0; i < PMJoinerCount; i++) + { + bs << (uint32_t)tJoiners[i]->size(); + bs << tJoiners[i]->getJoinType(); - //bs << (uint64_t) tJoiners[i]->smallNullValue(); + // bs << (uint64_t) tJoiners[i]->smallNullValue(); - bs << (uint8_t) tJoiners[i]->isTypelessJoin(); + bs << (uint8_t)tJoiners[i]->isTypelessJoin(); - if (tJoiners[i]->hasFEFilter()) - { - atLeastOneFE = true; -#ifdef JLF_DEBUG - cout << "serializing join FE object\n"; -#endif - bs << *tJoiners[i]->getFcnExpFilter(); - } - - if (!tJoiners[i]->isTypelessJoin()) - { - bs << (uint64_t) tJoiners[i]->smallNullValue(); - bs << (messageqcpp::ByteStream::quadbyte)tJoiners[i]->getLargeKeyColumn(); - //cout << "large key column is " << (uint32_t) tJoiners[i]->getLargeKeyColumn() << endl; - } - else - { - serializeVector(bs, tJoiners[i]->getLargeKeyColumns()); - bs << (uint32_t) tJoiners[i]->getKeyLength(); - // MCOL-4173 Notify PP if smallSide and largeSide have different column widths - // and send smallSide RG to PP. - bool joinHasSkewedKeyColumn = tJoiners[i]->joinHasSkewedKeyColumn(); - bs << (uint8_t) joinHasSkewedKeyColumn; - if (!smallSideRGSent && joinHasSkewedKeyColumn) - { - idbassert(!smallSideRGs.empty()); - bs << smallSideRGs[0]; - serializeVector(bs, tJoiners[i]->getSmallKeyColumns()); - smallSideRGSent = true; - } - } - } - - if (atLeastOneFE) - bs << joinFERG; - - if (sendTupleJoinRowGroupData) - { -#ifdef JLF_DEBUG - cout << "sending smallside data\n"; -#endif - serializeVector(bs, smallSideRGs); - bs << largeSideRG; - bs << joinedRG; // TODO: I think we can omit joinedRG if (!(fe2 || aggregatorPM)) -// cout << "joined RG: " << joinedRG.toString() << endl; - } - } - } - - bs << filterCount; - - for (i = 0; i < filterCount; ++i) - { -// cout << "serializing step " << i << endl; - filterSteps[i]->createCommand(bs); - } - - bs << projectCount; - - for (i = 0; i < projectCount; ++i) - projectSteps[i]->createCommand(bs); - - // aggregate step only when output is row group - if (ot == ROW_GROUP) - { - if (aggregatorPM.get() != NULL) + if (tJoiners[i]->hasFEFilter()) { - bs << (uint8_t) 1; - bs << aggregateRGPM; - bs << *(aggregatorPM.get()); + atLeastOneFE = true; +#ifdef JLF_DEBUG + cout << "serializing join FE object\n"; +#endif + bs << *tJoiners[i]->getFcnExpFilter(); + } + + if (!tJoiners[i]->isTypelessJoin()) + { + bs << (uint64_t)tJoiners[i]->smallNullValue(); + bs << (messageqcpp::ByteStream::quadbyte)tJoiners[i]->getLargeKeyColumn(); + // cout << "large key column is " << (uint32_t) tJoiners[i]->getLargeKeyColumn() << endl; } else { - bs << (uint8_t) 0; + serializeVector(bs, tJoiners[i]->getLargeKeyColumns()); + bs << (uint32_t)tJoiners[i]->getKeyLength(); + // MCOL-4173 Notify PP if smallSide and largeSide have different column widths + // and send smallSide RG to PP. + bool joinHasSkewedKeyColumn = tJoiners[i]->joinHasSkewedKeyColumn(); + bs << (uint8_t)joinHasSkewedKeyColumn; + if (!smallSideRGSent && joinHasSkewedKeyColumn) + { + idbassert(!smallSideRGs.empty()); + bs << smallSideRGs[0]; + serializeVector(bs, tJoiners[i]->getSmallKeyColumns()); + smallSideRGSent = true; + } } - } + } - // decide which rowgroup is received by PrimProc - if (ot == ROW_GROUP) + if (atLeastOneFE) + bs << joinFERG; + + if (sendTupleJoinRowGroupData) + { +#ifdef JLF_DEBUG + cout << "sending smallside data\n"; +#endif + serializeVector(bs, smallSideRGs); + bs << largeSideRG; + bs << joinedRG; // TODO: I think we can omit joinedRG if (!(fe2 || aggregatorPM)) + // cout << "joined RG: " << joinedRG.toString() << endl; + } + } + } + + bs << filterCount; + + for (i = 0; i < filterCount; ++i) + { + // cout << "serializing step " << i << endl; + filterSteps[i]->createCommand(bs); + } + + bs << projectCount; + + for (i = 0; i < projectCount; ++i) + projectSteps[i]->createCommand(bs); + + // aggregate step only when output is row group + if (ot == ROW_GROUP) + { + if (aggregatorPM.get() != NULL) { - primprocRG.reset(new RowGroup[threadCount]); - - for (uint32_t i = 0; i < threadCount; i++) - if (aggregatorPM) - primprocRG[i] = aggregateRGPM; - else if (fe2) - primprocRG[i] = fe2Output; - //This shouldn't be necessary. As of 2-17-14, PrimProc - //will only send joined results if fe2 || aggregatorPM, - //so it will never send back data for joinedRG. - //else if ((flags & HAS_JOINER) && sendTupleJoinRowGroupData) - // primprocRG[i] = joinedRG; - else - primprocRG[i] = projectionRG; + bs << (uint8_t)1; + bs << aggregateRGPM; + bs << *(aggregatorPM.get()); } + else + { + bs << (uint8_t)0; + } + } + // decide which rowgroup is received by PrimProc + if (ot == ROW_GROUP) + { + primprocRG.reset(new RowGroup[threadCount]); + + for (uint32_t i = 0; i < threadCount; i++) + if (aggregatorPM) + primprocRG[i] = aggregateRGPM; + else if (fe2) + primprocRG[i] = fe2Output; + // This shouldn't be necessary. As of 2-17-14, PrimProc + // will only send joined results if fe2 || aggregatorPM, + // so it will never send back data for joinedRG. + // else if ((flags & HAS_JOINER) && sendTupleJoinRowGroupData) + // primprocRG[i] = joinedRG; + else + primprocRG[i] = projectionRG; + } } /** @@ -1229,203 +1223,203 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const void BatchPrimitiveProcessorJL::runBPP(ByteStream& bs, uint32_t pmNum) { - ISMPacketHeader ism; - uint32_t i; + ISMPacketHeader ism; + uint32_t i; - /* XXXPAT: BPPJL currently reuses the ism Size fields for other things to - save bandwidth. They're completely unused otherwise. We need to throw out all unused - fields of every defined header. */ + /* XXXPAT: BPPJL currently reuses the ism Size fields for other things to + save bandwidth. They're completely unused otherwise. We need to throw out all unused + fields of every defined header. */ - bs.restart(); + bs.restart(); - memset((void*)&ism, 0, sizeof(ism)); - ism.Command = BATCH_PRIMITIVE_RUN; + memset((void*)&ism, 0, sizeof(ism)); + ism.Command = BATCH_PRIMITIVE_RUN; - // TODO: this causes the code using this ism to send on only one socket, even if more than one socket is defined for each PM. - ism.Interleave = pmNum; -// ism.Interleave = pmNum - 1; + // TODO: this causes the code using this ism to send on only one socket, even if more than one socket is + // defined for each PM. + ism.Interleave = pmNum; + // ism.Interleave = pmNum - 1; - /* ... and the Size field is used for the "weight" of processing a BPP - where 1 is one Command on one logical block. */ - ism.Size = count * (filterCount + projectCount); + /* ... and the Size field is used for the "weight" of processing a BPP + where 1 is one Command on one logical block. */ + ism.Size = count * (filterCount + projectCount); - bs.append((uint8_t*) &ism, sizeof(ism)); + bs.append((uint8_t*)&ism, sizeof(ism)); - /* The next 4 vars are for BPPSeeder; BPP itself skips them */ - bs << sessionID; - bs << stepID; - bs << uniqueID; - bs << _priority; + /* The next 4 vars are for BPPSeeder; BPP itself skips them */ + bs << sessionID; + bs << stepID; + bs << uniqueID; + bs << _priority; - bs << dbRoot; - bs << count; + bs << dbRoot; + bs << count; - if (_hasScan) - idbassert(ridCount == 0); - else if (!sendRowGroups) - idbassert(ridCount > 0 && (ridMap != 0 || sendAbsRids)); - else - idbassert(inputRG.getRowCount() > 0); + if (_hasScan) + idbassert(ridCount == 0); + else if (!sendRowGroups) + idbassert(ridCount > 0 && (ridMap != 0 || sendAbsRids)); + else + idbassert(inputRG.getRowCount() > 0); - if (sendRowGroups) - { - uint32_t rgSize = inputRG.getDataSize(); - bs << rgSize; - bs.append(inputRG.getData(), rgSize); - } + if (sendRowGroups) + { + uint32_t rgSize = inputRG.getDataSize(); + bs << rgSize; + bs.append(inputRG.getData(), rgSize); + } + else + { + bs << ridCount; + + if (sendAbsRids) + bs.append((uint8_t*)absRids.get(), ridCount << 3); else { - bs << ridCount; - - if (sendAbsRids) - bs.append((uint8_t*) absRids.get(), ridCount << 3); - else - { - bs << ridMap; - bs << baseRid; - bs.append((uint8_t*) relRids, ridCount << 1); - } - - if (sendValues) - bs.append((uint8_t*) values, ridCount << 3); + bs << ridMap; + bs << baseRid; + bs.append((uint8_t*)relRids, ridCount << 1); } - for (i = 0; i < filterCount; i++) - filterSteps[i]->runCommand(bs); + if (sendValues) + bs.append((uint8_t*)values, ridCount << 3); + } - for (i = 0; i < projectCount; i++) - projectSteps[i]->runCommand(bs); + for (i = 0; i < filterCount; i++) + filterSteps[i]->runCommand(bs); + + for (i = 0; i < projectCount; i++) + projectSteps[i]->runCommand(bs); } - void BatchPrimitiveProcessorJL::runErrorBPP(ByteStream& bs) { - ISMPacketHeader ism; - bs.restart(); + ISMPacketHeader ism; + bs.restart(); - memset((void*)&ism, 0, sizeof(ism)); - ism.Command = BATCH_PRIMITIVE_RUN; - ism.Status = status; - ism.Size = sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader); + memset((void*)&ism, 0, sizeof(ism)); + ism.Command = BATCH_PRIMITIVE_RUN; + ism.Status = status; + ism.Size = sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader); - bs.append((uint8_t*) &ism, sizeof(ism)); + bs.append((uint8_t*)&ism, sizeof(ism)); - bs << (messageqcpp::ByteStream::quadbyte)sessionID; - bs << (messageqcpp::ByteStream::quadbyte)stepID; - bs << uniqueID; - bs << count; - bs << ridCount; + bs << (messageqcpp::ByteStream::quadbyte)sessionID; + bs << (messageqcpp::ByteStream::quadbyte)stepID; + bs << uniqueID; + bs << count; + bs << ridCount; } void BatchPrimitiveProcessorJL::destroyBPP(ByteStream& bs) const { - ISMPacketHeader ism; + ISMPacketHeader ism; -// if (!(sessionID & 0x80000000)) -// cout << "step ID " << stepID << " added " << rowCounter << " rows, processed " -// << rowsProcessed << " rows" << endl; + // if (!(sessionID & 0x80000000)) + // cout << "step ID " << stepID << " added " << rowCounter << " rows, processed " + // << rowsProcessed << " rows" << endl; - memset((void*)&ism, 0, sizeof(ism)); - ism.Command = BATCH_PRIMITIVE_DESTROY; + memset((void*)&ism, 0, sizeof(ism)); + ism.Command = BATCH_PRIMITIVE_DESTROY; - bs.append((uint8_t*) &ism, sizeof(ism)); - /* XXXPAT: We can get rid of sessionID and stepID; - it's there for easier debugging only */ - bs << (messageqcpp::ByteStream::quadbyte)sessionID; - bs << (messageqcpp::ByteStream::quadbyte)stepID; - bs << uniqueID; + bs.append((uint8_t*)&ism, sizeof(ism)); + /* XXXPAT: We can get rid of sessionID and stepID; + it's there for easier debugging only */ + bs << (messageqcpp::ByteStream::quadbyte)sessionID; + bs << (messageqcpp::ByteStream::quadbyte)stepID; + bs << uniqueID; } void BatchPrimitiveProcessorJL::useJoiners(const vector >& j) { - pos = 0; - joinerNum = 0; - tJoiners = j; + pos = 0; + joinerNum = 0; + tJoiners = j; - PMJoinerCount = 0; - tlKeyLens.reset(new uint32_t[tJoiners.size()]); + PMJoinerCount = 0; + tlKeyLens.reset(new uint32_t[tJoiners.size()]); - for (uint32_t i = 0; i < tJoiners.size(); i++) + for (uint32_t i = 0; i < tJoiners.size(); i++) + { + if (tJoiners[i]->inPM()) { - if (tJoiners[i]->inPM()) - { - PMJoinerCount++; - smallSideKeys.push_back(tJoiners[i]->getSmallKeyColumns()); - smallSideRGs.push_back(tJoiners[i]->getSmallRG()); + PMJoinerCount++; + smallSideKeys.push_back(tJoiners[i]->getSmallKeyColumns()); + smallSideRGs.push_back(tJoiners[i]->getSmallRG()); - if (tJoiners[i]->isTypelessJoin()) - tlKeyLens[i] = tJoiners[i]->getKeyLength(); + if (tJoiners[i]->isTypelessJoin()) + tlKeyLens[i] = tJoiners[i]->getKeyLength(); - if (tJoiners[i]->hasFEFilter()) - sendTupleJoinRowGroupData = true; - - if (tJoiners[i]->smallOuterJoin()) - hasSmallOuterJoin = true; - } - } - - largeSideRG = tJoiners[0]->getLargeRG(); - - if (aggregatorPM || fe2) - { + if (tJoiners[i]->hasFEFilter()) sendTupleJoinRowGroupData = true; -#ifdef JLF_DEBUG - cout << "will send small side row data\n"; -#endif + + if (tJoiners[i]->smallOuterJoin()) + hasSmallOuterJoin = true; } - posByJoinerNum.reset(new uint32_t[PMJoinerCount]); - memset(posByJoinerNum.get(), 0, PMJoinerCount * sizeof(uint32_t)); + } + + largeSideRG = tJoiners[0]->getLargeRG(); + + if (aggregatorPM || fe2) + { + sendTupleJoinRowGroupData = true; +#ifdef JLF_DEBUG + cout << "will send small side row data\n"; +#endif + } + posByJoinerNum.reset(new uint32_t[PMJoinerCount]); + memset(posByJoinerNum.get(), 0, PMJoinerCount * sizeof(uint32_t)); } // helper fcn to interleave small side data by joinernum bool BatchPrimitiveProcessorJL::pickNextJoinerNum() { - uint i; - // find the next joiner that still has more data to send. Set joinerNum & pos. - for (i = 0; i < PMJoinerCount; i++) - { - joinerNum = (joinerNum + 1) % PMJoinerCount; - if (posByJoinerNum[joinerNum] != tJoiners[joinerNum]->getSmallSide()->size()) - break; - } - if (i == PMJoinerCount) - return false; - pos = posByJoinerNum[joinerNum]; - return true; + uint i; + // find the next joiner that still has more data to send. Set joinerNum & pos. + for (i = 0; i < PMJoinerCount; i++) + { + joinerNum = (joinerNum + 1) % PMJoinerCount; + if (posByJoinerNum[joinerNum] != tJoiners[joinerNum]->getSmallSide()->size()) + break; + } + if (i == PMJoinerCount) + return false; + pos = posByJoinerNum[joinerNum]; + return true; } - + /* This algorithm relies on the joiners being sorted by size atm */ /* XXXPAT: Going to interleave across joiners to take advantage of the new locking env in PrimProc */ bool BatchPrimitiveProcessorJL::nextTupleJoinerMsg(ByteStream& bs) { - uint32_t size = 0, toSend, i, j; - ISMPacketHeader ism; - Row r; - vector* tSmallSide; - joiner::TypelessData tlData; - uint32_t smallKeyCol; - uint32_t largeKeyCol; - uint64_t smallkey; - bool isNull; - bool bSignedUnsigned; - - bool moreMsgs = pickNextJoinerNum(); - - if (!moreMsgs) - { - /* last message */ -// cout << "sending last joiner msg\n"; - ism.Command = BATCH_PRIMITIVE_END_JOINER; - bs.load((uint8_t*) &ism, sizeof(ism)); - bs << (messageqcpp::ByteStream::quadbyte)sessionID; - bs << (messageqcpp::ByteStream::quadbyte)stepID; - bs << uniqueID; - return false; - } - - memset((void*)&ism, 0, sizeof(ism)); - tSmallSide = tJoiners[joinerNum]->getSmallSide(); - size = tSmallSide->size(); + uint32_t size = 0, toSend, i, j; + ISMPacketHeader ism; + Row r; + vector* tSmallSide; + joiner::TypelessData tlData; + uint32_t smallKeyCol; + uint32_t largeKeyCol; + uint64_t smallkey; + bool isNull; + bool bSignedUnsigned; + + bool moreMsgs = pickNextJoinerNum(); + + if (!moreMsgs) + { + /* last message */ + // cout << "sending last joiner msg\n"; + ism.Command = BATCH_PRIMITIVE_END_JOINER; + bs.load((uint8_t*)&ism, sizeof(ism)); + bs << (messageqcpp::ByteStream::quadbyte)sessionID; + bs << (messageqcpp::ByteStream::quadbyte)stepID; + bs << uniqueID; + return false; + } + + memset((void*)&ism, 0, sizeof(ism)); + tSmallSide = tJoiners[joinerNum]->getSmallSide(); + size = tSmallSide->size(); #if 0 if (joinerNum == PMJoinerCount - 1 && pos == size) @@ -1448,322 +1442,322 @@ bool BatchPrimitiveProcessorJL::nextTupleJoinerMsg(ByteStream& bs) pos = 0; } #endif - - ism.Command = BATCH_PRIMITIVE_ADD_JOINER; - bs.load((uint8_t*) &ism, sizeof(ism)); - bs << (messageqcpp::ByteStream::quadbyte)sessionID; - bs << (messageqcpp::ByteStream::quadbyte)stepID; - bs << uniqueID; - smallSideRGs[joinerNum].initRow(&r); + ism.Command = BATCH_PRIMITIVE_ADD_JOINER; + bs.load((uint8_t*)&ism, sizeof(ism)); + bs << (messageqcpp::ByteStream::quadbyte)sessionID; + bs << (messageqcpp::ByteStream::quadbyte)stepID; + bs << uniqueID; - unsigned metasize = 12; //12 = sizeof(struct JoinerElements) + smallSideRGs[joinerNum].initRow(&r); - if (tJoiners[joinerNum]->isTypelessJoin()) metasize = 0; + unsigned metasize = 12; // 12 = sizeof(struct JoinerElements) - unsigned rowunits = fJoinerChunkSize / (r.getSize() + metasize); - toSend = std::min(size - pos, rowunits); - bs << toSend; - bs << pos; - bs << joinerNum; + if (tJoiners[joinerNum]->isTypelessJoin()) + metasize = 0; - if (tJoiners[joinerNum]->isTypelessJoin()) + unsigned rowunits = fJoinerChunkSize / (r.getSize() + metasize); + toSend = std::min(size - pos, rowunits); + bs << toSend; + bs << pos; + bs << joinerNum; + + if (tJoiners[joinerNum]->isTypelessJoin()) + { + utils::FixedAllocator fa(tlKeyLens[joinerNum], true); + + for (i = pos; i < pos + toSend; i++) { - utils::FixedAllocator fa(tlKeyLens[joinerNum], true); + r.setPointer((*tSmallSide)[i]); + isNull = false; + bSignedUnsigned = tJoiners[joinerNum]->isSignedUnsignedJoin(); - for (i = pos; i < pos + toSend; i++) + for (j = 0; j < smallSideKeys[joinerNum].size(); j++) + { + isNull |= r.isNullValue(smallSideKeys[joinerNum][j]); + + if (UNLIKELY(bSignedUnsigned)) { - r.setPointer((*tSmallSide)[i]); - isNull = false; - bSignedUnsigned = tJoiners[joinerNum]->isSignedUnsignedJoin(); + // BUG 5628 If this is a signed/unsigned join column and the sign bit is set on either side, + // then it should not compare. Send null to PM to prevent compare + smallKeyCol = smallSideKeys[joinerNum][j]; + largeKeyCol = tJoiners[joinerNum]->getLargeKeyColumns()[j]; - for (j = 0; j < smallSideKeys[joinerNum].size(); j++) - { - isNull |= r.isNullValue(smallSideKeys[joinerNum][j]); - - if (UNLIKELY(bSignedUnsigned)) - { - // BUG 5628 If this is a signed/unsigned join column and the sign bit is set on either side, - // then it should not compare. Send null to PM to prevent compare - smallKeyCol = smallSideKeys[joinerNum][j]; - largeKeyCol = tJoiners[joinerNum]->getLargeKeyColumns()[j]; - - if (r.isUnsigned(smallKeyCol) != largeSideRG.isUnsigned(largeKeyCol)) - { - if (r.isUnsigned(smallKeyCol)) - smallkey = r.getUintField(smallKeyCol); - else - smallkey = r.getIntField(smallKeyCol); - - if (smallkey & 0x8000000000000000ULL) - { - isNull = true; - break; - } - } - } - } - - if (!isNull) - { - tlData = makeTypelessKey(r, smallSideKeys[joinerNum], - tlKeyLens[joinerNum], &fa, - largeSideRG, tJoiners[joinerNum]->getLargeKeyColumns()); - if (tlData.len == 0) - { - isNull = true; - } - } - bs << (uint8_t) isNull; - if (!isNull) - { - tlData.serialize(bs); - bs << i; - } - } - } - else - { -#pragma pack(push,1) - struct JoinerElements - { - int64_t key; - uint32_t value; - } *arr; -#pragma pack(pop) - bs.needAtLeast(toSend * sizeof(JoinerElements)); - arr = (JoinerElements*) bs.getInputPtr(); - - smallKeyCol = smallSideKeys[joinerNum][0]; - bSignedUnsigned = r.isUnsigned(smallKeyCol) != largeSideRG.isUnsigned(tJoiners[joinerNum]->getLargeKeyColumns()[0]); - j = 0; - - for (i = pos, j = 0; i < pos + toSend; ++i, ++j) - { - r.setPointer((*tSmallSide)[i]); - - if ( r.getColType(smallKeyCol)== CalpontSystemCatalog::LONGDOUBLE) - { - // Small side is a long double. Since CS can't store larger than DOUBLE, - // we need to convert to whatever type large side is -- double or int64 - long double smallkeyld = r.getLongDoubleField(smallKeyCol); - switch (largeSideRG.getColType(tJoiners[joinerNum]->getLargeKeyColumns()[0])) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - if (smallkeyld > MAX_DOUBLE || smallkeyld < MIN_DOUBLE) - { - smallkey = joblist::UBIGINTEMPTYROW; - } - else - { - double d = (double)smallkeyld; - smallkey = *(int64_t*)&d; - } - break; - } - default: - { - if (r.isUnsigned(smallKeyCol) && smallkeyld > MAX_UBIGINT) - { - smallkey = joblist::UBIGINTEMPTYROW; - } - else if (smallkeyld > MAX_BIGINT || smallkeyld < MIN_BIGINT) - { - smallkey = joblist::UBIGINTEMPTYROW; - } - else - { - smallkey = (int64_t)smallkeyld; - } - break; - } - } - } - else if (r.isUnsigned(smallKeyCol)) - smallkey = r.getUintField(smallKeyCol); + if (r.isUnsigned(smallKeyCol) != largeSideRG.isUnsigned(largeKeyCol)) + { + if (r.isUnsigned(smallKeyCol)) + smallkey = r.getUintField(smallKeyCol); else - smallkey = r.getIntField(smallKeyCol); + smallkey = r.getIntField(smallKeyCol); - // If this is a compare signed vs unsigned and the sign bit is on for this value, then all compares - // against the large side should fall. UBIGINTEMPTYROW is not a valid value, so nothing will match. - if (bSignedUnsigned && (smallkey & 0x8000000000000000ULL)) - smallkey = joblist::UBIGINTEMPTYROW; - - arr[j].key = (int64_t)smallkey; - arr[j].value = i; -// cout << "sending " << arr[j].key << ", " << arr[j].value << endl; + if (smallkey & 0x8000000000000000ULL) + { + isNull = true; + break; + } + } } + } - bs.advanceInputPtr(toSend * sizeof(JoinerElements)); - } - - if (sendTupleJoinRowGroupData) - { - RowGroup& smallSide = smallSideRGs[joinerNum]; - RGData tmpData(smallSide, toSend); - Row tmpRow; - - smallSide.setData(&tmpData); - smallSide.initRow(&tmpRow); - smallSide.getRow(0, &tmpRow); - - for (i = pos; i < pos + toSend; i++, tmpRow.nextRow()) + if (!isNull) + { + tlData = makeTypelessKey(r, smallSideKeys[joinerNum], tlKeyLens[joinerNum], &fa, largeSideRG, + tJoiners[joinerNum]->getLargeKeyColumns()); + if (tlData.len == 0) { - r.setPointer((*tSmallSide)[i]); - copyRow(r, &tmpRow); + isNull = true; } + } + bs << (uint8_t)isNull; + if (!isNull) + { + tlData.serialize(bs); + bs << i; + } + } + } + else + { +#pragma pack(push, 1) + struct JoinerElements + { + int64_t key; + uint32_t value; + } * arr; +#pragma pack(pop) + bs.needAtLeast(toSend * sizeof(JoinerElements)); + arr = (JoinerElements*)bs.getInputPtr(); - smallSide.setRowCount(toSend); - tmpData.serialize(bs, smallSide.getDataSize()); + smallKeyCol = smallSideKeys[joinerNum][0]; + bSignedUnsigned = + r.isUnsigned(smallKeyCol) != largeSideRG.isUnsigned(tJoiners[joinerNum]->getLargeKeyColumns()[0]); + j = 0; + + for (i = pos, j = 0; i < pos + toSend; ++i, ++j) + { + r.setPointer((*tSmallSide)[i]); + + if (r.getColType(smallKeyCol) == CalpontSystemCatalog::LONGDOUBLE) + { + // Small side is a long double. Since CS can't store larger than DOUBLE, + // we need to convert to whatever type large side is -- double or int64 + long double smallkeyld = r.getLongDoubleField(smallKeyCol); + switch (largeSideRG.getColType(tJoiners[joinerNum]->getLargeKeyColumns()[0])) + { + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + if (smallkeyld > MAX_DOUBLE || smallkeyld < MIN_DOUBLE) + { + smallkey = joblist::UBIGINTEMPTYROW; + } + else + { + double d = (double)smallkeyld; + smallkey = *(int64_t*)&d; + } + break; + } + default: + { + if (r.isUnsigned(smallKeyCol) && smallkeyld > MAX_UBIGINT) + { + smallkey = joblist::UBIGINTEMPTYROW; + } + else if (smallkeyld > MAX_BIGINT || smallkeyld < MIN_BIGINT) + { + smallkey = joblist::UBIGINTEMPTYROW; + } + else + { + smallkey = (int64_t)smallkeyld; + } + break; + } + } + } + else if (r.isUnsigned(smallKeyCol)) + smallkey = r.getUintField(smallKeyCol); + else + smallkey = r.getIntField(smallKeyCol); + + // If this is a compare signed vs unsigned and the sign bit is on for this value, then all compares + // against the large side should fall. UBIGINTEMPTYROW is not a valid value, so nothing will match. + if (bSignedUnsigned && (smallkey & 0x8000000000000000ULL)) + smallkey = joblist::UBIGINTEMPTYROW; + + arr[j].key = (int64_t)smallkey; + arr[j].value = i; + // cout << "sending " << arr[j].key << ", " << arr[j].value << endl; } - pos += toSend; - posByJoinerNum[joinerNum] = pos; - return true; + bs.advanceInputPtr(toSend * sizeof(JoinerElements)); + } + + if (sendTupleJoinRowGroupData) + { + RowGroup& smallSide = smallSideRGs[joinerNum]; + RGData tmpData(smallSide, toSend); + Row tmpRow; + + smallSide.setData(&tmpData); + smallSide.initRow(&tmpRow); + smallSide.getRow(0, &tmpRow); + + for (i = pos; i < pos + toSend; i++, tmpRow.nextRow()) + { + r.setPointer((*tSmallSide)[i]); + copyRow(r, &tmpRow); + } + + smallSide.setRowCount(toSend); + tmpData.serialize(bs, smallSide.getDataSize()); + } + + pos += toSend; + posByJoinerNum[joinerNum] = pos; + return true; } void BatchPrimitiveProcessorJL::setProjectionRowGroup(const rowgroup::RowGroup& rg) { - ot = ROW_GROUP; - projectionRG = rg; + ot = ROW_GROUP; + projectionRG = rg; } void BatchPrimitiveProcessorJL::setJoinedRowGroup(const rowgroup::RowGroup& rg) { - joinedRG = rg; + joinedRG = rg; } void BatchPrimitiveProcessorJL::setInputRowGroup(const rowgroup::RowGroup& rg) { - sendRowGroups = true; - sendAbsRids = false; - sendValues = false; - inputRG = rg; + sendRowGroups = true; + sendAbsRids = false; + sendValues = false; + inputRG = rg; } void BatchPrimitiveProcessorJL::addAggregateStep(const rowgroup::SP_ROWAGG_PM_t& aggpm, - const rowgroup::RowGroup& argpm) + const rowgroup::RowGroup& argpm) { - aggregatorPM = aggpm; - aggregateRGPM = argpm; + aggregatorPM = aggpm; + aggregateRGPM = argpm; - if (tJoiners.size() > 0) - sendTupleJoinRowGroupData = true; + if (tJoiners.size() > 0) + sendTupleJoinRowGroupData = true; } /* OR hacks */ void BatchPrimitiveProcessorJL::setBOP(uint32_t op) { - bop = op; + bop = op; - if (op == BOP_OR && filterCount > 1) + if (op == BOP_OR && filterCount > 1) + { + for (int i = 1; i < filterCount; ++i) { - for (int i = 1; i < filterCount; ++i) - { - ColumnCommandJL* colcmd = dynamic_cast(filterSteps[i].get()); + ColumnCommandJL* colcmd = dynamic_cast(filterSteps[i].get()); - if (colcmd != NULL) - colcmd->scan(false); - } + if (colcmd != NULL) + colcmd->scan(false); } + } } void BatchPrimitiveProcessorJL::setForHJ(bool b) { - forHJ = b; + forHJ = b; } void BatchPrimitiveProcessorJL::setFEGroup1(boost::shared_ptr fe, - const RowGroup& input) + const RowGroup& input) { - fe1 = fe; - fe1Input = input; + fe1 = fe; + fe1Input = input; } void BatchPrimitiveProcessorJL::setFEGroup2(boost::shared_ptr fe, - const RowGroup& output) + const RowGroup& output) { - fe2 = fe; - fe2Output = output; + fe2 = fe; + fe2Output = output; - if (tJoiners.size() > 0 && PMJoinerCount > 0) - sendTupleJoinRowGroupData = true; + if (tJoiners.size() > 0 && PMJoinerCount > 0) + sendTupleJoinRowGroupData = true; } const string BatchPrimitiveProcessorJL::toMiniString() const { - ostringstream oss; - int i; - set colset; - string col; + ostringstream oss; + int i; + set colset; + string col; - for (i = 0; i < filterCount; i++) + for (i = 0; i < filterCount; i++) + { + col = filterSteps[i]->getColName(); + // FilterCommandJL has two referenced columns, needs special handling. + FilterCommandJL* filterCmd = dynamic_cast(filterSteps[i].get()); + + if (filterCmd == NULL) { - col = filterSteps[i]->getColName(); - // FilterCommandJL has two referenced columns, needs special handling. - FilterCommandJL* filterCmd = dynamic_cast(filterSteps[i].get()); - - if (filterCmd == NULL) - { - colset.insert(col); - } - else - { - // is a FilterCommandJL - size_t sep = col.find(','); - colset.insert(col.substr(0, sep)); - - if (sep != string::npos) - colset.insert(col.substr(++sep)); - } + colset.insert(col); } - - for (i = 0; i < projectCount; i++) + else { - col = projectSteps[i]->getColName(); - colset.insert(col); + // is a FilterCommandJL + size_t sep = col.find(','); + colset.insert(col.substr(0, sep)); + + if (sep != string::npos) + colset.insert(col.substr(++sep)); } + } - set::iterator iter = colset.begin(); - oss << '(' << *iter++; + for (i = 0; i < projectCount; i++) + { + col = projectSteps[i]->getColName(); + colset.insert(col); + } - while (iter != colset.end()) - oss << ',' << *iter++; + set::iterator iter = colset.begin(); + oss << '(' << *iter++; - oss << ')'; + while (iter != colset.end()) + oss << ',' << *iter++; - return oss.str(); + oss << ')'; + + return oss.str(); } void BatchPrimitiveProcessorJL::setJoinFERG(const RowGroup& rg) { - joinFERG = rg; + joinFERG = rg; } void BatchPrimitiveProcessorJL::abortProcessing(ByteStream* bs) { - ISMPacketHeader ism; + ISMPacketHeader ism; - memset((void*)&ism, 0, sizeof(ism)); - ism.Command = BATCH_PRIMITIVE_ABORT; + memset((void*)&ism, 0, sizeof(ism)); + ism.Command = BATCH_PRIMITIVE_ABORT; - bs->load((uint8_t*) &ism, sizeof(ism)); - *bs << uniqueID; + bs->load((uint8_t*)&ism, sizeof(ism)); + *bs << uniqueID; } void BatchPrimitiveProcessorJL::deliverStringTableRowGroup(bool b) { - if (aggregatorPM) - aggregateRGPM.setUseStringTable(b); - else if (fe2) - fe2Output.setUseStringTable(b); - else - projectionRG.setUseStringTable(b); + if (aggregatorPM) + aggregateRGPM.setUseStringTable(b); + else if (fe2) + fe2Output.setUseStringTable(b); + else + projectionRG.setUseStringTable(b); } - -} +} // namespace joblist diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index cdc03e262..46c8a5839 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -48,328 +48,324 @@ namespace joblist { -const uint32_t LOGICAL_BLOCKS_CONVERTER = 23; // 10 + 13. 13 to convert to logical blocks, +const uint32_t LOGICAL_BLOCKS_CONVERTER = 23; // 10 + 13. 13 to convert to logical blocks, // 10 to convert to groups of 1024 logical blocks -const uint8_t CP_FLAG_AND_LBID = 9; // # bytes used for CP boolean and the lbid - // used by BatchPrimitiveProcessorJL::countThisMsg() +const uint8_t CP_FLAG_AND_LBID = 9; // # bytes used for CP boolean and the lbid + // used by BatchPrimitiveProcessorJL::countThisMsg() // forward reference struct JobInfo; class BatchPrimitiveProcessorJL { -public: - /* Constructor used by the JobStep */ - explicit BatchPrimitiveProcessorJL(const ResourceManager* rm); - ~BatchPrimitiveProcessorJL(); + public: + /* Constructor used by the JobStep */ + explicit BatchPrimitiveProcessorJL(const ResourceManager* rm); + ~BatchPrimitiveProcessorJL(); - /* Interface used by the JobStep */ + /* Interface used by the JobStep */ - /* Some accessors */ - inline bool hasScan() - { - return _hasScan; - } + /* Some accessors */ + inline bool hasScan() + { + return _hasScan; + } - /* For initializing the object */ - inline void setSessionID(uint32_t num) - { - sessionID = num; - } - inline void setStepID(uint32_t num) - { - stepID = num; - } - inline void setUniqueID(uint32_t id) - { - uniqueID = id; - } - inline void setQueryContext(const BRM::QueryContext& qc) - { - versionInfo = qc; - } - inline void setTxnID(uint32_t num) - { - txnID = num; - } - inline void setOutputType(BPSOutputType o) - { - ot = o; + /* For initializing the object */ + inline void setSessionID(uint32_t num) + { + sessionID = num; + } + inline void setStepID(uint32_t num) + { + stepID = num; + } + inline void setUniqueID(uint32_t id) + { + uniqueID = id; + } + inline void setQueryContext(const BRM::QueryContext& qc) + { + versionInfo = qc; + } + inline void setTxnID(uint32_t num) + { + txnID = num; + } + inline void setOutputType(BPSOutputType o) + { + ot = o; - if (ot == TUPLE || ot == ROW_GROUP) needRidsAtDelivery = true; - } - inline void setNeedRidsAtDelivery(bool b) - { - needRidsAtDelivery = b; - } - inline void setTraceFlags(uint32_t flags) - { - LBIDTrace = ((flags & execplan::CalpontSelectExecutionPlan::TRACE_LBIDS) != 0); - } - inline uint32_t getRidCount() - { - return ridCount; - } - inline void setThreadCount(uint32_t tc) - { - threadCount = tc; - } + if (ot == TUPLE || ot == ROW_GROUP) + needRidsAtDelivery = true; + } + inline void setNeedRidsAtDelivery(bool b) + { + needRidsAtDelivery = b; + } + inline void setTraceFlags(uint32_t flags) + { + LBIDTrace = ((flags & execplan::CalpontSelectExecutionPlan::TRACE_LBIDS) != 0); + } + inline uint32_t getRidCount() + { + return ridCount; + } + inline void setThreadCount(uint32_t tc) + { + threadCount = tc; + } - void addFilterStep(const pColScanStep&, std::vector lastScannedLBID); - void addFilterStep(const PseudoColStep&); - void addFilterStep(const pColStep&); - void addFilterStep(const pDictionaryStep&); - void addFilterStep(const FilterStep&); - void addProjectStep(const PseudoColStep&); - void addProjectStep(const pColStep&); - void addProjectStep(const PassThruStep&); - void addProjectStep(const pColStep&, const pDictionaryStep&); - void addProjectStep(const PassThruStep&, const pDictionaryStep&); + void addFilterStep(const pColScanStep&, std::vector lastScannedLBID); + void addFilterStep(const PseudoColStep&); + void addFilterStep(const pColStep&); + void addFilterStep(const pDictionaryStep&); + void addFilterStep(const FilterStep&); + void addProjectStep(const PseudoColStep&); + void addProjectStep(const pColStep&); + void addProjectStep(const PassThruStep&); + void addProjectStep(const pColStep&, const pDictionaryStep&); + void addProjectStep(const PassThruStep&, const pDictionaryStep&); - void createBPP(messageqcpp::ByteStream&) const; - void destroyBPP(messageqcpp::ByteStream&) const; + void createBPP(messageqcpp::ByteStream&) const; + void destroyBPP(messageqcpp::ByteStream&) const; - /* Call this one last */ - // void addDeliveryStep(const DeliveryStep &); + /* Call this one last */ + // void addDeliveryStep(const DeliveryStep &); - /* At runtime, feed input here */ - void addElementType(const ElementType&, uint32_t dbroot); - void addElementType(const StringElementType&, uint32_t dbroot); - //void setRowGroupData(const rowgroup::RowGroup &); + /* At runtime, feed input here */ + void addElementType(const ElementType&, uint32_t dbroot); + void addElementType(const StringElementType&, uint32_t dbroot); + // void setRowGroupData(const rowgroup::RowGroup &); - void runBPP(messageqcpp::ByteStream&, uint32_t pmNum); - void abortProcessing(messageqcpp::ByteStream*); + void runBPP(messageqcpp::ByteStream&, uint32_t pmNum); + void abortProcessing(messageqcpp::ByteStream*); - /* After serializing a BPP object, reset it and it's ready for more input */ - void reset(); + /* After serializing a BPP object, reset it and it's ready for more input */ + void reset(); - /* The JobStep calls these to initialize a BPP that starts with a column scan */ - void setLBID(uint64_t lbid, const BRM::EMEntry& scannedExtent); - inline void setCount(uint16_t c) - { - idbassert(c > 0); - count = c; - } + /* The JobStep calls these to initialize a BPP that starts with a column scan */ + void setLBID(uint64_t lbid, const BRM::EMEntry& scannedExtent); + inline void setCount(uint16_t c) + { + idbassert(c > 0); + count = c; + } - /* Turn a ByteStream into ElementTypes or StringElementTypes */ - void getElementTypes(messageqcpp::ByteStream& in, std::vector* out, - bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, - uint32_t* touchedBlocks) const; - void getStringElementTypes(messageqcpp::ByteStream& in, - std::vector* out, bool* validCPData, uint64_t* lbid, - int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, - uint32_t* touchedBlocks) const; - /* (returns the row count) */ -// uint32_t getTableBand(messageqcpp::ByteStream &in, messageqcpp::ByteStream *out, -// bool *validCPData, uint64_t *lbid, int64_t *min, int64_t *max, -// uint32_t *cachedIO, uint32_t *physIO, uint32_t *touchedBlocks) const; - void getTuples(messageqcpp::ByteStream& in, std::vector* out, - bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, - uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const; - void deserializeAggregateResults(messageqcpp::ByteStream* in, - std::vector* out) const; - void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, - bool* validCPData, uint64_t* lbid, int128_t* min, int128_t* max, - uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID, bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const; - void deserializeAggregateResult(messageqcpp::ByteStream* in, - std::vector* out) const; - bool countThisMsg(messageqcpp::ByteStream& in) const; + /* Turn a ByteStream into ElementTypes or StringElementTypes */ + void getElementTypes(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, + uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks) const; + void getStringElementTypes(messageqcpp::ByteStream& in, std::vector* out, + bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, + uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const; + /* (returns the row count) */ + // uint32_t getTableBand(messageqcpp::ByteStream &in, messageqcpp::ByteStream *out, + // bool *validCPData, uint64_t *lbid, int64_t *min, int64_t *max, + // uint32_t *cachedIO, uint32_t *physIO, uint32_t *touchedBlocks) const; + void getTuples(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, uint64_t* lbid, + int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks) const; + void deserializeAggregateResults(messageqcpp::ByteStream* in, std::vector* out) const; + void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, + uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, bool* hasBinaryColumn, + const execplan::CalpontSystemCatalog::ColType& colType) const; + void deserializeAggregateResult(messageqcpp::ByteStream* in, std::vector* out) const; + bool countThisMsg(messageqcpp::ByteStream& in) const; - void setStatus(uint16_t s) - { - status = s; - } - uint16_t getStatus() const - { - return status; - } - void runErrorBPP(messageqcpp::ByteStream&); -// uint32_t getErrorTableBand(uint16_t error, messageqcpp::ByteStream *out); -// boost::shared_array getErrorRowGroupData(uint16_t error) const; - rowgroup::RGData getErrorRowGroupData(uint16_t error) const; + void setStatus(uint16_t s) + { + status = s; + } + uint16_t getStatus() const + { + return status; + } + void runErrorBPP(messageqcpp::ByteStream&); + // uint32_t getErrorTableBand(uint16_t error, messageqcpp::ByteStream *out); + // boost::shared_array getErrorRowGroupData(uint16_t error) const; + rowgroup::RGData getErrorRowGroupData(uint16_t error) const; - // @bug 1098 - std::vector& getFilterSteps() - { - return filterSteps; - } - std::vector& getProjectSteps() - { - return projectSteps; - } + // @bug 1098 + std::vector& getFilterSteps() + { + return filterSteps; + } + std::vector& getProjectSteps() + { + return projectSteps; + } - std::string toString() const; + std::string toString() const; - /* RowGroup additions */ - void setProjectionRowGroup(const rowgroup::RowGroup& rg); - void setInputRowGroup(const rowgroup::RowGroup& rg); + /* RowGroup additions */ + void setProjectionRowGroup(const rowgroup::RowGroup& rg); + void setInputRowGroup(const rowgroup::RowGroup& rg); - /* Aggregation */ - void addAggregateStep(const rowgroup::SP_ROWAGG_PM_t&, const rowgroup::RowGroup&); - void setJoinedRowGroup(const rowgroup::RowGroup& rg); + /* Aggregation */ + void addAggregateStep(const rowgroup::SP_ROWAGG_PM_t&, const rowgroup::RowGroup&); + void setJoinedRowGroup(const rowgroup::RowGroup& rg); - /* Tuple hashjoin */ - void useJoiners(const std::vector >&); - bool nextTupleJoinerMsg(messageqcpp::ByteStream&); -// void setSmallSideKeyColumn(uint32_t col); + /* Tuple hashjoin */ + void useJoiners(const std::vector >&); + bool nextTupleJoinerMsg(messageqcpp::ByteStream&); + // void setSmallSideKeyColumn(uint32_t col); - /* OR hacks */ - void setBOP(uint32_t op); // BOP_AND or BOP_OR, default is BOP_AND - void setForHJ(bool b); // default is false + /* OR hacks */ + void setBOP(uint32_t op); // BOP_AND or BOP_OR, default is BOP_AND + void setForHJ(bool b); // default is false - /* self-join */ - void jobInfo(const JobInfo* jobInfo) - { - fJobInfo = jobInfo; - } + /* self-join */ + void jobInfo(const JobInfo* jobInfo) + { + fJobInfo = jobInfo; + } - /* Function & Expression support */ - void setFEGroup1(boost::shared_ptr, - const rowgroup::RowGroup& input); - void setFEGroup2(boost::shared_ptr, - const rowgroup::RowGroup& output); - void setJoinFERG(const rowgroup::RowGroup& rg); + /* Function & Expression support */ + void setFEGroup1(boost::shared_ptr, const rowgroup::RowGroup& input); + void setFEGroup2(boost::shared_ptr, const rowgroup::RowGroup& output); + void setJoinFERG(const rowgroup::RowGroup& rg); - /* This fcn determines whether or not the containing TBPS obj will process results - from a join or put the RG data right in the output datalist. */ - bool pmSendsFinalResult() const - { - //isn't aware of UM-only joins. Function name is a bit misleading. - return (tJoiners.size() == 0 || (PMJoinerCount > 0 && (fe2 || aggregatorPM))); - } - const std::string toMiniString() const; + /* This fcn determines whether or not the containing TBPS obj will process results + from a join or put the RG data right in the output datalist. */ + bool pmSendsFinalResult() const + { + // isn't aware of UM-only joins. Function name is a bit misleading. + return (tJoiners.size() == 0 || (PMJoinerCount > 0 && (fe2 || aggregatorPM))); + } + const std::string toMiniString() const; - void priority(uint32_t p) - { - _priority = p; - }; - uint32_t priority() - { - return _priority; - } + void priority(uint32_t p) + { + _priority = p; + }; + uint32_t priority() + { + return _priority; + } - void deliverStringTableRowGroup(bool b); + void deliverStringTableRowGroup(bool b); - void setUuid(const boost::uuids::uuid& u) - { - uuid = u; - } + void setUuid(const boost::uuids::uuid& u) + { + uuid = u; + } -private: - //void setLBIDForScan(uint64_t rid, uint32_t dbroot); + private: + // void setLBIDForScan(uint64_t rid, uint32_t dbroot); - BPSOutputType ot; + BPSOutputType ot; - bool needToSetLBID; + bool needToSetLBID; - BRM::QueryContext versionInfo; - uint32_t txnID; - uint32_t sessionID; - uint32_t stepID; - uint32_t uniqueID; + BRM::QueryContext versionInfo; + uint32_t txnID; + uint32_t sessionID; + uint32_t stepID; + uint32_t uniqueID; - // # of times to loop over the command arrays - // ... This is 1, except when the first command is a scan, in which case - // this single BPP object produces count responses. - uint16_t count; + // # of times to loop over the command arrays + // ... This is 1, except when the first command is a scan, in which case + // this single BPP object produces count responses. + uint16_t count; - /* XXXPAT: tradeoff here. Memory wasted by statically allocating all of these - arrays on the UM (most aren't used) vs more dynamic allocation - on the PM */ + /* XXXPAT: tradeoff here. Memory wasted by statically allocating all of these + arrays on the UM (most aren't used) vs more dynamic allocation + on the PM */ - uint64_t baseRid; // first abs RID of the logical block + uint64_t baseRid; // first abs RID of the logical block - uint16_t relRids[LOGICAL_BLOCK_RIDS]; - boost::scoped_array absRids; - // TODO MCOL-641 Do we need uint128_t buffers here? - // When would sendValues=true, in which case values[] - // is sent to primproc? - uint64_t values[LOGICAL_BLOCK_RIDS]; - uint16_t ridCount; - bool needStrValues; + uint16_t relRids[LOGICAL_BLOCK_RIDS]; + boost::scoped_array absRids; + // TODO MCOL-641 Do we need uint128_t buffers here? + // When would sendValues=true, in which case values[] + // is sent to primproc? + uint64_t values[LOGICAL_BLOCK_RIDS]; + uint16_t ridCount; + bool needStrValues; - uint16_t wideColumnsWidths; + uint16_t wideColumnsWidths; - std::vector filterSteps; - std::vector projectSteps; - //@bug 1136 - uint16_t filterCount; - uint16_t projectCount; - bool needRidsAtDelivery; - uint16_t ridMap; + std::vector filterSteps; + std::vector projectSteps; + //@bug 1136 + uint16_t filterCount; + uint16_t projectCount; + bool needRidsAtDelivery; + uint16_t ridMap; -// TableBand templateTB; - uint32_t tableOID; - boost::scoped_array tablePositions; - uint32_t tableColumnCount; - bool sendValues; - bool sendAbsRids; - bool _hasScan; - bool LBIDTrace; + // TableBand templateTB; + uint32_t tableOID; + boost::scoped_array tablePositions; + uint32_t tableColumnCount; + bool sendValues; + bool sendAbsRids; + bool _hasScan; + bool LBIDTrace; - /* for tuple return type */ - std::vector colWidths; - uint32_t tupleLength; -// uint32_t rowCounter; // for debugging -// uint32_t rowsProcessed; - uint16_t status; + /* for tuple return type */ + std::vector colWidths; + uint32_t tupleLength; + // uint32_t rowCounter; // for debugging + // uint32_t rowsProcessed; + uint16_t status; - /* for Joiner serialization */ - bool pickNextJoinerNum(); - uint32_t pos, joinerNum; - boost::shared_ptr > smallSide; - boost::scoped_array posByJoinerNum; + /* for Joiner serialization */ + bool pickNextJoinerNum(); + uint32_t pos, joinerNum; + boost::shared_ptr > smallSide; + boost::scoped_array posByJoinerNum; - /* for RowGroup return type */ - rowgroup::RowGroup inputRG, projectionRG; - bool sendRowGroups; - uint32_t valueColumn; + /* for RowGroup return type */ + rowgroup::RowGroup inputRG, projectionRG; + bool sendRowGroups; + uint32_t valueColumn; - /* for PM Aggregation */ - rowgroup::RowGroup joinedRG; - rowgroup::SP_ROWAGG_PM_t aggregatorPM; - rowgroup::RowGroup aggregateRGPM; + /* for PM Aggregation */ + rowgroup::RowGroup joinedRG; + rowgroup::SP_ROWAGG_PM_t aggregatorPM; + rowgroup::RowGroup aggregateRGPM; - /* UM portion of the PM join alg */ - std::vector > tJoiners; - std::vector smallSideRGs; - rowgroup::RowGroup largeSideRG; - std::vector > smallSideKeys; - boost::scoped_array tlKeyLens; - bool sendTupleJoinRowGroupData; - uint32_t PMJoinerCount; + /* UM portion of the PM join alg */ + std::vector > tJoiners; + std::vector smallSideRGs; + rowgroup::RowGroup largeSideRG; + std::vector > smallSideKeys; + boost::scoped_array tlKeyLens; + bool sendTupleJoinRowGroupData; + uint32_t PMJoinerCount; - /* OR hack */ - uint8_t bop; // BOP_AND or BOP_OR - bool forHJ; // indicate if feeding a hashjoin, doJoin does not cover smallside + /* OR hack */ + uint8_t bop; // BOP_AND or BOP_OR + bool forHJ; // indicate if feeding a hashjoin, doJoin does not cover smallside - /* Self-join */ - const JobInfo* fJobInfo; + /* Self-join */ + const JobInfo* fJobInfo; - /* Functions & Expressions support */ - boost::shared_ptr fe1, fe2; - rowgroup::RowGroup fe1Input, fe2Output; - rowgroup::RowGroup joinFERG; + /* Functions & Expressions support */ + boost::shared_ptr fe1, fe2; + rowgroup::RowGroup fe1Input, fe2Output; + rowgroup::RowGroup joinFERG; - mutable boost::scoped_array primprocRG; // the format of the data received from PrimProc - uint32_t threadCount; + mutable boost::scoped_array + primprocRG; // the format of the data received from PrimProc + uint32_t threadCount; - unsigned fJoinerChunkSize; - uint32_t dbRoot; - bool hasSmallOuterJoin; + unsigned fJoinerChunkSize; + uint32_t dbRoot; + bool hasSmallOuterJoin; - uint32_t _priority; + uint32_t _priority; - boost::uuids::uuid uuid; + boost::uuids::uuid uuid; - friend class CommandJL; - friend class ColumnCommandJL; - friend class PassThruCommandJL; + friend class CommandJL; + friend class ColumnCommandJL; + friend class PassThruCommandJL; }; -} +} // namespace joblist #endif // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/bpp-jl.h b/dbcon/joblist/bpp-jl.h index aefbd1d79..fc64fae2f 100644 --- a/dbcon/joblist/bpp-jl.h +++ b/dbcon/joblist/bpp-jl.h @@ -43,4 +43,3 @@ #include "rtscommand-jl.h" #include "filtercommand-jl.h" #include "pseudocc-jl.h" - diff --git a/dbcon/joblist/bucketdl.h b/dbcon/joblist/bucketdl.h index 9df73b966..28fe22892 100644 --- a/dbcon/joblist/bucketdl.h +++ b/dbcon/joblist/bucketdl.h @@ -24,7 +24,6 @@ * class XXX interface */ - #include "wsdl.h" #include "hasher.h" #include "bucketreuse.h" @@ -38,673 +37,666 @@ #define _BUCKETDL_HPP_ namespace joblist { - /** @brief class BucketDL * */ -template +template class BucketDL : public DataList { - typedef DataList base; - typedef boost::shared_ptr< TupleWSDL > TWSSP; - typedef std::vector TWSVec; + typedef DataList base; + typedef boost::shared_ptr TWSSP; + typedef std::vector TWSVec; - enum ElementMode - { - RID_MODE, - RID_VALUE - }; + enum ElementMode + { + RID_MODE, + RID_VALUE + }; -public: - /** Main constuctor. - * @param numBuckets The number of buckets to create - * @param numConsumers The number of consumers that will eventually read this DL. - * @param maxElementsPerBucket The maximum # of elements each bucket should keep in memory. - * @param hash The function object that calculates which bucket an elements goes into on insertion. - */ + public: + /** Main constuctor. + * @param numBuckets The number of buckets to create + * @param numConsumers The number of consumers that will eventually read this DL. + * @param maxElementsPerBucket The maximum # of elements each bucket should keep in memory. + * @param hash The function object that calculates which bucket an elements goes into on insertion. + */ - BucketDL(uint32_t numBuckets, uint32_t numConsumers, uint32_t maxElementsPerBucket, ResourceManager* rm, - boost::function hash = utils::Hasher()); + BucketDL(uint32_t numBuckets, uint32_t numConsumers, uint32_t maxElementsPerBucket, ResourceManager* rm, + boost::function hash = utils::Hasher()); - virtual ~BucketDL(); + virtual ~BucketDL(); - // datalist interface. insert() and endOfInput() are the only - // datalist function that makes sense. The consumer side functions - // are stubs. Consumers know they're consuming a BucketDL. - void insert(const element_t& e); - void insert(const std::vector& e); - void insert(const element_t* array, uint64_t arrayCount); + // datalist interface. insert() and endOfInput() are the only + // datalist function that makes sense. The consumer side functions + // are stubs. Consumers know they're consuming a BucketDL. + void insert(const element_t& e); + void insert(const std::vector& e); + void insert(const element_t* array, uint64_t arrayCount); - void insert(TupleType& e); - void insert(std::vector& e); + void insert(TupleType& e); + void insert(std::vector& e); - void endOfInput(); - uint64_t getIterator(); - bool next(uint64_t it, element_t* e); - void setMultipleProducers(bool); + void endOfInput(); + uint64_t getIterator(); + bool next(uint64_t it, element_t* e); + void setMultipleProducers(bool); - // BucketDL consumer fcns - uint64_t getIterator(uint64_t bucket); - bool next(uint64_t bucket, uint64_t it, element_t* e); + // BucketDL consumer fcns + uint64_t getIterator(uint64_t bucket); + bool next(uint64_t bucket, uint64_t it, element_t* e); - /** Returns the size of the specified bucket */ - uint64_t size(uint64_t bucket); + /** Returns the size of the specified bucket */ + uint64_t size(uint64_t bucket); - /** Returns the total number of elements stored */ - uint64_t totalSize(); + /** Returns the total number of elements stored */ + uint64_t totalSize(); - /** Returns the number of buckets */ - uint64_t bucketCount(); + /** Returns the number of buckets */ + uint64_t bucketCount(); - /** Sets the value to pass down to element_t::getHashString() */ - void setHashMode(uint64_t mode); + /** Sets the value to pass down to element_t::getHashString() */ + void setHashMode(uint64_t mode); - /** Sets the value to RID_only or rid_value mode */ - void setElementMode(uint64_t mode); + /** Sets the value to RID_only or rid_value mode */ + void setElementMode(uint64_t mode); - /** Total number of files and filespace used for temp files */ - void totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) const; + /** Total number of files and filespace used for temp files */ + void totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) const; - const uint32_t hashLen() const - { - return fHashLen; - } - void hashLen (const uint32_t hashLen) - { - fHashLen = hashLen; - } + const uint32_t hashLen() const + { + return fHashLen; + } + void hashLen(const uint32_t hashLen) + { + fHashLen = hashLen; + } - const uint32_t elementLen() const - { - return fElementLen; - } - void elementLen (const uint64_t ridSize, const uint64_t dataSize); + const uint32_t elementLen() const + { + return fElementLen; + } + void elementLen(const uint64_t ridSize, const uint64_t dataSize); - /** This class does employ temp disk */ - virtual bool useDisk() const - { - return true; - } + /** This class does employ temp disk */ + virtual bool useDisk() const + { + return true; + } - /** Sets the size of the element components that are saved to disk */ - virtual void setDiskElemSize(uint32_t size1st, uint32_t size2nd); + /** Sets the size of the element components that are saved to disk */ + virtual void setDiskElemSize(uint32_t size1st, uint32_t size2nd); - /** Accessor and mutator to the BucketReuseControlEntry */ - void reuseControl(BucketReuseControlEntry* control, bool readonly); - BucketReuseControlEntry* reuseControl() - { - return fReuseControl; - } + /** Accessor and mutator to the BucketReuseControlEntry */ + void reuseControl(BucketReuseControlEntry* control, bool readonly); + BucketReuseControlEntry* reuseControl() + { + return fReuseControl; + } - /** Restores the buckets' set number and start positions */ - void restoreBucketInformation(); + /** Restores the buckets' set number and start positions */ + void restoreBucketInformation(); - /** @brief return tuple rid size */ - const uint64_t ridSize() const - { - return fRidSize; - } + /** @brief return tuple rid size */ + const uint64_t ridSize() const + { + return fRidSize; + } - /** @brief return tuple data size */ - const uint64_t dataSize() const - { - return fDataSize; - } + /** @brief return tuple data size */ + const uint64_t dataSize() const + { + return fDataSize; + } - /** Enables disk I/O time logging */ - void enableDiskIoTrace(); + /** Enables disk I/O time logging */ + void enableDiskIoTrace(); - /** Returns the disk I/O time in seconds */ - bool totalDiskIoTime(uint64_t& w, uint64_t& r); + /** Returns the disk I/O time in seconds */ + bool totalDiskIoTime(uint64_t& w, uint64_t& r); - /** Returns the reference of the disk I/O info list */ - std::list& diskIoInfoList(uint64_t bucket); + /** Returns the reference of the disk I/O info list */ + std::list& diskIoInfoList(uint64_t bucket); -protected: - - -private: - // Declare default constructors but don't define to disable their use - explicit BucketDL(); - explicit BucketDL(const BucketDL&); - BucketDL& operator=(const BucketDL&); - - ResourceManager* fRm; - WSDL** buckets; - WSDL** rbuckets; - TWSVec fTBuckets; - uint64_t numBuckets; - uint64_t numConsumers; - uint64_t maxElements; - boost::function hashFcn; - uint64_t hashMode; - uint64_t bucketMask; - bool multiProducer; - bool fTraceOn; - uint64_t elementMode; - uint32_t fHashLen; // @bug 844. hash length for tuple type - uint64_t fRidSize; // @bug 844. - uint64_t fDataSize; - uint64_t fElementLen; - uint64_t bucketDoneCount; - BucketReuseControlEntry* fReuseControl; + protected: + private: + // Declare default constructors but don't define to disable their use + explicit BucketDL(); + explicit BucketDL(const BucketDL&); + BucketDL& operator=(const BucketDL&); + ResourceManager* fRm; + WSDL** buckets; + WSDL** rbuckets; + TWSVec fTBuckets; + uint64_t numBuckets; + uint64_t numConsumers; + uint64_t maxElements; + boost::function hashFcn; + uint64_t hashMode; + uint64_t bucketMask; + bool multiProducer; + bool fTraceOn; + uint64_t elementMode; + uint32_t fHashLen; // @bug 844. hash length for tuple type + uint64_t fRidSize; // @bug 844. + uint64_t fDataSize; + uint64_t fElementLen; + uint64_t bucketDoneCount; + BucketReuseControlEntry* fReuseControl; }; -template +template BucketDL::BucketDL(uint32_t nb, uint32_t nc, uint32_t me, ResourceManager* rm, - boost::function hash) - : base(), fRm(rm), buckets(0), rbuckets(0), fTraceOn(false), fHashLen(0), fElementLen(0), - bucketDoneCount(0), fReuseControl(NULL) + boost::function hash) + : base() + , fRm(rm) + , buckets(0) + , rbuckets(0) + , fTraceOn(false) + , fHashLen(0) + , fElementLen(0) + , bucketDoneCount(0) + , fReuseControl(NULL) { - uint32_t i; - uint64_t mask; + uint32_t i; + uint64_t mask; - numBuckets = nb; - numConsumers = nc; - maxElements = me; - hashFcn = hash; - hashMode = 0; - elementMode = RID_MODE; - multiProducer = false; + numBuckets = nb; + numConsumers = nc; + maxElements = me; + hashFcn = hash; + hashMode = 0; + elementMode = RID_MODE; + multiProducer = false; - // initialize buckets - if (typeid (element_t) == typeid(TupleType)) - { - for (i = 0; i < numBuckets; i++) - fTBuckets.push_back(TWSSP(new TupleWSDL(numConsumers, maxElements, fRm))); - } - else - { - rbuckets = new WSDL* [numBuckets]; + // initialize buckets + if (typeid(element_t) == typeid(TupleType)) + { + for (i = 0; i < numBuckets; i++) + fTBuckets.push_back(TWSSP(new TupleWSDL(numConsumers, maxElements, fRm))); + } + else + { + rbuckets = new WSDL*[numBuckets]; - for (i = 0; i < numBuckets; i++) - rbuckets[i] = new WSDL - (numConsumers, maxElements, rm); - } + for (i = 0; i < numBuckets; i++) + rbuckets[i] = new WSDL(numConsumers, maxElements, rm); + } - for (i = 1, mask = 1, bucketMask = 0; i <= 64; i++) - { - mask <<= 1; - bucketMask = (bucketMask << 1) | 1; + for (i = 1, mask = 1, bucketMask = 0; i <= 64; i++) + { + mask <<= 1; + bucketMask = (bucketMask << 1) | 1; - if (numBuckets & mask) - break; - } + if (numBuckets & mask) + break; + } - for (i++, mask <<= 1; i <= 64; i++, mask <<= 1) - if (numBuckets & mask) - throw std::runtime_error("BucketDL: The number of buckets should be a power of 2."); + for (i++, mask <<= 1; i <= 64; i++, mask <<= 1) + if (numBuckets & mask) + throw std::runtime_error("BucketDL: The number of buckets should be a power of 2."); } -template +template BucketDL::~BucketDL() { - if (typeid(element_t) == typeid(TupleType)) - return; + if (typeid(element_t) == typeid(TupleType)) + return; - uint64_t i; + uint64_t i; - if (elementMode == RID_MODE) - { - for (i = 0; i < numBuckets; i++) - delete rbuckets[i]; + if (elementMode == RID_MODE) + { + for (i = 0; i < numBuckets; i++) + delete rbuckets[i]; - delete [] rbuckets; - } - else - { - for (i = 0; i < numBuckets; i++) - delete buckets[i]; + delete[] rbuckets; + } + else + { + for (i = 0; i < numBuckets; i++) + delete buckets[i]; - delete [] buckets; - } + delete[] buckets; + } } -template +template void BucketDL::setMultipleProducers(bool b) { - multiProducer = b; - uint64_t i; + multiProducer = b; + uint64_t i; - if (typeid(element_t) == typeid(TupleType)) + if (typeid(element_t) == typeid(TupleType)) + { + for (i = 0; i < numBuckets; i++) + fTBuckets[i]->setMultipleProducers(b); + } + else + { + if (elementMode == RID_MODE) { - for (i = 0; i < numBuckets; i++) - fTBuckets[i]->setMultipleProducers(b); + for (i = 0; i < numBuckets; i++) + rbuckets[i]->setMultipleProducers(b); } else { - if (elementMode == RID_MODE) - { - for (i = 0; i < numBuckets; i++) - rbuckets[i]->setMultipleProducers(b); - } - else - { - for (i = 0; i < numBuckets; i++) - buckets[i]->setMultipleProducers(b); - } + for (i = 0; i < numBuckets; i++) + buckets[i]->setMultipleProducers(b); } + } } -template +template void BucketDL::insert(const element_t& e) { - /* Need the element type to provide what to hash, which conflicts - with the standard meaning of "<<". For our currently-defined element types, - this would be the rid field only, not the entire contents of the structure */ + /* Need the element type to provide what to hash, which conflicts + with the standard meaning of "<<". For our currently-defined element types, + this would be the rid field only, not the entire contents of the structure */ - uint64_t bucket, len = fHashLen; - const char* hashStr; + uint64_t bucket, len = fHashLen; + const char* hashStr; - hashStr = e.getHashString(hashMode, &len); - bucket = hashFcn(hashStr, len) & bucketMask; - - if (elementMode == RID_MODE) - { - RIDElementType rid(e.first); - rbuckets[bucket]->insert(rid); - } - else - buckets[bucket]->insert(e); + hashStr = e.getHashString(hashMode, &len); + bucket = hashFcn(hashStr, len) & bucketMask; + if (elementMode == RID_MODE) + { + RIDElementType rid(e.first); + rbuckets[bucket]->insert(rid); + } + else + buckets[bucket]->insert(e); } -template +template void BucketDL::insert(TupleType& e) { - uint64_t bucket, len = fHashLen; - const char* hashStr; + uint64_t bucket, len = fHashLen; + const char* hashStr; - hashStr = e.getHashString(hashMode, &len); + hashStr = e.getHashString(hashMode, &len); - bucket = hashFcn(hashStr, len) & bucketMask; - fTBuckets[bucket]->insert(e); + bucket = hashFcn(hashStr, len) & bucketMask; + fTBuckets[bucket]->insert(e); } -template +template void BucketDL::insert(std::vector& v) { - std::vector::iterator it, end; + std::vector::iterator it, end; + if (multiProducer) + base::lock(); + + try + { + end = v.end(); + + for (it = v.begin(); it != end; ++it) + fTBuckets[hashFcn(it->second, fHashLen) & bucketMask]->insert_nolock(*it); + } + catch (...) + { if (multiProducer) - base::lock(); + base::unlock(); - try - { - end = v.end(); + throw; + } - for (it = v.begin(); it != end; ++it) - fTBuckets[hashFcn(it->second, fHashLen) & bucketMask]->insert_nolock(*it); - } - catch ( ... ) - { - if (multiProducer) - base::unlock(); - - throw; - } - - if (multiProducer) - base::unlock(); + if (multiProducer) + base::unlock(); } -template +template void BucketDL::insert(const std::vector& v) { - typename std::vector::const_iterator it, end; - const char* hashStr; - uint64_t len = fHashLen; + typename std::vector::const_iterator it, end; + const char* hashStr; + uint64_t len = fHashLen; - if (multiProducer) - base::lock(); + if (multiProducer) + base::lock(); - try + try + { + end = v.end(); + + if (elementMode == RID_MODE) { - end = v.end(); + RIDElementType rid; - if (elementMode == RID_MODE) - { - RIDElementType rid; - - for (it = v.begin(); it != end; ++it) - { - hashStr = it->getHashString(hashMode, &len); - rid.first = (*it).first; - rbuckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(rid); - } - } - else - for (it = v.begin(); it != end; ++it) - { - hashStr = it->getHashString(hashMode, &len); - buckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(*it); - } + for (it = v.begin(); it != end; ++it) + { + hashStr = it->getHashString(hashMode, &len); + rid.first = (*it).first; + rbuckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(rid); + } } - catch ( ... ) - { - if (multiProducer) - base::unlock(); - - throw; - } - + else + for (it = v.begin(); it != end; ++it) + { + hashStr = it->getHashString(hashMode, &len); + buckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(*it); + } + } + catch (...) + { if (multiProducer) - base::unlock(); + base::unlock(); + + throw; + } + + if (multiProducer) + base::unlock(); } -template +template void BucketDL::insert(const element_t* array, uint64_t arrayCount) { - const char* hashStr; - uint64_t len = fHashLen; + const char* hashStr; + uint64_t len = fHashLen; - if (multiProducer) - base::lock(); + if (multiProducer) + base::lock(); - try + try + { + if (elementMode == RID_MODE) { - if (elementMode == RID_MODE) - { - RIDElementType rid; + RIDElementType rid; - for (uint64_t i = 0; i < arrayCount; ++i) - { - hashStr = array[i].getHashString(hashMode, &len); - rid.first = array[i].first; - rbuckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(rid); - } - } - else - for (uint64_t i = 0; i < arrayCount; ++i) - { - hashStr = array[i].getHashString(hashMode, &len); - buckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock( - array[i]); - } + for (uint64_t i = 0; i < arrayCount; ++i) + { + hashStr = array[i].getHashString(hashMode, &len); + rid.first = array[i].first; + rbuckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(rid); + } } - catch ( ... ) - { - if (multiProducer) - base::unlock(); - - throw; - } - + else + for (uint64_t i = 0; i < arrayCount; ++i) + { + hashStr = array[i].getHashString(hashMode, &len); + buckets[hashFcn(hashStr, len) & bucketMask]->insert_nolock(array[i]); + } + } + catch (...) + { if (multiProducer) - base::unlock(); + base::unlock(); + + throw; + } + + if (multiProducer) + base::unlock(); } -template +template uint64_t BucketDL::getIterator() { - throw std::logic_error("don't call BucketDL::getIterator(), call getIterator(uint32_t)"); + throw std::logic_error("don't call BucketDL::getIterator(), call getIterator(uint32_t)"); } -template +template void BucketDL::endOfInput() { - uint64_t i; - uint64_t saveSize = 0; //debug + uint64_t i; + uint64_t saveSize = 0; // debug - if (typeid(element_t) == typeid(TupleType)) + if (typeid(element_t) == typeid(TupleType)) + { + for (i = 0; i < numBuckets; i++) { - for (i = 0; i < numBuckets; i++) - { - fTBuckets[i]->endOfInput(); - saveSize += fTBuckets[i]->saveSize(); - } - - //std::cout << "bucketdl-" << this << " saveSize=" << saveSize << std::endl; - } - else if (elementMode == RID_MODE) - { - for (i = 0; i < numBuckets; i++) - rbuckets[i]->endOfInput(); - } - else - { - for (i = 0; i < numBuckets; i++) - buckets[i]->endOfInput(); + fTBuckets[i]->endOfInput(); + saveSize += fTBuckets[i]->saveSize(); } - if (fReuseControl != NULL && fReuseControl->userNotified() == false) - fReuseControl->notifyUsers(); + // std::cout << "bucketdl-" << this << " saveSize=" << saveSize << std::endl; + } + else if (elementMode == RID_MODE) + { + for (i = 0; i < numBuckets; i++) + rbuckets[i]->endOfInput(); + } + else + { + for (i = 0; i < numBuckets; i++) + buckets[i]->endOfInput(); + } + if (fReuseControl != NULL && fReuseControl->userNotified() == false) + fReuseControl->notifyUsers(); } -template +template bool BucketDL::next(uint64_t it, element_t* e) { - throw std::logic_error("don't call BucketDL::next(uint32_t, element_t), call next(uint32_t, uint32_t, element_t"); + throw std::logic_error( + "don't call BucketDL::next(uint32_t, element_t), call next(uint32_t, uint32_t, element_t"); } -template +template uint64_t BucketDL::getIterator(uint64_t bucket) { - if (typeid(element_t) == typeid(TupleType)) - return fTBuckets[bucket]->getIterator(); + if (typeid(element_t) == typeid(TupleType)) + return fTBuckets[bucket]->getIterator(); - if (elementMode == RID_MODE) - return rbuckets[bucket]->getIterator(); - else - return buckets[bucket]->getIterator(); + if (elementMode == RID_MODE) + return rbuckets[bucket]->getIterator(); + else + return buckets[bucket]->getIterator(); } -template +template bool BucketDL::next(uint64_t bucket, uint64_t it, element_t* e) { - if (typeid(element_t) == typeid(TupleType)) - return fTBuckets[bucket]->next(it, reinterpret_cast(e)); + if (typeid(element_t) == typeid(TupleType)) + return fTBuckets[bucket]->next(it, reinterpret_cast(e)); - bool ret; + bool ret; - if (elementMode == RID_MODE) - { - RIDElementType rid; - ret = rbuckets[bucket]->next(it, &rid); - e->first = rid.first; - } - else - { - ret = buckets[bucket]->next(it, e); - } + if (elementMode == RID_MODE) + { + RIDElementType rid; + ret = rbuckets[bucket]->next(it, &rid); + e->first = rid.first; + } + else + { + ret = buckets[bucket]->next(it, e); + } - if (ret != true && fReuseControl != NULL) - { - // because not all the buckets are consumed at the same time, - // getIterator(i) maybe called sequentially by one or more threads, - // need to make sure all consumers are done with all the buckets - base::lock(); + if (ret != true && fReuseControl != NULL) + { + // because not all the buckets are consumed at the same time, + // getIterator(i) maybe called sequentially by one or more threads, + // need to make sure all consumers are done with all the buckets + base::lock(); - if (++bucketDoneCount == numConsumers * numBuckets) - BucketReuseManager::instance()->userDeregister(fReuseControl); + if (++bucketDoneCount == numConsumers * numBuckets) + BucketReuseManager::instance()->userDeregister(fReuseControl); - base::unlock(); - } + base::unlock(); + } - return ret; + return ret; } -template +template uint64_t BucketDL::size(uint64_t bucket) { - if (typeid(element_t) == typeid(TupleType)) - return fTBuckets[bucket]->totalSize(); + if (typeid(element_t) == typeid(TupleType)) + return fTBuckets[bucket]->totalSize(); - if (elementMode == RID_MODE) - return rbuckets[bucket]->totalSize(); - else - return buckets[bucket]->totalSize(); + if (elementMode == RID_MODE) + return rbuckets[bucket]->totalSize(); + else + return buckets[bucket]->totalSize(); } -template +template uint64_t BucketDL::bucketCount() { - return numBuckets; + return numBuckets; } -template +template uint64_t BucketDL::totalSize() { - uint64_t ret = 0; - uint64_t i; + uint64_t ret = 0; + uint64_t i; - if (typeid(element_t) == typeid(TupleType)) - for (i = 0; i < numBuckets; i++) - ret += fTBuckets[i]->totalSize(); - else if (elementMode == RID_MODE) + if (typeid(element_t) == typeid(TupleType)) + for (i = 0; i < numBuckets; i++) + ret += fTBuckets[i]->totalSize(); + else if (elementMode == RID_MODE) + { + for (i = 0; i < numBuckets; i++) + ret += rbuckets[i]->totalSize(); + } + else + { + for (i = 0; i < numBuckets; i++) + ret += buckets[i]->totalSize(); + } + + return ret; +} + +template +void BucketDL::setHashMode(uint64_t mode) +{ + hashMode = mode; + // Make elementMode the same as hashMode unless setElementMode + // is explicitly called by the caller, like filterstep. + setElementMode(mode); +} + +template +void BucketDL::setElementMode(uint64_t mode) +{ + uint64_t i; + + if (typeid(element_t) == typeid(TupleType)) + return; + + if (elementMode != mode) + { + if (elementMode == RID_MODE) { - for (i = 0; i < numBuckets; i++) - ret += rbuckets[i]->totalSize(); + for (i = 0; i < numBuckets; i++) + delete rbuckets[i]; + + delete[] rbuckets; } else { - for (i = 0; i < numBuckets; i++) - ret += buckets[i]->totalSize(); + for (i = 0; i < numBuckets; i++) + delete buckets[i]; + + delete[] buckets; } - return ret; -} + elementMode = mode; -template -void BucketDL::setHashMode(uint64_t mode) -{ - hashMode = mode; - // Make elementMode the same as hashMode unless setElementMode - // is explicitly called by the caller, like filterstep. - setElementMode (mode); -} - -template -void BucketDL::setElementMode(uint64_t mode) -{ - uint64_t i; - - if (typeid(element_t) == typeid(TupleType)) - return; - - if (elementMode != mode) + if (elementMode == RID_MODE) { - if (elementMode == RID_MODE) - { - for (i = 0; i < numBuckets; i++) - delete rbuckets[i]; + rbuckets = new WSDL*[numBuckets]; - delete [] rbuckets; - } - else - { - for (i = 0; i < numBuckets; i++) - delete buckets[i]; - - delete [] buckets; - } - - elementMode = mode; - - if (elementMode == RID_MODE) - { - rbuckets = new WSDL* [numBuckets]; - - for (i = 0; i < numBuckets; i++) - { - rbuckets[i] = new WSDL - (numConsumers, maxElements, - base::getDiskElemSize1st(), - base::getDiskElemSize2nd(), fRm); - rbuckets[i]->setMultipleProducers(multiProducer); - rbuckets[i]->traceOn(fTraceOn); - } - } - else - { - buckets = new WSDL* [numBuckets]; - - for (i = 0; i < numBuckets; i++) - { - buckets[i] = new WSDL - (numConsumers, maxElements, - base::getDiskElemSize1st(), - base::getDiskElemSize2nd(), fRm); - buckets[i]->setMultipleProducers(multiProducer); - buckets[i]->traceOn(fTraceOn); - } - } + for (i = 0; i < numBuckets; i++) + { + rbuckets[i] = new WSDL(numConsumers, maxElements, base::getDiskElemSize1st(), + base::getDiskElemSize2nd(), fRm); + rbuckets[i]->setMultipleProducers(multiProducer); + rbuckets[i]->traceOn(fTraceOn); + } } + else + { + buckets = new WSDL*[numBuckets]; -// std::cout << "bucketdl-" << this << " setElementMode " << hashMode << std::endl; + for (i = 0; i < numBuckets; i++) + { + buckets[i] = new WSDL(numConsumers, maxElements, base::getDiskElemSize1st(), + base::getDiskElemSize2nd(), fRm); + buckets[i]->setMultipleProducers(multiProducer); + buckets[i]->traceOn(fTraceOn); + } + } + } + // std::cout << "bucketdl-" << this << " setElementMode " << hashMode << std::endl; } // // Returns the number of temp files and the space taken up by those files // (in bytes) by this Bucket collection. // -template -void BucketDL::totalFileCounts( - uint64_t& numFiles, - uint64_t& numBytes) const +template +void BucketDL::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) const { - numFiles = 0; - numBytes = 0; + numFiles = 0; + numBytes = 0; - if (typeid(element_t) == typeid(TupleType)) + if (typeid(element_t) == typeid(TupleType)) + { + for (uint64_t i = 0; i < numBuckets; i++) { - for (uint64_t i = 0; i < numBuckets; i++) - { - uint64_t setCnt = fTBuckets[i]->initialSetCount(); + uint64_t setCnt = fTBuckets[i]->initialSetCount(); - if (setCnt > 1) - { - numFiles += fTBuckets[i]->numberOfTempFiles(); - numBytes += fTBuckets[i]->saveSize(); - } - } + if (setCnt > 1) + { + numFiles += fTBuckets[i]->numberOfTempFiles(); + numBytes += fTBuckets[i]->saveSize(); + } } - else if (elementMode == RID_MODE) + } + else if (elementMode == RID_MODE) + { + for (uint64_t i = 0; i < numBuckets; i++) { - for (uint64_t i = 0; i < numBuckets; i++) - { - uint64_t setCnt = rbuckets[i]->initialSetCount(); + uint64_t setCnt = rbuckets[i]->initialSetCount(); - if (setCnt > 1) - { - //std::cout << "BDL: bucket " << i << " has " << setCnt << - // " sets" << std::endl; + if (setCnt > 1) + { + // std::cout << "BDL: bucket " << i << " has " << setCnt << + // " sets" << std::endl; - numFiles += rbuckets[i]->numberOfTempFiles(); - numBytes += rbuckets[i]->saveSize; - } - } + numFiles += rbuckets[i]->numberOfTempFiles(); + numBytes += rbuckets[i]->saveSize; + } } - else + } + else + { + for (uint64_t i = 0; i < numBuckets; i++) { - for (uint64_t i = 0; i < numBuckets; i++) - { - uint64_t setCnt = buckets[i]->initialSetCount(); + uint64_t setCnt = buckets[i]->initialSetCount(); - if (setCnt > 1) - { - //std::cout << "BDL: bucket " << i << " has " << setCnt << - // " sets" << std::endl; + if (setCnt > 1) + { + // std::cout << "BDL: bucket " << i << " has " << setCnt << + // " sets" << std::endl; - numFiles += buckets[i]->numberOfTempFiles(); - numBytes += buckets[i]->saveSize; - } - } + numFiles += buckets[i]->numberOfTempFiles(); + numBytes += buckets[i]->saveSize; + } } + } } -template +template void BucketDL::elementLen(const uint64_t ridSize, const uint64_t dataSize) { - fElementLen = ridSize + dataSize; - uint64_t i; + fElementLen = ridSize + dataSize; + uint64_t i; - if (typeid(element_t) == typeid(TupleType)) - { - for (i = 0; i < numBuckets; i++) - fTBuckets[i]->tupleSize(ridSize, dataSize); - } + if (typeid(element_t) == typeid(TupleType)) + { + for (i = 0; i < numBuckets; i++) + fTBuckets[i]->tupleSize(ridSize, dataSize); + } } // @@ -712,146 +704,146 @@ void BucketDL::elementLen(const uint64_t ridSize, const uint64_t data // size1st is the size in bytes of element_t.first. // size2nd is the size in bytes of element_t.second. // -template +template void BucketDL::setDiskElemSize(uint32_t size1st, uint32_t size2nd) { - base::fElemDiskFirstSize = size1st; - base::fElemDiskSecondSize = size2nd; + base::fElemDiskFirstSize = size1st; + base::fElemDiskSecondSize = size2nd; - //...Forward this size information to our internal WSDL containers. - // @todo compress for tuplewsdl - if (typeid(element_t) == typeid(TupleType)) - return; - - if (elementMode == RID_MODE) - { - for (uint64_t i = 0; i < numBuckets; i++) - { - rbuckets[i]->setDiskElemSize ( size1st, size2nd ); - } - } - else - { - for (uint64_t i = 0; i < numBuckets; i++) - { - buckets[i]->setDiskElemSize ( size1st, size2nd ); - } - } - - if (fReuseControl != NULL) - { - fReuseControl->dataSize().first = size1st; - fReuseControl->dataSize().second = size2nd; - } -} - -template -void BucketDL::reuseControl(BucketReuseControlEntry* control, bool readonly) -{ - // @todo reuse for tuplewsdl - if (typeid(element_t) == typeid(TupleType)) - return; - - if (control == NULL) - return; - - fReuseControl = control; - std::vector& infoVec = fReuseControl->restoreInfoVec(); - infoVec.resize(numBuckets); + //...Forward this size information to our internal WSDL containers. + // @todo compress for tuplewsdl + if (typeid(element_t) == typeid(TupleType)) + return; + if (elementMode == RID_MODE) + { for (uint64_t i = 0; i < numBuckets; i++) { - std::stringstream ss; - ss << control->baseName() << "." << i; - - if (elementMode == RID_MODE) - rbuckets[i]->setReuseInfo(&(infoVec[i]), ss.str().c_str(), readonly); - else - buckets[i]->setReuseInfo(&(infoVec[i]), ss.str().c_str(), readonly); + rbuckets[i]->setDiskElemSize(size1st, size2nd); } + } + else + { + for (uint64_t i = 0; i < numBuckets; i++) + { + buckets[i]->setDiskElemSize(size1st, size2nd); + } + } + + if (fReuseControl != NULL) + { + fReuseControl->dataSize().first = size1st; + fReuseControl->dataSize().second = size2nd; + } } -template +template +void BucketDL::reuseControl(BucketReuseControlEntry* control, bool readonly) +{ + // @todo reuse for tuplewsdl + if (typeid(element_t) == typeid(TupleType)) + return; + + if (control == NULL) + return; + + fReuseControl = control; + std::vector& infoVec = fReuseControl->restoreInfoVec(); + infoVec.resize(numBuckets); + + for (uint64_t i = 0; i < numBuckets; i++) + { + std::stringstream ss; + ss << control->baseName() << "." << i; + + if (elementMode == RID_MODE) + rbuckets[i]->setReuseInfo(&(infoVec[i]), ss.str().c_str(), readonly); + else + buckets[i]->setReuseInfo(&(infoVec[i]), ss.str().c_str(), readonly); + } +} + +template void BucketDL::restoreBucketInformation() { - if (typeid(element_t) == typeid(TupleType)) - return; + if (typeid(element_t) == typeid(TupleType)) + return; - std::vector& infoVec = fReuseControl->restoreInfoVec(); + std::vector& infoVec = fReuseControl->restoreInfoVec(); - if (elementMode == RID_MODE) - for (uint64_t i = 0; i < numBuckets; i++) - rbuckets[i]->restoreSetForReuse(infoVec[i]); - else - for (uint64_t i = 0; i < numBuckets; i++) - buckets[i]->restoreSetForReuse(infoVec[i]); + if (elementMode == RID_MODE) + for (uint64_t i = 0; i < numBuckets; i++) + rbuckets[i]->restoreSetForReuse(infoVec[i]); + else + for (uint64_t i = 0; i < numBuckets; i++) + buckets[i]->restoreSetForReuse(infoVec[i]); } -template +template void BucketDL::enableDiskIoTrace() { - fTraceOn = true; + fTraceOn = true; - if (typeid(element_t) == typeid(TupleType)) - { - for (uint64_t bucket = 0; bucket < numBuckets; bucket++) - fTBuckets[bucket]->traceOn(fTraceOn); + if (typeid(element_t) == typeid(TupleType)) + { + for (uint64_t bucket = 0; bucket < numBuckets; bucket++) + fTBuckets[bucket]->traceOn(fTraceOn); - return; - } + return; + } - if (elementMode == RID_MODE) - for (uint64_t bucket = 0; bucket < numBuckets; bucket++) - rbuckets[bucket]->traceOn(fTraceOn); - else - for (uint64_t bucket = 0; bucket < numBuckets; bucket++) - buckets[bucket]->traceOn(fTraceOn); + if (elementMode == RID_MODE) + for (uint64_t bucket = 0; bucket < numBuckets; bucket++) + rbuckets[bucket]->traceOn(fTraceOn); + else + for (uint64_t bucket = 0; bucket < numBuckets; bucket++) + buckets[bucket]->traceOn(fTraceOn); } -template +template bool BucketDL::totalDiskIoTime(uint64_t& w, uint64_t& r) { - boost::posix_time::time_duration wTime(0, 0, 0, 0); - boost::posix_time::time_duration rTime(0, 0, 0, 0); - bool diskIo = false; + boost::posix_time::time_duration wTime(0, 0, 0, 0); + boost::posix_time::time_duration rTime(0, 0, 0, 0); + bool diskIo = false; - for (uint64_t bucket = 0; bucket < numBuckets; bucket++) + for (uint64_t bucket = 0; bucket < numBuckets; bucket++) + { + std::list& infoList = diskIoInfoList(bucket); + std::list::iterator k = infoList.begin(); + + while (k != infoList.end()) { - std::list& infoList = diskIoInfoList(bucket); - std::list::iterator k = infoList.begin(); + if (k->fWrite == true) + wTime += k->fEnd - k->fStart; + else + rTime += k->fEnd - k->fStart; - while (k != infoList.end()) - { - if (k->fWrite == true) - wTime += k->fEnd - k->fStart; - else - rTime += k->fEnd - k->fStart; - - k++; - } - - if (infoList.size() > 0) - diskIo = true; + k++; } - w = wTime.total_seconds(); - r = rTime.total_seconds(); + if (infoList.size() > 0) + diskIo = true; + } - return diskIo; + w = wTime.total_seconds(); + r = rTime.total_seconds(); + + return diskIo; } -template +template std::list& BucketDL::diskIoInfoList(uint64_t bucket) { - if (typeid(element_t) == typeid(TupleType)) - return (fTBuckets[bucket]->diskIoList()); + if (typeid(element_t) == typeid(TupleType)) + return (fTBuckets[bucket]->diskIoList()); - if (elementMode == RID_MODE) - return (rbuckets[bucket]->diskIoList()); - else - return (buckets[bucket]->diskIoList()); + if (elementMode == RID_MODE) + return (rbuckets[bucket]->diskIoList()); + else + return (buckets[bucket]->diskIoList()); } -} //namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 4bf6356a3..8e42a73e6 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -43,90 +43,91 @@ using namespace messageqcpp; namespace joblist { - ColumnCommandJL::ColumnCommandJL(const pColScanStep& scan, vector lastLBID) { - BRM::DBRM dbrm; - isScan = true; + BRM::DBRM dbrm; + isScan = true; - /* grab necessary vars from scan */ - traceFlags = scan.fTraceFlags; - filterString = scan.fFilterString; - filterCount = scan.fFilterCount; - colType = scan.fColType; - BOP = scan.fBOP; - extents = scan.extents; - OID = scan.fOid; - colName = scan.fName; - rpbShift = scan.rpbShift; - fIsDict = scan.fIsDict; - fLastLbid = lastLBID; + /* grab necessary vars from scan */ + traceFlags = scan.fTraceFlags; + filterString = scan.fFilterString; + filterCount = scan.fFilterCount; + colType = scan.fColType; + BOP = scan.fBOP; + extents = scan.extents; + OID = scan.fOid; + colName = scan.fName; + rpbShift = scan.rpbShift; + fIsDict = scan.fIsDict; + fLastLbid = lastLBID; - //cout << "CCJL inherited lastlbids: "; - //for (uint32_t i = 0; i < lastLBID.size(); i++) - // cout << lastLBID[i] << " "; - //cout << endl; + // cout << "CCJL inherited lastlbids: "; + // for (uint32_t i = 0; i < lastLBID.size(); i++) + // cout << lastLBID[i] << " "; + // cout << endl; -// cout << "Init columncommand from scan with OID " << OID << endl; - /* I think modmask isn't necessary for scans */ - divShift = scan.divShift; - modMask = (1 << divShift) - 1; + // cout << "Init columncommand from scan with OID " << OID << endl; + /* I think modmask isn't necessary for scans */ + divShift = scan.divShift; + modMask = (1 << divShift) - 1; - // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation. - fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; - // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). - fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; - config::Config* cf = config::Config::makeConfig(); - string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); + // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use + // in RID calculation. + fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; + // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; + config::Config* cf = config::Config::makeConfig(); + string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); - if ( fpc.length() != 0 ) - fFilesPerColumnPartition = cf->uFromText(fpc); + if (fpc.length() != 0) + fFilesPerColumnPartition = cf->uFromText(fpc); } ColumnCommandJL::ColumnCommandJL(const pColStep& step) { - BRM::DBRM dbrm; + BRM::DBRM dbrm; - isScan = false; + isScan = false; - /* grab necessary vars from step */ - traceFlags = step.fTraceFlags; - filterString = step.fFilterString; - filterCount = step.fFilterCount; - colType = step.fColType; - BOP = step.fBOP; - extents = step.extents; - divShift = step.divShift; - modMask = step.modMask; - rpbShift = step.rpbShift; - OID = step.fOid; - colName = step.fName; - fIsDict = step.fIsDict; - ResourceManager* rm = ResourceManager::instance(); - numDBRoots = rm->getDBRootCount(); + /* grab necessary vars from step */ + traceFlags = step.fTraceFlags; + filterString = step.fFilterString; + filterCount = step.fFilterCount; + colType = step.fColType; + BOP = step.fBOP; + extents = step.extents; + divShift = step.divShift; + modMask = step.modMask; + rpbShift = step.rpbShift; + OID = step.fOid; + colName = step.fName; + fIsDict = step.fIsDict; + ResourceManager* rm = ResourceManager::instance(); + numDBRoots = rm->getDBRootCount(); - // grab the last LBID for this column. It's a _minor_ optimization for the block loader. - //dbrm.getLastLocalHWM((BRM::OID_t)OID, dbroot, partNum, segNum, lastHWM); - //dbrm.lookupLocal((BRM::OID_t)OID, partNum, segNum, lastHWM, fLastLbid); - /* - fLastLbid.resize(numDBRoots); - for (uint32_t i = 0; i < numDBRoots; i++) { - dbrm.getLastLocalHWM2((BRM::OID_t)OID, i+1, partNum, segNum, lastHWM); - dbrm.lookupLocal((BRM::OID_t)OID, partNum, segNum, lastHWM, fLastLbid[i]); - } - */ + // grab the last LBID for this column. It's a _minor_ optimization for the block loader. + // dbrm.getLastLocalHWM((BRM::OID_t)OID, dbroot, partNum, segNum, lastHWM); + // dbrm.lookupLocal((BRM::OID_t)OID, partNum, segNum, lastHWM, fLastLbid); + /* + fLastLbid.resize(numDBRoots); + for (uint32_t i = 0; i < numDBRoots; i++) { + dbrm.getLastLocalHWM2((BRM::OID_t)OID, i+1, partNum, segNum, lastHWM); + dbrm.lookupLocal((BRM::OID_t)OID, partNum, segNum, lastHWM, fLastLbid[i]); + } + */ -// cout << "Init columncommand from step with OID " << OID << " and width " << colType.colWidth << endl; + // cout << "Init columncommand from step with OID " << OID << " and width " << colType.colWidth << endl; - // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation. - fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; - // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). - fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; - config::Config* cf = config::Config::makeConfig(); - string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); + // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use + // in RID calculation. + fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; + // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; + config::Config* cf = config::Config::makeConfig(); + string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); - if ( fpc.length() != 0 ) - fFilesPerColumnPartition = cf->uFromText(fpc); + if (fpc.length() != 0) + fFilesPerColumnPartition = cf->uFromText(fpc); } ColumnCommandJL::~ColumnCommandJL() @@ -136,167 +137,153 @@ ColumnCommandJL::~ColumnCommandJL() // The other leg is in PP, its name is ColumnCommand::makeCommand. void ColumnCommandJL::createCommand(ByteStream& bs) const { - bs << (uint8_t) COLUMN_COMMAND; - colType.serialize(bs); - bs << (uint8_t) isScan; - bs << traceFlags; - bs << filterString; - bs << BOP; - bs << filterCount; - serializeInlineVector(bs, fLastLbid); + bs << (uint8_t)COLUMN_COMMAND; + colType.serialize(bs); + bs << (uint8_t)isScan; + bs << traceFlags; + bs << filterString; + bs << BOP; + bs << filterCount; + serializeInlineVector(bs, fLastLbid); - CommandJL::createCommand(bs); + CommandJL::createCommand(bs); } void ColumnCommandJL::runCommand(ByteStream& bs) const { - bs << lbid; + bs << lbid; } void ColumnCommandJL::setLBID(uint64_t rid, uint32_t dbRoot) { - uint32_t partNum; - uint16_t segNum; - uint8_t extentNum; - uint16_t blockNum; - uint32_t colWidth; - uint32_t i; + uint32_t partNum; + uint16_t segNum; + uint8_t extentNum; + uint16_t blockNum; + uint32_t colWidth; + uint32_t i; - idbassert(extents.size() > 0); - colWidth = extents[0].colWid; - rowgroup::getLocationFromRid(rid, &partNum, &segNum, &extentNum, &blockNum); + idbassert(extents.size() > 0); + colWidth = extents[0].colWid; + rowgroup::getLocationFromRid(rid, &partNum, &segNum, &extentNum, &blockNum); - for (i = 0; i < extents.size(); i++) + for (i = 0; i < extents.size(); i++) + { + if (extents[i].dbRoot == dbRoot && extents[i].partitionNum == partNum && + extents[i].segmentNum == segNum && extents[i].blockOffset == (extentNum * colWidth * 1024)) { - if (extents[i].dbRoot == dbRoot && - extents[i].partitionNum == partNum && - extents[i].segmentNum == segNum && - extents[i].blockOffset == (extentNum * colWidth * 1024)) - { - - lbid = extents[i].range.start + (blockNum * colWidth); - currentExtentIndex = i; - /* - ostringstream os; - os << "CCJL: rid=" << rid << "; dbroot=" << dbRoot << "; partitionNum=" << partNum - << "; segmentNum=" << segNum << "; extentNum = " << (int) extentNum << - "; blockNum = " << blockNum << "; OID=" << OID << " LBID=" << lbid; - cout << os.str() << endl; - */ - return; - } + lbid = extents[i].range.start + (blockNum * colWidth); + currentExtentIndex = i; + /* + ostringstream os; + os << "CCJL: rid=" << rid << "; dbroot=" << dbRoot << "; partitionNum=" << partNum + << "; segmentNum=" << segNum << "; extentNum = " << (int) extentNum << + "; blockNum = " << blockNum << "; OID=" << OID << " LBID=" << lbid; + cout << os.str() << endl; + */ + return; } + } - throw logic_error("ColumnCommandJL: setLBID didn't find the extent for the rid."); + throw logic_error("ColumnCommandJL: setLBID didn't find the extent for the rid."); -// ostringstream os; -// os << "CCJL: rid=" << rid << "; dbroot=" << dbRoot << "; partitionNum=" << partitionNum << "; segmentNum=" << segmentNum << "; stripeWithinPartition=" << -// stripeWithinPartition << "; OID=" << OID << " LBID=" << lbid; -// BRM::log(os.str()); + // ostringstream os; + // os << "CCJL: rid=" << rid << "; dbroot=" << dbRoot << "; partitionNum=" << partitionNum << "; + //segmentNum=" << segmentNum << "; stripeWithinPartition=" << stripeWithinPartition << "; OID=" << OID << " + //LBID=" << lbid; BRM::log(os.str()); } inline uint32_t ColumnCommandJL::getFBO(uint64_t lbid) { - uint32_t i; - uint64_t lastLBID; + uint32_t i; + uint64_t lastLBID; - for (i = 0; i < extents.size(); i++) + for (i = 0; i < extents.size(); i++) + { + lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; + + if (lbid >= (uint64_t)extents[i].range.start && lbid <= lastLBID) { - lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; + // @Bug 2889. Change for drop partition. Treat the FBO as if all of the partitions are + // still there as one or more partitions may have been dropped. Get the original index + // for this partition (i.e. what the index would be if all of the partitions were still + // there). The RIDs wind up being calculated off of this FBO for use in DML and DML + // needs calculates the partition number, segment number, etc. off of the RID and needs + // to remain the same when partitions are dropped. + // + // originalIndex = extents in partitions above + + // extents in this partition in stripes above + + // file offset in this stripe + uint32_t originalIndex = + (extents[i].partitionNum * fExtentsPerSegmentFile * fFilesPerColumnPartition) + + ((extents[i].blockOffset / extents[i].colWid / 1024) * fFilesPerColumnPartition) + + extents[i].segmentNum; - if (lbid >= (uint64_t) extents[i].range.start && lbid <= lastLBID) - { - - // @Bug 2889. Change for drop partition. Treat the FBO as if all of the partitions are - // still there as one or more partitions may have been dropped. Get the original index - // for this partition (i.e. what the index would be if all of the partitions were still - // there). The RIDs wind up being calculated off of this FBO for use in DML and DML - // needs calculates the partition number, segment number, etc. off of the RID and needs - // to remain the same when partitions are dropped. - // - // originalIndex = extents in partitions above + - // extents in this partition in stripes above + - // file offset in this stripe - uint32_t originalIndex = - (extents[i].partitionNum * fExtentsPerSegmentFile * fFilesPerColumnPartition) + - ((extents[i].blockOffset / extents[i].colWid / 1024) * fFilesPerColumnPartition) + - extents[i].segmentNum; - - return (lbid - extents[i].range.start) + (originalIndex << divShift); - } + return (lbid - extents[i].range.start) + (originalIndex << divShift); } + } - throw logic_error("ColumnCommandJL: didn't find the FBO?"); + throw logic_error("ColumnCommandJL: didn't find the FBO?"); } uint8_t ColumnCommandJL::getTableColumnType() { - switch (colType.colWidth) - { - case 8: - return TableColumn::UINT64; + switch (colType.colWidth) + { + case 8: return TableColumn::UINT64; - case 4: - return TableColumn::UINT32; + case 4: return TableColumn::UINT32; - case 2: - return TableColumn::UINT16; + case 2: return TableColumn::UINT16; - case 1: - return TableColumn::UINT8; + case 1: return TableColumn::UINT8; - // TODO MCOL-641 - case 16: - //fallthrough + // TODO MCOL-641 + case 16: + // fallthrough - default: - throw logic_error("ColumnCommandJL: bad column width"); - } + default: throw logic_error("ColumnCommandJL: bad column width"); + } } string ColumnCommandJL::toString() { - ostringstream ret; + ostringstream ret; - ret << "ColumnCommandJL: " << filterCount << " filters colwidth=" << - colType.colWidth << " oid=" << OID << " name=" << colName; + ret << "ColumnCommandJL: " << filterCount << " filters colwidth=" << colType.colWidth << " oid=" << OID + << " name=" << colName; - if (isScan) - ret << " (scan)"; + if (isScan) + ret << " (scan)"; - if (isDict()) - ret << " (tokens)"; - else if (datatypes::isCharType(colType.colDataType)) - ret << " (is char)"; + if (isDict()) + ret << " (tokens)"; + else if (datatypes::isCharType(colType.colDataType)) + ret << " (is char)"; - return ret.str(); + return ret.str(); } uint16_t ColumnCommandJL::getWidth() { - return colType.colWidth; + return colType.colWidth; } void ColumnCommandJL::reloadExtents() { - int err; - BRM::DBRM dbrm; + int err; + BRM::DBRM dbrm; - err = dbrm.getExtents(OID, extents); + err = dbrm.getExtents(OID, extents); - if (err) - { - ostringstream os; - os << "pColStep: BRM lookup error. Could not get extents for OID " << OID; - throw runtime_error(os.str()); - } + if (err) + { + ostringstream os; + os << "pColStep: BRM lookup error. Could not get extents for OID " << OID; + throw runtime_error(os.str()); + } - sort(extents.begin(), extents.end(), BRM::ExtentSorter()); + sort(extents.begin(), extents.end(), BRM::ExtentSorter()); } - - - - - -}; +}; // namespace joblist diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index 9206f484d..e00019025 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -37,100 +37,98 @@ namespace joblist { - class ColumnCommandJL : public CommandJL { -public: - ColumnCommandJL(const pColScanStep&, std::vector lastLBID); - ColumnCommandJL(const pColStep&); - virtual ~ColumnCommandJL(); + public: + ColumnCommandJL(const pColScanStep&, std::vector lastLBID); + ColumnCommandJL(const pColStep&); + virtual ~ColumnCommandJL(); - virtual void createCommand(messageqcpp::ByteStream& bs) const; - virtual void runCommand(messageqcpp::ByteStream& bs) const; - void setLBID(uint64_t rid, uint32_t dbroot); - uint8_t getTableColumnType(); - virtual std::string toString(); - uint16_t getWidth(); - CommandType getCommandType() - { - return COLUMN_COMMAND; - } - // @bug 1098 - uint8_t getBOP() const - { - return BOP; - } - const messageqcpp::ByteStream& getFilterString() - { - return filterString; - } - uint16_t getFilterCount() const - { - return filterCount; - } - const std::vector& getExtents() - { - return extents; - } - const execplan::CalpontSystemCatalog::ColType& getColType() const - { - return colType; - } - bool isDict() const - { - return fIsDict; - } + virtual void createCommand(messageqcpp::ByteStream& bs) const; + virtual void runCommand(messageqcpp::ByteStream& bs) const; + void setLBID(uint64_t rid, uint32_t dbroot); + uint8_t getTableColumnType(); + virtual std::string toString(); + uint16_t getWidth(); + CommandType getCommandType() + { + return COLUMN_COMMAND; + } + // @bug 1098 + uint8_t getBOP() const + { + return BOP; + } + const messageqcpp::ByteStream& getFilterString() + { + return filterString; + } + uint16_t getFilterCount() const + { + return filterCount; + } + const std::vector& getExtents() + { + return extents; + } + const execplan::CalpontSystemCatalog::ColType& getColType() const + { + return colType; + } + bool isDict() const + { + return fIsDict; + } - void scan(bool b) - { - isScan = b; - } - bool scan() - { - return isScan; - } + void scan(bool b) + { + isScan = b; + } + bool scan() + { + return isScan; + } - void reloadExtents(); + void reloadExtents(); -protected: - uint32_t currentExtentIndex; - messageqcpp::ByteStream filterString; - std::vector extents; - execplan::CalpontSystemCatalog::ColType colType; + protected: + uint32_t currentExtentIndex; + messageqcpp::ByteStream filterString; + std::vector extents; + execplan::CalpontSystemCatalog::ColType colType; -private: - ColumnCommandJL(); - ColumnCommandJL(const ColumnCommandJL&); + private: + ColumnCommandJL(); + ColumnCommandJL(const ColumnCommandJL&); - uint32_t getFBO(uint64_t lbid); + uint32_t getFBO(uint64_t lbid); - bool isScan; - uint64_t lbid; - uint32_t traceFlags; // probably move this to Command - uint8_t BOP; - uint32_t rpbShift, divShift, modMask; - uint16_t filterCount; - std::vector fLastLbid; + bool isScan; + uint64_t lbid; + uint32_t traceFlags; // probably move this to Command + uint8_t BOP; + uint32_t rpbShift, divShift, modMask; + uint16_t filterCount; + std::vector fLastLbid; - bool fIsDict; + bool fIsDict; - // @Bug 2889. Added two members below for drop partition enhancement. - // RJD: make sure that we keep enough significant digits around for partition math - uint64_t fFilesPerColumnPartition; - uint64_t fExtentsPerSegmentFile; + // @Bug 2889. Added two members below for drop partition enhancement. + // RJD: make sure that we keep enough significant digits around for partition math + uint64_t fFilesPerColumnPartition; + uint64_t fExtentsPerSegmentFile; - uint32_t numDBRoots; - uint32_t dbroot; + uint32_t numDBRoots; + uint32_t dbroot; - static const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 32; + static const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 32; - public: - // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile) - static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; + public: + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile) + static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; }; -} +} // namespace joblist #endif // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/command-jl.cpp b/dbcon/joblist/command-jl.cpp index 8bab37f95..663fb153b 100644 --- a/dbcon/joblist/command-jl.cpp +++ b/dbcon/joblist/command-jl.cpp @@ -25,35 +25,31 @@ using namespace messageqcpp; namespace joblist { - -CommandJL::CommandJL() : - bpp(0), - OID(0), - tupleKey(0xFFFFFFFF) +CommandJL::CommandJL() : bpp(0), OID(0), tupleKey(0xFFFFFFFF) { - queryUuid = bu::nil_generator()(); - stepUuid = bu::nil_generator()(); + queryUuid = bu::nil_generator()(); + stepUuid = bu::nil_generator()(); } -CommandJL::CommandJL(const CommandJL& c) : - bpp(c.bpp), - OID(c.OID), - tupleKey(c.tupleKey), - colName(c.colName), - queryUuid(c.queryUuid), - stepUuid(c.stepUuid) +CommandJL::CommandJL(const CommandJL& c) + : bpp(c.bpp) + , OID(c.OID) + , tupleKey(c.tupleKey) + , colName(c.colName) + , queryUuid(c.queryUuid) + , stepUuid(c.stepUuid) { } -CommandJL::~CommandJL() { }; +CommandJL::~CommandJL(){}; void CommandJL::createCommand(ByteStream& bs) const { - bs << OID; - bs << tupleKey; - bs << queryUuid; - bs << stepUuid; - // no need to send column name to PM as of rel.2.2.5. + bs << OID; + bs << tupleKey; + bs << queryUuid; + bs << stepUuid; + // no need to send column name to PM as of rel.2.2.5. } -} +} // namespace joblist diff --git a/dbcon/joblist/command-jl.h b/dbcon/joblist/command-jl.h index 00a7be8cb..d29177cc7 100644 --- a/dbcon/joblist/command-jl.h +++ b/dbcon/joblist/command-jl.h @@ -28,90 +28,88 @@ namespace joblist { - class BatchPrimitiveProcessorJL; class CommandJL { -public: - CommandJL(); - CommandJL(const CommandJL&); - virtual ~CommandJL(); + public: + CommandJL(); + CommandJL(const CommandJL&); + virtual ~CommandJL(); - // converts a rid or dictionary token to an LBID. For ColumnCommandJL it's a RID, - // for a DictStep it's a token. - virtual void setLBID(uint64_t data, uint32_t dbroot) = 0; - virtual uint8_t getTableColumnType() = 0; - virtual std::string toString() = 0; - virtual void createCommand(messageqcpp::ByteStream&) const; - virtual void runCommand(messageqcpp::ByteStream&) const = 0; - uint32_t getOID() const - { - return OID; - } - const std::string& getColName() const - { - return colName; - } - void setTupleKey(uint32_t tkey) - { - tupleKey = tkey; - } - uint32_t getTupleKey() const - { - return tupleKey; - } - virtual uint16_t getWidth() = 0; - const boost::uuids::uuid& getQueryUuid() const - { - return queryUuid; - } - void setQueryUuid(const boost::uuids::uuid& u) - { - queryUuid = u; - } - const boost::uuids::uuid& getStepUuid() const - { - return stepUuid; - } - void setStepUuid(const boost::uuids::uuid& u) - { - stepUuid = u; - } + // converts a rid or dictionary token to an LBID. For ColumnCommandJL it's a RID, + // for a DictStep it's a token. + virtual void setLBID(uint64_t data, uint32_t dbroot) = 0; + virtual uint8_t getTableColumnType() = 0; + virtual std::string toString() = 0; + virtual void createCommand(messageqcpp::ByteStream&) const; + virtual void runCommand(messageqcpp::ByteStream&) const = 0; + uint32_t getOID() const + { + return OID; + } + const std::string& getColName() const + { + return colName; + } + void setTupleKey(uint32_t tkey) + { + tupleKey = tkey; + } + uint32_t getTupleKey() const + { + return tupleKey; + } + virtual uint16_t getWidth() = 0; + const boost::uuids::uuid& getQueryUuid() const + { + return queryUuid; + } + void setQueryUuid(const boost::uuids::uuid& u) + { + queryUuid = u; + } + const boost::uuids::uuid& getStepUuid() const + { + return stepUuid; + } + void setStepUuid(const boost::uuids::uuid& u) + { + stepUuid = u; + } - void setBatchPrimitiveProcessor(BatchPrimitiveProcessorJL* b) - { - bpp = b; - } + void setBatchPrimitiveProcessor(BatchPrimitiveProcessorJL* b) + { + bpp = b; + } - enum CommandType - { - NONE, - COLUMN_COMMAND, - DICT_STEP, - DICT_SCAN, - PASS_THRU, - RID_TO_STRING, - FILTER_COMMAND, - PSEUDO_COLUMN - }; + enum CommandType + { + NONE, + COLUMN_COMMAND, + DICT_STEP, + DICT_SCAN, + PASS_THRU, + RID_TO_STRING, + FILTER_COMMAND, + PSEUDO_COLUMN + }; - virtual CommandType getCommandType() = 0; + virtual CommandType getCommandType() = 0; -protected: - BatchPrimitiveProcessorJL* bpp; - uint32_t OID; - uint32_t tupleKey; - std::string colName; // for stats - boost::uuids::uuid queryUuid; - boost::uuids::uuid stepUuid; - -private: + protected: + BatchPrimitiveProcessorJL* bpp; + uint32_t OID; + uint32_t tupleKey; + std::string colName; // for stats + boost::uuids::uuid queryUuid; + boost::uuids::uuid stepUuid; + private: }; typedef boost::shared_ptr SCommand; -} +} // namespace joblist #endif diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 1722c89d8..e0475c113 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -59,831 +59,773 @@ using namespace querytele; namespace joblist { - -CrossEngineStep::CrossEngineStep( - const std::string& schema, - const std::string& table, - const std::string& alias, - const JobInfo& jobInfo) : - BatchPrimitive(jobInfo), - fRowsRetrieved(0), - fRowsReturned(0), - fRowsPerGroup(256), - fOutputDL(NULL), - fOutputIterator(0), - fRunner(0), - fEndOfResult(false), - fSchema(schema), - fTable(table), - fAlias(alias), - fColumnCount(0), - fFeInstance(funcexp::FuncExp::instance()) +CrossEngineStep::CrossEngineStep(const std::string& schema, const std::string& table, + const std::string& alias, const JobInfo& jobInfo) + : BatchPrimitive(jobInfo) + , fRowsRetrieved(0) + , fRowsReturned(0) + , fRowsPerGroup(256) + , fOutputDL(NULL) + , fOutputIterator(0) + , fRunner(0) + , fEndOfResult(false) + , fSchema(schema) + , fTable(table) + , fAlias(alias) + , fColumnCount(0) + , fFeInstance(funcexp::FuncExp::instance()) { - fExtendedInfo = "CES: "; - getMysqldInfo(jobInfo); - fQtc.stepParms().stepType = StepTeleStats::T_CES; - mysql = new utils::LibMySQL(); + fExtendedInfo = "CES: "; + getMysqldInfo(jobInfo); + fQtc.stepParms().stepType = StepTeleStats::T_CES; + mysql = new utils::LibMySQL(); } - CrossEngineStep::~CrossEngineStep() { - delete mysql; + delete mysql; } - void CrossEngineStep::setOutputRowGroup(const rowgroup::RowGroup& rg) { - fRowGroupOut = fRowGroupDelivered = fRowGroupAdded = rg; + fRowGroupOut = fRowGroupDelivered = fRowGroupAdded = rg; } - void CrossEngineStep::deliverStringTableRowGroup(bool b) { - // results are either using setField, or mapping to delivered row group - fRowGroupDelivered.setUseStringTable(b); + // results are either using setField, or mapping to delivered row group + fRowGroupDelivered.setUseStringTable(b); } - bool CrossEngineStep::deliverStringTableRowGroup() const { - return fRowGroupDelivered.usesStringTable(); + return fRowGroupDelivered.usesStringTable(); } - void CrossEngineStep::addFcnJoinExp(const std::vector& fe) { - fFeFcnJoin = fe; + fFeFcnJoin = fe; } - void CrossEngineStep::addFcnExpGroup1(const boost::shared_ptr& fe) { - fFeFilters.push_back(fe); + fFeFilters.push_back(fe); } - void CrossEngineStep::setFE1Input(const rowgroup::RowGroup& rg) { - fRowGroupFe1 = rg; + fRowGroupFe1 = rg; } - void CrossEngineStep::setFcnExpGroup3(const std::vector& fe) { - fFeSelects = fe; + fFeSelects = fe; } - void CrossEngineStep::setFE23Output(const rowgroup::RowGroup& rg) { - fRowGroupFe3 = fRowGroupDelivered = fRowGroupAdded = rg; + fRowGroupFe3 = fRowGroupDelivered = fRowGroupAdded = rg; } - void CrossEngineStep::makeMappings() { - fFe1Column.reset(new int[fColumnCount]); + fFe1Column.reset(new int[fColumnCount]); - for (uint64_t i = 0; i < fColumnCount; ++i) - fFe1Column[i] = -1; + for (uint64_t i = 0; i < fColumnCount; ++i) + fFe1Column[i] = -1; - if (fFeFilters.size() > 0 || fFeFcnJoin.size() > 0) + if (fFeFilters.size() > 0 || fFeFcnJoin.size() > 0) + { + const std::vector& colInFe1 = fRowGroupFe1.getKeys(); + + for (uint64_t i = 0; i < colInFe1.size(); i++) { - const std::vector& colInFe1 = fRowGroupFe1.getKeys(); + map::iterator it = fColumnMap.find(colInFe1[i]); - for (uint64_t i = 0; i < colInFe1.size(); i++) - { - map::iterator it = fColumnMap.find(colInFe1[i]); - - if (it != fColumnMap.end()) - fFe1Column[it->second] = i; - } - - fFeMapping1 = makeMapping(fRowGroupFe1, fRowGroupOut); + if (it != fColumnMap.end()) + fFe1Column[it->second] = i; } - if (!fFeSelects.empty()) - fFeMapping3 = makeMapping(fRowGroupOut, fRowGroupFe3); -} + fFeMapping1 = makeMapping(fRowGroupFe1, fRowGroupOut); + } + if (!fFeSelects.empty()) + fFeMapping3 = makeMapping(fRowGroupOut, fRowGroupFe3); +} void CrossEngineStep::setField(int i, const char* value, unsigned long length, MYSQL_FIELD* field, Row& row) { - CalpontSystemCatalog::ColDataType colType = row.getColType(i); + CalpontSystemCatalog::ColDataType colType = row.getColType(i); - if ((colType == CalpontSystemCatalog::CHAR || colType == CalpontSystemCatalog::VARCHAR) && - row.getColumnWidth(i) > 8) + if ((colType == CalpontSystemCatalog::CHAR || colType == CalpontSystemCatalog::VARCHAR) && + row.getColumnWidth(i) > 8) + { + if (value != NULL) + row.setStringField(value, i); + else + row.setStringField("", i); + } + else if ((colType == CalpontSystemCatalog::BLOB) || (colType == CalpontSystemCatalog::TEXT) || + (colType == CalpontSystemCatalog::VARBINARY)) + { + if (value != NULL) + row.setVarBinaryField((const uint8_t*)value, length, i); + else + row.setVarBinaryField(NULL, 0, i); + } + else + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = colType; + ct.colWidth = row.getColumnWidth(i); + + if (colType == CalpontSystemCatalog::DECIMAL || colType == CalpontSystemCatalog::UDECIMAL) { - if (value != NULL) - row.setStringField(value, i); - else - row.setStringField("", i); - } - else if ((colType == CalpontSystemCatalog::BLOB) || - (colType == CalpontSystemCatalog::TEXT) || - (colType == CalpontSystemCatalog::VARBINARY)) - { - if (value != NULL) - row.setVarBinaryField((const uint8_t*)value, length, i); - else - row.setVarBinaryField(NULL, 0, i); + ct.scale = field->decimals; + ct.precision = field->length; + + if (ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + row.setInt128Field(convertValueNum(value, ct), i); + return; + } } else { - CalpontSystemCatalog::ColType ct; - ct.colDataType = colType; - ct.colWidth = row.getColumnWidth(i); - - if (colType == CalpontSystemCatalog::DECIMAL || - colType == CalpontSystemCatalog::UDECIMAL) - { - ct.scale = field->decimals; - ct.precision = field->length; - - if (ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - row.setInt128Field(convertValueNum(value, ct), i); - return; - } - } - else - { - ct.scale = row.getScale(i); - ct.precision = row.getPrecision(i); - } - - row.setIntField(convertValueNum(value, ct), i); + ct.scale = row.getScale(i); + ct.precision = row.getPrecision(i); } -} + row.setIntField(convertValueNum(value, ct), i); + } +} inline void CrossEngineStep::addRow(RGData& data) { - fRowDelivered.setRid(fRowsReturned % fRowsPerGroup); - fRowDelivered.nextRow(); - fRowGroupAdded.incRowCount(); + fRowDelivered.setRid(fRowsReturned % fRowsPerGroup); + fRowDelivered.nextRow(); + fRowGroupAdded.incRowCount(); - if (++fRowsReturned % fRowsPerGroup == 0) - { - fOutputDL->insert(data); - data.reinit(fRowGroupAdded, fRowsPerGroup); - fRowGroupAdded.setData(&data); - fRowGroupAdded.resetRowGroup(fRowsReturned); - fRowGroupAdded.getRow(0, &fRowDelivered); - } + if (++fRowsReturned % fRowsPerGroup == 0) + { + fOutputDL->insert(data); + data.reinit(fRowGroupAdded, fRowsPerGroup); + fRowGroupAdded.setData(&data); + fRowGroupAdded.resetRowGroup(fRowsReturned); + fRowGroupAdded.getRow(0, &fRowDelivered); + } } - // simplified version of convertValueNum() in jlf_execplantojoblist.cpp. template -T CrossEngineStep::convertValueNum( - const char* str, const CalpontSystemCatalog::ColType& ct) +T CrossEngineStep::convertValueNum(const char* str, const CalpontSystemCatalog::ColType& ct) { - T rv = 0; - bool pushWarning = false; - bool nullFlag = (str == NULL); - boost::any anyVal = ct.convertColumnData((nullFlag ? "" : str), pushWarning, fTimeZone, nullFlag, true, false); + T rv = 0; + bool pushWarning = false; + bool nullFlag = (str == NULL); + boost::any anyVal = + ct.convertColumnData((nullFlag ? "" : str), pushWarning, fTimeZone, nullFlag, true, false); - // Out of range values are treated as NULL as discussed during design review. - if (pushWarning) - { - anyVal = ct.getNullValueForType(); - } + // Out of range values are treated as NULL as discussed during design review. + if (pushWarning) + { + anyVal = ct.getNullValueForType(); + } - switch (ct.colDataType) - { - case CalpontSystemCatalog::BIT: - rv = boost::any_cast(anyVal); - break; + switch (ct.colDataType) + { + case CalpontSystemCatalog::BIT: rv = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::TINYINT: #if BOOST_VERSION >= 105200 - rv = boost::any_cast(anyVal); + rv = boost::any_cast(anyVal); #else - rv = boost::any_cast(anyVal); + rv = boost::any_cast(anyVal); #endif - break; + break; - case CalpontSystemCatalog::UTINYINT: - rv = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::UTINYINT: rv = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::SMALLINT: - rv = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::SMALLINT: rv = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::USMALLINT: - rv = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::USMALLINT: rv = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: #ifdef _MSC_VER - rv = boost::any_cast(anyVal); + rv = boost::any_cast(anyVal); #else - rv = boost::any_cast(anyVal); + rv = boost::any_cast(anyVal); #endif - break; + break; - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: #ifdef _MSC_VER - rv = boost::any_cast(anyVal); + rv = boost::any_cast(anyVal); #else - rv = boost::any_cast(anyVal); + rv = boost::any_cast(anyVal); #endif - break; + break; - case CalpontSystemCatalog::BIGINT: - rv = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::BIGINT: rv = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::UBIGINT: - rv = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::UBIGINT: rv = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float f = boost::any_cast(anyVal); + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float f = boost::any_cast(anyVal); - //N.B. There is a bug in boost::any or in gcc where, if you store a nan, - // you will get back a nan, but not necessarily the same bits that you put in. - // This only seems to be for float (double seems to work). - if (isnan(f)) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - f = *tfp; - } + // N.B. There is a bug in boost::any or in gcc where, if you store a nan, + // you will get back a nan, but not necessarily the same bits that you put in. + // This only seems to be for float (double seems to work). + if (isnan(f)) + { + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + f = *tfp; + } - float* fp = &f; - int32_t* ip = reinterpret_cast(fp); - rv = *ip; - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double d = boost::any_cast(anyVal); - double* dp = &d; - int64_t* ip = reinterpret_cast(dp); - rv = *ip; - } - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - { - std::string i = boost::any_cast(anyVal); - // bug 1932, pad nulls up to the size of v - i.resize(sizeof(rv), 0); - rv = *((uint64_t*) i.data()); - } - break; - - case CalpontSystemCatalog::DATE: - rv = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DATETIME: - rv = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIMESTAMP: - rv = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIME: - rv = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) - rv = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) - rv = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) -#ifdef _MSC_VER - rv = boost::any_cast(anyVal); - -#else - rv = boost::any_cast(anyVal); -#endif - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - rv = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - rv = boost::any_cast(anyVal); - break; - - default: - break; + float* fp = &f; + int32_t* ip = reinterpret_cast(fp); + rv = *ip; } + break; - return rv; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double d = boost::any_cast(anyVal); + double* dp = &d; + int64_t* ip = reinterpret_cast(dp); + rv = *ip; + } + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: + { + std::string i = boost::any_cast(anyVal); + // bug 1932, pad nulls up to the size of v + i.resize(sizeof(rv), 0); + rv = *((uint64_t*)i.data()); + } + break; + + case CalpontSystemCatalog::DATE: rv = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DATETIME: rv = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::TIMESTAMP: rv = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::TIME: rv = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + rv = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) + rv = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) +#ifdef _MSC_VER + rv = boost::any_cast(anyVal); + +#else + rv = boost::any_cast(anyVal); +#endif + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + rv = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + rv = boost::any_cast(anyVal); + break; + + default: break; + } + + return rv; } - void CrossEngineStep::getMysqldInfo(const JobInfo& jobInfo) { - if (jobInfo.rm->getMysqldInfo(fHost, fUser, fPasswd, fPort) == false) - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_CROSS_ENGINE_CONFIG), - ERR_CROSS_ENGINE_CONFIG); + if (jobInfo.rm->getMysqldInfo(fHost, fUser, fPasswd, fPort) == false) + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_CROSS_ENGINE_CONFIG), ERR_CROSS_ENGINE_CONFIG); } - void CrossEngineStep::run() { -// idbassert(!fDelivery); + // idbassert(!fDelivery); - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for cross engine step."); + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for cross engine step."); - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); - if (fDelivery == true) - { - fOutputIterator = fOutputDL->getIterator(); - } + if (fDelivery == true) + { + fOutputIterator = fOutputDL->getIterator(); + } - fRunner = jobstepThreadPool.invoke(Runner(this)); + fRunner = jobstepThreadPool.invoke(Runner(this)); } - void CrossEngineStep::join() { - if (fRunner) - jobstepThreadPool.join(fRunner); + if (fRunner) + jobstepThreadPool.join(fRunner); } - void CrossEngineStep::execute() { - int ret = 0; - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; + int ret = 0; + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; - try - { - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); + try + { + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); - ret = mysql->init(fHost.c_str(), fPort, fUser.c_str(), fPasswd.c_str(), fSchema.c_str()); + ret = mysql->init(fHost.c_str(), fPort, fUser.c_str(), fPasswd.c_str(), fSchema.c_str()); - if (ret != 0) - mysql->handleMySqlError(mysql->getError().c_str(), ret); + if (ret != 0) + mysql->handleMySqlError(mysql->getError().c_str(), ret); - std::string query(makeQuery()); - fLogger->logMessage(logging::LOG_TYPE_INFO, "QUERY to foreign engine: " + query); + std::string query(makeQuery()); + fLogger->logMessage(logging::LOG_TYPE_INFO, "QUERY to foreign engine: " + query); - if (traceOn()) - cout << "QUERY: " << query << endl; - - ret = mysql->run(query.c_str()); - - if (ret != 0) - mysql->handleMySqlError(mysql->getError().c_str(), ret); - - int num_fields = mysql->getFieldCount(); - - char** rowIn; // input - //shared_array rgDataDelivered; // output - RGData rgDataDelivered; - fRowGroupAdded.initRow(&fRowDelivered); - // use getDataSize() i/o getMaxDataSize() to make sure there are 8192 rows. - rgDataDelivered.reinit(fRowGroupAdded, fRowsPerGroup); - fRowGroupAdded.setData(&rgDataDelivered); - fRowGroupAdded.resetRowGroup(0); - fRowGroupAdded.getRow(0, &fRowDelivered); - - if (traceOn()) - dlTimes.setFirstReadTime(); - - // Any functions to evaluate - makeMappings(); - bool doFE1 = ((fFeFcnJoin.size() > 0) || (fFeFilters.size() > 0)); - bool doFE3 = (fFeSelects.size() > 0); - - if (!doFE1 && !doFE3) - { - while ((rowIn = mysql->nextRow()) && !cancelled()) - { - for (int i = 0; i < num_fields; i++) - setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), fRowDelivered); - - addRow(rgDataDelivered); - } - } - - else if (doFE1 && !doFE3) // FE in WHERE clause only - { - shared_array rgDataFe1; // functions in where clause - Row rowFe1; // row for fe evaluation - fRowGroupFe1.initRow(&rowFe1, true); - rgDataFe1.reset(new uint8_t[rowFe1.getSize()]); - rowFe1.setData(rgDataFe1.get()); - - while ((rowIn = mysql->nextRow()) && !cancelled()) - { - // Parse the columns used in FE1 first, the other column may not need be parsed. - for (int i = 0; i < num_fields; i++) - { - if (fFe1Column[i] != -1) - setField(fFe1Column[i], rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe1); - } - - if (fFeFilters.size() > 0) - { - bool feBreak = false; - - for (std::vector >::iterator it = fFeFilters.begin(); it != fFeFilters.end(); it++) - { - if (fFeInstance->evaluate(rowFe1, (*it).get()) == false) - { - feBreak = true; - break; - } - } - - if (feBreak) - continue; - } - - // evaluate the FE join column - fFeInstance->evaluate(rowFe1, fFeFcnJoin); - - // Pass throug the parsed columns, and parse the remaining columns. - applyMapping(fFeMapping1, rowFe1, &fRowDelivered); - - for (int i = 0; i < num_fields; i++) - { - if (fFe1Column[i] == -1) - setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), fRowDelivered); - } - - addRow(rgDataDelivered); - } - } - - else if (!doFE1 && doFE3) // FE in SELECT clause only - { - shared_array rgDataFe3; // functions in select clause - Row rowFe3; // row for fe evaluation - fRowGroupOut.initRow(&rowFe3, true); - rgDataFe3.reset(new uint8_t[rowFe3.getSize()]); - rowFe3.setData(rgDataFe3.get()); - - while ((rowIn = mysql->nextRow()) && !cancelled()) - { - for (int i = 0; i < num_fields; i++) - setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe3); - - fFeInstance->evaluate(rowFe3, fFeSelects); - - applyMapping(fFeMapping3, rowFe3, &fRowDelivered); - - addRow(rgDataDelivered); - } - } - - else // FE in SELECT clause, FE join and WHERE clause - { - shared_array rgDataFe1; // functions in where clause - Row rowFe1; // row for fe1 evaluation - fRowGroupFe1.initRow(&rowFe1, true); - rgDataFe1.reset(new uint8_t[rowFe1.getSize()]); - rowFe1.setData(rgDataFe1.get()); - - shared_array rgDataFe3; // functions in select clause - Row rowFe3; // row for fe3 evaluation - fRowGroupOut.initRow(&rowFe3, true); - rgDataFe3.reset(new uint8_t[rowFe3.getSize()]); - rowFe3.setData(rgDataFe3.get()); - - while ((rowIn = mysql->nextRow()) && !cancelled()) - { - // Parse the columns used in FE1 first, the other column may not need be parsed. - for (int i = 0; i < num_fields; i++) - { - if (fFe1Column[i] != -1) - setField(fFe1Column[i], rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe1); - } - - if (fFeFilters.size() > 0) - { - bool feBreak = false; - - for (std::vector >::iterator it = fFeFilters.begin(); it != fFeFilters.end(); it++) - { - if (fFeInstance->evaluate(rowFe1, (*it).get()) == false) - { - feBreak = true; - break; - } - } - - if (feBreak) - continue; - } - - // evaluate the FE join column - fFeInstance->evaluate(rowFe1, fFeFcnJoin); - - // Pass throug the parsed columns, and parse the remaining columns. - applyMapping(fFeMapping1, rowFe1, &rowFe3); - - for (int i = 0; i < num_fields; i++) - { - if (fFe1Column[i] == -1) - setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe3); - } - - fFeInstance->evaluate(rowFe3, fFeSelects); - applyMapping(fFeMapping3, rowFe3, &fRowDelivered); - - addRow(rgDataDelivered); - } - } - - //INSERT_ADAPTER(fOutputDL, rgDataDelivered); - fOutputDL->insert(rgDataDelivered); - fRowsRetrieved = mysql->getRowCount(); - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_CROSS_ENGINE_CONNECT, - logging::ERR_ALWAYS_CRITICAL, - "CrossEngineStep::execute()"); - } - - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - fEndOfResult = true; - fOutputDL->endOfInput(); - - // Bug 3136, let mini stats to be formatted if traceOn. if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } -} + cout << "QUERY: " << query << endl; + ret = mysql->run(query.c_str()); + + if (ret != 0) + mysql->handleMySqlError(mysql->getError().c_str(), ret); + + int num_fields = mysql->getFieldCount(); + + char** rowIn; // input + // shared_array rgDataDelivered; // output + RGData rgDataDelivered; + fRowGroupAdded.initRow(&fRowDelivered); + // use getDataSize() i/o getMaxDataSize() to make sure there are 8192 rows. + rgDataDelivered.reinit(fRowGroupAdded, fRowsPerGroup); + fRowGroupAdded.setData(&rgDataDelivered); + fRowGroupAdded.resetRowGroup(0); + fRowGroupAdded.getRow(0, &fRowDelivered); + + if (traceOn()) + dlTimes.setFirstReadTime(); + + // Any functions to evaluate + makeMappings(); + bool doFE1 = ((fFeFcnJoin.size() > 0) || (fFeFilters.size() > 0)); + bool doFE3 = (fFeSelects.size() > 0); + + if (!doFE1 && !doFE3) + { + while ((rowIn = mysql->nextRow()) && !cancelled()) + { + for (int i = 0; i < num_fields; i++) + setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), fRowDelivered); + + addRow(rgDataDelivered); + } + } + + else if (doFE1 && !doFE3) // FE in WHERE clause only + { + shared_array rgDataFe1; // functions in where clause + Row rowFe1; // row for fe evaluation + fRowGroupFe1.initRow(&rowFe1, true); + rgDataFe1.reset(new uint8_t[rowFe1.getSize()]); + rowFe1.setData(rgDataFe1.get()); + + while ((rowIn = mysql->nextRow()) && !cancelled()) + { + // Parse the columns used in FE1 first, the other column may not need be parsed. + for (int i = 0; i < num_fields; i++) + { + if (fFe1Column[i] != -1) + setField(fFe1Column[i], rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe1); + } + + if (fFeFilters.size() > 0) + { + bool feBreak = false; + + for (std::vector >::iterator it = fFeFilters.begin(); + it != fFeFilters.end(); it++) + { + if (fFeInstance->evaluate(rowFe1, (*it).get()) == false) + { + feBreak = true; + break; + } + } + + if (feBreak) + continue; + } + + // evaluate the FE join column + fFeInstance->evaluate(rowFe1, fFeFcnJoin); + + // Pass throug the parsed columns, and parse the remaining columns. + applyMapping(fFeMapping1, rowFe1, &fRowDelivered); + + for (int i = 0; i < num_fields; i++) + { + if (fFe1Column[i] == -1) + setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), fRowDelivered); + } + + addRow(rgDataDelivered); + } + } + + else if (!doFE1 && doFE3) // FE in SELECT clause only + { + shared_array rgDataFe3; // functions in select clause + Row rowFe3; // row for fe evaluation + fRowGroupOut.initRow(&rowFe3, true); + rgDataFe3.reset(new uint8_t[rowFe3.getSize()]); + rowFe3.setData(rgDataFe3.get()); + + while ((rowIn = mysql->nextRow()) && !cancelled()) + { + for (int i = 0; i < num_fields; i++) + setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe3); + + fFeInstance->evaluate(rowFe3, fFeSelects); + + applyMapping(fFeMapping3, rowFe3, &fRowDelivered); + + addRow(rgDataDelivered); + } + } + + else // FE in SELECT clause, FE join and WHERE clause + { + shared_array rgDataFe1; // functions in where clause + Row rowFe1; // row for fe1 evaluation + fRowGroupFe1.initRow(&rowFe1, true); + rgDataFe1.reset(new uint8_t[rowFe1.getSize()]); + rowFe1.setData(rgDataFe1.get()); + + shared_array rgDataFe3; // functions in select clause + Row rowFe3; // row for fe3 evaluation + fRowGroupOut.initRow(&rowFe3, true); + rgDataFe3.reset(new uint8_t[rowFe3.getSize()]); + rowFe3.setData(rgDataFe3.get()); + + while ((rowIn = mysql->nextRow()) && !cancelled()) + { + // Parse the columns used in FE1 first, the other column may not need be parsed. + for (int i = 0; i < num_fields; i++) + { + if (fFe1Column[i] != -1) + setField(fFe1Column[i], rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe1); + } + + if (fFeFilters.size() > 0) + { + bool feBreak = false; + + for (std::vector >::iterator it = fFeFilters.begin(); + it != fFeFilters.end(); it++) + { + if (fFeInstance->evaluate(rowFe1, (*it).get()) == false) + { + feBreak = true; + break; + } + } + + if (feBreak) + continue; + } + + // evaluate the FE join column + fFeInstance->evaluate(rowFe1, fFeFcnJoin); + + // Pass throug the parsed columns, and parse the remaining columns. + applyMapping(fFeMapping1, rowFe1, &rowFe3); + + for (int i = 0; i < num_fields; i++) + { + if (fFe1Column[i] == -1) + setField(i, rowIn[i], mysql->getFieldLength(i), mysql->getField(i), rowFe3); + } + + fFeInstance->evaluate(rowFe3, fFeSelects); + applyMapping(fFeMapping3, rowFe3, &fRowDelivered); + + addRow(rgDataDelivered); + } + } + + // INSERT_ADAPTER(fOutputDL, rgDataDelivered); + fOutputDL->insert(rgDataDelivered); + fRowsRetrieved = mysql->getRowCount(); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_CROSS_ENGINE_CONNECT, logging::ERR_ALWAYS_CRITICAL, + "CrossEngineStep::execute()"); + } + + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + fEndOfResult = true; + fOutputDL->endOfInput(); + + // Bug 3136, let mini stats to be formatted if traceOn. + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } +} void CrossEngineStep::setBPP(JobStep* jobStep) { - pColStep* pcs = dynamic_cast(jobStep); - pColScanStep* pcss = NULL; - pDictionaryStep* pds = NULL; - pDictionaryScan* pdss = NULL; - FilterStep* fs = NULL; - std::string bop = " AND "; + pColStep* pcs = dynamic_cast(jobStep); + pColScanStep* pcss = NULL; + pDictionaryStep* pds = NULL; + pDictionaryScan* pdss = NULL; + FilterStep* fs = NULL; + std::string bop = " AND "; - if (pcs != 0) - { - if (dynamic_cast(pcs) != NULL) - throw logic_error("No Psedo Column for foreign engine."); + if (pcs != 0) + { + if (dynamic_cast(pcs) != NULL) + throw logic_error("No Psedo Column for foreign engine."); - if (pcs->BOP() == BOP_OR) - bop = " OR "; + if (pcs->BOP() == BOP_OR) + bop = " OR "; - addFilterStr(pcs->getFilters(), bop); - } - else if ((pcss = dynamic_cast(jobStep)) != NULL) - { - if (pcss->BOP() == BOP_OR) - bop = " OR "; + addFilterStr(pcs->getFilters(), bop); + } + else if ((pcss = dynamic_cast(jobStep)) != NULL) + { + if (pcss->BOP() == BOP_OR) + bop = " OR "; - addFilterStr(pcss->getFilters(), bop); - } - else if ((pds = dynamic_cast(jobStep)) != NULL) - { - if (pds->BOP() == BOP_OR) - bop = " OR "; + addFilterStr(pcss->getFilters(), bop); + } + else if ((pds = dynamic_cast(jobStep)) != NULL) + { + if (pds->BOP() == BOP_OR) + bop = " OR "; - addFilterStr(pds->getFilters(), bop); - } - else if ((pdss = dynamic_cast(jobStep)) != NULL) - { - if (pds->BOP() == BOP_OR) - bop = " OR "; + addFilterStr(pds->getFilters(), bop); + } + else if ((pdss = dynamic_cast(jobStep)) != NULL) + { + if (pds->BOP() == BOP_OR) + bop = " OR "; - addFilterStr(pdss->getFilters(), bop); - } - else if ((fs = dynamic_cast(jobStep)) != NULL) - { - addFilterStr(fs->getFilters(), bop); - } + addFilterStr(pdss->getFilters(), bop); + } + else if ((fs = dynamic_cast(jobStep)) != NULL) + { + addFilterStr(fs->getFilters(), bop); + } } void CrossEngineStep::addFilterStr(const std::vector& f, const std::string& bop) { - if (f.size() == 0) - return; + if (f.size() == 0) + return; - std::string filterStr; + std::string filterStr; - for (uint64_t i = 0; i < f.size(); i++) - { - if (f[i]->data().empty()) - continue; - - if (!filterStr.empty()) - filterStr += bop; - - filterStr += f[i]->data(); - } + for (uint64_t i = 0; i < f.size(); i++) + { + if (f[i]->data().empty()) + continue; if (!filterStr.empty()) - { - if (!fWhereClause.empty()) - fWhereClause += " AND (" + filterStr + ")"; - else - fWhereClause += " WHERE (" + filterStr + ")"; - } -} + filterStr += bop; + filterStr += f[i]->data(); + } + + if (!filterStr.empty()) + { + if (!fWhereClause.empty()) + fWhereClause += " AND (" + filterStr + ")"; + else + fWhereClause += " WHERE (" + filterStr + ")"; + } +} void CrossEngineStep::setProjectBPP(JobStep* jobStep1, JobStep*) { - fColumnMap[jobStep1->tupleId()] = fColumnCount++; + fColumnMap[jobStep1->tupleId()] = fColumnCount++; - if (!fSelectClause.empty()) - fSelectClause += ", "; - else - fSelectClause += "SELECT "; + if (!fSelectClause.empty()) + fSelectClause += ", "; + else + fSelectClause += "SELECT "; - fSelectClause += "`" + jobStep1->name() + "`"; + fSelectClause += "`" + jobStep1->name() + "`"; } - std::string CrossEngineStep::makeQuery() { - ostringstream oss; - oss << fSelectClause << " FROM `" << fTable << "`"; + ostringstream oss; + oss << fSelectClause << " FROM `" << fTable << "`"; - if (fTable.compare(fAlias) != 0) - oss << " `" << fAlias << "`"; + if (fTable.compare(fAlias) != 0) + oss << " `" << fAlias << "`"; - if (!fWhereClause.empty()) - oss << fWhereClause; + if (!fWhereClause.empty()) + oss << fWhereClause; - // the std::string must consist of a single SQL statement without a terminating semicolon ; or \g. - // oss << ";"; - return oss.str(); + // the std::string must consist of a single SQL statement without a terminating semicolon ; or \g. + // oss << ";"; + return oss.str(); } const RowGroup& CrossEngineStep::getOutputRowGroup() const { - return fRowGroupOut; + return fRowGroupOut; } - const RowGroup& CrossEngineStep::getDeliveredRowGroup() const { - return fRowGroupDelivered; + return fRowGroupDelivered; } - uint32_t CrossEngineStep::nextBand(messageqcpp::ByteStream& bs) { - //shared_array rgDataOut; - RGData rgDataOut; - bool more = false; - uint32_t rowCount = 0; + // shared_array rgDataOut; + RGData rgDataOut; + bool more = false; + uint32_t rowCount = 0; - try + try + { + bs.restart(); + + more = fOutputDL->next(fOutputIterator, &rgDataOut); + + if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + if (more && !cancelled()) { - bs.restart(); - + fRowGroupDelivered.setData(&rgDataOut); + fRowGroupDelivered.serializeRGData(bs); + rowCount = fRowGroupDelivered.getRowCount(); + } + else + { + while (more) more = fOutputDL->next(fOutputIterator, &rgDataOut); - if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - if (more && !cancelled()) - { - fRowGroupDelivered.setData(&rgDataOut); - fRowGroupDelivered.serializeRGData(bs); - rowCount = fRowGroupDelivered.getRowCount(); - } - else - { - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - - fEndOfResult = true; - } + fEndOfResult = true; } - catch (...) + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_DELIVERY, logging::ERR_ALWAYS_CRITICAL, + "CrossEngineStep::nextBand()"); + while (more) + more = fOutputDL->next(fOutputIterator, &rgDataOut); + + fEndOfResult = true; + } + + if (fEndOfResult) + { + // send an empty / error band + rgDataOut.reinit(fRowGroupDelivered, 0); + fRowGroupDelivered.setData(&rgDataOut); + fRowGroupDelivered.resetRowGroup(0); + fRowGroupDelivered.setStatus(status()); + fRowGroupDelivered.serializeRGData(bs); + + if (traceOn()) { - handleException(std::current_exception(), - logging::ERR_IN_DELIVERY, - logging::ERR_ALWAYS_CRITICAL, - "CrossEngineStep::nextBand()"); - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - - fEndOfResult = true; + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); } - if (fEndOfResult) - { - // send an empty / error band - rgDataOut.reinit(fRowGroupDelivered, 0); - fRowGroupDelivered.setData(&rgDataOut); - fRowGroupDelivered.resetRowGroup(0); - fRowGroupDelivered.setStatus(status()); - fRowGroupDelivered.serializeRGData(bs); + if (traceOn()) + printCalTrace(); + } - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - } - - if (traceOn()) - printCalTrace(); - } - - return rowCount; + return rowCount; } - const std::string CrossEngineStep::toString() const { - ostringstream oss; - oss << "CrossEngineStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "CrossEngineStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " in:"; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); - oss << " out:"; + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } - void CrossEngineStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; rows retrieved-" << fRowsRetrieved - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString << "; rows retrieved-" + << fRowsRetrieved << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } - void CrossEngineStep::formatMiniStats() { - ostringstream oss; - oss << "CES " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "CES " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); } -} +} // namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/crossenginestep.h b/dbcon/joblist/crossenginestep.h index bce3ee048..71c7386f1 100644 --- a/dbcon/joblist/crossenginestep.h +++ b/dbcon/joblist/crossenginestep.h @@ -17,7 +17,6 @@ // $Id: crossenginestep.h 9620 2013-06-13 15:51:52Z pleblanc $ - #ifndef JOBLIST_CROSSENGINESTEP_H #define JOBLIST_CROSSENGINESTEP_H @@ -36,11 +35,11 @@ namespace utils class LibMySQL; } -namespace execplan +namespace execplan { class ParseTree; class ReturnedColumn; -} +} // namespace execplan namespace funcexp { @@ -54,191 +53,203 @@ namespace joblist */ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep { -public: - /** @brief CrossEngineStep constructor - */ - CrossEngineStep( - const std::string& schema, - const std::string& table, - const std::string& alias, - const JobInfo& jobInfo); + public: + /** @brief CrossEngineStep constructor + */ + CrossEngineStep(const std::string& schema, const std::string& table, const std::string& alias, + const JobInfo& jobInfo); - /** @brief CrossEngineStep destructor - */ - ~CrossEngineStep(); + /** @brief CrossEngineStep destructor + */ + ~CrossEngineStep(); - /** @brief virtual void Run method - */ - void run(); + /** @brief virtual void Run method + */ + void run(); - /** @brief virtual void join method - */ - void join(); + /** @brief virtual void join method + */ + void join(); - /** @brief virtual string toString method - */ - const std::string toString() const; + /** @brief virtual string toString method + */ + const std::string toString() const; - // from BatchPrimitive - bool getFeederFlag() const + // from BatchPrimitive + bool getFeederFlag() const + { + return false; + } + uint64_t getLastTupleId() const + { + return 0; + } + uint32_t getStepCount() const + { + return 1; + } + void setBPP(JobStep* jobStep); + void setFirstStepType(PrimitiveStepType firstStepType) + { + } + void setIsProjectionOnly() + { + } + void setLastTupleId(uint64_t id) + { + } + void setOutputType(BPSOutputType outputType) + { + } + void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2); + void setStepCount() + { + } + void setSwallowRows(const bool swallowRows) + { + } + void setBppStep() + { + } + void dec(DistributedEngineComm* dec) + { + } + const OIDVector& getProjectOids() const + { + return fOIDVector; + } + uint64_t blksSkipped() const + { + return 0; + } + bool wasStepRun() const + { + return fRunExecuted; + } + BPSOutputType getOutputType() const + { + return ROW_GROUP; + } + uint64_t getRows() const + { + return fRowsReturned; + } + const std::string& schemaName() const + { + return fSchema; + } + const std::string& tableName() const + { + return fTable; + } + const std::string& tableAlias() const + { + return fAlias; + } + void setJobInfo(const JobInfo* jobInfo) + { + } + void setOutputRowGroup(const rowgroup::RowGroup&); + const rowgroup::RowGroup& getOutputRowGroup() const; + + // from DECEventListener + void newPMOnline(uint32_t) + { + } + + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; + uint32_t nextBand(messageqcpp::ByteStream& bs); + + void addFcnJoinExp(const std::vector&); + void addFcnExpGroup1(const boost::shared_ptr&); + void setFE1Input(const rowgroup::RowGroup&); + void setFcnExpGroup3(const std::vector&); + void setFE23Output(const rowgroup::RowGroup&); + + void addFilter(JobStep* jobStep); + void addProject(JobStep* jobStep); + + protected: + virtual void execute(); + virtual void getMysqldInfo(const JobInfo&); + virtual void makeMappings(); + virtual void addFilterStr(const std::vector&, const std::string&); + virtual std::string makeQuery(); + virtual void setField(int, const char*, unsigned long, MYSQL_FIELD*, rowgroup::Row&); + inline void addRow(rowgroup::RGData&); + // inline void addRow(boost::shared_array&); + template + T convertValueNum(const char*, const execplan::CalpontSystemCatalog::ColType&); + virtual void formatMiniStats(); + virtual void printCalTrace(); + + uint64_t fRowsRetrieved; + uint64_t fRowsReturned; + uint64_t fRowsPerGroup; + + // output rowgroup and row + rowgroup::RowGroup fRowGroupOut; + rowgroup::RowGroup fRowGroupDelivered; + rowgroup::RowGroup fRowGroupAdded; + rowgroup::Row fRowDelivered; + + // for datalist + RowGroupDL* fOutputDL; + uint64_t fOutputIterator; + + class Runner + { + public: + Runner(CrossEngineStep* step) : fStep(step) { - return false; } - uint64_t getLastTupleId() const + void operator()() { - return 0; + utils::setThreadName("CESRunner"); + fStep->execute(); } - uint32_t getStepCount () const - { - return 1; - } - void setBPP(JobStep* jobStep); - void setFirstStepType(PrimitiveStepType firstStepType) {} - void setIsProjectionOnly() {} - void setLastTupleId(uint64_t id) {} - void setOutputType(BPSOutputType outputType) {} - void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2); - void setStepCount() {} - void setSwallowRows(const bool swallowRows) {} - void setBppStep() {} - void dec(DistributedEngineComm* dec) {} - const OIDVector& getProjectOids() const - { - return fOIDVector; - } - uint64_t blksSkipped() const - { - return 0; - } - bool wasStepRun() const - { - return fRunExecuted; - } - BPSOutputType getOutputType() const - { - return ROW_GROUP; - } - uint64_t getRows() const - { - return fRowsReturned; - } - const std::string& schemaName() const - { - return fSchema; - } - const std::string& tableName() const - { - return fTable; - } - const std::string& tableAlias() const - { - return fAlias; - } - void setJobInfo(const JobInfo* jobInfo) {} - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - // from DECEventListener - void newPMOnline(uint32_t) {} + CrossEngineStep* fStep; + }; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + uint64_t fRunner; // thread pool handle + OIDVector fOIDVector; + bool fEndOfResult; + bool fRunExecuted; - void addFcnJoinExp(const std::vector&); - void addFcnExpGroup1(const boost::shared_ptr&); - void setFE1Input(const rowgroup::RowGroup&); - void setFcnExpGroup3(const std::vector&); - void setFE23Output(const rowgroup::RowGroup&); + // MySQL server info + std::string fHost; + std::string fUser; + std::string fPasswd; + std::string fSchema; + std::string fTable; + std::string fAlias; + unsigned int fPort; - void addFilter(JobStep* jobStep); - void addProject(JobStep* jobStep); + // returned columns and primitive filters + std::string fWhereClause; + std::string fSelectClause; + // Function & Expression columns + std::vector > fFeFilters; + std::vector > fFeSelects; + std::vector > fFeFcnJoin; + std::map fColumnMap; // projected key position (k->p) + uint64_t fColumnCount; + boost::scoped_array fFe1Column; + boost::shared_array fFeMapping1; + boost::shared_array fFeMapping3; + rowgroup::RowGroup fRowGroupFe1; + rowgroup::RowGroup fRowGroupFe3; -protected: - virtual void execute(); - virtual void getMysqldInfo(const JobInfo&); - virtual void makeMappings(); - virtual void addFilterStr(const std::vector&, const std::string&); - virtual std::string makeQuery(); - virtual void setField(int, const char*, unsigned long, MYSQL_FIELD*, rowgroup::Row&); - inline void addRow(rowgroup::RGData&); - //inline void addRow(boost::shared_array&); - template - T convertValueNum( - const char*, const execplan::CalpontSystemCatalog::ColType&); - virtual void formatMiniStats(); - virtual void printCalTrace(); - - uint64_t fRowsRetrieved; - uint64_t fRowsReturned; - uint64_t fRowsPerGroup; - - // output rowgroup and row - rowgroup::RowGroup fRowGroupOut; - rowgroup::RowGroup fRowGroupDelivered; - rowgroup::RowGroup fRowGroupAdded; - rowgroup::Row fRowDelivered; - - // for datalist - RowGroupDL* fOutputDL; - uint64_t fOutputIterator; - - class Runner - { - public: - Runner(CrossEngineStep* step) : fStep(step) { } - void operator()() - { - utils::setThreadName("CESRunner"); - fStep->execute(); - } - - CrossEngineStep* fStep; - }; - - uint64_t fRunner; // thread pool handle - OIDVector fOIDVector; - bool fEndOfResult; - bool fRunExecuted; - - // MySQL server info - std::string fHost; - std::string fUser; - std::string fPasswd; - std::string fSchema; - std::string fTable; - std::string fAlias; - unsigned int fPort; - - // returned columns and primitive filters - std::string fWhereClause; - std::string fSelectClause; - - // Function & Expression columns - std::vector > fFeFilters; - std::vector > fFeSelects; - std::vector > fFeFcnJoin; - std::map fColumnMap; // projected key position (k->p) - uint64_t fColumnCount; - boost::scoped_array fFe1Column; - boost::shared_array fFeMapping1; - boost::shared_array fFeMapping3; - rowgroup::RowGroup fRowGroupFe1; - rowgroup::RowGroup fRowGroupFe3; - - funcexp::FuncExp* fFeInstance; - utils::LibMySQL* mysql; + funcexp::FuncExp* fFeInstance; + utils::LibMySQL* mysql; }; - -} // namespace - - +} // namespace joblist #endif // JOBLIST_CROSSENGINESTEP_H // vim:ts=4 sw=4: - - diff --git a/dbcon/joblist/datalist.h b/dbcon/joblist/datalist.h index 9eb6777cc..805c3c2f3 100644 --- a/dbcon/joblist/datalist.h +++ b/dbcon/joblist/datalist.h @@ -33,12 +33,11 @@ namespace joblist { - /* - Need to implement the following members in element_t: - copy constructor - = operator - < operator if using BandedDL + Need to implement the following members in element_t: + copy constructor + = operator + < operator if using BandedDL */ /** @brief class DataList @@ -85,140 +84,139 @@ namespace joblist * level. To use this class, be sure you have the \< operator defined for * whatever element type you're storing. */ -template +template class DataList { -public: - typedef element_t value_type; + public: + typedef element_t value_type; - DataList(); - DataList(const DataList& dl); - virtual ~DataList(); + DataList(); + DataList(const DataList& dl); + virtual ~DataList(); - DataList& operator=(const DataList& dl); + DataList& operator=(const DataList& dl); - virtual void insert(const element_t& e) = 0; - virtual void insert(const std::vector& v) = 0; - virtual uint64_t getIterator() = 0; - virtual bool next(uint64_t it, element_t* e) = 0; - virtual void endOfInput(); - virtual void setMultipleProducers(bool b) = 0; - virtual uint64_t totalSize() - { - return 0; - } - virtual bool totalDiskIoTime(uint64_t& w, uint64_t& r) - { - return false; - } + virtual void insert(const element_t& e) = 0; + virtual void insert(const std::vector& v) = 0; + virtual uint64_t getIterator() = 0; + virtual bool next(uint64_t it, element_t* e) = 0; + virtual void endOfInput(); + virtual void setMultipleProducers(bool b) = 0; + virtual uint64_t totalSize() + { + return 0; + } + virtual bool totalDiskIoTime(uint64_t& w, uint64_t& r) + { + return false; + } - virtual void OID(execplan::CalpontSystemCatalog::OID OID) - { - fOID = OID; - } - virtual execplan::CalpontSystemCatalog::OID OID() const - { - return fOID; - } + virtual void OID(execplan::CalpontSystemCatalog::OID OID) + { + fOID = OID; + } + virtual execplan::CalpontSystemCatalog::OID OID() const + { + return fOID; + } - //...Following methods indicate whether this datalist employs temp disk; - //...and if so, the num bytes for element_t.first and element_t.second. - //...Currently support sizes: (8,8), (8,4), (4,8), and (4,4). - virtual bool useDisk() const - { - return false; - } - virtual void setDiskElemSize(uint32_t size1st, uint32_t size2nd); - uint32_t getDiskElemSize1st() const - { - return fElemDiskFirstSize; - } - uint32_t getDiskElemSize2nd() const - { - return fElemDiskSecondSize; - } + //...Following methods indicate whether this datalist employs temp disk; + //...and if so, the num bytes for element_t.first and element_t.second. + //...Currently support sizes: (8,8), (8,4), (4,8), and (4,4). + virtual bool useDisk() const + { + return false; + } + virtual void setDiskElemSize(uint32_t size1st, uint32_t size2nd); + uint32_t getDiskElemSize1st() const + { + return fElemDiskFirstSize; + } + uint32_t getDiskElemSize2nd() const + { + return fElemDiskSecondSize; + } -protected: - void lock(); - void unlock(); + protected: + void lock(); + void unlock(); - boost::mutex& getMutex() - { - return mutex; // why in the world is this necessary in FIFO? - } + boost::mutex& getMutex() + { + return mutex; // why in the world is this necessary in FIFO? + } - boost::mutex mutex; - bool noMoreInput; - uint64_t consumersFinished; - uint32_t fElemDiskFirstSize; //byte size of element.first saved to disk - uint32_t fElemDiskSecondSize;//byte size of element.second saved to disk + boost::mutex mutex; + bool noMoreInput; + uint64_t consumersFinished; + uint32_t fElemDiskFirstSize; // byte size of element.first saved to disk + uint32_t fElemDiskSecondSize; // byte size of element.second saved to disk -private: - execplan::CalpontSystemCatalog::OID fOID; + private: + execplan::CalpontSystemCatalog::OID fOID; }; -template -DataList::DataList() : - noMoreInput(false), consumersFinished(0), - fElemDiskFirstSize(sizeof(uint64_t)), fElemDiskSecondSize(sizeof(uint64_t)), - fOID(0) -{ - //pthread_mutex_init(&mutex, NULL); -}; +template +DataList::DataList() + : noMoreInput(false) + , consumersFinished(0) + , fElemDiskFirstSize(sizeof(uint64_t)) + , fElemDiskSecondSize(sizeof(uint64_t)) + , fOID(0){ + // pthread_mutex_init(&mutex, NULL); + }; -template +template DataList::DataList(const DataList& dl) { - noMoreInput = dl.noMoreInput; - //pthread_mutex_init(&mutex, NULL); - fOID = dl.fOID; - consumersFinished = dl.consumersFinished; - fElemDiskFirstSize = dl.fElemDiskFirstSize; - fElemDiskSecondSize = dl.fElemDiskSecondSize; + noMoreInput = dl.noMoreInput; + // pthread_mutex_init(&mutex, NULL); + fOID = dl.fOID; + consumersFinished = dl.consumersFinished; + fElemDiskFirstSize = dl.fElemDiskFirstSize; + fElemDiskSecondSize = dl.fElemDiskSecondSize; }; -template -DataList::~DataList() +template +DataList::~DataList(){ + // pthread_mutex_destroy(&mutex); +}; + +template +DataList& DataList::operator=(const DataList& dl) { - //pthread_mutex_destroy(&mutex); + noMoreInput = dl.noMoreInput; + fOID = dl.fOID; + consumersFinished = dl.consumersFinished; + fElemDiskFirstSize = dl.fElemDiskFirstSize; + fElemDiskSecondSize = dl.fElemDiskSecondSize; }; -template -DataList& DataList::operator= -(const DataList& dl) -{ - noMoreInput = dl.noMoreInput; - fOID = dl.fOID; - consumersFinished = dl.consumersFinished; - fElemDiskFirstSize = dl.fElemDiskFirstSize; - fElemDiskSecondSize = dl.fElemDiskSecondSize; -}; - -template +template void DataList::endOfInput() { - noMoreInput = true; + noMoreInput = true; }; -template +template void DataList::lock() { - mutex.lock(); //pthread_mutex_lock(&mutex); + mutex.lock(); // pthread_mutex_lock(&mutex); }; -template +template void DataList::unlock() { - mutex.unlock(); //pthread_mutex_unlock(&mutex); + mutex.unlock(); // pthread_mutex_unlock(&mutex); }; -template +template void DataList::setDiskElemSize(uint32_t size1st, uint32_t size2nd) { - fElemDiskFirstSize = size1st; - fElemDiskSecondSize = size2nd; + fElemDiskFirstSize = size1st; + fElemDiskSecondSize = size2nd; } -} // namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/datalistimpl.h b/dbcon/joblist/datalistimpl.h index 4cb8c984d..971e518b1 100644 --- a/dbcon/joblist/datalistimpl.h +++ b/dbcon/joblist/datalistimpl.h @@ -38,238 +38,235 @@ namespace joblist { - /** @brief class DataListImpl * */ -template +template class DataListImpl : public DataList { + public: + DataListImpl(uint32_t numConsumers); + DataListImpl(const DataListImpl& dl); + virtual ~DataListImpl(); -public: - DataListImpl(uint32_t numConsumers); - DataListImpl(const DataListImpl& dl); - virtual ~DataListImpl(); + DataListImpl& operator=(const DataListImpl& dl); - DataListImpl& operator=(const DataListImpl& dl); + // derived classes need to lock around these fcns + virtual void insert(const element_t& e); + virtual void insert(const std::vector& v); + virtual uint64_t getIterator(); + virtual bool next(uint64_t it, element_t* e); - // derived classes need to lock around these fcns - virtual void insert(const element_t& e); - virtual void insert(const std::vector& v); - virtual uint64_t getIterator(); - virtual bool next(uint64_t it, element_t* e); + virtual void setNumConsumers(uint32_t); + virtual uint32_t getNumConsumers() const; + void resetNumConsumers(uint32_t numConsumers); - virtual void setNumConsumers(uint32_t); - virtual uint32_t getNumConsumers() const; - void resetNumConsumers(uint32_t numConsumers); + protected: + bool endOfData(uint64_t id) const; + void eraseUpTo(uint64_t id); + void reset(); + virtual void shrink(); + typename container_t::iterator iInsert(const element_t& e); -protected: - bool endOfData(uint64_t id) const; - void eraseUpTo(uint64_t id); - void reset(); - virtual void shrink(); - typename container_t::iterator iInsert(const element_t& e); + container_t* c; + typename container_t::iterator* cIterators; + uint64_t numConsumers; + uint64_t itIndex; - container_t* c; - typename container_t::iterator* cIterators; - uint64_t numConsumers; - uint64_t itIndex; - -private: - explicit DataListImpl(); + private: + explicit DataListImpl(); }; -template -DataListImpl::DataListImpl() : DataList(), c(0), cIterators(0), numConsumers(0), - itIndex(0) -{ } +template +DataListImpl::DataListImpl() + : DataList(), c(0), cIterators(0), numConsumers(0), itIndex(0) +{ +} -template +template void DataListImpl::setNumConsumers(uint32_t nc) { - resetNumConsumers(nc); + resetNumConsumers(nc); } -template +template uint32_t DataListImpl::getNumConsumers() const { - return numConsumers; + return numConsumers; } -template +template void DataListImpl::resetNumConsumers(uint32_t nc) { - if (itIndex != 0) - throw std::logic_error("DataListImpl::resetNumConsumers(): attempt to change numConsumers " - "after iterators have been issued"); + if (itIndex != 0) + throw std::logic_error( + "DataListImpl::resetNumConsumers(): attempt to change numConsumers " + "after iterators have been issued"); - uint32_t i; + uint32_t i; - numConsumers = nc; - delete [] cIterators; - cIterators = new typename container_t::iterator[numConsumers]; + numConsumers = nc; + delete[] cIterators; + cIterators = new typename container_t::iterator[numConsumers]; - for (i = 0; i < numConsumers; i++) - cIterators[i] = c->end(); + for (i = 0; i < numConsumers; i++) + cIterators[i] = c->end(); } -template +template DataListImpl::DataListImpl(uint32_t nc) : DataList() { - uint32_t i; + uint32_t i; - numConsumers = nc; - itIndex = 0; - c = new container_t(); - cIterators = new typename container_t::iterator[numConsumers]; + numConsumers = nc; + itIndex = 0; + c = new container_t(); + cIterators = new typename container_t::iterator[numConsumers]; - for (i = 0; i < numConsumers; i++) - cIterators[i] = c->end(); + for (i = 0; i < numConsumers; i++) + cIterators[i] = c->end(); } -template -DataListImpl::DataListImpl -(const DataListImpl& dl) - : DataList(dl) +template +DataListImpl::DataListImpl(const DataListImpl& dl) + : DataList(dl) { - int i; + int i; - c = dl.c; - numConsumers = dl.numConsumers; - itIndex = dl.itIndex; + c = dl.c; + numConsumers = dl.numConsumers; + itIndex = dl.itIndex; - //delete [] cIterators; - cIterators = new typename container_t::iterator[numConsumers]; + // delete [] cIterators; + cIterators = new typename container_t::iterator[numConsumers]; - for (i = 0; i < numConsumers; i++) - cIterators[i] = dl.cIterators[i]; + for (i = 0; i < numConsumers; i++) + cIterators[i] = dl.cIterators[i]; }; -template +template DataListImpl::~DataListImpl() { - delete c; - delete [] cIterators; + delete c; + delete[] cIterators; }; // lock at a higher level -template -DataListImpl& DataListImpl::operator= -(const DataListImpl& dl) +template +DataListImpl& DataListImpl::operator=( + const DataListImpl& dl) { - uint64_t i; + uint64_t i; - static_cast >(*this) = - static_cast >(dl); - delete c; - c = dl.c; - numConsumers = dl.numConsumers; - itIndex = dl.itIndex; + static_cast >(*this) = static_cast >(dl); + delete c; + c = dl.c; + numConsumers = dl.numConsumers; + itIndex = dl.itIndex; - delete [] cIterators; - cIterators = new typename container_t::iterator[numConsumers]; + delete[] cIterators; + cIterators = new typename container_t::iterator[numConsumers]; - for (i = 0; i < numConsumers; i++) - cIterators[i] = dl.cIterators[i]; + for (i = 0; i < numConsumers; i++) + cIterators[i] = dl.cIterators[i]; - return *this; + return *this; }; -template +template uint64_t DataListImpl::getIterator() { - if (itIndex >= numConsumers) - { - std::ostringstream oss; - oss << "DataListImpl::getIterator(): caller attempted to grab too many iterators: " << - "have " << numConsumers << " asked for " << (itIndex + 1); - throw std::logic_error(oss.str().c_str()); - } + if (itIndex >= numConsumers) + { + std::ostringstream oss; + oss << "DataListImpl::getIterator(): caller attempted to grab too many iterators: " + << "have " << numConsumers << " asked for " << (itIndex + 1); + throw std::logic_error(oss.str().c_str()); + } - cIterators[itIndex] = c->begin(); - return itIndex++; + cIterators[itIndex] = c->begin(); + return itIndex++; } -template +template inline void DataListImpl::insert(const element_t& e) { - c->insert(c->end(), e); + c->insert(c->end(), e); } -template +template inline void DataListImpl::insert(const std::vector& v) { - if (typeid(container_t) == typeid(std::vector)) - { - std::vector* vc = (std::vector*) c; - vc->insert(vc->end(), v.begin(), v.end()); - } - else - throw std::logic_error("insert(vector) isn't supported for non-vector-based DLs yet"); + if (typeid(container_t) == typeid(std::vector)) + { + std::vector* vc = (std::vector*)c; + vc->insert(vc->end(), v.begin(), v.end()); + } + else + throw std::logic_error("insert(vector) isn't supported for non-vector-based DLs yet"); } -template +template inline typename container_t::iterator DataListImpl::iInsert(const element_t& e) { - return c->insert(c->end(), e); + return c->insert(c->end(), e); } -template +template inline bool DataListImpl::next(uint64_t id, element_t* e) { + // idbassert(id < numConsumers); + if (c == 0 || cIterators[id] == c->end()) + return false; -// idbassert(id < numConsumers); - if (c == 0 || cIterators[id] == c->end()) - return false; - - *e = *(cIterators[id]); - cIterators[id]++; - return true; + *e = *(cIterators[id]); + cIterators[id]++; + return true; } -template +template bool DataListImpl::endOfData(uint64_t id) const { - idbassert(id < numConsumers); - return (cIterators[id] == c->end()); + idbassert(id < numConsumers); + return (cIterators[id] == c->end()); } -template +template void DataListImpl::eraseUpTo(uint64_t id) { - idbassert(id < numConsumers); + idbassert(id < numConsumers); #ifdef DEBUG - uint64_t i; - typename container_t::iterator it; + uint64_t i; + typename container_t::iterator it; - for (it = c->begin(), i = 0; it != cIterators[id]; it++, i++) ; + for (it = c->begin(), i = 0; it != cIterators[id]; it++, i++) + ; - std::cout << "DLI: erasing " << i << " elements" << std::endl; + std::cout << "DLI: erasing " << i << " elements" << std::endl; #endif - c->erase(c->begin(), cIterators[id]); + c->erase(c->begin(), cIterators[id]); }; -template +template void DataListImpl::reset() { - uint64_t i; + uint64_t i; - delete c; - c = new container_t(); + delete c; + c = new container_t(); - for (i = 0; i < numConsumers; i++) - cIterators[i] = c->begin(); + for (i = 0; i < numConsumers; i++) + cIterators[i] = c->begin(); } -template +template void DataListImpl::shrink() { - - delete c; - c = 0; + delete c; + c = 0; } -} //namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/dictstep-jl.cpp b/dbcon/joblist/dictstep-jl.cpp index 352fee68b..2c3c51737 100644 --- a/dbcon/joblist/dictstep-jl.cpp +++ b/dbcon/joblist/dictstep-jl.cpp @@ -35,31 +35,30 @@ using namespace messageqcpp; namespace joblist { - DictStepJL::DictStepJL() { } DictStepJL::DictStepJL(const pDictionaryStep& dict) { - BOP = dict.fBOP; - OID = dict.oid(); - colName = dict.name(); - compressionType = dict.colType().compressionType; + BOP = dict.fBOP; + OID = dict.oid(); + colName = dict.name(); + compressionType = dict.colType().compressionType; - hasEqFilter = dict.hasEqualityFilter; + hasEqFilter = dict.hasEqualityFilter; - if (hasEqFilter) - { - //cout << "saw eqfilter\n"; - eqOp = dict.tmpCOP; - eqFilter = dict.eqFilter; - } - else - filterString = dict.fFilterString; + if (hasEqFilter) + { + // cout << "saw eqfilter\n"; + eqOp = dict.tmpCOP; + eqFilter = dict.eqFilter; + } + else + filterString = dict.fFilterString; - filterCount = dict.fFilterCount; - charsetNumber = dict.fColType.charsetNumber; + filterCount = dict.fFilterCount; + charsetNumber = dict.fColType.charsetNumber; } DictStepJL::~DictStepJL() @@ -68,56 +67,57 @@ DictStepJL::~DictStepJL() void DictStepJL::setLBID(uint64_t token, uint32_t dbroot) { -// lbid = token >> 10; // the lbid is calculated on the PM + // lbid = token >> 10; // the lbid is calculated on the PM } void DictStepJL::createCommand(ByteStream& bs) const { - bs << (uint8_t) DICT_STEP; - bs << BOP; - bs << (uint8_t)compressionType; - bs << charsetNumber; - bs << filterCount; - bs << (uint8_t) hasEqFilter; + bs << (uint8_t)DICT_STEP; + bs << BOP; + bs << (uint8_t)compressionType; + bs << charsetNumber; + bs << filterCount; + bs << (uint8_t)hasEqFilter; - if (hasEqFilter) - { - idbassert(filterCount == eqFilter.size()); - bs << eqOp; + if (hasEqFilter) + { + idbassert(filterCount == eqFilter.size()); + bs << eqOp; - for (uint32_t i = 0; i < filterCount; i++) - bs << eqFilter[i]; - } - else - bs << filterString; - CommandJL::createCommand(bs); + for (uint32_t i = 0; i < filterCount; i++) + bs << eqFilter[i]; + } + else + bs << filterString; + CommandJL::createCommand(bs); } void DictStepJL::runCommand(ByteStream& bs) const -{ } +{ +} uint8_t DictStepJL::getTableColumnType() { - throw logic_error("Don't call DictStepJL::getTableColumn(); it's not a projection step"); + throw logic_error("Don't call DictStepJL::getTableColumn(); it's not a projection step"); } string DictStepJL::toString() { - ostringstream os; + ostringstream os; - os << "DictStepJL: " << filterCount << " filters, BOP=" << (int) BOP - << ", oid=" << OID << " name=" << colName << endl; - return os.str(); + os << "DictStepJL: " << filterCount << " filters, BOP=" << (int)BOP << ", oid=" << OID + << " name=" << colName << endl; + return os.str(); } uint16_t DictStepJL::getWidth() { - return colWidth; + return colWidth; } void DictStepJL::setWidth(uint16_t w) { - colWidth = w; + colWidth = w; } -}; // namespace +}; // namespace joblist diff --git a/dbcon/joblist/dictstep-jl.h b/dbcon/joblist/dictstep-jl.h index a9782acf4..bdfa1fe80 100644 --- a/dbcon/joblist/dictstep-jl.h +++ b/dbcon/joblist/dictstep-jl.h @@ -37,48 +37,48 @@ namespace joblist { - class DictStepJL : public CommandJL { -public: - DictStepJL(); - DictStepJL(const pDictionaryStep&); - virtual ~DictStepJL(); + public: + DictStepJL(); + DictStepJL(const pDictionaryStep&); + virtual ~DictStepJL(); - void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For ColumnCommandJL it's a RID, for a DictStep it's a token. - uint8_t getTableColumnType(); - std::string toString(); + void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For + // ColumnCommandJL it's a RID, for a DictStep it's a token. + uint8_t getTableColumnType(); + std::string toString(); - /* XXXPAT: The width is only valid for projection steps and the output - type is TUPLE at the moment. */ - void setWidth(uint16_t); - uint16_t getWidth(); + /* XXXPAT: The width is only valid for projection steps and the output + type is TUPLE at the moment. */ + void setWidth(uint16_t); + uint16_t getWidth(); - CommandType getCommandType() - { - return DICT_STEP; - } + CommandType getCommandType() + { + return DICT_STEP; + } - void createCommand(messageqcpp::ByteStream&) const; - void runCommand(messageqcpp::ByteStream&) const; + void createCommand(messageqcpp::ByteStream&) const; + void runCommand(messageqcpp::ByteStream&) const; -private: - DictStepJL(const DictStepJL&); + private: + DictStepJL(const DictStepJL&); -// uint64_t lbid; -// uint32_t fbo; - uint32_t traceFlags; // probably move this to Command - uint8_t BOP; - uint16_t colWidth; - int compressionType; - messageqcpp::ByteStream filterString; - uint32_t filterCount; - std::vector eqFilter; - bool hasEqFilter; - uint8_t eqOp; // COMPARE_EQ or COMPARE_NE - uint32_t charsetNumber; + // uint64_t lbid; + // uint32_t fbo; + uint32_t traceFlags; // probably move this to Command + uint8_t BOP; + uint16_t colWidth; + int compressionType; + messageqcpp::ByteStream filterString; + uint32_t filterCount; + std::vector eqFilter; + bool hasEqFilter; + uint8_t eqOp; // COMPARE_EQ or COMPARE_NE + uint32_t charsetNumber; }; -}; // namespace +}; // namespace joblist #endif diff --git a/dbcon/joblist/diskjoinstep.cpp b/dbcon/joblist/diskjoinstep.cpp index ee184a061..87be21660 100644 --- a/dbcon/joblist/diskjoinstep.cpp +++ b/dbcon/joblist/diskjoinstep.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "diskjoinstep.h" #include "exceptclasses.h" #include "resourcemanager.h" @@ -27,627 +26,625 @@ using namespace logging; namespace joblist { - -DiskJoinStep::DiskJoinStep() { } - -DiskJoinStep::DiskJoinStep(TupleHashJoinStep* t, int djsIndex, int joinIndex, bool lastOne) : JobStep(*t), thjs(t), - mainThread(0), joinerIndex(joinIndex), closedOutput(false) +DiskJoinStep::DiskJoinStep() { - /* - grab all relevant vars from THJS - make largeRG and outputRG - make the RG mappings - init a JoinPartition - load the existing RGData into JoinPartition - */ +} - largeRG = thjs->largeRG + thjs->outputRG; +DiskJoinStep::DiskJoinStep(TupleHashJoinStep* t, int djsIndex, int joinIndex, bool lastOne) + : JobStep(*t), thjs(t), mainThread(0), joinerIndex(joinIndex), closedOutput(false) +{ + /* + grab all relevant vars from THJS + make largeRG and outputRG + make the RG mappings + init a JoinPartition + load the existing RGData into JoinPartition + */ - if (lastOne) - outputRG = thjs->outputRG; - else - outputRG = largeRG; + largeRG = thjs->largeRG + thjs->outputRG; - smallRG = thjs->smallRGs[joinerIndex]; - largeKeyCols = thjs->largeSideKeys[joinerIndex]; - smallKeyCols = thjs->smallSideKeys[joinerIndex]; + if (lastOne) + outputRG = thjs->outputRG; + else + outputRG = largeRG; - /* Should not be necessary if we can use THJS's logic to do the join */ - fe = thjs->getJoinFilter(joinerIndex); + smallRG = thjs->smallRGs[joinerIndex]; + largeKeyCols = thjs->largeSideKeys[joinerIndex]; + smallKeyCols = thjs->smallSideKeys[joinerIndex]; - if (fe) - { - joinFERG = thjs->joinFilterRG; - SjoinFEMapping = makeMapping(smallRG, joinFERG); - LjoinFEMapping = makeMapping(largeRG, joinFERG); - } + /* Should not be necessary if we can use THJS's logic to do the join */ + fe = thjs->getJoinFilter(joinerIndex); - joiner = thjs->djsJoiners[djsIndex]; - joinType = joiner->getJoinType(); - typeless = joiner->isTypelessJoin(); - joiner->clearData(); - joiner->setInUM(); + if (fe) + { + joinFERG = thjs->joinFilterRG; + SjoinFEMapping = makeMapping(smallRG, joinFERG); + LjoinFEMapping = makeMapping(largeRG, joinFERG); + } - LOMapping = makeMapping(largeRG, outputRG); - SOMapping = makeMapping(smallRG, outputRG); + joiner = thjs->djsJoiners[djsIndex]; + joinType = joiner->getJoinType(); + typeless = joiner->isTypelessJoin(); + joiner->clearData(); + joiner->setInUM(); - /* Link up */ - largeDL = thjs->fifos[djsIndex]; - outputDL = thjs->fifos[djsIndex + 1]; - smallDL = thjs->smallDLs[joinerIndex]; - largeIt = largeDL->getIterator(); + LOMapping = makeMapping(largeRG, outputRG); + SOMapping = makeMapping(smallRG, outputRG); - smallUsage = thjs->djsSmallUsage; - smallLimit = thjs->djsSmallLimit; - largeLimit = thjs->djsLargeLimit; - partitionSize = thjs->djsPartitionSize; + /* Link up */ + largeDL = thjs->fifos[djsIndex]; + outputDL = thjs->fifos[djsIndex + 1]; + smallDL = thjs->smallDLs[joinerIndex]; + largeIt = largeDL->getIterator(); - if (smallLimit == 0) - smallLimit = numeric_limits::max(); + smallUsage = thjs->djsSmallUsage; + smallLimit = thjs->djsSmallLimit; + largeLimit = thjs->djsLargeLimit; + partitionSize = thjs->djsPartitionSize; - if (largeLimit == 0) - largeLimit = numeric_limits::max(); + if (smallLimit == 0) + smallLimit = numeric_limits::max(); - uint64_t totalUMMemory = thjs->resourceManager->getConfiguredUMMemLimit(); - jp.reset(new JoinPartition(largeRG, smallRG, smallKeyCols, largeKeyCols, typeless, - (joinType & ANTI) && (joinType & MATCHNULLS), (bool) fe, totalUMMemory, partitionSize)); + if (largeLimit == 0) + largeLimit = numeric_limits::max(); - if (cancelled()) - { - // drain inputs, close output - smallReader(); // only small input is supplying input at this point - // largeReader(); - outputDL->endOfInput(); - closedOutput = true; - } + uint64_t totalUMMemory = thjs->resourceManager->getConfiguredUMMemLimit(); + jp.reset(new JoinPartition(largeRG, smallRG, smallKeyCols, largeKeyCols, typeless, + (joinType & ANTI) && (joinType & MATCHNULLS), (bool)fe, totalUMMemory, + partitionSize)); - largeIterationCount = 0; - lastLargeIteration = false; - fMiniInfo.clear(); - fExtendedInfo.clear(); + if (cancelled()) + { + // drain inputs, close output + smallReader(); // only small input is supplying input at this point + // largeReader(); + outputDL->endOfInput(); + closedOutput = true; + } + + largeIterationCount = 0; + lastLargeIteration = false; + fMiniInfo.clear(); + fExtendedInfo.clear(); } DiskJoinStep::~DiskJoinStep() { - abort(); + abort(); - if (mainThread) - { - jobstepThreadPool.join(mainThread); - mainThread = 0; - } + if (mainThread) + { + jobstepThreadPool.join(mainThread); + mainThread = 0; + } - if (jp) - atomicops::atomicSub(smallUsage.get(), jp->getSmallSideDiskUsage()); + if (jp) + atomicops::atomicSub(smallUsage.get(), jp->getSmallSideDiskUsage()); } void DiskJoinStep::loadExistingData(vector& data) { - int64_t memUsage; - uint32_t i; + int64_t memUsage; + uint32_t i; - for (i = 0; i < data.size() && !cancelled(); i++) - { - memUsage = jp->insertSmallSideRGData(data[i]); - atomicops::atomicAdd(smallUsage.get(), memUsage); - } + for (i = 0; i < data.size() && !cancelled(); i++) + { + memUsage = jp->insertSmallSideRGData(data[i]); + atomicops::atomicAdd(smallUsage.get(), memUsage); + } } void DiskJoinStep::run() { - mainThread = jobstepThreadPool.invoke(Runner(this)); + mainThread = jobstepThreadPool.invoke(Runner(this)); } void DiskJoinStep::join() { - if (mainThread) - { - jobstepThreadPool.join(mainThread); - mainThread = 0; - } + if (mainThread) + { + jobstepThreadPool.join(mainThread); + mainThread = 0; + } - if (jp) - { - atomicops::atomicSub(smallUsage.get(), jp->getSmallSideDiskUsage()); - //int64_t memUsage; - //memUsage = atomicops::atomicSub(smallUsage.get(), jp->getSmallSideDiskUsage()); - //cout << "join(): small side usage was: " << jp->getSmallSideDiskUsage() << " final shared mem usage = " << memUsage << endl; - jp.reset(); - } + if (jp) + { + atomicops::atomicSub(smallUsage.get(), jp->getSmallSideDiskUsage()); + // int64_t memUsage; + // memUsage = atomicops::atomicSub(smallUsage.get(), jp->getSmallSideDiskUsage()); + // cout << "join(): small side usage was: " << jp->getSmallSideDiskUsage() << " final shared mem usage = " + // << memUsage << endl; + jp.reset(); + } } void DiskJoinStep::smallReader() { - RGData rgData; - bool more = true; - int64_t memUsage = 0, combinedMemUsage = 0; - int rowCount = 0; - RowGroup l_smallRG = smallRG; + RGData rgData; + bool more = true; + int64_t memUsage = 0, combinedMemUsage = 0; + int rowCount = 0; + RowGroup l_smallRG = smallRG; - try + try + { + while (more && !cancelled()) { - while (more && !cancelled()) + more = smallDL->next(0, &rgData); + + if (more) + { + l_smallRG.setData(&rgData); + rowCount += l_smallRG.getRowCount(); + memUsage = jp->insertSmallSideRGData(rgData); + combinedMemUsage = atomicops::atomicAdd(smallUsage.get(), memUsage); + + // cout << "memusage = " << memUsage << " total = " << combinedMemUsage << endl; + if (combinedMemUsage > smallLimit) { - more = smallDL->next(0, &rgData); - - if (more) - { - l_smallRG.setData(&rgData); - rowCount += l_smallRG.getRowCount(); - memUsage = jp->insertSmallSideRGData(rgData); - combinedMemUsage = atomicops::atomicAdd(smallUsage.get(), memUsage); - - //cout << "memusage = " << memUsage << " total = " << combinedMemUsage << endl; - if (combinedMemUsage > smallLimit) - { - errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_DBJ_DISK_USAGE_LIMIT)); - status(ERR_DBJ_DISK_USAGE_LIMIT); - cout << "DJS small reader: exceeded disk space limit" << endl; - abort(); - } - } + errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_DBJ_DISK_USAGE_LIMIT)); + status(ERR_DBJ_DISK_USAGE_LIMIT); + cout << "DJS small reader: exceeded disk space limit" << endl; + abort(); } - - //cout << "(" << joinerIndex << ") read the small side data, combined mem usage= " << combinedMemUsage << " rowcount = " << rowCount << endl; - if (!cancelled()) - { - memUsage = jp->doneInsertingSmallData(); - combinedMemUsage = atomicops::atomicAdd(smallUsage.get(), memUsage); - - //cout << "2memusage = " << memUsage << " total = " << combinedMemUsage << endl; - if (combinedMemUsage > smallLimit) - { - errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_DBJ_DISK_USAGE_LIMIT)); - status(ERR_DBJ_DISK_USAGE_LIMIT); - cout << "DJS small reader: exceeded disk space limit" << endl; - abort(); - } - } - } // try - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_ALWAYS_CRITICAL, - "DiskJoinStep::smallReader()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - abort(); + } } - while (more) - more = smallDL->next(0, &rgData); + // cout << "(" << joinerIndex << ") read the small side data, combined mem usage= " << combinedMemUsage << + // " rowcount = " << rowCount << endl; + if (!cancelled()) + { + memUsage = jp->doneInsertingSmallData(); + combinedMemUsage = atomicops::atomicAdd(smallUsage.get(), memUsage); + + // cout << "2memusage = " << memUsage << " total = " << combinedMemUsage << endl; + if (combinedMemUsage > smallLimit) + { + errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_DBJ_DISK_USAGE_LIMIT)); + status(ERR_DBJ_DISK_USAGE_LIMIT); + cout << "DJS small reader: exceeded disk space limit" << endl; + abort(); + } + } + } // try + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_ALWAYS_CRITICAL, + "DiskJoinStep::smallReader()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + abort(); + } + + while (more) + more = smallDL->next(0, &rgData); } void DiskJoinStep::largeReader() { - RGData rgData; - bool more = true; - int64_t largeSize = 0; - int rowCount = 0; - RowGroup l_largeRG = largeRG; + RGData rgData; + bool more = true; + int64_t largeSize = 0; + int rowCount = 0; + RowGroup l_largeRG = largeRG; - largeIterationCount++; - //cout << "iteration " << largeIterationCount << " largeLimit=" << largeLimit << endl; + largeIterationCount++; + // cout << "iteration " << largeIterationCount << " largeLimit=" << largeLimit << endl; - try + try + { + while (more && !cancelled() && largeSize < largeLimit) { - while (more && !cancelled() && largeSize < largeLimit) - { - more = largeDL->next(largeIt, &rgData); + more = largeDL->next(largeIt, &rgData); - if (more) - { - l_largeRG.setData(&rgData); - rowCount += l_largeRG.getRowCount(); - //cout << "large side raw data: " << largeRG.toString() << endl; + if (more) + { + l_largeRG.setData(&rgData); + rowCount += l_largeRG.getRowCount(); + // cout << "large side raw data: " << largeRG.toString() << endl; - largeSize += jp->insertLargeSideRGData(rgData); - } - } - - jp->doneInsertingLargeData(); - - //cout << "(" << joinerIndex << ") read the large side data rowcount = " << rowCount << endl; - if (!more) - lastLargeIteration = true; - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_ALWAYS_CRITICAL, - "DiskJoinStep::largeReader()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - abort(); + largeSize += jp->insertLargeSideRGData(rgData); + } } - if (cancelled()) - while (more) - more = largeDL->next(largeIt, &rgData); + jp->doneInsertingLargeData(); + + // cout << "(" << joinerIndex << ") read the large side data rowcount = " << rowCount << endl; + if (!more) + lastLargeIteration = true; + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_ALWAYS_CRITICAL, + "DiskJoinStep::largeReader()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + abort(); + } + + if (cancelled()) + while (more) + more = largeDL->next(largeIt, &rgData); } - void DiskJoinStep::loadFcn() { - boost::shared_ptr out; - bool ret; + boost::shared_ptr out; + bool ret; - try + try + { + do { - do - { - out.reset(new LoaderOutput()); - ret = jp->getNextPartition(&out->smallData, &out->partitionID, &out->jp); + out.reset(new LoaderOutput()); + ret = jp->getNextPartition(&out->smallData, &out->partitionID, &out->jp); - if (ret) - { - //cout << "loaded partition " << out->partitionID << " smallData = " << out->smallData.size() << endl; - loadFIFO->insert(out); - } - } - while (ret && !cancelled()); - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_ALWAYS_CRITICAL, - "DiskJoinStep::loadFcn()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - abort(); - } + if (ret) + { + // cout << "loaded partition " << out->partitionID << " smallData = " << out->smallData.size() << + // endl; + loadFIFO->insert(out); + } + } while (ret && !cancelled()); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_ALWAYS_CRITICAL, + "DiskJoinStep::loadFcn()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + abort(); + } - loadFIFO->endOfInput(); + loadFIFO->endOfInput(); } void DiskJoinStep::buildFcn() { - boost::shared_ptr in; - boost::shared_ptr out; - bool more = true; - int it = loadFIFO->getIterator(); - int i, j; - Row smallRow; - RowGroup l_smallRG = smallRG; + boost::shared_ptr in; + boost::shared_ptr out; + bool more = true; + int it = loadFIFO->getIterator(); + int i, j; + Row smallRow; + RowGroup l_smallRG = smallRG; - l_smallRG.initRow(&smallRow); + l_smallRG.initRow(&smallRow); - while (1) + while (1) + { + // cout << "getting a partition from the loader" << endl; + more = loadFIFO->next(it, &in); + + if (!more || cancelled()) + goto out; + + out.reset(new BuilderOutput()); + out->smallData = in->smallData; + out->partitionID = in->partitionID; + out->jp = in->jp; + out->tupleJoiner = joiner->copyForDiskJoin(); + + // cout << "building a tuplejoiner" << endl; + for (i = 0; i < (int)in->smallData.size(); i++) { - //cout << "getting a partition from the loader" << endl; - more = loadFIFO->next(it, &in); + l_smallRG.setData(&in->smallData[i]); + l_smallRG.getRow(0, &smallRow); - if (!more || cancelled()) - goto out; - - out.reset(new BuilderOutput()); - out->smallData = in->smallData; - out->partitionID = in->partitionID; - out->jp = in->jp; - out->tupleJoiner = joiner->copyForDiskJoin(); - - //cout << "building a tuplejoiner" << endl; - for (i = 0; i < (int) in->smallData.size(); i++) - { - l_smallRG.setData(&in->smallData[i]); - l_smallRG.getRow(0, &smallRow); - - for (j = 0; j < (int) l_smallRG.getRowCount(); j++, smallRow.nextRow()) - out->tupleJoiner->insert(smallRow, (largeIterationCount == 1)); - } - - out->tupleJoiner->doneInserting(); - buildFIFO->insert(out); + for (j = 0; j < (int)l_smallRG.getRowCount(); j++, smallRow.nextRow()) + out->tupleJoiner->insert(smallRow, (largeIterationCount == 1)); } + out->tupleJoiner->doneInserting(); + buildFIFO->insert(out); + } + out: - while (more) - more = loadFIFO->next(it, &in); + while (more) + more = loadFIFO->next(it, &in); - buildFIFO->endOfInput(); + buildFIFO->endOfInput(); } void DiskJoinStep::joinFcn() { + /* This function mostly serves as an adapter between the + input data and the joinOneRG() fcn in THJS. */ - /* This function mostly serves as an adapter between the - input data and the joinOneRG() fcn in THJS. */ + boost::shared_ptr in; + bool more = true; + int it = buildFIFO->getIterator(); + int i, j; + vector joinResults; + RowGroup l_largeRG = largeRG, l_smallRG = smallRG; + RowGroup l_outputRG = outputRG; + Row l_largeRow; + Row l_joinFERow, l_outputRow, baseRow; + vector > joinMatches; + boost::shared_array smallRowTemplates(new Row[1]); + vector > joiners; + boost::shared_array > colMappings, fergMappings; + boost::scoped_array > smallNullMem; + boost::scoped_array joinFEMem; + Row smallNullRow; - boost::shared_ptr in; - bool more = true; - int it = buildFIFO->getIterator(); - int i, j; - vector joinResults; - RowGroup l_largeRG = largeRG, l_smallRG = smallRG; - RowGroup l_outputRG = outputRG; - Row l_largeRow; - Row l_joinFERow, l_outputRow, baseRow; - vector > joinMatches; - boost::shared_array smallRowTemplates(new Row[1]); - vector > joiners; - boost::shared_array > colMappings, fergMappings; - boost::scoped_array > smallNullMem; - boost::scoped_array joinFEMem; - Row smallNullRow; + boost::scoped_array baseRowMem; - boost::scoped_array baseRowMem; + if (joiner->hasFEFilter()) + { + joinFERG.initRow(&l_joinFERow, true); + joinFEMem.reset(new uint8_t[l_joinFERow.getSize()]); + l_joinFERow.setData(joinFEMem.get()); + } - if (joiner->hasFEFilter()) + outputRG.initRow(&l_outputRow); + outputRG.initRow(&baseRow, true); + + largeRG.initRow(&l_largeRow); + + baseRowMem.reset(new uint8_t[baseRow.getSize()]); + baseRow.setData(baseRowMem.get()); + joinMatches.push_back(vector()); + smallRG.initRow(&smallRowTemplates[0]); + joiners.resize(1); + + colMappings.reset(new boost::shared_array[2]); + colMappings[0] = SOMapping; + colMappings[1] = LOMapping; + + if (fe) + { + fergMappings.reset(new boost::shared_array[2]); + fergMappings[0] = SjoinFEMapping; + fergMappings[1] = LjoinFEMapping; + } + + l_smallRG.initRow(&smallNullRow, true); + smallNullMem.reset(new boost::scoped_array[1]); + smallNullMem[0].reset(new uint8_t[smallNullRow.getSize()]); + smallNullRow.setData(smallNullMem[0].get()); + smallNullRow.initToNull(); + + try + { + while (1) { - joinFERG.initRow(&l_joinFERow, true); - joinFEMem.reset(new uint8_t[l_joinFERow.getSize()]); - l_joinFERow.setData(joinFEMem.get()); - } + more = buildFIFO->next(it, &in); - outputRG.initRow(&l_outputRow); - outputRG.initRow(&baseRow, true); + if (!more || cancelled()) + goto out; - largeRG.initRow(&l_largeRow); + joiners[0] = in->tupleJoiner; + boost::shared_ptr largeData; + largeData = in->jp->getNextLargeRGData(); - baseRowMem.reset(new uint8_t[baseRow.getSize()]); - baseRow.setData(baseRowMem.get()); - joinMatches.push_back(vector()); - smallRG.initRow(&smallRowTemplates[0]); - joiners.resize(1); + while (largeData) + { + l_largeRG.setData(largeData.get()); + thjs->joinOneRG(0, joinResults, l_largeRG, l_outputRG, l_largeRow, l_joinFERow, l_outputRow, baseRow, + joinMatches, smallRowTemplates, outputDL.get(), &joiners, &colMappings, &fergMappings, + &smallNullMem); - colMappings.reset(new boost::shared_array[2]); - colMappings[0] = SOMapping; - colMappings[1] = LOMapping; - - if (fe) - { - fergMappings.reset(new boost::shared_array[2]); - fergMappings[0] = SjoinFEMapping; - fergMappings[1] = LjoinFEMapping; - } - - l_smallRG.initRow(&smallNullRow, true); - smallNullMem.reset(new boost::scoped_array[1]); - smallNullMem[0].reset(new uint8_t[smallNullRow.getSize()]); - smallNullRow.setData(smallNullMem[0].get()); - smallNullRow.initToNull(); - - try - { - while (1) + for (j = 0; j < (int)joinResults.size(); j++) { - more = buildFIFO->next(it, &in); - - if (!more || cancelled()) - goto out; - - joiners[0] = in->tupleJoiner; - boost::shared_ptr largeData; - largeData = in->jp->getNextLargeRGData(); - - while (largeData) - { - l_largeRG.setData(largeData.get()); - thjs->joinOneRG(0, joinResults, l_largeRG, l_outputRG, l_largeRow, l_joinFERow, - l_outputRow, baseRow, joinMatches, smallRowTemplates, outputDL.get(), - &joiners, &colMappings, &fergMappings, &smallNullMem); - - for (j = 0; j < (int) joinResults.size(); j++) - { - //l_outputRG.setData(&joinResults[j]); - //cout << "got joined output " << l_outputRG.toString() << endl; - outputDL->insert(joinResults[j]); - } - thjs->returnMemory(); - joinResults.clear(); - largeData = in->jp->getNextLargeRGData(); - } - - if (joinType & SMALLOUTER) - { - if (!lastLargeIteration) - { - /* TODO: an optimization would be to detect whether any new rows were marked and if not - suppress the save operation */ - vector unmatched; - in->tupleJoiner->getUnmarkedRows(&unmatched); - //cout << "***** saving partition " << in->partitionID << " unmarked count=" << unmatched.size() << " total count=" - // << in->tupleJoiner->size() << " vector size=" << in->smallData.size() << endl; - in->jp->saveSmallSidePartition(in->smallData); - } - else - { - //cout << "finishing small-outer output" << endl; - vector unmatched; - RGData rgData(l_outputRG); - Row outputRow; - - l_outputRG.setData(&rgData); - l_outputRG.resetRowGroup(0); - l_outputRG.initRow(&outputRow); - l_outputRG.getRow(0, &outputRow); - - l_largeRG.initRow(&l_largeRow, true); - boost::scoped_array largeNullMem(new uint8_t[l_largeRow.getSize()]); - l_largeRow.setData(largeNullMem.get()); - l_largeRow.initToNull(); - - in->tupleJoiner->getUnmarkedRows(&unmatched); - - //cout << " small-outer count=" << unmatched.size() << endl; - for (i = 0; i < (int) unmatched.size(); i++) - { - smallRowTemplates[0].setData(unmatched[i]); - applyMapping(LOMapping, l_largeRow, &outputRow); - applyMapping(SOMapping, smallRowTemplates[0], &outputRow); - l_outputRG.incRowCount(); - - if (l_outputRG.getRowCount() == 8192) - { - outputDL->insert(rgData); - //cout << "inserting a full RG" << endl; - if (thjs) - { - if (!thjs->getMemory(l_outputRG.getMaxDataSize())) - { - // calculate guess of size required for error message - uint64_t memReqd = (unmatched.size() * outputRG.getDataSize(1)) / 1048576; - Message::Args args; - args.add(memReqd); - args.add(thjs->resourceManager->getConfiguredUMMemLimit() / 1048576); - std::cerr << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_RESULT_TOO_BIG, args) - << " @" << __FILE__ << ":" << __LINE__; - throw logging::IDBExcept(logging::ERR_JOIN_RESULT_TOO_BIG, args); - } - } - - rgData.reinit(l_outputRG); - l_outputRG.setData(&rgData); - l_outputRG.resetRowGroup(0); - l_outputRG.getRow(0, &outputRow); - } - else - outputRow.nextRow(); - } - - if (l_outputRG.getRowCount() > 0) - { - //cout << "inserting an rg with " << l_outputRG.getRowCount() << endl; - outputDL->insert(rgData); - } - if (thjs) - { - thjs->returnMemory(); - } - } - } + // l_outputRG.setData(&joinResults[j]); + // cout << "got joined output " << l_outputRG.toString() << endl; + outputDL->insert(joinResults[j]); } - } // the try stmt above; need to reformat. - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_ALWAYS_CRITICAL, - "DiskJoinStep::joinFcn()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - abort(); + thjs->returnMemory(); + joinResults.clear(); + largeData = in->jp->getNextLargeRGData(); + } + + if (joinType & SMALLOUTER) + { + if (!lastLargeIteration) + { + /* TODO: an optimization would be to detect whether any new rows were marked and if not + suppress the save operation */ + vector unmatched; + in->tupleJoiner->getUnmarkedRows(&unmatched); + // cout << "***** saving partition " << in->partitionID << " unmarked count=" << unmatched.size() << + // " total count=" + // << in->tupleJoiner->size() << " vector size=" << in->smallData.size() << endl; + in->jp->saveSmallSidePartition(in->smallData); + } + else + { + // cout << "finishing small-outer output" << endl; + vector unmatched; + RGData rgData(l_outputRG); + Row outputRow; + + l_outputRG.setData(&rgData); + l_outputRG.resetRowGroup(0); + l_outputRG.initRow(&outputRow); + l_outputRG.getRow(0, &outputRow); + + l_largeRG.initRow(&l_largeRow, true); + boost::scoped_array largeNullMem(new uint8_t[l_largeRow.getSize()]); + l_largeRow.setData(largeNullMem.get()); + l_largeRow.initToNull(); + + in->tupleJoiner->getUnmarkedRows(&unmatched); + + // cout << " small-outer count=" << unmatched.size() << endl; + for (i = 0; i < (int)unmatched.size(); i++) + { + smallRowTemplates[0].setData(unmatched[i]); + applyMapping(LOMapping, l_largeRow, &outputRow); + applyMapping(SOMapping, smallRowTemplates[0], &outputRow); + l_outputRG.incRowCount(); + + if (l_outputRG.getRowCount() == 8192) + { + outputDL->insert(rgData); + // cout << "inserting a full RG" << endl; + if (thjs) + { + if (!thjs->getMemory(l_outputRG.getMaxDataSize())) + { + // calculate guess of size required for error message + uint64_t memReqd = (unmatched.size() * outputRG.getDataSize(1)) / 1048576; + Message::Args args; + args.add(memReqd); + args.add(thjs->resourceManager->getConfiguredUMMemLimit() / 1048576); + std::cerr << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_RESULT_TOO_BIG, + args) + << " @" << __FILE__ << ":" << __LINE__; + throw logging::IDBExcept(logging::ERR_JOIN_RESULT_TOO_BIG, args); + } + } + + rgData.reinit(l_outputRG); + l_outputRG.setData(&rgData); + l_outputRG.resetRowGroup(0); + l_outputRG.getRow(0, &outputRow); + } + else + outputRow.nextRow(); + } + + if (l_outputRG.getRowCount() > 0) + { + // cout << "inserting an rg with " << l_outputRG.getRowCount() << endl; + outputDL->insert(rgData); + } + if (thjs) + { + thjs->returnMemory(); + } + } + } } + } // the try stmt above; need to reformat. + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_ALWAYS_CRITICAL, + "DiskJoinStep::joinFcn()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + abort(); + } out: - while (more) - more = buildFIFO->next(it, &in); + while (more) + more = buildFIFO->next(it, &in); - if (lastLargeIteration || cancelled()) - { - reportStats(); - outputDL->endOfInput(); - closedOutput = true; - } + if (lastLargeIteration || cancelled()) + { + reportStats(); + outputDL->endOfInput(); + closedOutput = true; + } } void DiskJoinStep::mainRunner() { - /* - Read from smallDL, insert into small side - Read from largeDL, insert into large side - Start the processing threads - */ + /* + Read from smallDL, insert into small side + Read from largeDL, insert into large side + Start the processing threads + */ + try + { + smallReader(); + + while (!lastLargeIteration && !cancelled()) + { + // cout << "large iteration " << largeIterationCount << endl; + jp->initForLargeSideFeed(); + largeReader(); + // cout << "done reading iteration " << largeIterationCount-1 << endl; + + jp->initForProcessing(); + + if (cancelled()) + break; + + loadFIFO.reset( + new FIFO >(1, 1)); // double buffering should be good enough + buildFIFO.reset(new FIFO >(1, 1)); + + std::vector thrds; + thrds.reserve(3); + thrds.push_back(jobstepThreadPool.invoke(Loader(this))); + thrds.push_back(jobstepThreadPool.invoke(Builder(this))); + thrds.push_back(jobstepThreadPool.invoke(Joiner(this))); + jobstepThreadPool.join(thrds); + } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_ALWAYS_CRITICAL, + "DiskJoinStep::mainRunner()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + abort(); + } + + // make sure all inputs were drained & output closed + if (cancelled()) + { try { - smallReader(); - - while (!lastLargeIteration && !cancelled()) - { - //cout << "large iteration " << largeIterationCount << endl; - jp->initForLargeSideFeed(); - largeReader(); - //cout << "done reading iteration " << largeIterationCount-1 << endl; - - jp->initForProcessing(); - - if (cancelled()) - break; - - loadFIFO.reset(new FIFO >(1, 1)); // double buffering should be good enough - buildFIFO.reset(new FIFO >(1, 1)); - - std::vector thrds; - thrds.reserve(3); - thrds.push_back(jobstepThreadPool.invoke(Loader(this))); - thrds.push_back(jobstepThreadPool.invoke(Builder(this))); - thrds.push_back(jobstepThreadPool.invoke(Joiner(this))); - jobstepThreadPool.join(thrds); - } + jp->initForLargeSideFeed(); } catch (...) { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_ALWAYS_CRITICAL, - "DiskJoinStep::mainRunner()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - abort(); - } + } // doesn't matter if this fails to open the large-file - // make sure all inputs were drained & output closed - if (cancelled()) + largeReader(); // large reader will only drain the fifo when cancelled() + + if (!closedOutput) { - try - { - jp->initForLargeSideFeed(); - } - catch (...) { } // doesn't matter if this fails to open the large-file - - largeReader(); // large reader will only drain the fifo when cancelled() - - if (!closedOutput) - { - outputDL->endOfInput(); - closedOutput = true; - } + outputDL->endOfInput(); + closedOutput = true; } + } } const string DiskJoinStep::toString() const { - return "DiskJoinStep\n"; + return "DiskJoinStep\n"; } void DiskJoinStep::reportStats() { - ostringstream os1, os2; + ostringstream os1, os2; - os1 << "DiskJoinStep: joined (large) " << alias() << " to (small) " << joiner->getTableName() << ". Processing stages: " << largeIterationCount << - ", disk usage small/large: " << jp->getMaxSmallSize() << "/" << jp->getMaxLargeSize() << - ", total bytes read/written: " << jp->getBytesRead() << "/" << jp->getBytesWritten() << endl; - fExtendedInfo = os1.str(); + os1 << "DiskJoinStep: joined (large) " << alias() << " to (small) " << joiner->getTableName() + << ". Processing stages: " << largeIterationCount + << ", disk usage small/large: " << jp->getMaxSmallSize() << "/" << jp->getMaxLargeSize() + << ", total bytes read/written: " << jp->getBytesRead() << "/" << jp->getBytesWritten() << endl; + fExtendedInfo = os1.str(); - /* TODO: Can this report anything more useful in miniInfo? */ - int64_t bytesToReport = jp->getBytesRead() + jp->getBytesWritten(); - char units; + /* TODO: Can this report anything more useful in miniInfo? */ + int64_t bytesToReport = jp->getBytesRead() + jp->getBytesWritten(); + char units; - if (bytesToReport > (1 << 30)) - { - bytesToReport >>= 30; - units = 'G'; - } - else if (bytesToReport > (1 << 20)) - { - bytesToReport >>= 20; - units = 'M'; - } - else if (bytesToReport > (1 << 10)) - { - bytesToReport >>= 10; - units = 'K'; - } - else units = ' '; + if (bytesToReport > (1 << 30)) + { + bytesToReport >>= 30; + units = 'G'; + } + else if (bytesToReport > (1 << 20)) + { + bytesToReport >>= 20; + units = 'M'; + } + else if (bytesToReport > (1 << 10)) + { + bytesToReport >>= 10; + units = 'K'; + } + else + units = ' '; - os2 << "DJS UM " << alias() << "-" << joiner->getTableName() << " - - " << bytesToReport << - units << " - - -------- -\n"; + os2 << "DJS UM " << alias() << "-" << joiner->getTableName() << " - - " << bytesToReport << units + << " - - -------- -\n"; - fMiniInfo = os2.str(); + fMiniInfo = os2.str(); - if (traceOn()) - logEnd(os1.str().c_str()); + if (traceOn()) + logEnd(os1.str().c_str()); } - -} +} // namespace joblist diff --git a/dbcon/joblist/diskjoinstep.h b/dbcon/joblist/diskjoinstep.h index fbdda890a..ef0a7c6da 100644 --- a/dbcon/joblist/diskjoinstep.h +++ b/dbcon/joblist/diskjoinstep.h @@ -27,129 +27,136 @@ namespace joblist { - class DiskJoinStep : public JobStep { -public: - DiskJoinStep(); - DiskJoinStep(TupleHashJoinStep*, int djsIndex, int joinerIndex, bool lastOne); - virtual ~DiskJoinStep(); + public: + DiskJoinStep(); + DiskJoinStep(TupleHashJoinStep*, int djsIndex, int joinerIndex, bool lastOne); + virtual ~DiskJoinStep(); - void run(); - void join(); - const std::string toString() const; + void run(); + void join(); + const std::string toString() const; - void loadExistingData(std::vector& data); - uint32_t getIterationCount() + void loadExistingData(std::vector& data); + uint32_t getIterationCount() + { + return largeIterationCount; + } + + protected: + private: + boost::shared_ptr jp; + rowgroup::RowGroup largeRG, smallRG, outputRG, joinFERG; + std::vector largeKeyCols, smallKeyCols; + boost::shared_ptr largeDL, outputDL; + RowGroupDL* smallDL; + + boost::shared_array LOMapping, SOMapping, SjoinFEMapping, LjoinFEMapping; + TupleHashJoinStep* thjs; + boost::shared_ptr fe; + bool typeless; + JoinType joinType; + boost::shared_ptr joiner; // the same instance THJS uses + + /* main thread, started by JobStep::run() */ + void mainRunner(); + struct Runner + { + Runner(DiskJoinStep* d) : djs(d) { - return largeIterationCount; } - -protected: -private: - boost::shared_ptr jp; - rowgroup::RowGroup largeRG, smallRG, outputRG, joinFERG; - std::vector largeKeyCols, smallKeyCols; - boost::shared_ptr largeDL, outputDL; - RowGroupDL* smallDL; - - boost::shared_array LOMapping, SOMapping, SjoinFEMapping, LjoinFEMapping; - TupleHashJoinStep* thjs; - boost::shared_ptr fe; - bool typeless; - JoinType joinType; - boost::shared_ptr joiner; // the same instance THJS uses - - /* main thread, started by JobStep::run() */ - void mainRunner(); - struct Runner + void operator()() { - Runner(DiskJoinStep* d) : djs(d) { } - void operator()() - { - utils::setThreadName("DJSMainRunner"); - djs->mainRunner(); - } - DiskJoinStep* djs; - }; + utils::setThreadName("DJSMainRunner"); + djs->mainRunner(); + } + DiskJoinStep* djs; + }; - void smallReader(); - void largeReader(); - int largeIt; - bool lastLargeIteration; - uint32_t largeIterationCount; + void smallReader(); + void largeReader(); + int largeIt; + bool lastLargeIteration; + uint32_t largeIterationCount; - uint64_t mainThread; // thread handle from thread pool + uint64_t mainThread; // thread handle from thread pool - /* Loader structs */ - struct LoaderOutput + /* Loader structs */ + struct LoaderOutput + { + std::vector smallData; + uint64_t partitionID; + joiner::JoinPartition* jp; + }; + boost::shared_ptr > > loadFIFO; + + struct Loader + { + Loader(DiskJoinStep* d) : djs(d) { - std::vector smallData; - uint64_t partitionID; - joiner::JoinPartition* jp; - }; - boost::shared_ptr > > loadFIFO; - - struct Loader + } + void operator()() { - Loader(DiskJoinStep* d) : djs(d) { } - void operator()() - { - utils::setThreadName("DJSLoader"); - djs->loadFcn(); - } - DiskJoinStep* djs; - }; - void loadFcn(); + utils::setThreadName("DJSLoader"); + djs->loadFcn(); + } + DiskJoinStep* djs; + }; + void loadFcn(); - /* Builder structs */ - struct BuilderOutput + /* Builder structs */ + struct BuilderOutput + { + boost::shared_ptr tupleJoiner; + std::vector smallData; + uint64_t partitionID; + joiner::JoinPartition* jp; + }; + + boost::shared_ptr > > buildFIFO; + + struct Builder + { + Builder(DiskJoinStep* d) : djs(d) { - boost::shared_ptr tupleJoiner; - std::vector smallData; - uint64_t partitionID; - joiner::JoinPartition* jp; - }; - - boost::shared_ptr > > buildFIFO; - - struct Builder + } + void operator()() { - Builder(DiskJoinStep* d) : djs(d) { } - void operator()() - { - utils::setThreadName("DJSBuilder"); - djs->buildFcn(); - } - DiskJoinStep* djs; - }; - void buildFcn(); + utils::setThreadName("DJSBuilder"); + djs->buildFcn(); + } + DiskJoinStep* djs; + }; + void buildFcn(); - /* Joining structs */ - struct Joiner + /* Joining structs */ + struct Joiner + { + Joiner(DiskJoinStep* d) : djs(d) { - Joiner(DiskJoinStep* d) : djs(d) { } - void operator()() - { - utils::setThreadName("DJSJoiner"); - djs->joinFcn(); - } - DiskJoinStep* djs; - }; - void joinFcn(); + } + void operator()() + { + utils::setThreadName("DJSJoiner"); + djs->joinFcn(); + } + DiskJoinStep* djs; + }; + void joinFcn(); - // limits & usage - boost::shared_ptr smallUsage; - int64_t smallLimit; - int64_t largeLimit; - uint64_t partitionSize; + // limits & usage + boost::shared_ptr smallUsage; + int64_t smallLimit; + int64_t largeLimit; + uint64_t partitionSize; - void reportStats(); + void reportStats(); - uint32_t joinerIndex; - bool closedOutput; + uint32_t joinerIndex; + bool closedOutput; }; -} +} // namespace joblist -#endif // DISKJOINSTEP_H +#endif // DISKJOINSTEP_H diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index 0b91d8b74..026cbf516 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -70,119 +70,112 @@ using namespace joblist; namespace { - -void writeToLog(const char* file, int line, const string& msg, LOG_TYPE logto = LOG_TYPE_INFO) +void writeToLog(const char* file, int line, const string& msg, LOG_TYPE logto = LOG_TYPE_INFO) { - LoggingID lid(05); - MessageLog ml(lid); - Message::Args args; - Message m(0); - args.add(file); - args.add("@"); - args.add(line); - args.add(msg); - m.format(args); + LoggingID lid(05); + MessageLog ml(lid); + Message::Args args; + Message m(0); + args.add(file); + args.add("@"); + args.add(line); + args.add(msg); + m.format(args); - switch (logto) - { - case LOG_TYPE_DEBUG: - ml.logDebugMessage(m); - break; + switch (logto) + { + case LOG_TYPE_DEBUG: ml.logDebugMessage(m); break; - case LOG_TYPE_INFO: - ml.logInfoMessage(m); - break; + case LOG_TYPE_INFO: ml.logInfoMessage(m); break; - case LOG_TYPE_WARNING: - ml.logWarningMessage(m); - break; + case LOG_TYPE_WARNING: ml.logWarningMessage(m); break; - case LOG_TYPE_ERROR: - ml.logWarningMessage(m); - break; + case LOG_TYPE_ERROR: ml.logWarningMessage(m); break; - case LOG_TYPE_CRITICAL: - ml.logCriticalMessage(m); - break; - } + case LOG_TYPE_CRITICAL: ml.logCriticalMessage(m); break; + } } // @bug 1463. this function is added for PM failover. for dual/more nic PM, // this function is used to get the module name -string getModuleNameByIPAddr(oam::ModuleTypeConfig moduletypeconfig, - string ipAddress) +string getModuleNameByIPAddr(oam::ModuleTypeConfig moduletypeconfig, string ipAddress) { - string modulename = ""; - DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); + string modulename = ""; + DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); - for ( ; pt != moduletypeconfig.ModuleNetworkList.end() ; pt++) + for (; pt != moduletypeconfig.ModuleNetworkList.end(); pt++) + { + modulename = (*pt).DeviceName; + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for (; pt1 != (*pt).hostConfigList.end(); pt1++) { - modulename = (*pt).DeviceName; - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) - { - if (ipAddress == (*pt1).IPAddr) - return modulename; - } + if (ipAddress == (*pt1).IPAddr) + return modulename; } + } - return modulename; + return modulename; } struct EngineCommRunner { - EngineCommRunner(joblist::DistributedEngineComm* jl, - boost::shared_ptr cl, uint32_t connectionIndex) : jbl(jl), client(cl), - connIndex(connectionIndex) {} - joblist::DistributedEngineComm* jbl; - boost::shared_ptr client; - uint32_t connIndex; - void operator()() + EngineCommRunner(joblist::DistributedEngineComm* jl, boost::shared_ptr cl, + uint32_t connectionIndex) + : jbl(jl), client(cl), connIndex(connectionIndex) + { + } + joblist::DistributedEngineComm* jbl; + boost::shared_ptr client; + uint32_t connIndex; + void operator()() + { + // cout << "Listening on client at 0x" << hex << (ptrdiff_t)client << dec << endl; + try { - //cout << "Listening on client at 0x" << hex << (ptrdiff_t)client << dec << endl; - try - { - jbl->Listen(client, connIndex); - } - catch (std::exception& ex) - { - string what(ex.what()); - cerr << "exception caught in EngineCommRunner: " << what << endl; - - if (what.find("St9bad_alloc") != string::npos) - { - writeToLog(__FILE__, __LINE__, what, LOG_TYPE_CRITICAL); -// abort(); - } - else writeToLog(__FILE__, __LINE__, what); - } - catch (...) - { - string msg("exception caught in EngineCommRunner."); - writeToLog(__FILE__, __LINE__, msg); - cerr << msg << endl; - } + jbl->Listen(client, connIndex); } + catch (std::exception& ex) + { + string what(ex.what()); + cerr << "exception caught in EngineCommRunner: " << what << endl; + + if (what.find("St9bad_alloc") != string::npos) + { + writeToLog(__FILE__, __LINE__, what, LOG_TYPE_CRITICAL); + // abort(); + } + else + writeToLog(__FILE__, __LINE__, what); + } + catch (...) + { + string msg("exception caught in EngineCommRunner."); + writeToLog(__FILE__, __LINE__, msg); + cerr << msg << endl; + } + } }; template struct QueueShutdown : public unary_function { - void operator()(T& x) - { - x.shutdown(); - } + void operator()(T& x) + { + x.shutdown(); + } }; -} +} // namespace /** Debug macro */ #define THROTTLE_DEBUG 0 #if THROTTLE_DEBUG #define THROTTLEDEBUG std::cout #else -#define THROTTLEDEBUG if (false) std::cout +#define THROTTLEDEBUG \ + if (false) \ + std::cout #endif namespace joblist @@ -192,952 +185,951 @@ DistributedEngineComm* DistributedEngineComm::fInstance = 0; /*static*/ DistributedEngineComm* DistributedEngineComm::instance(ResourceManager* rm, bool isExeMgr) { - if (fInstance == 0) - fInstance = new DistributedEngineComm(rm, isExeMgr); + if (fInstance == 0) + fInstance = new DistributedEngineComm(rm, isExeMgr); - return fInstance; + return fInstance; } /*static*/ void DistributedEngineComm::reset() { - delete fInstance; - fInstance = 0; + delete fInstance; + fInstance = 0; } -DistributedEngineComm::DistributedEngineComm(ResourceManager* rm, bool isExeMgr) : - fRm(rm), - fLBIDShift(fRm->getPsLBID_Shift()), - pmCount(0), - fIsExeMgr(isExeMgr) +DistributedEngineComm::DistributedEngineComm(ResourceManager* rm, bool isExeMgr) + : fRm(rm), fLBIDShift(fRm->getPsLBID_Shift()), pmCount(0), fIsExeMgr(isExeMgr) { - - Setup(); + Setup(); } DistributedEngineComm::~DistributedEngineComm() { - Close(); - fInstance = 0; + Close(); + fInstance = 0; } void DistributedEngineComm::Setup() { - // This is here to ensure that this function does not get invoked multiple times simultaneously. - boost::mutex::scoped_lock setupLock(fSetupMutex); + // This is here to ensure that this function does not get invoked multiple times simultaneously. + boost::mutex::scoped_lock setupLock(fSetupMutex); - makeBusy(true); + makeBusy(true); - // This needs to be here to ensure that whenever Setup function is called, the lists are - // empty. It's possible junk was left behind if exception. - ClientList::iterator iter; + // This needs to be here to ensure that whenever Setup function is called, the lists are + // empty. It's possible junk was left behind if exception. + ClientList::iterator iter; - for (iter = newClients.begin(); iter != newClients.end(); ++iter) - { - (*iter)->shutdown(); - } + for (iter = newClients.begin(); iter != newClients.end(); ++iter) + { + (*iter)->shutdown(); + } - newClients.clear(); - newLocks.clear(); + newClients.clear(); + newLocks.clear(); - uint32_t newPmCount = fRm->getPsCount(); - throttleThreshold = fRm->getDECThrottleThreshold(); - tbpsThreadCount = fRm->getJlNumScanReceiveThreads(); - fDECConnectionsPerQuery = fRm->getDECConnectionsPerQuery(); - unsigned numConnections = getNumConnections(); - oam::Oam oam; - ModuleTypeConfig moduletypeconfig; + uint32_t newPmCount = fRm->getPsCount(); + throttleThreshold = fRm->getDECThrottleThreshold(); + tbpsThreadCount = fRm->getJlNumScanReceiveThreads(); + fDECConnectionsPerQuery = fRm->getDECConnectionsPerQuery(); + unsigned numConnections = getNumConnections(); + oam::Oam oam; + ModuleTypeConfig moduletypeconfig; + + try + { + oam.getSystemConfig("pm", moduletypeconfig); + } + catch (...) + { + writeToLog(__FILE__, __LINE__, "oam.getSystemConfig error, unknown exception", LOG_TYPE_ERROR); + throw runtime_error("Setup failed"); + } + + if (newPmCount == 0) + writeToLog(__FILE__, __LINE__, "Got a config file with 0 PMs", LOG_TYPE_CRITICAL); + + // This needs to make sense when compared to the extent size + // fLBIDShift = static_cast(config::Config::uFromText(fConfig->getConfig(section, + // "LBID_Shift"))); + + auto* config = fRm->getConfig(); + std::vector pmsAddressesAndPorts; + for (size_t i = 1; i <= newPmCount; ++i) + { + std::string pmConfigNodeName("PMS"); + pmConfigNodeName.append(std::to_string(i)); + // The port returned by getAddressAndPort can be 0 but this will be handled by + // MessageQueueClient::connect + pmsAddressesAndPorts.push_back(messageqcpp::getAddressAndPort(config, pmConfigNodeName)); + } + + // numConnections must be calculated as number of PMs * number of connections per PM. + // This happens earlier in getNumConnections(). + for (size_t i = 0; i < numConnections; ++i) + { + size_t connectionId = i % newPmCount; + boost::shared_ptr cl(new MessageQueueClient( + pmsAddressesAndPorts[connectionId].first, pmsAddressesAndPorts[connectionId].second)); + boost::shared_ptr nl(new boost::mutex()); try { - oam.getSystemConfig("pm", moduletypeconfig); + if (cl->connect()) + { + newClients.push_back(cl); + // assign the module name + cl->moduleName(getModuleNameByIPAddr(moduletypeconfig, cl->addr2String())); + newLocks.push_back(nl); + StartClientListener(cl, i); + } + else + { + throw runtime_error("Connection refused from PMS" + std::to_string(connectionId)); + } + } + catch (std::exception& ex) + { + if (i < newPmCount) + newPmCount--; + + writeToLog(__FILE__, __LINE__, + "Could not connect to PMS" + std::to_string(connectionId) + ": " + ex.what(), + LOG_TYPE_ERROR); + if (newPmCount == 0) + { + writeToLog(__FILE__, __LINE__, "No more PMs to try to connect to", LOG_TYPE_ERROR); + break; + } } catch (...) { - writeToLog(__FILE__, __LINE__, "oam.getSystemConfig error, unknown exception", LOG_TYPE_ERROR); - throw runtime_error("Setup failed"); + if (i < newPmCount) + newPmCount--; + + writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId), + LOG_TYPE_ERROR); + if (newPmCount == 0) + { + writeToLog(__FILE__, __LINE__, "No more PMs to try to connect to", LOG_TYPE_ERROR); + break; + } } + } - if (newPmCount == 0) - writeToLog(__FILE__, __LINE__, "Got a config file with 0 PMs", - LOG_TYPE_CRITICAL); + // for every entry in newClients up to newPmCount, scan for the same ip in the + // first pmCount. If there is no match, it's a new node, + // call the event listeners' newPMOnline() callbacks. + boost::mutex::scoped_lock lock(eventListenerLock); - //This needs to make sense when compared to the extent size - // fLBIDShift = static_cast(config::Config::uFromText(fConfig->getConfig(section, "LBID_Shift"))); + for (uint32_t i = 0; i < newPmCount; i++) + { + uint32_t j; - auto* config = fRm->getConfig(); - std::vector pmsAddressesAndPorts; - for (size_t i = 1; i <= newPmCount; ++i) + for (j = 0; j < pmCount; j++) { - std::string pmConfigNodeName("PMS"); - pmConfigNodeName.append(std::to_string(i)); - // The port returned by getAddressAndPort can be 0 but this will be handled by - // MessageQueueClient::connect - pmsAddressesAndPorts.push_back(messageqcpp::getAddressAndPort(config, - pmConfigNodeName)); + if (!fPmConnections.empty() && j < fPmConnections.size() && + newClients[i]->isSameAddr(*fPmConnections[j])) + { + break; + } } - // numConnections must be calculated as number of PMs * number of connections per PM. - // This happens earlier in getNumConnections(). - for (size_t i = 0; i < numConnections; ++i) - { - size_t connectionId = i % newPmCount; - boost::shared_ptr cl(new MessageQueueClient(pmsAddressesAndPorts[connectionId].first, - pmsAddressesAndPorts[connectionId].second)); - boost::shared_ptr nl(new boost::mutex()); + if (j == pmCount) + for (uint32_t k = 0; k < eventListeners.size(); k++) + eventListeners[k]->newPMOnline(i); + } - try - { - if (cl->connect()) - { - newClients.push_back(cl); - // assign the module name - cl->moduleName(getModuleNameByIPAddr(moduletypeconfig, cl->addr2String())); - newLocks.push_back(nl); - StartClientListener(cl, i); - } - else - { - throw runtime_error("Connection refused from PMS" + std::to_string(connectionId)); - } - } - catch (std::exception& ex) - { - if (i < newPmCount) - newPmCount--; + lock.unlock(); - writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId) + ": " + ex.what(), LOG_TYPE_ERROR); - if (newPmCount == 0) - { - writeToLog(__FILE__, __LINE__, "No more PMs to try to connect to", LOG_TYPE_ERROR); - break; - } - } - catch (...) - { - if (i < newPmCount) - newPmCount--; + fWlock.swap(newLocks); + fPmConnections.swap(newClients); + // memory barrier to prevent the pmCount assignment migrating upward + atomicops::atomicMb(); + pmCount = newPmCount; - writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId), LOG_TYPE_ERROR); - if (newPmCount == 0) - { - writeToLog(__FILE__, __LINE__, "No more PMs to try to connect to", LOG_TYPE_ERROR); - break; - } - } - } - - // for every entry in newClients up to newPmCount, scan for the same ip in the - // first pmCount. If there is no match, it's a new node, - // call the event listeners' newPMOnline() callbacks. - boost::mutex::scoped_lock lock(eventListenerLock); - - for (uint32_t i = 0; i < newPmCount; i++) - { - uint32_t j; - - for (j = 0; j < pmCount; j++) - { - if (!fPmConnections.empty() && j < fPmConnections.size() && - newClients[i]->isSameAddr(*fPmConnections[j])) - { - break; - } - } - - if (j == pmCount) - for (uint32_t k = 0; k < eventListeners.size(); k++) - eventListeners[k]->newPMOnline(i); - } - - lock.unlock(); - - fWlock.swap(newLocks); - fPmConnections.swap(newClients); - // memory barrier to prevent the pmCount assignment migrating upward - atomicops::atomicMb(); - pmCount = newPmCount; - - newLocks.clear(); - newClients.clear(); + newLocks.clear(); + newClients.clear(); } int DistributedEngineComm::Close() { - //cout << "DistributedEngineComm::Close() called" << endl; + // cout << "DistributedEngineComm::Close() called" << endl; - makeBusy(false); - // for each MessageQueueClient in pmConnections delete the MessageQueueClient; - fPmConnections.clear(); - fPmReader.clear(); - return 0; + makeBusy(false); + // for each MessageQueueClient in pmConnections delete the MessageQueueClient; + fPmConnections.clear(); + fPmReader.clear(); + return 0; } void DistributedEngineComm::Listen(boost::shared_ptr client, uint32_t connIndex) { - SBS sbs; + SBS sbs; - try + try + { + while (Busy()) { - while (Busy()) - { - Stats stats; - //TODO: This call blocks so setting Busy() in another thread doesn't work here... - sbs = client->read(0, NULL, &stats); + Stats stats; + // TODO: This call blocks so setting Busy() in another thread doesn't work here... + sbs = client->read(0, NULL, &stats); - if (sbs->length() != 0) - { - addDataToOutput(sbs, connIndex, &stats); - } - else // got zero bytes on read, nothing more will come - goto Error; - } - - return; - } - catch (std::exception& e) - { - cerr << "DEC Caught EXCEPTION: " << e.what() << endl; - goto Error; - } - catch (...) - { - cerr << "DEC Caught UNKNOWN EXCEPT" << endl; + if (sbs->length() != 0) + { + addDataToOutput(sbs, connIndex, &stats); + } + else // got zero bytes on read, nothing more will come goto Error; } + return; + } + catch (std::exception& e) + { + cerr << "DEC Caught EXCEPTION: " << e.what() << endl; + goto Error; + } + catch (...) + { + cerr << "DEC Caught UNKNOWN EXCEPT" << endl; + goto Error; + } + Error: - // @bug 488 - error condition! push 0 length bs to messagequeuemap and - // eventually let jobstep error out. - boost::mutex::scoped_lock lk(fMlock); + // @bug 488 - error condition! push 0 length bs to messagequeuemap and + // eventually let jobstep error out. + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok; + sbs.reset(new ByteStream(0)); + + for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok) + { + map_tok->second->queue.clear(); + (void)atomicops::atomicInc(&map_tok->second->unackedWork[0]); + map_tok->second->queue.push(sbs); + } + lk.unlock(); + + if (fIsExeMgr) + { + // std::cout << "WARNING: DEC READ 0 LENGTH BS FROM " + // << client->otherEnd()<< " OR GOT AN EXCEPTION READING" << std::endl; + decltype(pmCount) originalPMCount = pmCount; + // Re-establish if a remote PM restarted. + std::this_thread::sleep_for(std::chrono::seconds(3)); + Setup(); + if (originalPMCount != pmCount) + { + ostringstream os; + os << "DEC: lost connection to " << client->addr2String(); + writeToLog(__FILE__, __LINE__, os.str(), LOG_TYPE_ERROR); + } + + /* + // reset the pmconnection vector + ClientList tempConns; + boost::mutex::scoped_lock onErrLock(fOnErrMutex); + string moduleName = client->moduleName(); + //cout << "moduleName=" << moduleName << endl; + for ( uint32_t i = 0; i < fPmConnections.size(); i++) + { + if (moduleName != fPmConnections[i]->moduleName()) + tempConns.push_back(fPmConnections[i]); + //else + //cout << "DEC remove PM" << fPmConnections[i]->otherEnd() << " moduleName=" << + fPmConnections[i]->moduleName() << endl; + } + + if (tempConns.size() == fPmConnections.size()) return; + + fPmConnections.swap(tempConns); + pmCount = (pmCount == 0 ? 0 : pmCount - 1); + //cout << "PMCOUNT=" << pmCount << endl; + + // log it + ostringstream os; + os << "DEC: lost connection to " << client->addr2String(); + writeToLog(__FILE__, __LINE__, os.str(), LOG_TYPE_CRITICAL); + */ + } + return; +} + +void DistributedEngineComm::addQueue(uint32_t key, bool sendACKs) +{ + bool b; + + boost::mutex* lock = new boost::mutex(); + condition* cond = new condition(); + uint32_t firstPMInterleavedConnectionId = + key % (fPmConnections.size() / pmCount) * fDECConnectionsPerQuery * pmCount % fPmConnections.size(); + boost::shared_ptr mqe(new MQE(pmCount, firstPMInterleavedConnectionId)); + + mqe->queue = StepMsgQueue(lock, cond); + mqe->sendACKs = sendACKs; + mqe->throttled = false; + + boost::mutex::scoped_lock lk(fMlock); + b = fSessionMessages.insert(pair >(key, mqe)).second; + + if (!b) + { + ostringstream os; + os << "DEC: attempt to add a queue with a duplicate ID " << key << endl; + throw runtime_error(os.str()); + } +} + +void DistributedEngineComm::removeQueue(uint32_t key) +{ + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + return; + + map_tok->second->queue.shutdown(); + map_tok->second->queue.clear(); + fSessionMessages.erase(map_tok); +} + +void DistributedEngineComm::shutdownQueue(uint32_t key) +{ + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + return; + + map_tok->second->queue.shutdown(); + map_tok->second->queue.clear(); +} + +void DistributedEngineComm::read(uint32_t key, SBS& bs) +{ + boost::shared_ptr mqe; + + // Find the StepMsgQueueList for this session + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + { + ostringstream os; + + os << "DEC: attempt to read(bs) from a nonexistent queue\n"; + throw runtime_error(os.str()); + } + + mqe = map_tok->second; + lk.unlock(); + + // this method can block: you can't hold any locks here... + TSQSize_t queueSize = mqe->queue.pop(&bs); + + if (bs && mqe->sendACKs) + { + boost::mutex::scoped_lock lk(ackLock); + + if (mqe->throttled && !mqe->hasBigMsgs && queueSize.size <= disableThreshold) + setFlowControl(false, key, mqe); + + vector v; + v.push_back(bs); + sendAcks(key, v, mqe, queueSize.size); + } + + if (!bs) + bs.reset(new ByteStream()); +} + +const ByteStream DistributedEngineComm::read(uint32_t key) +{ + SBS sbs; + boost::shared_ptr mqe; + + // Find the StepMsgQueueList for this session + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + { + ostringstream os; + + os << "DEC: read(): attempt to read from a nonexistent queue\n"; + throw runtime_error(os.str()); + } + + mqe = map_tok->second; + lk.unlock(); + + TSQSize_t queueSize = mqe->queue.pop(&sbs); + + if (sbs && mqe->sendACKs) + { + boost::mutex::scoped_lock lk(ackLock); + + if (mqe->throttled && !mqe->hasBigMsgs && queueSize.size <= disableThreshold) + setFlowControl(false, key, mqe); + + vector v; + v.push_back(sbs); + sendAcks(key, v, mqe, queueSize.size); + } + + if (!sbs) + sbs.reset(new ByteStream()); + + return *sbs; +} + +void DistributedEngineComm::read_all(uint32_t key, vector& v) +{ + boost::shared_ptr mqe; + + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + { + ostringstream os; + os << "DEC: read_all(): attempt to read from a nonexistent queue\n"; + throw runtime_error(os.str()); + } + + mqe = map_tok->second; + lk.unlock(); + + mqe->queue.pop_all(v); + + if (mqe->sendACKs) + { + boost::mutex::scoped_lock lk(ackLock); + sendAcks(key, v, mqe, 0); + } +} + +void DistributedEngineComm::read_some(uint32_t key, uint32_t divisor, vector& v, bool* flowControlOn) +{ + boost::shared_ptr mqe; + + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + { + ostringstream os; + + os << "DEC: read_some(): attempt to read from a nonexistent queue\n"; + throw runtime_error(os.str()); + } + + mqe = map_tok->second; + lk.unlock(); + + TSQSize_t queueSize = mqe->queue.pop_some(divisor, v, 1); // need to play with the min # + + if (flowControlOn) + *flowControlOn = false; + + if (mqe->sendACKs) + { + boost::mutex::scoped_lock lk(ackLock); + + if (mqe->throttled && !mqe->hasBigMsgs && queueSize.size <= disableThreshold) + setFlowControl(false, key, mqe); + + sendAcks(key, v, mqe, queueSize.size); + + if (flowControlOn) + *flowControlOn = mqe->throttled; + } +} + +void DistributedEngineComm::sendAcks(uint32_t uniqueID, const vector& msgs, boost::shared_ptr mqe, + size_t queueSize) +{ + ISMPacketHeader* ism; + uint32_t l_msgCount = msgs.size(); + + /* If the current queue size > target, do nothing. + * If the original queue size > target, ACK the msgs below the target. + */ + if (!mqe->throttled || queueSize >= mqe->targetQueueSize) + { + /* no acks will be sent, but update unackedwork to keep the #s accurate */ + uint16_t numack = 0; + uint32_t sockidx = 0; + + while (l_msgCount > 0) + { + nextPMToACK(mqe, l_msgCount, &sockidx, &numack); + idbassert(numack <= l_msgCount); + l_msgCount -= numack; + } + + return; + } + + size_t totalMsgSize = 0; + + for (uint32_t i = 0; i < msgs.size(); i++) + totalMsgSize += msgs[i]->lengthWithHdrOverhead(); + + if (queueSize + totalMsgSize > mqe->targetQueueSize) + { + /* update unackedwork for the overage that will never be acked */ + int64_t overage = queueSize + totalMsgSize - mqe->targetQueueSize; + uint16_t numack = 0; + uint32_t sockidx = 0; + uint32_t msgsToIgnore; + + for (msgsToIgnore = 0; overage >= 0; msgsToIgnore++) + overage -= msgs[msgsToIgnore]->lengthWithHdrOverhead(); + + if (overage < 0) + msgsToIgnore--; + + l_msgCount = msgs.size() - msgsToIgnore; // this num gets acked + + while (msgsToIgnore > 0) + { + nextPMToACK(mqe, msgsToIgnore, &sockidx, &numack); + idbassert(numack <= msgsToIgnore); + msgsToIgnore -= numack; + } + } + + if (l_msgCount > 0) + { + ByteStream msg(sizeof(ISMPacketHeader)); + uint16_t* toAck; + vector pmAcked(pmCount, false); + + ism = (ISMPacketHeader*)msg.getInputPtr(); + // The only var checked by ReadThread is the Command var. The others + // are wasted space. We hijack the Size, & Flags fields for the + // params to the ACK msg. + + ism->Interleave = uniqueID; + ism->Command = BATCH_PRIMITIVE_ACK; + toAck = &ism->Size; + + msg.advanceInputPtr(sizeof(ISMPacketHeader)); + + while (l_msgCount > 0) + { + /* could have to send up to pmCount ACKs */ + uint32_t sockIndex = 0; + + /* This will reset the ACK field in the Bytestream directly, and nothing + * else needs to change if multiple msgs are sent. */ + nextPMToACK(mqe, l_msgCount, &sockIndex, toAck); + idbassert(*toAck <= l_msgCount); + l_msgCount -= *toAck; + pmAcked[sockIndex] = true; + writeToClient(sockIndex, msg); + } + + // @bug4436, when no more unacked work, send an ack to all PMs that haven't been acked. + // This is apply to the big message case only. For small messages, the flow control is + // disabled when the queue size is below the disableThreshold. + if (mqe->hasBigMsgs) + { + uint64_t totalUnackedWork = 0; + + for (uint32_t i = 0; i < pmCount; i++) + totalUnackedWork += mqe->unackedWork[i]; + + if (totalUnackedWork == 0) + { + *toAck = 1; + + for (uint32_t i = 0; i < pmCount; i++) + { + if (!pmAcked[i]) + writeToClient(i, msg); + } + } + } + } +} + +void DistributedEngineComm::nextPMToACK(boost::shared_ptr mqe, uint32_t maxAck, uint32_t* sockIndex, + uint16_t* numToAck) +{ + uint32_t i; + uint32_t& nextIndex = mqe->ackSocketIndex; + + /* Other threads can be touching mqe->unackedWork at the same time, but because of + * the locking env, mqe->unackedWork can only grow; whatever gets latched in this fcn + * is a safe minimum at the point of use. */ + + if (mqe->unackedWork[nextIndex] >= maxAck) + { + (void)atomicops::atomicSub(&mqe->unackedWork[nextIndex], maxAck); + *sockIndex = nextIndex; + // FIXME: we're going to truncate here from 32 to 16 bits. Hopefully this will always fit... + *numToAck = maxAck; + + if (pmCount > 0) + nextIndex = (nextIndex + 1) % pmCount; + + return; + } + else + { + for (i = 0; i < pmCount; i++) + { + uint32_t curVal = mqe->unackedWork[nextIndex]; + uint32_t unackedWork = (curVal > maxAck ? maxAck : curVal); + + if (unackedWork > 0) + { + (void)atomicops::atomicSub(&mqe->unackedWork[nextIndex], unackedWork); + *sockIndex = nextIndex; + *numToAck = unackedWork; + + if (pmCount > 0) + nextIndex = (nextIndex + 1) % pmCount; + + return; + } + + if (pmCount > 0) + nextIndex = (nextIndex + 1) % pmCount; + } + + cerr << "DEC::nextPMToACK(): Couldn't find a PM to ACK! "; + + for (i = 0; i < pmCount; i++) + cerr << mqe->unackedWork[i] << " "; + + cerr << " max: " << maxAck; + cerr << endl; + // make sure the returned vars are legitimate + *sockIndex = nextIndex; + *numToAck = maxAck / pmCount; + + if (pmCount > 0) + nextIndex = (nextIndex + 1) % pmCount; + + return; + } +} + +void DistributedEngineComm::setFlowControl(bool enabled, uint32_t uniqueID, boost::shared_ptr mqe) +{ + mqe->throttled = enabled; + ByteStream msg(sizeof(ISMPacketHeader)); + ISMPacketHeader* ism = (ISMPacketHeader*)msg.getInputPtr(); + + ism->Interleave = uniqueID; + ism->Command = BATCH_PRIMITIVE_ACK; + ism->Size = (enabled ? 0 : -1); + +#ifdef VALGRIND + /* XXXPAT: For testing in valgrind, init the vars that don't get used */ + ism->Flags = 0; + ism->Type = 0; + ism->MsgCount = 0; + ism->Status = 0; +#endif + + msg.advanceInputPtr(sizeof(ISMPacketHeader)); + + for (uint32_t i = 0; i < mqe->pmCount; i++) + writeToClient(i, msg); +} + +void DistributedEngineComm::write(uint32_t senderID, ByteStream& msg) +{ + ISMPacketHeader* ism = (ISMPacketHeader*)msg.buf(); + uint32_t dest; + uint32_t numConn = fPmConnections.size(); + + if (numConn > 0) + { + switch (ism->Command) + { + case BATCH_PRIMITIVE_CREATE: + /* Disable flow control initially */ + msg << (uint32_t)-1; + /* FALLTHRU */ + + case BATCH_PRIMITIVE_DESTROY: + case BATCH_PRIMITIVE_ADD_JOINER: + case BATCH_PRIMITIVE_END_JOINER: + case BATCH_PRIMITIVE_ABORT: + case DICT_CREATE_EQUALITY_FILTER: + case DICT_DESTROY_EQUALITY_FILTER: + /* XXXPAT: This relies on the assumption that the first pmCount "PMS*" + entries in the config file point to unique PMs */ + uint32_t i; + + for (i = 0; i < pmCount; i++) + writeToClient(i, msg, senderID); + + return; + + case BATCH_PRIMITIVE_RUN: + case DICT_TOKEN_BY_SCAN_COMPARE: + // for efficiency, writeToClient() grabs the interleaving factor for the caller, + // and decides the final connection index because it already grabs the + // caller's queue information + dest = ism->Interleave; + writeToClient(dest, msg, senderID, true); + break; + + default: idbassert_s(0, "Unknown message type"); + } + } + else + { + writeToLog(__FILE__, __LINE__, "No PrimProcs are running", LOG_TYPE_DEBUG); + throw IDBExcept(ERR_NO_PRIMPROC); + } +} + +void DistributedEngineComm::write(messageqcpp::ByteStream& msg, uint32_t connection) +{ + ISMPacketHeader* ism = (ISMPacketHeader*)msg.buf(); + PrimitiveHeader* pm = (PrimitiveHeader*)(ism + 1); + uint32_t senderID = pm->UniqueID; + + boost::mutex::scoped_lock lk(fMlock, boost::defer_lock_t()); + MessageQueueMap::iterator it; + // This keeps mqe's stats from being freed until end of function + boost::shared_ptr mqe; + Stats* senderStats = NULL; + + lk.lock(); + it = fSessionMessages.find(senderID); + + if (it != fSessionMessages.end()) + { + mqe = it->second; + senderStats = &(mqe->stats); + } + + lk.unlock(); + + newClients[connection]->write(msg, NULL, senderStats); +} + +void DistributedEngineComm::StartClientListener(boost::shared_ptr cl, uint32_t connIndex) +{ + boost::thread* thrd = new boost::thread(EngineCommRunner(this, cl, connIndex)); + fPmReader.push_back(thrd); +} + +void DistributedEngineComm::addDataToOutput(SBS sbs, uint32_t connIndex, Stats* stats) +{ + ISMPacketHeader* hdr = (ISMPacketHeader*)(sbs->buf()); + PrimitiveHeader* p = (PrimitiveHeader*)(hdr + 1); + uint32_t uniqueId = p->UniqueID; + boost::shared_ptr mqe; + + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(uniqueId); + + if (map_tok == fSessionMessages.end()) + { + // For debugging... + // cerr << "DistributedEngineComm::AddDataToOutput: tried to add a message to a dead session: " << + // uniqueId << ", size " << sbs->length() << ", step id " << p->StepID << endl; + return; + } + + mqe = map_tok->second; + lk.unlock(); + + if (pmCount > 0) + { + (void)atomicops::atomicInc(&mqe->unackedWork[connIndex % pmCount]); + } + + TSQSize_t queueSize = mqe->queue.push(sbs); + + if (mqe->sendACKs) + { + boost::mutex::scoped_lock lk(ackLock); + uint64_t msgSize = sbs->lengthWithHdrOverhead(); + + if (!mqe->throttled && msgSize > (targetRecvQueueSize / 2)) + doHasBigMsgs(mqe, (300 * 1024 * 1024 > 3 * msgSize ? 300 * 1024 * 1024 + : 3 * msgSize)); // buffer at least 3 big msgs + + if (!mqe->throttled && queueSize.size >= mqe->targetQueueSize) + setFlowControl(true, uniqueId, mqe); + } + + if (stats) + mqe->stats.dataRecvd(stats->dataRecvd()); +} + +void DistributedEngineComm::doHasBigMsgs(boost::shared_ptr mqe, uint64_t targetSize) +{ + mqe->hasBigMsgs = true; + + if (mqe->targetQueueSize < targetSize) + mqe->targetQueueSize = targetSize; +} + +int DistributedEngineComm::writeToClient(size_t aPMIndex, const ByteStream& bs, uint32_t senderUniqueID, + bool doInterleaving) +{ + boost::mutex::scoped_lock lk(fMlock, boost::defer_lock_t()); + MessageQueueMap::iterator it; + // Keep mqe's stats from being freed early + boost::shared_ptr mqe; + Stats* senderStats = NULL; + + if (fPmConnections.size() == 0) + return 0; + + uint32_t connectionId = aPMIndex; + if (senderUniqueID != numeric_limits::max()) + { + lk.lock(); + it = fSessionMessages.find(senderUniqueID); + + if (it != fSessionMessages.end()) + { + mqe = it->second; + senderStats = &(mqe->stats); + size_t pmIndex = aPMIndex % mqe->pmCount; + connectionId = it->second->getNextConnectionId(pmIndex, fPmConnections.size(), fDECConnectionsPerQuery); + } + + lk.unlock(); + } + + try + { + ClientList::value_type client = fPmConnections[connectionId]; + + if (!client->isAvailable()) + return 0; + + boost::mutex::scoped_lock lk(*(fWlock[connectionId])); + client->write(bs, NULL, senderStats); + return 0; + } + catch (...) + { + // @bug 488. error out under such condition instead of re-trying other connection, + // by pushing 0 size bytestream to messagequeue and throw exception + SBS sbs; + lk.lock(); + // std::cout << "WARNING: DEC WRITE BROKEN PIPE. PMS index = " << index << std::endl; MessageQueueMap::iterator map_tok; sbs.reset(new ByteStream(0)); for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok) { - map_tok->second->queue.clear(); - (void)atomicops::atomicInc(&map_tok->second->unackedWork[0]); - map_tok->second->queue.push(sbs); - } - lk.unlock(); - - if (fIsExeMgr) - { - //std::cout << "WARNING: DEC READ 0 LENGTH BS FROM " - // << client->otherEnd()<< " OR GOT AN EXCEPTION READING" << std::endl; - decltype(pmCount) originalPMCount = pmCount; - // Re-establish if a remote PM restarted. - std::this_thread::sleep_for(std::chrono::seconds(3)); - Setup(); - if (originalPMCount != pmCount) - { - ostringstream os; - os << "DEC: lost connection to " << client->addr2String(); - writeToLog(__FILE__, __LINE__, os.str(), LOG_TYPE_ERROR); - } - -/* - // reset the pmconnection vector - ClientList tempConns; - boost::mutex::scoped_lock onErrLock(fOnErrMutex); - string moduleName = client->moduleName(); - //cout << "moduleName=" << moduleName << endl; - for ( uint32_t i = 0; i < fPmConnections.size(); i++) - { - if (moduleName != fPmConnections[i]->moduleName()) - tempConns.push_back(fPmConnections[i]); - //else - //cout << "DEC remove PM" << fPmConnections[i]->otherEnd() << " moduleName=" << fPmConnections[i]->moduleName() << endl; - } - - if (tempConns.size() == fPmConnections.size()) return; - - fPmConnections.swap(tempConns); - pmCount = (pmCount == 0 ? 0 : pmCount - 1); - //cout << "PMCOUNT=" << pmCount << endl; - - // log it - ostringstream os; - os << "DEC: lost connection to " << client->addr2String(); - writeToLog(__FILE__, __LINE__, os.str(), LOG_TYPE_CRITICAL); -*/ - } - return; -} - -void DistributedEngineComm::addQueue(uint32_t key, bool sendACKs) -{ - bool b; - - boost::mutex* lock = new boost::mutex(); - condition* cond = new condition(); - uint32_t firstPMInterleavedConnectionId = key % (fPmConnections.size() / pmCount) * fDECConnectionsPerQuery * pmCount % fPmConnections.size(); - boost::shared_ptr mqe(new MQE(pmCount, firstPMInterleavedConnectionId)); - - mqe->queue = StepMsgQueue(lock, cond); - mqe->sendACKs = sendACKs; - mqe->throttled = false; - - boost::mutex::scoped_lock lk ( fMlock ); - b = fSessionMessages.insert(pair >(key, mqe)).second; - - if (!b) - { - ostringstream os; - os << "DEC: attempt to add a queue with a duplicate ID " << key << endl; - throw runtime_error(os.str()); - } -} - -void DistributedEngineComm::removeQueue(uint32_t key) -{ - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - return; - - map_tok->second->queue.shutdown(); - map_tok->second->queue.clear(); - fSessionMessages.erase(map_tok); -} - -void DistributedEngineComm::shutdownQueue(uint32_t key) -{ - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - return; - - map_tok->second->queue.shutdown(); - map_tok->second->queue.clear(); -} - -void DistributedEngineComm::read(uint32_t key, SBS& bs) -{ - boost::shared_ptr mqe; - - //Find the StepMsgQueueList for this session - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - { - ostringstream os; - - os << "DEC: attempt to read(bs) from a nonexistent queue\n"; - throw runtime_error(os.str()); - } - - mqe = map_tok->second; - lk.unlock(); - - //this method can block: you can't hold any locks here... - TSQSize_t queueSize = mqe->queue.pop(&bs); - - if (bs && mqe->sendACKs) - { - boost::mutex::scoped_lock lk(ackLock); - - if (mqe->throttled && !mqe->hasBigMsgs && queueSize.size <= disableThreshold) - setFlowControl(false, key, mqe); - - vector v; - v.push_back(bs); - sendAcks(key, v, mqe, queueSize.size); - } - - if (!bs) - bs.reset(new ByteStream()); -} - -const ByteStream DistributedEngineComm::read(uint32_t key) -{ - SBS sbs; - boost::shared_ptr mqe; - - //Find the StepMsgQueueList for this session - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - { - ostringstream os; - - os << "DEC: read(): attempt to read from a nonexistent queue\n"; - throw runtime_error(os.str()); - } - - mqe = map_tok->second; - lk.unlock(); - - TSQSize_t queueSize = mqe->queue.pop(&sbs); - - if (sbs && mqe->sendACKs) - { - boost::mutex::scoped_lock lk(ackLock); - - if (mqe->throttled && !mqe->hasBigMsgs && queueSize.size <= disableThreshold) - setFlowControl(false, key, mqe); - - vector v; - v.push_back(sbs); - sendAcks(key, v, mqe, queueSize.size); - } - - if (!sbs) - sbs.reset(new ByteStream()); - - return *sbs; -} - -void DistributedEngineComm::read_all(uint32_t key, vector& v) -{ - boost::shared_ptr mqe; - - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - { - ostringstream os; - os << "DEC: read_all(): attempt to read from a nonexistent queue\n"; - throw runtime_error(os.str()); - } - - mqe = map_tok->second; - lk.unlock(); - - mqe->queue.pop_all(v); - - if (mqe->sendACKs) - { - boost::mutex::scoped_lock lk(ackLock); - sendAcks(key, v, mqe, 0); - } -} - -void DistributedEngineComm::read_some(uint32_t key, uint32_t divisor, vector& v, - bool* flowControlOn) -{ - boost::shared_ptr mqe; - - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - { - ostringstream os; - - os << "DEC: read_some(): attempt to read from a nonexistent queue\n"; - throw runtime_error(os.str()); - } - - mqe = map_tok->second; - lk.unlock(); - - TSQSize_t queueSize = mqe->queue.pop_some(divisor, v, 1); // need to play with the min # - - if (flowControlOn) - *flowControlOn = false; - - if (mqe->sendACKs) - { - boost::mutex::scoped_lock lk(ackLock); - - if (mqe->throttled && !mqe->hasBigMsgs && queueSize.size <= disableThreshold) - setFlowControl(false, key, mqe); - - sendAcks(key, v, mqe, queueSize.size); - - if (flowControlOn) - *flowControlOn = mqe->throttled; - } -} - -void DistributedEngineComm::sendAcks(uint32_t uniqueID, const vector& msgs, - boost::shared_ptr mqe, size_t queueSize) -{ - ISMPacketHeader* ism; - uint32_t l_msgCount = msgs.size(); - - /* If the current queue size > target, do nothing. - * If the original queue size > target, ACK the msgs below the target. - */ - if (!mqe->throttled || queueSize >= mqe->targetQueueSize) - { - /* no acks will be sent, but update unackedwork to keep the #s accurate */ - uint16_t numack = 0; - uint32_t sockidx = 0; - - while (l_msgCount > 0) - { - nextPMToACK(mqe, l_msgCount, &sockidx, &numack); - idbassert(numack <= l_msgCount); - l_msgCount -= numack; - } - - return; - } - - size_t totalMsgSize = 0; - - for (uint32_t i = 0; i < msgs.size(); i++) - totalMsgSize += msgs[i]->lengthWithHdrOverhead(); - - if (queueSize + totalMsgSize > mqe->targetQueueSize) - { - /* update unackedwork for the overage that will never be acked */ - int64_t overage = queueSize + totalMsgSize - mqe->targetQueueSize; - uint16_t numack = 0; - uint32_t sockidx = 0; - uint32_t msgsToIgnore; - - for (msgsToIgnore = 0; overage >= 0; msgsToIgnore++) - overage -= msgs[msgsToIgnore]->lengthWithHdrOverhead(); - - if (overage < 0) - msgsToIgnore--; - - l_msgCount = msgs.size() - msgsToIgnore; // this num gets acked - - while (msgsToIgnore > 0) - { - nextPMToACK(mqe, msgsToIgnore, &sockidx, &numack); - idbassert(numack <= msgsToIgnore); - msgsToIgnore -= numack; - } - } - - if (l_msgCount > 0) - { - ByteStream msg(sizeof(ISMPacketHeader)); - uint16_t* toAck; - vector pmAcked(pmCount, false); - - ism = (ISMPacketHeader*) msg.getInputPtr(); - // The only var checked by ReadThread is the Command var. The others - // are wasted space. We hijack the Size, & Flags fields for the - // params to the ACK msg. - - ism->Interleave = uniqueID; - ism->Command = BATCH_PRIMITIVE_ACK; - toAck = &ism->Size; - - msg.advanceInputPtr(sizeof(ISMPacketHeader)); - - while (l_msgCount > 0) - { - /* could have to send up to pmCount ACKs */ - uint32_t sockIndex = 0; - - /* This will reset the ACK field in the Bytestream directly, and nothing - * else needs to change if multiple msgs are sent. */ - nextPMToACK(mqe, l_msgCount, &sockIndex, toAck); - idbassert(*toAck <= l_msgCount); - l_msgCount -= *toAck; - pmAcked[sockIndex] = true; - writeToClient(sockIndex, msg); - } - - // @bug4436, when no more unacked work, send an ack to all PMs that haven't been acked. - // This is apply to the big message case only. For small messages, the flow control is - // disabled when the queue size is below the disableThreshold. - if (mqe->hasBigMsgs) - { - uint64_t totalUnackedWork = 0; - - for (uint32_t i = 0; i < pmCount; i++) - totalUnackedWork += mqe->unackedWork[i]; - - if (totalUnackedWork == 0) - { - *toAck = 1; - - for (uint32_t i = 0; i < pmCount; i++) - { - if (!pmAcked[i]) - writeToClient(i, msg); - } - } - } - } -} - -void DistributedEngineComm::nextPMToACK(boost::shared_ptr mqe, uint32_t maxAck, - uint32_t* sockIndex, uint16_t* numToAck) -{ - uint32_t i; - uint32_t& nextIndex = mqe->ackSocketIndex; - - /* Other threads can be touching mqe->unackedWork at the same time, but because of - * the locking env, mqe->unackedWork can only grow; whatever gets latched in this fcn - * is a safe minimum at the point of use. */ - - if (mqe->unackedWork[nextIndex] >= maxAck) - { - (void)atomicops::atomicSub(&mqe->unackedWork[nextIndex], maxAck); - *sockIndex = nextIndex; - //FIXME: we're going to truncate here from 32 to 16 bits. Hopefully this will always fit... - *numToAck = maxAck; - - if (pmCount > 0) - nextIndex = (nextIndex + 1) % pmCount; - - return; - } - else - { - for (i = 0; i < pmCount; i++) - { - uint32_t curVal = mqe->unackedWork[nextIndex]; - uint32_t unackedWork = (curVal > maxAck ? maxAck : curVal); - - if (unackedWork > 0) - { - (void)atomicops::atomicSub(&mqe->unackedWork[nextIndex], unackedWork); - *sockIndex = nextIndex; - *numToAck = unackedWork; - - if (pmCount > 0) - nextIndex = (nextIndex + 1) % pmCount; - - return; - } - - if (pmCount > 0) - nextIndex = (nextIndex + 1) % pmCount; - } - - cerr << "DEC::nextPMToACK(): Couldn't find a PM to ACK! "; - - for (i = 0; i < pmCount; i++) - cerr << mqe->unackedWork[i] << " "; - - cerr << " max: " << maxAck; - cerr << endl; - //make sure the returned vars are legitimate - *sockIndex = nextIndex; - *numToAck = maxAck / pmCount; - - if (pmCount > 0) - nextIndex = (nextIndex + 1) % pmCount; - - return; - } -} - -void DistributedEngineComm::setFlowControl(bool enabled, uint32_t uniqueID, boost::shared_ptr mqe) -{ - mqe->throttled = enabled; - ByteStream msg(sizeof(ISMPacketHeader)); - ISMPacketHeader* ism = (ISMPacketHeader*) msg.getInputPtr(); - - ism->Interleave = uniqueID; - ism->Command = BATCH_PRIMITIVE_ACK; - ism->Size = (enabled ? 0 : -1); - -#ifdef VALGRIND - /* XXXPAT: For testing in valgrind, init the vars that don't get used */ - ism->Flags = 0; - ism->Type = 0; - ism->MsgCount = 0; - ism->Status = 0; -#endif - - msg.advanceInputPtr(sizeof(ISMPacketHeader)); - - for (uint32_t i = 0; i < mqe->pmCount; i++) - writeToClient(i, msg); -} - -void DistributedEngineComm::write(uint32_t senderID, ByteStream& msg) -{ - ISMPacketHeader* ism = (ISMPacketHeader*) msg.buf(); - uint32_t dest; - uint32_t numConn = fPmConnections.size(); - - if (numConn > 0) - { - switch (ism->Command) - { - case BATCH_PRIMITIVE_CREATE: - /* Disable flow control initially */ - msg << (uint32_t) - 1; - /* FALLTHRU */ - - case BATCH_PRIMITIVE_DESTROY: - case BATCH_PRIMITIVE_ADD_JOINER: - case BATCH_PRIMITIVE_END_JOINER: - case BATCH_PRIMITIVE_ABORT: - case DICT_CREATE_EQUALITY_FILTER: - case DICT_DESTROY_EQUALITY_FILTER: - /* XXXPAT: This relies on the assumption that the first pmCount "PMS*" - entries in the config file point to unique PMs */ - uint32_t i; - - for (i = 0; i < pmCount; i++) - writeToClient(i, msg, senderID); - - return; - - case BATCH_PRIMITIVE_RUN: - case DICT_TOKEN_BY_SCAN_COMPARE: - // for efficiency, writeToClient() grabs the interleaving factor for the caller, - // and decides the final connection index because it already grabs the - // caller's queue information - dest = ism->Interleave; - writeToClient(dest, msg, senderID, true); - break; - - default: - idbassert_s(0, "Unknown message type"); - } - } - else - { - writeToLog(__FILE__, __LINE__, "No PrimProcs are running", LOG_TYPE_DEBUG); - throw IDBExcept(ERR_NO_PRIMPROC); - } -} - -void DistributedEngineComm::write(messageqcpp::ByteStream& msg, uint32_t connection) -{ - ISMPacketHeader* ism = (ISMPacketHeader*) msg.buf(); - PrimitiveHeader* pm = (PrimitiveHeader*) (ism + 1); - uint32_t senderID = pm->UniqueID; - - boost::mutex::scoped_lock lk(fMlock, boost::defer_lock_t()); - MessageQueueMap::iterator it; - // This keeps mqe's stats from being freed until end of function - boost::shared_ptr mqe; - Stats* senderStats = NULL; - - lk.lock(); - it = fSessionMessages.find(senderID); - - if (it != fSessionMessages.end()) - { - mqe = it->second; - senderStats = &(mqe->stats); + map_tok->second->queue.clear(); + (void)atomicops::atomicInc(&map_tok->second->unackedWork[0]); + map_tok->second->queue.push(sbs); } lk.unlock(); + /* + // reconfig the connection array + ClientList tempConns; + { + //cout << "WARNING: DEC WRITE BROKEN PIPE " << fPmConnections[index]->otherEnd()<< + endl; boost::mutex::scoped_lock onErrLock(fOnErrMutex); string moduleName = + fPmConnections[index]->moduleName(); + //cout << "module name = " << moduleName << endl; + if (index >= fPmConnections.size()) return 0; - newClients[connection]->write(msg, NULL, senderStats); -} + for (uint32_t i = 0; i < fPmConnections.size(); i++) + { + if (moduleName != fPmConnections[i]->moduleName()) + tempConns.push_back(fPmConnections[i]); + } + if (tempConns.size() == fPmConnections.size()) return 0; + fPmConnections.swap(tempConns); + pmCount = (pmCount == 0 ? 0 : pmCount - 1); + } + // send alarm + ALARMManager alarmMgr; + string alarmItem("UNKNOWN"); -void DistributedEngineComm::StartClientListener(boost::shared_ptr cl, uint32_t connIndex) -{ - boost::thread* thrd = new boost::thread(EngineCommRunner(this, cl, connIndex)); - fPmReader.push_back(thrd); -} - -void DistributedEngineComm::addDataToOutput(SBS sbs, uint32_t connIndex, Stats* stats) -{ - ISMPacketHeader* hdr = (ISMPacketHeader*)(sbs->buf()); - PrimitiveHeader* p = (PrimitiveHeader*)(hdr + 1); - uint32_t uniqueId = p->UniqueID; - boost::shared_ptr mqe; - - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(uniqueId); - - if (map_tok == fSessionMessages.end()) + if (index < fPmConnections.size()) { - // For debugging... - //cerr << "DistributedEngineComm::AddDataToOutput: tried to add a message to a dead session: " << uniqueId << ", size " << sbs->length() << ", step id " << p->StepID << endl; - return; + alarmItem = fPmConnections[index]->addr2String(); } - mqe = map_tok->second; - lk.unlock(); - - if (pmCount > 0) - { - (void)atomicops::atomicInc(&mqe->unackedWork[connIndex % pmCount]); - } - - TSQSize_t queueSize = mqe->queue.push(sbs); - - if (mqe->sendACKs) - { - boost::mutex::scoped_lock lk(ackLock); - uint64_t msgSize = sbs->lengthWithHdrOverhead(); - - if (!mqe->throttled && msgSize > (targetRecvQueueSize / 2)) - doHasBigMsgs(mqe, (300 * 1024 * 1024 > 3 * msgSize ? - 300 * 1024 * 1024 : 3 * msgSize)); //buffer at least 3 big msgs - - if (!mqe->throttled && queueSize.size >= mqe->targetQueueSize) - setFlowControl(true, uniqueId, mqe); - } - - if (stats) - mqe->stats.dataRecvd(stats->dataRecvd()); -} - -void DistributedEngineComm::doHasBigMsgs(boost::shared_ptr mqe, uint64_t targetSize) -{ - mqe->hasBigMsgs = true; - - if (mqe->targetQueueSize < targetSize) - mqe->targetQueueSize = targetSize; -} - -int DistributedEngineComm::writeToClient(size_t aPMIndex, const ByteStream& bs, uint32_t senderUniqueID, bool doInterleaving) -{ - boost::mutex::scoped_lock lk(fMlock, boost::defer_lock_t()); - MessageQueueMap::iterator it; - // Keep mqe's stats from being freed early - boost::shared_ptr mqe; - Stats* senderStats = NULL; - - if (fPmConnections.size() == 0) - return 0; - - uint32_t connectionId = aPMIndex; - if (senderUniqueID != numeric_limits::max()) - { - lk.lock(); - it = fSessionMessages.find(senderUniqueID); - - if (it != fSessionMessages.end()) - { - mqe = it->second; - senderStats = &(mqe->stats); - size_t pmIndex = aPMIndex % mqe->pmCount; - connectionId = it->second->getNextConnectionId(pmIndex, - fPmConnections.size(), - fDECConnectionsPerQuery); - } - - lk.unlock(); - } - - try - { - ClientList::value_type client = fPmConnections[connectionId]; - - if (!client->isAvailable()) return 0; - - boost::mutex::scoped_lock lk(*(fWlock[connectionId])); - client->write(bs, NULL, senderStats); - return 0; - } - catch (...) - { - // @bug 488. error out under such condition instead of re-trying other connection, - // by pushing 0 size bytestream to messagequeue and throw exception - SBS sbs; - lk.lock(); - //std::cout << "WARNING: DEC WRITE BROKEN PIPE. PMS index = " << index << std::endl; - MessageQueueMap::iterator map_tok; - sbs.reset(new ByteStream(0)); - - for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok) - { - map_tok->second->queue.clear(); - (void)atomicops::atomicInc(&map_tok->second->unackedWork[0]); - map_tok->second->queue.push(sbs); - } - - lk.unlock(); - /* - // reconfig the connection array - ClientList tempConns; - { - //cout << "WARNING: DEC WRITE BROKEN PIPE " << fPmConnections[index]->otherEnd()<< endl; - boost::mutex::scoped_lock onErrLock(fOnErrMutex); - string moduleName = fPmConnections[index]->moduleName(); - //cout << "module name = " << moduleName << endl; - if (index >= fPmConnections.size()) return 0; - - for (uint32_t i = 0; i < fPmConnections.size(); i++) - { - if (moduleName != fPmConnections[i]->moduleName()) - tempConns.push_back(fPmConnections[i]); - } - if (tempConns.size() == fPmConnections.size()) return 0; - fPmConnections.swap(tempConns); - pmCount = (pmCount == 0 ? 0 : pmCount - 1); - } - // send alarm - ALARMManager alarmMgr; - string alarmItem("UNKNOWN"); - - if (index < fPmConnections.size()) - { - alarmItem = fPmConnections[index]->addr2String(); - } - - alarmItem.append(" PrimProc"); - alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET); - */ - throw runtime_error("DistributedEngineComm::write: Broken Pipe error"); - } + alarmItem.append(" PrimProc"); + alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET); + */ + throw runtime_error("DistributedEngineComm::write: Broken Pipe error"); + } } uint32_t DistributedEngineComm::size(uint32_t key) { - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - if (map_tok == fSessionMessages.end()) - throw runtime_error("DEC::size() attempt to get the size of a nonexistant queue!"); + if (map_tok == fSessionMessages.end()) + throw runtime_error("DEC::size() attempt to get the size of a nonexistant queue!"); - boost::shared_ptr mqe = map_tok->second; - //TODO: should probably check that this is a valid iter... - lk.unlock(); - return mqe->queue.size().count; + boost::shared_ptr mqe = map_tok->second; + // TODO: should probably check that this is a valid iter... + lk.unlock(); + return mqe->queue.size().count; } void DistributedEngineComm::addDECEventListener(DECEventListener* l) { - boost::mutex::scoped_lock lk(eventListenerLock); - eventListeners.push_back(l); + boost::mutex::scoped_lock lk(eventListenerLock); + eventListeners.push_back(l); } void DistributedEngineComm::removeDECEventListener(DECEventListener* l) { - boost::mutex::scoped_lock lk(eventListenerLock); - std::vector newListeners; - uint32_t s = eventListeners.size(); + boost::mutex::scoped_lock lk(eventListenerLock); + std::vector newListeners; + uint32_t s = eventListeners.size(); - for (uint32_t i = 0; i < s; i++) - if (eventListeners[i] != l) - newListeners.push_back(eventListeners[i]); + for (uint32_t i = 0; i < s; i++) + if (eventListeners[i] != l) + newListeners.push_back(eventListeners[i]); - eventListeners.swap(newListeners); + eventListeners.swap(newListeners); } Stats DistributedEngineComm::getNetworkStats(uint32_t uniqueID) { - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator it; - Stats empty; + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator it; + Stats empty; - it = fSessionMessages.find(uniqueID); + it = fSessionMessages.find(uniqueID); - if (it != fSessionMessages.end()) - return it->second->stats; + if (it != fSessionMessages.end()) + return it->second->stats; - return empty; + return empty; } DistributedEngineComm::MQE::MQE(const uint32_t pCount, const uint32_t initialInterleaverValue) - : ackSocketIndex(0), pmCount(pCount), hasBigMsgs(false), targetQueueSize(targetRecvQueueSize) + : ackSocketIndex(0), pmCount(pCount), hasBigMsgs(false), targetQueueSize(targetRecvQueueSize) { - unackedWork.reset(new volatile uint32_t[pmCount]); - interleaver.reset(new uint32_t[pmCount]); - memset((void*) unackedWork.get(), 0, pmCount * sizeof(uint32_t)); - uint32_t interleaverValue = initialInterleaverValue; - initialConnectionId = initialInterleaverValue; - for (size_t pmId = 0; pmId < pmCount; ++pmId) - { - interleaver.get()[pmId] = interleaverValue++; - } + unackedWork.reset(new volatile uint32_t[pmCount]); + interleaver.reset(new uint32_t[pmCount]); + memset((void*)unackedWork.get(), 0, pmCount * sizeof(uint32_t)); + uint32_t interleaverValue = initialInterleaverValue; + initialConnectionId = initialInterleaverValue; + for (size_t pmId = 0; pmId < pmCount; ++pmId) + { + interleaver.get()[pmId] = interleaverValue++; + } } // Here is the assumed connections distribution schema in a pmConnections vector. @@ -1147,20 +1139,20 @@ uint32_t DistributedEngineComm::MQE::getNextConnectionId(const size_t pmIndex, const uint32_t DECConnectionsPerQuery) { - uint32_t nextConnectionId = (interleaver[pmIndex] + pmCount) % pmConnectionsNumber; - // Wrap around the connection id. - if ((nextConnectionId - pmIndex) % DECConnectionsPerQuery == 0) - { - // The sum must be < connections vector size. - nextConnectionId = initialConnectionId + pmIndex; - interleaver[pmIndex] = nextConnectionId; - } - else - { - interleaver[pmIndex] = nextConnectionId; - } - return nextConnectionId; + uint32_t nextConnectionId = (interleaver[pmIndex] + pmCount) % pmConnectionsNumber; + // Wrap around the connection id. + if ((nextConnectionId - pmIndex) % DECConnectionsPerQuery == 0) + { + // The sum must be < connections vector size. + nextConnectionId = initialConnectionId + pmIndex; + interleaver[pmIndex] = nextConnectionId; + } + else + { + interleaver[pmIndex] = nextConnectionId; + } + return nextConnectionId; } -} +} // namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/distributedenginecomm.h b/dbcon/joblist/distributedenginecomm.h index 01f2cf194..3d473250b 100644 --- a/dbcon/joblist/distributedenginecomm.h +++ b/dbcon/joblist/distributedenginecomm.h @@ -71,14 +71,13 @@ class Config; */ namespace joblist { - class DECEventListener { -public: - virtual ~DECEventListener() { }; + public: + virtual ~DECEventListener(){}; - /* Do whatever needs to be done to init the new PM */ - virtual void newPMOnline(uint32_t newConnectionNumber) = 0; + /* Do whatever needs to be done to init the new PM */ + virtual void newPMOnline(uint32_t newConnectionNumber) = 0; }; /** @@ -86,224 +85,222 @@ public: */ class DistributedEngineComm { -public: - /** - * Constructors - */ - EXPORT virtual ~DistributedEngineComm(); + public: + /** + * Constructors + */ + EXPORT virtual ~DistributedEngineComm(); - EXPORT static DistributedEngineComm* instance(ResourceManager* rm, bool isExeMgr = false); + EXPORT static DistributedEngineComm* instance(ResourceManager* rm, bool isExeMgr = false); - /** @brief delete the static instance - * This has the effect of causing the connection to be rebuilt - */ - EXPORT static void reset(); + /** @brief delete the static instance + * This has the effect of causing the connection to be rebuilt + */ + EXPORT static void reset(); - /** @brief currently a nop - * - */ - EXPORT int Open() - { - return 0; - } + /** @brief currently a nop + * + */ + EXPORT int Open() + { + return 0; + } - EXPORT void addQueue(uint32_t key, bool sendACKs = false); - EXPORT void removeQueue(uint32_t key); - EXPORT void shutdownQueue(uint32_t key); + EXPORT void addQueue(uint32_t key, bool sendACKs = false); + EXPORT void removeQueue(uint32_t key); + EXPORT void shutdownQueue(uint32_t key); - /** @brief read a primitive response - * - * Returns the next message in the inbound queue for session sessionId and step stepId. - * @todo See if we can't save a copy by returning a const& - */ + /** @brief read a primitive response + * + * Returns the next message in the inbound queue for session sessionId and step stepId. + * @todo See if we can't save a copy by returning a const& + */ - EXPORT const messageqcpp::ByteStream read(uint32_t key); - /** @brief read a primitve response - * - * Returns the next message in the inbound queue for session sessionId and step stepId. - * @param bs A pointer to the ByteStream to fill in. - * @note: saves a copy vs read(uint32_t, uint32_t). - */ - EXPORT void read(uint32_t key, messageqcpp::SBS&); + EXPORT const messageqcpp::ByteStream read(uint32_t key); + /** @brief read a primitve response + * + * Returns the next message in the inbound queue for session sessionId and step stepId. + * @param bs A pointer to the ByteStream to fill in. + * @note: saves a copy vs read(uint32_t, uint32_t). + */ + EXPORT void read(uint32_t key, messageqcpp::SBS&); - /** @brief read a primitve response - * - * Returns the next message in the inbound queue for session sessionId and step stepId. - */ - EXPORT void read_all(uint32_t key, std::vector& v); + /** @brief read a primitve response + * + * Returns the next message in the inbound queue for session sessionId and step stepId. + */ + EXPORT void read_all(uint32_t key, std::vector& v); - /** reads queuesize/divisor msgs */ - EXPORT void read_some(uint32_t key, uint32_t divisor, std::vector& v, - bool* flowControlOn = NULL); + /** reads queuesize/divisor msgs */ + EXPORT void read_some(uint32_t key, uint32_t divisor, std::vector& v, + bool* flowControlOn = NULL); - /** @brief Write a primitive message - * - * Writes a primitive message to a primitive server. Msg needs to conatin an ISMPacketHeader. The - * LBID is extracted from the ISMPacketHeader and used to determine the actual P/M to send to. - */ - EXPORT void write(uint32_t key, messageqcpp::ByteStream& msg); + /** @brief Write a primitive message + * + * Writes a primitive message to a primitive server. Msg needs to conatin an ISMPacketHeader. The + * LBID is extracted from the ISMPacketHeader and used to determine the actual P/M to send to. + */ + EXPORT void write(uint32_t key, messageqcpp::ByteStream& msg); - //EXPORT void throttledWrite(const messageqcpp::ByteStream& msg); + // EXPORT void throttledWrite(const messageqcpp::ByteStream& msg); - /** @brief Special write function for use only by newPMOnline event handlers - */ - EXPORT void write(messageqcpp::ByteStream& msg, uint32_t connection); + /** @brief Special write function for use only by newPMOnline event handlers + */ + EXPORT void write(messageqcpp::ByteStream& msg, uint32_t connection); - /** @brief Shutdown this object - * - * Closes all the connections created during Setup() and cleans up other stuff. - */ - EXPORT int Close(); + /** @brief Shutdown this object + * + * Closes all the connections created during Setup() and cleans up other stuff. + */ + EXPORT int Close(); - /** @brief Start listening for primitive responses - * - * Starts the current thread listening on the client socket for primitive response messages. Will not return - * until busy() returns false or a zero-length response is received. - */ - EXPORT void Listen(boost::shared_ptr client, uint32_t connIndex); + /** @brief Start listening for primitive responses + * + * Starts the current thread listening on the client socket for primitive response messages. Will not return + * until busy() returns false or a zero-length response is received. + */ + EXPORT void Listen(boost::shared_ptr client, uint32_t connIndex); - /** @brief set/unset busy flag - * - * Set or unset the busy flag so Listen() can return. - */ - EXPORT void makeBusy(bool b) - { - fBusy = b; - } + /** @brief set/unset busy flag + * + * Set or unset the busy flag so Listen() can return. + */ + EXPORT void makeBusy(bool b) + { + fBusy = b; + } - /** @brief fBusy accessor - * - */ - EXPORT bool Busy() const - { - return fBusy; - } + /** @brief fBusy accessor + * + */ + EXPORT bool Busy() const + { + return fBusy; + } - /** @brief Returns the size of the queue for the specified jobstep - */ - EXPORT uint32_t size(uint32_t key); + /** @brief Returns the size of the queue for the specified jobstep + */ + EXPORT uint32_t size(uint32_t key); - EXPORT void Setup(); + EXPORT void Setup(); - EXPORT void addDECEventListener(DECEventListener*); - EXPORT void removeDECEventListener(DECEventListener*); + EXPORT void addDECEventListener(DECEventListener*); + EXPORT void removeDECEventListener(DECEventListener*); - uint64_t connectedPmServers() const - { - return fPmConnections.size(); - } - uint32_t getPmCount() const - { - return pmCount; - } + uint64_t connectedPmServers() const + { + return fPmConnections.size(); + } + uint32_t getPmCount() const + { + return pmCount; + } - unsigned getNumConnections() const - { - unsigned cpp = (fIsExeMgr ? fRm->getPsConnectionsPerPrimProc() : 1); - return fRm->getPsCount() * cpp; - } + unsigned getNumConnections() const + { + unsigned cpp = (fIsExeMgr ? fRm->getPsConnectionsPerPrimProc() : 1); + return fRm->getPsCount() * cpp; + } - messageqcpp::Stats getNetworkStats(uint32_t uniqueID); + messageqcpp::Stats getNetworkStats(uint32_t uniqueID); - friend class ::TestDistributedEngineComm; + friend class ::TestDistributedEngineComm; -private: - typedef std::vector ReaderList; - typedef std::vector > ClientList; + private: + typedef std::vector ReaderList; + typedef std::vector > ClientList; - //A queue of ByteStreams coming in from PrimProc heading for a JobStep - typedef ThreadSafeQueue StepMsgQueue; + // A queue of ByteStreams coming in from PrimProc heading for a JobStep + typedef ThreadSafeQueue StepMsgQueue; - /* To keep some state associated with the connection. These aren't copyable. */ - struct MQE : public boost::noncopyable - { - MQE(const uint32_t pmCount, const uint32_t initialInterleaverValue); - uint32_t getNextConnectionId(const size_t pmIndex, - const size_t pmConnectionsNumber, - const uint32_t DECConnectionsPerQuery); - messageqcpp::Stats stats; - StepMsgQueue queue; - uint32_t ackSocketIndex; - boost::scoped_array unackedWork; - boost::scoped_array interleaver; - uint32_t initialConnectionId; - uint32_t pmCount; - // non-BPP primitives don't do ACKs - bool sendACKs; + /* To keep some state associated with the connection. These aren't copyable. */ + struct MQE : public boost::noncopyable + { + MQE(const uint32_t pmCount, const uint32_t initialInterleaverValue); + uint32_t getNextConnectionId(const size_t pmIndex, const size_t pmConnectionsNumber, + const uint32_t DECConnectionsPerQuery); + messageqcpp::Stats stats; + StepMsgQueue queue; + uint32_t ackSocketIndex; + boost::scoped_array unackedWork; + boost::scoped_array interleaver; + uint32_t initialConnectionId; + uint32_t pmCount; + // non-BPP primitives don't do ACKs + bool sendACKs; - // This var will allow us to toggle flow control for BPP instances when - // the UM is keeping up. Send -1 as the ACK value to disable flow control - // on the PM side, positive value to reenable it. Not yet impl'd on the UM side. - bool throttled; + // This var will allow us to toggle flow control for BPP instances when + // the UM is keeping up. Send -1 as the ACK value to disable flow control + // on the PM side, positive value to reenable it. Not yet impl'd on the UM side. + bool throttled; - // This var signifies that the PM can return msgs big enough to keep toggling - // FC on and off. We force FC on in that case and maintain a larger buffer. - bool hasBigMsgs; + // This var signifies that the PM can return msgs big enough to keep toggling + // FC on and off. We force FC on in that case and maintain a larger buffer. + bool hasBigMsgs; - uint64_t targetQueueSize; - }; + uint64_t targetQueueSize; + }; - //The mapping of session ids to StepMsgQueueLists - typedef std::map > MessageQueueMap; + // The mapping of session ids to StepMsgQueueLists + typedef std::map > MessageQueueMap; - explicit DistributedEngineComm(ResourceManager* rm, bool isExeMgr); + explicit DistributedEngineComm(ResourceManager* rm, bool isExeMgr); - void StartClientListener(boost::shared_ptr cl, uint32_t connIndex); + void StartClientListener(boost::shared_ptr cl, uint32_t connIndex); - /** @brief Add a message to the queue - * - */ - void addDataToOutput(messageqcpp::SBS, uint32_t connIndex, messageqcpp::Stats* statsToAdd); + /** @brief Add a message to the queue + * + */ + void addDataToOutput(messageqcpp::SBS, uint32_t connIndex, messageqcpp::Stats* statsToAdd); - /** @brief Writes data to the client at the index - * - * Continues trying to write data to the client at the next index until all clients have been tried. - */ - int writeToClient(size_t index, const messageqcpp::ByteStream& bs, - uint32_t senderID = std::numeric_limits::max(), bool doInterleaving = false); + /** @brief Writes data to the client at the index + * + * Continues trying to write data to the client at the next index until all clients have been tried. + */ + int writeToClient(size_t index, const messageqcpp::ByteStream& bs, + uint32_t senderID = std::numeric_limits::max(), bool doInterleaving = false); - static DistributedEngineComm* fInstance; - ResourceManager* fRm; + static DistributedEngineComm* fInstance; + ResourceManager* fRm; - ClientList fPmConnections; // all the pm servers - ReaderList fPmReader; // all the reader threads for the pm servers - MessageQueueMap fSessionMessages; // place to put messages from the pm server to be returned by the Read method - boost::mutex fMlock; //sessionMessages mutex - std::vector > fWlock; //PrimProc socket write mutexes - bool fBusy; - unsigned fLBIDShift; - volatile uint32_t pmCount; - boost::mutex fOnErrMutex; // to lock function scope to reset pmconnections under error condition - boost::mutex fSetupMutex; + ClientList fPmConnections; // all the pm servers + ReaderList fPmReader; // all the reader threads for the pm servers + MessageQueueMap + fSessionMessages; // place to put messages from the pm server to be returned by the Read method + boost::mutex fMlock; // sessionMessages mutex + std::vector > fWlock; // PrimProc socket write mutexes + bool fBusy; + unsigned fLBIDShift; + volatile uint32_t pmCount; + boost::mutex fOnErrMutex; // to lock function scope to reset pmconnections under error condition + boost::mutex fSetupMutex; - // event listener data - std::vector eventListeners; - boost::mutex eventListenerLock; + // event listener data + std::vector eventListeners; + boost::mutex eventListenerLock; - ClientList newClients; - std::vector > newLocks; + ClientList newClients; + std::vector > newLocks; - bool fIsExeMgr; + bool fIsExeMgr; - // send-side throttling vars - uint64_t throttleThreshold; - static const uint32_t targetRecvQueueSize = 50000000; - static const uint32_t disableThreshold = 10000000; - uint32_t tbpsThreadCount; - uint32_t fDECConnectionsPerQuery; - - void sendAcks(uint32_t uniqueID, const std::vector& msgs, - boost::shared_ptr mqe, size_t qSize); - void nextPMToACK(boost::shared_ptr mqe, uint32_t maxAck, uint32_t* sockIndex, - uint16_t* numToAck); - void setFlowControl(bool enable, uint32_t uniqueID, boost::shared_ptr mqe); - void doHasBigMsgs(boost::shared_ptr mqe, uint64_t targetSize); - boost::mutex ackLock; + // send-side throttling vars + uint64_t throttleThreshold; + static const uint32_t targetRecvQueueSize = 50000000; + static const uint32_t disableThreshold = 10000000; + uint32_t tbpsThreadCount; + uint32_t fDECConnectionsPerQuery; + void sendAcks(uint32_t uniqueID, const std::vector& msgs, boost::shared_ptr mqe, + size_t qSize); + void nextPMToACK(boost::shared_ptr mqe, uint32_t maxAck, uint32_t* sockIndex, uint16_t* numToAck); + void setFlowControl(bool enable, uint32_t uniqueID, boost::shared_ptr mqe); + void doHasBigMsgs(boost::shared_ptr mqe, uint64_t targetSize); + boost::mutex ackLock; }; -} +} // namespace joblist #undef EXPORT -#endif // DISTENGINECOMM_H +#endif // DISTENGINECOMM_H diff --git a/dbcon/joblist/elementcompression.h b/dbcon/joblist/elementcompression.h index 669d48103..f24613dfc 100644 --- a/dbcon/joblist/elementcompression.h +++ b/dbcon/joblist/elementcompression.h @@ -33,14 +33,13 @@ #include "elementtype.h" #ifndef __GNUC__ -# ifndef __attribute__ -# define __attribute__(x) -# endif +#ifndef __attribute__ +#define __attribute__(x) +#endif #endif namespace joblist { - //------------------------------------------------------------------------------ // Structs used for storing compressed element types externally to disk. // Note the use of "packed" to keep the compiler from padding the struct. @@ -49,25 +48,25 @@ namespace joblist //------------------------------------------------------------------------------ struct CompElement64Rid32Val { - uint64_t first; - uint32_t second; + uint64_t first; + uint32_t second; } __attribute__((__packed__)); struct CompElement32Rid64Val { - uint32_t first; - uint64_t second; + uint32_t first; + uint64_t second; } __attribute__((__packed__)); struct CompElement32Rid32Val { - uint32_t first; - uint32_t second; + uint32_t first; + uint32_t second; }; struct CompElement32RidOnly { - uint32_t first; + uint32_t first; }; //------------------------------------------------------------------------------ @@ -79,314 +78,186 @@ struct CompElement32RidOnly //------------------------------------------------------------------------------ class ElementCompression { -public: - // - //...Utilities to compress from 64 bit to 32 bit for RID and/or Value - // - template - static void compress( - std::vector& vIn, - std::vector& vOut); - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of ElementType to 32RidOnly not supported"); - } + public: + // + //...Utilities to compress from 64 bit to 32 bit for RID and/or Value + // + template + static void compress(std::vector& vIn, std::vector& vOut); + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of ElementType to 32RidOnly not supported"); + } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of StringElementType to 64/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of StringElementType to 32/64 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of StringElementType to 32/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of StringElementType to 32RidOnly not supported"); - } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of StringElementType to 64/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of StringElementType to 32/64 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of StringElementType to 32/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of StringElementType to 32RidOnly not supported"); + } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of DoubleElementType to 64/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of DoubleElementType to 32/64 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of DoubleElementType to 32/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of DoubleElementType to 32RidOnly not supported"); - } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of DoubleElementType to 64/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of DoubleElementType to 32/64 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of DoubleElementType to 32/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of DoubleElementType to 32RidOnly not supported"); + } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of RIDElementType to 64/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of RIDElementType to 32/64 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of RIDElementType to 32/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut); + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of RIDElementType to 64/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of RIDElementType to 32/64 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of RIDElementType to 32/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut); - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of TupleType to 64/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of TupleType to 32/64 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of TupleType to 32/32 not supported"); - } - static void compress( - std::vector& vIn, - std::vector& vOut) - { - throw std::logic_error( - "Compression of TupleType to 32/32 not supported"); - } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of TupleType to 64/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of TupleType to 32/64 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of TupleType to 32/32 not supported"); + } + static void compress(std::vector& vIn, std::vector& vOut) + { + throw std::logic_error("Compression of TupleType to 32/32 not supported"); + } - // - //...Utilities to expand from 32 bit to 64 bit for RID and/or Value - // - template - static void expand( - std::vector& vIn, - ElementType* vOut); - static void expand( - std::vector& vIn, - ElementType* vOut) - { - throw std::logic_error( - "Expansion to ElementType from 32RidOnly not supported"); - } + // + //...Utilities to expand from 32 bit to 64 bit for RID and/or Value + // + template + static void expand(std::vector& vIn, ElementType* vOut); + static void expand(std::vector& vIn, ElementType* vOut) + { + throw std::logic_error("Expansion to ElementType from 32RidOnly not supported"); + } - static void expand( - std::vector& vIn, - StringElementType* vOut) - { - throw std::logic_error( - "Expansion to StringElementType from 64/32 not supported"); - } - static void expand( - std::vector& vIn, - StringElementType* vOut) - { - throw std::logic_error( - "Expansion to StringElementType from 32/64 not supported"); - } - static void expand( - std::vector& vIn, - StringElementType* vOut) - { - throw std::logic_error( - "Expansion to StringElementType from 32/32 not supported"); - } - static void expand( - std::vector& vIn, - StringElementType* vOut) - { - throw std::logic_error( - "Expansion to StringElementType from 32RidOnly not supported"); - } + static void expand(std::vector& vIn, StringElementType* vOut) + { + throw std::logic_error("Expansion to StringElementType from 64/32 not supported"); + } + static void expand(std::vector& vIn, StringElementType* vOut) + { + throw std::logic_error("Expansion to StringElementType from 32/64 not supported"); + } + static void expand(std::vector& vIn, StringElementType* vOut) + { + throw std::logic_error("Expansion to StringElementType from 32/32 not supported"); + } + static void expand(std::vector& vIn, StringElementType* vOut) + { + throw std::logic_error("Expansion to StringElementType from 32RidOnly not supported"); + } - static void expand( - std::vector& vIn, - DoubleElementType* vOut) - { - throw std::logic_error( - "Expansion to DoubleElementType from 64/32 not supported"); - } - static void expand( - std::vector& vIn, - DoubleElementType* vOut) - { - throw std::logic_error( - "Expansion to DoubleElementType from 32/64 not supported"); - } - static void expand( - std::vector& vIn, - DoubleElementType* vOut) - { - throw std::logic_error( - "Expansion to DoubleElementType from 32/32 not supported"); - } - static void expand( - std::vector& vIn, - DoubleElementType* vOut) - { - throw std::logic_error( - "Expansion to DoubleElementType from 32RidOnly not supported"); - } + static void expand(std::vector& vIn, DoubleElementType* vOut) + { + throw std::logic_error("Expansion to DoubleElementType from 64/32 not supported"); + } + static void expand(std::vector& vIn, DoubleElementType* vOut) + { + throw std::logic_error("Expansion to DoubleElementType from 32/64 not supported"); + } + static void expand(std::vector& vIn, DoubleElementType* vOut) + { + throw std::logic_error("Expansion to DoubleElementType from 32/32 not supported"); + } + static void expand(std::vector& vIn, DoubleElementType* vOut) + { + throw std::logic_error("Expansion to DoubleElementType from 32RidOnly not supported"); + } - static void expand( - std::vector& vIn, - RIDElementType* vOut) - { - throw std::logic_error( - "Expansion to RIDElementType from 64/32 not supported"); - } - static void expand( - std::vector& vIn, - RIDElementType* vOut) - { - throw std::logic_error( - "Expansion to RIDElementType from 32/64 not supported"); - } - static void expand( - std::vector& vIn, - RIDElementType* vOut) - { - throw std::logic_error( - "Expansion to RIDElementType from 32/32 not supported"); - } - static void expand( - std::vector& vIn, - RIDElementType* vOut); + static void expand(std::vector& vIn, RIDElementType* vOut) + { + throw std::logic_error("Expansion to RIDElementType from 64/32 not supported"); + } + static void expand(std::vector& vIn, RIDElementType* vOut) + { + throw std::logic_error("Expansion to RIDElementType from 32/64 not supported"); + } + static void expand(std::vector& vIn, RIDElementType* vOut) + { + throw std::logic_error("Expansion to RIDElementType from 32/32 not supported"); + } + static void expand(std::vector& vIn, RIDElementType* vOut); - static void expand( - std::vector& vIn, - TupleType* vOut) - { - throw std::logic_error( - "Expansion to TupleType from 64/32 not supported"); - } - static void expand( - std::vector& vIn, - TupleType* vOut) - { - throw std::logic_error( - "Expansion to TupleType from 32/64 not supported"); - } - static void expand( - std::vector& vIn, - TupleType* vOut) - { - throw std::logic_error( - "Expansion to TupleType from 32/32 not supported"); - } - static void expand( - std::vector& vIn, - TupleType* vOut) - { - throw std::logic_error( - "Expansion to TupleType from 32/32 not supported"); - } + static void expand(std::vector& vIn, TupleType* vOut) + { + throw std::logic_error("Expansion to TupleType from 64/32 not supported"); + } + static void expand(std::vector& vIn, TupleType* vOut) + { + throw std::logic_error("Expansion to TupleType from 32/64 not supported"); + } + static void expand(std::vector& vIn, TupleType* vOut) + { + throw std::logic_error("Expansion to TupleType from 32/32 not supported"); + } + static void expand(std::vector& vIn, TupleType* vOut) + { + throw std::logic_error("Expansion to TupleType from 32/32 not supported"); + } - // - //...Utilities to write a single element with a compressed 32 bit RID. - // - static void writeWith32Rid( - const ElementType& e, - std::fstream& fFile); - static void writeWith32Rid( - const DoubleElementType& e, - std::fstream& fFile) - { - throw std::logic_error( - "Compress/Write of 32 RID DoubleElementType not supported"); - } - static void writeWith32Rid( - const StringElementType& e, - std::fstream& fFile); - static void writeWith32Rid( - const RIDElementType& e, - std::fstream& fFile); - static void writeWith32Rid( - const TupleType& e, - std::fstream& fFile) - { - throw std::logic_error( - "Compress/Write of 32 RID TupleType not supported"); - } + // + //...Utilities to write a single element with a compressed 32 bit RID. + // + static void writeWith32Rid(const ElementType& e, std::fstream& fFile); + static void writeWith32Rid(const DoubleElementType& e, std::fstream& fFile) + { + throw std::logic_error("Compress/Write of 32 RID DoubleElementType not supported"); + } + static void writeWith32Rid(const StringElementType& e, std::fstream& fFile); + static void writeWith32Rid(const RIDElementType& e, std::fstream& fFile); + static void writeWith32Rid(const TupleType& e, std::fstream& fFile) + { + throw std::logic_error("Compress/Write of 32 RID TupleType not supported"); + } - // - //...Utilities to read a single element with a compressed 32 bit RID. - // - static void readWith32Rid( - ElementType& e, - std::fstream& fFile); - static void readWith32Rid( - DoubleElementType& e, - std::fstream& fFile) - { - throw std::logic_error( - "Read/Expand of 32 RID DoubleElementType not supported"); - } - static void readWith32Rid( - StringElementType& e, - std::fstream& fFile); - static void readWith32Rid( - RIDElementType& e, - std::fstream& fFile); - static void readWith32Rid( - TupleType& e, - std::fstream& fFile) - { - throw std::logic_error( - "Read/Expand of 32 RID TupleType not supported"); - } + // + //...Utilities to read a single element with a compressed 32 bit RID. + // + static void readWith32Rid(ElementType& e, std::fstream& fFile); + static void readWith32Rid(DoubleElementType& e, std::fstream& fFile) + { + throw std::logic_error("Read/Expand of 32 RID DoubleElementType not supported"); + } + static void readWith32Rid(StringElementType& e, std::fstream& fFile); + static void readWith32Rid(RIDElementType& e, std::fstream& fFile); + static void readWith32Rid(TupleType& e, std::fstream& fFile) + { + throw std::logic_error("Read/Expand of 32 RID TupleType not supported"); + } }; //------------------------------------------------------------------------------ @@ -399,18 +270,16 @@ public: //...Compress RID/Value to 32 bit RID, 32 bit value // template -/* static */ inline void -ElementCompression::compress( - std::vector& vIn, - std::vector& vOut) +/* static */ inline void ElementCompression::compress(std::vector& vIn, + std::vector& vOut) { - uint64_t count = vIn.size(); + uint64_t count = vIn.size(); - for (unsigned int i = 0; i < count; i++) - { - vOut[i].first = vIn[i].first; - vOut[i].second = vIn[i].second; - } + for (unsigned int i = 0; i < count; i++) + { + vOut[i].first = vIn[i].first; + vOut[i].second = vIn[i].second; + } } // @@ -419,17 +288,15 @@ ElementCompression::compress( //...sion of compress() because this version does not access or copy the //..."second" data member. // -/* static */ inline void -ElementCompression::compress( - std::vector& vIn, - std::vector& vOut) +/* static */ inline void ElementCompression::compress(std::vector& vIn, + std::vector& vOut) { - uint64_t count = vIn.size(); + uint64_t count = vIn.size(); - for (unsigned int i = 0; i < count; i++) - { - vOut[i].first = vIn[i].first; - } + for (unsigned int i = 0; i < count; i++) + { + vOut[i].first = vIn[i].first; + } } //------------------------------------------------------------------------------ @@ -442,18 +309,15 @@ ElementCompression::compress( //...Expand RID/Value from 32 bit RID, 32 bit value // template -/* static */ inline void -ElementCompression::expand( - std::vector& vIn, - ElementType* vOut) +/* static */ inline void ElementCompression::expand(std::vector& vIn, ElementType* vOut) { - uint64_t count = vIn.size(); + uint64_t count = vIn.size(); - for (unsigned int i = 0; i < count; i++) - { - vOut[i].first = vIn[i].first; - vOut[i].second = vIn[i].second; - } + for (unsigned int i = 0; i < count; i++) + { + vOut[i].first = vIn[i].first; + vOut[i].second = vIn[i].second; + } } // @@ -462,99 +326,79 @@ ElementCompression::expand( //...sion of expand() because this version does not access or copy the //..."second" data member. // -/* static */ inline void -ElementCompression::expand( - std::vector& vIn, - RIDElementType* vOut) +/* static */ inline void ElementCompression::expand(std::vector& vIn, + RIDElementType* vOut) { - uint64_t count = vIn.size(); + uint64_t count = vIn.size(); - for (unsigned int i = 0; i < count; i++) - { - vOut[i].first = vIn[i].first; - } + for (unsigned int i = 0; i < count; i++) + { + vOut[i].first = vIn[i].first; + } } //------------------------------------------------------------------------------ // Inline utilities to write a single element with a compressed 32 bit RID. //------------------------------------------------------------------------------ -/* static */ inline void -ElementCompression::writeWith32Rid( - const ElementType& e, - std::fstream& fFile) +/* static */ inline void ElementCompression::writeWith32Rid(const ElementType& e, std::fstream& fFile) { - CompElement32Rid64Val eCompressed; - eCompressed.first = e.first; - eCompressed.second = e.second; - fFile.write((char*) &eCompressed, sizeof(CompElement32Rid64Val)); + CompElement32Rid64Val eCompressed; + eCompressed.first = e.first; + eCompressed.second = e.second; + fFile.write((char*)&eCompressed, sizeof(CompElement32Rid64Val)); } -/* static */ inline void -ElementCompression::writeWith32Rid( - const StringElementType& e, - std::fstream& fFile) +/* static */ inline void ElementCompression::writeWith32Rid(const StringElementType& e, std::fstream& fFile) { - uint32_t rid = e.first; - uint16_t dlen = e.second.length(); + uint32_t rid = e.first; + uint16_t dlen = e.second.length(); - fFile.write((char*)&rid, sizeof(rid) ); - fFile.write((char*)&dlen, sizeof(dlen)); - fFile.write( e.second.c_str(), dlen ); + fFile.write((char*)&rid, sizeof(rid)); + fFile.write((char*)&dlen, sizeof(dlen)); + fFile.write(e.second.c_str(), dlen); } -/* static */ inline void -ElementCompression::writeWith32Rid( - const RIDElementType& e, - std::fstream& fFile) +/* static */ inline void ElementCompression::writeWith32Rid(const RIDElementType& e, std::fstream& fFile) { - CompElement32RidOnly eCompressed; - eCompressed.first = e.first; - fFile.write((char*) &eCompressed, sizeof(CompElement32RidOnly)); + CompElement32RidOnly eCompressed; + eCompressed.first = e.first; + fFile.write((char*)&eCompressed, sizeof(CompElement32RidOnly)); } //------------------------------------------------------------------------------ // Inline utilities to read a single element with a compressed 32 bit RID. //------------------------------------------------------------------------------ -/* static */ inline void -ElementCompression::readWith32Rid( - ElementType& e, - std::fstream& fFile) +/* static */ inline void ElementCompression::readWith32Rid(ElementType& e, std::fstream& fFile) { - CompElement32Rid64Val eCompressed; - fFile.read((char*) &eCompressed, sizeof(CompElement32Rid64Val)); - e.first = eCompressed.first; - e.second = eCompressed.second; + CompElement32Rid64Val eCompressed; + fFile.read((char*)&eCompressed, sizeof(CompElement32Rid64Val)); + e.first = eCompressed.first; + e.second = eCompressed.second; } -/* static */ inline void -ElementCompression::readWith32Rid( - StringElementType& e, - std::fstream& fFile) +/* static */ inline void ElementCompression::readWith32Rid(StringElementType& e, std::fstream& fFile) { - uint32_t rid = 0; - uint16_t dlen = 0; - char d[32768]; + uint32_t rid = 0; + uint16_t dlen = 0; + char d[32768]; - fFile.read((char*)&rid, sizeof(rid) ); - fFile.read((char*)&dlen, sizeof(dlen)); - fFile.read( d, dlen ); + fFile.read((char*)&rid, sizeof(rid)); + fFile.read((char*)&dlen, sizeof(dlen)); + fFile.read(d, dlen); - e.first = rid; - e.second.assign(d, dlen); + e.first = rid; + e.second.assign(d, dlen); } -/* static */ inline void -ElementCompression::readWith32Rid( - RIDElementType& e, - std::fstream& fFile) +/* static */ inline void ElementCompression::readWith32Rid(RIDElementType& e, std::fstream& fFile) { - CompElement32RidOnly eCompressed; - fFile.read((char*) &eCompressed, sizeof(CompElement32RidOnly)); - e.first = eCompressed.first; + CompElement32RidOnly eCompressed; + fFile.read((char*)&eCompressed, sizeof(CompElement32RidOnly)); + e.first = eCompressed.first; } -} // end of joblist namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/elementtype.cpp b/dbcon/joblist/elementtype.cpp index 1c96b4caa..61134a83b 100644 --- a/dbcon/joblist/elementtype.cpp +++ b/dbcon/joblist/elementtype.cpp @@ -30,102 +30,100 @@ using namespace joblist; namespace joblist { +StringElementType::StringElementType(){}; -StringElementType::StringElementType() { }; +StringElementType::StringElementType(uint64_t f, const std::string& s) : first(f), second(s){}; -StringElementType::StringElementType(uint64_t f, const std::string& s) : first(f), second(s) { }; +DoubleElementType::DoubleElementType(){}; -DoubleElementType::DoubleElementType() { }; - -DoubleElementType::DoubleElementType(uint64_t f, double s) : first(f), second(s) { }; +DoubleElementType::DoubleElementType(uint64_t f, double s) : first(f), second(s){}; RIDElementType::RIDElementType() { - first = static_cast(-1); + first = static_cast(-1); }; -RIDElementType::RIDElementType(uint64_t f) : first(f) { }; +RIDElementType::RIDElementType(uint64_t f) : first(f){}; const string ElementType::toString() const { - ostringstream oss; - oss << first << '/' << second; - return oss.str(); + ostringstream oss; + oss << first << '/' << second; + return oss.str(); } istream& operator>>(istream& in, ElementType& rhs) { - in.read((char*) &rhs, sizeof(ElementType)); - return in; + in.read((char*)&rhs, sizeof(ElementType)); + return in; } ostream& operator<<(ostream& out, const ElementType& rhs) { - out.write((char*) &rhs, sizeof(ElementType)); - return out; + out.write((char*)&rhs, sizeof(ElementType)); + return out; } ostream& operator<<(std::ostream& out, const StringElementType& rhs) { - uint64_t r = rhs.first; - int16_t dlen = rhs.second.length(); + uint64_t r = rhs.first; + int16_t dlen = rhs.second.length(); - out.write((char*)&r, sizeof(r)); - out.write( (char*)&dlen, sizeof(dlen)); - out.write( rhs.second.c_str(), dlen); + out.write((char*)&r, sizeof(r)); + out.write((char*)&dlen, sizeof(dlen)); + out.write(rhs.second.c_str(), dlen); - return out; + return out; } istream& operator>>(std::istream& out, StringElementType& rhs) { - uint64_t r; - int16_t dlen; - char d[32768]; // atm 32k is the largest possible val for the length of strings stored + uint64_t r; + int16_t dlen; + char d[32768]; // atm 32k is the largest possible val for the length of strings stored - out.read((char*)&r, sizeof(r)); - out.read( (char*)&dlen, sizeof(dlen)); - out.read( (char*)&d, dlen); + out.read((char*)&r, sizeof(r)); + out.read((char*)&dlen, sizeof(dlen)); + out.read((char*)&d, dlen); - rhs.first = r; - rhs.second = string(d, dlen); + rhs.first = r; + rhs.second = string(d, dlen); - return out; + return out; } istream& operator>>(istream& in, DoubleElementType& rhs) { - in.read((char*) &rhs, sizeof(DoubleElementType)); - return in; + in.read((char*)&rhs, sizeof(DoubleElementType)); + return in; } ostream& operator<<(ostream& out, const DoubleElementType& rhs) { - out.write((char*) &rhs, sizeof(DoubleElementType)); - return out; + out.write((char*)&rhs, sizeof(DoubleElementType)); + return out; } istream& operator>>(istream& is, RIDElementType& dl) { - is.read((char*)&dl, sizeof(RIDElementType)); - return is; + is.read((char*)&dl, sizeof(RIDElementType)); + return is; } ostream& operator<<(ostream& os, const RIDElementType& dl) { - os.write((char*)&dl, sizeof(RIDElementType)); - return os; + os.write((char*)&dl, sizeof(RIDElementType)); + return os; } istream& operator>>(istream& is, TupleType& dl) { - throw std::logic_error("TupleType >> not implemented"); + throw std::logic_error("TupleType >> not implemented"); } ostream& operator<<(ostream& os, const TupleType& dl) { - throw std::logic_error("TupleType << not implemented"); + throw std::logic_error("TupleType << not implemented"); } -} // namespace joblist - +} // namespace joblist diff --git a/dbcon/joblist/elementtype.h b/dbcon/joblist/elementtype.h index 401f10bf4..0d1f6ed7c 100644 --- a/dbcon/joblist/elementtype.h +++ b/dbcon/joblist/elementtype.h @@ -32,51 +32,43 @@ #include #ifndef __GNUC__ -# ifndef __attribute__ -# define __attribute__(x) -# endif +#ifndef __attribute__ +#define __attribute__(x) +#endif #endif namespace joblist { - /** @brief struct ElementType * */ struct ElementType { - typedef uint64_t first_type; - typedef uint64_t second_type; - uint64_t first; - uint64_t second; + typedef uint64_t first_type; + typedef uint64_t second_type; + uint64_t first; + uint64_t second; - ElementType() : first(static_cast(-1)), - second(static_cast(-1)) - { }; - ElementType(uint64_t f, uint64_t s) : first(f), second(s) { }; + ElementType() : first(static_cast(-1)), second(static_cast(-1)){}; + ElementType(uint64_t f, uint64_t s) : first(f), second(s){}; - const char* getHashString(uint64_t mode, uint64_t* len) const + const char* getHashString(uint64_t mode, uint64_t* len) const + { + switch (mode) { - switch (mode) - { - case 0: - *len = 8; - return (char*) &first; + case 0: *len = 8; return (char*)&first; - case 1: - *len = 8; - return (char*) &second; + case 1: *len = 8; return (char*)&second; - default: - throw std::logic_error("ElementType: invalid mode in getHashString()."); - } + default: throw std::logic_error("ElementType: invalid mode in getHashString()."); } - inline bool operator<(const ElementType& e) const - { - return (first < e.first); - } - const std::string toString() const; + } + inline bool operator<(const ElementType& e) const + { + return (first < e.first); + } + const std::string toString() const; }; /** @brief struct StringElementType @@ -84,36 +76,30 @@ struct ElementType */ struct StringElementType { - typedef uint64_t first_type; - typedef std::string second_type; + typedef uint64_t first_type; + typedef std::string second_type; - uint64_t first; - std::string second; + uint64_t first; + std::string second; - StringElementType(); - StringElementType(uint64_t f, const std::string& s); + StringElementType(); + StringElementType(uint64_t f, const std::string& s); - const char* getHashString(uint64_t mode, uint64_t* len) const + const char* getHashString(uint64_t mode, uint64_t* len) const + { + switch (mode) { - switch (mode) - { - case 0: - *len = sizeof(first); - return (char*) &first; + case 0: *len = sizeof(first); return (char*)&first; - case 1: - *len = second.size(); - return (char*) second.data(); + case 1: *len = second.size(); return (char*)second.data(); - default: - throw std::logic_error("StringElementType: invalid mode in getHashString()."); - } + default: throw std::logic_error("StringElementType: invalid mode in getHashString()."); } - inline bool operator<(const StringElementType& e) const - { - return (first < e.first); - } - + } + inline bool operator<(const StringElementType& e) const + { + return (first < e.first); + } }; /** @brief struct DoubleElementType @@ -121,66 +107,60 @@ struct StringElementType */ struct DoubleElementType { + uint64_t first; + double second; - uint64_t first; - double second; + DoubleElementType(); + DoubleElementType(uint64_t f, double s); - DoubleElementType(); - DoubleElementType(uint64_t f, double s); - - typedef double second_type; - const char* getHashString(uint64_t mode, uint64_t* len) const + typedef double second_type; + const char* getHashString(uint64_t mode, uint64_t* len) const + { + switch (mode) { - switch (mode) - { - case 0: - *len = sizeof(first); - return (char*) &first; + case 0: *len = sizeof(first); return (char*)&first; - case 1: - *len = sizeof(second); - return (char*) &second; + case 1: *len = sizeof(second); return (char*)&second; - default: - throw std::logic_error("StringElementType: invalid mode in getHashString()."); - } - } - inline bool operator<(const DoubleElementType& e) const - { - return (first < e.first); + default: throw std::logic_error("StringElementType: invalid mode in getHashString()."); } + } + inline bool operator<(const DoubleElementType& e) const + { + return (first < e.first); + } }; -template +template struct RowWrapper { - uint64_t count; - static const uint64_t ElementsPerGroup = 8192; - element_t et[8192]; + uint64_t count; + static const uint64_t ElementsPerGroup = 8192; + element_t et[8192]; - RowWrapper(): count(0) - { - } + RowWrapper() : count(0) + { + } - inline RowWrapper(const RowWrapper& rg) : count(rg.count) - { - for (uint32_t i = 0; i < count; ++i) - et[i] = rg.et[i]; - } + inline RowWrapper(const RowWrapper& rg) : count(rg.count) + { + for (uint32_t i = 0; i < count; ++i) + et[i] = rg.et[i]; + } - ~RowWrapper() - { - } + ~RowWrapper() + { + } - inline RowWrapper& operator=(const RowWrapper& rg) - { - count = rg.count; + inline RowWrapper& operator=(const RowWrapper& rg) + { + count = rg.count; - for (uint32_t i = 0; i < count; ++i) - et[i] = rg.et[i]; + for (uint32_t i = 0; i < count; ++i) + et[i] = rg.et[i]; - return *this; - } + return *this; + } }; /** @brief struct RIDElementType @@ -188,21 +168,21 @@ struct RowWrapper */ struct RIDElementType { - uint64_t first; + uint64_t first; - RIDElementType(); - RIDElementType(uint64_t f); + RIDElementType(); + RIDElementType(uint64_t f); - const char* getHashString(uint64_t mode, uint64_t* len) const - { - *len = 8; - return (char*) &first; - } + const char* getHashString(uint64_t mode, uint64_t* len) const + { + *len = 8; + return (char*)&first; + } - bool operator<(const RIDElementType& e) const - { - return (first < e.first); - } + bool operator<(const RIDElementType& e) const + { + return (first < e.first); + } }; /** @brief struct TupleType @@ -213,34 +193,38 @@ struct RIDElementType struct TupleType { - uint64_t first; - char* second; - TupleType() {} - TupleType (uint64_t f, char* s): first(f), second(s) {} + uint64_t first; + char* second; + TupleType() + { + } + TupleType(uint64_t f, char* s) : first(f), second(s) + { + } - /** @brief delete a tuple - * - * this function should be called by the tuple user outside - * the datalist if necessary - */ - void deleter() - { - delete [] second; - } + /** @brief delete a tuple + * + * this function should be called by the tuple user outside + * the datalist if necessary + */ + void deleter() + { + delete[] second; + } - /** @brief get hash string - * @note params mode and len are ignored here. they are carried - * just to keep a consistent interface with the other element type - */ - const char* getHashString(uint64_t mode, uint64_t* len) const - { - return (char*)second; - } + /** @brief get hash string + * @note params mode and len are ignored here. they are carried + * just to keep a consistent interface with the other element type + */ + const char* getHashString(uint64_t mode, uint64_t* len) const + { + return (char*)second; + } - bool operator<(const TupleType& e) const - { - return (first < e.first); - } + bool operator<(const TupleType& e) const + { + return (first < e.first); + } }; typedef RowWrapper UintRowGroup; @@ -257,7 +241,7 @@ extern std::istream& operator>>(std::istream& in, RIDElementType& rhs); extern std::ostream& operator<<(std::ostream& out, const RIDElementType& rhs); extern std::istream& operator>>(std::istream& in, TupleType& rhs); extern std::ostream& operator<<(std::ostream& out, const TupleType& rhs); -} +} // namespace joblist #ifndef NO_DATALISTS @@ -272,45 +256,44 @@ extern std::ostream& operator<<(std::ostream& out, const TupleType& rhs); namespace joblist { - ///** @brief type BandedDataList // * // */ -//typedef BandedDL BandedDataList; +// typedef BandedDL BandedDataList; ///** @brief type StringDataList // * // */ -//typedef BandedDL StringDataList; +// typedef BandedDL StringDataList; /** @brief type StringFifoDataList * */ -//typedef FIFO StringFifoDataList; +// typedef FIFO StringFifoDataList; typedef FIFO StringFifoDataList; ///** @brief type StringBucketDataList // * // */ -//typedef BucketDL StringBucketDataList; +// typedef BucketDL StringBucketDataList; ///** @brief type WorkingSetDataList // * // */ -//typedef WSDL WorkingSetDataList; +// typedef WSDL WorkingSetDataList; /** @brief type FifoDataList * */ -//typedef FIFO FifoDataList; +// typedef FIFO FifoDataList; typedef FIFO FifoDataList; ///** @brief type BucketDataList // * // */ -//typedef BucketDL BucketDataList; +// typedef BucketDL BucketDataList; ///** @brief type ConstantDataList_t // * // */ -//typedef ConstantDataList ConstantDataList_t; +// typedef ConstantDataList ConstantDataList_t; ///** @brief type StringConstantDataList_t // * // */ -//typedef ConstantDataList StringConstantDataList_t; +// typedef ConstantDataList StringConstantDataList_t; /** @brief type DataList_t * */ @@ -322,36 +305,36 @@ typedef DataList StrDataList; ///** @brief type DoubleDataList // * // */ -//typedef DataList DoubleDataList; +// typedef DataList DoubleDataList; ///** @brief type TupleDataList // * // */ -//typedef DataList TupleDataList; +// typedef DataList TupleDataList; ///** @brief type SortedWSDL // * // */ -//typedef SWSDL SortedWSDL; +// typedef SWSDL SortedWSDL; ///** @brief type StringSortedWSDL // * // */ -//typedef SWSDL StringSortedWSDL; +// typedef SWSDL StringSortedWSDL; ///** @brief type ZonedDL // * // */ -//typedef ZDL ZonedDL; +// typedef ZDL ZonedDL; ///** @brief type StringZonedDL // * // */ -//typedef ZDL StringZonedDL; +// typedef ZDL StringZonedDL; // ///** @brief type TupleBucketDL // * // */ -//typedef BucketDL TupleBucketDataList; +// typedef BucketDL TupleBucketDataList; typedef FIFO RowGroupDL; -} +} // namespace joblist #include #include @@ -363,269 +346,275 @@ namespace joblist */ class AnyDataList { -public: - AnyDataList() : fDl3(0), fDl6(0), fDl9(0), fDisown(false) { } - ~AnyDataList() + public: + AnyDataList() : fDl3(0), fDl6(0), fDl9(0), fDisown(false) + { + } + ~AnyDataList() + { + if (!fDisown) { - if (!fDisown) - { - delete fDl3; - delete fDl6; - delete fDl9; - } + delete fDl3; + delete fDl6; + delete fDl9; } + } -// AnyDataList() : fDl1(0), fDl2(0), fDl3(0), fDl4(0), fDl5(0), fDl6(0), fDl7(0), fDl8(0), fDl9(0), -// fDl10(0), fDl11(0), fDl12(0), fDl13(0), fDl14(0), fDl15(0), fDl16(0), fDl17(0), fDl18(0), -// fDl19(0), fDl20(0), fDisown(false) { } -// ~AnyDataList() { if (!fDisown) { delete fDl1; delete fDl2; delete fDl3; delete fDl4; -// delete fDl5; delete fDl6; delete fDl7; delete fDl8; delete fDl9; delete fDl10; delete fDl11; -// delete fDl12; delete fDl13; delete fDl14; delete fDl15; delete fDl16; delete fDl17; -// delete fDl18; delete fDl19; delete fDl20; } } + // AnyDataList() : fDl1(0), fDl2(0), fDl3(0), fDl4(0), fDl5(0), fDl6(0), fDl7(0), fDl8(0), fDl9(0), + // fDl10(0), fDl11(0), fDl12(0), fDl13(0), fDl14(0), fDl15(0), fDl16(0), fDl17(0), fDl18(0), + // fDl19(0), fDl20(0), fDisown(false) { } + // ~AnyDataList() { if (!fDisown) { delete fDl1; delete fDl2; delete fDl3; delete fDl4; + // delete fDl5; delete fDl6; delete fDl7; delete fDl8; delete fDl9; delete fDl10; delete fDl11; + // delete fDl12; delete fDl13; delete fDl14; delete fDl15; delete fDl16; delete fDl17; + // delete fDl18; delete fDl19; delete fDl20; } } - // disown() fixes the problem of multiple ownership of a single DL, - // or one on the stack + // disown() fixes the problem of multiple ownership of a single DL, + // or one on the stack - //In the world of bad ideas these are at the top. The whole point of this class is to manage - // dynamically allocated data in an automatic way. These 2 methods circumvent this, and they - // are not necessary in any event, because you can safely share AnyDataList's via a AnyDataListSPtr. - inline void disown() __attribute__ ((deprecated)) - { - fDisown = true; - } - inline void posess() __attribute__ ((deprecated)) - { - fDisown = false; - } + // In the world of bad ideas these are at the top. The whole point of this class is to manage + // dynamically allocated data in an automatic way. These 2 methods circumvent this, and they + // are not necessary in any event, because you can safely share AnyDataList's via a AnyDataListSPtr. + inline void disown() __attribute__((deprecated)) + { + fDisown = true; + } + inline void posess() __attribute__((deprecated)) + { + fDisown = false; + } -// inline void bandedDL(BandedDataList* dl) { fDl1 = dl; } -// inline BandedDataList* bandedDL() { return fDl1; } -// inline const BandedDataList* bandedDL() const { return fDl1; } -// -// inline void workingSetDL(WorkingSetDataList* dl) { fDl2 = dl; } -// inline WorkingSetDataList* workingSetDL() { return fDl2; } -// inline const WorkingSetDataList* workingSetDL() const { return fDl2; } -// - inline void fifoDL(FifoDataList* dl) - { - fDl3 = dl; - } - inline FifoDataList* fifoDL() - { - return fDl3; - } - inline const FifoDataList* fifoDL() const - { - return fDl3; - } -// -// inline void bucketDL(BucketDataList* dl) { fDl4 = dl; } -// inline BucketDataList* bucketDL() { return fDl4; } -// inline const BucketDataList* bucketDL() const { return fDl4; } -// -// inline void constantDL(ConstantDataList_t* dl) { fDl5 = dl; } -// inline ConstantDataList_t* constantDL() { return fDl5; } -// inline const ConstantDataList_t* constantDL() const { return fDl5; } -// -// inline void sortedWSDL(SortedWSDL* dl) { fDl13 = dl; } -// inline SortedWSDL* sortedWSDL() { return fDl13; } -// inline const SortedWSDL* sortedWSDL() const { return fDl13; } -// -// inline void zonedDL(ZonedDL* dl) { fDl15 = dl; } -// inline ZonedDL* zonedDL() { return fDl15; } -// inline const ZonedDL* zonedDL() const { return fDl15; } -// - inline void stringDL(StringFifoDataList* dl) - { - fDl6 = dl; - } - inline StringFifoDataList* stringDL() - { - return fDl6; - } - inline const StringFifoDataList* stringDL() const - { - return fDl6; - } -// -// inline void stringBandedDL(StringDataList* dl) { fDl10 = dl; } -// inline StringDataList* stringBandedDL() { return fDl10; } -// inline const StringDataList* stringBandedDL() const { return fDl10; } -// -// inline void stringBucketDL(StringBucketDataList* dl) { fDl11 = dl; } -// inline StringBucketDataList* stringBucketDL() { return fDl11; } -// inline const StringBucketDataList* stringBucketDL() const { return fDl11; } -// -// inline void stringConstantDL(StringConstantDataList_t* dl) { fDl12 = dl; } -// inline StringConstantDataList_t* stringConstantDL() { return fDl12; } -// inline const StringConstantDataList_t* stringConstantDL() const { return fDl12; } -// -// inline void stringSortedWSDL(StringSortedWSDL* dl) { fDl14 = dl; } -// inline StringSortedWSDL* stringSortedWSDL() { return fDl14; } -// inline const StringSortedWSDL* stringSortedWSDL() const { return fDl14; } -// -// inline void stringZonedDL(StringZonedDL* dl) { fDl16 = dl; } -// inline StringZonedDL* stringZonedDL() { return fDl16; } -// inline const StringZonedDL* stringZonedDL() const { return fDl16; } -// -// inline void tupleBucketDL(TupleBucketDataList* dl) { fDl18 = dl; } -// inline TupleBucketDataList* tupleBucketDL() { return fDl18; } -// inline const TupleBucketDataList* tupleBucketDL() const { return fDl18; } -// -// inline void deliveryWSDL(DeliveryWSDL *dl) { fDl19 = dl; } -// inline DeliveryWSDL * deliveryWSDL() { return fDl19; } -// inline const DeliveryWSDL * deliveryWSDL() const { return fDl19; } + // inline void bandedDL(BandedDataList* dl) { fDl1 = dl; } + // inline BandedDataList* bandedDL() { return fDl1; } + // inline const BandedDataList* bandedDL() const { return fDl1; } + // + // inline void workingSetDL(WorkingSetDataList* dl) { fDl2 = dl; } + // inline WorkingSetDataList* workingSetDL() { return fDl2; } + // inline const WorkingSetDataList* workingSetDL() const { return fDl2; } + // + inline void fifoDL(FifoDataList* dl) + { + fDl3 = dl; + } + inline FifoDataList* fifoDL() + { + return fDl3; + } + inline const FifoDataList* fifoDL() const + { + return fDl3; + } + // + // inline void bucketDL(BucketDataList* dl) { fDl4 = dl; } + // inline BucketDataList* bucketDL() { return fDl4; } + // inline const BucketDataList* bucketDL() const { return fDl4; } + // + // inline void constantDL(ConstantDataList_t* dl) { fDl5 = dl; } + // inline ConstantDataList_t* constantDL() { return fDl5; } + // inline const ConstantDataList_t* constantDL() const { return fDl5; } + // + // inline void sortedWSDL(SortedWSDL* dl) { fDl13 = dl; } + // inline SortedWSDL* sortedWSDL() { return fDl13; } + // inline const SortedWSDL* sortedWSDL() const { return fDl13; } + // + // inline void zonedDL(ZonedDL* dl) { fDl15 = dl; } + // inline ZonedDL* zonedDL() { return fDl15; } + // inline const ZonedDL* zonedDL() const { return fDl15; } + // + inline void stringDL(StringFifoDataList* dl) + { + fDl6 = dl; + } + inline StringFifoDataList* stringDL() + { + return fDl6; + } + inline const StringFifoDataList* stringDL() const + { + return fDl6; + } + // + // inline void stringBandedDL(StringDataList* dl) { fDl10 = dl; } + // inline StringDataList* stringBandedDL() { return fDl10; } + // inline const StringDataList* stringBandedDL() const { return fDl10; } + // + // inline void stringBucketDL(StringBucketDataList* dl) { fDl11 = dl; } + // inline StringBucketDataList* stringBucketDL() { return fDl11; } + // inline const StringBucketDataList* stringBucketDL() const { return fDl11; } + // + // inline void stringConstantDL(StringConstantDataList_t* dl) { fDl12 = dl; } + // inline StringConstantDataList_t* stringConstantDL() { return fDl12; } + // inline const StringConstantDataList_t* stringConstantDL() const { return fDl12; } + // + // inline void stringSortedWSDL(StringSortedWSDL* dl) { fDl14 = dl; } + // inline StringSortedWSDL* stringSortedWSDL() { return fDl14; } + // inline const StringSortedWSDL* stringSortedWSDL() const { return fDl14; } + // + // inline void stringZonedDL(StringZonedDL* dl) { fDl16 = dl; } + // inline StringZonedDL* stringZonedDL() { return fDl16; } + // inline const StringZonedDL* stringZonedDL() const { return fDl16; } + // + // inline void tupleBucketDL(TupleBucketDataList* dl) { fDl18 = dl; } + // inline TupleBucketDataList* tupleBucketDL() { return fDl18; } + // inline const TupleBucketDataList* tupleBucketDL() const { return fDl18; } + // + // inline void deliveryWSDL(DeliveryWSDL *dl) { fDl19 = dl; } + // inline DeliveryWSDL * deliveryWSDL() { return fDl19; } + // inline const DeliveryWSDL * deliveryWSDL() const { return fDl19; } - inline void rowGroupDL(boost::shared_ptr dl) - { - fDl20 = dl; - } - inline void rowGroupDL(RowGroupDL* dl) - { - fDl20.reset(dl); - } - inline RowGroupDL* rowGroupDL() - { - return fDl20.get(); - } - inline const RowGroupDL* rowGroupDL() const - { - return fDl20.get(); - } + inline void rowGroupDL(boost::shared_ptr dl) + { + fDl20 = dl; + } + inline void rowGroupDL(RowGroupDL* dl) + { + fDl20.reset(dl); + } + inline RowGroupDL* rowGroupDL() + { + return fDl20.get(); + } + inline const RowGroupDL* rowGroupDL() const + { + return fDl20.get(); + } - DataList_t* dataList() - { - if (fDl3 != NULL) return reinterpret_cast(fDl3); - else if (fDl9 != NULL) return fDl9; + DataList_t* dataList() + { + if (fDl3 != NULL) + return reinterpret_cast(fDl3); + else if (fDl9 != NULL) + return fDl9; - return reinterpret_cast(fDl20.get()); -// if (fDl1 != NULL) return fDl1; -// else if (fDl2 != NULL) return fDl2; -// else if (fDl3 != NULL) return reinterpret_cast(fDl3); -// else if (fDl4 != NULL) return fDl4; -// else if (fDl9 != NULL) return fDl9; -// else if (fDl13 != NULL) return fDl13; -// else if (fDl15 != NULL) return fDl15; -// else if (fDl19 != NULL) return fDl19; -// else if (fDl20 != NULL) return reinterpret_cast(fDl20); -// else return fDl5; - } -// - StrDataList* stringDataList() - { -// if (fDl6 != NULL) return reinterpret_cast(fDl6); -// else if (fDl10 != NULL) return fDl10; -// else if (fDl11 != NULL) return fDl11; -// else if (fDl12 != NULL) return fDl12; -// else if (fDl14 != NULL) return fDl14; -// else if (fDl16 != NULL) return fDl16; -// return fDl8; - return reinterpret_cast(fDl6); - } -// -// TupleDataList* tupleDataList() { -// if (fDl18 != NULL) return fDl18; -// return fDl17; -// } -// -// /* fDl{7,8} store base class pointers. For consistency, maybe strDataList -// should consider fDl6 also. */ -// inline StrDataList * strDataList() -// { return fDl8; } -// -// inline void strDataList(StrDataList *d) -// { fDl8 = d; } -// -// inline DoubleDataList * doubleDL() -// { return fDl7; } -// -// inline void doubleDL(DoubleDataList *d) -// { fDl7 = d; } + return reinterpret_cast(fDl20.get()); + // if (fDl1 != NULL) return fDl1; + // else if (fDl2 != NULL) return fDl2; + // else if (fDl3 != NULL) return reinterpret_cast(fDl3); + // else if (fDl4 != NULL) return fDl4; + // else if (fDl9 != NULL) return fDl9; + // else if (fDl13 != NULL) return fDl13; + // else if (fDl15 != NULL) return fDl15; + // else if (fDl19 != NULL) return fDl19; + // else if (fDl20 != NULL) return reinterpret_cast(fDl20); + // else return fDl5; + } + // + StrDataList* stringDataList() + { + // if (fDl6 != NULL) return reinterpret_cast(fDl6); + // else if (fDl10 != NULL) return fDl10; + // else if (fDl11 != NULL) return fDl11; + // else if (fDl12 != NULL) return fDl12; + // else if (fDl14 != NULL) return fDl14; + // else if (fDl16 != NULL) return fDl16; + // return fDl8; + return reinterpret_cast(fDl6); + } + // + // TupleDataList* tupleDataList() { + // if (fDl18 != NULL) return fDl18; + // return fDl17; + // } + // + // /* fDl{7,8} store base class pointers. For consistency, maybe strDataList + // should consider fDl6 also. */ + // inline StrDataList * strDataList() + // { return fDl8; } + // + // inline void strDataList(StrDataList *d) + // { fDl8 = d; } + // + // inline DoubleDataList * doubleDL() + // { return fDl7; } + // + // inline void doubleDL(DoubleDataList *d) + // { fDl7 = d; } - enum DataListTypes - { - UNKNOWN_DATALIST, /*!< 0 Unknown DataList */ - BANDED_DATALIST, /*!< 1 Banded DataList */ - WORKING_SET_DATALIST, /*!< 2 WSDL */ - FIFO_DATALIST, /*!< 3 FIFO */ - BUCKET_DATALIST, /*!< 4 Bucket */ - CONSTANT_DATALIST, /*!< 5 Constant */ - STRING_DATALIST, /*!< 6 String */ - DOUBLE_DATALIST, /*!< 7 Double */ - STRINGFIFO_DATALIST, /*!< 8 String FIFO */ - STRINGBANDED_DATALIST, /*!< 9 String Banded */ - STRINGBUCKET_DATALIST, /*!< 10 String Bucket */ - STRINGCONSTANT_DATALIST, /*!< 11 String Constant */ - SORTED_WORKING_SET_DATALIST, /*!< 12 Sorted WSDL */ - STRINGSORTED_WORKING_SET_DATALIST, /*!< 13 String Sorted WSDL */ - ZONED_DATALIST, /*!< 14 Zoned Datalist */ - STRINGZONED_DATALIST, /*!< 15 String Zoned Datalist */ - TUPLEBUCKET_DATALIST, /*!< 16 Tuple Bucket Datalist */ - TUPLE_DATALIST, /*!< 17 Tuple Datalist */ - DELIVERYWSDL, /*!< 18 Delivery WSDL */ - ROWGROUP_DATALIST - }; + enum DataListTypes + { + UNKNOWN_DATALIST, /*!< 0 Unknown DataList */ + BANDED_DATALIST, /*!< 1 Banded DataList */ + WORKING_SET_DATALIST, /*!< 2 WSDL */ + FIFO_DATALIST, /*!< 3 FIFO */ + BUCKET_DATALIST, /*!< 4 Bucket */ + CONSTANT_DATALIST, /*!< 5 Constant */ + STRING_DATALIST, /*!< 6 String */ + DOUBLE_DATALIST, /*!< 7 Double */ + STRINGFIFO_DATALIST, /*!< 8 String FIFO */ + STRINGBANDED_DATALIST, /*!< 9 String Banded */ + STRINGBUCKET_DATALIST, /*!< 10 String Bucket */ + STRINGCONSTANT_DATALIST, /*!< 11 String Constant */ + SORTED_WORKING_SET_DATALIST, /*!< 12 Sorted WSDL */ + STRINGSORTED_WORKING_SET_DATALIST, /*!< 13 String Sorted WSDL */ + ZONED_DATALIST, /*!< 14 Zoned Datalist */ + STRINGZONED_DATALIST, /*!< 15 String Zoned Datalist */ + TUPLEBUCKET_DATALIST, /*!< 16 Tuple Bucket Datalist */ + TUPLE_DATALIST, /*!< 17 Tuple Datalist */ + DELIVERYWSDL, /*!< 18 Delivery WSDL */ + ROWGROUP_DATALIST + }; - static DataListTypes dlType(const DataList_t* dl); - static DataListTypes strDlType(const StrDataList* dl); -// static DataListTypes tupleDlType(const TupleDataList* dl); - uint32_t getNumConsumers() - { -// if (fDl1 != NULL) return fDl1->getNumConsumers(); -// else if (fDl2 != NULL) return fDl2->getNumConsumers(); -// else if (fDl3 != NULL) return fDl3->getNumConsumers(); -// else if (fDl6 != NULL) return fDl6->getNumConsumers(); -// else if (fDl10 != NULL) return fDl10->getNumConsumers(); -// else if (fDl13 != NULL) return fDl13->getNumConsumers(); -// else if (fDl14 != NULL) return fDl14->getNumConsumers(); -// else if (fDl15 != NULL) return fDl15->getNumConsumers(); -// else if (fDl16 != NULL) return fDl16->getNumConsumers(); -// else if (fDl4 != NULL) return 1; -// else if (fDl11 != NULL) return 1; -// else if (fDl18 != NULL) return 1; -// else if (fDl19 != NULL) return fDl19->getNumConsumers(); -// else if (fDl20 != NULL) return 1; -// else return 0; + static DataListTypes dlType(const DataList_t* dl); + static DataListTypes strDlType(const StrDataList* dl); + // static DataListTypes tupleDlType(const TupleDataList* dl); + uint32_t getNumConsumers() + { + // if (fDl1 != NULL) return fDl1->getNumConsumers(); + // else if (fDl2 != NULL) return fDl2->getNumConsumers(); + // else if (fDl3 != NULL) return fDl3->getNumConsumers(); + // else if (fDl6 != NULL) return fDl6->getNumConsumers(); + // else if (fDl10 != NULL) return fDl10->getNumConsumers(); + // else if (fDl13 != NULL) return fDl13->getNumConsumers(); + // else if (fDl14 != NULL) return fDl14->getNumConsumers(); + // else if (fDl15 != NULL) return fDl15->getNumConsumers(); + // else if (fDl16 != NULL) return fDl16->getNumConsumers(); + // else if (fDl4 != NULL) return 1; + // else if (fDl11 != NULL) return 1; + // else if (fDl18 != NULL) return 1; + // else if (fDl19 != NULL) return fDl19->getNumConsumers(); + // else if (fDl20 != NULL) return 1; + // else return 0; - if (fDl20) return 1; - else if (fDl3 != NULL) return fDl3->getNumConsumers(); - else if (fDl6 != NULL) return fDl6->getNumConsumers(); + if (fDl20) + return 1; + else if (fDl3 != NULL) + return fDl3->getNumConsumers(); + else if (fDl6 != NULL) + return fDl6->getNumConsumers(); - return 0; - } + return 0; + } - //There is no operator==() because 2 AnyDataList's are equal if they point to the same DL, but the only way - // that could be is if they are the _same_ AnyDatalist, since, by convention, AnyDataList's are only - // moved around as shared_ptr's (AnyDataListSPtr). Indeed, it is an error if two different AnyDataList - // objects point to the same DL. - // bool operator==(const AnyDataList& rhs); + // There is no operator==() because 2 AnyDataList's are equal if they point to the same DL, but the only way + // that could be is if they are the _same_ AnyDatalist, since, by convention, AnyDataList's are only + // moved around as shared_ptr's (AnyDataListSPtr). Indeed, it is an error if two different AnyDataList + // objects point to the same DL. + // bool operator==(const AnyDataList& rhs); -private: - AnyDataList(const AnyDataList& rhs); - AnyDataList& operator=(const AnyDataList& rhs); - -// BandedDataList* fDl1; -// WorkingSetDataList* fDl2; - FifoDataList* fDl3; -// BucketDataList* fDl4; -// ConstantDataList_t* fDl5; - StringFifoDataList* fDl6; -// DoubleDataList* fDl7; -// StrDataList* fDl8; - DataList_t* fDl9; -// StringDataList* fDl10; -// StringBucketDataList* fDl11; -// StringConstantDataList_t* fDl12; -// SortedWSDL* fDl13; -// StringSortedWSDL* fDl14; -// ZonedDL* fDl15; -// StringZonedDL* fDl16; -// TupleDataList* fDl17; -// TupleBucketDataList *fDl18; -// DeliveryWSDL *fDl19; - boost::shared_ptr fDl20; - bool fDisown; + private: + AnyDataList(const AnyDataList& rhs); + AnyDataList& operator=(const AnyDataList& rhs); + // BandedDataList* fDl1; + // WorkingSetDataList* fDl2; + FifoDataList* fDl3; + // BucketDataList* fDl4; + // ConstantDataList_t* fDl5; + StringFifoDataList* fDl6; + // DoubleDataList* fDl7; + // StrDataList* fDl8; + DataList_t* fDl9; + // StringDataList* fDl10; + // StringBucketDataList* fDl11; + // StringConstantDataList_t* fDl12; + // SortedWSDL* fDl13; + // StringSortedWSDL* fDl14; + // ZonedDL* fDl15; + // StringZonedDL* fDl16; + // TupleDataList* fDl17; + // TupleBucketDataList *fDl18; + // DeliveryWSDL *fDl19; + boost::shared_ptr fDl20; + bool fDisown; }; /** @brief type AnyDataListSPtr @@ -643,13 +632,12 @@ extern std::ostream& operator<<(std::ostream& os, const AnyDataListSPtr& dl); //...Manipulators for controlling the inclusion of the datalist's //...OID in the AnyDataListSPtr's output stream operator. // -extern std::ostream& showOidInDL ( std::ostream& strm ); -extern std::ostream& omitOidInDL ( std::ostream& strm ); +extern std::ostream& showOidInDL(std::ostream& strm); +extern std::ostream& omitOidInDL(std::ostream& strm); -} +} // namespace joblist #endif #endif // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/errorinfo.h b/dbcon/joblist/errorinfo.h index 6b37e3733..710cb89c1 100644 --- a/dbcon/joblist/errorinfo.h +++ b/dbcon/joblist/errorinfo.h @@ -17,7 +17,6 @@ // $Id: jobstep.h 9636 2013-06-20 14:23:36Z rdempsey $ - /** @file */ #ifndef JOBLIST_ERROR_INFO_H_ @@ -28,32 +27,31 @@ namespace joblist { - /** @brief struct ErrorInfo * * struct ErrorInfo stores the error code and message */ struct ErrorInfo { - ErrorInfo() : errCode(0) { } - uint32_t errCode; - std::string errMsg; - // for backward compat - ErrorInfo(uint16_t v) : errCode(v) { } - ErrorInfo& operator=(uint16_t v) - { - errCode = v; - errMsg.clear(); - return *this; - } + ErrorInfo() : errCode(0) + { + } + uint32_t errCode; + std::string errMsg; + // for backward compat + ErrorInfo(uint16_t v) : errCode(v) + { + } + ErrorInfo& operator=(uint16_t v) + { + errCode = v; + errMsg.clear(); + return *this; + } }; typedef boost::shared_ptr SErrorInfo; - -} +} // namespace joblist #endif // JOBLIST_ERROR_INFO_H_ // vim:ts=4 sw=4: - - - diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 9e949196d..2f5b96afc 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -17,7 +17,6 @@ // $Id: expressionstep.cpp 9681 2013-07-11 22:58:05Z xlou $ - //#define NDEBUG #include #include @@ -56,183 +55,175 @@ using namespace rowgroup; namespace joblist { -ExpressionStep::ExpressionStep() : - fExpressionFilter(NULL), - fExpressionId(-1), - fVarBinOK(false), - fSelectFilter(false), - fAssociatedJoinId(0), - fDoJoin(false), - fVirtual(false) +ExpressionStep::ExpressionStep() + : fExpressionFilter(NULL) + , fExpressionId(-1) + , fVarBinOK(false) + , fSelectFilter(false) + , fAssociatedJoinId(0) + , fDoJoin(false) + , fVirtual(false) { } -ExpressionStep::ExpressionStep(const JobInfo& jobInfo) : - JobStep(jobInfo), - fExpressionFilter(NULL), - fExpressionId(-1), - fVarBinOK(false), - fSelectFilter(false), - fAssociatedJoinId(0), - fDoJoin(false), - fVirtual(false) +ExpressionStep::ExpressionStep(const JobInfo& jobInfo) + : JobStep(jobInfo) + , fExpressionFilter(NULL) + , fExpressionId(-1) + , fVarBinOK(false) + , fSelectFilter(false) + , fAssociatedJoinId(0) + , fDoJoin(false) + , fVirtual(false) { } -ExpressionStep::ExpressionStep(const ExpressionStep& rhs) : - JobStep(rhs), - fExpression(rhs.expression()), - fExpressionFilter(NULL), - fExpressionId(rhs.expressionId()), - fAliases(rhs.aliases()), - fViews(rhs.views()), - fSchemas(rhs.schemas()), - fTableKeys(rhs.tableKeys()), - fColumnKeys(rhs.columnKeys()), - fVarBinOK(rhs.fVarBinOK), - fSelectFilter(rhs.fSelectFilter), - fAssociatedJoinId(rhs.fAssociatedJoinId), - fDoJoin(rhs.fDoJoin), - fVirtual(rhs.fVirtual) +ExpressionStep::ExpressionStep(const ExpressionStep& rhs) + : JobStep(rhs) + , fExpression(rhs.expression()) + , fExpressionFilter(NULL) + , fExpressionId(rhs.expressionId()) + , fAliases(rhs.aliases()) + , fViews(rhs.views()) + , fSchemas(rhs.schemas()) + , fTableKeys(rhs.tableKeys()) + , fColumnKeys(rhs.columnKeys()) + , fVarBinOK(rhs.fVarBinOK) + , fSelectFilter(rhs.fSelectFilter) + , fAssociatedJoinId(rhs.fAssociatedJoinId) + , fDoJoin(rhs.fDoJoin) + , fVirtual(rhs.fVirtual) { - if (rhs.expressionFilter() != NULL) - fExpressionFilter = new ParseTree(*(rhs.expressionFilter())); + if (rhs.expressionFilter() != NULL) + fExpressionFilter = new ParseTree(*(rhs.expressionFilter())); } - ExpressionStep::~ExpressionStep() { - if (fExpressionFilter != NULL) - delete fExpressionFilter; + if (fExpressionFilter != NULL) + delete fExpressionFilter; } - void ExpressionStep::run() { } - void ExpressionStep::join() { } - void ExpressionStep::expression(const SRCP exp, JobInfo& jobInfo) { - fExpression = exp; + fExpression = exp; - // set expression Id - ArithmeticColumn* ac = dynamic_cast(fExpression.get()); - FunctionColumn* fc = dynamic_cast(fExpression.get()); - fExpressionId = exp.get()->expressionId(); + // set expression Id + ArithmeticColumn* ac = dynamic_cast(fExpression.get()); + FunctionColumn* fc = dynamic_cast(fExpression.get()); + fExpressionId = exp.get()->expressionId(); - if (ac != NULL || fc != NULL) - addColumn(exp.get(), jobInfo); + if (ac != NULL || fc != NULL) + addColumn(exp.get(), jobInfo); } - void ExpressionStep::expressionFilter(const Filter* filter, JobInfo& jobInfo) { - Filter* f = filter->clone(); - fExpressionFilter = new ParseTree(f); - idbassert(fExpressionFilter != NULL); + Filter* f = filter->clone(); + fExpressionFilter = new ParseTree(f); + idbassert(fExpressionFilter != NULL); - if (fExpressionFilter == NULL) - { - std::ostringstream errmsg; - errmsg << "ExpressionStep: Failed to create a new ParseTree"; - cerr << boldStart << errmsg.str() << boldStop << endl; - throw runtime_error(errmsg.str()); - } + if (fExpressionFilter == NULL) + { + std::ostringstream errmsg; + errmsg << "ExpressionStep: Failed to create a new ParseTree"; + cerr << boldStart << errmsg.str() << boldStop << endl; + throw runtime_error(errmsg.str()); + } - addFilter(fExpressionFilter, jobInfo); + addFilter(fExpressionFilter, jobInfo); - // populate the oid vectors - SimpleFilter* sf = dynamic_cast(f); + // populate the oid vectors + SimpleFilter* sf = dynamic_cast(f); - if (sf != NULL && sf->op()->data() == "=") - functionJoinCheck(sf, jobInfo); + if (sf != NULL && sf->op()->data() == "=") + functionJoinCheck(sf, jobInfo); } - void ExpressionStep::expressionFilter(const ParseTree* filter, JobInfo& jobInfo) { - fExpressionFilter = new ParseTree(); - idbassert(fExpressionFilter != NULL); + fExpressionFilter = new ParseTree(); + idbassert(fExpressionFilter != NULL); - if (fExpressionFilter == NULL) - { - std::ostringstream errmsg; - errmsg << "ExpressionStep: Failed to create a new ParseTree"; - cerr << boldStart << errmsg.str() << boldStop << endl; - throw runtime_error(errmsg.str()); - } + if (fExpressionFilter == NULL) + { + std::ostringstream errmsg; + errmsg << "ExpressionStep: Failed to create a new ParseTree"; + cerr << boldStart << errmsg.str() << boldStop << endl; + throw runtime_error(errmsg.str()); + } - fExpressionFilter->copyTree(*filter); + fExpressionFilter->copyTree(*filter); - addFilter(fExpressionFilter, jobInfo); + addFilter(fExpressionFilter, jobInfo); } - void ExpressionStep::addSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { - addColumn(sf->lhs(), jobInfo); - addColumn(sf->rhs(), jobInfo); + addColumn(sf->lhs(), jobInfo); + addColumn(sf->rhs(), jobInfo); } - void ExpressionStep::addFilter(ParseTree* filter, JobInfo& jobInfo) { - stack filterStack; + stack filterStack; - while (filter || !filterStack.empty()) + while (filter || !filterStack.empty()) + { + if (filter != NULL) { - if (filter != NULL) - { - filterStack.push(filter); - filter = filter->left(); - } - else if (!filterStack.empty()) - { - filter = filterStack.top(); - filterStack.pop(); - - TreeNode* tn = filter->data(); - filter = filter->right(); - - ReturnedColumn* rc = dynamic_cast(tn); - SimpleFilter* sf = dynamic_cast(tn); - ConstantFilter* cf = dynamic_cast(tn); - Operator* op = dynamic_cast(tn); - - if (rc != NULL) - { - addColumn(rc, jobInfo); - } - else if (sf != NULL) - { - addSimpleFilter(sf, jobInfo); - } - else if (cf != NULL) - { - const ConstantFilter::FilterList& fs = cf->filterList(); - - for (ConstantFilter::FilterList::const_iterator i = fs.begin(); i != fs.end(); i++) - { - SimpleFilter* f = dynamic_cast(i->get()); - - if (f != NULL) - addSimpleFilter(f, jobInfo); - else - throw logic_error("unknow filter type in constant filter."); - } - } - else if (op == NULL) - { - throw logic_error("tree node not handled in Expression step."); - } - } + filterStack.push(filter); + filter = filter->left(); } + else if (!filterStack.empty()) + { + filter = filterStack.top(); + filterStack.pop(); + + TreeNode* tn = filter->data(); + filter = filter->right(); + + ReturnedColumn* rc = dynamic_cast(tn); + SimpleFilter* sf = dynamic_cast(tn); + ConstantFilter* cf = dynamic_cast(tn); + Operator* op = dynamic_cast(tn); + + if (rc != NULL) + { + addColumn(rc, jobInfo); + } + else if (sf != NULL) + { + addSimpleFilter(sf, jobInfo); + } + else if (cf != NULL) + { + const ConstantFilter::FilterList& fs = cf->filterList(); + + for (ConstantFilter::FilterList::const_iterator i = fs.begin(); i != fs.end(); i++) + { + SimpleFilter* f = dynamic_cast(i->get()); + + if (f != NULL) + addSimpleFilter(f, jobInfo); + else + throw logic_error("unknow filter type in constant filter."); + } + } + else if (op == NULL) + { + throw logic_error("tree node not handled in Expression step."); + } + } + } #if 0 // have to workaround correlation on exp, the following approach does not work // extract simple columns from parse tree @@ -259,552 +250,544 @@ void ExpressionStep::addFilter(ParseTree* filter, JobInfo& jobInfo) #endif - return; + return; } - void ExpressionStep::addColumn(ReturnedColumn* rc, JobInfo& jobInfo) { - const vector* scs = NULL; - const vector* wcs = NULL; - ArithmeticColumn* ac = NULL; - FunctionColumn* fc = NULL; - SimpleColumn* sc = NULL; - WindowFunctionColumn* wc = NULL; + const vector* scs = NULL; + const vector* wcs = NULL; + ArithmeticColumn* ac = NULL; + FunctionColumn* fc = NULL; + SimpleColumn* sc = NULL; + WindowFunctionColumn* wc = NULL; - // workaround for exp(sc) in (sub) where correlation is set on exp, but not sc. - // populate it to sc for correct scope resolution - uint64_t correlated = rc->joinInfo(); + // workaround for exp(sc) in (sub) where correlation is set on exp, but not sc. + // populate it to sc for correct scope resolution + uint64_t correlated = rc->joinInfo(); - if (NULL != (ac = dynamic_cast(rc))) + if (NULL != (ac = dynamic_cast(rc))) + { + scs = &(ac->simpleColumnList()); + wcs = &(ac->windowfunctionColumnList()); + } + else if (NULL != (fc = dynamic_cast(rc))) + { + scs = &(fc->simpleColumnList()); + wcs = &(fc->windowfunctionColumnList()); + fVarBinOK = ((strcmp(fc->functionName().c_str(), "hex") == 0) || + (strcmp(fc->functionName().c_str(), "octet_length") == 0) || + (strcmp(fc->functionName().c_str(), "length") == 0)); + } + + if (scs != NULL || wcs != NULL) + { + if (scs != NULL) { - scs = &(ac->simpleColumnList()); - wcs = &(ac->windowfunctionColumnList()); - } - else if (NULL != (fc = dynamic_cast(rc))) - { - scs = &(fc->simpleColumnList()); - wcs = &(fc->windowfunctionColumnList()); - fVarBinOK = ((strcmp(fc->functionName().c_str(), "hex") == 0) || - (strcmp(fc->functionName().c_str(), "octet_length") == 0) || - (strcmp(fc->functionName().c_str(), "length") == 0)); + vector::const_iterator cit = scs->begin(); + vector::const_iterator end = scs->end(); + + while (cit != end) + { + SimpleColumn* sc = *cit; + sc->joinInfo(sc->joinInfo() | correlated); + populateColumnInfo(*cit, jobInfo); + ++cit; + } } - if (scs != NULL || wcs != NULL) + if (wcs != NULL) { - if (scs != NULL) - { - vector::const_iterator cit = scs->begin(); - vector::const_iterator end = scs->end(); + vector::const_iterator cit = wcs->begin(); + vector::const_iterator end = wcs->end(); - while (cit != end) - { - SimpleColumn* sc = *cit; - sc->joinInfo(sc->joinInfo() | correlated); - populateColumnInfo(*cit, jobInfo); - ++cit; - } - } + while (cit != end) + { + populateColumnInfo(*cit, jobInfo); + ++cit; + } + } + } + else if (NULL != (sc = dynamic_cast(rc))) + { + populateColumnInfo(sc, jobInfo); + } + else if (NULL != (wc = dynamic_cast(rc))) + { + populateColumnInfo(rc, jobInfo); + } + else if (NULL != (dynamic_cast(rc))) + { + populateColumnInfo(rc, jobInfo); + } + else + { + ConstantColumn* cc = dynamic_cast(rc); - if (wcs != NULL) - { - vector::const_iterator cit = wcs->begin(); - vector::const_iterator end = wcs->end(); - - while (cit != end) - { - populateColumnInfo(*cit, jobInfo); - ++cit; - } - } - } - else if (NULL != (sc = dynamic_cast(rc))) + // idbassert(cc != NULL) + if (cc == NULL) { - populateColumnInfo(sc, jobInfo); - } - else if (NULL != (wc = dynamic_cast(rc))) - { - populateColumnInfo(rc, jobInfo); - } - else if (NULL != (dynamic_cast(rc))) - { - populateColumnInfo(rc, jobInfo); - } - else - { - ConstantColumn* cc = dynamic_cast(rc); - -// idbassert(cc != NULL) - if (cc == NULL) - { - std::ostringstream errmsg; - errmsg << "ExpressionStep: " << typeid(*rc).name() << " in expression."; - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } + std::ostringstream errmsg; + errmsg << "ExpressionStep: " << typeid(*rc).name() << " in expression."; + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); } + } } - void ExpressionStep::populateColumnInfo(ReturnedColumn* rc, JobInfo& jobInfo) { - // As of bug3695, make sure varbinary is not used in function expression. - if ((rc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - rc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) - throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); + // As of bug3695, make sure varbinary is not used in function expression. + if ((rc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + rc->resultType().colDataType == CalpontSystemCatalog::BLOB) && + !fVarBinOK) + throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); - SimpleColumn* sc = dynamic_cast(rc); - WindowFunctionColumn* wc = NULL; - AggregateColumn* ac = NULL; + SimpleColumn* sc = dynamic_cast(rc); + WindowFunctionColumn* wc = NULL; + AggregateColumn* ac = NULL; - if (NULL != sc) - return populateColumnInfo(sc, jobInfo); - else if (NULL != (wc = dynamic_cast(rc))) - return populateColumnInfo(wc, jobInfo); - else if (NULL != (ac = dynamic_cast(rc))) - return populateColumnInfo(ac, jobInfo); - else // for now only allow simple and windowfunction column, more work to do. - throw runtime_error("Error in parsing expression."); + if (NULL != sc) + return populateColumnInfo(sc, jobInfo); + else if (NULL != (wc = dynamic_cast(rc))) + return populateColumnInfo(wc, jobInfo); + else if (NULL != (ac = dynamic_cast(rc))) + return populateColumnInfo(ac, jobInfo); + else // for now only allow simple and windowfunction column, more work to do. + throw runtime_error("Error in parsing expression."); } - void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) { - // As of bug3695, make sure varbinary is not used in function expression. - if ((sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - sc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) - throw runtime_error ("VARBINARY/BLOB in filter or function is not supported."); + // As of bug3695, make sure varbinary is not used in function expression. + if ((sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + sc->resultType().colDataType == CalpontSystemCatalog::BLOB) && + !fVarBinOK) + throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); - CalpontSystemCatalog::OID tblOid = joblist::tableOid(sc, jobInfo.csc); - string alias = extractTableAlias(sc); - string view = sc->viewName(); - string schema = sc->schemaName(); - fTableOids.push_back(tblOid); - CalpontSystemCatalog::ColType ct; + CalpontSystemCatalog::OID tblOid = joblist::tableOid(sc, jobInfo.csc); + string alias = extractTableAlias(sc); + string view = sc->viewName(); + string schema = sc->schemaName(); + fTableOids.push_back(tblOid); + CalpontSystemCatalog::ColType ct; - if (schema.empty()) + if (schema.empty()) + { + sc->oid(tblOid + 1 + sc->colPosition()); + ct = sc->resultType(); + } + else if (sc->isColumnStore() == false) + { + ct = sc->colType(); + } + else + { + ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (dynamic_cast(sc) == NULL) { - sc->oid(tblOid + 1 + sc->colPosition()); - ct = sc->resultType(); + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; } - else if (sc->isColumnStore() == false) + // X + if (ct.scale == 0) // keep passed original ct for decimal type + sc->resultType(ct); // update from mysql type to calpont type + } + + fAliases.push_back(alias); + fViews.push_back(view); + fSchemas.push_back(schema); + fTableKeys.push_back(makeTableKey(jobInfo, sc)); + fColumns.push_back(sc); + + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + fColumnKeys.push_back(ti.key); + + // @bug 2990, MySQL timestamp/time/date/datetime type is different from IDB type + if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || + ti.dtype == CalpontSystemCatalog::TIME || ti.dtype == CalpontSystemCatalog::TIMESTAMP) + { + if (ti.dtype != ct.colDataType) { - ct = sc->colType(); + ct.colWidth = ti.width; + ct.colDataType = ti.dtype; + ct.scale = ti.scale; + ct.precision = ti.precision; + sc->resultType(ct); } - else - { - ct = sc->colType(); + } -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (dynamic_cast(sc) == NULL) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } -//X - if (ct.scale == 0) // keep passed original ct for decimal type - sc->resultType(ct); // update from mysql type to calpont type - } + CalpontSystemCatalog::OID dictOid = joblist::isDictCol(ct); - fAliases.push_back(alias); - fViews.push_back(view); - fSchemas.push_back(schema); - fTableKeys.push_back(makeTableKey(jobInfo, sc)); - fColumns.push_back(sc); - - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); - fColumnKeys.push_back(ti.key); - - // @bug 2990, MySQL timestamp/time/date/datetime type is different from IDB type - if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || - ti.dtype == CalpontSystemCatalog::TIME || - ti.dtype == CalpontSystemCatalog::TIMESTAMP) - { - if (ti.dtype != ct.colDataType) - { - ct.colWidth = ti.width; - ct.colDataType = ti.dtype; - ct.scale = ti.scale; - ct.precision = ti.precision; - sc->resultType(ct); - } - } - - CalpontSystemCatalog::OID dictOid = joblist::isDictCol(ct); - - if (dictOid > 0) - { - uint32_t tupleKey = ti.key; - jobInfo.tokenOnly[tupleKey] = false; - jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); - ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); - jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; - } + if (dictOid > 0) + { + uint32_t tupleKey = ti.key; + jobInfo.tokenOnly[tupleKey] = false; + jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; + } } - void ExpressionStep::populateColumnInfo(WindowFunctionColumn* wc, JobInfo& jobInfo) { - // As of bug3695, make sure varbinary is not used in function expression. - if ((wc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - wc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) - throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); + // As of bug3695, make sure varbinary is not used in function expression. + if ((wc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + wc->resultType().colDataType == CalpontSystemCatalog::BLOB) && + !fVarBinOK) + throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); - // This is for window function in IN/EXISTS sub-query. - // In 4.0 implementation, the window function is cloned to where clause in a simple filter. - // This can be identified because SQL syntax does NOT allow window function in where clause. - // Workaround here until a better way found. - TupleInfo ti(setExpTupleInfo(wc->resultType(), wc->expressionId(), wc->alias(), jobInfo)); - uint64_t wcKey = ti.key; - string alias(""); - string view(""); - string schema(""); - fTableOids.push_back(jobInfo.keyInfo->tupleKeyToTableOid[wcKey]); - fAliases.push_back(alias); - fViews.push_back(view); - fSchemas.push_back(schema); - fTableKeys.push_back(jobInfo.keyInfo->colKeyToTblKey[wcKey]); - fColumnKeys.push_back(wcKey); - fColumns.push_back(wc); + // This is for window function in IN/EXISTS sub-query. + // In 4.0 implementation, the window function is cloned to where clause in a simple filter. + // This can be identified because SQL syntax does NOT allow window function in where clause. + // Workaround here until a better way found. + TupleInfo ti(setExpTupleInfo(wc->resultType(), wc->expressionId(), wc->alias(), jobInfo)); + uint64_t wcKey = ti.key; + string alias(""); + string view(""); + string schema(""); + fTableOids.push_back(jobInfo.keyInfo->tupleKeyToTableOid[wcKey]); + fAliases.push_back(alias); + fViews.push_back(view); + fSchemas.push_back(schema); + fTableKeys.push_back(jobInfo.keyInfo->colKeyToTblKey[wcKey]); + fColumnKeys.push_back(wcKey); + fColumns.push_back(wc); } - void ExpressionStep::populateColumnInfo(AggregateColumn* ac, JobInfo& jobInfo) { - // As of bug3695, make sure varbinary is not used in function expression. - if ((ac->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - ac->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) - throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); + // As of bug3695, make sure varbinary is not used in function expression. + if ((ac->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + ac->resultType().colDataType == CalpontSystemCatalog::BLOB) && + !fVarBinOK) + throw runtime_error("VARBINARY/BLOB in filter or function is not supported."); - // This is for aggregate function in IN/EXISTS sub-query. - TupleInfo ti(setExpTupleInfo(ac->resultType(), ac->expressionId(), ac->alias(), jobInfo)); - uint64_t acKey = ti.key; - string alias(""); - string view(""); - string schema(""); - fTableOids.push_back(jobInfo.keyInfo->tupleKeyToTableOid[acKey]); - fAliases.push_back(alias); - fViews.push_back(view); - fSchemas.push_back(schema); - fTableKeys.push_back(jobInfo.keyInfo->colKeyToTblKey[acKey]); - fColumnKeys.push_back(acKey); - fColumns.push_back(ac); + // This is for aggregate function in IN/EXISTS sub-query. + TupleInfo ti(setExpTupleInfo(ac->resultType(), ac->expressionId(), ac->alias(), jobInfo)); + uint64_t acKey = ti.key; + string alias(""); + string view(""); + string schema(""); + fTableOids.push_back(jobInfo.keyInfo->tupleKeyToTableOid[acKey]); + fAliases.push_back(alias); + fViews.push_back(view); + fSchemas.push_back(schema); + fTableKeys.push_back(jobInfo.keyInfo->colKeyToTblKey[acKey]); + fColumnKeys.push_back(acKey); + fColumns.push_back(ac); } - void ExpressionStep::updateInputIndex(map& indexMap, const JobInfo& jobInfo) { - // expression is handled as function join already - if (fDoJoin) - return; + // expression is handled as function join already + if (fDoJoin) + return; - if (jobInfo.trace) - cout << "Input indices of Expression:" << (int64_t) fExpressionId << endl; + if (jobInfo.trace) + cout << "Input indices of Expression:" << (int64_t)fExpressionId << endl; - for (vector::iterator it = fColumns.begin(); it != fColumns.end(); ++it) + for (vector::iterator it = fColumns.begin(); it != fColumns.end(); ++it) + { + SimpleColumn* sc = dynamic_cast(*it); + + if (sc != NULL) { - SimpleColumn* sc = dynamic_cast(*it); + CalpontSystemCatalog::OID oid = sc->oid(); + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct; + uint32_t key = fColumnKeys[std::distance(fColumns.begin(), it)]; - if (sc != NULL) + if (sc->schemaName().empty()) + { + ct = sc->resultType(); + } + else if (sc->isColumnStore() == false) + { + ct = sc->colType(); + } + else + { + ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (dynamic_cast(sc) == NULL) { - CalpontSystemCatalog::OID oid = sc->oid(); - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct; - uint32_t key = fColumnKeys[std::distance(fColumns.begin(), it)]; - - if (sc->schemaName().empty()) - { - ct = sc->resultType(); - } - else if (sc->isColumnStore() == false) - { - ct = sc->colType(); - } - else - { - ct = sc->colType(); - -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (dynamic_cast(sc) == NULL) - { - ct = jobInfo.csc->colType(oid); - ct.charsetNumber =sc->colType().charsetNumber; - } - -//X - dictOid = joblist::isDictCol(ct); - - if (dictOid > 0) - key = jobInfo.keyInfo->dictKeyMap[key]; - } - - sc->inputIndex(indexMap[key]); - - if (jobInfo.trace) - cout << "OID/key:" << (dictOid ? dictOid : oid) << "/" << key << "(" << sc->tableAlias() << "):"; - } - else - { - (*it)->inputIndex(indexMap[getExpTupleKey(jobInfo, (*it)->expressionId())]); - - if (jobInfo.trace) - cout << "EID:" << (*it)->expressionId(); + ct = jobInfo.csc->colType(oid); + ct.charsetNumber = sc->colType().charsetNumber; } - if (jobInfo.trace) - cout << (*it)->inputIndex() << endl; + // X + dictOid = joblist::isDictCol(ct); + + if (dictOid > 0) + key = jobInfo.keyInfo->dictKeyMap[key]; + } + + sc->inputIndex(indexMap[key]); + + if (jobInfo.trace) + cout << "OID/key:" << (dictOid ? dictOid : oid) << "/" << key << "(" << sc->tableAlias() << "):"; + } + else + { + (*it)->inputIndex(indexMap[getExpTupleKey(jobInfo, (*it)->expressionId())]); + + if (jobInfo.trace) + cout << "EID:" << (*it)->expressionId(); } - // if substutes exist, update the original column - for (map::iterator k = fSubMap.begin(); k != fSubMap.end(); k++) - k->second->inputIndex(k->first->inputIndex()); -} + if (jobInfo.trace) + cout << (*it)->inputIndex() << endl; + } + // if substutes exist, update the original column + for (map::iterator k = fSubMap.begin(); k != fSubMap.end(); k++) + k->second->inputIndex(k->first->inputIndex()); +} void ExpressionStep::updateOutputIndex(map& indexMap, const JobInfo& jobInfo) { - fExpression->outputIndex(indexMap[getExpTupleKey(jobInfo, fExpressionId)]); + fExpression->outputIndex(indexMap[getExpTupleKey(jobInfo, fExpressionId)]); - if (jobInfo.trace) - { - cout << "output index of Expression:" << (int64_t) fExpressionId << ":" - << fExpression->outputIndex() << endl << endl; - } + if (jobInfo.trace) + { + cout << "output index of Expression:" << (int64_t)fExpressionId << ":" << fExpression->outputIndex() + << endl + << endl; + } } void ExpressionStep::updateColumnOidAlias(JobInfo& jobInfo) { - for (size_t i = 0; i < fColumns.size(); i++) + for (size_t i = 0; i < fColumns.size(); i++) + { + SimpleColumn* sc = dynamic_cast(fColumns[i]); + + // virtual table columns + if (sc != NULL && sc->schemaName().empty()) { - SimpleColumn* sc = dynamic_cast(fColumns[i]); - - // virtual table columns - if (sc != NULL && sc->schemaName().empty()) - { - fTableOids[i] = joblist::tableOid(sc, jobInfo.csc); - fAliases[i] = extractTableAlias(sc); - } + fTableOids[i] = joblist::tableOid(sc, jobInfo.csc); + fAliases[i] = extractTableAlias(sc); } + } } - void ExpressionStep::substitute(uint64_t i, const SSC& ssc) { - fVsc.insert(ssc); // save a local copy in case the virtual table in subquery be out of scope. - fSubMap[ssc.get()] = fColumns[i]; - fColumns[i] = ssc.get(); + fVsc.insert(ssc); // save a local copy in case the virtual table in subquery be out of scope. + fSubMap[ssc.get()] = fColumns[i]; + fColumns[i] = ssc.get(); } - void ExpressionStep::functionJoinCheck(SimpleFilter* sf, JobInfo& jobInfo) { - if ((sf->lhs()->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - sf->lhs()->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) - throw runtime_error("VARBINARY/BLOB in join is not supported."); + if ((sf->lhs()->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + sf->lhs()->resultType().colDataType == CalpontSystemCatalog::BLOB) && + !fVarBinOK) + throw runtime_error("VARBINARY/BLOB in join is not supported."); - // only handle one & only one table at each side, and not the same table - fFunctionJoinInfo.reset(new FunctionJoinInfo); + // only handle one & only one table at each side, and not the same table + fFunctionJoinInfo.reset(new FunctionJoinInfo); - if ((parseFuncJoinColumn(sf->lhs(), jobInfo) == false) || - (parseFuncJoinColumn(sf->rhs(), jobInfo) == false) || - (fFunctionJoinInfo->fTableKey[0] == fFunctionJoinInfo->fTableKey[1])) + if ((parseFuncJoinColumn(sf->lhs(), jobInfo) == false) || + (parseFuncJoinColumn(sf->rhs(), jobInfo) == false) || + (fFunctionJoinInfo->fTableKey[0] == fFunctionJoinInfo->fTableKey[1])) + { + // not convertible + fFunctionJoinInfo.reset(); + return; + } + + if ((!compatibleColumnTypes(sf->lhs()->resultType(), sf->rhs()->resultType(), true)) && + (fFunctionJoinInfo->fTableKey.size() == 2)) + { + uint32_t t1 = fFunctionJoinInfo->fTableKey[0]; + uint32_t t2 = fFunctionJoinInfo->fTableKey[1]; + jobInfo.incompatibleJoinMap[t1] = t2; + jobInfo.incompatibleJoinMap[t2] = t1; + + // not convertible + fFunctionJoinInfo.reset(); + return; + } + + SJSTEP sjstep; + ReturnedColumn* sfLhs = sf->lhs(); + ReturnedColumn* sfRhs = sf->rhs(); + + if (dynamic_cast(sfLhs) == NULL) + { + SRCP lhs(sfLhs->clone()); + ExpressionStep* esLhs = new ExpressionStep(jobInfo); + esLhs->expression(lhs, jobInfo); + sjstep.reset(esLhs); + } + + fFunctionJoinInfo->fStep.push_back(sjstep); + sjstep.reset(); + + if (dynamic_cast(sfRhs) == NULL) + { + SRCP rhs(sfRhs->clone()); + ExpressionStep* esRhs = new ExpressionStep(jobInfo); + esRhs->expression(rhs, jobInfo); + sjstep.reset(esRhs); + } + + fFunctionJoinInfo->fStep.push_back(sjstep); + + JoinType jt = INNER; + + if (sfLhs->returnAll()) + jt = LEFTOUTER; + else if (sfRhs->returnAll()) + jt = RIGHTOUTER; + + uint64_t joinInfo = sfLhs->joinInfo() | sfRhs->joinInfo(); + int64_t correlatedSide = 0; + ExpressionStep* ve = NULL; + + if (joinInfo != 0) + { + if (joinInfo & JOIN_SEMI) + jt |= SEMI; + + if (joinInfo & JOIN_ANTI) + jt |= ANTI; + + if (joinInfo & JOIN_SCALAR) + jt |= SCALAR; + + if (joinInfo & JOIN_NULL_MATCH) + jt |= MATCHNULLS; + + if (joinInfo & JOIN_CORRELATED) + jt |= CORRELATED; + + if (joinInfo & JOIN_OUTER_SELECT) + jt |= LARGEOUTER; + + if (sfLhs->joinInfo() & JOIN_CORRELATED) { - // not convertible - fFunctionJoinInfo.reset(); - return; + correlatedSide = 1; + ve = dynamic_cast(fFunctionJoinInfo->fStep[1].get()); } - - if ((!compatibleColumnTypes(sf->lhs()->resultType(), sf->rhs()->resultType(), true)) && - (fFunctionJoinInfo->fTableKey.size() == 2)) + else if (sfRhs->joinInfo() & JOIN_CORRELATED) { - uint32_t t1 = fFunctionJoinInfo->fTableKey[0]; - uint32_t t2 = fFunctionJoinInfo->fTableKey[1]; - jobInfo.incompatibleJoinMap[t1] = t2; - jobInfo.incompatibleJoinMap[t2] = t1; - - // not convertible - fFunctionJoinInfo.reset(); - return; + correlatedSide = 2; + ve = dynamic_cast(fFunctionJoinInfo->fStep[0].get()); } + } - SJSTEP sjstep; - ReturnedColumn* sfLhs = sf->lhs(); - ReturnedColumn* sfRhs = sf->rhs(); + if (ve != NULL) + ve->virtualStep(); - if (dynamic_cast(sfLhs) == NULL) - { - SRCP lhs(sfLhs->clone()); - ExpressionStep* esLhs = new ExpressionStep(jobInfo); - esLhs->expression(lhs, jobInfo); - sjstep.reset(esLhs); - } + fFunctionJoinInfo->fJoinType = jt; + fFunctionJoinInfo->fCorrelatedSide = correlatedSide; + fFunctionJoinInfo->fJoinId = ++jobInfo.joinNum; - fFunctionJoinInfo->fStep.push_back(sjstep); - sjstep.reset(); - - if (dynamic_cast(sfRhs) == NULL) - { - SRCP rhs(sfRhs->clone()); - ExpressionStep* esRhs = new ExpressionStep(jobInfo); - esRhs->expression(rhs, jobInfo); - sjstep.reset(esRhs); - } - - fFunctionJoinInfo->fStep.push_back(sjstep); - - JoinType jt = INNER; - - if (sfLhs->returnAll()) - jt = LEFTOUTER; - else if (sfRhs->returnAll()) - jt = RIGHTOUTER; - - uint64_t joinInfo = sfLhs->joinInfo() | sfRhs->joinInfo(); - int64_t correlatedSide = 0; - ExpressionStep* ve = NULL; - - if (joinInfo != 0) - { - if (joinInfo & JOIN_SEMI) - jt |= SEMI; - - if (joinInfo & JOIN_ANTI) - jt |= ANTI; - - if (joinInfo & JOIN_SCALAR) - jt |= SCALAR; - - if (joinInfo & JOIN_NULL_MATCH) - jt |= MATCHNULLS; - - if (joinInfo & JOIN_CORRELATED) - jt |= CORRELATED; - - if (joinInfo & JOIN_OUTER_SELECT) - jt |= LARGEOUTER; - - if (sfLhs->joinInfo() & JOIN_CORRELATED) - { - correlatedSide = 1; - ve = dynamic_cast(fFunctionJoinInfo->fStep[1].get()); - } - else if (sfRhs->joinInfo() & JOIN_CORRELATED) - { - correlatedSide = 2; - ve = dynamic_cast(fFunctionJoinInfo->fStep[0].get()); - } - } - - if (ve != NULL) - ve->virtualStep(); - - fFunctionJoinInfo->fJoinType = jt; - fFunctionJoinInfo->fCorrelatedSide = correlatedSide; - fFunctionJoinInfo->fJoinId = ++jobInfo.joinNum; - - jobInfo.functionJoins.push_back(this); + jobInfo.functionJoins.push_back(this); } - bool ExpressionStep::parseFuncJoinColumn(ReturnedColumn* rc, JobInfo& jobInfo) { - set tids; // tables used in the expression - set cids; // columns used in the expression - uint32_t key = -1; // join key - uint32_t tid = -1; // table Id of the simple column - bool isSc = false; // rc is a simple column + set tids; // tables used in the expression + set cids; // columns used in the expression + uint32_t key = -1; // join key + uint32_t tid = -1; // table Id of the simple column + bool isSc = false; // rc is a simple column - ArithmeticColumn* ac = dynamic_cast(rc); - FunctionColumn* fc = dynamic_cast(rc); - SimpleColumn* sc = dynamic_cast(rc); + ArithmeticColumn* ac = dynamic_cast(rc); + FunctionColumn* fc = dynamic_cast(rc); + SimpleColumn* sc = dynamic_cast(rc); - if (sc != NULL) + if (sc != NULL) + { + isSc = true; + key = getTupleKey(jobInfo, sc); + tid = getTableKey(jobInfo, key); + + if (jobInfo.keyInfo->dictKeyMap.find(key) != jobInfo.keyInfo->dictKeyMap.end()) + key = jobInfo.keyInfo->dictKeyMap[key]; + + tids.insert(tid); + cids.insert(key); + } + else if (ac != NULL || fc != NULL) + { + TupleInfo ti(setExpTupleInfo(rc, jobInfo)); + key = ti.key; + + for (uint32_t i = 0; i < rc->simpleColumnList().size(); i++) { - isSc = true; - key = getTupleKey(jobInfo, sc); - tid = getTableKey(jobInfo, key); - - if (jobInfo.keyInfo->dictKeyMap.find(key) != jobInfo.keyInfo->dictKeyMap.end()) - key = jobInfo.keyInfo->dictKeyMap[key]; - - tids.insert(tid); - cids.insert(key); + sc = rc->simpleColumnList()[i]; + uint32_t cid = getTupleKey(jobInfo, sc); + tid = getTableKey(jobInfo, cid); + tids.insert(tid); + cids.insert(cid); } - else if (ac != NULL || fc != NULL) - { - TupleInfo ti(setExpTupleInfo(rc, jobInfo)); - key = ti.key; + } - for (uint32_t i = 0; i < rc->simpleColumnList().size(); i++) - { - sc = rc->simpleColumnList()[i]; - uint32_t cid = getTupleKey(jobInfo, sc); - tid = getTableKey(jobInfo, cid); - tids.insert(tid); - cids.insert(cid); - } - } + int32_t tableOid = -1; + int32_t oid = -1; + string alias; + string view; + string schema; - int32_t tableOid = -1; - int32_t oid = -1; - string alias; - string view; - string schema; + if (sc && tids.size() == 1) + { + tableOid = joblist::tableOid(sc, jobInfo.csc); + oid = sc->oid(); + alias = extractTableAlias(sc); + view = sc->viewName(); + schema = sc->schemaName(); + } + else if (dynamic_cast(rc) || dynamic_cast(rc) || + dynamic_cast(rc) || dynamic_cast(rc)) + { + tableOid = execplan::CNX_VTABLE_ID; + oid = rc->expressionId(); + alias = jobInfo.subAlias; + } + else + { + return false; + } - if (sc && tids.size() == 1) - { - tableOid = joblist::tableOid(sc, jobInfo.csc); - oid = sc->oid(); - alias = extractTableAlias(sc); - view = sc->viewName(); - schema = sc->schemaName(); - } - else if (dynamic_cast(rc) || dynamic_cast(rc) || - dynamic_cast(rc) || dynamic_cast(rc)) - { - tableOid = execplan::CNX_VTABLE_ID; - oid = rc->expressionId(); - alias = jobInfo.subAlias; - } - else - { - return false; - } + if (isSc == false) + jobInfo.keyInfo->functionJoinKeys.insert(key); - if (isSc == false) - jobInfo.keyInfo->functionJoinKeys.insert(key); + fFunctionJoinInfo->fExpression.push_back(rc); + fFunctionJoinInfo->fJoinKey.push_back(key); + fFunctionJoinInfo->fTableKey.push_back(tid); + fFunctionJoinInfo->fColumnKeys.push_back(cids); + fFunctionJoinInfo->fTableOid.push_back(tableOid); + fFunctionJoinInfo->fOid.push_back(oid); + fFunctionJoinInfo->fSequence.push_back(rc->sequence()); + fFunctionJoinInfo->fAlias.push_back(alias); + fFunctionJoinInfo->fView.push_back(view); + fFunctionJoinInfo->fSchema.push_back(schema); - fFunctionJoinInfo->fExpression.push_back(rc); - fFunctionJoinInfo->fJoinKey.push_back(key); - fFunctionJoinInfo->fTableKey.push_back(tid); - fFunctionJoinInfo->fColumnKeys.push_back(cids); - fFunctionJoinInfo->fTableOid.push_back(tableOid); - fFunctionJoinInfo->fOid.push_back(oid); - fFunctionJoinInfo->fSequence.push_back(rc->sequence()); - fFunctionJoinInfo->fAlias.push_back(alias); - fFunctionJoinInfo->fView.push_back(view); - fFunctionJoinInfo->fSchema.push_back(schema); - - return true; + return true; } - const string ExpressionStep::toString() const { - ostringstream oss; - oss << "ExpressionStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "ExpressionStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " in:"; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); - return oss.str(); + return oss.str(); } - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/expressionstep.h b/dbcon/joblist/expressionstep.h index 1974ad42e..3b8446f75 100644 --- a/dbcon/joblist/expressionstep.h +++ b/dbcon/joblist/expressionstep.h @@ -17,7 +17,6 @@ // $Id: expressionstep.h 9632 2013-06-18 22:18:20Z xlou $ - /** @file * class ExpStep interface */ @@ -29,7 +28,6 @@ #include "jobstep.h" #include "filter.h" - namespace execplan { // forward reference @@ -37,239 +35,235 @@ class ReturnedColumn; class SimpleColumn; class SimpleFilter; class WindowFunctionColumn; -}; - +}; // namespace execplan namespace joblist { - struct JobInfo; struct FunctionJoinInfo; class ExpressionStep : public JobStep { -public: - // constructors - ExpressionStep(); - ExpressionStep(const JobInfo&); - // destructor constructors - virtual ~ExpressionStep(); + public: + // constructors + ExpressionStep(); + ExpressionStep(const JobInfo&); + // destructor constructors + virtual ~ExpressionStep(); - // inherited methods - void run(); - void join(); - const std::string toString() const; + // inherited methods + void run(); + void join(); + const std::string toString() const; - execplan::CalpontSystemCatalog::OID oid() const - { - return 0; - } - execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOids.empty() ? 0 : fTableOids.front(); - } - using JobStep::alias; - std::string alias() const - { - return fAliases.empty() ? "" : fAliases.front(); - } - using JobStep::view; - std::string view() const - { - return fViews.empty() ? "" : fViews.front(); - } - using JobStep::schema; - std::string schema() const - { - return fSchemas.empty() ? "" : fSchemas.front(); - } - uint32_t tableKey() const - { - return fTableKeys.empty() ? -1 : fTableKeys.front(); - } - uint32_t columnKey() const - { - return fColumnKeys.empty() ? -1 : fColumnKeys.front(); - } + execplan::CalpontSystemCatalog::OID oid() const + { + return 0; + } + execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOids.empty() ? 0 : fTableOids.front(); + } + using JobStep::alias; + std::string alias() const + { + return fAliases.empty() ? "" : fAliases.front(); + } + using JobStep::view; + std::string view() const + { + return fViews.empty() ? "" : fViews.front(); + } + using JobStep::schema; + std::string schema() const + { + return fSchemas.empty() ? "" : fSchemas.front(); + } + uint32_t tableKey() const + { + return fTableKeys.empty() ? -1 : fTableKeys.front(); + } + uint32_t columnKey() const + { + return fColumnKeys.empty() ? -1 : fColumnKeys.front(); + } - void expression(const execplan::SRCP exp, JobInfo& jobInfo); - execplan::SRCP expression() const - { - return fExpression; - } + void expression(const execplan::SRCP exp, JobInfo& jobInfo); + execplan::SRCP expression() const + { + return fExpression; + } - virtual void expressionFilter(const execplan::Filter* filter, JobInfo& jobInfo); - virtual void expressionFilter(const execplan::ParseTree* filter, JobInfo& jobInfo); - execplan::ParseTree* expressionFilter() const - { - return fExpressionFilter; - } + virtual void expressionFilter(const execplan::Filter* filter, JobInfo& jobInfo); + virtual void expressionFilter(const execplan::ParseTree* filter, JobInfo& jobInfo); + execplan::ParseTree* expressionFilter() const + { + return fExpressionFilter; + } - void expressionId(uint64_t eid) - { - fExpressionId = eid; - } - uint64_t expressionId() const - { - return fExpressionId; - } + void expressionId(uint64_t eid) + { + fExpressionId = eid; + } + uint64_t expressionId() const + { + return fExpressionId; + } - const std::vector& tableOids() const - { - return fTableOids; - } - const std::vector& aliases() const - { - return fAliases; - } - const std::vector& views() const - { - return fViews; - } - const std::vector& schemas() const - { - return fSchemas; - } - const std::vector& tableKeys() const - { - return fTableKeys; - } - const std::vector& columnKeys() const - { - return fColumnKeys; - } + const std::vector& tableOids() const + { + return fTableOids; + } + const std::vector& aliases() const + { + return fAliases; + } + const std::vector& views() const + { + return fViews; + } + const std::vector& schemas() const + { + return fSchemas; + } + const std::vector& tableKeys() const + { + return fTableKeys; + } + const std::vector& columnKeys() const + { + return fColumnKeys; + } - std::vector& tableOids() - { - return fTableOids; - } - std::vector& aliases() - { - return fAliases; - } - std::vector& views() - { - return fViews; - } - std::vector& schemas() - { - return fSchemas; - } - std::vector& tableKeys() - { - return fTableKeys; - } - std::vector& columnKeys() - { - return fColumnKeys; - } + std::vector& tableOids() + { + return fTableOids; + } + std::vector& aliases() + { + return fAliases; + } + std::vector& views() + { + return fViews; + } + std::vector& schemas() + { + return fSchemas; + } + std::vector& tableKeys() + { + return fTableKeys; + } + std::vector& columnKeys() + { + return fColumnKeys; + } - virtual void updateInputIndex(std::map& indexMap, const JobInfo& jobInfo); - virtual void updateOutputIndex(std::map& indexMap, const JobInfo& jobInfo); - virtual void updateColumnOidAlias(JobInfo& jobInfo); + virtual void updateInputIndex(std::map& indexMap, const JobInfo& jobInfo); + virtual void updateOutputIndex(std::map& indexMap, const JobInfo& jobInfo); + virtual void updateColumnOidAlias(JobInfo& jobInfo); - std::vector& columns() - { - return fColumns; - } - void substitute(uint64_t, const SSC&); + std::vector& columns() + { + return fColumns; + } + void substitute(uint64_t, const SSC&); - void selectFilter(bool b) - { - fSelectFilter = b; - } - bool selectFilter() const - { - return fSelectFilter; - } + void selectFilter(bool b) + { + fSelectFilter = b; + } + bool selectFilter() const + { + return fSelectFilter; + } - void associatedJoinId(uint64_t i) - { - fAssociatedJoinId = i; - } - uint64_t associatedJoinId() const - { - return fAssociatedJoinId; - } + void associatedJoinId(uint64_t i) + { + fAssociatedJoinId = i; + } + uint64_t associatedJoinId() const + { + return fAssociatedJoinId; + } - void functionJoin(bool b) - { - fDoJoin = b; - } - bool functionJoin() const - { - return fDoJoin; - } + void functionJoin(bool b) + { + fDoJoin = b; + } + bool functionJoin() const + { + return fDoJoin; + } - void virtualStep(bool b) - { - fVirtual = b; - } - bool virtualStep() const - { - return fVirtual; - } + void virtualStep(bool b) + { + fVirtual = b; + } + bool virtualStep() const + { + return fVirtual; + } - boost::shared_ptr& functionJoinInfo() - { - return fFunctionJoinInfo; - } - void resetJoinInfo() - { - fFunctionJoinInfo.reset(); - } + boost::shared_ptr& functionJoinInfo() + { + return fFunctionJoinInfo; + } + void resetJoinInfo() + { + fFunctionJoinInfo.reset(); + } -protected: - virtual void addColumn(execplan::ReturnedColumn* rc, JobInfo& jobInfo); - virtual void addFilter(execplan::ParseTree* filter, JobInfo& jobInfo); - virtual void addSimpleFilter(execplan::SimpleFilter* sf, JobInfo& jobInfo); - virtual void populateColumnInfo(execplan::ReturnedColumn* rc, JobInfo& jobInfo); - virtual void populateColumnInfo(execplan::SimpleColumn* sc, JobInfo& jobInfo); - virtual void populateColumnInfo(execplan::WindowFunctionColumn* wc, JobInfo& jobInfo); - virtual void populateColumnInfo(execplan::AggregateColumn* ac, JobInfo& jobInfo); - virtual void functionJoinCheck(execplan::SimpleFilter* sf, JobInfo& jobInfo); - virtual bool parseFuncJoinColumn(ReturnedColumn* rc, JobInfo& jobInfo); + protected: + virtual void addColumn(execplan::ReturnedColumn* rc, JobInfo& jobInfo); + virtual void addFilter(execplan::ParseTree* filter, JobInfo& jobInfo); + virtual void addSimpleFilter(execplan::SimpleFilter* sf, JobInfo& jobInfo); + virtual void populateColumnInfo(execplan::ReturnedColumn* rc, JobInfo& jobInfo); + virtual void populateColumnInfo(execplan::SimpleColumn* sc, JobInfo& jobInfo); + virtual void populateColumnInfo(execplan::WindowFunctionColumn* wc, JobInfo& jobInfo); + virtual void populateColumnInfo(execplan::AggregateColumn* ac, JobInfo& jobInfo); + virtual void functionJoinCheck(execplan::SimpleFilter* sf, JobInfo& jobInfo); + virtual bool parseFuncJoinColumn(ReturnedColumn* rc, JobInfo& jobInfo); + // expression + execplan::SRCP fExpression; + execplan::ParseTree* fExpressionFilter; + uint64_t fExpressionId; - // expression - execplan::SRCP fExpression; - execplan::ParseTree* fExpressionFilter; - uint64_t fExpressionId; + // columns accessed + std::vector fTableOids; + std::vector fAliases; + std::vector fViews; + std::vector fSchemas; + std::vector fTableKeys; + std::vector fColumnKeys; + std::vector fColumns; - // columns accessed - std::vector fTableOids; - std::vector fAliases; - std::vector fViews; - std::vector fSchemas; - std::vector fTableKeys; - std::vector fColumnKeys; - std::vector fColumns; + private: + // disable copy constructor + // Cannot copy fColumns, which depends on fExpressionFilter. + ExpressionStep(const ExpressionStep&); -private: - // disable copy constructor - // Cannot copy fColumns, which depends on fExpressionFilter. - ExpressionStep(const ExpressionStep&); + // for VARBINARY, only support limited number of functions: like hex(), length(), etc. + bool fVarBinOK; - // for VARBINARY, only support limited number of functions: like hex(), length(), etc. - bool fVarBinOK; + // @bug 3780, for select filter + bool fSelectFilter; - // @bug 3780, for select filter - bool fSelectFilter; + // @bug 3037, outer join with additional comparison + uint64_t fAssociatedJoinId; - // @bug 3037, outer join with additional comparison - uint64_t fAssociatedJoinId; + // @bug 4531, for window function in IN/EXISTS sub-query. + std::map fSubMap; + std::set fVsc; // for substitute wc with vsc - // @bug 4531, for window function in IN/EXISTS sub-query. - std::map fSubMap; - std::set fVsc; // for substitute wc with vsc - - // @bug 3683, function join - boost::shared_ptr fFunctionJoinInfo; - bool fDoJoin; - bool fVirtual; + // @bug 3683, function join + boost::shared_ptr fFunctionJoinInfo; + bool fDoJoin; + bool fVirtual; }; -} - -#endif // JOBLIST_EXPRESSION_STEP_H +} // namespace joblist +#endif // JOBLIST_EXPRESSION_STEP_H diff --git a/dbcon/joblist/fifo.h b/dbcon/joblist/fifo.h index 429572ed5..62bf89352 100644 --- a/dbcon/joblist/fifo.h +++ b/dbcon/joblist/fifo.h @@ -20,7 +20,6 @@ * *****************************************************************************/ - /** @file * class XXX interface */ @@ -38,147 +37,144 @@ namespace joblist { - /** @brief class FIFO * */ /* This derives from DataListImpl > but it manages its own data */ -template +template class FIFO : public DataListImpl, element_t> { -private: - typedef DataListImpl, element_t> base; + private: + typedef DataListImpl, element_t> base; -public: - enum ElementMode - { - RID_ONLY, - RID_VALUE - }; + public: + enum ElementMode + { + RID_ONLY, + RID_VALUE + }; - FIFO(uint32_t numConsumers, uint32_t maxElements); - virtual ~FIFO(); + FIFO(uint32_t numConsumers, uint32_t maxElements); + virtual ~FIFO(); - /* DataList interface */ - inline void insert(const element_t& e); - inline void insert(const std::vector& v); - inline bool next(uint64_t it, element_t* e); - uint64_t getIterator(); - void endOfInput(); - void setMultipleProducers(bool b); + /* DataList interface */ + inline void insert(const element_t& e); + inline void insert(const std::vector& v); + inline bool next(uint64_t it, element_t* e); + uint64_t getIterator(); + void endOfInput(); + void setMultipleProducers(bool b); - /* Use this insert() to detect when insertion fills up buffer. */ - /* When this happens, call waitTillReadyForInserts() before resuming*/ - /* with more inserts. */ - inline void insert(const element_t& e, - bool& bufferFullBlocked, bool& consumptionStarted); - inline void waitTillReadyForInserts(); - inline bool isOutputBlocked() const; + /* Use this insert() to detect when insertion fills up buffer. */ + /* When this happens, call waitTillReadyForInserts() before resuming*/ + /* with more inserts. */ + inline void insert(const element_t& e, bool& bufferFullBlocked, bool& consumptionStarted); + inline void waitTillReadyForInserts(); + inline bool isOutputBlocked() const; - void OID(execplan::CalpontSystemCatalog::OID oid) - { - base::OID(oid); - } - execplan::CalpontSystemCatalog::OID OID() const - { - return base::OID(); - } + void OID(execplan::CalpontSystemCatalog::OID oid) + { + base::OID(oid); + } + execplan::CalpontSystemCatalog::OID OID() const + { + return base::OID(); + } - inline void dropToken() { }; - inline void dropToken(uint32_t) { }; + inline void dropToken(){}; + inline void dropToken(uint32_t){}; - // Counters that reflect how many many times this FIFO blocked on reads/writes - uint64_t blockedWriteCount() const; - uint64_t blockedReadCount() const; + // Counters that reflect how many many times this FIFO blocked on reads/writes + uint64_t blockedWriteCount() const; + uint64_t blockedReadCount() const; - // @bug 653 set number of consumers when it is empty. - void setNumConsumers( uint32_t nc ); + // @bug 653 set number of consumers when it is empty. + void setNumConsumers(uint32_t nc); - void inOrder(bool order) - { - fInOrder = order; - } - bool inOrder() const - { - return fInOrder; - } + void inOrder(bool order) + { + fInOrder = order; + } + bool inOrder() const + { + return fInOrder; + } - // Default behavior tracks number of element inserts. If application - // is inserting complex elements (ex: RowWrappers), then application - // should call totalSize() mutator to record accurate element count. - // This is not a blocking call. If totalSize() accessor is called - // prior to completion, the "current" total size will be returned. - void totalSize(const uint64_t totSize) - { - fTotSize = totSize; - } - uint64_t totalSize() - { - return fTotSize; - } + // Default behavior tracks number of element inserts. If application + // is inserting complex elements (ex: RowWrappers), then application + // should call totalSize() mutator to record accurate element count. + // This is not a blocking call. If totalSize() accessor is called + // prior to completion, the "current" total size will be returned. + void totalSize(const uint64_t totSize) + { + fTotSize = totSize; + } + uint64_t totalSize() + { + return fTotSize; + } - void maxElements(uint64_t max); - uint64_t maxElements() - { - return fMaxElements; - } + void maxElements(uint64_t max); + uint64_t maxElements() + { + return fMaxElements; + } - // FIFO only uses elementmode to control how the elements are saved - // to temp disk, should the application choose to page to disk. If - // a FIFO is converted to another datalist, this enum should be copied - // over to the datalist, as a ZDL for example can use the element mode - // for other reasons. - void setElementMode(uint32_t mode) - { - fElementMode = mode; - } - uint32_t getElementMode() const - { - return fElementMode; - } + // FIFO only uses elementmode to control how the elements are saved + // to temp disk, should the application choose to page to disk. If + // a FIFO is converted to another datalist, this enum should be copied + // over to the datalist, as a ZDL for example can use the element mode + // for other reasons. + void setElementMode(uint32_t mode) + { + fElementMode = mode; + } + uint32_t getElementMode() const + { + return fElementMode; + } - // Total number of files and filespace used for temp files - void setTotalFileCounts(uint64_t numFiles, uint64_t numBytes); - void totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) const; + // Total number of files and filespace used for temp files + void setTotalFileCounts(uint64_t numFiles, uint64_t numBytes); + void totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) const; - // returns true if there might be more data to read, - // false if there is no more data. Similar to next(), but - // does not return data. - bool more(uint64_t id); -protected: + // returns true if there might be more data to read, + // false if there is no more data. Similar to next(), but + // does not return data. + bool more(uint64_t id); -private: - boost::condition finishedConsuming, moreData; + protected: + private: + boost::condition finishedConsuming, moreData; - element_t* pBuffer; - element_t* cBuffer; - uint64_t ppos; - uint64_t* cpos; - uint64_t cDone; - uint64_t fMaxElements; - uint64_t cWaiting; - uint64_t fTotSize; - bool fInOrder; - uint64_t fConsumerFinishedCount; - volatile bool fConsumptionStarted; - uint32_t fElementMode; - uint64_t fNumFiles; - uint64_t fNumBytes; + element_t* pBuffer; + element_t* cBuffer; + uint64_t ppos; + uint64_t* cpos; + uint64_t cDone; + uint64_t fMaxElements; + uint64_t cWaiting; + uint64_t fTotSize; + bool fInOrder; + uint64_t fConsumerFinishedCount; + volatile bool fConsumptionStarted; + uint32_t fElementMode; + uint64_t fNumFiles; + uint64_t fNumBytes; - // Counters that reflect how many many times this FIFO blocked - // on reads and writes due to the FIFO being empty or full. - uint64_t blockedInsertWriteCount; - uint64_t blockedNextReadCount; + // Counters that reflect how many many times this FIFO blocked + // on reads and writes due to the FIFO being empty or full. + uint64_t blockedInsertWriteCount; + uint64_t blockedNextReadCount; - FIFO& operator=(const FIFO&); - FIFO(const FIFO&); - FIFO(); - - void signalPs(); - bool swapBuffers(bool waitIfBlocked = true); - bool waitForSwap(uint64_t id); + FIFO& operator=(const FIFO&); + FIFO(const FIFO&); + FIFO(); + void signalPs(); + bool swapBuffers(bool waitIfBlocked = true); + bool waitForSwap(uint64_t id); }; // #define FIFO_DEBUG @@ -187,332 +183,331 @@ private: // Need to bench both ways before changing it. // #define ONE_CS -template +template FIFO::FIFO(uint32_t con, uint32_t max) : DataListImpl, element_t>(con) { - fMaxElements = max; - pBuffer = 0; - cBuffer = 0; - cpos = new uint64_t[con]; - ppos = 0; - cWaiting = 0; - fTotSize = 0; - fInOrder = false; - fConsumerFinishedCount = 0; - fConsumptionStarted = false; - fElementMode = RID_ONLY; - fNumFiles = 0; - fNumBytes = 0; + fMaxElements = max; + pBuffer = 0; + cBuffer = 0; + cpos = new uint64_t[con]; + ppos = 0; + cWaiting = 0; + fTotSize = 0; + fInOrder = false; + fConsumerFinishedCount = 0; + fConsumptionStarted = false; + fElementMode = RID_ONLY; + fNumFiles = 0; + fNumBytes = 0; - for (uint64_t i = 0; i < con; ++i) - cpos[i] = fMaxElements; + for (uint64_t i = 0; i < con; ++i) + cpos[i] = fMaxElements; - cDone = con; + cDone = con; - blockedInsertWriteCount = blockedNextReadCount = 0; + blockedInsertWriteCount = blockedNextReadCount = 0; } -template +template FIFO::FIFO() { - throw std::logic_error("don't use FIFO()"); + throw std::logic_error("don't use FIFO()"); } -template +template FIFO::FIFO(const FIFO& f) { - throw std::logic_error("don't use FIFO(FIFO &)"); + throw std::logic_error("don't use FIFO(FIFO &)"); } -template +template FIFO& FIFO::operator=(const FIFO& f) { - throw std::logic_error("don't use FIFO:: ="); + throw std::logic_error("don't use FIFO:: ="); } -template +template FIFO::~FIFO() { - if (pBuffer) - delete [] pBuffer; + if (pBuffer) + delete[] pBuffer; - if (cBuffer) - delete [] cBuffer; + if (cBuffer) + delete[] cBuffer; - delete [] cpos; + delete[] cpos; } // if waitIfBlocked is false, the return value indicates if the swap blocked -template +template bool FIFO::swapBuffers(bool waitIfBlocked) { - element_t* tmp; + element_t* tmp; - boost::mutex::scoped_lock scoped(base::mutex); + boost::mutex::scoped_lock scoped(base::mutex); - if (cDone < base::numConsumers) + if (cDone < base::numConsumers) + { + blockedInsertWriteCount++; + + if (!waitIfBlocked) + return true; + + while (cDone < base::numConsumers) + finishedConsuming.wait(scoped); + } + + tmp = pBuffer; + pBuffer = cBuffer; + cBuffer = tmp; + cDone = 0; + ppos = 0; + memset(cpos, 0, sizeof(*cpos) * base::numConsumers); + + if (cWaiting) + { + moreData.notify_all(); + cWaiting = 0; + } + + return false; +} + +template +inline void FIFO::insert(const element_t& e) +{ + if (!pBuffer) + { + pBuffer = new element_t[fMaxElements]; + cBuffer = new element_t[fMaxElements]; + } + + pBuffer[ppos++] = e; + fTotSize++; + + if (ppos == fMaxElements) + swapBuffers(); +} + +// version of insert that will return rather than block if insert would block +template +inline void FIFO::insert(const element_t& e, bool& bufferFullBlocked, bool& consumptionStarted) +{ + if (!pBuffer) + { + pBuffer = new element_t[fMaxElements]; + cBuffer = new element_t[fMaxElements]; + } + + pBuffer[ppos++] = e; + fTotSize++; + + bufferFullBlocked = false; + consumptionStarted = fConsumptionStarted; + + if (ppos == fMaxElements) + bufferFullBlocked = swapBuffers(false); +} + +template +inline void FIFO::waitTillReadyForInserts() +{ + if (ppos == fMaxElements) + swapBuffers(); +} + +template +inline bool FIFO::isOutputBlocked() const +{ + if (ppos == fMaxElements) + return true; + else + return false; +} + +template +inline void FIFO::insert(const std::vector& e) +{ + typename std::vector::const_iterator it = e.begin(); + typename std::vector::const_iterator end = e.end(); + + while (it != end) + { + insert(*it); + ++it; + } +} + +template +bool FIFO::waitForSwap(uint64_t id) +{ + boost::mutex::scoped_lock scoped(base::mutex); + +#ifdef ONE_CS + + if (cpos[id] == fMaxElements) + if (++cDone == base::numConsumers) + finishedConsuming.notify_all() +#endif + while (cpos[id] == fMaxElements && !base::noMoreInput) + { + ++cWaiting; + blockedNextReadCount++; + moreData.wait(scoped); + } + + if (cpos[id] == fMaxElements) + { + // Before we free the lock, let's check to see if all our consumers + // are finished, in which case we can delete our data buffers. + if (++fConsumerFinishedCount == base::numConsumers) { - blockedInsertWriteCount++; - - if (!waitIfBlocked) - return true; - - while (cDone < base::numConsumers) - finishedConsuming.wait(scoped); + delete[] pBuffer; + delete[] cBuffer; + pBuffer = 0; + cBuffer = 0; } + return false; + } + + return true; +} + +template +bool FIFO::more(uint64_t id) +{ + boost::mutex::scoped_lock scoped(base::mutex); + return !(cpos[id] == fMaxElements && base::noMoreInput); +} + +template +void FIFO::signalPs() +{ + boost::mutex::scoped_lock scoped(base::mutex); + + if (++cDone == base::numConsumers) + finishedConsuming.notify_all(); +} + +template +inline bool FIFO::next(uint64_t id, element_t* out) +{ + base::mutex.lock(); + fConsumptionStarted = true; + + if (cpos[id] >= fMaxElements) + { + base::mutex.unlock(); + if (!waitForSwap(id)) + return false; + base::mutex.lock(); + } + + *out = cBuffer[cpos[id]++]; + +#ifndef ONE_CS + + if (cpos[id] == fMaxElements) + { + base::mutex.unlock(); + signalPs(); + return true; + } +#endif + base::mutex.unlock(); + return true; +} + +template +void FIFO::endOfInput() +{ + element_t* tmp; + + boost::mutex::scoped_lock scoped(base::mutex); + + if (ppos != 0) + { + while (cDone < base::numConsumers) + finishedConsuming.wait(scoped); + + fMaxElements = ppos; tmp = pBuffer; pBuffer = cBuffer; cBuffer = tmp; cDone = 0; - ppos = 0; memset(cpos, 0, sizeof(*cpos) * base::numConsumers); + } - if (cWaiting) - { - moreData.notify_all(); - cWaiting = 0; - } + base::endOfInput(); - return false; + if (cWaiting) + moreData.notify_all(); } -template -inline void FIFO::insert(const element_t& e) -{ - if (!pBuffer) - { - pBuffer = new element_t[fMaxElements]; - cBuffer = new element_t[fMaxElements]; - } - - pBuffer[ppos++] = e; - fTotSize++; - - if (ppos == fMaxElements) - swapBuffers(); -} - -// version of insert that will return rather than block if insert would block -template -inline void FIFO::insert(const element_t& e, - bool& bufferFullBlocked, bool& consumptionStarted) -{ - if (!pBuffer) - { - pBuffer = new element_t[fMaxElements]; - cBuffer = new element_t[fMaxElements]; - } - - pBuffer[ppos++] = e; - fTotSize++; - - bufferFullBlocked = false; - consumptionStarted = fConsumptionStarted; - - if (ppos == fMaxElements) - bufferFullBlocked = swapBuffers(false); -} - -template -inline void FIFO::waitTillReadyForInserts() -{ - if (ppos == fMaxElements) - swapBuffers(); -} - -template -inline bool FIFO::isOutputBlocked() const -{ - if (ppos == fMaxElements) - return true; - else - return false; -} - -template -inline void FIFO::insert(const std::vector& e) -{ - typename std::vector::const_iterator it = e.begin(); - typename std::vector::const_iterator end = e.end(); - - while (it != end) - { - insert(*it); - ++it; - } -} - -template -bool FIFO::waitForSwap(uint64_t id) -{ - boost::mutex::scoped_lock scoped(base::mutex); - -#ifdef ONE_CS - - if (cpos[id] == fMaxElements) - if (++cDone == base::numConsumers) - finishedConsuming.notify_all() -#endif - while (cpos[id] == fMaxElements && !base::noMoreInput) - { - ++cWaiting; - blockedNextReadCount++; - moreData.wait(scoped); - } - - if (cpos[id] == fMaxElements) - { - // Before we free the lock, let's check to see if all our consumers - // are finished, in which case we can delete our data buffers. - if (++fConsumerFinishedCount == base::numConsumers) - { - delete [] pBuffer; - delete [] cBuffer; - pBuffer = 0; - cBuffer = 0; - } - - return false; - } - - return true; -} - -template -bool FIFO::more(uint64_t id) -{ - boost::mutex::scoped_lock scoped(base::mutex); - return !(cpos[id] == fMaxElements && base::noMoreInput); -} - -template -void FIFO::signalPs() -{ - boost::mutex::scoped_lock scoped(base::mutex); - - if (++cDone == base::numConsumers) - finishedConsuming.notify_all(); -} - -template -inline bool FIFO::next(uint64_t id, element_t* out) -{ - base::mutex.lock(); - fConsumptionStarted = true; - - if (cpos[id] >= fMaxElements) - { - base::mutex.unlock(); - if (!waitForSwap(id)) - return false; - base::mutex.lock(); - } - - *out = cBuffer[cpos[id]++]; - -#ifndef ONE_CS - - if (cpos[id] == fMaxElements) - { - base::mutex.unlock(); - signalPs(); - return true; - } -#endif - base::mutex.unlock(); - return true; -} - -template -void FIFO::endOfInput() -{ - element_t* tmp; - - boost::mutex::scoped_lock scoped(base::mutex); - - if (ppos != 0) - { - while (cDone < base::numConsumers) - finishedConsuming.wait(scoped); - - fMaxElements = ppos; - tmp = pBuffer; - pBuffer = cBuffer; - cBuffer = tmp; - cDone = 0; - memset(cpos, 0, sizeof(*cpos) * base::numConsumers); - } - - base::endOfInput(); - - if (cWaiting) - moreData.notify_all(); -} - -template +template uint64_t FIFO::getIterator() { - uint64_t ret; + uint64_t ret; - boost::mutex::scoped_lock scoped(base::mutex); - ret = base::getIterator(); - return ret; + boost::mutex::scoped_lock scoped(base::mutex); + ret = base::getIterator(); + return ret; } -template +template uint64_t FIFO::blockedWriteCount() const { - return blockedInsertWriteCount; + return blockedInsertWriteCount; } -template +template uint64_t FIFO::blockedReadCount() const { - return blockedNextReadCount; + return blockedNextReadCount; } -template +template void FIFO::setMultipleProducers(bool b) { - if (b) - throw std::logic_error("FIFO: setMultipleProducers() doesn't work yet"); + if (b) + throw std::logic_error("FIFO: setMultipleProducers() doesn't work yet"); } //@bug 653 -template -void FIFO::setNumConsumers( uint32_t nc ) +template +void FIFO::setNumConsumers(uint32_t nc) { - delete [] cpos; - base::setNumConsumers(nc); - cpos = new uint64_t[nc]; + delete[] cpos; + base::setNumConsumers(nc); + cpos = new uint64_t[nc]; - for (uint64_t i = 0; i < nc; ++i) - cpos[i] = fMaxElements; + for (uint64_t i = 0; i < nc; ++i) + cpos[i] = fMaxElements; - cDone = nc; + cDone = nc; } //@bug 864 -template +template void FIFO::maxElements(uint64_t max) { - if (fMaxElements != max) - { - fMaxElements = max; + if (fMaxElements != max) + { + fMaxElements = max; - if (pBuffer) - delete [] pBuffer; + if (pBuffer) + delete[] pBuffer; - if (cBuffer) - delete [] cBuffer; + if (cBuffer) + delete[] cBuffer; - pBuffer = 0; - cBuffer = 0; + pBuffer = 0; + cBuffer = 0; - for (uint64_t i = 0; i < base::numConsumers; ++i) - cpos[i] = fMaxElements; - } + for (uint64_t i = 0; i < base::numConsumers; ++i) + cpos[i] = fMaxElements; + } } // @@ -520,22 +515,20 @@ void FIFO::maxElements(uint64_t max) // (in bytes) by this FIFO collection, if the application code chose to cache // to disk. // -template +template void FIFO::setTotalFileCounts(uint64_t numFiles, uint64_t numBytes) { - fNumFiles = numFiles; - fNumBytes = numBytes; + fNumFiles = numFiles; + fNumBytes = numBytes; } -template -void FIFO::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) -const +template +void FIFO::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) const { - numFiles = fNumFiles; - numBytes = fNumBytes; + numFiles = fNumFiles; + numBytes = fNumBytes; } -} // namespace +} // namespace joblist #endif // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/filtercommand-jl.cpp b/dbcon/joblist/filtercommand-jl.cpp index 47d6a4836..f7c4ce73a 100644 --- a/dbcon/joblist/filtercommand-jl.cpp +++ b/dbcon/joblist/filtercommand-jl.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: filtercommand-jl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: filtercommand-jl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "bpp-jl.h" @@ -28,63 +28,52 @@ using namespace messageqcpp; namespace joblist { - -FilterCommandJL::FilterCommandJL(const FilterStep& step) : - fBOP(step.BOP()), fColType(step.colType()) +FilterCommandJL::FilterCommandJL(const FilterStep& step) : fBOP(step.BOP()), fColType(step.colType()) { - OID = 0; - colName = step.name(); + OID = 0; + colName = step.name(); } - FilterCommandJL::~FilterCommandJL() { } - void FilterCommandJL::setLBID(uint64_t rid, uint32_t dbroot) { } - uint8_t FilterCommandJL::getTableColumnType() { - throw logic_error("Don't call FilterCommandJL::getTableColumn(); it's not a projection step"); + throw logic_error("Don't call FilterCommandJL::getTableColumn(); it's not a projection step"); } - CommandJL::CommandType FilterCommandJL::getCommandType() { - return FILTER_COMMAND; + return FILTER_COMMAND; } - string FilterCommandJL::toString() { - ostringstream ret; + ostringstream ret; - ret << "FilterCommandJL: " << colName << " BOP=" << (uint32_t) fBOP; - return ret.str(); + ret << "FilterCommandJL: " << colName << " BOP=" << (uint32_t)fBOP; + return ret.str(); } - void FilterCommandJL::createCommand(ByteStream& bs) const { - bs << (uint8_t) FILTER_COMMAND; - bs << fBOP; - CommandJL::createCommand(bs); + bs << (uint8_t)FILTER_COMMAND; + bs << fBOP; + CommandJL::createCommand(bs); } - void FilterCommandJL::runCommand(ByteStream& bs) const { } - uint16_t FilterCommandJL::getWidth() { - return fColType.colWidth; + return fColType.colWidth; } - -}; +}; // namespace joblist diff --git a/dbcon/joblist/filtercommand-jl.h b/dbcon/joblist/filtercommand-jl.h index 99ec566a0..df4210ca7 100644 --- a/dbcon/joblist/filtercommand-jl.h +++ b/dbcon/joblist/filtercommand-jl.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: filtercommand-jl.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: filtercommand-jl.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file * class FilterCommand interface */ @@ -32,35 +32,32 @@ namespace joblist { - class FilterCommandJL : public CommandJL { -public: - FilterCommandJL(const FilterStep&); - virtual ~FilterCommandJL(); + public: + FilterCommandJL(const FilterStep&); + virtual ~FilterCommandJL(); - void setLBID(uint64_t rid, uint32_t dbroot); - uint8_t getTableColumnType(); - CommandType getCommandType(); - std::string toString(); - void createCommand(messageqcpp::ByteStream& bs) const; - void runCommand(messageqcpp::ByteStream& bs) const; - uint16_t getWidth(); - uint8_t getBOP() const - { - return fBOP; - }; + void setLBID(uint64_t rid, uint32_t dbroot); + uint8_t getTableColumnType(); + CommandType getCommandType(); + std::string toString(); + void createCommand(messageqcpp::ByteStream& bs) const; + void runCommand(messageqcpp::ByteStream& bs) const; + uint16_t getWidth(); + uint8_t getBOP() const + { + return fBOP; + }; -private: - FilterCommandJL(); - FilterCommandJL(const FilterCommandJL&); + private: + FilterCommandJL(); + FilterCommandJL(const FilterCommandJL&); - uint8_t fBOP; - execplan::CalpontSystemCatalog::ColType fColType; + uint8_t fBOP; + execplan::CalpontSystemCatalog::ColType fColType; }; -}; - - -#endif // JOBLIST_FILTERCOMMANDJL_H_ +}; // namespace joblist +#endif // JOBLIST_FILTERCOMMANDJL_H_ diff --git a/dbcon/joblist/filterstep.cpp b/dbcon/joblist/filterstep.cpp index f90bc4b45..c5d08e891 100644 --- a/dbcon/joblist/filterstep.cpp +++ b/dbcon/joblist/filterstep.cpp @@ -33,13 +33,12 @@ using namespace execplan; using namespace joblist; // move to header file -//const uint32_t defaultFlushInterval = 0x2000; +// const uint32_t defaultFlushInterval = 0x2000; namespace joblist { - ////@bug 686. Make filterstep doing jobs in seperate thread and return to main immediately. //// So the other job steps can start. -//struct FSRunner +// struct FSRunner //{ // FSRunner(FilterStep* p) : joiner(p) // {} @@ -61,12 +60,8 @@ namespace joblist // } //}; -FilterStep::FilterStep( - const execplan::CalpontSystemCatalog::ColType& colType, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fTableOID(0), - fColType(colType) +FilterStep::FilterStep(const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo) + : JobStep(jobInfo), fTableOID(0), fColType(colType) { } @@ -76,150 +71,151 @@ FilterStep::~FilterStep() void FilterStep::join() { -// runner->join(); + // runner->join(); } void FilterStep::setBOP(int8_t b) { - fBOP = b; + fBOP = b; } - void FilterStep::run() { -// if (traceOn()) -// { -// syslogStartStep(16, // exemgr subsystem -// std::string("FilterStep")); // step name -// } -// -// runner.reset(new boost::thread(FSRunner(this))); + // if (traceOn()) + // { + // syslogStartStep(16, // exemgr subsystem + // std::string("FilterStep")); // step name + // } + // + // runner.reset(new boost::thread(FSRunner(this))); } void FilterStep::doFilter() { -// idbassert(fInputJobStepAssociation.outSize() == 2); -// idbassert(fOutputJobStepAssociation.outSize() == 1); -// StringFifoDataList* fStrFAp = 0; -// StringFifoDataList* fStrFBp = 0; -// StringFifoDataList* strFifo = 0; -// StrDataList* strResult = 0; -// -// FifoDataList* fFBP = 0; -// FifoDataList* fFAP = 0; -// FifoDataList* fifo = 0; -// DataList_t* result = 0; -// -// TimeSet timer; -//try -//{ -// -// fFAP = fInputJobStepAssociation.outAt(0)->fifoDL(); -// if ( !fFAP ) -// { -// fStrFAp = fInputJobStepAssociation.outAt(0)->stringDL(); -// idbassert(fStrFAp); -// -// fStrFBp = fInputJobStepAssociation.outAt(1)->stringDL(); -// idbassert(fStrFBp); -// -// strFifo = fOutputJobStepAssociation.outAt(0)->stringDL(); -// -// strResult = fOutputJobStepAssociation.outAt(0)->stringDataList(); -// } -// else -// { -// fFBP = fInputJobStepAssociation.outAt(1)->fifoDL(); -// idbassert(fFBP); -// fifo = fOutputJobStepAssociation.outAt(0)->fifoDL(); -// -// result = fOutputJobStepAssociation.outAt(0)->dataList(); -// } -// ostringstream ss; //tester -// ss << "Filter step id " << fStepId << " threw an exception"; -// throw runtime_error(ss.str()); -// resultCount = 0; -// if (fTableOID >= 3000 && dlTimes.FirstReadTime().tv_sec==0) -// { -// dlTimes.setFirstReadTime(); -// } -// uint32_t cop = BOP(); -// -// if (0 == status()) -// { -// FilterOperation filterOP; -// if ( !fFAP ) -// { -// if ( strFifo ) -// { -// filterOP.filter( cop, *fStrFAp, *fStrFBp, *strFifo, resultCount, timer); -// } -// else -// { -// filterOP.filter( cop, *fStrFAp, *fStrFBp, *strResult, resultCount, timer); -// } -// } -// else -// { -// if ( fifo ) -// { -// filterOP.filter( cop, *fFAP, *fFBP, *fifo, resultCount, timer); -// } -// else -// { -// filterOP.filter( cop, *fFAP, *fFBP, *result,resultCount, timer ); -// } -// } -// } // status() == 0 -//}//try -//catch (std::exception &e) -//{ -// std::cout << "FilterStep caught: " << e.what() << std::endl; -// unblockDataLists(fifo, strFifo, strResult, result); -// catchHandler(e.what()); -// status(logging::filterStepErr); -//} -//catch (...) -//{ -// string msg("FSRunner caught something not an exception!"); -// std::cout << msg << std::endl; -// unblockDataLists(fifo, strFifo, strResult, result); -// catchHandler(msg); -// status(logging::filterStepErr); -//} -// -// -// if (fTableOID >= 3000) -// dlTimes.setEndOfInputTime(); -// //...Print job step completion information -// if (fTableOID >= 3000 && traceOn()) -// { -// time_t finTime = time(0); -// char finTimeString[50]; -// ctime_r(&finTime, finTimeString); -// finTimeString[strlen(finTimeString)-1 ] = '\0'; -// -// ostringstream logStr; -// logStr << "ses:" << fSessionId << " st: " << fStepId << -// " finished at " << finTimeString << -// "; 1st read " << dlTimes.FirstReadTimeString() << -// "; EOI " << dlTimes.EndOfInputTimeString() -// <<"; Output:"<fifoDL(); + // if ( !fFAP ) + // { + // fStrFAp = fInputJobStepAssociation.outAt(0)->stringDL(); + // idbassert(fStrFAp); + // + // fStrFBp = fInputJobStepAssociation.outAt(1)->stringDL(); + // idbassert(fStrFBp); + // + // strFifo = fOutputJobStepAssociation.outAt(0)->stringDL(); + // + // strResult = fOutputJobStepAssociation.outAt(0)->stringDataList(); + // } + // else + // { + // fFBP = fInputJobStepAssociation.outAt(1)->fifoDL(); + // idbassert(fFBP); + // fifo = fOutputJobStepAssociation.outAt(0)->fifoDL(); + // + // result = fOutputJobStepAssociation.outAt(0)->dataList(); + // } + // ostringstream ss; //tester + // ss << "Filter step id " << fStepId << " threw an exception"; + // throw runtime_error(ss.str()); + // resultCount = 0; + // if (fTableOID >= 3000 && dlTimes.FirstReadTime().tv_sec==0) + // { + // dlTimes.setFirstReadTime(); + // } + // uint32_t cop = BOP(); + // + // if (0 == status()) + // { + // FilterOperation filterOP; + // if ( !fFAP ) + // { + // if ( strFifo ) + // { + // filterOP.filter( cop, *fStrFAp, *fStrFBp, *strFifo, resultCount, timer); + // } + // else + // { + // filterOP.filter( cop, *fStrFAp, *fStrFBp, *strResult, resultCount, timer); + // } + // } + // else + // { + // if ( fifo ) + // { + // filterOP.filter( cop, *fFAP, *fFBP, *fifo, resultCount, timer); + // } + // else + // { + // filterOP.filter( cop, *fFAP, *fFBP, *result,resultCount, timer ); + // } + // } + // } // status() == 0 + //}//try + // catch (std::exception &e) + //{ + // std::cout << "FilterStep caught: " << e.what() << std::endl; + // unblockDataLists(fifo, strFifo, strResult, result); + // catchHandler(e.what()); + // status(logging::filterStepErr); + //} + // catch (...) + //{ + // string msg("FSRunner caught something not an exception!"); + // std::cout << msg << std::endl; + // unblockDataLists(fifo, strFifo, strResult, result); + // catchHandler(msg); + // status(logging::filterStepErr); + //} + // + // + // if (fTableOID >= 3000) + // dlTimes.setEndOfInputTime(); + // //...Print job step completion information + // if (fTableOID >= 3000 && traceOn()) + // { + // time_t finTime = time(0); + // char finTimeString[50]; + // ctime_r(&finTime, finTimeString); + // finTimeString[strlen(finTimeString)-1 ] = '\0'; + // + // ostringstream logStr; + // logStr << "ses:" << fSessionId << " st: " << fStepId << + // " finished at " << finTimeString << + // "; 1st read " << dlTimes.FirstReadTimeString() << + // "; EOI " << dlTimes.EndOfInputTimeString() + // <<"; Output:"<endOfInput(); // else if (strFifo) strFifo->endOfInput(); @@ -227,37 +223,33 @@ void FilterStep::doFilter() // else if (result) result->endOfInput(); //} - void FilterStep::addFilter(const execplan::Filter* f) { - if (NULL != f) - fFilters.push_back(f); + if (NULL != f) + fFilters.push_back(f); } - const string FilterStep::toString() const { - ostringstream oss; - size_t idlsz; + ostringstream oss; + size_t idlsz; - idlsz = fInputJobStepAssociation.outSize(); - idbassert(idlsz == 2); + idlsz = fInputJobStepAssociation.outSize(); + idbassert(idlsz == 2); - oss << "FilterStep ses:" << fSessionId << " txn:" << fTxnId << - " st:" << fStepId; - oss << " in tb/col1:" << fTableOID << "/"; - oss << " " << fInputJobStepAssociation.outAt(0); // output will include oid - oss << " in tb/col2:" << fTableOID << "/"; - oss << " " << fInputJobStepAssociation.outAt(1); + oss << "FilterStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + oss << " in tb/col1:" << fTableOID << "/"; + oss << " " << fInputJobStepAssociation.outAt(0); // output will include oid + oss << " in tb/col2:" << fTableOID << "/"; + oss << " " << fInputJobStepAssociation.outAt(1); - idlsz = fOutputJobStepAssociation.outSize(); - idbassert(idlsz == 1); + idlsz = fOutputJobStepAssociation.outSize(); + idbassert(idlsz == 1); - oss << endl << " out tb/col:" << fTableOID << "/"; - oss << " " << fOutputJobStepAssociation.outAt(0);// output will include oid - - return oss.str(); -} + oss << endl << " out tb/col:" << fTableOID << "/"; + oss << " " << fOutputJobStepAssociation.outAt(0); // output will include oid + return oss.str(); } +} // namespace joblist diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 73757734b..5eb0e1b10 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -18,7 +18,6 @@ // $Id: groupconcat.cpp 9705 2013-07-17 20:06:07Z pleblanc $ - #include //#define NDEBUG #include @@ -60,1037 +59,995 @@ using namespace ordering; namespace joblist { - - // GroupConcatInfo class implementation GroupConcatInfo::GroupConcatInfo() { } - GroupConcatInfo::~GroupConcatInfo() { } - void GroupConcatInfo::prepGroupConcat(JobInfo& jobInfo) { - RetColsVector::iterator i = jobInfo.groupConcatCols.begin(); + RetColsVector::iterator i = jobInfo.groupConcatCols.begin(); - while (i != jobInfo.groupConcatCols.end()) + while (i != jobInfo.groupConcatCols.end()) + { + GroupConcatColumn* gcc = dynamic_cast(i->get()); + const RowColumn* rcp = dynamic_cast(gcc->aggParms()[0].get()); + + SP_GroupConcat groupConcat(new GroupConcat); + groupConcat->fSeparator = gcc->separator(); + groupConcat->fDistinct = gcc->distinct(); + groupConcat->fSize = gcc->resultType().colWidth; + groupConcat->fRm = jobInfo.rm; + groupConcat->fSessionMemLimit = jobInfo.umMemLimit; + groupConcat->fTimeZone = jobInfo.timeZone; + + int key = -1; + const vector& cols = rcp->columnVec(); + + for (uint64_t j = 0, k = 0; j < cols.size(); j++) { - GroupConcatColumn* gcc = dynamic_cast(i->get()); - const RowColumn* rcp = dynamic_cast(gcc->aggParms()[0].get()); + const ConstantColumn* cc = dynamic_cast(cols[j].get()); - SP_GroupConcat groupConcat(new GroupConcat); - groupConcat->fSeparator = gcc->separator(); - groupConcat->fDistinct = gcc->distinct(); - groupConcat->fSize = gcc->resultType().colWidth; - groupConcat->fRm = jobInfo.rm; - groupConcat->fSessionMemLimit = jobInfo.umMemLimit; - groupConcat->fTimeZone = jobInfo.timeZone; - - int key = -1; - const vector& cols = rcp->columnVec(); - - for (uint64_t j = 0, k = 0; j < cols.size(); j++) - { - const ConstantColumn* cc = dynamic_cast(cols[j].get()); - - if (cc == NULL) - { - key = getColumnKey(cols[j], jobInfo); - fColumns.insert(key); - groupConcat->fGroupCols.push_back(make_pair(key, k++)); - } - else - { - groupConcat->fConstCols.push_back(make_pair(cc->constval(), j)); - } - } - - vector& orderCols = gcc->orderCols(); - - for (vector::iterator k = orderCols.begin(); k != orderCols.end(); k++) - { - if (dynamic_cast(k->get()) != NULL) - continue; - - key = getColumnKey(*k, jobInfo); - fColumns.insert(key); - groupConcat->fOrderCols.push_back(make_pair(key, k->get()->asc())); - } - - fGroupConcat.push_back(groupConcat); - - i++; + if (cc == NULL) + { + key = getColumnKey(cols[j], jobInfo); + fColumns.insert(key); + groupConcat->fGroupCols.push_back(make_pair(key, k++)); + } + else + { + groupConcat->fConstCols.push_back(make_pair(cc->constval(), j)); + } } - // Rare case: all columns in group_concat are constant columns, use a column in column map. - if (jobInfo.groupConcatCols.size() > 0 && fColumns.size() == 0) + vector& orderCols = gcc->orderCols(); + + for (vector::iterator k = orderCols.begin(); k != orderCols.end(); k++) { - int key = -1; + if (dynamic_cast(k->get()) != NULL) + continue; - for (vector::iterator i = jobInfo.tableList.begin(); - i != jobInfo.tableList.end() && key == -1; - i++) - { - if (jobInfo.columnMap[*i].size() > 0) - { - key = *(jobInfo.columnMap[*i].begin()); - } - } - - if (key != -1) - { - fColumns.insert(key); - } - else - { - throw runtime_error("Empty column map."); - } + key = getColumnKey(*k, jobInfo); + fColumns.insert(key); + groupConcat->fOrderCols.push_back(make_pair(key, k->get()->asc())); } -} + fGroupConcat.push_back(groupConcat); -uint32_t GroupConcatInfo::getColumnKey(const SRCP& srcp, JobInfo& jobInfo) -{ - int colKey = -1; - const SimpleColumn* sc = dynamic_cast(srcp.get()); + i++; + } - if (sc != NULL) + // Rare case: all columns in group_concat are constant columns, use a column in column map. + if (jobInfo.groupConcatCols.size() > 0 && fColumns.size() == 0) + { + int key = -1; + + for (vector::iterator i = jobInfo.tableList.begin(); i != jobInfo.tableList.end() && key == -1; + i++) { - if (sc->schemaName().empty()) - { - // bug3839, handle columns from subquery. - SimpleColumn tmp(*sc, jobInfo.sessionId); - tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - colKey = getTupleKey(jobInfo, &tmp); - } - else - { - colKey = getTupleKey(jobInfo, sc); - } + if (jobInfo.columnMap[*i].size() > 0) + { + key = *(jobInfo.columnMap[*i].begin()); + } + } - // check if this is a dictionary column - if (jobInfo.keyInfo->dictKeyMap.find(colKey) != jobInfo.keyInfo->dictKeyMap.end()) - colKey = jobInfo.keyInfo->dictKeyMap[colKey]; + if (key != -1) + { + fColumns.insert(key); } else { - const ArithmeticColumn* ac = dynamic_cast(srcp.get()); - const FunctionColumn* fc = dynamic_cast(srcp.get()); - - if (ac != NULL || fc != NULL) - { - colKey = getExpTupleKey(jobInfo, srcp->expressionId()); - } - else - { - cerr << "Unsupported GROUP_CONCAT column. " << srcp->toString() << endl; - throw runtime_error("Unsupported GROUP_CONCAT column."); - } + throw runtime_error("Empty column map."); } - - return colKey; + } } +uint32_t GroupConcatInfo::getColumnKey(const SRCP& srcp, JobInfo& jobInfo) +{ + int colKey = -1; + const SimpleColumn* sc = dynamic_cast(srcp.get()); + + if (sc != NULL) + { + if (sc->schemaName().empty()) + { + // bug3839, handle columns from subquery. + SimpleColumn tmp(*sc, jobInfo.sessionId); + tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + colKey = getTupleKey(jobInfo, &tmp); + } + else + { + colKey = getTupleKey(jobInfo, sc); + } + + // check if this is a dictionary column + if (jobInfo.keyInfo->dictKeyMap.find(colKey) != jobInfo.keyInfo->dictKeyMap.end()) + colKey = jobInfo.keyInfo->dictKeyMap[colKey]; + } + else + { + const ArithmeticColumn* ac = dynamic_cast(srcp.get()); + const FunctionColumn* fc = dynamic_cast(srcp.get()); + + if (ac != NULL || fc != NULL) + { + colKey = getExpTupleKey(jobInfo, srcp->expressionId()); + } + else + { + cerr << "Unsupported GROUP_CONCAT column. " << srcp->toString() << endl; + throw runtime_error("Unsupported GROUP_CONCAT column."); + } + } + + return colKey; +} void GroupConcatInfo::mapColumns(const RowGroup& projRG) { - map projColumnMap; - const vector& keysProj = projRG.getKeys(); + map projColumnMap; + const vector& keysProj = projRG.getKeys(); - for (uint64_t i = 0; i < projRG.getColumnCount(); i++) - projColumnMap[keysProj[i]] = i; + for (uint64_t i = 0; i < projRG.getColumnCount(); i++) + projColumnMap[keysProj[i]] = i; - for (vector::iterator k = fGroupConcat.begin(); k != fGroupConcat.end(); k++) + for (vector::iterator k = fGroupConcat.begin(); k != fGroupConcat.end(); k++) + { + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); + + vector >::iterator i1 = (*k)->fGroupCols.begin(); + + while (i1 != (*k)->fGroupCols.end()) { - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); + map::iterator j = projColumnMap.find(i1->first); - vector >::iterator i1 = (*k)->fGroupCols.begin(); + if (j == projColumnMap.end()) + { + cerr << "Concat Key:" << i1->first << " is not projected." << endl; + throw runtime_error("Project error."); + } - while (i1 != (*k)->fGroupCols.end()) - { - map::iterator j = projColumnMap.find(i1->first); + pos.push_back(pos.back() + projRG.getColumnWidth(j->second)); + oids.push_back(projRG.getOIDs()[j->second]); + keys.push_back(projRG.getKeys()[j->second]); + types.push_back(projRG.getColTypes()[j->second]); + csNums.push_back(projRG.getCharsetNumber(j->second)); + scale.push_back(projRG.getScale()[j->second]); + precision.push_back(projRG.getPrecision()[j->second]); - if (j == projColumnMap.end()) - { - cerr << "Concat Key:" << i1->first << " is not projected." << endl; - throw runtime_error("Project error."); - } - - pos.push_back(pos.back() + projRG.getColumnWidth(j->second)); - oids.push_back(projRG.getOIDs()[j->second]); - keys.push_back(projRG.getKeys()[j->second]); - types.push_back(projRG.getColTypes()[j->second]); - csNums.push_back(projRG.getCharsetNumber(j->second)); - scale.push_back(projRG.getScale()[j->second]); - precision.push_back(projRG.getPrecision()[j->second]); - - i1++; - } - - vector >::iterator i2 = (*k)->fOrderCols.begin(); - - while (i2 != (*k)->fOrderCols.end()) - { - map::iterator j = projColumnMap.find(i2->first); - - if (j == projColumnMap.end()) - { - cerr << "Order Key:" << i2->first << " is not projected." << endl; - throw runtime_error("Project error."); - } - - vector::iterator i3 = find(keys.begin(), keys.end(), j->first); - int idx = 0; - - if (i3 == keys.end()) - { - idx = keys.size(); - - pos.push_back(pos.back() + projRG.getColumnWidth(j->second)); - oids.push_back(projRG.getOIDs()[j->second]); - keys.push_back(projRG.getKeys()[j->second]); - types.push_back(projRG.getColTypes()[j->second]); - csNums.push_back(projRG.getCharsetNumber(j->second)); - scale.push_back(projRG.getScale()[j->second]); - precision.push_back(projRG.getPrecision()[j->second]); - } - else - { - idx = std::distance(keys.begin(), i3); - } - - (*k)->fOrderCond.push_back(make_pair(idx, i2->second)); - - i2++; - } - - (*k)->fRowGroup = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, projRG.getStringTableThreshold(), false); - (*k)->fMapping = makeMapping(projRG, (*k)->fRowGroup); + i1++; } -} + vector >::iterator i2 = (*k)->fOrderCols.begin(); + + while (i2 != (*k)->fOrderCols.end()) + { + map::iterator j = projColumnMap.find(i2->first); + + if (j == projColumnMap.end()) + { + cerr << "Order Key:" << i2->first << " is not projected." << endl; + throw runtime_error("Project error."); + } + + vector::iterator i3 = find(keys.begin(), keys.end(), j->first); + int idx = 0; + + if (i3 == keys.end()) + { + idx = keys.size(); + + pos.push_back(pos.back() + projRG.getColumnWidth(j->second)); + oids.push_back(projRG.getOIDs()[j->second]); + keys.push_back(projRG.getKeys()[j->second]); + types.push_back(projRG.getColTypes()[j->second]); + csNums.push_back(projRG.getCharsetNumber(j->second)); + scale.push_back(projRG.getScale()[j->second]); + precision.push_back(projRG.getPrecision()[j->second]); + } + else + { + idx = std::distance(keys.begin(), i3); + } + + (*k)->fOrderCond.push_back(make_pair(idx, i2->second)); + + i2++; + } + + (*k)->fRowGroup = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, + projRG.getStringTableThreshold(), false); + (*k)->fMapping = makeMapping(projRG, (*k)->fRowGroup); + } +} shared_array GroupConcatInfo::makeMapping(const RowGroup& in, const RowGroup& out) { - // For some reason using the rowgroup mapping fcns don't work completely right in this class - shared_array mapping(new int[out.getColumnCount()]); + // For some reason using the rowgroup mapping fcns don't work completely right in this class + shared_array mapping(new int[out.getColumnCount()]); - for (uint64_t i = 0; i < out.getColumnCount(); i++) + for (uint64_t i = 0; i < out.getColumnCount(); i++) + { + for (uint64_t j = 0; j < in.getColumnCount(); j++) { - for (uint64_t j = 0; j < in.getColumnCount(); j++) - { - if ((out.getKeys()[i] == in.getKeys()[j])) - { - mapping[i] = j; - break; - } - } + if ((out.getKeys()[i] == in.getKeys()[j])) + { + mapping[i] = j; + break; + } } + } - return mapping; + return mapping; } - const string GroupConcatInfo::toString() const { - ostringstream oss; - oss << "GroupConcatInfo: toString() to be implemented."; - oss << endl; + ostringstream oss; + oss << "GroupConcatInfo: toString() to be implemented."; + oss << endl; - return oss.str(); + return oss.str(); } - GroupConcatAgUM::GroupConcatAgUM(rowgroup::SP_GroupConcat& gcc) : GroupConcatAg(gcc) { - initialize(); + initialize(); } GroupConcatAgUM::~GroupConcatAgUM() { } - void GroupConcatAgUM::initialize() { - if (fGroupConcat->fDistinct || fGroupConcat->fOrderCols.size() > 0) - fConcator.reset(new GroupConcatOrderBy()); - else - fConcator.reset(new GroupConcatNoOrder()); + if (fGroupConcat->fDistinct || fGroupConcat->fOrderCols.size() > 0) + fConcator.reset(new GroupConcatOrderBy()); + else + fConcator.reset(new GroupConcatNoOrder()); - fConcator->initialize(fGroupConcat); + fConcator->initialize(fGroupConcat); - fGroupConcat->fRowGroup.initRow(&fRow, true); - fData.reset(new uint8_t[fRow.getSize()]); - fRow.setData(fData.get()); + fGroupConcat->fRowGroup.initRow(&fRow, true); + fData.reset(new uint8_t[fRow.getSize()]); + fRow.setData(fData.get()); } - void GroupConcatAgUM::processRow(const rowgroup::Row& inRow) { - applyMapping(fGroupConcat->fMapping, inRow); - fConcator->processRow(fRow); + applyMapping(fGroupConcat->fMapping, inRow); + fConcator->processRow(fRow); } - void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i) { - uint8_t* data = inRow.getData(); - joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); + uint8_t* data = inRow.getData(); + joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); - fConcator->merge(gccAg->concator().get()); + fConcator->merge(gccAg->concator().get()); } - void GroupConcatAgUM::getResult(uint8_t* buff) { - fConcator->getResult(buff, fGroupConcat->fSeparator); + fConcator->getResult(buff, fGroupConcat->fSeparator); } uint8_t* GroupConcatAgUM::getResult() { - return fConcator->getResult(fGroupConcat->fSeparator); + return fConcator->getResult(fGroupConcat->fSeparator); } - void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, const Row& row) { - // For some reason the rowgroup mapping fcns don't work right in this class. - for (uint64_t i = 0; i < fRow.getColumnCount(); i++) + // For some reason the rowgroup mapping fcns don't work right in this class. + for (uint64_t i = 0; i < fRow.getColumnCount(); i++) + { + if (fRow.getColumnWidth(i) > datatypes::MAXLEGACYWIDTH) { - if (fRow.getColumnWidth(i) > datatypes::MAXLEGACYWIDTH) - { - if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || - fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR || - fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT) - { - fRow.setStringField(row.getConstString(mapping[i]), i); - } - else if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE) - { - fRow.setLongDoubleField(row.getLongDoubleField(mapping[i]), i); - } - else if (datatypes::isWideDecimalType(fRow.getColType(i), fRow.getColumnWidth(i))) - { - row.copyBinaryField(fRow, i, mapping[i]); - } - } - else - { - fRow.setIntField(row.getIntField(mapping[i]), i); - } + if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || + fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR || + fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT) + { + fRow.setStringField(row.getConstString(mapping[i]), i); + } + else if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + fRow.setLongDoubleField(row.getLongDoubleField(mapping[i]), i); + } + else if (datatypes::isWideDecimalType(fRow.getColType(i), fRow.getColumnWidth(i))) + { + row.copyBinaryField(fRow, i, mapping[i]); + } } + else + { + fRow.setIntField(row.getIntField(mapping[i]), i); + } + } } - // GroupConcator class implementation GroupConcator::GroupConcator() : fCurrentLength(0), fGroupConcatLen(0), fConstantLen(0) { } - GroupConcator::~GroupConcator() { } - void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) { - // MCOL-901 This value comes from the Server and it is - // too high(3MB) to allocate it for every instance. - fGroupConcatLen = gcc->fSize; - fCurrentLength -= strlen(gcc->fSeparator.c_str()); - fTimeZone = gcc->fTimeZone; + // MCOL-901 This value comes from the Server and it is + // too high(3MB) to allocate it for every instance. + fGroupConcatLen = gcc->fSize; + fCurrentLength -= strlen(gcc->fSeparator.c_str()); + fTimeZone = gcc->fTimeZone; - fConstCols = gcc->fConstCols; - fConstantLen = strlen(gcc->fSeparator.c_str()); + fConstCols = gcc->fConstCols; + fConstantLen = strlen(gcc->fSeparator.c_str()); - for (uint64_t i = 0; i < fConstCols.size(); i++) - fConstantLen += strlen(fConstCols[i].first.c_str()); + for (uint64_t i = 0; i < fConstCols.size(); i++) + fConstantLen += strlen(fConstCols[i].first.c_str()); } - void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) { - const CalpontSystemCatalog::ColDataType* types = row.getColTypes(); - vector::iterator i = fConcatColumns.begin(); - vector >::iterator j = fConstCols.begin(); + const CalpontSystemCatalog::ColDataType* types = row.getColTypes(); + vector::iterator i = fConcatColumns.begin(); + vector >::iterator j = fConstCols.begin(); - uint64_t groupColCount = fConcatColumns.size() + fConstCols.size(); + uint64_t groupColCount = fConcatColumns.size() + fConstCols.size(); - for (uint64_t k = 0; k < groupColCount; k++) + for (uint64_t k = 0; k < groupColCount; k++) + { + if (j != fConstCols.end() && k == j->second) { - if (j != fConstCols.end() && k == j->second) - { - oss << j->first; - j++; - continue; - } - - switch (types[*i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - int64_t intVal = row.getIntField(*i); - - oss << intVal; - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - oss << fixed << row.getDecimalField(*i); - break; - } - - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - { - uint64_t uintVal = row.getUintField(*i); - int scale = (int) row.getScale(*i); - - if (scale == 0) - { - oss << uintVal; - } - else - { - oss << fixed << datatypes::Decimal(datatypes::TSInt128((int128_t) uintVal), - scale, - datatypes::INT128MAXPRECISION); - } - - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - oss << row.getStringField(*i).c_str(); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - oss << setprecision(15) << row.getDoubleField(*i); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - oss << setprecision(15) << row.getLongDoubleField(*i); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - oss << row.getFloatField(*i); - break; - } - - case CalpontSystemCatalog::DATE: - { - oss << DataConvert::dateToString(row.getUintField(*i)); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - oss << DataConvert::datetimeToString(row.getUintField(*i)); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - oss << DataConvert::timestampToString(row.getUintField(*i), fTimeZone); - break; - } - - case CalpontSystemCatalog::TIME: - { - oss << DataConvert::timeToString(row.getUintField(*i)); - break; - } - - default: - { - break; - } - } - - i++; - } -} - - -bool GroupConcator::concatColIsNull(const rowgroup::Row& row) -{ - bool ret = false; - - for (vector::iterator i = fConcatColumns.begin(); i != fConcatColumns.end(); i++) - { - if (row.isNullValue(*i)) - { - ret = true; - break; - } + oss << j->first; + j++; + continue; } - return ret; -} - - -int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) -{ - int64_t rowLen = fConstantLen; // fixed constant and separator length - const CalpontSystemCatalog::ColDataType* types = row.getColTypes(); - - // null values are already skipped. - for (vector::iterator i = fConcatColumns.begin(); i != fConcatColumns.end(); i++) + switch (types[*i]) { - if (row.isNullValue(*i)) - continue; + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + { + int64_t intVal = row.getIntField(*i); - int64_t fieldLen = 0; + oss << intVal; - switch (types[*i]) + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + oss << fixed << row.getDecimalField(*i); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + uint64_t uintVal = row.getUintField(*i); + int scale = (int)row.getScale(*i); + + if (scale == 0) { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - int64_t v = row.getIntField(*i); - - if (v < 0) fieldLen++; - - while ((v /= 10) != 0) fieldLen++; - - fieldLen += 1; - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - { - uint64_t v = row.getUintField(*i); - - while ((v /= 10) != 0) fieldLen++; - - fieldLen += 1; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - fieldLen += 1; - - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - fieldLen += row.getConstString(*i).length(); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::LONGDOUBLE: - { - fieldLen = 1; // minimum length - break; - } - - case CalpontSystemCatalog::DATE: - { - fieldLen = 10; // YYYY-MM-DD - break; - } - - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - { - fieldLen = 19; // YYYY-MM-DD HH24:MI:SS - // Decimal point and milliseconds - uint64_t colPrecision = row.getPrecision(*i); - - if (colPrecision > 0 && colPrecision < 7) - { - fieldLen += colPrecision + 1; - } - - break; - } - - case CalpontSystemCatalog::TIME: - { - fieldLen = 10; // -HHH:MI:SS - // Decimal point and milliseconds - uint64_t colPrecision = row.getPrecision(*i); - - if (colPrecision > 0 && colPrecision < 7) - { - fieldLen += colPrecision + 1; - } - - break; - } - - default: - { - break; - } - } - - rowLen += fieldLen; - } - - return rowLen; -} - - -const string GroupConcator::toString() const -{ - ostringstream oss; - oss << "GroupConcat size-" << fGroupConcatLen; - oss << "Concat cols: "; - vector::const_iterator i = fConcatColumns.begin(); - vector >::const_iterator j = fConstCols.begin(); - uint64_t groupColCount = fConcatColumns.size() + fConstCols.size(); - - for (uint64_t k = 0; k < groupColCount; k++) - { - if (j != fConstCols.end() && k == j->second) - { - oss << 'c' << " "; - j++; + oss << uintVal; } else { - oss << (*i) << " "; - i++; + oss << fixed + << datatypes::Decimal(datatypes::TSInt128((int128_t)uintVal), scale, + datatypes::INT128MAXPRECISION); } + + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + { + oss << row.getStringField(*i).c_str(); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + oss << setprecision(15) << row.getDoubleField(*i); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + oss << setprecision(15) << row.getLongDoubleField(*i); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + oss << row.getFloatField(*i); + break; + } + + case CalpontSystemCatalog::DATE: + { + oss << DataConvert::dateToString(row.getUintField(*i)); + break; + } + + case CalpontSystemCatalog::DATETIME: + { + oss << DataConvert::datetimeToString(row.getUintField(*i)); + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + oss << DataConvert::timestampToString(row.getUintField(*i), fTimeZone); + break; + } + + case CalpontSystemCatalog::TIME: + { + oss << DataConvert::timeToString(row.getUintField(*i)); + break; + } + + default: + { + break; + } } - oss << endl; - - return oss.str(); + i++; + } } +bool GroupConcator::concatColIsNull(const rowgroup::Row& row) +{ + bool ret = false; + + for (vector::iterator i = fConcatColumns.begin(); i != fConcatColumns.end(); i++) + { + if (row.isNullValue(*i)) + { + ret = true; + break; + } + } + + return ret; +} + +int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) +{ + int64_t rowLen = fConstantLen; // fixed constant and separator length + const CalpontSystemCatalog::ColDataType* types = row.getColTypes(); + + // null values are already skipped. + for (vector::iterator i = fConcatColumns.begin(); i != fConcatColumns.end(); i++) + { + if (row.isNullValue(*i)) + continue; + + int64_t fieldLen = 0; + + switch (types[*i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + { + int64_t v = row.getIntField(*i); + + if (v < 0) + fieldLen++; + + while ((v /= 10) != 0) + fieldLen++; + + fieldLen += 1; + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + uint64_t v = row.getUintField(*i); + + while ((v /= 10) != 0) + fieldLen++; + + fieldLen += 1; + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + fieldLen += 1; + + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + { + fieldLen += row.getConstString(*i).length(); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + case CalpontSystemCatalog::LONGDOUBLE: + { + fieldLen = 1; // minimum length + break; + } + + case CalpontSystemCatalog::DATE: + { + fieldLen = 10; // YYYY-MM-DD + break; + } + + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + { + fieldLen = 19; // YYYY-MM-DD HH24:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + + break; + } + + case CalpontSystemCatalog::TIME: + { + fieldLen = 10; // -HHH:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + + break; + } + + default: + { + break; + } + } + + rowLen += fieldLen; + } + + return rowLen; +} + +const string GroupConcator::toString() const +{ + ostringstream oss; + oss << "GroupConcat size-" << fGroupConcatLen; + oss << "Concat cols: "; + vector::const_iterator i = fConcatColumns.begin(); + vector >::const_iterator j = fConstCols.begin(); + uint64_t groupColCount = fConcatColumns.size() + fConstCols.size(); + + for (uint64_t k = 0; k < groupColCount; k++) + { + if (j != fConstCols.end() && k == j->second) + { + oss << 'c' << " "; + j++; + } + else + { + oss << (*i) << " "; + i++; + } + } + + oss << endl; + + return oss.str(); +} // GroupConcatOrderBy class implementation GroupConcatOrderBy::GroupConcatOrderBy() { - fRule.fIdbCompare = this; + fRule.fIdbCompare = this; } - GroupConcatOrderBy::~GroupConcatOrderBy() { } - void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc) { - GroupConcator::initialize(gcc); + GroupConcator::initialize(gcc); - fOrderByCond.resize(0); + fOrderByCond.resize(0); - for (uint64_t i = 0; i < gcc->fOrderCond.size(); i++) - fOrderByCond.push_back(IdbSortSpec(gcc->fOrderCond[i].first, gcc->fOrderCond[i].second)); + for (uint64_t i = 0; i < gcc->fOrderCond.size(); i++) + fOrderByCond.push_back(IdbSortSpec(gcc->fOrderCond[i].first, gcc->fOrderCond[i].second)); - fDistinct = gcc->fDistinct; - fRowsPerRG = 128; - fErrorCode = ERR_AGGREGATION_TOO_BIG; - fRm = gcc->fRm; - fSessionMemLimit = gcc->fSessionMemLimit; + fDistinct = gcc->fDistinct; + fRowsPerRG = 128; + fErrorCode = ERR_AGGREGATION_TOO_BIG; + fRm = gcc->fRm; + fSessionMemLimit = gcc->fSessionMemLimit; - vector >::iterator i = gcc->fGroupCols.begin(); + vector >::iterator i = gcc->fGroupCols.begin(); - while (i != gcc->fGroupCols.end()) - fConcatColumns.push_back((*(i++)).second); + while (i != gcc->fGroupCols.end()) + fConcatColumns.push_back((*(i++)).second); - IdbOrderBy::initialize(gcc->fRowGroup); + IdbOrderBy::initialize(gcc->fRowGroup); } - uint64_t GroupConcatOrderBy::getKeyLength() const { - // only distinct the concatenated columns - return fConcatColumns.size(); // cols 0 to fConcatColumns.size() - 1 will be compared + // only distinct the concatenated columns + return fConcatColumns.size(); // cols 0 to fConcatColumns.size() - 1 will be compared } - void GroupConcatOrderBy::processRow(const rowgroup::Row& row) { - // check if this is a distinct row - if (fDistinct && fDistinctMap->find(row.getPointer()) != fDistinctMap->end()) - return; + // check if this is a distinct row + if (fDistinct && fDistinctMap->find(row.getPointer()) != fDistinctMap->end()) + return; - // this row is skipped if any concatenated column is null. - if (concatColIsNull(row)) - return; + // this row is skipped if any concatenated column is null. + if (concatColIsNull(row)) + return; - // if the row count is less than the limit - if (fCurrentLength < fGroupConcatLen) + // if the row count is less than the limit + if (fCurrentLength < fGroupConcatLen) + { + copyRow(row, &fRow0); + // the RID is no meaning here, use it to store the estimated length. + int16_t estLen = lengthEstimate(fRow0); + fRow0.setRid(estLen); + OrderByRow newRow(fRow0, fRule); + fOrderByQueue.push(newRow); + fCurrentLength += estLen; + + // add to the distinct map + if (fDistinct) + fDistinctMap->insert(fRow0.getPointer()); + + fRowGroup.incRowCount(); + fRow0.nextRow(); + + if (fRowGroup.getRowCount() >= fRowsPerRG) { - copyRow(row, &fRow0); - // the RID is no meaning here, use it to store the estimated length. - int16_t estLen = lengthEstimate(fRow0); - fRow0.setRid(estLen); - OrderByRow newRow(fRow0, fRule); - fOrderByQueue.push(newRow); - fCurrentLength += estLen; + fDataQueue.push(fData); - // add to the distinct map - if (fDistinct) - fDistinctMap->insert(fRow0.getPointer()); + uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - fRowGroup.incRowCount(); - fRow0.nextRow(); + if (!fRm->getMemory(newSize, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += newSize; - if (fRowGroup.getRowCount() >= fRowsPerRG) - { - fDataQueue.push(fData); + fData.reinit(fRowGroup, fRowsPerRG); + fRowGroup.setData(&fData); + fRowGroup.resetRowGroup(0); + fRowGroup.getRow(0, &fRow0); + } + } - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); + else if (fOrderByCond.size() > 0 && fRule.less(row.getPointer(), fOrderByQueue.top().fData)) + { + OrderByRow swapRow = fOrderByQueue.top(); + fRow1.setData(swapRow.fData); + fOrderByQueue.pop(); + fCurrentLength -= fRow1.getRelRid(); + fRow2.setData(swapRow.fData); - if (!fRm->getMemory(newSize, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += newSize; - - fData.reinit(fRowGroup, fRowsPerRG); - fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); - fRowGroup.getRow(0, &fRow0); - } + if (!fDistinct) + { + copyRow(row, &fRow1); + } + else + { + // only the copyRow does useful work here + fDistinctMap->erase(swapRow.fData); + copyRow(row, &fRow2); + fDistinctMap->insert(swapRow.fData); } - else if (fOrderByCond.size() > 0 && fRule.less(row.getPointer(), fOrderByQueue.top().fData)) - { - OrderByRow swapRow = fOrderByQueue.top(); - fRow1.setData(swapRow.fData); - fOrderByQueue.pop(); - fCurrentLength -= fRow1.getRelRid(); - fRow2.setData(swapRow.fData); + int16_t estLen = lengthEstimate(fRow2); + fRow2.setRid(estLen); + fCurrentLength += estLen; - if (!fDistinct) - { - copyRow(row, &fRow1); - } - else - { - // only the copyRow does useful work here - fDistinctMap->erase(swapRow.fData); - copyRow(row, &fRow2); - fDistinctMap->insert(swapRow.fData); - } - - int16_t estLen = lengthEstimate(fRow2); - fRow2.setRid(estLen); - fCurrentLength += estLen; - - fOrderByQueue.push(swapRow); - } + fOrderByQueue.push(swapRow); + } } - void GroupConcatOrderBy::merge(GroupConcator* gc) { - GroupConcatOrderBy* go = dynamic_cast(gc); + GroupConcatOrderBy* go = dynamic_cast(gc); - while (go->fOrderByQueue.empty() == false) + while (go->fOrderByQueue.empty() == false) + { + const OrderByRow& row = go->fOrderByQueue.top(); + + // check if the distinct row already exists + if (fDistinct && fDistinctMap->find(row.fData) != fDistinctMap->end()) { - const OrderByRow& row = go->fOrderByQueue.top(); - - // check if the distinct row already exists - if (fDistinct && fDistinctMap->find(row.fData) != fDistinctMap->end()) - { - ; // no op; - } - - // if the row count is less than the limit - else if (fCurrentLength < fGroupConcatLen) - { - fOrderByQueue.push(row); - row1.setData(row.fData); - fCurrentLength += row1.getRelRid(); - - // add to the distinct map - if (fDistinct) - fDistinctMap->insert(row.fData); - - } - - else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) - { - OrderByRow swapRow = fOrderByQueue.top(); - row1.setData(swapRow.fData); - fOrderByQueue.pop(); - fCurrentLength -= row1.getRelRid(); - - if (fDistinct) - { - fDistinctMap->erase(swapRow.fData); - fDistinctMap->insert(row.fData); - } - - row1.setData(row.fData); - fCurrentLength += row1.getRelRid(); - - fOrderByQueue.push(row); - } - - go->fOrderByQueue.pop(); + ; // no op; } -} + // if the row count is less than the limit + else if (fCurrentLength < fGroupConcatLen) + { + fOrderByQueue.push(row); + row1.setData(row.fData); + fCurrentLength += row1.getRelRid(); + + // add to the distinct map + if (fDistinct) + fDistinctMap->insert(row.fData); + } + + else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) + { + OrderByRow swapRow = fOrderByQueue.top(); + row1.setData(swapRow.fData); + fOrderByQueue.pop(); + fCurrentLength -= row1.getRelRid(); + + if (fDistinct) + { + fDistinctMap->erase(swapRow.fData); + fDistinctMap->insert(row.fData); + } + + row1.setData(row.fData); + fCurrentLength += row1.getRelRid(); + + fOrderByQueue.push(row); + } + + go->fOrderByQueue.pop(); + } +} void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) { - ostringstream oss; - bool addSep = false; + ostringstream oss; + bool addSep = false; - // need to reverse the order - stack rowStack; + // need to reverse the order + stack rowStack; - while (fOrderByQueue.size() > 0) - { - rowStack.push(fOrderByQueue.top()); - fOrderByQueue.pop(); - } + while (fOrderByQueue.size() > 0) + { + rowStack.push(fOrderByQueue.top()); + fOrderByQueue.pop(); + } - while (rowStack.size() > 0) - { - if (addSep) - oss << sep; - else - addSep = true; + while (rowStack.size() > 0) + { + if (addSep) + oss << sep; + else + addSep = true; - const OrderByRow& topRow = rowStack.top(); - fRow0.setData(topRow.fData); - outputRow(oss, fRow0); - rowStack.pop(); - } + const OrderByRow& topRow = rowStack.top(); + fRow0.setData(topRow.fData); + outputRow(oss, fRow0); + rowStack.pop(); + } - int64_t resultSize = oss.str().size(); - resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; - fOutputString.reset(new uint8_t[resultSize + 2]); - fOutputString[resultSize] = '\0'; - fOutputString[resultSize + 1] = '\0'; + int64_t resultSize = oss.str().size(); + resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; + fOutputString.reset(new uint8_t[resultSize + 2]); + fOutputString[resultSize] = '\0'; + fOutputString[resultSize + 1] = '\0'; - strncpy((char*)fOutputString.get(), - oss.str().c_str(), resultSize); + strncpy((char*)fOutputString.get(), oss.str().c_str(), resultSize); } uint8_t* GroupConcator::getResult(const string& sep) { - getResult(fOutputString.get(), sep); - return fOutputString.get(); + getResult(fOutputString.get(), sep); + return fOutputString.get(); } const string GroupConcatOrderBy::toString() const { - string baseStr = GroupConcator::toString(); + string baseStr = GroupConcator::toString(); - ostringstream oss; - oss << "OrderBy cols: "; - vector::const_iterator i = fOrderByCond.begin(); + ostringstream oss; + oss << "OrderBy cols: "; + vector::const_iterator i = fOrderByCond.begin(); - for (; i != fOrderByCond.end(); i++) - oss << "(" << i->fIndex << "," - << ((i->fAsc) ? "Asc" : "Desc") << "," - << ((i->fNf) ? "null first" : "null last") << ") "; + for (; i != fOrderByCond.end(); i++) + oss << "(" << i->fIndex << "," << ((i->fAsc) ? "Asc" : "Desc") << "," + << ((i->fNf) ? "null first" : "null last") << ") "; - if (fDistinct) - oss << endl << " distinct"; + if (fDistinct) + oss << endl << " distinct"; - oss << endl; + oss << endl; - return (baseStr + oss.str()); + return (baseStr + oss.str()); } - // GroupConcatNoOrder class implementation -GroupConcatNoOrder::GroupConcatNoOrder() : - fRowsPerRG(128), fErrorCode(ERR_AGGREGATION_TOO_BIG), fMemSize(0), fRm(NULL) +GroupConcatNoOrder::GroupConcatNoOrder() + : fRowsPerRG(128), fErrorCode(ERR_AGGREGATION_TOO_BIG), fMemSize(0), fRm(NULL) { } - GroupConcatNoOrder::~GroupConcatNoOrder() { - if (fRm) - fRm->returnMemory(fMemSize, fSessionMemLimit); + if (fRm) + fRm->returnMemory(fMemSize, fSessionMemLimit); } - void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) { - GroupConcator::initialize(gcc); + GroupConcator::initialize(gcc); - fRowGroup = gcc->fRowGroup; - fRowsPerRG = 128; - fErrorCode = ERR_AGGREGATION_TOO_BIG; - fRm = gcc->fRm; - fSessionMemLimit = gcc->fSessionMemLimit; + fRowGroup = gcc->fRowGroup; + fRowsPerRG = 128; + fErrorCode = ERR_AGGREGATION_TOO_BIG; + fRm = gcc->fRm; + fSessionMemLimit = gcc->fSessionMemLimit; - vector >::iterator i = gcc->fGroupCols.begin(); + vector >::iterator i = gcc->fGroupCols.begin(); - while (i != gcc->fGroupCols.end()) - fConcatColumns.push_back((*(i++)).second); + while (i != gcc->fGroupCols.end()) + fConcatColumns.push_back((*(i++)).second); - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); + uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - if (!fRm->getMemory(newSize, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += newSize; - - fData.reinit(fRowGroup, fRowsPerRG); - fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); - fRowGroup.initRow(&fRow); - fRowGroup.getRow(0, &fRow); + if (!fRm->getMemory(newSize, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += newSize; + + fData.reinit(fRowGroup, fRowsPerRG); + fRowGroup.setData(&fData); + fRowGroup.resetRowGroup(0); + fRowGroup.initRow(&fRow); + fRowGroup.getRow(0, &fRow); } - void GroupConcatNoOrder::processRow(const rowgroup::Row& row) { - // if the row count is less than the limit - if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) + // if the row count is less than the limit + if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) + { + copyRow(row, &fRow); + + // the RID is no meaning here, use it to store the estimated length. + int16_t estLen = lengthEstimate(fRow); + fRow.setRid(estLen); + fCurrentLength += estLen; + fRowGroup.incRowCount(); + fRow.nextRow(); + + if (fRowGroup.getRowCount() >= fRowsPerRG) { - copyRow(row, &fRow); + uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - // the RID is no meaning here, use it to store the estimated length. - int16_t estLen = lengthEstimate(fRow); - fRow.setRid(estLen); - fCurrentLength += estLen; - fRowGroup.incRowCount(); - fRow.nextRow(); + if (!fRm->getMemory(newSize, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += newSize; - if (fRowGroup.getRowCount() >= fRowsPerRG) - { - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - - if (!fRm->getMemory(newSize, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += newSize; - - fDataQueue.push(fData); - fData.reinit(fRowGroup, fRowsPerRG); - fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); - fRowGroup.getRow(0, &fRow); - } + fDataQueue.push(fData); + fData.reinit(fRowGroup, fRowsPerRG); + fRowGroup.setData(&fData); + fRowGroup.resetRowGroup(0); + fRowGroup.getRow(0, &fRow); } + } } - void GroupConcatNoOrder::merge(GroupConcator* gc) { - GroupConcatNoOrder* in = dynamic_cast(gc); + GroupConcatNoOrder* in = dynamic_cast(gc); - while (in->fDataQueue.size() > 0) - { - fDataQueue.push(in->fDataQueue.front()); - in->fDataQueue.pop(); - } + while (in->fDataQueue.size() > 0) + { + fDataQueue.push(in->fDataQueue.front()); + in->fDataQueue.pop(); + } - fDataQueue.push(in->fData); - fMemSize += in->fMemSize; - in->fMemSize = 0; + fDataQueue.push(in->fData); + fMemSize += in->fMemSize; + in->fMemSize = 0; } - void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) { - ostringstream oss; - bool addSep = false; + ostringstream oss; + bool addSep = false; - fDataQueue.push(fData); + fDataQueue.push(fData); - while (fDataQueue.size() > 0) + while (fDataQueue.size() > 0) + { + fRowGroup.setData(&fDataQueue.front()); + fRowGroup.getRow(0, &fRow); + + for (uint64_t i = 0; i < fRowGroup.getRowCount(); i++) { - fRowGroup.setData(&fDataQueue.front()); - fRowGroup.getRow(0, &fRow); + if (addSep) + oss << sep; + else + addSep = true; - for (uint64_t i = 0; i < fRowGroup.getRowCount(); i++) - { - if (addSep) - oss << sep; - else - addSep = true; - - outputRow(oss, fRow); - fRow.nextRow(); - } - - fDataQueue.pop(); + outputRow(oss, fRow); + fRow.nextRow(); } - int64_t resultSize = oss.str().size(); - resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; - fOutputString.reset(new uint8_t[resultSize + 2]); - fOutputString[resultSize] = '\0'; - fOutputString[resultSize + 1] = '\0'; + fDataQueue.pop(); + } - strncpy((char*)fOutputString.get(), - oss.str().c_str(), resultSize); + int64_t resultSize = oss.str().size(); + resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; + fOutputString.reset(new uint8_t[resultSize + 2]); + fOutputString[resultSize] = '\0'; + fOutputString[resultSize + 1] = '\0'; + + strncpy((char*)fOutputString.get(), oss.str().c_str(), resultSize); } - const string GroupConcatNoOrder::toString() const { - return GroupConcator::toString(); + return GroupConcator::toString(); } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/groupconcat.h b/dbcon/joblist/groupconcat.h index e0bc243c1..ba348e2b7 100644 --- a/dbcon/joblist/groupconcat.h +++ b/dbcon/joblist/groupconcat.h @@ -17,7 +17,6 @@ // $Id: groupconcat.h 9705 2013-07-17 20:06:07Z pleblanc $ - /** @file */ #ifndef GROUP_CONCAT_H @@ -29,10 +28,10 @@ #include #include -#include "returnedcolumn.h" // SRCP -#include "rowgroup.h" // RowGroup -#include "rowaggregation.h" // SP_GroupConcat -#include "limitedorderby.h" // IdbOrderBy +#include "returnedcolumn.h" // SRCP +#include "rowgroup.h" // RowGroup +#include "rowaggregation.h" // SP_GroupConcat +#include "limitedorderby.h" // IdbOrderBy #if defined(_MSC_VER) && defined(JOBLIST_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -42,155 +41,148 @@ namespace joblist { - // forward reference struct JobInfo; -class GroupConcator; -class ResourceManager; - +class GroupConcator; +class ResourceManager; class GroupConcatInfo { -public: - GroupConcatInfo(); - virtual ~GroupConcatInfo(); + public: + GroupConcatInfo(); + virtual ~GroupConcatInfo(); - void prepGroupConcat(JobInfo&); - void mapColumns(const rowgroup::RowGroup&); + void prepGroupConcat(JobInfo&); + void mapColumns(const rowgroup::RowGroup&); - std::set& columns() - { - return fColumns; - } - std::vector& groupConcat() - { - return fGroupConcat; - } + std::set& columns() + { + return fColumns; + } + std::vector& groupConcat() + { + return fGroupConcat; + } - const std::string toString() const; + const std::string toString() const; -protected: - uint32_t getColumnKey(const execplan::SRCP& srcp, JobInfo& jobInfo); - boost::shared_array makeMapping(const rowgroup::RowGroup&, const rowgroup::RowGroup&); + protected: + uint32_t getColumnKey(const execplan::SRCP& srcp, JobInfo& jobInfo); + boost::shared_array makeMapping(const rowgroup::RowGroup&, const rowgroup::RowGroup&); - std::set fColumns; - std::vector fGroupConcat; + std::set fColumns; + std::vector fGroupConcat; }; - class GroupConcatAgUM : public rowgroup::GroupConcatAg { -public: - EXPORT GroupConcatAgUM(rowgroup::SP_GroupConcat&); - EXPORT ~GroupConcatAgUM(); + public: + EXPORT GroupConcatAgUM(rowgroup::SP_GroupConcat&); + EXPORT ~GroupConcatAgUM(); - using rowgroup::GroupConcatAg::merge; - void initialize(); - void processRow(const rowgroup::Row&); - EXPORT void merge(const rowgroup::Row&, int64_t); - boost::scoped_ptr& concator() - { - return fConcator; - } + using rowgroup::GroupConcatAg::merge; + void initialize(); + void processRow(const rowgroup::Row&); + EXPORT void merge(const rowgroup::Row&, int64_t); + boost::scoped_ptr& concator() + { + return fConcator; + } - EXPORT void getResult(uint8_t*); - EXPORT uint8_t* getResult(); + EXPORT void getResult(uint8_t*); + EXPORT uint8_t* getResult(); -protected: - void applyMapping(const boost::shared_array&, const rowgroup::Row&); + protected: + void applyMapping(const boost::shared_array&, const rowgroup::Row&); - boost::scoped_ptr fConcator; - boost::scoped_array fData; - rowgroup::Row fRow; - bool fNoOrder; + boost::scoped_ptr fConcator; + boost::scoped_array fData; + rowgroup::Row fRow; + bool fNoOrder; }; - // GROUP_CONCAT base class GroupConcator { -public: - GroupConcator(); - virtual ~GroupConcator(); + public: + GroupConcator(); + virtual ~GroupConcator(); - virtual void initialize(const rowgroup::SP_GroupConcat&); - virtual void processRow(const rowgroup::Row&) = 0; + virtual void initialize(const rowgroup::SP_GroupConcat&); + virtual void processRow(const rowgroup::Row&) = 0; - virtual void merge(GroupConcator*) = 0; - virtual void getResult(uint8_t* buff, const std::string& sep) = 0; - virtual uint8_t* getResult(const std::string& sep); + virtual void merge(GroupConcator*) = 0; + virtual void getResult(uint8_t* buff, const std::string& sep) = 0; + virtual uint8_t* getResult(const std::string& sep); - virtual const std::string toString() const; + virtual const std::string toString() const; -protected: - virtual bool concatColIsNull(const rowgroup::Row&); - virtual void outputRow(std::ostringstream&, const rowgroup::Row&); - virtual int64_t lengthEstimate(const rowgroup::Row&); + protected: + virtual bool concatColIsNull(const rowgroup::Row&); + virtual void outputRow(std::ostringstream&, const rowgroup::Row&); + virtual int64_t lengthEstimate(const rowgroup::Row&); - std::vector fConcatColumns; - std::vector > fConstCols; - int64_t fCurrentLength; - int64_t fGroupConcatLen; - int64_t fConstantLen; - boost::scoped_array fOutputString; - std::string fTimeZone; + std::vector fConcatColumns; + std::vector > fConstCols; + int64_t fCurrentLength; + int64_t fGroupConcatLen; + int64_t fConstantLen; + boost::scoped_array fOutputString; + std::string fTimeZone; }; - // For GROUP_CONCAT withour distinct or orderby class GroupConcatNoOrder : public GroupConcator { -public: - GroupConcatNoOrder(); - virtual ~GroupConcatNoOrder(); + public: + GroupConcatNoOrder(); + virtual ~GroupConcatNoOrder(); - void initialize(const rowgroup::SP_GroupConcat&); - void processRow(const rowgroup::Row&); + void initialize(const rowgroup::SP_GroupConcat&); + void processRow(const rowgroup::Row&); - void merge(GroupConcator*); - using GroupConcator::getResult; - void getResult(uint8_t* buff, const std::string& sep); + void merge(GroupConcator*); + using GroupConcator::getResult; + void getResult(uint8_t* buff, const std::string& sep); - const std::string toString() const; + const std::string toString() const; -protected: - rowgroup::RowGroup fRowGroup; - rowgroup::Row fRow; - rowgroup::RGData fData; - std::queue fDataQueue; - uint64_t fRowsPerRG; - uint64_t fErrorCode; - uint64_t fMemSize; - ResourceManager* fRm; - boost::shared_ptr fSessionMemLimit; + protected: + rowgroup::RowGroup fRowGroup; + rowgroup::Row fRow; + rowgroup::RGData fData; + std::queue fDataQueue; + uint64_t fRowsPerRG; + uint64_t fErrorCode; + uint64_t fMemSize; + ResourceManager* fRm; + boost::shared_ptr fSessionMemLimit; }; - // ORDER BY used in GROUP_CONCAT class // This version is for GROUP_CONCAT, the size is limited by the group_concat_max_len. class GroupConcatOrderBy : public GroupConcator, public ordering::IdbOrderBy { -public: - GroupConcatOrderBy(); - virtual ~GroupConcatOrderBy(); + public: + GroupConcatOrderBy(); + virtual ~GroupConcatOrderBy(); - using ordering::IdbOrderBy::initialize; - void initialize(const rowgroup::SP_GroupConcat&); - void processRow(const rowgroup::Row&); - uint64_t getKeyLength() const; + using ordering::IdbOrderBy::initialize; + void initialize(const rowgroup::SP_GroupConcat&); + void processRow(const rowgroup::Row&); + uint64_t getKeyLength() const; - void merge(GroupConcator*); - using GroupConcator::getResult; - void getResult(uint8_t* buff, const std::string& sep); + void merge(GroupConcator*); + using GroupConcator::getResult; + void getResult(uint8_t* buff, const std::string& sep); - const std::string toString() const; + const std::string toString() const; -protected: + protected: }; -} +} // namespace joblist #undef EXPORT #endif // GROUP_CONCAT_H - diff --git a/dbcon/joblist/jl_logger.cpp b/dbcon/joblist/jl_logger.cpp index 2ada20df7..905848a3c 100644 --- a/dbcon/joblist/jl_logger.cpp +++ b/dbcon/joblist/jl_logger.cpp @@ -36,51 +36,45 @@ namespace boost::mutex logMutex; }; - namespace joblist { - -Logger::Logger() : fLogId(5), - fImpl(new logging::Logger(5)) +Logger::Logger() : fLogId(5), fImpl(new logging::Logger(5)) { - MsgMap msgMap; + MsgMap msgMap; - msgMap[LogDefaultMsg] = Message(LogDefaultMsg); - msgMap[LogSQLTrace] = Message(LogSQLTrace); - msgMap[LogNoPrimProcs] = Message(LogNoPrimProcs); - msgMap[LogMakeJobList] = Message(LogMakeJobList); - msgMap[LogRDRequest] = Message(LogRDRequest); - msgMap[LogRDRequestWait] = Message(LogRDRequestWait); - msgMap[LogRDReturn] = Message(LogRDReturn); - msgMap[LogRMResourceChange] = Message(LogRMResourceChange); - msgMap[LogRMResourceChangeError] = Message(LogRMResourceChangeError); + msgMap[LogDefaultMsg] = Message(LogDefaultMsg); + msgMap[LogSQLTrace] = Message(LogSQLTrace); + msgMap[LogNoPrimProcs] = Message(LogNoPrimProcs); + msgMap[LogMakeJobList] = Message(LogMakeJobList); + msgMap[LogRDRequest] = Message(LogRDRequest); + msgMap[LogRDRequestWait] = Message(LogRDRequestWait); + msgMap[LogRDReturn] = Message(LogRDReturn); + msgMap[LogRMResourceChange] = Message(LogRMResourceChange); + msgMap[LogRMResourceChangeError] = Message(LogRMResourceChangeError); - fImpl->msgMap(msgMap); + fImpl->msgMap(msgMap); } -void catchHandler(const string& ex, - int c, SErrorInfo& ei, - unsigned sid, - logging::LOG_TYPE level) +void catchHandler(const string& ex, int c, SErrorInfo& ei, unsigned sid, logging::LOG_TYPE level) { - boost::mutex::scoped_lock lk(logMutex); + boost::mutex::scoped_lock lk(logMutex); - if (ei->errCode == 0) - { - ei->errMsg = ex; - ei->errCode = c; - } + if (ei->errCode == 0) + { + ei->errMsg = ex; + ei->errCode = c; + } - Logger log; - log.setLoggingSession(sid); - log.logMessage(level, ex); + Logger log; + log.setLoggingSession(sid); + log.logMessage(level, ex); } const string Logger::logMessage(logging::LOG_TYPE logLevel, unsigned idbErrorCode) { - string errMsg = logging::IDBErrorInfo::instance()->errorMsg(idbErrorCode); - logMessage(logLevel, errMsg); - return errMsg; + string errMsg = logging::IDBErrorInfo::instance()->errorMsg(idbErrorCode); + logMessage(logLevel, errMsg); + return errMsg; } -} +} // namespace joblist diff --git a/dbcon/joblist/jl_logger.h b/dbcon/joblist/jl_logger.h index cfa4da08e..d82ddb688 100644 --- a/dbcon/joblist/jl_logger.h +++ b/dbcon/joblist/jl_logger.h @@ -47,54 +47,54 @@ const unsigned LogRDReturn = logging::M0064; const unsigned LogRMResourceChange = logging::M0066; const unsigned LogRMResourceChangeError = logging::M0067; - /** @brief message log wrapper class */ class Logger { -public: - Logger(); + public: + Logger(); - const std::string logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, - const logging::Message::Args& args, const logging::LoggingID& logInfo) - { - return fImpl->logMessage(logLevel, mid, args, logInfo); - } + const std::string logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, + const logging::Message::Args& args, const logging::LoggingID& logInfo) + { + return fImpl->logMessage(logLevel, mid, args, logInfo); + } - const std::string logMessage(logging::LOG_TYPE logLevel, const std::string& msg, logging::Message::MessageID mid = LogDefaultMsg ) - { - logging::Message::Args args; - args.add(msg); - return fImpl->logMessage(logLevel, mid, args, fLogId); - } + const std::string logMessage(logging::LOG_TYPE logLevel, const std::string& msg, + logging::Message::MessageID mid = LogDefaultMsg) + { + logging::Message::Args args; + args.add(msg); + return fImpl->logMessage(logLevel, mid, args, fLogId); + } - const std::string logMessage(logging::LOG_TYPE logLevel, unsigned idbErrorCode); + const std::string logMessage(logging::LOG_TYPE logLevel, unsigned idbErrorCode); - void setLoggingSession(unsigned sid) - { - fLogId.fSessionID = sid; - } - void setLoggingTxn(unsigned txn) - { - fLogId.fTxnID = txn; - } - void setLoggingThd(unsigned thr) - { - fLogId.fThdID = thr; - } -private: - // defaults okay - //Logger(const Logger& rhs); - //Logger& operator=(const Logger& rhs); - logging::LoggingID fLogId; + void setLoggingSession(unsigned sid) + { + fLogId.fSessionID = sid; + } + void setLoggingTxn(unsigned txn) + { + fLogId.fTxnID = txn; + } + void setLoggingThd(unsigned thr) + { + fLogId.fThdID = thr; + } - logging::SPL fImpl; + private: + // defaults okay + // Logger(const Logger& rhs); + // Logger& operator=(const Logger& rhs); + logging::LoggingID fLogId; + + logging::SPL fImpl; }; typedef boost::shared_ptr SPJL; void catchHandler(const std::string& s, int c, SErrorInfo& errorInfo, unsigned sid = 0, logging::LOG_TYPE = logging::LOG_TYPE_CRITICAL); -} +} // namespace joblist #endif - diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 4dabc57f3..6ae8724e6 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -18,7 +18,6 @@ // $Id: jlf_common.cpp 9655 2013-06-25 23:08:13Z xlou $ - #include "calpontsystemcatalog.h" #include "aggregatecolumn.h" #include "pseudocolumn.h" @@ -42,727 +41,653 @@ using namespace joblist; namespace { - // @brief Returns unique key for a column, table, or expresssion. -uint32_t uniqTupleKey(JobInfo& jobInfo, - CalpontSystemCatalog::OID& o, - CalpontSystemCatalog::OID& t, - const string& cn, - const string& ca, - const string& tn, - const string& ta, - const string& sn, - const string& vw, - uint32_t pi, - uint64_t en, - bool correlated = false) +uint32_t uniqTupleKey(JobInfo& jobInfo, CalpontSystemCatalog::OID& o, CalpontSystemCatalog::OID& t, + const string& cn, const string& ca, const string& tn, const string& ta, + const string& sn, const string& vw, uint32_t pi, uint64_t en, bool correlated = false) { - uint64_t subId = jobInfo.subId; + uint64_t subId = jobInfo.subId; - if (correlated && jobInfo.pJobInfo) - subId = jobInfo.pJobInfo->subId; + if (correlated && jobInfo.pJobInfo) + subId = jobInfo.pJobInfo->subId; - string alias(ta); -// if (!ca.empty()) -// alias += "." + ca; - string nm(ta); + string alias(ta); + // if (!ca.empty()) + // alias += "." + ca; + string nm(ta); - if (!cn.empty()) - nm += "." + cn; + if (!cn.empty()) + nm += "." + cn; - UniqId id(o, ta, sn, vw, pi, subId); - TupleKeyMap::iterator iter = jobInfo.keyInfo->tupleKeyMap.find(id); + UniqId id(o, ta, sn, vw, pi, subId); + TupleKeyMap::iterator iter = jobInfo.keyInfo->tupleKeyMap.find(id); - if (iter != jobInfo.keyInfo->tupleKeyMap.end()) - return iter->second; + if (iter != jobInfo.keyInfo->tupleKeyMap.end()) + return iter->second; - uint32_t newId = jobInfo.keyInfo->nextKey++; -//cout << "new id: " << newId << " -- " << o << ", " << pi << ", " << nm << ", " << vw << ", " << sn << ", " << subId << endl; - jobInfo.keyInfo->tupleKeyMap[id] = newId; - jobInfo.keyInfo->tupleKeyVec.push_back(id); - jobInfo.keyInfo->tupleKeyToTableOid.insert(make_pair(newId, t)); - jobInfo.keyInfo->crossEngine.push_back((en != 0)); + uint32_t newId = jobInfo.keyInfo->nextKey++; + // cout << "new id: " << newId << " -- " << o << ", " << pi << ", " << nm << ", " << vw << ", " << sn << ", + // " << subId << endl; + jobInfo.keyInfo->tupleKeyMap[id] = newId; + jobInfo.keyInfo->tupleKeyVec.push_back(id); + jobInfo.keyInfo->tupleKeyToTableOid.insert(make_pair(newId, t)); + jobInfo.keyInfo->crossEngine.push_back((en != 0)); - string ss = vw; + string ss = vw; - if (ss.length() > 0) - ss += "."; + if (ss.length() > 0) + ss += "."; - if (sn.length() > 0) - ss += sn + "."; + if (sn.length() > 0) + ss += sn + "."; - if (o != t) + if (o != t) + { + string name = cn; + + if (!ca.empty()) // has an alias + name = ca; + else if (ta.compare(0, 5, "$sub_") && ta.compare(0, 4, "$exp")) // compare != 0 + name = ss + ta + "." + name; + + jobInfo.keyInfo->tupleKeyToName.push_back(name); + jobInfo.keyInfo->keyName.insert(make_pair(newId, cn)); + } + else + { + string name = ta; + bool useAlias = true; + + if (name.empty()) { - string name = cn; + name = tn; + useAlias = false; + } - if (!ca.empty()) // has an alias - name = ca; - else if (ta.compare(0, 5, "$sub_") && ta.compare(0, 4, "$exp")) // compare != 0 - name = ss + ta + "." + name; - - jobInfo.keyInfo->tupleKeyToName.push_back(name); - jobInfo.keyInfo->keyName.insert(make_pair(newId, cn)); + if (tn.compare(0, 4, "$sub") && tn.compare(0, 4, "$exp")) + { + if (!useAlias) + name = ss + name; + } + else if (tn.compare(0, 4, "$sub") == 0) + { + name = "sub-query"; } else { - string name = ta; - bool useAlias = true; - - if (name.empty()) - { - name = tn; - useAlias = false; - } - - if (tn.compare(0, 4, "$sub") && tn.compare(0, 4, "$exp")) - { - if (!useAlias) - name = ss + name; - } - else if (tn.compare(0, 4, "$sub") == 0) - { - name = "sub-query"; - } - else - { - name = "expression"; - } - - jobInfo.keyInfo->tupleKeyToName.push_back(name); - jobInfo.keyInfo->keyName.insert(make_pair(newId, tn)); + name = "expression"; } - return newId; -} + jobInfo.keyInfo->tupleKeyToName.push_back(name); + jobInfo.keyInfo->keyName.insert(make_pair(newId, tn)); + } + return newId; +} // @brief Returns a suitably fudged column width uint32_t fudgeWidth(const CalpontSystemCatalog::ColType& ict, CalpontSystemCatalog::OID oid) { - CalpontSystemCatalog::OID dictOid = isDictCol(ict); - CalpontSystemCatalog::ColType ct = ict; + CalpontSystemCatalog::OID dictOid = isDictCol(ict); + CalpontSystemCatalog::ColType ct = ict; - if (ct.colDataType != CalpontSystemCatalog::VARBINARY && ct.colDataType != CalpontSystemCatalog::BLOB) - { - if (ct.colDataType == CalpontSystemCatalog::VARCHAR || ct.colDataType == CalpontSystemCatalog::TEXT) - ct.colWidth++; + if (ct.colDataType != CalpontSystemCatalog::VARBINARY && ct.colDataType != CalpontSystemCatalog::BLOB) + { + if (ct.colDataType == CalpontSystemCatalog::VARCHAR || ct.colDataType == CalpontSystemCatalog::TEXT) + ct.colWidth++; - //Round colWidth up - if (ct.colWidth == 3) - ct.colWidth = 4; - else if (ct.colWidth == 5 || ct.colWidth == 6 || ct.colWidth == 7) - ct.colWidth = 8; - else if (dictOid > 0 && oid != dictOid) // token column - ct.colWidth = 8; - } - else - { - ct.colWidth += 2; // insert the length bytes - } + // Round colWidth up + if (ct.colWidth == 3) + ct.colWidth = 4; + else if (ct.colWidth == 5 || ct.colWidth == 6 || ct.colWidth == 7) + ct.colWidth = 8; + else if (dictOid > 0 && oid != dictOid) // token column + ct.colWidth = 8; + } + else + { + ct.colWidth += 2; // insert the length bytes + } - return ct.colWidth; + return ct.colWidth; } - // @brief Set some tuple info -TupleInfo setTupleInfo_(const CalpontSystemCatalog::ColType& ct, - CalpontSystemCatalog::OID col_oid, - JobInfo& jobInfo, - CalpontSystemCatalog::OID tbl_oid, - const string& col_name, - const string& col_alias, - const string& sch_name, - const string& tbl_name, - const string& tbl_alias, - const string& vw_name, - bool correlated = false, - uint32_t pc_id = 0, - uint64_t engine = 0) +TupleInfo setTupleInfo_(const CalpontSystemCatalog::ColType& ct, CalpontSystemCatalog::OID col_oid, + JobInfo& jobInfo, CalpontSystemCatalog::OID tbl_oid, const string& col_name, + const string& col_alias, const string& sch_name, const string& tbl_name, + const string& tbl_alias, const string& vw_name, bool correlated = false, + uint32_t pc_id = 0, uint64_t engine = 0) { - // get the unique tupleOids for this column - uint32_t tbl_key = uniqTupleKey(jobInfo, tbl_oid, tbl_oid, "", "", tbl_name, tbl_alias, - sch_name, vw_name, 0, engine, correlated); - uint32_t col_key = uniqTupleKey(jobInfo, col_oid, tbl_oid, col_name, col_alias, tbl_name, - tbl_alias, sch_name, vw_name, pc_id, engine, correlated); - //If this is the first time we've seen this col, add it to the tim - TupleInfoMap::iterator it = jobInfo.keyInfo->tupleInfoMap.find(col_key); - TupleInfo ti; + // get the unique tupleOids for this column + uint32_t tbl_key = uniqTupleKey(jobInfo, tbl_oid, tbl_oid, "", "", tbl_name, tbl_alias, sch_name, vw_name, + 0, engine, correlated); + uint32_t col_key = uniqTupleKey(jobInfo, col_oid, tbl_oid, col_name, col_alias, tbl_name, tbl_alias, + sch_name, vw_name, pc_id, engine, correlated); + // If this is the first time we've seen this col, add it to the tim + TupleInfoMap::iterator it = jobInfo.keyInfo->tupleInfoMap.find(col_key); + TupleInfo ti; - if (it != jobInfo.keyInfo->tupleInfoMap.end()) - { - //We've seen the key - ti = it->second; - } - else - { - //Haven't even seen the table yet, much less this col - ti = TupleInfo(fudgeWidth(ct, col_oid), col_oid, col_key, tbl_key, - ct.scale, ct.precision, ct.colDataType, ct.charsetNumber); - jobInfo.keyInfo->tupleInfoMap[col_key] = ti; - jobInfo.keyInfo->colKeyToTblKey[col_key] = tbl_key; - jobInfo.keyInfo->colKeyToTblKey[tbl_key] = tbl_key; - jobInfo.keyInfo->colType[col_key] = ct; - jobInfo.keyInfo->pseudoType[col_key] = pc_id; - } + if (it != jobInfo.keyInfo->tupleInfoMap.end()) + { + // We've seen the key + ti = it->second; + } + else + { + // Haven't even seen the table yet, much less this col + ti = TupleInfo(fudgeWidth(ct, col_oid), col_oid, col_key, tbl_key, ct.scale, ct.precision, ct.colDataType, + ct.charsetNumber); + jobInfo.keyInfo->tupleInfoMap[col_key] = ti; + jobInfo.keyInfo->colKeyToTblKey[col_key] = tbl_key; + jobInfo.keyInfo->colKeyToTblKey[tbl_key] = tbl_key; + jobInfo.keyInfo->colType[col_key] = ct; + jobInfo.keyInfo->pseudoType[col_key] = pc_id; + } - if (pc_id > 0 && jobInfo.pseudoColTable.find(tbl_key) == jobInfo.pseudoColTable.end()) - jobInfo.pseudoColTable.insert(tbl_key); + if (pc_id > 0 && jobInfo.pseudoColTable.find(tbl_key) == jobInfo.pseudoColTable.end()) + jobInfo.pseudoColTable.insert(tbl_key); - return ti; + return ti; } - -uint32_t getTupleKey_(const JobInfo& jobInfo, - CalpontSystemCatalog::OID oid, - const string& colName, - const string& tblAlias, - const string& schema, - const string& view, - bool correlated = false, - uint32_t pseudo = 0, - uint64_t engine = 0) +uint32_t getTupleKey_(const JobInfo& jobInfo, CalpontSystemCatalog::OID oid, const string& colName, + const string& tblAlias, const string& schema, const string& view, + bool correlated = false, uint32_t pseudo = 0, uint64_t engine = 0) { - uint64_t subId = jobInfo.subId; + uint64_t subId = jobInfo.subId; - if (correlated && jobInfo.pJobInfo) - subId = jobInfo.pJobInfo->subId; + if (correlated && jobInfo.pJobInfo) + subId = jobInfo.pJobInfo->subId; - string alias(tblAlias); - string name(tblAlias); + string alias(tblAlias); + string name(tblAlias); - if (!colName.empty()) - name += "." + colName; + if (!colName.empty()) + name += "." + colName; -// if (!colAlias.empty()) -// alias += "." + colAlias; - UniqId id(oid, tblAlias, schema, view, pseudo, subId); - TupleKeyMap::const_iterator iter = jobInfo.keyInfo->tupleKeyMap.find(id); + // if (!colAlias.empty()) + // alias += "." + colAlias; + UniqId id(oid, tblAlias, schema, view, pseudo, subId); + TupleKeyMap::const_iterator iter = jobInfo.keyInfo->tupleKeyMap.find(id); - if (iter != jobInfo.keyInfo->tupleKeyMap.end()) - return iter->second; + if (iter != jobInfo.keyInfo->tupleKeyMap.end()) + return iter->second; - // dictionaryscan tableOid is 0 in the tuplehashjoin. - if (oid != 0) - { - ostringstream strstm; - strstm << "(" << oid << ", "; + // dictionaryscan tableOid is 0 in the tuplehashjoin. + if (oid != 0) + { + ostringstream strstm; + strstm << "(" << oid << ", "; - if (!alias.empty()) - strstm << alias; + if (!alias.empty()) + strstm << alias; - if (!view.empty()) - strstm << ", " << view; + if (!view.empty()) + strstm << ", " << view; - strstm << ") not found in tuple info map."; + strstm << ") not found in tuple info map."; - Message::Args args; - args.add(strstm.str()); - jobInfo.logger->logMessage(LOG_TYPE_DEBUG, LogMakeJobList, args, - LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); - cerr << strstm.str() << endl; + Message::Args args; + args.add(strstm.str()); + jobInfo.logger->logMessage(LOG_TYPE_DEBUG, LogMakeJobList, args, + LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); + cerr << strstm.str() << endl; -//cout << "not found: " << oid << ", " << name << ", " << view << ", " << schema << ", " << subId << endl; - throw logic_error("column is not found in info map."); - } + // cout << "not found: " << oid << ", " << name << ", " << view << ", " << schema << ", " << subId << + // endl; + throw logic_error("column is not found in info map."); + } - return static_cast(-1); + return static_cast(-1); } -} - +} // namespace namespace joblist { - -UniqId::UniqId(const execplan::SimpleColumn* sc) : - fId(sc->oid()), - //fName(extractTableAlias(sc)+"."+sc->columnName()), - fTable(extractTableAlias(sc)), - fSchema(sc->schemaName()), - fView(sc->viewName()), - fPseudo(0), - fSubId(-1) +UniqId::UniqId(const execplan::SimpleColumn* sc) + : fId(sc->oid()) + , + // fName(extractTableAlias(sc)+"."+sc->columnName()), + fTable(extractTableAlias(sc)) + , fSchema(sc->schemaName()) + , fView(sc->viewName()) + , fPseudo(0) + , fSubId(-1) { - const PseudoColumn* pc = dynamic_cast(sc); - uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; - fPseudo = pseudoType; + const PseudoColumn* pc = dynamic_cast(sc); + uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; + fPseudo = pseudoType; } - -UniqId::UniqId(int o, const execplan::SimpleColumn* sc) : - fId(o), - //fName(extractTableAlias(sc)+"."+sc->columnName()), - fTable(extractTableAlias(sc)), - fSchema(sc->schemaName()), - fView(sc->viewName()), - fPseudo(0), - fSubId(-1) +UniqId::UniqId(int o, const execplan::SimpleColumn* sc) + : fId(o) + , + // fName(extractTableAlias(sc)+"."+sc->columnName()), + fTable(extractTableAlias(sc)) + , fSchema(sc->schemaName()) + , fView(sc->viewName()) + , fPseudo(0) + , fSubId(-1) { } - string UniqId::toString() const { - ostringstream strstm; - strstm << fId << ":" << fTable << ":" << fSchema << ":" << fView << ":" - << fPseudo << ":" << (int64_t)fSubId; - return strstm.str(); + ostringstream strstm; + strstm << fId << ":" << fTable << ":" << fSchema << ":" << fView << ":" << fPseudo << ":" + << (int64_t)fSubId; + return strstm.str(); } - //------------------------------------------------------------------------------ // Returns the table alias for the specified column //------------------------------------------------------------------------------ string extractTableAlias(const SimpleColumn* sc) { - return sc->tableAlias(); + return sc->tableAlias(); } - //------------------------------------------------------------------------------ // Returns the table alias for the specified column //------------------------------------------------------------------------------ string extractTableAlias(const SSC& sc) { - return sc->tableAlias(); + return sc->tableAlias(); } - //------------------------------------------------------------------------------ // Returns OID associated with colType if it is a dictionary column, else // the value returned for the OID is 0. //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { - if (colType.isWideDecimalType()) return 0; - - if (colType.colWidth > 8) return colType.ddn.dictOID; - - if (colType.colDataType == CalpontSystemCatalog::VARCHAR && - colType.colWidth > 7) return colType.ddn.dictOID; - - if (colType.colDataType == CalpontSystemCatalog::VARBINARY || - colType.colDataType == CalpontSystemCatalog::BLOB || - colType.colDataType == CalpontSystemCatalog::TEXT) - return colType.ddn.dictOID; - + if (colType.isWideDecimalType()) return 0; -} + if (colType.colWidth > 8) + return colType.ddn.dictOID; + + if (colType.colDataType == CalpontSystemCatalog::VARCHAR && colType.colWidth > 7) + return colType.ddn.dictOID; + + if (colType.colDataType == CalpontSystemCatalog::VARBINARY || + colType.colDataType == CalpontSystemCatalog::BLOB || colType.colDataType == CalpontSystemCatalog::TEXT) + return colType.ddn.dictOID; + + return 0; +} //------------------------------------------------------------------------------ // Determines if colType is a character column //------------------------------------------------------------------------------ bool isCharCol(const CalpontSystemCatalog::ColType& colType) { - switch (colType.colDataType) - { - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - return true; - break; + switch (colType.colDataType) + { + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: return true; break; - default: - return false; - break; - } + default: return false; break; + } - return false; + return false; } - //------------------------------------------------------------------------------ // Returns OID associated with a table //------------------------------------------------------------------------------ CalpontSystemCatalog::OID tableOid(const SimpleColumn* sc, boost::shared_ptr cat) { - if (sc->schemaName().empty()) - return execplan::CNX_VTABLE_ID; + if (sc->schemaName().empty()) + return execplan::CNX_VTABLE_ID; - if (sc->isColumnStore() == false) - return 0; + if (sc->isColumnStore() == false) + return 0; - CalpontSystemCatalog::ROPair p = cat->tableRID(make_table(sc->schemaName(), - sc->tableName())); - return p.objnum; + CalpontSystemCatalog::ROPair p = cat->tableRID(make_table(sc->schemaName(), sc->tableName())); + return p.objnum; } -uint32_t getTupleKey(JobInfo& jobInfo, - const execplan::SimpleColumn* sc, - bool add) +uint32_t getTupleKey(JobInfo& jobInfo, const execplan::SimpleColumn* sc, bool add) { - int key = -1; - const PseudoColumn* pc = dynamic_cast(sc); - uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; + int key = -1; + const PseudoColumn* pc = dynamic_cast(sc); + uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; - if (sc == NULL) + if (sc == NULL) + { + return -1; + } + + if (add) + { + // setTupleInfo first if add is true, ok if already set. + if (sc->schemaName().empty()) { - return -1; - } - - if (add) - { - // setTupleInfo first if add is true, ok if already set. - if (sc->schemaName().empty()) - { - SimpleColumn tmp(*sc, jobInfo.sessionId); - tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - key = getTupleKey(jobInfo, &tmp); // sub-query should be there - } - else - { - CalpontSystemCatalog::ColType ct = sc->colType(); - string alias(extractTableAlias(sc)); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); - key = ti.key; - - - CalpontSystemCatalog::OID dictOid = isDictCol(ct); - - if (dictOid > 0) - { - ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); - jobInfo.keyInfo->dictKeyMap[key] = ti.key; - key = ti.key; - } - } + SimpleColumn tmp(*sc, jobInfo.sessionId); + tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + key = getTupleKey(jobInfo, &tmp); // sub-query should be there } else { - // TupleInfo is expected to be set already - return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc), - sc->schemaName(), sc->viewName(), - ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), - pseudoType, (sc->isColumnStore() ? 0 : 1)); - } + CalpontSystemCatalog::ColType ct = sc->colType(); + string alias(extractTableAlias(sc)); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + key = ti.key; - return key; + CalpontSystemCatalog::OID dictOid = isDictCol(ct); + + if (dictOid > 0) + { + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[key] = ti.key; + key = ti.key; + } + } + } + else + { + // TupleInfo is expected to be set already + return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc), sc->schemaName(), + sc->viewName(), ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), pseudoType, + (sc->isColumnStore() ? 0 : 1)); + } + + return key; } - + uint32_t getTupleKey(JobInfo& jobInfo, const SRCP& srcp, bool add) { - int key = -1; + int key = -1; - if (add) + if (add) + { + // setTupleInfo first if add is true, ok if already set. + const SimpleColumn* sc = dynamic_cast(srcp.get()); + + if (sc != NULL) { - // setTupleInfo first if add is true, ok if already set. - const SimpleColumn* sc = dynamic_cast(srcp.get()); + if (sc->schemaName().empty()) + { + SimpleColumn tmp(*sc, jobInfo.sessionId); + tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + key = getTupleKey(jobInfo, &tmp); // sub-query should be there + } + else + { + CalpontSystemCatalog::ColType ct = sc->colType(); + string alias(extractTableAlias(sc)); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + key = ti.key; - if (sc != NULL) + CalpontSystemCatalog::OID dictOid = isDictCol(ct); + + if (dictOid > 0) { - if (sc->schemaName().empty()) - { - SimpleColumn tmp(*sc, jobInfo.sessionId); - tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - key = getTupleKey(jobInfo, &tmp); // sub-query should be there - } - else - { - CalpontSystemCatalog::ColType ct = sc->colType(); - string alias(extractTableAlias(sc)); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); - key = ti.key; - - CalpontSystemCatalog::OID dictOid = isDictCol(ct); - - if (dictOid > 0) - { - ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); - jobInfo.keyInfo->dictKeyMap[key] = ti.key; - key = ti.key; - } - } - } - else - { - CalpontSystemCatalog::ColType ct = srcp->resultType(); - TupleInfo ti(setExpTupleInfo(ct, srcp->expressionId(), srcp->alias(), jobInfo)); - key = ti.key; + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[key] = ti.key; + key = ti.key; } + } } else { - // TupleInfo is expected to be set already - const SimpleColumn* sc = dynamic_cast(srcp.get()); - - if (sc != NULL) - { - if (sc->schemaName().empty()) - { - SimpleColumn tmp(*sc, jobInfo.sessionId); - tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - key = getTupleKey(jobInfo, &tmp); - } - else - { - key = getTupleKey(jobInfo, sc); - } - - // check if this is a dictionary column - if (jobInfo.keyInfo->dictKeyMap.find(key) != jobInfo.keyInfo->dictKeyMap.end()) - key = jobInfo.keyInfo->dictKeyMap[key]; - } - else - { - key = getExpTupleKey(jobInfo, srcp->expressionId()); - } + CalpontSystemCatalog::ColType ct = srcp->resultType(); + TupleInfo ti(setExpTupleInfo(ct, srcp->expressionId(), srcp->alias(), jobInfo)); + key = ti.key; } + } + else + { + // TupleInfo is expected to be set already + const SimpleColumn* sc = dynamic_cast(srcp.get()); - return key; + if (sc != NULL) + { + if (sc->schemaName().empty()) + { + SimpleColumn tmp(*sc, jobInfo.sessionId); + tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + key = getTupleKey(jobInfo, &tmp); + } + else + { + key = getTupleKey(jobInfo, sc); + } + + // check if this is a dictionary column + if (jobInfo.keyInfo->dictKeyMap.find(key) != jobInfo.keyInfo->dictKeyMap.end()) + key = jobInfo.keyInfo->dictKeyMap[key]; + } + else + { + key = getExpTupleKey(jobInfo, srcp->expressionId()); + } + } + + return key; } - uint32_t getTableKey(const JobInfo& jobInfo, execplan::CalpontSystemCatalog::OID tableOid, const string& alias, const string& schema, const string& view) { - return getTupleKey_(jobInfo, tableOid, "", alias, schema, view); + return getTupleKey_(jobInfo, tableOid, "", alias, schema, view); } - uint32_t getTableKey(const JobInfo& jobInfo, uint32_t cid) { - return jobInfo.keyInfo->colKeyToTblKey[cid]; + return jobInfo.keyInfo->colKeyToTblKey[cid]; } - void updateTableKey(uint32_t cid, uint32_t tid, JobInfo& jobInfo) { - jobInfo.keyInfo->colKeyToTblKey[cid] = tid; + jobInfo.keyInfo->colKeyToTblKey[cid] = tid; } - uint32_t getTableKey(JobInfo& jobInfo, JobStep* js) { - CalpontSystemCatalog::OID tableOid = js->tableOid(); - return getTupleKey_(jobInfo, tableOid, "", js->alias(), js->schema(), js->view()); + CalpontSystemCatalog::OID tableOid = js->tableOid(); + return getTupleKey_(jobInfo, tableOid, "", js->alias(), js->schema(), js->view()); } - uint32_t makeTableKey(JobInfo& jobInfo, const execplan::SimpleColumn* sc) { - CalpontSystemCatalog::OID o = tableOid(sc, jobInfo.csc); - return uniqTupleKey(jobInfo, o, o, "", "", sc->tableName(), extractTableAlias(sc), - sc->schemaName(), sc->viewName(), 0, (sc->isColumnStore() ? 0 : 1), - ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0)); + CalpontSystemCatalog::OID o = tableOid(sc, jobInfo.csc); + return uniqTupleKey(jobInfo, o, o, "", "", sc->tableName(), extractTableAlias(sc), sc->schemaName(), + sc->viewName(), 0, (sc->isColumnStore() ? 0 : 1), + ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0)); } - -uint32_t makeTableKey(JobInfo& jobInfo, - CalpontSystemCatalog::OID o, - const string& tn, - const string& ta, - const string& sn, - const string& vn, - uint64_t en) +uint32_t makeTableKey(JobInfo& jobInfo, CalpontSystemCatalog::OID o, const string& tn, const string& ta, + const string& sn, const string& vn, uint64_t en) { - return uniqTupleKey(jobInfo, o, o, "", "", tn, ta, sn, vn, 0, en); + return uniqTupleKey(jobInfo, o, o, "", "", tn, ta, sn, vn, 0, en); } - TupleInfo getTupleInfo(uint32_t columnKey, const JobInfo& jobInfo) { - TupleInfoMap::const_iterator cit = jobInfo.keyInfo->tupleInfoMap.find(columnKey); + TupleInfoMap::const_iterator cit = jobInfo.keyInfo->tupleInfoMap.find(columnKey); - if ((cit == jobInfo.keyInfo->tupleInfoMap.end()) || - (cit->second.dtype == CalpontSystemCatalog::BIT)) - { - ostringstream strstm; - strstm << "TupleInfo for (" << jobInfo.keyInfo->tupleKeyVec[columnKey].fId << "," - << jobInfo.keyInfo->tupleKeyVec[columnKey].fTable; + if ((cit == jobInfo.keyInfo->tupleInfoMap.end()) || (cit->second.dtype == CalpontSystemCatalog::BIT)) + { + ostringstream strstm; + strstm << "TupleInfo for (" << jobInfo.keyInfo->tupleKeyVec[columnKey].fId << "," + << jobInfo.keyInfo->tupleKeyVec[columnKey].fTable; - if (jobInfo.keyInfo->tupleKeyVec[columnKey].fView.length() > 0) - strstm << "," << jobInfo.keyInfo->tupleKeyVec[columnKey].fView; + if (jobInfo.keyInfo->tupleKeyVec[columnKey].fView.length() > 0) + strstm << "," << jobInfo.keyInfo->tupleKeyVec[columnKey].fView; - strstm << ") could not be found." << endl; - cerr << strstm.str(); + strstm << ") could not be found." << endl; + cerr << strstm.str(); - Message::Args args; - args.add(strstm.str()); - jobInfo.logger->logMessage(LOG_TYPE_DEBUG, LogMakeJobList, args, - LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); + Message::Args args; + args.add(strstm.str()); + jobInfo.logger->logMessage(LOG_TYPE_DEBUG, LogMakeJobList, args, + LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); - throw runtime_error("column's tuple info could not be found"); - } + throw runtime_error("column's tuple info could not be found"); + } - return cit->second; + return cit->second; } - TupleInfo setTupleInfo(const execplan::CalpontSystemCatalog::ColType& ct, - execplan::CalpontSystemCatalog::OID col_oid, - JobInfo& jobInfo, - execplan::CalpontSystemCatalog::OID tbl_oid, - const execplan::SimpleColumn* sc, + execplan::CalpontSystemCatalog::OID col_oid, JobInfo& jobInfo, + execplan::CalpontSystemCatalog::OID tbl_oid, const execplan::SimpleColumn* sc, const string& alias) { - const PseudoColumn* pc = dynamic_cast(sc); - uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; - return setTupleInfo_(ct, col_oid, jobInfo, tbl_oid, sc->columnName(), sc->alias(), - sc->schemaName(), sc->tableName(), alias, sc->viewName(), - ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), - pseudoType, (sc->isColumnStore() ? 0 : 1)); + const PseudoColumn* pc = dynamic_cast(sc); + uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; + return setTupleInfo_(ct, col_oid, jobInfo, tbl_oid, sc->columnName(), sc->alias(), sc->schemaName(), + sc->tableName(), alias, sc->viewName(), + ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), pseudoType, + (sc->isColumnStore() ? 0 : 1)); } - TupleInfo setExpTupleInfo(const execplan::CalpontSystemCatalog::ColType& ct, uint64_t expressionId, const string& alias, JobInfo& jobInfo, bool cr) { - // pretend all expressions belong to "virtual" table EXPRESSION, (CNX_EXP_TABLE_ID, expression) - // CNX_EXP_TABLE_ID(999) is not for user table or column, there will be no confilict in queries. - JobInfo* ji = &jobInfo; + // pretend all expressions belong to "virtual" table EXPRESSION, (CNX_EXP_TABLE_ID, expression) + // CNX_EXP_TABLE_ID(999) is not for user table or column, there will be no confilict in queries. + JobInfo* ji = &jobInfo; - if (cr && jobInfo.pJobInfo) - ji = jobInfo.pJobInfo; + if (cr && jobInfo.pJobInfo) + ji = jobInfo.pJobInfo; - string expAlias("$exp"); + string expAlias("$exp"); - if (!(ji->subAlias.empty())) - expAlias += ji->subAlias; + if (!(ji->subAlias.empty())) + expAlias += ji->subAlias; - return setTupleInfo_( - ct, expressionId, jobInfo, CNX_EXP_TABLE_ID, "", alias, "", "$exp", expAlias, "", cr); + return setTupleInfo_(ct, expressionId, jobInfo, CNX_EXP_TABLE_ID, "", alias, "", "$exp", expAlias, "", cr); } - TupleInfo setExpTupleInfo(const execplan::ReturnedColumn* rc, JobInfo& jobInfo) { - return setExpTupleInfo(rc->resultType(), rc->expressionId(), rc->alias(), jobInfo, - ((rc->joinInfo() & execplan::JOIN_CORRELATED) != 0)); + return setExpTupleInfo(rc->resultType(), rc->expressionId(), rc->alias(), jobInfo, + ((rc->joinInfo() & execplan::JOIN_CORRELATED) != 0)); } - uint32_t getExpTupleKey(const JobInfo& jobInfo, uint64_t eid, bool cr) { - const JobInfo* ji = &jobInfo; + const JobInfo* ji = &jobInfo; - if (cr && jobInfo.pJobInfo) - ji = jobInfo.pJobInfo; + if (cr && jobInfo.pJobInfo) + ji = jobInfo.pJobInfo; - string expAlias("$exp"); + string expAlias("$exp"); - if (!(ji->subAlias.empty())) - expAlias += ji->subAlias; + if (!(ji->subAlias.empty())) + expAlias += ji->subAlias; - return getTupleKey_(jobInfo, eid, "", expAlias, "", "", cr); + return getTupleKey_(jobInfo, eid, "", expAlias, "", "", cr); } - void addAggregateColumn(ReturnedColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo) { - uint32_t eid = agc->expressionId(); - setExpTupleInfo(agc->resultType(), eid, agc->alias(), jobInfo); + uint32_t eid = agc->expressionId(); + setExpTupleInfo(agc->resultType(), eid, agc->alias(), jobInfo); - vector >::iterator i; + vector >::iterator i; - for (i = jobInfo.aggEidIndexList.begin(); i != jobInfo.aggEidIndexList.end(); ++i) + for (i = jobInfo.aggEidIndexList.begin(); i != jobInfo.aggEidIndexList.end(); ++i) + { + if (i->first == (int)eid) + break; + } + + if (idx < 0 && i != jobInfo.aggEidIndexList.end()) + { + agc->inputIndex(i->second); + jobInfo.cloneAggregateColMap.insert(make_pair(vec[i->second].get(), agc)); + } + else + { + SRCP srcp; + + if (idx < 0) { - if (i->first == (int) eid) - break; - } - - if (idx < 0 && i != jobInfo.aggEidIndexList.end()) - { - agc->inputIndex(i->second); - jobInfo.cloneAggregateColMap.insert(make_pair(vec[i->second].get(), agc)); + srcp.reset(agc->clone()); + idx = vec.size(); + vec.push_back(srcp); } else { - SRCP srcp; - - if (idx < 0) - { - srcp.reset(agc->clone()); - idx = vec.size(); - vec.push_back(srcp); - } - else - { - srcp = vec[idx]; - } - - jobInfo.aggEidIndexList.push_back(make_pair(eid, idx)); - agc->inputIndex(idx); - jobInfo.cloneAggregateColMap.insert(make_pair(srcp.get(), agc)); + srcp = vec[idx]; } + + jobInfo.aggEidIndexList.push_back(make_pair(eid, idx)); + agc->inputIndex(idx); + jobInfo.cloneAggregateColMap.insert(make_pair(srcp.get(), agc)); + } } - -bool operator < (const struct UniqId& x, const struct UniqId& y) +bool operator<(const struct UniqId& x, const struct UniqId& y) { - return ( - (x.fId < y.fId) || - (x.fId == y.fId && x.fTable < y.fTable) || - (x.fId == y.fId && x.fTable == y.fTable && x.fSchema < y.fSchema) || - (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView < y.fView) || - (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView == y.fView && - x.fPseudo < y.fPseudo) || - (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView == y.fView && - x.fPseudo == y.fPseudo && x.fSubId < y.fSubId)); + return ((x.fId < y.fId) || (x.fId == y.fId && x.fTable < y.fTable) || + (x.fId == y.fId && x.fTable == y.fTable && x.fSchema < y.fSchema) || + (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView < y.fView) || + (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView == y.fView && + x.fPseudo < y.fPseudo) || + (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView == y.fView && + x.fPseudo == y.fPseudo && x.fSubId < y.fSubId)); } - -bool operator == (const struct UniqId& x, const struct UniqId& y) +bool operator==(const struct UniqId& x, const struct UniqId& y) { - return ( - x.fId == y.fId && - x.fTable == y.fTable && - x.fSchema == y.fSchema && - x.fView == y.fView && - x.fPseudo == y.fPseudo && - x.fSubId == y.fSubId); + return (x.fId == y.fId && x.fTable == y.fTable && x.fSchema == y.fSchema && x.fView == y.fView && + x.fPseudo == y.fPseudo && x.fSubId == y.fSubId); } - void updateDerivedColumn(JobInfo& jobInfo, SimpleColumn* sc, CalpontSystemCatalog::ColType& ct) { - sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - map::iterator i = - jobInfo.vtableColTypes.find(UniqId(sc)); + map::iterator i = jobInfo.vtableColTypes.find(UniqId(sc)); - if (i != jobInfo.vtableColTypes.end()) - ct = i->second; + if (i != jobInfo.vtableColTypes.end()) + ct = i->second; } - bool filterWithDictionary(execplan::CalpontSystemCatalog::OID dictOid, uint64_t n) { - // if n == 0, no dictionary scan, alway filter with dictionary. - if (n == 0) - return true; + // if n == 0, no dictionary scan, alway filter with dictionary. + if (n == 0) + return true; - // if n == ulong_max, always use dictionary scan - if (n == ULONG_MAX) - return false; + // if n == ulong_max, always use dictionary scan + if (n == ULONG_MAX) + return false; - vector entries; - DBRM dbrm; + vector entries; + DBRM dbrm; - if (dbrm.getExtents(dictOid, entries) != 0) - return false; // Just do pdictionaryscan and let job step handle this. + if (dbrm.getExtents(dictOid, entries) != 0) + return false; // Just do pdictionaryscan and let job step handle this. - vector::iterator it = entries.begin(); - bool ret = false; - n--; // HWM starts at 0 + vector::iterator it = entries.begin(); + bool ret = false; + n--; // HWM starts at 0 - while (it != entries.end()) + while (it != entries.end()) + { + if (it->HWM > n) { - if (it->HWM > n) - { - ret = true; - break; - } - - it++; + ret = true; + break; } - return ret; -} + it++; + } + return ret; +} // @Bug 1230 & 1955 // Don't allow join/compare on "incompatible" cols @@ -772,140 +697,129 @@ bool filterWithDictionary(execplan::CalpontSystemCatalog::OID dictOid, uint64_t // date to date // datetime to datetime // string to string -bool compatibleColumnTypes(const CalpontSystemCatalog::ColType& ct1, - const CalpontSystemCatalog::ColType& ct2, - bool forJoin) +bool compatibleColumnTypes(const CalpontSystemCatalog::ColType& ct1, const CalpontSystemCatalog::ColType& ct2, + bool forJoin) { - return compatibleColumnTypes(ct1.colDataType, ct1.scale, ct2.colDataType, ct2.scale, forJoin); + return compatibleColumnTypes(ct1.colDataType, ct1.scale, ct2.colDataType, ct2.scale, forJoin); } - bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_t scale1, - const CalpontSystemCatalog::ColDataType& dt2, uint32_t scale2, - bool forJoin) + const CalpontSystemCatalog::ColDataType& dt2, uint32_t scale2, bool forJoin) { - // disable VARBINARY used in join - if (dt1 == CalpontSystemCatalog::VARBINARY || - dt2 == CalpontSystemCatalog::VARBINARY) - throw runtime_error("Comparsion between VARBINARY columns is not supported."); + // disable VARBINARY used in join + if (dt1 == CalpontSystemCatalog::VARBINARY || dt2 == CalpontSystemCatalog::VARBINARY) + throw runtime_error("Comparsion between VARBINARY columns is not supported."); - switch (dt1) - { - case CalpontSystemCatalog::BIT: - if (dt2 != CalpontSystemCatalog::BIT) return false; + switch (dt1) + { + case CalpontSystemCatalog::BIT: + if (dt2 != CalpontSystemCatalog::BIT) + return false; - break; + break; - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - if (dt2 != CalpontSystemCatalog::TINYINT && - dt2 != CalpontSystemCatalog::SMALLINT && - dt2 != CalpontSystemCatalog::MEDINT && - dt2 != CalpontSystemCatalog::INT&& - dt2 != CalpontSystemCatalog::BIGINT && - dt2 != CalpontSystemCatalog::DECIMAL && - dt2 != CalpontSystemCatalog::UTINYINT && - dt2 != CalpontSystemCatalog::USMALLINT && - dt2 != CalpontSystemCatalog::UMEDINT && - dt2 != CalpontSystemCatalog::UINT && - dt2 != CalpontSystemCatalog::UBIGINT && - dt2 != CalpontSystemCatalog::UDECIMAL) return false; + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UDECIMAL: + if (dt2 != CalpontSystemCatalog::TINYINT && dt2 != CalpontSystemCatalog::SMALLINT && + dt2 != CalpontSystemCatalog::MEDINT && dt2 != CalpontSystemCatalog::INT && + dt2 != CalpontSystemCatalog::BIGINT && dt2 != CalpontSystemCatalog::DECIMAL && + dt2 != CalpontSystemCatalog::UTINYINT && dt2 != CalpontSystemCatalog::USMALLINT && + dt2 != CalpontSystemCatalog::UMEDINT && dt2 != CalpontSystemCatalog::UINT && + dt2 != CalpontSystemCatalog::UBIGINT && dt2 != CalpontSystemCatalog::UDECIMAL) + return false; - if (scale2 != scale1) return false; + if (scale2 != scale1) + return false; - break; + break; - case CalpontSystemCatalog::DATE: - if (dt2 != CalpontSystemCatalog::DATE) return false; + case CalpontSystemCatalog::DATE: + if (dt2 != CalpontSystemCatalog::DATE) + return false; - break; + break; - case CalpontSystemCatalog::DATETIME: - if (dt2 != CalpontSystemCatalog::DATETIME) return false; + case CalpontSystemCatalog::DATETIME: + if (dt2 != CalpontSystemCatalog::DATETIME) + return false; - break; + break; - case CalpontSystemCatalog::TIMESTAMP: - if (dt2 != CalpontSystemCatalog::TIMESTAMP) return false; + case CalpontSystemCatalog::TIMESTAMP: + if (dt2 != CalpontSystemCatalog::TIMESTAMP) + return false; - break; + break; - case CalpontSystemCatalog::TIME: - if (dt2 != CalpontSystemCatalog::TIME) return false; + case CalpontSystemCatalog::TIME: + if (dt2 != CalpontSystemCatalog::TIME) + return false; - break; + break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: - // @bug 1495 compound/string join - if (dt2 != CalpontSystemCatalog::VARCHAR && - dt2 != CalpontSystemCatalog::CHAR && - dt2 != CalpontSystemCatalog::TEXT) - return false; + // @bug 1495 compound/string join + if (dt2 != CalpontSystemCatalog::VARCHAR && dt2 != CalpontSystemCatalog::CHAR && + dt2 != CalpontSystemCatalog::TEXT) + return false; - break; + break; - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - if (dt2 != CalpontSystemCatalog::VARBINARY && - dt2 != CalpontSystemCatalog::BLOB) return false; + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + if (dt2 != CalpontSystemCatalog::VARBINARY && dt2 != CalpontSystemCatalog::BLOB) + return false; - break; + break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - if (forJoin && (dt2 != CalpontSystemCatalog::FLOAT && - dt2 != CalpontSystemCatalog::FLOAT)) return false; - else if (dt2 != CalpontSystemCatalog::FLOAT && - dt2 != CalpontSystemCatalog::DOUBLE && - dt2 != CalpontSystemCatalog::UFLOAT && - dt2 != CalpontSystemCatalog::UDOUBLE) return false; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + if (forJoin && (dt2 != CalpontSystemCatalog::FLOAT && dt2 != CalpontSystemCatalog::FLOAT)) + return false; + else if (dt2 != CalpontSystemCatalog::FLOAT && dt2 != CalpontSystemCatalog::DOUBLE && + dt2 != CalpontSystemCatalog::UFLOAT && dt2 != CalpontSystemCatalog::UDOUBLE) + return false; - break; + break; - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - if (forJoin && (dt2 != CalpontSystemCatalog::DOUBLE && - dt2 != CalpontSystemCatalog::UDOUBLE)) return false; - else if (dt2 != CalpontSystemCatalog::FLOAT && - dt2 != CalpontSystemCatalog::DOUBLE && - dt2 != CalpontSystemCatalog::UFLOAT && - dt2 != CalpontSystemCatalog::UDOUBLE) return false; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + if (forJoin && (dt2 != CalpontSystemCatalog::DOUBLE && dt2 != CalpontSystemCatalog::UDOUBLE)) + return false; + else if (dt2 != CalpontSystemCatalog::FLOAT && dt2 != CalpontSystemCatalog::DOUBLE && + dt2 != CalpontSystemCatalog::UFLOAT && dt2 != CalpontSystemCatalog::UDOUBLE) + return false; - break; + break; - case CalpontSystemCatalog::LONGDOUBLE: - if (forJoin && (dt2 != CalpontSystemCatalog::LONGDOUBLE)) - return false; - else if (dt2 != CalpontSystemCatalog::FLOAT && - dt2 != CalpontSystemCatalog::DOUBLE && - dt2 != CalpontSystemCatalog::UFLOAT && - dt2 != CalpontSystemCatalog::UDOUBLE && - dt2 != CalpontSystemCatalog::LONGDOUBLE) return false; + case CalpontSystemCatalog::LONGDOUBLE: + if (forJoin && (dt2 != CalpontSystemCatalog::LONGDOUBLE)) + return false; + else if (dt2 != CalpontSystemCatalog::FLOAT && dt2 != CalpontSystemCatalog::DOUBLE && + dt2 != CalpontSystemCatalog::UFLOAT && dt2 != CalpontSystemCatalog::UDOUBLE && + dt2 != CalpontSystemCatalog::LONGDOUBLE) + return false; - break; + break; - default: - return false; - break; - } + default: return false; break; + } - return true; + return true; } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 582f81621..2988172f6 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -18,7 +18,6 @@ // $Id: jlf_common.h 9702 2013-07-17 19:08:07Z xlou $ - /** @file jlf_common.h * */ @@ -54,8 +53,7 @@ namespace execplan { class AggregateColumn; class SimpleColumn; -} - +} // namespace execplan namespace joblist { @@ -63,9 +61,9 @@ namespace joblist const std::string boldStart = "\033[0;1m"; const std::string boldStop = "\033[0;39m"; -const int8_t CONST_COL_NONE = 0; +const int8_t CONST_COL_NONE = 0; const int8_t CONST_COL_EXIST = 1; -const int8_t CONST_COL_ONLY = 2; +const int8_t CONST_COL_ONLY = 2; // pretend all expressions belong to "virtual" table EXPRESSION, (CNX_EXP_TABLE_ID, expression) // CNX_EXP_TABLE_ID(999) is not for user table or column, there will be no confilict in queries. @@ -73,31 +71,38 @@ const int32_t CNX_EXP_TABLE_ID = 999; struct TupleInfo { - TupleInfo(uint32_t w = 0, uint32_t o = 0, uint32_t k = -1, uint32_t t = -1, uint32_t s = 0, uint32_t p = 0, - execplan::CalpontSystemCatalog::ColDataType dt = execplan::CalpontSystemCatalog::BIT, uint32_t csn = 8) : - width(w), oid(o), key(k), tkey(t), scale(s), precision(p), dtype(dt), csNum(csn) { } - ~TupleInfo() { } + TupleInfo(uint32_t w = 0, uint32_t o = 0, uint32_t k = -1, uint32_t t = -1, uint32_t s = 0, uint32_t p = 0, + execplan::CalpontSystemCatalog::ColDataType dt = execplan::CalpontSystemCatalog::BIT, + uint32_t csn = 8) + : width(w), oid(o), key(k), tkey(t), scale(s), precision(p), dtype(dt), csNum(csn) + { + } + ~TupleInfo() + { + } - uint32_t width; - uint32_t oid; - uint32_t key; - uint32_t tkey; - uint32_t scale; - uint32_t precision; - execplan::CalpontSystemCatalog::ColDataType dtype; - uint32_t csNum; // For collations + uint32_t width; + uint32_t oid; + uint32_t key; + uint32_t tkey; + uint32_t scale; + uint32_t precision; + execplan::CalpontSystemCatalog::ColDataType dtype; + uint32_t csNum; // For collations }; // for compound join struct JoinData { - int64_t fJoinId; - std::vector fLeftKeys; - std::vector fRightKeys; - std::vector fTypes; // joblisttypes.h: INNER, LEFTOUTER, RIGHTOUTER - bool fTypeless; + int64_t fJoinId; + std::vector fLeftKeys; + std::vector fRightKeys; + std::vector fTypes; // joblisttypes.h: INNER, LEFTOUTER, RIGHTOUTER + bool fTypeless; - JoinData() : fJoinId(-1), fTypeless(false) {} + JoinData() : fJoinId(-1), fTypeless(false) + { + } }; typedef std::stack JobStepVectorStack; @@ -105,61 +110,66 @@ typedef std::map TupleInfoVector; typedef std::map TupleInfoMap; -//for subquery support +// for subquery support struct UniqId { - int fId; // OID for real table, sequence # for subquery -// std::string fName; // name (table alias + [column name, if column]) - std::string fTable; // table name (table alias) - std::string fSchema; // schema name - std::string fView; // view name - uint32_t fPseudo; // pseudo type -// uint64_t fEngine; // InfiniDB == 0 - uint64_t fSubId; // subquery ID + int fId; // OID for real table, sequence # for subquery + // std::string fName; // name (table alias + [column name, if column]) + std::string fTable; // table name (table alias) + std::string fSchema; // schema name + std::string fView; // view name + uint32_t fPseudo; // pseudo type + // uint64_t fEngine; // InfiniDB == 0 + uint64_t fSubId; // subquery ID - UniqId() : fId(-1), fSubId(-1) {} - UniqId(int i, const std::string& t, const std::string& s, const std::string& v, - uint32_t pi = 0, uint64_t l = -1) : - fId(i), fTable(t), fSchema(s), fView(v), fPseudo(pi), fSubId(l) {} - UniqId(const execplan::SimpleColumn* sc); - UniqId(int o, const execplan::SimpleColumn* sc); + UniqId() : fId(-1), fSubId(-1) + { + } + UniqId(int i, const std::string& t, const std::string& s, const std::string& v, uint32_t pi = 0, + uint64_t l = -1) + : fId(i), fTable(t), fSchema(s), fView(v), fPseudo(pi), fSubId(l) + { + } + UniqId(const execplan::SimpleColumn* sc); + UniqId(int o, const execplan::SimpleColumn* sc); - std::string toString() const; + std::string toString() const; }; -bool operator < (const struct UniqId& x, const struct UniqId& y); -bool operator == (const struct UniqId& x, const struct UniqId& y); +bool operator<(const struct UniqId& x, const struct UniqId& y); +bool operator==(const struct UniqId& x, const struct UniqId& y); typedef std::map TupleKeyMap; -//typedef vector RetColsVector; +// typedef vector RetColsVector; typedef execplan::CalpontSelectExecutionPlan::ReturnedColumnList RetColsVector; -//join data between table pairs +// join data between table pairs typedef std::map, JoinData> TableJoinMap; struct TupleKeyInfo { - uint32_t nextKey; - TupleKeyMap tupleKeyMap; - std::vector tupleKeyVec; - std::vector tupleKeyToName; - std::vector crossEngine; + uint32_t nextKey; + TupleKeyMap tupleKeyMap; + std::vector tupleKeyVec; + std::vector tupleKeyToName; + std::vector crossEngine; - // TODO: better organize these structs - std::map tupleKeyToTableOid; - std::map colType; - std::map token2DictTypeMap; - std::map keyName; - std::map colKeyToTblKey; - std::map dictKeyMap; // map token key to dictionary key - DictOidToColOidMap dictOidToColOid; // map dictionary OID to column OID - std::map pseudoType; // key to pseudo column type - std::set functionJoinKeys; // key used in function join - TupleInfoMap tupleInfoMap; + // TODO: better organize these structs + std::map tupleKeyToTableOid; + std::map colType; + std::map token2DictTypeMap; + std::map keyName; + std::map colKeyToTblKey; + std::map dictKeyMap; // map token key to dictionary key + DictOidToColOidMap dictOidToColOid; // map dictionary OID to column OID + std::map pseudoType; // key to pseudo column type + std::set functionJoinKeys; // key used in function join + TupleInfoMap tupleInfoMap; - TupleKeyInfo() : nextKey(0) {} + TupleKeyInfo() : nextKey(0) + { + } }; - //------------------------------------------------------------------------------ /** @brief This struct maintains state for the query processing * @@ -167,215 +177,215 @@ struct TupleKeyInfo //------------------------------------------------------------------------------ struct JobInfo { - JobInfo(ResourceManager* r) : - rm(r), - sessionId(0), - txnId(0), - statementId(0), - maxBuckets(rm->getHjMaxBuckets()), - maxElems(rm->getHjMaxElems()), - flushInterval(rm->getJLFlushInterval()), - fifoSize(rm->getJlFifoSize()), - fifoSizeLargeSideHj(rm->getHjFifoSizeLargeSide()), - scanLbidReqLimit(rm->getJlScanLbidReqLimit()), - scanLbidReqThreshold(rm->getJlScanLbidReqThreshold()), - tempSaveSize(rm->getScTempSaveSize()), - logger(new Logger()), - traceFlags(0), - tupleDLMaxSize(rm->getTwMaxSize()), - tupleMaxBuckets(rm->getTwMaxBuckets()), - projectingTableOID(0), - isExeMgr(false), - trace(false), - tryTuples(false), - constantCol(CONST_COL_NONE), - hasDistinct(false), - hasAggregation(false), - limitStart(0), - limitCount(-1), - joinNum(0), - subLevel(0), - subNum(0), - subId(0), - pJobInfo(NULL), - constantFalse(false), - cntStarPos(-1), - stringScanThreshold(1), - wfqLimitStart(0), - wfqLimitCount(-1) - { } - ResourceManager* rm; - uint32_t sessionId; - uint32_t txnId; - BRM::QueryContext verId; - uint32_t statementId; - std::string queryType; - boost::shared_ptr csc; - int maxBuckets; - uint64_t maxElems; - JobStepVectorStack stack; - uint32_t flushInterval; - uint32_t fifoSize; - uint32_t fifoSizeLargeSideHj; - //...joblist does not use scanLbidReqLimit and SdanLbidReqThreshold. - //...They are actually used by pcolscan and pdictionaryscan, but - //...we have joblist get and report the values here since they - //...are global to the job. - uint32_t scanLbidReqLimit; - uint32_t scanLbidReqThreshold; - uint32_t tempSaveSize; - SPJL logger; - uint32_t traceFlags; - uint64_t tupleDLMaxSize; - uint32_t tupleMaxBuckets; - SErrorInfo errorInfo; - execplan::CalpontSystemCatalog::OID* projectingTableOID; // DeliveryWSDLs get a reference to this - bool isExeMgr; - bool trace; - bool tryTuples; - int8_t constantCol; - TupleInfoVector pjColList; + JobInfo(ResourceManager* r) + : rm(r) + , sessionId(0) + , txnId(0) + , statementId(0) + , maxBuckets(rm->getHjMaxBuckets()) + , maxElems(rm->getHjMaxElems()) + , flushInterval(rm->getJLFlushInterval()) + , fifoSize(rm->getJlFifoSize()) + , fifoSizeLargeSideHj(rm->getHjFifoSizeLargeSide()) + , scanLbidReqLimit(rm->getJlScanLbidReqLimit()) + , scanLbidReqThreshold(rm->getJlScanLbidReqThreshold()) + , tempSaveSize(rm->getScTempSaveSize()) + , logger(new Logger()) + , traceFlags(0) + , tupleDLMaxSize(rm->getTwMaxSize()) + , tupleMaxBuckets(rm->getTwMaxBuckets()) + , projectingTableOID(0) + , isExeMgr(false) + , trace(false) + , tryTuples(false) + , constantCol(CONST_COL_NONE) + , hasDistinct(false) + , hasAggregation(false) + , limitStart(0) + , limitCount(-1) + , joinNum(0) + , subLevel(0) + , subNum(0) + , subId(0) + , pJobInfo(NULL) + , constantFalse(false) + , cntStarPos(-1) + , stringScanThreshold(1) + , wfqLimitStart(0) + , wfqLimitCount(-1) + { + } + ResourceManager* rm; + uint32_t sessionId; + uint32_t txnId; + BRM::QueryContext verId; + uint32_t statementId; + std::string queryType; + boost::shared_ptr csc; + int maxBuckets; + uint64_t maxElems; + JobStepVectorStack stack; + uint32_t flushInterval; + uint32_t fifoSize; + uint32_t fifoSizeLargeSideHj; + //...joblist does not use scanLbidReqLimit and SdanLbidReqThreshold. + //...They are actually used by pcolscan and pdictionaryscan, but + //...we have joblist get and report the values here since they + //...are global to the job. + uint32_t scanLbidReqLimit; + uint32_t scanLbidReqThreshold; + uint32_t tempSaveSize; + SPJL logger; + uint32_t traceFlags; + uint64_t tupleDLMaxSize; + uint32_t tupleMaxBuckets; + SErrorInfo errorInfo; + execplan::CalpontSystemCatalog::OID* projectingTableOID; // DeliveryWSDLs get a reference to this + bool isExeMgr; + bool trace; + bool tryTuples; + int8_t constantCol; + TupleInfoVector pjColList; - // aggregation - bool hasDistinct; - bool hasAggregation; - std::vector groupByColVec; - std::vector distinctColVec; - std::vector expressionVec; - std::vector > returnedColVec; + // aggregation + bool hasDistinct; + bool hasAggregation; + std::vector groupByColVec; + std::vector distinctColVec; + std::vector expressionVec; + std::vector > returnedColVec; - // order by and limit - std::vector > orderByColVec; - uint64_t limitStart; - uint64_t limitCount; - uint32_t orderByThreads; + // order by and limit + std::vector > orderByColVec; + uint64_t limitStart; + uint64_t limitCount; + uint32_t orderByThreads; - // tupleInfo - boost::shared_ptr keyInfo; + // tupleInfo + boost::shared_ptr keyInfo; - // skip dictionary step if the real string is not necessary to projected. - // In most case, the string is used for return or comparison, so default is false. - // when setting to false, no need to check: false overwrites true; - // When setting to true, need check: true cannot overwrite false. - std::map tokenOnly; + // skip dictionary step if the real string is not necessary to projected. + // In most case, the string is used for return or comparison, so default is false. + // when setting to false, no need to check: false overwrites true; + // When setting to true, need check: true cannot overwrite false. + std::map tokenOnly; - // unique ID list of the tables in from clause - std::vector tableList; + // unique ID list of the tables in from clause + std::vector tableList; - // table join map - TableJoinMap tableJoinMap; + // table join map + TableJoinMap tableJoinMap; - // for expression - JobStepVector crossTableExpressions; - JobStepVector returnedExpressions; + // for expression + JobStepVector crossTableExpressions; + JobStepVector returnedExpressions; - // @bug3683, function join - std::vector functionJoins; // store expressions can be converted to joins + // @bug3683, function join + std::vector functionJoins; // store expressions can be converted to joins - // for function on aggregation - RetColsVector deliveredCols; // columns to be sent to connector - RetColsVector nonConstCols; // none constant columns - RetColsVector nonConstDelCols; // delivered none constant columns - RetColsVector projectionCols; // columns for projection - std::multimap cloneAggregateColMap; - std::vector > aggEidIndexList; + // for function on aggregation + RetColsVector deliveredCols; // columns to be sent to connector + RetColsVector nonConstCols; // none constant columns + RetColsVector nonConstDelCols; // delivered none constant columns + RetColsVector projectionCols; // columns for projection + std::multimap cloneAggregateColMap; + std::vector > aggEidIndexList; - // table pairs with incompatible join which is treated as expression - std::map incompatibleJoinMap; + // table pairs with incompatible join which is treated as expression + std::map incompatibleJoinMap; - // bug 1573 & 3391, having - SJSTEP havingStep; - JobStepVector havingStepVec; + // bug 1573 & 3391, having + SJSTEP havingStep; + JobStepVector havingStepVec; - // bug 2634, 5311 and 5374, outjoin and predicates - std::set outerOnTable; - std::set tableHasIsNull; - JobStepVector outerJoinExpressions; + // bug 2634, 5311 and 5374, outjoin and predicates + std::set outerOnTable; + std::set tableHasIsNull; + JobStepVector outerJoinExpressions; - // bug 3759, join in order - // mixed outer join - std::map tableSize; - int64_t joinNum; + // bug 3759, join in order + // mixed outer join + std::map tableSize; + int64_t joinNum; - // for subquery - boost::shared_ptr subCount; // # of subqueries in the query statement - int subLevel; // subquery level - int subNum; // # of subqueries @ level n - int subId; // id of current subquery - JobInfo* pJobInfo; // jobinfo of outer query - bool constantFalse; // has constant false filter - std::string subAlias; // unique alias to identify the subquery - JobStepVector correlateSteps; - JobStepVector selectAndFromSubs; - std::set returnColSet; - std::map vtableColTypes; + // for subquery + boost::shared_ptr subCount; // # of subqueries in the query statement + int subLevel; // subquery level + int subNum; // # of subqueries @ level n + int subId; // id of current subquery + JobInfo* pJobInfo; // jobinfo of outer query + bool constantFalse; // has constant false filter + std::string subAlias; // unique alias to identify the subquery + JobStepVector correlateSteps; + JobStepVector selectAndFromSubs; + std::set returnColSet; + std::map vtableColTypes; - // step to process orderby, limit and fill in constants - SJSTEP annexStep; + // step to process orderby, limit and fill in constants + SJSTEP annexStep; - // @bug3475, aggregate constant column - std::map constAggregate; - int64_t cntStarPos; // position of count(*) + // @bug3475, aggregate constant column + std::map constAggregate; + int64_t cntStarPos; // position of count(*) - // @bug3321, dictionary scan setting, HWM = stringScanThreshold -1 - uint64_t stringScanThreshold; + // @bug3321, dictionary scan setting, HWM = stringScanThreshold -1 + uint64_t stringScanThreshold; - // @bug3362, group_concat - RetColsVector groupConcatCols; - GroupConcatInfo groupConcatInfo; + // @bug3362, group_concat + RetColsVector groupConcatCols; + GroupConcatInfo groupConcatInfo; - // @bug3736, column map - std::map > columnMap; + // @bug3736, column map + std::map > columnMap; - // @bug3438, joblist for trace/stats - JobList* jobListPtr; // just reference, NOT delete by JobInfo + // @bug3438, joblist for trace/stats + JobList* jobListPtr; // just reference, NOT delete by JobInfo - // WORKAROUND for join FE limitation (join Id to expression tables map) - std::map > joinFeTableMap; + // WORKAROUND for join FE limitation (join Id to expression tables map) + std::map > joinFeTableMap; - uint32_t stringTableThreshold; + uint32_t stringTableThreshold; - // @bug4531, Window Function support - RetColsVector windowCols; - RetColsVector windowExps; - RetColsVector windowDels; - std::set windowSet; - RetColsVector wfqOrderby; - uint64_t wfqLimitStart; - uint64_t wfqLimitCount; - // workaround for expression of windowfunction in IN/EXISTS sub-query - //std::map exprWinfuncListMap; + // @bug4531, Window Function support + RetColsVector windowCols; + RetColsVector windowExps; + RetColsVector windowDels; + std::set windowSet; + RetColsVector wfqOrderby; + uint64_t wfqLimitStart; + uint64_t wfqLimitCount; + // workaround for expression of windowfunction in IN/EXISTS sub-query + // std::map exprWinfuncListMap; - // Flag to tell us we are in local PM only query mode - uint32_t localQuery; + // Flag to tell us we are in local PM only query mode + uint32_t localQuery; - boost::uuids::uuid uuid; + boost::uuids::uuid uuid; - // @bug4021, column map for all pseudo column queries - std::map tableColMap; - std::set pseudoColTable; + // @bug4021, column map for all pseudo column queries + std::map tableColMap; + std::set pseudoColTable; - /* Disk-based join vars */ - boost::shared_ptr smallSideUsage; - boost::shared_ptr umMemLimit; - int64_t smallSideLimit; // need to get these from a session var in execplan - int64_t largeSideLimit; - uint64_t partitionSize; - bool isDML; - std::string timeZone; + /* Disk-based join vars */ + boost::shared_ptr smallSideUsage; + boost::shared_ptr umMemLimit; + int64_t smallSideLimit; // need to get these from a session var in execplan + int64_t largeSideLimit; + uint64_t partitionSize; + bool isDML; + std::string timeZone; - // This is for tracking any dynamically allocated ParseTree objects - // in simpleScalarFilterToParseTree() for later deletion in - // ~csep() or csep.unserialize() - std::vector dynamicParseTreeVec; + // This is for tracking any dynamically allocated ParseTree objects + // in simpleScalarFilterToParseTree() for later deletion in + // ~csep() or csep.unserialize() + std::vector dynamicParseTreeVec; -private: - //defaults okay - //JobInfo(const JobInfo& rhs); - //JobInfo& operator=(const JobInfo& rhs); + private: + // defaults okay + // JobInfo(const JobInfo& rhs); + // JobInfo& operator=(const JobInfo& rhs); }; - //------------------------------------------------------------------------------ // namespace scoped functions //------------------------------------------------------------------------------ @@ -404,45 +414,26 @@ bool isCharCol(const execplan::CalpontSystemCatalog::ColType& colType); * */ execplan::CalpontSystemCatalog::OID tableOid(const execplan::SimpleColumn* sc, - boost::shared_ptr cat); + boost::shared_ptr cat); /** @brief Returns the unique ID to be used in tupleInfo * */ -uint32_t getTupleKey(JobInfo& jobInfo, - const execplan::SimpleColumn* sc, - bool add = false); -uint32_t getTableKey(const JobInfo& jobInfo, - execplan::CalpontSystemCatalog::OID tableOid, - const std::string& alias, - const std::string& schema, - const std::string& view); -uint32_t getTupleKey(JobInfo& jobInfo, - const execplan::SRCP& srcp, - bool add = false); -uint32_t getTableKey(const JobInfo& jobInfo, - uint32_t cid); -uint32_t getTableKey(JobInfo& jobInfo, - JobStep* js); +uint32_t getTupleKey(JobInfo& jobInfo, const execplan::SimpleColumn* sc, bool add = false); +uint32_t getTableKey(const JobInfo& jobInfo, execplan::CalpontSystemCatalog::OID tableOid, + const std::string& alias, const std::string& schema, const std::string& view); +uint32_t getTupleKey(JobInfo& jobInfo, const execplan::SRCP& srcp, bool add = false); +uint32_t getTableKey(const JobInfo& jobInfo, uint32_t cid); +uint32_t getTableKey(JobInfo& jobInfo, JobStep* js); -void updateTableKey(uint32_t cid, - uint32_t tid, - JobInfo& jobInfo); +void updateTableKey(uint32_t cid, uint32_t tid, JobInfo& jobInfo); -uint32_t getExpTupleKey(const JobInfo& jobInfo, - uint64_t eid, - bool cr = false); - -uint32_t makeTableKey(JobInfo& jobInfo, - const execplan::SimpleColumn* sc); -uint32_t makeTableKey(JobInfo& jobInfo, - execplan::CalpontSystemCatalog::OID tableOid, - const std::string& tbl_name, - const std::string& tbl_alias, - const std::string& sch_name, - const std::string& vw_name, - uint64_t engine = 0); +uint32_t getExpTupleKey(const JobInfo& jobInfo, uint64_t eid, bool cr = false); +uint32_t makeTableKey(JobInfo& jobInfo, const execplan::SimpleColumn* sc); +uint32_t makeTableKey(JobInfo& jobInfo, execplan::CalpontSystemCatalog::OID tableOid, + const std::string& tbl_name, const std::string& tbl_alias, const std::string& sch_name, + const std::string& vw_name, uint64_t engine = 0); /** @brief Returns the tupleInfo associate with the (table, column) key pair * @@ -453,20 +444,15 @@ TupleInfo getTupleInfo(uint32_t columnKey, const JobInfo& jobInfo); * */ TupleInfo setTupleInfo(const execplan::CalpontSystemCatalog::ColType& ct, - execplan::CalpontSystemCatalog::OID col_oid, - JobInfo& jobInfo, - execplan::CalpontSystemCatalog::OID tbl_oid, - const execplan::SimpleColumn* sc, + execplan::CalpontSystemCatalog::OID col_oid, JobInfo& jobInfo, + execplan::CalpontSystemCatalog::OID tbl_oid, const execplan::SimpleColumn* sc, const std::string& alias); /** @brief set tuple info for expressions * */ -TupleInfo setExpTupleInfo(const execplan::CalpontSystemCatalog::ColType& ct, - uint64_t expressionId, - const std::string& alias, - JobInfo& jobInfo, - bool rc = false); +TupleInfo setExpTupleInfo(const execplan::CalpontSystemCatalog::ColType& ct, uint64_t expressionId, + const std::string& alias, JobInfo& jobInfo, bool rc = false); TupleInfo setExpTupleInfo(const execplan::ReturnedColumn* rc, JobInfo& jobInfo); @@ -475,26 +461,22 @@ TupleInfo setExpTupleInfo(const execplan::ReturnedColumn* rc, JobInfo& jobInfo); */ void addAggregateColumn(execplan::ReturnedColumn*, int, RetColsVector&, JobInfo&); -void makeJobSteps(execplan::CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, - JobStepVector& querySteps, JobStepVector& projectSteps, - DeliveredTableMap& deliverySteps); +void makeJobSteps(execplan::CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, + JobStepVector& projectSteps, DeliveredTableMap& deliverySteps); void makeUnionJobSteps(execplan::CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, JobStepVector&, DeliveredTableMap& deliverySteps); -void updateDerivedColumn(JobInfo&, execplan::SimpleColumn*, - execplan::CalpontSystemCatalog::ColType&); +void updateDerivedColumn(JobInfo&, execplan::SimpleColumn*, execplan::CalpontSystemCatalog::ColType&); bool filterWithDictionary(execplan::CalpontSystemCatalog::OID dictOid, uint64_t n); bool compatibleColumnTypes(const execplan::CalpontSystemCatalog::ColType& ct1, - const execplan::CalpontSystemCatalog::ColType& ct2, - bool forJoin = true); + const execplan::CalpontSystemCatalog::ColType& ct2, bool forJoin = true); bool compatibleColumnTypes(const execplan::CalpontSystemCatalog::ColDataType& dt1, uint32_t scale1, const execplan::CalpontSystemCatalog::ColDataType& dt2, uint32_t scale2, - bool forJoin = true); + bool forJoin = true); - -} // end of jlf_common namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 5416f9581..a9c0c254b 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -17,7 +17,6 @@ // $Id: jlf_execplantojoblist.cpp 9702 2013-07-17 19:08:07Z xlou $ - #include "jlf_execplantojoblist.h" #include @@ -87,14 +86,13 @@ using namespace logging; #include "jlf_tuplejoblist.h" #include "mcs_decimal.h" - #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" - // for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to 'typeid' +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" +// for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to +// 'typeid' #endif - namespace { using namespace joblist; @@ -104,7 +102,7 @@ const string boldStop = "\033[0;39m"; typedef stack TreeNodeStack; //@bug 598 self-join -//typedef std::pair OIDAliasPair; //self-join +// typedef std::pair OIDAliasPair; //self-join const Operator opeq("="); const Operator oplt("<"); @@ -129,1843 +127,1751 @@ const Operator opNOTLIKE("NOT LIKE"); const Operator opisnotnull("isnotnull"); const Operator opisnull("isnull"); - const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); - /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ -template +template void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val) { - T& n = val; - bool pushWarning = false; - boost::any anyVal = ct.convertColumnData("", pushWarning, timeZone, true, false, false); + T& n = val; + bool pushWarning = false; + boost::any anyVal = ct.convertColumnData("", pushWarning, timeZone, true, false, false); - switch (ct.colDataType) + switch (ct.colDataType) + { + case CalpontSystemCatalog::BIT: + // n = boost::any_cast(anyVal); + break; + + case CalpontSystemCatalog::TINYINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::UTINYINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::SMALLINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::USMALLINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::BIGINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::UBIGINT: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: { - case CalpontSystemCatalog::BIT: - //n = boost::any_cast(anyVal); - break; + float f = boost::any_cast(anyVal); - case CalpontSystemCatalog::TINYINT: - n = boost::any_cast(anyVal); - break; + // N.B. There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, + // but not necessarily the same bits that you put in. This only seems to be for float (double seems + // to work). + if (isnan(f)) + { + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + f = *tfp; + } - case CalpontSystemCatalog::UTINYINT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::SMALLINT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::USMALLINT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::BIGINT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::UBIGINT: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float f = boost::any_cast(anyVal); - - //N.B. There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, - // but not necessarily the same bits that you put in. This only seems to be for float (double seems - // to work). - if (isnan(f)) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - f = *tfp; - } - - float* fp = &f; - int32_t* ip = reinterpret_cast(fp); - n = *ip; - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double d = boost::any_cast(anyVal); - double* dp = &d; - int64_t* ip = reinterpret_cast(dp); - n = *ip; - } - break; - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - case CalpontSystemCatalog::CHAR: - - // @bug 532 - where is null was not returning null rows for char(1) - char(8). - // @Bug 972 Varchar should be treated differently from char - if ( (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7) || - (ct.colDataType == CalpontSystemCatalog::VARBINARY && ct.colWidth <= 7) || - (ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) ) - { - const string& i = boost::any_cast(anyVal); - - //n = *((uint64_t *) i.c_str()); - /* this matches what dataconvert is returning; not valid to copy - * 8 bytes every time. */ - if (ct.colDataType == CalpontSystemCatalog::CHAR) - { - switch (ct.colWidth) - { - case 1: - n = *((uint8_t*) i.data()); - break; - - case 2: - n = *((uint16_t*) i.data()); - break; - - case 3: - case 4: - n = *((uint32_t*) i.data()); - break; - - default: - n = *((uint64_t*) i.data()); - break; - } - } - else - { - switch (ct.colWidth) - { - case 1: - n = *((uint16_t*) i.data()); - break; - - case 2: - n = *((uint32_t*) i.data()); - break; - - default: - n = *((uint64_t*) i.data()); - break; - } - } - - } - else - { - WriteEngine::Token t = boost::any_cast(anyVal); - n = *(uint64_t*)&t; - } - - break; - - case CalpontSystemCatalog::DATE: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DATETIME: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIMESTAMP: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIME: - n = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - n = boost::any_cast(anyVal); - - break; - - default: - break; + float* fp = &f; + int32_t* ip = reinterpret_cast(fp); + n = *ip; } + break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double d = boost::any_cast(anyVal); + double* dp = &d; + int64_t* ip = reinterpret_cast(dp); + n = *ip; + } + break; + + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: + case CalpontSystemCatalog::CHAR: + + // @bug 532 - where is null was not returning null rows for char(1) - char(8). + // @Bug 972 Varchar should be treated differently from char + if ((ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7) || + (ct.colDataType == CalpontSystemCatalog::VARBINARY && ct.colWidth <= 7) || + (ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8)) + { + const string& i = boost::any_cast(anyVal); + + // n = *((uint64_t *) i.c_str()); + /* this matches what dataconvert is returning; not valid to copy + * 8 bytes every time. */ + if (ct.colDataType == CalpontSystemCatalog::CHAR) + { + switch (ct.colWidth) + { + case 1: n = *((uint8_t*)i.data()); break; + + case 2: n = *((uint16_t*)i.data()); break; + + case 3: + case 4: n = *((uint32_t*)i.data()); break; + + default: n = *((uint64_t*)i.data()); break; + } + } + else + { + switch (ct.colWidth) + { + case 1: n = *((uint16_t*)i.data()); break; + + case 2: n = *((uint32_t*)i.data()); break; + + default: n = *((uint64_t*)i.data()); break; + } + } + } + else + { + WriteEngine::Token t = boost::any_cast(anyVal); + n = *(uint64_t*)&t; + } + + break; + + case CalpontSystemCatalog::DATE: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DATETIME: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::TIMESTAMP: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::TIME: n = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + n = boost::any_cast(anyVal); + + break; + + default: break; + } } template -void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, const string& timeZone, T& v) +void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, + const string& timeZone, T& v) { - if (str.size() == 0 || isNull ) - { - valueNullNum(ct, timeZone, v); - return; - } + if (str.size() == 0 || isNull) + { + valueNullNum(ct, timeZone, v); + return; + } + v = 0; + rf = 0; + bool pushWarning = false; + boost::any anyVal = ct.convertColumnData(str, pushWarning, timeZone, false, true, false); - v = 0; - rf = 0; - bool pushWarning = false; - boost::any anyVal = ct.convertColumnData(str, pushWarning, timeZone, false, true, false); + switch (ct.colDataType) + { + case CalpontSystemCatalog::BIT: v = boost::any_cast(anyVal); break; - switch (ct.colDataType) - { - case CalpontSystemCatalog::BIT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::TINYINT: #if BOOST_VERSION >= 105200 - v = boost::any_cast(anyVal); + v = boost::any_cast(anyVal); #else - v = boost::any_cast(anyVal); + v = boost::any_cast(anyVal); #endif - break; + break; - case CalpontSystemCatalog::UTINYINT: - v = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::UTINYINT: v = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::SMALLINT: - v = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::SMALLINT: v = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::USMALLINT: - v = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::USMALLINT: v = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: #ifdef _MSC_VER - v = boost::any_cast(anyVal); + v = boost::any_cast(anyVal); #else - v = boost::any_cast(anyVal); + v = boost::any_cast(anyVal); #endif - break; + break; - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: #ifdef _MSC_VER - v = boost::any_cast(anyVal); + v = boost::any_cast(anyVal); #else - v = boost::any_cast(anyVal); + v = boost::any_cast(anyVal); #endif - break; + break; - case CalpontSystemCatalog::BIGINT: - v = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::BIGINT: v = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::UBIGINT: - v = boost::any_cast(anyVal); - break; + case CalpontSystemCatalog::UBIGINT: v = boost::any_cast(anyVal); break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float f = boost::any_cast(anyVal); - - //N.B. There is a bug in boost::any or in gcc where, if you store a nan, - // you will get back a nan, but not necessarily the same bits that you put in. - // This only seems to be for float (double seems to work). - if (isnan(f)) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - f = *tfp; - } - - float* fp = &f; - int32_t* ip = reinterpret_cast(fp); - v = *ip; - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double d = boost::any_cast(anyVal); - double* dp = &d; - int64_t* ip = reinterpret_cast(dp); - v = *ip; - } - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - { - string i = boost::any_cast(anyVal); - // bug 1932, pad nulls up to the size of v - i.resize(sizeof(v), 0); - v = *((uint64_t*) i.data()); - - if (pushWarning) - rf = ROUND_POS; - } - break; - - case CalpontSystemCatalog::DATE: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DATETIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIMESTAMP: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) -#ifdef _MSC_VER - v = boost::any_cast(anyVal); - -#else - v = boost::any_cast(anyVal); -#endif - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - v = boost::any_cast(anyVal); - - break; - - default: - break; - } - - if ((ct.colDataType == CalpontSystemCatalog::TINYINT || - ct.colDataType == CalpontSystemCatalog::SMALLINT || - ct.colDataType == CalpontSystemCatalog::MEDINT || - ct.colDataType == CalpontSystemCatalog::INT || - ct.colDataType == CalpontSystemCatalog::BIGINT || - ct.colDataType == CalpontSystemCatalog::DECIMAL || - ct.colDataType == CalpontSystemCatalog::UTINYINT || - ct.colDataType == CalpontSystemCatalog::USMALLINT || - ct.colDataType == CalpontSystemCatalog::UMEDINT || - ct.colDataType == CalpontSystemCatalog::UINT || - ct.colDataType == CalpontSystemCatalog::UBIGINT || - ct.colDataType == CalpontSystemCatalog::UDECIMAL) && - pushWarning) + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: { - // get rid of leading white spaces and parentheses - string data(str); - size_t fpos = data.find_first_of(" \t()"); + float f = boost::any_cast(anyVal); - while (string::npos != fpos) - { - data.erase(fpos, 1); - fpos = data.find_first_of(" \t()"); - } + // N.B. There is a bug in boost::any or in gcc where, if you store a nan, + // you will get back a nan, but not necessarily the same bits that you put in. + // This only seems to be for float (double seems to work). + if (isnan(f)) + { + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + f = *tfp; + } - rf = (data[0] == '-') ? ROUND_NEG : ROUND_POS; + float* fp = &f; + int32_t* ip = reinterpret_cast(fp); + v = *ip; } + break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double d = boost::any_cast(anyVal); + double* dp = &d; + int64_t* ip = reinterpret_cast(dp); + v = *ip; + } + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: + { + string i = boost::any_cast(anyVal); + // bug 1932, pad nulls up to the size of v + i.resize(sizeof(v), 0); + v = *((uint64_t*)i.data()); + + if (pushWarning) + rf = ROUND_POS; + } + break; + + case CalpontSystemCatalog::DATE: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DATETIME: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::TIMESTAMP: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::TIME: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) +#ifdef _MSC_VER + v = boost::any_cast(anyVal); + +#else + v = boost::any_cast(anyVal); +#endif + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + v = boost::any_cast(anyVal); + + break; + + default: break; + } + + if ((ct.colDataType == CalpontSystemCatalog::TINYINT || ct.colDataType == CalpontSystemCatalog::SMALLINT || + ct.colDataType == CalpontSystemCatalog::MEDINT || ct.colDataType == CalpontSystemCatalog::INT || + ct.colDataType == CalpontSystemCatalog::BIGINT || ct.colDataType == CalpontSystemCatalog::DECIMAL || + ct.colDataType == CalpontSystemCatalog::UTINYINT || + ct.colDataType == CalpontSystemCatalog::USMALLINT || ct.colDataType == CalpontSystemCatalog::UMEDINT || + ct.colDataType == CalpontSystemCatalog::UINT || ct.colDataType == CalpontSystemCatalog::UBIGINT || + ct.colDataType == CalpontSystemCatalog::UDECIMAL) && + pushWarning) + { + // get rid of leading white spaces and parentheses + string data(str); + size_t fpos = data.find_first_of(" \t()"); + + while (string::npos != fpos) + { + data.erase(fpos, 1); + fpos = data.find_first_of(" \t()"); + } + + rf = (data[0] == '-') ? ROUND_NEG : ROUND_POS; + } } -//TODO: make this totaly case-insensitive +// TODO: make this totaly case-insensitive int8_t op2num(const SOP& sop) { - if (*sop == opeq) - return COMPARE_EQ; - else if (*sop == oplt) - return COMPARE_LT; - else if (*sop == ople) - return COMPARE_LE; - else if (*sop == opgt) - return COMPARE_GT; - else if (*sop == opge) - return COMPARE_GE; - else if (*sop == opne) - return COMPARE_NE; - else if (*sop == oplike || *sop == opLIKE) - return COMPARE_LIKE; - else if (*sop == opis || *sop == opIS || *sop == opisnull) - return COMPARE_EQ; - else if (*sop == opisnot || *sop == opISNOT || *sop == opisnotnull) - return COMPARE_NE; - else if (*sop == opnotlike || *sop == opNOTLIKE) - return COMPARE_NLIKE; - else - cerr << boldStart << "op2num: Unhandled operator >" << *sop << '<' << boldStop << endl; + if (*sop == opeq) + return COMPARE_EQ; + else if (*sop == oplt) + return COMPARE_LT; + else if (*sop == ople) + return COMPARE_LE; + else if (*sop == opgt) + return COMPARE_GT; + else if (*sop == opge) + return COMPARE_GE; + else if (*sop == opne) + return COMPARE_NE; + else if (*sop == oplike || *sop == opLIKE) + return COMPARE_LIKE; + else if (*sop == opis || *sop == opIS || *sop == opisnull) + return COMPARE_EQ; + else if (*sop == opisnot || *sop == opISNOT || *sop == opisnotnull) + return COMPARE_NE; + else if (*sop == opnotlike || *sop == opNOTLIKE) + return COMPARE_NLIKE; + else + cerr << boldStart << "op2num: Unhandled operator >" << *sop << '<' << boldStop << endl; - return COMPARE_NIL; + return COMPARE_NIL; } int8_t bop2num(const SOP& sop) { - if (*sop == opand || *sop == opAND) - return BOP_AND; - else if (*sop == opor || *sop == opOR) - return BOP_OR; - else if (*sop == opxor || *sop == opXOR) - return BOP_XOR; - else - cerr << boldStart << "bop2num: Unhandled operator " << *sop << boldStop << endl; + if (*sop == opand || *sop == opAND) + return BOP_AND; + else if (*sop == opor || *sop == opOR) + return BOP_OR; + else if (*sop == opxor || *sop == opXOR) + return BOP_XOR; + else + cerr << boldStart << "bop2num: Unhandled operator " << *sop << boldStop << endl; - return BOP_NONE; + return BOP_NONE; } enum TreeNodeType { - TREENODE, - FILTER, - CONSTANTFILTER, - EXISTSFILTER, - SELECTFILTER, - SIMPLEFILTER, - OUTERJOINONFILTER, - OPERATOR, - RETURNEDCOLUMN, - AGGREGATECOLUMN, - WINDOWFUNCTIONCOLUMN, - ARITHMETICCOLUMN, - CONSTANTCOLUMN, - FUNCTIONCOLUMN, - SIMPLECOLUMN, - TREENODEIMPL, - SIMPLESCALARFILTER, - UNKNOWN, + TREENODE, + FILTER, + CONSTANTFILTER, + EXISTSFILTER, + SELECTFILTER, + SIMPLEFILTER, + OUTERJOINONFILTER, + OPERATOR, + RETURNEDCOLUMN, + AGGREGATECOLUMN, + WINDOWFUNCTIONCOLUMN, + ARITHMETICCOLUMN, + CONSTANTCOLUMN, + FUNCTIONCOLUMN, + SIMPLECOLUMN, + TREENODEIMPL, + SIMPLESCALARFILTER, + UNKNOWN, }; TreeNodeType TreeNode2Type(const TreeNode* tn) { - //The indentation here is to show inheritance only. - if (typeid(*tn) == typeid(TreeNode)) - return TREENODE; + // The indentation here is to show inheritance only. + if (typeid(*tn) == typeid(TreeNode)) + return TREENODE; - if (typeid(*tn) == typeid(Filter)) - return FILTER; + if (typeid(*tn) == typeid(Filter)) + return FILTER; - if (typeid(*tn) == typeid(ConstantFilter)) - return CONSTANTFILTER; + if (typeid(*tn) == typeid(ConstantFilter)) + return CONSTANTFILTER; - if (typeid(*tn) == typeid(ExistsFilter)) - return EXISTSFILTER; + if (typeid(*tn) == typeid(ExistsFilter)) + return EXISTSFILTER; - if (typeid(*tn) == typeid(SelectFilter)) - return SELECTFILTER; + if (typeid(*tn) == typeid(SelectFilter)) + return SELECTFILTER; - if (typeid(*tn) == typeid(SimpleFilter)) - return SIMPLEFILTER; + if (typeid(*tn) == typeid(SimpleFilter)) + return SIMPLEFILTER; - if (typeid(*tn) == typeid(OuterJoinOnFilter)) - return OUTERJOINONFILTER; + if (typeid(*tn) == typeid(OuterJoinOnFilter)) + return OUTERJOINONFILTER; - if (typeid(*tn) == typeid(Operator) || - typeid(*tn) == typeid(PredicateOperator) || - typeid(*tn) == typeid(LogicOperator)) - return OPERATOR; + if (typeid(*tn) == typeid(Operator) || typeid(*tn) == typeid(PredicateOperator) || + typeid(*tn) == typeid(LogicOperator)) + return OPERATOR; - if (typeid(*tn) == typeid(ReturnedColumn)) - return RETURNEDCOLUMN; + if (typeid(*tn) == typeid(ReturnedColumn)) + return RETURNEDCOLUMN; - if (typeid(*tn) == typeid(AggregateColumn)) - return AGGREGATECOLUMN; + if (typeid(*tn) == typeid(AggregateColumn)) + return AGGREGATECOLUMN; - if (typeid(*tn) == typeid(WindowFunctionColumn)) - return WINDOWFUNCTIONCOLUMN; + if (typeid(*tn) == typeid(WindowFunctionColumn)) + return WINDOWFUNCTIONCOLUMN; - if (typeid(*tn) == typeid(ArithmeticColumn)) - return ARITHMETICCOLUMN; + if (typeid(*tn) == typeid(ArithmeticColumn)) + return ARITHMETICCOLUMN; - if (typeid(*tn) == typeid(ConstantColumn)) - return CONSTANTCOLUMN; + if (typeid(*tn) == typeid(ConstantColumn)) + return CONSTANTCOLUMN; - if (typeid(*tn) == typeid(FunctionColumn)) - return FUNCTIONCOLUMN; + if (typeid(*tn) == typeid(FunctionColumn)) + return FUNCTIONCOLUMN; - if (typeid(*tn) == typeid(SimpleColumn)) - return SIMPLECOLUMN; + if (typeid(*tn) == typeid(SimpleColumn)) + return SIMPLECOLUMN; - if (typeid(*tn) == typeid(SimpleColumn_INT<1>) || - typeid(*tn) == typeid(SimpleColumn_INT<2>) || - typeid(*tn) == typeid(SimpleColumn_INT<4>) || - typeid(*tn) == typeid(SimpleColumn_INT<8>) || - typeid(*tn) == typeid(SimpleColumn_UINT<1>) || - typeid(*tn) == typeid(SimpleColumn_UINT<2>) || - typeid(*tn) == typeid(SimpleColumn_UINT<4>) || - typeid(*tn) == typeid(SimpleColumn_UINT<8>)) - return SIMPLECOLUMN; + if (typeid(*tn) == typeid(SimpleColumn_INT<1>) || typeid(*tn) == typeid(SimpleColumn_INT<2>) || + typeid(*tn) == typeid(SimpleColumn_INT<4>) || typeid(*tn) == typeid(SimpleColumn_INT<8>) || + typeid(*tn) == typeid(SimpleColumn_UINT<1>) || typeid(*tn) == typeid(SimpleColumn_UINT<2>) || + typeid(*tn) == typeid(SimpleColumn_UINT<4>) || typeid(*tn) == typeid(SimpleColumn_UINT<8>)) + return SIMPLECOLUMN; - if (typeid(*tn) == typeid(SimpleColumn_Decimal<1>) || - typeid(*tn) == typeid(SimpleColumn_Decimal<2>) || - typeid(*tn) == typeid(SimpleColumn_Decimal<4>) || - typeid(*tn) == typeid(SimpleColumn_Decimal<8>)) - return SIMPLECOLUMN; + if (typeid(*tn) == typeid(SimpleColumn_Decimal<1>) || typeid(*tn) == typeid(SimpleColumn_Decimal<2>) || + typeid(*tn) == typeid(SimpleColumn_Decimal<4>) || typeid(*tn) == typeid(SimpleColumn_Decimal<8>)) + return SIMPLECOLUMN; - if (typeid(*tn) == typeid(PseudoColumn)) - return SIMPLECOLUMN; + if (typeid(*tn) == typeid(PseudoColumn)) + return SIMPLECOLUMN; - if (typeid(*tn) == typeid(TreeNodeImpl)) - return TREENODEIMPL; + if (typeid(*tn) == typeid(TreeNodeImpl)) + return TREENODEIMPL; - if (typeid(*tn) == typeid(SimpleScalarFilter)) - return SIMPLESCALARFILTER; + if (typeid(*tn) == typeid(SimpleScalarFilter)) + return SIMPLESCALARFILTER; - return UNKNOWN; + return UNKNOWN; } - const JobStepVector doColFilter(const SimpleColumn* sc1, const SimpleColumn* sc2, JobInfo& jobInfo, const SOP& sop, SimpleFilter* sf) { - //The idea here is to take the two SC's and pipe them into a filter step. - //The output of the filter step is one DL that is the minimum rid list met the condition. - CalpontSystemCatalog::OID tableOid1 = tableOid(sc1, jobInfo.csc); - CalpontSystemCatalog::OID tableOid2 = tableOid(sc2, jobInfo.csc); - string alias1(extractTableAlias(sc1)); - string alias2(extractTableAlias(sc2)); - CalpontSystemCatalog::ColType ct1 = sc1->colType(); - CalpontSystemCatalog::ColType ct2 = sc2->colType(); - const PseudoColumn* pc1 = dynamic_cast(sc1); - const PseudoColumn* pc2 = dynamic_cast(sc2); - -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (!sc1->schemaName().empty() && sc1->isColumnStore() && !pc1) - { - ct1 = jobInfo.csc->colType(sc1->oid()); - ct1.charsetNumber =sc1->colType().charsetNumber; - } - if (!sc2->schemaName().empty() && sc2->isColumnStore() && !pc2) - { - ct2 = jobInfo.csc->colType(sc2->oid()); - ct2.charsetNumber =sc2->colType().charsetNumber; - } -//X - int8_t op = op2num(sop); - - pColStep* pcs1 = NULL; - - if (pc1 == NULL) - pcs1 = new pColStep(sc1->oid(), tableOid1, ct1, jobInfo); - else - pcs1 = new PseudoColStep(sc1->oid(), tableOid1, pc1->pseudoType(), ct1, jobInfo); - - CalpontSystemCatalog::OID dictOid1 = isDictCol(ct1); - pcs1->alias(alias1); - pcs1->view(sc1->viewName()); - pcs1->name(sc1->columnName()); - pcs1->schema(sc1->schemaName()); - pcs1->cardinality(sc1->cardinality()); - pcs1->setFeederFlag(true); - - pColStep* pcs2 = NULL; - - if (pc2 == NULL) - pcs2 = new pColStep(sc2->oid(), tableOid2, ct2, jobInfo); - else - pcs2 = new PseudoColStep(sc2->oid(), tableOid2, pc2->pseudoType(), ct2, jobInfo); - - CalpontSystemCatalog::OID dictOid2 = isDictCol(ct2); - pcs2->alias(alias2); - pcs2->view(sc2->viewName()); - pcs2->name(sc2->columnName()); - pcs2->schema(sc2->schemaName()); - pcs2->cardinality(sc2->cardinality()); - pcs2->setFeederFlag(true); - - //Associate the steps - JobStepVector jsv; - - TupleInfo ti1(setTupleInfo(ct1, sc1->oid(), jobInfo, tableOid1, sc1, alias1)); - pcs1->tupleId(ti1.key); - TupleInfo ti2(setTupleInfo(ct2, sc2->oid(), jobInfo, tableOid2, sc2, alias2)); - pcs2->tupleId(ti2.key); - - // check if they are string columns greater than 8 bytes. - if ((!isDictCol(ct1)) && (!isDictCol(ct2))) - { - // not strings, no need for dictionary steps, output fifo datalist - AnyDataListSPtr spdl1(new AnyDataList()); - FifoDataList* dl1 = new FifoDataList(1, jobInfo.fifoSize); - spdl1->fifoDL(dl1); - dl1->OID(sc1->oid()); - - JobStepAssociation outJs1; - outJs1.outAdd(spdl1); - pcs1->outputAssociation(outJs1); - - AnyDataListSPtr spdl2(new AnyDataList()); - FifoDataList* dl2 = new FifoDataList(1, jobInfo.fifoSize); - spdl2->fifoDL(dl2); - dl2->OID(sc2->oid()); - - JobStepAssociation outJs2; - outJs2.outAdd(spdl2); - pcs2->outputAssociation(outJs2); - pcs2->inputAssociation(outJs1); - - - FilterStep* filt = new FilterStep(ct1, jobInfo); - filt->alias(extractTableAlias(sc1)); - filt->tableOid(tableOid1); - filt->name(pcs1->name() + "," + pcs2->name()); - filt->view(pcs1->view()); - filt->schema(pcs1->schema()); - filt->addFilter(sf); - - if (op) - filt->setBOP(op); - - JobStepAssociation outJs3; - outJs3.outAdd(spdl1); - outJs3.outAdd(spdl2); - filt->inputAssociation(outJs3); - - SJSTEP step; - step.reset(pcs1); - jsv.push_back(step); - step.reset(pcs2); - jsv.push_back(step); - step.reset(filt); - jsv.push_back(step); - } - else if ((isCharCol(ct1)) && (isCharCol(ct2))) - { - //check where any column is greater than eight bytes - if ((isDictCol(ct1) != 0 ) && (isDictCol(ct2) != 0 )) - { - // extra steps for string column greater than eight bytes -- from token to string - pDictionaryStep* pdss1 = new pDictionaryStep(dictOid1, tableOid1, ct1, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid1] = sc1->oid(); - pdss1->alias(extractTableAlias(sc1)); - pdss1->view(sc1->viewName()); - pdss1->name(sc1->columnName()); - pdss1->schema(sc1->schemaName()); - pdss1->cardinality(sc1->cardinality()); - - // data list for column 1 step 1 (pcolstep) output - AnyDataListSPtr spdl11(new AnyDataList()); - FifoDataList* dl11 = new FifoDataList(1, jobInfo.fifoSize); - spdl11->fifoDL(dl11); - dl11->OID(sc1->oid()); - - JobStepAssociation outJs1; - outJs1.outAdd(spdl11); - pcs1->outputAssociation(outJs1); - - // data list for column 1 step 2 (pdictionarystep) output - AnyDataListSPtr spdl12(new AnyDataList()); - StringFifoDataList* dl12 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl12->stringDL(dl12); - dl12->OID(sc1->oid()); - - JobStepAssociation outJs2; - outJs2.outAdd(spdl12); - pdss1->outputAssociation(outJs2); - - //Associate pcs1 with pdss1 - JobStepAssociation outJs11; - outJs11.outAdd(spdl11); - pdss1->inputAssociation(outJs11); - - - pDictionaryStep* pdss2 = new pDictionaryStep(dictOid2, tableOid2, ct2, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid2] = sc2->oid(); - pdss2->alias(extractTableAlias(sc2)); - pdss2->view(sc2->viewName()); - pdss2->name(sc2->columnName()); - pdss2->schema(sc2->schemaName()); - pdss2->cardinality(sc2->cardinality()); - - - // data list for column 2 step 1 (pcolstep) output - AnyDataListSPtr spdl21(new AnyDataList()); - FifoDataList* dl21 = new FifoDataList(1, jobInfo.fifoSize); - spdl21->fifoDL(dl21); - dl21->OID(sc2->oid()); - - JobStepAssociation outJs3; - outJs3.outAdd(spdl21); - pcs2->outputAssociation(outJs3); - pcs2->inputAssociation(outJs2); - //Associate pcs2 with pdss2 - JobStepAssociation outJs22; - outJs22.outAdd(spdl21); - pdss2->inputAssociation(outJs22); - - // data list for column 2 step 2 (pdictionarystep) output - AnyDataListSPtr spdl22(new AnyDataList()); - StringFifoDataList* dl22 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl22->stringDL(dl22); - dl22->OID(sc2->oid()); - - JobStepAssociation outJs4; - outJs4.outAdd(spdl22); - pdss2->outputAssociation(outJs4); - - FilterStep* filt = new FilterStep(ct1, jobInfo); - filt->alias(extractTableAlias(sc1)); - filt->tableOid(tableOid1); - filt->name(pcs1->name() + "," + pcs2->name()); - filt->view(pcs1->view()); - filt->schema(pcs1->schema()); - filt->addFilter(sf); - - if (op) - filt->setBOP((op)); - - JobStepAssociation outJs5; - outJs5.outAdd(spdl12); - outJs5.outAdd(spdl22); - filt->inputAssociation(outJs5); - - SJSTEP step; - step.reset(pcs1); - jsv.push_back(step); - step.reset(pdss1); - jsv.push_back(step); - step.reset(pcs2); - jsv.push_back(step); - step.reset(pdss2); - jsv.push_back(step); - step.reset(filt); - jsv.push_back(step); - - TupleInfo ti1(setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc1, alias1)); - pdss1->tupleId(ti1.key); - jobInfo.keyInfo->dictKeyMap[pcs1->tupleId()] = ti1.key; - jobInfo.tokenOnly[pcs1->tupleId()] = false; - - TupleInfo ti2(setTupleInfo(ct2, dictOid2, jobInfo, tableOid2, sc2, alias2)); - pdss2->tupleId(ti2.key); - jobInfo.keyInfo->dictKeyMap[pcs2->tupleId()] = ti2.key; - jobInfo.tokenOnly[pcs2->tupleId()] = false; - } - else if ((isDictCol(ct1) != 0 ) && (isDictCol(ct2) == 0 )) //col1 is dictionary column - { - // extra steps for string column greater than eight bytes -- from token to string - pDictionaryStep* pdss1 = new pDictionaryStep(dictOid1, tableOid1, ct1, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid1] = sc1->oid(); - pdss1->alias(extractTableAlias(sc1)); - pdss1->view(sc1->viewName()); - pdss1->name(sc1->columnName()); - pdss1->schema(sc1->schemaName()); - pdss1->cardinality(sc1->cardinality()); - - // data list for column 1 step 1 (pcolstep) output - AnyDataListSPtr spdl11(new AnyDataList()); - FifoDataList* dl11 = new FifoDataList(1, jobInfo.fifoSize); - spdl11->fifoDL(dl11); - dl11->OID(sc1->oid()); - - JobStepAssociation outJs1; - outJs1.outAdd(spdl11); - pcs1->outputAssociation(outJs1); - - // data list for column 1 step 2 (pdictionarystep) output - AnyDataListSPtr spdl12(new AnyDataList()); - StringFifoDataList* dl12 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl12->stringDL(dl12); - dl12->OID(sc1->oid()); - - JobStepAssociation outJs2; - outJs2.outAdd(spdl12); - pdss1->outputAssociation(outJs2); - - //Associate pcs1 with pdss1 - JobStepAssociation outJs11; - outJs11.outAdd(spdl11); - pdss1->inputAssociation(outJs11); - - // data list for column 2 step 1 (pcolstep) output - AnyDataListSPtr spdl21(new AnyDataList()); - FifoDataList* dl21 = new FifoDataList(1, jobInfo.fifoSize); - spdl21->fifoDL(dl21); - dl21->OID(sc2->oid()); - - JobStepAssociation outJs3; - outJs3.outAdd(spdl21); - pcs2->outputAssociation(outJs3); - pcs2->inputAssociation(outJs2); - - FilterStep* filt = new FilterStep(ct1, jobInfo); - filt->alias(extractTableAlias(sc1)); - filt->tableOid(tableOid1); - filt->view(pcs1->view()); - filt->schema(pcs1->schema()); - filt->addFilter(sf); - - if (op) - filt->setBOP((op)); - - JobStepAssociation outJs5; - outJs5.outAdd(spdl12); - outJs5.outAdd(spdl21); - filt->inputAssociation(outJs5); - - SJSTEP step; - step.reset(pcs1); - jsv.push_back(step); - step.reset(pdss1); - jsv.push_back(step); - step.reset(pcs2); - jsv.push_back(step); - step.reset(filt); - jsv.push_back(step); - - TupleInfo ti1(setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc1, alias1)); - pdss1->tupleId(ti1.key); - jobInfo.keyInfo->dictKeyMap[pcs1->tupleId()] = ti1.key; - jobInfo.tokenOnly[pcs1->tupleId()] = false; - } - else // if ((isDictCol(ct1) == 0 ) && (isDictCol(ct2) !=0 )) //col2 is dictionary column - { - // extra steps for string column greater than eight bytes -- from token to string - // data list for column 1 step 1 (pcolstep) output - AnyDataListSPtr spdl11(new AnyDataList()); - FifoDataList* dl11 = new FifoDataList(1, jobInfo.fifoSize); - spdl11->fifoDL(dl11); - dl11->OID(sc1->oid()); - - JobStepAssociation outJs1; - outJs1.outAdd(spdl11); - pcs1->outputAssociation(outJs1); - - // data list for column 1 step 2 (pdictionarystep) output - AnyDataListSPtr spdl12(new AnyDataList()); - StringFifoDataList* dl12 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl12->stringDL(dl12); - dl12->OID(sc1->oid()); - - - pDictionaryStep* pdss2 = new pDictionaryStep(dictOid2, tableOid2, ct2, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid2] = sc2->oid(); - pdss2->alias(extractTableAlias(sc2)); - pdss2->view(sc2->viewName()); - pdss2->name(sc2->columnName()); - pdss2->schema(sc2->schemaName()); - pdss2->cardinality(sc2->cardinality()); - - - // data list for column 2 step 1 (pcolstep) output - AnyDataListSPtr spdl21(new AnyDataList()); - FifoDataList* dl21 = new FifoDataList(1, jobInfo.fifoSize); - spdl21->fifoDL(dl21); - dl21->OID(sc2->oid()); - - JobStepAssociation outJs3; - outJs3.outAdd(spdl21); - pcs2->outputAssociation(outJs3); - pcs2->inputAssociation(outJs1); - //Associate pcs2 with pdss2 - JobStepAssociation outJs22; - outJs22.outAdd(spdl21); - pdss2->inputAssociation(outJs22); - - // data list for column 2 step 2 (pdictionarystep) output - AnyDataListSPtr spdl22(new AnyDataList()); - StringFifoDataList* dl22 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl22->stringDL(dl22); - dl22->OID(sc2->oid()); - - JobStepAssociation outJs4; - outJs4.outAdd(spdl22); - pdss2->outputAssociation(outJs4); - - FilterStep* filt = new FilterStep(ct1, jobInfo); - filt->alias(extractTableAlias(sc1)); - filt->tableOid(tableOid1); - filt->view(pcs1->view()); - filt->schema(pcs1->schema()); - filt->addFilter(sf); - - if (op) - filt->setBOP((op)); - - JobStepAssociation outJs5; - outJs5.outAdd(spdl11); - outJs5.outAdd(spdl22); - filt->inputAssociation(outJs5); - - SJSTEP step; - step.reset(pcs1); - jsv.push_back(step); - step.reset(pcs2); - jsv.push_back(step); - step.reset(pdss2); - jsv.push_back(step); - step.reset(filt); - jsv.push_back(step); - - TupleInfo ti2(setTupleInfo(ct2, dictOid2, jobInfo, tableOid2, sc2, alias2)); - pdss2->tupleId(ti2.key); - jobInfo.keyInfo->dictKeyMap[pcs2->tupleId()] = ti2.key; - jobInfo.tokenOnly[pcs2->tupleId()] = false; - } - } - else - { - cerr << boldStart << "Filterstep: Filter with different type is not supported " << boldStop << endl; - throw QueryDataExcept("Filter with different column types is not supported.", incompatFilterCols); - } - - return jsv; -} - - -const JobStepVector doFilterExpression(const SimpleColumn* sc1, const SimpleColumn* sc2, JobInfo& jobInfo, const SOP& sop) -{ - JobStepVector jsv; - SJSTEP sjstep; - ExpressionStep* es = new ExpressionStep(jobInfo); - - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 2"); - - SimpleFilter sf; - sf.op(sop); - sf.lhs(sc1->clone()); - sf.rhs(sc2->clone()); - es->expressionFilter(&sf, jobInfo); - - sjstep.reset(es); - jsv.push_back(sjstep); - - return jsv; -} - -const JobStepVector doJoin( - SimpleColumn* sc1, SimpleColumn* sc2, JobInfo& jobInfo, const SOP& sop, SimpleFilter* sf) -{ - if ((sc1->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - sc1->resultType().colDataType == CalpontSystemCatalog::BLOB)) - throw runtime_error("VARBINARY/BLOB in join is not supported."); - - //The idea here is to take the two SC's and pipe them into a HJ step. The output of the HJ step - // is 2 DL's (one for each table) that are the minimum rid list for each side of the join. - CalpontSystemCatalog::OID tableOid1 = tableOid(sc1, jobInfo.csc); - CalpontSystemCatalog::OID tableOid2 = tableOid(sc2, jobInfo.csc); - string alias1(extractTableAlias(sc1)); - string alias2(extractTableAlias(sc2)); - string view1(sc1->viewName()); - string view2(sc2->viewName()); - string schema1(sc1->schemaName()); - string schema2(sc2->schemaName()); - - CalpontSystemCatalog::ColType ct1 = sc1->colType(); - CalpontSystemCatalog::ColType ct2 = sc2->colType(); - PseudoColumn* pc1 = dynamic_cast(sc1); - PseudoColumn* pc2 = dynamic_cast(sc2); - -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (!sc1->schemaName().empty() && sc1->isColumnStore() && !pc1) - { - ct1 = jobInfo.csc->colType(sc1->oid()); - ct1.charsetNumber =sc1->colType().charsetNumber; - } - if (!sc2->schemaName().empty() && sc2->isColumnStore() && !pc2) - { - ct2 = jobInfo.csc->colType(sc2->oid()); - ct2.charsetNumber =sc2->colType().charsetNumber; - } -//X - uint64_t joinInfo = sc1->joinInfo() | sc2->joinInfo(); - - if (sc1->schemaName().empty()) - { - SimpleColumn* tmp = (sc1); - updateDerivedColumn(jobInfo, tmp, ct1); - } - - if (sc2->schemaName().empty()) - { - SimpleColumn* tmp = (sc2); - updateDerivedColumn(jobInfo, tmp, ct2); - } - - //@bug 566 Dont do a hash join if the table and the alias are the same - //Bug 590 - if (tableOid1 == tableOid2 && alias1 == alias2 && view1 == view2 && joinInfo == 0) - { - if (sc1->schemaName().empty() || !compatibleColumnTypes(ct1, ct2, false)) - { - return doFilterExpression(sc1, sc2, jobInfo, sop); - } - - JobStepVector colFilter = doColFilter(sc1, sc2, jobInfo, sop, sf); - //jsv.insert(jsv.end(), colFilter.begin(), colFilter.end()); - return colFilter; - } - - // different tables - if (!compatibleColumnTypes(ct1, ct2, true)) - { - JobStepVector jsv; - jsv = doFilterExpression(sc1, sc2, jobInfo, sop); - uint32_t t1 = makeTableKey(jobInfo, sc1); - uint32_t t2 = makeTableKey(jobInfo, sc2); - jobInfo.incompatibleJoinMap[t1] = t2; - jobInfo.incompatibleJoinMap[t2] = t1; - - return jsv; - } - - pColStep* pcs1 = NULL; - CalpontSystemCatalog::OID oid1 = sc1->oid(); - CalpontSystemCatalog::OID dictOid1 = isDictCol(ct1); - - if (sc1->schemaName().empty() == false) - { - if (pc1 == NULL) - pcs1 = new pColStep(oid1, tableOid1, ct1, jobInfo); - else - pcs1 = new PseudoColStep(oid1, tableOid1, pc1->pseudoType(), ct1, jobInfo); - - pcs1->alias(alias1); - pcs1->view(view1); - pcs1->name(sc1->columnName()); - pcs1->schema(sc1->schemaName()); - pcs1->cardinality(sc1->cardinality()); - } - - pColStep* pcs2 = NULL; - CalpontSystemCatalog::OID oid2 = sc2->oid(); - CalpontSystemCatalog::OID dictOid2 = isDictCol(ct2); - - if (sc2->schemaName().empty() == false) - { - if (pc2 == NULL) - pcs2 = new pColStep(oid2, tableOid2, ct2, jobInfo); - else - pcs2 = new PseudoColStep(oid2, tableOid2, pc2->pseudoType(), ct2, jobInfo); - - pcs2->alias(alias2); - pcs2->view(view2); - pcs2->name(sc2->columnName()); - pcs2->schema(sc2->schemaName()); - pcs2->cardinality(sc2->cardinality()); - } - - JoinType jt = INNER; - - if (sc1->returnAll() && sc2->returnAll()) - { - // Due to current connector limitation, INNER may have both returnAll set. - // @bug3037, compound outer join may have both flag set if not the 1st pair. - jt = INNER; - } - else if (sc1->returnAll()) - { - jt = LEFTOUTER; - } - else if (sc2->returnAll()) - { - jt = RIGHTOUTER; - } - - //Associate the steps - JobStepVector jsv; - SJSTEP step; - - // bug 1495 compound join, v-table handles string join and compound join the same way + // The idea here is to take the two SC's and pipe them into a filter step. + // The output of the filter step is one DL that is the minimum rid list met the condition. + CalpontSystemCatalog::OID tableOid1 = tableOid(sc1, jobInfo.csc); + CalpontSystemCatalog::OID tableOid2 = tableOid(sc2, jobInfo.csc); + string alias1(extractTableAlias(sc1)); + string alias2(extractTableAlias(sc2)); + CalpontSystemCatalog::ColType ct1 = sc1->colType(); + CalpontSystemCatalog::ColType ct2 = sc2->colType(); + const PseudoColumn* pc1 = dynamic_cast(sc1); + const PseudoColumn* pc2 = dynamic_cast(sc2); + + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (!sc1->schemaName().empty() && sc1->isColumnStore() && !pc1) + { + ct1 = jobInfo.csc->colType(sc1->oid()); + ct1.charsetNumber = sc1->colType().charsetNumber; + } + if (!sc2->schemaName().empty() && sc2->isColumnStore() && !pc2) + { + ct2 = jobInfo.csc->colType(sc2->oid()); + ct2.charsetNumber = sc2->colType().charsetNumber; + } + // X + int8_t op = op2num(sop); + + pColStep* pcs1 = NULL; + + if (pc1 == NULL) + pcs1 = new pColStep(sc1->oid(), tableOid1, ct1, jobInfo); + else + pcs1 = new PseudoColStep(sc1->oid(), tableOid1, pc1->pseudoType(), ct1, jobInfo); + + CalpontSystemCatalog::OID dictOid1 = isDictCol(ct1); + pcs1->alias(alias1); + pcs1->view(sc1->viewName()); + pcs1->name(sc1->columnName()); + pcs1->schema(sc1->schemaName()); + pcs1->cardinality(sc1->cardinality()); + pcs1->setFeederFlag(true); + + pColStep* pcs2 = NULL; + + if (pc2 == NULL) + pcs2 = new pColStep(sc2->oid(), tableOid2, ct2, jobInfo); + else + pcs2 = new PseudoColStep(sc2->oid(), tableOid2, pc2->pseudoType(), ct2, jobInfo); + + CalpontSystemCatalog::OID dictOid2 = isDictCol(ct2); + pcs2->alias(alias2); + pcs2->view(sc2->viewName()); + pcs2->name(sc2->columnName()); + pcs2->schema(sc2->schemaName()); + pcs2->cardinality(sc2->cardinality()); + pcs2->setFeederFlag(true); + + // Associate the steps + JobStepVector jsv; + + TupleInfo ti1(setTupleInfo(ct1, sc1->oid(), jobInfo, tableOid1, sc1, alias1)); + pcs1->tupleId(ti1.key); + TupleInfo ti2(setTupleInfo(ct2, sc2->oid(), jobInfo, tableOid2, sc2, alias2)); + pcs2->tupleId(ti2.key); + + // check if they are string columns greater than 8 bytes. + if ((!isDictCol(ct1)) && (!isDictCol(ct2))) + { + // not strings, no need for dictionary steps, output fifo datalist AnyDataListSPtr spdl1(new AnyDataList()); - RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); - spdl1->rowGroupDL(dl1); - dl1->OID(oid1); + FifoDataList* dl1 = new FifoDataList(1, jobInfo.fifoSize); + spdl1->fifoDL(dl1); + dl1->OID(sc1->oid()); - if (pcs1) - { - JobStepAssociation outJs1; - outJs1.outAdd(spdl1); - pcs1->outputAssociation(outJs1); - - step.reset(pcs1); - jsv.push_back(step); - } + JobStepAssociation outJs1; + outJs1.outAdd(spdl1); + pcs1->outputAssociation(outJs1); AnyDataListSPtr spdl2(new AnyDataList()); - RowGroupDL* dl2 = new RowGroupDL(1, jobInfo.fifoSize); - spdl2->rowGroupDL(dl2); - dl2->OID(oid2); + FifoDataList* dl2 = new FifoDataList(1, jobInfo.fifoSize); + spdl2->fifoDL(dl2); + dl2->OID(sc2->oid()); - if (pcs2) - { - JobStepAssociation outJs2; - outJs2.outAdd(spdl2); - pcs2->outputAssociation(outJs2); + JobStepAssociation outJs2; + outJs2.outAdd(spdl2); + pcs2->outputAssociation(outJs2); + pcs2->inputAssociation(outJs1); - step.reset(pcs2); - jsv.push_back(step); - } + FilterStep* filt = new FilterStep(ct1, jobInfo); + filt->alias(extractTableAlias(sc1)); + filt->tableOid(tableOid1); + filt->name(pcs1->name() + "," + pcs2->name()); + filt->view(pcs1->view()); + filt->schema(pcs1->schema()); + filt->addFilter(sf); - TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); - thj->tableOid1(tableOid1); - thj->tableOid2(tableOid2); - thj->alias1(alias1); - thj->alias2(alias2); - thj->view1(view1); - thj->view2(view2); - thj->schema1(schema1); - thj->schema2(schema2); - thj->oid1(oid1); - thj->oid2(oid2); - thj->dictOid1(dictOid1); - thj->dictOid2(dictOid2); - thj->sequence1(sc1->sequence()); - thj->sequence2(sc2->sequence()); - thj->column1(sc1); - thj->column2(sc2); - - // MCOL-334 joins in views need to have higher priority than SEMI/ANTI - if (!view1.empty() && view1 == view2) - { - thj->joinId(-1); - } - else - { - thj->joinId((joinInfo == 0) ? (++jobInfo.joinNum) : 0); - } - - // Check if SEMI/ANTI join. - // INNER/OUTER join and SEMI/ANTI are mutually exclusive, - if (joinInfo != 0) - { - // @bug3998, keep the OUTER join type - // jt = INIT; - - if (joinInfo & JOIN_SEMI) - jt |= SEMI; - - if (joinInfo & JOIN_ANTI) - jt |= ANTI; - - if (joinInfo & JOIN_SCALAR) - jt |= SCALAR; - - if (joinInfo & JOIN_NULL_MATCH) - jt |= MATCHNULLS; - - if (joinInfo & JOIN_CORRELATED) - jt |= CORRELATED; - - if (joinInfo & JOIN_OUTER_SELECT) - jt |= LARGEOUTER; - - if (sc1->joinInfo() & JOIN_CORRELATED) - thj->correlatedSide(1); - else if (sc2->joinInfo() & JOIN_CORRELATED) - thj->correlatedSide(2); - } - - thj->setJoinType(jt); + if (op) + filt->setBOP(op); JobStepAssociation outJs3; outJs3.outAdd(spdl1); outJs3.outAdd(spdl2); - thj->inputAssociation(outJs3); - step.reset(thj); - - TupleInfo ti1(setTupleInfo(ct1, oid1, jobInfo, tableOid1, sc1, alias1)); - - if (pcs1) - { - pcs1->tupleId(ti1.key); - thj->tupleId1(ti1.key); - - if (dictOid1 > 0) - { - ti1 = setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc1, alias1); - jobInfo.keyInfo->dictOidToColOid[dictOid1] = oid1; - jobInfo.keyInfo->dictKeyMap[pcs1->tupleId()] = ti1.key; - jobInfo.tokenOnly[pcs1->tupleId()] = false; -// thj->tupleId1(ti1.key); - } - } - else - { - thj->tupleId1(getTupleKey(jobInfo, sc1)); - } - - TupleInfo ti2(setTupleInfo(ct2, oid2, jobInfo, tableOid2, sc2, alias2)); - - if (pcs2) - { - pcs2->tupleId(ti2.key); - thj->tupleId2(pcs2->tupleId()); - - if (dictOid2 > 0) - { - TupleInfo ti2(setTupleInfo(ct2, dictOid2, jobInfo, tableOid2, sc2, alias2)); - jobInfo.keyInfo->dictOidToColOid[dictOid2] = oid2; - jobInfo.keyInfo->dictKeyMap[pcs2->tupleId()] = ti2.key; - jobInfo.tokenOnly[pcs2->tupleId()] = false; -// thj->tupleId2(ti2.key); - } - } - else - { - thj->tupleId2(getTupleKey(jobInfo, sc2)); - } + filt->inputAssociation(outJs3); + SJSTEP step; + step.reset(pcs1); jsv.push_back(step); + step.reset(pcs2); + jsv.push_back(step); + step.reset(filt); + jsv.push_back(step); + } + else if ((isCharCol(ct1)) && (isCharCol(ct2))) + { + // check where any column is greater than eight bytes + if ((isDictCol(ct1) != 0) && (isDictCol(ct2) != 0)) + { + // extra steps for string column greater than eight bytes -- from token to string + pDictionaryStep* pdss1 = new pDictionaryStep(dictOid1, tableOid1, ct1, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid1] = sc1->oid(); + pdss1->alias(extractTableAlias(sc1)); + pdss1->view(sc1->viewName()); + pdss1->name(sc1->columnName()); + pdss1->schema(sc1->schemaName()); + pdss1->cardinality(sc1->cardinality()); - return jsv; + // data list for column 1 step 1 (pcolstep) output + AnyDataListSPtr spdl11(new AnyDataList()); + FifoDataList* dl11 = new FifoDataList(1, jobInfo.fifoSize); + spdl11->fifoDL(dl11); + dl11->OID(sc1->oid()); + + JobStepAssociation outJs1; + outJs1.outAdd(spdl11); + pcs1->outputAssociation(outJs1); + + // data list for column 1 step 2 (pdictionarystep) output + AnyDataListSPtr spdl12(new AnyDataList()); + StringFifoDataList* dl12 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl12->stringDL(dl12); + dl12->OID(sc1->oid()); + + JobStepAssociation outJs2; + outJs2.outAdd(spdl12); + pdss1->outputAssociation(outJs2); + + // Associate pcs1 with pdss1 + JobStepAssociation outJs11; + outJs11.outAdd(spdl11); + pdss1->inputAssociation(outJs11); + + pDictionaryStep* pdss2 = new pDictionaryStep(dictOid2, tableOid2, ct2, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid2] = sc2->oid(); + pdss2->alias(extractTableAlias(sc2)); + pdss2->view(sc2->viewName()); + pdss2->name(sc2->columnName()); + pdss2->schema(sc2->schemaName()); + pdss2->cardinality(sc2->cardinality()); + + // data list for column 2 step 1 (pcolstep) output + AnyDataListSPtr spdl21(new AnyDataList()); + FifoDataList* dl21 = new FifoDataList(1, jobInfo.fifoSize); + spdl21->fifoDL(dl21); + dl21->OID(sc2->oid()); + + JobStepAssociation outJs3; + outJs3.outAdd(spdl21); + pcs2->outputAssociation(outJs3); + pcs2->inputAssociation(outJs2); + // Associate pcs2 with pdss2 + JobStepAssociation outJs22; + outJs22.outAdd(spdl21); + pdss2->inputAssociation(outJs22); + + // data list for column 2 step 2 (pdictionarystep) output + AnyDataListSPtr spdl22(new AnyDataList()); + StringFifoDataList* dl22 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl22->stringDL(dl22); + dl22->OID(sc2->oid()); + + JobStepAssociation outJs4; + outJs4.outAdd(spdl22); + pdss2->outputAssociation(outJs4); + + FilterStep* filt = new FilterStep(ct1, jobInfo); + filt->alias(extractTableAlias(sc1)); + filt->tableOid(tableOid1); + filt->name(pcs1->name() + "," + pcs2->name()); + filt->view(pcs1->view()); + filt->schema(pcs1->schema()); + filt->addFilter(sf); + + if (op) + filt->setBOP((op)); + + JobStepAssociation outJs5; + outJs5.outAdd(spdl12); + outJs5.outAdd(spdl22); + filt->inputAssociation(outJs5); + + SJSTEP step; + step.reset(pcs1); + jsv.push_back(step); + step.reset(pdss1); + jsv.push_back(step); + step.reset(pcs2); + jsv.push_back(step); + step.reset(pdss2); + jsv.push_back(step); + step.reset(filt); + jsv.push_back(step); + + TupleInfo ti1(setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc1, alias1)); + pdss1->tupleId(ti1.key); + jobInfo.keyInfo->dictKeyMap[pcs1->tupleId()] = ti1.key; + jobInfo.tokenOnly[pcs1->tupleId()] = false; + + TupleInfo ti2(setTupleInfo(ct2, dictOid2, jobInfo, tableOid2, sc2, alias2)); + pdss2->tupleId(ti2.key); + jobInfo.keyInfo->dictKeyMap[pcs2->tupleId()] = ti2.key; + jobInfo.tokenOnly[pcs2->tupleId()] = false; + } + else if ((isDictCol(ct1) != 0) && (isDictCol(ct2) == 0)) // col1 is dictionary column + { + // extra steps for string column greater than eight bytes -- from token to string + pDictionaryStep* pdss1 = new pDictionaryStep(dictOid1, tableOid1, ct1, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid1] = sc1->oid(); + pdss1->alias(extractTableAlias(sc1)); + pdss1->view(sc1->viewName()); + pdss1->name(sc1->columnName()); + pdss1->schema(sc1->schemaName()); + pdss1->cardinality(sc1->cardinality()); + + // data list for column 1 step 1 (pcolstep) output + AnyDataListSPtr spdl11(new AnyDataList()); + FifoDataList* dl11 = new FifoDataList(1, jobInfo.fifoSize); + spdl11->fifoDL(dl11); + dl11->OID(sc1->oid()); + + JobStepAssociation outJs1; + outJs1.outAdd(spdl11); + pcs1->outputAssociation(outJs1); + + // data list for column 1 step 2 (pdictionarystep) output + AnyDataListSPtr spdl12(new AnyDataList()); + StringFifoDataList* dl12 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl12->stringDL(dl12); + dl12->OID(sc1->oid()); + + JobStepAssociation outJs2; + outJs2.outAdd(spdl12); + pdss1->outputAssociation(outJs2); + + // Associate pcs1 with pdss1 + JobStepAssociation outJs11; + outJs11.outAdd(spdl11); + pdss1->inputAssociation(outJs11); + + // data list for column 2 step 1 (pcolstep) output + AnyDataListSPtr spdl21(new AnyDataList()); + FifoDataList* dl21 = new FifoDataList(1, jobInfo.fifoSize); + spdl21->fifoDL(dl21); + dl21->OID(sc2->oid()); + + JobStepAssociation outJs3; + outJs3.outAdd(spdl21); + pcs2->outputAssociation(outJs3); + pcs2->inputAssociation(outJs2); + + FilterStep* filt = new FilterStep(ct1, jobInfo); + filt->alias(extractTableAlias(sc1)); + filt->tableOid(tableOid1); + filt->view(pcs1->view()); + filt->schema(pcs1->schema()); + filt->addFilter(sf); + + if (op) + filt->setBOP((op)); + + JobStepAssociation outJs5; + outJs5.outAdd(spdl12); + outJs5.outAdd(spdl21); + filt->inputAssociation(outJs5); + + SJSTEP step; + step.reset(pcs1); + jsv.push_back(step); + step.reset(pdss1); + jsv.push_back(step); + step.reset(pcs2); + jsv.push_back(step); + step.reset(filt); + jsv.push_back(step); + + TupleInfo ti1(setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc1, alias1)); + pdss1->tupleId(ti1.key); + jobInfo.keyInfo->dictKeyMap[pcs1->tupleId()] = ti1.key; + jobInfo.tokenOnly[pcs1->tupleId()] = false; + } + else // if ((isDictCol(ct1) == 0 ) && (isDictCol(ct2) !=0 )) //col2 is dictionary column + { + // extra steps for string column greater than eight bytes -- from token to string + // data list for column 1 step 1 (pcolstep) output + AnyDataListSPtr spdl11(new AnyDataList()); + FifoDataList* dl11 = new FifoDataList(1, jobInfo.fifoSize); + spdl11->fifoDL(dl11); + dl11->OID(sc1->oid()); + + JobStepAssociation outJs1; + outJs1.outAdd(spdl11); + pcs1->outputAssociation(outJs1); + + // data list for column 1 step 2 (pdictionarystep) output + AnyDataListSPtr spdl12(new AnyDataList()); + StringFifoDataList* dl12 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl12->stringDL(dl12); + dl12->OID(sc1->oid()); + + pDictionaryStep* pdss2 = new pDictionaryStep(dictOid2, tableOid2, ct2, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid2] = sc2->oid(); + pdss2->alias(extractTableAlias(sc2)); + pdss2->view(sc2->viewName()); + pdss2->name(sc2->columnName()); + pdss2->schema(sc2->schemaName()); + pdss2->cardinality(sc2->cardinality()); + + // data list for column 2 step 1 (pcolstep) output + AnyDataListSPtr spdl21(new AnyDataList()); + FifoDataList* dl21 = new FifoDataList(1, jobInfo.fifoSize); + spdl21->fifoDL(dl21); + dl21->OID(sc2->oid()); + + JobStepAssociation outJs3; + outJs3.outAdd(spdl21); + pcs2->outputAssociation(outJs3); + pcs2->inputAssociation(outJs1); + // Associate pcs2 with pdss2 + JobStepAssociation outJs22; + outJs22.outAdd(spdl21); + pdss2->inputAssociation(outJs22); + + // data list for column 2 step 2 (pdictionarystep) output + AnyDataListSPtr spdl22(new AnyDataList()); + StringFifoDataList* dl22 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl22->stringDL(dl22); + dl22->OID(sc2->oid()); + + JobStepAssociation outJs4; + outJs4.outAdd(spdl22); + pdss2->outputAssociation(outJs4); + + FilterStep* filt = new FilterStep(ct1, jobInfo); + filt->alias(extractTableAlias(sc1)); + filt->tableOid(tableOid1); + filt->view(pcs1->view()); + filt->schema(pcs1->schema()); + filt->addFilter(sf); + + if (op) + filt->setBOP((op)); + + JobStepAssociation outJs5; + outJs5.outAdd(spdl11); + outJs5.outAdd(spdl22); + filt->inputAssociation(outJs5); + + SJSTEP step; + step.reset(pcs1); + jsv.push_back(step); + step.reset(pcs2); + jsv.push_back(step); + step.reset(pdss2); + jsv.push_back(step); + step.reset(filt); + jsv.push_back(step); + + TupleInfo ti2(setTupleInfo(ct2, dictOid2, jobInfo, tableOid2, sc2, alias2)); + pdss2->tupleId(ti2.key); + jobInfo.keyInfo->dictKeyMap[pcs2->tupleId()] = ti2.key; + jobInfo.tokenOnly[pcs2->tupleId()] = false; + } + } + else + { + cerr << boldStart << "Filterstep: Filter with different type is not supported " << boldStop << endl; + throw QueryDataExcept("Filter with different column types is not supported.", incompatFilterCols); + } + + return jsv; } +const JobStepVector doFilterExpression(const SimpleColumn* sc1, const SimpleColumn* sc2, JobInfo& jobInfo, + const SOP& sop) +{ + JobStepVector jsv; + SJSTEP sjstep; + ExpressionStep* es = new ExpressionStep(jobInfo); + + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 2"); + + SimpleFilter sf; + sf.op(sop); + sf.lhs(sc1->clone()); + sf.rhs(sc2->clone()); + es->expressionFilter(&sf, jobInfo); + + sjstep.reset(es); + jsv.push_back(sjstep); + + return jsv; +} + +const JobStepVector doJoin(SimpleColumn* sc1, SimpleColumn* sc2, JobInfo& jobInfo, const SOP& sop, + SimpleFilter* sf) +{ + if ((sc1->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + sc1->resultType().colDataType == CalpontSystemCatalog::BLOB)) + throw runtime_error("VARBINARY/BLOB in join is not supported."); + + // The idea here is to take the two SC's and pipe them into a HJ step. The output of the HJ step + // is 2 DL's (one for each table) that are the minimum rid list for each side of the join. + CalpontSystemCatalog::OID tableOid1 = tableOid(sc1, jobInfo.csc); + CalpontSystemCatalog::OID tableOid2 = tableOid(sc2, jobInfo.csc); + string alias1(extractTableAlias(sc1)); + string alias2(extractTableAlias(sc2)); + string view1(sc1->viewName()); + string view2(sc2->viewName()); + string schema1(sc1->schemaName()); + string schema2(sc2->schemaName()); + + CalpontSystemCatalog::ColType ct1 = sc1->colType(); + CalpontSystemCatalog::ColType ct2 = sc2->colType(); + PseudoColumn* pc1 = dynamic_cast(sc1); + PseudoColumn* pc2 = dynamic_cast(sc2); + + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (!sc1->schemaName().empty() && sc1->isColumnStore() && !pc1) + { + ct1 = jobInfo.csc->colType(sc1->oid()); + ct1.charsetNumber = sc1->colType().charsetNumber; + } + if (!sc2->schemaName().empty() && sc2->isColumnStore() && !pc2) + { + ct2 = jobInfo.csc->colType(sc2->oid()); + ct2.charsetNumber = sc2->colType().charsetNumber; + } + // X + uint64_t joinInfo = sc1->joinInfo() | sc2->joinInfo(); + + if (sc1->schemaName().empty()) + { + SimpleColumn* tmp = (sc1); + updateDerivedColumn(jobInfo, tmp, ct1); + } + + if (sc2->schemaName().empty()) + { + SimpleColumn* tmp = (sc2); + updateDerivedColumn(jobInfo, tmp, ct2); + } + + //@bug 566 Dont do a hash join if the table and the alias are the same + // Bug 590 + if (tableOid1 == tableOid2 && alias1 == alias2 && view1 == view2 && joinInfo == 0) + { + if (sc1->schemaName().empty() || !compatibleColumnTypes(ct1, ct2, false)) + { + return doFilterExpression(sc1, sc2, jobInfo, sop); + } + + JobStepVector colFilter = doColFilter(sc1, sc2, jobInfo, sop, sf); + // jsv.insert(jsv.end(), colFilter.begin(), colFilter.end()); + return colFilter; + } + + // different tables + if (!compatibleColumnTypes(ct1, ct2, true)) + { + JobStepVector jsv; + jsv = doFilterExpression(sc1, sc2, jobInfo, sop); + uint32_t t1 = makeTableKey(jobInfo, sc1); + uint32_t t2 = makeTableKey(jobInfo, sc2); + jobInfo.incompatibleJoinMap[t1] = t2; + jobInfo.incompatibleJoinMap[t2] = t1; + + return jsv; + } + + pColStep* pcs1 = NULL; + CalpontSystemCatalog::OID oid1 = sc1->oid(); + CalpontSystemCatalog::OID dictOid1 = isDictCol(ct1); + + if (sc1->schemaName().empty() == false) + { + if (pc1 == NULL) + pcs1 = new pColStep(oid1, tableOid1, ct1, jobInfo); + else + pcs1 = new PseudoColStep(oid1, tableOid1, pc1->pseudoType(), ct1, jobInfo); + + pcs1->alias(alias1); + pcs1->view(view1); + pcs1->name(sc1->columnName()); + pcs1->schema(sc1->schemaName()); + pcs1->cardinality(sc1->cardinality()); + } + + pColStep* pcs2 = NULL; + CalpontSystemCatalog::OID oid2 = sc2->oid(); + CalpontSystemCatalog::OID dictOid2 = isDictCol(ct2); + + if (sc2->schemaName().empty() == false) + { + if (pc2 == NULL) + pcs2 = new pColStep(oid2, tableOid2, ct2, jobInfo); + else + pcs2 = new PseudoColStep(oid2, tableOid2, pc2->pseudoType(), ct2, jobInfo); + + pcs2->alias(alias2); + pcs2->view(view2); + pcs2->name(sc2->columnName()); + pcs2->schema(sc2->schemaName()); + pcs2->cardinality(sc2->cardinality()); + } + + JoinType jt = INNER; + + if (sc1->returnAll() && sc2->returnAll()) + { + // Due to current connector limitation, INNER may have both returnAll set. + // @bug3037, compound outer join may have both flag set if not the 1st pair. + jt = INNER; + } + else if (sc1->returnAll()) + { + jt = LEFTOUTER; + } + else if (sc2->returnAll()) + { + jt = RIGHTOUTER; + } + + // Associate the steps + JobStepVector jsv; + SJSTEP step; + + // bug 1495 compound join, v-table handles string join and compound join the same way + AnyDataListSPtr spdl1(new AnyDataList()); + RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); + spdl1->rowGroupDL(dl1); + dl1->OID(oid1); + + if (pcs1) + { + JobStepAssociation outJs1; + outJs1.outAdd(spdl1); + pcs1->outputAssociation(outJs1); + + step.reset(pcs1); + jsv.push_back(step); + } + + AnyDataListSPtr spdl2(new AnyDataList()); + RowGroupDL* dl2 = new RowGroupDL(1, jobInfo.fifoSize); + spdl2->rowGroupDL(dl2); + dl2->OID(oid2); + + if (pcs2) + { + JobStepAssociation outJs2; + outJs2.outAdd(spdl2); + pcs2->outputAssociation(outJs2); + + step.reset(pcs2); + jsv.push_back(step); + } + + TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); + thj->tableOid1(tableOid1); + thj->tableOid2(tableOid2); + thj->alias1(alias1); + thj->alias2(alias2); + thj->view1(view1); + thj->view2(view2); + thj->schema1(schema1); + thj->schema2(schema2); + thj->oid1(oid1); + thj->oid2(oid2); + thj->dictOid1(dictOid1); + thj->dictOid2(dictOid2); + thj->sequence1(sc1->sequence()); + thj->sequence2(sc2->sequence()); + thj->column1(sc1); + thj->column2(sc2); + + // MCOL-334 joins in views need to have higher priority than SEMI/ANTI + if (!view1.empty() && view1 == view2) + { + thj->joinId(-1); + } + else + { + thj->joinId((joinInfo == 0) ? (++jobInfo.joinNum) : 0); + } + + // Check if SEMI/ANTI join. + // INNER/OUTER join and SEMI/ANTI are mutually exclusive, + if (joinInfo != 0) + { + // @bug3998, keep the OUTER join type + // jt = INIT; + + if (joinInfo & JOIN_SEMI) + jt |= SEMI; + + if (joinInfo & JOIN_ANTI) + jt |= ANTI; + + if (joinInfo & JOIN_SCALAR) + jt |= SCALAR; + + if (joinInfo & JOIN_NULL_MATCH) + jt |= MATCHNULLS; + + if (joinInfo & JOIN_CORRELATED) + jt |= CORRELATED; + + if (joinInfo & JOIN_OUTER_SELECT) + jt |= LARGEOUTER; + + if (sc1->joinInfo() & JOIN_CORRELATED) + thj->correlatedSide(1); + else if (sc2->joinInfo() & JOIN_CORRELATED) + thj->correlatedSide(2); + } + + thj->setJoinType(jt); + + JobStepAssociation outJs3; + outJs3.outAdd(spdl1); + outJs3.outAdd(spdl2); + thj->inputAssociation(outJs3); + step.reset(thj); + + TupleInfo ti1(setTupleInfo(ct1, oid1, jobInfo, tableOid1, sc1, alias1)); + + if (pcs1) + { + pcs1->tupleId(ti1.key); + thj->tupleId1(ti1.key); + + if (dictOid1 > 0) + { + ti1 = setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc1, alias1); + jobInfo.keyInfo->dictOidToColOid[dictOid1] = oid1; + jobInfo.keyInfo->dictKeyMap[pcs1->tupleId()] = ti1.key; + jobInfo.tokenOnly[pcs1->tupleId()] = false; + // thj->tupleId1(ti1.key); + } + } + else + { + thj->tupleId1(getTupleKey(jobInfo, sc1)); + } + + TupleInfo ti2(setTupleInfo(ct2, oid2, jobInfo, tableOid2, sc2, alias2)); + + if (pcs2) + { + pcs2->tupleId(ti2.key); + thj->tupleId2(pcs2->tupleId()); + + if (dictOid2 > 0) + { + TupleInfo ti2(setTupleInfo(ct2, dictOid2, jobInfo, tableOid2, sc2, alias2)); + jobInfo.keyInfo->dictOidToColOid[dictOid2] = oid2; + jobInfo.keyInfo->dictKeyMap[pcs2->tupleId()] = ti2.key; + jobInfo.tokenOnly[pcs2->tupleId()] = false; + // thj->tupleId2(ti2.key); + } + } + else + { + thj->tupleId2(getTupleKey(jobInfo, sc2)); + } + + jsv.push_back(step); + + return jsv; +} const JobStepVector doSemiJoin(const SimpleColumn* sc, const ReturnedColumn* rc, JobInfo& jobInfo) { - if ((sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || - sc->resultType().colDataType == CalpontSystemCatalog::BLOB)) - throw runtime_error("VARBINARY/BLOB in join is not supported."); + if ((sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + sc->resultType().colDataType == CalpontSystemCatalog::BLOB)) + throw runtime_error("VARBINARY/BLOB in join is not supported."); - CalpontSystemCatalog::OID tableOid1 = tableOid(sc, jobInfo.csc); - CalpontSystemCatalog::OID tableOid2 = execplan::CNX_VTABLE_ID; - string alias1(extractTableAlias(sc)); - string alias2(jobInfo.subAlias); - CalpontSystemCatalog::ColType ct1 = sc->colType(); - CalpontSystemCatalog::ColType ct2 = rc->resultType(); - const PseudoColumn* pc1 = dynamic_cast(sc); + CalpontSystemCatalog::OID tableOid1 = tableOid(sc, jobInfo.csc); + CalpontSystemCatalog::OID tableOid2 = execplan::CNX_VTABLE_ID; + string alias1(extractTableAlias(sc)); + string alias2(jobInfo.subAlias); + CalpontSystemCatalog::ColType ct1 = sc->colType(); + CalpontSystemCatalog::ColType ct2 = rc->resultType(); + const PseudoColumn* pc1 = dynamic_cast(sc); -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (!sc->schemaName().empty() && sc->isColumnStore() && !pc1) - { - ct1 = jobInfo.csc->colType(sc->oid()); - ct1.charsetNumber =sc->colType().charsetNumber; - } -//X - JobStepVector jsv; - SJSTEP step; + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (!sc->schemaName().empty() && sc->isColumnStore() && !pc1) + { + ct1 = jobInfo.csc->colType(sc->oid()); + ct1.charsetNumber = sc->colType().charsetNumber; + } + // X + JobStepVector jsv; + SJSTEP step; - CalpontSystemCatalog::OID dictOid1 = 0; - uint64_t tupleId1 = -1; - uint64_t tupleId2 = -1; + CalpontSystemCatalog::OID dictOid1 = 0; + uint64_t tupleId1 = -1; + uint64_t tupleId2 = -1; - if (sc->schemaName().empty() == false) - { - pColStep* pcs1 = NULL; + if (sc->schemaName().empty() == false) + { + pColStep* pcs1 = NULL; - if (pc1 == NULL) - pcs1 = new pColStep(sc->oid(), tableOid1, ct1, jobInfo); - else - pcs1 = new PseudoColStep(sc->oid(), tableOid1, pc1->pseudoType(), ct1, jobInfo); + if (pc1 == NULL) + pcs1 = new pColStep(sc->oid(), tableOid1, ct1, jobInfo); + else + pcs1 = new PseudoColStep(sc->oid(), tableOid1, pc1->pseudoType(), ct1, jobInfo); - dictOid1 = isDictCol(ct1); - pcs1->alias(alias1); - pcs1->view(sc->viewName()); - pcs1->name(sc->columnName()); - pcs1->schema(sc->schemaName()); - pcs1->cardinality(sc->cardinality()); + dictOid1 = isDictCol(ct1); + pcs1->alias(alias1); + pcs1->view(sc->viewName()); + pcs1->name(sc->columnName()); + pcs1->schema(sc->schemaName()); + pcs1->cardinality(sc->cardinality()); - step.reset(pcs1); - jsv.push_back(step); - - TupleInfo ti1(setTupleInfo(ct1, sc->oid(), jobInfo, tableOid1, sc, alias1)); - pcs1->tupleId(ti1.key); - tupleId1 = ti1.key; - - if (dictOid1 > 0) - { - ti1 = setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc, alias1); - jobInfo.keyInfo->dictOidToColOid[dictOid1] = sc->oid(); - jobInfo.keyInfo->dictKeyMap[tupleId1] = ti1.key; - jobInfo.tokenOnly[pcs1->tupleId()] = false; - } - } - - TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); - thj->tableOid1(tableOid1); - thj->tableOid2(tableOid2); - thj->alias1(alias1); - thj->view1(sc->viewName()); - thj->schema1(sc->schemaName()); - thj->oid1(sc->oid()); - thj->oid2(tableOid2 + 1 + rc->sequence()); - thj->alias2(alias2); - thj->dictOid1(dictOid1); - thj->dictOid2(0); - thj->sequence1(sc->sequence()); - thj->sequence2(rc->sequence()); - thj->column1(sc); - thj->column2(rc); - thj->joinId(0); - thj->tupleId1(tupleId1); - thj->tupleId2(tupleId2); - - // set join type - JoinType jt = INIT; - uint64_t joinInfo = sc->joinInfo(); - - if (joinInfo & JOIN_SEMI) - jt |= SEMI; - - if (joinInfo & JOIN_ANTI) - jt |= ANTI; - - if (joinInfo & JOIN_SCALAR) - jt |= SCALAR; - - if (joinInfo & JOIN_NULL_MATCH) - jt |= MATCHNULLS; - - if (joinInfo & JOIN_CORRELATED) - jt |= CORRELATED; - - if (joinInfo & JOIN_OUTER_SELECT) - jt |= LARGEOUTER; - - thj->setJoinType(jt); - - // set correlated side - thj->correlatedSide(1); - - step.reset(thj); + step.reset(pcs1); jsv.push_back(step); - return jsv; -} + TupleInfo ti1(setTupleInfo(ct1, sc->oid(), jobInfo, tableOid1, sc, alias1)); + pcs1->tupleId(ti1.key); + tupleId1 = ti1.key; + + if (dictOid1 > 0) + { + ti1 = setTupleInfo(ct1, dictOid1, jobInfo, tableOid1, sc, alias1); + jobInfo.keyInfo->dictOidToColOid[dictOid1] = sc->oid(); + jobInfo.keyInfo->dictKeyMap[tupleId1] = ti1.key; + jobInfo.tokenOnly[pcs1->tupleId()] = false; + } + } + + TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); + thj->tableOid1(tableOid1); + thj->tableOid2(tableOid2); + thj->alias1(alias1); + thj->view1(sc->viewName()); + thj->schema1(sc->schemaName()); + thj->oid1(sc->oid()); + thj->oid2(tableOid2 + 1 + rc->sequence()); + thj->alias2(alias2); + thj->dictOid1(dictOid1); + thj->dictOid2(0); + thj->sequence1(sc->sequence()); + thj->sequence2(rc->sequence()); + thj->column1(sc); + thj->column2(rc); + thj->joinId(0); + thj->tupleId1(tupleId1); + thj->tupleId2(tupleId2); + + // set join type + JoinType jt = INIT; + uint64_t joinInfo = sc->joinInfo(); + + if (joinInfo & JOIN_SEMI) + jt |= SEMI; + + if (joinInfo & JOIN_ANTI) + jt |= ANTI; + + if (joinInfo & JOIN_SCALAR) + jt |= SCALAR; + + if (joinInfo & JOIN_NULL_MATCH) + jt |= MATCHNULLS; + + if (joinInfo & JOIN_CORRELATED) + jt |= CORRELATED; + + if (joinInfo & JOIN_OUTER_SELECT) + jt |= LARGEOUTER; + + thj->setJoinType(jt); + + // set correlated side + thj->correlatedSide(1); + + step.reset(thj); + jsv.push_back(step); + return jsv; +} SJSTEP expressionToFuncJoin(ExpressionStep* es, JobInfo& jobInfo) { - idbassert(es); - boost::shared_ptr fji = es->functionJoinInfo(); - es->functionJoin(true); - TupleHashJoinStep* thjs = new TupleHashJoinStep(jobInfo); - thjs->tableOid1(fji->fTableOid[0]); - thjs->tableOid2(fji->fTableOid[1]); - thjs->oid1(fji->fOid[0]); - thjs->oid2(fji->fOid[1]); - thjs->alias1(fji->fAlias[0]); - thjs->alias2(fji->fAlias[1]); - thjs->view1(fji->fView[0]); - thjs->view2(fji->fView[1]); - thjs->schema1(fji->fSchema[0]); - thjs->schema2(fji->fSchema[1]); - thjs->column1(fji->fExpression[0]); - thjs->column2(fji->fExpression[1]); - thjs->sequence1(fji->fSequence[0]); - thjs->sequence2(fji->fSequence[1]); - thjs->joinId(fji->fJoinId); - thjs->setJoinType(fji->fJoinType); - thjs->correlatedSide(fji->fCorrelatedSide); - thjs->funcJoinInfo(fji); - thjs->tupleId1(fji->fJoinKey[0]); - thjs->tupleId2(fji->fJoinKey[1]); + idbassert(es); + boost::shared_ptr fji = es->functionJoinInfo(); + es->functionJoin(true); + TupleHashJoinStep* thjs = new TupleHashJoinStep(jobInfo); + thjs->tableOid1(fji->fTableOid[0]); + thjs->tableOid2(fji->fTableOid[1]); + thjs->oid1(fji->fOid[0]); + thjs->oid2(fji->fOid[1]); + thjs->alias1(fji->fAlias[0]); + thjs->alias2(fji->fAlias[1]); + thjs->view1(fji->fView[0]); + thjs->view2(fji->fView[1]); + thjs->schema1(fji->fSchema[0]); + thjs->schema2(fji->fSchema[1]); + thjs->column1(fji->fExpression[0]); + thjs->column2(fji->fExpression[1]); + thjs->sequence1(fji->fSequence[0]); + thjs->sequence2(fji->fSequence[1]); + thjs->joinId(fji->fJoinId); + thjs->setJoinType(fji->fJoinType); + thjs->correlatedSide(fji->fCorrelatedSide); + thjs->funcJoinInfo(fji); + thjs->tupleId1(fji->fJoinKey[0]); + thjs->tupleId2(fji->fJoinKey[1]); - updateTableKey(fji->fJoinKey[0], fji->fTableKey[0], jobInfo); - updateTableKey(fji->fJoinKey[1], fji->fTableKey[1], jobInfo); + updateTableKey(fji->fJoinKey[0], fji->fTableKey[0], jobInfo); + updateTableKey(fji->fJoinKey[1], fji->fTableKey[1], jobInfo); - return SJSTEP(thjs); + return SJSTEP(thjs); } - const JobStepVector doExpressionFilter(const ParseTree* n, JobInfo& jobInfo) { - JobStepVector jsv; - ExpressionStep* es = new ExpressionStep(jobInfo); + JobStepVector jsv; + ExpressionStep* es = new ExpressionStep(jobInfo); - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 1"); + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 1"); - es->expressionFilter(n, jobInfo); - SJSTEP sjstep(es); - jsv.push_back(sjstep); + es->expressionFilter(n, jobInfo); + SJSTEP sjstep(es); + jsv.push_back(sjstep); - return jsv; + return jsv; } - const JobStepVector doExpressionFilter(const Filter* f, JobInfo& jobInfo) { - JobStepVector jsv; - ExpressionStep* es = new ExpressionStep(jobInfo); + JobStepVector jsv; + ExpressionStep* es = new ExpressionStep(jobInfo); - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 2"); + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 2"); - es->expressionFilter(f, jobInfo); - SJSTEP sjstep(es); - jsv.push_back(sjstep); + es->expressionFilter(f, jobInfo); + SJSTEP sjstep(es); + jsv.push_back(sjstep); - // @bug3683, support in/exists suquery function join - const SimpleFilter* sf = dynamic_cast(f); + // @bug3683, support in/exists suquery function join + const SimpleFilter* sf = dynamic_cast(f); - if (sf != NULL) + if (sf != NULL) + { + const ReturnedColumn* lrc = static_cast(sf->lhs()); + const ReturnedColumn* rrc = static_cast(sf->rhs()); + + if (lrc->joinInfo()) { - const ReturnedColumn* lrc = static_cast(sf->lhs()); - const ReturnedColumn* rrc = static_cast(sf->rhs()); + const ReturnedColumn* lac = dynamic_cast(lrc); + const ReturnedColumn* lfc = dynamic_cast(lrc); + const ReturnedColumn* lsc = dynamic_cast(lrc); - if (lrc->joinInfo()) - { - const ReturnedColumn* lac = dynamic_cast(lrc); - const ReturnedColumn* lfc = dynamic_cast(lrc); - const ReturnedColumn* lsc = dynamic_cast(lrc); - - if ((lac || lfc || lsc) && es->functionJoinInfo()) - jsv.push_back(expressionToFuncJoin(es, jobInfo)); - } - else if (rrc->joinInfo()) - { - const ReturnedColumn* rac = dynamic_cast(lrc); - const ReturnedColumn* rfc = dynamic_cast(lrc); - const ReturnedColumn* rsc = dynamic_cast(lrc); - - if ((rac || rfc || rsc) && es->functionJoinInfo()) - jsv.push_back(expressionToFuncJoin(es, jobInfo)); - } + if ((lac || lfc || lsc) && es->functionJoinInfo()) + jsv.push_back(expressionToFuncJoin(es, jobInfo)); } + else if (rrc->joinInfo()) + { + const ReturnedColumn* rac = dynamic_cast(lrc); + const ReturnedColumn* rfc = dynamic_cast(lrc); + const ReturnedColumn* rsc = dynamic_cast(lrc); - return jsv; + if ((rac || rfc || rsc) && es->functionJoinInfo()) + jsv.push_back(expressionToFuncJoin(es, jobInfo)); + } + } + + return jsv; } - const JobStepVector doConstantBooleanFilter(const ParseTree* n, JobInfo& jobInfo) { - JobStepVector jsv; - TupleConstantBooleanStep* tcbs = new - TupleConstantBooleanStep(jobInfo, n->data()->getBoolVal()); + JobStepVector jsv; + TupleConstantBooleanStep* tcbs = new TupleConstantBooleanStep(jobInfo, n->data()->getBoolVal()); - if (tcbs == NULL) - throw runtime_error("Failed to create Constant Boolean Step"); + if (tcbs == NULL) + throw runtime_error("Failed to create Constant Boolean Step"); - SJSTEP sjstep(tcbs); - jsv.push_back(sjstep); + SJSTEP sjstep(tcbs); + jsv.push_back(sjstep); - return jsv; + return jsv; } - bool optimizeIdbPatitionSimpleFilter(SimpleFilter* sf, JobStepVector& jsv, JobInfo& jobInfo) { - //@bug5848, not equal filter is not optimized. - if (sf->op()->op() != opeq.op()) - return false; + //@bug5848, not equal filter is not optimized. + if (sf->op()->op() != opeq.op()) + return false; - const FunctionColumn* fc = static_cast(sf->lhs()); - const ConstantColumn* cc = static_cast(sf->rhs()); + const FunctionColumn* fc = static_cast(sf->lhs()); + const ConstantColumn* cc = static_cast(sf->rhs()); - if (fc == NULL) - { - cc = static_cast(sf->lhs()); - fc = static_cast(sf->rhs()); - } + if (fc == NULL) + { + cc = static_cast(sf->lhs()); + fc = static_cast(sf->rhs()); + } - // not a function or not idbparttition - if (fc == NULL || cc == NULL || fc->functionName().compare("idbpartition") != 0) - return false; + // not a function or not idbparttition + if (fc == NULL || cc == NULL || fc->functionName().compare("idbpartition") != 0) + return false; - // make sure the cc has 3 tokens - vector cv; - boost::split(cv, cc->constval(), boost::is_any_of(".")); + // make sure the cc has 3 tokens + vector cv; + boost::split(cv, cc->constval(), boost::is_any_of(".")); - if (cv.size() != 3) - return false; + if (cv.size() != 3) + return false; - // construct separate filters, then make job steps - JobStepVector v; - SOP sop = sf->op(); - const funcexp::FunctionParm& parms = fc->functionParms(); + // construct separate filters, then make job steps + JobStepVector v; + SOP sop = sf->op(); + const funcexp::FunctionParm& parms = fc->functionParms(); - for (uint64_t i = 0; i < 3; i++) - { - // very weird parms order is: db root, physical partition, segment - // logical partition string : physical partition, segment, db root - ReturnedColumn* lhs = dynamic_cast(parms[(i + 1) % 3]->data()); - ConstantColumn* rhs = new ConstantColumn(cv[i]); - SimpleFilter* f = new SimpleFilter(sop, lhs->clone(), rhs); - v = doSimpleFilter(f, jobInfo); - jsv.insert(jsv.end(), v.begin(), v.end()); - } + for (uint64_t i = 0; i < 3; i++) + { + // very weird parms order is: db root, physical partition, segment + // logical partition string : physical partition, segment, db root + ReturnedColumn* lhs = dynamic_cast(parms[(i + 1) % 3]->data()); + ConstantColumn* rhs = new ConstantColumn(cv[i]); + SimpleFilter* f = new SimpleFilter(sop, lhs->clone(), rhs); + v = doSimpleFilter(f, jobInfo); + jsv.insert(jsv.end(), v.begin(), v.end()); + } - return true; + return true; } const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { - JobStepVector jsv; + JobStepVector jsv; - if (sf == 0) return jsv; + if (sf == 0) + return jsv; - //cout << "doSimpleFilter " << endl; - //cout << *sf << endl; - ReturnedColumn* lhs; - ReturnedColumn* rhs; - SOP sop; + // cout << "doSimpleFilter " << endl; + // cout << *sf << endl; + ReturnedColumn* lhs; + ReturnedColumn* rhs; + SOP sop; - lhs = sf->lhs(); - rhs = sf->rhs(); - sop = sf->op(); + lhs = sf->lhs(); + rhs = sf->rhs(); + sop = sf->op(); - TreeNodeType lhsType; - TreeNodeType rhsType; + TreeNodeType lhsType; + TreeNodeType rhsType; - lhsType = TreeNode2Type(lhs); - rhsType = TreeNode2Type(rhs); + lhsType = TreeNode2Type(lhs); + rhsType = TreeNode2Type(rhs); - CalpontSystemCatalog::OID tbl_oid = 0; + CalpontSystemCatalog::OID tbl_oid = 0; - SJSTEP sjstep; + SJSTEP sjstep; - if (lhsType == SIMPLECOLUMN && rhsType == CONSTANTCOLUMN ) + if (lhsType == SIMPLECOLUMN && rhsType == CONSTANTCOLUMN) + { + int8_t cop = op2num(sop); + const SimpleColumn* sc = static_cast(lhs); + const ConstantColumn* cc = static_cast(rhs); + string alias(extractTableAlias(sc)); + string view(sc->viewName()); + string schema(sc->schemaName()); + tbl_oid = tableOid(sc, jobInfo.csc); + + if (sc->joinInfo() != 0 && cc->sequence() != -1) { - int8_t cop = op2num(sop); - const SimpleColumn* sc = static_cast(lhs); - const ConstantColumn* cc = static_cast(rhs); - string alias(extractTableAlias(sc)); - string view(sc->viewName()); - string schema(sc->schemaName()); - tbl_oid = tableOid(sc, jobInfo.csc); + // correlated, like in 'c1 in select 1' type sub queries. + return doSemiJoin(sc, cc, jobInfo); + } + else if (sc->schemaName().empty()) + { + // bug 3749, mark outer join table with isNull filter + if (ConstantColumn::NULLDATA == cc->type() && (opis == *sop || opisnull == *sop)) + jobInfo.tableHasIsNull.insert(getTableKey(jobInfo, tbl_oid, alias, "", view)); - if (sc->joinInfo() != 0 && cc->sequence() != -1) - { - // correlated, like in 'c1 in select 1' type sub queries. - return doSemiJoin(sc, cc, jobInfo); - } - else if (sc->schemaName().empty()) - { - // bug 3749, mark outer join table with isNull filter - if (ConstantColumn::NULLDATA == cc->type() && (opis == *sop || opisnull == *sop)) - jobInfo.tableHasIsNull.insert(getTableKey(jobInfo, tbl_oid, alias, "", view)); + return doExpressionFilter(sf, jobInfo); + } - return doExpressionFilter(sf, jobInfo); - } + string constval(cc->constval()); - string constval(cc->constval()); + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct = sc->colType(); + const PseudoColumn* pc = dynamic_cast(sc); - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct = sc->colType(); - const PseudoColumn* pc = dynamic_cast(sc); + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) + { + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + // X -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } -//X + //@bug 339 nulls are not stored in dictionary + if ((dictOid = isDictCol(ct)) > 0 && ConstantColumn::NULLDATA != cc->type()) + { + if (jobInfo.trace) + cout << "Emit pTokenByScan/pCol for SimpleColumn op ConstantColumn" << endl; - //@bug 339 nulls are not stored in dictionary - if ((dictOid = isDictCol(ct)) > 0 && ConstantColumn::NULLDATA != cc->type()) - { - if (jobInfo.trace) - cout << "Emit pTokenByScan/pCol for SimpleColumn op ConstantColumn" << endl; + // dictionary, cannot be pseudo column + pColStep* pcs = new pColStep(sc->oid(), tbl_oid, ct, jobInfo); + pcs->alias(alias); + pcs->view(view); + pcs->name(sc->columnName()); + pcs->schema(sc->schemaName()); + pcs->cardinality(sc->cardinality()); - // dictionary, cannot be pseudo column - pColStep* pcs = new pColStep(sc->oid(), tbl_oid, ct, jobInfo); - pcs->alias(alias); - pcs->view(view); - pcs->name(sc->columnName()); - pcs->schema(sc->schemaName()); - pcs->cardinality(sc->cardinality()); + if (ct.colDataType == execplan::CalpontSystemCatalog::TEXT || + ct.colDataType == execplan::CalpontSystemCatalog::BLOB || + filterWithDictionary(dictOid, jobInfo.stringScanThreshold)) + { + pDictionaryStep* pds = new pDictionaryStep(dictOid, tbl_oid, ct, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); + pds->alias(alias); + pds->view(view); + pds->name(sc->columnName()); + pds->schema(sc->schemaName()); + pds->cardinality(sc->cardinality()); - if (ct.colDataType == execplan::CalpontSystemCatalog::TEXT || - ct.colDataType == execplan::CalpontSystemCatalog::BLOB || - filterWithDictionary(dictOid, jobInfo.stringScanThreshold)) - { - pDictionaryStep* pds = new pDictionaryStep(dictOid, tbl_oid, ct, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); - pds->alias(alias); - pds->view(view); - pds->name(sc->columnName()); - pds->schema(sc->schemaName()); - pds->cardinality(sc->cardinality()); + // Add the filter + pds->addFilter(cop, constval); - //Add the filter - pds->addFilter(cop, constval); + // data list for pcolstep output + AnyDataListSPtr spdl1(new AnyDataList()); + FifoDataList* dl1 = new FifoDataList(1, jobInfo.fifoSize); + spdl1->fifoDL(dl1); + dl1->OID(sc->oid()); - // data list for pcolstep output - AnyDataListSPtr spdl1(new AnyDataList()); - FifoDataList* dl1 = new FifoDataList(1, jobInfo.fifoSize); - spdl1->fifoDL(dl1); - dl1->OID(sc->oid()); + JobStepAssociation outJs1; + outJs1.outAdd(spdl1); + pcs->outputAssociation(outJs1); - JobStepAssociation outJs1; - outJs1.outAdd(spdl1); - pcs->outputAssociation(outJs1); + // data list for pdictionarystep output + AnyDataListSPtr spdl2(new AnyDataList()); + StringFifoDataList* dl2 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl2->stringDL(dl2); + dl2->OID(sc->oid()); - // data list for pdictionarystep output - AnyDataListSPtr spdl2(new AnyDataList()); - StringFifoDataList* dl2 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl2->stringDL(dl2); - dl2->OID(sc->oid()); + JobStepAssociation outJs2; + outJs2.outAdd(spdl2); + pds->outputAssociation(outJs2); - JobStepAssociation outJs2; - outJs2.outAdd(spdl2); - pds->outputAssociation(outJs2); + // Associate pcs with pds + JobStepAssociation outJs; + outJs.outAdd(spdl1); + pds->inputAssociation(outJs); - //Associate pcs with pds - JobStepAssociation outJs; - outJs.outAdd(spdl1); - pds->inputAssociation(outJs); + sjstep.reset(pcs); + jsv.push_back(sjstep); + sjstep.reset(pds); + jsv.push_back(sjstep); - sjstep.reset(pcs); - jsv.push_back(sjstep); - sjstep.reset(pds); - jsv.push_back(sjstep); + // save for expression transformation + pds->addFilter(sf); - // save for expression transformation - pds->addFilter(sf); + // token column + CalpontSystemCatalog::ColType tct; + tct.colDataType = CalpontSystemCatalog::BIGINT; + tct.colWidth = 8; + tct.scale = 0; + tct.precision = 0; + tct.compressionType = ct.compressionType; + TupleInfo ti(setTupleInfo(tct, sc->oid(), jobInfo, tbl_oid, sc, alias)); + jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; + pcs->tupleId(ti.key); - // token column - CalpontSystemCatalog::ColType tct; - tct.colDataType = CalpontSystemCatalog::BIGINT; - tct.colWidth = 8; - tct.scale = 0; - tct.precision = 0; - tct.compressionType = ct.compressionType; - TupleInfo ti(setTupleInfo(tct, sc->oid(), jobInfo, tbl_oid, sc, alias)); - jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; - pcs->tupleId(ti.key); + // string column + ti = setTupleInfo(ct, dictOid, jobInfo, tbl_oid, sc, alias); + pds->tupleId(ti.key); + jobInfo.keyInfo->dictKeyMap[pcs->tupleId()] = ti.key; + jobInfo.tokenOnly[ti.key] = false; + } + else + { + pDictionaryScan* pds = new pDictionaryScan(dictOid, tbl_oid, ct, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); + pds->alias(alias); + pds->view(view); + pds->name(sc->columnName()); + pds->schema(sc->schemaName()); + pds->cardinality(sc->cardinality()); - // string column - ti = setTupleInfo(ct, dictOid, jobInfo, tbl_oid, sc, alias); - pds->tupleId(ti.key); - jobInfo.keyInfo->dictKeyMap[pcs->tupleId()] = ti.key; - jobInfo.tokenOnly[ti.key] = false; - } - else - { - pDictionaryScan* pds = new pDictionaryScan(dictOid, tbl_oid, ct, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); - pds->alias(alias); - pds->view(view); - pds->name(sc->columnName()); - pds->schema(sc->schemaName()); - pds->cardinality(sc->cardinality()); + // Add the filter + pds->addFilter(cop, constval); - //Add the filter - pds->addFilter(cop, constval); + // save for expression transformation + pds->addFilter(sf); - // save for expression transformation - pds->addFilter(sf); + TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); + thj->tableOid1(0); + thj->tableOid2(tbl_oid); + thj->alias1(alias); + thj->alias2(alias); + thj->view1(view); + thj->view2(view); + thj->schema1(schema); + thj->schema2(schema); + thj->oid1(sc->oid()); + thj->oid2(sc->oid()); + thj->joinId(0); + thj->setJoinType(INNER); - TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); - thj->tableOid1(0); - thj->tableOid2(tbl_oid); - thj->alias1(alias); - thj->alias2(alias); - thj->view1(view); - thj->view2(view); - thj->schema1(schema); - thj->schema2(schema); - thj->oid1(sc->oid()); - thj->oid2(sc->oid()); - thj->joinId(0); - thj->setJoinType(INNER); + CalpontSystemCatalog::ColType dct; + dct.colDataType = CalpontSystemCatalog::BIGINT; + dct.colWidth = 8; + dct.scale = 0; + dct.precision = 0; + dct.compressionType = ct.compressionType; - CalpontSystemCatalog::ColType dct; - dct.colDataType = CalpontSystemCatalog::BIGINT; - dct.colWidth = 8; - dct.scale = 0; - dct.precision = 0; - dct.compressionType = ct.compressionType; + TupleInfo ti(setTupleInfo(dct, sc->oid(), jobInfo, tbl_oid, sc, alias)); + jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; + pds->tupleId(ti.key); // pcs, pds use same tuple key, both 8-byte column + pcs->tupleId(ti.key); + thj->tupleId1(ti.key); + thj->tupleId2(ti.key); - TupleInfo ti(setTupleInfo(dct, sc->oid(), jobInfo, tbl_oid, sc, alias)); - jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; - pds->tupleId(ti.key); // pcs, pds use same tuple key, both 8-byte column - pcs->tupleId(ti.key); - thj->tupleId1(ti.key); - thj->tupleId2(ti.key); + if (jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) + jobInfo.tokenOnly[ti.key] = true; - if (jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) - jobInfo.tokenOnly[ti.key] = true; + // Associate the steps + AnyDataListSPtr spdl1(new AnyDataList()); - //Associate the steps - AnyDataListSPtr spdl1(new AnyDataList()); + RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); + spdl1->rowGroupDL(dl1); + dl1->OID(dictOid); - RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); - spdl1->rowGroupDL(dl1); - dl1->OID(dictOid); + JobStepAssociation outJs1; + outJs1.outAdd(spdl1); + pds->outputAssociation(outJs1); - JobStepAssociation outJs1; - outJs1.outAdd(spdl1); - pds->outputAssociation(outJs1); + AnyDataListSPtr spdl2(new AnyDataList()); - AnyDataListSPtr spdl2(new AnyDataList()); + RowGroupDL* dl2 = new RowGroupDL(1, jobInfo.fifoSize); + spdl2->rowGroupDL(dl2); + dl2->OID(sc->oid()); - RowGroupDL* dl2 = new RowGroupDL(1, jobInfo.fifoSize); - spdl2->rowGroupDL(dl2); - dl2->OID(sc->oid()); + JobStepAssociation outJs2; + outJs2.outAdd(spdl2); + pcs->outputAssociation(outJs2); - JobStepAssociation outJs2; - outJs2.outAdd(spdl2); - pcs->outputAssociation(outJs2); + JobStepAssociation outJs3; + outJs3.outAdd(spdl1); + outJs3.outAdd(spdl2); + sjstep.reset(pds); + jsv.push_back(sjstep); + sjstep.reset(pcs); + jsv.push_back(sjstep); - JobStepAssociation outJs3; - outJs3.outAdd(spdl1); - outJs3.outAdd(spdl2); - sjstep.reset(pds); - jsv.push_back(sjstep); - sjstep.reset(pcs); - jsv.push_back(sjstep); - - thj->inputAssociation(outJs3); - sjstep.reset(thj); - jsv.push_back(sjstep); - } - } - else if ( ConstantColumn::NULLDATA != cc->type() && - (cop & COMPARE_LIKE) ) // both like and not like - { - return doExpressionFilter(sf, jobInfo); - } - else - { - // @bug 1151 string longer than colwidth of char/varchar. - int64_t value = 0; - int128_t value128 = 0; - uint8_t rf = 0; + thj->inputAssociation(outJs3); + sjstep.reset(thj); + jsv.push_back(sjstep); + } + } + else if (ConstantColumn::NULLDATA != cc->type() && (cop & COMPARE_LIKE)) // both like and not like + { + return doExpressionFilter(sf, jobInfo); + } + else + { + // @bug 1151 string longer than colwidth of char/varchar. + int64_t value = 0; + int128_t value128 = 0; + uint8_t rf = 0; #ifdef FAILED_ATOI_IS_ZERO - //if cvn throws (because there's non-digit data in the string, treat that as zero rather than - // throwing - try - { - bool isNull = ConstantColumn::NULLDATA == cc->type(); - convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); + // if cvn throws (because there's non-digit data in the string, treat that as zero rather than + // throwing + try + { + bool isNull = ConstantColumn::NULLDATA == cc->type(); + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); - if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) - { - float f = cc->getFloatVal(); - value = *(reinterpret_cast(&f)); - } - else if (ct.colDataType == CalpontSystemCatalog::DOUBLE && !isNull) - { - double d = cc->getDoubleVal(); - value = *(reinterpret_cast(&d)); - } - } - catch (...) - { - switch (ct.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - value = 0; - rf = 0; - break; + if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) + { + float f = cc->getFloatVal(); + value = *(reinterpret_cast(&f)); + } + else if (ct.colDataType == CalpontSystemCatalog::DOUBLE && !isNull) + { + double d = cc->getDoubleVal(); + value = *(reinterpret_cast(&d)); + } + } + catch (...) + { + switch (ct.colDataType) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + value = 0; + rf = 0; + break; - default: - throw; - break; - } - } + default: throw; break; + } + } #else - bool isNull = ConstantColumn::NULLDATA == cc->type(); + bool isNull = ConstantColumn::NULLDATA == cc->type(); - if (ct.isWideDecimalType()) - convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); - else - convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); + if (ct.isWideDecimalType()) + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); + else + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); - if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) - { - float f = cc->getFloatVal(); - value = *(reinterpret_cast(&f)); - } - else if (ct.colDataType == CalpontSystemCatalog::DOUBLE && !isNull) - { - double d = cc->getDoubleVal(); - value = *(reinterpret_cast(&d)); - } + if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) + { + float f = cc->getFloatVal(); + value = *(reinterpret_cast(&f)); + } + else if (ct.colDataType == CalpontSystemCatalog::DOUBLE && !isNull) + { + double d = cc->getDoubleVal(); + value = *(reinterpret_cast(&d)); + } #endif - // @bug 2584, make "= null" to COMPARE_NIL. - if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) - cop = COMPARE_NIL; + // @bug 2584, make "= null" to COMPARE_NIL. + if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) + cop = COMPARE_NIL; - if (jobInfo.trace) - cout << "doSimpleFilter Emit pCol for SimpleColumn op ConstantColumn = " << value << - " (" << cc->constval() << ')' << endl; + if (jobInfo.trace) + cout << "doSimpleFilter Emit pCol for SimpleColumn op ConstantColumn = " << value << " (" + << cc->constval() << ')' << endl; - if (sf->indexFlag() == 0) - { - pColStep* pcs = NULL; + if (sf->indexFlag() == 0) + { + pColStep* pcs = NULL; - if (pc == NULL) - pcs = new pColStep(sc->oid(), tbl_oid, ct, jobInfo); - else - pcs = new PseudoColStep(sc->oid(), tbl_oid, pc->pseudoType(), ct, jobInfo); + if (pc == NULL) + pcs = new pColStep(sc->oid(), tbl_oid, ct, jobInfo); + else + pcs = new PseudoColStep(sc->oid(), tbl_oid, pc->pseudoType(), ct, jobInfo); - if (sc->isColumnStore()) - { - if (ct.isWideDecimalType()) - pcs->addFilter(cop, value128, rf); - else - pcs->addFilter(cop, value, rf); - } + if (sc->isColumnStore()) + { + if (ct.isWideDecimalType()) + pcs->addFilter(cop, value128, rf); + else + pcs->addFilter(cop, value, rf); + } - pcs->alias(alias); - pcs->view(view); - pcs->name(sc->columnName()); - pcs->schema(sc->schemaName()); - pcs->cardinality(sf->cardinality()); + pcs->alias(alias); + pcs->view(view); + pcs->name(sc->columnName()); + pcs->schema(sc->schemaName()); + pcs->cardinality(sf->cardinality()); - sjstep.reset(pcs); - jsv.push_back(sjstep); + sjstep.reset(pcs); + jsv.push_back(sjstep); - // save for expression transformation - pcs->addFilter(sf); + // save for expression transformation + pcs->addFilter(sf); - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tbl_oid, sc, alias)); - pcs->tupleId(ti.key); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tbl_oid, sc, alias)); + pcs->tupleId(ti.key); - if (dictOid > 0) // cc->type() == ConstantColumn::NULLDATA - { - if (jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) - jobInfo.tokenOnly[ti.key] = true; - } + if (dictOid > 0) // cc->type() == ConstantColumn::NULLDATA + { + if (jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) + jobInfo.tokenOnly[ti.key] = true; + } - if (ConstantColumn::NULLDATA == cc->type() && - (opis == *sop || opisnull == *sop)) - jobInfo.tableHasIsNull.insert( - getTableKey(jobInfo, tbl_oid, alias, sc->schemaName(), view)); - } - else - { - throw runtime_error("Uses indexes?"); + if (ConstantColumn::NULLDATA == cc->type() && (opis == *sop || opisnull == *sop)) + jobInfo.tableHasIsNull.insert(getTableKey(jobInfo, tbl_oid, alias, sc->schemaName(), view)); + } + else + { + throw runtime_error("Uses indexes?"); #if 0 CalpontSystemCatalog::IndexName indexName; indexName.schema = sc->schemaName(); @@ -2011,574 +1917,566 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) sjstep.reset(pidxL); jsv.push_back(sjstep); #endif - } - } + } } - else if (lhsType == SIMPLECOLUMN && rhsType == SIMPLECOLUMN) + } + else if (lhsType == SIMPLECOLUMN && rhsType == SIMPLECOLUMN) + { + // This is a join (different table) or filter step (same table) + SimpleColumn* sc1 = static_cast(lhs); + SimpleColumn* sc2 = static_cast(rhs); + + // @bug 1496. handle non equal operator as expression in v-table mode + if ((sc1->tableName() != sc2->tableName() || sc1->tableAlias() != sc2->tableAlias() || + sc1->viewName() != sc2->viewName()) && + (sop->data() != "=")) { - //This is a join (different table) or filter step (same table) - SimpleColumn* sc1 = static_cast(lhs); - SimpleColumn* sc2 = static_cast(rhs); + return doExpressionFilter(sf, jobInfo); + } - // @bug 1496. handle non equal operator as expression in v-table mode - if ((sc1->tableName() != sc2->tableName() || - sc1->tableAlias() != sc2->tableAlias() || - sc1->viewName() != sc2->viewName()) - && (sop->data() != "=")) - { - return doExpressionFilter(sf, jobInfo); - } + // @bug 1349. no-op rules: + // 1. If two columns of a simple filter are of the two different tables, and the + // filter operator is not "=", no-op this simple filter, + // 2. If a join filter has "ANTI" option, no op this filter before ANTI hashjoin + // is supported in ExeMgr. + // @bug 1933. Throw exception instead of no-op for MySQL virtual table + // (no connector re-filter). + if (sf->joinFlag() == SimpleFilter::ANTI) + throw runtime_error("Anti join is not currently supported"); - // @bug 1349. no-op rules: - // 1. If two columns of a simple filter are of the two different tables, and the - // filter operator is not "=", no-op this simple filter, - // 2. If a join filter has "ANTI" option, no op this filter before ANTI hashjoin - // is supported in ExeMgr. - // @bug 1933. Throw exception instead of no-op for MySQL virtual table - // (no connector re-filter). - if (sf->joinFlag() == SimpleFilter::ANTI) - throw runtime_error("Anti join is not currently supported"); + // Do a simple column join + JobStepVector join = doJoin(sc1, sc2, jobInfo, sop, sf); + // set cardinality for the hashjoin step. hj result card <= larger input card + uint32_t card = 0; - // Do a simple column join - JobStepVector join = doJoin(sc1, sc2, jobInfo, sop, sf); - // set cardinality for the hashjoin step. hj result card <= larger input card - uint32_t card = 0; + if (sf->cardinality() > sc1->cardinality() && sf->cardinality() > sc2->cardinality()) + card = (sc1->cardinality() > sc2->cardinality() ? sc1->cardinality() : sc2->cardinality()); + else + card = sf->cardinality(); + + join[join.size() - 1].get()->cardinality(card); + + jsv.insert(jsv.end(), join.begin(), join.end()); + } + else if (lhsType == CONSTANTCOLUMN && rhsType == SIMPLECOLUMN) + { + // swap the two and process as normal + SOP opsop(sop->opposite()); + sf->op(opsop); + sf->lhs(rhs); + sf->rhs(lhs); + jsv = doSimpleFilter(sf, jobInfo); + + if (jsv.empty()) + throw runtime_error("Unhandled SimpleFilter"); + } + else if (lhsType == ARITHMETICCOLUMN || rhsType == ARITHMETICCOLUMN || lhsType == FUNCTIONCOLUMN || + rhsType == FUNCTIONCOLUMN) + { + const ReturnedColumn* rc = static_cast(lhs); + + if (rc && (!rc->joinInfo()) && rhsType == AGGREGATECOLUMN) + throw IDBExcept(ERR_AGG_IN_WHERE); + + // @bug5848, CP elimination for idbPartition(col) + JobStepVector sv; + + if (optimizeIdbPatitionSimpleFilter(sf, sv, jobInfo)) + jsv.swap(sv); + else + jsv = doExpressionFilter(sf, jobInfo); + } + else if (lhsType == SIMPLECOLUMN && + (rhsType == AGGREGATECOLUMN || rhsType == ARITHMETICCOLUMN || rhsType == FUNCTIONCOLUMN)) + { + const SimpleColumn* sc = static_cast(lhs); + const ReturnedColumn* rc = static_cast(rhs); + + if (sc->joinInfo() != 0) + return doSemiJoin(sc, rc, jobInfo); + else if (rhsType == AGGREGATECOLUMN) + throw IDBExcept(ERR_AGG_IN_WHERE); + else + throw logic_error("doSimpleFilter: Unhandled SimpleFilter."); + } + else if (rhsType == SIMPLECOLUMN && + (lhsType == AGGREGATECOLUMN || lhsType == ARITHMETICCOLUMN || lhsType == FUNCTIONCOLUMN)) + { + const SimpleColumn* sc = static_cast(rhs); + const ReturnedColumn* rc = static_cast(lhs); + + if (sc->joinInfo() != 0) + return doSemiJoin(sc, rc, jobInfo); + else if (lhsType == AGGREGATECOLUMN) + throw IDBExcept(ERR_AGG_IN_WHERE); + else + throw logic_error("doSimpleFilter: Unhandled SimpleFilter."); + } + else if (rhsType == WINDOWFUNCTIONCOLUMN) + { + // workaroud for IN subquery with window function + // window function as arguments of a function is not covered !! + jsv = doExpressionFilter(sf, jobInfo); + } + else if (lhsType == CONSTANTCOLUMN && rhsType == CONSTANTCOLUMN) + { + jsv = doExpressionFilter(sf, jobInfo); + } + else + { + cerr << boldStart << "doSimpleFilter: Unhandled SimpleFilter: left = " << lhsType + << ", right = " << rhsType << boldStop << endl; + + throw logic_error("doSimpleFilter: Unhandled SimpleFilter."); + } + + return jsv; +} + +const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) +{ + JobStepVector jsv; + + if (oj == 0) + return jsv; + + // cout << "doOuterJoinOnFilter " << endl; + // cout << *oj << endl; + + // Parse the join on filter to join steps and an expression step, if any. + stack nodeStack; // stack used for pre-order traverse + set doneNodes; // solved joins and simple filters + map cpMap; // link for node removal + JobStepVector join; // join step with its projection steps + bool keepFilters = false; // keep filters for cross engine step + + // To compromise the front end difficulty on setting outer attributes. + set tablesInOuter; + + // root + ParseTree* filters = new ParseTree(*(oj->pt().get())); + nodeStack.push(filters); + cpMap[filters] = NULL; + + // @bug5311, optimization for outer join on clause + set tablesInJoin; + + // @bug3683, function join + vector > fjCandidates; + + // while stack is not empty + while (!nodeStack.empty()) + { + ParseTree* cn = nodeStack.top(); // current node + nodeStack.pop(); + TreeNode* tn = cn->data(); + Operator* op = dynamic_cast(tn); // AND | OR + + if (op != NULL && (*op == opOR || *op == opor)) + continue; + + // join is expressed as SimpleFilter + SimpleFilter* sf = dynamic_cast(tn); + + if (sf != NULL) + { + if (sf->joinFlag() == SimpleFilter::ANTI) + throw runtime_error("Anti join is not currently supported"); + + ReturnedColumn* lhs = sf->lhs(); + ReturnedColumn* rhs = sf->rhs(); + SOP sop = sf->op(); + + SimpleColumn* sc1 = dynamic_cast(lhs); + SimpleColumn* sc2 = dynamic_cast(rhs); + + if ((sc1 != NULL && sc2 != NULL) && (sop->data() == "=") && + (sc1->tableName() != sc2->tableName() || sc1->tableAlias() != sc2->tableAlias() || + sc1->viewName() != sc2->viewName())) + { + // @bug3037, workaround on join order, wish this can be corrected soon, + // cascade outer table attribute. + CalpontSystemCatalog::OID tableOid1 = tableOid(sc1, jobInfo.csc); + uint64_t tid1 = + getTableKey(jobInfo, tableOid1, sc1->tableAlias(), sc1->schemaName(), sc1->viewName()); + CalpontSystemCatalog::OID tableOid2 = tableOid(sc2, jobInfo.csc); + uint64_t tid2 = + getTableKey(jobInfo, tableOid2, sc2->tableAlias(), sc2->schemaName(), sc2->viewName()); + + if (tablesInOuter.find(tid1) != tablesInOuter.end()) + sc1->returnAll(true); + else if (tablesInOuter.find(tid2) != tablesInOuter.end()) + sc2->returnAll(true); + + if (sc1->returnAll() && !sc2->returnAll()) + tablesInOuter.insert(tid1); + else if (!sc1->returnAll() && sc2->returnAll()) + tablesInOuter.insert(tid2); + + tablesInJoin.insert(tid1); + tablesInJoin.insert(tid2); + + join = doJoin(sc1, sc2, jobInfo, sop, sf); + // set cardinality for the hashjoin step. + uint32_t card = sf->cardinality(); if (sf->cardinality() > sc1->cardinality() && sf->cardinality() > sc2->cardinality()) - card = (sc1->cardinality() > sc2->cardinality() ? sc1->cardinality() : sc2->cardinality()); - else - card = sf->cardinality(); + card = ((sc1->cardinality() > sc2->cardinality()) ? sc1->cardinality() : sc2->cardinality()); join[join.size() - 1].get()->cardinality(card); jsv.insert(jsv.end(), join.begin(), join.end()); + doneNodes.insert(cn); + } + else + { + ExpressionStep* es = new ExpressionStep(jobInfo); + + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 2"); + + SJSTEP sjstep; + es->expressionFilter(sf, jobInfo); + sjstep.reset(es); + + if (es->functionJoinInfo()) + fjCandidates.push_back(make_pair(cn, sjstep)); + } } - else if (lhsType == CONSTANTCOLUMN && rhsType == SIMPLECOLUMN) + + // Add right and left to the stack. + ParseTree* right = cn->right(); + + if (right != NULL) { - //swap the two and process as normal - SOP opsop(sop->opposite()); - sf->op(opsop); - sf->lhs(rhs); - sf->rhs(lhs); - jsv = doSimpleFilter(sf, jobInfo); - - if (jsv.empty()) - throw runtime_error("Unhandled SimpleFilter"); + cpMap[right] = cn; + nodeStack.push(right); } - else if (lhsType == ARITHMETICCOLUMN || rhsType == ARITHMETICCOLUMN || - lhsType == FUNCTIONCOLUMN || rhsType == FUNCTIONCOLUMN) + + ParseTree* left = cn->left(); + + if (left != NULL) { - const ReturnedColumn* rc = static_cast(lhs); - - if (rc && (!rc->joinInfo()) && rhsType == AGGREGATECOLUMN) - throw IDBExcept(ERR_AGG_IN_WHERE); - - // @bug5848, CP elimination for idbPartition(col) - JobStepVector sv; - - if (optimizeIdbPatitionSimpleFilter(sf, sv, jobInfo)) - jsv.swap(sv); - else - jsv = doExpressionFilter(sf, jobInfo); + cpMap[left] = cn; + nodeStack.push(left); } - else if (lhsType == SIMPLECOLUMN && - (rhsType == AGGREGATECOLUMN || rhsType == ARITHMETICCOLUMN || rhsType == FUNCTIONCOLUMN)) + } + + // check if there are any join steps in jsv. + bool isOk = true; + TupleHashJoinStep* thjs = NULL; + + for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) + { + if (dynamic_cast(i->get()) != thjs) + thjs = dynamic_cast(i->get()); + } + + // no simple column join found, try function join + if (thjs == NULL) + { + // check any expressions can be converted to function joins. + vector >::iterator i = fjCandidates.begin(); + + while (i != fjCandidates.end()) { - const SimpleColumn* sc = static_cast(lhs); - const ReturnedColumn* rc = static_cast(rhs); + ExpressionStep* es = dynamic_cast((i->second).get()); + SJSTEP sjstep = expressionToFuncJoin(es, jobInfo); + idbassert(sjstep.get()); + jsv.push_back(sjstep); - if (sc->joinInfo() != 0) - return doSemiJoin(sc, rc, jobInfo); - else if (rhsType == AGGREGATECOLUMN) - throw IDBExcept(ERR_AGG_IN_WHERE); - else - throw logic_error("doSimpleFilter: Unhandled SimpleFilter."); + doneNodes.insert(i->first); + + i++; } - else if (rhsType == SIMPLECOLUMN && - (lhsType == AGGREGATECOLUMN || lhsType == ARITHMETICCOLUMN || lhsType == FUNCTIONCOLUMN)) - { - const SimpleColumn* sc = static_cast(rhs); - const ReturnedColumn* rc = static_cast(lhs); + } - if (sc->joinInfo() != 0) - return doSemiJoin(sc, rc, jobInfo); - else if (lhsType == AGGREGATECOLUMN) - throw IDBExcept(ERR_AGG_IN_WHERE); - else - throw logic_error("doSimpleFilter: Unhandled SimpleFilter."); - } - else if (rhsType == WINDOWFUNCTIONCOLUMN) - { - // workaroud for IN subquery with window function - // window function as arguments of a function is not covered !! - jsv = doExpressionFilter(sf, jobInfo); - } - else if (lhsType == CONSTANTCOLUMN && rhsType == CONSTANTCOLUMN) - { - jsv = doExpressionFilter(sf, jobInfo); - } - else - { - cerr << boldStart << "doSimpleFilter: Unhandled SimpleFilter: left = " << lhsType << - ", right = " << rhsType << boldStop << endl; + // check again if we got a join step. + for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) + { + if (dynamic_cast(i->get()) != thjs) + thjs = dynamic_cast(i->get()); + } - throw logic_error("doSimpleFilter: Unhandled SimpleFilter."); - } - - return jsv; -} - - -const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) -{ - JobStepVector jsv; - - if (oj == 0) return jsv; - - //cout << "doOuterJoinOnFilter " << endl; - //cout << *oj << endl; - - // Parse the join on filter to join steps and an expression step, if any. - stack nodeStack; // stack used for pre-order traverse - set doneNodes; // solved joins and simple filters - map cpMap; // link for node removal - JobStepVector join; // join step with its projection steps - bool keepFilters = false; // keep filters for cross engine step - - // To compromise the front end difficulty on setting outer attributes. - set tablesInOuter; - - // root - ParseTree* filters = new ParseTree(*(oj->pt().get())); + if (thjs != NULL) + { + // @bug5311, optimization for outer join on clause, move out small side simple filters. + // some repeat code, but better done after join sides settled. nodeStack.push(filters); cpMap[filters] = NULL; - // @bug5311, optimization for outer join on clause - set tablesInJoin; - - // @bug3683, function join - vector > fjCandidates; - // while stack is not empty while (!nodeStack.empty()) { - ParseTree* cn = nodeStack.top(); // current node - nodeStack.pop(); - TreeNode* tn = cn->data(); - Operator* op = dynamic_cast(tn); // AND | OR + ParseTree* cn = nodeStack.top(); // current node + nodeStack.pop(); + TreeNode* tn = cn->data(); + Operator* op = dynamic_cast(tn); // AND | OR - if (op != NULL && (*op == opOR || *op == opor)) + if (op != NULL && (*op == opOR || *op == opor)) + continue; + + SimpleFilter* sf = dynamic_cast(tn); + + if ((sf != NULL) && (doneNodes.find(cn) == doneNodes.end())) + { + // joins are done, this is not a join. + ReturnedColumn* lhs = sf->lhs(); + ReturnedColumn* rhs = sf->rhs(); + SOP sop = sf->op(); + + // handle simple-simple | simple-constant | constant-simple + SimpleColumn* sc = NULL; + SimpleColumn* sc1 = dynamic_cast(lhs); + SimpleColumn* sc2 = dynamic_cast(rhs); + + if ((sc1 != NULL && sc2 != NULL) && + (sc1->tableName() == sc2->tableName() && sc1->tableAlias() == sc2->tableAlias() && + sc1->viewName() == sc2->viewName())) + { + sc = sc1; // same table, just check sc1 + } + else if (sc1 != NULL && dynamic_cast(rhs) != NULL) + { + sc = sc1; + } + else if (sc2 != NULL && dynamic_cast(lhs) != NULL) + { + sc = sc2; + } + + if (sc != NULL) + { + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + uint64_t tid = getTableKey(jobInfo, tblOid, sc->tableAlias(), sc->schemaName(), sc->viewName()); + + // skip outer table filters or table not directly involved in the outer join + if (tablesInOuter.find(tid) != tablesInOuter.end() || tablesInJoin.find(tid) == tablesInJoin.end()) continue; - // join is expressed as SimpleFilter - SimpleFilter* sf = dynamic_cast(tn); + JobStepVector sfv = doSimpleFilter(sf, jobInfo); + ExpressionStep* es = NULL; - if (sf != NULL) + for (JobStepVector::iterator k = sfv.begin(); k != sfv.end(); k++) + { + k->get()->onClauseFilter(true); + + if ((es = dynamic_cast(k->get())) != NULL) + es->associatedJoinId(thjs->joinId()); + } + + jsv.insert(jsv.end(), sfv.begin(), sfv.end()); + + // MCOL-1182 if we are doing a join between a cross engine + // step and a constant then keep the filter for the cross + // engine step instead of deleting it further down. + if (!sc->isColumnStore()) + { + keepFilters = true; + } + + doneNodes.insert(cn); + } + } + + // Add right and left to the stack. + ParseTree* right = cn->right(); + + if (right != NULL) + { + cpMap[right] = cn; + nodeStack.push(right); + } + + ParseTree* left = cn->left(); + + if (left != NULL) + { + cpMap[left] = cn; + nodeStack.push(left); + } + } + + // remove joins from the original filters + ParseTree* nullTree = NULL; + + for (set::iterator i = doneNodes.begin(); i != doneNodes.end() && isOk; i++) + { + ParseTree* c = *i; + map::iterator j = cpMap.find(c); + + if (j == cpMap.end()) + { + isOk = false; + continue; + } + + ParseTree* p = j->second; + + if (p == NULL) + { + filters = NULL; + + if (!keepFilters) { - if (sf->joinFlag() == SimpleFilter::ANTI) - throw runtime_error("Anti join is not currently supported"); + delete c; + } + } + else + { + map::iterator k = cpMap.find(p); - ReturnedColumn* lhs = sf->lhs(); - ReturnedColumn* rhs = sf->rhs(); - SOP sop = sf->op(); + if (k == cpMap.end()) + { + isOk = false; + continue; + } - SimpleColumn* sc1 = dynamic_cast(lhs); - SimpleColumn* sc2 = dynamic_cast(rhs); + ParseTree* pp = k->second; - if ((sc1 != NULL && sc2 != NULL) && (sop->data() == "=") && - (sc1->tableName() != sc2->tableName() || - sc1->tableAlias() != sc2->tableAlias() || - sc1->viewName() != sc2->viewName())) - { - // @bug3037, workaround on join order, wish this can be corrected soon, - // cascade outer table attribute. - CalpontSystemCatalog::OID tableOid1 = tableOid(sc1, jobInfo.csc); - uint64_t tid1 = getTableKey( - jobInfo, tableOid1, sc1->tableAlias(), sc1->schemaName(), sc1->viewName()); - CalpontSystemCatalog::OID tableOid2 = tableOid(sc2, jobInfo.csc); - uint64_t tid2 = getTableKey( - jobInfo, tableOid2, sc2->tableAlias(), sc2->schemaName(), sc2->viewName()); + if (pp == NULL) + { + if (p->left() == c) + filters = p->right(); + else + filters = p->left(); - if (tablesInOuter.find(tid1) != tablesInOuter.end()) - sc1->returnAll(true); - else if (tablesInOuter.find(tid2) != tablesInOuter.end()) - sc2->returnAll(true); - - if (sc1->returnAll() && !sc2->returnAll()) - tablesInOuter.insert(tid1); - else if (!sc1->returnAll() && sc2->returnAll()) - tablesInOuter.insert(tid2); - - tablesInJoin.insert(tid1); - tablesInJoin.insert(tid2); - - join = doJoin(sc1, sc2, jobInfo, sop, sf); - // set cardinality for the hashjoin step. - uint32_t card = sf->cardinality(); - - if (sf->cardinality() > sc1->cardinality() && - sf->cardinality() > sc2->cardinality()) - card = ((sc1->cardinality() > sc2->cardinality()) ? - sc1->cardinality() : sc2->cardinality()); - - join[join.size() - 1].get()->cardinality(card); - - jsv.insert(jsv.end(), join.begin(), join.end()); - doneNodes.insert(cn); - } + cpMap[filters] = NULL; + } + else + { + if (p->left() == c) + { + if (pp->left() == p) + pp->left(p->right()); else - { - ExpressionStep* es = new ExpressionStep(jobInfo); + pp->right(p->right()); - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 2"); - - SJSTEP sjstep; - es->expressionFilter(sf, jobInfo); - sjstep.reset(es); - - if (es->functionJoinInfo()) - fjCandidates.push_back(make_pair(cn, sjstep)); - } - } - - // Add right and left to the stack. - ParseTree* right = cn->right(); - - if (right != NULL) - { - cpMap[right] = cn; - nodeStack.push(right); - } - - ParseTree* left = cn->left(); - - if (left != NULL) - { - cpMap[left] = cn; - nodeStack.push(left); - } - } - - // check if there are any join steps in jsv. - bool isOk = true; - TupleHashJoinStep* thjs = NULL; - - for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) - { - if (dynamic_cast(i->get()) != thjs) - thjs = dynamic_cast(i->get()); - } - - // no simple column join found, try function join - if (thjs == NULL) - { - // check any expressions can be converted to function joins. - vector >::iterator i = fjCandidates.begin(); - - while (i != fjCandidates.end()) - { - ExpressionStep* es = dynamic_cast((i->second).get()); - SJSTEP sjstep = expressionToFuncJoin(es, jobInfo); - idbassert(sjstep.get()); - jsv.push_back(sjstep); - - doneNodes.insert(i->first); - - i++; - } - } - - // check again if we got a join step. - for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) - { - if (dynamic_cast(i->get()) != thjs) - thjs = dynamic_cast(i->get()); - } - - if (thjs != NULL) - { - // @bug5311, optimization for outer join on clause, move out small side simple filters. - // some repeat code, but better done after join sides settled. - nodeStack.push(filters); - cpMap[filters] = NULL; - - // while stack is not empty - while (!nodeStack.empty()) - { - ParseTree* cn = nodeStack.top(); // current node - nodeStack.pop(); - TreeNode* tn = cn->data(); - Operator* op = dynamic_cast(tn); // AND | OR - - if (op != NULL && (*op == opOR || *op == opor)) - continue; - - SimpleFilter* sf = dynamic_cast(tn); - - if ((sf != NULL) && (doneNodes.find(cn) == doneNodes.end())) - { - // joins are done, this is not a join. - ReturnedColumn* lhs = sf->lhs(); - ReturnedColumn* rhs = sf->rhs(); - SOP sop = sf->op(); - - // handle simple-simple | simple-constant | constant-simple - SimpleColumn* sc = NULL; - SimpleColumn* sc1 = dynamic_cast(lhs); - SimpleColumn* sc2 = dynamic_cast(rhs); - - if ((sc1 != NULL && sc2 != NULL) && - (sc1->tableName() == sc2->tableName() && - sc1->tableAlias() == sc2->tableAlias() && - sc1->viewName() == sc2->viewName())) - { - sc = sc1; // same table, just check sc1 - } - else if (sc1 != NULL && dynamic_cast(rhs) != NULL) - { - sc = sc1; - } - else if (sc2 != NULL && dynamic_cast(lhs) != NULL) - { - sc = sc2; - } - - if (sc != NULL) - { - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - uint64_t tid = getTableKey( - jobInfo, tblOid, sc->tableAlias(), sc->schemaName(), sc->viewName()); - - // skip outer table filters or table not directly involved in the outer join - if (tablesInOuter.find(tid) != tablesInOuter.end() || - tablesInJoin.find(tid) == tablesInJoin.end()) - continue; - - JobStepVector sfv = doSimpleFilter(sf, jobInfo); - ExpressionStep* es = NULL; - - for (JobStepVector::iterator k = sfv.begin(); k != sfv.end(); k++) - { - k->get()->onClauseFilter(true); - - if ((es = dynamic_cast(k->get())) != NULL) - es->associatedJoinId(thjs->joinId()); - } - - jsv.insert(jsv.end(), sfv.begin(), sfv.end()); - - // MCOL-1182 if we are doing a join between a cross engine - // step and a constant then keep the filter for the cross - // engine step instead of deleting it further down. - if (!sc->isColumnStore()) - { - keepFilters = true; - } - - doneNodes.insert(cn); - } - } - - // Add right and left to the stack. - ParseTree* right = cn->right(); - - if (right != NULL) - { - cpMap[right] = cn; - nodeStack.push(right); - } - - ParseTree* left = cn->left(); - - if (left != NULL) - { - cpMap[left] = cn; - nodeStack.push(left); - } - } - - // remove joins from the original filters - ParseTree* nullTree = NULL; - - for (set::iterator i = doneNodes.begin(); i != doneNodes.end() && isOk; i++) - { - ParseTree* c = *i; - map::iterator j = cpMap.find(c); - - if (j == cpMap.end()) - { - isOk = false; - continue; - } - - ParseTree* p = j->second; - - if (p == NULL) - { - filters = NULL; - - if (!keepFilters) - { - delete c; - } - } + cpMap[p->right()] = pp; + } + else + { + if (pp->left() == p) + pp->left(p->left()); else - { - map::iterator k = cpMap.find(p); + pp->right(p->left()); - if (k == cpMap.end()) - { - isOk = false; - continue; - } - - ParseTree* pp = k->second; - - if (pp == NULL) - { - if (p->left() == c) - filters = p->right(); - else - filters = p->left(); - - cpMap[filters] = NULL; - } - else - { - if (p->left() == c) - { - if (pp->left() == p) - pp->left(p->right()); - else - pp->right(p->right()); - - cpMap[p->right()] = pp; - } - else - { - if (pp->left() == p) - pp->left(p->left()); - else - pp->right(p->left()); - - cpMap[p->left()] = pp; - } - } - - p->left(nullTree); - p->right(nullTree); - - if (!keepFilters) - { - delete p; - delete c; - } - } + cpMap[p->left()] = pp; + } } - // construct an expression step, if additional comparison exists. - if (isOk && filters != NULL && filters->data() != NULL) + p->left(nullTree); + p->right(nullTree); + + if (!keepFilters) { - ExpressionStep* es = new ExpressionStep(jobInfo); - - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 1"); - - es->expressionFilter(filters, jobInfo); - es->associatedJoinId(thjs->joinId()); - SJSTEP sjstep(es); - jsv.push_back(sjstep); + delete p; + delete c; } + } } - else + + // construct an expression step, if additional comparison exists. + if (isOk && filters != NULL && filters->data() != NULL) { - // Due to Calpont view handling, some joins may treated as expressions. - ExpressionStep* es = new ExpressionStep(jobInfo); + ExpressionStep* es = new ExpressionStep(jobInfo); - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 1"); + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 1"); - es->expressionFilter(filters, jobInfo); - SJSTEP sjstep(es); - jsv.push_back(sjstep); + es->expressionFilter(filters, jobInfo); + es->associatedJoinId(thjs->joinId()); + SJSTEP sjstep(es); + jsv.push_back(sjstep); } + } + else + { + // Due to Calpont view handling, some joins may treated as expressions. + ExpressionStep* es = new ExpressionStep(jobInfo); - delete filters; + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 1"); - if (!isOk) - { - throw runtime_error("Failed to parse join condition."); - } + es->expressionFilter(filters, jobInfo); + SJSTEP sjstep(es); + jsv.push_back(sjstep); + } - if (jobInfo.trace) - { - ostringstream oss; - oss << "\nOuterJoinOn steps: " << endl; - ostream_iterator oIter(oss, "\n"); - copy(jsv.begin(), jsv.end(), oIter); - cout << oss.str(); - } + delete filters; - return jsv; + if (!isOk) + { + throw runtime_error("Failed to parse join condition."); + } + + if (jobInfo.trace) + { + ostringstream oss; + oss << "\nOuterJoinOn steps: " << endl; + ostream_iterator oIter(oss, "\n"); + copy(jsv.begin(), jsv.end(), oIter); + cout << oss.str(); + } + + return jsv; } bool tryCombineDictionary(JobStepVector& jsv1, JobStepVector& jsv2, int8_t bop) { - JobStepVector::iterator it2 = jsv2.end() - 1; - - // already checked: (typeid(*(it2->get()) != typeid(pDictionaryStep)) - if (typeid(*((it2 - 1)->get())) != typeid(pColStep)) - return false; - - pDictionaryStep* ipdsp = dynamic_cast(it2->get()); - bool onClauseFilter = ipdsp->onClauseFilter(); - - JobStepVector::iterator iter = jsv1.begin(); - JobStepVector::iterator end = jsv1.end(); - - if (bop == BOP_OR) - { - iter = end - 1; - } - - while (iter != end) - { - pDictionaryStep* pdsp = dynamic_cast(iter->get()); - - if (pdsp != NULL && pdsp->onClauseFilter() == onClauseFilter) - { - // If the OID's match and the BOP's match and the previous step is pcolstep, - // then append the filters. - if ((ipdsp->tupleId() == pdsp->tupleId()) && - (dynamic_cast((iter - 1)->get()) != NULL)) - { - if (pdsp->BOP() == BOP_NONE) - { - if (ipdsp->BOP() == BOP_NONE || ipdsp->BOP() == bop) - { - pdsp->appendFilter(ipdsp->filterString(), ipdsp->filterCount()); - pdsp->setBOP(bop); - pdsp->appendFilter(ipdsp->getFilters()); - return true; - } - } - else if (pdsp->BOP() == bop) - { - if (ipdsp->BOP() == BOP_NONE || ipdsp->BOP() == bop) - { - pdsp->appendFilter(ipdsp->filterString(), ipdsp->filterCount()); - pdsp->appendFilter(ipdsp->getFilters()); - return true; - } - } - } - } - - ++iter; - } + JobStepVector::iterator it2 = jsv2.end() - 1; + // already checked: (typeid(*(it2->get()) != typeid(pDictionaryStep)) + if (typeid(*((it2 - 1)->get())) != typeid(pColStep)) return false; + + pDictionaryStep* ipdsp = dynamic_cast(it2->get()); + bool onClauseFilter = ipdsp->onClauseFilter(); + + JobStepVector::iterator iter = jsv1.begin(); + JobStepVector::iterator end = jsv1.end(); + + if (bop == BOP_OR) + { + iter = end - 1; + } + + while (iter != end) + { + pDictionaryStep* pdsp = dynamic_cast(iter->get()); + + if (pdsp != NULL && pdsp->onClauseFilter() == onClauseFilter) + { + // If the OID's match and the BOP's match and the previous step is pcolstep, + // then append the filters. + if ((ipdsp->tupleId() == pdsp->tupleId()) && (dynamic_cast((iter - 1)->get()) != NULL)) + { + if (pdsp->BOP() == BOP_NONE) + { + if (ipdsp->BOP() == BOP_NONE || ipdsp->BOP() == bop) + { + pdsp->appendFilter(ipdsp->filterString(), ipdsp->filterCount()); + pdsp->setBOP(bop); + pdsp->appendFilter(ipdsp->getFilters()); + return true; + } + } + else if (pdsp->BOP() == bop) + { + if (ipdsp->BOP() == BOP_NONE || ipdsp->BOP() == bop) + { + pdsp->appendFilter(ipdsp->filterString(), ipdsp->filterCount()); + pdsp->appendFilter(ipdsp->getFilters()); + return true; + } + } + } + } + + ++iter; + } + + return false; } bool tryCombineDictionaryScan(JobStepVector& jsv1, JobStepVector& jsv2, int8_t bop) @@ -2646,540 +2544,541 @@ bool tryCombineDictionaryScan(JobStepVector& jsv1, JobStepVector& jsv2, int8_t b #endif - return false; + return false; } // We want to search the existing filters in the stack for this column to see if we can just add the // filters for this step to a previous pColStep or pDictionary. bool tryCombineFilters(JobStepVector& jsv1, JobStepVector& jsv2, int8_t bop) { - // A couple of tests to make sure we're operating on the right things... - if (jsv1.size() < 1) return false; - - // if filter by pDictionary, there are two steps: pcolstep and pdictionarystep. - if (jsv2.size() == 2 && typeid(*jsv2.back().get()) == typeid(pDictionaryStep)) - return tryCombineDictionary(jsv1, jsv2, bop); - - // if filter by pDictionaryScan, there are three steps: pdictionaryscan, pcolstep and join. - if (jsv2.size() == 3 && typeid(*jsv2.front().get()) == typeid(pDictionaryScan)) - return tryCombineDictionaryScan(jsv1, jsv2, bop); - - // non-dictionary filters - if (jsv2.size() != 1) return false; - - pColStep* ipcsp = dynamic_cast(jsv2.back().get()); - - if (ipcsp == NULL) - return false; - - bool onClauseFilter = ipcsp->onClauseFilter(); - - JobStepVector::iterator iter = jsv1.begin(); - JobStepVector::iterator end = jsv1.end(); - - // only try last step in jsv1 if operator is OR. - if (bop == BOP_OR) - { - iter = end - 1; - } - - while (iter != end) - { - pColStep* pcsp = dynamic_cast(iter->get()); - - if (pcsp != NULL && pcsp->onClauseFilter() == onClauseFilter) - { - idbassert(pcsp); - - // If the OID's match and the BOP's match then append the filters - if (ipcsp->tupleId() == pcsp->tupleId()) - { - if (pcsp->BOP() == BOP_NONE) - { - if (ipcsp->BOP() == BOP_NONE || ipcsp->BOP() == bop) - { - pcsp->appendFilter(ipcsp->filterString(), ipcsp->filterCount()); - pcsp->setBOP(bop); - pcsp->appendFilter(ipcsp->getFilters()); - return true; - } - } - else if (pcsp->BOP() == bop) - { - if (ipcsp->BOP() == BOP_NONE || ipcsp->BOP() == bop) - { - pcsp->appendFilter(ipcsp->filterString(), ipcsp->filterCount()); - pcsp->appendFilter(ipcsp->getFilters()); - return true; - } - } - } - } - - ++iter; - } - + // A couple of tests to make sure we're operating on the right things... + if (jsv1.size() < 1) return false; -} + // if filter by pDictionary, there are two steps: pcolstep and pdictionarystep. + if (jsv2.size() == 2 && typeid(*jsv2.back().get()) == typeid(pDictionaryStep)) + return tryCombineDictionary(jsv1, jsv2, bop); + + // if filter by pDictionaryScan, there are three steps: pdictionaryscan, pcolstep and join. + if (jsv2.size() == 3 && typeid(*jsv2.front().get()) == typeid(pDictionaryScan)) + return tryCombineDictionaryScan(jsv1, jsv2, bop); + + // non-dictionary filters + if (jsv2.size() != 1) + return false; + + pColStep* ipcsp = dynamic_cast(jsv2.back().get()); + + if (ipcsp == NULL) + return false; + + bool onClauseFilter = ipcsp->onClauseFilter(); + + JobStepVector::iterator iter = jsv1.begin(); + JobStepVector::iterator end = jsv1.end(); + + // only try last step in jsv1 if operator is OR. + if (bop == BOP_OR) + { + iter = end - 1; + } + + while (iter != end) + { + pColStep* pcsp = dynamic_cast(iter->get()); + + if (pcsp != NULL && pcsp->onClauseFilter() == onClauseFilter) + { + idbassert(pcsp); + + // If the OID's match and the BOP's match then append the filters + if (ipcsp->tupleId() == pcsp->tupleId()) + { + if (pcsp->BOP() == BOP_NONE) + { + if (ipcsp->BOP() == BOP_NONE || ipcsp->BOP() == bop) + { + pcsp->appendFilter(ipcsp->filterString(), ipcsp->filterCount()); + pcsp->setBOP(bop); + pcsp->appendFilter(ipcsp->getFilters()); + return true; + } + } + else if (pcsp->BOP() == bop) + { + if (ipcsp->BOP() == BOP_NONE || ipcsp->BOP() == bop) + { + pcsp->appendFilter(ipcsp->filterString(), ipcsp->filterCount()); + pcsp->appendFilter(ipcsp->getFilters()); + return true; + } + } + } + } + + ++iter; + } + + return false; +} const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) { - JobStepVector jsv; + JobStepVector jsv; - if (cf == 0) return jsv; + if (cf == 0) + return jsv; - SOP op = cf->op(); - // default op is 'and' - string opStr("and"); + SOP op = cf->op(); + // default op is 'and' + string opStr("and"); - if (op) opStr = ba::to_lower_copy(op->data()); + if (op) + opStr = ba::to_lower_copy(op->data()); - SJSTEP sjstep; + SJSTEP sjstep; - if (opStr == "and" || opStr == "or") + if (opStr == "and" || opStr == "or") + { + SOP sop; + + const SSC sc = boost::dynamic_pointer_cast(cf->col()); + + // if column from subquery + if (!sc || sc->schemaName().empty()) { - SOP sop; + return doExpressionFilter(cf, jobInfo); + } - const SSC sc = boost::dynamic_pointer_cast(cf->col()); + ConstantFilter::FilterList fl = cf->filterList(); + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct = sc.get()->colType(); + PseudoColumn* pc = dynamic_cast(sc.get()); - // if column from subquery - if (!sc || sc->schemaName().empty()) + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) + { + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + + // X + CalpontSystemCatalog::OID tbOID = tableOid(sc.get(), jobInfo.csc); + string alias(extractTableAlias(sc)); + string view(sc->viewName()); + string schema(sc->schemaName()); + + if ((dictOid = isDictCol(ct)) > 0) + { + if (jobInfo.trace) + cout << "Emit pTokenByScan/pCol for SimpleColumn op ConstantColumn " + "[op ConstantColumn]" + << endl; + + pColStep* pcs = new pColStep(sc->oid(), tbOID, ct, jobInfo); + pcs->alias(alias); + pcs->view(view); + pcs->name(sc->columnName()); + pcs->schema(sc->schemaName()); + pcs->cardinality(sc->cardinality()); + + if (ct.colDataType == execplan::CalpontSystemCatalog::TEXT || + ct.colDataType == execplan::CalpontSystemCatalog::BLOB || + filterWithDictionary(dictOid, jobInfo.stringScanThreshold)) + { + pDictionaryStep* pds = new pDictionaryStep(dictOid, tbOID, ct, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); + pds->alias(alias); + pds->view(view); + pds->name(sc->columnName()); + pds->schema(sc->schemaName()); + pds->cardinality(sc->cardinality()); + + if (op) + pds->setBOP(bop2num(op)); + + // Add the filter(s) + for (unsigned i = 0; i < fl.size(); i++) { - return doExpressionFilter(cf, jobInfo); + const SSFP sf = fl[i]; + const ConstantColumn* cc; + cc = static_cast(sf->rhs()); + sop = sf->op(); + + // add each filter to pColStep + int8_t cop = op2num(sop); + + // @bug 2584, make "= null" to COMPARE_NIL. + if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) + cop = COMPARE_NIL; + + string value = cc->constval(); + // Because, on a filter, we want to compare ignoring trailing spaces + boost::algorithm::trim_right_if(value, boost::is_any_of(" ")); + + pds->addFilter(cop, value); } - ConstantFilter::FilterList fl = cf->filterList(); - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct = sc.get()->colType(); - PseudoColumn* pc = dynamic_cast(sc.get()); + // data list for pcolstep output + AnyDataListSPtr spdl1(new AnyDataList()); + FifoDataList* dl1 = new FifoDataList(1, jobInfo.fifoSize); + spdl1->fifoDL(dl1); + dl1->OID(sc->oid()); -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) + JobStepAssociation outJs1; + outJs1.outAdd(spdl1); + pcs->outputAssociation(outJs1); + + // data list for pdictionarystep output + AnyDataListSPtr spdl2(new AnyDataList()); + StringFifoDataList* dl2 = new StringFifoDataList(1, jobInfo.fifoSize); + spdl2->stringDL(dl2); + dl2->OID(sc->oid()); + + JobStepAssociation outJs2; + outJs2.outAdd(spdl2); + pds->outputAssociation(outJs2); + + // Associate pcs with pds + JobStepAssociation outJs; + outJs.outAdd(spdl1); + pds->inputAssociation(outJs); + + sjstep.reset(pcs); + jsv.push_back(sjstep); + sjstep.reset(pds); + jsv.push_back(sjstep); + + // save for expression transformation + pds->addFilter(cf); + + // token column + CalpontSystemCatalog::ColType tct; + tct.colDataType = CalpontSystemCatalog::BIGINT; + tct.colWidth = 8; + tct.scale = 0; + tct.precision = 0; + tct.compressionType = ct.compressionType; + TupleInfo ti(setTupleInfo(tct, sc->oid(), jobInfo, tbOID, sc.get(), alias)); + jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; + pcs->tupleId(ti.key); + + // string column + ti = setTupleInfo(ct, dictOid, jobInfo, tbOID, sc.get(), alias); + pds->tupleId(ti.key); + jobInfo.keyInfo->dictKeyMap[pcs->tupleId()] = ti.key; + jobInfo.tokenOnly[ti.key] = false; + } + else + { + pDictionaryScan* pds = new pDictionaryScan(dictOid, tbOID, ct, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid] = sc.get()->oid(); + pds->alias(alias); + pds->view(view); + pds->name(sc.get()->columnName()); + pds->schema(sc.get()->schemaName()); + + if (op) + pds->setBOP(bop2num(op)); + + // Add the filter(s) + for (unsigned i = 0; i < fl.size(); i++) { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; + const SSFP sf = fl[i]; + const ConstantColumn* cc; + cc = static_cast(sf->rhs()); + sop = sf->op(); + + // add each filter to pColStep + int8_t cop = op2num(sop); + + // @bug 2584, make "= null" to COMPARE_NIL. + if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) + cop = COMPARE_NIL; + + string value = cc->constval(); + // Because, on a filter, we want to compare ignoring trailing spaces + boost::algorithm::trim_right_if(value, boost::is_any_of(" ")); + + pds->addFilter(cop, value); } -//X - CalpontSystemCatalog::OID tbOID = tableOid(sc.get(), jobInfo.csc); - string alias(extractTableAlias(sc)); - string view(sc->viewName()); - string schema(sc->schemaName()); + // save for expression transformation + pds->addFilter(cf); - if ((dictOid = isDictCol(ct)) > 0) - { - if (jobInfo.trace) - cout << "Emit pTokenByScan/pCol for SimpleColumn op ConstantColumn " - "[op ConstantColumn]" << endl; + TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); + thj->tableOid1(0); + thj->tableOid2(tbOID); + thj->alias1(alias); + thj->alias2(alias); + thj->view1(view); + thj->view2(view); + thj->schema1(schema); + thj->schema2(schema); + thj->oid1(sc->oid()); + thj->oid2(sc->oid()); + thj->joinId(0); + thj->setJoinType(INNER); - pColStep* pcs = new pColStep(sc->oid(), tbOID, ct, jobInfo); - pcs->alias(alias); - pcs->view(view); - pcs->name(sc->columnName()); - pcs->schema(sc->schemaName()); - pcs->cardinality(sc->cardinality()); + // Associate the steps + AnyDataListSPtr spdl1(new AnyDataList()); + RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); + spdl1->rowGroupDL(dl1); + dl1->OID(dictOid); - if (ct.colDataType == execplan::CalpontSystemCatalog::TEXT || - ct.colDataType == execplan::CalpontSystemCatalog::BLOB || - filterWithDictionary(dictOid, jobInfo.stringScanThreshold)) - { - pDictionaryStep* pds = new pDictionaryStep(dictOid, tbOID, ct, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid] = sc->oid(); - pds->alias(alias); - pds->view(view); - pds->name(sc->columnName()); - pds->schema(sc->schemaName()); - pds->cardinality(sc->cardinality()); + JobStepAssociation outJs1; + outJs1.outAdd(spdl1); + pds->outputAssociation(outJs1); - if (op) - pds->setBOP(bop2num(op)); + AnyDataListSPtr spdl2(new AnyDataList()); + RowGroupDL* dl2 = new RowGroupDL(1, jobInfo.fifoSize); + spdl2->rowGroupDL(dl2); + dl2->OID(sc->oid()); - //Add the filter(s) - for (unsigned i = 0; i < fl.size(); i++) - { - const SSFP sf = fl[i]; - const ConstantColumn* cc; - cc = static_cast(sf->rhs()); - sop = sf->op(); + JobStepAssociation outJs2; + outJs2.outAdd(spdl2); + pcs->outputAssociation(outJs2); - //add each filter to pColStep - int8_t cop = op2num(sop); + JobStepAssociation outJs3; + outJs3.outAdd(spdl1); + outJs3.outAdd(spdl2); + thj->inputAssociation(outJs3); - // @bug 2584, make "= null" to COMPARE_NIL. - if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) - cop = COMPARE_NIL; + sjstep.reset(pds); + jsv.push_back(sjstep); + sjstep.reset(pcs); + jsv.push_back(sjstep); + sjstep.reset(thj); + jsv.push_back(sjstep); - string value = cc->constval(); - // Because, on a filter, we want to compare ignoring trailing spaces - boost::algorithm::trim_right_if(value, boost::is_any_of(" ")); + CalpontSystemCatalog::ColType dct; + dct.colDataType = CalpontSystemCatalog::BIGINT; + dct.colWidth = 8; + dct.scale = 0; + dct.precision = 0; + dct.compressionType = ct.compressionType; - pds->addFilter(cop, value); - } + TupleInfo ti(setTupleInfo(dct, sc->oid(), jobInfo, tbOID, sc.get(), alias)); + jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; + pds->tupleId(ti.key); // pcs, pds use same tuple key, both 8-byte column + pcs->tupleId(ti.key); + thj->tupleId1(ti.key); + thj->tupleId2(ti.key); - // data list for pcolstep output - AnyDataListSPtr spdl1(new AnyDataList()); - FifoDataList* dl1 = new FifoDataList(1, jobInfo.fifoSize); - spdl1->fifoDL(dl1); - dl1->OID(sc->oid()); - - JobStepAssociation outJs1; - outJs1.outAdd(spdl1); - pcs->outputAssociation(outJs1); - - // data list for pdictionarystep output - AnyDataListSPtr spdl2(new AnyDataList()); - StringFifoDataList* dl2 = new StringFifoDataList(1, jobInfo.fifoSize); - spdl2->stringDL(dl2); - dl2->OID(sc->oid()); - - JobStepAssociation outJs2; - outJs2.outAdd(spdl2); - pds->outputAssociation(outJs2); - - //Associate pcs with pds - JobStepAssociation outJs; - outJs.outAdd(spdl1); - pds->inputAssociation(outJs); - - sjstep.reset(pcs); - jsv.push_back(sjstep); - sjstep.reset(pds); - jsv.push_back(sjstep); - - // save for expression transformation - pds->addFilter(cf); - - // token column - CalpontSystemCatalog::ColType tct; - tct.colDataType = CalpontSystemCatalog::BIGINT; - tct.colWidth = 8; - tct.scale = 0; - tct.precision = 0; - tct.compressionType = ct.compressionType; - TupleInfo ti(setTupleInfo(tct, sc->oid(), jobInfo, tbOID, sc.get(), alias)); - jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; - pcs->tupleId(ti.key); - - // string column - ti = setTupleInfo(ct, dictOid, jobInfo, tbOID, sc.get(), alias); - pds->tupleId(ti.key); - jobInfo.keyInfo->dictKeyMap[pcs->tupleId()] = ti.key; - jobInfo.tokenOnly[ti.key] = false; - } - else - { - pDictionaryScan* pds = new pDictionaryScan(dictOid, tbOID, ct, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid] = sc.get()->oid(); - pds->alias(alias); - pds->view(view); - pds->name(sc.get()->columnName()); - pds->schema(sc.get()->schemaName()); - - if (op) - pds->setBOP(bop2num(op)); - - //Add the filter(s) - for (unsigned i = 0; i < fl.size(); i++) - { - const SSFP sf = fl[i]; - const ConstantColumn* cc; - cc = static_cast(sf->rhs()); - sop = sf->op(); - - //add each filter to pColStep - int8_t cop = op2num(sop); - - // @bug 2584, make "= null" to COMPARE_NIL. - if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) - cop = COMPARE_NIL; - - string value = cc->constval(); - // Because, on a filter, we want to compare ignoring trailing spaces - boost::algorithm::trim_right_if(value, boost::is_any_of(" ")); - - pds->addFilter(cop, value); - } - - // save for expression transformation - pds->addFilter(cf); - - TupleHashJoinStep* thj = new TupleHashJoinStep(jobInfo); - thj->tableOid1(0); - thj->tableOid2(tbOID); - thj->alias1(alias); - thj->alias2(alias); - thj->view1(view); - thj->view2(view); - thj->schema1(schema); - thj->schema2(schema); - thj->oid1(sc->oid()); - thj->oid2(sc->oid()); - thj->joinId(0); - thj->setJoinType(INNER); - - //Associate the steps - AnyDataListSPtr spdl1(new AnyDataList()); - RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); - spdl1->rowGroupDL(dl1); - dl1->OID(dictOid); - - JobStepAssociation outJs1; - outJs1.outAdd(spdl1); - pds->outputAssociation(outJs1); - - AnyDataListSPtr spdl2(new AnyDataList()); - RowGroupDL* dl2 = new RowGroupDL(1, jobInfo.fifoSize); - spdl2->rowGroupDL(dl2); - dl2->OID(sc->oid()); - - JobStepAssociation outJs2; - outJs2.outAdd(spdl2); - pcs->outputAssociation(outJs2); - - JobStepAssociation outJs3; - outJs3.outAdd(spdl1); - outJs3.outAdd(spdl2); - thj->inputAssociation(outJs3); - - sjstep.reset(pds); - jsv.push_back(sjstep); - sjstep.reset(pcs); - jsv.push_back(sjstep); - sjstep.reset(thj); - jsv.push_back(sjstep); - - CalpontSystemCatalog::ColType dct; - dct.colDataType = CalpontSystemCatalog::BIGINT; - dct.colWidth = 8; - dct.scale = 0; - dct.precision = 0; - dct.compressionType = ct.compressionType; - - TupleInfo ti(setTupleInfo(dct, sc->oid(), jobInfo, tbOID, sc.get(), alias)); - jobInfo.keyInfo->token2DictTypeMap[ti.key] = ct; - pds->tupleId(ti.key); // pcs, pds use same tuple key, both 8-byte column - pcs->tupleId(ti.key); - thj->tupleId1(ti.key); - thj->tupleId2(ti.key); - - if (jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) - jobInfo.tokenOnly[ti.key] = true; - } - } - else - { - if (jobInfo.trace) - cout << "Emit pCol for SimpleColumn op ConstantColumn [op ConstantColumn]" << endl; - - CalpontSystemCatalog::OID tblOid = tableOid(sc.get(), jobInfo.csc); - string alias(extractTableAlias(sc)); - pColStep* pcs = NULL; - - if (pc == NULL) - pcs = new pColStep(sc->oid(), tblOid, ct, jobInfo); - else - pcs = new PseudoColStep(sc->oid(), tblOid, pc->pseudoType(), ct, jobInfo); - - pcs->alias(extractTableAlias(sc)); - pcs->view(sc->viewName()); - pcs->name(sc->columnName()); - pcs->schema(sc->schemaName()); - - if (sc->isColumnStore()) - { - if (op) - pcs->setBOP(bop2num(op)); - - for (unsigned i = 0; i < fl.size(); i++) - { - const SSFP sf = fl[i]; - const ConstantColumn* cc; - cc = static_cast(sf->rhs()); - sop = sf->op(); - - //add each filter to pColStep - int8_t cop = op2num(sop); - int64_t value = 0; - int128_t value128 = 0; - string constval = cc->constval(); - - // @bug 1151 string longer than colwidth of char/varchar. - uint8_t rf = 0; - bool isNull = ConstantColumn::NULLDATA == cc->type(); - - if (ct.isWideDecimalType()) - convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); - else - convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); - - if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) - { - float f = cc->getFloatVal(); - value = *(reinterpret_cast(&f)); - } - else if (ct.colDataType == CalpontSystemCatalog::DOUBLE && !isNull) - { - double d = cc->getDoubleVal(); - value = *(reinterpret_cast(&d)); - } - - // @bug 2584, make "= null" to COMPARE_NIL. - if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) - cop = COMPARE_NIL; - - if (ct.isWideDecimalType()) - pcs->addFilter(cop, value128, rf); - else - pcs->addFilter(cop, value, rf); - } - } - - // save for expression transformation - pcs->addFilter(cf); - - sjstep.reset(pcs); - jsv.push_back(sjstep); - -//XXX use this before connector sets colType in sc correctly. - CalpontSystemCatalog::ColType ct = sc->colType(); - - if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc.get(), alias)); -//X TupleInfo ti(setTupleInfo(sc->colType(), sc->oid(), jobInfo, tblOid, sc.get(), alias)); - pcs->tupleId(ti.key); - } + if (jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) + jobInfo.tokenOnly[ti.key] = true; + } } else { - cerr << boldStart << "doConstantFilter: Can only handle 'and' and 'or' right now, got '" - << opStr << "'" << boldStop << endl; - throw logic_error("doConstantFilter: Not handled operation type."); + if (jobInfo.trace) + cout << "Emit pCol for SimpleColumn op ConstantColumn [op ConstantColumn]" << endl; + + CalpontSystemCatalog::OID tblOid = tableOid(sc.get(), jobInfo.csc); + string alias(extractTableAlias(sc)); + pColStep* pcs = NULL; + + if (pc == NULL) + pcs = new pColStep(sc->oid(), tblOid, ct, jobInfo); + else + pcs = new PseudoColStep(sc->oid(), tblOid, pc->pseudoType(), ct, jobInfo); + + pcs->alias(extractTableAlias(sc)); + pcs->view(sc->viewName()); + pcs->name(sc->columnName()); + pcs->schema(sc->schemaName()); + + if (sc->isColumnStore()) + { + if (op) + pcs->setBOP(bop2num(op)); + + for (unsigned i = 0; i < fl.size(); i++) + { + const SSFP sf = fl[i]; + const ConstantColumn* cc; + cc = static_cast(sf->rhs()); + sop = sf->op(); + + // add each filter to pColStep + int8_t cop = op2num(sop); + int64_t value = 0; + int128_t value128 = 0; + string constval = cc->constval(); + + // @bug 1151 string longer than colwidth of char/varchar. + uint8_t rf = 0; + bool isNull = ConstantColumn::NULLDATA == cc->type(); + + if (ct.isWideDecimalType()) + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); + else + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); + + if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) + { + float f = cc->getFloatVal(); + value = *(reinterpret_cast(&f)); + } + else if (ct.colDataType == CalpontSystemCatalog::DOUBLE && !isNull) + { + double d = cc->getDoubleVal(); + value = *(reinterpret_cast(&d)); + } + + // @bug 2584, make "= null" to COMPARE_NIL. + if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) + cop = COMPARE_NIL; + + if (ct.isWideDecimalType()) + pcs->addFilter(cop, value128, rf); + else + pcs->addFilter(cop, value, rf); + } + } + + // save for expression transformation + pcs->addFilter(cf); + + sjstep.reset(pcs); + jsv.push_back(sjstep); + + // XXX use this before connector sets colType in sc correctly. + CalpontSystemCatalog::ColType ct = sc->colType(); + + if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) + { + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc.get(), alias)); + // X TupleInfo ti(setTupleInfo(sc->colType(), sc->oid(), jobInfo, tblOid, sc.get(), + // alias)); + pcs->tupleId(ti.key); } + } + else + { + cerr << boldStart << "doConstantFilter: Can only handle 'and' and 'or' right now, got '" << opStr << "'" + << boldStop << endl; + throw logic_error("doConstantFilter: Not handled operation type."); + } - return jsv; + return jsv; } - const JobStepVector doFunctionFilter(const ParseTree* n, JobInfo& jobInfo) { - FunctionColumn* fc = dynamic_cast(n->data()); - idbassert(fc); + FunctionColumn* fc = dynamic_cast(n->data()); + idbassert(fc); - JobStepVector jsv; - string functionName = ba::to_lower_copy(fc->functionName()); + JobStepVector jsv; + string functionName = ba::to_lower_copy(fc->functionName()); - if (functionName.compare("in") == 0 || functionName.compare(" in ") == 0) + if (functionName.compare("in") == 0 || functionName.compare(" in ") == 0) + { + const funcexp::FunctionParm& parms = fc->functionParms(); + FunctionColumn* parm0Fc = dynamic_cast(parms[0]->data()); + PseudoColumn* parm0Pc = dynamic_cast(parms[0]->data()); + vector > constParms(3); + uint64_t constParmsCount = 0; + + if (parm0Fc && parm0Fc->functionName().compare("idbpartition") == 0) { - const funcexp::FunctionParm& parms = fc->functionParms(); - FunctionColumn* parm0Fc = dynamic_cast(parms[0]->data()); - PseudoColumn* parm0Pc = dynamic_cast(parms[0]->data()); - vector > constParms(3); - uint64_t constParmsCount = 0; + for (uint64_t i = 1; i < parms.size(); i++) + { + ConstantColumn* cc = dynamic_cast(parms[i]->data()); - if (parm0Fc && parm0Fc->functionName().compare("idbpartition") == 0) + if (cc) { - for (uint64_t i = 1; i < parms.size(); i++) - { - ConstantColumn* cc = dynamic_cast(parms[i]->data()); + vector cv; + boost::split(cv, cc->constval(), boost::is_any_of(".")); - if (cc) - { - vector cv; - boost::split(cv, cc->constval(), boost::is_any_of(".")); - - if (cv.size() == 3) - { - constParms[1].push_back(cv[0]); - constParms[2].push_back(cv[1]); - constParms[0].push_back(cv[2]); - constParmsCount++; - } - } - } - - if (constParmsCount == (parms.size() - 1)) - { - const funcexp::FunctionParm& pcs = parm0Fc->functionParms(); - - for (uint64_t i = 0; i < 3; i++) - { - ConstantFilter* cf = new ConstantFilter(); - SOP sop(new LogicOperator("or")); - PseudoColumn* pc = dynamic_cast(pcs[i]->data()); - idbassert(pc); - SSC col(pc->clone()); - cf->op(sop); - cf->col(col); - sop.reset(new PredicateOperator("=")); - - for (uint64_t j = 0; j < constParmsCount; j++) - { - SimpleFilter* f = new SimpleFilter(sop, col->clone(), - new ConstantColumn(constParms[i][j])); - cf->pushFilter(f); - } - - JobStepVector sv = doConstantFilter(cf, jobInfo); - delete cf; - - jsv.insert(jsv.end(), sv.begin(), sv.end()); - } - } - - // put the separate filtered resulted together - JobStepVector sv = doExpressionFilter(n, jobInfo); - jsv.insert(jsv.end(), sv.begin(), sv.end()); + if (cv.size() == 3) + { + constParms[1].push_back(cv[0]); + constParms[2].push_back(cv[1]); + constParms[0].push_back(cv[2]); + constParmsCount++; + } } - else if (parm0Pc != NULL) + } + + if (constParmsCount == (parms.size() - 1)) + { + const funcexp::FunctionParm& pcs = parm0Fc->functionParms(); + + for (uint64_t i = 0; i < 3; i++) { - for (uint64_t i = 1; i < parms.size(); i++) - { - ConstantColumn* cc = dynamic_cast(parms[i]->data()); + ConstantFilter* cf = new ConstantFilter(); + SOP sop(new LogicOperator("or")); + PseudoColumn* pc = dynamic_cast(pcs[i]->data()); + idbassert(pc); + SSC col(pc->clone()); + cf->op(sop); + cf->col(col); + sop.reset(new PredicateOperator("=")); - if (cc) - { - constParms[0].push_back(cc->constval()); - constParmsCount++; - } - } + for (uint64_t j = 0; j < constParmsCount; j++) + { + SimpleFilter* f = new SimpleFilter(sop, col->clone(), new ConstantColumn(constParms[i][j])); + cf->pushFilter(f); + } - if (constParmsCount == (parms.size() - 1)) - { - ConstantFilter* cf = new ConstantFilter(); - SOP sop(new LogicOperator("or")); - SSC col(parm0Pc->clone()); - cf->op(sop); - cf->col(col); - sop.reset(new PredicateOperator("=")); + JobStepVector sv = doConstantFilter(cf, jobInfo); + delete cf; - for (uint64_t j = 0; j < constParmsCount; j++) - { - SimpleFilter* f = new SimpleFilter(sop, col->clone(), - new ConstantColumn(constParms[0][j])); - cf->pushFilter(f); - } - - JobStepVector sv = doConstantFilter(cf, jobInfo); - delete cf; - - jsv.insert(jsv.end(), sv.begin(), sv.end()); - } + jsv.insert(jsv.end(), sv.begin(), sv.end()); } + } + + // put the separate filtered resulted together + JobStepVector sv = doExpressionFilter(n, jobInfo); + jsv.insert(jsv.end(), sv.begin(), sv.end()); } + else if (parm0Pc != NULL) + { + for (uint64_t i = 1; i < parms.size(); i++) + { + ConstantColumn* cc = dynamic_cast(parms[i]->data()); - if (jsv.empty()) - jsv = doExpressionFilter(n, jobInfo); + if (cc) + { + constParms[0].push_back(cc->constval()); + constParmsCount++; + } + } - return jsv; + if (constParmsCount == (parms.size() - 1)) + { + ConstantFilter* cf = new ConstantFilter(); + SOP sop(new LogicOperator("or")); + SSC col(parm0Pc->clone()); + cf->op(sop); + cf->col(col); + sop.reset(new PredicateOperator("=")); + + for (uint64_t j = 0; j < constParmsCount; j++) + { + SimpleFilter* f = new SimpleFilter(sop, col->clone(), new ConstantColumn(constParms[0][j])); + cf->pushFilter(f); + } + + JobStepVector sv = doConstantFilter(cf, jobInfo); + delete cf; + + jsv.insert(jsv.end(), sv.begin(), sv.end()); + } + } + } + + if (jsv.empty()) + jsv = doExpressionFilter(n, jobInfo); + + return jsv; } - #if 0 void doAND(JobStepVector& jsv, JobInfo& jobInfo) { @@ -3245,327 +3144,310 @@ void doOR(const ParseTree* n, JobStepVector& jsv, JobInfo& jobInfo, bool tryComb } #endif - void doOR(ParseTree* n, JobInfo& jobInfo, bool tryCombine) { - JobStepVector jsv; + JobStepVector jsv; - // convert simple scalar filter sub to parse tree to be evaluated by expression + // convert simple scalar filter sub to parse tree to be evaluated by expression + { + // travering in post-order because a node may be expanded + ParseTree* node = n; + stack nodeStack; + ParseTree* lastVisit = NULL; + + while ((node || !nodeStack.empty())) { - // travering in post-order because a node may be expanded - ParseTree* node = n; - stack nodeStack; - ParseTree* lastVisit = NULL; + if (node) + { + nodeStack.push(node); + node = node->left(); + } + else + { + ParseTree* top = nodeStack.top(); + ParseTree* right = top->right(); - while ((node || !nodeStack.empty())) + if (right && lastVisit != right) { - if (node) - { - nodeStack.push(node); - node = node->left(); - } - else - { - ParseTree* top = nodeStack.top(); - ParseTree* right = top->right(); - - if (right && lastVisit != right) - { - node = right; - } - else - { - nodeStack.pop(); - - TreeNode* tn = top->data(); - - if (TreeNode2Type(tn) == SIMPLESCALARFILTER) - { - SimpleScalarFilter* sf = dynamic_cast(tn); - ParseTree* parseTree = NULL; - - if (simpleScalarFilterToParseTree(sf, parseTree, jobInfo)) - { - ParseTree* ccp = top; - delete ccp->data(); - ccp->left(parseTree->left()); - ccp->right(parseTree->right()); - ccp->data(parseTree->data()); - jobInfo.dynamicParseTreeVec.push_back(parseTree); - } - else if (parseTree) - { - delete parseTree; - parseTree = NULL; - } - } - - lastVisit = top; - } - } - } - } - - if (tryCombine) - { - // travering OR branch n in post-order iteratively - ParseTree* node = n; - stack nodeStack; - ParseTree* lastVisit = NULL; - bool okToCombine = true; - - while ((node || !nodeStack.empty()) && okToCombine) - { - if (node) - { - nodeStack.push(node); - node = node->left(); - } - else - { - ParseTree* top = nodeStack.top(); - ParseTree* right = top->right(); - - if (right && lastVisit != right) - { - node = right; - } - else - { - nodeStack.pop(); - lastVisit = top; - - TreeNode* tn = top->data(); - JobStepVector nsv; - - switch (TreeNode2Type(tn)) - { - case SIMPLEFILTER: - { - nsv = doSimpleFilter(dynamic_cast(tn), jobInfo); - break; - } - - case OPERATOR: - { - const Operator* op = static_cast(tn); - - if (*op == opAND || *op == opand || *op == opXOR || *op == opxor) - okToCombine = false; - else if (*op != opOR && *op != opor) - throw logic_error("doOR: unknow operator type."); - - break; - } - - case CONSTANTFILTER: - { - nsv = doConstantFilter(dynamic_cast(tn), jobInfo); - break; - } - - case SIMPLESCALARFILTER: - { - // not possible, should be converted in the block above - break; - } - - case OUTERJOINONFILTER: - case FUNCTIONCOLUMN: - case ARITHMETICCOLUMN: - case SIMPLECOLUMN: - case CONSTANTCOLUMN: - case EXISTSFILTER: - case SELECTFILTER: - { - okToCombine = false; - break; - } - - case UNKNOWN: - { - cerr << boldStart << "doOR: Unknown" << boldStop << endl; - throw logic_error("doOR: unknow type."); - break; - } - - default: - { - cerr << boldStart << "doOR: Not handled: " << TreeNode2Type(tn) - << boldStop << endl; - throw logic_error("doOR: Not handled treeNode type."); - break; - } - } - - if (nsv.size() > 0 && okToCombine) - { - if (jsv.empty()) - jsv = nsv; - else - okToCombine = tryCombineFilters(jsv, nsv, BOP_OR); - } - } - } - } - - if (!okToCombine) - jsv.clear(); - } - - if (jsv.empty()) - { - // OR is processed as an expression - jsv = doExpressionFilter(n, jobInfo); - } - - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); -} - - -} // end of unnamed namespace - - -namespace joblist -{ - -// This method is the entry point into the execution plan to joblist -// conversion performed by the functions in this file. -// @bug6131, pre-order traversing -/* static */ void -JLF_ExecPlanToJobList::walkTree(execplan::ParseTree* n, JobInfo& jobInfo) -{ - TreeNode* tn = n->data(); - JobStepVector jsv; - TreeNodeType tnType = TreeNode2Type(tn); - - switch (tnType) - { - case SIMPLEFILTER: - jsv = doSimpleFilter(dynamic_cast(tn), jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, true); - break; - - case OUTERJOINONFILTER: - jsv = doOuterJoinOnFilter(dynamic_cast(tn), jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, true); - break; - - case OPERATOR: - break; - - case CONSTANTFILTER: - jsv = doConstantFilter(dynamic_cast(tn), jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); - break; - - case FUNCTIONCOLUMN: - jsv = doFunctionFilter(n, jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); - break; - - case ARITHMETICCOLUMN: - jsv = doExpressionFilter(n, jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); - break; - - case SIMPLECOLUMN: - jsv = doExpressionFilter(n, jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); - break; - - case CONSTANTCOLUMN: - jsv = doConstantBooleanFilter(n, jobInfo); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); - break; - - case SIMPLESCALARFILTER: - doSimpleScalarFilter(n, jobInfo); - break; - - case EXISTSFILTER: - doExistsFilter(n, jobInfo); - break; - - case SELECTFILTER: - doSelectFilter(n, jobInfo); - break; - - case UNKNOWN: - cerr << boldStart << "walkTree: Unknown" << boldStop << endl; - throw logic_error("walkTree: unknow type."); - break; - - default: - /* - TREENODE, - FILTER, - RETURNEDCOLUMN, - AGGREGATECOLUMN, - TREENODEIMPL, - */ - cerr << boldStart << "walkTree: Not handled: " << TreeNode2Type(tn) << boldStop << endl; - throw logic_error("walkTree: Not handled treeNode type."); - break; - } - - //cout << *tn << endl; - - - if (tnType == OPERATOR) - { - const Operator* op = static_cast(tn); - - if (*op == opAND || *op == opand) - { - /*doAND(jsv, jobInfo)*/; - - if (n->left()) - walkTree(n->left(), jobInfo); - - if (n->right()) - walkTree(n->right(), jobInfo); - } - else if (*op == opOR || *op == opor) - { - doOR(n, jobInfo, true); - } - else if (*op == opXOR || *op == opxor) - { - doOR(n, jobInfo, false); + node = right; } else { - cerr << boldStart - << "walkTree: only know how to handle 'and' and 'or' right now, got: " << *op - << boldStop << endl; - throw logic_error("walkTree: unknow operator type."); + nodeStack.pop(); + + TreeNode* tn = top->data(); + + if (TreeNode2Type(tn) == SIMPLESCALARFILTER) + { + SimpleScalarFilter* sf = dynamic_cast(tn); + ParseTree* parseTree = NULL; + + if (simpleScalarFilterToParseTree(sf, parseTree, jobInfo)) + { + ParseTree* ccp = top; + delete ccp->data(); + ccp->left(parseTree->left()); + ccp->right(parseTree->right()); + ccp->data(parseTree->data()); + jobInfo.dynamicParseTreeVec.push_back(parseTree); + } + else if (parseTree) + { + delete parseTree; + parseTree = NULL; + } + } + + lastVisit = top; } + } } + } + + if (tryCombine) + { + // travering OR branch n in post-order iteratively + ParseTree* node = n; + stack nodeStack; + ParseTree* lastVisit = NULL; + bool okToCombine = true; + + while ((node || !nodeStack.empty()) && okToCombine) + { + if (node) + { + nodeStack.push(node); + node = node->left(); + } + else + { + ParseTree* top = nodeStack.top(); + ParseTree* right = top->right(); + + if (right && lastVisit != right) + { + node = right; + } + else + { + nodeStack.pop(); + lastVisit = top; + + TreeNode* tn = top->data(); + JobStepVector nsv; + + switch (TreeNode2Type(tn)) + { + case SIMPLEFILTER: + { + nsv = doSimpleFilter(dynamic_cast(tn), jobInfo); + break; + } + + case OPERATOR: + { + const Operator* op = static_cast(tn); + + if (*op == opAND || *op == opand || *op == opXOR || *op == opxor) + okToCombine = false; + else if (*op != opOR && *op != opor) + throw logic_error("doOR: unknow operator type."); + + break; + } + + case CONSTANTFILTER: + { + nsv = doConstantFilter(dynamic_cast(tn), jobInfo); + break; + } + + case SIMPLESCALARFILTER: + { + // not possible, should be converted in the block above + break; + } + + case OUTERJOINONFILTER: + case FUNCTIONCOLUMN: + case ARITHMETICCOLUMN: + case SIMPLECOLUMN: + case CONSTANTCOLUMN: + case EXISTSFILTER: + case SELECTFILTER: + { + okToCombine = false; + break; + } + + case UNKNOWN: + { + cerr << boldStart << "doOR: Unknown" << boldStop << endl; + throw logic_error("doOR: unknow type."); + break; + } + + default: + { + cerr << boldStart << "doOR: Not handled: " << TreeNode2Type(tn) << boldStop << endl; + throw logic_error("doOR: Not handled treeNode type."); + break; + } + } + + if (nsv.size() > 0 && okToCombine) + { + if (jsv.empty()) + jsv = nsv; + else + okToCombine = tryCombineFilters(jsv, nsv, BOP_OR); + } + } + } + } + + if (!okToCombine) + jsv.clear(); + } + + if (jsv.empty()) + { + // OR is processed as an expression + jsv = doExpressionFilter(n, jobInfo); + } + + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); } +} // end of unnamed namespace -void JLF_ExecPlanToJobList::addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bool tryCombine) +namespace joblist { - idbassert(jobInfo.stack.size() < 2); +// This method is the entry point into the execution plan to joblist +// conversion performed by the functions in this file. +// @bug6131, pre-order traversing +/* static */ void JLF_ExecPlanToJobList::walkTree(execplan::ParseTree* n, JobInfo& jobInfo) +{ + TreeNode* tn = n->data(); + JobStepVector jsv; + TreeNodeType tnType = TreeNode2Type(tn); - if (jobInfo.stack.size() > 0) + switch (tnType) + { + case SIMPLEFILTER: + jsv = doSimpleFilter(dynamic_cast(tn), jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, true); + break; + + case OUTERJOINONFILTER: + jsv = doOuterJoinOnFilter(dynamic_cast(tn), jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, true); + break; + + case OPERATOR: break; + + case CONSTANTFILTER: + jsv = doConstantFilter(dynamic_cast(tn), jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + break; + + case FUNCTIONCOLUMN: + jsv = doFunctionFilter(n, jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + break; + + case ARITHMETICCOLUMN: + jsv = doExpressionFilter(n, jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + break; + + case SIMPLECOLUMN: + jsv = doExpressionFilter(n, jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + break; + + case CONSTANTCOLUMN: + jsv = doConstantBooleanFilter(n, jobInfo); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + break; + + case SIMPLESCALARFILTER: doSimpleScalarFilter(n, jobInfo); break; + + case EXISTSFILTER: doExistsFilter(n, jobInfo); break; + + case SELECTFILTER: doSelectFilter(n, jobInfo); break; + + case UNKNOWN: + cerr << boldStart << "walkTree: Unknown" << boldStop << endl; + throw logic_error("walkTree: unknow type."); + break; + + default: + /* + TREENODE, + FILTER, + RETURNEDCOLUMN, + AGGREGATECOLUMN, + TREENODEIMPL, + */ + cerr << boldStart << "walkTree: Not handled: " << TreeNode2Type(tn) << boldStop << endl; + throw logic_error("walkTree: Not handled treeNode type."); + break; + } + + // cout << *tn << endl; + + if (tnType == OPERATOR) + { + const Operator* op = static_cast(tn); + + if (*op == opAND || *op == opand) { - JobStepVector& jsv = jobInfo.stack.top(); + /*doAND(jsv, jobInfo)*/; - if (tryCombine == false || tryCombineFilters(jsv, nsv, BOP_AND) == false) - jsv.insert(jsv.end(), nsv.begin(), nsv.end()); + if (n->left()) + walkTree(n->left(), jobInfo); + + if (n->right()) + walkTree(n->right(), jobInfo); + } + else if (*op == opOR || *op == opor) + { + doOR(n, jobInfo, true); + } + else if (*op == opXOR || *op == opxor) + { + doOR(n, jobInfo, false); } else { - jobInfo.stack.push(nsv); + cerr << boldStart << "walkTree: only know how to handle 'and' and 'or' right now, got: " << *op + << boldStop << endl; + throw logic_error("walkTree: unknow operator type."); } + } } +void JLF_ExecPlanToJobList::addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bool tryCombine) +{ + idbassert(jobInfo.stack.size() < 2); -} // end of joblist namespace + if (jobInfo.stack.size() > 0) + { + JobStepVector& jsv = jobInfo.stack.top(); + + if (tryCombine == false || tryCombineFilters(jsv, nsv, BOP_AND) == false) + jsv.insert(jsv.end(), nsv.begin(), nsv.end()); + } + else + { + jobInfo.stack.push(nsv); + } +} + +} // namespace joblist // vim:ts=4 sw=4: #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif diff --git a/dbcon/joblist/jlf_execplantojoblist.h b/dbcon/joblist/jlf_execplantojoblist.h index 6f9fb5daf..eae4b8617 100644 --- a/dbcon/joblist/jlf_execplantojoblist.h +++ b/dbcon/joblist/jlf_execplantojoblist.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: jlf_execplantojoblist.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: jlf_execplantojoblist.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file jlf_execplantojoblist.h * @@ -31,12 +31,10 @@ #include "calpontselectexecutionplan.h" #include "calpontsystemcatalog.h" - #include "jlf_common.h" namespace joblist { - //------------------------------------------------------------------------------ /** @brief Class makes initial pass at converting Calpont Exec Plan to a joblist * @@ -44,27 +42,27 @@ namespace joblist //------------------------------------------------------------------------------ class JLF_ExecPlanToJobList { -public: - /** @brief This function is the entry point into CEP to joblist conversion - * - * @param ParseTree (in) is CEP to be translated to a joblist - * @param JobInfo& (in/out) is the JobInfo reference that is loaded - */ - static void walkTree(execplan::ParseTree* n, JobInfo& jobInfo); + public: + /** @brief This function is the entry point into CEP to joblist conversion + * + * @param ParseTree (in) is CEP to be translated to a joblist + * @param JobInfo& (in/out) is the JobInfo reference that is loaded + */ + static void walkTree(execplan::ParseTree* n, JobInfo& jobInfo); - /** @brief This function add new job steps to the job step vector in JobInfo - * - * @param JobStepVector& (in) is a vector of new job steps - * @param JobInfo& (in/out) is the JobInfo reference that is loaded - * @param bool (in) is combine job step possible - */ - static void addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bool tryCombine); + /** @brief This function add new job steps to the job step vector in JobInfo + * + * @param JobStepVector& (in) is a vector of new job steps + * @param JobInfo& (in/out) is the JobInfo reference that is loaded + * @param bool (in) is combine job step possible + */ + static void addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bool tryCombine); -private: - // Disable constructor since this class only contains a static method - JLF_ExecPlanToJobList(); + private: + // Disable constructor since this class only contains a static method + JLF_ExecPlanToJobList(); }; -} // end of joblist namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/jlf_graphics.cpp b/dbcon/joblist/jlf_graphics.cpp index f01f695b2..7365e1c4e 100644 --- a/dbcon/joblist/jlf_graphics.cpp +++ b/dbcon/joblist/jlf_graphics.cpp @@ -35,390 +35,391 @@ using namespace joblist; #include "jlf_graphics.h" - #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" - // for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to 'typeid' +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" +// for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to +// 'typeid' #endif - namespace jlf_graphics { - ostream& writeDotCmds(ostream& dotFile, const JobStepVector& query, const JobStepVector& project) { - // Graphic view draw - dotFile << "digraph G {" << endl; - JobStepVector::iterator qsi; - JobStepVector::iterator psi; - int ctn = 0; + // Graphic view draw + dotFile << "digraph G {" << endl; + JobStepVector::iterator qsi; + JobStepVector::iterator psi; + int ctn = 0; - // merge in the subquery steps - JobStepVector querySteps = query; - JobStepVector projectSteps = project; + // merge in the subquery steps + JobStepVector querySteps = query; + JobStepVector projectSteps = project; + { + SubQueryStep* subquery = NULL; + qsi = querySteps.begin(); + + while (qsi != querySteps.end()) { - SubQueryStep* subquery = NULL; + if ((subquery = dynamic_cast(qsi->get())) != NULL) + { + querySteps.erase(qsi); + JobStepVector subSteps = subquery->subJoblist()->querySteps(); + querySteps.insert(querySteps.end(), subSteps.begin(), subSteps.end()); qsi = querySteps.begin(); - - while (qsi != querySteps.end()) - { - if ((subquery = dynamic_cast(qsi->get())) != NULL) - { - querySteps.erase(qsi); - JobStepVector subSteps = subquery->subJoblist()->querySteps(); - querySteps.insert(querySteps.end(), subSteps.begin(), subSteps.end()); - qsi = querySteps.begin(); - } - else - { - qsi++; - } - } + } + else + { + qsi++; + } } + } - for (qsi = querySteps.begin(); qsi != querySteps.end(); ctn++, qsi++) + for (qsi = querySteps.begin(); qsi != querySteps.end(); ctn++, qsi++) + { + // if (dynamic_cast(qsi->get()) != NULL) + // continue; + + uint16_t stepidIn = qsi->get()->stepId(); + dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; + + if (typeid(*(qsi->get())) == typeid(pColStep)) { -// if (dynamic_cast(qsi->get()) != NULL) -// continue; - - uint16_t stepidIn = qsi->get()->stepId(); - dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; - - if (typeid(*(qsi->get())) == typeid(pColStep)) - { - dotFile << "(" << qsi->get()->tableOid() << "/" << qsi->get()->oid() << ")" << "\""; - dotFile << " shape=ellipse"; - } - else if (typeid(*(qsi->get())) == typeid(pColScanStep)) - { - dotFile << "(" << qsi->get()->tableOid() << "/" << qsi->get()->oid() << ")" << "\""; - dotFile << " shape=box"; - } -// else if (typeid(*(qsi->get())) == typeid(HashJoinStep) || -// typeid(*(qsi->get())) == typeid(StringHashJoinStep)) -// { -// dotFile << "\""; -// dotFile << " shape=diamond"; -// } - else if (typeid(*(qsi->get())) == typeid(TupleHashJoinStep)) - { - dotFile << "\""; - dotFile << " shape=diamond peripheries=2"; - } -// else if (typeid(*(qsi->get())) == typeid(UnionStep) || -// typeid(*(qsi->get())) == typeid(TupleUnion) ) - else if (typeid(*(qsi->get())) == typeid(TupleUnion)) - { - dotFile << "\""; - dotFile << " shape=triangle"; - } - else if (typeid(*(qsi->get())) == typeid(pDictionaryStep)) - { - dotFile << "\""; - dotFile << " shape=trapezium"; - } - else if (typeid(*(qsi->get())) == typeid(FilterStep)) - { - dotFile << "\""; - dotFile << " shape=house orientation=180"; - } -// else if (typeid(*(qsi->get())) == typeid(ReduceStep)) -// { -// dotFile << "\""; -// dotFile << " shape=triangle orientation=180"; -// } -// else if (typeid(*(qsi->get())) == typeid(BatchPrimitiveStep) || typeid(*(qsi->get())) == typeid(TupleBPS)) - else if (typeid(*(qsi->get())) == typeid(TupleBPS)) - { - bool isTuple = false; - BatchPrimitive* bps = dynamic_cast(qsi->get()); - - if (dynamic_cast(bps) != 0) - isTuple = true; - - dotFile << "(" << bps->tableOid() << "/" << bps->oid() ; - OIDVector projectOids = bps->getProjectOids(); - - if ( projectOids.size() > 0 ) - { - dotFile << "\\l"; - dotFile << "PC: "; - } - - for ( unsigned int i = 0; i < projectOids.size(); i++ ) - { - dotFile << projectOids[i] << " "; - - if ( (i + 1) % 3 == 0 ) - dotFile << "\\l"; - } - - dotFile << ")\""; - dotFile << " shape=box style=bold"; - - if (isTuple) - dotFile << " peripheries=2"; - } - else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) - { - BatchPrimitive* bps = dynamic_cast(qsi->get()); - dotFile << "(" << bps->alias() << ")\""; - dotFile << " shape=box style=bold"; - dotFile << " peripheries=2"; - } - else if (typeid(*(qsi->get())) == typeid(TupleAggregateStep)) - { - dotFile << "\""; - dotFile << " shape=triangle orientation=180"; - } - else if (typeid(*(qsi->get())) == typeid(TupleAnnexStep)) - { - dotFile << "\""; - dotFile << " shape=star"; - } - else if (typeid(*(qsi->get())) == typeid(WindowFunctionStep)) - { - dotFile << "\""; - dotFile << " shape=triangle orientation=180"; - dotFile << " peripheries=2"; - } -// else if (typeid(*(qsi->get())) == typeid(AggregateFilterStep)) -// { -// dotFile << "\""; -// dotFile << " shape=hexagon peripheries=2 style=bold"; -// } -// else if (typeid(*(qsi->get())) == typeid(BucketReuseStep)) -// { -// dotFile << "(" << qsi->get()->tableOid() << "/" << qsi->get()->oid() << ")" << "\""; -// dotFile << " shape=box style=dashed"; -// } - else - dotFile << "\""; - - dotFile << "]" << endl; - - for (unsigned int i = 0; i < qsi->get()->outputAssociation().outSize(); i++) - { - ptrdiff_t dloutptr; - DataList_t* dlout = qsi->get()->outputAssociation().outAt(i)->dataList(); - uint32_t numConsumers = qsi->get()->outputAssociation().outAt(i)->getNumConsumers(); - - if (dlout) - { - dloutptr = (ptrdiff_t)dlout; - } - else - { - StrDataList* sdl = qsi->get()->outputAssociation().outAt(i)->stringDataList(); - dloutptr = (ptrdiff_t)sdl; - } - - for (unsigned int k = 0; k < querySteps.size(); k++) - { - uint16_t stepidOut = querySteps[k].get()->stepId(); - JobStepAssociation queryInputSA = querySteps[k].get()->inputAssociation(); - - for (unsigned int j = 0; j < queryInputSA.outSize(); j++) - { - ptrdiff_t dlinptr; - DataList_t* dlin = queryInputSA.outAt(j)->dataList(); - StrDataList* sdl = 0; - - if (dlin) - dlinptr = (ptrdiff_t)dlin; - else - { - sdl = queryInputSA.outAt(j)->stringDataList(); - dlinptr = (ptrdiff_t)sdl; - } - - if ((ptrdiff_t)dloutptr == (ptrdiff_t)dlinptr) - { - dotFile << stepidIn << " -> " << stepidOut; - - if (dlin) - { - dotFile << " [label=\"[" << AnyDataList::dlType(dlin) - << "/" << numConsumers << "]\"]" << endl; - } - else - { - dotFile << " [label=\"[" << AnyDataList::strDlType(sdl) - << "/" << numConsumers << "]\"]" << endl; - } - } - } - } - - for (psi = projectSteps.begin(); psi < projectSteps.end(); psi++) - { - uint16_t stepidOut = psi->get()->stepId(); - JobStepAssociation projectInputSA = psi->get()->inputAssociation(); - - for (unsigned int j = 0; j < projectInputSA.outSize(); j++) - { - ptrdiff_t dlinptr; - DataList_t* dlin = projectInputSA.outAt(j)->dataList(); - StrDataList* sdl = 0; - - if (dlin) - dlinptr = (ptrdiff_t)dlin; - else - { - sdl = projectInputSA.outAt(j)->stringDataList(); - dlinptr = (ptrdiff_t)sdl; - } - - if (dloutptr == dlinptr) - //if ((ptrdiff_t)dlout == (ptrdiff_t)dlin) - { - dotFile << stepidIn << " -> " << stepidOut; - - if (dlin) - { - dotFile << " [label=\"[" << AnyDataList::dlType(dlin) - << "/" << numConsumers << "]\"]" << endl; - } - else - { - dotFile << " [label=\"[" << AnyDataList::strDlType(sdl) - << "/" << numConsumers << "]\"]" << endl; - } - } - } - } - } + dotFile << "(" << qsi->get()->tableOid() << "/" << qsi->get()->oid() << ")" + << "\""; + dotFile << " shape=ellipse"; } - - for (psi = projectSteps.begin(), ctn = 0; psi != projectSteps.end(); ctn++, psi++) + else if (typeid(*(qsi->get())) == typeid(pColScanStep)) { - uint16_t stepidIn = psi->get()->stepId(); - dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; + dotFile << "(" << qsi->get()->tableOid() << "/" << qsi->get()->oid() << ")" + << "\""; + dotFile << " shape=box"; + } + // else if (typeid(*(qsi->get())) == typeid(HashJoinStep) || + // typeid(*(qsi->get())) == typeid(StringHashJoinStep)) + // { + // dotFile << "\""; + // dotFile << " shape=diamond"; + // } + else if (typeid(*(qsi->get())) == typeid(TupleHashJoinStep)) + { + dotFile << "\""; + dotFile << " shape=diamond peripheries=2"; + } + // else if (typeid(*(qsi->get())) == typeid(UnionStep) || + // typeid(*(qsi->get())) == typeid(TupleUnion) ) + else if (typeid(*(qsi->get())) == typeid(TupleUnion)) + { + dotFile << "\""; + dotFile << " shape=triangle"; + } + else if (typeid(*(qsi->get())) == typeid(pDictionaryStep)) + { + dotFile << "\""; + dotFile << " shape=trapezium"; + } + else if (typeid(*(qsi->get())) == typeid(FilterStep)) + { + dotFile << "\""; + dotFile << " shape=house orientation=180"; + } + // else if (typeid(*(qsi->get())) == typeid(ReduceStep)) + // { + // dotFile << "\""; + // dotFile << " shape=triangle orientation=180"; + // } + // else if (typeid(*(qsi->get())) == typeid(BatchPrimitiveStep) || typeid(*(qsi->get())) == + //typeid(TupleBPS)) + else if (typeid(*(qsi->get())) == typeid(TupleBPS)) + { + bool isTuple = false; + BatchPrimitive* bps = dynamic_cast(qsi->get()); - if (typeid(*(psi->get())) == typeid(pColStep)) - { - dotFile << "(" << psi->get()->tableOid() << "/" << psi->get()->oid() << ")" << "\""; - dotFile << " shape=ellipse"; - } - else if (typeid(*(psi->get())) == typeid(pColScanStep)) - { - dotFile << "(" << psi->get()->tableOid() << "/" << psi->get()->oid() << ")" << "\""; - dotFile << " shape=box"; - } - else if (typeid(*(psi->get())) == typeid(pDictionaryStep)) - { - dotFile << "\""; - dotFile << " shape=trapezium"; - } - else if (typeid(*(psi->get())) == typeid(PassThruStep)) - { - dotFile << "(" << psi->get()->tableOid() << "/" << psi->get()->oid() << ")" << "\""; - dotFile << " shape=octagon"; - } -// else if (typeid(*(psi->get())) == typeid(BatchPrimitiveStep) || typeid(*(psi->get())) == typeid(TupleBPS)) - else if (typeid(*(psi->get())) == typeid(TupleBPS)) - { - bool isTuple = false; - BatchPrimitive* bps = dynamic_cast(psi->get()); + if (dynamic_cast(bps) != 0) + isTuple = true; - if (dynamic_cast(bps) != 0) - isTuple = true; + dotFile << "(" << bps->tableOid() << "/" << bps->oid(); + OIDVector projectOids = bps->getProjectOids(); - dotFile << "(" << bps->tableOid() << ":\\l"; - OIDVector projectOids = bps->getProjectOids(); + if (projectOids.size() > 0) + { + dotFile << "\\l"; + dotFile << "PC: "; + } - for ( unsigned int i = 0; i < projectOids.size(); i++ ) + for (unsigned int i = 0; i < projectOids.size(); i++) + { + dotFile << projectOids[i] << " "; + + if ((i + 1) % 3 == 0) + dotFile << "\\l"; + } + + dotFile << ")\""; + dotFile << " shape=box style=bold"; + + if (isTuple) + dotFile << " peripheries=2"; + } + else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) + { + BatchPrimitive* bps = dynamic_cast(qsi->get()); + dotFile << "(" << bps->alias() << ")\""; + dotFile << " shape=box style=bold"; + dotFile << " peripheries=2"; + } + else if (typeid(*(qsi->get())) == typeid(TupleAggregateStep)) + { + dotFile << "\""; + dotFile << " shape=triangle orientation=180"; + } + else if (typeid(*(qsi->get())) == typeid(TupleAnnexStep)) + { + dotFile << "\""; + dotFile << " shape=star"; + } + else if (typeid(*(qsi->get())) == typeid(WindowFunctionStep)) + { + dotFile << "\""; + dotFile << " shape=triangle orientation=180"; + dotFile << " peripheries=2"; + } + // else if (typeid(*(qsi->get())) == typeid(AggregateFilterStep)) + // { + // dotFile << "\""; + // dotFile << " shape=hexagon peripheries=2 style=bold"; + // } + // else if (typeid(*(qsi->get())) == typeid(BucketReuseStep)) + // { + // dotFile << "(" << qsi->get()->tableOid() << "/" << qsi->get()->oid() << ")" << "\""; + // dotFile << " shape=box style=dashed"; + // } + else + dotFile << "\""; + + dotFile << "]" << endl; + + for (unsigned int i = 0; i < qsi->get()->outputAssociation().outSize(); i++) + { + ptrdiff_t dloutptr; + DataList_t* dlout = qsi->get()->outputAssociation().outAt(i)->dataList(); + uint32_t numConsumers = qsi->get()->outputAssociation().outAt(i)->getNumConsumers(); + + if (dlout) + { + dloutptr = (ptrdiff_t)dlout; + } + else + { + StrDataList* sdl = qsi->get()->outputAssociation().outAt(i)->stringDataList(); + dloutptr = (ptrdiff_t)sdl; + } + + for (unsigned int k = 0; k < querySteps.size(); k++) + { + uint16_t stepidOut = querySteps[k].get()->stepId(); + JobStepAssociation queryInputSA = querySteps[k].get()->inputAssociation(); + + for (unsigned int j = 0; j < queryInputSA.outSize(); j++) + { + ptrdiff_t dlinptr; + DataList_t* dlin = queryInputSA.outAt(j)->dataList(); + StrDataList* sdl = 0; + + if (dlin) + dlinptr = (ptrdiff_t)dlin; + else + { + sdl = queryInputSA.outAt(j)->stringDataList(); + dlinptr = (ptrdiff_t)sdl; + } + + if ((ptrdiff_t)dloutptr == (ptrdiff_t)dlinptr) + { + dotFile << stepidIn << " -> " << stepidOut; + + if (dlin) { - dotFile << projectOids[i] << " "; - - if ( (i + 1) % 3 == 0 ) - dotFile << "\\l"; - } - - dotFile << ")\"" ; - dotFile << " shape=box style=bold"; - - if (isTuple) - dotFile << " peripheries=2"; - } - else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) - { - BatchPrimitive* bps = dynamic_cast(qsi->get()); - dotFile << "(" << bps->alias() << ")\""; - dotFile << " shape=box style=bold"; - dotFile << " peripheries=2"; - } - else - dotFile << "\""; - - dotFile << "]" << endl; - - for (unsigned int i = 0; i < psi->get()->outputAssociation().outSize(); i++) - { - ptrdiff_t dloutptr; - DataList_t* dlout = psi->get()->outputAssociation().outAt(i)->dataList(); - uint32_t numConsumers = psi->get()->outputAssociation().outAt(i)->getNumConsumers(); - - if (dlout) - { - dloutptr = (ptrdiff_t)dlout; + dotFile << " [label=\"[" << AnyDataList::dlType(dlin) << "/" << numConsumers << "]\"]" << endl; } else { - StrDataList* sdl = psi->get()->outputAssociation().outAt(i)->stringDataList(); - dloutptr = (ptrdiff_t)sdl; - } - - for (unsigned int k = ctn + 1; k < projectSteps.size(); k++) - { - uint16_t stepidOut = projectSteps[k].get()->stepId(); - JobStepAssociation projectInputSA = projectSteps[k].get()->inputAssociation(); - - for (unsigned int j = 0; j < projectInputSA.outSize(); j++) - { - ptrdiff_t dlinptr; - DataList_t* dlin = projectInputSA.outAt(j)->dataList(); - StrDataList* sdl = 0; - - if (dlin) - dlinptr = (ptrdiff_t)dlin; - else - { - sdl = projectInputSA.outAt(j)->stringDataList(); - dlinptr = (ptrdiff_t)sdl; - } - - if ((ptrdiff_t)dloutptr == (ptrdiff_t)dlinptr) - { - dotFile << stepidIn << " -> " << stepidOut; - - if (dlin) - { - dotFile << " [label=\"[" << AnyDataList::dlType(dlin) - << "/" << numConsumers << "]\"]" << endl; - } - else - { - dotFile << " [label=\"[" << AnyDataList::strDlType(sdl) - << "/" << numConsumers << "]\"]" << endl; - } - } - } + dotFile << " [label=\"[" << AnyDataList::strDlType(sdl) << "/" << numConsumers << "]\"]" + << endl; } + } } + } + + for (psi = projectSteps.begin(); psi < projectSteps.end(); psi++) + { + uint16_t stepidOut = psi->get()->stepId(); + JobStepAssociation projectInputSA = psi->get()->inputAssociation(); + + for (unsigned int j = 0; j < projectInputSA.outSize(); j++) + { + ptrdiff_t dlinptr; + DataList_t* dlin = projectInputSA.outAt(j)->dataList(); + StrDataList* sdl = 0; + + if (dlin) + dlinptr = (ptrdiff_t)dlin; + else + { + sdl = projectInputSA.outAt(j)->stringDataList(); + dlinptr = (ptrdiff_t)sdl; + } + + if (dloutptr == dlinptr) + // if ((ptrdiff_t)dlout == (ptrdiff_t)dlin) + { + dotFile << stepidIn << " -> " << stepidOut; + + if (dlin) + { + dotFile << " [label=\"[" << AnyDataList::dlType(dlin) << "/" << numConsumers << "]\"]" << endl; + } + else + { + dotFile << " [label=\"[" << AnyDataList::strDlType(sdl) << "/" << numConsumers << "]\"]" + << endl; + } + } + } + } } + } - dotFile << "}" << endl; + for (psi = projectSteps.begin(), ctn = 0; psi != projectSteps.end(); ctn++, psi++) + { + uint16_t stepidIn = psi->get()->stepId(); + dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; - return dotFile; + if (typeid(*(psi->get())) == typeid(pColStep)) + { + dotFile << "(" << psi->get()->tableOid() << "/" << psi->get()->oid() << ")" + << "\""; + dotFile << " shape=ellipse"; + } + else if (typeid(*(psi->get())) == typeid(pColScanStep)) + { + dotFile << "(" << psi->get()->tableOid() << "/" << psi->get()->oid() << ")" + << "\""; + dotFile << " shape=box"; + } + else if (typeid(*(psi->get())) == typeid(pDictionaryStep)) + { + dotFile << "\""; + dotFile << " shape=trapezium"; + } + else if (typeid(*(psi->get())) == typeid(PassThruStep)) + { + dotFile << "(" << psi->get()->tableOid() << "/" << psi->get()->oid() << ")" + << "\""; + dotFile << " shape=octagon"; + } + // else if (typeid(*(psi->get())) == typeid(BatchPrimitiveStep) || typeid(*(psi->get())) == + //typeid(TupleBPS)) + else if (typeid(*(psi->get())) == typeid(TupleBPS)) + { + bool isTuple = false; + BatchPrimitive* bps = dynamic_cast(psi->get()); + if (dynamic_cast(bps) != 0) + isTuple = true; + + dotFile << "(" << bps->tableOid() << ":\\l"; + OIDVector projectOids = bps->getProjectOids(); + + for (unsigned int i = 0; i < projectOids.size(); i++) + { + dotFile << projectOids[i] << " "; + + if ((i + 1) % 3 == 0) + dotFile << "\\l"; + } + + dotFile << ")\""; + dotFile << " shape=box style=bold"; + + if (isTuple) + dotFile << " peripheries=2"; + } + else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) + { + BatchPrimitive* bps = dynamic_cast(qsi->get()); + dotFile << "(" << bps->alias() << ")\""; + dotFile << " shape=box style=bold"; + dotFile << " peripheries=2"; + } + else + dotFile << "\""; + + dotFile << "]" << endl; + + for (unsigned int i = 0; i < psi->get()->outputAssociation().outSize(); i++) + { + ptrdiff_t dloutptr; + DataList_t* dlout = psi->get()->outputAssociation().outAt(i)->dataList(); + uint32_t numConsumers = psi->get()->outputAssociation().outAt(i)->getNumConsumers(); + + if (dlout) + { + dloutptr = (ptrdiff_t)dlout; + } + else + { + StrDataList* sdl = psi->get()->outputAssociation().outAt(i)->stringDataList(); + dloutptr = (ptrdiff_t)sdl; + } + + for (unsigned int k = ctn + 1; k < projectSteps.size(); k++) + { + uint16_t stepidOut = projectSteps[k].get()->stepId(); + JobStepAssociation projectInputSA = projectSteps[k].get()->inputAssociation(); + + for (unsigned int j = 0; j < projectInputSA.outSize(); j++) + { + ptrdiff_t dlinptr; + DataList_t* dlin = projectInputSA.outAt(j)->dataList(); + StrDataList* sdl = 0; + + if (dlin) + dlinptr = (ptrdiff_t)dlin; + else + { + sdl = projectInputSA.outAt(j)->stringDataList(); + dlinptr = (ptrdiff_t)sdl; + } + + if ((ptrdiff_t)dloutptr == (ptrdiff_t)dlinptr) + { + dotFile << stepidIn << " -> " << stepidOut; + + if (dlin) + { + dotFile << " [label=\"[" << AnyDataList::dlType(dlin) << "/" << numConsumers << "]\"]" << endl; + } + else + { + dotFile << " [label=\"[" << AnyDataList::strDlType(sdl) << "/" << numConsumers << "]\"]" + << endl; + } + } + } + } + } + } + + dotFile << "}" << endl; + + return dotFile; } -} // end namespace jlf_graphics +} // end namespace jlf_graphics // vim:ts=4 sw=4 syntax=cpp: #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif diff --git a/dbcon/joblist/jlf_graphics.h b/dbcon/joblist/jlf_graphics.h index 049eb98a1..f67a55614 100644 --- a/dbcon/joblist/jlf_graphics.h +++ b/dbcon/joblist/jlf_graphics.h @@ -28,14 +28,12 @@ namespace jlf_graphics { - /** Format a stream of dot commands -* Format a stream of dot commands -*/ + * Format a stream of dot commands + */ std::ostream& writeDotCmds(std::ostream& dotFile, const joblist::JobStepVector& querySteps, const joblist::JobStepVector& projectSteps); -} +} // namespace jlf_graphics #endif - diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 106c9d6ab..5b612d4ae 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -18,7 +18,6 @@ // $Id: jlf_subquery.cpp 6419 2010-03-30 04:28:32Z xlou $ - #include #include #include @@ -63,288 +62,280 @@ using namespace logging; #include "jlf_subquery.h" using namespace joblist; - namespace { - void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row, const string& timeZone) { - ostringstream oss; - int64_t data = 0; + ostringstream oss; + int64_t data = 0; - switch (row.getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - if (row.getScale(i) == 0) - { - oss << row.getIntField(i); - *cc = new ConstantColumn(oss.str(), row.getIntField(i)); - break; - } - /* fall through */ - /* else > 0 */ + switch (row.getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + if (row.getScale(i) == 0) + { + oss << row.getIntField(i); + *cc = new ConstantColumn(oss.str(), row.getIntField(i)); + break; + } + /* fall through */ + /* else > 0 */ - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (row.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) - { - int128_t val; - row.getInt128Field(i, val); + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (row.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + int128_t val; + row.getInt128Field(i, val); - IDB_Decimal dec(0, row.getScale(i), row.getPrecision(i), val); + IDB_Decimal dec(0, row.getScale(i), row.getPrecision(i), val); - *cc = new ConstantColumn(dec.toString(true), dec); - } - else - { - data = row.getIntField(i); + *cc = new ConstantColumn(dec.toString(true), dec); + } + else + { + data = row.getIntField(i); - IDB_Decimal dec(data, row.getScale(i), row.getPrecision(i)); + IDB_Decimal dec(data, row.getScale(i), row.getPrecision(i)); - *cc = new ConstantColumn(dec.toString(), dec); - } - break; + *cc = new ConstantColumn(dec.toString(), dec); + } + break; - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - oss << row.getUintField(i); - *cc = new ConstantColumn(oss.str(), row.getUintField(i)); - break; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + oss << row.getUintField(i); + *cc = new ConstantColumn(oss.str(), row.getUintField(i)); + break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - oss << fixed << row.getFloatField(i); - *cc = new ConstantColumn(oss.str(), (double) row.getFloatField(i)); - break; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + oss << fixed << row.getFloatField(i); + *cc = new ConstantColumn(oss.str(), (double)row.getFloatField(i)); + break; - case CalpontSystemCatalog::DOUBLE: - oss << fixed << row.getDoubleField(i); - *cc = new ConstantColumn(oss.str(), row.getDoubleField(i)); - break; + case CalpontSystemCatalog::DOUBLE: + oss << fixed << row.getDoubleField(i); + *cc = new ConstantColumn(oss.str(), row.getDoubleField(i)); + break; - case CalpontSystemCatalog::LONGDOUBLE: - oss << fixed << row.getLongDoubleField(i); - *cc = new ConstantColumn(oss.str(), row.getLongDoubleField(i)); - break; + case CalpontSystemCatalog::LONGDOUBLE: + oss << fixed << row.getLongDoubleField(i); + *cc = new ConstantColumn(oss.str(), row.getLongDoubleField(i)); + break; - case CalpontSystemCatalog::DATE: - oss << dataconvert::DataConvert::dateToString(row.getUintField<4>(i)); - *cc = new ConstantColumn(oss.str()); - break; + case CalpontSystemCatalog::DATE: + oss << dataconvert::DataConvert::dateToString(row.getUintField<4>(i)); + *cc = new ConstantColumn(oss.str()); + break; - case CalpontSystemCatalog::DATETIME: - oss << dataconvert::DataConvert::datetimeToString(row.getUintField<8>(i)); - *cc = new ConstantColumn(oss.str()); - break; + case CalpontSystemCatalog::DATETIME: + oss << dataconvert::DataConvert::datetimeToString(row.getUintField<8>(i)); + *cc = new ConstantColumn(oss.str()); + break; - case CalpontSystemCatalog::TIMESTAMP: - oss << dataconvert::DataConvert::timestampToString(row.getUintField<8>(i), timeZone); - *cc = new ConstantColumn(oss.str()); - break; + case CalpontSystemCatalog::TIMESTAMP: + oss << dataconvert::DataConvert::timestampToString(row.getUintField<8>(i), timeZone); + *cc = new ConstantColumn(oss.str()); + break; - case CalpontSystemCatalog::TIME: - oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); - *cc = new ConstantColumn(oss.str()); - break; + case CalpontSystemCatalog::TIME: + oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); + *cc = new ConstantColumn(oss.str()); + break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::BLOB: - oss << (char*) (row.getStringField(i).c_str()); - *cc = new ConstantColumn(oss.str()); - break; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::BLOB: + oss << (char*)(row.getStringField(i).c_str()); + *cc = new ConstantColumn(oss.str()); + break; - default: - oss << "Unsupported data type: " << row.getColTypes()[i]; - throw QueryDataExcept(oss.str(), dataTypeErr); - } + default: + oss << "Unsupported data type: " << row.getColTypes()[i]; + throw QueryDataExcept(oss.str(), dataTypeErr); + } } - void sfInHaving(ParseTree* pt, void*) { - SelectFilter* sf = dynamic_cast(pt->data()); + SelectFilter* sf = dynamic_cast(pt->data()); - if (sf != NULL) - throw IDBExcept(ERR_NON_SUPPORT_HAVING); + if (sf != NULL) + throw IDBExcept(ERR_NON_SUPPORT_HAVING); } - void ssfInHaving(ParseTree* pt, void* obj) { - JobInfo* jobInfo = reinterpret_cast(obj); - SimpleScalarFilter* ssf = dynamic_cast(pt->data()); + JobInfo* jobInfo = reinterpret_cast(obj); + SimpleScalarFilter* ssf = dynamic_cast(pt->data()); - if (ssf != NULL) + if (ssf != NULL) + { + ParseTree* parseTree = NULL; + + if (simpleScalarFilterToParseTree(ssf, parseTree, *jobInfo)) { - ParseTree* parseTree = NULL; + // replace simple scalar filter with simple filters + delete pt->data(); + pt->left(parseTree->left()); + pt->right(parseTree->right()); + pt->data(parseTree->data()); - if (simpleScalarFilterToParseTree(ssf, parseTree, *jobInfo)) - { - // replace simple scalar filter with simple filters - delete pt->data(); - pt->left(parseTree->left()); - pt->right(parseTree->right()); - pt->data(parseTree->data()); - - jobInfo->dynamicParseTreeVec.push_back(parseTree); - // don't delete the parseTree, it has been placed in the plan. - // Instead, we use the dynamicParseTreeVec above for deletion - // in ~csep() or csep.unserialize(). - // delete parseTree; - } - else - { - // not a scalar result - // replace simple scalar filter with simple filters - delete pt->data(); - pt->data(NULL); - delete parseTree; - jobInfo->constantFalse = true; - } + jobInfo->dynamicParseTreeVec.push_back(parseTree); + // don't delete the parseTree, it has been placed in the plan. + // Instead, we use the dynamicParseTreeVec above for deletion + // in ~csep() or csep.unserialize(). + // delete parseTree; } + else + { + // not a scalar result + // replace simple scalar filter with simple filters + delete pt->data(); + pt->data(NULL); + delete parseTree; + jobInfo->constantFalse = true; + } + } } - void getCorrelatedFilters(ParseTree* pt, void* obj) { - SimpleFilter* sf = dynamic_cast(pt->data()); + SimpleFilter* sf = dynamic_cast(pt->data()); - if (sf != NULL) + if (sf != NULL) + { + ReturnedColumn* rc1 = dynamic_cast(sf->lhs()); + ReturnedColumn* rc2 = dynamic_cast(sf->rhs()); + + bool correlated = false; + + if (rc1 != NULL && rc1->joinInfo() != 0) + correlated = true; + + if (rc2 != NULL && rc2->joinInfo() != 0) + correlated = true; + + if (correlated) { - ReturnedColumn* rc1 = dynamic_cast(sf->lhs()); - ReturnedColumn* rc2 = dynamic_cast(sf->rhs()); + ParseTree** correlatedFilters = reinterpret_cast(obj); - bool correlated = false; + if (*correlatedFilters == NULL) + { + *correlatedFilters = new ParseTree(sf); + } + else + { + ParseTree* left = *correlatedFilters; + *correlatedFilters = new ParseTree(new LogicOperator("and")); + (*correlatedFilters)->left(left); + (*correlatedFilters)->right(new ParseTree(sf)); + } - if (rc1 != NULL && rc1->joinInfo() != 0) - correlated = true; - - if (rc2 != NULL && rc2->joinInfo() != 0) - correlated = true; - - if (correlated) - { - ParseTree** correlatedFilters = reinterpret_cast(obj); - - if (*correlatedFilters == NULL) - { - *correlatedFilters = new ParseTree(sf); - } - else - { - ParseTree* left = *correlatedFilters; - *correlatedFilters = new ParseTree(new LogicOperator("and")); - (*correlatedFilters)->left(left); - (*correlatedFilters)->right(new ParseTree(sf)); - } - - pt->data(NULL); - } + pt->data(NULL); } + } } - ParseTree* trim(ParseTree*& pt) { - ParseTree* lhs = pt->left(); + ParseTree* lhs = pt->left(); - if (lhs) - pt->left(trim(lhs)); + if (lhs) + pt->left(trim(lhs)); - ParseTree* rhs = pt->right(); + ParseTree* rhs = pt->right(); - if (rhs) - pt->right(trim(rhs)); + if (rhs) + pt->right(trim(rhs)); - if ((lhs == NULL) && (rhs == NULL) && (pt->data() == NULL)) - { - delete pt; - pt = NULL; - } - else if ((lhs == NULL || rhs == NULL) && dynamic_cast(pt->data())) - { - idbassert(dynamic_cast(pt->data())->data() == "and"); - ParseTree* br = pt; - ParseTree* nl = NULL; // the left()/right() are overloaded + if ((lhs == NULL) && (rhs == NULL) && (pt->data() == NULL)) + { + delete pt; + pt = NULL; + } + else if ((lhs == NULL || rhs == NULL) && dynamic_cast(pt->data())) + { + idbassert(dynamic_cast(pt->data())->data() == "and"); + ParseTree* br = pt; + ParseTree* nl = NULL; // the left()/right() are overloaded - if (lhs == NULL && rhs != NULL) - pt = rhs; - else if (lhs != NULL && rhs == NULL) - pt = lhs; - else - pt = NULL; + if (lhs == NULL && rhs != NULL) + pt = rhs; + else if (lhs != NULL && rhs == NULL) + pt = lhs; + else + pt = NULL; - br->left(nl); - br->right(nl); - delete br; - } + br->left(nl); + br->right(nl); + delete br; + } - return pt; + return pt; } - void handleNotIn(JobStepVector& jsv, JobInfo* jobInfo) { - // convert CORRELATED join (but not MATCHNULLS) to expression. - for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) - { - TupleHashJoinStep* thjs = dynamic_cast(i->get()); + // convert CORRELATED join (but not MATCHNULLS) to expression. + for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) + { + TupleHashJoinStep* thjs = dynamic_cast(i->get()); - if (!thjs || !(thjs->getJoinType() & CORRELATED) || (thjs->getJoinType() & MATCHNULLS) ) - continue; + if (!thjs || !(thjs->getJoinType() & CORRELATED) || (thjs->getJoinType() & MATCHNULLS)) + continue; - ReturnedColumn* lhs = thjs->column1()->clone(); - ReturnedColumn* rhs = thjs->column2()->clone(); + ReturnedColumn* lhs = thjs->column1()->clone(); + ReturnedColumn* rhs = thjs->column2()->clone(); - SOP sop(new PredicateOperator("=")); - sop->setOpType(lhs->resultType(), rhs->resultType()); - sop->resultType(sop->operationType()); - SimpleFilter* sf = new SimpleFilter(sop, lhs, rhs); + SOP sop(new PredicateOperator("=")); + sop->setOpType(lhs->resultType(), rhs->resultType()); + sop->resultType(sop->operationType()); + SimpleFilter* sf = new SimpleFilter(sop, lhs, rhs); - ExpressionStep* es = new ExpressionStep(*jobInfo); + ExpressionStep* es = new ExpressionStep(*jobInfo); - if (es == NULL) - throw runtime_error("Failed to create ExpressionStep 2"); + if (es == NULL) + throw runtime_error("Failed to create ExpressionStep 2"); - es->expressionFilter(sf, *jobInfo); - es->resetJoinInfo(); // Not to be done as join - i->reset(es); + es->expressionFilter(sf, *jobInfo); + es->resetJoinInfo(); // Not to be done as join + i->reset(es); - delete sf; - } + delete sf; + } } - bool isNotInSubquery(JobStepVector& jsv) { - // use MATCHNULLS(execplan::JOIN_NULL_MATCH) to identify NOT IN and NOT EXIST - bool notIn = false; + // use MATCHNULLS(execplan::JOIN_NULL_MATCH) to identify NOT IN and NOT EXIST + bool notIn = false; - for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) + for (JobStepVector::iterator i = jsv.begin(); i != jsv.end(); i++) + { + TupleHashJoinStep* thjs = dynamic_cast(i->get()); + + if (thjs) { - TupleHashJoinStep* thjs = dynamic_cast(i->get()); - - if (thjs) - { - if (thjs->getJoinType() & MATCHNULLS) - { - // only NOT IN will be specially treated. - notIn = true; - break; - } - } + if (thjs->getJoinType() & MATCHNULLS) + { + // only NOT IN will be specially treated. + notIn = true; + break; + } } + } - return notIn; + return notIn; } // This fcn is currently unused. Will keep it in the code for now. @@ -380,523 +371,508 @@ void alterCsepInExistsFilter(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) void doCorrelatedExists(const ExistsFilter* ef, JobInfo& jobInfo) { - // Transformer sub to a subquery step. - SErrorInfo errorInfo(jobInfo.errorInfo); - SubQueryTransformer transformer(&jobInfo, errorInfo); - CalpontSelectExecutionPlan* csep = ef->sub().get(); - //alterCsepInExistsFilter(csep, jobInfo); - SJSTEP subQueryStep = transformer.makeSubQueryStep(csep); + // Transformer sub to a subquery step. + SErrorInfo errorInfo(jobInfo.errorInfo); + SubQueryTransformer transformer(&jobInfo, errorInfo); + CalpontSelectExecutionPlan* csep = ef->sub().get(); + // alterCsepInExistsFilter(csep, jobInfo); + SJSTEP subQueryStep = transformer.makeSubQueryStep(csep); - // @bug3524, special handling of not in. - JobStepVector& jsv = transformer.correlatedSteps(); + // @bug3524, special handling of not in. + JobStepVector& jsv = transformer.correlatedSteps(); - if (isNotInSubquery(jsv) == true) - handleNotIn(jsv, transformer.subJobInfo()); + if (isNotInSubquery(jsv) == true) + handleNotIn(jsv, transformer.subJobInfo()); - transformer.updateCorrelateInfo(); - jsv.push_back(subQueryStep); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + transformer.updateCorrelateInfo(); + jsv.push_back(subQueryStep); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); } - void doNonCorrelatedExists(const ExistsFilter* ef, JobInfo& jobInfo) { - // Assume: exists-subquery without a from clause, like exists (select 1) - // always evalustes to true - bool noFrom = (ef->sub()->tableList().size() == 0); - bool exists = !ef->notExists(); + // Assume: exists-subquery without a from clause, like exists (select 1) + // always evalustes to true + bool noFrom = (ef->sub()->tableList().size() == 0); + bool exists = !ef->notExists(); - if (!noFrom) + if (!noFrom) + { + // Transformer sub to a scalar result set. + SErrorInfo errorInfo(new ErrorInfo()); + SimpleScalarTransformer transformer(&jobInfo, errorInfo, true); + transformer.makeSubQueryStep(ef->sub().get()); + + // @bug 2839. error out in-relelvant correlated column case + if (!transformer.correlatedSteps().empty()) { - // Transformer sub to a scalar result set. - SErrorInfo errorInfo(new ErrorInfo()); - SimpleScalarTransformer transformer(&jobInfo, errorInfo, true); - transformer.makeSubQueryStep(ef->sub().get()); + JobStepVector::const_iterator it = transformer.correlatedSteps().begin(); + string tn; - // @bug 2839. error out in-relelvant correlated column case - if (!transformer.correlatedSteps().empty()) + for (; it != transformer.correlatedSteps().end(); ++it) + { + // 1. if tuplehashjoin, check alias1 and alias2; otherwise, check alias + // 2. alias start with "$sub_" + TupleHashJoinStep* thjs = dynamic_cast(it->get()); + + if (thjs) { - JobStepVector::const_iterator it = transformer.correlatedSteps().begin(); - string tn; - - for (; it != transformer.correlatedSteps().end(); ++it) - { - // 1. if tuplehashjoin, check alias1 and alias2; otherwise, check alias - // 2. alias start with "$sub_" - TupleHashJoinStep* thjs = dynamic_cast(it->get()); - - if (thjs) - { - if (thjs->alias1().empty() || thjs->alias1().compare(0, 5, "$sub_")) - tn = thjs->alias2(); - else - tn = thjs->alias1(); - } - else - { - tn = it->get()->alias(); - } - } - - Message::Args args; - - if (tn.empty() || tn.compare(0, 5, "$sub_")) - tn = "sub-query"; - - args.add(tn); - throw IDBExcept(ERR_MISS_JOIN_IN_SUB, args); + if (thjs->alias1().empty() || thjs->alias1().compare(0, 5, "$sub_")) + tn = thjs->alias2(); + else + tn = thjs->alias1(); } - - // Catch more_than_1_row exception only - try + else { - transformer.run(); + tn = it->get()->alias(); } - catch (MoreThan1RowExcept&) - { - // no-op - }; + } - // Check if the exists condition is satisfied. - //((!transformer.emptyResultSet() && !ef->notExists()) || - // ( transformer.emptyResultSet() && ef->notExists())) - exists = (transformer.emptyResultSet() == ef->notExists()); + Message::Args args; + + if (tn.empty() || tn.compare(0, 5, "$sub_")) + tn = "sub-query"; + + args.add(tn); + throw IDBExcept(ERR_MISS_JOIN_IN_SUB, args); } - JobStepVector jsv; - SJSTEP tcs(new TupleConstantBooleanStep(jobInfo, exists)); - jsv.push_back(tcs); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); -} + // Catch more_than_1_row exception only + try + { + transformer.run(); + } + catch (MoreThan1RowExcept&) + { + // no-op + }; + // Check if the exists condition is satisfied. + //((!transformer.emptyResultSet() && !ef->notExists()) || + // ( transformer.emptyResultSet() && ef->notExists())) + exists = (transformer.emptyResultSet() == ef->notExists()); + } + + JobStepVector jsv; + SJSTEP tcs(new TupleConstantBooleanStep(jobInfo, exists)); + jsv.push_back(tcs); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); +} const SRCP doSelectSubquery(CalpontExecutionPlan* ep, SRCP& sc, JobInfo& jobInfo) { - CalpontSelectExecutionPlan* csep = dynamic_cast(ep); - SRCP rc; - SErrorInfo errorInfo(jobInfo.errorInfo); - string subView = dynamic_cast(sc.get())->viewName(); - SubQueryTransformer transformer(&jobInfo, errorInfo, subView); - transformer.setVarbinaryOK(); - SJSTEP subQueryStep = transformer.makeSubQueryStep(csep); + CalpontSelectExecutionPlan* csep = dynamic_cast(ep); + SRCP rc; + SErrorInfo errorInfo(jobInfo.errorInfo); + string subView = dynamic_cast(sc.get())->viewName(); + SubQueryTransformer transformer(&jobInfo, errorInfo, subView); + transformer.setVarbinaryOK(); + SJSTEP subQueryStep = transformer.makeSubQueryStep(csep); - if (transformer.correlatedSteps().size() > 0) + if (transformer.correlatedSteps().size() > 0) + { + transformer.updateCorrelateInfo(); + JobStepVector jsv = transformer.correlatedSteps(); + jsv.push_back(subQueryStep); + jobInfo.selectAndFromSubs.insert(jobInfo.selectAndFromSubs.end(), jsv.begin(), jsv.end()); + const RetColsVector& retCol = csep->returnedCols(); + + for (uint64_t i = 0; i < retCol.size(); i++) { - transformer.updateCorrelateInfo(); - JobStepVector jsv = transformer.correlatedSteps(); - jsv.push_back(subQueryStep); - jobInfo.selectAndFromSubs.insert(jobInfo.selectAndFromSubs.end(), jsv.begin(), jsv.end()); - const RetColsVector& retCol = csep->returnedCols(); - - for (uint64_t i = 0; i < retCol.size(); i++) - { - if (retCol[i]->colSource() == 0) - { - rc = transformer.virtualTable().columns()[i]; - break; - } - } + if (retCol[i]->colSource() == 0) + { + rc = transformer.virtualTable().columns()[i]; + break; + } } - else + } + else + { + // Non-correlated subquery + // Do not catch exceptions here, let caller handle them. + SimpleScalarTransformer simpleTransformer(transformer); + simpleTransformer.run(); + + // Costruct a simple column based on the scalar result. + ConstantColumn* cc = NULL; + + if (simpleTransformer.emptyResultSet() == false) { - // Non-correlated subquery - // Do not catch exceptions here, let caller handle them. - SimpleScalarTransformer simpleTransformer(transformer); - simpleTransformer.run(); + // set value for cc + const Row& row = simpleTransformer.resultRow(); - // Costruct a simple column based on the scalar result. - ConstantColumn* cc = NULL; - - if (simpleTransformer.emptyResultSet() == false) - { - // set value for cc - const Row& row = simpleTransformer.resultRow(); - - if (!row.isNullValue(0)) - getColumnValue(&cc, 0, row, jobInfo.timeZone); - } - - // Empty set or null value - if (cc == NULL) - { - cc = new ConstantColumn(""); - cc->type(ConstantColumn::NULLDATA); - } - - rc.reset(cc); + if (!row.isNullValue(0)) + getColumnValue(&cc, 0, row, jobInfo.timeZone); } - return rc; -} - - + // Empty set or null value + if (cc == NULL) + { + cc = new ConstantColumn(""); + cc->type(ConstantColumn::NULLDATA); + } + + rc.reset(cc); + } + + return rc; } +} // namespace namespace joblist { - bool simpleScalarFilterToParseTree(SimpleScalarFilter* sf, ParseTree*& pt, JobInfo& jobInfo) { - const vector& cols = sf->cols(); - CalpontSelectExecutionPlan* csep = sf->sub().get(); - SOP sop = sf->op(); + const vector& cols = sf->cols(); + CalpontSelectExecutionPlan* csep = sf->sub().get(); + SOP sop = sf->op(); - // For row construct, supports only =, <> in Release 1.1. - // Other operators are errored out by connector. - string lop("and"); + // For row construct, supports only =, <> in Release 1.1. + // Other operators are errored out by connector. + string lop("and"); - if ((cols.size() > 1) && (sop->data() == "<>")) - lop = "or"; + if ((cols.size() > 1) && (sop->data() == "<>")) + lop = "or"; - // Transformer sub to a scalar result. - SErrorInfo errorInfo(jobInfo.errorInfo); - SimpleScalarTransformer transformer(&jobInfo, errorInfo, false); - transformer.makeSubQueryStep(csep); + // Transformer sub to a scalar result. + SErrorInfo errorInfo(jobInfo.errorInfo); + SimpleScalarTransformer transformer(&jobInfo, errorInfo, false); + transformer.makeSubQueryStep(csep); - // Do not catch exceptions here, let caller handle them. - transformer.run(); + // Do not catch exceptions here, let caller handle them. + transformer.run(); - // if subquery errored out - if (errorInfo->errCode) + // if subquery errored out + if (errorInfo->errCode) + { + ostringstream oss; + oss << "Sub-query failed: "; + + if (errorInfo->errMsg.empty()) { - ostringstream oss; - oss << "Sub-query failed: "; - - if (errorInfo->errMsg.empty()) - { - oss << "error code " << errorInfo->errCode; - errorInfo->errMsg = oss.str(); - } - - throw runtime_error(errorInfo->errMsg); + oss << "error code " << errorInfo->errCode; + errorInfo->errMsg = oss.str(); } - // Construct simple filters based on the scalar result. - bool isScalar = false; + throw runtime_error(errorInfo->errMsg); + } - if (transformer.emptyResultSet() == false) + // Construct simple filters based on the scalar result. + bool isScalar = false; + + if (transformer.emptyResultSet() == false) + { + const Row& row = transformer.resultRow(); + uint64_t i = 0; + + for (; i < cols.size(); i++) { - const Row& row = transformer.resultRow(); - uint64_t i = 0; + // = null is always false + if (row.isNullValue(i) == true) + break; - for (; i < cols.size(); i++) - { - // = null is always false - if (row.isNullValue(i) == true) - break; + // set fResult for cc + ConstantColumn* cc = NULL; + getColumnValue(&cc, i, row, jobInfo.timeZone); + sop->setOpType(cols[i]->resultType(), cc->resultType()); - // set fResult for cc - ConstantColumn* cc = NULL; - getColumnValue(&cc, i, row, jobInfo.timeZone); - sop->setOpType(cols[i]->resultType(), cc->resultType()); + SimpleFilter* sf = new SimpleFilter(sop, cols[i]->clone(), cc); - SimpleFilter* sf = new SimpleFilter(sop, cols[i]->clone(), cc); - - if (i == 0) - { - pt = new ParseTree(sf); - } - else - { - ParseTree* left = pt; - pt = new ParseTree(new LogicOperator(lop)); - pt->left(left); - pt->right(new ParseTree(sf)); - } - } - - if (i >= cols.size()) - isScalar = true; + if (i == 0) + { + pt = new ParseTree(sf); + } + else + { + ParseTree* left = pt; + pt = new ParseTree(new LogicOperator(lop)); + pt->left(left); + pt->right(new ParseTree(sf)); + } } - return isScalar; + if (i >= cols.size()) + isScalar = true; + } + + return isScalar; } - void doSimpleScalarFilter(ParseTree* p, JobInfo& jobInfo) { - SimpleScalarFilter* sf = dynamic_cast(p->data()); - idbassert(sf != NULL); - ParseTree* parseTree = NULL; + SimpleScalarFilter* sf = dynamic_cast(p->data()); + idbassert(sf != NULL); + ParseTree* parseTree = NULL; - // Parse filters to job step. - if (simpleScalarFilterToParseTree(sf, parseTree, jobInfo)) - { - // update the plan for supporting OR in future. - ParseTree* ccp = (p); - delete ccp->data(); - ccp->left(parseTree->left()); - ccp->right(parseTree->right()); - ccp->data(parseTree->data()); + // Parse filters to job step. + if (simpleScalarFilterToParseTree(sf, parseTree, jobInfo)) + { + // update the plan for supporting OR in future. + ParseTree* ccp = (p); + delete ccp->data(); + ccp->left(parseTree->left()); + ccp->right(parseTree->right()); + ccp->data(parseTree->data()); - // create job steps for each simple filter - JLF_ExecPlanToJobList::walkTree(parseTree, jobInfo); + // create job steps for each simple filter + JLF_ExecPlanToJobList::walkTree(parseTree, jobInfo); - jobInfo.dynamicParseTreeVec.push_back(parseTree); - // don't delete the parseTree, it has been placed in the plan. - // Instead, we use the dynamicParseTreeVec above for deletion - // in ~csep() or csep.unserialize(). - // delete parseTree; - } - else - { - // not a scalar result - delete parseTree; - JobStepVector jsv; - SJSTEP tcs(new TupleConstantBooleanStep(jobInfo, false)); - jsv.push_back(tcs); - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); - } + jobInfo.dynamicParseTreeVec.push_back(parseTree); + // don't delete the parseTree, it has been placed in the plan. + // Instead, we use the dynamicParseTreeVec above for deletion + // in ~csep() or csep.unserialize(). + // delete parseTree; + } + else + { + // not a scalar result + delete parseTree; + JobStepVector jsv; + SJSTEP tcs(new TupleConstantBooleanStep(jobInfo, false)); + jsv.push_back(tcs); + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + } } - void doExistsFilter(const ParseTree* p, JobInfo& jobInfo) { - const ExistsFilter* ef = dynamic_cast(p->data()); - idbassert(ef != NULL); + const ExistsFilter* ef = dynamic_cast(p->data()); + idbassert(ef != NULL); - if (ef->correlated()) - doCorrelatedExists(ef, jobInfo); - else - doNonCorrelatedExists(ef, jobInfo); + if (ef->correlated()) + doCorrelatedExists(ef, jobInfo); + else + doNonCorrelatedExists(ef, jobInfo); } - void doSelectFilter(const ParseTree* p, JobInfo& jobInfo) { - const SelectFilter* sf = dynamic_cast(p->data()); - idbassert(sf != NULL); + const SelectFilter* sf = dynamic_cast(p->data()); + idbassert(sf != NULL); - SErrorInfo errorInfo(jobInfo.errorInfo); - SubQueryTransformer transformer(&jobInfo, errorInfo); - SJSTEP subQueryStep = transformer.makeSubQueryStep(sf->sub().get()); - transformer.updateCorrelateInfo(); - JobStepVector jsv = transformer.correlatedSteps(); + SErrorInfo errorInfo(jobInfo.errorInfo); + SubQueryTransformer transformer(&jobInfo, errorInfo); + SJSTEP subQueryStep = transformer.makeSubQueryStep(sf->sub().get()); + transformer.updateCorrelateInfo(); + JobStepVector jsv = transformer.correlatedSteps(); - jsv.push_back(subQueryStep); + jsv.push_back(subQueryStep); - const vector& cols = sf->cols(); - SOP sop = sf->op(); + const vector& cols = sf->cols(); + SOP sop = sf->op(); - // For row construct, supports only =, <> in Release 1.1. - // Other operators are errored out by connector. - string lop("and"); + // For row construct, supports only =, <> in Release 1.1. + // Other operators are errored out by connector. + string lop("and"); - if ((cols.size() > 1) && (sop->data() == "<>")) - lop = "or"; + if ((cols.size() > 1) && (sop->data() == "<>")) + lop = "or"; - // @bug3780, select filters are not additional comparison, but fe2. - // When parsing the sub query, correlated columns may be added as group by column, - // s is the start position of the original selected columns. - uint64_t s = sf->returnedColPos(); - ParseTree* pt = NULL; - const VirtualTable& vt = transformer.virtualTable(); + // @bug3780, select filters are not additional comparison, but fe2. + // When parsing the sub query, correlated columns may be added as group by column, + // s is the start position of the original selected columns. + uint64_t s = sf->returnedColPos(); + ParseTree* pt = NULL; + const VirtualTable& vt = transformer.virtualTable(); - for (uint64_t i = 0; i < cols.size(); i++) + for (uint64_t i = 0; i < cols.size(); i++) + { + ReturnedColumn* lhs = cols[i].get()->clone(); + ReturnedColumn* rhs = vt.columns()[s + i].get()->clone(); + sop->setOpType(lhs->resultType(), rhs->resultType()); + + if (i == 0) { - ReturnedColumn* lhs = cols[i].get()->clone(); - ReturnedColumn* rhs = vt.columns()[s + i].get()->clone(); - sop->setOpType(lhs->resultType(), rhs->resultType()); - - if (i == 0) - { - pt = new ParseTree(new SimpleFilter(sop, lhs, rhs)); - } - else - { - ParseTree* left = pt; - pt = new ParseTree(new LogicOperator(lop)); - pt->left(left); - pt->right(new ParseTree(new SimpleFilter(sop, lhs, rhs))); - } + pt = new ParseTree(new SimpleFilter(sop, lhs, rhs)); } - - if (pt != NULL) + else { - ExpressionStep* es = new ExpressionStep(jobInfo); - es->expressionFilter(pt, jobInfo); - es->selectFilter(true); - delete pt; - - jsv.push_back(SJSTEP(es)); + ParseTree* left = pt; + pt = new ParseTree(new LogicOperator(lop)); + pt->left(left); + pt->right(new ParseTree(new SimpleFilter(sop, lhs, rhs))); } + } - JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); + if (pt != NULL) + { + ExpressionStep* es = new ExpressionStep(jobInfo); + es->expressionFilter(pt, jobInfo); + es->selectFilter(true); + delete pt; + + jsv.push_back(SJSTEP(es)); + } + + JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); } - void preprocessHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - ParseTree* havings = (csep->having()); - idbassert(havings != NULL); // check having exists before calling this function. + ParseTree* havings = (csep->having()); + idbassert(havings != NULL); // check having exists before calling this function. - // check select filter in having - havings->walk(sfInHaving, &jobInfo); + // check select filter in having + havings->walk(sfInHaving, &jobInfo); - // check simple scalar filters in having - havings->walk(ssfInHaving, &jobInfo); + // check simple scalar filters in having + havings->walk(ssfInHaving, &jobInfo); - // check correlated columns in having - ParseTree* correlatedFilters = NULL; - havings->walk(getCorrelatedFilters, &correlatedFilters); - trim(havings); + // check correlated columns in having + ParseTree* correlatedFilters = NULL; + havings->walk(getCorrelatedFilters, &correlatedFilters); + trim(havings); - if (havings == NULL) - { - csep->having(NULL); - } + if (havings == NULL) + { + csep->having(NULL); + } - if (correlatedFilters != NULL) - { - ParseTree* newFilters = new ParseTree(new LogicOperator("and")); - newFilters->left(csep->filters()); - newFilters->right(correlatedFilters); + if (correlatedFilters != NULL) + { + ParseTree* newFilters = new ParseTree(new LogicOperator("and")); + newFilters->left(csep->filters()); + newFilters->right(correlatedFilters); - csep->filters(newFilters); - csep->having(havings); - } + csep->filters(newFilters); + csep->having(havings); + } } - int doFromSubquery(CalpontExecutionPlan* ep, const string& alias, const string& view, JobInfo& jobInfo) { - CalpontSelectExecutionPlan* csep = dynamic_cast(ep); - SErrorInfo errorInfo(jobInfo.errorInfo); - SubQueryTransformer transformer(&jobInfo, errorInfo, alias, view); - transformer.setVarbinaryOK(); - SJSTEP subQueryStep = transformer.makeSubQueryStep(csep, true); - subQueryStep->view(view); - SJSTEP subAd(new SubAdapterStep(subQueryStep, jobInfo)); - jobInfo.selectAndFromSubs.push_back(subAd); + CalpontSelectExecutionPlan* csep = dynamic_cast(ep); + SErrorInfo errorInfo(jobInfo.errorInfo); + SubQueryTransformer transformer(&jobInfo, errorInfo, alias, view); + transformer.setVarbinaryOK(); + SJSTEP subQueryStep = transformer.makeSubQueryStep(csep, true); + subQueryStep->view(view); + SJSTEP subAd(new SubAdapterStep(subQueryStep, jobInfo)); + jobInfo.selectAndFromSubs.push_back(subAd); - return CNX_VTABLE_ID; + return CNX_VTABLE_ID; } - void addOrderByAndLimit(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - jobInfo.limitStart = csep->limitStart(); - jobInfo.limitCount = csep->limitNum(); - jobInfo.orderByThreads = csep->orderByThreads(); + jobInfo.limitStart = csep->limitStart(); + jobInfo.limitCount = csep->limitNum(); + jobInfo.orderByThreads = csep->orderByThreads(); - CalpontSelectExecutionPlan::OrderByColumnList& orderByCols = csep->orderByCols(); + CalpontSelectExecutionPlan::OrderByColumnList& orderByCols = csep->orderByCols(); - for (uint64_t i = 0; i < orderByCols.size(); i++) + for (uint64_t i = 0; i < orderByCols.size(); i++) + { + // skip constant columns + if (dynamic_cast(orderByCols[i].get()) != NULL) + continue; + + uint32_t tupleKey = -1; + SimpleColumn* sc = dynamic_cast(orderByCols[i].get()); + + if (sc != NULL) { - // skip constant columns - if (dynamic_cast(orderByCols[i].get()) != NULL) - continue; + CalpontSystemCatalog::OID oid = sc->oid(); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct; + string alias(extractTableAlias(sc)); + string view(sc->viewName()); + string schema(sc->schemaName()); - uint32_t tupleKey = -1; - SimpleColumn* sc = dynamic_cast(orderByCols[i].get()); + // string name(sc->columnName()); + if (!sc->schemaName().empty()) + { + ct = sc->colType(); - if (sc != NULL) + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (sc->isColumnStore() && !(dynamic_cast(sc))) { - CalpontSystemCatalog::OID oid = sc->oid(); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct; - string alias(extractTableAlias(sc)); - string view(sc->viewName()); - string schema(sc->schemaName()); + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + // X + dictOid = isDictCol(ct); + } + else + { + if (sc->colPosition() == -1) + { + sc = dynamic_cast(jobInfo.deliveredCols[sc->orderPos()].get()); -// string name(sc->columnName()); - if (!sc->schemaName().empty()) - { - ct = sc->colType(); - -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (sc->isColumnStore() && !(dynamic_cast(sc))) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } -//X - dictOid = isDictCol(ct); - } - else - { - if (sc->colPosition() == -1) - { - sc = dynamic_cast(jobInfo.deliveredCols[sc->orderPos()].get()); - - // If sc is NULL it's most likely a scaler subquery - if (sc == NULL) - { - const ReturnedColumn* rc = dynamic_cast(orderByCols[i].get()); - uint32_t eid = rc->expressionId(); - // If eid is -1, then there's no corresponding - // entry in tupleKeyMap and it will assert down the line - // Don't add the order by. It won't work and ordering on - // a singleton is a waste anyway. - if ((int32_t)eid == -1) - continue; - CalpontSystemCatalog::ColType ct = rc->resultType(); - tupleKey = getExpTupleKey(jobInfo, eid); - jobInfo.orderByColVec.push_back(make_pair(tupleKey, orderByCols[i]->asc())); - continue; - } - } - else - { - sc->oid((tblOid + 1) + sc->colPosition()); - } - - oid = sc->oid(); - ct = jobInfo.vtableColTypes[UniqId(oid, alias, "", "")]; - } - - tupleKey = getTupleKey(jobInfo, sc); - - // for dictionary columns, replace the token oid with string oid - if (dictOid > 0) - { - tupleKey = jobInfo.keyInfo->dictKeyMap[tupleKey]; - } + // If sc is NULL it's most likely a scaler subquery + if (sc == NULL) + { + const ReturnedColumn* rc = dynamic_cast(orderByCols[i].get()); + uint32_t eid = rc->expressionId(); + // If eid is -1, then there's no corresponding + // entry in tupleKeyMap and it will assert down the line + // Don't add the order by. It won't work and ordering on + // a singleton is a waste anyway. + if ((int32_t)eid == -1) + continue; + CalpontSystemCatalog::ColType ct = rc->resultType(); + tupleKey = getExpTupleKey(jobInfo, eid); + jobInfo.orderByColVec.push_back(make_pair(tupleKey, orderByCols[i]->asc())); + continue; + } } else { - const ReturnedColumn* rc = dynamic_cast(orderByCols[i].get()); - uint64_t eid = rc->expressionId(); - CalpontSystemCatalog::ColType ct = rc->resultType(); - tupleKey = getExpTupleKey(jobInfo, eid); + sc->oid((tblOid + 1) + sc->colPosition()); } - jobInfo.orderByColVec.push_back(make_pair(tupleKey, orderByCols[i]->asc())); - } -} + oid = sc->oid(); + ct = jobInfo.vtableColTypes[UniqId(oid, alias, "", "")]; + } + tupleKey = getTupleKey(jobInfo, sc); + + // for dictionary columns, replace the token oid with string oid + if (dictOid > 0) + { + tupleKey = jobInfo.keyInfo->dictKeyMap[tupleKey]; + } + } + else + { + const ReturnedColumn* rc = dynamic_cast(orderByCols[i].get()); + uint64_t eid = rc->expressionId(); + CalpontSystemCatalog::ColType ct = rc->resultType(); + tupleKey = getExpTupleKey(jobInfo, eid); + } + + jobInfo.orderByColVec.push_back(make_pair(tupleKey, orderByCols[i]->asc())); + } +} void preprocessSelectSubquery(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - RetColsVector& retCols = (csep->returnedCols()); - CalpontSelectExecutionPlan::SelectList::const_iterator sub = csep->selectSubList().begin(); + RetColsVector& retCols = (csep->returnedCols()); + CalpontSelectExecutionPlan::SelectList::const_iterator sub = csep->selectSubList().begin(); - for (RetColsVector::iterator i = retCols.begin(); i != retCols.end(); i++) + for (RetColsVector::iterator i = retCols.begin(); i != retCols.end(); i++) + { + if ((*i)->colSource() == execplan::SELECT_SUB) { - if ((*i)->colSource() == execplan::SELECT_SUB) - { - (*i) = doSelectSubquery(sub->get(), *i, jobInfo); - sub++; - } + (*i) = doSelectSubquery(sub->get(), *i, jobInfo); + sub++; } + } } - SJSTEP doUnionSub(CalpontExecutionPlan* ep, JobInfo& jobInfo) { - CalpontSelectExecutionPlan* csep = dynamic_cast(ep); - SErrorInfo errorInfo(jobInfo.errorInfo); - SubQueryTransformer transformer(&jobInfo, errorInfo); - transformer.setVarbinaryOK(); - SJSTEP subQueryStep = transformer.makeSubQueryStep(csep, false); - SJSTEP subAd(new SubAdapterStep(subQueryStep, jobInfo)); - return subAd; + CalpontSelectExecutionPlan* csep = dynamic_cast(ep); + SErrorInfo errorInfo(jobInfo.errorInfo); + SubQueryTransformer transformer(&jobInfo, errorInfo); + transformer.setVarbinaryOK(); + SJSTEP subQueryStep = transformer.makeSubQueryStep(csep, false); + SJSTEP subAd(new SubAdapterStep(subQueryStep, jobInfo)); + return subAd; } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/jlf_subquery.h b/dbcon/joblist/jlf_subquery.h index d5c9d8bac..8550a9840 100644 --- a/dbcon/joblist/jlf_subquery.h +++ b/dbcon/joblist/jlf_subquery.h @@ -17,7 +17,6 @@ // $Id: jlf_subquery.h 6383 2010-03-24 04:38:39Z xlou $ - /** @file */ #ifndef JLF_SUBQUERY_H @@ -27,22 +26,19 @@ namespace execplan { - class ParseTree; class SimpleScalarFilter; -} - +} // namespace execplan namespace joblist { - bool simpleScalarFilterToParseTree(SimpleScalarFilter* sf, ParseTree*& pt, JobInfo& jobInfo); void addOrderByAndLimit(execplan::CalpontSelectExecutionPlan*, JobInfo&); void doExistsFilter(const execplan::ParseTree*, JobInfo&); -int doFromSubquery(execplan::CalpontExecutionPlan*, const std::string&, const std::string&, JobInfo&); +int doFromSubquery(execplan::CalpontExecutionPlan*, const std::string&, const std::string&, JobInfo&); void doSelectFilter(const execplan::ParseTree*, JobInfo&); @@ -54,7 +50,6 @@ void preprocessSelectSubquery(execplan::CalpontSelectExecutionPlan*, JobInfo&); SJSTEP doUnionSub(execplan::CalpontExecutionPlan*, JobInfo&); -} +} // namespace joblist #endif // JLF_SUBQUERY_H - diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 6ff50a4b5..7678d2788 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -73,291 +73,314 @@ using namespace joblist; #include "statistics.h" - #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" - // for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to 'typeid' +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" +// for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to +// 'typeid' #endif - namespace { - - // construct a pcolstep from column key void tupleKeyToProjectStep(uint32_t key, JobStepVector& jsv, JobInfo& jobInfo) { - // this JSA is for pcolstep construct, is not taking input/output - // because the pcolstep is to be added into TBPS - CalpontSystemCatalog::OID oid = jobInfo.keyInfo->tupleKeyVec[key].fId; - DictOidToColOidMap::iterator mit = jobInfo.keyInfo->dictOidToColOid.find(oid); + // this JSA is for pcolstep construct, is not taking input/output + // because the pcolstep is to be added into TBPS + CalpontSystemCatalog::OID oid = jobInfo.keyInfo->tupleKeyVec[key].fId; + DictOidToColOidMap::iterator mit = jobInfo.keyInfo->dictOidToColOid.find(oid); - // if the key is for a dictionary, start with its token key - if (mit != jobInfo.keyInfo->dictOidToColOid.end()) + // if the key is for a dictionary, start with its token key + if (mit != jobInfo.keyInfo->dictOidToColOid.end()) + { + oid = mit->second; + + for (map::iterator i = jobInfo.keyInfo->dictKeyMap.begin(); + i != jobInfo.keyInfo->dictKeyMap.end(); i++) { - oid = mit->second; - - for (map::iterator i = jobInfo.keyInfo->dictKeyMap.begin(); - i != jobInfo.keyInfo->dictKeyMap.end(); - i++) - { - if (key == i->second) - { - key = i->first; - break; - } - } - - jobInfo.tokenOnly[key] = false; + if (key == i->second) + { + key = i->first; + break; + } } - CalpontSystemCatalog::OID tableOid = jobInfo.keyInfo->tupleKeyToTableOid[key]; -// JobStepAssociation dummyJsa; -// AnyDataListSPtr adl(new AnyDataList()); -// RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); -// dl->OID(oid); -// adl->rowGroupDL(dl); -// dummyJsa.outAdd(adl); + jobInfo.tokenOnly[key] = false; + } - CalpontSystemCatalog::ColType ct = jobInfo.keyInfo->colType[key]; + CalpontSystemCatalog::OID tableOid = jobInfo.keyInfo->tupleKeyToTableOid[key]; + // JobStepAssociation dummyJsa; + // AnyDataListSPtr adl(new AnyDataList()); + // RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + // dl->OID(oid); + // adl->rowGroupDL(dl); + // dummyJsa.outAdd(adl); - if (jobInfo.keyInfo->token2DictTypeMap.find(key) != jobInfo.keyInfo->token2DictTypeMap.end()) - ct = jobInfo.keyInfo->token2DictTypeMap[key]; + CalpontSystemCatalog::ColType ct = jobInfo.keyInfo->colType[key]; - uint32_t pt = jobInfo.keyInfo->pseudoType[key]; + if (jobInfo.keyInfo->token2DictTypeMap.find(key) != jobInfo.keyInfo->token2DictTypeMap.end()) + ct = jobInfo.keyInfo->token2DictTypeMap[key]; - SJSTEP sjs; + uint32_t pt = jobInfo.keyInfo->pseudoType[key]; - if (pt == 0) - sjs.reset(new pColStep(oid, tableOid, ct, jobInfo)); - else - sjs.reset(new PseudoColStep(oid, tableOid, pt, ct, jobInfo)); + SJSTEP sjs; - sjs->alias(jobInfo.keyInfo->tupleKeyVec[key].fTable); - sjs->view(jobInfo.keyInfo->tupleKeyVec[key].fView); - sjs->schema(jobInfo.keyInfo->tupleKeyVec[key].fSchema); - sjs->name(jobInfo.keyInfo->keyName[key]); - sjs->tupleId(key); + if (pt == 0) + sjs.reset(new pColStep(oid, tableOid, ct, jobInfo)); + else + sjs.reset(new PseudoColStep(oid, tableOid, pt, ct, jobInfo)); + + sjs->alias(jobInfo.keyInfo->tupleKeyVec[key].fTable); + sjs->view(jobInfo.keyInfo->tupleKeyVec[key].fView); + sjs->schema(jobInfo.keyInfo->tupleKeyVec[key].fSchema); + sjs->name(jobInfo.keyInfo->keyName[key]); + sjs->tupleId(key); + + jsv.push_back(sjs); + + bool tokenOnly = false; + map::iterator toIt = jobInfo.tokenOnly.find(key); + + if (toIt != jobInfo.tokenOnly.end()) + tokenOnly = toIt->second; + + if (sjs.get()->isDictCol() && !tokenOnly) + { + // Need a dictionary step + uint32_t dictKey = jobInfo.keyInfo->dictKeyMap[key]; + CalpontSystemCatalog::OID dictOid = jobInfo.keyInfo->tupleKeyVec[dictKey].fId; + sjs.reset(new pDictionaryStep(dictOid, tableOid, ct, jobInfo)); + sjs->alias(jobInfo.keyInfo->tupleKeyVec[dictKey].fTable); + sjs->view(jobInfo.keyInfo->tupleKeyVec[dictKey].fView); + sjs->schema(jobInfo.keyInfo->tupleKeyVec[dictKey].fSchema); + sjs->name(jobInfo.keyInfo->keyName[dictKey]); + sjs->tupleId(dictKey); + + jobInfo.keyInfo->dictOidToColOid[dictOid] = oid; jsv.push_back(sjs); - - bool tokenOnly = false; - map::iterator toIt = jobInfo.tokenOnly.find(key); - - if (toIt != jobInfo.tokenOnly.end()) - tokenOnly = toIt->second; - - if (sjs.get()->isDictCol() && !tokenOnly) - { - // Need a dictionary step - uint32_t dictKey = jobInfo.keyInfo->dictKeyMap[key]; - CalpontSystemCatalog::OID dictOid = jobInfo.keyInfo->tupleKeyVec[dictKey].fId; - sjs.reset(new pDictionaryStep(dictOid, tableOid, ct, jobInfo)); - sjs->alias(jobInfo.keyInfo->tupleKeyVec[dictKey].fTable); - sjs->view(jobInfo.keyInfo->tupleKeyVec[dictKey].fView); - sjs->schema(jobInfo.keyInfo->tupleKeyVec[dictKey].fSchema); - sjs->name(jobInfo.keyInfo->keyName[dictKey]); - sjs->tupleId(dictKey); - - jobInfo.keyInfo->dictOidToColOid[dictOid] = oid; - - jsv.push_back(sjs); - } + } } - -inline void addColumnToRG(uint32_t cid, vector& pos, vector& oids, - vector& keys, vector& scale, vector& precision, - vector& types, vector& csNums, JobInfo& jobInfo) +inline void addColumnToRG(uint32_t cid, vector& pos, vector& oids, vector& keys, + vector& scale, vector& precision, + vector& types, vector& csNums, + JobInfo& jobInfo) { - TupleInfo ti(getTupleInfo(cid, jobInfo)); - pos.push_back(pos.back() + ti.width); - oids.push_back(ti.oid); - keys.push_back(ti.key); - types.push_back(ti.dtype); - csNums.push_back(ti.csNum); - scale.push_back(ti.scale); - precision.push_back(ti.precision); + TupleInfo ti(getTupleInfo(cid, jobInfo)); + pos.push_back(pos.back() + ti.width); + oids.push_back(ti.oid); + keys.push_back(ti.key); + types.push_back(ti.dtype); + csNums.push_back(ti.csNum); + scale.push_back(ti.scale); + precision.push_back(ti.precision); } - inline void addColumnInExpToRG(uint32_t cid, vector& pos, vector& oids, vector& keys, vector& scale, vector& precision, - vector& types, vector& csNums, JobInfo& jobInfo) + vector& types, vector& csNums, + JobInfo& jobInfo) { - if (jobInfo.keyInfo->dictKeyMap.find(cid) != jobInfo.keyInfo->dictKeyMap.end()) - cid = jobInfo.keyInfo->dictKeyMap[cid]; + if (jobInfo.keyInfo->dictKeyMap.find(cid) != jobInfo.keyInfo->dictKeyMap.end()) + cid = jobInfo.keyInfo->dictKeyMap[cid]; - if (find(keys.begin(), keys.end(), cid) == keys.end()) - addColumnToRG(cid, pos, oids, keys, scale, precision, types, csNums, jobInfo); + if (find(keys.begin(), keys.end(), cid) == keys.end()) + addColumnToRG(cid, pos, oids, keys, scale, precision, types, csNums, jobInfo); } - inline void addColumnsToRG(uint32_t tid, vector& pos, vector& oids, vector& keys, vector& scale, vector& precision, - vector& types, - vector& csNums, + vector& types, vector& csNums, TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - // -- the selected columns - vector& pjCol = tableInfoMap[tid].fProjectCols; + // -- the selected columns + vector& pjCol = tableInfoMap[tid].fProjectCols; - for (unsigned i = 0; i < pjCol.size(); i++) - { - addColumnToRG(pjCol[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); - } + for (unsigned i = 0; i < pjCol.size(); i++) + { + addColumnToRG(pjCol[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); + } - // -- any columns will be used in cross-table exps - vector& exp2 = tableInfoMap[tid].fColsInExp2; + // -- any columns will be used in cross-table exps + vector& exp2 = tableInfoMap[tid].fColsInExp2; - for (unsigned i = 0; i < exp2.size(); i++) - { - addColumnInExpToRG(exp2[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); - } + for (unsigned i = 0; i < exp2.size(); i++) + { + addColumnInExpToRG(exp2[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); + } - // -- any columns will be used in returned exps - vector& expr = tableInfoMap[tid].fColsInRetExp; + // -- any columns will be used in returned exps + vector& expr = tableInfoMap[tid].fColsInRetExp; - for (unsigned i = 0; i < expr.size(); i++) - { - addColumnInExpToRG(expr[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); - } + for (unsigned i = 0; i < expr.size(); i++) + { + addColumnInExpToRG(expr[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); + } - // -- any columns will be used in final outer join expression - vector& expo = tableInfoMap[tid].fColsInOuter; + // -- any columns will be used in final outer join expression + vector& expo = tableInfoMap[tid].fColsInOuter; - for (unsigned i = 0; i < expo.size(); i++) - { - addColumnInExpToRG(expo[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); - } + for (unsigned i = 0; i < expo.size(); i++) + { + addColumnInExpToRG(expo[i], pos, oids, keys, scale, precision, types, csNums, jobInfo); + } } - -void constructJoinedRowGroup(RowGroup& rg, uint32_t large, uint32_t prev, bool root, - set& tableSet, TableInfoMap& tableInfoMap, JobInfo& jobInfo) +void constructJoinedRowGroup(RowGroup& rg, uint32_t large, uint32_t prev, bool root, set& tableSet, + TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - // Construct the output rowgroup for the join. - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); + // Construct the output rowgroup for the join. + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); - // -- start with the join keys - // lead by joinkeys -- to have more controls on joins - // [loop throuh the key list to support compound join] - if (root == false) // not root - { - vector& joinKeys = jobInfo.tableJoinMap[make_pair(large, prev)].fLeftKeys; + // -- start with the join keys + // lead by joinkeys -- to have more controls on joins + // [loop throuh the key list to support compound join] + if (root == false) // not root + { + vector& joinKeys = jobInfo.tableJoinMap[make_pair(large, prev)].fLeftKeys; - for (vector::iterator i = joinKeys.begin(); i != joinKeys.end(); i++) - addColumnToRG(*i, pos, oids, keys, scale, precision, types, csNums, jobInfo); - } + for (vector::iterator i = joinKeys.begin(); i != joinKeys.end(); i++) + addColumnToRG(*i, pos, oids, keys, scale, precision, types, csNums, jobInfo); + } - // -- followed by the columns in select or expression - for (set::iterator i = tableSet.begin(); i != tableSet.end(); i++) - addColumnsToRG(*i, pos, oids, keys, scale, precision, types, csNums, tableInfoMap, jobInfo); + // -- followed by the columns in select or expression + for (set::iterator i = tableSet.begin(); i != tableSet.end(); i++) + addColumnsToRG(*i, pos, oids, keys, scale, precision, types, csNums, tableInfoMap, jobInfo); - RowGroup tmpRg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - rg = tmpRg; + RowGroup tmpRg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + rg = tmpRg; } - void constructJoinedRowGroup(RowGroup& rg, set& tableSet, TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - // Construct the output rowgroup for the join. - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); + // Construct the output rowgroup for the join. + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); - for (set::iterator i = tableSet.begin(); i != tableSet.end(); i++) + for (set::iterator i = tableSet.begin(); i != tableSet.end(); i++) + { + // columns in select or expression + addColumnsToRG(*i, pos, oids, keys, scale, precision, types, csNums, tableInfoMap, jobInfo); + + // keys to be joined if not already in the rowgroup + vector& adjList = tableInfoMap[*i].fAdjacentList; + + for (vector::iterator j = adjList.begin(); j != adjList.end(); j++) { - // columns in select or expression - addColumnsToRG(*i, pos, oids, keys, scale, precision, types, csNums, tableInfoMap, jobInfo); + if (find(tableSet.begin(), tableSet.end(), *j) == tableSet.end()) + { + // not joined + vector& joinKeys = jobInfo.tableJoinMap[make_pair(*i, *j)].fLeftKeys; - // keys to be joined if not already in the rowgroup - vector& adjList = tableInfoMap[*i].fAdjacentList; - - for (vector::iterator j = adjList.begin(); j != adjList.end(); j++) + for (vector::iterator k = joinKeys.begin(); k != joinKeys.end(); k++) { - if (find(tableSet.begin(), tableSet.end(), *j) == tableSet.end()) - { - // not joined - vector& joinKeys = jobInfo.tableJoinMap[make_pair(*i, *j)].fLeftKeys; - - for (vector::iterator k = joinKeys.begin(); k != joinKeys.end(); k++) - { - if (find(keys.begin(), keys.end(), *k) == keys.end()) - addColumnToRG(*k, pos, oids, keys, scale, precision, types, csNums, jobInfo); - } - } + if (find(keys.begin(), keys.end(), *k) == keys.end()) + addColumnToRG(*k, pos, oids, keys, scale, precision, types, csNums, jobInfo); } + } } + } - RowGroup tmpRg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - rg = tmpRg; + RowGroup tmpRg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + rg = tmpRg; } - void updateExp2Cols(JobStepVector& expSteps, TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - for (JobStepVector::iterator it = expSteps.begin(); it != expSteps.end(); it++) + for (JobStepVector::iterator it = expSteps.begin(); it != expSteps.end(); it++) + { + ExpressionStep* exps = dynamic_cast(it->get()); + const vector& tables = exps->tableKeys(); + const vector& columns = exps->columnKeys(); + + for (uint64_t i = 0; i < tables.size(); ++i) { - ExpressionStep* exps = dynamic_cast(it->get()); - const vector& tables = exps->tableKeys(); - const vector& columns = exps->columnKeys(); + vector& exp2 = tableInfoMap[tables[i]].fColsInExp2; + vector::iterator cit = find(exp2.begin(), exp2.end(), columns[i]); - for (uint64_t i = 0; i < tables.size(); ++i) - { - vector& exp2 = tableInfoMap[tables[i]].fColsInExp2; - vector::iterator cit = find(exp2.begin(), exp2.end(), columns[i]); - - if (cit != exp2.end()) - exp2.erase(cit); - } + if (cit != exp2.end()) + exp2.erase(cit); } + } } - void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps, JobInfo& jobInfo) { - SJSTEP spjs = querySteps.back(); - BatchPrimitive* bps = dynamic_cast(spjs.get()); - TupleHashJoinStep* thjs = dynamic_cast(spjs.get()); - SubAdapterStep* sas = dynamic_cast(spjs.get()); + SJSTEP spjs = querySteps.back(); + BatchPrimitive* bps = dynamic_cast(spjs.get()); + TupleHashJoinStep* thjs = dynamic_cast(spjs.get()); + SubAdapterStep* sas = dynamic_cast(spjs.get()); - if (!bps && !thjs && !sas) - throw runtime_error("Bad last step"); + if (!bps && !thjs && !sas) + throw runtime_error("Bad last step"); - // original output rowgroup of the step - TupleJobStep* tjs = dynamic_cast(spjs.get()); - const RowGroup* rg0 = &(tjs->getOutputRowGroup()); + // original output rowgroup of the step + TupleJobStep* tjs = dynamic_cast(spjs.get()); + const RowGroup* rg0 = &(tjs->getOutputRowGroup()); - if (jobInfo.trace) cout << "Output RowGroup 0: " << rg0->toString() << endl; + if (jobInfo.trace) + cout << "Output RowGroup 0: " << rg0->toString() << endl; - // Construct a rowgroup that matches the select columns - TupleInfoVector v = jobInfo.pjColList; - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; + // Construct a rowgroup that matches the select columns + TupleInfoVector v = jobInfo.pjColList; + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); + + for (unsigned i = 0; i < v.size(); i++) + { + pos.push_back(pos.back() + v[i].width); + oids.push_back(v[i].oid); + keys.push_back(v[i].key); + types.push_back(v[i].dtype); + csNums.push_back(v[i].csNum); + scale.push_back(v[i].scale); + precision.push_back(v[i].precision); + } + + RowGroup rg1(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + + // evaluate the returned/groupby expressions if any + JobStepVector& expSteps = jobInfo.returnedExpressions; + + if (expSteps.size() > 0) + { + // create a RG has the keys not in rg0 + pos.clear(); + oids.clear(); + keys.clear(); + scale.clear(); + precision.clear(); + types.clear(); + csNums.clear(); pos.push_back(2); + const vector& keys0 = rg0->getKeys(); + for (unsigned i = 0; i < v.size(); i++) { + if (find(keys0.begin(), keys0.end(), v[i].key) == keys0.end()) + { pos.push_back(pos.back() + v[i].width); oids.push_back(v[i].oid); keys.push_back(v[i].key); @@ -365,2112 +388,2044 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps, csNums.push_back(v[i].csNum); scale.push_back(v[i].scale); precision.push_back(v[i].precision); + } } - RowGroup rg1(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - - // evaluate the returned/groupby expressions if any - JobStepVector& expSteps = jobInfo.returnedExpressions; - - if (expSteps.size() > 0) - { - // create a RG has the keys not in rg0 - pos.clear(); - oids.clear(); - keys.clear(); - scale.clear(); - precision.clear(); - types.clear(); - csNums.clear(); - pos.push_back(2); - - const vector& keys0 = rg0->getKeys(); - - for (unsigned i = 0; i < v.size(); i++) - { - if (find(keys0.begin(), keys0.end(), v[i].key) == keys0.end()) - { - pos.push_back(pos.back() + v[i].width); - oids.push_back(v[i].oid); - keys.push_back(v[i].key); - types.push_back(v[i].dtype); - csNums.push_back(v[i].csNum); - scale.push_back(v[i].scale); - precision.push_back(v[i].precision); - } - } - - // for v0.9.3.0, the output and input to the expression are in the same row - // add the returned column into the rg0 as rg01 - RowGroup rg01 = *rg0 + RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - - if (jobInfo.trace) cout << "Output RowGroup 01: " << rg01.toString() << endl; - - map keyToIndexMap0; // maps key to the index in the input RG - - for (uint64_t i = 0; i < rg01.getKeys().size(); ++i) - keyToIndexMap0.insert(make_pair(rg01.getKeys()[i], i)); - - vector exps; // columns to be evaluated - - for (JobStepVector::iterator eit = expSteps.begin(); eit != expSteps.end(); ++eit) - { - ExpressionStep* es = dynamic_cast(eit->get()); - es->updateInputIndex(keyToIndexMap0, jobInfo); - es->updateOutputIndex(keyToIndexMap0, jobInfo); // same row as input - exps.push_back(es->expression()); - } - - // last step can be tbps (no join) or thjs, either one can have a group 3 expression - if (bps || thjs) - { - tjs->setOutputRowGroup(rg01); - tjs->setFcnExpGroup3(exps); - tjs->setFE23Output(rg1); - } - else if (sas) - { - sas->setFeRowGroup(rg01); - sas->addExpression(exps); - sas->setOutputRowGroup(rg1); - } - } - else - { - if (thjs && thjs->hasFcnExpGroup2()) - thjs->setFE23Output(rg1); - else - tjs->setOutputRowGroup(rg1); - } - - if (jobInfo.trace) cout << "Output RowGroup 1: " << rg1.toString() << endl; - - if (jobInfo.hasAggregation == false) - { - if (thjs != NULL) //setup a few things for the final thjs step... - thjs->outputAssociation(JobStepAssociation()); - - deliverySteps[CNX_VTABLE_ID] = spjs; - } - else - { - TupleDeliveryStep* tds = dynamic_cast(spjs.get()); - idbassert(tds != NULL); - SJSTEP ads = TupleAggregateStep::prepAggregate(spjs, jobInfo); - querySteps.push_back(ads); - - if (ads.get() != NULL) - deliverySteps[CNX_VTABLE_ID] = ads; - else - throw std::logic_error("Failed to prepare Aggregation Delivery Step."); - } - - if (jobInfo.havingStep) - { - TupleDeliveryStep* ds = - dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); - - AnyDataListSPtr spdlIn(new AnyDataList()); - RowGroupDL* dlIn = new RowGroupDL(1, jobInfo.fifoSize); - dlIn->OID(CNX_VTABLE_ID); - spdlIn->rowGroupDL(dlIn); - JobStepAssociation jsaIn; - jsaIn.outAdd(spdlIn); - dynamic_cast(ds)->outputAssociation(jsaIn); - jobInfo.havingStep->inputAssociation(jsaIn); - - AnyDataListSPtr spdlOut(new AnyDataList()); - RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); - dlOut->OID(CNX_VTABLE_ID); - spdlOut->rowGroupDL(dlOut); - JobStepAssociation jsaOut; - jsaOut.outAdd(spdlOut); - jobInfo.havingStep->outputAssociation(jsaOut); - - querySteps.push_back(jobInfo.havingStep); - dynamic_cast(jobInfo.havingStep.get()) - ->initialize(ds->getDeliveredRowGroup(), jobInfo); - deliverySteps[CNX_VTABLE_ID] = jobInfo.havingStep; - } - - if (jobInfo.windowCols.size() > 0) - { - spjs = querySteps.back(); - SJSTEP ws = WindowFunctionStep::makeWindowFunctionStep(spjs, jobInfo); - idbassert(ws.get()); - querySteps.push_back(ws); - deliverySteps[CNX_VTABLE_ID] = ws; - } - - // TODO MCOL-894 we don't need to run sorting|distinct - // every time -// if ((jobInfo.limitCount != (uint64_t) - 1) || -// (jobInfo.constantCol == CONST_COL_EXIST) || -// (jobInfo.hasDistinct)) -// { - if (jobInfo.annexStep.get() == NULL) - jobInfo.annexStep.reset(new TupleAnnexStep(jobInfo)); - - TupleAnnexStep* tas = dynamic_cast(jobInfo.annexStep.get()); - tas->setLimit(jobInfo.limitStart, jobInfo.limitCount); - - if (jobInfo.orderByColVec.size() > 0) - { - tas->addOrderBy(new LimitedOrderBy()); - if (jobInfo.orderByThreads > 1) - tas->setParallelOp(); - tas->setMaxThreads(jobInfo.orderByThreads); - } - - if (jobInfo.constantCol == CONST_COL_EXIST) - tas->addConstant(new TupleConstantStep(jobInfo)); - - if (jobInfo.hasDistinct) - tas->setDistinct(); - -// } - - if (jobInfo.annexStep) - { - TupleDeliveryStep* ds = - dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); - RowGroup rg2 = ds->getDeliveredRowGroup(); - - if (jobInfo.trace) cout << "Output RowGroup 2: " << rg2.toString() << endl; - - AnyDataListSPtr spdlIn(new AnyDataList()); - RowGroupDL* dlIn; - if (jobInfo.orderByColVec.size() > 0) - dlIn = new RowGroupDL(jobInfo.orderByThreads, jobInfo.fifoSize); - else - dlIn = new RowGroupDL(1, jobInfo.fifoSize); - dlIn->OID(CNX_VTABLE_ID); - spdlIn->rowGroupDL(dlIn); - JobStepAssociation jsaIn; - jsaIn.outAdd(spdlIn); - dynamic_cast(ds)->outputAssociation(jsaIn); - jobInfo.annexStep->inputAssociation(jsaIn); - - AnyDataListSPtr spdlOut(new AnyDataList()); - RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); - dlOut->OID(CNX_VTABLE_ID); - spdlOut->rowGroupDL(dlOut); - JobStepAssociation jsaOut; - jsaOut.outAdd(spdlOut); - jobInfo.annexStep->outputAssociation(jsaOut); - - querySteps.push_back(jobInfo.annexStep); - dynamic_cast(jobInfo.annexStep.get())->initialize(rg2, jobInfo); - deliverySteps[CNX_VTABLE_ID] = jobInfo.annexStep; - } - - // Check if constant false - if (jobInfo.constantFalse) - { - TupleConstantBooleanStep* tcs = new TupleConstantBooleanStep(jobInfo, false); - tcs->outputAssociation(querySteps.back().get()->outputAssociation()); - TupleDeliveryStep* tds = - dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); - tcs->initialize(tds->getDeliveredRowGroup(), jobInfo); - - JobStepVector::iterator it = querySteps.begin(); - - while (it != querySteps.end()) - { - if ((dynamic_cast(it->get()) != NULL) || - (dynamic_cast(it->get()) != NULL)) - break; - - it++; - } - - SJSTEP bs(tcs); - - if (it != querySteps.end()) - tcs->outputAssociation((*it)->inputAssociation()); - else - deliverySteps[CNX_VTABLE_ID] = bs; - - querySteps.erase(querySteps.begin(), it); - querySteps.insert(querySteps.begin(), bs); - } + // for v0.9.3.0, the output and input to the expression are in the same row + // add the returned column into the rg0 as rg01 + RowGroup rg01 = *rg0 + RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, + jobInfo.stringTableThreshold); if (jobInfo.trace) + cout << "Output RowGroup 01: " << rg01.toString() << endl; + + map keyToIndexMap0; // maps key to the index in the input RG + + for (uint64_t i = 0; i < rg01.getKeys().size(); ++i) + keyToIndexMap0.insert(make_pair(rg01.getKeys()[i], i)); + + vector exps; // columns to be evaluated + + for (JobStepVector::iterator eit = expSteps.begin(); eit != expSteps.end(); ++eit) { - TupleDeliveryStep* ds = dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); - - if (ds) cout << "Delivered RowGroup: " << ds->getDeliveredRowGroup().toString() << endl; + ExpressionStep* es = dynamic_cast(eit->get()); + es->updateInputIndex(keyToIndexMap0, jobInfo); + es->updateOutputIndex(keyToIndexMap0, jobInfo); // same row as input + exps.push_back(es->expression()); } -} + // last step can be tbps (no join) or thjs, either one can have a group 3 expression + if (bps || thjs) + { + tjs->setOutputRowGroup(rg01); + tjs->setFcnExpGroup3(exps); + tjs->setFE23Output(rg1); + } + else if (sas) + { + sas->setFeRowGroup(rg01); + sas->addExpression(exps); + sas->setOutputRowGroup(rg1); + } + } + else + { + if (thjs && thjs->hasFcnExpGroup2()) + thjs->setFE23Output(rg1); + else + tjs->setOutputRowGroup(rg1); + } + + if (jobInfo.trace) + cout << "Output RowGroup 1: " << rg1.toString() << endl; + + if (jobInfo.hasAggregation == false) + { + if (thjs != NULL) // setup a few things for the final thjs step... + thjs->outputAssociation(JobStepAssociation()); + + deliverySteps[CNX_VTABLE_ID] = spjs; + } + else + { + TupleDeliveryStep* tds = dynamic_cast(spjs.get()); + idbassert(tds != NULL); + SJSTEP ads = TupleAggregateStep::prepAggregate(spjs, jobInfo); + querySteps.push_back(ads); + + if (ads.get() != NULL) + deliverySteps[CNX_VTABLE_ID] = ads; + else + throw std::logic_error("Failed to prepare Aggregation Delivery Step."); + } + + if (jobInfo.havingStep) + { + TupleDeliveryStep* ds = dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); + + AnyDataListSPtr spdlIn(new AnyDataList()); + RowGroupDL* dlIn = new RowGroupDL(1, jobInfo.fifoSize); + dlIn->OID(CNX_VTABLE_ID); + spdlIn->rowGroupDL(dlIn); + JobStepAssociation jsaIn; + jsaIn.outAdd(spdlIn); + dynamic_cast(ds)->outputAssociation(jsaIn); + jobInfo.havingStep->inputAssociation(jsaIn); + + AnyDataListSPtr spdlOut(new AnyDataList()); + RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); + dlOut->OID(CNX_VTABLE_ID); + spdlOut->rowGroupDL(dlOut); + JobStepAssociation jsaOut; + jsaOut.outAdd(spdlOut); + jobInfo.havingStep->outputAssociation(jsaOut); + + querySteps.push_back(jobInfo.havingStep); + dynamic_cast(jobInfo.havingStep.get())->initialize(ds->getDeliveredRowGroup(), jobInfo); + deliverySteps[CNX_VTABLE_ID] = jobInfo.havingStep; + } + + if (jobInfo.windowCols.size() > 0) + { + spjs = querySteps.back(); + SJSTEP ws = WindowFunctionStep::makeWindowFunctionStep(spjs, jobInfo); + idbassert(ws.get()); + querySteps.push_back(ws); + deliverySteps[CNX_VTABLE_ID] = ws; + } + + // TODO MCOL-894 we don't need to run sorting|distinct + // every time + // if ((jobInfo.limitCount != (uint64_t) - 1) || + // (jobInfo.constantCol == CONST_COL_EXIST) || + // (jobInfo.hasDistinct)) + // { + if (jobInfo.annexStep.get() == NULL) + jobInfo.annexStep.reset(new TupleAnnexStep(jobInfo)); + + TupleAnnexStep* tas = dynamic_cast(jobInfo.annexStep.get()); + tas->setLimit(jobInfo.limitStart, jobInfo.limitCount); + + if (jobInfo.orderByColVec.size() > 0) + { + tas->addOrderBy(new LimitedOrderBy()); + if (jobInfo.orderByThreads > 1) + tas->setParallelOp(); + tas->setMaxThreads(jobInfo.orderByThreads); + } + + if (jobInfo.constantCol == CONST_COL_EXIST) + tas->addConstant(new TupleConstantStep(jobInfo)); + + if (jobInfo.hasDistinct) + tas->setDistinct(); + + // } + + if (jobInfo.annexStep) + { + TupleDeliveryStep* ds = dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); + RowGroup rg2 = ds->getDeliveredRowGroup(); + + if (jobInfo.trace) + cout << "Output RowGroup 2: " << rg2.toString() << endl; + + AnyDataListSPtr spdlIn(new AnyDataList()); + RowGroupDL* dlIn; + if (jobInfo.orderByColVec.size() > 0) + dlIn = new RowGroupDL(jobInfo.orderByThreads, jobInfo.fifoSize); + else + dlIn = new RowGroupDL(1, jobInfo.fifoSize); + dlIn->OID(CNX_VTABLE_ID); + spdlIn->rowGroupDL(dlIn); + JobStepAssociation jsaIn; + jsaIn.outAdd(spdlIn); + dynamic_cast(ds)->outputAssociation(jsaIn); + jobInfo.annexStep->inputAssociation(jsaIn); + + AnyDataListSPtr spdlOut(new AnyDataList()); + RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); + dlOut->OID(CNX_VTABLE_ID); + spdlOut->rowGroupDL(dlOut); + JobStepAssociation jsaOut; + jsaOut.outAdd(spdlOut); + jobInfo.annexStep->outputAssociation(jsaOut); + + querySteps.push_back(jobInfo.annexStep); + dynamic_cast(jobInfo.annexStep.get())->initialize(rg2, jobInfo); + deliverySteps[CNX_VTABLE_ID] = jobInfo.annexStep; + } + + // Check if constant false + if (jobInfo.constantFalse) + { + TupleConstantBooleanStep* tcs = new TupleConstantBooleanStep(jobInfo, false); + tcs->outputAssociation(querySteps.back().get()->outputAssociation()); + TupleDeliveryStep* tds = dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); + tcs->initialize(tds->getDeliveredRowGroup(), jobInfo); + + JobStepVector::iterator it = querySteps.begin(); + + while (it != querySteps.end()) + { + if ((dynamic_cast(it->get()) != NULL) || + (dynamic_cast(it->get()) != NULL)) + break; + + it++; + } + + SJSTEP bs(tcs); + + if (it != querySteps.end()) + tcs->outputAssociation((*it)->inputAssociation()); + else + deliverySteps[CNX_VTABLE_ID] = bs; + + querySteps.erase(querySteps.begin(), it); + querySteps.insert(querySteps.begin(), bs); + } + + if (jobInfo.trace) + { + TupleDeliveryStep* ds = dynamic_cast(deliverySteps[CNX_VTABLE_ID].get()); + + if (ds) + cout << "Delivered RowGroup: " << ds->getDeliveredRowGroup().toString() << endl; + } +} // add the project steps into the query TBPS and construct the output rowgroup void addProjectStepsToBps(TableInfoMap::iterator& mit, BatchPrimitive* bps, JobInfo& jobInfo) { - // make sure we have a good tuple bps - if (bps == NULL) - throw runtime_error("BPS is null"); + // make sure we have a good tuple bps + if (bps == NULL) + throw runtime_error("BPS is null"); - // construct a pcolstep for each joinkey to be projected - vector& joinKeys = mit->second.fJoinKeys; - JobStepVector keySteps; - vector fjKeys; + // construct a pcolstep for each joinkey to be projected + vector& joinKeys = mit->second.fJoinKeys; + JobStepVector keySteps; + vector fjKeys; - for (vector::iterator kit = joinKeys.begin(); kit != joinKeys.end(); kit++) + for (vector::iterator kit = joinKeys.begin(); kit != joinKeys.end(); kit++) + { + if (jobInfo.keyInfo.get()->tupleKeyToTableOid[*kit] != CNX_EXP_TABLE_ID) + tupleKeyToProjectStep(*kit, keySteps, jobInfo); + else + fjKeys.push_back(*kit); + } + + // construct pcolstep for columns in expresssions + JobStepVector expSteps; + vector& exp1 = mit->second.fColsInExp1; + + for (vector::iterator kit = exp1.begin(); kit != exp1.end(); kit++) + tupleKeyToProjectStep(*kit, expSteps, jobInfo); + + vector& exp2 = mit->second.fColsInExp2; + + for (vector::iterator kit = exp2.begin(); kit != exp2.end(); kit++) + tupleKeyToProjectStep(*kit, expSteps, jobInfo); + + vector& expRet = mit->second.fColsInRetExp; + + for (vector::iterator kit = expRet.begin(); kit != expRet.end(); kit++) + tupleKeyToProjectStep(*kit, expSteps, jobInfo); + + vector& expOut = mit->second.fColsInOuter; + + for (vector::iterator kit = expOut.begin(); kit != expOut.end(); kit++) + tupleKeyToProjectStep(*kit, expSteps, jobInfo); + + vector& expFj = mit->second.fColsInFuncJoin; + + for (vector::iterator kit = expFj.begin(); kit != expFj.end(); kit++) + tupleKeyToProjectStep(*kit, expSteps, jobInfo); + + // for output rowgroup + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); + + // this psv is a copy of the project steps, the original vector in mit is not changed + JobStepVector psv = mit->second.fProjectSteps; // columns being selected + psv.insert(psv.begin(), keySteps.begin(), keySteps.end()); // add joinkeys to project + psv.insert(psv.end(), expSteps.begin(), expSteps.end()); // add expressions to project + set seenCols; // columns already processed + + // for passthru conversion + // passthru is disabled (default lastTupleId to -1) unless the TupleBPS::bop is BOP_AND. + uint64_t lastTupleId = -1; + TupleBPS* tbps = dynamic_cast(bps); + + if (tbps != NULL && tbps->getBOP() == BOP_AND && exp1.size() == 0) + lastTupleId = tbps->getLastTupleId(); + + for (JobStepVector::iterator it = psv.begin(); it != psv.end(); it++) + { + JobStep* js = it->get(); + uint32_t tupleKey = js->tupleId(); + + if (seenCols.find(tupleKey) != seenCols.end()) + continue; + + // update processed column set + seenCols.insert(tupleKey); + + // if the projected column is the last accessed predicate + pColStep* pcol = dynamic_cast(js); + + if (pcol != NULL && js->tupleId() == lastTupleId) { - if (jobInfo.keyInfo.get()->tupleKeyToTableOid[*kit] != CNX_EXP_TABLE_ID) - tupleKeyToProjectStep(*kit, keySteps, jobInfo); - else - fjKeys.push_back(*kit); + PassThruStep* pts = new PassThruStep(*pcol); + + if (dynamic_cast(pcol)) + pts->pseudoType(dynamic_cast(pcol)->pseudoColumnId()); + + pts->alias(pcol->alias()); + pts->view(pcol->view()); + pts->name(pcol->name()); + pts->tupleId(pcol->tupleId()); + it->reset(pts); } - // construct pcolstep for columns in expresssions - JobStepVector expSteps; - vector& exp1 = mit->second.fColsInExp1; + // add projected column to TBPS + bool tokenOnly = false; + map::iterator toIt = jobInfo.tokenOnly.find(js->tupleId()); - for (vector::iterator kit = exp1.begin(); kit != exp1.end(); kit++) - tupleKeyToProjectStep(*kit, expSteps, jobInfo); + if (toIt != jobInfo.tokenOnly.end()) + tokenOnly = toIt->second; - vector& exp2 = mit->second.fColsInExp2; - - for (vector::iterator kit = exp2.begin(); kit != exp2.end(); kit++) - tupleKeyToProjectStep(*kit, expSteps, jobInfo); - - vector& expRet = mit->second.fColsInRetExp; - - for (vector::iterator kit = expRet.begin(); kit != expRet.end(); kit++) - tupleKeyToProjectStep(*kit, expSteps, jobInfo); - - vector& expOut = mit->second.fColsInOuter; - - for (vector::iterator kit = expOut.begin(); kit != expOut.end(); kit++) - tupleKeyToProjectStep(*kit, expSteps, jobInfo); - - vector& expFj = mit->second.fColsInFuncJoin; - - for (vector::iterator kit = expFj.begin(); kit != expFj.end(); kit++) - tupleKeyToProjectStep(*kit, expSteps, jobInfo); - - // for output rowgroup - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); - - // this psv is a copy of the project steps, the original vector in mit is not changed - JobStepVector psv = mit->second.fProjectSteps; // columns being selected - psv.insert(psv.begin(), keySteps.begin(), keySteps.end()); // add joinkeys to project - psv.insert(psv.end(), expSteps.begin(), expSteps.end()); // add expressions to project - set seenCols; // columns already processed - - // for passthru conversion - // passthru is disabled (default lastTupleId to -1) unless the TupleBPS::bop is BOP_AND. - uint64_t lastTupleId = -1; - TupleBPS* tbps = dynamic_cast(bps); - - if (tbps != NULL && tbps->getBOP() == BOP_AND && exp1.size() == 0) - lastTupleId = tbps->getLastTupleId(); - - for (JobStepVector::iterator it = psv.begin(); it != psv.end(); it++) + if (it->get()->isDictCol() && !tokenOnly) { - JobStep* js = it->get(); - uint32_t tupleKey = js->tupleId(); + // if (jobInfo.trace && bps->tableOid() >= 3000) + // cout << "1 setting project BPP for " << tbps->toString() << " with " + //<< it->get()->toString() << " and " << (it+1)->get()->toString() << endl; + bps->setProjectBPP(it->get(), (it + 1)->get()); - if (seenCols.find(tupleKey) != seenCols.end()) - continue; + // this is a two-step project step, remove the token step from id vector + vector& pjv = mit->second.fProjectCols; + uint32_t tokenKey = js->tupleId(); - // update processed column set - seenCols.insert(tupleKey); - - // if the projected column is the last accessed predicate - pColStep* pcol = dynamic_cast(js); - - if (pcol != NULL && js->tupleId() == lastTupleId) + for (vector::iterator i = pjv.begin(); i != pjv.end(); ++i) + { + if (*i == tokenKey) { - PassThruStep* pts = new PassThruStep(*pcol); - - if (dynamic_cast(pcol)) - pts->pseudoType(dynamic_cast(pcol)->pseudoColumnId()); - - pts->alias(pcol->alias()); - pts->view(pcol->view()); - pts->name(pcol->name()); - pts->tupleId(pcol->tupleId()); - it->reset(pts); + pjv.erase(i); + break; } + } - // add projected column to TBPS - bool tokenOnly = false; - map::iterator toIt = jobInfo.tokenOnly.find(js->tupleId()); - - if (toIt != jobInfo.tokenOnly.end()) - tokenOnly = toIt->second; - - if (it->get()->isDictCol() && !tokenOnly) - { -// if (jobInfo.trace && bps->tableOid() >= 3000) -// cout << "1 setting project BPP for " << tbps->toString() << " with " << -// it->get()->toString() << " and " << (it+1)->get()->toString() << endl; - bps->setProjectBPP(it->get(), (it + 1)->get()); - - // this is a two-step project step, remove the token step from id vector - vector& pjv = mit->second.fProjectCols; - uint32_t tokenKey = js->tupleId(); - - for (vector::iterator i = pjv.begin(); i != pjv.end(); ++i) - { - if (*i == tokenKey) - { - pjv.erase(i); - break; - } - } - - // move to the dictionary step - js = (++it)->get(); - tupleKey = js->tupleId(); - seenCols.insert(tupleKey); - } - else - { -// if (jobInfo.trace && bps->tableOid() >= 3000) -// cout << "2 setting project BPP for " << tbps->toString() << " with " << -// it->get()->toString() << " and " << "NULL" << endl; - bps->setProjectBPP(it->get(), NULL); - } - - // add the tuple info of the column into the RowGroup - TupleInfo ti(getTupleInfo(tupleKey, jobInfo)); - pos.push_back(pos.back() + ti.width); - oids.push_back(ti.oid); - keys.push_back(ti.key); - types.push_back(ti.dtype); - csNums.push_back(ti.csNum); - scale.push_back(ti.scale); - precision.push_back(ti.precision); + // move to the dictionary step + js = (++it)->get(); + tupleKey = js->tupleId(); + seenCols.insert(tupleKey); + } + else + { + // if (jobInfo.trace && bps->tableOid() >= 3000) + // cout << "2 setting project BPP for " << tbps->toString() << " with " + //<< it->get()->toString() << " and " << "NULL" << endl; + bps->setProjectBPP(it->get(), NULL); } - // add function join columns - for (vector::iterator i = fjKeys.begin(); i != fjKeys.end(); i++) - { - TupleInfo ti(getTupleInfo(*i, jobInfo)); - pos.push_back(pos.back() + ti.width); - oids.push_back(ti.oid); - keys.push_back(ti.key); - types.push_back(ti.dtype); - csNums.push_back(ti.csNum); - scale.push_back(ti.scale); - precision.push_back(ti.precision); - } + // add the tuple info of the column into the RowGroup + TupleInfo ti(getTupleInfo(tupleKey, jobInfo)); + pos.push_back(pos.back() + ti.width); + oids.push_back(ti.oid); + keys.push_back(ti.key); + types.push_back(ti.dtype); + csNums.push_back(ti.csNum); + scale.push_back(ti.scale); + precision.push_back(ti.precision); + } - // construct RowGroup - RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + // add function join columns + for (vector::iterator i = fjKeys.begin(); i != fjKeys.end(); i++) + { + TupleInfo ti(getTupleInfo(*i, jobInfo)); + pos.push_back(pos.back() + ti.width); + oids.push_back(ti.oid); + keys.push_back(ti.key); + types.push_back(ti.dtype); + csNums.push_back(ti.csNum); + scale.push_back(ti.scale); + precision.push_back(ti.precision); + } - // fix the output association - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - spdl->rowGroupDL(dl); - dl->OID(mit->first); - JobStepAssociation jsa; - jsa.outAdd(spdl); - bps->outputAssociation(jsa); - bps->setOutputRowGroup(rg); + // construct RowGroup + RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + + // fix the output association + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + spdl->rowGroupDL(dl); + dl->OID(mit->first); + JobStepAssociation jsa; + jsa.outAdd(spdl); + bps->outputAssociation(jsa); + bps->setOutputRowGroup(rg); } - // add one-table expression steps into the query TBPS void addExpresssionStepsToBps(TableInfoMap::iterator& mit, SJSTEP& sjsp, JobInfo& jobInfo) { - BatchPrimitive* bps = dynamic_cast(sjsp.get()); - CalpontSystemCatalog::OID tableOid = mit->second.fTableOid; - JobStepVector& exps = mit->second.fOneTableExpSteps; - JobStepVector& fjs = mit->second.fFuncJoinExps; - ExpressionStep* exp0 = NULL; + BatchPrimitive* bps = dynamic_cast(sjsp.get()); + CalpontSystemCatalog::OID tableOid = mit->second.fTableOid; + JobStepVector& exps = mit->second.fOneTableExpSteps; + JobStepVector& fjs = mit->second.fFuncJoinExps; + ExpressionStep* exp0 = NULL; - if (exps.size() > 0) - exp0 = dynamic_cast(exps[0].get()); + if (exps.size() > 0) + exp0 = dynamic_cast(exps[0].get()); + else + exp0 = dynamic_cast(fjs[0].get()); + + if (bps == NULL) + { + if (tableOid > 0) + { + uint32_t key0 = exp0->columnKey(); + CalpontSystemCatalog::ColType ct = jobInfo.keyInfo->colType[key0]; + map::iterator dkMit; + + if (jobInfo.keyInfo->token2DictTypeMap.find(key0) != jobInfo.keyInfo->token2DictTypeMap.end()) + ct = jobInfo.keyInfo->token2DictTypeMap[key0]; + + scoped_ptr pcss(new pColScanStep(exp0->oid(), tableOid, ct, jobInfo)); + + sjsp.reset(new TupleBPS(*pcss, jobInfo)); + TupleBPS* tbps = dynamic_cast(sjsp.get()); + tbps->setJobInfo(&jobInfo); + tbps->setFirstStepType(SCAN); + + // add the first column to BPP's filterSteps + tbps->setBPP(pcss.get()); + + bps = tbps; + } else - exp0 = dynamic_cast(fjs[0].get()); - - if (bps == NULL) { - if (tableOid > 0) - { - uint32_t key0 = exp0->columnKey(); - CalpontSystemCatalog::ColType ct = jobInfo.keyInfo->colType[key0]; - map::iterator dkMit; + sjsp.reset(new CrossEngineStep(mit->second.fSchema, mit->second.fName, mit->second.fAlias, jobInfo)); - if (jobInfo.keyInfo->token2DictTypeMap.find(key0) != - jobInfo.keyInfo->token2DictTypeMap.end()) - ct = jobInfo.keyInfo->token2DictTypeMap[key0]; + bps = dynamic_cast(sjsp.get()); + } + } - scoped_ptr pcss( - new pColScanStep(exp0->oid(), tableOid, ct, jobInfo)); + // rowgroup for evaluating the one table expression + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); - sjsp.reset(new TupleBPS(*pcss, jobInfo)); - TupleBPS* tbps = dynamic_cast(sjsp.get()); - tbps->setJobInfo(&jobInfo); - tbps->setFirstStepType(SCAN); + vector cols; + JobStepVector& fjExp = mit->second.fFuncJoinExps; - // add the first column to BPP's filterSteps - tbps->setBPP(pcss.get()); + for (JobStepVector::iterator it = fjExp.begin(); it != fjExp.end(); it++) + { + ExpressionStep* e = dynamic_cast(it->get()); + cols.push_back(getExpTupleKey(jobInfo, e->expressionId())); + } - bps = tbps; - } - else - { - sjsp.reset(new CrossEngineStep(mit->second.fSchema, - mit->second.fName, - mit->second.fAlias, - jobInfo)); + cols.insert(cols.end(), mit->second.fColsInExp1.begin(), mit->second.fColsInExp1.end()); + cols.insert(cols.end(), mit->second.fColsInFuncJoin.begin(), mit->second.fColsInFuncJoin.end()); + uint32_t index = 0; // index in the rowgroup + map keyToIndexMap; // maps key to the index in the RG - bps = dynamic_cast(sjsp.get()); - } + for (vector::iterator kit = cols.begin(); kit != cols.end(); kit++) + { + uint32_t key = *kit; + + if (jobInfo.keyInfo->dictKeyMap.find(key) != jobInfo.keyInfo->dictKeyMap.end()) + key = jobInfo.keyInfo->dictKeyMap[key]; + + // check if this key is already in + if (keyToIndexMap.find(key) != keyToIndexMap.end()) + continue; + + // update processed column set + keyToIndexMap.insert(make_pair(key, index++)); + + // add the tuple info of the column into the RowGroup + TupleInfo ti(getTupleInfo(key, jobInfo)); + pos.push_back(pos.back() + ti.width); + oids.push_back(ti.oid); + keys.push_back(ti.key); + types.push_back(ti.dtype); + csNums.push_back(ti.csNum); + scale.push_back(ti.scale); + precision.push_back(ti.precision); + } + + // construct RowGroup and add to TBPS + RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + bps->setFE1Input(rg); + + if (jobInfo.trace) + cout << "FE1 input RowGroup: " << rg.toString() << endl << endl; + + // add the expression steps into TBPS, the input-indices are set in SCs. + for (JobStepVector::iterator it = exps.begin(); it != exps.end(); it++) + { + ExpressionStep* e = dynamic_cast(it->get()); + + if (e->functionJoin()) + continue; + + e->updateInputIndex(keyToIndexMap, jobInfo); + boost::shared_ptr sppt(new ParseTree); + sppt->copyTree(*(e->expressionFilter())); + bps->addFcnExpGroup1(sppt); + } + + // add the function join expression steps into TBPS, too + if (fjs.size() > 0) + { + vector fjCols; + + for (JobStepVector::iterator it = fjs.begin(); it != fjs.end(); it++) + { + ExpressionStep* e = dynamic_cast(it->get()); + + if (e->virtualStep()) + continue; + + e->updateInputIndex(keyToIndexMap, jobInfo); + e->updateOutputIndex(keyToIndexMap, jobInfo); + fjCols.push_back(e->expression()); } - // rowgroup for evaluating the one table expression - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); - - vector cols; - JobStepVector& fjExp = mit->second.fFuncJoinExps; - - for (JobStepVector::iterator it = fjExp.begin(); it != fjExp.end(); it++) - { - ExpressionStep* e = dynamic_cast(it->get()); - cols.push_back(getExpTupleKey(jobInfo, e->expressionId())); - } - - cols.insert(cols.end(), mit->second.fColsInExp1.begin(), mit->second.fColsInExp1.end()); - cols.insert(cols.end(), mit->second.fColsInFuncJoin.begin(), mit->second.fColsInFuncJoin.end()); - uint32_t index = 0; // index in the rowgroup - map keyToIndexMap; // maps key to the index in the RG - - for (vector::iterator kit = cols.begin(); kit != cols.end(); kit++) - { - uint32_t key = *kit; - - if (jobInfo.keyInfo->dictKeyMap.find(key) != jobInfo.keyInfo->dictKeyMap.end()) - key = jobInfo.keyInfo->dictKeyMap[key]; - - // check if this key is already in - if (keyToIndexMap.find(key) != keyToIndexMap.end()) - continue; - - // update processed column set - keyToIndexMap.insert(make_pair(key, index++)); - - // add the tuple info of the column into the RowGroup - TupleInfo ti(getTupleInfo(key, jobInfo)); - pos.push_back(pos.back() + ti.width); - oids.push_back(ti.oid); - keys.push_back(ti.key); - types.push_back(ti.dtype); - csNums.push_back(ti.csNum); - scale.push_back(ti.scale); - precision.push_back(ti.precision); - } - - // construct RowGroup and add to TBPS - RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - bps->setFE1Input(rg); - - if (jobInfo.trace) cout << "FE1 input RowGroup: " << rg.toString() << endl << endl; - - // add the expression steps into TBPS, the input-indices are set in SCs. - for (JobStepVector::iterator it = exps.begin(); it != exps.end(); it++) - { - ExpressionStep* e = dynamic_cast(it->get()); - - if (e->functionJoin()) - continue; - - e->updateInputIndex(keyToIndexMap, jobInfo); - boost::shared_ptr sppt(new ParseTree); - sppt->copyTree(*(e->expressionFilter())); - bps->addFcnExpGroup1(sppt); - } - - // add the function join expression steps into TBPS, too - if (fjs.size() > 0) - { - vector fjCols; - - for (JobStepVector::iterator it = fjs.begin(); it != fjs.end(); it++) - { - ExpressionStep* e = dynamic_cast(it->get()); - - if (e->virtualStep()) - continue; - - e->updateInputIndex(keyToIndexMap, jobInfo); - e->updateOutputIndex(keyToIndexMap, jobInfo); - fjCols.push_back(e->expression()); - } - - bps->addFcnJoinExp(fjCols); - } + bps->addFcnJoinExp(fjCols); + } } - bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) { - TableInfo& tableInfo = mit->second; - JobStepVector& qsv = tableInfo.fQuerySteps; - JobStepVector newSteps; // store combined steps - RowGroup rgOut; // rowgroup of combined steps - CalpontSystemCatalog::OID tableOid = tableInfo.fTableOid; + TableInfo& tableInfo = mit->second; + JobStepVector& qsv = tableInfo.fQuerySteps; + JobStepVector newSteps; // store combined steps + RowGroup rgOut; // rowgroup of combined steps + CalpontSystemCatalog::OID tableOid = tableInfo.fTableOid; - if (tableOid != CNX_VTABLE_ID) + if (tableOid != CNX_VTABLE_ID) + { + // real table + if (qsv.size() == 0) { - // real table - if (qsv.size() == 0) + // find a column in FE1, FE2, or FE3 + uint32_t key = -1; + + if (tableInfo.fColsInExp1.size() > 0) + key = tableInfo.fColsInExp1[0]; + else if (tableInfo.fColsInExp2.size() > 0) + key = tableInfo.fColsInExp2[0]; + else if (tableInfo.fColsInRetExp.size() > 0) + key = tableInfo.fColsInRetExp[0]; + else if (tableInfo.fColsInOuter.size() > 0) + key = tableInfo.fColsInOuter[0]; + else if (tableInfo.fColsInColMap.size() > 0) + key = tableInfo.fColsInColMap[0]; + else + throw runtime_error("No query step"); + + // construct a pcolscanstep to initialize the tbps + CalpontSystemCatalog::OID oid = jobInfo.keyInfo->tupleKeyVec[key].fId; + CalpontSystemCatalog::ColType ct = jobInfo.keyInfo->colType[key]; + map::iterator dkMit; + + if (jobInfo.keyInfo->token2DictTypeMap.find(key) != jobInfo.keyInfo->token2DictTypeMap.end()) + ct = jobInfo.keyInfo->token2DictTypeMap[key]; + + SJSTEP sjs(new pColScanStep(oid, tableOid, ct, jobInfo)); + sjs->alias(jobInfo.keyInfo->tupleKeyVec[key].fTable); + sjs->view(jobInfo.keyInfo->tupleKeyVec[key].fView); + sjs->schema(jobInfo.keyInfo->tupleKeyVec[key].fSchema); + sjs->name(jobInfo.keyInfo->keyName[key]); + sjs->tupleId(key); + qsv.push_back(sjs); + } + + SJSTEP sjsp; // shared_ptr for the new BatchPrimitive + BatchPrimitive* bps = NULL; // pscan/pcol/filter/etc combined to + vector pdsVec; // pds for string filters + JobStepVector::iterator begin = qsv.begin(); + JobStepVector::iterator end = qsv.end(); + JobStepVector::iterator it = begin; + + // make sure there is a pcolscan if there is a pcolstep + while (it != end) + { + if (typeid(*(it->get())) == typeid(pColScanStep)) + break; + + if (typeid(*(it->get())) == typeid(pColStep)) + { + pColStep* pcs = dynamic_cast(it->get()); + (*it).reset(new pColScanStep(*pcs)); + break; + } + + it++; + } + + // ---- predicates ---- + // setup TBPS and dictionaryscan + it = begin; + + while (it != end) + { + if (typeid(*(it->get())) == typeid(pColScanStep)) + { + if (bps == NULL) { - // find a column in FE1, FE2, or FE3 - uint32_t key = -1; + if (tableOid > 0) + { + sjsp.reset(new TupleBPS(*(dynamic_cast(it->get())), jobInfo)); + TupleBPS* tbps = dynamic_cast(sjsp.get()); + tbps->setJobInfo(&jobInfo); + tbps->setFirstStepType(SCAN); + bps = tbps; + } + else + { + sjsp.reset( + new CrossEngineStep(mit->second.fSchema, mit->second.fName, mit->second.fAlias, jobInfo)); + bps = dynamic_cast(sjsp.get()); + } + } + else + { + pColScanStep* pcss = dynamic_cast(it->get()); + (*it).reset(new pColStep(*pcss)); + } + } - if (tableInfo.fColsInExp1.size() > 0) - key = tableInfo.fColsInExp1[0]; - else if (tableInfo.fColsInExp2.size() > 0) - key = tableInfo.fColsInExp2[0]; - else if (tableInfo.fColsInRetExp.size() > 0) - key = tableInfo.fColsInRetExp[0]; - else if (tableInfo.fColsInOuter.size() > 0) - key = tableInfo.fColsInOuter[0]; - else if (tableInfo.fColsInColMap.size() > 0) - key = tableInfo.fColsInColMap[0]; - else - throw runtime_error("No query step"); + unsigned itInc = 1; // iterator increase number + unsigned numOfStepsAddToBps = 0; // # steps to be added into TBPS - // construct a pcolscanstep to initialize the tbps - CalpontSystemCatalog::OID oid = jobInfo.keyInfo->tupleKeyVec[key].fId; - CalpontSystemCatalog::ColType ct = jobInfo.keyInfo->colType[key]; - map::iterator dkMit; + if ((std::distance(it, end) > 2 && dynamic_cast(it->get()) != NULL && + (dynamic_cast((it + 1)->get()) != NULL || + dynamic_cast((it + 1)->get()) != NULL) && + dynamic_cast((it + 2)->get()) != NULL) || + (std::distance(it, end) > 1 && dynamic_cast(it->get()) != NULL && + dynamic_cast((it + 1)->get()) != NULL)) + { + // string access predicate + // setup pDictionaryScan + pDictionaryScan* pds = dynamic_cast(it->get()); + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); - if (jobInfo.keyInfo->token2DictTypeMap.find(key) != - jobInfo.keyInfo->token2DictTypeMap.end()) - ct = jobInfo.keyInfo->token2DictTypeMap[key]; + pos.push_back(2 + 8); + CalpontSystemCatalog::OID coid = jobInfo.keyInfo->dictOidToColOid[pds->oid()]; + oids.push_back(coid); + uint32_t keyId = pds->tupleId(); + keys.push_back(keyId); + types.push_back(CalpontSystemCatalog::BIGINT); + csNums.push_back(pds->colType().charsetNumber); + scale.push_back(0); + precision.push_back(0); - SJSTEP sjs(new pColScanStep(oid, tableOid, ct, jobInfo)); - sjs->alias(jobInfo.keyInfo->tupleKeyVec[key].fTable); - sjs->view(jobInfo.keyInfo->tupleKeyVec[key].fView); - sjs->schema(jobInfo.keyInfo->tupleKeyVec[key].fSchema); - sjs->name(jobInfo.keyInfo->keyName[key]); - sjs->tupleId(key); - qsv.push_back(sjs); + RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, + jobInfo.stringTableThreshold); + + if (jobInfo.trace) + cout << "RowGroup pds(and): " << rg.toString() << endl; + + pds->setOutputRowGroup(rg); + newSteps.push_back(*it); + + DictionaryScanInfo pdsInfo; + pdsInfo.fTokenId = keyId; + pdsInfo.fDl = pds->outputAssociation().outAt(0); + pdsInfo.fRowGroup = rg; + pdsVec.push_back(pdsInfo); + + // save the token join to the last + itInc = 1; + numOfStepsAddToBps = 0; + } + else if (std::distance(begin, it) > 1 && + (dynamic_cast((it - 1)->get()) != NULL || + dynamic_cast((it - 2)->get()) != NULL) && + dynamic_cast(it->get()) != NULL) + { + // save the token join to the last, by pdsInfo + itInc = 1; + numOfStepsAddToBps = 0; + } + else if (std::distance(it, end) > 2 && dynamic_cast((it + 1)->get()) != NULL && + dynamic_cast((it + 2)->get()) != NULL) + { + itInc = 3; + numOfStepsAddToBps = 3; + } + else if (std::distance(it, end) > 3 && dynamic_cast((it + 1)->get()) != NULL && + dynamic_cast((it + 2)->get()) != NULL && + dynamic_cast((it + 3)->get()) != NULL) + { + itInc = 4; + numOfStepsAddToBps = 4; + } + else if (std::distance(it, end) > 3 && dynamic_cast((it + 1)->get()) != NULL && + dynamic_cast((it + 2)->get()) != NULL && + dynamic_cast((it + 3)->get()) != NULL) + { + itInc = 4; + numOfStepsAddToBps = 4; + } + else if (std::distance(it, end) > 4 && dynamic_cast((it + 1)->get()) != NULL && + dynamic_cast((it + 2)->get()) != NULL && + dynamic_cast((it + 3)->get()) != NULL && + dynamic_cast((it + 4)->get()) != NULL) + { + itInc = 5; + numOfStepsAddToBps = 5; + } + else if (std::distance(it, end) > 1 && + (dynamic_cast(it->get()) != NULL || + dynamic_cast(it->get()) != NULL) && + dynamic_cast((it + 1)->get()) != NULL) + { + itInc = 2; + numOfStepsAddToBps = 2; + } + else if (dynamic_cast(it->get()) != NULL) + { + pColStep* pcol = dynamic_cast(it->get()); + + if (pcol->getFilters().size() == 0) + { + // not an access predicate, pcol for token will be added later if necessary + numOfStepsAddToBps = 0; + } + else + { + numOfStepsAddToBps = 1; } - SJSTEP sjsp; // shared_ptr for the new BatchPrimitive - BatchPrimitive* bps = NULL; // pscan/pcol/filter/etc combined to - vector pdsVec; // pds for string filters - JobStepVector::iterator begin = qsv.begin(); - JobStepVector::iterator end = qsv.end(); - JobStepVector::iterator it = begin; + itInc = 1; + } + else if (dynamic_cast(it->get()) != NULL) + { + numOfStepsAddToBps = 1; + itInc = 1; + } + else + { + // Not a combinable step, or step pattern not recognized. + cerr << boldStart << "Try to combine " << typeid(*(it->get())).name() << ": " << it->get()->oid() + << " into TBPS" << boldStop << endl; + return false; + } - // make sure there is a pcolscan if there is a pcolstep - while (it != end) + // now add the steps into the TBPS + if (numOfStepsAddToBps > 0 && bps == NULL) + throw runtime_error("BPS not created 1"); + + for (unsigned i = 0; i < numOfStepsAddToBps; i++) + { + bps->setBPP((it + i)->get()); + bps->setStepCount(); + bps->setLastTupleId((it + i)->get()->tupleId()); + } + + it += itInc; + } + + // add one-table expression steps to TBPS + if (tableInfo.fOneTableExpSteps.size() > 0 || tableInfo.fFuncJoinExps.size() > 0) + addExpresssionStepsToBps(mit, sjsp, jobInfo); + + // add TBPS to the step vector + newSteps.push_back(sjsp); + + // ---- projects ---- + // now, add the joinkeys to the project step vector + addProjectStepsToBps(mit, bps, jobInfo); + + // rowgroup has the joinkeys and selected columns + // this is the expected output of this table + rgOut = bps->getOutputRowGroup(); + + // add token joins + if (pdsVec.size() > 0) + { + // ---- token joins ---- + // construct a TupleHashJoinStep + TupleBPS* tbps = dynamic_cast(bps); + TupleHashJoinStep* thjs = new TupleHashJoinStep(jobInfo); + thjs->tableOid1(0); + thjs->tableOid2(tableInfo.fTableOid); + thjs->alias1(tableInfo.fAlias); + thjs->alias2(tableInfo.fAlias); + thjs->view1(tableInfo.fView); + thjs->view2(tableInfo.fView); + thjs->schema1(tableInfo.fSchema); + thjs->schema2(tableInfo.fSchema); + thjs->setLargeSideBPS(tbps); + thjs->joinId(-1); // token join is a filter force it done before other joins + thjs->setJoinType(INNER); + thjs->tokenJoin(mit->first); + tbps->incWaitToRunStepCnt(); + SJSTEP spthjs(thjs); + + // rowgroup of the TBPS side + // start with the expected output of the table, tokens will be appended + RowGroup rgTbps = rgOut; + + // input jobstepassociation + // 1. small sides -- pdictionaryscan steps + vector rgPdsVec; + map addedCol; + vector jointypes; + vector typeless; + vector> smallKeyIndices; + vector> largeKeyIndices; + vector tableNames; + JobStepAssociation inJsa; + + for (vector::iterator i = pdsVec.begin(); i != pdsVec.end(); i++) + { + // add the token steps to the TBPS + uint32_t tupleKey = i->fTokenId; + map::iterator k = addedCol.find(tupleKey); + unsigned largeSideIndex = rgTbps.getColumnCount(); + + if (k == addedCol.end()) { - if (typeid(*(it->get())) == typeid(pColScanStep)) - break; + SJSTEP sjs(new pColStep(jobInfo.keyInfo->tupleKeyVec[tupleKey].fId, tableInfo.fTableOid, + jobInfo.keyInfo->token2DictTypeMap[tupleKey], jobInfo)); + sjs->alias(tableInfo.fAlias); + sjs->view(tableInfo.fView); + sjs->schema(tableInfo.fSchema); + sjs->name(jobInfo.keyInfo->keyName[tupleKey]); + sjs->tupleId(tupleKey); + bps->setProjectBPP(sjs.get(), NULL); - if (typeid(*(it->get())) == typeid(pColStep)) - { - pColStep* pcs = dynamic_cast(it->get()); - (*it).reset(new pColScanStep(*pcs)); - break; - } - - it++; + // Update info, which will be used to config the hashjoin later. + rgTbps += i->fRowGroup; + addedCol[tupleKey] = largeSideIndex; + } + else + { + largeSideIndex = k->second; } - // ---- predicates ---- - // setup TBPS and dictionaryscan - it = begin; + inJsa.outAdd(i->fDl); + tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[tupleKey].fTable); + rgPdsVec.push_back(i->fRowGroup); + jointypes.push_back(INNER); + typeless.push_back(false); + smallKeyIndices.push_back(vector(1, 0)); + largeKeyIndices.push_back(vector(1, largeSideIndex)); + } - while (it != end) - { - if (typeid(*(it->get())) == typeid(pColScanStep)) - { - if (bps == NULL) - { - if (tableOid > 0) - { - sjsp.reset(new TupleBPS(*(dynamic_cast(it->get())), jobInfo)); - TupleBPS* tbps = dynamic_cast(sjsp.get()); - tbps->setJobInfo(&jobInfo); - tbps->setFirstStepType(SCAN); - bps = tbps; - } - else - { - sjsp.reset(new CrossEngineStep(mit->second.fSchema, - mit->second.fName, - mit->second.fAlias, - jobInfo)); - bps = dynamic_cast(sjsp.get()); - } - } - else - { - pColScanStep* pcss = dynamic_cast(it->get()); - (*it).reset(new pColStep(*pcss)); - } - } + // 2. large side + if (jobInfo.trace) + cout << "RowGroup bps(and): " << rgTbps.toString() << endl; - unsigned itInc = 1; // iterator increase number - unsigned numOfStepsAddToBps = 0; // # steps to be added into TBPS + bps->setOutputRowGroup(rgTbps); + inJsa.outAdd(bps->outputAssociation().outAt(0)); - if ((std::distance(it, end) > 2 && - dynamic_cast(it->get()) != NULL && - (dynamic_cast((it + 1)->get()) != NULL || - dynamic_cast((it + 1)->get()) != NULL) && - dynamic_cast((it + 2)->get()) != NULL) || - (std::distance(it, end) > 1 && - dynamic_cast(it->get()) != NULL && - dynamic_cast((it + 1)->get()) != NULL)) - { - // string access predicate - // setup pDictionaryScan - pDictionaryScan* pds = dynamic_cast(it->get()); - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); + // set input jobstepassociation + thjs->inputAssociation(inJsa); + thjs->setLargeSideDLIndex(inJsa.outSize() - 1); - pos.push_back(2 + 8); - CalpontSystemCatalog::OID coid = jobInfo.keyInfo->dictOidToColOid[pds->oid()]; - oids.push_back(coid); - uint32_t keyId = pds->tupleId(); - keys.push_back(keyId); - types.push_back(CalpontSystemCatalog::BIGINT); - csNums.push_back(pds->colType().charsetNumber); - scale.push_back(0); - precision.push_back(0); + // output jobstepassociation + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + spdl->rowGroupDL(dl); + dl->OID(mit->first); + JobStepAssociation jsaOut; + jsaOut.outAdd(spdl); + thjs->outputAssociation(jsaOut); - RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + // config the tuplehashjoin + thjs->configSmallSideRG(rgPdsVec, tableNames); + thjs->configLargeSideRG(rgTbps); + thjs->configJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); + thjs->setOutputRowGroup(rgOut); + newSteps.push_back(spthjs); + } + } + else + { + // table derived from subquery + SubQueryStep* subStep = NULL; + SubAdapterStep* adaStep = NULL; - if (jobInfo.trace) cout << "RowGroup pds(and): " << rg.toString() << endl; + for (JobStepVector::iterator it = qsv.begin(); it != qsv.end(); it++) + { + if (((subStep = dynamic_cast(it->get())) != NULL) || + ((adaStep = dynamic_cast(it->get())) != NULL)) + newSteps.push_back(*it); + } - pds->setOutputRowGroup(rg); - newSteps.push_back(*it); + if (subStep == NULL && adaStep == NULL) + throw runtime_error("No step for subquery."); - DictionaryScanInfo pdsInfo; - pdsInfo.fTokenId = keyId; - pdsInfo.fDl = pds->outputAssociation().outAt(0); - pdsInfo.fRowGroup = rg; - pdsVec.push_back(pdsInfo); - - // save the token join to the last - itInc = 1; - numOfStepsAddToBps = 0; - } - else if (std::distance(begin, it) > 1 && - (dynamic_cast((it - 1)->get()) != NULL || - dynamic_cast((it - 2)->get()) != NULL) && - dynamic_cast(it->get()) != NULL) - { - // save the token join to the last, by pdsInfo - itInc = 1; - numOfStepsAddToBps = 0; - } - else if (std::distance(it, end) > 2 && - dynamic_cast((it + 1)->get()) != NULL && - dynamic_cast((it + 2)->get()) != NULL) - { - itInc = 3; - numOfStepsAddToBps = 3; - } - else if (std::distance(it, end) > 3 && - dynamic_cast((it + 1)->get()) != NULL && - dynamic_cast((it + 2)->get()) != NULL && - dynamic_cast((it + 3)->get()) != NULL) - { - itInc = 4; - numOfStepsAddToBps = 4; - } - else if (std::distance(it, end) > 3 && - dynamic_cast((it + 1)->get()) != NULL && - dynamic_cast((it + 2)->get()) != NULL && - dynamic_cast((it + 3)->get()) != NULL) - { - itInc = 4; - numOfStepsAddToBps = 4; - } - else if (std::distance(it, end) > 4 && - dynamic_cast((it + 1)->get()) != NULL && - dynamic_cast((it + 2)->get()) != NULL && - dynamic_cast((it + 3)->get()) != NULL && - dynamic_cast((it + 4)->get()) != NULL) - { - itInc = 5; - numOfStepsAddToBps = 5; - } - else if (std::distance(it, end) > 1 && - (dynamic_cast(it->get()) != NULL || - dynamic_cast(it->get()) != NULL) && - dynamic_cast((it + 1)->get()) != NULL) - { - itInc = 2; - numOfStepsAddToBps = 2; - } - else if (dynamic_cast(it->get()) != NULL) - { - pColStep* pcol = dynamic_cast(it->get()); - - if (pcol->getFilters().size() == 0) - { - // not an access predicate, pcol for token will be added later if necessary - numOfStepsAddToBps = 0; - } - else - { - numOfStepsAddToBps = 1; - } - - itInc = 1; - } - else if (dynamic_cast(it->get()) != NULL) - { - numOfStepsAddToBps = 1; - itInc = 1; - } - else - { - // Not a combinable step, or step pattern not recognized. - cerr << boldStart << "Try to combine " << typeid(*(it->get())).name() << ": " - << it->get()->oid() << " into TBPS" << boldStop << endl; - return false; - } - - // now add the steps into the TBPS - if (numOfStepsAddToBps > 0 && bps == NULL) - throw runtime_error("BPS not created 1"); - - for (unsigned i = 0; i < numOfStepsAddToBps; i++) - { - bps->setBPP((it + i)->get()); - bps->setStepCount(); - bps->setLastTupleId((it + i)->get()->tupleId()); - } - - it += itInc; - } - - // add one-table expression steps to TBPS - if (tableInfo.fOneTableExpSteps.size() > 0 || tableInfo.fFuncJoinExps.size() > 0) - addExpresssionStepsToBps(mit, sjsp, jobInfo); - - // add TBPS to the step vector - newSteps.push_back(sjsp); - - // ---- projects ---- - // now, add the joinkeys to the project step vector - addProjectStepsToBps(mit, bps, jobInfo); - - // rowgroup has the joinkeys and selected columns - // this is the expected output of this table - rgOut = bps->getOutputRowGroup(); - - // add token joins - if (pdsVec.size() > 0) - { - // ---- token joins ---- - // construct a TupleHashJoinStep - TupleBPS* tbps = dynamic_cast(bps); - TupleHashJoinStep* thjs = new TupleHashJoinStep(jobInfo); - thjs->tableOid1(0); - thjs->tableOid2(tableInfo.fTableOid); - thjs->alias1(tableInfo.fAlias); - thjs->alias2(tableInfo.fAlias); - thjs->view1(tableInfo.fView); - thjs->view2(tableInfo.fView); - thjs->schema1(tableInfo.fSchema); - thjs->schema2(tableInfo.fSchema); - thjs->setLargeSideBPS(tbps); - thjs->joinId(-1); // token join is a filter force it done before other joins - thjs->setJoinType(INNER); - thjs->tokenJoin(mit->first); - tbps->incWaitToRunStepCnt(); - SJSTEP spthjs(thjs); - - // rowgroup of the TBPS side - // start with the expected output of the table, tokens will be appended - RowGroup rgTbps = rgOut; - - // input jobstepassociation - // 1. small sides -- pdictionaryscan steps - vector rgPdsVec; - map addedCol; - vector jointypes; - vector typeless; - vector > smallKeyIndices; - vector > largeKeyIndices; - vector tableNames; - JobStepAssociation inJsa; - - for (vector::iterator i = pdsVec.begin(); i != pdsVec.end(); i++) - { - // add the token steps to the TBPS - uint32_t tupleKey = i->fTokenId; - map::iterator k = addedCol.find(tupleKey); - unsigned largeSideIndex = rgTbps.getColumnCount(); - - if (k == addedCol.end()) - { - SJSTEP sjs(new pColStep(jobInfo.keyInfo->tupleKeyVec[tupleKey].fId, - tableInfo.fTableOid, - jobInfo.keyInfo->token2DictTypeMap[tupleKey], - jobInfo)); - sjs->alias(tableInfo.fAlias); - sjs->view(tableInfo.fView); - sjs->schema(tableInfo.fSchema); - sjs->name(jobInfo.keyInfo->keyName[tupleKey]); - sjs->tupleId(tupleKey); - bps->setProjectBPP(sjs.get(), NULL); - - // Update info, which will be used to config the hashjoin later. - rgTbps += i->fRowGroup; - addedCol[tupleKey] = largeSideIndex; - } - else - { - largeSideIndex = k->second; - } - - inJsa.outAdd(i->fDl); - tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[tupleKey].fTable); - rgPdsVec.push_back(i->fRowGroup); - jointypes.push_back(INNER); - typeless.push_back(false); - smallKeyIndices.push_back(vector(1, 0)); - largeKeyIndices.push_back(vector(1, largeSideIndex)); - } - - // 2. large side - if (jobInfo.trace) cout << "RowGroup bps(and): " << rgTbps.toString() << endl; - - bps->setOutputRowGroup(rgTbps); - inJsa.outAdd(bps->outputAssociation().outAt(0)); - - // set input jobstepassociation - thjs->inputAssociation(inJsa); - thjs->setLargeSideDLIndex(inJsa.outSize() - 1); - - // output jobstepassociation - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - spdl->rowGroupDL(dl); - dl->OID(mit->first); - JobStepAssociation jsaOut; - jsaOut.outAdd(spdl); - thjs->outputAssociation(jsaOut); - - // config the tuplehashjoin - thjs->configSmallSideRG(rgPdsVec, tableNames); - thjs->configLargeSideRG(rgTbps); - thjs->configJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); - thjs->setOutputRowGroup(rgOut); - newSteps.push_back(spthjs); - } + if (subStep) + { + rgOut = subStep->getOutputRowGroup(); } else { - // table derived from subquery - SubQueryStep* subStep = NULL; - SubAdapterStep* adaStep = NULL; + // add one-table expression steps to the adapter + if (tableInfo.fOneTableExpSteps.size() > 0) + adaStep->addExpression(tableInfo.fOneTableExpSteps, jobInfo); - for (JobStepVector::iterator it = qsv.begin(); it != qsv.end(); it++) + // add function join steps + if (tableInfo.fFuncJoinExps.size() > 0) + { + // fe rowgroup info + RowGroup feRg = adaStep->getFeRowGroup(); + + if (feRg.getColumnCount() == 0) + feRg = adaStep->getOutputRowGroup(); + + const vector& feKeys = feRg.getKeys(); + map keyToIndexMapFe; + + for (uint64_t i = 0; i < feKeys.size(); ++i) + keyToIndexMapFe.insert(make_pair(feKeys[i], i)); + + // output rowgroup info + const RowGroup& outRg = adaStep->getOutputRowGroup(); + const vector& outKeys = outRg.getKeys(); + map keyToIndexMapOut; + + for (uint64_t i = 0; i < outKeys.size(); ++i) + keyToIndexMapOut.insert(make_pair(outKeys[i], i)); + + // make sure the function join columns are present in the rgs + vector fjKeys; + vector fjCols; + TupleInfoVector tis; + uint64_t lastFeIdx = feKeys.size(); + JobStepVector& fjs = tableInfo.fFuncJoinExps; + + for (JobStepVector::iterator it = fjs.begin(); it != fjs.end(); it++) { - if (((subStep = dynamic_cast(it->get())) != NULL) || - ((adaStep = dynamic_cast(it->get())) != NULL)) - newSteps.push_back(*it); + ExpressionStep* e = dynamic_cast(it->get()); + TupleInfo ti = setExpTupleInfo(e->expression().get(), jobInfo); + + if (find(feKeys.begin(), feKeys.end(), ti.key) == feKeys.end()) + { + tis.push_back(ti); + keyToIndexMapFe.insert(make_pair(ti.key, lastFeIdx++)); + } + + e->updateInputIndex(keyToIndexMapFe, jobInfo); + e->updateOutputIndex(keyToIndexMapFe, jobInfo); + fjCols.push_back(e->expression()); } - if (subStep == NULL && adaStep == NULL) - throw runtime_error("No step for subquery."); + // additional fields in the rowgroup + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); - if (subStep) + for (unsigned i = 0; i < tis.size(); i++) { - rgOut = subStep->getOutputRowGroup(); + pos.push_back(pos.back() + tis[i].width); + oids.push_back(tis[i].oid); + keys.push_back(tis[i].key); + types.push_back(tis[i].dtype); + csNums.push_back(tis[i].csNum); + scale.push_back(tis[i].scale); + precision.push_back(tis[i].precision); } - else + + RowGroup addRg(oids.size(), pos, oids, keys, types, csNums, scale, precision, + jobInfo.stringTableThreshold); + + RowGroup feRg1 = feRg; + RowGroup outRg1 = outRg; + + if (addRg.getColumnCount() > 0) { - // add one-table expression steps to the adapter - if (tableInfo.fOneTableExpSteps.size() > 0) - adaStep->addExpression(tableInfo.fOneTableExpSteps, jobInfo); - - // add function join steps - if (tableInfo.fFuncJoinExps.size() > 0) - { - // fe rowgroup info - RowGroup feRg = adaStep->getFeRowGroup(); - - if (feRg.getColumnCount() == 0) - feRg = adaStep->getOutputRowGroup(); - - const vector& feKeys = feRg.getKeys(); - map keyToIndexMapFe; - - for (uint64_t i = 0; i < feKeys.size(); ++i) - keyToIndexMapFe.insert(make_pair(feKeys[i], i)); - - // output rowgroup info - const RowGroup& outRg = adaStep->getOutputRowGroup(); - const vector& outKeys = outRg.getKeys(); - map keyToIndexMapOut; - - for (uint64_t i = 0; i < outKeys.size(); ++i) - keyToIndexMapOut.insert(make_pair(outKeys[i], i)); - - // make sure the function join columns are present in the rgs - vector fjKeys; - vector fjCols; - TupleInfoVector tis; - uint64_t lastFeIdx = feKeys.size(); - JobStepVector& fjs = tableInfo.fFuncJoinExps; - - for (JobStepVector::iterator it = fjs.begin(); it != fjs.end(); it++) - { - ExpressionStep* e = dynamic_cast(it->get()); - TupleInfo ti = setExpTupleInfo(e->expression().get(), jobInfo); - - if (find(feKeys.begin(), feKeys.end(), ti.key) == feKeys.end()) - { - tis.push_back(ti); - keyToIndexMapFe.insert(make_pair(ti.key, lastFeIdx++)); - } - - e->updateInputIndex(keyToIndexMapFe, jobInfo); - e->updateOutputIndex(keyToIndexMapFe, jobInfo); - fjCols.push_back(e->expression()); - } - - // additional fields in the rowgroup - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); - - for (unsigned i = 0; i < tis.size(); i++) - { - pos.push_back(pos.back() + tis[i].width); - oids.push_back(tis[i].oid); - keys.push_back(tis[i].key); - types.push_back(tis[i].dtype); - csNums.push_back(tis[i].csNum); - scale.push_back(tis[i].scale); - precision.push_back(tis[i].precision); - } - - RowGroup addRg(oids.size(), pos, oids, keys, types, csNums, scale, precision, - jobInfo.stringTableThreshold); - - RowGroup feRg1 = feRg; - RowGroup outRg1 = outRg; - - if (addRg.getColumnCount() > 0) - { - feRg1 += addRg; - outRg1 += addRg; - } - - adaStep->addFcnJoinExp(fjCols); - adaStep->setFeRowGroup(feRg1); - adaStep->setOutputRowGroup(outRg1); - } - - rgOut = adaStep->getOutputRowGroup(); + feRg1 += addRg; + outRg1 += addRg; } + + adaStep->addFcnJoinExp(fjCols); + adaStep->setFeRowGroup(feRg1); + adaStep->setOutputRowGroup(outRg1); + } + + rgOut = adaStep->getOutputRowGroup(); } + } - tableInfo.fDl = newSteps.back()->outputAssociation().outAt(0); - tableInfo.fRowGroup = rgOut; + tableInfo.fDl = newSteps.back()->outputAssociation().outAt(0); + tableInfo.fRowGroup = rgOut; - if (jobInfo.trace) - cout << "RowGroup for " << mit->first << " : " << mit->second.fRowGroup.toString() << endl; + if (jobInfo.trace) + cout << "RowGroup for " << mit->first << " : " << mit->second.fRowGroup.toString() << endl; - qsv.swap(newSteps); + qsv.swap(newSteps); - return true; + return true; } - -bool addFunctionJoin(vector& joinedTables, JobStepVector& joinSteps, - set& nodeSet, set >& pathSet, - TableInfoMap& tableInfoMap, JobInfo& jobInfo) +bool addFunctionJoin(vector& joinedTables, JobStepVector& joinSteps, set& nodeSet, + set>& pathSet, TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - // @bug3683, adding function joins for not joined tables, one pair at a time. - // design review comment: - // all convertable expressions between a pair of tables should be done all, or none. - vector::iterator i = jobInfo.functionJoins.begin(); // candidates - set > functionJoinPairs; // pairs - bool added = false; // new node added + // @bug3683, adding function joins for not joined tables, one pair at a time. + // design review comment: + // all convertable expressions between a pair of tables should be done all, or none. + vector::iterator i = jobInfo.functionJoins.begin(); // candidates + set> functionJoinPairs; // pairs + bool added = false; // new node added - // for function join tables' scope checking, not to try semi join inside subquery. - set tables; // tables to join - tables.insert(jobInfo.tableList.begin(), jobInfo.tableList.end()); + // for function join tables' scope checking, not to try semi join inside subquery. + set tables; // tables to join + tables.insert(jobInfo.tableList.begin(), jobInfo.tableList.end()); - // table pairs to be joined by function joins - TableJoinMap::iterator m1 = jobInfo.tableJoinMap.end(); - TableJoinMap::iterator m2 = jobInfo.tableJoinMap.end(); + // table pairs to be joined by function joins + TableJoinMap::iterator m1 = jobInfo.tableJoinMap.end(); + TableJoinMap::iterator m2 = jobInfo.tableJoinMap.end(); - for (; (i != jobInfo.functionJoins.end()); i++) + for (; (i != jobInfo.functionJoins.end()); i++) + { + ExpressionStep* es = dynamic_cast((*i)); + idbassert(es); + + if (es->functionJoin()) + continue; // already converted to a join + + boost::shared_ptr fji = es->functionJoinInfo(); + uint32_t key1 = fji->fJoinKey[0]; + uint32_t key2 = fji->fJoinKey[1]; + uint32_t tid1 = fji->fTableKey[0]; + uint32_t tid2 = fji->fTableKey[1]; + + if (nodeSet.find(tid1) != nodeSet.end() && nodeSet.find(tid2) != nodeSet.end()) + continue; // both connected, will be a cycle if added. + + if (nodeSet.find(tid1) == nodeSet.end() && nodeSet.find(tid2) == nodeSet.end()) + continue; // both isolated, wait until one is connected. + + if (tables.find(tid1) == tables.end() || tables.find(tid2) == tables.end()) + continue; // sub-query case + + // one & only one is already connected + pair p(tid1, tid2); + + if (functionJoinPairs.empty()) { - ExpressionStep* es = dynamic_cast((*i)); - idbassert(es); + functionJoinPairs.insert(make_pair(tid1, tid2)); + functionJoinPairs.insert(make_pair(tid2, tid1)); + tableInfoMap[tid1].fAdjacentList.push_back(tid2); + tableInfoMap[tid2].fAdjacentList.push_back(tid1); - if (es->functionJoin()) - continue; // already converted to a join + if (find(joinedTables.begin(), joinedTables.end(), tid1) == joinedTables.end()) + { + joinedTables.push_back(tid1); + nodeSet.insert(tid1); + pathSet.insert(make_pair(tid2, tid1)); + } + else + { + joinedTables.push_back(tid2); + nodeSet.insert(tid2); + pathSet.insert(make_pair(tid1, tid2)); + } - boost::shared_ptr fji = es->functionJoinInfo(); - uint32_t key1 = fji->fJoinKey[0]; - uint32_t key2 = fji->fJoinKey[1]; - uint32_t tid1 = fji->fTableKey[0]; - uint32_t tid2 = fji->fTableKey[1]; + added = true; - if (nodeSet.find(tid1) != nodeSet.end() && nodeSet.find(tid2) != nodeSet.end()) - continue; // both connected, will be a cycle if added. + m1 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid1, tid2), JoinData())); + m2 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid2, tid1), JoinData())); - if (nodeSet.find(tid1) == nodeSet.end() && nodeSet.find(tid2) == nodeSet.end()) - continue; // both isolated, wait until one is connected. + if (m1 == jobInfo.tableJoinMap.end() || m2 == jobInfo.tableJoinMap.end()) + throw runtime_error("Bad table map."); - if (tables.find(tid1) == tables.end() || tables.find(tid2) == tables.end()) - continue; // sub-query case + TupleInfo ti1 = getTupleInfo(key1, jobInfo); + TupleInfo ti2 = getTupleInfo(key2, jobInfo); - // one & only one is already connected - pair p(tid1, tid2); - - if (functionJoinPairs.empty()) - { - functionJoinPairs.insert(make_pair(tid1, tid2)); - functionJoinPairs.insert(make_pair(tid2, tid1)); - tableInfoMap[tid1].fAdjacentList.push_back(tid2); - tableInfoMap[tid2].fAdjacentList.push_back(tid1); - - if (find(joinedTables.begin(), joinedTables.end(), tid1) == joinedTables.end()) - { - joinedTables.push_back(tid1); - nodeSet.insert(tid1); - pathSet.insert(make_pair(tid2, tid1)); - } - else - { - joinedTables.push_back(tid2); - nodeSet.insert(tid2); - pathSet.insert(make_pair(tid1, tid2)); - } - - added = true; - - m1 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid1, tid2), JoinData())); - m2 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid2, tid1), JoinData())); - - if (m1 == jobInfo.tableJoinMap.end() || m2 == jobInfo.tableJoinMap.end()) - throw runtime_error("Bad table map."); - - TupleInfo ti1 = getTupleInfo(key1, jobInfo); - TupleInfo ti2 = getTupleInfo(key2, jobInfo); - - // Enable Typeless JOIN for char and wide decimal types. - if (datatypes::isCharType(ti1.dtype) || - (datatypes::isWideDecimalType(ti1.dtype, ti1.width) || - datatypes::isWideDecimalType(ti2.dtype, ti2.width))) - m1->second.fTypeless = m2->second.fTypeless = true; // ti2 is compatible - else - m1->second.fTypeless = m2->second.fTypeless = false; - } - else if (functionJoinPairs.find(p) == functionJoinPairs.end()) - { - continue; // different path - } - else - { - // path already added, do compound join - m1->second.fTypeless = m2->second.fTypeless = true; - } - - // simple or compound function join - es->functionJoin(true); - updateTableKey(key1, tid1, jobInfo); - updateTableKey(key2, tid2, jobInfo); - - tableInfoMap[tid1].fJoinKeys.push_back(key1); - tableInfoMap[tid2].fJoinKeys.push_back(key2); - - if (fji->fStep[0]) - tableInfoMap[tid1].fFuncJoinExps.push_back(fji->fStep[0]); - - if (fji->fStep[1]) - tableInfoMap[tid2].fFuncJoinExps.push_back(fji->fStep[1]); - - vector& cols1 = tableInfoMap[tid1].fColsInFuncJoin; - cols1.insert(cols1.end(), fji->fColumnKeys[0].begin(), fji->fColumnKeys[0].end()); - vector& cols2 = tableInfoMap[tid2].fColsInFuncJoin; - cols2.insert(cols2.end(), fji->fColumnKeys[1].begin(), fji->fColumnKeys[1].end()); - - // construct a join step - TupleHashJoinStep* thjs = new TupleHashJoinStep(jobInfo); - thjs->tableOid1(fji->fTableOid[0]); - thjs->tableOid2(fji->fTableOid[1]); - thjs->oid1(fji->fOid[0]); - thjs->oid2(fji->fOid[1]); - thjs->alias1(fji->fAlias[0]); - thjs->alias2(fji->fAlias[1]); - thjs->view1(fji->fView[0]); - thjs->view2(fji->fView[1]); - thjs->schema1(fji->fSchema[0]); - thjs->schema2(fji->fSchema[1]); - thjs->column1(fji->fExpression[0]); - thjs->column2(fji->fExpression[1]); - thjs->sequence1(fji->fSequence[0]); - thjs->sequence2(fji->fSequence[1]); - thjs->joinId(fji->fJoinId); - thjs->setJoinType(fji->fJoinType); - thjs->funcJoinInfo(fji); - thjs->tupleId1(key1); - thjs->tupleId2(key2); - SJSTEP spjs(thjs); - - // check correlated info - JoinType joinType = fji->fJoinType; - - if (!(joinType & CORRELATED)) - { - joinSteps.push_back(spjs); - - // Keep a map of the join (table, key) pairs - m1->second.fLeftKeys.push_back(key1); - m1->second.fRightKeys.push_back(key2); - - m2->second.fLeftKeys.push_back(key2); - m2->second.fRightKeys.push_back(key1); - - // Keep a map of the join type between the keys. - // OUTER join and SEMI/ANTI join are mutually exclusive. - if (joinType == LEFTOUTER) - { - m1->second.fTypes.push_back(SMALLOUTER); - m2->second.fTypes.push_back(LARGEOUTER); - jobInfo.outerOnTable.insert(tid2); - } - else if (joinType == RIGHTOUTER) - { - m1->second.fTypes.push_back(LARGEOUTER); - m2->second.fTypes.push_back(SMALLOUTER); - jobInfo.outerOnTable.insert(tid1); - } - else if ((joinType & SEMI) && - ((joinType & LEFTOUTER) == LEFTOUTER || (joinType & RIGHTOUTER) == RIGHTOUTER)) - { - // @bug3998, DML UPDATE borrows "SEMI" flag, - // allowing SEMI and LARGEOUTER combination to support update with outer join. - if ((joinType & LEFTOUTER) == LEFTOUTER) - { - joinType ^= LEFTOUTER; - m1->second.fTypes.push_back(joinType); - m2->second.fTypes.push_back(joinType | LARGEOUTER); - jobInfo.outerOnTable.insert(tid2); - } - else - { - joinType ^= RIGHTOUTER; - m1->second.fTypes.push_back(joinType | LARGEOUTER); - m2->second.fTypes.push_back(joinType); - jobInfo.outerOnTable.insert(tid1); - } - } - else - { - m1->second.fTypes.push_back(joinType); - m2->second.fTypes.push_back(joinType); - } - - // need id to keep the join order - m1->second.fJoinId = m2->second.fJoinId = thjs->joinId(); - } - else - { - // one of the tables is in outer query - jobInfo.correlateSteps.push_back(spjs); - } + // Enable Typeless JOIN for char and wide decimal types. + if (datatypes::isCharType(ti1.dtype) || (datatypes::isWideDecimalType(ti1.dtype, ti1.width) || + datatypes::isWideDecimalType(ti2.dtype, ti2.width))) + m1->second.fTypeless = m2->second.fTypeless = true; // ti2 is compatible + else + m1->second.fTypeless = m2->second.fTypeless = false; + } + else if (functionJoinPairs.find(p) == functionJoinPairs.end()) + { + continue; // different path + } + else + { + // path already added, do compound join + m1->second.fTypeless = m2->second.fTypeless = true; } - return added; + // simple or compound function join + es->functionJoin(true); + updateTableKey(key1, tid1, jobInfo); + updateTableKey(key2, tid2, jobInfo); + + tableInfoMap[tid1].fJoinKeys.push_back(key1); + tableInfoMap[tid2].fJoinKeys.push_back(key2); + + if (fji->fStep[0]) + tableInfoMap[tid1].fFuncJoinExps.push_back(fji->fStep[0]); + + if (fji->fStep[1]) + tableInfoMap[tid2].fFuncJoinExps.push_back(fji->fStep[1]); + + vector& cols1 = tableInfoMap[tid1].fColsInFuncJoin; + cols1.insert(cols1.end(), fji->fColumnKeys[0].begin(), fji->fColumnKeys[0].end()); + vector& cols2 = tableInfoMap[tid2].fColsInFuncJoin; + cols2.insert(cols2.end(), fji->fColumnKeys[1].begin(), fji->fColumnKeys[1].end()); + + // construct a join step + TupleHashJoinStep* thjs = new TupleHashJoinStep(jobInfo); + thjs->tableOid1(fji->fTableOid[0]); + thjs->tableOid2(fji->fTableOid[1]); + thjs->oid1(fji->fOid[0]); + thjs->oid2(fji->fOid[1]); + thjs->alias1(fji->fAlias[0]); + thjs->alias2(fji->fAlias[1]); + thjs->view1(fji->fView[0]); + thjs->view2(fji->fView[1]); + thjs->schema1(fji->fSchema[0]); + thjs->schema2(fji->fSchema[1]); + thjs->column1(fji->fExpression[0]); + thjs->column2(fji->fExpression[1]); + thjs->sequence1(fji->fSequence[0]); + thjs->sequence2(fji->fSequence[1]); + thjs->joinId(fji->fJoinId); + thjs->setJoinType(fji->fJoinType); + thjs->funcJoinInfo(fji); + thjs->tupleId1(key1); + thjs->tupleId2(key2); + SJSTEP spjs(thjs); + + // check correlated info + JoinType joinType = fji->fJoinType; + + if (!(joinType & CORRELATED)) + { + joinSteps.push_back(spjs); + + // Keep a map of the join (table, key) pairs + m1->second.fLeftKeys.push_back(key1); + m1->second.fRightKeys.push_back(key2); + + m2->second.fLeftKeys.push_back(key2); + m2->second.fRightKeys.push_back(key1); + + // Keep a map of the join type between the keys. + // OUTER join and SEMI/ANTI join are mutually exclusive. + if (joinType == LEFTOUTER) + { + m1->second.fTypes.push_back(SMALLOUTER); + m2->second.fTypes.push_back(LARGEOUTER); + jobInfo.outerOnTable.insert(tid2); + } + else if (joinType == RIGHTOUTER) + { + m1->second.fTypes.push_back(LARGEOUTER); + m2->second.fTypes.push_back(SMALLOUTER); + jobInfo.outerOnTable.insert(tid1); + } + else if ((joinType & SEMI) && + ((joinType & LEFTOUTER) == LEFTOUTER || (joinType & RIGHTOUTER) == RIGHTOUTER)) + { + // @bug3998, DML UPDATE borrows "SEMI" flag, + // allowing SEMI and LARGEOUTER combination to support update with outer join. + if ((joinType & LEFTOUTER) == LEFTOUTER) + { + joinType ^= LEFTOUTER; + m1->second.fTypes.push_back(joinType); + m2->second.fTypes.push_back(joinType | LARGEOUTER); + jobInfo.outerOnTable.insert(tid2); + } + else + { + joinType ^= RIGHTOUTER; + m1->second.fTypes.push_back(joinType | LARGEOUTER); + m2->second.fTypes.push_back(joinType); + jobInfo.outerOnTable.insert(tid1); + } + } + else + { + m1->second.fTypes.push_back(joinType); + m2->second.fTypes.push_back(joinType); + } + + // need id to keep the join order + m1->second.fJoinId = m2->second.fJoinId = thjs->joinId(); + } + else + { + // one of the tables is in outer query + jobInfo.correlateSteps.push_back(spjs); + } + } + + return added; } void collectCycles(JoinGraph& joinGraph, const JobInfo& jobInfo, TableInfoMap& tableInfoMap, - uint32_t currentTable, uint32_t prevTable, JoinEdges& edgesToTransform, - Cycles& cycles) + uint32_t currentTable, uint32_t prevTable, JoinEdges& edgesToTransform, Cycles& cycles) { - // Mark as visited. - joinGraph[currentTable].fVisited = true; - joinGraph[currentTable].fParent = prevTable; + // Mark as visited. + joinGraph[currentTable].fVisited = true; + joinGraph[currentTable].fParent = prevTable; - // For each adjacent node. - for (const auto adjNode : joinGraph[currentTable].fAdjacentList) + // For each adjacent node. + for (const auto adjNode : joinGraph[currentTable].fAdjacentList) + { + // If visited and not a back edge consider as a cycle. + if (joinGraph[adjNode].fVisited && prevTable != adjNode) { - // If visited and not a back edge consider as a cycle. - if (joinGraph[adjNode].fVisited && prevTable != adjNode) + Cycle cycle; + const auto edgeForward = make_pair(currentTable, adjNode); + const auto edgeBackward = make_pair(adjNode, currentTable); + + if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) + { + edgesToTransform.insert(edgeForward); + cycle.push_back(edgeForward); + } + + auto nodeIt = currentTable; + auto nextNode = joinGraph[nodeIt].fParent; + // Walk back until we find node `adjNode` we identified before. + while (nextNode != UINT_MAX && nextNode != adjNode) + { + const auto edgeForward = make_pair(nextNode, nodeIt); + const auto edgeBackward = make_pair(nodeIt, nextNode); + + if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) { - Cycle cycle; - const auto edgeForward = make_pair(currentTable, adjNode); - const auto edgeBackward = make_pair(adjNode, currentTable); - - if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) - { - edgesToTransform.insert(edgeForward); - cycle.push_back(edgeForward); - } - - auto nodeIt = currentTable; - auto nextNode = joinGraph[nodeIt].fParent; - // Walk back until we find node `adjNode` we identified before. - while (nextNode != UINT_MAX && nextNode != adjNode) - { - const auto edgeForward = make_pair(nextNode, nodeIt); - const auto edgeBackward = make_pair(nodeIt, nextNode); - - if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) - { - edgesToTransform.insert(edgeForward); - cycle.push_back(edgeForward); - } - - nodeIt = nextNode; - nextNode = joinGraph[nodeIt].fParent; - } - - // Add the last edge. - if (nextNode != UINT_MAX) - { - const auto edgeForward = make_pair(nextNode, nodeIt); - const auto edgeBackward = make_pair(nodeIt, nextNode); - - if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) - { - edgesToTransform.insert(edgeForward); - cycle.push_back(edgeForward); - } - } - - if (jobInfo.trace && cycle.size()) - { - std::cout << "Collected cycle (while walking join graph): " << std::endl; - for (const auto& edge : cycle) - { - cout << "Edge: " << edge.first << " -> " << edge.second << endl; - auto it = jobInfo.tableJoinMap.find(edge); - - cout << "Left keys: " << endl; - for (auto key : it->second.fLeftKeys) - { - cout << "Key: " << key - << " column oid: " << jobInfo.keyInfo->tupleKeyVec[key].fId << endl; - } - - cout << "Right keys: " << endl; - for (auto key : it->second.fRightKeys) - { - cout << "Key: " << key - << " column oid: " << jobInfo.keyInfo->tupleKeyVec[key].fId << endl; - } - } - } - - // Collect the cycle. - if (cycle.size()) - cycles.push_back(std::move(cycle)); + edgesToTransform.insert(edgeForward); + cycle.push_back(edgeForward); } - // If not visited - go there. - else if (joinGraph[adjNode].fVisited == false) + + nodeIt = nextNode; + nextNode = joinGraph[nodeIt].fParent; + } + + // Add the last edge. + if (nextNode != UINT_MAX) + { + const auto edgeForward = make_pair(nextNode, nodeIt); + const auto edgeBackward = make_pair(nodeIt, nextNode); + + if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) { - collectCycles(joinGraph, jobInfo, tableInfoMap, adjNode, currentTable, edgesToTransform, - cycles); + edgesToTransform.insert(edgeForward); + cycle.push_back(edgeForward); } + } + + if (jobInfo.trace && cycle.size()) + { + std::cout << "Collected cycle (while walking join graph): " << std::endl; + for (const auto& edge : cycle) + { + cout << "Edge: " << edge.first << " -> " << edge.second << endl; + auto it = jobInfo.tableJoinMap.find(edge); + + cout << "Left keys: " << endl; + for (auto key : it->second.fLeftKeys) + { + cout << "Key: " << key << " column oid: " << jobInfo.keyInfo->tupleKeyVec[key].fId << endl; + } + + cout << "Right keys: " << endl; + for (auto key : it->second.fRightKeys) + { + cout << "Key: " << key << " column oid: " << jobInfo.keyInfo->tupleKeyVec[key].fId << endl; + } + } + } + + // Collect the cycle. + if (cycle.size()) + cycles.push_back(std::move(cycle)); } + // If not visited - go there. + else if (joinGraph[adjNode].fVisited == false) + { + collectCycles(joinGraph, jobInfo, tableInfoMap, adjNode, currentTable, edgesToTransform, cycles); + } + } } void removeFromList(uint32_t tableId, std::vector& adjList) { - auto tableIdIt = std::find(adjList.begin(), adjList.end(), tableId); - if (tableIdIt != adjList.end()) - adjList.erase(tableIdIt); + auto tableIdIt = std::find(adjList.begin(), adjList.end(), tableId); + if (tableIdIt != adjList.end()) + adjList.erase(tableIdIt); } bool isForeignKeyForeignKeyLink(TableInfoMap& infoMap, const JobInfo& jobInfo, const pair& edge, statistics::StatisticsManager* statisticsManager) { - const auto it = jobInfo.tableJoinMap.find(edge); - std::vector leftKeys, rightKeys; - std::vector lOid, rOid; + const auto it = jobInfo.tableJoinMap.find(edge); + std::vector leftKeys, rightKeys; + std::vector lOid, rOid; - for (auto key : it->second.fLeftKeys) - { - auto oid = jobInfo.keyInfo->tupleKeyVec[key].fId; - if (!statisticsManager->hasKey(oid)) - return false; - - auto keyType = statisticsManager->getKeyType(oid); - leftKeys.push_back(keyType); - lOid.push_back(oid); - - if (jobInfo.trace) - std::cout << "OID " << oid << " with key type " << (uint32_t) keyType << std::endl; - } - - for (auto key : it->second.fRightKeys) - { - auto oid = jobInfo.keyInfo->tupleKeyVec[key].fId; - if (!statisticsManager->hasKey(oid)) - return false; - - auto keyType = statisticsManager->getKeyType(oid); - rightKeys.push_back(keyType); - rOid.push_back(oid); - - if (jobInfo.trace) - std::cout << "OID " << oid << " with key type " << (uint32_t) keyType << std::endl; - } - - if (rightKeys.size() == 0 || leftKeys.size() == 0) + for (auto key : it->second.fLeftKeys) + { + auto oid = jobInfo.keyInfo->tupleKeyVec[key].fId; + if (!statisticsManager->hasKey(oid)) return false; - statistics::KeyType leftType = statistics::KeyType::PK; - for (auto keyType : leftKeys) - { - if (keyType == statistics::KeyType::FK) - { - leftType = keyType; - break; - } - } + auto keyType = statisticsManager->getKeyType(oid); + leftKeys.push_back(keyType); + lOid.push_back(oid); - statistics::KeyType rightType = statistics::KeyType::PK; - for (auto keyType : rightKeys) - { - if (keyType == statistics::KeyType::FK) - { - rightType = keyType; - break; - } - } + if (jobInfo.trace) + std::cout << "OID " << oid << " with key type " << (uint32_t)keyType << std::endl; + } - if (rightType == statistics::KeyType::FK && leftType == statistics::KeyType::FK) - { - if (jobInfo.trace) - { - std::cout << "Found FK <-> FK connection " << lOid.front() << " <-> " << rOid.front() - << std::endl; - } - return true; - } + for (auto key : it->second.fRightKeys) + { + auto oid = jobInfo.keyInfo->tupleKeyVec[key].fId; + if (!statisticsManager->hasKey(oid)) + return false; + auto keyType = statisticsManager->getKeyType(oid); + rightKeys.push_back(keyType); + rOid.push_back(oid); + + if (jobInfo.trace) + std::cout << "OID " << oid << " with key type " << (uint32_t)keyType << std::endl; + } + + if (rightKeys.size() == 0 || leftKeys.size() == 0) return false; + + statistics::KeyType leftType = statistics::KeyType::PK; + for (auto keyType : leftKeys) + { + if (keyType == statistics::KeyType::FK) + { + leftType = keyType; + break; + } + } + + statistics::KeyType rightType = statistics::KeyType::PK; + for (auto keyType : rightKeys) + { + if (keyType == statistics::KeyType::FK) + { + rightType = keyType; + break; + } + } + + if (rightType == statistics::KeyType::FK && leftType == statistics::KeyType::FK) + { + if (jobInfo.trace) + { + std::cout << "Found FK <-> FK connection " << lOid.front() << " <-> " << rOid.front() << std::endl; + } + return true; + } + + return false; } void chooseEdgeToTransform(TableInfoMap& infoMap, const JobInfo& jobInfo, Cycle& cycle, JoinEdges& edgesToTransform, std::pair& resultEdge) { - auto* statisticsManager = statistics::StatisticsManager::instance(); + auto* statisticsManager = statistics::StatisticsManager::instance(); - for (auto& edgeForward : cycle) + for (auto& edgeForward : cycle) + { + if (isForeignKeyForeignKeyLink(infoMap, jobInfo, edgeForward, statisticsManager)) { - if (isForeignKeyForeignKeyLink(infoMap, jobInfo, edgeForward, statisticsManager)) - { - const auto edgeBackward = std::make_pair(edgeForward.second, edgeForward.first); - if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) - { - edgesToTransform.insert(edgeForward); - resultEdge = edgeForward; - return; - } - } + const auto edgeBackward = std::make_pair(edgeForward.second, edgeForward.first); + if (!edgesToTransform.count(edgeForward) && !edgesToTransform.count(edgeBackward)) + { + edgesToTransform.insert(edgeForward); + resultEdge = edgeForward; + return; + } } + } - if (jobInfo.trace) - std::cout << "FK FK key not found, removing the first one " << std::endl; + if (jobInfo.trace) + std::cout << "FK FK key not found, removing the first one " << std::endl; - // FIXME: Use size of columns to possible largest cardinality. - // Take just a first. - edgesToTransform.insert(cycle.front()); - resultEdge = cycle.front(); + // FIXME: Use size of columns to possible largest cardinality. + // Take just a first. + edgesToTransform.insert(cycle.front()); + resultEdge = cycle.front(); } void breakCyclesAndCollectEdges(TableInfoMap& infoMap, const JobInfo& jobInfo, Cycles& cycles, JoinEdges& edgesToTransform) { - for (auto& cycle : cycles) + for (auto& cycle : cycles) + { + std::pair edgeForward; + if (cycle.size() == 0) + continue; + + chooseEdgeToTransform(infoMap, jobInfo, cycle, edgesToTransform, edgeForward); + + if (jobInfo.trace) { - std::pair edgeForward; - if (cycle.size() == 0) - continue; - - chooseEdgeToTransform(infoMap, jobInfo, cycle, edgesToTransform, edgeForward); - - if (jobInfo.trace) - { - std::cout << "Remove " << edgeForward.first << " from adjlist of " << edgeForward.second - << std::endl; - } - - // If not present add the edge. - auto tableInfoIt = jobInfo.tableJoinMap.find(edgeForward); - - auto& firstExp2 = infoMap[edgeForward.first].fColsInExp2; - firstExp2.insert(firstExp2.end(), tableInfoIt->second.fLeftKeys.begin(), - tableInfoIt->second.fLeftKeys.end()); - auto& secondExp2 = infoMap[edgeForward.second].fColsInExp2; - secondExp2.insert(secondExp2.end(), tableInfoIt->second.fRightKeys.begin(), - tableInfoIt->second.fRightKeys.end()); - - removeFromList(edgeForward.first, infoMap[edgeForward.second].fAdjacentList); - removeFromList(edgeForward.second, infoMap[edgeForward.first].fAdjacentList); + std::cout << "Remove " << edgeForward.first << " from adjlist of " << edgeForward.second << std::endl; } + + // If not present add the edge. + auto tableInfoIt = jobInfo.tableJoinMap.find(edgeForward); + + auto& firstExp2 = infoMap[edgeForward.first].fColsInExp2; + firstExp2.insert(firstExp2.end(), tableInfoIt->second.fLeftKeys.begin(), + tableInfoIt->second.fLeftKeys.end()); + auto& secondExp2 = infoMap[edgeForward.second].fColsInExp2; + secondExp2.insert(secondExp2.end(), tableInfoIt->second.fRightKeys.begin(), + tableInfoIt->second.fRightKeys.end()); + + removeFromList(edgeForward.first, infoMap[edgeForward.second].fAdjacentList); + removeFromList(edgeForward.second, infoMap[edgeForward.first].fAdjacentList); + } } void initJoinGraph(const TableInfoMap& infoMap, JoinGraph& joinGraph) { - for (const auto& infoPair : infoMap) - { - JoinTableNode joinTableNode; - // Copy adjacent list. - joinTableNode.fAdjacentList = infoPair.second.fAdjacentList; - joinGraph[infoPair.first] = joinTableNode; - } + for (const auto& infoPair : infoMap) + { + JoinTableNode joinTableNode; + // Copy adjacent list. + joinTableNode.fAdjacentList = infoPair.second.fAdjacentList; + joinGraph[infoPair.first] = joinTableNode; + } } void collectEdgesAndBreakCycles(TableInfoMap& infoMap, const JobInfo& jobInfo, JoinEdges& edgesToTransform) { - JoinGraph joinGraph; - initJoinGraph(infoMap, joinGraph); - Cycles cycles; + JoinGraph joinGraph; + initJoinGraph(infoMap, joinGraph); + Cycles cycles; - collectCycles(joinGraph, jobInfo, infoMap, - /*currentTable=*/joinGraph.begin()->first, - /*prevTable=*/UINT_MAX, edgesToTransform, cycles); + collectCycles(joinGraph, jobInfo, infoMap, + /*currentTable=*/joinGraph.begin()->first, + /*prevTable=*/UINT_MAX, edgesToTransform, cycles); - if (jobInfo.trace) + if (jobInfo.trace) + { + std::cout << "Collected cycles (after walking join graph): " << std::endl; + for (const auto& cycle : cycles) { - std::cout << "Collected cycles (after walking join graph): " << std::endl; - for (const auto& cycle : cycles) - { - std::cout << "Collected cycle: " << std::endl; - for (const auto& edge : cycle) - { - std::cout << edge.first << " -> " << edge.second << std::endl; - } - } + std::cout << "Collected cycle: " << std::endl; + for (const auto& edge : cycle) + { + std::cout << edge.first << " -> " << edge.second << std::endl; + } } + } - edgesToTransform.clear(); - // Finally break the cycles by removing collected edges from the graph. - breakCyclesAndCollectEdges(infoMap, jobInfo, cycles, edgesToTransform); + edgesToTransform.clear(); + // Finally break the cycles by removing collected edges from the graph. + breakCyclesAndCollectEdges(infoMap, jobInfo, cycles, edgesToTransform); } -void spanningTreeCheck(TableInfoMap& tableInfoMap, JobStepVector joinSteps, - JobInfo& jobInfo, JoinEdges& edgesToTransform) +void spanningTreeCheck(TableInfoMap& tableInfoMap, JobStepVector joinSteps, JobInfo& jobInfo, + JoinEdges& edgesToTransform) { - bool spanningTree = true; - unsigned errcode = 0; - Message::Args args; + bool spanningTree = true; + unsigned errcode = 0; + Message::Args args; - if (jobInfo.trace) + if (jobInfo.trace) + { + cout << "Table Connection:" << endl; + + for (TableInfoMap::iterator i = tableInfoMap.begin(); i != tableInfoMap.end(); i++) { - cout << "Table Connection:" << endl; + cout << i->first << " :"; + vector::iterator j = i->second.fAdjacentList.begin(); - for (TableInfoMap::iterator i = tableInfoMap.begin(); i != tableInfoMap.end(); i++) + while (j != i->second.fAdjacentList.end()) + cout << " " << *j++; + + cout << endl; + } + + cout << endl; + } + + if (tableInfoMap.size() < 1) + { + spanningTree = false; + cerr << boldStart << "No table information." << boldStop << endl; + throw logic_error("No table information."); + } + else if (tableInfoMap.size() > 1) + { + // 1a. make sure all tables are joined if not a single table query. + set nodeSet; + set> pathSet; + vector joinedTables; + joinedTables.push_back((tableInfoMap.begin())->first); + + for (size_t i = 0; i < joinedTables.size(); i++) + { + vector& v = tableInfoMap[joinedTables[i]].fAdjacentList; + nodeSet.insert(joinedTables[i]); + + for (vector::iterator j = v.begin(); j != v.end(); j++) + { + if (nodeSet.find(*j) == nodeSet.end()) + joinedTables.push_back(*j); + + nodeSet.insert(*j); + pathSet.insert(make_pair(joinedTables[i], *j)); + } + } + + // 1b. convert expressions to function joins if not connected with simple column joins. + bool fjAdded = false; + + while (joinedTables.size() < tableInfoMap.size() && + addFunctionJoin(joinedTables, joinSteps, nodeSet, pathSet, tableInfoMap, jobInfo)) + { + fjAdded = true; + + for (size_t i = joinedTables.size() - 1; i < joinedTables.size(); i++) + { + vector& v = tableInfoMap[joinedTables[i]].fAdjacentList; + + for (vector::iterator j = v.begin(); j != v.end(); j++) { - cout << i->first << " :"; - vector::iterator j = i->second.fAdjacentList.begin(); + if (find(joinedTables.begin(), joinedTables.end(), *j) == joinedTables.end()) + joinedTables.push_back(*j); - while (j != i->second.fAdjacentList.end()) - cout << " " << *j++; - - cout << endl; + nodeSet.insert(*j); + pathSet.insert(make_pair(joinedTables[i], *j)); } + } + } + + if (jobInfo.trace && fjAdded) + { + cout << "Table Connection after adding function join:" << endl; + + for (TableInfoMap::iterator i = tableInfoMap.begin(); i != tableInfoMap.end(); i++) + { + cout << i->first << " :"; + vector::iterator j = i->second.fAdjacentList.begin(); + + while (j != i->second.fAdjacentList.end()) + cout << " " << *j++; cout << endl; + } + + cout << endl; } - if (tableInfoMap.size() < 1) + // Check that join is compatible + set views1; + set tables1; + string errStr; + + vector::iterator k = joinedTables.begin(); + + k = joinedTables.begin(); + + for (; k != joinedTables.end(); k++) { + if (jobInfo.keyInfo->tupleKeyVec[*k].fView.empty()) + tables1.insert(jobInfo.keyInfo->tupleKeyToName[*k]); + else + views1.insert(jobInfo.keyInfo->tupleKeyVec[*k].fView); + + if (jobInfo.incompatibleJoinMap.find(*k) != jobInfo.incompatibleJoinMap.end()) + { + errcode = ERR_INCOMPATIBLE_JOIN; + + uint32_t key2 = jobInfo.incompatibleJoinMap[*k]; + + if (jobInfo.keyInfo->tupleKeyVec[*k].fView.length() > 0) + { + string view2 = jobInfo.keyInfo->tupleKeyVec[key2].fView; + + if (jobInfo.keyInfo->tupleKeyVec[*k].fView == view2) + { + // same view + errStr += "Tables in '" + view2 + "' have"; + } + else if (view2.empty()) + { + // view and real table + errStr += "'" + jobInfo.keyInfo->tupleKeyVec[*k].fView + "' and '" + + jobInfo.keyInfo->tupleKeyToName[key2] + "' have"; + } + else + { + // two views + errStr += "'" + jobInfo.keyInfo->tupleKeyVec[*k].fView + "' and '" + view2 + "' have"; + } + } + else + { + string view2 = jobInfo.keyInfo->tupleKeyVec[key2].fView; + + if (view2.empty()) + { + // two real tables + errStr += "'" + jobInfo.keyInfo->tupleKeyToName[*k] + "' and '" + + jobInfo.keyInfo->tupleKeyToName[key2] + "' have"; + } + else + { + // real table and view + errStr += "'" + jobInfo.keyInfo->tupleKeyToName[*k] + "' and '" + view2 + "' have"; + } + } + + args.add(errStr); spanningTree = false; - cerr << boldStart << "No table information." << boldStop << endl; - throw logic_error("No table information."); + break; + } } - else if (tableInfoMap.size() > 1) + + // 1c. check again if all tables are joined after pulling in function joins. + if (joinedTables.size() < tableInfoMap.size()) { - // 1a. make sure all tables are joined if not a single table query. - set nodeSet; - set > pathSet; - vector joinedTables; - joinedTables.push_back((tableInfoMap.begin())->first); + vector notJoinedTables; - for (size_t i = 0; i < joinedTables.size(); i++) + for (TableInfoMap::iterator i = tableInfoMap.begin(); i != tableInfoMap.end(); i++) + { + if (find(joinedTables.begin(), joinedTables.end(), i->first) == joinedTables.end()) + notJoinedTables.push_back(i->first); + } + + set views2; + set tables2; + k = notJoinedTables.begin(); + + for (; k != notJoinedTables.end(); k++) + { + if (jobInfo.keyInfo->tupleKeyVec[*k].fView.empty()) + tables2.insert(jobInfo.keyInfo->tupleKeyToName[*k]); + else + views2.insert(jobInfo.keyInfo->tupleKeyVec[*k].fView); + } + + if (errStr.empty()) + { + errcode = ERR_MISS_JOIN; + + // 1. check if all tables in a view are joined + for (set::iterator s = views1.begin(); s != views1.end(); s++) { - vector& v = tableInfoMap[joinedTables[i]].fAdjacentList; - nodeSet.insert(joinedTables[i]); - - for (vector::iterator j = v.begin(); j != v.end(); j++) - { - if (nodeSet.find(*j) == nodeSet.end()) - joinedTables.push_back(*j); - - nodeSet.insert(*j); - pathSet.insert(make_pair(joinedTables[i], *j)); - } + if (views2.find(*s) != views2.end()) + { + errStr = "Tables in '" + (*s) + "' are"; + } } - // 1b. convert expressions to function joins if not connected with simple column joins. - bool fjAdded = false; - - while (joinedTables.size() < tableInfoMap.size() && - addFunctionJoin(joinedTables, joinSteps, nodeSet, pathSet, tableInfoMap, jobInfo)) + // 2. tables and views are joined + if (errStr.empty()) { - fjAdded = true; + string set1; - for (size_t i = joinedTables.size() - 1; i < joinedTables.size(); i++) - { - vector& v = tableInfoMap[joinedTables[i]].fAdjacentList; - - for (vector::iterator j = v.begin(); j != v.end(); j++) - { - if (find(joinedTables.begin(), joinedTables.end(), *j) == joinedTables.end()) - joinedTables.push_back(*j); - - nodeSet.insert(*j); - pathSet.insert(make_pair(joinedTables[i], *j)); - } - } - } - - - if (jobInfo.trace && fjAdded) - { - cout << "Table Connection after adding function join:" << endl; - - for (TableInfoMap::iterator i = tableInfoMap.begin(); i != tableInfoMap.end(); i++) - { - cout << i->first << " :"; - vector::iterator j = i->second.fAdjacentList.begin(); - - while (j != i->second.fAdjacentList.end()) - cout << " " << *j++; - - cout << endl; - } - - cout << endl; - } - - // Check that join is compatible - set views1; - set tables1; - string errStr; - - vector::iterator k = joinedTables.begin(); - - k = joinedTables.begin(); - - for (; k != joinedTables.end(); k++) - { - if (jobInfo.keyInfo->tupleKeyVec[*k].fView.empty()) - tables1.insert(jobInfo.keyInfo->tupleKeyToName[*k]); + for (set::iterator s = views1.begin(); s != views1.end(); s++) + { + if (set1.empty()) + set1 = "'"; else - views1.insert(jobInfo.keyInfo->tupleKeyVec[*k].fView); + set1 += ", "; - if (jobInfo.incompatibleJoinMap.find(*k) != jobInfo.incompatibleJoinMap.end()) - { - errcode = ERR_INCOMPATIBLE_JOIN; + set1 += (*s); + } - uint32_t key2 = jobInfo.incompatibleJoinMap[*k]; - - if (jobInfo.keyInfo->tupleKeyVec[*k].fView.length() > 0) - { - string view2 = jobInfo.keyInfo->tupleKeyVec[key2].fView; - - if (jobInfo.keyInfo->tupleKeyVec[*k].fView == view2) - { - // same view - errStr += "Tables in '" + view2 + "' have"; - } - else if (view2.empty()) - { - // view and real table - errStr += "'" + jobInfo.keyInfo->tupleKeyVec[*k].fView + "' and '" + - jobInfo.keyInfo->tupleKeyToName[key2] + "' have"; - } - else - { - // two views - errStr += "'" + jobInfo.keyInfo->tupleKeyVec[*k].fView + "' and '" + - view2 + "' have"; - } - } - else - { - string view2 = jobInfo.keyInfo->tupleKeyVec[key2].fView; - - if (view2.empty()) - { - // two real tables - errStr += "'" + jobInfo.keyInfo->tupleKeyToName[*k] + "' and '" + - jobInfo.keyInfo->tupleKeyToName[key2] + "' have"; - } - else - { - // real table and view - errStr += "'" + jobInfo.keyInfo->tupleKeyToName[*k] + "' and '" + - view2 + "' have"; - } - } - - args.add(errStr); - spanningTree = false; - break; - } - - } - - // 1c. check again if all tables are joined after pulling in function joins. - if (joinedTables.size() < tableInfoMap.size()) - { - vector notJoinedTables; - - for (TableInfoMap::iterator i = tableInfoMap.begin(); i != tableInfoMap.end(); i++) - { - if (find(joinedTables.begin(), joinedTables.end(), i->first) == joinedTables.end()) - notJoinedTables.push_back(i->first); - } - - set views2; - set tables2; - k = notJoinedTables.begin(); - - for (; k != notJoinedTables.end(); k++) - { - if (jobInfo.keyInfo->tupleKeyVec[*k].fView.empty()) - tables2.insert(jobInfo.keyInfo->tupleKeyToName[*k]); - else - views2.insert(jobInfo.keyInfo->tupleKeyVec[*k].fView); - } - - if (errStr.empty()) - { - errcode = ERR_MISS_JOIN; - - // 1. check if all tables in a view are joined - for (set::iterator s = views1.begin(); s != views1.end(); s++) - { - if (views2.find(*s) != views2.end()) - { - errStr = "Tables in '" + (*s) + "' are"; - } - } - - // 2. tables and views are joined - if (errStr.empty()) - { - string set1; - - for (set::iterator s = views1.begin(); s != views1.end(); s++) - { - if (set1.empty()) - set1 = "'"; - else - set1 += ", "; - - set1 += (*s); - } - - for (set::iterator s = tables1.begin(); s != tables1.end(); s++) - { - if (set1.empty()) - set1 = "'"; - else - set1 += ", "; - - set1 += (*s); - } - - string set2; - - for (set::iterator s = views2.begin(); s != views2.end(); s++) - { - if (set2.empty()) - set2 = "'"; - else - set2 += ", "; - - set2 += (*s); - } - - for (set::iterator s = tables2.begin(); s != tables2.end(); s++) - { - if (set2.empty()) - set2 = "'"; - else - set2 += ", "; - - set2 += (*s); - } - - errStr = set1 + "' and " + set2 + "' are"; - args.add(errStr); - spanningTree = false; - } - } - - } - - // 2. Cycles. - if (spanningTree && (nodeSet.size() - pathSet.size() / 2 != 1)) - { - // 2.1. Inner. - if (jobInfo.outerOnTable.size() == 0) - { - collectEdgesAndBreakCycles(tableInfoMap, jobInfo, edgesToTransform); - } - // 2.2. Outer. + for (set::iterator s = tables1.begin(); s != tables1.end(); s++) + { + if (set1.empty()) + set1 = "'"; else - { - errcode = ERR_CIRCULAR_JOIN; - spanningTree = false; - } + set1 += ", "; + + set1 += (*s); + } + + string set2; + + for (set::iterator s = views2.begin(); s != views2.end(); s++) + { + if (set2.empty()) + set2 = "'"; + else + set2 += ", "; + + set2 += (*s); + } + + for (set::iterator s = tables2.begin(); s != tables2.end(); s++) + { + if (set2.empty()) + set2 = "'"; + else + set2 += ", "; + + set2 += (*s); + } + + errStr = set1 + "' and " + set2 + "' are"; + args.add(errStr); + spanningTree = false; } + } } - if (!spanningTree) + // 2. Cycles. + if (spanningTree && (nodeSet.size() - pathSet.size() / 2 != 1)) { - cerr << boldStart << IDBErrorInfo::instance()->errorMsg(errcode, args) << boldStop << endl; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + // 2.1. Inner. + if (jobInfo.outerOnTable.size() == 0) + { + collectEdgesAndBreakCycles(tableInfoMap, jobInfo, edgesToTransform); + } + // 2.2. Outer. + else + { + errcode = ERR_CIRCULAR_JOIN; + spanningTree = false; + } } + } + + if (!spanningTree) + { + cerr << boldStart << IDBErrorInfo::instance()->errorMsg(errcode, args) << boldStop << endl; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } } void outjoinPredicateAdjust(TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - set::iterator i = jobInfo.outerOnTable.begin(); + set::iterator i = jobInfo.outerOnTable.begin(); - for (; i != jobInfo.outerOnTable.end(); i++) + for (; i != jobInfo.outerOnTable.end(); i++) + { + // resetTableFilters(tableInfoMap[*i], jobInfo) + TableInfo& tblInfo = tableInfoMap[*i]; + + if (tblInfo.fTableOid != CNX_VTABLE_ID) { - // resetTableFilters(tableInfoMap[*i], jobInfo) - TableInfo& tblInfo = tableInfoMap[*i]; + JobStepVector::iterator k = tblInfo.fQuerySteps.begin(); + JobStepVector onClauseFilterSteps; //@bug5887, 5311 - if (tblInfo.fTableOid != CNX_VTABLE_ID) + for (; k != tblInfo.fQuerySteps.end(); k++) + { + if ((*k)->onClauseFilter()) { - JobStepVector::iterator k = tblInfo.fQuerySteps.begin(); - JobStepVector onClauseFilterSteps; //@bug5887, 5311 - - for (; k != tblInfo.fQuerySteps.end(); k++) - { - if ((*k)->onClauseFilter()) - { - onClauseFilterSteps.push_back(*k); - continue; - } - - uint32_t colKey = -1; - pColStep* pcs = dynamic_cast(k->get()); - pColScanStep* pcss = dynamic_cast(k->get()); - pDictionaryScan* pdss = dynamic_cast(k->get()); - pDictionaryStep* pdsp = dynamic_cast(k->get()); - vector* filters = NULL; - int8_t bop = -1; - - if (pcs != NULL) - { - filters = &(pcs->getFilters()); - bop = pcs->BOP(); - colKey = pcs->tupleId(); - } - else if (pcss != NULL) - { - filters = &(pcss->getFilters()); - bop = pcss->BOP(); - colKey = pcss->tupleId(); - } - else if (pdss != NULL) - { - filters = &(pdss->getFilters()); - bop = pdss->BOP(); - colKey = pdss->tupleId(); - } - else if (pdsp != NULL) - { - filters = &(pdsp->getFilters()); - bop = pdsp->BOP(); - colKey = pdsp->tupleId(); - } - - if (filters != NULL && filters->size() > 0) - { - ParseTree* pt = new ParseTree((*filters)[0]->clone()); - - for (size_t i = 1; i < filters->size(); i++) - { - ParseTree* left = pt; - ParseTree* right = - new ParseTree((*filters)[i]->clone()); - ParseTree* op = (BOP_OR == bop) ? - new ParseTree(new LogicOperator("or")) : - new ParseTree(new LogicOperator("and")); - op->left(left); - op->right(right); - - pt = op; - } - - ExpressionStep* es = new ExpressionStep(jobInfo); - - if (es == NULL) - throw runtime_error ("Failed to new ExpressionStep 2"); - - es->expressionFilter(pt, jobInfo); - SJSTEP sjstep(es); - jobInfo.outerJoinExpressions.push_back(sjstep); - tblInfo.fColsInOuter.push_back(colKey); - - delete pt; - } - } - - // Do not apply the primitive filters if there is an "IS NULL" in where clause. - if (jobInfo.tableHasIsNull.find(*i) != jobInfo.tableHasIsNull.end()) - tblInfo.fQuerySteps = onClauseFilterSteps; + onClauseFilterSteps.push_back(*k); + continue; } - jobInfo.outerJoinExpressions.insert(jobInfo.outerJoinExpressions.end(), - tblInfo.fOneTableExpSteps.begin(), tblInfo.fOneTableExpSteps.end()); - tblInfo.fOneTableExpSteps.clear(); + uint32_t colKey = -1; + pColStep* pcs = dynamic_cast(k->get()); + pColScanStep* pcss = dynamic_cast(k->get()); + pDictionaryScan* pdss = dynamic_cast(k->get()); + pDictionaryStep* pdsp = dynamic_cast(k->get()); + vector* filters = NULL; + int8_t bop = -1; - tblInfo.fColsInOuter.insert(tblInfo.fColsInOuter.end(), - tblInfo.fColsInExp1.begin(), tblInfo.fColsInExp1.end()); + if (pcs != NULL) + { + filters = &(pcs->getFilters()); + bop = pcs->BOP(); + colKey = pcs->tupleId(); + } + else if (pcss != NULL) + { + filters = &(pcss->getFilters()); + bop = pcss->BOP(); + colKey = pcss->tupleId(); + } + else if (pdss != NULL) + { + filters = &(pdss->getFilters()); + bop = pdss->BOP(); + colKey = pdss->tupleId(); + } + else if (pdsp != NULL) + { + filters = &(pdsp->getFilters()); + bop = pdsp->BOP(); + colKey = pdsp->tupleId(); + } + + if (filters != NULL && filters->size() > 0) + { + ParseTree* pt = new ParseTree((*filters)[0]->clone()); + + for (size_t i = 1; i < filters->size(); i++) + { + ParseTree* left = pt; + ParseTree* right = new ParseTree((*filters)[i]->clone()); + ParseTree* op = (BOP_OR == bop) ? new ParseTree(new LogicOperator("or")) + : new ParseTree(new LogicOperator("and")); + op->left(left); + op->right(right); + + pt = op; + } + + ExpressionStep* es = new ExpressionStep(jobInfo); + + if (es == NULL) + throw runtime_error("Failed to new ExpressionStep 2"); + + es->expressionFilter(pt, jobInfo); + SJSTEP sjstep(es); + jobInfo.outerJoinExpressions.push_back(sjstep); + tblInfo.fColsInOuter.push_back(colKey); + + delete pt; + } + } + + // Do not apply the primitive filters if there is an "IS NULL" in where clause. + if (jobInfo.tableHasIsNull.find(*i) != jobInfo.tableHasIsNull.end()) + tblInfo.fQuerySteps = onClauseFilterSteps; } -} + jobInfo.outerJoinExpressions.insert(jobInfo.outerJoinExpressions.end(), tblInfo.fOneTableExpSteps.begin(), + tblInfo.fOneTableExpSteps.end()); + tblInfo.fOneTableExpSteps.clear(); + + tblInfo.fColsInOuter.insert(tblInfo.fColsInOuter.end(), tblInfo.fColsInExp1.begin(), + tblInfo.fColsInExp1.end()); + } +} uint32_t getLargestTable(JobInfo& jobInfo, TableInfoMap& tableInfoMap, bool overrideLargeSideEstimate) { - // Subquery in FROM clause assumptions: - // hint will be ignored, if the 1st table in FROM clause is a derived table. - if (jobInfo.keyInfo->tupleKeyVec[jobInfo.tableList[0]].fId < 3000) - overrideLargeSideEstimate = false; + // Subquery in FROM clause assumptions: + // hint will be ignored, if the 1st table in FROM clause is a derived table. + if (jobInfo.keyInfo->tupleKeyVec[jobInfo.tableList[0]].fId < 3000) + overrideLargeSideEstimate = false; - // Bug 2123. Added logic to dynamically determine the large side table unless the SQL statement - // contained a hint saying to skip the estimation and use the FIRST table in the from clause. - // Prior to this, we were using the LAST table in the from clause. We switched it as there - // were some outer join sqls that couldn't be written with the large table last. - // Default to the first table in the from clause if: - // the user set the hint; or - // there is only one table in the query. - uint32_t ret = jobInfo.tableList.front(); - - if (jobInfo.tableList.size() <= 1) - { - return ret; - } - - // Algorithm to dynamically determine the largest table. - uint64_t largestCardinality = 0; - uint64_t estimatedRowCount = 0; - - // Loop through the tables and find the one with the largest estimated cardinality. - for (uint32_t i = 0; i < jobInfo.tableList.size(); i++) - { - jobInfo.tableSize[jobInfo.tableList[i]] = 0; - TableInfoMap::iterator it = tableInfoMap.find(jobInfo.tableList[i]); - - if (it != tableInfoMap.end()) - { - // @Bug 3771. Loop through the query steps until the tupleBPS is found instead of - // just looking at the first one. Tables in the query that included a filter on a - // dictionary column were not getting their row count estimated. - for (JobStepVector::iterator jsIt = it->second.fQuerySteps.begin(); - jsIt != it->second.fQuerySteps.end(); jsIt++) - { - TupleBPS* tupleBPS = dynamic_cast((*jsIt).get()); - - if (tupleBPS != NULL) - { - estimatedRowCount = tupleBPS->getEstimatedRowCount(); - jobInfo.tableSize[jobInfo.tableList[i]] = estimatedRowCount; - - if (estimatedRowCount > largestCardinality) - { - ret = jobInfo.tableList[i]; - largestCardinality = estimatedRowCount; - } - - break; - } - } - } - } - - // select /*! INFINIDB_ORDERED */ - if (overrideLargeSideEstimate) - { - ret = jobInfo.tableList.front(); - jobInfo.tableSize[ret] = numeric_limits::max(); - } + // Bug 2123. Added logic to dynamically determine the large side table unless the SQL statement + // contained a hint saying to skip the estimation and use the FIRST table in the from clause. + // Prior to this, we were using the LAST table in the from clause. We switched it as there + // were some outer join sqls that couldn't be written with the large table last. + // Default to the first table in the from clause if: + // the user set the hint; or + // there is only one table in the query. + uint32_t ret = jobInfo.tableList.front(); + if (jobInfo.tableList.size() <= 1) + { return ret; -} + } + // Algorithm to dynamically determine the largest table. + uint64_t largestCardinality = 0; + uint64_t estimatedRowCount = 0; + + // Loop through the tables and find the one with the largest estimated cardinality. + for (uint32_t i = 0; i < jobInfo.tableList.size(); i++) + { + jobInfo.tableSize[jobInfo.tableList[i]] = 0; + TableInfoMap::iterator it = tableInfoMap.find(jobInfo.tableList[i]); + + if (it != tableInfoMap.end()) + { + // @Bug 3771. Loop through the query steps until the tupleBPS is found instead of + // just looking at the first one. Tables in the query that included a filter on a + // dictionary column were not getting their row count estimated. + for (JobStepVector::iterator jsIt = it->second.fQuerySteps.begin(); + jsIt != it->second.fQuerySteps.end(); jsIt++) + { + TupleBPS* tupleBPS = dynamic_cast((*jsIt).get()); + + if (tupleBPS != NULL) + { + estimatedRowCount = tupleBPS->getEstimatedRowCount(); + jobInfo.tableSize[jobInfo.tableList[i]] = estimatedRowCount; + + if (estimatedRowCount > largestCardinality) + { + ret = jobInfo.tableList[i]; + largestCardinality = estimatedRowCount; + } + + break; + } + } + } + } + + // select /*! INFINIDB_ORDERED */ + if (overrideLargeSideEstimate) + { + ret = jobInfo.tableList.front(); + jobInfo.tableSize[ret] = numeric_limits::max(); + } + + return ret; +} uint32_t getPrevLarge(uint32_t n, TableInfoMap& tableInfoMap) { - // root node : no previous node; - // other node: only one immediate previous node; - int prev = -1; - vector& adjList = tableInfoMap[n].fAdjacentList; + // root node : no previous node; + // other node: only one immediate previous node; + int prev = -1; + vector& adjList = tableInfoMap[n].fAdjacentList; - for (vector::iterator i = adjList.begin(); i != adjList.end() && prev < 0; i++) - { - if (tableInfoMap[*i].fVisited == true) - prev = *i; - } + for (vector::iterator i = adjList.begin(); i != adjList.end() && prev < 0; i++) + { + if (tableInfoMap[*i].fVisited == true) + prev = *i; + } - return prev; + return prev; } - uint32_t getKeyIndex(uint32_t key, const RowGroup& rg) { - vector::const_iterator i = rg.getKeys().begin(); + vector::const_iterator i = rg.getKeys().begin(); - for (; i != rg.getKeys().end(); ++i) - if (key == *i) - break; + for (; i != rg.getKeys().end(); ++i) + if (key == *i) + break; - if (i == rg.getKeys().end()) - throw runtime_error("No key found."); + if (i == rg.getKeys().end()) + throw runtime_error("No key found."); - return std::distance(rg.getKeys().begin(), i); + return std::distance(rg.getKeys().begin(), i); } - bool joinInfoCompare(const SP_JoinInfo& a, const SP_JoinInfo& b) { - return (a->fJoinData.fJoinId < b->fJoinData.fJoinId); + return (a->fJoinData.fJoinId < b->fJoinData.fJoinId); } - string joinTypeToString(const JoinType& joinType) { - string ret; + string ret; - if (joinType & INNER) - ret = "inner"; - else if (joinType & LARGEOUTER) - ret = "largeOuter"; - else if (joinType & SMALLOUTER) - ret = "smallOuter"; + if (joinType & INNER) + ret = "inner"; + else if (joinType & LARGEOUTER) + ret = "largeOuter"; + else if (joinType & SMALLOUTER) + ret = "smallOuter"; - if (joinType & SEMI) - ret += "+semi"; + if (joinType & SEMI) + ret += "+semi"; - if (joinType & ANTI) - ret += "+ant"; + if (joinType & ANTI) + ret += "+ant"; - if (joinType & SCALAR) - ret += "+scalar"; + if (joinType & SCALAR) + ret += "+scalar"; - if (joinType & MATCHNULLS) - ret += "+matchnulls"; + if (joinType & MATCHNULLS) + ret += "+matchnulls"; - if (joinType & WITHFCNEXP) - ret += "+exp"; + if (joinType & WITHFCNEXP) + ret += "+exp"; - if (joinType & CORRELATED) - ret += "+correlated"; + if (joinType & CORRELATED) + ret += "+correlated"; - return ret; + return ret; } void matchEdgesInRowGroup(const JobInfo& jobInfo, const RowGroup& rg, JoinEdges& edgesToTransform, PostJoinFilterKeys& postJoinFilterKeys) { + if (jobInfo.trace) + { + cout << "\nTrying to match the RowGroup to apply a post join " + "filter\n"; + } + + std::vector> takenEdges; + for (const auto& edge : edgesToTransform) + { + auto it = jobInfo.tableJoinMap.find(edge); + std::vector currentKeys; + // Combine keys. + currentKeys = it->second.fLeftKeys; + currentKeys.insert(currentKeys.end(), it->second.fRightKeys.begin(), it->second.fRightKeys.end()); + + // Rowgroup keys. + const auto& rgKeys = rg.getKeys(); + uint32_t keyIndex = 0; + uint32_t keySize = currentKeys.size(); + + // Search for keys in result rowgroup. + while (keyIndex < keySize) + { + auto keyIt = std::find(rgKeys.begin(), rgKeys.end(), currentKeys[keyIndex]); + // We have to match all keys. + if (keyIt == rgKeys.end()) + break; + + ++keyIndex; + } + if (jobInfo.trace) { - cout << "\nTrying to match the RowGroup to apply a post join " - "filter\n"; + if (keyIndex == keySize) + cout << "\nRowGroup matched\n"; + else + cout << "\nRowGroup not matched\n"; + + cout << rg.toString() << endl; + cout << "For the following keys:\n"; + for (auto key : currentKeys) + cout << key << " "; + cout << endl; } - std::vector> takenEdges; - for (const auto& edge : edgesToTransform) + // All keys matched in current Rowgroup. + if (keyIndex == keySize) { - auto it = jobInfo.tableJoinMap.find(edge); - std::vector currentKeys; - // Combine keys. - currentKeys = it->second.fLeftKeys; - currentKeys.insert(currentKeys.end(), it->second.fRightKeys.begin(), - it->second.fRightKeys.end()); - - // Rowgroup keys. - const auto& rgKeys = rg.getKeys(); - uint32_t keyIndex = 0; - uint32_t keySize = currentKeys.size(); - - // Search for keys in result rowgroup. - while (keyIndex < keySize) - { - auto keyIt = std::find(rgKeys.begin(), rgKeys.end(), currentKeys[keyIndex]); - // We have to match all keys. - if (keyIt == rgKeys.end()) - break; - - ++keyIndex; - } - - if (jobInfo.trace) - { - if (keyIndex == keySize) - cout << "\nRowGroup matched\n"; - else - cout << "\nRowGroup not matched\n"; - - cout << rg.toString() << endl; - cout << "For the following keys:\n"; - for (auto key : currentKeys) - cout << key << " "; - cout << endl; - } - - // All keys matched in current Rowgroup. - if (keyIndex == keySize) - { - // Add macthed keys. - postJoinFilterKeys.push_back(make_pair(edge, currentKeys)); - takenEdges.push_back(edge); - } + // Add macthed keys. + postJoinFilterKeys.push_back(make_pair(edge, currentKeys)); + takenEdges.push_back(edge); } + } - // Erase taken edges. - for (const auto& edge : takenEdges) - { - auto it = edgesToTransform.find(edge); - edgesToTransform.erase(it); - } + // Erase taken edges. + for (const auto& edge : takenEdges) + { + auto it = edgesToTransform.find(edge); + edgesToTransform.erase(it); + } } void createPostJoinFilters(const JobInfo& jobInfo, TableInfoMap& tableInfoMap, @@ -2478,1375 +2433,1354 @@ void createPostJoinFilters(const JobInfo& jobInfo, TableInfoMap& tableInfoMap, const std::map& keyToIndexMap, std::vector& postJoinFilters) { - for (const auto& p : postJoinFilterKeys) + for (const auto& p : postJoinFilterKeys) + { + const auto& edge = p.first; + const auto& keys = p.second; + + if (jobInfo.trace) + cout << "\nRestore a cycle as a post join filter\n"; + + uint32_t leftKeyIndex = 0; + uint32_t rightKeyIndex = keys.size() / 2; + // Left end is where right starts. + const uint32_t leftSize = rightKeyIndex; + + while (leftKeyIndex < leftSize) { - const auto& edge = p.first; - const auto& keys = p.second; + // Keys. + auto leftKey = keys[leftKeyIndex]; + auto rightKey = keys[rightKeyIndex]; - if (jobInfo.trace) - cout << "\nRestore a cycle as a post join filter\n"; + // Column oids. + auto leftOid = jobInfo.keyInfo->tupleKeyVec[leftKey].fId; + auto rightOid = jobInfo.keyInfo->tupleKeyVec[rightKey].fId; - uint32_t leftKeyIndex = 0; - uint32_t rightKeyIndex = keys.size() / 2; - // Left end is where right starts. - const uint32_t leftSize = rightKeyIndex; + // Column types. + auto leftType = jobInfo.keyInfo->colType[keys[leftKeyIndex]]; + auto rightType = jobInfo.keyInfo->colType[keys[rightKeyIndex]]; - while (leftKeyIndex < leftSize) - { - // Keys. - auto leftKey = keys[leftKeyIndex]; - auto rightKey = keys[rightKeyIndex]; + CalpontSystemCatalog::TableColName leftTableColName; + CalpontSystemCatalog::TableColName rightTableColName; - // Column oids. - auto leftOid = jobInfo.keyInfo->tupleKeyVec[leftKey].fId; - auto rightOid = jobInfo.keyInfo->tupleKeyVec[rightKey].fId; + // Check for the dict. + if (joblist::isDictCol(leftType) && joblist::isDictCol(rightType)) + { + leftTableColName = jobInfo.csc->dictColName(leftOid); + rightTableColName = jobInfo.csc->dictColName(rightOid); + } + else + { + leftTableColName = jobInfo.csc->colName(leftOid); + rightTableColName = jobInfo.csc->colName(rightOid); + } - // Column types. - auto leftType = jobInfo.keyInfo->colType[keys[leftKeyIndex]]; - auto rightType = jobInfo.keyInfo->colType[keys[rightKeyIndex]]; + // Create columns. + auto* leftColumn = + new SimpleColumn(leftTableColName.schema, leftTableColName.table, leftTableColName.column); - CalpontSystemCatalog::TableColName leftTableColName; - CalpontSystemCatalog::TableColName rightTableColName; + auto* rightColumn = + new SimpleColumn(rightTableColName.schema, rightTableColName.table, rightTableColName.column); - // Check for the dict. - if (joblist::isDictCol(leftType) && joblist::isDictCol(rightType)) - { - leftTableColName = jobInfo.csc->dictColName(leftOid); - rightTableColName = jobInfo.csc->dictColName(rightOid); - } - else - { - leftTableColName = jobInfo.csc->colName(leftOid); - rightTableColName = jobInfo.csc->colName(rightOid); - } + // Set column indices in the result Rowgroup. + auto leftIndexIt = keyToIndexMap.find(leftKey); + if (leftIndexIt != keyToIndexMap.end()) + { + leftColumn->inputIndex(leftIndexIt->second); + } + else + { + std::cerr << "Cannot find key: " << leftKey << " in the IndexMap " << std::endl; + throw logic_error("Post join filter: Cannot find key in the index map"); + } - // Create columns. - auto* leftColumn = new SimpleColumn(leftTableColName.schema, leftTableColName.table, - leftTableColName.column); + auto rightIndexIt = keyToIndexMap.find(rightKey); + if (rightIndexIt != keyToIndexMap.end()) + { + rightColumn->inputIndex(rightIndexIt->second); + } + else + { + std::cerr << "Cannot find key: " << rightKey << " in the IndexMap " << std::endl; + throw logic_error("Post join filter: Cannot find key in the index map"); + } - auto* rightColumn = new SimpleColumn(rightTableColName.schema, rightTableColName.table, - rightTableColName.column); + // Create an eq operator. + SOP eqPredicateOperator(new PredicateOperator("=")); - // Set column indices in the result Rowgroup. - auto leftIndexIt = keyToIndexMap.find(leftKey); - if (leftIndexIt != keyToIndexMap.end()) - { - leftColumn->inputIndex(leftIndexIt->second); - } - else - { - std::cerr << "Cannot find key: " << leftKey << " in the IndexMap " << std::endl; - throw logic_error("Post join filter: Cannot find key in the index map"); - } + // Set a type. + eqPredicateOperator->setOpType(leftColumn->resultType(), rightColumn->resultType()); - auto rightIndexIt = keyToIndexMap.find(rightKey); - if (rightIndexIt != keyToIndexMap.end()) - { - rightColumn->inputIndex(rightIndexIt->second); - } - else - { - std::cerr << "Cannot find key: " << rightKey << " in the IndexMap " << std::endl; - throw logic_error("Post join filter: Cannot find key in the index map"); - } + // Create a post join filter. + SimpleFilter* joinFilter = new SimpleFilter(eqPredicateOperator, leftColumn, rightColumn); - // Create an eq operator. - SOP eqPredicateOperator(new PredicateOperator("=")); + postJoinFilters.push_back(joinFilter); - // Set a type. - eqPredicateOperator->setOpType(leftColumn->resultType(), rightColumn->resultType()); + // Erase keys from fColsInExp2. + auto& firstExp2 = tableInfoMap[edge.first].fColsInExp2; + auto keyItInExp2 = std::find(firstExp2.begin(), firstExp2.end(), leftKey); + if (keyItInExp2 != firstExp2.end()) + firstExp2.erase(keyItInExp2); - // Create a post join filter. - SimpleFilter* joinFilter = - new SimpleFilter(eqPredicateOperator, leftColumn, rightColumn); + auto& secondExp2 = tableInfoMap[edge.second].fColsInExp2; + keyItInExp2 = std::find(secondExp2.begin(), secondExp2.end(), rightKey); + if (keyItInExp2 != secondExp2.end()) + secondExp2.erase(keyItInExp2); - postJoinFilters.push_back(joinFilter); - - // Erase keys from fColsInExp2. - auto& firstExp2 = tableInfoMap[edge.first].fColsInExp2; - auto keyItInExp2 = std::find(firstExp2.begin(), firstExp2.end(), leftKey); - if (keyItInExp2 != firstExp2.end()) - firstExp2.erase(keyItInExp2); - - auto& secondExp2 = tableInfoMap[edge.second].fColsInExp2; - keyItInExp2 = std::find(secondExp2.begin(), secondExp2.end(), rightKey); - if (keyItInExp2 != secondExp2.end()) - secondExp2.erase(keyItInExp2); - - ++leftKeyIndex; - ++rightKeyIndex; - } + ++leftKeyIndex; + ++rightKeyIndex; } + } } SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo& jobInfo, vector& joinOrder, JoinEdges& edgesToTransform) { - vector smallSides; - tableInfoMap[large].fVisited = true; - tableInfoMap[large].fJoinedTables.insert(large); - set& tableSet = tableInfoMap[large].fJoinedTables; - vector& adjList = tableInfoMap[large].fAdjacentList; - uint32_t prevLarge = (uint32_t) getPrevLarge(large, tableInfoMap); - bool root = (prevLarge == (uint32_t) - 1) ? true : false; - uint32_t link = large; - uint32_t cId = -1; + vector smallSides; + tableInfoMap[large].fVisited = true; + tableInfoMap[large].fJoinedTables.insert(large); + set& tableSet = tableInfoMap[large].fJoinedTables; + vector& adjList = tableInfoMap[large].fAdjacentList; + uint32_t prevLarge = (uint32_t)getPrevLarge(large, tableInfoMap); + bool root = (prevLarge == (uint32_t)-1) ? true : false; + uint32_t link = large; + uint32_t cId = -1; - // Get small sides ready. - for (vector::iterator i = adjList.begin(); i != adjList.end(); i++) + // Get small sides ready. + for (vector::iterator i = adjList.begin(); i != adjList.end(); i++) + { + if (tableInfoMap[*i].fVisited == false) { - if (tableInfoMap[*i].fVisited == false) - { - cId = *i; - smallSides.push_back( - joinToLargeTable(*i, tableInfoMap, jobInfo, joinOrder, edgesToTransform)); + cId = *i; + smallSides.push_back(joinToLargeTable(*i, tableInfoMap, jobInfo, joinOrder, edgesToTransform)); - tableSet.insert(tableInfoMap[*i].fJoinedTables.begin(), - tableInfoMap[*i].fJoinedTables.end()); - } + tableSet.insert(tableInfoMap[*i].fJoinedTables.begin(), tableInfoMap[*i].fJoinedTables.end()); } + } - // Join with its small sides, if not a leaf node. - if (smallSides.size() > 0) + // Join with its small sides, if not a leaf node. + if (smallSides.size() > 0) + { + // non-leaf node, need a join + SJSTEP spjs = tableInfoMap[large].fQuerySteps.back(); + BatchPrimitive* bps = dynamic_cast(spjs.get()); + SubAdapterStep* tsas = dynamic_cast(spjs.get()); + TupleHashJoinStep* thjs = dynamic_cast(spjs.get()); + + // @bug6158, try to put BPS on large side if possible + if (tsas && smallSides.size() == 1) { - // non-leaf node, need a join - SJSTEP spjs = tableInfoMap[large].fQuerySteps.back(); - BatchPrimitive* bps = dynamic_cast(spjs.get()); - SubAdapterStep* tsas = dynamic_cast(spjs.get()); - TupleHashJoinStep* thjs = dynamic_cast(spjs.get()); - - // @bug6158, try to put BPS on large side if possible - if (tsas && smallSides.size() == 1) - { - SJSTEP sspjs = tableInfoMap[cId].fQuerySteps.back(); - BatchPrimitive* sbps = dynamic_cast(sspjs.get()); - TupleHashJoinStep* sthjs = dynamic_cast(sspjs.get()); - - if (sbps || (sthjs && sthjs->tokenJoin() == cId)) - { - SP_JoinInfo largeJoinInfo(new JoinInfo); - largeJoinInfo->fTableOid = tableInfoMap[large].fTableOid; - largeJoinInfo->fAlias = tableInfoMap[large].fAlias; - largeJoinInfo->fView = tableInfoMap[large].fView; - largeJoinInfo->fSchema = tableInfoMap[large].fSchema; - - largeJoinInfo->fDl = tableInfoMap[large].fDl; - largeJoinInfo->fRowGroup = tableInfoMap[large].fRowGroup; - - TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(large, cId)); - - if (mit == jobInfo.tableJoinMap.end()) - throw runtime_error("Join step not found."); - - largeJoinInfo->fJoinData = mit->second; - - // switch large and small sides - joinOrder.back() = large; - large = cId; - smallSides[0] = largeJoinInfo; - tableInfoMap[large].fJoinedTables = tableSet; - - bps = sbps; - thjs = sthjs; - tsas = NULL; - } - } - - if (!bps && !thjs && !tsas) - { - if (dynamic_cast(spjs.get())) - throw IDBExcept(ERR_NON_SUPPORT_SUB_QUERY_TYPE); - - throw runtime_error("Not supported join."); - } - - size_t dcf = 0; // for dictionary column filters, 0 if thjs is null. - RowGroup largeSideRG = tableInfoMap[large].fRowGroup; - - if (thjs && thjs->tokenJoin() == large) - { - dcf = thjs->getLargeKeys().size(); - largeSideRG = thjs->getLargeRowGroup(); - } - - // info for debug trace - vector tableNames; - vector traces; - - // sort the smallsides base on the joinId - sort(smallSides.begin(), smallSides.end(), joinInfoCompare); - int64_t lastJoinId = smallSides.back()->fJoinData.fJoinId; - - // get info to config the TupleHashjoin - DataListVec smallSideDLs; - vector smallSideRGs; - vector jointypes; - vector typeless; - vector > smallKeyIndices; - vector > largeKeyIndices; - - for (vector::iterator i = smallSides.begin(); i != smallSides.end(); i++) - { - JoinInfo* info = i->get(); - smallSideDLs.push_back(info->fDl); - smallSideRGs.push_back(info->fRowGroup); - jointypes.push_back(info->fJoinData.fTypes[0]); - typeless.push_back(info->fJoinData.fTypeless); - - vector smallIndices; - vector largeIndices; - const vector& keys1 = info->fJoinData.fLeftKeys; - const vector& keys2 = info->fJoinData.fRightKeys; - vector::const_iterator k1 = keys1.begin(); - vector::const_iterator k2 = keys2.begin(); - uint32_t stid = getTableKey(jobInfo, *k1); - tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[stid].fTable); - - for (; k1 != keys1.end(); ++k1, ++k2) - { - smallIndices.push_back(getKeyIndex(*k1, info->fRowGroup)); - largeIndices.push_back(getKeyIndex(*k2, largeSideRG)); - } - - smallKeyIndices.push_back(smallIndices); - largeKeyIndices.push_back(largeIndices); - - if (jobInfo.trace) - { - // small side column - ostringstream smallKey, smallIndex; - string alias1 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys1.front())]; - smallKey << alias1 << "-"; - - for (k1 = keys1.begin(); k1 != keys1.end(); ++k1) - { - CalpontSystemCatalog::OID oid1 = jobInfo.keyInfo->tupleKeyVec[*k1].fId; - CalpontSystemCatalog::TableColName tcn1 = jobInfo.csc->colName(oid1); - smallKey << "(" << tcn1.column << ":" << oid1 << ":" << *k1 << ")"; - smallIndex << " " << getKeyIndex(*k1, info->fRowGroup); - } - - // large side column - ostringstream largeKey, largeIndex; - string alias2 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys2.front())]; - largeKey << alias2 << "-"; - - for (k2 = keys2.begin(); k2 != keys2.end(); ++k2) - { - CalpontSystemCatalog::OID oid2 = jobInfo.keyInfo->tupleKeyVec[*k2].fId; - CalpontSystemCatalog::TableColName tcn2 = jobInfo.csc->colName(oid2); - largeKey << "(" << tcn2.column << ":" << oid2 << ":" << *k2 << ")"; - largeIndex << " " << getKeyIndex(*k2, largeSideRG); - } - - ostringstream oss; - oss << smallKey.str() << " join on " << largeKey.str() - << " joinType: " << info->fJoinData.fTypes.front() - << "(" << joinTypeToString(info->fJoinData.fTypes.front()) << ")" - << (info->fJoinData.fTypeless ? " " : " !") << "typeless" << endl; - oss << "smallSideIndex-largeSideIndex :" << smallIndex.str() << " --" - << largeIndex.str() << endl; - oss << "small side RG" << endl << info->fRowGroup.toString() << endl; - traces.push_back(oss.str()); - } - } - - if (jobInfo.trace) - { - ostringstream oss; - oss << "large side RG" << endl << largeSideRG.toString() << endl; - traces.push_back(oss.str()); - } - - if (bps || tsas) - { - thjs = new TupleHashJoinStep(jobInfo); - thjs->tableOid1(smallSides[0]->fTableOid); - thjs->tableOid2(tableInfoMap[large].fTableOid); - thjs->alias1(smallSides[0]->fAlias); - thjs->alias2(tableInfoMap[large].fAlias); - thjs->view1(smallSides[0]->fView); - thjs->view2(tableInfoMap[large].fView); - thjs->schema1(smallSides[0]->fSchema); - thjs->schema2(tableInfoMap[large].fSchema); - thjs->setLargeSideBPS(bps); - thjs->joinId(lastJoinId); - - if (dynamic_cast(bps) != NULL) - bps->incWaitToRunStepCnt(); - - SJSTEP spjs(thjs); - - JobStepAssociation inJsa; - inJsa.outAdd(smallSideDLs, 0); - inJsa.outAdd(tableInfoMap[large].fDl); - thjs->inputAssociation(inJsa); - thjs->setLargeSideDLIndex(inJsa.outSize() - 1); - - JobStepAssociation outJsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - spdl->rowGroupDL(dl); - dl->OID(large); - outJsa.outAdd(spdl); - thjs->outputAssociation(outJsa); - - thjs->configSmallSideRG(smallSideRGs, tableNames); - thjs->configLargeSideRG(tableInfoMap[large].fRowGroup); - thjs->configJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); - - tableInfoMap[large].fQuerySteps.push_back(spjs); - tableInfoMap[large].fDl = spdl; - } - else - { - JobStepAssociation inJsa = thjs->inputAssociation(); - - if (inJsa.outSize() < 2) - throw runtime_error("Not enough input to a hashjoin."); - - size_t last = inJsa.outSize() - 1; - inJsa.outAdd(smallSideDLs, last); - thjs->inputAssociation(inJsa); - thjs->setLargeSideDLIndex(inJsa.outSize() - 1); - - thjs->addSmallSideRG(smallSideRGs, tableNames); - thjs->addJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); - } - - RowGroup rg; - constructJoinedRowGroup(rg, link, prevLarge, root, tableSet, tableInfoMap, jobInfo); - thjs->setOutputRowGroup(rg); - tableInfoMap[large].fRowGroup = rg; - - if (jobInfo.trace) - { - cout << boldStart << "\n====== join info ======\n" << boldStop; - - for (vector::iterator t = traces.begin(); t != traces.end(); ++t) - cout << *t; - - cout << "RowGroup join result: " << endl << rg.toString() << endl << endl; - } - - // check if any cross-table expressions can be evaluated after the join - JobStepVector readyExpSteps; - JobStepVector& expSteps = jobInfo.crossTableExpressions; - JobStepVector::iterator eit = expSteps.begin(); - - while (eit != expSteps.end()) - { - ExpressionStep* exp = dynamic_cast(eit->get()); - - if (exp == NULL) - throw runtime_error("Not an expression."); - - if (exp->functionJoin()) - { - eit++; - continue; // done as join - } - - const vector& tables = exp->tableKeys(); - uint64_t i = 0; - - for (; i < tables.size(); i++) - { - if (tableSet.find(tables[i]) == tableSet.end()) - break; - } - - // all tables for this expression are joined - if (tables.size() == i) - { - readyExpSteps.push_back(*eit); - eit = expSteps.erase(eit); - } - else - { - eit++; - } - } - - // if root, handle the delayed outer join filters - if (root && jobInfo.outerJoinExpressions.size() > 0) - readyExpSteps.insert(readyExpSteps.end(), - jobInfo.outerJoinExpressions.begin(), - jobInfo.outerJoinExpressions.end()); - - PostJoinFilterKeys postJoinFilterKeys; - if (edgesToTransform.size()) - matchEdgesInRowGroup(jobInfo, rg, edgesToTransform, postJoinFilterKeys); - - // check additional compares for semi-join. - if (readyExpSteps.size() > 0 || postJoinFilterKeys.size() > 0) - { - // tables have additional comparisons - map correlateTables; // index in thjs - map correlateCompare; // expression - // map keys to the indices in the RG - map keyToIndexMap; - - const auto& rowGroupKeys = rg.getKeys(); - for (uint64_t i = 0, e = rowGroupKeys.size(); i < e; ++i) - keyToIndexMap.insert(make_pair(rowGroupKeys[i], i)); - - if (readyExpSteps.size() > 0) - { - for (size_t i = 0; i != smallSides.size(); i++) - { - if ((jointypes[i] & SEMI) || (jointypes[i] & ANTI) || (jointypes[i] & SCALAR)) - { - uint32_t tid = - getTableKey(jobInfo, smallSides[i]->fTableOid, smallSides[i]->fAlias, - smallSides[i]->fSchema, smallSides[i]->fView); - correlateTables[tid] = i; - correlateCompare[tid] = NULL; - } - } - } - - if (readyExpSteps.size() > 0 && correlateTables.size() > 0) - { - // separate additional compare for each table pair - JobStepVector::iterator eit = readyExpSteps.begin(); - - while (eit != readyExpSteps.end()) - { - ExpressionStep* e = dynamic_cast(eit->get()); - - if (e->selectFilter()) - { - // @bug3780, leave select filter to normal expression - eit++; - continue; - } - - const vector& tables = e->tableKeys(); - map::iterator j = correlateTables.end(); - - for (size_t i = 0; i < tables.size(); i++) - { - j = correlateTables.find(tables[i]); - - if (j != correlateTables.end()) - break; - } - - if (j == correlateTables.end()) - { - eit++; - continue; - } - - // map the input column index - e->updateInputIndex(keyToIndexMap, jobInfo); - ParseTree* pt = correlateCompare[j->first]; - - if (pt == NULL) - { - // first expression - pt = new ParseTree; - pt->copyTree(*(e->expressionFilter())); - } - else - { - // combine the expressions - ParseTree* left = pt; - ParseTree* right = new ParseTree; - right->copyTree(*(e->expressionFilter())); - pt = new ParseTree(new LogicOperator("and")); - pt->left(left); - pt->right(right); - } - - correlateCompare[j->first] = pt; - eit = readyExpSteps.erase(eit); - } - - map::iterator k = correlateTables.begin(); - - while (k != correlateTables.end()) - { - ParseTree* pt = correlateCompare[k->first]; - - if (pt != NULL) - { - boost::shared_ptr sppt(pt); - thjs->addJoinFilter(sppt, dcf + k->second); - } - - k++; - } - - thjs->setJoinFilterInputRG(rg); - } - - // normal expression if any. - if (readyExpSteps.size() > 0 || postJoinFilterKeys.size() > 0) - { - // add the expression steps in where clause can be solved by this join to bps - ParseTree* pt = NULL; - - std::vector postJoinFilters; - createPostJoinFilters(jobInfo, tableInfoMap, postJoinFilterKeys, keyToIndexMap, - postJoinFilters); - - if (jobInfo.trace) - { - cout << "Filters created " << endl; - for (auto* filter : postJoinFilters) - cout << filter->toString() << endl; - } - - for (auto* joinFilter : postJoinFilters) - { - if (pt == nullptr) - { - pt = new ParseTree(joinFilter); - } - else - { - ParseTree* left = pt; - ParseTree* right = new ParseTree(joinFilter); - pt = new ParseTree(new LogicOperator("and")); - pt->left(left); - pt->right(right); - } - } - - JobStepVector::iterator eit = readyExpSteps.begin(); - - for (; eit != readyExpSteps.end(); eit++) - { - // map the input column index - ExpressionStep* e = dynamic_cast(eit->get()); - e->updateInputIndex(keyToIndexMap, jobInfo); - - if (pt == NULL) - { - // first expression - pt = new ParseTree; - pt->copyTree(*(e->expressionFilter())); - } - else - { - // combine the expressions - ParseTree* left = pt; - ParseTree* right = new ParseTree; - right->copyTree(*(e->expressionFilter())); - pt = new ParseTree(new LogicOperator("and")); - pt->left(left); - pt->right(right); - } - } - - if (pt) - { - boost::shared_ptr sppt(pt); - thjs->addFcnExpGroup2(sppt); - } - } - - // update the fColsInExp2 and construct the output RG - updateExp2Cols(readyExpSteps, tableInfoMap, jobInfo); - constructJoinedRowGroup(rg, link, prevLarge, root, tableSet, tableInfoMap, jobInfo); - - if (thjs->hasFcnExpGroup2()) - thjs->setFE23Output(rg); - else - thjs->setOutputRowGroup(rg); - - tableInfoMap[large].fRowGroup = rg; - - if (jobInfo.trace) - { - cout << "RowGroup of " << tableInfoMap[large].fAlias << " after EXP G2: " << endl - << rg.toString() << endl << endl; - } - } - } - - - // Prepare the current table info to join with its large side. - SP_JoinInfo joinInfo(new JoinInfo); - joinInfo->fTableOid = tableInfoMap[large].fTableOid; - joinInfo->fAlias = tableInfoMap[large].fAlias; - joinInfo->fView = tableInfoMap[large].fView; - joinInfo->fSchema = tableInfoMap[large].fSchema; - - joinInfo->fDl = tableInfoMap[large].fDl; - joinInfo->fRowGroup = tableInfoMap[large].fRowGroup; - - if (root == false) // not root - { - TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(link, prevLarge)); + SJSTEP sspjs = tableInfoMap[cId].fQuerySteps.back(); + BatchPrimitive* sbps = dynamic_cast(sspjs.get()); + TupleHashJoinStep* sthjs = dynamic_cast(sspjs.get()); + + if (sbps || (sthjs && sthjs->tokenJoin() == cId)) + { + SP_JoinInfo largeJoinInfo(new JoinInfo); + largeJoinInfo->fTableOid = tableInfoMap[large].fTableOid; + largeJoinInfo->fAlias = tableInfoMap[large].fAlias; + largeJoinInfo->fView = tableInfoMap[large].fView; + largeJoinInfo->fSchema = tableInfoMap[large].fSchema; + + largeJoinInfo->fDl = tableInfoMap[large].fDl; + largeJoinInfo->fRowGroup = tableInfoMap[large].fRowGroup; + + TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(large, cId)); if (mit == jobInfo.tableJoinMap.end()) - throw runtime_error("Join step not found."); + throw runtime_error("Join step not found."); - joinInfo->fJoinData = mit->second; + largeJoinInfo->fJoinData = mit->second; + + // switch large and small sides + joinOrder.back() = large; + large = cId; + smallSides[0] = largeJoinInfo; + tableInfoMap[large].fJoinedTables = tableSet; + + bps = sbps; + thjs = sthjs; + tsas = NULL; + } } - joinOrder.push_back(large); + if (!bps && !thjs && !tsas) + { + if (dynamic_cast(spjs.get())) + throw IDBExcept(ERR_NON_SUPPORT_SUB_QUERY_TYPE); - return joinInfo; + throw runtime_error("Not supported join."); + } + + size_t dcf = 0; // for dictionary column filters, 0 if thjs is null. + RowGroup largeSideRG = tableInfoMap[large].fRowGroup; + + if (thjs && thjs->tokenJoin() == large) + { + dcf = thjs->getLargeKeys().size(); + largeSideRG = thjs->getLargeRowGroup(); + } + + // info for debug trace + vector tableNames; + vector traces; + + // sort the smallsides base on the joinId + sort(smallSides.begin(), smallSides.end(), joinInfoCompare); + int64_t lastJoinId = smallSides.back()->fJoinData.fJoinId; + + // get info to config the TupleHashjoin + DataListVec smallSideDLs; + vector smallSideRGs; + vector jointypes; + vector typeless; + vector> smallKeyIndices; + vector> largeKeyIndices; + + for (vector::iterator i = smallSides.begin(); i != smallSides.end(); i++) + { + JoinInfo* info = i->get(); + smallSideDLs.push_back(info->fDl); + smallSideRGs.push_back(info->fRowGroup); + jointypes.push_back(info->fJoinData.fTypes[0]); + typeless.push_back(info->fJoinData.fTypeless); + + vector smallIndices; + vector largeIndices; + const vector& keys1 = info->fJoinData.fLeftKeys; + const vector& keys2 = info->fJoinData.fRightKeys; + vector::const_iterator k1 = keys1.begin(); + vector::const_iterator k2 = keys2.begin(); + uint32_t stid = getTableKey(jobInfo, *k1); + tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[stid].fTable); + + for (; k1 != keys1.end(); ++k1, ++k2) + { + smallIndices.push_back(getKeyIndex(*k1, info->fRowGroup)); + largeIndices.push_back(getKeyIndex(*k2, largeSideRG)); + } + + smallKeyIndices.push_back(smallIndices); + largeKeyIndices.push_back(largeIndices); + + if (jobInfo.trace) + { + // small side column + ostringstream smallKey, smallIndex; + string alias1 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys1.front())]; + smallKey << alias1 << "-"; + + for (k1 = keys1.begin(); k1 != keys1.end(); ++k1) + { + CalpontSystemCatalog::OID oid1 = jobInfo.keyInfo->tupleKeyVec[*k1].fId; + CalpontSystemCatalog::TableColName tcn1 = jobInfo.csc->colName(oid1); + smallKey << "(" << tcn1.column << ":" << oid1 << ":" << *k1 << ")"; + smallIndex << " " << getKeyIndex(*k1, info->fRowGroup); + } + + // large side column + ostringstream largeKey, largeIndex; + string alias2 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys2.front())]; + largeKey << alias2 << "-"; + + for (k2 = keys2.begin(); k2 != keys2.end(); ++k2) + { + CalpontSystemCatalog::OID oid2 = jobInfo.keyInfo->tupleKeyVec[*k2].fId; + CalpontSystemCatalog::TableColName tcn2 = jobInfo.csc->colName(oid2); + largeKey << "(" << tcn2.column << ":" << oid2 << ":" << *k2 << ")"; + largeIndex << " " << getKeyIndex(*k2, largeSideRG); + } + + ostringstream oss; + oss << smallKey.str() << " join on " << largeKey.str() + << " joinType: " << info->fJoinData.fTypes.front() << "(" + << joinTypeToString(info->fJoinData.fTypes.front()) << ")" + << (info->fJoinData.fTypeless ? " " : " !") << "typeless" << endl; + oss << "smallSideIndex-largeSideIndex :" << smallIndex.str() << " --" << largeIndex.str() << endl; + oss << "small side RG" << endl << info->fRowGroup.toString() << endl; + traces.push_back(oss.str()); + } + } + + if (jobInfo.trace) + { + ostringstream oss; + oss << "large side RG" << endl << largeSideRG.toString() << endl; + traces.push_back(oss.str()); + } + + if (bps || tsas) + { + thjs = new TupleHashJoinStep(jobInfo); + thjs->tableOid1(smallSides[0]->fTableOid); + thjs->tableOid2(tableInfoMap[large].fTableOid); + thjs->alias1(smallSides[0]->fAlias); + thjs->alias2(tableInfoMap[large].fAlias); + thjs->view1(smallSides[0]->fView); + thjs->view2(tableInfoMap[large].fView); + thjs->schema1(smallSides[0]->fSchema); + thjs->schema2(tableInfoMap[large].fSchema); + thjs->setLargeSideBPS(bps); + thjs->joinId(lastJoinId); + + if (dynamic_cast(bps) != NULL) + bps->incWaitToRunStepCnt(); + + SJSTEP spjs(thjs); + + JobStepAssociation inJsa; + inJsa.outAdd(smallSideDLs, 0); + inJsa.outAdd(tableInfoMap[large].fDl); + thjs->inputAssociation(inJsa); + thjs->setLargeSideDLIndex(inJsa.outSize() - 1); + + JobStepAssociation outJsa; + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + spdl->rowGroupDL(dl); + dl->OID(large); + outJsa.outAdd(spdl); + thjs->outputAssociation(outJsa); + + thjs->configSmallSideRG(smallSideRGs, tableNames); + thjs->configLargeSideRG(tableInfoMap[large].fRowGroup); + thjs->configJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); + + tableInfoMap[large].fQuerySteps.push_back(spjs); + tableInfoMap[large].fDl = spdl; + } + else + { + JobStepAssociation inJsa = thjs->inputAssociation(); + + if (inJsa.outSize() < 2) + throw runtime_error("Not enough input to a hashjoin."); + + size_t last = inJsa.outSize() - 1; + inJsa.outAdd(smallSideDLs, last); + thjs->inputAssociation(inJsa); + thjs->setLargeSideDLIndex(inJsa.outSize() - 1); + + thjs->addSmallSideRG(smallSideRGs, tableNames); + thjs->addJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); + } + + RowGroup rg; + constructJoinedRowGroup(rg, link, prevLarge, root, tableSet, tableInfoMap, jobInfo); + thjs->setOutputRowGroup(rg); + tableInfoMap[large].fRowGroup = rg; + + if (jobInfo.trace) + { + cout << boldStart << "\n====== join info ======\n" << boldStop; + + for (vector::iterator t = traces.begin(); t != traces.end(); ++t) + cout << *t; + + cout << "RowGroup join result: " << endl << rg.toString() << endl << endl; + } + + // check if any cross-table expressions can be evaluated after the join + JobStepVector readyExpSteps; + JobStepVector& expSteps = jobInfo.crossTableExpressions; + JobStepVector::iterator eit = expSteps.begin(); + + while (eit != expSteps.end()) + { + ExpressionStep* exp = dynamic_cast(eit->get()); + + if (exp == NULL) + throw runtime_error("Not an expression."); + + if (exp->functionJoin()) + { + eit++; + continue; // done as join + } + + const vector& tables = exp->tableKeys(); + uint64_t i = 0; + + for (; i < tables.size(); i++) + { + if (tableSet.find(tables[i]) == tableSet.end()) + break; + } + + // all tables for this expression are joined + if (tables.size() == i) + { + readyExpSteps.push_back(*eit); + eit = expSteps.erase(eit); + } + else + { + eit++; + } + } + + // if root, handle the delayed outer join filters + if (root && jobInfo.outerJoinExpressions.size() > 0) + readyExpSteps.insert(readyExpSteps.end(), jobInfo.outerJoinExpressions.begin(), + jobInfo.outerJoinExpressions.end()); + + PostJoinFilterKeys postJoinFilterKeys; + if (edgesToTransform.size()) + matchEdgesInRowGroup(jobInfo, rg, edgesToTransform, postJoinFilterKeys); + + // check additional compares for semi-join. + if (readyExpSteps.size() > 0 || postJoinFilterKeys.size() > 0) + { + // tables have additional comparisons + map correlateTables; // index in thjs + map correlateCompare; // expression + // map keys to the indices in the RG + map keyToIndexMap; + + const auto& rowGroupKeys = rg.getKeys(); + for (uint64_t i = 0, e = rowGroupKeys.size(); i < e; ++i) + keyToIndexMap.insert(make_pair(rowGroupKeys[i], i)); + + if (readyExpSteps.size() > 0) + { + for (size_t i = 0; i != smallSides.size(); i++) + { + if ((jointypes[i] & SEMI) || (jointypes[i] & ANTI) || (jointypes[i] & SCALAR)) + { + uint32_t tid = getTableKey(jobInfo, smallSides[i]->fTableOid, smallSides[i]->fAlias, + smallSides[i]->fSchema, smallSides[i]->fView); + correlateTables[tid] = i; + correlateCompare[tid] = NULL; + } + } + } + + if (readyExpSteps.size() > 0 && correlateTables.size() > 0) + { + // separate additional compare for each table pair + JobStepVector::iterator eit = readyExpSteps.begin(); + + while (eit != readyExpSteps.end()) + { + ExpressionStep* e = dynamic_cast(eit->get()); + + if (e->selectFilter()) + { + // @bug3780, leave select filter to normal expression + eit++; + continue; + } + + const vector& tables = e->tableKeys(); + map::iterator j = correlateTables.end(); + + for (size_t i = 0; i < tables.size(); i++) + { + j = correlateTables.find(tables[i]); + + if (j != correlateTables.end()) + break; + } + + if (j == correlateTables.end()) + { + eit++; + continue; + } + + // map the input column index + e->updateInputIndex(keyToIndexMap, jobInfo); + ParseTree* pt = correlateCompare[j->first]; + + if (pt == NULL) + { + // first expression + pt = new ParseTree; + pt->copyTree(*(e->expressionFilter())); + } + else + { + // combine the expressions + ParseTree* left = pt; + ParseTree* right = new ParseTree; + right->copyTree(*(e->expressionFilter())); + pt = new ParseTree(new LogicOperator("and")); + pt->left(left); + pt->right(right); + } + + correlateCompare[j->first] = pt; + eit = readyExpSteps.erase(eit); + } + + map::iterator k = correlateTables.begin(); + + while (k != correlateTables.end()) + { + ParseTree* pt = correlateCompare[k->first]; + + if (pt != NULL) + { + boost::shared_ptr sppt(pt); + thjs->addJoinFilter(sppt, dcf + k->second); + } + + k++; + } + + thjs->setJoinFilterInputRG(rg); + } + + // normal expression if any. + if (readyExpSteps.size() > 0 || postJoinFilterKeys.size() > 0) + { + // add the expression steps in where clause can be solved by this join to bps + ParseTree* pt = NULL; + + std::vector postJoinFilters; + createPostJoinFilters(jobInfo, tableInfoMap, postJoinFilterKeys, keyToIndexMap, postJoinFilters); + + if (jobInfo.trace) + { + cout << "Filters created " << endl; + for (auto* filter : postJoinFilters) + cout << filter->toString() << endl; + } + + for (auto* joinFilter : postJoinFilters) + { + if (pt == nullptr) + { + pt = new ParseTree(joinFilter); + } + else + { + ParseTree* left = pt; + ParseTree* right = new ParseTree(joinFilter); + pt = new ParseTree(new LogicOperator("and")); + pt->left(left); + pt->right(right); + } + } + + JobStepVector::iterator eit = readyExpSteps.begin(); + + for (; eit != readyExpSteps.end(); eit++) + { + // map the input column index + ExpressionStep* e = dynamic_cast(eit->get()); + e->updateInputIndex(keyToIndexMap, jobInfo); + + if (pt == NULL) + { + // first expression + pt = new ParseTree; + pt->copyTree(*(e->expressionFilter())); + } + else + { + // combine the expressions + ParseTree* left = pt; + ParseTree* right = new ParseTree; + right->copyTree(*(e->expressionFilter())); + pt = new ParseTree(new LogicOperator("and")); + pt->left(left); + pt->right(right); + } + } + + if (pt) + { + boost::shared_ptr sppt(pt); + thjs->addFcnExpGroup2(sppt); + } + } + + // update the fColsInExp2 and construct the output RG + updateExp2Cols(readyExpSteps, tableInfoMap, jobInfo); + constructJoinedRowGroup(rg, link, prevLarge, root, tableSet, tableInfoMap, jobInfo); + + if (thjs->hasFcnExpGroup2()) + thjs->setFE23Output(rg); + else + thjs->setOutputRowGroup(rg); + + tableInfoMap[large].fRowGroup = rg; + + if (jobInfo.trace) + { + cout << "RowGroup of " << tableInfoMap[large].fAlias << " after EXP G2: " << endl + << rg.toString() << endl + << endl; + } + } + } + + // Prepare the current table info to join with its large side. + SP_JoinInfo joinInfo(new JoinInfo); + joinInfo->fTableOid = tableInfoMap[large].fTableOid; + joinInfo->fAlias = tableInfoMap[large].fAlias; + joinInfo->fView = tableInfoMap[large].fView; + joinInfo->fSchema = tableInfoMap[large].fSchema; + + joinInfo->fDl = tableInfoMap[large].fDl; + joinInfo->fRowGroup = tableInfoMap[large].fRowGroup; + + if (root == false) // not root + { + TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(link, prevLarge)); + + if (mit == jobInfo.tableJoinMap.end()) + throw runtime_error("Join step not found."); + + joinInfo->fJoinData = mit->second; + } + + joinOrder.push_back(large); + + return joinInfo; } - bool joinStepCompare(const SJSTEP& a, const SJSTEP& b) { - return (dynamic_cast(a.get())->joinId() < - dynamic_cast(b.get())->joinId()); + return (dynamic_cast(a.get())->joinId() < + dynamic_cast(b.get())->joinId()); } - struct JoinOrderData { - uint32_t fTid1; - uint32_t fTid2; - uint32_t fJoinId; + uint32_t fTid1; + uint32_t fTid2; + uint32_t fJoinId; }; - void getJoinOrder(vector& joins, JobStepVector& joinSteps, JobInfo& jobInfo) { - sort(joinSteps.begin(), joinSteps.end(), joinStepCompare); + sort(joinSteps.begin(), joinSteps.end(), joinStepCompare); - for (JobStepVector::iterator i = joinSteps.begin(); i < joinSteps.end(); i++) + for (JobStepVector::iterator i = joinSteps.begin(); i < joinSteps.end(); i++) + { + TupleHashJoinStep* thjs = dynamic_cast(i->get()); + JoinOrderData jo; + jo.fTid1 = getTableKey(jobInfo, thjs->tupleId1()); + jo.fTid2 = getTableKey(jobInfo, thjs->tupleId2()); + jo.fJoinId = thjs->joinId(); + + // not fastest, but good for a small list + vector::iterator j; + + for (j = joins.begin(); j < joins.end(); j++) { - TupleHashJoinStep* thjs = dynamic_cast(i->get()); - JoinOrderData jo; - jo.fTid1 = getTableKey(jobInfo, thjs->tupleId1()); - jo.fTid2 = getTableKey(jobInfo, thjs->tupleId2()); - jo.fJoinId = thjs->joinId(); - - // not fastest, but good for a small list - vector::iterator j; - - for (j = joins.begin(); j < joins.end(); j++) - { - if ((j->fTid1 == jo.fTid1 && j->fTid2 == jo.fTid2) || - (j->fTid1 == jo.fTid2 && j->fTid2 == jo.fTid1)) - { - j->fJoinId = jo.fJoinId; - break; - } - } - - // insert unique join pair - if (j == joins.end()) - joins.push_back(jo); + if ((j->fTid1 == jo.fTid1 && j->fTid2 == jo.fTid2) || (j->fTid1 == jo.fTid2 && j->fTid2 == jo.fTid1)) + { + j->fJoinId = jo.fJoinId; + break; + } } + + // insert unique join pair + if (j == joins.end()) + joins.push_back(jo); + } } inline void updateJoinSides(uint32_t small, uint32_t large, map& joinInfoMap, vector& smallSides, TableInfoMap& tableInfoMap, JobInfo& jobInfo) { - TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(small, large)); + TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(small, large)); - if (mit == jobInfo.tableJoinMap.end()) - throw runtime_error("Join step not found."); + if (mit == jobInfo.tableJoinMap.end()) + throw runtime_error("Join step not found."); - joinInfoMap[small]->fJoinData = mit->second; - tableInfoMap[small].fJoinedTables.insert(small); - smallSides.push_back(joinInfoMap[small]); + joinInfoMap[small]->fJoinData = mit->second; + tableInfoMap[small].fJoinedTables.insert(small); + smallSides.push_back(joinInfoMap[small]); - tableInfoMap[large].fJoinedTables.insert( - tableInfoMap[small].fJoinedTables.begin(), tableInfoMap[small].fJoinedTables.end()); - tableInfoMap[large].fJoinedTables.insert(large); + tableInfoMap[large].fJoinedTables.insert(tableInfoMap[small].fJoinedTables.begin(), + tableInfoMap[small].fJoinedTables.end()); + tableInfoMap[large].fJoinedTables.insert(large); } - // For OUTER JOIN bug @2422/2633/3437/3759, join table based on join order. // The largest table will be always the streaming table, other tables are always on small side. void joinTablesInOrder(uint32_t largest, JobStepVector& joinSteps, TableInfoMap& tableInfoMap, JobInfo& jobInfo, vector& joinOrder) { - // populate the tableInfo for join - map joinInfoMap; // + // populate the tableInfo for join + map joinInfoMap; // - // > - // large priority: - // -1 - must be on small side, like derived tables for semi join; - // 0 - prefer to be on small side, like FROM subquery; - // 1 - can be on either large or small side; - // 2 - must be on large side. - map > joinStepMap; + // > + // large priority: + // -1 - must be on small side, like derived tables for semi join; + // 0 - prefer to be on small side, like FROM subquery; + // 1 - can be on either large or small side; + // 2 - must be on large side. + map> joinStepMap; - BatchPrimitive* bps = NULL; - SubAdapterStep* tsas = NULL; - TupleHashJoinStep* thjs = NULL; + BatchPrimitive* bps = NULL; + SubAdapterStep* tsas = NULL; + TupleHashJoinStep* thjs = NULL; - for (vector::iterator i = jobInfo.tableList.begin(); i < jobInfo.tableList.end(); i++) + for (vector::iterator i = jobInfo.tableList.begin(); i < jobInfo.tableList.end(); i++) + { + SP_JoinInfo joinInfo(new JoinInfo); + joinInfo->fTableOid = tableInfoMap[*i].fTableOid; + joinInfo->fAlias = tableInfoMap[*i].fAlias; + joinInfo->fView = tableInfoMap[*i].fView; + joinInfo->fSchema = tableInfoMap[*i].fSchema; + + joinInfo->fDl = tableInfoMap[*i].fDl; + joinInfo->fRowGroup = tableInfoMap[*i].fRowGroup; + + joinInfoMap[*i] = joinInfo; + + SJSTEP spjs = tableInfoMap[*i].fQuerySteps.back(); + bps = dynamic_cast(spjs.get()); + tsas = dynamic_cast(spjs.get()); + thjs = dynamic_cast(spjs.get()); + TupleBPS* tbps = dynamic_cast(spjs.get()); + + if (*i == largest) + joinStepMap[*i] = make_pair(spjs, 2); + else if (tbps || thjs) + joinStepMap[*i] = make_pair(spjs, 1); + else if (tsas) + joinStepMap[*i] = make_pair(spjs, 0); + else + joinStepMap[*i] = make_pair(spjs, -1); + } + + // sort the join steps based on join ID. + vector joins; + getJoinOrder(joins, joinSteps, jobInfo); + + // join the steps + int64_t lastJoinId = -1; + uint32_t large = (uint32_t)-1; + uint32_t small = (uint32_t)-1; + uint32_t prevLarge = (uint32_t)-1; + bool umstream = false; + vector joinedTable; + uint32_t lastJoin = joins.size() - 1; + bool isSemijoin = true; + + for (uint64_t js = 0; js < joins.size(); js++) + { + set smallSideTid; + + if (joins[js].fJoinId != 0) + isSemijoin = false; + + vector smallSides; + uint32_t tid1 = joins[js].fTid1; + uint32_t tid2 = joins[js].fTid2; + lastJoinId = joins[js].fJoinId; + + // largest has already joined, and this join cannot be merged. + if (prevLarge == largest && tid1 != largest && tid2 != largest) + umstream = true; + + if (joinStepMap[tid1].second > joinStepMap[tid2].second) // high priority { - SP_JoinInfo joinInfo(new JoinInfo); - joinInfo->fTableOid = tableInfoMap[*i].fTableOid; - joinInfo->fAlias = tableInfoMap[*i].fAlias; - joinInfo->fView = tableInfoMap[*i].fView; - joinInfo->fSchema = tableInfoMap[*i].fSchema; - - joinInfo->fDl = tableInfoMap[*i].fDl; - joinInfo->fRowGroup = tableInfoMap[*i].fRowGroup; - - joinInfoMap[*i] = joinInfo; - - SJSTEP spjs = tableInfoMap[*i].fQuerySteps.back(); - bps = dynamic_cast(spjs.get()); - tsas = dynamic_cast(spjs.get()); - thjs = dynamic_cast(spjs.get()); - TupleBPS* tbps = dynamic_cast(spjs.get()); - - if (*i == largest) - joinStepMap[*i] = make_pair(spjs, 2); - else if (tbps || thjs) - joinStepMap[*i] = make_pair(spjs, 1); - else if (tsas) - joinStepMap[*i] = make_pair(spjs, 0); - else - joinStepMap[*i] = make_pair(spjs, -1); + large = tid1; + small = tid2; + } + else if (joinStepMap[tid1].second == joinStepMap[tid2].second && + jobInfo.tableSize[tid1] >= jobInfo.tableSize[tid2]) // favor t1 for hint + { + large = tid1; + small = tid2; + } + else + { + large = tid2; + small = tid1; } - // sort the join steps based on join ID. - vector joins; - getJoinOrder(joins, joinSteps, jobInfo); + updateJoinSides(small, large, joinInfoMap, smallSides, tableInfoMap, jobInfo); - // join the steps - int64_t lastJoinId = -1; - uint32_t large = (uint32_t) - 1; - uint32_t small = (uint32_t) - 1; - uint32_t prevLarge = (uint32_t) - 1; - bool umstream = false; - vector joinedTable; - uint32_t lastJoin = joins.size() - 1; - bool isSemijoin = true; + if (find(joinedTable.begin(), joinedTable.end(), small) == joinedTable.end()) + joinedTable.push_back(small); - for (uint64_t js = 0; js < joins.size(); js++) + smallSideTid.insert(small); + + // merge in the next step if the large side is the same + for (uint64_t ns = js + 1; ns < joins.size(); js++, ns++) { - set smallSideTid; + uint32_t tid1 = joins[ns].fTid1; + uint32_t tid2 = joins[ns].fTid2; + uint32_t small = (uint32_t)-1; - if (joins[js].fJoinId != 0) - isSemijoin = false; + if ((tid1 == large) && ((joinStepMap[tid1].second > joinStepMap[tid2].second) || + (joinStepMap[tid1].second == joinStepMap[tid2].second && + jobInfo.tableSize[tid1] >= jobInfo.tableSize[tid2]))) + { + small = tid2; + } + else if ((tid2 == large) && ((joinStepMap[tid2].second > joinStepMap[tid1].second) || + (joinStepMap[tid2].second == joinStepMap[tid1].second && + jobInfo.tableSize[tid2] >= jobInfo.tableSize[tid1]))) + { + small = tid1; + } + else + { + break; + } - vector smallSides; - uint32_t tid1 = joins[js].fTid1; - uint32_t tid2 = joins[js].fTid2; - lastJoinId = joins[js].fJoinId; + // check if FE needs table in previous smallsides + if (jobInfo.joinFeTableMap[joins[ns].fJoinId].size() > 0) + { + set& tids = jobInfo.joinFeTableMap[joins[ns].fJoinId]; - // largest has already joined, and this join cannot be merged. - if (prevLarge == largest && tid1 != largest && tid2 != largest) - umstream = true; - - if (joinStepMap[tid1].second > joinStepMap[tid2].second) // high priority + for (set::iterator si = smallSideTid.begin(); si != smallSideTid.end(); si++) { - large = tid1; - small = tid2; + if (tids.find(*si) != tids.end()) + throw runtime_error( + "On clause filter involving a table not directly involved in the outer join is currently not " + "supported."); } - else if (joinStepMap[tid1].second == joinStepMap[tid2].second && - jobInfo.tableSize[tid1] >= jobInfo.tableSize[tid2]) // favor t1 for hint + } + + updateJoinSides(small, large, joinInfoMap, smallSides, tableInfoMap, jobInfo); + lastJoinId = joins[ns].fJoinId; + + if (find(joinedTable.begin(), joinedTable.end(), small) == joinedTable.end()) + joinedTable.push_back(small); + + smallSideTid.insert(small); + } + + joinedTable.push_back(large); + + SJSTEP spjs = joinStepMap[large].first; + bps = dynamic_cast(spjs.get()); + tsas = dynamic_cast(spjs.get()); + thjs = dynamic_cast(spjs.get()); + + if (!bps && !thjs && !tsas) + { + if (dynamic_cast(spjs.get())) + throw IDBExcept(ERR_NON_SUPPORT_SUB_QUERY_TYPE); + + throw runtime_error("Not supported join."); + } + + size_t startPos = 0; // start point to add new smallsides + RowGroup largeSideRG = joinInfoMap[large]->fRowGroup; + + if (thjs && thjs->tokenJoin() == large) + largeSideRG = thjs->getLargeRowGroup(); + + // get info to config the TupleHashjoin + vector traces; + vector tableNames; + DataListVec smallSideDLs; + vector smallSideRGs; + vector jointypes; + vector typeless; + vector> smallKeyIndices; + vector> largeKeyIndices; + + // bug5764, make sure semi joins acting as filter is after outer join. + { + // the inner table filters have to be performed after outer join + vector semijoins; + vector smallouts; + + for (size_t i = 0; i < smallSides.size(); i++) + { + // find the the small-outer and semi-join joins + JoinType jt = smallSides[i]->fJoinData.fTypes[0]; + + if (jt & SMALLOUTER) + smallouts.push_back(i); + else if (jt & (SEMI | ANTI | SCALAR | CORRELATED)) + semijoins.push_back(i); + } + + // check the join order, re-arrange if necessary + if (smallouts.size() > 0 && semijoins.size() > 0) + { + uint64_t lastSmallOut = smallouts.back(); + uint64_t lastSemijoin = semijoins.back(); + + if (lastSmallOut > lastSemijoin) { - large = tid1; - small = tid2; - } - else - { - large = tid2; - small = tid1; - } + vector temp1; + vector temp2; + size_t j = 0; - updateJoinSides(small, large, joinInfoMap, smallSides, tableInfoMap, jobInfo); - - if (find(joinedTable.begin(), joinedTable.end(), small) == joinedTable.end()) - joinedTable.push_back(small); - - smallSideTid.insert(small); - - // merge in the next step if the large side is the same - for (uint64_t ns = js + 1; ns < joins.size(); js++, ns++) - { - uint32_t tid1 = joins[ns].fTid1; - uint32_t tid2 = joins[ns].fTid2; - uint32_t small = (uint32_t) - 1; - - if ((tid1 == large) && - ((joinStepMap[tid1].second > joinStepMap[tid2].second) || - (joinStepMap[tid1].second == joinStepMap[tid2].second && - jobInfo.tableSize[tid1] >= jobInfo.tableSize[tid2]))) + for (size_t i = 0; i < smallSides.size(); i++) + { + if (j < semijoins.size() && i == semijoins[j]) { - small = tid2; - } - else if ((tid2 == large) && - ((joinStepMap[tid2].second > joinStepMap[tid1].second) || - (joinStepMap[tid2].second == joinStepMap[tid1].second && - jobInfo.tableSize[tid2] >= jobInfo.tableSize[tid1]))) - { - small = tid1; + temp1.push_back(smallSides[i]); + j++; } else { - break; + temp2.push_back(smallSides[i]); } - // check if FE needs table in previous smallsides - if (jobInfo.joinFeTableMap[joins[ns].fJoinId].size() > 0) - { - set& tids = jobInfo.joinFeTableMap[joins[ns].fJoinId]; + if (i == lastSmallOut) + temp2.insert(temp2.end(), temp1.begin(), temp1.end()); + } - for (set::iterator si = smallSideTid.begin(); si != smallSideTid.end(); si++) - { - if (tids.find(*si) != tids.end()) - throw runtime_error("On clause filter involving a table not directly involved in the outer join is currently not supported."); - } - } - - updateJoinSides(small, large, joinInfoMap, smallSides, tableInfoMap, jobInfo); - lastJoinId = joins[ns].fJoinId; - - if (find(joinedTable.begin(), joinedTable.end(), small) == joinedTable.end()) - joinedTable.push_back(small); - - smallSideTid.insert(small); + smallSides = temp2; } + } + } - joinedTable.push_back(large); + for (vector::iterator i = smallSides.begin(); i != smallSides.end(); i++) + { + JoinInfo* info = i->get(); + smallSideDLs.push_back(info->fDl); + smallSideRGs.push_back(info->fRowGroup); + jointypes.push_back(info->fJoinData.fTypes[0]); + typeless.push_back(info->fJoinData.fTypeless); - SJSTEP spjs = joinStepMap[large].first; - bps = dynamic_cast(spjs.get()); - tsas = dynamic_cast(spjs.get()); - thjs = dynamic_cast(spjs.get()); + vector smallIndices; + vector largeIndices; + const vector& keys1 = info->fJoinData.fLeftKeys; + const vector& keys2 = info->fJoinData.fRightKeys; + vector::const_iterator k1 = keys1.begin(); + vector::const_iterator k2 = keys2.begin(); + uint32_t stid = getTableKey(jobInfo, *k1); + tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[stid].fTable); - if (!bps && !thjs && !tsas) + for (; k1 != keys1.end(); ++k1, ++k2) + { + smallIndices.push_back(getKeyIndex(*k1, info->fRowGroup)); + largeIndices.push_back(getKeyIndex(*k2, largeSideRG)); + } + + smallKeyIndices.push_back(smallIndices); + largeKeyIndices.push_back(largeIndices); + + if (jobInfo.trace) + { + // small side column + ostringstream smallKey, smallIndex; + string alias1 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys1.front())]; + smallKey << alias1 << "-"; + + for (k1 = keys1.begin(); k1 != keys1.end(); ++k1) { - if (dynamic_cast(spjs.get())) - throw IDBExcept(ERR_NON_SUPPORT_SUB_QUERY_TYPE); - - throw runtime_error("Not supported join."); + CalpontSystemCatalog::OID oid1 = jobInfo.keyInfo->tupleKeyVec[*k1].fId; + CalpontSystemCatalog::TableColName tcn1 = jobInfo.csc->colName(oid1); + smallKey << "(" << tcn1.column << ":" << oid1 << ":" << *k1 << ")"; + smallIndex << " " << getKeyIndex(*k1, info->fRowGroup); } - size_t startPos = 0; // start point to add new smallsides - RowGroup largeSideRG = joinInfoMap[large]->fRowGroup; + // large side column + ostringstream largeKey, largeIndex; + string alias2 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys2.front())]; + largeKey << alias2 << "-"; - if (thjs && thjs->tokenJoin() == large) - largeSideRG = thjs->getLargeRowGroup(); - - // get info to config the TupleHashjoin - vector traces; - vector tableNames; - DataListVec smallSideDLs; - vector smallSideRGs; - vector jointypes; - vector typeless; - vector > smallKeyIndices; - vector > largeKeyIndices; - - // bug5764, make sure semi joins acting as filter is after outer join. + for (k2 = keys2.begin(); k2 != keys2.end(); ++k2) { - // the inner table filters have to be performed after outer join - vector semijoins; - vector smallouts; - - for (size_t i = 0; i < smallSides.size(); i++) - { - // find the the small-outer and semi-join joins - JoinType jt = smallSides[i]->fJoinData.fTypes[0]; - - if (jt & SMALLOUTER) - smallouts.push_back(i); - else if (jt & (SEMI | ANTI | SCALAR | CORRELATED)) - semijoins.push_back(i); - } - - // check the join order, re-arrange if necessary - if (smallouts.size() > 0 && semijoins.size() > 0) - { - uint64_t lastSmallOut = smallouts.back(); - uint64_t lastSemijoin = semijoins.back(); - - if (lastSmallOut > lastSemijoin) - { - vector temp1; - vector temp2; - size_t j = 0; - - for (size_t i = 0; i < smallSides.size(); i++) - { - if (j < semijoins.size() && i == semijoins[j]) - { - temp1.push_back(smallSides[i]); - j++; - } - else - { - temp2.push_back(smallSides[i]); - } - - if (i == lastSmallOut) - temp2.insert(temp2.end(), temp1.begin(), temp1.end()); - } - - smallSides = temp2; - } - } + CalpontSystemCatalog::OID oid2 = jobInfo.keyInfo->tupleKeyVec[*k2].fId; + CalpontSystemCatalog::TableColName tcn2 = jobInfo.csc->colName(oid2); + largeKey << "(" << tcn2.column << ":" << oid2 << ":" << *k2 << ")"; + largeIndex << " " << getKeyIndex(*k2, largeSideRG); } - for (vector::iterator i = smallSides.begin(); i != smallSides.end(); i++) + ostringstream oss; + oss << smallKey.str() << " join on " << largeKey.str() + << " joinType: " << info->fJoinData.fTypes.front() << "(" + << joinTypeToString(info->fJoinData.fTypes.front()) << ")" + << (info->fJoinData.fTypeless ? " " : " !") << "typeless" << endl; + oss << "smallSideIndex-largeSideIndex :" << smallIndex.str() << " --" << largeIndex.str() << endl; + oss << "small side RG" << endl << info->fRowGroup.toString() << endl; + traces.push_back(oss.str()); + } + } + + if (jobInfo.trace) + { + ostringstream oss; + oss << "large side RG" << endl << largeSideRG.toString() << endl; + traces.push_back(oss.str()); + } + + if (bps || tsas || umstream || (thjs && joinStepMap[large].second < 1)) + { + thjs = new TupleHashJoinStep(jobInfo); + thjs->tableOid1(smallSides[0]->fTableOid); + thjs->tableOid2(tableInfoMap[large].fTableOid); + thjs->alias1(smallSides[0]->fAlias); + thjs->alias2(tableInfoMap[large].fAlias); + thjs->view1(smallSides[0]->fView); + thjs->view2(tableInfoMap[large].fView); + thjs->schema1(smallSides[0]->fSchema); + thjs->schema2(tableInfoMap[large].fSchema); + thjs->setLargeSideBPS(bps); + thjs->joinId(lastJoinId); + + if (dynamic_cast(bps) != NULL) + bps->incWaitToRunStepCnt(); + + spjs.reset(thjs); + + JobStepAssociation inJsa; + inJsa.outAdd(smallSideDLs, 0); + inJsa.outAdd(joinInfoMap[large]->fDl); + thjs->inputAssociation(inJsa); + thjs->setLargeSideDLIndex(inJsa.outSize() - 1); + + JobStepAssociation outJsa; + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + spdl->rowGroupDL(dl); + dl->OID(large); + outJsa.outAdd(spdl); + thjs->outputAssociation(outJsa); + + thjs->configSmallSideRG(smallSideRGs, tableNames); + thjs->configLargeSideRG(joinInfoMap[large]->fRowGroup); + thjs->configJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); + + tableInfoMap[large].fQuerySteps.push_back(spjs); + tableInfoMap[large].fDl = spdl; + } + else // thjs && joinStepMap[large].second >= 1 + { + JobStepAssociation inJsa = thjs->inputAssociation(); + + if (inJsa.outSize() < 2) + throw runtime_error("Not enough input to a hashjoin."); + + startPos = inJsa.outSize() - 1; + inJsa.outAdd(smallSideDLs, startPos); + thjs->inputAssociation(inJsa); + thjs->setLargeSideDLIndex(inJsa.outSize() - 1); + + thjs->addSmallSideRG(smallSideRGs, tableNames); + thjs->addJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); + } + + RowGroup rg; + set& tableSet = tableInfoMap[large].fJoinedTables; + constructJoinedRowGroup(rg, tableSet, tableInfoMap, jobInfo); + thjs->setOutputRowGroup(rg); + tableInfoMap[large].fRowGroup = rg; + tableSet.insert(large); + + if (jobInfo.trace) + { + cout << boldStart << "\n====== join info ======\n" << boldStop; + + for (vector::iterator t = traces.begin(); t != traces.end(); ++t) + cout << *t; + + cout << "RowGroup join result: " << endl << rg.toString() << endl << endl; + } + + // on clause filter association + map joinIdIndexMap; + + for (size_t i = 0; i < smallSides.size(); i++) + { + if (smallSides[i]->fJoinData.fJoinId > 0) + joinIdIndexMap[smallSides[i]->fJoinData.fJoinId] = i; + } + + // check if any cross-table expressions can be evaluated after the join + JobStepVector readyExpSteps; + JobStepVector& expSteps = jobInfo.crossTableExpressions; + JobStepVector::iterator eit = expSteps.begin(); + + while (eit != expSteps.end()) + { + ExpressionStep* exp = dynamic_cast(eit->get()); + + if (exp == NULL) + throw runtime_error("Not an expression."); + + if (exp->functionJoin()) + { + eit++; + continue; // done as join + } + + const vector& tables = exp->tableKeys(); + uint64_t i = 0; + + for (; i < tables.size(); i++) + { + if (tableInfoMap[large].fJoinedTables.find(tables[i]) == tableInfoMap[large].fJoinedTables.end()) + break; + } + + // all tables for this expression are joined + bool ready = (tables.size() == i); + + // for on clause condition, need check join ID + if (ready && exp->associatedJoinId() != 0) + { + map::iterator x = joinIdIndexMap.find(exp->associatedJoinId()); + ready = (x != joinIdIndexMap.end()); + } + + if (ready) + { + readyExpSteps.push_back(*eit); + eit = expSteps.erase(eit); + } + else + { + eit++; + } + } + + // if last join step, handle the delayed outer join filters + if (js == lastJoin && jobInfo.outerJoinExpressions.size() > 0) + readyExpSteps.insert(readyExpSteps.end(), jobInfo.outerJoinExpressions.begin(), + jobInfo.outerJoinExpressions.end()); + + // check additional compares for semi-join + if (readyExpSteps.size() > 0) + { + map keyToIndexMap; // map keys to the indices in the RG + + const auto& rowGroupKeys = rg.getKeys(); + for (uint64_t i = 0, e = rowGroupKeys.size(); i < e; ++i) + keyToIndexMap.insert(make_pair(rowGroupKeys[i], i)); + + // tables have additional comparisons + map correlateTables; // index in thjs + map correlateCompare; // expression + + for (size_t i = 0; i != smallSides.size(); i++) + { + if ((jointypes[i] & SEMI) || (jointypes[i] & ANTI) || (jointypes[i] & SCALAR)) { - JoinInfo* info = i->get(); - smallSideDLs.push_back(info->fDl); - smallSideRGs.push_back(info->fRowGroup); - jointypes.push_back(info->fJoinData.fTypes[0]); - typeless.push_back(info->fJoinData.fTypeless); - - vector smallIndices; - vector largeIndices; - const vector& keys1 = info->fJoinData.fLeftKeys; - const vector& keys2 = info->fJoinData.fRightKeys; - vector::const_iterator k1 = keys1.begin(); - vector::const_iterator k2 = keys2.begin(); - uint32_t stid = getTableKey(jobInfo, *k1); - tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[stid].fTable); - - for (; k1 != keys1.end(); ++k1, ++k2) - { - smallIndices.push_back(getKeyIndex(*k1, info->fRowGroup)); - largeIndices.push_back(getKeyIndex(*k2, largeSideRG)); - } - - smallKeyIndices.push_back(smallIndices); - largeKeyIndices.push_back(largeIndices); - - if (jobInfo.trace) - { - // small side column - ostringstream smallKey, smallIndex; - string alias1 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys1.front())]; - smallKey << alias1 << "-"; - - for (k1 = keys1.begin(); k1 != keys1.end(); ++k1) - { - CalpontSystemCatalog::OID oid1 = jobInfo.keyInfo->tupleKeyVec[*k1].fId; - CalpontSystemCatalog::TableColName tcn1 = jobInfo.csc->colName(oid1); - smallKey << "(" << tcn1.column << ":" << oid1 << ":" << *k1 << ")"; - smallIndex << " " << getKeyIndex(*k1, info->fRowGroup); - } - - // large side column - ostringstream largeKey, largeIndex; - string alias2 = jobInfo.keyInfo->keyName[getTableKey(jobInfo, keys2.front())]; - largeKey << alias2 << "-"; - - for (k2 = keys2.begin(); k2 != keys2.end(); ++k2) - { - CalpontSystemCatalog::OID oid2 = jobInfo.keyInfo->tupleKeyVec[*k2].fId; - CalpontSystemCatalog::TableColName tcn2 = jobInfo.csc->colName(oid2); - largeKey << "(" << tcn2.column << ":" << oid2 << ":" << *k2 << ")"; - largeIndex << " " << getKeyIndex(*k2, largeSideRG); - } - - ostringstream oss; - oss << smallKey.str() << " join on " << largeKey.str() - << " joinType: " << info->fJoinData.fTypes.front() - << "(" << joinTypeToString(info->fJoinData.fTypes.front()) << ")" - << (info->fJoinData.fTypeless ? " " : " !") << "typeless" << endl; - oss << "smallSideIndex-largeSideIndex :" << smallIndex.str() << " --" - << largeIndex.str() << endl; - oss << "small side RG" << endl << info->fRowGroup.toString() << endl; - traces.push_back(oss.str()); - } + uint32_t tid = getTableKey(jobInfo, smallSides[i]->fTableOid, smallSides[i]->fAlias, + smallSides[i]->fSchema, smallSides[i]->fView); + correlateTables[tid] = i; + correlateCompare[tid] = NULL; } + } - if (jobInfo.trace) + if (correlateTables.size() > 0) + { + // separate additional compare for each table pair + JobStepVector::iterator eit = readyExpSteps.begin(); + + while (eit != readyExpSteps.end()) { - ostringstream oss; - oss << "large side RG" << endl << largeSideRG.toString() << endl; - traces.push_back(oss.str()); + ExpressionStep* e = dynamic_cast(eit->get()); + + if (e->selectFilter()) + { + // @bug3780, leave select filter to normal expression + eit++; + continue; + } + + const vector& tables = e->tableKeys(); + map::iterator j = correlateTables.end(); + + for (size_t i = 0; i < tables.size(); i++) + { + j = correlateTables.find(tables[i]); + + if (j != correlateTables.end()) + break; + } + + if (j == correlateTables.end()) + { + eit++; + continue; + } + + // map the input column index + e->updateInputIndex(keyToIndexMap, jobInfo); + ParseTree* pt = correlateCompare[j->first]; + + if (pt == NULL) + { + // first expression + pt = new ParseTree; + pt->copyTree(*(e->expressionFilter())); + } + else + { + // combine the expressions + ParseTree* left = pt; + ParseTree* right = new ParseTree; + right->copyTree(*(e->expressionFilter())); + pt = new ParseTree(new LogicOperator("and")); + pt->left(left); + pt->right(right); + } + + correlateCompare[j->first] = pt; + eit = readyExpSteps.erase(eit); } - if (bps || tsas || umstream || (thjs && joinStepMap[large].second < 1)) + map::iterator k = correlateTables.begin(); + + while (k != correlateTables.end()) { - thjs = new TupleHashJoinStep(jobInfo); - thjs->tableOid1(smallSides[0]->fTableOid); - thjs->tableOid2(tableInfoMap[large].fTableOid); - thjs->alias1(smallSides[0]->fAlias); - thjs->alias2(tableInfoMap[large].fAlias); - thjs->view1(smallSides[0]->fView); - thjs->view2(tableInfoMap[large].fView); - thjs->schema1(smallSides[0]->fSchema); - thjs->schema2(tableInfoMap[large].fSchema); - thjs->setLargeSideBPS(bps); - thjs->joinId(lastJoinId); + ParseTree* pt = correlateCompare[k->first]; - if (dynamic_cast(bps) != NULL) - bps->incWaitToRunStepCnt(); + if (pt != NULL) + { + boost::shared_ptr sppt(pt); + thjs->addJoinFilter(sppt, startPos + k->second); + } - spjs.reset(thjs); - - JobStepAssociation inJsa; - inJsa.outAdd(smallSideDLs, 0); - inJsa.outAdd(joinInfoMap[large]->fDl); - thjs->inputAssociation(inJsa); - thjs->setLargeSideDLIndex(inJsa.outSize() - 1); - - JobStepAssociation outJsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - spdl->rowGroupDL(dl); - dl->OID(large); - outJsa.outAdd(spdl); - thjs->outputAssociation(outJsa); - - thjs->configSmallSideRG(smallSideRGs, tableNames); - thjs->configLargeSideRG(joinInfoMap[large]->fRowGroup); - thjs->configJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); - - tableInfoMap[large].fQuerySteps.push_back(spjs); - tableInfoMap[large].fDl = spdl; + k++; } - else // thjs && joinStepMap[large].second >= 1 + + thjs->setJoinFilterInputRG(rg); + } + + // normal expression if any + if (readyExpSteps.size() > 0) + { + // add the expression steps in where clause can be solved by this join to bps + ParseTree* pt = NULL; + JobStepVector::iterator eit = readyExpSteps.begin(); + + for (; eit != readyExpSteps.end(); eit++) { - JobStepAssociation inJsa = thjs->inputAssociation(); + // map the input column index + ExpressionStep* e = dynamic_cast(eit->get()); + e->updateInputIndex(keyToIndexMap, jobInfo); - if (inJsa.outSize() < 2) - throw runtime_error("Not enough input to a hashjoin."); + // short circuit on clause expressions + map::iterator x = joinIdIndexMap.find(e->associatedJoinId()); - startPos = inJsa.outSize() - 1; - inJsa.outAdd(smallSideDLs, startPos); - thjs->inputAssociation(inJsa); - thjs->setLargeSideDLIndex(inJsa.outSize() - 1); + if (x != joinIdIndexMap.end()) + { + ParseTree* joinFilter = new ParseTree; + joinFilter->copyTree(*(e->expressionFilter())); + boost::shared_ptr sppt(joinFilter); + thjs->addJoinFilter(sppt, startPos + x->second); + thjs->setJoinFilterInputRG(rg); + continue; + } - thjs->addSmallSideRG(smallSideRGs, tableNames); - thjs->addJoinKeyIndex(jointypes, typeless, smallKeyIndices, largeKeyIndices); + if (pt == NULL) + { + // first expression + pt = new ParseTree; + pt->copyTree(*(e->expressionFilter())); + } + else + { + // combine the expressions + ParseTree* left = pt; + ParseTree* right = new ParseTree; + right->copyTree(*(e->expressionFilter())); + pt = new ParseTree(new LogicOperator("and")); + pt->left(left); + pt->right(right); + } } - RowGroup rg; - set& tableSet = tableInfoMap[large].fJoinedTables; - constructJoinedRowGroup(rg, tableSet, tableInfoMap, jobInfo); + if (pt != NULL) + { + boost::shared_ptr sppt(pt); + thjs->addFcnExpGroup2(sppt); + } + } + + // update the fColsInExp2 and construct the output RG + updateExp2Cols(readyExpSteps, tableInfoMap, jobInfo); + constructJoinedRowGroup(rg, tableSet, tableInfoMap, jobInfo); + + if (thjs->hasFcnExpGroup2()) + thjs->setFE23Output(rg); + else thjs->setOutputRowGroup(rg); - tableInfoMap[large].fRowGroup = rg; - tableSet.insert(large); - if (jobInfo.trace) - { - cout << boldStart << "\n====== join info ======\n" << boldStop; + tableInfoMap[large].fRowGroup = rg; - for (vector::iterator t = traces.begin(); t != traces.end(); ++t) - cout << *t; - - cout << "RowGroup join result: " << endl << rg.toString() << endl << endl; - } - - // on clause filter association - map joinIdIndexMap; - - for (size_t i = 0; i < smallSides.size(); i++) - { - if (smallSides[i]->fJoinData.fJoinId > 0) - joinIdIndexMap[smallSides[i]->fJoinData.fJoinId] = i; - } - - // check if any cross-table expressions can be evaluated after the join - JobStepVector readyExpSteps; - JobStepVector& expSteps = jobInfo.crossTableExpressions; - JobStepVector::iterator eit = expSteps.begin(); - - while (eit != expSteps.end()) - { - ExpressionStep* exp = dynamic_cast(eit->get()); - - if (exp == NULL) - throw runtime_error("Not an expression."); - - if (exp->functionJoin()) - { - eit++; - continue; // done as join - } - - const vector& tables = exp->tableKeys(); - uint64_t i = 0; - - for (; i < tables.size(); i++) - { - if (tableInfoMap[large].fJoinedTables.find(tables[i]) == - tableInfoMap[large].fJoinedTables.end()) - break; - } - - // all tables for this expression are joined - bool ready = (tables.size() == i); - - // for on clause condition, need check join ID - if (ready && exp->associatedJoinId() != 0) - { - map::iterator x = joinIdIndexMap.find(exp->associatedJoinId()); - ready = (x != joinIdIndexMap.end()); - } - - if (ready) - { - readyExpSteps.push_back(*eit); - eit = expSteps.erase(eit); - } - else - { - eit++; - } - } - - // if last join step, handle the delayed outer join filters - if (js == lastJoin && jobInfo.outerJoinExpressions.size() > 0) - readyExpSteps.insert(readyExpSteps.end(), - jobInfo.outerJoinExpressions.begin(), - jobInfo.outerJoinExpressions.end()); - - // check additional compares for semi-join - if (readyExpSteps.size() > 0) - { - map keyToIndexMap; // map keys to the indices in the RG - - const auto& rowGroupKeys = rg.getKeys(); - for (uint64_t i = 0, e = rowGroupKeys.size(); i < e; ++i) - keyToIndexMap.insert(make_pair(rowGroupKeys[i], i)); - - // tables have additional comparisons - map correlateTables; // index in thjs - map correlateCompare; // expression - - for (size_t i = 0; i != smallSides.size(); i++) - { - if ((jointypes[i] & SEMI) || (jointypes[i] & ANTI) || (jointypes[i] & SCALAR)) - { - uint32_t tid = getTableKey(jobInfo, - smallSides[i]->fTableOid, - smallSides[i]->fAlias, - smallSides[i]->fSchema, - smallSides[i]->fView); - correlateTables[tid] = i; - correlateCompare[tid] = NULL; - } - } - - if (correlateTables.size() > 0) - { - // separate additional compare for each table pair - JobStepVector::iterator eit = readyExpSteps.begin(); - - while (eit != readyExpSteps.end()) - { - ExpressionStep* e = dynamic_cast(eit->get()); - - if (e->selectFilter()) - { - // @bug3780, leave select filter to normal expression - eit++; - continue; - } - - const vector& tables = e->tableKeys(); - map::iterator j = correlateTables.end(); - - for (size_t i = 0; i < tables.size(); i++) - { - j = correlateTables.find(tables[i]); - - if (j != correlateTables.end()) - break; - } - - if (j == correlateTables.end()) - { - eit++; - continue; - } - - // map the input column index - e->updateInputIndex(keyToIndexMap, jobInfo); - ParseTree* pt = correlateCompare[j->first]; - - if (pt == NULL) - { - // first expression - pt = new ParseTree; - pt->copyTree(*(e->expressionFilter())); - } - else - { - // combine the expressions - ParseTree* left = pt; - ParseTree* right = new ParseTree; - right->copyTree(*(e->expressionFilter())); - pt = new ParseTree(new LogicOperator("and")); - pt->left(left); - pt->right(right); - } - - correlateCompare[j->first] = pt; - eit = readyExpSteps.erase(eit); - } - - map::iterator k = correlateTables.begin(); - - while (k != correlateTables.end()) - { - ParseTree* pt = correlateCompare[k->first]; - - if (pt != NULL) - { - boost::shared_ptr sppt(pt); - thjs->addJoinFilter(sppt, startPos + k->second); - } - - k++; - } - - thjs->setJoinFilterInputRG(rg); - } - - // normal expression if any - if (readyExpSteps.size() > 0) - { - // add the expression steps in where clause can be solved by this join to bps - ParseTree* pt = NULL; - JobStepVector::iterator eit = readyExpSteps.begin(); - - for (; eit != readyExpSteps.end(); eit++) - { - // map the input column index - ExpressionStep* e = dynamic_cast(eit->get()); - e->updateInputIndex(keyToIndexMap, jobInfo); - - // short circuit on clause expressions - map::iterator x = joinIdIndexMap.find(e->associatedJoinId()); - - if (x != joinIdIndexMap.end()) - { - ParseTree* joinFilter = new ParseTree; - joinFilter->copyTree(*(e->expressionFilter())); - boost::shared_ptr sppt(joinFilter); - thjs->addJoinFilter(sppt, startPos + x->second); - thjs->setJoinFilterInputRG(rg); - continue; - } - - if (pt == NULL) - { - // first expression - pt = new ParseTree; - pt->copyTree(*(e->expressionFilter())); - } - else - { - // combine the expressions - ParseTree* left = pt; - ParseTree* right = new ParseTree; - right->copyTree(*(e->expressionFilter())); - pt = new ParseTree(new LogicOperator("and")); - pt->left(left); - pt->right(right); - } - } - - if (pt != NULL) - { - boost::shared_ptr sppt(pt); - thjs->addFcnExpGroup2(sppt); - } - } - - // update the fColsInExp2 and construct the output RG - updateExp2Cols(readyExpSteps, tableInfoMap, jobInfo); - constructJoinedRowGroup(rg, tableSet, tableInfoMap, jobInfo); - - if (thjs->hasFcnExpGroup2()) - thjs->setFE23Output(rg); - else - thjs->setOutputRowGroup(rg); - - tableInfoMap[large].fRowGroup = rg; - - if (jobInfo.trace) - { - cout << "RowGroup of " << tableInfoMap[large].fAlias << " after EXP G2: " << endl - << rg.toString() << endl << endl; - } - } - - // update the info maps - int l = (joinStepMap[large].second == 2) ? 2 : 0; - - if (isSemijoin) - joinStepMap[large] = make_pair(spjs, joinStepMap[large].second); - else - joinStepMap[large] = make_pair(spjs, l); - - for (set::iterator i = tableSet.begin(); i != tableSet.end(); i++) - { - joinInfoMap[*i]->fDl = tableInfoMap[large].fDl; - joinInfoMap[*i]->fRowGroup = tableInfoMap[large].fRowGroup; - - if (*i != large) - { - //@bug6117, token should be done for small side tables. - SJSTEP smallJs = joinStepMap[*i].first; - TupleHashJoinStep* smallThjs = dynamic_cast(smallJs.get()); - - if (smallThjs && smallThjs->tokenJoin()) - smallThjs->tokenJoin(-1); - - // Set join priority for smallsides. - joinStepMap[*i] = make_pair(spjs, l); - - // Mark joined tables, smalls and large, as a group. - tableInfoMap[*i].fJoinedTables = tableInfoMap[large].fJoinedTables; - } - } - - prevLarge = large; + if (jobInfo.trace) + { + cout << "RowGroup of " << tableInfoMap[large].fAlias << " after EXP G2: " << endl + << rg.toString() << endl + << endl; + } } - // Keep join order by the table last used for picking the right delivery step. + // update the info maps + int l = (joinStepMap[large].second == 2) ? 2 : 0; + + if (isSemijoin) + joinStepMap[large] = make_pair(spjs, joinStepMap[large].second); + else + joinStepMap[large] = make_pair(spjs, l); + + for (set::iterator i = tableSet.begin(); i != tableSet.end(); i++) { - for (vector::reverse_iterator i = joinedTable.rbegin(); i < joinedTable.rend(); i++) - { - if (find(joinOrder.begin(), joinOrder.end(), *i) == joinOrder.end()) - joinOrder.push_back(*i); - } + joinInfoMap[*i]->fDl = tableInfoMap[large].fDl; + joinInfoMap[*i]->fRowGroup = tableInfoMap[large].fRowGroup; - const uint64_t n = joinOrder.size(); - const uint64_t h = n / 2; - const uint64_t e = n - 1; + if (*i != large) + { + //@bug6117, token should be done for small side tables. + SJSTEP smallJs = joinStepMap[*i].first; + TupleHashJoinStep* smallThjs = dynamic_cast(smallJs.get()); - for (uint64_t i = 0; i < h; i++) - std::swap(joinOrder[i], joinOrder[e - i]); + if (smallThjs && smallThjs->tokenJoin()) + smallThjs->tokenJoin(-1); + + // Set join priority for smallsides. + joinStepMap[*i] = make_pair(spjs, l); + + // Mark joined tables, smalls and large, as a group. + tableInfoMap[*i].fJoinedTables = tableInfoMap[large].fJoinedTables; + } } + + prevLarge = large; + } + + // Keep join order by the table last used for picking the right delivery step. + { + for (vector::reverse_iterator i = joinedTable.rbegin(); i < joinedTable.rend(); i++) + { + if (find(joinOrder.begin(), joinOrder.end(), *i) == joinOrder.end()) + joinOrder.push_back(*i); + } + + const uint64_t n = joinOrder.size(); + const uint64_t h = n / 2; + const uint64_t e = n - 1; + + for (uint64_t i = 0; i < h; i++) + std::swap(joinOrder[i], joinOrder[e - i]); + } } inline void joinTables(JobStepVector& joinSteps, TableInfoMap& tableInfoMap, JobInfo& jobInfo, vector& joinOrder, JoinEdges& edgesToTransform, const bool overrideLargeSideEstimate) { - uint32_t largestTable = - getLargestTable(jobInfo, tableInfoMap, overrideLargeSideEstimate); + uint32_t largestTable = getLargestTable(jobInfo, tableInfoMap, overrideLargeSideEstimate); - if (jobInfo.outerOnTable.size() == 0) - joinToLargeTable(largestTable, tableInfoMap, jobInfo, joinOrder, edgesToTransform); - else - joinTablesInOrder(largestTable, joinSteps, tableInfoMap, jobInfo, joinOrder); + if (jobInfo.outerOnTable.size() == 0) + joinToLargeTable(largestTable, tableInfoMap, jobInfo, joinOrder, edgesToTransform); + else + joinTablesInOrder(largestTable, joinSteps, tableInfoMap, jobInfo, joinOrder); } - void makeNoTableJobStep(JobStepVector& querySteps, JobStepVector& projectSteps, DeliveredTableMap& deliverySteps, JobInfo& jobInfo) { - querySteps.clear(); - projectSteps.clear(); - deliverySteps.clear(); - querySteps.push_back(TupleConstantStep::addConstantStep(jobInfo)); - deliverySteps[CNX_VTABLE_ID] = querySteps.back(); + querySteps.clear(); + projectSteps.clear(); + deliverySteps.clear(); + querySteps.push_back(TupleConstantStep::addConstantStep(jobInfo)); + deliverySteps[CNX_VTABLE_ID] = querySteps.back(); } -} - - +} // namespace namespace joblist { @@ -3854,784 +3788,779 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte DeliveredTableMap& deliverySteps, JobInfo& jobInfo, const bool overrideLargeSideEstimate) { - if (jobInfo.trace) + if (jobInfo.trace) + { + const boost::shared_ptr& keyInfo = jobInfo.keyInfo; + cout << "query steps:" << endl; + + for (JobStepVector::iterator i = querySteps.begin(); i != querySteps.end(); ++i) { - const boost::shared_ptr& keyInfo = jobInfo.keyInfo; - cout << "query steps:" << endl; + TupleHashJoinStep* thjs = dynamic_cast(i->get()); - for (JobStepVector::iterator i = querySteps.begin(); i != querySteps.end(); ++i) - { - TupleHashJoinStep* thjs = dynamic_cast(i->get()); - - if (thjs == NULL) - { - int64_t id = ((*i)->tupleId() != (uint64_t) - 1) ? (*i)->tupleId() : -1; - cout << typeid(*(i->get())).name() << ": " << (*i)->oid() << " " << id << " " - << (int)((id != -1) ? getTableKey(jobInfo, id) : -1) << endl; - } - else - { - int64_t id1 = (thjs->tupleId1() != (uint64_t) - 1) ? thjs->tupleId1() : -1; - int64_t id2 = (thjs->tupleId2() != (uint64_t) - 1) ? thjs->tupleId2() : -1; - cout << typeid(*thjs).name() << ": " << thjs->oid1() << " " << id1 << " " - << (int)((id1 != -1) ? getTableKey(jobInfo, id1) : -1) << " - " - << thjs->getJoinType() << " - " << thjs->oid2() << " " << id2 << " " - << (int)((id2 != -1) ? getTableKey(jobInfo, id2) : -1) << endl; - } - } - - cout << "project steps:" << endl; - - for (JobStepVector::iterator i = projectSteps.begin(); i != projectSteps.end(); ++i) - { - cout << typeid(*(i->get())).name() << ": " << (*i)->oid() << " " - << (*i)->tupleId() << " " << getTableKey(jobInfo, (*i)->tupleId()) << endl; - } - - cout << "delivery steps:" << endl; - - for (DeliveredTableMap::iterator i = deliverySteps.begin(); i != deliverySteps.end(); ++i) - cout << typeid(*(i->second.get())).name() << endl; - - cout << "\nTable Info: (key oid name alias view sub)" << endl; - - for (uint32_t i = 0; i < keyInfo->tupleKeyVec.size(); ++i) - { - int64_t tid = jobInfo.keyInfo->colKeyToTblKey[i]; - - if (tid == i) - { - CalpontSystemCatalog::OID oid = keyInfo->tupleKeyVec[i].fId; - string alias = keyInfo->tupleKeyVec[i].fTable; - - if (alias.length() < 1) alias = "N/A"; - - string name = keyInfo->keyName[i]; - - if (name.empty()) name = "unknown"; - - string view = keyInfo->tupleKeyVec[i].fView; - - if (view.length() < 1) view = "N/A"; - - int sid = keyInfo->tupleKeyVec[i].fSubId; - cout << i << "\t" << oid << "\t" << name << "\t" << alias - << "\t" << view << "\t" << hex << sid << dec << endl; - } - } - - cout << "\nTupleKey vector: (tupleKey oid tableKey name alias view sub)" << endl; - - for (uint32_t i = 0; i < keyInfo->tupleKeyVec.size(); ++i) - { - CalpontSystemCatalog::OID oid = keyInfo->tupleKeyVec[i].fId; - int64_t tid = jobInfo.keyInfo->colKeyToTblKey[i]; - string alias = keyInfo->tupleKeyVec[i].fTable; - - if (alias.length() < 1) alias = "N/A"; - - // Expression IDs are borrowed from systemcatalog IDs, which are not used in tuple. - string name = keyInfo->keyName[i]; - - if (keyInfo->dictOidToColOid.find(oid) != keyInfo->dictOidToColOid.end()) - { - name += "[d]"; // indicate this is a dictionary column - } - - if (jobInfo.keyInfo->pseudoType[i] > 0) - { - name += "[p]"; // indicate this is a pseudo column - } - - if (name.empty()) - { - name = "unknown"; - } - - string view = keyInfo->tupleKeyVec[i].fView; - - if (view.length() < 1) view = "N/A"; - - int sid = keyInfo->tupleKeyVec[i].fSubId; - cout << i << "\t" << oid << "\t" << tid << "\t" << name << "\t" << alias - << "\t" << view << "\t" << hex << sid << dec << endl; - } - - cout << endl; + if (thjs == NULL) + { + int64_t id = ((*i)->tupleId() != (uint64_t)-1) ? (*i)->tupleId() : -1; + cout << typeid(*(i->get())).name() << ": " << (*i)->oid() << " " << id << " " + << (int)((id != -1) ? getTableKey(jobInfo, id) : -1) << endl; + } + else + { + int64_t id1 = (thjs->tupleId1() != (uint64_t)-1) ? thjs->tupleId1() : -1; + int64_t id2 = (thjs->tupleId2() != (uint64_t)-1) ? thjs->tupleId2() : -1; + cout << typeid(*thjs).name() << ": " << thjs->oid1() << " " << id1 << " " + << (int)((id1 != -1) ? getTableKey(jobInfo, id1) : -1) << " - " << thjs->getJoinType() << " - " + << thjs->oid2() << " " << id2 << " " << (int)((id2 != -1) ? getTableKey(jobInfo, id2) : -1) + << endl; + } } + cout << "project steps:" << endl; - // @bug 2771, handle no table select query - if (jobInfo.tableList.size() < 1) + for (JobStepVector::iterator i = projectSteps.begin(); i != projectSteps.end(); ++i) { - makeNoTableJobStep(querySteps, projectSteps, deliverySteps, jobInfo); - return; + cout << typeid(*(i->get())).name() << ": " << (*i)->oid() << " " << (*i)->tupleId() << " " + << getTableKey(jobInfo, (*i)->tupleId()) << endl; } - // Create a step vector for each table in the from clause. - TableInfoMap tableInfoMap; + cout << "delivery steps:" << endl; - for (uint64_t i = 0; i < jobInfo.tableList.size(); i++) + for (DeliveredTableMap::iterator i = deliverySteps.begin(); i != deliverySteps.end(); ++i) + cout << typeid(*(i->second.get())).name() << endl; + + cout << "\nTable Info: (key oid name alias view sub)" << endl; + + for (uint32_t i = 0; i < keyInfo->tupleKeyVec.size(); ++i) { - uint32_t tableUid = jobInfo.tableList[i]; - tableInfoMap[tableUid] = TableInfo(); - tableInfoMap[tableUid].fTableOid = jobInfo.keyInfo->tupleKeyVec[tableUid].fId; - tableInfoMap[tableUid].fName = jobInfo.keyInfo->keyName[tableUid]; - tableInfoMap[tableUid].fAlias = jobInfo.keyInfo->tupleKeyVec[tableUid].fTable; - tableInfoMap[tableUid].fView = jobInfo.keyInfo->tupleKeyVec[tableUid].fView; - tableInfoMap[tableUid].fSchema = jobInfo.keyInfo->tupleKeyVec[tableUid].fSchema; - tableInfoMap[tableUid].fSubId = jobInfo.keyInfo->tupleKeyVec[tableUid].fSubId; - tableInfoMap[tableUid].fColsInColMap = jobInfo.columnMap[tableUid]; + int64_t tid = jobInfo.keyInfo->colKeyToTblKey[i]; + + if (tid == i) + { + CalpontSystemCatalog::OID oid = keyInfo->tupleKeyVec[i].fId; + string alias = keyInfo->tupleKeyVec[i].fTable; + + if (alias.length() < 1) + alias = "N/A"; + + string name = keyInfo->keyName[i]; + + if (name.empty()) + name = "unknown"; + + string view = keyInfo->tupleKeyVec[i].fView; + + if (view.length() < 1) + view = "N/A"; + + int sid = keyInfo->tupleKeyVec[i].fSubId; + cout << i << "\t" << oid << "\t" << name << "\t" << alias << "\t" << view << "\t" << hex << sid << dec + << endl; + } } - // Set of the columns being projected. - for (TupleInfoVector::iterator i = jobInfo.pjColList.begin(); i != jobInfo.pjColList.end(); i++) - jobInfo.returnColSet.insert(i->key); + cout << "\nTupleKey vector: (tupleKey oid tableKey name alias view sub)" << endl; - // Strip constantbooleanquerySteps - for (uint64_t i = 0; i < querySteps.size(); ) + for (uint32_t i = 0; i < keyInfo->tupleKeyVec.size(); ++i) { - TupleConstantBooleanStep* bs = dynamic_cast(querySteps[i].get()); - ExpressionStep* es = dynamic_cast(querySteps[i].get()); + CalpontSystemCatalog::OID oid = keyInfo->tupleKeyVec[i].fId; + int64_t tid = jobInfo.keyInfo->colKeyToTblKey[i]; + string alias = keyInfo->tupleKeyVec[i].fTable; - if (bs != NULL) + if (alias.length() < 1) + alias = "N/A"; + + // Expression IDs are borrowed from systemcatalog IDs, which are not used in tuple. + string name = keyInfo->keyName[i]; + + if (keyInfo->dictOidToColOid.find(oid) != keyInfo->dictOidToColOid.end()) + { + name += "[d]"; // indicate this is a dictionary column + } + + if (jobInfo.keyInfo->pseudoType[i] > 0) + { + name += "[p]"; // indicate this is a pseudo column + } + + if (name.empty()) + { + name = "unknown"; + } + + string view = keyInfo->tupleKeyVec[i].fView; + + if (view.length() < 1) + view = "N/A"; + + int sid = keyInfo->tupleKeyVec[i].fSubId; + cout << i << "\t" << oid << "\t" << tid << "\t" << name << "\t" << alias << "\t" << view << "\t" << hex + << sid << dec << endl; + } + + cout << endl; + } + + // @bug 2771, handle no table select query + if (jobInfo.tableList.size() < 1) + { + makeNoTableJobStep(querySteps, projectSteps, deliverySteps, jobInfo); + return; + } + + // Create a step vector for each table in the from clause. + TableInfoMap tableInfoMap; + + for (uint64_t i = 0; i < jobInfo.tableList.size(); i++) + { + uint32_t tableUid = jobInfo.tableList[i]; + tableInfoMap[tableUid] = TableInfo(); + tableInfoMap[tableUid].fTableOid = jobInfo.keyInfo->tupleKeyVec[tableUid].fId; + tableInfoMap[tableUid].fName = jobInfo.keyInfo->keyName[tableUid]; + tableInfoMap[tableUid].fAlias = jobInfo.keyInfo->tupleKeyVec[tableUid].fTable; + tableInfoMap[tableUid].fView = jobInfo.keyInfo->tupleKeyVec[tableUid].fView; + tableInfoMap[tableUid].fSchema = jobInfo.keyInfo->tupleKeyVec[tableUid].fSchema; + tableInfoMap[tableUid].fSubId = jobInfo.keyInfo->tupleKeyVec[tableUid].fSubId; + tableInfoMap[tableUid].fColsInColMap = jobInfo.columnMap[tableUid]; + } + + // Set of the columns being projected. + for (TupleInfoVector::iterator i = jobInfo.pjColList.begin(); i != jobInfo.pjColList.end(); i++) + jobInfo.returnColSet.insert(i->key); + + // Strip constantbooleanquerySteps + for (uint64_t i = 0; i < querySteps.size();) + { + TupleConstantBooleanStep* bs = dynamic_cast(querySteps[i].get()); + ExpressionStep* es = dynamic_cast(querySteps[i].get()); + + if (bs != NULL) + { + // cosntant step + if (bs->boolValue() == false) + jobInfo.constantFalse = true; + + querySteps.erase(querySteps.begin() + i); + } + else if (es != NULL && es->tableKeys().size() == 0) + { + // constant expression + ParseTree* p = es->expressionFilter(); // filter + + if (p != NULL) + { + Row r; // dummy row + + if (funcexp::FuncExp::instance()->evaluate(r, p) == false) + jobInfo.constantFalse = true; + + querySteps.erase(querySteps.begin() + i); + } + } + else + { + i++; + } + } + + // double check if the function join canditates are still there. + JobStepVector steps = querySteps; + + for (int64_t i = jobInfo.functionJoins.size() - 1; i >= 0; i--) + { + bool exist = false; + + for (JobStepVector::iterator j = steps.begin(); j != steps.end() && !exist; ++j) + { + if (jobInfo.functionJoins[i] == j->get()) + exist = true; + } + + if (!exist) + jobInfo.functionJoins.erase(jobInfo.functionJoins.begin() + i); + } + + // Concatenate query and project steps + steps.insert(steps.end(), projectSteps.begin(), projectSteps.end()); + + // Make sure each query step has an output DL + // This is necessary for toString() method on most steps + for (JobStepVector::iterator it = steps.begin(); it != steps.end(); ++it) + { + // if (dynamic_cast(it->get())) + // continue; + + if (it->get()->outputAssociation().outSize() == 0) + { + JobStepAssociation jsa; + AnyDataListSPtr adl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + dl->OID(it->get()->oid()); + adl->rowGroupDL(dl); + jsa.outAdd(adl); + it->get()->outputAssociation(jsa); + } + } + + // Populate the TableInfo map with the job steps keyed by table ID. + JobStepVector joinSteps; + JobStepVector& expSteps = jobInfo.crossTableExpressions; + JobStepVector::iterator it = querySteps.begin(); + JobStepVector::iterator end = querySteps.end(); + + while (it != end) + { + // Separate table joins from other predicates. + TupleHashJoinStep* thjs = dynamic_cast(it->get()); + ExpressionStep* exps = dynamic_cast(it->get()); + SubAdapterStep* subs = dynamic_cast(it->get()); + + if (thjs != NULL && thjs->tupleId1() != thjs->tupleId2()) + { + // simple column and constant column semi join + if (thjs->tableOid2() == 0 && thjs->schema2().empty()) + { + jobInfo.correlateSteps.push_back(*it++); + continue; + } + + // check correlated join step + JoinType joinType = thjs->getJoinType(); + + if (joinType & CORRELATED) + { + // one of the tables is in outer query + jobInfo.correlateSteps.push_back(*it++); + continue; + } + + // Save the join topology. + uint32_t key1 = thjs->tupleId1(); + uint32_t key2 = thjs->tupleId2(); + uint32_t tid1 = getTableKey(jobInfo, key1); + uint32_t tid2 = getTableKey(jobInfo, key2); + + if (thjs->dictOid1() > 0) + key1 = jobInfo.keyInfo->dictKeyMap[key1]; + + if (thjs->dictOid2() > 0) + key2 = jobInfo.keyInfo->dictKeyMap[key2]; + + // not correlated + joinSteps.push_back(*it); + tableInfoMap[tid1].fJoinKeys.push_back(key1); + tableInfoMap[tid2].fJoinKeys.push_back(key2); + + // save the function join expressions + boost::shared_ptr fji = thjs->funcJoinInfo(); + + if (fji) + { + if (fji->fStep[0]) { - // cosntant step - if (bs->boolValue() == false) - jobInfo.constantFalse = true; - - querySteps.erase(querySteps.begin() + i); + tableInfoMap[tid1].fFuncJoinExps.push_back(fji->fStep[0]); + vector& cols = tableInfoMap[tid1].fColsInFuncJoin; + cols.insert(cols.end(), fji->fColumnKeys[0].begin(), fji->fColumnKeys[0].end()); } - else if (es != NULL && es->tableKeys().size() == 0) + + if (fji->fStep[1]) { - // constant expression - ParseTree* p = es->expressionFilter(); // filter + tableInfoMap[tid2].fFuncJoinExps.push_back(fji->fStep[1]); + vector& cols = tableInfoMap[tid2].fColsInFuncJoin; + cols.insert(cols.end(), fji->fColumnKeys[1].begin(), fji->fColumnKeys[1].end()); + } + } - if (p != NULL) - { - Row r; // dummy row + // keep a join map + pair tablePair(tid1, tid2); + TableJoinMap::iterator m1 = jobInfo.tableJoinMap.find(tablePair); + TableJoinMap::iterator m2 = jobInfo.tableJoinMap.end(); - if (funcexp::FuncExp::instance()->evaluate(r, p) == false) - jobInfo.constantFalse = true; + if (m1 == jobInfo.tableJoinMap.end()) + { + tableInfoMap[tid1].fAdjacentList.push_back(tid2); + tableInfoMap[tid2].fAdjacentList.push_back(tid1); - querySteps.erase(querySteps.begin() + i); - } + m1 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid1, tid2), JoinData())); + m2 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid2, tid1), JoinData())); + + TupleInfo ti1(getTupleInfo(key1, jobInfo)); + TupleInfo ti2(getTupleInfo(key2, jobInfo)); + + if (ti1.width > 8 || ti2.width > 8) + { + if (ti1.dtype == execplan::CalpontSystemCatalog::LONGDOUBLE || + ti2.dtype == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + m1->second.fTypeless = m2->second.fTypeless = false; + } + else + { + m1->second.fTypeless = m2->second.fTypeless = true; + } } else { - i++; + m1->second.fTypeless = m2->second.fTypeless = isCharType(ti1.dtype) || isCharType(ti2.dtype); } - } + } + else + { + m2 = jobInfo.tableJoinMap.find(make_pair(tid2, tid1)); + m1->second.fTypeless = m2->second.fTypeless = true; + } - // double check if the function join canditates are still there. - JobStepVector steps = querySteps; + if (m1 == jobInfo.tableJoinMap.end() || m2 == jobInfo.tableJoinMap.end()) + throw runtime_error("Bad table map."); - for (int64_t i = jobInfo.functionJoins.size() - 1; i >= 0; i--) - { - bool exist = false; + // Keep a map of the join (table, key) pairs + m1->second.fLeftKeys.push_back(key1); + m1->second.fRightKeys.push_back(key2); - for (JobStepVector::iterator j = steps.begin(); j != steps.end() && !exist; ++j) + m2->second.fLeftKeys.push_back(key2); + m2->second.fRightKeys.push_back(key1); + + // Keep a map of the join type between the keys. + // OUTER join and SEMI/ANTI join are mutually exclusive. + if (joinType == LEFTOUTER) + { + m1->second.fTypes.push_back(SMALLOUTER); + m2->second.fTypes.push_back(LARGEOUTER); + jobInfo.outerOnTable.insert(tid2); + } + else if (joinType == RIGHTOUTER) + { + m1->second.fTypes.push_back(LARGEOUTER); + m2->second.fTypes.push_back(SMALLOUTER); + jobInfo.outerOnTable.insert(tid1); + } + else if ((joinType & SEMI) && + ((joinType & LEFTOUTER) == LEFTOUTER || (joinType & RIGHTOUTER) == RIGHTOUTER)) + { + // @bug3998, DML UPDATE borrows "SEMI" flag, + // allowing SEMI and LARGEOUTER combination to support update with outer join. + if ((joinType & LEFTOUTER) == LEFTOUTER) { - if (jobInfo.functionJoins[i] == j->get()) - exist = true; + joinType ^= LEFTOUTER; + m1->second.fTypes.push_back(joinType); + m2->second.fTypes.push_back(joinType | LARGEOUTER); + jobInfo.outerOnTable.insert(tid2); } - - if (!exist) - jobInfo.functionJoins.erase(jobInfo.functionJoins.begin() + i); - } - - // Concatenate query and project steps - steps.insert(steps.end(), projectSteps.begin(), projectSteps.end()); - - // Make sure each query step has an output DL - // This is necessary for toString() method on most steps - for (JobStepVector::iterator it = steps.begin(); it != steps.end(); ++it) - { - //if (dynamic_cast(it->get())) - // continue; - - if (it->get()->outputAssociation().outSize() == 0) - { - JobStepAssociation jsa; - AnyDataListSPtr adl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(it->get()->oid()); - adl->rowGroupDL(dl); - jsa.outAdd(adl); - it->get()->outputAssociation(jsa); - } - } - - // Populate the TableInfo map with the job steps keyed by table ID. - JobStepVector joinSteps; - JobStepVector& expSteps = jobInfo.crossTableExpressions; - JobStepVector::iterator it = querySteps.begin(); - JobStepVector::iterator end = querySteps.end(); - - while (it != end) - { - // Separate table joins from other predicates. - TupleHashJoinStep* thjs = dynamic_cast(it->get()); - ExpressionStep* exps = dynamic_cast(it->get()); - SubAdapterStep* subs = dynamic_cast(it->get()); - - if (thjs != NULL && thjs->tupleId1() != thjs->tupleId2()) - { - // simple column and constant column semi join - if (thjs->tableOid2() == 0 && thjs->schema2().empty()) - { - jobInfo.correlateSteps.push_back(*it++); - continue; - } - - // check correlated join step - JoinType joinType = thjs->getJoinType(); - - if (joinType & CORRELATED) - { - // one of the tables is in outer query - jobInfo.correlateSteps.push_back(*it++); - continue; - } - - // Save the join topology. - uint32_t key1 = thjs->tupleId1(); - uint32_t key2 = thjs->tupleId2(); - uint32_t tid1 = getTableKey(jobInfo, key1); - uint32_t tid2 = getTableKey(jobInfo, key2); - - if (thjs->dictOid1() > 0) - key1 = jobInfo.keyInfo->dictKeyMap[key1]; - - if (thjs->dictOid2() > 0) - key2 = jobInfo.keyInfo->dictKeyMap[key2]; - - // not correlated - joinSteps.push_back(*it); - tableInfoMap[tid1].fJoinKeys.push_back(key1); - tableInfoMap[tid2].fJoinKeys.push_back(key2); - - // save the function join expressions - boost::shared_ptr fji = thjs->funcJoinInfo(); - - if (fji) - { - if (fji->fStep[0]) - { - tableInfoMap[tid1].fFuncJoinExps.push_back(fji->fStep[0]); - vector& cols = tableInfoMap[tid1].fColsInFuncJoin; - cols.insert(cols.end(), fji->fColumnKeys[0].begin(), fji->fColumnKeys[0].end()); - } - - if (fji->fStep[1]) - { - tableInfoMap[tid2].fFuncJoinExps.push_back(fji->fStep[1]); - vector& cols = tableInfoMap[tid2].fColsInFuncJoin; - cols.insert(cols.end(), fji->fColumnKeys[1].begin(), fji->fColumnKeys[1].end()); - } - } - - // keep a join map - pair tablePair(tid1, tid2); - TableJoinMap::iterator m1 = jobInfo.tableJoinMap.find(tablePair); - TableJoinMap::iterator m2 = jobInfo.tableJoinMap.end(); - - if (m1 == jobInfo.tableJoinMap.end()) - { - tableInfoMap[tid1].fAdjacentList.push_back(tid2); - tableInfoMap[tid2].fAdjacentList.push_back(tid1); - - m1 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid1, tid2), JoinData())); - m2 = jobInfo.tableJoinMap.insert(m1, make_pair(make_pair(tid2, tid1), JoinData())); - - TupleInfo ti1(getTupleInfo(key1, jobInfo)); - TupleInfo ti2(getTupleInfo(key2, jobInfo)); - - if (ti1.width > 8 || ti2.width > 8) - { - if (ti1.dtype == execplan::CalpontSystemCatalog::LONGDOUBLE - || ti2.dtype == execplan::CalpontSystemCatalog::LONGDOUBLE) - { - m1->second.fTypeless = m2->second.fTypeless = false; - } - else - { - m1->second.fTypeless = m2->second.fTypeless = true; - } - } - else - { - m1->second.fTypeless = m2->second.fTypeless = isCharType(ti1.dtype) || - isCharType(ti2.dtype); - } - } - else - { - m2 = jobInfo.tableJoinMap.find(make_pair(tid2, tid1)); - m1->second.fTypeless = m2->second.fTypeless = true; - } - - if (m1 == jobInfo.tableJoinMap.end() || m2 == jobInfo.tableJoinMap.end()) - throw runtime_error("Bad table map."); - - // Keep a map of the join (table, key) pairs - m1->second.fLeftKeys.push_back(key1); - m1->second.fRightKeys.push_back(key2); - - m2->second.fLeftKeys.push_back(key2); - m2->second.fRightKeys.push_back(key1); - - // Keep a map of the join type between the keys. - // OUTER join and SEMI/ANTI join are mutually exclusive. - if (joinType == LEFTOUTER) - { - m1->second.fTypes.push_back(SMALLOUTER); - m2->second.fTypes.push_back(LARGEOUTER); - jobInfo.outerOnTable.insert(tid2); - } - else if (joinType == RIGHTOUTER) - { - m1->second.fTypes.push_back(LARGEOUTER); - m2->second.fTypes.push_back(SMALLOUTER); - jobInfo.outerOnTable.insert(tid1); - } - else if ((joinType & SEMI) && - ((joinType & LEFTOUTER) == LEFTOUTER || (joinType & RIGHTOUTER) == RIGHTOUTER)) - { - // @bug3998, DML UPDATE borrows "SEMI" flag, - // allowing SEMI and LARGEOUTER combination to support update with outer join. - if ((joinType & LEFTOUTER) == LEFTOUTER) - { - joinType ^= LEFTOUTER; - m1->second.fTypes.push_back(joinType); - m2->second.fTypes.push_back(joinType | LARGEOUTER); - jobInfo.outerOnTable.insert(tid2); - } - else - { - joinType ^= RIGHTOUTER; - m1->second.fTypes.push_back(joinType | LARGEOUTER); - m2->second.fTypes.push_back(joinType); - jobInfo.outerOnTable.insert(tid1); - } - } - else - { - m1->second.fTypes.push_back(joinType); - m2->second.fTypes.push_back(joinType); - } - - // need id to keep the join order - m1->second.fJoinId = m2->second.fJoinId = thjs->joinId(); - } - // Separate the expressions - else if (exps != NULL && subs == NULL) - { - const vector& tables = exps->tableKeys(); - const vector& columns = exps->columnKeys(); - bool tableInOuterQuery = false; - set tableSet; // involved unique tables - - for (uint64_t i = 0; i < tables.size(); ++i) - { - if (find(jobInfo.tableList.begin(), jobInfo.tableList.end(), tables[i]) != - jobInfo.tableList.end()) - tableSet.insert(tables[i]); - else - tableInOuterQuery = true; - } - - if (tableInOuterQuery) - { - // all columns in subquery scope to be projected - for (uint64_t i = 0; i < tables.size(); ++i) - { - // outer-query columns - if (tableSet.find(tables[i]) == tableSet.end()) - continue; - - // subquery columns - uint32_t c = columns[i]; - - if (jobInfo.returnColSet.find(c) == jobInfo.returnColSet.end()) - { - tableInfoMap[tables[i]].fProjectCols.push_back(c); - jobInfo.pjColList.push_back(getTupleInfo(c, jobInfo)); - jobInfo.returnColSet.insert(c); - const SimpleColumn* sc = - dynamic_cast(exps->columns()[i]); - - if (sc != NULL) - jobInfo.deliveredCols.push_back(SRCP(sc->clone())); - } - } - - jobInfo.correlateSteps.push_back(*it++); - continue; - } - - // is the expression cross tables? - if (tableSet.size() == 1 && exps->associatedJoinId() == 0) - { - // single table and not in join on clause - uint32_t tid = tables[0]; - - for (uint64_t i = 0; i < columns.size(); ++i) - tableInfoMap[tid].fColsInExp1.push_back(columns[i]); - - tableInfoMap[tid].fOneTableExpSteps.push_back(*it); - } - else - { - // WORKAROUND for limitation on join with filter - if (exps->associatedJoinId() != 0) - { - for (uint64_t i = 0; i < exps->columns().size(); ++i) - { - jobInfo.joinFeTableMap[exps->associatedJoinId()].insert(tables[i]); - } - } - - // resolve after join: cross table or on clause conditions - for (uint64_t i = 0; i < columns.size(); ++i) - { - uint32_t cid = columns[i]; - uint32_t tid = getTableKey(jobInfo, cid); - tableInfoMap[tid].fColsInExp2.push_back(cid); - } - - expSteps.push_back(*it); - } - } - // Separate the other steps by unique ID. else { - uint32_t tid = -1; - uint64_t cid = (*it)->tupleId(); - - if (cid != (uint64_t) - 1) - tid = getTableKey(jobInfo, (*it)->tupleId()); - else - tid = getTableKey(jobInfo, it->get()); - - if (find(jobInfo.tableList.begin(), jobInfo.tableList.end(), tid) != - jobInfo.tableList.end()) - { - tableInfoMap[tid].fQuerySteps.push_back(*it); - } - else - { - jobInfo.correlateSteps.push_back(*it); - } + joinType ^= RIGHTOUTER; + m1->second.fTypes.push_back(joinType | LARGEOUTER); + m2->second.fTypes.push_back(joinType); + jobInfo.outerOnTable.insert(tid1); } + } + else + { + m1->second.fTypes.push_back(joinType); + m2->second.fTypes.push_back(joinType); + } - it++; + // need id to keep the join order + m1->second.fJoinId = m2->second.fJoinId = thjs->joinId(); } - - // @bug2634, delay isNull filter on outerjoin key - // @bug5374, delay predicates for outerjoin - outjoinPredicateAdjust(tableInfoMap, jobInfo); - - // @bug4021, make sure there is real column to scan - for (TableInfoMap::iterator it = tableInfoMap.begin(); it != tableInfoMap.end(); it++) + // Separate the expressions + else if (exps != NULL && subs == NULL) { - uint32_t tableUid = it->first; + const vector& tables = exps->tableKeys(); + const vector& columns = exps->columnKeys(); + bool tableInOuterQuery = false; + set tableSet; // involved unique tables - if (jobInfo.pseudoColTable.find(tableUid) == jobInfo.pseudoColTable.end()) + for (uint64_t i = 0; i < tables.size(); ++i) + { + if (find(jobInfo.tableList.begin(), jobInfo.tableList.end(), tables[i]) != jobInfo.tableList.end()) + tableSet.insert(tables[i]); + else + tableInOuterQuery = true; + } + + if (tableInOuterQuery) + { + // all columns in subquery scope to be projected + for (uint64_t i = 0; i < tables.size(); ++i) + { + // outer-query columns + if (tableSet.find(tables[i]) == tableSet.end()) continue; - JobStepVector& steps = tableInfoMap[tableUid].fQuerySteps; - JobStepVector::iterator s = steps.begin(); - JobStepVector::iterator p = steps.end(); + // subquery columns + uint32_t c = columns[i]; - for (; s != steps.end(); s++) - { - if (typeid(*(s->get())) == typeid(pColScanStep) || - typeid(*(s->get())) == typeid(pColStep)) - break; + if (jobInfo.returnColSet.find(c) == jobInfo.returnColSet.end()) + { + tableInfoMap[tables[i]].fProjectCols.push_back(c); + jobInfo.pjColList.push_back(getTupleInfo(c, jobInfo)); + jobInfo.returnColSet.insert(c); + const SimpleColumn* sc = dynamic_cast(exps->columns()[i]); - // @bug5893, iterator to the first pseudocolumn - if (typeid(*(s->get())) == typeid(PseudoColStep) && p == steps.end()) - p = s; + if (sc != NULL) + jobInfo.deliveredCols.push_back(SRCP(sc->clone())); + } } - if (s == steps.end()) + jobInfo.correlateSteps.push_back(*it++); + continue; + } + + // is the expression cross tables? + if (tableSet.size() == 1 && exps->associatedJoinId() == 0) + { + // single table and not in join on clause + uint32_t tid = tables[0]; + + for (uint64_t i = 0; i < columns.size(); ++i) + tableInfoMap[tid].fColsInExp1.push_back(columns[i]); + + tableInfoMap[tid].fOneTableExpSteps.push_back(*it); + } + else + { + // WORKAROUND for limitation on join with filter + if (exps->associatedJoinId() != 0) { - map::iterator t = jobInfo.tableColMap.find(tableUid); - - if (t == jobInfo.tableColMap.end()) - { - string msg = jobInfo.keyInfo->tupleKeyToName[tableUid]; - msg += " has no column in column map."; - throw runtime_error(msg); - } - - SimpleColumn* sc = dynamic_cast(t->second.get()); - CalpontSystemCatalog::OID oid = sc->oid(); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - CalpontSystemCatalog::ColType ct = sc->colType(); - string alias(extractTableAlias(sc)); - SJSTEP sjs(new pColScanStep(oid, tblOid, ct, jobInfo)); - sjs->alias(alias); - sjs->view(sc->viewName()); - sjs->schema(sc->schemaName()); - sjs->name(sc->columnName()); - TupleInfo ti(setTupleInfo(ct, oid, jobInfo, tblOid, sc, alias)); - sjs->tupleId(ti.key); - steps.insert(steps.begin(), sjs); - - if (isDictCol(ct) && jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) - jobInfo.tokenOnly[ti.key] = true; + for (uint64_t i = 0; i < exps->columns().size(); ++i) + { + jobInfo.joinFeTableMap[exps->associatedJoinId()].insert(tables[i]); + } } - else if (s > p) + + // resolve after join: cross table or on clause conditions + for (uint64_t i = 0; i < columns.size(); ++i) { - // @bug5893, make sure a pcol is in front of any pseudo step. - SJSTEP t = *s; - *s = *p; - *p = t; + uint32_t cid = columns[i]; + uint32_t tid = getTableKey(jobInfo, cid); + tableInfoMap[tid].fColsInExp2.push_back(cid); } + + expSteps.push_back(*it); + } } - - // @bug3767, error out scalar subquery with aggregation and correlated additional comparison. - if (jobInfo.hasAggregation && (jobInfo.correlateSteps.size() > 0)) + // Separate the other steps by unique ID. + else { - // expression filter - ExpressionStep* exp = NULL; + uint32_t tid = -1; + uint64_t cid = (*it)->tupleId(); - for (it = jobInfo.correlateSteps.begin(); it != jobInfo.correlateSteps.end(); it++) - { - if (((exp = dynamic_cast(it->get())) != NULL) && (!exp->functionJoin())) - break; + if (cid != (uint64_t)-1) + tid = getTableKey(jobInfo, (*it)->tupleId()); + else + tid = getTableKey(jobInfo, it->get()); - exp = NULL; - } - - // correlated join step - TupleHashJoinStep* thjs = NULL; - - for (it = jobInfo.correlateSteps.begin(); it != jobInfo.correlateSteps.end(); it++) - { - if ((thjs = dynamic_cast(it->get())) != NULL) - break; - } - - // @bug5202, error out not equal correlation and aggregation in subquery. - if ((exp != NULL) && (thjs != NULL) && (thjs->getJoinType() & CORRELATED)) - throw IDBExcept( - IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_NEQ_AGG_SUB), - ERR_NON_SUPPORT_NEQ_AGG_SUB); + if (find(jobInfo.tableList.begin(), jobInfo.tableList.end(), tid) != jobInfo.tableList.end()) + { + tableInfoMap[tid].fQuerySteps.push_back(*it); + } + else + { + jobInfo.correlateSteps.push_back(*it); + } } - it = projectSteps.begin(); - end = projectSteps.end(); + it++; + } - while (it != end) + // @bug2634, delay isNull filter on outerjoin key + // @bug5374, delay predicates for outerjoin + outjoinPredicateAdjust(tableInfoMap, jobInfo); + + // @bug4021, make sure there is real column to scan + for (TableInfoMap::iterator it = tableInfoMap.begin(); it != tableInfoMap.end(); it++) + { + uint32_t tableUid = it->first; + + if (jobInfo.pseudoColTable.find(tableUid) == jobInfo.pseudoColTable.end()) + continue; + + JobStepVector& steps = tableInfoMap[tableUid].fQuerySteps; + JobStepVector::iterator s = steps.begin(); + JobStepVector::iterator p = steps.end(); + + for (; s != steps.end(); s++) { - uint32_t tid = getTableKey(jobInfo, (*it)->tupleId()); - tableInfoMap[tid].fProjectSteps.push_back(*it); - tableInfoMap[tid].fProjectCols.push_back((*it)->tupleId()); - it++; + if (typeid(*(s->get())) == typeid(pColScanStep) || typeid(*(s->get())) == typeid(pColStep)) + break; + + // @bug5893, iterator to the first pseudocolumn + if (typeid(*(s->get())) == typeid(PseudoColStep) && p == steps.end()) + p = s; } - for (TupleInfoVector::iterator j = jobInfo.pjColList.begin(); j != jobInfo.pjColList.end(); j++) + if (s == steps.end()) { - if (jobInfo.keyInfo->tupleKeyVec[j->tkey].fId == CNX_EXP_TABLE_ID) - continue; + map::iterator t = jobInfo.tableColMap.find(tableUid); - vector& projectCols = tableInfoMap[j->tkey].fProjectCols; + if (t == jobInfo.tableColMap.end()) + { + string msg = jobInfo.keyInfo->tupleKeyToName[tableUid]; + msg += " has no column in column map."; + throw runtime_error(msg); + } - if (find(projectCols.begin(), projectCols.end(), j->key) == projectCols.end()) - projectCols.push_back(j->key); + SimpleColumn* sc = dynamic_cast(t->second.get()); + CalpontSystemCatalog::OID oid = sc->oid(); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + CalpontSystemCatalog::ColType ct = sc->colType(); + string alias(extractTableAlias(sc)); + SJSTEP sjs(new pColScanStep(oid, tblOid, ct, jobInfo)); + sjs->alias(alias); + sjs->view(sc->viewName()); + sjs->schema(sc->schemaName()); + sjs->name(sc->columnName()); + TupleInfo ti(setTupleInfo(ct, oid, jobInfo, tblOid, sc, alias)); + sjs->tupleId(ti.key); + steps.insert(steps.begin(), sjs); + + if (isDictCol(ct) && jobInfo.tokenOnly.find(ti.key) == jobInfo.tokenOnly.end()) + jobInfo.tokenOnly[ti.key] = true; } - - JobStepVector& retExp = jobInfo.returnedExpressions; - - for (it = retExp.begin(); it != retExp.end(); ++it) + else if (s > p) { - ExpressionStep* exp = dynamic_cast(it->get()); + // @bug5893, make sure a pcol is in front of any pseudo step. + SJSTEP t = *s; + *s = *p; + *p = t; + } + } - if (exp == NULL) - throw runtime_error("Not an expression."); + // @bug3767, error out scalar subquery with aggregation and correlated additional comparison. + if (jobInfo.hasAggregation && (jobInfo.correlateSteps.size() > 0)) + { + // expression filter + ExpressionStep* exp = NULL; - for (uint64_t i = 0; i < exp->columnKeys().size(); ++i) - { - tableInfoMap[exp->tableKeys()[i]].fColsInRetExp.push_back(exp->columnKeys()[i]); - } + for (it = jobInfo.correlateSteps.begin(); it != jobInfo.correlateSteps.end(); it++) + { + if (((exp = dynamic_cast(it->get())) != NULL) && (!exp->functionJoin())) + break; + + exp = NULL; } - // reset all step vector - querySteps.clear(); - projectSteps.clear(); - deliverySteps.clear(); + // correlated join step + TupleHashJoinStep* thjs = NULL; - JoinEdges edgesToTransform; - // Check if the tables and joins can be used to construct a spanning tree. - spanningTreeCheck(tableInfoMap, joinSteps, jobInfo, edgesToTransform); + for (it = jobInfo.correlateSteps.begin(); it != jobInfo.correlateSteps.end(); it++) + { + if ((thjs = dynamic_cast(it->get())) != NULL) + break; + } - // 1. combine job steps for each table - TableInfoMap::iterator mit; + // @bug5202, error out not equal correlation and aggregation in subquery. + if ((exp != NULL) && (thjs != NULL) && (thjs->getJoinType() & CORRELATED)) + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_NEQ_AGG_SUB), + ERR_NON_SUPPORT_NEQ_AGG_SUB); + } - for (mit = tableInfoMap.begin(); mit != tableInfoMap.end(); mit++) - if (combineJobStepsByTable(mit, jobInfo) == false) - throw runtime_error("combineJobStepsByTable failed."); + it = projectSteps.begin(); + end = projectSteps.end(); - // 2. join the combined steps together to form the spanning tree - vector joinOrder; - joinTables(joinSteps, tableInfoMap, jobInfo, joinOrder, edgesToTransform, - overrideLargeSideEstimate); + while (it != end) + { + uint32_t tid = getTableKey(jobInfo, (*it)->tupleId()); + tableInfoMap[tid].fProjectSteps.push_back(*it); + tableInfoMap[tid].fProjectCols.push_back((*it)->tupleId()); + it++; + } - // 3. put the steps together - for (vector::iterator i = joinOrder.begin(); i != joinOrder.end(); ++i) - querySteps.insert(querySteps.end(), - tableInfoMap[*i].fQuerySteps.begin(), - tableInfoMap[*i].fQuerySteps.end()); + for (TupleInfoVector::iterator j = jobInfo.pjColList.begin(); j != jobInfo.pjColList.end(); j++) + { + if (jobInfo.keyInfo->tupleKeyVec[j->tkey].fId == CNX_EXP_TABLE_ID) + continue; - adjustLastStep(querySteps, deliverySteps, jobInfo); // to match the select clause + vector& projectCols = tableInfoMap[j->tkey].fProjectCols; + + if (find(projectCols.begin(), projectCols.end(), j->key) == projectCols.end()) + projectCols.push_back(j->key); + } + + JobStepVector& retExp = jobInfo.returnedExpressions; + + for (it = retExp.begin(); it != retExp.end(); ++it) + { + ExpressionStep* exp = dynamic_cast(it->get()); + + if (exp == NULL) + throw runtime_error("Not an expression."); + + for (uint64_t i = 0; i < exp->columnKeys().size(); ++i) + { + tableInfoMap[exp->tableKeys()[i]].fColsInRetExp.push_back(exp->columnKeys()[i]); + } + } + + // reset all step vector + querySteps.clear(); + projectSteps.clear(); + deliverySteps.clear(); + + JoinEdges edgesToTransform; + // Check if the tables and joins can be used to construct a spanning tree. + spanningTreeCheck(tableInfoMap, joinSteps, jobInfo, edgesToTransform); + + // 1. combine job steps for each table + TableInfoMap::iterator mit; + + for (mit = tableInfoMap.begin(); mit != tableInfoMap.end(); mit++) + if (combineJobStepsByTable(mit, jobInfo) == false) + throw runtime_error("combineJobStepsByTable failed."); + + // 2. join the combined steps together to form the spanning tree + vector joinOrder; + joinTables(joinSteps, tableInfoMap, jobInfo, joinOrder, edgesToTransform, overrideLargeSideEstimate); + + // 3. put the steps together + for (vector::iterator i = joinOrder.begin(); i != joinOrder.end(); ++i) + querySteps.insert(querySteps.end(), tableInfoMap[*i].fQuerySteps.begin(), + tableInfoMap[*i].fQuerySteps.end()); + + adjustLastStep(querySteps, deliverySteps, jobInfo); // to match the select clause } - SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& jobInfo) { - vector inputRGs; - vector distinct; - uint64_t colCount = jobInfo.deliveredCols.size(); + vector inputRGs; + vector distinct; + uint64_t colCount = jobInfo.deliveredCols.size(); - vector oids; - vector keys; - vector scale; - vector precision; - vector width; - vector types; - vector csNums; - JobStepAssociation jsaToUnion; + vector oids; + vector keys; + vector scale; + vector precision; + vector width; + vector types; + vector csNums; + JobStepAssociation jsaToUnion; - // bug4388, share code with connector for column type coversion - vector > queryColTypes; + // bug4388, share code with connector for column type coversion + vector> queryColTypes; + + for (uint64_t j = 0; j < colCount; ++j) + queryColTypes.push_back(vector(queries.size())); + + for (uint64_t i = 0; i < queries.size(); i++) + { + SJSTEP& spjs = queries[i]; + TupleDeliveryStep* tds = dynamic_cast(spjs.get()); + + if (tds == NULL) + { + throw runtime_error("Not a deliverable step."); + } + + const RowGroup& rg = tds->getDeliveredRowGroup(); + inputRGs.push_back(rg); + + const vector& scaleIn = rg.getScale(); + const vector& precisionIn = rg.getPrecision(); + const vector& typesIn = rg.getColTypes(); + const vector& csNumsIn = rg.getCharsetNumbers(); for (uint64_t j = 0; j < colCount; ++j) - queryColTypes.push_back(vector(queries.size())); - - for (uint64_t i = 0; i < queries.size(); i++) { - SJSTEP& spjs = queries[i]; - TupleDeliveryStep* tds = dynamic_cast(spjs.get()); - - if (tds == NULL) - { - throw runtime_error("Not a deliverable step."); - } - - const RowGroup& rg = tds->getDeliveredRowGroup(); - inputRGs.push_back(rg); - - const vector& scaleIn = rg.getScale(); - const vector& precisionIn = rg.getPrecision(); - const vector& typesIn = rg.getColTypes(); - const vector& csNumsIn = rg.getCharsetNumbers(); - - for (uint64_t j = 0; j < colCount; ++j) - { - queryColTypes[j][i].colDataType = typesIn[j]; - queryColTypes[j][i].charsetNumber = csNumsIn[j]; - queryColTypes[j][i].scale = scaleIn[j]; - queryColTypes[j][i].precision = precisionIn[j]; - queryColTypes[j][i].colWidth = rg.getColumnWidth(j); - } - - if (i == 0) - { - const vector& oidsIn = rg.getOIDs(); - const vector& keysIn = rg.getKeys(); - oids.insert(oids.end(), oidsIn.begin(), oidsIn.begin() + colCount); - keys.insert(keys.end(), keysIn.begin(), keysIn.begin() + colCount); - } - - // if all union types are UNION_ALL, distinctUnionNum is 0. - distinct.push_back(distinctUnionNum > i); - - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - spdl->rowGroupDL(dl); - dl->OID(CNX_VTABLE_ID); - JobStepAssociation jsa; - jsa.outAdd(spdl); - spjs->outputAssociation(jsa); - jsaToUnion.outAdd(spdl); + queryColTypes[j][i].colDataType = typesIn[j]; + queryColTypes[j][i].charsetNumber = csNumsIn[j]; + queryColTypes[j][i].scale = scaleIn[j]; + queryColTypes[j][i].precision = precisionIn[j]; + queryColTypes[j][i].colWidth = rg.getColumnWidth(j); } + if (i == 0) + { + const vector& oidsIn = rg.getOIDs(); + const vector& keysIn = rg.getKeys(); + oids.insert(oids.end(), oidsIn.begin(), oidsIn.begin() + colCount); + keys.insert(keys.end(), keysIn.begin(), keysIn.begin() + colCount); + } + + // if all union types are UNION_ALL, distinctUnionNum is 0. + distinct.push_back(distinctUnionNum > i); + AnyDataListSPtr spdl(new AnyDataList()); RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); spdl->rowGroupDL(dl); dl->OID(CNX_VTABLE_ID); JobStepAssociation jsa; jsa.outAdd(spdl); - TupleUnion* unionStep = new TupleUnion(CNX_VTABLE_ID, jobInfo); - unionStep->inputAssociation(jsaToUnion); - unionStep->outputAssociation(jsa); + spjs->outputAssociation(jsa); + jsaToUnion.outAdd(spdl); + } - // get unioned column types - for (uint64_t j = 0; j < colCount; ++j) - { - CalpontSystemCatalog::ColType colType = CalpontSystemCatalog::ColType::convertUnionColType(queryColTypes[j]); - types.push_back(colType.colDataType); - csNums.push_back(colType.charsetNumber); - scale.push_back(colType.scale); - precision.push_back(colType.precision); - width.push_back(colType.colWidth); - } + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + spdl->rowGroupDL(dl); + dl->OID(CNX_VTABLE_ID); + JobStepAssociation jsa; + jsa.outAdd(spdl); + TupleUnion* unionStep = new TupleUnion(CNX_VTABLE_ID, jobInfo); + unionStep->inputAssociation(jsaToUnion); + unionStep->outputAssociation(jsa); - vector pos; - pos.push_back(2); + // get unioned column types + for (uint64_t j = 0; j < colCount; ++j) + { + CalpontSystemCatalog::ColType colType = + CalpontSystemCatalog::ColType::convertUnionColType(queryColTypes[j]); + types.push_back(colType.colDataType); + csNums.push_back(colType.charsetNumber); + scale.push_back(colType.scale); + precision.push_back(colType.precision); + width.push_back(colType.colWidth); + } - for (uint64_t i = 0; i < oids.size(); ++i) - pos.push_back(pos[i] + width[i]); + vector pos; + pos.push_back(2); - unionStep->setInputRowGroups(inputRGs); - unionStep->setDistinctFlags(distinct); - unionStep->setOutputRowGroup(RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold)); + for (uint64_t i = 0; i < oids.size(); ++i) + pos.push_back(pos[i] + width[i]); - // Fix for bug 4388 adjusts the result type at connector side, this workaround is obsolete. - // bug 3067, update the returned column types. - // This is a workaround as the connector always uses the first query' returned columns. - // ct.colDataType = types[i]; - // ct.scale = scale[i]; - // ct.colWidth = width[i]; + unionStep->setInputRowGroups(inputRGs); + unionStep->setDistinctFlags(distinct); + unionStep->setOutputRowGroup( + RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold)); - for (size_t i = 0; i < jobInfo.deliveredCols.size(); i++) - { - CalpontSystemCatalog::ColType ct = jobInfo.deliveredCols[i]->resultType(); -//XXX remove after connector change - ct.colDataType = types[i]; - ct.scale = scale[i]; - ct.colWidth = width[i]; + // Fix for bug 4388 adjusts the result type at connector side, this workaround is obsolete. + // bug 3067, update the returned column types. + // This is a workaround as the connector always uses the first query' returned columns. + // ct.colDataType = types[i]; + // ct.scale = scale[i]; + // ct.colWidth = width[i]; - // varchar/varbinary column width has been fudged, see fudgeWidth in jlf_common.cpp. - if (ct.colDataType == CalpontSystemCatalog::VARCHAR) - ct.colWidth--; - else if (ct.colDataType == CalpontSystemCatalog::VARBINARY) - ct.colWidth -= 2; + for (size_t i = 0; i < jobInfo.deliveredCols.size(); i++) + { + CalpontSystemCatalog::ColType ct = jobInfo.deliveredCols[i]->resultType(); + // XXX remove after connector change + ct.colDataType = types[i]; + ct.scale = scale[i]; + ct.colWidth = width[i]; - jobInfo.deliveredCols[i]->resultType(ct); - } + // varchar/varbinary column width has been fudged, see fudgeWidth in jlf_common.cpp. + if (ct.colDataType == CalpontSystemCatalog::VARCHAR) + ct.colWidth--; + else if (ct.colDataType == CalpontSystemCatalog::VARBINARY) + ct.colWidth -= 2; - if (jobInfo.trace) - { - cout << boldStart << "\ninput RGs: (distinct=" << distinctUnionNum << ")\n" << boldStop; + jobInfo.deliveredCols[i]->resultType(ct); + } - for (vector::iterator i = inputRGs.begin(); i != inputRGs.end(); i++) - cout << i->toString() << endl << endl; + if (jobInfo.trace) + { + cout << boldStart << "\ninput RGs: (distinct=" << distinctUnionNum << ")\n" << boldStop; - cout << boldStart << "output RG:\n" << boldStop - << unionStep->getDeliveredRowGroup().toString() << endl; - } + for (vector::iterator i = inputRGs.begin(); i != inputRGs.end(); i++) + cout << i->toString() << endl << endl; - return SJSTEP(unionStep); + cout << boldStart << "output RG:\n" << boldStop << unionStep->getDeliveredRowGroup().toString() << endl; + } + + return SJSTEP(unionStep); } -} +} // namespace joblist // vim:ts=4 sw=4: #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif - diff --git a/dbcon/joblist/jlf_tuplejoblist.h b/dbcon/joblist/jlf_tuplejoblist.h index c09fd2ff4..63af004b1 100644 --- a/dbcon/joblist/jlf_tuplejoblist.h +++ b/dbcon/joblist/jlf_tuplejoblist.h @@ -17,7 +17,6 @@ // $Id: jlf_tuplejoblist.h 9655 2013-06-25 23:08:13Z xlou $ - /** @file */ #ifndef JLF_TUPLEJOBLIST_H @@ -29,124 +28,122 @@ namespace joblist { - // data to construct and config a tuple hashjoin [used by small side table] struct JoinInfo { - execplan::CalpontSystemCatalog::OID fTableOid; - std::string fAlias; - std::string fSchema; - std::string fView; - AnyDataListSPtr fDl; // output data list - rowgroup::RowGroup fRowGroup; // rowgroup meta data for the data list - // colOid and alias can be retrieved from JobInfo.tupleKeyVec using join key. + execplan::CalpontSystemCatalog::OID fTableOid; + std::string fAlias; + std::string fSchema; + std::string fView; + AnyDataListSPtr fDl; // output data list + rowgroup::RowGroup fRowGroup; // rowgroup meta data for the data list + // colOid and alias can be retrieved from JobInfo.tupleKeyVec using join key. - // @bug 1495 compound join - JoinData fJoinData; + // @bug 1495 compound join + JoinData fJoinData; - JoinInfo() : fTableOid(-1) {} + JoinInfo() : fTableOid(-1) + { + } }; typedef boost::shared_ptr SP_JoinInfo; - // data to construct and config a token hashjoin for string access predicates struct DictionaryScanInfo { - uint32_t fTokenId; // token unique id - AnyDataListSPtr fDl; // data list - rowgroup::RowGroup fRowGroup; // rowgroup meta data for the data list + uint32_t fTokenId; // token unique id + AnyDataListSPtr fDl; // data list + rowgroup::RowGroup fRowGroup; // rowgroup meta data for the data list - DictionaryScanInfo() : fTokenId(-1) { } + DictionaryScanInfo() : fTokenId(-1) + { + } }; - // steps and joins of a table in a query struct TableInfo { - execplan::CalpontSystemCatalog::OID fTableOid; - std::string fName; - std::string fAlias; - std::string fSchema; - std::string fView; - uint64_t fSubId; - JobStepVector fQuerySteps; - JobStepVector fProjectSteps; - JobStepVector fFuncJoinExps; - JobStepVector fOneTableExpSteps; - std::vector fProjectCols; - std::vector fColsInExp1; // 1 table expression - std::vector fColsInExp2; // 2 or more tables in expression - std::vector fColsInRetExp; // expression in selection/group by clause - std::vector fColsInOuter; // delayed outer join filter - std::vector fColsInFuncJoin; // expression for function join - std::vector fColsInColMap; // columns in column map - std::vector fJoinKeys; - std::vector fAdjacentList; // tables joined with - bool fVisited; + execplan::CalpontSystemCatalog::OID fTableOid; + std::string fName; + std::string fAlias; + std::string fSchema; + std::string fView; + uint64_t fSubId; + JobStepVector fQuerySteps; + JobStepVector fProjectSteps; + JobStepVector fFuncJoinExps; + JobStepVector fOneTableExpSteps; + std::vector fProjectCols; + std::vector fColsInExp1; // 1 table expression + std::vector fColsInExp2; // 2 or more tables in expression + std::vector fColsInRetExp; // expression in selection/group by clause + std::vector fColsInOuter; // delayed outer join filter + std::vector fColsInFuncJoin; // expression for function join + std::vector fColsInColMap; // columns in column map + std::vector fJoinKeys; + std::vector fAdjacentList; // tables joined with + bool fVisited; - AnyDataListSPtr fDl; // output data list - rowgroup::RowGroup fRowGroup; // output rowgroup meta data - std::set fJoinedTables; // tables directly/indirectly joined to this table + AnyDataListSPtr fDl; // output data list + rowgroup::RowGroup fRowGroup; // output rowgroup meta data + std::set fJoinedTables; // tables directly/indirectly joined to this table - TableInfo() : fTableOid(-1), fVisited(false) {} + TableInfo() : fTableOid(-1), fVisited(false) + { + } }; typedef std::map TableInfoMap; - struct FunctionJoinInfo { - std::vector fTableKey; - std::vector fJoinKey; - std::vector fTableOid; - std::vector fOid; - std::vector fSequence; - std::vector fAlias; - std::vector fView; - std::vector fSchema; - JobStepVector fStep; - JoinType fJoinType; - int64_t fJoinId; - int64_t fCorrelatedSide; - std::vector > fColumnKeys; - std::vector fExpression; + std::vector fTableKey; + std::vector fJoinKey; + std::vector fTableOid; + std::vector fOid; + std::vector fSequence; + std::vector fAlias; + std::vector fView; + std::vector fSchema; + JobStepVector fStep; + JoinType fJoinType; + int64_t fJoinId; + int64_t fCorrelatedSide; + std::vector> fColumnKeys; + std::vector fExpression; - FunctionJoinInfo() : fJoinType(0), fJoinId(0), fCorrelatedSide(0) {} + FunctionJoinInfo() : fJoinType(0), fJoinId(0), fCorrelatedSide(0) + { + } }; typedef boost::shared_ptr SP_FuncJoinInfo; -typedef std::pair KeyPair; - +typedef std::pair KeyPair; // combine and associate tuple job steps -void associateTupleJobSteps(JobStepVector& querySteps, - JobStepVector& projectSteps, - DeliveredTableMap& deliverySteps, - JobInfo& jobInfo, +void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSteps, + DeliveredTableMap& deliverySteps, JobInfo& jobInfo, const bool overrideLargeSideEstimate); - // make BOP_OR an expression step void orExpresssion(const execplan::Operator* op, JobInfo& jobInfo); // union the queries and return the tuple union step -SJSTEP unionQueries(JobStepVector& queries, - uint64_t distinctUnionNum, - JobInfo& jobInfo); +SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& jobInfo); struct JoinTableNode { - bool fVisited; - uint32_t fParent; - std::vector fAdjacentList; - JoinTableNode() : fVisited(false), fParent(-1) {} + bool fVisited; + uint32_t fParent; + std::vector fAdjacentList; + JoinTableNode() : fVisited(false), fParent(-1) + { + } }; using JoinGraph = std::map; using JoinEdges = std::set>; using Cycles = std::vector>>; using Cycle = std::vector>; -using PostJoinFilterKeys = - std::vector, std::vector>>; -} +using PostJoinFilterKeys = std::vector, std::vector>>; +} // namespace joblist #endif // JLF_TUPLEJOBLIST_H - diff --git a/dbcon/joblist/joblist.cpp b/dbcon/joblist/joblist.cpp index 6c6bceb54..fd58bde04 100644 --- a/dbcon/joblist/joblist.cpp +++ b/dbcon/joblist/joblist.cpp @@ -47,44 +47,41 @@ using namespace execplan; #include "atomicops.h" - #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" - // for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to 'typeid' +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" +// for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to +// 'typeid' #endif namespace joblist { -int JobList::fPmsConfigured = 0; +int JobList::fPmsConfigured = 0; struct JSJoiner { - JSJoiner(JobStep* j): js(j) { } - void operator()() - { - js->abort(); - js->join(); - } - JobStep* js; + JSJoiner(JobStep* j) : js(j) + { + } + void operator()() + { + js->abort(); + js->join(); + } + JobStep* js; }; -JobList::JobList(bool isEM) : - fIsRunning(false), - fIsExeMgr(isEM), - fPmsConnected(0), - projectingTableOID(0), - fAborted(0), - fPriority(50) +JobList::JobList(bool isEM) + : fIsRunning(false), fIsExeMgr(isEM), fPmsConnected(0), projectingTableOID(0), fAborted(0), fPriority(50) { } JobList::~JobList() { - try + try + { + if (fIsRunning) { - if (fIsRunning) - { #if 0 // This logic creates a set of threads to wind down the query vector joiners; @@ -116,300 +113,297 @@ JobList::~JobList() nullStep.jobstepThreadPool.join(joiners); #endif - // This logic stops the query steps one at a time - JobStepVector::iterator iter; - JobStepVector::iterator end; - end = fQuery.end(); + // This logic stops the query steps one at a time + JobStepVector::iterator iter; + JobStepVector::iterator end; + end = fQuery.end(); - for (iter = fQuery.begin(); iter != end; ++iter) - { - (*iter)->abort(); - } + for (iter = fQuery.begin(); iter != end; ++iter) + { + (*iter)->abort(); + } - // Stop all the projection steps - end = fProject.end(); + // Stop all the projection steps + end = fProject.end(); - for (iter = fProject.begin(); iter != end; ++iter) - { - (*iter)->abort(); - } + for (iter = fProject.begin(); iter != end; ++iter) + { + (*iter)->abort(); + } - // Wait for all the query steps to end - end = fQuery.end(); + // Wait for all the query steps to end + end = fQuery.end(); - for (iter = fQuery.begin(); iter != end; ++iter) - { - (*iter)->join(); - } + for (iter = fQuery.begin(); iter != end; ++iter) + { + (*iter)->join(); + } - // Wait for all the projection steps to end - end = fProject.end(); + // Wait for all the projection steps to end + end = fProject.end(); - for (iter = fProject.begin(); iter != end; ++iter) - { - (*iter)->join(); - } - } - } - catch (exception& ex) - { - cerr << "JobList::~JobList: exception caught: " << ex.what() << endl; - } - catch (...) - { - cerr << "JobList::~JobList: exception caught!" << endl; + for (iter = fProject.begin(); iter != end; ++iter) + { + (*iter)->join(); + } } + } + catch (exception& ex) + { + cerr << "JobList::~JobList: exception caught: " << ex.what() << endl; + } + catch (...) + { + cerr << "JobList::~JobList: exception caught!" << endl; + } } int JobList::doQuery() { - // Don't start the steps if there is no PrimProc connection. - if (fPmsConfigured < 1 || fPmsConnected < fPmsConfigured) - return 0; + // Don't start the steps if there is no PrimProc connection. + if (fPmsConfigured < 1 || fPmsConnected < fPmsConfigured) + return 0; - JobStep* js; + JobStep* js; - // Set the priority on the jobsteps - for (uint32_t i = 0; i < fQuery.size(); i++) - fQuery[i]->priority(fPriority); + // Set the priority on the jobsteps + for (uint32_t i = 0; i < fQuery.size(); i++) + fQuery[i]->priority(fPriority); - for (uint32_t i = 0; i < fProject.size(); i++) - fProject[i]->priority(fPriority); + for (uint32_t i = 0; i < fProject.size(); i++) + fProject[i]->priority(fPriority); - // I put this logging in a separate loop rather than including it in the - // other loop that calls run(), to insure that these logging msgs would - // not be interleaved with any logging coming from the calls to run(). - JobStepVector::iterator iter2 = fQuery.begin(); - JobStepVector::iterator end2 = fQuery.end(); - int rc = -1; + // I put this logging in a separate loop rather than including it in the + // other loop that calls run(), to insure that these logging msgs would + // not be interleaved with any logging coming from the calls to run(). + JobStepVector::iterator iter2 = fQuery.begin(); + JobStepVector::iterator end2 = fQuery.end(); + int rc = -1; - while (iter2 != end2) + while (iter2 != end2) + { + js = iter2->get(); + + if (js->traceOn()) { - js = iter2->get(); - - if (js->traceOn()) - { - if (js->delayedRun()) - { - std::ostringstream oss; - oss << "Session: " << js->sessionId() << - "; delaying start of query step " << js->stepId() << - "; waitStepCount-" << js->waitToRunStepCnt() << std::endl; - std::cout << oss.str(); - } - } - - ++iter2; + if (js->delayedRun()) + { + std::ostringstream oss; + oss << "Session: " << js->sessionId() << "; delaying start of query step " << js->stepId() + << "; waitStepCount-" << js->waitToRunStepCnt() << std::endl; + std::cout << oss.str(); + } } - iter2 = fProject.begin(); - end2 = fProject.end(); + ++iter2; + } - while (iter2 != end2) + iter2 = fProject.begin(); + end2 = fProject.end(); + + while (iter2 != end2) + { + js = iter2->get(); + + if (js->traceOn()) { - js = iter2->get(); - - if (js->traceOn()) - { - if (js->delayedRun()) - { - std::ostringstream oss; - oss << "Session: " << js->sessionId() << - "; delaying start of project step " << js->stepId() << - "; waitStepCount-" << js->waitToRunStepCnt() << std::endl; - std::cout << oss.str(); - } - } - - ++iter2; + if (js->delayedRun()) + { + std::ostringstream oss; + oss << "Session: " << js->sessionId() << "; delaying start of project step " << js->stepId() + << "; waitStepCount-" << js->waitToRunStepCnt() << std::endl; + std::cout << oss.str(); + } } - JobStepVector::iterator iter = fQuery.begin(); - JobStepVector::iterator end = fQuery.end(); + ++iter2; + } - // Start the query running - while (iter != end) + JobStepVector::iterator iter = fQuery.begin(); + JobStepVector::iterator end = fQuery.end(); + + // Start the query running + while (iter != end) + { + js = iter->get(); + + if (!js->delayedRun()) { - js = iter->get(); - - if (!js->delayedRun()) - { - js->run(); - } - - ++iter; + js->run(); } - iter = fProject.begin(); - end = fProject.end(); + ++iter; + } - // Fire up the projection steps - while (iter != end) + iter = fProject.begin(); + end = fProject.end(); + + // Fire up the projection steps + while (iter != end) + { + if (!iter->get()->delayedRun()) { - if (!iter->get()->delayedRun()) - { - iter->get()->run(); - } - - ++iter; + iter->get()->run(); } - fIsRunning = true; - rc = 0; - return rc; + ++iter; + } + + fIsRunning = true; + rc = 0; + return rc; } int JobList::putEngineComm(DistributedEngineComm* dec) { - int retryCnt = 0; + int retryCnt = 0; - if (fPmsConfigured == 0) - { - logging::LoggingID lid(05); - logging::MessageLog ml(lid); - logging::Message::Args args; - logging::Message m(0); - // We failed to get a connection - args.add("There are no PMs configured. Can't perform Query"); - args.add(retryCnt); - m.format(args); - ml.logDebugMessage(m); + if (fPmsConfigured == 0) + { + logging::LoggingID lid(05); + logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message m(0); + // We failed to get a connection + args.add("There are no PMs configured. Can't perform Query"); + args.add(retryCnt); + m.format(args); + ml.logDebugMessage(m); - if (!errInfo) - errInfo.reset(new ErrorInfo); + if (!errInfo) + errInfo.reset(new ErrorInfo); - errInfo->errCode = logging::ERR_NO_PRIMPROC; - errInfo->errMsg = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NO_PRIMPROC); - return errInfo->errCode; - } + errInfo->errCode = logging::ERR_NO_PRIMPROC; + errInfo->errMsg = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NO_PRIMPROC); + return errInfo->errCode; + } - // Check to be sure all PrimProcs are attached. + // Check to be sure all PrimProcs are attached. + fPmsConnected = dec->connectedPmServers(); + + while (fPmsConnected < fPmsConfigured) + { + sleep(1); fPmsConnected = dec->connectedPmServers(); - while (fPmsConnected < fPmsConfigured) + // Give up after 20 seconds. Primproc isn't coming back + if (retryCnt >= 20) { - sleep(1); - fPmsConnected = dec->connectedPmServers(); - - // Give up after 20 seconds. Primproc isn't coming back - if (retryCnt >= 20) - { - break; - } - - ++retryCnt; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - oamCache->forceReload(); - dec->Setup(); + break; } - if (retryCnt > 0) + ++retryCnt; + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + oamCache->forceReload(); + dec->Setup(); + } + + if (retryCnt > 0) + { + logging::LoggingID lid(05); + logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message m(0); + + if (fPmsConnected < fPmsConfigured) { - logging::LoggingID lid(05); - logging::MessageLog ml(lid); - logging::Message::Args args; - logging::Message m(0); + // We failed to get a connection + args.add("Failed to get all PrimProc connections. Retry count"); + args.add(retryCnt); + m.format(args); + ml.logDebugMessage(m); - if (fPmsConnected < fPmsConfigured) - { - // We failed to get a connection - args.add("Failed to get all PrimProc connections. Retry count"); - args.add(retryCnt); - m.format(args); - ml.logDebugMessage(m); + if (!errInfo) + errInfo.reset(new ErrorInfo); - if (!errInfo) - errInfo.reset(new ErrorInfo); + errInfo->errCode = logging::ERR_NO_PRIMPROC; + errInfo->errMsg = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NO_PRIMPROC); + return errInfo->errCode; + } + else + { + // Log that we had to reconnect + args.add("PrimProc reconnected. Retry count"); + args.add(retryCnt); + m.format(args); + ml.logDebugMessage(m); + } + } - errInfo->errCode = logging::ERR_NO_PRIMPROC; - errInfo->errMsg = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NO_PRIMPROC); - return errInfo->errCode; - } - else - { - // Log that we had to reconnect - args.add("PrimProc reconnected. Retry count"); - args.add(retryCnt); - m.format(args); - ml.logDebugMessage(m); - } + JobStepVector::iterator iter = fQuery.begin(); + JobStepVector::iterator end = fQuery.end(); + + while (iter != end) + { + SJSTEP sjstep = *iter; + JobStep* jsp = sjstep.get(); + + if (typeid(*jsp) == typeid(pDictionaryScan)) + { + pDictionaryScan* step = dynamic_cast(jsp); + step->dec(dec); + } + else if (typeid(*jsp) == typeid(TupleBPS)) + { + BatchPrimitive* step = dynamic_cast(jsp); + step->setBppStep(); + step->dec(dec); } - JobStepVector::iterator iter = fQuery.begin(); - JobStepVector::iterator end = fQuery.end(); + ++iter; + } - while (iter != end) + iter = fProject.begin(); + end = fProject.end(); + + while (iter != end) + { + SJSTEP sjstep = *iter; + JobStep* jsp = sjstep.get(); + + if (typeid(*jsp) == typeid(TupleBPS)) { - SJSTEP sjstep = *iter; - JobStep* jsp = sjstep.get(); - - if (typeid(*jsp) == typeid(pDictionaryScan)) - { - pDictionaryScan* step = dynamic_cast(jsp); - step->dec(dec); - } - else if (typeid(*jsp) == typeid(TupleBPS)) - { - BatchPrimitive* step = dynamic_cast(jsp); - step->setBppStep(); - step->dec(dec); - } - - ++iter; + BatchPrimitive* step = dynamic_cast(jsp); + step->setBppStep(); + step->dec(dec); } - iter = fProject.begin(); - end = fProject.end(); + ++iter; + } - while (iter != end) - { - SJSTEP sjstep = *iter; - JobStep* jsp = sjstep.get(); - - if (typeid(*jsp) == typeid(TupleBPS)) - { - BatchPrimitive* step = dynamic_cast(jsp); - step->setBppStep(); - step->dec(dec); - } - - ++iter; - } - - return 0; + return 0; } // -- TupleJobList /* returns row count */ uint32_t TupleJobList::projectTable(CalpontSystemCatalog::OID, messageqcpp::ByteStream& bs) { - uint32_t ret = ds->nextBand(bs); - moreData = (ret != 0); - return ret; + uint32_t ret = ds->nextBand(bs); + moreData = (ret != 0); + return ret; } const rowgroup::RowGroup& TupleJobList::getOutputRowGroup() const { - if (fDeliveredTables.empty()) - throw runtime_error("Empty delivery!"); + if (fDeliveredTables.empty()) + throw runtime_error("Empty delivery!"); - TupleDeliveryStep* tds = - dynamic_cast(fDeliveredTables.begin()->second.get()); + TupleDeliveryStep* tds = dynamic_cast(fDeliveredTables.begin()->second.get()); - if (tds == NULL) - throw runtime_error("Not a TupleDeliveryStep!!"); + if (tds == NULL) + throw runtime_error("Not a TupleDeliveryStep!!"); - return tds->getDeliveredRowGroup(); + return tds->getDeliveredRowGroup(); } void TupleJobList::setDeliveryFlag(bool f) { - DeliveredTableMap::iterator iter = fDeliveredTables.begin(); - SJSTEP dstep = iter->second; - ds = dynamic_cast(dstep.get()); + DeliveredTableMap::iterator iter = fDeliveredTables.begin(); + SJSTEP dstep = iter->second; + ds = dynamic_cast(dstep.get()); - if (ds) // if not dummy step - dstep->delivery(f); + if (ds) // if not dummy step + dstep->delivery(f); } //------------------------------------------------------------------------------ @@ -421,784 +415,794 @@ void TupleJobList::setDeliveryFlag(bool f) //------------------------------------------------------------------------------ void JobList::querySummary(bool extendedStats) { - fMiniInfo += "\n"; + fMiniInfo += "\n"; - try + try + { + // bug3438, print subquery info prior to outer query + for (vector::iterator i = subqueryJoblists.begin(); i != subqueryJoblists.end(); i++) { - // bug3438, print subquery info prior to outer query - for (vector::iterator i = subqueryJoblists.begin(); i != subqueryJoblists.end(); i++) - { - i->get()->querySummary(extendedStats); - fStats += i->get()->queryStats(); - fExtendedInfo += i->get()->extendedInfo(); - fMiniInfo += i->get()->miniInfo(); - } - - JobStepVector::const_iterator qIter = fQuery.begin(); - JobStepVector::const_iterator qEnd = fQuery.end(); - JobStep* js; - - // - //...Add up the I/O and msg counts for the query job steps - // - while ( qIter != qEnd ) - { - js = qIter->get(); - - fStats.fPhyIO += js->phyIOCount(); - fStats.fCacheIO += js->cacheIOCount(); - - if (typeid(*js) == typeid(TupleBPS)) - { - fStats.fMsgRcvCnt += js->blockTouched(); - } - else - { - fStats.fMsgRcvCnt += js->msgsRcvdCount(); - } - - fStats.fMsgBytesIn += js->msgBytesIn(); - fStats.fMsgBytesOut += js->msgBytesOut(); - - //...As long as we only have 2 job steps that involve casual - //...partioning, we just define blkSkipped() in those 2 classes, - //...and use typeid to find/invoke those methods. If we start - //...adding blkSkipped() to more classes, we should probably make - //...blkSkipped() a pure virtual method of the base JobStep class - //...so that we don't have to do this type checking and casting. - uint64_t skipCnt = 0; - - if (typeid(*js) == typeid(pColStep)) - skipCnt = (dynamic_cast(js))->blksSkipped (); - else if (typeid(*js) == typeid(pColScanStep)) - skipCnt = (dynamic_cast(js))->blksSkipped (); - else if (typeid(*js) == typeid(TupleBPS)) - skipCnt = (dynamic_cast(js))->blksSkipped (); - - fStats.fCPBlocksSkipped += skipCnt; - - if (extendedStats) - { - string ei; - int max = 0; - ei = js->extendedInfo(); - - while (max < 4 && ei.size() <= 10) - { - ei = js->extendedInfo(); - max++; - } - - fExtendedInfo += ei; - fMiniInfo += js->miniInfo() + "\n"; - } - - ++qIter; - } - - JobStepVector::const_iterator pIter = fProject.begin(); - JobStepVector::const_iterator pEnd = fProject.end(); - - // - //...Add up the I/O and msg counts for the projection job steps - // - while ( pIter != pEnd ) - { - js = pIter->get(); - - fStats.fPhyIO += js->phyIOCount(); - fStats.fCacheIO += js->cacheIOCount(); - - if (typeid(*js) == typeid(TupleBPS)) - { - fStats.fMsgRcvCnt += js->blockTouched(); - } - else - { - fStats.fMsgRcvCnt += js->msgsRcvdCount(); - } - - fStats.fMsgBytesIn += js->msgBytesIn(); - fStats.fMsgBytesOut += js->msgBytesOut(); - - uint64_t skipCnt = 0; - - if (typeid(*js) == typeid(pColStep)) - skipCnt = (dynamic_cast(js))->blksSkipped (); - else if (typeid(*js) == typeid(pColScanStep)) - skipCnt = (dynamic_cast(js))->blksSkipped (); - else if (typeid(*js) == typeid(TupleBPS)) - skipCnt = (dynamic_cast(js))->blksSkipped (); - - fStats.fCPBlocksSkipped += skipCnt; - ++pIter; - } - - if ((!fProject.empty()) && extendedStats) - { - DeliveredTableMap::iterator dsi = fDeliveredTables.begin(); - - while (dsi != fDeliveredTables.end()) - { - js = dynamic_cast(dsi->second.get()); - string ei; - int max = 0; - ei = js->extendedInfo(); - - while (max < 4 && ei.size() <= 10) - { - ei = js->extendedInfo(); - max++; - } - - fExtendedInfo += ei; - fMiniInfo += js->miniInfo() + "\n"; - - ++dsi; - } - } - } - catch (exception& ex) - { - cerr << "JobList::querySummary: exception caught: " << ex.what() << endl; - return; - } - catch (...) - { - cerr << "JobList::querySummary: exception caught!" << endl; - return; + i->get()->querySummary(extendedStats); + fStats += i->get()->queryStats(); + fExtendedInfo += i->get()->extendedInfo(); + fMiniInfo += i->get()->miniInfo(); } + JobStepVector::const_iterator qIter = fQuery.begin(); + JobStepVector::const_iterator qEnd = fQuery.end(); + JobStep* js; + + // + //...Add up the I/O and msg counts for the query job steps + // + while (qIter != qEnd) + { + js = qIter->get(); + + fStats.fPhyIO += js->phyIOCount(); + fStats.fCacheIO += js->cacheIOCount(); + + if (typeid(*js) == typeid(TupleBPS)) + { + fStats.fMsgRcvCnt += js->blockTouched(); + } + else + { + fStats.fMsgRcvCnt += js->msgsRcvdCount(); + } + + fStats.fMsgBytesIn += js->msgBytesIn(); + fStats.fMsgBytesOut += js->msgBytesOut(); + + //...As long as we only have 2 job steps that involve casual + //...partioning, we just define blkSkipped() in those 2 classes, + //...and use typeid to find/invoke those methods. If we start + //...adding blkSkipped() to more classes, we should probably make + //...blkSkipped() a pure virtual method of the base JobStep class + //...so that we don't have to do this type checking and casting. + uint64_t skipCnt = 0; + + if (typeid(*js) == typeid(pColStep)) + skipCnt = (dynamic_cast(js))->blksSkipped(); + else if (typeid(*js) == typeid(pColScanStep)) + skipCnt = (dynamic_cast(js))->blksSkipped(); + else if (typeid(*js) == typeid(TupleBPS)) + skipCnt = (dynamic_cast(js))->blksSkipped(); + + fStats.fCPBlocksSkipped += skipCnt; + + if (extendedStats) + { + string ei; + int max = 0; + ei = js->extendedInfo(); + + while (max < 4 && ei.size() <= 10) + { + ei = js->extendedInfo(); + max++; + } + + fExtendedInfo += ei; + fMiniInfo += js->miniInfo() + "\n"; + } + + ++qIter; + } + + JobStepVector::const_iterator pIter = fProject.begin(); + JobStepVector::const_iterator pEnd = fProject.end(); + + // + //...Add up the I/O and msg counts for the projection job steps + // + while (pIter != pEnd) + { + js = pIter->get(); + + fStats.fPhyIO += js->phyIOCount(); + fStats.fCacheIO += js->cacheIOCount(); + + if (typeid(*js) == typeid(TupleBPS)) + { + fStats.fMsgRcvCnt += js->blockTouched(); + } + else + { + fStats.fMsgRcvCnt += js->msgsRcvdCount(); + } + + fStats.fMsgBytesIn += js->msgBytesIn(); + fStats.fMsgBytesOut += js->msgBytesOut(); + + uint64_t skipCnt = 0; + + if (typeid(*js) == typeid(pColStep)) + skipCnt = (dynamic_cast(js))->blksSkipped(); + else if (typeid(*js) == typeid(pColScanStep)) + skipCnt = (dynamic_cast(js))->blksSkipped(); + else if (typeid(*js) == typeid(TupleBPS)) + skipCnt = (dynamic_cast(js))->blksSkipped(); + + fStats.fCPBlocksSkipped += skipCnt; + ++pIter; + } + + if ((!fProject.empty()) && extendedStats) + { + DeliveredTableMap::iterator dsi = fDeliveredTables.begin(); + + while (dsi != fDeliveredTables.end()) + { + js = dynamic_cast(dsi->second.get()); + string ei; + int max = 0; + ei = js->extendedInfo(); + + while (max < 4 && ei.size() <= 10) + { + ei = js->extendedInfo(); + max++; + } + + fExtendedInfo += ei; + fMiniInfo += js->miniInfo() + "\n"; + + ++dsi; + } + } + } + catch (exception& ex) + { + cerr << "JobList::querySummary: exception caught: " << ex.what() << endl; return; + } + catch (...) + { + cerr << "JobList::querySummary: exception caught!" << endl; + return; + } + + return; } // @bug 828. Added additional information to the graph at the end of execution void JobList::graph(uint32_t sessionID) { - // Graphic view draw - ostringstream oss; - struct timeval tvbuf; - gettimeofday(&tvbuf, 0); - struct tm tmbuf; - localtime_r(reinterpret_cast(&tvbuf.tv_sec), &tmbuf); - oss << "jobstep_results." << setfill('0') - << setw(4) << (tmbuf.tm_year + 1900) - << setw(2) << (tmbuf.tm_mon + 1) - << setw(2) << (tmbuf.tm_mday) - << setw(2) << (tmbuf.tm_hour) - << setw(2) << (tmbuf.tm_min) - << setw(2) << (tmbuf.tm_sec) - << setw(6) << (tvbuf.tv_usec) - << ".dot"; - string jsrname(oss.str()); - //it's too late to set this here. ExeMgr has already returned ei to dm... - //fExtendedInfo += "Graphs are in " + jsrname; - std::ofstream dotFile(jsrname.c_str(), std::ios::out); - dotFile << "digraph G {" << std::endl; - JobStepVector::iterator qsi; - JobStepVector::iterator psi; - DeliveredTableMap::iterator dsi; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - CalpontSystemCatalog::TableColName tcn; - uint64_t outSize = 0; - uint64_t msgs = 0; - uint64_t pio = 0; - int ctn = 0; - bool diskIo = false; - uint64_t saveTime = 0; - uint64_t loadTime = 0; + // Graphic view draw + ostringstream oss; + struct timeval tvbuf; + gettimeofday(&tvbuf, 0); + struct tm tmbuf; + localtime_r(reinterpret_cast(&tvbuf.tv_sec), &tmbuf); + oss << "jobstep_results." << setfill('0') << setw(4) << (tmbuf.tm_year + 1900) << setw(2) + << (tmbuf.tm_mon + 1) << setw(2) << (tmbuf.tm_mday) << setw(2) << (tmbuf.tm_hour) << setw(2) + << (tmbuf.tm_min) << setw(2) << (tmbuf.tm_sec) << setw(6) << (tvbuf.tv_usec) << ".dot"; + string jsrname(oss.str()); + // it's too late to set this here. ExeMgr has already returned ei to dm... + // fExtendedInfo += "Graphs are in " + jsrname; + std::ofstream dotFile(jsrname.c_str(), std::ios::out); + dotFile << "digraph G {" << std::endl; + JobStepVector::iterator qsi; + JobStepVector::iterator psi; + DeliveredTableMap::iterator dsi; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::TableColName tcn; + uint64_t outSize = 0; + uint64_t msgs = 0; + uint64_t pio = 0; + int ctn = 0; + bool diskIo = false; + uint64_t saveTime = 0; + uint64_t loadTime = 0; - // merge in the subquery steps - JobStepVector querySteps = fQuery; + // merge in the subquery steps + JobStepVector querySteps = fQuery; + { + SubQueryStep* subquery = NULL; + qsi = querySteps.begin(); + + while (qsi != querySteps.end()) { - SubQueryStep* subquery = NULL; + if ((subquery = dynamic_cast(qsi->get())) != NULL) + { + querySteps.erase(qsi); + JobStepVector subSteps = subquery->subJoblist()->querySteps(); + querySteps.insert(querySteps.end(), subSteps.begin(), subSteps.end()); qsi = querySteps.begin(); + } + else + { + qsi++; + } + } + } - while (qsi != querySteps.end()) + for (qsi = querySteps.begin(); qsi != querySteps.end(); ctn++, qsi++) + { + // HashJoinStep* hjs = 0; + + // if (dynamic_cast(qsi->get()) != NULL) + // continue; + + // @bug 1042. clear column name first at each loop + tcn.column = ""; + + uint16_t stepidIn = qsi->get()->stepId(); + dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; + + // @Bug 1033. colName was being called for dictionary steps that don't have column names. + // Added if condition below. + if (typeid(*(qsi->get())) == typeid(pColScanStep) || typeid(*(qsi->get())) == typeid(pColStep)) + tcn = csc->colName(qsi->get()->oid()); + + dotFile << "("; + + if (!tcn.column.empty()) + dotFile << tcn.column << "/"; + + if (typeid(*(qsi->get())) == typeid(TupleBPS)) + { + BatchPrimitive* bps = dynamic_cast(qsi->get()); + OIDVector projectOids = bps->getProjectOids(); + + if (projectOids.size() > 0) + { + dotFile << "\\l"; + dotFile << "PC:"; + dotFile << "\\l"; + + for (unsigned int i = 0; i < projectOids.size(); i++) { - if ((subquery = dynamic_cast(qsi->get())) != NULL) - { - querySteps.erase(qsi); - JobStepVector subSteps = subquery->subJoblist()->querySteps(); - querySteps.insert(querySteps.end(), subSteps.begin(), subSteps.end()); - qsi = querySteps.begin(); - } - else - { - qsi++; - } + tcn = csc->colName(projectOids[i]); + + if (!tcn.column.empty()) + dotFile << tcn.column << " "; + + if ((i + 1) % 3 == 0) + dotFile << "\\l"; } + } + else + { + tcn = csc->colName(qsi->get()->oid()); + dotFile << tcn.column << "/"; + } + } + else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) + { + tcn.schema = qsi->get()->schema(); + tcn.table = qsi->get()->alias(); } - for (qsi = querySteps.begin(); qsi != querySteps.end(); ctn++, qsi++) + dotFile << JSTimeStamp::tsdiffstr(qsi->get()->dlTimes.EndOfInputTime(), + qsi->get()->dlTimes.FirstReadTime()) + << "s"; + + dotFile << ")"; + + // oracle predict card + dotFile << "\\l#: " << (*qsi)->cardinality(); + + if (typeid(*(qsi->get())) == typeid(pColStep)) { - //HashJoinStep* hjs = 0; + dotFile << "\"" + << " shape=ellipse"; + } + else if (typeid(*(qsi->get())) == typeid(pColScanStep)) + { + dotFile << "\"" + << " shape=box"; + } + else if (typeid(*(qsi->get())) == typeid(TupleBPS)) + { + BatchPrimitive* bps = dynamic_cast(qsi->get()); - //if (dynamic_cast(qsi->get()) != NULL) - // continue; - - // @bug 1042. clear column name first at each loop - tcn.column = ""; - - uint16_t stepidIn = qsi->get()->stepId(); - dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; - - // @Bug 1033. colName was being called for dictionary steps that don't have column names. - // Added if condition below. - if ( typeid(*(qsi->get())) == typeid(pColScanStep) || - typeid(*(qsi->get())) == typeid(pColStep)) - tcn = csc->colName(qsi->get()->oid()); - - dotFile << "("; - - if (!tcn.column.empty()) - dotFile << tcn.column << "/"; + // if BPS not run, a BucketReuseStep was substituted, so draw dashed + if (bps->wasStepRun()) + { + dotFile << "\"" + << " shape=box style=bold"; if (typeid(*(qsi->get())) == typeid(TupleBPS)) - { - BatchPrimitive* bps = dynamic_cast(qsi->get()); - OIDVector projectOids = bps->getProjectOids(); + dotFile << " peripheries=2"; + } + else + dotFile << "\"" + << " shape=box style=dashed"; + } + else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) + { + dotFile << "\"" + << " shape=box style=dashed"; + } + else if (typeid(*(qsi->get())) == typeid(TupleAggregateStep)) + { + dotFile << "\"" + << " shape=triangle orientation=180"; + } + else if (typeid(*(qsi->get())) == typeid(TupleAnnexStep)) + { + dotFile << "\"" + << " shape=star"; + } + else if (typeid(*(qsi->get())) == typeid(WindowFunctionStep)) + { + dotFile << "\"" + << " shape=triangle orientation=180 peripheries=2"; + } + else if (typeid(*(qsi->get())) == typeid(TupleHashJoinStep)) + { + dotFile << "\""; + dotFile << " shape=diamond style=dashed peripheries=2"; + } + else if (typeid(*(qsi->get())) == typeid(TupleUnion)) + { + dotFile << "\"" + << " shape=triangle"; + } + else if (typeid(*(qsi->get())) == typeid(pDictionaryStep)) + { + dotFile << "\"" + << " shape=trapezium"; + } + else if (typeid(*(qsi->get())) == typeid(FilterStep)) + { + dotFile << "\"" + << " shape=house orientation=180"; + } + else if (typeid(*(qsi->get())) == typeid(TupleBPS)) + { + dotFile << "\"" + << " shape=box style=bold"; + dotFile << " peripheries=2"; + } + else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) + { + dotFile << "\"" + << " shape=box style=bold"; + dotFile << " peripheries=2"; + } + else + dotFile << "\""; - if ( projectOids.size() > 0 ) + dotFile << "]" << endl; + + // msgsRecived, physicalIO, cacheIO + msgs = qsi->get()->msgsRcvdCount(); + pio = qsi->get()->phyIOCount(); + + for (unsigned int i = 0; i < qsi->get()->outputAssociation().outSize(); i++) + { + ptrdiff_t dloutptr = 0; + DataList_t* dlout; + StrDataList* sdl; + // TupleDataList* tdl; + + if ((dlout = qsi->get()->outputAssociation().outAt(i)->dataList())) + { + dloutptr = (ptrdiff_t)dlout; + outSize = dlout->totalSize(); + diskIo = dlout->totalDiskIoTime(saveTime, loadTime); + } + else if ((sdl = qsi->get()->outputAssociation().outAt(i)->stringDataList())) + { + dloutptr = (ptrdiff_t)sdl; + outSize = sdl->totalSize(); + diskIo = sdl->totalDiskIoTime(saveTime, loadTime); + } + + // if HashJoinStep, determine if output fifo was cached to disk + bool hjTempDiskFlag = false; + + for (unsigned int k = 0; k < querySteps.size(); k++) + { + uint16_t stepidOut = querySteps[k].get()->stepId(); + JobStepAssociation queryInputSA = querySteps[k].get()->inputAssociation(); + + for (unsigned int j = 0; j < queryInputSA.outSize(); j++) + { + ptrdiff_t dlinptr = 0; + DataList_t* dlin = queryInputSA.outAt(j)->dataList(); + StrDataList* sdl = 0; + + if (dlin) + dlinptr = (ptrdiff_t)dlin; + else if ((sdl = queryInputSA.outAt(j)->stringDataList())) + { + dlinptr = (ptrdiff_t)sdl; + } + + if (dloutptr == dlinptr) + { + dotFile << stepidIn << " -> " << stepidOut; + + dotFile << " [label=\" r: " << outSize; + + if (hjTempDiskFlag) { - dotFile << "\\l"; - dotFile << "PC:"; - dotFile << "\\l"; - - for ( unsigned int i = 0; i < projectOids.size(); i++ ) - { - tcn = csc->colName(projectOids[i]); - - if (!tcn.column.empty()) - dotFile << tcn.column << " "; - - if ( (i + 1) % 3 == 0 ) - dotFile << "\\l"; - } - } - else - { - tcn = csc->colName(qsi->get()->oid()); - dotFile << tcn.column << "/"; - } - } - else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) - { - tcn.schema = qsi->get()->schema(); - tcn.table = qsi->get()->alias(); - } - - dotFile << JSTimeStamp::tsdiffstr(qsi->get()->dlTimes.EndOfInputTime(), - qsi->get()->dlTimes.FirstReadTime()) << "s"; - - dotFile << ")"; - - // oracle predict card - dotFile << "\\l#: " << (*qsi)->cardinality(); - - if (typeid(*(qsi->get())) == typeid(pColStep)) - { - dotFile << "\"" << " shape=ellipse"; - } - else if (typeid(*(qsi->get())) == typeid(pColScanStep)) - { - dotFile << "\"" << " shape=box"; - } - else if (typeid(*(qsi->get())) == typeid(TupleBPS)) - { - BatchPrimitive* bps = - dynamic_cast(qsi->get()); - - // if BPS not run, a BucketReuseStep was substituted, so draw dashed - if (bps->wasStepRun()) - { - dotFile << "\"" << " shape=box style=bold"; - - if (typeid(*(qsi->get())) == typeid(TupleBPS)) - dotFile << " peripheries=2"; - } - else - dotFile << "\"" << " shape=box style=dashed"; - } - else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) - { - dotFile << "\"" << " shape=box style=dashed"; - } - else if (typeid(*(qsi->get())) == typeid(TupleAggregateStep)) - { - dotFile << "\"" << " shape=triangle orientation=180"; - } - else if (typeid(*(qsi->get())) == typeid(TupleAnnexStep)) - { - dotFile << "\"" << " shape=star"; - } - else if (typeid(*(qsi->get())) == typeid(WindowFunctionStep)) - { - dotFile << "\"" << " shape=triangle orientation=180 peripheries=2"; - } - else if (typeid(*(qsi->get())) == typeid(TupleHashJoinStep)) - { - dotFile << "\""; - dotFile << " shape=diamond style=dashed peripheries=2"; - } - else if (typeid(*(qsi->get())) == typeid(TupleUnion) ) - { - dotFile << "\"" << " shape=triangle"; - } - else if (typeid(*(qsi->get())) == typeid(pDictionaryStep)) - { - dotFile << "\"" << " shape=trapezium"; - } - else if (typeid(*(qsi->get())) == typeid(FilterStep)) - { - dotFile << "\"" << " shape=house orientation=180"; - } - else if (typeid(*(qsi->get())) == typeid(TupleBPS)) - { - dotFile << "\"" << " shape=box style=bold"; - dotFile << " peripheries=2"; - } - else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) - { - dotFile << "\"" << " shape=box style=bold"; - dotFile << " peripheries=2"; - } - else - dotFile << "\""; - - dotFile << "]" << endl; - - // msgsRecived, physicalIO, cacheIO - msgs = qsi->get()->msgsRcvdCount(); - pio = qsi->get()->phyIOCount(); - - for (unsigned int i = 0; i < qsi->get()->outputAssociation().outSize(); i++) - { - ptrdiff_t dloutptr = 0; - DataList_t* dlout; - StrDataList* sdl; -// TupleDataList* tdl; - - if ((dlout = qsi->get()->outputAssociation().outAt(i)->dataList())) - { - dloutptr = (ptrdiff_t)dlout; - outSize = dlout->totalSize(); - diskIo = dlout->totalDiskIoTime(saveTime, loadTime); - } - else if ((sdl = qsi->get()->outputAssociation().outAt(i)->stringDataList())) - { - dloutptr = (ptrdiff_t)sdl; - outSize = sdl->totalSize(); - diskIo = sdl->totalDiskIoTime(saveTime, loadTime); + dotFile << "*"; } - // if HashJoinStep, determine if output fifo was cached to disk - bool hjTempDiskFlag = false; + dotFile << "\\l"; - for (unsigned int k = 0; k < querySteps.size(); k++) + if (msgs != 0) { - uint16_t stepidOut = querySteps[k].get()->stepId(); - JobStepAssociation queryInputSA = querySteps[k].get()->inputAssociation(); + dotFile << " m: " << msgs << "\\l"; - for (unsigned int j = 0; j < queryInputSA.outSize(); j++) - { - ptrdiff_t dlinptr = 0; - DataList_t* dlin = queryInputSA.outAt(j)->dataList(); - StrDataList* sdl = 0; + if (typeid(*(qsi->get())) == typeid(TupleBPS)) + { + dotFile << " b: " << qsi->get()->blockTouched() << "\\l"; + } - if (dlin) - dlinptr = (ptrdiff_t)dlin; - else if ((sdl = queryInputSA.outAt(j)->stringDataList())) - { - dlinptr = (ptrdiff_t)sdl; - } - - if (dloutptr == dlinptr) - { - dotFile << stepidIn << " -> " << stepidOut; - - dotFile << " [label=\" r: " << outSize; - - if (hjTempDiskFlag) - { - dotFile << "*"; - } - - dotFile << "\\l"; - - if (msgs != 0) - { - dotFile << " m: " << msgs << "\\l"; - - if (typeid(*(qsi->get())) == typeid(TupleBPS)) - { - dotFile << " b: " << qsi->get()->blockTouched() << "\\l"; - } - - dotFile << " p: " << pio << "\\l"; - } - - if (diskIo == true) - { - dotFile << " wr: " << saveTime << "s\\l"; - dotFile << " rd: " << loadTime << "s\\l"; - } - - dotFile << "\"]" << endl; - } - } + dotFile << " p: " << pio << "\\l"; } - for (psi = fProject.begin(); psi < fProject.end(); psi++) + if (diskIo == true) { - uint16_t stepidOut = psi->get()->stepId(); - JobStepAssociation projectInputSA = psi->get()->inputAssociation(); - - for (unsigned int j = 0; j < projectInputSA.outSize(); j++) - { - ptrdiff_t dlinptr; - DataList_t* dlin = projectInputSA.outAt(j)->dataList(); - StrDataList* sdl = 0; - - if (dlin) - { - dlinptr = (ptrdiff_t)dlin; - } - else - { - sdl = projectInputSA.outAt(j)->stringDataList(); - dlinptr = (ptrdiff_t)sdl; - } - - if (dloutptr == dlinptr) - { - dotFile << stepidIn << " -> " << stepidOut; - - dotFile << " [label=\" r: " << outSize; - - if (hjTempDiskFlag) - { - dotFile << "*"; - } - - dotFile << "\\l"; - - if (msgs != 0) - { - dotFile << " m: " << msgs << "\\l"; - dotFile << " p: " << pio << "\\l"; - } - - if (diskIo == true) - { - dotFile << " wr: " << saveTime << "s\\l"; - dotFile << " rd: " << loadTime << "s\\l"; - } - - dotFile << "\"]" << endl; - } - } + dotFile << " wr: " << saveTime << "s\\l"; + dotFile << " rd: " << loadTime << "s\\l"; } - } - //@Bug 921 - if (typeid(*(qsi->get())) == typeid(TupleBPS)) - { - BatchPrimitive* bps = dynamic_cast(qsi->get()); - CalpontSystemCatalog::OID tableOIDProject = bps->tableOid(); - - if (bps->getOutputType() == TABLE_BAND || bps->getOutputType() == ROW_GROUP) - { - outSize = bps->getRows(); - - for (dsi = fDeliveredTables.begin(); dsi != fDeliveredTables.end(); dsi++) - { - BatchPrimitive* bpsDelivery = dynamic_cast((dsi->second).get()); - TupleHashJoinStep* thjDelivery = dynamic_cast((dsi->second).get()); - - if (bpsDelivery) - { - CalpontSystemCatalog::OID tableOID = bpsDelivery->tableOid(); - dotFile << tableOID << " [label=" << bpsDelivery->alias() << - " shape=plaintext]" << endl; - JobStepAssociation deliveryInputSA = bpsDelivery->inputAssociation(); - - if (tableOIDProject == tableOID) - { - dotFile << stepidIn << " -> " << tableOID; - - dotFile << " [label=\" r: " << outSize << "\\l"; - dotFile << " m: " << bpsDelivery->msgsRcvdCount() << "\\l"; - dotFile << " b: " << bpsDelivery->blockTouched() << "\\l"; - dotFile << " p: " << bpsDelivery->phyIOCount() << "\\l"; - dotFile << "\"]" << endl; - } - } - else if (thjDelivery) - { - CalpontSystemCatalog::OID tableOID = thjDelivery->tableOid(); - dotFile << tableOID << " [label=" << "vtable" << " shape=plaintext]" << endl; - JobStepAssociation deliveryInputSA = thjDelivery->inputAssociation(); - - if (tableOIDProject == tableOID) - { - dotFile << stepidIn << " -> " << tableOID; - - dotFile << " [label=\" r: " << outSize << "\\l"; - dotFile << " m: " << thjDelivery->msgsRcvdCount() << "\\l"; - dotFile << " b: " << thjDelivery->blockTouched() << "\\l"; - dotFile << " p: " << thjDelivery->phyIOCount() << "\\l"; - dotFile << "\"]" << endl; - } - } - } - } - } - else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) - { - outSize = dynamic_cast(qsi->get())->getRows(); - dotFile << "0" << " [label=" << qsi->get()->alias() << " shape=plaintext]" << endl; - dotFile << stepidIn << " -> 0"; - dotFile << " [label=\" r: " << outSize << "\\l"; dotFile << "\"]" << endl; + } } + } + + for (psi = fProject.begin(); psi < fProject.end(); psi++) + { + uint16_t stepidOut = psi->get()->stepId(); + JobStepAssociation projectInputSA = psi->get()->inputAssociation(); + + for (unsigned int j = 0; j < projectInputSA.outSize(); j++) + { + ptrdiff_t dlinptr; + DataList_t* dlin = projectInputSA.outAt(j)->dataList(); + StrDataList* sdl = 0; + + if (dlin) + { + dlinptr = (ptrdiff_t)dlin; + } + else + { + sdl = projectInputSA.outAt(j)->stringDataList(); + dlinptr = (ptrdiff_t)sdl; + } + + if (dloutptr == dlinptr) + { + dotFile << stepidIn << " -> " << stepidOut; + + dotFile << " [label=\" r: " << outSize; + + if (hjTempDiskFlag) + { + dotFile << "*"; + } + + dotFile << "\\l"; + + if (msgs != 0) + { + dotFile << " m: " << msgs << "\\l"; + dotFile << " p: " << pio << "\\l"; + } + + if (diskIo == true) + { + dotFile << " wr: " << saveTime << "s\\l"; + dotFile << " rd: " << loadTime << "s\\l"; + } + + dotFile << "\"]" << endl; + } + } + } } - for (psi = fProject.begin(), ctn = 0; psi != fProject.end(); ctn++, psi++) + //@Bug 921 + if (typeid(*(qsi->get())) == typeid(TupleBPS)) { - tcn.column = ""; - uint16_t stepidIn = psi->get()->stepId(); - dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; - tcn = csc->colName(psi->get()->oid()); - dotFile << "("; - BatchPrimitive* bps = 0; + BatchPrimitive* bps = dynamic_cast(qsi->get()); + CalpontSystemCatalog::OID tableOIDProject = bps->tableOid(); - if (typeid(*(psi->get())) == typeid(TupleBPS)) - { - bps = dynamic_cast(psi->get()); - OIDVector projectOids = bps->getProjectOids(); + if (bps->getOutputType() == TABLE_BAND || bps->getOutputType() == ROW_GROUP) + { + outSize = bps->getRows(); - for ( unsigned int i = 0; i < projectOids.size(); i++ ) - { - tcn = csc->colName(projectOids[i]); - - if (!tcn.column.empty()) - { - dotFile << tcn.column; - - if ( i != (projectOids.size() - 1) ) - dotFile << "/ "; - } - - if ( (i + 1) % 3 == 0 ) - dotFile << "\\l"; - } - } - else - { - if (!tcn.column.empty()) - dotFile << tcn.column << "/"; - } - - dotFile << JSTimeStamp::tsdiffstr(psi->get()->dlTimes.EndOfInputTime(), psi->get()->dlTimes.FirstReadTime()) << "s"; - dotFile << ")"; - - if (typeid(*(psi->get())) == typeid(pColStep)) - { - dotFile << "\"" << " shape=ellipse"; - } - else if (typeid(*(psi->get())) == typeid(pColScanStep)) - { - dotFile << "\"" << " shape=box"; - } - else if (typeid(*(psi->get())) == typeid(TupleBPS)) - { - dotFile << "\"" << " shape=box style=bold"; - - if (typeid(*(psi->get())) == typeid(TupleBPS)) - dotFile << " peripheries=2"; - } - else if (typeid(*(psi->get())) == typeid(pDictionaryStep)) - { - dotFile << "\"" << " shape=trapezium"; - } - else if (typeid(*(psi->get())) == typeid(PassThruStep)) - { - dotFile << "\"" << " shape=octagon"; - } - else if (typeid(*(psi->get())) == typeid(FilterStep)) - { - dotFile << "\"" << " shape=house orientation=180"; - } - else - dotFile << "\""; - - dotFile << "]" << endl; - - // msgsRecived, physicalIO, cacheIO - msgs = psi->get()->msgsRcvdCount(); - pio = psi->get()->phyIOCount(); - - CalpontSystemCatalog::OID tableOIDProject = 0; - - if (bps) - tableOIDProject = bps->tableOid(); - - //@Bug 921 for (dsi = fDeliveredTables.begin(); dsi != fDeliveredTables.end(); dsi++) { - BatchPrimitive* dbps = dynamic_cast((dsi->second).get()); + BatchPrimitive* bpsDelivery = dynamic_cast((dsi->second).get()); + TupleHashJoinStep* thjDelivery = dynamic_cast((dsi->second).get()); - if (dbps) + if (bpsDelivery) + { + CalpontSystemCatalog::OID tableOID = bpsDelivery->tableOid(); + dotFile << tableOID << " [label=" << bpsDelivery->alias() << " shape=plaintext]" << endl; + JobStepAssociation deliveryInputSA = bpsDelivery->inputAssociation(); + + if (tableOIDProject == tableOID) { - outSize = dbps->getRows(); - CalpontSystemCatalog::OID tableOID = dbps->tableOid(); - dotFile << tableOID << " [label=" << dbps->alias() << " shape=plaintext]" << endl; - JobStepAssociation deliveryInputSA = dbps->inputAssociation(); + dotFile << stepidIn << " -> " << tableOID; - if ( tableOIDProject == tableOID ) - { - dotFile << stepidIn << " -> " << tableOID; - - dotFile << " [label=\" r: " << outSize << "\\l"; - dotFile << " m: " << dbps->msgsRcvdCount() << "\\l"; - dotFile << " b: " << dbps->blockTouched() << "\\l"; - dotFile << " p: " << dbps->phyIOCount() << "\\l"; - dotFile << "\"]" << endl; - - } + dotFile << " [label=\" r: " << outSize << "\\l"; + dotFile << " m: " << bpsDelivery->msgsRcvdCount() << "\\l"; + dotFile << " b: " << bpsDelivery->blockTouched() << "\\l"; + dotFile << " p: " << bpsDelivery->phyIOCount() << "\\l"; + dotFile << "\"]" << endl; } + } + else if (thjDelivery) + { + CalpontSystemCatalog::OID tableOID = thjDelivery->tableOid(); + dotFile << tableOID << " [label=" + << "vtable" + << " shape=plaintext]" << endl; + JobStepAssociation deliveryInputSA = thjDelivery->inputAssociation(); + + if (tableOIDProject == tableOID) + { + dotFile << stepidIn << " -> " << tableOID; + + dotFile << " [label=\" r: " << outSize << "\\l"; + dotFile << " m: " << thjDelivery->msgsRcvdCount() << "\\l"; + dotFile << " b: " << thjDelivery->blockTouched() << "\\l"; + dotFile << " p: " << thjDelivery->phyIOCount() << "\\l"; + dotFile << "\"]" << endl; + } + } } + } + } + else if (typeid(*(qsi->get())) == typeid(CrossEngineStep)) + { + outSize = dynamic_cast(qsi->get())->getRows(); + dotFile << "0" + << " [label=" << qsi->get()->alias() << " shape=plaintext]" << endl; + dotFile << stepidIn << " -> 0"; + dotFile << " [label=\" r: " << outSize << "\\l"; + dotFile << "\"]" << endl; + } + } + + for (psi = fProject.begin(), ctn = 0; psi != fProject.end(); ctn++, psi++) + { + tcn.column = ""; + uint16_t stepidIn = psi->get()->stepId(); + dotFile << stepidIn << " [label=\"st_" << stepidIn << " "; + tcn = csc->colName(psi->get()->oid()); + dotFile << "("; + BatchPrimitive* bps = 0; + + if (typeid(*(psi->get())) == typeid(TupleBPS)) + { + bps = dynamic_cast(psi->get()); + OIDVector projectOids = bps->getProjectOids(); + + for (unsigned int i = 0; i < projectOids.size(); i++) + { + tcn = csc->colName(projectOids[i]); + + if (!tcn.column.empty()) + { + dotFile << tcn.column; + + if (i != (projectOids.size() - 1)) + dotFile << "/ "; + } + + if ((i + 1) % 3 == 0) + dotFile << "\\l"; + } + } + else + { + if (!tcn.column.empty()) + dotFile << tcn.column << "/"; } - dotFile << "}" << std::endl; - dotFile.close(); + dotFile << JSTimeStamp::tsdiffstr(psi->get()->dlTimes.EndOfInputTime(), + psi->get()->dlTimes.FirstReadTime()) + << "s"; + dotFile << ")"; + + if (typeid(*(psi->get())) == typeid(pColStep)) + { + dotFile << "\"" + << " shape=ellipse"; + } + else if (typeid(*(psi->get())) == typeid(pColScanStep)) + { + dotFile << "\"" + << " shape=box"; + } + else if (typeid(*(psi->get())) == typeid(TupleBPS)) + { + dotFile << "\"" + << " shape=box style=bold"; + + if (typeid(*(psi->get())) == typeid(TupleBPS)) + dotFile << " peripheries=2"; + } + else if (typeid(*(psi->get())) == typeid(pDictionaryStep)) + { + dotFile << "\"" + << " shape=trapezium"; + } + else if (typeid(*(psi->get())) == typeid(PassThruStep)) + { + dotFile << "\"" + << " shape=octagon"; + } + else if (typeid(*(psi->get())) == typeid(FilterStep)) + { + dotFile << "\"" + << " shape=house orientation=180"; + } + else + dotFile << "\""; + + dotFile << "]" << endl; + + // msgsRecived, physicalIO, cacheIO + msgs = psi->get()->msgsRcvdCount(); + pio = psi->get()->phyIOCount(); + + CalpontSystemCatalog::OID tableOIDProject = 0; + + if (bps) + tableOIDProject = bps->tableOid(); + + //@Bug 921 + for (dsi = fDeliveredTables.begin(); dsi != fDeliveredTables.end(); dsi++) + { + BatchPrimitive* dbps = dynamic_cast((dsi->second).get()); + + if (dbps) + { + outSize = dbps->getRows(); + CalpontSystemCatalog::OID tableOID = dbps->tableOid(); + dotFile << tableOID << " [label=" << dbps->alias() << " shape=plaintext]" << endl; + JobStepAssociation deliveryInputSA = dbps->inputAssociation(); + + if (tableOIDProject == tableOID) + { + dotFile << stepidIn << " -> " << tableOID; + + dotFile << " [label=\" r: " << outSize << "\\l"; + dotFile << " m: " << dbps->msgsRcvdCount() << "\\l"; + dotFile << " b: " << dbps->blockTouched() << "\\l"; + dotFile << " p: " << dbps->phyIOCount() << "\\l"; + dotFile << "\"]" << endl; + } + } + } + } + + dotFile << "}" << std::endl; + dotFile.close(); } void JobList::validate() const { -// uint32_t i; -// DeliveredTableMap::const_iterator it; + // uint32_t i; + // DeliveredTableMap::const_iterator it; - /* Make sure there's at least one query step and that they're the right type */ - idbassert(fQuery.size() > 0); -// for (i = 0; i < fQuery.size(); i++) -// idbassert(dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) || -// dynamic_cast(fQuery[i].get()) -// ); -// -// /* Make sure there's at least one projected table and that they're the right type */ -// idbassert(fDeliveredTables.size() > 0); -// for (i = 0; i < fProject.size(); i++) -// idbassert(dynamic_cast(fProject[i].get())); -// -// /* Check that all JobSteps use the right status pointer */ -// for (i = 0; i < fQuery.size(); i++) { -// idbassert(fQuery[i]->errorInfo().get() == errorInfo().get()); -// } -// for (i = 0; i < fProject.size(); i++) { -// idbassert(fProject[i]->errorInfo().get() == errorInfo().get()); -// } -// for (it = fDeliveredTables.begin(); it != fDeliveredTables.end(); ++it) { -// idbassert(it->second->errorInfo().get() == errorInfo().get()); -// } + /* Make sure there's at least one query step and that they're the right type */ + idbassert(fQuery.size() > 0); + // for (i = 0; i < fQuery.size(); i++) + // idbassert(dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) || + // dynamic_cast(fQuery[i].get()) + // ); + // + // /* Make sure there's at least one projected table and that they're the right type */ + // idbassert(fDeliveredTables.size() > 0); + // for (i = 0; i < fProject.size(); i++) + // idbassert(dynamic_cast(fProject[i].get())); + // + // /* Check that all JobSteps use the right status pointer */ + // for (i = 0; i < fQuery.size(); i++) { + // idbassert(fQuery[i]->errorInfo().get() == errorInfo().get()); + // } + // for (i = 0; i < fProject.size(); i++) { + // idbassert(fProject[i]->errorInfo().get() == errorInfo().get()); + // } + // for (it = fDeliveredTables.begin(); it != fDeliveredTables.end(); ++it) { + // idbassert(it->second->errorInfo().get() == errorInfo().get()); + // } } void TupleJobList::validate() const { - uint32_t i, j; - DeliveredTableMap::const_iterator it; + uint32_t i, j; + DeliveredTableMap::const_iterator it; - idbassert(fQuery.size() > 0); + idbassert(fQuery.size() > 0); - for (i = 0; i < fQuery.size(); i++) - { - idbassert(dynamic_cast(fQuery[i].get()) || - dynamic_cast(fQuery[i].get()) || - dynamic_cast(fQuery[i].get()) || - dynamic_cast(fQuery[i].get()) || - dynamic_cast(fQuery[i].get()) - ); - } + for (i = 0; i < fQuery.size(); i++) + { + idbassert(dynamic_cast(fQuery[i].get()) || dynamic_cast(fQuery[i].get()) || + dynamic_cast(fQuery[i].get()) || + dynamic_cast(fQuery[i].get()) || dynamic_cast(fQuery[i].get())); + } - /* Duplicate check */ - for (i = 0; i < fQuery.size(); i++) - for (j = i + 1; j < fQuery.size(); j++) - idbassert(fQuery[i].get() != fQuery[j].get()); + /* Duplicate check */ + for (i = 0; i < fQuery.size(); i++) + for (j = i + 1; j < fQuery.size(); j++) + idbassert(fQuery[i].get() != fQuery[j].get()); - /**** XXXPAT: An indication of a possible problem: The next assertion fails - occasionally */ -// idbassert(fDeliveredTables.begin()->first == 100); //fake oid for the vtable -// cout << "Delivered TableOID is " << fDeliveredTables.begin()->first << endl; - idbassert(fProject.size() == 0); - idbassert(fDeliveredTables.size() == 1); - idbassert(dynamic_cast(fDeliveredTables.begin()->second.get())); + /**** XXXPAT: An indication of a possible problem: The next assertion fails + occasionally */ + // idbassert(fDeliveredTables.begin()->first == 100); //fake oid for the vtable + // cout << "Delivered TableOID is " << fDeliveredTables.begin()->first << endl; + idbassert(fProject.size() == 0); + idbassert(fDeliveredTables.size() == 1); + idbassert(dynamic_cast(fDeliveredTables.begin()->second.get())); - /* Check that all JobSteps use the right status pointer */ - for (i = 0; i < fQuery.size(); i++) - idbassert(fQuery[i]->errorInfo().get() == errorInfo().get()); + /* Check that all JobSteps use the right status pointer */ + for (i = 0; i < fQuery.size(); i++) + idbassert(fQuery[i]->errorInfo().get() == errorInfo().get()); - for (i = 0; i < fProject.size(); i++) - idbassert(fProject[i]->errorInfo().get() == errorInfo().get()); + for (i = 0; i < fProject.size(); i++) + idbassert(fProject[i]->errorInfo().get() == errorInfo().get()); - for (it = fDeliveredTables.begin(); it != fDeliveredTables.end(); ++it) - idbassert(it->second->errorInfo().get() == errorInfo().get()); + for (it = fDeliveredTables.begin(); it != fDeliveredTables.end(); ++it) + idbassert(it->second->errorInfo().get() == errorInfo().get()); } void JobList::abort() { - uint32_t i; + uint32_t i; - //If we're not currently aborting, then start aborting... - if (atomicops::atomicCAS(&fAborted, 0, 1)) - { - for (i = 0; i < fQuery.size(); i++) - fQuery[i]->abort(); + // If we're not currently aborting, then start aborting... + if (atomicops::atomicCAS(&fAborted, 0, 1)) + { + for (i = 0; i < fQuery.size(); i++) + fQuery[i]->abort(); - for (i = 0; i < fProject.size(); i++) - fProject[i]->abort(); - - - } + for (i = 0; i < fProject.size(); i++) + fProject[i]->abort(); + } } void JobList::abortOnLimit(JobStep* js) { - //If we're not currently aborting, then start aborting... - if (atomicops::atomicCAS(&fAborted, 0, 1)) + // If we're not currently aborting, then start aborting... + if (atomicops::atomicCAS(&fAborted, 0, 1)) + { + // @bug4848, enhance and unify limit handling. + for (uint32_t i = 0; i < fQuery.size(); i++) { - // @bug4848, enhance and unify limit handling. - for (uint32_t i = 0; i < fQuery.size(); i++) - { - if (fQuery[i].get() == js) - break; + if (fQuery[i].get() == js) + break; - fQuery[i]->abort(); - } + fQuery[i]->abort(); } + } } string JobList::toString() const { - uint32_t i; - string ret; + uint32_t i; + string ret; - ret = "Filter Steps:\n"; + ret = "Filter Steps:\n"; - for (i = 0; i < fQuery.size(); i++) - ret += fQuery[i]->toString(); + for (i = 0; i < fQuery.size(); i++) + ret += fQuery[i]->toString(); - //ret += "\nProjection Steps:\n"; - //for (i = 0; i < fProject.size(); i++) - // ret += fProject[i]->toString(); - ret += "\n"; - return ret; + // ret += "\nProjection Steps:\n"; + // for (i = 0; i < fProject.size(); i++) + // ret += fProject[i]->toString(); + ret += "\n"; + return ret; } TupleJobList::TupleJobList(bool isEM) : JobList(isEM), ds(NULL), moreData(true) @@ -1207,27 +1211,26 @@ TupleJobList::TupleJobList(bool isEM) : JobList(isEM), ds(NULL), moreData(true) TupleJobList::~TupleJobList() { - abort(); + abort(); } void TupleJobList::abort() { - if (fAborted == 0 && fIsRunning) - { - JobList::abort(); - messageqcpp::ByteStream bs; + if (fAborted == 0 && fIsRunning) + { + JobList::abort(); + messageqcpp::ByteStream bs; - if (ds && moreData) - while (ds->nextBand(bs)); - } + if (ds && moreData) + while (ds->nextBand(bs)) + ; + } } - -} +} // namespace joblist #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/joblist.h b/dbcon/joblist/joblist.h index ce85db6d3..61f498a27 100644 --- a/dbcon/joblist/joblist.h +++ b/dbcon/joblist/joblist.h @@ -18,7 +18,6 @@ // $Id: joblist.h 9623 2013-06-13 19:56:23Z rdempsey $ - /** @file */ #ifndef JOBLIST_JOBLIST_H_ @@ -36,9 +35,9 @@ #include "bytestream.h" #ifndef __GNUC__ -# ifndef __attribute__ -# define __attribute__(x) -# endif +#ifndef __attribute__ +#define __attribute__(x) +#endif #endif #if defined(_MSC_VER) && defined(JOBLIST_DLLEXPORT) @@ -49,7 +48,6 @@ namespace joblist { - typedef std::vector JobStepVector; typedef std::map DeliveredTableMap; @@ -63,204 +61,205 @@ typedef boost::shared_ptr SJLP; class JobList { -public: - explicit JobList(bool isEM = false); - virtual ~JobList(); - virtual int doQuery(); + public: + explicit JobList(bool isEM = false); + virtual ~JobList(); + virtual int doQuery(); - /* returns row count */ - virtual uint32_t projectTable(execplan::CalpontSystemCatalog::OID, messageqcpp::ByteStream&) = 0; - virtual int putEngineComm(DistributedEngineComm*); + /* returns row count */ + virtual uint32_t projectTable(execplan::CalpontSystemCatalog::OID, messageqcpp::ByteStream&) = 0; + virtual int putEngineComm(DistributedEngineComm*); - virtual void addQuery(const JobStepVector& query) - { - fQuery = query; - } - virtual void addProject(const JobStepVector& project) - { - fProject = project; - } - virtual void addDelivery(const DeliveredTableMap& delivery) - { - fDeliveredTables = delivery; - } - virtual void PutEngineComm(const DistributedEngineComm*) __attribute__((deprecated)) { } - virtual const DeliveredTableMap& deliveredTables() const - { - return fDeliveredTables; - } - virtual void querySummary(bool extendedStats); - virtual void graph(uint32_t sessionID); + virtual void addQuery(const JobStepVector& query) + { + fQuery = query; + } + virtual void addProject(const JobStepVector& project) + { + fProject = project; + } + virtual void addDelivery(const DeliveredTableMap& delivery) + { + fDeliveredTables = delivery; + } + virtual void PutEngineComm(const DistributedEngineComm*) __attribute__((deprecated)) + { + } + virtual const DeliveredTableMap& deliveredTables() const + { + return fDeliveredTables; + } + virtual void querySummary(bool extendedStats); + virtual void graph(uint32_t sessionID); - virtual const SErrorInfo& errorInfo() const - { - return errInfo; - } - virtual void errorInfo(SErrorInfo sp) - { - errInfo = sp; - } + virtual const SErrorInfo& errorInfo() const + { + return errInfo; + } + virtual void errorInfo(SErrorInfo sp) + { + errInfo = sp; + } - virtual uint32_t status() const - { - return errInfo->errCode; - } - virtual void status(uint32_t ec) - { - errInfo->errCode = ec; - } - virtual const std::string& errMsg() const - { - return errInfo->errMsg; - } - virtual void errMsg(const std::string& s) - { - errInfo->errMsg = s; - } + virtual uint32_t status() const + { + return errInfo->errCode; + } + virtual void status(uint32_t ec) + { + errInfo->errCode = ec; + } + virtual const std::string& errMsg() const + { + return errInfo->errMsg; + } + virtual void errMsg(const std::string& s) + { + errInfo->errMsg = s; + } - virtual execplan::CalpontSystemCatalog::OID* projectingTableOIDPtr() - { - return &projectingTableOID; - } + virtual execplan::CalpontSystemCatalog::OID* projectingTableOIDPtr() + { + return &projectingTableOID; + } - /** Does some light validation on the final joblist - * - * Currently only verifies that all JobSteps are not tuple oriented. - */ - virtual void validate() const; + /** Does some light validation on the final joblist + * + * Currently only verifies that all JobSteps are not tuple oriented. + */ + virtual void validate() const; - querystats::QueryStats& queryStats() - { - return fStats; - } - void queryStats(const querystats::QueryStats& stats) - { - fStats = stats; - } - const std::string& extendedInfo() const - { - return fExtendedInfo; - } - void extendedInfo(const std::string& extendedInfo) - { - fExtendedInfo = extendedInfo; - } - const std::string& miniInfo() const - { - return fMiniInfo; - } - void miniInfo(const std::string& miniInfo) - { - fMiniInfo = miniInfo; - } + querystats::QueryStats& queryStats() + { + return fStats; + } + void queryStats(const querystats::QueryStats& stats) + { + fStats = stats; + } + const std::string& extendedInfo() const + { + return fExtendedInfo; + } + void extendedInfo(const std::string& extendedInfo) + { + fExtendedInfo = extendedInfo; + } + const std::string& miniInfo() const + { + return fMiniInfo; + } + void miniInfo(const std::string& miniInfo) + { + fMiniInfo = miniInfo; + } - void addSubqueryJobList(const SJLP& sjl) - { - subqueryJoblists.push_back(sjl); - } + void addSubqueryJobList(const SJLP& sjl) + { + subqueryJoblists.push_back(sjl); + } - /** Stop the running query - * - * This notifies the joblist to abort the running query. It returns right away, not - * when the joblist has actually stopped. The caller may need to drain some data - * through projectTable() for the joblist to abort completely. - */ - EXPORT virtual void abort(); - EXPORT virtual bool aborted() - { - return (fAborted != 0); - } + /** Stop the running query + * + * This notifies the joblist to abort the running query. It returns right away, not + * when the joblist has actually stopped. The caller may need to drain some data + * through projectTable() for the joblist to abort completely. + */ + EXPORT virtual void abort(); + EXPORT virtual bool aborted() + { + return (fAborted != 0); + } - std::string toString() const; + std::string toString() const; - void priority(uint32_t p) - { - fPriority = p; - } - uint32_t priority() - { - return fPriority; - } + void priority(uint32_t p) + { + fPriority = p; + } + uint32_t priority() + { + return fPriority; + } - // @bug4848, enhance and unify limit handling. - EXPORT virtual void abortOnLimit(JobStep* js); + // @bug4848, enhance and unify limit handling. + EXPORT virtual void abortOnLimit(JobStep* js); - static void setPMsConfigured(int pms) - { - fPmsConfigured = pms; - } + static void setPMsConfigured(int pms) + { + fPmsConfigured = pms; + } -protected: - //defaults okay - //JobList(const JobList& rhs); - //JobList& operator=(const JobList& rhs); - bool fIsRunning; - bool fIsExeMgr; - int fPmsConnected; + protected: + // defaults okay + // JobList(const JobList& rhs); + // JobList& operator=(const JobList& rhs); + bool fIsRunning; + bool fIsExeMgr; + int fPmsConnected; - // Dirty pool kludge. Contains the number of PMs configured in Columnstore.xml. - // This kludge reduces the number of calls needed to config.Config, which are expensive. - static int fPmsConfigured; + // Dirty pool kludge. Contains the number of PMs configured in Columnstore.xml. + // This kludge reduces the number of calls needed to config.Config, which are expensive. + static int fPmsConfigured; - DeliveredTableMap fDeliveredTables; - execplan::CalpontSystemCatalog::OID projectingTableOID; //DeliveryWSDLs get a reference to this - SErrorInfo errInfo; - JobStepVector fQuery; - JobStepVector fProject; + DeliveredTableMap fDeliveredTables; + execplan::CalpontSystemCatalog::OID projectingTableOID; // DeliveryWSDLs get a reference to this + SErrorInfo errInfo; + JobStepVector fQuery; + JobStepVector fProject; - // @bug3438, get stats/trace from subqueries - querystats::QueryStats fStats; - std::string fExtendedInfo; - std::string fMiniInfo; - std::vector subqueryJoblists; + // @bug3438, get stats/trace from subqueries + querystats::QueryStats fStats; + std::string fExtendedInfo; + std::string fMiniInfo; + std::vector subqueryJoblists; - volatile uint32_t fAborted; + volatile uint32_t fAborted; - uint32_t fPriority; //higher #s = higher priority + uint32_t fPriority; // higher #s = higher priority }; class TupleJobList : public JobList { -public: - TupleJobList(bool isEM = false); - virtual ~TupleJobList(); + public: + TupleJobList(bool isEM = false); + virtual ~TupleJobList(); - EXPORT uint32_t projectTable(execplan::CalpontSystemCatalog::OID, messageqcpp::ByteStream&); - EXPORT const rowgroup::RowGroup& getOutputRowGroup() const; - TupleDeliveryStep* getDeliveryStep() - { - return ds; - } - const JobStepVector& querySteps() const - { - return fQuery; - } - void setDeliveryFlag(bool f); - void abort(); + EXPORT uint32_t projectTable(execplan::CalpontSystemCatalog::OID, messageqcpp::ByteStream&); + EXPORT const rowgroup::RowGroup& getOutputRowGroup() const; + TupleDeliveryStep* getDeliveryStep() + { + return ds; + } + const JobStepVector& querySteps() const + { + return fQuery; + } + void setDeliveryFlag(bool f); + void abort(); - /** Does some light validation on the final joblist - * - * Currently verifies that JobSteps are all tuple-oriented, that - * there's one and only one projection step, and that its fake table OID is 100. - * @note The fake OID check is disabled atm because it's not always 100 although it's supposed to be. - */ - EXPORT void validate() const; + /** Does some light validation on the final joblist + * + * Currently verifies that JobSteps are all tuple-oriented, that + * there's one and only one projection step, and that its fake table OID is 100. + * @note The fake OID check is disabled atm because it's not always 100 although it's supposed to be. + */ + EXPORT void validate() const; -private: - //defaults okay - //TupleJobList(const TupleJobList& rhs); - //TupleJobList& operator=(const TupleJobList& rhs); + private: + // defaults okay + // TupleJobList(const TupleJobList& rhs); + // TupleJobList& operator=(const TupleJobList& rhs); - TupleDeliveryStep* ds; - bool moreData; // used to prevent calling nextBand beyond the last RowGroup + TupleDeliveryStep* ds; + bool moreData; // used to prevent calling nextBand beyond the last RowGroup }; typedef boost::shared_ptr STJLP; -} +} // namespace joblist #undef EXPORT #endif // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 96e9d2760..06bdf5920 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -93,420 +93,411 @@ using namespace rowgroup; #include "mcsv1_udaf.h" #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" - // for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to 'typeid' +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpotentially-evaluated-expression" +// for warnings on typeid :expression with side effects will be evaluated despite being used as an operand to +// 'typeid' #endif - namespace { using namespace joblist; void projectSimpleColumn(const SimpleColumn* sc, JobStepVector& jsv, JobInfo& jobInfo) { - if (sc == NULL) - throw logic_error("projectSimpleColumn: sc is null"); + if (sc == NULL) + throw logic_error("projectSimpleColumn: sc is null"); - CalpontSystemCatalog::OID oid = sc->oid(); - CalpontSystemCatalog::OID tbl_oid = tableOid(sc, jobInfo.csc); - string alias(extractTableAlias(sc)); - string view(sc->viewName()); - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct; - pColStep* pcs = NULL; - pDictionaryStep* pds = NULL; - bool tokenOnly = false; - TupleInfo ti; + CalpontSystemCatalog::OID oid = sc->oid(); + CalpontSystemCatalog::OID tbl_oid = tableOid(sc, jobInfo.csc); + string alias(extractTableAlias(sc)); + string view(sc->viewName()); + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct; + pColStep* pcs = NULL; + pDictionaryStep* pds = NULL; + bool tokenOnly = false; + TupleInfo ti; - if (!sc->schemaName().empty()) + if (!sc->schemaName().empty()) + { + SJSTEP sjstep; + + // always tuples after release 3.0 + // if (!jobInfo.tryTuples) + // jobInfo.tables.insert(make_table(sc->schemaName(), sc->tableName())); + + // if (jobInfo.trace) + // cout << "doProject Emit pCol for SimpleColumn " << oid << endl; + + const PseudoColumn* pc = dynamic_cast(sc); + ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (sc->isColumnStore() && !pc) { - SJSTEP sjstep; - -// always tuples after release 3.0 -// if (!jobInfo.tryTuples) -// jobInfo.tables.insert(make_table(sc->schemaName(), sc->tableName())); - -// if (jobInfo.trace) -// cout << "doProject Emit pCol for SimpleColumn " << oid << endl; - - const PseudoColumn* pc = dynamic_cast(sc); - ct = sc->colType(); - -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (sc->isColumnStore() && !pc) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } -//X - if (pc == NULL) - pcs = new pColStep(oid, tbl_oid, ct, jobInfo); - else - pcs = new PseudoColStep(oid, tbl_oid, pc->pseudoType(), ct, jobInfo); - - pcs->alias(alias); - pcs->view(view); - pcs->name(sc->columnName()); - pcs->cardinality(sc->cardinality()); - //pcs->setOrderRids(true); - - sjstep.reset(pcs); - jsv.push_back(sjstep); - - dictOid = isDictCol(ct); - ti = setTupleInfo(ct, oid, jobInfo, tbl_oid, sc, alias); - pcs->tupleId(ti.key); - - if (dictOid > 0 && jobInfo.hasAggregation) - { - map::iterator it = - jobInfo.tokenOnly.find(getTupleKey(jobInfo, sc)); - - if (it != jobInfo.tokenOnly.end()) - tokenOnly = it->second; - } - - if (dictOid > 0 && !tokenOnly) - { - //This is a double-step step -// if (jobInfo.trace) -// cout << "doProject Emit pGetSignature for SimpleColumn " << dictOid << endl; - - pds = new pDictionaryStep(dictOid, tbl_oid, ct, jobInfo); - jobInfo.keyInfo->dictOidToColOid[dictOid] = oid; - pds->alias(alias); - pds->view(view); - pds->name(sc->columnName()); - pds->cardinality(sc->cardinality()); - //pds->setOrderRids(true); - - //Associate these two linked steps - JobStepAssociation outJs; - AnyDataListSPtr spdl1(new AnyDataList()); - RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); - spdl1->rowGroupDL(dl1); - dl1->OID(oid); - - // not a tokenOnly column - setTupleInfo(ct, dictOid, jobInfo, tbl_oid, sc, alias); - jobInfo.tokenOnly[getTupleKey(jobInfo, sc)] = false; - outJs.outAdd(spdl1); - - pcs->outputAssociation(outJs); - pds->inputAssociation(outJs); - - sjstep.reset(pds); - jsv.push_back(sjstep); - - oid = dictOid; // dictionary column - ti = setTupleInfo(ct, oid, jobInfo, tbl_oid, sc, alias); - pds->tupleId(ti.key); - jobInfo.keyInfo->dictKeyMap[pcs->tupleId()] = ti.key; - } + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; } - else // must be vtable mode + // X + if (pc == NULL) + pcs = new pColStep(oid, tbl_oid, ct, jobInfo); + else + pcs = new PseudoColStep(oid, tbl_oid, pc->pseudoType(), ct, jobInfo); + + pcs->alias(alias); + pcs->view(view); + pcs->name(sc->columnName()); + pcs->cardinality(sc->cardinality()); + // pcs->setOrderRids(true); + + sjstep.reset(pcs); + jsv.push_back(sjstep); + + dictOid = isDictCol(ct); + ti = setTupleInfo(ct, oid, jobInfo, tbl_oid, sc, alias); + pcs->tupleId(ti.key); + + if (dictOid > 0 && jobInfo.hasAggregation) { - oid = (tbl_oid + 1) + sc->colPosition(); - ct = jobInfo.vtableColTypes[UniqId(oid, alias, "", "")]; - ti = setTupleInfo(ct, oid, jobInfo, tbl_oid, sc, alias); + map::iterator it = jobInfo.tokenOnly.find(getTupleKey(jobInfo, sc)); + + if (it != jobInfo.tokenOnly.end()) + tokenOnly = it->second; } - if (dictOid > 0 && tokenOnly) + if (dictOid > 0 && !tokenOnly) { - // scale is not used by string columns - // borrow it to indicate token is used in projection, not the real string. - ti.scale = 8; - } + // This is a double-step step + // if (jobInfo.trace) + // cout << "doProject Emit pGetSignature for SimpleColumn " << dictOid << + //endl; - jobInfo.pjColList.push_back(ti); + pds = new pDictionaryStep(dictOid, tbl_oid, ct, jobInfo); + jobInfo.keyInfo->dictOidToColOid[dictOid] = oid; + pds->alias(alias); + pds->view(view); + pds->name(sc->columnName()); + pds->cardinality(sc->cardinality()); + // pds->setOrderRids(true); + + // Associate these two linked steps + JobStepAssociation outJs; + AnyDataListSPtr spdl1(new AnyDataList()); + RowGroupDL* dl1 = new RowGroupDL(1, jobInfo.fifoSize); + spdl1->rowGroupDL(dl1); + dl1->OID(oid); + + // not a tokenOnly column + setTupleInfo(ct, dictOid, jobInfo, tbl_oid, sc, alias); + jobInfo.tokenOnly[getTupleKey(jobInfo, sc)] = false; + outJs.outAdd(spdl1); + + pcs->outputAssociation(outJs); + pds->inputAssociation(outJs); + + sjstep.reset(pds); + jsv.push_back(sjstep); + + oid = dictOid; // dictionary column + ti = setTupleInfo(ct, oid, jobInfo, tbl_oid, sc, alias); + pds->tupleId(ti.key); + jobInfo.keyInfo->dictKeyMap[pcs->tupleId()] = ti.key; + } + } + else // must be vtable mode + { + oid = (tbl_oid + 1) + sc->colPosition(); + ct = jobInfo.vtableColTypes[UniqId(oid, alias, "", "")]; + ti = setTupleInfo(ct, oid, jobInfo, tbl_oid, sc, alias); + } + + if (dictOid > 0 && tokenOnly) + { + // scale is not used by string columns + // borrow it to indicate token is used in projection, not the real string. + ti.scale = 8; + } + + jobInfo.pjColList.push_back(ti); } const JobStepVector doProject(const RetColsVector& retCols, JobInfo& jobInfo) { - JobStepVector jsv; - SJSTEP sjstep; + JobStepVector jsv; + SJSTEP sjstep; - for (unsigned i = 0; i < retCols.size(); i++) + for (unsigned i = 0; i < retCols.size(); i++) + { + const SimpleColumn* sc = dynamic_cast(retCols[i].get()); + const WindowFunctionColumn* wc = NULL; + + if (sc != NULL) { - const SimpleColumn* sc = dynamic_cast(retCols[i].get()); - const WindowFunctionColumn* wc = NULL; - - if (sc != NULL) - { - projectSimpleColumn(sc, jsv, jobInfo); - } - else if ((wc = dynamic_cast(retCols[i].get())) != NULL) - { - //put place hold column in projection list - uint64_t eid = wc->expressionId(); - CalpontSystemCatalog::ColType ct = wc->resultType(); - TupleInfo ti(setExpTupleInfo(ct, eid, retCols[i].get()->alias(), jobInfo)); - jobInfo.pjColList.push_back(ti); - } - else - { - const ArithmeticColumn* ac = NULL; - const FunctionColumn* fc = NULL; - const ConstantColumn* cc = NULL; - uint64_t eid = -1; - CalpontSystemCatalog::ColType ct; - ExpressionStep* es = new ExpressionStep(jobInfo); - es->expression(retCols[i], jobInfo); - sjstep.reset(es); - - if ((ac = dynamic_cast(retCols[i].get())) != NULL) - { - eid = ac->expressionId(); - ct = ac->resultType(); - } - else if ((fc = dynamic_cast(retCols[i].get())) != NULL) - { - eid = fc->expressionId(); - ct = fc->resultType(); - } - else if ((cc = dynamic_cast(retCols[i].get())) != NULL) - { - eid = cc->expressionId(); - ct = cc->resultType(); - } - else - { - std::ostringstream errmsg; - errmsg << "doProject: unhandled returned column: " << typeid(*retCols[i]).name(); - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } - - // set expression tuple Info - TupleInfo ti(setExpTupleInfo(ct, eid, retCols[i].get()->alias(), jobInfo)); - uint32_t key = ti.key; - - if (retCols[i]->windowfunctionColumnList().size() > 0) - jobInfo.expressionVec.push_back(key); - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) - == jobInfo.expressionVec.end()) - { - jobInfo.returnedExpressions.push_back(sjstep); - } - - //put place hold column in projection list - jobInfo.pjColList.push_back(ti); - } + projectSimpleColumn(sc, jsv, jobInfo); } + else if ((wc = dynamic_cast(retCols[i].get())) != NULL) + { + // put place hold column in projection list + uint64_t eid = wc->expressionId(); + CalpontSystemCatalog::ColType ct = wc->resultType(); + TupleInfo ti(setExpTupleInfo(ct, eid, retCols[i].get()->alias(), jobInfo)); + jobInfo.pjColList.push_back(ti); + } + else + { + const ArithmeticColumn* ac = NULL; + const FunctionColumn* fc = NULL; + const ConstantColumn* cc = NULL; + uint64_t eid = -1; + CalpontSystemCatalog::ColType ct; + ExpressionStep* es = new ExpressionStep(jobInfo); + es->expression(retCols[i], jobInfo); + sjstep.reset(es); - return jsv; + if ((ac = dynamic_cast(retCols[i].get())) != NULL) + { + eid = ac->expressionId(); + ct = ac->resultType(); + } + else if ((fc = dynamic_cast(retCols[i].get())) != NULL) + { + eid = fc->expressionId(); + ct = fc->resultType(); + } + else if ((cc = dynamic_cast(retCols[i].get())) != NULL) + { + eid = cc->expressionId(); + ct = cc->resultType(); + } + else + { + std::ostringstream errmsg; + errmsg << "doProject: unhandled returned column: " << typeid(*retCols[i]).name(); + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); + } + + // set expression tuple Info + TupleInfo ti(setExpTupleInfo(ct, eid, retCols[i].get()->alias(), jobInfo)); + uint32_t key = ti.key; + + if (retCols[i]->windowfunctionColumnList().size() > 0) + jobInfo.expressionVec.push_back(key); + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) == + jobInfo.expressionVec.end()) + { + jobInfo.returnedExpressions.push_back(sjstep); + } + + // put place hold column in projection list + jobInfo.pjColList.push_back(ti); + } + } + + return jsv; } void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - TupleHavingStep* ths = new TupleHavingStep(jobInfo); - ths->expressionFilter(csep->having(), jobInfo); - jobInfo.havingStep.reset(ths); + TupleHavingStep* ths = new TupleHavingStep(jobInfo); + ths->expressionFilter(csep->having(), jobInfo); + jobInfo.havingStep.reset(ths); - // simple columns in select clause - set scInSelect; + // simple columns in select clause + set scInSelect; - for (RetColsVector::iterator i = jobInfo.nonConstCols.begin(); - i != jobInfo.nonConstCols.end(); - i++) + for (RetColsVector::iterator i = jobInfo.nonConstCols.begin(); i != jobInfo.nonConstCols.end(); i++) + { + SimpleColumn* sc = dynamic_cast(i->get()); + + if (sc != NULL) { - SimpleColumn* sc = dynamic_cast(i->get()); + if (sc->schemaName().empty()) + sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - if (sc != NULL) - { - if (sc->schemaName().empty()) - sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - - scInSelect.insert(UniqId(sc)); - } + scInSelect.insert(UniqId(sc)); } + } - // simple columns in gruop by clause - set scInGroupBy; + // simple columns in gruop by clause + set scInGroupBy; - for (RetColsVector::iterator i = csep->groupByCols().begin(); - i != csep->groupByCols().end(); - i++) + for (RetColsVector::iterator i = csep->groupByCols().begin(); i != csep->groupByCols().end(); i++) + { + SimpleColumn* sc = dynamic_cast(i->get()); + + if (sc != NULL) { - SimpleColumn* sc = dynamic_cast(i->get()); - - if (sc != NULL) + if (sc->schemaName().empty() && sc->oid() == 0) + { + if (sc->colPosition() == -1) { - if (sc->schemaName().empty() && sc->oid() == 0) - { - if (sc->colPosition() == -1) - { - // from select subquery - SRCP ss = csep->returnedCols()[sc->orderPos()]; - (*i) = ss; - } - else - { - sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - } - } - - scInGroupBy.insert(UniqId(sc)); - } - } - - bool aggInHaving = false; - const vector& columns = ths->columns(); - - for (vector::const_iterator i = columns.begin(); i != columns.end(); i++) - { - // evaluate aggregate columns in having - AggregateColumn* agc = dynamic_cast(*i); - - if (agc) - { - addAggregateColumn(agc, -1, jobInfo.nonConstCols, jobInfo); - aggInHaving = true; + // from select subquery + SRCP ss = csep->returnedCols()[sc->orderPos()]; + (*i) = ss; } else { - // simple columns used in having and in group by clause must be in rowgroup - SimpleColumn* sc = dynamic_cast(*i); - - if (sc != NULL) - { - if (sc->schemaName().empty()) - sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - - UniqId scId(sc); - - if (scInGroupBy.find(scId) != scInGroupBy.end() && - scInSelect.find(scId) == scInSelect.end()) - { - jobInfo.nonConstCols.push_back(SRCP(sc->clone())); - } - } + sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); } - } + } - if (aggInHaving == false) + scInGroupBy.insert(UniqId(sc)); + } + } + + bool aggInHaving = false; + const vector& columns = ths->columns(); + + for (vector::const_iterator i = columns.begin(); i != columns.end(); i++) + { + // evaluate aggregate columns in having + AggregateColumn* agc = dynamic_cast(*i); + + if (agc) { - // treated the same as where clause if no aggregate column in having. - jobInfo.havingStep.reset(); - - // parse the having expression - ParseTree* filters = csep->having(); - - if (filters != 0) - { - JLF_ExecPlanToJobList::walkTree(filters, jobInfo); - } - - if (!jobInfo.stack.empty()) - { - idbassert(jobInfo.stack.size() == 1); - jobInfo.havingStepVec = jobInfo.stack.top(); - jobInfo.stack.pop(); - } + addAggregateColumn(agc, -1, jobInfo.nonConstCols, jobInfo); + aggInHaving = true; } + else + { + // simple columns used in having and in group by clause must be in rowgroup + SimpleColumn* sc = dynamic_cast(*i); + + if (sc != NULL) + { + if (sc->schemaName().empty()) + sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + + UniqId scId(sc); + + if (scInGroupBy.find(scId) != scInGroupBy.end() && scInSelect.find(scId) == scInSelect.end()) + { + jobInfo.nonConstCols.push_back(SRCP(sc->clone())); + } + } + } + } + + if (aggInHaving == false) + { + // treated the same as where clause if no aggregate column in having. + jobInfo.havingStep.reset(); + + // parse the having expression + ParseTree* filters = csep->having(); + + if (filters != 0) + { + JLF_ExecPlanToJobList::walkTree(filters, jobInfo); + } + + if (!jobInfo.stack.empty()) + { + idbassert(jobInfo.stack.size() == 1); + jobInfo.havingStepVec = jobInfo.stack.top(); + jobInfo.stack.pop(); + } + } } -void preProcessFunctionOnAggregation(const vector& scs, - const vector& aggs, - const vector& wcs, - JobInfo& jobInfo) +void preProcessFunctionOnAggregation(const vector& scs, const vector& aggs, + const vector& wcs, JobInfo& jobInfo) { - // append the simple columns if not already projected - set scProjected; + // append the simple columns if not already projected + set scProjected; - for (RetColsVector::iterator i = jobInfo.projectionCols.begin(); - i != jobInfo.projectionCols.end(); - i++) + for (RetColsVector::iterator i = jobInfo.projectionCols.begin(); i != jobInfo.projectionCols.end(); i++) + { + SimpleColumn* sc = dynamic_cast(i->get()); + + if (sc != NULL) { - SimpleColumn* sc = dynamic_cast(i->get()); + if (sc->schemaName().empty()) + sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - if (sc != NULL) - { - if (sc->schemaName().empty()) - sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - - scProjected.insert(UniqId(sc)); - } + scProjected.insert(UniqId(sc)); } + } - for (vector::const_iterator i = scs.begin(); i != scs.end(); i++) + for (vector::const_iterator i = scs.begin(); i != scs.end(); i++) + { + if (scProjected.find(UniqId(*i)) == scProjected.end()) { - if (scProjected.find(UniqId(*i)) == scProjected.end()) - { - jobInfo.projectionCols.push_back(SRCP((*i)->clone())); - scProjected.insert(UniqId(*i)); - } + jobInfo.projectionCols.push_back(SRCP((*i)->clone())); + scProjected.insert(UniqId(*i)); } + } - // append the aggregate columns in arithmetic/function column to the projection list - for (vector::const_iterator i = aggs.begin(); i != aggs.end(); i++) + // append the aggregate columns in arithmetic/function column to the projection list + for (vector::const_iterator i = aggs.begin(); i != aggs.end(); i++) + { + addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo); + if (wcs.size() > 0) { - addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo); - if (wcs.size() > 0) - { - jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone())); - } + jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone())); } + } } void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) + for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) + { + if (NULL == dynamic_cast(jobInfo.deliveredCols[i].get())) + jobInfo.nonConstCols.push_back(jobInfo.deliveredCols[i]); + } + + // save the original delivered non constant columns + jobInfo.nonConstDelCols = jobInfo.nonConstCols; + + if (jobInfo.nonConstCols.size() != jobInfo.deliveredCols.size()) + { + jobInfo.constantCol = CONST_COL_EXIST; + + // bug 2531, all constant column. + if (jobInfo.nonConstCols.size() == 0) { - if (NULL == dynamic_cast(jobInfo.deliveredCols[i].get())) - jobInfo.nonConstCols.push_back(jobInfo.deliveredCols[i]); + if (csep->columnMap().size() > 0) + jobInfo.nonConstCols.push_back((*(csep->columnMap().begin())).second); + else + jobInfo.constantCol = CONST_COL_ONLY; } + } - // save the original delivered non constant columns - jobInfo.nonConstDelCols = jobInfo.nonConstCols; + for (uint64_t i = 0; i < jobInfo.nonConstCols.size(); i++) + { + AggregateColumn* agc = dynamic_cast(jobInfo.nonConstCols[i].get()); - if (jobInfo.nonConstCols.size() != jobInfo.deliveredCols.size()) + if (agc) + addAggregateColumn(agc, i, jobInfo.nonConstCols, jobInfo); + } + + if (csep->having() != NULL) + checkHavingClause(csep, jobInfo); + + jobInfo.projectionCols = jobInfo.nonConstCols; + + for (uint64_t i = 0; i < jobInfo.nonConstCols.size(); i++) + { + const ArithmeticColumn* ac = dynamic_cast(jobInfo.nonConstCols[i].get()); + const FunctionColumn* fc = dynamic_cast(jobInfo.nonConstCols[i].get()); + + if (ac != NULL && ac->aggColumnList().size() > 0) { - jobInfo.constantCol = CONST_COL_EXIST; - - // bug 2531, all constant column. - if (jobInfo.nonConstCols.size() == 0) - { - if (csep->columnMap().size() > 0) - jobInfo.nonConstCols.push_back((*(csep->columnMap().begin())).second); - else - jobInfo.constantCol = CONST_COL_ONLY; - } + jobInfo.nonConstCols[i]->outputIndex(i); + preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), + ac->windowfunctionColumnList(), jobInfo); } - - for (uint64_t i = 0; i < jobInfo.nonConstCols.size(); i++) + else if (fc != NULL && fc->aggColumnList().size() > 0) { - AggregateColumn* agc = dynamic_cast(jobInfo.nonConstCols[i].get()); - - if (agc) - addAggregateColumn(agc, i, jobInfo.nonConstCols, jobInfo); - } - - if (csep->having() != NULL) - checkHavingClause(csep, jobInfo); - - jobInfo.projectionCols = jobInfo.nonConstCols; - - for (uint64_t i = 0; i < jobInfo.nonConstCols.size(); i++) - { - const ArithmeticColumn* ac = - dynamic_cast(jobInfo.nonConstCols[i].get()); - const FunctionColumn* fc = - dynamic_cast(jobInfo.nonConstCols[i].get()); - - if (ac != NULL && ac->aggColumnList().size() > 0) - { - jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo); - } - else if (fc != NULL && fc->aggColumnList().size() > 0) - { - jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo); - } + jobInfo.nonConstCols[i]->outputIndex(i); + preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), + fc->windowfunctionColumnList(), jobInfo); } + } } /* @@ -522,308 +513,304 @@ the fSequence is not set, so orderPos is used to locate the column. */ void checkGroupByCols(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - // order by columns may be not in the select and [group by] clause - const CalpontSelectExecutionPlan::OrderByColumnList& orderByCols = csep->orderByCols(); + // order by columns may be not in the select and [group by] clause + const CalpontSelectExecutionPlan::OrderByColumnList& orderByCols = csep->orderByCols(); - for (uint64_t i = 0; i < orderByCols.size(); i++) + for (uint64_t i = 0; i < orderByCols.size(); i++) + { + if (orderByCols[i]->orderPos() == (uint64_t)(-1)) { - if (orderByCols[i]->orderPos() == (uint64_t)(-1)) + // @bug 4531, skip window functions, should be already added. + if (dynamic_cast(orderByCols[i].get()) != NULL || + orderByCols[i]->windowfunctionColumnList().size() > 0) + continue; + + jobInfo.deliveredCols.push_back(orderByCols[i]); + + // @bug 3025 + // Append the non-aggregate orderby column to group by, if there is group by clause. + // Duplicates will be removed by next if block. + if (csep->groupByCols().size() > 0) + { + // Not an aggregate column and not an expression of aggregation. + if (dynamic_cast(orderByCols[i].get()) == NULL && + orderByCols[i]->aggColumnList().empty()) + csep->groupByCols().push_back(orderByCols[i]); + } + } + } + + if (csep->groupByCols().size() > 0) + { + set colInGroupBy; + RetColsVector uniqGbCols; + + for (RetColsVector::iterator i = csep->groupByCols().begin(); i != csep->groupByCols().end(); i++) + { + // skip constant columns + if (dynamic_cast(i->get()) != NULL) + continue; + + ReturnedColumn* rc = i->get(); + SimpleColumn* sc = dynamic_cast(rc); + + bool selectSubquery = false; + + if (sc && sc->schemaName().empty() && sc->oid() == 0) + { + if (sc->colPosition() == -1) { - // @bug 4531, skip window functions, should be already added. - if (dynamic_cast(orderByCols[i].get()) != NULL || - orderByCols[i]->windowfunctionColumnList().size() > 0) - continue; + // from select subquery + // sc->orderPos() should NOT be -1 because it is a SELECT sub-query. + SRCP ss = csep->returnedCols()[sc->orderPos()]; + (*i) = ss; + selectSubquery = true; - jobInfo.deliveredCols.push_back(orderByCols[i]); - - // @bug 3025 - // Append the non-aggregate orderby column to group by, if there is group by clause. - // Duplicates will be removed by next if block. - if (csep->groupByCols().size() > 0) - { - // Not an aggregate column and not an expression of aggregation. - if (dynamic_cast(orderByCols[i].get()) == NULL && - orderByCols[i]->aggColumnList().empty()) - csep->groupByCols().push_back(orderByCols[i]); - } + // At this point whatever sc pointed to is invalid + // update the rc and sc + rc = ss.get(); + sc = dynamic_cast(rc); } + else + { + sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + } + } + + UniqId col; + + if (sc) + col = UniqId(sc); + else + col = UniqId(rc->expressionId(), rc->alias(), "", ""); + + if (colInGroupBy.find(col) == colInGroupBy.end() || selectSubquery) + { + colInGroupBy.insert(col); + uniqGbCols.push_back(*i); + } } - if (csep->groupByCols().size() > 0) - { - set colInGroupBy; - RetColsVector uniqGbCols; - - for (RetColsVector::iterator i = csep->groupByCols().begin(); - i != csep->groupByCols().end(); - i++) - { - // skip constant columns - if (dynamic_cast(i->get()) != NULL) - continue; - - ReturnedColumn* rc = i->get(); - SimpleColumn* sc = dynamic_cast(rc); - - bool selectSubquery = false; - - if (sc && sc->schemaName().empty() && sc->oid() == 0) - { - if (sc->colPosition() == -1) - { - // from select subquery - // sc->orderPos() should NOT be -1 because it is a SELECT sub-query. - SRCP ss = csep->returnedCols()[sc->orderPos()]; - (*i) = ss; - selectSubquery = true; - - // At this point whatever sc pointed to is invalid - // update the rc and sc - rc = ss.get(); - sc = dynamic_cast(rc); - } - else - { - sc->oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - } - } - - UniqId col; - - if (sc) - col = UniqId(sc); - else - col = UniqId(rc->expressionId(), rc->alias(), "", ""); - - if (colInGroupBy.find(col) == colInGroupBy.end() || selectSubquery) - { - colInGroupBy.insert(col); - uniqGbCols.push_back(*i); - } - } - - if (csep->groupByCols().size() != uniqGbCols.size()) - (csep)->groupByCols(uniqGbCols); - } + if (csep->groupByCols().size() != uniqGbCols.size()) + (csep)->groupByCols(uniqGbCols); + } } void checkAggregation(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - checkGroupByCols(csep, jobInfo); - checkReturnedColumns(csep, jobInfo); - RetColsVector& retCols = jobInfo.projectionCols; + checkGroupByCols(csep, jobInfo); + checkReturnedColumns(csep, jobInfo); + RetColsVector& retCols = jobInfo.projectionCols; - jobInfo.hasDistinct = csep->distinct(); + jobInfo.hasDistinct = csep->distinct(); - // DISTINCT with window functions must be done in tupleannexstep - if (csep->distinct() == true && jobInfo.windowDels.size() == 0) + // DISTINCT with window functions must be done in tupleannexstep + if (csep->distinct() == true && jobInfo.windowDels.size() == 0) + { + jobInfo.hasAggregation = true; + } + else if (csep->groupByCols().size() > 0) + { + // groupby without aggregate functions is supported. + jobInfo.hasAggregation = true; + } + else + { + for (uint64_t i = 0; i < retCols.size(); i++) { + if (dynamic_cast(retCols[i].get()) != NULL) + { jobInfo.hasAggregation = true; + break; + } } - else if (csep->groupByCols().size() > 0) - { - // groupby without aggregate functions is supported. - jobInfo.hasAggregation = true; - } - else - { - for (uint64_t i = 0; i < retCols.size(); i++) - { - if (dynamic_cast(retCols[i].get()) != NULL) - { - jobInfo.hasAggregation = true; - break; - } - } - } + } } void updateAggregateColType(AggregateColumn* ac, const SRCP& srcp, int op, JobInfo& jobInfo) { - CalpontSystemCatalog::ColType ct; - const SimpleColumn* sc = dynamic_cast(srcp.get()); - const ArithmeticColumn* ar = NULL; - const FunctionColumn* fc = NULL; + CalpontSystemCatalog::ColType ct; + const SimpleColumn* sc = dynamic_cast(srcp.get()); + const ArithmeticColumn* ar = NULL; + const FunctionColumn* fc = NULL; - if (sc != NULL) - ct = sc->resultType(); - else if ((ar = dynamic_cast(srcp.get())) != NULL) - ct = ar->resultType(); - else if ((fc = dynamic_cast(srcp.get())) != NULL) - ct = fc->resultType(); + if (sc != NULL) + ct = sc->resultType(); + else if ((ar = dynamic_cast(srcp.get())) != NULL) + ct = ar->resultType(); + else if ((fc = dynamic_cast(srcp.get())) != NULL) + ct = fc->resultType(); - if (op == AggregateColumn::STDDEV_POP || op == AggregateColumn::STDDEV_SAMP || - op == AggregateColumn::VAR_POP || op == AggregateColumn::VAR_SAMP) + if (op == AggregateColumn::STDDEV_POP || op == AggregateColumn::STDDEV_SAMP || + op == AggregateColumn::VAR_POP || op == AggregateColumn::VAR_SAMP) + { + ct.colWidth = sizeof(double); + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.scale = 0; + ct.precision = -1; + } + else if (op == AggregateColumn::UDAF) + { + UDAFColumn* udafc = dynamic_cast(ac); + + if (udafc) { - ct.colWidth = sizeof(double); - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.scale = 0; - ct.precision = -1; - } - else if (op == AggregateColumn::UDAF) - { - UDAFColumn* udafc = dynamic_cast(ac); - - if (udafc) - { - mcsv1sdk::mcsv1Context& udafContext = udafc->getContext(); - ct.colDataType = udafContext.getResultType(); - ct.colWidth = udafContext.getColWidth(); - ct.scale = udafContext.getScale(); - ct.precision = udafContext.getPrecision(); - } - else - { - ct = ac->resultType(); - } + mcsv1sdk::mcsv1Context& udafContext = udafc->getContext(); + ct.colDataType = udafContext.getResultType(); + ct.colWidth = udafContext.getColWidth(); + ct.scale = udafContext.getScale(); + ct.precision = udafContext.getPrecision(); } else { - ct = ac->resultType(); + ct = ac->resultType(); } + } + else + { + ct = ac->resultType(); + } - ac->resultType(ct); + ac->resultType(ct); - // update the original if this aggregate column is cloned from function on aggregation - pair::iterator, - multimap::iterator> range = - jobInfo.cloneAggregateColMap.equal_range(ac); + // update the original if this aggregate column is cloned from function on aggregation + pair::iterator, + multimap::iterator> + range = jobInfo.cloneAggregateColMap.equal_range(ac); - for (multimap::iterator i = range.first; i != range.second; ++i) - (i->second)->resultType(ct); + for (multimap::iterator i = range.first; i != range.second; ++i) + (i->second)->resultType(ct); } - const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - vector projectKeys; // projected column keys -- unique - RetColsVector pcv; // projected column vector -- may have duplicates + vector projectKeys; // projected column keys -- unique + RetColsVector pcv; // projected column vector -- may have duplicates - // add the groupby cols in the front part of the project column vector (pcv) - const CalpontSelectExecutionPlan::GroupByColumnList& groupByCols = csep->groupByCols(); - uint64_t lastGroupByPos = 0; + // add the groupby cols in the front part of the project column vector (pcv) + const CalpontSelectExecutionPlan::GroupByColumnList& groupByCols = csep->groupByCols(); + uint64_t lastGroupByPos = 0; - for (uint64_t i = 0; i < groupByCols.size(); i++) + for (uint64_t i = 0; i < groupByCols.size(); i++) + { + pcv.push_back(groupByCols[i]); + lastGroupByPos++; + + const SimpleColumn* sc = dynamic_cast(groupByCols[i].get()); + const ArithmeticColumn* ac = NULL; + const FunctionColumn* fc = NULL; + + if (sc != NULL) { - pcv.push_back(groupByCols[i]); - lastGroupByPos++; + CalpontSystemCatalog::OID gbOid = sc->oid(); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct; + string alias(extractTableAlias(sc)); + string view(sc->viewName()); - const SimpleColumn* sc = dynamic_cast(groupByCols[i].get()); - const ArithmeticColumn* ac = NULL; - const FunctionColumn* fc = NULL; + if (!sc->schemaName().empty()) + { + ct = sc->colType(); - if (sc != NULL) + // XXX use this before connector sets colType in sc correctly. + if (sc->isColumnStore() && dynamic_cast(sc) == NULL) { - CalpontSystemCatalog::OID gbOid = sc->oid(); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct; - string alias(extractTableAlias(sc)); - string view(sc->viewName()); - - if (!sc->schemaName().empty()) - { - ct = sc->colType(); - -//XXX use this before connector sets colType in sc correctly. - if (sc->isColumnStore() && dynamic_cast(sc) == NULL) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } -//X - dictOid = isDictCol(ct); - } - else - { - gbOid = (tblOid + 1) + sc->colPosition(); - ct = jobInfo.vtableColTypes[UniqId(gbOid, alias, "", "")]; - } - - // As of bug3695, make sure varbinary is not used in group by. - if (ct.colDataType == CalpontSystemCatalog::VARBINARY) - throw runtime_error ("VARBINARY in group by is not supported."); - - TupleInfo ti(setTupleInfo(ct, gbOid, jobInfo, tblOid, sc, alias)); - uint32_t tupleKey = ti.key; - - if (find(projectKeys.begin(), projectKeys.end(), tupleKey) == projectKeys.end()) - projectKeys.push_back(tupleKey); - - // for dictionary columns, replace the token oid with string oid - if (dictOid > 0) - { - jobInfo.tokenOnly[tupleKey] = false; - ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); - jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; - tupleKey = ti.key; - } - - jobInfo.groupByColVec.push_back(tupleKey); + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; } - else if ((ac = dynamic_cast(groupByCols[i].get())) != NULL) - { - uint64_t eid = ac->expressionId(); - CalpontSystemCatalog::ColType ct = ac->resultType(); - TupleInfo ti(setExpTupleInfo(ct, eid, ac->alias(), jobInfo)); - uint32_t tupleKey = ti.key; - jobInfo.groupByColVec.push_back(tupleKey); + // X + dictOid = isDictCol(ct); + } + else + { + gbOid = (tblOid + 1) + sc->colPosition(); + ct = jobInfo.vtableColTypes[UniqId(gbOid, alias, "", "")]; + } - if (find(projectKeys.begin(), projectKeys.end(), tupleKey) == projectKeys.end()) - projectKeys.push_back(tupleKey); - } - else if ((fc = dynamic_cast(groupByCols[i].get())) != NULL) - { - uint64_t eid = fc->expressionId(); - CalpontSystemCatalog::ColType ct = fc->resultType(); - TupleInfo ti(setExpTupleInfo(ct, eid, fc->alias(), jobInfo)); - uint32_t tupleKey = ti.key; - jobInfo.groupByColVec.push_back(tupleKey); + // As of bug3695, make sure varbinary is not used in group by. + if (ct.colDataType == CalpontSystemCatalog::VARBINARY) + throw runtime_error("VARBINARY in group by is not supported."); - if (find(projectKeys.begin(), projectKeys.end(), tupleKey) == projectKeys.end()) - projectKeys.push_back(tupleKey); - } - else - { - std::ostringstream errmsg; - errmsg << "doAggProject: unsupported group by column: " - << typeid(*groupByCols[i]).name(); - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } + TupleInfo ti(setTupleInfo(ct, gbOid, jobInfo, tblOid, sc, alias)); + uint32_t tupleKey = ti.key; + + if (find(projectKeys.begin(), projectKeys.end(), tupleKey) == projectKeys.end()) + projectKeys.push_back(tupleKey); + + // for dictionary columns, replace the token oid with string oid + if (dictOid > 0) + { + jobInfo.tokenOnly[tupleKey] = false; + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; + tupleKey = ti.key; + } + + jobInfo.groupByColVec.push_back(tupleKey); } - - // process the returned columns - RetColsVector& retCols = jobInfo.projectionCols; - SRCP srcp; - - for (uint64_t i = 0; i < retCols.size(); i++) + else if ((ac = dynamic_cast(groupByCols[i].get())) != NULL) { - GroupConcatColumn* gcc = dynamic_cast(retCols[i].get()); + uint64_t eid = ac->expressionId(); + CalpontSystemCatalog::ColType ct = ac->resultType(); + TupleInfo ti(setExpTupleInfo(ct, eid, ac->alias(), jobInfo)); + uint32_t tupleKey = ti.key; + jobInfo.groupByColVec.push_back(tupleKey); - if (gcc != NULL) - { - srcp = gcc->aggParms()[0]; - const RowColumn* rcp = dynamic_cast(srcp.get()); + if (find(projectKeys.begin(), projectKeys.end(), tupleKey) == projectKeys.end()) + projectKeys.push_back(tupleKey); + } + else if ((fc = dynamic_cast(groupByCols[i].get())) != NULL) + { + uint64_t eid = fc->expressionId(); + CalpontSystemCatalog::ColType ct = fc->resultType(); + TupleInfo ti(setExpTupleInfo(ct, eid, fc->alias(), jobInfo)); + uint32_t tupleKey = ti.key; + jobInfo.groupByColVec.push_back(tupleKey); - const vector& cols = rcp->columnVec(); + if (find(projectKeys.begin(), projectKeys.end(), tupleKey) == projectKeys.end()) + projectKeys.push_back(tupleKey); + } + else + { + std::ostringstream errmsg; + errmsg << "doAggProject: unsupported group by column: " << typeid(*groupByCols[i]).name(); + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); + } + } - for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) - { - if (dynamic_cast(j->get()) == NULL) - retCols.push_back(*j); - } + // process the returned columns + RetColsVector& retCols = jobInfo.projectionCols; + SRCP srcp; - vector& orderCols = gcc->orderCols(); + for (uint64_t i = 0; i < retCols.size(); i++) + { + GroupConcatColumn* gcc = dynamic_cast(retCols[i].get()); - for (vector::iterator k = orderCols.begin(); k != orderCols.end(); k++) - { - if (dynamic_cast(k->get()) == NULL) - retCols.push_back(*k); - } + if (gcc != NULL) + { + srcp = gcc->aggParms()[0]; + const RowColumn* rcp = dynamic_cast(srcp.get()); - continue; - } + const vector& cols = rcp->columnVec(); + + for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) + { + if (dynamic_cast(j->get()) == NULL) + retCols.push_back(*j); + } + + vector& orderCols = gcc->orderCols(); + + for (vector::iterator k = orderCols.begin(); k != orderCols.end(); k++) + { + if (dynamic_cast(k->get()) == NULL) + retCols.push_back(*k); + } + + continue; + } #if 0 // MCOL-1201 Add support for multi-parameter UDAnF @@ -859,1499 +846,1478 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo } #endif - srcp = retCols[i]; - const AggregateColumn* ag = dynamic_cast(retCols[i].get()); + srcp = retCols[i]; + const AggregateColumn* ag = dynamic_cast(retCols[i].get()); - // bug 3728 Make a dummy expression for srcp if it is an - // expression. This is needed to fill in some stuff. - // Note that es.expression does nothing if the item is not an expression. - if (ag == NULL) - { - // Not an aggregate. Make a dummy expression for the item - ExpressionStep es; - es.expression(srcp, jobInfo); - } - else - { - // MCOL-1201 multi-argument aggregate. make a dummy expression - // step for each argument that is an expression. - for (uint32_t i = 0; i < ag->aggParms().size(); ++i) - { - srcp = ag->aggParms()[i]; - ExpressionStep es; - es.expression(srcp, jobInfo); - } - } - } - - map dictMap; // bug 1853, the tupleKey - dictoid map - - for (uint64_t i = 0; i < retCols.size(); i++) + // bug 3728 Make a dummy expression for srcp if it is an + // expression. This is needed to fill in some stuff. + // Note that es.expression does nothing if the item is not an expression. + if (ag == NULL) { - srcp = retCols[i]; - const SimpleColumn* sc = dynamic_cast(srcp.get()); - AggregateColumn* aggc = dynamic_cast(srcp.get()); - bool doDistinct = (csep->distinct() && csep->groupByCols().empty()); + // Not an aggregate. Make a dummy expression for the item + ExpressionStep es; + es.expression(srcp, jobInfo); + } + else + { + // MCOL-1201 multi-argument aggregate. make a dummy expression + // step for each argument that is an expression. + for (uint32_t i = 0; i < ag->aggParms().size(); ++i) + { + srcp = ag->aggParms()[i]; + ExpressionStep es; + es.expression(srcp, jobInfo); + } + } + } + + map dictMap; // bug 1853, the tupleKey - dictoid map + + for (uint64_t i = 0; i < retCols.size(); i++) + { + srcp = retCols[i]; + const SimpleColumn* sc = dynamic_cast(srcp.get()); + AggregateColumn* aggc = dynamic_cast(srcp.get()); + bool doDistinct = (csep->distinct() && csep->groupByCols().empty()); // Use this instead of the above line to mimic MariaDB's sql_mode = 'ONLY_FULL_GROUP_BY' // bool doDistinct = (csep->distinct() && // csep->groupByCols().empty() && // !jobInfo.hasAggregation); - uint32_t tupleKey = -1; - string alias; - string view; + uint32_t tupleKey = -1; + string alias; + string view; - // returned column could be groupby column, a simplecoulumn not an aggregatecolumn - int op = 0; - CalpontSystemCatalog::OID dictOid = 0; - CalpontSystemCatalog::ColType ct, aggCt; + // returned column could be groupby column, a simplecoulumn not an aggregatecolumn + int op = 0; + CalpontSystemCatalog::OID dictOid = 0; + CalpontSystemCatalog::ColType ct, aggCt; - if (aggc) + if (aggc) + { + GroupConcatColumn* gcc = dynamic_cast(retCols[i].get()); + + if (gcc != NULL) + { + jobInfo.groupConcatCols.push_back(retCols[i]); + + uint64_t eid = gcc->expressionId(); + ct = gcc->resultType(); + TupleInfo ti(setExpTupleInfo(ct, eid, gcc->alias(), jobInfo)); + tupleKey = ti.key; + jobInfo.returnedColVec.push_back(make_pair(tupleKey, gcc->aggOp())); + // not a tokenOnly column. Mark all the columns involved + srcp = gcc->aggParms()[0]; + const RowColumn* rowCol = dynamic_cast(srcp.get()); + + if (rowCol) { - GroupConcatColumn* gcc = dynamic_cast(retCols[i].get()); + const std::vector& cols = rowCol->columnVec(); - if (gcc != NULL) + for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) + { + sc = dynamic_cast(j->get()); + + if (sc) { - jobInfo.groupConcatCols.push_back(retCols[i]); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + alias = extractTableAlias(sc); + ct = sc->colType(); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + jobInfo.tokenOnly[ti.key] = false; + } + } + } - uint64_t eid = gcc->expressionId(); - ct = gcc->resultType(); - TupleInfo ti(setExpTupleInfo(ct, eid, gcc->alias(), jobInfo)); - tupleKey = ti.key; - jobInfo.returnedColVec.push_back(make_pair(tupleKey, gcc->aggOp())); - // not a tokenOnly column. Mark all the columns involved - srcp = gcc->aggParms()[0]; - const RowColumn* rowCol = dynamic_cast(srcp.get()); + continue; + } + else + { + // Aggregate column not group concat + AggParms& aggParms = aggc->aggParms(); - if (rowCol) - { - const std::vector& cols = rowCol->columnVec(); + for (uint32_t parm = 0; parm < aggParms.size(); ++parm) + { + // Only do the optimization of converting to count(*) if + // there is only one parameter. + if (aggParms.size() == 1 && aggc->constCol().get() != NULL) + { + // replace the aggregate on constant with a count(*) + SRCP clone; + UDAFColumn* udafc = dynamic_cast(aggc); - for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) - { - sc = dynamic_cast(j->get()); - - if (sc) - { - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - alias = extractTableAlias(sc); - ct = sc->colType(); - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); - jobInfo.tokenOnly[ti.key] = false; - } - } - } - - continue; + if (udafc) + { + clone.reset(new UDAFColumn(*udafc, aggc->sessionID())); } else { - // Aggregate column not group concat - AggParms& aggParms = aggc->aggParms(); - - for (uint32_t parm = 0; parm < aggParms.size(); ++parm) - { - // Only do the optimization of converting to count(*) if - // there is only one parameter. - if (aggParms.size() == 1 && aggc->constCol().get() != NULL) - { - // replace the aggregate on constant with a count(*) - SRCP clone; - UDAFColumn* udafc = dynamic_cast(aggc); - - if (udafc) - { - clone.reset(new UDAFColumn(*udafc, aggc->sessionID())); - } - else - { - clone.reset(new AggregateColumn(*aggc, aggc->sessionID())); - } - - jobInfo.constAggregate.insert(make_pair(i, clone)); - aggc->aggOp(AggregateColumn::COUNT_ASTERISK); - aggc->distinct(false); - } - - srcp = aggParms[parm]; - sc = dynamic_cast(srcp.get()); - - if (parm == 0) - { - op = aggc->aggOp(); - } - else - { - op = AggregateColumn::MULTI_PARM; - } - - doDistinct = aggc->distinct(); - - if (aggParms.size() == 1) - { - // Set the col type based on the single parm. - // Changing col type based on a parm if multiple parms - // doesn't really make sense. - if (op != AggregateColumn::SUM && op != AggregateColumn::DISTINCT_SUM && - op != AggregateColumn::AVG && op != AggregateColumn::DISTINCT_AVG) - { - updateAggregateColType(aggc, srcp, op, jobInfo); - } - } - - aggCt = aggc->resultType(); - - // As of bug3695, make sure varbinary is not used in aggregation. - // TODO: allow for UDAF - if (sc != NULL && sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY) - throw runtime_error ("VARBINARY in aggregate function is not supported."); - - // Project the parm columns or expressions - if (sc != NULL) - { - CalpontSystemCatalog::OID retOid = sc->oid(); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - alias = extractTableAlias(sc); - view = sc->viewName(); - - if (!sc->schemaName().empty()) - { - ct = sc->colType(); - - //XXX use this before connector sets colType in sc correctly. - if (sc->isColumnStore() && dynamic_cast(sc) == NULL) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } - - //X - dictOid = isDictCol(ct); - } - else - { - retOid = (tblOid + 1) + sc->colPosition(); - ct = jobInfo.vtableColTypes[UniqId(retOid, alias, "", "")]; - } - - TupleInfo ti(setTupleInfo(ct, retOid, jobInfo, tblOid, sc, alias)); - tupleKey = ti.key; - - // this is a string column - if (dictOid > 0) - { - map::iterator findit = jobInfo.tokenOnly.find(tupleKey); - - // if the column has never seen, and the op is count: possible need count only. - if (AggregateColumn::COUNT == op || AggregateColumn::COUNT_ASTERISK == op) - { - if (findit == jobInfo.tokenOnly.end()) - jobInfo.tokenOnly[tupleKey] = true; - } - // if aggregate other than count, token is not enough. - else if (op != 0 || doDistinct) - { - jobInfo.tokenOnly[tupleKey] = false; - } - - findit = jobInfo.tokenOnly.find(tupleKey); - - if (!(findit != jobInfo.tokenOnly.end() && findit->second == true)) - { - dictMap[tupleKey] = dictOid; - jobInfo.keyInfo->dictOidToColOid[dictOid] = retOid; - ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); - jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; - } - } - } - else - { - const ArithmeticColumn* ac = NULL; - const FunctionColumn* fc = NULL; - const WindowFunctionColumn* wc = NULL; - bool hasAggCols = false; - bool hasWndCols = false; - - if ((ac = dynamic_cast(srcp.get())) != NULL) - { - if (ac->aggColumnList().size() > 0) - hasAggCols = true; - if (ac->windowfunctionColumnList().size() > 0) - hasWndCols = true; - } - else if ((fc = dynamic_cast(srcp.get())) != NULL) - { - if (fc->aggColumnList().size() > 0) - hasAggCols = true; - if (fc->windowfunctionColumnList().size() > 0) - hasWndCols = true; - } - else if (dynamic_cast(srcp.get()) != NULL) - { - std::ostringstream errmsg; - errmsg << "Invalid aggregate function nesting."; - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } - else if (dynamic_cast(srcp.get()) != NULL) - { - } - else if ((wc = dynamic_cast(srcp.get())) == NULL) - { - std::ostringstream errmsg; - errmsg << "doAggProject: unsupported column: " << typeid(*(srcp.get())).name(); - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } - - uint64_t eid = srcp.get()->expressionId(); - ct = srcp.get()->resultType(); - TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); - tupleKey = ti.key; - - if (hasAggCols && !hasWndCols) - jobInfo.expressionVec.push_back(tupleKey); - } - - // add to project list - vector::iterator keyIt = find(projectKeys.begin(), projectKeys.end(), tupleKey); - - if (keyIt == projectKeys.end()) - { - RetColsVector::iterator it = pcv.end(); - - if (doDistinct) - it = pcv.insert(pcv.begin() + lastGroupByPos++, srcp); - else - it = pcv.insert(pcv.end(), srcp); - - projectKeys.insert(projectKeys.begin() + std::distance(pcv.begin(), it), tupleKey); - } - else if (doDistinct) // @bug4250, move forward distinct column if necessary. - { - uint32_t pos = std::distance(projectKeys.begin(), keyIt); - - if (pos >= lastGroupByPos) - { - pcv[pos] = pcv[lastGroupByPos]; - pcv[lastGroupByPos] = srcp; - projectKeys[pos] = projectKeys[lastGroupByPos]; - projectKeys[lastGroupByPos] = tupleKey; - lastGroupByPos++; - } - } - - if (doDistinct && dictOid > 0) - tupleKey = jobInfo.keyInfo->dictKeyMap[tupleKey]; - - // remember the columns to be returned - jobInfo.returnedColVec.push_back(make_pair(tupleKey, op)); - - // bug 1499 distinct processing, save unique distinct columns - if (doDistinct && - (jobInfo.distinctColVec.end() == - find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey))) - { - jobInfo.distinctColVec.push_back(tupleKey); - } - } + clone.reset(new AggregateColumn(*aggc, aggc->sessionID())); } + + jobInfo.constAggregate.insert(make_pair(i, clone)); + aggc->aggOp(AggregateColumn::COUNT_ASTERISK); + aggc->distinct(false); + } + + srcp = aggParms[parm]; + sc = dynamic_cast(srcp.get()); + + if (parm == 0) + { + op = aggc->aggOp(); + } + else + { + op = AggregateColumn::MULTI_PARM; + } + + doDistinct = aggc->distinct(); + + if (aggParms.size() == 1) + { + // Set the col type based on the single parm. + // Changing col type based on a parm if multiple parms + // doesn't really make sense. + if (op != AggregateColumn::SUM && op != AggregateColumn::DISTINCT_SUM && + op != AggregateColumn::AVG && op != AggregateColumn::DISTINCT_AVG) + { + updateAggregateColType(aggc, srcp, op, jobInfo); + } + } + + aggCt = aggc->resultType(); + + // As of bug3695, make sure varbinary is not used in aggregation. + // TODO: allow for UDAF + if (sc != NULL && sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY) + throw runtime_error("VARBINARY in aggregate function is not supported."); + + // Project the parm columns or expressions + if (sc != NULL) + { + CalpontSystemCatalog::OID retOid = sc->oid(); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + alias = extractTableAlias(sc); + view = sc->viewName(); + + if (!sc->schemaName().empty()) + { + ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + if (sc->isColumnStore() && dynamic_cast(sc) == NULL) + { + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + + // X + dictOid = isDictCol(ct); + } + else + { + retOid = (tblOid + 1) + sc->colPosition(); + ct = jobInfo.vtableColTypes[UniqId(retOid, alias, "", "")]; + } + + TupleInfo ti(setTupleInfo(ct, retOid, jobInfo, tblOid, sc, alias)); + tupleKey = ti.key; + + // this is a string column + if (dictOid > 0) + { + map::iterator findit = jobInfo.tokenOnly.find(tupleKey); + + // if the column has never seen, and the op is count: possible need count only. + if (AggregateColumn::COUNT == op || AggregateColumn::COUNT_ASTERISK == op) + { + if (findit == jobInfo.tokenOnly.end()) + jobInfo.tokenOnly[tupleKey] = true; + } + // if aggregate other than count, token is not enough. + else if (op != 0 || doDistinct) + { + jobInfo.tokenOnly[tupleKey] = false; + } + + findit = jobInfo.tokenOnly.find(tupleKey); + + if (!(findit != jobInfo.tokenOnly.end() && findit->second == true)) + { + dictMap[tupleKey] = dictOid; + jobInfo.keyInfo->dictOidToColOid[dictOid] = retOid; + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; + } + } + } + else + { + const ArithmeticColumn* ac = NULL; + const FunctionColumn* fc = NULL; + const WindowFunctionColumn* wc = NULL; + bool hasAggCols = false; + bool hasWndCols = false; + + if ((ac = dynamic_cast(srcp.get())) != NULL) + { + if (ac->aggColumnList().size() > 0) + hasAggCols = true; + if (ac->windowfunctionColumnList().size() > 0) + hasWndCols = true; + } + else if ((fc = dynamic_cast(srcp.get())) != NULL) + { + if (fc->aggColumnList().size() > 0) + hasAggCols = true; + if (fc->windowfunctionColumnList().size() > 0) + hasWndCols = true; + } + else if (dynamic_cast(srcp.get()) != NULL) + { + std::ostringstream errmsg; + errmsg << "Invalid aggregate function nesting."; + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); + } + else if (dynamic_cast(srcp.get()) != NULL) + { + } + else if ((wc = dynamic_cast(srcp.get())) == NULL) + { + std::ostringstream errmsg; + errmsg << "doAggProject: unsupported column: " << typeid(*(srcp.get())).name(); + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); + } + + uint64_t eid = srcp.get()->expressionId(); + ct = srcp.get()->resultType(); + TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); + tupleKey = ti.key; + + if (hasAggCols && !hasWndCols) + jobInfo.expressionVec.push_back(tupleKey); + } + + // add to project list + vector::iterator keyIt = find(projectKeys.begin(), projectKeys.end(), tupleKey); + + if (keyIt == projectKeys.end()) + { + RetColsVector::iterator it = pcv.end(); + + if (doDistinct) + it = pcv.insert(pcv.begin() + lastGroupByPos++, srcp); + else + it = pcv.insert(pcv.end(), srcp); + + projectKeys.insert(projectKeys.begin() + std::distance(pcv.begin(), it), tupleKey); + } + else if (doDistinct) // @bug4250, move forward distinct column if necessary. + { + uint32_t pos = std::distance(projectKeys.begin(), keyIt); + + if (pos >= lastGroupByPos) + { + pcv[pos] = pcv[lastGroupByPos]; + pcv[lastGroupByPos] = srcp; + projectKeys[pos] = projectKeys[lastGroupByPos]; + projectKeys[lastGroupByPos] = tupleKey; + lastGroupByPos++; + } + } + + if (doDistinct && dictOid > 0) + tupleKey = jobInfo.keyInfo->dictKeyMap[tupleKey]; + + // remember the columns to be returned + jobInfo.returnedColVec.push_back(make_pair(tupleKey, op)); + + // bug 1499 distinct processing, save unique distinct columns + if (doDistinct && (jobInfo.distinctColVec.end() == + find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey))) + { + jobInfo.distinctColVec.push_back(tupleKey); + } + } + } + } + else + { + // Not an Aggregate + // simple column selected + if (sc != NULL) + { + // one column only need project once + CalpontSystemCatalog::OID retOid = sc->oid(); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + alias = extractTableAlias(sc); + view = sc->viewName(); + + if (!sc->schemaName().empty()) + { + ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + if (sc->isColumnStore() && dynamic_cast(sc) == NULL) + { + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + + // X + dictOid = isDictCol(ct); } else { - // Not an Aggregate - // simple column selected - if (sc != NULL) - { - // one column only need project once - CalpontSystemCatalog::OID retOid = sc->oid(); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - alias = extractTableAlias(sc); - view = sc->viewName(); - - if (!sc->schemaName().empty()) - { - ct = sc->colType(); - - //XXX use this before connector sets colType in sc correctly. - if (sc->isColumnStore() && dynamic_cast(sc) == NULL) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } - - //X - dictOid = isDictCol(ct); - } - else - { - retOid = (tblOid + 1) + sc->colPosition(); - ct = jobInfo.vtableColTypes[UniqId(retOid, alias, "", "")]; - } - - TupleInfo ti(setTupleInfo(ct, retOid, jobInfo, tblOid, sc, alias)); - tupleKey = ti.key; - - // this is a string column - if (dictOid > 0) - { - map::iterator findit = jobInfo.tokenOnly.find(tupleKey); - - // if the column has never seen, and the op is count: possible need count only. - if (AggregateColumn::COUNT == op || AggregateColumn::COUNT_ASTERISK == op) - { - if (findit == jobInfo.tokenOnly.end()) - jobInfo.tokenOnly[tupleKey] = true; - } - // if aggregate other than count, token is not enough. - else if (op != 0 || doDistinct) - { - jobInfo.tokenOnly[tupleKey] = false; - } - - findit = jobInfo.tokenOnly.find(tupleKey); - - if (!(findit != jobInfo.tokenOnly.end() && findit->second == true)) - { - dictMap[tupleKey] = dictOid; - jobInfo.keyInfo->dictOidToColOid[dictOid] = retOid; - ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); - jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; - } - } - } - else - { - const ArithmeticColumn* ac = NULL; - const FunctionColumn* fc = NULL; - const WindowFunctionColumn* wc = NULL; - bool hasAggCols = false; - bool hasWndCols = false; - - if ((ac = dynamic_cast(srcp.get())) != NULL) - { - if (ac->aggColumnList().size() > 0) - hasAggCols = true; - if (ac->windowfunctionColumnList().size() > 0) - hasWndCols = true; - } - else if ((fc = dynamic_cast(srcp.get())) != NULL) - { - if (fc->aggColumnList().size() > 0) - hasAggCols = true; - if (fc->windowfunctionColumnList().size() > 0) - hasWndCols = true; - } - else if (dynamic_cast(srcp.get()) != NULL) - { - std::ostringstream errmsg; - errmsg << "Invalid aggregate function nesting."; - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } - else if (dynamic_cast(srcp.get()) != NULL) - { - } - else if ((wc = dynamic_cast(srcp.get())) == NULL) - { - std::ostringstream errmsg; - errmsg << "doAggProject: unsupported column: " << typeid(*(srcp.get())).name(); - cerr << boldStart << errmsg.str() << boldStop << endl; - throw logic_error(errmsg.str()); - } - - uint64_t eid = srcp.get()->expressionId(); - ct = srcp.get()->resultType(); - TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); - tupleKey = ti.key; - - if (hasAggCols && !hasWndCols) - jobInfo.expressionVec.push_back(tupleKey); - } - - // add to project list - vector::iterator keyIt = find(projectKeys.begin(), projectKeys.end(), tupleKey); - - if (keyIt == projectKeys.end()) - { - RetColsVector::iterator it = pcv.end(); - - if (doDistinct) - it = pcv.insert(pcv.begin() + lastGroupByPos++, srcp); - else - it = pcv.insert(pcv.end(), srcp); - - projectKeys.insert(projectKeys.begin() + std::distance(pcv.begin(), it), tupleKey); - } - else if (doDistinct) // @bug4250, move forward distinct column if necessary. - { - uint32_t pos = std::distance(projectKeys.begin(), keyIt); - - if (pos >= lastGroupByPos) - { - pcv[pos] = pcv[lastGroupByPos]; - pcv[lastGroupByPos] = srcp; - projectKeys[pos] = projectKeys[lastGroupByPos]; - projectKeys[lastGroupByPos] = tupleKey; - lastGroupByPos++; - } - } - - if (doDistinct && dictOid > 0) - tupleKey = jobInfo.keyInfo->dictKeyMap[tupleKey]; - - // remember the columns to be returned - jobInfo.returnedColVec.push_back(make_pair(tupleKey, op)); - - // bug 1499 distinct processing, save unique distinct columns - if (doDistinct && - (jobInfo.distinctColVec.end() == - find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey))) - { - jobInfo.distinctColVec.push_back(tupleKey); - } + retOid = (tblOid + 1) + sc->colPosition(); + ct = jobInfo.vtableColTypes[UniqId(retOid, alias, "", "")]; } - } + TupleInfo ti(setTupleInfo(ct, retOid, jobInfo, tblOid, sc, alias)); + tupleKey = ti.key; - // for dictionary columns not count only, replace the token oid with string oid - for (vector >::iterator it = jobInfo.returnedColVec.begin(); - it != jobInfo.returnedColVec.end(); it++) - { - // if the column is a dictionary column and not count only - bool tokenOnly = false; - map::iterator i = jobInfo.tokenOnly.find(it->first); - - if (i != jobInfo.tokenOnly.end()) - tokenOnly = i->second; - - if (dictMap.find(it->first) != dictMap.end() && !tokenOnly) + // this is a string column + if (dictOid > 0) { - uint32_t tupleKey = jobInfo.keyInfo->dictKeyMap[it->first]; - int op = it->second; - *it = make_pair(tupleKey, op); + map::iterator findit = jobInfo.tokenOnly.find(tupleKey); + + // if the column has never seen, and the op is count: possible need count only. + if (AggregateColumn::COUNT == op || AggregateColumn::COUNT_ASTERISK == op) + { + if (findit == jobInfo.tokenOnly.end()) + jobInfo.tokenOnly[tupleKey] = true; + } + // if aggregate other than count, token is not enough. + else if (op != 0 || doDistinct) + { + jobInfo.tokenOnly[tupleKey] = false; + } + + findit = jobInfo.tokenOnly.find(tupleKey); + + if (!(findit != jobInfo.tokenOnly.end() && findit->second == true)) + { + dictMap[tupleKey] = dictOid; + jobInfo.keyInfo->dictOidToColOid[dictOid] = retOid; + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[tupleKey] = ti.key; + } } + } + else + { + const ArithmeticColumn* ac = NULL; + const FunctionColumn* fc = NULL; + const WindowFunctionColumn* wc = NULL; + bool hasAggCols = false; + bool hasWndCols = false; + + if ((ac = dynamic_cast(srcp.get())) != NULL) + { + if (ac->aggColumnList().size() > 0) + hasAggCols = true; + if (ac->windowfunctionColumnList().size() > 0) + hasWndCols = true; + } + else if ((fc = dynamic_cast(srcp.get())) != NULL) + { + if (fc->aggColumnList().size() > 0) + hasAggCols = true; + if (fc->windowfunctionColumnList().size() > 0) + hasWndCols = true; + } + else if (dynamic_cast(srcp.get()) != NULL) + { + std::ostringstream errmsg; + errmsg << "Invalid aggregate function nesting."; + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); + } + else if (dynamic_cast(srcp.get()) != NULL) + { + } + else if ((wc = dynamic_cast(srcp.get())) == NULL) + { + std::ostringstream errmsg; + errmsg << "doAggProject: unsupported column: " << typeid(*(srcp.get())).name(); + cerr << boldStart << errmsg.str() << boldStop << endl; + throw logic_error(errmsg.str()); + } + + uint64_t eid = srcp.get()->expressionId(); + ct = srcp.get()->resultType(); + TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); + tupleKey = ti.key; + + if (hasAggCols && !hasWndCols) + jobInfo.expressionVec.push_back(tupleKey); + } + + // add to project list + vector::iterator keyIt = find(projectKeys.begin(), projectKeys.end(), tupleKey); + + if (keyIt == projectKeys.end()) + { + RetColsVector::iterator it = pcv.end(); + + if (doDistinct) + it = pcv.insert(pcv.begin() + lastGroupByPos++, srcp); + else + it = pcv.insert(pcv.end(), srcp); + + projectKeys.insert(projectKeys.begin() + std::distance(pcv.begin(), it), tupleKey); + } + else if (doDistinct) // @bug4250, move forward distinct column if necessary. + { + uint32_t pos = std::distance(projectKeys.begin(), keyIt); + + if (pos >= lastGroupByPos) + { + pcv[pos] = pcv[lastGroupByPos]; + pcv[lastGroupByPos] = srcp; + projectKeys[pos] = projectKeys[lastGroupByPos]; + projectKeys[lastGroupByPos] = tupleKey; + lastGroupByPos++; + } + } + + if (doDistinct && dictOid > 0) + tupleKey = jobInfo.keyInfo->dictKeyMap[tupleKey]; + + // remember the columns to be returned + jobInfo.returnedColVec.push_back(make_pair(tupleKey, op)); + + // bug 1499 distinct processing, save unique distinct columns + if (doDistinct && (jobInfo.distinctColVec.end() == + find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey))) + { + jobInfo.distinctColVec.push_back(tupleKey); + } } + } - return doProject(pcv, jobInfo); + // for dictionary columns not count only, replace the token oid with string oid + for (vector >::iterator it = jobInfo.returnedColVec.begin(); + it != jobInfo.returnedColVec.end(); it++) + { + // if the column is a dictionary column and not count only + bool tokenOnly = false; + map::iterator i = jobInfo.tokenOnly.find(it->first); + + if (i != jobInfo.tokenOnly.end()) + tokenOnly = i->second; + + if (dictMap.find(it->first) != dictMap.end() && !tokenOnly) + { + uint32_t tupleKey = jobInfo.keyInfo->dictKeyMap[it->first]; + int op = it->second; + *it = make_pair(tupleKey, op); + } + } + + return doProject(pcv, jobInfo); } - template class Uniqer : public unary_function { -private: - typedef typename T::mapped_type Mt_; - class Pred : public unary_function + private: + typedef typename T::mapped_type Mt_; + class Pred : public unary_function + { + public: + Pred(const Mt_& retCol) : fRetCol(retCol) { - public: - Pred(const Mt_& retCol) : fRetCol(retCol) { } - bool operator()(const Mt_ rc) const - { - return fRetCol->sameColumn(rc.get()); - } - private: - const Mt_& fRetCol; - }; -public: - void operator()(typename T::value_type mapItem) - { - Pred pred(mapItem.second); - RetColsVector::iterator iter; - iter = find_if(fRetColsVec.begin(), fRetColsVec.end(), pred); - - if (iter == fRetColsVec.end()) - { - //Add this ReturnedColumn - fRetColsVec.push_back(mapItem.second); - } } - RetColsVector fRetColsVec; + bool operator()(const Mt_ rc) const + { + return fRetCol->sameColumn(rc.get()); + } + + private: + const Mt_& fRetCol; + }; + + public: + void operator()(typename T::value_type mapItem) + { + Pred pred(mapItem.second); + RetColsVector::iterator iter; + iter = find_if(fRetColsVec.begin(), fRetColsVec.end(), pred); + + if (iter == fRetColsVec.end()) + { + // Add this ReturnedColumn + fRetColsVec.push_back(mapItem.second); + } + } + RetColsVector fRetColsVec; }; uint16_t numberSteps(JobStepVector& steps, uint16_t stepNo, uint32_t flags) { - JobStepVector::iterator iter = steps.begin(); - JobStepVector::iterator end = steps.end(); + JobStepVector::iterator iter = steps.begin(); + JobStepVector::iterator end = steps.end(); - while (iter != end) - { - // don't number the delimiters - //if (dynamic_cast(iter->get()) != NULL) - //{ - // ++iter; - // continue; - //} + while (iter != end) + { + // don't number the delimiters + // if (dynamic_cast(iter->get()) != NULL) + //{ + // ++iter; + // continue; + //} - JobStep* pJobStep = iter->get(); - pJobStep->stepId(stepNo); - pJobStep->setTraceFlags(flags); - stepNo++; - ++iter; - } + JobStep* pJobStep = iter->get(); + pJobStep->stepId(stepNo); + pJobStep->setTraceFlags(flags); + stepNo++; + ++iter; + } - return stepNo; + return stepNo; } void changePcolStepToPcolScan(JobStepVector::iterator& it, JobStepVector::iterator& end) { - // make sure no pseudo column is a scan column - idbassert(dynamic_cast(it->get()) == NULL); + // make sure no pseudo column is a scan column + idbassert(dynamic_cast(it->get()) == NULL); - pColStep* colStep = dynamic_cast(it->get()); - pColScanStep* scanStep = 0; + pColStep* colStep = dynamic_cast(it->get()); + pColScanStep* scanStep = 0; - //Might be a pDictionaryScan step - if (colStep) + // Might be a pDictionaryScan step + if (colStep) + { + scanStep = new pColScanStep(*colStep); + } + else + { + // If we have a pDictionaryScan-pColStep duo, then change the pColStep + if (typeid(*(it->get())) == typeid(pDictionaryScan) && std::distance(it, end) > 1 && + typeid(*((it + 1)->get())) == typeid(pColStep)) { - scanStep = new pColScanStep(*colStep); - } - else - { - //If we have a pDictionaryScan-pColStep duo, then change the pColStep - if (typeid(*(it->get())) == typeid(pDictionaryScan) && - std::distance(it, end) > 1 && - typeid(*((it + 1)->get())) == typeid(pColStep)) - { - ++it; - colStep = dynamic_cast(it->get()); - scanStep = new pColScanStep(*colStep); - } + ++it; + colStep = dynamic_cast(it->get()); + scanStep = new pColScanStep(*colStep); } + } - if (scanStep) - { - it->reset(scanStep); - } + if (scanStep) + { + it->reset(scanStep); + } } // optimize filter order // perform none string filters first because string filter joins the tokens. void optimizeFilterOrder(JobStepVector& qsv) { - // move all none string filters - uint64_t pdsPos = 0; + // move all none string filters + uint64_t pdsPos = 0; -// int64_t orbranch = 0; - for (; pdsPos < qsv.size(); ++pdsPos) + // int64_t orbranch = 0; + for (; pdsPos < qsv.size(); ++pdsPos) + { + // skip the or branches + // OrDelimiterLhs* lhs = dynamic_cast(qsv[pdsPos].get()); + // if (lhs != NULL) + // { + // orbranch++; + // continue; + // } + // + // if (orbranch > 0) + // { + // UnionStep* us = dynamic_cast(qsv[pdsPos].get()); + // if (us) + // orbranch--; + // } + // else { - // skip the or branches -// OrDelimiterLhs* lhs = dynamic_cast(qsv[pdsPos].get()); -// if (lhs != NULL) -// { -// orbranch++; -// continue; -// } -// -// if (orbranch > 0) -// { -// UnionStep* us = dynamic_cast(qsv[pdsPos].get()); -// if (us) -// orbranch--; -// } -// else - { - pDictionaryScan* pds = dynamic_cast(qsv[pdsPos].get()); + pDictionaryScan* pds = dynamic_cast(qsv[pdsPos].get()); - if (pds) - break; - } + if (pds) + break; } + } - // no pDictionaryScan step - if (pdsPos >= qsv.size()) - return; + // no pDictionaryScan step + if (pdsPos >= qsv.size()) + return; - // get the filter steps that are not in or branches - vector pcolIdVec; - JobStepVector pcolStepVec; + // get the filter steps that are not in or branches + vector pcolIdVec; + JobStepVector pcolStepVec; -// orbranch = 0; - for (uint64_t i = pdsPos; i < qsv.size(); ++i) + // orbranch = 0; + for (uint64_t i = pdsPos; i < qsv.size(); ++i) + { + // OrDelimiterLhs* lhs = dynamic_cast(qsv[pdsPos].get()); + // if (lhs != NULL) + // { + // orbranch++; + // continue; + // } + + // if (orbranch > 0) + // { + // UnionStep* us = dynamic_cast(qsv[pdsPos].get()); + // if (us) + // orbranch--; + // } + // else { -// OrDelimiterLhs* lhs = dynamic_cast(qsv[pdsPos].get()); -// if (lhs != NULL) -// { -// orbranch++; -// continue; -// } + pColStep* pcol = dynamic_cast(qsv[i].get()); -// if (orbranch > 0) -// { -// UnionStep* us = dynamic_cast(qsv[pdsPos].get()); -// if (us) -// orbranch--; -// } -// else - { - pColStep* pcol = dynamic_cast(qsv[i].get()); - - if (pcol != NULL && pcol->filterCount() > 0) - pcolIdVec.push_back(i); - } + if (pcol != NULL && pcol->filterCount() > 0) + pcolIdVec.push_back(i); } + } - for (vector::reverse_iterator r = pcolIdVec.rbegin(); r < pcolIdVec.rend(); ++r) - { - pcolStepVec.push_back(qsv[*r]); - qsv.erase(qsv.begin() + (*r)); - } + for (vector::reverse_iterator r = pcolIdVec.rbegin(); r < pcolIdVec.rend(); ++r) + { + pcolStepVec.push_back(qsv[*r]); + qsv.erase(qsv.begin() + (*r)); + } - qsv.insert(qsv.begin() + pdsPos, pcolStepVec.rbegin(), pcolStepVec.rend()); + qsv.insert(qsv.begin() + pdsPos, pcolStepVec.rbegin(), pcolStepVec.rend()); } void exceptionHandler(JobList* joblist, const JobInfo& jobInfo, const string& logMsg, logging::LOG_TYPE logLevel = LOG_TYPE_ERROR) { - cerr << "### JobListFactory ses:" << jobInfo.sessionId << " caught: " << logMsg << endl; - Message::Args args; - args.add(logMsg); - jobInfo.logger->logMessage(logLevel, LogMakeJobList, args, - LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); - // dummy delivery map, workaround for (qb == 2) in main.cpp - DeliveredTableMap dtm; - SJSTEP dummyStep; - dtm[0] = dummyStep; - joblist->addDelivery(dtm); + cerr << "### JobListFactory ses:" << jobInfo.sessionId << " caught: " << logMsg << endl; + Message::Args args; + args.add(logMsg); + jobInfo.logger->logMessage(logLevel, LogMakeJobList, args, + LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); + // dummy delivery map, workaround for (qb == 2) in main.cpp + DeliveredTableMap dtm; + SJSTEP dummyStep; + dtm[0] = dummyStep; + joblist->addDelivery(dtm); } - -void parseExecutionPlan(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, - JobStepVector& querySteps, JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) +void parseExecutionPlan(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, + JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) { - ParseTree* filters = csep->filters(); - jobInfo.deliveredCols = csep->returnedCols(); + ParseTree* filters = csep->filters(); + jobInfo.deliveredCols = csep->returnedCols(); - if (filters != 0) + if (filters != 0) + { + JLF_ExecPlanToJobList::walkTree(filters, jobInfo); + } + + if (jobInfo.trace) + cout << endl << "Stack: " << endl; + + if (!jobInfo.stack.empty()) + { + idbassert(jobInfo.stack.size() == 1); + querySteps = jobInfo.stack.top(); + jobInfo.stack.pop(); + + // do some filter order optimization + optimizeFilterOrder(querySteps); + } + + if (jobInfo.selectAndFromSubs.size() > 0) + { + querySteps.insert(querySteps.begin(), jobInfo.selectAndFromSubs.begin(), jobInfo.selectAndFromSubs.end()); + } + + // bug4531, window function support + WindowFunctionStep::checkWindowFunction(csep, jobInfo); + + // bug3391, move forward the aggregation check for no aggregate having clause. + checkAggregation(csep, jobInfo); + + // include filters in having clause, if any. + if (jobInfo.havingStepVec.size() > 0) + querySteps.insert(querySteps.begin(), jobInfo.havingStepVec.begin(), jobInfo.havingStepVec.end()); + + // Need to change the leading pColStep to a pColScanStep + // Keep a list of the (table OIDs,alias) that we've already processed for @bug 598 self-join + set seenTableIds; + + // Stack of seenTables to make sure the left-hand side and right-hand have the same content + stack > seenTableStack; + + if (!querySteps.empty()) + { + JobStepVector::iterator iter = querySteps.begin(); + JobStepVector::iterator end = querySteps.end(); + + for (; iter != end; ++iter) { - JLF_ExecPlanToJobList::walkTree(filters, jobInfo); - } + idbassert(iter->get()); - if (jobInfo.trace) - cout << endl << "Stack: " << endl; + // As of bug3695, make sure varbinary is not used in filters. + if (typeid(*(iter->get())) == typeid(pColStep)) + { + // only pcolsteps, no pcolscan yet. + pColStep* pcol = dynamic_cast(iter->get()); - if (!jobInfo.stack.empty()) - { - idbassert(jobInfo.stack.size() == 1); - querySteps = jobInfo.stack.top(); - jobInfo.stack.pop(); - - // do some filter order optimization - optimizeFilterOrder(querySteps); - } - - if (jobInfo.selectAndFromSubs.size() > 0) - { - querySteps.insert(querySteps.begin(), - jobInfo.selectAndFromSubs.begin(), jobInfo.selectAndFromSubs.end()); - } - - // bug4531, window function support - WindowFunctionStep::checkWindowFunction(csep, jobInfo); - - // bug3391, move forward the aggregation check for no aggregate having clause. - checkAggregation(csep, jobInfo); - - // include filters in having clause, if any. - if (jobInfo.havingStepVec.size() > 0) - querySteps.insert(querySteps.begin(), - jobInfo.havingStepVec.begin(), jobInfo.havingStepVec.end()); - - //Need to change the leading pColStep to a pColScanStep - //Keep a list of the (table OIDs,alias) that we've already processed for @bug 598 self-join - set seenTableIds; - - //Stack of seenTables to make sure the left-hand side and right-hand have the same content - stack > seenTableStack; - - if (!querySteps.empty()) - { - JobStepVector::iterator iter = querySteps.begin(); - JobStepVector::iterator end = querySteps.end(); - - for (; iter != end; ++iter) + if (pcol->colType().colDataType == CalpontSystemCatalog::VARBINARY) { - idbassert(iter->get()); + if (pcol->filterCount() != 1) + throw runtime_error("VARBINARY in filter or function is not supported."); - // As of bug3695, make sure varbinary is not used in filters. - if (typeid(*(iter->get())) == typeid(pColStep)) - { - // only pcolsteps, no pcolscan yet. - pColStep* pcol = dynamic_cast(iter->get()); + // error out if the filter is not "is null" or "is not null" + // should block "= null" and "!= null" ??? + messageqcpp::ByteStream filter = pcol->filterString(); + uint8_t op = 0; + filter >> op; + bool nullOp = (op == COMPARE_EQ || op == COMPARE_NE || op == COMPARE_NIL); + filter >> op; // skip roundFlag + uint64_t value = 0; + filter >> value; + nullOp = nullOp && (value == 0xfffffffffffffffeULL); - if (pcol->colType().colDataType == CalpontSystemCatalog::VARBINARY) - { - if (pcol->filterCount() != 1) - throw runtime_error ("VARBINARY in filter or function is not supported."); - - // error out if the filter is not "is null" or "is not null" - // should block "= null" and "!= null" ??? - messageqcpp::ByteStream filter = pcol->filterString(); - uint8_t op = 0; - filter >> op; - bool nullOp = (op == COMPARE_EQ || op == COMPARE_NE || op == COMPARE_NIL); - filter >> op; // skip roundFlag - uint64_t value = 0; - filter >> value; - nullOp = nullOp && (value == 0xfffffffffffffffeULL); - - if (nullOp == false) - throw runtime_error ("VARBINARY in filter or function is not supported."); - } - } - -// // save the current seentable for right-hand side -// if (typeid(*(iter->get())) == typeid(OrDelimiterLhs)) -// { -// seenTableStack.push(seenTableIds); -// continue; -// } -// -// // restore the seentable -// else if (typeid(*(iter->get())) == typeid(OrDelimiterRhs)) -// { -// seenTableIds = seenTableStack.top(); -// seenTableStack.pop(); -// continue; -// } - - if (typeid(*(iter->get())) == typeid(pColStep)) - { - pColStep* colStep = dynamic_cast(iter->get()); - string alias(colStep->alias()); - string view(colStep->view()); - //If this is the first time we've seen this table or alias - uint32_t tableId = 0; - tableId = getTableKey(jobInfo, colStep->tupleId()); - - if (seenTableIds.find(tableId) == seenTableIds.end()) - changePcolStepToPcolScan(iter, end); - - //Mark this OID as seen - seenTableIds.insert(tableId); - } + if (nullOp == false) + throw runtime_error("VARBINARY in filter or function is not supported."); } - } + } - //build the project steps - if (jobInfo.deliveredCols.empty()) - { - throw logic_error("No delivery column."); - } + // // save the current seentable for right-hand side + // if (typeid(*(iter->get())) == typeid(OrDelimiterLhs)) + // { + // seenTableStack.push(seenTableIds); + // continue; + // } + // + // // restore the seentable + // else if (typeid(*(iter->get())) == typeid(OrDelimiterRhs)) + // { + // seenTableIds = seenTableStack.top(); + // seenTableStack.pop(); + // continue; + // } - // if any aggregate columns - if (jobInfo.hasAggregation == true) - { - projectSteps = doAggProject(csep, jobInfo); + if (typeid(*(iter->get())) == typeid(pColStep)) + { + pColStep* colStep = dynamic_cast(iter->get()); + string alias(colStep->alias()); + string view(colStep->view()); + // If this is the first time we've seen this table or alias + uint32_t tableId = 0; + tableId = getTableKey(jobInfo, colStep->tupleId()); + + if (seenTableIds.find(tableId) == seenTableIds.end()) + changePcolStepToPcolScan(iter, end); + + // Mark this OID as seen + seenTableIds.insert(tableId); + } } + } + + // build the project steps + if (jobInfo.deliveredCols.empty()) + { + throw logic_error("No delivery column."); + } + + // if any aggregate columns + if (jobInfo.hasAggregation == true) + { + projectSteps = doAggProject(csep, jobInfo); + } + else + { + projectSteps = doProject(jobInfo.nonConstCols, jobInfo); + } + + // bug3736, have jobInfo include the column map info. + const CalpontSelectExecutionPlan::ColumnMap& retCols = csep->columnMap(); + CalpontSelectExecutionPlan::ColumnMap::const_iterator i = retCols.begin(); + + for (; i != retCols.end(); i++) + { + SimpleColumn* sc = dynamic_cast(i->second.get()); + + if (sc && !sc->schemaName().empty()) + { + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + CalpontSystemCatalog::ColType ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + if (sc->isColumnStore() && dynamic_cast(sc) == NULL) + { + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; + } + + // X + + string alias(extractTableAlias(sc)); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + uint32_t colKey = ti.key; + uint32_t tblKey = getTableKey(jobInfo, colKey); + jobInfo.columnMap[tblKey].push_back(colKey); + + if (jobInfo.tableColMap.find(tblKey) == jobInfo.tableColMap.end()) + jobInfo.tableColMap[tblKey] = i->second; + } + } + + // special case, select without a table, like: select 1; + if (jobInfo.constantCol == CONST_COL_ONLY) + return; + + // If there are no filters (select * from table;) then add one simple scan + // TODO: more work here... + // @bug 497 fix. populate a map of tableoid for querysteps. tablescan + // cols whose table does not belong to the map + typedef set tableIDMap_t; + tableIDMap_t tableIDMap; + JobStepVector::iterator qsiter = querySteps.begin(); + JobStepVector::iterator qsend = querySteps.end(); + uint32_t tableId = 0; + + while (qsiter != qsend) + { + JobStep* js = qsiter->get(); + + if (js->tupleId() != (uint64_t)-1) + tableId = getTableKey(jobInfo, js->tupleId()); + + tableIDMap.insert(tableId); + ++qsiter; + } + + JobStepVector::iterator jsiter = projectSteps.begin(); + JobStepVector::iterator jsend = projectSteps.end(); + + while (jsiter != jsend) + { + JobStep* js = jsiter->get(); + + if (js->tupleId() != (uint64_t)-1) + tableId = getTableKey(jobInfo, js->tupleId()); else + tableId = getTableKey(jobInfo, js); + + if (typeid(*(jsiter->get())) == typeid(pColStep) && tableIDMap.find(tableId) == tableIDMap.end()) { - projectSteps = doProject(jobInfo.nonConstCols, jobInfo); + SJSTEP step0 = *jsiter; + pColStep* colStep = dynamic_cast(step0.get()); + pColScanStep* scanStep = new pColScanStep(*colStep); + // clear out any output association so we get a nice, new one during association + scanStep->outputAssociation(JobStepAssociation()); + step0.reset(scanStep); + querySteps.push_back(step0); + js = step0.get(); + tableId = getTableKey(jobInfo, js->tupleId()); + tableIDMap.insert(tableId); } - // bug3736, have jobInfo include the column map info. - const CalpontSelectExecutionPlan::ColumnMap& retCols = csep->columnMap(); - CalpontSelectExecutionPlan::ColumnMap::const_iterator i = retCols.begin(); - - for (; i != retCols.end(); i++) - { - SimpleColumn* sc = dynamic_cast(i->second.get()); - - if (sc && !sc->schemaName().empty()) - { - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - CalpontSystemCatalog::ColType ct = sc->colType(); - -//XXX use this before connector sets colType in sc correctly. - if (sc->isColumnStore() && dynamic_cast(sc) == NULL) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } - -//X - - string alias(extractTableAlias(sc)); - TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); - uint32_t colKey = ti.key; - uint32_t tblKey = getTableKey(jobInfo, colKey); - jobInfo.columnMap[tblKey].push_back(colKey); - - if (jobInfo.tableColMap.find(tblKey) == jobInfo.tableColMap.end()) - jobInfo.tableColMap[tblKey] = i->second; - } - } - - // special case, select without a table, like: select 1; - if (jobInfo.constantCol == CONST_COL_ONLY) - return; - - //If there are no filters (select * from table;) then add one simple scan - //TODO: more work here... - // @bug 497 fix. populate a map of tableoid for querysteps. tablescan - // cols whose table does not belong to the map - typedef set tableIDMap_t; - tableIDMap_t tableIDMap; - JobStepVector::iterator qsiter = querySteps.begin(); - JobStepVector::iterator qsend = querySteps.end(); - uint32_t tableId = 0; - - while (qsiter != qsend) - { - JobStep* js = qsiter->get(); - - if (js->tupleId() != (uint64_t) - 1) - tableId = getTableKey(jobInfo, js->tupleId()); - - tableIDMap.insert(tableId); - ++qsiter; - } - - JobStepVector::iterator jsiter = projectSteps.begin(); - JobStepVector::iterator jsend = projectSteps.end(); - - while (jsiter != jsend) - { - JobStep* js = jsiter->get(); - - if (js->tupleId() != (uint64_t) - 1) - tableId = getTableKey(jobInfo, js->tupleId()); - else - tableId = getTableKey(jobInfo, js); - - if (typeid(*(jsiter->get())) == typeid(pColStep) && - tableIDMap.find(tableId) == tableIDMap.end()) - { - SJSTEP step0 = *jsiter; - pColStep* colStep = dynamic_cast(step0.get()); - pColScanStep* scanStep = new pColScanStep(*colStep); - //clear out any output association so we get a nice, new one during association - scanStep->outputAssociation(JobStepAssociation()); - step0.reset(scanStep); - querySteps.push_back(step0); - js = step0.get(); - tableId = getTableKey(jobInfo, js->tupleId()); - tableIDMap.insert(tableId); - } - - ++jsiter; - } + ++jsiter; + } } - // v-table mode -void makeVtableModeSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, - JobStepVector& querySteps, JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) +void makeVtableModeSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, + JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) { - // special case for outer query order by limit -- return all - if (jobInfo.subId == 0 && csep->hasOrderBy() && !csep->specHandlerProcessed()) + // special case for outer query order by limit -- return all + if (jobInfo.subId == 0 && csep->hasOrderBy() && !csep->specHandlerProcessed()) + { + jobInfo.limitCount = (uint64_t)-1; + } + // support order by and limit in sub-query/union or + // GROUP BY handler processed outer query order + else if (csep->orderByCols().size() > 0) + { + addOrderByAndLimit(csep, jobInfo); + } + // limit without order by in any query + else + { + jobInfo.limitStart = csep->limitStart(); + jobInfo.limitCount = csep->limitNum(); + } + + // Bug 2123. Added overrideLargeSideEstimate parm below. True if the query was written + // with a hint telling us to skip the estimatation process for determining the large side + // table and instead use the table order in the from clause. + associateTupleJobSteps(querySteps, projectSteps, deliverySteps, jobInfo, csep->overrideLargeSideEstimate()); + uint16_t stepNo = jobInfo.subId * 10000; + numberSteps(querySteps, stepNo, jobInfo.traceFlags); + // SJSTEP ds = deliverySteps.begin()->second; + idbassert(deliverySteps.begin()->second.get()); + // ds->stepId(stepNo); + // ds->setTraceFlags(jobInfo.traceFlags); +} + +void makeAnalyzeTableJobSteps(MCSAnalyzeTableExecutionPlan* caep, JobInfo& jobInfo, JobStepVector& querySteps, + DeliveredTableMap& deliverySteps) +{ + JobStepVector projectSteps; + const auto& retCols = caep->returnedCols(); + + if (retCols.size() == 0) + return; + + // This is strange, but to generate a valid `out rowgroup` in AnnexStep we have to initialize + // `nonConstDelCols`, otherwise we will get an empty result in ExeMgr. Why don't just use `in + // row group`? + jobInfo.nonConstDelCols = retCols; + + // Iterate over `returned columns` and create a `pColStep` for each. + for (uint32_t i = 0; i < retCols.size(); i++) + { + const SimpleColumn* sc = dynamic_cast(retCols[i].get()); + CalpontSystemCatalog::OID oid = sc->oid(); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + CalpontSystemCatalog::ColType colType; + + if (!sc->schemaName().empty()) { - jobInfo.limitCount = (uint64_t) - 1; + SJSTEP sjstep; + colType = sc->colType(); + auto* pcs = new pColStep(oid, tblOid, colType, jobInfo); + pcs->alias(extractTableAlias(sc)); + pcs->view(sc->viewName()); + pcs->name(sc->columnName()); + pcs->cardinality(sc->cardinality()); + + auto ti = setTupleInfo(colType, oid, jobInfo, tblOid, sc, pcs->alias()); + pcs->tupleId(ti.key); + + sjstep.reset(pcs); + projectSteps.push_back(sjstep); } - // support order by and limit in sub-query/union or - // GROUP BY handler processed outer query order - else if (csep->orderByCols().size() > 0) + } + + if (projectSteps.size() == 0) + return; + + // Transform the first `pColStep` to `pColScanStep`. + SJSTEP firstStep = projectSteps.front(); + pColStep* colStep = static_cast(firstStep.get()); + // Create a `pColScanStep` using first `pColStep`. + pColScanStep* scanStep = new pColScanStep(*colStep); + scanStep->outputAssociation(JobStepAssociation()); + // Update first step. + firstStep.reset(scanStep); + + // Create tuple BPS step. + TupleBPS* tbps = new TupleBPS(*scanStep, jobInfo); + tbps->setFirstStepType(SCAN); + + // One `filter` step is scan step. + tbps->setBPP(scanStep); + tbps->setStepCount(); + + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); + + bool passThruCreated = false; + for (JobStepVector::iterator it = projectSteps.begin(); it != projectSteps.end(); it++) + { + JobStep* js = it->get(); + auto* colStep = static_cast(js); + + // TODO: Hoist the condition branch from the cycle, it will look ugly, but probaby faster. + if (UNLIKELY(!passThruCreated)) { - addOrderByAndLimit(csep, jobInfo); + PassThruStep* pts = new PassThruStep(*colStep); + passThruCreated = true; + pts->alias(colStep->alias()); + pts->view(colStep->view()); + pts->name(colStep->name()); + pts->tupleId(colStep->tupleId()); + tbps->setProjectBPP(pts, NULL); } - // limit without order by in any query else { - jobInfo.limitStart = csep->limitStart(); - jobInfo.limitCount = csep->limitNum(); + tbps->setProjectBPP(it->get(), NULL); } - // Bug 2123. Added overrideLargeSideEstimate parm below. True if the query was written - // with a hint telling us to skip the estimatation process for determining the large side - // table and instead use the table order in the from clause. - associateTupleJobSteps(querySteps, projectSteps, deliverySteps, - jobInfo, csep->overrideLargeSideEstimate()); - uint16_t stepNo = jobInfo.subId * 10000; - numberSteps(querySteps, stepNo, jobInfo.traceFlags); -// SJSTEP ds = deliverySteps.begin()->second; - idbassert(deliverySteps.begin()->second.get()); -// ds->stepId(stepNo); -// ds->setTraceFlags(jobInfo.traceFlags); + TupleInfo ti(getTupleInfo(colStep->tupleId(), jobInfo)); + pos.push_back(pos.back() + ti.width); + oids.push_back(ti.oid); + keys.push_back(ti.key); + types.push_back(ti.dtype); + csNums.push_back(ti.csNum); + scale.push_back(ti.scale); + precision.push_back(ti.precision); + } + + RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, 20); + + SJSTEP sjsp; + sjsp.reset(tbps); + // Add tuple BPS step to query steps. + querySteps.push_back(sjsp); + + // Delivery step. + SJSTEP annexStep; + auto* tas = new TupleAnnexStep(jobInfo); + annexStep.reset(tas); + + // Create input `RowGroupDL`. + RowGroupDL* dlIn = new RowGroupDL(1, jobInfo.fifoSize); + dlIn->OID(CNX_VTABLE_ID); + AnyDataListSPtr spdlIn(new AnyDataList()); + spdlIn->rowGroupDL(dlIn); + JobStepAssociation jsaIn; + jsaIn.outAdd(spdlIn); + + // Create output `RowGroupDL`. + RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); + dlOut->OID(CNX_VTABLE_ID); + AnyDataListSPtr spdlOut(new AnyDataList()); + spdlOut->rowGroupDL(dlOut); + JobStepAssociation jsaOut; + jsaOut.outAdd(spdlOut); + + // Set input and output. + tbps->setOutputRowGroup(rg); + tbps->outputAssociation(jsaIn); + annexStep->inputAssociation(jsaIn); + annexStep->outputAssociation(jsaOut); + + // Initialize. + tas->initialize(rg, jobInfo); + + // Add `annexStep` to delivery steps and to query steps. + deliverySteps[CNX_VTABLE_ID] = annexStep; + querySteps.push_back(annexStep); + + if (jobInfo.trace) + { + std::cout << "TupleBPS created: " << std::endl; + std::cout << tbps->toString() << std::endl; + std::cout << "Result row group: " << std::endl; + std::cout << rg.toString() << std::endl; + } } -void makeAnalyzeTableJobSteps(MCSAnalyzeTableExecutionPlan* caep, JobInfo& jobInfo, - JobStepVector& querySteps, DeliveredTableMap& deliverySteps) -{ - JobStepVector projectSteps; - const auto& retCols = caep->returnedCols(); - - if (retCols.size() == 0) - return; - - // This is strange, but to generate a valid `out rowgroup` in AnnexStep we have to initialize - // `nonConstDelCols`, otherwise we will get an empty result in ExeMgr. Why don't just use `in - // row group`? - jobInfo.nonConstDelCols = retCols; - - // Iterate over `returned columns` and create a `pColStep` for each. - for (uint32_t i = 0; i < retCols.size(); i++) - { - const SimpleColumn* sc = dynamic_cast(retCols[i].get()); - CalpontSystemCatalog::OID oid = sc->oid(); - CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); - CalpontSystemCatalog::ColType colType; - - if (!sc->schemaName().empty()) - { - SJSTEP sjstep; - colType = sc->colType(); - auto* pcs = new pColStep(oid, tblOid, colType, jobInfo); - pcs->alias(extractTableAlias(sc)); - pcs->view(sc->viewName()); - pcs->name(sc->columnName()); - pcs->cardinality(sc->cardinality()); - - auto ti = setTupleInfo(colType, oid, jobInfo, tblOid, sc, pcs->alias()); - pcs->tupleId(ti.key); - - sjstep.reset(pcs); - projectSteps.push_back(sjstep); - } - } - - if (projectSteps.size() == 0) - return; - - // Transform the first `pColStep` to `pColScanStep`. - SJSTEP firstStep = projectSteps.front(); - pColStep* colStep = static_cast(firstStep.get()); - // Create a `pColScanStep` using first `pColStep`. - pColScanStep* scanStep = new pColScanStep(*colStep); - scanStep->outputAssociation(JobStepAssociation()); - // Update first step. - firstStep.reset(scanStep); - - // Create tuple BPS step. - TupleBPS* tbps = new TupleBPS(*scanStep, jobInfo); - tbps->setFirstStepType(SCAN); - - // One `filter` step is scan step. - tbps->setBPP(scanStep); - tbps->setStepCount(); - - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); - - bool passThruCreated = false; - for (JobStepVector::iterator it = projectSteps.begin(); it != projectSteps.end(); it++) - { - JobStep* js = it->get(); - auto* colStep = static_cast(js); - - // TODO: Hoist the condition branch from the cycle, it will look ugly, but probaby faster. - if (UNLIKELY(!passThruCreated)) - { - PassThruStep* pts = new PassThruStep(*colStep); - passThruCreated = true; - pts->alias(colStep->alias()); - pts->view(colStep->view()); - pts->name(colStep->name()); - pts->tupleId(colStep->tupleId()); - tbps->setProjectBPP(pts, NULL); - } - else - { - tbps->setProjectBPP(it->get(), NULL); - } - - TupleInfo ti(getTupleInfo(colStep->tupleId(), jobInfo)); - pos.push_back(pos.back() + ti.width); - oids.push_back(ti.oid); - keys.push_back(ti.key); - types.push_back(ti.dtype); - csNums.push_back(ti.csNum); - scale.push_back(ti.scale); - precision.push_back(ti.precision); - } - - RowGroup rg(oids.size(), pos, oids, keys, types, csNums, scale, precision, 20); - - SJSTEP sjsp; - sjsp.reset(tbps); - // Add tuple BPS step to query steps. - querySteps.push_back(sjsp); - - // Delivery step. - SJSTEP annexStep; - auto* tas = new TupleAnnexStep(jobInfo); - annexStep.reset(tas); - - // Create input `RowGroupDL`. - RowGroupDL* dlIn = new RowGroupDL(1, jobInfo.fifoSize); - dlIn->OID(CNX_VTABLE_ID); - AnyDataListSPtr spdlIn(new AnyDataList()); - spdlIn->rowGroupDL(dlIn); - JobStepAssociation jsaIn; - jsaIn.outAdd(spdlIn); - - // Create output `RowGroupDL`. - RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); - dlOut->OID(CNX_VTABLE_ID); - AnyDataListSPtr spdlOut(new AnyDataList()); - spdlOut->rowGroupDL(dlOut); - JobStepAssociation jsaOut; - jsaOut.outAdd(spdlOut); - - // Set input and output. - tbps->setOutputRowGroup(rg); - tbps->outputAssociation(jsaIn); - annexStep->inputAssociation(jsaIn); - annexStep->outputAssociation(jsaOut); - - // Initialize. - tas->initialize(rg, jobInfo); - - // Add `annexStep` to delivery steps and to query steps. - deliverySteps[CNX_VTABLE_ID] = annexStep; - querySteps.push_back(annexStep); - - if (jobInfo.trace) - { - std::cout << "TupleBPS created: " << std::endl; - std::cout << tbps->toString() << std::endl; - std::cout << "Result row group: " << std::endl; - std::cout << rg.toString() << std::endl; - } -} - -} // namespace +} // namespace namespace joblist { - -void makeJobSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, - JobStepVector& querySteps, JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) +void makeJobSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, + JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) { - // v-table mode, switch to tuple methods and return the tuple joblist. - //@Bug 1958 Build table list only for tryTuples. - const CalpontSelectExecutionPlan::SelectList& fromSubquery = csep->derivedTableList(); - int i = 0; + // v-table mode, switch to tuple methods and return the tuple joblist. + //@Bug 1958 Build table list only for tryTuples. + const CalpontSelectExecutionPlan::SelectList& fromSubquery = csep->derivedTableList(); + int i = 0; - for (CalpontSelectExecutionPlan::TableList::const_iterator it = csep->tableList().begin(); - it != csep->tableList().end(); - it++) - { - CalpontSystemCatalog::OID oid; + for (CalpontSelectExecutionPlan::TableList::const_iterator it = csep->tableList().begin(); + it != csep->tableList().end(); it++) + { + CalpontSystemCatalog::OID oid; - if (it->schema.empty()) - oid = doFromSubquery(fromSubquery[i++].get(), it->alias, it->view, jobInfo); - else if (it->fisColumnStore) - oid = jobInfo.csc->tableRID(*it).objnum; - else - oid = 0; + if (it->schema.empty()) + oid = doFromSubquery(fromSubquery[i++].get(), it->alias, it->view, jobInfo); + else if (it->fisColumnStore) + oid = jobInfo.csc->tableRID(*it).objnum; + else + oid = 0; - uint32_t tableUid = makeTableKey(jobInfo, oid, it->table, it->alias, it->schema, it->view); - jobInfo.tableList.push_back(tableUid); - } + uint32_t tableUid = makeTableKey(jobInfo, oid, it->table, it->alias, it->schema, it->view); + jobInfo.tableList.push_back(tableUid); + } - // add select suqueries - preprocessSelectSubquery(csep, jobInfo); + // add select suqueries + preprocessSelectSubquery(csep, jobInfo); - // semi-join may appear in having clause - if (csep->having() != NULL) - preprocessHavingClause(csep, jobInfo); + // semi-join may appear in having clause + if (csep->having() != NULL) + preprocessHavingClause(csep, jobInfo); - // parse plan and make jobstep list - parseExecutionPlan(csep, jobInfo, querySteps, projectSteps, deliverySteps); - makeVtableModeSteps(csep, jobInfo, querySteps, projectSteps, deliverySteps); + // parse plan and make jobstep list + parseExecutionPlan(csep, jobInfo, querySteps, projectSteps, deliverySteps); + makeVtableModeSteps(csep, jobInfo, querySteps, projectSteps, deliverySteps); } -void makeUnionJobSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, - JobStepVector& querySteps, JobStepVector&, DeliveredTableMap& deliverySteps) +void makeUnionJobSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, + JobStepVector&, DeliveredTableMap& deliverySteps) { - CalpontSelectExecutionPlan::SelectList& selectVec = csep->unionVec(); - uint8_t distinctUnionNum = csep->distinctUnionNum(); - RetColsVector unionRetCols = csep->returnedCols(); - JobStepVector unionFeeders; + CalpontSelectExecutionPlan::SelectList& selectVec = csep->unionVec(); + uint8_t distinctUnionNum = csep->distinctUnionNum(); + RetColsVector unionRetCols = csep->returnedCols(); + JobStepVector unionFeeders; - for (CalpontSelectExecutionPlan::SelectList::iterator cit = selectVec.begin(); - cit != selectVec.end(); - cit++) - { - // @bug4848, enhance and unify limit handling. - SJSTEP sub = doUnionSub(cit->get(), jobInfo); - querySteps.push_back(sub); - unionFeeders.push_back(sub); - } + for (CalpontSelectExecutionPlan::SelectList::iterator cit = selectVec.begin(); cit != selectVec.end(); + cit++) + { + // @bug4848, enhance and unify limit handling. + SJSTEP sub = doUnionSub(cit->get(), jobInfo); + querySteps.push_back(sub); + unionFeeders.push_back(sub); + } - jobInfo.deliveredCols = unionRetCols; - SJSTEP unionStep(unionQueries(unionFeeders, distinctUnionNum, jobInfo)); - querySteps.push_back(unionStep); - uint16_t stepNo = jobInfo.subId * 10000; - numberSteps(querySteps, stepNo, jobInfo.traceFlags); - deliverySteps[execplan::CNX_VTABLE_ID] = unionStep; + jobInfo.deliveredCols = unionRetCols; + SJSTEP unionStep(unionQueries(unionFeeders, distinctUnionNum, jobInfo)); + querySteps.push_back(unionStep); + uint16_t stepNo = jobInfo.subId * 10000; + numberSteps(querySteps, stepNo, jobInfo.traceFlags); + deliverySteps[execplan::CNX_VTABLE_ID] = unionStep; } -} // namespace joblist +} // namespace joblist namespace { - -void handleException(std::exception_ptr e, JobList* jl, JobInfo& jobInfo, unsigned& errCode, - string& emsg) +void handleException(std::exception_ptr e, JobList* jl, JobInfo& jobInfo, unsigned& errCode, string& emsg) { + try + { + std::rethrow_exception(e); + } + catch (IDBExcept& iex) + { + jobInfo.errorInfo->errCode = iex.errorCode(); + errCode = iex.errorCode(); + exceptionHandler(jl, jobInfo, iex.what(), LOG_TYPE_DEBUG); + emsg = iex.what(); + } + catch (const std::exception& ex) + { + jobInfo.errorInfo->errCode = makeJobListErr; + errCode = makeJobListErr; + exceptionHandler(jl, jobInfo, ex.what()); + emsg = ex.what(); + } + catch (...) + { + jobInfo.errorInfo->errCode = makeJobListErr; + errCode = makeJobListErr; + exceptionHandler(jl, jobInfo, "an exception"); + emsg = "An unknown internal joblist error"; + } + delete jl; + jl = nullptr; +} + +SJLP makeJobList_(CalpontExecutionPlan* cplan, ResourceManager* rm, bool isExeMgr, unsigned& errCode, + string& emsg) +{ + // TODO: This part requires a proper refactoring, we have to move common methods from + // `CalpontSelectExecutionPlan` to the base class. I have no idea what's a point of + // `CalpontExecutionPlan` as base class without any meaningful virtual functions and common + // fields. The main thing I concern about - to make a huge refactoring of this before a week + // of release, because there is no time to test it and I do not want to introduce an errors in + // the existing code. Hope will make it on the next iteration of statistics development. + CalpontSelectExecutionPlan* csep = dynamic_cast(cplan); + if (csep != nullptr) + { + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(csep->sessionID()); + + static config::Config* sysConfig = config::Config::makeConfig(); + int pmsConfigured = atoi(sysConfig->getConfig("PrimitiveServers", "Count").c_str()); + + // We have to go ahead and create JobList now so we can store the joblist's + // projectTableOID pointer in JobInfo for use during jobstep creation. + SErrorInfo errorInfo(new ErrorInfo()); + boost::shared_ptr keyInfo(new TupleKeyInfo); + boost::shared_ptr subCount(new int); + *subCount = 0; + JobList* jl = new TupleJobList(isExeMgr); + jl->setPMsConfigured(pmsConfigured); + jl->priority(csep->priority()); + jl->errorInfo(errorInfo); + rm->setTraceFlags(csep->traceFlags()); + + // Stuff a util struct with some stuff we always need + JobInfo jobInfo(rm); + jobInfo.sessionId = csep->sessionID(); + jobInfo.txnId = csep->txnID(); + jobInfo.verId = csep->verID(); + jobInfo.statementId = csep->statementID(); + jobInfo.queryType = csep->queryType(); + jobInfo.csc = csc; + // TODO: clean up the vestiges of the bool trace + jobInfo.trace = csep->traceOn(); + jobInfo.traceFlags = csep->traceFlags(); + jobInfo.isExeMgr = isExeMgr; + // jobInfo.tryTuples = tryTuples; // always tuples after release 3.0 + jobInfo.stringScanThreshold = csep->stringScanThreshold(); + jobInfo.errorInfo = errorInfo; + jobInfo.keyInfo = keyInfo; + jobInfo.subCount = subCount; + jobInfo.projectingTableOID = jl->projectingTableOIDPtr(); + jobInfo.jobListPtr = jl; + jobInfo.stringTableThreshold = csep->stringTableThreshold(); + jobInfo.localQuery = csep->localQuery(); + jobInfo.uuid = csep->uuid(); + jobInfo.timeZone = csep->timeZone(); + + /* disk-based join vars */ + jobInfo.smallSideLimit = csep->djsSmallSideLimit(); + jobInfo.largeSideLimit = csep->djsLargeSideLimit(); + jobInfo.partitionSize = csep->djsPartitionSize(); + jobInfo.umMemLimit.reset(new int64_t); + *(jobInfo.umMemLimit) = csep->umMemLimit(); + jobInfo.isDML = csep->isDML(); + + jobInfo.smallSideUsage.reset(new int64_t); + *jobInfo.smallSideUsage = 0; + + // set fifoSize to 1 for CalpontSystemCatalog query + if (csep->sessionID() & 0x80000000) + jobInfo.fifoSize = 1; + else if (csep->traceOn()) + cout << (*csep) << endl; + try { - std::rethrow_exception(e); + JobStepVector querySteps; + JobStepVector projectSteps; + DeliveredTableMap deliverySteps; + + if (csep->unionVec().size() == 0) + makeJobSteps(csep, jobInfo, querySteps, projectSteps, deliverySteps); + else + makeUnionJobSteps(csep, jobInfo, querySteps, projectSteps, deliverySteps); + + uint16_t stepNo = numberSteps(querySteps, 0, jobInfo.traceFlags); + stepNo = numberSteps(projectSteps, stepNo, jobInfo.traceFlags); + + struct timeval stTime; + + if (jobInfo.trace) + { + ostringstream oss; + oss << endl; + oss << endl << "job parms: " << endl; + oss << "maxBuckets = " << jobInfo.maxBuckets << ", maxElems = " << jobInfo.maxElems + << ", flushInterval = " << jobInfo.flushInterval << ", fifoSize = " << jobInfo.fifoSize + << ", ScanLimit/Threshold = " << jobInfo.scanLbidReqLimit << "/" << jobInfo.scanLbidReqThreshold + << endl; + oss << "UUID: " << jobInfo.uuid << endl; + oss << endl << "job filter steps: " << endl; + ostream_iterator oIter(oss, "\n"); + copy(querySteps.begin(), querySteps.end(), oIter); + oss << endl << "job project steps: " << endl; + copy(projectSteps.begin(), projectSteps.end(), oIter); + oss << endl << "job delivery steps: " << endl; + DeliveredTableMap::iterator dsi = deliverySteps.begin(); + + while (dsi != deliverySteps.end()) + { + oss << dynamic_cast(dsi->second.get()) << endl; + ++dsi; + } + + oss << endl; + gettimeofday(&stTime, 0); + + struct tm tmbuf; +#ifdef _MSC_VER + errno_t p = 0; + time_t t = stTime.tv_sec; + p = localtime_s(&tmbuf, &t); + + if (p != 0) + memset(&tmbuf, 0, sizeof(tmbuf)); + +#else + localtime_r(&stTime.tv_sec, &tmbuf); +#endif + ostringstream tms; + tms << setfill('0') << setw(4) << (tmbuf.tm_year + 1900) << setw(2) << (tmbuf.tm_mon + 1) << setw(2) + << (tmbuf.tm_mday) << setw(2) << (tmbuf.tm_hour) << setw(2) << (tmbuf.tm_min) << setw(2) + << (tmbuf.tm_sec) << setw(6) << (stTime.tv_usec); + string tmstr(tms.str()); + string jsrname("jobstep." + tmstr + ".dot"); + ofstream dotFile(jsrname.c_str()); + jlf_graphics::writeDotCmds(dotFile, querySteps, projectSteps); + + char timestamp[80]; +#ifdef _MSC_VER + t = stTime.tv_sec; + p = ctime_s(timestamp, 80, &t); + + if (p != 0) + strcpy(timestamp, "UNKNOWN"); + +#else + ctime_r((const time_t*)&stTime.tv_sec, timestamp); +#endif + oss << "runtime updates: start at " << timestamp; + cout << oss.str(); + Message::Args args; + args.add(oss.str()); + jobInfo.logger->logMessage(LOG_TYPE_DEBUG, LogSQLTrace, args, + LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); + cout << flush; + } + else + { + gettimeofday(&stTime, 0); + } + + // Finish initializing the JobList object + jl->addQuery(querySteps); + jl->addProject(projectSteps); + jl->addDelivery(deliverySteps); + csep->setDynamicParseTreeVec(jobInfo.dynamicParseTreeVec); + + dynamic_cast(jl)->setDeliveryFlag(true); } catch (IDBExcept& iex) { - jobInfo.errorInfo->errCode = iex.errorCode(); - errCode = iex.errorCode(); - exceptionHandler(jl, jobInfo, iex.what(), LOG_TYPE_DEBUG); - emsg = iex.what(); + jobInfo.errorInfo->errCode = iex.errorCode(); + errCode = iex.errorCode(); + exceptionHandler(jl, jobInfo, iex.what(), LOG_TYPE_DEBUG); + emsg = iex.what(); + goto bailout; } catch (const std::exception& ex) { - jobInfo.errorInfo->errCode = makeJobListErr; - errCode = makeJobListErr; - exceptionHandler(jl, jobInfo, ex.what()); - emsg = ex.what(); + jobInfo.errorInfo->errCode = makeJobListErr; + errCode = makeJobListErr; + exceptionHandler(jl, jobInfo, ex.what()); + emsg = ex.what(); + goto bailout; } catch (...) { - jobInfo.errorInfo->errCode = makeJobListErr; - errCode = makeJobListErr; - exceptionHandler(jl, jobInfo, "an exception"); - emsg = "An unknown internal joblist error"; + jobInfo.errorInfo->errCode = makeJobListErr; + errCode = makeJobListErr; + exceptionHandler(jl, jobInfo, "an exception"); + emsg = "An unknown internal joblist error"; + goto bailout; } + + goto done; + + bailout: delete jl; - jl = nullptr; -} + jl = 0; -SJLP makeJobList_( - CalpontExecutionPlan* cplan, - ResourceManager* rm, - bool isExeMgr, - unsigned& errCode, string& emsg) -{ - // TODO: This part requires a proper refactoring, we have to move common methods from - // `CalpontSelectExecutionPlan` to the base class. I have no idea what's a point of - // `CalpontExecutionPlan` as base class without any meaningful virtual functions and common - // fields. The main thing I concern about - to make a huge refactoring of this before a week - // of release, because there is no time to test it and I do not want to introduce an errors in - // the existing code. Hope will make it on the next iteration of statistics development. - CalpontSelectExecutionPlan* csep = dynamic_cast(cplan); - if (csep != nullptr) + if (emsg.empty()) + emsg = "An unknown internal joblist error"; + + done: + SJLP jlp(jl); + return jlp; + } + else + { + auto* caep = dynamic_cast(cplan); + JobList* jl = nullptr; + + if (caep == nullptr) { - boost::shared_ptr csc = - CalpontSystemCatalog::makeCalpontSystemCatalog(csep->sessionID()); + SJLP jlp(jl); + std::cerr << "Ivalid execution plan" << std::endl; + return jlp; + } - static config::Config* sysConfig = config::Config::makeConfig(); - int pmsConfigured = atoi(sysConfig->getConfig("PrimitiveServers", "Count").c_str()); + jl = new TupleJobList(isExeMgr); + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(caep->sessionID()); - // We have to go ahead and create JobList now so we can store the joblist's - // projectTableOID pointer in JobInfo for use during jobstep creation. - SErrorInfo errorInfo(new ErrorInfo()); - boost::shared_ptr keyInfo(new TupleKeyInfo); - boost::shared_ptr subCount(new int); - *subCount = 0; - JobList* jl = new TupleJobList(isExeMgr); - jl->setPMsConfigured(pmsConfigured); - jl->priority(csep->priority()); - jl->errorInfo(errorInfo); - rm->setTraceFlags(csep->traceFlags()); + static config::Config* sysConfig = config::Config::makeConfig(); + uint32_t pmsConfigured = atoi(sysConfig->getConfig("PrimitiveServers", "Count").c_str()); - // Stuff a util struct with some stuff we always need - JobInfo jobInfo(rm); - jobInfo.sessionId = csep->sessionID(); - jobInfo.txnId = csep->txnID(); - jobInfo.verId = csep->verID(); - jobInfo.statementId = csep->statementID(); - jobInfo.queryType = csep->queryType(); - jobInfo.csc = csc; - // TODO: clean up the vestiges of the bool trace - jobInfo.trace = csep->traceOn(); - jobInfo.traceFlags = csep->traceFlags(); - jobInfo.isExeMgr = isExeMgr; - // jobInfo.tryTuples = tryTuples; // always tuples after release 3.0 - jobInfo.stringScanThreshold = csep->stringScanThreshold(); - jobInfo.errorInfo = errorInfo; - jobInfo.keyInfo = keyInfo; - jobInfo.subCount = subCount; - jobInfo.projectingTableOID = jl->projectingTableOIDPtr(); - jobInfo.jobListPtr = jl; - jobInfo.stringTableThreshold = csep->stringTableThreshold(); - jobInfo.localQuery = csep->localQuery(); - jobInfo.uuid = csep->uuid(); - jobInfo.timeZone = csep->timeZone(); + SErrorInfo errorInfo(new ErrorInfo()); + boost::shared_ptr keyInfo(new TupleKeyInfo); + boost::shared_ptr subCount(new int); + *subCount = 0; - /* disk-based join vars */ - jobInfo.smallSideLimit = csep->djsSmallSideLimit(); - jobInfo.largeSideLimit = csep->djsLargeSideLimit(); - jobInfo.partitionSize = csep->djsPartitionSize(); - jobInfo.umMemLimit.reset(new int64_t); - *(jobInfo.umMemLimit) = csep->umMemLimit(); - jobInfo.isDML = csep->isDML(); + jl->setPMsConfigured(pmsConfigured); + jl->priority(caep->priority()); + jl->errorInfo(errorInfo); - jobInfo.smallSideUsage.reset(new int64_t); - *jobInfo.smallSideUsage = 0; + JobInfo jobInfo(rm); + jobInfo.sessionId = caep->sessionID(); + jobInfo.txnId = caep->txnID(); + jobInfo.verId = caep->verID(); + jobInfo.statementId = caep->statementID(); + jobInfo.csc = csc; + jobInfo.trace = caep->traceOn(); + jobInfo.isExeMgr = isExeMgr; + // TODO: Implement it when we have a dict column. + jobInfo.stringScanThreshold = 20; + jobInfo.errorInfo = errorInfo; + jobInfo.keyInfo = keyInfo; + jobInfo.subCount = subCount; + jobInfo.projectingTableOID = jl->projectingTableOIDPtr(); + jobInfo.jobListPtr = jl; + jobInfo.localQuery = caep->localQuery(); + jobInfo.timeZone = caep->timeZone(); - // set fifoSize to 1 for CalpontSystemCatalog query - if (csep->sessionID() & 0x80000000) - jobInfo.fifoSize = 1; - else if (csep->traceOn()) - cout << (*csep) << endl; + try + { + JobStepVector querySteps; + DeliveredTableMap deliverySteps; - try - { - JobStepVector querySteps; - JobStepVector projectSteps; - DeliveredTableMap deliverySteps; + // Parse exe plan and create a jobstesp from it. + makeAnalyzeTableJobSteps(caep, jobInfo, querySteps, deliverySteps); - if (csep->unionVec().size() == 0) - makeJobSteps(csep, jobInfo, querySteps, projectSteps, deliverySteps); - else - makeUnionJobSteps(csep, jobInfo, querySteps, projectSteps, deliverySteps); - - uint16_t stepNo = numberSteps(querySteps, 0, jobInfo.traceFlags); - stepNo = numberSteps(projectSteps, stepNo, jobInfo.traceFlags); - - struct timeval stTime; - - if (jobInfo.trace) - { - ostringstream oss; - oss << endl; - oss << endl << "job parms: " << endl; - oss << "maxBuckets = " << jobInfo.maxBuckets << ", maxElems = " << jobInfo.maxElems - << ", flushInterval = " << jobInfo.flushInterval - << ", fifoSize = " << jobInfo.fifoSize - << ", ScanLimit/Threshold = " << jobInfo.scanLbidReqLimit << "/" - << jobInfo.scanLbidReqThreshold << endl; - oss << "UUID: " << jobInfo.uuid << endl; - oss << endl << "job filter steps: " << endl; - ostream_iterator oIter(oss, "\n"); - copy(querySteps.begin(), querySteps.end(), oIter); - oss << endl << "job project steps: " << endl; - copy(projectSteps.begin(), projectSteps.end(), oIter); - oss << endl << "job delivery steps: " << endl; - DeliveredTableMap::iterator dsi = deliverySteps.begin(); - - while (dsi != deliverySteps.end()) - { - oss << dynamic_cast(dsi->second.get()) << endl; - ++dsi; - } - - oss << endl; - gettimeofday(&stTime, 0); - - struct tm tmbuf; -#ifdef _MSC_VER - errno_t p = 0; - time_t t = stTime.tv_sec; - p = localtime_s(&tmbuf, &t); - - if (p != 0) - memset(&tmbuf, 0, sizeof(tmbuf)); - -#else - localtime_r(&stTime.tv_sec, &tmbuf); -#endif - ostringstream tms; - tms << setfill('0') << setw(4) << (tmbuf.tm_year + 1900) << setw(2) - << (tmbuf.tm_mon + 1) << setw(2) << (tmbuf.tm_mday) << setw(2) - << (tmbuf.tm_hour) << setw(2) << (tmbuf.tm_min) << setw(2) << (tmbuf.tm_sec) - << setw(6) << (stTime.tv_usec); - string tmstr(tms.str()); - string jsrname("jobstep." + tmstr + ".dot"); - ofstream dotFile(jsrname.c_str()); - jlf_graphics::writeDotCmds(dotFile, querySteps, projectSteps); - - char timestamp[80]; -#ifdef _MSC_VER - t = stTime.tv_sec; - p = ctime_s(timestamp, 80, &t); - - if (p != 0) - strcpy(timestamp, "UNKNOWN"); - -#else - ctime_r((const time_t*) &stTime.tv_sec, timestamp); -#endif - oss << "runtime updates: start at " << timestamp; - cout << oss.str(); - Message::Args args; - args.add(oss.str()); - jobInfo.logger->logMessage(LOG_TYPE_DEBUG, LogSQLTrace, args, - LoggingID(5, jobInfo.sessionId, jobInfo.txnId, 0)); - cout << flush; - } - else - { - gettimeofday(&stTime, 0); - } - - // Finish initializing the JobList object - jl->addQuery(querySteps); - jl->addProject(projectSteps); - jl->addDelivery(deliverySteps); - csep->setDynamicParseTreeVec(jobInfo.dynamicParseTreeVec); - - dynamic_cast(jl)->setDeliveryFlag(true); - } - catch (IDBExcept& iex) - { - jobInfo.errorInfo->errCode = iex.errorCode(); - errCode = iex.errorCode(); - exceptionHandler(jl, jobInfo, iex.what(), LOG_TYPE_DEBUG); - emsg = iex.what(); - goto bailout; - } - catch (const std::exception& ex) - { - jobInfo.errorInfo->errCode = makeJobListErr; - errCode = makeJobListErr; - exceptionHandler(jl, jobInfo, ex.what()); - emsg = ex.what(); - goto bailout; - } - catch (...) - { - jobInfo.errorInfo->errCode = makeJobListErr; - errCode = makeJobListErr; - exceptionHandler(jl, jobInfo, "an exception"); - emsg = "An unknown internal joblist error"; - goto bailout; - } - - goto done; - - bailout: + if (!querySteps.size()) + { delete jl; - jl = 0; + // Indicates that query steps is empty. + errCode = logging::statisticsJobListEmpty; + jl = nullptr; + goto out; + } - if (emsg.empty()) - emsg = "An unknown internal joblist error"; + numberSteps(querySteps, 0, jobInfo.traceFlags); - done: - SJLP jlp(jl); - return jlp; + jl->addQuery(querySteps); + jl->addDelivery(deliverySteps); + + dynamic_cast(jl)->setDeliveryFlag(true); } - else + catch (...) { - auto* caep = dynamic_cast(cplan); - JobList* jl = nullptr; - - if (caep == nullptr) - { - SJLP jlp(jl); - std::cerr << "Ivalid execution plan" << std::endl; - return jlp; - } - - jl = new TupleJobList(isExeMgr); - boost::shared_ptr csc = - CalpontSystemCatalog::makeCalpontSystemCatalog(caep->sessionID()); - - static config::Config* sysConfig = config::Config::makeConfig(); - uint32_t pmsConfigured = atoi(sysConfig->getConfig("PrimitiveServers", "Count").c_str()); - - SErrorInfo errorInfo(new ErrorInfo()); - boost::shared_ptr keyInfo(new TupleKeyInfo); - boost::shared_ptr subCount(new int); - *subCount = 0; - - jl->setPMsConfigured(pmsConfigured); - jl->priority(caep->priority()); - jl->errorInfo(errorInfo); - - JobInfo jobInfo(rm); - jobInfo.sessionId = caep->sessionID(); - jobInfo.txnId = caep->txnID(); - jobInfo.verId = caep->verID(); - jobInfo.statementId = caep->statementID(); - jobInfo.csc = csc; - jobInfo.trace = caep->traceOn(); - jobInfo.isExeMgr = isExeMgr; - // TODO: Implement it when we have a dict column. - jobInfo.stringScanThreshold = 20; - jobInfo.errorInfo = errorInfo; - jobInfo.keyInfo = keyInfo; - jobInfo.subCount = subCount; - jobInfo.projectingTableOID = jl->projectingTableOIDPtr(); - jobInfo.jobListPtr = jl; - jobInfo.localQuery = caep->localQuery(); - jobInfo.timeZone = caep->timeZone(); - - try - { - JobStepVector querySteps; - DeliveredTableMap deliverySteps; - - // Parse exe plan and create a jobstesp from it. - makeAnalyzeTableJobSteps(caep, jobInfo, querySteps, deliverySteps); - - if (!querySteps.size()) - { - delete jl; - // Indicates that query steps is empty. - errCode = logging::statisticsJobListEmpty; - jl = nullptr; - goto out; - } - - numberSteps(querySteps, 0, jobInfo.traceFlags); - - jl->addQuery(querySteps); - jl->addDelivery(deliverySteps); - - dynamic_cast(jl)->setDeliveryFlag(true); - } - catch (...) - { - handleException(std::current_exception(), jl, jobInfo, errCode, emsg); - } - - out: - SJLP jlp(jl); - return jlp; + handleException(std::current_exception(), jl, jobInfo, errCode, emsg); } + + out: + SJLP jlp(jl); + return jlp; + } } -} // namespace +} // namespace namespace joblist { - /* static */ -SJLP JobListFactory::makeJobList( - CalpontExecutionPlan* cplan, - ResourceManager* rm, - bool tryTuple, - bool isExeMgr) +SJLP JobListFactory::makeJobList(CalpontExecutionPlan* cplan, ResourceManager* rm, bool tryTuple, + bool isExeMgr) { - SJLP ret; - string emsg; - unsigned errCode = 0; + SJLP ret; + string emsg; + unsigned errCode = 0; - ret = makeJobList_(cplan, rm, isExeMgr, errCode, emsg); + ret = makeJobList_(cplan, rm, isExeMgr, errCode, emsg); - if (!ret) - { - ret.reset(new TupleJobList(isExeMgr)); - SErrorInfo errorInfo(new ErrorInfo); - errorInfo->errCode = errCode; - errorInfo->errMsg = emsg; - ret->errorInfo(errorInfo); - } + if (!ret) + { + ret.reset(new TupleJobList(isExeMgr)); + SErrorInfo errorInfo(new ErrorInfo); + errorInfo->errCode = errCode; + errorInfo->errMsg = emsg; + ret->errorInfo(errorInfo); + } - return ret; + return ret; } -} +} // namespace joblist // vim:ts=4 sw=4: #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif diff --git a/dbcon/joblist/joblistfactory.h b/dbcon/joblist/joblistfactory.h index 9ccef730f..a3efd9b96 100644 --- a/dbcon/joblist/joblistfactory.h +++ b/dbcon/joblist/joblistfactory.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: joblistfactory.h 9655 2013-06-25 23:08:13Z xlou $ -* -* -***********************************************************************/ + * $Id: joblistfactory.h 9655 2013-06-25 23:08:13Z xlou $ + * + * + ***********************************************************************/ /** @file */ #ifndef JOBLISTFACTORY_H @@ -42,7 +42,6 @@ class CalpontExecutionPlan; namespace joblist { - class ResourceManager; /** @brief create a JobList object from a CalpontExecutionPlan object * @@ -57,25 +56,19 @@ class ResourceManager; */ class JobListFactory { + public: + /** @brief static JobList constructor method + * + * @param cplan the CalpontExecutionPlan from which the JobList is constructed + */ + EXPORT static SJLP makeJobList(execplan::CalpontExecutionPlan* cplan, ResourceManager* rm, + bool tryTuple = false, bool isExeMgr = false); -public: - /** @brief static JobList constructor method - * - * @param cplan the CalpontExecutionPlan from which the JobList is constructed - */ - EXPORT static SJLP makeJobList( - execplan::CalpontExecutionPlan* cplan, - ResourceManager* rm, - bool tryTuple = false, - bool isExeMgr = false); - -private: - + private: }; -} +} // namespace joblist #undef EXPORT -#endif //JOBLISTFACTORY_H - +#endif // JOBLISTFACTORY_H diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 6d5664c57..d4c6fb213 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /** -* $Id: joblisttypes.h 9251 2013-02-01 21:21:35Z dhall $ -*/ + * $Id: joblisttypes.h 9251 2013-02-01 21:21:35Z dhall $ + */ /** @file */ #ifndef JOBLISTTYPES_ @@ -30,15 +30,14 @@ namespace joblist { - const uint64_t BIGINTNULL = 0x8000000000000000ULL; const uint64_t BIGINTEMPTYROW = 0x8000000000000001ULL; const uint32_t INTNULL = 0x80000000; const uint32_t INTEMPTYROW = 0x80000001; const uint16_t SMALLINTNULL = 0x8000; const uint16_t SMALLINTEMPTYROW = 0x8001; -const uint8_t TINYINTNULL = 0x80; -const uint8_t TINYINTEMPTYROW = 0x81; +const uint8_t TINYINTNULL = 0x80; +const uint8_t TINYINTEMPTYROW = 0x81; const uint64_t UBIGINTNULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t UBIGINTEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; @@ -46,8 +45,8 @@ const uint32_t UINTNULL = 0xFFFFFFFE; const uint32_t UINTEMPTYROW = 0xFFFFFFFF; const uint16_t USMALLINTNULL = 0xFFFE; const uint16_t USMALLINTEMPTYROW = 0xFFFF; -const uint8_t UTINYINTNULL = 0xFE; -const uint8_t UTINYINTEMPTYROW = 0xFF; +const uint8_t UTINYINTNULL = 0xFE; +const uint8_t UTINYINTEMPTYROW = 0xFF; const uint32_t FLOATNULL = 0xFFAAAAAA; const uint32_t FLOATEMPTYROW = 0xFFAAAAAB; @@ -64,8 +63,8 @@ const uint64_t TIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint64_t TIMESTAMPNULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t TIMESTAMPEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; -const uint8_t CHAR1NULL = 0xFE; -const uint8_t CHAR1EMPTYROW = 0xFF; +const uint8_t CHAR1NULL = 0xFE; +const uint8_t CHAR1EMPTYROW = 0xFF; const uint16_t CHAR2NULL = 0xFEFF; const uint16_t CHAR2EMPTYROW = 0xFFFF; const uint32_t CHAR4NULL = 0xFEFFFFFF; @@ -73,12 +72,12 @@ const uint32_t CHAR4EMPTYROW = 0xFFFFFFFF; const uint64_t CHAR8NULL = 0xFEFFFFFFFFFFFFFFULL; const uint64_t CHAR8EMPTYROW = 0xFFFFFFFFFFFFFFFFULL; -const int8_t NULL_INT8 = TINYINTNULL; +const int8_t NULL_INT8 = TINYINTNULL; const int16_t NULL_INT16 = SMALLINTNULL; const int32_t NULL_INT32 = INTNULL; const int64_t NULL_INT64 = BIGINTNULL; -const uint8_t NULL_UINT8 = UTINYINTNULL; +const uint8_t NULL_UINT8 = UTINYINTNULL; const uint16_t NULL_UINT16 = USMALLINTNULL; const uint32_t NULL_UINT32 = UINTNULL; const uint64_t NULL_UINT64 = UBIGINTNULL; @@ -89,19 +88,10 @@ const std::string CPSTRNOTFOUND("_CpNoTf_"); /** @brief enum used to define the join type. */ typedef uint32_t JoinType; -const JoinType INIT = 0, - INNER = 0x1, - LARGEOUTER = 0x2, - SMALLOUTER = 0x4, - LEFTOUTER = 0x3, // deprecated, using impossible values for backward compat - RIGHTOUTER = 0x5, // deprecated.. - SEMI = 0x8, - ANTI = 0x10, - SCALAR = 0x20, - MATCHNULLS = 0x40, - WITHFCNEXP = 0x80, - CORRELATED = 0x100; -} +const JoinType INIT = 0, INNER = 0x1, LARGEOUTER = 0x2, SMALLOUTER = 0x4, + LEFTOUTER = 0x3, // deprecated, using impossible values for backward compat + RIGHTOUTER = 0x5, // deprecated.. + SEMI = 0x8, ANTI = 0x10, SCALAR = 0x20, MATCHNULLS = 0x40, WITHFCNEXP = 0x80, CORRELATED = 0x100; +} // namespace joblist #endif - diff --git a/dbcon/joblist/jobstep.cpp b/dbcon/joblist/jobstep.cpp index 3ececcae4..36d13f35e 100644 --- a/dbcon/joblist/jobstep.cpp +++ b/dbcon/joblist/jobstep.cpp @@ -45,169 +45,157 @@ using namespace querytele; namespace { - int toInt(const string& val) { - if (val.length() == 0) return -1; + if (val.length() == 0) + return -1; - return static_cast(config::Config::fromText(val)); -} + return static_cast(config::Config::fromText(val)); } +} // namespace namespace joblist { -boost::mutex JobStep::fLogMutex; //=PTHREAD_MUTEX_INITIALIZER; +boost::mutex JobStep::fLogMutex; //=PTHREAD_MUTEX_INITIALIZER; threadpool::ThreadPool JobStep::jobstepThreadPool(defaultJLThreadPoolSize, 0); ostream& operator<<(ostream& os, const JobStep* rhs) { - os << rhs->toString(); - return os; + os << rhs->toString(); + return os; } //------------------------------------------------------------------------------ // constructor //------------------------------------------------------------------------------ -JobStep::JobStep(const JobInfo& j) : - fSessionId(j.sessionId), - fTxnId(j.txnId), - fVerId(j.verId), - fStatementId(j.statementId), - fStepId(0), - fTupleId(-1), - fTraceFlags(0), - fCardinality(0), - fDelayedRunFlag(false), - fDelivery(false), - fOnClauseFilter(false), - fDie(false), - fWaitToRunStepCnt(0), - fPriority(1), - fErrorInfo(j.errorInfo), - fLogger(j.logger), - fLocalQuery(j.localQuery), - fQueryUuid(j.uuid), - fProgress(0), - fStartTime(-1), - fTimeZone(j.timeZone) +JobStep::JobStep(const JobInfo& j) + : fSessionId(j.sessionId) + , fTxnId(j.txnId) + , fVerId(j.verId) + , fStatementId(j.statementId) + , fStepId(0) + , fTupleId(-1) + , fTraceFlags(0) + , fCardinality(0) + , fDelayedRunFlag(false) + , fDelivery(false) + , fOnClauseFilter(false) + , fDie(false) + , fWaitToRunStepCnt(0) + , fPriority(1) + , fErrorInfo(j.errorInfo) + , fLogger(j.logger) + , fLocalQuery(j.localQuery) + , fQueryUuid(j.uuid) + , fProgress(0) + , fStartTime(-1) + , fTimeZone(j.timeZone) { - QueryTeleServerParms tsp; - string teleServerHost(Config::makeConfig()->getConfig("QueryTele", "Host")); + QueryTeleServerParms tsp; + string teleServerHost(Config::makeConfig()->getConfig("QueryTele", "Host")); - if (!teleServerHost.empty()) + if (!teleServerHost.empty()) + { + int teleServerPort = toInt(Config::makeConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) { - int teleServerPort = toInt(Config::makeConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - tsp.host = teleServerHost; - tsp.port = teleServerPort; - } + tsp.host = teleServerHost; + tsp.port = teleServerPort; } + } - fQtc.serverParms(tsp); - //fStepUuid = bu::random_generator()(); - fStepUuid = QueryTeleClient::genUUID(); + fQtc.serverParms(tsp); + // fStepUuid = bu::random_generator()(); + fStepUuid = QueryTeleClient::genUUID(); } //------------------------------------------------------------------------------ // Log a syslog msg for the start of this specified job step //------------------------------------------------------------------------------ -void JobStep::syslogStartStep ( - uint32_t subSystem, - const string& stepName) const +void JobStep::syslogStartStep(uint32_t subSystem, const string& stepName) const { - LoggingID logId ( subSystem, sessionId(), txnId() ); - MessageLog msgLog ( logId ); + LoggingID logId(subSystem, sessionId(), txnId()); + MessageLog msgLog(logId); - Message msgStartStep ( M0030 ); - Message::Args args; - args.add ( (uint64_t)statementId() ); // statement id for this job step - args.add ( (int)stepId() ); // step id for this job step - args.add ( stepName ); // step name for this job step - msgStartStep.format ( args ); - msgLog.logDebugMessage ( msgStartStep ); + Message msgStartStep(M0030); + Message::Args args; + args.add((uint64_t)statementId()); // statement id for this job step + args.add((int)stepId()); // step id for this job step + args.add(stepName); // step name for this job step + msgStartStep.format(args); + msgLog.logDebugMessage(msgStartStep); } //------------------------------------------------------------------------------ // Log a syslog message for the end of this specified job step //------------------------------------------------------------------------------ -void JobStep::syslogEndStep ( - uint32_t subSystem, - uint64_t blockedDLInput, - uint64_t blockedDLOutput, - uint64_t msgBytesInput, - uint64_t msgBytesOutput) const +void JobStep::syslogEndStep(uint32_t subSystem, uint64_t blockedDLInput, uint64_t blockedDLOutput, + uint64_t msgBytesInput, uint64_t msgBytesOutput) const { - LoggingID logId ( subSystem, sessionId(), txnId() ); - MessageLog msgLog ( logId ); + LoggingID logId(subSystem, sessionId(), txnId()); + MessageLog msgLog(logId); - Message msgEndStep ( M0031 ); - Message::Args args; - args.add ( (uint64_t)statementId() ); // statement id for this job step - args.add ( (int)stepId() ); // step id for this job step - args.add ( blockedDLInput ); // blocked datalist input (ex: fifo) - args.add ( blockedDLOutput ); // blocked datalist output(ex: fifo) - args.add ( msgBytesInput ); // incoming msg byte count - args.add ( msgBytesOutput ); // outgoing msg byte count - msgEndStep.format ( args ); - msgLog.logDebugMessage ( msgEndStep ); + Message msgEndStep(M0031); + Message::Args args; + args.add((uint64_t)statementId()); // statement id for this job step + args.add((int)stepId()); // step id for this job step + args.add(blockedDLInput); // blocked datalist input (ex: fifo) + args.add(blockedDLOutput); // blocked datalist output(ex: fifo) + args.add(msgBytesInput); // incoming msg byte count + args.add(msgBytesOutput); // outgoing msg byte count + msgEndStep.format(args); + msgLog.logDebugMessage(msgEndStep); } //------------------------------------------------------------------------------ // Log a syslog message for the physical vs cache block I/O counts //------------------------------------------------------------------------------ -void JobStep::syslogReadBlockCounts ( - uint32_t subSystem, - uint64_t physicalReadCount, - uint64_t cacheReadCount, - uint64_t casualPartBlocks) const +void JobStep::syslogReadBlockCounts(uint32_t subSystem, uint64_t physicalReadCount, uint64_t cacheReadCount, + uint64_t casualPartBlocks) const { - LoggingID logId ( subSystem, sessionId(), txnId() ); - MessageLog msgLog ( logId ); + LoggingID logId(subSystem, sessionId(), txnId()); + MessageLog msgLog(logId); - Message msgEndStep ( M0032 ); - Message::Args args; - args.add ( (uint64_t)statementId() ); // statement id for this job step - args.add ( (int)stepId() ); // step id for this job step - args.add ( (int)oid() ); // step id for this job step - args.add ( physicalReadCount ); // blocked datalist input (ex: fifo) - args.add ( cacheReadCount ); // blocked datalist output(ex: fifo) - args.add ( casualPartBlocks ); // casual partition block hits - msgEndStep.format ( args ); - msgLog.logDebugMessage ( msgEndStep ); + Message msgEndStep(M0032); + Message::Args args; + args.add((uint64_t)statementId()); // statement id for this job step + args.add((int)stepId()); // step id for this job step + args.add((int)oid()); // step id for this job step + args.add(physicalReadCount); // blocked datalist input (ex: fifo) + args.add(cacheReadCount); // blocked datalist output(ex: fifo) + args.add(casualPartBlocks); // casual partition block hits + msgEndStep.format(args); + msgLog.logDebugMessage(msgEndStep); } //------------------------------------------------------------------------------ // Log a syslog msg for the effective start/end times for this step // (lastWriteTime denotes when the EndOfInput marker was written out). //------------------------------------------------------------------------------ -void JobStep::syslogProcessingTimes ( - uint32_t subSystem, - const struct timeval& firstReadTime, - const struct timeval& lastReadTime, - const struct timeval& firstWriteTime, - const struct timeval& lastWriteTime) const +void JobStep::syslogProcessingTimes(uint32_t subSystem, const struct timeval& firstReadTime, + const struct timeval& lastReadTime, const struct timeval& firstWriteTime, + const struct timeval& lastWriteTime) const { - LoggingID logId ( subSystem, sessionId(), txnId() ); - MessageLog msgLog ( logId ); - Message msgStartStep ( M0046 ); - Message::Args args; + LoggingID logId(subSystem, sessionId(), txnId()); + MessageLog msgLog(logId); + Message msgStartStep(M0046); + Message::Args args; - args.add ( (uint64_t)statementId() ); // statement id for this job step - args.add ( (int)stepId() ); // step id for this job step - args.add ( JSTimeStamp::format(firstReadTime) ); // when first DL input element read - args.add ( JSTimeStamp::format(lastReadTime) ); // when last DL input element read - args.add ( JSTimeStamp::format(firstWriteTime));// when first DL output elem written - args.add ( JSTimeStamp::format(lastWriteTime)); // when EndOfInput written to DL out - msgStartStep.format ( args ); - msgLog.logDebugMessage ( msgStartStep ); + args.add((uint64_t)statementId()); // statement id for this job step + args.add((int)stepId()); // step id for this job step + args.add(JSTimeStamp::format(firstReadTime)); // when first DL input element read + args.add(JSTimeStamp::format(lastReadTime)); // when last DL input element read + args.add(JSTimeStamp::format(firstWriteTime)); // when first DL output elem written + args.add(JSTimeStamp::format(lastWriteTime)); // when EndOfInput written to DL out + msgStartStep.format(args); + msgLog.logDebugMessage(msgStartStep); } bool JobStep::traceOn() const { - return fTraceFlags & execplan::CalpontSelectExecutionPlan::TRACE_LOG; + return fTraceFlags & execplan::CalpontSelectExecutionPlan::TRACE_LOG; } ////////////////////////////////////////////////////////////////////// @@ -221,49 +209,39 @@ bool JobStep::traceOn() const // critErrorCode is this a crit IDBExcept or not // methodName method name to log ////////////////////////////////////////////////////////////////////// -void JobStep::handleException(std::exception_ptr e, - const int errorCode, - const unsigned infoErrorCode, - const std::string& methodName) +void JobStep::handleException(std::exception_ptr e, const int errorCode, const unsigned infoErrorCode, + const std::string& methodName) { - try - { - std::rethrow_exception(e); - } - catch (const IDBExcept& iex) - { - std::cerr << methodName << " caught a internal exception. " - << std::endl; + try + { + std::rethrow_exception(e); + } + catch (const IDBExcept& iex) + { + std::cerr << methodName << " caught a internal exception. " << std::endl; - catchHandler(methodName + " " + iex.what(), iex.errorCode(), - fErrorInfo, fSessionId, (iex.errorCode() == infoErrorCode - ? LOG_TYPE_INFO - : LOG_TYPE_CRITICAL)); - } - catch (boost::exception& e) - { - std::cerr << methodName << " caught a boost::exception. " - << std::endl; - catchHandler(methodName + " caught " + boost::diagnostic_information(e), - errorCode, fErrorInfo, fSessionId); - } - catch (const std::exception& ex) - { - std::cerr << methodName << " caught an exception. " << std::endl; - catchHandler(methodName + " caught " + ex.what(), errorCode, - fErrorInfo, fSessionId); - } - catch (...) - { - std::ostringstream oss; + catchHandler(methodName + " " + iex.what(), iex.errorCode(), fErrorInfo, fSessionId, + (iex.errorCode() == infoErrorCode ? LOG_TYPE_INFO : LOG_TYPE_CRITICAL)); + } + catch (boost::exception& e) + { + std::cerr << methodName << " caught a boost::exception. " << std::endl; + catchHandler(methodName + " caught " + boost::diagnostic_information(e), errorCode, fErrorInfo, + fSessionId); + } + catch (const std::exception& ex) + { + std::cerr << methodName << " caught an exception. " << std::endl; + catchHandler(methodName + " caught " + ex.what(), errorCode, fErrorInfo, fSessionId); + } + catch (...) + { + std::ostringstream oss; - std::cerr << methodName << " caught an unknown exception." - << std::endl; - catchHandler(methodName + " caught an unknown exception ", - errorCode, fErrorInfo, fSessionId); - } + std::cerr << methodName << " caught an unknown exception." << std::endl; + catchHandler(methodName + " caught an unknown exception ", errorCode, fErrorInfo, fSessionId); + } } -} //namespace joblist +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index 88dbeb94c..2051e2f45 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -17,7 +17,6 @@ // $Id: jobstep.h 9636 2013-06-20 14:23:36Z rdempsey $ - /** @file */ #ifndef JOBLIST_JOBSTEP_H_ @@ -48,506 +47,514 @@ #include "branchpred.h" #ifndef __GNUC__ -# ifndef __attribute__ -# define __attribute__(x) -# endif +#ifndef __attribute__ +#define __attribute__(x) +#endif #endif namespace joblist { - /** @brief class JobStepAssociation mediator class to connect/control JobSteps and DataLists * * Class JobStepAssociation connects/controls JobSteps and DalaLists */ class JobStepAssociation { -public: - JobStepAssociation() { } - virtual ~JobStepAssociation() {} + public: + JobStepAssociation() + { + } + virtual ~JobStepAssociation() + { + } - void inAdd(const AnyDataListSPtr& spdl) __attribute__((deprecated)) - { - fInDataList.push_back(spdl); - } - void outAdd(const AnyDataListSPtr& spdl) - { - fOutDataList.push_back(spdl); - } - void outAdd(const AnyDataListSPtr& spdl, size_t pos) - { - if (pos > fOutDataList.size()) throw std::logic_error("Insert position is beyond end."); + void inAdd(const AnyDataListSPtr& spdl) __attribute__((deprecated)) + { + fInDataList.push_back(spdl); + } + void outAdd(const AnyDataListSPtr& spdl) + { + fOutDataList.push_back(spdl); + } + void outAdd(const AnyDataListSPtr& spdl, size_t pos) + { + if (pos > fOutDataList.size()) + throw std::logic_error("Insert position is beyond end."); - fOutDataList.insert(fOutDataList.begin() + pos, spdl); - } - void outAdd(const DataListVec& spdlVec, size_t pos) - { - if (pos > fOutDataList.size()) throw std::logic_error("Insert position is beyond end."); + fOutDataList.insert(fOutDataList.begin() + pos, spdl); + } + void outAdd(const DataListVec& spdlVec, size_t pos) + { + if (pos > fOutDataList.size()) + throw std::logic_error("Insert position is beyond end."); - fOutDataList.insert(fOutDataList.begin() + pos, spdlVec.begin(), spdlVec.end()); - } - size_t inSize() const __attribute__((deprecated)) - { - return fInDataList.size(); - } - size_t outSize() const - { - return fOutDataList.size(); - } - const AnyDataListSPtr& inAt(size_t i) const __attribute__((deprecated)) - { - return fInDataList.at(i); - } - const AnyDataListSPtr& outAt(size_t i) const - { - return fOutDataList.at(i); - } - AnyDataListSPtr& outAt(size_t i) - { - return fOutDataList.at(i); - } + fOutDataList.insert(fOutDataList.begin() + pos, spdlVec.begin(), spdlVec.end()); + } + size_t inSize() const __attribute__((deprecated)) + { + return fInDataList.size(); + } + size_t outSize() const + { + return fOutDataList.size(); + } + const AnyDataListSPtr& inAt(size_t i) const __attribute__((deprecated)) + { + return fInDataList.at(i); + } + const AnyDataListSPtr& outAt(size_t i) const + { + return fOutDataList.at(i); + } + AnyDataListSPtr& outAt(size_t i) + { + return fOutDataList.at(i); + } -private: - DataListVec fInDataList; - DataListVec fOutDataList; + private: + DataListVec fInDataList; + DataListVec fOutDataList; }; - // forward reference struct JobInfo; - /** @brief class JobStep abstract class describing a query execution step * * Class JobStep is an abstract class that describes a query execution step */ class JobStep { -public: + public: + /** constructor + */ + JobStep() + { + } + JobStep(const JobInfo&); + /** destructor + */ + virtual ~JobStep() + { /*pthread_mutex_destroy(&mutex);*/ + } + /** @brief virtual void Run method + */ + virtual void run() = 0; + virtual void abort() + { + fDie = true; + } + /** @brief virtual void join method + */ + virtual void join() = 0; + /** @brief virtual string toString method + */ + virtual const std::string toString() const = 0; - /** constructor - */ - JobStep() { } - JobStep(const JobInfo&); - /** destructor - */ - virtual ~JobStep() { /*pthread_mutex_destroy(&mutex);*/ } - /** @brief virtual void Run method - */ - virtual void run() = 0; - virtual void abort() - { - fDie = true; - } - /** @brief virtual void join method - */ - virtual void join() = 0; - /** @brief virtual string toString method - */ - virtual const std::string toString() const = 0; + /** @brief virtual JobStepAssociation * inputAssociation method + */ + virtual void inputAssociation(const JobStepAssociation& inputAssociation) + { + fInputJobStepAssociation = inputAssociation; + } + virtual const JobStepAssociation& inputAssociation() const + { + return fInputJobStepAssociation; + } + /** @brief virtual JobStepAssociation * outputAssociation method + */ + virtual void outputAssociation(const JobStepAssociation& outputAssociation) + { + fOutputJobStepAssociation = outputAssociation; + } + virtual const JobStepAssociation& outputAssociation() const + { + return fOutputJobStepAssociation; + } - /** @brief virtual JobStepAssociation * inputAssociation method - */ - virtual void inputAssociation(const JobStepAssociation& inputAssociation) - { - fInputJobStepAssociation = inputAssociation; - } - virtual const JobStepAssociation& inputAssociation() const - { - return fInputJobStepAssociation; - } - /** @brief virtual JobStepAssociation * outputAssociation method - */ - virtual void outputAssociation(const JobStepAssociation& outputAssociation) - { - fOutputJobStepAssociation = outputAssociation; - } - virtual const JobStepAssociation& outputAssociation() const - { - return fOutputJobStepAssociation; - } + virtual void stepId(uint16_t stepId) + { + fStepId = stepId; + } + virtual uint16_t stepId() const + { + return fStepId; + } + virtual uint32_t sessionId() const + { + return fSessionId; + } + virtual uint32_t txnId() const + { + return fTxnId; + } + virtual uint32_t statementId() const + { + return fStatementId; + } + virtual void logger(const SPJL& logger) + { + fLogger = logger; + } - virtual void stepId(uint16_t stepId) - { - fStepId = stepId; - } - virtual uint16_t stepId() const - { - return fStepId; - } - virtual uint32_t sessionId() const - { - return fSessionId; - } - virtual uint32_t txnId() const - { - return fTxnId; - } - virtual uint32_t statementId() const - { - return fStatementId; - } - virtual void logger(const SPJL& logger) - { - fLogger = logger; - } + virtual bool isDictCol() const + { + return 0; + } + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return 0; + } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return 0; + } + // @bug 598 Added alias for self-join + virtual std::string alias() const + { + return fAlias; + } + virtual void alias(const std::string& alias) + { + fAlias = alias; + } + // @bug 3401 & 3402, view support + virtual std::string view() const + { + return fView; + } + virtual void view(const std::string& vw) + { + fView = vw; + } + // @bug 3438, stats with column name + virtual std::string name() const + { + return fName; + } + virtual void name(const std::string& nm) + { + fName = nm; + } + virtual std::string schema() const + { + return fSchema; + } + virtual void schema(const std::string& s) + { + fSchema = s; + } + // @bug 3398, add columns' unique tuple ID to job step + virtual uint64_t tupleId() const + { + return fTupleId; + } + virtual void tupleId(uint64_t id) + { + fTupleId = id; + } - virtual bool isDictCol() const - { - return 0; - } - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return 0; - } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return 0; - } - // @bug 598 Added alias for self-join - virtual std::string alias() const - { - return fAlias; - } - virtual void alias(const std::string& alias) - { - fAlias = alias; - } - // @bug 3401 & 3402, view support - virtual std::string view() const - { - return fView; - } - virtual void view(const std::string& vw) - { - fView = vw; - } - // @bug 3438, stats with column name - virtual std::string name() const - { - return fName; - } - virtual void name(const std::string& nm) - { - fName = nm; - } - virtual std::string schema() const - { - return fSchema; - } - virtual void schema(const std::string& s) - { - fSchema = s; - } - // @bug 3398, add columns' unique tuple ID to job step - virtual uint64_t tupleId() const - { - return fTupleId; - } - virtual void tupleId(uint64_t id) - { - fTupleId = id; - } + //...Final I/O blk count, msg rcv count, etc for this job step. These + //...methods do not use a mutex lock to acquire values, because it is + //...assumed they are called after all processing is complete. + virtual uint64_t phyIOCount() const + { + return 0; + } + virtual uint64_t cacheIOCount() const + { + return 0; + } + virtual uint64_t msgsRcvdCount() const + { + return 0; + } + virtual uint64_t msgBytesIn() const + { + return 0; + } + virtual uint64_t msgBytesOut() const + { + return 0; + } + virtual uint64_t blockTouched() const + { + return 0; + } + virtual uint64_t cardinality() const + { + return fCardinality; + } + virtual void cardinality(const uint64_t cardinality) + { + fCardinality = cardinality; + } - //...Final I/O blk count, msg rcv count, etc for this job step. These - //...methods do not use a mutex lock to acquire values, because it is - //...assumed they are called after all processing is complete. - virtual uint64_t phyIOCount ( ) const - { - return 0; - } - virtual uint64_t cacheIOCount ( ) const - { - return 0; - } - virtual uint64_t msgsRcvdCount ( ) const - { - return 0; - } - virtual uint64_t msgBytesIn ( ) const - { - return 0; - } - virtual uint64_t msgBytesOut ( ) const - { - return 0; - } - virtual uint64_t blockTouched ( ) const - { - return 0; - } - virtual uint64_t cardinality ( ) const - { - return fCardinality; - } - virtual void cardinality ( const uint64_t cardinality ) - { - fCardinality = cardinality; - } + // functions to delay/control jobstep execution; decWaitToRunStepCnt() per- + // forms atomic decrement op because it is accessed by multiple threads. + bool delayedRun() const + { + return fDelayedRunFlag; + } + int waitToRunStepCnt() + { + return fWaitToRunStepCnt; + } + void incWaitToRunStepCnt() + { + fDelayedRunFlag = true; + ++fWaitToRunStepCnt; + } + int decWaitToRunStepCnt() + { + return atomicops::atomicDec(&fWaitToRunStepCnt); + } + void resetDelayedRun() + { + fDelayedRunFlag = false; + fWaitToRunStepCnt = 0; + } - // functions to delay/control jobstep execution; decWaitToRunStepCnt() per- - // forms atomic decrement op because it is accessed by multiple threads. - bool delayedRun() const - { - return fDelayedRunFlag; - } - int waitToRunStepCnt() - { - return fWaitToRunStepCnt; - } - void incWaitToRunStepCnt() - { - fDelayedRunFlag = true; - ++fWaitToRunStepCnt; - } - int decWaitToRunStepCnt() - { - return atomicops::atomicDec(&fWaitToRunStepCnt); - } - void resetDelayedRun() - { - fDelayedRunFlag = false; - fWaitToRunStepCnt = 0; - } + void logEnd(const char* s) + { + fLogMutex.lock(); // pthread_mutex_lock(&mutex); + std::cout << s << std::endl; + fLogMutex.unlock(); // pthread_mutex_unlock(&mutex); + } + void syslogStartStep(uint32_t subSystem, const std::string& stepName) const; + void syslogEndStep(uint32_t subSystem, uint64_t blockedDLInput, uint64_t blockedDLOutput, + uint64_t msgBytesInput = 0, uint64_t msgBytesOutput = 0) const; + void syslogReadBlockCounts(uint32_t subSystem, uint64_t physicalReadCount, uint64_t cacheReadCount, + uint64_t casualPartBlocks) const; + void syslogProcessingTimes(uint32_t subSystem, const struct timeval& firstReadTime, + const struct timeval& lastReadTime, const struct timeval& firstWriteTime, + const struct timeval& lastWriteTime) const; + void setTrace(bool trace) __attribute__((deprecated)); + bool traceOn() const; + void setTraceFlags(uint32_t flags) + { + fTraceFlags = flags; + } + JSTimeStamp dlTimes; - void logEnd(const char* s) - { - fLogMutex.lock(); //pthread_mutex_lock(&mutex); - std::cout << s << std::endl; - fLogMutex.unlock(); //pthread_mutex_unlock(&mutex); - } - void syslogStartStep(uint32_t subSystem, - const std::string& stepName) const; - void syslogEndStep (uint32_t subSystem, - uint64_t blockedDLInput, - uint64_t blockedDLOutput, - uint64_t msgBytesInput = 0, - uint64_t msgBytesOutput = 0 ) const; - void syslogReadBlockCounts (uint32_t subSystem, - uint64_t physicalReadCount, - uint64_t cacheReadCount, - uint64_t casualPartBlocks ) const; - void syslogProcessingTimes (uint32_t subSystem, - const struct timeval& firstReadTime, - const struct timeval& lastReadTime, - const struct timeval& firstWriteTime, - const struct timeval& lastWriteTime) const; - void setTrace(bool trace) __attribute__((deprecated)); - bool traceOn() const; - void setTraceFlags(uint32_t flags) - { - fTraceFlags = flags; - } - JSTimeStamp dlTimes; + const std::string& extendedInfo() const + { + return fExtendedInfo; + } + const std::string& miniInfo() const + { + return fMiniInfo; + } - const std::string& extendedInfo() const - { - return fExtendedInfo; - } - const std::string& miniInfo() const - { - return fMiniInfo; - } + uint32_t priority() + { + return fPriority; + } + void priority(uint32_t p) + { + fPriority = p; + } - uint32_t priority() - { - return fPriority; - } - void priority(uint32_t p) - { - fPriority = p; - } + uint32_t status() const + { + return fErrorInfo->errCode; + } + void status(uint32_t s) + { + fErrorInfo->errCode = s; + } + std::string errorMessage() + { + return fErrorInfo->errMsg; + } + void errorMessage(const std::string& s) + { + fErrorInfo->errMsg = s; + } + const SErrorInfo& errorInfo() const + { + return fErrorInfo; + } + SErrorInfo& errorInfo() + { + return fErrorInfo; + } + void errorInfo(SErrorInfo& sp) + { + fErrorInfo = sp; + } - uint32_t status() const + bool cancelled() + { + return (fErrorInfo->errCode > 0 || fDie); + } + + virtual bool stringTableFriendly() + { + return false; + } + + bool delivery() const + { + return fDelivery; + } + void delivery(bool b) + { + fDelivery = b; + } + + const boost::uuids::uuid& queryUuid() const + { + return fQueryUuid; + } + + //@bug5887, distinguish on clause filter and where clause filter + bool onClauseFilter() const + { + return fOnClauseFilter; + } + void onClauseFilter(bool b) + { + fOnClauseFilter = b; + } + + void timeZone(const std::string& timezone) + { + fTimeZone = timezone; + } + const std::string timeZone() const + { + return fTimeZone; + } + + void handleException(std::exception_ptr e, const int errorCode, const unsigned infoErrorCode, + const std::string& methodName); + + static threadpool::ThreadPool jobstepThreadPool; + + protected: + //@bug6088, for telemetry posting + static const int64_t STEP_TELE_INTERVAL = 5000; // now, this is the browser refresh rate + void postStepStartTele(querytele::StepTeleStats& sts) + { + fStartTime = fLastStepTeleTime = sts.start_time = querytele::QueryTeleClient::timeNowms(); + fQtc.postStepTele(sts); + } + void postStepProgressTele(querytele::StepTeleStats& sts) + { + int64_t crntTime = querytele::QueryTeleClient::timeNowms(); + + if ((crntTime - fLastStepTeleTime) >= STEP_TELE_INTERVAL) { - return fErrorInfo->errCode; - } - void status(uint32_t s) - { - fErrorInfo->errCode = s; - } - std::string errorMessage() - { - return fErrorInfo->errMsg; - } - void errorMessage(const std::string& s) - { - fErrorInfo->errMsg = s; - } - const SErrorInfo& errorInfo() const - { - return fErrorInfo; - } - SErrorInfo& errorInfo() - { - return fErrorInfo; - } - void errorInfo(SErrorInfo& sp) - { - fErrorInfo = sp; + // interval between step telemetry + sts.start_time = fStartTime; + fQtc.postStepTele(sts); + fLastStepTeleTime = crntTime; } + } + void postStepSummaryTele(querytele::StepTeleStats& sts) + { + sts.start_time = fStartTime; + sts.end_time = fLastStepTeleTime = querytele::QueryTeleClient::timeNowms(); + fQtc.postStepTele(sts); + } - bool cancelled() - { - return (fErrorInfo->errCode > 0 || fDie); - } + JobStepAssociation fInputJobStepAssociation; + JobStepAssociation fOutputJobStepAssociation; - virtual bool stringTableFriendly() - { - return false; - } + uint32_t fSessionId; + uint32_t fTxnId; + BRM::QueryContext fVerId; + uint32_t fStatementId; - bool delivery() const - { - return fDelivery; - } - void delivery(bool b) - { - fDelivery = b; - } + uint32_t fStepId; + uint64_t fTupleId; - const boost::uuids::uuid& queryUuid() const - { - return fQueryUuid; - } + std::string fAlias; + std::string fView; + std::string fName; + std::string fSchema; + uint32_t fTraceFlags; + uint64_t fCardinality; + bool fDelayedRunFlag; + bool fDelivery; + bool fOnClauseFilter; + volatile bool fDie; + volatile uint32_t fWaitToRunStepCnt; + std::string fExtendedInfo; + std::string fMiniInfo; - //@bug5887, distinguish on clause filter and where clause filter - bool onClauseFilter() const - { - return fOnClauseFilter; - } - void onClauseFilter(bool b) - { - fOnClauseFilter = b; - } + uint32_t fPriority; - void timeZone(const std::string& timezone) - { - fTimeZone = timezone; - } - const std::string timeZone() const - { - return fTimeZone; - } + SErrorInfo fErrorInfo; + SPJL fLogger; - void handleException(std::exception_ptr e, - const int errorCode, - const unsigned infoErrorCode, - const std::string& methodName); + uint32_t fLocalQuery; - static threadpool::ThreadPool jobstepThreadPool; -protected: + boost::uuids::uuid fQueryUuid; + boost::uuids::uuid fStepUuid; + querytele::QueryTeleClient fQtc; + uint64_t fProgress; + int64_t fStartTime; + int64_t fLastStepTeleTime; + std::string fTimeZone; - //@bug6088, for telemetry posting - static const int64_t STEP_TELE_INTERVAL = 5000; // now, this is the browser refresh rate - void postStepStartTele(querytele::StepTeleStats& sts) - { - fStartTime = fLastStepTeleTime = sts.start_time = querytele::QueryTeleClient::timeNowms(); - fQtc.postStepTele(sts); - } - void postStepProgressTele(querytele::StepTeleStats& sts) - { - int64_t crntTime = querytele::QueryTeleClient::timeNowms(); + private: + static boost::mutex fLogMutex; - if ((crntTime - fLastStepTeleTime) >= STEP_TELE_INTERVAL) - { - // interval between step telemetry - sts.start_time = fStartTime; - fQtc.postStepTele(sts); - fLastStepTeleTime = crntTime; - } - } - void postStepSummaryTele(querytele::StepTeleStats& sts) - { - sts.start_time = fStartTime; - sts.end_time = fLastStepTeleTime = querytele::QueryTeleClient::timeNowms(); - fQtc.postStepTele(sts); - } - - JobStepAssociation fInputJobStepAssociation; - JobStepAssociation fOutputJobStepAssociation; - - uint32_t fSessionId; - uint32_t fTxnId; - BRM::QueryContext fVerId; - uint32_t fStatementId; - - uint32_t fStepId; - uint64_t fTupleId; - - std::string fAlias; - std::string fView; - std::string fName; - std::string fSchema; - uint32_t fTraceFlags; - uint64_t fCardinality; - bool fDelayedRunFlag; - bool fDelivery; - bool fOnClauseFilter; - volatile bool fDie; - volatile uint32_t fWaitToRunStepCnt; - std::string fExtendedInfo; - std::string fMiniInfo; - - uint32_t fPriority; - - SErrorInfo fErrorInfo; - SPJL fLogger; - - uint32_t fLocalQuery; - - boost::uuids::uuid fQueryUuid; - boost::uuids::uuid fStepUuid; - querytele::QueryTeleClient fQtc; - uint64_t fProgress; - int64_t fStartTime; - int64_t fLastStepTeleTime; - std::string fTimeZone; - -private: - static boost::mutex fLogMutex; - - friend class CommandJL; + friend class CommandJL; }; - class TupleJobStep { -public: - TupleJobStep() { } - virtual ~TupleJobStep() { } - virtual void setOutputRowGroup(const rowgroup::RowGroup&) = 0; - virtual void setFcnExpGroup3(const std::vector&) {} - virtual void setFE23Output(const rowgroup::RowGroup&) {} - virtual const rowgroup::RowGroup& getOutputRowGroup() const = 0; + public: + TupleJobStep() + { + } + virtual ~TupleJobStep() + { + } + virtual void setOutputRowGroup(const rowgroup::RowGroup&) = 0; + virtual void setFcnExpGroup3(const std::vector&) + { + } + virtual void setFE23Output(const rowgroup::RowGroup&) + { + } + virtual const rowgroup::RowGroup& getOutputRowGroup() const = 0; }; - class TupleDeliveryStep : public TupleJobStep { -public: - virtual ~TupleDeliveryStep() { } - virtual uint32_t nextBand(messageqcpp::ByteStream& bs) = 0; - virtual const rowgroup::RowGroup& getDeliveredRowGroup() const = 0; - virtual void deliverStringTableRowGroup(bool b) = 0; - virtual bool deliverStringTableRowGroup() const = 0; + public: + virtual ~TupleDeliveryStep() + { + } + virtual uint32_t nextBand(messageqcpp::ByteStream& bs) = 0; + virtual const rowgroup::RowGroup& getDeliveredRowGroup() const = 0; + virtual void deliverStringTableRowGroup(bool b) = 0; + virtual bool deliverStringTableRowGroup() const = 0; }; class NullStep : public JobStep { -public: - /** @brief virtual void Run method - */ - virtual void run() {} - /** @brief virtual void join method - */ - virtual void join() {} - /** @brief virtual string toString method - */ - virtual const std::string toString() const - { - return "NullStep"; - } + public: + /** @brief virtual void Run method + */ + virtual void run() + { + } + /** @brief virtual void join method + */ + virtual void join() + { + } + /** @brief virtual string toString method + */ + virtual const std::string toString() const + { + return "NullStep"; + } }; // calls rhs->toString() @@ -558,8 +565,7 @@ typedef boost::shared_ptr JobStepAssociationSPtr; typedef boost::shared_ptr SJSTEP; -} +} // namespace joblist #endif // JOBLIST_JOBSTEP_H_ // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/jobstepassociation.cpp b/dbcon/joblist/jobstepassociation.cpp index e00cddabd..953e457ff 100644 --- a/dbcon/joblist/jobstepassociation.cpp +++ b/dbcon/joblist/jobstepassociation.cpp @@ -16,16 +16,14 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: jobstepassociation.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: jobstepassociation.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include "jobstep.h" using namespace joblist; namespace joblist { - } - diff --git a/dbcon/joblist/largedatalist.h b/dbcon/joblist/largedatalist.h index 7aedead4b..0dd56e6ea 100644 --- a/dbcon/joblist/largedatalist.h +++ b/dbcon/joblist/largedatalist.h @@ -51,16 +51,16 @@ #include "exceptclasses.h" #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_DIRECT -# define O_DIRECT 0 +#define O_DIRECT 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif namespace joblist @@ -72,25 +72,27 @@ const uint32_t defaultMaxElements = 0x2000000; // 32M // saved to temporary disk. enum CompressionModeType { - COMPRESS_NO_COMPRESS = 1, // no compression of RID or data value - COMPRESS_TO_64_32 = 2, // compress to 64 bit RID, 32 bit value - COMPRESS_TO_32_64 = 3, // compress to 32 bit RID, 64 bit value - COMPRESS_TO_32_32 = 4, // compress to 32 bit RID, 32 bit value - COMPRESS_TO_32 = 5, // compress RID only to 32 bit RID only - COMPRESS_TO_32_STR = 6 // compress to 32 bit RID for a StringElementType + COMPRESS_NO_COMPRESS = 1, // no compression of RID or data value + COMPRESS_TO_64_32 = 2, // compress to 64 bit RID, 32 bit value + COMPRESS_TO_32_64 = 3, // compress to 32 bit RID, 64 bit value + COMPRESS_TO_32_32 = 4, // compress to 32 bit RID, 32 bit value + COMPRESS_TO_32 = 5, // compress RID only to 32 bit RID only + COMPRESS_TO_32_STR = 6 // compress to 32 bit RID for a StringElementType }; #ifndef _DISKIOINFO__ #define _DISKIOINFO__ struct DiskIoInfo { - boost::posix_time::ptime fStart; - boost::posix_time::ptime fEnd; - uint64_t fBytes; - bool fWrite; + boost::posix_time::ptime fStart; + boost::posix_time::ptime fEnd; + uint64_t fBytes; + bool fWrite; - // c: byte count; b: is write operation? - DiskIoInfo(bool b = false) : fBytes(0), fWrite(b) {} + // c: byte count; b: is write operation? + DiskIoInfo(bool b = false) : fBytes(0), fWrite(b) + { + } }; #endif //_DISKIOINFO__ @@ -100,208 +102,208 @@ struct DiskIoInfo /** @brief class LargeDataList * */ -template +template class LargeDataList : public DataListImpl { - typedef DataListImpl base; + typedef DataListImpl base; -public: - LargeDataList(uint32_t numConsumers, - uint32_t elementSaveSize1, - uint32_t elementSaveSize2, - const ResourceManager* rm); - virtual ~LargeDataList(); + public: + LargeDataList(uint32_t numConsumers, uint32_t elementSaveSize1, uint32_t elementSaveSize2, + const ResourceManager* rm); + virtual ~LargeDataList(); - virtual void endOfInput(); - virtual void insert(const element_t& e); - virtual uint64_t totalSize(); - virtual void setMultipleProducers(bool); - virtual uint64_t numberOfTempFiles() const; - virtual void setDiskElemSize(uint32_t size1st, uint32_t size2nd); - virtual void setReuseInfo(SetRestoreInfo* info, const std::string filename, bool readonly); - virtual void restoreSetForReuse(const struct SetRestoreInfo& info); - virtual void traceOn(bool b) - { - fTraceOn = b; - } - std::list& diskIoList() - { - return fDiskIoList; - } + virtual void endOfInput(); + virtual void insert(const element_t& e); + virtual uint64_t totalSize(); + virtual void setMultipleProducers(bool); + virtual uint64_t numberOfTempFiles() const; + virtual void setDiskElemSize(uint32_t size1st, uint32_t size2nd); + virtual void setReuseInfo(SetRestoreInfo* info, const std::string filename, bool readonly); + virtual void restoreSetForReuse(const struct SetRestoreInfo& info); + virtual void traceOn(bool b) + { + fTraceOn = b; + } + std::list& diskIoList() + { + return fDiskIoList; + } -protected: - // @bug 721. add append option - virtual void load(uint64_t setNumber, bool append = false); + protected: + // @bug 721. add append option + virtual void load(uint64_t setNumber, bool append = false); - /* Warning!! The container must have enough memory reserved, it must be - contiguous, and element_t must be inline data!! */ - // @bug 721. add append option - virtual void load_contiguous(uint64_t setNumber, bool append = false); + /* Warning!! The container must have enough memory reserved, it must be + contiguous, and element_t must be inline data!! */ + // @bug 721. add append option + virtual void load_contiguous(uint64_t setNumber, bool append = false); - // these save methods return the number of bytes written by the save - // operation - virtual uint64_t save(); - virtual uint64_t save(container_t* c); - virtual uint64_t save_contiguous(); - virtual uint64_t save_contiguous(container_t* c); + // these save methods return the number of bytes written by the save + // operation + virtual uint64_t save(); + virtual uint64_t save(container_t* c); + virtual uint64_t save_contiguous(); + virtual uint64_t save_contiguous(container_t* c); - virtual void registerNewSet(); - virtual std::string getFilename(); + virtual void registerNewSet(); + virtual std::string getFilename(); - virtual bool next(uint64_t it, element_t* e); - virtual bool next_nowait(uint64_t it, element_t* e); - virtual void waitForConsumePhase(); - virtual void resetIterators(); - virtual void resetIterators_nowait(); - void removeFile(); - bool saveForReuse() - { - return fSaveForReuse; - } + virtual bool next(uint64_t it, element_t* e); + virtual bool next_nowait(uint64_t it, element_t* e); + virtual void waitForConsumePhase(); + virtual void resetIterators(); + virtual void resetIterators_nowait(); + void removeFile(); + bool saveForReuse() + { + return fSaveForReuse; + } - std::string path; - uint64_t loadedSet, setCount; - int64_t phase; // 0 = produce phase, 1 = consume phase - uint64_t totSize; - bool multipleProducers; - bool fTraceOn; - std::list fDiskIoList; + std::string path; + uint64_t loadedSet, setCount; + int64_t phase; // 0 = produce phase, 1 = consume phase + uint64_t totSize; + bool multipleProducers; + bool fTraceOn; + std::list fDiskIoList; -private: - // Declare but don't define default and copy constuctor, and assignment - // operator to disable their use. - explicit LargeDataList(); - LargeDataList(const LargeDataList&); - LargeDataList& operator=(const LargeDataList&); + private: + // Declare but don't define default and copy constuctor, and assignment + // operator to disable their use. + explicit LargeDataList(); + LargeDataList(const LargeDataList&); + LargeDataList& operator=(const LargeDataList&); - void createTempFile(); - void save_contiguousCompressed(container_t* c); - void save_noncontiguousCompressed(container_t* c); - template - void writeContiguousCompressed(container_t* c); - void load_contiguousCompressed(bool append, uint64_t count); - template - void readContiguousCompressed( - uint64_t count, - element_t* pElementData); - void setCompressionMode(); // set current compression mode - void saveRestoreInfo(); + void createTempFile(); + void save_contiguousCompressed(container_t* c); + void save_noncontiguousCompressed(container_t* c); + template + void writeContiguousCompressed(container_t* c); + void load_contiguousCompressed(bool append, uint64_t count); + template + void readContiguousCompressed(uint64_t count, element_t* pElementData); + void setCompressionMode(); // set current compression mode + void saveRestoreInfo(); - boost::condition consumePhase; // consumers block here until endOfInput() - uint64_t filenameCounter; - std::string fFilename; // LDL file name - std::vector fSetStartPositions; //file offsets - std::fstream fFile; // stream used to store the LDL file - uint64_t fLoadedSetCount; // number of sets that've been loaded - CompressionModeType fCompressMode; // compression used when saved to disk + boost::condition consumePhase; // consumers block here until endOfInput() + uint64_t filenameCounter; + std::string fFilename; // LDL file name + std::vector fSetStartPositions; // file offsets + std::fstream fFile; // stream used to store the LDL file + uint64_t fLoadedSetCount; // number of sets that've been loaded + CompressionModeType fCompressMode; // compression used when saved to disk - bool fReUse; // flag for reuse - bool fSaveForReuse; // flag to save the restore infomation - SetRestoreInfo* fRestoreInfo; // point to the restore data in control + bool fReUse; // flag for reuse + bool fSaveForReuse; // flag to save the restore infomation + SetRestoreInfo* fRestoreInfo; // point to the restore data in control }; -template -LargeDataList::LargeDataList(uint32_t nc, uint32_t elementSaveSize1st, uint32_t elementSaveSize2nd, const ResourceManager* rm): - base(nc), path(rm.getScTempDiskPath()), fTraceOn(false), fReUse(false), fSaveForReuse(false), fRestoreInfo(NULL) +template +LargeDataList::LargeDataList(uint32_t nc, uint32_t elementSaveSize1st, + uint32_t elementSaveSize2nd, const ResourceManager* rm) + : base(nc) + , path(rm.getScTempDiskPath()) + , fTraceOn(false) + , fReUse(false) + , fSaveForReuse(false) + , fRestoreInfo(NULL) { - - loadedSet = 0; - setCount = 1; - filenameCounter = 1; - phase = 0; - totSize = 0; - multipleProducers = false; - fLoadedSetCount = 0; - setDiskElemSize ( elementSaveSize1st, elementSaveSize2nd ); + loadedSet = 0; + setCount = 1; + filenameCounter = 1; + phase = 0; + totSize = 0; + multipleProducers = false; + fLoadedSetCount = 0; + setDiskElemSize(elementSaveSize1st, elementSaveSize2nd); } -template +template LargeDataList::~LargeDataList() { - std::vector::iterator it; + std::vector::iterator it; - //pthread_cond_destroy(&consumePhase); - removeFile(); + // pthread_cond_destroy(&consumePhase); + removeFile(); } - -template +template inline void LargeDataList::removeFile() { - if (!fFilename.empty() && !fReUse) - { - unlink(fFilename.c_str()); - fFilename = ""; - } + if (!fFilename.empty() && !fReUse) + { + unlink(fFilename.c_str()); + fFilename = ""; + } } - -template +template void LargeDataList::setMultipleProducers(bool b) { - multipleProducers = b; + multipleProducers = b; } -template +template void LargeDataList::endOfInput() { - if (fSaveForReuse == true) - saveRestoreInfo(); + if (fSaveForReuse == true) + saveRestoreInfo(); - base::endOfInput(); - phase = 1; - consumePhase.notify_all(); //pthread_cond_broadcast(&consumePhase); + base::endOfInput(); + phase = 1; + consumePhase.notify_all(); // pthread_cond_broadcast(&consumePhase); } -template +template std::string LargeDataList::getFilename() { - std::stringstream o; + std::stringstream o; - o << path << "/LDL-0x" << std::hex << (ptrdiff_t)this << std::dec << "-" << filenameCounter++; - return o.str(); + o << path << "/LDL-0x" << std::hex << (ptrdiff_t)this << std::dec << "-" << filenameCounter++; + return o.str(); } -template -void LargeDataList::setReuseInfo( - SetRestoreInfo* info, const std::string filename, bool readonly) +template +void LargeDataList::setReuseInfo(SetRestoreInfo* info, const std::string filename, + bool readonly) { - fReUse = true; - fSaveForReuse = !readonly; - fRestoreInfo = info; - fFilename = filename; - std::ios_base::openmode mode = std::ios_base::in | std::ios_base::binary; + fReUse = true; + fSaveForReuse = !readonly; + fRestoreInfo = info; + fFilename = filename; + std::ios_base::openmode mode = std::ios_base::in | std::ios_base::binary; - // if need to create the file - if (fSaveForReuse) mode |= std::ios_base::out | std::ios_base::trunc; + // if need to create the file + if (fSaveForReuse) + mode |= std::ios_base::out | std::ios_base::trunc; - fFile.open(filename.c_str(), mode); + fFile.open(filename.c_str(), mode); - if (!(fFile.is_open())) - { - std::string errMsg("Error opening BucketReuse file "); - errMsg += filename; - perror(errMsg.c_str()); - throw logging::LargeDataListExcept(errMsg); - } + if (!(fFile.is_open())) + { + std::string errMsg("Error opening BucketReuse file "); + errMsg += filename; + perror(errMsg.c_str()); + throw logging::LargeDataListExcept(errMsg); + } } -template +template void LargeDataList::saveRestoreInfo() { - fRestoreInfo->fSetCount = setCount; - fRestoreInfo->fTotalSize = totSize; - fRestoreInfo->fSetStartPositions = fSetStartPositions; + fRestoreInfo->fSetCount = setCount; + fRestoreInfo->fTotalSize = totSize; + fRestoreInfo->fSetStartPositions = fSetStartPositions; } -template +template void LargeDataList::restoreSetForReuse(const struct SetRestoreInfo& info) { - setCount = info.fSetCount; - totSize = info.fTotalSize; - fSetStartPositions = info.fSetStartPositions; + setCount = info.fSetCount; + totSize = info.fTotalSize; + fSetStartPositions = info.fSetStartPositions; - load(0); + load(0); } //------------------------------------------------------------------------------ @@ -310,15 +312,15 @@ void LargeDataList::restoreSetForReuse(const struct SetR // will help protect the application code from knowing this, in case we // want to change the implementation to sometimes output more than 1 file. //------------------------------------------------------------------------------ -template +template uint64_t LargeDataList::numberOfTempFiles() const { -// uint64_t nFiles = 0; -// if ( !fFilename.empty() ) -// nFiles = 1; -// -// return nFiles; - return ((!fSetStartPositions.empty() && !fReUse) ? 1 : 0); + // uint64_t nFiles = 0; + // if ( !fFilename.empty() ) + // nFiles = 1; + // + // return nFiles; + return ((!fSetStartPositions.empty() && !fReUse) ? 1 : 0); } //------------------------------------------------------------------------------ @@ -327,272 +329,268 @@ uint64_t LargeDataList::numberOfTempFiles() const // sets for a LargeDataList. // exceptions: runtime_error thrown if file creation or file open fails //------------------------------------------------------------------------------ -template +template void LargeDataList::createTempFile() { - int64_t fd; + int64_t fd; - /* Is there a good way to do this through an fstream? */ - do - { - fFilename = this->getFilename(); - fd = open(fFilename.c_str(), O_CREAT | O_RDWR | O_EXCL | O_BINARY, 0666); - } - while (fd < 0 && errno == EEXIST); + /* Is there a good way to do this through an fstream? */ + do + { + fFilename = this->getFilename(); + fd = open(fFilename.c_str(), O_CREAT | O_RDWR | O_EXCL | O_BINARY, 0666); + } while (fd < 0 && errno == EEXIST); - if (fd < 0) - { - std::ostringstream errmsg; - errmsg << "LargeDataList::createTempFile(): could not save to disk (" - << errno << ") - " << std::strerror(errno); - std::cerr << errmsg.str() << std::endl; - throw std::runtime_error(errmsg.str()); - } + if (fd < 0) + { + std::ostringstream errmsg; + errmsg << "LargeDataList::createTempFile(): could not save to disk (" << errno << ") - " + << std::strerror(errno); + std::cerr << errmsg.str() << std::endl; + throw std::runtime_error(errmsg.str()); + } - close(fd); + close(fd); - //std::cout << "Creating/opening file: " << fFilename << std::endl; - fFile.open(fFilename.c_str(), - std::ios_base::in | std::ios_base::out | std::ios_base::binary ); + // std::cout << "Creating/opening file: " << fFilename << std::endl; + fFile.open(fFilename.c_str(), std::ios_base::in | std::ios_base::out | std::ios_base::binary); - if (!(fFile.is_open())) - { - std::string errMsg("Error opening temp file "); - errMsg += fFilename; - perror(errMsg.c_str()); - throw logging::LargeDataListExcept(errMsg); - } + if (!(fFile.is_open())) + { + std::string errMsg("Error opening temp file "); + errMsg += fFilename; + perror(errMsg.c_str()); + throw logging::LargeDataListExcept(errMsg); + } } // Need to grab the mutex at a higher level /* - File format: - int: # of elements - element_t[# of elements] stored according to ostream& element_t::operator<< + File format: + int: # of elements + element_t[# of elements] stored according to ostream& element_t::operator<< */ -template +template uint64_t LargeDataList::save() { - container_t* c = (base::c); - return save(c); + container_t* c = (base::c); + return save(c); } // Need to grab the mutex at a higher level /* - File format: - int: # of elements - element_t[# of elements] stored according to ostream& element_t::operator<< + File format: + int: # of elements + element_t[# of elements] stored according to ostream& element_t::operator<< */ -template -uint64_t LargeDataList::save(container_t* c ) +template +uint64_t LargeDataList::save(container_t* c) { - uint64_t count, nBytesWritten = 0; - typename container_t::iterator it; + uint64_t count, nBytesWritten = 0; + typename container_t::iterator it; #ifdef PROFILE - struct timespec ts1, ts2; - clock_gettime(CLOCK_REALTIME, &ts1); + struct timespec ts1, ts2; + clock_gettime(CLOCK_REALTIME, &ts1); #endif - /* XXXPAT: catch exceptions in save/load or at the higher level? */ - try + /* XXXPAT: catch exceptions in save/load or at the higher level? */ + try + { + //...Create the temporary LDL file (if necessary) + if (fFilename.empty()) + createTempFile(); + + // Save our file offset for this set, to be used when we load the set. + std::fstream::pos_type firstByte = fFile.tellp(); + fSetStartPositions.push_back(firstByte); + // std::cout << "Saving " << fFilename << "; count-" << c->size() << + // "; tellp-" << fFile.tellp() << std::endl; + + DiskIoInfo info(true); + + if (fTraceOn) + info.fStart = boost::posix_time::microsec_clock::local_time(); + + count = c->size(); + fFile.write((char*)&count, sizeof(count)); + + if (fCompressMode == COMPRESS_NO_COMPRESS) { - //...Create the temporary LDL file (if necessary) - if ( fFilename.empty() ) - createTempFile(); - - // Save our file offset for this set, to be used when we load the set. - std::fstream::pos_type firstByte = fFile.tellp(); - fSetStartPositions.push_back( firstByte ); - //std::cout << "Saving " << fFilename << "; count-" << c->size() << - // "; tellp-" << fFile.tellp() << std::endl; - - DiskIoInfo info(true); - - if (fTraceOn) info.fStart = boost::posix_time::microsec_clock::local_time(); - - count = c->size(); - fFile.write((char*) &count, sizeof(count)); - - if (fCompressMode == COMPRESS_NO_COMPRESS) - { - std::copy(c->begin(), - c->end(), - std::ostream_iterator(fFile)); - } - else - { - save_noncontiguousCompressed ( c ); - } - - nBytesWritten = fFile.tellp() - firstByte; - - if (fTraceOn) - { - info.fEnd = boost::posix_time::microsec_clock::local_time(); - info.fBytes = nBytesWritten; - fDiskIoList.push_back(info); - } + std::copy(c->begin(), c->end(), std::ostream_iterator(fFile)); } - catch (const std::runtime_error& e) + else { - if (fFile.is_open()) - fFile.close(); - - std::string msg("Error occurred saving non-contiguous container into file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg + e.what()); + save_noncontiguousCompressed(c); } - catch (...) + + nBytesWritten = fFile.tellp() - firstByte; + + if (fTraceOn) { - if (fFile.is_open()) - fFile.close(); - - std::string msg("Error occurred saving non-contiguous container into file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg); + info.fEnd = boost::posix_time::microsec_clock::local_time(); + info.fBytes = nBytesWritten; + fDiskIoList.push_back(info); } + } + catch (const std::runtime_error& e) + { + if (fFile.is_open()) + fFile.close(); + + std::string msg("Error occurred saving non-contiguous container into file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg + e.what()); + } + catch (...) + { + if (fFile.is_open()) + fFile.close(); + + std::string msg("Error occurred saving non-contiguous container into file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg); + } #ifdef PROFILE - clock_gettime(CLOCK_REALTIME, &ts2); - /* What should we do with this profile info? */ + clock_gettime(CLOCK_REALTIME, &ts2); + /* What should we do with this profile info? */ #endif - return nBytesWritten; + return nBytesWritten; } -template +template uint64_t LargeDataList::save_contiguous() { - std::vector* c = reinterpret_cast *>(base::c); - return save_contiguous(c); + std::vector* c = reinterpret_cast*>(base::c); + return save_contiguous(c); } -template +template uint64_t LargeDataList::save_contiguous(container_t* c) { - uint64_t count, nBytesWritten = 0; - typename container_t::iterator it; + uint64_t count, nBytesWritten = 0; + typename container_t::iterator it; #ifdef PROFILE - struct timespec ts1, ts2; - clock_gettime(CLOCK_REALTIME, &ts1); + struct timespec ts1, ts2; + clock_gettime(CLOCK_REALTIME, &ts1); #endif - /* XXXPAT: catch exceptions in save/load or at the higher level? */ - try + /* XXXPAT: catch exceptions in save/load or at the higher level? */ + try + { + //...Create the temporary LDL file (if necessary) + if (fFilename.empty()) + createTempFile(); + + // Save our file offset for this set, to be used when we load the set. + std::fstream::pos_type firstByte = fFile.tellp(); + fSetStartPositions.push_back(firstByte); + // std::cout << "SavingC " << fFilename << "; count-" << c->size() << + // "; tellp-" << fFile.tellp() << std::endl; + + DiskIoInfo info(true); + + if (fTraceOn) + info.fStart = boost::posix_time::microsec_clock::local_time(); + + count = c->size(); + fFile.write((char*)&count, sizeof(count)); + + // Perform compression of data, as it is saved, "if" applicable + if (fCompressMode == COMPRESS_NO_COMPRESS) { - //...Create the temporary LDL file (if necessary) - if ( fFilename.empty() ) - createTempFile(); - - // Save our file offset for this set, to be used when we load the set. - std::fstream::pos_type firstByte = fFile.tellp(); - fSetStartPositions.push_back( firstByte ); - //std::cout << "SavingC " << fFilename << "; count-" << c->size() << - // "; tellp-" << fFile.tellp() << std::endl; - - DiskIoInfo info(true); - - if (fTraceOn) info.fStart = boost::posix_time::microsec_clock::local_time(); - - count = c->size(); - fFile.write((char*) &count, sizeof(count)); - - // Perform compression of data, as it is saved, "if" applicable - if (fCompressMode == COMPRESS_NO_COMPRESS) - { - fFile.write((char*) (c->begin().operator->()), - sizeof(element_t) * count); - } - else - { - save_contiguousCompressed ( c ); - } - - nBytesWritten = fFile.tellp() - firstByte; - - if (fTraceOn) - { - info.fEnd = boost::posix_time::microsec_clock::local_time(); - info.fBytes = nBytesWritten; - fDiskIoList.push_back(info); - } + fFile.write((char*)(c->begin().operator->()), sizeof(element_t) * count); } - catch (const std::runtime_error& e) + else { - if (fFile.is_open()) - fFile.close(); - - std::string msg("Error occurred saving contiguous container into file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg + e.what()); + save_contiguousCompressed(c); } - catch (...) + + nBytesWritten = fFile.tellp() - firstByte; + + if (fTraceOn) { - if (fFile.is_open()) - fFile.close(); - - std::string msg("Error occurred saving contiguous container into file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg); + info.fEnd = boost::posix_time::microsec_clock::local_time(); + info.fBytes = nBytesWritten; + fDiskIoList.push_back(info); } + } + catch (const std::runtime_error& e) + { + if (fFile.is_open()) + fFile.close(); + + std::string msg("Error occurred saving contiguous container into file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg + e.what()); + } + catch (...) + { + if (fFile.is_open()) + fFile.close(); + + std::string msg("Error occurred saving contiguous container into file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg); + } #ifdef PROFILE - clock_gettime(CLOCK_REALTIME, &ts2); - /* What should we do with this profile info? */ + clock_gettime(CLOCK_REALTIME, &ts2); + /* What should we do with this profile info? */ #endif - return nBytesWritten; + return nBytesWritten; } //------------------------------------------------------------------------------ // Compress and save contiguous data to temp file. // c - container to be saved //------------------------------------------------------------------------------ -template -void LargeDataList::save_contiguousCompressed( - container_t* c) +template +void LargeDataList::save_contiguousCompressed(container_t* c) { - //...Compress and write out the elements based on the compression mode - switch (fCompressMode) + //...Compress and write out the elements based on the compression mode + switch (fCompressMode) + { + case COMPRESS_TO_64_32: { - case COMPRESS_TO_64_32: - { - writeContiguousCompressed(c); - break; - } - - case COMPRESS_TO_32_64: - { - writeContiguousCompressed(c); - break; - } - - case COMPRESS_TO_32_32: - { - writeContiguousCompressed(c); - break; - } - - case COMPRESS_TO_32: - { - writeContiguousCompressed(c); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "save_contiguousCompressed() called " - " without compression mode being set"; - std::cerr << errmsg << std::endl; - throw std::logic_error( errmsg.str() ); - break; - } + writeContiguousCompressed(c); + break; } + + case COMPRESS_TO_32_64: + { + writeContiguousCompressed(c); + break; + } + + case COMPRESS_TO_32_32: + { + writeContiguousCompressed(c); + break; + } + + case COMPRESS_TO_32: + { + writeContiguousCompressed(c); + break; + } + + default: + { + std::ostringstream errmsg; + errmsg << "save_contiguousCompressed() called " + " without compression mode being set"; + std::cerr << errmsg << std::endl; + throw std::logic_error(errmsg.str()); + break; + } + } } //------------------------------------------------------------------------------ @@ -600,396 +598,393 @@ void LargeDataList::save_contiguousCompressed( // to a vector of saveElement_t; and then saves the data to temp file. // c - container to be written to temporary disk file. //------------------------------------------------------------------------------ -template -template -void LargeDataList::writeContiguousCompressed( - container_t* c) +template +template +void LargeDataList::writeContiguousCompressed(container_t* c) { - uint64_t count = c->size(); - std::vector* v = reinterpret_cast *>(c); + uint64_t count = c->size(); + std::vector* v = reinterpret_cast*>(c); - //...copy/compress data into vector of saveElement_t - std::vector cSave; - cSave.resize( count ); - ElementCompression::compress ( *v, cSave ); + //...copy/compress data into vector of saveElement_t + std::vector cSave; + cSave.resize(count); + ElementCompression::compress(*v, cSave); - //...write saveElement_t vector to temp file - fFile.write((char*) (cSave.begin().operator->()), - (sizeof(saveElement_t)*count)); + //...write saveElement_t vector to temp file + fFile.write((char*)(cSave.begin().operator->()), (sizeof(saveElement_t) * count)); } //------------------------------------------------------------------------------ // Compress and save noncontiguous data to temp file. // c - container to be saved //------------------------------------------------------------------------------ -template -void LargeDataList::save_noncontiguousCompressed( - container_t* c) +template +void LargeDataList::save_noncontiguousCompressed(container_t* c) { - //...Compress and write out the elements based on the compression mode. - //...The only compression currently supported here is compression of the - //...RID from 64 to 32 bits. - switch (fCompressMode) + //...Compress and write out the elements based on the compression mode. + //...The only compression currently supported here is compression of the + //...RID from 64 to 32 bits. + switch (fCompressMode) + { + case COMPRESS_TO_32_64: + case COMPRESS_TO_32: + case COMPRESS_TO_32_STR: { + typename container_t::const_iterator iter = c->begin(); + + while (iter != c->end()) + { + ElementCompression::writeWith32Rid(*iter, fFile); + ++iter; + } + + break; + } + + default: + { + std::ostringstream errmsg; + errmsg << "save_noncontigousCompressed incorrectly called for " + "compression mode " + << fCompressMode; + std::cerr << errmsg.str() << std::endl; + throw std::logic_error(errmsg.str()); + break; + } + } +} + +template +void LargeDataList::load(uint64_t setNumber, bool append) +{ + uint64_t i, count; + std::vector* v; + std::set* s; + +#ifdef PROFILE + struct timespec ts1, ts2; + clock_gettime(CLOCK_REALTIME, &ts1); +#endif + + if (loadedSet == setNumber && phase != 0) + { + resetIterators(); + return; + } + + /* XXXPAT: How to handle errors here? Specifically, unless the entire load + is successful, things will be left in a relatively undefined state. Do we + have to care about things like that here? Initial guess: no. */ + try + { + DiskIoInfo info; + + if (fTraceOn) + info.fStart = boost::posix_time::microsec_clock::local_time(); + + // Position the file to the correct file offset for this set. + fFile.seekg(fSetStartPositions.at(setNumber)); + // std::cout << "Loading filename-" << fFilename << + // "; set-" << setNumber << "; fPos-" << + // fSetStartPositions.at(setNumber) << std::endl; + + std::streampos startPos = fFile.tellg(); + fFile.read((char*)&count, sizeof(count)); + // std::cout << "really slow load, count=" << count << std::endl; + + // Specific logic to handle saving of a std::vector + if (typeid(*base::c) == typeid(std::vector)) + { + v = reinterpret_cast*>(base::c); + + // @bug 721. merge all saving sets to current loaded set 0 and sort + if (!append) + v->resize(0); + + if (count > v->size()) + v->reserve(count); + + switch (fCompressMode) + { case COMPRESS_TO_32_64: case COMPRESS_TO_32: case COMPRESS_TO_32_STR: { - typename container_t::const_iterator iter = c->begin(); + element_t e; - while ( iter != c->end() ) - { - ElementCompression::writeWith32Rid ( *iter, fFile ); - ++iter; - } + for (i = 0; i < count; ++i) + { + ElementCompression::readWith32Rid(e, fFile); + v->push_back(e); + } - break; + break; } default: { - std::ostringstream errmsg; - errmsg << "save_noncontigousCompressed incorrectly called for " - "compression mode " << fCompressMode; - std::cerr << errmsg.str() << std::endl; - throw std::logic_error( errmsg.str() ); - break; - } - } -} + std::istream_iterator it(fFile); -template -void LargeDataList::load(uint64_t setNumber, bool append) -{ - uint64_t i, count; - std::vector* v; - std::set* s; + for (i = 0; i < count; ++i) + { + v->push_back(*it); + + // Increment stream iterator except for last element. + // We don't want to go past the end of this set. + if ((i + 1) < count) + ++it; // advance to next element in file + } + + break; + } + } + } + // Specific logic to handle saving of a std::set + else if (typeid(*base::c) == typeid(std::set)) + { + s = reinterpret_cast*>(base::c); + + if (!append) + s->clear(); + + switch (fCompressMode) + { + case COMPRESS_TO_32_64: + case COMPRESS_TO_32: + case COMPRESS_TO_32_STR: + { + element_t e; + + for (i = 0; i < count; ++i) + { + ElementCompression::readWith32Rid(e, fFile); + s->insert(e); + } + + break; + } + + default: + { + std::istream_iterator it(fFile); + + for (i = 0; i < count; ++i) + { + s->insert(*it); + + // Increment stream iterator except for last element. + // We don't want to go past the end of this set. + if ((i + 1) < count) + ++it; // advance to next element in file + } + + break; + } + } + } + else + { + /* this is a slow fallback. If we need it, we should write + a specialization for whatever container c is */ + if (!append) + { + delete base::c; + base::c = new container_t(); + } + + switch (fCompressMode) + { + case COMPRESS_TO_32_64: + case COMPRESS_TO_32: + case COMPRESS_TO_32_STR: + { + element_t e; + + for (i = 0; i < count; ++i) + { + ElementCompression::readWith32Rid(e, fFile); + base::insert(e); + } + + break; + } + + default: + { + std::istream_iterator it(fFile); + + for (i = 0; i < count; ++i) + { + // std::cout << "inserting " << loaded << std::endl; + // we might want to use the derived class for inserting + // instead but we will want to add to the base class + // interface to support reentrancy. Possibly an + // "insert_nolock()" would be sufficient. + base::insert(*it); + + // Increment stream iterator except for last element. + // We don't want to go past the end of this set. + if ((i + 1) < count) + ++it; // advance to next element in file + } + + break; + } + } + } + + // std::cout << "... done" << std::endl; + + if (fTraceOn) + { + info.fEnd = boost::posix_time::microsec_clock::local_time(); + info.fBytes = fFile.tellg() - startPos; + fDiskIoList.push_back(info); + } + } + catch (const std::runtime_error& e) + { + if (fFile.is_open()) + fFile.close(); + + std::string msg("Error occurred loading non-contiguous container from file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg + e.what()); + } + catch (...) + { + if (fFile.is_open()) + fFile.close(); + + std::string msg("Error occurred loading non-contiguous container from file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg); + } + + resetIterators_nowait(); + loadedSet = setNumber; + fLoadedSetCount++; + + //...Close the file once all the sets have been loaded. We could compare + //...setNumber to see if it is the last set, but we make no assumptions + //...about the order in which the sets are loaded; so we instead track + //...the number of sets loaded, and use that to know when we are done. + if (fLoadedSetCount == fSetStartPositions.size()) + { + fFile.close(); + } #ifdef PROFILE - struct timespec ts1, ts2; - clock_gettime(CLOCK_REALTIME, &ts1); -#endif - - if (loadedSet == setNumber && phase != 0) - { - resetIterators(); - return; - } - - /* XXXPAT: How to handle errors here? Specifically, unless the entire load - is successful, things will be left in a relatively undefined state. Do we - have to care about things like that here? Initial guess: no. */ - try - { - DiskIoInfo info; - - if (fTraceOn) info.fStart = boost::posix_time::microsec_clock::local_time(); - - // Position the file to the correct file offset for this set. - fFile.seekg( fSetStartPositions.at(setNumber) ); - //std::cout << "Loading filename-" << fFilename << - // "; set-" << setNumber << "; fPos-" << - // fSetStartPositions.at(setNumber) << std::endl; - - std::streampos startPos = fFile.tellg(); - fFile.read((char*) &count, sizeof(count)); - //std::cout << "really slow load, count=" << count << std::endl; - - // Specific logic to handle saving of a std::vector - if (typeid(*base::c) == typeid(std::vector)) - { - v = reinterpret_cast *>(base::c); - - // @bug 721. merge all saving sets to current loaded set 0 and sort - if (!append) - v->resize(0); - - if (count > v->size()) - v->reserve( count ); - - switch (fCompressMode) - { - case COMPRESS_TO_32_64: - case COMPRESS_TO_32: - case COMPRESS_TO_32_STR: - { - element_t e; - - for (i = 0; i < count; ++i) - { - ElementCompression::readWith32Rid ( e, fFile ); - v->push_back(e); - } - - break; - } - - default: - { - std::istream_iterator it(fFile); - - for (i = 0; i < count; ++i) - { - v->push_back(*it); - - // Increment stream iterator except for last element. - // We don't want to go past the end of this set. - if ((i + 1) < count) - ++it; // advance to next element in file - } - - break; - } - } - } - // Specific logic to handle saving of a std::set - else if (typeid(*base::c) == typeid(std::set)) - { - s = reinterpret_cast *>(base::c); - - if ( !append) - s->clear(); - - switch (fCompressMode) - { - case COMPRESS_TO_32_64: - case COMPRESS_TO_32: - case COMPRESS_TO_32_STR: - { - element_t e; - - for (i = 0; i < count; ++i) - { - ElementCompression::readWith32Rid ( e, fFile ); - s->insert(e); - } - - break; - } - - default: - { - std::istream_iterator it(fFile); - - for (i = 0; i < count; ++i) - { - s->insert(*it); - - // Increment stream iterator except for last element. - // We don't want to go past the end of this set. - if ((i + 1) < count) - ++it; // advance to next element in file - } - - break; - } - } - } - else - { - /* this is a slow fallback. If we need it, we should write - a specialization for whatever container c is */ - if (!append) - { - delete base::c; - base::c = new container_t(); - } - - switch (fCompressMode) - { - case COMPRESS_TO_32_64: - case COMPRESS_TO_32: - case COMPRESS_TO_32_STR: - { - element_t e; - - for (i = 0; i < count; ++i) - { - ElementCompression::readWith32Rid ( e, fFile ); - base::insert(e); - } - - break; - } - - default: - { - std::istream_iterator it(fFile); - - for (i = 0; i < count; ++i) - { - //std::cout << "inserting " << loaded << std::endl; - // we might want to use the derived class for inserting - // instead but we will want to add to the base class - // interface to support reentrancy. Possibly an - // "insert_nolock()" would be sufficient. - base::insert(*it); - - // Increment stream iterator except for last element. - // We don't want to go past the end of this set. - if ((i + 1) < count) - ++it; // advance to next element in file - } - - break; - } - } - } - -// std::cout << "... done" << std::endl; - - if (fTraceOn) - { - info.fEnd = boost::posix_time::microsec_clock::local_time(); - info.fBytes = fFile.tellg() - startPos; - fDiskIoList.push_back(info); - } - } - catch (const std::runtime_error& e) - { - if (fFile.is_open()) - fFile.close(); - - std::string msg("Error occurred loading non-contiguous container from file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg + e.what()); - } - catch (...) - { - if (fFile.is_open()) - fFile.close(); - - std::string msg("Error occurred loading non-contiguous container from file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg); - } - - resetIterators_nowait(); - loadedSet = setNumber; - fLoadedSetCount++; - - //...Close the file once all the sets have been loaded. We could compare - //...setNumber to see if it is the last set, but we make no assumptions - //...about the order in which the sets are loaded; so we instead track - //...the number of sets loaded, and use that to know when we are done. - if ( fLoadedSetCount == fSetStartPositions.size() ) - { - fFile.close(); - } - -#ifdef PROFILE - clock_gettime(CLOCK_REALTIME, &ts2); - /* What should we do with this profile info? */ + clock_gettime(CLOCK_REALTIME, &ts2); + /* What should we do with this profile info? */ #endif } -template +template void LargeDataList::load_contiguous(uint64_t setNumber, bool append) { - uint64_t count; - // char *buf = NULL; - std::vector* v; + uint64_t count; + // char *buf = NULL; + std::vector* v; #ifdef PROFILE - struct timespec ts1, ts2; - clock_gettime(CLOCK_REALTIME, &ts1); + struct timespec ts1, ts2; + clock_gettime(CLOCK_REALTIME, &ts1); #endif - if (loadedSet == setNumber && phase != 0) + if (loadedSet == setNumber && phase != 0) + { + resetIterators(); + return; + } + + /* XXXPAT: How to handle errors here? Specifically, unless the entire load + is successful, things will be left in a relatively undefined state. Do we + have to care about things like that here? Initial guess: no. */ + + try + { + DiskIoInfo info; + + if (fTraceOn) + info.fStart = boost::posix_time::microsec_clock::local_time(); + + v = reinterpret_cast*>(base::c); + // Position the file to the correct file offset for this set. + fFile.seekg(fSetStartPositions.at(setNumber)); + // std::cout << "LoadingC filename-" << fFilename << + // "; set-" << setNumber << "; fPos-" << + // fSetStartPositions.at(setNumber) << std::endl; + + std::streampos startPos = fFile.tellg(); + fFile.read((char*)&count, sizeof(count)); + + // Perform expansion of data, as it is loaded, "if" applicable + if (fCompressMode == COMPRESS_NO_COMPRESS) { - resetIterators(); - return; + if (append) + { + // @bug 721. append to current set for sorting purpose + uint64_t ctn = base::c->size(); + base::c->resize(ctn + count); + fFile.read((char*)((v->begin() + ctn).operator->()), count * sizeof(element_t)); + } + else + { + if (count != base::c->size()) + base::c->resize(count); + + fFile.read((char*)(v->begin().operator->()), count * sizeof(element_t)); + } + } + else + { + load_contiguousCompressed(append, count); } - /* XXXPAT: How to handle errors here? Specifically, unless the entire load - is successful, things will be left in a relatively undefined state. Do we - have to care about things like that here? Initial guess: no. */ - - try + if (fTraceOn) { - DiskIoInfo info; - - if (fTraceOn) info.fStart = boost::posix_time::microsec_clock::local_time(); - - v = reinterpret_cast *>(base::c); - // Position the file to the correct file offset for this set. - fFile.seekg( fSetStartPositions.at(setNumber) ); - //std::cout << "LoadingC filename-" << fFilename << - // "; set-" << setNumber << "; fPos-" << - // fSetStartPositions.at(setNumber) << std::endl; - - std::streampos startPos = fFile.tellg(); - fFile.read((char*) &count, sizeof(count)); - - // Perform expansion of data, as it is loaded, "if" applicable - if (fCompressMode == COMPRESS_NO_COMPRESS) - { - if (append) - { - // @bug 721. append to current set for sorting purpose - uint64_t ctn = base::c->size(); - base::c->resize(ctn + count); - fFile.read((char*) ((v->begin() + ctn).operator->()), - count * sizeof(element_t)); - } - else - { - if (count != base::c->size()) - base::c->resize( count ); - - fFile.read((char*) (v->begin().operator->()), - count * sizeof(element_t)); - } - } - else - { - load_contiguousCompressed( append, count ); - } - - if (fTraceOn) - { - info.fEnd = boost::posix_time::microsec_clock::local_time(); - info.fBytes = fFile.tellg() - startPos; - fDiskIoList.push_back(info); - } + info.fEnd = boost::posix_time::microsec_clock::local_time(); + info.fBytes = fFile.tellg() - startPos; + fDiskIoList.push_back(info); } - catch (const std::runtime_error& e) - { - if (fFile.is_open()) - fFile.close(); + } + catch (const std::runtime_error& e) + { + if (fFile.is_open()) + fFile.close(); - std::string msg("Error occurred loading contiguous container from file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg + e.what()); - } - catch (...) - { - if (fFile.is_open()) - fFile.close(); + std::string msg("Error occurred loading contiguous container from file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg + e.what()); + } + catch (...) + { + if (fFile.is_open()) + fFile.close(); - std::string msg("Error occurred loading contiguous container from file " + fFilename + " "); - std::cerr << msg << std::endl; - throw logging::LargeDataListExcept(msg); - } + std::string msg("Error occurred loading contiguous container from file " + fFilename + " "); + std::cerr << msg << std::endl; + throw logging::LargeDataListExcept(msg); + } - resetIterators_nowait(); - loadedSet = setNumber; - fLoadedSetCount++; + resetIterators_nowait(); + loadedSet = setNumber; + fLoadedSetCount++; - //...Close the file once all the sets have been loaded. We could compare - //...setNumber to see if it is the last set, but we make no assumptions - //...about the order in which the sets are loaded; so we instead track - //...the number of sets loaded, and use that to know when we are done. - if ( fLoadedSetCount == fSetStartPositions.size() ) - { - fFile.close(); - } + //...Close the file once all the sets have been loaded. We could compare + //...setNumber to see if it is the last set, but we make no assumptions + //...about the order in which the sets are loaded; so we instead track + //...the number of sets loaded, and use that to know when we are done. + if (fLoadedSetCount == fSetStartPositions.size()) + { + fFile.close(); + } #ifdef PROFILE - clock_gettime(CLOCK_REALTIME, &ts2); - /* What should we do with this profile info? */ + clock_gettime(CLOCK_REALTIME, &ts2); + /* What should we do with this profile info? */ #endif - } //------------------------------------------------------------------------------ @@ -997,68 +992,63 @@ void LargeDataList::load_contiguous(uint64_t setNumber, // append - flag that indicates whether data is to be appended to container // count - the number of elements to be read and expanded //------------------------------------------------------------------------------ -template +template void LargeDataList::load_contiguousCompressed(bool append, uint64_t count) { - std::vector* v = - reinterpret_cast *>(base::c); - element_t* pElementData = 0; + std::vector* v = reinterpret_cast*>(base::c); + element_t* pElementData = 0; - if (append) - { - uint64_t currentCount = base::c->size(); - base::c->resize(currentCount + count); - pElementData = ((v->begin() + currentCount).operator->()); - } - else - { - if (count != base::c->size()) - base::c->resize( count ); + if (append) + { + uint64_t currentCount = base::c->size(); + base::c->resize(currentCount + count); + pElementData = ((v->begin() + currentCount).operator->()); + } + else + { + if (count != base::c->size()) + base::c->resize(count); - pElementData = (v->begin().operator->()); + pElementData = (v->begin().operator->()); + } + + //...Read in and expand the elements based on the compression mode + switch (fCompressMode) + { + case COMPRESS_TO_64_32: + { + readContiguousCompressed(count, pElementData); + break; } - //...Read in and expand the elements based on the compression mode - switch (fCompressMode) + case COMPRESS_TO_32_64: { - case COMPRESS_TO_64_32: - { - readContiguousCompressed - ( count, pElementData); - break; - } - - case COMPRESS_TO_32_64: - { - readContiguousCompressed - ( count, pElementData); - break; - } - - case COMPRESS_TO_32_32: - { - readContiguousCompressed - ( count, pElementData); - break; - } - - case COMPRESS_TO_32: - { - readContiguousCompressed - ( count, pElementData); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "load_contiguousCompressed() called " - " without compression mode being set"; - std::cerr << errmsg.str() << std::endl; - throw std::logic_error( errmsg.str() ); - break; - } + readContiguousCompressed(count, pElementData); + break; } + + case COMPRESS_TO_32_32: + { + readContiguousCompressed(count, pElementData); + break; + } + + case COMPRESS_TO_32: + { + readContiguousCompressed(count, pElementData); + break; + } + + default: + { + std::ostringstream errmsg; + errmsg << "load_contiguousCompressed() called " + " without compression mode being set"; + std::cerr << errmsg.str() << std::endl; + throw std::logic_error(errmsg.str()); + break; + } + } } //------------------------------------------------------------------------------ @@ -1067,150 +1057,144 @@ void LargeDataList::load_contiguousCompressed(bool appen // count - (input) number of elements to be read from temp file // pElementData - (output) element_t data that has been read and expanded //------------------------------------------------------------------------------ -template -template -void LargeDataList::readContiguousCompressed( - uint64_t count, - element_t* pElementData) +template +template +void LargeDataList::readContiguousCompressed(uint64_t count, element_t* pElementData) { - //...read data from temp file into saveElement_t vector - std::vector cLoad; - cLoad.resize( count ); - fFile.read((char*) (cLoad.begin().operator->()), - (sizeof(saveElement_t)*count)); + //...read data from temp file into saveElement_t vector + std::vector cLoad; + cLoad.resize(count); + fFile.read((char*)(cLoad.begin().operator->()), (sizeof(saveElement_t) * count)); - //...copy/expand data into element_t container - ElementCompression::expand ( cLoad, pElementData ); + //...copy/expand data into element_t container + ElementCompression::expand(cLoad, pElementData); } -template +template inline bool LargeDataList::next(uint64_t it, element_t* e) { - bool ret; + bool ret; - waitForConsumePhase(); - ret = base::next(it, e); - return ret; + waitForConsumePhase(); + ret = base::next(it, e); + return ret; } -template +template inline bool LargeDataList::next_nowait(uint64_t it, element_t* e) { - bool ret; - ret = base::next(it, e); - return ret; + bool ret; + ret = base::next(it, e); + return ret; } -template +template inline void LargeDataList::waitForConsumePhase() { - while (phase == 0) - consumePhase.wait(this->mutex); //pthread_cond_wait(&consumePhase, &(this->mutex)); + while (phase == 0) + consumePhase.wait(this->mutex); // pthread_cond_wait(&consumePhase, &(this->mutex)); } -template +template void LargeDataList::registerNewSet() { - delete base::c; - base::c = new container_t(); - loadedSet++; - setCount++; + delete base::c; + base::c = new container_t(); + loadedSet++; + setCount++; } -template +template uint64_t LargeDataList::totalSize() { - waitForConsumePhase(); - return totSize; + waitForConsumePhase(); + return totSize; } -template +template inline void LargeDataList::insert(const element_t& e) { - totSize++; - base::insert(e); + totSize++; + base::insert(e); } -template +template void LargeDataList::resetIterators() { - waitForConsumePhase(); + waitForConsumePhase(); - for (int i = 0; i < (int) base::numConsumers; ++i) - base::cIterators[i] = base::c->begin(); + for (int i = 0; i < (int)base::numConsumers; ++i) + base::cIterators[i] = base::c->begin(); } -template +template void LargeDataList::resetIterators_nowait() { - for (uint64_t i = 0; i < base::numConsumers; ++i) - { - base::cIterators[i] = base::c->begin(); - } + for (uint64_t i = 0; i < base::numConsumers; ++i) + { + base::cIterators[i] = base::c->begin(); + } } //------------------------------------------------------------------------------ // Set save element size values stored in our base class, and update the // compression mode enumeration. //------------------------------------------------------------------------------ -template -void LargeDataList::setDiskElemSize( - uint32_t elementSaveSize1st, uint32_t elementSaveSize2nd ) +template +void LargeDataList::setDiskElemSize(uint32_t elementSaveSize1st, + uint32_t elementSaveSize2nd) { - base::setDiskElemSize ( elementSaveSize1st, elementSaveSize2nd ); + base::setDiskElemSize(elementSaveSize1st, elementSaveSize2nd); - // update our compression mode enumeration to reflect the save element size - setCompressionMode(); + // update our compression mode enumeration to reflect the save element size + setCompressionMode(); } //------------------------------------------------------------------------------ // Sets compression mode enumeration based on the current sizes for // save element size1st (RID) and save element size2nd (value). //------------------------------------------------------------------------------ -template +template void LargeDataList::setCompressionMode() { - const uint32_t COMPRESS_4BYTE_LENGTH = 4; - fCompressMode = COMPRESS_NO_COMPRESS; + const uint32_t COMPRESS_4BYTE_LENGTH = 4; + fCompressMode = COMPRESS_NO_COMPRESS; - uint32_t size1st = base::getDiskElemSize1st(); + uint32_t size1st = base::getDiskElemSize1st(); - if ( typeid(element_t) == typeid(RIDElementType) ) + if (typeid(element_t) == typeid(RIDElementType)) + { + if (size1st == COMPRESS_4BYTE_LENGTH) { - if ( size1st == COMPRESS_4BYTE_LENGTH ) - { - fCompressMode = COMPRESS_TO_32; - } + fCompressMode = COMPRESS_TO_32; } - else if ( typeid(element_t) == typeid(StringElementType) ) + } + else if (typeid(element_t) == typeid(StringElementType)) + { + if (size1st == COMPRESS_4BYTE_LENGTH) { - if ( size1st == COMPRESS_4BYTE_LENGTH ) - { - fCompressMode = COMPRESS_TO_32_STR; - } + fCompressMode = COMPRESS_TO_32_STR; } - else - { - uint32_t size2nd = base::getDiskElemSize2nd(); + } + else + { + uint32_t size2nd = base::getDiskElemSize2nd(); - if (( size1st == COMPRESS_4BYTE_LENGTH) && - ( size2nd != COMPRESS_4BYTE_LENGTH)) - { - fCompressMode = COMPRESS_TO_32_64; - } - else if (( size1st != COMPRESS_4BYTE_LENGTH) && - ( size2nd == COMPRESS_4BYTE_LENGTH)) - { - fCompressMode = COMPRESS_TO_64_32; - } - else if (( size1st == COMPRESS_4BYTE_LENGTH) && - ( size2nd == COMPRESS_4BYTE_LENGTH)) - { - fCompressMode = COMPRESS_TO_32_32; - } + if ((size1st == COMPRESS_4BYTE_LENGTH) && (size2nd != COMPRESS_4BYTE_LENGTH)) + { + fCompressMode = COMPRESS_TO_32_64; } + else if ((size1st != COMPRESS_4BYTE_LENGTH) && (size2nd == COMPRESS_4BYTE_LENGTH)) + { + fCompressMode = COMPRESS_TO_64_32; + } + else if ((size1st == COMPRESS_4BYTE_LENGTH) && (size2nd == COMPRESS_4BYTE_LENGTH)) + { + fCompressMode = COMPRESS_TO_32_32; + } + } } -} // namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/largehashjoin.cpp b/dbcon/joblist/largehashjoin.cpp index 66182aaa3..cad81a30f 100644 --- a/dbcon/joblist/largehashjoin.cpp +++ b/dbcon/joblist/largehashjoin.cpp @@ -44,81 +44,82 @@ namespace { void logDiskIoInfo(uint64_t stepId, const AnyDataListSPtr& spdl) { - boost::mutex::scoped_lock lk(fileLock_g); - string umDiskioLog = string(MCSLOGDIR) + "/trace/umdiskio.log"; - string umDiskioBak = string(MCSLOGDIR) + "/trace/umdiskio.bak"; + boost::mutex::scoped_lock lk(fileLock_g); + string umDiskioLog = string(MCSLOGDIR) + "/trace/umdiskio.log"; + string umDiskioBak = string(MCSLOGDIR) + "/trace/umdiskio.bak"; - ofstream umDiskIoFile(umDiskioLog.c_str(), ios_base::app); + ofstream umDiskIoFile(umDiskioLog.c_str(), ios_base::app); - CalpontSystemCatalog::OID oid; - uint64_t maxBuckets = 0; - list* infoList = NULL; - string bkt("bkt"); - BucketDL* bdl = spdl->bucketDL(); - BucketDL* sbdl = spdl->stringBucketDL(); - ZDL* zdl = spdl->zonedDL(); - ZDL* szdl = spdl->stringZonedDL(); + CalpontSystemCatalog::OID oid; + uint64_t maxBuckets = 0; + list* infoList = NULL; + string bkt("bkt"); + BucketDL* bdl = spdl->bucketDL(); + BucketDL* sbdl = spdl->stringBucketDL(); + ZDL* zdl = spdl->zonedDL(); + ZDL* szdl = spdl->stringZonedDL(); - if (bdl != NULL) - { - maxBuckets = bdl->bucketCount(); - oid = bdl->OID(); - } - else if (zdl != NULL) - { - maxBuckets = zdl->bucketCount(); - oid = zdl->OID(); - bkt = "zdl"; - } - else if (sbdl != NULL) - { - maxBuckets = sbdl->bucketCount(); - oid = sbdl->OID(); - } - else if (szdl != NULL) - { - maxBuckets = szdl->bucketCount(); - oid = szdl->OID(); - bkt = "zdl"; - } - else - { - // not logged for now. - return; - } + if (bdl != NULL) + { + maxBuckets = bdl->bucketCount(); + oid = bdl->OID(); + } + else if (zdl != NULL) + { + maxBuckets = zdl->bucketCount(); + oid = zdl->OID(); + bkt = "zdl"; + } + else if (sbdl != NULL) + { + maxBuckets = sbdl->bucketCount(); + oid = sbdl->OID(); + } + else if (szdl != NULL) + { + maxBuckets = szdl->bucketCount(); + oid = szdl->OID(); + bkt = "zdl"; + } + else + { + // not logged for now. + return; + } - for (uint64_t i = 0; i < maxBuckets; i++) + for (uint64_t i = 0; i < maxBuckets; i++) + { + if (bdl) + infoList = &(bdl->diskIoInfoList(i)); + else if (zdl) + infoList = &(zdl->diskIoInfoList(i)); + else if (sbdl) + infoList = &(sbdl->diskIoInfoList(i)); + else if (szdl) + infoList = &(szdl->diskIoInfoList(i)); + + for (list::iterator j = infoList->begin(); j != infoList->end(); j++) { - if (bdl) infoList = &(bdl->diskIoInfoList(i)); - else if (zdl) infoList = &(zdl->diskIoInfoList(i)); - else if (sbdl) infoList = &(sbdl->diskIoInfoList(i)); - else if (szdl) infoList = &(szdl->diskIoInfoList(i)); - - for (list::iterator j = infoList->begin(); j != infoList->end(); j++) - { - boost::posix_time::time_duration td = j->fEnd - j->fStart; - umDiskIoFile << setfill('0') - << "st" << setw(2) << stepId << "oid" << oid << bkt << setw(3) << i - << (j->fWrite ? " writes " : " reads ") << setw(7) << setfill(' ') - << j->fBytes << " bytes, at " << j->fStart << " duration " - << td.total_microseconds() << " mcs @ " - << (j->fBytes / td.total_microseconds()) << "MB/s" << endl; - } + boost::posix_time::time_duration td = j->fEnd - j->fStart; + umDiskIoFile << setfill('0') << "st" << setw(2) << stepId << "oid" << oid << bkt << setw(3) << i + << (j->fWrite ? " writes " : " reads ") << setw(7) << setfill(' ') << j->fBytes + << " bytes, at " << j->fStart << " duration " << td.total_microseconds() << " mcs @ " + << (j->fBytes / td.total_microseconds()) << "MB/s" << endl; } + } - streampos curPos = umDiskIoFile.tellp( ); - umDiskIoFile.close(); - - // move the current file to bak when size above .5 G, so total log is 1 G - if (curPos > 0x20000000) - { - string cmd = "/bin/mv " + umDiskioLog + " " + umDiskioBak; - (void)system(cmd.c_str()); - } -} + streampos curPos = umDiskIoFile.tellp(); + umDiskIoFile.close(); + // move the current file to bak when size above .5 G, so total log is 1 G + if (curPos > 0x20000000) + { + string cmd = "/bin/mv " + umDiskioLog + " " + umDiskioBak; + (void)system(cmd.c_str()); + } } +} // namespace namespace joblist { @@ -127,656 +128,633 @@ const uint64_t ZDL_VEC_SIZE = 4096; // So the other job steps can start. struct HJRunner { - HJRunner(LargeHashJoin* p) : joiner(p) - {} - LargeHashJoin* joiner; - void operator()() + HJRunner(LargeHashJoin* p) : joiner(p) + { + } + LargeHashJoin* joiner; + void operator()() + { + try { - try - { - joiner->doHashJoin(); - } - catch (std::exception& e) - { - ostringstream errMsg; - errMsg << "HJRunner caught: " << e.what(); - joiner->errorLogging(errMsg.str()); - joiner->unblockDatalists(logging::largeHashJoinErr); - } - catch (...) - { - string msg("HJRunner caught something not an exception!"); - joiner->errorLogging(msg); - joiner->unblockDatalists(logging::largeHashJoinErr); - } + joiner->doHashJoin(); } + catch (std::exception& e) + { + ostringstream errMsg; + errMsg << "HJRunner caught: " << e.what(); + joiner->errorLogging(errMsg.str()); + joiner->unblockDatalists(logging::largeHashJoinErr); + } + catch (...) + { + string msg("HJRunner caught something not an exception!"); + joiner->errorLogging(msg); + joiner->unblockDatalists(logging::largeHashJoinErr); + } + } }; struct StringHJRunner { - StringHJRunner(StringHashJoinStep* p) : joiner(p) - {} - StringHashJoinStep* joiner; - void operator()() + StringHJRunner(StringHashJoinStep* p) : joiner(p) + { + } + StringHashJoinStep* joiner; + void operator()() + { + try { - try - { - joiner->doStringHashJoin(); - } - catch (std::exception& e) - { - ostringstream errMsg; - errMsg << "StringHJRunner caught: " << e.what(); - joiner->errorLogging(errMsg.str()); - joiner->unblockDatalists(logging::stringHashJoinStepErr); - } - catch (...) - { - string msg("StringHJRunner caught something not an exception!"); - joiner->errorLogging(msg); - joiner->unblockDatalists(logging::stringHashJoinStepErr); - } + joiner->doStringHashJoin(); } + catch (std::exception& e) + { + ostringstream errMsg; + errMsg << "StringHJRunner caught: " << e.what(); + joiner->errorLogging(errMsg.str()); + joiner->unblockDatalists(logging::stringHashJoinStepErr); + } + catch (...) + { + string msg("StringHJRunner caught something not an exception!"); + joiner->errorLogging(msg); + joiner->unblockDatalists(logging::stringHashJoinStepErr); + } + } }; // Thread function used by HashJoin // -template +template void* HashJoinByBucket_thr(void* arg) { - typename HashJoin::thrParams_t* params = (typename HashJoin::thrParams_t*)arg; - HashJoin* hjPtr = params->hjptr; - const uint32_t thrIdx = params->thrIdx; - long set1Size = 0; - long set2Size = 0; - bool sendAllHashSet = false; - bool sendAllSearchSet = false; + typename HashJoin::thrParams_t* params = (typename HashJoin::thrParams_t*)arg; + HashJoin* hjPtr = params->hjptr; + const uint32_t thrIdx = params->thrIdx; + long set1Size = 0; + long set2Size = 0; + bool sendAllHashSet = false; + bool sendAllSearchSet = false; - try + try + { + for (uint32_t idx = 0, bucketIdx = params->startBucket; idx < params->numBuckets; idx++, bucketIdx++) { - for (uint32_t idx = 0, bucketIdx = params->startBucket; - idx < params->numBuckets; - idx++, bucketIdx++) +#ifdef DEBUG + cout << "\tJoinByBucket() thr " << dec << thrIdx << " bkt " << bucketIdx << "/" + << hjPtr->Set1()->bucketCount() << "/" << params->numBuckets << endl; +#endif + + JoinType joinType = hjPtr->getJoinType(); + + set1Size = hjPtr->Set1()->size(bucketIdx); + set2Size = hjPtr->Set2()->size(bucketIdx); + + if (set1Size <= 0 && set2Size <= 0) + { + continue; + } + else + { + if (set1Size > set2Size) { + hjPtr->setSearchSet(hjPtr->Set1()->getBDL(), thrIdx); + hjPtr->setHashSet(hjPtr->Set2()->getBDL(), thrIdx); + hjPtr->setSearchResult(hjPtr->Result1(), thrIdx); + hjPtr->setHashResult(hjPtr->Result2(), thrIdx); + sendAllHashSet = (joinType == RIGHTOUTER); + sendAllSearchSet = (joinType == LEFTOUTER); + } + else + { + hjPtr->setHashSet(hjPtr->Set1()->getBDL(), thrIdx); + hjPtr->setSearchSet(hjPtr->Set2()->getBDL(), thrIdx); + hjPtr->setHashResult(hjPtr->Result1(), thrIdx); + hjPtr->setSearchResult(hjPtr->Result2(), thrIdx); + sendAllHashSet = (joinType == LEFTOUTER); + sendAllSearchSet = (joinType == RIGHTOUTER); + } // if set1Size > set2Size ... + + } // if set1Size <=0 . . . + + params->timeset.setTimer(createHashStr); + hjPtr->createHash(hjPtr->HashSet(thrIdx), hjPtr->HashTable(thrIdx), bucketIdx, sendAllHashSet, + hjPtr->HashResult(thrIdx), params->dlTimes, params->die); + params->timeset.holdTimer(createHashStr); #ifdef DEBUG - cout << "\tJoinByBucket() thr " << dec << thrIdx - << " bkt " << bucketIdx - << "/" << hjPtr->Set1()->bucketCount() - << "/" << params->numBuckets << endl; + long hashSetTotal = 0; + long searchSetTotal = 0; + + for (uint32_t j = 0; j < hjPtr->HashSet(thrIdx)->bucketCount(); j++) + hashSetTotal += hjPtr->HashSet(thrIdx)->bucketCount(); // are bucketDL + + for (uint32_t j = 0; j < hjPtr->HashSet(thrIdx)->bucketCount(); j++) + searchSetTotal += hjPtr->SearchSet(thrIdx)->size(j); // can be any datalist + + cout << "\t\tJoinByBucket() thr " << dec << thrIdx << " bkt " << bucketIdx << " hashSize " + << hashSetTotal << " searchSize " << searchSetTotal << endl; #endif - JoinType joinType = hjPtr->getJoinType(); + bool more; + e_t e; + e_t e2; + const uint64_t InvalidRID = static_cast(-1); + int iter = hjPtr->SearchSet(thrIdx)->getIterator(bucketIdx); - set1Size = hjPtr->Set1()->size(bucketIdx); - set2Size = hjPtr->Set2()->size(bucketIdx); + ZonedDL* zdl1 = dynamic_cast(hjPtr->SearchResult(thrIdx)); + ZonedDL* zdl2 = dynamic_cast(hjPtr->HashResult(thrIdx)); + vector vec1; + vector vec2; - if ( set1Size <= 0 && set2Size <= 0 ) + std::pair::hashIter_t, typename HashJoin::hashIter_t> hashItPair; + typename HashJoin::hashIter_t hashIt; + typename HashJoin::hash_t* ht = hjPtr->HashTable(thrIdx); + params->timeset.setTimer(hashJoinStr); + + for (more = hjPtr->SearchSet(thrIdx)->next(bucketIdx, iter, &e); more && !(*params->die); + more = hjPtr->SearchSet(thrIdx)->next(bucketIdx, iter, &e)) + { + // If sendAllSearchSet=true, keep all of the search set. If this is + // a right outer, we are dealing with a join such as + // col1 = col2 (+) + // where col1 is the SearchSet and col2 is the HashSet. We want to include + // all of col1 in this case regardless of whether there is a matching col2. + if (sendAllSearchSet) + { + if (zdl1) + { + vec1.push_back(e); + + if (vec1.size() >= ZDL_VEC_SIZE) { - continue; + params->timeset.setTimer(insertResultsStr); + hjPtr->SearchResult(thrIdx)->insert(vec1); + vec1.clear(); + params->timeset.holdTimer(insertResultsStr); } - else + } + else + hjPtr->SearchResult(thrIdx)->insert(e); + } + + hashIt = ht->find(e.second); + + if (hashIt != ht->end()) + { +#ifdef DEBUG + + if (hjPtr->SearchResult(thrIdx)->OID() >= 3000) + cout << "JoinByBucket() SearchResult add " << bucketIdx << " [" << e.first << "][" << e.second + << "]" << endl; + + uint32_t a = 0; + e_t b = e_t(); +#endif + + // If sendAllSearchSet=false, we already added the search result + // before the if condition above. + if (!sendAllSearchSet) + { + if (zdl1) { - if (set1Size > set2Size) - { - hjPtr->setSearchSet(hjPtr->Set1()->getBDL(), thrIdx); - hjPtr->setHashSet(hjPtr->Set2()->getBDL(), thrIdx); - hjPtr->setSearchResult(hjPtr->Result1(), thrIdx); - hjPtr->setHashResult(hjPtr->Result2(), thrIdx); - sendAllHashSet = (joinType == RIGHTOUTER); - sendAllSearchSet = (joinType == LEFTOUTER); - } - else - { - hjPtr->setHashSet(hjPtr->Set1()->getBDL(), thrIdx); - hjPtr->setSearchSet(hjPtr->Set2()->getBDL(), thrIdx); - hjPtr->setHashResult(hjPtr->Result1(), thrIdx); - hjPtr->setSearchResult(hjPtr->Result2(), thrIdx); - sendAllHashSet = (joinType == LEFTOUTER); - sendAllSearchSet = (joinType == RIGHTOUTER); - } //if set1Size > set2Size ... + vec1.push_back(e); - } // if set1Size <=0 . . . - - params->timeset.setTimer(createHashStr); - hjPtr->createHash(hjPtr->HashSet(thrIdx), - hjPtr->HashTable(thrIdx), - bucketIdx, - sendAllHashSet, - hjPtr->HashResult(thrIdx), - params->dlTimes, params->die); - params->timeset.holdTimer(createHashStr); - -#ifdef DEBUG - long hashSetTotal = 0; - long searchSetTotal = 0; - - for (uint32_t j = 0; j < hjPtr->HashSet(thrIdx)->bucketCount(); j++) - hashSetTotal += hjPtr->HashSet(thrIdx)->bucketCount(); // are bucketDL - - for (uint32_t j = 0; j < hjPtr->HashSet(thrIdx)->bucketCount(); j++) - searchSetTotal += hjPtr->SearchSet(thrIdx)->size(j); // can be any datalist - - cout << "\t\tJoinByBucket() thr " << dec << thrIdx - << " bkt " << bucketIdx - << " hashSize " << hashSetTotal - << " searchSize " << searchSetTotal << endl; -#endif - - bool more; - e_t e; - e_t e2; - const uint64_t InvalidRID = static_cast(-1); - int iter = hjPtr->SearchSet(thrIdx)->getIterator(bucketIdx); - - ZonedDL* zdl1 = dynamic_cast(hjPtr->SearchResult(thrIdx)); - ZonedDL* zdl2 = dynamic_cast(hjPtr->HashResult(thrIdx)); - vector vec1; - vector vec2; - - std::pair::hashIter_t, typename HashJoin::hashIter_t> hashItPair; - typename HashJoin::hashIter_t hashIt; - typename HashJoin::hash_t* ht = hjPtr->HashTable(thrIdx); - params->timeset.setTimer(hashJoinStr); - - for (more = hjPtr->SearchSet(thrIdx)->next(bucketIdx, iter, &e); - more && !(*params->die); - more = hjPtr->SearchSet(thrIdx)->next(bucketIdx, iter, &e) ) - { - - // If sendAllSearchSet=true, keep all of the search set. If this is - // a right outer, we are dealing with a join such as - // col1 = col2 (+) - // where col1 is the SearchSet and col2 is the HashSet. We want to include - // all of col1 in this case regardless of whether there is a matching col2. - if (sendAllSearchSet) - { - if (zdl1) - { - vec1.push_back(e); - - if (vec1.size() >= ZDL_VEC_SIZE) - { - params->timeset.setTimer(insertResultsStr); - hjPtr->SearchResult(thrIdx)->insert(vec1); - vec1.clear(); - params->timeset.holdTimer(insertResultsStr); - } - } - else - hjPtr->SearchResult(thrIdx)->insert(e); - } - - hashIt = ht->find(e.second); - - if (hashIt != ht->end()) - { -#ifdef DEBUG - - if (hjPtr->SearchResult(thrIdx)->OID() >= 3000) - cout << "JoinByBucket() SearchResult add " << bucketIdx - << " [" << e.first << "][" << e.second << "]" << endl; - - uint32_t a = 0; - e_t b = e_t(); -#endif - - // If sendAllSearchSet=false, we already added the search result - // before the if condition above. - if (!sendAllSearchSet) - { - if (zdl1) - { - vec1.push_back(e); - - if (vec1.size() >= ZDL_VEC_SIZE) - { - params->timeset.setTimer(insertResultsStr); - hjPtr->SearchResult(thrIdx)->insert(vec1); - vec1.clear(); - params->timeset.holdTimer(insertResultsStr); - } - } - else - hjPtr->SearchResult(thrIdx)->insert(e); - } - - // If sendAllHashSet=false, add the hash results to the output datalist. - // If it is a left outer join then we already added all of the right side rows - // in the bucket in the createHash call earlier in this function. - if (!sendAllHashSet) - { - - // If the matching pair has it's RID set to invalid, it's already been encountered, - // so no reason to add it to the output datalist or keep searching for more matching values. - if (hashIt->second != InvalidRID) - { - - // If the matching pair has it's RID set to invalid, it's already been encountered, - hashItPair = ht->equal_range(e.second); - - for (hashIt = hashItPair.first; hashIt != hashItPair.second; hashIt++) - { - e2.first = hashIt->second; - e2.second = e.second; - - if (zdl2) - { - vec2.push_back(e2); - - if (vec2.size() >= ZDL_VEC_SIZE) - { - params->timeset.setTimer(insertResultsStr); - hjPtr->HashResult(thrIdx)->insert(vec2); - vec2.clear(); - params->timeset.holdTimer(insertResultsStr); - } - } - else - hjPtr->HashResult(thrIdx)->insert(e2); - -#ifdef DEBUG - a++; - b = v.second; -#endif - - // Set the RID to invalid rid now that it's been matched and added to the output datalist. - // This will keep us from duplicating it if it is matched again. - hashIt->second = InvalidRID; - - } - -#ifdef DEBUG - cout << "\t\tadded " << b << " " << a << " times" << endl << endl; -#endif - } - - } - - } // if hashIt != hashIt->end() - - } // for ( hjPtr... - - params->timeset.holdTimer(hashJoinStr); - - params->timeset.setTimer(insertLastResultsStr); - - if (vec1.size() != 0) - { + if (vec1.size() >= ZDL_VEC_SIZE) + { + params->timeset.setTimer(insertResultsStr); hjPtr->SearchResult(thrIdx)->insert(vec1); vec1.clear(); + params->timeset.holdTimer(insertResultsStr); + } } + else + hjPtr->SearchResult(thrIdx)->insert(e); + } - if (vec2.size() != 0) + // If sendAllHashSet=false, add the hash results to the output datalist. + // If it is a left outer join then we already added all of the right side rows + // in the bucket in the createHash call earlier in this function. + if (!sendAllHashSet) + { + // If the matching pair has it's RID set to invalid, it's already been encountered, + // so no reason to add it to the output datalist or keep searching for more matching values. + if (hashIt->second != InvalidRID) { - hjPtr->HashResult(thrIdx)->insert(vec2); - vec2.clear(); + // If the matching pair has it's RID set to invalid, it's already been encountered, + hashItPair = ht->equal_range(e.second); + + for (hashIt = hashItPair.first; hashIt != hashItPair.second; hashIt++) + { + e2.first = hashIt->second; + e2.second = e.second; + + if (zdl2) + { + vec2.push_back(e2); + + if (vec2.size() >= ZDL_VEC_SIZE) + { + params->timeset.setTimer(insertResultsStr); + hjPtr->HashResult(thrIdx)->insert(vec2); + vec2.clear(); + params->timeset.holdTimer(insertResultsStr); + } + } + else + hjPtr->HashResult(thrIdx)->insert(e2); + +#ifdef DEBUG + a++; + b = v.second; +#endif + + // Set the RID to invalid rid now that it's been matched and added to the output datalist. + // This will keep us from duplicating it if it is matched again. + hashIt->second = InvalidRID; + } + +#ifdef DEBUG + cout << "\t\tadded " << b << " " << a << " times" << endl << endl; +#endif } + } - params->timeset.holdTimer(insertLastResultsStr); + } // if hashIt != hashIt->end() - // typename HashJoin::hash_t* ht = hjPtr->HashTable(thrIdx); - ht->clear(); + } // for ( hjPtr... - } // for (bucketIdx... - } // try -// We don't have to call JSA.endOfInput() for this exception, because -// the parent thread takes care of that in performThreadedJoin(). - catch (const logging::LargeDataListExcept& ex) + params->timeset.holdTimer(hashJoinStr); + + params->timeset.setTimer(insertLastResultsStr); + + if (vec1.size() != 0) + { + hjPtr->SearchResult(thrIdx)->insert(vec1); + vec1.clear(); + } + + if (vec2.size() != 0) + { + hjPtr->HashResult(thrIdx)->insert(vec2); + vec2.clear(); + } + + params->timeset.holdTimer(insertLastResultsStr); + + // typename HashJoin::hash_t* ht = hjPtr->HashTable(thrIdx); + ht->clear(); + + } // for (bucketIdx... + } // try + // We don't have to call JSA.endOfInput() for this exception, because + // the parent thread takes care of that in performThreadedJoin(). + catch (const logging::LargeDataListExcept& ex) + { + ostringstream errMsg; + + if (typeid(e_t) == typeid(StringElementType)) { - ostringstream errMsg; - - if (typeid(e_t) == typeid(StringElementType)) - { - errMsg << "HashJoinByBucket_thr: caught LDL error: " << - ex.what(); - hjPtr->status(logging::stringHashJoinStepLargeDataListFileErr); - } - else - { - errMsg << "HashJoinByBucket_thr: caught LDL error: " << ex.what(); - hjPtr->status(logging::largeHashJoinLargeDataListFileErr); - } - - cerr << errMsg.str() << endl; - catchHandler(errMsg.str(), hjPtr->sessionId()); + errMsg << "HashJoinByBucket_thr: caught LDL error: " << ex.what(); + hjPtr->status(logging::stringHashJoinStepLargeDataListFileErr); } - catch (const exception& ex) + else { - ostringstream errMsg; - - if (typeid(e_t) == typeid(StringElementType)) - { - errMsg << "HashJoinByBucket_thr: caught: " << ex.what(); - hjPtr->status(logging::stringHashJoinStepErr); - } - else - { - errMsg << "HashJoinByBucket_thr: caught: " << ex.what(); - hjPtr->status(logging::largeHashJoinErr); - } - - cerr << errMsg.str() << endl; - catchHandler(errMsg.str(), hjPtr->sessionId()); - } - catch (...) - { - ostringstream errMsg; - - if (typeid(e_t) == typeid(StringElementType)) - { - errMsg << "HashJoinByBucket_thr: caught unknown exception: "; - hjPtr->status(logging::stringHashJoinStepErr); - } - else - { - errMsg << "HashJoinByBucket_thr: caught unknown exception"; - hjPtr->status(logging::largeHashJoinErr); - } - - cerr << errMsg.str() << endl; - catchHandler(errMsg.str(), hjPtr->sessionId()); + errMsg << "HashJoinByBucket_thr: caught LDL error: " << ex.what(); + hjPtr->status(logging::largeHashJoinLargeDataListFileErr); } - return NULL; -} // HashJoinByBucket_thr + cerr << errMsg.str() << endl; + catchHandler(errMsg.str(), hjPtr->sessionId()); + } + catch (const exception& ex) + { + ostringstream errMsg; -LargeHashJoin::LargeHashJoin(JoinType joinType, - uint32_t sessionId, - uint32_t txnId, - uint32_t statementId, - ResourceManager* rm ): - fSessionId(sessionId), fTxnId(txnId), - fStepId(0), fStatementId(statementId), fTableOID1(0), fTableOID2(0), - fJoinType(joinType), fRm(rm), fAlias1(), fAlias2() + if (typeid(e_t) == typeid(StringElementType)) + { + errMsg << "HashJoinByBucket_thr: caught: " << ex.what(); + hjPtr->status(logging::stringHashJoinStepErr); + } + else + { + errMsg << "HashJoinByBucket_thr: caught: " << ex.what(); + hjPtr->status(logging::largeHashJoinErr); + } + + cerr << errMsg.str() << endl; + catchHandler(errMsg.str(), hjPtr->sessionId()); + } + catch (...) + { + ostringstream errMsg; + + if (typeid(e_t) == typeid(StringElementType)) + { + errMsg << "HashJoinByBucket_thr: caught unknown exception: "; + hjPtr->status(logging::stringHashJoinStepErr); + } + else + { + errMsg << "HashJoinByBucket_thr: caught unknown exception"; + hjPtr->status(logging::largeHashJoinErr); + } + + cerr << errMsg.str() << endl; + catchHandler(errMsg.str(), hjPtr->sessionId()); + } + + return NULL; +} // HashJoinByBucket_thr + +LargeHashJoin::LargeHashJoin(JoinType joinType, uint32_t sessionId, uint32_t txnId, uint32_t statementId, + ResourceManager* rm) + : fSessionId(sessionId) + , fTxnId(txnId) + , fStepId(0) + , fStatementId(statementId) + , fTableOID1(0) + , fTableOID2(0) + , fJoinType(joinType) + , fRm(rm) + , fAlias1() + , fAlias2() { -// fConfig = config::Config::makeConfig(); -// fJoinType = joinType; + // fConfig = config::Config::makeConfig(); + // fJoinType = joinType; } LargeHashJoin::~LargeHashJoin() { - if (traceOn()) - { - for (uint64_t i = 0; i < fInputJobStepAssociation.outSize(); i++) - logDiskIoInfo(fStepId, fInputJobStepAssociation.outAt(i)); + if (traceOn()) + { + for (uint64_t i = 0; i < fInputJobStepAssociation.outSize(); i++) + logDiskIoInfo(fStepId, fInputJobStepAssociation.outAt(i)); - for (uint64_t i = 0; i < fOutputJobStepAssociation.outSize(); i++) - logDiskIoInfo(fStepId, fOutputJobStepAssociation.outAt(i)); - } + for (uint64_t i = 0; i < fOutputJobStepAssociation.outSize(); i++) + logDiskIoInfo(fStepId, fOutputJobStepAssociation.outAt(i)); + } } void LargeHashJoin::join() { - runner->join(); + runner->join(); } void LargeHashJoin::run() { - if (traceOn()) - { - syslogStartStep(16, // exemgr subsystem - std::string("LargeHashJoin")); // step name - } + if (traceOn()) + { + syslogStartStep(16, // exemgr subsystem + std::string("LargeHashJoin")); // step name + } - runner.reset(new boost::thread(HJRunner(this))); + runner.reset(new boost::thread(HJRunner(this))); } void LargeHashJoin::unblockDatalists(uint16_t status) { - fOutputJobStepAssociation.status(status); - fOutputJobStepAssociation.outAt(0)->dataList()->endOfInput(); - fOutputJobStepAssociation.outAt(1)->dataList()->endOfInput(); + fOutputJobStepAssociation.status(status); + fOutputJobStepAssociation.outAt(0)->dataList()->endOfInput(); + fOutputJobStepAssociation.outAt(1)->dataList()->endOfInput(); } void LargeHashJoin::errorLogging(const string& msg) const { - ostringstream errMsg; - errMsg << "Step " << stepId() << "; " << msg; - cerr << errMsg.str() << endl; - catchHandler( errMsg.str(), sessionId() ); + ostringstream errMsg; + errMsg << "Step " << stepId() << "; " << msg; + cerr << errMsg.str() << endl; + catchHandler(errMsg.str(), sessionId()); } void LargeHashJoin::doHashJoin() { - string val; + string val; - idbassert(fInputJobStepAssociation.outSize() >= 2); - idbassert(fOutputJobStepAssociation.outSize() >= 2); - BucketDataList* Ap = 0; - BucketDataList* Bp = 0; - BucketDataList* tAp = 0; - BucketDataList* tBp = 0; - DataList_t* inDL1 = 0; - DataList_t* inDL2 = 0; - inDL1 = fInputJobStepAssociation.outAt(0)->dataList(); - inDL2 = fInputJobStepAssociation.outAt(1)->dataList(); - idbassert(inDL1); - idbassert(inDL2); + idbassert(fInputJobStepAssociation.outSize() >= 2); + idbassert(fOutputJobStepAssociation.outSize() >= 2); + BucketDataList* Ap = 0; + BucketDataList* Bp = 0; + BucketDataList* tAp = 0; + BucketDataList* tBp = 0; + DataList_t* inDL1 = 0; + DataList_t* inDL2 = 0; + inDL1 = fInputJobStepAssociation.outAt(0)->dataList(); + inDL2 = fInputJobStepAssociation.outAt(1)->dataList(); + idbassert(inDL1); + idbassert(inDL2); - HashJoin* hj = 0; - double createWorkTime = 0; - double hashWorkTime = 0; - double insertWorkTime = 0; - DataList_t* resultA = fOutputJobStepAssociation.outAt(0)->dataList(); - DataList_t* resultB = fOutputJobStepAssociation.outAt(1)->dataList(); + HashJoin* hj = 0; + double createWorkTime = 0; + double hashWorkTime = 0; + double insertWorkTime = 0; + DataList_t* resultA = fOutputJobStepAssociation.outAt(0)->dataList(); + DataList_t* resultB = fOutputJobStepAssociation.outAt(1)->dataList(); - if (0 < fInputJobStepAssociation.status()) + if (0 < fInputJobStepAssociation.status()) + { + unblockDatalists(fInputJobStepAssociation.status()); + } + else + { + string currentAction("preparing join"); + + try { - unblockDatalists(fInputJobStepAssociation.status()); + // If we're given BucketDL's, use them + if (typeid(*inDL1) == typeid(BucketDataList)) + { + if (typeid(*inDL2) != typeid(BucketDataList)) + { + throw logic_error("LargeHashJoin::run: expected either 0 or 2 BucketDL's!"); + } + + Ap = dynamic_cast(inDL1); + Bp = dynamic_cast(inDL2); + } + else + { + throw logic_error("HashJoin will take only BucketDLs as inputs"); + int maxBuckets = fRm.getHjMaxBuckets(); + joblist::ridtype_t maxElems = fRm.getHjMaxElems(); + + BucketDataList* tAp = new BucketDataList(maxBuckets, 1, maxElems, fRm); + BucketDataList* tBp = new BucketDataList(maxBuckets, 1, maxElems, fRm); + tAp->setHashMode(1); + tBp->setHashMode(1); + + ElementType element; + int id; + + id = inDL1->getIterator(); + + while (inDL1->next(id, &element)) + { + tAp->insert(element); + } + + tAp->endOfInput(); + + id = inDL2->getIterator(); + + while (inDL2->next(id, &element)) + { + tBp->insert(element); + } + + tBp->endOfInput(); + + Ap = tAp; + Bp = tBp; + } + + unsigned numThreads = fRm.getHjNumThreads(); + + BDLWrapper setA(Ap); + BDLWrapper setB(Bp); + + hj = new HashJoin(setA, setB, resultA, resultB, fJoinType, &dlTimes, + fOutputJobStepAssociation.statusPtr(), sessionId(), &die); + + if (fTableOID2 >= 3000) + { + ostringstream logStr2; + logStr2 << "LargeHashJoin::run: ses:" << fSessionId << " st:" << fStepId + << " input sizes: " << setA.size() << "/" << setB.size() << endl; + cout << logStr2.str(); + } + + currentAction = "performing join"; + + if (fTableOID2 >= 3000) + { + dlTimes.setFirstReadTime(); + dlTimes.setEndOfInputTime(dlTimes.FirstReadTime()); + } + + hj->performJoin(numThreads); + + } // try + catch (const logging::LargeDataListExcept& ex) + { + ostringstream errMsg; + errMsg << __FILE__ << " doHashJoin: " << currentAction << ", caught LDL error: " << ex.what(); + errorLogging(errMsg.str()); + unblockDatalists(logging::largeHashJoinLargeDataListFileErr); } - else + catch (const exception& ex) { - - string currentAction("preparing join"); - - try - { - //If we're given BucketDL's, use them - if (typeid(*inDL1) == typeid(BucketDataList)) - { - - if (typeid(*inDL2) != typeid(BucketDataList)) - { - throw logic_error("LargeHashJoin::run: expected either 0 or 2 BucketDL's!"); - } - - Ap = dynamic_cast(inDL1); - Bp = dynamic_cast(inDL2); - } - else - { - throw logic_error("HashJoin will take only BucketDLs as inputs"); - int maxBuckets = fRm.getHjMaxBuckets(); - joblist::ridtype_t maxElems = fRm.getHjMaxElems(); - - BucketDataList* tAp = new BucketDataList(maxBuckets, 1, maxElems, fRm); - BucketDataList* tBp = new BucketDataList(maxBuckets, 1, maxElems, fRm); - tAp->setHashMode(1); - tBp->setHashMode(1); - - ElementType element; - int id; - - id = inDL1->getIterator(); - - while (inDL1->next(id, &element)) - { - tAp->insert(element); - } - - tAp->endOfInput(); - - id = inDL2->getIterator(); - - while (inDL2->next(id, &element)) - { - tBp->insert(element); - } - - tBp->endOfInput(); - - Ap = tAp; - Bp = tBp; - } - - unsigned numThreads = fRm.getHjNumThreads(); - - BDLWrapper< ElementType > setA(Ap); - BDLWrapper< ElementType > setB(Bp); - - hj = new HashJoin(setA, setB, resultA, resultB, fJoinType, &dlTimes, fOutputJobStepAssociation.statusPtr(), sessionId(), &die); - - if (fTableOID2 >= 3000) - { - ostringstream logStr2; - logStr2 << "LargeHashJoin::run: ses:" << fSessionId << - " st:" << fStepId << - " input sizes: " << setA.size() << "/" << setB.size() << endl; - cout << logStr2.str(); - } - - currentAction = "performing join"; - - if (fTableOID2 >= 3000) - { - dlTimes.setFirstReadTime(); - dlTimes.setEndOfInputTime( dlTimes.FirstReadTime() ); - } - - hj->performJoin(numThreads); - - } // try - catch (const logging::LargeDataListExcept& ex) - { - ostringstream errMsg; - errMsg << __FILE__ << " doHashJoin: " << - currentAction << ", caught LDL error: " << ex.what(); - errorLogging(errMsg.str()); - unblockDatalists(logging::largeHashJoinLargeDataListFileErr); - } - catch (const exception& ex) - { - ostringstream errMsg; - errMsg << __FILE__ << " doHashJoin: " << - currentAction << ", caught: " << ex.what(); - errorLogging(errMsg.str()); - unblockDatalists(logging::largeHashJoinErr); - } - catch (...) - { - ostringstream errMsg; - errMsg << __FILE__ << " doHashJoin: " << - currentAction << ", caught unknown exception"; - errorLogging(errMsg.str()); - unblockDatalists(logging::largeHashJoinErr); - } - - if (hj) - { - //..hashWorkTime is the time to perform the hashjoin excluding the - // the output insertion time. insertWorkTime is the sum or total - // of both insert times. The end result is that createWorkTime + - // hashWorkTime + insertWorkTime roughly equates to the total work - // time. - createWorkTime = hj->getTimeSet()->totalTime(createHashStr); - hashWorkTime = hj->getTimeSet()->totalTime(hashJoinStr) - - hj->getTimeSet()->totalTime(insertResultsStr); - insertWorkTime = hj->getTimeSet()->totalTime(insertResultsStr) + - hj->getTimeSet()->totalTime(insertLastResultsStr); - } - - } // (fInputJobStepAssociation.status() == 0) - - if (fTableOID2 >= 3000 && traceOn()) + ostringstream errMsg; + errMsg << __FILE__ << " doHashJoin: " << currentAction << ", caught: " << ex.what(); + errorLogging(errMsg.str()); + unblockDatalists(logging::largeHashJoinErr); + } + catch (...) { - time_t finTime = time(0); - char finTimeString[50]; - ctime_r(&finTime, finTimeString); - finTimeString[strlen(finTimeString) - 1 ] = '\0'; - - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << - " finished at " << finTimeString << - "; 1st read " << dlTimes.FirstReadTimeString() << - "; EOI " << dlTimes.EndOfInputTimeString() << endl - << "\tLargeHashJoin::run: output sizes: " - << resultA->totalSize() << "/" << resultB->totalSize() - << " run time: " << - JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << fixed << setprecision(2) - << "s\n\tTotal work times: create hash: " << createWorkTime - << "s, hash join: " << hashWorkTime - << "s, insert results: " << insertWorkTime << "s\n" - << "\tJob completion status " << fInputJobStepAssociation.status() << endl; - logEnd(logStr.str().c_str()); - - syslogProcessingTimes(16, // exemgr subsystem - dlTimes.FirstReadTime(), // use join start time for first read time - dlTimes.EndOfInputTime(), // use join end time for last read time - dlTimes.FirstReadTime(), // use join start time for first write time - dlTimes.EndOfInputTime()); // use join end time for last write time - syslogEndStep(16, // exemgr subsystem - 0, // no blocked datalist input to report - 0); // no blocked datalist output to report + ostringstream errMsg; + errMsg << __FILE__ << " doHashJoin: " << currentAction << ", caught unknown exception"; + errorLogging(errMsg.str()); + unblockDatalists(logging::largeHashJoinErr); } - delete hj; - delete tAp; - delete tBp; + if (hj) + { + //..hashWorkTime is the time to perform the hashjoin excluding the + // the output insertion time. insertWorkTime is the sum or total + // of both insert times. The end result is that createWorkTime + + // hashWorkTime + insertWorkTime roughly equates to the total work + // time. + createWorkTime = hj->getTimeSet()->totalTime(createHashStr); + hashWorkTime = hj->getTimeSet()->totalTime(hashJoinStr) - hj->getTimeSet()->totalTime(insertResultsStr); + insertWorkTime = + hj->getTimeSet()->totalTime(insertResultsStr) + hj->getTimeSet()->totalTime(insertLastResultsStr); + } + + } // (fInputJobStepAssociation.status() == 0) + + if (fTableOID2 >= 3000 && traceOn()) + { + time_t finTime = time(0); + char finTimeString[50]; + ctime_r(&finTime, finTimeString); + finTimeString[strlen(finTimeString) - 1] = '\0'; + + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << finTimeString << "; 1st read " + << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() << endl + << "\tLargeHashJoin::run: output sizes: " << resultA->totalSize() << "/" << resultB->totalSize() + << " run time: " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << fixed << setprecision(2) << "s\n\tTotal work times: create hash: " << createWorkTime + << "s, hash join: " << hashWorkTime << "s, insert results: " << insertWorkTime << "s\n" + << "\tJob completion status " << fInputJobStepAssociation.status() << endl; + logEnd(logStr.str().c_str()); + + syslogProcessingTimes(16, // exemgr subsystem + dlTimes.FirstReadTime(), // use join start time for first read time + dlTimes.EndOfInputTime(), // use join end time for last read time + dlTimes.FirstReadTime(), // use join start time for first write time + dlTimes.EndOfInputTime()); // use join end time for last write time + syslogEndStep(16, // exemgr subsystem + 0, // no blocked datalist input to report + 0); // no blocked datalist output to report + } + + delete hj; + delete tAp; + delete tBp; } const string LargeHashJoin::toString() const { - ostringstream oss; - CalpontSystemCatalog::OID oid1 = 0; - CalpontSystemCatalog::OID oid2 = 0; - DataList_t* dl1; - DataList_t* dl2; - size_t idlsz; + ostringstream oss; + CalpontSystemCatalog::OID oid1 = 0; + CalpontSystemCatalog::OID oid2 = 0; + DataList_t* dl1; + DataList_t* dl2; + size_t idlsz; - idlsz = fInputJobStepAssociation.outSize(); - idbassert(idlsz == 2); - dl1 = fInputJobStepAssociation.outAt(0)->dataList(); + idlsz = fInputJobStepAssociation.outSize(); + idbassert(idlsz == 2); + dl1 = fInputJobStepAssociation.outAt(0)->dataList(); - if (dl1) oid1 = dl1->OID(); + if (dl1) + oid1 = dl1->OID(); - dl2 = fInputJobStepAssociation.outAt(1)->dataList(); + dl2 = fInputJobStepAssociation.outAt(1)->dataList(); - if (dl2) oid2 = dl2->OID(); + if (dl2) + oid2 = dl2->OID(); - oss << "LargeHashJoin ses:" << fSessionId << " st:" << fStepId; - oss << omitOidInDL; - oss << " in tb/col1:" << fTableOID1 << "/" << oid1; - oss << " " << fInputJobStepAssociation.outAt(0); - oss << " in tb/col2:" << fTableOID2 << "/" << oid2; - oss << " " << fInputJobStepAssociation.outAt(1); + oss << "LargeHashJoin ses:" << fSessionId << " st:" << fStepId; + oss << omitOidInDL; + oss << " in tb/col1:" << fTableOID1 << "/" << oid1; + oss << " " << fInputJobStepAssociation.outAt(0); + oss << " in tb/col2:" << fTableOID2 << "/" << oid2; + oss << " " << fInputJobStepAssociation.outAt(1); - idlsz = fOutputJobStepAssociation.outSize(); - idbassert(idlsz == 2); - dl1 = fOutputJobStepAssociation.outAt(0)->dataList(); + idlsz = fOutputJobStepAssociation.outSize(); + idbassert(idlsz == 2); + dl1 = fOutputJobStepAssociation.outAt(0)->dataList(); - if (dl1) oid1 = dl1->OID(); + if (dl1) + oid1 = dl1->OID(); - dl2 = fOutputJobStepAssociation.outAt(1)->dataList(); + dl2 = fOutputJobStepAssociation.outAt(1)->dataList(); - if (dl2) oid2 = dl2->OID(); + if (dl2) + oid2 = dl2->OID(); - oss << endl << " "; - oss << " out tb/col1:" << fTableOID1 << "/" << oid1; - oss << " " << fOutputJobStepAssociation.outAt(0); - oss << " out tb/col2:" << fTableOID2 << "/" << oid2; - oss << " " << fOutputJobStepAssociation.outAt(1) << endl; + oss << endl << " "; + oss << " out tb/col1:" << fTableOID1 << "/" << oid1; + oss << " " << fOutputJobStepAssociation.outAt(0); + oss << " out tb/col2:" << fTableOID2 << "/" << oid2; + oss << " " << fOutputJobStepAssociation.outAt(1) << endl; - return oss.str(); + return oss.str(); } -StringHashJoinStep::StringHashJoinStep(JoinType joinType, - uint32_t sessionId, - uint32_t txnId, - uint32_t statementId, - ResourceManager* rm): - LargeHashJoin(joinType, sessionId, txnId, statementId, rm) +StringHashJoinStep::StringHashJoinStep(JoinType joinType, uint32_t sessionId, uint32_t txnId, + uint32_t statementId, ResourceManager* rm) + : LargeHashJoin(joinType, sessionId, txnId, statementId, rm) { } @@ -786,390 +764,384 @@ StringHashJoinStep::~StringHashJoinStep() void StringHashJoinStep::run() { - if (traceOn()) - { - syslogStartStep(16, // exemgr subsystem - std::string("StringHashJoinStep")); // step name - } + if (traceOn()) + { + syslogStartStep(16, // exemgr subsystem + std::string("StringHashJoinStep")); // step name + } - runner.reset(new boost::thread(StringHJRunner(this))); + runner.reset(new boost::thread(StringHJRunner(this))); } void StringHashJoinStep::doStringHashJoin() { - string val; + string val; - idbassert(fInputJobStepAssociation.outSize() >= 2); - idbassert(fOutputJobStepAssociation.outSize() >= 2); - DataList* inDL1 = fInputJobStepAssociation.outAt(0)->stringDataList(); - DataList* inDL2 = fInputJobStepAssociation.outAt(1)->stringDataList(); - idbassert(inDL1); - idbassert(inDL2); + idbassert(fInputJobStepAssociation.outSize() >= 2); + idbassert(fOutputJobStepAssociation.outSize() >= 2); + DataList* inDL1 = fInputJobStepAssociation.outAt(0)->stringDataList(); + DataList* inDL2 = fInputJobStepAssociation.outAt(1)->stringDataList(); + idbassert(inDL1); + idbassert(inDL2); - BucketDL* Ap = 0; - BucketDL* Bp = 0; - BucketDL* tAp = 0; - BucketDL* tBp = 0; + BucketDL* Ap = 0; + BucketDL* Bp = 0; + BucketDL* tAp = 0; + BucketDL* tBp = 0; - HashJoin* hj = 0; - double createWorkTime = 0; - double hashWorkTime = 0; - double insertWorkTime = 0; - DataList_t* resultA = fOutputJobStepAssociation.outAt(0)->dataList(); - DataList_t* resultB = fOutputJobStepAssociation.outAt(1)->dataList(); - struct timeval start_time; - gettimeofday(&start_time, 0); - struct timeval end_time = start_time; - ZonedDL* bdl1 = 0; - ZonedDL* bdl2 = 0; + HashJoin* hj = 0; + double createWorkTime = 0; + double hashWorkTime = 0; + double insertWorkTime = 0; + DataList_t* resultA = fOutputJobStepAssociation.outAt(0)->dataList(); + DataList_t* resultB = fOutputJobStepAssociation.outAt(1)->dataList(); + struct timeval start_time; + gettimeofday(&start_time, 0); + struct timeval end_time = start_time; + ZonedDL* bdl1 = 0; + ZonedDL* bdl2 = 0; - // result from hashjoinstep is expected to be BandedDataList - // but the HashJoin returns StringDataList - // also, the null is reported as "_CpNuLl_" by pDictionStep - // create two StringDataList for the intermediate result BDL - // @bug 721. use zdl. - StringZonedDL* dlA = new StringZonedDL(1, fRm); - dlA->setMultipleProducers(true); - StringZonedDL* dlB = new StringZonedDL(1, fRm); - dlB->setMultipleProducers(true); + // result from hashjoinstep is expected to be BandedDataList + // but the HashJoin returns StringDataList + // also, the null is reported as "_CpNuLl_" by pDictionStep + // create two StringDataList for the intermediate result BDL + // @bug 721. use zdl. + StringZonedDL* dlA = new StringZonedDL(1, fRm); + dlA->setMultipleProducers(true); + StringZonedDL* dlB = new StringZonedDL(1, fRm); + dlB->setMultipleProducers(true); - if (0 < fInputJobStepAssociation.status() ) + if (0 < fInputJobStepAssociation.status()) + { + unblockDatalists(fInputJobStepAssociation.status()); + } + else + { + string currentAction("preparing join"); + + try { - unblockDatalists(fInputJobStepAssociation.status()); + // If we're given BucketDL's, use them + if (typeid(*inDL1) == typeid(BucketDL)) + { + if (typeid(*inDL2) != typeid(BucketDL)) + { + throw logic_error("StringHashJoinStep::run: expected either 0 or 2 BucketDL's!"); + } + + Ap = dynamic_cast*>(inDL1); + Bp = dynamic_cast*>(inDL2); + } + else + { + int maxBuckets = fRm.getHjMaxBuckets(); + joblist::ridtype_t maxElems = fRm.getHjMaxElems(); + + // int maxBuckets=4; + // joblist::ridtype_t maxElems=1024*8; + // val = fConfig->getConfig("HashJoin", "MaxBuckets"); // same as HashJoin + // if (val.size() > 0) + // maxBuckets = static_cast(config::Config::fromText(val)); + // if (maxBuckets <=0) + // maxBuckets=4; + // val = fConfig->getConfig("HashJoin", "MaxElems"); // same as HashJoin + // if (val.size() >0) + // maxElems = config::Config::uFromText(val); + // if (maxElems<=0) + // maxElems=1024*8; + + tAp = new BucketDL(maxBuckets, 1, maxElems, fRm); + tBp = new BucketDL(maxBuckets, 1, maxElems, fRm); + tAp->setHashMode(1); + tBp->setHashMode(1); + + StringElementType element; + int id = inDL1->getIterator(); + + while (inDL1->next(id, &element)) + { + tAp->insert(element); + } + + tAp->endOfInput(); + + id = inDL2->getIterator(); + + while (inDL2->next(id, &element)) + { + tBp->insert(element); + } + + tBp->endOfInput(); + + Ap = tAp; + Bp = tBp; + } + + unsigned numThreads = fRm.getHjNumThreads(); + // unsigned numThreads = 0; + // val = fConfig->getConfig("HashJoin", "NumThreads"); + // if (val.size() > 0) + // numThreads = static_cast(config::Config::uFromText(val)); + // if (numThreads <= 0) + // numThreads = 4; + + BDLWrapper setA(Ap); + BDLWrapper setB(Bp); + + HashJoin* hj = + new HashJoin(setA, setB, dlA, dlB, fJoinType, &dlTimes, + fOutputJobStepAssociation.statusPtr(), sessionId(), &die); + + if ((dlA == NULL) || (dlB == NULL) || (hj == NULL)) + { + ostringstream oss; + oss << "StringHashJoinStep::run() null pointer from new -- "; + oss << "StringDataList A(0x" << hex << (ptrdiff_t)dlA << "), B(0x" << (ptrdiff_t)dlB + << "), HashJoin hj(0x" << (ptrdiff_t)hj << ")"; + throw(runtime_error(oss.str().c_str())); + } + + // leave this in + if (fTableOID2 >= 3000) + { + ostringstream logStr2; + logStr2 << "StringHashJoinStep::run: ses:" << fSessionId << " st:" << fStepId + << " input sizes: " << setA.size() << "/" << setB.size() << endl; + cout << logStr2.str(); + } + + currentAction = "performing join"; + + if (fTableOID2 >= 3000) + { + dlTimes.setFirstReadTime(); + dlTimes.setEndOfInputTime(dlTimes.FirstReadTime()); + } + + hj->performJoin(numThreads); + + currentAction = "after join"; + + // convert from StringElementType to ElementType by grabbing the rid + // take _CpNuLl_ out of the result + StringElementType se; + ElementType e; + int id = dlA->getIterator(); + + bdl1 = dynamic_cast(resultA); + bdl2 = dynamic_cast(resultB); + vector v; + v.reserve(ZDL_VEC_SIZE); + + if (bdl1) + { + while (dlA->next(id, &se)) + { + if (se.second != CPNULLSTRMARK) + { + e.first = se.first; + v.push_back(e); + + if (v.size() >= ZDL_VEC_SIZE) + { + resultA->insert(v); + v.clear(); + } + } + } + + if (v.size() > 0) + resultA->insert(v); + + resultA->endOfInput(); + } + + else + { + while (dlA->next(id, &se)) + { + if (se.second != CPNULLSTRMARK) + { + e.first = se.first; + resultA->insert(e); + } + } + + resultA->endOfInput(); + } + + id = dlB->getIterator(); + + if (bdl2) + { + v.clear(); + + while (dlB->next(id, &se)) + { + if (se.second != CPNULLSTRMARK) + { + e.first = se.first; + v.push_back(e); + + if (v.size() >= ZDL_VEC_SIZE) + { + resultB->insert(v); + v.clear(); + } + } + } + + if (v.size() > 0) + resultB->insert(v); + + resultB->endOfInput(); + } + else + { + while (dlB->next(id, &se)) + { + if (se.second != CPNULLSTRMARK) + { + e.first = se.first; + resultB->insert(e); + } + } + + resultB->endOfInput(); + } + } // try + catch (const logging::LargeDataListExcept& ex) + { + ostringstream errMsg; + errMsg << __FILE__ << " doStringHashJoin: " << currentAction << ", caught LDL error: " << ex.what(); + errorLogging(errMsg.str()); + unblockDatalists(logging::stringHashJoinStepLargeDataListFileErr); + dlA->endOfInput(); + dlB->endOfInput(); } - else + catch (const exception& ex) { - - string currentAction("preparing join"); - - try - { - //If we're given BucketDL's, use them - if (typeid(*inDL1) == typeid(BucketDL)) - { - if (typeid(*inDL2) != typeid(BucketDL)) - { - throw logic_error("StringHashJoinStep::run: expected either 0 or 2 BucketDL's!"); - } - - Ap = dynamic_cast*>(inDL1); - Bp = dynamic_cast*>(inDL2); - } - else - { - int maxBuckets = fRm.getHjMaxBuckets(); - joblist::ridtype_t maxElems = fRm.getHjMaxElems(); - -// int maxBuckets=4; -// joblist::ridtype_t maxElems=1024*8; -// val = fConfig->getConfig("HashJoin", "MaxBuckets"); // same as HashJoin -// if (val.size() > 0) -// maxBuckets = static_cast(config::Config::fromText(val)); -// if (maxBuckets <=0) -// maxBuckets=4; -// val = fConfig->getConfig("HashJoin", "MaxElems"); // same as HashJoin -// if (val.size() >0) -// maxElems = config::Config::uFromText(val); -// if (maxElems<=0) -// maxElems=1024*8; - - tAp = new BucketDL(maxBuckets, 1, maxElems, fRm); - tBp = new BucketDL(maxBuckets, 1, maxElems, fRm); - tAp->setHashMode(1); - tBp->setHashMode(1); - - StringElementType element; - int id = inDL1->getIterator(); - - while (inDL1->next(id, &element)) - { - tAp->insert(element); - } - - tAp->endOfInput(); - - id = inDL2->getIterator(); - - while (inDL2->next(id, &element)) - { - tBp->insert(element); - } - - tBp->endOfInput(); - - Ap = tAp; - Bp = tBp; - } - - unsigned numThreads = fRm.getHjNumThreads(); -// unsigned numThreads = 0; -// val = fConfig->getConfig("HashJoin", "NumThreads"); -// if (val.size() > 0) -// numThreads = static_cast(config::Config::uFromText(val)); -// if (numThreads <= 0) -// numThreads = 4; - - BDLWrapper< StringElementType > setA(Ap); - BDLWrapper< StringElementType > setB(Bp); - - HashJoin* hj = - new HashJoin(setA, setB, dlA, dlB, fJoinType, &dlTimes, fOutputJobStepAssociation.statusPtr(), sessionId(), &die); - - if ((dlA == NULL) || (dlB == NULL) || (hj == NULL)) - { - ostringstream oss; - oss << "StringHashJoinStep::run() null pointer from new -- "; - oss << "StringDataList A(0x" << hex << (ptrdiff_t)dlA << "), B(0x" - << (ptrdiff_t)dlB << "), HashJoin hj(0x" << (ptrdiff_t)hj << ")"; - throw (runtime_error(oss.str().c_str())); - } - - // leave this in - if (fTableOID2 >= 3000) - { - ostringstream logStr2; - logStr2 << "StringHashJoinStep::run: ses:" << fSessionId << - " st:" << fStepId << - " input sizes: " << setA.size() << "/" << setB.size() << endl; - cout << logStr2.str(); - } - - currentAction = "performing join"; - - if (fTableOID2 >= 3000) - { - dlTimes.setFirstReadTime(); - dlTimes.setEndOfInputTime( dlTimes.FirstReadTime() ); - } - - hj->performJoin(numThreads); - - currentAction = "after join"; - - // convert from StringElementType to ElementType by grabbing the rid - // take _CpNuLl_ out of the result - StringElementType se; - ElementType e; - int id = dlA->getIterator(); - - bdl1 = dynamic_cast(resultA); - bdl2 = dynamic_cast(resultB); - vector v; - v.reserve(ZDL_VEC_SIZE); - - if (bdl1) - { - while (dlA->next(id, &se)) - { - if (se.second != CPNULLSTRMARK) - { - e.first = se.first; - v.push_back(e); - - if (v.size() >= ZDL_VEC_SIZE) - { - resultA->insert(v); - v.clear(); - } - } - } - - if (v.size() > 0) - resultA->insert(v); - - resultA->endOfInput(); - } - - else - { - while (dlA->next(id, &se)) - { - if (se.second != CPNULLSTRMARK) - { - e.first = se.first; - resultA->insert(e); - } - } - - resultA->endOfInput(); - } - - id = dlB->getIterator(); - - if (bdl2) - { - v.clear(); - - while (dlB->next(id, &se)) - { - if (se.second != CPNULLSTRMARK) - { - e.first = se.first; - v.push_back(e); - - if (v.size() >= ZDL_VEC_SIZE) - { - resultB->insert(v); - v.clear(); - } - } - } - - if (v.size() > 0) - resultB->insert(v); - - resultB->endOfInput(); - } - else - { - while (dlB->next(id, &se)) - { - if (se.second != CPNULLSTRMARK) - { - e.first = se.first; - resultB->insert(e); - } - } - - resultB->endOfInput(); - } - } // try - catch (const logging::LargeDataListExcept& ex) - { - ostringstream errMsg; - errMsg << __FILE__ << " doStringHashJoin: " << - currentAction << ", caught LDL error: " << ex.what(); - errorLogging(errMsg.str()); - unblockDatalists(logging::stringHashJoinStepLargeDataListFileErr); - dlA->endOfInput(); - dlB->endOfInput(); - } - catch (const exception& ex) - { - ostringstream errMsg; - errMsg << __FILE__ << " doStringHashJoin: " << - currentAction << ", caught: " << ex.what(); - errorLogging(errMsg.str()); - unblockDatalists(logging::stringHashJoinStepErr); - dlA->endOfInput(); - dlB->endOfInput(); - } - catch (...) - { - ostringstream errMsg; - errMsg << __FILE__ << " doStringHashJoin: " << - currentAction << ", caught unknown exception"; - errorLogging(errMsg.str()); - unblockDatalists(logging::stringHashJoinStepErr); - dlA->endOfInput(); - dlB->endOfInput(); - } - - gettimeofday(&end_time, 0); - - if (fTableOID2 >= 3000) dlTimes.setEndOfInputTime(); - - if (hj) - { - //..hashWorkTime is the time to perform the hashjoin excluding the - // the output insertion time. insertWorkTime is the sum or total - // of both insert times. The end result is that createWorkTime + - // hashWorkTime + insertWorkTime roughly equates to the total work - // time. - createWorkTime = hj->getTimeSet()->totalTime(createHashStr); - hashWorkTime = hj->getTimeSet()->totalTime(hashJoinStr) - - hj->getTimeSet()->totalTime(insertResultsStr); - insertWorkTime = hj->getTimeSet()->totalTime(insertResultsStr) + - hj->getTimeSet()->totalTime(insertLastResultsStr); - } - - } // (fInputJobStepAssociation.status() == 0) - - if (fTableOID2 >= 3000 && traceOn()) + ostringstream errMsg; + errMsg << __FILE__ << " doStringHashJoin: " << currentAction << ", caught: " << ex.what(); + errorLogging(errMsg.str()); + unblockDatalists(logging::stringHashJoinStepErr); + dlA->endOfInput(); + dlB->endOfInput(); + } + catch (...) { - time_t finTime = time(0); - char finTimeString[50]; - ctime_r(&finTime, finTimeString); - finTimeString[strlen(finTimeString) - 1 ] = '\0'; - - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << - " finished at " << finTimeString << - "; 1st read " << dlTimes.FirstReadTimeString() << - "; EOI " << dlTimes.EndOfInputTimeString() << endl - << "\tStringHashJoinStep::run: output sizes: " - << dlA->totalSize() << "/" << dlB->totalSize() << " ["; - - if (bdl1 && bdl2) - logStr << bdl1->totalSize() << "/" << bdl2->totalSize(); - - logStr << "] run time: " << - JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << fixed << setprecision(2) - << "s\n\tTotal work times: create hash: " << createWorkTime - << "s, hash join: " << hashWorkTime - << "s, insert results: " << insertWorkTime << "s\n" - << "\tJob completion status " << fInputJobStepAssociation.status() << endl; - logEnd(logStr.str().c_str()); - - syslogProcessingTimes(16, // exemgr subsystem - start_time, // use join start time for first read time - end_time, // use join end time for last read time - start_time, // use join start time for first write time - end_time); // use join end time for last write time - syslogEndStep(16, // exemgr subsystem - 0, // no blocked datalist input to report - 0); // no blocked datalist output to report + ostringstream errMsg; + errMsg << __FILE__ << " doStringHashJoin: " << currentAction << ", caught unknown exception"; + errorLogging(errMsg.str()); + unblockDatalists(logging::stringHashJoinStepErr); + dlA->endOfInput(); + dlB->endOfInput(); } - delete hj; - delete tAp; - delete tBp; - delete dlA; - delete dlB; + gettimeofday(&end_time, 0); + + if (fTableOID2 >= 3000) + dlTimes.setEndOfInputTime(); + + if (hj) + { + //..hashWorkTime is the time to perform the hashjoin excluding the + // the output insertion time. insertWorkTime is the sum or total + // of both insert times. The end result is that createWorkTime + + // hashWorkTime + insertWorkTime roughly equates to the total work + // time. + createWorkTime = hj->getTimeSet()->totalTime(createHashStr); + hashWorkTime = hj->getTimeSet()->totalTime(hashJoinStr) - hj->getTimeSet()->totalTime(insertResultsStr); + insertWorkTime = + hj->getTimeSet()->totalTime(insertResultsStr) + hj->getTimeSet()->totalTime(insertLastResultsStr); + } + + } // (fInputJobStepAssociation.status() == 0) + + if (fTableOID2 >= 3000 && traceOn()) + { + time_t finTime = time(0); + char finTimeString[50]; + ctime_r(&finTime, finTimeString); + finTimeString[strlen(finTimeString) - 1] = '\0'; + + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << finTimeString << "; 1st read " + << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() << endl + << "\tStringHashJoinStep::run: output sizes: " << dlA->totalSize() << "/" << dlB->totalSize() + << " ["; + + if (bdl1 && bdl2) + logStr << bdl1->totalSize() << "/" << bdl2->totalSize(); + + logStr << "] run time: " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << fixed << setprecision(2) << "s\n\tTotal work times: create hash: " << createWorkTime + << "s, hash join: " << hashWorkTime << "s, insert results: " << insertWorkTime << "s\n" + << "\tJob completion status " << fInputJobStepAssociation.status() << endl; + logEnd(logStr.str().c_str()); + + syslogProcessingTimes(16, // exemgr subsystem + start_time, // use join start time for first read time + end_time, // use join end time for last read time + start_time, // use join start time for first write time + end_time); // use join end time for last write time + syslogEndStep(16, // exemgr subsystem + 0, // no blocked datalist input to report + 0); // no blocked datalist output to report + } + + delete hj; + delete tAp; + delete tBp; + delete dlA; + delete dlB; } const string StringHashJoinStep::toString() const { - ostringstream oss; - CalpontSystemCatalog::OID oid1 = 0; - CalpontSystemCatalog::OID oid2 = 0; + ostringstream oss; + CalpontSystemCatalog::OID oid1 = 0; + CalpontSystemCatalog::OID oid2 = 0; - size_t idlsz = fInputJobStepAssociation.outSize(); - idbassert(idlsz == 2); - DataList* dl1 = fInputJobStepAssociation.outAt(0)->stringDataList(); + size_t idlsz = fInputJobStepAssociation.outSize(); + idbassert(idlsz == 2); + DataList* dl1 = fInputJobStepAssociation.outAt(0)->stringDataList(); - if (dl1) oid1 = dl1->OID(); + if (dl1) + oid1 = dl1->OID(); - DataList* dl2 = fInputJobStepAssociation.outAt(1)->stringDataList(); + DataList* dl2 = fInputJobStepAssociation.outAt(1)->stringDataList(); - if (dl2) oid2 = dl2->OID(); + if (dl2) + oid2 = dl2->OID(); - oss << "StringHashJoinStep ses:" << fSessionId << " st:" << fStepId; - oss << omitOidInDL; - oss << " in tb/col1:" << fTableOID1 << "/" << oid1; - oss << " " << fInputJobStepAssociation.outAt(0); - oss << " in tb/col2:" << fTableOID2 << "/" << oid2; - oss << " " << fInputJobStepAssociation.outAt(1); + oss << "StringHashJoinStep ses:" << fSessionId << " st:" << fStepId; + oss << omitOidInDL; + oss << " in tb/col1:" << fTableOID1 << "/" << oid1; + oss << " " << fInputJobStepAssociation.outAt(0); + oss << " in tb/col2:" << fTableOID2 << "/" << oid2; + oss << " " << fInputJobStepAssociation.outAt(1); - idlsz = fOutputJobStepAssociation.outSize(); - idbassert(idlsz == 2); - DataList_t* dl3 = fOutputJobStepAssociation.outAt(0)->dataList(); + idlsz = fOutputJobStepAssociation.outSize(); + idbassert(idlsz == 2); + DataList_t* dl3 = fOutputJobStepAssociation.outAt(0)->dataList(); - if (dl3) oid1 = dl3->OID(); + if (dl3) + oid1 = dl3->OID(); - DataList_t* dl4 = fOutputJobStepAssociation.outAt(1)->dataList(); + DataList_t* dl4 = fOutputJobStepAssociation.outAt(1)->dataList(); - if (dl4) oid2 = dl4->OID(); + if (dl4) + oid2 = dl4->OID(); - oss << endl << " "; - oss << " out tb/col1:" << fTableOID1 << "/" << oid1; - oss << " " << fOutputJobStepAssociation.outAt(0); - oss << " out tb/col2:" << fTableOID2 << "/" << oid2; - oss << " " << fOutputJobStepAssociation.outAt(1); - - return oss.str(); -} + oss << endl << " "; + oss << " out tb/col1:" << fTableOID1 << "/" << oid1; + oss << " " << fOutputJobStepAssociation.outAt(0); + oss << " out tb/col2:" << fTableOID2 << "/" << oid2; + oss << " " << fOutputJobStepAssociation.outAt(1); + return oss.str(); } +} // namespace joblist diff --git a/dbcon/joblist/largehashjoin.h b/dbcon/joblist/largehashjoin.h index c5315b355..cea003f44 100644 --- a/dbcon/joblist/largehashjoin.h +++ b/dbcon/joblist/largehashjoin.h @@ -52,18 +52,16 @@ namespace std { namespace tr1 { -template<> -struct hash - : public std::unary_function +template <> +struct hash : public std::unary_function { - std::size_t - operator()(long long unsigned int val) const - { - return static_cast(val); - } + std::size_t operator()(long long unsigned int val) const + { + return static_cast(val); + } }; -} -} +} // namespace tr1 +} // namespace std #endif #endif @@ -80,14 +78,11 @@ struct hash #include "jl_logger.h" #ifdef PROFILE -extern void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - struct timespec& diff); +extern void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff); #endif namespace joblist { - const string createHashStr("create hash"); const string hashJoinStr("hash join"); const string insertResultsStr("insert results"); @@ -99,30 +94,31 @@ void* HashJoinByBucket_thr(void* arg); /** @brief class HjHasher * */ -template +template class HjHasher { -private: - utils::Hasher hasher; + private: + utils::Hasher hasher; -public: - uint32_t operator()(const typename element_t::second_type& v) const - { - return hasher((const char*) &v, sizeof(typename element_t::second_type)); - } + public: + uint32_t operator()(const typename element_t::second_type& v) const + { + return hasher((const char*)&v, sizeof(typename element_t::second_type)); + } }; // template specialization for string -template<> +template <> class HjHasher { -private: - utils::Hasher hasher; -public: - uint32_t operator()(const std::string& v) const - { - return hasher(v.c_str(), (uint32_t) v.length()); - } + private: + utils::Hasher hasher; + + public: + uint32_t operator()(const std::string& v) const + { + return hasher(v.c_str(), (uint32_t)v.length()); + } }; /** @brief class HashJoin @@ -131,163 +127,160 @@ public: template class HashJoin { + public: + typedef std::list hashList_t; + typedef typename std::list::iterator hashListIter_t; -public: - typedef std::list hashList_t; - typedef typename std::list::iterator hashListIter_t; + // @Bug 867 - Changed the unordered_map to an unordered_multimap and changed the value to be RIDs rather + // than a list of ElementType to reduce memory utilization and to increase the performance of loading the + // map. typedef std::tr1::unordered_map, + // HjHasher > hash_t; + typedef + typename std::tr1::unordered_multimap + hash_t; + typedef typename std::tr1::unordered_multimap::iterator hashIter_t; + typedef typename std::tr1::unordered_multimap::value_type hashPair_t; - // @Bug 867 - Changed the unordered_map to an unordered_multimap and changed the value to be RIDs rather than a list of ElementType to reduce - // memory utilization and to increase the performance of loading the map. - // typedef std::tr1::unordered_map, HjHasher > hash_t; - typedef typename std::tr1::unordered_multimap hash_t; - typedef typename std::tr1::unordered_multimap::iterator hashIter_t; - typedef typename std::tr1::unordered_multimap::value_type hashPair_t; + // allow each thread to have its own pointers + struct control_struct + { + hash_t hashTbl; + BucketDL* searchSet; + BucketDL* hashSet; + DataList* searchResult; + DataList* hashResult; + }; - // allow each thread to have its own pointers - struct control_struct - { - hash_t hashTbl; - BucketDL* searchSet; - BucketDL* hashSet; - DataList* searchResult; - DataList* hashResult; - }; - - boost::scoped_array controls; //TODO: needs to be >= HJ threads from Columnstore.xml - typedef struct thrParams_struct - { - HashJoin* hjptr; - uint32_t startBucket; - uint32_t numBuckets; - uint32_t thrIdx; - TimeSet timeset; - JSTimeStamp dlTimes; - volatile bool* die; - } thrParams_t; - - HashJoin(joblist::BDLWrapper& set1, - joblist::BDLWrapper& set2, - joblist::DataList* result1, - joblist::DataList* result2, - JoinType joinType, - JSTimeStamp* dlTimes, - const SErrorInfo& status, - uint32_t sessionId, - volatile bool* die); - - HashJoin(); - HashJoin(const HashJoin& hj); - JoinType getJoinType() - { - return fJoinType; - } - virtual ~HashJoin(); - virtual int performJoin(const uint32_t thrCount = 1); - virtual int performThreadedJoin(const uint32_t numThreads); - - joblist::BDLWrapper* Set1() - { - return & fSet1; - } - joblist::BDLWrapper* Set2() - { - return & fSet2; - } - joblist::DataList* Result1() - { - return fResult1; - } - joblist::DataList* Result2() - { - return fResult2; - } - - void setSearchSet(BucketDL* bdl, const uint32_t idx) - { - controls[idx].searchSet = bdl; - } - void setHashSet(BucketDL* bdl, const uint32_t idx) - { - controls[idx].hashSet = bdl; - } - - BucketDL* SearchSet(const uint32_t idx) - { - return controls[idx].searchSet; - } - BucketDL* HashSet(const uint32_t idx) - { - return controls[idx].hashSet; - } - - void setSearchResult(DataList* bdl, const uint32_t idx) - { - controls[idx].searchResult = bdl; - } - void setHashResult(DataList* bdl, const uint32_t idx) - { - controls[idx].hashResult = bdl; - } - - joblist::DataList* SearchResult(const uint32_t idx) - { - return controls[idx].searchResult; - } - joblist::DataList* HashResult(const uint32_t idx) - { - return controls[idx].hashResult; - } - - hash_t* HashTable(const uint64_t i ) - { - return &(controls[i].hashTbl); - } - - void createHash(BucketDL* bdlptr, - hash_t* destHashTbl, - const uint32_t idx, - bool populateResult, // true if bdlptr is opposite an outer join - joblist::DataList* result, // populated if populateResult true - JSTimeStamp& thrDlTimes, volatile bool* die); - void init(); - TimeSet* getTimeSet() - { - return &fTimeSet; - } - uint16_t status() const - { - return fStatus->errCode; - } - void status(uint16_t s) - { - fStatus->errCode; - } - uint32_t sessionId() - { - return fSessionId; - } - -private: - // input sets - joblist::BDLWrapper fSet1; - joblist::BDLWrapper fSet2; - - // result sets - joblist::DataList* fResult1; - joblist::DataList* fResult2; - - // convenience pointers - BucketDL* fSearchSet; - BucketDL* fHashSet; - joblist::DataList* fSearchResult; - joblist::DataList* fHashResult; - - JoinType fJoinType; - JSTimeStamp* dlTimes; - TimeSet fTimeSet; - SErrorInfo fStatus; - uint32_t fSessionId; + boost::scoped_array controls; // TODO: needs to be >= HJ threads from + // Columnstore.xml + typedef struct thrParams_struct + { + HashJoin* hjptr; + uint32_t startBucket; + uint32_t numBuckets; + uint32_t thrIdx; + TimeSet timeset; + JSTimeStamp dlTimes; volatile bool* die; + } thrParams_t; + + HashJoin(joblist::BDLWrapper& set1, joblist::BDLWrapper& set2, + joblist::DataList* result1, joblist::DataList* result2, JoinType joinType, + JSTimeStamp* dlTimes, const SErrorInfo& status, uint32_t sessionId, volatile bool* die); + + HashJoin(); + HashJoin(const HashJoin& hj); + JoinType getJoinType() + { + return fJoinType; + } + virtual ~HashJoin(); + virtual int performJoin(const uint32_t thrCount = 1); + virtual int performThreadedJoin(const uint32_t numThreads); + + joblist::BDLWrapper* Set1() + { + return &fSet1; + } + joblist::BDLWrapper* Set2() + { + return &fSet2; + } + joblist::DataList* Result1() + { + return fResult1; + } + joblist::DataList* Result2() + { + return fResult2; + } + + void setSearchSet(BucketDL* bdl, const uint32_t idx) + { + controls[idx].searchSet = bdl; + } + void setHashSet(BucketDL* bdl, const uint32_t idx) + { + controls[idx].hashSet = bdl; + } + + BucketDL* SearchSet(const uint32_t idx) + { + return controls[idx].searchSet; + } + BucketDL* HashSet(const uint32_t idx) + { + return controls[idx].hashSet; + } + + void setSearchResult(DataList* bdl, const uint32_t idx) + { + controls[idx].searchResult = bdl; + } + void setHashResult(DataList* bdl, const uint32_t idx) + { + controls[idx].hashResult = bdl; + } + + joblist::DataList* SearchResult(const uint32_t idx) + { + return controls[idx].searchResult; + } + joblist::DataList* HashResult(const uint32_t idx) + { + return controls[idx].hashResult; + } + + hash_t* HashTable(const uint64_t i) + { + return &(controls[i].hashTbl); + } + + void createHash(BucketDL* bdlptr, hash_t* destHashTbl, const uint32_t idx, + bool populateResult, // true if bdlptr is opposite an outer join + joblist::DataList* result, // populated if populateResult true + JSTimeStamp& thrDlTimes, volatile bool* die); + void init(); + TimeSet* getTimeSet() + { + return &fTimeSet; + } + uint16_t status() const + { + return fStatus->errCode; + } + void status(uint16_t s) + { + fStatus->errCode; + } + uint32_t sessionId() + { + return fSessionId; + } + + private: + // input sets + joblist::BDLWrapper fSet1; + joblist::BDLWrapper fSet2; + + // result sets + joblist::DataList* fResult1; + joblist::DataList* fResult2; + + // convenience pointers + BucketDL* fSearchSet; + BucketDL* fHashSet; + joblist::DataList* fSearchResult; + joblist::DataList* fHashResult; + + JoinType fJoinType; + JSTimeStamp* dlTimes; + TimeSet fTimeSet; + SErrorInfo fStatus; + uint32_t fSessionId; + volatile bool* die; }; template @@ -296,299 +289,284 @@ HashJoin::HashJoin() } template -HashJoin::HashJoin(joblist::BDLWrapper& set1, - joblist::BDLWrapper& set2, - joblist::DataList* result1, - joblist::DataList* result2, - JoinType joinType, - JSTimeStamp* dlt, - const SErrorInfo& status, - uint32_t sessionId, volatile bool* d) : - fTimeSet(), fStatus(status), fSessionId(sessionId) +HashJoin::HashJoin(joblist::BDLWrapper& set1, joblist::BDLWrapper& set2, + joblist::DataList* result1, joblist::DataList* result2, + JoinType joinType, JSTimeStamp* dlt, const SErrorInfo& status, + uint32_t sessionId, volatile bool* d) + : fTimeSet(), fStatus(status), fSessionId(sessionId) { - fSet1 = set1; - fSet2 = set2; - fResult1 = result1; - fResult2 = result2; - fSearchResult = NULL; - fHashResult = NULL; - fJoinType = joinType; - die = d; - init(); - dlTimes = dlt; + fSet1 = set1; + fSet2 = set2; + fResult1 = result1; + fResult2 = result2; + fSearchResult = NULL; + fHashResult = NULL; + fJoinType = joinType; + die = d; + init(); + dlTimes = dlt; } template void HashJoin::init() { - controls.reset(new control_struct[32]); + controls.reset(new control_struct[32]); - for (int idx = 0; idx < 32; idx++) - { - HashTable(idx)->clear(); - setSearchSet(NULL, idx); - setHashSet(NULL, idx); - setSearchResult(NULL, idx); - setHashResult(NULL, idx); - } + for (int idx = 0; idx < 32; idx++) + { + HashTable(idx)->clear(); + setSearchSet(NULL, idx); + setHashSet(NULL, idx); + setSearchResult(NULL, idx); + setHashResult(NULL, idx); + } } template HashJoin::~HashJoin() { - controls.reset(); + controls.reset(); } template int HashJoin::performThreadedJoin(const uint32_t numThreads) { - //boost::thread thrArr[numThreads]; - boost::scoped_array thrArr(new boost::thread[numThreads]); - //typename HashJoin::thrParams_t params[numThreads]; - boost::scoped_array::thrParams_t> params(new typename HashJoin::thrParams_t[numThreads]); - uint32_t maxThreads = numThreads; - int realCnt = 0; - uint32_t bucketsPerThr = 0; - uint32_t totalBuckets = 0; + // boost::thread thrArr[numThreads]; + boost::scoped_array thrArr(new boost::thread[numThreads]); + // typename HashJoin::thrParams_t params[numThreads]; + boost::scoped_array::thrParams_t> params( + new typename HashJoin::thrParams_t[numThreads]); + uint32_t maxThreads = numThreads; + int realCnt = 0; + uint32_t bucketsPerThr = 0; + uint32_t totalBuckets = 0; - // TODO: maybe this should throw an exception - if (maxThreads <= 0 || maxThreads > 32) - { - maxThreads = 1; + // TODO: maybe this should throw an exception + if (maxThreads <= 0 || maxThreads > 32) + { + maxThreads = 1; #ifdef DEBUG - cerr << "HashJoin: invalid requested thread value n=" << numThreads << endl; + cerr << "HashJoin: invalid requested thread value n=" << numThreads << endl; +#endif + } + + // s/b equal buckets so check Set1() buckets + if (Set1()->bucketCount() < maxThreads) + { + maxThreads = Set1()->bucketCount(); + } + + bucketsPerThr = (uint32_t)(Set1()->bucketCount() / maxThreads); + uint32_t idx = 0; + + for (idx = 0; idx < maxThreads && totalBuckets < Set1()->bucketCount(); idx++) + { + params[idx].hjptr = this; + params[idx].startBucket = totalBuckets; + params[idx].numBuckets = bucketsPerThr; + params[idx].thrIdx = idx; + params[idx].die = die; + totalBuckets += bucketsPerThr; + + if ((totalBuckets + bucketsPerThr) > Set1()->bucketCount()) + bucketsPerThr = Set1()->bucketCount() - totalBuckets; + +#ifdef DEBUG + cout << "thr i " << idx << " [" << params[idx].startBucket << ", " << params[idx].numBuckets << "] " + << totalBuckets << " " << (int)bucketsPerThr << endl; +#endif + + } // for( + + // add the remaining buckets to the last thread + // instead of adding a thread + if (totalBuckets < Set1()->bucketCount()) + { + idx--; + params[idx].numBuckets += bucketsPerThr; + } + +#ifdef DEBUG + cout << "thr i " << idx << " [" << params[idx].startBucket << ", " << params[idx].numBuckets << "] " + << totalBuckets << " " << (int)bucketsPerThr << "-" << endl; +#endif + + try + { + for (idx = 0; idx < maxThreads; idx++) + { + int ret = 0; + // ret = pthread_create(&thrArr[idx], NULL, HashJoinByBucket_thr, ¶ms[idx]); + boost::thread t(HashJoinByBucket_thr, ¶ms[idx]); + thrArr[idx].swap(t); + + if (ret != 0) + throw logic_error("HashJoin: pthread_create failure"); + else + { + realCnt++; + } + +#ifdef DEBUG + cout << "Started thr " << idx << endl; #endif } - // s/b equal buckets so check Set1() buckets - if (Set1()->bucketCount() < maxThreads) + idbassert((unsigned)realCnt == maxThreads); + } // try + catch (std::exception& e) + { + std::ostringstream errMsg; + + if (typeid(element_t) == typeid(StringElementType)) { - maxThreads = Set1()->bucketCount(); + errMsg << "performThreadedJoin: caught: " << e.what(); + *fStatus = logging::stringHashJoinStepErr; + } + else + { + errMsg << "performThreadedJoin: caught: " << e.what(); + *fStatus = logging::largeHashJoinErr; } - bucketsPerThr = (uint32_t)(Set1()->bucketCount() / maxThreads); - uint32_t idx = 0; + std::cerr << errMsg.str() << std::endl; + catchHandler(errMsg.str(), sessionId()); + } + catch (...) + { + std::ostringstream errMsg; - for (idx = 0; idx < maxThreads && totalBuckets < Set1()->bucketCount(); idx++) + if (typeid(element_t) == typeid(StringElementType)) { - params[idx].hjptr = this; - params[idx].startBucket = totalBuckets; - params[idx].numBuckets = bucketsPerThr; - params[idx].thrIdx = idx; - params[idx].die = die; - totalBuckets += bucketsPerThr; + errMsg << "performThreadedJoin: unknown exception"; + *fStatus = logging::stringHashJoinStepErr; + } + else + { + errMsg << "performThreadedJoin: caught: unknown exception"; + *fStatus = logging::largeHashJoinErr; + } - if ( (totalBuckets + bucketsPerThr) > Set1()->bucketCount() ) - bucketsPerThr = Set1()->bucketCount() - totalBuckets; + std::cerr << errMsg.str() << std::endl; + catchHandler(errMsg.str(), sessionId()); + } + for (int idx = 0; idx < realCnt; idx++) + { + thrArr[idx].join(); // pthread_join(thrArr[idx], NULL); #ifdef DEBUG - cout << "thr i " << idx - << " [" << params[idx].startBucket << ", " << params[idx].numBuckets << "] " - << totalBuckets << " " << (int)bucketsPerThr << endl; + cout << "HJ " << hex << this << dec << ": Joining thr " << idx << endl; #endif + } - } // for( + Result1()->endOfInput(); + Result2()->endOfInput(); - // add the remaining buckets to the last thread - // instead of adding a thread - if (totalBuckets < Set1()->bucketCount()) - { - idx--; - params[idx].numBuckets += bucketsPerThr; + if (realCnt > 0) + dlTimes->setFirstReadTime(params[0].dlTimes.FirstReadTime()); - } + dlTimes->setEndOfInputTime(); -#ifdef DEBUG - cout << "thr i " << idx << " [" << params[idx].startBucket << ", " - << params[idx].numBuckets << "] " - << totalBuckets << " " << (int)bucketsPerThr << "-" << endl; -#endif + for (int i = 0; i < realCnt; i++) + { + fTimeSet += params[i].timeset; - try - { - for (idx = 0; idx < maxThreads; idx++) - { + // Select earliest read time as overall firstReadTime + if (params[i].dlTimes.FirstReadTime().tv_sec < dlTimes->FirstReadTime().tv_sec) + dlTimes->setFirstReadTime(params[i].dlTimes.FirstReadTime()); + } - int ret = 0; - //ret = pthread_create(&thrArr[idx], NULL, HashJoinByBucket_thr, ¶ms[idx]); - boost::thread t(HashJoinByBucket_thr, ¶ms[idx]); - thrArr[idx].swap(t); + controls.reset(); - if (ret != 0) - throw logic_error("HashJoin: pthread_create failure"); - else - { - realCnt++; - } - -#ifdef DEBUG - cout << "Started thr " << idx << endl; -#endif - } - - idbassert((unsigned)realCnt == maxThreads); - } // try - catch (std::exception& e) - { - std::ostringstream errMsg; - - if (typeid(element_t) == typeid(StringElementType)) - { - errMsg << "performThreadedJoin: caught: " << e.what(); - *fStatus = logging::stringHashJoinStepErr; - } - else - { - errMsg << "performThreadedJoin: caught: " << e.what(); - *fStatus = logging::largeHashJoinErr; - } - - std::cerr << errMsg.str() << std::endl; - catchHandler(errMsg.str(), sessionId()); - } - catch (...) - { - std::ostringstream errMsg; - - if (typeid(element_t) == typeid(StringElementType)) - { - errMsg << "performThreadedJoin: unknown exception"; - *fStatus = logging::stringHashJoinStepErr; - } - else - { - errMsg << "performThreadedJoin: caught: unknown exception"; - *fStatus = logging::largeHashJoinErr; - } - - std::cerr << errMsg.str() << std::endl; - catchHandler(errMsg.str(), sessionId()); - } - - for (int idx = 0; idx < realCnt; idx++) - { - thrArr[idx].join(); //pthread_join(thrArr[idx], NULL); -#ifdef DEBUG - cout << "HJ " << hex << this << dec << ": Joining thr " << idx << endl; -#endif - } - - Result1()->endOfInput(); - Result2()->endOfInput(); - - if (realCnt > 0) - dlTimes->setFirstReadTime( params[0].dlTimes.FirstReadTime() ); - - dlTimes->setEndOfInputTime(); - - for (int i = 0; i < realCnt; i++) - { - fTimeSet += params[i].timeset; - - // Select earliest read time as overall firstReadTime - if ( params[i].dlTimes.FirstReadTime().tv_sec < dlTimes->FirstReadTime().tv_sec ) - dlTimes->setFirstReadTime( params[i].dlTimes.FirstReadTime() ); - } - - controls.reset(); - - return realCnt; + return realCnt; } // defaults to 1 thread template int HashJoin::performJoin(const uint32_t thrCount) { - return performThreadedJoin(thrCount); + return performThreadedJoin(thrCount); } // create a hash table from the elements in the bucketDL at bdlptr[bucketNum] // template -void HashJoin::createHash(BucketDL* srcBucketDL, - hash_t* destHashTbl, - const uint32_t bucketNum, - bool populateResult, - joblist::DataList* result, - JSTimeStamp& thrDlTimes, volatile bool* die) +void HashJoin::createHash(BucketDL* srcBucketDL, hash_t* destHashTbl, + const uint32_t bucketNum, bool populateResult, + joblist::DataList* result, JSTimeStamp& thrDlTimes, + volatile bool* die) { - bool more; - element_t e; - element_t temp; + bool more; + element_t e; + element_t temp; #ifdef DEBUG - int idx = 0; + int idx = 0; #endif #ifdef PROFILE - timespec ts1, ts2, diff; - clock_gettime(CLOCK_REALTIME, &ts1); + timespec ts1, ts2, diff; + clock_gettime(CLOCK_REALTIME, &ts1); #endif - uint32_t bucketIter = srcBucketDL->getIterator((int)bucketNum); - // @bug 828. catch hashjoin starting time - more = srcBucketDL->next(bucketNum, bucketIter, &e); + uint32_t bucketIter = srcBucketDL->getIterator((int)bucketNum); + // @bug 828. catch hashjoin starting time + more = srcBucketDL->next(bucketNum, bucketIter, &e); - if (thrDlTimes.FirstReadTime().tv_sec == 0) + if (thrDlTimes.FirstReadTime().tv_sec == 0) + { + thrDlTimes.setFirstReadTime(); + } + + for (; more & !(*die); more = srcBucketDL->next(bucketNum, bucketIter, &e)) + { +#ifdef DEBUG + cout << "createHash() bkt " << bucketNum << " idx " << idx << " find(" << e.second << ")" << endl; +#endif + + // If the bucket dl is the other side of an outer join, we want to go ahead and populate the output + // data list with every row. For example, if the where clause is: + // where colA (+) = colB + // and the passed bucket datalist contains colb, we will go ahead and populate the output datalist here + // because all of colB should be returned regardless of whether there is a matching colA. + if (populateResult) { - thrDlTimes.setFirstReadTime(); + result->insert(e); } - for (; more & !(*die); more = srcBucketDL->next(bucketNum, bucketIter, &e)) + try { + // std::list tmp(1,e); + destHashTbl->insert( + std::pair(e.second, e.first)); + } + catch (exception& exc) + { + std::ostringstream errMsg; + errMsg << "Exception in createHash() " << exc.what(); + std::cerr << errMsg.str() << endl; + catchHandler(errMsg.str(), sessionId()); + throw; // rethrow + } + catch (...) + { + std::string errMsg("Unknown exception in createHash()"); + std::cerr << errMsg << endl; + catchHandler(errMsg, sessionId()); + throw; // rethrow + } + + } // for (more... #ifdef DEBUG - cout << "createHash() bkt " << bucketNum - << " idx " << idx << " find(" << e.second << ")" << endl; -#endif - - // If the bucket dl is the other side of an outer join, we want to go ahead and populate the output - // data list with every row. For example, if the where clause is: - // where colA (+) = colB - // and the passed bucket datalist contains colb, we will go ahead and populate the output datalist here - // because all of colB should be returned regardless of whether there is a matching colA. - if (populateResult) - { - result->insert(e); - } - - try - { - // std::list tmp(1,e); - destHashTbl->insert(std::pair - (e.second, e.first)); - } - catch (exception& exc) - { - std::ostringstream errMsg; - errMsg << "Exception in createHash() " << exc.what(); - std::cerr << errMsg.str() << endl; - catchHandler(errMsg.str(), sessionId()); - throw; // rethrow - } - catch (...) - { - std::string errMsg ("Unknown exception in createHash()"); - std::cerr << errMsg << endl; - catchHandler(errMsg, sessionId()); - throw; // rethrow - } - - } // for (more... - -#ifdef DEBUG - cout << "createHash() bkt " << bucketNum << " complete" << endl; + cout << "createHash() bkt " << bucketNum << " complete" << endl; #endif #ifdef PROFILE - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - std::cout << "Time to create hash for bucket pair " - << bucketNum << ": " - << diff.tv_sec << "s " - << diff.tv_nsec << "ns" - << std::endl; + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + std::cout << "Time to create hash for bucket pair " << bucketNum << ": " << diff.tv_sec << "s " + << diff.tv_nsec << "ns" << std::endl; #endif -} // createHash +} // createHash - -} //namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 53bed0168..26080f0c2 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -15,11 +15,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - /****************************************************************************** -* $Id: lbidlist.cpp 9655 2013-06-25 23:08:13Z xlou $ -* -******************************************************************************/ + * $Id: lbidlist.cpp 9655 2013-06-25 23:08:13Z xlou $ + * + ******************************************************************************/ #include #include "primitivemsg.h" #include "blocksize.h" @@ -32,7 +31,7 @@ #include "mcs_decimal.h" #define IS_VERBOSE (fDebug >= 4) -#define IS_DETAIL (fDebug >= 3) +#define IS_DETAIL (fDebug >= 3) #define IS_SUMMARY (fDebug >= 2) using namespace std; @@ -41,130 +40,123 @@ using namespace BRM; namespace joblist { - LBIDList::LBIDList() { - throw logic_error("Don't use LBIDList()"); + throw logic_error("Don't use LBIDList()"); } /** @LBIDList(oid, debug) -* -* Create a new LBIDList structure. -* Used to apply CP logic but cannot perform updates of CP data -*/ + * + * Create a new LBIDList structure. + * Used to apply CP logic but cannot perform updates of CP data + */ LBIDList::LBIDList(const int debug) { - fDebug = debug; + fDebug = debug; } /** @LBIDList(oid, debug) -* -* Create a new LBIDList structure. -* ctor that prepares the object for updates -* of the CP data. -*/ + * + * Create a new LBIDList structure. + * ctor that prepares the object for updates + * of the CP data. + */ -LBIDList::LBIDList(const CalpontSystemCatalog::OID oid, - const int debug) +LBIDList::LBIDList(const CalpontSystemCatalog::OID oid, const int debug) { - init(oid, debug); + init(oid, debug); } /** @LBIDList::Init() Initializes a LBIDList structure -* -* Create a new LBIDList structure and initialize it -*/ + * + * Create a new LBIDList structure and initialize it + */ -void LBIDList::init(const CalpontSystemCatalog::OID oid, - const int debug) +void LBIDList::init(const CalpontSystemCatalog::OID oid, const int debug) { - LBIDRange LBIDR; - fDebug = debug; - int err = 0; + LBIDRange LBIDR; + fDebug = debug; + int err = 0; #ifdef DEBUG - if (IS_VERBOSE) - cout << "LBIDList Init for " << oid - << " size = " << LBIDRanges.size() - << " " << this << endl; + if (IS_VERBOSE) + cout << "LBIDList Init for " << oid << " size = " << LBIDRanges.size() << " " << this << endl; #endif - if (!em) - { - em.reset(new DBRM); - } + if (!em) + { + em.reset(new DBRM); + } - try - { - err = em->lookup(oid, LBIDRanges); - } - catch (exception& e) - { - cout << "LBIDList::init(): DBRM lookup error: " << e.what() << endl; - throw; - } + try + { + err = em->lookup(oid, LBIDRanges); + } + catch (exception& e) + { + cout << "LBIDList::init(): DBRM lookup error: " << e.what() << endl; + throw; + } - if (err) - { - cout << "Lookup error ret " << err << endl; - throw runtime_error("LBIDList::init(): DBRM lookup failure"); - } + if (err) + { + cout << "Lookup error ret " << err << endl; + throw runtime_error("LBIDList::init(): DBRM lookup failure"); + } #ifdef DEBUG - const int RangeCount = LBIDRanges.size(); + const int RangeCount = LBIDRanges.size(); + + if (IS_VERBOSE) + cout << "LBIDList Init got " << RangeCount << " ranges " << endl; + + for (int i = 0; i < RangeCount; i++) + { + LBIDR = LBIDRanges.at(i); if (IS_VERBOSE) - cout << "LBIDList Init got " << RangeCount << " ranges " << endl; - - for (int i = 0; i < RangeCount; i++) - { - LBIDR = LBIDRanges.at(i); - - if (IS_VERBOSE) - cout << "Start = " << LBIDR.start << ", Len = " << LBIDR.size << endl; - } + cout << "Start = " << LBIDR.start << ", Len = " << LBIDR.size << endl; + } #endif - } LBIDList::~LBIDList() { - std::vector::value_type ptr; + std::vector::value_type ptr; - while (!lbidPartitionVector.empty()) - { - ptr = lbidPartitionVector.back(); - lbidPartitionVector.pop_back(); - delete ptr; - } + while (!lbidPartitionVector.empty()) + { + ptr = lbidPartitionVector.back(); + lbidPartitionVector.pop_back(); + delete ptr; + } #ifdef DEBUG - if (IS_VERBOSE) - cout << "LBIDList::~LBIDList() object deleted " << this << endl << endl; + if (IS_VERBOSE) + cout << "LBIDList::~LBIDList() object deleted " << this << endl << endl; #endif } - void LBIDList::Dump(long Index, int Count) const { - LBIDRange LBIDR; + LBIDRange LBIDR; - const int RangeCount = LBIDRanges.size(); - cout << "LBIDList::Dump with " << RangeCount << "ranges" << endl; + const int RangeCount = LBIDRanges.size(); + cout << "LBIDList::Dump with " << RangeCount << "ranges" << endl; - for (int i = 0; i < RangeCount; i++) - { - LBIDR = LBIDRanges.at(i); - cout << "Start = " << LBIDR.start << ", Len = " << LBIDR.size << endl; - } + for (int i = 0; i < RangeCount; i++) + { + LBIDR = LBIDRanges.at(i); + cout << "Start = " << LBIDR.start << ", Len = " << LBIDR.size << endl; + } - cout << endl; + cout << endl; } // Store max/min structure for update later. lbidPartitionVector serves a list of lbid,max,min to update @@ -175,367 +167,349 @@ bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType colDataType) { - bool bRet = true; - MinMaxPartition* mmp = NULL; - LBIDRange LBIDR; - const int RangeCount = LBIDRanges.size(); - int32_t seq32 = 0; + bool bRet = true; + MinMaxPartition* mmp = NULL; + LBIDRange LBIDR; + const int RangeCount = LBIDRanges.size(); + int32_t seq32 = 0; - for (int i = 0; i < RangeCount; i++) + for (int i = 0; i < RangeCount; i++) + { + LBIDR = LBIDRanges.at(i); + + if (lbid == LBIDR.start) { - LBIDR = LBIDRanges.at(i); + int retVal = -1; - if (lbid == LBIDR.start) - { - int retVal = -1; + if (pEMEntries && pEMEntries->size() > 0) + { + // @bug 2968 - Get CP info from snapshot of ExtentMap taken at + // the start of the query. + retVal = getMinMaxFromEntries(min, max, seq32, lbid, *pEMEntries); + } + else + { + if (em) + retVal = em->getExtentMaxMin(lbid, max, min, seq32); + } - if (pEMEntries && pEMEntries->size() > 0) - { - // @bug 2968 - Get CP info from snapshot of ExtentMap taken at - // the start of the query. - retVal = getMinMaxFromEntries(min, max, seq32, lbid, *pEMEntries); - } - else - { - if (em) retVal = em->getExtentMaxMin(lbid, max, min, seq32); - } - - seq = seq32; + seq = seq32; #ifdef DEBUG - if (IS_VERBOSE) - cout << i - << " GetMinMax() ret " << retVal - << " Min " << min - << " Max " << max - << " lbid " << lbid - << " seq32 " << seq32 << endl; + if (IS_VERBOSE) + cout << i << " GetMinMax() ret " << retVal << " Min " << min << " Max " << max << " lbid " << lbid + << " seq32 " << seq32 << endl; #endif - if (retVal != BRM::CP_VALID) // invalid extent that needs to be validated - { + if (retVal != BRM::CP_VALID) // invalid extent that needs to be validated + { #ifdef DEBUG - cout << "Added Mx/Mn to partitionVector" << endl; + cout << "Added Mx/Mn to partitionVector" << endl; #endif - mmp = new MinMaxPartition(); - mmp->lbid = (int64_t)LBIDR.start; - mmp->lbidmax = (int64_t)(LBIDR.start + LBIDR.size); - mmp->seq = seq32; - - if (isUnsigned(colDataType)) - { - mmp->max = 0; - mmp->min = static_cast(numeric_limits::max()); - } - else - { - if (typeid(T) == typeid(int128_t)) - { - mmp->bigMax = datatypes::Decimal::minInt128; - mmp->bigMin = datatypes::Decimal::maxInt128; - } - else - { - mmp->max = numeric_limits::min(); - mmp->min = numeric_limits::max(); - } - } - - mmp->isValid = retVal; - mmp->blksScanned = 0; - lbidPartitionVector.push_back(mmp); - bRet = false; - } - - return bRet; - } - } - - return false; -} - -template -bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, - int64_t lbid, const tr1::unordered_map& entries, - execplan::CalpontSystemCatalog::ColDataType colDataType) -{ - tr1::unordered_map::const_iterator it = entries.find(lbid); - - if (it == entries.end()) - return false; - - const BRM::EMEntry& entry = it->second; - - if (entry.partition.cprange.isValid != BRM::CP_VALID) - { - MinMaxPartition* mmp; mmp = new MinMaxPartition(); - mmp->lbid = lbid; - mmp->lbidmax = lbid + (entry.range.size * 1024); - mmp->seq = entry.partition.cprange.sequenceNum; + mmp->lbid = (int64_t)LBIDR.start; + mmp->lbidmax = (int64_t)(LBIDR.start + LBIDR.size); + mmp->seq = seq32; if (isUnsigned(colDataType)) { - mmp->max = 0; - mmp->min = static_cast(numeric_limits::max()); + mmp->max = 0; + mmp->min = static_cast(numeric_limits::max()); } else { - if (typeid(T) == typeid(int128_t)) - { - mmp->bigMax = datatypes::Decimal::minInt128; - mmp->bigMin = datatypes::Decimal::maxInt128; - } - else - { - mmp->max = numeric_limits::min(); - mmp->min = numeric_limits::max(); - } + if (typeid(T) == typeid(int128_t)) + { + mmp->bigMax = datatypes::Decimal::minInt128; + mmp->bigMin = datatypes::Decimal::maxInt128; + } + else + { + mmp->max = numeric_limits::min(); + mmp->min = numeric_limits::max(); + } } - mmp->isValid = entry.partition.cprange.isValid; + mmp->isValid = retVal; mmp->blksScanned = 0; lbidPartitionVector.push_back(mmp); - return false; - } + bRet = false; + } - // *DRRTUY min/max should be 16 aligned here - if (typeid(T) == typeid(int128_t)) + return bRet; + } + } + + return false; +} + +template +bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, + const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType) +{ + tr1::unordered_map::const_iterator it = entries.find(lbid); + + if (it == entries.end()) + return false; + + const BRM::EMEntry& entry = it->second; + + if (entry.partition.cprange.isValid != BRM::CP_VALID) + { + MinMaxPartition* mmp; + mmp = new MinMaxPartition(); + mmp->lbid = lbid; + mmp->lbidmax = lbid + (entry.range.size * 1024); + mmp->seq = entry.partition.cprange.sequenceNum; + + if (isUnsigned(colDataType)) { - *min = entry.partition.cprange.bigLoVal; - *max = entry.partition.cprange.bigHiVal; + mmp->max = 0; + mmp->min = static_cast(numeric_limits::max()); } else { - *min = entry.partition.cprange.loVal; - *max = entry.partition.cprange.hiVal; + if (typeid(T) == typeid(int128_t)) + { + mmp->bigMax = datatypes::Decimal::minInt128; + mmp->bigMin = datatypes::Decimal::maxInt128; + } + else + { + mmp->max = numeric_limits::min(); + mmp->min = numeric_limits::max(); + } } - *seq = entry.partition.cprange.sequenceNum; + mmp->isValid = entry.partition.cprange.isValid; + mmp->blksScanned = 0; + lbidPartitionVector.push_back(mmp); + return false; + } - return true; + // *DRRTUY min/max should be 16 aligned here + if (typeid(T) == typeid(int128_t)) + { + *min = entry.partition.cprange.bigLoVal; + *max = entry.partition.cprange.bigHiVal; + } + else + { + *min = entry.partition.cprange.loVal; + *max = entry.partition.cprange.hiVal; + } + + *seq = entry.partition.cprange.sequenceNum; + + return true; } - // Get the min, max, and sequence number for the specified LBID by searching // the given vector of ExtentMap entries. template -int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, - int64_t lbid, const std::vector& EMEntries) +int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, + const std::vector& EMEntries) { - for (unsigned i = 0; i < EMEntries.size(); i++) + for (unsigned i = 0; i < EMEntries.size(); i++) + { + int64_t lastLBID = EMEntries[i].range.start + (EMEntries[i].range.size * 1024) - 1; + + if (lbid >= EMEntries[i].range.start && lbid <= lastLBID) { - int64_t lastLBID = EMEntries[i].range.start + (EMEntries[i].range.size * 1024) - 1; - - if (lbid >= EMEntries[i].range.start && lbid <= lastLBID) - { - // *DRRTUY min/max should be 16 aligned here - if (typeid(T) == typeid(int128_t)) - { - min = EMEntries[i].partition.cprange.bigLoVal; - max = EMEntries[i].partition.cprange.bigHiVal; - } - else - { - min = EMEntries[i].partition.cprange.loVal; - max = EMEntries[i].partition.cprange.hiVal; - } - seq = EMEntries[i].partition.cprange.sequenceNum; - return EMEntries[i].partition.cprange.isValid; - } + // *DRRTUY min/max should be 16 aligned here + if (typeid(T) == typeid(int128_t)) + { + min = EMEntries[i].partition.cprange.bigLoVal; + max = EMEntries[i].partition.cprange.bigHiVal; + } + else + { + min = EMEntries[i].partition.cprange.loVal; + max = EMEntries[i].partition.cprange.hiVal; + } + seq = EMEntries[i].partition.cprange.sequenceNum; + return EMEntries[i].partition.cprange.isValid; } + } - return BRM::CP_INVALID; + return BRM::CP_INVALID; } template -void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, - const CalpontSystemCatalog::ColType & type, +void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, const CalpontSystemCatalog::ColType& type, bool validData) { - MinMaxPartition* mmp = NULL; + MinMaxPartition* mmp = NULL; #ifdef DEBUG - cout << "UpdateMinMax() Mn/Mx " << min << "/" << max - << " lbid " << lbid - << " sz " << lbidPartitionVector.size() << endl; + cout << "UpdateMinMax() Mn/Mx " << min << "/" << max << " lbid " << lbid << " sz " + << lbidPartitionVector.size() << endl; #endif - for (uint32_t i = 0; i < lbidPartitionVector.size(); i++) + for (uint32_t i = 0; i < lbidPartitionVector.size(); i++) + { + mmp = lbidPartitionVector.at(i); + + if ((lbid >= mmp->lbid) && (lbid < mmp->lbidmax)) { + mmp->blksScanned++; - mmp = lbidPartitionVector.at(i); +#ifdef DEBUG - if ( (lbid >= mmp->lbid) && (lbid < mmp->lbidmax) ) + if (IS_VERBOSE) + cout << "UpdateMinMax() old Mn/Mx " << mmp->min << "/" << mmp->max << " lbid " << lbid << " seq " + << mmp->seq << " valid " << mmp->isValid << endl; + +#endif + + if (!validData) + { + // cout << "Invalidating an extent b/c of a versioned block!\n"; + mmp->isValid = BRM::CP_UPDATING; + return; + } + + if (mmp->isValid == BRM::CP_INVALID) + { + if (datatypes::isCharType(type.colDataType)) { - mmp->blksScanned++; + datatypes::Charset cs(const_cast(type).getCharset()); + if (datatypes::TCharShort::strnncollsp(cs, min, mmp->min, type.colWidth) < 0 || + mmp->min == numeric_limits::max()) + mmp->min = min; -#ifdef DEBUG - - if (IS_VERBOSE) - cout << "UpdateMinMax() old Mn/Mx " - << mmp->min << "/" << mmp->max - << " lbid " << lbid - << " seq " << mmp->seq - << " valid " << mmp->isValid << endl; - -#endif - - if (!validData) - { - //cout << "Invalidating an extent b/c of a versioned block!\n"; - mmp->isValid = BRM::CP_UPDATING; - return; - } - - if (mmp->isValid == BRM::CP_INVALID) - { - if (datatypes::isCharType(type.colDataType)) - { - datatypes::Charset cs(const_cast(type).getCharset()); - if (datatypes::TCharShort::strnncollsp(cs, min, mmp->min, type.colWidth) < 0 || - mmp->min == numeric_limits::max()) - mmp->min = min; - - if (datatypes::TCharShort::strnncollsp(cs, max, mmp->max, type.colWidth) > 0 || - mmp->max == numeric_limits::min()) - mmp->max = max; - } - else if (datatypes::isUnsigned(type.colDataType)) - { - if (static_cast(min) < static_cast(mmp->min)) - mmp->min = min; - - if (static_cast(max) > static_cast(mmp->max)) - mmp->max = max; - } - else - { - if (typeid(T) == typeid(int128_t)) - { - if (min < mmp->bigMin) - { - mmp->bigMin = min; - } - - if (max > mmp->bigMax) - { - mmp->bigMax = max; - } - } - else - { - if (min < mmp->min) - mmp->min = min; - - if (max > mmp->max) - mmp->max = max; - } - } - - } - -#ifdef DEBUG - - if (IS_VERBOSE) - cout << "UpdateMinMax() new Mn/Mx " - << mmp->min << "/" << mmp->max << " " - << " lbid " << lbid - << " seq " << mmp->seq - << " valid " << mmp->isValid << endl; - -#endif - return; + if (datatypes::TCharShort::strnncollsp(cs, max, mmp->max, type.colWidth) > 0 || + mmp->max == numeric_limits::min()) + mmp->max = max; } + else if (datatypes::isUnsigned(type.colDataType)) + { + if (static_cast(min) < static_cast(mmp->min)) + mmp->min = min; + + if (static_cast(max) > static_cast(mmp->max)) + mmp->max = max; + } + else + { + if (typeid(T) == typeid(int128_t)) + { + if (min < mmp->bigMin) + { + mmp->bigMin = min; + } + + if (max > mmp->bigMax) + { + mmp->bigMax = max; + } + } + else + { + if (min < mmp->min) + mmp->min = min; + + if (max > mmp->max) + mmp->max = max; + } + } + } + +#ifdef DEBUG + + if (IS_VERBOSE) + cout << "UpdateMinMax() new Mn/Mx " << mmp->min << "/" << mmp->max << " " + << " lbid " << lbid << " seq " << mmp->seq << " valid " << mmp->isValid << endl; + +#endif + return; } + } } void LBIDList::UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType) { - MinMaxPartition* mmp = NULL; + MinMaxPartition* mmp = NULL; #ifdef DEBUG - if (IS_VERBOSE) - cout << "LBIDList::UpdateAllPartitionInfo() size " << lbidPartitionVector.size() << endl; + if (IS_VERBOSE) + cout << "LBIDList::UpdateAllPartitionInfo() size " << lbidPartitionVector.size() << endl; #endif - // @bug 1970 - Added new dbrm interface that takes a vector of CPInfo objects to set the min and max for multiple - // extents at a time. This cuts the number of calls way down and improves performance. - CPInfo cpInfo; - vector vCpInfo; + // @bug 1970 - Added new dbrm interface that takes a vector of CPInfo objects to set the min and max for + // multiple extents at a time. This cuts the number of calls way down and improves performance. + CPInfo cpInfo; + vector vCpInfo; - uint32_t cpUpdateInterval = 25000; + uint32_t cpUpdateInterval = 25000; - for (uint32_t i = 0; i < lbidPartitionVector.size(); i++) + for (uint32_t i = 0; i < lbidPartitionVector.size(); i++) + { + mmp = lbidPartitionVector.at(i); + + //@bug4543: only update extentmap CP if blks were scanned for this extent + if ((mmp->isValid == BRM::CP_INVALID) && (mmp->blksScanned > 0)) { + cpInfo.firstLbid = mmp->lbid; + cpInfo.isBinaryColumn = (colType.colWidth == 16); + if (cpInfo.isBinaryColumn) + { + cpInfo.bigMax = mmp->bigMax; + cpInfo.bigMin = mmp->bigMin; + } + else + { + cpInfo.max = mmp->max; + cpInfo.min = mmp->min; + } + cpInfo.seqNum = (int32_t)mmp->seq; + vCpInfo.push_back(cpInfo); - mmp = lbidPartitionVector.at(i); - - //@bug4543: only update extentmap CP if blks were scanned for this extent - if ((mmp->isValid == BRM::CP_INVALID) && (mmp->blksScanned > 0)) - { - cpInfo.firstLbid = mmp->lbid; - cpInfo.isBinaryColumn = (colType.colWidth == 16); - if (cpInfo.isBinaryColumn) - { - cpInfo.bigMax = mmp->bigMax; - cpInfo.bigMin = mmp->bigMin; - } - else - { - cpInfo.max = mmp->max; - cpInfo.min = mmp->min; - } - cpInfo.seqNum = (int32_t)mmp->seq; - vCpInfo.push_back(cpInfo); - - // Limit the number of extents to update at a time. A map is created within the call and this will prevent unbounded - // memory. Probably will never approach this limit but just in case. - if ((i + 1) % cpUpdateInterval == 0 || (i + 1) == lbidPartitionVector.size()) - { - em->setExtentsMaxMin(vCpInfo); - vCpInfo.clear(); - } - -#ifdef DEBUG - - if (IS_VERBOSE) - cout << "LBIDList::UpdateAllPartitionInfo() updated mmp.lbid " << mmp->lbid - << " mmp->max " << mmp->max - << " mmp->min " << mmp->min - << " seq " << mmp->seq - << " blksScanned " << mmp->blksScanned - << endl; - -#endif - mmp->isValid = BRM::CP_VALID; - } - - //delete mmp; - } - - // Send the last batch of CP info to BRM. - if (!vCpInfo.empty()) - { + // Limit the number of extents to update at a time. A map is created within the call and this will + // prevent unbounded memory. Probably will never approach this limit but just in case. + if ((i + 1) % cpUpdateInterval == 0 || (i + 1) == lbidPartitionVector.size()) + { em->setExtentsMaxMin(vCpInfo); + vCpInfo.clear(); + } + +#ifdef DEBUG + + if (IS_VERBOSE) + cout << "LBIDList::UpdateAllPartitionInfo() updated mmp.lbid " << mmp->lbid << " mmp->max " + << mmp->max << " mmp->min " << mmp->min << " seq " << mmp->seq << " blksScanned " + << mmp->blksScanned << endl; + +#endif + mmp->isValid = BRM::CP_VALID; } + + // delete mmp; + } + + // Send the last batch of CP info to BRM. + if (!vCpInfo.empty()) + { + em->setExtentsMaxMin(vCpInfo); + } } bool LBIDList::IsRangeBoundary(uint64_t lbid) { - const int RangeCount = LBIDRanges.size(); - LBIDRange LBIDR; + const int RangeCount = LBIDRanges.size(); + LBIDRange LBIDR; - for (int i = 0; i < RangeCount; i++) - { - LBIDR = LBIDRanges.at(i); + for (int i = 0; i < RangeCount; i++) + { + LBIDR = LBIDRanges.at(i); - if (lbid == (uint64_t)(LBIDR.start)) - return true; - } + if (lbid == (uint64_t)(LBIDR.start)) + return true; + } - return false; + return false; } /* test datatype for casual partitioning predicate optimization @@ -546,37 +520,33 @@ bool LBIDList::IsRangeBoundary(uint64_t lbid) bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType type, const uint8_t size) const { - switch (type) - { - case CalpontSystemCatalog::CHAR: - return size < 9; + switch (type) + { + case CalpontSystemCatalog::CHAR: return size < 9; - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - return size < 8; + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: return size < 8; - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - return true; + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UDECIMAL: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: return true; - default: - return false; - } + default: return false; + } } /* Check for casual partitioning predicate optimization. This function applies the predicate using @@ -586,394 +556,368 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t * returns false if casual partitioning predicate optimization has eliminated the scan. */ -template +template inline bool LBIDList::compareVal(const T& Min, const T& Max, const T& value, char op, uint8_t lcf) { - switch (op) - { - case COMPARE_LT: - case COMPARE_NGE: - if (value <= Min) - { - return false; - } + switch (op) + { + case COMPARE_LT: + case COMPARE_NGE: + if (value <= Min) + { + return false; + } - break; + break; - case COMPARE_LE: - case COMPARE_NGT: - if (value < Min) - { - return false; - } + case COMPARE_LE: + case COMPARE_NGT: + if (value < Min) + { + return false; + } - break; + break; - case COMPARE_GT: - case COMPARE_NLE: - if (value >= Max) - { - return false; - } + case COMPARE_GT: + case COMPARE_NLE: + if (value >= Max) + { + return false; + } - break; + break; - case COMPARE_GE: - case COMPARE_NLT: - if (value > Max) - { - return false; - } + case COMPARE_GE: + case COMPARE_NLT: + if (value > Max) + { + return false; + } - break; + break; - case COMPARE_EQ: - if (value < Min || value > Max || lcf > 0) - { - return false; - } + case COMPARE_EQ: + if (value < Min || value > Max || lcf > 0) + { + return false; + } - break; + break; - case COMPARE_NE: + case COMPARE_NE: - // @bug 3087 - if ( value == Min && value == Max && lcf == 0) - { - return false; - } + // @bug 3087 + if (value == Min && value == Max && lcf == 0) + { + return false; + } - break; - } + break; + } - return true; + return true; } - -static inline bool compareStr(const datatypes::Charset &cs, - const utils::ConstString &Min, - const utils::ConstString &Max, - const utils::ConstString &value, - char op, uint8_t lcf) +static inline bool compareStr(const datatypes::Charset& cs, const utils::ConstString& Min, + const utils::ConstString& Max, const utils::ConstString& value, char op, + uint8_t lcf) { - switch (op) - { - case COMPARE_LT: - case COMPARE_NGE: - return cs.strnncollsp(value, Min) > 0; + switch (op) + { + case COMPARE_LT: + case COMPARE_NGE: return cs.strnncollsp(value, Min) > 0; - case COMPARE_LE: - case COMPARE_NGT: - return cs.strnncollsp(value, Min) >= 0; + case COMPARE_LE: + case COMPARE_NGT: return cs.strnncollsp(value, Min) >= 0; - case COMPARE_GT: - case COMPARE_NLE: - return cs.strnncollsp(value, Max) < 0; + case COMPARE_GT: + case COMPARE_NLE: return cs.strnncollsp(value, Max) < 0; - case COMPARE_GE: - case COMPARE_NLT: - return cs.strnncollsp(value, Max) <= 0; + case COMPARE_GE: + case COMPARE_NLT: return cs.strnncollsp(value, Max) <= 0; - case COMPARE_EQ: - return cs.strnncollsp(value, Min) >= 0 && - cs.strnncollsp(value, Max) <= 0 && - lcf <= 0; + case COMPARE_EQ: return cs.strnncollsp(value, Min) >= 0 && cs.strnncollsp(value, Max) <= 0 && lcf <= 0; - case COMPARE_NE: + case COMPARE_NE: - // @bug 3087 - return cs.strnncollsp(value, Min) != 0 || - cs.strnncollsp(value, Max) != 0 || - lcf != 0; - } + // @bug 3087 + return cs.strnncollsp(value, Min) != 0 || cs.strnncollsp(value, Max) != 0 || lcf != 0; + } - return false; + return false; } - -template -bool LBIDList::checkSingleValue(T min, T max, T value, - const execplan::CalpontSystemCatalog::ColType & type) +template +bool LBIDList::checkSingleValue(T min, T max, T value, const execplan::CalpontSystemCatalog::ColType& type) { - if (isCharType(type.colDataType)) - { - // MCOL-641 LBIDList::CasualPartitionDataType() returns false if - // width > 8 for a character type, so T cannot be int128_t here - datatypes::Charset cs(const_cast(type).getCharset()); - return datatypes::TCharShort::strnncollsp(cs, value, min, type.colWidth) >= 0 && - datatypes::TCharShort::strnncollsp(cs, value, max, type.colWidth) <= 0; - } - else if (isUnsigned(type.colDataType)) - { - return (static_cast(value) >= static_cast(min) && - static_cast(value) <= static_cast(max)); - } - else - { - return (value >= min && value <= max); - } + if (isCharType(type.colDataType)) + { + // MCOL-641 LBIDList::CasualPartitionDataType() returns false if + // width > 8 for a character type, so T cannot be int128_t here + datatypes::Charset cs(const_cast(type).getCharset()); + return datatypes::TCharShort::strnncollsp(cs, value, min, type.colWidth) >= 0 && + datatypes::TCharShort::strnncollsp(cs, value, max, type.colWidth) <= 0; + } + else if (isUnsigned(type.colDataType)) + { + return (static_cast(value) >= static_cast(min) && + static_cast(value) <= static_cast(max)); + } + else + { + return (value >= min && value <= max); + } } -template +template bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, - const execplan::CalpontSystemCatalog::ColType & type) + const execplan::CalpontSystemCatalog::ColType& type) { - if (isCharType(type.colDataType)) - { - // MCOL-641 LBIDList::CasualPartitionDataType() returns false if - // width > 8 for a character type, so T cannot be int128_t here - datatypes::Charset cs(const_cast(type).getCharset()); - return datatypes::TCharShort::strnncollsp(cs, tmin, max, type.colWidth) <= 0 && - datatypes::TCharShort::strnncollsp(cs, tmax, min, type.colWidth) >= 0; - } - else if (isUnsigned(type.colDataType)) - { - return (static_cast(tmin) <= static_cast(max) && - static_cast(tmax) >= static_cast(min)); - } - else - { - return (tmin <= max && tmax >= min); - } + if (isCharType(type.colDataType)) + { + // MCOL-641 LBIDList::CasualPartitionDataType() returns false if + // width > 8 for a character type, so T cannot be int128_t here + datatypes::Charset cs(const_cast(type).getCharset()); + return datatypes::TCharShort::strnncollsp(cs, tmin, max, type.colWidth) <= 0 && + datatypes::TCharShort::strnncollsp(cs, tmax, min, type.colWidth) >= 0; + } + else if (isUnsigned(type.colDataType)) + { + return (static_cast(tmin) <= static_cast(max) && + static_cast(tmax) >= static_cast(min)); + } + else + { + return (tmin <= max && tmax >= min); + } } bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, - const messageqcpp::ByteStream* bs, - const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, - const uint8_t BOP) + const messageqcpp::ByteStream* bs, const uint16_t NOPS, + const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP) { + int length = bs->length(), pos = 0; + const char* MsgDataPtr = (const char*)bs->buf(); + bool scan = true; + int64_t value = 0; + int128_t bigValue = 0; + bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); + bool bIsChar = datatypes::isCharType(ct.colDataType); - int length = bs->length(), pos = 0; - const char* MsgDataPtr = (const char*) bs->buf(); - bool scan = true; - int64_t value = 0; - int128_t bigValue = 0; - bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); - bool bIsChar = datatypes::isCharType(ct.colDataType); + for (int i = 0; i < NOPS; i++) + { + scan = true; + pos += ct.colWidth + 2; // predicate + op + lcf - for (int i = 0; i < NOPS; i++) + if (pos > length) { - scan = true; - pos += ct.colWidth + 2; // predicate + op + lcf - - if (pos > length) - { #ifdef DEBUG - cout << "CasualPartitionPredicate: Filter parsing went beyond the end of the filter string!" << endl; + cout << "CasualPartitionPredicate: Filter parsing went beyond the end of the filter string!" << endl; #endif - return true; - } + return true; + } - char op = *MsgDataPtr++; - uint8_t lcf = *(uint8_t*)MsgDataPtr++; + char op = *MsgDataPtr++; + uint8_t lcf = *(uint8_t*)MsgDataPtr++; - if (bIsUnsigned) + if (bIsUnsigned) + { + switch (ct.colWidth) + { + case 1: { - switch (ct.colWidth) - { - case 1: - { - uint8_t val = *(int8_t*)MsgDataPtr; - value = val; - break; - } - - case 2: - { - uint16_t val = *(int16_t*)MsgDataPtr; - value = val; - break; - } - - case 4: - { - uint32_t val = *(int32_t*)MsgDataPtr; - value = val; - break; - } - - case 8: - { - uint64_t val = *(int64_t*)MsgDataPtr; - value = static_cast(val); - break; - } - } + uint8_t val = *(int8_t*)MsgDataPtr; + value = val; + break; } - else + + case 2: { - switch (ct.colWidth) - { - case 1: - { - int8_t val = *(int8_t*)MsgDataPtr; - value = val; - break; - } - - case 2: - { - int16_t val = *(int16_t*)MsgDataPtr; - value = val; - break; - } - - case 4: - { - int32_t val = *(int32_t*)MsgDataPtr; - value = val; - break; - } - - case 8: - { - int64_t val = *(int64_t*)MsgDataPtr; - value = val; - break; - } - - case 16: - { - datatypes::TSInt128::assignPtrPtr(&bigValue, MsgDataPtr); - break; - } - } + uint16_t val = *(int16_t*)MsgDataPtr; + value = val; + break; } - MsgDataPtr += ct.colWidth; - - if (ct.isWideDecimalType() && execplan::isNull(bigValue, ct)) + case 4: { - continue; - } - else if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. - { - continue; + uint32_t val = *(int32_t*)MsgDataPtr; + value = val; + break; } - if (bIsChar) + case 8: { - datatypes::Charset cs(ct.charsetNumber); - utils::ConstString sMin((const char *) &cpRange.loVal, ct.colWidth); - utils::ConstString sMax((const char *) &cpRange.hiVal, ct.colWidth); - utils::ConstString sVal((const char *) &value, ct.colWidth); - scan = compareStr(cs, sMin.rtrimZero(), - sMax.rtrimZero(), - sVal.rtrimZero(), op, lcf); -// cout << "scan=" << (uint32_t) scan << endl; + uint64_t val = *(int64_t*)MsgDataPtr; + value = static_cast(val); + break; } - else if (bIsUnsigned) + } + } + else + { + switch (ct.colWidth) + { + case 1: { - scan = compareVal(static_cast(cpRange.loVal), static_cast(cpRange.hiVal), static_cast(value), op, lcf); - } - else - { - if (ct.colWidth != datatypes::MAXDECIMALWIDTH) - { - scan = compareVal(cpRange.loVal, cpRange.hiVal, value, op, lcf); - } - else - { - scan = compareVal(cpRange.bigLoVal, cpRange.bigHiVal, bigValue, op, lcf); - } + int8_t val = *(int8_t*)MsgDataPtr; + value = val; + break; } - if (BOP == BOP_AND && !scan) + case 2: { - break; + int16_t val = *(int16_t*)MsgDataPtr; + value = val; + break; } - if (BOP == BOP_OR && scan) + case 4: { - break; + int32_t val = *(int32_t*)MsgDataPtr; + value = val; + break; } - //TODO: What about BOP_NONE? + case 8: + { + int64_t val = *(int64_t*)MsgDataPtr; + value = val; + break; + } - } // for() + case 16: + { + datatypes::TSInt128::assignPtrPtr(&bigValue, MsgDataPtr); + break; + } + } + } + + MsgDataPtr += ct.colWidth; + + if (ct.isWideDecimalType() && execplan::isNull(bigValue, ct)) + { + continue; + } + else if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. + { + continue; + } + + if (bIsChar) + { + datatypes::Charset cs(ct.charsetNumber); + utils::ConstString sMin((const char*)&cpRange.loVal, ct.colWidth); + utils::ConstString sMax((const char*)&cpRange.hiVal, ct.colWidth); + utils::ConstString sVal((const char*)&value, ct.colWidth); + scan = compareStr(cs, sMin.rtrimZero(), sMax.rtrimZero(), sVal.rtrimZero(), op, lcf); + // cout << "scan=" << (uint32_t) scan << endl; + } + else if (bIsUnsigned) + { + scan = compareVal(static_cast(cpRange.loVal), static_cast(cpRange.hiVal), + static_cast(value), op, lcf); + } + else + { + if (ct.colWidth != datatypes::MAXDECIMALWIDTH) + { + scan = compareVal(cpRange.loVal, cpRange.hiVal, value, op, lcf); + } + else + { + scan = compareVal(cpRange.bigLoVal, cpRange.bigHiVal, bigValue, op, lcf); + } + } + + if (BOP == BOP_AND && !scan) + { + break; + } + + if (BOP == BOP_OR && scan) + { + break; + } + + // TODO: What about BOP_NONE? + + } // for() #ifdef DEBUG - if (IS_VERBOSE) - cout << "CPPredicate " << (scan == true ? "TRUE" : "FALSE") << endl; + if (IS_VERBOSE) + cout << "CPPredicate " << (scan == true ? "TRUE" : "FALSE") << endl; #endif - return scan; -} // CasualPartitioningPredicate + return scan; +} // CasualPartitioningPredicate void LBIDList::copyLbidList(const LBIDList& rhs) { - em = rhs.em; - vector::value_type ptr; + em = rhs.em; + vector::value_type ptr; - while (!lbidPartitionVector.empty()) - { - ptr = lbidPartitionVector.back(); - lbidPartitionVector.pop_back(); - delete ptr; - } + while (!lbidPartitionVector.empty()) + { + ptr = lbidPartitionVector.back(); + lbidPartitionVector.pop_back(); + delete ptr; + } - lbidPartitionVector.clear(); //Overkill... - vector::const_iterator iter = rhs.lbidPartitionVector.begin(); - vector::const_iterator end = rhs.lbidPartitionVector.end(); + lbidPartitionVector.clear(); // Overkill... + vector::const_iterator iter = rhs.lbidPartitionVector.begin(); + vector::const_iterator end = rhs.lbidPartitionVector.end(); - while (iter != end) - { - MinMaxPartition* mmp = new MinMaxPartition(); - *mmp = **iter; - lbidPartitionVector.push_back(mmp); - ++iter; - } + while (iter != end) + { + MinMaxPartition* mmp = new MinMaxPartition(); + *mmp = **iter; + lbidPartitionVector.push_back(mmp); + ++iter; + } - LBIDRanges = rhs.LBIDRanges; - fDebug = rhs.fDebug; + LBIDRanges = rhs.LBIDRanges; + fDebug = rhs.fDebug; } -template -bool LBIDList::GetMinMax(int128_t& min, int128_t& max, int64_t& seq, int64_t lbid, - const std::vector* pEMEntries, - execplan::CalpontSystemCatalog::ColDataType colDataType); -template -bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, - const std::vector* pEMEntries, - execplan::CalpontSystemCatalog::ColDataType colDataType); +template bool LBIDList::GetMinMax(int128_t& min, int128_t& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType colDataType); +template bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType colDataType); -template -bool LBIDList::GetMinMax(int128_t* min, int128_t* max, int64_t* seq, - int64_t lbid, const tr1::unordered_map& entries, - execplan::CalpontSystemCatalog::ColDataType colDataType); +template bool LBIDList::GetMinMax(int128_t* min, int128_t* max, int64_t* seq, int64_t lbid, + const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType); -template -bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, - int64_t lbid, const tr1::unordered_map& entries, - execplan::CalpontSystemCatalog::ColDataType colDataType); +template bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, int64_t lbid, + const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType); -template -void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, - const execplan::CalpontSystemCatalog::ColType & type, bool validData = true); +template void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, + const execplan::CalpontSystemCatalog::ColType& type, + bool validData = true); -template -void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, - const execplan::CalpontSystemCatalog::ColType & type, bool validData = true); +template void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, + const execplan::CalpontSystemCatalog::ColType& type, + bool validData = true); -template -bool LBIDList::checkSingleValue(int128_t min, int128_t max, int128_t value, - const execplan::CalpontSystemCatalog::ColType & type); +template bool LBIDList::checkSingleValue(int128_t min, int128_t max, int128_t value, + const execplan::CalpontSystemCatalog::ColType& type); -template -bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, - const execplan::CalpontSystemCatalog::ColType & type); +template bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, + const execplan::CalpontSystemCatalog::ColType& type); -template -bool LBIDList::checkRangeOverlap(int128_t min, int128_t max, int128_t tmin, int128_t tmax, - const execplan::CalpontSystemCatalog::ColType & type); +template bool LBIDList::checkRangeOverlap(int128_t min, int128_t max, int128_t tmin, int128_t tmax, + const execplan::CalpontSystemCatalog::ColType& type); -template -bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, - const execplan::CalpontSystemCatalog::ColType & type); +template bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, + const execplan::CalpontSystemCatalog::ColType& type); -} //namespace joblist +} // namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 5009d5039..ef89b5386 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: lbidlist.h 9655 2013-06-25 23:08:13Z xlou $ -* -* -***********************************************************************/ + * $Id: lbidlist.h 9655 2013-06-25 23:08:13Z xlou $ + * + * + ***********************************************************************/ /** @file */ #ifndef JOBLIST_LBIDLIST_H @@ -40,7 +40,6 @@ namespace joblist { - typedef BRM::LBIDRange_v LBIDRangeVector; /** @brief struct MinMaxPartition @@ -48,21 +47,21 @@ typedef BRM::LBIDRange_v LBIDRangeVector; */ struct MinMaxPartition { - int64_t lbid; - int64_t lbidmax; - int64_t seq; - int isValid; - uint32_t blksScanned; - union - { - int128_t bigMin; - int64_t min; - }; - union - { - int128_t bigMax; - int64_t max; - }; + int64_t lbid; + int64_t lbidmax; + int64_t seq; + int isValid; + uint32_t blksScanned; + union + { + int128_t bigMin; + int64_t min; + }; + union + { + int128_t bigMax; + int64_t max; + }; }; /** @brief class LBIDList @@ -70,94 +69,87 @@ struct MinMaxPartition */ class LBIDList { -public: + public: + explicit LBIDList(const execplan::CalpontSystemCatalog::OID oid, const int debug); - explicit LBIDList(const execplan::CalpontSystemCatalog::OID oid, - const int debug); + explicit LBIDList(const int debug); - explicit LBIDList(const int debug); + void init(const execplan::CalpontSystemCatalog::OID oid, const int debug); - void init(const execplan::CalpontSystemCatalog::OID oid, - const int debug); + virtual ~LBIDList(); - virtual ~LBIDList(); + void Dump(long Index, int Count) const; + uint32_t GetRangeSize() const + { + return LBIDRanges.size() ? LBIDRanges.at(0).size : 0; + } - void Dump(long Index, int Count) const; - uint32_t GetRangeSize() const - { - return LBIDRanges.size() ? LBIDRanges.at(0).size : 0; - } + // Functions to handle min/max values per lbid for casual partitioning; + // If pEMEntries is provided, then min/max will be extracted from that + // vector, else extents in BRM will be searched. If type is unsigned, caller + // should static cast returned min and max to uint64_t/int128_t + template + bool GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType type); - // Functions to handle min/max values per lbid for casual partitioning; - // If pEMEntries is provided, then min/max will be extracted from that - // vector, else extents in BRM will be searched. If type is unsigned, caller - // should static cast returned min and max to uint64_t/int128_t - template - bool GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, - const std::vector* pEMEntries, - execplan::CalpontSystemCatalog::ColDataType type); + template + bool GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, + const std::tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType type); - template - bool GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, - const std::tr1::unordered_map& entries, - execplan::CalpontSystemCatalog::ColDataType type); + template + void UpdateMinMax(T min, T max, int64_t lbid, const execplan::CalpontSystemCatalog::ColType& type, + bool validData = true); - template - void UpdateMinMax(T min, T max, int64_t lbid, - const execplan::CalpontSystemCatalog::ColType & type, bool validData = true); + void UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType); - void UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType); + bool IsRangeBoundary(uint64_t lbid); - bool IsRangeBoundary(uint64_t lbid); + bool CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, + const messageqcpp::ByteStream* MsgDataPtr, const uint16_t NOPS, + const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP); - bool CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, - const messageqcpp::ByteStream* MsgDataPtr, - const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, - const uint8_t BOP); + template + bool checkSingleValue(T min, T max, T value, const execplan::CalpontSystemCatalog::ColType& type); - template - bool checkSingleValue(T min, T max, T value, - const execplan::CalpontSystemCatalog::ColType & type); + template + bool checkRangeOverlap(T min, T max, T tmin, T tmax, const execplan::CalpontSystemCatalog::ColType& type); - template - bool checkRangeOverlap(T min, T max, T tmin, T tmax, - const execplan::CalpontSystemCatalog::ColType & type); + // check the column data type and the column size to determine if it + // is a data type to apply casual paritioning. + bool CasualPartitionDataType(const execplan::CalpontSystemCatalog::ColDataType type, + const uint8_t size) const; - // check the column data type and the column size to determine if it - // is a data type to apply casual paritioning. - bool CasualPartitionDataType(const execplan::CalpontSystemCatalog::ColDataType type, const uint8_t size) const; + LBIDList(const LBIDList& rhs) + { + copyLbidList(rhs); + } - LBIDList(const LBIDList& rhs) - { - copyLbidList(rhs); - } + LBIDList& operator=(const LBIDList& rhs) + { + copyLbidList(rhs); + return *this; + } - LBIDList& operator=(const LBIDList& rhs) - { - copyLbidList(rhs); - return *this; - } + private: + LBIDList(); -private: - LBIDList(); + void copyLbidList(const LBIDList& rhs); - void copyLbidList(const LBIDList& rhs); + template + inline bool compareVal(const T& Min, const T& Max, const T& value, char op, uint8_t lcf); - template - inline bool compareVal(const T& Min, const T& Max, const T& value, char op, uint8_t lcf); + template + int getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, + const std::vector& EMEntries); - template - int getMinMaxFromEntries(T& min, T& max, int32_t& seq, - int64_t lbid, const std::vector& EMEntries); + boost::shared_ptr em; + std::vector lbidPartitionVector; + LBIDRangeVector LBIDRanges; + int fDebug; - boost::shared_ptr em; - std::vector lbidPartitionVector; - LBIDRangeVector LBIDRanges; - int fDebug; +}; // LBIDList -}; // LBIDList - - -} // joblist +} // namespace joblist #endif diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index ab3c1615b..3aee6fc02 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -17,7 +17,6 @@ // $Id: limitedorderby.cpp 9581 2013-05-31 13:46:14Z pleblanc $ - #include //#define NDEBUG #include @@ -41,280 +40,267 @@ using namespace ordering; namespace joblist { - -const uint64_t LimitedOrderBy::fMaxUncommited = 102400; // 100KiB - make it configurable? +const uint64_t LimitedOrderBy::fMaxUncommited = 102400; // 100KiB - make it configurable? // LimitedOrderBy class implementation LimitedOrderBy::LimitedOrderBy() : fStart(0), fCount(-1), fUncommitedMemory(0) { - fRule.fIdbCompare = this; + fRule.fIdbCompare = this; } - LimitedOrderBy::~LimitedOrderBy() { } - -void LimitedOrderBy::initialize(const RowGroup& rg, const JobInfo& jobInfo, bool invertRules, bool isMultiThreaded) +void LimitedOrderBy::initialize(const RowGroup& rg, const JobInfo& jobInfo, bool invertRules, + bool isMultiThreaded) { - fRm = jobInfo.rm; - fSessionMemLimit = jobInfo.umMemLimit; - fErrorCode = ERR_LIMIT_TOO_BIG; + fRm = jobInfo.rm; + fSessionMemLimit = jobInfo.umMemLimit; + fErrorCode = ERR_LIMIT_TOO_BIG; - // locate column position in the rowgroup - map keyToIndexMap; + // locate column position in the rowgroup + map keyToIndexMap; - for (uint64_t i = 0; i < rg.getKeys().size(); ++i) - { - if (keyToIndexMap.find(rg.getKeys()[i]) == keyToIndexMap.end()) - keyToIndexMap.insert(make_pair(rg.getKeys()[i], i)); - } + for (uint64_t i = 0; i < rg.getKeys().size(); ++i) + { + if (keyToIndexMap.find(rg.getKeys()[i]) == keyToIndexMap.end()) + keyToIndexMap.insert(make_pair(rg.getKeys()[i], i)); + } - vector >::const_iterator i = jobInfo.orderByColVec.begin(); + vector >::const_iterator i = jobInfo.orderByColVec.begin(); - for ( ; i != jobInfo.orderByColVec.end(); i++) - { - map::iterator j = keyToIndexMap.find(i->first); - idbassert(j != keyToIndexMap.end()); + for (; i != jobInfo.orderByColVec.end(); i++) + { + map::iterator j = keyToIndexMap.find(i->first); + idbassert(j != keyToIndexMap.end()); - fOrderByCond.push_back(IdbSortSpec(j->second, i->second ^ invertRules)); - } + fOrderByCond.push_back(IdbSortSpec(j->second, i->second ^ invertRules)); + } - // limit row count info - if (isMultiThreaded) - { - // CS can't apply offset at the first stage - // otherwise it looses records. - fStart = 0; - fCount = jobInfo.limitStart+jobInfo.limitCount; - } - else - { - fStart = jobInfo.limitStart; - fCount = jobInfo.limitCount; - } + // limit row count info + if (isMultiThreaded) + { + // CS can't apply offset at the first stage + // otherwise it looses records. + fStart = 0; + fCount = jobInfo.limitStart + jobInfo.limitCount; + } + else + { + fStart = jobInfo.limitStart; + fCount = jobInfo.limitCount; + } - IdbOrderBy::initialize(rg); + IdbOrderBy::initialize(rg); } // This must return a proper number of key columns and // not just a column count. uint64_t LimitedOrderBy::getKeyLength() const { - return fRow0.getColumnCount(); + return fRow0.getColumnCount(); } - void LimitedOrderBy::processRow(const rowgroup::Row& row) { - // check if this is a distinct row - if (fDistinct && fDistinctMap->find(row.getPointer()) != fDistinctMap->end()) - return; + // check if this is a distinct row + if (fDistinct && fDistinctMap->find(row.getPointer()) != fDistinctMap->end()) + return; - // @bug5312, limit count is 0, do nothing. - if (fCount == 0) - return; + // @bug5312, limit count is 0, do nothing. + if (fCount == 0) + return; - // if the row count is less than the limit - if (fOrderByQueue.size() < fStart + fCount) + // if the row count is less than the limit + if (fOrderByQueue.size() < fStart + fCount) + { + copyRow(row, &fRow0); + OrderByRow newRow(fRow0, fRule); + fOrderByQueue.push(newRow); + + uint64_t memSizeInc = sizeof(newRow); + fUncommitedMemory += memSizeInc; + if (fUncommitedMemory >= fMaxUncommited) { - copyRow(row, &fRow0); - OrderByRow newRow(fRow0, fRule); - fOrderByQueue.push(newRow); - - uint64_t memSizeInc = sizeof(newRow); - fUncommitedMemory += memSizeInc; - if (fUncommitedMemory >= fMaxUncommited) - { - if (!fRm->getMemory(fUncommitedMemory, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" - << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += fUncommitedMemory; - fUncommitedMemory = 0; - } - - // add to the distinct map - if (fDistinct) - fDistinctMap->insert(fRow0.getPointer()); - - fRowGroup.incRowCount(); - fRow0.nextRow(); - - if (fRowGroup.getRowCount() >= fRowsPerRG) - { - fDataQueue.push(fData); - uint64_t newSize = fRowGroup.getSizeWithStrings() - fRowGroup.getHeaderSize(); - - if (!fRm->getMemory(newSize, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += newSize; - - fData.reinit(fRowGroup, fRowsPerRG); - fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); - fRowGroup.getRow(0, &fRow0); - } + if (!fRm->getMemory(fUncommitedMemory, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += fUncommitedMemory; + fUncommitedMemory = 0; } - else if (fOrderByCond.size() > 0 && fRule.less(row.getPointer(), fOrderByQueue.top().fData)) + // add to the distinct map + if (fDistinct) + fDistinctMap->insert(fRow0.getPointer()); + + fRowGroup.incRowCount(); + fRow0.nextRow(); + + if (fRowGroup.getRowCount() >= fRowsPerRG) { - OrderByRow swapRow = fOrderByQueue.top(); - row1.setData(swapRow.fData); - copyRow(row, &row1); + fDataQueue.push(fData); + uint64_t newSize = fRowGroup.getSizeWithStrings() - fRowGroup.getHeaderSize(); - if (fDistinct) - { - fDistinctMap->erase(fOrderByQueue.top().fData); - fDistinctMap->insert(row1.getPointer()); - } + if (!fRm->getMemory(newSize, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += newSize; - fOrderByQueue.pop(); - fOrderByQueue.push(swapRow); + fData.reinit(fRowGroup, fRowsPerRG); + fRowGroup.setData(&fData); + fRowGroup.resetRowGroup(0); + fRowGroup.getRow(0, &fRow0); } + } + + else if (fOrderByCond.size() > 0 && fRule.less(row.getPointer(), fOrderByQueue.top().fData)) + { + OrderByRow swapRow = fOrderByQueue.top(); + row1.setData(swapRow.fData); + copyRow(row, &row1); + + if (fDistinct) + { + fDistinctMap->erase(fOrderByQueue.top().fData); + fDistinctMap->insert(row1.getPointer()); + } + + fOrderByQueue.pop(); + fOrderByQueue.push(swapRow); + } } /* - * The f() copies top element from an ordered queue into a row group. It + * The f() copies top element from an ordered queue into a row group. It * does this backwards to syncronise sorting orientation with the server. * The top row from the queue goes last into the returned set. - */ + */ void LimitedOrderBy::finalize() { - if (fUncommitedMemory > 0) + if (fUncommitedMemory > 0) + { + if (!fRm->getMemory(fUncommitedMemory, fSessionMemLimit)) { - if (!fRm->getMemory(fUncommitedMemory, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" - << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += fUncommitedMemory; - fUncommitedMemory = 0; + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += fUncommitedMemory; + fUncommitedMemory = 0; + } + + queue tempQueue; + if (fRowGroup.getRowCount() > 0) + fDataQueue.push(fData); + + if (fOrderByQueue.size() > 0) + { + // *DRRTUY Very memory intensive. CS needs to account active + // memory only and release memory if needed. + uint64_t memSizeInc = fRowGroup.getSizeWithStrings() - fRowGroup.getHeaderSize(); + + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += memSizeInc; + + uint64_t offset = 0; + uint64_t i = 0; + // Reduce queue size by an offset value if it applicable. + uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ? fOrderByQueue.size() - fStart : 0; + list tempRGDataList; + + if (fCount <= queueSizeWoOffset) + { + offset = fCount % fRowsPerRG; + if (!offset && fCount > 0) + offset = fRowsPerRG; + } + else + { + offset = queueSizeWoOffset % fRowsPerRG; + if (!offset && queueSizeWoOffset > 0) + offset = fRowsPerRG; } - queue tempQueue; - if (fRowGroup.getRowCount() > 0) - fDataQueue.push(fData); + list::iterator tempListIter = tempRGDataList.begin(); - if (fOrderByQueue.size() > 0) + i = 0; + uint32_t rSize = fRow0.getSize(); + uint64_t preLastRowNumb = fRowsPerRG - 1; + fData.reinit(fRowGroup, fRowsPerRG); + fRowGroup.setData(&fData); + fRowGroup.resetRowGroup(0); + // *DRRTUY This approach won't work with + // OFSET > fRowsPerRG + offset = offset != 0 ? offset - 1 : offset; + fRowGroup.getRow(offset, &fRow0); + + while ((fOrderByQueue.size() > fStart) && (i++ < fCount)) { - // *DRRTUY Very memory intensive. CS needs to account active - // memory only and release memory if needed. - uint64_t memSizeInc = fRowGroup.getSizeWithStrings() - fRowGroup.getHeaderSize(); + const OrderByRow& topRow = fOrderByQueue.top(); + row1.setData(topRow.fData); + copyRow(row1, &fRow0); + fRowGroup.incRowCount(); + offset--; + fRow0.prevRow(rSize); + fOrderByQueue.pop(); + + // if RG has fRowsPerRG rows + if (offset == (uint64_t)-1) + { + tempRGDataList.push_front(fData); if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); } fMemSize += memSizeInc; - - uint64_t offset = 0; - uint64_t i = 0; - // Reduce queue size by an offset value if it applicable. - uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ? - fOrderByQueue.size() - fStart : 0; - list tempRGDataList; - if ( fCount <= queueSizeWoOffset ) - { - offset = fCount % fRowsPerRG; - if(!offset && fCount > 0) - offset = fRowsPerRG; - } - else - { - offset = queueSizeWoOffset % fRowsPerRG; - if(!offset && queueSizeWoOffset > 0) - offset = fRowsPerRG; - } - - list::iterator tempListIter = tempRGDataList.begin(); - - i = 0; - uint32_t rSize = fRow0.getSize(); - uint64_t preLastRowNumb = fRowsPerRG - 1; fData.reinit(fRowGroup, fRowsPerRG); fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); - // *DRRTUY This approach won't work with - // OFSET > fRowsPerRG - offset = offset != 0 ? offset - 1 : offset; - fRowGroup.getRow(offset, &fRow0); - - while ((fOrderByQueue.size() > fStart) && (i++ < fCount)) - { - const OrderByRow& topRow = fOrderByQueue.top(); - row1.setData(topRow.fData); - copyRow(row1, &fRow0); - fRowGroup.incRowCount(); - offset--; - fRow0.prevRow(rSize); - fOrderByQueue.pop(); - - // if RG has fRowsPerRG rows - if(offset == (uint64_t)-1) - { - tempRGDataList.push_front(fData); - - if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += memSizeInc; - - fData.reinit(fRowGroup, fRowsPerRG); - fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); // ? - fRowGroup.getRow(preLastRowNumb, &fRow0); - offset = preLastRowNumb; - } - } - // Push the last/only group into the queue. - if (fRowGroup.getRowCount() > 0) - tempRGDataList.push_front(fData); - - for(tempListIter = tempRGDataList.begin(); tempListIter != tempRGDataList.end(); tempListIter++) - tempQueue.push(*tempListIter); - - fDataQueue = tempQueue; + fRowGroup.resetRowGroup(0); // ? + fRowGroup.getRow(preLastRowNumb, &fRow0); + offset = preLastRowNumb; + } } -} + // Push the last/only group into the queue. + if (fRowGroup.getRowCount() > 0) + tempRGDataList.push_front(fData); + for (tempListIter = tempRGDataList.begin(); tempListIter != tempRGDataList.end(); tempListIter++) + tempQueue.push(*tempListIter); + + fDataQueue = tempQueue; + } +} const string LimitedOrderBy::toString() const { - ostringstream oss; - oss << "OrderBy cols: "; - vector::const_iterator i = fOrderByCond.begin(); + ostringstream oss; + oss << "OrderBy cols: "; + vector::const_iterator i = fOrderByCond.begin(); - for (; i != fOrderByCond.end(); i++) - oss << "(" << i->fIndex << "," - << ((i->fAsc) ? "Asc" : "Desc") << "," - << ((i->fNf) ? "null first" : "null last") << ") "; + for (; i != fOrderByCond.end(); i++) + oss << "(" << i->fIndex << "," << ((i->fAsc) ? "Asc" : "Desc") << "," + << ((i->fNf) ? "null first" : "null last") << ") "; - oss << " start-" << fStart << " count-" << fCount; + oss << " start-" << fStart << " count-" << fCount; - if (fDistinct) - oss << " distinct"; + if (fDistinct) + oss << " distinct"; - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/limitedorderby.h b/dbcon/joblist/limitedorderby.h index d54acf56e..decc2ed5b 100644 --- a/dbcon/joblist/limitedorderby.h +++ b/dbcon/joblist/limitedorderby.h @@ -17,7 +17,6 @@ // $Id: limitedorderby.h 9414 2013-04-22 22:18:30Z xlou $ - /** @file */ #ifndef LIMITED_ORDER_BY_H @@ -27,48 +26,40 @@ #include "rowgroup.h" #include "../../utils/windowfunction/idborderby.h" - namespace joblist { - - // forward reference struct JobInfo; - // ORDER BY with LIMIT class // This version is for subqueries, limit the result set to fit in memory, // use ORDER BY to make the results consistent. // The actual output are the first or last # of rows, which are NOT ordered. class LimitedOrderBy : public ordering::IdbOrderBy { -public: - LimitedOrderBy(); - virtual ~LimitedOrderBy(); - using ordering::IdbOrderBy::initialize; - void initialize(const rowgroup::RowGroup&, - const JobInfo&, - bool invertRules = false, - bool isMultiThreded = false); - void processRow(const rowgroup::Row&); - uint64_t getKeyLength() const; - uint64_t getLimitCount() const - { - return fCount; - } - const std::string toString() const; + public: + LimitedOrderBy(); + virtual ~LimitedOrderBy(); + using ordering::IdbOrderBy::initialize; + void initialize(const rowgroup::RowGroup&, const JobInfo&, bool invertRules = false, + bool isMultiThreded = false); + void processRow(const rowgroup::Row&); + uint64_t getKeyLength() const; + uint64_t getLimitCount() const + { + return fCount; + } + const std::string toString() const; - void finalize(); + void finalize(); -protected: - uint64_t fStart; - uint64_t fCount; - uint64_t fUncommitedMemory; - static const uint64_t fMaxUncommited; + protected: + uint64_t fStart; + uint64_t fCount; + uint64_t fUncommitedMemory; + static const uint64_t fMaxUncommited; }; - -} +} // namespace joblist #endif // LIMITED_ORDER_BY_H - diff --git a/dbcon/joblist/orderby-tests.cpp b/dbcon/joblist/orderby-tests.cpp index 792f65596..1342937f7 100644 --- a/dbcon/joblist/orderby-tests.cpp +++ b/dbcon/joblist/orderby-tests.cpp @@ -48,346 +48,341 @@ using namespace messageqcpp; // Timer class used by this tdriver to output elapsed times, etc. class Timer { -public: - void start(const string& message) + public: + void start(const string& message) + { + if (!fHeaderDisplayed) { - if (!fHeaderDisplayed) - { - header(); - fHeaderDisplayed = true; - } - - gettimeofday(&fTvStart, 0); - cout << timestr() << " Start " << message << endl; + header(); + fHeaderDisplayed = true; } - void stop(const string& message) - { - time_t now; - time(&now); - string secondsElapsed; - getTimeElapsed(secondsElapsed); - cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; - } + gettimeofday(&fTvStart, 0); + cout << timestr() << " Start " << message << endl; + } - Timer() : fHeaderDisplayed(false) {} + void stop(const string& message) + { + time_t now; + time(&now); + string secondsElapsed; + getTimeElapsed(secondsElapsed); + cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; + } -private: + Timer() : fHeaderDisplayed(false) + { + } - struct timeval fTvStart; - bool fHeaderDisplayed; + private: + struct timeval fTvStart; + bool fHeaderDisplayed; - double getTimeElapsed(string& seconds) - { - struct timeval tvStop; - gettimeofday(&tvStop, 0); - double secondsElapsed = - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); - ostringstream oss; - oss << secondsElapsed; - seconds = oss.str(); - seconds.resize(8, '0'); - return secondsElapsed; - } + double getTimeElapsed(string& seconds) + { + struct timeval tvStop; + gettimeofday(&tvStop, 0); + double secondsElapsed = + (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); + ostringstream oss; + oss << secondsElapsed; + seconds = oss.str(); + seconds.resize(8, '0'); + return secondsElapsed; + } - string timestr() - { - struct tm tm; - struct timeval tv; + string timestr() + { + struct tm tm; + struct timeval tv; - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec << '.' - << setw(6) << tv.tv_usec - ; - return oss.str(); - } + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec + << '.' << setw(6) << tv.tv_usec; + return oss.str(); + } - void header() - { - cout << endl; - cout << "Time Seconds Activity" << endl; - } + void header() + { + cout << endl; + cout << "Time Seconds Activity" << endl; + } }; class FilterDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(FilterDriver); - CPPUNIT_TEST_SUITE(FilterDriver); + CPPUNIT_TEST(ORDERBY_TIME_TEST); - CPPUNIT_TEST(ORDERBY_TIME_TEST); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + void orderByTest_nRGs(uint64_t numRows, uint64_t limit, uint64_t maxThreads, bool parallelExecution, + bool generateRandValues, bool hasDistinct) + { + Timer timer; + // This test creates TAS for single bigint column and run sorting on it. -private: - void orderByTest_nRGs(uint64_t numRows, uint64_t limit, - uint64_t maxThreads, - bool parallelExecution, - bool generateRandValues, - bool hasDistinct) + cout << endl; + cout << "------------------------------------------------------------" << endl; + timer.start("insert"); + + stringstream ss2; + ss2.flush(); + ss2 << "loading " << numRows << " rows into initial RowGroup."; + std::string message = ss2.str(); + timer.start(message); + + ResourceManager* rm = ResourceManager::instance(true); + + joblist::JobInfo jobInfo = joblist::JobInfo(rm); + // 1st column is the sorting key + uint8_t tupleKey = 1; + uint8_t offset = 0; + uint16_t rowsPerRG = 8192; // hardcoded max rows in RowGroup value + uint32_t numberOfRGs = numRows / rowsPerRG; + + // set sorting rules + // true - ascending order, false otherwise + jobInfo.orderByColVec.push_back(make_pair(tupleKey, false)); + jobInfo.limitStart = offset; + jobInfo.limitCount = limit; + jobInfo.hasDistinct = hasDistinct; + // JobInfo doesn't set these SP in ctor + jobInfo.umMemLimit.reset(new int64_t); + *(jobInfo.umMemLimit) = MEMORY_LIMIT; + SErrorInfo errorInfo(new ErrorInfo()); + jobInfo.errorInfo = errorInfo; + uint32_t oid = 3001; + // populate JobInfo.nonConstDelCols with dummy shared_pointers + // to notify TupleAnnexStep::initialise() about number of non-constant columns + execplan::SRCP srcp1, srcp2; + jobInfo.nonConstDelCols.push_back(srcp1); + jobInfo.nonConstDelCols.push_back(srcp2); + + // create two columns RG. 1st is the sorting key, second is the data column + std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector charSetNumVec; + std::vector types; + offsets.push_back(2); + offsets.push_back(10); + offsets.push_back(18); + roids.push_back(oid); + roids.push_back(oid); + tkeys.push_back(1); + tkeys.push_back(1); + types.push_back(execplan::CalpontSystemCatalog::UBIGINT); + types.push_back(execplan::CalpontSystemCatalog::UBIGINT); + cscale.push_back(0); + cscale.push_back(0); + cprecision.push_back(20); + cprecision.push_back(20); + charSetNumVec.push_back(8); + charSetNumVec.push_back(8); + rowgroup::RowGroup inRG(2, // column count + offsets, // oldOffset + roids, // column oids + tkeys, // keys + types, // types + charSetNumVec, // charset numbers + cscale, // scale + cprecision, // precision + 20, // sTableThreshold + false // useStringTable + ); + rowgroup::RowGroup jobInfoRG(inRG); + joblist::TupleAnnexStep tns = joblist::TupleAnnexStep(jobInfo); + tns.addOrderBy(new joblist::LimitedOrderBy()); + tns.delivery(true); + // activate parallel sort + if (parallelExecution) { - Timer timer; - // This test creates TAS for single bigint column and run sorting on it. + tns.setParallelOp(); + } + tns.setMaxThreads(maxThreads); + tns.initialize(jobInfoRG, jobInfo); + tns.setLimit(0, limit); - cout << endl; - cout << "------------------------------------------------------------" << endl; - timer.start("insert"); + // Create JobStepAssociation mediator class ins to connect DL and JS + joblist::AnyDataListSPtr spdlIn(new AnyDataList()); + // joblist::RowGroupDL* dlIn = new RowGroupDL(maxThreads, jobInfo.fifoSize); + joblist::RowGroupDL* dlIn = new RowGroupDL(maxThreads, 21001); + dlIn->OID(oid); + spdlIn->rowGroupDL(dlIn); + joblist::JobStepAssociation jsaIn; + jsaIn.outAdd(spdlIn); + tns.inputAssociation(jsaIn); - stringstream ss2; - ss2.flush(); - ss2 << "loading " << numRows << " rows into initial RowGroup."; - std::string message = ss2.str(); - timer.start(message); - - ResourceManager* rm = ResourceManager::instance(true); - - joblist::JobInfo jobInfo = joblist::JobInfo(rm); - // 1st column is the sorting key - uint8_t tupleKey = 1; - uint8_t offset = 0; - uint16_t rowsPerRG = 8192; // hardcoded max rows in RowGroup value - uint32_t numberOfRGs = numRows / rowsPerRG; - - // set sorting rules - // true - ascending order, false otherwise - jobInfo.orderByColVec.push_back(make_pair(tupleKey, false)); - jobInfo.limitStart = offset; - jobInfo.limitCount = limit; - jobInfo.hasDistinct = hasDistinct; - // JobInfo doesn't set these SP in ctor - jobInfo.umMemLimit.reset(new int64_t); - *(jobInfo.umMemLimit) = MEMORY_LIMIT; - SErrorInfo errorInfo(new ErrorInfo()); - jobInfo.errorInfo = errorInfo; - uint32_t oid =3001; - // populate JobInfo.nonConstDelCols with dummy shared_pointers - // to notify TupleAnnexStep::initialise() about number of non-constant columns - execplan::SRCP srcp1, srcp2; - jobInfo.nonConstDelCols.push_back(srcp1); - jobInfo.nonConstDelCols.push_back(srcp2); - - // create two columns RG. 1st is the sorting key, second is the data column - std::vector offsets, roids, tkeys, cscale, cprecision; - std::vector charSetNumVec; - std::vector types; - offsets.push_back(2); offsets.push_back(10); offsets.push_back(18); - roids.push_back(oid); roids.push_back(oid); - tkeys.push_back(1); tkeys.push_back(1); - types.push_back(execplan::CalpontSystemCatalog::UBIGINT); - types.push_back(execplan::CalpontSystemCatalog::UBIGINT); - cscale.push_back(0); cscale.push_back(0); - cprecision.push_back(20); cprecision.push_back(20); - charSetNumVec.push_back(8); charSetNumVec.push_back(8); - rowgroup::RowGroup inRG(2, //column count - offsets, //oldOffset - roids, // column oids - tkeys, //keys - types, // types - charSetNumVec, // charset numbers - cscale, //scale - cprecision, // precision - 20, // sTableThreshold - false //useStringTable - ); - rowgroup::RowGroup jobInfoRG(inRG); - joblist::TupleAnnexStep tns = joblist::TupleAnnexStep(jobInfo); - tns.addOrderBy(new joblist::LimitedOrderBy()); - tns.delivery(true); - // activate parallel sort - if(parallelExecution) + uint64_t base = 42; + uint64_t maxInt = 0; + if (generateRandValues) + ::srand(base); + uint64_t nextValue; + for (uint32_t i = 1; i <= numberOfRGs; i++) + { + // create RGData with the RG structure and manually populate RG + // Use reinint(numberOfRGs) to preset an array + rowgroup::RGData rgD = rowgroup::RGData(inRG); + inRG.setData(&rgD); + rowgroup::Row r; + inRG.initRow(&r); + uint32_t rowSize = r.getSize(); + inRG.getRow(0, &r); + // populate RGData + // for(uint64_t i = rowsPerRG+1; i > 0; i--) + for (uint64_t i = 0; i < rowsPerRG; i++) // Worst case scenario for PQ + { + // TBD Use set..._offset methods to avoid offset calculation instructions + if (generateRandValues) { - tns.setParallelOp(); + nextValue = ::rand(); } - tns.setMaxThreads(maxThreads); - tns.initialize(jobInfoRG, jobInfo); - tns.setLimit(0, limit); - - // Create JobStepAssociation mediator class ins to connect DL and JS - joblist::AnyDataListSPtr spdlIn(new AnyDataList()); - //joblist::RowGroupDL* dlIn = new RowGroupDL(maxThreads, jobInfo.fifoSize); - joblist::RowGroupDL* dlIn = new RowGroupDL(maxThreads, 21001); - dlIn->OID(oid); - spdlIn->rowGroupDL(dlIn); - joblist::JobStepAssociation jsaIn; - jsaIn.outAdd(spdlIn); - tns.inputAssociation(jsaIn); - - uint64_t base = 42; - uint64_t maxInt = 0; - if(generateRandValues) - ::srand(base); - uint64_t nextValue; - for(uint32_t i = 1; i <= numberOfRGs; i++) - { - // create RGData with the RG structure and manually populate RG - // Use reinint(numberOfRGs) to preset an array - rowgroup::RGData rgD = rowgroup::RGData(inRG); - inRG.setData(&rgD); - rowgroup::Row r; - inRG.initRow(&r); - uint32_t rowSize = r.getSize(); - inRG.getRow(0, &r); - // populate RGData - //for(uint64_t i = rowsPerRG+1; i > 0; i--) - for(uint64_t i = 0; i < rowsPerRG; i++) // Worst case scenario for PQ - { - // TBD Use set..._offset methods to avoid offset calculation instructions - if(generateRandValues) - { - nextValue = ::rand(); - } - else - { - nextValue = base + i; - } - - r.setUintField<8>(nextValue, 0); - r.setUintField<8>(nextValue, 1); - if (maxInt < nextValue) - { - maxInt = nextValue; - } - r.nextRow(rowSize); - } - base += 1; - inRG.setRowCount(rowsPerRG); - - // Insert RGData into input DL - dlIn->insert(rgD); - } - // end of input signal - std::cout << "orderByTest_nRGs input DataList totalSize " << dlIn->totalSize() << std::endl; - dlIn->endOfInput(); - timer.stop(message); - - joblist::AnyDataListSPtr spdlOut(new AnyDataList()); - // Set the ring buffer big enough to take RGData-s with results b/c - // there is nobody to read out of the buffer. - joblist::RowGroupDL* dlOut = new RowGroupDL(1, numberOfRGs); - dlOut->OID(oid); - spdlOut->rowGroupDL(dlOut); - joblist::JobStepAssociation jsaOut; - jsaOut.outAdd(spdlOut); - //uint64_t outputDLIter = dlOut->getIterator(); - tns.outputAssociation(jsaOut); - - // Run Annex Step - message = "Sorting"; - timer.start(message); - tns.run(); - tns.join(); - timer.stop(message); - - // serialize RGData into bs and later back - // to follow ExeMgr whilst getting TAS result RowGroup - messageqcpp::ByteStream bs; - uint32_t result = 0; - rowgroup::RowGroup outRG(inRG); - rowgroup::RGData outRGData(outRG); - result = tns.nextBand(bs); - /*bool more = false; - do + else { - dlOut->next(outputDLIter, &outRGData); - } while (more);*/ - std::cout << "orderByTest_nRGs result " << result << std::endl; - //CPPUNIT_ASSERT( result == limit ); - outRGData.deserialize(bs); - outRG.setData(&outRGData); - - //std::cout << "orderByTest_nRGs output RG " << outRG.toString() << std::endl; - std::cout << "maxInt " << maxInt << std::endl; - { - rowgroup::Row r; - outRG.initRow(&r); - outRG.getRow(0, &r); - CPPUNIT_ASSERT(limit == outRG.getRowCount() || outRG.getRowCount() == 8192); - CPPUNIT_ASSERT_EQUAL(maxInt, r.getUintField(1)); + nextValue = base + i; } - cout << "------------------------------------------------------------" << endl; + r.setUintField<8>(nextValue, 0); + r.setUintField<8>(nextValue, 1); + if (maxInt < nextValue) + { + maxInt = nextValue; + } + r.nextRow(rowSize); + } + base += 1; + inRG.setRowCount(rowsPerRG); + + // Insert RGData into input DL + dlIn->insert(rgD); } + // end of input signal + std::cout << "orderByTest_nRGs input DataList totalSize " << dlIn->totalSize() << std::endl; + dlIn->endOfInput(); + timer.stop(message); + joblist::AnyDataListSPtr spdlOut(new AnyDataList()); + // Set the ring buffer big enough to take RGData-s with results b/c + // there is nobody to read out of the buffer. + joblist::RowGroupDL* dlOut = new RowGroupDL(1, numberOfRGs); + dlOut->OID(oid); + spdlOut->rowGroupDL(dlOut); + joblist::JobStepAssociation jsaOut; + jsaOut.outAdd(spdlOut); + // uint64_t outputDLIter = dlOut->getIterator(); + tns.outputAssociation(jsaOut); - void ORDERBY_TIME_TEST() + // Run Annex Step + message = "Sorting"; + timer.start(message); + tns.run(); + tns.join(); + timer.stop(message); + + // serialize RGData into bs and later back + // to follow ExeMgr whilst getting TAS result RowGroup + messageqcpp::ByteStream bs; + uint32_t result = 0; + rowgroup::RowGroup outRG(inRG); + rowgroup::RGData outRGData(outRG); + result = tns.nextBand(bs); + /*bool more = false; + do { - uint64_t numRows = 8192; - uint64_t maxThreads = 8; - // limit == 100000 is still twice as good to sort in parallel - // limit == 1000000 however is better to sort using single threaded sorting - uint64_t limit = 100000; - bool parallel = true; - bool woParallel = false; - bool generateRandValues = true; - bool hasDistinct = true; - bool noDistinct = false; - orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, noDistinct); - orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, noDistinct); - orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, hasDistinct); - orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, hasDistinct); - } - void QUICK_TEST() + dlOut->next(outputDLIter, &outRGData); + } while (more);*/ + std::cout << "orderByTest_nRGs result " << result << std::endl; + // CPPUNIT_ASSERT( result == limit ); + outRGData.deserialize(bs); + outRG.setData(&outRGData); + + // std::cout << "orderByTest_nRGs output RG " << outRG.toString() << std::endl; + std::cout << "maxInt " << maxInt << std::endl; { - float f = 1.1; - double d = 1.2; - uint64_t i = 1; - uint64_t* i_ptr = &i; - double* d_ptr = &d; - uint64_t* i2_ptr = (uint64_t*) d_ptr; - float* f_ptr = &f; - i_ptr = (uint64_t*) f_ptr; - - cout << "*i_ptr=" << *i_ptr << endl; - cout << "*i2_ptr=" << *i2_ptr << endl; - f_ptr = (float*) i_ptr; - - cout << "*f_ptr=" << *f_ptr << endl; - - cout << endl; - - if (d > i) - cout << "1.2 is greater than 1." << endl; - - if (f > i) - cout << "1.1 is greater than 1." << endl; - - if (d > f) - cout << "1.2 is greater than 1.1" << endl; - - if (*i_ptr < *i2_ptr) - cout << "1.1 < 1.2 when represented as uint64_t." << endl; - - cout << "sizeof(f) = " << sizeof(f) << endl; - cout << "sizeof(i) = " << sizeof(i) << endl; - cout << "sizeof(d) = " << sizeof(d) << endl; - - double dbl = 9.7; - double dbl2 = 1.3; - i_ptr = (uint64_t*) &dbl; - i2_ptr = (uint64_t*) &dbl2; - cout << endl; - cout << "9.7 as int is " << *i_ptr << endl; - cout << "9.7 as int is " << *i2_ptr << endl; - cout << "1.2 < 9.7 is " << (*i_ptr < *i2_ptr) << endl; + rowgroup::Row r; + outRG.initRow(&r); + outRG.getRow(0, &r); + CPPUNIT_ASSERT(limit == outRG.getRowCount() || outRG.getRowCount() == 8192); + CPPUNIT_ASSERT_EQUAL(maxInt, r.getUintField(1)); } + + cout << "------------------------------------------------------------" << endl; + } + + void ORDERBY_TIME_TEST() + { + uint64_t numRows = 8192; + uint64_t maxThreads = 8; + // limit == 100000 is still twice as good to sort in parallel + // limit == 1000000 however is better to sort using single threaded sorting + uint64_t limit = 100000; + bool parallel = true; + bool woParallel = false; + bool generateRandValues = true; + bool hasDistinct = true; + bool noDistinct = false; + orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, noDistinct); + orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, noDistinct); + orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, hasDistinct); + orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, hasDistinct); + } + void QUICK_TEST() + { + float f = 1.1; + double d = 1.2; + uint64_t i = 1; + uint64_t* i_ptr = &i; + double* d_ptr = &d; + uint64_t* i2_ptr = (uint64_t*)d_ptr; + float* f_ptr = &f; + i_ptr = (uint64_t*)f_ptr; + + cout << "*i_ptr=" << *i_ptr << endl; + cout << "*i2_ptr=" << *i2_ptr << endl; + f_ptr = (float*)i_ptr; + + cout << "*f_ptr=" << *f_ptr << endl; + + cout << endl; + + if (d > i) + cout << "1.2 is greater than 1." << endl; + + if (f > i) + cout << "1.1 is greater than 1." << endl; + + if (d > f) + cout << "1.2 is greater than 1.1" << endl; + + if (*i_ptr < *i2_ptr) + cout << "1.1 < 1.2 when represented as uint64_t." << endl; + + cout << "sizeof(f) = " << sizeof(f) << endl; + cout << "sizeof(i) = " << sizeof(i) << endl; + cout << "sizeof(d) = " << sizeof(d) << endl; + + double dbl = 9.7; + double dbl2 = 1.3; + i_ptr = (uint64_t*)&dbl; + i2_ptr = (uint64_t*)&dbl2; + cout << endl; + cout << "9.7 as int is " << *i_ptr << endl; + cout << "9.7 as int is " << *i2_ptr << endl; + cout << "1.2 < 9.7 is " << (*i_ptr < *i2_ptr) << endl; + } }; CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index 9fcf79131..b512e37b3 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -42,46 +42,34 @@ using namespace messageqcpp; namespace joblist { - /* I think this class literally does nothing */ PassThruCommandJL::PassThruCommandJL(const PassThruStep& p) { - OID = p.oid(); - colName = p.name(); - colWidth = p.colWidth; + OID = p.oid(); + colName = p.name(); + colWidth = p.colWidth; -// cout << "PassThru col width = " << (int) colWidth << " for OID " << OID << endl; - /* Is this ever a dictionary column? */ - if (p.isDictColumn) - tableColumnType = TableColumn::STRING; - else - switch (colWidth) - { - case 1: - tableColumnType = TableColumn::UINT8; - break; + // cout << "PassThru col width = " << (int) colWidth << " for OID " << OID << endl; + /* Is this ever a dictionary column? */ + if (p.isDictColumn) + tableColumnType = TableColumn::STRING; + else + switch (colWidth) + { + case 1: tableColumnType = TableColumn::UINT8; break; - case 2: - tableColumnType = TableColumn::UINT16; - break; + case 2: tableColumnType = TableColumn::UINT16; break; - case 4: - tableColumnType = TableColumn::UINT32; - break; + case 4: tableColumnType = TableColumn::UINT32; break; - case 8: - tableColumnType = TableColumn::UINT64; - break; - - case 16: - case 32: - tableColumnType = TableColumn::STRING; - break; - - default: - throw logic_error("PassThruCommandJL(): bad column width?"); - } + case 8: tableColumnType = TableColumn::UINT64; break; + + case 16: + case 32: tableColumnType = TableColumn::STRING; break; + + default: throw logic_error("PassThruCommandJL(): bad column width?"); + } } PassThruCommandJL::~PassThruCommandJL() @@ -94,9 +82,9 @@ void PassThruCommandJL::setLBID(uint64_t l, uint32_t dbroot) void PassThruCommandJL::createCommand(ByteStream& bs) const { - bs << (uint8_t) PASS_THRU; - bs << colWidth; - CommandJL::createCommand(bs); + bs << (uint8_t)PASS_THRU; + bs << colWidth; + CommandJL::createCommand(bs); } void PassThruCommandJL::runCommand(ByteStream& bs) const @@ -105,22 +93,21 @@ void PassThruCommandJL::runCommand(ByteStream& bs) const uint8_t PassThruCommandJL::getTableColumnType() { - return tableColumnType; + return tableColumnType; } string PassThruCommandJL::toString() { - ostringstream oss; - oss << "PassThruCommandJL: colwidth=" << static_cast(colWidth) << " oid=" << OID - << " colName=" << colName; - return oss.str(); + ostringstream oss; + oss << "PassThruCommandJL: colwidth=" << static_cast(colWidth) << " oid=" << OID + << " colName=" << colName; + return oss.str(); } uint16_t PassThruCommandJL::getWidth() { - return colWidth; + return colWidth; } -}; +}; // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/passthrucommand-jl.h b/dbcon/joblist/passthrucommand-jl.h index 642452fc8..d47ba7018 100644 --- a/dbcon/joblist/passthrucommand-jl.h +++ b/dbcon/joblist/passthrucommand-jl.h @@ -37,33 +37,33 @@ namespace joblist { - class PassThruCommandJL : public CommandJL { -public: - PassThruCommandJL(const PassThruStep&); - virtual ~PassThruCommandJL(); + public: + PassThruCommandJL(const PassThruStep&); + virtual ~PassThruCommandJL(); - void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For ColumnCommandJL it's a RID, for a DictStep it's a token. - uint8_t getTableColumnType(); - std::string toString(); + void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For + // ColumnCommandJL it's a RID, for a DictStep it's a token. + uint8_t getTableColumnType(); + std::string toString(); - void createCommand(messageqcpp::ByteStream&) const; - void runCommand(messageqcpp::ByteStream&) const; - uint16_t getWidth(); - CommandType getCommandType() - { - return PASS_THRU; - } + void createCommand(messageqcpp::ByteStream&) const; + void runCommand(messageqcpp::ByteStream&) const; + uint16_t getWidth(); + CommandType getCommandType() + { + return PASS_THRU; + } -private: - PassThruCommandJL(); - PassThruCommandJL(const PassThruCommandJL&); + private: + PassThruCommandJL(); + PassThruCommandJL(const PassThruCommandJL&); - uint8_t colWidth; - uint8_t tableColumnType; + uint8_t colWidth; + uint8_t tableColumnType; }; -}; +}; // namespace joblist #endif diff --git a/dbcon/joblist/passthrustep.cpp b/dbcon/joblist/passthrustep.cpp index 426b60f2f..7eca8dff5 100644 --- a/dbcon/joblist/passthrustep.cpp +++ b/dbcon/joblist/passthrustep.cpp @@ -34,11 +34,11 @@ using namespace boost; #include "primitivestep.h" using namespace execplan; -//namespace { -//using namespace joblist; -//class ptt +// namespace { +// using namespace joblist; +// class ptt //{ -//public: +// public: // ptt(DataList_t* i, DataList_t* o, bool swallowRows, PassThruStep* pStep) : // idl(i), odl(o), fSwallowRows(swallowRows), fpStep(pStep) { } // @@ -73,7 +73,8 @@ using namespace execplan; // while (more) // { // // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is -// // TRACE_FLAGS::TRACE_NO_ROWS4. Running with this on will swallow rows at +// // TRACE_FLAGS::TRACE_NO_ROWS4. Running with this on will swallow rows +//at // // projection. // /* XXXPAT: If this feeds a pDictionary, is fSwallowRows always false? */ // if (!fSwallowRows || isDictColumn) @@ -94,21 +95,13 @@ using namespace execplan; // for (uint64_t i = 0; i < rwIn.count; ++i) // { // switch (colWidth) { -// case 1: pos[rwOut.count++] = rwIn.et[i].second; break; -// case 2: ((uint16_t *) pos)[rwOut.count++] = rwIn.et[i].second; -// break; -// case 3: -// case 4: ((uint32_t *) pos)[rwOut.count++] = rwIn.et[i].second; -// break; -// case 5: -// case 6: -// case 7: -// case 8: ((uint64_t *) pos)[rwOut.count++] = rwIn.et[i].second; -// break; -// default: -// cout << "PassThruStep: bad column width of " << +// case 1: pos[rwOut.count++] = rwIn.et[i].second; +//break; case 2: ((uint16_t *) pos)[rwOut.count++] = rwIn.et[i].second; break; case 3: case 4: ((uint32_t *) +//pos)[rwOut.count++] = rwIn.et[i].second; break; case 5: case 6: case 7: case 8: ((uint64_t *) +//pos)[rwOut.count++] = rwIn.et[i].second; break; default: cout << "PassThruStep: bad column width of " << // fpStep->getColWidth() << endl; -// throw logic_error("PassThruStep: bad column width"); +// throw logic_error("PassThruStep: bad column +//width"); // } // // if (rwOut.count == rwOut.ElementsPerGroup) @@ -131,7 +124,8 @@ using namespace execplan; // while (more) // { // // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is -// // TRACE_FLAGS::TRACE_NO_ROWS4. Running with this on will swallow rows at +// // TRACE_FLAGS::TRACE_NO_ROWS4. Running with this on will swallow rows +//at // // projection. // // if (!fSwallowRows || isDictColumn) @@ -153,9 +147,8 @@ using namespace execplan; // case 8: ((uint64_t *) pos)[rwOut.count++] = e.second; // break; // default: -// cout << "PassThruStep: bad column width of " << -// fpStep->getColWidth() << endl; -// throw logic_error("PassThruStep: bad column width"); +// cout << "PassThruStep: bad column width of " +//<< fpStep->getColWidth() << endl; throw logic_error("PassThruStep: bad column width"); // } // } // if (rwOut.count == rwOut.ElementsPerGroup) @@ -193,7 +186,7 @@ using namespace execplan; // } // } // -//private: +// private: // //ptt(const ptt& rhs); // //ptt& operator=(const ptt& rhs); // @@ -207,63 +200,58 @@ using namespace execplan; namespace joblist { - -PassThruStep::PassThruStep( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tableOid, - const execplan::CalpontSystemCatalog::ColType& colType, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fOid(oid), - fTableOid(tableOid), - isEM(jobInfo.isExeMgr), - fSwallowRows(false), - fRm(jobInfo.rm) +PassThruStep::PassThruStep(execplan::CalpontSystemCatalog::OID oid, + execplan::CalpontSystemCatalog::OID tableOid, + const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo) + : JobStep(jobInfo) + , fOid(oid) + , fTableOid(tableOid) + , isEM(jobInfo.isExeMgr) + , fSwallowRows(false) + , fRm(jobInfo.rm) { - colWidth = colType.colWidth; - realWidth = colType.colWidth; - isDictColumn = ((colType.colDataType == CalpontSystemCatalog::VARCHAR && colType.colWidth > 7) - || (colType.colDataType == CalpontSystemCatalog::CHAR && colType.colWidth > 8) - || (colType.colDataType == CalpontSystemCatalog::TEXT) - || (colType.colDataType == CalpontSystemCatalog::BLOB)); - fColType = colType; - fPseudoType = 0; - + colWidth = colType.colWidth; + realWidth = colType.colWidth; + isDictColumn = ((colType.colDataType == CalpontSystemCatalog::VARCHAR && colType.colWidth > 7) || + (colType.colDataType == CalpontSystemCatalog::CHAR && colType.colWidth > 8) || + (colType.colDataType == CalpontSystemCatalog::TEXT) || + (colType.colDataType == CalpontSystemCatalog::BLOB)); + fColType = colType; + fPseudoType = 0; } PassThruStep::PassThruStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.fRm) { - fInputJobStepAssociation = rhs.inputAssociation(); - fOutputJobStepAssociation = rhs.outputAssociation(); - colWidth = rhs.fColType.colWidth; - realWidth = rhs.realWidth; - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - fSwallowRows = rhs.getSwallowRows(); - isDictColumn = rhs.isDictCol(); - fColType = rhs.colType(); - isEM = rhs.isExeMgr(); + fInputJobStepAssociation = rhs.inputAssociation(); + fOutputJobStepAssociation = rhs.outputAssociation(); + colWidth = rhs.fColType.colWidth; + realWidth = rhs.realWidth; + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + fSwallowRows = rhs.getSwallowRows(); + isDictColumn = rhs.isDictCol(); + fColType = rhs.colType(); + isEM = rhs.isExeMgr(); - const PseudoColStep* pcs = dynamic_cast(&rhs); - - if (pcs) - fPseudoType = pcs->pseudoColumnId(); + const PseudoColStep* pcs = dynamic_cast(&rhs); + if (pcs) + fPseudoType = pcs->pseudoColumnId(); } PassThruStep::PassThruStep(const PseudoColStep& rhs) : JobStep(rhs), fRm(rhs.fRm) { - fInputJobStepAssociation = rhs.inputAssociation(); - fOutputJobStepAssociation = rhs.outputAssociation(); - colWidth = rhs.fColType.colWidth; - realWidth = rhs.realWidth; - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - fSwallowRows = rhs.getSwallowRows(); - isDictColumn = rhs.isDictCol(); - fColType = rhs.colType(); - fPseudoType = rhs.pseudoColumnId(); - isEM = rhs.isExeMgr(); + fInputJobStepAssociation = rhs.inputAssociation(); + fOutputJobStepAssociation = rhs.outputAssociation(); + colWidth = rhs.fColType.colWidth; + realWidth = rhs.realWidth; + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + fSwallowRows = rhs.getSwallowRows(); + isDictColumn = rhs.isDictCol(); + fColType = rhs.colType(); + fPseudoType = rhs.pseudoColumnId(); + isEM = rhs.isExeMgr(); } PassThruStep::~PassThruStep() @@ -272,48 +260,47 @@ PassThruStep::~PassThruStep() void PassThruStep::run() { -// if (traceOn()) -// { -// syslogStartStep(16, // exemgr subsystem -// std::string("PassThruStep")); // step name -// } -// -// DataList_t* idl = fInputJobStepAssociation.outAt(0)->dataList(); -// idbassert(idl); -// DataList_t* odl = fOutputJobStepAssociation.outAt(0)->dataList(); -// idbassert(odl); -// ptt ptt(idl, odl, fSwallowRows, this); -// fPTThd = new boost::thread(ptt); + // if (traceOn()) + // { + // syslogStartStep(16, // exemgr subsystem + // std::string("PassThruStep")); // step name + // } + // + // DataList_t* idl = fInputJobStepAssociation.outAt(0)->dataList(); + // idbassert(idl); + // DataList_t* odl = fOutputJobStepAssociation.outAt(0)->dataList(); + // idbassert(odl); + // ptt ptt(idl, odl, fSwallowRows, this); + // fPTThd = new boost::thread(ptt); } void PassThruStep::join() { -// fPTThd->join(); -// delete fPTThd; + // fPTThd->join(); + // delete fPTThd; } const string PassThruStep::toString() const { - ostringstream oss; - oss << "PassThruStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId << - " tb/col:" << fTableOid << "/" << fOid; + ostringstream oss; + oss << "PassThruStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId + << " tb/col:" << fTableOid << "/" << fOid; - if (alias().length()) oss << " alias:" << alias(); + if (alias().length()) + oss << " alias:" << alias(); - oss << " " << omitOidInDL - << fOutputJobStepAssociation.outAt(0) << showOidInDL; - oss << " in:"; + oss << " " << omitOidInDL << fOutputJobStepAssociation.outAt(0) << showOidInDL; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - { - oss << fInputJobStepAssociation.outAt(i) << ", "; - } + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + oss << fInputJobStepAssociation.outAt(i) << ", "; + } - if (fSwallowRows) - oss << " (sink)"; - - return oss.str(); -} + if (fSwallowRows) + oss << " (sink)"; + return oss.str(); } +} // namespace joblist diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 7c0d50657..53dfbcfb0 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: pcolscan.cpp 9655 2013-06-25 23:08:13Z xlou $ -* -* -***********************************************************************/ + * $Id: pcolscan.cpp 9655 2013-06-25 23:08:13Z xlou $ + * + * + ***********************************************************************/ #include #include @@ -43,7 +43,6 @@ using namespace logging; #include "logicoperator.h" using namespace execplan; - #include "distributedenginecomm.h" #include "primitivemsg.h" #include "timestamp.h" @@ -54,13 +53,12 @@ using namespace execplan; //#define DEBUG 1 //#define DEBUG2 1 - namespace { //// const uint32_t defaultScanLbidReqLimit = 10000; //// const uint32_t defaultScanLbidReqThreshold = 5000; // -//struct pColScanStepPrimitive +// struct pColScanStepPrimitive //{ // pColScanStepPrimitive(pColScanStep* pColScanStep) : fPColScanStep(pColScanStep) // {} @@ -81,14 +79,14 @@ namespace // if (msg.find("there are no primitive processors") != string::npos) // { // SPJL logger = fPColScanStep->logger(); -// logger->logMessage(LOG_TYPE_CRITICAL, LogNoPrimProcs, Message::Args(), LoggingID(5)); -// exit(1); +// logger->logMessage(LOG_TYPE_CRITICAL, LogNoPrimProcs, Message::Args(), +//LoggingID(5)); exit(1); // } // } // } //}; // -//struct pColScanStepAggregater +// struct pColScanStepAggregater //{ // pColScanStepAggregater(pColScanStep* pColScanStep, uint64_t index) : // fPColScanStepCol(pColScanStep), fThreadId(index) @@ -104,154 +102,149 @@ namespace // } // catch(std::exception& re) // { -// cerr << fPColScanStepCol->toString() << ": receive thread threw an exception: " << re.what() << endl; +// cerr << fPColScanStepCol->toString() << ": receive thread threw an exception: " << +//re.what() << endl; // } // } //}; // -} // end of local unnamed namespace - +} // namespace namespace joblist { - -pColScanStep::pColScanStep( - CalpontSystemCatalog::OID o, - CalpontSystemCatalog::OID t, - const CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fRm(jobInfo.rm), - fMsgHeader(), - fNumThreads(fRm->getJlNumScanReceiveThreads()), - fFilterCount(0), - fOid(o), - fTableOid(t), - fColType(ct), - fBOP(BOP_OR), - sentCount(0), - recvCount(0), - fScanLbidReqLimit(fRm->getJlScanLbidReqLimit()), - fScanLbidReqThreshold(fRm->getJlScanLbidReqThreshold()), - fStopSending(false), - fSingleThread(false), - fPhysicalIO(0), - fCacheIO(0), - fNumBlksSkipped(0), - fMsgBytesIn(0), - fMsgBytesOut(0), - fMsgsToPm(0) +pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, + const CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) + : JobStep(jobInfo) + , fRm(jobInfo.rm) + , fMsgHeader() + , fNumThreads(fRm->getJlNumScanReceiveThreads()) + , fFilterCount(0) + , fOid(o) + , fTableOid(t) + , fColType(ct) + , fBOP(BOP_OR) + , sentCount(0) + , recvCount(0) + , fScanLbidReqLimit(fRm->getJlScanLbidReqLimit()) + , fScanLbidReqThreshold(fRm->getJlScanLbidReqThreshold()) + , fStopSending(false) + , fSingleThread(false) + , fPhysicalIO(0) + , fCacheIO(0) + , fNumBlksSkipped(0) + , fMsgBytesIn(0) + , fMsgBytesOut(0) + , fMsgsToPm(0) { - if (fTableOid == 0) // cross engine support - return; + if (fTableOid == 0) // cross engine support + return; - int err, i, mask; + int err, i, mask; - finishedSending = false; - recvWaiting = 0; - recvExited = 0; - rDoNothing = false; - fIsDict = false; + finishedSending = false; + recvWaiting = 0; + recvExited = 0; + rDoNothing = false; + fIsDict = false; - //If this is a dictionary column, fudge the numbers... - if ( fColType.colDataType == CalpontSystemCatalog::VARCHAR ) + // If this is a dictionary column, fudge the numbers... + if (fColType.colDataType == CalpontSystemCatalog::VARCHAR) + { + if (8 > fColType.colWidth && 4 <= fColType.colWidth) + fColType.colDataType = CalpontSystemCatalog::CHAR; + + fColType.colWidth++; + } + + // If this is a dictionary column, fudge the numbers... + if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY) || + (fColType.colDataType == CalpontSystemCatalog::BLOB) || + (fColType.colDataType == CalpontSystemCatalog::TEXT)) + { + fColType.colWidth = 8; + fIsDict = true; + } + // MCOL-641 WIP + else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::DECIMAL && + fColType.colDataType != CalpontSystemCatalog::UDECIMAL) + { + fColType.colWidth = 8; + fIsDict = true; + // TODO: is this right? + fColType.colDataType = CalpontSystemCatalog::VARCHAR; + } + + // Round colWidth up + if (fColType.colWidth == 3) + fColType.colWidth = 4; + else if (fColType.colWidth == 5 || fColType.colWidth == 6 || fColType.colWidth == 7) + fColType.colWidth = 8; + + err = dbrm.lookup(fOid, lbidRanges); + + if (err) + throw runtime_error("pColScan: BRM LBID range lookup failure (1)"); + + err = dbrm.getExtents(fOid, extents); + + if (err) + throw runtime_error("pColScan: BRM HWM lookup failure (4)"); + + sort(extents.begin(), extents.end(), BRM::ExtentSorter()); + numExtents = extents.size(); + extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; + + if (fOid > 3000) + { + lbidList.reset(new LBIDList(fOid, 0)); + } + + /* calculate shortcuts for rid-based arithmetic */ + for (i = 1, mask = 1; i <= 32; i++) + { + mask <<= 1; + + if (extentSize & mask) { - if (8 > fColType.colWidth && 4 <= fColType.colWidth ) - fColType.colDataType = CalpontSystemCatalog::CHAR; - - fColType.colWidth++; + divShift = i; + break; } + } - //If this is a dictionary column, fudge the numbers... - if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY) - || (fColType.colDataType == CalpontSystemCatalog::BLOB) - || (fColType.colDataType == CalpontSystemCatalog::TEXT)) + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (extentSize & mask) + throw runtime_error("pColScan: Extent size must be a power of 2 in blocks"); + + ridsPerBlock = BLOCK_SIZE / fColType.colWidth; + + for (i = 1, mask = 1; i <= 32; i++) + { + mask <<= 1; + + if (ridsPerBlock & mask) { - fColType.colWidth = 8; - fIsDict = true; - } - // MCOL-641 WIP - else if (fColType.colWidth > 8 - && fColType.colDataType != CalpontSystemCatalog::DECIMAL - && fColType.colDataType != CalpontSystemCatalog::UDECIMAL) - { - fColType.colWidth = 8; - fIsDict = true; - //TODO: is this right? - fColType.colDataType = CalpontSystemCatalog::VARCHAR; + rpbShift = i; + break; } + } - //Round colWidth up - if (fColType.colWidth == 3) - fColType.colWidth = 4; - else if (fColType.colWidth == 5 || fColType.colWidth == 6 || fColType.colWidth == 7) - fColType.colWidth = 8; - - err = dbrm.lookup(fOid, lbidRanges); - - if (err) - throw runtime_error("pColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getExtents(fOid, extents); - - if (err) - throw runtime_error("pColScan: BRM HWM lookup failure (4)"); - - sort(extents.begin(), extents.end(), BRM::ExtentSorter()); - numExtents = extents.size(); - extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; - - if (fOid > 3000) - { - lbidList.reset(new LBIDList(fOid, 0)); - } - - /* calculate shortcuts for rid-based arithmetic */ - for (i = 1, mask = 1; i <= 32; i++) - { - mask <<= 1; - - if (extentSize & mask) - { - divShift = i; - break; - } - } - - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (extentSize & mask) - throw runtime_error("pColScan: Extent size must be a power of 2 in blocks"); - - ridsPerBlock = BLOCK_SIZE / fColType.colWidth; - - for (i = 1, mask = 1; i <= 32; i++) - { - mask <<= 1; - - if (ridsPerBlock & mask) - { - rpbShift = i; - break; - } - } - - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (ridsPerBlock & mask) - throw runtime_error("pColScan: Block size and column width must be a power of 2"); + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (ridsPerBlock & mask) + throw runtime_error("pColScan: Block size and column width must be a power of 2"); } pColScanStep::~pColScanStep() { - //pthread_mutex_destroy(&mutex); - //pthread_mutex_destroy(&dlMutex); - //pthread_mutex_destroy(&cpMutex); - //pthread_cond_destroy(&condvar); - //pthread_cond_destroy(&condvarWakeupProducer); - //delete lbidList; -// delete [] fProducerThread; -// if (fDec) -// fDec->removeQueue(uniqueID); // in case it gets aborted + // pthread_mutex_destroy(&mutex); + // pthread_mutex_destroy(&dlMutex); + // pthread_mutex_destroy(&cpMutex); + // pthread_cond_destroy(&condvar); + // pthread_cond_destroy(&condvarWakeupProducer); + // delete lbidList; + // delete [] fProducerThread; + // if (fDec) + // fDec->removeQueue(uniqueID); // in case it gets aborted } //------------------------------------------------------------------------------ @@ -259,987 +252,976 @@ pColScanStep::~pColScanStep() //------------------------------------------------------------------------------ void pColScanStep::initializeConfigParms() { -// const string section ( "JobList" ); -// const string sendLimitName ( "ScanLbidReqLimit" ); -// const string sendThresholdName ( "ScanLbidReqThreshold" ); -// const string numReadThreadsName ( "NumScanReceiveThreads" ); -// Config* cf = Config::makeConfig(); + // const string section ( "JobList" ); + // const string sendLimitName ( "ScanLbidReqLimit" ); + // const string sendThresholdName ( "ScanLbidReqThreshold" ); + // const string numReadThreadsName ( "NumScanReceiveThreads" ); + // Config* cf = Config::makeConfig(); -// string strVal; + // string strVal; - //...Get the tuning parameters that throttle msgs sent to primproc - //...fScanLbidReqLimit puts a cap on how many LBID's we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fScanLbidReqThreshold is the level at which the number of outstanding - //... LBID reqs must fall below, before the producer can send more LBIDs. -// strVal = cf->getConfig(section, sendLimitName); -// if (strVal.size() > 0) -// fScanLbidReqLimit = static_cast(Config::uFromText(strVal)); -// -// strVal = cf->getConfig(section, sendThresholdName); -// if (strVal.size() > 0) -// fScanLbidReqThreshold = static_cast(Config::uFromText(strVal)); -// -// fNumThreads = 8; -// strVal = cf->getConfig(section, numReadThreadsName); -// if (strVal.size() > 0) -// fNumThreads = static_cast(Config::uFromText(strVal)); + //...Get the tuning parameters that throttle msgs sent to primproc + //...fScanLbidReqLimit puts a cap on how many LBID's we will request from + //... primproc, before pausing to let the consumer thread catch up. + //... Without this limit, there is a chance that PrimProc could flood + //... ExeMgr with thousands of messages that will consume massive + //... amounts of memory for a 100 gigabyte database. + //...fScanLbidReqThreshold is the level at which the number of outstanding + //... LBID reqs must fall below, before the producer can send more LBIDs. + // strVal = cf->getConfig(section, sendLimitName); + // if (strVal.size() > 0) + // fScanLbidReqLimit = static_cast(Config::uFromText(strVal)); + // + // strVal = cf->getConfig(section, sendThresholdName); + // if (strVal.size() > 0) + // fScanLbidReqThreshold = static_cast(Config::uFromText(strVal)); + // + // fNumThreads = 8; + // strVal = cf->getConfig(section, numReadThreadsName); + // if (strVal.size() > 0) + // fNumThreads = static_cast(Config::uFromText(strVal)); -// fProducerThread = new SPTHD[fNumThreads]; + // fProducerThread = new SPTHD[fNumThreads]; } void pColScanStep::startPrimitiveThread() { -// fConsumerThread.reset(new boost::thread(pColScanStepPrimitive(this))); + // fConsumerThread.reset(new boost::thread(pColScanStepPrimitive(this))); } void pColScanStep::startAggregationThread() { -// for (uint32_t i = 0; i < fNumThreads; i++) -// fProducerThread[i].reset(new boost::thread(pColScanStepAggregater(this, i))); + // for (uint32_t i = 0; i < fNumThreads; i++) + // fProducerThread[i].reset(new boost::thread(pColScanStepAggregater(this, i))); } void pColScanStep::run() { -// if (traceOn()) -// { -// syslogStartStep(16, // exemgr subsystem -// std::string("pColScanStep")); // step name -// } -// -// //"consume" input datalist. In this case, there is no IDL, we just send one primitive -// startPrimitiveThread(); -// //produce output datalist -// //Don't start this yet...see below -// //startAggregationThread(); + // if (traceOn()) + // { + // syslogStartStep(16, // exemgr subsystem + // std::string("pColScanStep")); // step name + // } + // + // //"consume" input datalist. In this case, there is no IDL, we just send one primitive + // startPrimitiveThread(); + // //produce output datalist + // //Don't start this yet...see below + // //startAggregationThread(); } void pColScanStep::join() { -// fConsumerThread->join(); -// if ( !fSingleThread ) { -// for (uint32_t i = 0; i < fNumThreads; i++) -// fProducerThread[i]->join(); -// } + // fConsumerThread->join(); + // if ( !fSingleThread ) { + // for (uint32_t i = 0; i < fNumThreads; i++) + // fProducerThread[i]->join(); + // } } void pColScanStep::sendPrimitiveMessages() { -// //The presence of an input DL means we (probably) have a pDictionaryScan step feeding this scan step -// // a list of tokens to get the rids for. Convert the input tokens to a filter string. -// if (fInputJobStepAssociation.outSize() > 0) -// { -// addFilters(); -// if (fTableOid >= 3000) -// cout << toString() << endl; -// //If we got no input rids (as opposed to no input DL at all) then there were no matching rows from -// // the previous step, so this step should not return any rows either. This would be the case, for -// // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. -// if (fFilterCount == 0) { -// rDoNothing=true; -// startAggregationThread(); -// return; -// } -// } -// -// startAggregationThread(); -// -// /* for all the blocks that need to be sent -// * build out a message for each block with the primitive message header filled -// * out and all the NOPS and BOP structures as well. -// * Then serialize it to a BytStream and then send it on its way -// */ -// -// LBIDRange_v::iterator it; -// uint64_t fbo; -// -// ISMPacketHeader ism; -// ism.Flags = planFlagsToPrimFlags(fTraceFlags); -// ism.Command=COL_BY_SCAN_RANGE; -// ism.Size = sizeof(ISMPacketHeader) + sizeof(ColByScanRangeRequestHeader) + fFilterString.length(); -// ism.Type=2; -// //bool firstWrite = true; -// -// //...Counter used to track the number of LBIDs we are requesting from -// //...primproc in the current set of msgs, till we reach fScanLbidReqLimit -// uint32_t runningLbidCount = 0; -// bool exitLoop = false; -// const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); -// -// for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) -// { -// BRM::LBID_t lbid = (*it).start; -// -// fbo = getFBO(lbid); -// if (hwm < fbo) -// continue; -// -// if (fOid >= 3000 && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth) ) -// { -// int64_t Min=0; -// int64_t Max=0; -// int64_t SeqNum=0; -// bool MinMaxValid=true; -// bool cpPredicate=true; -// -// // can we consolidate these crit sections? -// cpMutex.lock(); //pthread_mutex_lock(&cpMutex); -// MinMaxValid = lbidList->GetMinMax(Min, Max, SeqNum, lbid, 0); -// -// if (MinMaxValid) -// { -// cpPredicate=lbidList->CasualPartitionPredicate(Min, -// Max, -// &fFilterString, -// fFilterCount, -// fColType, -// fBOP) || ignoreCP; -// } -// cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); -// -// if (cpPredicate==false){ //don't scan this extent -//#ifdef DEBUG -// cout << "Scan Skip " << lbid << endl; -//#endif -// //...Track the number of LBIDs we skip due to Casual Partioning. -// //...We use the same equation we use to initialize remainingLbids -// //...in the code that follows down below this. -// fNumBlksSkipped += ( (hwm > (fbo + it->size - 1)) ? -// (it->size) : (hwm - fbo + 1) ); -// continue; -// } -//#ifdef DEBUG -// else -// cout << "Scan " << lbid << endl; -//#endif -// -// } -// -// LBID_t msgLbidStart = it->start; -// uint32_t remainingLbids = -// ( (hwm > (fbo + it->size - 1)) ? (it->size) : (hwm - fbo + 1) ); -// uint32_t msgLbidCount = 0; -// -// while ( remainingLbids > 0 ) -// { -// //...Break up this range of LBIDs when we reach the msg size -// //...limit for one request (fScanLbidReqLimit) -// if ( (runningLbidCount + remainingLbids) >= fScanLbidReqLimit ) -// { -// msgLbidCount = fScanLbidReqLimit - runningLbidCount; -// sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); -// //...Wait for the consuming thread to catch up if our -// //...backlog of work is >= the allowable threshold limit -// -// mutex.lock(); //pthread_mutex_lock(&mutex); -// sentCount += msgLbidCount; -// if (recvWaiting) -// condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to resume -// -// while ( ((sentCount - recvCount) >= fScanLbidReqThreshold) -// && !fStopSending ) -// { -// sendWaiting = true; -//#ifdef DEBUG2 -// if (fOid >= 3000) -// cout << "pColScanStep producer WAITING: " << -// "st:" << fStepId << -// "; sentCount-" << sentCount << -// "; recvCount-" << recvCount << -// "; threshold-" << fScanLbidReqThreshold << endl; -//#endif -// condvarWakeupProducer.wait(mutex); //pthread_cond_wait ( &condvarWakeupProducer, &mutex ); -//#ifdef DEBUG2 -// if (fOid >= 3000) -// cout << "pColScanStep producer RESUMING: " << -// "st:" << fStepId << endl; -//#endif -// sendWaiting = false; -// } -// -// //...Set flag to quit if consumer thread tells us to -// if (fStopSending) -// exitLoop = true; -// -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// runningLbidCount = 0; -// } -// else -// { -// msgLbidCount = remainingLbids; -// -// sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); -// mutex.lock(); //pthread_mutex_lock(&mutex); -// sentCount += msgLbidCount; -// if (recvWaiting) -// condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to resume -// -// //...Set flag to quit if consumer thread tells us to -// if (fStopSending) -// exitLoop = true; -// -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// runningLbidCount += msgLbidCount; -// } -// -// //...If consuming thread has quit, then we should do the same. -// //...This can happen if consuming thread receives empty ByteStream -// if (exitLoop) -// break; -// -// remainingLbids -= msgLbidCount; -// msgLbidStart += msgLbidCount; -// } -// -// if (exitLoop) -// break; -// -// } // end of loop through LBID ranges to be requested from primproc -// -// mutex.lock(); //pthread_mutex_lock(&mutex); -// finishedSending = true; -// if (recvWaiting) -// condvar.notify_all(); //pthread_cond_broadcast(&condvar); -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -//// cerr << "send side exiting" << endl; -// -//#ifdef DEBUG2 -// if (fOid >= 3000) -// { -// time_t t = time(0); -// char timeString[50]; -// ctime_r(&t, timeString); -// timeString[strlen(timeString)-1 ] = '\0'; -// cout << "pColScanStep Finished sending primitives for: " << -// fOid << " at " << timeString << endl; -// } -//#endif -// + // //The presence of an input DL means we (probably) have a pDictionaryScan step feeding this scan step + // // a list of tokens to get the rids for. Convert the input tokens to a filter string. + // if (fInputJobStepAssociation.outSize() > 0) + // { + // addFilters(); + // if (fTableOid >= 3000) + // cout << toString() << endl; + // //If we got no input rids (as opposed to no input DL at all) then there were no matching rows from + // // the previous step, so this step should not return any rows either. This would be the case, for + // // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. + // if (fFilterCount == 0) { + // rDoNothing=true; + // startAggregationThread(); + // return; + // } + // } + // + // startAggregationThread(); + // + // /* for all the blocks that need to be sent + // * build out a message for each block with the primitive message header filled + // * out and all the NOPS and BOP structures as well. + // * Then serialize it to a BytStream and then send it on its way + // */ + // + // LBIDRange_v::iterator it; + // uint64_t fbo; + // + // ISMPacketHeader ism; + // ism.Flags = planFlagsToPrimFlags(fTraceFlags); + // ism.Command=COL_BY_SCAN_RANGE; + // ism.Size = sizeof(ISMPacketHeader) + sizeof(ColByScanRangeRequestHeader) + fFilterString.length(); + // ism.Type=2; + // //bool firstWrite = true; + // + // //...Counter used to track the number of LBIDs we are requesting from + // //...primproc in the current set of msgs, till we reach fScanLbidReqLimit + // uint32_t runningLbidCount = 0; + // bool exitLoop = false; + // const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); + // + // for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + // { + // BRM::LBID_t lbid = (*it).start; + // + // fbo = getFBO(lbid); + // if (hwm < fbo) + // continue; + // + // if (fOid >= 3000 && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth) ) + // { + // int64_t Min=0; + // int64_t Max=0; + // int64_t SeqNum=0; + // bool MinMaxValid=true; + // bool cpPredicate=true; + // + // // can we consolidate these crit sections? + // cpMutex.lock(); //pthread_mutex_lock(&cpMutex); + // MinMaxValid = lbidList->GetMinMax(Min, Max, SeqNum, lbid, 0); + // + // if (MinMaxValid) + // { + // cpPredicate=lbidList->CasualPartitionPredicate(Min, + // Max, + // &fFilterString, + // fFilterCount, + // fColType, + // fBOP) || + //ignoreCP; + // } + // cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); + // + // if (cpPredicate==false){ //don't scan this extent + //#ifdef DEBUG + // cout << "Scan Skip " << lbid << endl; + //#endif + // //...Track the number of LBIDs we skip due to Casual Partioning. + // //...We use the same equation we use to initialize remainingLbids + // //...in the code that follows down below this. + // fNumBlksSkipped += ( (hwm > (fbo + it->size - 1)) ? + // (it->size) : (hwm - fbo + 1) ); + // continue; + // } + //#ifdef DEBUG + // else + // cout << "Scan " << lbid << endl; + //#endif + // + // } + // + // LBID_t msgLbidStart = it->start; + // uint32_t remainingLbids = + // ( (hwm > (fbo + it->size - 1)) ? (it->size) : (hwm - fbo + 1) ); + // uint32_t msgLbidCount = 0; + // + // while ( remainingLbids > 0 ) + // { + // //...Break up this range of LBIDs when we reach the msg size + // //...limit for one request (fScanLbidReqLimit) + // if ( (runningLbidCount + remainingLbids) >= fScanLbidReqLimit ) + // { + // msgLbidCount = fScanLbidReqLimit - runningLbidCount; + // sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); + // //...Wait for the consuming thread to catch up if our + // //...backlog of work is >= the allowable threshold limit + // + // mutex.lock(); //pthread_mutex_lock(&mutex); + // sentCount += msgLbidCount; + // if (recvWaiting) + // condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to + //resume + // + // while ( ((sentCount - recvCount) >= fScanLbidReqThreshold) + // && !fStopSending ) + // { + // sendWaiting = true; + //#ifdef DEBUG2 + // if (fOid >= 3000) + // cout << "pColScanStep producer WAITING: " << + // "st:" << fStepId << + // "; sentCount-" << sentCount << + // "; recvCount-" << recvCount << + // "; threshold-" << fScanLbidReqThreshold << endl; + //#endif + // condvarWakeupProducer.wait(mutex); //pthread_cond_wait ( &condvarWakeupProducer, &mutex + //); #ifdef DEBUG2 if (fOid >= 3000) cout << "pColScanStep producer RESUMING: " << "st:" << fStepId << endl; + //#endif + // sendWaiting = false; + // } + // + // //...Set flag to quit if consumer thread tells us to + // if (fStopSending) + // exitLoop = true; + // + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // runningLbidCount = 0; + // } + // else + // { + // msgLbidCount = remainingLbids; + // + // sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); + // mutex.lock(); //pthread_mutex_lock(&mutex); + // sentCount += msgLbidCount; + // if (recvWaiting) + // condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to + //resume + // + // //...Set flag to quit if consumer thread tells us to + // if (fStopSending) + // exitLoop = true; + // + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // runningLbidCount += msgLbidCount; + // } + // + // //...If consuming thread has quit, then we should do the same. + // //...This can happen if consuming thread receives empty ByteStream + // if (exitLoop) + // break; + // + // remainingLbids -= msgLbidCount; + // msgLbidStart += msgLbidCount; + // } + // + // if (exitLoop) + // break; + // + // } // end of loop through LBID ranges to be requested from primproc + // + // mutex.lock(); //pthread_mutex_lock(&mutex); + // finishedSending = true; + // if (recvWaiting) + // condvar.notify_all(); //pthread_cond_broadcast(&condvar); + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + //// cerr << "send side exiting" << endl; + // + //#ifdef DEBUG2 + // if (fOid >= 3000) + // { + // time_t t = time(0); + // char timeString[50]; + // ctime_r(&t, timeString); + // timeString[strlen(timeString)-1 ] = '\0'; + // cout << "pColScanStep Finished sending primitives for: " << + // fOid << " at " << timeString << endl; + // } + //#endif + // } //------------------------------------------------------------------------------ // Construct and send a single primitive message to primproc //------------------------------------------------------------------------------ -void pColScanStep::sendAPrimitiveMessage( - ISMPacketHeader& ism, - BRM::LBID_t msgLbidStart, - uint32_t msgLbidCount -) +void pColScanStep::sendAPrimitiveMessage(ISMPacketHeader& ism, BRM::LBID_t msgLbidStart, + uint32_t msgLbidCount) { -// ByteStream bs; -// -// bs.load(reinterpret_cast(&ism), sizeof(ism)); -// -// fMsgHeader.LBID = msgLbidStart; -// fMsgHeader.DataSize = fColType.colWidth; -// fMsgHeader.DataType = fColType.colDataType; -// fMsgHeader.CompType = fColType.compressionType; -// if (fFilterCount > 0) -// fMsgHeader.OutputType = 3; // pairs -// else -// fMsgHeader.OutputType = OT_DATAVALUE; -// fMsgHeader.BOP = fBOP; -// fMsgHeader.NOPS = fFilterCount; -// fMsgHeader.NVALS = 0; -// fMsgHeader.Count = msgLbidCount; -// fMsgHeader.Hdr.SessionID = fSessionId; -// //fMsgHeader.Hdr.StatementID = 0; -// fMsgHeader.Hdr.TransactionID = fTxnId; -// fMsgHeader.Hdr.VerID = fVerId; -// fMsgHeader.Hdr.StepID = fStepId; -// fMsgHeader.Hdr.UniqueID = uniqueID; -// -// bs.append(reinterpret_cast(&fMsgHeader), -// sizeof(fMsgHeader)); -// bs += fFilterString; -// -//#ifdef DEBUG2 -// if (fOid >= 3000) -// cout << "pColScanStep producer st: " << fStepId << -// ": sending req for lbid start " << msgLbidStart << -// "; lbid count " << msgLbidCount << endl; -//#endif -// -// fMsgBytesOut += bs.lengthWithHdrOverhead(); -// fDec->write(bs); -// fMsgsToPm++; + // ByteStream bs; + // + // bs.load(reinterpret_cast(&ism), sizeof(ism)); + // + // fMsgHeader.LBID = msgLbidStart; + // fMsgHeader.DataSize = fColType.colWidth; + // fMsgHeader.DataType = fColType.colDataType; + // fMsgHeader.CompType = fColType.compressionType; + // if (fFilterCount > 0) + // fMsgHeader.OutputType = 3; // pairs + // else + // fMsgHeader.OutputType = OT_DATAVALUE; + // fMsgHeader.BOP = fBOP; + // fMsgHeader.NOPS = fFilterCount; + // fMsgHeader.NVALS = 0; + // fMsgHeader.Count = msgLbidCount; + // fMsgHeader.Hdr.SessionID = fSessionId; + // //fMsgHeader.Hdr.StatementID = 0; + // fMsgHeader.Hdr.TransactionID = fTxnId; + // fMsgHeader.Hdr.VerID = fVerId; + // fMsgHeader.Hdr.StepID = fStepId; + // fMsgHeader.Hdr.UniqueID = uniqueID; + // + // bs.append(reinterpret_cast(&fMsgHeader), + // sizeof(fMsgHeader)); + // bs += fFilterString; + // + //#ifdef DEBUG2 + // if (fOid >= 3000) + // cout << "pColScanStep producer st: " << fStepId << + // ": sending req for lbid start " << msgLbidStart << + // "; lbid count " << msgLbidCount << endl; + //#endif + // + // fMsgBytesOut += bs.lengthWithHdrOverhead(); + // fDec->write(bs); + // fMsgsToPm++; } struct CPInfo { - CPInfo(int64_t MIN, int64_t MAX, uint64_t l) : min(MIN), max(MAX), LBID(l) { }; - int64_t min; - int64_t max; - uint64_t LBID; + CPInfo(int64_t MIN, int64_t MAX, uint64_t l) : min(MIN), max(MAX), LBID(l){}; + int64_t min; + int64_t max; + uint64_t LBID; }; void pColScanStep::receivePrimitiveMessages(uint64_t tid) { -// AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); -// DataList_t* dlp = dl->dataList(); -// FifoDataList *fifo = dl->fifoDL(); -// BucketDL *bucket = dynamic_cast *>(dlp); -// ZDL *zdl = dynamic_cast *>(dlp); -// int64_t l_ridsReturned = 0; -// uint64_t l_physicalIO = 0, l_cachedIO = 0; -// uint64_t fbo; -// uint64_t ridBase; -// vector v; -// UintRowGroup rw; -// vector > bsv; -// uint32_t i, k, size, bsLength; -// bool lastThread = false; -// vector cpv; -// -// if (bucket || zdl) -// dlp->setMultipleProducers(true); -// -// mutex.lock(); //pthread_mutex_lock(&mutex); -// -// // count the LBIDs -// for (; !rDoNothing; ) { -// -// // sync with the send side -// while (!finishedSending && sentCount == recvCount) { -// recvWaiting++; -// condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); -// recvWaiting--; -// } -// if (sentCount == recvCount && finishedSending) { -//// cout << "done recving" << endl; -// break; -// } -// -// fDec->read_some(uniqueID, fNumThreads, bsv); -// for (unsigned int jj=0; jjlengthWithHdrOverhead(); -// } -// -// -// size = bsv.size(); -// -// if (size == 0) { -// /* XXXPAT: Need to give other threads a chance to update recvCount. -// As of 2/25/08, each BS contains multiple responses. With the current -// protocol, the exact # isn't known until they're processed. Threads -// waiting for more input will have to busy wait on either recvCount -// being updated or input. It's only an issue at the tail end of the -// responses, and probably doesn't matter then either. */ -// -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// usleep(1000); // 1ms Good? -// mutex.lock(); //pthread_mutex_lock(&mutex); -// continue; -// } -// -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) -// dlTimes.setFirstReadTime(); -// if (fOid>=3000) dlTimes.setLastReadTime(); -// -//// cerr << "got a response of " << size << " msgs\n"; -// -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// uint32_t msgCount = 0; -// for (i = 0; i < size; i++) { -// const ByteStream::byte* bsp = bsv[i]->buf(); -// -// bsLength = bsv[i]->length(); -// k = 0; -// while (k < bsLength) { -// ++msgCount; -//// cout << "got msg " << msgCount << " k = " << k << endl; -// k += sizeof(ISMPacketHeader); -// const ColResultHeader* crh = reinterpret_cast(&bsp[k]); -// // get the ColumnResultHeader out of the bytestream -// k += sizeof(ColResultHeader); -// -// l_cachedIO += crh->CacheIO; -// l_physicalIO += crh->PhysicalIO; -// fbo = getFBO(crh->LBID); -// ridBase = fbo << rpbShift; -// -// for(int j = 0; j < crh->NVALS; j++) -// { -// uint64_t dv; -// uint64_t rid; -// -// if (crh->OutputType == OT_DATAVALUE) { -// if (isEmptyVal(&bsp[k])) { -// k += fColType.colWidth; -// continue; -// } -// rid = j + ridBase; -// } -// else { -// rid = *((const uint16_t *) &bsp[k]) + ridBase; -// k += sizeof(uint16_t); -// } -// -// switch (fColType.colWidth) { -// case 8: dv = *((const uint64_t *) &bsp[k]); k += 8; break; -// case 4: dv = *((const uint32_t *) &bsp[k]); k += 4; break; -// case 2: dv = *((const uint16_t *) &bsp[k]); k += 2; break; -// case 1: dv = *((const uint8_t *) &bsp[k]); ++k; break; -// default: -// throw runtime_error("pColStep: invalid column width!"); -// } -// -// v.push_back(ElementType(rid, dv)); -// ++l_ridsReturned; -//#ifdef DEBUG -//// if (fOid >=3000) -//// cout << " -- inserting <" << rid << ", " << dv << ">" << endl; -//#endif -// // per row operations... -// } // for -// -// // per block operations... -// -//#ifdef DEBUG -// cout << "recvPrimMsgs Oid " << fOid -// << " valid " << crh->ValidMinMax -// << " LBID " << crh->LBID -// << " Mn/Mx " << crh->Min << "/" << crh->Max -// << " nvals " << crh->NVALS -// << "/" << l_ridsReturned << endl; -//#endif -// if (fOid >= 3000 && crh->ValidMinMax) -// cpv.push_back(CPInfo(crh->Min, crh->Max, crh->LBID)); -// } -// // per ByteStream operations... -// } -// // per read operations.... -// -// if (bucket) { -// if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) -// dlTimes.setFirstInsertTime(); -// -// bucket->insert(v); -// } -// else if (zdl) { -// zdl->insert(v); -// } -// else { -// size = v.size(); -// if (size>0) -// if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) -// dlTimes.setFirstInsertTime(); -// -// dlMutex.lock(); //pthread_mutex_lock(&dlMutex); -// for (i = 0; i < size; ++i) { -// rw.et[rw.count++] = v[i]; -// if (rw.count == rw.ElementsPerGroup) -// { -// fifo->insert(rw); -// rw.count = 0; -// } -// } -// -// if (rw.count > 0) -// { -// fifo->insert(rw); -// rw.count = 0; -// } -// -// dlMutex.unlock(); //pthread_mutex_unlock(&dlMutex); -// } -// v.clear(); -// -// size = cpv.size(); -// if (size > 0) { -// cpMutex.lock(); //pthread_mutex_lock(&cpMutex); -// for (i = 0; i < size; i++) { -// CPInfo *cpi = &(cpv[i]); -// lbidList->UpdateMinMax(cpi->min, cpi->max, cpi->LBID, fColType.colDataType); -// } -// cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); -// cpv.clear(); -// } -// -// mutex.lock(); //pthread_mutex_lock(&mutex); -// recvCount += msgCount; -// //...If producer is waiting, and we have gone below our threshold value, -// //...then we signal the producer to request more data from primproc -// if ( (sendWaiting) && ( (sentCount - recvCount) < fScanLbidReqThreshold ) ) -// { -//#ifdef DEBUG2 -// if (fOid >= 3000) -// cout << "pColScanStep consumer signaling producer for more data: "<< -// "st:" << fStepId << -// "; sentCount-" << sentCount << -// "; recvCount-" << recvCount << -// "; threshold-" << fScanLbidReqThreshold << endl; -//#endif -// condvarWakeupProducer.notify_one(); //pthread_cond_signal(&condvarWakeupProducer); -// } -// } // end of loop to read LBID responses from primproc -// -// fPhysicalIO += l_physicalIO; -// fCacheIO += l_cachedIO; -// ridsReturned += l_ridsReturned; -//// cerr << "out of the main loop " << recvExited << endl; -// if (++recvExited == fNumThreads) { -// //...Casual partitioning could cause us to do no processing. In that -// //...case these time stamps did not get set. So we set them here. -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { -// dlTimes.setFirstReadTime(); -// dlTimes.setLastReadTime(); -// dlTimes.setFirstInsertTime(); -// } -// if (fOid>=3000) dlTimes.setEndOfInputTime(); -// -// //@bug 699: Reset StepMsgQueue -// fDec->removeQueue(uniqueID); -// -// if (fifo) -// fifo->endOfInput(); -// else -// dlp->endOfInput(); -// lastThread = true; -// } -// -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// if (fTableOid >= 3000 && lastThread) -// { -// //...Construct timestamp using ctime_r() instead of ctime() not -// //...necessarily due to re-entrancy, but because we want to strip -// //...the newline ('\n') off the end of the formatted string. -// time_t t = time(0); -// char timeString[50]; -// ctime_r(&t, timeString); -// timeString[strlen(timeString)-1 ] = '\0'; -// -// FifoDataList* pFifo = 0; -// uint64_t totalBlockedReadCount = 0; -// uint64_t totalBlockedWriteCount = 0; -// -// //...Sum up the blocked FIFO reads for all input associations -// size_t inDlCnt = fInputJobStepAssociation.outSize(); -// for (size_t iDataList=0; iDataListfifoDL(); -// if (pFifo) -// { -// totalBlockedReadCount += pFifo->blockedReadCount(); -// } -// } -// -// //...Sum up the blocked FIFO writes for all output associations -// size_t outDlCnt = fOutputJobStepAssociation.outSize(); -// for (size_t iDataList=0; iDataListfifoDL(); -// if (pFifo) -// { -// totalBlockedWriteCount += pFifo->blockedWriteCount(); -// } -// } -// -// //...Roundoff inbound msg byte count to nearest KB for display; -// //...no need to do so for outbound, because it should be small. -// uint64_t msgBytesInKB = fMsgBytesIn >> 10; -// if (fMsgBytesIn & 512) -// msgBytesInKB++; -// // @bug 807 -// if (fifo) -// fifo->totalSize(ridsReturned); -// -// if (traceOn()) -// { -// //...Print job step completion information -// ostringstream logStr; -// logStr << "ses:" << fSessionId << -// " st: " << fStepId << " finished at " << -// timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << -// fCacheIO << "; MsgsSent-" << fMsgsToPm << "; MsgsRcvd-" << recvCount << -// "; BlockedFifoIn/Out-" << totalBlockedReadCount << -// "/" << totalBlockedWriteCount << -// "; output size-" << ridsReturned << endl << -// "\tPartitionBlocksEliminated-" << fNumBlksSkipped << -// "; MsgBytesIn-" << msgBytesInKB << "KB" << -// "; MsgBytesOut-" << fMsgBytesOut << "B" << endl << -// "\t1st read " << dlTimes.FirstReadTimeString() << -// "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << -// JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << -// "s" << endl; -// -// logEnd(logStr.str().c_str()); -// -// syslogReadBlockCounts(16, // exemgr subsystem -// fPhysicalIO, // # blocks read from disk -// fCacheIO, // # blocks read from cache -// fNumBlksSkipped); // # casual partition block hits -// syslogProcessingTimes(16, // exemgr subsystem -// dlTimes.FirstReadTime(), // first datalist read -// dlTimes.LastReadTime(), // last datalist read -// dlTimes.FirstInsertTime(), // first datalist write -// dlTimes.EndOfInputTime()); // last (endOfInput) datalist write -// syslogEndStep(16, // exemgr subsystem -// totalBlockedReadCount, // blocked datalist input -// totalBlockedWriteCount, // blocked datalist output -// fMsgBytesIn, // incoming msg byte count -// fMsgBytesOut); // outgoing msg byte count -// } -// -// } -// -// if (fOid >=3000 && lastThread) -// lbidList->UpdateAllPartitionInfo(); -// -//// cerr << "recv thread exiting" << endl; + // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); + // DataList_t* dlp = dl->dataList(); + // FifoDataList *fifo = dl->fifoDL(); + // BucketDL *bucket = dynamic_cast *>(dlp); + // ZDL *zdl = dynamic_cast *>(dlp); + // int64_t l_ridsReturned = 0; + // uint64_t l_physicalIO = 0, l_cachedIO = 0; + // uint64_t fbo; + // uint64_t ridBase; + // vector v; + // UintRowGroup rw; + // vector > bsv; + // uint32_t i, k, size, bsLength; + // bool lastThread = false; + // vector cpv; + // + // if (bucket || zdl) + // dlp->setMultipleProducers(true); + // + // mutex.lock(); //pthread_mutex_lock(&mutex); + // + // // count the LBIDs + // for (; !rDoNothing; ) { + // + // // sync with the send side + // while (!finishedSending && sentCount == recvCount) { + // recvWaiting++; + // condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); + // recvWaiting--; + // } + // if (sentCount == recvCount && finishedSending) { + //// cout << "done recving" << endl; + // break; + // } + // + // fDec->read_some(uniqueID, fNumThreads, bsv); + // for (unsigned int jj=0; jjlengthWithHdrOverhead(); + // } + // + // + // size = bsv.size(); + // + // if (size == 0) { + // /* XXXPAT: Need to give other threads a chance to update recvCount. + // As of 2/25/08, each BS contains multiple responses. With the current + // protocol, the exact # isn't known until they're processed. Threads + // waiting for more input will have to busy wait on either recvCount + // being updated or input. It's only an issue at the tail end of the + // responses, and probably doesn't matter then either. */ + // + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // usleep(1000); // 1ms Good? + // mutex.lock(); //pthread_mutex_lock(&mutex); + // continue; + // } + // + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) + // dlTimes.setFirstReadTime(); + // if (fOid>=3000) dlTimes.setLastReadTime(); + // + //// cerr << "got a response of " << size << " msgs\n"; + // + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // uint32_t msgCount = 0; + // for (i = 0; i < size; i++) { + // const ByteStream::byte* bsp = bsv[i]->buf(); + // + // bsLength = bsv[i]->length(); + // k = 0; + // while (k < bsLength) { + // ++msgCount; + //// cout << "got msg " << msgCount << " k = " << k << endl; + // k += sizeof(ISMPacketHeader); + // const ColResultHeader* crh = reinterpret_cast(&bsp[k]); + // // get the ColumnResultHeader out of the bytestream + // k += sizeof(ColResultHeader); + // + // l_cachedIO += crh->CacheIO; + // l_physicalIO += crh->PhysicalIO; + // fbo = getFBO(crh->LBID); + // ridBase = fbo << rpbShift; + // + // for(int j = 0; j < crh->NVALS; j++) + // { + // uint64_t dv; + // uint64_t rid; + // + // if (crh->OutputType == OT_DATAVALUE) { + // if (isEmptyVal(&bsp[k])) { + // k += fColType.colWidth; + // continue; + // } + // rid = j + ridBase; + // } + // else { + // rid = *((const uint16_t *) &bsp[k]) + ridBase; + // k += sizeof(uint16_t); + // } + // + // switch (fColType.colWidth) { + // case 8: dv = *((const uint64_t *) &bsp[k]); k += 8; break; + // case 4: dv = *((const uint32_t *) &bsp[k]); k += 4; break; + // case 2: dv = *((const uint16_t *) &bsp[k]); k += 2; break; + // case 1: dv = *((const uint8_t *) &bsp[k]); ++k; break; + // default: + // throw runtime_error("pColStep: invalid column + //width!"); + // } + // + // v.push_back(ElementType(rid, dv)); + // ++l_ridsReturned; + //#ifdef DEBUG + //// if (fOid >=3000) + //// cout << " -- inserting <" << rid << ", " << dv << ">" << + ///endl; + //#endif + // // per row operations... + // } // for + // + // // per block operations... + // + //#ifdef DEBUG + // cout << "recvPrimMsgs Oid " << fOid + // << " valid " << crh->ValidMinMax + // << " LBID " << crh->LBID + // << " Mn/Mx " << crh->Min << "/" << crh->Max + // << " nvals " << crh->NVALS + // << "/" << l_ridsReturned << endl; + //#endif + // if (fOid >= 3000 && crh->ValidMinMax) + // cpv.push_back(CPInfo(crh->Min, crh->Max, crh->LBID)); + // } + // // per ByteStream operations... + // } + // // per read operations.... + // + // if (bucket) { + // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) + // dlTimes.setFirstInsertTime(); + // + // bucket->insert(v); + // } + // else if (zdl) { + // zdl->insert(v); + // } + // else { + // size = v.size(); + // if (size>0) + // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) + // dlTimes.setFirstInsertTime(); + // + // dlMutex.lock(); //pthread_mutex_lock(&dlMutex); + // for (i = 0; i < size; ++i) { + // rw.et[rw.count++] = v[i]; + // if (rw.count == rw.ElementsPerGroup) + // { + // fifo->insert(rw); + // rw.count = 0; + // } + // } + // + // if (rw.count > 0) + // { + // fifo->insert(rw); + // rw.count = 0; + // } + // + // dlMutex.unlock(); //pthread_mutex_unlock(&dlMutex); + // } + // v.clear(); + // + // size = cpv.size(); + // if (size > 0) { + // cpMutex.lock(); //pthread_mutex_lock(&cpMutex); + // for (i = 0; i < size; i++) { + // CPInfo *cpi = &(cpv[i]); + // lbidList->UpdateMinMax(cpi->min, cpi->max, cpi->LBID, fColType.colDataType); + // } + // cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); + // cpv.clear(); + // } + // + // mutex.lock(); //pthread_mutex_lock(&mutex); + // recvCount += msgCount; + // //...If producer is waiting, and we have gone below our threshold value, + // //...then we signal the producer to request more data from primproc + // if ( (sendWaiting) && ( (sentCount - recvCount) < fScanLbidReqThreshold ) ) + // { + //#ifdef DEBUG2 + // if (fOid >= 3000) + // cout << "pColScanStep consumer signaling producer for more data: "<< + // "st:" << fStepId << + // "; sentCount-" << sentCount << + // "; recvCount-" << recvCount << + // "; threshold-" << fScanLbidReqThreshold << endl; + //#endif + // condvarWakeupProducer.notify_one(); //pthread_cond_signal(&condvarWakeupProducer); + // } + // } // end of loop to read LBID responses from primproc + // + // fPhysicalIO += l_physicalIO; + // fCacheIO += l_cachedIO; + // ridsReturned += l_ridsReturned; + //// cerr << "out of the main loop " << recvExited << endl; + // if (++recvExited == fNumThreads) { + // //...Casual partitioning could cause us to do no processing. In that + // //...case these time stamps did not get set. So we set them here. + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { + // dlTimes.setFirstReadTime(); + // dlTimes.setLastReadTime(); + // dlTimes.setFirstInsertTime(); + // } + // if (fOid>=3000) dlTimes.setEndOfInputTime(); + // + // //@bug 699: Reset StepMsgQueue + // fDec->removeQueue(uniqueID); + // + // if (fifo) + // fifo->endOfInput(); + // else + // dlp->endOfInput(); + // lastThread = true; + // } + // + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // if (fTableOid >= 3000 && lastThread) + // { + // //...Construct timestamp using ctime_r() instead of ctime() not + // //...necessarily due to re-entrancy, but because we want to strip + // //...the newline ('\n') off the end of the formatted string. + // time_t t = time(0); + // char timeString[50]; + // ctime_r(&t, timeString); + // timeString[strlen(timeString)-1 ] = '\0'; + // + // FifoDataList* pFifo = 0; + // uint64_t totalBlockedReadCount = 0; + // uint64_t totalBlockedWriteCount = 0; + // + // //...Sum up the blocked FIFO reads for all input associations + // size_t inDlCnt = fInputJobStepAssociation.outSize(); + // for (size_t iDataList=0; iDataListfifoDL(); + // if (pFifo) + // { + // totalBlockedReadCount += pFifo->blockedReadCount(); + // } + // } + // + // //...Sum up the blocked FIFO writes for all output associations + // size_t outDlCnt = fOutputJobStepAssociation.outSize(); + // for (size_t iDataList=0; iDataListfifoDL(); + // if (pFifo) + // { + // totalBlockedWriteCount += pFifo->blockedWriteCount(); + // } + // } + // + // //...Roundoff inbound msg byte count to nearest KB for display; + // //...no need to do so for outbound, because it should be small. + // uint64_t msgBytesInKB = fMsgBytesIn >> 10; + // if (fMsgBytesIn & 512) + // msgBytesInKB++; + // // @bug 807 + // if (fifo) + // fifo->totalSize(ridsReturned); + // + // if (traceOn()) + // { + // //...Print job step completion information + // ostringstream logStr; + // logStr << "ses:" << fSessionId << + // " st: " << fStepId << " finished at " << + // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << + // fCacheIO << "; MsgsSent-" << fMsgsToPm << "; MsgsRcvd-" << recvCount << + // "; BlockedFifoIn/Out-" << totalBlockedReadCount << + // "/" << totalBlockedWriteCount << + // "; output size-" << ridsReturned << endl << + // "\tPartitionBlocksEliminated-" << fNumBlksSkipped << + // "; MsgBytesIn-" << msgBytesInKB << "KB" << + // "; MsgBytesOut-" << fMsgBytesOut << "B" << endl << + // "\t1st read " << dlTimes.FirstReadTimeString() << + // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << + // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << + // "s" << endl; + // + // logEnd(logStr.str().c_str()); + // + // syslogReadBlockCounts(16, // exemgr subsystem + // fPhysicalIO, // # blocks read from disk + // fCacheIO, // # blocks read from cache + // fNumBlksSkipped); // # casual partition block hits + // syslogProcessingTimes(16, // exemgr subsystem + // dlTimes.FirstReadTime(), // first datalist read + // dlTimes.LastReadTime(), // last datalist read + // dlTimes.FirstInsertTime(), // first datalist write + // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write + // syslogEndStep(16, // exemgr subsystem + // totalBlockedReadCount, // blocked datalist input + // totalBlockedWriteCount, // blocked datalist output + // fMsgBytesIn, // incoming msg byte count + // fMsgBytesOut); // outgoing msg byte count + // } + // + // } + // + // if (fOid >=3000 && lastThread) + // lbidList->UpdateAllPartitionInfo(); + // + //// cerr << "recv thread exiting" << endl; } void pColScanStep::addFilter(int8_t COP, float value) { - fFilterString << (uint8_t) COP; - fFilterString << (uint8_t) 0; - fFilterString << *((uint32_t*) &value); - fFilterCount++; + fFilterString << (uint8_t)COP; + fFilterString << (uint8_t)0; + fFilterString << *((uint32_t*)&value); + fFilterCount++; } void pColScanStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) { - int8_t tmp8; - int16_t tmp16; - int32_t tmp32; + int8_t tmp8; + int16_t tmp16; + int32_t tmp32; - fFilterString << (uint8_t) COP; - fFilterString << roundFlag; + fFilterString << (uint8_t)COP; + fFilterString << roundFlag; - // converts to a type of the appropriate width, then bitwise - // copies into the filter ByteStream - switch (fColType.colWidth) - { - case 1: - tmp8 = value; - fFilterString << *((uint8_t*) &tmp8); - break; + // converts to a type of the appropriate width, then bitwise + // copies into the filter ByteStream + switch (fColType.colWidth) + { + case 1: + tmp8 = value; + fFilterString << *((uint8_t*)&tmp8); + break; - case 2: - tmp16 = value; - fFilterString << *((uint16_t*) &tmp16); - break; + case 2: + tmp16 = value; + fFilterString << *((uint16_t*)&tmp16); + break; - case 4: - tmp32 = value; - fFilterString << *((uint32_t*) &tmp32); - break; + case 4: + tmp32 = value; + fFilterString << *((uint32_t*)&tmp32); + break; - case 8: - fFilterString << *((uint64_t*) &value); - break; + case 8: fFilterString << *((uint64_t*)&value); break; - default: - ostringstream o; + default: + ostringstream o; - o << "pColScanStep: CalpontSystemCatalog says OID " << fOid << - " has a width of " << fColType.colWidth; - throw runtime_error(o.str()); - } + o << "pColScanStep: CalpontSystemCatalog says OID " << fOid << " has a width of " << fColType.colWidth; + throw runtime_error(o.str()); + } - fFilterCount++; + fFilterCount++; } void pColScanStep::setBOP(int8_t B) { - fBOP = B; + fBOP = B; } -void pColScanStep::setSingleThread( bool b) +void pColScanStep::setSingleThread(bool b) { - fSingleThread = b; - fNumThreads = 1; + fSingleThread = b; + fNumThreads = 1; } void pColScanStep::setOutputType(int8_t OutputType) { - fOutputType = OutputType; + fOutputType = OutputType; } const string pColScanStep::toString() const { - ostringstream oss; - oss << "pColScanStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId << - " tb/col:" << fTableOid << "/" << fOid; + ostringstream oss; + oss << "pColScanStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId + << " tb/col:" << fTableOid << "/" << fOid; - if (alias().length()) oss << " alias:" << alias(); + if (alias().length()) + oss << " alias:" << alias(); - oss << " " << omitOidInDL - << fOutputJobStepAssociation.outAt(0) << showOidInDL; - oss << " nf:" << fFilterCount; - oss << " in:"; + oss << " " << omitOidInDL << fOutputJobStepAssociation.outAt(0) << showOidInDL; + oss << " nf:" << fFilterCount; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - { - oss << fInputJobStepAssociation.outAt(i) << ", "; - } + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + oss << fInputJobStepAssociation.outAt(i) << ", "; + } - return oss.str(); + return oss.str(); } uint64_t pColScanStep::getFBO(uint64_t lbid) { - uint32_t i; - uint64_t lastLBID; + uint32_t i; + uint64_t lastLBID; - for (i = 0; i < numExtents; i++) - { - lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; + for (i = 0; i < numExtents; i++) + { + lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; -// lastLBID = extents[i].range.start + (extents[i].range.size * 1024) - 1; -// cerr << "start: " << extents[i].range.start << " end:" << lastLBID <= (uint64_t) extents[i].range.start && lbid <= lastLBID) -// return (lbid - extents[i].range.start) + (extentSize * i); - return (lbid - extents[i].range.start) + (i << divShift); - } + // lastLBID = extents[i].range.start + (extents[i].range.size * 1024) - 1; + // cerr << "start: " << extents[i].range.start << " end:" << lastLBID <= (uint64_t)extents[i].range.start && lbid <= lastLBID) + // return (lbid - extents[i].range.start) + (extentSize * i); + return (lbid - extents[i].range.start) + (i << divShift); + } - cerr << "pColScan: didn't find the FBO?\n"; - throw logic_error("pColScan: didn't find the FBO?"); + cerr << "pColScan: didn't find the FBO?\n"; + throw logic_error("pColScan: didn't find the FBO?"); } -pColScanStep::pColScanStep(const pColStep& rhs) : - JobStep(rhs), - fRm(rhs.resourceManager()), - fMsgHeader() +pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resourceManager()), fMsgHeader() { - fNumThreads = fRm->getJlNumScanReceiveThreads(); - fFilterCount = rhs.filterCount(); - fFilterString = rhs.filterString(); - isFilterFeeder = rhs.getFeederFlag(); - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - fColType = rhs.colType(); - fBOP = rhs.BOP(); - fIsDict = rhs.isDictCol(); - sentCount = 0; - recvCount = 0; - fScanLbidReqLimit = fRm->getJlScanLbidReqLimit(); - fScanLbidReqThreshold = fRm->getJlScanLbidReqThreshold(); - fStopSending = false; - fSingleThread = false; - fPhysicalIO = 0; - fCacheIO = 0; - fNumBlksSkipped = 0; - fMsgBytesIn = 0; - fMsgBytesOut = 0; - fMsgsToPm = 0; - fCardinality = rhs.cardinality(); - fFilters = rhs.fFilters; - fOnClauseFilter = rhs.onClauseFilter(); + fNumThreads = fRm->getJlNumScanReceiveThreads(); + fFilterCount = rhs.filterCount(); + fFilterString = rhs.filterString(); + isFilterFeeder = rhs.getFeederFlag(); + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + fColType = rhs.colType(); + fBOP = rhs.BOP(); + fIsDict = rhs.isDictCol(); + sentCount = 0; + recvCount = 0; + fScanLbidReqLimit = fRm->getJlScanLbidReqLimit(); + fScanLbidReqThreshold = fRm->getJlScanLbidReqThreshold(); + fStopSending = false; + fSingleThread = false; + fPhysicalIO = 0; + fCacheIO = 0; + fNumBlksSkipped = 0; + fMsgBytesIn = 0; + fMsgBytesOut = 0; + fMsgsToPm = 0; + fCardinality = rhs.cardinality(); + fFilters = rhs.fFilters; + fOnClauseFilter = rhs.onClauseFilter(); - if (fTableOid == 0) // cross engine support - return; + if (fTableOid == 0) // cross engine support + return; - int err; + int err; - err = dbrm.lookup(fOid, lbidRanges); + err = dbrm.lookup(fOid, lbidRanges); - if (err) - throw runtime_error("pColScan: BRM LBID range lookup failure (1)"); + if (err) + throw runtime_error("pColScan: BRM LBID range lookup failure (1)"); - err = dbrm.getExtents(fOid, extents); + err = dbrm.getExtents(fOid, extents); - if (err) - throw runtime_error("pColScan: BRM HWM lookup failure (4)"); + if (err) + throw runtime_error("pColScan: BRM HWM lookup failure (4)"); - sort(extents.begin(), extents.end(), BRM::ExtentSorter()); - numExtents = extents.size(); - extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; - lbidList = rhs.lbidList; - //pthread_mutex_init(&mutex, NULL); - //pthread_mutex_init(&dlMutex, NULL); - //pthread_mutex_init(&cpMutex, NULL); - //pthread_cond_init(&condvar, NULL); - //pthread_cond_init(&condvarWakeupProducer, NULL); - finishedSending = sendWaiting = rDoNothing = false; - recvWaiting = 0; - recvExited = 0; + sort(extents.begin(), extents.end(), BRM::ExtentSorter()); + numExtents = extents.size(); + extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; + lbidList = rhs.lbidList; + // pthread_mutex_init(&mutex, NULL); + // pthread_mutex_init(&dlMutex, NULL); + // pthread_mutex_init(&cpMutex, NULL); + // pthread_cond_init(&condvar, NULL); + // pthread_cond_init(&condvarWakeupProducer, NULL); + finishedSending = sendWaiting = rDoNothing = false; + recvWaiting = 0; + recvExited = 0; - /* calculate some shortcuts for extent-based arithmetic */ - ridsPerBlock = rhs.ridsPerBlock; - rpbShift = rhs.rpbShift; - divShift = rhs.divShift; + /* calculate some shortcuts for extent-based arithmetic */ + ridsPerBlock = rhs.ridsPerBlock; + rpbShift = rhs.rpbShift; + divShift = rhs.divShift; -// initializeConfigParms ( ); - fTraceFlags = rhs.fTraceFlags; -// uniqueID = UniqueNumberGenerator::instance()->getUnique32(); -// if (fDec) -// fDec->addQueue(uniqueID); -// fProducerThread = new SPTHD[fNumThreads]; + // initializeConfigParms ( ); + fTraceFlags = rhs.fTraceFlags; + // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + // if (fDec) + // fDec->addQueue(uniqueID); + // fProducerThread = new SPTHD[fNumThreads]; } void pColScanStep::addFilters() { - AnyDataListSPtr dl = fInputJobStepAssociation.outAt(0); - DataList_t* bdl = dl->dataList(); - idbassert(bdl); - int it = -1; - bool more; - ElementType e; - int64_t token; + AnyDataListSPtr dl = fInputJobStepAssociation.outAt(0); + DataList_t* bdl = dl->dataList(); + idbassert(bdl); + int it = -1; + bool more; + ElementType e; + int64_t token; - try - { - it = bdl->getIterator(); - } - catch (std::exception& ex) - { - cerr << "pColScanStep::addFilters: caught exception: " - << ex.what() << " stepno: " << fStepId << endl; - throw; - } - catch (...) - { - cerr << "pColScanStep::addFilters: caught exception" << endl; - throw; - } + try + { + it = bdl->getIterator(); + } + catch (std::exception& ex) + { + cerr << "pColScanStep::addFilters: caught exception: " << ex.what() << " stepno: " << fStepId << endl; + throw; + } + catch (...) + { + cerr << "pColScanStep::addFilters: caught exception" << endl; + throw; + } - fBOP = BOP_OR; + fBOP = BOP_OR; + more = bdl->next(it, &e); + + while (more) + { + token = e.second; + addFilter(COMPARE_EQ, token); more = bdl->next(it, &e); + } - while (more) - { - token = e.second; - addFilter(COMPARE_EQ, token); - more = bdl->next(it, &e); - } - - return; + return; } bool pColScanStep::isEmptyVal(const uint8_t* val8) const { - const int width = fColType.colWidth; + const int width = fColType.colWidth; - switch (fColType.colDataType) + switch (fColType.colDataType) + { + case CalpontSystemCatalog::UTINYINT: { - case CalpontSystemCatalog::UTINYINT: - { - return (*val8 == joblist::UTINYINTEMPTYROW); - } - - case CalpontSystemCatalog::USMALLINT: - { - const uint16_t* val16 = reinterpret_cast(val8); - return (*val16 == joblist::USMALLINTEMPTYROW); - } - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - { - const uint32_t* val32 = reinterpret_cast(val8); - return (*val32 == joblist::UINTEMPTYROW); - } - - case CalpontSystemCatalog::UBIGINT: - { - const uint64_t* val64 = reinterpret_cast(val8); - return (*val64 == joblist::BIGINTEMPTYROW); - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - const uint32_t* val32 = reinterpret_cast(val8); - return (*val32 == joblist::FLOATEMPTYROW); - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - const uint64_t* val64 = reinterpret_cast(val8); - return (*val64 == joblist::DOUBLEEMPTYROW); - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - if (width == 1) - { - return (*val8 == joblist::CHAR1EMPTYROW); - } - else if (width == 2) - { - const uint16_t* val16 = reinterpret_cast(val8); - return (*val16 == joblist::CHAR2EMPTYROW); - } - else if (width <= 4) - { - const uint32_t* val32 = reinterpret_cast(val8); - return (*val32 == joblist::CHAR4EMPTYROW); - } - else if (width <= 8) - { - const uint64_t* val64 = reinterpret_cast(val8); - return (*val64 == joblist::CHAR8EMPTYROW); - } - - default: - break; + return (*val8 == joblist::UTINYINTEMPTYROW); } - switch (width) + case CalpontSystemCatalog::USMALLINT: { - case 1: - { - return (*val8 == joblist::TINYINTEMPTYROW); - } - - case 2: - { - const uint16_t* val16 = reinterpret_cast(val8); - return (*val16 == joblist::SMALLINTEMPTYROW); - } - - case 4: - { - const uint32_t* val32 = reinterpret_cast(val8); - return (*val32 == joblist::INTEMPTYROW); - } - - case 8: - { - const uint64_t* val64 = reinterpret_cast(val8); - return (*val64 == joblist::BIGINTEMPTYROW); - } - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(33); - - colWidth.add(width); - msg.format(colWidth); - logger.logErrorMessage(msg); - return false; + const uint16_t* val16 = reinterpret_cast(val8); + return (*val16 == joblist::USMALLINTEMPTYROW); } - return false; + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + { + const uint32_t* val32 = reinterpret_cast(val8); + return (*val32 == joblist::UINTEMPTYROW); + } + + case CalpontSystemCatalog::UBIGINT: + { + const uint64_t* val64 = reinterpret_cast(val8); + return (*val64 == joblist::BIGINTEMPTYROW); + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + const uint32_t* val32 = reinterpret_cast(val8); + return (*val32 == joblist::FLOATEMPTYROW); + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + const uint64_t* val64 = reinterpret_cast(val8); + return (*val64 == joblist::DOUBLEEMPTYROW); + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + if (width == 1) + { + return (*val8 == joblist::CHAR1EMPTYROW); + } + else if (width == 2) + { + const uint16_t* val16 = reinterpret_cast(val8); + return (*val16 == joblist::CHAR2EMPTYROW); + } + else if (width <= 4) + { + const uint32_t* val32 = reinterpret_cast(val8); + return (*val32 == joblist::CHAR4EMPTYROW); + } + else if (width <= 8) + { + const uint64_t* val64 = reinterpret_cast(val8); + return (*val64 == joblist::CHAR8EMPTYROW); + } + + default: break; + } + + switch (width) + { + case 1: + { + return (*val8 == joblist::TINYINTEMPTYROW); + } + + case 2: + { + const uint16_t* val16 = reinterpret_cast(val8); + return (*val16 == joblist::SMALLINTEMPTYROW); + } + + case 4: + { + const uint32_t* val32 = reinterpret_cast(val8); + return (*val32 == joblist::INTEMPTYROW); + } + + case 8: + { + const uint64_t* val64 = reinterpret_cast(val8); + return (*val64 == joblist::BIGINTEMPTYROW); + } + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(33); + + colWidth.add(width); + msg.format(colWidth); + logger.logErrorMessage(msg); + return false; + } + + return false; } - void pColScanStep::addFilter(const Filter* f) { - if (NULL != f) - fFilters.push_back(f); + if (NULL != f) + fFilters.push_back(f); } - void pColScanStep::appendFilter(const std::vector& fs) { - fFilters.insert(fFilters.end(), fs.begin(), fs.end()); + fFilters.insert(fFilters.end(), fs.begin(), fs.end()); } -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index c087525ac..eefaf02e1 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: pcolstep.cpp 9655 2013-06-25 23:08:13Z xlou $ -* -* -***********************************************************************/ + * $Id: pcolstep.cpp 9655 2013-06-25 23:08:13Z xlou $ + * + * + ***********************************************************************/ #include #include #include @@ -99,395 +99,393 @@ struct pColStepAggregator }; #endif -pColStep::pColStep( - CalpontSystemCatalog::OID o, - CalpontSystemCatalog::OID t, - const CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fRm(jobInfo.rm), - sysCat(jobInfo.csc), - fOid(o), - fTableOid(t), - fColType(ct), - fFilterCount(0), - fBOP(BOP_NONE), - ridList(0), - msgsSent(0), - msgsRecvd(0), - finishedSending(false), - recvWaiting(false), - fIsDict(false), - isEM(jobInfo.isExeMgr), - ridCount(0), - fFlushInterval(jobInfo.flushInterval), - fSwallowRows(false), - fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()), - fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()), - fStopSending(false), - isFilterFeeder(false), - fPhysicalIO(0), - fCacheIO(0), - fNumBlksSkipped(0), - fMsgBytesIn(0), - fMsgBytesOut(0) +pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, + const CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) + : JobStep(jobInfo) + , fRm(jobInfo.rm) + , sysCat(jobInfo.csc) + , fOid(o) + , fTableOid(t) + , fColType(ct) + , fFilterCount(0) + , fBOP(BOP_NONE) + , ridList(0) + , msgsSent(0) + , msgsRecvd(0) + , finishedSending(false) + , recvWaiting(false) + , fIsDict(false) + , isEM(jobInfo.isExeMgr) + , ridCount(0) + , fFlushInterval(jobInfo.flushInterval) + , fSwallowRows(false) + , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) + , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) + , fStopSending(false) + , isFilterFeeder(false) + , fPhysicalIO(0) + , fCacheIO(0) + , fNumBlksSkipped(0) + , fMsgBytesIn(0) + , fMsgBytesOut(0) { - if (fTableOid == 0) // cross engine support - return; + if (fTableOid == 0) // cross engine support + return; - int err, i; - uint32_t mask; + int err, i; + uint32_t mask; - if (fFlushInterval == 0 || !isEM) - fOutputType = OT_BOTH; - else - fOutputType = OT_TOKEN; + if (fFlushInterval == 0 || !isEM) + fOutputType = OT_BOTH; + else + fOutputType = OT_TOKEN; - if (fOid < 1000) - throw runtime_error("pColStep: invalid column"); + if (fOid < 1000) + throw runtime_error("pColStep: invalid column"); - if (!compress::CompressInterface::isCompressionAvail(fColType.compressionType)) - { - ostringstream oss; - oss << "Unsupported compression type " << fColType.compressionType; - oss << " for " << sysCat->colName(fOid); + if (!compress::CompressInterface::isCompressionAvail(fColType.compressionType)) + { + ostringstream oss; + oss << "Unsupported compression type " << fColType.compressionType; + oss << " for " << sysCat->colName(fOid); #ifdef SKIP_IDB_COMPRESSION - oss << ". It looks you're running Community binaries on an Enterprise database."; + oss << ". It looks you're running Community binaries on an Enterprise database."; #endif - throw runtime_error(oss.str()); - } + throw runtime_error(oss.str()); + } - realWidth = fColType.colWidth; + realWidth = fColType.colWidth; - if ( fColType.colDataType == CalpontSystemCatalog::VARCHAR ) + if (fColType.colDataType == CalpontSystemCatalog::VARCHAR) + { + if (8 > fColType.colWidth && 4 <= fColType.colWidth) + fColType.colDataType = CalpontSystemCatalog::CHAR; + + fColType.colWidth++; + } + + // If this is a dictionary column, fudge the numbers... + if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY) || + (fColType.colDataType == CalpontSystemCatalog::BLOB) || + (fColType.colDataType == CalpontSystemCatalog::TEXT)) + { + fColType.colWidth = 8; + fIsDict = true; + } + // WIP MCOL-641 + else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::DECIMAL && + fColType.colDataType != CalpontSystemCatalog::UDECIMAL) + { + fColType.colWidth = 8; + fIsDict = true; + // TODO: is this right? + fColType.colDataType = CalpontSystemCatalog::VARCHAR; + } + + // Round colWidth up + if (fColType.colWidth == 3) + fColType.colWidth = 4; + else if (fColType.colWidth == 5 || fColType.colWidth == 6 || fColType.colWidth == 7) + fColType.colWidth = 8; + + idbassert(fColType.colWidth > 0); + ridsPerBlock = BLOCK_SIZE / fColType.colWidth; + + /* calculate some shortcuts for extent and block based arithmetic */ + extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; + + for (i = 1, mask = 1, modMask = 0; i <= 32; i++) + { + mask <<= 1; + modMask = (modMask << 1) | 1; + + if (extentSize & mask) { - if (8 > fColType.colWidth && 4 <= fColType.colWidth ) - fColType.colDataType = CalpontSystemCatalog::CHAR; - - fColType.colWidth++; + divShift = i; + break; } + } - //If this is a dictionary column, fudge the numbers... - if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY) - || (fColType.colDataType == CalpontSystemCatalog::BLOB) - || (fColType.colDataType == CalpontSystemCatalog::TEXT)) + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (extentSize & mask) + throw runtime_error("pColStep: Extent size must be a power of 2 in blocks"); + + /* calculate shortcuts for rid-based arithmetic */ + for (i = 1, mask = 1, rpbMask = 0; i <= 32; i++) + { + mask <<= 1; + rpbMask = (rpbMask << 1) | 1; + + if (ridsPerBlock & mask) { - fColType.colWidth = 8; - fIsDict = true; + rpbShift = i; + break; } - // WIP MCOL-641 - else if (fColType.colWidth > 8 - && fColType.colDataType != CalpontSystemCatalog::DECIMAL - && fColType.colDataType != CalpontSystemCatalog::UDECIMAL) + } + + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (ridsPerBlock & mask) + throw runtime_error("pColStep: Block size and column width must be a power of 2"); + + for (i = 0, mask = 1, blockSizeShift = 0; i < 32; i++) + { + if (mask == BLOCK_SIZE) { - fColType.colWidth = 8; - fIsDict = true; - //TODO: is this right? - fColType.colDataType = CalpontSystemCatalog::VARCHAR; + blockSizeShift = i; + break; } - //Round colWidth up - if (fColType.colWidth == 3) - fColType.colWidth = 4; - else if (fColType.colWidth == 5 || fColType.colWidth == 6 || fColType.colWidth == 7) - fColType.colWidth = 8; + mask <<= 1; + } - idbassert(fColType.colWidth > 0); - ridsPerBlock = BLOCK_SIZE / fColType.colWidth; + if (i == 32) + throw runtime_error("pColStep: Block size must be a power of 2"); - /* calculate some shortcuts for extent and block based arithmetic */ - extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; + err = dbrm.getExtents(o, extents); - for (i = 1, mask = 1, modMask = 0; i <= 32; i++) - { - mask <<= 1; - modMask = (modMask << 1) | 1; + if (err) + { + ostringstream os; + os << "pColStep: BRM lookup error. Could not get extents for OID " << o; + throw runtime_error(os.str()); + } - if (extentSize & mask) - { - divShift = i; - break; - } - } + if (fOid > 3000) + { + lbidList.reset(new LBIDList(fOid, 0)); + } - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (extentSize & mask) - throw runtime_error("pColStep: Extent size must be a power of 2 in blocks"); - - /* calculate shortcuts for rid-based arithmetic */ - for (i = 1, mask = 1, rpbMask = 0; i <= 32; i++) - { - mask <<= 1; - rpbMask = (rpbMask << 1) | 1; - - if (ridsPerBlock & mask) - { - rpbShift = i; - break; - } - } - - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (ridsPerBlock & mask) - throw runtime_error("pColStep: Block size and column width must be a power of 2"); - - for (i = 0, mask = 1, blockSizeShift = 0; i < 32; i++) - { - if (mask == BLOCK_SIZE) - { - blockSizeShift = i; - break; - } - - mask <<= 1; - } - - if (i == 32) - throw runtime_error("pColStep: Block size must be a power of 2"); - - err = dbrm.getExtents(o, extents); - - if (err) - { - ostringstream os; - os << "pColStep: BRM lookup error. Could not get extents for OID " << o; - throw runtime_error(os.str()); - } - - if (fOid > 3000) - { - lbidList.reset(new LBIDList(fOid, 0)); - } - - sort(extents.begin(), extents.end(), ExtentSorter()); - numExtents = extents.size(); -// uniqueID = UniqueNumberGenerator::instance()->getUnique32(); -// if (fDec) -// fDec->addQueue(uniqueID); -// initializeConfigParms ( ); + sort(extents.begin(), extents.end(), ExtentSorter()); + numExtents = extents.size(); + // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + // if (fDec) + // fDec->addQueue(uniqueID); + // initializeConfigParms ( ); } -pColStep::pColStep(const pColScanStep& rhs) : - JobStep(rhs), - fRm(rhs.resourceManager()), - fOid(rhs.oid()), - fTableOid(rhs.tableOid()), - fColType(rhs.colType()), - fFilterCount(rhs.filterCount()), - fBOP(rhs.BOP()), - ridList(0), - fFilterString(rhs.filterString()), - msgsSent(0), - msgsRecvd(0), - finishedSending(false), - recvWaiting(false), - fIsDict(rhs.isDictCol()), - ridCount(0), - // Per Cindy, it's save to put fFlushInterval to be 0 - fFlushInterval(0), - fSwallowRows(false), - fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()), - fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()), - fStopSending(false), - fPhysicalIO(0), - fCacheIO(0), - fNumBlksSkipped(0), - fMsgBytesIn(0), - fMsgBytesOut(0), - fFilters(rhs.getFilters()) +pColStep::pColStep(const pColScanStep& rhs) + : JobStep(rhs) + , fRm(rhs.resourceManager()) + , fOid(rhs.oid()) + , fTableOid(rhs.tableOid()) + , fColType(rhs.colType()) + , fFilterCount(rhs.filterCount()) + , fBOP(rhs.BOP()) + , ridList(0) + , fFilterString(rhs.filterString()) + , msgsSent(0) + , msgsRecvd(0) + , finishedSending(false) + , recvWaiting(false) + , fIsDict(rhs.isDictCol()) + , ridCount(0) + , + // Per Cindy, it's save to put fFlushInterval to be 0 + fFlushInterval(0) + , fSwallowRows(false) + , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) + , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) + , fStopSending(false) + , fPhysicalIO(0) + , fCacheIO(0) + , fNumBlksSkipped(0) + , fMsgBytesIn(0) + , fMsgBytesOut(0) + , fFilters(rhs.getFilters()) { - int err, i; - uint32_t mask; + int err, i; + uint32_t mask; - if (fTableOid == 0) // cross engine support - return; + if (fTableOid == 0) // cross engine support + return; - if (fOid < 1000) - throw runtime_error("pColStep: invalid column"); + if (fOid < 1000) + throw runtime_error("pColStep: invalid column"); - ridsPerBlock = rhs.getRidsPerBlock(); - /* calculate some shortcuts for extent and block based arithmetic */ - extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; + ridsPerBlock = rhs.getRidsPerBlock(); + /* calculate some shortcuts for extent and block based arithmetic */ + extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; - for (i = 1, mask = 1, modMask = 0; i <= 32; i++) + for (i = 1, mask = 1, modMask = 0; i <= 32; i++) + { + mask <<= 1; + modMask = (modMask << 1) | 1; + + if (extentSize & mask) { - mask <<= 1; - modMask = (modMask << 1) | 1; + divShift = i; + break; + } + } - if (extentSize & mask) - { - divShift = i; - break; - } + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (extentSize & mask) + throw runtime_error("pColStep: Extent size must be a power of 2 in blocks"); + + /* calculate shortcuts for rid-based arithmetic */ + for (i = 1, mask = 1, rpbMask = 0; i <= 32; i++) + { + mask <<= 1; + rpbMask = (rpbMask << 1) | 1; + + if (ridsPerBlock & mask) + { + rpbShift = i; + break; + } + } + + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (ridsPerBlock & mask) + throw runtime_error("pColStep: Block size and column width must be a power of 2"); + + for (i = 0, mask = 1, blockSizeShift = 0; i < 32; i++) + { + if (mask == BLOCK_SIZE) + { + blockSizeShift = i; + break; } - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (extentSize & mask) - throw runtime_error("pColStep: Extent size must be a power of 2 in blocks"); + mask <<= 1; + } - /* calculate shortcuts for rid-based arithmetic */ - for (i = 1, mask = 1, rpbMask = 0; i <= 32; i++) - { - mask <<= 1; - rpbMask = (rpbMask << 1) | 1; + if (i == 32) + throw runtime_error("pColStep: Block size must be a power of 2"); - if (ridsPerBlock & mask) - { - rpbShift = i; - break; - } - } + err = dbrm.getExtents(fOid, extents); - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (ridsPerBlock & mask) - throw runtime_error("pColStep: Block size and column width must be a power of 2"); + if (err) + { + ostringstream os; + os << "pColStep: BRM lookup error. Could not get extents for OID " << fOid; + throw runtime_error(os.str()); + } - for (i = 0, mask = 1, blockSizeShift = 0; i < 32; i++) - { - if (mask == BLOCK_SIZE) - { - blockSizeShift = i; - break; - } + lbidList = rhs.getlbidList(); - mask <<= 1; - } + sort(extents.begin(), extents.end(), ExtentSorter()); + numExtents = extents.size(); - if (i == 32) - throw runtime_error("pColStep: Block size must be a power of 2"); + fOnClauseFilter = rhs.onClauseFilter(); - err = dbrm.getExtents(fOid, extents); - - if (err) - { - ostringstream os; - os << "pColStep: BRM lookup error. Could not get extents for OID " << fOid; - throw runtime_error(os.str()); - } - - lbidList = rhs.getlbidList(); - - sort(extents.begin(), extents.end(), ExtentSorter()); - numExtents = extents.size(); - - fOnClauseFilter = rhs.onClauseFilter(); - -// uniqueID = UniqueNumberGenerator::instance()->getUnique32(); -// if (fDec) -// fDec->addQueue(uniqueID); -// initializeConfigParms ( ); + // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + // if (fDec) + // fDec->addQueue(uniqueID); + // initializeConfigParms ( ); } -pColStep::pColStep(const PassThruStep& rhs) : - JobStep(rhs), - fRm(rhs.resourceManager()), - fOid(rhs.oid()), - fTableOid(rhs.tableOid()), - fColType(rhs.colType()), - fFilterCount(0), - fBOP(BOP_NONE), - ridList(0), - msgsSent(0), - msgsRecvd(0), - finishedSending(false), - recvWaiting(false), - fIsDict(rhs.isDictCol()), - ridCount(0), - // Per Cindy, it's save to put fFlushInterval to be 0 - fFlushInterval(0), - fSwallowRows(false), - fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()), - fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()), - fStopSending(false), - fPhysicalIO(0), - fCacheIO(0), - fNumBlksSkipped(0), - fMsgBytesIn(0), - fMsgBytesOut(0) +pColStep::pColStep(const PassThruStep& rhs) + : JobStep(rhs) + , fRm(rhs.resourceManager()) + , fOid(rhs.oid()) + , fTableOid(rhs.tableOid()) + , fColType(rhs.colType()) + , fFilterCount(0) + , fBOP(BOP_NONE) + , ridList(0) + , msgsSent(0) + , msgsRecvd(0) + , finishedSending(false) + , recvWaiting(false) + , fIsDict(rhs.isDictCol()) + , ridCount(0) + , + // Per Cindy, it's save to put fFlushInterval to be 0 + fFlushInterval(0) + , fSwallowRows(false) + , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) + , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) + , fStopSending(false) + , fPhysicalIO(0) + , fCacheIO(0) + , fNumBlksSkipped(0) + , fMsgBytesIn(0) + , fMsgBytesOut(0) { - int err, i; - uint32_t mask; + int err, i; + uint32_t mask; - if (fTableOid == 0) // cross engine support - return; + if (fTableOid == 0) // cross engine support + return; - if (fOid < 1000) - throw runtime_error("pColStep: invalid column"); + if (fOid < 1000) + throw runtime_error("pColStep: invalid column"); - ridsPerBlock = BLOCK_SIZE / fColType.colWidth; - /* calculate some shortcuts for extent and block based arithmetic */ - extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; + ridsPerBlock = BLOCK_SIZE / fColType.colWidth; + /* calculate some shortcuts for extent and block based arithmetic */ + extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; - for (i = 1, mask = 1, modMask = 0; i <= 32; i++) + for (i = 1, mask = 1, modMask = 0; i <= 32; i++) + { + mask <<= 1; + modMask = (modMask << 1) | 1; + + if (extentSize & mask) { - mask <<= 1; - modMask = (modMask << 1) | 1; + divShift = i; + break; + } + } - if (extentSize & mask) - { - divShift = i; - break; - } + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (extentSize & mask) + throw runtime_error("pColStep: Extent size must be a power of 2 in blocks"); + + /* calculate shortcuts for rid-based arithmetic */ + for (i = 1, mask = 1, rpbMask = 0; i <= 32; i++) + { + mask <<= 1; + rpbMask = (rpbMask << 1) | 1; + + if (ridsPerBlock & mask) + { + rpbShift = i; + break; + } + } + + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (ridsPerBlock & mask) + throw runtime_error("pColStep: Block size and column width must be a power of 2"); + + for (i = 0, mask = 1, blockSizeShift = 0; i < 32; i++) + { + if (mask == BLOCK_SIZE) + { + blockSizeShift = i; + break; } - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (extentSize & mask) - throw runtime_error("pColStep: Extent size must be a power of 2 in blocks"); + mask <<= 1; + } - /* calculate shortcuts for rid-based arithmetic */ - for (i = 1, mask = 1, rpbMask = 0; i <= 32; i++) - { - mask <<= 1; - rpbMask = (rpbMask << 1) | 1; + if (i == 32) + throw runtime_error("pColStep: Block size must be a power of 2"); - if (ridsPerBlock & mask) - { - rpbShift = i; - break; - } - } + err = dbrm.getExtents(fOid, extents); - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (ridsPerBlock & mask) - throw runtime_error("pColStep: Block size and column width must be a power of 2"); + if (err) + { + ostringstream os; + os << "pColStep: BRM lookup error. Could not get extents for OID " << fOid; + throw runtime_error(os.str()); + } - for (i = 0, mask = 1, blockSizeShift = 0; i < 32; i++) - { - if (mask == BLOCK_SIZE) - { - blockSizeShift = i; - break; - } - - mask <<= 1; - } - - if (i == 32) - throw runtime_error("pColStep: Block size must be a power of 2"); - - err = dbrm.getExtents(fOid, extents); - - if (err) - { - ostringstream os; - os << "pColStep: BRM lookup error. Could not get extents for OID " << fOid; - throw runtime_error(os.str()); - } - - sort(extents.begin(), extents.end(), ExtentSorter()); - numExtents = extents.size(); -// uniqueID = UniqueNumberGenerator::instance()->getUnique32(); -// if (fDec) -// fDec->addQueue(uniqueID); -// initializeConfigParms ( ); + sort(extents.begin(), extents.end(), ExtentSorter()); + numExtents = extents.size(); + // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + // if (fDec) + // fDec->addQueue(uniqueID); + // initializeConfigParms ( ); } pColStep::~pColStep() { - // join? - //delete lbidList; -// if (fDec) -// fDec->removeQueue(uniqueID); + // join? + // delete lbidList; + // if (fDec) + // fDec->removeQueue(uniqueID); } //------------------------------------------------------------------------------ @@ -495,1022 +493,1022 @@ pColStep::~pColStep() //------------------------------------------------------------------------------ void pColStep::initializeConfigParms() { + // const string section ( "JobList" ); + // const string sendLimitName ( "ProjectBlockReqLimit" ); + // const string sendThresholdName ( "ProjectBlockReqThreshold" ); + // Config* cf = Config::makeConfig(); + // + // string strVal; + // uint64_t numVal; -// const string section ( "JobList" ); -// const string sendLimitName ( "ProjectBlockReqLimit" ); -// const string sendThresholdName ( "ProjectBlockReqThreshold" ); -// Config* cf = Config::makeConfig(); -// -// string strVal; -// uint64_t numVal; + //...Get the tuning parameters that throttle msgs sent to primproc + //...fFilterRowReqLimit puts a cap on how many rids we will request from + //... primproc, before pausing to let the consumer thread catch up. + //... Without this limit, there is a chance that PrimProc could flood + //... ExeMgr with thousands of messages that will consume massive + //... amounts of memory for a 100 gigabyte database. + //...fFilterRowReqThreshhold is the level at which the number of outstanding + //... rids must fall below, before the producer can send more rids. - //...Get the tuning parameters that throttle msgs sent to primproc - //...fFilterRowReqLimit puts a cap on how many rids we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fFilterRowReqThreshhold is the level at which the number of outstanding - //... rids must fall below, before the producer can send more rids. - -// strVal = cf->getConfig(section, sendLimitName); -// if (strVal.size() > 0) -// { -// errno = 0; -// numVal = Config::uFromText(strVal); -// if ( errno == 0 ) -// fProjectBlockReqLimit = (uint32_t)numVal; -// } -// -// strVal = cf->getConfig(section, sendThresholdName); -// if (strVal.size() > 0) -// { -// errno = 0; -// numVal = Config::uFromText(strVal); -// if ( errno == 0 ) -// fProjectBlockReqThreshold = (uint32_t)numVal; -// } + // strVal = cf->getConfig(section, sendLimitName); + // if (strVal.size() > 0) + // { + // errno = 0; + // numVal = Config::uFromText(strVal); + // if ( errno == 0 ) + // fProjectBlockReqLimit = (uint32_t)numVal; + // } + // + // strVal = cf->getConfig(section, sendThresholdName); + // if (strVal.size() > 0) + // { + // errno = 0; + // numVal = Config::uFromText(strVal); + // if ( errno == 0 ) + // fProjectBlockReqThreshold = (uint32_t)numVal; + // } } void pColStep::startPrimitiveThread() { -// pThread.reset(new boost::thread(pColStepPrimitive(this))); + // pThread.reset(new boost::thread(pColStepPrimitive(this))); } void pColStep::startAggregationThread() { -// cThread.reset(new boost::thread(pColStepAggregator(this))); + // cThread.reset(new boost::thread(pColStepAggregator(this))); } void pColStep::run() { -// if (traceOn()) -// { -// syslogStartStep(16, // exemgr subsystem -// std::string("pColStep")); // step name -// } -// -// size_t sz = fInputJobStepAssociation.outSize(); -// idbassert(sz > 0); -// const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); -// DataList_t* dlp = dl->dataList(); -// DataList* strDlp = dl->stringDataList(); -// if ( dlp ) -// setRidList(dlp); -// else -// { -// setStrRidList( strDlp ); -// } -// //Sort can be set through the jobstep or the input JSA if fFlushinterval is 0 -// fToSort = (fFlushInterval) ? 0 : (!fToSort) ? fInputJobStepAssociation.toSort() : fToSort; -// fToSort = 0; -// //pthread_mutex_init(&mutex, NULL); -// //pthread_cond_init(&condvar, NULL); -// //pthread_cond_init(&flushed, NULL); -// startPrimitiveThread(); -// startAggregationThread(); + // if (traceOn()) + // { + // syslogStartStep(16, // exemgr subsystem + // std::string("pColStep")); // step name + // } + // + // size_t sz = fInputJobStepAssociation.outSize(); + // idbassert(sz > 0); + // const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); + // DataList_t* dlp = dl->dataList(); + // DataList* strDlp = dl->stringDataList(); + // if ( dlp ) + // setRidList(dlp); + // else + // { + // setStrRidList( strDlp ); + // } + // //Sort can be set through the jobstep or the input JSA if fFlushinterval is 0 + // fToSort = (fFlushInterval) ? 0 : (!fToSort) ? fInputJobStepAssociation.toSort() : fToSort; + // fToSort = 0; + // //pthread_mutex_init(&mutex, NULL); + // //pthread_cond_init(&condvar, NULL); + // //pthread_cond_init(&flushed, NULL); + // startPrimitiveThread(); + // startAggregationThread(); } void pColStep::join() { -// pThread->join(); -// cThread->join(); -// //pthread_mutex_destroy(&mutex); -// //pthread_cond_destroy(&condvar); -// //pthread_cond_destroy(&flushed); + // pThread->join(); + // cThread->join(); + // //pthread_mutex_destroy(&mutex); + // //pthread_cond_destroy(&condvar); + // //pthread_cond_destroy(&flushed); } void pColStep::addFilter(int8_t COP, float value) { - fFilterString << (uint8_t) COP; - fFilterString << (uint8_t) 0; - fFilterString << *((uint32_t*) &value); - fFilterCount++; + fFilterString << (uint8_t)COP; + fFilterString << (uint8_t)0; + fFilterString << *((uint32_t*)&value); + fFilterCount++; } void pColStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) { - int8_t tmp8; - int16_t tmp16; - int32_t tmp32; + int8_t tmp8; + int16_t tmp16; + int32_t tmp32; - fFilterString << (uint8_t) COP; - fFilterString << roundFlag; + fFilterString << (uint8_t)COP; + fFilterString << roundFlag; - // converts to a type of the appropriate width, then bitwise - // copies into the filter ByteStream - switch (fColType.colWidth) - { - case 1: - tmp8 = value; - fFilterString << *((uint8_t*) &tmp8); - break; + // converts to a type of the appropriate width, then bitwise + // copies into the filter ByteStream + switch (fColType.colWidth) + { + case 1: + tmp8 = value; + fFilterString << *((uint8_t*)&tmp8); + break; - case 2: - tmp16 = value; - fFilterString << *((uint16_t*) &tmp16); - break; + case 2: + tmp16 = value; + fFilterString << *((uint16_t*)&tmp16); + break; - case 4: - tmp32 = value; - fFilterString << *((uint32_t*) &tmp32); - break; + case 4: + tmp32 = value; + fFilterString << *((uint32_t*)&tmp32); + break; - case 8: - fFilterString << *((uint64_t*) &value); - break; + case 8: fFilterString << *((uint64_t*)&value); break; - default: - ostringstream o; + default: + ostringstream o; - o << "pColStep: CalpontSystemCatalog says OID " << fOid << - " has a width of " << fColType.colWidth; - throw runtime_error(o.str()); - } + o << "pColStep: CalpontSystemCatalog says OID " << fOid << " has a width of " << fColType.colWidth; + throw runtime_error(o.str()); + } - fFilterCount++; + fFilterCount++; } void pColStep::addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag) { - fFilterString << (uint8_t) COP; - fFilterString << roundFlag; + fFilterString << (uint8_t)COP; + fFilterString << roundFlag; - // bitwise copies into the filter ByteStream - fFilterString << *reinterpret_cast(&value); + // bitwise copies into the filter ByteStream + fFilterString << *reinterpret_cast(&value); - fFilterCount++; + fFilterCount++; } void pColStep::setRidList(DataList* dl) { - ridList = dl; + ridList = dl; } void pColStep::setStrRidList(DataList* strDl) { - strRidList = strDl; + strRidList = strDl; } void pColStep::setBOP(int8_t b) { - fBOP = b; + fBOP = b; } void pColStep::setOutputType(int8_t OutputType) { - fOutputType = OutputType; + fOutputType = OutputType; } void pColStep::setSwallowRows(const bool swallowRows) { - fSwallowRows = swallowRows; + fSwallowRows = swallowRows; } void pColStep::sendPrimitiveMessages() { -// int it = -1; -// int msgRidCount = 0; -// int ridListIdx = 0; -// bool more = false; -// uint64_t absoluteRID = 0; -// int64_t msgLBID = -1; -// int64_t nextLBID = -1; -// int64_t msgLargeBlock = -1; -// int64_t nextLargeBlock = -1; -// uint16_t blockRelativeRID; -// uint32_t msgCount = 0; -// uint32_t sentBlockCount = 0; -// int msgsSkip=0; -// bool scan=false; -// bool scanThisBlock=false; -// ElementType e; -// UintRowGroup rw; -// StringElementType strE; -// StringRowGroup strRw; -// -// ByteStream msgRidList; -// ByteStream primMsg(MAX_BUFFER_SIZE); //the MAX_BUFFER_SIZE as of 8/20 -// -// NewColRequestHeader hdr; -// -// AnyDataListSPtr dl; -// FifoDataList *fifo = NULL; -// StringFifoDataList* strFifo = NULL; -// -// const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); -// -// //The presence of more than 1 input DL means we (probably) have a pDictionaryScan step feeding this step -// // a list of tokens to get the rids for. Convert the input tokens to a filter string. We also have a rid -// // list as the second input dl -// if (fInputJobStepAssociation.outSize() > 1) -// { -// addFilters(); -// if (fTableOid >= 3000) -// cout << toString() << endl; -// //If we got no input rids (as opposed to no input DL at all) then there were no matching rows from -// // the previous step, so this step should not return any rows either. This would be the case, for -// // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. -// if (fFilterCount == 0) -// { -// goto done; -// } -// } -// -// // determine which ranges/extents to eliminate from this step -// -//#ifdef DEBUG -// if (fOid>=3000) -// cout << "oid " << fOid << endl; -//#endif -// -// scanFlags.resize(numExtents); -// -// for (uint32_t idx=0; idx CasualPartitionPredicate( -// extents[idx].partition.cprange.loVal, -// extents[idx].partition.cprange.hiVal, -// &fFilterString, -// fFilterCount, -// fColType, -// fBOP) || ignoreCP; -// scanFlags[idx]=flag; -//#ifdef DEBUG -// if (fOid >= 3000 && flushInterval == 0) -// cout << (flag ? " will scan " : " will not scan ") -// << "extent with range " << extents[idx].partition.cprange.loVal -// << "-" << extents[idx].partition.cprange.hiVal << endl; -//#endif -// -// } -// -//// if (fOid>=3000) -//// cout << " " << scanFlags[idx]; -// } -//// if (scanFlags.size()>0) -//// cout << endl; -// -// // If there was more than 1 input DL, the first is a list of filters and the second is a list of rids, -// // otherwise the first is the list of rids. -// if (fInputJobStepAssociation.outSize() > 1) -// ridListIdx = 1; -// else -// ridListIdx = 0; -// -// dl = fInputJobStepAssociation.outAt(ridListIdx); -// ridList = dl->dataList(); -// if ( ridList ) -// { -// fifo = dl->fifoDL(); -// -// if (fifo) -// it = fifo->getIterator(); -// else -// it = ridList->getIterator(); -// } -// else -// { -// strRidList = dl->stringDataList(); -// strFifo = dl->stringDL(); -// -// if (strFifo) -// it = strFifo->getIterator(); -// else -// it = strRidList->getIterator(); -// } -// -// if (ridList) -// { -// if (fifo) -// { -// more = fifo->next(it, &rw); -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { -// dlTimes.setFirstReadTime(); -// } -// absoluteRID = rw.et[0].first; -// } -// else -// { -// more = ridList->next(it, &e); -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { -// dlTimes.setFirstReadTime(); -// } -// absoluteRID = e.first; -// rw.count = 1; -// } -// } -// else -// { -// if (strFifo) -// { -// more = strFifo->next(it, &strRw); -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { -// dlTimes.setFirstReadTime(); -// } -// absoluteRID = strRw.et[0].first; -// } -// else -// { -// more = strRidList->next(it, &strE); -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { -// dlTimes.setFirstReadTime(); -// } -// absoluteRID = strE.first; -// strRw.count = 1; -// } -// } -// -// if (more) -// msgLBID = getLBID(absoluteRID, scan); -// scanThisBlock = scan; -// msgLargeBlock = absoluteRID >> blockSizeShift; -// -// while (more || msgRidCount > 0) { -// uint64_t rwCount; -// if ( ridList) -// rwCount = rw.count; -// else -// rwCount = strRw.count; -// -// for (uint64_t i = 0; ((i < rwCount) || (!more && msgRidCount > 0)); ) -// { -// if ( ridList) -// { -// if (fifo) -// absoluteRID = rw.et[i].first; -// else -// absoluteRID = e.first; -// } -// else -// { -// if (strFifo) -// absoluteRID = strRw.et[i].first; -// else -// absoluteRID = strE.first; -// } -// -// if (more) { -// nextLBID = getLBID(absoluteRID, scan); -// nextLargeBlock = absoluteRID >> blockSizeShift; -// } -// -// //XXXPAT: need to prove N & S here -// if (nextLBID == msgLBID && more) { -//// blockRelativeRID = absoluteRID % ridsPerBlock; -// blockRelativeRID = absoluteRID & rpbMask; -// msgRidList << blockRelativeRID; -// msgRidCount++; -// ++i; -// } -// else { -// //Bug 831: move building msg after the check of scanThisBlock -// if (scanThisBlock==true) -// { -// hdr.ism.Interleave=0; -// hdr.ism.Flags=planFlagsToPrimFlags(fTraceFlags); -// hdr.ism.Command=COL_BY_SCAN; -// hdr.ism.Size=sizeof(NewColRequestHeader) + fFilterString.length() + -// msgRidList.length(); -// hdr.ism.Type=2; -// -// hdr.hdr.SessionID = fSessionId; -// //hdr.hdr.StatementID = 0; -// hdr.hdr.TransactionID = fTxnId; -// hdr.hdr.VerID = fVerId; -// hdr.hdr.StepID = fStepId; -// hdr.hdr.UniqueID = uniqueID; -// -// hdr.LBID = msgLBID; -//// idbassert(hdr.LBID >= 0); -// hdr.DataSize = fColType.colWidth; -// hdr.DataType = fColType.colDataType; -// hdr.CompType = fColType.compressionType; -// hdr.OutputType = fOutputType; -// hdr.BOP = fBOP; -// hdr.NOPS = fFilterCount; -// hdr.NVALS = msgRidCount; -// hdr.sort = fToSort; -// -// primMsg.append((const uint8_t *) &hdr, sizeof(NewColRequestHeader)); -// primMsg += fFilterString; -// primMsg += msgRidList; -// ridCount += msgRidCount; -// ++sentBlockCount; -// -//#ifdef DEBUG -// if (flushInterval == 0 && fOid >= 3000) -// cout << "sending a prim msg for LBID " << msgLBID << endl; -//#endif -// ++msgCount; -//// cout << "made a primitive\n"; -// if (msgLargeBlock != nextLargeBlock || !more) { -//// cout << "writing " << msgCount << " primitives\n"; -// fMsgBytesOut += primMsg.lengthWithHdrOverhead(); -// fDec->write(primMsg); -// msgsSent += msgCount; -// msgCount = 0; -// primMsg.restart(); -// msgLargeBlock = nextLargeBlock; -// -// // @bug 769 - Added "&& !fSwallowRows" condition below to fix problem with -// // caltraceon(16) not working for tpch01 and some other queries. If a query -// // ever held off requesting more blocks, it would lock and never finish. -// //Bug 815 -// if (( sentBlockCount >= fProjectBlockReqLimit) && !fSwallowRows && -// (( msgsSent - msgsRecvd) > fProjectBlockReqThreshold)) -// { -// mutex.lock(); //pthread_mutex_lock(&mutex); -// fStopSending = true; -// -// // @bug 836. Wake up the receiver if he's sleeping. -// if (recvWaiting) -// condvar.notify_one(); //pthread_cond_signal(&condvar); -// flushed.wait(mutex); //pthread_cond_wait(&flushed, &mutex); -// fStopSending = false; -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// sentBlockCount = 0; -// } -// } -// } -// else -// { -// msgsSkip++; -// } -// msgLBID = nextLBID; -// msgRidList.restart(); -// msgRidCount = 0; -// -// mutex.lock(); //pthread_mutex_lock(&mutex); -// -// if (scanThisBlock) { -// if (recvWaiting) -// condvar.notify_one(); //pthread_cond_signal(&condvar); -// #ifdef DEBUG -//// cout << "msgsSent++ = " << msgsSent << endl; -// #endif -// } -// scanThisBlock = scan; -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// // break the for loop -// if (!more) -// break; -// } -// } // for rw.count -// -// if (more) -// { -// if ( ridList ) -// { -// if (fifo) -// { -// rw.count = 0; -// more = fifo->next(it, &rw); -// } -// else -// { -// rw.count = 1; -// more = ridList->next(it, &e); -// } -// } -// else -// { -// if (strFifo) -// { -// strRw.count = 0; -// more = strFifo->next(it, &strRw); -// } -// else -// { -// strRw.count = 1; -// more = strRidList->next(it, &strE); -// } -// } -// } -// } -// -// if (fOid>=3000) dlTimes.setLastReadTime(); -// -//done: -// mutex.lock(); //pthread_mutex_lock(&mutex); -// finishedSending = true; -// if (recvWaiting) -// condvar.notify_one(); //pthread_cond_signal(&condvar); -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -//#ifdef DEBUG -// if (fOid >=3000) -// cout << "pColStep msgSent " -// << msgsSent << "/" << msgsSkip -// << " rids " << ridCount -// << " oid " << fOid << " " << msgLBID << endl; -//#endif -// //...Track the number of LBIDs we skip due to Casual Partioning. -// fNumBlksSkipped += msgsSkip; + // int it = -1; + // int msgRidCount = 0; + // int ridListIdx = 0; + // bool more = false; + // uint64_t absoluteRID = 0; + // int64_t msgLBID = -1; + // int64_t nextLBID = -1; + // int64_t msgLargeBlock = -1; + // int64_t nextLargeBlock = -1; + // uint16_t blockRelativeRID; + // uint32_t msgCount = 0; + // uint32_t sentBlockCount = 0; + // int msgsSkip=0; + // bool scan=false; + // bool scanThisBlock=false; + // ElementType e; + // UintRowGroup rw; + // StringElementType strE; + // StringRowGroup strRw; + // + // ByteStream msgRidList; + // ByteStream primMsg(MAX_BUFFER_SIZE); //the MAX_BUFFER_SIZE as of 8/20 + // + // NewColRequestHeader hdr; + // + // AnyDataListSPtr dl; + // FifoDataList *fifo = NULL; + // StringFifoDataList* strFifo = NULL; + // + // const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); + // + // //The presence of more than 1 input DL means we (probably) have a pDictionaryScan step feeding this + //step + // // a list of tokens to get the rids for. Convert the input tokens to a filter string. We also have a + //rid + // // list as the second input dl + // if (fInputJobStepAssociation.outSize() > 1) + // { + // addFilters(); + // if (fTableOid >= 3000) + // cout << toString() << endl; + // //If we got no input rids (as opposed to no input DL at all) then there were no matching rows + //from + // // the previous step, so this step should not return any rows either. This would be the case, + //for + // // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. + // if (fFilterCount == 0) + // { + // goto done; + // } + // } + // + // // determine which ranges/extents to eliminate from this step + // + //#ifdef DEBUG + // if (fOid>=3000) + // cout << "oid " << fOid << endl; + //#endif + // + // scanFlags.resize(numExtents); + // + // for (uint32_t idx=0; idx CasualPartitionPredicate( + // extents[idx].partition.cprange.loVal, + // extents[idx].partition.cprange.hiVal, + // &fFilterString, + // fFilterCount, + // fColType, + // fBOP) || ignoreCP; + // scanFlags[idx]=flag; + //#ifdef DEBUG + // if (fOid >= 3000 && flushInterval == 0) + // cout << (flag ? " will scan " : " will not scan ") + // << "extent with range " << extents[idx].partition.cprange.loVal + // << "-" << extents[idx].partition.cprange.hiVal << endl; + //#endif + // + // } + // + //// if (fOid>=3000) + //// cout << " " << scanFlags[idx]; + // } + //// if (scanFlags.size()>0) + //// cout << endl; + // + // // If there was more than 1 input DL, the first is a list of filters and the second is a list of rids, + // // otherwise the first is the list of rids. + // if (fInputJobStepAssociation.outSize() > 1) + // ridListIdx = 1; + // else + // ridListIdx = 0; + // + // dl = fInputJobStepAssociation.outAt(ridListIdx); + // ridList = dl->dataList(); + // if ( ridList ) + // { + // fifo = dl->fifoDL(); + // + // if (fifo) + // it = fifo->getIterator(); + // else + // it = ridList->getIterator(); + // } + // else + // { + // strRidList = dl->stringDataList(); + // strFifo = dl->stringDL(); + // + // if (strFifo) + // it = strFifo->getIterator(); + // else + // it = strRidList->getIterator(); + // } + // + // if (ridList) + // { + // if (fifo) + // { + // more = fifo->next(it, &rw); + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { + // dlTimes.setFirstReadTime(); + // } + // absoluteRID = rw.et[0].first; + // } + // else + // { + // more = ridList->next(it, &e); + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { + // dlTimes.setFirstReadTime(); + // } + // absoluteRID = e.first; + // rw.count = 1; + // } + // } + // else + // { + // if (strFifo) + // { + // more = strFifo->next(it, &strRw); + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { + // dlTimes.setFirstReadTime(); + // } + // absoluteRID = strRw.et[0].first; + // } + // else + // { + // more = strRidList->next(it, &strE); + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { + // dlTimes.setFirstReadTime(); + // } + // absoluteRID = strE.first; + // strRw.count = 1; + // } + // } + // + // if (more) + // msgLBID = getLBID(absoluteRID, scan); + // scanThisBlock = scan; + // msgLargeBlock = absoluteRID >> blockSizeShift; + // + // while (more || msgRidCount > 0) { + // uint64_t rwCount; + // if ( ridList) + // rwCount = rw.count; + // else + // rwCount = strRw.count; + // + // for (uint64_t i = 0; ((i < rwCount) || (!more && msgRidCount > 0)); ) + // { + // if ( ridList) + // { + // if (fifo) + // absoluteRID = rw.et[i].first; + // else + // absoluteRID = e.first; + // } + // else + // { + // if (strFifo) + // absoluteRID = strRw.et[i].first; + // else + // absoluteRID = strE.first; + // } + // + // if (more) { + // nextLBID = getLBID(absoluteRID, scan); + // nextLargeBlock = absoluteRID >> blockSizeShift; + // } + // + // //XXXPAT: need to prove N & S here + // if (nextLBID == msgLBID && more) { + //// blockRelativeRID = absoluteRID % ridsPerBlock; + // blockRelativeRID = absoluteRID & rpbMask; + // msgRidList << blockRelativeRID; + // msgRidCount++; + // ++i; + // } + // else { + // //Bug 831: move building msg after the check of scanThisBlock + // if (scanThisBlock==true) + // { + // hdr.ism.Interleave=0; + // hdr.ism.Flags=planFlagsToPrimFlags(fTraceFlags); + // hdr.ism.Command=COL_BY_SCAN; + // hdr.ism.Size=sizeof(NewColRequestHeader) + fFilterString.length() + + // msgRidList.length(); + // hdr.ism.Type=2; + // + // hdr.hdr.SessionID = fSessionId; + // //hdr.hdr.StatementID = 0; + // hdr.hdr.TransactionID = fTxnId; + // hdr.hdr.VerID = fVerId; + // hdr.hdr.StepID = fStepId; + // hdr.hdr.UniqueID = uniqueID; + // + // hdr.LBID = msgLBID; + //// idbassert(hdr.LBID >= 0); + // hdr.DataSize = fColType.colWidth; + // hdr.DataType = fColType.colDataType; + // hdr.CompType = fColType.compressionType; + // hdr.OutputType = fOutputType; + // hdr.BOP = fBOP; + // hdr.NOPS = fFilterCount; + // hdr.NVALS = msgRidCount; + // hdr.sort = fToSort; + // + // primMsg.append((const uint8_t *) &hdr, sizeof(NewColRequestHeader)); + // primMsg += fFilterString; + // primMsg += msgRidList; + // ridCount += msgRidCount; + // ++sentBlockCount; + // + //#ifdef DEBUG + // if (flushInterval == 0 && fOid >= 3000) + // cout << "sending a prim msg for LBID " << msgLBID << endl; + //#endif + // ++msgCount; + //// cout << "made a primitive\n"; + // if (msgLargeBlock != nextLargeBlock || !more) { + //// cout << "writing " << msgCount << " primitives\n"; + // fMsgBytesOut += primMsg.lengthWithHdrOverhead(); + // fDec->write(primMsg); + // msgsSent += msgCount; + // msgCount = 0; + // primMsg.restart(); + // msgLargeBlock = nextLargeBlock; + // + // // @bug 769 - Added "&& !fSwallowRows" condition below to fix problem + //with + // // caltraceon(16) not working for tpch01 and some other queries. If a + //query + // // ever held off requesting more blocks, it would lock and never finish. + // //Bug 815 + // if (( sentBlockCount >= fProjectBlockReqLimit) && !fSwallowRows + //&& + // (( msgsSent - msgsRecvd) > fProjectBlockReqThreshold)) + // { + // mutex.lock(); //pthread_mutex_lock(&mutex); + // fStopSending = true; + // + // // @bug 836. Wake up the receiver if he's sleeping. + // if (recvWaiting) + // condvar.notify_one(); + ////pthread_cond_signal(&condvar); flushed.wait(mutex); //pthread_cond_wait(&flushed, &mutex); fStopSending + //= false; mutex.unlock(); //pthread_mutex_unlock(&mutex); sentBlockCount = 0; + // } + // } + // } + // else + // { + // msgsSkip++; + // } + // msgLBID = nextLBID; + // msgRidList.restart(); + // msgRidCount = 0; + // + // mutex.lock(); //pthread_mutex_lock(&mutex); + // + // if (scanThisBlock) { + // if (recvWaiting) + // condvar.notify_one(); //pthread_cond_signal(&condvar); + // #ifdef DEBUG + //// cout << "msgsSent++ = " << msgsSent << endl; + // #endif + // } + // scanThisBlock = scan; + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // // break the for loop + // if (!more) + // break; + // } + // } // for rw.count + // + // if (more) + // { + // if ( ridList ) + // { + // if (fifo) + // { + // rw.count = 0; + // more = fifo->next(it, &rw); + // } + // else + // { + // rw.count = 1; + // more = ridList->next(it, &e); + // } + // } + // else + // { + // if (strFifo) + // { + // strRw.count = 0; + // more = strFifo->next(it, &strRw); + // } + // else + // { + // strRw.count = 1; + // more = strRidList->next(it, &strE); + // } + // } + // } + // } + // + // if (fOid>=3000) dlTimes.setLastReadTime(); + // + // done: + // mutex.lock(); //pthread_mutex_lock(&mutex); + // finishedSending = true; + // if (recvWaiting) + // condvar.notify_one(); //pthread_cond_signal(&condvar); + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + //#ifdef DEBUG + // if (fOid >=3000) + // cout << "pColStep msgSent " + // << msgsSent << "/" << msgsSkip + // << " rids " << ridCount + // << " oid " << fOid << " " << msgLBID << endl; + //#endif + // //...Track the number of LBIDs we skip due to Casual Partioning. + // fNumBlksSkipped += msgsSkip; } void pColStep::receivePrimitiveMessages() { -// int64_t ridResults = 0; -// AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); -// DataList_t* dlp = dl->dataList(); -// uint64_t fbo; -// FifoDataList *fifo = dl->fifoDL(); -// UintRowGroup rw; -// uint64_t ridBase; -// boost::shared_ptr bs; -// uint32_t i = 0, length; -// -// while (1) { -// // sync with the send side -// mutex.lock(); //pthread_mutex_lock(&mutex); -// while (!finishedSending && msgsSent == msgsRecvd) { -// recvWaiting = true; -// #ifdef DEBUG -// cout << "c sleeping" << endl; -// #endif -// // @bug 836. Wake up the sender if he's sleeping. -// if (fStopSending) -// flushed.notify_one(); //pthread_cond_signal(&flushed); -// condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); -// #ifdef DEBUG -// cout << "c waking" << endl; -// #endif -// recvWaiting = false; -// } -// if (msgsSent == msgsRecvd) { -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// break; -// } -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// // do the recv -// fDec->read(uniqueID, bs); -// fMsgBytesIn += bs->lengthWithHdrOverhead(); -// -// // no more messages, and buffered messages should be already processed by now. -// if (bs->length() == 0) break; -// -// #ifdef DEBUG -// cout << "msgsRecvd++ = " << msgsRecvd << ". RidResults = " << ridResults << endl; -// cout << "Got a ColResultHeader!: " << bs.length() << " bytes" << endl; -// #endif -// -// const ByteStream::byte* bsp = bs->buf(); -// -// // get the ISMPacketHeader out of the bytestream -// //const ISMPacketHeader* ism = reinterpret_cast(bsp); -// -// // get the ColumnResultHeader out of the bytestream -// const ColResultHeader* crh = reinterpret_cast -// (&bsp[sizeof(ISMPacketHeader)]); -// -// bool firstRead = true; -// length = bs->length(); -// -// i = 0; -// uint32_t msgCount = 0; -// while (i < length) { -// ++msgCount; -// -// i += sizeof(ISMPacketHeader); -// crh = reinterpret_cast(&bsp[i]); -// // double check the sequence number is increased by one each time -// i += sizeof(ColResultHeader); -// -// fCacheIO += crh->CacheIO; -// fPhysicalIO += crh->PhysicalIO; -// -// // From this point on the rest of the bytestream is the data that comes back from the primitive server -// // This needs to be fed to a datalist that is retrieved from the outputassociation object. -// -// fbo = getFBO(crh->LBID); -// ridBase = fbo << rpbShift; -// -// #ifdef DEBUG -//// cout << " NVALS = " << crh->NVALS << " fbo = " << fbo << " lbid = " << crh->LBID << endl; -// #endif -// -// //Check output type -// if ( fOutputType == OT_RID ) -// { -// ridResults += crh->NVALS; -// } -// -// /* XXXPAT: This clause is executed when ExeMgr calls the new nextBand(BS) fcn. -// -// TODO: both classes have to agree -// on which nextBand() variant will be called. pColStep -// currently has to infer that from flushInterval and the -// Table OID. It would be better to have a more explicit form -// of agreement. -// -// The goal of the nextBand(BS) fcn is to avoid iterating over -// every row except at unserialization. This clause copies -// the raw results from the PrimProc response directly into -// the memory used for the ElementType array. DeliveryStep -// will also treat the ElementType array as raw memory and -// serialize that. TableColumn now parses the packed data -// instead of whole ElementTypes. -// */ -// else if (fOutputType == OT_TOKEN && fFlushInterval > 0 && !fIsDict) { -// -// if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) -// dlTimes.setFirstInsertTime(); -// ridResults += crh->NVALS; -// -// /* memcpy the bytestream into the output set */ -// uint32_t toCopy, bsPos = 0; -// uint8_t *pos; -// while (bsPos < crh->NVALS) { -// toCopy = (crh->NVALS - bsPos > rw.ElementsPerGroup - rw.count ? -// rw.ElementsPerGroup - rw.count : crh->NVALS - bsPos); -// pos = ((uint8_t *) &rw.et[0]) + (rw.count * fColType.colWidth); -// memcpy(pos, &bsp[i], toCopy * fColType.colWidth); -// bsPos += toCopy; -// i += toCopy * fColType.colWidth; -// rw.count += toCopy; -// if (rw.count == rw.ElementsPerGroup) { -// if (!fSwallowRows) -// fifo->insert(rw); -// rw.count = 0; -// } -// } -// } -// else if ( fOutputType == OT_TOKEN) -// { -// uint64_t dv; -// uint64_t rid; -// -// if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) -// dlTimes.setFirstInsertTime(); -// ridResults += crh->NVALS; -// for(int j = 0; j < crh->NVALS; ++j) -// { -// // XXXPAT: Only use this when the RID doesn't matter or when -// // the response contains every row. -// -// rid = j + ridBase; -// switch (fColType.colWidth) { -// case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; break; -// case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; -// case 2: dv = *((const uint16_t *) &bsp[i]); i += 2; break; -// case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; -// default: -// throw runtime_error("pColStep: invalid column width!"); -// } -// -// // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is on. -// // This options swallows rows in the project steps. -// if (!fSwallowRows) -// { -// if (fifo) -// { -// rw.et[rw.count].first = rid; -// rw.et[rw.count++].second = dv; -// if (rw.count == rw.ElementsPerGroup) -// { -// fifo->insert(rw); -// rw.count = 0; -// } -// } -// else -// { -// dlp->insert(ElementType(rid, dv)); -// } -// #ifdef DEBUG -// //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << endl; -// #endif -// } -// } -// } -// else if ( fOutputType == OT_BOTH ) -// { -// ridResults += crh->NVALS; -// for(int j = 0; j < crh->NVALS; ++j) -// { -// uint64_t dv; -// uint64_t rid; -// -// rid = *((const uint16_t *) &bsp[i]) + ridBase; -// i += sizeof(uint16_t); -// switch (fColType.colWidth) { -// case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; break; -// case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; -// case 2: dv = *((const uint16_t *) &bsp[i]); i += 2; break; -// case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; -// default: -// throw runtime_error("pColStep: invalid column width!"); -// } -// -// // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is on. -// // This options swallows rows in the project steps. -// if (!fSwallowRows) { -// if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) -// dlTimes.setFirstInsertTime(); -// if(fifo) -// { -//// rw.et[rw.count++] = ElementType(rid, dv); -// rw.et[rw.count].first = rid; -// rw.et[rw.count++].second = dv; -// if (rw.count == rw.ElementsPerGroup) -// { -// fifo->insert(rw); -// rw.count = 0; -// } -// } -// else -// { -// dlp->insert(ElementType(rid, dv)); -// } -// #ifdef DEBUG -// //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << endl; -// #endif -// } -// } -// } -// } // unpacking the BS -// -// //Bug 815: Check whether we have enough to process -// //++lockCount; -// mutex.lock(); //pthread_mutex_lock(&mutex); -// if ( fStopSending && ((msgsSent - msgsRecvd ) <= fProjectBlockReqThreshold) ) -// { -// flushed.notify_one(); //pthread_cond_signal(&flushed); -// } -// mutex.unlock(); //pthread_mutex_unlock(&mutex); -// -// firstRead = false; -// msgsRecvd += msgCount; -// } // read loop -// // done reading -// -// if (fifo && rw.count > 0) -// fifo->insert(rw); -// -// //...Casual partitioning could cause us to do no processing. In that -// //...case these time stamps did not get set. So we set them here. -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { -// dlTimes.setFirstReadTime(); -// dlTimes.setLastReadTime(); -// dlTimes.setFirstInsertTime(); -// } -// if (fOid>=3000) dlTimes.setEndOfInputTime(); -// -// //@bug 699: Reset StepMsgQueue -// fDec->removeQueue(uniqueID); -// -// if (fifo) -// fifo->endOfInput(); -// else -// dlp->endOfInput(); -// -// if (fTableOid >= 3000) -// { -// //...Construct timestamp using ctime_r() instead of ctime() not -// //...necessarily due to re-entrancy, but because we want to strip -// //...the newline ('\n') off the end of the formatted string. -// time_t t = time(0); -// char timeString[50]; -// ctime_r(&t, timeString); -// timeString[strlen(timeString)-1 ] = '\0'; -// -// FifoDataList* pFifo = 0; -// uint64_t totalBlockedReadCount = 0; -// uint64_t totalBlockedWriteCount = 0; -// -// //...Sum up the blocked FIFO reads for all input associations -// size_t inDlCnt = fInputJobStepAssociation.outSize(); -// for (size_t iDataList=0; iDataListfifoDL(); -// if (pFifo) -// { -// totalBlockedReadCount += pFifo->blockedReadCount(); -// } -// } -// -// //...Sum up the blocked FIFO writes for all output associations -// size_t outDlCnt = fOutputJobStepAssociation.outSize(); -// for (size_t iDataList=0; iDataListfifoDL(); -// if (pFifo) -// { -// totalBlockedWriteCount += pFifo->blockedWriteCount(); -// } -// } -// -// //...Roundoff msg byte counts to nearest KB for display -// uint64_t msgBytesInKB = fMsgBytesIn >> 10; -// uint64_t msgBytesOutKB = fMsgBytesOut >> 10; -// if (fMsgBytesIn & 512) -// msgBytesInKB++; -// if (fMsgBytesOut & 512) -// msgBytesOutKB++; -// -// // @bug 828 -// if (fifo) -// fifo->totalSize(ridResults); -// -// if (traceOn()) -// { -// //...Print job step completion information -// ostringstream logStr; -// logStr << "ses:" << fSessionId << -// " st: " << fStepId << " finished at " << -// timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << -// fCacheIO << "; MsgsRvcd-" << msgsRecvd << -// "; BlockedFifoIn/Out-" << totalBlockedReadCount << -// "/" << totalBlockedWriteCount << -// "; output size-" << ridResults << endl << -// "\tPartitionBlocksEliminated-" << fNumBlksSkipped << -// "; MsgBytesIn-" << msgBytesInKB << "KB" << -// "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << -// "\t1st read " << dlTimes.FirstReadTimeString() << -// "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << -// JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << -// "s" << endl; -// -// logEnd(logStr.str().c_str()); -// -// syslogReadBlockCounts(16, // exemgr sybsystem -// fPhysicalIO, // # blocks read from disk -// fCacheIO, // # blocks read from cache -// fNumBlksSkipped); // # casual partition block hits -// syslogProcessingTimes(16, // exemgr subsystem -// dlTimes.FirstReadTime(), // first datalist read -// dlTimes.LastReadTime(), // last datalist read -// dlTimes.FirstInsertTime(), // first datalist write -// dlTimes.EndOfInputTime()); // last (endOfInput) datalist write -// syslogEndStep(16, // exemgr subsystem -// totalBlockedReadCount, // blocked datalist input -// totalBlockedWriteCount, // blocked datalist output -// fMsgBytesIn, // incoming msg byte count -// fMsgBytesOut); // outgoing msg byte count -// } -// } + // int64_t ridResults = 0; + // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); + // DataList_t* dlp = dl->dataList(); + // uint64_t fbo; + // FifoDataList *fifo = dl->fifoDL(); + // UintRowGroup rw; + // uint64_t ridBase; + // boost::shared_ptr bs; + // uint32_t i = 0, length; + // + // while (1) { + // // sync with the send side + // mutex.lock(); //pthread_mutex_lock(&mutex); + // while (!finishedSending && msgsSent == msgsRecvd) { + // recvWaiting = true; + // #ifdef DEBUG + // cout << "c sleeping" << endl; + // #endif + // // @bug 836. Wake up the sender if he's sleeping. + // if (fStopSending) + // flushed.notify_one(); //pthread_cond_signal(&flushed); + // condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); + // #ifdef DEBUG + // cout << "c waking" << endl; + // #endif + // recvWaiting = false; + // } + // if (msgsSent == msgsRecvd) { + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // break; + // } + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // // do the recv + // fDec->read(uniqueID, bs); + // fMsgBytesIn += bs->lengthWithHdrOverhead(); + // + // // no more messages, and buffered messages should be already processed by now. + // if (bs->length() == 0) break; + // + // #ifdef DEBUG + // cout << "msgsRecvd++ = " << msgsRecvd << ". RidResults = " << ridResults << endl; + // cout << "Got a ColResultHeader!: " << bs.length() << " bytes" << endl; + // #endif + // + // const ByteStream::byte* bsp = bs->buf(); + // + // // get the ISMPacketHeader out of the bytestream + // //const ISMPacketHeader* ism = reinterpret_cast(bsp); + // + // // get the ColumnResultHeader out of the bytestream + // const ColResultHeader* crh = reinterpret_cast + // (&bsp[sizeof(ISMPacketHeader)]); + // + // bool firstRead = true; + // length = bs->length(); + // + // i = 0; + // uint32_t msgCount = 0; + // while (i < length) { + // ++msgCount; + // + // i += sizeof(ISMPacketHeader); + // crh = reinterpret_cast(&bsp[i]); + // // double check the sequence number is increased by one each time + // i += sizeof(ColResultHeader); + // + // fCacheIO += crh->CacheIO; + // fPhysicalIO += crh->PhysicalIO; + // + // // From this point on the rest of the bytestream is the data that comes back from the + //primitive server + // // This needs to be fed to a datalist that is retrieved from the outputassociation + //object. + // + // fbo = getFBO(crh->LBID); + // ridBase = fbo << rpbShift; + // + // #ifdef DEBUG + //// cout << " NVALS = " << crh->NVALS << " fbo = " << fbo << " lbid = " << crh->LBID << + ///endl; + // #endif + // + // //Check output type + // if ( fOutputType == OT_RID ) + // { + // ridResults += crh->NVALS; + // } + // + // /* XXXPAT: This clause is executed when ExeMgr calls the new nextBand(BS) fcn. + // + // TODO: both classes have to agree + // on which nextBand() variant will be called. pColStep + // currently has to infer that from flushInterval and the + // Table OID. It would be better to have a more explicit form + // of agreement. + // + // The goal of the nextBand(BS) fcn is to avoid iterating over + // every row except at unserialization. This clause copies + // the raw results from the PrimProc response directly into + // the memory used for the ElementType array. DeliveryStep + // will also treat the ElementType array as raw memory and + // serialize that. TableColumn now parses the packed data + // instead of whole ElementTypes. + // */ + // else if (fOutputType == OT_TOKEN && fFlushInterval > 0 && !fIsDict) { + // + // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) + // dlTimes.setFirstInsertTime(); + // ridResults += crh->NVALS; + // + // /* memcpy the bytestream into the output set */ + // uint32_t toCopy, bsPos = 0; + // uint8_t *pos; + // while (bsPos < crh->NVALS) { + // toCopy = (crh->NVALS - bsPos > rw.ElementsPerGroup - rw.count ? + // rw.ElementsPerGroup - rw.count : crh->NVALS - bsPos); + // pos = ((uint8_t *) &rw.et[0]) + (rw.count * fColType.colWidth); + // memcpy(pos, &bsp[i], toCopy * fColType.colWidth); + // bsPos += toCopy; + // i += toCopy * fColType.colWidth; + // rw.count += toCopy; + // if (rw.count == rw.ElementsPerGroup) { + // if (!fSwallowRows) + // fifo->insert(rw); + // rw.count = 0; + // } + // } + // } + // else if ( fOutputType == OT_TOKEN) + // { + // uint64_t dv; + // uint64_t rid; + // + // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) + // dlTimes.setFirstInsertTime(); + // ridResults += crh->NVALS; + // for(int j = 0; j < crh->NVALS; ++j) + // { + // // XXXPAT: Only use this when the RID doesn't matter or when + // // the response contains every row. + // + // rid = j + ridBase; + // switch (fColType.colWidth) { + // case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; break; + // case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; + // case 2: dv = *((const uint16_t *) &bsp[i]); i += 2; break; + // case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; + // default: + // throw runtime_error("pColStep: invalid column + //width!"); + // } + // + // // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is + //on. + // // This options swallows rows in the project steps. + // if (!fSwallowRows) + // { + // if (fifo) + // { + // rw.et[rw.count].first = rid; + // rw.et[rw.count++].second = dv; + // if (rw.count == rw.ElementsPerGroup) + // { + // fifo->insert(rw); + // rw.count = 0; + // } + // } + // else + // { + // dlp->insert(ElementType(rid, dv)); + // } + // #ifdef DEBUG + // //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << + //endl; #endif + // } + // } + // } + // else if ( fOutputType == OT_BOTH ) + // { + // ridResults += crh->NVALS; + // for(int j = 0; j < crh->NVALS; ++j) + // { + // uint64_t dv; + // uint64_t rid; + // + // rid = *((const uint16_t *) &bsp[i]) + ridBase; + // i += sizeof(uint16_t); + // switch (fColType.colWidth) { + // case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; + //break; case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; case 2: dv = *((const uint16_t *) + //&bsp[i]); i += 2; break; case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; default: throw + //runtime_error("pColStep: invalid column width!"); + // } + // + // // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is + //on. + // // This options swallows rows in the project steps. + // if (!fSwallowRows) { + // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) + // dlTimes.setFirstInsertTime(); + // if(fifo) + // { + //// rw.et[rw.count++] = ElementType(rid, dv); + // rw.et[rw.count].first = rid; + // rw.et[rw.count++].second = dv; + // if (rw.count == rw.ElementsPerGroup) + // { + // fifo->insert(rw); + // rw.count = 0; + // } + // } + // else + // { + // dlp->insert(ElementType(rid, dv)); + // } + // #ifdef DEBUG + // //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << + //endl; #endif + // } + // } + // } + // } // unpacking the BS + // + // //Bug 815: Check whether we have enough to process + // //++lockCount; + // mutex.lock(); //pthread_mutex_lock(&mutex); + // if ( fStopSending && ((msgsSent - msgsRecvd ) <= fProjectBlockReqThreshold) ) + // { + // flushed.notify_one(); //pthread_cond_signal(&flushed); + // } + // mutex.unlock(); //pthread_mutex_unlock(&mutex); + // + // firstRead = false; + // msgsRecvd += msgCount; + // } // read loop + // // done reading + // + // if (fifo && rw.count > 0) + // fifo->insert(rw); + // + // //...Casual partitioning could cause us to do no processing. In that + // //...case these time stamps did not get set. So we set them here. + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { + // dlTimes.setFirstReadTime(); + // dlTimes.setLastReadTime(); + // dlTimes.setFirstInsertTime(); + // } + // if (fOid>=3000) dlTimes.setEndOfInputTime(); + // + // //@bug 699: Reset StepMsgQueue + // fDec->removeQueue(uniqueID); + // + // if (fifo) + // fifo->endOfInput(); + // else + // dlp->endOfInput(); + // + // if (fTableOid >= 3000) + // { + // //...Construct timestamp using ctime_r() instead of ctime() not + // //...necessarily due to re-entrancy, but because we want to strip + // //...the newline ('\n') off the end of the formatted string. + // time_t t = time(0); + // char timeString[50]; + // ctime_r(&t, timeString); + // timeString[strlen(timeString)-1 ] = '\0'; + // + // FifoDataList* pFifo = 0; + // uint64_t totalBlockedReadCount = 0; + // uint64_t totalBlockedWriteCount = 0; + // + // //...Sum up the blocked FIFO reads for all input associations + // size_t inDlCnt = fInputJobStepAssociation.outSize(); + // for (size_t iDataList=0; iDataListfifoDL(); + // if (pFifo) + // { + // totalBlockedReadCount += pFifo->blockedReadCount(); + // } + // } + // + // //...Sum up the blocked FIFO writes for all output associations + // size_t outDlCnt = fOutputJobStepAssociation.outSize(); + // for (size_t iDataList=0; iDataListfifoDL(); + // if (pFifo) + // { + // totalBlockedWriteCount += pFifo->blockedWriteCount(); + // } + // } + // + // //...Roundoff msg byte counts to nearest KB for display + // uint64_t msgBytesInKB = fMsgBytesIn >> 10; + // uint64_t msgBytesOutKB = fMsgBytesOut >> 10; + // if (fMsgBytesIn & 512) + // msgBytesInKB++; + // if (fMsgBytesOut & 512) + // msgBytesOutKB++; + // + // // @bug 828 + // if (fifo) + // fifo->totalSize(ridResults); + // + // if (traceOn()) + // { + // //...Print job step completion information + // ostringstream logStr; + // logStr << "ses:" << fSessionId << + // " st: " << fStepId << " finished at " << + // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << + // fCacheIO << "; MsgsRvcd-" << msgsRecvd << + // "; BlockedFifoIn/Out-" << totalBlockedReadCount << + // "/" << totalBlockedWriteCount << + // "; output size-" << ridResults << endl << + // "\tPartitionBlocksEliminated-" << fNumBlksSkipped << + // "; MsgBytesIn-" << msgBytesInKB << "KB" << + // "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << + // "\t1st read " << dlTimes.FirstReadTimeString() << + // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << + // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << + // "s" << endl; + // + // logEnd(logStr.str().c_str()); + // + // syslogReadBlockCounts(16, // exemgr sybsystem + // fPhysicalIO, // # blocks read from disk + // fCacheIO, // # blocks read from cache + // fNumBlksSkipped); // # casual partition block hits + // syslogProcessingTimes(16, // exemgr subsystem + // dlTimes.FirstReadTime(), // first datalist read + // dlTimes.LastReadTime(), // last datalist read + // dlTimes.FirstInsertTime(), // first datalist write + // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write + // syslogEndStep(16, // exemgr subsystem + // totalBlockedReadCount, // blocked datalist input + // totalBlockedWriteCount, // blocked datalist output + // fMsgBytesIn, // incoming msg byte count + // fMsgBytesOut); // outgoing msg byte count + // } + // } } const string pColStep::toString() const { - ostringstream oss; - oss << "pColStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId << - " tb/col:" << fTableOid << "/" << fOid; + ostringstream oss; + oss << "pColStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId + << " tb/col:" << fTableOid << "/" << fOid; - if (alias().length()) oss << " alias:" << alias(); + if (alias().length()) + oss << " alias:" << alias(); - if (view().length()) oss << " view:" << view(); + if (view().length()) + oss << " view:" << view(); - if (fOutputJobStepAssociation.outSize() > 0) - oss << " " << omitOidInDL - << fOutputJobStepAssociation.outAt(0) << showOidInDL; - else - oss << " (no output yet)"; + if (fOutputJobStepAssociation.outSize() > 0) + oss << " " << omitOidInDL << fOutputJobStepAssociation.outAt(0) << showOidInDL; + else + oss << " (no output yet)"; - oss << " nf:" << fFilterCount; - oss << " in:"; + oss << " nf:" << fFilterCount; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - { - oss << fInputJobStepAssociation.outAt(i) << ", "; - } + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + oss << fInputJobStepAssociation.outAt(i) << ", "; + } - if (fSwallowRows) - oss << " (sink)"; + if (fSwallowRows) + oss << " (sink)"; - return oss.str(); + return oss.str(); } void pColStep::addFilters() { - AnyDataListSPtr dl = fInputJobStepAssociation.outAt(0); - DataList_t* bdl = dl->dataList(); - FifoDataList* fifo = fInputJobStepAssociation.outAt(0)->fifoDL(); + AnyDataListSPtr dl = fInputJobStepAssociation.outAt(0); + DataList_t* bdl = dl->dataList(); + FifoDataList* fifo = fInputJobStepAssociation.outAt(0)->fifoDL(); - idbassert(bdl); - int it = -1; - bool more; - ElementType e; - int64_t token; + idbassert(bdl); + int it = -1; + bool more; + ElementType e; + int64_t token; - if (fifo != NULL) + if (fifo != NULL) + { + try { - try - { - it = fifo->getIterator(); - } - catch (exception& ex) - { - cerr << "pColStep::addFilters: caught exception: " << ex.what() << " stepno: " << - fStepId << endl; - } - catch (...) - { - cerr << "pColStep::addFilters: caught exception" << endl; - } - - fBOP = BOP_OR; - UintRowGroup rw; - - more = fifo->next(it, &rw); - - while (more) - { - for (uint64_t i = 0; i < rw.count; ++i) - addFilter(COMPARE_EQ, (int64_t) rw.et[i].second); - - more = fifo->next(it, &rw); - } + it = fifo->getIterator(); } - else + catch (exception& ex) { - try - { - it = bdl->getIterator(); - } - catch (exception& ex) - { - cerr << "pColStep::addFilters: caught exception: " << ex.what() << " stepno: " << - fStepId << endl; - } - catch (...) - { - cerr << "pColStep::addFilters: caught exception" << endl; - } - - fBOP = BOP_OR; - - more = bdl->next(it, &e); - - while (more) - { - token = e.second; - addFilter(COMPARE_EQ, token); - - more = bdl->next(it, &e); - } + cerr << "pColStep::addFilters: caught exception: " << ex.what() << " stepno: " << fStepId << endl; + } + catch (...) + { + cerr << "pColStep::addFilters: caught exception" << endl; } - return; + fBOP = BOP_OR; + UintRowGroup rw; + + more = fifo->next(it, &rw); + + while (more) + { + for (uint64_t i = 0; i < rw.count; ++i) + addFilter(COMPARE_EQ, (int64_t)rw.et[i].second); + + more = fifo->next(it, &rw); + } + } + else + { + try + { + it = bdl->getIterator(); + } + catch (exception& ex) + { + cerr << "pColStep::addFilters: caught exception: " << ex.what() << " stepno: " << fStepId << endl; + } + catch (...) + { + cerr << "pColStep::addFilters: caught exception" << endl; + } + + fBOP = BOP_OR; + + more = bdl->next(it, &e); + + while (more) + { + token = e.second; + addFilter(COMPARE_EQ, token); + + more = bdl->next(it, &e); + } + } + + return; } /* This exists to avoid a DBRM lookup for every rid. */ inline uint64_t pColStep::getLBID(uint64_t rid, bool& scan) { - uint32_t extentIndex, extentOffset; - uint64_t fbo; - fbo = rid >> rpbShift; - extentIndex = fbo >> divShift; - extentOffset = fbo & modMask; - scan = (scanFlags[extentIndex] != 0); - return extents[extentIndex].range.start + extentOffset; + uint32_t extentIndex, extentOffset; + uint64_t fbo; + fbo = rid >> rpbShift; + extentIndex = fbo >> divShift; + extentOffset = fbo & modMask; + scan = (scanFlags[extentIndex] != 0); + return extents[extentIndex].range.start + extentOffset; } inline uint64_t pColStep::getFBO(uint64_t lbid) { - uint32_t i; - uint64_t lastLBID; + uint32_t i; + uint64_t lastLBID; - for (i = 0; i < numExtents; i++) - { - lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; + for (i = 0; i < numExtents; i++) + { + lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; - if (lbid >= (uint64_t) extents[i].range.start && lbid <= lastLBID) - return (lbid - extents[i].range.start) + (i << divShift); - } + if (lbid >= (uint64_t)extents[i].range.start && lbid <= lastLBID) + return (lbid - extents[i].range.start) + (i << divShift); + } - cerr << "pColStep: didn't find the FBO?\n"; - throw logic_error("pColStep: didn't find the FBO?"); + cerr << "pColStep: didn't find the FBO?\n"; + throw logic_error("pColStep: didn't find the FBO?"); } - void pColStep::appendFilter(const messageqcpp::ByteStream& filter, unsigned count) { - fFilterString += filter; - fFilterCount += count; + fFilterString += filter; + fFilterCount += count; } - void pColStep::addFilter(const Filter* f) { - if (NULL != f) - fFilters.push_back(f); + if (NULL != f) + fFilters.push_back(f); } - void pColStep::appendFilter(const std::vector& fs) { - fFilters.insert(fFilters.end(), fs.begin(), fs.end()); + fFilters.insert(fFilters.end(), fs.begin(), fs.end()); } -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/pdictionary.cpp b/dbcon/joblist/pdictionary.cpp index 97ace8f77..71a941713 100644 --- a/dbcon/joblist/pdictionary.cpp +++ b/dbcon/joblist/pdictionary.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: pdictionary.cpp 9655 2013-06-25 23:08:13Z xlou $ -* -* -***********************************************************************/ + * $Id: pdictionary.cpp 9655 2013-06-25 23:08:13Z xlou $ + * + * + ***********************************************************************/ #include #include @@ -52,8 +52,7 @@ using namespace BRM; namespace joblist { - -//struct pDictionaryStepPrimitive +// struct pDictionaryStepPrimitive //{ // pDictionaryStepPrimitive(pDictionaryStep* pDictStep) : fPDictionaryStep(pDictStep) // {} @@ -72,7 +71,7 @@ namespace joblist // //}; // -//struct pDictStepAggregator +// struct pDictStepAggregator //{ // pDictStepAggregator(pDictionaryStep* pDictStep) : fPDictStep(pDictStep) // {} @@ -89,495 +88,478 @@ namespace joblist // } //}; - - -pDictionaryStep::pDictionaryStep( - CalpontSystemCatalog::OID o, - CalpontSystemCatalog::OID t, - const CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fOid(o), - fTableOid(t), - fBOP(BOP_NONE), - msgsSent(0), - msgsRecvd(0), - finishedSending(false), - recvWaiting(false), - ridCount(0), - fColType(ct), - pThread(0), - cThread(0), - fFilterCount(0), - requestList(0), - fInterval(jobInfo.flushInterval), - fPhysicalIO(0), - fCacheIO(0), - fMsgBytesIn(0), - fMsgBytesOut(0), - fRm(jobInfo.rm), - hasEqualityFilter(false) +pDictionaryStep::pDictionaryStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, + const CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) + : JobStep(jobInfo) + , fOid(o) + , fTableOid(t) + , fBOP(BOP_NONE) + , msgsSent(0) + , msgsRecvd(0) + , finishedSending(false) + , recvWaiting(false) + , ridCount(0) + , fColType(ct) + , pThread(0) + , cThread(0) + , fFilterCount(0) + , requestList(0) + , fInterval(jobInfo.flushInterval) + , fPhysicalIO(0) + , fCacheIO(0) + , fMsgBytesIn(0) + , fMsgBytesOut(0) + , fRm(jobInfo.rm) + , hasEqualityFilter(false) { -// uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); -// fColType.compressionType = fColType.ddn.compressionType = ct; + // fColType.compressionType = fColType.ddn.compressionType = ct; } pDictionaryStep::~pDictionaryStep() { -// if (fDec) -// fDec->removeQueue(uniqueID); + // if (fDec) + // fDec->removeQueue(uniqueID); } void pDictionaryStep::startPrimitiveThread() { -// pThread.reset(new boost::thread(pDictionaryStepPrimitive(this))); + // pThread.reset(new boost::thread(pDictionaryStepPrimitive(this))); } void pDictionaryStep::startAggregationThread() { -// cThread.reset(new boost::thread(pDictStepAggregator(this))); + // cThread.reset(new boost::thread(pDictStepAggregator(this))); } void pDictionaryStep::run() { -// if (traceOn()) -// { -// syslogStartStep(16, // exemgr subsystem -// std::string("pDictionaryStep")); // step name -// } -// -// const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); -// DataList_t* dlp = dl->dataList(); -// setInputList(dlp); -// -// startPrimitiveThread(); -// startAggregationThread(); + // if (traceOn()) + // { + // syslogStartStep(16, // exemgr subsystem + // std::string("pDictionaryStep")); // step name + // } + // + // const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); + // DataList_t* dlp = dl->dataList(); + // setInputList(dlp); + // + // startPrimitiveThread(); + // startAggregationThread(); } void pDictionaryStep::join() { -// pThread->join(); -// cThread->join(); + // pThread->join(); + // cThread->join(); } void pDictionaryStep::setInputList(DataList_t* dl) { - requestList = dl; + requestList = dl; } void pDictionaryStep::setBOP(int8_t b) { - fBOP = b; + fBOP = b; } void pDictionaryStep::addFilter(int8_t COP, const string& value) { - fFilterString << (uint8_t) COP; - fFilterString << (uint16_t) value.size(); - fFilterString.append((const uint8_t*) value.c_str(), value.size()); - fFilterCount++; + fFilterString << (uint8_t)COP; + fFilterString << (uint16_t)value.size(); + fFilterString.append((const uint8_t*)value.c_str(), value.size()); + fFilterCount++; - if (fFilterCount == 1 && (COP == COMPARE_EQ || COP == COMPARE_NE)) - { - hasEqualityFilter = true; - tmpCOP = COP; - } + if (fFilterCount == 1 && (COP == COMPARE_EQ || COP == COMPARE_NE)) + { + hasEqualityFilter = true; + tmpCOP = COP; + } - if (hasEqualityFilter) + if (hasEqualityFilter) + { + if (COP != tmpCOP) { - if (COP != tmpCOP) - { - hasEqualityFilter = false; - eqFilter.clear(); - } - else - eqFilter.push_back(value); + hasEqualityFilter = false; + eqFilter.clear(); } + else + eqFilter.push_back(value); + } } void pDictionaryStep::sendPrimitiveMessages() { -// int it = -1; -// int msgRidCount = 0; -// bool more; -// int64_t sigToken, msgLBID, nextLBID = -1; -// uint16_t sigOrd; -// ByteStream msgRidList, primMsg(65536); //the MAX_BUFFER_SIZE as of 8/20 -// DictSignatureRequestHeader hdr; -// ISMPacketHeader ism; -// OldGetSigParams pt; -// FifoDataList* fifo = fInputJobStepAssociation.outAt(0)->fifoDL(); -// UintRowGroup rw; -// -///* XXXPAT: Does this primitive need to care about the HWM as a sanity check, given -//that a ridlist is supplied? */ -// -// if (fifo == 0) -// throw logic_error("Use p_colscanrange instead here"); -// -// try{ -// it = fifo->getIterator(); -// }catch(exception& ex) { -// cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception: " << ex.what() << endl; -// }catch(...) { -// cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception" << endl; -// } -// -// more = fifo->next(it, &rw); -// -// sigToken = rw.et[0].second; -// msgLBID = sigToken >> 10; -// while (more || msgRidCount > 0) { -// for (uint64_t i = 0; ((i < rw.count) || (!more && msgRidCount > 0)); ) -// { -// if (more) -// { -// ridCount++; -// sigToken = rw.et[i].second; -// nextLBID = sigToken >> 10; -//#ifdef DEBUG -// cout << "sigToken = " << sigToken << " lbid = " << nextLBID << endl; -//#endif -// } -// -// // @bug 472 -// if (nextLBID == msgLBID && more && msgRidCount < 8000) { //XXXPAT: need to prove N & S here -// sigOrd = sigToken & 0x3ff; -// pt.rid = (nextLBID >= 0 ? rw.et[i].first : 0x8000000000000000LL | rw.et[i].first); -// pt.offsetIndex = sigOrd; -// msgRidList.append(reinterpret_cast(&pt), sizeof(pt)); -// msgRidCount++; -// ++i; -//#ifdef DEBUG -// cout << "added signature ordinal " << sigOrd << endl; -//#endif -// } -// else { -//#ifdef DEBUG -// cout << "sending a prim msg" << endl; -//#endif -// -// // send the primitive, start constructing the next msg -// ism.Interleave=0; -// ism.Flags=planFlagsToPrimFlags(fTraceFlags); -// ism.Command=DICT_SIGNATURE; -// ism.Size=sizeof(DictSignatureRequestHeader) + msgRidList.length(); -// ism.Type=2; -// -// hdr.Hdr.SessionID = fSessionId; -// //hdr.Hdr.StatementID = 0; -// hdr.Hdr.TransactionID = fTxnId; -// hdr.Hdr.VerID = fVerId; -// hdr.Hdr.StepID = fStepId; -// hdr.Hdr.UniqueID = uniqueID; -// -// hdr.LBID = msgLBID; -// idbassert(msgRidCount <= 8000); -// hdr.NVALS = msgRidCount; -// hdr.CompType = fColType.ddn.compressionType; -// -// primMsg.load((const uint8_t *) &ism, sizeof(ism)); -// primMsg.append((const uint8_t *) &hdr, sizeof(DictSignatureRequestHeader)); -// primMsg += msgRidList; -// fMsgBytesOut += primMsg.lengthWithHdrOverhead(); -// fDec->write(primMsg); -// -// msgLBID = nextLBID; -// primMsg.restart(); -// msgRidList.restart(); -// msgRidCount = 0; -// -// mutex.lock(); -// msgsSent++; -// if (recvWaiting) -// condvar.notify_one(); -//#ifdef DEBUG -// cout << "msgsSent++ = " << msgsSent << endl; -//#endif -// mutex.unlock(); -// -// if (!more) -// break; -// } -// } // rw.count -// -// if (more) -// { -// rw.count = 0; -// more = fifo->next(it, &rw); -// } -// } -// -// mutex.lock(); -// finishedSending = true; -// if (recvWaiting) -// condvar.notify_one(); -// mutex.unlock(); + // int it = -1; + // int msgRidCount = 0; + // bool more; + // int64_t sigToken, msgLBID, nextLBID = -1; + // uint16_t sigOrd; + // ByteStream msgRidList, primMsg(65536); //the MAX_BUFFER_SIZE as of 8/20 + // DictSignatureRequestHeader hdr; + // ISMPacketHeader ism; + // OldGetSigParams pt; + // FifoDataList* fifo = fInputJobStepAssociation.outAt(0)->fifoDL(); + // UintRowGroup rw; + // + ///* XXXPAT: Does this primitive need to care about the HWM as a sanity check, given + // that a ridlist is supplied? */ + // + // if (fifo == 0) + // throw logic_error("Use p_colscanrange instead here"); + // + // try{ + // it = fifo->getIterator(); + // }catch(exception& ex) { + // cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception: " << ex.what() << endl; + // }catch(...) { + // cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception" << endl; + // } + // + // more = fifo->next(it, &rw); + // + // sigToken = rw.et[0].second; + // msgLBID = sigToken >> 10; + // while (more || msgRidCount > 0) { + // for (uint64_t i = 0; ((i < rw.count) || (!more && msgRidCount > 0)); ) + // { + // if (more) + // { + // ridCount++; + // sigToken = rw.et[i].second; + // nextLBID = sigToken >> 10; + //#ifdef DEBUG + // cout << "sigToken = " << sigToken << " lbid = " << nextLBID << endl; + //#endif + // } + // + // // @bug 472 + // if (nextLBID == msgLBID && more && msgRidCount < 8000) { //XXXPAT: need to prove N & S + //here sigOrd = sigToken & 0x3ff; pt.rid = (nextLBID >= 0 ? rw.et[i].first : 0x8000000000000000LL | + //rw.et[i].first); pt.offsetIndex = sigOrd; msgRidList.append(reinterpret_cast(&pt), + //sizeof(pt)); msgRidCount++; + // ++i; + //#ifdef DEBUG + // cout << "added signature ordinal " << sigOrd << endl; + //#endif + // } + // else { + //#ifdef DEBUG + // cout << "sending a prim msg" << endl; + //#endif + // + // // send the primitive, start constructing the next msg + // ism.Interleave=0; + // ism.Flags=planFlagsToPrimFlags(fTraceFlags); + // ism.Command=DICT_SIGNATURE; + // ism.Size=sizeof(DictSignatureRequestHeader) + msgRidList.length(); + // ism.Type=2; + // + // hdr.Hdr.SessionID = fSessionId; + // //hdr.Hdr.StatementID = 0; + // hdr.Hdr.TransactionID = fTxnId; + // hdr.Hdr.VerID = fVerId; + // hdr.Hdr.StepID = fStepId; + // hdr.Hdr.UniqueID = uniqueID; + // + // hdr.LBID = msgLBID; + // idbassert(msgRidCount <= 8000); + // hdr.NVALS = msgRidCount; + // hdr.CompType = fColType.ddn.compressionType; + // + // primMsg.load((const uint8_t *) &ism, sizeof(ism)); + // primMsg.append((const uint8_t *) &hdr, sizeof(DictSignatureRequestHeader)); + // primMsg += msgRidList; + // fMsgBytesOut += primMsg.lengthWithHdrOverhead(); + // fDec->write(primMsg); + // + // msgLBID = nextLBID; + // primMsg.restart(); + // msgRidList.restart(); + // msgRidCount = 0; + // + // mutex.lock(); + // msgsSent++; + // if (recvWaiting) + // condvar.notify_one(); + //#ifdef DEBUG + // cout << "msgsSent++ = " << msgsSent << endl; + //#endif + // mutex.unlock(); + // + // if (!more) + // break; + // } + // } // rw.count + // + // if (more) + // { + // rw.count = 0; + // more = fifo->next(it, &rw); + // } + // } + // + // mutex.lock(); + // finishedSending = true; + // if (recvWaiting) + // condvar.notify_one(); + // mutex.unlock(); } void pDictionaryStep::receivePrimitiveMessages() { -// int64_t ridResults = 0; -// AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); -// StrDataList* dlp = dl->stringDataList(); -// StringFifoDataList *fifo = fOutputJobStepAssociation.outAt(0)->stringDL(); -// StringRowGroup rw; -// -// while (1) { -// -// // sync with the send side -// mutex.lock(); -// -// while (!finishedSending && msgsSent==msgsRecvd) { -// recvWaiting = true; -// condvar.wait(mutex); -// if (msgsSent == msgsRecvd) { -// mutex.unlock(); -// break; -// } -// recvWaiting = false; -// } -// -// if (finishedSending != 0 && msgsRecvd >= msgsSent) { -// goto junk; -// } -// mutex.unlock(); -// -// // do the recv -// -// ByteStream bs = fDec->read(uniqueID); -// fMsgBytesIn += bs.lengthWithHdrOverhead(); -// if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) -// dlTimes.setFirstReadTime(); -// if (fOid>=3000) dlTimes.setLastReadTime(); -// -// msgsRecvd++; -// if (bs.length() == 0) -// break; -// -// const ByteStream::byte* bsp = bs.buf(); -// -// // get the ResultHeader out of the bytestream -// const DictOutput* drh = reinterpret_cast(bsp); -// -// bsp += sizeof(DictOutput); -// -// fCacheIO += drh->CacheIO; -// fPhysicalIO += drh->PhysicalIO; -// -// // From this point on the rest of the bytestream is the data that comes back from the primitive server -// // This needs to be fed to a datalist that is retrieved from the outputassociation object. -// -// char d[8192]; -//// memset(d, 0, 8192); -// if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) -// dlTimes.setFirstInsertTime(); -// for(int j = 0; j < drh->NVALS; j++) -// { -// const uint64_t* ridp = (const uint64_t*)bsp; -// bsp += sizeof(*ridp); -// uint64_t rid = *ridp; -// const uint16_t* lenp = (const uint16_t*)bsp; -// bsp += sizeof(*lenp); -// uint16_t len = *lenp; -// memcpy(d, bsp, len); -// bsp += len; -// d[len] = 0; -// if (rid == 0xFFFFFFFFFFFFFFFFULL) -// { -// strcpy(d, CPNULLSTRMARK.c_str()); -// } -//#ifdef FIFO_SINK -// if (fOid < 3000) -//#endif -// if (fifo) -// { -// rw.et[rw.count++] = StringElementType(rid, d); -// if (rw.count == rw.ElementsPerGroup) -// { -// fifo->insert(rw); -// rw.count = 0; -// } -// } -// else -// { -// dlp->insert(StringElementType(rid, d)); -// } -// -//#ifdef DEBUG -// cout << " -- inserting <" << rid << ", " << d << ">" << endl; -//#endif -// ridResults++; -// -// } -// } -// -//junk: -// -// if (fifo && rw.count > 0) -// fifo->insert(rw); -// -// //@bug 699: Reset StepMsgQueue -// fDec->removeQueue(uniqueID); -// -// if (fOid>=3000) dlTimes.setEndOfInputTime(); -// dlp->endOfInput(); -// -// if (fTableOid >= 3000) -// { -// //...Construct timestamp using ctime_r() instead of ctime() not -// //...necessarily due to re-entrancy, but because we want to strip -// //...the newline ('\n') off the end of the formatted string. -// time_t t = time(0); -// char timeString[50]; -// ctime_r(&t, timeString); -// timeString[strlen(timeString)-1 ] = '\0'; -// -// FifoDataList* pFifo = 0; -// uint64_t totalBlockedReadCount = 0; -// uint64_t totalBlockedWriteCount = 0; -// -// //...Sum up the blocked FIFO reads for all input associations -// size_t inDlCnt = fInputJobStepAssociation.outSize(); -// for (size_t iDataList=0; iDataListfifoDL(); -// if (pFifo) -// { -// totalBlockedReadCount += pFifo->blockedReadCount(); -// } -// } -// -// //...Sum up the blocked FIFO writes for all output associations -// size_t outDlCnt = fOutputJobStepAssociation.outSize(); -// for (size_t iDataList=0; iDataListfifoDL(); -// if (pFifo) -// { -// totalBlockedWriteCount += pFifo->blockedWriteCount(); -// } -// } -// -// -// -// //...Roundoff msg byte counts to nearest KB for display -// uint64_t msgBytesInKB = fMsgBytesIn >> 10; -// uint64_t msgBytesOutKB = fMsgBytesOut >> 10; -// if (fMsgBytesIn & 512) -// msgBytesInKB++; -// if (fMsgBytesOut & 512) -// msgBytesOutKB++; -// -// // @bug 807 -// if (fifo) -// fifo->totalSize(ridResults); -// -// if (traceOn()) -// { -// //...Print job step completion information -// ostringstream logStr; -// logStr << "ses:" << fSessionId << " st: " << fStepId << -// " finished at " << -// timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << -// fCacheIO << "; MsgsRcvd-" << msgsRecvd << -// "; BlockedFifoIn/Out-" << totalBlockedReadCount << -// "/" << totalBlockedWriteCount << -// "; output size-" << ridResults << endl << -// "\tMsgBytesIn-" << msgBytesInKB << "KB" << -// "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << -// "\t1st read " << dlTimes.FirstReadTimeString() << -// "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << -// JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << -// "s" << endl; -// -// logEnd(logStr.str().c_str()); -// -// syslogReadBlockCounts(16, // exemgr subsystem -// fPhysicalIO, // # blocks read from disk -// fCacheIO, // # blocks read from cache -// 0); // # casual partition block hits -// syslogProcessingTimes(16, // exemgr subsystem -// dlTimes.FirstReadTime(), // first datalist read -// dlTimes.LastReadTime(), // last datalist read -// dlTimes.FirstInsertTime(), // first datalist write -// dlTimes.EndOfInputTime()); // last (endOfInput) datalist write -// syslogEndStep(16, // exemgr subsystem -// totalBlockedReadCount, // blocked datalist input -// totalBlockedWriteCount, // blocked datalist output -// fMsgBytesIn, // incoming msg byte count -// fMsgBytesOut); // outgoing msg byte count -// } -// } -// + // int64_t ridResults = 0; + // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); + // StrDataList* dlp = dl->stringDataList(); + // StringFifoDataList *fifo = fOutputJobStepAssociation.outAt(0)->stringDL(); + // StringRowGroup rw; + // + // while (1) { + // + // // sync with the send side + // mutex.lock(); + // + // while (!finishedSending && msgsSent==msgsRecvd) { + // recvWaiting = true; + // condvar.wait(mutex); + // if (msgsSent == msgsRecvd) { + // mutex.unlock(); + // break; + // } + // recvWaiting = false; + // } + // + // if (finishedSending != 0 && msgsRecvd >= msgsSent) { + // goto junk; + // } + // mutex.unlock(); + // + // // do the recv + // + // ByteStream bs = fDec->read(uniqueID); + // fMsgBytesIn += bs.lengthWithHdrOverhead(); + // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) + // dlTimes.setFirstReadTime(); + // if (fOid>=3000) dlTimes.setLastReadTime(); + // + // msgsRecvd++; + // if (bs.length() == 0) + // break; + // + // const ByteStream::byte* bsp = bs.buf(); + // + // // get the ResultHeader out of the bytestream + // const DictOutput* drh = reinterpret_cast(bsp); + // + // bsp += sizeof(DictOutput); + // + // fCacheIO += drh->CacheIO; + // fPhysicalIO += drh->PhysicalIO; + // + // // From this point on the rest of the bytestream is the data that comes back from the primitive + //server + // // This needs to be fed to a datalist that is retrieved from the outputassociation object. + // + // char d[8192]; + //// memset(d, 0, 8192); + // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) + // dlTimes.setFirstInsertTime(); + // for(int j = 0; j < drh->NVALS; j++) + // { + // const uint64_t* ridp = (const uint64_t*)bsp; + // bsp += sizeof(*ridp); + // uint64_t rid = *ridp; + // const uint16_t* lenp = (const uint16_t*)bsp; + // bsp += sizeof(*lenp); + // uint16_t len = *lenp; + // memcpy(d, bsp, len); + // bsp += len; + // d[len] = 0; + // if (rid == 0xFFFFFFFFFFFFFFFFULL) + // { + // strcpy(d, CPNULLSTRMARK.c_str()); + // } + //#ifdef FIFO_SINK + // if (fOid < 3000) + //#endif + // if (fifo) + // { + // rw.et[rw.count++] = StringElementType(rid, d); + // if (rw.count == rw.ElementsPerGroup) + // { + // fifo->insert(rw); + // rw.count = 0; + // } + // } + // else + // { + // dlp->insert(StringElementType(rid, d)); + // } + // + //#ifdef DEBUG + // cout << " -- inserting <" << rid << ", " << d << ">" << endl; + //#endif + // ridResults++; + // + // } + // } + // + // junk: + // + // if (fifo && rw.count > 0) + // fifo->insert(rw); + // + // //@bug 699: Reset StepMsgQueue + // fDec->removeQueue(uniqueID); + // + // if (fOid>=3000) dlTimes.setEndOfInputTime(); + // dlp->endOfInput(); + // + // if (fTableOid >= 3000) + // { + // //...Construct timestamp using ctime_r() instead of ctime() not + // //...necessarily due to re-entrancy, but because we want to strip + // //...the newline ('\n') off the end of the formatted string. + // time_t t = time(0); + // char timeString[50]; + // ctime_r(&t, timeString); + // timeString[strlen(timeString)-1 ] = '\0'; + // + // FifoDataList* pFifo = 0; + // uint64_t totalBlockedReadCount = 0; + // uint64_t totalBlockedWriteCount = 0; + // + // //...Sum up the blocked FIFO reads for all input associations + // size_t inDlCnt = fInputJobStepAssociation.outSize(); + // for (size_t iDataList=0; iDataListfifoDL(); + // if (pFifo) + // { + // totalBlockedReadCount += pFifo->blockedReadCount(); + // } + // } + // + // //...Sum up the blocked FIFO writes for all output associations + // size_t outDlCnt = fOutputJobStepAssociation.outSize(); + // for (size_t iDataList=0; iDataListfifoDL(); + // if (pFifo) + // { + // totalBlockedWriteCount += pFifo->blockedWriteCount(); + // } + // } + // + // + // + // //...Roundoff msg byte counts to nearest KB for display + // uint64_t msgBytesInKB = fMsgBytesIn >> 10; + // uint64_t msgBytesOutKB = fMsgBytesOut >> 10; + // if (fMsgBytesIn & 512) + // msgBytesInKB++; + // if (fMsgBytesOut & 512) + // msgBytesOutKB++; + // + // // @bug 807 + // if (fifo) + // fifo->totalSize(ridResults); + // + // if (traceOn()) + // { + // //...Print job step completion information + // ostringstream logStr; + // logStr << "ses:" << fSessionId << " st: " << fStepId << + // " finished at " << + // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << + // fCacheIO << "; MsgsRcvd-" << msgsRecvd << + // "; BlockedFifoIn/Out-" << totalBlockedReadCount << + // "/" << totalBlockedWriteCount << + // "; output size-" << ridResults << endl << + // "\tMsgBytesIn-" << msgBytesInKB << "KB" << + // "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << + // "\t1st read " << dlTimes.FirstReadTimeString() << + // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << + // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << + // "s" << endl; + // + // logEnd(logStr.str().c_str()); + // + // syslogReadBlockCounts(16, // exemgr subsystem + // fPhysicalIO, // # blocks read from disk + // fCacheIO, // # blocks read from cache + // 0); // # casual partition block hits + // syslogProcessingTimes(16, // exemgr subsystem + // dlTimes.FirstReadTime(), // first datalist read + // dlTimes.LastReadTime(), // last datalist read + // dlTimes.FirstInsertTime(), // first datalist write + // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write + // syslogEndStep(16, // exemgr subsystem + // totalBlockedReadCount, // blocked datalist input + // totalBlockedWriteCount, // blocked datalist output + // fMsgBytesIn, // incoming msg byte count + // fMsgBytesOut); // outgoing msg byte count + // } + // } + // } const string pDictionaryStep::toString() const { - ostringstream oss; + ostringstream oss; - oss << "pDictionaryStep ses:" - << fSessionId << " txn:" - << fTxnId << " ver:" - << fVerId << " st:" - << fStepId << " tb/col:" - << fTableOid << "/" << fOid; - oss << " " << omitOidInDL - << fOutputJobStepAssociation.outAt(0) << showOidInDL; - oss << " in:"; + oss << "pDictionaryStep ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId + << " tb/col:" << fTableOid << "/" << fOid; + oss << " " << omitOidInDL << fOutputJobStepAssociation.outAt(0) << showOidInDL; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - { - oss << fInputJobStepAssociation.outAt(i) << ", "; - } + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + oss << fInputJobStepAssociation.outAt(i) << ", "; + } #ifdef FIFO_SINK - if (fOid < 3000) - oss << " (sink)"; + if (fOid < 3000) + oss << " (sink)"; #endif - return oss.str(); - } + return oss.str(); +} void pDictionaryStep::appendFilter(const messageqcpp::ByteStream& filter, unsigned count) { - ByteStream bs(filter); // need to preserve the input BS - uint8_t* buf; - uint8_t COP; - uint16_t size; - string value; + ByteStream bs(filter); // need to preserve the input BS + uint8_t* buf; + uint8_t COP; + uint16_t size; + string value; - while (bs.length() > 0) - { - bs >> COP; - bs >> size; - buf = bs.buf(); - value = string((char*) buf, size); - addFilter(COP, value); - bs.advance(size); - } + while (bs.length() > 0) + { + bs >> COP; + bs >> size; + buf = bs.buf(); + value = string((char*)buf, size); + addFilter(COP, value); + bs.advance(size); + } - - - - //fFilterString += filter; - //fFilterCount += count; + // fFilterString += filter; + // fFilterCount += count; } - void pDictionaryStep::addFilter(const Filter* f) { - if (NULL != f) - fFilters.push_back(f); + if (NULL != f) + fFilters.push_back(f); } - void pDictionaryStep::appendFilter(const std::vector& fs) { - fFilters.insert(fFilters.end(), fs.begin(), fs.end()); + fFilters.insert(fFilters.end(), fs.begin(), fs.end()); } - -} //namespace +} // namespace joblist diff --git a/dbcon/joblist/pdictionaryscan.cpp b/dbcon/joblist/pdictionaryscan.cpp index c5e8745ae..df795b862 100644 --- a/dbcon/joblist/pdictionaryscan.cpp +++ b/dbcon/joblist/pdictionaryscan.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: pdictionaryscan.cpp 9655 2013-06-25 23:08:13Z xlou $ -* -* -***********************************************************************/ + * $Id: pdictionaryscan.cpp 9655 2013-06-25 23:08:13Z xlou $ + * + * + ***********************************************************************/ #include #include #include @@ -69,154 +69,149 @@ using namespace querytele; namespace joblist { - struct pDictionaryScanPrimitive { - pDictionaryScanPrimitive(pDictionaryScan* pDictScan) : fPDictScan(pDictScan) - {} - pDictionaryScan* fPDictScan; - void operator()() + pDictionaryScanPrimitive(pDictionaryScan* pDictScan) : fPDictScan(pDictScan) + { + } + pDictionaryScan* fPDictScan; + void operator()() + { + try { - try - { - utils::setThreadName("DSSScan"); - fPDictScan->sendPrimitiveMessages(); - } - catch (runtime_error& re) - { - catchHandler(re.what(), ERR_DICTIONARY_SCAN, fPDictScan->errorInfo(), fPDictScan->sessionId()); - } - catch (...) - { - catchHandler("pDictionaryScan send caught an unknown exception", - ERR_DICTIONARY_SCAN, fPDictScan->errorInfo(), fPDictScan->sessionId()); - } - + utils::setThreadName("DSSScan"); + fPDictScan->sendPrimitiveMessages(); } + catch (runtime_error& re) + { + catchHandler(re.what(), ERR_DICTIONARY_SCAN, fPDictScan->errorInfo(), fPDictScan->sessionId()); + } + catch (...) + { + catchHandler("pDictionaryScan send caught an unknown exception", ERR_DICTIONARY_SCAN, + fPDictScan->errorInfo(), fPDictScan->sessionId()); + } + } }; struct pDictionaryScanAggregator { - pDictionaryScanAggregator(pDictionaryScan* pDictScan) : fPDictScan(pDictScan) - {} - pDictionaryScan* fPDictScan; - void operator()() + pDictionaryScanAggregator(pDictionaryScan* pDictScan) : fPDictScan(pDictScan) + { + } + pDictionaryScan* fPDictScan; + void operator()() + { + try { - try - { - utils::setThreadName("DSSAgg"); - fPDictScan->receivePrimitiveMessages(); - } - catch (runtime_error& re) - { - catchHandler(re.what(), ERR_DICTIONARY_SCAN, fPDictScan->errorInfo(), fPDictScan->sessionId()); - } - catch (...) - { - catchHandler("pDictionaryScan receive caught an unknown exception", - ERR_DICTIONARY_SCAN, fPDictScan->errorInfo(), fPDictScan->sessionId()); - } - + utils::setThreadName("DSSAgg"); + fPDictScan->receivePrimitiveMessages(); } + catch (runtime_error& re) + { + catchHandler(re.what(), ERR_DICTIONARY_SCAN, fPDictScan->errorInfo(), fPDictScan->sessionId()); + } + catch (...) + { + catchHandler("pDictionaryScan receive caught an unknown exception", ERR_DICTIONARY_SCAN, + fPDictScan->errorInfo(), fPDictScan->sessionId()); + } + } }; - -pDictionaryScan::pDictionaryScan( - CalpontSystemCatalog::OID o, - CalpontSystemCatalog::OID t, - const CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fDec(NULL), - sysCat(jobInfo.csc), - fOid(o), - fTableOid(t), - fFilterCount(0), - fBOP(BOP_NONE), - msgsSent(0), - msgsRecvd(0), - finishedSending(false), - recvWaiting(false), - sendWaiting(false), - ridCount(0), - ridList(0), - fColType(ct), - pThread(0), - cThread(0), - fScanLbidReqLimit(jobInfo.rm->getJlScanLbidReqLimit()), - fScanLbidReqThreshold(jobInfo.rm->getJlScanLbidReqThreshold()), - fStopSending(false), - fSingleThread(false), - fPhysicalIO(0), - fCacheIO(0), - fMsgBytesIn(0), - fMsgBytesOut(0), - fMsgsToPm(0), - fMsgsExpect(0), - fRm(jobInfo.rm), - isEquality(false) +pDictionaryScan::pDictionaryScan(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, + const CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) + : JobStep(jobInfo) + , fDec(NULL) + , sysCat(jobInfo.csc) + , fOid(o) + , fTableOid(t) + , fFilterCount(0) + , fBOP(BOP_NONE) + , msgsSent(0) + , msgsRecvd(0) + , finishedSending(false) + , recvWaiting(false) + , sendWaiting(false) + , ridCount(0) + , ridList(0) + , fColType(ct) + , pThread(0) + , cThread(0) + , fScanLbidReqLimit(jobInfo.rm->getJlScanLbidReqLimit()) + , fScanLbidReqThreshold(jobInfo.rm->getJlScanLbidReqThreshold()) + , fStopSending(false) + , fSingleThread(false) + , fPhysicalIO(0) + , fCacheIO(0) + , fMsgBytesIn(0) + , fMsgBytesOut(0) + , fMsgsToPm(0) + , fMsgsExpect(0) + , fRm(jobInfo.rm) + , isEquality(false) { - int err; - DBRM dbrm; + int err; + DBRM dbrm; - // Get list of non outOfService extents for the OID of interest - err = dbrm.lookup(fOid, fDictlbids); + // Get list of non outOfService extents for the OID of interest + err = dbrm.lookup(fOid, fDictlbids); - if (err) + if (err) + { + ostringstream oss; + oss << "pDictionaryScan: lookup error (2)! For OID-" << fOid; + throw runtime_error(oss.str()); + } + + err = dbrm.getExtents(fOid, extents); + + if (err) + { + ostringstream oss; + oss << "pDictionaryScan: dbrm.getExtents error! For OID-" << fOid; + throw runtime_error(oss.str()); + } + + sort(extents.begin(), extents.end(), ExtentSorter()); + numExtents = extents.size(); + extentSize = (fRm->getExtentRows() * 8) / BLOCK_SIZE; + + uint64_t i = 1, mask = 1; + + for (; i <= 32; i++) + { + mask <<= 1; + + if (extentSize & mask) { - ostringstream oss; - oss << "pDictionaryScan: lookup error (2)! For OID-" << fOid; - throw runtime_error(oss.str()); + divShift = i; + break; } + } - err = dbrm.getExtents(fOid, extents); + for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) + if (extentSize & mask) + throw runtime_error("pDictionaryScan: Extent size must be a power of 2 in blocks"); - if (err) - { - ostringstream oss; - oss << "pDictionaryScan: dbrm.getExtents error! For OID-" << fOid; - throw runtime_error(oss.str()); - } + fCOP1 = COMPARE_NIL; + fCOP2 = COMPARE_NIL; - sort(extents.begin(), extents.end(), ExtentSorter()); - numExtents = extents.size(); - extentSize = (fRm->getExtentRows() * 8) / BLOCK_SIZE; - - uint64_t i = 1, mask = 1; - - for (; i <= 32; i++) - { - mask <<= 1; - - if (extentSize & mask) - { - divShift = i; - break; - } - } - - for (i++, mask <<= 1; i <= 32; i++, mask <<= 1) - if (extentSize & mask) - throw runtime_error("pDictionaryScan: Extent size must be a power of 2 in blocks"); - - fCOP1 = COMPARE_NIL; - fCOP2 = COMPARE_NIL; - - uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - initializeConfigParms(); - fExtendedInfo = "DSS: "; - fQtc.stepParms().stepType = StepTeleStats::T_DSS; + uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + initializeConfigParms(); + fExtendedInfo = "DSS: "; + fQtc.stepParms().stepType = StepTeleStats::T_DSS; } pDictionaryScan::~pDictionaryScan() { - if (fDec) - { - if (isEquality) - destroyEqualityFilter(); + if (fDec) + { + if (isEquality) + destroyEqualityFilter(); - fDec->removeQueue(uniqueID); - } + fDec->removeQueue(uniqueID); + } } //------------------------------------------------------------------------------ @@ -224,744 +219,701 @@ pDictionaryScan::~pDictionaryScan() //------------------------------------------------------------------------------ void pDictionaryScan::initializeConfigParms() { - fLogicalBlocksPerScan = fRm->getJlLogicalBlocksPerScan(); + fLogicalBlocksPerScan = fRm->getJlLogicalBlocksPerScan(); } void pDictionaryScan::startPrimitiveThread() { - pThread = jobstepThreadPool.invoke(pDictionaryScanPrimitive(this)); + pThread = jobstepThreadPool.invoke(pDictionaryScanPrimitive(this)); } void pDictionaryScan::startAggregationThread() { - cThread = jobstepThreadPool.invoke(pDictionaryScanAggregator(this)); + cThread = jobstepThreadPool.invoke(pDictionaryScanAggregator(this)); } void pDictionaryScan::run() { - if (traceOn()) - { - syslogStartStep(16, // exemgr subsystem - std::string("pDictionaryScan")); // step name - } + if (traceOn()) + { + syslogStartStep(16, // exemgr subsystem + std::string("pDictionaryScan")); // step name + } - //For now, we cannot handle an input DL to this step - if (fInputJobStepAssociation.outSize() > 0) - throw logic_error("pDictionaryScan::run: don't know what to do with an input DL!"); + // For now, we cannot handle an input DL to this step + if (fInputJobStepAssociation.outSize() > 0) + throw logic_error("pDictionaryScan::run: don't know what to do with an input DL!"); - if (isEquality) - serializeEqualityFilter(); + if (isEquality) + serializeEqualityFilter(); - startPrimitiveThread(); - startAggregationThread(); + startPrimitiveThread(); + startAggregationThread(); } void pDictionaryScan::join() { - jobstepThreadPool.join(pThread); - jobstepThreadPool.join(cThread); + jobstepThreadPool.join(pThread); + jobstepThreadPool.join(cThread); - if (isEquality && fDec) - { - destroyEqualityFilter(); - isEquality = false; - } + if (isEquality && fDec) + { + destroyEqualityFilter(); + isEquality = false; + } } void pDictionaryScan::addFilter(int8_t COP, const string& value) { -// uint8_t* s = (uint8_t*)alloca(value.size() * sizeof(uint8_t)); + // uint8_t* s = (uint8_t*)alloca(value.size() * sizeof(uint8_t)); -// memcpy(s, value.data(), value.size()); -// fFilterString << (uint16_t) value.size(); -// fFilterString.append(s, value.size()); - fFilterCount++; + // memcpy(s, value.data(), value.size()); + // fFilterString << (uint16_t) value.size(); + // fFilterString.append(s, value.size()); + fFilterCount++; - if (fFilterCount == 1) + if (fFilterCount == 1) + { + fCOP1 = COP; + + if (COP == COMPARE_EQ || COP == COMPARE_NE) { - fCOP1 = COP; - - if (COP == COMPARE_EQ || COP == COMPARE_NE) - { - isEquality = true; - equalityFilter.push_back(value); - } + isEquality = true; + equalityFilter.push_back(value); } + } - if (fFilterCount == 2) + if (fFilterCount == 2) + { + fCOP2 = COP; + + // This static_cast should be safe since COP's are small, non-negative numbers + if ((COP == COMPARE_EQ || COP == COMPARE_NE) && COP == static_cast(fCOP1)) { - fCOP2 = COP; - - // This static_cast should be safe since COP's are small, non-negative numbers - if ((COP == COMPARE_EQ || COP == COMPARE_NE) && COP == static_cast(fCOP1)) - { - isEquality = true; - equalityFilter.push_back(value); - } - else - { - isEquality = false; - equalityFilter.clear(); - } - } - - if (fFilterCount > 2 && isEquality) - { - fFilterString.reset(); - equalityFilter.push_back(value); + isEquality = true; + equalityFilter.push_back(value); } else { - fFilterString << (uint16_t) value.size(); - fFilterString.append((const uint8_t*)value.data(), value.size()); + isEquality = false; + equalityFilter.clear(); } + } + + if (fFilterCount > 2 && isEquality) + { + fFilterString.reset(); + equalityFilter.push_back(value); + } + else + { + fFilterString << (uint16_t)value.size(); + fFilterString.append((const uint8_t*)value.data(), value.size()); + } } void pDictionaryScan::setRidList(DataList* dl) { - ridList = dl; + ridList = dl; } void pDictionaryScan::setBOP(int8_t b) { - fBOP = b; + fBOP = b; } void pDictionaryScan::sendPrimitiveMessages() { - LBIDRange_v::iterator it; - HWM_t hwm; - uint32_t fbo; - ByteStream primMsg(65536); - DBRM dbrm; - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - BRM::OID_t oid; - boost::shared_ptr > dbRootConnectionMap; - boost::shared_ptr > dbRootPMMap; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - int localPMId = oamCache->getLocalPMId(); + LBIDRange_v::iterator it; + HWM_t hwm; + uint32_t fbo; + ByteStream primMsg(65536); + DBRM dbrm; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + BRM::OID_t oid; + boost::shared_ptr > dbRootConnectionMap; + boost::shared_ptr > dbRootPMMap; + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + int localPMId = oamCache->getLocalPMId(); - try + try + { + dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); + dbRootPMMap = oamCache->getDBRootToPMMap(); + + it = fDictlbids.begin(); + + for (; it != fDictlbids.end() && !cancelled(); it++) { - dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); - dbRootPMMap = oamCache->getDBRootToPMMap(); + LBID_t msgLbidStart = it->start; + dbrm.lookupLocal(msgLbidStart, (BRM::VER_t)fVerId.currentScn, false, oid, dbroot, partNum, segNum, fbo); - it = fDictlbids.begin(); + // Bug5741 If we are local only and this doesn't belongs to us, skip it + if (fLocalQuery == execplan::CalpontSelectExecutionPlan::LOCAL_QUERY) + { + if (localPMId == 0) + throw IDBExcept(ERR_LOCAL_QUERY_UM); - for (; it != fDictlbids.end() && !cancelled(); it++) + if (dbRootPMMap->find(dbroot)->second != localPMId) + continue; + } + + // Retrieve the HWM blk for the segment file specified by the oid, + // partition, and segment number. The extState arg indicates + // whether the hwm block is outOfService or not, but we already + // filtered out the outOfService extents when we constructed the + // fDictlbids list, so extState is extraneous info at this point. + int extState; + dbrm.getLocalHWM(oid, partNum, segNum, hwm, extState); + + uint32_t remainingLbids = fMsgsExpect = ((hwm > (fbo + it->size - 1)) ? (it->size) : (hwm - fbo + 1)); + + uint32_t msgLbidCount = fLogicalBlocksPerScan; + + while (remainingLbids && !cancelled()) + { + if (remainingLbids < msgLbidCount) + msgLbidCount = remainingLbids; + + if (dbRootConnectionMap->find(dbroot) == dbRootConnectionMap->end()) { - LBID_t msgLbidStart = it->start; - dbrm.lookupLocal(msgLbidStart, - (BRM::VER_t)fVerId.currentScn, - false, - oid, - dbroot, - partNum, - segNum, - fbo); + // MCOL-259 force a reload of the xml. This usualy fixes it. + Logger log; + log.logMessage(logging::LOG_TYPE_DEBUG, + "dictionary forcing reload of columnstore.xml for dbRootConnectionMap"); + oamCache->forceReload(); + dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); - // Bug5741 If we are local only and this doesn't belongs to us, skip it - if (fLocalQuery == execplan::CalpontSelectExecutionPlan::LOCAL_QUERY) - { - if (localPMId == 0) - throw IDBExcept(ERR_LOCAL_QUERY_UM); + if (dbRootConnectionMap->find(dbroot) == dbRootConnectionMap->end()) + { + log.logMessage(logging::LOG_TYPE_DEBUG, "dictionary still not in dbRootConnectionMap"); + throw IDBExcept(ERR_DATA_OFFLINE); + } + } - if (dbRootPMMap->find(dbroot)->second != localPMId) - continue; - } + sendAPrimitiveMessage(primMsg, msgLbidStart, msgLbidCount, (*dbRootConnectionMap)[dbroot]); + primMsg.restart(); - // Retrieve the HWM blk for the segment file specified by the oid, - // partition, and segment number. The extState arg indicates - // whether the hwm block is outOfService or not, but we already - // filtered out the outOfService extents when we constructed the - // fDictlbids list, so extState is extraneous info at this point. - int extState; - dbrm.getLocalHWM(oid, partNum, segNum, hwm, extState); + mutex.lock(); + msgsSent += msgLbidCount; - uint32_t remainingLbids = fMsgsExpect = - ( (hwm > (fbo + it->size - 1)) ? (it->size) : (hwm - fbo + 1) ); + if (recvWaiting) + condvar.notify_all(); - uint32_t msgLbidCount = fLogicalBlocksPerScan; + while ((msgsSent - msgsRecvd) > fScanLbidReqThreshold) + { + sendWaiting = true; + condvarWakeupProducer.wait(mutex); + sendWaiting = false; + } - while ( remainingLbids && !cancelled()) - { - if ( remainingLbids < msgLbidCount) - msgLbidCount = remainingLbids; + mutex.unlock(); - if (dbRootConnectionMap->find(dbroot) == dbRootConnectionMap->end()) - { - // MCOL-259 force a reload of the xml. This usualy fixes it. - Logger log; - log.logMessage(logging::LOG_TYPE_DEBUG, "dictionary forcing reload of columnstore.xml for dbRootConnectionMap"); - oamCache->forceReload(); - dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); + remainingLbids -= msgLbidCount; + msgLbidStart += msgLbidCount; + } + } // end of loop through LBID ranges to be requested from primproc + } // try + catch (...) + { + handleException(std::current_exception(), logging::ERR_DICTIONARY_SCAN, logging::ERR_ALWAYS_CRITICAL, + "pDictionaryScan::sendPrimitiveMessages()"); + sendError(ERR_DICTIONARY_SCAN); + } - if (dbRootConnectionMap->find(dbroot) == dbRootConnectionMap->end()) - { - log.logMessage(logging::LOG_TYPE_DEBUG, "dictionary still not in dbRootConnectionMap"); - throw IDBExcept(ERR_DATA_OFFLINE); - } - } + mutex.lock(); + finishedSending = true; - sendAPrimitiveMessage(primMsg, msgLbidStart, msgLbidCount, (*dbRootConnectionMap)[dbroot]); - primMsg.restart(); + if (recvWaiting) + { + condvar.notify_one(); + } - mutex.lock(); - msgsSent += msgLbidCount; - - if (recvWaiting) - condvar.notify_all(); - - while (( msgsSent - msgsRecvd) > fScanLbidReqThreshold) - { - sendWaiting = true; - condvarWakeupProducer.wait(mutex); - sendWaiting = false; - } - - mutex.unlock(); - - remainingLbids -= msgLbidCount; - msgLbidStart += msgLbidCount; - } - } // end of loop through LBID ranges to be requested from primproc - }//try - catch (...) - { - handleException(std::current_exception(), - logging::ERR_DICTIONARY_SCAN, - logging::ERR_ALWAYS_CRITICAL, - "pDictionaryScan::sendPrimitiveMessages()"); - sendError(ERR_DICTIONARY_SCAN); - } - - mutex.lock(); - finishedSending = true; - - if (recvWaiting) - { - condvar.notify_one(); - } - - mutex.unlock(); + mutex.unlock(); #ifdef DEBUG2 - if (fOid >= 3000) - { - time_t t = time(0); - char timeString[50]; - ctime_r(&t, timeString); - timeString[strlen(timeString) - 1 ] = '\0'; - cout << "pDictionaryScan Finished sending primitives for: " << - fOid << " at " << timeString << endl; - } + if (fOid >= 3000) + { + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + cout << "pDictionaryScan Finished sending primitives for: " << fOid << " at " << timeString << endl; + } #endif - } void pDictionaryScan::sendError(uint16_t s) { - status(s); + status(s); } //------------------------------------------------------------------------------ // Construct and send a single primitive message to primproc //------------------------------------------------------------------------------ -void pDictionaryScan::sendAPrimitiveMessage( - ByteStream& primMsg, - BRM::LBID_t msgLbidStart, - uint32_t msgLbidCount, - uint16_t pm -) +void pDictionaryScan::sendAPrimitiveMessage(ByteStream& primMsg, BRM::LBID_t msgLbidStart, + uint32_t msgLbidCount, uint16_t pm) { - DictTokenByScanRequestHeader hdr; - void *hdrp = static_cast(&hdr); - memset(hdrp, 0, sizeof(hdr)); + DictTokenByScanRequestHeader hdr; + void* hdrp = static_cast(&hdr); + memset(hdrp, 0, sizeof(hdr)); - hdr.ism.Interleave = pm; - hdr.ism.Flags = planFlagsToPrimFlags(fTraceFlags); - hdr.ism.Command = DICT_TOKEN_BY_SCAN_COMPARE; - hdr.ism.Size = sizeof(DictTokenByScanRequestHeader) + - fFilterString.length(); - hdr.ism.Type = 2; + hdr.ism.Interleave = pm; + hdr.ism.Flags = planFlagsToPrimFlags(fTraceFlags); + hdr.ism.Command = DICT_TOKEN_BY_SCAN_COMPARE; + hdr.ism.Size = sizeof(DictTokenByScanRequestHeader) + fFilterString.length(); + hdr.ism.Type = 2; - hdr.Hdr.SessionID = fSessionId; - hdr.Hdr.TransactionID = fTxnId; - hdr.Hdr.VerID = fVerId.currentScn; - hdr.Hdr.StepID = fStepId; - hdr.Hdr.UniqueID = uniqueID; - hdr.Hdr.Priority = priority(); + hdr.Hdr.SessionID = fSessionId; + hdr.Hdr.TransactionID = fTxnId; + hdr.Hdr.VerID = fVerId.currentScn; + hdr.Hdr.StepID = fStepId; + hdr.Hdr.UniqueID = uniqueID; + hdr.Hdr.Priority = priority(); - hdr.LBID = msgLbidStart; - idbassert(utils::is_nonnegative(hdr.LBID)); - hdr.OutputType = OT_TOKEN; - hdr.BOP = fBOP; - hdr.COP1 = fCOP1; - hdr.COP2 = fCOP2; - hdr.NVALS = fFilterCount; - hdr.Count = msgLbidCount; - hdr.CompType = fColType.ddn.compressionType; - hdr.charsetNumber = fColType.charsetNumber; - idbassert(hdr.Count > 0); + hdr.LBID = msgLbidStart; + idbassert(utils::is_nonnegative(hdr.LBID)); + hdr.OutputType = OT_TOKEN; + hdr.BOP = fBOP; + hdr.COP1 = fCOP1; + hdr.COP2 = fCOP2; + hdr.NVALS = fFilterCount; + hdr.Count = msgLbidCount; + hdr.CompType = fColType.ddn.compressionType; + hdr.charsetNumber = fColType.charsetNumber; + idbassert(hdr.Count > 0); - if (isEquality) - hdr.flags |= HAS_EQ_FILTER; + if (isEquality) + hdr.flags |= HAS_EQ_FILTER; - if (fSessionId & 0x80000000) - hdr.flags |= IS_SYSCAT; + if (fSessionId & 0x80000000) + hdr.flags |= IS_SYSCAT; + /* TODO: Need to figure out how to get the full fVerID into this msg. + * XXXPAT: The way I did it is IMO the least kludgy, while requiring only a couple + * changes. + * The old msg was: TokenByScanRequestHeader + fFilterString + * The new msg is: TokenByScanRequestHeader + fVerId + old message + * Prepending verid wastes a few bytes that go over the network, but that is better + * than putting it in the middle or at the end in terms of simplicity & memory usage, + * given the current code. + */ - /* TODO: Need to figure out how to get the full fVerID into this msg. - * XXXPAT: The way I did it is IMO the least kludgy, while requiring only a couple - * changes. - * The old msg was: TokenByScanRequestHeader + fFilterString - * The new msg is: TokenByScanRequestHeader + fVerId + old message - * Prepending verid wastes a few bytes that go over the network, but that is better - * than putting it in the middle or at the end in terms of simplicity & memory usage, - * given the current code. - */ + primMsg.load((const uint8_t*)&hdr, sizeof(DictTokenByScanRequestHeader)); + primMsg << fVerId; + primMsg.append((const uint8_t*)&hdr, sizeof(DictTokenByScanRequestHeader)); + primMsg += fFilterString; - primMsg.load((const uint8_t*) &hdr, sizeof(DictTokenByScanRequestHeader)); - primMsg << fVerId; - primMsg.append((const uint8_t*) &hdr, sizeof(DictTokenByScanRequestHeader)); - primMsg += fFilterString; - - //cout << "Sending rqst LBIDS " << msgLbidStart - // << " hdr.Count " << hdr.Count - // << " filterCount " << fFilterCount << endl; + // cout << "Sending rqst LBIDS " << msgLbidStart + // << " hdr.Count " << hdr.Count + // << " filterCount " << fFilterCount << endl; #ifdef DEBUG2 - if (fOid >= 3000) - cout << "pDictionaryScan producer st: " << fStepId << - ": sending req for lbid start " << msgLbidStart << - "; lbid count " << msgLbidCount << endl; + if (fOid >= 3000) + cout << "pDictionaryScan producer st: " << fStepId << ": sending req for lbid start " << msgLbidStart + << "; lbid count " << msgLbidCount << endl; #endif - try - { - fDec->write(uniqueID, primMsg); - } - catch (...) - { - abort(); - handleException(std::current_exception(), - logging::ERR_DICTIONARY_SCAN, - logging::ERR_ALWAYS_CRITICAL, - "pDictionaryScan::sendAPrimitiveMessage()"); - sendError(ERR_DICTIONARY_SCAN); - } + try + { + fDec->write(uniqueID, primMsg); + } + catch (...) + { + abort(); + handleException(std::current_exception(), logging::ERR_DICTIONARY_SCAN, logging::ERR_ALWAYS_CRITICAL, + "pDictionaryScan::sendAPrimitiveMessage()"); + sendError(ERR_DICTIONARY_SCAN); + } - fMsgsToPm++; + fMsgsToPm++; } void pDictionaryScan::receivePrimitiveMessages() { - RowGroupDL* rgFifo = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - boost::shared_ptr bs; - RGData rgData; - Row r; + RowGroupDL* rgFifo = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + boost::shared_ptr bs; + RGData rgData; + Row r; - fRidResults = 0; + fRidResults = 0; - idbassert(fOutputRowGroup.getColumnCount() > 0); - fOutputRowGroup.initRow(&r); + idbassert(fOutputRowGroup.getColumnCount() > 0); + fOutputRowGroup.initRow(&r); + rgData = RGData(fOutputRowGroup); + fOutputRowGroup.setData(&rgData); + fOutputRowGroup.resetRowGroup(0); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + + if (fOid >= 3000) + { + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = fMsgsExpect; + postStepStartTele(sts); + } + + uint16_t error = status(); + + //...Be careful here. Mutex is locked prior to entering the loop, so + //...any continue statement in the loop must be sure the mutex is locked. + // error condition will not go through loop + if (!error) + mutex.lock(); + + try + { + while (!error) + { + // sync with the send side + while (!finishedSending && msgsSent == msgsRecvd) + { + recvWaiting = true; + condvar.wait(mutex); + recvWaiting = false; + } + + if (finishedSending && (msgsSent == msgsRecvd)) + { + mutex.unlock(); + break; + } + + mutex.unlock(); + + fDec->read(uniqueID, bs); + + if (fOid >= 3000 && traceOn() && dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + if (fOid >= 3000 && traceOn()) + dlTimes.setLastReadTime(); + + if (bs->length() == 0) + { + mutex.lock(); + fStopSending = true; + condvarWakeupProducer.notify_one(); + mutex.unlock(); + break; + } + + ISMPacketHeader* hdr = (ISMPacketHeader*)(bs->buf()); + error = hdr->Status; + + if (!error) + { + const ByteStream::byte* bsp = bs->buf(); + + // get the ResultHeader out of the bytestream + const TokenByScanResultHeader* crh = reinterpret_cast(bsp); + bsp += sizeof(TokenByScanResultHeader); + + fCacheIO += crh->CacheIO; + fPhysicalIO += crh->PhysicalIO; + + // From this point on the rest of the bytestream is the data that comes back from the primitive server + // This needs to be fed to a datalist that is retrieved from the outputassociation object. + + PrimToken pt; + uint64_t token; +#ifdef DEBUG + cout << "dict step " << fStepId << " NVALS = " << crh->NVALS << endl; +#endif + + if (fOid >= 3000 && traceOn() && dlTimes.FirstInsertTime().tv_sec == 0) + dlTimes.setFirstInsertTime(); + + for (int j = 0; j < crh->NVALS && !cancelled(); j++) + { + memcpy(&pt, bsp, sizeof(pt)); + bsp += sizeof(pt); + uint64_t rid = fRidResults++; + token = (pt.LBID << 10) | pt.offset; + + fOutputRowGroup.getRow(fOutputRowGroup.getRowCount(), &r); + // load r up w/ values + r.setRid(rid); + r.setUintField<8>(token, 0); + fOutputRowGroup.incRowCount(); + + if (fOutputRowGroup.getRowCount() == 8192) + { + // INSERT_ADAPTER(rgFifo, rgData); + // fOutputRowGroup.convertToInlineDataInPlace(); + rgFifo->insert(rgData); + rgData = RGData(fOutputRowGroup); + fOutputRowGroup.setData(&rgData); + fOutputRowGroup.resetRowGroup(0); + } + } + + mutex.lock(); + msgsRecvd++; + + if (fOid >= 3000) + { + uint64_t progress = msgsRecvd * 100 / fMsgsExpect; + + if (progress > fProgress) + { + fProgress = progress; + sts.msg_type = StepTeleStats::ST_PROGRESS; + sts.total_units_of_work = fMsgsExpect; + sts.units_of_work_completed = msgsRecvd; + postStepProgressTele(sts); + } + } + + //...If producer is waiting, and we have gone below our threshold value, + //...then we signal the producer to request more data from primproc + if ((sendWaiting) && ((msgsSent - msgsRecvd) < fScanLbidReqThreshold)) + { +#ifdef DEBUG2 + + if (fOid >= 3000) + cout << "pDictionaryScan consumer signaling producer for " + "more data: " + << "st:" << fStepId << "; sentCount-" << msgsSent << "; recvCount-" << msgsRecvd + << "; threshold-" << fScanLbidReqThreshold << endl; + +#endif + condvarWakeupProducer.notify_one(); + } + } // if !error + else + { + mutex.lock(); + fStopSending = true; + condvarWakeupProducer.notify_one(); + mutex.unlock(); + string errMsg; + + // bs->advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + //*bs >> errMsg; + if (error < 1000) + { + logging::ErrorCodes errorcodes; + errMsg = errorcodes.errorString(error); + } + else + { + errMsg = IDBErrorInfo::instance()->errorMsg(error); + } + + errorMessage(errMsg); + status(error); + } + } // end of loop to read LBID responses from primproc + } + catch (const LargeDataListExcept& ex) + { + catchHandler(ex.what(), ERR_DICTIONARY_SCAN, fErrorInfo, fSessionId); + mutex.unlock(); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_DICTIONARY_SCAN, logging::ERR_ALWAYS_CRITICAL, + "pDictionaryScan::receivePrimitiveMessages()"); + mutex.unlock(); + } + + if (fOutputRowGroup.getRowCount() > 0) + { + // fOutputRowGroup.convertToInlineDataInPlace(); + // INSERT_ADAPTER(rgFifo, rgData); + rgFifo->insert(rgData); rgData = RGData(fOutputRowGroup); fOutputRowGroup.setData(&rgData); fOutputRowGroup.resetRowGroup(0); + } - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; + Stats stats = fDec->getNetworkStats(uniqueID); + fMsgBytesIn = stats.dataRecvd(); + fMsgBytesOut = stats.dataSent(); - if (fOid >= 3000) + //@bug 699: Reset StepMsgQueue + fDec->removeQueue(uniqueID); + + if (fTableOid >= 3000) + { + //...Construct timestamp using ctime_r() instead of ctime() not + //...necessarily due to re-entrancy, but because we want to strip + //...the newline ('\n') off the end of the formatted string. + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + + //...Roundoff inbound msg byte count to nearest KB for display; + //...no need to do so for outbound, because it should be small. + uint64_t msgBytesInKB = fMsgBytesIn >> 10; + + if (fMsgBytesIn & 512) + msgBytesInKB++; + + if (traceOn()) { - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = fMsgsExpect; - postStepStartTele(sts); + dlTimes.setEndOfInputTime(); + + //...Print job step completion information + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString << "; PhyI/O-" + << fPhysicalIO << "; CacheI/O-" << fCacheIO << "; MsgsSent-" << fMsgsToPm << "; MsgsRcvd-" + << msgsRecvd << "; output size-" << fRidResults << endl + << "\tMsgBytesIn-" << msgBytesInKB + << "KB" + "; MsgBytesOut-" + << fMsgBytesOut << "B" << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s" << endl + << "\tJob completion status " << status() << endl; + + logEnd(logStr.str().c_str()); + + syslogReadBlockCounts(16, // exemgr subsystem + fPhysicalIO, // # blocks read from disk + fCacheIO, // # blocks read from cache + 0); // # casual partition block hits + syslogProcessingTimes(16, // exemgr subsystem + dlTimes.FirstReadTime(), // first datalist read + dlTimes.LastReadTime(), // last datalist read + dlTimes.FirstInsertTime(), // first datlist write + dlTimes.EndOfInputTime()); // last (endOfInput) datalist write + syslogEndStep(16, // exemgr subsystem + fMsgBytesIn, // incoming msg byte count + fMsgBytesOut); // outgoing msg byte count + fExtendedInfo += toString() + logStr.str(); + formatMiniStats(); } - uint16_t error = status(); + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.phy_io = fPhysicalIO; + sts.cache_io = fCacheIO; + sts.msg_rcv_cnt = sts.total_units_of_work = sts.units_of_work_completed = msgsRecvd; + sts.msg_bytes_in = fMsgBytesIn; + sts.msg_bytes_out = fMsgBytesOut; + sts.rows = fRidResults; + postStepSummaryTele(sts); + } - //...Be careful here. Mutex is locked prior to entering the loop, so - //...any continue statement in the loop must be sure the mutex is locked. - //error condition will not go through loop - if (!error) mutex.lock(); - - try - { - while (!error) - { - - // sync with the send side - while (!finishedSending && msgsSent == msgsRecvd) - { - recvWaiting = true; - condvar.wait(mutex); - recvWaiting = false; - } - - if (finishedSending && (msgsSent == msgsRecvd)) - { - mutex.unlock(); - break; - } - - mutex.unlock(); - - fDec->read(uniqueID, bs); - - if (fOid >= 3000 && traceOn() && dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - if (fOid >= 3000 && traceOn()) - dlTimes.setLastReadTime(); - - if (bs->length() == 0) - { - mutex.lock(); - fStopSending = true; - condvarWakeupProducer.notify_one(); - mutex.unlock(); - break; - } - - ISMPacketHeader* hdr = (ISMPacketHeader*)(bs->buf()); - error = hdr->Status; - - if (! error) - { - const ByteStream::byte* bsp = bs->buf(); - - // get the ResultHeader out of the bytestream - const TokenByScanResultHeader* crh = reinterpret_cast(bsp); - bsp += sizeof(TokenByScanResultHeader); - - fCacheIO += crh->CacheIO; - fPhysicalIO += crh->PhysicalIO; - - // From this point on the rest of the bytestream is the data that comes back from the primitive server - // This needs to be fed to a datalist that is retrieved from the outputassociation object. - - PrimToken pt; - uint64_t token; -#ifdef DEBUG - cout << "dict step " << fStepId << " NVALS = " << crh->NVALS << endl; -#endif - - if (fOid >= 3000 && traceOn() && dlTimes.FirstInsertTime().tv_sec == 0) - dlTimes.setFirstInsertTime(); - - for (int j = 0; j < crh->NVALS && !cancelled(); j++) - { - memcpy(&pt, bsp, sizeof(pt)); - bsp += sizeof(pt); - uint64_t rid = fRidResults++; - token = (pt.LBID << 10) | pt.offset; - - fOutputRowGroup.getRow(fOutputRowGroup.getRowCount(), &r); - // load r up w/ values - r.setRid(rid); - r.setUintField<8>(token, 0); - fOutputRowGroup.incRowCount(); - - if (fOutputRowGroup.getRowCount() == 8192) - { - //INSERT_ADAPTER(rgFifo, rgData); - //fOutputRowGroup.convertToInlineDataInPlace(); - rgFifo->insert(rgData); - rgData = RGData(fOutputRowGroup); - fOutputRowGroup.setData(&rgData); - fOutputRowGroup.resetRowGroup(0); - } - } - - mutex.lock(); - msgsRecvd++; - - if (fOid >= 3000) - { - uint64_t progress = msgsRecvd * 100 / fMsgsExpect; - - if (progress > fProgress) - { - fProgress = progress; - sts.msg_type = StepTeleStats::ST_PROGRESS; - sts.total_units_of_work = fMsgsExpect; - sts.units_of_work_completed = msgsRecvd; - postStepProgressTele(sts); - } - } - - //...If producer is waiting, and we have gone below our threshold value, - //...then we signal the producer to request more data from primproc - if ( (sendWaiting) && - ( (msgsSent - msgsRecvd) < fScanLbidReqThreshold ) ) - { -#ifdef DEBUG2 - - if (fOid >= 3000) - cout << "pDictionaryScan consumer signaling producer for " - "more data: " << - "st:" << fStepId << - "; sentCount-" << msgsSent << - "; recvCount-" << msgsRecvd << - "; threshold-" << fScanLbidReqThreshold << endl; - -#endif - condvarWakeupProducer.notify_one(); - } - } //if !error - else - { - mutex.lock(); - fStopSending = true; - condvarWakeupProducer.notify_one(); - mutex.unlock(); - string errMsg; - - //bs->advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - //*bs >> errMsg; - if (error < 1000) - { - logging::ErrorCodes errorcodes; - errMsg = errorcodes.errorString(error); - } - else - { - errMsg = IDBErrorInfo::instance()->errorMsg(error); - } - - errorMessage(errMsg); - status(error); - } - } // end of loop to read LBID responses from primproc - } - catch (const LargeDataListExcept& ex) - { - catchHandler(ex.what(), ERR_DICTIONARY_SCAN, fErrorInfo, fSessionId); - mutex.unlock(); - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_DICTIONARY_SCAN, - logging::ERR_ALWAYS_CRITICAL, - "pDictionaryScan::receivePrimitiveMessages()"); - mutex.unlock(); - } - - if (fOutputRowGroup.getRowCount() > 0) - { - //fOutputRowGroup.convertToInlineDataInPlace(); - //INSERT_ADAPTER(rgFifo, rgData); - rgFifo->insert(rgData); - rgData = RGData(fOutputRowGroup); - fOutputRowGroup.setData(&rgData); - fOutputRowGroup.resetRowGroup(0); - } - - Stats stats = fDec->getNetworkStats(uniqueID); - fMsgBytesIn = stats.dataRecvd(); - fMsgBytesOut = stats.dataSent(); - - //@bug 699: Reset StepMsgQueue - fDec->removeQueue(uniqueID); - - if (fTableOid >= 3000) - { - //...Construct timestamp using ctime_r() instead of ctime() not - //...necessarily due to re-entrancy, but because we want to strip - //...the newline ('\n') off the end of the formatted string. - time_t t = time(0); - char timeString[50]; - ctime_r(&t, timeString); - timeString[strlen(timeString) - 1 ] = '\0'; - - //...Roundoff inbound msg byte count to nearest KB for display; - //...no need to do so for outbound, because it should be small. - uint64_t msgBytesInKB = fMsgBytesIn >> 10; - - if (fMsgBytesIn & 512) - msgBytesInKB++; - - if (traceOn()) - { - dlTimes.setEndOfInputTime(); - - //...Print job step completion information - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << - timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - fCacheIO << "; MsgsSent-" << fMsgsToPm << "; MsgsRcvd-" << msgsRecvd << - "; output size-" << fRidResults << endl << - "\tMsgBytesIn-" << msgBytesInKB << "KB" - "; MsgBytesOut-" << fMsgBytesOut << "B" << endl << - "\t1st read " << dlTimes.FirstReadTimeString() << - "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << "s" << endl - << "\tJob completion status " << status() << endl; - - logEnd(logStr.str().c_str()); - - syslogReadBlockCounts(16, // exemgr subsystem - fPhysicalIO, // # blocks read from disk - fCacheIO, // # blocks read from cache - 0); // # casual partition block hits - syslogProcessingTimes(16, // exemgr subsystem - dlTimes.FirstReadTime(), // first datalist read - dlTimes.LastReadTime(), // last datalist read - dlTimes.FirstInsertTime(), // first datlist write - dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - syslogEndStep(16, // exemgr subsystem - fMsgBytesIn, // incoming msg byte count - fMsgBytesOut); // outgoing msg byte count - fExtendedInfo += toString() + logStr.str(); - formatMiniStats(); - } - - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.phy_io = fPhysicalIO; - sts.cache_io = fCacheIO; - sts.msg_rcv_cnt = sts.total_units_of_work = sts.units_of_work_completed = msgsRecvd; - sts.msg_bytes_in = fMsgBytesIn; - sts.msg_bytes_out = fMsgBytesOut; - sts.rows = fRidResults; - postStepSummaryTele(sts); - } - - rgFifo->endOfInput(); + rgFifo->endOfInput(); } const string pDictionaryScan::toString() const { - ostringstream oss; - oss << "pDictionaryScan ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId - << " st:" << fStepId << " alias: " << (fAlias.length() ? fAlias : "none") - << " tb/col:" << fTableOid << "/" << fOid; - oss << " " << omitOidInDL - << fOutputJobStepAssociation.outAt(0) << showOidInDL; - oss << " nf:" << fFilterCount; - oss << " in:"; + ostringstream oss; + oss << "pDictionaryScan ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId + << " alias: " << (fAlias.length() ? fAlias : "none") << " tb/col:" << fTableOid << "/" << fOid; + oss << " " << omitOidInDL << fOutputJobStepAssociation.outAt(0) << showOidInDL; + oss << " nf:" << fFilterCount; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - { - oss << fInputJobStepAssociation.outAt(i) << ", "; - } + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + oss << fInputJobStepAssociation.outAt(i) << ", "; + } - return oss.str(); + return oss.str(); } void pDictionaryScan::formatMiniStats() { - ostringstream oss; - oss << "DSS " - << "PM " - << fAlias << " " - << fTableOid << " (" << fName << ") " - << fPhysicalIO << " " - << fCacheIO << " " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRidResults << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "DSS " + << "PM " << fAlias << " " << fTableOid << " (" << fName << ") " << fPhysicalIO << " " << fCacheIO << " " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRidResults << " "; + fMiniInfo += oss.str(); } - void pDictionaryScan::addFilter(const Filter* f) { - if (NULL != f) - fFilters.push_back(f); + if (NULL != f) + fFilters.push_back(f); } - void pDictionaryScan::appendFilter(const std::vector& fs) { - fFilters.insert(fFilters.end(), fs.begin(), fs.end()); + fFilters.insert(fFilters.end(), fs.begin(), fs.end()); } - void pDictionaryScan::appendFilter(const messageqcpp::ByteStream& filter, unsigned count) { - fFilterString += filter; - fFilterCount += count; + fFilterString += filter; + fFilterCount += count; } - void pDictionaryScan::serializeEqualityFilter() { - ByteStream msg; - ISMPacketHeader ism; - uint32_t i; - vector empty; + ByteStream msg; + ISMPacketHeader ism; + uint32_t i; + vector empty; - void *ismp = static_cast(&ism); - memset(ismp, 0, sizeof(ISMPacketHeader)); - ism.Command = DICT_CREATE_EQUALITY_FILTER; - msg.load((uint8_t*) &ism, sizeof(ISMPacketHeader)); - msg << uniqueID; - msg << (uint32_t) colType().charsetNumber; - msg << (uint32_t) equalityFilter.size(); + void* ismp = static_cast(&ism); + memset(ismp, 0, sizeof(ISMPacketHeader)); + ism.Command = DICT_CREATE_EQUALITY_FILTER; + msg.load((uint8_t*)&ism, sizeof(ISMPacketHeader)); + msg << uniqueID; + msg << (uint32_t)colType().charsetNumber; + msg << (uint32_t)equalityFilter.size(); - for (i = 0; i < equalityFilter.size(); i++) - msg << equalityFilter[i]; + for (i = 0; i < equalityFilter.size(); i++) + msg << equalityFilter[i]; - try - { - fDec->write(uniqueID, msg); - } - catch (...) - { - abort(); - handleException(std::current_exception(), - logging::ERR_DICTIONARY_SCAN, - logging::ERR_ALWAYS_CRITICAL, - "pDictionaryScan::serializeEqualityFilter()"); - } + try + { + fDec->write(uniqueID, msg); + } + catch (...) + { + abort(); + handleException(std::current_exception(), logging::ERR_DICTIONARY_SCAN, logging::ERR_ALWAYS_CRITICAL, + "pDictionaryScan::serializeEqualityFilter()"); + } - empty.swap(equalityFilter); + empty.swap(equalityFilter); } void pDictionaryScan::destroyEqualityFilter() { - ByteStream msg; - ISMPacketHeader ism; + ByteStream msg; + ISMPacketHeader ism; - void *ismp = static_cast(&ism); - memset(ismp, 0, sizeof(ISMPacketHeader)); - ism.Command = DICT_DESTROY_EQUALITY_FILTER; - msg.load((uint8_t*) &ism, sizeof(ISMPacketHeader)); - msg << uniqueID; + void* ismp = static_cast(&ism); + memset(ismp, 0, sizeof(ISMPacketHeader)); + ism.Command = DICT_DESTROY_EQUALITY_FILTER; + msg.load((uint8_t*)&ism, sizeof(ISMPacketHeader)); + msg << uniqueID; - try - { - fDec->write(uniqueID, msg); - } - catch (...) - { - abort(); - handleException(std::current_exception(), - logging::ERR_DICTIONARY_SCAN, - logging::ERR_ALWAYS_CRITICAL, - "pDictionaryScan::destroyEqualityFilter()"); - } + try + { + fDec->write(uniqueID, msg); + } + catch (...) + { + abort(); + handleException(std::current_exception(), logging::ERR_DICTIONARY_SCAN, logging::ERR_ALWAYS_CRITICAL, + "pDictionaryScan::destroyEqualityFilter()"); + } } void pDictionaryScan::abort() { - JobStep::abort(); + JobStep::abort(); - if (fDec) - fDec->shutdownQueue(uniqueID); + if (fDec) + fDec->shutdownQueue(uniqueID); } - - - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/primitivemsg.cpp b/dbcon/joblist/primitivemsg.cpp index c07c5d7ba..47e7b8e2f 100644 --- a/dbcon/joblist/primitivemsg.cpp +++ b/dbcon/joblist/primitivemsg.cpp @@ -33,43 +33,42 @@ namespace joblist { void PrimitiveMsg::send() { - throw logic_error("somehow ended up in PrimitiveMsg::send()!"); + throw logic_error("somehow ended up in PrimitiveMsg::send()!"); } void PrimitiveMsg::buildPrimitiveMessage(ISMPACKETCOMMAND, void*, void*) { - throw logic_error("somehow ended up in PrimitiveMsg::buildPrimitiveMessage()!"); + throw logic_error("somehow ended up in PrimitiveMsg::buildPrimitiveMessage()!"); } void PrimitiveMsg::receive() { - throw logic_error("somehow ended up in PrimitiveMsg::receive()!"); + throw logic_error("somehow ended up in PrimitiveMsg::receive()!"); } void PrimitiveMsg::sendPrimitiveMessages() { - throw logic_error("somehow ended up in PrimitiveMsg::sendPrimitiveMessages()!"); + throw logic_error("somehow ended up in PrimitiveMsg::sendPrimitiveMessages()!"); } void PrimitiveMsg::receivePrimitiveMessages() { - throw logic_error("somehow ended up in PrimitiveMsg::receivePrimitiveMessages()!"); + throw logic_error("somehow ended up in PrimitiveMsg::receivePrimitiveMessages()!"); } // Unfortuneately we have 32 bits in the execplan flags, but only 16 that can be sent to // PrimProc, so we have to convert them (throwing some away). uint16_t PrimitiveMsg::planFlagsToPrimFlags(uint32_t planFlags) { - uint16_t flags = 0; + uint16_t flags = 0; - if (planFlags & CalpontSelectExecutionPlan::TRACE_LBIDS) - flags |= PF_LBID_TRACE; + if (planFlags & CalpontSelectExecutionPlan::TRACE_LBIDS) + flags |= PF_LBID_TRACE; - if (planFlags & CalpontSelectExecutionPlan::PM_PROFILE) - flags |= PF_PM_PROF; - - return flags; -} + if (planFlags & CalpontSelectExecutionPlan::PM_PROFILE) + flags |= PF_PM_PROF; + return flags; } +} // namespace joblist diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index 5b4bbfe05..3d467a8ec 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /* -* $Id: primitivemsg.h 9655 2013-06-25 23:08:13Z xlou $ -*/ + * $Id: primitivemsg.h 9655 2013-06-25 23:08:13Z xlou $ + */ /** @file */ @@ -33,11 +33,11 @@ #include -#pragma pack(push,1) +#pragma pack(push, 1) #ifdef _MSC_VER -#pragma warning (push) -#pragma warning (disable : 4200) +#pragma warning(push) +#pragma warning(disable : 4200) #endif // from blocksize.h @@ -46,74 +46,60 @@ const int32_t DATA_BLOCK_SIZE = BLOCK_SIZE; const int8_t COMPARE_NIL = 0x00; const int8_t COMPARE_LT = 0x01; const int8_t COMPARE_EQ = 0x02; -const int8_t COMPARE_LE = (COMPARE_LT | COMPARE_EQ); //0x03 +const int8_t COMPARE_LE = (COMPARE_LT | COMPARE_EQ); // 0x03 const int8_t COMPARE_GT = 0x04; -const int8_t COMPARE_NE = (COMPARE_LT | COMPARE_GT); //0x05 -const int8_t COMPARE_GE = (COMPARE_GT | COMPARE_EQ); //0x06 +const int8_t COMPARE_NE = (COMPARE_LT | COMPARE_GT); // 0x05 +const int8_t COMPARE_GE = (COMPARE_GT | COMPARE_EQ); // 0x06 const int8_t COMPARE_NOT = 0x08; -const int8_t COMPARE_NLT = (COMPARE_LT | COMPARE_NOT); //0x09 -const int8_t COMPARE_NLE = (COMPARE_LE | COMPARE_NOT); //0x0b -const int8_t COMPARE_NGT = (COMPARE_GT | COMPARE_NOT); //0x0c -const int8_t COMPARE_NGE = (COMPARE_GE | COMPARE_NOT); //0x0e +const int8_t COMPARE_NLT = (COMPARE_LT | COMPARE_NOT); // 0x09 +const int8_t COMPARE_NLE = (COMPARE_LE | COMPARE_NOT); // 0x0b +const int8_t COMPARE_NGT = (COMPARE_GT | COMPARE_NOT); // 0x0c +const int8_t COMPARE_NGE = (COMPARE_GE | COMPARE_NOT); // 0x0e const int8_t COMPARE_LIKE = 0x10; -const int8_t COMPARE_NLIKE = (COMPARE_LIKE | COMPARE_NOT); //0x18 - +const int8_t COMPARE_NLIKE = (COMPARE_LIKE | COMPARE_NOT); // 0x18 namespace primitives { - using RIDType = uint16_t; using NVALSType = uint16_t; using utils::ConstString; -class StringComparator: public datatypes::Charset +class StringComparator : public datatypes::Charset { -public: - StringComparator(const Charset &cs) - :Charset(cs) - { } - bool op(int * error, uint8_t COP, - const ConstString &str1, - const ConstString &str2) const + public: + StringComparator(const Charset& cs) : Charset(cs) + { + } + bool op(int* error, uint8_t COP, const ConstString& str1, const ConstString& str2) const + { + if (COP & COMPARE_LIKE) + return like(COP & COMPARE_NOT, str1, str2); + + int cmp = strnncollsp(str1, str2); + + switch (COP) { - if (COP & COMPARE_LIKE) - return like(COP & COMPARE_NOT, str1, str2); + case COMPARE_NIL: return false; - int cmp = strnncollsp(str1, str2); + case COMPARE_LT: return cmp < 0; - switch (COP) - { - case COMPARE_NIL: - return false; + case COMPARE_EQ: return cmp == 0; - case COMPARE_LT: - return cmp < 0; + case COMPARE_LE: return cmp <= 0; - case COMPARE_EQ: - return cmp == 0; + case COMPARE_GT: return cmp > 0; - case COMPARE_LE: - return cmp <= 0; + case COMPARE_NE: return cmp != 0; - case COMPARE_GT: - return cmp > 0; - - case COMPARE_NE: - return cmp != 0; - - case COMPARE_GE: - return cmp >= 0; - - default: - *error |= 1; - return false; - } + case COMPARE_GE: return cmp >= 0; + default: *error |= 1; return false; } + } }; -} // namespace primities +} // namespace primitives // BOP (Binary Operation) values // used to tell if the operations are all be true or @@ -129,29 +115,29 @@ public: #define OT_RID 1 #define OT_TOKEN 2 -#define OT_BOTH 3 // both = RID & TOKEN +#define OT_BOTH 3 // both = RID & TOKEN #define OT_DATAVALUE 4 -#define OT_INPUTARG OT_RID // reuse OT_RID's flag in dictionary primitives. Specifies that +#define OT_INPUTARG OT_RID // reuse OT_RID's flag in dictionary primitives. Specifies that // the filter argument that matched should be part of the result set. // (only makes sense when BOP = OR). -#define OT_AGGREGATE 8 // specifies that aggregate data should be generated +#define OT_AGGREGATE 8 // specifies that aggregate data should be generated // Packet Header Types enum PACKETTYPE { - NULL1 = 0, - NULL2 = 0X0F, - DATA = 1, - CMD = 2, - FLOW = 3, - CONFIG = 9 + NULL1 = 0, + NULL2 = 0X0F, + DATA = 1, + CMD = 2, + FLOW = 3, + CONFIG = 9 }; enum TYPEFLOWCOMMAND { - ACK = 1, - NACK = 2 + ACK = 1, + NACK = 2 }; // Define the ISM Commands @@ -163,77 +149,77 @@ enum TYPEFLOWCOMMAND #define PRIM_DICTBASE 200 #define PRIM_DELIVERBASE 250 -//8 bits only! +// 8 bits only! enum ISMPACKETCOMMAND { - //max of 50-10=40 commands - LOCAL_JOIN_BY_RID = PRIM_LOCALBASE + 0, - LOCAL_OR_BY_RID = PRIM_LOCALBASE + 1, - ARITHMETIC_FUNCTION = PRIM_LOCALBASE + 2, - FUNCTION_CALL = PRIM_LOCALBASE + 3, - FUNCTION_CALL_VECTOR = PRIM_LOCALBASE + 4, - LOCAL_COMPARE_BY_VALUE = PRIM_LOCALBASE + 5, - LOCAL_JOIN_BY_VALUE = PRIM_LOCALBASE + 6, - BATCH_PRIMITIVE_CREATE = PRIM_LOCALBASE + 7, - BATCH_PRIMITIVE_RUN = PRIM_LOCALBASE + 8, - BATCH_PRIMITIVE_DESTROY = PRIM_LOCALBASE + 9, - BATCH_PRIMITIVE_ADD_JOINER = PRIM_LOCALBASE + 10, - BATCH_PRIMITIVE_END_JOINER = PRIM_LOCALBASE + 11, - BATCH_PRIMITIVE_ACK = PRIM_LOCALBASE + 12, - BATCH_PRIMITIVE_ABORT = PRIM_LOCALBASE + 13, + // max of 50-10=40 commands + LOCAL_JOIN_BY_RID = PRIM_LOCALBASE + 0, + LOCAL_OR_BY_RID = PRIM_LOCALBASE + 1, + ARITHMETIC_FUNCTION = PRIM_LOCALBASE + 2, + FUNCTION_CALL = PRIM_LOCALBASE + 3, + FUNCTION_CALL_VECTOR = PRIM_LOCALBASE + 4, + LOCAL_COMPARE_BY_VALUE = PRIM_LOCALBASE + 5, + LOCAL_JOIN_BY_VALUE = PRIM_LOCALBASE + 6, + BATCH_PRIMITIVE_CREATE = PRIM_LOCALBASE + 7, + BATCH_PRIMITIVE_RUN = PRIM_LOCALBASE + 8, + BATCH_PRIMITIVE_DESTROY = PRIM_LOCALBASE + 9, + BATCH_PRIMITIVE_ADD_JOINER = PRIM_LOCALBASE + 10, + BATCH_PRIMITIVE_END_JOINER = PRIM_LOCALBASE + 11, + BATCH_PRIMITIVE_ACK = PRIM_LOCALBASE + 12, + BATCH_PRIMITIVE_ABORT = PRIM_LOCALBASE + 13, - //max of 100-50=50 commands - COL_RESULTS = PRIM_COLBASE + 0, - COL_AGG_RESULTS = PRIM_COLBASE + 1, - COL_BY_SCAN = PRIM_COLBASE + 2, - COL_BY_RID = PRIM_COLBASE + 3, - COL_AGG_BY_SCAN = PRIM_COLBASE + 4, - COL_AGG_BY_RID = PRIM_COLBASE + 5, - COL_JOIN_BY_SCAN = PRIM_COLBASE + 6, - COL_FILTER_BY_TOKEN = PRIM_COLBASE + 7, - COL_FILTER_BY_RID_VAL = PRIM_COLBASE + 8, - COL_BY_SCAN_RANGE = PRIM_COLBASE + 9, - COL_LOOPBACK = PRIM_COLBASE + 10, + // max of 100-50=50 commands + COL_RESULTS = PRIM_COLBASE + 0, + COL_AGG_RESULTS = PRIM_COLBASE + 1, + COL_BY_SCAN = PRIM_COLBASE + 2, + COL_BY_RID = PRIM_COLBASE + 3, + COL_AGG_BY_SCAN = PRIM_COLBASE + 4, + COL_AGG_BY_RID = PRIM_COLBASE + 5, + COL_JOIN_BY_SCAN = PRIM_COLBASE + 6, + COL_FILTER_BY_TOKEN = PRIM_COLBASE + 7, + COL_FILTER_BY_RID_VAL = PRIM_COLBASE + 8, + COL_BY_SCAN_RANGE = PRIM_COLBASE + 9, + COL_LOOPBACK = PRIM_COLBASE + 10, - //max of 190-100=90 commands - INDEX_RESULTS = PRIM_INDEXBASE + 0, // Obsolete ? - INDEX_SCAN_RESULTS = PRIM_INDEXBASE + 1, // p_IdxScan results - INDEX_WALK_RESULTS = PRIM_INDEXBASE + 2, // p_IdxWalk results - INDEX_LIST_RESULTS = PRIM_INDEXBASE + 3, // p_IdxList results - INDEX_LIST_AGGREGATE_RESULTS = PRIM_INDEXBASE + 4, // p_IdxListAggregate results - INDEX_SCAN_AGGREGATE_RESULTS = PRIM_INDEXBASE + 5, // p_IdxScanAggregate results - INDEX_BY_SCAN = PRIM_INDEXBASE + 6, // p_IdxScan - INDEX_BY_COMPARE = PRIM_INDEXBASE + 7, // Obsolete ? - INDEX_WALK = PRIM_INDEXBASE + 8, // p_IdxWalk - INDEX_LIST = PRIM_INDEXBASE + 9, // p_IdxList - INDEX_LIST_AGGREGATE = PRIM_INDEXBASE + 10, // p_IdxListAggregate - INDEX_SCAN_AGGREGATE = PRIM_INDEXBASE + 11, // p_IdxScanAggregate + // max of 190-100=90 commands + INDEX_RESULTS = PRIM_INDEXBASE + 0, // Obsolete ? + INDEX_SCAN_RESULTS = PRIM_INDEXBASE + 1, // p_IdxScan results + INDEX_WALK_RESULTS = PRIM_INDEXBASE + 2, // p_IdxWalk results + INDEX_LIST_RESULTS = PRIM_INDEXBASE + 3, // p_IdxList results + INDEX_LIST_AGGREGATE_RESULTS = PRIM_INDEXBASE + 4, // p_IdxListAggregate results + INDEX_SCAN_AGGREGATE_RESULTS = PRIM_INDEXBASE + 5, // p_IdxScanAggregate results + INDEX_BY_SCAN = PRIM_INDEXBASE + 6, // p_IdxScan + INDEX_BY_COMPARE = PRIM_INDEXBASE + 7, // Obsolete ? + INDEX_WALK = PRIM_INDEXBASE + 8, // p_IdxWalk + INDEX_LIST = PRIM_INDEXBASE + 9, // p_IdxList + INDEX_LIST_AGGREGATE = PRIM_INDEXBASE + 10, // p_IdxListAggregate + INDEX_SCAN_AGGREGATE = PRIM_INDEXBASE + 11, // p_IdxScanAggregate - //max of 200-190=10 commands - CACHE_OP_RESULTS = PRIM_CACHEBASE + 0, // Response msg - CACHE_FLUSH = PRIM_CACHEBASE + 1, // Flush the entire block cache - CACHE_CLEAN_VSS = PRIM_CACHEBASE + 2, // Clean out indicated VSS entries - CACHE_DROP_FDS = PRIM_CACHEBASE + 3, // Drop the whole file descriptor cache - FLUSH_ALL_VERSION = PRIM_CACHEBASE + 4, // Drop all versions of specified LBIDs - CACHE_FLUSH_BY_OID = PRIM_CACHEBASE + 5, // Drop all versions of all LBIDs for the given OIDs - CACHE_FLUSH_PARTITION = PRIM_CACHEBASE + 6, // Drop a partition - CACHE_PURGE_FDS = PRIM_CACHEBASE + 7, // Purge the file descriptor cache for the modified files + // max of 200-190=10 commands + CACHE_OP_RESULTS = PRIM_CACHEBASE + 0, // Response msg + CACHE_FLUSH = PRIM_CACHEBASE + 1, // Flush the entire block cache + CACHE_CLEAN_VSS = PRIM_CACHEBASE + 2, // Clean out indicated VSS entries + CACHE_DROP_FDS = PRIM_CACHEBASE + 3, // Drop the whole file descriptor cache + FLUSH_ALL_VERSION = PRIM_CACHEBASE + 4, // Drop all versions of specified LBIDs + CACHE_FLUSH_BY_OID = PRIM_CACHEBASE + 5, // Drop all versions of all LBIDs for the given OIDs + CACHE_FLUSH_PARTITION = PRIM_CACHEBASE + 6, // Drop a partition + CACHE_PURGE_FDS = PRIM_CACHEBASE + 7, // Purge the file descriptor cache for the modified files - //max of 250-200=50 commands - DICT_RESULTS = PRIM_DICTBASE + 0, - DICT_TOKEN_BY_INDEX_COMPARE = PRIM_DICTBASE + 1, - DICT_TOKEN_BY_SCAN_COMPARE = PRIM_DICTBASE + 2, - DICT_SIGNATURE = PRIM_DICTBASE + 3, - DICT_AGGREGATE = PRIM_DICTBASE + 4, - DICT_AGGREGATE_RESULTS = PRIM_DICTBASE + 5, - DICT_SCAN_COMPARE_RESULTS = PRIM_DICTBASE + 6, - DICT_SIGNATURE_RANGE = PRIM_DICTBASE + 7, - DICT_CREATE_EQUALITY_FILTER = PRIM_DICTBASE + 8, - DICT_DESTROY_EQUALITY_FILTER = PRIM_DICTBASE + 9, + // max of 250-200=50 commands + DICT_RESULTS = PRIM_DICTBASE + 0, + DICT_TOKEN_BY_INDEX_COMPARE = PRIM_DICTBASE + 1, + DICT_TOKEN_BY_SCAN_COMPARE = PRIM_DICTBASE + 2, + DICT_SIGNATURE = PRIM_DICTBASE + 3, + DICT_AGGREGATE = PRIM_DICTBASE + 4, + DICT_AGGREGATE_RESULTS = PRIM_DICTBASE + 5, + DICT_SCAN_COMPARE_RESULTS = PRIM_DICTBASE + 6, + DICT_SIGNATURE_RANGE = PRIM_DICTBASE + 7, + DICT_CREATE_EQUALITY_FILTER = PRIM_DICTBASE + 8, + DICT_DESTROY_EQUALITY_FILTER = PRIM_DICTBASE + 9, - //max of 256-250=6 commands - DELIVER_TOKEN_RESULTS = PRIM_DELIVERBASE + 0, - DELIVER_RID_RESULTS = PRIM_DELIVERBASE + 1 + // max of 256-250=6 commands + DELIVER_TOKEN_RESULTS = PRIM_DELIVERBASE + 0, + DELIVER_RID_RESULTS = PRIM_DELIVERBASE + 1 }; #undef PRIM_LOCALBASE @@ -243,30 +229,30 @@ enum ISMPACKETCOMMAND #undef PRIM_DELIVERBASE /* Flags for BPP messages */ -const uint16_t NEED_STR_VALUES = 0x01; //1; -const uint16_t GOT_ABS_RIDS = 0x02; //2; -const uint16_t GOT_VALUES = 0x04; //4; -const uint16_t LBID_TRACE = 0x08; //8; -const uint16_t HAS_JOINER = 0x10; //16; -const uint16_t SEND_RIDS_AT_DELIVERY = 0x20; //32; -const uint16_t HAS_ROWGROUP = 0x40; //64; -const uint16_t JOIN_ROWGROUP_DATA = 0x80; //128 -const uint16_t HAS_WIDE_COLUMNS = 0x100; //256; +const uint16_t NEED_STR_VALUES = 0x01; // 1; +const uint16_t GOT_ABS_RIDS = 0x02; // 2; +const uint16_t GOT_VALUES = 0x04; // 4; +const uint16_t LBID_TRACE = 0x08; // 8; +const uint16_t HAS_JOINER = 0x10; // 16; +const uint16_t SEND_RIDS_AT_DELIVERY = 0x20; // 32; +const uint16_t HAS_ROWGROUP = 0x40; // 64; +const uint16_t JOIN_ROWGROUP_DATA = 0x80; // 128 +const uint16_t HAS_WIDE_COLUMNS = 0x100; // 256; -//TODO: put this in a namespace to stop global ns pollution +// TODO: put this in a namespace to stop global ns pollution enum PrimFlags { - PF_LBID_TRACE = 0x01, /*!< Enable LBID tracing in PrimProc */ - PF_PM_PROF = 0x02, /*!< Enable LBID tracing in PrimProc */ + PF_LBID_TRACE = 0x01, /*!< Enable LBID tracing in PrimProc */ + PF_PM_PROF = 0x02, /*!< Enable LBID tracing in PrimProc */ }; enum BPSOutputType { - BPS_ELEMENT_TYPE, - STRING_ELEMENT_TYPE, - TABLE_BAND, - TUPLE, - ROW_GROUP + BPS_ELEMENT_TYPE, + STRING_ELEMENT_TYPE, + TABLE_BAND, + TUPLE, + ROW_GROUP }; // Constant Message Header structures @@ -290,15 +276,16 @@ struct VBCPacketHeader // Changing this structure one !MUST! align ColResultHeader struct ISMPacketHeader { - ISMPacketHeader(): Interleave(0), Flags(0), Command(0), Size(0), Type(0), MsgCount(0), Status(0) {} - uint32_t Interleave; - uint16_t Flags; - uint8_t Command; - uint16_t Size; - unsigned Type: 4; - unsigned MsgCount: 4; - uint16_t Status; - + ISMPacketHeader() : Interleave(0), Flags(0), Command(0), Size(0), Type(0), MsgCount(0), Status(0) + { + } + uint32_t Interleave; + uint16_t Flags; + uint8_t Command; + uint16_t Size; + unsigned Type : 4; + unsigned MsgCount : 4; + uint16_t Status; }; // Primitive request/response structure Header @@ -306,12 +293,12 @@ struct ISMPacketHeader // Changing this structure one !MUST! align ColResultHeader struct PrimitiveHeader { - uint32_t SessionID; // Front end Session Identifier - uint32_t TransactionID; // Front end Transaction Identifier - uint32_t VerID; // DB Version ID used for this Session/Statement - uint32_t StepID; // Internal Primitive Sequence number - uint32_t UniqueID; // Unique ID for DEC and BPP - uint32_t Priority; // Priority level of the user + uint32_t SessionID; // Front end Session Identifier + uint32_t TransactionID; // Front end Transaction Identifier + uint32_t VerID; // DB Version ID used for this Session/Statement + uint32_t StepID; // Internal Primitive Sequence number + uint32_t UniqueID; // Unique ID for DEC and BPP + uint32_t Priority; // Priority level of the user }; #if 0 @@ -335,181 +322,172 @@ struct DiskResultsHeader struct ColLoopback { - PrimitiveHeader Hdr; // 64 bit header + PrimitiveHeader Hdr; // 64 bit header }; - -struct ColRequestHeaderDataType: public datatypes::Charset +struct ColRequestHeaderDataType : public datatypes::Charset { - int32_t CompType; - uint16_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file - ColRequestHeaderDataType() - :Charset(my_charset_bin), - CompType(0), - DataSize(0), - DataType(0) - { } - ColRequestHeaderDataType(const execplan::CalpontSystemCatalog::ColType &rhs) - :Charset(rhs.charsetNumber), - CompType(rhs.compressionType), - DataSize(rhs.colWidth), - DataType(rhs.colDataType) - { } + int32_t CompType; + uint16_t DataSize; + uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + ColRequestHeaderDataType() : Charset(my_charset_bin), CompType(0), DataSize(0), DataType(0) + { + } + ColRequestHeaderDataType(const execplan::CalpontSystemCatalog::ColType& rhs) + : Charset(rhs.charsetNumber) + , CompType(rhs.compressionType) + , DataSize(rhs.colWidth) + , DataType(rhs.colDataType) + { + } }; - // COL_BY_SCAN -//Tied to ColByScanRangeRequestHeader and NewColRequestHeader. Check other headers if modifying. +// Tied to ColByScanRangeRequestHeader and NewColRequestHeader. Check other headers if modifying. struct ColByScanRequestHeader { - PrimitiveHeader Hdr; // 64 bit header - uint64_t LBID; - ColRequestHeaderDataType colType; - uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both - uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR - uint16_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7680-8191 - uint16_t NOPS; - uint16_t NVALS; - uint8_t sort; - ColByScanRequestHeader() - :LBID(0), OutputType(0), BOP(0), - RidFlags(0), NOPS(0), NVALS(0), sort(0) - { - memset(&Hdr, 0, sizeof(Hdr)); - } + PrimitiveHeader Hdr; // 64 bit header + uint64_t LBID; + ColRequestHeaderDataType colType; + uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both + uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR + uint16_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7680-8191 + uint16_t NOPS; + uint16_t NVALS; + uint8_t sort; + ColByScanRequestHeader() : LBID(0), OutputType(0), BOP(0), RidFlags(0), NOPS(0), NVALS(0), sort(0) + { + memset(&Hdr, 0, sizeof(Hdr)); + } }; // COL_BY_SCAN_RANGE -//Tied to ColByScanRequestHeader and NewColRequestHeader. Check other headers if modifying. +// Tied to ColByScanRequestHeader and NewColRequestHeader. Check other headers if modifying. struct ColByScanRangeRequestHeader { - PrimitiveHeader Hdr; // 64 bit header - uint64_t LBID; // starting LBID - ColRequestHeaderDataType colType; - uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both - uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR - uint16_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7680-8191 - uint16_t NOPS; - uint16_t NVALS; - uint8_t sort; - uint16_t Count; //Number of LBID's - ColByScanRangeRequestHeader() - :LBID(0), OutputType(0), - BOP(0), RidFlags(0), NOPS(0), NVALS(0), - sort(0), Count(0) - { - memset(&Hdr, 0, sizeof(Hdr)); - } + PrimitiveHeader Hdr; // 64 bit header + uint64_t LBID; // starting LBID + ColRequestHeaderDataType colType; + uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both + uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR + uint16_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7680-8191 + uint16_t NOPS; + uint16_t NVALS; + uint8_t sort; + uint16_t Count; // Number of LBID's + ColByScanRangeRequestHeader() + : LBID(0), OutputType(0), BOP(0), RidFlags(0), NOPS(0), NVALS(0), sort(0), Count(0) + { + memset(&Hdr, 0, sizeof(Hdr)); + } }; // COL_BY_RID struct ColByRIDRequestHeader { - PrimitiveHeader Hdr; // 64 bit header - uint64_t LBID; - ColRequestHeaderDataType colType; - uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both - uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR - uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair - uint16_t NOPS; - uint16_t NVALS; - uint8_t sort; - ColByRIDRequestHeader(); // QQ? Not used? + PrimitiveHeader Hdr; // 64 bit header + uint64_t LBID; + ColRequestHeaderDataType colType; + uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both + uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR + uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair + uint16_t NOPS; + uint16_t NVALS; + uint8_t sort; + ColByRIDRequestHeader(); // QQ? Not used? }; // COL_AGG_BY_SCAN struct ColAggByScanRequestHeader { - PrimitiveHeader Hdr; // 64 bit header - uint64_t LBID; - uint16_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file - uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both - uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR - uint8_t ExtraNotUsed; - uint16_t NOPS; - uint16_t NVALS; + PrimitiveHeader Hdr; // 64 bit header + uint64_t LBID; + uint16_t DataSize; + uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both + uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR + uint8_t ExtraNotUsed; + uint16_t NOPS; + uint16_t NVALS; }; // COL_AGG_BY_RID struct ColAggByRIDRequestHeader { - PrimitiveHeader Hdr; // 64 bit header - uint64_t LBID; - ColRequestHeaderDataType colType; - uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both - uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR - uint8_t ExtraNotUsed; - uint16_t NOPS; - uint16_t NVALS; - ColAggByRIDRequestHeader(); // Not used? + PrimitiveHeader Hdr; // 64 bit header + uint64_t LBID; + ColRequestHeaderDataType colType; + uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both + uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR + uint8_t ExtraNotUsed; + uint16_t NOPS; + uint16_t NVALS; + ColAggByRIDRequestHeader(); // Not used? }; // Loopback Results struct LoopbackResultHeader { - PrimitiveHeader Hdr; + PrimitiveHeader Hdr; }; // Column Results - // Column Aggregate results struct ColAggResultHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - uint64_t MIN; // Minimum value in this block (signed) - uint64_t MAX; // Maximum value in this block (signed) - uint64_t SUM; // Sum of values in this block (unsigned) - uint32_t SUMOverflow; // Overflow of sum (unsigned) - uint16_t NVALS; // Number of values in this block - uint16_t Pad1; + PrimitiveHeader Hdr; + uint64_t LBID; + uint64_t MIN; // Minimum value in this block (signed) + uint64_t MAX; // Maximum value in this block (signed) + uint64_t SUM; // Sum of values in this block (unsigned) + uint32_t SUMOverflow; // Overflow of sum (unsigned) + uint16_t NVALS; // Number of values in this block + uint16_t Pad1; }; // INDEX_BY_SCAN struct IndexByScanRequestHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - uint32_t State; - uint8_t Flags; - uint8_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file - uint8_t ExtraNotUsed; - uint16_t NVALS; + PrimitiveHeader Hdr; + uint64_t LBID; + uint32_t State; + uint8_t Flags; + uint8_t DataSize; + uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + uint8_t ExtraNotUsed; + uint16_t NVALS; }; // INDEX_BY_COMPARE struct IndexByCompareRequestHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - uint32_t State; - uint8_t Flags; - uint8_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file - uint8_t ExtraNotUsed; - uint16_t NVALS; + PrimitiveHeader Hdr; + uint64_t LBID; + uint32_t State; + uint8_t Flags; + uint8_t DataSize; + uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + uint8_t ExtraNotUsed; + uint16_t NVALS; }; struct IndexResultHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - uint32_t State; - uint16_t NVALS; - uint16_t Pad; + PrimitiveHeader Hdr; + uint64_t LBID; + uint32_t State; + uint16_t NVALS; + uint16_t Pad; }; // p_IdxWalk @@ -520,20 +498,20 @@ struct IndexResultHeader #ifdef __cplusplus struct IndexWalkHeader { - ISMPacketHeader ism; - PrimitiveHeader Hdr; - uint64_t SearchString[2]; - const std::vector* SearchStrings; //used only if NVALS > 2 - uint8_t SSlen; // width of the search argument in BITS - uint8_t Shift; // initialize to zero when first sending to primitive - uint8_t BOP; - uint8_t COP1; - uint8_t COP2; - uint8_t State; //right now this is only 1 or 0, specifying entire subtrees - uint16_t NVALS; - uint64_t LBID: 36; - uint8_t SubBlock: 5; - uint8_t SBEntry: 5; + ISMPacketHeader ism; + PrimitiveHeader Hdr; + uint64_t SearchString[2]; + const std::vector* SearchStrings; // used only if NVALS > 2 + uint8_t SSlen; // width of the search argument in BITS + uint8_t Shift; // initialize to zero when first sending to primitive + uint8_t BOP; + uint8_t COP1; + uint8_t COP2; + uint8_t State; // right now this is only 1 or 0, specifying entire subtrees + uint16_t NVALS; + uint64_t LBID : 36; + uint8_t SubBlock : 5; + uint8_t SBEntry : 5; }; #endif @@ -541,76 +519,76 @@ struct IndexWalkHeader struct IndexListHeader { - ISMPacketHeader ism; - PrimitiveHeader Hdr; - uint64_t LBID; - uint16_t NVALS; - uint16_t Pad1; - uint32_t Pad2; - // As the input parameter, what follows is IndexListParam[NVALS] - // As the output parameter, what follows is IndexListResult[NVALS] + ISMPacketHeader ism; + PrimitiveHeader Hdr; + uint64_t LBID; + uint16_t NVALS; + uint16_t Pad1; + uint32_t Pad2; + // As the input parameter, what follows is IndexListParam[NVALS] + // As the output parameter, what follows is IndexListResult[NVALS] }; // p_IdxList parameter struct IndexListParam { - uint64_t type : 3; // 0 - header, 4 - subblock, 5 - block - uint64_t spare : 15; - uint64_t fbo : 36; - uint64_t sbid : 5; - uint64_t entry : 5; -// int64_t listValue; + uint64_t type : 3; // 0 - header, 4 - subblock, 5 - block + uint64_t spare : 15; + uint64_t fbo : 36; + uint64_t sbid : 5; + uint64_t entry : 5; + // int64_t listValue; }; struct IndexListEntry { - uint64_t type : 3; - uint64_t spare : 5; - uint64_t ridCt : 10; - uint64_t value : 46; + uint64_t type : 3; + uint64_t spare : 5; + uint64_t ridCt : 10; + uint64_t value : 46; }; struct IndexListResult { - IndexListEntry entry; - int64_t listValue; + IndexListEntry entry; + int64_t listValue; }; enum IndexListType { - LIST_SIZE = 0, - EMPTY_LIST_PTR = 1, - EMPTY_PTR = 2, - RID = 3, - LLP_SUBBLK = 4, - LLP_BLK = 5, - PARENT = 6, - NOT_IN_USE = 7 + LIST_SIZE = 0, + EMPTY_LIST_PTR = 1, + EMPTY_PTR = 2, + RID = 3, + LLP_SUBBLK = 4, + LLP_BLK = 5, + PARENT = 6, + NOT_IN_USE = 7 }; // DICT_TOKEN_BY_INDEX_COMPARE struct DictTokenByIndexRequestHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - uint16_t NVALS; - uint16_t Pad1; - uint32_t Pad2; + PrimitiveHeader Hdr; + uint64_t LBID; + uint16_t NVALS; + uint16_t Pad1; + uint32_t Pad2; }; // DICT_TOKEN_BY_SCAN_COMPARE struct DataValue { - uint16_t len; - char data[]; + uint16_t len; + char data[]; }; struct PrimToken { - uint64_t LBID; - uint16_t offset; // measured in bytes - uint16_t len; // # of bytes + uint64_t LBID; + uint16_t offset; // measured in bytes + uint16_t len; // # of bytes }; // Masks for the flags member of TokenByScanRequestHeader @@ -619,96 +597,95 @@ struct PrimToken struct TokenByScanRequestHeader { - ISMPacketHeader ism; - PrimitiveHeader Hdr; - uint64_t LBID; - int32_t CompType; - uint8_t COP1; - uint8_t COP2; - uint8_t BOP; - uint8_t OutputType; - uint16_t NVALS; - uint16_t flags; - uint32_t Pad2; - uint16_t Count; - uint32_t charsetNumber; -}; // what follows is NVALS DataValues. + ISMPacketHeader ism; + PrimitiveHeader Hdr; + uint64_t LBID; + int32_t CompType; + uint8_t COP1; + uint8_t COP2; + uint8_t BOP; + uint8_t OutputType; + uint16_t NVALS; + uint16_t flags; + uint32_t Pad2; + uint16_t Count; + uint32_t charsetNumber; +}; // what follows is NVALS DataValues. // compatibility with Ron's stuff. typedef TokenByScanRequestHeader DictTokenByScanRequestHeader; struct TokenByScanResultHeader { - ISMPacketHeader ism; - PrimitiveHeader Hdr; - uint32_t NBYTES; - uint16_t NVALS; - uint16_t Pad1; - uint32_t CacheIO; // I/O count from buffer cache - uint32_t PhysicalIO; // Physical I/O count from disk + ISMPacketHeader ism; + PrimitiveHeader Hdr; + uint32_t NBYTES; + uint16_t NVALS; + uint16_t Pad1; + uint32_t CacheIO; // I/O count from buffer cache + uint32_t PhysicalIO; // Physical I/O count from disk }; // what follows is NVALS Tokens or DataValues. // DICT_SIGNATURE struct DictSignatureRequestHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - int32_t CompType; - uint16_t NVALS; - uint16_t Pad1; - uint32_t Pad2; + PrimitiveHeader Hdr; + uint64_t LBID; + int32_t CompType; + uint16_t NVALS; + uint16_t Pad1; + uint32_t Pad2; }; -//Tied to DictSignatureRequestHeader, note if modifying either. +// Tied to DictSignatureRequestHeader, note if modifying either. struct DictSignatureRangeRequestHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - int32_t CompType; - uint16_t NVALS; - uint16_t Pad1; - uint32_t Pad2; - uint16_t Count; + PrimitiveHeader Hdr; + uint64_t LBID; + int32_t CompType; + uint16_t NVALS; + uint16_t Pad1; + uint32_t Pad2; + uint16_t Count; }; - // DICT_AGGREGATE struct DictAggregateRequestHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - int32_t CompType; - uint16_t NVALS; - uint16_t Pad1; - uint32_t Pad2; + PrimitiveHeader Hdr; + uint64_t LBID; + int32_t CompType; + uint16_t NVALS; + uint16_t Pad1; + uint32_t Pad2; }; struct DictResultHeader { - PrimitiveHeader Hdr; - uint64_t LBID; - uint16_t NVALS; - uint16_t Pad1; - uint32_t Pad2; + PrimitiveHeader Hdr; + uint64_t LBID; + uint16_t NVALS; + uint16_t Pad1; + uint32_t Pad2; }; struct AggregateSignatureRequestHeader { - ISMPacketHeader ism; - PrimitiveHeader hdr; - uint16_t NVALS; - PrimToken tokens[]; + ISMPacketHeader ism; + PrimitiveHeader hdr; + uint16_t NVALS; + PrimToken tokens[]; }; struct AggregateSignatureResultHeader { - ISMPacketHeader ism; - PrimitiveHeader hdr; - uint16_t Count; - // these implicitly follow the header - // DataValue min; - // DataValue max; + ISMPacketHeader ism; + PrimitiveHeader hdr; + uint16_t Count; + // these implicitly follow the header + // DataValue min; + // DataValue max; }; /* An array of these structures defines a filter applied by p_Col. The length @@ -717,197 +694,196 @@ struct AggregateSignatureResultHeader */ struct ColArgs { - uint8_t COP; - uint8_t rf; // rounding flag: indicates if val is truncated or saturated - // for further evaluation of an equal condiction - int8_t val[]; + uint8_t COP; + uint8_t rf; // rounding flag: indicates if val is truncated or saturated + // for further evaluation of an equal condiction + int8_t val[]; }; // const for rf const uint8_t ROUND_POS = 0x01; // actual value larger/longer than the stored value const uint8_t ROUND_NEG = 0x80; // actual value less than the stored value -//Tied to ColByScanRequestHeader and ColByScanRangeRequestHeader. Check other headers if modifying. +// Tied to ColByScanRequestHeader and ColByScanRangeRequestHeader. Check other headers if modifying. struct NewColRequestHeader { - ISMPacketHeader ism; - PrimitiveHeader hdr; - uint64_t LBID; - ColRequestHeaderDataType colType; - uint8_t OutputType; // OT_DATAVALUE, OT_RID, or OT_BOTH - uint8_t BOP; -// uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair (deprecated) - uint16_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7680-8191 - uint16_t NOPS; - uint16_t NVALS; - uint8_t sort; //1 to sort - // this follows the header - // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) - // uint16_t Rids[NVALS] (each rid is relative to the given block) + ISMPacketHeader ism; + PrimitiveHeader hdr; + uint64_t LBID; + ColRequestHeaderDataType colType; + uint8_t OutputType; // OT_DATAVALUE, OT_RID, or OT_BOTH + uint8_t BOP; + // uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair (deprecated) + uint16_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7680-8191 + uint16_t NOPS; + uint16_t NVALS; + uint8_t sort; // 1 to sort + // this follows the header + // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) + // uint16_t Rids[NVALS] (each rid is relative to the given block) - // QQ: The below constructor is never used. - // This struct is used in a cast only, in a hackish way. - NewColRequestHeader(); - inline uint16_t* getRIDArrayPtr(const int width) const + // QQ: The below constructor is never used. + // This struct is used in a cast only, in a hackish way. + NewColRequestHeader(); + inline uint16_t* getRIDArrayPtr(const int width) const + { + return (NVALS > 0) ? reinterpret_cast((uint8_t*)this + sizeof(NewColRequestHeader) + + (NOPS * getFilterSize(width))) + : nullptr; + } + // WIP change to unsigned if needed + inline int getFilterSize(const int width) const + { + return sizeof(uint8_t) + sizeof(uint8_t) + width; + } + inline uint8_t* getFilterStringPtr() + { + return reinterpret_cast(this) + sizeof(NewColRequestHeader); + } + inline void sortRIDArrayIfNeeded(const int width) + { + // NVALS means number of RIDs in the context of the f(). + if (NVALS > 0) { - return (NVALS > 0) ? reinterpret_cast((uint8_t*)this + - sizeof(NewColRequestHeader) + - (NOPS * getFilterSize(width))) - : nullptr; - } - // WIP change to unsigned if needed - inline int getFilterSize(const int width) const - { - return sizeof(uint8_t) + sizeof(uint8_t) + width; - } - inline uint8_t* getFilterStringPtr() - { - return reinterpret_cast(this) + sizeof(NewColRequestHeader); - } - inline void sortRIDArrayIfNeeded(const int width) - { - // NVALS means number of RIDs in the context of the f(). - if (NVALS > 0) - { - uint16_t* ridArray = getRIDArrayPtr(width); - if (sort == 1) - std::sort(ridArray, ridArray + NVALS); - } + uint16_t* ridArray = getRIDArrayPtr(width); + if (sort == 1) + std::sort(ridArray, ridArray + NVALS); } + } }; struct NewColAggRequestHeader { - ISMPacketHeader ism; - PrimitiveHeader hdr; - uint64_t LBID; - ColRequestHeaderDataType colType; - uint8_t OutputType; - uint8_t BOP; - uint8_t ExtraNotUsed; - uint16_t NOPS; - uint16_t NVALS; - // this follows the header - // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) - // uint16_t Rids[NVALS] (each rid is relative to the given block) - NewColAggRequestHeader(); // QQ: not used + ISMPacketHeader ism; + PrimitiveHeader hdr; + uint64_t LBID; + ColRequestHeaderDataType colType; + uint8_t OutputType; + uint8_t BOP; + uint8_t ExtraNotUsed; + uint16_t NOPS; + uint16_t NVALS; + // this follows the header + // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) + // uint16_t Rids[NVALS] (each rid is relative to the given block) + NewColAggRequestHeader(); // QQ: not used }; // The size of the structure !MUST! be aligned by the max(sizeof(DataType)) supported by MCS. struct ColResultHeader { - PrimitiveHeader hdr; - int128_t Min; // Minimum value in this block for signed data types - int128_t Max; // Maximum value in this block for signed data types - ISMPacketHeader ism; - uint64_t LBID; - uint16_t RidFlags; - primitives::NVALSType NVALS; - uint16_t ValidMinMax; // 1 if Min/Max are valid, otherwise 0 - uint32_t OutputType; - uint32_t CacheIO; // I/O count from buffer cache - uint32_t PhysicalIO; // Physical I/O count from disk - char padding[34]; - // if OutputType was OT_DATAVALUE, what follows is DataType[NVALS] - // if OutputType was OT_RID, what follows is uint16_t Rids[NVALS] - // if OutputType was OT_BOTH, what follows is uint16_t Rids[NVALS] DataType[NVALS] + PrimitiveHeader hdr; + int128_t Min; // Minimum value in this block for signed data types + int128_t Max; // Maximum value in this block for signed data types + ISMPacketHeader ism; + uint64_t LBID; + uint16_t RidFlags; + primitives::NVALSType NVALS; + uint16_t ValidMinMax; // 1 if Min/Max are valid, otherwise 0 + uint32_t OutputType; + uint32_t CacheIO; // I/O count from buffer cache + uint32_t PhysicalIO; // Physical I/O count from disk + char padding[34]; + // if OutputType was OT_DATAVALUE, what follows is DataType[NVALS] + // if OutputType was OT_RID, what follows is uint16_t Rids[NVALS] + // if OutputType was OT_BOTH, what follows is uint16_t Rids[NVALS] DataType[NVALS] }; namespace primitives { - constexpr static uint32_t ColResultHeaderFirstValueOffset = sizeof(ColResultHeader) + sizeof(RIDType) * BLOCK_SIZE; - constexpr static uint32_t RID2FirstValueOffset = sizeof(RIDType) * BLOCK_SIZE; +constexpr static uint32_t ColResultHeaderFirstValueOffset = + sizeof(ColResultHeader) + sizeof(RIDType) * BLOCK_SIZE; +constexpr static uint32_t RID2FirstValueOffset = sizeof(RIDType) * BLOCK_SIZE; - template - inline T* getValuesArrayPosition(uint8_t* out, const NVALSType offset) - { - return reinterpret_cast(out + offset * sizeof(T)); - } - - inline primitives::RIDType* getRIDArrayPosition(uint8_t* out, const NVALSType offset) - { - return getValuesArrayPosition(out, offset); - } - - inline uint8_t* getFirstValueArrayPosition(ColResultHeader* outMsg) - { - return reinterpret_cast(outMsg) + ColResultHeaderFirstValueOffset; - } - - inline uint8_t* getFirstRIDArrayPosition(ColResultHeader* outMsg) - { - return reinterpret_cast(&outMsg[1]); - } +template +inline T* getValuesArrayPosition(uint8_t* out, const NVALSType offset) +{ + return reinterpret_cast(out + offset * sizeof(T)); } +inline primitives::RIDType* getRIDArrayPosition(uint8_t* out, const NVALSType offset) +{ + return getValuesArrayPosition(out, offset); +} + +inline uint8_t* getFirstValueArrayPosition(ColResultHeader* outMsg) +{ + return reinterpret_cast(outMsg) + ColResultHeaderFirstValueOffset; +} + +inline uint8_t* getFirstRIDArrayPosition(ColResultHeader* outMsg) +{ + return reinterpret_cast(&outMsg[1]); +} +} // namespace primitives + /* additional types to support p_dictionary */ struct DictFilterElement { - uint8_t COP; - uint16_t len; // this is the length of data, not the size of the entire entry - uint8_t data[]; + uint8_t COP; + uint16_t len; // this is the length of data, not the size of the entire entry + uint8_t data[]; }; struct DictInput { - ISMPacketHeader ism; - PrimitiveHeader hdr; - uint64_t LBID; - uint8_t BOP; - uint8_t InputFlags; // 1 -> 64-bit RID, 64-bit token pairs (old p_GetSignature behavior), - // 0-> new behavior - uint8_t OutputType; - uint16_t NOPS; - uint16_t NVALS; - PrimToken tokens[]; // NVALS of these. - // DictFilterElement[NOPS] filter; + ISMPacketHeader ism; + PrimitiveHeader hdr; + uint64_t LBID; + uint8_t BOP; + uint8_t InputFlags; // 1 -> 64-bit RID, 64-bit token pairs (old p_GetSignature behavior), + // 0-> new behavior + uint8_t OutputType; + uint16_t NOPS; + uint16_t NVALS; + PrimToken tokens[]; // NVALS of these. + // DictFilterElement[NOPS] filter; }; struct DictAggregate { - uint16_t Count; - // DataValue min; - // DataValue max; + uint16_t Count; + // DataValue min; + // DataValue max; }; struct DictOutput { - ISMPacketHeader ism; - PrimitiveHeader hdr; - uint64_t LBID; - uint16_t NVALS; - uint16_t Pad; - uint32_t NBYTES; - uint32_t CacheIO; // I/O count from buffer cache - uint32_t PhysicalIO; // Physical I/O count from disk - // What follows this header depends on OutputType. - // for each NVAL, what follows is ... - // if OutputType | OT_RID & InputFlags==1, the 64-bit RID associated with the input token - // if OutputType | OT_TOKEN, a PrimToken - // if OutputType | OT_DATAVALUE, a DataValue containing the string in the dict block - // if OutputType | OT_INPUT, a DataValue containing the first filter string... - // ... that matched (only makes sense when BOP is OR). - // DictAggregate agg; (if OutputType | OT_AGGREGATE) -}; //same as TokenByScanResultHeader at the moment + ISMPacketHeader ism; + PrimitiveHeader hdr; + uint64_t LBID; + uint16_t NVALS; + uint16_t Pad; + uint32_t NBYTES; + uint32_t CacheIO; // I/O count from buffer cache + uint32_t PhysicalIO; // Physical I/O count from disk + // What follows this header depends on OutputType. + // for each NVAL, what follows is ... + // if OutputType | OT_RID & InputFlags==1, the 64-bit RID associated with the input token + // if OutputType | OT_TOKEN, a PrimToken + // if OutputType | OT_DATAVALUE, a DataValue containing the string in the dict block + // if OutputType | OT_INPUT, a DataValue containing the first filter string... + // ... that matched (only makes sense when BOP is OR). + // DictAggregate agg; (if OutputType | OT_AGGREGATE) +}; // same as TokenByScanResultHeader at the moment struct OldGetSigParams { - uint64_t rid; - uint16_t offsetIndex; + uint64_t rid; + uint16_t offsetIndex; }; struct LbidAtVer { - uint64_t LBID; - uint32_t Ver; + uint64_t LBID; + uint32_t Ver; }; #ifdef _MSC_VER -#pragma warning (pop) +#pragma warning(pop) #endif #pragma pack(pop) -#endif //JOBLIST_PRIMITIVE_H +#endif // JOBLIST_PRIMITIVE_H // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index b92e461a5..5f1e28665 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -18,7 +18,6 @@ // $Id: primitivestep.h 9688 2013-07-15 19:27:22Z pleblanc $ - /** @file */ #ifndef JOBLIST_PRIMITIVESTEP_H @@ -70,7 +69,6 @@ namespace joblist { - /* Forward decl's to support the batch primitive classes */ struct JobInfo; class CommandJL; @@ -82,331 +80,326 @@ class pColScanStep; class PassThruStep; class PseudoColStep; - typedef boost::shared_ptr SP_LBIDList; typedef std::vector OIDVector; -typedef std::vector > OIDIntVector; - +typedef std::vector> OIDIntVector; enum PrimitiveStepType { - SCAN, - COLSTEP, - DICTIONARYSCAN, - DICTIONARY, - PASSTHRU, - AGGRFILTERSTEP + SCAN, + COLSTEP, + DICTIONARYSCAN, + DICTIONARY, + PASSTHRU, + AGGRFILTERSTEP }; - /** @brief class PrimitiveMsg * */ class PrimitiveMsg { -public: - /** @brief virtual void Send method - */ - virtual void send(); - /** @brief virtual void Receive method - */ - virtual void receive(); - /** @brief virtual void BuildPrimitiveMessage method - */ - virtual void buildPrimitiveMessage(ISMPACKETCOMMAND cmd, void* filterValues, void* ridArray); - virtual void sendPrimitiveMessages(); - virtual void receivePrimitiveMessages(); + public: + /** @brief virtual void Send method + */ + virtual void send(); + /** @brief virtual void Receive method + */ + virtual void receive(); + /** @brief virtual void BuildPrimitiveMessage method + */ + virtual void buildPrimitiveMessage(ISMPACKETCOMMAND cmd, void* filterValues, void* ridArray); + virtual void sendPrimitiveMessages(); + virtual void receivePrimitiveMessages(); - PrimitiveMsg() { } + PrimitiveMsg() + { + } - virtual ~PrimitiveMsg() { } + virtual ~PrimitiveMsg() + { + } - uint16_t planFlagsToPrimFlags(uint32_t planFlags); + uint16_t planFlagsToPrimFlags(uint32_t planFlags); -private: + private: }; - class pColScanStep; class pColStep : public JobStep, public PrimitiveMsg { + typedef std::pair element_t; - typedef std::pair element_t; + public: + /** @brief pColStep constructor + * @param flushInterval The interval in msgs at which the sending side should + * wait for the receiveing side to catch up. 0 (default) means never. + */ + pColStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, + const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); -public: - /** @brief pColStep constructor - * @param flushInterval The interval in msgs at which the sending side should - * wait for the receiveing side to catch up. 0 (default) means never. - */ - pColStep( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tableOid, - const execplan::CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo); + pColStep(const pColScanStep& rhs); - pColStep(const pColScanStep& rhs); + pColStep(const PassThruStep& rhs); - pColStep(const PassThruStep& rhs); + virtual ~pColStep(); - virtual ~pColStep(); + /** @brief Starts processing. Set at least the RID list before calling. + * + * Starts processing. Set at least the RID list before calling this. + */ + virtual void run(); + /** @brief Sync's the caller with the end of execution. + * + * Does nothing. Returns when this instance is finished. + */ + virtual void join(); - /** @brief Starts processing. Set at least the RID list before calling. - * - * Starts processing. Set at least the RID list before calling this. - */ - virtual void run(); - /** @brief Sync's the caller with the end of execution. - * - * Does nothing. Returns when this instance is finished. - */ - virtual void join(); + virtual const std::string toString() const; - virtual const std::string toString() const; + virtual bool isDictCol() const + { + return fIsDict; + }; + bool isExeMgr() const + { + return isEM; + } - virtual bool isDictCol() const - { - return fIsDict; - }; - bool isExeMgr() const - { - return isEM; - } + /** @brief Set config parameters for this JobStep. + * + * Set the config parameters this JobStep. + */ + void initializeConfigParms(); - /** @brief Set config parameters for this JobStep. - * - * Set the config parameters this JobStep. - */ - void initializeConfigParms(); + /** @brief The main loop for the send-side thread + * + * The main loop for the primitive-issuing thread. Don't call it directly. + */ + void sendPrimitiveMessages(); - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); + /** @brief The main loop for the recv-side thread + * + * The main loop for the receive-side thread. Don't call it directly. + */ + void receivePrimitiveMessages(); - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - void receivePrimitiveMessages(); + /** @brief Add a filter. Use this interface when the column stores anything but 4-byte floats. + * + * Add a filter. Use this interface when the column stores anything but 4-byte floats. + */ + void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); + void addFilter(int8_t COP, float value); + void addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag = 0); - /** @brief Add a filter. Use this interface when the column stores anything but 4-byte floats. - * - * Add a filter. Use this interface when the column stores anything but 4-byte floats. - */ - void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); - void addFilter(int8_t COP, float value); - void addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag = 0); + /** @brief Sets the DataList to get RID values from. + * + * Sets the DataList to get RID values from. Filtering by RID distinguishes + * this class from pColScan. Use pColScan if the every RID should be considered; it's + * faster at that. + */ + void setRidList(DataList* rids); - /** @brief Sets the DataList to get RID values from. - * - * Sets the DataList to get RID values from. Filtering by RID distinguishes - * this class from pColScan. Use pColScan if the every RID should be considered; it's - * faster at that. - */ - void setRidList(DataList* rids); + /** @brief Sets the String DataList to get RID values from. + * + * Sets the string DataList to get RID values from. Filtering by RID distinguishes + * this class from pColScan. Use pColScan if the every RID should be considered; it's + * faster at that. + */ + void setStrRidList(DataList* strDl); - /** @brief Sets the String DataList to get RID values from. - * - * Sets the string DataList to get RID values from. Filtering by RID distinguishes - * this class from pColScan. Use pColScan if the every RID should be considered; it's - * faster at that. - */ - void setStrRidList(DataList* strDl); + /** @brief Set the binary operator for the filter predicate (BOP_AND or BOP_OR). + * + * Set the binary operator for the filter predicate (BOP_AND or BOP_OR). + */ + void setBOP(int8_t BOP); - /** @brief Set the binary operator for the filter predicate (BOP_AND or BOP_OR). - * - * Set the binary operator for the filter predicate (BOP_AND or BOP_OR). - */ - void setBOP(int8_t BOP); + /** @brief Set the output type. + * + * Set the output type (1 = RID, 2 = Token, 3 = Both). + */ + void setOutputType(int8_t OutputType); - /** @brief Set the output type. - * - * Set the output type (1 = RID, 2 = Token, 3 = Both). - */ - void setOutputType(int8_t OutputType); + /** @brief Set the swallowRows flag. + * + * + * If true, no rows will be inserted to the output datalists. + */ + void setSwallowRows(const bool swallowRows); - /** @brief Set the swallowRows flag. - * - * - * If true, no rows will be inserted to the output datalists. - */ - void setSwallowRows(const bool swallowRows); + /** @brief Get the swallowRows flag. + * + * + * If true, no rows will be inserted to the output datalists. + */ + bool getSwallowRows() const + { + return fSwallowRows; + } - /** @brief Get the swallowRows flag. - * - * - * If true, no rows will be inserted to the output datalists. - */ - bool getSwallowRows() const - { - return fSwallowRows; - } + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return fOid; + } - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return fOid; - } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } + uint32_t filterCount() const + { + return fFilterCount; + } + const messageqcpp::ByteStream& filterString() const + { + return fFilterString; + } + int8_t BOP() const + { + return fBOP; + } + const execplan::CalpontSystemCatalog::ColType& colType() const + { + return fColType; + } + void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); + uint32_t flushInterval() const + { + return fFlushInterval; + } + bool getFeederFlag() const + { + return isFilterFeeder; + } - uint32_t filterCount() const - { - return fFilterCount; - } - const messageqcpp::ByteStream& filterString() const - { - return fFilterString; - } - int8_t BOP() const - { - return fBOP; - } - const execplan::CalpontSystemCatalog::ColType& colType() const - { - return fColType; - } - void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); - uint32_t flushInterval() const - { - return fFlushInterval; - } - bool getFeederFlag() const - { - return isFilterFeeder; - } + void setFeederFlag(bool filterFeeder) + { + isFilterFeeder = filterFeeder; + } + virtual uint64_t phyIOCount() const + { + return fPhysicalIO; + } + virtual uint64_t cacheIOCount() const + { + return fCacheIO; + } + virtual uint64_t msgsRcvdCount() const + { + return msgsRecvd; + } + virtual uint64_t msgBytesIn() const + { + return fMsgBytesIn; + } + virtual uint64_t msgBytesOut() const + { + return fMsgBytesOut; + } - void setFeederFlag (bool filterFeeder) - { - isFilterFeeder = filterFeeder; - } - virtual uint64_t phyIOCount () const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount () const - { - return fCacheIO; - } - virtual uint64_t msgsRcvdCount () const - { - return msgsRecvd; - } - virtual uint64_t msgBytesIn () const - { - return fMsgBytesIn; - } - virtual uint64_t msgBytesOut () const - { - return fMsgBytesOut; - } + //...Currently only supported by pColStep and pColScanStep, so didn't bother + //...to define abstract method in base class, but if start adding to other + //...classes, then should consider adding pure virtual method to JobStep. + uint64_t blksSkipped() const + { + return fNumBlksSkipped; + } + ResourceManager* resourceManager() const + { + return fRm; + } - //...Currently only supported by pColStep and pColScanStep, so didn't bother - //...to define abstract method in base class, but if start adding to other - //...classes, then should consider adding pure virtual method to JobStep. - uint64_t blksSkipped () const - { - return fNumBlksSkipped; - } - ResourceManager* resourceManager() const - { - return fRm; - } + SP_LBIDList getlbidList() const + { + return lbidList; + } - SP_LBIDList getlbidList() const - { - return lbidList; - } + void addFilter(const execplan::Filter* f); + void appendFilter(const std::vector& fs); + std::vector& getFilters() + { + return fFilters; + } - void addFilter(const execplan::Filter* f); - void appendFilter(const std::vector& fs); - std::vector& getFilters() - { - return fFilters; - } + protected: + void addFilters(); -protected: - void addFilters(); + private: + /** @brief constructor for completeness + */ + explicit pColStep(); -private: + /** @brief StartPrimitiveThread + * Utility function to start worker thread that sends primitive messages + */ + void startPrimitiveThread(); + /** @brief StartAggregationThread + * Utility function to start worker thread that receives result aggregation from primitive servers + */ + void startAggregationThread(); + uint64_t getLBID(uint64_t rid, bool& scan); + uint64_t getFBO(uint64_t lbid); - /** @brief constructor for completeness - */ - explicit pColStep(); + ResourceManager* fRm; + boost::shared_ptr sysCat; + execplan::CalpontSystemCatalog::OID fOid; + execplan::CalpontSystemCatalog::OID fTableOid; + execplan::CalpontSystemCatalog::ColType fColType; + uint32_t fFilterCount; + int8_t fBOP; + int8_t fOutputType; + uint16_t realWidth; + DataList_t* ridList; + StrDataList* strRidList; + messageqcpp::ByteStream fFilterString; + std::vector extents; + uint32_t extentSize, divShift, modMask, ridsPerBlock, rpbShift, blockSizeShift, numExtents; + uint64_t rpbMask; + uint64_t msgsSent, msgsRecvd; + bool finishedSending, recvWaiting, fIsDict; + bool isEM; + int64_t ridCount; + uint32_t fFlushInterval; - /** @brief StartPrimitiveThread - * Utility function to start worker thread that sends primitive messages - */ - void startPrimitiveThread(); - /** @brief StartAggregationThread - * Utility function to start worker thread that receives result aggregation from primitive servers - */ - void startAggregationThread(); - uint64_t getLBID(uint64_t rid, bool& scan); - uint64_t getFBO(uint64_t lbid); + // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is TRACE_FLAGS::TRACE_NO_ROWS4. + // Running with this one will swallow rows at projection. + bool fSwallowRows; + uint32_t fProjectBlockReqLimit; // max number of rids to send in a scan + // request to primproc + uint32_t fProjectBlockReqThreshold; // min level of rids backlog before + // consumer will tell producer to send + // more rids scan requests to primproc - ResourceManager* fRm; - boost::shared_ptr sysCat; - execplan::CalpontSystemCatalog::OID fOid; - execplan::CalpontSystemCatalog::OID fTableOid; - execplan::CalpontSystemCatalog::ColType fColType; - uint32_t fFilterCount; - int8_t fBOP; - int8_t fOutputType; - uint16_t realWidth; - DataList_t* ridList; - StrDataList* strRidList; - messageqcpp::ByteStream fFilterString; - std::vector extents; - uint32_t extentSize, divShift, modMask, ridsPerBlock, rpbShift, blockSizeShift, numExtents; - uint64_t rpbMask; - uint64_t msgsSent, msgsRecvd; - bool finishedSending, recvWaiting, fIsDict; - bool isEM; - int64_t ridCount; - uint32_t fFlushInterval; + volatile bool fStopSending; + bool isFilterFeeder; + uint64_t fPhysicalIO; // total physical I/O count + uint64_t fCacheIO; // total cache I/O count + uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP + uint64_t fMsgBytesIn; // total byte count for incoming messages + uint64_t fMsgBytesOut; // total byte count for outcoming messages - // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is TRACE_FLAGS::TRACE_NO_ROWS4. - // Running with this one will swallow rows at projection. - bool fSwallowRows; - uint32_t fProjectBlockReqLimit; // max number of rids to send in a scan - // request to primproc - uint32_t fProjectBlockReqThreshold; // min level of rids backlog before - // consumer will tell producer to send - // more rids scan requests to primproc + BRM::DBRM dbrm; - volatile bool fStopSending; - bool isFilterFeeder; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count - uint64_t fNumBlksSkipped;//total number of block scans skipped due to CP - uint64_t fMsgBytesIn; // total byte count for incoming messages - uint64_t fMsgBytesOut; // total byte count for outcoming messages + // boost::shared_ptr cThread; //consumer thread + // boost::shared_ptr pThread; //producer thread + boost::mutex mutex; + boost::condition condvar; + boost::condition flushed; + SP_LBIDList lbidList; + std::vector scanFlags; // use to keep track of which extents to eliminate from this step + uint32_t uniqueID; - BRM::DBRM dbrm; + //@bug 2634 + //@bug 3128 change ParseTree* to vector + std::vector fFilters; - // boost::shared_ptr cThread; //consumer thread - // boost::shared_ptr pThread; //producer thread - boost::mutex mutex; - boost::condition condvar; - boost::condition flushed; - SP_LBIDList lbidList; - std::vector scanFlags; // use to keep track of which extents to eliminate from this step - uint32_t uniqueID; - - //@bug 2634 - //@bug 3128 change ParseTree* to vector - std::vector fFilters; - - friend class pColScanStep; - friend class PassThruStep; - friend class ColumnCommandJL; - friend class RTSCommandJL; - friend class BatchPrimitiveStep; - friend class TupleBPS; + friend class pColScanStep; + friend class PassThruStep; + friend class ColumnCommandJL; + friend class RTSCommandJL; + friend class BatchPrimitiveStep; + friend class TupleBPS; }; /** @brief the pColScan Step @@ -423,265 +416,258 @@ private: class pColScanStep : public JobStep, public PrimitiveMsg { -public: - /** @brief pColScanStep constructor - */ - pColScanStep( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tableOid, - const execplan::CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo); + public: + /** @brief pColScanStep constructor + */ + pColScanStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, + const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - pColScanStep(const pColStep& rhs); - ~pColScanStep(); + pColScanStep(const pColStep& rhs); + ~pColScanStep(); - /** @brief Starts processing. - * - * Starts processing. - */ - virtual void run(); + /** @brief Starts processing. + * + * Starts processing. + */ + virtual void run(); - /** @brief Sync's the caller with the end of execution. - * - * Does nothing. Returns when this instance is finished. - */ - virtual void join(); + /** @brief Sync's the caller with the end of execution. + * + * Does nothing. Returns when this instance is finished. + */ + virtual void join(); - virtual bool isDictCol() const - { - return fIsDict; - }; + virtual bool isDictCol() const + { + return fIsDict; + }; - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); + /** @brief The main loop for the send-side thread + * + * The main loop for the primitive-issuing thread. Don't call it directly. + */ + void sendPrimitiveMessages(); - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - using PrimitiveMsg::receivePrimitiveMessages; - void receivePrimitiveMessages(uint64_t i = 0); + /** @brief The main loop for the recv-side thread + * + * The main loop for the receive-side thread. Don't call it directly. + */ + using PrimitiveMsg::receivePrimitiveMessages; + void receivePrimitiveMessages(uint64_t i = 0); - /** @brief Add a filter when the column is a 4-byte float type - * - * Add a filter when the column is a 4-byte float type - */ - void addFilter(int8_t COP, float value); + /** @brief Add a filter when the column is a 4-byte float type + * + * Add a filter when the column is a 4-byte float type + */ + void addFilter(int8_t COP, float value); - /** @brief Add a filter when the column is anything but a 4-byte float type. - * - * Add a filter when the column is anything but a 4-byte float type, including - * 8-byte doubles. - */ - void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); + /** @brief Add a filter when the column is anything but a 4-byte float type. + * + * Add a filter when the column is anything but a 4-byte float type, including + * 8-byte doubles. + */ + void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); - /** @brief Set the binary operator for the filter predicates - * - * Set the binary operator for the filter predicates (BOP_AND or BOP_OR). - * It is initialized to OR. - */ - void setBOP(int8_t BOP); // AND or OR - int8_t BOP() const - { - return fBOP; - } + /** @brief Set the binary operator for the filter predicates + * + * Set the binary operator for the filter predicates (BOP_AND or BOP_OR). + * It is initialized to OR. + */ + void setBOP(int8_t BOP); // AND or OR + int8_t BOP() const + { + return fBOP; + } - bool getFeederFlag() const - { - return isFilterFeeder; - } + bool getFeederFlag() const + { + return isFilterFeeder; + } - void setFeederFlag (bool filterFeeder) - { - isFilterFeeder = filterFeeder; - } - /** @brief Get the string of the filter predicates - * - * Get the filter string constructed from the predicates - */ - messageqcpp::ByteStream filterString() const - { - return fFilterString; - } + void setFeederFlag(bool filterFeeder) + { + isFilterFeeder = filterFeeder; + } + /** @brief Get the string of the filter predicates + * + * Get the filter string constructed from the predicates + */ + messageqcpp::ByteStream filterString() const + { + return fFilterString; + } - void setSingleThread(bool b); - bool getSingleThread() - { - return fSingleThread; - } + void setSingleThread(bool b); + bool getSingleThread() + { + return fSingleThread; + } - /** @brief Set the output type. - * - * Set the output type (1 = RID, 2 = Token, 3 = Both).pColScan - */ - void setOutputType(int8_t OutputType); - uint32_t filterCount() const - { - return fFilterCount; - } + /** @brief Set the output type. + * + * Set the output type (1 = RID, 2 = Token, 3 = Both).pColScan + */ + void setOutputType(int8_t OutputType); + uint32_t filterCount() const + { + return fFilterCount; + } - virtual const std::string toString() const; + virtual const std::string toString() const; - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return fOid; - } + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return fOid; + } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } - const execplan::CalpontSystemCatalog::ColType& colType() const - { - return fColType; - } - ResourceManager* resourceManager() const - { - return fRm; - } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } + const execplan::CalpontSystemCatalog::ColType& colType() const + { + return fColType; + } + ResourceManager* resourceManager() const + { + return fRm; + } - virtual uint64_t phyIOCount () const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount () const - { - return fCacheIO; - } - virtual uint64_t msgsRcvdCount () const - { - return recvCount; - } - virtual uint64_t msgBytesIn () const - { - return fMsgBytesIn; - } - virtual uint64_t msgBytesOut () const - { - return fMsgBytesOut; - } - uint32_t getRidsPerBlock() const - { - return ridsPerBlock; - } + virtual uint64_t phyIOCount() const + { + return fPhysicalIO; + } + virtual uint64_t cacheIOCount() const + { + return fCacheIO; + } + virtual uint64_t msgsRcvdCount() const + { + return recvCount; + } + virtual uint64_t msgBytesIn() const + { + return fMsgBytesIn; + } + virtual uint64_t msgBytesOut() const + { + return fMsgBytesOut; + } + uint32_t getRidsPerBlock() const + { + return ridsPerBlock; + } - //...Currently only supported by pColStep and pColScanStep, so didn't bother - //...to define abstract method in base class, but if start adding to other - //...classes, then should consider adding pure virtual method to JobStep. - uint64_t blksSkipped () const - { - return fNumBlksSkipped; - } + //...Currently only supported by pColStep and pColScanStep, so didn't bother + //...to define abstract method in base class, but if start adding to other + //...classes, then should consider adding pure virtual method to JobStep. + uint64_t blksSkipped() const + { + return fNumBlksSkipped; + } - std::string udfName() const - { - return fUdfName; - }; - void udfName(const std::string& name) - { - fUdfName = name; - } + std::string udfName() const + { + return fUdfName; + }; + void udfName(const std::string& name) + { + fUdfName = name; + } - SP_LBIDList getlbidList() const - { - return lbidList; - } + SP_LBIDList getlbidList() const + { + return lbidList; + } - void addFilter(const execplan::Filter* f); - void appendFilter(const std::vector& fs); - std::vector& getFilters() - { - return fFilters; - } - const std::vector& getFilters() const - { - return fFilters; - } + void addFilter(const execplan::Filter* f); + void appendFilter(const std::vector& fs); + std::vector& getFilters() + { + return fFilters; + } + const std::vector& getFilters() const + { + return fFilters; + } -protected: - void addFilters(); + protected: + void addFilters(); -private: - //defaults okay? - //pColScanStep(const pColScanStep& rhs); - //pColScanStep& operator=(const pColScanStep& rhs); + private: + // defaults okay? + // pColScanStep(const pColScanStep& rhs); + // pColScanStep& operator=(const pColScanStep& rhs); - typedef boost::shared_ptr SPTHD; - void startPrimitiveThread(); - void startAggregationThread(); - void initializeConfigParms(); - void sendAPrimitiveMessage ( - ISMPacketHeader& ism, - BRM::LBID_t msgLbidStart, - uint32_t msgLbidCount); - uint64_t getFBO(uint64_t lbid); - bool isEmptyVal(const uint8_t* val8) const; + typedef boost::shared_ptr SPTHD; + void startPrimitiveThread(); + void startAggregationThread(); + void initializeConfigParms(); + void sendAPrimitiveMessage(ISMPacketHeader& ism, BRM::LBID_t msgLbidStart, uint32_t msgLbidCount); + uint64_t getFBO(uint64_t lbid); + bool isEmptyVal(const uint8_t* val8) const; - ResourceManager* fRm; - ColByScanRangeRequestHeader fMsgHeader; - SPTHD fConsumerThread; - /// number of threads on the receive side - uint32_t fNumThreads; + ResourceManager* fRm; + ColByScanRangeRequestHeader fMsgHeader; + SPTHD fConsumerThread; + /// number of threads on the receive side + uint32_t fNumThreads; - SPTHD* fProducerThread; - messageqcpp::ByteStream fFilterString; - uint32_t fFilterCount; - execplan::CalpontSystemCatalog::OID fOid; - execplan::CalpontSystemCatalog::OID fTableOid; - execplan::CalpontSystemCatalog::ColType fColType; - int8_t fBOP; - int8_t fOutputType; - uint32_t sentCount; - uint32_t recvCount; - BRM::LBIDRange_v lbidRanges; - BRM::DBRM dbrm; - SP_LBIDList lbidList; + SPTHD* fProducerThread; + messageqcpp::ByteStream fFilterString; + uint32_t fFilterCount; + execplan::CalpontSystemCatalog::OID fOid; + execplan::CalpontSystemCatalog::OID fTableOid; + execplan::CalpontSystemCatalog::ColType fColType; + int8_t fBOP; + int8_t fOutputType; + uint32_t sentCount; + uint32_t recvCount; + BRM::LBIDRange_v lbidRanges; + BRM::DBRM dbrm; + SP_LBIDList lbidList; - boost::mutex mutex; - boost::mutex dlMutex; - boost::mutex cpMutex; - boost::condition condvar; - boost::condition condvarWakeupProducer; - bool finishedSending, sendWaiting, rDoNothing, fIsDict; - uint32_t recvWaiting, recvExited; + boost::mutex mutex; + boost::mutex dlMutex; + boost::mutex cpMutex; + boost::condition condvar; + boost::condition condvarWakeupProducer; + bool finishedSending, sendWaiting, rDoNothing, fIsDict; + uint32_t recvWaiting, recvExited; - std::vector extents; - uint32_t extentSize, divShift, ridsPerBlock, rpbShift, numExtents; -// config::Config *fConfig; + std::vector extents; + uint32_t extentSize, divShift, ridsPerBlock, rpbShift, numExtents; + // config::Config *fConfig; - uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan - // request to primproc - uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before - // consumer will tell producer to send - // more LBID scan requests to primproc + uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan + // request to primproc + uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before + // consumer will tell producer to send + // more LBID scan requests to primproc - bool fStopSending; - bool fSingleThread; - bool isFilterFeeder; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count - uint64_t fNumBlksSkipped;//total number of block scans skipped due to CP - uint64_t fMsgBytesIn; // total byte count for incoming messages - uint64_t fMsgBytesOut; // total byte count for outcoming messages - uint32_t fMsgsToPm; // total number of messages sent to PMs - uint32_t uniqueID; - std::string fUdfName; + bool fStopSending; + bool fSingleThread; + bool isFilterFeeder; + uint64_t fPhysicalIO; // total physical I/O count + uint64_t fCacheIO; // total cache I/O count + uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP + uint64_t fMsgBytesIn; // total byte count for incoming messages + uint64_t fMsgBytesOut; // total byte count for outcoming messages + uint32_t fMsgsToPm; // total number of messages sent to PMs + uint32_t uniqueID; + std::string fUdfName; - //@bug 2634 - //@bug 3128 change ParseTree* to vector - std::vector fFilters; + //@bug 2634 + //@bug 3128 change ParseTree* to vector + std::vector fFilters; - friend class ColumnCommandJL; - friend class BatchPrimitiveProcessorJL; - friend class BucketReuseStep; - friend class BatchPrimitiveStep; - friend class TupleBPS; + friend class ColumnCommandJL; + friend class BatchPrimitiveProcessorJL; + friend class BucketReuseStep; + friend class BatchPrimitiveStep; + friend class TupleBPS; }; - #if 0 class pIdxStep : public JobStep { @@ -712,136 +698,132 @@ protected: */ class pDictionaryStep : public JobStep, public PrimitiveMsg { + public: + /** @brief pDictionaryStep constructor + */ -public: - /** @brief pDictionaryStep constructor - */ + pDictionaryStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tabelOid, + const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - pDictionaryStep( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tabelOid, - const execplan::CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo); + virtual ~pDictionaryStep(); - virtual ~pDictionaryStep(); + /** @brief virtual void Run method + */ + virtual void run(); + virtual void join(); + // void setOutList(StringDataList* rids); + void setInputList(DataList_t* rids); + void setBOP(int8_t b); + void sendPrimitiveMessages(); + void receivePrimitiveMessages(); - /** @brief virtual void Run method - */ - virtual void run(); - virtual void join(); - //void setOutList(StringDataList* rids); - void setInputList(DataList_t* rids); - void setBOP(int8_t b); - void sendPrimitiveMessages(); - void receivePrimitiveMessages(); + virtual const std::string toString() const; - virtual const std::string toString() const; + execplan::CalpontSystemCatalog::ColType& colType() + { + return fColType; + } + execplan::CalpontSystemCatalog::ColType colType() const + { + return fColType; + } - execplan::CalpontSystemCatalog::ColType& colType() - { - return fColType; - } - execplan::CalpontSystemCatalog::ColType colType() const - { - return fColType; - } + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return fOid; + } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } + virtual uint64_t phyIOCount() const + { + return fPhysicalIO; + } + virtual uint64_t cacheIOCount() const + { + return fCacheIO; + } + virtual uint64_t msgsRcvdCount() const + { + return msgsRecvd; + } + virtual uint64_t msgBytesIn() const + { + return fMsgBytesIn; + } + virtual uint64_t msgBytesOut() const + { + return fMsgBytesOut; + } + void addFilter(int8_t COP, const std::string& value); + uint32_t filterCount() const + { + return fFilterCount; + } + messageqcpp::ByteStream filterString() const + { + return fFilterString; + } - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return fOid; - } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } - virtual uint64_t phyIOCount () const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount () const - { - return fCacheIO; - } - virtual uint64_t msgsRcvdCount () const - { - return msgsRecvd; - } - virtual uint64_t msgBytesIn () const - { - return fMsgBytesIn; - } - virtual uint64_t msgBytesOut () const - { - return fMsgBytesOut; - } - void addFilter(int8_t COP, const std::string& value); - uint32_t filterCount() const - { - return fFilterCount; - } - messageqcpp::ByteStream filterString() const - { - return fFilterString; - } + // @bug3321, add filters into pDictionary + void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); + void addFilter(const execplan::Filter* f); + void appendFilter(const std::vector& fs); + std::vector& getFilters() + { + return fFilters; + } + int8_t BOP() const + { + return fBOP; + } - // @bug3321, add filters into pDictionary - void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); - void addFilter(const execplan::Filter* f); - void appendFilter(const std::vector& fs); - std::vector& getFilters() - { - return fFilters; - } - int8_t BOP() const - { - return fBOP; - } + private: + pDictionaryStep(); + void startPrimitiveThread(); + void startAggregationThread(); -private: - pDictionaryStep(); - void startPrimitiveThread(); - void startAggregationThread(); + boost::shared_ptr sysCat; + execplan::CalpontSystemCatalog::OID fOid; + execplan::CalpontSystemCatalog::OID fTableOid; + uint32_t fBOP; + uint32_t msgsSent; + uint32_t msgsRecvd; + uint32_t finishedSending; + uint32_t recvWaiting; + int64_t ridCount; + execplan::CalpontSystemCatalog::ColType fColType; + uint64_t pThread; // producer thread + uint64_t cThread; // producer thread - boost::shared_ptr sysCat; - execplan::CalpontSystemCatalog::OID fOid; - execplan::CalpontSystemCatalog::OID fTableOid; - uint32_t fBOP; - uint32_t msgsSent; - uint32_t msgsRecvd; - uint32_t finishedSending; - uint32_t recvWaiting; - int64_t ridCount; - execplan::CalpontSystemCatalog::ColType fColType; - uint64_t pThread; //producer thread - uint64_t cThread; //producer thread + messageqcpp::ByteStream fFilterString; + uint32_t fFilterCount; - messageqcpp::ByteStream fFilterString; - uint32_t fFilterCount; + DataList_t* requestList; + // StringDataList* stringList; + boost::mutex mutex; + boost::condition condvar; + uint32_t fInterval; + uint64_t fPhysicalIO; // total physical I/O count + uint64_t fCacheIO; // total cache I/O count + uint64_t fMsgBytesIn; // total byte count for incoming messages + uint64_t fMsgBytesOut; // total byte count for outcoming messages + uint32_t uniqueID; + ResourceManager* fRm; - DataList_t* requestList; - //StringDataList* stringList; - boost::mutex mutex; - boost::condition condvar; - uint32_t fInterval; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count - uint64_t fMsgBytesIn; // total byte count for incoming messages - uint64_t fMsgBytesOut; // total byte count for outcoming messages - uint32_t uniqueID; - ResourceManager* fRm; + //@bug 3128 change ParseTree* to vector + std::vector fFilters; - //@bug 3128 change ParseTree* to vector - std::vector fFilters; + bool hasEqualityFilter; + int8_t tmpCOP; + std::vector eqFilter; - bool hasEqualityFilter; - int8_t tmpCOP; - std::vector eqFilter; - - friend class DictStepJL; - friend class RTSCommandJL; - friend class BucketReuseStep; - friend class BatchPrimitiveStep; - friend class TupleBPS; + friend class DictStepJL; + friend class RTSCommandJL; + friend class BucketReuseStep; + friend class BatchPrimitiveStep; + friend class TupleBPS; }; /** @brief class pDictionaryScan @@ -849,249 +831,245 @@ private: */ class pDictionaryScan : public JobStep, public PrimitiveMsg { -public: + public: + /** @brief pDictionaryScan constructor + */ - /** @brief pDictionaryScan constructor - */ + pDictionaryScan(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, + const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - pDictionaryScan( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tableOid, - const execplan::CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo); + ~pDictionaryScan(); - ~pDictionaryScan(); + /** @brief virtual void Run method + */ + virtual void run(); + virtual void join(); + void setInputList(DataList_t* rids); + void setBOP(int8_t b); + void sendPrimitiveMessages(); + void receivePrimitiveMessages(); + void setSingleThread(); + virtual const std::string toString() const; - /** @brief virtual void Run method - */ - virtual void run(); - virtual void join(); - void setInputList(DataList_t* rids); - void setBOP(int8_t b); - void sendPrimitiveMessages(); - void receivePrimitiveMessages(); - void setSingleThread(); - virtual const std::string toString() const; + void setRidList(DataList* rids); - void setRidList(DataList* rids); + /** @brief Add a filter. Use this interface when the column stores anything but 4-byte floats. + * + * Add a filter. Use this interface when the column stores anything but 4-byte floats. + */ + void addFilter(int8_t COP, const std::string& value); // all but FLOATS can use this interface - /** @brief Add a filter. Use this interface when the column stores anything but 4-byte floats. - * - * Add a filter. Use this interface when the column stores anything but 4-byte floats. - */ - void addFilter(int8_t COP, const std::string& value); // all but FLOATS can use this interface + /** @brief Set the DistributedEngineComm object this instance should use + * + * Set the DistributedEngineComm object this instance should use + */ + void dec(DistributedEngineComm* dec) + { + if (fDec) + fDec->removeQueue(uniqueID); - /** @brief Set the DistributedEngineComm object this instance should use - * - * Set the DistributedEngineComm object this instance should use - */ - void dec(DistributedEngineComm* dec) - { - if (fDec) fDec->removeQueue(uniqueID); + fDec = dec; - fDec = dec; + if (fDec) + fDec->addQueue(uniqueID); + } - if (fDec) fDec->addQueue(uniqueID); - } + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return fOid; + } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return fOid; - } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } + uint64_t phyIOCount() const + { + return fPhysicalIO; + } + uint64_t cacheIOCount() const + { + return fCacheIO; + } + uint64_t msgsRcvdCount() const + { + return msgsRecvd; + } + uint64_t msgBytesIn() const + { + return fMsgBytesIn; + } + uint64_t msgBytesOut() const + { + return fMsgBytesOut; + } - uint64_t phyIOCount () const - { - return fPhysicalIO; - } - uint64_t cacheIOCount () const - { - return fCacheIO; - } - uint64_t msgsRcvdCount () const - { - return msgsRecvd; - } - uint64_t msgBytesIn () const - { - return fMsgBytesIn; - } - uint64_t msgBytesOut () const - { - return fMsgBytesOut; - } + BPSOutputType getOutputType() const + { + return fOutType; + } + void setOutputType(BPSOutputType ot) + { + fOutType = ot; + } + void setOutputRowGroup(const rowgroup::RowGroup& rg) + { + fOutputRowGroup = rg; + } + const rowgroup::RowGroup& getOutputRowGroup() const + { + return fOutputRowGroup; + } - BPSOutputType getOutputType() const - { - return fOutType; - } - void setOutputType(BPSOutputType ot) - { - fOutType = ot; - } - void setOutputRowGroup(const rowgroup::RowGroup& rg) - { - fOutputRowGroup = rg; - } - const rowgroup::RowGroup& getOutputRowGroup() const - { - return fOutputRowGroup; - } + // @bug3321, add interface for combining filters. + int8_t BOP() const + { + return fBOP; + } + void addFilter(const execplan::Filter* f); + void appendFilter(const std::vector& fs); + std::vector& getFilters() + { + return fFilters; + } + messageqcpp::ByteStream filterString() const + { + return fFilterString; + } + uint32_t filterCount() const + { + return fFilterCount; + } + void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); - // @bug3321, add interface for combining filters. - int8_t BOP() const - { - return fBOP; - } - void addFilter(const execplan::Filter* f); - void appendFilter(const std::vector& fs); - std::vector& getFilters() - { - return fFilters; - } - messageqcpp::ByteStream filterString() const - { - return fFilterString; - } - uint32_t filterCount() const - { - return fFilterCount; - } - void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); + virtual void abort(); - virtual void abort(); - - const execplan::CalpontSystemCatalog::ColType& colType() const - { - return fColType; - } + const execplan::CalpontSystemCatalog::ColType& colType() const + { + return fColType; + } -protected: - void sendError(uint16_t error); + protected: + void sendError(uint16_t error); -private: - pDictionaryScan(); - void startPrimitiveThread(); - void startAggregationThread(); - void initializeConfigParms(); - void sendAPrimitiveMessage( - messageqcpp::ByteStream& primMsg, - BRM::LBID_t msgLbidStart, - uint32_t msgLbidCount, uint16_t dbroot); - void formatMiniStats(); + private: + pDictionaryScan(); + void startPrimitiveThread(); + void startAggregationThread(); + void initializeConfigParms(); + void sendAPrimitiveMessage(messageqcpp::ByteStream& primMsg, BRM::LBID_t msgLbidStart, + uint32_t msgLbidCount, uint16_t dbroot); + void formatMiniStats(); - DistributedEngineComm* fDec; - boost::shared_ptr sysCat; - execplan::CalpontSystemCatalog::OID fOid; - execplan::CalpontSystemCatalog::OID fTableOid; - uint32_t fFilterCount; - uint32_t fBOP; - uint32_t fCOP1; - uint32_t fCOP2; - uint32_t msgsSent; - uint32_t msgsRecvd; - uint32_t finishedSending; - uint32_t recvWaiting; - uint32_t sendWaiting; - int64_t ridCount; - uint32_t fLogicalBlocksPerScan; - DataList* ridList; - messageqcpp::ByteStream fFilterString; - execplan::CalpontSystemCatalog::ColType fColType; - uint64_t pThread; //producer thread. thread pool handle - uint64_t cThread; //consumer thread. thread pool handle - DataList_t* requestList; - //StringDataList* stringList; - boost::mutex mutex; - boost::condition condvar; - boost::condition condvarWakeupProducer; - BRM::LBIDRange_v fDictlbids; - std::vector extents; - uint64_t extentSize; - uint64_t divShift; - uint64_t numExtents; - uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan - // request to primproc - uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before - // consumer will tell producer to send - bool fStopSending; - bool fSingleThread; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count - uint64_t fMsgBytesIn; // total byte count for incoming messages - uint64_t fMsgBytesOut; // total byte count for outcoming messages - uint32_t fMsgsToPm; // total number of messages sent to PMs - uint32_t fMsgsExpect; // total blocks to scan - uint32_t uniqueID; - ResourceManager* fRm; - BPSOutputType fOutType; - rowgroup::RowGroup fOutputRowGroup; - uint64_t fRidResults; + DistributedEngineComm* fDec; + boost::shared_ptr sysCat; + execplan::CalpontSystemCatalog::OID fOid; + execplan::CalpontSystemCatalog::OID fTableOid; + uint32_t fFilterCount; + uint32_t fBOP; + uint32_t fCOP1; + uint32_t fCOP2; + uint32_t msgsSent; + uint32_t msgsRecvd; + uint32_t finishedSending; + uint32_t recvWaiting; + uint32_t sendWaiting; + int64_t ridCount; + uint32_t fLogicalBlocksPerScan; + DataList* ridList; + messageqcpp::ByteStream fFilterString; + execplan::CalpontSystemCatalog::ColType fColType; + uint64_t pThread; // producer thread. thread pool handle + uint64_t cThread; // consumer thread. thread pool handle + DataList_t* requestList; + // StringDataList* stringList; + boost::mutex mutex; + boost::condition condvar; + boost::condition condvarWakeupProducer; + BRM::LBIDRange_v fDictlbids; + std::vector extents; + uint64_t extentSize; + uint64_t divShift; + uint64_t numExtents; + uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan + // request to primproc + uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before + // consumer will tell producer to send + bool fStopSending; + bool fSingleThread; + uint64_t fPhysicalIO; // total physical I/O count + uint64_t fCacheIO; // total cache I/O count + uint64_t fMsgBytesIn; // total byte count for incoming messages + uint64_t fMsgBytesOut; // total byte count for outcoming messages + uint32_t fMsgsToPm; // total number of messages sent to PMs + uint32_t fMsgsExpect; // total blocks to scan + uint32_t uniqueID; + ResourceManager* fRm; + BPSOutputType fOutType; + rowgroup::RowGroup fOutputRowGroup; + uint64_t fRidResults; - //@bug 2634 - //@bug 3128 change ParseTree* to vector - std::vector fFilters; + //@bug 2634 + //@bug 3128 change ParseTree* to vector + std::vector fFilters; - bool isEquality; - std::vector equalityFilter; - void serializeEqualityFilter(); - void destroyEqualityFilter(); + bool isEquality; + std::vector equalityFilter; + void serializeEqualityFilter(); + void destroyEqualityFilter(); }; - class BatchPrimitive : public JobStep, public PrimitiveMsg, public DECEventListener { -public: - - BatchPrimitive(const JobInfo& jobInfo) : JobStep(jobInfo) {} - virtual bool getFeederFlag() const = 0; - virtual uint64_t getLastTupleId() const = 0; - virtual uint32_t getStepCount () const = 0; - virtual void setBPP(JobStep* jobStep) = 0; - virtual void setFirstStepType(PrimitiveStepType firstStepType) = 0; - virtual void setIsProjectionOnly() = 0; - virtual void setLastTupleId(uint64_t) = 0; - virtual void setOutputType(BPSOutputType outputType) = 0; - virtual void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2) = 0; - virtual void setStepCount() = 0; - virtual void setSwallowRows(const bool swallowRows) = 0; - virtual void setBppStep() = 0; - virtual void dec(DistributedEngineComm* dec) = 0; - virtual const OIDVector& getProjectOids() const = 0; - virtual uint64_t blksSkipped() const = 0; - virtual bool wasStepRun() const = 0; - virtual BPSOutputType getOutputType() const = 0; - virtual uint64_t getRows() const = 0; - virtual void setJobInfo(const JobInfo* jobInfo) = 0; - virtual void setOutputRowGroup(const rowgroup::RowGroup& rg) = 0; - virtual const rowgroup::RowGroup& getOutputRowGroup() const = 0; - virtual void addFcnJoinExp(const std::vector& fe) = 0; - virtual void addFcnExpGroup1(const boost::shared_ptr& fe) = 0; - virtual void setFE1Input(const rowgroup::RowGroup& feInput) = 0; - virtual void setFcnExpGroup3(const std::vector& fe) = 0; - virtual void setFE23Output(const rowgroup::RowGroup& rg) = 0; + public: + BatchPrimitive(const JobInfo& jobInfo) : JobStep(jobInfo) + { + } + virtual bool getFeederFlag() const = 0; + virtual uint64_t getLastTupleId() const = 0; + virtual uint32_t getStepCount() const = 0; + virtual void setBPP(JobStep* jobStep) = 0; + virtual void setFirstStepType(PrimitiveStepType firstStepType) = 0; + virtual void setIsProjectionOnly() = 0; + virtual void setLastTupleId(uint64_t) = 0; + virtual void setOutputType(BPSOutputType outputType) = 0; + virtual void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2) = 0; + virtual void setStepCount() = 0; + virtual void setSwallowRows(const bool swallowRows) = 0; + virtual void setBppStep() = 0; + virtual void dec(DistributedEngineComm* dec) = 0; + virtual const OIDVector& getProjectOids() const = 0; + virtual uint64_t blksSkipped() const = 0; + virtual bool wasStepRun() const = 0; + virtual BPSOutputType getOutputType() const = 0; + virtual uint64_t getRows() const = 0; + virtual void setJobInfo(const JobInfo* jobInfo) = 0; + virtual void setOutputRowGroup(const rowgroup::RowGroup& rg) = 0; + virtual const rowgroup::RowGroup& getOutputRowGroup() const = 0; + virtual void addFcnJoinExp(const std::vector& fe) = 0; + virtual void addFcnExpGroup1(const boost::shared_ptr& fe) = 0; + virtual void setFE1Input(const rowgroup::RowGroup& feInput) = 0; + virtual void setFcnExpGroup3(const std::vector& fe) = 0; + virtual void setFE23Output(const rowgroup::RowGroup& rg) = 0; }; struct _CPInfo { - _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool val) : min(MIN), max(MAX), LBID(l), valid(val){}; - _CPInfo(int128_t BIGMIN, int128_t BIGMAX, uint64_t l, bool val) - : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val){}; - union - { - int128_t bigMin; - int64_t min; - }; - union - { - int128_t bigMax; - int64_t max; - }; - uint64_t LBID; - bool valid; + _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool val) : min(MIN), max(MAX), LBID(l), valid(val){}; + _CPInfo(int128_t BIGMIN, int128_t BIGMAX, uint64_t l, bool val) + : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val){}; + union + { + int128_t bigMin; + int64_t min; + }; + union + { + int128_t bigMax; + int64_t max; + }; + uint64_t LBID; + bool valid; }; /** @brief class TupleBPS @@ -1099,286 +1077,292 @@ struct _CPInfo */ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep { -public: - TupleBPS(const pColStep& rhs, const JobInfo& jobInfo); - TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo); - TupleBPS(const pDictionaryStep& rhs, const JobInfo& jobInfo); - TupleBPS(const pDictionaryScan& rhs, const JobInfo& jobInfo); - TupleBPS(const PassThruStep& rhs, const JobInfo& jobInfo); - virtual ~TupleBPS(); + public: + TupleBPS(const pColStep& rhs, const JobInfo& jobInfo); + TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo); + TupleBPS(const pDictionaryStep& rhs, const JobInfo& jobInfo); + TupleBPS(const pDictionaryScan& rhs, const JobInfo& jobInfo); + TupleBPS(const PassThruStep& rhs, const JobInfo& jobInfo); + virtual ~TupleBPS(); - /** @brief Starts processing. - * - * Starts processing. - */ - virtual void run(); - /** @brief Sync's the caller with the end of execution. - * - * Does nothing. Returns when this instance is finished. - */ - virtual void join(); + /** @brief Starts processing. + * + * Starts processing. + */ + virtual void run(); + /** @brief Sync's the caller with the end of execution. + * + * Does nothing. Returns when this instance is finished. + */ + virtual void join(); - void abort(); - void abort_nolock(); + void abort(); + void abort_nolock(); - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); + /** @brief The main loop for the send-side thread + * + * The main loop for the primitive-issuing thread. Don't call it directly. + */ + void sendPrimitiveMessages(); - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - void receiveMultiPrimitiveMessages(); + /** @brief The main loop for the recv-side thread + * + * The main loop for the receive-side thread. Don't call it directly. + */ + void receiveMultiPrimitiveMessages(); - // Processes the vector of `bytestream` starting from `begin` index to the `end` index, non inclusive. - void processByteStreamVector(vector>& bsv, - const uint32_t begin, const uint32_t end, vector<_CPInfo>& cpv, - RowGroupDL* dlp, const uint32_t threadID); + // Processes the vector of `bytestream` starting from `begin` index to the `end` index, non inclusive. + void processByteStreamVector(vector>& bsv, const uint32_t begin, + const uint32_t end, vector<_CPInfo>& cpv, RowGroupDL* dlp, + const uint32_t threadID); - /** @brief Add a filter when the column is anything but a 4-byte float type. - * - * Add a filter when the column is anything but a 4-byte float type, including - * 8-byte doubles. - */ - void setBPP(JobStep* jobStep); - void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2); - bool scanit(uint64_t rid); - void storeCasualPartitionInfo(const bool estimateRowCounts); - bool getFeederFlag() const - { - return isFilterFeeder; - } - void setFeederFlag (bool filterFeeder) - { - isFilterFeeder = filterFeeder; - } - void setSwallowRows(const bool swallowRows) - { - fSwallowRows = swallowRows; - } - bool getSwallowRows() const - { - return fSwallowRows; - } + /** @brief Add a filter when the column is anything but a 4-byte float type. + * + * Add a filter when the column is anything but a 4-byte float type, including + * 8-byte doubles. + */ + void setBPP(JobStep* jobStep); + void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2); + bool scanit(uint64_t rid); + void storeCasualPartitionInfo(const bool estimateRowCounts); + bool getFeederFlag() const + { + return isFilterFeeder; + } + void setFeederFlag(bool filterFeeder) + { + isFilterFeeder = filterFeeder; + } + void setSwallowRows(const bool swallowRows) + { + fSwallowRows = swallowRows; + } + bool getSwallowRows() const + { + return fSwallowRows; + } - /* Base class interface fcn that can go away */ - void setOutputType(BPSOutputType) { } //Can't change the ot of a TupleBPS - BPSOutputType getOutputType() const - { - return ROW_GROUP; - } - void setBppStep() { } - void setIsProjectionOnly() { } + /* Base class interface fcn that can go away */ + void setOutputType(BPSOutputType) + { + } // Can't change the ot of a TupleBPS + BPSOutputType getOutputType() const + { + return ROW_GROUP; + } + void setBppStep() + { + } + void setIsProjectionOnly() + { + } - uint64_t getRows() const - { - return ridsReturned; - } - void setFirstStepType(PrimitiveStepType firstStepType) - { - ffirstStepType = firstStepType; - } - PrimitiveStepType getPrimitiveStepType () - { - return ffirstStepType; - } - void setStepCount() - { - fStepCount++; - } - uint32_t getStepCount () const - { - return fStepCount; - } - void setLastTupleId(uint64_t id) - { - fLastTupleId = id; - } - uint64_t getLastTupleId() const - { - return fLastTupleId; - } + uint64_t getRows() const + { + return ridsReturned; + } + void setFirstStepType(PrimitiveStepType firstStepType) + { + ffirstStepType = firstStepType; + } + PrimitiveStepType getPrimitiveStepType() + { + return ffirstStepType; + } + void setStepCount() + { + fStepCount++; + } + uint32_t getStepCount() const + { + return fStepCount; + } + void setLastTupleId(uint64_t id) + { + fLastTupleId = id; + } + uint64_t getLastTupleId() const + { + return fLastTupleId; + } - /** @brief Set the DistributedEngineComm object this instance should use - * - * Set the DistributedEngineComm object this instance should use - */ - void dec(DistributedEngineComm* dec); + /** @brief Set the DistributedEngineComm object this instance should use + * + * Set the DistributedEngineComm object this instance should use + */ + void dec(DistributedEngineComm* dec); - virtual void stepId(uint16_t stepId); - virtual uint16_t stepId() const - { - return fStepId; - } - virtual const std::string toString() const; + virtual void stepId(uint16_t stepId); + virtual uint16_t stepId() const + { + return fStepId; + } + virtual const std::string toString() const; - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return fOid; - } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } - const execplan::CalpontSystemCatalog::ColType& colType() const - { - return fColType; - } - const OIDVector& getProjectOids() const - { - return projectOids; - } - virtual uint64_t phyIOCount () const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount () const - { - return fCacheIO; - } - virtual uint64_t msgsRcvdCount () const - { - return msgsRecvd; - } - virtual uint64_t msgBytesIn () const - { - return fMsgBytesIn; - } - virtual uint64_t msgBytesOut () const - { - return fMsgBytesOut; - } - virtual uint64_t blockTouched () const - { - return fBlockTouched; - } - uint32_t nextBand(messageqcpp::ByteStream& bs); + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return fOid; + } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } + const execplan::CalpontSystemCatalog::ColType& colType() const + { + return fColType; + } + const OIDVector& getProjectOids() const + { + return projectOids; + } + virtual uint64_t phyIOCount() const + { + return fPhysicalIO; + } + virtual uint64_t cacheIOCount() const + { + return fCacheIO; + } + virtual uint64_t msgsRcvdCount() const + { + return msgsRecvd; + } + virtual uint64_t msgBytesIn() const + { + return fMsgBytesIn; + } + virtual uint64_t msgBytesOut() const + { + return fMsgBytesOut; + } + virtual uint64_t blockTouched() const + { + return fBlockTouched; + } + uint32_t nextBand(messageqcpp::ByteStream& bs); - //...Currently only supported by pColStep and pColScanStep, so didn't bother - //...to define abstract method in base class, but if start adding to other - //...classes, then should consider adding pure virtual method to JobStep. - uint64_t blksSkipped () const - { - return fNumBlksSkipped; - } + //...Currently only supported by pColStep and pColScanStep, so didn't bother + //...to define abstract method in base class, but if start adding to other + //...classes, then should consider adding pure virtual method to JobStep. + uint64_t blksSkipped() const + { + return fNumBlksSkipped; + } - uint32_t getUniqueID() - { - return uniqueID; - } - void useJoiner(boost::shared_ptr); - void useJoiners(const std::vector >&); - bool wasStepRun() const - { - return fRunExecuted; - } + uint32_t getUniqueID() + { + return uniqueID; + } + void useJoiner(boost::shared_ptr); + void useJoiners(const std::vector>&); + bool wasStepRun() const + { + return fRunExecuted; + } - // DEC event listener interface - void newPMOnline(uint32_t connectionNumber); + // DEC event listener interface + void newPMOnline(uint32_t connectionNumber); - void setInputRowGroup(const rowgroup::RowGroup& rg); - void setOutputRowGroup(const rowgroup::RowGroup& rg); - const rowgroup::RowGroup& getOutputRowGroup() const; + void setInputRowGroup(const rowgroup::RowGroup& rg); + void setOutputRowGroup(const rowgroup::RowGroup& rg); + const rowgroup::RowGroup& getOutputRowGroup() const; - void setAggregateStep(const rowgroup::SP_ROWAGG_PM_t& agg, const rowgroup::RowGroup& rg); + void setAggregateStep(const rowgroup::SP_ROWAGG_PM_t& agg, const rowgroup::RowGroup& rg); - /* This is called by TupleHashJoin only */ - void setJoinedResultRG(const rowgroup::RowGroup& rg); + /* This is called by TupleHashJoin only */ + void setJoinedResultRG(const rowgroup::RowGroup& rg); - /* OR hacks */ - void setBOP(uint8_t op); // BOP_AND or BOP_OR - uint8_t getBOP() - { - return bop; - } + /* OR hacks */ + void setBOP(uint8_t op); // BOP_AND or BOP_OR + uint8_t getBOP() + { + return bop; + } - void setJobInfo(const JobInfo* jobInfo); + void setJobInfo(const JobInfo* jobInfo); - // @bug 2123. Added getEstimatedRowCount function. - /* @brief estimates the number of rows that will be returned for use in determining the - * large side table for hashjoins. - */ - uint64_t getEstimatedRowCount(); + // @bug 2123. Added getEstimatedRowCount function. + /* @brief estimates the number of rows that will be returned for use in determining the + * large side table for hashjoins. + */ + uint64_t getEstimatedRowCount(); - /* Functions & Expressions support. - Group 1 is for single-table filters only at the moment. Group 1 objects - are registered by JLF on the TBPS object directly because there is no join - involved. + /* Functions & Expressions support. + Group 1 is for single-table filters only at the moment. Group 1 objects + are registered by JLF on the TBPS object directly because there is no join + involved. - Group 2 is for cross-table filters only and should be registered on the - join instance by the JLF. When the query starts running, the join object - decides whether the Group 2 instance should run on the PM and UM, then - registers it with the TBPS. + Group 2 is for cross-table filters only and should be registered on the + join instance by the JLF. When the query starts running, the join object + decides whether the Group 2 instance should run on the PM and UM, then + registers it with the TBPS. - Group 3 is for selected columns whether or not its calculation is single- - table or cross-table. If it's single-table and there's no join instance, - JLF should register that object with the TBPS for that table. If it's - cross-table, then JLF should register it with the join step. - */ - void addFcnJoinExp(const std::vector& fe); - void addFcnExpGroup1(const boost::shared_ptr& fe); - void setFE1Input(const rowgroup::RowGroup& feInput); + Group 3 is for selected columns whether or not its calculation is single- + table or cross-table. If it's single-table and there's no join instance, + JLF should register that object with the TBPS for that table. If it's + cross-table, then JLF should register it with the join step. + */ + void addFcnJoinExp(const std::vector& fe); + void addFcnExpGroup1(const boost::shared_ptr& fe); + void setFE1Input(const rowgroup::RowGroup& feInput); - /* for use by the THJS only... */ - void setFcnExpGroup2(const boost::shared_ptr& fe2, - const rowgroup::RowGroup& output, bool runFE2onPM); + /* for use by the THJS only... */ + void setFcnExpGroup2(const boost::shared_ptr& fe2, + const rowgroup::RowGroup& output, bool runFE2onPM); - /* Functions & Expressions in select and groupby clause. - JLF should use these only if there isn't a join. If there is, call the - equivalent fcn on THJS instead */ - void setFcnExpGroup3(const std::vector& fe); - void setFE23Output(const rowgroup::RowGroup& rg); - bool hasFcnExpGroup3() - { - return (fe2 != NULL); - } + /* Functions & Expressions in select and groupby clause. + JLF should use these only if there isn't a join. If there is, call the + equivalent fcn on THJS instead */ + void setFcnExpGroup3(const std::vector& fe); + void setFE23Output(const rowgroup::RowGroup& rg); + bool hasFcnExpGroup3() + { + return (fe2 != NULL); + } - // rowgroup to connector - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + // rowgroup to connector + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; - /* Interface for adding add'l predicates for casual partitioning. - * This fcn checks for any intersection between the values in vals - * and the range of a given extent. If there is no intersection, that extent - * won't be processed. For every extent in OID, it effectively calculates - * ((vals[0] >= min && vals[0] <= max) || (vals[1] >= min && vals[1] <= max)) ... - * && (previous calculation for that extent). - * Note that it is an adder not a setter. For an extent to be scanned, all calls - * must have a non-empty intersection. - */ - void addCPPredicates(uint32_t OID, const std::vector& vals, bool isRange, - bool isSmallSideWideDecimal); + /* Interface for adding add'l predicates for casual partitioning. + * This fcn checks for any intersection between the values in vals + * and the range of a given extent. If there is no intersection, that extent + * won't be processed. For every extent in OID, it effectively calculates + * ((vals[0] >= min && vals[0] <= max) || (vals[1] >= min && vals[1] <= max)) ... + * && (previous calculation for that extent). + * Note that it is an adder not a setter. For an extent to be scanned, all calls + * must have a non-empty intersection. + */ + void addCPPredicates(uint32_t OID, const std::vector& vals, bool isRange, + bool isSmallSideWideDecimal); - /* semijoin adds */ - void setJoinFERG(const rowgroup::RowGroup& rg); + /* semijoin adds */ + void setJoinFERG(const rowgroup::RowGroup& rg); - /* To cover over the race between creating extents in each column. Mitigates - * bug 3607.*/ - bool goodExtentCount(); - void reloadExtentLists(); - void initExtentMarkers(); // need a better name for this + /* To cover over the race between creating extents in each column. Mitigates + * bug 3607.*/ + bool goodExtentCount(); + void reloadExtentLists(); + void initExtentMarkers(); // need a better name for this - virtual bool stringTableFriendly() - { - return true; - } - ResourceManager* resourceManager() const - { - return fRm; - } - bool runFEonPM() const - { - return bRunFEonPM; - } - -protected: - void sendError(uint16_t status); + virtual bool stringTableFriendly() + { + return true; + } + ResourceManager* resourceManager() const + { + return fRm; + } + bool runFEonPM() const + { + return bRunFEonPM; + } -private: + protected: + void sendError(uint16_t status); + + private: void formatMiniStats(); void startPrimitiveThread(); @@ -1399,14 +1383,14 @@ private: uint16_t fNumSteps; int fColWidth; uint32_t fStepCount; - bool fCPEvaluated; // @bug 2123 - uint64_t fEstimatedRows; // @bug 2123 + bool fCPEvaluated; // @bug 2123 + uint64_t fEstimatedRows; // @bug 2123 /// number of threads on the receive side uint32_t fMaxNumThreads; uint32_t fNumThreads; PrimitiveStepType ffirstStepType; bool isFilterFeeder; - std::vector fProducerThreads; // thread pool handles + std::vector fProducerThreads; // thread pool handles std::vector fProcessorThreads; messageqcpp::ByteStream fFilterString; uint32_t fFilterCount; @@ -1429,30 +1413,31 @@ private: uint32_t recvWaiting; uint32_t recvExited; uint64_t ridsReturned; - std::map> extentsMap; + std::map> + extentsMap; std::vector scannedExtents; OIDVector projectOids; uint32_t extentSize, divShift, rpbShift, numExtents, modMask; - uint32_t fRequestSize; // the number of logical extents per batch of requests sent to PrimProc. - uint32_t fProcessorThreadsPerScan; // The number of messages sent per logical extent. + uint32_t fRequestSize; // the number of logical extents per batch of requests sent to PrimProc. + uint32_t fProcessorThreadsPerScan; // The number of messages sent per logical extent. bool fSwallowRows; - uint32_t fMaxOutstandingRequests; // The number of logical extents have not processed by PrimProc - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count - uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP - uint64_t fMsgBytesIn; // total byte count for incoming messages - uint64_t fMsgBytesOut; // total byte count for outcoming messages - uint64_t fBlockTouched; // total blocks touched - uint32_t fExtentsPerSegFile; // config num of Extents Per Segment File + uint32_t fMaxOutstandingRequests; // The number of logical extents have not processed by PrimProc + uint64_t fPhysicalIO; // total physical I/O count + uint64_t fCacheIO; // total cache I/O count + uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP + uint64_t fMsgBytesIn; // total byte count for incoming messages + uint64_t fMsgBytesOut; // total byte count for outcoming messages + uint64_t fBlockTouched; // total blocks touched + uint32_t fExtentsPerSegFile; // config num of Extents Per Segment File // uint64_t cThread; //consumer thread. thread handle from thread pool - uint64_t pThread; // producer thread. thread handle from thread pool + uint64_t pThread; // producer thread. thread handle from thread pool boost::mutex tplMutex; boost::mutex dlMutex; boost::mutex cpMutex; boost::mutex serializeJoinerMutex; boost::condition condvarWakeupProducer, condvar; - std::vector scanFlags; // use to keep track of which extents to eliminate from this step + std::vector scanFlags; // use to keep track of which extents to eliminate from this step bool BPPIsAllocated; uint32_t uniqueID; ResourceManager* fRm; @@ -1464,22 +1449,22 @@ private: std::vector> tjoiners; bool doJoin, hasPMJoin, hasUMJoin; - std::vector joinerMatchesRGs; // parses the small-side matches from joiner + std::vector joinerMatchesRGs; // parses the small-side matches from joiner uint32_t smallSideCount; int smallOuterJoiner; - bool fRunExecuted; // was the run method executed for this step - rowgroup::RowGroup inputRowGroup; // for parsing the data read from the datalist - rowgroup::RowGroup primRowGroup; // for parsing the data received from the PM - rowgroup::RowGroup outputRowGroup; // if there's a join, these are the joined + bool fRunExecuted; // was the run method executed for this step + rowgroup::RowGroup inputRowGroup; // for parsing the data read from the datalist + rowgroup::RowGroup primRowGroup; // for parsing the data received from the PM + rowgroup::RowGroup outputRowGroup; // if there's a join, these are the joined // result, otherwise it's = to primRowGroup // aggregation support rowgroup::SP_ROWAGG_PM_t fAggregatorPm; rowgroup::RowGroup fAggRowGroupPm; // OR hacks - uint8_t bop; // BOP_AND or BOP_OR + uint8_t bop; // BOP_AND or BOP_OR // temporary hack to make sure JobList only calls run and join once boost::mutex jlLock; @@ -1514,70 +1499,69 @@ private: class JoinLocalData { - public: - JoinLocalData() = delete; - JoinLocalData(const JoinLocalData&) = delete; - JoinLocalData(JoinLocalData&&) = delete; - JoinLocalData& operator=(const JoinLocalData&) = delete; - JoinLocalData& operator=(JoinLocalData&&) = delete; - ~JoinLocalData() = default; + public: + JoinLocalData() = delete; + JoinLocalData(const JoinLocalData&) = delete; + JoinLocalData(JoinLocalData&&) = delete; + JoinLocalData& operator=(const JoinLocalData&) = delete; + JoinLocalData& operator=(JoinLocalData&&) = delete; + ~JoinLocalData() = default; - JoinLocalData(TupleBPS* pTupleBPS, rowgroup::RowGroup& primRowGroup, rowgroup::RowGroup& outputRowGroup, - boost::shared_ptr& fe2, rowgroup::RowGroup& fe2Output, - std::vector& joinerMatchesRGs, rowgroup::RowGroup& joinFERG, - std::vector>& tjoiners, uint32_t smallSideCount, - bool doJoin); + JoinLocalData(TupleBPS* pTupleBPS, rowgroup::RowGroup& primRowGroup, rowgroup::RowGroup& outputRowGroup, + boost::shared_ptr& fe2, rowgroup::RowGroup& fe2Output, + std::vector& joinerMatchesRGs, rowgroup::RowGroup& joinFERG, + std::vector>& tjoiners, uint32_t smallSideCount, + bool doJoin); - friend class TupleBPS; + friend class TupleBPS; - private: - uint64_t generateJoinResultSet(const uint32_t depth, - std::vector& outputData, - RowGroupDL* dlp); - void processFE2(vector& rgData); + private: + uint64_t generateJoinResultSet(const uint32_t depth, std::vector& outputData, + RowGroupDL* dlp); + void processFE2(vector& rgData); - TupleBPS* tbps; // Parent - rowgroup::RowGroup local_primRG; - rowgroup::RowGroup local_outputRG; + TupleBPS* tbps; // Parent + rowgroup::RowGroup local_primRG; + rowgroup::RowGroup local_outputRG; - uint32_t cachedIO_Thread = 0; - uint32_t physIO_Thread = 0; - uint32_t touchedBlocks_Thread = 0; - int64_t ridsReturned_Thread = 0; + uint32_t cachedIO_Thread = 0; + uint32_t physIO_Thread = 0; + uint32_t touchedBlocks_Thread = 0; + int64_t ridsReturned_Thread = 0; - // On init. - boost::shared_ptr fe2; - rowgroup::RowGroup fe2Output; - std::vector joinerMatchesRGs; - rowgroup::RowGroup joinFERG; - std::vector> tjoiners; + // On init. + boost::shared_ptr fe2; + rowgroup::RowGroup fe2Output; + std::vector joinerMatchesRGs; + rowgroup::RowGroup joinFERG; + std::vector> tjoiners; - uint32_t smallSideCount; - bool doJoin; + uint32_t smallSideCount; + bool doJoin; - // Join vars. - vector> joinerOutput; - rowgroup::Row largeSideRow; - rowgroup::Row joinedBaseRow; - rowgroup::Row largeNull; - rowgroup::Row joinFERow; // LSR clean - boost::scoped_array smallSideRows; - boost::scoped_array smallNulls; - boost::scoped_array joinedBaseRowData; - boost::scoped_array joinFERowData; - boost::shared_array largeMapping; - vector> smallMappings; - vector> fergMappings; - rowgroup::RGData joinedData; - boost::scoped_array largeNullMemory; - boost::scoped_array> smallNullMemory; - uint32_t matchCount; - - rowgroup::Row postJoinRow; - rowgroup::RowGroup local_fe2Output; - rowgroup::RGData local_fe2Data; - rowgroup::Row local_fe2OutRow; - funcexp::FuncExpWrapper local_fe2; + // Join vars. + vector> joinerOutput; + rowgroup::Row largeSideRow; + rowgroup::Row joinedBaseRow; + rowgroup::Row largeNull; + rowgroup::Row joinFERow; // LSR clean + boost::scoped_array smallSideRows; + boost::scoped_array smallNulls; + boost::scoped_array joinedBaseRowData; + boost::scoped_array joinFERowData; + boost::shared_array largeMapping; + vector> smallMappings; + vector> fergMappings; + rowgroup::RGData joinedData; + boost::scoped_array largeNullMemory; + boost::scoped_array> smallNullMemory; + uint32_t matchCount; + + rowgroup::Row postJoinRow; + rowgroup::RowGroup local_fe2Output; + rowgroup::RGData local_fe2Data; + rowgroup::Row local_fe2OutRow; + funcexp::FuncExpWrapper local_fe2; }; // We will initialize the actual value in TupleBPS `initializeConfigParms` function. @@ -1588,21 +1572,21 @@ private: std::shared_ptr getJoinLocalDataByIndex(uint32_t index) { - idbassert(index < fNumProcessorThreads && joinLocalDataPool.size() == fNumProcessorThreads); - return joinLocalDataPool[index]; + idbassert(index < fNumProcessorThreads && joinLocalDataPool.size() == fNumProcessorThreads); + return joinLocalDataPool[index]; } void initializeJoinLocalDataPool(uint32_t numThreads) { - idbassert(numThreads <= fMaxNumProcessorThreads); - for (uint32_t i = 0; i < numThreads; ++i) - { - joinLocalDataPool.push_back(std::shared_ptr( - new JoinLocalData(this, primRowGroup, outputRowGroup, fe2, fe2Output, joinerMatchesRGs, joinFERG, - tjoiners, smallSideCount, doJoin))); - } + idbassert(numThreads <= fMaxNumProcessorThreads); + for (uint32_t i = 0; i < numThreads; ++i) + { + joinLocalDataPool.push_back(std::shared_ptr( + new JoinLocalData(this, primRowGroup, outputRowGroup, fe2, fe2Output, joinerMatchesRGs, joinFERG, + tjoiners, smallSideCount, doJoin))); + } - fNumProcessorThreads = numThreads; + fNumProcessorThreads = numThreads; } // Join local data vector. @@ -1611,108 +1595,107 @@ private: /* shared nothing support */ struct Job { - Job(uint32_t d, uint32_t n, uint32_t b, boost::shared_ptr& bs) - : dbroot(d), connectionNum(n), expectedResponses(b), msg(bs) - { - } - uint32_t dbroot; - uint32_t connectionNum; - uint32_t expectedResponses; - boost::shared_ptr msg; - }; + Job(uint32_t d, uint32_t n, uint32_t b, boost::shared_ptr& bs) + : dbroot(d), connectionNum(n), expectedResponses(b), msg(bs) + { + } + uint32_t dbroot; + uint32_t connectionNum; + uint32_t expectedResponses; + boost::shared_ptr msg; + }; - void prepCasualPartitioning(); - void makeJobs(std::vector* jobs); - void interleaveJobs(std::vector* jobs) const; - void sendJobs(const std::vector& jobs); - uint32_t numDBRoots; + void prepCasualPartitioning(); + void makeJobs(std::vector* jobs); + void interleaveJobs(std::vector* jobs) const; + void sendJobs(const std::vector& jobs); + uint32_t numDBRoots; - /* Pseudo column filter processing. Think about refactoring into a separate class. */ - bool processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr > dbRootPMMap) const; - template - bool processOneFilterType(int8_t colWidth, T value, uint32_t type) const; - template - bool processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, - uint32_t filterCount) const; - bool processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int64_t min, int64_t max, - const uint8_t* filterString, uint32_t filterCount) const; - bool processLBIDFilter(const BRM::EMEntry& emEntry) const; - template - bool compareSingleValue(uint8_t COP, T val1, T val2) const; - bool compareRange(uint8_t COP, int64_t min, int64_t max, int64_t val) const; - bool hasPCFilter, hasPMFilter, hasRIDFilter, hasSegmentFilter, hasDBRootFilter, hasSegmentDirFilter, - hasPartitionFilter, hasMaxFilter, hasMinFilter, hasLBIDFilter, hasExtentIDFilter; - }; + /* Pseudo column filter processing. Think about refactoring into a separate class. */ + bool processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr> dbRootPMMap) const; + template + bool processOneFilterType(int8_t colWidth, T value, uint32_t type) const; + template + bool processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, + uint32_t filterCount) const; + bool processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int64_t min, int64_t max, + const uint8_t* filterString, uint32_t filterCount) const; + bool processLBIDFilter(const BRM::EMEntry& emEntry) const; + template + bool compareSingleValue(uint8_t COP, T val1, T val2) const; + bool compareRange(uint8_t COP, int64_t min, int64_t max, int64_t val) const; + bool hasPCFilter, hasPMFilter, hasRIDFilter, hasSegmentFilter, hasDBRootFilter, hasSegmentDirFilter, + hasPartitionFilter, hasMaxFilter, hasMinFilter, hasLBIDFilter, hasExtentIDFilter; +}; /** @brief class FilterStep * */ class FilterStep : public JobStep { -public: + public: + FilterStep(const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo); + ~FilterStep(); - FilterStep(const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo); - ~FilterStep(); + /** @brief virtual void Run method + */ + void run(); + void join(); - /** @brief virtual void Run method - */ - void run(); - void join(); + const std::string toString() const; - const std::string toString() const; + execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOID; + } + void tableOid(execplan::CalpontSystemCatalog::OID tableOid) + { + fTableOID = tableOid; + } + const execplan::CalpontSystemCatalog::ColType& colType() const + { + return fColType; + } + void setBOP(int8_t b); + int8_t BOP() const + { + return fBOP; + } + friend struct FSRunner; - execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOID; - } - void tableOid(execplan::CalpontSystemCatalog::OID tableOid) - { - fTableOID = tableOid; - } - const execplan::CalpontSystemCatalog::ColType& colType() const - { - return fColType; - } - void setBOP(int8_t b); - int8_t BOP() const - { - return fBOP; - } - friend struct FSRunner; + void addFilter(const execplan::Filter* f); + std::vector& getFilters() + { + return fFilters; + } - void addFilter(const execplan::Filter* f); - std::vector& getFilters() - { - return fFilters; - } + protected: + // void unblockDataLists(FifoDataList* fifo, StringFifoDataList* strFifo, StrDataList* strResult, + //DataList_t* result); -protected: -// void unblockDataLists(FifoDataList* fifo, StringFifoDataList* strFifo, StrDataList* strResult, DataList_t* result); + private: + // This i/f is not meaningful in this step + execplan::CalpontSystemCatalog::OID oid() const + { + return 0; + } + void doFilter(); // @bug 686 -private: + // config::Config *fConfig; - //This i/f is not meaningful in this step - execplan::CalpontSystemCatalog::OID oid() const - { - return 0; - } - void doFilter(); // @bug 686 + execplan::CalpontSystemCatalog::OID fTableOID; + execplan::CalpontSystemCatalog::ColType fColType; + int8_t fBOP; + // int64_t runner; // thread handle from thread pool -// config::Config *fConfig; + // @bug 687 Add data and friend declarations for concurrent filter steps. + std::vector fSortedA; // used to internally sort input data + std::vector fSortedB; + // FifoDataList* fFAp; // Used to internally pass data to + // FifoDataList* fFBp; // FilterOperation thread. + uint64_t resultCount; - execplan::CalpontSystemCatalog::OID fTableOID; - execplan::CalpontSystemCatalog::ColType fColType; - int8_t fBOP; - // int64_t runner; // thread handle from thread pool - - // @bug 687 Add data and friend declarations for concurrent filter steps. - std::vector fSortedA; // used to internally sort input data - std::vector fSortedB; -// FifoDataList* fFAp; // Used to internally pass data to -// FifoDataList* fFBp; // FilterOperation thread. - uint64_t resultCount; - - std::vector fFilters; + std::vector fFilters; }; /** @brief class PassThruStep @@ -1720,155 +1703,143 @@ private: */ class PassThruStep : public JobStep, public PrimitiveMsg { + typedef std::pair element_t; - typedef std::pair element_t; + public: + /** @brief PassThruStep constructor + */ + PassThruStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, + const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo); -public: - /** @brief PassThruStep constructor - */ - PassThruStep( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tableOid, - const execplan::CalpontSystemCatalog::ColType& colType, - const JobInfo& jobInfo); + PassThruStep(const pColStep& rhs); + PassThruStep(const PseudoColStep& rhs); - PassThruStep(const pColStep& rhs); - PassThruStep(const PseudoColStep& rhs); + virtual ~PassThruStep(); - virtual ~PassThruStep(); + /** @brief Starts processing. Set at least the RID list before calling. + * + * Starts processing. Set at least the RID list before calling this. + */ + virtual void run(); - /** @brief Starts processing. Set at least the RID list before calling. - * - * Starts processing. Set at least the RID list before calling this. - */ - virtual void run(); + /** @brief Sync's the caller with the end of execution. + * + * Does nothing. Returns when this instance is finished. + */ + virtual void join(); - /** @brief Sync's the caller with the end of execution. - * - * Does nothing. Returns when this instance is finished. - */ - virtual void join(); + virtual const std::string toString() const; - virtual const std::string toString() const; + virtual execplan::CalpontSystemCatalog::OID oid() const + { + return fOid; + } - virtual execplan::CalpontSystemCatalog::OID oid() const - { - return fOid; - } + virtual execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } - virtual execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } + uint8_t getColWidth() const + { + return colWidth; + } + bool isDictCol() const + { + return isDictColumn; + } + bool isExeMgr() const + { + return isEM; + } + const execplan::CalpontSystemCatalog::ColType& colType() const + { + return fColType; + } + ResourceManager* resourceManager() const + { + return fRm; + } - uint8_t getColWidth() const - { - return colWidth; - } - bool isDictCol() const - { - return isDictColumn; - } - bool isExeMgr() const - { - return isEM; - } - const execplan::CalpontSystemCatalog::ColType& colType() const - { - return fColType; - } - ResourceManager* resourceManager() const - { - return fRm; - } + void pseudoType(uint32_t p) + { + fPseudoType = p; + } + uint32_t pseudoType() const + { + return fPseudoType; + } - void pseudoType(uint32_t p) - { - fPseudoType = p; - } - uint32_t pseudoType() const - { - return fPseudoType; - } + protected: + private: + /** @brief constructor for completeness + */ + explicit PassThruStep(); -protected: + uint64_t getLBID(uint64_t rid, bool& scan); + uint64_t getFBO(uint64_t lbid); -private: + boost::shared_ptr catalog; + execplan::CalpontSystemCatalog::OID fOid; + execplan::CalpontSystemCatalog::OID fTableOid; + uint8_t colWidth; + uint16_t realWidth; + uint32_t fPseudoType; + execplan::CalpontSystemCatalog::ColType fColType; + bool isDictColumn; + bool isEM; - /** @brief constructor for completeness - */ - explicit PassThruStep(); + // boost::thread* fPTThd; - uint64_t getLBID(uint64_t rid, bool& scan); - uint64_t getFBO(uint64_t lbid); - - boost::shared_ptr catalog; - execplan::CalpontSystemCatalog::OID fOid; - execplan::CalpontSystemCatalog::OID fTableOid; - uint8_t colWidth; - uint16_t realWidth; - uint32_t fPseudoType; - execplan::CalpontSystemCatalog::ColType fColType; - bool isDictColumn; - bool isEM; - -// boost::thread* fPTThd; - - // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is TRACE_FLAGS::TRACE_NO_ROWS4. - // Running with this one will swallow rows at projection. - bool fSwallowRows; - ResourceManager* fRm; - friend class PassThruCommandJL; - friend class RTSCommandJL; - friend class BatchPrimitiveStep; - friend class TupleBPS; + // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is TRACE_FLAGS::TRACE_NO_ROWS4. + // Running with this one will swallow rows at projection. + bool fSwallowRows; + ResourceManager* fRm; + friend class PassThruCommandJL; + friend class RTSCommandJL; + friend class BatchPrimitiveStep; + friend class TupleBPS; }; class PseudoColStep : public pColStep { -public: - /** @brief PseudoColStep constructor - */ - PseudoColStep( - execplan::CalpontSystemCatalog::OID oid, - execplan::CalpontSystemCatalog::OID tableOid, - uint32_t pId, - const execplan::CalpontSystemCatalog::ColType& ct, - const JobInfo& jobInfo) : - pColStep(oid, tableOid, ct, jobInfo), - fPseudoColumnId(pId) - {} + public: + /** @brief PseudoColStep constructor + */ + PseudoColStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, + uint32_t pId, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) + : pColStep(oid, tableOid, ct, jobInfo), fPseudoColumnId(pId) + { + } - PseudoColStep(const PassThruStep& rhs) : - pColStep(rhs), - fPseudoColumnId(rhs.pseudoType()) - {} + PseudoColStep(const PassThruStep& rhs) : pColStep(rhs), fPseudoColumnId(rhs.pseudoType()) + { + } - virtual ~PseudoColStep() {} + virtual ~PseudoColStep() + { + } - uint32_t pseudoColumnId() const - { - return fPseudoColumnId; - } - void pseudoColumnId(uint32_t pId) - { - fPseudoColumnId = pId; - } + uint32_t pseudoColumnId() const + { + return fPseudoColumnId; + } + void pseudoColumnId(uint32_t pId) + { + fPseudoColumnId = pId; + } -protected: - uint32_t fPseudoColumnId; + protected: + uint32_t fPseudoColumnId; -private: - /** @brief disabled constuctor - */ - PseudoColStep(const pColScanStep&); - PseudoColStep(const pColStep&); + private: + /** @brief disabled constuctor + */ + PseudoColStep(const pColScanStep&); + PseudoColStep(const pColStep&); }; - -} +} // namespace joblist #endif // JOBLIST_PRIMITIVESTEP_H // vim:ts=4 sw=4: - - diff --git a/dbcon/joblist/pseudocc-jl.cpp b/dbcon/joblist/pseudocc-jl.cpp index 7a48fb096..321ca5c8c 100644 --- a/dbcon/joblist/pseudocc-jl.cpp +++ b/dbcon/joblist/pseudocc-jl.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "pseudocc-jl.h" #include "pseudocolumn.h" #include "nullvaluemanip.h" @@ -26,7 +25,6 @@ using namespace execplan; namespace joblist { - PseudoCCJL::PseudoCCJL(const PseudoColStep& pcs) : ColumnCommandJL(pcs), function(pcs.pseudoColumnId()) { } @@ -37,82 +35,78 @@ PseudoCCJL::~PseudoCCJL() void PseudoCCJL::createCommand(ByteStream& bs) const { - bs << (uint8_t) PSEUDO_COLUMN; - bs << function; - ColumnCommandJL::createCommand(bs); + bs << (uint8_t)PSEUDO_COLUMN; + bs << function; + ColumnCommandJL::createCommand(bs); } void PseudoCCJL::runCommand(ByteStream& bs) const { - if (function == PSEUDO_EXTENTMAX) + if (function == PSEUDO_EXTENTMAX) + { + if (!colType.isWideDecimalType()) { - if (!colType.isWideDecimalType()) - { - int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; - int64_t min = extents[currentExtentIndex].partition.cprange.loVal; + int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; + int64_t min = extents[currentExtentIndex].partition.cprange.loVal; - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << max; - else - bs << utils::getNullValue(colType.colDataType, colType.colWidth); - } - else - { - int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; - int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; - - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << (uint128_t) max; - else - { - int128_t int128Null; - datatypes::Decimal::setWideDecimalNullValue(int128Null); - bs << (uint128_t) int128Null; - } - } + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << max; + else + bs << utils::getNullValue(colType.colDataType, colType.colWidth); } - else if (function == PSEUDO_EXTENTMIN) + else { - if (!colType.isWideDecimalType()) - { - int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; - int64_t min = extents[currentExtentIndex].partition.cprange.loVal; + int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; + int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << min; - else - bs << utils::getNullValue(colType.colDataType, colType.colWidth); - } - else - { - int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; - int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; - - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << (uint128_t) min; - else - { - int128_t int128Null; - datatypes::Decimal::setWideDecimalNullValue(int128Null); - bs << (uint128_t) int128Null; - } - } + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << (uint128_t)max; + else + { + int128_t int128Null; + datatypes::Decimal::setWideDecimalNullValue(int128Null); + bs << (uint128_t)int128Null; + } } - else if (function == PSEUDO_EXTENTID) - bs << extents[currentExtentIndex].range.start; + } + else if (function == PSEUDO_EXTENTMIN) + { + if (!colType.isWideDecimalType()) + { + int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; + int64_t min = extents[currentExtentIndex].partition.cprange.loVal; - ColumnCommandJL::runCommand(bs); + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << min; + else + bs << utils::getNullValue(colType.colDataType, colType.colWidth); + } + else + { + int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; + int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; + + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << (uint128_t)min; + else + { + int128_t int128Null; + datatypes::Decimal::setWideDecimalNullValue(int128Null); + bs << (uint128_t)int128Null; + } + } + } + else if (function == PSEUDO_EXTENTID) + bs << extents[currentExtentIndex].range.start; + + ColumnCommandJL::runCommand(bs); } string PseudoCCJL::toString() { - ostringstream oss; - oss << "PseudoColumnJL fcn: " << function << " on: " << ColumnCommandJL::toString(); - return oss.str(); + ostringstream oss; + oss << "PseudoColumnJL fcn: " << function << " on: " << ColumnCommandJL::toString(); + return oss.str(); } - - - - -} +} // namespace joblist diff --git a/dbcon/joblist/pseudocc-jl.h b/dbcon/joblist/pseudocc-jl.h index 1fd7e4100..040f2c74a 100644 --- a/dbcon/joblist/pseudocc-jl.h +++ b/dbcon/joblist/pseudocc-jl.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef PSEUDOCCJL_H #define PSEUDOCCJL_H @@ -23,25 +22,24 @@ namespace joblist { - class PseudoCCJL : public ColumnCommandJL { -public: - PseudoCCJL(const PseudoColStep&); - virtual ~PseudoCCJL(); + public: + PseudoCCJL(const PseudoColStep&); + virtual ~PseudoCCJL(); - virtual void createCommand(messageqcpp::ByteStream&) const; - virtual void runCommand(messageqcpp::ByteStream&) const; - virtual std::string toString(); - uint32_t getFunction() const - { - return function; - } -protected: -private: - uint32_t function; + virtual void createCommand(messageqcpp::ByteStream&) const; + virtual void runCommand(messageqcpp::ByteStream&) const; + virtual std::string toString(); + uint32_t getFunction() const + { + return function; + } + + protected: + private: + uint32_t function; }; - -} -#endif // PSEUDOCCJL_H +} // namespace joblist +#endif // PSEUDOCCJL_H diff --git a/dbcon/joblist/resource.h b/dbcon/joblist/resource.h index 8b1c6eda5..0d041c6d5 100644 --- a/dbcon/joblist/resource.h +++ b/dbcon/joblist/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/dbcon/joblist/resourcedistributor.cpp b/dbcon/joblist/resourcedistributor.cpp index ad2fb2cea..d9ef545a0 100644 --- a/dbcon/joblist/resourcedistributor.cpp +++ b/dbcon/joblist/resourcedistributor.cpp @@ -31,137 +31,135 @@ using namespace std; namespace joblist { - const unsigned maxSessionsDefault = 100; uint64_t ResourceDistributor::requestResource(uint32_t sessionID) { - uint64_t resource = getSessionResource(sessionID); + uint64_t resource = getSessionResource(sessionID); - return requestResource(sessionID, resource); + return requestResource(sessionID, resource); } uint64_t ResourceDistributor::requestResource(uint32_t sessionID, uint64_t resource) { + if (fTraceOn) + logMessage(logging::LOG_TYPE_DEBUG, LogRDRequest, resource, sessionID); + + boost::mutex::scoped_lock lk(fResourceLock); + + while (fTotalResource < resource) + { if (fTraceOn) - logMessage(logging::LOG_TYPE_DEBUG, LogRDRequest, resource, sessionID); + logMessage(logging::LOG_TYPE_DEBUG, LogRDRequestWait, resource, sessionID); - boost::mutex::scoped_lock lk (fResourceLock ); + fResourceAvailable.wait(lk); - while (fTotalResource < resource) - { - if (fTraceOn) - logMessage(logging::LOG_TYPE_DEBUG, LogRDRequestWait, resource, sessionID); - - fResourceAvailable.wait(lk); - - if (fTraceOn) - logMessage(logging::LOG_TYPE_DEBUG, LogRDRequest, resource, sessionID); - - } - - fTotalResource -= resource; - - return resource; -} - - - -void ResourceDistributor::returnResource(uint64_t resource) -{ if (fTraceOn) - logMessage(logging::LOG_TYPE_DEBUG, LogRDReturn, resource); + logMessage(logging::LOG_TYPE_DEBUG, LogRDRequest, resource, sessionID); + } - boost::mutex::scoped_lock lk (fResourceLock ); - fTotalResource += resource; + fTotalResource -= resource; - fResourceAvailable.notify_all(); + return resource; } -void ResourceDistributor::logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value, uint32_t sessionID) +void ResourceDistributor::returnResource(uint64_t resource) { - logging::Message::Args args; - args.add(fJob); - args.add(fIdentity); - args.add(fTotalResource); + if (fTraceOn) + logMessage(logging::LOG_TYPE_DEBUG, LogRDReturn, resource); - if (value) args.add(value); + boost::mutex::scoped_lock lk(fResourceLock); + fTotalResource += resource; - Logger log; - log.logMessage(logLevel, mid, args, logging::LoggingID(5, sessionID)); + fResourceAvailable.notify_all(); } -void LockedSessionMap::updateAging(uint32_t sessionID) +void ResourceDistributor::logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, + uint64_t value, uint32_t sessionID) { - boost::mutex::scoped_lock lock(fSessionLock); - SessionList::iterator pos = find(fSessionAgingList.begin(), fSessionAgingList.end(), sessionID); + logging::Message::Args args; + args.add(fJob); + args.add(fIdentity); + args.add(fTotalResource); - if (fSessionAgingList.end() != pos) - fSessionAgingList.splice(fSessionAgingList.end(), fSessionAgingList, find(fSessionAgingList.begin(), fSessionAgingList.end(), sessionID)); - else - fSessionAgingList.push_back(sessionID); + if (value) + args.add(value); + + Logger log; + log.logMessage(logLevel, mid, args, logging::LoggingID(5, sessionID)); } -uint64_t LockedSessionMap::getSessionResource(uint32_t sessionID) +void LockedSessionMap::updateAging(uint32_t sessionID) { - SessionMap::const_iterator it = fSessionMap.find(sessionID); + boost::mutex::scoped_lock lock(fSessionLock); + SessionList::iterator pos = find(fSessionAgingList.begin(), fSessionAgingList.end(), sessionID); - if (fSessionMap.end() != it) - { - updateAging(sessionID); - return it->second; - } + if (fSessionAgingList.end() != pos) + fSessionAgingList.splice(fSessionAgingList.end(), fSessionAgingList, + find(fSessionAgingList.begin(), fSessionAgingList.end(), sessionID)); + else + fSessionAgingList.push_back(sessionID); +} - return fResourceBlock; +uint64_t LockedSessionMap::getSessionResource(uint32_t sessionID) +{ + SessionMap::const_iterator it = fSessionMap.find(sessionID); + + if (fSessionMap.end() != it) + { + updateAging(sessionID); + return it->second; + } + + return fResourceBlock; } bool LockedSessionMap::addSession(uint32_t sessionID, uint64_t resource, uint64_t limit) { - bool ret = true; + bool ret = true; - if (resource > limit) - { - resource = limit; - ret = false; - } + if (resource > limit) + { + resource = limit; + ret = false; + } - boost::mutex::scoped_lock maplock(fMapLock); - fSessionMap[sessionID] = resource; - updateAging(sessionID); + boost::mutex::scoped_lock maplock(fMapLock); + fSessionMap[sessionID] = resource; + updateAging(sessionID); - if (fMaxSessions < fSessionMap.size()) - { - boost::mutex::scoped_lock lock(fSessionLock); - uint32_t oldsession = fSessionAgingList.front(); - fSessionMap.erase(oldsession); - fSessionAgingList.erase(fSessionAgingList.begin()); - } - - return ret; + if (fMaxSessions < fSessionMap.size()) + { + boost::mutex::scoped_lock lock(fSessionLock); + uint32_t oldsession = fSessionAgingList.front(); + fSessionMap.erase(oldsession); + fSessionAgingList.erase(fSessionAgingList.begin()); + } + return ret; } void LockedSessionMap::removeSession(uint32_t sessionID) { - boost::mutex::scoped_lock maplock(fMapLock); - fSessionMap.erase(sessionID); - boost::mutex::scoped_lock listlock(fSessionLock); - fSessionAgingList.erase(find(fSessionAgingList.begin(), fSessionAgingList.end(), sessionID)); + boost::mutex::scoped_lock maplock(fMapLock); + fSessionMap.erase(sessionID); + boost::mutex::scoped_lock listlock(fSessionLock); + fSessionAgingList.erase(find(fSessionAgingList.begin(), fSessionAgingList.end(), sessionID)); } -ostream& operator<<(ostream& os, const LockedSessionMap& lsm) +ostream& operator<<(ostream& os, const LockedSessionMap& lsm) { - os << "Default Resource Block: " << lsm.fResourceBlock << "\tMax Number of saved sessions: " << lsm.fMaxSessions << endl; - os << "Session Map:\tsessionID\tvalue\n"; - LockedSessionMap::SessionMap::const_iterator smIt = lsm.fSessionMap.begin(), smEnd = lsm.fSessionMap.end(); + os << "Default Resource Block: " << lsm.fResourceBlock + << "\tMax Number of saved sessions: " << lsm.fMaxSessions << endl; + os << "Session Map:\tsessionID\tvalue\n"; + LockedSessionMap::SessionMap::const_iterator smIt = lsm.fSessionMap.begin(), smEnd = lsm.fSessionMap.end(); - for (; smIt != smEnd; ++smIt) - os << "\t\t" << smIt->first << "\t\t" << smIt->second << endl; + for (; smIt != smEnd; ++smIt) + os << "\t\t" << smIt->first << "\t\t" << smIt->second << endl; - os << "\nAging List:\tsessionID\n\t\t"; - copy(lsm.fSessionAgingList.begin(), lsm.fSessionAgingList.end(), ostream_iterator(os, "\n\t\t")); - os << endl; - return os; + os << "\nAging List:\tsessionID\n\t\t"; + copy(lsm.fSessionAgingList.begin(), lsm.fSessionAgingList.end(), ostream_iterator(os, "\n\t\t")); + os << endl; + return os; } -} //namespace - +} // namespace joblist diff --git a/dbcon/joblist/resourcedistributor.h b/dbcon/joblist/resourcedistributor.h index c27d1fac4..6704716ef 100644 --- a/dbcon/joblist/resourcedistributor.h +++ b/dbcon/joblist/resourcedistributor.h @@ -37,7 +37,6 @@ namespace joblist { - /** @brief ResourceDistributor * Manages a resource. Distributes up to fTotalResource on request. * Expects the requester to return the resource when finished. @@ -59,86 +58,91 @@ extern const unsigned maxSessionsDefault; class LockedSessionMap { -public: - LockedSessionMap(uint64_t resource, unsigned maxSessions = maxSessionsDefault): fResourceBlock(resource), fMaxSessions(maxSessions) {} - typedef std::map SessionMap; - typedef std::list SessionList; - bool addSession(uint32_t sessionID, uint64_t resource, uint64_t limit = std::numeric_limits::max()); - void removeSession(uint32_t sessionID); - uint64_t getSessionResource(uint32_t sessionID); - friend std::ostream& operator<<(std::ostream& os, const LockedSessionMap& lsm); - -private: - void updateAging(uint32_t sessionID); - boost::mutex fMapLock; - SessionMap fSessionMap; - uint64_t fResourceBlock; - boost::mutex fSessionLock; - SessionList fSessionAgingList; - const unsigned fMaxSessions; - + public: + LockedSessionMap(uint64_t resource, unsigned maxSessions = maxSessionsDefault) + : fResourceBlock(resource), fMaxSessions(maxSessions) + { + } + typedef std::map SessionMap; + typedef std::list SessionList; + bool addSession(uint32_t sessionID, uint64_t resource, + uint64_t limit = std::numeric_limits::max()); + void removeSession(uint32_t sessionID); + uint64_t getSessionResource(uint32_t sessionID); + friend std::ostream& operator<<(std::ostream& os, const LockedSessionMap& lsm); + private: + void updateAging(uint32_t sessionID); + boost::mutex fMapLock; + SessionMap fSessionMap; + uint64_t fResourceBlock; + boost::mutex fSessionLock; + SessionList fSessionAgingList; + const unsigned fMaxSessions; }; - class ResourceDistributor { -public: + public: + ResourceDistributor(const std::string& job, const std::string& identity, uint64_t totalResource, + uint64_t resourceBlock, bool trace) + : fJob(job) + , fIdentity(identity) + , fTotalResource(totalResource) + , fSessionMap(resourceBlock) + , fTraceOn(trace) + { + } - ResourceDistributor(const std::string& job, const std::string& identity, uint64_t totalResource, uint64_t resourceBlock, bool trace) : - fJob(job), fIdentity(identity), fTotalResource(totalResource), fSessionMap(resourceBlock), fTraceOn(trace) - {} + virtual ~ResourceDistributor() + { + } - virtual ~ResourceDistributor() {} + typedef std::map SessionMap; - typedef std::map SessionMap; + uint64_t requestResource(uint32_t sessionID); + uint64_t requestResource(uint32_t sessionID, uint64_t resource); + void returnResource(uint64_t resource); - uint64_t requestResource(uint32_t sessionID); - uint64_t requestResource(uint32_t sessionID, uint64_t resource); - void returnResource(uint64_t resource); + uint64_t getSessionResource(uint32_t sessionID) + { + return fSessionMap.getSessionResource(sessionID); + } + uint64_t getTotalResource() const + { + return fTotalResource; + } - uint64_t getSessionResource(uint32_t sessionID) - { - return fSessionMap.getSessionResource(sessionID); - } + void setTrace(bool trace) + { + fTraceOn = trace; + } - uint64_t getTotalResource() const - { - return fTotalResource; - } + bool addSession(uint32_t sessionID, uint64_t resource) + { + return fSessionMap.addSession(sessionID, resource, fTotalResource); + } + void removeSession(uint32_t sessionID) + { + fSessionMap.removeSession(sessionID); + } + private: + void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0, + uint32_t sessionId = 0); - void setTrace(bool trace) - { - fTraceOn = trace; - } - - bool addSession(uint32_t sessionID, uint64_t resource) - { - return fSessionMap.addSession(sessionID, resource, fTotalResource); - } - void removeSession(uint32_t sessionID) - { - fSessionMap.removeSession(sessionID); - } - -private: - - void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0, uint32_t sessionId = 0); - - std::string fJob; - std::string fIdentity; - uint64_t fTotalResource; - uint64_t fResourceBlock; - boost::mutex fResourceLock; - boost::condition fResourceAvailable; - - LockedSessionMap fSessionMap; - uint32_t fTraceOn; + std::string fJob; + std::string fIdentity; + uint64_t fTotalResource; + uint64_t fResourceBlock; + boost::mutex fResourceLock; + boost::condition fResourceAvailable; + LockedSessionMap fSessionMap; + uint32_t fTraceOn; }; -} +} // namespace joblist #endif diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index ca4e1c7a7..f011d8177 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -43,18 +43,17 @@ using namespace config; namespace joblist { - -//const string ResourceManager::fExeMgrStr("ExeMgr1"); +// const string ResourceManager::fExeMgrStr("ExeMgr1"); const string ResourceManager::fHashJoinStr("HashJoin"); const string ResourceManager::fHashBucketReuseStr("HashBucketReuse"); const string ResourceManager::fJobListStr("JobList"); const string ResourceManager::fPrimitiveServersStr("PrimitiveServers"); -//const string ResourceManager::fSystemConfigStr("SystemConfig"); +// const string ResourceManager::fSystemConfigStr("SystemConfig"); const string ResourceManager::fTupleWSDLStr("TupleWSDL"); const string ResourceManager::fZDLStr("ZDL"); const string ResourceManager::fExtentMapStr("ExtentMap"); -//const string ResourceManager::fDMLProcStr("DMLProc"); -//const string ResourceManager::fBatchInsertStr("BatchInsert"); +// const string ResourceManager::fDMLProcStr("DMLProc"); +// const string ResourceManager::fBatchInsertStr("BatchInsert"); const string ResourceManager::fOrderByLimitStr("OrderByLimit"); const string ResourceManager::fRowAggregationStr("RowAggregation"); @@ -63,362 +62,413 @@ boost::mutex mx; ResourceManager* ResourceManager::instance(bool runningInExeMgr) { - boost::mutex::scoped_lock lk(mx); + boost::mutex::scoped_lock lk(mx); - if (!fInstance) - fInstance = new ResourceManager(runningInExeMgr); + if (!fInstance) + fInstance = new ResourceManager(runningInExeMgr); - return fInstance; + return fInstance; } -ResourceManager::ResourceManager(bool runningInExeMgr) : - fExeMgrStr("ExeMgr1"), - fSystemConfigStr("SystemConfig"), - fDMLProcStr("DMLProc"), - fBatchInsertStr("BatchInsert"), - fConfig(Config::makeConfig()), - fNumCores(8), - fHjNumThreads(defaultNumThreads), - fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan), - fJlNumScanReceiveThreads(defaultScanReceiveThreads), - fTwNumThreads(defaultNumThreads), - fJlMaxOutstandingRequests(defaultMaxOutstandingRequests), - fHJUmMaxMemorySmallSideDistributor(fHashJoinStr, - "UmMaxMemorySmallSide", - getUintVal(fHashJoinStr, "TotalUmMaxMemorySmallSide", defaultTotalUmMemory), - getUintVal(fHashJoinStr, "UmMaxMemorySmallSide", defaultHJUmMaxMemorySmallSide), - 0), - fHJPmMaxMemorySmallSideSessionMap( - getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide)), - isExeMgr(runningInExeMgr) +ResourceManager::ResourceManager(bool runningInExeMgr) + : fExeMgrStr("ExeMgr1") + , fSystemConfigStr("SystemConfig") + , fDMLProcStr("DMLProc") + , fBatchInsertStr("BatchInsert") + , fConfig(Config::makeConfig()) + , fNumCores(8) + , fHjNumThreads(defaultNumThreads) + , fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan) + , fJlNumScanReceiveThreads(defaultScanReceiveThreads) + , fTwNumThreads(defaultNumThreads) + , fJlMaxOutstandingRequests(defaultMaxOutstandingRequests) + , fHJUmMaxMemorySmallSideDistributor( + fHashJoinStr, "UmMaxMemorySmallSide", + getUintVal(fHashJoinStr, "TotalUmMaxMemorySmallSide", defaultTotalUmMemory), + getUintVal(fHashJoinStr, "UmMaxMemorySmallSide", defaultHJUmMaxMemorySmallSide), 0) + , fHJPmMaxMemorySmallSideSessionMap( + getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide)) + , isExeMgr(runningInExeMgr) { - int temp; - int configNumCores = -1; + int temp; + int configNumCores = -1; - fTraceFlags = 0; - //See if we want to override the calculated #cores - temp = getIntVal(fJobListStr, "NumCores", -1); + fTraceFlags = 0; + // See if we want to override the calculated #cores + temp = getIntVal(fJobListStr, "NumCores", -1); - if (temp > 0) - configNumCores = temp; + if (temp > 0) + configNumCores = temp; - if (configNumCores <= 0) + if (configNumCores <= 0) + { + // count the actual #cores + utils::CGroupConfigurator cg; + fNumCores = cg.getNumCores(); + + if (fNumCores <= 0) + fNumCores = 8; + } + else + fNumCores = configNumCores; + + // based on the #cores, calculate some thread parms + if (fNumCores > 0) + { + fHjNumThreads = fNumCores; + fJlNumScanReceiveThreads = fNumCores; + fTwNumThreads = fNumCores; + } + + // possibly override any calculated values + temp = getIntVal(fHashJoinStr, "NumThreads", -1); + + if (temp > 0) + fHjNumThreads = temp; + + temp = getIntVal(fJobListStr, "ProcessorThreadsPerScan", -1); + + if (temp > 0) + fJlProcessorThreadsPerScan = temp; + + temp = getIntVal(fJobListStr, "MaxOutstandingRequests", -1); + + if (temp > 0) + fJlMaxOutstandingRequests = temp; + else + { + oam::Oam oam; + oam::ModuleTypeConfig moduletypeconfig; + oam.getSystemConfig("pm", moduletypeconfig); + const uint temp = moduletypeconfig.ModuleCount * fNumCores * 4 / fJlProcessorThreadsPerScan; + const uint minMaxOutstandingRequests = + std::max(static_cast(moduletypeconfig.ModuleCount * 2), defaultMaxOutstandingRequests); + fJlMaxOutstandingRequests = temp > minMaxOutstandingRequests ? temp : minMaxOutstandingRequests; + } + + temp = getIntVal(fJobListStr, "NumScanReceiveThreads", -1); + + if (temp > 0) + fJlNumScanReceiveThreads = temp; + + fDECConnectionsPerQuery = getUintVal(fJobListStr, "DECConnectionsPerQuery", 0); + fDECConnectionsPerQuery = + (fDECConnectionsPerQuery) ? fDECConnectionsPerQuery : getPsConnectionsPerPrimProc(); + + temp = getIntVal(fTupleWSDLStr, "NumThreads", -1); + + if (temp > 0) + fTwNumThreads = temp; + + pmJoinMemLimit = getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide); + + // Need to use different limits if this instance isn't running on the UM, + // or if it's an ExeMgr running on a PM node + if (!isExeMgr) + totalUmMemLimit = pmJoinMemLimit; + else + { + // Installation.PMwithUM = y by default so RM prefers HashJoin.TotalPmUmMemory + // if it exists. + string whichLimit = "TotalUmMemory"; + string pmWithUM = fConfig->getConfig("Installation", "PMwithUM"); + + if (pmWithUM == "y" || pmWithUM == "Y") { - //count the actual #cores + oam::Oam OAM; + oam::oamModuleInfo_t moduleInfo = OAM.getModuleInfo(); + string& moduleType = boost::get<1>(moduleInfo); + + if (moduleType == "pm" || moduleType == "PM") + { + string doesItExist = fConfig->getConfig(fHashJoinStr, "TotalPmUmMemory"); + + if (!doesItExist.empty()) + whichLimit = "TotalPmUmMemory"; + } + } + + string umtxt = fConfig->getConfig(fHashJoinStr, whichLimit); + + if (umtxt.empty()) + totalUmMemLimit = defaultTotalUmMemory; + else + { + // is it an absolute or a percentage? + if (umtxt.find('%') != string::npos) + { utils::CGroupConfigurator cg; - fNumCores = cg.getNumCores(); + uint64_t totalMem = cg.getTotalMemory(); + totalUmMemLimit = atoll(umtxt.c_str()) / 100.0 * (double)totalMem; - if (fNumCores <= 0) - fNumCores = 8; + if (totalUmMemLimit == 0 || totalUmMemLimit == LLONG_MIN || + totalUmMemLimit == LLONG_MAX) // some garbage in the xml entry + totalUmMemLimit = defaultTotalUmMemory; + } + else // an absolute; use the existing converter + { + totalUmMemLimit = getIntVal(fHashJoinStr, whichLimit, defaultTotalUmMemory); + } } + } + + configuredUmMemLimit = totalUmMemLimit; + // cout << "RM: total UM memory = " << totalUmMemLimit << endl; + + // multi-thread aggregate + string nt, nb, nr; + nt = fConfig->getConfig("RowAggregation", "RowAggrThreads"); + + if (nt.empty()) + { + if (numCores() > 0) + fAggNumThreads = numCores(); else - fNumCores = configNumCores; + fAggNumThreads = 1; + } + else + fAggNumThreads = fConfig->uFromText(nt); - //based on the #cores, calculate some thread parms - if (fNumCores > 0) - { - fHjNumThreads = fNumCores; - fJlNumScanReceiveThreads = fNumCores; - fTwNumThreads = fNumCores; - } + nb = fConfig->getConfig("RowAggregation", "RowAggrBuckets"); - //possibly override any calculated values - temp = getIntVal(fHashJoinStr, "NumThreads", -1); + if (nb.empty()) + fAggNumBuckets = fAggNumThreads * 4; + else + fAggNumBuckets = fConfig->uFromText(nb); - if (temp > 0) - fHjNumThreads = temp; + nr = fConfig->getConfig("RowAggregation", "RowAggrRowGroupsPerThread"); - temp = getIntVal(fJobListStr, "ProcessorThreadsPerScan", -1); + if (nr.empty()) + fAggNumRowGroups = 20; + else + fAggNumRowGroups = fConfig->uFromText(nr); + // window function + string wt = fConfig->getConfig("WindowFunction", "WorkThreads"); - if (temp > 0) - fJlProcessorThreadsPerScan = temp; + if (wt.empty()) + fWindowFunctionThreads = numCores(); + else + fWindowFunctionThreads = fConfig->uFromText(wt); - temp = getIntVal(fJobListStr, "MaxOutstandingRequests", -1); + // hdfs info + string hdfs = fConfig->getConfig("SystemConfig", "DataFilePlugin"); - if (temp > 0) - fJlMaxOutstandingRequests = temp; - else - { - oam::Oam oam; - oam::ModuleTypeConfig moduletypeconfig; - oam.getSystemConfig("pm", moduletypeconfig); - const uint temp = moduletypeconfig.ModuleCount * fNumCores * 4 / fJlProcessorThreadsPerScan; - const uint minMaxOutstandingRequests = std::max(static_cast(moduletypeconfig.ModuleCount * 2), defaultMaxOutstandingRequests); - fJlMaxOutstandingRequests = temp > minMaxOutstandingRequests ? temp : minMaxOutstandingRequests; - } + if (hdfs.find("hdfs") != string::npos) + fUseHdfs = true; + else + fUseHdfs = false; - temp = getIntVal(fJobListStr, "NumScanReceiveThreads", -1); + fAllowedDiskAggregation = + getBoolVal(fRowAggregationStr, "AllowDiskBasedAggregation", defaultAllowDiskAggregation); - if (temp > 0) - fJlNumScanReceiveThreads = temp; + fMaxBPPSendQueue = getUintVal(fPrimitiveServersStr, "MaxBPPSendQueue", defaultMaxBPPSendQueue); - fDECConnectionsPerQuery = getUintVal(fJobListStr, "DECConnectionsPerQuery", 0); - fDECConnectionsPerQuery = (fDECConnectionsPerQuery) - ? fDECConnectionsPerQuery - : getPsConnectionsPerPrimProc(); - - temp = getIntVal(fTupleWSDLStr, "NumThreads", -1); - - if (temp > 0) - fTwNumThreads = temp; - - pmJoinMemLimit = getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", - defaultHJPmMaxMemorySmallSide); - - // Need to use different limits if this instance isn't running on the UM, - // or if it's an ExeMgr running on a PM node - if (!isExeMgr) - totalUmMemLimit = pmJoinMemLimit; - else - { - // Installation.PMwithUM = y by default so RM prefers HashJoin.TotalPmUmMemory - // if it exists. - string whichLimit = "TotalUmMemory"; - string pmWithUM = fConfig->getConfig("Installation", "PMwithUM"); - - if (pmWithUM == "y" || pmWithUM == "Y") - { - oam::Oam OAM; - oam::oamModuleInfo_t moduleInfo = OAM.getModuleInfo(); - string& moduleType = boost::get<1>(moduleInfo); - - if (moduleType == "pm" || moduleType == "PM") - { - string doesItExist = fConfig->getConfig(fHashJoinStr, "TotalPmUmMemory"); - - if (!doesItExist.empty()) - whichLimit = "TotalPmUmMemory"; - } - } - - string umtxt = fConfig->getConfig(fHashJoinStr, whichLimit); - - if (umtxt.empty()) - totalUmMemLimit = defaultTotalUmMemory; - else - { - // is it an absolute or a percentage? - if (umtxt.find('%') != string::npos) - { - utils::CGroupConfigurator cg; - uint64_t totalMem = cg.getTotalMemory(); - totalUmMemLimit = atoll(umtxt.c_str()) / 100.0 * (double) totalMem; - - if (totalUmMemLimit == 0 || totalUmMemLimit == LLONG_MIN || - totalUmMemLimit == LLONG_MAX) // some garbage in the xml entry - totalUmMemLimit = defaultTotalUmMemory; - } - else // an absolute; use the existing converter - { - totalUmMemLimit = getIntVal(fHashJoinStr, whichLimit, - defaultTotalUmMemory); - } - } - } - - configuredUmMemLimit = totalUmMemLimit; - //cout << "RM: total UM memory = " << totalUmMemLimit << endl; - - // multi-thread aggregate - string nt, nb, nr; - nt = fConfig->getConfig("RowAggregation", "RowAggrThreads"); - - if (nt.empty()) - { - if ( numCores() > 0 ) - fAggNumThreads = numCores(); - else - fAggNumThreads = 1; - } - else - fAggNumThreads = fConfig->uFromText(nt); - - nb = fConfig->getConfig("RowAggregation", "RowAggrBuckets"); - - if (nb.empty()) - fAggNumBuckets = fAggNumThreads * 4; - else - fAggNumBuckets = fConfig->uFromText(nb); - - nr = fConfig->getConfig("RowAggregation", "RowAggrRowGroupsPerThread"); - - if (nr.empty()) - fAggNumRowGroups = 20; - else - fAggNumRowGroups = fConfig->uFromText(nr); - - // window function - string wt = fConfig->getConfig("WindowFunction", "WorkThreads"); - - if (wt.empty()) - fWindowFunctionThreads = numCores(); - else - fWindowFunctionThreads = fConfig->uFromText(wt); - - // hdfs info - string hdfs = fConfig->getConfig("SystemConfig", "DataFilePlugin"); - - if ( hdfs.find("hdfs") != string::npos) - fUseHdfs = true; - else - fUseHdfs = false; - - fAllowedDiskAggregation = getBoolVal(fRowAggregationStr, - "AllowDiskBasedAggregation", - defaultAllowDiskAggregation); - - fMaxBPPSendQueue = getUintVal(fPrimitiveServersStr, "MaxBPPSendQueue", defaultMaxBPPSendQueue); - - if (!load_encryption_keys()) - { - Logger log; - log.logMessage(logging::LOG_TYPE_ERROR, "Error loading CEJ password encryption keys"); - } + if (!load_encryption_keys()) + { + Logger log; + log.logMessage(logging::LOG_TYPE_ERROR, "Error loading CEJ password encryption keys"); + } } int ResourceManager::getEmPriority() const { - int temp = getIntVal(fExeMgrStr, "Priority", defaultEMPriority); - // config file priority is 40..1 (highest..lowest) - // convert to -20..19 (highest..lowest, defaults to -1) - int val; + int temp = getIntVal(fExeMgrStr, "Priority", defaultEMPriority); + // config file priority is 40..1 (highest..lowest) + // convert to -20..19 (highest..lowest, defaults to -1) + int val; - // @Bug3385 - the ExeMgr priority was being set backwards with 1 being the highest instead of the lowest. - if (temp < 1) - val = 19; - else if (temp > 40) - val = -20; - else - val = 20 - temp; + // @Bug3385 - the ExeMgr priority was being set backwards with 1 being the highest instead of the lowest. + if (temp < 1) + val = 19; + else if (temp > 40) + val = -20; + else + val = 20 - temp; - return val; + return val; } void ResourceManager::addHJPmMaxSmallSideMap(uint32_t sessionID, uint64_t mem) { - if (fHJPmMaxMemorySmallSideSessionMap.addSession(sessionID, mem, - fHJUmMaxMemorySmallSideDistributor.getTotalResource())) - logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, defaultHJPmMaxMemorySmallSide, - "PmMaxMemorySmallSide", LogRMResourceChange ); - else - { - logResourceChangeMessage(logging::LOG_TYPE_WARNING, sessionID, mem, - fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "PmMaxMemorySmallSide", - LogRMResourceChangeError); + if (fHJPmMaxMemorySmallSideSessionMap.addSession(sessionID, mem, + fHJUmMaxMemorySmallSideDistributor.getTotalResource())) + logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, defaultHJPmMaxMemorySmallSide, + "PmMaxMemorySmallSide", LogRMResourceChange); + else + { + logResourceChangeMessage(logging::LOG_TYPE_WARNING, sessionID, mem, + fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "PmMaxMemorySmallSide", + LogRMResourceChangeError); - logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, - fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "PmMaxMemorySmallSide", - LogRMResourceChangeError); - } + logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, + fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "PmMaxMemorySmallSide", + LogRMResourceChangeError); + } } void ResourceManager::addHJUmMaxSmallSideMap(uint32_t sessionID, uint64_t mem) { - if (fHJUmMaxMemorySmallSideDistributor.addSession(sessionID, mem)) - logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, defaultHJUmMaxMemorySmallSide, - "UmMaxMemorySmallSide", LogRMResourceChange); - else - { - logResourceChangeMessage(logging::LOG_TYPE_WARNING, sessionID, mem, - fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "UmMaxMemorySmallSide", - LogRMResourceChangeError); - - logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, - fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "UmMaxMemorySmallSide", - LogRMResourceChangeError); - - } + if (fHJUmMaxMemorySmallSideDistributor.addSession(sessionID, mem)) + logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, defaultHJUmMaxMemorySmallSide, + "UmMaxMemorySmallSide", LogRMResourceChange); + else + { + logResourceChangeMessage(logging::LOG_TYPE_WARNING, sessionID, mem, + fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "UmMaxMemorySmallSide", + LogRMResourceChangeError); + logResourceChangeMessage(logging::LOG_TYPE_INFO, sessionID, mem, + fHJUmMaxMemorySmallSideDistributor.getTotalResource(), "UmMaxMemorySmallSide", + LogRMResourceChangeError); + } } -void ResourceManager::logResourceChangeMessage(logging::LOG_TYPE logType, uint32_t sessionID, uint64_t newvalue, - uint64_t value, const string& source, logging::Message::MessageID mid) +void ResourceManager::logResourceChangeMessage(logging::LOG_TYPE logType, uint32_t sessionID, + uint64_t newvalue, uint64_t value, const string& source, + logging::Message::MessageID mid) { - logging::Message::Args args; - args.add(source); - args.add(newvalue); - args.add(value); - Logger log; - log.logMessage(logType, mid, args, logging::LoggingID(5, sessionID)); + logging::Message::Args args; + args.add(source); + args.add(newvalue); + args.add(value); + Logger log; + log.logMessage(logType, mid, args, logging::LoggingID(5, sessionID)); } -void ResourceManager::emServerThreads() { } -void ResourceManager::emServerQueueSize() { } -void ResourceManager::emSecondsBetweenMemChecks() { } -void ResourceManager::emMaxPct() { } -void ResourceManager::emPriority() { } -void ResourceManager::emExecQueueSize() { } - -void ResourceManager::hjNumThreads() { } -void ResourceManager::hjMaxBuckets() { } -void ResourceManager::hjMaxElems() { } -void ResourceManager::hjFifoSizeLargeSide() { } -void ResourceManager::hjPmMaxMemorySmallSide() { } - -void ResourceManager::jlFlushInterval() { } -void ResourceManager::jlFifoSize() { } -void ResourceManager::jlScanLbidReqLimit() { } -void ResourceManager::jlScanLbidReqThreshold() { } -void ResourceManager::jlProjectBlockReqLimit() { } -void ResourceManager::jlProjectBlockReqThreshold() { } -void ResourceManager::jlNumScanReceiveThreads() { } - - -void ResourceManager::psCount() { } -void ResourceManager::psConnectionsPerPrimProc() { } -void ResourceManager::psLBID_Shift() { } - -void ResourceManager::scTempDiskPath() { } -void ResourceManager::scTempSaveSize() { } -void ResourceManager::scWorkingDir() { } - - -void ResourceManager::twMaxSize() { } -void ResourceManager::twInitialCapacity() { } -void ResourceManager::twMaxBuckets () { } -void ResourceManager::twNumThreads() { } -void ResourceManager::zdl_MaxElementsInMem() { } -void ResourceManager::zdl_MaxElementsPerBucket () { } - -void ResourceManager::hbrPredicate() { } - -bool ResourceManager::getMysqldInfo( - std::string& h, std::string& u, std::string& w, unsigned int& p) const +void ResourceManager::emServerThreads() { - static const std::string hostUserUnassignedValue("unassigned"); - // MCS will read username and pass from disk if the config changed. - u = getStringVal("CrossEngineSupport", "User", hostUserUnassignedValue); - std::string encryptedPW = getStringVal("CrossEngineSupport", "Password", ""); - //This will return back the plaintext password if there is no MCSDATADIR/.secrets file present - w = decrypt_password(encryptedPW); - // MCS will not read username and pass from disk if the config changed. - h = getStringVal("CrossEngineSupport", "Host", hostUserUnassignedValue); - p = getUintVal("CrossEngineSupport", "Port", 0); - u = getStringVal("CrossEngineSupport", "User", "unassigned"); +} +void ResourceManager::emServerQueueSize() +{ +} +void ResourceManager::emSecondsBetweenMemChecks() +{ +} +void ResourceManager::emMaxPct() +{ +} +void ResourceManager::emPriority() +{ +} +void ResourceManager::emExecQueueSize() +{ +} - bool rc = true; +void ResourceManager::hjNumThreads() +{ +} +void ResourceManager::hjMaxBuckets() +{ +} +void ResourceManager::hjMaxElems() +{ +} +void ResourceManager::hjFifoSizeLargeSide() +{ +} +void ResourceManager::hjPmMaxMemorySmallSide() +{ +} - if ((h.compare("unassigned") == 0) || - (u.compare("unassigned") == 0) || - (p == 0)) - rc = false; +void ResourceManager::jlFlushInterval() +{ +} +void ResourceManager::jlFifoSize() +{ +} +void ResourceManager::jlScanLbidReqLimit() +{ +} +void ResourceManager::jlScanLbidReqThreshold() +{ +} +void ResourceManager::jlProjectBlockReqLimit() +{ +} +void ResourceManager::jlProjectBlockReqThreshold() +{ +} +void ResourceManager::jlNumScanReceiveThreads() +{ +} - return rc; +void ResourceManager::psCount() +{ +} +void ResourceManager::psConnectionsPerPrimProc() +{ +} +void ResourceManager::psLBID_Shift() +{ +} + +void ResourceManager::scTempDiskPath() +{ +} +void ResourceManager::scTempSaveSize() +{ +} +void ResourceManager::scWorkingDir() +{ +} + +void ResourceManager::twMaxSize() +{ +} +void ResourceManager::twInitialCapacity() +{ +} +void ResourceManager::twMaxBuckets() +{ +} +void ResourceManager::twNumThreads() +{ +} +void ResourceManager::zdl_MaxElementsInMem() +{ +} +void ResourceManager::zdl_MaxElementsPerBucket() +{ +} + +void ResourceManager::hbrPredicate() +{ +} + +bool ResourceManager::getMysqldInfo(std::string& h, std::string& u, std::string& w, unsigned int& p) const +{ + static const std::string hostUserUnassignedValue("unassigned"); + // MCS will read username and pass from disk if the config changed. + u = getStringVal("CrossEngineSupport", "User", hostUserUnassignedValue); + std::string encryptedPW = getStringVal("CrossEngineSupport", "Password", ""); + // This will return back the plaintext password if there is no MCSDATADIR/.secrets file present + w = decrypt_password(encryptedPW); + // MCS will not read username and pass from disk if the config changed. + h = getStringVal("CrossEngineSupport", "Host", hostUserUnassignedValue); + p = getUintVal("CrossEngineSupport", "Port", 0); + u = getStringVal("CrossEngineSupport", "User", "unassigned"); + + bool rc = true; + + if ((h.compare("unassigned") == 0) || (u.compare("unassigned") == 0) || (p == 0)) + rc = false; + + return rc; } bool ResourceManager::queryStatsEnabled() const { - std::string val(getStringVal("QueryStats", "Enabled", "N" )); - boost::to_upper(val); - return "Y" == val; + std::string val(getStringVal("QueryStats", "Enabled", "N")); + boost::to_upper(val); + return "Y" == val; } bool ResourceManager::userPriorityEnabled() const { - std::string val(getStringVal("UserPriority", "Enabled", "N" )); - boost::to_upper(val); - return "Y" == val; + std::string val(getStringVal("UserPriority", "Enabled", "N")); + boost::to_upper(val); + return "Y" == val; } // Counts memory. This funtion doesn't actually malloc, just counts against two limits @@ -426,47 +476,46 @@ bool ResourceManager::userPriorityEnabled() const // If both have space, return true. bool ResourceManager::getMemory(int64_t amount, boost::shared_ptr& sessionLimit, bool patience) { - bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); - bool ret2 = sessionLimit ? (atomicops::atomicSub(sessionLimit.get(), amount) >= 0) : ret1; + bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); + bool ret2 = sessionLimit ? (atomicops::atomicSub(sessionLimit.get(), amount) >= 0) : ret1; - uint32_t retryCounter = 0, maxRetries = 20; // 10s delay + uint32_t retryCounter = 0, maxRetries = 20; // 10s delay - while (patience && !(ret1 && ret2) && retryCounter++ < maxRetries) - { - atomicops::atomicAdd(&totalUmMemLimit, amount); - sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0; - usleep(500000); - ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); - ret2 = sessionLimit ? (atomicops::atomicSub(sessionLimit.get(), amount) >= 0) : ret1; - } - if (!(ret1 && ret2)) - { - // If we didn't get any memory, restore the counters. - atomicops::atomicAdd(&totalUmMemLimit, amount); - sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0; - } - return (ret1 && ret2); + while (patience && !(ret1 && ret2) && retryCounter++ < maxRetries) + { + atomicops::atomicAdd(&totalUmMemLimit, amount); + sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0; + usleep(500000); + ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); + ret2 = sessionLimit ? (atomicops::atomicSub(sessionLimit.get(), amount) >= 0) : ret1; + } + if (!(ret1 && ret2)) + { + // If we didn't get any memory, restore the counters. + atomicops::atomicAdd(&totalUmMemLimit, amount); + sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0; + } + return (ret1 && ret2); } // Don't care about session memory bool ResourceManager::getMemory(int64_t amount, bool patience) { - bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); - - uint32_t retryCounter = 0, maxRetries = 20; // 10s delay - - while (patience && !ret1 && retryCounter++ < maxRetries) - { - atomicops::atomicAdd(&totalUmMemLimit, amount); - usleep(500000); - ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); - } - if (!ret1) - { - // If we didn't get any memory, restore the counters. - atomicops::atomicAdd(&totalUmMemLimit, amount); - } - return ret1; + bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); + + uint32_t retryCounter = 0, maxRetries = 20; // 10s delay + + while (patience && !ret1 && retryCounter++ < maxRetries) + { + atomicops::atomicAdd(&totalUmMemLimit, amount); + usleep(500000); + ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); + } + if (!ret1) + { + // If we didn't get any memory, restore the counters. + atomicops::atomicAdd(&totalUmMemLimit, amount); + } + return ret1; } - -} //namespace +} // namespace joblist diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 476b57bcc..f9d15ab33 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -46,14 +46,14 @@ namespace joblist { -//aggfilterstep -const uint32_t defaultNumThreads = 8; -//joblistfactory +// aggfilterstep +const uint32_t defaultNumThreads = 8; +// joblistfactory const uint32_t defaultFlushInterval = 8 * 1024; const uint32_t defaultFifoSize = 10; const uint32_t defaultHJFifoSizeLargeSide = 128; -const uint64_t defaultHJMaxElems = 512 * 1024; //hashjoin uses 8192 -const int defaultHJMaxBuckets = 32; //hashjoin uses 4 +const uint64_t defaultHJMaxElems = 512 * 1024; // hashjoin uses 8192 +const int defaultHJMaxBuckets = 32; // hashjoin uses 4 const uint64_t defaultHJPmMaxMemorySmallSide = 1 * 1024 * 1024 * 1024ULL; const uint64_t defaultHJUmMaxMemorySmallSide = 4 * 1024 * 1024 * 1024ULL; const uint32_t defaultTempSaveSize = defaultHJMaxElems; @@ -64,41 +64,43 @@ const uint32_t defaultTupleDLMaxSize = 64 * 1024; const uint32_t defaultJLThreadPoolSize = 100; -//pcolscan.cpp -const uint32_t defaultScanLbidReqLimit = 10000; +// pcolscan.cpp +const uint32_t defaultScanLbidReqLimit = 10000; const uint32_t defaultScanLbidReqThreshold = 5000; -const uint32_t defaultLogicalBlocksPerScan = 1024; // added for bug 1264. -const uint32_t defaultScanBlockThreshhold = 10000; //in jobstep.h +const uint32_t defaultLogicalBlocksPerScan = 1024; // added for bug 1264. +const uint32_t defaultScanBlockThreshhold = 10000; // in jobstep.h const uint32_t defaultScanReceiveThreads = 8; -//pcolstep.cpp -const uint32_t defaultProjectBlockReqLimit = 32 * 1024; -const uint32_t defaultProjectBlockReqThreshold = 16 * 1024; //256 in jobstep.h +// pcolstep.cpp +const uint32_t defaultProjectBlockReqLimit = 32 * 1024; +const uint32_t defaultProjectBlockReqThreshold = 16 * 1024; // 256 in jobstep.h -//BatchPrimitiveStep -const uint32_t defaultRequestSize = 1; +// BatchPrimitiveStep +const uint32_t defaultRequestSize = 1; const uint32_t defaultMaxOutstandingRequests = 20; const uint32_t defaultProcessorThreadsPerScan = 16; const uint32_t defaultJoinerChunkSize = 16 * 1024 * 1024; -// I estimate that the average non-cloud columnstore node has 64GB. I've seen from 16GB to 256GB. Cloud can be as low as 4GB -// However, ExeMgr has a targetRecvQueueSize hardcoded to 50,000,000, so some number greater than this makes sense. Seriously greater doesn't make sense, -// so I went with 5x. If there are a number of simultaneous queries that return giant result sets, then 0.25 GB each seems reasonable. -// This is only for the return queue. We still need room for all the processing, and if a single node system, for ExeMgr as well. -// On small systems, I recommend we use a smaller value. -// I believe a larger value will not improve anything since at this point, we're just filling a queue much faster than it can be emptied. -// Even if we make this default larger, giant results will still eventually block. Just with less memory available for other processing. -const uint64_t defaultMaxBPPSendQueue = 250000000; // ~250MB +// I estimate that the average non-cloud columnstore node has 64GB. I've seen from 16GB to 256GB. Cloud can be +// as low as 4GB However, ExeMgr has a targetRecvQueueSize hardcoded to 50,000,000, so some number greater +// than this makes sense. Seriously greater doesn't make sense, so I went with 5x. If there are a number of +// simultaneous queries that return giant result sets, then 0.25 GB each seems reasonable. This is only for +// the return queue. We still need room for all the processing, and if a single node system, for ExeMgr as +// well. On small systems, I recommend we use a smaller value. I believe a larger value will not improve +// anything since at this point, we're just filling a queue much faster than it can be emptied. Even if we +// make this default larger, giant results will still eventually block. Just with less memory available for +// other processing. +const uint64_t defaultMaxBPPSendQueue = 250000000; // ~250MB -//bucketreuse +// bucketreuse const std::string defaultTempDiskPath = "/tmp"; -const std::string defaultWorkingDir = "."; //"/tmp"; +const std::string defaultWorkingDir = "."; //"/tmp"; -//largedatalist +// largedatalist const uint32_t defaultLDLMaxElements = 32 * 1024 * 1024; -//zdl +// zdl const uint64_t defaultMaxElementsInMem = 32 * 1024 * 1024; const uint64_t defaultNumBuckets = 128; const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; @@ -106,24 +108,24 @@ const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; const int defaultEMServerThreads = 50; const int defaultEMSecondsBetweenMemChecks = 1; const int defaultEMMaxPct = 95; -const int defaultEMPriority = 21; // @Bug 3385 +const int defaultEMPriority = 21; // @Bug 3385 const int defaultEMExecQueueSize = 20; - const uint64_t defaultInitialCapacity = 1024 * 1024; -const int defaultTWMaxBuckets = 256; -const int defaultPSCount = 0; -const int defaultConnectionsPerPrimProc = 1; +const int defaultTWMaxBuckets = 256; +const int defaultPSCount = 0; +const int defaultConnectionsPerPrimProc = 1; const uint32_t defaultLBID_Shift = 13; const uint64_t defaultExtentRows = 8 * 1024 * 1024; // DMLProc -// @bug 1886. Knocked a 0 off the default below dropping it from 4M down to 256K. Delete was consuming too much memory. -const uint64_t defaultDMLMaxDeleteRows = 256 * 1024; +// @bug 1886. Knocked a 0 off the default below dropping it from 4M down to 256K. Delete was consuming too +// much memory. +const uint64_t defaultDMLMaxDeleteRows = 256 * 1024; // Connector // @bug 2048. To control batch insert memory usage. -const uint64_t defaultRowsPerBatch = 10000; +const uint64_t defaultRowsPerBatch = 10000; /* HJ CP feedback, see bug #1465 */ const uint32_t defaultHjCPUniqueLimit = 100; @@ -143,543 +145,551 @@ const bool defaultAllowDiskAggregation = false; */ class ResourceManager { -public: + public: + /** @brief ctor + * + */ + EXPORT ResourceManager(bool runningInExeMgr = false); + static ResourceManager* instance(bool runningInExeMgr = false); + // ResourceManager(const config::Config *cf); + // ResourceManager(const std::string& config); + // passed by ExeMgr and DistributedEngineComm to MessageQueueServer or -Client + config::Config* getConfig() + { + return fConfig; + } - /** @brief ctor - * - */ - EXPORT ResourceManager(bool runningInExeMgr = false); - static ResourceManager* instance(bool runningInExeMgr = false); -// ResourceManager(const config::Config *cf); -// ResourceManager(const std::string& config); -//passed by ExeMgr and DistributedEngineComm to MessageQueueServer or -Client - config::Config* getConfig() - { - return fConfig; - } + /** @brief dtor + */ + virtual ~ResourceManager() + { + } - /** @brief dtor - */ - virtual ~ResourceManager() {} + typedef std::map MemMap; - typedef std::map MemMap; + // @MCOL-513 - Added threadpool to ExeMgr + int getEmServerThreads() const + { + return getIntVal(fExeMgrStr, "ThreadPoolSize", defaultEMServerThreads); + } + std::string getExeMgrThreadPoolDebug() const + { + return getStringVal(fExeMgrStr, "ThreadPoolDebug", "N"); + } - // @MCOL-513 - Added threadpool to ExeMgr - int getEmServerThreads() const - { - return getIntVal(fExeMgrStr, "ThreadPoolSize", defaultEMServerThreads); - } - std::string getExeMgrThreadPoolDebug() const - { - return getStringVal(fExeMgrStr, "ThreadPoolDebug", "N"); - } + int getEmSecondsBetweenMemChecks() const + { + return getUintVal(fExeMgrStr, "SecondsBetweenMemChecks", defaultEMSecondsBetweenMemChecks); + } + int getEmMaxPct() const + { + return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); + } + EXPORT int getEmPriority() const; + int getEmExecQueueSize() const + { + return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); + } - int getEmSecondsBetweenMemChecks() const - { - return getUintVal(fExeMgrStr, "SecondsBetweenMemChecks", defaultEMSecondsBetweenMemChecks); - } - int getEmMaxPct() const - { - return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); - } - EXPORT int getEmPriority() const; - int getEmExecQueueSize() const - { - return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); - } + bool getAllowDiskAggregation() const + { + return fAllowedDiskAggregation; + } - bool getAllowDiskAggregation() const - { - return fAllowedDiskAggregation; - } + uint64_t getDECConnectionsPerQuery() const + { + return fDECConnectionsPerQuery; + } - uint64_t getDECConnectionsPerQuery() const - { - return fDECConnectionsPerQuery; - } + int getHjMaxBuckets() const + { + return getUintVal(fHashJoinStr, "MaxBuckets", defaultHJMaxBuckets); + } + unsigned getHjNumThreads() const + { + return fHjNumThreads; + } // getUintVal(fHashJoinStr, "NumThreads", defaultNumThreads); } + uint64_t getHjMaxElems() const + { + return getUintVal(fHashJoinStr, "MaxElems", defaultHJMaxElems); + } + uint32_t getHjFifoSizeLargeSide() const + { + return getUintVal(fHashJoinStr, "FifoSizeLargeSide", defaultHJFifoSizeLargeSide); + } + uint32_t getHjCPUniqueLimit() const + { + return getUintVal(fHashJoinStr, "CPUniqueLimit", defaultHjCPUniqueLimit); + } + uint64_t getPMJoinMemLimit() const + { + return pmJoinMemLimit; + } - int getHjMaxBuckets() const - { - return getUintVal(fHashJoinStr, "MaxBuckets", defaultHJMaxBuckets); - } - unsigned getHjNumThreads() const - { - return fHjNumThreads; - } //getUintVal(fHashJoinStr, "NumThreads", defaultNumThreads); } - uint64_t getHjMaxElems() const - { - return getUintVal(fHashJoinStr, "MaxElems", defaultHJMaxElems); - } - uint32_t getHjFifoSizeLargeSide() const - { - return getUintVal(fHashJoinStr, "FifoSizeLargeSide", defaultHJFifoSizeLargeSide); - } - uint32_t getHjCPUniqueLimit() const - { - return getUintVal(fHashJoinStr, "CPUniqueLimit", defaultHjCPUniqueLimit); - } - uint64_t getPMJoinMemLimit() const - { - return pmJoinMemLimit; - } + uint32_t getJLFlushInterval() const + { + return getUintVal(fJobListStr, "FlushInterval", defaultFlushInterval); + } + uint32_t getJlFifoSize() const + { + return getUintVal(fJobListStr, "FifoSize", defaultFifoSize); + } + uint32_t getJlScanLbidReqLimit() const + { + return getUintVal(fJobListStr, "ScanLbidReqLimit", defaultScanLbidReqLimit); + } + uint32_t getJlScanLbidReqThreshold() const + { + return getUintVal(fJobListStr, "ScanLbidReqThreshold", defaultScanLbidReqThreshold); + } - uint32_t getJLFlushInterval() const - { - return getUintVal(fJobListStr, "FlushInterval", defaultFlushInterval); - } - uint32_t getJlFifoSize() const - { - return getUintVal(fJobListStr, "FifoSize", defaultFifoSize); - } - uint32_t getJlScanLbidReqLimit() const - { - return getUintVal(fJobListStr, "ScanLbidReqLimit", defaultScanLbidReqLimit); - } - uint32_t getJlScanLbidReqThreshold() const - { - return getUintVal(fJobListStr, "ScanLbidReqThreshold", defaultScanLbidReqThreshold); - } + // @MCOL-513 - Added threadpool to JobSteps + int getJLThreadPoolSize() const + { + return getIntVal(fJobListStr, "ThreadPoolSize", defaultJLThreadPoolSize); + } + std::string getJlThreadPoolDebug() const + { + return getStringVal(fJobListStr, "ThreadPoolDebug", "N"); + } + std::string getDMLJlThreadPoolDebug() const + { + return getStringVal(fJobListStr, "DMLThreadPoolDebug", "N"); + } - // @MCOL-513 - Added threadpool to JobSteps - int getJLThreadPoolSize() const - { - return getIntVal(fJobListStr, "ThreadPoolSize", defaultJLThreadPoolSize); - } - std::string getJlThreadPoolDebug() const - { - return getStringVal(fJobListStr, "ThreadPoolDebug", "N"); - } - std::string getDMLJlThreadPoolDebug() const - { - return getStringVal(fJobListStr, "DMLThreadPoolDebug", "N"); - } + // @bug 1264 - Added LogicalBlocksPerScan configurable which determines the number of blocks contained in + // each BPS scan request. + uint32_t getJlLogicalBlocksPerScan() const + { + return getUintVal(fJobListStr, "LogicalBlocksPerScan", defaultLogicalBlocksPerScan); + } + uint32_t getJlProjectBlockReqLimit() const + { + return getUintVal(fJobListStr, "ProjectBlockReqLimit", defaultProjectBlockReqLimit); + } + uint32_t getJlProjectBlockReqThreshold() const + { + return getUintVal(fJobListStr, "ProjectBlockReqThreshold", defaultProjectBlockReqThreshold); + } + uint32_t getJlNumScanReceiveThreads() const + { + return fJlNumScanReceiveThreads; + } // getUintVal(fJobListStr, "NumScanReceiveThreads", defaultScanReceiveThreads); } - // @bug 1264 - Added LogicalBlocksPerScan configurable which determines the number of blocks contained in each BPS scan request. - uint32_t getJlLogicalBlocksPerScan() const - { - return getUintVal(fJobListStr, "LogicalBlocksPerScan", defaultLogicalBlocksPerScan); - } - uint32_t getJlProjectBlockReqLimit() const - { - return getUintVal(fJobListStr, "ProjectBlockReqLimit", defaultProjectBlockReqLimit ); - } - uint32_t getJlProjectBlockReqThreshold() const - { - return getUintVal(fJobListStr, "ProjectBlockReqThreshold", defaultProjectBlockReqThreshold); - } - uint32_t getJlNumScanReceiveThreads() const - { - return fJlNumScanReceiveThreads; - } //getUintVal(fJobListStr, "NumScanReceiveThreads", defaultScanReceiveThreads); } + // @bug 1424,1298 + uint32_t getJlProcessorThreadsPerScan() const + { + return fJlProcessorThreadsPerScan; + } // getUintVal(fJobListStr,"ProcessorThreadsPerScan", defaultProcessorThreadsPerScan); } + uint32_t getJlRequestSize() const + { + return getUintVal(fJobListStr, "RequestSize", defaultRequestSize); + } + uint32_t getJlMaxOutstandingRequests() const + { + return fJlMaxOutstandingRequests; + // getUintVal(fJobListStr, "MaxOutstandingRequests", defaultMaxOutstandingRequests); + } + uint32_t getJlJoinerChunkSize() const + { + return getUintVal(fJobListStr, "JoinerChunkSize", defaultJoinerChunkSize); + } - // @bug 1424,1298 - uint32_t getJlProcessorThreadsPerScan() const - { - return fJlProcessorThreadsPerScan; - } //getUintVal(fJobListStr,"ProcessorThreadsPerScan", defaultProcessorThreadsPerScan); } - uint32_t getJlRequestSize() const - { - return getUintVal(fJobListStr, "RequestSize", defaultRequestSize ); - } - uint32_t getJlMaxOutstandingRequests() const - { - return fJlMaxOutstandingRequests; - //getUintVal(fJobListStr, "MaxOutstandingRequests", defaultMaxOutstandingRequests); - } - uint32_t getJlJoinerChunkSize() const - { - return getUintVal(fJobListStr, "JoinerChunkSize", defaultJoinerChunkSize); - } + int getPsCount() const + { + return getUintVal(fPrimitiveServersStr, "Count", defaultPSCount); + } + int getPsConnectionsPerPrimProc() const + { + return getUintVal(fPrimitiveServersStr, "ConnectionsPerPrimProc", defaultConnectionsPerPrimProc); + } + uint32_t getPsLBID_Shift() const + { + return getUintVal(fPrimitiveServersStr, "LBID_Shift", defaultLBID_Shift); + } - int getPsCount() const - { - return getUintVal(fPrimitiveServersStr, "Count", defaultPSCount ); - } - int getPsConnectionsPerPrimProc() const - { - return getUintVal(fPrimitiveServersStr, "ConnectionsPerPrimProc", defaultConnectionsPerPrimProc); - } - uint32_t getPsLBID_Shift() const - { - return getUintVal(fPrimitiveServersStr, "LBID_Shift", defaultLBID_Shift ); - } + std::string getScTempDiskPath() const + { + return startup::StartUp::tmpDir(); + } + uint64_t getScTempSaveSize() const + { + return getUintVal(fSystemConfigStr, "TempSaveSize", defaultTempSaveSize); + } + std::string getScWorkingDir() const + { + return startup::StartUp::tmpDir(); + } - std::string getScTempDiskPath() const - { - return startup::StartUp::tmpDir(); - } - uint64_t getScTempSaveSize() const - { - return getUintVal(fSystemConfigStr, "TempSaveSize", defaultTempSaveSize); - } - std::string getScWorkingDir() const - { - return startup::StartUp::tmpDir(); - } + uint32_t getTwMaxSize() const + { + return getUintVal(fTupleWSDLStr, "MaxSize", defaultTupleDLMaxSize); + } + uint64_t getTwInitialCapacity() const + { + return getUintVal(fTupleWSDLStr, "InitialCapacity", defaultInitialCapacity); + } + int getTwMaxBuckets() const + { + return getUintVal(fTupleWSDLStr, "MaxBuckets", defaultTWMaxBuckets); + } + uint8_t getTwNumThreads() const + { + return fTwNumThreads; + } // getUintVal(fTupleWSDLStr, "NumThreads", defaultNumThreads ); } + uint64_t getZdl_MaxElementsInMem() const + { + return getUintVal(fZDLStr, "ZDL_MaxElementsInMem", defaultMaxElementsInMem); + } + uint64_t getZdl_MaxElementsPerBucket() const + { + return getUintVal(fZDLStr, "ZDL_MaxElementsPerBucket", defaultMaxElementsPerBuckert); + } - uint32_t getTwMaxSize() const - { - return getUintVal(fTupleWSDLStr, "MaxSize", defaultTupleDLMaxSize ); - } - uint64_t getTwInitialCapacity() const - { - return getUintVal(fTupleWSDLStr, "InitialCapacity", defaultInitialCapacity ); - } - int getTwMaxBuckets () const - { - return getUintVal(fTupleWSDLStr, "MaxBuckets", defaultTWMaxBuckets ); - } - uint8_t getTwNumThreads() const - { - return fTwNumThreads; - } //getUintVal(fTupleWSDLStr, "NumThreads", defaultNumThreads ); } - uint64_t getZdl_MaxElementsInMem() const - { - return getUintVal(fZDLStr, "ZDL_MaxElementsInMem", defaultMaxElementsInMem ); - } - uint64_t getZdl_MaxElementsPerBucket () const - { - return getUintVal(fZDLStr, "ZDL_MaxElementsPerBucket", defaultMaxElementsPerBuckert ); - } + uint64_t getExtentRows() const + { + return getUintVal(fExtentMapStr, "ExtentRows", defaultExtentRows); + } - uint64_t getExtentRows() const - { - return getUintVal(fExtentMapStr, "ExtentRows", defaultExtentRows ); - } + uint32_t getDBRootCount() const + { + return getUintVal(fSystemConfigStr, "DBRootCount", 1); + } + uint32_t getPMCount() const + { + return getUintVal(fPrimitiveServersStr, "Count", 1); + } - uint32_t getDBRootCount() const - { - return getUintVal(fSystemConfigStr, "DBRootCount", 1); - } - uint32_t getPMCount() const - { - return getUintVal(fPrimitiveServersStr, "Count", 1); - } + std::vector getHbrPredicate() const + { + std::vector columns; + fConfig->getConfig(fHashBucketReuseStr, "Predicate", columns); + return columns; + } - std::vector getHbrPredicate() const - { - std::vector columns; - fConfig->getConfig(fHashBucketReuseStr, "Predicate", columns); - return columns; - } + uint64_t getDMLMaxDeleteRows() const + { + return getUintVal(fDMLProcStr, "MaxDeleteRows", defaultDMLMaxDeleteRows); + } - uint64_t getDMLMaxDeleteRows () const - { - return getUintVal(fDMLProcStr, "MaxDeleteRows", defaultDMLMaxDeleteRows); - } + uint64_t getRowsPerBatch() const + { + return getUintVal(fBatchInsertStr, "RowsPerBatch", defaultRowsPerBatch); + } - uint64_t getRowsPerBatch() const - { - return getUintVal(fBatchInsertStr, "RowsPerBatch", defaultRowsPerBatch); - } + uint8_t getUseCpimport() const + { + int val = getIntVal(fBatchInsertStr, "UseCpimport", defaultUseCpimport); + return val; + } - uint8_t getUseCpimport() const - { - int val = getIntVal(fBatchInsertStr, "UseCpimport", defaultUseCpimport); - return val; - } + uint64_t getOrderByLimitMaxMemory() const + { + return getUintVal(fOrderByLimitStr, "MaxMemory", defaultOrderByLimitMaxMemory); + } - uint64_t getOrderByLimitMaxMemory() const - { - return getUintVal(fOrderByLimitStr, "MaxMemory", defaultOrderByLimitMaxMemory); - } + uint64_t getDECThrottleThreshold() const + { + return getUintVal(fJobListStr, "DECThrottleThreshold", defaultDECThrottleThreshold); + } - uint64_t getDECThrottleThreshold() const - { - return getUintVal(fJobListStr, "DECThrottleThreshold", defaultDECThrottleThreshold); - } + uint64_t getMaxBPPSendQueue() const + { + return fMaxBPPSendQueue; + } - uint64_t getMaxBPPSendQueue() const - { - return fMaxBPPSendQueue; - } - - EXPORT void emServerThreads(); - EXPORT void emServerQueueSize(); - EXPORT void emSecondsBetweenMemChecks(); - EXPORT void emMaxPct(); - EXPORT void emPriority(); - EXPORT void emExecQueueSize(); + EXPORT void emServerThreads(); + EXPORT void emServerQueueSize(); + EXPORT void emSecondsBetweenMemChecks(); + EXPORT void emMaxPct(); + EXPORT void emPriority(); + EXPORT void emExecQueueSize(); - EXPORT void hjNumThreads(); - EXPORT void hjMaxBuckets(); - EXPORT void hjMaxElems(); - EXPORT void hjFifoSizeLargeSide(); - EXPORT void hjPmMaxMemorySmallSide(); + EXPORT void hjNumThreads(); + EXPORT void hjMaxBuckets(); + EXPORT void hjMaxElems(); + EXPORT void hjFifoSizeLargeSide(); + EXPORT void hjPmMaxMemorySmallSide(); - /* new HJ/Union/Aggregation mem interface, used by TupleBPS */ - /* sessionLimit is a pointer to the var holding the session-scope limit, should be JobInfo.umMemLimit - for the query. */ - /* Temporary parameter 'patience', will wait for up to 10s to get the memory. */ - EXPORT bool getMemory(int64_t amount, boost::shared_ptr& sessionLimit, bool patience = true); - EXPORT bool getMemory(int64_t amount, bool patience = true); - inline void returnMemory(int64_t amount) - { - atomicops::atomicAdd(&totalUmMemLimit, amount); - } - inline void returnMemory(int64_t amount, boost::shared_ptr& sessionLimit) - { - atomicops::atomicAdd(&totalUmMemLimit, amount); - sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount): 0; - } - inline int64_t availableMemory() const - { - return totalUmMemLimit; - } + /* new HJ/Union/Aggregation mem interface, used by TupleBPS */ + /* sessionLimit is a pointer to the var holding the session-scope limit, should be JobInfo.umMemLimit + for the query. */ + /* Temporary parameter 'patience', will wait for up to 10s to get the memory. */ + EXPORT bool getMemory(int64_t amount, boost::shared_ptr& sessionLimit, bool patience = true); + EXPORT bool getMemory(int64_t amount, bool patience = true); + inline void returnMemory(int64_t amount) + { + atomicops::atomicAdd(&totalUmMemLimit, amount); + } + inline void returnMemory(int64_t amount, boost::shared_ptr& sessionLimit) + { + atomicops::atomicAdd(&totalUmMemLimit, amount); + sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0; + } + inline int64_t availableMemory() const + { + return totalUmMemLimit; + } - /* old HJ mem interface, used by HashJoin */ - uint64_t getHjPmMaxMemorySmallSide(uint32_t sessionID) - { - return fHJPmMaxMemorySmallSideSessionMap.getSessionResource(sessionID); - } - uint64_t getHjUmMaxMemorySmallSide(uint32_t sessionID) - { - return fHJUmMaxMemorySmallSideDistributor.getSessionResource(sessionID); - } - uint64_t getHjTotalUmMaxMemorySmallSide() const - { - return fHJUmMaxMemorySmallSideDistributor.getTotalResource(); - } + /* old HJ mem interface, used by HashJoin */ + uint64_t getHjPmMaxMemorySmallSide(uint32_t sessionID) + { + return fHJPmMaxMemorySmallSideSessionMap.getSessionResource(sessionID); + } + uint64_t getHjUmMaxMemorySmallSide(uint32_t sessionID) + { + return fHJUmMaxMemorySmallSideDistributor.getSessionResource(sessionID); + } + uint64_t getHjTotalUmMaxMemorySmallSide() const + { + return fHJUmMaxMemorySmallSideDistributor.getTotalResource(); + } - EXPORT void addHJUmMaxSmallSideMap(uint32_t sessionID, uint64_t mem); + EXPORT void addHJUmMaxSmallSideMap(uint32_t sessionID, uint64_t mem); - void removeHJUmMaxSmallSideMap(uint32_t sessionID) - { - fHJUmMaxMemorySmallSideDistributor.removeSession(sessionID); - } + void removeHJUmMaxSmallSideMap(uint32_t sessionID) + { + fHJUmMaxMemorySmallSideDistributor.removeSession(sessionID); + } - EXPORT void addHJPmMaxSmallSideMap(uint32_t sessionID, uint64_t mem); - void removeHJPmMaxSmallSideMap(uint32_t sessionID) - { - fHJPmMaxMemorySmallSideSessionMap.removeSession(sessionID); - } + EXPORT void addHJPmMaxSmallSideMap(uint32_t sessionID, uint64_t mem); + void removeHJPmMaxSmallSideMap(uint32_t sessionID) + { + fHJPmMaxMemorySmallSideSessionMap.removeSession(sessionID); + } - void removeSessionMaps(uint32_t sessionID) - { - fHJPmMaxMemorySmallSideSessionMap.removeSession(sessionID); - fHJUmMaxMemorySmallSideDistributor.removeSession(sessionID); - } + void removeSessionMaps(uint32_t sessionID) + { + fHJPmMaxMemorySmallSideSessionMap.removeSession(sessionID); + fHJUmMaxMemorySmallSideDistributor.removeSession(sessionID); + } - uint64_t requestHJMaxMemorySmallSide(uint32_t sessionID, uint64_t amount) - { - return fHJUmMaxMemorySmallSideDistributor.requestResource(sessionID, amount); - } + uint64_t requestHJMaxMemorySmallSide(uint32_t sessionID, uint64_t amount) + { + return fHJUmMaxMemorySmallSideDistributor.requestResource(sessionID, amount); + } - uint64_t requestHJUmMaxMemorySmallSide(uint32_t sessionID) - { - return fHJUmMaxMemorySmallSideDistributor.requestResource(sessionID); - } - void returnHJUmMaxMemorySmallSide(uint64_t mem) - { - fHJUmMaxMemorySmallSideDistributor.returnResource(mem); - } + uint64_t requestHJUmMaxMemorySmallSide(uint32_t sessionID) + { + return fHJUmMaxMemorySmallSideDistributor.requestResource(sessionID); + } + void returnHJUmMaxMemorySmallSide(uint64_t mem) + { + fHJUmMaxMemorySmallSideDistributor.returnResource(mem); + } + EXPORT void jlFlushInterval(); + EXPORT void jlFifoSize(); + EXPORT void jlScanLbidReqLimit(); + EXPORT void jlScanLbidReqThreshold(); + EXPORT void jlProjectBlockReqLimit(); + EXPORT void jlProjectBlockReqThreshold(); + EXPORT void jlNumScanReceiveThreads(); - EXPORT void jlFlushInterval(); - EXPORT void jlFifoSize(); - EXPORT void jlScanLbidReqLimit(); - EXPORT void jlScanLbidReqThreshold(); - EXPORT void jlProjectBlockReqLimit(); - EXPORT void jlProjectBlockReqThreshold(); - EXPORT void jlNumScanReceiveThreads(); + EXPORT void psCount(); + EXPORT void psConnectionsPerPrimProc(); + EXPORT void psLBID_Shift(); - EXPORT void psCount(); - EXPORT void psConnectionsPerPrimProc() ; - EXPORT void psLBID_Shift(); + EXPORT void scTempDiskPath(); + EXPORT void scTempSaveSize(); + EXPORT void scWorkingDir(); - EXPORT void scTempDiskPath(); - EXPORT void scTempSaveSize() ; - EXPORT void scWorkingDir(); + EXPORT void twMaxSize(); + EXPORT void twInitialCapacity(); + EXPORT void twMaxBuckets(); + EXPORT void twNumThreads(); - EXPORT void twMaxSize(); - EXPORT void twInitialCapacity() ; - EXPORT void twMaxBuckets () ; - EXPORT void twNumThreads(); + EXPORT void zdl_MaxElementsInMem(); + EXPORT void zdl_MaxElementsPerBucket(); - EXPORT void zdl_MaxElementsInMem(); - EXPORT void zdl_MaxElementsPerBucket() ; + EXPORT void hbrPredicate(); - EXPORT void hbrPredicate(); + void setTraceFlags(uint32_t flags) + { + fTraceFlags = flags; + fHJUmMaxMemorySmallSideDistributor.setTrace( + ((fTraceFlags & execplan::CalpontSelectExecutionPlan::TRACE_RESRCMGR) != 0)); + } + bool rmtraceOn() const + { + return ((fTraceFlags & execplan::CalpontSelectExecutionPlan::TRACE_RESRCMGR) != 0); + } - void setTraceFlags(uint32_t flags) - { - fTraceFlags = flags; - fHJUmMaxMemorySmallSideDistributor.setTrace(((fTraceFlags & execplan::CalpontSelectExecutionPlan::TRACE_RESRCMGR) != 0)); - } - bool rmtraceOn() const - { - return ((fTraceFlags & execplan::CalpontSelectExecutionPlan::TRACE_RESRCMGR) != 0); - } + void numCores(unsigned numCores) + { + fNumCores = numCores; + } + unsigned numCores() const + { + return fNumCores; + } - void numCores(unsigned numCores) - { - fNumCores = numCores; - } - unsigned numCores() const - { - return fNumCores; - } + void aggNumThreads(uint32_t numThreads) + { + fAggNumThreads = numThreads; + } + uint32_t aggNumThreads() const + { + return fAggNumThreads; + } - void aggNumThreads(uint32_t numThreads) - { - fAggNumThreads = numThreads; - } - uint32_t aggNumThreads() const - { - return fAggNumThreads; - } + void aggNumBuckets(uint32_t numBuckets) + { + fAggNumBuckets = numBuckets; + } + uint32_t aggNumBuckets() const + { + return fAggNumBuckets; + } - void aggNumBuckets(uint32_t numBuckets) - { - fAggNumBuckets = numBuckets; - } - uint32_t aggNumBuckets() const - { - return fAggNumBuckets; - } + void aggNumRowGroups(uint32_t numRowGroups) + { + fAggNumRowGroups = numRowGroups; + } + uint32_t aggNumRowGroups() const + { + return fAggNumRowGroups; + } - void aggNumRowGroups(uint32_t numRowGroups) - { - fAggNumRowGroups = numRowGroups; - } - uint32_t aggNumRowGroups() const - { - return fAggNumRowGroups; - } + void windowFunctionThreads(uint32_t n) + { + fWindowFunctionThreads = n; + } + uint32_t windowFunctionThreads() const + { + return fWindowFunctionThreads; + } - void windowFunctionThreads(uint32_t n) - { - fWindowFunctionThreads = n; - } - uint32_t windowFunctionThreads() const - { - return fWindowFunctionThreads; - } + bool useHdfs() const + { + return fUseHdfs; + } - bool useHdfs() const - { - return fUseHdfs; - } + EXPORT bool getMysqldInfo(std::string& h, std::string& u, std::string& w, unsigned int& p) const; + EXPORT bool queryStatsEnabled() const; + EXPORT bool userPriorityEnabled() const; - EXPORT bool getMysqldInfo(std::string& h, std::string& u, std::string& w, unsigned int& p) const; - EXPORT bool queryStatsEnabled() const; - EXPORT bool userPriorityEnabled() const; + uint64_t getConfiguredUMMemLimit() const + { + return configuredUmMemLimit; + } - uint64_t getConfiguredUMMemLimit() const - { - return configuredUmMemLimit; - } -private: + private: + void logResourceChangeMessage(logging::LOG_TYPE logType, uint32_t sessionID, uint64_t newvalue, + uint64_t value, const std::string& source, logging::Message::MessageID mid); + /** @brief get name's value from section + * + * get name's value from section in the current config file or default value . + * @param section the name of the config file section to search + * @param name the param name whose value is to be returned + * @param defVal the default value returned if the value is missing + */ + std::string getStringVal(const std::string& section, const std::string& name, + const std::string& defVal) const; - void logResourceChangeMessage(logging::LOG_TYPE logType, uint32_t sessionID, uint64_t newvalue, uint64_t value, const std::string& source, logging::Message::MessageID mid); - /** @brief get name's value from section - * - * get name's value from section in the current config file or default value . - * @param section the name of the config file section to search - * @param name the param name whose value is to be returned - * @param defVal the default value returned if the value is missing - */ - std::string getStringVal(const std::string& section, const std::string& name, const std::string& defVal) const; + template + IntType getUintVal(const std::string& section, const std::string& name, IntType defval) const; - template - IntType getUintVal(const std::string& section, const std::string& name, IntType defval) const; + template + IntType getIntVal(const std::string& section, const std::string& name, IntType defval) const; - template - IntType getIntVal(const std::string& section, const std::string& name, IntType defval) const; + bool getBoolVal(const std::string& section, const std::string& name, bool defval) const; - bool getBoolVal(const std::string& section, const std::string& name, bool defval) const; + void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0, + uint32_t sessionId = 0); - void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0, uint32_t sessionId = 0); + /*static const*/ std::string fExeMgrStr; + static const std::string fHashJoinStr; + static const std::string fHashBucketReuseStr; + static const std::string fJobListStr; + static const std::string fPrimitiveServersStr; + /*static const*/ std::string fSystemConfigStr; + static const std::string fTupleWSDLStr; + static const std::string fZDLStr; + static const std::string fExtentMapStr; + /*static const*/ std::string fDMLProcStr; + /*static const*/ std::string fBatchInsertStr; + static const std::string fOrderByLimitStr; + static const std::string fRowAggregationStr; + config::Config* fConfig; + static ResourceManager* fInstance; + uint32_t fTraceFlags; - /*static const*/ std::string fExeMgrStr; - static const std::string fHashJoinStr; - static const std::string fHashBucketReuseStr; - static const std::string fJobListStr; - static const std::string fPrimitiveServersStr; - /*static const*/ std::string fSystemConfigStr; - static const std::string fTupleWSDLStr; - static const std::string fZDLStr; - static const std::string fExtentMapStr; - /*static const*/ std::string fDMLProcStr; - /*static const*/ std::string fBatchInsertStr; - static const std::string fOrderByLimitStr; - static const std::string fRowAggregationStr; - config::Config* fConfig; - static ResourceManager* fInstance; - uint32_t fTraceFlags; + unsigned fNumCores; + unsigned fHjNumThreads; + uint32_t fJlProcessorThreadsPerScan; + uint32_t fJlNumScanReceiveThreads; + uint8_t fTwNumThreads; + uint32_t fJlMaxOutstandingRequests; - unsigned fNumCores; - unsigned fHjNumThreads; - uint32_t fJlProcessorThreadsPerScan; - uint32_t fJlNumScanReceiveThreads; - uint8_t fTwNumThreads; - uint32_t fJlMaxOutstandingRequests; + /* old HJ support */ + ResourceDistributor fHJUmMaxMemorySmallSideDistributor; + LockedSessionMap fHJPmMaxMemorySmallSideSessionMap; - /* old HJ support */ - ResourceDistributor fHJUmMaxMemorySmallSideDistributor; - LockedSessionMap fHJPmMaxMemorySmallSideSessionMap; + /* new HJ/Union/Aggregation support */ + volatile int64_t totalUmMemLimit; // mem limit for join, union, and aggregation on the UM + int64_t configuredUmMemLimit; + uint64_t pmJoinMemLimit; // mem limit on individual PM joins - /* new HJ/Union/Aggregation support */ - volatile int64_t totalUmMemLimit; // mem limit for join, union, and aggregation on the UM - int64_t configuredUmMemLimit; - uint64_t pmJoinMemLimit; // mem limit on individual PM joins + /* multi-thread aggregate */ + uint32_t fAggNumThreads; + uint32_t fAggNumBuckets; + uint32_t fAggNumRowGroups; - /* multi-thread aggregate */ - uint32_t fAggNumThreads; - uint32_t fAggNumBuckets; - uint32_t fAggNumRowGroups; + // window function + uint32_t fWindowFunctionThreads; - // window function - uint32_t fWindowFunctionThreads; - - - bool isExeMgr; - bool fUseHdfs; - bool fAllowedDiskAggregation{false}; - uint64_t fDECConnectionsPerQuery; - uint64_t fMaxBPPSendQueue = 250000000; + bool isExeMgr; + bool fUseHdfs; + bool fAllowedDiskAggregation{false}; + uint64_t fDECConnectionsPerQuery; + uint64_t fMaxBPPSendQueue = 250000000; }; - -inline std::string ResourceManager::getStringVal(const std::string& section, const std::string& name, const std::string& defval) const +inline std::string ResourceManager::getStringVal(const std::string& section, const std::string& name, + const std::string& defval) const { - std::string val = fConfig->getConfig(section, name); + std::string val = fConfig->getConfig(section, name); #ifdef DEBUGRM - std::cout << "RM getStringVal for " << section << " : " << name << " val: " << val << " default: " << defval << std::endl; + std::cout << "RM getStringVal for " << section << " : " << name << " val: " << val << " default: " << defval + << std::endl; #endif - return (0 == val.length() ? defval : val); + return (0 == val.length() ? defval : val); } -template -inline IntType ResourceManager::getUintVal(const std::string& section, const std::string& name, IntType defval) const +template +inline IntType ResourceManager::getUintVal(const std::string& section, const std::string& name, + IntType defval) const { - IntType val = fConfig->uFromText(fConfig->getConfig(section, name)); + IntType val = fConfig->uFromText(fConfig->getConfig(section, name)); #ifdef DEBUGRM - std::cout << "RM getUintVal val: " << section << " : " << name << " val: " << val << " default: " << defval << std::endl; + std::cout << "RM getUintVal val: " << section << " : " << name << " val: " << val << " default: " << defval + << std::endl; #endif - return ( 0 == val ? defval : val ); - + return (0 == val ? defval : val); } -template -inline IntType ResourceManager::getIntVal(const std::string& section, const std::string& name, IntType defval) const +template +inline IntType ResourceManager::getIntVal(const std::string& section, const std::string& name, + IntType defval) const { - std::string retStr = fConfig->getConfig(section, name); + std::string retStr = fConfig->getConfig(section, name); #ifdef DEBUGRM - std::cout << "RM getIntVal val: " << section << " : " << name << " val: " << retStr << " default: " << defval << std::endl; + std::cout << "RM getIntVal val: " << section << " : " << name << " val: " << retStr + << " default: " << defval << std::endl; #endif - return ( 0 == retStr.length() ? defval : fConfig->fromText(retStr) ); + return (0 == retStr.length() ? defval : fConfig->fromText(retStr)); } -inline bool ResourceManager::getBoolVal(const std::string& section, const std::string& name, bool defval) const +inline bool ResourceManager::getBoolVal(const std::string& section, const std::string& name, + bool defval) const { auto retStr = fConfig->getConfig(section, name); - return ( 0 == retStr.length() ? defval : (retStr == "y" || retStr == "Y") ); + return (0 == retStr.length() ? defval : (retStr == "y" || retStr == "Y")); } -} +} // namespace joblist #undef EXPORT #endif - diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 133d1df64..9babae7c3 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -15,11 +15,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - /****************************************************************************** -* $Id: rowestimator.cpp 5642 2009-08-10 21:04:59Z wweeks $ -* -******************************************************************************/ + * $Id: rowestimator.cpp 5642 2009-08-10 21:04:59Z wweeks $ + * + ******************************************************************************/ #include #include "primitivemsg.h" #include "blocksize.h" @@ -41,628 +40,588 @@ using namespace execplan; using namespace BRM; using namespace logging; - namespace joblist { - // Returns the sum of the days through a particular month where 1 is Jan, 2 is Feb, ... -// This is used for converting a Calpont date to an integer representing the day number since the year 0 for use in -// calculating the number of distinct days in a range. It doesn't account for leap years as these are rough estimates -// and only need to be accurate within an order of magnitude. +// This is used for converting a Calpont date to an integer representing the day number since the year 0 for +// use in calculating the number of distinct days in a range. It doesn't account for leap years as these are +// rough estimates and only need to be accurate within an order of magnitude. uint32_t RowEstimator::daysThroughMonth(uint32_t mth) { - switch (mth) - { - case 0: - return 0; + switch (mth) + { + case 0: return 0; - case 1: - return 31; + case 1: return 31; - case 2: - return 59; + case 2: return 59; - case 3: - return 90; + case 3: return 90; - case 4: - return 120; + case 4: return 120; - case 5: - return 151; + case 5: return 151; - case 6: - return 181; + case 6: return 181; - case 7: - return 212; + case 7: return 212; - case 8: - return 243; + case 8: return 243; - case 9: - return 273; + case 9: return 273; - case 10: - return 304; + case 10: return 304; - case 11: - return 334; + case 11: return 334; - default: - return 365; - } + default: return 365; + } } // This function takes a column value and if necessary adjusts it based on rules defined in the requirements. // The values are adjusted so that one can be subtracted from another to find a range, compare, etc. -uint64_t RowEstimator::adjustValue(const execplan::CalpontSystemCatalog::ColType& ct, - const uint64_t& value) +uint64_t RowEstimator::adjustValue(const execplan::CalpontSystemCatalog::ColType& ct, const uint64_t& value) { - switch (ct.colDataType) + switch (ct.colDataType) + { + // Use day precision for dates. We'll use day relative to the year 0 without worrying about leap + // years. This is for row count estimation and we are close enough for hand grenades. + case CalpontSystemCatalog::DATE: { - // Use day precision for dates. We'll use day relative to the year 0 without worrying about leap - // years. This is for row count estimation and we are close enough for hand grenades. - case CalpontSystemCatalog::DATE: - { - dataconvert::Date dt(value); - return dt.year * 365 + daysThroughMonth(dt.month - 1) + dt.day; - } - - // Use second precision for datetime estimates. We'll use number of seconds since the year 0 - // without worrying about leap years. - case CalpontSystemCatalog::DATETIME: - { - dataconvert::DateTime dtm(value); - // 86,400 seconds in a day. - return (dtm.year * 365 + daysThroughMonth(dtm.month - 1) + dtm.day - 1) * 86400 + - dtm.hour * 3600 + dtm.minute * 60 + dtm.second; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp ts(value); - return ts.second; - } - - // Use the first character only for estimating chars and varchar ranges. - // TODO: Use dictionary column HWM for dictionary columns. - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - // Last byte is the first character in the string. - return (0xFF & value); - - default: - return value; + dataconvert::Date dt(value); + return dt.year * 365 + daysThroughMonth(dt.month - 1) + dt.day; } + + // Use second precision for datetime estimates. We'll use number of seconds since the year 0 + // without worrying about leap years. + case CalpontSystemCatalog::DATETIME: + { + dataconvert::DateTime dtm(value); + // 86,400 seconds in a day. + return (dtm.year * 365 + daysThroughMonth(dtm.month - 1) + dtm.day - 1) * 86400 + dtm.hour * 3600 + + dtm.minute * 60 + dtm.second; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp ts(value); + return ts.second; + } + + // Use the first character only for estimating chars and varchar ranges. + // TODO: Use dictionary column HWM for dictionary columns. + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + // Last byte is the first character in the string. + return (0xFF & value); + + default: return value; + } } // Estimates the number of distinct values given a min/max range. When the range has not been set, // rules from the requirements are used based on the column type. -template -uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const T& min, - const T& max, - const char cpStatus) +template +uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, const T& min, + const T& max, const char cpStatus) { - T ret = 10; + T ret = 10; - // If no casual partitioning info available for extent. These rules were defined in the requirements. - if (cpStatus != BRM::CP_VALID) + // If no casual partitioning info available for extent. These rules were defined in the requirements. + if (cpStatus != BRM::CP_VALID) + { + switch (ct.colDataType) { - switch (ct.colDataType) - { - case CalpontSystemCatalog::BIT: - return 2; + case CalpontSystemCatalog::BIT: return 2; - // Return limit/2 for integers where limit is number of possible values. - case CalpontSystemCatalog::TINYINT: - return (1 << 8) / 2; + // Return limit/2 for integers where limit is number of possible values. + case CalpontSystemCatalog::TINYINT: return (1 << 8) / 2; - case CalpontSystemCatalog::UTINYINT: - return (1 << 8); + case CalpontSystemCatalog::UTINYINT: return (1 << 8); - case CalpontSystemCatalog::SMALLINT: - return (1 << 16) / 2; + case CalpontSystemCatalog::SMALLINT: return (1 << 16) / 2; - case CalpontSystemCatalog::USMALLINT: - return (1 << 16); + case CalpontSystemCatalog::USMALLINT: return (1 << 16); - // Next group all have range greater than 8M (# of rows in an extent), use 8M/2 as the estimate. - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return fRowsPerExtent / 2; + // Next group all have range greater than 8M (# of rows in an extent), use 8M/2 as the estimate. + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return fRowsPerExtent / 2; - // Use 1000 for dates. - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - return 1000; + // Use 1000 for dates. + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: return 1000; - // Use 10 for CHARs and VARCHARs. We'll use 10 for whatever else. - // Todo: Requirement say use dictionary HWM if dictionary column, not sure that will be doable. - default: - return 10; - } - } - else - { - ret = max - min + 1; + // Use 10 for CHARs and VARCHARs. We'll use 10 for whatever else. + // Todo: Requirement say use dictionary HWM if dictionary column, not sure that will be doable. + default: return 10; } + } + else + { + ret = max - min + 1; + } - if (ret > fRowsPerExtent) - { - ret = fRowsPerExtent; - } + if (ret > fRowsPerExtent) + { + ret = fRowsPerExtent; + } - return ret; + return ret; } -// Returns a floating point number between 0 and 1 representing the percentage of matching rows for the given predicate against -// the given range. This function is used for estimating an individual operation such as col1 = 2. -template -float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, - char op, uint8_t lcf, +// Returns a floating point number between 0 and 1 representing the percentage of matching rows for the given +// predicate against the given range. This function is used for estimating an individual operation such as +// col1 = 2. +template +float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, uint32_t distinctValues, char cpStatus, const execplan::CalpontSystemCatalog::ColType& ct) { - float factor = 1.0; + float factor = 1.0; - switch (op) - { - case COMPARE_LT: - case COMPARE_NGE: - if (cpStatus == BRM::CP_VALID) - { - if (!ct.isWideDecimalType()) - factor = (1.0 * value - min) / (max - min + 1); - else - factor = ((float128_t) value - min) / (max - min + 1); - } + switch (op) + { + case COMPARE_LT: + case COMPARE_NGE: + if (cpStatus == BRM::CP_VALID) + { + if (!ct.isWideDecimalType()) + factor = (1.0 * value - min) / (max - min + 1); + else + factor = ((float128_t)value - min) / (max - min + 1); + } - break; + break; - case COMPARE_LE: - case COMPARE_NGT: - if (cpStatus == BRM::CP_VALID) - { - if (!ct.isWideDecimalType()) - factor = (1.0 * value - min + 1) / (max - min + 1); - else - factor = ((float128_t) value - min + 1) / (max - min + 1); - } + case COMPARE_LE: + case COMPARE_NGT: + if (cpStatus == BRM::CP_VALID) + { + if (!ct.isWideDecimalType()) + factor = (1.0 * value - min + 1) / (max - min + 1); + else + factor = ((float128_t)value - min + 1) / (max - min + 1); + } - break; + break; - case COMPARE_GT: - case COMPARE_NLE: - if (cpStatus == BRM::CP_VALID) - { - if (!ct.isWideDecimalType()) - factor = (1.0 * max - value) / (1.0 * max - min + 1); - else - factor = ((float128_t) max - value) / (max - min + 1); - } + case COMPARE_GT: + case COMPARE_NLE: + if (cpStatus == BRM::CP_VALID) + { + if (!ct.isWideDecimalType()) + factor = (1.0 * max - value) / (1.0 * max - min + 1); + else + factor = ((float128_t)max - value) / (max - min + 1); + } - break; + break; - case COMPARE_GE: - case COMPARE_NLT: - if (cpStatus == BRM::CP_VALID) - { - // TODO: Best way to convert to floating point arithmetic? - if (!ct.isWideDecimalType()) - factor = (1.0 * max - value + 1) / (max - min + 1); - else - factor = ((float128_t) max - value + 1) / (max - min + 1); - } + case COMPARE_GE: + case COMPARE_NLT: + if (cpStatus == BRM::CP_VALID) + { + // TODO: Best way to convert to floating point arithmetic? + if (!ct.isWideDecimalType()) + factor = (1.0 * max - value + 1) / (max - min + 1); + else + factor = ((float128_t)max - value + 1) / (max - min + 1); + } - break; + break; - case COMPARE_EQ: - factor = 1.0 / distinctValues; - break; + case COMPARE_EQ: factor = 1.0 / distinctValues; break; - case COMPARE_NE: - factor = 1.0 - (1.0 / distinctValues); - break; - } + case COMPARE_NE: factor = 1.0 - (1.0 / distinctValues); break; + } - if (factor < 0.0) - { - factor = 0.0; - } - else if (factor > 1.0) - { - factor = 1.0; - } + if (factor < 0.0) + { + factor = 0.0; + } + else if (factor > 1.0) + { + factor = 1.0; + } - return factor; + return factor; } // Estimate the percentage of rows that will be returned for a particular extent. // This function provides the estimate for entire filter such as "col 1 < 100 or col1 > 10000". -float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, - const messageqcpp::ByteStream* bs, - const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, - const uint8_t BOP, - const uint32_t& rowsInExtent) +float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const messageqcpp::ByteStream* bs, + const uint16_t NOPS, + const execplan::CalpontSystemCatalog::ColType& ct, + const uint8_t BOP, const uint32_t& rowsInExtent) { - bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); - float factor = 1.0; - float tempFactor = 1.0; + bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); + float factor = 1.0; + float tempFactor = 1.0; - uint64_t adjustedMin = 0, adjustedMax = 0; - uint128_t adjustedBigMin, adjustedBigMax; - uint32_t distinctValuesEstimate; + uint64_t adjustedMin = 0, adjustedMax = 0; + uint128_t adjustedBigMin, adjustedBigMax; + uint32_t distinctValuesEstimate; - // Adjust values based on column type and estimate the - if (!ct.isWideDecimalType()) + // Adjust values based on column type and estimate the + if (!ct.isWideDecimalType()) + { + adjustedMin = adjustValue(ct, emEntry.partition.cprange.loVal); + adjustedMax = adjustValue(ct, emEntry.partition.cprange.hiVal); + distinctValuesEstimate = + estimateDistinctValues(ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); + } + else + { + adjustedBigMin = emEntry.partition.cprange.bigLoVal; + adjustedBigMax = emEntry.partition.cprange.bigHiVal; + distinctValuesEstimate = + estimateDistinctValues(ct, adjustedBigMin, adjustedBigMax, emEntry.partition.cprange.isValid); + } + + // Loop through the operations and estimate the percentage of rows that will qualify. + // For example, there are two operations for "col1 > 5 and col1 < 10": + // 1) col1 > 5 + // 2) col2 < 10 + int length = bs->length(), pos = 0; + const char* msgDataPtr = (const char*)bs->buf(); + int64_t value = 0; + int128_t bigValue = 0; + bool firstQualifyingOrCondition = true; + uint16_t comparisonLimit = (NOPS <= fMaxComparisons) ? NOPS : fMaxComparisons; + + for (int i = 0; i < comparisonLimit; i++) + { + pos += ct.colWidth + 2; // predicate + op + lcf + + // TODO: Stole this condition from lbidlist. + // Investigate whether this can happen / should throw an error. + if (pos > length) { - adjustedMin = adjustValue(ct, emEntry.partition.cprange.loVal); - adjustedMax = adjustValue(ct, emEntry.partition.cprange.hiVal); - distinctValuesEstimate = estimateDistinctValues( - ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); + return factor; + } + + // Get the comparison value for the condition. + char op = *msgDataPtr++; + uint8_t lcf = *(uint8_t*)msgDataPtr++; + + if (bIsUnsigned) + { + switch (ct.colWidth) + { + case 1: + { + uint8_t val = *(uint8_t*)msgDataPtr; + value = val; + break; + } + + case 2: + { + uint16_t val = *(uint16_t*)msgDataPtr; + value = val; + break; + } + + case 4: + { + uint32_t val = *(uint32_t*)msgDataPtr; + value = val; + break; + } + + case 16: + { + if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + datatypes::TSInt128 val(reinterpret_cast(msgDataPtr)); + bigValue = val.getValue(); + break; + } + } + // fallthrough + + case 8: + default: + { + uint64_t val = *(uint64_t*)msgDataPtr; + value = static_cast(val); + break; + } + } } else { - adjustedBigMin = emEntry.partition.cprange.bigLoVal; - adjustedBigMax = emEntry.partition.cprange.bigHiVal; - distinctValuesEstimate = estimateDistinctValues( - ct, adjustedBigMin, adjustedBigMax, emEntry.partition.cprange.isValid); + switch (ct.colWidth) + { + case 1: + { + int8_t val = *(int8_t*)msgDataPtr; + value = val; + break; + } + + case 2: + { + int16_t val = *(int16_t*)msgDataPtr; + value = val; + break; + } + + case 4: + { + int32_t val = *(int32_t*)msgDataPtr; + value = val; + break; + } + + case 16: + { + if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + datatypes::TSInt128 val(reinterpret_cast(msgDataPtr)); + bigValue = val.getValue(); + break; + } + } + // fallthrough + + case 8: + default: + { + int64_t val = *(int64_t*)msgDataPtr; + value = val; + break; + } + } } + // TODO: Investigate whether condition below should throw an error. + msgDataPtr += ct.colWidth; - // Loop through the operations and estimate the percentage of rows that will qualify. - // For example, there are two operations for "col1 > 5 and col1 < 10": - // 1) col1 > 5 - // 2) col2 < 10 - int length = bs->length(), pos = 0; - const char* msgDataPtr = (const char*) bs->buf(); - int64_t value = 0; - int128_t bigValue = 0; - bool firstQualifyingOrCondition = true; - uint16_t comparisonLimit = (NOPS <= fMaxComparisons) ? NOPS : fMaxComparisons; - - for (int i = 0; i < comparisonLimit; i++) + if (pos > length) { - pos += ct.colWidth + 2; // predicate + op + lcf - - // TODO: Stole this condition from lbidlist. - // Investigate whether this can happen / should throw an error. - if (pos > length) - { - return factor; - } - - // Get the comparison value for the condition. - char op = *msgDataPtr++; - uint8_t lcf = *(uint8_t*)msgDataPtr++; - - if (bIsUnsigned) - { - switch (ct.colWidth) - { - case 1: - { - uint8_t val = *(uint8_t*)msgDataPtr; - value = val; - break; - } - - case 2: - { - uint16_t val = *(uint16_t*)msgDataPtr; - value = val; - break; - } - - case 4: - { - uint32_t val = *(uint32_t*)msgDataPtr; - value = val; - break; - } - - case 16: - { - if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) - { - datatypes::TSInt128 val(reinterpret_cast(msgDataPtr)); - bigValue = val.getValue(); - break; - } - } - // fallthrough - - case 8: - default: - { - uint64_t val = *(uint64_t*)msgDataPtr; - value = static_cast(val); - break; - } - } - } - else - { - switch (ct.colWidth) - { - case 1: - { - int8_t val = *(int8_t*)msgDataPtr; - value = val; - break; - } - - case 2: - { - int16_t val = *(int16_t*)msgDataPtr; - value = val; - break; - } - - case 4: - { - int32_t val = *(int32_t*)msgDataPtr; - value = val; - break; - } - - case 16: - { - if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) - { - datatypes::TSInt128 val(reinterpret_cast(msgDataPtr)); - bigValue = val.getValue(); - break; - } - } - // fallthrough - - case 8: - default: - { - int64_t val = *(int64_t*)msgDataPtr; - value = val; - break; - } - } - } - - // TODO: Investigate whether condition below should throw an error. - msgDataPtr += ct.colWidth; - - if (pos > length) - { - return factor; - } - -#if ROW_EST_DEBUG - cout << " Min-" << emEntry.partition.cprange.loVal << - ", Max-" << emEntry.partition.cprange.hiVal << - ", Val-" << value; -#endif - - // Get the factor for the individual operation. - if (bIsUnsigned) - { - if (!ct.isWideDecimalType()) - { - tempFactor = estimateOpFactor( - adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); - } - else - { - tempFactor = estimateOpFactor( - adjustedBigMin, adjustedBigMax, bigValue, op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); - } - } - else - { - if (!ct.isWideDecimalType()) - { - tempFactor = estimateOpFactor( - adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); - } - else - { - tempFactor = estimateOpFactor( - adjustedBigMin, adjustedBigMax, bigValue, op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); - } - } - -#if ROW_EST_DEBUG - cout << ", OperatorFactor-" << tempFactor << ", DistinctValsEst-" << distinctValuesEstimate << endl; -#endif - - // Use it in the overall factor. - if (BOP == BOP_AND) - { - // TODO: Handle betweens correctly (same as a >= 5 and a <= 10) - factor *= tempFactor; - } - else if (BOP == BOP_OR) - { - if (firstQualifyingOrCondition) - { - factor = tempFactor; - firstQualifyingOrCondition = false; - } - else - { - factor += tempFactor; - } - } - else - { - factor = tempFactor; - } - - } // for() - - if (factor > 1.0) - { - factor = 1.0; + return factor; } #if ROW_EST_DEBUG + cout << " Min-" << emEntry.partition.cprange.loVal << ", Max-" << emEntry.partition.cprange.hiVal + << ", Val-" << value; +#endif - if (NOPS > 1) - cout << " FilterFactor-" << factor << endl; + // Get the factor for the individual operation. + if (bIsUnsigned) + { + if (!ct.isWideDecimalType()) + { + tempFactor = + estimateOpFactor(adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + else + { + tempFactor = + estimateOpFactor(adjustedBigMin, adjustedBigMax, bigValue, op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + } + else + { + if (!ct.isWideDecimalType()) + { + tempFactor = estimateOpFactor(adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + else + { + tempFactor = + estimateOpFactor(adjustedBigMin, adjustedBigMax, bigValue, op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + } + +#if ROW_EST_DEBUG + cout << ", OperatorFactor-" << tempFactor << ", DistinctValsEst-" << distinctValuesEstimate << endl; +#endif + + // Use it in the overall factor. + if (BOP == BOP_AND) + { + // TODO: Handle betweens correctly (same as a >= 5 and a <= 10) + factor *= tempFactor; + } + else if (BOP == BOP_OR) + { + if (firstQualifyingOrCondition) + { + factor = tempFactor; + firstQualifyingOrCondition = false; + } + else + { + factor += tempFactor; + } + } + else + { + factor = tempFactor; + } + + } // for() + + if (factor > 1.0) + { + factor = 1.0; + } + +#if ROW_EST_DEBUG + + if (NOPS > 1) + cout << " FilterFactor-" << factor << endl; #endif - return factor; + return factor; } -// This function returns the estimated row count for the entire TupleBPS. It samples the last 20 (configurable) extents to -// calculate the estimate. +// This function returns the estimated row count for the entire TupleBPS. It samples the last 20 +// (configurable) extents to calculate the estimate. uint64_t RowEstimator::estimateRows(const vector& cpColVec, - const std::vector& scanFlags, - BRM::DBRM& dbrm, + const std::vector& scanFlags, BRM::DBRM& dbrm, const execplan::CalpontSystemCatalog::OID oid) { #if ROW_EST_DEBUG - StopWatch stopwatch; - stopwatch.start("estimateRows"); + StopWatch stopwatch; + stopwatch.start("estimateRows"); #endif - uint32_t rowsInLastExtent = fRowsPerExtent; - uint32_t extentRows = 0; - HWM_t hwm = 0; - float factor = 1.0; - float tempFactor = 1.0; + uint32_t rowsInLastExtent = fRowsPerExtent; + uint32_t extentRows = 0; + HWM_t hwm = 0; + float factor = 1.0; + float tempFactor = 1.0; - ColumnCommandJL* colCmd = 0; - uint32_t extentsSampled = 0; - uint64_t totalRowsToBeScanned = 0; - uint32_t estimatedExtentRowCount = 0; - uint64_t estimatedRowCount = 0; - //vector *extents = NULL; + ColumnCommandJL* colCmd = 0; + uint32_t extentsSampled = 0; + uint64_t totalRowsToBeScanned = 0; + uint32_t estimatedExtentRowCount = 0; + uint64_t estimatedRowCount = 0; + // vector *extents = NULL; - // Nothing to do if no scanFlags. - if (scanFlags.size() == 0 || cpColVec.size() == 0) + // Nothing to do if no scanFlags. + if (scanFlags.size() == 0 || cpColVec.size() == 0) + { + // TODO: Probably should throw an error here. + return 0; + } + + // Use the HWM for the estimated row count in the last extent. + colCmd = cpColVec[0]; + const vector& extents = colCmd->getExtents(); + hwm = extents.back().HWM; // extents is sorted by "global" fbo + rowsInLastExtent = ((hwm + 1) * fBlockSize / colCmd->getColType().colWidth) % fRowsPerExtent; + + // Sum up the total number of scanned rows. + int32_t idx = scanFlags.size() - 1; + + while (idx >= 0) + { + if (scanFlags[idx]) { - // TODO: Probably should throw an error here. - return 0; + extentRows = (idx == (int)scanFlags.size() - 1 ? rowsInLastExtent : fRowsPerExtent); + + // Get the predicate factor. +#if ROW_EST_DEBUG + cout << endl; + cout << "Ext-" << idx << ", rowsToScan-" << extentRows << endl; +#endif + factor = 1.0; + + for (uint32_t j = 0; j < cpColVec.size(); j++) + { + colCmd = cpColVec[j]; + // RowEstimator rowEstimator; +#if ROW_EST_DEBUG + stopwatch.start("estimateRowReturnFactor"); +#endif + // tempFactor = rowEstimator.estimateRowReturnFactor( + tempFactor = estimateRowReturnFactor(colCmd->getExtents()[idx], &(colCmd->getFilterString()), + colCmd->getFilterCount(), colCmd->getColType(), colCmd->getBOP(), + extentRows); +#if ROW_EST_DEBUG + stopwatch.stop("estimateRowReturnFactor"); +#endif + + factor *= tempFactor; + } + + extentsSampled++; + totalRowsToBeScanned += extentRows; + estimatedExtentRowCount = uint64_t(ceil(factor * extentRows)); + + if (estimatedExtentRowCount <= 0) + estimatedExtentRowCount = 1; + + estimatedRowCount += estimatedExtentRowCount; +#if ROW_EST_DEBUG + cout << "ExtentFactor-" << factor << ", EstimatedRows-" << estimatedExtentRowCount << endl; +#endif } - // Use the HWM for the estimated row count in the last extent. - colCmd = cpColVec[0]; - const vector& extents = colCmd->getExtents(); - hwm = extents.back().HWM; // extents is sorted by "global" fbo - rowsInLastExtent = ((hwm + 1) * fBlockSize / colCmd->getColType().colWidth) % fRowsPerExtent; + // if (extentsSampled == fExtentsToSample || idx == 0) + //{ + // done = true; + //} + // else + //{ + idx--; + //} + } - // Sum up the total number of scanned rows. - int32_t idx = scanFlags.size() - 1; + // If there are more extents than we sampled, add the row counts for the qualifying extents + // that we didn't sample to the count of rows that will be scanned. + // XXXPAT: Modified this fcn to sample all extents. Leaving this here due to level of arcana + // involved. :) + if (false && (extentsSampled >= fExtentsToSample) && (idx > 0)) + { + factor = (1.0 * estimatedRowCount) / (1.0 * totalRowsToBeScanned); +#if ROW_EST_DEBUG + cout << "overall factor-" << factor << endl; +#endif - while (idx >= 0) + for (int32_t i = 0; i < idx; i++) { - if (scanFlags[idx]) - { - extentRows = (idx == (int) scanFlags.size() - 1 ? rowsInLastExtent : fRowsPerExtent); - - // Get the predicate factor. -#if ROW_EST_DEBUG - cout << endl; - cout << "Ext-" << idx << ", rowsToScan-" << extentRows << endl; -#endif - factor = 1.0; - - for (uint32_t j = 0; j < cpColVec.size(); j++) - { - colCmd = cpColVec[j]; - //RowEstimator rowEstimator; -#if ROW_EST_DEBUG - stopwatch.start("estimateRowReturnFactor"); -#endif - //tempFactor = rowEstimator.estimateRowReturnFactor( - tempFactor = estimateRowReturnFactor( - colCmd->getExtents()[idx], - &(colCmd->getFilterString()), - colCmd->getFilterCount(), - colCmd->getColType(), - colCmd->getBOP(), - extentRows); -#if ROW_EST_DEBUG - stopwatch.stop("estimateRowReturnFactor"); -#endif - - factor *= tempFactor; - } - - extentsSampled++; - totalRowsToBeScanned += extentRows; - estimatedExtentRowCount = uint64_t(ceil(factor * extentRows)); - - if (estimatedExtentRowCount <= 0) estimatedExtentRowCount = 1; - - estimatedRowCount += estimatedExtentRowCount; -#if ROW_EST_DEBUG - cout << "ExtentFactor-" << factor << ", EstimatedRows-" << estimatedExtentRowCount << endl; -#endif - } - - //if (extentsSampled == fExtentsToSample || idx == 0) - //{ - //done = true; - //} - //else - //{ - idx--; - //} + if (scanFlags[i]) + { + // Don't take the expense of checking to see if the last extent was one that wasn't + // sampled. It will more than likely have been the first extent sampled since we + // are doing them in reverse order. If not, the amount of rows not populated isn't + // that significant since there are many qualifying extents. + totalRowsToBeScanned += fRowsPerExtent; + } } - // If there are more extents than we sampled, add the row counts for the qualifying extents - // that we didn't sample to the count of rows that will be scanned. - // XXXPAT: Modified this fcn to sample all extents. Leaving this here due to level of arcana - // involved. :) - if (false && (extentsSampled >= fExtentsToSample) && (idx > 0)) - { - factor = (1.0 * estimatedRowCount) / (1.0 * totalRowsToBeScanned); -#if ROW_EST_DEBUG - cout << "overall factor-" << factor << endl; -#endif - - for (int32_t i = 0; i < idx; i++) - { - if (scanFlags[i]) - { - // Don't take the expense of checking to see if the last extent was one that wasn't - // sampled. It will more than likely have been the first extent sampled since we - // are doing them in reverse order. If not, the amount of rows not populated isn't - // that significant since there are many qualifying extents. - totalRowsToBeScanned += fRowsPerExtent; - } - } - - estimatedRowCount = uint64_t(ceil(factor * totalRowsToBeScanned)); - } + estimatedRowCount = uint64_t(ceil(factor * totalRowsToBeScanned)); + } #if ROW_EST_DEBUG - cout << "Oid-" << oid << ", TotalEstimatedRows-" << estimatedRowCount << endl; - stopwatch.stop("estimateRows"); - stopwatch.finish(); + cout << "Oid-" << oid << ", TotalEstimatedRows-" << estimatedRowCount << endl; + stopwatch.stop("estimateRows"); + stopwatch.finish(); #endif - return estimatedRowCount; + return estimatedRowCount; } // @Bug 3503. Fix to use the number of extents to estimate the number of rows in queries that do @@ -670,19 +629,17 @@ uint64_t RowEstimator::estimateRows(const vector& cpColVec, // We use an estimate of 100% of the rows regardless of any dictionary filters. uint64_t RowEstimator::estimateRowsForNonCPColumn(ColumnCommandJL& colCmd) { - uint64_t estimatedRows = 0; - int numExtents = colCmd.getExtents().size(); + uint64_t estimatedRows = 0; + int numExtents = colCmd.getExtents().size(); - if (numExtents > 0) - { - HWM_t hwm = colCmd.getExtents()[numExtents - 1].HWM; - uint32_t rowsInLastExtent = - ((hwm + 1) * fBlockSize / colCmd.getColType().colWidth) % fRowsPerExtent; - estimatedRows = fRowsPerExtent * (numExtents - 1) + rowsInLastExtent; - } + if (numExtents > 0) + { + HWM_t hwm = colCmd.getExtents()[numExtents - 1].HWM; + uint32_t rowsInLastExtent = ((hwm + 1) * fBlockSize / colCmd.getColType().colWidth) % fRowsPerExtent; + estimatedRows = fRowsPerExtent * (numExtents - 1) + rowsInLastExtent; + } - return estimatedRows; + return estimatedRows; } - -} //namespace joblist +} // namespace joblist diff --git a/dbcon/joblist/rowestimator.h b/dbcon/joblist/rowestimator.h index a4c702a44..8a54ff4f0 100644 --- a/dbcon/joblist/rowestimator.h +++ b/dbcon/joblist/rowestimator.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: rowestimator.h 5449 2009-06-19 19:58:27Z wweeks $ -* -* -***********************************************************************/ + * $Id: rowestimator.h 5449 2009-06-19 19:58:27Z wweeks $ + * + * + ***********************************************************************/ /** @file */ #ifndef JOBLIST_ROWESTIMATOR_H @@ -37,117 +37,117 @@ namespace joblist { - /** @brief estimates row counts for a TupleBPS. * - * Class RowEstimator uses Casual Partitioning information and the filter string pertaining to a particular TupleBPS object - * to estimate cardinality. It is used to determine which table to use as the large side table in a multijoin operation. + * Class RowEstimator uses Casual Partitioning information and the filter string pertaining to a particular + * TupleBPS object to estimate cardinality. It is used to determine which table to use as the large side + * table in a multijoin operation. */ class RowEstimator { -public: - /** @brief ctor - */ - RowEstimator(): fExtentsToSample(20), fIntDistinctAdjust(1), fDecDistinctAdjust(1), fChar1DistinctAdjust(1), - fChar2Thru7DistinctAdjust(1), fDictDistinctAdjust(1), fDateDistinctAdjust(1) { } + public: + /** @brief ctor + */ + RowEstimator() + : fExtentsToSample(20) + , fIntDistinctAdjust(1) + , fDecDistinctAdjust(1) + , fChar1DistinctAdjust(1) + , fChar2Thru7DistinctAdjust(1) + , fDictDistinctAdjust(1) + , fDateDistinctAdjust(1) + { + } - /** @brief estimate the number of rows that will be returned for a particular tuple batch primitive step. - * - * @param cpColVec vector of ColumnCommandJL pointers associated to the step. - * @param scanFlags vector of flags with one entry per extent populated by the casual - * partitioning evaluation, each that will be scanned are true, the ones that - * were eliminated by casual partitioining are false. - * @param dbrm DBRM object used to get the HWM. - * @parm oid The objectid for the first column in the step. - * - */ - uint64_t estimateRows(const std::vector& cpColVec, - const std::vector & scanFlags, - BRM::DBRM& dbrm, - const execplan::CalpontSystemCatalog::OID oid); + /** @brief estimate the number of rows that will be returned for a particular tuple batch primitive step. + * + * @param cpColVec vector of ColumnCommandJL pointers associated to the step. + * @param scanFlags vector of flags with one entry per extent populated by the casual + * partitioning evaluation, each that will be scanned are true, the ones that + * were eliminated by casual partitioining are false. + * @param dbrm DBRM object used to get the HWM. + * @parm oid The objectid for the first column in the step. + * + */ + uint64_t estimateRows(const std::vector& cpColVec, const std::vector& scanFlags, + BRM::DBRM& dbrm, const execplan::CalpontSystemCatalog::OID oid); - /** @brief Estimate the number of rows that will be returned for a particular table given - * a ColumnCommandJL for non casual partitioning column. Added for bug 3503. - * - * @param colCmd The ColumnCommandJL. - * - */ - uint64_t estimateRowsForNonCPColumn(ColumnCommandJL& colCmd); + /** @brief Estimate the number of rows that will be returned for a particular table given + * a ColumnCommandJL for non casual partitioning column. Added for bug 3503. + * + * @param colCmd The ColumnCommandJL. + * + */ + uint64_t estimateRowsForNonCPColumn(ColumnCommandJL& colCmd); -private: - /** @brief adjusts column values so that they can be compared via ranges. - * - * This function provides a value for dates, datetimes, and strings that can be used for distinct value estimation and - * comparisons. - * - * @param ct The column type. - * @param value The column value. - * - */ - uint64_t adjustValue(const execplan::CalpontSystemCatalog::ColType& ct, - const uint64_t& value); + private: + /** @brief adjusts column values so that they can be compared via ranges. + * + * This function provides a value for dates, datetimes, and strings that can be used for distinct value + * estimation and comparisons. + * + * @param ct The column type. + * @param value The column value. + * + */ + uint64_t adjustValue(const execplan::CalpontSystemCatalog::ColType& ct, const uint64_t& value); - uint32_t daysThroughMonth(uint32_t mth); + uint32_t daysThroughMonth(uint32_t mth); - template - uint32_t estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const T& min, - const T& max, - const char cpStatus); + template + uint32_t estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, const T& min, + const T& max, const char cpStatus); - /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify the given individual operation. - * - * This function works for a single operation such as "col1 = 5". - * - * @param ct The column type. - * @param min The minimum value in the range. - * @param max The maximum value in the range. - * @parm cpStatus The status of the CP data (whether it's valid). - * - */ - template - float estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, - uint32_t distinctValues, char cpStatus, - const execplan::CalpontSystemCatalog::ColType& ct); + /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify the given individual + * operation. + * + * This function works for a single operation such as "col1 = 5". + * + * @param ct The column type. + * @param min The minimum value in the range. + * @param max The maximum value in the range. + * @parm cpStatus The status of the CP data (whether it's valid). + * + */ + template + float estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, + uint32_t distinctValues, char cpStatus, + const execplan::CalpontSystemCatalog::ColType& ct); - /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify - * the given operation(s). - * - * This function works for multiple operations against the same column such as - * "col1 = 5 or col1 = 10". It calls estimateOpFactor for each individual operation. - * - * @param emEntry The extent map entry for the extent being evaluated. - * @param msgDataPtr The filter string. - * @param ct The column type. - * @param BOP The binary operator for the filter predicates (eg. OR for col1 = 5 or col1 = 10) - * @param rowsInExtent The number of rows in the extent being evaluated. - * - */ - float estimateRowReturnFactor(const BRM::EMEntry& emEntry, - const messageqcpp::ByteStream* msgDataPtr, - const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, - const uint8_t BOP, - const uint32_t& rowsInExtent); + /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify + * the given operation(s). + * + * This function works for multiple operations against the same column such as + * "col1 = 5 or col1 = 10". It calls estimateOpFactor for each individual operation. + * + * @param emEntry The extent map entry for the extent being evaluated. + * @param msgDataPtr The filter string. + * @param ct The column type. + * @param BOP The binary operator for the filter predicates (eg. OR for col1 = 5 or col1 = 10) + * @param rowsInExtent The number of rows in the extent being evaluated. + * + */ + float estimateRowReturnFactor(const BRM::EMEntry& emEntry, const messageqcpp::ByteStream* msgDataPtr, + const uint16_t NOPS, const execplan::CalpontSystemCatalog::ColType& ct, + const uint8_t BOP, const uint32_t& rowsInExtent); - // Configurables read from Columnstore.xml - future. - uint32_t fExtentsToSample; - uint32_t fIntDistinctAdjust; - uint32_t fDecDistinctAdjust; - uint32_t fChar1DistinctAdjust; - uint32_t fChar2Thru7DistinctAdjust; - uint32_t fDictDistinctAdjust; - uint32_t fDateDistinctAdjust; + // Configurables read from Columnstore.xml - future. + uint32_t fExtentsToSample; + uint32_t fIntDistinctAdjust; + uint32_t fDecDistinctAdjust; + uint32_t fChar1DistinctAdjust; + uint32_t fChar2Thru7DistinctAdjust; + uint32_t fDictDistinctAdjust; + uint32_t fDateDistinctAdjust; - static const uint32_t fRowsPerExtent = 8192 * 1024; - static const uint32_t fBlockSize = 8192; // Block size in bytes. + static const uint32_t fRowsPerExtent = 8192 * 1024; + static const uint32_t fBlockSize = 8192; // Block size in bytes. - // Limits the number of comparisons for each extent. Example, in clause w/ 1000 values will limit the checks to - // the number below. - static const uint32_t fMaxComparisons = 10; + // Limits the number of comparisons for each extent. Example, in clause w/ 1000 values will limit the + // checks to the number below. + static const uint32_t fMaxComparisons = 10; -}; // RowEstimator +}; // RowEstimator - -} // joblist +} // namespace joblist #endif diff --git a/dbcon/joblist/rtscommand-jl.cpp b/dbcon/joblist/rtscommand-jl.cpp index aa6507e56..15326eca4 100644 --- a/dbcon/joblist/rtscommand-jl.cpp +++ b/dbcon/joblist/rtscommand-jl.cpp @@ -31,35 +31,32 @@ #include "bpp-jl.h" #include "tablecolumn.h" - using namespace std; using namespace messageqcpp; namespace joblist { - RTSCommandJL::RTSCommandJL(const pColStep& c, const pDictionaryStep& d) { - - col.reset(new ColumnCommandJL(c)); - dict.reset(new DictStepJL(d)); - /* XXXPAT: Need to validate the width; critical for tuple return functionality */ - dict->setWidth(c.realWidth); - OID = d.oid(); - colName = d.name(); - passThru = 0; + col.reset(new ColumnCommandJL(c)); + dict.reset(new DictStepJL(d)); + /* XXXPAT: Need to validate the width; critical for tuple return functionality */ + dict->setWidth(c.realWidth); + OID = d.oid(); + colName = d.name(); + passThru = 0; } RTSCommandJL::RTSCommandJL(const PassThruStep& p, const pDictionaryStep& d) { - execplan::CalpontSystemCatalog::ColType colType; + execplan::CalpontSystemCatalog::ColType colType; - dict.reset(new DictStepJL(d)); - /* XXXPAT: Need to validate the width; critical for tuple return functionality */ - dict->setWidth(p.realWidth); - OID = d.oid(); - colName = d.name(); - passThru = 1; + dict.reset(new DictStepJL(d)); + /* XXXPAT: Need to validate the width; critical for tuple return functionality */ + dict->setWidth(p.realWidth); + OID = d.oid(); + colName = d.name(); + passThru = 1; } RTSCommandJL::~RTSCommandJL() @@ -68,55 +65,55 @@ RTSCommandJL::~RTSCommandJL() void RTSCommandJL::setLBID(uint64_t data, uint32_t dbroot) { - if (!passThru) - col->setLBID(data, dbroot); + if (!passThru) + col->setLBID(data, dbroot); - dict->setLBID(data, dbroot); + dict->setLBID(data, dbroot); } void RTSCommandJL::createCommand(ByteStream& bs) const { - bs << (uint8_t) RID_TO_STRING; - bs << passThru; + bs << (uint8_t)RID_TO_STRING; + bs << passThru; - if (!passThru) - col->createCommand(bs); + if (!passThru) + col->createCommand(bs); - dict->createCommand(bs); - CommandJL::createCommand(bs); + dict->createCommand(bs); + CommandJL::createCommand(bs); } void RTSCommandJL::runCommand(ByteStream& bs) const { - if (!passThru) - col->runCommand(bs); + if (!passThru) + col->runCommand(bs); - dict->runCommand(bs); + dict->runCommand(bs); } uint8_t RTSCommandJL::getTableColumnType() { - return TableColumn::STRING; + return TableColumn::STRING; } string RTSCommandJL::toString() { - ostringstream ret; + ostringstream ret; - ret << "RTSCommandJL: oid=" << OID << " colName=" << colName << endl; - ret << " "; + ret << "RTSCommandJL: oid=" << OID << " colName=" << colName << endl; + ret << " "; - if (!passThru) - ret << col->toString() << endl; + if (!passThru) + ret << col->toString() << endl; - ret << " "; - ret << dict->toString(); - return ret.str(); + ret << " "; + ret << dict->toString(); + return ret.str(); } uint16_t RTSCommandJL::getWidth() { - return dict->getWidth(); + return dict->getWidth(); } -}; +}; // namespace joblist diff --git a/dbcon/joblist/rtscommand-jl.h b/dbcon/joblist/rtscommand-jl.h index f1a2084d4..b0bd8eca0 100644 --- a/dbcon/joblist/rtscommand-jl.h +++ b/dbcon/joblist/rtscommand-jl.h @@ -38,39 +38,39 @@ namespace joblist { - class RTSCommandJL : public CommandJL { -public: - RTSCommandJL(const pColStep&, const pDictionaryStep&); - RTSCommandJL(const PassThruStep&, const pDictionaryStep&); - virtual ~RTSCommandJL(); + public: + RTSCommandJL(const pColStep&, const pDictionaryStep&); + RTSCommandJL(const PassThruStep&, const pDictionaryStep&); + virtual ~RTSCommandJL(); - void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For ColumnCommandJL it's a RID, for a DictStep it's a token. - uint8_t getTableColumnType(); - std::string toString(); - bool isPassThru() - { - return (passThru != 0); - } - uint16_t getWidth(); - CommandType getCommandType() - { - return RID_TO_STRING; - } + void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For + // ColumnCommandJL it's a RID, for a DictStep it's a token. + uint8_t getTableColumnType(); + std::string toString(); + bool isPassThru() + { + return (passThru != 0); + } + uint16_t getWidth(); + CommandType getCommandType() + { + return RID_TO_STRING; + } - void createCommand(messageqcpp::ByteStream&) const; - void runCommand(messageqcpp::ByteStream&) const; + void createCommand(messageqcpp::ByteStream&) const; + void runCommand(messageqcpp::ByteStream&) const; -private: - RTSCommandJL(); - RTSCommandJL(const RTSCommandJL&); + private: + RTSCommandJL(); + RTSCommandJL(const RTSCommandJL&); - boost::scoped_ptr col; - boost::scoped_ptr dict; - uint8_t passThru; + boost::scoped_ptr col; + boost::scoped_ptr dict; + uint8_t passThru; }; -}; +}; // namespace joblist #endif diff --git a/dbcon/joblist/subquerystep.cpp b/dbcon/joblist/subquerystep.cpp index 2d39b4a43..a019c24f8 100644 --- a/dbcon/joblist/subquerystep.cpp +++ b/dbcon/joblist/subquerystep.cpp @@ -18,7 +18,6 @@ // $Id: subquerystep.cpp 6370 2010-03-18 02:58:09Z xlou $ - #include //#define NDEBUG #include @@ -55,16 +54,12 @@ using namespace querytele; #include "subquerystep.h" using namespace joblist; - namespace joblist { - -SubQueryStep::SubQueryStep(const JobInfo& jobInfo) - : JobStep(jobInfo) - , fRowsReturned(0) +SubQueryStep::SubQueryStep(const JobInfo& jobInfo) : JobStep(jobInfo), fRowsReturned(0) { - fExtendedInfo = "SQS: "; - fQtc.stepParms().stepType = StepTeleStats::T_SQS; + fExtendedInfo = "SQS: "; + fQtc.stepParms().stepType = StepTeleStats::T_SQS; } SubQueryStep::~SubQueryStep() @@ -73,495 +68,469 @@ SubQueryStep::~SubQueryStep() void SubQueryStep::run() { - fSubJobList->doQuery(); + fSubJobList->doQuery(); } void SubQueryStep::join() { - if (fRunner) - fRunner->join(); + if (fRunner) + fRunner->join(); } void SubQueryStep::abort() { - JobStep::abort(); - fSubJobList->abort(); + JobStep::abort(); + fSubJobList->abort(); } const string SubQueryStep::toString() const { - ostringstream oss; - oss << "SubQueryStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "SubQueryStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - if (fOutputJobStepAssociation.outSize() > 0) - { - oss << " out:"; + if (fOutputJobStepAssociation.outSize() > 0) + { + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); - } + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); + } - return oss.str(); + return oss.str(); } - /* void SubQueryStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString )-1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at "<< timeString - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time (0); + char timeString[50]; + ctime_r (&t, timeString); + timeString[strlen (timeString )-1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at "<< timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() + << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" + << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } void SubQueryStep::formatMiniStats() { - ostringstream oss; - oss << "SQS " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "SQS " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " + << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); } */ - SubAdapterStep::SubAdapterStep(SJSTEP& s, const JobInfo& jobInfo) - : JobStep(jobInfo) - , fTableOid(s->tableOid()) - , fSubStep(s) - , fRowsInput(0) - , fRowsReturned(0) - , fEndOfResult(false) - , fInputIterator(0) - , fOutputIterator(0) - , fRunner(0) + : JobStep(jobInfo) + , fTableOid(s->tableOid()) + , fSubStep(s) + , fRowsInput(0) + , fRowsReturned(0) + , fEndOfResult(false) + , fInputIterator(0) + , fOutputIterator(0) + , fRunner(0) { - fAlias = s->alias(); - fView = s->view(); - fInputJobStepAssociation = s->outputAssociation(); - fRowGroupIn = dynamic_cast(s.get())->getOutputRowGroup(); - setOutputRowGroup(fRowGroupIn); + fAlias = s->alias(); + fView = s->view(); + fInputJobStepAssociation = s->outputAssociation(); + fRowGroupIn = dynamic_cast(s.get())->getOutputRowGroup(); + setOutputRowGroup(fRowGroupIn); } - SubAdapterStep::~SubAdapterStep() { } void SubAdapterStep::abort() { - JobStep::abort(); + JobStep::abort(); - if (fSubStep) - fSubStep->abort(); + if (fSubStep) + fSubStep->abort(); } void SubAdapterStep::run() { - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for subquery adapter step."); + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for subquery adapter step."); - fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fInputDL == NULL) - throw logic_error("Input is not a RowGroup data list."); + if (fInputDL == NULL) + throw logic_error("Input is not a RowGroup data list."); - fInputIterator = fInputDL->getIterator(); + fInputIterator = fInputDL->getIterator(); - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for non-delivery subquery adapter step."); + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for non-delivery subquery adapter step."); - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); - if (fDelivery) - fOutputIterator = fOutputDL->getIterator(); + if (fDelivery) + fOutputIterator = fOutputDL->getIterator(); - fRunner = jobstepThreadPool.invoke(Runner(this)); + fRunner = jobstepThreadPool.invoke(Runner(this)); } - void SubAdapterStep::join() { - if (fRunner) - jobstepThreadPool.join(fRunner); + if (fRunner) + jobstepThreadPool.join(fRunner); } - uint32_t SubAdapterStep::nextBand(messageqcpp::ByteStream& bs) { - RGData rgDataOut; - bool more = false; - uint32_t rowCount = 0; + RGData rgDataOut; + bool more = false; + uint32_t rowCount = 0; - try + try + { + bs.restart(); + + more = fOutputDL->next(fOutputIterator, &rgDataOut); + + if (!more || cancelled()) { - bs.restart(); - + //@bug4459. + while (more) more = fOutputDL->next(fOutputIterator, &rgDataOut); - if (!more || cancelled()) - { - //@bug4459. - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - - fEndOfResult = true; - } - - if (more && !fEndOfResult) - { - fRowGroupDeliver.setData(&rgDataOut); - fRowGroupDeliver.serializeRGData(bs); - rowCount = fRowGroupDeliver.getRowCount(); - } + fEndOfResult = true; } - catch (...) + + if (more && !fEndOfResult) { - handleException(std::current_exception(), - logging::ERR_IN_DELIVERY, - logging::ERR_ALWAYS_CRITICAL, - "SubAdapterStep::nextBand()"); - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - fEndOfResult = true; + fRowGroupDeliver.setData(&rgDataOut); + fRowGroupDeliver.serializeRGData(bs); + rowCount = fRowGroupDeliver.getRowCount(); } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_DELIVERY, logging::ERR_ALWAYS_CRITICAL, + "SubAdapterStep::nextBand()"); + while (more) + more = fOutputDL->next(fOutputIterator, &rgDataOut); + fEndOfResult = true; + } - if (fEndOfResult) - { - // send an empty / error band - RGData rgData(fRowGroupDeliver, 0); - fRowGroupDeliver.setData(&rgData); - fRowGroupDeliver.resetRowGroup(0); - fRowGroupDeliver.setStatus(status()); - fRowGroupDeliver.serializeRGData(bs); - } + if (fEndOfResult) + { + // send an empty / error band + RGData rgData(fRowGroupDeliver, 0); + fRowGroupDeliver.setData(&rgData); + fRowGroupDeliver.resetRowGroup(0); + fRowGroupDeliver.setStatus(status()); + fRowGroupDeliver.serializeRGData(bs); + } - return rowCount; + return rowCount; } - void SubAdapterStep::setFeRowGroup(const rowgroup::RowGroup& rg) { - fRowGroupFe = rg; + fRowGroupFe = rg; } - void SubAdapterStep::setOutputRowGroup(const rowgroup::RowGroup& rg) { - fRowGroupOut = fRowGroupDeliver = rg; + fRowGroupOut = fRowGroupDeliver = rg; - if (fRowGroupFe.getColumnCount() == 0) - fIndexMap = makeMapping(fRowGroupIn, fRowGroupOut); - else - fIndexMap = makeMapping(fRowGroupFe, fRowGroupOut); + if (fRowGroupFe.getColumnCount() == 0) + fIndexMap = makeMapping(fRowGroupIn, fRowGroupOut); + else + fIndexMap = makeMapping(fRowGroupFe, fRowGroupOut); - checkDupOutputColumns(); + checkDupOutputColumns(); } - void SubAdapterStep::checkDupOutputColumns() { - map keymap; // map - fDupColumns.clear(); - const vector& keys = fRowGroupDeliver.getKeys(); + map keymap; // map + fDupColumns.clear(); + const vector& keys = fRowGroupDeliver.getKeys(); - for (uint32_t i = 0; i < keys.size(); i++) - { - map::iterator j = keymap.find(keys[i]); + for (uint32_t i = 0; i < keys.size(); i++) + { + map::iterator j = keymap.find(keys[i]); - if (j == keymap.end()) - keymap.insert(make_pair(keys[i], i)); // map key to col index - else - fDupColumns.push_back(make_pair(i, j->second)); // dest/src index pair - } + if (j == keymap.end()) + keymap.insert(make_pair(keys[i], i)); // map key to col index + else + fDupColumns.push_back(make_pair(i, j->second)); // dest/src index pair + } } - void SubAdapterStep::dupOutputColumns(Row& row) { - for (uint64_t i = 0; i < fDupColumns.size(); i++) - row.copyField(fDupColumns[i].first, fDupColumns[i].second); + for (uint64_t i = 0; i < fDupColumns.size(); i++) + row.copyField(fDupColumns[i].first, fDupColumns[i].second); } - void SubAdapterStep::outputRow(Row& rowIn, Row& rowOut) { - applyMapping(fIndexMap, rowIn, &rowOut); + applyMapping(fIndexMap, rowIn, &rowOut); - if (fDupColumns.size() > 0) - dupOutputColumns(rowOut); + if (fDupColumns.size() > 0) + dupOutputColumns(rowOut); - fRowGroupOut.incRowCount(); - rowOut.nextRow(); + fRowGroupOut.incRowCount(); + rowOut.nextRow(); } - void SubAdapterStep::deliverStringTableRowGroup(bool b) { - fRowGroupOut.setUseStringTable(b); - fRowGroupDeliver.setUseStringTable(b); + fRowGroupOut.setUseStringTable(b); + fRowGroupDeliver.setUseStringTable(b); } - bool SubAdapterStep::deliverStringTableRowGroup() const { - idbassert(fRowGroupOut.usesStringTable() == fRowGroupDeliver.usesStringTable()); - return fRowGroupDeliver.usesStringTable(); + idbassert(fRowGroupOut.usesStringTable() == fRowGroupDeliver.usesStringTable()); + return fRowGroupDeliver.usesStringTable(); } - const string SubAdapterStep::toString() const { - ostringstream oss; - oss << "SubAdapterStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "SubAdapterStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - if (fInputJobStepAssociation.outSize() > 0) - oss << fInputJobStepAssociation.outAt(0); + if (fInputJobStepAssociation.outSize() > 0) + oss << fInputJobStepAssociation.outAt(0); - if (fOutputJobStepAssociation.outSize() > 0) - oss << fOutputJobStepAssociation.outAt(0); + if (fOutputJobStepAssociation.outSize() > 0) + oss << fOutputJobStepAssociation.outAt(0); - return oss.str(); + return oss.str(); } - void SubAdapterStep::execute() { - RGData rgDataIn; - RGData rgDataOut; - Row rowIn; - Row rowFe; - Row rowOut; - fRowGroupIn.initRow(&rowIn); - fRowGroupOut.initRow(&rowOut); + RGData rgDataIn; + RGData rgDataOut; + Row rowIn; + Row rowFe; + Row rowOut; + fRowGroupIn.initRow(&rowIn); + fRowGroupOut.initRow(&rowOut); - RGData rowFeData; - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - bool usesFE = false; + RGData rowFeData; + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + bool usesFE = false; - if (fRowGroupFe.getColumnCount() > 0) - { - usesFE = true; - fRowGroupFe.initRow(&rowFe, true); - rowFeData = RGData(fRowGroupFe, 1); - fRowGroupFe.setData(&rowFeData); - fRowGroupFe.getRow(0, &rowFe); - } + if (fRowGroupFe.getColumnCount() > 0) + { + usesFE = true; + fRowGroupFe.initRow(&rowFe, true); + rowFeData = RGData(fRowGroupFe, 1); + fRowGroupFe.setData(&rowFeData); + fRowGroupFe.getRow(0, &rowFe); + } - bool more = false; + bool more = false; - try - { - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); + try + { + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); - fSubStep->run(); + fSubStep->run(); - more = fInputDL->next(fInputIterator, &rgDataIn); - - if (traceOn()) dlTimes.setFirstReadTime(); - - while (more && !cancelled()) - { - fRowGroupIn.setData(&rgDataIn); - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - - fRowGroupIn.getRow(0, &rowIn); - fRowGroupOut.getRow(0, &rowOut); - - fRowsInput += fRowGroupIn.getRowCount(); - - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) - { - if (fExpression.get() == NULL) - { - outputRow(rowIn, rowOut); - } - else if (!usesFE) - { - if (fExpression->evaluate(&rowIn)) - { - outputRow(rowIn, rowOut); - } - } - else - { - copyRow(rowIn, &rowFe, rowIn.getColumnCount()); - - //memcpy(rowFe.getData(), rowIn.getData(), rowIn.getSize()); - if (fExpression->evaluate(&rowFe)) - { - outputRow(rowFe, rowOut); - } - } - - rowIn.nextRow(); - } - - if (fRowGroupOut.getRowCount() > 0) - { - fRowsReturned += fRowGroupOut.getRowCount(); - fOutputDL->insert(rgDataOut); - } - - more = fInputDL->next(fInputIterator, &rgDataIn); - } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_ALWAYS_CRITICAL, - "SubAdapterStep::execute()"); - } - - if (cancelled()) - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); + more = fInputDL->next(fInputIterator, &rgDataIn); if (traceOn()) + dlTimes.setFirstReadTime(); + + while (more && !cancelled()) { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } + fRowGroupIn.setData(&rgDataIn); + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); + fRowGroupIn.getRow(0, &rowIn); + fRowGroupOut.getRow(0, &rowOut); - // Bug 3136, let mini stats to be formatted if traceOn. - fOutputDL->endOfInput(); -} + fRowsInput += fRowGroupIn.getRowCount(); - -void SubAdapterStep::addExpression(const JobStepVector& exps, JobInfo& jobInfo) -{ - // maps key to the index in the RG - map keyToIndexMap; - const vector& keys = fRowGroupIn.getKeys(); - - for (size_t i = 0; i < keys.size(); i++) - keyToIndexMap[keys[i]] = i; - - // combine the expression to one parse tree - ParseTree* filter = NULL; - - for (JobStepVector::const_iterator it = exps.begin(); it != exps.end(); it++) - { - ExpressionStep* e = dynamic_cast(it->get()); - idbassert(e); - - e->updateInputIndex(keyToIndexMap, jobInfo); - - if (filter != NULL) + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) + { + if (fExpression.get() == NULL) { - ParseTree* left = filter; - ParseTree* right = new ParseTree(); - right->copyTree(*(e->expressionFilter())); - filter = new ParseTree(new LogicOperator("and")); - filter->left(left); - filter->right(right); + outputRow(rowIn, rowOut); + } + else if (!usesFE) + { + if (fExpression->evaluate(&rowIn)) + { + outputRow(rowIn, rowOut); + } } else { - filter = new ParseTree(); - filter->copyTree(*(e->expressionFilter())); + copyRow(rowIn, &rowFe, rowIn.getColumnCount()); + + // memcpy(rowFe.getData(), rowIn.getData(), rowIn.getSize()); + if (fExpression->evaluate(&rowFe)) + { + outputRow(rowFe, rowOut); + } } + + rowIn.nextRow(); + } + + if (fRowGroupOut.getRowCount() > 0) + { + fRowsReturned += fRowGroupOut.getRowCount(); + fOutputDL->insert(rgDataOut); + } + + more = fInputDL->next(fInputIterator, &rgDataIn); } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_ALWAYS_CRITICAL, + "SubAdapterStep::execute()"); + } - // add to the expression wrapper - if (fExpression.get() == NULL) - fExpression.reset(new funcexp::FuncExpWrapper()); + if (cancelled()) + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); - fExpression->addFilter(boost::shared_ptr(filter)); + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + // Bug 3136, let mini stats to be formatted if traceOn. + fOutputDL->endOfInput(); } +void SubAdapterStep::addExpression(const JobStepVector& exps, JobInfo& jobInfo) +{ + // maps key to the index in the RG + map keyToIndexMap; + const vector& keys = fRowGroupIn.getKeys(); + + for (size_t i = 0; i < keys.size(); i++) + keyToIndexMap[keys[i]] = i; + + // combine the expression to one parse tree + ParseTree* filter = NULL; + + for (JobStepVector::const_iterator it = exps.begin(); it != exps.end(); it++) + { + ExpressionStep* e = dynamic_cast(it->get()); + idbassert(e); + + e->updateInputIndex(keyToIndexMap, jobInfo); + + if (filter != NULL) + { + ParseTree* left = filter; + ParseTree* right = new ParseTree(); + right->copyTree(*(e->expressionFilter())); + filter = new ParseTree(new LogicOperator("and")); + filter->left(left); + filter->right(right); + } + else + { + filter = new ParseTree(); + filter->copyTree(*(e->expressionFilter())); + } + } + + // add to the expression wrapper + if (fExpression.get() == NULL) + fExpression.reset(new funcexp::FuncExpWrapper()); + + fExpression->addFilter(boost::shared_ptr(filter)); +} void SubAdapterStep::addExpression(const vector& exps) { - // add to the function wrapper - if (fExpression.get() == NULL) - fExpression.reset(new funcexp::FuncExpWrapper()); + // add to the function wrapper + if (fExpression.get() == NULL) + fExpression.reset(new funcexp::FuncExpWrapper()); - for (vector::const_iterator i = exps.begin(); i != exps.end(); i++) - fExpression->addReturnedColumn(*i); + for (vector::const_iterator i = exps.begin(); i != exps.end(); i++) + fExpression->addReturnedColumn(*i); } - void SubAdapterStep::addFcnJoinExp(const vector& exps) { - addExpression(exps); + addExpression(exps); } - void SubAdapterStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; total rows input-" << fRowsInput - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows input-" << fRowsInput << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } - void SubAdapterStep::formatMiniStats() { - /* - ostringstream oss; - oss << "SAS " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); - */ + /* + ostringstream oss; + oss << "SAS " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " + << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); + */ } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/subquerystep.h b/dbcon/joblist/subquerystep.h index e7c479ba2..fb0eca13c 100644 --- a/dbcon/joblist/subquerystep.h +++ b/dbcon/joblist/subquerystep.h @@ -17,7 +17,6 @@ // $Id: subquerystep.h 6370 2010-03-18 02:58:09Z xlou $ - /** @file */ #ifndef SUBQUERY_STEP_H @@ -39,246 +38,241 @@ struct JobInfo; class SubQueryStep : public JobStep { + public: + /** @brief SubQueryStep constructor + */ + SubQueryStep(const JobInfo&); -public: - /** @brief SubQueryStep constructor - */ - SubQueryStep(const JobInfo&); + /** @brief SubQueryStep destructor + */ + ~SubQueryStep(); - /** @brief SubQueryStep destructor - */ - ~SubQueryStep(); + /** @brief virtual void run method + */ + void run(); - /** @brief virtual void run method - */ - void run(); + /** @brief virtual void join method + */ + void join(); - /** @brief virtual void join method - */ - void join(); + /** @brief virtual void abort method + */ + void abort(); - /** @brief virtual void abort method - */ - void abort(); + /** @brief virtual get table OID + * @returns OID + */ + execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } - /** @brief virtual get table OID - * @returns OID - */ - execplan::CalpontSystemCatalog::OID tableOid() const - { - return fTableOid; - } + /** @brief virtual set table OID + */ + void tableOid(const execplan::CalpontSystemCatalog::OID& id) + { + fTableOid = id; + } - /** @brief virtual set table OID - */ - void tableOid(const execplan::CalpontSystemCatalog::OID& id) - { - fTableOid = id; - } + /** @brief virtual output info to a string + * @returns string + */ + const std::string toString() const; - /** @brief virtual output info to a string - * @returns string - */ - const std::string toString() const; + /** @brief virtual set the output rowgroup + */ + virtual void setOutputRowGroup(const rowgroup::RowGroup& rg) + { + fOutputRowGroup = rg; + } - /** @brief virtual set the output rowgroup - */ - virtual void setOutputRowGroup(const rowgroup::RowGroup& rg) - { - fOutputRowGroup = rg; - } + /** @brief virtual get the output rowgroup + * @returns RowGroup + */ + virtual const rowgroup::RowGroup& getOutputRowGroup() const + { + return fOutputRowGroup; + } - /** @brief virtual get the output rowgroup - * @returns RowGroup - */ - virtual const rowgroup::RowGroup& getOutputRowGroup() const - { - return fOutputRowGroup; - } + /** @brief virtual set the sub-query's joblist + */ + virtual void subJoblist(const STJLP& sjl) + { + fSubJobList = sjl; + } - /** @brief virtual set the sub-query's joblist - */ - virtual void subJoblist(const STJLP& sjl) - { - fSubJobList = sjl; - } + /** @brief virtual get the sub-query's joblist + * @returns boost::shared_ptr + */ + virtual const STJLP& subJoblist() const + { + return fSubJobList; + } - /** @brief virtual get the sub-query's joblist - * @returns boost::shared_ptr - */ - virtual const STJLP& subJoblist() const - { - return fSubJobList; - } + protected: + uint64_t fRowsReturned; + execplan::CalpontSystemCatalog::OID fTableOid; + std::vector fColumnOids; + rowgroup::RowGroup fOutputRowGroup; + STJLP fSubJobList; -protected: - uint64_t fRowsReturned; - - execplan::CalpontSystemCatalog::OID fTableOid; - std::vector fColumnOids; - rowgroup::RowGroup fOutputRowGroup; - STJLP fSubJobList; - - boost::scoped_ptr fRunner; + boost::scoped_ptr fRunner; }; - class SubAdapterStep : public JobStep, public TupleDeliveryStep { + public: + /** @brief SubAdapterStep constructor + */ + SubAdapterStep(SJSTEP& s, const JobInfo&); -public: - /** @brief SubAdapterStep constructor - */ - SubAdapterStep(SJSTEP& s, const JobInfo&); + /** @brief SubAdapterStep destructor + */ + ~SubAdapterStep(); - /** @brief SubAdapterStep destructor - */ - ~SubAdapterStep(); + /** @brief virtual void run method + */ + void run(); - /** @brief virtual void run method - */ - void run(); + /** @brief virtual void join method + */ + void join(); - /** @brief virtual void join method - */ - void join(); + /** @brief virtual void abort method + */ + void abort(); - /** @brief virtual void abort method - */ - void abort(); + /** @brief virtual get table OID + * @returns OID + */ + execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOid; + } - /** @brief virtual get table OID - * @returns OID - */ - execplan::CalpontSystemCatalog::OID tableOid() const + /** @brief virtual set table OID + */ + void tableOid(const execplan::CalpontSystemCatalog::OID& id) + { + fTableOid = id; + } + + /** @brief virtual output info to a string + * @returns string + */ + const std::string toString() const; + + /** @brief virtual set the output rowgroup + */ + void setOutputRowGroup(const rowgroup::RowGroup& rg); + + /** @brief virtual get the output rowgroup + * @returns RowGroup + */ + const rowgroup::RowGroup& getOutputRowGroup() const + { + return fRowGroupOut; + } + + /** @brief TupleDeliveryStep's pure virtual methods nextBand + * @returns row count + */ + uint32_t nextBand(messageqcpp::ByteStream& bs); + + /** @brief Delivered Row Group + * @returns RowGroup + */ + const rowgroup::RowGroup& getDeliveredRowGroup() const + { + return fRowGroupDeliver; + } + + /** @brief Turn on/off string table delivery + */ + void deliverStringTableRowGroup(bool b); + + /** @brief Check useStringTable flag on delivered RowGroup + * @returns boolean + */ + bool deliverStringTableRowGroup() const; + + /** @brief set the rowgroup for FE to work on + */ + void setFeRowGroup(const rowgroup::RowGroup& rg); + + /** @brief get the rowgroup for FE + * @returns RowGroup + */ + const rowgroup::RowGroup& getFeRowGroup() const + { + return fRowGroupFe; + } + + /** @brief get subquery step + */ + const SJSTEP& subStep() const + { + return fSubStep; + } + + /** @brief add filters (expression steps) + */ + void addExpression(const JobStepVector&, JobInfo&); + + /** @brief add function columns (returned columns) + */ + void addExpression(const std::vector&); + + /** @brief add function join expresssion + */ + void addFcnJoinExp(const std::vector&); + + protected: + void execute(); + void outputRow(rowgroup::Row&, rowgroup::Row&); + void checkDupOutputColumns(); + void dupOutputColumns(rowgroup::Row&); + void printCalTrace(); + void formatMiniStats(); + + execplan::CalpontSystemCatalog::OID fTableOid; + rowgroup::RowGroup fRowGroupIn; + rowgroup::RowGroup fRowGroupOut; + rowgroup::RowGroup fRowGroupFe; + rowgroup::RowGroup fRowGroupDeliver; + SJSTEP fSubStep; + uint64_t fRowsInput; + uint64_t fRowsReturned; + bool fEndOfResult; + boost::shared_array fIndexMap; + std::vector > fDupColumns; + + RowGroupDL* fInputDL; + RowGroupDL* fOutputDL; + uint64_t fInputIterator; + uint64_t fOutputIterator; + + class Runner + { + public: + Runner(SubAdapterStep* step) : fStep(step) { - return fTableOid; + } + void operator()() + { + utils::setThreadName("SQSRunner"); + fStep->execute(); } - /** @brief virtual set table OID - */ - void tableOid(const execplan::CalpontSystemCatalog::OID& id) - { - fTableOid = id; - } + SubAdapterStep* fStep; + }; + uint64_t fRunner; // thread pool handle - /** @brief virtual output info to a string - * @returns string - */ - const std::string toString() const; - - /** @brief virtual set the output rowgroup - */ - void setOutputRowGroup(const rowgroup::RowGroup& rg); - - /** @brief virtual get the output rowgroup - * @returns RowGroup - */ - const rowgroup::RowGroup& getOutputRowGroup() const - { - return fRowGroupOut; - } - - /** @brief TupleDeliveryStep's pure virtual methods nextBand - * @returns row count - */ - uint32_t nextBand(messageqcpp::ByteStream& bs); - - /** @brief Delivered Row Group - * @returns RowGroup - */ - const rowgroup::RowGroup& getDeliveredRowGroup() const - { - return fRowGroupDeliver; - } - - /** @brief Turn on/off string table delivery - */ - void deliverStringTableRowGroup(bool b); - - /** @brief Check useStringTable flag on delivered RowGroup - * @returns boolean - */ - bool deliverStringTableRowGroup() const; - - /** @brief set the rowgroup for FE to work on - */ - void setFeRowGroup(const rowgroup::RowGroup& rg); - - /** @brief get the rowgroup for FE - * @returns RowGroup - */ - const rowgroup::RowGroup& getFeRowGroup() const - { - return fRowGroupFe; - } - - /** @brief get subquery step - */ - const SJSTEP& subStep() const - { - return fSubStep; - } - - /** @brief add filters (expression steps) - */ - void addExpression(const JobStepVector&, JobInfo&); - - /** @brief add function columns (returned columns) - */ - void addExpression(const std::vector&); - - /** @brief add function join expresssion - */ - void addFcnJoinExp(const std::vector&); - - -protected: - void execute(); - void outputRow(rowgroup::Row&, rowgroup::Row&); - void checkDupOutputColumns(); - void dupOutputColumns(rowgroup::Row&); - void printCalTrace(); - void formatMiniStats(); - - execplan::CalpontSystemCatalog::OID fTableOid; - rowgroup::RowGroup fRowGroupIn; - rowgroup::RowGroup fRowGroupOut; - rowgroup::RowGroup fRowGroupFe; - rowgroup::RowGroup fRowGroupDeliver; - SJSTEP fSubStep; - uint64_t fRowsInput; - uint64_t fRowsReturned; - bool fEndOfResult; - boost::shared_array fIndexMap; - std::vector > fDupColumns; - - RowGroupDL* fInputDL; - RowGroupDL* fOutputDL; - uint64_t fInputIterator; - uint64_t fOutputIterator; - - class Runner - { - public: - Runner(SubAdapterStep* step) : fStep(step) { } - void operator()() - { - utils::setThreadName("SQSRunner"); - fStep->execute(); - } - - SubAdapterStep* fStep; - }; - uint64_t fRunner; // thread pool handle - - boost::scoped_ptr fExpression; + boost::scoped_ptr fExpression; }; - -} +} // namespace joblist #endif // SUBQUERY_STEP_H // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index be54eaedd..816b6aa72 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -18,7 +18,6 @@ // $Id: subquerytransformer.cpp 6406 2010-03-26 19:18:37Z xlou $ - #include //#define NDEBUG #include @@ -49,605 +48,578 @@ using namespace logging; #include "subquerystep.h" #include "subquerytransformer.h" - namespace joblist { - -SubQueryTransformer::SubQueryTransformer(JobInfo* jobInfo, SErrorInfo& err) : - fOutJobInfo(jobInfo), fSubJobInfo(NULL), fErrorInfo(err) +SubQueryTransformer::SubQueryTransformer(JobInfo* jobInfo, SErrorInfo& err) + : fOutJobInfo(jobInfo), fSubJobInfo(NULL), fErrorInfo(err) { } - -SubQueryTransformer::SubQueryTransformer(JobInfo* jobInfo, SErrorInfo& err, - const string& view) : - fOutJobInfo(jobInfo), fSubJobInfo(NULL), fErrorInfo(err) +SubQueryTransformer::SubQueryTransformer(JobInfo* jobInfo, SErrorInfo& err, const string& view) + : fOutJobInfo(jobInfo), fSubJobInfo(NULL), fErrorInfo(err) { - fVtable.view(view); + fVtable.view(view); } - -SubQueryTransformer::SubQueryTransformer(JobInfo* jobInfo, SErrorInfo& err, - const string& alias, - const string& view) : - fOutJobInfo(jobInfo), fSubJobInfo(NULL), fErrorInfo(err) +SubQueryTransformer::SubQueryTransformer(JobInfo* jobInfo, SErrorInfo& err, const string& alias, + const string& view) + : fOutJobInfo(jobInfo), fSubJobInfo(NULL), fErrorInfo(err) { - fVtable.alias(alias); - fVtable.view(view); + fVtable.alias(alias); + fVtable.view(view); } - SubQueryTransformer::~SubQueryTransformer() { - // OK to delete NULL ptr - delete fSubJobInfo; - fSubJobInfo = NULL; + // OK to delete NULL ptr + delete fSubJobInfo; + fSubJobInfo = NULL; } - SJSTEP& SubQueryTransformer::makeSubQueryStep(execplan::CalpontSelectExecutionPlan* csep, - bool subInFromClause) + bool subInFromClause) { - if (fOutJobInfo->trace) - cout << (*csep) << endl; + if (fOutJobInfo->trace) + cout << (*csep) << endl; - // Setup job info, job list and error status relation. - fSubJobInfo = new JobInfo(fOutJobInfo->rm); - fSubJobInfo->sessionId = fOutJobInfo->sessionId; - fSubJobInfo->txnId = fOutJobInfo->txnId; - fSubJobInfo->verId = fOutJobInfo->verId; - fSubJobInfo->statementId = fOutJobInfo->statementId; - fSubJobInfo->queryType = fOutJobInfo->queryType; - fSubJobInfo->csc = fOutJobInfo->csc; - fSubJobInfo->trace = fOutJobInfo->trace; - fSubJobInfo->traceFlags = fOutJobInfo->traceFlags; - fSubJobInfo->isExeMgr = fOutJobInfo->isExeMgr; - fSubJobInfo->subLevel = fOutJobInfo->subLevel + 1; - fSubJobInfo->keyInfo = fOutJobInfo->keyInfo; - fSubJobInfo->stringScanThreshold = fOutJobInfo->stringScanThreshold; - fSubJobInfo->tryTuples = true; - fSubJobInfo->errorInfo = fErrorInfo; - fOutJobInfo->subNum++; - fSubJobInfo->subCount = fOutJobInfo->subCount; - fSubJobInfo->subId = ++(*(fSubJobInfo->subCount)); - fSubJobInfo->pJobInfo = fOutJobInfo; - fSubJobList.reset(new TupleJobList(true)); - fSubJobList->priority(csep->priority()); - fSubJobInfo->projectingTableOID = fSubJobList->projectingTableOIDPtr(); - fSubJobInfo->jobListPtr = fSubJobList.get(); - fSubJobInfo->stringTableThreshold = fOutJobInfo->stringTableThreshold; - fSubJobInfo->localQuery = fOutJobInfo->localQuery; - fSubJobInfo->uuid = fOutJobInfo->uuid; - fOutJobInfo->jobListPtr->addSubqueryJobList(fSubJobList); + // Setup job info, job list and error status relation. + fSubJobInfo = new JobInfo(fOutJobInfo->rm); + fSubJobInfo->sessionId = fOutJobInfo->sessionId; + fSubJobInfo->txnId = fOutJobInfo->txnId; + fSubJobInfo->verId = fOutJobInfo->verId; + fSubJobInfo->statementId = fOutJobInfo->statementId; + fSubJobInfo->queryType = fOutJobInfo->queryType; + fSubJobInfo->csc = fOutJobInfo->csc; + fSubJobInfo->trace = fOutJobInfo->trace; + fSubJobInfo->traceFlags = fOutJobInfo->traceFlags; + fSubJobInfo->isExeMgr = fOutJobInfo->isExeMgr; + fSubJobInfo->subLevel = fOutJobInfo->subLevel + 1; + fSubJobInfo->keyInfo = fOutJobInfo->keyInfo; + fSubJobInfo->stringScanThreshold = fOutJobInfo->stringScanThreshold; + fSubJobInfo->tryTuples = true; + fSubJobInfo->errorInfo = fErrorInfo; + fOutJobInfo->subNum++; + fSubJobInfo->subCount = fOutJobInfo->subCount; + fSubJobInfo->subId = ++(*(fSubJobInfo->subCount)); + fSubJobInfo->pJobInfo = fOutJobInfo; + fSubJobList.reset(new TupleJobList(true)); + fSubJobList->priority(csep->priority()); + fSubJobInfo->projectingTableOID = fSubJobList->projectingTableOIDPtr(); + fSubJobInfo->jobListPtr = fSubJobList.get(); + fSubJobInfo->stringTableThreshold = fOutJobInfo->stringTableThreshold; + fSubJobInfo->localQuery = fOutJobInfo->localQuery; + fSubJobInfo->uuid = fOutJobInfo->uuid; + fOutJobInfo->jobListPtr->addSubqueryJobList(fSubJobList); - fSubJobInfo->smallSideLimit = fOutJobInfo->smallSideLimit; - fSubJobInfo->largeSideLimit = fOutJobInfo->largeSideLimit; - fSubJobInfo->smallSideUsage = fOutJobInfo->smallSideUsage; - fSubJobInfo->partitionSize = fOutJobInfo->partitionSize; - fSubJobInfo->umMemLimit = fOutJobInfo->umMemLimit; - fSubJobInfo->isDML = fOutJobInfo->isDML; + fSubJobInfo->smallSideLimit = fOutJobInfo->smallSideLimit; + fSubJobInfo->largeSideLimit = fOutJobInfo->largeSideLimit; + fSubJobInfo->smallSideUsage = fOutJobInfo->smallSideUsage; + fSubJobInfo->partitionSize = fOutJobInfo->partitionSize; + fSubJobInfo->umMemLimit = fOutJobInfo->umMemLimit; + fSubJobInfo->isDML = fOutJobInfo->isDML; - // Update v-table's alias. - fVtable.name("$sub"); + // Update v-table's alias. + fVtable.name("$sub"); - if (fVtable.alias().empty()) + if (fVtable.alias().empty()) + { + ostringstream oss; + oss << "$sub_" << fSubJobInfo->subId << "_" << fSubJobInfo->subLevel << "_" << fOutJobInfo->subNum; + fVtable.alias(oss.str()); + } + + fSubJobInfo->subAlias = fVtable.alias(); //@bug5844, unique alias for sub + + // Make the jobsteps out of the execution plan. + JobStepVector querySteps; + JobStepVector projectSteps; + DeliveredTableMap deliverySteps; + + if (csep->unionVec().size() == 0) + makeJobSteps(csep, *fSubJobInfo, querySteps, projectSteps, deliverySteps); + else if (subInFromClause) + makeUnionJobSteps(csep, *fSubJobInfo, querySteps, projectSteps, deliverySteps); + else + throw IDBExcept(ERR_UNION_IN_SUBQUERY); + + if (fSubJobInfo->trace) + { + ostringstream oss; + oss << boldStart << "\nsubquery " << fSubJobInfo->subLevel << "." << fOutJobInfo->subNum + << " steps:" << boldStop << endl; + ostream_iterator oIter(oss, "\n"); + copy(querySteps.begin(), querySteps.end(), oIter); + cout << oss.str(); + } + + // Add steps to the joblist. + fSubJobList->addQuery(querySteps); + fSubJobList->addDelivery(deliverySteps); + fSubJobList->putEngineComm(DistributedEngineComm::instance(fOutJobInfo->rm)); + csep->setDynamicParseTreeVec(fSubJobInfo->dynamicParseTreeVec); + + // Get the correlated steps + fCorrelatedSteps = fSubJobInfo->correlateSteps; + fSubReturnedCols = fSubJobInfo->deliveredCols; + + // Convert subquery to step. + SubQueryStep* sqs = new SubQueryStep(*fSubJobInfo); + sqs->tableOid(fVtable.tableOid()); + sqs->alias(fVtable.alias()); + sqs->subJoblist(fSubJobList); + sqs->setOutputRowGroup(fSubJobList->getOutputRowGroup()); + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, fSubJobInfo->fifoSize); + spdl->rowGroupDL(dl); + dl->OID(fVtable.tableOid()); + JobStepAssociation jsa; + jsa.outAdd(spdl); + (querySteps.back())->outputAssociation(jsa); + sqs->outputAssociation(jsa); + fSubQueryStep.reset(sqs); + + // Update the v-table columns and rowgroup + vector pos; + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + pos.push_back(2); + + CalpontSystemCatalog::OID tblOid = fVtable.tableOid(); + string tableName = fVtable.name(); + string alias = fVtable.alias(); + const RowGroup& rg = fSubJobList->getOutputRowGroup(); + Row row; + rg.initRow(&row); + uint64_t outputCols = + rg.getColumnCount() < fSubReturnedCols.size() ? rg.getColumnCount() : fSubReturnedCols.size(); + + for (uint64_t i = 0; i < outputCols; i++) + { + fVtable.addColumn(fSubReturnedCols[i]); + + // make sure the column type is the same as rowgroup + CalpontSystemCatalog::ColType ct = fVtable.columnType(i); + CalpontSystemCatalog::ColDataType colDataTypeInRg = row.getColTypes()[i]; + + if (dynamic_cast(fSubReturnedCols[i].get()) != NULL || + dynamic_cast(fSubReturnedCols[i].get()) != NULL) { - ostringstream oss; - oss << "$sub_" - << fSubJobInfo->subId - << "_" << fSubJobInfo->subLevel - << "_" << fOutJobInfo->subNum; - fVtable.alias(oss.str()); - } + // skip char/varchar/varbinary column because the colWidth in row is fudged. + if (colDataTypeInRg != CalpontSystemCatalog::VARCHAR && colDataTypeInRg != CalpontSystemCatalog::CHAR && + colDataTypeInRg != CalpontSystemCatalog::VARBINARY && + colDataTypeInRg != CalpontSystemCatalog::TEXT && colDataTypeInRg != CalpontSystemCatalog::BLOB) + { + ct.colWidth = row.getColumnWidth(i); + ct.colDataType = row.getColTypes()[i]; + ct.charsetNumber = row.getCharsetNumber(i); + ct.scale = row.getScale(i); - fSubJobInfo->subAlias = fVtable.alias(); //@bug5844, unique alias for sub - - // Make the jobsteps out of the execution plan. - JobStepVector querySteps; - JobStepVector projectSteps; - DeliveredTableMap deliverySteps; - - if (csep->unionVec().size() == 0) - makeJobSteps(csep, *fSubJobInfo, querySteps, projectSteps, deliverySteps); - else if (subInFromClause) - makeUnionJobSteps(csep, *fSubJobInfo, querySteps, projectSteps, deliverySteps); - else - throw IDBExcept(ERR_UNION_IN_SUBQUERY); - - if (fSubJobInfo->trace) - { - ostringstream oss; - oss << boldStart - << "\nsubquery " << fSubJobInfo->subLevel << "." << fOutJobInfo->subNum << " steps:" - << boldStop << endl; - ostream_iterator oIter(oss, "\n"); - copy(querySteps.begin(), querySteps.end(), oIter); - cout << oss.str(); - } - - // Add steps to the joblist. - fSubJobList->addQuery(querySteps); - fSubJobList->addDelivery(deliverySteps); - fSubJobList->putEngineComm(DistributedEngineComm::instance(fOutJobInfo->rm)); - csep->setDynamicParseTreeVec(fSubJobInfo->dynamicParseTreeVec); - - // Get the correlated steps - fCorrelatedSteps = fSubJobInfo->correlateSteps; - fSubReturnedCols = fSubJobInfo->deliveredCols; - - // Convert subquery to step. - SubQueryStep* sqs = - new SubQueryStep(*fSubJobInfo); - sqs->tableOid(fVtable.tableOid()); - sqs->alias(fVtable.alias()); - sqs->subJoblist(fSubJobList); - sqs->setOutputRowGroup(fSubJobList->getOutputRowGroup()); - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, fSubJobInfo->fifoSize); - spdl->rowGroupDL(dl); - dl->OID(fVtable.tableOid()); - JobStepAssociation jsa; - jsa.outAdd(spdl); - (querySteps.back())->outputAssociation(jsa); - sqs->outputAssociation(jsa); - fSubQueryStep.reset(sqs); - - // Update the v-table columns and rowgroup - vector pos; - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - pos.push_back(2); - - CalpontSystemCatalog::OID tblOid = fVtable.tableOid(); - string tableName = fVtable.name(); - string alias = fVtable.alias(); - const RowGroup& rg = fSubJobList->getOutputRowGroup(); - Row row; - rg.initRow(&row); - uint64_t outputCols = rg.getColumnCount() < fSubReturnedCols.size() ? - rg.getColumnCount() : fSubReturnedCols.size() ; - - for (uint64_t i = 0; i < outputCols; i++) - { - fVtable.addColumn(fSubReturnedCols[i]); - - // make sure the column type is the same as rowgroup - CalpontSystemCatalog::ColType ct = fVtable.columnType(i); - CalpontSystemCatalog::ColDataType colDataTypeInRg = row.getColTypes()[i]; - - if (dynamic_cast(fSubReturnedCols[i].get()) != NULL || - dynamic_cast(fSubReturnedCols[i].get()) != NULL) + if (colDataTypeInRg != CalpontSystemCatalog::FLOAT && + colDataTypeInRg != CalpontSystemCatalog::UFLOAT && + colDataTypeInRg != CalpontSystemCatalog::DOUBLE && + colDataTypeInRg != CalpontSystemCatalog::UDOUBLE && + colDataTypeInRg != CalpontSystemCatalog::LONGDOUBLE) { - // skip char/varchar/varbinary column because the colWidth in row is fudged. - if (colDataTypeInRg != CalpontSystemCatalog::VARCHAR && - colDataTypeInRg != CalpontSystemCatalog::CHAR && - colDataTypeInRg != CalpontSystemCatalog::VARBINARY && - colDataTypeInRg != CalpontSystemCatalog::TEXT && - colDataTypeInRg != CalpontSystemCatalog::BLOB) - { - ct.colWidth = row.getColumnWidth(i); - ct.colDataType = row.getColTypes()[i]; - ct.charsetNumber = row.getCharsetNumber(i); - ct.scale = row.getScale(i); - - if (colDataTypeInRg != CalpontSystemCatalog::FLOAT && - colDataTypeInRg != CalpontSystemCatalog::UFLOAT && - colDataTypeInRg != CalpontSystemCatalog::DOUBLE && - colDataTypeInRg != CalpontSystemCatalog::UDOUBLE && - colDataTypeInRg != CalpontSystemCatalog::LONGDOUBLE) - { - if (ct.scale != 0 && ct.precision != -1) - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - ct.precision = row.getPrecision(i); - fVtable.columnType(ct, i); - } - } - // MySQL timestamp/time/date/datetime type is different from IDB type - else if (colDataTypeInRg == CalpontSystemCatalog::DATE || - colDataTypeInRg == CalpontSystemCatalog::DATETIME || - colDataTypeInRg == CalpontSystemCatalog::TIMESTAMP || - colDataTypeInRg == CalpontSystemCatalog::TIME) - { - ct.colWidth = row.getColumnWidth(i); - ct.colDataType = row.getColTypes()[i]; - ct.scale = row.getScale(i); - ct.precision = row.getPrecision(i); - fVtable.columnType(ct, i); + if (ct.scale != 0 && ct.precision != -1) + ct.colDataType = CalpontSystemCatalog::DECIMAL; } - // build tuple info to export to outer query - TupleInfo ti(setTupleInfo(fVtable.columnType(i), fVtable.columnOid(i), *fOutJobInfo, - tblOid, fVtable.columns()[i].get(), alias)); - - if (i < rg.getColumnCount()) - { - pos.push_back(pos.back() + ti.width); - oids.push_back(ti.oid); - keys.push_back(ti.key); - types.push_back(ti.dtype); - csNums.push_back(ti.csNum); - scale.push_back(ti.scale); - precision.push_back(ti.precision); - } - - fOutJobInfo->vtableColTypes[UniqId(fVtable.columnOid(i), fVtable.alias(), "", "")] = - fVtable.columnType(i); + ct.precision = row.getPrecision(i); + fVtable.columnType(ct, i); + } + } + // MySQL timestamp/time/date/datetime type is different from IDB type + else if (colDataTypeInRg == CalpontSystemCatalog::DATE || + colDataTypeInRg == CalpontSystemCatalog::DATETIME || + colDataTypeInRg == CalpontSystemCatalog::TIMESTAMP || + colDataTypeInRg == CalpontSystemCatalog::TIME) + { + ct.colWidth = row.getColumnWidth(i); + ct.colDataType = row.getColTypes()[i]; + ct.scale = row.getScale(i); + ct.precision = row.getPrecision(i); + fVtable.columnType(ct, i); } - RowGroup rg1(oids.size(), pos, oids, keys, types, csNums, scale, precision, csep->stringTableThreshold()); - rg1.setUseStringTable(rg.usesStringTable()); + // build tuple info to export to outer query + TupleInfo ti(setTupleInfo(fVtable.columnType(i), fVtable.columnOid(i), *fOutJobInfo, tblOid, + fVtable.columns()[i].get(), alias)); - dynamic_cast(fSubQueryStep.get())->setOutputRowGroup(rg1); + if (i < rg.getColumnCount()) + { + pos.push_back(pos.back() + ti.width); + oids.push_back(ti.oid); + keys.push_back(ti.key); + types.push_back(ti.dtype); + csNums.push_back(ti.csNum); + scale.push_back(ti.scale); + precision.push_back(ti.precision); + } - return fSubQueryStep; + fOutJobInfo->vtableColTypes[UniqId(fVtable.columnOid(i), fVtable.alias(), "", "")] = + fVtable.columnType(i); + } + + RowGroup rg1(oids.size(), pos, oids, keys, types, csNums, scale, precision, csep->stringTableThreshold()); + rg1.setUseStringTable(rg.usesStringTable()); + + dynamic_cast(fSubQueryStep.get())->setOutputRowGroup(rg1); + + return fSubQueryStep; } - void SubQueryTransformer::checkCorrelateInfo(TupleHashJoinStep* thjs, const JobInfo& jobInfo) { - int pos = (thjs->correlatedSide() == 1) ? thjs->sequence2() : thjs->sequence1(); + int pos = (thjs->correlatedSide() == 1) ? thjs->sequence2() : thjs->sequence1(); - if (pos == -1 || (size_t) pos >= fVtable.columns().size()) - { - uint64_t id = (thjs->correlatedSide() == 1) ? thjs->tupleId2() : thjs->tupleId1(); - string alias = jobInfo.keyInfo->tupleKeyVec[id].fTable; - string name = jobInfo.keyInfo->keyName[id]; + if (pos == -1 || (size_t)pos >= fVtable.columns().size()) + { + uint64_t id = (thjs->correlatedSide() == 1) ? thjs->tupleId2() : thjs->tupleId1(); + string alias = jobInfo.keyInfo->tupleKeyVec[id].fTable; + string name = jobInfo.keyInfo->keyName[id]; - if (!name.empty() && alias.length() > 0) - name = alias + "." + name; + if (!name.empty() && alias.length() > 0) + name = alias + "." + name; - Message::Args args; - args.add(name); - string errMsg(IDBErrorInfo::instance()->errorMsg(ERR_CORRELATE_COL_MISSING, args)); - cerr << errMsg << ": " << pos << endl; - throw IDBExcept(errMsg, ERR_CORRELATE_COL_MISSING); - } + Message::Args args; + args.add(name); + string errMsg(IDBErrorInfo::instance()->errorMsg(ERR_CORRELATE_COL_MISSING, args)); + cerr << errMsg << ": " << pos << endl; + throw IDBExcept(errMsg, ERR_CORRELATE_COL_MISSING); + } } - void SubQueryTransformer::updateCorrelateInfo() { - // put vtable into the table list to resolve correlated filters - // Temp fix for @bug3932 until outer join has no dependency on table order. - // Insert at [1], not to mess with OUTER join and hint(INFINIDB_ORDERED -- bug2317). - fOutJobInfo->tableList.insert(fOutJobInfo->tableList.begin() + 1, makeTableKey( - *fOutJobInfo, fVtable.tableOid(), fVtable.name(), fVtable.alias(), "", fVtable.view())); + // put vtable into the table list to resolve correlated filters + // Temp fix for @bug3932 until outer join has no dependency on table order. + // Insert at [1], not to mess with OUTER join and hint(INFINIDB_ORDERED -- bug2317). + fOutJobInfo->tableList.insert( + fOutJobInfo->tableList.begin() + 1, + makeTableKey(*fOutJobInfo, fVtable.tableOid(), fVtable.name(), fVtable.alias(), "", fVtable.view())); - // tables in outer level - set outTables; + // tables in outer level + set outTables; - for (uint64_t i = 0; i < fOutJobInfo->tableList.size(); i++) - outTables.insert(fOutJobInfo->tableList[i]); + for (uint64_t i = 0; i < fOutJobInfo->tableList.size(); i++) + outTables.insert(fOutJobInfo->tableList[i]); - // tables in subquery level - set subTables; + // tables in subquery level + set subTables; - for (uint64_t i = 0; i < fSubJobInfo->tableList.size(); i++) - subTables.insert(fSubJobInfo->tableList[i]); + for (uint64_t i = 0; i < fSubJobInfo->tableList.size(); i++) + subTables.insert(fSubJobInfo->tableList[i]); - // any function joins - fOutJobInfo->functionJoins.insert(fOutJobInfo->functionJoins.end(), - fSubJobInfo->functionJoins.begin(), fSubJobInfo->functionJoins.end()); + // any function joins + fOutJobInfo->functionJoins.insert(fOutJobInfo->functionJoins.end(), fSubJobInfo->functionJoins.begin(), + fSubJobInfo->functionJoins.end()); - // Update correlated steps - const map& subMap = fVtable.columnMap(); + // Update correlated steps + const map& subMap = fVtable.columnMap(); - for (JobStepVector::iterator i = fCorrelatedSteps.begin(); i != fCorrelatedSteps.end(); i++) + for (JobStepVector::iterator i = fCorrelatedSteps.begin(); i != fCorrelatedSteps.end(); i++) + { + TupleHashJoinStep* thjs = dynamic_cast(i->get()); + ExpressionStep* es = dynamic_cast(i->get()); + + if (thjs) { - TupleHashJoinStep* thjs = dynamic_cast(i->get()); - ExpressionStep* es = dynamic_cast(i->get()); + if (thjs->getJoinType() & CORRELATED) + { + checkCorrelateInfo(thjs, *fSubJobInfo); + thjs->setJoinType(thjs->getJoinType() ^ CORRELATED); - if (thjs) + if (thjs->correlatedSide() == 1) { - if (thjs->getJoinType() & CORRELATED) - { - checkCorrelateInfo(thjs, *fSubJobInfo); - thjs->setJoinType(thjs->getJoinType() ^ CORRELATED); + int pos = thjs->sequence2(); + thjs->tableOid2(fVtable.tableOid()); + thjs->oid2(fVtable.columnOid(pos)); + thjs->alias2(fVtable.alias()); + thjs->view2(fVtable.columns()[pos]->viewName()); + thjs->schema2(fVtable.columns()[pos]->schemaName()); + thjs->dictOid2(0); + thjs->sequence2(-1); + thjs->joinId(fOutJobInfo->joinNum++); - if (thjs->correlatedSide() == 1) - { - int pos = thjs->sequence2(); - thjs->tableOid2(fVtable.tableOid()); - thjs->oid2(fVtable.columnOid(pos)); - thjs->alias2(fVtable.alias()); - thjs->view2(fVtable.columns()[pos]->viewName()); - thjs->schema2(fVtable.columns()[pos]->schemaName()); - thjs->dictOid2(0); - thjs->sequence2(-1); - thjs->joinId(fOutJobInfo->joinNum++); - - CalpontSystemCatalog::ColType ct2 = fVtable.columnType(pos); - TupleInfo ti(setTupleInfo(ct2, thjs->oid2(), *fOutJobInfo, thjs->tableOid2(), - fVtable.columns()[pos].get(), thjs->alias2())); - thjs->tupleId2(ti.key); - } - else - { - int pos = thjs->sequence1(); - thjs->tableOid1(fVtable.tableOid()); - thjs->oid1(fVtable.columnOid(pos)); - thjs->alias1(fVtable.alias()); - thjs->view1(fVtable.columns()[pos]->viewName()); - thjs->schema1(fVtable.columns()[pos]->schemaName()); - thjs->dictOid1(0); - thjs->sequence1(-1); - thjs->joinId(fOutJobInfo->joinNum++); - - CalpontSystemCatalog::ColType ct1 = fVtable.columnType(pos); - TupleInfo ti(setTupleInfo(ct1, thjs->oid1(), *fOutJobInfo, thjs->tableOid1(), - fVtable.columns()[pos].get(), thjs->alias1())); - thjs->tupleId1(ti.key); - } - } - - /* - else // oid1 == 0, dictionary column - { - CalpontSystemCatalog::ColType dct; - dct.colDataType = CalpontSystemCatalog::BIGINT; - dct.colWidth = 8; - dct.scale = 0; - dct.precision = 0; - } - */ - } - else if (es) - { - // already handled by function join - if (es->functionJoin()) - continue; - - vector& scList = es->columns(); - vector& tableOids = es->tableOids(); - vector& aliases = es->aliases(); - vector& views = es->views(); - vector& schemas = es->schemas(); - vector& tableKeys = es->tableKeys(); - vector& columnKeys = es->columnKeys(); - - // update simple columns - for (uint64_t j = 0; j < scList.size(); j++) - { - SimpleColumn* sc = dynamic_cast(scList[j]); - - if (sc != NULL) - { - if (subTables.find(tableKeys[j]) != subTables.end()) - { - const map::const_iterator k = - subMap.find(UniqId(sc->oid(), aliases[j], schemas[j], views[j])); - - if (k == subMap.end()) - throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); - - sc->schemaName(""); - sc->tableName(fVtable.name()); - sc->tableAlias(fVtable.alias()); - sc->viewName(fVtable.view()); - sc->oid(fVtable.columnOid(k->second)); - sc->columnName(fVtable.columns()[k->second]->columnName()); - const CalpontSystemCatalog::ColType& ct = fVtable.columnType(k->second); - TupleInfo ti = setTupleInfo( - ct, sc->oid(), *fOutJobInfo, fVtable.tableOid(), sc, fVtable.alias()); - - tableOids[j] = execplan::CNX_VTABLE_ID; - aliases[j] = sc->tableAlias(); - views[j] = sc->viewName(); - schemas[j] = sc->schemaName(); - columnKeys[j] = ti.key; - tableKeys[j] = getTableKey(*fOutJobInfo, ti.key); - } - else - { - const CalpontSystemCatalog::ColType& - ct = fOutJobInfo->keyInfo->colType[columnKeys[j]]; - sc->joinInfo(0); - TupleInfo ti = setTupleInfo( - ct, sc->oid(), *fOutJobInfo, tableOids[j], sc, aliases[j]); - - columnKeys[j] = ti.key; - tableKeys[j] = getTableKey(*fOutJobInfo, ti.key); - } - } - else if (dynamic_cast(scList[j]) != NULL) - { - // workaround for window function IN/EXISTS subquery - const map::const_iterator k = - subMap.find(UniqId(scList[j]->expressionId(), "", "", "")); - - if (k == subMap.end()) - throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); - - sc = fVtable.columns()[k->second].get(); - es->substitute(j, fVtable.columns()[k->second]); - CalpontSystemCatalog::ColType ct = sc->colType(); - string alias(extractTableAlias(sc)); - CalpontSystemCatalog::OID tblOid = fVtable.tableOid(); - TupleInfo ti(setTupleInfo(ct, sc->oid(), *fOutJobInfo, tblOid, sc, alias)); - - tableOids[j] = execplan::CNX_VTABLE_ID; - columnKeys[j] = ti.key; - tableKeys[j] = getTableKey(*fOutJobInfo, ti.key); - } - } - - es->updateColumnOidAlias(*fOutJobInfo); - - // update function join info - boost::shared_ptr& fji = es->functionJoinInfo(); - - if (fji && fji->fCorrelatedSide) - { - // replace sub side with a virtual column - int64_t subSide = (fji->fCorrelatedSide == 1) ? 1 : 0; - ReturnedColumn* rc = fji->fExpression[subSide]; - SimpleColumn* sc = dynamic_cast(rc); - - if (sc == NULL) - { - UniqId colId = UniqId(rc->expressionId(), "", "", ""); - const map::const_iterator k = subMap.find(colId); - - if (k == subMap.end()) - throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); - - SSC vc = fVtable.columns()[k->second]; - sc = vc.get(); - } - - // virtual table info in outer query - TupleInfo ti(setTupleInfo(sc->colType(), sc->oid(), *fOutJobInfo, - fVtable.tableOid(), sc, fVtable.alias())); - - set cids; - cids.insert(ti.key); - fji->fJoinKey[subSide] = ti.key; - fji->fTableKey[subSide] = ti.tkey; - fji->fColumnKeys[subSide] = cids; - fji->fTableOid[subSide] = fVtable.tableOid(); - fji->fAlias[subSide] = fVtable.alias(); - fji->fView[subSide] = fVtable.view(); - fji->fSchema[subSide] = ""; - - // turn off correlated flag - fji->fExpression[fji->fCorrelatedSide - 1]->joinInfo(0); - fji->fJoinType ^= CORRELATED; - fji->fJoinId = 0; - } + CalpontSystemCatalog::ColType ct2 = fVtable.columnType(pos); + TupleInfo ti(setTupleInfo(ct2, thjs->oid2(), *fOutJobInfo, thjs->tableOid2(), + fVtable.columns()[pos].get(), thjs->alias2())); + thjs->tupleId2(ti.key); } else { - JobStep* j = i->get(); - uint32_t tid = -1; - uint64_t cid = j->tupleId(); + int pos = thjs->sequence1(); + thjs->tableOid1(fVtable.tableOid()); + thjs->oid1(fVtable.columnOid(pos)); + thjs->alias1(fVtable.alias()); + thjs->view1(fVtable.columns()[pos]->viewName()); + thjs->schema1(fVtable.columns()[pos]->schemaName()); + thjs->dictOid1(0); + thjs->sequence1(-1); + thjs->joinId(fOutJobInfo->joinNum++); - if (cid != (uint64_t) - 1) - tid = getTableKey(*fOutJobInfo, cid); - else - tid = getTableKey(*fOutJobInfo, j); - - if (outTables.find(tid) == outTables.end()) - { - if (subMap.find( - UniqId(j->oid(), j->alias(), j->schema(), j->view(), 0)) != subMap.end()) - //throw CorrelateFailExcept(); - throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); - } + CalpontSystemCatalog::ColType ct1 = fVtable.columnType(pos); + TupleInfo ti(setTupleInfo(ct1, thjs->oid1(), *fOutJobInfo, thjs->tableOid1(), + fVtable.columns()[pos].get(), thjs->alias1())); + thjs->tupleId1(ti.key); } - } -} + } + /* + else // oid1 == 0, dictionary column + { + CalpontSystemCatalog::ColType dct; + dct.colDataType = CalpontSystemCatalog::BIGINT; + dct.colWidth = 8; + dct.scale = 0; + dct.precision = 0; + } + */ + } + else if (es) + { + // already handled by function join + if (es->functionJoin()) + continue; + + vector& scList = es->columns(); + vector& tableOids = es->tableOids(); + vector& aliases = es->aliases(); + vector& views = es->views(); + vector& schemas = es->schemas(); + vector& tableKeys = es->tableKeys(); + vector& columnKeys = es->columnKeys(); + + // update simple columns + for (uint64_t j = 0; j < scList.size(); j++) + { + SimpleColumn* sc = dynamic_cast(scList[j]); + + if (sc != NULL) + { + if (subTables.find(tableKeys[j]) != subTables.end()) + { + const map::const_iterator k = + subMap.find(UniqId(sc->oid(), aliases[j], schemas[j], views[j])); + + if (k == subMap.end()) + throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); + + sc->schemaName(""); + sc->tableName(fVtable.name()); + sc->tableAlias(fVtable.alias()); + sc->viewName(fVtable.view()); + sc->oid(fVtable.columnOid(k->second)); + sc->columnName(fVtable.columns()[k->second]->columnName()); + const CalpontSystemCatalog::ColType& ct = fVtable.columnType(k->second); + TupleInfo ti = setTupleInfo(ct, sc->oid(), *fOutJobInfo, fVtable.tableOid(), sc, fVtable.alias()); + + tableOids[j] = execplan::CNX_VTABLE_ID; + aliases[j] = sc->tableAlias(); + views[j] = sc->viewName(); + schemas[j] = sc->schemaName(); + columnKeys[j] = ti.key; + tableKeys[j] = getTableKey(*fOutJobInfo, ti.key); + } + else + { + const CalpontSystemCatalog::ColType& ct = fOutJobInfo->keyInfo->colType[columnKeys[j]]; + sc->joinInfo(0); + TupleInfo ti = setTupleInfo(ct, sc->oid(), *fOutJobInfo, tableOids[j], sc, aliases[j]); + + columnKeys[j] = ti.key; + tableKeys[j] = getTableKey(*fOutJobInfo, ti.key); + } + } + else if (dynamic_cast(scList[j]) != NULL) + { + // workaround for window function IN/EXISTS subquery + const map::const_iterator k = + subMap.find(UniqId(scList[j]->expressionId(), "", "", "")); + + if (k == subMap.end()) + throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); + + sc = fVtable.columns()[k->second].get(); + es->substitute(j, fVtable.columns()[k->second]); + CalpontSystemCatalog::ColType ct = sc->colType(); + string alias(extractTableAlias(sc)); + CalpontSystemCatalog::OID tblOid = fVtable.tableOid(); + TupleInfo ti(setTupleInfo(ct, sc->oid(), *fOutJobInfo, tblOid, sc, alias)); + + tableOids[j] = execplan::CNX_VTABLE_ID; + columnKeys[j] = ti.key; + tableKeys[j] = getTableKey(*fOutJobInfo, ti.key); + } + } + + es->updateColumnOidAlias(*fOutJobInfo); + + // update function join info + boost::shared_ptr& fji = es->functionJoinInfo(); + + if (fji && fji->fCorrelatedSide) + { + // replace sub side with a virtual column + int64_t subSide = (fji->fCorrelatedSide == 1) ? 1 : 0; + ReturnedColumn* rc = fji->fExpression[subSide]; + SimpleColumn* sc = dynamic_cast(rc); + + if (sc == NULL) + { + UniqId colId = UniqId(rc->expressionId(), "", "", ""); + const map::const_iterator k = subMap.find(colId); + + if (k == subMap.end()) + throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); + + SSC vc = fVtable.columns()[k->second]; + sc = vc.get(); + } + + // virtual table info in outer query + TupleInfo ti( + setTupleInfo(sc->colType(), sc->oid(), *fOutJobInfo, fVtable.tableOid(), sc, fVtable.alias())); + + set cids; + cids.insert(ti.key); + fji->fJoinKey[subSide] = ti.key; + fji->fTableKey[subSide] = ti.tkey; + fji->fColumnKeys[subSide] = cids; + fji->fTableOid[subSide] = fVtable.tableOid(); + fji->fAlias[subSide] = fVtable.alias(); + fji->fView[subSide] = fVtable.view(); + fji->fSchema[subSide] = ""; + + // turn off correlated flag + fji->fExpression[fji->fCorrelatedSide - 1]->joinInfo(0); + fji->fJoinType ^= CORRELATED; + fji->fJoinId = 0; + } + } + else + { + JobStep* j = i->get(); + uint32_t tid = -1; + uint64_t cid = j->tupleId(); + + if (cid != (uint64_t)-1) + tid = getTableKey(*fOutJobInfo, cid); + else + tid = getTableKey(*fOutJobInfo, j); + + if (outTables.find(tid) == outTables.end()) + { + if (subMap.find(UniqId(j->oid(), j->alias(), j->schema(), j->view(), 0)) != subMap.end()) + // throw CorrelateFailExcept(); + throw IDBExcept(logging::ERR_NON_SUPPORT_SUB_QUERY_TYPE); + } + } + } +} void SubQueryTransformer::run() { - // not to be called for base class + // not to be called for base class } - // ------ SimpleScalarTransformer ------ -SimpleScalarTransformer::SimpleScalarTransformer(JobInfo* jobInfo, SErrorInfo& status, bool e) : - SubQueryTransformer(jobInfo, status), - fInputDl(NULL), - fDlIterator(-1), - fEmptyResultSet(true), - fExistFilter(e) +SimpleScalarTransformer::SimpleScalarTransformer(JobInfo* jobInfo, SErrorInfo& status, bool e) + : SubQueryTransformer(jobInfo, status) + , fInputDl(NULL) + , fDlIterator(-1) + , fEmptyResultSet(true) + , fExistFilter(e) { } - -SimpleScalarTransformer::SimpleScalarTransformer(const SubQueryTransformer& rhs) : - SubQueryTransformer(rhs.outJobInfo(), rhs.errorInfo()), - fInputDl(NULL), - fDlIterator(-1), - fEmptyResultSet(true), - fExistFilter(false) +SimpleScalarTransformer::SimpleScalarTransformer(const SubQueryTransformer& rhs) + : SubQueryTransformer(rhs.outJobInfo(), rhs.errorInfo()) + , fInputDl(NULL) + , fDlIterator(-1) + , fEmptyResultSet(true) + , fExistFilter(false) { - fSubJobList = rhs.subJobList(); - fSubQueryStep = rhs.subQueryStep(); + fSubJobList = rhs.subJobList(); + fSubQueryStep = rhs.subQueryStep(); } - SimpleScalarTransformer::~SimpleScalarTransformer() { } - void SimpleScalarTransformer::run() { - // set up receiver - fRowGroup = dynamic_cast(fSubQueryStep.get())->getOutputRowGroup(); - fRowGroup.initRow(&fRow, true); - fInputDl = fSubQueryStep->outputAssociation().outAt(0)->rowGroupDL(); - fDlIterator = fInputDl->getIterator(); + // set up receiver + fRowGroup = dynamic_cast(fSubQueryStep.get())->getOutputRowGroup(); + fRowGroup.initRow(&fRow, true); + fInputDl = fSubQueryStep->outputAssociation().outAt(0)->rowGroupDL(); + fDlIterator = fInputDl->getIterator(); - // run the subquery - fSubJobList->doQuery(); + // run the subquery + fSubJobList->doQuery(); - // retrieve the scalar result - getScalarResult(); + // retrieve the scalar result + getScalarResult(); - // check result count - if (fErrorInfo->errCode == ERR_MORE_THAN_1_ROW) - throw MoreThan1RowExcept(); + // check result count + if (fErrorInfo->errCode == ERR_MORE_THAN_1_ROW) + throw MoreThan1RowExcept(); } - void SimpleScalarTransformer::getScalarResult() { - RGData rgData; - bool more; + RGData rgData; + bool more; - more = fInputDl->next(fDlIterator, &rgData); + more = fInputDl->next(fDlIterator, &rgData); - while (more) + while (more) + { + fRowGroup.setData(&rgData); + + // Only need one row for scalar filter + if (fEmptyResultSet && fRowGroup.getRowCount() == 1) { - fRowGroup.setData(&rgData); + fEmptyResultSet = false; + Row row; + fRowGroup.initRow(&row); + fRowGroup.getRow(0, &row); + fRowData.reset(new uint8_t[fRow.getSize()]); + fRow.setData(fRowData.get()); + copyRow(row, &fRow); - // Only need one row for scalar filter - if (fEmptyResultSet && fRowGroup.getRowCount() == 1) - { - fEmptyResultSet = false; - Row row; - fRowGroup.initRow(&row); - fRowGroup.getRow(0, &row); - fRowData.reset(new uint8_t[fRow.getSize()]); - fRow.setData(fRowData.get()); - copyRow(row, &fRow); - - // For exist filter, stop the query after one or more rows retrieved. - if (fExistFilter) - { - fErrorInfo->errMsg = IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW); - fErrorInfo->errCode = ERR_MORE_THAN_1_ROW; - } - } - - // more than 1 row case: - // not empty set, and get new rows - // empty set, but get more than 1 rows - else if (fRowGroup.getRowCount() > 0) - { - // Stop the query after some rows retrieved. - fEmptyResultSet = false; - fErrorInfo->errMsg = IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW); - fErrorInfo->errCode = ERR_MORE_THAN_1_ROW; - } - - // For scalar filter, have to check all blocks to ensure only one row. - if (fErrorInfo->errCode != 0) - while (more) more = fInputDl->next(fDlIterator, &rgData); - else - more = fInputDl->next(fDlIterator, &rgData); + // For exist filter, stop the query after one or more rows retrieved. + if (fExistFilter) + { + fErrorInfo->errMsg = IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW); + fErrorInfo->errCode = ERR_MORE_THAN_1_ROW; + } } + + // more than 1 row case: + // not empty set, and get new rows + // empty set, but get more than 1 rows + else if (fRowGroup.getRowCount() > 0) + { + // Stop the query after some rows retrieved. + fEmptyResultSet = false; + fErrorInfo->errMsg = IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW); + fErrorInfo->errCode = ERR_MORE_THAN_1_ROW; + } + + // For scalar filter, have to check all blocks to ensure only one row. + if (fErrorInfo->errCode != 0) + while (more) + more = fInputDl->next(fDlIterator, &rgData); + else + more = fInputDl->next(fDlIterator, &rgData); + } } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/subquerytransformer.h b/dbcon/joblist/subquerytransformer.h index ffea628f2..5ab897ce4 100644 --- a/dbcon/joblist/subquerytransformer.h +++ b/dbcon/joblist/subquerytransformer.h @@ -17,13 +17,11 @@ // $Id: subquerytransformer.h 6370 2010-03-18 02:58:09Z xlou $ - /** @file */ #ifndef SUBQUERY_TRANSFORMER_H #define SUBQUERY_TRANSFORMER_H - #include "elementtype.h" #include "jobstep.h" #include "joblist.h" @@ -32,213 +30,199 @@ namespace execplan { - class CalpontSelectExecutionPlan; } - namespace joblist { - struct JobInfo; -class TupleHashJoinStep; +class TupleHashJoinStep; class SubQueryTransformer { + public: + /** @brief SubQueryTransformer constructor + * @param jobInfo + * @param errorInfo + */ + SubQueryTransformer(JobInfo*, SErrorInfo&); -public: - /** @brief SubQueryTransformer constructor - * @param jobInfo - * @param errorInfo - */ - SubQueryTransformer(JobInfo*, SErrorInfo&); + /** @brief SubQueryTransformer constructor + * @param jobInfo + * @param errorInfo + * @param view + */ + SubQueryTransformer(JobInfo*, SErrorInfo&, const std::string&); - /** @brief SubQueryTransformer constructor - * @param jobInfo - * @param errorInfo - * @param view - */ - SubQueryTransformer(JobInfo*, SErrorInfo&, const std::string&); + /** @brief SubQueryTransformer constructor + * @param jobInfo + * @param errorInfo + * @param alias + * @param view + */ + SubQueryTransformer(JobInfo*, SErrorInfo&, const std::string&, const std::string&); - /** @brief SubQueryTransformer constructor - * @param jobInfo - * @param errorInfo - * @param alias - * @param view - */ - SubQueryTransformer( - JobInfo*, SErrorInfo&, const std::string&, const std::string&); + /** @brief SubQueryTransformer destructor + */ + virtual ~SubQueryTransformer(); - /** @brief SubQueryTransformer destructor - */ - virtual ~SubQueryTransformer(); + /** @brief virtual make a subquery step + * @param csep the execution plan + * @param b if the subquery is in FROM clause + * @returns boost::shared_ptr + */ + virtual SJSTEP& makeSubQueryStep(execplan::CalpontSelectExecutionPlan*, bool b = false); - /** @brief virtual make a subquery step - * @param csep the execution plan - * @param b if the subquery is in FROM clause - * @returns boost::shared_ptr - */ - virtual SJSTEP& makeSubQueryStep(execplan::CalpontSelectExecutionPlan*, bool b = false); + /** @brief virtual make a virtual table + * @param csep the execution plan + * @returns const VirtualTable& + */ + virtual void updateCorrelateInfo(); - /** @brief virtual make a virtual table - * @param csep the execution plan - * @returns const VirtualTable& - */ - virtual void updateCorrelateInfo(); + /** @brief virtual void run method + */ + virtual void run(); - /** @brief virtual void run method - */ - virtual void run(); + /** @brief virtual outer query jobinfo + * @returns JobInfo* + */ + virtual JobInfo* outJobInfo() const + { + return fOutJobInfo; + } - /** @brief virtual outer query jobinfo - * @returns JobInfo* - */ - virtual JobInfo* outJobInfo() const - { - return fOutJobInfo; - } + /** @brief virtual subquery jobinfo + * @returns JobInfo* + */ + virtual JobInfo* subJobInfo() const + { + return fSubJobInfo; + } - /** @brief virtual subquery jobinfo - * @returns JobInfo* - */ - virtual JobInfo* subJobInfo() const - { - return fSubJobInfo; - } + /** @brief virtual error info pointer + * @returns SErrorInfo& + */ + virtual SErrorInfo& errorInfo() const + { + return fErrorInfo; + } - /** @brief virtual error info pointer - * @returns SErrorInfo& - */ - virtual SErrorInfo& errorInfo() const - { - return fErrorInfo; - } + /** @brief virtual joblist + * @returns STJLP& + */ + virtual const STJLP& subJobList() const + { + return fSubJobList; + } - /** @brief virtual joblist - * @returns STJLP& - */ - virtual const STJLP& subJobList() const - { - return fSubJobList; - } + /** @brief virtual subquery step + * @returns SJSTEP& + */ + virtual const SJSTEP& subQueryStep() const + { + return fSubQueryStep; + } - /** @brief virtual subquery step - * @returns SJSTEP& - */ - virtual const SJSTEP& subQueryStep() const - { - return fSubQueryStep; - } + /** @brief virtual get correlated steps + * @returns const JobStepVector& + */ + virtual JobStepVector& correlatedSteps() + { + return fCorrelatedSteps; + } - /** @brief virtual get correlated steps - * @returns const JobStepVector& - */ - virtual JobStepVector& correlatedSteps() - { - return fCorrelatedSteps; - } + /** @brief get virtual table + * @returns const VirtualTable& + */ + const VirtualTable& virtualTable() const + { + return fVtable; + } - /** @brief get virtual table - * @returns const VirtualTable& - */ - const VirtualTable& virtualTable() const - { - return fVtable; - } + /** @brief set varbinary support flag + */ + void setVarbinaryOK() + { + fVtable.varbinaryOK(true); + } - /** @brief set varbinary support flag - */ - void setVarbinaryOK() - { - fVtable.varbinaryOK(true); - } + protected: + void checkCorrelateInfo(TupleHashJoinStep*, const JobInfo&); + JobInfo* fOutJobInfo; + JobInfo* fSubJobInfo; + SErrorInfo& fErrorInfo; + JobStepVector fCorrelatedSteps; + RetColsVector fSubReturnedCols; + STJLP fSubJobList; + SJSTEP fSubQueryStep; + VirtualTable fVtable; -protected: - - void checkCorrelateInfo(TupleHashJoinStep*, const JobInfo&); - - JobInfo* fOutJobInfo; - JobInfo* fSubJobInfo; - SErrorInfo& fErrorInfo; - JobStepVector fCorrelatedSteps; - RetColsVector fSubReturnedCols; - STJLP fSubJobList; - SJSTEP fSubQueryStep; - VirtualTable fVtable; - - // disable copy constructor and assignment operator - SubQueryTransformer(const SubQueryTransformer&); - SubQueryTransformer& operator=(const SubQueryTransformer&); - + // disable copy constructor and assignment operator + SubQueryTransformer(const SubQueryTransformer&); + SubQueryTransformer& operator=(const SubQueryTransformer&); }; - class SimpleScalarTransformer : public SubQueryTransformer { + public: + /** @brief SimpleScalarTransformer constructor + * @param jobInfo + * @param errorInfo for error info + * @param existFilter indicate if exist filter + */ + SimpleScalarTransformer(JobInfo* jobInfo, SErrorInfo& errInfo, bool existFilter); -public: - /** @brief SimpleScalarTransformer constructor - * @param jobInfo - * @param errorInfo for error info - * @param existFilter indicate if exist filter - */ - SimpleScalarTransformer(JobInfo* jobInfo, SErrorInfo& errInfo, bool existFilter); + /** @brief SimpleScalarTransformer constructor + * @param SubQueryTransformer + */ + SimpleScalarTransformer(const SubQueryTransformer& rhs); - /** @brief SimpleScalarTransformer constructor - * @param SubQueryTransformer - */ - SimpleScalarTransformer(const SubQueryTransformer& rhs); + /** @brief SimpleScalarTransformer destructor + */ + virtual ~SimpleScalarTransformer(); - /** @brief SimpleScalarTransformer destructor - */ - virtual ~SimpleScalarTransformer(); + /** @brief virtual void run method + */ + void run(); - /** @brief virtual void run method - */ - void run(); + /** @brief virtual get scalar result + * @param jobInfo + */ + virtual void getScalarResult(); - /** @brief virtual get scalar result - * @param jobInfo - */ - virtual void getScalarResult(); + /** @brief check if result set is empty + * @returns bool + */ + bool emptyResultSet() const + { + return fEmptyResultSet; + } - /** @brief check if result set is empty - * @returns bool - */ - bool emptyResultSet() const - { - return fEmptyResultSet; - } + /** @brief retrieve result row + * @returns Row + */ + const rowgroup::Row& resultRow() const + { + return fRow; + } - /** @brief retrieve result row - * @returns Row - */ - const rowgroup::Row& resultRow() const - { - return fRow; - } - - -protected: - RowGroupDL* fInputDl; - int fDlIterator; - rowgroup::RowGroup fRowGroup; - rowgroup::Row fRow; - boost::scoped_array fRowData; - bool fEmptyResultSet; - bool fExistFilter; - - // disable copy constructor and assignment operator - SimpleScalarTransformer(const SimpleScalarTransformer&); - SimpleScalarTransformer& operator=(const SimpleScalarTransformer&); + protected: + RowGroupDL* fInputDl; + int fDlIterator; + rowgroup::RowGroup fRowGroup; + rowgroup::Row fRow; + boost::scoped_array fRowData; + bool fEmptyResultSet; + bool fExistFilter; + // disable copy constructor and assignment operator + SimpleScalarTransformer(const SimpleScalarTransformer&); + SimpleScalarTransformer& operator=(const SimpleScalarTransformer&); }; - -} +} // namespace joblist #endif // SUBQUERY_STEP_H // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/tablecolumn.cpp b/dbcon/joblist/tablecolumn.cpp index c971a1966..be20292c8 100644 --- a/dbcon/joblist/tablecolumn.cpp +++ b/dbcon/joblist/tablecolumn.cpp @@ -33,266 +33,264 @@ using namespace messageqcpp; namespace joblist { - /** @brief constructor -*/ -TableColumn::TableColumn(const execplan::CalpontSystemCatalog::OID columnOID, const supportedType columnType) : - fColumnOID(columnOID), fIsNullColumn(true), fColumnType(columnType) + */ +TableColumn::TableColumn(const execplan::CalpontSystemCatalog::OID columnOID, const supportedType columnType) + : fColumnOID(columnOID), fIsNullColumn(true), fColumnType(columnType) { - preserialized.reset(new ByteStream()); + preserialized.reset(new ByteStream()); } TableColumn::TableColumn() : fColumnOID(0), fIsNullColumn(true), fColumnType(UNDEFINED) { - preserialized.reset(new ByteStream()); + preserialized.reset(new ByteStream()); }; void TableColumn::serialize() { -// cerr << "pre-serializing" << endl; - messageqcpp::ByteStream::octbyte rowCount; - messageqcpp::ByteStream::octbyte oid; - messageqcpp::ByteStream::byte nullFlag; - messageqcpp::ByteStream::byte columnType; + // cerr << "pre-serializing" << endl; + messageqcpp::ByteStream::octbyte rowCount; + messageqcpp::ByteStream::octbyte oid; + messageqcpp::ByteStream::byte nullFlag; + messageqcpp::ByteStream::byte columnType; - oid = fColumnOID; - *preserialized << oid; + oid = fColumnOID; + *preserialized << oid; - columnType = fColumnType; - *preserialized << columnType; + columnType = fColumnType; + *preserialized << columnType; - if (fIsNullColumn) - nullFlag = 1; - else - nullFlag = 0; + if (fIsNullColumn) + nullFlag = 1; + else + nullFlag = 0; - *preserialized << nullFlag; + *preserialized << nullFlag; - if (!fIsNullColumn) + if (!fIsNullColumn) + { + if (fColumnType == UINT64) { - if (fColumnType == UINT64) - { - rowCount = fIntValues->size(); - *preserialized << rowCount; - preserialized->append((uint8_t*) & (*fIntValues)[0], 8 * rowCount); - } - else if (fColumnType == STRING) - { - rowCount = fStrValues->size(); - *preserialized << rowCount; - - for (uint32_t i = 0; i < rowCount; i++) - *preserialized << (*fStrValues)[i]; - } + rowCount = fIntValues->size(); + *preserialized << rowCount; + preserialized->append((uint8_t*)&(*fIntValues)[0], 8 * rowCount); } + else if (fColumnType == STRING) + { + rowCount = fStrValues->size(); + *preserialized << rowCount; + + for (uint32_t i = 0; i < rowCount; i++) + *preserialized << (*fStrValues)[i]; + } + } } /** @brief serializes the object into the passed byte stream. -*/ + */ void TableColumn::serialize(messageqcpp::ByteStream& b) { - if (preserialized->length() != 0) + if (preserialized->length() != 0) + { + b += *preserialized; + preserialized->reset(); + // cerr << "returning a preserialized column" << endl; + return; + } + + messageqcpp::ByteStream::octbyte rowCount; + messageqcpp::ByteStream::octbyte oid; + messageqcpp::ByteStream::byte nullFlag; + messageqcpp::ByteStream::byte columnType; + + oid = fColumnOID; + b << oid; + + columnType = fColumnType; + b << columnType; + + if (fIsNullColumn) + nullFlag = 1; + else + nullFlag = 0; + + b << nullFlag; + + if (!fIsNullColumn) + { + if (fColumnType == UINT64) { - b += *preserialized; - preserialized->reset(); -// cerr << "returning a preserialized column" << endl; - return; + rowCount = fIntValues->size(); + b << rowCount; + b.append((uint8_t*)&(*fIntValues)[0], 8 * rowCount); } - - messageqcpp::ByteStream::octbyte rowCount; - messageqcpp::ByteStream::octbyte oid; - messageqcpp::ByteStream::byte nullFlag; - messageqcpp::ByteStream::byte columnType; - - oid = fColumnOID; - b << oid; - - columnType = fColumnType; - b << columnType; - - if (fIsNullColumn) - nullFlag = 1; - else - nullFlag = 0; - - b << nullFlag; - - if (!fIsNullColumn) + else if (fColumnType == STRING) { - if (fColumnType == UINT64) - { - rowCount = fIntValues->size(); - b << rowCount; - b.append((uint8_t*) & (*fIntValues)[0], 8 * rowCount); - } - else if (fColumnType == STRING) - { - rowCount = fStrValues->size(); - b << rowCount; + rowCount = fStrValues->size(); + b << rowCount; - for (uint32_t i = 0; i < rowCount; i++) - b << (*fStrValues)[i]; - } + for (uint32_t i = 0; i < rowCount; i++) + b << (*fStrValues)[i]; } - - + } } /** @brief inflates the object from the passed byte stream. -*/ + */ void TableColumn::unserialize(messageqcpp::ByteStream& b) { - messageqcpp::ByteStream::octbyte rowCount; - messageqcpp::ByteStream::octbyte oid; - messageqcpp::ByteStream::byte nullFlag; - messageqcpp::ByteStream::byte columnType; - uint32_t val32; - uint16_t val16; - uint8_t val8; + messageqcpp::ByteStream::octbyte rowCount; + messageqcpp::ByteStream::octbyte oid; + messageqcpp::ByteStream::byte nullFlag; + messageqcpp::ByteStream::byte columnType; + uint32_t val32; + uint16_t val16; + uint8_t val8; - b >> oid; - fColumnOID = oid; + b >> oid; + fColumnOID = oid; -// cout << "UN: oid = " << oid << endl; + // cout << "UN: oid = " << oid << endl; - b >> columnType; + b >> columnType; - /* Fudge fColumnType for onlookers. */ + /* Fudge fColumnType for onlookers. */ + if (columnType != STRING) + fColumnType = UINT64; + else + fColumnType = STRING; + + b >> nullFlag; + fIsNullColumn = (nullFlag != 0); + // cout << "UN (" << oid << "): is null: " << (int) nullFlag << endl; + + if (!fIsNullColumn) + { + b >> rowCount; + + // cout << "UN (" << oid << "): rowCount = " << rowCount << endl; if (columnType != STRING) - fColumnType = UINT64; - else - fColumnType = STRING; + fIntValues.reset(new std::vector()); - b >> nullFlag; - fIsNullColumn = (nullFlag != 0); -// cout << "UN (" << oid << "): is null: " << (int) nullFlag << endl; - - if (!fIsNullColumn) + /* XXXPAT: A switch on fColumnType is more concise, but I suspect this is + a little faster b/c of fewer jumps in the loop. Since it's a row-by-row operation, it + has to scream. */ + if (columnType == UINT8) { - b >> rowCount; + // cout << "UN (" << oid << "): is an 8\n"; + fIntValues->reserve(rowCount); -// cout << "UN (" << oid << "): rowCount = " << rowCount << endl; - if (columnType != STRING) - fIntValues.reset(new std::vector()); - - /* XXXPAT: A switch on fColumnType is more concise, but I suspect this is - a little faster b/c of fewer jumps in the loop. Since it's a row-by-row operation, it - has to scream. */ - if (columnType == UINT8) - { -// cout << "UN (" << oid << "): is an 8\n"; - fIntValues->reserve(rowCount); - - for (uint32_t i = 0; i < rowCount; ++i) - { - b >> val8; -// cout << "UN (" << oid << "): " << (int) val8 << " at " << i << endl; - fIntValues->push_back(val8); - } - } - else if (columnType == UINT16) - { -// cout << "UN (" << oid << "): is a 16\n"; - fIntValues->reserve(rowCount); - - for (uint32_t i = 0; i < rowCount; ++i) - { - b >> val16; -// cout << "UN (" << oid << "): " << val16 << " at " << i << endl; - fIntValues->push_back(val16); - } - } - else if (columnType == UINT32) - { -// cout << "UN (" << oid << "): is a 32\n"; - fIntValues->reserve(rowCount); - - for (uint32_t i = 0; i < rowCount; ++i) - { - b >> val32; -// cout << "UN (" << oid << "): " << val32 << " at " << i << endl; - fIntValues->push_back(val32); - } - } - else if (columnType == UINT64) - { - fIntValues->resize(rowCount); - memcpy(&(*fIntValues)[0], b.buf(), 8 * rowCount); - b.advance(8 * rowCount); - } - else if (columnType == STRING) - { - fStrValues.reset(new std::vector()); - fStrValues->reserve(rowCount); - std::string value; - - for (uint32_t i = 0; i < rowCount; i++) - { - b >> value; -// cout << "UN: " << value << endl; - fStrValues->push_back(value); - } - } + for (uint32_t i = 0; i < rowCount; ++i) + { + b >> val8; + // cout << "UN (" << oid << "): " << (int) val8 << " at " << i << endl; + fIntValues->push_back(val8); + } } + else if (columnType == UINT16) + { + // cout << "UN (" << oid << "): is a 16\n"; + fIntValues->reserve(rowCount); + + for (uint32_t i = 0; i < rowCount; ++i) + { + b >> val16; + // cout << "UN (" << oid << "): " << val16 << " at " << i << endl; + fIntValues->push_back(val16); + } + } + else if (columnType == UINT32) + { + // cout << "UN (" << oid << "): is a 32\n"; + fIntValues->reserve(rowCount); + + for (uint32_t i = 0; i < rowCount; ++i) + { + b >> val32; + // cout << "UN (" << oid << "): " << val32 << " at " << i << endl; + fIntValues->push_back(val32); + } + } + else if (columnType == UINT64) + { + fIntValues->resize(rowCount); + memcpy(&(*fIntValues)[0], b.buf(), 8 * rowCount); + b.advance(8 * rowCount); + } + else if (columnType == STRING) + { + fStrValues.reset(new std::vector()); + fStrValues->reserve(rowCount); + std::string value; + + for (uint32_t i = 0; i < rowCount; i++) + { + b >> value; + // cout << "UN: " << value << endl; + fStrValues->push_back(value); + } + } + } } -/** @brief adds the column and it's values to the passed NJLSysDataList or appends the values if the column is already included in the NJLSysDataList. -*/ -void TableColumn::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList, const std::vector& rids) +/** @brief adds the column and it's values to the passed NJLSysDataList or appends the values if the column is + * already included in the NJLSysDataList. + */ +void TableColumn::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList, + const std::vector& rids) { + execplan::ColumnResult* cr; + int idx = sysDataList.findColumn(fColumnOID); - execplan::ColumnResult* cr; - int idx = sysDataList.findColumn(fColumnOID); + if (idx >= 0) + { + cr = sysDataList.sysDataVec[idx]; + } + else + { + cr = new execplan::ColumnResult(); + cr->SetColumnOID(fColumnOID); + sysDataList.push_back(cr); + } - if (idx >= 0) + if (fColumnType == UINT64) + { + uint32_t vsize = fIntValues->size(); + bool putRids = (rids.size() == vsize); + + for (uint32_t i = 0; i < vsize; i++) { - cr = sysDataList.sysDataVec[idx]; + cr->PutData((*fIntValues)[i]); + + if (putRids) + { + cr->PutRid(rids[i]); + } + else + { + cr->PutRid(0); + } } - else + } + else + { + uint32_t vsize = fStrValues->size(); + bool putRids = (rids.size() == vsize); + + for (uint32_t i = 0; i < vsize; i++) { - cr = new execplan::ColumnResult(); - cr->SetColumnOID(fColumnOID); - sysDataList.push_back(cr); - } - - if (fColumnType == UINT64) - { - uint32_t vsize = fIntValues->size(); - bool putRids = (rids.size() == vsize); - - for (uint32_t i = 0; i < vsize; i++) - { - cr->PutData((*fIntValues)[i]); - - if (putRids) - { - cr->PutRid(rids[i]); - } - else - { - cr->PutRid(0); - } - } - } - else - { - uint32_t vsize = fStrValues->size(); - bool putRids = (rids.size() == vsize); - - for (uint32_t i = 0; i < vsize; i++) - { - cr->PutStringData((*fStrValues)[i]); - - if (putRids) - { - cr->PutRid(rids[i]); - } - else - { - cr->PutRid(0); - } - } + cr->PutStringData((*fStrValues)[i]); + + if (putRids) + { + cr->PutRid(rids[i]); + } + else + { + cr->PutRid(0); + } } + } } #if 0 void TableColumn::addToSysDataRids(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList, const std::vector& rids) @@ -331,5 +329,4 @@ void TableColumn::addToSysDataRids(execplan::CalpontSystemCatalog::NJLSysDataLis } #endif -} // namespace - +} // namespace joblist diff --git a/dbcon/joblist/tablecolumn.h b/dbcon/joblist/tablecolumn.h index 63b3cfba8..4a245bb49 100644 --- a/dbcon/joblist/tablecolumn.h +++ b/dbcon/joblist/tablecolumn.h @@ -43,7 +43,6 @@ namespace joblist { - /** @brief create a JobList object from a CalpontExecutionPlan object * * Class TableColumn contains a column and it's values. TableColumn objects are contained in a @@ -51,111 +50,111 @@ namespace joblist */ class TableColumn { -public: + public: + /** @brief enum with the supported value types. + */ + enum supportedType + { + UINT8, + UINT16, + UINT32, + UINT64, + STRING, + UNDEFINED + }; - /** @brief enum with the supported value types. - */ - enum supportedType - { - UINT8, - UINT16, - UINT32, - UINT64, - STRING, - UNDEFINED - }; + /** @brief constructor + */ + EXPORT TableColumn(const execplan::CalpontSystemCatalog::OID columnOID, const supportedType columnType); - /** @brief constructor - */ - EXPORT TableColumn(const execplan::CalpontSystemCatalog::OID columnOID, const supportedType columnType); + EXPORT TableColumn(); - EXPORT TableColumn(); + /** @brief getter for the column's OID. + */ + inline execplan::CalpontSystemCatalog::OID getColumnOID() const + { + return fColumnOID; + } - /** @brief getter for the column's OID. - */ - inline execplan::CalpontSystemCatalog::OID getColumnOID() const - { - return fColumnOID; - } + /** @brief getter for the column's values. + */ + inline const boost::shared_ptr > getIntValues() + { + return fIntValues; + } - /** @brief getter for the column's values. - */ - inline const boost::shared_ptr > getIntValues() - { - return fIntValues; - } + inline const boost::shared_ptr > getStrValues() + { + return fStrValues; + } - inline const boost::shared_ptr > getStrValues() - { - return fStrValues; - } + inline bool isNullColumn() const + { + return fIsNullColumn; + } - inline bool isNullColumn() const - { - return fIsNullColumn; - } + // pre-build the bytestream to be returned + EXPORT void serialize(); - // pre-build the bytestream to be returned - EXPORT void serialize(); + /** @brief serializes the object into the passed byte stream. + */ + EXPORT void serialize(messageqcpp::ByteStream& b); - /** @brief serializes the object into the passed byte stream. - */ - EXPORT void serialize(messageqcpp::ByteStream& b); + /** @brief inflates the object from the passed byte stream. + */ + EXPORT void unserialize(messageqcpp::ByteStream& b); - /** @brief inflates the object from the passed byte stream. - */ - EXPORT void unserialize(messageqcpp::ByteStream& b); - - /** @brief adds the column and it's values to the passed NJLSysDataList or appends the values if the column is already included in the NJLSysDataList. - */ - EXPORT void addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList, const std::vector& rids); + /** @brief adds the column and it's values to the passed NJLSysDataList or appends the values if the column + * is already included in the NJLSysDataList. + */ + EXPORT void addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList, + const std::vector& rids); #if 0 EXPORT void addToSysDataRids(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList, const std::vector& rids); #endif - inline void setIntValues(boost::shared_ptr > sv) - { - fIntValues = sv; - fIsNullColumn = fIntValues->empty(); - } + inline void setIntValues(boost::shared_ptr > sv) + { + fIntValues = sv; + fIsNullColumn = fIntValues->empty(); + } - inline void setStrValues(boost::shared_ptr > sv) - { - fStrValues = sv; - fIsNullColumn = fStrValues->empty(); - } + inline void setStrValues(boost::shared_ptr > sv) + { + fStrValues = sv; + fIsNullColumn = fStrValues->empty(); + } - inline supportedType getColumnType() - { - return fColumnType; - } + inline supportedType getColumnType() + { + return fColumnType; + } #ifdef TC_CHECK_RIDS - const std::vector& rids() const - { - return fRids; - } + const std::vector& rids() const + { + return fRids; + } #endif -private: - execplan::CalpontSystemCatalog::OID fColumnOID; - boost::shared_ptr > fIntValues; - boost::shared_ptr > fStrValues; - bool fIsNullColumn; - supportedType fColumnType; - boost::shared_ptr preserialized; + private: + execplan::CalpontSystemCatalog::OID fColumnOID; + boost::shared_ptr > fIntValues; + boost::shared_ptr > fStrValues; + bool fIsNullColumn; + supportedType fColumnType; + boost::shared_ptr preserialized; #ifdef TC_CHECK_RIDS - std::vector fRids; + std::vector fRids; #endif - // defaults okay - //TableColumn(const TableColumn& rhs); // no copies - //TableColumn& operator=(const TableColumn& rhs); // no assignments + // defaults okay + // TableColumn(const TableColumn& rhs); // no copies + // TableColumn& operator=(const TableColumn& rhs); // no assignments }; #undef EXPORT -} // namespace +} // namespace joblist #endif - diff --git a/dbcon/joblist/tdriver-agg.cpp b/dbcon/joblist/tdriver-agg.cpp index 636d0c86a..63da5ef46 100644 --- a/dbcon/joblist/tdriver-agg.cpp +++ b/dbcon/joblist/tdriver-agg.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: tdriver-agg.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: tdriver-agg.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -66,110 +66,111 @@ const uint32_t MAX_SIZE = 0x100000; const uint32_t MAX_ELEMENTS = 0x20000; const uint32_t NUM_THREADS = 4; const string datapath = "/home/zzhu/genii/tools/dbbuilder/lineitem.tbl"; -//const string datapath="/usr/local/mariadb/columnstore/bin/lineitem.tbl"; +// const string datapath="/usr/local/mariadb/columnstore/bin/lineitem.tbl"; int numConsumers = 1; int numRuns = 1; int printInterval = numRuns * 100000; JSTimeStamp dlTimes; -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } } struct ThreadParms { - BucketDL* bdl; - TupleBucketDataList* tbdl; - int count; - int threadNumber; + BucketDL* bdl; + TupleBucketDataList* tbdl; + int count; + int threadNumber; }; void* bucketConsumer(void* arg) { - ThreadParms* parms = reinterpret_cast(arg); - // BucketDL *bdl = reinterpret_cast *>(arg); - BucketDL* bdl = parms->bdl; - int numBucketsToConsume = parms->count; - int threadNumber = parms->threadNumber; - int64_t i; - ElementType val; - uint32_t it; - int bucketIndex; + ThreadParms* parms = reinterpret_cast(arg); + // BucketDL *bdl = reinterpret_cast *>(arg); + BucketDL* bdl = parms->bdl; + int numBucketsToConsume = parms->count; + int threadNumber = parms->threadNumber; + int64_t i; + ElementType val; + uint32_t it; + int bucketIndex; - int cnt = 0; + int cnt = 0; - for (i = 0; i < numBucketsToConsume; i++) - { - bucketIndex = (threadNumber * numBucketsToConsume) + i; - it = bdl->getIterator(bucketIndex); + for (i = 0; i < numBucketsToConsume; i++) + { + bucketIndex = (threadNumber * numBucketsToConsume) + i; + it = bdl->getIterator(bucketIndex); - while (bdl->next(bucketIndex, it, &val)) - cnt++; - } + while (bdl->next(bucketIndex, it, &val)) + cnt++; + } - cout << "consumer " << threadNumber << "consumed " << cnt << endl; - return NULL; + cout << "consumer " << threadNumber << "consumed " << cnt << endl; + return NULL; } void* tupleConsumer(void* arg) { - ThreadParms* parms = reinterpret_cast(arg); - // BucketDL *bdl = reinterpret_cast *>(arg); - TupleBucketDataList* tbdl = parms->tbdl; - int numBucketsToConsume = parms->count; - int threadNumber = parms->threadNumber; - int i; - //ElementType val; - TupleType val; - uint32_t it; - int bucketIndex; + ThreadParms* parms = reinterpret_cast(arg); + // BucketDL *bdl = reinterpret_cast *>(arg); + TupleBucketDataList* tbdl = parms->tbdl; + int numBucketsToConsume = parms->count; + int threadNumber = parms->threadNumber; + int i; + // ElementType val; + TupleType val; + uint32_t it; + int bucketIndex; - int cnt = 0; + int cnt = 0; - for (i = 0; i < numBucketsToConsume; i++) - { - bucketIndex = (threadNumber * numBucketsToConsume) + i; - it = tbdl->getIterator(bucketIndex); + for (i = 0; i < numBucketsToConsume; i++) + { + bucketIndex = (threadNumber * numBucketsToConsume) + i; + it = tbdl->getIterator(bucketIndex); - while (tbdl->next(bucketIndex, it, &val)) - cnt++; - } + while (tbdl->next(bucketIndex, it, &val)) + cnt++; + } - cout << "consumer " << threadNumber << "consumed " << cnt << endl; - return NULL; + cout << "consumer " << threadNumber << "consumed " << cnt << endl; + return NULL; } /** @brief class TupleHasher -* -*/ + * + */ class TupleHasher1 { -private: - Hasher fHasher; - uint32_t fHashLen; + private: + Hasher fHasher; + uint32_t fHashLen; -public: - TupleHasher1(uint32_t len) : fHashLen(len) {} - uint32_t operator()(const char* v) const - { - return fHasher(v, fHashLen); - } - /* - uint32_t operator()(const uint64_t v) const - { - return v; - }*/ + public: + TupleHasher1(uint32_t len) : fHashLen(len) + { + } + uint32_t operator()(const char* v) const + { + return fHasher(v, fHashLen); + } + /* + uint32_t operator()(const uint64_t v) const + { + return v; + }*/ }; /** @brief class TupleComparator @@ -177,32 +178,33 @@ public: */ class TupleComparator1 { -private: - uint32_t fCmpLen; + private: + uint32_t fCmpLen; -public: - TupleComparator1(uint32_t len) : fCmpLen(len) {} - bool operator()(const char* v1, const char* v2) const - { - return (memcmp(v1, v2, fCmpLen) == 0); - } - /* - bool operator()(const uint64_t v1, const uint64_t v2) const - { - return (v1 == v2); - }*/ + public: + TupleComparator1(uint32_t len) : fCmpLen(len) + { + } + bool operator()(const char* v1, const char* v2) const + { + return (memcmp(v1, v2, fCmpLen) == 0); + } + /* + bool operator()(const uint64_t v1, const uint64_t v2) const + { + return (v1 == v2); + }*/ }; - void* aggregator(void* arg) { - uint32_t fHashLen = 4; - TupleType tt; - tt.second = new char[fHashLen]; - ThreadParms* parms = reinterpret_cast(arg); - int threadNumber = parms->threadNumber; - struct timespec ts1, ts2, diff; - uint32_t size = 585938; + uint32_t fHashLen = 4; + TupleType tt; + tt.second = new char[fHashLen]; + ThreadParms* parms = reinterpret_cast(arg); + int threadNumber = parms->threadNumber; + struct timespec ts1, ts2, diff; + uint32_t size = 585938; #if 0 //typedef std::vector RIDVec; typedef std::pair Results; @@ -341,834 +343,794 @@ void* aggregator(void* arg) #endif #if 1 - //typedef std::vector RIDVec; - //typedef std::pair Results; - typedef std::tr1::unordered_map TupleHashMap1; - typedef std::tr1::unordered_map::iterator TupleHMIter1; - typedef boost::shared_ptr SHMP1; - vector vt; - vt.reserve(size / 10 * 4); + // typedef std::vector RIDVec; + // typedef std::pair Results; + typedef std::tr1::unordered_map TupleHashMap1; + typedef std::tr1::unordered_map::iterator TupleHMIter1; + typedef boost::shared_ptr SHMP1; + vector vt; + vt.reserve(size / 10 * 4); - SHMP1 shmp; - TupleHasher1 tupleHash(fHashLen); - TupleComparator1 tupleComp(fHashLen); + SHMP1 shmp; + TupleHasher1 tupleHash(fHashLen); + TupleComparator1 tupleComp(fHashLen); - shmp.reset(new TupleHashMap1(1, tupleHash, tupleComp)); - TupleHMIter1 iter; - int64_t rr; + shmp.reset(new TupleHashMap1(1, tupleHash, tupleComp)); + TupleHMIter1 iter; + int64_t rr; - uint32_t val = 0; + uint32_t val = 0; - for (uint32_t k = 0; k < NUM_BUCKETS / 4; k++) + for (uint32_t k = 0; k < NUM_BUCKETS / 4; k++) + { + clock_gettime(CLOCK_REALTIME, &ts1); + + for (uint32_t j = 0; j < 4; j++) { - clock_gettime(CLOCK_REALTIME, &ts1); + for (uint32_t i = 0; i < size / 10; i++) + { + memcpy(tt.second, &i, 4); + vt.push_back(tt); - for (uint32_t j = 0; j < 4; j++) + iter = shmp->find(tt.second); + + if (iter == shmp->end()) { - for (uint32_t i = 0; i < size / 10; i++) - { - memcpy(tt.second, &i, 4); - vt.push_back(tt); - - iter = shmp->find(tt.second); - - if (iter == shmp->end()) - { - //rr.second.clear(); - //getAggResult(tt, rr.first); - //rr.second.push_back(tt.first); - char* hashStr = new char[fHashLen]; - memcpy(hashStr, tt.second, fHashLen); - shmp->insert(std::pair (hashStr, i)); - } - else - { - //updateAggResult(tt, hashIt->second.first); - //iter->second.second.push_back(tt.first); - iter->second += i; - } - } + // rr.second.clear(); + // getAggResult(tt, rr.first); + // rr.second.push_back(tt.first); + char* hashStr = new char[fHashLen]; + memcpy(hashStr, tt.second, fHashLen); + shmp->insert(std::pair(hashStr, i)); } - - cout << "thread " << threadNumber << " hashmap " << k << " size=" << shmp->size() << endl; - - // loop through vector to find match in map for rid - vector::iterator it; - int val = 0; - - for (it = vt.begin(); it != vt.end(); it++) + else { - iter = shmp->find(it->second); - - if (iter != shmp->end()) - val++; + // updateAggResult(tt, hashIt->second.first); + // iter->second.second.push_back(tt.first); + iter->second += i; } - - cout << "val = " << val << endl; - - // clean up hashmap memory - for (iter = shmp->begin(); iter != shmp->end(); iter++) - delete [] iter->first; - - shmp->clear(); - vt.clear(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "thread " << threadNumber << "do aggregation took " << diff.tv_sec << "s " << diff.tv_nsec << " ns" << endl; + } } -#endif + cout << "thread " << threadNumber << " hashmap " << k << " size=" << shmp->size() << endl; + // loop through vector to find match in map for rid + vector::iterator it; + int val = 0; + + for (it = vt.begin(); it != vt.end(); it++) + { + iter = shmp->find(it->second); + + if (iter != shmp->end()) + val++; + } + + cout << "val = " << val << endl; + + // clean up hashmap memory + for (iter = shmp->begin(); iter != shmp->end(); iter++) + delete[] iter->first; + + shmp->clear(); + vt.clear(); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "thread " << threadNumber << "do aggregation took " << diff.tv_sec << "s " << diff.tv_nsec + << " ns" << endl; + } + +#endif } void* TBDL_producer(void* arg) { - ThreadParms* parms = reinterpret_cast(arg); - TupleBucketDataList* tbdl = parms->tbdl; - TupleType t; - vector vt; + ThreadParms* parms = reinterpret_cast(arg); + TupleBucketDataList* tbdl = parms->tbdl; + TupleType t; + vector vt; - for (uint32_t i = 0; i < 150000000; i++) + for (uint32_t i = 0; i < 150000000; i++) + { + t.first = i; + t.second = new char[4]; + memcpy(t.second, &i, 4); + vt.push_back(t); + + if (vt.size() == 8192) { - t.first = i; - t.second = new char[4]; - memcpy(t.second, &i, 4); - vt.push_back(t); - - if (vt.size() == 8192) - { - tbdl->insert(vt); - vt.clear(); - } + tbdl->insert(vt); + vt.clear(); } + } - if (vt.size() > 0) - tbdl->insert(vt); + if (vt.size() > 0) + tbdl->insert(vt); } void* BDL_producer(void* arg) { - ThreadParms* parms = reinterpret_cast(arg); - BucketDL* bdl = parms->bdl; - TupleType t; - ElementType e; - vector vt; + ThreadParms* parms = reinterpret_cast(arg); + BucketDL* bdl = parms->bdl; + TupleType t; + ElementType e; + vector vt; - for (uint32_t i = 0; i < 150000000; i++) + for (uint32_t i = 0; i < 150000000; i++) + { + e.first = i; + e.second = i; + // t.second = new char[4]; + // memcpy(t.second, &i, 4); + vt.push_back(e); + + if (vt.size() == 8192) { - e.first = i; - e.second = i; - //t.second = new char[4]; - //memcpy(t.second, &i, 4); - vt.push_back(e); - - if (vt.size() == 8192) - { - bdl->insert(vt); - vt.clear(); - } + bdl->insert(vt); + vt.clear(); } + } - if (vt.size() > 0) - bdl->insert(vt); + if (vt.size() > 0) + bdl->insert(vt); } class AggDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(AggDriver); + // CPPUNIT_TEST(aggFilter_group2); + // CPPUNIT_TEST(tuplewsdl); + // CPPUNIT_TEST(bucketdl); + // CPPUNIT_TEST(aggFilter_group1); + // CPPUNIT_TEST(hashjoin); + // CPPUNIT_TEST(hashmap_tuple); + // CPPUNIT_TEST(hashmap_tuple_multi); + // CPPUNIT_TEST(hashmap_ET); + // CPPUNIT_TEST(tuplewsdl_multi); + CPPUNIT_TEST(bdl_multi); + CPPUNIT_TEST_SUITE_END(); + typedef boost::shared_ptr SSFP; - CPPUNIT_TEST_SUITE(AggDriver); -//CPPUNIT_TEST(aggFilter_group2); -//CPPUNIT_TEST(tuplewsdl); -//CPPUNIT_TEST(bucketdl); -//CPPUNIT_TEST(aggFilter_group1); -//CPPUNIT_TEST(hashjoin); -//CPPUNIT_TEST(hashmap_tuple); -//CPPUNIT_TEST(hashmap_tuple_multi); -//CPPUNIT_TEST(hashmap_ET); -//CPPUNIT_TEST(tuplewsdl_multi); - CPPUNIT_TEST(bdl_multi); - CPPUNIT_TEST_SUITE_END(); - typedef boost::shared_ptr SSFP; + private: + ResourceManager fRm; -private: - ResourceManager fRm; -public: - void setUp1(JobStepAssociation& in, JobStepAssociation& out) + public: + void setUp1(JobStepAssociation& in, JobStepAssociation& out) + { + // sleep(20); + // input - TupleBucket + AnyDataListSPtr adl1(new AnyDataList()); + TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, numConsumers, MAX_SIZE, fRm); + tbdl->setMultipleProducers(0); + // tbdl->setElementMode(1); + + adl1->tupleBucketDL(tbdl); + in.outAdd(adl1); + + // output - ElementType Bucket + AnyDataListSPtr adl2(new AnyDataList()); + BucketDataList* bdl = new BucketDataList(NUM_BUCKETS, numConsumers, MAX_ELEMENTS, fRm); + bdl->setElementMode(1); + adl2->bucketDL(bdl); + out.outAdd(adl2); + + // prepare the input datalist + // ifstream ifs(datapath.c_str(), ios::in); + uint64_t l_orderkey; + uint64_t l_quantity; + char data[1000]; + // char row[50]; + char* tok; + uint64_t count = 0; + uint32_t rid = 0; + TupleType* tt = new TupleType(); + tbdl->hashLen(sizeof(l_orderkey)); + uint32_t size = sizeof(l_orderkey) + sizeof(l_quantity); + // tbdl->elementLen(size + sizeof(rid)); + tbdl->elementLen(sizeof(rid), size); + // char* second = new char[size]; + vector v; + + timer1.start("input"); + ifstream ifs(datapath.c_str(), ios::in); + + while (!ifs.eof()) { - //sleep(20); - // input - TupleBucket - AnyDataListSPtr adl1(new AnyDataList()); - TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, numConsumers, MAX_SIZE, fRm); - tbdl->setMultipleProducers(0); - //tbdl->setElementMode(1); + ifs.getline(data, 1000); + tok = strtok(data, "|"); + count = 0; - adl1->tupleBucketDL(tbdl); - in.outAdd(adl1); + while (tok != NULL) + { + if (count == 0) + l_orderkey = atol(tok); - // output - ElementType Bucket - AnyDataListSPtr adl2(new AnyDataList()); - BucketDataList* bdl = new BucketDataList(NUM_BUCKETS, numConsumers, MAX_ELEMENTS, fRm); - bdl->setElementMode(1); - adl2->bucketDL(bdl); - out.outAdd(adl2); + if (count == 4) + l_quantity = atof(tok) * 100; - // prepare the input datalist - //ifstream ifs(datapath.c_str(), ios::in); - uint64_t l_orderkey; - uint64_t l_quantity; - char data[1000]; - //char row[50]; - char* tok; - uint64_t count = 0; - uint32_t rid = 0; - TupleType* tt = new TupleType(); - tbdl->hashLen(sizeof(l_orderkey)); - uint32_t size = sizeof(l_orderkey) + sizeof(l_quantity); - //tbdl->elementLen(size + sizeof(rid)); - tbdl->elementLen(sizeof(rid), size); - //char* second = new char[size]; - vector v; + count++; + tok = strtok(NULL, "|"); + } - timer1.start("input"); - ifstream ifs(datapath.c_str(), ios::in); + for (int i = 0; i < numRuns; i++) + { + l_orderkey += 6000000 * i; + tt->first = rid; + tt->second = new char[size]; + memcpy(tt->second, &l_orderkey, sizeof(l_orderkey)); + memcpy(tt->second + sizeof(l_orderkey), &l_quantity, sizeof(l_quantity)); + v.push_back(*tt); - while (!ifs.eof()) + if (v.size() == 2048) { - ifs.getline(data, 1000); - tok = strtok(data, "|"); - count = 0; - - while (tok != NULL) - { - if (count == 0) - l_orderkey = atol(tok); - - if (count == 4) - l_quantity = atof(tok) * 100; - - count++; - tok = strtok(NULL, "|"); - } - - for (int i = 0; i < numRuns; i++) - { - l_orderkey += 6000000 * i; - tt->first = rid; - tt->second = new char[size]; - memcpy(tt->second, &l_orderkey, sizeof(l_orderkey)); - memcpy(tt->second + sizeof(l_orderkey), &l_quantity, sizeof(l_quantity)); - v.push_back(*tt); - - if (v.size() == 2048) - { - tbdl->insert(v); - v.clear(); - } - - rid++; - } - - if (rid % printInterval == 0) - cout << rid << " " << l_orderkey << " " << l_quantity << endl; + tbdl->insert(v); + v.clear(); } - ifs.close(); + rid++; + } - if (v.size() > 0) - { - tbdl->insert(v); - //for (uint32_t i = 0; i < v.size(); i++) - // delete [] (v[i].second); - v.clear(); - } - - tbdl->endOfInput(); - timer1.stop("input"); - cout << "input size=" << tbdl->totalSize() << endl; - //ifs.close(); - delete tt; + if (rid % printInterval == 0) + cout << rid << " " << l_orderkey << " " << l_quantity << endl; } - int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull ) + ifs.close(); + + if (v.size() > 0) { - //if (str.size() == 0 || isNull ) return valueNullNum(ct); - - int64_t v = 0; - - boost::any anyVal = DataConvert::convertColumnData(ct, str, false); - - switch (ct.colDataType) - { - case CalpontSystemCatalog::BIT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TINYINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::SMALLINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::BIGINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::FLOAT: - { - float i = boost::any_cast(anyVal); - v = (Int64)i; - } - break; - - case CalpontSystemCatalog::DOUBLE: - { - double i = boost::any_cast(anyVal); - v = (Int64)i; - } - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::CLOB: - { - //v = boost::any_cast(anyVal); - string i = boost::any_cast(anyVal); - v = *((Int64*) i.c_str()); - } - - break; - - case CalpontSystemCatalog::DATE: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DATETIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DECIMAL: - v = boost::any_cast(anyVal); - break; - - default: - break; - } - - return v; + tbdl->insert(v); + // for (uint32_t i = 0; i < v.size(); i++) + // delete [] (v[i].second); + v.clear(); } - void tuplewsdl_multi() + tbdl->endOfInput(); + timer1.stop("input"); + cout << "input size=" << tbdl->totalSize() << endl; + // ifs.close(); + delete tt; + } + + int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull) + { + // if (str.size() == 0 || isNull ) return valueNullNum(ct); + + int64_t v = 0; + + boost::any anyVal = DataConvert::convertColumnData(ct, str, false); + + switch (ct.colDataType) { - TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, 1, MAX_SIZE, fRm); - tbdl->setMultipleProducers(true); - tbdl->setElementMode(1); // RID_VALUE - tbdl->hashLen(4); - tbdl->elementLen(4, 12); - pthread_t producer[4]; - ThreadParms producerThreadParms[4]; + case CalpontSystemCatalog::BIT: v = boost::any_cast(anyVal); break; - timer1.start("insert-tbdl"); + case CalpontSystemCatalog::TINYINT: v = boost::any_cast(anyVal); break; - for (uint32_t i = 0; i < 4; i++) - { - producerThreadParms[i].tbdl = tbdl; - producerThreadParms[i].threadNumber = i; - //producerThreadParms[i].count = ::count; - pthread_create(&producer[i], NULL, - TBDL_producer, &producerThreadParms[i]); - } + case CalpontSystemCatalog::SMALLINT: v = boost::any_cast(anyVal); break; - for (uint32_t i = 0; i < 4; i++) - pthread_join(producer[i], NULL); + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: v = boost::any_cast(anyVal); break; - tbdl->endOfInput(); - timer1.stop("insert-tbdl"); - cout << "tbdl finish insert " << tbdl->totalSize() << endl; - //timer1.finish(); + case CalpontSystemCatalog::BIGINT: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::FLOAT: + { + float i = boost::any_cast(anyVal); + v = (Int64)i; + } + break; + + case CalpontSystemCatalog::DOUBLE: + { + double i = boost::any_cast(anyVal); + v = (Int64)i; + } + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::CLOB: + { + // v = boost::any_cast(anyVal); + string i = boost::any_cast(anyVal); + v = *((Int64*)i.c_str()); + } + + break; + + case CalpontSystemCatalog::DATE: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DATETIME: v = boost::any_cast(anyVal); break; + + case CalpontSystemCatalog::DECIMAL: v = boost::any_cast(anyVal); break; + + default: break; } - void bdl_multi() + return v; + } + + void tuplewsdl_multi() + { + TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, 1, MAX_SIZE, fRm); + tbdl->setMultipleProducers(true); + tbdl->setElementMode(1); // RID_VALUE + tbdl->hashLen(4); + tbdl->elementLen(4, 12); + pthread_t producer[4]; + ThreadParms producerThreadParms[4]; + + timer1.start("insert-tbdl"); + + for (uint32_t i = 0; i < 4; i++) { - BucketDL* bdl = new BucketDL(128, 1, MAX_ELEMENTS, fRm); - bdl->setMultipleProducers(true); - bdl->setElementMode(1); // RID_VALUE - bdl->setDiskElemSize ( 4, 4 ); - pthread_t producer[4]; - ThreadParms producerThreadParms[4]; - - timer1.start("insert-bdl"); - - for (uint32_t i = 0; i < 4; i++) - { - producerThreadParms[i].bdl = bdl; - producerThreadParms[i].threadNumber = i; - //producerThreadParms[i].count = ::count; - pthread_create(&producer[i], NULL, - BDL_producer, &producerThreadParms[i]); - } - - for (uint32_t i = 0; i < 4; i++) - pthread_join(producer[i], NULL); - - bdl->endOfInput(); - timer1.stop("insert-bdl"); - cout << "bdl finish insert " << bdl->totalSize() << endl; - timer1.finish(); + producerThreadParms[i].tbdl = tbdl; + producerThreadParms[i].threadNumber = i; + // producerThreadParms[i].count = ::count; + pthread_create(&producer[i], NULL, TBDL_producer, &producerThreadParms[i]); } - void tuplewsdl() + for (uint32_t i = 0; i < 4; i++) + pthread_join(producer[i], NULL); + + tbdl->endOfInput(); + timer1.stop("insert-tbdl"); + cout << "tbdl finish insert " << tbdl->totalSize() << endl; + // timer1.finish(); + } + + void bdl_multi() + { + BucketDL* bdl = new BucketDL(128, 1, MAX_ELEMENTS, fRm); + bdl->setMultipleProducers(true); + bdl->setElementMode(1); // RID_VALUE + bdl->setDiskElemSize(4, 4); + pthread_t producer[4]; + ThreadParms producerThreadParms[4]; + + timer1.start("insert-bdl"); + + for (uint32_t i = 0; i < 4; i++) { - TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, numConsumers, MAX_SIZE, fRm); - // prepare the input datalist - //ifstream ifs(datapath.c_str(), ios::in); - uint64_t l_orderkey, l_quantity; - char data[1000]; - //char row[50]; - char* tok; - uint64_t count = 0, rid = 0; - TupleType* tt = new TupleType(); - tbdl->hashLen(sizeof(l_orderkey)); - uint32_t size = sizeof(l_orderkey) + sizeof(l_quantity); - //tbdl->elementLen(size + sizeof(rid)); - tbdl->elementLen(sizeof(rid), size); - //char* second = new char[size]; - vector v; - - timer1.start("tuple-input"); - int i; - - for (i = 0; i < numRuns; i++) - { - ifstream ifs(datapath.c_str(), ios::in); - - while (!ifs.eof()) - { - ifs.getline(data, 1000); - tok = strtok(data, "|"); - count = 0; - - while (tok != NULL) - { - if (count == 0) - l_orderkey = 6000000 * i + atol(tok); - - if (count == 4) - l_quantity = atof(tok) * 100; - - count++; - tok = strtok(NULL, "|"); - } - - tt->first = rid; - tt->second = new char[size]; - memcpy(tt->second, &l_orderkey, sizeof(l_orderkey)); - memcpy(tt->second + sizeof(l_orderkey), &l_quantity, sizeof(l_quantity)); - v.push_back(*tt); - - if (v.size() == 2048) - { - tbdl->insert(v); - - for (uint32_t i = 0; i < v.size(); i++) - v[i].deleter(); - - v.clear(); - } - - rid++; - - if (rid % printInterval == 0) - cout << rid << " " << l_orderkey << " " << l_quantity << endl; - } - - ifs.close(); - } - - if (v.size() > 0) - { - tbdl->insert(v); - - for (uint32_t i = 0; i < v.size(); i++) - delete [] (v[i].second); - - v.clear(); - } - - tbdl->endOfInput(); - timer1.stop("tuple-input"); - cout << "input size=" << tbdl->totalSize() << endl; - //ifs.close(); - delete tt; - - ThreadParms thparms; - thparms.count = NUM_BUCKETS / NUM_THREADS; - thparms.tbdl = tbdl; - pthread_t consumer[NUM_THREADS]; - - ThreadParms thParms[NUM_THREADS]; - timer1.start("tuple-consumer"); - - for (i = 0; i < NUM_THREADS; i++) - { - thparms.threadNumber = i; - thParms[i] = thparms; - pthread_create(&consumer[i], NULL, tupleConsumer, &thParms[i]); - } - - for (i = 0; i < NUM_THREADS; i++) - pthread_join(consumer[i], NULL); - - timer1.stop("tuple-consumer"); - - timer1.finish(); - delete tbdl; + producerThreadParms[i].bdl = bdl; + producerThreadParms[i].threadNumber = i; + // producerThreadParms[i].count = ::count; + pthread_create(&producer[i], NULL, BDL_producer, &producerThreadParms[i]); } - void aggFilter_group1() + for (uint32_t i = 0; i < 4; i++) + pthread_join(producer[i], NULL); + + bdl->endOfInput(); + timer1.stop("insert-bdl"); + cout << "bdl finish insert " << bdl->totalSize() << endl; + timer1.finish(); + } + + void tuplewsdl() + { + TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, numConsumers, MAX_SIZE, fRm); + // prepare the input datalist + // ifstream ifs(datapath.c_str(), ios::in); + uint64_t l_orderkey, l_quantity; + char data[1000]; + // char row[50]; + char* tok; + uint64_t count = 0, rid = 0; + TupleType* tt = new TupleType(); + tbdl->hashLen(sizeof(l_orderkey)); + uint32_t size = sizeof(l_orderkey) + sizeof(l_quantity); + // tbdl->elementLen(size + sizeof(rid)); + tbdl->elementLen(sizeof(rid), size); + // char* second = new char[size]; + vector v; + + timer1.start("tuple-input"); + int i; + + for (i = 0; i < numRuns; i++) { - /* - int tm; - for (int m = 0; m < 1024; m++) - { - char *a = new char[5*1024*1024]; - memset(a,0,5*1024*1024); - memcpy(&tm, a, 4); - } - */ - //for (int i = 0; i < 10; i++) { - JobStepAssociation in, out; - setUp1(in, out); - CalpontSystemCatalog::TableName tn("test", "lineitem"); - string s_l_orderkey = "test.lineitem.l_orderkey"; - string s_l_quantity = "test.lineitem.l_quantity"; - uint32_t sessionid = 1; - uint32_t txnId = 1; - uint32_t verId = 1; - uint16_t stepID = 0; - uint32_t statementID = 1; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionid); + ifstream ifs(datapath.c_str(), ios::in); - SRCP srcp; - SimpleColumn* l_orderkey = new SimpleColumn(s_l_orderkey, sessionid); - SimpleColumn* l_quantity = new SimpleColumn(s_l_quantity, sessionid); - CalpontSystemCatalog::ROPair ropair = csc->tableRID(tn); - CalpontSystemCatalog::OID tableoid = ropair.objnum; + while (!ifs.eof()) + { + ifs.getline(data, 1000); + tok = strtok(data, "|"); + count = 0; - // sum(l_quantity) group by l_ordereky - srcp.reset(l_quantity); - AggregateColumn* ac = new AggregateColumn("sum", srcp.get()->clone(), sessionid); - - ac->addProjectCol(srcp); - srcp.reset(l_orderkey); - ac->addGroupByCol(srcp); - - // constant column 318 - ConstantColumn* cc = new ConstantColumn(318, ConstantColumn::NUM); - // > - Operator* op = new Operator(">"); - SOP sop(op); - // sum(l_quantity) > 318 - SSFP ssfp; - SimpleFilter* sf = new SimpleFilter(sop, ac, cc); - ssfp.reset(sf); - SJSTEP step; - - AggregateFilterStep* afs = - new AggregateFilterStep(in, - out, - ac->functionName(), - ac->groupByColList(), - ac->projectColList(), - ac->functionParms(), - tableoid, - sessionid, - txnId, - verId, - stepID, - statementID, fRm); - - // output column l_orderkey; - afs->outputCol(dynamic_cast(srcp.get())->oid()); - step.reset(afs); - // one column case, nomalize filter value - CalpontSystemCatalog::ColType ct; - int64_t intVal; - string strVal; - - //int8_t cop = op2num(sop); - if (typeid((*ac->functionParms().get())) == typeid(SimpleColumn)) + while (tok != NULL) { - SimpleColumn* sc = reinterpret_cast(ac->functionParms().get()); - ct = csc->colType(sc->oid()); - intVal = convertValueNum(cc->constval(), ct, false); - afs->addFilter(COMPARE_GT, intVal); - } - else - { - if (cc->type() == ConstantColumn::NUM) - { - intVal = atol(cc->constval().c_str()); - afs->addFilter(COMPARE_GT, intVal, false); - } - else if (cc->type() == ConstantColumn::LITERAL) - afs->addFilter(COMPARE_GT, cc->constval(), false); + if (count == 0) + l_orderkey = 6000000 * i + atol(tok); + + if (count == 4) + l_quantity = atof(tok) * 100; + + count++; + tok = strtok(NULL, "|"); } - timer1.start("agg"); - afs->run(); - afs->join(); - timer1.stop("agg"); - timer1.finish(); - //} + tt->first = rid; + tt->second = new char[size]; + memcpy(tt->second, &l_orderkey, sizeof(l_orderkey)); + memcpy(tt->second + sizeof(l_orderkey), &l_quantity, sizeof(l_quantity)); + v.push_back(*tt); + + if (v.size() == 2048) + { + tbdl->insert(v); + + for (uint32_t i = 0; i < v.size(); i++) + v[i].deleter(); + + v.clear(); + } + + rid++; + + if (rid % printInterval == 0) + cout << rid << " " << l_orderkey << " " << l_quantity << endl; + } + + ifs.close(); } - void bucketdl() + if (v.size() > 0) { - BucketDL* bdl = new BucketDL(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); - bdl->setElementMode(1); - ElementType* et = new ElementType(); + tbdl->insert(v); - uint64_t l_orderkey, l_quantity; - char data[1000]; - char* tok; - uint64_t count = 0, rid = 0; - TupleType* tt = new TupleType(); + for (uint32_t i = 0; i < v.size(); i++) + delete[](v[i].second); - vector v; - timer1.start("bucket-input"); - int i; - - for (i = 0; i < numRuns; i++) - { - ifstream ifs(datapath.c_str(), ios::in); - - while (!ifs.eof()) - { - ifs.getline(data, 1000); - tok = strtok(data, "|"); - count = 0; - - while (tok != NULL) - { - if (count == 0) - l_orderkey = 6000000 * i + atol(tok); - - if (count == 4) - l_quantity = atof(tok) * 100; - - count++; - tok = strtok(NULL, "|"); - } - - et->first = rid; - et->second = rid; - //tt->second = new char[16]; - //memcpy(tt->second, &l_orderkey, sizeof(l_orderkey)); - //memcpy(tt->second+sizeof(l_orderkey), &l_quantity, sizeof(l_quantity)); - v.push_back(*et); - - if (v.size() == 2048) - { - bdl->insert(v); - v.clear(); - } - - rid++; - - //delete [] tt->second; - if (rid % printInterval == 0) - cout << rid << " " << l_orderkey << " " << l_quantity << endl; - } - - ifs.close(); - } - - if (v.size() > 0) - { - bdl->insert(v); - v.clear(); - } - - bdl->endOfInput(); - timer1.stop("bucket-input"); - cout << "input size=" << bdl->totalSize() << endl; - delete tt; - delete et; - - ThreadParms thparms; - thparms.count = NUM_BUCKETS / NUM_THREADS; - thparms.bdl = bdl; - pthread_t consumer[NUM_THREADS]; - - ThreadParms thParms[NUM_THREADS]; - timer1.start("bucket-consumer"); - - for (i = 0; i < NUM_THREADS; i++) - { - thparms.threadNumber = i; - thParms[i] = thparms; - pthread_create(&consumer[i], NULL, bucketConsumer, &thParms[i]); - } - - for (i = 0; i < NUM_THREADS; i++) - pthread_join(consumer[i], NULL); - - timer1.stop("bucket-consumer"); - - timer1.finish(); - delete bdl; + v.clear(); } - void aggFilter_group2() + tbdl->endOfInput(); + timer1.stop("tuple-input"); + cout << "input size=" << tbdl->totalSize() << endl; + // ifs.close(); + delete tt; + + ThreadParms thparms; + thparms.count = NUM_BUCKETS / NUM_THREADS; + thparms.tbdl = tbdl; + pthread_t consumer[NUM_THREADS]; + + ThreadParms thParms[NUM_THREADS]; + timer1.start("tuple-consumer"); + + for (i = 0; i < NUM_THREADS; i++) { - CalpontSystemCatalog::TableName tn("test", "lineitem"); - string s_l_orderkey = "test.lineitem.l_orderkey"; - string s_l_quantity = "test.lineitem.l_quantity"; - string s_l_partkey = "test.lineitem.l_partkey"; - uint32_t sessionid = 1; - uint32_t txnId = 1; - uint32_t verId = 1; - uint16_t stepID = 0; - uint32_t statementID = 1; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionid); - - SRCP srcp; - SimpleColumn* l_orderkey = new SimpleColumn(s_l_orderkey, sessionid); - SimpleColumn* l_quantity = new SimpleColumn(s_l_quantity, sessionid); - SimpleColumn* l_partkey = new SimpleColumn(s_l_partkey, sessionid); - CalpontSystemCatalog::ROPair ropair = csc->tableRID(tn); - CalpontSystemCatalog::OID tableoid = ropair.objnum; - - // sum(l_quantity) group by l_ordereky - AggregateColumn* ac = new AggregateColumn("sum", l_quantity, sessionid); - srcp.reset(l_quantity); - ac->addProjectCol(srcp); - srcp.reset(l_orderkey); - ac->addGroupByCol(srcp); - srcp.reset(l_partkey); - ac->addGroupByCol(srcp); - - // constant column 318 - ConstantColumn* cc = new ConstantColumn(318, ConstantColumn::NUM); - // > - Operator* op = new Operator(">"); - SOP sop(op); - // sum(l_quantity) > 318 - SimpleFilter* sf = new SimpleFilter(sop, ac, cc); - - // aggregate filter step - /** @brief Constructor */ - JobStepAssociation in; - JobStepAssociation out; - AggregateFilterStep* afs = - new AggregateFilterStep(in, - out, - ac->functionName(), - ac->groupByColList(), - ac->projectColList(), - ac->functionParms(), - tableoid, - sessionid, - txnId, - verId, - stepID, - statementID, fRm); - afs->run(); - afs->join(); + thparms.threadNumber = i; + thParms[i] = thparms; + pthread_create(&consumer[i], NULL, tupleConsumer, &thParms[i]); } - void hashjoin() + for (i = 0; i < NUM_THREADS; i++) + pthread_join(consumer[i], NULL); + + timer1.stop("tuple-consumer"); + + timer1.finish(); + delete tbdl; + } + + void aggFilter_group1() + { + /* + int tm; + for (int m = 0; m < 1024; m++) + { + char *a = new char[5*1024*1024]; + memset(a,0,5*1024*1024); + memcpy(&tm, a, 4); + } + */ + // for (int i = 0; i < 10; i++) { + JobStepAssociation in, out; + setUp1(in, out); + CalpontSystemCatalog::TableName tn("test", "lineitem"); + string s_l_orderkey = "test.lineitem.l_orderkey"; + string s_l_quantity = "test.lineitem.l_quantity"; + uint32_t sessionid = 1; + uint32_t txnId = 1; + uint32_t verId = 1; + uint16_t stepID = 0; + uint32_t statementID = 1; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionid); + + SRCP srcp; + SimpleColumn* l_orderkey = new SimpleColumn(s_l_orderkey, sessionid); + SimpleColumn* l_quantity = new SimpleColumn(s_l_quantity, sessionid); + CalpontSystemCatalog::ROPair ropair = csc->tableRID(tn); + CalpontSystemCatalog::OID tableoid = ropair.objnum; + + // sum(l_quantity) group by l_ordereky + srcp.reset(l_quantity); + AggregateColumn* ac = new AggregateColumn("sum", srcp.get()->clone(), sessionid); + + ac->addProjectCol(srcp); + srcp.reset(l_orderkey); + ac->addGroupByCol(srcp); + + // constant column 318 + ConstantColumn* cc = new ConstantColumn(318, ConstantColumn::NUM); + // > + Operator* op = new Operator(">"); + SOP sop(op); + // sum(l_quantity) > 318 + SSFP ssfp; + SimpleFilter* sf = new SimpleFilter(sop, ac, cc); + ssfp.reset(sf); + SJSTEP step; + + AggregateFilterStep* afs = new AggregateFilterStep(in, out, ac->functionName(), ac->groupByColList(), + ac->projectColList(), ac->functionParms(), tableoid, + sessionid, txnId, verId, stepID, statementID, fRm); + + // output column l_orderkey; + afs->outputCol(dynamic_cast(srcp.get())->oid()); + step.reset(afs); + // one column case, nomalize filter value + CalpontSystemCatalog::ColType ct; + int64_t intVal; + string strVal; + + // int8_t cop = op2num(sop); + if (typeid((*ac->functionParms().get())) == typeid(SimpleColumn)) { - BucketDL< ElementType > A(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); - BucketDL< ElementType > B(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); - BucketDL< ElementType > C(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); - BucketDL< ElementType > D(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); - A.setDiskElemSize( 4, 4 ); - B.setDiskElemSize( 4, 4 ); - A.setElementMode(1); - A.setHashMode(1); - B.setElementMode(1); - B.setHashMode(1); - int i; - timer1.start("insert"); - - for (i = 0; i < 6000000 * numRuns; i++) - A.insert(ElementType(i, i)); - - A.endOfInput(); - - for (i = 0; i < 1500000 * numRuns; i++) - B.insert(ElementType(i, i + 1000000)); - - B.endOfInput(); - timer1.stop("insert"); - C.setElementMode(1); - D.setElementMode(1); - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - timer1.start("hashjoin"); - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &dlTimes); - hj->performJoin(); - timer1.stop("hashjoin"); - timer1.finish(); - cout << "hash join output size: " << resultA->totalSize() << "/" << resultB->totalSize() << endl; + SimpleColumn* sc = reinterpret_cast(ac->functionParms().get()); + ct = csc->colType(sc->oid()); + intVal = convertValueNum(cc->constval(), ct, false); + afs->addFilter(COMPARE_GT, intVal); + } + else + { + if (cc->type() == ConstantColumn::NUM) + { + intVal = atol(cc->constval().c_str()); + afs->addFilter(COMPARE_GT, intVal, false); + } + else if (cc->type() == ConstantColumn::LITERAL) + afs->addFilter(COMPARE_GT, cc->constval(), false); } - typedef struct Elem + timer1.start("agg"); + afs->run(); + afs->join(); + timer1.stop("agg"); + timer1.finish(); + //} + } + + void bucketdl() + { + BucketDL* bdl = new BucketDL(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); + bdl->setElementMode(1); + ElementType* et = new ElementType(); + + uint64_t l_orderkey, l_quantity; + char data[1000]; + char* tok; + uint64_t count = 0, rid = 0; + TupleType* tt = new TupleType(); + + vector v; + timer1.start("bucket-input"); + int i; + + for (i = 0; i < numRuns; i++) { - char* hashStr; - //int64_t result; - vector rids; - } Elem; + ifstream ifs(datapath.c_str(), ios::in); + while (!ifs.eof()) + { + ifs.getline(data, 1000); + tok = strtok(data, "|"); + count = 0; - void hashmap_tuple() - { - int tm; - - for (int m = 0; m < 1024; m++) + while (tok != NULL) { - char* a = new char[5 * 1024 * 1024]; - memset(a, 0, 5 * 1024 * 1024); - memcpy(&tm, a, 4); + if (count == 0) + l_orderkey = 6000000 * i + atol(tok); + + if (count == 4) + l_quantity = atof(tok) * 100; + + count++; + tok = strtok(NULL, "|"); } - uint32_t fHashLen = 4; - TupleType tt; - tt.second = new char[fHashLen]; - uint64_t size = 585937; - uint64_t i = 0, j = 0; + et->first = rid; + et->second = rid; + // tt->second = new char[16]; + // memcpy(tt->second, &l_orderkey, sizeof(l_orderkey)); + // memcpy(tt->second+sizeof(l_orderkey), &l_quantity, sizeof(l_quantity)); + v.push_back(*et); + + if (v.size() == 2048) + { + bdl->insert(v); + v.clear(); + } + + rid++; + + // delete [] tt->second; + if (rid % printInterval == 0) + cout << rid << " " << l_orderkey << " " << l_quantity << endl; + } + + ifs.close(); + } + + if (v.size() > 0) + { + bdl->insert(v); + v.clear(); + } + + bdl->endOfInput(); + timer1.stop("bucket-input"); + cout << "input size=" << bdl->totalSize() << endl; + delete tt; + delete et; + + ThreadParms thparms; + thparms.count = NUM_BUCKETS / NUM_THREADS; + thparms.bdl = bdl; + pthread_t consumer[NUM_THREADS]; + + ThreadParms thParms[NUM_THREADS]; + timer1.start("bucket-consumer"); + + for (i = 0; i < NUM_THREADS; i++) + { + thparms.threadNumber = i; + thParms[i] = thparms; + pthread_create(&consumer[i], NULL, bucketConsumer, &thParms[i]); + } + + for (i = 0; i < NUM_THREADS; i++) + pthread_join(consumer[i], NULL); + + timer1.stop("bucket-consumer"); + + timer1.finish(); + delete bdl; + } + + void aggFilter_group2() + { + CalpontSystemCatalog::TableName tn("test", "lineitem"); + string s_l_orderkey = "test.lineitem.l_orderkey"; + string s_l_quantity = "test.lineitem.l_quantity"; + string s_l_partkey = "test.lineitem.l_partkey"; + uint32_t sessionid = 1; + uint32_t txnId = 1; + uint32_t verId = 1; + uint16_t stepID = 0; + uint32_t statementID = 1; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionid); + + SRCP srcp; + SimpleColumn* l_orderkey = new SimpleColumn(s_l_orderkey, sessionid); + SimpleColumn* l_quantity = new SimpleColumn(s_l_quantity, sessionid); + SimpleColumn* l_partkey = new SimpleColumn(s_l_partkey, sessionid); + CalpontSystemCatalog::ROPair ropair = csc->tableRID(tn); + CalpontSystemCatalog::OID tableoid = ropair.objnum; + + // sum(l_quantity) group by l_ordereky + AggregateColumn* ac = new AggregateColumn("sum", l_quantity, sessionid); + srcp.reset(l_quantity); + ac->addProjectCol(srcp); + srcp.reset(l_orderkey); + ac->addGroupByCol(srcp); + srcp.reset(l_partkey); + ac->addGroupByCol(srcp); + + // constant column 318 + ConstantColumn* cc = new ConstantColumn(318, ConstantColumn::NUM); + // > + Operator* op = new Operator(">"); + SOP sop(op); + // sum(l_quantity) > 318 + SimpleFilter* sf = new SimpleFilter(sop, ac, cc); + + // aggregate filter step + /** @brief Constructor */ + JobStepAssociation in; + JobStepAssociation out; + AggregateFilterStep* afs = new AggregateFilterStep(in, out, ac->functionName(), ac->groupByColList(), + ac->projectColList(), ac->functionParms(), tableoid, + sessionid, txnId, verId, stepID, statementID, fRm); + afs->run(); + afs->join(); + } + + void hashjoin() + { + BucketDL A(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); + BucketDL B(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); + BucketDL C(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); + BucketDL D(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm); + A.setDiskElemSize(4, 4); + B.setDiskElemSize(4, 4); + A.setElementMode(1); + A.setHashMode(1); + B.setElementMode(1); + B.setHashMode(1); + int i; + timer1.start("insert"); + + for (i = 0; i < 6000000 * numRuns; i++) + A.insert(ElementType(i, i)); + + A.endOfInput(); + + for (i = 0; i < 1500000 * numRuns; i++) + B.insert(ElementType(i, i + 1000000)); + + B.endOfInput(); + timer1.stop("insert"); + C.setElementMode(1); + D.setElementMode(1); + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + timer1.start("hashjoin"); + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &dlTimes); + hj->performJoin(); + timer1.stop("hashjoin"); + timer1.finish(); + cout << "hash join output size: " << resultA->totalSize() << "/" << resultB->totalSize() << endl; + } + + typedef struct Elem + { + char* hashStr; + // int64_t result; + vector rids; + } Elem; + + void hashmap_tuple() + { + int tm; + + for (int m = 0; m < 1024; m++) + { + char* a = new char[5 * 1024 * 1024]; + memset(a, 0, 5 * 1024 * 1024); + memcpy(&tm, a, 4); + } + + uint32_t fHashLen = 4; + TupleType tt; + tt.second = new char[fHashLen]; + uint64_t size = 585937; + uint64_t i = 0, j = 0; #if 1 - pthread_t agg[NUM_THREADS]; - ThreadParms thParms[NUM_THREADS]; - ThreadParms thParm; - struct timespec ts1, ts2, diff; + pthread_t agg[NUM_THREADS]; + ThreadParms thParms[NUM_THREADS]; + ThreadParms thParm; + struct timespec ts1, ts2, diff; - clock_gettime(CLOCK_REALTIME, &ts1); + clock_gettime(CLOCK_REALTIME, &ts1); - for (i = 0; i < NUM_THREADS; i++) - { - thParm.threadNumber = i; - thParms[i] = thParm; - pthread_create(&agg[i], NULL, aggregator, &thParms[i]); - } + for (i = 0; i < NUM_THREADS; i++) + { + thParm.threadNumber = i; + thParms[i] = thParm; + pthread_create(&agg[i], NULL, aggregator, &thParms[i]); + } - for (i = 0; i < NUM_THREADS; i++) - pthread_join(agg[i], NULL); + for (i = 0; i < NUM_THREADS; i++) + pthread_join(agg[i], NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "aggregation took " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "aggregation took " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; #endif #if 0 //typedef std::vector RIDVec; @@ -1295,7 +1257,7 @@ public: shmp->clear(); #endif - } + } #if 0 void hashmap_tuple_multi() @@ -1378,74 +1340,74 @@ public: delete [] iter->first; } #endif - void hashmap_ET() + void hashmap_ET() + { + typedef std::list hashList_t; + typedef std::list::iterator hashListIter_t; + + // typedef std::tr1::unordered_multimap hash_t; + // typedef std::tr1::unordered_multimap::iterator + // hashIter_t; typedef std::tr1::unordered_multimap::value_type hashPair_t; + + typedef std::tr1::unordered_map hash_t; + typedef std::tr1::unordered_map::iterator hashIter_t; + typedef std::tr1::unordered_map::value_type hashPair_t; + + hash_t hashmap; + hashIter_t it; + hashList_t list; + + timer1.start("et_insert"); + ElementType et; + + // while (true) + for (int i = 0; i < 1200000; i++) { - typedef std::list hashList_t; - typedef std::list::iterator hashListIter_t; - - //typedef std::tr1::unordered_multimap hash_t; - //typedef std::tr1::unordered_multimap::iterator hashIter_t; - //typedef std::tr1::unordered_multimap::value_type hashPair_t; - - typedef std::tr1::unordered_map hash_t; - typedef std::tr1::unordered_map::iterator hashIter_t; - typedef std::tr1::unordered_map::value_type hashPair_t; - - hash_t hashmap; - hashIter_t it; - hashList_t list; - - timer1.start("et_insert"); - ElementType et; - - //while (true) - for (int i = 0; i < 1200000; i++) - { - list.push_back(et); - et.second = i; - hashmap.insert(hashPair_t(et.second, list)); - list.clear(); - } - - timer1.stop("et_insert"); - cout << "hashmap size=" << hashmap.size() << endl; - - int val = 0, i, j; - timer1.start("et_find"); - - for (j = 0; j < 4; j++) - for (i = 0; i < 1200000; i++) - { - et.second = i; - it = hashmap.find(i); - - if (it == hashmap.end()) - { - list.clear(); - //getAggResult(tt, rr.first); - list.push_back(et); - hashmap.insert(hashPair_t (et.second, list)); - } - else - { - it->second.push_back(et); - } - } - - timer1.stop("et_find"); - timer1.finish(); - cout << "hashmap size=" << hashmap.size() << endl; + list.push_back(et); + et.second = i; + hashmap.insert(hashPair_t(et.second, list)); + list.clear(); } + + timer1.stop("et_insert"); + cout << "hashmap size=" << hashmap.size() << endl; + + int val = 0, i, j; + timer1.start("et_find"); + + for (j = 0; j < 4; j++) + for (i = 0; i < 1200000; i++) + { + et.second = i; + it = hashmap.find(i); + + if (it == hashmap.end()) + { + list.clear(); + // getAggResult(tt, rr.first); + list.push_back(et); + hashmap.insert(hashPair_t(et.second, list)); + } + else + { + it->second.push_back(et); + } + } + + timer1.stop("et_find"); + timer1.finish(); + cout << "hashmap size=" << hashmap.size() << endl; + } }; CPPUNIT_TEST_SUITE_REGISTRATION(AggDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/joblist/tdriver-bdl.cpp b/dbcon/joblist/tdriver-bdl.cpp index dfa2f8e2a..87c48f3b4 100644 --- a/dbcon/joblist/tdriver-bdl.cpp +++ b/dbcon/joblist/tdriver-bdl.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: tdriver-bdl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: tdriver-bdl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -40,128 +40,122 @@ using namespace std; using namespace joblist; Stopwatch timer; -//dmc-uint64_t count = 20000000/*1000000*/; +// dmc-uint64_t count = 20000000/*1000000*/; uint64_t count = 8000000; const int NUM_PRODUCERS = 8; const int NUM_CONSUMERS = 4; -const int NUM_BUCKETS = 128; -const int MAX_ELEMENTS = 8192; -uint64_t readCounts[NUM_CONSUMERS]; +const int NUM_BUCKETS = 128; +const int MAX_ELEMENTS = 8192; +uint64_t readCounts[NUM_CONSUMERS]; struct ThreadParms { - void* bdl; - unsigned int threadNumber; - uint64_t count; // for producer this is the number of elements - // each producer is to write - // for consumer this is the number of buckets - // each consumer is to read + void* bdl; + unsigned int threadNumber; + uint64_t count; // for producer this is the number of elements + // each producer is to write + // for consumer this is the number of buckets + // each consumer is to read }; //------------------------------------------------------------------------------ -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } } //------------------------------------------------------------------------------ // thread callbacks and utilities for BDL testing //------------------------------------------------------------------------------ -template +template void* BDL_producer(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - BucketDL* bdl = reinterpret_cast *> - (pThreadParms->bdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + BucketDL* bdl = reinterpret_cast*>(pThreadParms->bdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = tNum; - i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); - i = i + NUM_PRODUCERS) - { - bdl->insert(ElemT(i, i)); - } + for (uint64_t i = tNum; i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); + i = i + NUM_PRODUCERS) + { + bdl->insert(ElemT(i, i)); + } - return NULL; + return NULL; } // // Can't use BDL_producer() to test RID only element type(s) // because we need an ElemT constructor that takes a single argument. // -template +template void* BDL_producer_ridonly(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - BucketDL* bdl = reinterpret_cast *> - (pThreadParms->bdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + BucketDL* bdl = reinterpret_cast*>(pThreadParms->bdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = tNum; - i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); - i = i + NUM_PRODUCERS) - { - bdl->insert(ElemT(i)); - } + for (uint64_t i = tNum; i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); + i = i + NUM_PRODUCERS) + { + bdl->insert(ElemT(i)); + } - return NULL; + return NULL; } -template +template void* BDL_consumer(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - BucketDL* bdl = reinterpret_cast *> - (pThreadParms->bdl); - uint64_t numBucketsToConsume = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + BucketDL* bdl = reinterpret_cast*>(pThreadParms->bdl); + uint64_t numBucketsToConsume = pThreadParms->count; - bool nextRet; - ElemT e; + bool nextRet; + ElemT e; - for (uint64_t i = 0; i < numBucketsToConsume; i++) + for (uint64_t i = 0; i < numBucketsToConsume; i++) + { + uint64_t bucketIndex = (tNum * numBucketsToConsume) + i; + uint64_t id = bdl->getIterator(bucketIndex); + + nextRet = bdl->next(bucketIndex, id, &e); + + while (nextRet) { - uint64_t bucketIndex = (tNum * numBucketsToConsume) + i; - uint64_t id = bdl->getIterator( bucketIndex ); - - nextRet = bdl->next(bucketIndex, id, &e); - - while (nextRet) - { - ::readCounts[tNum]++; - nextRet = bdl->next(bucketIndex, id, &e); - } + ::readCounts[tNum]++; + nextRet = bdl->next(bucketIndex, id, &e); } + } - return NULL; + return NULL; } -template +template void BDL_printFileStats(void* arg) { - BucketDL* bdl = reinterpret_cast *>(arg); - uint64_t nFiles; - uint64_t nBytes; - bdl->totalFileCounts(nFiles, nBytes); - uint32_t size1st = bdl->getDiskElemSize1st(); - uint32_t size2nd = bdl->getDiskElemSize2nd(); - cout << "NumberOfFiles: " << nFiles << endl; - cout << "NumberOfBytes: " << nBytes << endl; - cout << "ElementSize: " << size1st << "/" << size2nd << endl; + BucketDL* bdl = reinterpret_cast*>(arg); + uint64_t nFiles; + uint64_t nBytes; + bdl->totalFileCounts(nFiles, nBytes); + uint32_t size1st = bdl->getDiskElemSize1st(); + uint32_t size2nd = bdl->getDiskElemSize2nd(); + cout << "NumberOfFiles: " << nFiles << endl; + cout << "NumberOfBytes: " << nBytes << endl; + cout << "ElementSize: " << size1st << "/" << size2nd << endl; } //------------------------------------------------------------------------------ @@ -170,344 +164,330 @@ void BDL_printFileStats(void* arg) class DataListDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DataListDriver); + // CPPUNIT_TEST(configure); + // CPPUNIT_TEST(load_save); + CPPUNIT_TEST(BDL_bench_uncompressed); + CPPUNIT_TEST(BDL_bench_compressed_32_32); + CPPUNIT_TEST(BDL_bench_compressed_64_32); + CPPUNIT_TEST(BDL_bench_compressed_32_64); + CPPUNIT_TEST(BDL_bench_ridonly_uncompressed); + CPPUNIT_TEST(BDL_bench_ridonly_compressed_32); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE(DataListDriver); -//CPPUNIT_TEST(configure); -//CPPUNIT_TEST(load_save); - CPPUNIT_TEST(BDL_bench_uncompressed); - CPPUNIT_TEST(BDL_bench_compressed_32_32); - CPPUNIT_TEST(BDL_bench_compressed_64_32); - CPPUNIT_TEST(BDL_bench_compressed_32_64); - CPPUNIT_TEST(BDL_bench_ridonly_uncompressed); - CPPUNIT_TEST(BDL_bench_ridonly_compressed_32); - CPPUNIT_TEST_SUITE_END(); - -private: -public: - //-------------------------------------------------------------------------- - // setup method run prior to each unit test - //-------------------------------------------------------------------------- - void setUp() + private: + public: + //-------------------------------------------------------------------------- + // setup method run prior to each unit test + //-------------------------------------------------------------------------- + void setUp() + { + for (int i = 0; i < NUM_CONSUMERS; i++) { - for (int i = 0; i < NUM_CONSUMERS; i++) - { - ::readCounts[i] = 0; - } + ::readCounts[i] = 0; + } + } + + //-------------------------------------------------------------------------- + // validate results from a unit test + //-------------------------------------------------------------------------- + void validateResults() + { + uint64_t totalElementsRead = 0; + uint64_t totalElementsExpected = ::count * NUM_PRODUCERS; + + for (int i = 0; i < NUM_CONSUMERS; i++) + { + totalElementsRead += ::readCounts[i]; + cout << "consumer " << i << " read " << ::readCounts[i] << " elements " << endl; } - //-------------------------------------------------------------------------- - // validate results from a unit test - //-------------------------------------------------------------------------- - void validateResults() + CPPUNIT_ASSERT(totalElementsRead == totalElementsExpected); + } + + //-------------------------------------------------------------------------- + // BDL benchmark functions + //-------------------------------------------------------------------------- + void BDL_bench(char* testDesc, bool compress, uint32_t size1st, uint32_t size2nd) + { + typedef ElementType Element; + + string produceTag("bdl-produce_"); + string eofInputTag("bdl-endofinput_"); + string consumeTag("bdl-consume_"); + produceTag += testDesc; + eofInputTag += testDesc; + consumeTag += testDesc; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + uint32_t numBuckets = NUM_BUCKETS; + uint32_t maxElements = MAX_ELEMENTS; + BucketDL bdl(numBuckets, 1, maxElements); + bdl.setMultipleProducers(true); + bdl.setElementMode(1); // RID_VALUE + + if (compress) + bdl.setDiskElemSize(size1st, size2nd); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + timer.start(produceTag); + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) { - uint64_t totalElementsRead = 0; - uint64_t totalElementsExpected = ::count * NUM_PRODUCERS; - - for (int i = 0; i < NUM_CONSUMERS; i++) - { - totalElementsRead += ::readCounts[i]; - cout << "consumer " << i << " read " << ::readCounts[i] << - " elements " << endl; - } - - CPPUNIT_ASSERT(totalElementsRead == totalElementsExpected); + producerThreadParms[i].bdl = &bdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::count; + pthread_create(&producer[i], NULL, BDL_producer, &producerThreadParms[i]); } - //-------------------------------------------------------------------------- - // BDL benchmark functions - //-------------------------------------------------------------------------- - void BDL_bench(char* testDesc, bool compress, - uint32_t size1st, uint32_t size2nd) + for (i = 0; i < numOfConsumers; i++) { - typedef ElementType Element; - - string produceTag ("bdl-produce_"); - string eofInputTag("bdl-endofinput_"); - string consumeTag ("bdl-consume_"); - produceTag += testDesc; - eofInputTag += testDesc; - consumeTag += testDesc; - - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - uint32_t numBuckets = NUM_BUCKETS; - uint32_t maxElements = MAX_ELEMENTS; - BucketDL bdl(numBuckets, 1, maxElements); - bdl.setMultipleProducers(true); - bdl.setElementMode(1); // RID_VALUE - - if ( compress ) - bdl.setDiskElemSize ( size1st, size2nd ); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; - - timer.start(produceTag); - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].bdl = &bdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::count; - pthread_create(&producer[i], NULL, - BDL_producer, &producerThreadParms[i]); - } - - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].bdl = &bdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = NUM_BUCKETS / NUM_CONSUMERS; - pthread_create(&consumer[i], NULL, - BDL_consumer, &consumerThreadParms[i]); - } - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - timer.stop(produceTag); - - timer.start(eofInputTag); - bdl.endOfInput(); - timer.stop(eofInputTag); - - timer.start(consumeTag); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timer.stop(consumeTag); - - timer.finish(); - timespec_sub(ts1, ts2, diff); - cout << "compress state: " << (compress ? "on" : "off") << endl; - - if (compress) - cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "BDL_bench_" << testDesc << - ": producer & consumer passed " << - bdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; - - BDL_printFileStats(&bdl); - validateResults(); - cout << endl; + consumerThreadParms[i].bdl = &bdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = NUM_BUCKETS / NUM_CONSUMERS; + pthread_create(&consumer[i], NULL, BDL_consumer, &consumerThreadParms[i]); } - void BDL_bench_uncompressed() + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + timer.stop(produceTag); + + timer.start(eofInputTag); + bdl.endOfInput(); + timer.stop(eofInputTag); + + timer.start(consumeTag); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timer.stop(consumeTag); + + timer.finish(); + timespec_sub(ts1, ts2, diff); + cout << "compress state: " << (compress ? "on" : "off") << endl; + + if (compress) + cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "BDL_bench_" << testDesc << ": producer & consumer passed " << bdl.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + BDL_printFileStats(&bdl); + validateResults(); + cout << endl; + } + + void BDL_bench_uncompressed() + { + BDL_bench("uncompressed", false, 0, 0); + } + + void BDL_bench_compressed_32_32() + { + BDL_bench("compressed_32_32", true, 4, 4); + } + + void BDL_bench_compressed_64_32() + { + BDL_bench("compressed_64_32", true, 8, 4); + } + + void BDL_bench_compressed_32_64() + { + BDL_bench("compressed_32_64", true, 4, 8); + } + + void BDL_bench_ridonly(char* testDesc, bool compress, uint32_t size1st) + { + typedef RIDElementType Element; + uint32_t size2nd = 0; + + string produceTag("bdl-produce_ridonly"); + string eofInputTag("bdl-endofinput_ridonly"); + string consumeTag("bdl-consume_ridonly"); + produceTag += testDesc; + eofInputTag += testDesc; + consumeTag += testDesc; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + uint32_t numBuckets = NUM_BUCKETS; + uint32_t maxElements = MAX_ELEMENTS; + BucketDL bdl(numBuckets, 1, maxElements); + bdl.setMultipleProducers(true); + + if (compress) + bdl.setDiskElemSize(size1st, size2nd); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + timer.start(produceTag); + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) { - BDL_bench( "uncompressed", false, 0, 0 ); + producerThreadParms[i].bdl = &bdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::count; + pthread_create(&producer[i], NULL, BDL_producer_ridonly, &producerThreadParms[i]); } - void BDL_bench_compressed_32_32() + for (i = 0; i < numOfConsumers; i++) { - BDL_bench( "compressed_32_32", true, 4, 4 ); + consumerThreadParms[i].bdl = &bdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = NUM_BUCKETS / NUM_CONSUMERS; + pthread_create(&consumer[i], NULL, BDL_consumer, &consumerThreadParms[i]); } - void BDL_bench_compressed_64_32() - { - BDL_bench( "compressed_64_32", true, 8, 4 ); - } + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); - void BDL_bench_compressed_32_64() - { - BDL_bench( "compressed_32_64", true, 4, 8 ); - } + timer.stop(produceTag); - void BDL_bench_ridonly(char* testDesc, bool compress, - uint32_t size1st) - { - typedef RIDElementType Element; - uint32_t size2nd = 0; + timer.start(eofInputTag); + bdl.endOfInput(); + timer.stop(eofInputTag); - string produceTag ("bdl-produce_ridonly"); - string eofInputTag("bdl-endofinput_ridonly"); - string consumeTag ("bdl-consume_ridonly"); - produceTag += testDesc; - eofInputTag += testDesc; - consumeTag += testDesc; + timer.start(consumeTag); - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - uint32_t numBuckets = NUM_BUCKETS; - uint32_t maxElements = MAX_ELEMENTS; - BucketDL bdl(numBuckets, 1, maxElements); - bdl.setMultipleProducers(true); + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); - if ( compress ) - bdl.setDiskElemSize ( size1st, size2nd ); + clock_gettime(CLOCK_REALTIME, &ts2); + timer.stop(consumeTag); - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; + timer.finish(); + timespec_sub(ts1, ts2, diff); + cout << "compress state: " << (compress ? "on" : "off") << endl; - timer.start(produceTag); - clock_gettime(CLOCK_REALTIME, &ts1); + if (compress) + cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].bdl = &bdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::count; - pthread_create(&producer[i], NULL, - BDL_producer_ridonly, &producerThreadParms[i]); - } + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "BDL_bench_ridonly_" << testDesc << ": producer & consumer passed " << bdl.totalSize() + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].bdl = &bdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = NUM_BUCKETS / NUM_CONSUMERS; - pthread_create(&consumer[i], NULL, - BDL_consumer, &consumerThreadParms[i]); - } + BDL_printFileStats(&bdl); + validateResults(); + cout << endl; + } - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); + void BDL_bench_ridonly_uncompressed() + { + BDL_bench_ridonly("uncompressed", false, 0); + } - timer.stop(produceTag); + void BDL_bench_ridonly_compressed_32() + { + BDL_bench_ridonly("compressed_32", true, 4); + } - timer.start(eofInputTag); - bdl.endOfInput(); - timer.stop(eofInputTag); + //-------------------------------------------------------------------------- + // test the saving and loading of a bdl file + //-------------------------------------------------------------------------- + void load_save() + { + typedef ElementType Element; - timer.start(consumeTag); + vector v; - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); + for (uint32_t i = 0; i < ::count; i++) + v.push_back(Element(i, i)); - clock_gettime(CLOCK_REALTIME, &ts2); - timer.stop(consumeTag); + vector v1; + vector v2; + vector v3; - timer.finish(); - timespec_sub(ts1, ts2, diff); - cout << "compress state: " << (compress ? "on" : "off") << endl; + // save + ofstream f1; + ifstream f; + string filename = "bdl.txt"; + uint64_t cnt = v.size(); + f1.open(filename.c_str(), std::ios::binary); + f1.write((char*)&cnt, sizeof(cnt)); + f1.write((char*)(v.begin().operator->()), sizeof(Element) * cnt); + f.close(); - if (compress) - cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; + // load + v1.push_back(Element(3, 4)); + f.open(filename.c_str(), std::ios::binary); + timer.start("read"); + v1.resize(v1.size() + ::count); + f.read((char*)((v1.begin() + 1).operator->()), cnt * sizeof(Element)); + cout << v1.size() << endl; + timer.stop("read"); + cout << "E1: " << v1[0].first << endl; + f.close(); - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "BDL_bench_ridonly_" << testDesc << - ": producer & consumer passed " << - bdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; + f.open(filename.c_str(), std::ios::binary); + timer.start("assign"); + v2.assign(std::istream_iterator(f), std::istream_iterator()); + cout << v2.size() << endl; + timer.stop("assign"); + f.close(); - BDL_printFileStats(&bdl); - validateResults(); - cout << endl; - } + f.open(filename.c_str(), std::ios::binary); + timer.start("insert"); + v3.insert(v3.end(), std::istream_iterator(f), std::istream_iterator()); + cout << v3.size() << endl; + timer.stop("insert"); + f.close(); + timer.finish(); + } - void BDL_bench_ridonly_uncompressed() - { - BDL_bench_ridonly( "uncompressed", false, 0 ); - } + //-------------------------------------------------------------------------- + // test the reading of bdl configuration parameters + //-------------------------------------------------------------------------- + void configure() + { + timer.start("configure"); + config::Config* config = config::Config::makeConfig(); + std::string strBuckets; + std::string strElems; + std::string strThreads; + uint64_t maxBuckets; + uint64_t maxElems; + uint64_t numThreads; - void BDL_bench_ridonly_compressed_32() - { - BDL_bench_ridonly( "compressed_32", true, 4 ); - } + strBuckets = config->getConfig("HashJoin", "MaxBuckets"); + CPPUNIT_ASSERT(strBuckets.size() > 0); + maxBuckets = config::Config::uFromText(strBuckets); - //-------------------------------------------------------------------------- - // test the saving and loading of a bdl file - //-------------------------------------------------------------------------- - void load_save() - { - typedef ElementType Element; + strElems = config->getConfig("HashJoin", "MaxElems"); + CPPUNIT_ASSERT(strElems.size() > 0); + maxElems = config::Config::uFromText(strElems); - vector v; + strThreads = config->getConfig("HashJoin", "NumThreads"); + CPPUNIT_ASSERT(strThreads.size() > 0); + numThreads = config::Config::uFromText(strThreads); - for (uint32_t i = 0; i < ::count; i++) - v.push_back(Element(i, i)); + cout << "config: hashjoin:maxbuckets - " << maxBuckets << endl; + cout << "config: hashjoin:maxelems - " << maxElems << endl; + cout << "config: hashjoin:numThreads - " << numThreads << endl; - vector v1; - vector v2; - vector v3; - - // save - ofstream f1; - ifstream f; - string filename = "bdl.txt"; - uint64_t cnt = v.size(); - f1.open(filename.c_str(), std::ios::binary); - f1.write((char*) &cnt, sizeof(cnt)); - f1.write((char*) (v.begin().operator->()), sizeof(Element) * cnt); - f.close(); - - // load - v1.push_back(Element(3, 4)); - f.open(filename.c_str(), std::ios::binary); - timer.start("read"); - v1.resize(v1.size() +::count); - f.read((char*) ((v1.begin() + 1).operator->()), cnt * sizeof(Element)); - cout << v1.size() << endl; - timer.stop("read"); - cout << "E1: " << v1[0].first << endl; - f.close(); - - f.open(filename.c_str(), std::ios::binary); - timer.start("assign"); - v2.assign(std::istream_iterator(f), - std::istream_iterator()); - cout << v2.size() << endl; - timer.stop("assign"); - f.close(); - - f.open(filename.c_str(), std::ios::binary); - timer.start("insert"); - v3.insert(v3.end(), std::istream_iterator(f), - std::istream_iterator()); - cout << v3.size() << endl; - timer.stop("insert"); - f.close(); - timer.finish(); - } - - //-------------------------------------------------------------------------- - // test the reading of bdl configuration parameters - //-------------------------------------------------------------------------- - void configure() - { - timer.start("configure"); - config::Config* config = config::Config::makeConfig(); - std::string strBuckets; - std::string strElems; - std::string strThreads; - uint64_t maxBuckets; - uint64_t maxElems; - uint64_t numThreads; - - strBuckets = config->getConfig("HashJoin", "MaxBuckets"); - CPPUNIT_ASSERT(strBuckets.size() > 0); - maxBuckets = config::Config::uFromText(strBuckets); - - strElems = config->getConfig("HashJoin", "MaxElems"); - CPPUNIT_ASSERT(strElems.size() > 0); - maxElems = config::Config::uFromText(strElems); - - strThreads = config->getConfig("HashJoin", "NumThreads"); - CPPUNIT_ASSERT(strThreads.size() > 0); - numThreads = config::Config::uFromText(strThreads); - - cout << "config: hashjoin:maxbuckets - " << maxBuckets << endl; - cout << "config: hashjoin:maxelems - " << maxElems << endl; - cout << "config: hashjoin:numThreads - " << numThreads << endl; - - timer.stop("configure"); - timer.finish(); - } + timer.stop("configure"); + timer.finish(); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(DataListDriver); @@ -515,13 +495,11 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DataListDriver); //------------------------------------------------------------------------------ // main entry point //------------------------------------------------------------------------------ -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/joblist/tdriver-bru.cpp b/dbcon/joblist/tdriver-bru.cpp index 6bfd9028b..b8c89f939 100644 --- a/dbcon/joblist/tdriver-bru.cpp +++ b/dbcon/joblist/tdriver-bru.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: tdriver-bru.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: tdriver-bru.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -45,7 +45,6 @@ #include "bucketreuse.h" - // #undef CPPUNIT_ASSERT // #define CPPUNIT_ASSERT(x) @@ -53,7 +52,7 @@ using namespace std; using namespace boost; using namespace joblist; -//Stopwatch timer; +// Stopwatch timer; //------------------------------------------------------------------------------ // TestDriver class derived from CppUnit @@ -61,90 +60,102 @@ using namespace joblist; class BucketReUseDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(BucketReUseDriver); + CPPUNIT_TEST(parseConfig); + CPPUNIT_TEST(createFiles); + CPPUNIT_TEST(reuseFiles); + CPPUNIT_TEST(newversion); + CPPUNIT_TEST(concurrent); + CPPUNIT_TEST(concurrent_newversion); + CPPUNIT_TEST(concurrent_race); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE(BucketReUseDriver); - CPPUNIT_TEST(parseConfig); - CPPUNIT_TEST(createFiles); - CPPUNIT_TEST(reuseFiles); - CPPUNIT_TEST(newversion); - CPPUNIT_TEST(concurrent); - CPPUNIT_TEST(concurrent_newversion); - CPPUNIT_TEST(concurrent_race); - CPPUNIT_TEST_SUITE_END(); + private: + public: + //-------------------------------------------------------------------------- + // setup method run prior to each unit test, inherited from base + //-------------------------------------------------------------------------- + void setUp() + { + clock_gettime(CLOCK_REALTIME, &ts); + } -private: -public: - //-------------------------------------------------------------------------- - // setup method run prior to each unit test, inherited from base - //-------------------------------------------------------------------------- - void setUp() + //-------------------------------------------------------------------------- + // validate results from a unit test, inherited from base + //-------------------------------------------------------------------------- + void validateResults() + { + } + + //-------------------------------------------------------------------------- + // test functions + //-------------------------------------------------------------------------- + void parseConfig(); + void createFiles(); + void reuseFiles(); + void newversion(); + void concurrent(); + void concurrent_newversion(); + void concurrent_race(); + + private: + //-------------------------------------------------------------------------- + // initialize method + // oid : reference to the column OID used for test + //-------------------------------------------------------------------------- + void initControl(execplan::CalpontSystemCatalog::OID& oid); + + //-------------------------------------------------------------------------- + // validate results + // files: the filenames to be verified if exist + // exist: expect if the files are created or deleted + //-------------------------------------------------------------------------- + void validateFileExist(set& files, bool exist); + + static void* insertThread(void*); + static void* readThread(void*); + + static void* scanThread(void*); + static void* reuseThread(void*); + static void* raceThread(void*); + + struct ThreadArg + { + uint64_t id; // thread id + uint64_t version; // db version + uint64_t buckets; // max bucket numbers + uint64_t elements; // max number of elem per bucket + uint64_t total; // total number of elements + execplan::CalpontSystemCatalog::OID oid; // column OID + BucketDataList* dl; // datalist + + // for sync threads + bool* flag; + pthread_mutex_t* mutex; + pthread_cond_t* cond; + + // for file status check + set* files; + + ThreadArg() + : id(0) + , version(0) + , buckets(0) + , elements(0) + , total(0) + , dl(NULL) + , flag(NULL) + , mutex(NULL) + , cond(NULL) + , files(NULL) { - clock_gettime(CLOCK_REALTIME, &ts); } + }; - //-------------------------------------------------------------------------- - // validate results from a unit test, inherited from base - //-------------------------------------------------------------------------- - void validateResults() {} + static const string column; - //-------------------------------------------------------------------------- - // test functions - //-------------------------------------------------------------------------- - void parseConfig(); - void createFiles(); - void reuseFiles(); - void newversion(); - void concurrent(); - void concurrent_newversion(); - void concurrent_race(); - -private: - //-------------------------------------------------------------------------- - // initialize method - // oid : reference to the column OID used for test - //-------------------------------------------------------------------------- - void initControl(execplan::CalpontSystemCatalog::OID& oid); - - //-------------------------------------------------------------------------- - // validate results - // files: the filenames to be verified if exist - // exist: expect if the files are created or deleted - //-------------------------------------------------------------------------- - void validateFileExist(set& files, bool exist); - - static void* insertThread(void*); - static void* readThread(void*); - - static void* scanThread(void*); - static void* reuseThread(void*); - static void* raceThread(void*); - - struct ThreadArg - { - uint64_t id; // thread id - uint64_t version; // db version - uint64_t buckets; // max bucket numbers - uint64_t elements; // max number of elem per bucket - uint64_t total; // total number of elements - execplan::CalpontSystemCatalog::OID oid; // column OID - BucketDataList* dl; // datalist - - // for sync threads - bool* flag; - pthread_mutex_t* mutex; - pthread_cond_t* cond; - - // for file status check - set* files; - - ThreadArg() : id(0), version(0), buckets(0), elements(0), total(0), dl(NULL), - flag(NULL), mutex(NULL), cond(NULL), files(NULL) {} - }; - - static const string column; - -public: - static struct timespec ts; + public: + static struct timespec ts; }; CPPUNIT_TEST_SUITE_REGISTRATION(BucketReUseDriver); @@ -157,11 +168,11 @@ struct timespec BucketReUseDriver::ts; //------------------------------------------------------------------------------ int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } struct timespec atTime(); @@ -169,564 +180,678 @@ ostream& operator<<(ostream& os, const struct timespec& t); void BucketReUseDriver::parseConfig() { - cout << "ut: parseConfig start...\n" << endl; + cout << "ut: parseConfig start...\n" << endl; - // before run this test, make sure "tpch.lineitem.l_orderkey" is in the Columnstore.xml - BucketReuseManager* control = BucketReuseManager::instance(); - ResourceManager rm; - BucketReuseManager::instance()->startup(rm); + // before run this test, make sure "tpch.lineitem.l_orderkey" is in the Columnstore.xml + BucketReuseManager* control = BucketReuseManager::instance(); + ResourceManager rm; + BucketReuseManager::instance()->startup(rm); - // list all the predicates if any listed in the Columnstore.xml file - for (BucketReuseMap::iterator it = control->fControlMap.begin(); - it != control->fControlMap.end(); it++) - cout << *(it->second); + // list all the predicates if any listed in the Columnstore.xml file + for (BucketReuseMap::iterator it = control->fControlMap.begin(); it != control->fControlMap.end(); it++) + cout << *(it->second); - cout << endl; + cout << endl; - size_t schemap = column.find_first_of("."); - size_t columnp = column.find_last_of("."); - CPPUNIT_ASSERT(schemap != string::npos && columnp != string::npos); + size_t schemap = column.find_first_of("."); + size_t columnp = column.find_last_of("."); + CPPUNIT_ASSERT(schemap != string::npos && columnp != string::npos); - execplan::CalpontSystemCatalog::TableColName columnName; - columnName.schema = column.substr(0, schemap); - columnName.table = column.substr(schemap + 1, columnp - schemap - 1); - columnName.column = column.substr(columnp + 1); + execplan::CalpontSystemCatalog::TableColName columnName; + columnName.schema = column.substr(0, schemap); + columnName.table = column.substr(schemap + 1, columnp - schemap - 1); + columnName.column = column.substr(columnp + 1); - string filter = "allrows"; + string filter = "allrows"; - CPPUNIT_ASSERT(control->fControlMap.find(BucketFileKey(columnName, filter)) != - control->fControlMap.end()); + CPPUNIT_ASSERT(control->fControlMap.find(BucketFileKey(columnName, filter)) != control->fControlMap.end()); - cout << "ut: parseConfig done!\n" << endl; + cout << "ut: parseConfig done!\n" << endl; } - void BucketReUseDriver::createFiles() { - cout << "ut: createFiles start...\n" << endl; + cout << "ut: createFiles start...\n" << endl; - ThreadArg arg; - arg.id = 1; - arg.version = 1; - arg.buckets = 2; - arg.elements = 2; - arg.total = 1000000; + ThreadArg arg; + arg.id = 1; + arg.version = 1; + arg.buckets = 2; + arg.elements = 2; + arg.total = 1000000; - execplan::CalpontSystemCatalog::OID oid; - initControl(oid); - arg.oid = oid; + execplan::CalpontSystemCatalog::OID oid; + initControl(oid); + arg.oid = oid; - set files; - arg.files = &files; + set files; + arg.files = &files; - pthread_t t; - pthread_create(&t, NULL, scanThread, &arg); - pthread_join(t, NULL); + pthread_t t; + pthread_create(&t, NULL, scanThread, &arg); + pthread_join(t, NULL); - validateFileExist(files, true); + validateFileExist(files, true); - cout << "ut: createFiles done!\n" << endl; + cout << "ut: createFiles done!\n" << endl; } - void BucketReUseDriver::reuseFiles() { - cout << "ut: reuseFiles start...\n" << endl; + cout << "ut: reuseFiles start...\n" << endl; - ThreadArg arg1; - arg1.id = 1; - arg1.version = 1; - arg1.buckets = 2; - arg1.elements = 2; - arg1.total = 1000000; + ThreadArg arg1; + arg1.id = 1; + arg1.version = 1; + arg1.buckets = 2; + arg1.elements = 2; + arg1.total = 1000000; - execplan::CalpontSystemCatalog::OID oid; - initControl(oid); - arg1.oid = oid; + execplan::CalpontSystemCatalog::OID oid; + initControl(oid); + arg1.oid = oid; - set files1; - arg1.files = &files1; + set files1; + arg1.files = &files1; - // create the files - pthread_t t1; - pthread_create(&t1, NULL, scanThread, &arg1); - pthread_join(t1, NULL); + // create the files + pthread_t t1; + pthread_create(&t1, NULL, scanThread, &arg1); + pthread_join(t1, NULL); - validateFileExist(files1, true); + validateFileExist(files1, true); - // use new datalist, reuse case - ThreadArg arg2 = arg1; - arg2.id = 2; + // use new datalist, reuse case + ThreadArg arg2 = arg1; + arg2.id = 2; - set files2; - arg2.files = &files2; + set files2; + arg2.files = &files2; - pthread_t t2; - pthread_create(&t2, NULL, reuseThread, &arg2); - pthread_join(t2, NULL); + pthread_t t2; + pthread_create(&t2, NULL, reuseThread, &arg2); + pthread_join(t2, NULL); - validateFileExist(files2, true); + validateFileExist(files2, true); - cout << "ut: reuseFiles done!\n" << endl; + cout << "ut: reuseFiles done!\n" << endl; } - void BucketReUseDriver::newversion() { - cout << "ut: newversion start...\n" << endl; + cout << "ut: newversion start...\n" << endl; - ThreadArg arg1; - arg1.id = 1; - arg1.version = 1; - arg1.buckets = 2; - arg1.elements = 2; - arg1.total = 1000000; + ThreadArg arg1; + arg1.id = 1; + arg1.version = 1; + arg1.buckets = 2; + arg1.elements = 2; + arg1.total = 1000000; - execplan::CalpontSystemCatalog::OID oid; - initControl(oid); - arg1.oid = oid; + execplan::CalpontSystemCatalog::OID oid; + initControl(oid); + arg1.oid = oid; - set files1; - arg1.files = &files1; + set files1; + arg1.files = &files1; - // create the files - pthread_t t1; - pthread_create(&t1, NULL, scanThread, &arg1); - pthread_join(t1, NULL); + // create the files + pthread_t t1; + pthread_create(&t1, NULL, scanThread, &arg1); + pthread_join(t1, NULL); - validateFileExist(files1, true); + validateFileExist(files1, true); - // new version - ThreadArg arg2 = arg1; - arg2.id = 2; - arg2.version = 2; + // new version + ThreadArg arg2 = arg1; + arg2.id = 2; + arg2.version = 2; - set files2; - arg2.files = &files2; + set files2; + arg2.files = &files2; - pthread_t t2; - pthread_create(&t2, NULL, scanThread, &arg2); - pthread_join(t2, NULL); + pthread_t t2; + pthread_create(&t2, NULL, scanThread, &arg2); + pthread_join(t2, NULL); - pthread_yield(); + pthread_yield(); - validateFileExist(files1, false); - validateFileExist(files2, true); + validateFileExist(files1, false); + validateFileExist(files2, true); - // read from the new files with new datalist - ThreadArg arg3 = arg2; - arg3.id = 3; - arg3.dl = NULL; + // read from the new files with new datalist + ThreadArg arg3 = arg2; + arg3.id = 3; + arg3.dl = NULL; - set files3; - arg3.files = &files3; + set files3; + arg3.files = &files3; - pthread_t t3; - pthread_create(&t3, NULL, reuseThread, &arg3); - pthread_join(t3, NULL); + pthread_t t3; + pthread_create(&t3, NULL, reuseThread, &arg3); + pthread_join(t3, NULL); - validateFileExist(files3, true); + validateFileExist(files3, true); - cout << "ut: newversion done!\n" << endl; + cout << "ut: newversion done!\n" << endl; } void BucketReUseDriver::concurrent() { - cout << "ut: concurrent start...\n" << endl; + cout << "ut: concurrent start...\n" << endl; - ThreadArg arg1; - arg1.id = 1; - arg1.version = 1; - arg1.buckets = 2; - arg1.elements = 2; - arg1.total = 1000000; + ThreadArg arg1; + arg1.id = 1; + arg1.version = 1; + arg1.buckets = 2; + arg1.elements = 2; + arg1.total = 1000000; - execplan::CalpontSystemCatalog::OID oid; - initControl(oid); - arg1.oid = oid; + execplan::CalpontSystemCatalog::OID oid; + initControl(oid); + arg1.oid = oid; - set files1; - arg1.files = &files1; + set files1; + arg1.files = &files1; - // create the files - pthread_t t1; - pthread_create(&t1, NULL, scanThread, &arg1); + // create the files + pthread_t t1; + pthread_create(&t1, NULL, scanThread, &arg1); - sleep(1); + sleep(1); - // reuse case, current thread - ThreadArg arg2 = arg1; - arg2.id = 2; - set files2; - arg2.files = &files2; + // reuse case, current thread + ThreadArg arg2 = arg1; + arg2.id = 2; + set files2; + arg2.files = &files2; - pthread_t t2; - pthread_create(&t2, NULL, reuseThread, &arg2); + pthread_t t2; + pthread_create(&t2, NULL, reuseThread, &arg2); - pthread_join(t1, NULL); - validateFileExist(files1, true); + pthread_join(t1, NULL); + validateFileExist(files1, true); - pthread_join(t2, NULL); - validateFileExist(files2, true); + pthread_join(t2, NULL); + validateFileExist(files2, true); - cout << "ut: concurrent done!\n" << endl; + cout << "ut: concurrent done!\n" << endl; } void BucketReUseDriver::concurrent_newversion() { - cout << "ut: concurrent_newversion start...\n" << endl; + cout << "ut: concurrent_newversion start...\n" << endl; - bool flag = false; - pthread_mutex_t mutex; - pthread_mutex_init(&mutex, 0); - pthread_cond_t cond; - pthread_cond_init(&cond, 0); + bool flag = false; + pthread_mutex_t mutex; + pthread_mutex_init(&mutex, 0); + pthread_cond_t cond; + pthread_cond_init(&cond, 0); - ThreadArg arg1; - arg1.id = 1; - arg1.version = 1; - arg1.buckets = 2; - arg1.elements = 2; - arg1.total = 1000000; + ThreadArg arg1; + arg1.id = 1; + arg1.version = 1; + arg1.buckets = 2; + arg1.elements = 2; + arg1.total = 1000000; - execplan::CalpontSystemCatalog::OID oid; - initControl(oid); - arg1.oid = oid; + execplan::CalpontSystemCatalog::OID oid; + initControl(oid); + arg1.oid = oid; - set files1; - arg1.files = &files1; + set files1; + arg1.files = &files1; - // create the files - pthread_t t1; - pthread_create(&t1, NULL, scanThread, &arg1); + // create the files + pthread_t t1; + pthread_create(&t1, NULL, scanThread, &arg1); - // reuse case, current thread - ThreadArg arg2 = arg1; - arg2.id = 2; - set files2; - arg2.files = &files2; + // reuse case, current thread + ThreadArg arg2 = arg1; + arg2.id = 2; + set files2; + arg2.files = &files2; - sleep(1); + sleep(1); - pthread_t t2; - pthread_create(&t2, NULL, reuseThread, &arg2); + pthread_t t2; + pthread_create(&t2, NULL, reuseThread, &arg2); - // new version - ThreadArg arg3 = arg1; - arg3.id = 3; - arg3.version = 3; - arg3.flag = &flag; - arg3.mutex = &mutex; - arg3.cond = &cond; - set files3; - arg3.files = &files3; + // new version + ThreadArg arg3 = arg1; + arg3.id = 3; + arg3.version = 3; + arg3.flag = &flag; + arg3.mutex = &mutex; + arg3.cond = &cond; + set files3; + arg3.files = &files3; - pthread_t t3; - pthread_create(&t3, NULL, scanThread, &arg3); + pthread_t t3; + pthread_create(&t3, NULL, scanThread, &arg3); - sleep(3); + sleep(3); - pthread_mutex_lock(&mutex); - flag = true; - pthread_cond_broadcast(&cond); - pthread_mutex_unlock(&mutex); + pthread_mutex_lock(&mutex); + flag = true; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&mutex); - pthread_join(t1, NULL); - pthread_join(t2, NULL); - pthread_join(t3, NULL); + pthread_join(t1, NULL); + pthread_join(t2, NULL); + pthread_join(t3, NULL); - // let cleanup thread do its job - pthread_yield(); + // let cleanup thread do its job + pthread_yield(); - validateFileExist(files1, false); - validateFileExist(files2, false); - validateFileExist(files3, true); + validateFileExist(files1, false); + validateFileExist(files2, false); + validateFileExist(files3, true); - pthread_mutex_destroy(&mutex); - pthread_cond_destroy(&cond); + pthread_mutex_destroy(&mutex); + pthread_cond_destroy(&cond); - cout << "ut: concurrent_newversion done!\n" << endl; + cout << "ut: concurrent_newversion done!\n" << endl; } void BucketReUseDriver::concurrent_race() { - cout << "ut: concurrent_race start...\n" << endl; + cout << "ut: concurrent_race start...\n" << endl; - ThreadArg arg1; - arg1.id = 1; - arg1.version = 1; - arg1.buckets = 2; - arg1.elements = 2; - arg1.total = 2000000; + ThreadArg arg1; + arg1.id = 1; + arg1.version = 1; + arg1.buckets = 2; + arg1.elements = 2; + arg1.total = 2000000; - execplan::CalpontSystemCatalog::OID oid; - initControl(oid); - arg1.oid = oid; + execplan::CalpontSystemCatalog::OID oid; + initControl(oid); + arg1.oid = oid; - set files1; - arg1.files = &files1; + set files1; + arg1.files = &files1; - ThreadArg arg2 = arg1; - arg2.id = 2; - set files2; - arg2.files = &files2; + ThreadArg arg2 = arg1; + arg2.id = 2; + set files2; + arg2.files = &files2; - // start the version 1 threads - pthread_t t1; - pthread_t t2; - pthread_create(&t1, NULL, raceThread, &arg1); - pthread_create(&t2, NULL, raceThread, &arg2); + // start the version 1 threads + pthread_t t1; + pthread_t t2; + pthread_create(&t1, NULL, raceThread, &arg1); + pthread_create(&t2, NULL, raceThread, &arg2); - // let the version 1 threads register - sleep(1); + // let the version 1 threads register + sleep(1); - ThreadArg arg3 = arg1; - arg3.id = 3; - arg3.version = 4; - arg3.total = 1000000; - set files3; - arg3.files = &files3; + ThreadArg arg3 = arg1; + arg3.id = 3; + arg3.version = 4; + arg3.total = 1000000; + set files3; + arg3.files = &files3; - ThreadArg arg4 = arg3; - arg4.id = 4; - set files4; - arg4.files = &files4; + ThreadArg arg4 = arg3; + arg4.id = 4; + set files4; + arg4.files = &files4; - // start the version 4 threads - pthread_t t3; - pthread_t t4; - pthread_create(&t3, NULL, raceThread, &arg3); - pthread_create(&t4, NULL, raceThread, &arg4); + // start the version 4 threads + pthread_t t3; + pthread_t t4; + pthread_create(&t3, NULL, raceThread, &arg3); + pthread_create(&t4, NULL, raceThread, &arg4); - pthread_join(t3, NULL); - pthread_join(t4, NULL); - validateFileExist(files1, true); - validateFileExist(files4, true); + pthread_join(t3, NULL); + pthread_join(t4, NULL); + validateFileExist(files1, true); + validateFileExist(files4, true); - pthread_join(t1, NULL); - pthread_join(t2, NULL); - validateFileExist(files1, false); - validateFileExist(files4, true); + pthread_join(t1, NULL); + pthread_join(t2, NULL); + validateFileExist(files1, false); + validateFileExist(files4, true); - cout << "ut: concurrent_race done!\n" << endl; + cout << "ut: concurrent_race done!\n" << endl; } void BucketReUseDriver::initControl(execplan::CalpontSystemCatalog::OID& oid) { - // make sure the column for testing is in the map, "column" is a class variable - // to test with another column, only one place to change - // -- to validate the config parsing, run parseConfig -- - config::Config* cf = config::Config::makeConfig(); - vector columns; - cf->getConfig("HashBucketReuse", "Predicate", columns); + // make sure the column for testing is in the map, "column" is a class variable + // to test with another column, only one place to change + // -- to validate the config parsing, run parseConfig -- + config::Config* cf = config::Config::makeConfig(); + vector columns; + cf->getConfig("HashBucketReuse", "Predicate", columns); - bool found = true; + bool found = true; - for (vector::iterator it = columns.begin(); it != columns.end(); ++it) + for (vector::iterator it = columns.begin(); it != columns.end(); ++it) + { + if (it->compare(0, column.size(), column) == 0) { - if (it->compare(0, column.size(), column) == 0) - { - found = true; - break; - } + found = true; + break; } + } - string filter = "allrows"; - BucketReuseManager* control = BucketReuseManager::instance(); + string filter = "allrows"; + BucketReuseManager* control = BucketReuseManager::instance(); - if (found == false) - { - cout << "tpch.lineitem.l_orderkey is not in the Columnstore.xml!)" << endl; - cout << "insert it to countinue unit test" << endl; + if (found == false) + { + cout << "tpch.lineitem.l_orderkey is not in the Columnstore.xml!)" << endl; + cout << "insert it to countinue unit test" << endl; - size_t schemap = column.find_first_of("."); - size_t columnp = column.find_last_of("."); - CPPUNIT_ASSERT(schemap != string::npos && columnp != string::npos); - - execplan::CalpontSystemCatalog::TableColName tcn; - tcn.schema = column.substr(0, schemap); - tcn.table = column.substr(schemap + 1, columnp - schemap - 1); - tcn.column = column.substr(columnp + 1); - - control->fConfigMap.insert(pair(column, BucketFileKey(tcn, filter))); - } - - ResourceManager rm; - // now start the BucketReuseManager - BucketReuseManager::instance()->startup(rm); - - // get the oid for registration size_t schemap = column.find_first_of("."); size_t columnp = column.find_last_of("."); CPPUNIT_ASSERT(schemap != string::npos && columnp != string::npos); - execplan::CalpontSystemCatalog::TableColName columnName; - columnName.schema = column.substr(0, schemap); - columnName.table = column.substr(schemap + 1, columnp - schemap - 1); - columnName.column = column.substr(columnp + 1); + execplan::CalpontSystemCatalog::TableColName tcn; + tcn.schema = column.substr(0, schemap); + tcn.table = column.substr(schemap + 1, columnp - schemap - 1); + tcn.column = column.substr(columnp + 1); - CPPUNIT_ASSERT(control->fControlMap.find(BucketFileKey(columnName, filter)) != - control->fControlMap.end()); + control->fConfigMap.insert(pair(column, BucketFileKey(tcn, filter))); + } + + ResourceManager rm; + // now start the BucketReuseManager + BucketReuseManager::instance()->startup(rm); + + // get the oid for registration + size_t schemap = column.find_first_of("."); + size_t columnp = column.find_last_of("."); + CPPUNIT_ASSERT(schemap != string::npos && columnp != string::npos); + + execplan::CalpontSystemCatalog::TableColName columnName; + columnName.schema = column.substr(0, schemap); + columnName.table = column.substr(schemap + 1, columnp - schemap - 1); + columnName.column = column.substr(columnp + 1); + + CPPUNIT_ASSERT(control->fControlMap.find(BucketFileKey(columnName, filter)) != control->fControlMap.end()); } - void BucketReUseDriver::validateFileExist(set& files, bool exist) { - cout << "\ncheck if files exist or not:" << endl; + cout << "\ncheck if files exist or not:" << endl; - for (set::iterator i = files.begin(); i != files.end(); i++) + for (set::iterator i = files.begin(); i != files.end(); i++) + { + filesystem::path p(i->c_str()); + cout << (*i) << "-- "; + + if (exist) { - filesystem::path p(i->c_str()); - cout << (*i) << "-- "; - - if (exist) - { - CPPUNIT_ASSERT(filesystem::exists(p)); - cout << "OK" << endl; - } - else - { - CPPUNIT_ASSERT(!filesystem::exists(p)); - cout << "GONE" << endl; - } + CPPUNIT_ASSERT(filesystem::exists(p)); + cout << "OK" << endl; } + else + { + CPPUNIT_ASSERT(!filesystem::exists(p)); + cout << "GONE" << endl; + } + } - cout << endl; + cout << endl; } - void* BucketReUseDriver::insertThread(void* arg) { - ThreadArg* a = reinterpret_cast(arg); - BucketDataList* dl = a->dl; - CPPUNIT_ASSERT(dl != NULL); + ThreadArg* a = reinterpret_cast(arg); + BucketDataList* dl = a->dl; + CPPUNIT_ASSERT(dl != NULL); - cout << "thread " << a->id << " start at " << atTime() << endl; + cout << "thread " << a->id << " start at " << atTime() << endl; - BucketReuseControlEntry* entry = dl->reuseControl(); + BucketReuseControlEntry* entry = dl->reuseControl(); - for (uint64_t i = 0; i < a->buckets; i++) - { - stringstream ss; - ss << entry->baseName() << "." << i; - filesystem::path p(ss.str().c_str()); - a->files->insert(ss.str()); - } + for (uint64_t i = 0; i < a->buckets; i++) + { + stringstream ss; + ss << entry->baseName() << "." << i; + filesystem::path p(ss.str().c_str()); + a->files->insert(ss.str()); + } - ElementType e; + ElementType e; - for (uint64_t i = 0; i < a->total; i++) - { - e.first = i; - e.second = i * 10 + a->version; // include the version in values - dl->insert(e); - } + for (uint64_t i = 0; i < a->total; i++) + { + e.first = i; + e.second = i * 10 + a->version; // include the version in values + dl->insert(e); + } - cout << "thread[" << a->id << "] last element inserted at " << atTime() << endl; - dl->endOfInput(); + cout << "thread[" << a->id << "] last element inserted at " << atTime() << endl; + dl->endOfInput(); - cout << "thread " << a->id << " finished at " << atTime() << endl; + cout << "thread " << a->id << " finished at " << atTime() << endl; - return NULL; + return NULL; } - void* BucketReUseDriver::readThread(void* arg) { - ThreadArg* a = reinterpret_cast(arg); - BucketDataList* dl = a->dl; - CPPUNIT_ASSERT(dl != NULL); + ThreadArg* a = reinterpret_cast(arg); + BucketDataList* dl = a->dl; + CPPUNIT_ASSERT(dl != NULL); - cout << "thread " << a->id << " start at " << atTime() << endl; + cout << "thread " << a->id << " start at " << atTime() << endl; - BucketReuseControlEntry* entry = dl->reuseControl(); + BucketReuseControlEntry* entry = dl->reuseControl(); - for (uint64_t i = 0; i < a->buckets; i++) + for (uint64_t i = 0; i < a->buckets; i++) + { + stringstream ss; + ss << entry->baseName() << "." << i; + filesystem::path p(ss.str().c_str()); + a->files->insert(ss.str()); + } + + ElementType e; + uint64_t min = 0xffffffff, max = 0, count = 0; + uint64_t k[a->buckets]; // count of each bucket + bool firstRead = true; + + for (uint64_t i = 0; i < a->buckets; i++) + { + k[i] = 0; + uint64_t it = dl->getIterator(i); + + while (dl->next(i, it, &e)) { - stringstream ss; - ss << entry->baseName() << "." << i; - filesystem::path p(ss.str().c_str()); - a->files->insert(ss.str()); + if (firstRead) + { + cout << "thread[" << a->id << "] first read at " << atTime() << endl; + firstRead = false; + } + + if (e.second < min) + min = e.second; + + if (e.second > max) + max = e.second; + + // output the first 10 of each bucket or last 10 of the datalist + // if (count < 10 || (a->total - count) < 10 || k[i] < 10) + if (count < 2 || (a->total - count) < 2 || k[i] < 2) + cout << "thread[" << a->id << "] bucket:" << i << " e(" << e.first << ", " << e.second << ")" << endl; + + count++; + k[i]++; } + } - ElementType e; - uint64_t min = 0xffffffff, max = 0, count = 0; - uint64_t k[a->buckets]; // count of each bucket - bool firstRead = true; + cout << "\nthread[" << a->id << "] element: count = " << count << ", min/max = " << min << "/" << max + << ", elements in each bucket: "; - for (uint64_t i = 0; i < a->buckets; i++) - { - k[i] = 0; - uint64_t it = dl->getIterator(i); + for (uint64_t i = 0; i < a->buckets; i++) + cout << k[i] << " "; - while (dl->next(i, it, &e)) - { - if (firstRead) - { - cout << "thread[" << a->id << "] first read at " << atTime() << endl; - firstRead = false; - } + cout << endl; - if (e.second < min) min = e.second; + cout << "thread " << a->id << " finished at " << atTime() << endl; - if (e.second > max) max = e.second; - - // output the first 10 of each bucket or last 10 of the datalist - //if (count < 10 || (a->total - count) < 10 || k[i] < 10) - if (count < 2 || (a->total - count) < 2 || k[i] < 2) - cout << "thread[" << a->id << "] bucket:" << i - << " e(" << e.first << ", " << e.second << ")" << endl; - - count++; - k[i]++; - } - } - - cout << "\nthread[" << a->id << "] element: count = " << count - << ", min/max = " << min << "/" << max << ", elements in each bucket: "; - - for (uint64_t i = 0; i < a->buckets; i++) - cout << k[i] << " "; - - cout << endl; - - cout << "thread " << a->id << " finished at " << atTime() << endl; - - return NULL; + return NULL; } - void* BucketReUseDriver::scanThread(void* arg) { - ThreadArg* a = reinterpret_cast(arg); - BucketDataList* dl = a->dl; - CPPUNIT_ASSERT(dl == NULL); + ThreadArg* a = reinterpret_cast(arg); + BucketDataList* dl = a->dl; + CPPUNIT_ASSERT(dl == NULL); - if (a->cond != NULL) - { - pthread_mutex_lock(a->mutex); + if (a->cond != NULL) + { + pthread_mutex_lock(a->mutex); - while (*(a->flag) != true) - pthread_cond_wait(a->cond, a->mutex); + while (*(a->flag) != true) + pthread_cond_wait(a->cond, a->mutex); - pthread_mutex_unlock(a->mutex); - } + pthread_mutex_unlock(a->mutex); + } - cout << "thread " << a->id << " start at " << atTime() << endl; + cout << "thread " << a->id << " start at " << atTime() << endl; - string dummy; - bool scan = false; + string dummy; + bool scan = false; + boost::shared_ptr c = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0x80000000); + execplan::CalpontSystemCatalog::TableColName tcn = c->colName(a->oid); + BucketReuseControlEntry* entry = + BucketReuseManager::instance()->userRegister(tcn, dummy, a->version, a->buckets, scan); + CPPUNIT_ASSERT(scan == true); + + ResourceManager rm; + dl = new BucketDataList(a->buckets, 1, a->elements, rm); + dl->setElementMode(1); + dl->reuseControl(entry, !scan); + + ThreadArg arg1 = *a; + arg1.id *= 10; + arg1.dl = dl; + + pthread_t t1; + pthread_create(&t1, NULL, insertThread, &arg1); + + ThreadArg arg2 = arg1; + arg2.id += 1; + pthread_t t2; + pthread_create(&t2, NULL, readThread, &arg2); + + pthread_join(t1, NULL); + pthread_join(t2, NULL); + + delete dl; + + cout << "thread " << a->id << " finished at " << atTime() << endl; + + return NULL; +} + +void* BucketReUseDriver::reuseThread(void* arg) +{ + ThreadArg* a = reinterpret_cast(arg); + BucketDataList* dl = a->dl; + CPPUNIT_ASSERT(dl == NULL); + + if (a->cond != NULL) + { + pthread_mutex_lock(a->mutex); + + while (*(a->flag) != true) + pthread_cond_wait(a->cond, a->mutex); + + pthread_mutex_unlock(a->mutex); + } + + cout << "thread " << a->id << " start at " << atTime() << endl; + + string dummy; + bool scan = true; + boost::shared_ptr c = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0x80000000); + execplan::CalpontSystemCatalog::TableColName tcn = c->colName(a->oid); + BucketReuseControlEntry* entry = + BucketReuseManager::instance()->userRegister(tcn, dummy, a->version, a->buckets, scan); + CPPUNIT_ASSERT(scan == false); + + ResourceManager rm; + dl = new BucketDataList(a->buckets, 1, a->elements, rm); + dl->setElementMode(1); + dl->reuseControl(entry, !scan); + + ThreadArg arg1 = *a; + arg1.id *= 10; + arg1.dl = dl; + + pthread_t t1; + pthread_create(&t1, NULL, readThread, &arg1); + + if (entry->fileStatus() == BucketReuseControlEntry::progress_c) + { + boost::mutex::scoped_lock lock(BucketReuseManager::instance()->getMutex()); + dl->reuseControl()->stateChange().wait(lock); + } + else + { + CPPUNIT_ASSERT((entry->fileStatus() == BucketReuseControlEntry::using_c) || + (entry->fileStatus() == BucketReuseControlEntry::ready_c)); + } + + // the bucket files are ready + dl->restoreBucketInformation(); + dl->endOfInput(); + + pthread_join(t1, NULL); + + delete dl; + + cout << "thread " << a->id << " finished at " << atTime() << endl; + + return NULL; +} + +void* BucketReUseDriver::raceThread(void* arg) +{ + ThreadArg* a = reinterpret_cast(arg); + BucketDataList* dl = a->dl; + CPPUNIT_ASSERT(dl == NULL); + + if (a->cond != NULL) + { + pthread_mutex_lock(a->mutex); + + while (*(a->flag) != true) + pthread_cond_wait(a->cond, a->mutex); + + pthread_mutex_unlock(a->mutex); + } + + cout << "thread " << a->id << " start at " << atTime() << endl; + + string dummy; + bool scan = true; + ResourceManager rm; + BucketReuseControlEntry* entry = NULL; + { + boost::mutex::scoped_lock lock(BucketReuseManager::instance()->getMutex()); boost::shared_ptr c = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0x80000000); execplan::CalpontSystemCatalog::TableColName tcn = c->colName(a->oid); - BucketReuseControlEntry* entry = BucketReuseManager::instance()->userRegister( - tcn, dummy, a->version, a->buckets, scan); - CPPUNIT_ASSERT(scan == true); + entry = BucketReuseManager::instance()->userRegister(tcn, dummy, a->version, a->buckets, scan); - ResourceManager rm; dl = new BucketDataList(a->buckets, 1, a->elements, rm); dl->setElementMode(1); dl->reuseControl(entry, !scan); + } + + if (scan == true) + { + CPPUNIT_ASSERT(entry->fileStatus() == BucketReuseControlEntry::progress_c); ThreadArg arg1 = *a; arg1.id *= 10; @@ -742,47 +867,9 @@ void* BucketReUseDriver::scanThread(void* arg) pthread_join(t1, NULL); pthread_join(t2, NULL); - - delete dl; - - cout << "thread " << a->id << " finished at " << atTime() << endl; - - return NULL; -} - - -void* BucketReUseDriver::reuseThread(void* arg) -{ - ThreadArg* a = reinterpret_cast(arg); - BucketDataList* dl = a->dl; - CPPUNIT_ASSERT(dl == NULL); - - if (a->cond != NULL) - { - pthread_mutex_lock(a->mutex); - - while (*(a->flag) != true) - pthread_cond_wait(a->cond, a->mutex); - - pthread_mutex_unlock(a->mutex); - } - - cout << "thread " << a->id << " start at " << atTime() << endl; - - string dummy; - bool scan = true; - boost::shared_ptr c = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0x80000000); - execplan::CalpontSystemCatalog::TableColName tcn = c->colName(a->oid); - BucketReuseControlEntry* entry = BucketReuseManager::instance()->userRegister( - tcn, dummy, a->version, a->buckets, scan); - CPPUNIT_ASSERT(scan == false); - - ResourceManager rm; - dl = new BucketDataList(a->buckets, 1, a->elements, rm); - dl->setElementMode(1); - dl->reuseControl(entry, !scan); - + } + else + { ThreadArg arg1 = *a; arg1.id *= 10; arg1.dl = dl; @@ -792,13 +879,13 @@ void* BucketReUseDriver::reuseThread(void* arg) if (entry->fileStatus() == BucketReuseControlEntry::progress_c) { - boost::mutex::scoped_lock lock(BucketReuseManager::instance()->getMutex()); - dl->reuseControl()->stateChange().wait(lock); + boost::mutex::scoped_lock lock(BucketReuseManager::instance()->getMutex()); + dl->reuseControl()->stateChange().wait(lock); } else { - CPPUNIT_ASSERT((entry->fileStatus() == BucketReuseControlEntry::using_c) || - (entry->fileStatus() == BucketReuseControlEntry::ready_c)); + CPPUNIT_ASSERT((entry->fileStatus() == BucketReuseControlEntry::using_c) || + (entry->fileStatus() == BucketReuseControlEntry::ready_c)); } // the bucket files are ready @@ -806,127 +893,37 @@ void* BucketReUseDriver::reuseThread(void* arg) dl->endOfInput(); pthread_join(t1, NULL); + } - delete dl; + delete dl; - cout << "thread " << a->id << " finished at " << atTime() << endl; + cout << "thread " << a->id << " finished at " << atTime() << endl; - return NULL; + return NULL; } - -void* BucketReUseDriver::raceThread(void* arg) -{ - ThreadArg* a = reinterpret_cast(arg); - BucketDataList* dl = a->dl; - CPPUNIT_ASSERT(dl == NULL); - - if (a->cond != NULL) - { - pthread_mutex_lock(a->mutex); - - while (*(a->flag) != true) - pthread_cond_wait(a->cond, a->mutex); - - pthread_mutex_unlock(a->mutex); - } - - cout << "thread " << a->id << " start at " << atTime() << endl; - - string dummy; - bool scan = true; - ResourceManager rm; - BucketReuseControlEntry* entry = NULL; - { - boost::mutex::scoped_lock lock(BucketReuseManager::instance()->getMutex()); - boost::shared_ptr c = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0x80000000); - execplan::CalpontSystemCatalog::TableColName tcn = c->colName(a->oid); - entry = BucketReuseManager::instance()->userRegister( - tcn, dummy, a->version, a->buckets, scan); - - dl = new BucketDataList(a->buckets, 1, a->elements, rm); - dl->setElementMode(1); - dl->reuseControl(entry, !scan); - } - - if (scan == true) - { - CPPUNIT_ASSERT(entry->fileStatus() == BucketReuseControlEntry::progress_c); - - ThreadArg arg1 = *a; - arg1.id *= 10; - arg1.dl = dl; - - pthread_t t1; - pthread_create(&t1, NULL, insertThread, &arg1); - - ThreadArg arg2 = arg1; - arg2.id += 1; - pthread_t t2; - pthread_create(&t2, NULL, readThread, &arg2); - - pthread_join(t1, NULL); - pthread_join(t2, NULL); - } - else - { - ThreadArg arg1 = *a; - arg1.id *= 10; - arg1.dl = dl; - - pthread_t t1; - pthread_create(&t1, NULL, readThread, &arg1); - - if (entry->fileStatus() == BucketReuseControlEntry::progress_c) - { - boost::mutex::scoped_lock lock(BucketReuseManager::instance()->getMutex()); - dl->reuseControl()->stateChange().wait(lock); - } - else - { - CPPUNIT_ASSERT((entry->fileStatus() == BucketReuseControlEntry::using_c) || - (entry->fileStatus() == BucketReuseControlEntry::ready_c)); - } - - // the bucket files are ready - dl->restoreBucketInformation(); - dl->endOfInput(); - - pthread_join(t1, NULL); - } - - delete dl; - - cout << "thread " << a->id << " finished at " << atTime() << endl; - - return NULL; -} - - timespec atTime() { - timespec ts, ts1, ts2; - ts1 = BucketReUseDriver::ts; - clock_gettime(CLOCK_REALTIME, &ts2); + timespec ts, ts1, ts2; + ts1 = BucketReUseDriver::ts; + clock_gettime(CLOCK_REALTIME, &ts2); - if (ts2.tv_nsec < ts1.tv_nsec) - { - ts.tv_sec = ts2.tv_sec - ts1.tv_sec - 1; - ts.tv_nsec = ts2.tv_nsec + 1000000000 - ts1.tv_nsec; - } - else - { - ts.tv_sec = ts2.tv_sec - ts1.tv_sec; - ts.tv_nsec = ts2.tv_nsec - ts1.tv_nsec; - } + if (ts2.tv_nsec < ts1.tv_nsec) + { + ts.tv_sec = ts2.tv_sec - ts1.tv_sec - 1; + ts.tv_nsec = ts2.tv_nsec + 1000000000 - ts1.tv_nsec; + } + else + { + ts.tv_sec = ts2.tv_sec - ts1.tv_sec; + ts.tv_nsec = ts2.tv_nsec - ts1.tv_nsec; + } - return ts; + return ts; } ostream& operator<<(ostream& os, const struct timespec& t) { - os << t.tv_sec << "." << setw(9) << setfill('0') << t.tv_nsec << "s"; - return os; + os << t.tv_sec << "." << setw(9) << setfill('0') << t.tv_nsec << "s"; + return os; } - diff --git a/dbcon/joblist/tdriver-datalist.cpp b/dbcon/joblist/tdriver-datalist.cpp index fd9e9ad3f..2e5a05f34 100644 --- a/dbcon/joblist/tdriver-datalist.cpp +++ b/dbcon/joblist/tdriver-datalist.cpp @@ -45,1425 +45,1398 @@ Stopwatch timer; struct DLElement { -public: - int i; - bool operator<(const DLElement& c) const - { - return (i < c.i); - } - const char* getHashString(int mode, int* len) const - { - *len = 4; - return (char*) &i; - } + public: + int i; + bool operator<(const DLElement& c) const + { + return (i < c.i); + } + const char* getHashString(int mode, int* len) const + { + *len = 4; + return (char*)&i; + } }; istream& operator>>(istream& is, DLElement& dl) { - is.read((char*)&dl.i, sizeof(int)); - return is; + is.read((char*)&dl.i, sizeof(int)); + return is; } ostream& operator<<(ostream& os, const DLElement& dl) { - os.write((char*)&dl.i, sizeof(int)); - return os; + os.write((char*)&dl.i, sizeof(int)); + return os; } -int64_t count = 150000000/*1000000*/; -int maxElements = 32000000/*50000*/; //max elements in memory at once for the benchmarks -int buckets = 16; // # of buckets to use in bucketDL tests. -#define MAXINT64 (uint64_t) 0xffffffffffffffffLL +int64_t count = 150000000 /*1000000*/; +int maxElements = 32000000 /*50000*/; // max elements in memory at once for the benchmarks +int buckets = 16; // # of buckets to use in bucketDL tests. +#define MAXINT64 (uint64_t)0xffffffffffffffffLL /* Note, this is like what we'll use for banding operations */ class BandGrouper { -public: - uint32_t operator()(const char* data, uint32_t len) const - { - stringstream ss(string(data, len)); - ElementType dle; + public: + uint32_t operator()(const char* data, uint32_t len) const + { + stringstream ss(string(data, len)); + ElementType dle; - ss >> dle; -// cout << "bandGrouper sees " << dle.i << " returning " << dle.i/(::count/buckets) << endl; - return dle.first / (::count / buckets); - } + ss >> dle; + // cout << "bandGrouper sees " << dle.i << " returning " << + // dle.i/(::count/buckets) << endl; + return dle.first / (::count / buckets); + } }; -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } } int cCount = 0, pCount = 0; void* MP_FIFO_stress_producer(void* arg) { - FIFO* f = reinterpret_cast *>(arg); - uint64_t i; - ElementType val; - uint64_t id = ::pCount++; + FIFO* f = reinterpret_cast*>(arg); + uint64_t i; + ElementType val; + uint64_t id = ::pCount++; -// cout << "producer " << id << " started" << endl; - for (i = 0; i < (uint32_t) ::count / 10; i++) - { - val.first = (id << 60) | i; - val.second = val.first; - f->insert(val); - } + // cout << "producer " << id << " started" << endl; + for (i = 0; i < (uint32_t)::count / 10; i++) + { + val.first = (id << 60) | i; + val.second = val.first; + f->insert(val); + } - return NULL; + return NULL; } void* MP_FIFO_stress_consumer(void* arg) { - FIFO* f = reinterpret_cast *>(arg); - int it; - ElementType val; - bool ret; -// int id = ::cCount++; - uint64_t pCounter[10]; // 10 producers right now; make this a global var... - uint64_t pnum, pnext; + FIFO* f = reinterpret_cast*>(arg); + int it; + ElementType val; + bool ret; + // int id = ::cCount++; + uint64_t pCounter[10]; // 10 producers right now; make this a global var... + uint64_t pnum, pnext; - memset(pCounter, 0, 8 * 10); + memset(pCounter, 0, 8 * 10); -// cout << "consumer " << id << " started" << endl; - it = f->getIterator(); + // cout << "consumer " << id << " started" << endl; + it = f->getIterator(); + ret = f->next(it, &val); + + while (ret) + { + pnum = val.first >> 60; + pnext = pCounter[pnum]++; + // cerr << id << ": .first=0x" << hex << val.first << dec << " pnum:" << pnum << " val:" + // << pnext << endl; + CPPUNIT_ASSERT((val.first & 0xfffffffffffffffLL) == pnext); + CPPUNIT_ASSERT(val.second == val.first); ret = f->next(it, &val); + } - while (ret) - { - pnum = val.first >> 60; - pnext = pCounter[pnum]++; -// cerr << id << ": .first=0x" << hex << val.first << dec << " pnum:" << pnum << " val:" << pnext << endl; - CPPUNIT_ASSERT((val.first & 0xfffffffffffffffLL) == pnext); - CPPUNIT_ASSERT(val.second == val.first); - ret = f->next(it, &val); - } - - return NULL; + return NULL; } void* FIFO_stress_consumer(void* arg) { - FIFO* f = reinterpret_cast *>(arg); - uint64_t i; - int it; - ElementType val; - bool ret; - int id = ++::cCount; + FIFO* f = reinterpret_cast*>(arg); + uint64_t i; + int it; + ElementType val; + bool ret; + int id = ++::cCount; - cout << "consumer " << id << " started" << endl; - it = f->getIterator(); + cout << "consumer " << id << " started" << endl; + it = f->getIterator(); - for (i = 0; i < MAXINT64; i++) + for (i = 0; i < MAXINT64; i++) + { + // cout << id << ": " << i << " "; + ret = f->next(it, &val); + + if (!ret) { -// cout << id << ": " << i << " "; - ret = f->next(it, &val); - - if (!ret) - { - cout << "consumer " << id << " exiting" << endl; - return NULL; - } - - else - { -// cout << "first: " << val.first << " second: " << val.second << endl; - CPPUNIT_ASSERT(ret == true); - CPPUNIT_ASSERT(val.first == i && val.second == i); - } - + cout << "consumer " << id << " exiting" << endl; + return NULL; } - return NULL; + else + { + // cout << "first: " << val.first << " second: " << val.second << endl; + CPPUNIT_ASSERT(ret == true); + CPPUNIT_ASSERT(val.first == i && val.second == i); + } + } + + return NULL; } void* FIFO_bench_consumer(void* arg) { - FIFO* f = reinterpret_cast *>(arg); - int i, it, val; - bool ret; + FIFO* f = reinterpret_cast*>(arg); + int i, it, val; + bool ret; - it = f->getIterator(); + it = f->getIterator(); - for (i = 0; i < ::count; i++) - { -// cout << "t " << i << endl; - ret = f->next(it, &val); -// CPPUNIT_ASSERT(ret == true); - } + for (i = 0; i < ::count; i++) + { + // cout << "t " << i << endl; + ret = f->next(it, &val); + // CPPUNIT_ASSERT(ret == true); + } - return NULL; + return NULL; } void* FIFO_2_helper(void* arg) { - FIFO* f = reinterpret_cast *>(arg); - int i; - DLElement fe; + FIFO* f = reinterpret_cast*>(arg); + int i; + DLElement fe; - for (i = 0; i < ::count; i++) - { - fe.i = i; + for (i = 0; i < ::count; i++) + { + fe.i = i; #ifdef DEBUG - cout << "inserting " << i << endl; + cout << "inserting " << i << endl; #endif - f->insert(fe); - } + f->insert(fe); + } - sleep(5); // causes the test to block on moreData until endOfInput is called - f->endOfInput(); - return NULL; + sleep(5); // causes the test to block on moreData until endOfInput is called + f->endOfInput(); + return NULL; } void* WSDL_2_helper(void* arg) { - WSDL* w = reinterpret_cast *>(arg); - uint32_t i, id; - bool nextRet; - ElementType dle; + WSDL* w = reinterpret_cast*>(arg); + uint32_t i, id; + bool nextRet; + ElementType dle; - id = w->getIterator(); + id = w->getIterator(); -// cout << "id = " << id << endl; - for (i = 0; i < ::count; i++) - { - nextRet = w->next(id, &dle); -// cout << "tgot " << dle.i << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i && dle.second == i); - } + // cout << "id = " << id << endl; + for (i = 0; i < ::count; i++) + { + nextRet = w->next(id, &dle); + // cout << "tgot " << dle.i << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i && dle.second == i); + } - return NULL; + return NULL; } void* BandedDL_1_helper(void* arg) { - BandedDL* w = reinterpret_cast *>(arg); - uint32_t i, id; - bool nextRet; - ElementType dle; + BandedDL* w = reinterpret_cast*>(arg); + uint32_t i, id; + bool nextRet; + ElementType dle; - id = w->getIterator(); + id = w->getIterator(); -// cout << "id = " << id << endl; - for (i = 0; i < ::count; i++) - { - nextRet = w->next(id, &dle); -// cout << "tgot " << dle.i << endl; -// CPPUNIT_ASSERT(nextRet == true); -// CPPUNIT_ASSERT(dle.first == i && dle.second == i); - } + // cout << "id = " << id << endl; + for (i = 0; i < ::count; i++) + { + nextRet = w->next(id, &dle); + // cout << "tgot " << dle.i << endl; + // CPPUNIT_ASSERT(nextRet == true); + // CPPUNIT_ASSERT(dle.first == i && dle.second == i); + } - return NULL; + return NULL; } void* BandedDL_2_helper(void* arg) { - BandedDL* w = reinterpret_cast *>(arg); - uint32_t i, id; - bool nextRet; - StringElementType dle; + BandedDL* w = reinterpret_cast*>(arg); + uint32_t i, id; + bool nextRet; + StringElementType dle; - id = w->getIterator(); + id = w->getIterator(); -// cout << "id = " << id << endl; - for (i = 0; i < ::count; i++) - { - ostringstream os; + // cout << "id = " << id << endl; + for (i = 0; i < ::count; i++) + { + ostringstream os; - os << "blah blah" << i; - nextRet = w->next(id, &dle); -// cout << "tgot " << dle.i << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i && dle.second == os.str()); - } + os << "blah blah" << i; + nextRet = w->next(id, &dle); + // cout << "tgot " << dle.i << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i && dle.second == os.str()); + } - return NULL; + return NULL; } void* WSDL_bench_helper(void* arg) { - WSDL* w = reinterpret_cast *>(arg); - int i, id; - bool nextRet; - ElementType e; + WSDL* w = reinterpret_cast*>(arg); + int i, id; + bool nextRet; + ElementType e; - id = w->getIterator(); + id = w->getIterator(); - for (i = 0; i < ::count; i++) - nextRet = w->next(id, &e); + for (i = 0; i < ::count; i++) + nextRet = w->next(id, &e); - return NULL; + return NULL; } void* SWSDL_bench_helper(void* arg) { - SWSDL* sw = reinterpret_cast *>(arg); - int i, id; - bool nextRet; - ElementType e; + SWSDL* sw = reinterpret_cast*>(arg); + int i, id; + bool nextRet; + ElementType e; - id = sw->getIterator(); + id = sw->getIterator(); - for (i = 0; i < ::count * 8; i++) - nextRet = sw->next(id, &e); + for (i = 0; i < ::count * 8; i++) + nextRet = sw->next(id, &e); - return NULL; + return NULL; } void* BDL_bench_helper(void* arg) { - BandedDL* sw = reinterpret_cast *>(arg); - int i, id; - bool nextRet; - ElementType e; + BandedDL* sw = reinterpret_cast*>(arg); + int i, id; + bool nextRet; + ElementType e; - id = sw->getIterator(); + id = sw->getIterator(); - for (i = 0; i < ::count * 8; i++) - nextRet = sw->next(id, &e); + for (i = 0; i < ::count * 8; i++) + nextRet = sw->next(id, &e); - return NULL; + return NULL; } void* SWSDL_producer(void* arg) { - SWSDL* sw = reinterpret_cast *>(arg); + SWSDL* sw = reinterpret_cast*>(arg); - for (int i = 0; i < ::count; i++) - sw->insert(ElementType(i, i)); + for (int i = 0; i < ::count; i++) + sw->insert(ElementType(i, i)); - return NULL; + return NULL; } void* BDL_producer(void* arg) { - BandedDL* sw = reinterpret_cast *>(arg); - ElementType dle; + BandedDL* sw = reinterpret_cast*>(arg); + ElementType dle; - for (int i = 0; i < ::count; i++) - { - //dle.first = dle.second = i; - sw->insert(ElementType(i, i)); - } + for (int i = 0; i < ::count; i++) + { + // dle.first = dle.second = i; + sw->insert(ElementType(i, i)); + } - //sw->endOfInput(); - //cout << "end of input size is " << sw->totalSize() << endl; - return NULL; + // sw->endOfInput(); + // cout << "end of input size is " << sw->totalSize() << endl; + return NULL; } void* ZDL_producer(void* arg) { - ZDL* zdl = reinterpret_cast *>(arg); + ZDL* zdl = reinterpret_cast*>(arg); - for (int i = 0; i < ::count; i++) - { - zdl.insert(ElementType((int) N * rand() / (RAND_MAX + 1.0), i)); - } + for (int i = 0; i < ::count; i++) + { + zdl.insert(ElementType((int)N * rand() / (RAND_MAX + 1.0), i)); + } - return NULL; + return NULL; } void* ZDL_consumer(void* arg) { - ZDL* zdl = reinterpret_cast *>(arg); - int i, id; - bool nextRet; - ElementType e; + ZDL* zdl = reinterpret_cast*>(arg); + int i, id; + bool nextRet; + ElementType e; - id = zdl->getIterator(); + id = zdl->getIterator(); + nextRet = zdl->next(id, &e); + + while (nextRet) nextRet = zdl->next(id, &e); - while (nextRet) - nextRet = zdl->next(id, &e); - - return NULL; + return NULL; } void* SWSDL_consumer(void* arg) { - SWSDL* sw = reinterpret_cast *>(arg); - int i, id; - bool nextRet; - ElementType e; + SWSDL* sw = reinterpret_cast*>(arg); + int i, id; + bool nextRet; + ElementType e; - id = sw->getIterator(); - nextRet = sw->next(id, &e); + id = sw->getIterator(); + nextRet = sw->next(id, &e); - while (nextRet) - nextRet = zdl->next(id, &e); + while (nextRet) + nextRet = zdl->next(id, &e); - return NULL; + return NULL; } class DataListDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DataListDriver); - CPPUNIT_TEST_SUITE(DataListDriver); + // CPPUNIT_TEST(FIFO_1); + // CPPUNIT_TEST(FIFO_2); + // CPPUNIT_TEST(MP_FIFO_stress); //10 consumers 10 producers + // CPPUNIT_TEST(FIFO_stress); // 10 consumers 1 producer; doesn't stop + // CPPUNIT_TEST(FIFO_bench); + // CPPUNIT_TEST(FIFO_singlethread_bench); + // CPPUNIT_TEST(WSDL_1); + // CPPUNIT_TEST(WSDL_2); + // CPPUNIT_TEST(WSDL_3); // WSDL_1 with StringElementTypes + // CPPUNIT_TEST(WSDL_bench); + // CPPUNIT_TEST(WSDL_singlethread_bench); + // CPPUNIT_TEST(CON_DL_1); // ConstantDataList + // CPPUNIT_TEST(BucketDL_1); -//CPPUNIT_TEST(FIFO_1); -//CPPUNIT_TEST(FIFO_2); -//CPPUNIT_TEST(MP_FIFO_stress); //10 consumers 10 producers -// CPPUNIT_TEST(FIFO_stress); // 10 consumers 1 producer; doesn't stop -// CPPUNIT_TEST(FIFO_bench); -// CPPUNIT_TEST(FIFO_singlethread_bench); -//CPPUNIT_TEST(WSDL_1); -//CPPUNIT_TEST(WSDL_2); -//CPPUNIT_TEST(WSDL_3); // WSDL_1 with StringElementTypes -// CPPUNIT_TEST(WSDL_bench); -// CPPUNIT_TEST(WSDL_singlethread_bench); -//CPPUNIT_TEST(CON_DL_1); // ConstantDataList -//CPPUNIT_TEST(BucketDL_1); + // CPPUNIT_TEST(BandedDL_1); // WSDL_2 ported to BandedDLs + // CPPUNIT_TEST(BandedDL_2); // BandedDL_1 that uses StringElementTypes -//CPPUNIT_TEST(BandedDL_1); // WSDL_2 ported to BandedDLs -//CPPUNIT_TEST(BandedDL_2); // BandedDL_1 that uses StringElementTypes + // make sure the DataList consumer-side interface works. + // CPPUNIT_TEST(BandedDL_as_WSDL_1); -// make sure the DataList consumer-side interface works. -//CPPUNIT_TEST(BandedDL_as_WSDL_1); + // random inserts to bucketDL, create bandedDL, make sure it's in order + // CPPUNIT_TEST(BandedDL_as_WSDL_2); -// random inserts to bucketDL, create bandedDL, make sure it's in order -//CPPUNIT_TEST(BandedDL_as_WSDL_2); + /* tests prompted by feedback */ + // CPPUNIT_TEST(polymorphism_1); - /* tests prompted by feedback */ -//CPPUNIT_TEST(polymorphism_1); + // make sure we can store basic types (int in this case) + // CPPUNIT_TEST(polymorphism_2); -// make sure we can store basic types (int in this case) -//CPPUNIT_TEST(polymorphism_2); + // make sure we can create a FIFO, fill it, cast it to a DataList, and read the elements. + // CPPUNIT_TEST(polymorphism_3); + // CPPUNIT_TEST(SWSDL_bench); + // CPPUNIT_TEST(BDL_multiproducer_bench); + // CPPUNIT_TEST(BDL_consumer_bench); + // CPPUNIT_TEST(SWSDL_multiproducer_bench); + CPPUNIT_TEST(SWSDL_consumer_bench); + // CPPUNIT_TEST(BDL_singlethread_bench); + CPPUNIT_TEST(ZDL_bench); + CPPUNIT_TEST_SUITE_END(); -// make sure we can create a FIFO, fill it, cast it to a DataList, and read the elements. -//CPPUNIT_TEST(polymorphism_3); -//CPPUNIT_TEST(SWSDL_bench); -//CPPUNIT_TEST(BDL_multiproducer_bench); -//CPPUNIT_TEST(BDL_consumer_bench); -//CPPUNIT_TEST(SWSDL_multiproducer_bench); - CPPUNIT_TEST(SWSDL_consumer_bench); -//CPPUNIT_TEST(BDL_singlethread_bench); - CPPUNIT_TEST(ZDL_bench); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void FIFO_1() + { + int i, it; + DLElement dummy; + FIFO f(1, 20); + bool nextRet; -private: -public: - - void FIFO_1() + for (i = 0; i < 10; i++) { - int i, it; - DLElement dummy; - FIFO f(1, 20); - bool nextRet; - - for (i = 0; i < 10; i++) - { - dummy.i = i; - f.insert(dummy); - } - - f.endOfInput(); - it = f.getIterator(); - CPPUNIT_ASSERT(it == 0); - - for (i = 0; i < 10; i++) - { - nextRet = f.next(it, &dummy); - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dummy.i == i); - } + dummy.i = i; + f.insert(dummy); } - void FIFO_2() + f.endOfInput(); + it = f.getIterator(); + CPPUNIT_ASSERT(it == 0); + + for (i = 0; i < 10; i++) { - int i, it; - DLElement fe; - FIFO fifo(1, 2000); - pthread_t thread; - bool ret; + nextRet = f.next(it, &dummy); + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dummy.i == i); + } + } - fe.i = 0; - pthread_create(&thread, NULL, FIFO_2_helper, &fifo); - sleep(1); // make sure the thread sleeps after 2000 inserts - it = fifo.getIterator(); + void FIFO_2() + { + int i, it; + DLElement fe; + FIFO fifo(1, 2000); + pthread_t thread; + bool ret; - for (i = 0; i < ::count; i++) - { - if (i % 100000 == 0) - cout << i << "/" << ::count << endl; + fe.i = 0; + pthread_create(&thread, NULL, FIFO_2_helper, &fifo); + sleep(1); // make sure the thread sleeps after 2000 inserts + it = fifo.getIterator(); - ret = fifo.next(it, &fe); -// cout << "fe = " << fe.i << endl; - CPPUNIT_ASSERT(ret == true); - CPPUNIT_ASSERT(fe.i == i); - } + for (i = 0; i < ::count; i++) + { + if (i % 100000 == 0) + cout << i << "/" << ::count << endl; - ret = fifo.next(it, &fe); // should sleep until the thread calls endOfInput() - CPPUNIT_ASSERT(ret == false); - pthread_join(thread, NULL); + ret = fifo.next(it, &fe); + // cout << "fe = " << fe.i << endl; + CPPUNIT_ASSERT(ret == true); + CPPUNIT_ASSERT(fe.i == i); } - void MP_FIFO_stress() + ret = fifo.next(it, &fe); // should sleep until the thread calls endOfInput() + CPPUNIT_ASSERT(ret == false); + pthread_join(thread, NULL); + } + + void MP_FIFO_stress() + { + int64_t i; + FIFO f(10, 25000); + pthread_t consumer[10]; + pthread_t producer[10]; + + f.setMultipleProducers(true); + + for (i = 0; i < 10; i++) + pthread_create(&consumer[i], NULL, MP_FIFO_stress_consumer, &f); + + for (i = 0; i < 10; i++) + pthread_create(&producer[i], NULL, MP_FIFO_stress_producer, &f); + + for (i = 0; i < 10; i++) + pthread_join(producer[i], NULL); + + f.endOfInput(); + + /* + // for (i = 0; i < MAXINT64; i++) { + for (i = 0; i < ::count; i++) { + if ((i % 10000000) == 0) + cout << i << endl; + f.insert(ElementType(i, i)); + } + f.endOfInput(); + */ + for (i = 0; i < 10; i++) + pthread_join(consumer[i], NULL); + } + + void FIFO_stress() + { + int64_t i; + FIFO f(10, 25000); + pthread_t consumer[10]; + + for (i = 0; i < 10; i++) + pthread_create(&consumer[i], NULL, FIFO_stress_consumer, &f); + + // for (i = 0; i < MAXINT64; i++) { + for (i = 0; i < ::count; i++) { - int64_t i; - FIFO f(10, 25000); - pthread_t consumer[10]; - pthread_t producer[10]; + if ((i % 10000000) == 0) + cout << i << endl; - f.setMultipleProducers(true); - - for (i = 0; i < 10; i++) - pthread_create(&consumer[i], NULL, MP_FIFO_stress_consumer, &f); - - for (i = 0; i < 10; i++) - pthread_create(&producer[i], NULL, MP_FIFO_stress_producer, &f); - - for (i = 0; i < 10; i++) - pthread_join(producer[i], NULL); - - f.endOfInput(); - - /* - // for (i = 0; i < MAXINT64; i++) { - for (i = 0; i < ::count; i++) { - if ((i % 10000000) == 0) - cout << i << endl; - f.insert(ElementType(i, i)); - } - f.endOfInput(); - */ - for (i = 0; i < 10; i++) - pthread_join(consumer[i], NULL); + f.insert(ElementType(i, i)); } - void FIFO_stress() + f.endOfInput(); + + for (i = 0; i < 10; i++) + pthread_join(consumer[i], NULL); + } + + void FIFO_bench() + { + int i; + FIFO f(1, ::maxElements); + pthread_t consumer; + struct timespec tv1, tv2, diff; + + pthread_create(&consumer, NULL, FIFO_bench_consumer, &f); + clock_gettime(CLOCK_REALTIME, &tv1); + + for (i = 0; i < ::count; i++) { - int64_t i; - FIFO f(10, 25000); - pthread_t consumer[10]; - - for (i = 0; i < 10; i++) - pthread_create(&consumer[i], NULL, FIFO_stress_consumer, &f); - -// for (i = 0; i < MAXINT64; i++) { - for (i = 0; i < ::count; i++) - { - if ((i % 10000000) == 0) - cout << i << endl; - - f.insert(ElementType(i, i)); - } - - f.endOfInput(); - - for (i = 0; i < 10; i++) - pthread_join(consumer[i], NULL); + f.insert(i); } - void FIFO_bench() + f.endOfInput(); + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &tv2); + timespec_sub(tv1, tv2, diff); + cout << "producer & consumer passed " << ::count << " elements in " << diff.tv_sec << "s " << diff.tv_nsec + << "ns" << endl; + } + + void FIFO_singlethread_bench() + { + int i, it, e; + FIFO f(1, ::count); + bool more; + struct timespec tv1, tv2, diff; + + clock_gettime(CLOCK_REALTIME, &tv1); + + for (i = 0; i < ::count; i++) { - int i; - FIFO f(1, ::maxElements); - pthread_t consumer; - struct timespec tv1, tv2, diff; - - - pthread_create(&consumer, NULL, FIFO_bench_consumer, &f); - clock_gettime(CLOCK_REALTIME, &tv1); - - for (i = 0; i < ::count; i++) - { - f.insert(i); - } - - f.endOfInput(); - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &tv2); - timespec_sub(tv1, tv2, diff); - cout << "producer & consumer passed " << ::count << " elements in " << - diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; - + f.insert(i); } - void FIFO_singlethread_bench() + f.endOfInput(); + clock_gettime(CLOCK_REALTIME, &tv2); + timespec_sub(tv1, tv2, diff); + + cout << "FIFO_singlethreaded_bench: inserted " << ::count << " ints in " << diff.tv_sec << "s " + << diff.tv_nsec << "ns" << endl; + + it = f.getIterator(); + + for (i = 0; i < ::count; i++) { - int i, it, e; - FIFO f(1, ::count); - bool more; - struct timespec tv1, tv2, diff; - - clock_gettime(CLOCK_REALTIME, &tv1); - - for (i = 0; i < ::count; i++) - { - f.insert(i); - } - - f.endOfInput(); - clock_gettime(CLOCK_REALTIME, &tv2); - timespec_sub(tv1, tv2, diff); - - cout << "FIFO_singlethreaded_bench: inserted " << ::count << - " ints in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; - - it = f.getIterator(); - - for (i = 0; i < ::count; i++) - { - more = f.next(it, &e); - } - - clock_gettime(CLOCK_REALTIME, &tv1); - timespec_sub(tv2, tv1, diff); - cout << "FIFO_singlethread_bench: consumed, took " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; + more = f.next(it, &e); } + clock_gettime(CLOCK_REALTIME, &tv1); + timespec_sub(tv2, tv1, diff); + cout << "FIFO_singlethread_bench: consumed, took " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } - void WSDL_1() + void WSDL_1() + { + WSDL w(1, ::count / 100); + ElementType dle; + uint32_t i, id; + bool nextRet; + + for (i = 0; i < ::count; i++) { - WSDL w(1, ::count / 100); - ElementType dle; - uint32_t i, id; - bool nextRet; - - for (i = 0; i < ::count; i++) - { -// cout << "inserting " << i << endl; - dle.first = dle.second = i; - w.insert(dle); - } - - w.endOfInput(); - id = w.getIterator(); - - for (i = 0; i < ::count; i++) - { - nextRet = w.next(id, &dle); -// cout << i << ":got " << dle.first << " " << dle.second << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i && dle.second == i); - } + // cout << "inserting " << i << endl; + dle.first = dle.second = i; + w.insert(dle); } - void WSDL_2() + w.endOfInput(); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) { - WSDL w(2, ::count / 10); - ElementType dle; - uint32_t i, id, size; - bool nextRet; - pthread_t consumer1; + nextRet = w.next(id, &dle); + // cout << i << ":got " << dle.first << " " << dle.second << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i && dle.second == i); + } + } - pthread_create(&consumer1, NULL, WSDL_2_helper, &w); + void WSDL_2() + { + WSDL w(2, ::count / 10); + ElementType dle; + uint32_t i, id, size; + bool nextRet; + pthread_t consumer1; - // thread should wait here waiting on endOfInput() + pthread_create(&consumer1, NULL, WSDL_2_helper, &w); - for (i = 0; i < ::count; i++) - { - dle.first = dle.second = i; - w.insert(dle); - } + // thread should wait here waiting on endOfInput() - sleep(1); - w.endOfInput(); - - /* let the thread consume the loaded set; make sure that the load - doesn't happen until the main thread finishes with the loaded set */ - - sleep(1); - size = w.totalSize(); - id = w.getIterator(); - - for (i = 0; i < ::count; i++) - { - nextRet = w.next(id, &dle); -// cout << "got " << dle.i << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i && dle.second == i); - } - - pthread_join(consumer1, NULL); + for (i = 0; i < ::count; i++) + { + dle.first = dle.second = i; + w.insert(dle); } - void WSDL_3() + sleep(1); + w.endOfInput(); + + /* let the thread consume the loaded set; make sure that the load + doesn't happen until the main thread finishes with the loaded set */ + + sleep(1); + size = w.totalSize(); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) { - WSDL w(1, ::count / 10); - StringElementType dle; - uint32_t i, id; - bool nextRet; - - for (i = 0; i < ::count; i++) - { - ostringstream os; - - os << "blah blah blah" << i; -// cout << "inserting " << i << endl; - dle.first = i; - dle.second = os.str(); - w.insert(dle); - } - - w.endOfInput(); - id = w.getIterator(); - - for (i = 0; i < ::count; i++) - { - ostringstream os; - - os << "blah blah blah" << i; - nextRet = w.next(id, &dle); -// cout << i << ":got " << dle.first << " " << dle.second << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i && dle.second == os.str()); - } + nextRet = w.next(id, &dle); + // cout << "got " << dle.i << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i && dle.second == i); } - void WSDL_bench() + pthread_join(consumer1, NULL); + } + + void WSDL_3() + { + WSDL w(1, ::count / 10); + StringElementType dle; + uint32_t i, id; + bool nextRet; + + for (i = 0; i < ::count; i++) { - WSDL w(1, ::maxElements); - int i; - pthread_t consumer; - struct timespec ts1, ts2, diff; + ostringstream os; - clock_gettime(CLOCK_REALTIME, &ts1); - pthread_create(&consumer, NULL, WSDL_bench_helper, &w); - - for (i = 0; i < ::count; i++) - w.insert(ElementType(i, i)); - - w.endOfInput(); - cout << "end of input size is " << w.totalSize() << endl; - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "WSDL_bench: producer & consumer passed " << ::count << " elements in " << - diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + os << "blah blah blah" << i; + // cout << "inserting " << i << endl; + dle.first = i; + dle.second = os.str(); + w.insert(dle); } - void WSDL_singlethread_bench() + w.endOfInput(); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) { - WSDL w(1, ::maxElements); - int i, id, e; - bool nextRet; - struct timespec ts1, ts2, diff; + ostringstream os; - clock_gettime(CLOCK_REALTIME, &ts1); + os << "blah blah blah" << i; + nextRet = w.next(id, &dle); + // cout << i << ":got " << dle.first << " " << dle.second << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i && dle.second == os.str()); + } + } - for (i = 0; i < ::count; i++) - w.insert(i); + void WSDL_bench() + { + WSDL w(1, ::maxElements); + int i; + pthread_t consumer; + struct timespec ts1, ts2, diff; - w.endOfInput(); - id = w.getIterator(); + clock_gettime(CLOCK_REALTIME, &ts1); + pthread_create(&consumer, NULL, WSDL_bench_helper, &w); - for (i = 0; i < ::count; i++) - nextRet = w.next(id, &e); + for (i = 0; i < ::count; i++) + w.insert(ElementType(i, i)); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "WSDL_singlethread_bench: producer & consumer passed " << - ::count << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; + w.endOfInput(); + cout << "end of input size is " << w.totalSize() << endl; + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "WSDL_bench: producer & consumer passed " << ::count << " elements in " << diff.tv_sec << "s " + << diff.tv_nsec << "ns" << endl; + } + + void WSDL_singlethread_bench() + { + WSDL w(1, ::maxElements); + int i, id, e; + bool nextRet; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < ::count; i++) + w.insert(i); + + w.endOfInput(); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) + nextRet = w.next(id, &e); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "WSDL_singlethread_bench: producer & consumer passed " << ::count << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + void SWSDL_bench() + { + /* + SWSDL sw(1, ::maxElements); + int i; + pthread_t consumer; + pthread_t producer; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + pthread_create(&producer, NULL, SWSDL_producer, &sw); + pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); + + pthread_join(producer, NULL); + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << + diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + */ + SWSDL sw(1, ::maxElements); + int i; + pthread_t consumer; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); + + for (i = 0; i < ::count; i++) + sw.insert(ElementType(i, i)); + + sw.endOfInput(); + cout << "end of input size is " << sw.totalSize() << endl; + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << diff.tv_sec << "s " + << diff.tv_nsec << "ns" << endl; + } + + void SWSDL_multiproducer_bench() + { + SWSDL sw(1, ::maxElements); + sw.setMultipleProducers(true); + int i, id; + ElementType e; + bool nextRet; + uint32_t numOfThreads = 8; + pthread_t producer[numOfThreads]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfThreads; i++) + pthread_create(&producer[i], NULL, SWSDL_producer, &sw); + + for (i = 0; i < numOfThreads; i++) + pthread_join(producer[i], NULL); + + sw.endOfInput(); + cout << "end of input size is " << sw.totalSize() << endl; + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "# of Producer: " << numOfThreads << endl; + cout << "SWSDL_producer_phase_bench: producer & consumer passed " << ::count * numOfThreads + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + clock_gettime(CLOCK_REALTIME, &ts1); + id = sw.getIterator(); + + for (i = 0; i < ::count; i++) + nextRet = sw.next(id, &e); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + cout << "SWSDL_consumer_phase_bench: producer & consumer passed " << ::count * numOfThreads + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + void SWSDL_consumer_bench() + { + SWSDL sw(4); + sw.setMultipleProducers(true); + ElementType dle; + uint32_t i, id; + bool nextRet; + uint32_t numOfProducers = 8; + uint32_t numOfConsumers = 4; + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + pthread_create(&producer[i], NULL, SWSDL_producer, &sw); + + for (i = 0; i < numOfConsumers; i++) + pthread_create(&consumer[i], NULL, SWSDL_consumer, &sw); + + for (i = 0; i < numOfThreads; i++) + pthread_join(producer[i], NULL); + + sw.endOfInput(); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + cout << "# of Producer: " << numOfThreads << endl; + cout << "SWSDL_consumer_bench: producer & consumer passed " << ::count * numOfThreads << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + void BDL_multiproducer_bench() + { + BandedDL w(2); + w.setMultipleProducers(true); + ElementType dle; + uint32_t i, id; + bool nextRet; + uint32_t numOfThreads = 8; + pthread_t producer[numOfThreads]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfThreads; i++) + pthread_create(&producer[i], NULL, BDL_producer, &w); + + for (i = 0; i < numOfThreads; i++) + pthread_join(producer[i], NULL); + + w.endOfInput(); + cout << "end of input size is " << w.totalSize() << endl; + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "# of Producer: " << numOfThreads << endl; + cout << "BDL_producer_phase_bench: producer & consumer passed " << ::count * numOfThreads + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + clock_gettime(CLOCK_REALTIME, &ts1); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) + { + nextRet = w.next(id, &dle); } - void SWSDL_bench() + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + cout << "BDL_consumer_phase_bench: producer & consumer passed " << ::count * numOfThreads + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + /* + SWSDL sw(1, ::maxElements); + int i; + pthread_t consumer; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); + for (i = 0; i < ::count; i++) + sw.insert(ElementType(i, i)); + + sw.endOfInput(); + cout << "end of input size is " << sw.totalSize() << endl; + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << + diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + */ + } + + void BDL_consumer_bench() + { + BandedDL w(2); + w.setMultipleProducers(true); + ElementType dle; + uint32_t i, id; + bool nextRet; + uint32_t numOfThreads = 8; + pthread_t producer[numOfThreads]; + pthread_t consumer; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfThreads; i++) + pthread_create(&producer[i], NULL, BDL_producer, &w); + + for (i = 0; i < numOfThreads; i++) + pthread_join(producer[i], NULL); + + w.endOfInput(); + // pthread_create(&consumer, NULL, BDL_bench_helper, &w); + // cout << "end of input size is " << w.totalSize() << endl; + // clock_gettime(CLOCK_REALTIME, &ts2); + // timespec_sub(ts1, ts2, diff); + // cout << "# of Producer: " << numOfThreads << endl; + // cout << "BDL_producer_phase_bench: producer & consumer passed " << + // ::count * numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << + // "ns" << endl; + + // clock_gettime(CLOCK_REALTIME, &ts1); + id = w.getIterator(); + + for (i = 0; i < ::count * 8; i++) { - /* - SWSDL sw(1, ::maxElements); - int i; - pthread_t consumer; - pthread_t producer; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - pthread_create(&producer, NULL, SWSDL_producer, &sw); - pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); - - pthread_join(producer, NULL); - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << - diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; - */ - SWSDL sw(1, ::maxElements); - int i; - pthread_t consumer; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); - - for (i = 0; i < ::count; i++) - sw.insert(ElementType(i, i)); - - sw.endOfInput(); - cout << "end of input size is " << sw.totalSize() << endl; - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << - diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + nextRet = w.next(id, &dle); } - void SWSDL_multiproducer_bench() + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "end of input size is " << w.totalSize() << endl; + cout << "BDL_consumer_phase_bench: producer & consumer passed " << ::count * numOfThreads + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + /* + SWSDL sw(1, ::maxElements); + int i; + pthread_t consumer; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); + for (i = 0; i < ::count; i++) + sw.insert(ElementType(i, i)); + + sw.endOfInput(); + cout << "end of input size is " << sw.totalSize() << endl; + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << + diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + */ + } + + void BDL_singlethread_bench() + { + /*SWSDL sw(1, ::maxElements); + int i; + pthread_t consumer; + pthread_t producer; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + pthread_create(&producer, NULL, SWSDL_producer, &sw); + pthread_join(producer, NULL); + pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); + + pthread_join(consumer, NULL); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff);*/ + SWSDL sw(1, ::maxElements); + int i, id; + ElementType e; + bool nextRet; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + cout << "start of input size is " << ::count << endl; + + for (i = 0; i < ::count; i++) + sw.insert(ElementType(i, i)); + + // cout << "end of input size is " << sw.totalSize() << endl; + // sw.endOfInput(); + // id = sw.getIterator(); + for (i = 0; i < ::count; i++) + nextRet = sw.next(id, &e); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + // cout << "SWSDL_consumer_phase_bench: producer & consumer passed " << + // ::count << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << + // "ns" << endl; + } + + void CON_DL_1() + { + DLElement d, d2; + int i = 0; + bool nextRet; + + d.i = 1; + + ConstantDataList c(d); + + nextRet = c.next(i, &d2); + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(d.i == 1 && d2.i == 1); + + d.i = 2; + c.insert(d); + nextRet = c.next(i, &d2); + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(d.i == 2 && d2.i == 2); + } + + void BucketDL_1() + { + BucketDL bdl(16, 1, ::count / 100); + ElementType dle; + uint32_t i, it, eCount = 0, buckets; + bool nextRet; + + for (i = 0; i < ::count; i++) { - SWSDL sw(1, ::maxElements); - sw.setMultipleProducers(true); - int i, id; - ElementType e; - bool nextRet; - uint32_t numOfThreads = 8; - pthread_t producer[numOfThreads]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfThreads; i++) - pthread_create(&producer[i], NULL, SWSDL_producer, &sw); - - for (i = 0; i < numOfThreads; i++) - pthread_join(producer[i], NULL); - - sw.endOfInput(); - cout << "end of input size is " << sw.totalSize() << endl; - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "# of Producer: " << numOfThreads << endl; - cout << "SWSDL_producer_phase_bench: producer & consumer passed " << - ::count* numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; - clock_gettime(CLOCK_REALTIME, &ts1); - id = sw.getIterator(); - - for (i = 0; i < ::count; i++) - nextRet = sw.next(id, &e); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - cout << "SWSDL_consumer_phase_bench: producer & consumer passed " << - ::count* numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; + // if (i % 1000000 == 0) cout << i << endl; + dle.first = dle.second = i; + bdl.insert(dle); } - void SWSDL_consumer_bench() + bdl.endOfInput(); + // cout << "inserted " << ::count << " elements" << endl; + + buckets = bdl.bucketCount(); + + for (i = 0; i < buckets; i++) { - SWSDL sw(4); - sw.setMultipleProducers(true); - ElementType dle; - uint32_t i, id; - bool nextRet; - uint32_t numOfProducers = 8; - uint32_t numOfConsumers = 4; - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - struct timespec ts1, ts2, diff; + it = bdl.getIterator(i); - clock_gettime(CLOCK_REALTIME, &ts1); + do + { + nextRet = bdl.next(i, it, &dle); - for (i = 0; i < numOfProducers; i++) - pthread_create(&producer[i], NULL, SWSDL_producer, &sw); - - for (i = 0; i < numOfConsumers; i++) - pthread_create(&consumer[i], NULL, SWSDL_consumer, &sw); - - for (i = 0; i < numOfThreads; i++) - pthread_join(producer[i], NULL); - - sw.endOfInput(); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - cout << "# of Producer: " << numOfThreads << endl; - cout << "SWSDL_consumer_bench: producer & consumer passed " << - ::count* numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; + if (nextRet) + { + CPPUNIT_ASSERT(dle.first >= 0 && dle.first <= (uint64_t)::count); + CPPUNIT_ASSERT(++eCount <= ::count); + } + } while (nextRet == true); } + CPPUNIT_ASSERT(eCount == ::count); + } - void BDL_multiproducer_bench() + void BandedDL_1() + { + BandedDL w(2); + ElementType dle; + uint32_t i, id, size; + bool nextRet; + pthread_t consumer1; + + pthread_create(&consumer1, NULL, BandedDL_1_helper, &w); + + // thread should wait here waiting on endOfInput() + + for (i = 0; i < ::count; i++) { - BandedDL w(2); - w.setMultipleProducers(true); - ElementType dle; - uint32_t i, id; - bool nextRet; - uint32_t numOfThreads = 8; - pthread_t producer[numOfThreads]; - struct timespec ts1, ts2, diff; + dle.first = dle.second = i; + w.insert(dle); - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfThreads; i++) - pthread_create(&producer[i], NULL, BDL_producer, &w); - - for (i = 0; i < numOfThreads; i++) - pthread_join(producer[i], NULL); - - w.endOfInput(); - cout << "end of input size is " << w.totalSize() << endl; - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "# of Producer: " << numOfThreads << endl; - cout << "BDL_producer_phase_bench: producer & consumer passed " << - ::count* numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; - - clock_gettime(CLOCK_REALTIME, &ts1); - id = w.getIterator(); - - for (i = 0; i < ::count; i++) - { - nextRet = w.next(id, &dle); - } - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - cout << "BDL_consumer_phase_bench: producer & consumer passed " << - ::count* numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; - /* - SWSDL sw(1, ::maxElements); - int i; - pthread_t consumer; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); - for (i = 0; i < ::count; i++) - sw.insert(ElementType(i, i)); - - sw.endOfInput(); - cout << "end of input size is " << sw.totalSize() << endl; - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << - diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; - */ + if (((i + 1) % (::count / 10)) == 0) + { + cout << "inserted " << i + 1 << "/" << ::count << endl; + w.saveBand(); + } } - void BDL_consumer_bench() + sleep(1); + w.endOfInput(); + + /* let the thread consume the loaded set; make sure that the load + doesn't happen until the main thread finishes with the loaded set */ + + sleep(5); + size = w.totalSize(); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) { - BandedDL w(2); - w.setMultipleProducers(true); - ElementType dle; - uint32_t i, id; - bool nextRet; - uint32_t numOfThreads = 8; - pthread_t producer[numOfThreads]; - pthread_t consumer; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfThreads; i++) - pthread_create(&producer[i], NULL, BDL_producer, &w); - - for (i = 0; i < numOfThreads; i++) - pthread_join(producer[i], NULL); - - w.endOfInput(); - //pthread_create(&consumer, NULL, BDL_bench_helper, &w); - //cout << "end of input size is " << w.totalSize() << endl; - //clock_gettime(CLOCK_REALTIME, &ts2); - //timespec_sub(ts1, ts2, diff); - //cout << "# of Producer: " << numOfThreads << endl; - //cout << "BDL_producer_phase_bench: producer & consumer passed " << - // ::count * numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - // "ns" << endl; - - //clock_gettime(CLOCK_REALTIME, &ts1); - id = w.getIterator(); - - for (i = 0; i < ::count * 8; i++) - { - nextRet = w.next(id, &dle); - } - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "end of input size is " << w.totalSize() << endl; - cout << "BDL_consumer_phase_bench: producer & consumer passed " << - ::count* numOfThreads << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; - /* - SWSDL sw(1, ::maxElements); - int i; - pthread_t consumer; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); - for (i = 0; i < ::count; i++) - sw.insert(ElementType(i, i)); - - sw.endOfInput(); - cout << "end of input size is " << sw.totalSize() << endl; - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "SWSDL_bench: producer & consumer passed " << ::count << " elements in " << - diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; - */ + nextRet = w.next(id, &dle); + // cout << "got " << dle.i << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i); } - void BDL_singlethread_bench() + pthread_join(consumer1, NULL); + } + + void BandedDL_2() + { + BandedDL w(2); + StringElementType dle; + uint32_t i, id, size; + bool nextRet; + pthread_t consumer1; + + pthread_create(&consumer1, NULL, BandedDL_2_helper, &w); + + // thread should wait here waiting on endOfInput() + + for (i = 0; i < ::count; i++) { - /*SWSDL sw(1, ::maxElements); - int i; - pthread_t consumer; - pthread_t producer; - struct timespec ts1, ts2, diff; + ostringstream os; - clock_gettime(CLOCK_REALTIME, &ts1); - pthread_create(&producer, NULL, SWSDL_producer, &sw); - pthread_join(producer, NULL); - pthread_create(&consumer, NULL, SWSDL_bench_helper, &sw); + os << "blah blah" << i; - pthread_join(consumer, NULL); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff);*/ - SWSDL sw(1, ::maxElements); - int i, id; - ElementType e; - bool nextRet; - struct timespec ts1, ts2, diff; + dle.first = i; + dle.second = os.str(); + w.insert(dle); - clock_gettime(CLOCK_REALTIME, &ts1); - cout << "start of input size is " << ::count << endl; - - for (i = 0; i < ::count; i++) - sw.insert(ElementType(i, i)); - - //cout << "end of input size is " << sw.totalSize() << endl; - //sw.endOfInput(); - //id = sw.getIterator(); - for (i = 0; i < ::count; i++) - nextRet = sw.next(id, &e); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - //cout << "SWSDL_consumer_phase_bench: producer & consumer passed " << - // ::count << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - // "ns" << endl; + if (((i + 1) % (::count / 10)) == 0) + { + cout << "inserted " << i + 1 << "/" << ::count << endl; + w.saveBand(); + } } - void CON_DL_1() + sleep(1); + w.endOfInput(); + + /* let the thread consume the loaded set; make sure that the load + doesn't happen until the main thread finishes with the loaded set */ + + cout << "endofInput finished\n"; + + sleep(5); + size = w.totalSize(); + id = w.getIterator(); + + for (i = 0; i < ::count; i++) { - DLElement d, d2; - int i = 0; - bool nextRet; + ostringstream os; - d.i = 1; - - ConstantDataList c(d); - - nextRet = c.next(i, &d2); - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(d.i == 1 && d2.i == 1); - - d.i = 2; - c.insert(d); - nextRet = c.next(i, &d2); - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(d.i == 2 && d2.i == 2); + os << "blah blah" << i; + nextRet = w.next(id, &dle); + // cout << "got " << dle.i << endl; + CPPUNIT_ASSERT(nextRet == true); + CPPUNIT_ASSERT(dle.first == i && dle.second == os.str()); } - void BucketDL_1() + pthread_join(consumer1, NULL); + } + + void BandedDL_as_WSDL_1() + { + BandedDL bdl(1); + ElementType e; + uint32_t i, it; + bool more; + + for (i = 1; i <= ::count; i++) { - BucketDL bdl(16, 1, ::count / 100); - ElementType dle; - uint32_t i, it, eCount = 0, buckets; - bool nextRet; + e.first = e.second = i; + bdl.insert(e); - for (i = 0; i < ::count; i++) - { -// if (i % 1000000 == 0) cout << i << endl; - dle.first = dle.second = i; - bdl.insert(dle); - } - - bdl.endOfInput(); -// cout << "inserted " << ::count << " elements" << endl; - - buckets = bdl.bucketCount(); - - for (i = 0; i < buckets; i++) - { - it = bdl.getIterator(i); - - do - { - nextRet = bdl.next(i, it, &dle); - - if (nextRet) - { - CPPUNIT_ASSERT(dle.first >= 0 && dle.first <= (uint64_t) ::count); - CPPUNIT_ASSERT(++eCount <= ::count); - } - } - while (nextRet == true); - } - - CPPUNIT_ASSERT(eCount == ::count); + if (i % (::count / 10) == 0) + bdl.saveBand(); } - void BandedDL_1() + bdl.endOfInput(); + + it = bdl.getIterator(); + + for (i = 1; i <= ::count; i++) { - BandedDL w(2); - ElementType dle; - uint32_t i, id, size; - bool nextRet; - pthread_t consumer1; - - pthread_create(&consumer1, NULL, BandedDL_1_helper, &w); - - // thread should wait here waiting on endOfInput() - - for (i = 0; i < ::count; i++) - { - dle.first = dle.second = i; - w.insert(dle); - - if (((i + 1) % (::count / 10)) == 0) - { - cout << "inserted " << i + 1 << "/" << ::count << endl; - w.saveBand(); - } - } - - sleep(1); - w.endOfInput(); - - /* let the thread consume the loaded set; make sure that the load - doesn't happen until the main thread finishes with the loaded set */ - - sleep(5); - size = w.totalSize(); - id = w.getIterator(); - - for (i = 0; i < ::count; i++) - { - nextRet = w.next(id, &dle); -// cout << "got " << dle.i << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i); - } - - pthread_join(consumer1, NULL); + more = bdl.next(it, &e); + CPPUNIT_ASSERT(more == true); + CPPUNIT_ASSERT(e.first == i && e.second == i); } - void BandedDL_2() + more = bdl.next(it, &e); + CPPUNIT_ASSERT(more == false); + + bdl.restart(); + + /* make sure it can be read again */ + for (i = 1; i <= ::count; i++) { - BandedDL w(2); - StringElementType dle; - uint32_t i, id, size; - bool nextRet; - pthread_t consumer1; + more = bdl.next(it, &e); - pthread_create(&consumer1, NULL, BandedDL_2_helper, &w); + if (!more) + cerr << i << endl; - // thread should wait here waiting on endOfInput() - - for (i = 0; i < ::count; i++) - { - ostringstream os; - - os << "blah blah" << i; - - dle.first = i; - dle.second = os.str(); - w.insert(dle); - - if (((i + 1) % (::count / 10)) == 0) - { - cout << "inserted " << i + 1 << "/" << ::count << endl; - w.saveBand(); - } - } - - sleep(1); - w.endOfInput(); - - /* let the thread consume the loaded set; make sure that the load - doesn't happen until the main thread finishes with the loaded set */ - - cout << "endofInput finished\n"; - - sleep(5); - size = w.totalSize(); - id = w.getIterator(); - - for (i = 0; i < ::count; i++) - { - ostringstream os; - - os << "blah blah" << i; - nextRet = w.next(id, &dle); -// cout << "got " << dle.i << endl; - CPPUNIT_ASSERT(nextRet == true); - CPPUNIT_ASSERT(dle.first == i && dle.second == os.str()); - } - - pthread_join(consumer1, NULL); + CPPUNIT_ASSERT(more == true); + CPPUNIT_ASSERT(e.first == i && e.second == i); } - void BandedDL_as_WSDL_1() + more = bdl.next(it, &e); + CPPUNIT_ASSERT(more == false); + } + + void BandedDL_as_WSDL_2() + { + BucketDL bucketDL(::buckets, 1, 1000, BandGrouper()); + uint32_t last, i, it; + ElementType dle; + bool more; + + srand(time(NULL)); + + for (i = 0; i < ::count; i++) { - BandedDL bdl(1); - ElementType e; - uint32_t i, it; - bool more; - - for (i = 1; i <= ::count; i++) - { - e.first = e.second = i; - bdl.insert(e); - - if (i % (::count / 10) == 0) - bdl.saveBand(); - } - - bdl.endOfInput(); - - it = bdl.getIterator(); - - for (i = 1; i <= ::count; i++) - { - more = bdl.next(it, &e); - CPPUNIT_ASSERT(more == true); - CPPUNIT_ASSERT(e.first == i && e.second == i); - } - - more = bdl.next(it, &e); - CPPUNIT_ASSERT(more == false); - - bdl.restart(); - - /* make sure it can be read again */ - for (i = 1; i <= ::count; i++) - { - more = bdl.next(it, &e); - - if (!more) - cerr << i << endl; - - CPPUNIT_ASSERT(more == true); - CPPUNIT_ASSERT(e.first == i && e.second == i); - } - - more = bdl.next(it, &e); - CPPUNIT_ASSERT(more == false); + dle.first = dle.second = rand() % ::count; // artificial max + bucketDL.insert(dle); } - void BandedDL_as_WSDL_2() + bucketDL.endOfInput(); + + // cout << "made bucketdl" << endl; + + BandedDL bdl(bucketDL, 1); + + // cout << "made bandeddl" << endl; + + it = bdl.getIterator(); + last = 0; + // duplicates were removed, there won't be ::count elements in bdl + more = bdl.next(it, &dle); + + while (more) { - BucketDL bucketDL(::buckets, 1, 1000, BandGrouper()); - uint32_t last, i, it; - ElementType dle; - bool more; - - srand(time(NULL)); - - for (i = 0; i < ::count; i++) - { - dle.first = dle.second = rand() % ::count; // artificial max - bucketDL.insert(dle); - } - - bucketDL.endOfInput(); - -// cout << "made bucketdl" << endl; - - BandedDL bdl(bucketDL, 1); - -// cout << "made bandeddl" << endl; - - it = bdl.getIterator(); - last = 0; - // duplicates were removed, there won't be ::count elements in bdl - more = bdl.next(it, &dle); - - while (more) - { - CPPUNIT_ASSERT(dle.first < (uint64_t) ::count); -// cout << dle.first << " >= " << last << endl; - CPPUNIT_ASSERT(dle.first >= last); - last = dle.first; - more = bdl.next(it, &dle); - } - -// cerr << "read through bandeddl" << endl; - - more = bdl.next(it, &dle); - CPPUNIT_ASSERT(more == false); - + CPPUNIT_ASSERT(dle.first < (uint64_t)::count); + // cout << dle.first << " >= " << last << endl; + CPPUNIT_ASSERT(dle.first >= last); + last = dle.first; + more = bdl.next(it, &dle); } - void ZDL_bench() + // cerr << "read through bandeddl" << endl; + + more = bdl.next(it, &dle); + CPPUNIT_ASSERT(more == false); + } + + void ZDL_bench() + { + ZDL zdl(4); + zdl.setMultipleProducers(true); + ElementType dle; + uint32_t i, id; + bool nextRet; + uint32_t numOfProducers = 8; + uint32_t numOfConsumers = 4; + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + pthread_create(&producer[i], NULL, ZDL_producer, &zdl); + + for (i = 0; i < numOfConsumers; i++) + pthread_create(&consumer[i], NULL, ZDL_consumer, &zdl); + + for (i = 0; i < numOfThreads; i++) + pthread_join(producer[i], NULL); + + zdl.endOfInput(); + + // clock_gettime(CLOCK_REALTIME, &ts1); + id = w.getIterator(); + + for (i = 0; i < ::count * 8; i++) { - ZDL zdl(4); - zdl.setMultipleProducers(true); - ElementType dle; - uint32_t i, id; - bool nextRet; - uint32_t numOfProducers = 8; - uint32_t numOfConsumers = 4; - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - pthread_create(&producer[i], NULL, ZDL_producer, &zdl); - - for (i = 0; i < numOfConsumers; i++) - pthread_create(&consumer[i], NULL, ZDL_consumer, &zdl); - - for (i = 0; i < numOfThreads; i++) - pthread_join(producer[i], NULL); - - zdl.endOfInput(); - - //clock_gettime(CLOCK_REALTIME, &ts1); - id = w.getIterator(); - - for (i = 0; i < ::count * 8; i++) - { - nextRet = w.next(id, &dle); - } - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "end of input size is " << w.totalSize() << endl; - cout << "ZDL_consumer_producer_bench: producer & consumer passed " << - ::count* numOfProducers << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << - "ns" << endl; - + nextRet = w.next(id, &dle); } - /* Sanity test */ - void polymorphism_1() + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "end of input size is " << w.totalSize() << endl; + cout << "ZDL_consumer_producer_bench: producer & consumer passed " << ::count * numOfProducers + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + /* Sanity test */ + void polymorphism_1() + { + DataList* dl; + FIFO* f; + DLElement dle; + int it, i; + bool more; + + f = new FIFO(1, 20); + dl = (DataList*)f; + + it = dl->getIterator(); + + for (i = 0; i < 10; i++) { - DataList* dl; - FIFO* f; - DLElement dle; - int it, i; - bool more; - - f = new FIFO(1, 20); - dl = (DataList*) f; - - it = dl->getIterator(); - - for (i = 0; i < 10; i++) - { - dle.i = i; - dl->insert(dle); - } - - dl->endOfInput(); - - for (i = 0; i < 10; i++) - { - more = dl->next(it, &dle); -// cout << dle.i << endl; - CPPUNIT_ASSERT(more == true); - CPPUNIT_ASSERT(dle.i == i); - } - - delete f; + dle.i = i; + dl->insert(dle); } - /* Sanity test 2. Can we stuff basic datatypes into DataLists? Apparently not. WHY?*/ - void polymorphism_2() + dl->endOfInput(); + + for (i = 0; i < 10; i++) { - DataList* dl; - FIFO* f = new FIFO(1, 20); - int it, i, dle; - bool more; - - dl = (DataList*) f; - - it = dl->getIterator(); - - for (i = 0; i < 10; i++) - dl->insert(i); - - dl->endOfInput(); - - for (i = 0; i < 10; i++) - { - more = dl->next(it, &dle); - CPPUNIT_ASSERT(more == true); - CPPUNIT_ASSERT(dle == i); -// cout << "dle[" << i << "]: " << dle << endl; - } - - delete f; + more = dl->next(it, &dle); + // cout << dle.i << endl; + CPPUNIT_ASSERT(more == true); + CPPUNIT_ASSERT(dle.i == i); } - void polymorphism_3() + delete f; + } + + /* Sanity test 2. Can we stuff basic datatypes into DataLists? Apparently not. WHY?*/ + void polymorphism_2() + { + DataList* dl; + FIFO* f = new FIFO(1, 20); + int it, i, dle; + bool more; + + dl = (DataList*)f; + + it = dl->getIterator(); + + for (i = 0; i < 10; i++) + dl->insert(i); + + dl->endOfInput(); + + for (i = 0; i < 10; i++) { - DataList* dl; - FIFO* f; - int it, i; - DLElement dle; - bool more; - - f = new FIFO(1, 20); - dl = (DataList*) f; - - it = dl->getIterator(); - - for (i = 0; i < 10; i++) - { - dle.i = i; - f->insert(dle); - } - - f->endOfInput(); - - for (i = 0; i < 10; i++) - { -// cout << "next" << endl; - more = dl->next(it, &dle); - CPPUNIT_ASSERT(more == true); - CPPUNIT_ASSERT(dle.i == i); -// cout << "dle[" << i << "]: " << dle.i << endl; - } - - delete f; + more = dl->next(it, &dle); + CPPUNIT_ASSERT(more == true); + CPPUNIT_ASSERT(dle == i); + // cout << "dle[" << i << "]: " << dle << endl; } + delete f; + } + void polymorphism_3() + { + DataList* dl; + FIFO* f; + int it, i; + DLElement dle; + bool more; + f = new FIFO(1, 20); + dl = (DataList*)f; + it = dl->getIterator(); + for (i = 0; i < 10; i++) + { + dle.i = i; + f->insert(dle); + } + f->endOfInput(); + + for (i = 0; i < 10; i++) + { + // cout << "next" << endl; + more = dl->next(it, &dle); + CPPUNIT_ASSERT(more == true); + CPPUNIT_ASSERT(dle.i == i); + // cout << "dle[" << i << "]: " << dle.i << endl; + } + + delete f; + } }; CPPUNIT_TEST_SUITE_REGISTRATION(DataListDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/joblist/tdriver-dec.cpp b/dbcon/joblist/tdriver-dec.cpp index b766b05f2..efeaab4d7 100644 --- a/dbcon/joblist/tdriver-dec.cpp +++ b/dbcon/joblist/tdriver-dec.cpp @@ -34,180 +34,184 @@ using namespace joblist; class TestDistributedEngineComm { -public: - TestDistributedEngineComm(DistributedEngineComm* dec) : fDec(dec) { } - void addDataToOutput(const ByteStream& bs) - { - fDec->addDataToOutput(bs); - } + public: + TestDistributedEngineComm(DistributedEngineComm* dec) : fDec(dec) + { + } + void addDataToOutput(const ByteStream& bs) + { + fDec->addDataToOutput(bs); + } -private: - DistributedEngineComm* fDec; + private: + DistributedEngineComm* fDec; }; namespace { - const ByteStream buildBs(Int16 sessionId, Int16 stepId) { - uint32_t len = sizeof(ISMPacketHeader) + 2 * sizeof(PrimitiveHeader); - ByteStream::byte* bpr = new ByteStream::byte[len]; + uint32_t len = sizeof(ISMPacketHeader) + 2 * sizeof(PrimitiveHeader); + ByteStream::byte* bpr = new ByteStream::byte[len]; - ISMPacketHeader* hdr = reinterpret_cast(bpr); - PrimitiveHeader* p = reinterpret_cast(hdr + 1); + ISMPacketHeader* hdr = reinterpret_cast(bpr); + PrimitiveHeader* p = reinterpret_cast(hdr + 1); - p->SessionID = sessionId; - p->StepID = stepId; + p->SessionID = sessionId; + p->StepID = stepId; - ByteStream bs(bpr, len); - delete [] bpr; - return bs; + ByteStream bs(bpr, len); + delete[] bpr; + return bs; } void readBs(const ByteStream& bs, Int16& sessionId, Int16& stepId) { - const ISMPacketHeader* hdr = reinterpret_cast(bs.buf()); - const PrimitiveHeader* p = reinterpret_cast(hdr + 1); - sessionId = p->SessionID; - stepId = p->StepID; - return; + const ISMPacketHeader* hdr = reinterpret_cast(bs.buf()); + const PrimitiveHeader* p = reinterpret_cast(hdr + 1); + sessionId = p->SessionID; + stepId = p->StepID; + return; } class ThdFun1 { -public: - ThdFun1(DistributedEngineComm* dec, int sessionId, int stepId) : - fDec(dec), fSessionId(sessionId), fStepId(stepId) { } - void operator()() - { - ByteStream bs = fDec->read(fSessionId, fStepId); - idbassert(bs.length() == 0); - return; - } -private: - DistributedEngineComm* fDec; - int fSessionId; - int fStepId; + public: + ThdFun1(DistributedEngineComm* dec, int sessionId, int stepId) + : fDec(dec), fSessionId(sessionId), fStepId(stepId) + { + } + void operator()() + { + ByteStream bs = fDec->read(fSessionId, fStepId); + idbassert(bs.length() == 0); + return; + } + + private: + DistributedEngineComm* fDec; + int fSessionId; + int fStepId; }; -} +} // namespace int main(int argc, char** argv) { - int leakCheck = 0; + int leakCheck = 0; - if (argc > 1 && strcmp(argv[1], "--leakcheck") == 0) leakCheck = 1; + if (argc > 1 && strcmp(argv[1], "--leakcheck") == 0) + leakCheck = 1; - DistributedEngineComm* dec; + DistributedEngineComm* dec; - dec = DistributedEngineComm::instance("./config-dec.xml"); + dec = DistributedEngineComm::instance("./config-dec.xml"); - dec->addSession(12345); - dec->addStep(12345, 0); - dec->addStep(12345, 1); - dec->addStep(12345, 3); - dec->addStep(12345, 10); + dec->addSession(12345); + dec->addStep(12345, 0); + dec->addStep(12345, 1); + dec->addStep(12345, 3); + dec->addStep(12345, 10); - TestDistributedEngineComm tdec(dec); - ByteStream bs; + TestDistributedEngineComm tdec(dec); + ByteStream bs; - tdec.addDataToOutput(buildBs(12345, 0)); - tdec.addDataToOutput(buildBs(12345, 1)); - tdec.addDataToOutput(buildBs(12345, 3)); - tdec.addDataToOutput(buildBs(12345, 10)); + tdec.addDataToOutput(buildBs(12345, 0)); + tdec.addDataToOutput(buildBs(12345, 1)); + tdec.addDataToOutput(buildBs(12345, 3)); + tdec.addDataToOutput(buildBs(12345, 10)); - Int16 sessionId, stepId; - bs = dec->read(12345, 10); - readBs(bs, sessionId, stepId); - idbassert(sessionId == 12345); - idbassert(stepId == 10); + Int16 sessionId, stepId; + bs = dec->read(12345, 10); + readBs(bs, sessionId, stepId); + idbassert(sessionId == 12345); + idbassert(stepId == 10); + bs = dec->read(12345, 1); + readBs(bs, sessionId, stepId); + idbassert(sessionId == 12345); + idbassert(stepId == 1); + + bs = dec->read(12345, 0); + readBs(bs, sessionId, stepId); + idbassert(sessionId == 12345); + idbassert(stepId == 0); + + bs = dec->read(12345, 3); + readBs(bs, sessionId, stepId); + idbassert(sessionId == 12345); + idbassert(stepId == 3); + + unsigned i; + bs = buildBs(12345, 1); + // 1M seems a bit too much for a dev box + // 500K is about the max + const unsigned loopMax = 200000 / (leakCheck * 99 + 1); + + for (i = 0; i < loopMax; i++) + { + tdec.addDataToOutput(bs); + } + + for (i = 0; i < loopMax; i++) + { bs = dec->read(12345, 1); readBs(bs, sessionId, stepId); idbassert(sessionId == 12345); idbassert(stepId == 1); + } + + unsigned throws; + throws = 0; + + for (i = 0; i < loopMax; i++) + { + bs = buildBs(12345, (i % 10)); + + // some of these shoud throw since there's only a few steps added + try + { + tdec.addDataToOutput(bs); + } + catch (runtime_error& re) + { + throws++; + continue; + } + } + + idbassert(throws > 0); + + throws = 0; + + for (i = 0; i < loopMax; i++) + { + // some of these shoud throw since there's only a few steps added + try + { + bs = dec->read(12345, (i % 10)); + } + catch (runtime_error& re) + { + throws++; + continue; + } - bs = dec->read(12345, 0); readBs(bs, sessionId, stepId); idbassert(sessionId == 12345); - idbassert(stepId == 0); + idbassert(stepId == (i % 10)); + } - bs = dec->read(12345, 3); - readBs(bs, sessionId, stepId); - idbassert(sessionId == 12345); - idbassert(stepId == 3); + idbassert(throws > 0); - unsigned i; - bs = buildBs(12345, 1); - // 1M seems a bit too much for a dev box - // 500K is about the max - const unsigned loopMax = 200000 / (leakCheck * 99 + 1); + ThdFun1 fun1(dec, 12345, 1); + thread thd1(fun1); - for (i = 0; i < loopMax; i++) - { - tdec.addDataToOutput(bs); - } + dec->removeSession(12345); - for (i = 0; i < loopMax; i++) - { - bs = dec->read(12345, 1); - readBs(bs, sessionId, stepId); - idbassert(sessionId == 12345); - idbassert(stepId == 1); - } + thd1.join(); - unsigned throws; - throws = 0; + // delete dec; - for (i = 0; i < loopMax; i++) - { - bs = buildBs(12345, (i % 10)); - - //some of these shoud throw since there's only a few steps added - try - { - tdec.addDataToOutput(bs); - } - catch (runtime_error& re) - { - throws++; - continue; - } - } - - idbassert(throws > 0); - - throws = 0; - - for (i = 0; i < loopMax; i++) - { - //some of these shoud throw since there's only a few steps added - try - { - bs = dec->read(12345, (i % 10)); - } - catch (runtime_error& re) - { - throws++; - continue; - } - - readBs(bs, sessionId, stepId); - idbassert(sessionId == 12345); - idbassert(stepId == (i % 10)); - } - - idbassert(throws > 0); - - ThdFun1 fun1(dec, 12345, 1); - thread thd1(fun1); - - dec->removeSession(12345); - - thd1.join(); - - //delete dec; - - return 0; + return 0; } - diff --git a/dbcon/joblist/tdriver-deliver.cpp b/dbcon/joblist/tdriver-deliver.cpp index ceb3d09b6..4ad1ecc9f 100644 --- a/dbcon/joblist/tdriver-deliver.cpp +++ b/dbcon/joblist/tdriver-deliver.cpp @@ -44,18 +44,18 @@ using namespace std; using namespace joblist; using namespace execplan; +string schema("tpch100"); // dmc +string table("lineitem"); // create table d_step(c1 number); +string strtable("d_strstep"); // create table d_strstep(c1 char(10)); +string colstable("lineitem"); // create table d_colsstep(c1 number, c2 number, c3number, c4 number, c5 + // number, c6 number, c7 number, c8 number, c9 number,); -string schema("tpch100");//dmc -string table("lineitem"); //create table d_step(c1 number); -string strtable("d_strstep"); //create table d_strstep(c1 char(10)); -string colstable("lineitem"); //create table d_colsstep(c1 number, c2 number, c3number, c4 number, c5 number, c6 number, c7 number, c8 number, c9 number,); - -int dataSize = 100 * 1000 * 1000; // 8 MB -int bandSize = 8192 * 8; // 8 K +int dataSize = 100 * 1000 * 1000; // 8 MB +int bandSize = 8192 * 8; // 8 K int fifoSize = 128; int startingOid = 3416; // Oid for lineitem.l_orderkey on your database. // uint32_t flushInterval = 16384; // interval used in flushing table bands -uint32_t flushInterval = 16384; // interval used in flushing table bands +uint32_t flushInterval = 16384; // interval used in flushing table bands uint32_t columns; Stopwatch timer; @@ -69,84 +69,77 @@ Stopwatch timer; namespace { - class BSQueueMgr { -public: - /** @brief BSQueueMgr constructor. - * - * @param joblist (in) JobList where table bands reside - * @param tableOID (in) OID of the table to be projected - * @param maxQueueSize (in) Max # of table bands to be queued up - */ - explicit BSQueueMgr( - DeliveryStep* d, - int maxQueueSize = 1); + public: + /** @brief BSQueueMgr constructor. + * + * @param joblist (in) JobList where table bands reside + * @param tableOID (in) OID of the table to be projected + * @param maxQueueSize (in) Max # of table bands to be queued up + */ + explicit BSQueueMgr(DeliveryStep* d, int maxQueueSize = 1); - /** @brief TableBandQueueMgr destructor. - * - */ - ~BSQueueMgr( ); + /** @brief TableBandQueueMgr destructor. + * + */ + ~BSQueueMgr(); - /** @brief Main processing loop that projects the table bands. - * - */ - void project(); + /** @brief Main processing loop that projects the table bands. + * + */ + void project(); - /** @brief Called by second thread to acquire projected table band - * - * Acquires the next table band that has been projected, so that - * the calling thread can serialize and forward the table band to - * the next step. - * - * @return Next projected table band. Row count of 0 marks end of data - */ - messageqcpp::ByteStream* getNextByteStream(uint32_t& rowCount); + /** @brief Called by second thread to acquire projected table band + * + * Acquires the next table band that has been projected, so that + * the calling thread can serialize and forward the table band to + * the next step. + * + * @return Next projected table band. Row count of 0 marks end of data + */ + messageqcpp::ByteStream* getNextByteStream(uint32_t& rowCount); -private: - //Disable these by declaring but not defining - BSQueueMgr (const BSQueueMgr& rhs); - BSQueueMgr& operator=(const BSQueueMgr& rhs); + private: + // Disable these by declaring but not defining + BSQueueMgr(const BSQueueMgr& rhs); + BSQueueMgr& operator=(const BSQueueMgr& rhs); - struct QueueElement - { - messageqcpp::ByteStream* bs; - uint32_t rowCount; - }; + struct QueueElement + { + messageqcpp::ByteStream* bs; + uint32_t rowCount; + }; - DeliveryStep* ds; - unsigned int fMaxQueueSize; + DeliveryStep* ds; + unsigned int fMaxQueueSize; - //...Internal queue used to save table bands as they are projected. - std::queue fBSQueue; + //...Internal queue used to save table bands as they are projected. + std::queue fBSQueue; - //...Mutex to protect our internal table band queue, and the - //...condition variables used to signal when a table band has - //...been added to, or removed from the queue. - pthread_mutex_t fMutex; - pthread_cond_t fBSAdded; - pthread_cond_t fBSRemoved; + //...Mutex to protect our internal table band queue, and the + //...condition variables used to signal when a table band has + //...been added to, or removed from the queue. + pthread_mutex_t fMutex; + pthread_cond_t fBSAdded; + pthread_cond_t fBSRemoved; }; -BSQueueMgr::BSQueueMgr ( - DeliveryStep* d, - int maxQueueSize ) : - ds (d), - fMaxQueueSize(maxQueueSize) +BSQueueMgr::BSQueueMgr(DeliveryStep* d, int maxQueueSize) : ds(d), fMaxQueueSize(maxQueueSize) { - pthread_mutex_init ( &fMutex, 0 ); - pthread_cond_init ( &fBSAdded, 0 ); - pthread_cond_init ( &fBSRemoved, 0 ); + pthread_mutex_init(&fMutex, 0); + pthread_cond_init(&fBSAdded, 0); + pthread_cond_init(&fBSRemoved, 0); } //------------------------------------------------------------------------------ // BSQueueMgr destructor //------------------------------------------------------------------------------ -BSQueueMgr::~BSQueueMgr ( ) +BSQueueMgr::~BSQueueMgr() { - pthread_mutex_destroy( &fMutex); - pthread_cond_destroy ( &fBSAdded ); - pthread_cond_destroy ( &fBSRemoved ); + pthread_mutex_destroy(&fMutex); + pthread_cond_destroy(&fBSAdded); + pthread_cond_destroy(&fBSRemoved); } //------------------------------------------------------------------------------ @@ -156,40 +149,40 @@ BSQueueMgr::~BSQueueMgr ( ) // to continue adding projected table bands, until we reach a table band with // a row count of 0, which denotes the end of the table. //------------------------------------------------------------------------------ -void BSQueueMgr::project ( ) +void BSQueueMgr::project() { - bool moreData = true; + bool moreData = true; - //...Stay in loop to project table bands, until we reach a table band - //...having a row count of 0. + //...Stay in loop to project table bands, until we reach a table band + //...having a row count of 0. - while ( moreData ) + while (moreData) + { + uint32_t rowCount; + QueueElement qe; + + qe.bs = new ByteStream; + + rowCount = ds->nextBand(*(qe.bs)); + qe.rowCount = rowCount; + + pthread_mutex_lock(&fMutex); + + //...Wait for room in our queue before adding this table band + while (fBSQueue.size() >= fMaxQueueSize) { - uint32_t rowCount; - QueueElement qe; - - qe.bs = new ByteStream; - - rowCount = ds->nextBand(*(qe.bs)); - qe.rowCount = rowCount; - - pthread_mutex_lock( &fMutex ); - - //...Wait for room in our queue before adding this table band - while (fBSQueue.size() >= fMaxQueueSize ) - { - pthread_cond_wait( &fBSRemoved, &fMutex ); - } - - fBSQueue.push( qe ); - - if ( rowCount == 0 ) - moreData = false; - - pthread_cond_broadcast( &fBSAdded ); - - pthread_mutex_unlock( &fMutex ); + pthread_cond_wait(&fBSRemoved, &fMutex); } + + fBSQueue.push(qe); + + if (rowCount == 0) + moreData = false; + + pthread_cond_broadcast(&fBSAdded); + + pthread_mutex_unlock(&fMutex); + } } //------------------------------------------------------------------------------ @@ -199,40 +192,40 @@ void BSQueueMgr::project ( ) //------------------------------------------------------------------------------ ByteStream* BSQueueMgr::getNextByteStream(uint32_t& rowCount) { - QueueElement qe; + QueueElement qe; - pthread_mutex_lock( &fMutex ); + pthread_mutex_lock(&fMutex); - //...Wait for a table band to be added to our queue if the queue is empty - while ( fBSQueue.size() < 1 ) - { - pthread_cond_wait( &fBSAdded, &fMutex ); - } + //...Wait for a table band to be added to our queue if the queue is empty + while (fBSQueue.size() < 1) + { + pthread_cond_wait(&fBSAdded, &fMutex); + } - qe = fBSQueue.front( ); - fBSQueue.pop( ); - pthread_cond_broadcast(&fBSRemoved); + qe = fBSQueue.front(); + fBSQueue.pop(); + pthread_cond_broadcast(&fBSRemoved); - pthread_mutex_unlock( &fMutex ); + pthread_mutex_unlock(&fMutex); - rowCount = qe.rowCount; - return qe.bs; + rowCount = qe.rowCount; + return qe.bs; } struct BSProjectThread { - BSProjectThread ( BSQueueMgr* pMgr ) : - fBSQueueMgr ( pMgr ) { } + BSProjectThread(BSQueueMgr* pMgr) : fBSQueueMgr(pMgr) + { + } - BSQueueMgr* fBSQueueMgr; + BSQueueMgr* fBSQueueMgr; - void operator()() - { - fBSQueueMgr->project(); - } + void operator()() + { + fBSQueueMgr->project(); + } }; - // // A class that manages projection of table bands for a specific table // @@ -244,61 +237,58 @@ struct BSProjectThread // class TableBandQueueMgr { -public: - // - // TableBandQueueMgr constructor. - // - // pDStep (in) Delivery step where table bands reside - // maxQueueSize (in) Max # of table bands to be queued up - // - explicit TableBandQueueMgr( - DeliveryStep* pDStep, - int maxQueueSize = 1) : - fDStep (pDStep), - fMaxQueueSize(maxQueueSize) - { - pthread_mutex_init ( &fMutex, 0 ); - pthread_cond_init ( &fTableBandAdded, 0 ); - pthread_cond_init ( &fTableBandRemoved, 0 ); - } + public: + // + // TableBandQueueMgr constructor. + // + // pDStep (in) Delivery step where table bands reside + // maxQueueSize (in) Max # of table bands to be queued up + // + explicit TableBandQueueMgr(DeliveryStep* pDStep, int maxQueueSize = 1) + : fDStep(pDStep), fMaxQueueSize(maxQueueSize) + { + pthread_mutex_init(&fMutex, 0); + pthread_cond_init(&fTableBandAdded, 0); + pthread_cond_init(&fTableBandRemoved, 0); + } - // TableBandQueueMgr destructor. - ~TableBandQueueMgr( ) - { - pthread_mutex_destroy( &fMutex); - pthread_cond_destroy ( &fTableBandAdded ); - pthread_cond_destroy ( &fTableBandRemoved ); - } + // TableBandQueueMgr destructor. + ~TableBandQueueMgr() + { + pthread_mutex_destroy(&fMutex); + pthread_cond_destroy(&fTableBandAdded); + pthread_cond_destroy(&fTableBandRemoved); + } - // Main processing loop that projects the table bands. - void project(); + // Main processing loop that projects the table bands. + void project(); - // Called by second thread to acquire projected table band - // - // Acquires the next table band that has been projected, so that - // the calling thread can serialize and forward the table band to - // the next step. - // - // Returns next projected table band. Row count of 0 marks end of data - joblist::TableBand* getNextTableBand(); + // Called by second thread to acquire projected table band + // + // Acquires the next table band that has been projected, so that + // the calling thread can serialize and forward the table band to + // the next step. + // + // Returns next projected table band. Row count of 0 marks end of data + joblist::TableBand* getNextTableBand(); -private: - // Disable these by declaring but not defining - TableBandQueueMgr (const TableBandQueueMgr& rhs); - TableBandQueueMgr& operator=(const TableBandQueueMgr& rhs); + private: + // Disable these by declaring but not defining + TableBandQueueMgr(const TableBandQueueMgr& rhs); + TableBandQueueMgr& operator=(const TableBandQueueMgr& rhs); - DeliveryStep* fDStep; - unsigned int fMaxQueueSize; + DeliveryStep* fDStep; + unsigned int fMaxQueueSize; - // Internal queue used to save table bands as they are projected. - std::queue fTblBandQueue; + // Internal queue used to save table bands as they are projected. + std::queue fTblBandQueue; - // Mutex to protect our internal table band queue, and the - // condition variables used to signal when a table band has - // been added to, or removed from the queue. - pthread_mutex_t fMutex; - pthread_cond_t fTableBandAdded; - pthread_cond_t fTableBandRemoved; + // Mutex to protect our internal table band queue, and the + // condition variables used to signal when a table band has + // been added to, or removed from the queue. + pthread_mutex_t fMutex; + pthread_cond_t fTableBandAdded; + pthread_cond_t fTableBandRemoved; }; // @@ -308,35 +298,35 @@ private: // to continue adding projected table bands, until we reach a table band with // a row count of 0, which denotes the end of the table. // -void TableBandQueueMgr::project ( ) +void TableBandQueueMgr::project() { - bool moreData = true; + bool moreData = true; - //...Stay in loop to project table bands, until we reach a table band - //...having a row count of 0. + //...Stay in loop to project table bands, until we reach a table band + //...having a row count of 0. - while ( moreData ) + while (moreData) + { + joblist::TableBand* pTblBand = new joblist::TableBand; + *pTblBand = fDStep->nextBand(); + + pthread_mutex_lock(&fMutex); + + //...Wait for room in our queue before adding this table band + while (fTblBandQueue.size() >= fMaxQueueSize) { - joblist::TableBand* pTblBand = new joblist::TableBand; - *pTblBand = fDStep->nextBand( ); - - pthread_mutex_lock( &fMutex ); - - //...Wait for room in our queue before adding this table band - while ( fTblBandQueue.size( ) >= fMaxQueueSize ) - { - pthread_cond_wait( &fTableBandRemoved, &fMutex ); - } - - fTblBandQueue.push( pTblBand ); - - if ( pTblBand->getRowCount( ) == 0 ) - moreData = false; - - pthread_cond_broadcast( &fTableBandAdded ); - - pthread_mutex_unlock( &fMutex ); + pthread_cond_wait(&fTableBandRemoved, &fMutex); } + + fTblBandQueue.push(pTblBand); + + if (pTblBand->getRowCount() == 0) + moreData = false; + + pthread_cond_broadcast(&fTableBandAdded); + + pthread_mutex_unlock(&fMutex); + } } // @@ -346,30 +336,30 @@ void TableBandQueueMgr::project ( ) // joblist::TableBand* TableBandQueueMgr::getNextTableBand() { - joblist::TableBand* pTblBand = 0; + joblist::TableBand* pTblBand = 0; - pthread_mutex_lock( &fMutex ); + pthread_mutex_lock(&fMutex); - //...Wait for a table band to be added to our queue if the queue is empty - while ( fTblBandQueue.size() < 1 ) - { - pthread_cond_wait( &fTableBandAdded, &fMutex ); - } + //...Wait for a table band to be added to our queue if the queue is empty + while (fTblBandQueue.size() < 1) + { + pthread_cond_wait(&fTableBandAdded, &fMutex); + } - pTblBand = fTblBandQueue.front( ); - fTblBandQueue.pop( ); + pTblBand = fTblBandQueue.front(); + fTblBandQueue.pop(); - //...If the row count is 0, there is no need to notify our producing - //...thread, since that means we have reached the end of data. - if ( pTblBand->getRowCount( ) != 0 ) - pthread_cond_broadcast( &fTableBandRemoved ); + //...If the row count is 0, there is no need to notify our producing + //...thread, since that means we have reached the end of data. + if (pTblBand->getRowCount() != 0) + pthread_cond_broadcast(&fTableBandRemoved); - pthread_mutex_unlock( &fMutex ); + pthread_mutex_unlock(&fMutex); - return pTblBand; + return pTblBand; } -} // end of namespace +} // end of namespace // End of Class to manage table band projection in a separate thread //****************************************************************************** @@ -378,15 +368,15 @@ joblist::TableBand* TableBandQueueMgr::getNextTableBand() //------------------------------------------------------------------------------ // Drives thread to project table bands for delivery. //------------------------------------------------------------------------------ -void* projectThreadWrapper( void* pThreadData ) +void* projectThreadWrapper(void* pThreadData) { - TableBandQueueMgr* tableBandMgr = (TableBandQueueMgr*)pThreadData; + TableBandQueueMgr* tableBandMgr = (TableBandQueueMgr*)pThreadData; - //cout << "Starting thread to project columns..." << endl; - tableBandMgr->project(); - //cout << "Finished thread to project columns..." << endl; + // cout << "Starting thread to project columns..." << endl; + tableBandMgr->project(); + // cout << "Finished thread to project columns..." << endl; - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -394,14 +384,14 @@ void* projectThreadWrapper( void* pThreadData ) //------------------------------------------------------------------------------ void runStep(DeliveryStep& dstep, const string& message) { - string nextBandMsg (message ); - nextBandMsg += " - nextBand()"; - string serializeMsg (message ); - serializeMsg += " - serialize()"; - int nextBandCount = 0; + string nextBandMsg(message); + nextBandMsg += " - nextBand()"; + string serializeMsg(message); + serializeMsg += " - serialize()"; + int nextBandCount = 0; - ByteStream bs; - TableBand tb; + ByteStream bs; + TableBand tb; //...Perform table band projection and serialization in succession #if 0 @@ -424,53 +414,52 @@ void runStep(DeliveryStep& dstep, const string& message) //...Perform table band projection and serialization in parallel #else - string thrCreateMsg (message ); - thrCreateMsg += " - serialize-thrCreate"; - string thrJoinMsg (message ); - thrJoinMsg += " - serialize-thrJoin"; - string serializeWaitMsg(message ); - serializeWaitMsg += " - serialize-Wait"; + string thrCreateMsg(message); + thrCreateMsg += " - serialize-thrCreate"; + string thrJoinMsg(message); + thrJoinMsg += " - serialize-thrJoin"; + string serializeWaitMsg(message); + serializeWaitMsg += " - serialize-Wait"; - //...Would prefer to record this label in projectThreadWrapper, but - //...Stopwatch is not threadsafe, so safer to put here in main thread, - //...where the other Stopwatch times are recorded. Note that this - //...time will overlap the other timestamps we are recording. - // timer.start(nextBandMsg); + //...Would prefer to record this label in projectThreadWrapper, but + //...Stopwatch is not threadsafe, so safer to put here in main thread, + //...where the other Stopwatch times are recorded. Note that this + //...time will overlap the other timestamps we are recording. + // timer.start(nextBandMsg); - //...Start a second thread that will allow us to perform - //...table projections in parallel with band serialization - // timer.start(thrCreateMsg); - TableBandQueueMgr tableBandMgr(&dstep, 1); - pthread_t projectionThread; - pthread_create(&projectionThread, 0, - projectThreadWrapper, &tableBandMgr ); - // timer.stop (thrCreateMsg); + //...Start a second thread that will allow us to perform + //...table projections in parallel with band serialization + // timer.start(thrCreateMsg); + TableBandQueueMgr tableBandMgr(&dstep, 1); + pthread_t projectionThread; + pthread_create(&projectionThread, 0, projectThreadWrapper, &tableBandMgr); + // timer.stop (thrCreateMsg); - while (1) - { - //...The amount of time we spend waiting will help tell us how - //...much extra time is being spent constructing the table bands - // timer.start(serializeWaitMsg); - boost::scoped_ptr band(tableBandMgr.getNextTableBand()); - nextBandCount++; - // timer.stop (serializeWaitMsg); + while (1) + { + //...The amount of time we spend waiting will help tell us how + //...much extra time is being spent constructing the table bands + // timer.start(serializeWaitMsg); + boost::scoped_ptr band(tableBandMgr.getNextTableBand()); + nextBandCount++; + // timer.stop (serializeWaitMsg); - // timer.start(serializeMsg); - bs.reset(); - band->serialize(bs); - // timer.stop( serializeMsg); + // timer.start(serializeMsg); + bs.reset(); + band->serialize(bs); + // timer.stop( serializeMsg); - if (band->getRowCount() == 0) - break; - } + if (band->getRowCount() == 0) + break; + } - // timer.stop(nextBandMsg); + // timer.stop(nextBandMsg); - // timer.start(thrJoinMsg); - pthread_join(projectionThread, 0); - // timer.stop (thrJoinMsg); + // timer.start(thrJoinMsg); + pthread_join(projectionThread, 0); + // timer.stop (thrJoinMsg); #endif - cout << nextBandCount << " table bands delivered" << endl; + cout << nextBandCount << " table bands delivered" << endl; } //------------------------------------------------------------------------------ @@ -478,16 +467,16 @@ void runStep(DeliveryStep& dstep, const string& message) //------------------------------------------------------------------------------ void addElements(BandedDL* dl1) { - ElementType e; + ElementType e; - for (int i = 1; i <= dataSize; i++) - { - e.first = i; - e.second = i; - dl1->insert(e); - } + for (int i = 1; i <= dataSize; i++) + { + e.first = i; + e.second = i; + dl1->insert(e); + } - dl1->endOfInput(); + dl1->endOfInput(); } //------------------------------------------------------------------------------ @@ -495,16 +484,16 @@ void addElements(BandedDL* dl1) //------------------------------------------------------------------------------ void addElements(FIFO* dl1) { - StringElementType e; + StringElementType e; - for (int i = 1; i <= dataSize; i++) - { - e.first = i; - e.second = strtable; - dl1->insert(e); - } + for (int i = 1; i <= dataSize; i++) + { + e.first = i; + e.second = strtable; + dl1->insert(e); + } - dl1->endOfInput(); + dl1->endOfInput(); } //------------------------------------------------------------------------------ @@ -512,62 +501,60 @@ void addElements(FIFO* dl1) //------------------------------------------------------------------------------ void addElements(FIFO* dl1) { - ElementType e; - int wrapCount = 0; - UintRowGroup rg; + ElementType e; + int wrapCount = 0; + UintRowGroup rg; - for (int i = 1; i <= dataSize; i++) + for (int i = 1; i <= dataSize; i++) + { + e.first = i; + e.second = i; + + rg.et[wrapCount] = e; + wrapCount++; + + if (wrapCount == 8192 || i == dataSize) { - e.first = i; - e.second = i; - - rg.et[wrapCount] = e; - wrapCount++; - - if (wrapCount == 8192 || i == dataSize) - { - rg.count = wrapCount; - dl1->insert(rg); - wrapCount = 0; - } + rg.count = wrapCount; + dl1->insert(rg); + wrapCount = 0; } + } - dl1->endOfInput(); + dl1->endOfInput(); } //------------------------------------------------------------------------------ // Test delivery of a single numeric column //------------------------------------------------------------------------------ -void deliveryStep_1() //number column +void deliveryStep_1() // number column { - DistributedEngineComm* dec; - boost::shared_ptr cat; + DistributedEngineComm* dec; + boost::shared_ptr cat; - ResourceManager rm; - dec = DistributedEngineComm::instance(rm); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(1); + ResourceManager rm; + dec = DistributedEngineComm::instance(rm); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(1); - DBRM dbrm; - uint32_t uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); + DBRM dbrm; + uint32_t uniqueID = dbrm.getUnique32(); + dec->addQueue(uniqueID); - JobStepAssociation inJs; + JobStepAssociation inJs; - AnyDataListSPtr spdl1(new AnyDataList()); - FIFO* dl1 = new FIFO(1, 100); - addElements(dl1); + AnyDataListSPtr spdl1(new AnyDataList()); + FIFO* dl1 = new FIFO(1, 100); + addElements(dl1); - spdl1->fifoDL(dl1); + spdl1->fifoDL(dl1); - inJs.outAdd(spdl1); + inJs.outAdd(spdl1); - DeliveryStep dstep(inJs, JobStepAssociation(), - make_table(schema, table), - cat, 1, 1, 1, flushInterval); + DeliveryStep dstep(inJs, JobStepAssociation(), make_table(schema, table), cat, 1, 1, 1, flushInterval); - runStep(dstep, string("deliveryStep_1")); + runStep(dstep, string("deliveryStep_1")); - dec->removeQueue(uniqueID); + dec->removeQueue(uniqueID); } //------------------------------------------------------------------------------ @@ -576,51 +563,51 @@ void deliveryStep_1() //number column /* void deliveryStep_2() //string column { - DistributedEngineComm* dec; - boost::shared_ptr cat; + DistributedEngineComm* dec; + boost::shared_ptr cat; - dec = DistributedEngineComm::instance(); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(1); + dec = DistributedEngineComm::instance(); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(1); - dec->addSession(12345); - dec->addStep(12345, 0); + dec->addSession(12345); + dec->addStep(12345, 0); - JobStepAssociation inJs; + JobStepAssociation inJs; - AnyDataListSPtr spdl1(new AnyDataList()); - FIFO* dl1 = new FIFO(1,100); + AnyDataListSPtr spdl1(new AnyDataList()); + FIFO* dl1 = new FIFO(1,100); - addElements(dl1); - StringElementType e; + addElements(dl1); + StringElementType e; - spdl1->stringDL(dl1); + spdl1->stringDL(dl1); - inJs.outAdd(spdl1); + inJs.outAdd(spdl1); - DeliveryStep dstep(inJs, JobStepAssociation(), - make_table(schema, strtable), - cat, bandSize, 1, 1, 1, flushInterval); + DeliveryStep dstep(inJs, JobStepAssociation(), + make_table(schema, strtable), + cat, bandSize, 1, 1, 1, flushInterval); - runStep(dstep, string("deliveryStep_2")); + runStep(dstep, string("deliveryStep_2")); - dec->removeSession(12345); + dec->removeSession(12345); } */ //------------------------------------------------------------------------------ // Drives thread to add elements to the specified FIFO. //------------------------------------------------------------------------------ -void* addElementsThreadWrapper( void* pThreadData ) +void* addElementsThreadWrapper(void* pThreadData) { - FIFO* dl1 = (FIFO*)pThreadData; + FIFO* dl1 = (FIFO*)pThreadData; - //cout << "Starting thread to add elements for column " << - // dl1->OID() << endl; - addElements(dl1); - //cout << "Finished thread to add elements for column " << - // dl1->OID() << endl; + // cout << "Starting thread to add elements for column " << + // dl1->OID() << endl; + addElements(dl1); + // cout << "Finished thread to add elements for column " << + // dl1->OID() << endl; - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -628,60 +615,58 @@ void* addElementsThreadWrapper( void* pThreadData ) //------------------------------------------------------------------------------ void deliverTest(int numCols) { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ResourceManager rm; - dec = DistributedEngineComm::instance(rm); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(1); + DistributedEngineComm* dec; + boost::shared_ptr cat; + ResourceManager rm; + dec = DistributedEngineComm::instance(rm); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(1); - // Get the oid for the first column in the table - usually lineitem.l_orderkey. - CalpontSystemCatalog::TableName tableName = make_table(schema, colstable); - CalpontSystemCatalog::ROPair p = cat->tableRID(tableName); - startingOid = p.objnum + 1; + // Get the oid for the first column in the table - usually lineitem.l_orderkey. + CalpontSystemCatalog::TableName tableName = make_table(schema, colstable); + CalpontSystemCatalog::ROPair p = cat->tableRID(tableName); + startingOid = p.objnum + 1; - int startingOid = 3416; // Oid for lineitem.l_orderkey on your database. + int startingOid = 3416; // Oid for lineitem.l_orderkey on your database. - DBRM dbrm; - uint32_t uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); + DBRM dbrm; + uint32_t uniqueID = dbrm.getUnique32(); + dec->addQueue(uniqueID); - JobStepAssociation inJs; + JobStepAssociation inJs; - stringstream ss; - pthread_t threads[ numCols ]; + stringstream ss; + pthread_t threads[numCols]; - for (int i = 0; i < numCols; ++i) - { - AnyDataListSPtr spdl1(new AnyDataList()); + for (int i = 0; i < numCols; ++i) + { + AnyDataListSPtr spdl1(new AnyDataList()); - // Make FIFO big enough to contain the elements for a flush interval - FIFO* dl1 = new FIFO(1, fifoSize); -// BandedDL* dl1 = new BandedDL(1); + // Make FIFO big enough to contain the elements for a flush interval + FIFO* dl1 = new FIFO(1, fifoSize); + // BandedDL* dl1 = new BandedDL(1); - dl1->OID(i + startingOid); // lineitem first col object id - spdl1->fifoDL(dl1); -// spdl1->bandedDL(dl1); - inJs.outAdd(spdl1); + dl1->OID(i + startingOid); // lineitem first col object id + spdl1->fifoDL(dl1); + // spdl1->bandedDL(dl1); + inJs.outAdd(spdl1); - pthread_create(&threads[i], 0, addElementsThreadWrapper, dl1 ); - } + pthread_create(&threads[i], 0, addElementsThreadWrapper, dl1); + } - DeliveryStep dstep(inJs, JobStepAssociation(), tableName, - cat, 12345, 1, 1, flushInterval); + DeliveryStep dstep(inJs, JobStepAssociation(), tableName, cat, 12345, 1, 1, flushInterval); - ss << "DeliverStep test for " << numCols; + ss << "DeliverStep test for " << numCols; - string message = ss.str(); + string message = ss.str(); - runStep(dstep, ss.str()); + runStep(dstep, ss.str()); - for (int i = 0; i < numCols; ++i) - { - pthread_join(threads[i], 0); - } - - dec->removeQueue(uniqueID); + for (int i = 0; i < numCols; ++i) + { + pthread_join(threads[i], 0); + } + dec->removeQueue(uniqueID); } //------------------------------------------------------------------------------ @@ -690,180 +675,175 @@ void deliverTest(int numCols) //------------------------------------------------------------------------------ void testSizes() { + // Prompt for schema. + cout << "Enter Schema or Enter for " << schema << ": "; + string tmpSchema; + getline(cin, tmpSchema); - // Prompt for schema. - cout << "Enter Schema or Enter for " << schema << ": "; - string tmpSchema; - getline(cin, tmpSchema); + if (tmpSchema.length() > 0) + { + schema = tmpSchema; + } - if (tmpSchema.length() > 0) - { - schema = tmpSchema; - } + // Prompt for table. + cout << "Enter Table or Enter for " << colstable << ": "; + string tmpTable; + getline(cin, tmpTable); - // Prompt for table. - cout << "Enter Table or Enter for " << colstable << ": "; - string tmpTable; - getline(cin, tmpTable); + if (tmpTable.length() > 0) + { + colstable = tmpTable; + } - if (tmpTable.length() > 0) - { - colstable = tmpTable; - } + timer.start("Total"); + int maxCols = 9; + cout << endl; - timer.start("Total"); - int maxCols = 9; - cout << endl; + for (int i = 7; i <= maxCols; i++) + { + cout << endl << "Running test " << i << " of " << maxCols << endl; + stringstream ss; + ss << "Delivery test for " << dataSize << " rows and " << i << " columns"; + timer.start(ss.str()); + deliverTest(i); + timer.stop(ss.str()); + } - for (int i = 7; i <= maxCols; i++) - { - cout << endl << "Running test " << i << " of " << maxCols << endl; - stringstream ss; - ss << "Delivery test for " << dataSize << " rows and " << i << " columns"; - timer.start(ss.str()); - deliverTest(i); - timer.stop(ss.str()); - } - - timer.stop("Total"); - timer.finish(); + timer.stop("Total"); + timer.finish(); } void* nextBandBenchProducer(void* arg) { - FIFO* dl1 = (FIFO*) arg; - UintRowGroup rg; - uint64_t* arr; - uint32_t i; + FIFO* dl1 = (FIFO*)arg; + UintRowGroup rg; + uint64_t* arr; + uint32_t i; - arr = (uint64_t*) rg.et; + arr = (uint64_t*)rg.et; - for (i = 0; i < 8192; ++i) - arr[i] = i; + for (i = 0; i < 8192; ++i) + arr[i] = i; - rg.count = 8192; + rg.count = 8192; - for (i = 1; i <= dataSize / 8192; i++) - { -// cout << "inserting set " << i << endl; - dl1->insert(rg); - } + for (i = 1; i <= dataSize / 8192; i++) + { + // cout << "inserting set " << i << endl; + dl1->insert(rg); + } - dl1->endOfInput(); - return NULL; + dl1->endOfInput(); + return NULL; } void nextBandBenchmark() { - ByteStream bs; - pthread_t threads[columns]; - uint32_t i, rowCount = 1; - JobStepAssociation inJs; + ByteStream bs; + pthread_t threads[columns]; + uint32_t i, rowCount = 1; + JobStepAssociation inJs; - for (i = 0; i < columns; ++i) - { - AnyDataListSPtr spdl1(new AnyDataList()); + for (i = 0; i < columns; ++i) + { + AnyDataListSPtr spdl1(new AnyDataList()); - FIFO* dl1 = new FIFO(1, fifoSize); + FIFO* dl1 = new FIFO(1, fifoSize); - dl1->OID(i); // lineitem first col object id - spdl1->fifoDL(dl1); - inJs.outAdd(spdl1); + dl1->OID(i); // lineitem first col object id + spdl1->fifoDL(dl1); + inJs.outAdd(spdl1); - pthread_create(&threads[i], 0, nextBandBenchProducer, dl1 ); - cout << "started thread " << i << endl; - } + pthread_create(&threads[i], 0, nextBandBenchProducer, dl1); + cout << "started thread " << i << endl; + } - DeliveryStep ds(inJs, JobStepAssociation(), 8); - stringstream ss; + DeliveryStep ds(inJs, JobStepAssociation(), 8); + stringstream ss; - ss << "nextBandBenchmark with " << columns << " columns\n"; + ss << "nextBandBenchmark with " << columns << " columns\n"; - timer.start(ss.str()); + timer.start(ss.str()); - while (rowCount != 0) - { -// cout << "getting a BS\n"; - rowCount = ds.nextBand(bs); - bs.restart(); -// cout << "got a BS\n"; - } + while (rowCount != 0) + { + // cout << "getting a BS\n"; + rowCount = ds.nextBand(bs); + bs.restart(); + // cout << "got a BS\n"; + } - timer.stop(ss.str()); + timer.stop(ss.str()); - for (i = 0; i < columns; ++i) - pthread_join(threads[i], NULL); + for (i = 0; i < columns; ++i) + pthread_join(threads[i], NULL); } void queuedBSBenchmark(int queueLength) { - ByteStream* bs; - pthread_t threads[columns]; - uint32_t i, rowCount = 1; - JobStepAssociation inJs; + ByteStream* bs; + pthread_t threads[columns]; + uint32_t i, rowCount = 1; + JobStepAssociation inJs; - for (i = 0; i < columns; ++i) - { - AnyDataListSPtr spdl1(new AnyDataList()); + for (i = 0; i < columns; ++i) + { + AnyDataListSPtr spdl1(new AnyDataList()); - FIFO* dl1 = new FIFO(1, fifoSize); + FIFO* dl1 = new FIFO(1, fifoSize); - dl1->OID(i); // lineitem first col object id - spdl1->fifoDL(dl1); - inJs.outAdd(spdl1); + dl1->OID(i); // lineitem first col object id + spdl1->fifoDL(dl1); + inJs.outAdd(spdl1); - pthread_create(&threads[i], 0, nextBandBenchProducer, dl1 ); - cout << "started thread " << i << endl; - } + pthread_create(&threads[i], 0, nextBandBenchProducer, dl1); + cout << "started thread " << i << endl; + } - DeliveryStep ds(inJs, JobStepAssociation(), 8); - BSQueueMgr bsq(&ds, queueLength); - stringstream ss; - ss << "queuedBSBenchmark with " << columns << " columns and " << queueLength << " queue length\n"; + DeliveryStep ds(inJs, JobStepAssociation(), 8); + BSQueueMgr bsq(&ds, queueLength); + stringstream ss; + ss << "queuedBSBenchmark with " << columns << " columns and " << queueLength << " queue length\n"; - timer.start(ss.str()); - boost::thread(BSProjectThread(&bsq)); + timer.start(ss.str()); + boost::thread(BSProjectThread(&bsq)); - while (rowCount != 0) - { - bs = bsq.getNextByteStream(rowCount); - delete bs; - } + while (rowCount != 0) + { + bs = bsq.getNextByteStream(rowCount); + delete bs; + } - timer.stop(ss.str()); + timer.stop(ss.str()); - for (i = 0; i < columns; ++i) - pthread_join(threads[i], NULL); + for (i = 0; i < columns; ++i) + pthread_join(threads[i], NULL); } - - //------------------------------------------------------------------------------ // Main entry point //------------------------------------------------------------------------------ -int main( int argc, char** argv) +int main(int argc, char** argv) { - if (argc > 1) - columns = atoi(argv[1]); // override default number of rows - else - columns = 10; + if (argc > 1) + columns = atoi(argv[1]); // override default number of rows + else + columns = 10; - while (columns > 0) - { + while (columns > 0) + { + // testSizes(); + nextBandBenchmark(); -// testSizes(); - nextBandBenchmark(); + queuedBSBenchmark(10); + queuedBSBenchmark(9); + queuedBSBenchmark(8); + queuedBSBenchmark(5); + queuedBSBenchmark(4); + queuedBSBenchmark(1); + timer.finish(); + columns--; + } - queuedBSBenchmark(10); - queuedBSBenchmark(9); - queuedBSBenchmark(8); - queuedBSBenchmark(5); - queuedBSBenchmark(4); - queuedBSBenchmark(1); - timer.finish(); - columns--; - } - - return 0; + return 0; } - diff --git a/dbcon/joblist/tdriver-filter.cpp b/dbcon/joblist/tdriver-filter.cpp index d69099ef9..d14452ce7 100644 --- a/dbcon/joblist/tdriver-filter.cpp +++ b/dbcon/joblist/tdriver-filter.cpp @@ -33,7 +33,6 @@ using namespace joblist; #include "filteroperation.h" - #include #include #include "bytestream.h" @@ -49,565 +48,550 @@ using namespace messageqcpp; // Timer class used by this tdriver to output elapsed times, etc. class Timer { -public: - void start(const string& message) + public: + void start(const string& message) + { + if (!fHeaderDisplayed) { - if (!fHeaderDisplayed) - { - header(); - fHeaderDisplayed = true; - } - - gettimeofday(&fTvStart, 0); - cout << timestr() << " Start " << message << endl; + header(); + fHeaderDisplayed = true; } - void stop(const string& message) - { - time_t now; - time(&now); - string secondsElapsed; - getTimeElapsed(secondsElapsed); - cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; - } + gettimeofday(&fTvStart, 0); + cout << timestr() << " Start " << message << endl; + } - Timer() : fHeaderDisplayed(false) {} + void stop(const string& message) + { + time_t now; + time(&now); + string secondsElapsed; + getTimeElapsed(secondsElapsed); + cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; + } -private: + Timer() : fHeaderDisplayed(false) + { + } - struct timeval fTvStart; - bool fHeaderDisplayed; + private: + struct timeval fTvStart; + bool fHeaderDisplayed; - double getTimeElapsed(string& seconds) - { - struct timeval tvStop; - gettimeofday(&tvStop, 0); - double secondsElapsed = - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); - ostringstream oss; - oss << secondsElapsed; - seconds = oss.str(); - seconds.resize(8, '0'); - return secondsElapsed; - } + double getTimeElapsed(string& seconds) + { + struct timeval tvStop; + gettimeofday(&tvStop, 0); + double secondsElapsed = + (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); + ostringstream oss; + oss << secondsElapsed; + seconds = oss.str(); + seconds.resize(8, '0'); + return secondsElapsed; + } - string timestr() - { - struct tm tm; - struct timeval tv; + string timestr() + { + struct tm tm; + struct timeval tv; - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec << '.' - << setw(6) << tv.tv_usec - ; - return oss.str(); - } + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec + << '.' << setw(6) << tv.tv_usec; + return oss.str(); + } - void header() - { - cout << endl; - cout << "Time Seconds Activity" << endl; - } + void header() + { + cout << endl; + cout << "Time Seconds Activity" << endl; + } }; class FilterDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(FilterDriver); - CPPUNIT_TEST_SUITE(FilterDriver); + CPPUNIT_TEST(DOUBLE_TIME_TEST); - CPPUNIT_TEST(DOUBLE_TIME_TEST); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + // Creates two fifos with numRows and performs the passed comparison against their values. + // The first fifo will contain values 0..numRows-1. + // The second fifo will contain values numrows-1..0. + // Both bands will containt the same rids. + // Outputs timing results for loading each fifo, doing the comparison, and iterating over the results. + // Outputs the number of qualifying rows. + // Asserts that the number of qualifying rows is correct. + void filterTest(const FilterOperation::FilterOperator& fo, const int& numRows) + { + Timer timer; + cout << endl; + cout << "------------------------------------------------------------" << endl; + int i; + ElementType element; -private: - // Creates two fifos with numRows and performs the passed comparison against their values. - // The first fifo will contain values 0..numRows-1. - // The second fifo will contain values numrows-1..0. - // Both bands will containt the same rids. - // Outputs timing results for loading each fifo, doing the comparison, and iterating over the results. - // Outputs the number of qualifying rows. - // Asserts that the number of qualifying rows is correct. - void filterTest(const FilterOperation::FilterOperator& fo, const int& numRows) + stringstream ss; + ss << "loading first fifo with " << numRows << " rows."; + string message = ss.str(); + timer.start(message); + FIFO fifo1(1, numRows); + + for (i = 0; i < numRows; i++) { - Timer timer; - cout << endl; - cout << "------------------------------------------------------------" << endl; - int i; - ElementType element; - - stringstream ss; - ss << "loading first fifo with " << numRows << " rows."; - string message = ss.str(); - timer.start(message); - FIFO fifo1(1, numRows); - - for (i = 0; i < numRows; i++) - { - element.first = i; - element.second = i; - fifo1.insert(element); - } - - fifo1.endOfInput(); - timer.stop(message); - - stringstream ss2; - ss2.flush(); - ss2 << "loading second fifo with " << numRows << " rows."; - message = ss2.str(); - timer.start(message); - - FIFO fifo2(1, numRows); - - for (i = 0; i < numRows; i++) - { - element.first = i; - element.second = numRows - (i + 1); - fifo2.insert(element); - } - - fifo2.endOfInput(); - timer.stop(message); - - FIFO fifo3(1, numRows); - - DataList* dl1 = (DataList*) &fifo1; - DataList* dl2 = (DataList*) &fifo2; - DataList* dl3 = (DataList*) &fifo3; - int assertCount; - - switch (fo) - { - case FilterOperation::GT: - message = "GT Test"; - assertCount = numRows / 2; - break; - - case FilterOperation::LT: - message = "LT Test"; - assertCount = numRows / 2; - break; - - case FilterOperation::GTE: - message = "GTE Test"; - assertCount = (numRows / 2); - - if (numRows % 2 > 0) - assertCount++; - - break; - - case FilterOperation::LTE: - message = "LTE Test"; - assertCount = (numRows / 2); - - if (numRows % 2 > 0) - assertCount++; - - break; - - case FilterOperation::EQ: - message = "EQ Test"; - assertCount = numRows % 2; - break; - - case FilterOperation::NE: - message = "NE Test"; - assertCount = numRows - numRows % 2; - break; - - default: - break; - } - - timer.start(message); - FilterOperation filterOperation; - filterOperation.filter(fo, *dl1, *dl2, *dl3); - timer.stop(message); - fifo3.endOfInput(); - - timer.start("iterating over result datalist"); - bool more; - int it; - it = fifo3.getIterator(); - int count = 0; - - do - { - more = fifo3.next(it, &element); - - if (more) - { - count++; - // cout << element.fRid << " " << element.fValue << endl; - } - } - while (more); - - timer.stop("iterating over result datalist"); - - cout << count << " rows qualified." << endl; - idbassert(count == assertCount); + element.first = i; + element.second = i; + fifo1.insert(element); } - // Creates two string fifos with numRows and performs the passed comparison against their values. - // The first fifo will contain values 0..numRows-1. - // The second fifo will contain values numrows-1..0. - // Both bands will containt the same rids. - // Outputs timing results for loading each fifo, doing the comparison, and iterating over the results. - // Outputs the number of qualifying rows. - // Asserts that the number of qualifying rows is correct. - void stringFilterTest(const FilterOperation::FilterOperator& fo, const int& numRows) + fifo1.endOfInput(); + timer.stop(message); + + stringstream ss2; + ss2.flush(); + ss2 << "loading second fifo with " << numRows << " rows."; + message = ss2.str(); + timer.start(message); + + FIFO fifo2(1, numRows); + + for (i = 0; i < numRows; i++) { - Timer timer; - cout << endl; - cout << "------------------------------------------------------------" << endl; - int i; - StringElementType element; - - stringstream ss; - ss << "loading first fifo with " << numRows << " rows with StringElementType."; - string message = ss.str(); - timer.start(message); - FIFO fifo1(1, numRows); - - for (i = 0; i < numRows; i++) - { - element.first = i; - element.second = i; - fifo1.insert(element); - } - - fifo1.endOfInput(); - timer.stop(message); - - stringstream ss2; - ss2.flush(); - ss2 << "loading second fifo with " << numRows << " rows with StringElementType."; - message = ss2.str(); - timer.start(message); - - FIFO fifo2(1, numRows); - - for (i = 0; i < numRows; i++) - { - element.first = i; - element.second = numRows - (i + 1); - fifo2.insert(element); - } - - fifo2.endOfInput(); - timer.stop(message); - - FIFO fifo3(1, numRows); - - DataList* dl1 = (DataList*) &fifo1; - DataList* dl2 = (DataList*) &fifo2; - DataList* dl3 = (DataList*) &fifo3; - int assertCount; - - switch (fo) - { - case FilterOperation::GT: - message = "GT Test"; - assertCount = numRows / 2; - break; - - case FilterOperation::LT: - message = "LT Test"; - assertCount = numRows / 2; - break; - - case FilterOperation::GTE: - message = "GTE Test"; - assertCount = (numRows / 2); - - if (numRows % 2 > 0) - assertCount++; - - break; - - case FilterOperation::LTE: - message = "LTE Test"; - assertCount = (numRows / 2); - - if (numRows % 2 > 0) - assertCount++; - - break; - - case FilterOperation::EQ: - message = "EQ Test"; - assertCount = numRows % 2; - break; - - case FilterOperation::NE: - message = "NE Test"; - assertCount = numRows - numRows % 2; - break; - - default: - break; - } - - timer.start(message); - FilterOperation filterOperation; - filterOperation.filter(fo, *dl1, *dl2, *dl3); - timer.stop(message); - fifo3.endOfInput(); - - timer.start("iterating over result datalist"); - bool more; - int it; - it = fifo3.getIterator(); - int count = 0; - - do - { - more = fifo3.next(it, &element); - - if (more) - { - count++; - // cout << element.fRid << " " << element.fValue << endl; - } - } - while (more); - - timer.stop("iterating over result datalist"); - - cout << count << " rows qualified." << endl; - idbassert(count == assertCount); + element.first = i; + element.second = numRows - (i + 1); + fifo2.insert(element); } - // Creates two double fifos with numRows and performs the passed comparison against their values. - // The first fifo will contain values 0..numRows-1. - // The second fifo will contain values numrows-1..0. - // Both bands will containt the same rids. - // Outputs timing results for loading each fifo, doing the comparison, and iterating over the results. - // Outputs the number of qualifying rows. - // Asserts that the number of qualifying rows is correct. - void doubleFilterTest(const FilterOperation::FilterOperator& fo, const int& numRows) + fifo2.endOfInput(); + timer.stop(message); + + FIFO fifo3(1, numRows); + + DataList* dl1 = (DataList*)&fifo1; + DataList* dl2 = (DataList*)&fifo2; + DataList* dl3 = (DataList*)&fifo3; + int assertCount; + + switch (fo) { - Timer timer; - cout << endl; - cout << "------------------------------------------------------------" << endl; - int i; - DoubleElementType element; + case FilterOperation::GT: + message = "GT Test"; + assertCount = numRows / 2; + break; - stringstream ss; - ss << "loading first fifo with " << numRows << " rows of DoubleElementType."; - string message = ss.str(); - timer.start(message); - FIFO fifo1(1, numRows); + case FilterOperation::LT: + message = "LT Test"; + assertCount = numRows / 2; + break; - for (i = 0; i < numRows; i++) - { - element.first = i; - element.second = i + 0.1; - fifo1.insert(element); - } + case FilterOperation::GTE: + message = "GTE Test"; + assertCount = (numRows / 2); - fifo1.endOfInput(); - timer.stop(message); + if (numRows % 2 > 0) + assertCount++; - stringstream ss2; - ss2.flush(); - ss2 << "loading second fifo with " << numRows << " rows."; - message = ss2.str(); - timer.start(message); + break; - FIFO fifo2(1, numRows); + case FilterOperation::LTE: + message = "LTE Test"; + assertCount = (numRows / 2); - for (i = 0; i < numRows; i++) - { - element.first = i; - element.second = numRows - (i + 1) + 0.1; - fifo2.insert(element); - } + if (numRows % 2 > 0) + assertCount++; - fifo2.endOfInput(); - timer.stop(message); + break; - FIFO fifo3(1, numRows); + case FilterOperation::EQ: + message = "EQ Test"; + assertCount = numRows % 2; + break; - DataList* dl1 = (DataList*) &fifo1; - DataList* dl2 = (DataList*) &fifo2; - DataList* dl3 = (DataList*) &fifo3; - int assertCount; + case FilterOperation::NE: + message = "NE Test"; + assertCount = numRows - numRows % 2; + break; - switch (fo) - { - case FilterOperation::GT: - message = "GT Test"; - assertCount = numRows / 2; - break; - - case FilterOperation::LT: - message = "LT Test"; - assertCount = numRows / 2; - break; - - case FilterOperation::GTE: - message = "GTE Test"; - assertCount = (numRows / 2); - - if (numRows % 2 > 0) - assertCount++; - - break; - - case FilterOperation::LTE: - message = "LTE Test"; - assertCount = (numRows / 2); - - if (numRows % 2 > 0) - assertCount++; - - break; - - case FilterOperation::EQ: - message = "EQ Test"; - assertCount = numRows % 2; - break; - - case FilterOperation::NE: - message = "NE Test"; - assertCount = numRows - numRows % 2; - break; - - default: - break; - } - - timer.start(message); - FilterOperation filterOperation; - filterOperation.filter(fo, *dl1, *dl2, *dl3); - timer.stop(message); - fifo3.endOfInput(); - - timer.start("iterating over result datalist"); - bool more; - int it; - it = fifo3.getIterator(); - int count = 0; - - do - { - more = fifo3.next(it, &element); - - if (more) - { - count++; - // cout << element.fRid << " " << element.fValue << endl; - } - } - while (more); - - timer.stop("iterating over result datalist"); - - cout << count << " rows qualified." << endl; - idbassert(count == assertCount); + default: break; } -public: + timer.start(message); + FilterOperation filterOperation; + filterOperation.filter(fo, *dl1, *dl2, *dl3); + timer.stop(message); + fifo3.endOfInput(); - void TIME_TEST() + timer.start("iterating over result datalist"); + bool more; + int it; + it = fifo3.getIterator(); + int count = 0; + + do { - int numRows = 1000 * 1000 * 2; - filterTest(FilterOperation::GT, numRows); - filterTest(FilterOperation::LT, numRows); - filterTest(FilterOperation::LTE, numRows); - filterTest(FilterOperation::GTE, numRows); - filterTest(FilterOperation::EQ, numRows); - filterTest(FilterOperation::NE, numRows); + more = fifo3.next(it, &element); + + if (more) + { + count++; + // cout << element.fRid << " " << element.fValue << endl; + } + } while (more); + + timer.stop("iterating over result datalist"); + + cout << count << " rows qualified." << endl; + idbassert(count == assertCount); + } + + // Creates two string fifos with numRows and performs the passed comparison against their values. + // The first fifo will contain values 0..numRows-1. + // The second fifo will contain values numrows-1..0. + // Both bands will containt the same rids. + // Outputs timing results for loading each fifo, doing the comparison, and iterating over the results. + // Outputs the number of qualifying rows. + // Asserts that the number of qualifying rows is correct. + void stringFilterTest(const FilterOperation::FilterOperator& fo, const int& numRows) + { + Timer timer; + cout << endl; + cout << "------------------------------------------------------------" << endl; + int i; + StringElementType element; + + stringstream ss; + ss << "loading first fifo with " << numRows << " rows with StringElementType."; + string message = ss.str(); + timer.start(message); + FIFO fifo1(1, numRows); + + for (i = 0; i < numRows; i++) + { + element.first = i; + element.second = i; + fifo1.insert(element); } - void STRING_TIME_TEST() + fifo1.endOfInput(); + timer.stop(message); + + stringstream ss2; + ss2.flush(); + ss2 << "loading second fifo with " << numRows << " rows with StringElementType."; + message = ss2.str(); + timer.start(message); + + FIFO fifo2(1, numRows); + + for (i = 0; i < numRows; i++) { - int numRows = 1000 * 1000 * 2; - stringFilterTest(FilterOperation::GT, numRows); - stringFilterTest(FilterOperation::LT, numRows); - stringFilterTest(FilterOperation::LTE, numRows); - stringFilterTest(FilterOperation::GTE, numRows); - stringFilterTest(FilterOperation::EQ, numRows); - stringFilterTest(FilterOperation::NE, numRows); + element.first = i; + element.second = numRows - (i + 1); + fifo2.insert(element); } - void DOUBLE_TIME_TEST() + fifo2.endOfInput(); + timer.stop(message); + + FIFO fifo3(1, numRows); + + DataList* dl1 = (DataList*)&fifo1; + DataList* dl2 = (DataList*)&fifo2; + DataList* dl3 = (DataList*)&fifo3; + int assertCount; + + switch (fo) { - int numRows = 1000 * 1000 * 2; - doubleFilterTest(FilterOperation::GT, numRows); - doubleFilterTest(FilterOperation::LT, numRows); - doubleFilterTest(FilterOperation::LTE, numRows); - doubleFilterTest(FilterOperation::GTE, numRows); - doubleFilterTest(FilterOperation::EQ, numRows); - doubleFilterTest(FilterOperation::NE, numRows); + case FilterOperation::GT: + message = "GT Test"; + assertCount = numRows / 2; + break; + + case FilterOperation::LT: + message = "LT Test"; + assertCount = numRows / 2; + break; + + case FilterOperation::GTE: + message = "GTE Test"; + assertCount = (numRows / 2); + + if (numRows % 2 > 0) + assertCount++; + + break; + + case FilterOperation::LTE: + message = "LTE Test"; + assertCount = (numRows / 2); + + if (numRows % 2 > 0) + assertCount++; + + break; + + case FilterOperation::EQ: + message = "EQ Test"; + assertCount = numRows % 2; + break; + + case FilterOperation::NE: + message = "NE Test"; + assertCount = numRows - numRows % 2; + break; + + default: break; } - void QUICK_TEST() + timer.start(message); + FilterOperation filterOperation; + filterOperation.filter(fo, *dl1, *dl2, *dl3); + timer.stop(message); + fifo3.endOfInput(); + + timer.start("iterating over result datalist"); + bool more; + int it; + it = fifo3.getIterator(); + int count = 0; + + do { - float f = 1.1; - double d = 1.2; - uint64_t i = 1; - uint64_t* i_ptr = &i; - double* d_ptr = &d; - uint64_t* i2_ptr = (uint64_t*) d_ptr; - float* f_ptr = &f; - i_ptr = (uint64_t*) f_ptr; + more = fifo3.next(it, &element); - cout << "*i_ptr=" << *i_ptr << endl; - cout << "*i2_ptr=" << *i2_ptr << endl; - f_ptr = (float*) i_ptr; + if (more) + { + count++; + // cout << element.fRid << " " << element.fValue << endl; + } + } while (more); - cout << "*f_ptr=" << *f_ptr << endl; + timer.stop("iterating over result datalist"); - cout << endl; + cout << count << " rows qualified." << endl; + idbassert(count == assertCount); + } - if (d > i) - cout << "1.2 is greater than 1." << endl; + // Creates two double fifos with numRows and performs the passed comparison against their values. + // The first fifo will contain values 0..numRows-1. + // The second fifo will contain values numrows-1..0. + // Both bands will containt the same rids. + // Outputs timing results for loading each fifo, doing the comparison, and iterating over the results. + // Outputs the number of qualifying rows. + // Asserts that the number of qualifying rows is correct. + void doubleFilterTest(const FilterOperation::FilterOperator& fo, const int& numRows) + { + Timer timer; + cout << endl; + cout << "------------------------------------------------------------" << endl; + int i; + DoubleElementType element; - if (f > i) - cout << "1.1 is greater than 1." << endl; + stringstream ss; + ss << "loading first fifo with " << numRows << " rows of DoubleElementType."; + string message = ss.str(); + timer.start(message); + FIFO fifo1(1, numRows); - if (d > f) - cout << "1.2 is greater than 1.1" << endl; - - if (*i_ptr < *i2_ptr) - cout << "1.1 < 1.2 when represented as uint64_t." << endl; - - cout << "sizeof(f) = " << sizeof(f) << endl; - cout << "sizeof(i) = " << sizeof(i) << endl; - cout << "sizeof(d) = " << sizeof(d) << endl; - - double dbl = 9.7; - double dbl2 = 1.3; - i_ptr = (uint64_t*) &dbl; - i2_ptr = (uint64_t*) &dbl2; - cout << endl; - cout << "9.7 as int is " << *i_ptr << endl; - cout << "9.7 as int is " << *i2_ptr << endl; - cout << "1.2 < 9.7 is " << (*i_ptr < *i2_ptr) << endl; + for (i = 0; i < numRows; i++) + { + element.first = i; + element.second = i + 0.1; + fifo1.insert(element); } + + fifo1.endOfInput(); + timer.stop(message); + + stringstream ss2; + ss2.flush(); + ss2 << "loading second fifo with " << numRows << " rows."; + message = ss2.str(); + timer.start(message); + + FIFO fifo2(1, numRows); + + for (i = 0; i < numRows; i++) + { + element.first = i; + element.second = numRows - (i + 1) + 0.1; + fifo2.insert(element); + } + + fifo2.endOfInput(); + timer.stop(message); + + FIFO fifo3(1, numRows); + + DataList* dl1 = (DataList*)&fifo1; + DataList* dl2 = (DataList*)&fifo2; + DataList* dl3 = (DataList*)&fifo3; + int assertCount; + + switch (fo) + { + case FilterOperation::GT: + message = "GT Test"; + assertCount = numRows / 2; + break; + + case FilterOperation::LT: + message = "LT Test"; + assertCount = numRows / 2; + break; + + case FilterOperation::GTE: + message = "GTE Test"; + assertCount = (numRows / 2); + + if (numRows % 2 > 0) + assertCount++; + + break; + + case FilterOperation::LTE: + message = "LTE Test"; + assertCount = (numRows / 2); + + if (numRows % 2 > 0) + assertCount++; + + break; + + case FilterOperation::EQ: + message = "EQ Test"; + assertCount = numRows % 2; + break; + + case FilterOperation::NE: + message = "NE Test"; + assertCount = numRows - numRows % 2; + break; + + default: break; + } + + timer.start(message); + FilterOperation filterOperation; + filterOperation.filter(fo, *dl1, *dl2, *dl3); + timer.stop(message); + fifo3.endOfInput(); + + timer.start("iterating over result datalist"); + bool more; + int it; + it = fifo3.getIterator(); + int count = 0; + + do + { + more = fifo3.next(it, &element); + + if (more) + { + count++; + // cout << element.fRid << " " << element.fValue << endl; + } + } while (more); + + timer.stop("iterating over result datalist"); + + cout << count << " rows qualified." << endl; + idbassert(count == assertCount); + } + + public: + void TIME_TEST() + { + int numRows = 1000 * 1000 * 2; + filterTest(FilterOperation::GT, numRows); + filterTest(FilterOperation::LT, numRows); + filterTest(FilterOperation::LTE, numRows); + filterTest(FilterOperation::GTE, numRows); + filterTest(FilterOperation::EQ, numRows); + filterTest(FilterOperation::NE, numRows); + } + + void STRING_TIME_TEST() + { + int numRows = 1000 * 1000 * 2; + stringFilterTest(FilterOperation::GT, numRows); + stringFilterTest(FilterOperation::LT, numRows); + stringFilterTest(FilterOperation::LTE, numRows); + stringFilterTest(FilterOperation::GTE, numRows); + stringFilterTest(FilterOperation::EQ, numRows); + stringFilterTest(FilterOperation::NE, numRows); + } + + void DOUBLE_TIME_TEST() + { + int numRows = 1000 * 1000 * 2; + doubleFilterTest(FilterOperation::GT, numRows); + doubleFilterTest(FilterOperation::LT, numRows); + doubleFilterTest(FilterOperation::LTE, numRows); + doubleFilterTest(FilterOperation::GTE, numRows); + doubleFilterTest(FilterOperation::EQ, numRows); + doubleFilterTest(FilterOperation::NE, numRows); + } + + void QUICK_TEST() + { + float f = 1.1; + double d = 1.2; + uint64_t i = 1; + uint64_t* i_ptr = &i; + double* d_ptr = &d; + uint64_t* i2_ptr = (uint64_t*)d_ptr; + float* f_ptr = &f; + i_ptr = (uint64_t*)f_ptr; + + cout << "*i_ptr=" << *i_ptr << endl; + cout << "*i2_ptr=" << *i2_ptr << endl; + f_ptr = (float*)i_ptr; + + cout << "*f_ptr=" << *f_ptr << endl; + + cout << endl; + + if (d > i) + cout << "1.2 is greater than 1." << endl; + + if (f > i) + cout << "1.1 is greater than 1." << endl; + + if (d > f) + cout << "1.2 is greater than 1.1" << endl; + + if (*i_ptr < *i2_ptr) + cout << "1.1 < 1.2 when represented as uint64_t." << endl; + + cout << "sizeof(f) = " << sizeof(f) << endl; + cout << "sizeof(i) = " << sizeof(i) << endl; + cout << "sizeof(d) = " << sizeof(d) << endl; + + double dbl = 9.7; + double dbl2 = 1.3; + i_ptr = (uint64_t*)&dbl; + i2_ptr = (uint64_t*)&dbl2; + cout << endl; + cout << "9.7 as int is " << *i_ptr << endl; + cout << "9.7 as int is " << *i2_ptr << endl; + cout << "1.2 < 9.7 is " << (*i_ptr < *i2_ptr) << endl; + } }; CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/joblist/tdriver-function.cpp b/dbcon/joblist/tdriver-function.cpp index 6d1df5539..6657da65c 100644 --- a/dbcon/joblist/tdriver-function.cpp +++ b/dbcon/joblist/tdriver-function.cpp @@ -47,1464 +47,1452 @@ using namespace std; using namespace joblist; using namespace messageqcpp; - // void (*fp) (double, double); // Timer class used by this tdriver to output elapsed times, etc. class Timer { -public: - void start(const string& message) + public: + void start(const string& message) + { + if (!fHeaderDisplayed) { - if (!fHeaderDisplayed) - { - header(); - fHeaderDisplayed = true; - } - - gettimeofday(&fTvStart, 0); - cout << timestr() << " Start " << message << endl; + header(); + fHeaderDisplayed = true; } - void stop(const string& message) - { - time_t now; - time(&now); - string secondsElapsed; - getTimeElapsed(secondsElapsed); - cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; - } + gettimeofday(&fTvStart, 0); + cout << timestr() << " Start " << message << endl; + } - Timer() : fHeaderDisplayed(false) {} + void stop(const string& message) + { + time_t now; + time(&now); + string secondsElapsed; + getTimeElapsed(secondsElapsed); + cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; + } -private: + Timer() : fHeaderDisplayed(false) + { + } - struct timeval fTvStart; - bool fHeaderDisplayed; + private: + struct timeval fTvStart; + bool fHeaderDisplayed; - double getTimeElapsed(string& seconds) - { - struct timeval tvStop; - gettimeofday(&tvStop, 0); - double secondsElapsed = - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); - ostringstream oss; - oss << secondsElapsed; - seconds = oss.str(); - seconds.resize(8, '0'); - return secondsElapsed; - } + double getTimeElapsed(string& seconds) + { + struct timeval tvStop; + gettimeofday(&tvStop, 0); + double secondsElapsed = + (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); + ostringstream oss; + oss << secondsElapsed; + seconds = oss.str(); + seconds.resize(8, '0'); + return secondsElapsed; + } - string timestr() - { - struct tm tm; - struct timeval tv; + string timestr() + { + struct tm tm; + struct timeval tv; - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec << '.' - << setw(6) << tv.tv_usec - ; - return oss.str(); - } + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec + << '.' << setw(6) << tv.tv_usec; + return oss.str(); + } - void header() - { - cout << endl; - cout << "Time Seconds Activity" << endl; - } + void header() + { + cout << endl; + cout << "Time Seconds Activity" << endl; + } }; // Timer class used by this tdriver to output elapsed times, etc. class MultiTimer { + public: + void start(const string& message); + void stop(const string& message); + void finish() + { + // Calculate the total seconds elapsed. + struct timeval tvStop; + gettimeofday(&tvStop, 0); + double totalSeconds = + (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); + cout << endl; + cout << "Seconds Percentage Calls Description" << endl; -public: - void start(const string& message); - void stop(const string& message); - void finish() + // Add a last entry into the vector for total. + ProcessStats total; + total.fTotalSeconds = totalSeconds; + total.fProcess = "Total"; + total.fStartCount = 1; + fProcessStats.push_back(total); + + for (uint32_t i = 0; i < fProcessStats.size(); i++) { - - // Calculate the total seconds elapsed. - struct timeval tvStop; - gettimeofday(&tvStop, 0); - double totalSeconds = - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); - + if (i == (fProcessStats.size() - 1)) + { cout << endl; - cout << "Seconds Percentage Calls Description" << endl; + } - // Add a last entry into the vector for total. - ProcessStats total; - total.fTotalSeconds = totalSeconds; - total.fProcess = "Total"; - total.fStartCount = 1; - fProcessStats.push_back(total); - - for (uint32_t i = 0; i < fProcessStats.size(); i++) - { - - if (i == (fProcessStats.size() - 1)) - { - cout << endl; - } - - // Seconds. - string seconds; - ostringstream oss; - oss << fProcessStats[i].fTotalSeconds; - seconds = oss.str(); - seconds.resize(7, '0'); - cout << seconds << " "; - - // Percentage. - string percentage; - ostringstream oss2; - oss2 << (fProcessStats[i].fTotalSeconds / totalSeconds) * 100.0; - percentage = oss2.str(); - percentage.resize(5, ' '); - cout << percentage << "% "; - - // Times Initiated. - ostringstream oss3; - oss3 << fProcessStats[i].fStartCount; - string timesInitiated = oss3.str(); - timesInitiated.resize(10, ' '); - cout << timesInitiated << " "; - - // Description. - cout << fProcessStats[i].fProcess << endl; - } + // Seconds. + string seconds; + ostringstream oss; + oss << fProcessStats[i].fTotalSeconds; + seconds = oss.str(); + seconds.resize(7, '0'); + cout << seconds << " "; + // Percentage. + string percentage; + ostringstream oss2; + oss2 << (fProcessStats[i].fTotalSeconds / totalSeconds) * 100.0; + percentage = oss2.str(); + percentage.resize(5, ' '); + cout << percentage << "% "; + // Times Initiated. + ostringstream oss3; + oss3 << fProcessStats[i].fStartCount; + string timesInitiated = oss3.str(); + timesInitiated.resize(10, ' '); + cout << timesInitiated << " "; + // Description. + cout << fProcessStats[i].fProcess << endl; } - MultiTimer() : fStarted(false) {}; + } + MultiTimer() : fStarted(false){}; -private: - class ProcessStats + private: + class ProcessStats + { + public: + string fProcess; + struct timeval fTvProcessStarted; + double fTotalSeconds; + long fStartCount; + long fStopCount; + + ProcessStats() : fProcess(""), fTotalSeconds(0.0), fStartCount(0), fStopCount(0){}; + + void processStart() { - public: + gettimeofday(&fTvProcessStarted, 0); + fStartCount++; + } - string fProcess; - struct timeval fTvProcessStarted; - double fTotalSeconds; - long fStartCount; - long fStopCount; + void processStop() + { + struct timeval tvStop; + gettimeofday(&tvStop, 0); + fStopCount++; + fTotalSeconds += (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - + (fTvProcessStarted.tv_sec + (fTvProcessStarted.tv_usec / 1000000.0)); + } + }; - ProcessStats() : fProcess(""), fTotalSeconds(0.0), fStartCount(0), fStopCount(0) {}; - - void processStart() - { - gettimeofday(&fTvProcessStarted, 0); - fStartCount++; - } - - void processStop() - { - struct timeval tvStop; - gettimeofday(&tvStop, 0); - fStopCount++; - fTotalSeconds += - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvProcessStarted.tv_sec + (fTvProcessStarted.tv_usec / 1000000.0)); - - } - }; - - struct timeval fTvStart; - vector fProcessStats; - bool fStarted; + struct timeval fTvStart; + vector fProcessStats; + bool fStarted; }; void MultiTimer::stop(const string& message) { - bool found = false; - uint32_t idx = 0; + bool found = false; + uint32_t idx = 0; - for (uint32_t i = 0; i < fProcessStats.size(); i++) + for (uint32_t i = 0; i < fProcessStats.size(); i++) + { + if (fProcessStats[i].fProcess == message) { - if (fProcessStats[i].fProcess == message) - { - idx = i; - found = true; - break; - } + idx = i; + found = true; + break; } + } - if (!found) - { - throw std::runtime_error("MultiTimer::stop " + message + " called without calling start first."); - } + if (!found) + { + throw std::runtime_error("MultiTimer::stop " + message + " called without calling start first."); + } - fProcessStats[idx].processStop(); + fProcessStats[idx].processStop(); } void MultiTimer::start(const string& message) { - bool found = false; - uint32_t idx = 0; - ProcessStats processStats; + bool found = false; + uint32_t idx = 0; + ProcessStats processStats; - if (!fStarted) + if (!fStarted) + { + fStarted = true; + gettimeofday(&fTvStart, 0); + } + + for (uint32_t i = 0; i < fProcessStats.size(); i++) + { + if (fProcessStats[i].fProcess == message) { - fStarted = true; - gettimeofday(&fTvStart, 0); + idx = i; + found = true; + break; } + } - for (uint32_t i = 0; i < fProcessStats.size(); i++) - { - if (fProcessStats[i].fProcess == message) - { - idx = i; - found = true; - break; - } - } + if (!found) + { + fProcessStats.push_back(processStats); + idx = fProcessStats.size() - 1; + } - if (!found) - { - fProcessStats.push_back(processStats); - idx = fProcessStats.size() - 1; - } - - fProcessStats[idx].fProcess = message; - fProcessStats[idx].processStart(); + fProcessStats[idx].fProcess = message; + fProcessStats[idx].processStart(); } class functionDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(functionDriver); + CPPUNIT_TEST(FUNCTION_TEST); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE(functionDriver); - CPPUNIT_TEST(FUNCTION_TEST); - CPPUNIT_TEST_SUITE_END(); + private: + ResourceManager fRm; -private: - ResourceManager fRm; + void testDrdFunctions() + { + cout << endl; + cout << "double f(double) functions" << endl; + cout << "---------------------------" << endl; - void testDrdFunctions() + FunctionOperation* fop = FunctionOperation::instance(); + + // functions to test + list functionsToTest; + functionsToTest.push_back("abs"); + functionsToTest.push_back("acos"); + functionsToTest.push_back("asin"); + functionsToTest.push_back("atan"); + functionsToTest.push_back("ceil"); + functionsToTest.push_back("cos"); + functionsToTest.push_back("cosh"); + functionsToTest.push_back("exp"); + functionsToTest.push_back("floor"); + functionsToTest.push_back("ln"); + functionsToTest.push_back("log2"); + functionsToTest.push_back("log10"); + functionsToTest.push_back("sin"); + functionsToTest.push_back("sinh"); + functionsToTest.push_back("sqrt"); + functionsToTest.push_back("tan"); + functionsToTest.push_back("tanh"); + + // type vector is the same for drd functions + vector types; + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + + int numRows = 1; + DoubleElementType el0, el1; + list::iterator it; + + for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) { - cout << endl; - cout << "double f(double) functions" << endl; - cout << "---------------------------" << endl; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); + WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); - FunctionOperation* fop = FunctionOperation::instance(); + for (int i = 0; i < numRows; i++) + { + el1.first = i; + el1.second = (0.5); + wsdlIn1->insert(el1); + } - // functions to test - list functionsToTest; - functionsToTest.push_back("abs"); - functionsToTest.push_back("acos"); - functionsToTest.push_back("asin"); - functionsToTest.push_back("atan"); - functionsToTest.push_back("ceil"); - functionsToTest.push_back("cos"); - functionsToTest.push_back("cosh"); - functionsToTest.push_back("exp"); - functionsToTest.push_back("floor"); - functionsToTest.push_back("ln"); - functionsToTest.push_back("log2"); - functionsToTest.push_back("log10"); - functionsToTest.push_back("sin"); - functionsToTest.push_back("sinh"); - functionsToTest.push_back("sqrt"); - functionsToTest.push_back("tan"); - functionsToTest.push_back("tanh"); + wsdlIn1->endOfInput(); - // type vector is the same for drd functions - vector types; - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listOut(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->doubleDl(wsdlIn1); + parms.push_back(listOut); + parms.push_back(listIn1); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - int numRows = 1; - DoubleElementType el0, el1; - list::iterator it; + int id0 = wsdlOut->getIterator(); + int id1 = wsdlIn1->getIterator(); - for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) - { - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); + for (int i = 0; i < numRows; i++) + { + wsdlIn1->next(id1, &el1); + wsdlOut->next(id0, &el0); + cout << *it << "(" << el1.second << ")=" << el0.second << endl; + } + } + } - for (int i = 0; i < numRows; i++) - { - el1.first = i; - el1.second = (0.5); - wsdlIn1->insert(el1); - } + void testDrddFunctions() + { + cout << endl; + cout << "double f(double, double) functions" << endl; + cout << "---------------------------" << endl; - wsdlIn1->endOfInput(); + FunctionOperation* fop = FunctionOperation::instance(); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listOut(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->doubleDl(wsdlIn1); - parms.push_back(listOut); - parms.push_back(listIn1); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); + list functionsToTest; + functionsToTest.push_back("atan2"); + functionsToTest.push_back("power"); + functionsToTest.push_back("+"); + functionsToTest.push_back("-"); + functionsToTest.push_back("*"); + functionsToTest.push_back("/"); - int id0 = wsdlOut->getIterator(); - int id1 = wsdlIn1->getIterator(); + // type vector is the same for drdd functions + vector types; + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - for (int i = 0; i < numRows; i++) - { - wsdlIn1->next(id1, &el1); - wsdlOut->next(id0, &el0); - cout << *it << "(" << el1.second << ")=" << el0.second << endl; - } - } + int numRows = 1; + DoubleElementType el0, el1, el2; + list::iterator it; + + for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) + { + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); + WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); + WSDL* wsdlIn2 = new WSDL(2, numRows, fRm); + + for (int i = 0; i < numRows; i++) + { + el1.first = i; + el1.second = (0.5); + wsdlIn1->insert(el1); + wsdlIn2->insert(el1); // the data in two list is the same + } + + wsdlIn1->endOfInput(); + wsdlIn2->endOfInput(); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->doubleDl(wsdlIn1); + listIn2->doubleDl(wsdlIn2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); + + int id0 = wsdlOut->getIterator(); + int id1 = wsdlIn1->getIterator(); + int id2 = wsdlIn2->getIterator(); + + for (int i = 0; i < numRows; i++) + { + wsdlIn1->next(id1, &el1); + wsdlIn2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << *it << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } + } + } + + void testSrssFunctions() + { + cout << endl; + cout << "string f(string, string) functions" << endl; + cout << "---------------------------" << endl; + + FunctionOperation* fop = FunctionOperation::instance(); + + list functionsToTest; + functionsToTest.push_back("||"); + functionsToTest.push_back("concat"); + + // type vector is the same for drdd functions + vector types; + types.push_back(FunctionDataList::STRING_LISTTYPE); + types.push_back(FunctionDataList::STRING_LISTTYPE); + types.push_back(FunctionDataList::STRING_LISTTYPE); + + int numRows = 1; + StringElementType el0, el1, el2; + list::iterator it; + + for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) + { + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); + WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); + WSDL* wsdlIn2 = new WSDL(2, numRows, fRm); + + for (int i = 0; i < numRows; i++) + { + el1.first = i; + el1.second = "abc"; + wsdlIn1->insert(el1); + wsdlIn2->insert(el1); // the data in two list is the same + } + + wsdlIn1->endOfInput(); + wsdlIn2->endOfInput(); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->stringDl(wsdlOut); + listIn1->stringDl(wsdlIn1); + listIn2->stringDl(wsdlIn2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); + + int id0 = wsdlOut->getIterator(); + int id1 = wsdlIn1->getIterator(); + int id2 = wsdlIn2->getIterator(); + + for (int i = 0; i < numRows; i++) + { + wsdlIn1->next(id1, &el1); + wsdlIn2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << *it << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } + } + } + + void testSrsFunctions() + { + cout << endl; + cout << "string f(string) functions" << endl; + cout << "---------------------------" << endl; + + FunctionOperation* fop = FunctionOperation::instance(); + + list functionsToTest; + functionsToTest.push_back("upper"); + functionsToTest.push_back("asciistr"); + functionsToTest.push_back("trim"); + functionsToTest.push_back("ltrim"); + functionsToTest.push_back("rtrim"); + + // type vector is the same for drdd functions + vector types; + types.push_back(FunctionDataList::STRING_LISTTYPE); + types.push_back(FunctionDataList::STRING_LISTTYPE); + + int numRows = 2; + StringElementType el0, el1, el2; + list::iterator it; + + for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) + { + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); + WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); + + char abc[][10] = {" abc ", "second"}; + abc[0][3] = 222; // 0xDE + + for (int i = 0; i < numRows; i++) + { + el1.first = i; + el1.second = abc[i]; + wsdlIn1->insert(el1); + } + + wsdlIn1->endOfInput(); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FDLVec parms; + listOut->stringDl(wsdlOut); + listIn1->stringDl(wsdlIn1); + parms.push_back(listOut); + parms.push_back(listIn1); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); + + int id0 = wsdlOut->getIterator(); + int id1 = wsdlIn1->getIterator(); + + for (int i = 0; i < numRows; i++) + { + wsdlIn1->next(id1, &el1); + wsdlOut->next(id0, &el0); + cout << *it << "(\'" << el1.second << "\')=" + << "\'" << el0.second << "\'" << endl; + } + } + } + + void testAddFunctions() + { + cout << endl; + cout << "add(, ) functions" << endl; + cout << "---------------------------" << endl; + + FunctionOperation* fop = FunctionOperation::instance(); + + // the map only have one entry for add_ddd, which is mapped to add(any) + vector types; + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + + int numRows = 1; + + // double list + WSDL* wsdlInD1 = new WSDL(20, numRows, fRm); + WSDL* wsdlInD2 = new WSDL(20, numRows, fRm); + // uint32_t list + WSDL* wsdlInU1 = new WSDL(20, numRows, fRm); + WSDL* wsdlInU2 = new WSDL(20, numRows, fRm); + // string list + WSDL* wsdlInS1 = new WSDL(20, numRows, fRm); + WSDL* wsdlInS2 = new WSDL(20, numRows, fRm); + DoubleElementType eld; + ElementType elu; + StringElementType els; + + for (int i = 0; i < numRows; i++) + { + eld.first = i; + eld.second = (0.5); + wsdlInD1->insert(eld); + wsdlInD2->insert(eld); + elu.first = i; + elu.second = (6); + wsdlInU1->insert(elu); + wsdlInU2->insert(elu); + els.first = i; + els.second = "8.8"; + wsdlInS1->insert(els); + wsdlInS2->insert(els); } - void testDrddFunctions() + wsdlInD1->endOfInput(); + wsdlInD2->endOfInput(); + wsdlInU1->endOfInput(); + wsdlInU2->endOfInput(); + wsdlInS1->endOfInput(); + wsdlInS2->endOfInput(); + + // double add(double, double) { - cout << endl; - cout << "double f(double, double) functions" << endl; - cout << "---------------------------" << endl; + DoubleElementType el0, el1, el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - FunctionOperation* fop = FunctionOperation::instance(); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->doubleDl(wsdlInD1); + listIn2->doubleDl(wsdlInD2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - list functionsToTest; - functionsToTest.push_back("atan2"); - functionsToTest.push_back("power"); - functionsToTest.push_back("+"); - functionsToTest.push_back("-"); - functionsToTest.push_back("*"); - functionsToTest.push_back("/"); + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInD1->getIterator(); + int id2 = wsdlInD2->getIterator(); - // type vector is the same for drdd functions - vector types; - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - - int numRows = 1; - DoubleElementType el0, el1, el2; - list::iterator it; - - for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) - { - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); - WSDL* wsdlIn2 = new WSDL(2, numRows, fRm); - - for (int i = 0; i < numRows; i++) - { - el1.first = i; - el1.second = (0.5); - wsdlIn1->insert(el1); - wsdlIn2->insert(el1); // the data in two list is the same - } - - wsdlIn1->endOfInput(); - wsdlIn2->endOfInput(); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->doubleDl(wsdlIn1); - listIn2->doubleDl(wsdlIn2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlIn1->getIterator(); - int id2 = wsdlIn2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlIn1->next(id1, &el1); - wsdlIn2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << *it << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } + for (int i = 0; i < numRows; i++) + { + wsdlInD1->next(id1, &el1); + wsdlInD2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - void testSrssFunctions() + // double add(double, uint32_t) { - cout << endl; - cout << "string f(string, string) functions" << endl; - cout << "---------------------------" << endl; + DoubleElementType el0, el1; + ElementType el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - FunctionOperation* fop = FunctionOperation::instance(); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->doubleDl(wsdlInD1); + listIn2->uint64Dl(wsdlInU2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - list functionsToTest; - functionsToTest.push_back("||"); - functionsToTest.push_back("concat"); + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInD1->getIterator(); + int id2 = wsdlInU2->getIterator(); - // type vector is the same for drdd functions - vector types; - types.push_back(FunctionDataList::STRING_LISTTYPE); - types.push_back(FunctionDataList::STRING_LISTTYPE); - types.push_back(FunctionDataList::STRING_LISTTYPE); - - int numRows = 1; - StringElementType el0, el1, el2; - list::iterator it; - - for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) - { - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); - WSDL* wsdlIn2 = new WSDL(2, numRows, fRm); - - for (int i = 0; i < numRows; i++) - { - el1.first = i; - el1.second = "abc"; - wsdlIn1->insert(el1); - wsdlIn2->insert(el1); // the data in two list is the same - } - - wsdlIn1->endOfInput(); - wsdlIn2->endOfInput(); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->stringDl(wsdlOut); - listIn1->stringDl(wsdlIn1); - listIn2->stringDl(wsdlIn2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlIn1->getIterator(); - int id2 = wsdlIn2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlIn1->next(id1, &el1); - wsdlIn2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << *it << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } + for (int i = 0; i < numRows; i++) + { + wsdlInD1->next(id1, &el1); + wsdlInU2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - void testSrsFunctions() + // double add(double, string) { - cout << endl; - cout << "string f(string) functions" << endl; - cout << "---------------------------" << endl; + DoubleElementType el0, el1; + StringElementType el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - FunctionOperation* fop = FunctionOperation::instance(); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->doubleDl(wsdlInD1); + listIn2->stringDl(wsdlInS2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - list functionsToTest; - functionsToTest.push_back("upper"); - functionsToTest.push_back("asciistr"); - functionsToTest.push_back("trim"); - functionsToTest.push_back("ltrim"); - functionsToTest.push_back("rtrim"); + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInD1->getIterator(); + int id2 = wsdlInS2->getIterator(); - // type vector is the same for drdd functions - vector types; - types.push_back(FunctionDataList::STRING_LISTTYPE); - types.push_back(FunctionDataList::STRING_LISTTYPE); - - int numRows = 2; - StringElementType el0, el1, el2; - list::iterator it; - - for (it = functionsToTest.begin(); it != functionsToTest.end(); it++) - { - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr(*it, types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - WSDL* wsdlIn1 = new WSDL(2, numRows, fRm); - - char abc[][10] = {" abc ", "second"}; - abc[0][3] = 222; // 0xDE - - for (int i = 0; i < numRows; i++) - { - el1.first = i; - el1.second = abc[i]; - wsdlIn1->insert(el1); - } - - wsdlIn1->endOfInput(); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FDLVec parms; - listOut->stringDl(wsdlOut); - listIn1->stringDl(wsdlIn1); - parms.push_back(listOut); - parms.push_back(listIn1); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlIn1->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlIn1->next(id1, &el1); - wsdlOut->next(id0, &el0); - cout << *it << "(\'" << el1.second << "\')=" << "\'" << el0.second << "\'" << endl; - } - } + for (int i = 0; i < numRows; i++) + { + wsdlInD1->next(id1, &el1); + wsdlInS2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - void testAddFunctions() + // double add(uint32_t, double) { - cout << endl; - cout << "add(, ) functions" << endl; - cout << "---------------------------" << endl; + DoubleElementType el0, el2; + ElementType el1; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - FunctionOperation* fop = FunctionOperation::instance(); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->uint64Dl(wsdlInU1); + listIn2->doubleDl(wsdlInD2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - // the map only have one entry for add_ddd, which is mapped to add(any) - vector types; - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInU1->getIterator(); + int id2 = wsdlInD2->getIterator(); - int numRows = 1; - - // double list - WSDL* wsdlInD1 = new WSDL(20, numRows, fRm); - WSDL* wsdlInD2 = new WSDL(20, numRows, fRm); - // uint32_t list - WSDL* wsdlInU1 = new WSDL(20, numRows, fRm); - WSDL* wsdlInU2 = new WSDL(20, numRows, fRm); - // string list - WSDL* wsdlInS1 = new WSDL(20, numRows, fRm); - WSDL* wsdlInS2 = new WSDL(20, numRows, fRm); - DoubleElementType eld; - ElementType elu; - StringElementType els; - - for (int i = 0; i < numRows; i++) - { - eld.first = i; - eld.second = (0.5); - wsdlInD1->insert(eld); - wsdlInD2->insert(eld); - elu.first = i; - elu.second = (6); - wsdlInU1->insert(elu); - wsdlInU2->insert(elu); - els.first = i; - els.second = "8.8"; - wsdlInS1->insert(els); - wsdlInS2->insert(els); - } - - wsdlInD1->endOfInput(); - wsdlInD2->endOfInput(); - wsdlInU1->endOfInput(); - wsdlInU2->endOfInput(); - wsdlInS1->endOfInput(); - wsdlInS2->endOfInput(); - - // double add(double, double) - { - DoubleElementType el0, el1, el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->doubleDl(wsdlInD1); - listIn2->doubleDl(wsdlInD2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInD1->getIterator(); - int id2 = wsdlInD2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInD1->next(id1, &el1); - wsdlInD2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(double, uint32_t) - { - DoubleElementType el0, el1; - ElementType el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->doubleDl(wsdlInD1); - listIn2->uint64Dl(wsdlInU2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInD1->getIterator(); - int id2 = wsdlInU2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInD1->next(id1, &el1); - wsdlInU2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(double, string) - { - DoubleElementType el0, el1; - StringElementType el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->doubleDl(wsdlInD1); - listIn2->stringDl(wsdlInS2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInD1->getIterator(); - int id2 = wsdlInS2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInD1->next(id1, &el1); - wsdlInS2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(uint32_t, double) - { - DoubleElementType el0, el2; - ElementType el1; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->uint64Dl(wsdlInU1); - listIn2->doubleDl(wsdlInD2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInU1->getIterator(); - int id2 = wsdlInD2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInU1->next(id1, &el1); - wsdlInD2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(uint32_t, uint32_t) - { - DoubleElementType el0; - ElementType el1, el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->uint64Dl(wsdlInU1); - listIn2->uint64Dl(wsdlInU2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInU1->getIterator(); - int id2 = wsdlInU2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInU1->next(id1, &el1); - wsdlInU2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(uint32_t, string) - { - DoubleElementType el0; - ElementType el1; - StringElementType el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->uint64Dl(wsdlInU1); - listIn2->stringDl(wsdlInS2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInU1->getIterator(); - int id2 = wsdlInS2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInU1->next(id1, &el1); - wsdlInS2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(string, double) - { - DoubleElementType el0, el2; - StringElementType el1; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->stringDl(wsdlInS1); - listIn2->doubleDl(wsdlInD2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInS1->getIterator(); - int id2 = wsdlInD2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInS1->next(id1, &el1); - wsdlInD2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(string, uint32_t) - { - DoubleElementType el0; - StringElementType el1; - ElementType el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->stringDl(wsdlInS1); - listIn2->uint64Dl(wsdlInU2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInS1->getIterator(); - int id2 = wsdlInU2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInS1->next(id1, &el1); - wsdlInU2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } - - // double add(string, string) - { - DoubleElementType el0; - StringElementType el1, el2; - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - WSDL* wsdlOut = new WSDL(1, numRows, fRm); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->stringDl(wsdlInS1); - listIn2->stringDl(wsdlInS2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - wsdlOut->endOfInput(); - - int id0 = wsdlOut->getIterator(); - int id1 = wsdlInS1->getIterator(); - int id2 = wsdlInS2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlInS1->next(id1, &el1); - wsdlInS2->next(id2, &el2); - wsdlOut->next(id0, &el0); - cout << "+" << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; - } - } + for (int i = 0; i < numRows; i++) + { + wsdlInU1->next(id1, &el1); + wsdlInD2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - struct dateTime + // double add(uint32_t, uint32_t) { - unsigned msecond : 20; - unsigned second : 6; - unsigned minute : 6; - unsigned hour : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; + DoubleElementType el0; + ElementType el1, el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - dateTime(int y = 0xFFFF, int mn = 0xF, int d = 0x3F, int h = 0x3F, int mi = 0x3F, int s = 0x3F, int ms = 0xFFFFE) - { - year = y; - month = mn; - day = d; - hour = h; - minute = mi; - second = s; - msecond = ms; - } - }; + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->uint64Dl(wsdlInU1); + listIn2->uint64Dl(wsdlInU2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - void testDateFunctions() - { - cout << endl; - cout << "to_date function with formats" << endl; - cout << "-----------------------------" << endl; - - FunctionOperation* fop = FunctionOperation::instance(); - - vector types; - types.push_back(FunctionDataList::STRING_LISTTYPE); - types.push_back(FunctionDataList::UINT64_LISTTYPE); - types.push_back(FunctionDataList::STRING_CONST_LISTTYPE); - - // data list - AnyDataListSPtr spdl1(new AnyDataList()); - BandedDataList* dl1 = new BandedDataList(20, fRm); - spdl1->bandedDL(dl1); - - dateTime dt(2007, 11, 12, 16, 7, 8, 999); - uint64_t idt = *(reinterpret_cast(&dt)); - ElementType elu(0, idt); - - int numRows = 1; - - for (int i = 0; i < numRows; i++) - { - elu.first = i; - dl1->insert(elu); - } - - dl1->endOfInput(); - - char fmt[6][30] = { "YYYYMMDDHH", "YYYYMMDDHHMISS", "YYYYMMDDHHMISSFF", - "YYYY-MM-DD HH:MI:SS", "MON DD, RRRR", - "MM/DD/YY HH24:MI:SS.FF" - }; - - for (int i = 0; i < 6; i++) - { - AnyDataListSPtr spdl2(new AnyDataList()); - StringElementType els(0, fmt[i]); - ConstantDataList* dl2 = new ConstantDataList(els); - spdl2->stringConstantDL(dl2); - dl2->endOfInput(); - - JobStepAssociation inJs; - inJs.outAdd(spdl1); - inJs.outAdd(spdl2); -// jobstep->outputAssociation(outJs); // output of a job step - - AnyDataListSPtr spdlOut(new AnyDataList()); - StringDataList* dlOut = new StringDataList(1, fRm); - spdlOut->strDataList(dlOut); - JobStepAssociation outJs; - outJs.outAdd(spdlOut); -// funJobstep->inputAssociation(inJs); -// funJobstep->outputAssociation(outJs); - - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("to_char", types); - CPPUNIT_ASSERT(fp); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->stringDl(outJs.outAt(0)->stringDataList()); - listIn1->uint64Dl(inJs.outAt(0)->dataList()); - listIn2->stringDl(inJs.outAt(1)->stringDataList()); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - dlOut->endOfInput(); - - int id0 = dlOut->getIterator(); - int id1 = dl1->getIterator(); - int id2 = dl2->getIterator(); - StringElementType el0; - ElementType el1; - StringElementType el2; - - for (int i = 0; i < numRows; i++) - { - dl1->next(id1, &el1); - dl2->next(id2, &el2); - dlOut->next(id0, &el0); - cout << "to_char" << "(" << el1.second << ", \'" << el2.second << "\')=" << el0.second << endl; - } - } - - // default format - { - JobStepAssociation inJs; - inJs.outAdd(spdl1); -// jobstep->outputAssociation(outJs); // output of a job step - - AnyDataListSPtr spdlOut(new AnyDataList()); - StringDataList* dlOut = new StringDataList(1, fRm); - spdlOut->strDataList(dlOut); - JobStepAssociation outJs; - outJs.outAdd(spdlOut); -// funJobstep->inputAssociation(inJs); -// funJobstep->outputAssociation(outJs); - - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("to_char", types); - CPPUNIT_ASSERT(fp); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FDLVec parms; - listOut->stringDl(outJs.outAt(0)->stringDataList()); - listIn1->uint64Dl(inJs.outAt(0)->dataList()); - parms.push_back(listOut); - parms.push_back(listIn1); - fop->executeFunction(fop, fp, parms); - dlOut->endOfInput(); - - int id0 = dlOut->getIterator(); - int id1 = dl1->getIterator(); - StringElementType el0; - ElementType el1; - - for (int i = 0; i < numRows; i++) - { - dl1->next(id1, &el1); - dlOut->next(id0, &el0); - cout << "to_char" << "(" << el1.second << ")=" << el0.second << endl; - } - } + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInU1->getIterator(); + int id2 = wsdlInU2->getIterator(); + for (int i = 0; i < numRows; i++) + { + wsdlInU1->next(id1, &el1); + wsdlInU2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - void testToNumFunctions() + // double add(uint32_t, string) { - cout << endl; - cout << "to_num function, format ignored" << endl; - cout << "-------------------------------" << endl; + DoubleElementType el0; + ElementType el1; + StringElementType el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - FunctionOperation* fop = FunctionOperation::instance(); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->uint64Dl(wsdlInU1); + listIn2->stringDl(wsdlInS2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - vector types; - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::STRING_LISTTYPE); - types.push_back(FunctionDataList::STRING_CONST_LISTTYPE); - - AnyDataListSPtr spdl1(new AnyDataList()); - StringFifoDataList* dl1 = new StringFifoDataList(2, 100); - spdl1->stringDL(dl1); - - const int numRows = 6; - char num[numRows][30] = { "2007111218", "1234567.8", "-123456.78", - "2,007,111,218", "1.23E3", "-1.23E-3" - }; - - StringRowGroup rows; - - for (int i = 0; i < numRows; i++) - { - StringElementType els(0, num[i]); - rows.et[rows.count++] = els; - // dl1->insert(els); - } - - dl1->insert(rows); - dl1->endOfInput(); - - // format model, ignored - AnyDataListSPtr spdl2(new AnyDataList()); - StringElementType els(0, "dummy"); - ConstantDataList* dl2 = new ConstantDataList(els); - spdl2->stringConstantDL(dl2); - dl2->endOfInput(); - - JobStepAssociation inJs; - inJs.outAdd(spdl1); - inJs.outAdd(spdl2); -// jobstep->outputAssociation(outJs); // output of a job step - - AnyDataListSPtr spdlOut(new AnyDataList()); - FIFO* dlOut = new FIFO(1, 100); - spdlOut->doubleDL(dlOut); - JobStepAssociation outJs; - outJs.outAdd(spdlOut); -// funJobstep->inputAssociation(inJs); -// funJobstep->outputAssociation(outJs); - - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("to_number", types); - CPPUNIT_ASSERT(fp); - - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(outJs.outAt(0)->doubleDL()); - listIn1->stringDl(inJs.outAt(0)->stringDataList()); - listIn2->stringDl(inJs.outAt(1)->stringDataList()); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - fop->executeFunction(fop, fp, parms); - dlOut->endOfInput(); - - int id1 = dl1->getIterator(); - int id2 = dl2->getIterator(); - int id0 = dlOut->getIterator(); -// StringElementType el1; - StringRowGroup el1; - StringElementType el2; - DoubleElementType el0; - - for (int i = 0; i < numRows; i++) - { - dl1->next(id1, &el1); - dl2->next(id2, &el2); - dlOut->next(id0, &el0); - cout << "to_number" << "(\'" << el1.et[i].second << "\', \'" << el2.second << "\')=" << setw(20) << setprecision(10) << el0.second << endl; - } + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInU1->getIterator(); + int id2 = wsdlInS2->getIterator(); + for (int i = 0; i < numRows; i++) + { + wsdlInU1->next(id1, &el1); + wsdlInS2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - /* - void testDrdFunctions() - { - cout << endl; - cout << "double f(double) functions" << endl; - cout << "---------------------------" << endl; - FunctionOperation fo; - drdMap_t::iterator it; - drdMap_t map = fo.getDrdMap(); - int numRows = 1; - DoubleElementType el, el2; - for(it = map.begin(); it!= map.end(); it++) - { - CPPUNIT_ASSERT(fo.isDrdFunction(it->first)); - WSDL wsdlIn(2, numRows); - WSDL wsdlOut(1, numRows); - for(int i = 0; i < numRows; i++) { - el.first = i; - el.second = (0.5); - wsdlIn.insert(el); - } - wsdlIn.endOfInput(); - fo.executeDrdFunction(it->first, wsdlIn, wsdlOut); - wsdlOut.endOfInput(); - int id = wsdlIn.getIterator(); - int id2 = wsdlOut.getIterator(); - for(int i = 0; i < numRows; i++) { - wsdlIn.next(id, &el); - wsdlOut.next(id2, &el2); - cout << it->first << "(" << el.second << ")=" << el2.second << endl; - } - } - - - } - - void testDrddFunctions() - { - cout << endl; - cout << "double f(double, double) functions" << endl; - cout << "---------------------------" << endl; - FunctionOperation fo; - drddMap_t::iterator it; - drddMap_t map = fo.getDrddMap(); - int numRows = 1; - DoubleElementType el, el2, el3; - for(it = map.begin(); it!= map.end(); it++) - { - CPPUNIT_ASSERT(fo.isDrddFunction(it->first)); - WSDL wsdlIn(2, numRows); - WSDL wsdlIn2(2, numRows); - WSDL wsdlOut(1, numRows); - for(int i = 0; i < numRows; i++) { - el.first = i; - el.second = (0.5); - wsdlIn.insert(el); - wsdlIn2.insert(el); - } - wsdlIn.endOfInput(); - wsdlIn2.endOfInput(); - fo.executeDrddFunction(it->first, wsdlIn, wsdlIn2, wsdlOut); - wsdlOut.endOfInput(); - int id = wsdlIn.getIterator(); - int id2 = wsdlIn2.getIterator(); - int id3 = wsdlOut.getIterator(); - for(int i = 0; i < numRows; i++) { - wsdlIn.next(id, &el); - wsdlIn2.next(id2, &el2); - wsdlOut.next(id3, &el3); - cout << it->first << "(" << el.second << ", " << el2.second << ")=" << el3.second << endl; - } - } - - } - - void testSrssFunctions() - { - cout << endl; - cout << "string f(string, string) functions" << endl; - cout << "---------------------------" << endl; - FunctionOperation fo; - srssMap_t::iterator it; - srssMap_t map = fo.getSrssMap(); - int numRows = 1; - StringElementType el, el2, el3; - for(it = map.begin(); it!= map.end(); it++) - { - CPPUNIT_ASSERT(fo.isSrssFunction(it->first)); - WSDL wsdlIn(2, numRows); - WSDL wsdlIn2(2, numRows); - WSDL wsdlOut(1, numRows); - for(int i = 0; i < numRows; i++) { - el.first = i; - el.second = "abc"; - wsdlIn.insert(el); - wsdlIn2.insert(el); - } - wsdlIn.endOfInput(); - wsdlIn2.endOfInput(); - fo.executeSrssFunction(it->first, wsdlIn, wsdlIn2, wsdlOut); - wsdlOut.endOfInput(); - int id = wsdlIn.getIterator(); - int id2 = wsdlIn2.getIterator(); - int id3 = wsdlOut.getIterator(); - for(int i = 0; i < numRows; i++) { - wsdlIn.next(id, &el); - wsdlIn2.next(id2, &el2); - wsdlOut.next(id3, &el3); - cout << it->first << "(" << el.second << ", " << el2.second << ")=" << el3.second << endl; - } - } - - } - - void testSrsFunctions() - { - cout << endl; - cout << "string f(string) functions" << endl; - cout << "---------------------------" << endl; - FunctionOperation fo; - srsMap_t::iterator it; - srsMap_t map = fo.getSrsMap(); - int numRows = 2; - StringElementType elIn, elOut; - for(it = map.begin(); it!= map.end(); it++) - { - CPPUNIT_ASSERT(fo.isSrsFunction(it->first)); - WSDL wsdlIn(2, numRows); - WSDL wsdlOut(1, numRows); - char abc[][10] = {" abc ", "second"}; - abc[0][3] = 222; // 0xDE - for(int i = 0; i < numRows; i++) { - elIn.first = i; - elIn.second = abc[i]; - wsdlIn.insert(elIn); - } - wsdlIn.endOfInput(); - fo.executeSrsFunction(it->first, wsdlIn, wsdlOut); - wsdlOut.endOfInput(); - int idi = wsdlIn.getIterator(); - int ido = wsdlOut.getIterator(); - for(int i = 0; i < numRows; i++) { - wsdlIn.next(idi, &elIn); - wsdlOut.next(ido, &elOut); - cout << it->first << "(\"" << elIn.second << "\")=" << "\"" << elOut.second << "\"" << endl; - } - } - - } - */ - -public: - - void FUNCTION_TEST() + // double add(string, double) { - testDrdFunctions(); - testDrddFunctions(); - testSrssFunctions(); - testSrsFunctions(); - testAddFunctions(); - testDateFunctions(); - testToNumFunctions(); + DoubleElementType el0, el2; + StringElementType el1; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->stringDl(wsdlInS1); + listIn2->doubleDl(wsdlInD2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); + + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInS1->getIterator(); + int id2 = wsdlInD2->getIterator(); + + for (int i = 0; i < numRows; i++) + { + wsdlInS1->next(id1, &el1); + wsdlInD2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - // Executes an addition (+ function) between two large DataLists and displays timing results. - // Asserts that the output DataList contains the correct results. - void PERFORMANCE_TEST() + // double add(string, uint32_t) { - cout << endl << endl; - cout << "Performance Test" << endl; - cout << "----------------------------------------------------------------------------------------" << endl; - Timer timer; - int numRows = 1000 * 1000 * 2; - typedef WSDL DoubleWSDL; - DoubleWSDL* wsdlOut = new DoubleWSDL(1, numRows, fRm); - DoubleWSDL* wsdlIn1 = new DoubleWSDL(2, numRows, fRm); - DoubleWSDL* wsdlIn2 = new DoubleWSDL(2, numRows, fRm); + DoubleElementType el0; + StringElementType el1; + ElementType el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - stringstream ss; - ss << "Loading " << numRows << " DoubleElementTypes into a WSDL."; - string message = ss.str(); - timer.start(message); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->stringDl(wsdlInS1); + listIn2->uint64Dl(wsdlInU2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - DoubleElementType el; + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInS1->getIterator(); + int id2 = wsdlInU2->getIterator(); - for (int i = 0; i < numRows; i++) - { - el.first = i; - el.second = (i % 5); - wsdlIn1->insert(el); - } - - wsdlIn1->endOfInput(); - timer.stop(message); - - stringstream ss2; - ss2 << "Loading " << numRows << " DoubleElementTypes into a second WSDL."; - message = ss2.str(); - timer.start(message); - - for (int i = 0; i < numRows; i++) - { - el.first = i; - el.second = (i % 5); - wsdlIn2->insert(el); - } - - wsdlIn2->endOfInput(); - timer.stop(message); - - message = "Building FunctionDataList and vector"; - timer.start(message); - FunctionDataListSPtr listOut(new FunctionDataList()); - FunctionDataListSPtr listIn1(new FunctionDataList()); - FunctionDataListSPtr listIn2(new FunctionDataList()); - FDLVec parms; - listOut->doubleDl(wsdlOut); - listIn1->doubleDl(wsdlIn1); - listIn2->doubleDl(wsdlIn2); - parms.push_back(listOut); - parms.push_back(listIn1); - parms.push_back(listIn2); - - vector types; - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - types.push_back(FunctionDataList::DOUBLE_LISTTYPE); - timer.stop(message); - - message = "Executing addition and loading result into a third WSDL."; - timer.start(message); - FunctionOperation* fop = FunctionOperation::instance(); - FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); - CPPUNIT_ASSERT(fp); - fop->executeFunction(fop, fp, parms); - timer.stop(message); - wsdlOut->endOfInput(); - - message = "Iterating over results and doing asserts."; - timer.start(message); - DoubleElementType el0, el1, el2; - int id0 = wsdlOut->getIterator(); - int id1 = wsdlIn1->getIterator(); - int id2 = wsdlIn2->getIterator(); - - for (int i = 0; i < numRows; i++) - { - wsdlOut->next(id0, &el0); - wsdlIn1->next(id1, &el1); - wsdlIn2->next(id2, &el2); - - CPPUNIT_ASSERT(el0.first == el1.first); - CPPUNIT_ASSERT(el0.second == (el1.second + el2.second)); - } - - timer.stop(message); + for (int i = 0; i < numRows; i++) + { + wsdlInS1->next(id1, &el1); + wsdlInU2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } } - /* - void PERFORMANCE_TEST() - { - cout << endl << endl; - cout << "Performance Test" << endl; - cout << "----------------------------------------------------------------------------------------" << endl; - Timer timer; - int id; - int numRows = 1000 * 1000 * 2; - DoubleElementType el, el2, el3; - WSDL wsdlIn(2, numRows); - WSDL wsdlIn2(2, numRows); - WSDL wsdlOut(1, numRows); - stringstream ss; - ss << "Loading " << numRows << " DoubleElementTypes into a WSDL."; - string message = ss.str(); - timer.start(message); + // double add(string, string) + { + DoubleElementType el0; + StringElementType el1, el2; + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + WSDL* wsdlOut = new WSDL(1, numRows, fRm); - for(int i = 0; i < numRows; i++) { - el.first = i; - el.second = (i%5); - wsdlIn.insert(el); - } - wsdlIn.endOfInput(); - timer.stop(message); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->stringDl(wsdlInS1); + listIn2->stringDl(wsdlInS2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + wsdlOut->endOfInput(); - stringstream ss2; - ss2 << "Loading " << numRows << " DoubleElementTypes into a second WSDL."; - message = ss2.str(); - timer.start(message); + int id0 = wsdlOut->getIterator(); + int id1 = wsdlInS1->getIterator(); + int id2 = wsdlInS2->getIterator(); - for(int i = 0; i < numRows; i++) { - el.first = i; - el.second = (i%5); - wsdlIn2.insert(el); - } - wsdlIn2.endOfInput(); - timer.stop(message); + for (int i = 0; i < numRows; i++) + { + wsdlInS1->next(id1, &el1); + wsdlInS2->next(id2, &el2); + wsdlOut->next(id0, &el0); + cout << "+" + << "(" << el1.second << ", " << el2.second << ")=" << el0.second << endl; + } + } + } - message = "Executing addition and loading result into a third WSDL."; - timer.start(message); - FunctionOperation fo; - fo.executeDrddFunction("+", wsdlIn, wsdlIn2, wsdlOut); - timer.stop(message); - wsdlOut.endOfInput(); + struct dateTime + { + unsigned msecond : 20; + unsigned second : 6; + unsigned minute : 6; + unsigned hour : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; - message = "Iterating over results and doing asserts."; - timer.start(message); - id = wsdlIn.getIterator(); - int id2 = wsdlIn2.getIterator(); - int id3 = wsdlOut.getIterator(); - for(int i = 0; i < numRows; i++) { - wsdlIn.next(id, &el); - wsdlIn2.next(id2, &el2); - wsdlOut.next(id3, &el3); + dateTime(int y = 0xFFFF, int mn = 0xF, int d = 0x3F, int h = 0x3F, int mi = 0x3F, int s = 0x3F, + int ms = 0xFFFFE) + { + year = y; + month = mn; + day = d; + hour = h; + minute = mi; + second = s; + msecond = ms; + } + }; - CPPUNIT_ASSERT(el.first == el3.first); - CPPUNIT_ASSERT((el.second + el2.second) == el3.second); + void testDateFunctions() + { + cout << endl; + cout << "to_date function with formats" << endl; + cout << "-----------------------------" << endl; - } - timer.stop(message); - } - */ + FunctionOperation* fop = FunctionOperation::instance(); + + vector types; + types.push_back(FunctionDataList::STRING_LISTTYPE); + types.push_back(FunctionDataList::UINT64_LISTTYPE); + types.push_back(FunctionDataList::STRING_CONST_LISTTYPE); + + // data list + AnyDataListSPtr spdl1(new AnyDataList()); + BandedDataList* dl1 = new BandedDataList(20, fRm); + spdl1->bandedDL(dl1); + + dateTime dt(2007, 11, 12, 16, 7, 8, 999); + uint64_t idt = *(reinterpret_cast(&dt)); + ElementType elu(0, idt); + + int numRows = 1; + + for (int i = 0; i < numRows; i++) + { + elu.first = i; + dl1->insert(elu); + } + + dl1->endOfInput(); + + char fmt[6][30] = {"YYYYMMDDHH", "YYYYMMDDHHMISS", "YYYYMMDDHHMISSFF", + "YYYY-MM-DD HH:MI:SS", "MON DD, RRRR", "MM/DD/YY HH24:MI:SS.FF"}; + + for (int i = 0; i < 6; i++) + { + AnyDataListSPtr spdl2(new AnyDataList()); + StringElementType els(0, fmt[i]); + ConstantDataList* dl2 = new ConstantDataList(els); + spdl2->stringConstantDL(dl2); + dl2->endOfInput(); + + JobStepAssociation inJs; + inJs.outAdd(spdl1); + inJs.outAdd(spdl2); + // jobstep->outputAssociation(outJs); // output of a job step + + AnyDataListSPtr spdlOut(new AnyDataList()); + StringDataList* dlOut = new StringDataList(1, fRm); + spdlOut->strDataList(dlOut); + JobStepAssociation outJs; + outJs.outAdd(spdlOut); + // funJobstep->inputAssociation(inJs); + // funJobstep->outputAssociation(outJs); + + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("to_char", types); + CPPUNIT_ASSERT(fp); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->stringDl(outJs.outAt(0)->stringDataList()); + listIn1->uint64Dl(inJs.outAt(0)->dataList()); + listIn2->stringDl(inJs.outAt(1)->stringDataList()); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + dlOut->endOfInput(); + + int id0 = dlOut->getIterator(); + int id1 = dl1->getIterator(); + int id2 = dl2->getIterator(); + StringElementType el0; + ElementType el1; + StringElementType el2; + + for (int i = 0; i < numRows; i++) + { + dl1->next(id1, &el1); + dl2->next(id2, &el2); + dlOut->next(id0, &el0); + cout << "to_char" + << "(" << el1.second << ", \'" << el2.second << "\')=" << el0.second << endl; + } + } + + // default format + { + JobStepAssociation inJs; + inJs.outAdd(spdl1); + // jobstep->outputAssociation(outJs); // output of a job step + + AnyDataListSPtr spdlOut(new AnyDataList()); + StringDataList* dlOut = new StringDataList(1, fRm); + spdlOut->strDataList(dlOut); + JobStepAssociation outJs; + outJs.outAdd(spdlOut); + // funJobstep->inputAssociation(inJs); + // funJobstep->outputAssociation(outJs); + + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("to_char", types); + CPPUNIT_ASSERT(fp); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FDLVec parms; + listOut->stringDl(outJs.outAt(0)->stringDataList()); + listIn1->uint64Dl(inJs.outAt(0)->dataList()); + parms.push_back(listOut); + parms.push_back(listIn1); + fop->executeFunction(fop, fp, parms); + dlOut->endOfInput(); + + int id0 = dlOut->getIterator(); + int id1 = dl1->getIterator(); + StringElementType el0; + ElementType el1; + + for (int i = 0; i < numRows; i++) + { + dl1->next(id1, &el1); + dlOut->next(id0, &el0); + cout << "to_char" + << "(" << el1.second << ")=" << el0.second << endl; + } + } + } + + void testToNumFunctions() + { + cout << endl; + cout << "to_num function, format ignored" << endl; + cout << "-------------------------------" << endl; + + FunctionOperation* fop = FunctionOperation::instance(); + + vector types; + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::STRING_LISTTYPE); + types.push_back(FunctionDataList::STRING_CONST_LISTTYPE); + + AnyDataListSPtr spdl1(new AnyDataList()); + StringFifoDataList* dl1 = new StringFifoDataList(2, 100); + spdl1->stringDL(dl1); + + const int numRows = 6; + char num[numRows][30] = {"2007111218", "1234567.8", "-123456.78", "2,007,111,218", "1.23E3", "-1.23E-3"}; + + StringRowGroup rows; + + for (int i = 0; i < numRows; i++) + { + StringElementType els(0, num[i]); + rows.et[rows.count++] = els; + // dl1->insert(els); + } + + dl1->insert(rows); + dl1->endOfInput(); + + // format model, ignored + AnyDataListSPtr spdl2(new AnyDataList()); + StringElementType els(0, "dummy"); + ConstantDataList* dl2 = new ConstantDataList(els); + spdl2->stringConstantDL(dl2); + dl2->endOfInput(); + + JobStepAssociation inJs; + inJs.outAdd(spdl1); + inJs.outAdd(spdl2); + // jobstep->outputAssociation(outJs); // output of a job step + + AnyDataListSPtr spdlOut(new AnyDataList()); + FIFO* dlOut = new FIFO(1, 100); + spdlOut->doubleDL(dlOut); + JobStepAssociation outJs; + outJs.outAdd(spdlOut); + // funJobstep->inputAssociation(inJs); + // funJobstep->outputAssociation(outJs); + + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("to_number", types); + CPPUNIT_ASSERT(fp); + + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(outJs.outAt(0)->doubleDL()); + listIn1->stringDl(inJs.outAt(0)->stringDataList()); + listIn2->stringDl(inJs.outAt(1)->stringDataList()); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + fop->executeFunction(fop, fp, parms); + dlOut->endOfInput(); + + int id1 = dl1->getIterator(); + int id2 = dl2->getIterator(); + int id0 = dlOut->getIterator(); + // StringElementType el1; + StringRowGroup el1; + StringElementType el2; + DoubleElementType el0; + + for (int i = 0; i < numRows; i++) + { + dl1->next(id1, &el1); + dl2->next(id2, &el2); + dlOut->next(id0, &el0); + cout << "to_number" + << "(\'" << el1.et[i].second << "\', \'" << el2.second << "\')=" << setw(20) << setprecision(10) + << el0.second << endl; + } + } + + /* + void testDrdFunctions() + { + cout << endl; + cout << "double f(double) functions" << endl; + cout << "---------------------------" << endl; + FunctionOperation fo; + drdMap_t::iterator it; + drdMap_t map = fo.getDrdMap(); + int numRows = 1; + DoubleElementType el, el2; + for(it = map.begin(); it!= map.end(); it++) + { + CPPUNIT_ASSERT(fo.isDrdFunction(it->first)); + WSDL wsdlIn(2, numRows); + WSDL wsdlOut(1, numRows); + for(int i = 0; i < numRows; i++) { + el.first = i; + el.second = (0.5); + wsdlIn.insert(el); + } + wsdlIn.endOfInput(); + fo.executeDrdFunction(it->first, wsdlIn, wsdlOut); + wsdlOut.endOfInput(); + int id = wsdlIn.getIterator(); + int id2 = wsdlOut.getIterator(); + for(int i = 0; i < numRows; i++) { + wsdlIn.next(id, &el); + wsdlOut.next(id2, &el2); + cout << it->first << "(" << el.second << ")=" << el2.second << endl; + } + } + + + } + + void testDrddFunctions() + { + cout << endl; + cout << "double f(double, double) functions" << endl; + cout << "---------------------------" << endl; + FunctionOperation fo; + drddMap_t::iterator it; + drddMap_t map = fo.getDrddMap(); + int numRows = 1; + DoubleElementType el, el2, el3; + for(it = map.begin(); it!= map.end(); it++) + { + CPPUNIT_ASSERT(fo.isDrddFunction(it->first)); + WSDL wsdlIn(2, numRows); + WSDL wsdlIn2(2, numRows); + WSDL wsdlOut(1, numRows); + for(int i = 0; i < numRows; i++) { + el.first = i; + el.second = (0.5); + wsdlIn.insert(el); + wsdlIn2.insert(el); + } + wsdlIn.endOfInput(); + wsdlIn2.endOfInput(); + fo.executeDrddFunction(it->first, wsdlIn, wsdlIn2, wsdlOut); + wsdlOut.endOfInput(); + int id = wsdlIn.getIterator(); + int id2 = wsdlIn2.getIterator(); + int id3 = wsdlOut.getIterator(); + for(int i = 0; i < numRows; i++) { + wsdlIn.next(id, &el); + wsdlIn2.next(id2, &el2); + wsdlOut.next(id3, &el3); + cout << it->first << "(" << el.second << ", " << el2.second << ")=" << + el3.second << endl; + } + } + + } + + void testSrssFunctions() + { + cout << endl; + cout << "string f(string, string) functions" << endl; + cout << "---------------------------" << endl; + FunctionOperation fo; + srssMap_t::iterator it; + srssMap_t map = fo.getSrssMap(); + int numRows = 1; + StringElementType el, el2, el3; + for(it = map.begin(); it!= map.end(); it++) + { + CPPUNIT_ASSERT(fo.isSrssFunction(it->first)); + WSDL wsdlIn(2, numRows); + WSDL wsdlIn2(2, numRows); + WSDL wsdlOut(1, numRows); + for(int i = 0; i < numRows; i++) { + el.first = i; + el.second = "abc"; + wsdlIn.insert(el); + wsdlIn2.insert(el); + } + wsdlIn.endOfInput(); + wsdlIn2.endOfInput(); + fo.executeSrssFunction(it->first, wsdlIn, wsdlIn2, wsdlOut); + wsdlOut.endOfInput(); + int id = wsdlIn.getIterator(); + int id2 = wsdlIn2.getIterator(); + int id3 = wsdlOut.getIterator(); + for(int i = 0; i < numRows; i++) { + wsdlIn.next(id, &el); + wsdlIn2.next(id2, &el2); + wsdlOut.next(id3, &el3); + cout << it->first << "(" << el.second << ", " << el2.second << ")=" << + el3.second << endl; + } + } + + } + + void testSrsFunctions() + { + cout << endl; + cout << "string f(string) functions" << endl; + cout << "---------------------------" << endl; + FunctionOperation fo; + srsMap_t::iterator it; + srsMap_t map = fo.getSrsMap(); + int numRows = 2; + StringElementType elIn, elOut; + for(it = map.begin(); it!= map.end(); it++) + { + CPPUNIT_ASSERT(fo.isSrsFunction(it->first)); + WSDL wsdlIn(2, numRows); + WSDL wsdlOut(1, numRows); + char abc[][10] = {" abc ", "second"}; + abc[0][3] = 222; // 0xDE + for(int i = 0; i < numRows; i++) { + elIn.first = i; + elIn.second = abc[i]; + wsdlIn.insert(elIn); + } + wsdlIn.endOfInput(); + fo.executeSrsFunction(it->first, wsdlIn, wsdlOut); + wsdlOut.endOfInput(); + int idi = wsdlIn.getIterator(); + int ido = wsdlOut.getIterator(); + for(int i = 0; i < numRows; i++) { + wsdlIn.next(idi, &elIn); + wsdlOut.next(ido, &elOut); + cout << it->first << "(\"" << elIn.second << "\")=" << "\"" << elOut.second << "\"" << endl; + } + } + + } + */ + + public: + void FUNCTION_TEST() + { + testDrdFunctions(); + testDrddFunctions(); + testSrssFunctions(); + testSrsFunctions(); + testAddFunctions(); + testDateFunctions(); + testToNumFunctions(); + } + + // Executes an addition (+ function) between two large DataLists and displays timing results. + // Asserts that the output DataList contains the correct results. + void PERFORMANCE_TEST() + { + cout << endl << endl; + cout << "Performance Test" << endl; + cout << "----------------------------------------------------------------------------------------" + << endl; + Timer timer; + int numRows = 1000 * 1000 * 2; + typedef WSDL DoubleWSDL; + DoubleWSDL* wsdlOut = new DoubleWSDL(1, numRows, fRm); + DoubleWSDL* wsdlIn1 = new DoubleWSDL(2, numRows, fRm); + DoubleWSDL* wsdlIn2 = new DoubleWSDL(2, numRows, fRm); + + stringstream ss; + ss << "Loading " << numRows << " DoubleElementTypes into a WSDL."; + string message = ss.str(); + timer.start(message); + + DoubleElementType el; + + for (int i = 0; i < numRows; i++) + { + el.first = i; + el.second = (i % 5); + wsdlIn1->insert(el); + } + + wsdlIn1->endOfInput(); + timer.stop(message); + + stringstream ss2; + ss2 << "Loading " << numRows << " DoubleElementTypes into a second WSDL."; + message = ss2.str(); + timer.start(message); + + for (int i = 0; i < numRows; i++) + { + el.first = i; + el.second = (i % 5); + wsdlIn2->insert(el); + } + + wsdlIn2->endOfInput(); + timer.stop(message); + + message = "Building FunctionDataList and vector"; + timer.start(message); + FunctionDataListSPtr listOut(new FunctionDataList()); + FunctionDataListSPtr listIn1(new FunctionDataList()); + FunctionDataListSPtr listIn2(new FunctionDataList()); + FDLVec parms; + listOut->doubleDl(wsdlOut); + listIn1->doubleDl(wsdlIn1); + listIn2->doubleDl(wsdlIn2); + parms.push_back(listOut); + parms.push_back(listIn1); + parms.push_back(listIn2); + + vector types; + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + types.push_back(FunctionDataList::DOUBLE_LISTTYPE); + timer.stop(message); + + message = "Executing addition and loading result into a third WSDL."; + timer.start(message); + FunctionOperation* fop = FunctionOperation::instance(); + FunctionOperation::Function_t* fp = fop->getFunctionObjPtr("+", types); + CPPUNIT_ASSERT(fp); + fop->executeFunction(fop, fp, parms); + timer.stop(message); + wsdlOut->endOfInput(); + + message = "Iterating over results and doing asserts."; + timer.start(message); + DoubleElementType el0, el1, el2; + int id0 = wsdlOut->getIterator(); + int id1 = wsdlIn1->getIterator(); + int id2 = wsdlIn2->getIterator(); + + for (int i = 0; i < numRows; i++) + { + wsdlOut->next(id0, &el0); + wsdlIn1->next(id1, &el1); + wsdlIn2->next(id2, &el2); + + CPPUNIT_ASSERT(el0.first == el1.first); + CPPUNIT_ASSERT(el0.second == (el1.second + el2.second)); + } + + timer.stop(message); + } + /* + void PERFORMANCE_TEST() + { + cout << endl << endl; + cout << "Performance Test" << endl; + cout << + "----------------------------------------------------------------------------------------" << endl; Timer + timer; int id; int numRows = 1000 * 1000 * 2; DoubleElementType el, el2, el3; WSDL + wsdlIn(2, numRows); WSDL wsdlIn2(2, numRows); WSDL wsdlOut(1, + numRows); + + stringstream ss; + ss << "Loading " << numRows << " DoubleElementTypes into a WSDL."; + string message = ss.str(); + timer.start(message); + + for(int i = 0; i < numRows; i++) { + el.first = i; + el.second = (i%5); + wsdlIn.insert(el); + } + wsdlIn.endOfInput(); + timer.stop(message); + + stringstream ss2; + ss2 << "Loading " << numRows << " DoubleElementTypes into a second WSDL."; + message = ss2.str(); + timer.start(message); + + for(int i = 0; i < numRows; i++) { + el.first = i; + el.second = (i%5); + wsdlIn2.insert(el); + } + wsdlIn2.endOfInput(); + timer.stop(message); + + message = "Executing addition and loading result into a third WSDL."; + timer.start(message); + FunctionOperation fo; + fo.executeDrddFunction("+", wsdlIn, wsdlIn2, wsdlOut); + timer.stop(message); + wsdlOut.endOfInput(); + + message = "Iterating over results and doing asserts."; + timer.start(message); + id = wsdlIn.getIterator(); + int id2 = wsdlIn2.getIterator(); + int id3 = wsdlOut.getIterator(); + for(int i = 0; i < numRows; i++) { + wsdlIn.next(id, &el); + wsdlIn2.next(id2, &el2); + wsdlOut.next(id3, &el3); + + CPPUNIT_ASSERT(el.first == el3.first); + CPPUNIT_ASSERT((el.second + el2.second) == el3.second); + + } + timer.stop(message); + } + */ }; CPPUNIT_TEST_SUITE_REGISTRATION(functionDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/joblist/tdriver-hashjoin.cpp b/dbcon/joblist/tdriver-hashjoin.cpp index 347409c28..8dead6ca1 100644 --- a/dbcon/joblist/tdriver-hashjoin.cpp +++ b/dbcon/joblist/tdriver-hashjoin.cpp @@ -49,686 +49,671 @@ using namespace std; using namespace joblist; using namespace execplan; - class HashJoinTestDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(HashJoinTestDriver); - CPPUNIT_TEST_SUITE(HashJoinTestDriver); + CPPUNIT_TEST(HashJoin_1); + CPPUNIT_TEST(HashJoin_2); + CPPUNIT_TEST(HashJoin_3); + CPPUNIT_TEST(HashJoin_4); + CPPUNIT_TEST(HashJoin_5); - CPPUNIT_TEST(HashJoin_1); - CPPUNIT_TEST(HashJoin_2); - CPPUNIT_TEST(HashJoin_3); - CPPUNIT_TEST(HashJoin_4); - CPPUNIT_TEST(HashJoin_5); + CPPUNIT_TEST(LeftOuterJoin_1); + CPPUNIT_TEST(LeftOuterJoin_2); + CPPUNIT_TEST(LeftOuterJoin_3); + CPPUNIT_TEST(LeftOuterJoin_4); + CPPUNIT_TEST(RightOuterJoin_1); + CPPUNIT_TEST(RightOuterJoin_2); - CPPUNIT_TEST(LeftOuterJoin_1); - CPPUNIT_TEST(LeftOuterJoin_2); - CPPUNIT_TEST(LeftOuterJoin_3); - CPPUNIT_TEST(LeftOuterJoin_4); - CPPUNIT_TEST(RightOuterJoin_1); - CPPUNIT_TEST(RightOuterJoin_2); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + uint32_t elementCount(BucketDL* dl) const + { + int sz = 0; -private: + if (dl == NULL) + return 0; - uint32_t elementCount( BucketDL < ElementType>* dl ) const + for (uint32_t i = 0; i < dl->bucketCount(); i++) + sz += dl->size(i); + + return sz; + } + JSTimeStamp fTs; + ResourceManager fRm; + + public: + void HashJoin_1() + { + uint64_t maxElems = 32655; + int maxBuckets = 8; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + + A.insert(ElementType(1025, 1)); + A.insert(ElementType(1026, 2)); + A.insert(ElementType(1027, 3)); + A.insert(ElementType(1028, 4)); + + B.insert(ElementType(1034, 4)); + B.insert(ElementType(1035, 4)); + B.insert(ElementType(1036, 2)); + B.insert(ElementType(1037, 4)); + B.insert(ElementType(1041, 1)); + B.insert(ElementType(1042, 2)); + B.insert(ElementType(1043, 3)); + B.insert(ElementType(1044, 4)); + B.insert(ElementType(1045, 2)); + B.insert(ElementType(1046, 3)); + B.insert(ElementType(1047, 3)); + B.insert(ElementType(1048, 1)); + B.insert(ElementType(1049, 5)); + + A.endOfInput(); + B.endOfInput(); + + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); + + hj->performJoin(); + + int csize = elementCount(&C); + int dsize = elementCount(&D); + + // cout << "A " << setA.size() + // << " B " << setB.size() + // << " C " << csize + // << " D " << dsize + // << endl; + + CPPUNIT_ASSERT(csize == 4); + CPPUNIT_ASSERT(dsize == 12); + + } // HashJoin_1 + + void HashJoin_2() + { + uint64_t maxElems = 100; + int maxBuckets = 8; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + + // create A + for (uint64_t idx = 0; idx < maxElems; idx++) { - int sz = 0; - - if (dl == NULL) - return 0; - - for (uint32_t i = 0; i < dl->bucketCount(); i++) - sz += dl->size(i); - - return sz; - - } - JSTimeStamp fTs; - ResourceManager fRm; - -public: - - void HashJoin_1() - { - uint64_t maxElems = 32655; - int maxBuckets = 8; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - - A.insert(ElementType(1025, 1)); - A.insert(ElementType(1026, 2)); - A.insert(ElementType(1027, 3)); - A.insert(ElementType(1028, 4)); - - B.insert(ElementType(1034, 4)); - B.insert(ElementType(1035, 4)); - B.insert(ElementType(1036, 2)); - B.insert(ElementType(1037, 4)); - B.insert(ElementType(1041, 1)); - B.insert(ElementType(1042, 2)); - B.insert(ElementType(1043, 3)); - B.insert(ElementType(1044, 4)); - B.insert(ElementType(1045, 2)); - B.insert(ElementType(1046, 3)); - B.insert(ElementType(1047, 3)); - B.insert(ElementType(1048, 1)); - B.insert(ElementType(1049, 5)); - - A.endOfInput(); - B.endOfInput(); - - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); - - hj->performJoin(); - - int csize = elementCount(&C); - int dsize = elementCount(&D); - - //cout << "A " << setA.size() - // << " B " << setB.size() - // << " C " << csize - // << " D " << dsize - // << endl; - - CPPUNIT_ASSERT(csize == 4); - CPPUNIT_ASSERT(dsize == 12); - - } // HashJoin_1 - - void HashJoin_2() - { - uint64_t maxElems = 100; - int maxBuckets = 8; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - - // create A - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - A.insert(ElementType(idx, idx)); - B.insert(ElementType(idx + maxElems, idx)); - } - - - A.endOfInput(); - B.endOfInput(); - - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); - - //cout << "T2 sz " << csize << " " << dsize << endl; - CPPUNIT_ASSERT(csize == maxElems); - CPPUNIT_ASSERT(dsize == maxElems); - CPPUNIT_ASSERT(dsize == csize); - - } // HashJoin_2 - - void HashJoin_3() - { - uint64_t maxElems = 100; - int maxBuckets = 8; - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - - // create A - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - A.insert(ElementType(idx, idx)); - B.insert(ElementType(idx + maxElems, (-1)*idx)); - } - - A.endOfInput(); - B.endOfInput(); - - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); - - // both sets containt zero - //cout << "T3 sz " << csize << " " << dsize << endl; - - CPPUNIT_ASSERT(csize == 1); - CPPUNIT_ASSERT(dsize == 1); - CPPUNIT_ASSERT(dsize == csize); - } // HashJoin_3 - - void HashJoin_4() - { - uint64_t maxElems = 32655; - int maxBuckets = 128; - int setAMin = 0; - int setAMax = 10000; - int setARange = (setAMax - setAMin) + 1; - int setBMin = 0; - int setBMax = 10000; - int setBRange = (setBMax - setBMin) + 1; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - setARange = 0; - setBRange = 0; - // create A - srand(time(0)*getpid()); - long stime = clock(); - - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - uint64_t aVal = (rand() % setAMax) + 1; - uint64_t bVal = (rand() % setAMax) + 1; - A.insert(ElementType(idx, aVal) ); - B.insert(ElementType(idx + maxElems, bVal) ); - } - - A.endOfInput(); - B.endOfInput(); - long etime = clock(); - - //cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); - - stime = clock(); - hj->performJoin(); - etime = clock(); - //cout << "Join time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); - - //cout << "T4 sz " << csize << " " << dsize << endl; - - // TODO: determine what values to test - CPPUNIT_ASSERT(csize >= 1); - CPPUNIT_ASSERT(dsize >= 1); - - } // HashJoin_4 - - void HashJoin_5() - { - uint64_t maxElems = 32655; - int maxBuckets = 128; - const uint64_t modValue = 10; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - - // create A - for (uint64_t idx = 1; idx < maxElems ; idx++) - { - uint64_t aVal = idx; - uint64_t bVal = idx; - - if (bVal % modValue != 0) - bVal *= (-1); - - A.insert(ElementType(idx, aVal) ); - B.insert(ElementType(idx + maxElems, bVal) ); - } - - A.endOfInput(); - B.endOfInput(); - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); - - //cout << "T5 sz " << csize << " " << dsize << endl; - - // TODO: determine what values to test - CPPUNIT_ASSERT(csize == (uint64_t)(maxElems / modValue)); - CPPUNIT_ASSERT(dsize == (uint64_t)(maxElems / modValue)); - - } // HashJoin_5 - - void LeftOuterJoin_1() - { - - // Outer left join such as A (+) = B. - // All of Bs should be returned with matching As. - uint64_t maxElems = 32655; - int maxBuckets = 8; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - - A.insert(ElementType(1025, 1)); - A.insert(ElementType(1026, 2)); - A.insert(ElementType(1027, 3)); - A.insert(ElementType(1028, 4)); - A.insert(ElementType(1029, 99)); - - B.insert(ElementType(1034, 4)); - B.insert(ElementType(1035, 4)); - B.insert(ElementType(1036, 2)); - B.insert(ElementType(1037, 4)); - B.insert(ElementType(1041, 1)); - B.insert(ElementType(1042, 2)); - B.insert(ElementType(1043, 3)); - B.insert(ElementType(1044, 4)); - B.insert(ElementType(1045, 2)); - B.insert(ElementType(1046, 3)); - B.insert(ElementType(1047, 3)); - B.insert(ElementType(1048, 1)); - B.insert(ElementType(1049, 5)); - - A.endOfInput(); - B.endOfInput(); - - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); - - hj->performJoin(); - - int csize = elementCount(&C); - int dsize = elementCount(&D); - - //cout << "A " << setA.size() - // << " B " << setB.size() - // << " C " << csize - // << " D " << dsize - // << endl; - - CPPUNIT_ASSERT(csize == 4); - CPPUNIT_ASSERT(dsize == 13); - - } // HashJoin_1 - - - // Inserts 1,000,000 values in A. - // Inserts 666,668 values in B half of which match A. - // Peforms left outer join A (+) = B. - // Asserts that we returned 333,334 A values and all 666,668 B values. - void LeftOuterJoin_2() - { - uint64_t maxElems = 1000 * 1000; - int maxBuckets = 128; - int setAMin = 0; - int setAMax = 10000; - int setARange = (setAMax - setAMin) + 1; - int setBMin = 0; - int setBMax = 10000; - int setBRange = (setBMax - setBMin) + 1; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - setARange = 0; - setBRange = 0; - - ElementType el; - ElementType el2; - - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - el.first = idx; - el.second = idx; - el2.first = idx + maxElems; - el2.second = idx; - A.insert(el); - - if (idx % 3 == 0) - { - B.insert(el2); - el2.second = el2.first; - B.insert(el2); - } - } - - A.endOfInput(); - B.endOfInput(); - - //cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); -// cout << "csize=" << csize << "; dsize=" << dsize << endl; - CPPUNIT_ASSERT(csize == 333334); - CPPUNIT_ASSERT(dsize == 666668); - + A.insert(ElementType(idx, idx)); + B.insert(ElementType(idx + maxElems, idx)); } - // Inserts 0 values in A. - // Inserts 666,668 values in B half of which match A. - // Peforms left outer join A (+) = B. - // Asserts that we returned 0 A values and all 666,668 B values. - void LeftOuterJoin_3() + A.endOfInput(); + B.endOfInput(); + + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + + // cout << "T2 sz " << csize << " " << dsize << endl; + CPPUNIT_ASSERT(csize == maxElems); + CPPUNIT_ASSERT(dsize == maxElems); + CPPUNIT_ASSERT(dsize == csize); + + } // HashJoin_2 + + void HashJoin_3() + { + uint64_t maxElems = 100; + int maxBuckets = 8; + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + + // create A + for (uint64_t idx = 0; idx < maxElems; idx++) { - uint64_t maxElems = 1000 * 1000; - int maxBuckets = 128; - int setAMin = 0; - int setAMax = 10000; - int setARange = (setAMax - setAMin) + 1; - int setBMin = 0; - int setBMax = 10000; - int setBRange = (setBMax - setBMin) + 1; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - setARange = 0; - setBRange = 0; - - ElementType el; - ElementType el2; - - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - el.first = idx; - el.second = idx; - el2.first = idx + maxElems; - el2.second = idx; - - // A.insert(el); - if (idx % 3 == 0) - { - B.insert(el2); - el2.second = el2.first; - B.insert(el2); - } - } - - A.endOfInput(); - B.endOfInput(); - - //cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); -// cout << "csize=" << csize << "; dsize=" << dsize << endl; - CPPUNIT_ASSERT(csize == 0); - CPPUNIT_ASSERT(dsize == 666668); - + A.insert(ElementType(idx, idx)); + B.insert(ElementType(idx + maxElems, (-1) * idx)); } - // Inserts 1,000,000 values in A. - // Inserts 0 values in B half of which match A. - // Peforms left outer join A (+) = B. - // Asserts that we returned 0 A values and all 0 B values. - void LeftOuterJoin_4() + A.endOfInput(); + B.endOfInput(); + + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + + // both sets containt zero + // cout << "T3 sz " << csize << " " << dsize << endl; + + CPPUNIT_ASSERT(csize == 1); + CPPUNIT_ASSERT(dsize == 1); + CPPUNIT_ASSERT(dsize == csize); + } // HashJoin_3 + + void HashJoin_4() + { + uint64_t maxElems = 32655; + int maxBuckets = 128; + int setAMin = 0; + int setAMax = 10000; + int setARange = (setAMax - setAMin) + 1; + int setBMin = 0; + int setBMax = 10000; + int setBRange = (setBMax - setBMin) + 1; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + setARange = 0; + setBRange = 0; + // create A + srand(time(0) * getpid()); + long stime = clock(); + + for (uint64_t idx = 0; idx < maxElems; idx++) { - uint64_t maxElems = 1000 * 1000; - int maxBuckets = 128; - int setAMin = 0; - int setAMax = 10000; - int setARange = (setAMax - setAMin) + 1; - int setBMin = 0; - int setBMax = 10000; - int setBRange = (setBMax - setBMin) + 1; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - setARange = 0; - setBRange = 0; - - ElementType el; - ElementType el2; - - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - el.first = idx; - el.second = idx; - el2.first = idx + maxElems; - el2.second = idx; - A.insert(el); - /* - if(idx%3 == 0) { - B.insert(el2); - el2.second = el2.first; - B.insert(el2); - } - */ - } - - A.endOfInput(); - B.endOfInput(); - - //cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); -// cout << "csize=" << csize << "; dsize=" << dsize << endl; - CPPUNIT_ASSERT(csize == 0); - CPPUNIT_ASSERT(dsize == 0); - + uint64_t aVal = (rand() % setAMax) + 1; + uint64_t bVal = (rand() % setAMax) + 1; + A.insert(ElementType(idx, aVal)); + B.insert(ElementType(idx + maxElems, bVal)); } - void RightOuterJoin_1() + A.endOfInput(); + B.endOfInput(); + long etime = clock(); + + // cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); + + stime = clock(); + hj->performJoin(); + etime = clock(); + // cout << "Join time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + + // cout << "T4 sz " << csize << " " << dsize << endl; + + // TODO: determine what values to test + CPPUNIT_ASSERT(csize >= 1); + CPPUNIT_ASSERT(dsize >= 1); + + } // HashJoin_4 + + void HashJoin_5() + { + uint64_t maxElems = 32655; + int maxBuckets = 128; + const uint64_t modValue = 10; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + + // create A + for (uint64_t idx = 1; idx < maxElems; idx++) { + uint64_t aVal = idx; + uint64_t bVal = idx; - // Outer left join such as A (+) = B. - // All of Bs should be returned with matching As. - uint64_t maxElems = 32655; - int maxBuckets = 8; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - - A.insert(ElementType(1025, 1)); - A.insert(ElementType(1026, 2)); - A.insert(ElementType(1027, 3)); - A.insert(ElementType(1028, 4)); - A.insert(ElementType(1029, 99)); - - B.insert(ElementType(1034, 4)); - B.insert(ElementType(1035, 4)); - B.insert(ElementType(1036, 2)); - B.insert(ElementType(1037, 4)); - B.insert(ElementType(1041, 1)); - B.insert(ElementType(1042, 2)); - B.insert(ElementType(1043, 3)); - B.insert(ElementType(1044, 4)); - B.insert(ElementType(1045, 2)); - B.insert(ElementType(1046, 3)); - B.insert(ElementType(1047, 3)); - B.insert(ElementType(1048, 1)); - B.insert(ElementType(1049, 5)); - - A.endOfInput(); - B.endOfInput(); - - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, RIGHTOUTER, &fTs); - - hj->performJoin(); - - int csize = elementCount(&C); - int dsize = elementCount(&D); - - //cout << "A " << setA.size() - // << " B " << setB.size() - // << " C " << csize - // << " D " << dsize - // << endl; - - CPPUNIT_ASSERT(csize == 5); - CPPUNIT_ASSERT(dsize == 12); + if (bVal % modValue != 0) + bVal *= (-1); + A.insert(ElementType(idx, aVal)); + B.insert(ElementType(idx + maxElems, bVal)); } - // Inserts 1,000,000 values in A. - // Inserts 666,668 values in B half of which match A. - // Peforms right outer join A = B.(+) - // Asserts that we returned 1,000,000 A values and all 333,334 B values. - void RightOuterJoin_2() + A.endOfInput(); + B.endOfInput(); + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, INNER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + + // cout << "T5 sz " << csize << " " << dsize << endl; + + // TODO: determine what values to test + CPPUNIT_ASSERT(csize == (uint64_t)(maxElems / modValue)); + CPPUNIT_ASSERT(dsize == (uint64_t)(maxElems / modValue)); + + } // HashJoin_5 + + void LeftOuterJoin_1() + { + // Outer left join such as A (+) = B. + // All of Bs should be returned with matching As. + uint64_t maxElems = 32655; + int maxBuckets = 8; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + + A.insert(ElementType(1025, 1)); + A.insert(ElementType(1026, 2)); + A.insert(ElementType(1027, 3)); + A.insert(ElementType(1028, 4)); + A.insert(ElementType(1029, 99)); + + B.insert(ElementType(1034, 4)); + B.insert(ElementType(1035, 4)); + B.insert(ElementType(1036, 2)); + B.insert(ElementType(1037, 4)); + B.insert(ElementType(1041, 1)); + B.insert(ElementType(1042, 2)); + B.insert(ElementType(1043, 3)); + B.insert(ElementType(1044, 4)); + B.insert(ElementType(1045, 2)); + B.insert(ElementType(1046, 3)); + B.insert(ElementType(1047, 3)); + B.insert(ElementType(1048, 1)); + B.insert(ElementType(1049, 5)); + + A.endOfInput(); + B.endOfInput(); + + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); + + hj->performJoin(); + + int csize = elementCount(&C); + int dsize = elementCount(&D); + + // cout << "A " << setA.size() + // << " B " << setB.size() + // << " C " << csize + // << " D " << dsize + // << endl; + + CPPUNIT_ASSERT(csize == 4); + CPPUNIT_ASSERT(dsize == 13); + + } // HashJoin_1 + + // Inserts 1,000,000 values in A. + // Inserts 666,668 values in B half of which match A. + // Peforms left outer join A (+) = B. + // Asserts that we returned 333,334 A values and all 666,668 B values. + void LeftOuterJoin_2() + { + uint64_t maxElems = 1000 * 1000; + int maxBuckets = 128; + int setAMin = 0; + int setAMax = 10000; + int setARange = (setAMax - setAMin) + 1; + int setBMin = 0; + int setBMax = 10000; + int setBRange = (setBMax - setBMin) + 1; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + setARange = 0; + setBRange = 0; + + ElementType el; + ElementType el2; + + for (uint64_t idx = 0; idx < maxElems; idx++) { - uint64_t maxElems = 1000 * 1000; - int maxBuckets = 128; - int setAMin = 0; - int setAMax = 10000; - int setARange = (setAMax - setAMin) + 1; - int setBMin = 0; - int setBMax = 10000; - int setBRange = (setBMax - setBMin) + 1; - - BucketDL< ElementType > A(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > B(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > C(maxBuckets, 1, maxElems / maxBuckets, fRm); - BucketDL< ElementType > D(maxBuckets, 1, maxElems / maxBuckets, fRm); - A.setHashMode(1); - B.setHashMode(1); - setARange = 0; - setBRange = 0; - - ElementType el; - ElementType el2; - - for (uint64_t idx = 0; idx < maxElems ; idx++) - { - el.first = idx; - el.second = idx; - el2.first = idx + maxElems; - el2.second = idx; - A.insert(el); - - if (idx % 3 == 0) - { - B.insert(el2); - el2.second = el2.first; - B.insert(el2); - } - } - - A.endOfInput(); - B.endOfInput(); - - //cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; - BDLWrapper< ElementType > setA(&A); - BDLWrapper< ElementType > setB(&B); - DataList< ElementType >* resultA(&C); - DataList< ElementType >* resultB(&D); - - HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, RIGHTOUTER, &fTs); - - hj->performJoin(); - - uint64_t csize = elementCount(&C); - uint64_t dsize = elementCount(&D); -// cout << "csize=" << csize << "; dsize=" << dsize << endl; - CPPUNIT_ASSERT(csize == 1000000); - CPPUNIT_ASSERT(dsize == 333334); + el.first = idx; + el.second = idx; + el2.first = idx + maxElems; + el2.second = idx; + A.insert(el); + if (idx % 3 == 0) + { + B.insert(el2); + el2.second = el2.first; + B.insert(el2); + } } + A.endOfInput(); + B.endOfInput(); -}; // + // cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + // cout << "csize=" << csize << "; dsize=" << dsize << endl; + CPPUNIT_ASSERT(csize == 333334); + CPPUNIT_ASSERT(dsize == 666668); + } + + // Inserts 0 values in A. + // Inserts 666,668 values in B half of which match A. + // Peforms left outer join A (+) = B. + // Asserts that we returned 0 A values and all 666,668 B values. + void LeftOuterJoin_3() + { + uint64_t maxElems = 1000 * 1000; + int maxBuckets = 128; + int setAMin = 0; + int setAMax = 10000; + int setARange = (setAMax - setAMin) + 1; + int setBMin = 0; + int setBMax = 10000; + int setBRange = (setBMax - setBMin) + 1; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + setARange = 0; + setBRange = 0; + + ElementType el; + ElementType el2; + + for (uint64_t idx = 0; idx < maxElems; idx++) + { + el.first = idx; + el.second = idx; + el2.first = idx + maxElems; + el2.second = idx; + + // A.insert(el); + if (idx % 3 == 0) + { + B.insert(el2); + el2.second = el2.first; + B.insert(el2); + } + } + + A.endOfInput(); + B.endOfInput(); + + // cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + // cout << "csize=" << csize << "; dsize=" << dsize << endl; + CPPUNIT_ASSERT(csize == 0); + CPPUNIT_ASSERT(dsize == 666668); + } + + // Inserts 1,000,000 values in A. + // Inserts 0 values in B half of which match A. + // Peforms left outer join A (+) = B. + // Asserts that we returned 0 A values and all 0 B values. + void LeftOuterJoin_4() + { + uint64_t maxElems = 1000 * 1000; + int maxBuckets = 128; + int setAMin = 0; + int setAMax = 10000; + int setARange = (setAMax - setAMin) + 1; + int setBMin = 0; + int setBMax = 10000; + int setBRange = (setBMax - setBMin) + 1; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + setARange = 0; + setBRange = 0; + + ElementType el; + ElementType el2; + + for (uint64_t idx = 0; idx < maxElems; idx++) + { + el.first = idx; + el.second = idx; + el2.first = idx + maxElems; + el2.second = idx; + A.insert(el); + /* + if(idx%3 == 0) { + B.insert(el2); + el2.second = el2.first; + B.insert(el2); + } + */ + } + + A.endOfInput(); + B.endOfInput(); + + // cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, LEFTOUTER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + // cout << "csize=" << csize << "; dsize=" << dsize << endl; + CPPUNIT_ASSERT(csize == 0); + CPPUNIT_ASSERT(dsize == 0); + } + + void RightOuterJoin_1() + { + // Outer left join such as A (+) = B. + // All of Bs should be returned with matching As. + uint64_t maxElems = 32655; + int maxBuckets = 8; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + + A.insert(ElementType(1025, 1)); + A.insert(ElementType(1026, 2)); + A.insert(ElementType(1027, 3)); + A.insert(ElementType(1028, 4)); + A.insert(ElementType(1029, 99)); + + B.insert(ElementType(1034, 4)); + B.insert(ElementType(1035, 4)); + B.insert(ElementType(1036, 2)); + B.insert(ElementType(1037, 4)); + B.insert(ElementType(1041, 1)); + B.insert(ElementType(1042, 2)); + B.insert(ElementType(1043, 3)); + B.insert(ElementType(1044, 4)); + B.insert(ElementType(1045, 2)); + B.insert(ElementType(1046, 3)); + B.insert(ElementType(1047, 3)); + B.insert(ElementType(1048, 1)); + B.insert(ElementType(1049, 5)); + + A.endOfInput(); + B.endOfInput(); + + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, RIGHTOUTER, &fTs); + + hj->performJoin(); + + int csize = elementCount(&C); + int dsize = elementCount(&D); + + // cout << "A " << setA.size() + // << " B " << setB.size() + // << " C " << csize + // << " D " << dsize + // << endl; + + CPPUNIT_ASSERT(csize == 5); + CPPUNIT_ASSERT(dsize == 12); + } + + // Inserts 1,000,000 values in A. + // Inserts 666,668 values in B half of which match A. + // Peforms right outer join A = B.(+) + // Asserts that we returned 1,000,000 A values and all 333,334 B values. + void RightOuterJoin_2() + { + uint64_t maxElems = 1000 * 1000; + int maxBuckets = 128; + int setAMin = 0; + int setAMax = 10000; + int setARange = (setAMax - setAMin) + 1; + int setBMin = 0; + int setBMax = 10000; + int setBRange = (setBMax - setBMin) + 1; + + BucketDL A(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL B(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL C(maxBuckets, 1, maxElems / maxBuckets, fRm); + BucketDL D(maxBuckets, 1, maxElems / maxBuckets, fRm); + A.setHashMode(1); + B.setHashMode(1); + setARange = 0; + setBRange = 0; + + ElementType el; + ElementType el2; + + for (uint64_t idx = 0; idx < maxElems; idx++) + { + el.first = idx; + el.second = idx; + el2.first = idx + maxElems; + el2.second = idx; + A.insert(el); + + if (idx % 3 == 0) + { + B.insert(el2); + el2.second = el2.first; + B.insert(el2); + } + } + + A.endOfInput(); + B.endOfInput(); + + // cout << "Build time " << (float)(etime - stime)/(float)CLOCKS_PER_SEC << endl; + BDLWrapper setA(&A); + BDLWrapper setB(&B); + DataList* resultA(&C); + DataList* resultB(&D); + + HashJoin* hj = new HashJoin(setA, setB, resultA, resultB, RIGHTOUTER, &fTs); + + hj->performJoin(); + + uint64_t csize = elementCount(&C); + uint64_t dsize = elementCount(&D); + // cout << "csize=" << csize << "; dsize=" << dsize << endl; + CPPUNIT_ASSERT(csize == 1000000); + CPPUNIT_ASSERT(dsize == 333334); + } + +}; // CPPUNIT_TEST_SUITE_REGISTRATION(HashJoinTestDriver); -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); + runner.addTest(registry.makeTest()); - bool wasSuccessful = runner.run( "", false ); + bool wasSuccessful = runner.run("", false); - return (wasSuccessful ? 0 : 1); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/joblist/tdriver-index.cpp b/dbcon/joblist/tdriver-index.cpp index 9912a7667..e3f03fd08 100644 --- a/dbcon/joblist/tdriver-index.cpp +++ b/dbcon/joblist/tdriver-index.cpp @@ -42,318 +42,310 @@ using namespace std; using namespace joblist; using namespace execplan; - -CalpontSystemCatalog::TableColName testcol = { "tpch", "orders", "o_orderkey"}; -CalpontSystemCatalog::TableColName largecol = { "tpch", "lineitem", "l_orderkey"}; +CalpontSystemCatalog::TableColName testcol = {"tpch", "orders", "o_orderkey"}; +CalpontSystemCatalog::TableColName largecol = {"tpch", "lineitem", "l_orderkey"}; class JobStepDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(JobStepDriver); - CPPUNIT_TEST_SUITE(JobStepDriver); + // CPPUNIT_TEST(indexTest_ss1); + // CPPUNIT_TEST(indexTest_ss2); + // CPPUNIT_TEST(indexTest_many); + // CPPUNIT_TEST(indexTest_many2); + CPPUNIT_TEST(indexTest_lists); -// CPPUNIT_TEST(indexTest_ss1); -// CPPUNIT_TEST(indexTest_ss2); -// CPPUNIT_TEST(indexTest_many); -// CPPUNIT_TEST(indexTest_many2); - CPPUNIT_TEST(indexTest_lists); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + int getIndexOID(const CalpontSystemCatalog::TableColName& col, boost::shared_ptr cat) + { + const CalpontSystemCatalog::IndexNameList iNames = cat->colValueSysindexCol(col); - -private: - int getIndexOID(const CalpontSystemCatalog::TableColName& col, boost::shared_ptr cat) + if (0 == iNames.size()) { - const CalpontSystemCatalog::IndexNameList iNames = cat->colValueSysindexCol(col); - - if (0 == iNames.size()) - { - cout << "No index for " << col << endl; - return -1; - } - - CalpontSystemCatalog::IndexOID ixoid = cat->lookupIndexNbr(*iNames.begin()); - return ixoid.objnum; + cout << "No index for " << col << endl; + return -1; } -public: - // 1 search string - void indexTest_ss1() + CalpontSystemCatalog::IndexOID ixoid = cat->lookupIndexNbr(*iNames.begin()); + return ixoid.objnum; + } + + public: + // 1 search string + void indexTest_ss1() + { + ResourceManager rm; + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation walkJs; + + AnyDataListSPtr spdlw(new AnyDataList()); + BandedDL* dlw = new BandedDL(1, rm); + spdlw->bandedDL(dlw); + walkJs.outAdd(spdlw); + + int oid = getIndexOID(testcol, cat); // returns 3154 + + if (0 > oid) + return; + + pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); + + step0.addSearchStr(COMPARE_EQ, 3); + + step0.run(); + + step0.join(); + + ElementType e; + + JobStepAssociation outJs; + AnyDataListSPtr spdlo(new AnyDataList()); + BandedDL* dlo = new BandedDL(1, rm); + spdlo->bandedDL(dlo); + outJs.outAdd(spdlo); + + pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); + step1.run(); + step1.join(); + + int it = dlo->getIterator(); + int i = 1; + + while (dlo->next(it, &e)) { - ResourceManager rm; - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - JobStepAssociation inJs; - JobStepAssociation walkJs; - - AnyDataListSPtr spdlw(new AnyDataList()); - BandedDL* dlw = new BandedDL(1, rm); - spdlw->bandedDL(dlw); - walkJs.outAdd(spdlw); - - int oid = getIndexOID(testcol, cat); //returns 3154 - - if (0 > oid) return; - - pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); - - step0.addSearchStr(COMPARE_EQ, 3); - - step0.run(); - - step0.join(); - - - ElementType e; - - JobStepAssociation outJs; - AnyDataListSPtr spdlo(new AnyDataList()); - BandedDL* dlo = new BandedDL(1, rm); - spdlo->bandedDL(dlo); - outJs.outAdd(spdlo); - - - pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); - step1.run(); - step1.join(); - - int it = dlo->getIterator(); - int i = 1; - - while (dlo->next(it, &e) ) - { - cout << i++ << " \n"; - } - - + cout << i++ << " \n"; } - // 2 search strings - void indexTest_ss2() + } + // 2 search strings + void indexTest_ss2() + { + ResourceManager rm; + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation walkJs; + + AnyDataListSPtr spdlw(new AnyDataList()); + BandedDL* dlw = new BandedDL(1, rm); + spdlw->bandedDL(dlw); + + walkJs.outAdd(spdlw); + + int oid = getIndexOID(testcol, cat); // returns 3154 + + if (0 > oid) + return; + + pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); + + step0.addSearchStr(COMPARE_GT, 3); + step0.addSearchStr(COMPARE_LT, 60); + step0.setBOP(BOP_AND); + + step0.run(); + + step0.join(); + + JobStepAssociation outJs; + + AnyDataListSPtr spdlo(new AnyDataList()); + BandedDL* dlo = new BandedDL(1, rm); + spdlo->bandedDL(dlo); + outJs.outAdd(spdlo); + + pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); + + step1.run(); + step1.join(); + + int it = dlo->getIterator(); + int i = 1; + ElementType e; + + while (dlo->next(it, &e)) { - ResourceManager rm; - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - JobStepAssociation inJs; - JobStepAssociation walkJs; - - AnyDataListSPtr spdlw(new AnyDataList()); - BandedDL* dlw = new BandedDL(1, rm); - spdlw->bandedDL(dlw); - - - walkJs.outAdd(spdlw); - - int oid = getIndexOID(testcol, cat); //returns 3154 - - if (0 > oid) return; - - pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); - - step0.addSearchStr(COMPARE_GT, 3); - step0.addSearchStr(COMPARE_LT, 60); - step0.setBOP(BOP_AND); - - step0.run(); - - step0.join(); - - JobStepAssociation outJs; - - AnyDataListSPtr spdlo(new AnyDataList()); - BandedDL* dlo = new BandedDL(1, rm); - spdlo->bandedDL(dlo); - outJs.outAdd(spdlo); - - pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); - - step1.run(); - step1.join(); - - int it = dlo->getIterator(); - int i = 1; - ElementType e; - - while (dlo->next(it, &e) ) - { - cout << i++ << " \n"; - } + cout << i++ << " \n"; } - // input list of tokens - void indexTest_many() + } + // input list of tokens + void indexTest_many() + { + ResourceManager rm; + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + // dec->addSession(12345); + // dec->addStep(12345, 0); + + JobStepAssociation inJs; + AnyDataListSPtr spdli(new AnyDataList()); + BandedDL* dli = new BandedDL(1, rm); + spdli->bandedDL(dli); + ElementType e; + + for (e.second = 1; e.second < 100; ++e.second) { - ResourceManager rm; - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - -// dec->addSession(12345); -// dec->addStep(12345, 0); - - JobStepAssociation inJs; - AnyDataListSPtr spdli(new AnyDataList()); - BandedDL* dli = new BandedDL(1, rm); - spdli->bandedDL(dli); - ElementType e; - - for (e.second = 1; e.second < 100; ++e.second) - { - if (0 == e.second % 3 ) - dli->insert(e); - } - - dli->endOfInput(); - inJs.outAdd(spdli); - - JobStepAssociation walkJs; - - AnyDataListSPtr spdlw(new AnyDataList()); - BandedDL* dlw = new BandedDL(1, rm); - spdlw->bandedDL(dlw); - walkJs.outAdd(spdlw); - - int oid = getIndexOID(testcol, cat); //returns 3154 - - if (0 > oid) return; - - - pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); - - step0.run(); - step0.join(); - - JobStepAssociation outJs; - - AnyDataListSPtr spdlo(new AnyDataList()); - BandedDL* dlo = new BandedDL(1, rm); - spdlo->bandedDL(dlo); - outJs.outAdd(spdlo); - - pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); - - step1.run(); - step1.join(); -// dec->removeSession(12345); - - int it = dlo->getIterator(); - int i = 1; - - while (dlo->next(it, &e) ) - { - cout << i++ << " \n"; - } - } - // 2 tokens; should use search string - void indexTest_many2() - { - ResourceManager rm; - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - -// dec->addSession(12345); -// dec->addStep(12345, 0); - - JobStepAssociation inJs; - AnyDataListSPtr spdli(new AnyDataList()); - BandedDL* dli = new BandedDL(1, rm); - spdli->bandedDL(dli); - ElementType e; - e.second = 3; + if (0 == e.second % 3) dli->insert(e); - e.second = 32; - dli->insert(e); - - dli->endOfInput(); - inJs.outAdd(spdli); - - JobStepAssociation walkJs; - - AnyDataListSPtr spdlw(new AnyDataList()); - BandedDL* dlw = new BandedDL(1, rm); - spdlw->bandedDL(dlw); - walkJs.outAdd(spdlw); - - int oid = getIndexOID(testcol, cat); //returns 3154 - - if (0 > oid) return; - - pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); - - step0.run(); - step0.join(); - - JobStepAssociation outJs; - - AnyDataListSPtr spdlo(new AnyDataList()); - BandedDL* dlo = new BandedDL(1, rm); - spdlo->bandedDL(dlo); - outJs.outAdd(spdlo); - - pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); - - step1.run(); - step1.join(); -// dec->removeSession(12345); - - int it = dlo->getIterator(); - int i = 1; - - while (dlo->next(it, &e) ) - { - cout << i++ << " \n"; - } } - //Send enough data so that index list must send it back to primitives - void indexTest_lists() + + dli->endOfInput(); + inJs.outAdd(spdli); + + JobStepAssociation walkJs; + + AnyDataListSPtr spdlw(new AnyDataList()); + BandedDL* dlw = new BandedDL(1, rm); + spdlw->bandedDL(dlw); + walkJs.outAdd(spdlw); + + int oid = getIndexOID(testcol, cat); // returns 3154 + + if (0 > oid) + return; + + pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); + + step0.run(); + step0.join(); + + JobStepAssociation outJs; + + AnyDataListSPtr spdlo(new AnyDataList()); + BandedDL* dlo = new BandedDL(1, rm); + spdlo->bandedDL(dlo); + outJs.outAdd(spdlo); + + pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); + + step1.run(); + step1.join(); + // dec->removeSession(12345); + + int it = dlo->getIterator(); + int i = 1; + + while (dlo->next(it, &e)) { - ResourceManager rm; - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - JobStepAssociation inJs; - JobStepAssociation walkJs; - - AnyDataListSPtr spdlw(new AnyDataList()); - BandedDL* dlw = new BandedDL(1, rm); - spdlw->bandedDL(dlw); - walkJs.outAdd(spdlw); - - int oid = getIndexOID(largecol, cat); //returns 3152 - - if (0 > oid) return; - - - pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); - - step0.addSearchStr(COMPARE_GT, 3); - - step0.run(); - - step0.join(); - - JobStepAssociation outJs; - AnyDataListSPtr spdlo(new AnyDataList()); - BandedDL* dlo = new BandedDL(1, rm); - spdlo->bandedDL(dlo); - outJs.outAdd(spdlo); - - pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); - - step1.run(); - step1.join(); - //cout << "lists returned " << dlo->size() << " values.\n"; + cout << i++ << " \n"; } + } + // 2 tokens; should use search string + void indexTest_many2() + { + ResourceManager rm; + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + // dec->addSession(12345); + // dec->addStep(12345, 0); + JobStepAssociation inJs; + AnyDataListSPtr spdli(new AnyDataList()); + BandedDL* dli = new BandedDL(1, rm); + spdli->bandedDL(dli); + ElementType e; + e.second = 3; + dli->insert(e); + e.second = 32; + dli->insert(e); + + dli->endOfInput(); + inJs.outAdd(spdli); + + JobStepAssociation walkJs; + + AnyDataListSPtr spdlw(new AnyDataList()); + BandedDL* dlw = new BandedDL(1, rm); + spdlw->bandedDL(dlw); + walkJs.outAdd(spdlw); + + int oid = getIndexOID(testcol, cat); // returns 3154 + + if (0 > oid) + return; + + pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); + + step0.run(); + step0.join(); + + JobStepAssociation outJs; + + AnyDataListSPtr spdlo(new AnyDataList()); + BandedDL* dlo = new BandedDL(1, rm); + spdlo->bandedDL(dlo); + outJs.outAdd(spdlo); + + pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); + + step1.run(); + step1.join(); + // dec->removeSession(12345); + + int it = dlo->getIterator(); + int i = 1; + + while (dlo->next(it, &e)) + { + cout << i++ << " \n"; + } + } + // Send enough data so that index list must send it back to primitives + void indexTest_lists() + { + ResourceManager rm; + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + boost::shared_ptr cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation walkJs; + + AnyDataListSPtr spdlw(new AnyDataList()); + BandedDL* dlw = new BandedDL(1, rm); + spdlw->bandedDL(dlw); + walkJs.outAdd(spdlw); + + int oid = getIndexOID(largecol, cat); // returns 3152 + + if (0 > oid) + return; + + pIdxWalk step0(inJs, walkJs, dec, cat, oid, 12345, 999, 7, 0, 0, 0, 0); + + step0.addSearchStr(COMPARE_GT, 3); + + step0.run(); + + step0.join(); + + JobStepAssociation outJs; + AnyDataListSPtr spdlo(new AnyDataList()); + BandedDL* dlo = new BandedDL(1, rm); + spdlo->bandedDL(dlo); + outJs.outAdd(spdlo); + + pIdxList step1(walkJs, outJs, dec, cat, 12345, 999, 7, 0, 0, 0, 0); + + step1.run(); + step1.join(); + // cout << "lists returned " << dlo->size() << " values.\n"; + } }; CPPUNIT_TEST_SUITE_REGISTRATION(JobStepDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/joblist/tdriver-jobstep.cpp b/dbcon/joblist/tdriver-jobstep.cpp index 093856fc4..2a6870481 100644 --- a/dbcon/joblist/tdriver-jobstep.cpp +++ b/dbcon/joblist/tdriver-jobstep.cpp @@ -41,1038 +41,1029 @@ const uint64_t ZDL_VEC_SIZE = 4096; class JobStepDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(JobStepDriver); - CPPUNIT_TEST_SUITE(JobStepDriver); + /* These rely on Patrick's DB */ + // CPPUNIT_TEST(pColScan_1); + // CPPUNIT_TEST(pColStep_1); + // CPPUNIT_TEST(pColStep_2); + // CPPUNIT_TEST(pColStep_as_ProjectionStep_1); - /* These rely on Patrick's DB */ -// CPPUNIT_TEST(pColScan_1); -// CPPUNIT_TEST(pColStep_1); -// CPPUNIT_TEST(pColStep_2); -// CPPUNIT_TEST(pColStep_as_ProjectionStep_1); + // CPPUNIT_TEST(pnljoin_1); // value list, no rid list, no reduction step + // CPPUNIT_TEST(pnljoin_2); // value list, w/rid list, no reduction step + // CPPUNIT_TEST(pnljoin_3); // value list + rid list + reduction step -// CPPUNIT_TEST(pnljoin_1); // value list, no rid list, no reduction step -// CPPUNIT_TEST(pnljoin_2); // value list, w/rid list, no reduction step -// CPPUNIT_TEST(pnljoin_3); // value list + rid list + reduction step + CPPUNIT_TEST(reduceStep_1); // ElementType + CPPUNIT_TEST(reduceStep_2); // StringElementType + // CPPUNIT_TEST(reduceStep_3); // DoubleElementType + // CPPUNIT_TEST(reduceStep_4); // reduceStep_1 with BucketDLs as inputs - CPPUNIT_TEST(reduceStep_1); // ElementType - CPPUNIT_TEST(reduceStep_2); // StringElementType -//CPPUNIT_TEST(reduceStep_3); // DoubleElementType -//CPPUNIT_TEST(reduceStep_4); // reduceStep_1 with BucketDLs as inputs + CPPUNIT_TEST(unionStep_1); + // CPPUNIT_TEST(unionStep_2); + // CPPUNIT_TEST(unionStep_3); - CPPUNIT_TEST(unionStep_1); -//CPPUNIT_TEST(unionStep_2); -//CPPUNIT_TEST(unionStep_3); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + ResourceManager fRm; - ResourceManager fRm; + private: + public: + void pColScan_1() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; -private: -public: + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - void pColScan_1() + JobStepAssociation inJs; + JobStepAssociation outJs; + + AnyDataListSPtr spdl1(new AnyDataList()); + BandedDL* dl1 = new BandedDL(1, fRm); + spdl1->bandedDL(dl1); + outJs.outAdd(spdl1); + + pColScanStep step0(inJs, outJs, dec, cat, 1003, 1000, 12345, 999, 7, 0, 0, fRm); + int8_t cop; + int64_t filterValue; + cop = COMPARE_GE; + filterValue = 3010; + step0.addFilter(cop, filterValue); + cop = COMPARE_LE; + filterValue = 3318; + step0.addFilter(cop, filterValue); + step0.setBOP(BOP_AND); + inJs = outJs; + + step0.run(); + + step0.join(); + + DeliveryStep step1(inJs, outJs, make_table("CALPONTSYS", "SYSTABLE"), cat, 10000, 0, 0, 0); + inJs = outJs; + + step1.run(); + + step1.join(); + } + + void pColStep_1() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + int i, it; + bool more; + + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation outJs; + + AnyDataListSPtr spdl1(new AnyDataList()); + BandedDL* dl1 = new BandedDL(1, fRm); + spdl1->bandedDL(dl1); + inJs.outAdd(spdl1); + + AnyDataListSPtr spdl2(new AnyDataList()); + BandedDL* dl2 = new BandedDL(1, fRm); + spdl2->bandedDL(dl2); + outJs.outAdd(spdl2); + + for (i = 10; i < 15; i++) { - DistributedEngineComm* dec; - boost::shared_ptr cat; - - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - JobStepAssociation inJs; - JobStepAssociation outJs; - - AnyDataListSPtr spdl1(new AnyDataList()); - BandedDL* dl1 = new BandedDL(1, fRm); - spdl1->bandedDL(dl1); - outJs.outAdd(spdl1); - - pColScanStep step0(inJs, outJs, dec, cat, 1003, 1000, 12345, 999, 7, 0, 0, fRm); - int8_t cop; - int64_t filterValue; - cop = COMPARE_GE; - filterValue = 3010; - step0.addFilter(cop, filterValue); - cop = COMPARE_LE; - filterValue = 3318; - step0.addFilter(cop, filterValue); - step0.setBOP(BOP_AND); - inJs = outJs; - - step0.run(); - - step0.join(); - - DeliveryStep step1(inJs, outJs, make_table("CALPONTSYS", "SYSTABLE"), cat, 10000, 0, 0, 0); - inJs = outJs; - - step1.run(); - - step1.join(); + e.first = i; + dl1->insert(e); } - void pColStep_1() - { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - int i, it; - bool more; + dl1->endOfInput(); - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + pColStep p(inJs, outJs, dec, cat, 1003, 1000, 12346, 11, 11, 1, 0, fRm); - JobStepAssociation inJs; - JobStepAssociation outJs; + p.setRidList(dl1); // JSA should do this + p.run(); + p.join(); - AnyDataListSPtr spdl1(new AnyDataList()); - BandedDL* dl1 = new BandedDL(1, fRm); - spdl1->bandedDL(dl1); - inJs.outAdd(spdl1); + it = dl2->getIterator(); - AnyDataListSPtr spdl2(new AnyDataList()); - BandedDL* dl2 = new BandedDL(1, fRm); - spdl2->bandedDL(dl2); - outJs.outAdd(spdl2); - - for (i = 10; i < 15; i++) - { - e.first = i; - dl1->insert(e); - } - - dl1->endOfInput(); - - pColStep p(inJs, outJs, dec, cat, 1003, 1000, 12346, 11, 11, 1, 0, fRm); - - p.setRidList(dl1); // JSA should do this - p.run(); - p.join(); - - it = dl2->getIterator(); - - for (more = dl2->next(it, &e), i = 0; more; more = dl2->next(it, &e), i++) + for (more = dl2->next(it, &e), i = 0; more; more = dl2->next(it, &e), i++) #ifdef DEBUG - cout << "" << endl; + cout << "" << endl; #else - ; // walk the list silently + ; // walk the list silently #endif - CPPUNIT_ASSERT(i == 5); + CPPUNIT_ASSERT(i == 5); + } + + /* make sure it issues multiple primitive msgs correctly */ + void pColStep_2() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + int i, it; + bool more; + + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation outJs; + + AnyDataListSPtr spdl1(new AnyDataList()); + BandedDL* dl1 = new BandedDL(1, fRm); + spdl1->bandedDL(dl1); + inJs.outAdd(spdl1); + + AnyDataListSPtr spdl2(new AnyDataList()); + BandedDL* dl2 = new BandedDL(1, fRm); + spdl2->bandedDL(dl2); + outJs.outAdd(spdl2); + + for (i = 10; i < 10000; i++) + { + if (i % 2 == 0) // make it sparse + { + e.first = i; + dl1->insert(e); + } } - /* make sure it issues multiple primitive msgs correctly */ - void pColStep_2() - { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - int i, it; - bool more; + dl1->endOfInput(); - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + pColStep p(inJs, outJs, dec, cat, 1003, 1000, 12347, 11, 11, 1, 0, fRm); - JobStepAssociation inJs; - JobStepAssociation outJs; + p.setRidList(dl1); // JSA should do this + p.run(); + p.join(); - AnyDataListSPtr spdl1(new AnyDataList()); - BandedDL* dl1 = new BandedDL(1, fRm); - spdl1->bandedDL(dl1); - inJs.outAdd(spdl1); + it = dl2->getIterator(); - AnyDataListSPtr spdl2(new AnyDataList()); - BandedDL* dl2 = new BandedDL(1, fRm); - spdl2->bandedDL(dl2); - outJs.outAdd(spdl2); - - for (i = 10; i < 10000; i++) - { - if (i % 2 == 0) // make it sparse - { - e.first = i; - dl1->insert(e); - } - } - - dl1->endOfInput(); - - pColStep p(inJs, outJs, dec, cat, 1003, 1000, 12347, 11, 11, 1, 0, fRm); - - p.setRidList(dl1); // JSA should do this - p.run(); - p.join(); - - it = dl2->getIterator(); - - for (more = dl2->next(it, &e), i = 0; more; more = dl2->next(it, &e), i++) + for (more = dl2->next(it, &e), i = 0; more; more = dl2->next(it, &e), i++) #ifdef DEBUG - cout << "" << endl; + cout << "" << endl; #else - ; // walk the list silently + ; // walk the list silently #endif - CPPUNIT_ASSERT(i == 6); + CPPUNIT_ASSERT(i == 6); + } + + void pColStep_as_ProjectionStep_1() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + int i, it; + bool more; + + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation outJs; + + AnyDataListSPtr spdl1(new AnyDataList()); + BandedDL* dl1 = new BandedDL(1, fRm); + spdl1->bandedDL(dl1); + inJs.outAdd(spdl1); + + AnyDataListSPtr spdl2(new AnyDataList()); + BandedDL* dl2 = new BandedDL(1, fRm); + spdl2->bandedDL(dl2); + outJs.outAdd(spdl2); + + for (i = 1; i <= 21; i++) + { + e.first = i; + dl1->insert(e); } - void pColStep_as_ProjectionStep_1() - { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - int i, it; - bool more; + dl1->endOfInput(); - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + // flushInterval = 8 + pColStep p(inJs, outJs, dec, cat, 1003, 1000, 12348, 11, 11, 1, 2, fRm); - JobStepAssociation inJs; - JobStepAssociation outJs; + p.setRidList(dl1); // JSA should do this + p.run(); + p.join(); - AnyDataListSPtr spdl1(new AnyDataList()); - BandedDL* dl1 = new BandedDL(1, fRm); - spdl1->bandedDL(dl1); - inJs.outAdd(spdl1); + it = dl2->getIterator(); - AnyDataListSPtr spdl2(new AnyDataList()); - BandedDL* dl2 = new BandedDL(1, fRm); - spdl2->bandedDL(dl2); - outJs.outAdd(spdl2); - - for (i = 1; i <= 21; i++) - { - e.first = i; - dl1->insert(e); - } - - dl1->endOfInput(); - - // flushInterval = 8 - pColStep p(inJs, outJs, dec, cat, 1003, 1000, 12348, 11, 11, 1, 2, fRm); - - p.setRidList(dl1); // JSA should do this - p.run(); - p.join(); - - it = dl2->getIterator(); - - for (more = dl2->next(it, &e), i = 0; more; more = dl2->next(it, &e), i++) + for (more = dl2->next(it, &e), i = 0; more; more = dl2->next(it, &e), i++) #ifdef DEBUG - cout << "" << endl; + cout << "" << endl; #else - ; // walk the list silently + ; // walk the list silently #endif -// CPPUNIT_ASSERT(i == 5); + // CPPUNIT_ASSERT(i == 5); + } + + void reduceStep_1() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + WSDL* outDL; + ElementType e; + unsigned i; + int it; + bool more; + + ZDL* inDL = new ZDL(1, fRm); + ZDL* dDL = new ZDL(1, fRm); + outDL = new WSDL(1, 100000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->zonedDL(inDL); + driverADL->zonedDL(dDL); + outputADL->workingSetDL(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); + + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); + + // cout << "making input DataList" << endl; + vector vec1; + vector vec2; + + for (i = 0; i < ::count; i++) + { + e.first = i; + e.second = i + 1; + vec1.push_back(e); + + if (vec1.size() >= ZDL_VEC_SIZE) + { + inDL->insert(vec1); + vec1.clear(); + } + + // inDL->insert(e); } - void reduceStep_1() + if (!vec1.empty()) + inDL->insert(vec1); + + inDL->endOfInput(); + + // cout << "making driver DataList" << endl; + + for (i = 0; i < ::count; i += 2) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - WSDL* outDL; - ElementType e; - unsigned i; - int it; - bool more; + e.first = i; + e.second = i + 1; + vec2.push_back(e); + if (vec2.size() >= ZDL_VEC_SIZE) + { + dDL->insert(vec2); + vec2.clear(); + } - - ZDL* inDL = new ZDL(1, fRm); - ZDL* dDL = new ZDL(1, fRm); - outDL = new WSDL(1, 100000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->zonedDL(inDL); - driverADL->zonedDL(dDL); - outputADL->workingSetDL(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); - - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); - -// cout << "making input DataList" << endl; - vector vec1; - vector vec2; - - for (i = 0; i < ::count; i++) - { - e.first = i; - e.second = i + 1; - vec1.push_back(e); - - if (vec1.size() >= ZDL_VEC_SIZE) - { - inDL->insert(vec1); - vec1.clear(); - } - -// inDL->insert(e); - } - - if (!vec1.empty()) - inDL->insert(vec1); - - inDL->endOfInput(); - -// cout << "making driver DataList" << endl; - - for (i = 0; i < ::count; i += 2) - { - e.first = i; - e.second = i + 1; - vec2.push_back(e); - - if (vec2.size() >= ZDL_VEC_SIZE) - { - dDL->insert(vec2); - vec2.clear(); - } - -// dDL->insert(e); - } - - if (!vec2.empty()) - dDL->insert(vec2); - - dDL->endOfInput(); - -// cout << "reducing" << endl; - - ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); - rs.run(); - rs.join(); - - it = outDL->getIterator(); - more = outDL->next(it, &e); - i = 0; - - while (more) - { -// cout << i << ": first: " << e.first << " second: " << e.second << endl; - CPPUNIT_ASSERT(e.first < ::count); - CPPUNIT_ASSERT(e.first % 2 == 0); - more = outDL->next(it, &e); - i++; - } - + // dDL->insert(e); } - void reduceStep_2() + if (!vec2.empty()) + dDL->insert(vec2); + + dDL->endOfInput(); + + // cout << "reducing" << endl; + + ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); + rs.run(); + rs.join(); + + it = outDL->getIterator(); + more = outDL->next(it, &e); + i = 0; + + while (more) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - WSDL* outDL; - StringElementType e; - unsigned i; - int it; - bool more; + // cout << i << ": first: " << e.first << " second: " << e.second << endl; + CPPUNIT_ASSERT(e.first < ::count); + CPPUNIT_ASSERT(e.first % 2 == 0); + more = outDL->next(it, &e); + i++; + } + } -// inDL = new WSDL(1, 10000, fRm); -// dDL = new WSDL(1, 10000, fRm); - ZDL* inDL = new ZDL(1, fRm); - ZDL* dDL = new ZDL(1, fRm); + void reduceStep_2() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + WSDL* outDL; + StringElementType e; + unsigned i; + int it; + bool more; - outDL = new WSDL(1, 10000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->stringZonedDL(inDL); - driverADL->stringZonedDL(dDL); - outputADL->strDataList(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); + // inDL = new WSDL(1, 10000, fRm); + // dDL = new WSDL(1, 10000, fRm); + ZDL* inDL = new ZDL(1, fRm); + ZDL* dDL = new ZDL(1, fRm); - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); - vector vec1; - vector vec2; + outDL = new WSDL(1, 10000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->stringZonedDL(inDL); + driverADL->stringZonedDL(dDL); + outputADL->strDataList(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); - for (i = 0; i < ::count; i++) + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); + vector vec1; + vector vec2; + + for (i = 0; i < ::count; i++) + { + e.first = i; + e.second = string("blahblahblahblahblah"); + vec1.push_back(e); + + if (vec1.size() >= ZDL_VEC_SIZE) + { + inDL->insert(vec1); + vec1.clear(); + } + + if (0 == i % 2) + { + e.second = string("blahblahblah"); + vec2.push_back(e); + + if (vec2.size() >= ZDL_VEC_SIZE) { - e.first = i; - e.second = string("blahblahblahblahblah"); - vec1.push_back(e); - - if (vec1.size() >= ZDL_VEC_SIZE) - { - inDL->insert(vec1); - vec1.clear(); - } - - if (0 == i % 2) - { - e.second = string("blahblahblah"); - vec2.push_back(e); - - if (vec2.size() >= ZDL_VEC_SIZE) - { - dDL->insert(vec2); - vec2.clear(); - } - } - -// inDL->insert(e); + dDL->insert(vec2); + vec2.clear(); } + } - if (!vec1.empty()) - inDL->insert(vec1); - - inDL->endOfInput(); - - /* for (i = 0; i < ::count; i+=2) { - e.first = i; - e.second = string("blahblahblah"); - dDL->insert(e); - }*/ - if (!vec2.empty()) - dDL->insert(vec2); - - dDL->endOfInput(); - - ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); - rs.run(); - rs.join(); - - it = outDL->getIterator(); - more = outDL->next(it, &e); - i = 0; - - while (more) - { -// cout << i << ": first: " << e.first << " second: " << e.second << endl; - CPPUNIT_ASSERT(e.first < ::count); - CPPUNIT_ASSERT(e.first % 2 == 0); - more = outDL->next(it, &e); - i++; - } + // inDL->insert(e); } - void reduceStep_3() + if (!vec1.empty()) + inDL->insert(vec1); + + inDL->endOfInput(); + + /* for (i = 0; i < ::count; i+=2) { + e.first = i; + e.second = string("blahblahblah"); + dDL->insert(e); + }*/ + if (!vec2.empty()) + dDL->insert(vec2); + + dDL->endOfInput(); + + ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); + rs.run(); + rs.join(); + + it = outDL->getIterator(); + more = outDL->next(it, &e); + i = 0; + + while (more) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - WSDL* inDL, *dDL, *outDL; - DoubleElementType e; - unsigned i; - int it; - bool more; + // cout << i << ": first: " << e.first << " second: " << e.second << endl; + CPPUNIT_ASSERT(e.first < ::count); + CPPUNIT_ASSERT(e.first % 2 == 0); + more = outDL->next(it, &e); + i++; + } + } - inDL = new WSDL(1, 100000, fRm); - dDL = new WSDL(1, 100000, fRm); - outDL = new WSDL(1, 100000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->doubleDL(inDL); - driverADL->doubleDL(dDL); - outputADL->doubleDL(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); + void reduceStep_3() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + WSDL*inDL, *dDL, *outDL; + DoubleElementType e; + unsigned i; + int it; + bool more; - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); + inDL = new WSDL(1, 100000, fRm); + dDL = new WSDL(1, 100000, fRm); + outDL = new WSDL(1, 100000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->doubleDL(inDL); + driverADL->doubleDL(dDL); + outputADL->doubleDL(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); - for (i = 0; i < ::count; i++) - { - e.first = i; - e.second = ((double) i) + 0.1; - inDL->insert(e); - } + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); - inDL->endOfInput(); - - for (i = 0; i < ::count; i += 2) - { - e.first = i; - e.second = ((double) i) + 0.1; - dDL->insert(e); - } - - dDL->endOfInput(); - - ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); - rs.run(); - rs.join(); - - it = outDL->getIterator(); - more = outDL->next(it, &e); - i = 0; - - while (more) - { -// cout << i << ": first: " << e.first << " second: " << e.second << endl; - CPPUNIT_ASSERT(e.first < ::count); - CPPUNIT_ASSERT(e.first % 2 == 0); - more = outDL->next(it, &e); - i++; - } + for (i = 0; i < ::count; i++) + { + e.first = i; + e.second = ((double)i) + 0.1; + inDL->insert(e); } - void reduceStep_4() + inDL->endOfInput(); + + for (i = 0; i < ::count; i += 2) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - BucketDL* inDL, *dDL; - WSDL* outDL; - ElementType e; - unsigned i; - int it; - bool more; - - inDL = new BucketDL(10, 1, 100000, fRm); - dDL = new BucketDL(10, 1, 100000, fRm); - outDL = new WSDL(1, 100000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->bucketDL(inDL); - driverADL->bucketDL(dDL); - outputADL->workingSetDL(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); - - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); - -// cout << "making input DataList" << endl; - - for (i = 0; i < ::count; i++) - { - e.first = i; - e.second = i + 1; - inDL->insert(e); - } - - inDL->endOfInput(); - -// cout << "making driver DataList" << endl; - - for (i = 0; i < ::count; i += 2) - { - e.first = i; - e.second = i + 1; - dDL->insert(e); - } - - dDL->endOfInput(); - -// cout << "reducing" << endl; - - ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); - rs.run(); - rs.join(); - - it = outDL->getIterator(); - more = outDL->next(it, &e); - i = 0; - - while (more) - { -// cout << i << ": first: " << e.first << " second: " << e.second << endl; - CPPUNIT_ASSERT(e.first < ::count); - CPPUNIT_ASSERT(e.first % 2 == 0); - more = outDL->next(it, &e); - i++; - } + e.first = i; + e.second = ((double)i) + 0.1; + dDL->insert(e); } - void pnljoin_1() + dDL->endOfInput(); + + ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); + rs.run(); + rs.join(); + + it = outDL->getIterator(); + more = outDL->next(it, &e); + i = 0; + + while (more) { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - int it; - bool more; + // cout << i << ": first: " << e.first << " second: " << e.second << endl; + CPPUNIT_ASSERT(e.first < ::count); + CPPUNIT_ASSERT(e.first % 2 == 0); + more = outDL->next(it, &e); + i++; + } + } - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + void reduceStep_4() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + BucketDL*inDL, *dDL; + WSDL* outDL; + ElementType e; + unsigned i; + int it; + bool more; - JobStepAssociation inJs; - JobStepAssociation outJs; + inDL = new BucketDL(10, 1, 100000, fRm); + dDL = new BucketDL(10, 1, 100000, fRm); + outDL = new WSDL(1, 100000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->bucketDL(inDL); + driverADL->bucketDL(dDL); + outputADL->workingSetDL(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); - AnyDataListSPtr spdl1(new AnyDataList()); - WSDL* dl1 = new WSDL(1, 100, fRm); - spdl1->workingSetDL(dl1); - inJs.outAdd(spdl1); + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); - AnyDataListSPtr spdl2(new AnyDataList()); - WSDL* dl2 = new WSDL(1, 100, fRm); - spdl2->workingSetDL(dl2); - outJs.outAdd(spdl2); + // cout << "making input DataList" << endl; - /* These values are unique to Pat's DB files unfortunately. */ - /* Fill in the value list */ - dl1->insert(ElementType(1, 3179)); // row 10 in the target - dl1->insert(ElementType(2, 3191)); // row 12 - dl1->insert(ElementType(3, 3207)); // row 14 - dl1->insert(ElementType(4, 3318)); // row 20 OOO - dl1->insert(ElementType(5, 3242)); // row 16 - dl1->insert(ElementType(6, 3289)); // row 18 - dl1->insert(ElementType(7, 3191)); // duplicate value of row 14 + for (i = 0; i < ::count; i++) + { + e.first = i; + e.second = i + 1; + inDL->insert(e); + } - dl1->endOfInput(); - dl1->OID(1003); + inDL->endOfInput(); - PNLJoin joiner(inJs, outJs, dec, cat, 12349, 1000, 1, 0, fRm); + // cout << "making driver DataList" << endl; - joiner.run(); - joiner.join(); + for (i = 0; i < ::count; i += 2) + { + e.first = i; + e.second = i + 1; + dDL->insert(e); + } - it = dl2->getIterator(); - more = dl2->next(it, &e); + dDL->endOfInput(); - while (more) - { + // cout << "reducing" << endl; + + ReduceStep rs(in, out, 5, 1, 0, 0, 0, 0); + rs.run(); + rs.join(); + + it = outDL->getIterator(); + more = outDL->next(it, &e); + i = 0; + + while (more) + { + // cout << i << ": first: " << e.first << " second: " << e.second << endl; + CPPUNIT_ASSERT(e.first < ::count); + CPPUNIT_ASSERT(e.first % 2 == 0); + more = outDL->next(it, &e); + i++; + } + } + + void pnljoin_1() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + int it; + bool more; + + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation outJs; + + AnyDataListSPtr spdl1(new AnyDataList()); + WSDL* dl1 = new WSDL(1, 100, fRm); + spdl1->workingSetDL(dl1); + inJs.outAdd(spdl1); + + AnyDataListSPtr spdl2(new AnyDataList()); + WSDL* dl2 = new WSDL(1, 100, fRm); + spdl2->workingSetDL(dl2); + outJs.outAdd(spdl2); + + /* These values are unique to Pat's DB files unfortunately. */ + /* Fill in the value list */ + dl1->insert(ElementType(1, 3179)); // row 10 in the target + dl1->insert(ElementType(2, 3191)); // row 12 + dl1->insert(ElementType(3, 3207)); // row 14 + dl1->insert(ElementType(4, 3318)); // row 20 OOO + dl1->insert(ElementType(5, 3242)); // row 16 + dl1->insert(ElementType(6, 3289)); // row 18 + dl1->insert(ElementType(7, 3191)); // duplicate value of row 14 + + dl1->endOfInput(); + dl1->OID(1003); + + PNLJoin joiner(inJs, outJs, dec, cat, 12349, 1000, 1, 0, fRm); + + joiner.run(); + joiner.join(); + + it = dl2->getIterator(); + more = dl2->next(it, &e); + + while (more) + { #ifdef DEBUG - cout << "first: " << e.first << " second: " << e.second << endl; + cout << "first: " << e.first << " second: " << e.second << endl; #endif - more = dl2->next(it, &e); - } + more = dl2->next(it, &e); } - void pnljoin_2() + } + void pnljoin_2() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + int i, it; + bool more; + + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + JobStepAssociation inJs; + JobStepAssociation outJs; + + AnyDataListSPtr spdl1(new AnyDataList()); + WSDL* valueList = new WSDL(1, 100, fRm); + spdl1->workingSetDL(valueList); + inJs.outAdd(spdl1); + + AnyDataListSPtr spdl2(new AnyDataList()); + AnyDataListSPtr spdl3(new AnyDataList()); + WSDL* colResults = new WSDL(1, 100, fRm); + WSDL* inputRidList = new WSDL(1, 100, fRm); + spdl2->workingSetDL(colResults); + spdl3->workingSetDL(inputRidList); + outJs.outAdd(spdl2); + inJs.outAdd(spdl3); + + /* These values are unique to Pat's DB files unfortunately. */ + /* Fill in the value list */ + valueList->insert(ElementType(1, 3179)); // row 10 in the target + valueList->insert(ElementType(2, 3191)); // row 12 + valueList->insert(ElementType(3, 3207)); // row 14 + valueList->insert(ElementType(4, 3318)); // row 20 OOO + valueList->insert(ElementType(5, 3242)); // row 16 + valueList->insert(ElementType(6, 3289)); // row 18 + valueList->insert(ElementType(7, 3191)); // duplicate value of row 14 + + valueList->endOfInput(); + valueList->OID(1003); + + // supply a ridlist with row 16 missing; make sure it's missing in the result + for (i = 0; i < 25; i++) + if (i != 16) + inputRidList->insert(ElementType(i, i)); + + inputRidList->endOfInput(); + + PNLJoin joiner(inJs, outJs, dec, cat, 12349, 1000, 1, 0, fRm); + + joiner.run(); + joiner.join(); + + it = colResults->getIterator(); + more = colResults->next(it, &e); + + while (more) { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - int i, it; - bool more; - - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - JobStepAssociation inJs; - JobStepAssociation outJs; - - AnyDataListSPtr spdl1(new AnyDataList()); - WSDL* valueList = new WSDL(1, 100, fRm); - spdl1->workingSetDL(valueList); - inJs.outAdd(spdl1); - - AnyDataListSPtr spdl2(new AnyDataList()); - AnyDataListSPtr spdl3(new AnyDataList()); - WSDL* colResults = new WSDL(1, 100, fRm); - WSDL* inputRidList = new WSDL(1, 100, fRm); - spdl2->workingSetDL(colResults); - spdl3->workingSetDL(inputRidList); - outJs.outAdd(spdl2); - inJs.outAdd(spdl3); - - - /* These values are unique to Pat's DB files unfortunately. */ - /* Fill in the value list */ - valueList->insert(ElementType(1, 3179)); // row 10 in the target - valueList->insert(ElementType(2, 3191)); // row 12 - valueList->insert(ElementType(3, 3207)); // row 14 - valueList->insert(ElementType(4, 3318)); // row 20 OOO - valueList->insert(ElementType(5, 3242)); // row 16 - valueList->insert(ElementType(6, 3289)); // row 18 - valueList->insert(ElementType(7, 3191)); // duplicate value of row 14 - - valueList->endOfInput(); - valueList->OID(1003); - - // supply a ridlist with row 16 missing; make sure it's missing in the result - for (i = 0; i < 25; i++) - if (i != 16) - inputRidList->insert(ElementType(i, i)); - - inputRidList->endOfInput(); - - PNLJoin joiner(inJs, outJs, dec, cat, 12349, 1000, 1, 0, fRm); - - joiner.run(); - joiner.join(); - - it = colResults->getIterator(); - more = colResults->next(it, &e); - - while (more) - { #ifdef DEBUG - cout << "first: " << e.first << " second: " << e.second << endl; + cout << "first: " << e.first << " second: " << e.second << endl; #endif - more = colResults->next(it, &e); - } + more = colResults->next(it, &e); } + } - void pnljoin_3() + void pnljoin_3() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + int i, it; + bool more; + + dec = DistributedEngineComm::instance(fRm); + // dec = DistributedEngineComm::instance("./config-dec.xml"); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + + AnyDataListSPtr spdl1(new AnyDataList()); + AnyDataListSPtr spdl3(new AnyDataList()); + WSDL* valueList = new WSDL(1, 100, fRm); + WSDL* inputRidList = new WSDL(1, 100, fRm); + spdl1->workingSetDL(valueList); + spdl3->workingSetDL(inputRidList); + + JobStepAssociation inJs; + JobStepAssociation outJs; + + inJs.outAdd(spdl1); + inJs.outAdd(spdl3); + + AnyDataListSPtr spdl2(new AnyDataList()); + AnyDataListSPtr spdl4(new AnyDataList()); + WSDL* colResults = new WSDL(1, 100, fRm); + WSDL* reducedRidList = new WSDL(1, 100, fRm); + spdl2->workingSetDL(colResults); + spdl4->workingSetDL(reducedRidList); + outJs.outAdd(spdl2); + outJs.outAdd(spdl4); + + /* These values are unique to Pat's DB files unfortunately. */ + /* Fill in the value list */ + valueList->insert(ElementType(10, 3179)); // row 10 in the target + valueList->insert(ElementType(12, 3191)); // row 12 + valueList->insert( + ElementType(14541513, 3207)); // row 14 - on output should be in colResults, not in reducedridlist + valueList->insert(ElementType(20, 3318)); // row 20 OOO + valueList->insert(ElementType(16, 3242)); // row 16 + valueList->insert(ElementType(18, 3289)); // row 18 + + // XXXPAT: Duplicates here can end up in the reducedRidList. Technically it's + // correct, but do we want that or not? + // valueList->insert(ElementType(12, 3191)); // duplicate value of row 12 + + valueList->endOfInput(); + valueList->OID(1003); + + // supply a ridlist with row 16 missing; make sure it's missing in the result + for (i = 0; i < 25; i++) + if (i != 16) + inputRidList->insert(ElementType(i, i)); + + inputRidList->endOfInput(); + + PNLJoin joiner(inJs, outJs, dec, cat, 12349, 1000, 1, 0, fRm); + + joiner.run(); + joiner.join(); + + it = colResults->getIterator(); + more = colResults->next(it, &e); +#ifdef DEBUG + cout << "ColResults:" << endl; +#endif + + while (more) { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - int i, it; - bool more; - - dec = DistributedEngineComm::instance(fRm); - // dec = DistributedEngineComm::instance("./config-dec.xml"); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - AnyDataListSPtr spdl1(new AnyDataList()); - AnyDataListSPtr spdl3(new AnyDataList()); - WSDL* valueList = new WSDL(1, 100, fRm); - WSDL* inputRidList = new WSDL(1, 100, fRm); - spdl1->workingSetDL(valueList); - spdl3->workingSetDL(inputRidList); - - JobStepAssociation inJs; - JobStepAssociation outJs; - - inJs.outAdd(spdl1); - inJs.outAdd(spdl3); - - AnyDataListSPtr spdl2(new AnyDataList()); - AnyDataListSPtr spdl4(new AnyDataList()); - WSDL* colResults = new WSDL(1, 100, fRm); - WSDL* reducedRidList = new WSDL(1, 100, fRm); - spdl2->workingSetDL(colResults); - spdl4->workingSetDL(reducedRidList); - outJs.outAdd(spdl2); - outJs.outAdd(spdl4); - - /* These values are unique to Pat's DB files unfortunately. */ - /* Fill in the value list */ - valueList->insert(ElementType(10, 3179)); // row 10 in the target - valueList->insert(ElementType(12, 3191)); // row 12 - valueList->insert(ElementType(14541513, 3207)); // row 14 - on output should be in colResults, not in reducedridlist - valueList->insert(ElementType(20, 3318)); // row 20 OOO - valueList->insert(ElementType(16, 3242)); // row 16 - valueList->insert(ElementType(18, 3289)); // row 18 - - // XXXPAT: Duplicates here can end up in the reducedRidList. Technically it's - // correct, but do we want that or not? -// valueList->insert(ElementType(12, 3191)); // duplicate value of row 12 - - valueList->endOfInput(); - valueList->OID(1003); - - // supply a ridlist with row 16 missing; make sure it's missing in the result - for (i = 0; i < 25; i++) - if (i != 16) - inputRidList->insert(ElementType(i, i)); - - inputRidList->endOfInput(); - - PNLJoin joiner(inJs, outJs, dec, cat, 12349, 1000, 1, 0, fRm); - - joiner.run(); - joiner.join(); - - it = colResults->getIterator(); - more = colResults->next(it, &e); #ifdef DEBUG - cout << "ColResults:" << endl; + cout << " first: " << e.first << " second: " << e.second << endl; #endif - - while (more) - { -#ifdef DEBUG - cout << " first: " << e.first << " second: " << e.second << endl; -#endif - more = colResults->next(it, &e); - } - - it = reducedRidList->getIterator(); - more = reducedRidList->next(it, &e); -#ifdef DEBUG - cout << "Reduced Rid List:" << endl; -#endif - - while (more) - { -#ifdef DEBUG - cout << " first: " << e.first << " second: " << e.second << endl; -#endif - more = reducedRidList->next(it, &e); - } - + more = colResults->next(it, &e); } - void unionStep_1() + it = reducedRidList->getIterator(); + more = reducedRidList->next(it, &e); +#ifdef DEBUG + cout << "Reduced Rid List:" << endl; +#endif + + while (more) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - WSDL* outDL; - set s; - set::iterator sIt; - ElementType e; - unsigned i; - int it; - bool more; - - ZDL* inDL = new ZDL(1, fRm); - ZDL* dDL = new ZDL(1, fRm); - - outDL = new WSDL(1, 100000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->zonedDL(inDL); - driverADL->zonedDL(dDL); - - outputADL->workingSetDL(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); - - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); - -// cout << "making input DataList" << endl; - vector vec1; - vector vec2; - - for (i = 0; i < ::count; i++) - { - e.first = i; - e.second = i + 1; - vec1.push_back(e); - - if (vec1.size() >= ZDL_VEC_SIZE) - { - inDL->insert(vec1); - vec1.clear(); - } - -// inDL->insert(e); - } - - if (!vec1.empty()) - inDL->insert(vec1); - - inDL->endOfInput(); - -// cout << "making driver DataList" << endl; - - for (i = 0; i < ::count; i += 2) - { - e.first = i; - e.second = i + 1; - vec2.push_back(e); - - if (vec2.size() >= ZDL_VEC_SIZE) - { - dDL->insert(vec2); - vec2.clear(); - } - -// dDL->insert(e); - } - - if (!vec2.empty()) - dDL->insert(vec2); - - dDL->endOfInput(); - -// cout << "unionizing" << endl; - - UnionStep rs(in, out, 50, 5, 1, 0, 0, 0); - rs.run(); - rs.join(); - - CPPUNIT_ASSERT(outDL->totalSize() == inDL->totalSize()); - - it = outDL->getIterator(); - - for (i = 0, more = outDL->next(it, &e) ; more; more = outDL->next(it, &e), i++) - s.insert(e); - - CPPUNIT_ASSERT(outDL->totalSize() == i); - CPPUNIT_ASSERT(i == ::count); - - CPPUNIT_ASSERT(s.size() == outDL->totalSize()); // verifies no duplicates in outDL - - for (i = 0, sIt = s.begin(); sIt != s.end(); sIt++, i++) - CPPUNIT_ASSERT(sIt->first == i); - - CPPUNIT_ASSERT(i == ::count); // verifies they all exist. +#ifdef DEBUG + cout << " first: " << e.first << " second: " << e.second << endl; +#endif + more = reducedRidList->next(it, &e); } + } - void unionStep_2() + void unionStep_1() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + WSDL* outDL; + set s; + set::iterator sIt; + ElementType e; + unsigned i; + int it; + bool more; + + ZDL* inDL = new ZDL(1, fRm); + ZDL* dDL = new ZDL(1, fRm); + + outDL = new WSDL(1, 100000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->zonedDL(inDL); + driverADL->zonedDL(dDL); + + outputADL->workingSetDL(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); + + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); + + // cout << "making input DataList" << endl; + vector vec1; + vector vec2; + + for (i = 0; i < ::count; i++) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - WSDL* inDL, *dDL, *outDL; - set s; - set::iterator sIt; - ElementType e; - unsigned i; - int it; - bool more; + e.first = i; + e.second = i + 1; + vec1.push_back(e); - inDL = new WSDL(1, 100000, fRm); - dDL = new WSDL(1, 100000, fRm); - outDL = new WSDL(1, 100000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->workingSetDL(inDL); - driverADL->workingSetDL(dDL); - outputADL->workingSetDL(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); + if (vec1.size() >= ZDL_VEC_SIZE) + { + inDL->insert(vec1); + vec1.clear(); + } - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); - -// cout << "making input DataList" << endl; - - for (i = 0; i < ::count; i += 2) - { - e.first = i; - e.second = i + 1; - inDL->insert(e); - } - - inDL->endOfInput(); - -// cout << "making driver DataList" << endl; - - for (i = 0; i < ::count; i++) - { - e.first = i; - e.second = i + 1; - dDL->insert(e); - } - - dDL->endOfInput(); - -// cout << "unionizing" << endl; - - UnionStep rs(in, out, 50, 5, 1, 0, 0, 0); - rs.run(); - rs.join(); - - CPPUNIT_ASSERT(outDL->totalSize() == dDL->totalSize()); - - it = outDL->getIterator(); - - for (i = 0, more = outDL->next(it, &e) ; more; more = outDL->next(it, &e), i++) - s.insert(e); - - CPPUNIT_ASSERT(outDL->totalSize() == i); - CPPUNIT_ASSERT(i == ::count); - - CPPUNIT_ASSERT(s.size() == outDL->totalSize()); // verifies no duplicates in outDL - - for (i = 0, sIt = s.begin(); sIt != s.end(); sIt++, i++) - CPPUNIT_ASSERT(sIt->first == i); - - CPPUNIT_ASSERT(i == ::count); // verifies they all exist. + // inDL->insert(e); } - void unionStep_3() + if (!vec1.empty()) + inDL->insert(vec1); + + inDL->endOfInput(); + + // cout << "making driver DataList" << endl; + + for (i = 0; i < ::count; i += 2) { - JobStepAssociation in, out; - AnyDataList* inputADL, *driverADL, *outputADL; - AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; - WSDL* outDL; - BucketDL* inDL, *dDL; - set s; - set::iterator sIt; - ElementType e; - unsigned i; - int it; - bool more; + e.first = i; + e.second = i + 1; + vec2.push_back(e); - inDL = new BucketDL(10, 1, 100000, fRm); - dDL = new BucketDL(10, 1, 100000, fRm); - outDL = new WSDL(1, 100000, fRm); - inputADL = new AnyDataList(); - driverADL = new AnyDataList(); - outputADL = new AnyDataList(); - inputADL->bucketDL(inDL); - driverADL->bucketDL(dDL); - outputADL->workingSetDL(outDL); - inputSPtr.reset(inputADL); - driverSPtr.reset(driverADL); - outputSPtr.reset(outputADL); + if (vec2.size() >= ZDL_VEC_SIZE) + { + dDL->insert(vec2); + vec2.clear(); + } - in.outAdd(inputSPtr); - in.outAdd(driverSPtr); - out.outAdd(outputSPtr); - -// cout << "making input DataList" << endl; - - for (i = 0; i < ::count; i++) - { - e.first = i; - e.second = i + 1; - inDL->insert(e); - } - - inDL->endOfInput(); - -// cout << "making driver DataList" << endl; - - for (; i < ::count * 2; i++) - { - e.first = i; - e.second = i + 1; - dDL->insert(e); - } - - dDL->endOfInput(); - -// cout << "unionizing" << endl; - - UnionStep rs(in, out, 50, 5, 1, 0, 0, 0); - rs.run(); - rs.join(); - - CPPUNIT_ASSERT(outDL->totalSize() == 2 * ::count); - - it = outDL->getIterator(); - - for (i = 0, more = outDL->next(it, &e) ; more; more = outDL->next(it, &e), i++) - s.insert(e); - - CPPUNIT_ASSERT(outDL->totalSize() == i); - CPPUNIT_ASSERT(i == 2 * ::count); - - CPPUNIT_ASSERT(s.size() == outDL->totalSize()); // verifies no duplicates in outDL - - for (i = 0, sIt = s.begin(); sIt != s.end(); sIt++, i++) - CPPUNIT_ASSERT(sIt->first == i); - - CPPUNIT_ASSERT(i == 2 * ::count); // verifies they all exist. + // dDL->insert(e); } + if (!vec2.empty()) + dDL->insert(vec2); + dDL->endOfInput(); + // cout << "unionizing" << endl; + + UnionStep rs(in, out, 50, 5, 1, 0, 0, 0); + rs.run(); + rs.join(); + + CPPUNIT_ASSERT(outDL->totalSize() == inDL->totalSize()); + + it = outDL->getIterator(); + + for (i = 0, more = outDL->next(it, &e); more; more = outDL->next(it, &e), i++) + s.insert(e); + + CPPUNIT_ASSERT(outDL->totalSize() == i); + CPPUNIT_ASSERT(i == ::count); + + CPPUNIT_ASSERT(s.size() == outDL->totalSize()); // verifies no duplicates in outDL + + for (i = 0, sIt = s.begin(); sIt != s.end(); sIt++, i++) + CPPUNIT_ASSERT(sIt->first == i); + + CPPUNIT_ASSERT(i == ::count); // verifies they all exist. + } + + void unionStep_2() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + WSDL*inDL, *dDL, *outDL; + set s; + set::iterator sIt; + ElementType e; + unsigned i; + int it; + bool more; + + inDL = new WSDL(1, 100000, fRm); + dDL = new WSDL(1, 100000, fRm); + outDL = new WSDL(1, 100000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->workingSetDL(inDL); + driverADL->workingSetDL(dDL); + outputADL->workingSetDL(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); + + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); + + // cout << "making input DataList" << endl; + + for (i = 0; i < ::count; i += 2) + { + e.first = i; + e.second = i + 1; + inDL->insert(e); + } + + inDL->endOfInput(); + + // cout << "making driver DataList" << endl; + + for (i = 0; i < ::count; i++) + { + e.first = i; + e.second = i + 1; + dDL->insert(e); + } + + dDL->endOfInput(); + + // cout << "unionizing" << endl; + + UnionStep rs(in, out, 50, 5, 1, 0, 0, 0); + rs.run(); + rs.join(); + + CPPUNIT_ASSERT(outDL->totalSize() == dDL->totalSize()); + + it = outDL->getIterator(); + + for (i = 0, more = outDL->next(it, &e); more; more = outDL->next(it, &e), i++) + s.insert(e); + + CPPUNIT_ASSERT(outDL->totalSize() == i); + CPPUNIT_ASSERT(i == ::count); + + CPPUNIT_ASSERT(s.size() == outDL->totalSize()); // verifies no duplicates in outDL + + for (i = 0, sIt = s.begin(); sIt != s.end(); sIt++, i++) + CPPUNIT_ASSERT(sIt->first == i); + + CPPUNIT_ASSERT(i == ::count); // verifies they all exist. + } + + void unionStep_3() + { + JobStepAssociation in, out; + AnyDataList *inputADL, *driverADL, *outputADL; + AnyDataListSPtr inputSPtr, driverSPtr, outputSPtr; + WSDL* outDL; + BucketDL*inDL, *dDL; + set s; + set::iterator sIt; + ElementType e; + unsigned i; + int it; + bool more; + + inDL = new BucketDL(10, 1, 100000, fRm); + dDL = new BucketDL(10, 1, 100000, fRm); + outDL = new WSDL(1, 100000, fRm); + inputADL = new AnyDataList(); + driverADL = new AnyDataList(); + outputADL = new AnyDataList(); + inputADL->bucketDL(inDL); + driverADL->bucketDL(dDL); + outputADL->workingSetDL(outDL); + inputSPtr.reset(inputADL); + driverSPtr.reset(driverADL); + outputSPtr.reset(outputADL); + + in.outAdd(inputSPtr); + in.outAdd(driverSPtr); + out.outAdd(outputSPtr); + + // cout << "making input DataList" << endl; + + for (i = 0; i < ::count; i++) + { + e.first = i; + e.second = i + 1; + inDL->insert(e); + } + + inDL->endOfInput(); + + // cout << "making driver DataList" << endl; + + for (; i < ::count * 2; i++) + { + e.first = i; + e.second = i + 1; + dDL->insert(e); + } + + dDL->endOfInput(); + + // cout << "unionizing" << endl; + + UnionStep rs(in, out, 50, 5, 1, 0, 0, 0); + rs.run(); + rs.join(); + + CPPUNIT_ASSERT(outDL->totalSize() == 2 * ::count); + + it = outDL->getIterator(); + + for (i = 0, more = outDL->next(it, &e); more; more = outDL->next(it, &e), i++) + s.insert(e); + + CPPUNIT_ASSERT(outDL->totalSize() == i); + CPPUNIT_ASSERT(i == 2 * ::count); + + CPPUNIT_ASSERT(s.size() == outDL->totalSize()); // verifies no duplicates in outDL + + for (i = 0, sIt = s.begin(); sIt != s.end(); sIt++, i++) + CPPUNIT_ASSERT(sIt->first == i); + + CPPUNIT_ASSERT(i == 2 * ::count); // verifies they all exist. + } }; CPPUNIT_TEST_SUITE_REGISTRATION(JobStepDriver); -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/joblist/tdriver-pdict.cpp b/dbcon/joblist/tdriver-pdict.cpp index 2e3e1b5d6..3b7689bdb 100644 --- a/dbcon/joblist/tdriver-pdict.cpp +++ b/dbcon/joblist/tdriver-pdict.cpp @@ -39,110 +39,102 @@ using namespace execplan; class JobStepDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(JobStepDriver); - CPPUNIT_TEST_SUITE(JobStepDriver); + CPPUNIT_TEST(pDictStep_1); - CPPUNIT_TEST(pDictStep_1); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void pDictStep_1() + { + DistributedEngineComm* dec; + boost::shared_ptr cat; + ElementType e; + uint32_t i, it; + bool more; + BRM::DBRM dbrm; + const uint32_t dictOID = 2064; + ResourceManager rm; + dec = DistributedEngineComm::instance(rm); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); -private: -public: + JobStepAssociation inJs; + JobStepAssociation outJs; - void pDictStep_1() + // create request list + AnyDataListSPtr spdl1(new AnyDataList()); + FifoDataList* dl1 = new FifoDataList(1, 128); + spdl1->fifoDL(dl1); + inJs.outAdd(spdl1); + + AnyDataListSPtr spdl2(new AnyDataList()); + StringFifoDataList* dl2 = new StringFifoDataList(1, 128); + spdl2->stringDL(dl2); + outJs.outAdd(spdl2); + + int64_t lbid; + int err = dbrm.lookup(dictOID, 0, lbid); + CPPUNIT_ASSERT(err == 0); + + // populate the element pair + UintRowGroup rw; + + const uint32_t tcount = 10; + + for (i = 1; i <= tcount; i++) { - DistributedEngineComm* dec; - boost::shared_ptr cat; - ElementType e; - uint32_t i, it; - bool more; - BRM::DBRM dbrm; - const uint32_t dictOID = 2064; - ResourceManager rm; - dec = DistributedEngineComm::instance(rm); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - - JobStepAssociation inJs; - JobStepAssociation outJs; - - // create request list - AnyDataListSPtr spdl1(new AnyDataList()); - FifoDataList* dl1 = new FifoDataList(1, 128); - spdl1->fifoDL(dl1); - inJs.outAdd(spdl1); - - AnyDataListSPtr spdl2(new AnyDataList()); - StringFifoDataList* dl2 = new StringFifoDataList(1, 128); - spdl2->stringDL(dl2); - outJs.outAdd(spdl2); - - int64_t lbid; - int err = dbrm.lookup(dictOID, 0, lbid); - CPPUNIT_ASSERT(err == 0); - - // populate the element pair - UintRowGroup rw; - - const uint32_t tcount = 10; - - for (i = 1; i <= tcount; i++) - { - WriteEngine::Token token; // rid - token.op = i; // index of sig in block - token.fbo = lbid; // lbid of sig block to search - token.spare = 0; - // cast for ElementType second value - uint64_t* u = reinterpret_cast(&token); - CPPUNIT_ASSERT(u); - rw.et[rw.count].first = i; - rw.et[rw.count++].second = *u; - - if (rw.count == rw.ElementsPerGroup) - { - dl1->insert(rw); - } - - } - - if (rw.count > 0) - dl1->insert(rw); - - // close input set - dl1->endOfInput(); - - pDictionaryStep p(inJs, outJs, dec, cat, dictOID, 1000, 12346, 32000, 32000, 1, 0, rm); - p.dec(dec); - p.run(); - p.join(); -// StringElementType s; - it = dl2->getIterator(); - // dump the result set - StringRowGroup s; - - - for (more = dl2->next(it, &s), i = 0; more; more = dl2->next(it, &s), i++) - { - - for (uint64_t i = 0; i < s.count; ++i ) - cout << i << " " << endl; - } - - CPPUNIT_ASSERT(s.count == tcount); + WriteEngine::Token token; // rid + token.op = i; // index of sig in block + token.fbo = lbid; // lbid of sig block to search + token.spare = 0; + // cast for ElementType second value + uint64_t* u = reinterpret_cast(&token); + CPPUNIT_ASSERT(u); + rw.et[rw.count].first = i; + rw.et[rw.count++].second = *u; + if (rw.count == rw.ElementsPerGroup) + { + dl1->insert(rw); + } } + if (rw.count > 0) + dl1->insert(rw); + + // close input set + dl1->endOfInput(); + + pDictionaryStep p(inJs, outJs, dec, cat, dictOID, 1000, 12346, 32000, 32000, 1, 0, rm); + p.dec(dec); + p.run(); + p.join(); + // StringElementType s; + it = dl2->getIterator(); + // dump the result set + StringRowGroup s; + + for (more = dl2->next(it, &s), i = 0; more; more = dl2->next(it, &s), i++) + { + for (uint64_t i = 0; i < s.count; ++i) + cout << i << " " << endl; + } + + CPPUNIT_ASSERT(s.count == tcount); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(JobStepDriver); -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - - return (wasSuccessful ? 0 : 1); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/dbcon/joblist/tdriver-zdl.cpp b/dbcon/joblist/tdriver-zdl.cpp index 16a729f1c..7f3b30147 100644 --- a/dbcon/joblist/tdriver-zdl.cpp +++ b/dbcon/joblist/tdriver-zdl.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: tdriver-zdl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: tdriver-zdl.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ #include #include #include @@ -50,259 +50,247 @@ using namespace std; using namespace joblist; Stopwatch timer; -//dmc-uint64_t count = 20000000/*1000000*/; -uint64_t count1Set = 2000000; +// dmc-uint64_t count = 20000000/*1000000*/; +uint64_t count1Set = 2000000; uint64_t countMulSet = 8000000; -int maxElements = 16000000/*50000*/; //max elements in memory at once for the benchmarks +int maxElements = 16000000 /*50000*/; // max elements in memory at once for the benchmarks int id_sw = -1; pthread_mutex_t writeLock; const int NUM_PRODUCERS = 8; const int NUM_CONSUMERS = 4; -uint64_t readCounts[NUM_CONSUMERS]; +uint64_t readCounts[NUM_CONSUMERS]; struct ThreadParms { - void* zdl; - unsigned int threadNumber; - uint64_t count; // for producer this is the number of elements - // each producer is to write - // for consumer this is not currently used + void* zdl; + unsigned int threadNumber; + uint64_t count; // for producer this is the number of elements + // each producer is to write + // for consumer this is not currently used }; //------------------------------------------------------------------------------ -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } } //------------------------------------------------------------------------------ // thread callbacks for SWSDL testing //------------------------------------------------------------------------------ -template +template void* SWSDL_producer_1set_seq(void* arg) { - pthread_mutex_lock(&writeLock); - id_sw++; - pthread_mutex_unlock(&writeLock); - SWSDL* sw = reinterpret_cast *>(arg); + pthread_mutex_lock(&writeLock); + id_sw++; + pthread_mutex_unlock(&writeLock); + SWSDL* sw = reinterpret_cast*>(arg); - for (uint64_t i = id_sw; - i < (::count1Set * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - id_sw); - i = i + NUM_PRODUCERS) - { - sw->insert(ElemT(i, i)); - } + for (uint64_t i = id_sw; i < (::count1Set * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - id_sw); + i = i + NUM_PRODUCERS) + { + sw->insert(ElemT(i, i)); + } - return NULL; + return NULL; } -template +template void* SWSDL_producer_mulSet_seq(void* arg) { - pthread_mutex_lock(&writeLock); - id_sw++; - pthread_mutex_unlock(&writeLock); - SWSDL* sw = reinterpret_cast *>(arg); + pthread_mutex_lock(&writeLock); + id_sw++; + pthread_mutex_unlock(&writeLock); + SWSDL* sw = reinterpret_cast*>(arg); - for (uint64_t i = id_sw; - i < (::countMulSet * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - id_sw); - i = i + NUM_PRODUCERS) - { - sw->insert(ElemT(i, i)); - } + for (uint64_t i = id_sw; i < (::countMulSet * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - id_sw); + i = i + NUM_PRODUCERS) + { + sw->insert(ElemT(i, i)); + } - return NULL; + return NULL; } -template +template void* SWSDL_producer_1set_rand(void* arg) { - SWSDL* sw = reinterpret_cast *>(arg); + SWSDL* sw = reinterpret_cast*>(arg); - for (uint64_t i = 0; i < ::count1Set; i++) - { - sw->insert(ElemT((uint64_t)(::count1Set * rand() / (RAND_MAX + 1.0)), i)); - } + for (uint64_t i = 0; i < ::count1Set; i++) + { + sw->insert(ElemT((uint64_t)(::count1Set * rand() / (RAND_MAX + 1.0)), i)); + } - return NULL; + return NULL; } -template +template void* SWSDL_producer_mulSet_rand(void* arg) { - SWSDL* sw = reinterpret_cast *>(arg); + SWSDL* sw = reinterpret_cast*>(arg); - for (uint64_t i = 0; i < ::countMulSet; i++) - { - sw->insert(ElemT((uint64_t)(countMulSet * rand() / (RAND_MAX + 1.0)), i)); - } + for (uint64_t i = 0; i < ::countMulSet; i++) + { + sw->insert(ElemT((uint64_t)(countMulSet * rand() / (RAND_MAX + 1.0)), i)); + } - return NULL; + return NULL; } -template +template void* SWSDL_consumer(void* arg) { - SWSDL* sw = reinterpret_cast *>(arg); - uint64_t id; - bool nextRet; - ElemT e; + SWSDL* sw = reinterpret_cast*>(arg); + uint64_t id; + bool nextRet; + ElemT e; - id = sw->getIterator(); + id = sw->getIterator(); + nextRet = sw->next(id, &e); + + while (nextRet) nextRet = sw->next(id, &e); - while (nextRet) - nextRet = sw->next(id, &e); - - return NULL; + return NULL; } //------------------------------------------------------------------------------ // thread callbacks and utilities for ZDL testing //------------------------------------------------------------------------------ -template +template void* ZDL_producer_1set_seq(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - ZDL* zdl = reinterpret_cast *> - (pThreadParms->zdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + ZDL* zdl = reinterpret_cast*>(pThreadParms->zdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = tNum; - i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); - i = i + NUM_PRODUCERS) - { - zdl->insert(ElemT(i, i)); - } + for (uint64_t i = tNum; i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); + i = i + NUM_PRODUCERS) + { + zdl->insert(ElemT(i, i)); + } - return NULL; + return NULL; } -template +template void* ZDL_producer_mulSet_seq(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - ZDL* zdl = reinterpret_cast *> - (pThreadParms->zdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + ZDL* zdl = reinterpret_cast*>(pThreadParms->zdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = tNum; - i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); - i = i + NUM_PRODUCERS) - { - zdl->insert(ElemT(i, i)); - } + for (uint64_t i = tNum; i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); + i = i + NUM_PRODUCERS) + { + zdl->insert(ElemT(i, i)); + } - return NULL; + return NULL; } // // Can't use ZDL_producer_mulSet_seq() to test RID only element type(s) // because we need an ElemT constructor that takes a single argument. // -template +template void* ZDL_producer_mulSet_seq_ridonly(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - ZDL* zdl = reinterpret_cast *> - (pThreadParms->zdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + ZDL* zdl = reinterpret_cast*>(pThreadParms->zdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = tNum; - i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); - i = i + NUM_PRODUCERS) - { - zdl->insert(ElemT(i)); - } + for (uint64_t i = tNum; i < (elementCount * NUM_PRODUCERS) - ((NUM_PRODUCERS - 1) - tNum); + i = i + NUM_PRODUCERS) + { + zdl->insert(ElemT(i)); + } - return NULL; + return NULL; } -template +template void* ZDL_producer_1set_rand(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - //uint64_t tNum = pThreadParms->threadNumber; - ZDL* zdl = reinterpret_cast *> - (pThreadParms->zdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + // uint64_t tNum = pThreadParms->threadNumber; + ZDL* zdl = reinterpret_cast*>(pThreadParms->zdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = 0; i < elementCount; i++) - { - zdl->insert(ElemT((uint64_t)(elementCount * rand() / (RAND_MAX + 1.0)), i)); - } + for (uint64_t i = 0; i < elementCount; i++) + { + zdl->insert(ElemT((uint64_t)(elementCount * rand() / (RAND_MAX + 1.0)), i)); + } - return NULL; + return NULL; } -template +template void* ZDL_producer_mulSet_rand(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - //uint64_t tNum = pThreadParms->threadNumber; - ZDL* zdl = reinterpret_cast *> - (pThreadParms->zdl); - uint64_t elementCount = pThreadParms->count; + ThreadParms* pThreadParms = reinterpret_cast(arg); + // uint64_t tNum = pThreadParms->threadNumber; + ZDL* zdl = reinterpret_cast*>(pThreadParms->zdl); + uint64_t elementCount = pThreadParms->count; - for (uint64_t i = 0; i < elementCount; i++) - { - zdl->insert(ElemT((uint64_t)(elementCount * rand() / (RAND_MAX + 1.0)), i)); - } + for (uint64_t i = 0; i < elementCount; i++) + { + zdl->insert(ElemT((uint64_t)(elementCount * rand() / (RAND_MAX + 1.0)), i)); + } - return NULL; + return NULL; } -template +template void* ZDL_consumer(void* arg) { - ThreadParms* pThreadParms = reinterpret_cast(arg); - uint64_t tNum = pThreadParms->threadNumber; - ZDL* zdl = reinterpret_cast *> - (pThreadParms->zdl); + ThreadParms* pThreadParms = reinterpret_cast(arg); + uint64_t tNum = pThreadParms->threadNumber; + ZDL* zdl = reinterpret_cast*>(pThreadParms->zdl); - uint64_t id; - bool nextRet; - ElemT e; + uint64_t id; + bool nextRet; + ElemT e; - id = zdl->getIterator(); + id = zdl->getIterator(); + nextRet = zdl->next(id, &e); + + while (nextRet) + { + ::readCounts[tNum]++; nextRet = zdl->next(id, &e); + } - while (nextRet) - { - ::readCounts[tNum]++; - nextRet = zdl->next(id, &e); - } - - return NULL; + return NULL; } -template +template void ZDL_printFileStats(void* arg) { - ZDL* zdl = reinterpret_cast *>(arg); - uint64_t nFiles; - uint64_t nBytes; - zdl->totalFileCounts(nFiles, nBytes); - uint32_t size1st = zdl->getDiskElemSize1st(); - uint32_t size2nd = zdl->getDiskElemSize2nd(); - cout << "NumberOfFiles: " << nFiles << endl; - cout << "NumberOfBytes: " << nBytes << endl; - cout << "ElementSize: " << size1st << "/" << size2nd << endl; + ZDL* zdl = reinterpret_cast*>(arg); + uint64_t nFiles; + uint64_t nBytes; + zdl->totalFileCounts(nFiles, nBytes); + uint32_t size1st = zdl->getDiskElemSize1st(); + uint32_t size2nd = zdl->getDiskElemSize2nd(); + cout << "NumberOfFiles: " << nFiles << endl; + cout << "NumberOfBytes: " << nBytes << endl; + cout << "ElementSize: " << size1st << "/" << size2nd << endl; } //------------------------------------------------------------------------------ @@ -311,702 +299,668 @@ void ZDL_printFileStats(void* arg) class DataListDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DataListDriver); + // CPPUNIT_TEST(configure); + // CPPUNIT_TEST(load_save); + // CPPUNIT_TEST(SWSDL_bench_1set_seq); + // CPPUNIT_TEST(SWSDL_bench_1set_rand); + // CPPUNIT_TEST(SWSDL_bench_mulSet_seq); + // CPPUNIT_TEST(SWSDL_bench_mulSet_rand); + // CPPUNIT_TEST(ZDL_bench_1set_seq); + // CPPUNIT_TEST(ZDL_bench_1set_rand); + CPPUNIT_TEST(ZDL_bench_mulSet_seq_uncompressed); + CPPUNIT_TEST(ZDL_bench_mulSet_seq_compressed_32_32); + CPPUNIT_TEST(ZDL_bench_mulSet_seq_compressed_64_32); + CPPUNIT_TEST(ZDL_bench_mulSet_seq_compressed_32_64); + // CPPUNIT_TEST(ZDL_bench_mulSet_rand); + CPPUNIT_TEST(ZDL_bench_mulSet_seq_ridonly_uncompressed); + CPPUNIT_TEST(ZDL_bench_mulSet_seq_ridonly_compressed_32); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE(DataListDriver); -//CPPUNIT_TEST(configure); -//CPPUNIT_TEST(load_save); -//CPPUNIT_TEST(SWSDL_bench_1set_seq); -//CPPUNIT_TEST(SWSDL_bench_1set_rand); -//CPPUNIT_TEST(SWSDL_bench_mulSet_seq); -//CPPUNIT_TEST(SWSDL_bench_mulSet_rand); -//CPPUNIT_TEST(ZDL_bench_1set_seq); -//CPPUNIT_TEST(ZDL_bench_1set_rand); - CPPUNIT_TEST(ZDL_bench_mulSet_seq_uncompressed); - CPPUNIT_TEST(ZDL_bench_mulSet_seq_compressed_32_32); - CPPUNIT_TEST(ZDL_bench_mulSet_seq_compressed_64_32); - CPPUNIT_TEST(ZDL_bench_mulSet_seq_compressed_32_64); -//CPPUNIT_TEST(ZDL_bench_mulSet_rand); - CPPUNIT_TEST(ZDL_bench_mulSet_seq_ridonly_uncompressed); - CPPUNIT_TEST(ZDL_bench_mulSet_seq_ridonly_compressed_32); - CPPUNIT_TEST_SUITE_END(); + ResourceManager fRm; - ResourceManager fRm; -private: -public: - //-------------------------------------------------------------------------- - // setup method run prior to each unit test - //-------------------------------------------------------------------------- - void setUp() + private: + public: + //-------------------------------------------------------------------------- + // setup method run prior to each unit test + //-------------------------------------------------------------------------- + void setUp() + { + for (int i = 0; i < NUM_CONSUMERS; i++) { - for (int i = 0; i < NUM_CONSUMERS; i++) - { - ::readCounts[i] = 0; - } + ::readCounts[i] = 0; + } + } + + //-------------------------------------------------------------------------- + // validate results from a unit test + //-------------------------------------------------------------------------- + void validateResults(uint64_t totalElementsExpected) + { + for (int i = 0; i < NUM_CONSUMERS; i++) + { + cout << "consumer " << i << " read " << ::readCounts[i] << " elements" << endl; } - //-------------------------------------------------------------------------- - // validate results from a unit test - //-------------------------------------------------------------------------- - void validateResults(uint64_t totalElementsExpected) + cout << endl; + + for (int i = 0; i < NUM_CONSUMERS; i++) { - for (int i = 0; i < NUM_CONSUMERS; i++) - { - cout << "consumer " << i << " read " << ::readCounts[i] << - " elements" << endl; - } + CPPUNIT_ASSERT(readCounts[i] == totalElementsExpected); + } + } - cout << endl; + //-------------------------------------------------------------------------- + // SWSDL benchmark functions + //-------------------------------------------------------------------------- + void SWSDL_bench_1set_seq() + { + typedef ElementType Element; - for (int i = 0; i < NUM_CONSUMERS; i++) - { - CPPUNIT_ASSERT(readCounts[i] == totalElementsExpected); - } + id_sw = 0; + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + SWSDL sw(numOfConsumers, fRm); + sw.setMultipleProducers(true); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + pthread_create(&producer[i], NULL, SWSDL_producer_1set_seq, &sw); + + for (i = 0; i < numOfConsumers; i++) + pthread_create(&consumer[i], NULL, SWSDL_consumer, &sw); + + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + sw.endOfInput(); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "SWSDL_bench_1set_seq: producer & consumer passed " << sw.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + void SWSDL_bench_mulSet_seq() + { + typedef ElementType Element; + + id_sw = 0; + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + SWSDL sw(numOfConsumers, fRm); + sw.setMultipleProducers(true); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + struct timespec ts1, ts2, diff; + + timer.start("swsdl-produce"); + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + pthread_create(&producer[i], NULL, SWSDL_producer_mulSet_seq, &sw); + + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + timer.stop("swsdl-produce"); + timer.start("swsdl-endofinput"); + sw.endOfInput(); + timer.stop("swsdl-endofinput"); + + // timer.stop("swsdl-produce"); + timer.start("swsdl-consume"); + + for (i = 0; i < numOfConsumers; i++) + pthread_create(&consumer[i], NULL, SWSDL_consumer, &sw); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timer.stop("swsdl-consume"); + timer.finish(); + timespec_sub(ts1, ts2, diff); + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "SWSDL_bench_mulSet_seq: producer & consumer passed " << sw.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + void SWSDL_bench_1set_rand() + { + typedef ElementType Element; + + id_sw = 0; + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + SWSDL sw(numOfConsumers, ::maxElements, fRm); + sw.setMultipleProducers(true); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + pthread_create(&producer[i], NULL, SWSDL_producer_1set_rand, &sw); + + for (i = 0; i < numOfConsumers; i++) + pthread_create(&consumer[i], NULL, SWSDL_consumer, &sw); + + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + sw.endOfInput(); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "SWSDL_bench_1set_rand: producer & consumer passed " << sw.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + void SWSDL_bench_mulSet_rand() + { + typedef ElementType Element; + + id_sw = 0; + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + SWSDL sw(numOfConsumers, ::maxElements, fRm); + sw.setMultipleProducers(true); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + pthread_create(&producer[i], NULL, SWSDL_producer_mulSet_rand, &sw); + + for (i = 0; i < numOfConsumers; i++) + pthread_create(&consumer[i], NULL, SWSDL_consumer, &sw); + + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + sw.endOfInput(); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "SWSDL_bench_mulSet_rand: producer & consumer passed " << sw.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + } + + //-------------------------------------------------------------------------- + // ZDL benchmark functions + //-------------------------------------------------------------------------- + void ZDL_bench_1set_seq() + { + typedef ElementType Element; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + ZDL zdl(numOfConsumers, fRm); + zdl.setMultipleProducers(true); + zdl.setElementMode(1); // RID_VALUE + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + timer.start("zdl-produce_1set_seq"); + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) + { + producerThreadParms[i].zdl = &zdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::count1Set; + pthread_create(&producer[i], NULL, ZDL_producer_1set_seq, &producerThreadParms[i]); } - //-------------------------------------------------------------------------- - // SWSDL benchmark functions - //-------------------------------------------------------------------------- - void SWSDL_bench_1set_seq() + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + zdl.endOfInput(); + timer.stop("zdl-produce_1set_seq"); + + timer.start("zdl-consume_1set_seq"); + + for (i = 0; i < numOfConsumers; i++) { - typedef ElementType Element; - - id_sw = 0; - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - SWSDL sw(numOfConsumers, fRm); - sw.setMultipleProducers(true); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - pthread_create(&producer[i], NULL, - SWSDL_producer_1set_seq, &sw); - - for (i = 0; i < numOfConsumers; i++) - pthread_create(&consumer[i], NULL, - SWSDL_consumer, &sw); - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - sw.endOfInput(); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "SWSDL_bench_1set_seq: producer & consumer passed " << - sw.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; + consumerThreadParms[i].zdl = &zdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = 0; + pthread_create(&consumer[i], NULL, ZDL_consumer, &consumerThreadParms[i]); } - void SWSDL_bench_mulSet_seq() + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timer.stop("zdl-consume_1set_seq"); + + timer.finish(); + timespec_sub(ts1, ts2, diff); + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "ZDL_bench_1set_seq: producer & consumer passed " << zdl.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + ZDL_printFileStats(&zdl); + validateResults(::count1Set * NUM_PRODUCERS); + } + + void ZDL_bench_mulSet_seq(char* testDesc, bool compress, uint32_t size1st, uint32_t size2nd) + { + typedef ElementType Element; + + string produceTag("zdl-produce_"); + string eofInputTag("zdl-endofinput_"); + string consumeTag("zdl-consume_"); + produceTag += testDesc; + eofInputTag += testDesc; + consumeTag += testDesc; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + ZDL zdl(numOfConsumers, fRm); + zdl.setMultipleProducers(true); + zdl.setElementMode(1); // RID_VALUE + + if (compress) + zdl.setDiskElemSize(size1st, size2nd); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + timer.start(produceTag); + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) { - typedef ElementType Element; - - id_sw = 0; - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - SWSDL sw(numOfConsumers, fRm); - sw.setMultipleProducers(true); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - struct timespec ts1, ts2, diff; - - timer.start("swsdl-produce"); - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - pthread_create(&producer[i], NULL, - SWSDL_producer_mulSet_seq, &sw); - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - timer.stop("swsdl-produce"); - timer.start("swsdl-endofinput"); - sw.endOfInput(); - timer.stop("swsdl-endofinput"); - - //timer.stop("swsdl-produce"); - timer.start("swsdl-consume"); - - for (i = 0; i < numOfConsumers; i++) - pthread_create(&consumer[i], NULL, - SWSDL_consumer, &sw); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timer.stop("swsdl-consume"); - timer.finish(); - timespec_sub(ts1, ts2, diff); - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "SWSDL_bench_mulSet_seq: producer & consumer passed " << - sw.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; + producerThreadParms[i].zdl = &zdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::countMulSet; + pthread_create(&producer[i], NULL, ZDL_producer_mulSet_seq, &producerThreadParms[i]); } - void SWSDL_bench_1set_rand() + for (i = 0; i < numOfConsumers; i++) { - typedef ElementType Element; - - id_sw = 0; - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - SWSDL sw(numOfConsumers, ::maxElements, fRm); - sw.setMultipleProducers(true); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - pthread_create(&producer[i], NULL, - SWSDL_producer_1set_rand, &sw); - - for (i = 0; i < numOfConsumers; i++) - pthread_create(&consumer[i], NULL, - SWSDL_consumer, &sw); - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - sw.endOfInput(); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "SWSDL_bench_1set_rand: producer & consumer passed " << - sw.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; + consumerThreadParms[i].zdl = &zdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = 0; + pthread_create(&consumer[i], NULL, ZDL_consumer, &consumerThreadParms[i]); } - void SWSDL_bench_mulSet_rand() + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + timer.stop(produceTag); + + timer.start(eofInputTag); + zdl.endOfInput(); + timer.stop(eofInputTag); + + timer.start(consumeTag); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timer.stop(consumeTag); + + timer.finish(); + timespec_sub(ts1, ts2, diff); + cout << "compress state: " << (compress ? "on" : "off") << endl; + + if (compress) + cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "ZDL_bench_mulSet_seq_" << testDesc << ": producer & consumer passed " << zdl.totalSize() + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + ZDL_printFileStats(&zdl); + validateResults(::countMulSet * NUM_PRODUCERS); + } + + void ZDL_bench_mulSet_seq_uncompressed() + { + ZDL_bench_mulSet_seq("uncompressed", false, 0, 0); + } + + void ZDL_bench_mulSet_seq_compressed_32_32() + { + ZDL_bench_mulSet_seq("compressed_32_32", true, 4, 4); + } + + void ZDL_bench_mulSet_seq_compressed_64_32() + { + ZDL_bench_mulSet_seq("compressed_64_32", true, 8, 4); + } + + void ZDL_bench_mulSet_seq_compressed_32_64() + { + ZDL_bench_mulSet_seq("compressed_32_64", true, 4, 8); + } + + void ZDL_bench_1set_rand() + { + typedef ElementType Element; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + ZDL zdl(numOfConsumers, fRm); + zdl.setMultipleProducers(true); + zdl.setElementMode(1); // RID_VALUE + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) { - typedef ElementType Element; - - id_sw = 0; - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - SWSDL sw(numOfConsumers, ::maxElements, fRm); - sw.setMultipleProducers(true); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - pthread_create(&producer[i], NULL, - SWSDL_producer_mulSet_rand, &sw); - - for (i = 0; i < numOfConsumers; i++) - pthread_create(&consumer[i], NULL, - SWSDL_consumer, &sw); - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - sw.endOfInput(); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "SWSDL_bench_mulSet_rand: producer & consumer passed " << - sw.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; + producerThreadParms[i].zdl = &zdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::count1Set; + pthread_create(&producer[i], NULL, ZDL_producer_1set_rand, &producerThreadParms[i]); } - //-------------------------------------------------------------------------- - // ZDL benchmark functions - //-------------------------------------------------------------------------- - void ZDL_bench_1set_seq() + for (i = 0; i < numOfConsumers; i++) { - typedef ElementType Element; - - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - ZDL zdl(numOfConsumers, fRm); - zdl.setMultipleProducers(true); - zdl.setElementMode(1); // RID_VALUE - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; - - timer.start("zdl-produce_1set_seq"); - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].zdl = &zdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::count1Set; - pthread_create(&producer[i], NULL, - ZDL_producer_1set_seq, &producerThreadParms[i]); - } - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - zdl.endOfInput(); - timer.stop("zdl-produce_1set_seq"); - - timer.start("zdl-consume_1set_seq"); - - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].zdl = &zdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = 0; - pthread_create(&consumer[i], NULL, - ZDL_consumer, &consumerThreadParms[i]); - } - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timer.stop("zdl-consume_1set_seq"); - - timer.finish(); - timespec_sub(ts1, ts2, diff); - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "ZDL_bench_1set_seq: producer & consumer passed " << - zdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; - - ZDL_printFileStats(&zdl); - validateResults(::count1Set * NUM_PRODUCERS); + consumerThreadParms[i].zdl = &zdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = 0; + pthread_create(&consumer[i], NULL, ZDL_consumer, &consumerThreadParms[i]); } - void ZDL_bench_mulSet_seq(char* testDesc, bool compress, - uint32_t size1st, uint32_t size2nd) + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + zdl.endOfInput(); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "ZDL_bench_1set_rand: producer & consumer passed " << zdl.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + ZDL_printFileStats(&zdl); + validateResults(::count1Set); + } + + void ZDL_bench_mulSet_rand() + { + typedef ElementType Element; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + ZDL zdl(numOfConsumers, fRm); + zdl.setMultipleProducers(true); + zdl.setElementMode(1); // RID_VALUE + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) { - typedef ElementType Element; - - string produceTag ("zdl-produce_"); - string eofInputTag("zdl-endofinput_"); - string consumeTag ("zdl-consume_"); - produceTag += testDesc; - eofInputTag += testDesc; - consumeTag += testDesc; - - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - ZDL zdl(numOfConsumers, fRm); - zdl.setMultipleProducers(true); - zdl.setElementMode(1); // RID_VALUE - - if ( compress ) - zdl.setDiskElemSize ( size1st, size2nd ); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; - - timer.start(produceTag); - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].zdl = &zdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::countMulSet; - pthread_create(&producer[i], NULL, - ZDL_producer_mulSet_seq, &producerThreadParms[i]); - } - - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].zdl = &zdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = 0; - pthread_create(&consumer[i], NULL, - ZDL_consumer, &consumerThreadParms[i]); - } - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - timer.stop(produceTag); - - timer.start(eofInputTag); - zdl.endOfInput(); - timer.stop(eofInputTag); - - timer.start(consumeTag); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timer.stop(consumeTag); - - timer.finish(); - timespec_sub(ts1, ts2, diff); - cout << "compress state: " << (compress ? "on" : "off") << endl; - - if (compress) - cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "ZDL_bench_mulSet_seq_" << testDesc << - ": producer & consumer passed " << - zdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; - - ZDL_printFileStats(&zdl); - validateResults(::countMulSet * NUM_PRODUCERS); + producerThreadParms[i].zdl = &zdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::countMulSet; + pthread_create(&producer[i], NULL, ZDL_producer_mulSet_rand, &producerThreadParms[i]); } - void ZDL_bench_mulSet_seq_uncompressed() + for (i = 0; i < numOfConsumers; i++) { - ZDL_bench_mulSet_seq( "uncompressed", false, 0, 0 ); + consumerThreadParms[i].zdl = &zdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = 0; + pthread_create(&consumer[i], NULL, ZDL_consumer, &consumerThreadParms[i]); } - void ZDL_bench_mulSet_seq_compressed_32_32() + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + zdl.endOfInput(); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "ZDL_bench_mulSet_rand: producer & consumer passed " << zdl.totalSize() << " elements in " + << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + ZDL_printFileStats(&zdl); + validateResults(::countMulSet); + } + + void ZDL_bench_mulSet_seq_ridonly(char* testDesc, bool compress, uint32_t size1st) + { + typedef RIDElementType Element; + uint32_t size2nd = 0; + + string produceTag("zdl-produce_ridonly"); + string eofInputTag("zdl-endofinput_ridonly"); + string consumeTag("zdl-consume_ridonly"); + produceTag += testDesc; + eofInputTag += testDesc; + consumeTag += testDesc; + + uint32_t i; + uint32_t numOfProducers = NUM_PRODUCERS; + uint32_t numOfConsumers = NUM_CONSUMERS; + ZDL zdl(numOfConsumers, fRm); + zdl.setMultipleProducers(true); + + if (compress) + zdl.setDiskElemSize(size1st, size2nd); + + pthread_t producer[numOfProducers]; + pthread_t consumer[numOfConsumers]; + ThreadParms producerThreadParms[NUM_PRODUCERS]; + ThreadParms consumerThreadParms[NUM_CONSUMERS]; + struct timespec ts1, ts2, diff; + + timer.start(produceTag); + clock_gettime(CLOCK_REALTIME, &ts1); + + for (i = 0; i < numOfProducers; i++) { - ZDL_bench_mulSet_seq( "compressed_32_32", true, 4, 4 ); + producerThreadParms[i].zdl = &zdl; + producerThreadParms[i].threadNumber = i; + producerThreadParms[i].count = ::countMulSet; + pthread_create(&producer[i], NULL, ZDL_producer_mulSet_seq_ridonly, &producerThreadParms[i]); } - void ZDL_bench_mulSet_seq_compressed_64_32() + for (i = 0; i < numOfConsumers; i++) { - ZDL_bench_mulSet_seq( "compressed_64_32", true, 8, 4 ); + consumerThreadParms[i].zdl = &zdl; + consumerThreadParms[i].threadNumber = i; + consumerThreadParms[i].count = 0; + pthread_create(&consumer[i], NULL, ZDL_consumer, &consumerThreadParms[i]); } - void ZDL_bench_mulSet_seq_compressed_32_64() + for (i = 0; i < numOfProducers; i++) + pthread_join(producer[i], NULL); + + timer.stop(produceTag); + + timer.start(eofInputTag); + zdl.endOfInput(); + timer.stop(eofInputTag); + + timer.start(consumeTag); + + for (i = 0; i < numOfConsumers; i++) + pthread_join(consumer[i], NULL); + + clock_gettime(CLOCK_REALTIME, &ts2); + timer.stop(consumeTag); + + timer.finish(); + timespec_sub(ts1, ts2, diff); + cout << "compress state: " << (compress ? "on" : "off") << endl; + + if (compress) + cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; + + cout << "# of Producers: " << numOfProducers << endl; + cout << "# of Consumers: " << numOfConsumers << endl; + cout << "ZDL_bench_mulSet_seq_ridonly_" << testDesc << ": producer & consumer passed " << zdl.totalSize() + << " elements in " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl; + + ZDL_printFileStats(&zdl); + validateResults(::countMulSet * NUM_PRODUCERS); + } + + void ZDL_bench_mulSet_seq_ridonly_uncompressed() + { + ZDL_bench_mulSet_seq_ridonly("uncompressed", false, 0); + } + + void ZDL_bench_mulSet_seq_ridonly_compressed_32() + { + ZDL_bench_mulSet_seq_ridonly("compressed_32", true, 4); + } + + //-------------------------------------------------------------------------- + // test the saving and loading of a zdl file + //-------------------------------------------------------------------------- + void load_save() + { + typedef ElementType Element; + + vector v; + + for (uint32_t i = 0; i < ::count1Set * 8; i++) + v.push_back(Element(i, i)); + + vector v1; + vector v2; + vector v3; + + // save + ofstream f1; + ifstream f; + string filename = "zdl.txt"; + uint64_t ctn = v.size(); + f1.open(filename.c_str(), std::ios::binary); + f1.write((char*)&ctn, sizeof(ctn)); + f1.write((char*)(v.begin().operator->()), sizeof(Element) * ctn); + f.close(); + + // load + v1.push_back(Element(3, 4)); + f.open(filename.c_str(), std::ios::binary); + timer.start("read"); + v1.resize(v1.size() + ::count1Set * 8); + f.read((char*)((v1.begin() + 1).operator->()), ctn * sizeof(Element)); + cout << v1.size() << endl; + timer.stop("read"); + cout << "E1: " << v1[0].first << endl; + f.close(); + + f.open(filename.c_str(), std::ios::binary); + timer.start("assign"); + v2.assign(std::istream_iterator(f), std::istream_iterator()); + cout << v2.size() << endl; + timer.stop("assign"); + f.close(); + + f.open(filename.c_str(), std::ios::binary); + timer.start("insert"); + v3.insert(v3.end(), std::istream_iterator(f), std::istream_iterator()); + cout << v3.size() << endl; + timer.stop("insert"); + f.close(); + timer.finish(); + } + + //-------------------------------------------------------------------------- + // test the reading of zdl configuration parameters + //-------------------------------------------------------------------------- + void configure() + { + config::Config* config = config::Config::makeConfig(); + std::string strVal; + strVal = config->getConfig("ZDL", "MaxMemConsumption"); + uint64_t maxMemConsumption; + timer.start("configure"); + + for (int i = 0; i < 20; i++) { - ZDL_bench_mulSet_seq( "compressed_32_64", true, 4, 8 ); + if (strVal.size() > 0) + { + maxMemConsumption = config::Config::uFromText(strVal); + + if ((maxMemConsumption - 1) & maxMemConsumption) + throw std::runtime_error( + "ZDL: maxMemConsumption " + "should be a power of 2."); + } + else + maxMemConsumption = 1000000; } - void ZDL_bench_1set_rand() - { - typedef ElementType Element; - - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - ZDL zdl(numOfConsumers, fRm); - zdl.setMultipleProducers(true); - zdl.setElementMode(1); // RID_VALUE - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].zdl = &zdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::count1Set; - pthread_create(&producer[i], NULL, - ZDL_producer_1set_rand, &producerThreadParms[i]); - } - - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].zdl = &zdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = 0; - pthread_create(&consumer[i], NULL, - ZDL_consumer, &consumerThreadParms[i]); - } - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - zdl.endOfInput(); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "ZDL_bench_1set_rand: producer & consumer passed " << - zdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; - - ZDL_printFileStats(&zdl); - validateResults(::count1Set); - } - - void ZDL_bench_mulSet_rand() - { - typedef ElementType Element; - - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - ZDL zdl(numOfConsumers, fRm); - zdl.setMultipleProducers(true); - zdl.setElementMode(1); // RID_VALUE - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].zdl = &zdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::countMulSet; - pthread_create(&producer[i], NULL, - ZDL_producer_mulSet_rand, &producerThreadParms[i]); - } - - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].zdl = &zdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = 0; - pthread_create(&consumer[i], NULL, - ZDL_consumer, &consumerThreadParms[i]); - } - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - zdl.endOfInput(); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "ZDL_bench_mulSet_rand: producer & consumer passed " << - zdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; - - ZDL_printFileStats(&zdl); - validateResults(::countMulSet); - } - - void ZDL_bench_mulSet_seq_ridonly(char* testDesc, bool compress, - uint32_t size1st) - { - typedef RIDElementType Element; - uint32_t size2nd = 0; - - string produceTag ("zdl-produce_ridonly"); - string eofInputTag("zdl-endofinput_ridonly"); - string consumeTag ("zdl-consume_ridonly"); - produceTag += testDesc; - eofInputTag += testDesc; - consumeTag += testDesc; - - uint32_t i; - uint32_t numOfProducers = NUM_PRODUCERS; - uint32_t numOfConsumers = NUM_CONSUMERS; - ZDL zdl(numOfConsumers, fRm); - zdl.setMultipleProducers(true); - - if ( compress ) - zdl.setDiskElemSize ( size1st, size2nd ); - - pthread_t producer[numOfProducers]; - pthread_t consumer[numOfConsumers]; - ThreadParms producerThreadParms[NUM_PRODUCERS]; - ThreadParms consumerThreadParms[NUM_CONSUMERS]; - struct timespec ts1, ts2, diff; - - timer.start(produceTag); - clock_gettime(CLOCK_REALTIME, &ts1); - - for (i = 0; i < numOfProducers; i++) - { - producerThreadParms[i].zdl = &zdl; - producerThreadParms[i].threadNumber = i; - producerThreadParms[i].count = ::countMulSet; - pthread_create(&producer[i], NULL, - ZDL_producer_mulSet_seq_ridonly, - &producerThreadParms[i]); - } - - for (i = 0; i < numOfConsumers; i++) - { - consumerThreadParms[i].zdl = &zdl; - consumerThreadParms[i].threadNumber = i; - consumerThreadParms[i].count = 0; - pthread_create(&consumer[i], NULL, - ZDL_consumer, &consumerThreadParms[i]); - } - - for (i = 0; i < numOfProducers; i++) - pthread_join(producer[i], NULL); - - timer.stop(produceTag); - - timer.start(eofInputTag); - zdl.endOfInput(); - timer.stop(eofInputTag); - - timer.start(consumeTag); - - for (i = 0; i < numOfConsumers; i++) - pthread_join(consumer[i], NULL); - - clock_gettime(CLOCK_REALTIME, &ts2); - timer.stop(consumeTag); - - timer.finish(); - timespec_sub(ts1, ts2, diff); - cout << "compress state: " << (compress ? "on" : "off") << endl; - - if (compress) - cout << "size 1st/2nd: " << size1st << "/" << size2nd << endl; - - cout << "# of Producers: " << numOfProducers << endl; - cout << "# of Consumers: " << numOfConsumers << endl; - cout << "ZDL_bench_mulSet_seq_ridonly_" << testDesc << - ": producer & consumer passed " << - zdl.totalSize() << " elements in " << diff.tv_sec << "s " << - diff.tv_nsec << "ns" << endl; - - ZDL_printFileStats(&zdl); - validateResults(::countMulSet * NUM_PRODUCERS); - } - - void ZDL_bench_mulSet_seq_ridonly_uncompressed() - { - ZDL_bench_mulSet_seq_ridonly( "uncompressed", false, 0 ); - } - - void ZDL_bench_mulSet_seq_ridonly_compressed_32() - { - ZDL_bench_mulSet_seq_ridonly( "compressed_32", true, 4 ); - } - - //-------------------------------------------------------------------------- - // test the saving and loading of a zdl file - //-------------------------------------------------------------------------- - void load_save() - { - typedef ElementType Element; - - vector v; - - for (uint32_t i = 0; i < ::count1Set * 8; i++) - v.push_back(Element(i, i)); - - vector v1; - vector v2; - vector v3; - - // save - ofstream f1; - ifstream f; - string filename = "zdl.txt"; - uint64_t ctn = v.size(); - f1.open(filename.c_str(), std::ios::binary); - f1.write((char*) &ctn, sizeof(ctn)); - f1.write((char*) (v.begin().operator->()), sizeof(Element) * ctn); - f.close(); - - // load - v1.push_back(Element(3, 4)); - f.open(filename.c_str(), std::ios::binary); - timer.start("read"); - v1.resize(v1.size() +::count1Set * 8); - f.read((char*) ((v1.begin() + 1).operator->()), ctn * sizeof(Element)); - cout << v1.size() << endl; - timer.stop("read"); - cout << "E1: " << v1[0].first << endl; - f.close(); - - f.open(filename.c_str(), std::ios::binary); - timer.start("assign"); - v2.assign(std::istream_iterator(f), - std::istream_iterator()); - cout << v2.size() << endl; - timer.stop("assign"); - f.close(); - - f.open(filename.c_str(), std::ios::binary); - timer.start("insert"); - v3.insert(v3.end(), std::istream_iterator(f), - std::istream_iterator()); - cout << v3.size() << endl; - timer.stop("insert"); - f.close(); - timer.finish(); - } - - //-------------------------------------------------------------------------- - // test the reading of zdl configuration parameters - //-------------------------------------------------------------------------- - void configure() - { - config::Config* config = config::Config::makeConfig(); - std::string strVal; - strVal = config->getConfig("ZDL", "MaxMemConsumption"); - uint64_t maxMemConsumption; - timer.start("configure"); - - for (int i = 0; i < 20; i++) - { - if (strVal.size() > 0) - { - maxMemConsumption = config::Config::uFromText(strVal); - - if ((maxMemConsumption - 1) & maxMemConsumption) - throw std::runtime_error("ZDL: maxMemConsumption " - "should be a power of 2."); - } - else - maxMemConsumption = 1000000; - } - - timer.stop("configure"); - timer.finish(); - } + timer.stop("configure"); + timer.finish(); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(DataListDriver); @@ -1014,14 +968,12 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DataListDriver); //------------------------------------------------------------------------------ // main entry point //------------------------------------------------------------------------------ -int main( int argc, char** argv) +int main(int argc, char** argv) { - pthread_mutex_init(&writeLock, NULL); - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + pthread_mutex_init(&writeLock, NULL); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/dbcon/joblist/tdriver1.cpp b/dbcon/joblist/tdriver1.cpp index ca60ec34f..a865c797e 100644 --- a/dbcon/joblist/tdriver1.cpp +++ b/dbcon/joblist/tdriver1.cpp @@ -29,44 +29,43 @@ using namespace execplan; int main(int argc, char** argv) { - DistributedEngineComm* dec; - boost::shared_ptr cat; + DistributedEngineComm* dec; + boost::shared_ptr cat; - ResourceManager rm; - dec = DistributedEngineComm::instance(rm); - cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); + ResourceManager rm; + dec = DistributedEngineComm::instance(rm); + cat = CalpontSystemCatalog::makeCalpontSystemCatalog(); - JobStepAssociation inJs; - JobStepAssociation outJs; + JobStepAssociation inJs; + JobStepAssociation outJs; - AnyDataListSPtr spdl1(new AnyDataList()); - FifoDataList* dl1 = new FifoDataList(1, 128); - spdl1->fifoDL(dl1); - outJs.outAdd(spdl1); + AnyDataListSPtr spdl1(new AnyDataList()); + FifoDataList* dl1 = new FifoDataList(1, 128); + spdl1->fifoDL(dl1); + outJs.outAdd(spdl1); - pColScanStep step0(inJs, outJs, dec, cat, 1003, 1000, 12345, 999, 7, 0, 0, rm); - int8_t cop; - int64_t filterValue; - cop = COMPARE_GE; - filterValue = 3010; - step0.addFilter(cop, filterValue); - cop = COMPARE_LE; - filterValue = 3318; - step0.addFilter(cop, filterValue); - step0.setBOP(BOP_AND); - inJs = outJs; + pColScanStep step0(inJs, outJs, dec, cat, 1003, 1000, 12345, 999, 7, 0, 0, rm); + int8_t cop; + int64_t filterValue; + cop = COMPARE_GE; + filterValue = 3010; + step0.addFilter(cop, filterValue); + cop = COMPARE_LE; + filterValue = 3318; + step0.addFilter(cop, filterValue); + step0.setBOP(BOP_AND); + inJs = outJs; - step0.run(); + step0.run(); - step0.join(); + step0.join(); - DeliveryStep step1(inJs, outJs, make_table("CALPONTSYS", "SYSTABLE"), cat, 1000, 0, 1, 0); - inJs = outJs; + DeliveryStep step1(inJs, outJs, make_table("CALPONTSYS", "SYSTABLE"), cat, 1000, 0, 1, 0); + inJs = outJs; - step1.run(); + step1.run(); - step1.join(); + step1.join(); - return 0; + return 0; } - diff --git a/dbcon/joblist/tdriver2.cpp b/dbcon/joblist/tdriver2.cpp index 40489b9f9..9ac247c53 100644 --- a/dbcon/joblist/tdriver2.cpp +++ b/dbcon/joblist/tdriver2.cpp @@ -35,68 +35,69 @@ boost::shared_ptr csc; string toString(const string& tb, const string& col) { - ostringstream oss; + ostringstream oss; - OID_t colOID = csc->lookupOID(make_tcn("calpontsys", tb, col)); - oss << tb << " " << setw(24) << col << " " << colOID; - ColType_t ct = csc->colType(colOID); - oss << " " << setw(4) << ct.colWidth << " " << setw(4) << ct.ddn.dictOID << " " << setw(2) << ct.colPosition; + OID_t colOID = csc->lookupOID(make_tcn("calpontsys", tb, col)); + oss << tb << " " << setw(24) << col << " " << colOID; + ColType_t ct = csc->colType(colOID); + oss << " " << setw(4) << ct.colWidth << " " << setw(4) << ct.ddn.dictOID << " " << setw(2) + << ct.colPosition; - return oss.str(); + return oss.str(); } -} +} // namespace int main(int argc, char** argv) { - csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); - ROPair_t rp; + csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); + ROPair_t rp; - string calpontsys("calpontsys"); - string table; + string calpontsys("calpontsys"); + string table; - table = "systable"; - rp = csc->tableRID(make_table(calpontsys, table)); - cout << " " << table << ": " << rp.objnum << endl; - cout << toString(table, "tablename") << endl; - cout << toString(table, "schema") << endl; - cout << toString(table, "objectid") << endl; - cout << toString(table, "createdate") << endl; - cout << toString(table, "lastupdate") << endl; - cout << toString(table, "init") << endl; - cout << toString(table, "next") << endl; - cout << toString(table, "numofrows") << endl; - cout << toString(table, "avgrowlen") << endl; - cout << toString(table, "numofblocks") << endl; - cout << toString(table, "autoincrement") << endl; - cout << endl; + table = "systable"; + rp = csc->tableRID(make_table(calpontsys, table)); + cout << " " << table << ": " << rp.objnum << endl; + cout << toString(table, "tablename") << endl; + cout << toString(table, "schema") << endl; + cout << toString(table, "objectid") << endl; + cout << toString(table, "createdate") << endl; + cout << toString(table, "lastupdate") << endl; + cout << toString(table, "init") << endl; + cout << toString(table, "next") << endl; + cout << toString(table, "numofrows") << endl; + cout << toString(table, "avgrowlen") << endl; + cout << toString(table, "numofblocks") << endl; + cout << toString(table, "autoincrement") << endl; + cout << endl; - table = "syscolumn"; - rp = csc->tableRID(make_table(calpontsys, table)); - cout << " " << table << ": " << rp.objnum << endl; - cout << toString(table, "schema") << endl; - cout << toString(table, "tablename") << endl; - cout << toString(table, "columnname") << endl; - cout << toString(table, "objectid") << endl; - cout << toString(table, "dictobjectid") << endl; - cout << toString(table, "listobjectid") << endl; - cout << toString(table, "treeobjectid") << endl; - cout << toString(table, "datatype") << endl; - cout << toString(table, "columnlength") << endl; - cout << toString(table, "columnposition") << endl; - cout << toString(table, "lastupdate") << endl; - cout << toString(table, "defaultvalue") << endl; - cout << toString(table, "nullable") << endl; - cout << toString(table, "scale") << endl; - cout << toString(table, "prec") << endl; - cout << toString(table, "autoincrement") << endl; - cout << toString(table, "distcount") << endl; - cout << toString(table, "nullcount") << endl; - cout << toString(table, "minvalue") << endl; - cout << toString(table, "maxvalue") << endl; - cout << toString(table, "compressiontype") << endl; - cout << toString(table, "nextvalue") << endl; - cout << endl; + table = "syscolumn"; + rp = csc->tableRID(make_table(calpontsys, table)); + cout << " " << table << ": " << rp.objnum << endl; + cout << toString(table, "schema") << endl; + cout << toString(table, "tablename") << endl; + cout << toString(table, "columnname") << endl; + cout << toString(table, "objectid") << endl; + cout << toString(table, "dictobjectid") << endl; + cout << toString(table, "listobjectid") << endl; + cout << toString(table, "treeobjectid") << endl; + cout << toString(table, "datatype") << endl; + cout << toString(table, "columnlength") << endl; + cout << toString(table, "columnposition") << endl; + cout << toString(table, "lastupdate") << endl; + cout << toString(table, "defaultvalue") << endl; + cout << toString(table, "nullable") << endl; + cout << toString(table, "scale") << endl; + cout << toString(table, "prec") << endl; + cout << toString(table, "autoincrement") << endl; + cout << toString(table, "distcount") << endl; + cout << toString(table, "nullcount") << endl; + cout << toString(table, "minvalue") << endl; + cout << toString(table, "maxvalue") << endl; + cout << toString(table, "compressiontype") << endl; + cout << toString(table, "nextvalue") << endl; + cout << endl; #if 0 table = "sysindexcol"; rp = csc->tableRID(make_table(calpontsys, table)); @@ -157,6 +158,5 @@ int main(int argc, char** argv) cout << toString(table, "lastanalysisdate") << endl; cout << endl; #endif - return 0; + return 0; } - diff --git a/dbcon/joblist/threadsafequeue.h b/dbcon/joblist/threadsafequeue.h index efc677756..7b73de36f 100644 --- a/dbcon/joblist/threadsafequeue.h +++ b/dbcon/joblist/threadsafequeue.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: threadsafequeue.h 9655 2013-06-25 23:08:13Z xlou $ -*/ + * $Id: threadsafequeue.h 9655 2013-06-25 23:08:13Z xlou $ + */ /** @file */ #ifndef THREADSAFEQUEUE_H_ @@ -31,19 +31,18 @@ #include #if defined(_MSC_VER) && !defined(_WIN64) -# ifndef InterlockedAdd -# define InterlockedAdd64 InterlockedAdd -# define InterlockedAdd(x, y) ((x) + (y)) -# endif +#ifndef InterlockedAdd +#define InterlockedAdd64 InterlockedAdd +#define InterlockedAdd(x, y) ((x) + (y)) +#endif #endif namespace joblist { - struct TSQSize_t { - size_t size; - uint32_t count; + size_t size; + uint32_t count; }; /** @brief A thread-safe queue class @@ -57,24 +56,24 @@ struct TSQSize_t template class ThreadSafeQueue { -public: - typedef T value_type; + public: + typedef T value_type; - /** @brief constructor - * - * @warning this class takes ownership of the passed-in pointers. - */ - ThreadSafeQueue(boost::mutex* pimplLock = 0, boost::condition* pimplCond = 0) : - fShutdown(false), bytes(0), zeroCount(0) - { - fPimplLock.reset(pimplLock); - fPimplCond.reset(pimplCond); - } - /** @brief destructor - * - */ - ~ThreadSafeQueue() - { + /** @brief constructor + * + * @warning this class takes ownership of the passed-in pointers. + */ + ThreadSafeQueue(boost::mutex* pimplLock = 0, boost::condition* pimplCond = 0) + : fShutdown(false), bytes(0), zeroCount(0) + { + fPimplLock.reset(pimplLock); + fPimplCond.reset(pimplCond); + } + /** @brief destructor + * + */ + ~ThreadSafeQueue() + { #if 0 try @@ -86,7 +85,7 @@ public: } #endif - } + } #if 0 /** @brief get the head of the queue * @@ -114,242 +113,240 @@ public: return fImpl.front(); } #endif - /** @brief get the head of the queue - * - * Return a read/write reference to the head of the queue, leaving it there. This method will block - * until there is something to return. - */ - T& front() + /** @brief get the head of the queue + * + * Return a read/write reference to the head of the queue, leaving it there. This method will block + * until there is something to return. + */ + T& front() + { + if (fPimplLock == 0 || fPimplCond == 0) + throw std::runtime_error("TSQ: front(): no sync!"); + + boost::mutex::scoped_lock lk(*fPimplLock); + + if (fImpl.empty()) { - if (fPimplLock == 0 || fPimplCond == 0) - throw std::runtime_error("TSQ: front(): no sync!"); - - boost::mutex::scoped_lock lk(*fPimplLock); - - if (fImpl.empty()) - { - do - { - fPimplCond->wait(lk); - - if (fShutdown) return fBs0; - } - while (fImpl.empty()); - } - - return fImpl.front(); - } - - /** @brief put an item on the end of the queue - * - * Signals all threads waiting in front() to continue. - */ - TSQSize_t push(const T& v) - { - TSQSize_t ret = {0, 0}; - - if (fPimplLock == 0 || fPimplCond == 0) - throw std::runtime_error("TSQ: push(): no sync!"); + do + { + fPimplCond->wait(lk); if (fShutdown) - return ret; - - boost::mutex::scoped_lock lk(*fPimplLock); - fImpl.push(v); - bytes += v->lengthWithHdrOverhead(); - fPimplCond->notify_one(); - ret.size = bytes; - ret.count = static_cast(fImpl.size()); - return ret; + return fBs0; + } while (fImpl.empty()); } - /** @brief remove the front item in the queue - * - */ - TSQSize_t pop(T* out = NULL) + + return fImpl.front(); + } + + /** @brief put an item on the end of the queue + * + * Signals all threads waiting in front() to continue. + */ + TSQSize_t push(const T& v) + { + TSQSize_t ret = {0, 0}; + + if (fPimplLock == 0 || fPimplCond == 0) + throw std::runtime_error("TSQ: push(): no sync!"); + + if (fShutdown) + return ret; + + boost::mutex::scoped_lock lk(*fPimplLock); + fImpl.push(v); + bytes += v->lengthWithHdrOverhead(); + fPimplCond->notify_one(); + ret.size = bytes; + ret.count = static_cast(fImpl.size()); + return ret; + } + /** @brief remove the front item in the queue + * + */ + TSQSize_t pop(T* out = NULL) + { + TSQSize_t ret = {0, 0}; + + if (fPimplLock == 0) + throw std::runtime_error("TSQ: pop(): no sync!"); + + if (fShutdown) { - TSQSize_t ret = {0, 0}; + *out = fBs0; + return ret; + } - if (fPimplLock == 0) - throw std::runtime_error("TSQ: pop(): no sync!"); + boost::mutex::scoped_lock lk(*fPimplLock); - if (fShutdown) + if (out != NULL) + { + if (fImpl.empty()) + { + do { + if (fShutdown) + { *out = fBs0; return ret; - } + } - boost::mutex::scoped_lock lk(*fPimplLock); + fPimplCond->wait(lk); - if (out != NULL) - { - if (fImpl.empty()) - { - do - { - if (fShutdown) - { - *out = fBs0; - return ret; - } - - fPimplCond->wait(lk); - - if (fShutdown) - { - *out = fBs0; - return ret; - } - } - while (fImpl.empty()); - } - - *out = fImpl.front(); - bytes -= (*out)->lengthWithHdrOverhead(); - } - - fImpl.pop(); - ret.size = bytes; - ret.count = static_cast(fImpl.size()); - return ret; - } - - /* If there are less than min elements in the queue, this fcn will return nothing - * for up to 10 consecutive calls (poor man's timer). On the 11th, it will return - * the entire queue. Note, the zeroCount var is non-critical. Not a big deal if - * it gets fudged now and then. */ - TSQSize_t pop_some(uint32_t divisor, std::vector& t, uint32_t min = 1) - { - uint32_t curSize, workSize; - TSQSize_t ret = {0, 0}; - - if (fPimplLock == 0) - throw std::runtime_error("TSQ: pop_some(): no sync!"); - - t.clear(); - - if (fShutdown) + if (fShutdown) + { + *out = fBs0; return ret; + } + } while (fImpl.empty()); + } - boost::mutex::scoped_lock lk(*fPimplLock); - curSize = fImpl.size(); - - if (curSize < min) - { - workSize = 0; - zeroCount++; - } - else if (curSize / divisor <= min) - { - workSize = min; - zeroCount = 0; - } - else - { - workSize = curSize / divisor; - zeroCount = 0; - } - - if (zeroCount > 10) - { - workSize = curSize; - zeroCount = 0; - } - - for (uint32_t i = 0; i < workSize; ++i) - { - t.push_back(fImpl.front()); - bytes -= fImpl.front()->lengthWithHdrOverhead(); - fImpl.pop(); - } - - ret.count = fImpl.size(); - ret.size = bytes; - return ret; + *out = fImpl.front(); + bytes -= (*out)->lengthWithHdrOverhead(); } - inline void pop_all(std::vector& t) + fImpl.pop(); + ret.size = bytes; + ret.count = static_cast(fImpl.size()); + return ret; + } + + /* If there are less than min elements in the queue, this fcn will return nothing + * for up to 10 consecutive calls (poor man's timer). On the 11th, it will return + * the entire queue. Note, the zeroCount var is non-critical. Not a big deal if + * it gets fudged now and then. */ + TSQSize_t pop_some(uint32_t divisor, std::vector& t, uint32_t min = 1) + { + uint32_t curSize, workSize; + TSQSize_t ret = {0, 0}; + + if (fPimplLock == 0) + throw std::runtime_error("TSQ: pop_some(): no sync!"); + + t.clear(); + + if (fShutdown) + return ret; + + boost::mutex::scoped_lock lk(*fPimplLock); + curSize = fImpl.size(); + + if (curSize < min) { - pop_some(1, t); + workSize = 0; + zeroCount++; } - - /** @brief is the queue empty - * - */ - bool empty() const + else if (curSize / divisor <= min) { - if (fPimplLock == 0) - throw std::runtime_error("TSQ: empty(): no sync!"); - - boost::mutex::scoped_lock lk(*fPimplLock); - return fImpl.empty(); + workSize = min; + zeroCount = 0; } - /** @brief how many items are in the queue - * - */ - TSQSize_t size() const + else { - TSQSize_t ret; - - if (fPimplLock == 0) - throw std::runtime_error("TSQ: size(): no sync!"); - - boost::mutex::scoped_lock lk(*fPimplLock); - ret.size = bytes; - ret.count = fImpl.size(); - return ret; + workSize = curSize / divisor; + zeroCount = 0; } - /** @brief shutdown the queue - * - * cause all readers blocked in front() to return a default-constructed T - */ - void shutdown() + if (zeroCount > 10) { - fShutdown = true; - - if (fPimplCond != 0) - fPimplCond->notify_all(); - - return; + workSize = curSize; + zeroCount = 0; } - void clear() + for (uint32_t i = 0; i < workSize; ++i) { - if (fPimplLock == 0) - throw std::runtime_error("TSQ: clear(): no sync!"); - - boost::mutex::scoped_lock lk(*fPimplLock); - - while ( !fImpl.empty() ) - fImpl.pop(); - - bytes = 0; - return; + t.push_back(fImpl.front()); + bytes -= fImpl.front()->lengthWithHdrOverhead(); + fImpl.pop(); } -private: - typedef std::queue impl_type; - typedef boost::shared_ptr SPBM; - typedef boost::shared_ptr SPBC; + ret.count = fImpl.size(); + ret.size = bytes; + return ret; + } - //defaults okay - //ThreadSafeQueue(const ThreadSafeQueue& rhs); - //ThreadSafeQueue& operator=(const ThreadSafeQueue& rhs); + inline void pop_all(std::vector& t) + { + pop_some(1, t); + } - impl_type fImpl; - SPBM fPimplLock; - SPBC fPimplCond; - volatile bool fShutdown; - T fBs0; + /** @brief is the queue empty + * + */ + bool empty() const + { + if (fPimplLock == 0) + throw std::runtime_error("TSQ: empty(): no sync!"); + + boost::mutex::scoped_lock lk(*fPimplLock); + return fImpl.empty(); + } + /** @brief how many items are in the queue + * + */ + TSQSize_t size() const + { + TSQSize_t ret; + + if (fPimplLock == 0) + throw std::runtime_error("TSQ: size(): no sync!"); + + boost::mutex::scoped_lock lk(*fPimplLock); + ret.size = bytes; + ret.count = fImpl.size(); + return ret; + } + + /** @brief shutdown the queue + * + * cause all readers blocked in front() to return a default-constructed T + */ + void shutdown() + { + fShutdown = true; + + if (fPimplCond != 0) + fPimplCond->notify_all(); + + return; + } + + void clear() + { + if (fPimplLock == 0) + throw std::runtime_error("TSQ: clear(): no sync!"); + + boost::mutex::scoped_lock lk(*fPimplLock); + + while (!fImpl.empty()) + fImpl.pop(); + + bytes = 0; + return; + } + + private: + typedef std::queue impl_type; + typedef boost::shared_ptr SPBM; + typedef boost::shared_ptr SPBC; + + // defaults okay + // ThreadSafeQueue(const ThreadSafeQueue& rhs); + // ThreadSafeQueue& operator=(const ThreadSafeQueue& rhs); + + impl_type fImpl; + SPBM fPimplLock; + SPBC fPimplCond; + volatile bool fShutdown; + T fBs0; #ifdef _MSC_VER - volatile LONG bytes; + volatile LONG bytes; #else - size_t bytes; + size_t bytes; #endif - uint32_t zeroCount; // counts the # of times read_some returned 0 + uint32_t zeroCount; // counts the # of times read_some returned 0 }; -} +} // namespace joblist #endif - diff --git a/dbcon/joblist/timeset.h b/dbcon/joblist/timeset.h index 1088f1209..6389142a6 100644 --- a/dbcon/joblist/timeset.h +++ b/dbcon/joblist/timeset.h @@ -34,7 +34,6 @@ namespace joblist { - //------------------------------------------------------------------------------ /** @brief Maintains a group of simulated timers and their elapsed times. * @@ -44,62 +43,60 @@ namespace joblist //------------------------------------------------------------------------------ class TimeSet { - // TimerMap maps a timer to its start time - // ElapsedMap maps a timer to its cumulative elapsed time - typedef std::map TimerMap; - typedef std::map ElapsedMap; + // TimerMap maps a timer to its start time + // ElapsedMap maps a timer to its cumulative elapsed time + typedef std::map TimerMap; + typedef std::map ElapsedMap; -public: - TimeSet() : fTimer(), fElapsed() {} - void displayAll() const; - void display (const std::string& key) const; - double totalTime () const; - double totalTime (const std::string& key) const; - void holdTimer (const std::string& key); - void startTimer(const std::string& key); - void stopTimer (const std::string& key); - void setTimer (const std::string& key, bool start = true); - TimeSet& operator+=(const TimeSet& rhs); - void clear() - { - fTimer.clear(); - fElapsed.clear(); - } + public: + TimeSet() : fTimer(), fElapsed() + { + } + void displayAll() const; + void display(const std::string& key) const; + double totalTime() const; + double totalTime(const std::string& key) const; + void holdTimer(const std::string& key); + void startTimer(const std::string& key); + void stopTimer(const std::string& key); + void setTimer(const std::string& key, bool start = true); + TimeSet& operator+=(const TimeSet& rhs); + void clear() + { + fTimer.clear(); + fElapsed.clear(); + } -private: - void timespec_sub(const struct timespec& tv1, // start time - const struct timespec& tv2, // end time - struct timespec& diff) const; - void timespec_add(const struct timespec& tv1, - const struct timespec& tv2, - struct timespec& sum) const; + private: + void timespec_sub(const struct timespec& tv1, // start time + const struct timespec& tv2, // end time + struct timespec& diff) const; + void timespec_add(const struct timespec& tv1, const struct timespec& tv2, struct timespec& sum) const; - TimerMap fTimer; - ElapsedMap fElapsed; + TimerMap fTimer; + ElapsedMap fElapsed; }; //------------------------------------------------------------------------------ // Print the contents (to std::cout) of all timers in the ElapsedMap map. //------------------------------------------------------------------------------ -inline -void TimeSet::displayAll() const +inline void TimeSet::displayAll() const { - ElapsedMap::const_iterator itend = fElapsed.end(); + ElapsedMap::const_iterator itend = fElapsed.end(); - for (ElapsedMap::const_iterator it = fElapsed.begin(); it != itend; ++it) - { - double t; + for (ElapsedMap::const_iterator it = fElapsed.begin(); it != itend; ++it) + { + double t; #if defined(_MSC_VER) && defined(_my_pthread_h) - //FIXME - t = 0.0; + // FIXME + t = 0.0; #else - t = (double)it->second.tv_sec + - (double)it->second.tv_nsec / 1000000000.0; + t = (double)it->second.tv_sec + (double)it->second.tv_nsec / 1000000000.0; #endif - std::cout << "TimeSet " << it->first << ": " << t << "s\n"; - } + std::cout << "TimeSet " << it->first << ": " << t << "s\n"; + } - std::cout.flush(); + std::cout.flush(); } //------------------------------------------------------------------------------ @@ -109,94 +106,87 @@ void TimeSet::displayAll() const // then the contents of that timer will be printed. // key (in) - string that identifies the timer to be printed. //------------------------------------------------------------------------------ -inline -void TimeSet::display(const std::string& key) const +inline void TimeSet::display(const std::string& key) const { - ElapsedMap::const_iterator em = fElapsed.find(key); + ElapsedMap::const_iterator em = fElapsed.find(key); - if (fElapsed.end() != em) - { - double t; + if (fElapsed.end() != em) + { + double t; #if defined(_MSC_VER) && defined(_my_pthread_h) - t = 0.0; + t = 0.0; #else - t = (double)em->second.tv_sec + - (double)em->second.tv_nsec / 1000000000.0; + t = (double)em->second.tv_sec + (double)em->second.tv_nsec / 1000000000.0; #endif - std::cout << "TimeSet elapse " << em->first << ": " << t << "s\n"; - } - else + std::cout << "TimeSet elapse " << em->first << ": " << t << "s\n"; + } + else + { + TimerMap::const_iterator tm = fTimer.find(key); + + if (fTimer.end() != tm) { - TimerMap::const_iterator tm = fTimer.find(key); - - if (fTimer.end() != tm) - { - double t; + double t; #if defined(_MSC_VER) && defined(_my_pthread_h) - t = 0.0; + t = 0.0; #else - t = (double)tm->second.tv_sec + - (double)tm->second.tv_nsec / 1000000000.0; + t = (double)tm->second.tv_sec + (double)tm->second.tv_nsec / 1000000000.0; #endif - std::cout << "TimeSet start " << tm->first << ": " << t << "s\n"; - } + std::cout << "TimeSet start " << tm->first << ": " << t << "s\n"; } + } - std::cout.flush(); + std::cout.flush(); } //------------------------------------------------------------------------------ // Return sum of all timer elapsed times. //------------------------------------------------------------------------------ -inline -double TimeSet::totalTime() const +inline double TimeSet::totalTime() const { - struct timespec tSum = {0, 0}; - ElapsedMap::const_iterator itend = fElapsed.end(); + struct timespec tSum = {0, 0}; + ElapsedMap::const_iterator itend = fElapsed.end(); - for (ElapsedMap::const_iterator it = fElapsed.begin(); it != itend; ++it) - { + for (ElapsedMap::const_iterator it = fElapsed.begin(); it != itend; ++it) + { #if defined(_MSC_VER) && !defined(_my_pthread_h) - tSum.tv_sec += it->second.tv_sec; - tSum.tv_nsec += it->second.tv_nsec; + tSum.tv_sec += it->second.tv_sec; + tSum.tv_nsec += it->second.tv_nsec; #endif - } + } - double totSeconds; + double totSeconds; #if defined(_MSC_VER) && defined(_my_pthread_h) - totSeconds = 0.0; + totSeconds = 0.0; #else - totSeconds = (double)tSum.tv_sec + (double)tSum.tv_nsec / 1000000000.0; + totSeconds = (double)tSum.tv_sec + (double)tSum.tv_nsec / 1000000000.0; #endif - return totSeconds; + return totSeconds; } //------------------------------------------------------------------------------ // Return current elapsed time for the specified timer. // key (in) - string that identifies the timer of interest. //------------------------------------------------------------------------------ -inline -double TimeSet::totalTime(const std::string& key) const +inline double TimeSet::totalTime(const std::string& key) const { - ElapsedMap::const_iterator el = fElapsed.find(key); + ElapsedMap::const_iterator el = fElapsed.find(key); - if (fElapsed.end() != el) - { - double totSeconds; + if (fElapsed.end() != el) + { + double totSeconds; #if defined(_MSC_VER) && defined(_my_pthread_h) - totSeconds = 0.0; + totSeconds = 0.0; #else - totSeconds = - (double)el->second.tv_sec + - (double)el->second.tv_nsec / 1000000000.0; + totSeconds = (double)el->second.tv_sec + (double)el->second.tv_nsec / 1000000000.0; #endif - return totSeconds; - } - else - { - return 0; - } + return totSeconds; + } + else + { + return 0; + } } //------------------------------------------------------------------------------ @@ -207,26 +197,25 @@ double TimeSet::totalTime(const std::string& key) const // is time to reset the start time and effectively resume or restart the timer. // key (in) - string that identifies the timer to be "held". //------------------------------------------------------------------------------ -inline -void TimeSet::holdTimer(const std::string& key) +inline void TimeSet::holdTimer(const std::string& key) { - TimerMap::iterator it = fTimer.find(key); + TimerMap::iterator it = fTimer.find(key); - if (fTimer.end() != it) - { - struct timespec tEnd; - struct timespec tDiff; + if (fTimer.end() != it) + { + struct timespec tEnd; + struct timespec tDiff; #if defined(CLOCK_REALTIME) - clock_gettime(CLOCK_REALTIME, &tEnd); + clock_gettime(CLOCK_REALTIME, &tEnd); #elif defined(_MSC_VER) && defined(_my_pthread_h) - tEnd.tv.i64 = tEnd.max_timeout_msec = 0; + tEnd.tv.i64 = tEnd.max_timeout_msec = 0; #else - tEnd.tv_sec = tEnd.tv_nsec = 0; + tEnd.tv_sec = tEnd.tv_nsec = 0; #endif - timespec_sub( it->second, tEnd, tDiff ); - struct timespec tElapsed = fElapsed[key]; - timespec_add( tElapsed, tDiff, fElapsed[key] ); - } + timespec_sub(it->second, tEnd, tDiff); + struct timespec tElapsed = fElapsed[key]; + timespec_add(tElapsed, tDiff, fElapsed[key]); + } } //------------------------------------------------------------------------------ @@ -234,18 +223,17 @@ void TimeSet::holdTimer(const std::string& key) // Start time is set to current time. // key (in) - string that identifies the timer to be started. //------------------------------------------------------------------------------ -inline -void TimeSet::startTimer(const std::string& key) +inline void TimeSet::startTimer(const std::string& key) { - struct timespec ts; + struct timespec ts; #if defined(CLOCK_REALTIME) - clock_gettime(CLOCK_REALTIME, &ts); + clock_gettime(CLOCK_REALTIME, &ts); #elif defined(_MSC_VER) && defined(_my_pthread_h) - ts.tv.i64 = ts.max_timeout_msec = 0; + ts.tv.i64 = ts.max_timeout_msec = 0; #else - ts.tv_sec = ts.tv_nsec = 0; + ts.tv_sec = ts.tv_nsec = 0; #endif - fTimer[key] = ts; + fTimer[key] = ts; } //------------------------------------------------------------------------------ @@ -253,26 +241,25 @@ void TimeSet::startTimer(const std::string& key) // Elapsed time is recorded, and start time is set to current time. // key (in) - string that identifies the timer to be stopped. //------------------------------------------------------------------------------ -inline -void TimeSet::stopTimer(const std::string& key) +inline void TimeSet::stopTimer(const std::string& key) { - TimerMap::iterator it = fTimer.find(key); + TimerMap::iterator it = fTimer.find(key); - if (fTimer.end() != it) - { - struct timespec tEnd; - struct timespec tDiff; + if (fTimer.end() != it) + { + struct timespec tEnd; + struct timespec tDiff; #if defined(CLOCK_REALTIME) - clock_gettime(CLOCK_REALTIME, &tEnd); + clock_gettime(CLOCK_REALTIME, &tEnd); #elif defined(_MSC_VER) && defined(_my_pthread_h) - tEnd.tv.i64 = tEnd.max_timeout_msec = 0; + tEnd.tv.i64 = tEnd.max_timeout_msec = 0; #else - tEnd.tv_sec = tEnd.tv_nsec = 0; + tEnd.tv_sec = tEnd.tv_nsec = 0; #endif - timespec_sub( it->second, tEnd, tDiff ); - fElapsed[key] = tDiff; - fTimer[key] = tEnd; - } + timespec_sub(it->second, tEnd, tDiff); + fElapsed[key] = tDiff; + fTimer[key] = tEnd; + } } //------------------------------------------------------------------------------ @@ -280,31 +267,29 @@ void TimeSet::stopTimer(const std::string& key) // key (in) - string that identifies the timer to be started or stopped. // start (in) - boolean indicating whether to start or stop the timer. //------------------------------------------------------------------------------ -inline -void TimeSet::setTimer(const std::string& key, bool start /*= true*/) +inline void TimeSet::setTimer(const std::string& key, bool start /*= true*/) { - if (start) - startTimer(key); - else - stopTimer(key); + if (start) + startTimer(key); + else + stopTimer(key); } //------------------------------------------------------------------------------ // Adds the specified TimeSet elapsed times to "this" TimeSet. // rhs (in) - TimeSet with elapsed times to be added to "this" TimeSet. //------------------------------------------------------------------------------ -inline -TimeSet& TimeSet::operator+=(const TimeSet& rhs) +inline TimeSet& TimeSet::operator+=(const TimeSet& rhs) { - ElapsedMap::const_iterator itend = rhs.fElapsed.end(); + ElapsedMap::const_iterator itend = rhs.fElapsed.end(); - for (ElapsedMap::const_iterator it = rhs.fElapsed.begin(); it != itend; ++it) - { - struct timespec tLhs = fElapsed[it->first]; - timespec_add( tLhs, it->second, fElapsed[it->first] ); - } + for (ElapsedMap::const_iterator it = rhs.fElapsed.begin(); it != itend; ++it) + { + struct timespec tLhs = fElapsed[it->first]; + timespec_add(tLhs, it->second, fElapsed[it->first]); + } - return (*this); + return (*this); } //------------------------------------------------------------------------------ @@ -313,25 +298,24 @@ TimeSet& TimeSet::operator+=(const TimeSet& rhs) // tv2 (in) - second timespec value // diff (out) - value of tv2 - tv1 //------------------------------------------------------------------------------ -inline -void TimeSet::timespec_sub(const struct timespec& tv1, // start time - const struct timespec& tv2, // end time - struct timespec& diff) const +inline void TimeSet::timespec_sub(const struct timespec& tv1, // start time + const struct timespec& tv2, // end time + struct timespec& diff) const { #if defined(_MSC_VER) && defined(_my_pthread_h) - diff.tv.i64 = diff.max_timeout_msec = 0; + diff.tv.i64 = diff.max_timeout_msec = 0; #else - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv2.tv_nsec + 1000000000 - tv1.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv2.tv_nsec + 1000000000 - tv1.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } #endif } @@ -342,25 +326,24 @@ void TimeSet::timespec_sub(const struct timespec& tv1, // start time // tv2 (in) - second timespec value // sum (out) - sum of tv1 and tv2 //------------------------------------------------------------------------------ -inline -void TimeSet::timespec_add(const struct timespec& tv1, // start time - const struct timespec& tv2, // end time - struct timespec& sum) const +inline void TimeSet::timespec_add(const struct timespec& tv1, // start time + const struct timespec& tv2, // end time + struct timespec& sum) const { #if defined(_MSC_VER) && defined(_my_pthread_h) - sum.tv.i64 = sum.max_timeout_msec = 0; + sum.tv.i64 = sum.max_timeout_msec = 0; #else - sum.tv_sec = tv1.tv_sec + tv2.tv_sec; - sum.tv_nsec = tv1.tv_nsec + tv2.tv_nsec; + sum.tv_sec = tv1.tv_sec + tv2.tv_sec; + sum.tv_nsec = tv1.tv_nsec + tv2.tv_nsec; - if (sum.tv_nsec >= 1000000000) - { - sum.tv_sec += 1; - sum.tv_nsec -= 1000000000; - } + if (sum.tv_nsec >= 1000000000) + { + sum.tv_sec += 1; + sum.tv_nsec -= 1000000000; + } #endif } -}; +}; // namespace joblist #endif diff --git a/dbcon/joblist/timestamp.cpp b/dbcon/joblist/timestamp.cpp index cef39f64c..ecb531634 100644 --- a/dbcon/joblist/timestamp.cpp +++ b/dbcon/joblist/timestamp.cpp @@ -36,76 +36,74 @@ const struct timeval zerotime = {0, 0}; namespace joblist { - -JSTimeStamp::JSTimeStamp() : - fFirstInsertTime(zerotime), - fLastInsertTime(zerotime), - fEndofInputTime(zerotime), - fFirstReadTime(zerotime), - fLastReadTime(zerotime) +JSTimeStamp::JSTimeStamp() + : fFirstInsertTime(zerotime) + , fLastInsertTime(zerotime) + , fEndofInputTime(zerotime) + , fFirstReadTime(zerotime) + , fLastReadTime(zerotime) { } /* static */ const string JSTimeStamp::format(const struct timeval& tvbuf) { - string res; - char timeString[50]; - struct tm tmbuf; + string res; + char timeString[50]; + struct tm tmbuf; #ifdef _MSC_VER - errno_t p = 0; - time_t t = tvbuf.tv_sec; - p = localtime_s(&tmbuf, &t); + errno_t p = 0; + time_t t = tvbuf.tv_sec; + p = localtime_s(&tmbuf, &t); - if (p != 0) - { - memset(&tmbuf, 0, sizeof(tmbuf)); - strcpy(timeString, "UNKNOWN"); - } - else - { - if (strftime(timeString, 50, "%Y-%m-%d %H:%M:%S", &tmbuf) == 0) - strcpy(timeString, "UNKNOWN"); - } + if (p != 0) + { + memset(&tmbuf, 0, sizeof(tmbuf)); + strcpy(timeString, "UNKNOWN"); + } + else + { + if (strftime(timeString, 50, "%Y-%m-%d %H:%M:%S", &tmbuf) == 0) + strcpy(timeString, "UNKNOWN"); + } #else - localtime_r(&tvbuf.tv_sec, &tmbuf); - strftime(timeString, 50, "%F %T", &tmbuf); + localtime_r(&tvbuf.tv_sec, &tmbuf); + strftime(timeString, 50, "%F %T", &tmbuf); #endif - const int len = strlen(timeString); - snprintf(&timeString[len], (50 - len), ".%06lu", tvbuf.tv_usec); - res = timeString; - return res; + const int len = strlen(timeString); + snprintf(&timeString[len], (50 - len), ".%06lu", tvbuf.tv_usec); + res = timeString; + return res; } /* static */ const string JSTimeStamp::tsdiffstr(const struct timeval& t2, const struct timeval& t1) { - string res; - int ds; - int dus; - char timeString[50]; - dus = t2.tv_usec - t1.tv_usec; - ds = t2.tv_sec - t1.tv_sec; + string res; + int ds; + int dus; + char timeString[50]; + dus = t2.tv_usec - t1.tv_usec; + ds = t2.tv_sec - t1.tv_sec; - if (dus < 0) - { - ds--; - dus += 1000000; - } + if (dus < 0) + { + ds--; + dus += 1000000; + } - snprintf(timeString, 50, "%d.%06d", ds, dus); - res = timeString; - return res; + snprintf(timeString, 50, "%d.%06d", ds, dus); + res = timeString; + return res; } /* static */ const string JSTimeStamp::timeNow() { - struct timeval tv; - gettimeofday(&tv, 0); - return JSTimeStamp::format(tv); + struct timeval tv; + gettimeofday(&tv, 0); + return JSTimeStamp::format(tv); } -} //namespace joblist - +} // namespace joblist diff --git a/dbcon/joblist/timestamp.h b/dbcon/joblist/timestamp.h index 1720bb6fe..786f946db 100644 --- a/dbcon/joblist/timestamp.h +++ b/dbcon/joblist/timestamp.h @@ -30,117 +30,113 @@ namespace joblist { - class JSTimeStamp { + public: + JSTimeStamp(); + ~JSTimeStamp(){}; -public: - JSTimeStamp(); - ~JSTimeStamp() { }; + inline void setFirstInsertTime() + { + gettimeofday(&fFirstInsertTime, 0); + } + inline void setLastInsertTime() + { + gettimeofday(&fLastInsertTime, 0); + } + inline void setEndOfInputTime() + { + gettimeofday(&fEndofInputTime, 0); + } + inline void setFirstReadTime() + { + gettimeofday(&fFirstReadTime, 0); + } + inline void setLastReadTime() + { + gettimeofday(&fLastReadTime, 0); + } - inline void setFirstInsertTime() - { - gettimeofday(&fFirstInsertTime, 0); - } - inline void setLastInsertTime() - { - gettimeofday(&fLastInsertTime, 0); - } - inline void setEndOfInputTime() - { - gettimeofday(&fEndofInputTime, 0); - } - inline void setFirstReadTime() - { - gettimeofday(&fFirstReadTime, 0); - } - inline void setLastReadTime() - { - gettimeofday(&fLastReadTime, 0); - } + inline void setFirstInsertTime(const struct timeval& t) + { + fFirstInsertTime = t; + } + inline void setLastInsertTime(const struct timeval& t) + { + fLastInsertTime = t; + } + inline void setEndOfInputTime(const struct timeval& t) + { + fEndofInputTime = t; + } + inline void setFirstReadTime(const struct timeval& t) + { + fFirstReadTime = t; + } + inline void setLastReadTime(const struct timeval& t) + { + fLastReadTime = t; + } - inline void setFirstInsertTime(const struct timeval& t) - { - fFirstInsertTime = t; - } - inline void setLastInsertTime(const struct timeval& t) - { - fLastInsertTime = t; - } - inline void setEndOfInputTime(const struct timeval& t) - { - fEndofInputTime = t; - } - inline void setFirstReadTime(const struct timeval& t) - { - fFirstReadTime = t; - } - inline void setLastReadTime(const struct timeval& t) - { - fLastReadTime = t; - } + inline const struct timeval FirstInsertTime() const + { + return fFirstInsertTime; + } + inline const struct timeval LastInsertTime() const + { + return fLastInsertTime; + } + inline const struct timeval EndOfInputTime() const + { + return fEndofInputTime; + } + inline const struct timeval FirstReadTime() const + { + return fFirstReadTime; + } + inline const struct timeval LastReadTime() const + { + return fLastReadTime; + } - inline const struct timeval FirstInsertTime() const - { - return fFirstInsertTime; - } - inline const struct timeval LastInsertTime() const - { - return fLastInsertTime; - } - inline const struct timeval EndOfInputTime() const - { - return fEndofInputTime; - } - inline const struct timeval FirstReadTime() const - { - return fFirstReadTime; - } - inline const struct timeval LastReadTime() const - { - return fLastReadTime; - } + const std::string FirstInsertTimeString() const + { + return format(fFirstInsertTime); + } + const std::string LastInsertTimeString() const + { + return format(fLastInsertTime); + } + const std::string EndOfInputTimeString() const + { + return format(fEndofInputTime); + } + const std::string FirstReadTimeString() const + { + return format(fFirstReadTime); + } + const std::string LastReadTimeString() const + { + return format(fLastReadTime); + } - const std::string FirstInsertTimeString() const - { - return format(fFirstInsertTime); - } - const std::string LastInsertTimeString() const - { - return format(fLastInsertTime); - } - const std::string EndOfInputTimeString() const - { - return format(fEndofInputTime); - } - const std::string FirstReadTimeString() const - { - return format(fFirstReadTime); - } - const std::string LastReadTimeString() const - { - return format(fLastReadTime); - } + // returns str rep of t2 - t1 in seconds + static const std::string tsdiffstr(const struct timeval& t2, const struct timeval& t1); - //returns str rep of t2 - t1 in seconds - static const std::string tsdiffstr(const struct timeval& t2, const struct timeval& t1); + // returns str rep of tvbuf + static const std::string format(const struct timeval& tvbuf); - //returns str rep of tvbuf - static const std::string format(const struct timeval& tvbuf); + static const std::string timeNow(); - static const std::string timeNow(); - -protected: - -private: - struct timeval fFirstInsertTime; - struct timeval fLastInsertTime; - struct timeval fEndofInputTime; - struct timeval fFirstReadTime; - struct timeval fLastReadTime; + protected: + private: + struct timeval fFirstInsertTime; + struct timeval fLastInsertTime; + struct timeval fEndofInputTime; + struct timeval fFirstReadTime; + struct timeval fLastReadTime; }; -} //namespace joblist +} // namespace joblist #endif - diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 47def09ff..f9a6d7361 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -18,7 +18,6 @@ // $Id: tuple-bps.cpp 9705 2013-07-17 20:06:07Z pleblanc $ - #include //#define NDEBUG #include @@ -83,74 +82,77 @@ extern boost::mutex fileLock_g; namespace { -const uint32_t LOGICAL_EXTENT_CONVERTER = 10; // 10 + 13. 13 to convert to logical blocks, +const uint32_t LOGICAL_EXTENT_CONVERTER = 10; // 10 + 13. 13 to convert to logical blocks, // 10 to convert to groups of 1024 logical blocks const uint32_t DEFAULT_EXTENTS_PER_SEG_FILE = 2; -} +} // namespace /** Debug macro */ #define THROTTLE_DEBUG 0 #if THROTTLE_DEBUG #define THROTTLEDEBUG std::cout #else -#define THROTTLEDEBUG if (false) std::cout +#define THROTTLEDEBUG \ + if (false) \ + std::cout #endif namespace joblist { - struct TupleBPSPrimitive { - TupleBPSPrimitive(TupleBPS* batchPrimitiveStep) : - fBatchPrimitiveStep(batchPrimitiveStep) - {} - TupleBPS* fBatchPrimitiveStep; - void operator()() + TupleBPSPrimitive(TupleBPS* batchPrimitiveStep) : fBatchPrimitiveStep(batchPrimitiveStep) + { + } + TupleBPS* fBatchPrimitiveStep; + void operator()() + { + try { - try - { - utils::setThreadName("BPSPrimitive"); - fBatchPrimitiveStep->sendPrimitiveMessages(); - } - catch (std::exception& re) - { - cerr << "TupleBPS: send thread threw an exception: " << re.what() << - "\t" << this << endl; - catchHandler(re.what(), ERR_TUPLE_BPS, fBatchPrimitiveStep->errorInfo()); - } - catch (...) - { - string msg("TupleBPS: send thread threw an unknown exception "); - catchHandler(msg, ERR_TUPLE_BPS, fBatchPrimitiveStep->errorInfo()); - cerr << msg << this << endl; - } + utils::setThreadName("BPSPrimitive"); + fBatchPrimitiveStep->sendPrimitiveMessages(); } + catch (std::exception& re) + { + cerr << "TupleBPS: send thread threw an exception: " << re.what() << "\t" << this << endl; + catchHandler(re.what(), ERR_TUPLE_BPS, fBatchPrimitiveStep->errorInfo()); + } + catch (...) + { + string msg("TupleBPS: send thread threw an unknown exception "); + catchHandler(msg, ERR_TUPLE_BPS, fBatchPrimitiveStep->errorInfo()); + cerr << msg << this << endl; + } + } }; struct TupleBPSAggregators { - TupleBPSAggregators(TupleBPS* batchPrimitiveStep) : fBatchPrimitiveStepCols(batchPrimitiveStep) {} - TupleBPS* fBatchPrimitiveStepCols; + TupleBPSAggregators(TupleBPS* batchPrimitiveStep) : fBatchPrimitiveStepCols(batchPrimitiveStep) + { + } + TupleBPS* fBatchPrimitiveStepCols; - void operator()() + void operator()() + { + try { - try - { - utils::setThreadName("BPSAggregator"); - fBatchPrimitiveStepCols->receiveMultiPrimitiveMessages(); - } - catch (std::exception& re) - { - cerr << fBatchPrimitiveStepCols->toString() << ": receive thread threw an exception: " << re.what() << endl; - catchHandler(re.what(), ERR_TUPLE_BPS, fBatchPrimitiveStepCols->errorInfo()); - } - catch (...) - { - string msg("TupleBPS: recv thread threw an unknown exception "); - cerr << fBatchPrimitiveStepCols->toString() << msg << endl; - catchHandler(msg, ERR_TUPLE_BPS, fBatchPrimitiveStepCols->errorInfo()); - } + utils::setThreadName("BPSAggregator"); + fBatchPrimitiveStepCols->receiveMultiPrimitiveMessages(); } + catch (std::exception& re) + { + cerr << fBatchPrimitiveStepCols->toString() << ": receive thread threw an exception: " << re.what() + << endl; + catchHandler(re.what(), ERR_TUPLE_BPS, fBatchPrimitiveStepCols->errorInfo()); + } + catch (...) + { + string msg("TupleBPS: recv thread threw an unknown exception "); + cerr << fBatchPrimitiveStepCols->toString() << msg << endl; + catchHandler(msg, ERR_TUPLE_BPS, fBatchPrimitiveStepCols->errorInfo()); + } + } }; TupleBPS::JoinLocalData::JoinLocalData(TupleBPS* pTupleBPS, RowGroup& primRowGroup, RowGroup& outputRowGroup, @@ -160,225 +162,230 @@ TupleBPS::JoinLocalData::JoinLocalData(TupleBPS* pTupleBPS, RowGroup& primRowGro rowgroup::RowGroup& joinFERG, std::vector>& tjoiners, uint32_t smallSideCount, bool doJoin) - : tbps(pTupleBPS), local_primRG(primRowGroup), local_outputRG(outputRowGroup), fe2(fe2), - fe2Output(fe2Output), joinerMatchesRGs(joinerMatchesRGs), joinFERG(joinFERG), tjoiners(tjoiners), - smallSideCount(smallSideCount), doJoin(doJoin) + : tbps(pTupleBPS) + , local_primRG(primRowGroup) + , local_outputRG(outputRowGroup) + , fe2(fe2) + , fe2Output(fe2Output) + , joinerMatchesRGs(joinerMatchesRGs) + , joinFERG(joinFERG) + , tjoiners(tjoiners) + , smallSideCount(smallSideCount) + , doJoin(doJoin) { - if (doJoin || fe2) + if (doJoin || fe2) + { + local_outputRG.initRow(&postJoinRow); + } + + if (fe2) + { + local_fe2Output = fe2Output; + local_fe2Output.initRow(&local_fe2OutRow); + local_fe2Data.reinit(fe2Output); + local_fe2Output.setData(&local_fe2Data); + local_fe2 = *fe2; + } + + if (doJoin) + { + joinerOutput.resize(smallSideCount); + smallSideRows.reset(new Row[smallSideCount]); + smallNulls.reset(new Row[smallSideCount]); + smallMappings.resize(smallSideCount); + fergMappings.resize(smallSideCount + 1); + smallNullMemory.reset(new shared_array[smallSideCount]); + local_primRG.initRow(&largeSideRow); + local_outputRG.initRow(&joinedBaseRow, true); + joinedBaseRowData.reset(new uint8_t[joinedBaseRow.getSize()]); + joinedBaseRow.setData(joinedBaseRowData.get()); + joinedBaseRow.initToNull(); + largeMapping = makeMapping(local_primRG, local_outputRG); + + bool hasJoinFE = false; + + for (uint32_t i = 0; i < smallSideCount; ++i) { - local_outputRG.initRow(&postJoinRow); + joinerMatchesRGs[i].initRow(&(smallSideRows[i])); + smallMappings[i] = makeMapping(joinerMatchesRGs[i], local_outputRG); + + if (tjoiners[i]->hasFEFilter()) + { + fergMappings[i] = makeMapping(joinerMatchesRGs[i], joinFERG); + hasJoinFE = true; + } } - if (fe2) + if (hasJoinFE) { - local_fe2Output = fe2Output; - local_fe2Output.initRow(&local_fe2OutRow); - local_fe2Data.reinit(fe2Output); - local_fe2Output.setData(&local_fe2Data); - local_fe2 = *fe2; + joinFERG.initRow(&joinFERow, true); + joinFERowData.reset(new uint8_t[joinFERow.getSize()]); + memset(joinFERowData.get(), 0, joinFERow.getSize()); + joinFERow.setData(joinFERowData.get()); + fergMappings[smallSideCount] = makeMapping(local_primRG, joinFERG); } - if (doJoin) + for (uint32_t i = 0; i < smallSideCount; ++i) { - joinerOutput.resize(smallSideCount); - smallSideRows.reset(new Row[smallSideCount]); - smallNulls.reset(new Row[smallSideCount]); - smallMappings.resize(smallSideCount); - fergMappings.resize(smallSideCount + 1); - smallNullMemory.reset(new shared_array[smallSideCount]); - local_primRG.initRow(&largeSideRow); - local_outputRG.initRow(&joinedBaseRow, true); - joinedBaseRowData.reset(new uint8_t[joinedBaseRow.getSize()]); - joinedBaseRow.setData(joinedBaseRowData.get()); - joinedBaseRow.initToNull(); - largeMapping = makeMapping(local_primRG, local_outputRG); - - bool hasJoinFE = false; - - for (uint32_t i = 0; i < smallSideCount; ++i) - { - joinerMatchesRGs[i].initRow(&(smallSideRows[i])); - smallMappings[i] = makeMapping(joinerMatchesRGs[i], local_outputRG); - - if (tjoiners[i]->hasFEFilter()) - { - fergMappings[i] = makeMapping(joinerMatchesRGs[i], joinFERG); - hasJoinFE = true; - } - } - - if (hasJoinFE) - { - joinFERG.initRow(&joinFERow, true); - joinFERowData.reset(new uint8_t[joinFERow.getSize()]); - memset(joinFERowData.get(), 0, joinFERow.getSize()); - joinFERow.setData(joinFERowData.get()); - fergMappings[smallSideCount] = makeMapping(local_primRG, joinFERG); - } - - for (uint32_t i = 0; i < smallSideCount; ++i) - { - joinerMatchesRGs[i].initRow(&(smallNulls[i]), true); - smallNullMemory[i].reset(new uint8_t[smallNulls[i].getSize()]); - smallNulls[i].setData(smallNullMemory[i].get()); - smallNulls[i].initToNull(); - } - - local_primRG.initRow(&largeNull, true); - largeNullMemory.reset(new uint8_t[largeNull.getSize()]); - largeNull.setData(largeNullMemory.get()); - largeNull.initToNull(); + joinerMatchesRGs[i].initRow(&(smallNulls[i]), true); + smallNullMemory[i].reset(new uint8_t[smallNulls[i].getSize()]); + smallNulls[i].setData(smallNullMemory[i].get()); + smallNulls[i].initToNull(); } + + local_primRG.initRow(&largeNull, true); + largeNullMemory.reset(new uint8_t[largeNull.getSize()]); + largeNull.setData(largeNullMemory.get()); + largeNull.initToNull(); + } } -uint64_t TupleBPS::JoinLocalData::generateJoinResultSet(const uint32_t depth, +uint64_t TupleBPS::JoinLocalData::generateJoinResultSet(const uint32_t depth, std::vector& outputData, RowGroupDL* dlp) { - uint32_t i; - Row& smallRow = smallSideRows[depth]; - uint64_t memSizeForOutputRG = 0; - - if (depth < smallSideCount - 1) + uint32_t i; + Row& smallRow = smallSideRows[depth]; + uint64_t memSizeForOutputRG = 0; + + if (depth < smallSideCount - 1) + { + for (i = 0; i < joinerOutput[depth].size() && !tbps->cancelled(); i++) { - for (i = 0; i < joinerOutput[depth].size() && !tbps->cancelled(); i++) - { - smallRow.setPointer(joinerOutput[depth][i]); - applyMapping(smallMappings[depth], smallRow, &joinedBaseRow); - memSizeForOutputRG += generateJoinResultSet(depth + 1, outputData, dlp); - } + smallRow.setPointer(joinerOutput[depth][i]); + applyMapping(smallMappings[depth], smallRow, &joinedBaseRow); + memSizeForOutputRG += generateJoinResultSet(depth + 1, outputData, dlp); } - else + } + else + { + local_outputRG.getRow(local_outputRG.getRowCount(), &postJoinRow); + + for (i = 0; i < joinerOutput[depth].size() && !tbps->cancelled(); + i++, postJoinRow.nextRow(), local_outputRG.incRowCount()) { - local_outputRG.getRow(local_outputRG.getRowCount(), &postJoinRow); - - for (i = 0; i < joinerOutput[depth].size() && !tbps->cancelled(); i++, postJoinRow.nextRow(), - local_outputRG.incRowCount()) + smallRow.setPointer(joinerOutput[depth][i]); + + if (UNLIKELY(local_outputRG.getRowCount() == 8192)) + { + uint32_t dbRoot = local_outputRG.getDBRoot(); + uint64_t baseRid = local_outputRG.getBaseRid(); + outputData.push_back(joinedData); + // Don't let the join results buffer get out of control. + if (tbps->resourceManager()->getMemory(local_outputRG.getMaxDataSize(), false)) { - smallRow.setPointer(joinerOutput[depth][i]); - - if (UNLIKELY(local_outputRG.getRowCount() == 8192)) - { - uint32_t dbRoot = local_outputRG.getDBRoot(); - uint64_t baseRid = local_outputRG.getBaseRid(); - outputData.push_back(joinedData); - // Don't let the join results buffer get out of control. - if (tbps->resourceManager()->getMemory(local_outputRG.getMaxDataSize(), false)) - { - memSizeForOutputRG += local_outputRG.getMaxDataSize(); - } - else - { - // Don't wait for memory, just send the data on to DL. - RowGroup out(local_outputRG); - if (fe2 && tbps->runFEonPM()) - { - processFE2(outputData); - tbps->rgDataVecToDl(outputData, local_fe2Output, dlp); - } - else - { - tbps->rgDataVecToDl(outputData, out, dlp); - } - tbps->resourceManager()->returnMemory(memSizeForOutputRG); - memSizeForOutputRG = 0; - } - joinedData.reinit(local_outputRG); - local_outputRG.setData(&joinedData); - local_outputRG.resetRowGroup(baseRid); - local_outputRG.setDBRoot(dbRoot); - local_outputRG.getRow(0, &postJoinRow); - } - - applyMapping(smallMappings[depth], smallRow, &joinedBaseRow); - copyRow(joinedBaseRow, &postJoinRow); + memSizeForOutputRG += local_outputRG.getMaxDataSize(); } + else + { + // Don't wait for memory, just send the data on to DL. + RowGroup out(local_outputRG); + if (fe2 && tbps->runFEonPM()) + { + processFE2(outputData); + tbps->rgDataVecToDl(outputData, local_fe2Output, dlp); + } + else + { + tbps->rgDataVecToDl(outputData, out, dlp); + } + tbps->resourceManager()->returnMemory(memSizeForOutputRG); + memSizeForOutputRG = 0; + } + joinedData.reinit(local_outputRG); + local_outputRG.setData(&joinedData); + local_outputRG.resetRowGroup(baseRid); + local_outputRG.setDBRoot(dbRoot); + local_outputRG.getRow(0, &postJoinRow); + } + + applyMapping(smallMappings[depth], smallRow, &joinedBaseRow); + copyRow(joinedBaseRow, &postJoinRow); } - return memSizeForOutputRG; + } + return memSizeForOutputRG; } void TupleBPS::JoinLocalData::processFE2(vector& rgData) { - vector results; - RGData result; - uint32_t i, j; - bool ret; - - result = RGData(local_fe2Output); - local_fe2Output.setData(&result); - local_fe2Output.resetRowGroup(-1); - local_fe2Output.getRow(0, &local_fe2OutRow); - - for (i = 0; i < rgData.size(); i++) - { - local_outputRG.setData(&(rgData)[i]); - - if (local_fe2Output.getRowCount() == 0) - { - local_fe2Output.resetRowGroup(local_outputRG.getBaseRid()); - local_fe2Output.setDBRoot(local_outputRG.getDBRoot()); - } - - local_outputRG.getRow(0, &postJoinRow); - - for (j = 0; j < local_outputRG.getRowCount(); j++, postJoinRow.nextRow()) - { - ret = local_fe2.evaluate(&postJoinRow); - - if (ret) - { - applyMapping(tbps->fe2Mapping, postJoinRow, &local_fe2OutRow); - local_fe2OutRow.setRid(postJoinRow.getRelRid()); - local_fe2Output.incRowCount(); - local_fe2OutRow.nextRow(); - - if (local_fe2Output.getRowCount() == 8192 || - local_fe2Output.getDBRoot() != local_outputRG.getDBRoot() || - local_fe2Output.getBaseRid() != local_outputRG.getBaseRid() - ) - { - results.push_back(result); - result = RGData(local_fe2Output); - local_fe2Output.setData(&result); - local_fe2Output.resetRowGroup(local_outputRG.getBaseRid()); - local_fe2Output.setDBRoot(local_outputRG.getDBRoot()); - local_fe2Output.getRow(0, &local_fe2OutRow); - } - } - } - } - - if (local_fe2Output.getRowCount() > 0) - { - results.push_back(result); - } - - rgData.swap(results); -} + vector results; + RGData result; + uint32_t i, j; + bool ret; + result = RGData(local_fe2Output); + local_fe2Output.setData(&result); + local_fe2Output.resetRowGroup(-1); + local_fe2Output.getRow(0, &local_fe2OutRow); + + for (i = 0; i < rgData.size(); i++) + { + local_outputRG.setData(&(rgData)[i]); + + if (local_fe2Output.getRowCount() == 0) + { + local_fe2Output.resetRowGroup(local_outputRG.getBaseRid()); + local_fe2Output.setDBRoot(local_outputRG.getDBRoot()); + } + + local_outputRG.getRow(0, &postJoinRow); + + for (j = 0; j < local_outputRG.getRowCount(); j++, postJoinRow.nextRow()) + { + ret = local_fe2.evaluate(&postJoinRow); + + if (ret) + { + applyMapping(tbps->fe2Mapping, postJoinRow, &local_fe2OutRow); + local_fe2OutRow.setRid(postJoinRow.getRelRid()); + local_fe2Output.incRowCount(); + local_fe2OutRow.nextRow(); + + if (local_fe2Output.getRowCount() == 8192 || + local_fe2Output.getDBRoot() != local_outputRG.getDBRoot() || + local_fe2Output.getBaseRid() != local_outputRG.getBaseRid()) + { + results.push_back(result); + result = RGData(local_fe2Output); + local_fe2Output.setData(&result); + local_fe2Output.resetRowGroup(local_outputRG.getBaseRid()); + local_fe2Output.setDBRoot(local_outputRG.getDBRoot()); + local_fe2Output.getRow(0, &local_fe2OutRow); + } + } + } + } + + if (local_fe2Output.getRowCount() > 0) + { + results.push_back(result); + } + + rgData.swap(results); +} struct ByteStreamProcessor { - ByteStreamProcessor(TupleBPS* tbps, vector>& bsv, - const uint32_t begin, const uint32_t end, vector<_CPInfo>& cpv, RowGroupDL* dlp, - const uint32_t threadID) - : tbps(tbps), bsv(bsv), begin(begin), end(end), cpv(cpv), dlp(dlp), threadID(threadID) - { - } + ByteStreamProcessor(TupleBPS* tbps, vector>& bsv, + const uint32_t begin, const uint32_t end, vector<_CPInfo>& cpv, RowGroupDL* dlp, + const uint32_t threadID) + : tbps(tbps), bsv(bsv), begin(begin), end(end), cpv(cpv), dlp(dlp), threadID(threadID) + { + } - TupleBPS* tbps; - vector>& bsv; - uint32_t begin; - uint32_t end; - vector<_CPInfo>& cpv; - RowGroupDL* dlp; - uint32_t threadID; + TupleBPS* tbps; + vector>& bsv; + uint32_t begin; + uint32_t end; + vector<_CPInfo>& cpv; + RowGroupDL* dlp; + uint32_t threadID; - void operator()() - { - utils::setThreadName("ByteStreamProcessor"); - tbps->processByteStreamVector(bsv, begin, end, cpv, dlp, threadID); - } + void operator()() + { + utils::setThreadName("ByteStreamProcessor"); + tbps->processByteStreamVector(bsv, begin, end, cpv, dlp, threadID); + } }; //------------------------------------------------------------------------------ @@ -386,1171 +393,1131 @@ struct ByteStreamProcessor //------------------------------------------------------------------------------ void TupleBPS::initializeConfigParms() { - string strVal; + string strVal; + //...Get the tuning parameters that throttle msgs sent to primproc + //...fFilterRowReqLimit puts a cap on how many rids we will request from + //... primproc, before pausing to let the consumer thread catch up. + //... Without this limit, there is a chance that PrimProc could flood + //... ExeMgr with thousands of messages that will consume massive + //... amounts of memory for a 100 gigabyte database. + //...fFilterRowReqThreshold is the level at which the number of outstanding + //... rids must fall below, before the producer can send more rids. - //...Get the tuning parameters that throttle msgs sent to primproc - //...fFilterRowReqLimit puts a cap on how many rids we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fFilterRowReqThreshold is the level at which the number of outstanding - //... rids must fall below, before the producer can send more rids. + // These could go in constructor + fRequestSize = fRm->getJlRequestSize(); + fMaxOutstandingRequests = fRm->getJlMaxOutstandingRequests(); + fProcessorThreadsPerScan = fRm->getJlProcessorThreadsPerScan(); + fNumThreads = 0; - //These could go in constructor - fRequestSize = fRm->getJlRequestSize(); - fMaxOutstandingRequests = fRm->getJlMaxOutstandingRequests(); - fProcessorThreadsPerScan = fRm->getJlProcessorThreadsPerScan(); - fNumThreads = 0; + fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; - fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; + if (fRequestSize >= fMaxOutstandingRequests) + fRequestSize = 1; - if (fRequestSize >= fMaxOutstandingRequests) - fRequestSize = 1; + if ((fSessionId & 0x80000000) == 0) + { + fMaxNumThreads = fRm->getJlNumScanReceiveThreads(); + fMaxNumProcessorThreads = fMaxNumThreads; + } + else + { + fMaxNumThreads = 1; + fMaxNumProcessorThreads = 1; + } - if ((fSessionId & 0x80000000) == 0) - { - fMaxNumThreads = fRm->getJlNumScanReceiveThreads(); - fMaxNumProcessorThreads = fMaxNumThreads; - } - else - { - fMaxNumThreads = 1; - fMaxNumProcessorThreads = 1; - } - - // Reserve the max number of thread space. A bit of an optimization. - fProducerThreads.clear(); - fProducerThreads.reserve(fMaxNumThreads); + // Reserve the max number of thread space. A bit of an optimization. + fProducerThreads.clear(); + fProducerThreads.reserve(fMaxNumThreads); } -TupleBPS::TupleBPS(const pColStep& rhs, const JobInfo& jobInfo) : - BatchPrimitive(jobInfo), pThread(0), fRm(jobInfo.rm) +TupleBPS::TupleBPS(const pColStep& rhs, const JobInfo& jobInfo) + : BatchPrimitive(jobInfo), pThread(0), fRm(jobInfo.rm) { - fInputJobStepAssociation = rhs.inputAssociation(); - fOutputJobStepAssociation = rhs.outputAssociation(); - fDec = 0; - fSessionId = rhs.sessionId(); - fFilterCount = rhs.filterCount(); - fFilterString = rhs.filterString(); - isFilterFeeder = rhs.getFeederFlag(); - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - extentSize = rhs.extentSize; + fInputJobStepAssociation = rhs.inputAssociation(); + fOutputJobStepAssociation = rhs.outputAssociation(); + fDec = 0; + fSessionId = rhs.sessionId(); + fFilterCount = rhs.filterCount(); + fFilterString = rhs.filterString(); + isFilterFeeder = rhs.getFeederFlag(); + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + extentSize = rhs.extentSize; - scannedExtents = rhs.extents; - extentsMap[fOid] = tr1::unordered_map(); - tr1::unordered_map& ref = extentsMap[fOid]; + scannedExtents = rhs.extents; + extentsMap[fOid] = tr1::unordered_map(); + tr1::unordered_map& ref = extentsMap[fOid]; - for (uint32_t z = 0; z < rhs.extents.size(); z++) - ref[rhs.extents[z].range.start] = rhs.extents[z]; + for (uint32_t z = 0; z < rhs.extents.size(); z++) + ref[rhs.extents[z].range.start] = rhs.extents[z]; - lbidList = rhs.lbidList; - rpbShift = rhs.rpbShift; - divShift = rhs.divShift; - modMask = rhs.modMask; - numExtents = rhs.numExtents; - ridsRequested = 0; - ridsReturned = 0; - recvExited = 0; - totalMsgs = 0; - msgsSent = 0; - msgsRecvd = 0; - fMsgBytesIn = 0; - fMsgBytesOut = 0; - fBlockTouched = 0; - fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; - recvWaiting = 0; - fStepCount = 1; - fCPEvaluated = false; - fEstimatedRows = 0; - fColType = rhs.colType(); - alias(rhs.alias()); - view(rhs.view()); - name(rhs.name()); - fColWidth = fColType.colWidth; - fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); - initializeConfigParms(); - fBPP->setSessionID(fSessionId); - fBPP->setStepID(fStepId); - fBPP->setQueryContext(fVerId); - fBPP->setTxnID(fTxnId); - fTraceFlags = rhs.fTraceFlags; - fBPP->setTraceFlags(fTraceFlags); - fBPP->setOutputType(ROW_GROUP); - finishedSending = sendWaiting = false; - fNumBlksSkipped = 0; - fPhysicalIO = 0; - fCacheIO = 0; - BPPIsAllocated = false; - uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - fBPP->setUniqueID(uniqueID); - fBPP->setUuid(fStepUuid); - fCardinality = rhs.cardinality(); - doJoin = false; - hasPMJoin = false; - hasUMJoin = false; - fRunExecuted = false; - fSwallowRows = false; - smallOuterJoiner = -1; + lbidList = rhs.lbidList; + rpbShift = rhs.rpbShift; + divShift = rhs.divShift; + modMask = rhs.modMask; + numExtents = rhs.numExtents; + ridsRequested = 0; + ridsReturned = 0; + recvExited = 0; + totalMsgs = 0; + msgsSent = 0; + msgsRecvd = 0; + fMsgBytesIn = 0; + fMsgBytesOut = 0; + fBlockTouched = 0; + fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; + recvWaiting = 0; + fStepCount = 1; + fCPEvaluated = false; + fEstimatedRows = 0; + fColType = rhs.colType(); + alias(rhs.alias()); + view(rhs.view()); + name(rhs.name()); + fColWidth = fColType.colWidth; + fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); + initializeConfigParms(); + fBPP->setSessionID(fSessionId); + fBPP->setStepID(fStepId); + fBPP->setQueryContext(fVerId); + fBPP->setTxnID(fTxnId); + fTraceFlags = rhs.fTraceFlags; + fBPP->setTraceFlags(fTraceFlags); + fBPP->setOutputType(ROW_GROUP); + finishedSending = sendWaiting = false; + fNumBlksSkipped = 0; + fPhysicalIO = 0; + fCacheIO = 0; + BPPIsAllocated = false; + uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + fBPP->setUniqueID(uniqueID); + fBPP->setUuid(fStepUuid); + fCardinality = rhs.cardinality(); + doJoin = false; + hasPMJoin = false; + hasUMJoin = false; + fRunExecuted = false; + fSwallowRows = false; + smallOuterJoiner = -1; - // @1098 initialize scanFlags to be true - scanFlags.assign(numExtents, true); - runtimeCPFlags.assign(numExtents, true); - bop = BOP_AND; + // @1098 initialize scanFlags to be true + scanFlags.assign(numExtents, true); + runtimeCPFlags.assign(numExtents, true); + bop = BOP_AND; - runRan = joinRan = false; - fDelivery = false; - fExtendedInfo = "TBPS: "; - fQtc.stepParms().stepType = StepTeleStats::T_BPS; + runRan = joinRan = false; + fDelivery = false; + fExtendedInfo = "TBPS: "; + fQtc.stepParms().stepType = StepTeleStats::T_BPS; - hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = - hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; + hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = + hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; } -TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : - BatchPrimitive(jobInfo), fRm(jobInfo.rm) +TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : BatchPrimitive(jobInfo), fRm(jobInfo.rm) { - fInputJobStepAssociation = rhs.inputAssociation(); - fOutputJobStepAssociation = rhs.outputAssociation(); - fDec = 0; - fFilterCount = rhs.filterCount(); - fFilterString = rhs.filterString(); - isFilterFeeder = rhs.getFeederFlag(); - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - extentSize = rhs.extentSize; - lbidRanges = rhs.lbidRanges; + fInputJobStepAssociation = rhs.inputAssociation(); + fOutputJobStepAssociation = rhs.outputAssociation(); + fDec = 0; + fFilterCount = rhs.filterCount(); + fFilterString = rhs.filterString(); + isFilterFeeder = rhs.getFeederFlag(); + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + extentSize = rhs.extentSize; + lbidRanges = rhs.lbidRanges; - /* These lines are obsoleted by initExtentMarkers. Need to remove & retest. */ - scannedExtents = rhs.extents; - extentsMap[fOid] = tr1::unordered_map(); - tr1::unordered_map& ref = extentsMap[fOid]; + /* These lines are obsoleted by initExtentMarkers. Need to remove & retest. */ + scannedExtents = rhs.extents; + extentsMap[fOid] = tr1::unordered_map(); + tr1::unordered_map& ref = extentsMap[fOid]; - for (uint32_t z = 0; z < rhs.extents.size(); z++) - ref[rhs.extents[z].range.start] = rhs.extents[z]; + for (uint32_t z = 0; z < rhs.extents.size(); z++) + ref[rhs.extents[z].range.start] = rhs.extents[z]; - divShift = rhs.divShift; - totalMsgs = 0; - msgsSent = 0; - msgsRecvd = 0; - ridsReturned = 0; - ridsRequested = 0; - fNumBlksSkipped = 0; - fMsgBytesIn = 0; - fMsgBytesOut = 0; - fBlockTouched = 0; - fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; - recvWaiting = 0; - fSwallowRows = false; - fStepCount = 1; - fCPEvaluated = false; - fEstimatedRows = 0; - fColType = rhs.colType(); - alias(rhs.alias()); - view(rhs.view()); - name(rhs.name()); + divShift = rhs.divShift; + totalMsgs = 0; + msgsSent = 0; + msgsRecvd = 0; + ridsReturned = 0; + ridsRequested = 0; + fNumBlksSkipped = 0; + fMsgBytesIn = 0; + fMsgBytesOut = 0; + fBlockTouched = 0; + fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; + recvWaiting = 0; + fSwallowRows = false; + fStepCount = 1; + fCPEvaluated = false; + fEstimatedRows = 0; + fColType = rhs.colType(); + alias(rhs.alias()); + view(rhs.view()); + name(rhs.name()); - fColWidth = fColType.colWidth; - lbidList = rhs.lbidList; + fColWidth = fColType.colWidth; + lbidList = rhs.lbidList; - finishedSending = sendWaiting = false; - firstRead = true; - fSwallowRows = false; - recvExited = 0; - fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); - initializeConfigParms(); - fBPP->setSessionID(fSessionId); - fBPP->setQueryContext(fVerId); - fBPP->setTxnID(fTxnId); - fTraceFlags = rhs.fTraceFlags; - fBPP->setTraceFlags(fTraceFlags); - fBPP->setStepID(fStepId); - fBPP->setOutputType(ROW_GROUP); - fPhysicalIO = 0; - fCacheIO = 0; - BPPIsAllocated = false; - uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - fBPP->setUniqueID(uniqueID); - fBPP->setUuid(fStepUuid); - fCardinality = rhs.cardinality(); - doJoin = false; - hasPMJoin = false; - hasUMJoin = false; - fRunExecuted = false; - smallOuterJoiner = -1; - bop = BOP_AND; + finishedSending = sendWaiting = false; + firstRead = true; + fSwallowRows = false; + recvExited = 0; + fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); + initializeConfigParms(); + fBPP->setSessionID(fSessionId); + fBPP->setQueryContext(fVerId); + fBPP->setTxnID(fTxnId); + fTraceFlags = rhs.fTraceFlags; + fBPP->setTraceFlags(fTraceFlags); + fBPP->setStepID(fStepId); + fBPP->setOutputType(ROW_GROUP); + fPhysicalIO = 0; + fCacheIO = 0; + BPPIsAllocated = false; + uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + fBPP->setUniqueID(uniqueID); + fBPP->setUuid(fStepUuid); + fCardinality = rhs.cardinality(); + doJoin = false; + hasPMJoin = false; + hasUMJoin = false; + fRunExecuted = false; + smallOuterJoiner = -1; + bop = BOP_AND; - runRan = joinRan = false; - fDelivery = false; - fExtendedInfo = "TBPS: "; + runRan = joinRan = false; + fDelivery = false; + fExtendedInfo = "TBPS: "; - initExtentMarkers(); - fQtc.stepParms().stepType = StepTeleStats::T_BPS; + initExtentMarkers(); + fQtc.stepParms().stepType = StepTeleStats::T_BPS; - hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = - hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; + hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = + hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; } -TupleBPS::TupleBPS(const PassThruStep& rhs, const JobInfo& jobInfo) : - BatchPrimitive(jobInfo), fRm(jobInfo.rm) +TupleBPS::TupleBPS(const PassThruStep& rhs, const JobInfo& jobInfo) : BatchPrimitive(jobInfo), fRm(jobInfo.rm) { - fInputJobStepAssociation = rhs.inputAssociation(); - fOutputJobStepAssociation = rhs.outputAssociation(); - fDec = 0; - fFilterCount = 0; - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - ridsReturned = 0; - ridsRequested = 0; - fMsgBytesIn = 0; - fMsgBytesOut = 0; - fBlockTouched = 0; - fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; - recvExited = 0; - totalMsgs = 0; - msgsSent = 0; - msgsRecvd = 0; - recvWaiting = 0; - fStepCount = 1; - fCPEvaluated = false; - fEstimatedRows = 0; - fColType = rhs.colType(); - alias(rhs.alias()); - view(rhs.view()); - name(rhs.name()); - fColWidth = fColType.colWidth; - fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); - initializeConfigParms(); - fBPP->setSessionID(fSessionId); - fBPP->setStepID(fStepId); - fBPP->setQueryContext(fVerId); - fBPP->setTxnID(fTxnId); - fTraceFlags = rhs.fTraceFlags; - fBPP->setTraceFlags(fTraceFlags); - fBPP->setOutputType(ROW_GROUP); - finishedSending = sendWaiting = false; - fSwallowRows = false; - fNumBlksSkipped = 0; - fPhysicalIO = 0; - fCacheIO = 0; - BPPIsAllocated = false; - uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - fBPP->setUniqueID(uniqueID); - fBPP->setUuid(fStepUuid); - doJoin = false; - hasPMJoin = false; - hasUMJoin = false; - fRunExecuted = false; - isFilterFeeder = false; - smallOuterJoiner = -1; + fInputJobStepAssociation = rhs.inputAssociation(); + fOutputJobStepAssociation = rhs.outputAssociation(); + fDec = 0; + fFilterCount = 0; + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + ridsReturned = 0; + ridsRequested = 0; + fMsgBytesIn = 0; + fMsgBytesOut = 0; + fBlockTouched = 0; + fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; + recvExited = 0; + totalMsgs = 0; + msgsSent = 0; + msgsRecvd = 0; + recvWaiting = 0; + fStepCount = 1; + fCPEvaluated = false; + fEstimatedRows = 0; + fColType = rhs.colType(); + alias(rhs.alias()); + view(rhs.view()); + name(rhs.name()); + fColWidth = fColType.colWidth; + fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); + initializeConfigParms(); + fBPP->setSessionID(fSessionId); + fBPP->setStepID(fStepId); + fBPP->setQueryContext(fVerId); + fBPP->setTxnID(fTxnId); + fTraceFlags = rhs.fTraceFlags; + fBPP->setTraceFlags(fTraceFlags); + fBPP->setOutputType(ROW_GROUP); + finishedSending = sendWaiting = false; + fSwallowRows = false; + fNumBlksSkipped = 0; + fPhysicalIO = 0; + fCacheIO = 0; + BPPIsAllocated = false; + uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + fBPP->setUniqueID(uniqueID); + fBPP->setUuid(fStepUuid); + doJoin = false; + hasPMJoin = false; + hasUMJoin = false; + fRunExecuted = false; + isFilterFeeder = false; + smallOuterJoiner = -1; - // @1098 initialize scanFlags to be true - scanFlags.assign(numExtents, true); - runtimeCPFlags.assign(numExtents, true); - bop = BOP_AND; + // @1098 initialize scanFlags to be true + scanFlags.assign(numExtents, true); + runtimeCPFlags.assign(numExtents, true); + bop = BOP_AND; - runRan = joinRan = false; - fDelivery = false; - fExtendedInfo = "TBPS: "; - fQtc.stepParms().stepType = StepTeleStats::T_BPS; - - hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = - hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; + runRan = joinRan = false; + fDelivery = false; + fExtendedInfo = "TBPS: "; + fQtc.stepParms().stepType = StepTeleStats::T_BPS; + hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = + hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; } -TupleBPS::TupleBPS(const pDictionaryStep& rhs, const JobInfo& jobInfo) : - BatchPrimitive(jobInfo), fRm(jobInfo.rm) +TupleBPS::TupleBPS(const pDictionaryStep& rhs, const JobInfo& jobInfo) + : BatchPrimitive(jobInfo), fRm(jobInfo.rm) { - fInputJobStepAssociation = rhs.inputAssociation(); - fOutputJobStepAssociation = rhs.outputAssociation(); - fDec = 0; - fOid = rhs.oid(); - fTableOid = rhs.tableOid(); - totalMsgs = 0; - msgsSent = 0; - msgsRecvd = 0; - ridsReturned = 0; - ridsRequested = 0; - fNumBlksSkipped = 0; - fBlockTouched = 0; - fMsgBytesIn = 0; - fMsgBytesOut = 0; - fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; - recvWaiting = 0; - fSwallowRows = false; - fStepCount = 1; - fCPEvaluated = false; - fEstimatedRows = 0; - alias(rhs.alias()); - view(rhs.view()); - name(rhs.name()); - finishedSending = sendWaiting = false; - recvExited = 0; - fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); - initializeConfigParms(); - fBPP->setSessionID(fSessionId); - fBPP->setStepID(fStepId); - fBPP->setQueryContext(fVerId); - fBPP->setTxnID(fTxnId); - fTraceFlags = rhs.fTraceFlags; - fBPP->setTraceFlags(fTraceFlags); - fBPP->setOutputType(ROW_GROUP); - fPhysicalIO = 0; - fCacheIO = 0; - BPPIsAllocated = false; - uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - fBPP->setUniqueID(uniqueID); - fBPP->setUuid(fStepUuid); - fCardinality = rhs.cardinality(); - doJoin = false; - hasPMJoin = false; - hasUMJoin = false; - fRunExecuted = false; - isFilterFeeder = false; - smallOuterJoiner = -1; - // @1098 initialize scanFlags to be true - scanFlags.assign(numExtents, true); - runtimeCPFlags.assign(numExtents, true); - bop = BOP_AND; + fInputJobStepAssociation = rhs.inputAssociation(); + fOutputJobStepAssociation = rhs.outputAssociation(); + fDec = 0; + fOid = rhs.oid(); + fTableOid = rhs.tableOid(); + totalMsgs = 0; + msgsSent = 0; + msgsRecvd = 0; + ridsReturned = 0; + ridsRequested = 0; + fNumBlksSkipped = 0; + fBlockTouched = 0; + fMsgBytesIn = 0; + fMsgBytesOut = 0; + fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; + recvWaiting = 0; + fSwallowRows = false; + fStepCount = 1; + fCPEvaluated = false; + fEstimatedRows = 0; + alias(rhs.alias()); + view(rhs.view()); + name(rhs.name()); + finishedSending = sendWaiting = false; + recvExited = 0; + fBPP.reset(new BatchPrimitiveProcessorJL(fRm)); + initializeConfigParms(); + fBPP->setSessionID(fSessionId); + fBPP->setStepID(fStepId); + fBPP->setQueryContext(fVerId); + fBPP->setTxnID(fTxnId); + fTraceFlags = rhs.fTraceFlags; + fBPP->setTraceFlags(fTraceFlags); + fBPP->setOutputType(ROW_GROUP); + fPhysicalIO = 0; + fCacheIO = 0; + BPPIsAllocated = false; + uniqueID = UniqueNumberGenerator::instance()->getUnique32(); + fBPP->setUniqueID(uniqueID); + fBPP->setUuid(fStepUuid); + fCardinality = rhs.cardinality(); + doJoin = false; + hasPMJoin = false; + hasUMJoin = false; + fRunExecuted = false; + isFilterFeeder = false; + smallOuterJoiner = -1; + // @1098 initialize scanFlags to be true + scanFlags.assign(numExtents, true); + runtimeCPFlags.assign(numExtents, true); + bop = BOP_AND; - runRan = joinRan = false; - fDelivery = false; - fExtendedInfo = "TBPS: "; - fQtc.stepParms().stepType = StepTeleStats::T_BPS; - - hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = - hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; + runRan = joinRan = false; + fDelivery = false; + fExtendedInfo = "TBPS: "; + fQtc.stepParms().stepType = StepTeleStats::T_BPS; + hasPCFilter = hasPMFilter = hasRIDFilter = hasSegmentFilter = hasDBRootFilter = hasSegmentDirFilter = + hasPartitionFilter = hasMaxFilter = hasMinFilter = hasLBIDFilter = hasExtentIDFilter = false; } TupleBPS::~TupleBPS() { - if (fDec) + if (fDec) + { + fDec->removeDECEventListener(this); + + if (BPPIsAllocated) { - fDec->removeDECEventListener(this); + ByteStream bs; + fBPP->destroyBPP(bs); - if (BPPIsAllocated) - { - ByteStream bs; - fBPP->destroyBPP(bs); - - try - { - fDec->write(uniqueID, bs); - } - catch (const std::exception& e) - { - // log the exception - cerr << "~TupleBPS caught: " << e.what() << endl; - catchHandler(e.what(), ERR_TUPLE_BPS, fErrorInfo, fSessionId); - } - catch (...) - { - cerr << "~TupleBPS caught unknown exception" << endl; - catchHandler("~TupleBPS caught unknown exception", - ERR_TUPLE_BPS, fErrorInfo, fSessionId); - } - } - - fDec->removeQueue(uniqueID); + try + { + fDec->write(uniqueID, bs); + } + catch (const std::exception& e) + { + // log the exception + cerr << "~TupleBPS caught: " << e.what() << endl; + catchHandler(e.what(), ERR_TUPLE_BPS, fErrorInfo, fSessionId); + } + catch (...) + { + cerr << "~TupleBPS caught unknown exception" << endl; + catchHandler("~TupleBPS caught unknown exception", ERR_TUPLE_BPS, fErrorInfo, fSessionId); + } } + fDec->removeQueue(uniqueID); + } } void TupleBPS::setBPP(JobStep* jobStep) { - fCardinality = jobStep->cardinality(); + fCardinality = jobStep->cardinality(); - pColStep* pcsp = dynamic_cast(jobStep); + pColStep* pcsp = dynamic_cast(jobStep); - int colWidth = 0; + int colWidth = 0; - if (pcsp != 0) + if (pcsp != 0) + { + PseudoColStep* pseudo = dynamic_cast(jobStep); + + if (pseudo) { - PseudoColStep* pseudo = dynamic_cast(jobStep); + fBPP->addFilterStep(*pseudo); - if (pseudo) + if (pseudo->filterCount() > 0) + { + hasPCFilter = true; + + switch (pseudo->pseudoColumnId()) { - fBPP->addFilterStep(*pseudo); + case PSEUDO_EXTENTRELATIVERID: hasRIDFilter = true; break; - if (pseudo->filterCount() > 0) - { - hasPCFilter = true; + case PSEUDO_DBROOT: hasDBRootFilter = true; break; - switch (pseudo->pseudoColumnId()) - { - case PSEUDO_EXTENTRELATIVERID: - hasRIDFilter = true; - break; + case PSEUDO_PM: hasPMFilter = true; break; - case PSEUDO_DBROOT: - hasDBRootFilter = true; - break; + case PSEUDO_SEGMENT: hasSegmentFilter = true; break; - case PSEUDO_PM: - hasPMFilter = true; - break; + case PSEUDO_SEGMENTDIR: hasSegmentDirFilter = true; break; - case PSEUDO_SEGMENT: - hasSegmentFilter = true; - break; + case PSEUDO_EXTENTMIN: hasMinFilter = true; break; - case PSEUDO_SEGMENTDIR: - hasSegmentDirFilter = true; - break; + case PSEUDO_EXTENTMAX: hasMaxFilter = true; break; - case PSEUDO_EXTENTMIN: - hasMinFilter = true; - break; + case PSEUDO_BLOCKID: hasLBIDFilter = true; break; - case PSEUDO_EXTENTMAX: - hasMaxFilter = true; - break; + case PSEUDO_EXTENTID: hasExtentIDFilter = true; break; - case PSEUDO_BLOCKID: - hasLBIDFilter = true; - break; - - case PSEUDO_EXTENTID: - hasExtentIDFilter = true; - break; - - case PSEUDO_PARTITION: - hasPartitionFilter = true; - break; - } - } + case PSEUDO_PARTITION: hasPartitionFilter = true; break; } - else - fBPP->addFilterStep(*pcsp); + } + } + else + fBPP->addFilterStep(*pcsp); - extentsMap[pcsp->fOid] = tr1::unordered_map(); - tr1::unordered_map& ref = extentsMap[pcsp->fOid]; + extentsMap[pcsp->fOid] = tr1::unordered_map(); + tr1::unordered_map& ref = extentsMap[pcsp->fOid]; - for (uint32_t z = 0; z < pcsp->extents.size(); z++) - ref[pcsp->extents[z].range.start] = pcsp->extents[z]; + for (uint32_t z = 0; z < pcsp->extents.size(); z++) + ref[pcsp->extents[z].range.start] = pcsp->extents[z]; - colWidth = (pcsp->colType()).colWidth; - isFilterFeeder = pcsp->getFeederFlag(); + colWidth = (pcsp->colType()).colWidth; + isFilterFeeder = pcsp->getFeederFlag(); - // it17 does not allow combined AND/OR, this pcolstep is for hashjoin optimization. - if (bop == BOP_OR && isFilterFeeder == false) - fBPP->setForHJ(true); + // it17 does not allow combined AND/OR, this pcolstep is for hashjoin optimization. + if (bop == BOP_OR && isFilterFeeder == false) + fBPP->setForHJ(true); + } + else + { + pColScanStep* pcss = dynamic_cast(jobStep); + + if (pcss != 0) + { + fBPP->addFilterStep(*pcss, lastScannedLBID); + + extentsMap[pcss->fOid] = tr1::unordered_map(); + tr1::unordered_map& ref = extentsMap[pcss->fOid]; + + for (uint32_t z = 0; z < pcss->extents.size(); z++) + ref[pcss->extents[z].range.start] = pcss->extents[z]; + + colWidth = (pcss->colType()).colWidth; + isFilterFeeder = pcss->getFeederFlag(); } else { - pColScanStep* pcss = dynamic_cast(jobStep); + pDictionaryStep* pdsp = dynamic_cast(jobStep); - if (pcss != 0) + if (pdsp != 0) + { + fBPP->addFilterStep(*pdsp); + colWidth = (pdsp->colType()).colWidth; + } + else + { + FilterStep* pfsp = dynamic_cast(jobStep); + + if (pfsp) { - fBPP->addFilterStep(*pcss, lastScannedLBID); - - extentsMap[pcss->fOid] = tr1::unordered_map(); - tr1::unordered_map& ref = extentsMap[pcss->fOid]; - - for (uint32_t z = 0; z < pcss->extents.size(); z++) - ref[pcss->extents[z].range.start] = pcss->extents[z]; - - colWidth = (pcss->colType()).colWidth; - isFilterFeeder = pcss->getFeederFlag(); - } - else - { - pDictionaryStep* pdsp = dynamic_cast(jobStep); - - if (pdsp != 0) - { - fBPP->addFilterStep(*pdsp); - colWidth = (pdsp->colType()).colWidth; - } - else - { - FilterStep* pfsp = dynamic_cast(jobStep); - - if (pfsp) - { - fBPP->addFilterStep(*pfsp); - } - } - + fBPP->addFilterStep(*pfsp); } + } } + } - if (colWidth > fColWidth) - { - fColWidth = colWidth; - } + if (colWidth > fColWidth) + { + fColWidth = colWidth; + } } void TupleBPS::setProjectBPP(JobStep* jobStep1, JobStep* jobStep2) { - int colWidth = 0; + int colWidth = 0; - if (jobStep2 != NULL) + if (jobStep2 != NULL) + { + pDictionaryStep* pdsp = 0; + pColStep* pcsp = dynamic_cast(jobStep1); + + if (pcsp != 0) { - pDictionaryStep* pdsp = 0; - pColStep* pcsp = dynamic_cast(jobStep1); + pdsp = dynamic_cast(jobStep2); + fBPP->addProjectStep(*pcsp, *pdsp); - if (pcsp != 0) - { - pdsp = dynamic_cast(jobStep2); - fBPP->addProjectStep(*pcsp, *pdsp); + //@Bug 961 + if (!pcsp->isExeMgr()) + fBPP->setNeedRidsAtDelivery(true); - //@Bug 961 - if (!pcsp->isExeMgr()) - fBPP->setNeedRidsAtDelivery(true); - - colWidth = (pcsp->colType()).colWidth; - projectOids.push_back(jobStep1->oid()); - } - else - { - PassThruStep* psth = dynamic_cast(jobStep1); - - if (psth != 0) - { - pdsp = dynamic_cast(jobStep2); - fBPP->addProjectStep(*psth, *pdsp); - - //@Bug 961 - if (!psth->isExeMgr()) - fBPP->setNeedRidsAtDelivery(true); - - projectOids.push_back(jobStep1->oid()); - colWidth = (psth->colType()).colWidth; - } - } + colWidth = (pcsp->colType()).colWidth; + projectOids.push_back(jobStep1->oid()); } else { - pColStep* pcsp = dynamic_cast(jobStep1); + PassThruStep* psth = dynamic_cast(jobStep1); - if (pcsp != 0) + if (psth != 0) + { + pdsp = dynamic_cast(jobStep2); + fBPP->addProjectStep(*psth, *pdsp); + + //@Bug 961 + if (!psth->isExeMgr()) + fBPP->setNeedRidsAtDelivery(true); + + projectOids.push_back(jobStep1->oid()); + colWidth = (psth->colType()).colWidth; + } + } + } + else + { + pColStep* pcsp = dynamic_cast(jobStep1); + + if (pcsp != 0) + { + PseudoColStep* pseudo = dynamic_cast(jobStep1); + + if (pseudo) + { + fBPP->addProjectStep(*pseudo); + } + else + fBPP->addProjectStep(*pcsp); + + extentsMap[pcsp->fOid] = tr1::unordered_map(); + tr1::unordered_map& ref = extentsMap[pcsp->fOid]; + + for (uint32_t z = 0; z < pcsp->extents.size(); z++) + ref[pcsp->extents[z].range.start] = pcsp->extents[z]; + + //@Bug 961 + if (!pcsp->isExeMgr()) + fBPP->setNeedRidsAtDelivery(true); + + colWidth = (pcsp->colType()).colWidth; + projectOids.push_back(jobStep1->oid()); + } + else + { + PassThruStep* passthru = dynamic_cast(jobStep1); + + if (passthru != 0) + { + idbassert(!fBPP->getFilterSteps().empty()); + + if (static_cast(fBPP->getFilterSteps().back()->getOID()) != + passthru->oid()) { - PseudoColStep* pseudo = dynamic_cast(jobStep1); + SJSTEP pts; - if (pseudo) - { - fBPP->addProjectStep(*pseudo); - } - else - fBPP->addProjectStep(*pcsp); + if (passthru->pseudoType() == 0) + { + pts.reset(new pColStep(*passthru)); + pcsp = dynamic_cast(pts.get()); + } + else + { + pts.reset(new PseudoColStep(*passthru)); + pcsp = dynamic_cast(pts.get()); + } - extentsMap[pcsp->fOid] = tr1::unordered_map(); - tr1::unordered_map& ref = extentsMap[pcsp->fOid]; + fBPP->addProjectStep(*pcsp); - for (uint32_t z = 0; z < pcsp->extents.size(); z++) - ref[pcsp->extents[z].range.start] = pcsp->extents[z]; + if (!passthru->isExeMgr()) + fBPP->setNeedRidsAtDelivery(true); - //@Bug 961 - if (!pcsp->isExeMgr()) - fBPP->setNeedRidsAtDelivery(true); - - colWidth = (pcsp->colType()).colWidth; - projectOids.push_back(jobStep1->oid()); + colWidth = passthru->colType().colWidth; + projectOids.push_back(pts->oid()); } else { - PassThruStep* passthru = dynamic_cast(jobStep1); + fBPP->addProjectStep(*passthru); - if (passthru != 0) - { - idbassert(!fBPP->getFilterSteps().empty()); + //@Bug 961 + if (!passthru->isExeMgr()) + fBPP->setNeedRidsAtDelivery(true); - if (static_cast(fBPP->getFilterSteps().back()->getOID()) != passthru->oid()) - { - SJSTEP pts; - - if (passthru->pseudoType() == 0) - { - pts.reset(new pColStep(*passthru)); - pcsp = dynamic_cast(pts.get()); - } - else - { - pts.reset(new PseudoColStep(*passthru)); - pcsp = dynamic_cast(pts.get()); - } - - fBPP->addProjectStep(*pcsp); - - if (!passthru->isExeMgr()) - fBPP->setNeedRidsAtDelivery(true); - - colWidth = passthru->colType().colWidth; - projectOids.push_back(pts->oid()); - } - else - { - fBPP->addProjectStep(*passthru); - - //@Bug 961 - if (!passthru->isExeMgr()) - fBPP->setNeedRidsAtDelivery(true); - - colWidth = (passthru->colType()).colWidth; - projectOids.push_back(jobStep1->oid()); - } - } + colWidth = (passthru->colType()).colWidth; + projectOids.push_back(jobStep1->oid()); } + } } + } - if (colWidth > fColWidth) - { - fColWidth = colWidth; - } + if (colWidth > fColWidth) + { + fColWidth = colWidth; + } } void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) { - const vector& colCmdVec = fBPP->getFilterSteps(); - vector cpColVec; - vector lbidListVec; - ColumnCommandJL* colCmd = 0; + const vector& colCmdVec = fBPP->getFilterSteps(); + vector cpColVec; + vector lbidListVec; + ColumnCommandJL* colCmd = 0; - // @bug 2123. We call this earlier in the process for the hash join estimation process now. Return if we've already done the work. - if (fCPEvaluated) + // @bug 2123. We call this earlier in the process for the hash join estimation process now. Return if + // we've already done the work. + if (fCPEvaluated) + { + return; + } + + fCPEvaluated = true; + + if (colCmdVec.size() == 0) + return; + + for (uint32_t i = 0; i < colCmdVec.size(); i++) + { + colCmd = dynamic_cast(colCmdVec[i].get()); + + if (!colCmd || dynamic_cast(colCmdVec[i].get())) + continue; + + SP_LBIDList tmplbidList(new LBIDList(0)); + + if (tmplbidList->CasualPartitionDataType(colCmd->getColType().colDataType, colCmd->getColType().colWidth)) { - return; + lbidListVec.push_back(tmplbidList); + cpColVec.push_back(colCmd); } - fCPEvaluated = true; - - if (colCmdVec.size() == 0) - return; - - for (uint32_t i = 0; i < colCmdVec.size(); i++) + // @Bug 3503. Use the total table size as the estimate for non CP columns. + else if (fEstimatedRows == 0 && estimateRowCounts) { - colCmd = dynamic_cast(colCmdVec[i].get()); - - if (!colCmd || dynamic_cast(colCmdVec[i].get())) - continue; - - SP_LBIDList tmplbidList(new LBIDList(0)); - - if (tmplbidList->CasualPartitionDataType(colCmd->getColType().colDataType, colCmd->getColType().colWidth)) - { - lbidListVec.push_back(tmplbidList); - cpColVec.push_back(colCmd); - } - - // @Bug 3503. Use the total table size as the estimate for non CP columns. - else if (fEstimatedRows == 0 && estimateRowCounts) - { - RowEstimator rowEstimator; - fEstimatedRows = rowEstimator.estimateRowsForNonCPColumn(*colCmd); - } + RowEstimator rowEstimator; + fEstimatedRows = rowEstimator.estimateRowsForNonCPColumn(*colCmd); } + } + if (cpColVec.size() == 0) + return; - if (cpColVec.size() == 0) - return; + const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); - const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); + for (uint32_t idx = 0; idx < numExtents; idx++) + { + scanFlags[idx] = true; - for (uint32_t idx = 0; idx < numExtents; idx++) + for (uint32_t i = 0; i < cpColVec.size(); i++) { - scanFlags[idx] = true; + colCmd = cpColVec[i]; + const EMEntry& extent = colCmd->getExtents()[idx]; - for (uint32_t i = 0; i < cpColVec.size(); i++) - { - colCmd = cpColVec[i]; - const EMEntry& extent = colCmd->getExtents()[idx]; + /* If any column filter eliminates an extent, it doesn't get scanned */ + scanFlags[idx] = + scanFlags[idx] && (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || + lbidListVec[i]->CasualPartitionPredicate( + extent.partition.cprange, &(colCmd->getFilterString()), + colCmd->getFilterCount(), colCmd->getColType(), colCmd->getBOP())); - /* If any column filter eliminates an extent, it doesn't get scanned */ - scanFlags[idx] = scanFlags[idx] && - (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || - lbidListVec[i]->CasualPartitionPredicate( - extent.partition.cprange, - &(colCmd->getFilterString()), - colCmd->getFilterCount(), - colCmd->getColType(), - colCmd->getBOP()) - ); - - if (!scanFlags[idx]) - { - break; - } - } + if (!scanFlags[idx]) + { + break; + } } + } - // @bug 2123. Use the casual partitioning information to estimate the number of rows that will be returned for use in estimating - // the large side table for hashjoins. - if (estimateRowCounts) - { - RowEstimator rowEstimator; - fEstimatedRows = rowEstimator.estimateRows(cpColVec, scanFlags, dbrm, fOid); - } + // @bug 2123. Use the casual partitioning information to estimate the number of rows that will be returned + // for use in estimating the large side table for hashjoins. + if (estimateRowCounts) + { + RowEstimator rowEstimator; + fEstimatedRows = rowEstimator.estimateRows(cpColVec, scanFlags, dbrm, fOid); + } } void TupleBPS::startPrimitiveThread() { - pThread = jobstepThreadPool.invoke(TupleBPSPrimitive(this)); + pThread = jobstepThreadPool.invoke(TupleBPSPrimitive(this)); } void TupleBPS::startAggregationThread() { -// This block of code starts all threads up front -// fMaxNumThreads = 1; -// fNumThreads = fMaxNumThreads; -// for (uint32_t i = 0; i < fMaxNumThreads; i++) -// fProducerThreads.push_back(jobstepThreadPool.invoke(TupleBPSAggregators(this, i))); + // This block of code starts all threads up front + // fMaxNumThreads = 1; + // fNumThreads = fMaxNumThreads; + // for (uint32_t i = 0; i < fMaxNumThreads; i++) + // fProducerThreads.push_back(jobstepThreadPool.invoke(TupleBPSAggregators(this, i))); - fNumThreads++; - fProducerThreads.push_back(jobstepThreadPool.invoke(TupleBPSAggregators(this))); + fNumThreads++; + fProducerThreads.push_back(jobstepThreadPool.invoke(TupleBPSAggregators(this))); } void TupleBPS::startProcessingThread(TupleBPS* tbps, vector>& bsv, const uint32_t start, const uint32_t end, vector<_CPInfo>& cpv, RowGroupDL* dlp, const uint32_t threadID) { - fProcessorThreads.push_back( - jobstepThreadPool.invoke(ByteStreamProcessor(tbps, bsv, start, end, cpv, dlp, threadID))); + fProcessorThreads.push_back( + jobstepThreadPool.invoke(ByteStreamProcessor(tbps, bsv, start, end, cpv, dlp, threadID))); } void TupleBPS::serializeJoiner() { - ByteStream bs; - bool more = true; + ByteStream bs; + bool more = true; - /* false from nextJoinerMsg means it's the last msg, - it's not exactly the exit condition*/ - while (more) + /* false from nextJoinerMsg means it's the last msg, + it's not exactly the exit condition*/ + while (more) + { { - { - // code block to release the lock immediatly - boost::mutex::scoped_lock lk(serializeJoinerMutex); - more = fBPP->nextTupleJoinerMsg(bs); - } -#ifdef JLF_DEBUG - cout << "serializing joiner into " << bs.length() << " bytes" << endl; -#endif - fDec->write(uniqueID, bs); - bs.restart(); + // code block to release the lock immediatly + boost::mutex::scoped_lock lk(serializeJoinerMutex); + more = fBPP->nextTupleJoinerMsg(bs); } - +#ifdef JLF_DEBUG + cout << "serializing joiner into " << bs.length() << " bytes" << endl; +#endif + fDec->write(uniqueID, bs); + bs.restart(); + } } void TupleBPS::serializeJoiner(uint32_t conn) { - // We need this lock for TupleBPS::serializeJoiner() - boost::mutex::scoped_lock lk(serializeJoinerMutex); + // We need this lock for TupleBPS::serializeJoiner() + boost::mutex::scoped_lock lk(serializeJoinerMutex); - ByteStream bs; - bool more = true; + ByteStream bs; + bool more = true; - /* false from nextJoinerMsg means it's the last msg, - it's not exactly the exit condition*/ - while (more) - { - more = fBPP->nextTupleJoinerMsg(bs); - fDec->write(bs, conn); - bs.restart(); - } + /* false from nextJoinerMsg means it's the last msg, + it's not exactly the exit condition*/ + while (more) + { + more = fBPP->nextTupleJoinerMsg(bs); + fDec->write(bs, conn); + bs.restart(); + } } void TupleBPS::prepCasualPartitioning() { - uint32_t i; - int64_t min, max, seq; - int128_t bigMin, bigMax; - boost::mutex::scoped_lock lk(cpMutex); + uint32_t i; + int64_t min, max, seq; + int128_t bigMin, bigMax; + boost::mutex::scoped_lock lk(cpMutex); - for (i = 0; i < scannedExtents.size(); i++) + for (i = 0; i < scannedExtents.size(); i++) + { + if (fOid >= 3000) { - if (fOid >= 3000) - { - scanFlags[i] = scanFlags[i] && runtimeCPFlags[i]; + scanFlags[i] = scanFlags[i] && runtimeCPFlags[i]; - if (scanFlags[i] && lbidList->CasualPartitionDataType(fColType.colDataType, - fColType.colWidth)) - { - if (fColType.colWidth <= 8) - { - lbidList->GetMinMax(min, max, seq, (int64_t) scannedExtents[i].range.start, - &scannedExtents, fColType.colDataType); - } - else if (fColType.colWidth == 16) - { - lbidList->GetMinMax(bigMin, bigMax, seq, (int64_t) scannedExtents[i].range.start, - &scannedExtents, fColType.colDataType); - } - } + if (scanFlags[i] && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth)) + { + if (fColType.colWidth <= 8) + { + lbidList->GetMinMax(min, max, seq, (int64_t)scannedExtents[i].range.start, &scannedExtents, + fColType.colDataType); } - else - scanFlags[i] = true; + else if (fColType.colWidth == 16) + { + lbidList->GetMinMax(bigMin, bigMax, seq, (int64_t)scannedExtents[i].range.start, &scannedExtents, + fColType.colDataType); + } + } } + else + scanFlags[i] = true; + } } bool TupleBPS::goodExtentCount() { - uint32_t eCount = extentsMap.begin()->second.size(); - map > - ::iterator it; + uint32_t eCount = extentsMap.begin()->second.size(); + map>::iterator it; - for (it = extentsMap.begin(); it != extentsMap.end(); ++it) - if (it->second.size() != eCount) - return false; + for (it = extentsMap.begin(); it != extentsMap.end(); ++it) + if (it->second.size() != eCount) + return false; - return true; + return true; } void TupleBPS::initExtentMarkers() { - numDBRoots = fRm->getDBRootCount(); - lastExtent.resize(numDBRoots); - lastScannedLBID.resize(numDBRoots); + numDBRoots = fRm->getDBRootCount(); + lastExtent.resize(numDBRoots); + lastScannedLBID.resize(numDBRoots); - tr1::unordered_map& ref = extentsMap[fOid]; - tr1::unordered_map::iterator it; + tr1::unordered_map& ref = extentsMap[fOid]; + tr1::unordered_map::iterator it; - // Map part# and seg# to an extent count per segment file. - // Part# is 32 hi order bits of key, seg# is 32 lo order bits - std::tr1::unordered_map extentCountPerDbFile; + // Map part# and seg# to an extent count per segment file. + // Part# is 32 hi order bits of key, seg# is 32 lo order bits + std::tr1::unordered_map extentCountPerDbFile; - scannedExtents.clear(); + scannedExtents.clear(); - for (it = ref.begin(); it != ref.end(); ++it) + for (it = ref.begin(); it != ref.end(); ++it) + { + scannedExtents.push_back(it->second); + + //@bug 5322: 0 HWM may not mean full extent if 1 extent in file. + // Track how many extents are in each segment file + if (fExtentsPerSegFile > 1) { - scannedExtents.push_back(it->second); + EMEntry& e = it->second; + uint64_t key = ((uint64_t)e.partitionNum << 32) + e.segmentNum; + ++extentCountPerDbFile[key]; + } + } - //@bug 5322: 0 HWM may not mean full extent if 1 extent in file. - // Track how many extents are in each segment file - if (fExtentsPerSegFile > 1) - { - EMEntry& e = it->second; - uint64_t key = ((uint64_t)e.partitionNum << 32) + e.segmentNum; - ++extentCountPerDbFile[key]; - } + sort(scannedExtents.begin(), scannedExtents.end(), ExtentSorter()); + + numExtents = scannedExtents.size(); + // @1098 initialize scanFlags to be true + scanFlags.assign(numExtents, true); + runtimeCPFlags.assign(numExtents, true); + + for (uint32_t i = 0; i < numDBRoots; i++) + lastExtent[i] = -1; + + for (uint32_t i = 0; i < scannedExtents.size(); i++) + { + uint32_t dbRoot = scannedExtents[i].dbRoot - 1; + + /* Kludge to account for gaps in the dbroot mapping. */ + if (scannedExtents[i].dbRoot > numDBRoots) + { + lastExtent.resize(scannedExtents[i].dbRoot); + lastScannedLBID.resize(scannedExtents[i].dbRoot); + + for (uint32_t z = numDBRoots; z < scannedExtents[i].dbRoot; z++) + lastExtent[z] = -1; + + numDBRoots = scannedExtents[i].dbRoot; } - sort(scannedExtents.begin(), scannedExtents.end(), ExtentSorter()); + if ((scannedExtents[i].status == EXTENTAVAILABLE) && (lastExtent[dbRoot] < (int)i)) + lastExtent[dbRoot] = i; - numExtents = scannedExtents.size(); - // @1098 initialize scanFlags to be true - scanFlags.assign(numExtents, true); - runtimeCPFlags.assign(numExtents, true); - - for (uint32_t i = 0; i < numDBRoots; i++) - lastExtent[i] = -1; - - for (uint32_t i = 0; i < scannedExtents.size(); i++) + //@bug 5322: 0 HWM may not mean full extent if 1 extent in file. + // Use special status (EXTENTSTATUSMAX+1) to denote a single extent + // file with HWM 0; retrieve 1 block and not full extent. + if ((fExtentsPerSegFile > 1) && (scannedExtents[i].HWM == 0)) { - uint32_t dbRoot = scannedExtents[i].dbRoot - 1; + uint64_t key = ((uint64_t)scannedExtents[i].partitionNum << 32) + scannedExtents[i].segmentNum; - /* Kludge to account for gaps in the dbroot mapping. */ - if (scannedExtents[i].dbRoot > numDBRoots) - { - lastExtent.resize(scannedExtents[i].dbRoot); - lastScannedLBID.resize(scannedExtents[i].dbRoot); - - for (uint32_t z = numDBRoots; z < scannedExtents[i].dbRoot; z++) - lastExtent[z] = -1; - - numDBRoots = scannedExtents[i].dbRoot; - } - - if ((scannedExtents[i].status == EXTENTAVAILABLE) && (lastExtent[dbRoot] < (int) i)) - lastExtent[dbRoot] = i; - - //@bug 5322: 0 HWM may not mean full extent if 1 extent in file. - // Use special status (EXTENTSTATUSMAX+1) to denote a single extent - // file with HWM 0; retrieve 1 block and not full extent. - if ((fExtentsPerSegFile > 1) && (scannedExtents[i].HWM == 0)) - { - uint64_t key = ((uint64_t)scannedExtents[i].partitionNum << 32) + - scannedExtents[i].segmentNum; - - if (extentCountPerDbFile[key] == 1) - scannedExtents[i].status = EXTENTSTATUSMAX + 1; - } + if (extentCountPerDbFile[key] == 1) + scannedExtents[i].status = EXTENTSTATUSMAX + 1; } + } - // if only 1 block is written in the last extent, HWM is 0 and didn't get counted. - for (uint32_t i = 0; i < numDBRoots; i++) - { - if (lastExtent[i] != -1) - lastScannedLBID[i] = scannedExtents[lastExtent[i]].range.start + - (scannedExtents[lastExtent[i]].HWM - scannedExtents[lastExtent[i]].blockOffset); - else - lastScannedLBID[i] = -1; - } + // if only 1 block is written in the last extent, HWM is 0 and didn't get counted. + for (uint32_t i = 0; i < numDBRoots; i++) + { + if (lastExtent[i] != -1) + lastScannedLBID[i] = scannedExtents[lastExtent[i]].range.start + + (scannedExtents[lastExtent[i]].HWM - scannedExtents[lastExtent[i]].blockOffset); + else + lastScannedLBID[i] = -1; + } } void TupleBPS::reloadExtentLists() { - /* - * Iterate over each ColumnCommand instance - * - * 1) reload its extent array - * 2) update TupleBPS's extent array - * 3) update vars dependent on the extent layout (lastExtent, scanFlags, etc) - */ + /* + * Iterate over each ColumnCommand instance + * + * 1) reload its extent array + * 2) update TupleBPS's extent array + * 3) update vars dependent on the extent layout (lastExtent, scanFlags, etc) + */ - uint32_t i, j; - ColumnCommandJL* cc; - vector& filters = fBPP->getFilterSteps(); - vector& projections = fBPP->getProjectSteps(); - uint32_t oid; + uint32_t i, j; + ColumnCommandJL* cc; + vector& filters = fBPP->getFilterSteps(); + vector& projections = fBPP->getProjectSteps(); + uint32_t oid; - /* To reduce the race, make all CC's get new extents as close together - * as possible, then rebuild the local copies. - */ + /* To reduce the race, make all CC's get new extents as close together + * as possible, then rebuild the local copies. + */ - for (i = 0; i < filters.size(); i++) - { - cc = dynamic_cast(filters[i].get()); + for (i = 0; i < filters.size(); i++) + { + cc = dynamic_cast(filters[i].get()); - if (cc != NULL) - cc->reloadExtents(); - } + if (cc != NULL) + cc->reloadExtents(); + } - for (i = 0; i < projections.size(); i++) - { - cc = dynamic_cast(projections[i].get()); + for (i = 0; i < projections.size(); i++) + { + cc = dynamic_cast(projections[i].get()); - if (cc != NULL) - cc->reloadExtents(); - } + if (cc != NULL) + cc->reloadExtents(); + } - extentsMap.clear(); + extentsMap.clear(); - for (i = 0; i < filters.size(); i++) - { - cc = dynamic_cast(filters[i].get()); + for (i = 0; i < filters.size(); i++) + { + cc = dynamic_cast(filters[i].get()); - if (cc == NULL) - continue; + if (cc == NULL) + continue; - const vector& extents = cc->getExtents(); - oid = cc->getOID(); + const vector& extents = cc->getExtents(); + oid = cc->getOID(); - extentsMap[oid] = tr1::unordered_map(); - tr1::unordered_map& mref = extentsMap[oid]; + extentsMap[oid] = tr1::unordered_map(); + tr1::unordered_map& mref = extentsMap[oid]; - for (j = 0; j < extents.size(); j++) - mref[extents[j].range.start] = extents[j]; - } + for (j = 0; j < extents.size(); j++) + mref[extents[j].range.start] = extents[j]; + } - for (i = 0; i < projections.size(); i++) - { - cc = dynamic_cast(projections[i].get()); + for (i = 0; i < projections.size(); i++) + { + cc = dynamic_cast(projections[i].get()); - if (cc == NULL) - continue; + if (cc == NULL) + continue; - const vector& extents = cc->getExtents(); - oid = cc->getOID(); + const vector& extents = cc->getExtents(); + oid = cc->getOID(); - extentsMap[oid] = tr1::unordered_map(); - tr1::unordered_map& mref = extentsMap[oid]; + extentsMap[oid] = tr1::unordered_map(); + tr1::unordered_map& mref = extentsMap[oid]; - for (j = 0; j < extents.size(); j++) - mref[extents[j].range.start] = extents[j]; - } + for (j = 0; j < extents.size(); j++) + mref[extents[j].range.start] = extents[j]; + } - initExtentMarkers(); + initExtentMarkers(); } void TupleBPS::run() { - uint32_t i; - boost::mutex::scoped_lock lk(jlLock); - uint32_t retryCounter = 0; - const uint32_t retryMax = 1000; // 50s max; we've seen a 15s window so 50s should be 'safe' - const uint32_t waitInterval = 50000; // in us + uint32_t i; + boost::mutex::scoped_lock lk(jlLock); + uint32_t retryCounter = 0; + const uint32_t retryMax = 1000; // 50s max; we've seen a 15s window so 50s should be 'safe' + const uint32_t waitInterval = 50000; // in us - if (fRunExecuted) - return; + if (fRunExecuted) + return; - fRunExecuted = true; + fRunExecuted = true; - // make sure each numeric column has the same # of extents! See bugs 4564 and 3607 - try + // make sure each numeric column has the same # of extents! See bugs 4564 and 3607 + try + { + while (!goodExtentCount() && retryCounter++ < retryMax) { - while (!goodExtentCount() && retryCounter++ < retryMax) - { - usleep(waitInterval); - reloadExtentLists(); - } - } - catch (std::exception& e) - { - ostringstream os; - os << "TupleBPS: Could not get a consistent extent count for each column. Got '" - << e.what() << "'\n"; - catchHandler(os.str(), ERR_TUPLE_BPS, fErrorInfo, fSessionId); - fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); - return; + usleep(waitInterval); + reloadExtentLists(); } + } + catch (std::exception& e) + { + ostringstream os; + os << "TupleBPS: Could not get a consistent extent count for each column. Got '" << e.what() << "'\n"; + catchHandler(os.str(), ERR_TUPLE_BPS, fErrorInfo, fSessionId); + fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); + return; + } - if (retryCounter == retryMax) - { - catchHandler("TupleBPS: Could not get a consistent extent count for each column.", - ERR_TUPLE_BPS, fErrorInfo, fSessionId); - fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); - return; - } + if (retryCounter == retryMax) + { + catchHandler("TupleBPS: Could not get a consistent extent count for each column.", ERR_TUPLE_BPS, + fErrorInfo, fSessionId); + fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); + return; + } - if (traceOn()) - { - syslogStartStep(16, // exemgr subsystem - std::string("TupleBPS")); // step name - } + if (traceOn()) + { + syslogStartStep(16, // exemgr subsystem + std::string("TupleBPS")); // step name + } - ByteStream bs; + ByteStream bs; - if (fDelivery) - { - deliveryDL.reset(new RowGroupDL(1, 5)); - deliveryIt = deliveryDL->getIterator(); - } + if (fDelivery) + { + deliveryDL.reset(new RowGroupDL(1, 5)); + deliveryIt = deliveryDL->getIterator(); + } - fBPP->setThreadCount(fMaxNumProcessorThreads); + fBPP->setThreadCount(fMaxNumProcessorThreads); - if (doJoin) - for (i = 0; i < smallSideCount; i++) - tjoiners[i]->setThreadCount(fMaxNumProcessorThreads); + if (doJoin) + for (i = 0; i < smallSideCount; i++) + tjoiners[i]->setThreadCount(fMaxNumProcessorThreads); - if (fe1) - fBPP->setFEGroup1(fe1, fe1Input); + if (fe1) + fBPP->setFEGroup1(fe1, fe1Input); - if (fe2 && bRunFEonPM) - fBPP->setFEGroup2(fe2, fe2Output); + if (fe2 && bRunFEonPM) + fBPP->setFEGroup2(fe2, fe2Output); - if (fe2) - { - primRowGroup.initRow(&fe2InRow); - fe2Output.initRow(&fe2OutRow); - } + if (fe2) + { + primRowGroup.initRow(&fe2InRow); + fe2Output.initRow(&fe2OutRow); + } - try - { - fDec->addDECEventListener(this); - fBPP->priority(priority()); - fBPP->createBPP(bs); - fDec->write(uniqueID, bs); - BPPIsAllocated = true; + try + { + fDec->addDECEventListener(this); + fBPP->priority(priority()); + fBPP->createBPP(bs); + fDec->write(uniqueID, bs); + BPPIsAllocated = true; - if (doJoin && tjoiners[0]->inPM()) - serializeJoiner(); + if (doJoin && tjoiners[0]->inPM()) + serializeJoiner(); - prepCasualPartitioning(); - startPrimitiveThread(); - fProducerThreads.clear(); - fProducerThreads.reserve(fMaxNumThreads); - startAggregationThread(); - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_TUPLE_BPS, - logging::ERR_ALWAYS_CRITICAL, - "TupleBPS::run()"); - fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); - } + prepCasualPartitioning(); + startPrimitiveThread(); + fProducerThreads.clear(); + fProducerThreads.reserve(fMaxNumThreads); + startAggregationThread(); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_TUPLE_BPS, logging::ERR_ALWAYS_CRITICAL, + "TupleBPS::run()"); + fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); + } } void TupleBPS::join() { - boost::mutex::scoped_lock lk(jlLock); + boost::mutex::scoped_lock lk(jlLock); - if (joinRan) - return; + if (joinRan) + return; - joinRan = true; + joinRan = true; - if (fRunExecuted) + if (fRunExecuted) + { + if (msgsRecvd < msgsSent) { - if (msgsRecvd < msgsSent) - { - // wake up the sending thread, it should drain the input dl and exit - boost::unique_lock tplLock(tplMutex); - condvarWakeupProducer.notify_all(); - tplLock.unlock(); - } - - if (pThread) - jobstepThreadPool.join(pThread); - - jobstepThreadPool.join(fProducerThreads); - - if (BPPIsAllocated) - { - ByteStream bs; - fDec->removeDECEventListener(this); - fBPP->destroyBPP(bs); - - try - { - fDec->write(uniqueID, bs); - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_TUPLE_BPS, - logging::ERR_ALWAYS_CRITICAL, - "TupleBPS::join()"); - } - - BPPIsAllocated = false; - fDec->removeQueue(uniqueID); - tjoiners.clear(); - } + // wake up the sending thread, it should drain the input dl and exit + boost::unique_lock tplLock(tplMutex); + condvarWakeupProducer.notify_all(); + tplLock.unlock(); } + + if (pThread) + jobstepThreadPool.join(pThread); + + jobstepThreadPool.join(fProducerThreads); + + if (BPPIsAllocated) + { + ByteStream bs; + fDec->removeDECEventListener(this); + fBPP->destroyBPP(bs); + + try + { + fDec->write(uniqueID, bs); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_TUPLE_BPS, logging::ERR_ALWAYS_CRITICAL, + "TupleBPS::join()"); + } + + BPPIsAllocated = false; + fDec->removeQueue(uniqueID); + tjoiners.clear(); + } + } } void TupleBPS::sendError(uint16_t status) { - ByteStream msgBpp; - fBPP->setCount(1); - fBPP->setStatus(status); - fBPP->runErrorBPP(msgBpp); + ByteStream msgBpp; + fBPP->setCount(1); + fBPP->setStatus(status); + fBPP->runErrorBPP(msgBpp); - try - { - fDec->write(uniqueID, msgBpp); - } - catch (...) - { - // this fcn is only called in exception handlers - // let the first error take precedence - } + try + { + fDec->write(uniqueID, msgBpp); + } + catch (...) + { + // this fcn is only called in exception handlers + // let the first error take precedence + } - fBPP->reset(); - finishedSending = true; - condvar.notify_all(); - condvarWakeupProducer.notify_all(); + fBPP->reset(); + finishedSending = true; + condvar.notify_all(); + condvarWakeupProducer.notify_all(); } uint32_t TupleBPS::nextBand(ByteStream& bs) { - bool more = true; - RowGroup& realOutputRG = (fe2 ? fe2Output : primRowGroup); - RGData rgData; - uint32_t rowCount = 0; + bool more = true; + RowGroup& realOutputRG = (fe2 ? fe2Output : primRowGroup); + RGData rgData; + uint32_t rowCount = 0; - bs.restart(); + bs.restart(); - while (rowCount == 0 && more) - { - more = deliveryDL->next(deliveryIt, &rgData); + while (rowCount == 0 && more) + { + more = deliveryDL->next(deliveryIt, &rgData); - if (!more) - rgData = fBPP->getErrorRowGroupData(status()); + if (!more) + rgData = fBPP->getErrorRowGroupData(status()); - realOutputRG.setData(&rgData); - rowCount = realOutputRG.getRowCount(); + realOutputRG.setData(&rgData); + rowCount = realOutputRG.getRowCount(); - if ((more && rowCount > 0) || !more) - realOutputRG.serializeRGData(bs); - } + if ((more && rowCount > 0) || !more) + realOutputRG.serializeRGData(bs); + } - return rowCount; + return rowCount; } /* The current interleaving rotates over PMs, clustering jobs for a single PM @@ -1559,47 +1526,47 @@ uint32_t TupleBPS::nextBand(ByteStream& bs) */ void TupleBPS::interleaveJobs(vector* jobs) const { - vector newJobs; - uint32_t i; - uint32_t pmCount = 0; - scoped_array > bins; + vector newJobs; + uint32_t i; + uint32_t pmCount = 0; + scoped_array> bins; - // the input is grouped by dbroot + // the input is grouped by dbroot - if (pmCount == 1) - return; + if (pmCount == 1) + return; - /* Need to get the 'real' PM count */ - for (i = 0; i < jobs->size(); i++) - if (pmCount < (*jobs)[i].connectionNum + 1) - pmCount = (*jobs)[i].connectionNum + 1; + /* Need to get the 'real' PM count */ + for (i = 0; i < jobs->size(); i++) + if (pmCount < (*jobs)[i].connectionNum + 1) + pmCount = (*jobs)[i].connectionNum + 1; - bins.reset(new deque[pmCount]); + bins.reset(new deque[pmCount]); - // group by connection number - for (i = 0; i < jobs->size(); i++) - bins[(*jobs)[i].connectionNum].push_back((*jobs)[i]); + // group by connection number + for (i = 0; i < jobs->size(); i++) + bins[(*jobs)[i].connectionNum].push_back((*jobs)[i]); - // interleave by connection num - bool noWorkDone; + // interleave by connection num + bool noWorkDone; - while (newJobs.size() < jobs->size()) + while (newJobs.size() < jobs->size()) + { + noWorkDone = true; + + for (i = 0; i < pmCount; i++) { - noWorkDone = true; - - for (i = 0; i < pmCount; i++) - { - if (!bins[i].empty()) - { - newJobs.push_back(bins[i].front()); - bins[i].pop_front(); - noWorkDone = false; - } - } - - idbassert(!noWorkDone); + if (!bins[i].empty()) + { + newJobs.push_back(bins[i].front()); + bins[i].pop_front(); + noWorkDone = false; + } } + idbassert(!noWorkDone); + } + #if 0 /* Work in progress */ // up to this point, only the first connection to a PM is used. @@ -1618,1190 +1585,1178 @@ void TupleBPS::interleaveJobs(vector* jobs) const #endif - jobs->swap(newJobs); -// cout << "-------------\n"; -// for (i = 0; i < jobs->size(); i++) -// cout << "job " << i+1 << ": dbroot " << (*jobs)[i].dbroot << ", PM " -// << (*jobs)[i].connectionNum + 1 << endl; + jobs->swap(newJobs); + // cout << "-------------\n"; + // for (i = 0; i < jobs->size(); i++) + // cout << "job " << i+1 << ": dbroot " << (*jobs)[i].dbroot << ", PM " + // << (*jobs)[i].connectionNum + 1 << endl; } void TupleBPS::sendJobs(const vector& jobs) { - uint32_t i; - boost::unique_lock tplLock(tplMutex, boost::defer_lock); + uint32_t i; + boost::unique_lock tplLock(tplMutex, boost::defer_lock); - for (i = 0; i < jobs.size() && !cancelled(); i++) + for (i = 0; i < jobs.size() && !cancelled(); i++) + { + fDec->write(uniqueID, *(jobs[i].msg)); + tplLock.lock(); + msgsSent += jobs[i].expectedResponses; + + if (recvWaiting) + condvar.notify_all(); + + while ((msgsSent - msgsRecvd > fMaxOutstandingRequests << LOGICAL_EXTENT_CONVERTER) && !fDie) { - fDec->write(uniqueID, *(jobs[i].msg)); - tplLock.lock(); - msgsSent += jobs[i].expectedResponses; - - if (recvWaiting) - condvar.notify_all(); - - while ((msgsSent - msgsRecvd > fMaxOutstandingRequests << LOGICAL_EXTENT_CONVERTER) - && !fDie) - { - sendWaiting = true; - condvarWakeupProducer.wait(tplLock); - sendWaiting = false; - } - - tplLock.unlock(); + sendWaiting = true; + condvarWakeupProducer.wait(tplLock); + sendWaiting = false; } + + tplLock.unlock(); + } } -template +template bool TupleBPS::compareSingleValue(uint8_t COP, T val1, T val2) const { - switch (COP) - { - case COMPARE_LT: - case COMPARE_NGE: - return (val1 < val2); + switch (COP) + { + case COMPARE_LT: + case COMPARE_NGE: return (val1 < val2); - case COMPARE_LE: - case COMPARE_NGT: - return (val1 <= val2); + case COMPARE_LE: + case COMPARE_NGT: return (val1 <= val2); - case COMPARE_GT: - case COMPARE_NLE: - return (val1 > val2); + case COMPARE_GT: + case COMPARE_NLE: return (val1 > val2); - case COMPARE_GE: - case COMPARE_NLT: - return (val1 >= val2); + case COMPARE_GE: + case COMPARE_NLT: return (val1 >= val2); - case COMPARE_EQ: - return (val1 == val2); + case COMPARE_EQ: return (val1 == val2); - case COMPARE_NE: - return (val1 != val2); - } + case COMPARE_NE: return (val1 != val2); + } - return false; + return false; } /* (range COP val) comparisons */ bool TupleBPS::compareRange(uint8_t COP, int64_t min, int64_t max, int64_t val) const { - switch (COP) - { - case COMPARE_LT: - case COMPARE_NGE: - return (min < val); + switch (COP) + { + case COMPARE_LT: + case COMPARE_NGE: return (min < val); - case COMPARE_LE: - case COMPARE_NGT: - return (min <= val); + case COMPARE_LE: + case COMPARE_NGT: return (min <= val); - case COMPARE_GT: - case COMPARE_NLE: - return (max > val); + case COMPARE_GT: + case COMPARE_NLE: return (max > val); - case COMPARE_GE: - case COMPARE_NLT: - return (max >= val); + case COMPARE_GE: + case COMPARE_NLT: return (max >= val); - case COMPARE_EQ: // an 'in' comparison - return (val >= min && val <= max); + case COMPARE_EQ: // an 'in' comparison + return (val >= min && val <= max); - case COMPARE_NE: // 'not in' - return (val < min || val > max); - } + case COMPARE_NE: // 'not in' + return (val < min || val > max); + } - return false; + return false; } -bool TupleBPS::processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int64_t min, int64_t max, const uint8_t* filterString, - uint32_t filterCount) const +bool TupleBPS::processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int64_t min, int64_t max, + const uint8_t* filterString, uint32_t filterCount) const { - uint j; - bool ret = true; + uint j; + bool ret = true; - for (j = 0; j < filterCount; j++) + for (j = 0; j < filterCount; j++) + { + int8_t COP; + int64_t val2; + bool thisPredicate; + COP = *filterString++; + filterString++; // skip the round var, don't think that applies here + + switch (colWidth) { - int8_t COP; - int64_t val2; - bool thisPredicate; - COP = *filterString++; - filterString++; // skip the round var, don't think that applies here + case 1: + val2 = *((int8_t*)filterString); + filterString++; + break; - switch (colWidth) - { - case 1: - val2 = *((int8_t*) filterString); - filterString++; - break; + case 2: + val2 = *((int16_t*)filterString); + filterString += 2; + break; - case 2: - val2 = *((int16_t*) filterString); - filterString += 2; - break; + case 4: + val2 = *((int32_t*)filterString); + filterString += 4; + break; - case 4: - val2 = *((int32_t*) filterString); - filterString += 4; - break; + case 8: + val2 = *((int64_t*)filterString); + filterString += 8; + break; - case 8: - val2 = *((int64_t*) filterString); - filterString += 8; - break; - - default: - throw logic_error("invalid column width"); - } - - thisPredicate = compareRange(COP, min, max, val2); - - if (j == 0) - ret = thisPredicate; - - if (BOP == BOP_OR && thisPredicate) - return true; - else if (BOP == BOP_AND && !thisPredicate) - return false; + default: throw logic_error("invalid column width"); } - return ret; + thisPredicate = compareRange(COP, min, max, val2); + + if (j == 0) + ret = thisPredicate; + + if (BOP == BOP_OR && thisPredicate) + return true; + else if (BOP == BOP_AND && !thisPredicate) + return false; + } + + return ret; } -template +template bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, - uint32_t filterCount) const + uint32_t filterCount) const { - uint j; - bool ret = true; + uint j; + bool ret = true; - for (j = 0; j < filterCount; j++) + for (j = 0; j < filterCount; j++) + { + int8_t COP; + int64_t val2; + datatypes::TSInt128 bigVal2; + bool thisPredicate; + COP = *filterString++; + filterString++; // skip the round var, don't think that applies here + + switch (colWidth) { - int8_t COP; - int64_t val2; - datatypes::TSInt128 bigVal2; - bool thisPredicate; - COP = *filterString++; - filterString++; // skip the round var, don't think that applies here + case 1: + val2 = *((int8_t*)filterString); + filterString++; + break; - switch (colWidth) - { - case 1: - val2 = *((int8_t*) filterString); - filterString++; - break; + case 2: + val2 = *((int16_t*)filterString); + filterString += 2; + break; - case 2: - val2 = *((int16_t*) filterString); - filterString += 2; - break; + case 4: + val2 = *((int32_t*)filterString); + filterString += 4; + break; - case 4: - val2 = *((int32_t*) filterString); - filterString += 4; - break; + case 8: + val2 = *((int64_t*)filterString); + filterString += 8; + break; - case 8: - val2 = *((int64_t*) filterString); - filterString += 8; - break; + case 16: + bigVal2 = reinterpret_cast(filterString); + filterString += 16; + break; - case 16: - bigVal2 = reinterpret_cast(filterString); - filterString += 16; - break; - - default: - throw logic_error("invalid column width"); - } - - // Assumption is that colWidth > 0 - if (static_cast(colWidth) < datatypes::MAXDECIMALWIDTH) - thisPredicate = compareSingleValue(COP, (int64_t) val, val2); - else - thisPredicate = compareSingleValue(COP, (int128_t) val, bigVal2.getValue()); - - if (j == 0) - ret = thisPredicate; - - if (BOP == BOP_OR && thisPredicate) - return true; - else if (BOP == BOP_AND && !thisPredicate) - return false; + default: throw logic_error("invalid column width"); } - return ret; + // Assumption is that colWidth > 0 + if (static_cast(colWidth) < datatypes::MAXDECIMALWIDTH) + thisPredicate = compareSingleValue(COP, (int64_t)val, val2); + else + thisPredicate = compareSingleValue(COP, (int128_t)val, bigVal2.getValue()); + + if (j == 0) + ret = thisPredicate; + + if (BOP == BOP_OR && thisPredicate) + return true; + else if (BOP == BOP_AND && !thisPredicate) + return false; + } + + return ret; } -template +template bool TupleBPS::processOneFilterType(int8_t colWidth, T value, uint32_t type) const { - const vector& filters = fBPP->getFilterSteps(); - uint i; - bool ret = true; - bool firstPseudo = true; + const vector& filters = fBPP->getFilterSteps(); + uint i; + bool ret = true; + bool firstPseudo = true; - for (i = 0; i < filters.size(); i++) + for (i = 0; i < filters.size(); i++) + { + PseudoCCJL* pseudo = dynamic_cast(filters[i].get()); + + if (!pseudo || pseudo->getFunction() != type) + continue; + + int8_t BOP = pseudo->getBOP(); // I think this is the same as TupleBPS's bop var...? + + /* 1-byte COP, 1-byte 'round', colWidth-bytes value */ + const uint8_t* filterString = pseudo->getFilterString().buf(); + uint32_t filterCount = pseudo->getFilterCount(); + bool thisPredicate = processSingleFilterString(BOP, colWidth, value, filterString, filterCount); + + if (firstPseudo) { - PseudoCCJL* pseudo = dynamic_cast(filters[i].get()); - - if (!pseudo || pseudo->getFunction() != type) - continue; - - int8_t BOP = pseudo->getBOP(); // I think this is the same as TupleBPS's bop var...? - - /* 1-byte COP, 1-byte 'round', colWidth-bytes value */ - const uint8_t* filterString = pseudo->getFilterString().buf(); - uint32_t filterCount = pseudo->getFilterCount(); - bool thisPredicate = processSingleFilterString(BOP, colWidth, value, filterString, filterCount); - - if (firstPseudo) - { - firstPseudo = false; - ret = thisPredicate; - } - - if (bop == BOP_OR && thisPredicate) - return true; - else if (bop == BOP_AND && !thisPredicate) - return false; + firstPseudo = false; + ret = thisPredicate; } - return ret; + if (bop == BOP_OR && thisPredicate) + return true; + else if (bop == BOP_AND && !thisPredicate) + return false; + } + + return ret; } bool TupleBPS::processLBIDFilter(const EMEntry& emEntry) const { - const vector& filters = fBPP->getFilterSteps(); - uint i; - bool ret = true; - bool firstPseudo = true; - LBID_t firstLBID = emEntry.range.start; - LBID_t lastLBID = firstLBID + (emEntry.range.size * 1024) - 1; + const vector& filters = fBPP->getFilterSteps(); + uint i; + bool ret = true; + bool firstPseudo = true; + LBID_t firstLBID = emEntry.range.start; + LBID_t lastLBID = firstLBID + (emEntry.range.size * 1024) - 1; - for (i = 0; i < filters.size(); i++) + for (i = 0; i < filters.size(); i++) + { + PseudoCCJL* pseudo = dynamic_cast(filters[i].get()); + + if (!pseudo || pseudo->getFunction() != PSEUDO_BLOCKID) + continue; + + int8_t BOP = pseudo->getBOP(); // I think this is the same as TupleBPS's bop var...? + + /* 1-byte COP, 1-byte 'round', colWidth-bytes value */ + const uint8_t* filterString = pseudo->getFilterString().buf(); + uint32_t filterCount = pseudo->getFilterCount(); + bool thisPredicate = + processSingleFilterString_ranged(BOP, 8, firstLBID, lastLBID, filterString, filterCount); + + if (firstPseudo) { - PseudoCCJL* pseudo = dynamic_cast(filters[i].get()); - - if (!pseudo || pseudo->getFunction() != PSEUDO_BLOCKID) - continue; - - int8_t BOP = pseudo->getBOP(); // I think this is the same as TupleBPS's bop var...? - - /* 1-byte COP, 1-byte 'round', colWidth-bytes value */ - const uint8_t* filterString = pseudo->getFilterString().buf(); - uint32_t filterCount = pseudo->getFilterCount(); - bool thisPredicate = processSingleFilterString_ranged(BOP, 8, - firstLBID, lastLBID, filterString, filterCount); - - if (firstPseudo) - { - firstPseudo = false; - ret = thisPredicate; - } - - if (bop == BOP_OR && thisPredicate) - return true; - else if (bop == BOP_AND && !thisPredicate) - return false; + firstPseudo = false; + ret = thisPredicate; } - return ret; + if (bop == BOP_OR && thisPredicate) + return true; + else if (bop == BOP_AND && !thisPredicate) + return false; + } + + return ret; } -bool TupleBPS::processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr > dbRootPMMap) const +bool TupleBPS::processPseudoColFilters(uint32_t extentIndex, + boost::shared_ptr> dbRootPMMap) const { - if (!hasPCFilter) - return true; + if (!hasPCFilter) + return true; - const EMEntry& emEntry = scannedExtents[extentIndex]; + const EMEntry& emEntry = scannedExtents[extentIndex]; - if (bop == BOP_AND) - { - /* All Pseudocolumns have been promoted to 8-bytes except the casual partitioning filters */ - return (!hasPMFilter || processOneFilterType(8, (*dbRootPMMap)[emEntry.dbRoot], PSEUDO_PM)) - && (!hasSegmentFilter || processOneFilterType(8, emEntry.segmentNum, PSEUDO_SEGMENT)) - && (!hasDBRootFilter || processOneFilterType(8, emEntry.dbRoot, PSEUDO_DBROOT)) - && (!hasSegmentDirFilter || processOneFilterType(8, emEntry.partitionNum, PSEUDO_SEGMENTDIR)) - && (!hasExtentIDFilter || processOneFilterType(8, emEntry.range.start, PSEUDO_EXTENTID)) - && (!hasMaxFilter || (emEntry.partition.cprange.isValid == BRM::CP_VALID ? - (!fColType.isWideDecimalType() ? - processOneFilterType(emEntry.range.size, emEntry.partition.cprange.hiVal, PSEUDO_EXTENTMAX) : - processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigHiVal, PSEUDO_EXTENTMAX)) : true)) - && (!hasMinFilter || (emEntry.partition.cprange.isValid == BRM::CP_VALID ? - (!fColType.isWideDecimalType() ? - processOneFilterType(emEntry.range.size, emEntry.partition.cprange.loVal, PSEUDO_EXTENTMIN) : - processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigLoVal, PSEUDO_EXTENTMIN)) : true)) - && (!hasLBIDFilter || processLBIDFilter(emEntry)) - ; - } - else - { - return (hasPMFilter && processOneFilterType(8, (*dbRootPMMap)[emEntry.dbRoot], PSEUDO_PM)) - || (hasSegmentFilter && processOneFilterType(8, emEntry.segmentNum, PSEUDO_SEGMENT)) - || (hasDBRootFilter && processOneFilterType(8, emEntry.dbRoot, PSEUDO_DBROOT)) - || (hasSegmentDirFilter && processOneFilterType(8, emEntry.partitionNum, PSEUDO_SEGMENTDIR)) - || (hasExtentIDFilter && processOneFilterType(8, emEntry.range.start, PSEUDO_EXTENTID)) - || (hasMaxFilter && (emEntry.partition.cprange.isValid == BRM::CP_VALID ? - (!fColType.isWideDecimalType() ? - processOneFilterType(emEntry.range.size, emEntry.partition.cprange.hiVal, PSEUDO_EXTENTMAX) : - processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigHiVal, PSEUDO_EXTENTMAX)) : false)) - || (hasMinFilter && (emEntry.partition.cprange.isValid == BRM::CP_VALID ? - (!fColType.isWideDecimalType() ? - processOneFilterType(emEntry.range.size, emEntry.partition.cprange.loVal, PSEUDO_EXTENTMIN) : - processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigLoVal, PSEUDO_EXTENTMIN)) : false)) - || (hasLBIDFilter && processLBIDFilter(emEntry)) - ; - } + if (bop == BOP_AND) + { + /* All Pseudocolumns have been promoted to 8-bytes except the casual partitioning filters */ + return (!hasPMFilter || processOneFilterType(8, (*dbRootPMMap)[emEntry.dbRoot], PSEUDO_PM)) && + (!hasSegmentFilter || processOneFilterType(8, emEntry.segmentNum, PSEUDO_SEGMENT)) && + (!hasDBRootFilter || processOneFilterType(8, emEntry.dbRoot, PSEUDO_DBROOT)) && + (!hasSegmentDirFilter || processOneFilterType(8, emEntry.partitionNum, PSEUDO_SEGMENTDIR)) && + (!hasExtentIDFilter || processOneFilterType(8, emEntry.range.start, PSEUDO_EXTENTID)) && + (!hasMaxFilter || + (emEntry.partition.cprange.isValid == BRM::CP_VALID + ? (!fColType.isWideDecimalType() + ? processOneFilterType(emEntry.range.size, emEntry.partition.cprange.hiVal, + PSEUDO_EXTENTMAX) + : processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigHiVal, + PSEUDO_EXTENTMAX)) + : true)) && + (!hasMinFilter || + (emEntry.partition.cprange.isValid == BRM::CP_VALID + ? (!fColType.isWideDecimalType() + ? processOneFilterType(emEntry.range.size, emEntry.partition.cprange.loVal, + PSEUDO_EXTENTMIN) + : processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigLoVal, + PSEUDO_EXTENTMIN)) + : true)) && + (!hasLBIDFilter || processLBIDFilter(emEntry)); + } + else + { + return (hasPMFilter && processOneFilterType(8, (*dbRootPMMap)[emEntry.dbRoot], PSEUDO_PM)) || + (hasSegmentFilter && processOneFilterType(8, emEntry.segmentNum, PSEUDO_SEGMENT)) || + (hasDBRootFilter && processOneFilterType(8, emEntry.dbRoot, PSEUDO_DBROOT)) || + (hasSegmentDirFilter && processOneFilterType(8, emEntry.partitionNum, PSEUDO_SEGMENTDIR)) || + (hasExtentIDFilter && processOneFilterType(8, emEntry.range.start, PSEUDO_EXTENTID)) || + (hasMaxFilter && + (emEntry.partition.cprange.isValid == BRM::CP_VALID + ? (!fColType.isWideDecimalType() + ? processOneFilterType(emEntry.range.size, emEntry.partition.cprange.hiVal, + PSEUDO_EXTENTMAX) + : processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigHiVal, + PSEUDO_EXTENTMAX)) + : false)) || + (hasMinFilter && + (emEntry.partition.cprange.isValid == BRM::CP_VALID + ? (!fColType.isWideDecimalType() + ? processOneFilterType(emEntry.range.size, emEntry.partition.cprange.loVal, + PSEUDO_EXTENTMIN) + : processOneFilterType(fColType.colWidth, emEntry.partition.cprange.bigLoVal, + PSEUDO_EXTENTMIN)) + : false)) || + (hasLBIDFilter && processLBIDFilter(emEntry)); + } } - void TupleBPS::makeJobs(vector* jobs) { - boost::shared_ptr bs; - uint32_t i; - uint32_t lbidsToScan; - uint32_t blocksToScan; - uint32_t blocksPerJob; - LBID_t startingLBID; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - boost::shared_ptr > dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); - boost::shared_ptr > dbRootPMMap = oamCache->getDBRootToPMMap(); - int localPMId = oamCache->getLocalPMId(); + boost::shared_ptr bs; + uint32_t i; + uint32_t lbidsToScan; + uint32_t blocksToScan; + uint32_t blocksPerJob; + LBID_t startingLBID; + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + boost::shared_ptr> dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); + boost::shared_ptr> dbRootPMMap = oamCache->getDBRootToPMMap(); + int localPMId = oamCache->getLocalPMId(); - idbassert(ffirstStepType == SCAN); + idbassert(ffirstStepType == SCAN); - if (fOid >= 3000 && bop == BOP_AND) - storeCasualPartitionInfo(false); + if (fOid >= 3000 && bop == BOP_AND) + storeCasualPartitionInfo(false); - totalMsgs = 0; + totalMsgs = 0; - for (i = 0; i < scannedExtents.size(); i++) + for (i = 0; i < scannedExtents.size(); i++) + { + // the # of LBIDs to scan in this extent, if it will be scanned. + //@bug 5322: status EXTENTSTATUSMAX+1 means single block extent. + if ((scannedExtents[i].HWM == 0) && ((int)i < lastExtent[scannedExtents[i].dbRoot - 1]) && + (scannedExtents[i].status <= EXTENTSTATUSMAX)) + lbidsToScan = scannedExtents[i].range.size * 1024; + else + lbidsToScan = scannedExtents[i].HWM - scannedExtents[i].blockOffset + 1; + + // skip this extent if CP data rules it out or the scan has already passed + // the last extent for that DBRoot (import may be adding extents that shouldn't + // be read yet). Also skip if there's a pseudocolumn with a filter that would + // eliminate this extent + + bool inBounds = ((int)i <= lastExtent[scannedExtents[i].dbRoot - 1]); + + if (!inBounds) { - - // the # of LBIDs to scan in this extent, if it will be scanned. - //@bug 5322: status EXTENTSTATUSMAX+1 means single block extent. - if ((scannedExtents[i].HWM == 0) && - ((int) i < lastExtent[scannedExtents[i].dbRoot - 1]) && - (scannedExtents[i].status <= EXTENTSTATUSMAX)) - lbidsToScan = scannedExtents[i].range.size * 1024; - else - lbidsToScan = scannedExtents[i].HWM - scannedExtents[i].blockOffset + 1; - - // skip this extent if CP data rules it out or the scan has already passed - // the last extent for that DBRoot (import may be adding extents that shouldn't - // be read yet). Also skip if there's a pseudocolumn with a filter that would - // eliminate this extent - - bool inBounds = ((int)i <= lastExtent[scannedExtents[i].dbRoot - 1]); - - if (!inBounds) - { - continue; - } - - if (!scanFlags[i]) - { - fNumBlksSkipped += lbidsToScan; - continue; - } - - if (!processPseudoColFilters(i, dbRootPMMap)) - { - fNumBlksSkipped += lbidsToScan; - continue; - } - - //if (!scanFlags[i] || !inBounds) - // continue; - - /* Figure out many blocks have data in this extent - * Calc how many jobs to issue, - * Get the dbroot, - * construct the job msgs - */ - - // Bug5741 If we are local only and this doesn't belongs to us, skip it - if (fLocalQuery == execplan::CalpontSelectExecutionPlan::LOCAL_QUERY) - { - if (localPMId == 0) - { - throw IDBExcept(ERR_LOCAL_QUERY_UM); - } - - if (dbRootPMMap->find(scannedExtents[i].dbRoot)->second != localPMId) - continue; - } - - // a necessary DB root is offline - if (dbRootConnectionMap->find(scannedExtents[i].dbRoot) == dbRootConnectionMap->end()) - { - // MCOL-259 force a reload of the xml. This usualy fixes it. - Logger log; - log.logMessage(logging::LOG_TYPE_WARNING, "forcing reload of columnstore.xml for dbRootConnectionMap"); - oamCache->forceReload(); - dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); - - if (dbRootConnectionMap->find(scannedExtents[i].dbRoot) == dbRootConnectionMap->end()) - { - log.logMessage(logging::LOG_TYPE_WARNING, "dbroot still not in dbRootConnectionMap"); - throw IDBExcept(ERR_DATA_OFFLINE); - } - } - - - // the # of logical blocks in this extent - if (lbidsToScan % fColType.colWidth) - blocksToScan = lbidsToScan / fColType.colWidth + 1; - else - blocksToScan = lbidsToScan / fColType.colWidth; - - totalMsgs += blocksToScan; - - // how many logical blocks to process with a single job (& single thread on the PM) -#if defined(_MSC_VER) && BOOST_VERSION < 105200 - blocksPerJob = max(blocksToScan / fProcessorThreadsPerScan, 16UL); -#else - blocksPerJob = max(blocksToScan / fProcessorThreadsPerScan, 16U); -#endif - - startingLBID = scannedExtents[i].range.start; - - while (blocksToScan > 0) - { - uint32_t blocksThisJob = min(blocksToScan, blocksPerJob); - - fBPP->setLBID(startingLBID, scannedExtents[i]); - fBPP->setCount(blocksThisJob); - bs.reset(new ByteStream()); - fBPP->runBPP(*bs, (*dbRootConnectionMap)[scannedExtents[i].dbRoot]); - jobs->push_back(Job(scannedExtents[i].dbRoot, (*dbRootConnectionMap)[scannedExtents[i].dbRoot], - blocksThisJob, bs)); - blocksToScan -= blocksThisJob; - startingLBID += fColType.colWidth * blocksThisJob; - fBPP->reset(); - } + continue; } + if (!scanFlags[i]) + { + fNumBlksSkipped += lbidsToScan; + continue; + } + + if (!processPseudoColFilters(i, dbRootPMMap)) + { + fNumBlksSkipped += lbidsToScan; + continue; + } + + // if (!scanFlags[i] || !inBounds) + // continue; + + /* Figure out many blocks have data in this extent + * Calc how many jobs to issue, + * Get the dbroot, + * construct the job msgs + */ + + // Bug5741 If we are local only and this doesn't belongs to us, skip it + if (fLocalQuery == execplan::CalpontSelectExecutionPlan::LOCAL_QUERY) + { + if (localPMId == 0) + { + throw IDBExcept(ERR_LOCAL_QUERY_UM); + } + + if (dbRootPMMap->find(scannedExtents[i].dbRoot)->second != localPMId) + continue; + } + + // a necessary DB root is offline + if (dbRootConnectionMap->find(scannedExtents[i].dbRoot) == dbRootConnectionMap->end()) + { + // MCOL-259 force a reload of the xml. This usualy fixes it. + Logger log; + log.logMessage(logging::LOG_TYPE_WARNING, "forcing reload of columnstore.xml for dbRootConnectionMap"); + oamCache->forceReload(); + dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); + + if (dbRootConnectionMap->find(scannedExtents[i].dbRoot) == dbRootConnectionMap->end()) + { + log.logMessage(logging::LOG_TYPE_WARNING, "dbroot still not in dbRootConnectionMap"); + throw IDBExcept(ERR_DATA_OFFLINE); + } + } + + // the # of logical blocks in this extent + if (lbidsToScan % fColType.colWidth) + blocksToScan = lbidsToScan / fColType.colWidth + 1; + else + blocksToScan = lbidsToScan / fColType.colWidth; + + totalMsgs += blocksToScan; + + // how many logical blocks to process with a single job (& single thread on the PM) +#if defined(_MSC_VER) && BOOST_VERSION < 105200 + blocksPerJob = max(blocksToScan / fProcessorThreadsPerScan, 16UL); +#else + blocksPerJob = max(blocksToScan / fProcessorThreadsPerScan, 16U); +#endif + + startingLBID = scannedExtents[i].range.start; + + while (blocksToScan > 0) + { + uint32_t blocksThisJob = min(blocksToScan, blocksPerJob); + + fBPP->setLBID(startingLBID, scannedExtents[i]); + fBPP->setCount(blocksThisJob); + bs.reset(new ByteStream()); + fBPP->runBPP(*bs, (*dbRootConnectionMap)[scannedExtents[i].dbRoot]); + jobs->push_back( + Job(scannedExtents[i].dbRoot, (*dbRootConnectionMap)[scannedExtents[i].dbRoot], blocksThisJob, bs)); + blocksToScan -= blocksThisJob; + startingLBID += fColType.colWidth * blocksThisJob; + fBPP->reset(); + } + } } void TupleBPS::sendPrimitiveMessages() { - vector jobs; + vector jobs; - idbassert(ffirstStepType == SCAN); + idbassert(ffirstStepType == SCAN); - if (cancelled()) - goto abort; + if (cancelled()) + goto abort; - try - { - makeJobs(&jobs); - interleaveJobs(&jobs); - sendJobs(jobs); - } - catch (...) - { - sendError(logging::ERR_TUPLE_BPS); - handleException(std::current_exception(), logging::ERR_TUPLE_BPS, logging::ERR_ALWAYS_CRITICAL, - "st: " + std::to_string(fStepId) + " TupleBPS::sendPrimitiveMessages()"); - abort_nolock(); - } + try + { + makeJobs(&jobs); + interleaveJobs(&jobs); + sendJobs(jobs); + } + catch (...) + { + sendError(logging::ERR_TUPLE_BPS); + handleException(std::current_exception(), logging::ERR_TUPLE_BPS, logging::ERR_ALWAYS_CRITICAL, + "st: " + std::to_string(fStepId) + " TupleBPS::sendPrimitiveMessages()"); + abort_nolock(); + } abort: - boost::unique_lock tplLock(tplMutex); - finishedSending = true; - condvar.notify_all(); - tplLock.unlock(); + boost::unique_lock tplLock(tplMutex); + finishedSending = true; + condvar.notify_all(); + tplLock.unlock(); } void TupleBPS::processByteStreamVector(vector>& bsv, const uint32_t begin, const uint32_t end, vector<_CPInfo>& cpv, RowGroupDL* dlp, const uint32_t threadID) { - rowgroup::RGData rgData; - vector rgDatav; - vector fromPrimProc; - auto data = getJoinLocalDataByIndex(threadID); + rowgroup::RGData rgData; + vector rgDatav; + vector fromPrimProc; + auto data = getJoinLocalDataByIndex(threadID); - bool validCPData; - bool hasBinaryColumn; - int128_t min; - int128_t max; - uint64_t lbid; - uint32_t cachedIO; - uint32_t physIO; - uint32_t touchedBlocks; - int32_t memAmount = 0; + bool validCPData; + bool hasBinaryColumn; + int128_t min; + int128_t max; + uint64_t lbid; + uint32_t cachedIO; + uint32_t physIO; + uint32_t touchedBlocks; + int32_t memAmount = 0; - for (uint32_t i = begin; i < end; ++i) - { - messageqcpp::ByteStream* bs = bsv[i].get(); + for (uint32_t i = begin; i < end; ++i) + { + messageqcpp::ByteStream* bs = bsv[i].get(); - // @bug 488. when PrimProc node is down. error out - // An error condition. We are not going to do anymore. - ISMPacketHeader* hdr = (ISMPacketHeader*) (bs->buf()); + // @bug 488. when PrimProc node is down. error out + // An error condition. We are not going to do anymore. + ISMPacketHeader* hdr = (ISMPacketHeader*)(bs->buf()); #ifdef DEBUG_MPM - cout << "BS length: " << bs->length() << endl; + cout << "BS length: " << bs->length() << endl; #endif - if (bs->length() == 0 || hdr->Status > 0) - { - // PM errors mean this should abort right away instead of draining the PM backlog - // tplLock.lock(); + if (bs->length() == 0 || hdr->Status > 0) + { + // PM errors mean this should abort right away instead of draining the PM backlog + // tplLock.lock(); - if (bs->length() == 0) - { - errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_PRIMPROC_DOWN)); - status(ERR_PRIMPROC_DOWN); - } - else - { - string errMsg; + if (bs->length() == 0) + { + errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_PRIMPROC_DOWN)); + status(ERR_PRIMPROC_DOWN); + } + else + { + string errMsg; - bs->advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - *bs >> errMsg; - status(hdr->Status); - errorMessage(errMsg); - } + bs->advance(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + *bs >> errMsg; + status(hdr->Status); + errorMessage(errMsg); + } - // Sets `fDie` to true, so other threads can check `cancelled()`. - abort_nolock(); - return; - } - - bool unused; - fromPrimProc.clear(); - fBPP->getRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &min, &max, &cachedIO, - &physIO, &touchedBlocks, &unused, threadID, &hasBinaryColumn, - fColType); - - // Another layer of messiness. Need to refactor this fcn. - while (!fromPrimProc.empty() && !cancelled()) - { - rgData = fromPrimProc.back(); - fromPrimProc.pop_back(); - - data->local_primRG.setData(&rgData); - // TODO need the pre-join count even on PM joins... later - data->ridsReturned_Thread += data->local_primRG.getRowCount(); - - // TupleHashJoinStep::joinOneRG() is a port of the main join loop here. Any - // changes made here should also be made there and vice versa. - if (hasUMJoin || !fBPP->pmSendsFinalResult()) - { - data->joinedData = RGData(data->local_outputRG); - data->local_outputRG.setData(&data->joinedData); - data->local_outputRG.resetRowGroup(data->local_primRG.getBaseRid()); - data->local_outputRG.setDBRoot(data->local_primRG.getDBRoot()); - data->local_primRG.getRow(0, &data->largeSideRow); - - for (uint32_t k = 0; k < data->local_primRG.getRowCount() && !cancelled(); - k++, data->largeSideRow.nextRow()) - { - uint32_t matchCount = 0; - - for (uint32_t j = 0; j < smallSideCount; j++) - { - tjoiners[j]->match(data->largeSideRow, k, threadID, &(data->joinerOutput[j])); -#ifdef JLF_DEBUG - // Debugging code to print the matches - Row r; - joinerMatchesRGs[j].initRow(&r); - cout << data->joinerOutput[j].size() << " matches: \n"; - for (uint32_t z = 0; z < data->joinerOutput[j].size(); z++) - { - r.setPointer(data->joinerOutput[j][z]); - cout << " " << r.toString() << endl; - } -#endif - matchCount = data->joinerOutput[j].size(); - - if (tjoiners[j]->inUM()) - { - // Count the # of rows that pass the join filter - if (tjoiners[j]->hasFEFilter() && matchCount > 0) - { - vector newJoinerOutput; - applyMapping(data->fergMappings[smallSideCount], data->largeSideRow, - &data->joinFERow); - - for (uint32_t z = 0; z < data->joinerOutput[j].size(); z++) - { - data->smallSideRows[j].setPointer(data->joinerOutput[j][z]); - applyMapping(data->fergMappings[j], data->smallSideRows[j], - &data->joinFERow); - - if (!tjoiners[j]->evaluateFilter(data->joinFERow, threadID)) - matchCount--; - else - { - // The first match includes it in a SEMI join result and excludes it - // from an ANTI join result. If it's SEMI & SCALAR however, it needs - // to continue. - newJoinerOutput.push_back(data->joinerOutput[j][z]); - - if (tjoiners[j]->antiJoin() || - (tjoiners[j]->semiJoin() && !tjoiners[j]->scalar())) - break; - } - } - - // the filter eliminated all matches, need to join with the NULL row - if (matchCount == 0 && tjoiners[j]->largeOuterJoin()) - { - newJoinerOutput.push_back(Row::Pointer(data->smallNullMemory[j].get())); - matchCount = 1; - } - - data->joinerOutput[j].swap(newJoinerOutput); - } - - // XXXPAT: This has gone through enough revisions it would benefit - // from refactoring - - // If anti-join, reverse the result - if (tjoiners[j]->antiJoin()) - { - matchCount = (matchCount ? 0 : 1); - } - - if (matchCount == 0) - { - data->joinerOutput[j].clear(); - break; - } - else if (!tjoiners[j]->scalar() && - (tjoiners[j]->antiJoin() || tjoiners[j]->semiJoin())) - { - data->joinerOutput[j].clear(); - data->joinerOutput[j].push_back(Row::Pointer(data->smallNullMemory[j].get())); - matchCount = 1; - } - } - - if (matchCount == 0 && tjoiners[j]->innerJoin()) - break; - - // Scalar check - if (tjoiners[j]->scalar() && matchCount > 1) - { - errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW)); - status(ERR_MORE_THAN_1_ROW); - abort(); - } - - if (tjoiners[j]->smallOuterJoin()) - tjoiners[j]->markMatches(threadID, data->joinerOutput[j]); - } - - if (matchCount > 0) - { - applyMapping(data->largeMapping, data->largeSideRow, &data->joinedBaseRow); - data->joinedBaseRow.setRid(data->largeSideRow.getRelRid()); - memAmount += data->generateJoinResultSet( 0, rgDatav, dlp); - } - } // end of the for-loop in the join code - - if (data->local_outputRG.getRowCount() > 0) - { - rgDatav.push_back(data->joinedData); - } - } - else - { - rgDatav.push_back(rgData); - } - if (memAmount) - { - resourceManager()->returnMemory(memAmount); - memAmount = 0; - } - - // Execute UM F & E group 2 on rgDatav - if (fe2 && !bRunFEonPM && rgDatav.size() > 0 && !cancelled()) - { - data->processFE2(rgDatav); - rgDataVecToDl(rgDatav, data->local_fe2Output, dlp); - } - - data->cachedIO_Thread += cachedIO; - data->physIO_Thread += physIO; - data->touchedBlocks_Thread += touchedBlocks; - - if (fOid >= 3000 && ffirstStepType == SCAN && bop == BOP_AND) - { - if (fColType.colWidth <= 8) - { - cpv.push_back(_CPInfo((int64_t) min, (int64_t) max, lbid, validCPData)); - } - else if (fColType.colWidth == 16) - { - cpv.push_back(_CPInfo(min, max, lbid, validCPData)); - } - } - } // end of the per-rowgroup processing loop - - // insert the resulting rowgroup data from a single bytestream into dlp - if (rgDatav.size() > 0) - { - if (fe2 && bRunFEonPM) - rgDataVecToDl(rgDatav, data->local_fe2Output, dlp); - else - rgDataVecToDl(rgDatav, data->local_outputRG, dlp); - } + // Sets `fDie` to true, so other threads can check `cancelled()`. + abort_nolock(); + return; } + + bool unused; + fromPrimProc.clear(); + fBPP->getRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &min, &max, &cachedIO, &physIO, + &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); + + // Another layer of messiness. Need to refactor this fcn. + while (!fromPrimProc.empty() && !cancelled()) + { + rgData = fromPrimProc.back(); + fromPrimProc.pop_back(); + + data->local_primRG.setData(&rgData); + // TODO need the pre-join count even on PM joins... later + data->ridsReturned_Thread += data->local_primRG.getRowCount(); + + // TupleHashJoinStep::joinOneRG() is a port of the main join loop here. Any + // changes made here should also be made there and vice versa. + if (hasUMJoin || !fBPP->pmSendsFinalResult()) + { + data->joinedData = RGData(data->local_outputRG); + data->local_outputRG.setData(&data->joinedData); + data->local_outputRG.resetRowGroup(data->local_primRG.getBaseRid()); + data->local_outputRG.setDBRoot(data->local_primRG.getDBRoot()); + data->local_primRG.getRow(0, &data->largeSideRow); + + for (uint32_t k = 0; k < data->local_primRG.getRowCount() && !cancelled(); + k++, data->largeSideRow.nextRow()) + { + uint32_t matchCount = 0; + + for (uint32_t j = 0; j < smallSideCount; j++) + { + tjoiners[j]->match(data->largeSideRow, k, threadID, &(data->joinerOutput[j])); +#ifdef JLF_DEBUG + // Debugging code to print the matches + Row r; + joinerMatchesRGs[j].initRow(&r); + cout << data->joinerOutput[j].size() << " matches: \n"; + for (uint32_t z = 0; z < data->joinerOutput[j].size(); z++) + { + r.setPointer(data->joinerOutput[j][z]); + cout << " " << r.toString() << endl; + } +#endif + matchCount = data->joinerOutput[j].size(); + + if (tjoiners[j]->inUM()) + { + // Count the # of rows that pass the join filter + if (tjoiners[j]->hasFEFilter() && matchCount > 0) + { + vector newJoinerOutput; + applyMapping(data->fergMappings[smallSideCount], data->largeSideRow, &data->joinFERow); + + for (uint32_t z = 0; z < data->joinerOutput[j].size(); z++) + { + data->smallSideRows[j].setPointer(data->joinerOutput[j][z]); + applyMapping(data->fergMappings[j], data->smallSideRows[j], &data->joinFERow); + + if (!tjoiners[j]->evaluateFilter(data->joinFERow, threadID)) + matchCount--; + else + { + // The first match includes it in a SEMI join result and excludes it + // from an ANTI join result. If it's SEMI & SCALAR however, it needs + // to continue. + newJoinerOutput.push_back(data->joinerOutput[j][z]); + + if (tjoiners[j]->antiJoin() || (tjoiners[j]->semiJoin() && !tjoiners[j]->scalar())) + break; + } + } + + // the filter eliminated all matches, need to join with the NULL row + if (matchCount == 0 && tjoiners[j]->largeOuterJoin()) + { + newJoinerOutput.push_back(Row::Pointer(data->smallNullMemory[j].get())); + matchCount = 1; + } + + data->joinerOutput[j].swap(newJoinerOutput); + } + + // XXXPAT: This has gone through enough revisions it would benefit + // from refactoring + + // If anti-join, reverse the result + if (tjoiners[j]->antiJoin()) + { + matchCount = (matchCount ? 0 : 1); + } + + if (matchCount == 0) + { + data->joinerOutput[j].clear(); + break; + } + else if (!tjoiners[j]->scalar() && (tjoiners[j]->antiJoin() || tjoiners[j]->semiJoin())) + { + data->joinerOutput[j].clear(); + data->joinerOutput[j].push_back(Row::Pointer(data->smallNullMemory[j].get())); + matchCount = 1; + } + } + + if (matchCount == 0 && tjoiners[j]->innerJoin()) + break; + + // Scalar check + if (tjoiners[j]->scalar() && matchCount > 1) + { + errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW)); + status(ERR_MORE_THAN_1_ROW); + abort(); + } + + if (tjoiners[j]->smallOuterJoin()) + tjoiners[j]->markMatches(threadID, data->joinerOutput[j]); + } + + if (matchCount > 0) + { + applyMapping(data->largeMapping, data->largeSideRow, &data->joinedBaseRow); + data->joinedBaseRow.setRid(data->largeSideRow.getRelRid()); + memAmount += data->generateJoinResultSet(0, rgDatav, dlp); + } + } // end of the for-loop in the join code + + if (data->local_outputRG.getRowCount() > 0) + { + rgDatav.push_back(data->joinedData); + } + } + else + { + rgDatav.push_back(rgData); + } + if (memAmount) + { + resourceManager()->returnMemory(memAmount); + memAmount = 0; + } + + // Execute UM F & E group 2 on rgDatav + if (fe2 && !bRunFEonPM && rgDatav.size() > 0 && !cancelled()) + { + data->processFE2(rgDatav); + rgDataVecToDl(rgDatav, data->local_fe2Output, dlp); + } + + data->cachedIO_Thread += cachedIO; + data->physIO_Thread += physIO; + data->touchedBlocks_Thread += touchedBlocks; + + if (fOid >= 3000 && ffirstStepType == SCAN && bop == BOP_AND) + { + if (fColType.colWidth <= 8) + { + cpv.push_back(_CPInfo((int64_t)min, (int64_t)max, lbid, validCPData)); + } + else if (fColType.colWidth == 16) + { + cpv.push_back(_CPInfo(min, max, lbid, validCPData)); + } + } + } // end of the per-rowgroup processing loop + + // insert the resulting rowgroup data from a single bytestream into dlp + if (rgDatav.size() > 0) + { + if (fe2 && bRunFEonPM) + rgDataVecToDl(rgDatav, data->local_fe2Output, dlp); + else + rgDataVecToDl(rgDatav, data->local_outputRG, dlp); + } + } } void TupleBPS::receiveMultiPrimitiveMessages() { - AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - RowGroupDL* dlp = (fDelivery ? deliveryDL.get() : dl->rowGroupDL()); + AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); + RowGroupDL* dlp = (fDelivery ? deliveryDL.get() : dl->rowGroupDL()); - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; - uint32_t size = 0; + uint32_t size = 0; - // Based on the type of `tupleBPS` operation - initialize the `JoinLocalDataPool`. - // We initialize the max possible number of threads, because the right number of parallel threads will be - // chosen right before the `vector of bytestream` processing based on `ThreadPool` workload. - if (doJoin || fe2) - initializeJoinLocalDataPool(fMaxNumProcessorThreads); - else - initializeJoinLocalDataPool(1); + // Based on the type of `tupleBPS` operation - initialize the `JoinLocalDataPool`. + // We initialize the max possible number of threads, because the right number of parallel threads will be + // chosen right before the `vector of bytestream` processing based on `ThreadPool` workload. + if (doJoin || fe2) + initializeJoinLocalDataPool(fMaxNumProcessorThreads); + else + initializeJoinLocalDataPool(1); - vector> bsv; - boost::unique_lock tplLock(tplMutex, boost::defer_lock); + vector> bsv; + boost::unique_lock tplLock(tplMutex, boost::defer_lock); + + try + { + tplLock.lock(); + + while (1) + { + // sync with the send side + while (!finishedSending && msgsSent == msgsRecvd) + { + recvWaiting++; + condvar.wait(tplLock); + recvWaiting--; + } + + if (msgsSent == msgsRecvd && finishedSending) + break; + + bool flowControlOn; + fDec->read_some(uniqueID, fNumThreads, bsv, &flowControlOn); + size = bsv.size(); + + // @bug 4562 + if (traceOn() && fOid >= 3000 && dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + if (fOid >= 3000 && sts.msg_type == StepTeleStats::ST_INVALID && size > 0) + { + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = totalMsgs; + postStepStartTele(sts); + } + + for (uint32_t z = 0; z < size; z++) + { + if (bsv[z]->length() > 0 && fBPP->countThisMsg(*(bsv[z]))) + ++msgsRecvd; + } + + //@Bug 1424,1298 + + if (sendWaiting && ((msgsSent - msgsRecvd) <= (fMaxOutstandingRequests << LOGICAL_EXTENT_CONVERTER))) + { + condvarWakeupProducer.notify_one(); + THROTTLEDEBUG << "receiveMultiPrimitiveMessages wakes up sending side .. " + << " msgsSent: " << msgsSent << " msgsRecvd = " << msgsRecvd << endl; + } + + /* If there's an error and the joblist is being aborted, don't + sit around forever waiting for responses. */ + if (cancelled()) + { + if (sendWaiting) + condvarWakeupProducer.notify_one(); + + break; + } + + if (size == 0) + { + tplLock.unlock(); + usleep(2000 * fNumThreads); + tplLock.lock(); + continue; + } + + tplLock.unlock(); + + vector> cpInfos; + // Calculate the work sizes. + const uint32_t issuedThreads = jobstepThreadPool.getIssuedThreads(); + const uint32_t maxNumOfThreads = jobstepThreadPool.getMaxThreads(); + uint32_t numOfThreads = 1; + + if (issuedThreads < maxNumOfThreads && (doJoin || fe2)) + { + // We cannot parallel more than allocated data we have. + numOfThreads = std::min(maxNumOfThreads - issuedThreads, fNumProcessorThreads); + } + + uint32_t workSize = size / numOfThreads; + if (!workSize) + { + workSize = size; + numOfThreads = 1; + } + + vector workSizes; + // Calculate the work size for each thread. + workSizes.reserve(numOfThreads); + cpInfos.reserve(numOfThreads); + + for (uint32_t i = 0; i < numOfThreads; ++i) + { + workSizes.push_back(workSize); + cpInfos.push_back(vector<_CPInfo>()); + } + + const uint32_t moreWork = size % numOfThreads; + for (uint32_t i = 0; i < moreWork; ++i) + { + ++workSizes[i]; + } + + uint32_t start = 0; +#ifdef DEBUG_BSV + cout << "Number of threads: " << workSizes.size() << endl; +#endif + for (uint32_t i = 0, e = workSizes.size(); i < e; ++i) + { +#ifdef DEBUG_BSV + cout << "Thread # " << i << " work size " << workSizes[i] << endl; +#endif + uint32_t end = start + workSizes[i]; + startProcessingThread(this, bsv, start, end, cpInfos[i], dlp, i); + start = end; + } + + jobstepThreadPool.join(fProcessorThreads); + // Clear all. + fProcessorThreads.clear(); + bsv.clear(); + + // @bug 4562 + if (traceOn() && fOid >= 3000) + dlTimes.setFirstInsertTime(); + + // update casual partition + for (const auto& cpv : cpInfos) + { + size = cpv.size(); + if (size > 0 && !cancelled()) + { + for (uint32_t i = 0; i < size; i++) + { + if (fColType.colWidth > 8) + { + lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, fColType, cpv[i].valid); + } + else + { + lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType, cpv[i].valid); + } + } + } + } + + tplLock.lock(); + + if (fOid >= 3000) + { + uint64_t progress = msgsRecvd * 100 / totalMsgs; + bool postProgress = (progress > fProgress); + + if (postProgress) + { + fProgress = progress; + + sts.msg_type = StepTeleStats::ST_PROGRESS; + sts.total_units_of_work = totalMsgs; + sts.units_of_work_completed = msgsRecvd; + postStepProgressTele(sts); + } + } + + } // done reading + + } // try + catch (...) + { + handleException(std::current_exception(), logging::ERR_TUPLE_BPS, logging::ERR_ALWAYS_CRITICAL, + "st: " + std::to_string(fStepId) + " TupleBPS::receiveMultiPrimitiveMessages()"); + abort_nolock(); + } + + // We have one thread here and do not need to notify any waiting producer threads, because we are done with + // consuming messages from queue. + tplLock.unlock(); + + // Take just the first one. + auto data = getJoinLocalDataByIndex(0); + { + if (doJoin && smallOuterJoiner != -1 && !cancelled()) + { + vector rgDatav; + + // If this was a left outer join, this needs to put the unmatched + // rows from the joiner into the output + // XXXPAT: This might be a problem if later steps depend + // on sensible rids and/or sensible ordering + vector unmatched; +#ifdef JLF_DEBUG + cout << "finishing small-outer join output\n"; +#endif + uint32_t i = smallOuterJoiner; + tjoiners[i]->getUnmarkedRows(&unmatched); + data->joinedData = RGData(data->local_outputRG); + data->local_outputRG.setData(&data->joinedData); + data->local_outputRG.resetRowGroup(-1); + data->local_outputRG.getRow(0, &data->joinedBaseRow); + + for (uint32_t j = 0; j < unmatched.size(); j++) + { + data->smallSideRows[i].setPointer(unmatched[j]); + + for (uint32_t k = 0; k < smallSideCount; k++) + { + if (i == k) + applyMapping(data->smallMappings[i], data->smallSideRows[i], &data->joinedBaseRow); + else + applyMapping(data->smallMappings[k], data->smallNulls[k], &data->joinedBaseRow); + } + + applyMapping(data->largeMapping, data->largeNull, &data->joinedBaseRow); + data->joinedBaseRow.setRid(0); + data->joinedBaseRow.nextRow(); + data->local_outputRG.incRowCount(); + + if (data->local_outputRG.getRowCount() == 8192) + { + if (fe2) + { + rgDatav.push_back(data->joinedData); + data->processFE2(rgDatav); + + if (rgDatav.size() > 0) + rgDataToDl(rgDatav[0], data->local_fe2Output, dlp); + + rgDatav.clear(); + } + else + rgDataToDl(data->joinedData, data->local_outputRG, dlp); + + data->joinedData = RGData(data->local_outputRG); + data->local_outputRG.setData(&data->joinedData); + data->local_outputRG.resetRowGroup(-1); + data->local_outputRG.getRow(0, &data->joinedBaseRow); + } + } + + if (data->local_outputRG.getRowCount() > 0) + { + if (fe2) + { + rgDatav.push_back(data->joinedData); + data->processFE2(rgDatav); + + if (rgDatav.size() > 0) + rgDataToDl(rgDatav[0], data->local_fe2Output, dlp); + + rgDatav.clear(); + } + else + rgDataToDl(data->joinedData, data->local_outputRG, dlp); + } + } + + if (traceOn() && fOid >= 3000) + { + //...Casual partitioning could cause us to do no processing. In that + //...case these time stamps did not get set. So we set them here. + if (dlTimes.FirstReadTime().tv_sec == 0) + { + dlTimes.setFirstReadTime(); + dlTimes.setLastReadTime(); + dlTimes.setFirstInsertTime(); + } + + dlTimes.setEndOfInputTime(); + } + + ByteStream bs; try { - tplLock.lock(); - - while (1) - { - // sync with the send side - while (!finishedSending && msgsSent == msgsRecvd) - { - recvWaiting++; - condvar.wait(tplLock); - recvWaiting--; - } - - if (msgsSent == msgsRecvd && finishedSending) - break; - - bool flowControlOn; - fDec->read_some(uniqueID, fNumThreads, bsv, &flowControlOn); - size = bsv.size(); - - // @bug 4562 - if (traceOn() && fOid >= 3000 && dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - if (fOid >= 3000 && sts.msg_type == StepTeleStats::ST_INVALID && size > 0) - { - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = totalMsgs; - postStepStartTele(sts); - } - - for (uint32_t z = 0; z < size; z++) - { - if (bsv[z]->length() > 0 && fBPP->countThisMsg(*(bsv[z]))) - ++msgsRecvd; - } - - //@Bug 1424,1298 - - if (sendWaiting && ((msgsSent - msgsRecvd) <= - (fMaxOutstandingRequests << LOGICAL_EXTENT_CONVERTER))) - { - condvarWakeupProducer.notify_one(); - THROTTLEDEBUG << "receiveMultiPrimitiveMessages wakes up sending side .. " - << " msgsSent: " << msgsSent << " msgsRecvd = " << msgsRecvd << endl; - } - - /* If there's an error and the joblist is being aborted, don't - sit around forever waiting for responses. */ - if (cancelled()) - { - if (sendWaiting) - condvarWakeupProducer.notify_one(); - - break; - } - - if (size == 0) - { - tplLock.unlock(); - usleep(2000 * fNumThreads); - tplLock.lock(); - continue; - } - - tplLock.unlock(); - - vector> cpInfos; - // Calculate the work sizes. - const uint32_t issuedThreads = jobstepThreadPool.getIssuedThreads(); - const uint32_t maxNumOfThreads = jobstepThreadPool.getMaxThreads(); - uint32_t numOfThreads = 1; - - if (issuedThreads < maxNumOfThreads && (doJoin || fe2)) - { - // We cannot parallel more than allocated data we have. - numOfThreads = std::min(maxNumOfThreads - issuedThreads, fNumProcessorThreads); - } - - uint32_t workSize = size / numOfThreads; - if (!workSize) - { - workSize = size; - numOfThreads = 1; - } - - vector workSizes; - // Calculate the work size for each thread. - workSizes.reserve(numOfThreads); - cpInfos.reserve(numOfThreads); - - for (uint32_t i = 0; i < numOfThreads; ++i) - { - workSizes.push_back(workSize); - cpInfos.push_back(vector<_CPInfo>()); - } - - const uint32_t moreWork = size % numOfThreads; - for (uint32_t i = 0; i < moreWork; ++i) - { - ++workSizes[i]; - } - - uint32_t start = 0; -#ifdef DEBUG_BSV - cout << "Number of threads: " << workSizes.size() << endl; -#endif - for (uint32_t i = 0, e = workSizes.size(); i < e; ++i) - { - -#ifdef DEBUG_BSV - cout << "Thread # " << i << " work size " << workSizes[i] << endl; -#endif - uint32_t end = start + workSizes[i]; - startProcessingThread(this, bsv, start, end, cpInfos[i], dlp, i); - start = end; - } - - jobstepThreadPool.join(fProcessorThreads); - // Clear all. - fProcessorThreads.clear(); - bsv.clear(); - - // @bug 4562 - if (traceOn() && fOid >= 3000) - dlTimes.setFirstInsertTime(); - - // update casual partition - for (const auto& cpv : cpInfos) - { - size = cpv.size(); - if (size > 0 && !cancelled()) - { - for (uint32_t i = 0; i < size; i++) - { - if (fColType.colWidth > 8) - { - lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, fColType, - cpv[i].valid); - } - else - { - lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType, - cpv[i].valid); - } - } - } - } - - tplLock.lock(); - - if (fOid >= 3000) - { - uint64_t progress = msgsRecvd * 100 / totalMsgs; - bool postProgress = (progress > fProgress); - - if (postProgress) - { - fProgress = progress; - - sts.msg_type = StepTeleStats::ST_PROGRESS; - sts.total_units_of_work = totalMsgs; - sts.units_of_work_completed = msgsRecvd; - postStepProgressTele(sts); - } - } - - } // done reading - - } // try + if (BPPIsAllocated) + { + fDec->removeDECEventListener(this); + fBPP->destroyBPP(bs); + fDec->write(uniqueID, bs); + BPPIsAllocated = false; + } + } + // catch and do nothing. Let it continue with the clean up and profiling + catch (const std::exception& e) + { + cerr << "tuple-bps caught: " << e.what() << endl; + } catch (...) { - handleException(std::current_exception(), logging::ERR_TUPLE_BPS, logging::ERR_ALWAYS_CRITICAL, - "st: " + std::to_string(fStepId) + " TupleBPS::receiveMultiPrimitiveMessages()"); - abort_nolock(); + cerr << "tuple-bps caught unknown exception" << endl; } - // We have one thread here and do not need to notify any waiting producer threads, because we are done with - // consuming messages from queue. - tplLock.unlock(); + Stats stats = fDec->getNetworkStats(uniqueID); + fMsgBytesIn = stats.dataRecvd(); + fMsgBytesOut = stats.dataSent(); + fDec->removeQueue(uniqueID); + tjoiners.clear(); + } - // Take just the first one. - auto data = getJoinLocalDataByIndex(0); + //@Bug 1099 + ridsReturned += data->ridsReturned_Thread; + fPhysicalIO += data->physIO_Thread; + fCacheIO += data->cachedIO_Thread; + fBlockTouched += data->touchedBlocks_Thread; + + if (fTableOid >= 3000) + { + struct timeval tvbuf; + gettimeofday(&tvbuf, 0); + FIFO>* pFifo = 0; + uint64_t totalBlockedReadCount = 0; + uint64_t totalBlockedWriteCount = 0; + + //...Sum up the blocked FIFO reads for all input associations + size_t inDlCnt = fInputJobStepAssociation.outSize(); + + for (size_t iDataList = 0; iDataList < inDlCnt; iDataList++) { - if (doJoin && smallOuterJoiner != -1 && !cancelled()) - { - vector rgDatav; + pFifo = dynamic_cast>*>( + fInputJobStepAssociation.outAt(iDataList)->rowGroupDL()); - // If this was a left outer join, this needs to put the unmatched - // rows from the joiner into the output - // XXXPAT: This might be a problem if later steps depend - // on sensible rids and/or sensible ordering - vector unmatched; -#ifdef JLF_DEBUG - cout << "finishing small-outer join output\n"; -#endif - uint32_t i = smallOuterJoiner; - tjoiners[i]->getUnmarkedRows(&unmatched); - data->joinedData = RGData(data->local_outputRG); - data->local_outputRG.setData(&data->joinedData); - data->local_outputRG.resetRowGroup(-1); - data->local_outputRG.getRow(0, &data->joinedBaseRow); - - for (uint32_t j = 0; j < unmatched.size(); j++) - { - data->smallSideRows[i].setPointer(unmatched[j]); - - for (uint32_t k = 0; k < smallSideCount; k++) - { - if (i == k) - applyMapping(data->smallMappings[i], data->smallSideRows[i], &data->joinedBaseRow); - else - applyMapping(data->smallMappings[k], data->smallNulls[k], &data->joinedBaseRow); - } - - applyMapping(data->largeMapping, data->largeNull, &data->joinedBaseRow); - data->joinedBaseRow.setRid(0); - data->joinedBaseRow.nextRow(); - data->local_outputRG.incRowCount(); - - if (data->local_outputRG.getRowCount() == 8192) - { - if (fe2) - { - rgDatav.push_back(data->joinedData); - data->processFE2(rgDatav); - - if (rgDatav.size() > 0) - rgDataToDl(rgDatav[0], data->local_fe2Output, dlp); - - rgDatav.clear(); - } - else - rgDataToDl(data->joinedData, data->local_outputRG, dlp); - - data->joinedData = RGData(data->local_outputRG); - data->local_outputRG.setData(&data->joinedData); - data->local_outputRG.resetRowGroup(-1); - data->local_outputRG.getRow(0, &data->joinedBaseRow); - } - } - - if (data->local_outputRG.getRowCount() > 0) - { - if (fe2) - { - rgDatav.push_back(data->joinedData); - data->processFE2(rgDatav); - - if (rgDatav.size() > 0) - rgDataToDl(rgDatav[0], data->local_fe2Output, dlp); - - rgDatav.clear(); - } - else - rgDataToDl(data->joinedData, data->local_outputRG, dlp); - } - } - - if (traceOn() && fOid >= 3000) - { - //...Casual partitioning could cause us to do no processing. In that - //...case these time stamps did not get set. So we set them here. - if (dlTimes.FirstReadTime().tv_sec == 0) - { - dlTimes.setFirstReadTime(); - dlTimes.setLastReadTime(); - dlTimes.setFirstInsertTime(); - } - - dlTimes.setEndOfInputTime(); - } - - ByteStream bs; - - try - { - if (BPPIsAllocated) - { - fDec->removeDECEventListener(this); - fBPP->destroyBPP(bs); - fDec->write(uniqueID, bs); - BPPIsAllocated = false; - } - } - // catch and do nothing. Let it continue with the clean up and profiling - catch (const std::exception& e) - { - cerr << "tuple-bps caught: " << e.what() << endl; - } - catch (...) - { - cerr << "tuple-bps caught unknown exception" << endl; - } - - Stats stats = fDec->getNetworkStats(uniqueID); - fMsgBytesIn = stats.dataRecvd(); - fMsgBytesOut = stats.dataSent(); - fDec->removeQueue(uniqueID); - tjoiners.clear(); + if (pFifo) + { + totalBlockedReadCount += pFifo->blockedReadCount(); + } } - //@Bug 1099 - ridsReturned += data->ridsReturned_Thread; - fPhysicalIO += data->physIO_Thread; - fCacheIO += data->cachedIO_Thread; - fBlockTouched += data->touchedBlocks_Thread; + //...Sum up the blocked FIFO writes for all output associations + size_t outDlCnt = fOutputJobStepAssociation.outSize(); - if (fTableOid >= 3000) + for (size_t iDataList = 0; iDataList < outDlCnt; iDataList++) { - struct timeval tvbuf; - gettimeofday(&tvbuf, 0); - FIFO>* pFifo = 0; - uint64_t totalBlockedReadCount = 0; - uint64_t totalBlockedWriteCount = 0; + pFifo = dynamic_cast>*>(dlp); - //...Sum up the blocked FIFO reads for all input associations - size_t inDlCnt = fInputJobStepAssociation.outSize(); - - for (size_t iDataList = 0; iDataList < inDlCnt; iDataList++) - { - pFifo = dynamic_cast>*>( - fInputJobStepAssociation.outAt(iDataList)->rowGroupDL()); - - if (pFifo) - { - totalBlockedReadCount += pFifo->blockedReadCount(); - } - } - - //...Sum up the blocked FIFO writes for all output associations - size_t outDlCnt = fOutputJobStepAssociation.outSize(); - - for (size_t iDataList = 0; iDataList < outDlCnt; iDataList++) - { - pFifo = dynamic_cast>*>(dlp); - - if (pFifo) - { - totalBlockedWriteCount += pFifo->blockedWriteCount(); - } - } - - // Notify consumers as early as possible. - dlp->endOfInput(); - - //...Roundoff msg byte counts to nearest KB for display - uint64_t msgBytesInKB = fMsgBytesIn >> 10; - uint64_t msgBytesOutKB = fMsgBytesOut >> 10; - - if (fMsgBytesIn & 512) - msgBytesInKB++; - - if (fMsgBytesOut & 512) - msgBytesOutKB++; - - if (traceOn()) - { - // @bug 828 - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " - << JSTimeStamp::format(tvbuf) << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << fCacheIO - << "; MsgsSent-" << msgsSent << "; MsgsRvcd-" << msgsRecvd << "; BlocksTouched-" - << fBlockTouched << "; BlockedFifoIn/Out-" << totalBlockedReadCount << "/" - << totalBlockedWriteCount << "; output size-" << ridsReturned << endl - << "\tPartitionBlocksEliminated-" << fNumBlksSkipped << "; MsgBytesIn-" << msgBytesInKB - << "KB" - << "; MsgBytesOut-" << msgBytesOutKB << "KB" - << "; TotalMsgs-" << totalMsgs << endl - << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " - << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s\n\tUUID " << uuids::to_string(fStepUuid) << "\n\tQuery UUID " - << uuids::to_string(queryUuid()) << "\n\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - - syslogReadBlockCounts(16, // exemgr sybsystem - fPhysicalIO, // # blocks read from disk - fCacheIO, // # blocks read from cache - fNumBlksSkipped); // # casual partition block hits - syslogProcessingTimes(16, // exemgr subsystem - dlTimes.FirstReadTime(), // first datalist read - dlTimes.LastReadTime(), // last datalist read - dlTimes.FirstInsertTime(), // first datalist write - dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - syslogEndStep(16, // exemgr subsystem - totalBlockedReadCount, // blocked datalist input - totalBlockedWriteCount, // blocked datalist output - fMsgBytesIn, // incoming msg byte count - fMsgBytesOut); // outgoing msg byte count - fExtendedInfo += toString() + logStr.str(); - formatMiniStats(); - } - - { - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.phy_io = fPhysicalIO; - sts.cache_io = fCacheIO; - sts.msg_rcv_cnt = sts.total_units_of_work = sts.units_of_work_completed = msgsRecvd; - sts.cp_blocks_skipped = fNumBlksSkipped; - sts.msg_bytes_in = fMsgBytesIn; - sts.msg_bytes_out = fMsgBytesOut; - sts.rows = ridsReturned; - postStepSummaryTele(sts); - } - - if (ffirstStepType == SCAN && bop == BOP_AND && !cancelled()) - { - lbidList->UpdateAllPartitionInfo(fColType); - } + if (pFifo) + { + totalBlockedWriteCount += pFifo->blockedWriteCount(); + } } - else + + // Notify consumers as early as possible. + dlp->endOfInput(); + + //...Roundoff msg byte counts to nearest KB for display + uint64_t msgBytesInKB = fMsgBytesIn >> 10; + uint64_t msgBytesOutKB = fMsgBytesOut >> 10; + + if (fMsgBytesIn & 512) + msgBytesInKB++; + + if (fMsgBytesOut & 512) + msgBytesOutKB++; + + if (traceOn()) { - // Notify consumers. - dlp->endOfInput(); + // @bug 828 + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << JSTimeStamp::format(tvbuf) + << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << fCacheIO << "; MsgsSent-" << msgsSent + << "; MsgsRvcd-" << msgsRecvd << "; BlocksTouched-" << fBlockTouched << "; BlockedFifoIn/Out-" + << totalBlockedReadCount << "/" << totalBlockedWriteCount << "; output size-" << ridsReturned + << endl + << "\tPartitionBlocksEliminated-" << fNumBlksSkipped << "; MsgBytesIn-" << msgBytesInKB << "KB" + << "; MsgBytesOut-" << msgBytesOutKB << "KB" + << "; TotalMsgs-" << totalMsgs << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s\n\tUUID " << uuids::to_string(fStepUuid) << "\n\tQuery UUID " + << uuids::to_string(queryUuid()) << "\n\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + + syslogReadBlockCounts(16, // exemgr sybsystem + fPhysicalIO, // # blocks read from disk + fCacheIO, // # blocks read from cache + fNumBlksSkipped); // # casual partition block hits + syslogProcessingTimes(16, // exemgr subsystem + dlTimes.FirstReadTime(), // first datalist read + dlTimes.LastReadTime(), // last datalist read + dlTimes.FirstInsertTime(), // first datalist write + dlTimes.EndOfInputTime()); // last (endOfInput) datalist write + syslogEndStep(16, // exemgr subsystem + totalBlockedReadCount, // blocked datalist input + totalBlockedWriteCount, // blocked datalist output + fMsgBytesIn, // incoming msg byte count + fMsgBytesOut); // outgoing msg byte count + fExtendedInfo += toString() + logStr.str(); + formatMiniStats(); } + + { + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.phy_io = fPhysicalIO; + sts.cache_io = fCacheIO; + sts.msg_rcv_cnt = sts.total_units_of_work = sts.units_of_work_completed = msgsRecvd; + sts.cp_blocks_skipped = fNumBlksSkipped; + sts.msg_bytes_in = fMsgBytesIn; + sts.msg_bytes_out = fMsgBytesOut; + sts.rows = ridsReturned; + postStepSummaryTele(sts); + } + + if (ffirstStepType == SCAN && bop == BOP_AND && !cancelled()) + { + lbidList->UpdateAllPartitionInfo(fColType); + } + } + else + { + // Notify consumers. + dlp->endOfInput(); + } } const string TupleBPS::toString() const { - ostringstream oss; - oss << "TupleBPS ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId << - " tb/col:" << fTableOid << "/" << fOid; + ostringstream oss; + oss << "TupleBPS ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId << " st:" << fStepId + << " tb/col:" << fTableOid << "/" << fOid; - if (alias().length()) oss << " alias:" << alias(); + if (alias().length()) + oss << " alias:" << alias(); - if (view().length()) oss << " view:" << view(); + if (view().length()) + oss << " view:" << view(); #if 0 @@ -2811,585 +2766,572 @@ const string TupleBPS::toString() const #else - if (fDelivery) - oss << " is del "; - else - oss << " not del "; + if (fDelivery) + oss << " is del "; + else + oss << " not del "; - if (bop == BOP_OR) - oss << " BOP_OR "; + if (bop == BOP_OR) + oss << " BOP_OR "; - if (fDie) - oss << " aborting " << msgsSent << "/" << msgsRecvd << " " << uniqueID << " "; + if (fDie) + oss << " aborting " << msgsSent << "/" << msgsRecvd << " " << uniqueID << " "; - if (fOutputJobStepAssociation.outSize() > 0) - { - oss << fOutputJobStepAssociation.outAt(0); + if (fOutputJobStepAssociation.outSize() > 0) + { + oss << fOutputJobStepAssociation.outAt(0); - if (fOutputJobStepAssociation.outSize() > 1) - oss << " (too many outputs?)"; - } - else - { - oss << " (no outputs?)"; - } + if (fOutputJobStepAssociation.outSize() > 1) + oss << " (too many outputs?)"; + } + else + { + oss << " (no outputs?)"; + } #endif - oss << " nf:" << fFilterCount; - oss << " in:"; + oss << " nf:" << fFilterCount; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - { - oss << fInputJobStepAssociation.outAt(i); - } + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + oss << fInputJobStepAssociation.outAt(i); + } - oss << endl << " UUID: " << uuids::to_string(fStepUuid) << endl; - oss << " Query UUID: " << uuids::to_string(queryUuid()) << endl; - oss << " " << fBPP->toString() << endl; - return oss.str(); + oss << endl << " UUID: " << uuids::to_string(fStepUuid) << endl; + oss << " Query UUID: " << uuids::to_string(queryUuid()) << endl; + oss << " " << fBPP->toString() << endl; + return oss.str(); } /* This exists to avoid a DBRM lookup for every rid. */ inline bool TupleBPS::scanit(uint64_t rid) { - uint64_t fbo; - uint32_t extentIndex; + uint64_t fbo; + uint32_t extentIndex; - if (fOid < 3000) - return true; + if (fOid < 3000) + return true; - fbo = rid >> rpbShift; - extentIndex = fbo >> divShift; - return scanFlags[extentIndex] && runtimeCPFlags[extentIndex]; + fbo = rid >> rpbShift; + extentIndex = fbo >> divShift; + return scanFlags[extentIndex] && runtimeCPFlags[extentIndex]; } uint64_t TupleBPS::getFBO(uint64_t lbid) { - uint32_t i; - uint64_t lastLBID; + uint32_t i; + uint64_t lastLBID; - for (i = 0; i < numExtents; i++) - { - lastLBID = scannedExtents[i].range.start + (scannedExtents[i].range.size << 10) - 1; + for (i = 0; i < numExtents; i++) + { + lastLBID = scannedExtents[i].range.start + (scannedExtents[i].range.size << 10) - 1; - if (lbid >= (uint64_t) scannedExtents[i].range.start && lbid <= lastLBID) - return (lbid - scannedExtents[i].range.start) + (i << divShift); - } + if (lbid >= (uint64_t)scannedExtents[i].range.start && lbid <= lastLBID) + return (lbid - scannedExtents[i].range.start) + (i << divShift); + } - throw logic_error("TupleBPS: didn't find the FBO?"); + throw logic_error("TupleBPS: didn't find the FBO?"); } void TupleBPS::useJoiner(boost::shared_ptr tj) { - vector > v; - v.push_back(tj); - useJoiners(v); + vector> v; + v.push_back(tj); + useJoiners(v); } -void TupleBPS::useJoiners(const vector >& joiners) +void TupleBPS::useJoiners(const vector>& joiners) { - uint32_t i; + uint32_t i; - tjoiners = joiners; - doJoin = (joiners.size() != 0); + tjoiners = joiners; + doJoin = (joiners.size() != 0); - joinerMatchesRGs.clear(); - smallSideCount = tjoiners.size(); - hasPMJoin = false; - hasUMJoin = false; + joinerMatchesRGs.clear(); + smallSideCount = tjoiners.size(); + hasPMJoin = false; + hasUMJoin = false; - for (i = 0; i < smallSideCount; i++) - { - joinerMatchesRGs.push_back(tjoiners[i]->getSmallRG()); + for (i = 0; i < smallSideCount; i++) + { + joinerMatchesRGs.push_back(tjoiners[i]->getSmallRG()); - if (tjoiners[i]->inPM()) - hasPMJoin = true; - else - hasUMJoin = true; + if (tjoiners[i]->inPM()) + hasPMJoin = true; + else + hasUMJoin = true; - if (tjoiners[i]->getJoinType() & SMALLOUTER) - smallOuterJoiner = i; - } + if (tjoiners[i]->getJoinType() & SMALLOUTER) + smallOuterJoiner = i; + } - if (hasPMJoin) - fBPP->useJoiners(tjoiners); + if (hasPMJoin) + fBPP->useJoiners(tjoiners); } void TupleBPS::newPMOnline(uint32_t connectionNumber) { - ByteStream bs; + ByteStream bs; - fBPP->createBPP(bs); + fBPP->createBPP(bs); - try - { - fDec->write(bs, connectionNumber); + try + { + fDec->write(bs, connectionNumber); - if (hasPMJoin) - serializeJoiner(connectionNumber); - } - catch (IDBExcept& e) - { - abort(); - catchHandler(e.what(), e.errorCode(), fErrorInfo, fSessionId); - } + if (hasPMJoin) + serializeJoiner(connectionNumber); + } + catch (IDBExcept& e) + { + abort(); + catchHandler(e.what(), e.errorCode(), fErrorInfo, fSessionId); + } } void TupleBPS::setInputRowGroup(const rowgroup::RowGroup& rg) { - inputRowGroup = rg; - fBPP->setInputRowGroup(rg); + inputRowGroup = rg; + fBPP->setInputRowGroup(rg); } void TupleBPS::setOutputRowGroup(const rowgroup::RowGroup& rg) { - outputRowGroup = rg; - primRowGroup = rg; - fBPP->setProjectionRowGroup(rg); - checkDupOutputColumns(rg); + outputRowGroup = rg; + primRowGroup = rg; + fBPP->setProjectionRowGroup(rg); + checkDupOutputColumns(rg); - if (fe2) - fe2Mapping = makeMapping(outputRowGroup, fe2Output); + if (fe2) + fe2Mapping = makeMapping(outputRowGroup, fe2Output); } void TupleBPS::setJoinedResultRG(const rowgroup::RowGroup& rg) { - outputRowGroup = rg; - checkDupOutputColumns(rg); - fBPP->setJoinedRowGroup(rg); + outputRowGroup = rg; + checkDupOutputColumns(rg); + fBPP->setJoinedRowGroup(rg); - if (fe2) - fe2Mapping = makeMapping(outputRowGroup, fe2Output); + if (fe2) + fe2Mapping = makeMapping(outputRowGroup, fe2Output); } const rowgroup::RowGroup& TupleBPS::getOutputRowGroup() const { - return outputRowGroup; + return outputRowGroup; } void TupleBPS::setAggregateStep(const rowgroup::SP_ROWAGG_PM_t& agg, const rowgroup::RowGroup& rg) { - if (rg.getColumnCount() > 0) - { - fAggRowGroupPm = rg; - fAggregatorPm = agg; + if (rg.getColumnCount() > 0) + { + fAggRowGroupPm = rg; + fAggregatorPm = agg; - fBPP->addAggregateStep(agg, rg); - fBPP->setNeedRidsAtDelivery(false); - } + fBPP->addAggregateStep(agg, rg); + fBPP->setNeedRidsAtDelivery(false); + } } void TupleBPS::setBOP(uint8_t op) { - bop = op; - fBPP->setBOP(bop); + bop = op; + fBPP->setBOP(bop); } void TupleBPS::setJobInfo(const JobInfo* jobInfo) { - fBPP->jobInfo(jobInfo); + fBPP->jobInfo(jobInfo); } uint64_t TupleBPS::getEstimatedRowCount() { - // Call function that populates the scanFlags array based on the extents that qualify based on casual partitioning. - storeCasualPartitionInfo(true); - // TODO: Strip out the cout below after a few days of testing. + // Call function that populates the scanFlags array based on the extents that qualify based on casual + // partitioning. + storeCasualPartitionInfo(true); + // TODO: Strip out the cout below after a few days of testing. #ifdef JLF_DEBUG - cout << "OID-" << fOid << " EstimatedRowCount-" << fEstimatedRows << endl; + cout << "OID-" << fOid << " EstimatedRowCount-" << fEstimatedRows << endl; #endif - return fEstimatedRows; + return fEstimatedRows; } void TupleBPS::checkDupOutputColumns(const rowgroup::RowGroup& rg) { - // bug 1965, find if any duplicate columns selected - map keymap; // map - dupColumns.clear(); - const vector& keys = rg.getKeys(); + // bug 1965, find if any duplicate columns selected + map keymap; // map + dupColumns.clear(); + const vector& keys = rg.getKeys(); - for (uint32_t i = 0; i < keys.size(); i++) - { - map::iterator j = keymap.find(keys[i]); + for (uint32_t i = 0; i < keys.size(); i++) + { + map::iterator j = keymap.find(keys[i]); - if (j == keymap.end()) - keymap.insert(make_pair(keys[i], i)); // map key to col index - else - dupColumns.push_back(make_pair(i, j->second)); // dest/src index pair - } + if (j == keymap.end()) + keymap.insert(make_pair(keys[i], i)); // map key to col index + else + dupColumns.push_back(make_pair(i, j->second)); // dest/src index pair + } } void TupleBPS::dupOutputColumns(RGData& data, RowGroup& rg) { - rg.setData(&data); - dupOutputColumns(rg); + rg.setData(&data); + dupOutputColumns(rg); } void TupleBPS::dupOutputColumns(RowGroup& rg) { - Row workingRow; - rg.initRow(&workingRow); - rg.getRow(0, &workingRow); + Row workingRow; + rg.initRow(&workingRow); + rg.getRow(0, &workingRow); - for (uint64_t i = 0; i < rg.getRowCount(); i++) - { - for (uint64_t j = 0; j < dupColumns.size(); j++) - workingRow.copyField(dupColumns[j].first, dupColumns[j].second); + for (uint64_t i = 0; i < rg.getRowCount(); i++) + { + for (uint64_t j = 0; j < dupColumns.size(); j++) + workingRow.copyField(dupColumns[j].first, dupColumns[j].second); - workingRow.nextRow(); - } + workingRow.nextRow(); + } } void TupleBPS::stepId(uint16_t stepId) { - fStepId = stepId; - fBPP->setStepID(stepId); + fStepId = stepId; + fBPP->setStepID(stepId); } void TupleBPS::addFcnJoinExp(const vector& fe) { - if (!fe1) - fe1.reset(new funcexp::FuncExpWrapper()); + if (!fe1) + fe1.reset(new funcexp::FuncExpWrapper()); - for (uint32_t i = 0; i < fe.size(); i++) - fe1->addReturnedColumn(fe[i]); + for (uint32_t i = 0; i < fe.size(); i++) + fe1->addReturnedColumn(fe[i]); } void TupleBPS::addFcnExpGroup1(const boost::shared_ptr& fe) { - if (!fe1) - fe1.reset(new funcexp::FuncExpWrapper()); + if (!fe1) + fe1.reset(new funcexp::FuncExpWrapper()); - fe1->addFilter(fe); + fe1->addFilter(fe); } void TupleBPS::setFE1Input(const RowGroup& feInput) { - fe1Input = feInput; + fe1Input = feInput; } void TupleBPS::setFcnExpGroup2(const boost::shared_ptr& fe, const rowgroup::RowGroup& rg, bool runFE2onPM) { - fe2 = fe; - fe2Output = rg; - checkDupOutputColumns(rg); - fe2Mapping = makeMapping(outputRowGroup, fe2Output); - bRunFEonPM = runFE2onPM; + fe2 = fe; + fe2Output = rg; + checkDupOutputColumns(rg); + fe2Mapping = makeMapping(outputRowGroup, fe2Output); + bRunFEonPM = runFE2onPM; - if (bRunFEonPM) - fBPP->setFEGroup2(fe2, fe2Output); + if (bRunFEonPM) + fBPP->setFEGroup2(fe2, fe2Output); } void TupleBPS::setFcnExpGroup3(const vector& fe) { - if (!fe2) - fe2.reset(new funcexp::FuncExpWrapper()); + if (!fe2) + fe2.reset(new funcexp::FuncExpWrapper()); - for (uint32_t i = 0; i < fe.size(); i++) - fe2->addReturnedColumn(fe[i]); + for (uint32_t i = 0; i < fe.size(); i++) + fe2->addReturnedColumn(fe[i]); - // if this is called, there's no join, so it can always run on the PM - bRunFEonPM = true; - fBPP->setFEGroup2(fe2, fe2Output); + // if this is called, there's no join, so it can always run on the PM + bRunFEonPM = true; + fBPP->setFEGroup2(fe2, fe2Output); } void TupleBPS::setFE23Output(const rowgroup::RowGroup& feOutput) { - fe2Output = feOutput; - checkDupOutputColumns(feOutput); - fe2Mapping = makeMapping(outputRowGroup, fe2Output); + fe2Output = feOutput; + checkDupOutputColumns(feOutput); + fe2Mapping = makeMapping(outputRowGroup, fe2Output); - if (fe2 && bRunFEonPM) - fBPP->setFEGroup2(fe2, fe2Output); + if (fe2 && bRunFEonPM) + fBPP->setFEGroup2(fe2, fe2Output); } const rowgroup::RowGroup& TupleBPS::getDeliveredRowGroup() const { - if (fe2) - return fe2Output; + if (fe2) + return fe2Output; - return outputRowGroup; + return outputRowGroup; } void TupleBPS::deliverStringTableRowGroup(bool b) { - if (fe2) - fe2Output.setUseStringTable(b); - else if (doJoin) - outputRowGroup.setUseStringTable(b); - else - { - outputRowGroup.setUseStringTable(b); - primRowGroup.setUseStringTable(b); - } + if (fe2) + fe2Output.setUseStringTable(b); + else if (doJoin) + outputRowGroup.setUseStringTable(b); + else + { + outputRowGroup.setUseStringTable(b); + primRowGroup.setUseStringTable(b); + } - fBPP->deliverStringTableRowGroup(b); + fBPP->deliverStringTableRowGroup(b); } bool TupleBPS::deliverStringTableRowGroup() const { - if (fe2) - return fe2Output.usesStringTable(); + if (fe2) + return fe2Output.usesStringTable(); - return outputRowGroup.usesStringTable(); + return outputRowGroup.usesStringTable(); } void TupleBPS::formatMiniStats() { + ostringstream oss; + oss << "BPS " + << "PM " << alias() << " " << fTableOid << " " << fBPP->toMiniString() << " " << fPhysicalIO << " " + << fCacheIO << " " << fNumBlksSkipped << " " + << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " << ridsReturned + << " "; - ostringstream oss; - oss << "BPS " - << "PM " - << alias() << " " - << fTableOid << " " - << fBPP->toMiniString() << " " - << fPhysicalIO << " " - << fCacheIO << " " - << fNumBlksSkipped << " " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << ridsReturned << " "; - - fMiniInfo += oss.str(); + fMiniInfo += oss.str(); } void TupleBPS::rgDataToDl(RGData& rgData, RowGroup& rg, RowGroupDL* dlp) { - // bug 1965, populate duplicate columns if any. - if (dupColumns.size() > 0) - dupOutputColumns(rgData, rg); + // bug 1965, populate duplicate columns if any. + if (dupColumns.size() > 0) + dupOutputColumns(rgData, rg); - dlp->insert(rgData); + dlp->insert(rgData); } - void TupleBPS::rgDataVecToDl(vector& rgDatav, RowGroup& rg, RowGroupDL* dlp) { - uint64_t size = rgDatav.size(); + uint64_t size = rgDatav.size(); - if (size > 0 && !cancelled()) + if (size > 0 && !cancelled()) + { + dlMutex.lock(); + + for (uint64_t i = 0; i < size; i++) { - dlMutex.lock(); - - for (uint64_t i = 0; i < size; i++) - { - rgDataToDl(rgDatav[i], rg, dlp); - } - - dlMutex.unlock(); + rgDataToDl(rgDatav[i], rg, dlp); } - rgDatav.clear(); + dlMutex.unlock(); + } + + rgDatav.clear(); } void TupleBPS::setJoinFERG(const RowGroup& rg) { - joinFERG = rg; - fBPP->setJoinFERG(rg); + joinFERG = rg; + fBPP->setJoinFERG(rg); } void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool isRange, bool isSmallSideWideDecimal) { + if (fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP || fOid < 3000) + return; - if (fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP || fOid < 3000) + uint32_t i, j, k; + int64_t min, max, seq; + int128_t bigMin, bigMax; + bool isValid, intersection; + vector colCmdVec = fBPP->getFilterSteps(); + ColumnCommandJL* cmd; + + for (i = 0; i < fBPP->getProjectSteps().size(); i++) + colCmdVec.push_back(fBPP->getProjectSteps()[i]); + + LBIDList ll(OID, 0); + + /* Find the columncommand with that OID. + * Check that the column type is one handled by CP. + * For each extent in that OID, + * grab the min & max, + * OR together all of the intersection tests, + * AND it with the current CP flag. + */ + + for (i = 0; i < colCmdVec.size(); i++) + { + cmd = dynamic_cast(colCmdVec[i].get()); + + if (cmd != NULL && cmd->getOID() == OID) + { + const execplan::CalpontSystemCatalog::ColType& colType = cmd->getColType(); + + if (!ll.CasualPartitionDataType(colType.colDataType, colType.colWidth) || cmd->isDict()) return; - uint32_t i, j, k; - int64_t min, max, seq; - int128_t bigMin, bigMax; - bool isValid, intersection; - vector colCmdVec = fBPP->getFilterSteps(); - ColumnCommandJL* cmd; + // @bug 2989, use correct extents + tr1::unordered_map* extentsPtr = NULL; + vector extents; // in case the extents of OID is not in Map - for (i = 0; i < fBPP->getProjectSteps().size(); i++) - colCmdVec.push_back(fBPP->getProjectSteps()[i]); + // TODO: store the sorted vectors from the pcolscans/steps as a minor optimization + dbrm.getExtents(OID, extents); + sort(extents.begin(), extents.end(), ExtentSorter()); - LBIDList ll(OID, 0); + if (extentsMap.find(OID) != extentsMap.end()) + { + extentsPtr = &extentsMap[OID]; + } + else if (dbrm.getExtents(OID, extents) == 0) + { + extentsMap[OID] = tr1::unordered_map(); + tr1::unordered_map& mref = extentsMap[OID]; - /* Find the columncommand with that OID. - * Check that the column type is one handled by CP. - * For each extent in that OID, - * grab the min & max, - * OR together all of the intersection tests, - * AND it with the current CP flag. - */ + for (uint32_t z = 0; z < extents.size(); z++) + mref[extents[z].range.start] = extents[z]; - for (i = 0; i < colCmdVec.size(); i++) - { - cmd = dynamic_cast(colCmdVec[i].get()); + extentsPtr = &mref; + } - if (cmd != NULL && cmd->getOID() == OID) + if (colType.colWidth <= 8) + { + for (j = 0; j < extents.size(); j++) { - const execplan::CalpontSystemCatalog::ColType& colType = cmd->getColType(); + isValid = ll.GetMinMax(&min, &max, &seq, extents[j].range.start, *extentsPtr, colType.colDataType); - if (!ll.CasualPartitionDataType(colType.colDataType, colType.colWidth) - || cmd->isDict()) - return; - - // @bug 2989, use correct extents - tr1::unordered_map* extentsPtr = NULL; - vector extents; // in case the extents of OID is not in Map - - // TODO: store the sorted vectors from the pcolscans/steps as a minor optimization - dbrm.getExtents(OID, extents); - sort(extents.begin(), extents.end(), ExtentSorter()); - - if (extentsMap.find(OID) != extentsMap.end()) + if (isValid) + { + if (isRange) { - extentsPtr = &extentsMap[OID]; - } - else if (dbrm.getExtents(OID, extents) == 0) - { - extentsMap[OID] = tr1::unordered_map(); - tr1::unordered_map& mref = extentsMap[OID]; - - for (uint32_t z = 0; z < extents.size(); z++) - mref[extents[z].range.start] = extents[z]; - - extentsPtr = &mref; - } - - if (colType.colWidth <= 8) - { - for (j = 0; j < extents.size(); j++) - { - isValid = ll.GetMinMax(&min, &max, &seq, extents[j].range.start, *extentsPtr, - colType.colDataType); - - if (isValid) - { - if (isRange) - { - if (!isSmallSideWideDecimal) - { - runtimeCPFlags[j] = ll.checkRangeOverlap(min, max, (int64_t) vals[0], (int64_t) vals[1], - colType) && runtimeCPFlags[j]; - } - else - { - runtimeCPFlags[j] = ll.checkRangeOverlap((int128_t) min, (int128_t) max, vals[0], vals[1], - colType) && runtimeCPFlags[j]; - } - } - else - { - intersection = false; - - for (k = 0; k < vals.size(); k++) - { - if (!isSmallSideWideDecimal) - { - intersection = intersection || - ll.checkSingleValue(min, max, (int64_t) vals[k], colType); - } - else - { - intersection = intersection || - ll.checkSingleValue((int128_t) min, (int128_t) max, vals[k], colType); - } - } - - runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; - } - } - } + if (!isSmallSideWideDecimal) + { + runtimeCPFlags[j] = + ll.checkRangeOverlap(min, max, (int64_t)vals[0], (int64_t)vals[1], colType) && + runtimeCPFlags[j]; + } + else + { + runtimeCPFlags[j] = + ll.checkRangeOverlap((int128_t)min, (int128_t)max, vals[0], vals[1], colType) && + runtimeCPFlags[j]; + } } else { - for (j = 0; j < extents.size(); j++) + intersection = false; + + for (k = 0; k < vals.size(); k++) + { + if (!isSmallSideWideDecimal) { - isValid = ll.GetMinMax(&bigMin, &bigMax, &seq, extents[j].range.start, *extentsPtr, - colType.colDataType); - - if (isValid) - { - if (isRange) - runtimeCPFlags[j] = ll.checkRangeOverlap(bigMin, bigMax, vals[0], vals[1], - colType) && runtimeCPFlags[j]; - else - { - intersection = false; - - for (k = 0; k < vals.size(); k++) - intersection = intersection || - ll.checkSingleValue(bigMin, bigMax, vals[k], colType); - - runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; - } - } + intersection = intersection || ll.checkSingleValue(min, max, (int64_t)vals[k], colType); } - } + else + { + intersection = + intersection || ll.checkSingleValue((int128_t)min, (int128_t)max, vals[k], colType); + } + } - break; + runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + } + } } + } + else + { + for (j = 0; j < extents.size(); j++) + { + isValid = + ll.GetMinMax(&bigMin, &bigMax, &seq, extents[j].range.start, *extentsPtr, colType.colDataType); + + if (isValid) + { + if (isRange) + runtimeCPFlags[j] = + ll.checkRangeOverlap(bigMin, bigMax, vals[0], vals[1], colType) && runtimeCPFlags[j]; + else + { + intersection = false; + + for (k = 0; k < vals.size(); k++) + intersection = intersection || ll.checkSingleValue(bigMin, bigMax, vals[k], colType); + + runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + } + } + } + } + + break; } + } } void TupleBPS::dec(DistributedEngineComm* dec) { - if (fDec) - fDec->removeQueue(uniqueID); + if (fDec) + fDec->removeQueue(uniqueID); - fDec = dec; + fDec = dec; - if (fDec) - fDec->addQueue(uniqueID, true); + if (fDec) + fDec->addQueue(uniqueID, true); } void TupleBPS::abort_nolock() { - if (fDie) - return; + if (fDie) + return; - JobStep::abort(); + JobStep::abort(); - if (fDec && BPPIsAllocated) + if (fDec && BPPIsAllocated) + { + ByteStream bs; + fBPP->abortProcessing(&bs); + + try { - ByteStream bs; - fBPP->abortProcessing(&bs); - - try - { - fDec->write(uniqueID, bs); - } - catch (...) - { - // this throws only if there are no PMs left. If there are none, - // that is the cause of the abort and that will be reported to the - // front-end already. Nothing to do here. - } - - BPPIsAllocated = false; - fDec->shutdownQueue(uniqueID); + fDec->write(uniqueID, bs); + } + catch (...) + { + // this throws only if there are no PMs left. If there are none, + // that is the cause of the abort and that will be reported to the + // front-end already. Nothing to do here. } - condvarWakeupProducer.notify_all(); - condvar.notify_all(); + BPPIsAllocated = false; + fDec->shutdownQueue(uniqueID); + } + + condvarWakeupProducer.notify_all(); + condvar.notify_all(); } void TupleBPS::abort() { - boost::mutex::scoped_lock scoped(boost::mutex); - abort_nolock(); + boost::mutex::scoped_lock scoped(boost::mutex); + abort_nolock(); } -template -bool TupleBPS::processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const; -template -bool TupleBPS::processOneFilterType(int8_t colWidth, int128_t value, uint32_t type) const; +template bool TupleBPS::processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const; +template bool TupleBPS::processOneFilterType(int8_t colWidth, int128_t value, uint32_t type) const; -template -bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, - uint32_t filterCount) const; -template -bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int128_t val, const uint8_t* filterString, - uint32_t filterCount) const; +template bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, + const uint8_t* filterString, + uint32_t filterCount) const; +template bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int128_t val, + const uint8_t* filterString, + uint32_t filterCount) const; -template -bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const; -template -bool TupleBPS::compareSingleValue(uint8_t COP, int128_t val1, int128_t val2) const; +template bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const; +template bool TupleBPS::compareSingleValue(uint8_t COP, int128_t val1, int128_t val2) const; -} // namespace +} // namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 49b98abb8..b3ad0c209 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -18,7 +18,6 @@ // $Id: tupleaggregatestep.cpp 9732 2013-08-02 15:56:15Z pleblanc $ - //#define NDEBUG // Cross engine needs to be at top due to MySQL includes #define PREFER_MY_CONFIG_H @@ -72,61 +71,60 @@ using namespace querytele; //#include "stopwatch.cpp" -//Stopwatch timer; +// Stopwatch timer; namespace { - struct cmpTuple { - bool operator()(boost::tuple* > a, - boost::tuple* > b) - { - uint32_t keya = boost::get<0>(a); - uint32_t keyb = boost::get<0>(b); - int opa; - int opb; - mcsv1sdk::mcsv1_UDAF* pUDAFa; - mcsv1sdk::mcsv1_UDAF* pUDAFb; + bool operator()(boost::tuple*> a, + boost::tuple*> b) + { + uint32_t keya = boost::get<0>(a); + uint32_t keyb = boost::get<0>(b); + int opa; + int opb; + mcsv1sdk::mcsv1_UDAF* pUDAFa; + mcsv1sdk::mcsv1_UDAF* pUDAFb; - // If key is less than - if (keya < keyb) - return true; - if (keya == keyb) + // If key is less than + if (keya < keyb) + return true; + if (keya == keyb) + { + // test Op + opa = boost::get<1>(a); + opb = boost::get<1>(b); + if (opa < opb) + return true; + if (opa == opb) + { + // look at the UDAF object + pUDAFa = boost::get<2>(a); + pUDAFb = boost::get<2>(b); + if (pUDAFa < pUDAFb) + return true; + if (pUDAFa == pUDAFb) { - // test Op - opa = boost::get<1>(a); - opb = boost::get<1>(b); - if (opa < opb) - return true; - if (opa == opb) + std::vector* paramKeysa = boost::get<3>(a); + std::vector* paramKeysb = boost::get<3>(b); + if (paramKeysa == NULL || paramKeysb == NULL) + return false; + if (paramKeysa->size() < paramKeysb->size()) + return true; + if (paramKeysa->size() == paramKeysb->size()) + { + for (uint64_t i = 0; i < paramKeysa->size(); ++i) { - // look at the UDAF object - pUDAFa = boost::get<2>(a); - pUDAFb = boost::get<2>(b); - if (pUDAFa < pUDAFb) - return true; - if (pUDAFa == pUDAFb) - { - std::vector* paramKeysa = boost::get<3>(a); - std::vector* paramKeysb = boost::get<3>(b); - if (paramKeysa == NULL || paramKeysb == NULL) - return false; - if (paramKeysa->size() < paramKeysb->size()) - return true; - if (paramKeysa->size() == paramKeysb->size()) - { - for (uint64_t i = 0; i < paramKeysa->size(); ++i) - { - if ((*paramKeysa)[i] < (*paramKeysb)[i]) - return true; - } - } - } + if ((*paramKeysa)[i] < (*paramKeysb)[i]) + return true; } + } } - return false; + } } + return false; + } }; typedef vector> RowBucket; @@ -135,2308 +133,2208 @@ typedef vector RowBucketVec; // The AGG_MAP type is used to maintain a list of aggregate functions in order to // detect duplicates. Since all UDAF have the same op type (ROWAGG_UDAF), we add in // the function pointer in order to ensure uniqueness. -typedef map* >, uint64_t, cmpTuple> AGG_MAP; +typedef map*>, uint64_t, cmpTuple> + AGG_MAP; inline RowAggFunctionType functionIdMap(int planFuncId) { - switch (planFuncId) - { - case AggregateColumn::COUNT_ASTERISK: - return ROWAGG_COUNT_ASTERISK; + switch (planFuncId) + { + case AggregateColumn::COUNT_ASTERISK: return ROWAGG_COUNT_ASTERISK; - case AggregateColumn::COUNT: - return ROWAGG_COUNT_COL_NAME; + case AggregateColumn::COUNT: return ROWAGG_COUNT_COL_NAME; - case AggregateColumn::SUM: - return ROWAGG_SUM; + case AggregateColumn::SUM: return ROWAGG_SUM; - case AggregateColumn::AVG: - return ROWAGG_AVG; + case AggregateColumn::AVG: return ROWAGG_AVG; - case AggregateColumn::MIN: - return ROWAGG_MIN; + case AggregateColumn::MIN: return ROWAGG_MIN; - case AggregateColumn::MAX: - return ROWAGG_MAX; + case AggregateColumn::MAX: return ROWAGG_MAX; - case AggregateColumn::DISTINCT_COUNT: - return ROWAGG_COUNT_DISTINCT_COL_NAME; + case AggregateColumn::DISTINCT_COUNT: return ROWAGG_COUNT_DISTINCT_COL_NAME; - case AggregateColumn::DISTINCT_SUM: - return ROWAGG_DISTINCT_SUM; + case AggregateColumn::DISTINCT_SUM: return ROWAGG_DISTINCT_SUM; - case AggregateColumn::DISTINCT_AVG: - return ROWAGG_DISTINCT_AVG; + case AggregateColumn::DISTINCT_AVG: return ROWAGG_DISTINCT_AVG; - case AggregateColumn::STDDEV_POP: - return ROWAGG_STATS; + case AggregateColumn::STDDEV_POP: return ROWAGG_STATS; - case AggregateColumn::STDDEV_SAMP: - return ROWAGG_STATS; + case AggregateColumn::STDDEV_SAMP: return ROWAGG_STATS; - case AggregateColumn::VAR_POP: - return ROWAGG_STATS; + case AggregateColumn::VAR_POP: return ROWAGG_STATS; - case AggregateColumn::VAR_SAMP: - return ROWAGG_STATS; + case AggregateColumn::VAR_SAMP: return ROWAGG_STATS; - case AggregateColumn::BIT_AND: - return ROWAGG_BIT_AND; + case AggregateColumn::BIT_AND: return ROWAGG_BIT_AND; - case AggregateColumn::BIT_OR: - return ROWAGG_BIT_OR; + case AggregateColumn::BIT_OR: return ROWAGG_BIT_OR; - case AggregateColumn::BIT_XOR: - return ROWAGG_BIT_XOR; + case AggregateColumn::BIT_XOR: return ROWAGG_BIT_XOR; - case AggregateColumn::GROUP_CONCAT: - return ROWAGG_GROUP_CONCAT; + case AggregateColumn::GROUP_CONCAT: return ROWAGG_GROUP_CONCAT; - case AggregateColumn::CONSTANT: - return ROWAGG_CONSTANT; + case AggregateColumn::CONSTANT: return ROWAGG_CONSTANT; - case AggregateColumn::UDAF: - return ROWAGG_UDAF; + case AggregateColumn::UDAF: return ROWAGG_UDAF; - case AggregateColumn::MULTI_PARM: - return ROWAGG_MULTI_PARM; + case AggregateColumn::MULTI_PARM: return ROWAGG_MULTI_PARM; - default: - return ROWAGG_FUNCT_UNDEFINE; - } + default: return ROWAGG_FUNCT_UNDEFINE; + } } - inline RowAggFunctionType statsFuncIdMap(int planFuncId) { - switch (planFuncId) - { - case AggregateColumn::STDDEV_POP: - return ROWAGG_STDDEV_POP; + switch (planFuncId) + { + case AggregateColumn::STDDEV_POP: return ROWAGG_STDDEV_POP; - case AggregateColumn::STDDEV_SAMP: - return ROWAGG_STDDEV_SAMP; + case AggregateColumn::STDDEV_SAMP: return ROWAGG_STDDEV_SAMP; - case AggregateColumn::VAR_POP: - return ROWAGG_VAR_POP; + case AggregateColumn::VAR_POP: return ROWAGG_VAR_POP; - case AggregateColumn::VAR_SAMP: - return ROWAGG_VAR_SAMP; + case AggregateColumn::VAR_SAMP: return ROWAGG_VAR_SAMP; - default: - return ROWAGG_FUNCT_UNDEFINE; - } + default: return ROWAGG_FUNCT_UNDEFINE; + } } - inline string colTypeIdString(CalpontSystemCatalog::ColDataType type) { - switch (type) - { - case CalpontSystemCatalog::BIT: - return string("BIT"); + switch (type) + { + case CalpontSystemCatalog::BIT: return string("BIT"); - case CalpontSystemCatalog::TINYINT: - return string("TINYINT"); + case CalpontSystemCatalog::TINYINT: return string("TINYINT"); - case CalpontSystemCatalog::CHAR: - return string("CHAR"); + case CalpontSystemCatalog::CHAR: return string("CHAR"); - case CalpontSystemCatalog::SMALLINT: - return string("SMALLINT"); + case CalpontSystemCatalog::SMALLINT: return string("SMALLINT"); - case CalpontSystemCatalog::DECIMAL: - return string("DECIMAL"); + case CalpontSystemCatalog::DECIMAL: return string("DECIMAL"); - case CalpontSystemCatalog::MEDINT: - return string("MEDINT"); + case CalpontSystemCatalog::MEDINT: return string("MEDINT"); - case CalpontSystemCatalog::INT: - return string("INT"); + case CalpontSystemCatalog::INT: return string("INT"); - case CalpontSystemCatalog::FLOAT: - return string("FLOAT"); + case CalpontSystemCatalog::FLOAT: return string("FLOAT"); - case CalpontSystemCatalog::DATE: - return string("DATE"); + case CalpontSystemCatalog::DATE: return string("DATE"); - case CalpontSystemCatalog::BIGINT: - return string("BIGINT"); + case CalpontSystemCatalog::BIGINT: return string("BIGINT"); - case CalpontSystemCatalog::DOUBLE: - return string("DOUBLE"); + case CalpontSystemCatalog::DOUBLE: return string("DOUBLE"); - case CalpontSystemCatalog::LONGDOUBLE: - return string("LONGDOUBLE"); + case CalpontSystemCatalog::LONGDOUBLE: return string("LONGDOUBLE"); - case CalpontSystemCatalog::DATETIME: - return string("DATETIME"); + case CalpontSystemCatalog::DATETIME: return string("DATETIME"); - case CalpontSystemCatalog::TIMESTAMP: - return string("TIMESTAMP"); + case CalpontSystemCatalog::TIMESTAMP: return string("TIMESTAMP"); - case CalpontSystemCatalog::TIME: - return string("TIME"); + case CalpontSystemCatalog::TIME: return string("TIME"); - case CalpontSystemCatalog::VARCHAR: - return string("VARCHAR"); + case CalpontSystemCatalog::VARCHAR: return string("VARCHAR"); - case CalpontSystemCatalog::CLOB: - return string("CLOB"); + case CalpontSystemCatalog::CLOB: return string("CLOB"); - case CalpontSystemCatalog::BLOB: - return string("BLOB"); + case CalpontSystemCatalog::BLOB: return string("BLOB"); - case CalpontSystemCatalog::TEXT: - return string("TEXT"); + case CalpontSystemCatalog::TEXT: return string("TEXT"); - case CalpontSystemCatalog::UTINYINT: - return string("UTINYINT"); + case CalpontSystemCatalog::UTINYINT: return string("UTINYINT"); - case CalpontSystemCatalog::USMALLINT: - return string("USMALLINT"); + case CalpontSystemCatalog::USMALLINT: return string("USMALLINT"); - case CalpontSystemCatalog::UDECIMAL: - return string("UDECIMAL"); + case CalpontSystemCatalog::UDECIMAL: return string("UDECIMAL"); - case CalpontSystemCatalog::UMEDINT: - return string("UMEDINT"); + case CalpontSystemCatalog::UMEDINT: return string("UMEDINT"); - case CalpontSystemCatalog::UINT: - return string("UINT"); + case CalpontSystemCatalog::UINT: return string("UINT"); - case CalpontSystemCatalog::UFLOAT: - return string("UFLOAT"); + case CalpontSystemCatalog::UFLOAT: return string("UFLOAT"); - case CalpontSystemCatalog::UBIGINT: - return string("UBIGINT"); + case CalpontSystemCatalog::UBIGINT: return string("UBIGINT"); - case CalpontSystemCatalog::UDOUBLE: - return string("UDOUBLE"); + case CalpontSystemCatalog::UDOUBLE: return string("UDOUBLE"); - default: - return string("UNKNOWN"); - } + default: return string("UNKNOWN"); + } } - string keyName(uint64_t i, uint32_t key, const joblist::JobInfo& jobInfo) { - string name = jobInfo.projectionCols[i]->alias(); + string name = jobInfo.projectionCols[i]->alias(); - if (name.empty()) - { - name = jobInfo.keyInfo->tupleKeyToName[key]; - - if (jobInfo.keyInfo->tupleKeyVec[key].fId < 100) - name = "Expression/Function"; - } - - return name = "'" + name + "'"; -} + if (name.empty()) + { + name = jobInfo.keyInfo->tupleKeyToName[key]; + if (jobInfo.keyInfo->tupleKeyVec[key].fId < 100) + name = "Expression/Function"; + } + return name = "'" + name + "'"; } +} // namespace namespace joblist { - -void wideDecimalOrLongDouble(const uint64_t colProj, - const CalpontSystemCatalog::ColDataType type, - const vector& precisionProj, - const vector& scaleProj, - const vector& width, - vector& typeAgg, - vector& scaleAgg, - vector& precisionAgg, - vector& widthAgg) +void wideDecimalOrLongDouble(const uint64_t colProj, const CalpontSystemCatalog::ColDataType type, + const vector& precisionProj, const vector& scaleProj, + const vector& width, + vector& typeAgg, vector& scaleAgg, + vector& precisionAgg, vector& widthAgg) { - if ((type == CalpontSystemCatalog::DECIMAL - || type == CalpontSystemCatalog::UDECIMAL) - && datatypes::Decimal::isWideDecimalTypeByPrecision(precisionProj[colProj])) - { - typeAgg.push_back(type); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - widthAgg.push_back(width[colProj]); - } - else if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(type)) - { - typeAgg.push_back(CalpontSystemCatalog::DECIMAL); - scaleAgg.push_back(0); - precisionAgg.push_back(datatypes::INT128MAXPRECISION); - widthAgg.push_back(datatypes::MAXDECIMALWIDTH); - } - else - { - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); - scaleAgg.push_back(0); - precisionAgg.push_back(-1); - widthAgg.push_back(sizeof(long double)); - } + if ((type == CalpontSystemCatalog::DECIMAL || type == CalpontSystemCatalog::UDECIMAL) && + datatypes::Decimal::isWideDecimalTypeByPrecision(precisionProj[colProj])) + { + typeAgg.push_back(type); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + widthAgg.push_back(width[colProj]); + } + else if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(type)) + { + typeAgg.push_back(CalpontSystemCatalog::DECIMAL); + scaleAgg.push_back(0); + precisionAgg.push_back(datatypes::INT128MAXPRECISION); + widthAgg.push_back(datatypes::MAXDECIMALWIDTH); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + widthAgg.push_back(sizeof(long double)); + } } -TupleAggregateStep::TupleAggregateStep( - const SP_ROWAGG_UM_t& agg, - const RowGroup& rgOut, - const RowGroup& rgIn, - const JobInfo& jobInfo) : - JobStep(jobInfo), - fCatalog(jobInfo.csc), - fRowsReturned(0), - fDoneAggregate(false), - fEndOfResult(false), - fAggregator(agg), - fRowGroupOut(rgOut), - fRowGroupIn(rgIn), - fRunner(0), - fUmOnly(false), - fRm(jobInfo.rm), - fBucketNum(0), - fInputIter(-1), - fSessionMemLimit(jobInfo.umMemLimit) +TupleAggregateStep::TupleAggregateStep(const SP_ROWAGG_UM_t& agg, const RowGroup& rgOut, const RowGroup& rgIn, + const JobInfo& jobInfo) + : JobStep(jobInfo) + , fCatalog(jobInfo.csc) + , fRowsReturned(0) + , fDoneAggregate(false) + , fEndOfResult(false) + , fAggregator(agg) + , fRowGroupOut(rgOut) + , fRowGroupIn(rgIn) + , fRunner(0) + , fUmOnly(false) + , fRm(jobInfo.rm) + , fBucketNum(0) + , fInputIter(-1) + , fSessionMemLimit(jobInfo.umMemLimit) { - fRowGroupData.reinit(fRowGroupOut); - fRowGroupOut.setData(&fRowGroupData); - fAggregator->setInputOutput(fRowGroupIn, &fRowGroupOut); + fRowGroupData.reinit(fRowGroupOut); + fRowGroupOut.setData(&fRowGroupData); + fAggregator->setInputOutput(fRowGroupIn, &fRowGroupOut); - // decide if this needs to be multi-threaded - RowAggregationDistinct* multiAgg = dynamic_cast(fAggregator.get()); - fIsMultiThread = (multiAgg || fAggregator->aggMapKeyLength() > 0); + // decide if this needs to be multi-threaded + RowAggregationDistinct* multiAgg = dynamic_cast(fAggregator.get()); + fIsMultiThread = (multiAgg || fAggregator->aggMapKeyLength() > 0); - // initialize multi-thread variables - fNumOfThreads = fRm->aggNumThreads(); - fNumOfBuckets = fRm->aggNumBuckets(); - fNumOfRowGroups = fRm->aggNumRowGroups(); + // initialize multi-thread variables + fNumOfThreads = fRm->aggNumThreads(); + fNumOfBuckets = fRm->aggNumBuckets(); + fNumOfRowGroups = fRm->aggNumRowGroups(); - auto memLimit = std::min(fRm->availableMemory(), *fSessionMemLimit); - fNumOfBuckets = calcNumberOfBuckets(memLimit, - fNumOfThreads, - fNumOfBuckets, - fNumOfRowGroups, - fRowGroupIn.getRowSize(), - fRowGroupOut.getRowSize(), - fRm->getAllowDiskAggregation()); - fNumOfThreads = std::min(fNumOfThreads, fNumOfBuckets); + auto memLimit = std::min(fRm->availableMemory(), *fSessionMemLimit); + fNumOfBuckets = + calcNumberOfBuckets(memLimit, fNumOfThreads, fNumOfBuckets, fNumOfRowGroups, fRowGroupIn.getRowSize(), + fRowGroupOut.getRowSize(), fRm->getAllowDiskAggregation()); + fNumOfThreads = std::min(fNumOfThreads, fNumOfBuckets); - fMemUsage.reset(new uint64_t[fNumOfThreads]); - memset(fMemUsage.get(), 0, fNumOfThreads * sizeof(uint64_t)); + fMemUsage.reset(new uint64_t[fNumOfThreads]); + memset(fMemUsage.get(), 0, fNumOfThreads * sizeof(uint64_t)); - fExtendedInfo = "TAS: "; - fQtc.stepParms().stepType = StepTeleStats::T_TAS; + fExtendedInfo = "TAS: "; + fQtc.stepParms().stepType = StepTeleStats::T_TAS; } - TupleAggregateStep::~TupleAggregateStep() { - for (uint32_t i = 0; i < fNumOfThreads; i++) - fRm->returnMemory(fMemUsage[i], fSessionMemLimit); + for (uint32_t i = 0; i < fNumOfThreads; i++) + fRm->returnMemory(fMemUsage[i], fSessionMemLimit); - for (uint32_t i = 0; i < fAgg_mutex.size(); i++) - delete fAgg_mutex[i]; + for (uint32_t i = 0; i < fAgg_mutex.size(); i++) + delete fAgg_mutex[i]; } - void TupleAggregateStep::initializeMultiThread() { - RowGroupDL* dlIn = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - uint32_t i; + RowGroupDL* dlIn = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + uint32_t i; - if (dlIn == NULL) - throw logic_error("Input is not RowGroup data list in delivery step."); + if (dlIn == NULL) + throw logic_error("Input is not RowGroup data list in delivery step."); - if (fInputIter < 0) - fInputIter = dlIn->getIterator(); + if (fInputIter < 0) + fInputIter = dlIn->getIterator(); - fRowGroupIns.resize(fNumOfThreads); - fRowGroupOuts.resize(fNumOfBuckets); - fRowGroupDatas.resize(fNumOfBuckets); + fRowGroupIns.resize(fNumOfThreads); + fRowGroupOuts.resize(fNumOfBuckets); + fRowGroupDatas.resize(fNumOfBuckets); - rowgroup::SP_ROWAGG_UM_t agg; - RGData rgData; + rowgroup::SP_ROWAGG_UM_t agg; + RGData rgData; - for (i = 0; i < fNumOfBuckets; i++) - { - boost::mutex* lock = new boost::mutex(); - fAgg_mutex.push_back(lock); - fRowGroupOuts[i] = fRowGroupOut; - rgData.reinit(fRowGroupOut); - fRowGroupDatas[i] = rgData; - fRowGroupOuts[i].setData(&fRowGroupDatas[i]); - fRowGroupOuts[i].resetRowGroup(0); - } + for (i = 0; i < fNumOfBuckets; i++) + { + boost::mutex* lock = new boost::mutex(); + fAgg_mutex.push_back(lock); + fRowGroupOuts[i] = fRowGroupOut; + rgData.reinit(fRowGroupOut); + fRowGroupDatas[i] = rgData; + fRowGroupOuts[i].setData(&fRowGroupDatas[i]); + fRowGroupOuts[i].resetRowGroup(0); + } } - void TupleAggregateStep::run() { - if (fDelivery == false) - { - fRunner = jobstepThreadPool.invoke(Aggregator(this)); - } + if (fDelivery == false) + { + fRunner = jobstepThreadPool.invoke(Aggregator(this)); + } } - void TupleAggregateStep::join() { - if (fRunner) - jobstepThreadPool.join(fRunner); + if (fRunner) + jobstepThreadPool.join(fRunner); } - void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID) { - if (threadID >= fNumOfBuckets) - return; + if (threadID >= fNumOfBuckets) + return; - scoped_array rowBucketVecs(new RowBucketVec[fNumOfBuckets]); - scoped_array bucketDone(new bool[fNumOfBuckets]); - uint32_t hashlen = fAggregator->aggMapKeyLength(); + scoped_array rowBucketVecs(new RowBucketVec[fNumOfBuckets]); + scoped_array bucketDone(new bool[fNumOfBuckets]); + uint32_t hashlen = fAggregator->aggMapKeyLength(); - try + try + { + RowAggregationDistinct* aggDist = dynamic_cast(fAggregators[threadID].get()); + RowAggregationMultiDistinct* multiDist = + dynamic_cast(fAggregators[threadID].get()); + Row rowIn; + RowGroup* rowGroupIn = nullptr; + rowGroupIn = (aggDist->aggregator()->getOutputRowGroup()); + uint32_t bucketID; + std::vector> rgDataVec; + + if (multiDist) { - RowAggregationDistinct* aggDist = dynamic_cast(fAggregators[threadID].get()); - RowAggregationMultiDistinct* multiDist = dynamic_cast(fAggregators[threadID].get()); - Row rowIn; - RowGroup* rowGroupIn = nullptr; - rowGroupIn = (aggDist->aggregator()->getOutputRowGroup()); - uint32_t bucketID; - std::vector> rgDataVec; - - if (multiDist) - { - for (uint32_t i = 0; i < fNumOfBuckets; i++) - rowBucketVecs[i].resize(multiDist->subAggregators().size()); - } - else - { - for (uint32_t i = 0; i < fNumOfBuckets; i++) - rowBucketVecs[i].resize(1); - } - - // dispatch rows to bucket - if (multiDist) - { - for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) - { - rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); - rowGroupIn->initRow(&rowIn); - auto* subDistAgg = dynamic_cast(multiDist->subAggregators()[j].get()); - - while (subDistAgg->nextRowGroup()) - { - rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); - rgDataVec.emplace_back(subDistAgg->moveCurrentRGData()); - rowGroupIn->getRow(0, &rowIn); - - for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) - { - // The key is the groupby columns, which are the leading columns. - //uint8_t* hashMapKey = rowIn.getData() + 2; - //bucketID = hash.operator()(hashMapKey) & fBucketMask; - uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); - bucketID = hash % fNumOfBuckets; - rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); - rowIn.nextRow(); - } - } - } - } - else - { - rowGroupIn->initRow(&rowIn); - auto* subAgg = dynamic_cast(aggDist->aggregator().get()); - - while (subAgg->nextRowGroup()) - { - rowGroupIn->setData(aggDist->aggregator()->getOutputRowGroup()->getRGData()); - rgDataVec.emplace_back(subAgg->moveCurrentRGData()); - rowGroupIn->getRow(0, &rowIn); - - for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) - { - // The key is the groupby columns, which are the leading columns. - //uint8_t* hashMapKey = rowIn.getData() + 2; - //bucketID = hash.operator()(hashMapKey) & fBucketMask; - uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); - bucketID = hash % fNumOfBuckets; - rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); - rowIn.nextRow(); - } - } - } - - bool done = false; - - // reset bucketDone[] to be false - //memset(bucketDone, 0, sizeof(bucketDone)); - fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); - - while (!done && !cancelled()) - { - done = true; - - for (uint32_t c = 0; c < fNumOfBuckets && !cancelled(); c++) - { - if (!bucketDone[c] && fAgg_mutex[c]->try_lock()) - { - try - { - if (multiDist) - dynamic_cast(fAggregators[c].get())->doDistinctAggregation_rowVec(rowBucketVecs[c]); - else - dynamic_cast(fAggregators[c].get())->doDistinctAggregation_rowVec(rowBucketVecs[c][0]); - } - catch (...) - { - fAgg_mutex[c]->unlock(); - throw; - } - - fAgg_mutex[c]->unlock(); - bucketDone[c] = true; - rowBucketVecs[c][0].clear(); - } - else if (!bucketDone[c]) - { - done = false; - } - } - } - - if (cancelled()) - { - fEndOfResult = true; - } - - } // try - catch (...) - { - handleException(std::current_exception(), - logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, - "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); - fEndOfResult = true; - } - - - fDoneAggregate = true; - - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - } -} - - -uint32_t TupleAggregateStep::nextBand_singleThread(messageqcpp::ByteStream& bs) -{ - uint32_t rowCount = 0; - - try - { - if (!fDoneAggregate) - aggregateRowGroups(); - - if (fEndOfResult == false) - { - bs.restart(); - - // do the final aggregtion and deliver the results - // at least one RowGroup for aggregate results - if (dynamic_cast(fAggregator.get()) != NULL) - { - dynamic_cast(fAggregator.get())->doDistinctAggregation(); - } - - if (fAggregator->nextRowGroup()) - { - fAggregator->finalize(); - rowCount = fRowGroupOut.getRowCount(); - fRowsReturned += rowCount; - fRowGroupDelivered.setData(fRowGroupOut.getRGData()); - - if (fRowGroupOut.getColumnCount() != fRowGroupDelivered.getColumnCount()) - pruneAuxColumns(); - - fRowGroupDelivered.serializeRGData(bs); - } - else - { - fEndOfResult = true; - } - } - } // try - catch (...) - { - handleException(std::current_exception(), - logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, - "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); - fEndOfResult = true; - } - - if (fEndOfResult) - { - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - // send an empty / error band - RGData rgData(fRowGroupOut, 0); - fRowGroupOut.setData(&rgData); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setStatus(status()); - fRowGroupOut.serializeRGData(bs); - rowCount = 0; - - if (traceOn()) - printCalTrace(); - } - - return rowCount; -} - - -bool TupleAggregateStep::nextDeliveredRowGroup() -{ - for (; fBucketNum < fNumOfBuckets; fBucketNum++) - { - while (fAggregators[fBucketNum]->nextRowGroup()) - { - fAggregators[fBucketNum]->finalize(); - fRowGroupDelivered.setData(fAggregators[fBucketNum]->getOutputRowGroup()->getRGData()); - fRowGroupOut.setData(fAggregators[fBucketNum]->getOutputRowGroup()->getRGData()); - return true; - } - } - - fBucketNum = 0; - return false; -} - - -uint32_t TupleAggregateStep::nextBand(messageqcpp::ByteStream& bs) -{ - // use the orignal single thread model when no group by and distnct. - // @bug4314. DO NOT access fAggregtor before the first read of input, - // because hashjoin may not have finalized fAggregator. - if (!fIsMultiThread) - return nextBand_singleThread(bs); - - return doThreadedAggregate(bs, 0); -} - - -bool TupleAggregateStep::setPmHJAggregation(JobStep* step) -{ - TupleBPS* bps = dynamic_cast(step); - - if (bps != NULL) - { - fAggregatorUM->expression(fAggregator->expression()); - fAggregatorUM->constantAggregate(fAggregator->constantAggregate()); - fAggregator = fAggregatorUM; - fRowGroupIn = fRowGroupPMHJ; - fAggregator->setInputOutput(fRowGroupIn, &fRowGroupOut); - bps->setAggregateStep(fAggregatorPM, fRowGroupPMHJ); - } - - return (bps != NULL); -} - - -void TupleAggregateStep::configDeliveredRowGroup(const JobInfo& jobInfo) -{ - // configure the oids and keys - vector oids = fRowGroupOut.getOIDs(); - vector keys = fRowGroupOut.getKeys(); - vector >::const_iterator begin = jobInfo.aggEidIndexList.begin(); - vector >::const_iterator end = jobInfo.aggEidIndexList.end(); - - for (vector >::const_iterator i = begin; i != end; i++) - { - oids[i->second] = i->first; - keys[i->second] = getExpTupleKey(jobInfo, i->first); - } - - // correct the scale - vector scale = fRowGroupOut.getScale(); - vector precision = fRowGroupOut.getPrecision(); - - size_t retColCount = 0; - auto scaleIter = scale.begin(); - auto precisionIter = precision.begin(); - - if (jobInfo.havingStep) - { - retColCount = jobInfo.returnedColVec.size(); - - idbassert(jobInfo.returnedColVec.size() == jobInfo.projectionCols.size()); - - for (size_t i = 0; i < jobInfo.projectionCols.size() && - scaleIter != scale.end(); i++) - { - const auto& colType = jobInfo.projectionCols[i]->resultType(); - - if (colType.isWideDecimalType()) - { - *scaleIter = colType.scale; - *precisionIter = colType.precision; - } - - scaleIter++; precisionIter++; - } + for (uint32_t i = 0; i < fNumOfBuckets; i++) + rowBucketVecs[i].resize(multiDist->subAggregators().size()); } else { - retColCount = jobInfo.nonConstDelCols.size(); - - for (size_t i = 0; i < jobInfo.nonConstDelCols.size() && - scaleIter != scale.end(); i++) - { - const auto& colType = jobInfo.nonConstDelCols[i]->resultType(); - - if (colType.isWideDecimalType()) - { - *scaleIter = colType.scale; - *precisionIter = colType.precision; - } - - scaleIter++; precisionIter++; - } + for (uint32_t i = 0; i < fNumOfBuckets; i++) + rowBucketVecs[i].resize(1); } - vector::const_iterator offsets0 = fRowGroupOut.getOffsets().begin(); - vector::const_iterator types0 = - fRowGroupOut.getColTypes().begin(); - vector csNums = fRowGroupOut.getCharsetNumbers(); - vector::const_iterator precision0 = precision.begin(); - fRowGroupDelivered = RowGroup(retColCount, - vector(offsets0, offsets0 + retColCount + 1), - vector(oids.begin(), oids.begin() + retColCount), - vector(keys.begin(), keys.begin() + retColCount), - vector(types0, types0 + retColCount), - vector(csNums.begin(), csNums.begin() + retColCount), - vector(scale.begin(), scale.begin() + retColCount), - vector(precision0, precision0 + retColCount), - jobInfo.stringTableThreshold); + // dispatch rows to bucket + if (multiDist) + { + for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) + { + rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); + rowGroupIn->initRow(&rowIn); + auto* subDistAgg = dynamic_cast(multiDist->subAggregators()[j].get()); - if (jobInfo.trace) - cout << "delivered RG: " << fRowGroupDelivered.toString() << endl << endl; + while (subDistAgg->nextRowGroup()) + { + rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); + rgDataVec.emplace_back(subDistAgg->moveCurrentRGData()); + rowGroupIn->getRow(0, &rowIn); + for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) + { + // The key is the groupby columns, which are the leading columns. + // uint8_t* hashMapKey = rowIn.getData() + 2; + // bucketID = hash.operator()(hashMapKey) & fBucketMask; + uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); + bucketID = hash % fNumOfBuckets; + rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); + rowIn.nextRow(); + } + } + } + } + else + { + rowGroupIn->initRow(&rowIn); + auto* subAgg = dynamic_cast(aggDist->aggregator().get()); + + while (subAgg->nextRowGroup()) + { + rowGroupIn->setData(aggDist->aggregator()->getOutputRowGroup()->getRGData()); + rgDataVec.emplace_back(subAgg->moveCurrentRGData()); + rowGroupIn->getRow(0, &rowIn); + + for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) + { + // The key is the groupby columns, which are the leading columns. + // uint8_t* hashMapKey = rowIn.getData() + 2; + // bucketID = hash.operator()(hashMapKey) & fBucketMask; + uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); + bucketID = hash % fNumOfBuckets; + rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); + rowIn.nextRow(); + } + } + } + + bool done = false; + + // reset bucketDone[] to be false + // memset(bucketDone, 0, sizeof(bucketDone)); + fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); + + while (!done && !cancelled()) + { + done = true; + + for (uint32_t c = 0; c < fNumOfBuckets && !cancelled(); c++) + { + if (!bucketDone[c] && fAgg_mutex[c]->try_lock()) + { + try + { + if (multiDist) + dynamic_cast(fAggregators[c].get()) + ->doDistinctAggregation_rowVec(rowBucketVecs[c]); + else + dynamic_cast(fAggregators[c].get()) + ->doDistinctAggregation_rowVec(rowBucketVecs[c][0]); + } + catch (...) + { + fAgg_mutex[c]->unlock(); + throw; + } + + fAgg_mutex[c]->unlock(); + bucketDone[c] = true; + rowBucketVecs[c][0].clear(); + } + else if (!bucketDone[c]) + { + done = false; + } + } + } + + if (cancelled()) + { + fEndOfResult = true; + } + + } // try + catch (...) + { + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); + fEndOfResult = true; + } + + fDoneAggregate = true; + + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + } } +uint32_t TupleAggregateStep::nextBand_singleThread(messageqcpp::ByteStream& bs) +{ + uint32_t rowCount = 0; + + try + { + if (!fDoneAggregate) + aggregateRowGroups(); + + if (fEndOfResult == false) + { + bs.restart(); + + // do the final aggregtion and deliver the results + // at least one RowGroup for aggregate results + if (dynamic_cast(fAggregator.get()) != NULL) + { + dynamic_cast(fAggregator.get())->doDistinctAggregation(); + } + + if (fAggregator->nextRowGroup()) + { + fAggregator->finalize(); + rowCount = fRowGroupOut.getRowCount(); + fRowsReturned += rowCount; + fRowGroupDelivered.setData(fRowGroupOut.getRGData()); + + if (fRowGroupOut.getColumnCount() != fRowGroupDelivered.getColumnCount()) + pruneAuxColumns(); + + fRowGroupDelivered.serializeRGData(bs); + } + else + { + fEndOfResult = true; + } + } + } // try + catch (...) + { + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); + fEndOfResult = true; + } + + if (fEndOfResult) + { + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + // send an empty / error band + RGData rgData(fRowGroupOut, 0); + fRowGroupOut.setData(&rgData); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setStatus(status()); + fRowGroupOut.serializeRGData(bs); + rowCount = 0; + + if (traceOn()) + printCalTrace(); + } + + return rowCount; +} + +bool TupleAggregateStep::nextDeliveredRowGroup() +{ + for (; fBucketNum < fNumOfBuckets; fBucketNum++) + { + while (fAggregators[fBucketNum]->nextRowGroup()) + { + fAggregators[fBucketNum]->finalize(); + fRowGroupDelivered.setData(fAggregators[fBucketNum]->getOutputRowGroup()->getRGData()); + fRowGroupOut.setData(fAggregators[fBucketNum]->getOutputRowGroup()->getRGData()); + return true; + } + } + + fBucketNum = 0; + return false; +} + +uint32_t TupleAggregateStep::nextBand(messageqcpp::ByteStream& bs) +{ + // use the orignal single thread model when no group by and distnct. + // @bug4314. DO NOT access fAggregtor before the first read of input, + // because hashjoin may not have finalized fAggregator. + if (!fIsMultiThread) + return nextBand_singleThread(bs); + + return doThreadedAggregate(bs, 0); +} + +bool TupleAggregateStep::setPmHJAggregation(JobStep* step) +{ + TupleBPS* bps = dynamic_cast(step); + + if (bps != NULL) + { + fAggregatorUM->expression(fAggregator->expression()); + fAggregatorUM->constantAggregate(fAggregator->constantAggregate()); + fAggregator = fAggregatorUM; + fRowGroupIn = fRowGroupPMHJ; + fAggregator->setInputOutput(fRowGroupIn, &fRowGroupOut); + bps->setAggregateStep(fAggregatorPM, fRowGroupPMHJ); + } + + return (bps != NULL); +} + +void TupleAggregateStep::configDeliveredRowGroup(const JobInfo& jobInfo) +{ + // configure the oids and keys + vector oids = fRowGroupOut.getOIDs(); + vector keys = fRowGroupOut.getKeys(); + vector>::const_iterator begin = jobInfo.aggEidIndexList.begin(); + vector>::const_iterator end = jobInfo.aggEidIndexList.end(); + + for (vector>::const_iterator i = begin; i != end; i++) + { + oids[i->second] = i->first; + keys[i->second] = getExpTupleKey(jobInfo, i->first); + } + + // correct the scale + vector scale = fRowGroupOut.getScale(); + vector precision = fRowGroupOut.getPrecision(); + + size_t retColCount = 0; + auto scaleIter = scale.begin(); + auto precisionIter = precision.begin(); + + if (jobInfo.havingStep) + { + retColCount = jobInfo.returnedColVec.size(); + + idbassert(jobInfo.returnedColVec.size() == jobInfo.projectionCols.size()); + + for (size_t i = 0; i < jobInfo.projectionCols.size() && scaleIter != scale.end(); i++) + { + const auto& colType = jobInfo.projectionCols[i]->resultType(); + + if (colType.isWideDecimalType()) + { + *scaleIter = colType.scale; + *precisionIter = colType.precision; + } + + scaleIter++; + precisionIter++; + } + } + else + { + retColCount = jobInfo.nonConstDelCols.size(); + + for (size_t i = 0; i < jobInfo.nonConstDelCols.size() && scaleIter != scale.end(); i++) + { + const auto& colType = jobInfo.nonConstDelCols[i]->resultType(); + + if (colType.isWideDecimalType()) + { + *scaleIter = colType.scale; + *precisionIter = colType.precision; + } + + scaleIter++; + precisionIter++; + } + } + + vector::const_iterator offsets0 = fRowGroupOut.getOffsets().begin(); + vector::const_iterator types0 = fRowGroupOut.getColTypes().begin(); + vector csNums = fRowGroupOut.getCharsetNumbers(); + vector::const_iterator precision0 = precision.begin(); + fRowGroupDelivered = + RowGroup(retColCount, vector(offsets0, offsets0 + retColCount + 1), + vector(oids.begin(), oids.begin() + retColCount), + vector(keys.begin(), keys.begin() + retColCount), + vector(types0, types0 + retColCount), + vector(csNums.begin(), csNums.begin() + retColCount), + vector(scale.begin(), scale.begin() + retColCount), + vector(precision0, precision0 + retColCount), jobInfo.stringTableThreshold); + + if (jobInfo.trace) + cout << "delivered RG: " << fRowGroupDelivered.toString() << endl << endl; +} void TupleAggregateStep::setOutputRowGroup(const RowGroup& rg) { - fRowGroupOut = rg; - fRowGroupData.reinit(fRowGroupOut); - fRowGroupOut.setData(&fRowGroupData); - fAggregator->setInputOutput(fRowGroupIn, &fRowGroupOut); + fRowGroupOut = rg; + fRowGroupData.reinit(fRowGroupOut); + fRowGroupOut.setData(&fRowGroupData); + fAggregator->setInputOutput(fRowGroupIn, &fRowGroupOut); } - const RowGroup& TupleAggregateStep::getOutputRowGroup() const { - return fRowGroupOut; + return fRowGroupOut; } - const RowGroup& TupleAggregateStep::getDeliveredRowGroup() const { - return fRowGroupDelivered; + return fRowGroupDelivered; } - void TupleAggregateStep::savePmHJData(SP_ROWAGG_t& um, SP_ROWAGG_t& pm, RowGroup& rg) { - fAggregatorUM = dynamic_pointer_cast(um); - fAggregatorPM = pm; - fRowGroupPMHJ = rg; + fAggregatorUM = dynamic_pointer_cast(um); + fAggregatorPM = pm; + fRowGroupPMHJ = rg; } - void TupleAggregateStep::deliverStringTableRowGroup(bool b) { - fRowGroupDelivered.setUseStringTable(b); + fRowGroupDelivered.setUseStringTable(b); } - bool TupleAggregateStep::deliverStringTableRowGroup() const { - return fRowGroupDelivered.usesStringTable(); + return fRowGroupDelivered.usesStringTable(); } - const string TupleAggregateStep::toString() const { - ostringstream oss; - oss << "AggregateStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "AggregateStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " in:"; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); - if (fOutputJobStepAssociation.outSize() > 0) - { - oss << " out:"; + if (fOutputJobStepAssociation.outSize() > 0) + { + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); - } + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); + } - return oss.str(); + return oss.str(); } SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo) { - SJSTEP spjs; - TupleDeliveryStep* tds = dynamic_cast(step.get()); - TupleBPS* tbps = dynamic_cast(step.get()); - TupleHashJoinStep* thjs = dynamic_cast(step.get()); - SubAdapterStep* sas = dynamic_cast(step.get()); - CrossEngineStep* ces = dynamic_cast(step.get()); - vector rgs; // 0-ProjRG, 1-UMRG, [2-PMRG -- if 2 phases] - vector aggs; - SP_ROWAGG_UM_t aggUM; - bool distinctAgg = false; - int64_t constKey = -1; - vector constAggDataVec; + SJSTEP spjs; + TupleDeliveryStep* tds = dynamic_cast(step.get()); + TupleBPS* tbps = dynamic_cast(step.get()); + TupleHashJoinStep* thjs = dynamic_cast(step.get()); + SubAdapterStep* sas = dynamic_cast(step.get()); + CrossEngineStep* ces = dynamic_cast(step.get()); + vector rgs; // 0-ProjRG, 1-UMRG, [2-PMRG -- if 2 phases] + vector aggs; + SP_ROWAGG_UM_t aggUM; + bool distinctAgg = false; + int64_t constKey = -1; + vector constAggDataVec; - vector > returnedColVecOrig = jobInfo.returnedColVec; + vector> returnedColVecOrig = jobInfo.returnedColVec; - for (uint32_t idx = 0; idx < jobInfo.returnedColVec.size(); idx++) + for (uint32_t idx = 0; idx < jobInfo.returnedColVec.size(); idx++) + { + if (jobInfo.returnedColVec[idx].second == AggregateColumn::DISTINCT_COUNT || + jobInfo.returnedColVec[idx].second == AggregateColumn::DISTINCT_AVG || + jobInfo.returnedColVec[idx].second == AggregateColumn::DISTINCT_SUM) { - if (jobInfo.returnedColVec[idx].second == AggregateColumn::DISTINCT_COUNT || - jobInfo.returnedColVec[idx].second == AggregateColumn::DISTINCT_AVG || - jobInfo.returnedColVec[idx].second == AggregateColumn::DISTINCT_SUM - ) + distinctAgg = true; + } + + // Change COUNT_ASTERISK to CONSTANT if necessary. + // In joblistfactory, all aggregate(constant) are set to count(*) for easy process. + map::iterator it = jobInfo.constAggregate.find(idx); + + if (it != jobInfo.constAggregate.end()) + { + AggregateColumn* ac = dynamic_cast(it->second.get()); + + if (ac->aggOp() == AggregateColumn::COUNT_ASTERISK) + { + if (jobInfo.cntStarPos == -1) + jobInfo.cntStarPos = idx; + } + else + { + constKey = jobInfo.returnedColVec[idx].first; + CalpontSystemCatalog::ColType ct = ac->resultType(); + TupleInfo ti = setExpTupleInfo(ct, ac->expressionId(), ac->alias(), jobInfo); + jobInfo.returnedColVec[idx].first = ti.key; + jobInfo.returnedColVec[idx].second = AggregateColumn::CONSTANT; + + ConstantColumn* cc = dynamic_cast(ac->constCol().get()); + idbassert(cc != NULL); // @bug5261 + bool isNull = (ConstantColumn::NULLDATA == cc->type()); + + if (ac->aggOp() == AggregateColumn::UDAF) { - distinctAgg = true; + UDAFColumn* udafc = dynamic_cast(ac); + if (udafc) + { + constAggDataVec.push_back(ConstantAggData(cc->constval(), udafc->getContext().getName(), + functionIdMap(ac->aggOp()), isNull)); + } } - - // Change COUNT_ASTERISK to CONSTANT if necessary. - // In joblistfactory, all aggregate(constant) are set to count(*) for easy process. - map::iterator it = jobInfo.constAggregate.find(idx); - - if (it != jobInfo.constAggregate.end()) + else { - AggregateColumn* ac = dynamic_cast(it->second.get()); - - if (ac->aggOp() == AggregateColumn::COUNT_ASTERISK) - { - if (jobInfo.cntStarPos == -1) - jobInfo.cntStarPos = idx; - } - else - { - constKey = jobInfo.returnedColVec[idx].first; - CalpontSystemCatalog::ColType ct = ac->resultType(); - TupleInfo ti = - setExpTupleInfo(ct, ac->expressionId(), ac->alias(), jobInfo); - jobInfo.returnedColVec[idx].first = ti.key; - jobInfo.returnedColVec[idx].second = AggregateColumn::CONSTANT; - - ConstantColumn* cc = dynamic_cast(ac->constCol().get()); - idbassert(cc != NULL); // @bug5261 - bool isNull = (ConstantColumn::NULLDATA == cc->type()); - - if (ac->aggOp() == AggregateColumn::UDAF) - { - UDAFColumn* udafc = dynamic_cast(ac); - if (udafc) - { - constAggDataVec.push_back( - ConstantAggData(cc->constval(), udafc->getContext().getName(), - functionIdMap(ac->aggOp()), isNull)); - } - } - else - { - constAggDataVec.push_back( - ConstantAggData(cc->constval(), functionIdMap(ac->aggOp()), isNull)); - } - } + constAggDataVec.push_back(ConstantAggData(cc->constval(), functionIdMap(ac->aggOp()), isNull)); } + } + } + } + + // If there are aggregate(constant) columns, but no count(*), add a count(*). + if (constAggDataVec.size() > 0 && jobInfo.cntStarPos < 0) + { + jobInfo.cntStarPos = jobInfo.returnedColVec.size(); + jobInfo.returnedColVec.push_back(make_pair(constKey, AggregateColumn::COUNT_ASTERISK)); + } + + // preprocess the columns used by group_concat + jobInfo.groupConcatInfo.prepGroupConcat(jobInfo); + bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0 || sas || ces; + + rgs.push_back(tds->getDeliveredRowGroup()); + + // get rowgroup and aggregator + // For TupleHashJoin, we prepare for both PM and UM only aggregation + if (doUMOnly || thjs) + { + if (distinctAgg == true) + prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); + else + prep1PhaseAggregate(jobInfo, rgs, aggs); + + // TODO: fix this + if (doUMOnly) + rgs.push_back(rgs[0]); + } + + if (!doUMOnly) + { + if (distinctAgg == true) + prep2PhasesDistinctAggregate(jobInfo, rgs, aggs); + else + prep2PhasesAggregate(jobInfo, rgs, aggs); + } + + if (tbps != NULL) + { + // create delivery step + aggUM = dynamic_pointer_cast(aggs[0]); + spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[2], jobInfo)); + + if (doUMOnly) + dynamic_cast(spjs.get())->umOnly(true); + else + tbps->setAggregateStep(aggs[1], rgs[2]); + } + else if (thjs != NULL) + { + // create delivery step + aggUM = dynamic_pointer_cast(aggs[0]); + spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); + + if (doUMOnly) + dynamic_cast(spjs.get())->umOnly(true); + else + dynamic_cast(spjs.get())->savePmHJData(aggs[1], aggs[2], rgs[3]); + + // set input side + thjs->deliveryStep(spjs); + } + else + { + aggUM = dynamic_pointer_cast(aggs[0]); + spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); + } + + // Setup the input JobstepAssoctiation -- the mechanism + // whereby the previous step feeds data to this step. + // Otherwise, we need to create one and hook to the + // previous step as well as this aggregate step. + spjs->stepId(step->stepId() + 1); + + JobStepAssociation jsa; + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + dl->OID(execplan::CNX_VTABLE_ID); + spdl->rowGroupDL(dl); + jsa.outAdd(spdl); + + spjs->inputAssociation(jsa); // Aggregate input + + // Previous step output + step->outputAssociation(jsa); + + // add the aggregate on constants + if (constAggDataVec.size() > 0) + { + dynamic_cast(spjs.get())->addConstangAggregate(constAggDataVec); + jobInfo.returnedColVec.swap(returnedColVecOrig); // restore the original return columns + } + + // fix the delivered rowgroup data + dynamic_cast(spjs.get())->configDeliveredRowGroup(jobInfo); + + if (jobInfo.expressionVec.size() > 0) + dynamic_cast(spjs.get())->prepExpressionOnAggregate(aggUM, jobInfo); + + return spjs; +} + +void TupleAggregateStep::prep1PhaseAggregate(JobInfo& jobInfo, vector& rowgroups, + vector& aggregators) +{ + // check if there are any aggregate columns + vector> aggColVec; + vector>& returnedColVec = jobInfo.returnedColVec; + + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + if (returnedColVec[i].second != 0) + aggColVec.push_back(returnedColVec[i]); + } + + // populate the aggregate rowgroup: projectedRG -> aggregateRG + // + // Aggregate preparation by joblist factory: + // 1. get projected rowgroup (done by doAggProject) -- passed in + // 2. construct aggregate rowgroup -- output of UM + const RowGroup projRG = rowgroups[0]; + const vector& oidsProj = projRG.getOIDs(); + const vector& keysProj = projRG.getKeys(); + const vector& scaleProj = projRG.getScale(); + const vector& precisionProj = projRG.getPrecision(); + const vector& typeProj = projRG.getColTypes(); + const vector& csNumProj = projRG.getCharsetNumbers(); + + vector posAgg; + vector oidsAgg; + vector keysAgg; + vector scaleAgg; + vector precisionAgg; + vector typeAgg; + vector csNumAgg; + vector widthAgg; + vector groupBy; + vector functionVec; + uint32_t bigIntWidth = sizeof(int64_t); + uint32_t bigUintWidth = sizeof(uint64_t); + // For UDAF + uint32_t projColsUDAFIdx = 0; + UDAFColumn* udafc = NULL; + mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; + // for count column of average function + map avgFuncMap; + + // collect the projected column info, prepare for aggregation + vector width; + map projColPosMap; + + for (uint64_t i = 0; i < keysProj.size(); i++) + { + projColPosMap.insert(make_pair(keysProj[i], i)); + width.push_back(projRG.getColumnWidth(i)); + } + + // for groupby column + map groupbyMap; + + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + int64_t colProj = projColPosMap[jobInfo.groupByColVec[i]]; + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, -1)); + groupBy.push_back(groupby); + groupbyMap.insert(make_pair(jobInfo.groupByColVec[i], i)); + } + + // for distinct column + for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) + { + //@bug6126, continue if already in group by + if (groupbyMap.find(jobInfo.distinctColVec[i]) != groupbyMap.end()) + continue; + + int64_t colProj = projColPosMap[jobInfo.distinctColVec[i]]; + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, -1)); + groupBy.push_back(groupby); + groupbyMap.insert(make_pair(jobInfo.distinctColVec[i], i)); + } + + // populate the aggregate rowgroup + AGG_MAP aggFuncMap; + uint64_t outIdx = 0; + + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); + RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); + uint32_t key = returnedColVec[i].first; + + if (aggOp == ROWAGG_CONSTANT) + { + TupleInfo ti = getTupleInfo(key, jobInfo); + oidsAgg.push_back(ti.oid); + keysAgg.push_back(key); + scaleAgg.push_back(ti.scale); + precisionAgg.push_back(ti.precision); + typeAgg.push_back(ti.dtype); + csNumAgg.push_back(ti.csNum); + widthAgg.push_back(ti.width); + SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol(aggOp, stats, 0, outIdx, jobInfo.cntStarPos)); + functionVec.push_back(funct); + ++outIdx; + continue; } - // If there are aggregate(constant) columns, but no count(*), add a count(*). - if (constAggDataVec.size() > 0 && jobInfo.cntStarPos < 0) + if (aggOp == ROWAGG_GROUP_CONCAT) { - jobInfo.cntStarPos = jobInfo.returnedColVec.size(); - jobInfo.returnedColVec.push_back(make_pair(constKey, AggregateColumn::COUNT_ASTERISK)); + TupleInfo ti = getTupleInfo(key, jobInfo); + uint32_t ptrSize = sizeof(GroupConcatAg*); + uint32_t width = (ti.width >= ptrSize) ? ti.width : ptrSize; + oidsAgg.push_back(ti.oid); + keysAgg.push_back(key); + scaleAgg.push_back(ti.scale); + precisionAgg.push_back(ti.precision); + typeAgg.push_back(ti.dtype); + csNumAgg.push_back(ti.csNum); + widthAgg.push_back(width); + SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol(aggOp, stats, 0, outIdx, -1)); + functionVec.push_back(funct); + + ++outIdx; + continue; } - // preprocess the columns used by group_concat - jobInfo.groupConcatInfo.prepGroupConcat(jobInfo); - bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0 - || sas - || ces; - - rgs.push_back(tds->getDeliveredRowGroup()); - - // get rowgroup and aggregator - // For TupleHashJoin, we prepare for both PM and UM only aggregation - if (doUMOnly || thjs) + if (projColPosMap.find(key) == projColPosMap.end()) { - if (distinctAgg == true) - prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep1PhaseAggregate: " << emsg.str() << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + // make sure the colProj is correct + int64_t colProj = projColPosMap[key]; + + if (keysProj[colProj] != key) + { + ostringstream emsg; + emsg << "projection column map is out of sync."; + cerr << "prep1PhaseAggregate: " << emsg.str() << endl; + throw logic_error(emsg.str()); + } + + if (aggOp == ROWAGG_FUNCT_UNDEFINE) + { + // must be a groupby column or function on aggregation + // or used by group_concat + map::iterator it = groupbyMap.find(key); + + if (it != groupbyMap.end()) + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(width[colProj]); + + if (groupBy[it->second]->fOutputColumnIndex == (uint32_t)-1) + groupBy[it->second]->fOutputColumnIndex = outIdx; else - prep1PhaseAggregate(jobInfo, rgs, aggs); + functionVec.push_back(SP_ROWAGG_FUNC_t(new RowAggFunctionCol( + ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, groupBy[it->second]->fOutputColumnIndex))); - // TODO: fix this - if (doUMOnly) - rgs.push_back(rgs[0]); + ++outIdx; + continue; + } + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) != + jobInfo.expressionVec.end()) + { + TupleInfo ti = getTupleInfo(key, jobInfo); + oidsAgg.push_back(ti.oid); + keysAgg.push_back(key); + scaleAgg.push_back(ti.scale); + precisionAgg.push_back(ti.precision); + typeAgg.push_back(ti.dtype); + csNumAgg.push_back(ti.csNum); + widthAgg.push_back(ti.width); + ++outIdx; + continue; + } + else if (jobInfo.groupConcatInfo.columns().find(key) != jobInfo.groupConcatInfo.columns().end()) + { + // TODO: columns only for group_concat do not needed in result set. + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(width[colProj]); + ++outIdx; + continue; + } + else if (jobInfo.windowSet.find(key) != jobInfo.windowSet.end()) + { + // skip window columns/expression, which are computed later + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(width[colProj]); + ++outIdx; + continue; + } + else + { + Message::Args args; + args.add(keyName(i, key, jobInfo)); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + cerr << "prep1PhaseAggregate: " << emsg << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable + << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView << ", function=" << (int)aggOp << endl; + throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); + } } - if (!doUMOnly) + SP_ROWAGG_FUNC_t funct; + + if (aggOp == ROWAGG_UDAF) { - if (distinctAgg == true) - prep2PhasesDistinctAggregate(jobInfo, rgs, aggs); - else - prep2PhasesAggregate(jobInfo, rgs, aggs); - } - - if (tbps != NULL) - { - // create delivery step - aggUM = dynamic_pointer_cast(aggs[0]); - spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[2], jobInfo)); - - if (doUMOnly) - dynamic_cast(spjs.get())->umOnly(true); - else - tbps->setAggregateStep(aggs[1], rgs[2]); - } - else if (thjs != NULL) - { - // create delivery step - aggUM = dynamic_pointer_cast(aggs[0]); - spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); - - if (doUMOnly) - dynamic_cast(spjs.get())->umOnly(true); - else - dynamic_cast(spjs.get())->savePmHJData(aggs[1], aggs[2], rgs[3]); - - // set input side - thjs->deliveryStep(spjs); + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) + { + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) + { + for (uint64_t k = i + 1; + k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); + } + } + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, outIdx)); + break; + } + } + if (it == jobInfo.projectionCols.end()) + { + throw logic_error( + "(1)prep1PhaseAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); + } } else { - aggUM = dynamic_pointer_cast(aggs[0]); - spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); + funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, outIdx)); } - // Setup the input JobstepAssoctiation -- the mechanism - // whereby the previous step feeds data to this step. - // Otherwise, we need to create one and hook to the - // previous step as well as this aggregate step. - spjs->stepId(step->stepId() + 1); + functionVec.push_back(funct); - JobStepAssociation jsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(execplan::CNX_VTABLE_ID); - spdl->rowGroupDL(dl); - jsa.outAdd(spdl); - - spjs->inputAssociation(jsa); // Aggregate input - - //Previous step output - step->outputAssociation(jsa); - - // add the aggregate on constants - if (constAggDataVec.size() > 0) + switch (aggOp) { - dynamic_cast(spjs.get())->addConstangAggregate(constAggDataVec); - jobInfo.returnedColVec.swap(returnedColVecOrig); // restore the original return columns + case ROWAGG_MIN: + case ROWAGG_MAX: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(width[colProj]); + } + break; + + case ROWAGG_AVG: + avgFuncMap.insert(make_pair(key, funct)); + /* fall through */ + case ROWAGG_SUM: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("sum/average"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep1PhaseAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + wideDecimalOrLongDouble(colProj, typeProj[colProj], precisionProj, scaleProj, width, typeAgg, + scaleAgg, precisionAgg, widthAgg); + + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + csNumAgg.push_back(csNumProj[colProj]); + } + break; + + case ROWAGG_COUNT_COL_NAME: + case ROWAGG_COUNT_ASTERISK: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(0); + // work around count() in select subquery + precisionAgg.push_back(9999); + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(bigIntWidth); + } + break; + + case ROWAGG_STATS: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("variance/standard deviation"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep1PhaseAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(0); + typeAgg.push_back(CalpontSystemCatalog::DOUBLE); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(sizeof(double)); + } + break; + + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(0); + precisionAgg.push_back(-16); // for connector to skip null check + + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } + + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(bigIntWidth); + } + break; + + case ROWAGG_UDAF: + { + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); + + if (!udafFuncCol) + { + throw logic_error( + "(2)prep1PhaseAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + // Return column + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(udafFuncCol->fUDAFContext.getScale()); + precisionAgg.push_back(udafFuncCol->fUDAFContext.getPrecision()); + typeAgg.push_back(udafFuncCol->fUDAFContext.getResultType()); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(udafFuncCol->fUDAFContext.getColWidth()); + break; + } + + case ROWAGG_MULTI_PARM: + { + } + break; + + default: + { + ostringstream emsg; + emsg << "aggregate function (" << (uint64_t)aggOp << ") isn't supported"; + cerr << "prep1PhaseAggregate: " << emsg.str() << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); + } } - // fix the delivered rowgroup data - dynamic_cast(spjs.get())->configDeliveredRowGroup(jobInfo); + // find if this func is a duplicate + AGG_MAP::iterator iter = aggFuncMap.find( + boost::make_tuple(key, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - if (jobInfo.expressionVec.size() > 0) - dynamic_cast(spjs.get())->prepExpressionOnAggregate(aggUM, jobInfo); + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && iter != aggFuncMap.end()) + { + if (funct->fAggFunction == ROWAGG_AVG) + funct->fAggFunction = ROWAGG_DUP_AVG; + else if (funct->fAggFunction == ROWAGG_STATS) + funct->fAggFunction = ROWAGG_DUP_STATS; + else if (funct->fAggFunction == ROWAGG_UDAF) + funct->fAggFunction = ROWAGG_DUP_UDAF; + else + funct->fAggFunction = ROWAGG_DUP_FUNCT; - return spjs; + funct->fAuxColumnIndex = iter->second; + } + else + { + aggFuncMap.insert(make_pair( + boost::make_tuple(key, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + funct->fOutputColumnIndex)); + } + + if (aggOp != ROWAGG_MULTI_PARM) + { + ++outIdx; + } + } + + // now fix the AVG function, locate the count(column) position + for (uint64_t i = 0; i < functionVec.size(); i++) + { + if (functionVec[i]->fAggFunction != ROWAGG_COUNT_COL_NAME) + continue; + + // if the count(k) can be associated with an avg(k) + map::iterator k = + avgFuncMap.find(keysAgg[functionVec[i]->fOutputColumnIndex]); + + if (k != avgFuncMap.end()) + { + k->second->fAuxColumnIndex = functionVec[i]->fOutputColumnIndex; + functionVec[i]->fAggFunction = ROWAGG_COUNT_NO_OP; + } + } + + // there is avg(k), but no count(k) in the select list + uint64_t lastCol = outIdx; + + for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) + { + if (k->second->fAuxColumnIndex == (uint32_t)-1) + { + k->second->fAuxColumnIndex = lastCol++; + oidsAgg.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); + keysAgg.push_back(k->first); + scaleAgg.push_back(0); + precisionAgg.push_back(19); + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + widthAgg.push_back(bigIntWidth); + } + } + + // add auxiliary fields for UDAF and statistics functions + for (uint64_t i = 0; i < functionVec.size(); i++) + { + uint64_t j = functionVec[i]->fInputColumnIndex; + + if (functionVec[i]->fAggFunction == ROWAGG_UDAF) + { + // Column for index of UDAF UserData struct + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVec[i].get()); + + if (!udafFuncCol) + { + throw logic_error( + "(3)prep1PhaseAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + functionVec[i]->fAuxColumnIndex = lastCol++; + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); + scaleAgg.push_back(0); + precisionAgg.push_back(0); + precisionAgg.push_back(0); + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + csNumAgg.push_back(8); + widthAgg.push_back(bigUintWidth); + continue; + } + + if (functionVec[i]->fAggFunction != ROWAGG_STATS) + continue; + + functionVec[i]->fAuxColumnIndex = lastCol; + + // sum(x) + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAgg.push_back(8); + widthAgg.push_back(sizeof(long double)); + ++lastCol; + + // sum(x**2) + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAgg.push_back(8); + widthAgg.push_back(sizeof(long double)); + ++lastCol; + } + + // calculate the offset and create the rowaggregation, rowgroup + posAgg.push_back(2); + + for (uint64_t i = 0; i < oidsAgg.size(); i++) + posAgg.push_back(posAgg[i] + widthAgg[i]); + + RowGroup aggRG(oidsAgg.size(), posAgg, oidsAgg, keysAgg, typeAgg, csNumAgg, scaleAgg, precisionAgg, + jobInfo.stringTableThreshold); + SP_ROWAGG_UM_t rowAgg(new RowAggregationUM(groupBy, functionVec, jobInfo.rm, jobInfo.umMemLimit)); + rowAgg->timeZone(jobInfo.timeZone); + rowgroups.push_back(aggRG); + aggregators.push_back(rowAgg); + + // mapping the group_concat columns, if any. + if (jobInfo.groupConcatInfo.groupConcat().size() > 0) + { + jobInfo.groupConcatInfo.mapColumns(projRG); + rowAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); + } + + if (jobInfo.trace) + cout << "\n====== Aggregation RowGroups ======" << endl + << "projected RG: " << projRG.toString() << endl + << "aggregated RG: " << aggRG.toString() << endl; } - -void TupleAggregateStep::prep1PhaseAggregate( - JobInfo& jobInfo, vector& rowgroups, vector& aggregators) +void TupleAggregateStep::prep1PhaseDistinctAggregate(JobInfo& jobInfo, vector& rowgroups, + vector& aggregators) { - // check if there are any aggregate columns - vector > aggColVec; - vector >& returnedColVec = jobInfo.returnedColVec; + // check if there are any aggregate columns + vector> aggColVec; + vector>& returnedColVec = jobInfo.returnedColVec; - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - if (returnedColVec[i].second != 0) - aggColVec.push_back(returnedColVec[i]); - } + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + if (returnedColVec[i].second != 0) + aggColVec.push_back(returnedColVec[i]); + } - // populate the aggregate rowgroup: projectedRG -> aggregateRG - // - // Aggregate preparation by joblist factory: - // 1. get projected rowgroup (done by doAggProject) -- passed in - // 2. construct aggregate rowgroup -- output of UM - const RowGroup projRG = rowgroups[0]; - const vector& oidsProj = projRG.getOIDs(); - const vector& keysProj = projRG.getKeys(); - const vector& scaleProj = projRG.getScale(); - const vector& precisionProj = projRG.getPrecision(); - const vector& typeProj = projRG.getColTypes(); - const vector& csNumProj = projRG.getCharsetNumbers(); + // populate the aggregate rowgroup: projectedRG -> aggregateRG + // + // Aggregate preparation by joblist factory: + // 1. get projected rowgroup (done by doAggProject) -- passed in + // 2. construct aggregate rowgroup -- output of UM + const RowGroup projRG = rowgroups[0]; + const vector& oidsProj = projRG.getOIDs(); + const vector& keysProj = projRG.getKeys(); + const vector& scaleProj = projRG.getScale(); + const vector& precisionProj = projRG.getPrecision(); + const vector& typeProj = projRG.getColTypes(); + const vector& csNumProj = projRG.getCharsetNumbers(); - vector posAgg; - vector oidsAgg; - vector keysAgg; - vector scaleAgg; - vector precisionAgg; - vector typeAgg; - vector csNumAgg; - vector widthAgg; - vector groupBy; - vector functionVec; - uint32_t bigIntWidth = sizeof(int64_t); - uint32_t bigUintWidth = sizeof(uint64_t); - // For UDAF - uint32_t projColsUDAFIdx = 0; - UDAFColumn* udafc = NULL; - mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; - // for count column of average function - map avgFuncMap; + vector posAgg, posAggDist; + vector oidsAgg, oidsAggDist; + vector keysAgg, keysAggDist; + vector scaleAgg, scaleAggDist; + vector precisionAgg, precisionAggDist; + vector typeAgg, typeAggDist; + vector csNumAgg, csNumAggDist; + vector widthProj, widthAgg, widthAggDist; + vector groupBy, groupByNoDist; + vector functionVec1, functionVec2, functionNoDistVec; + uint32_t bigIntWidth = sizeof(int64_t); + // map key = column key, operation (enum), and UDAF pointer if UDAF. + AGG_MAP aggFuncMap; + // set avgSet; + list multiParmIndexes; + // fOR udaf + UDAFColumn* udafc = NULL; + mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; + uint32_t projColsUDAFIdx = 0; + uint32_t udafcParamIdx = 0; + + // for count column of average function + map avgFuncMap, avgDistFuncMap; + + // collect the projected column info, prepare for aggregation + vector width; + for (uint64_t i = 0; i < keysProj.size(); i++) + { + width.push_back(projRG.getColumnWidth(i)); + } + + // associate the columns between projected RG and aggregate RG on UM + // populated the aggregate columns + // the groupby columns are put in front, even not a returned column + // sum and count(column name) are omitted, if avg present + { + // project only unique oids, but they may be repeated in aggregation // collect the projected column info, prepare for aggregation - vector width; map projColPosMap; for (uint64_t i = 0; i < keysProj.size(); i++) { - projColPosMap.insert(make_pair(keysProj[i], i)); - width.push_back(projRG.getColumnWidth(i)); + projColPosMap.insert(make_pair(keysProj[i], i)); + widthProj.push_back(projRG.getColumnWidth(i)); } - // for groupby column - map groupbyMap; + // column index for aggregate rowgroup + uint64_t colAgg = 0; + // for groupby column for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) { - int64_t colProj = projColPosMap[jobInfo.groupByColVec[i]]; - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, -1)); - groupBy.push_back(groupby); - groupbyMap.insert(make_pair(jobInfo.groupByColVec[i], i)); + uint32_t key = jobInfo.groupByColVec[i]; + + if (projColPosMap.find(key) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep1PhaseDistinctAggregate: groupby " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + uint64_t colProj = projColPosMap[key]; + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAgg)); + groupBy.push_back(groupby); + + // copy down to aggregation rowgroup + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(widthProj[colProj]); + + aggFuncMap.insert(make_pair( + boost::make_tuple(keysAgg[colAgg], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + colAgg)); + colAgg++; } // for distinct column for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) { - //@bug6126, continue if already in group by - if (groupbyMap.find(jobInfo.distinctColVec[i]) != groupbyMap.end()) - continue; + uint32_t key = jobInfo.distinctColVec[i]; - int64_t colProj = projColPosMap[jobInfo.distinctColVec[i]]; - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, -1)); - groupBy.push_back(groupby); - groupbyMap.insert(make_pair(jobInfo.distinctColVec[i], i)); + if (projColPosMap.find(key) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep1PhaseDistinctAggregate: distinct " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + // check for dup distinct column -- @bug6126 + if (find(keysAgg.begin(), keysAgg.end(), key) != keysAgg.end()) + continue; + + uint64_t colProj = projColPosMap[key]; + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAgg)); + groupBy.push_back(groupby); + + // copy down to aggregation rowgroup + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(widthProj[colProj]); + + aggFuncMap.insert(make_pair( + boost::make_tuple(keysAgg[colAgg], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + colAgg)); + colAgg++; } - // populate the aggregate rowgroup - AGG_MAP aggFuncMap; + // vectors for aggregate functions + RowAggFunctionType aggOp = ROWAGG_FUNCT_UNDEFINE; + RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; + for (uint64_t i = 0; i < aggColVec.size(); i++) + { + pUDAFFunc = NULL; + uint32_t aggKey = aggColVec[i].first; + aggOp = functionIdMap(aggColVec[i].second); + RowAggFunctionType stats = statsFuncIdMap(aggColVec[i].second); + + // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) + if (aggOp != ROWAGG_MULTI_PARM) + prevAggOp = aggOp; + + // skip if this is a constant + if (aggOp == ROWAGG_CONSTANT) + continue; + + // skip if this is a group_concat + if (aggOp == ROWAGG_GROUP_CONCAT) + { + TupleInfo ti = getTupleInfo(aggKey, jobInfo); + uint32_t width = sizeof(GroupConcatAg*); + oidsAgg.push_back(ti.oid); + keysAgg.push_back(aggKey); + scaleAgg.push_back(ti.scale); + precisionAgg.push_back(ti.precision); + typeAgg.push_back(ti.dtype); + csNumAgg.push_back(ti.csNum); + widthAgg.push_back(width); + SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol(aggOp, stats, colAgg, colAgg, -1)); + functionVec1.push_back(funct); + aggFuncMap.insert(make_pair( + boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + colAgg)); + colAgg++; + + continue; + } + + if (projColPosMap.find(aggKey) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[aggKey] << "' isn't in tuple."; + cerr << "prep1PhaseDistinctAggregate: aggregate " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[aggKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[aggKey].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + // We skip distinct aggs, including extra parms. These are handled by adding them to group by list + // above. + if (aggOp == ROWAGG_DISTINCT_SUM || aggOp == ROWAGG_DISTINCT_AVG || + aggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) + continue; + if (aggOp == ROWAGG_MULTI_PARM && prevAggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) + continue; + + uint64_t colProj = projColPosMap[aggKey]; + + SP_ROWAGG_FUNC_t funct; + + if (aggOp == ROWAGG_UDAF) + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + + for (; it != jobInfo.projectionCols.end(); it++) + { + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) + { + for (uint64_t k = i + 1; + k < aggColVec.size() && aggColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(aggColVec[k].first); + } + } + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + break; + } + } + if (it == jobInfo.projectionCols.end()) + { + throw logic_error( + "(1)prep1PhaseDistinctAggregate: A UDAF function is called but there\'s not enough " + "UDAFColumns"); + } + } + else + { + funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAgg)); + } + + // skip if this is a duplicate + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && + aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL)) != + aggFuncMap.end()) + { + // skip if this is a duplicate + continue; + } + + functionVec1.push_back(funct); + aggFuncMap.insert(make_pair( + boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + colAgg)); + + switch (aggOp) + { + case ROWAGG_MIN: + case ROWAGG_MAX: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(widthProj[colProj]); + colAgg++; + } + break; + + case ROWAGG_SUM: + case ROWAGG_AVG: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("sum/average"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep1PhaseDistinctAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + csNumAgg.push_back(8); + wideDecimalOrLongDouble(colProj, typeProj[colProj], precisionProj, scaleProj, width, typeAgg, + scaleAgg, precisionAgg, widthAgg); + colAgg++; + + // has distinct step, put the count column for avg next to the sum + // let fall through to add a count column for average function + if (aggOp == ROWAGG_AVG) + funct->fAuxColumnIndex = colAgg; + else + break; + } + /* fall through */ + + case ROWAGG_COUNT_ASTERISK: + case ROWAGG_COUNT_COL_NAME: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(0); + // work around count() in select subquery + precisionAgg.push_back(9999); + + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } + + csNumAgg.push_back(8); + widthAgg.push_back(bigIntWidth); + colAgg++; + } + break; + + case ROWAGG_STATS: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("variance/standard deviation"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep1PhaseDistinctAggregate:: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + // count(x) + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(0); + typeAgg.push_back(CalpontSystemCatalog::DOUBLE); + csNumAgg.push_back(8); + widthAgg.push_back(sizeof(double)); + funct->fAuxColumnIndex = ++colAgg; + + // sum(x) + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAgg.push_back(8); + widthAgg.push_back(sizeof(long double)); + ++colAgg; + + // sum(x**2) + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAgg.push_back(8); + widthAgg.push_back(sizeof(long double)); + ++colAgg; + } + break; + + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(0); + precisionAgg.push_back(-16); // for connector to skip null check + + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } + + csNumAgg.push_back(8); + widthAgg.push_back(bigIntWidth); + colAgg++; + } + break; + + case ROWAGG_UDAF: + { + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); + + if (!udafFuncCol) + { + throw logic_error( + "(2)prep1PhaseDistinctAggregate A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + // Return column + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(udafFuncCol->fUDAFContext.getScale()); + precisionAgg.push_back(udafFuncCol->fUDAFContext.getPrecision()); + typeAgg.push_back(udafFuncCol->fUDAFContext.getResultType()); + csNumAgg.push_back(udafFuncCol->fUDAFContext.getCharsetNumber()); + widthAgg.push_back(udafFuncCol->fUDAFContext.getColWidth()); + ++colAgg; + // Column for index of UDAF UserData struct + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(0); + precisionAgg.push_back(0); + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + csNumAgg.push_back(8); + widthAgg.push_back(sizeof(uint64_t)); + funct->fAuxColumnIndex = colAgg++; + // If the first param is const + udafcParamIdx = 0; + ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) + { + funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; + } + ++udafcParamIdx; + break; + } + + case ROWAGG_MULTI_PARM: + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + typeAgg.push_back(typeProj[colProj]); + csNumAgg.push_back(csNumProj[colProj]); + widthAgg.push_back(widthProj[colProj]); + multiParmIndexes.push_back(colAgg); + ++colAgg; + // If the param is const + if (udafc) + { + if (udafcParamIdx > udafc->aggParms().size() - 1) + { + throw QueryDataExcept("prep1PhaseDistinctAggregate: UDAF multi function with too many parms", + aggregateFuncErr); + } + ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) + { + funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; + } + } + else if (prevAggOp != ROWAGG_COUNT_DISTINCT_COL_NAME) + { + throw QueryDataExcept("prep1PhaseDistinctAggregate: UDAF multi function with no parms", + aggregateFuncErr); + } + ++udafcParamIdx; + } + break; + + default: + { + ostringstream emsg; + emsg << "aggregate function (" << (uint64_t)aggOp << ") isn't supported"; + cerr << "prep1PhaseDistinctAggregate: " << emsg.str() << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); + } + } + } + } + + // populated the functionNoDistVec + { + // for (uint32_t idx = 0; idx < functionVec1.size(); idx++) + // { + // SP_ROWAGG_FUNC_t func1 = functionVec1[idx]; + // SP_ROWAGG_FUNC_t funct( + // new RowAggFunctionCol(func1->fAggFunction, + // func1->fStatsFunction, + // func1->fOutputColumnIndex, + // func1->fOutputColumnIndex, + // func1->fAuxColumnIndex)); + // functionNoDistVec.push_back(funct); + // } + functionNoDistVec = functionVec1; + } + + // associate the columns between the non-distinct aggregator and distinct aggregator + // populated the returned columns + // remove not returned groupby column + // add back sum or count(column name) if omitted due to avg column + // put count(column name) column to the end, if it is for avg only + { + // check if the count column for AVG is also a returned column, + // if so, replace the "-1" to actual position in returned vec. + AGG_MAP aggDupFuncMap; + projColsUDAFIdx = 0; + int64_t multiParms = 0; + + // copy over the groupby vector + // update the outputColumnIndex if returned + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(i, -1)); + groupByNoDist.push_back(groupby); + aggFuncMap.insert(make_pair( + boost::make_tuple(keysAgg[i], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), i)); + } + + // locate the return column position in aggregated rowgroup uint64_t outIdx = 0; + RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; + uint32_t prevRetKey = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { - RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); - RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); - uint32_t key = returnedColVec[i].first; + udafc = NULL; + pUDAFFunc = NULL; + uint32_t retKey = returnedColVec[i].first; + RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); + RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); + int colAgg = -1; - if (aggOp == ROWAGG_CONSTANT) + if (aggOp == ROWAGG_MULTI_PARM) + { + // Duplicate detection doesn't work for multi-parm` + + // If this function was earlier detected as a duplicate, unduplicate it. + SP_ROWAGG_FUNC_t funct = functionVec2.back(); + if (funct->fAggFunction == ROWAGG_DUP_FUNCT) + funct->fAggFunction = prevAggOp; + + // Remove it from aggDupFuncMap if it's in there. + funct->hasMultiParm = true; + AGG_MAP::iterator it = aggDupFuncMap.find(boost::make_tuple( + prevRetKey, prevAggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + if (it != aggDupFuncMap.end()) { - TupleInfo ti = getTupleInfo(key, jobInfo); - oidsAgg.push_back(ti.oid); - keysAgg.push_back(key); - scaleAgg.push_back(ti.scale); - precisionAgg.push_back(ti.precision); - typeAgg.push_back(ti.dtype); - csNumAgg.push_back(ti.csNum); - widthAgg.push_back(ti.width); - SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol( - aggOp, stats, 0, outIdx, jobInfo.cntStarPos)); - functionVec.push_back(funct); - ++outIdx; - continue; + aggDupFuncMap.erase(it); } - if (aggOp == ROWAGG_GROUP_CONCAT) + // Skip on final agg.: Extra parms for an aggregate have no work there. + ++multiParms; + continue; + } + else + { + // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) + prevAggOp = aggOp; + prevRetKey = returnedColVec[i].first; + } + + if (find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), retKey) != + jobInfo.distinctColVec.end()) + { + AGG_MAP::iterator it = aggFuncMap.find( + boost::make_tuple(retKey, 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (it != aggFuncMap.end()) { - TupleInfo ti = getTupleInfo(key, jobInfo); - uint32_t ptrSize = sizeof(GroupConcatAg*); - uint32_t width = (ti.width >= ptrSize) ? ti.width : ptrSize; - oidsAgg.push_back(ti.oid); - keysAgg.push_back(key); - scaleAgg.push_back(ti.scale); - precisionAgg.push_back(ti.precision); - typeAgg.push_back(ti.dtype); - csNumAgg.push_back(ti.csNum); - widthAgg.push_back(width); - SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol( - aggOp, stats, 0, outIdx, -1)); - functionVec.push_back(funct); - - ++outIdx; - continue; - } - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep1PhaseAggregate: " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - // make sure the colProj is correct - int64_t colProj = projColPosMap[key]; - - if (keysProj[colProj] != key) - { - ostringstream emsg; - emsg << "projection column map is out of sync."; - cerr << "prep1PhaseAggregate: " << emsg.str() << endl; - throw logic_error(emsg.str()); - } - - if (aggOp == ROWAGG_FUNCT_UNDEFINE) - { - // must be a groupby column or function on aggregation - // or used by group_concat - map::iterator it = groupbyMap.find(key); - - if (it != groupbyMap.end()) - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(width[colProj]); - - if (groupBy[it->second]->fOutputColumnIndex == (uint32_t) - 1) - groupBy[it->second]->fOutputColumnIndex = outIdx; - else - functionVec.push_back(SP_ROWAGG_FUNC_t( - new RowAggFunctionCol( - ROWAGG_DUP_FUNCT, - ROWAGG_FUNCT_UNDEFINE, - -1, - outIdx, - groupBy[it->second]->fOutputColumnIndex))); - - ++outIdx; - continue; - } - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) != - jobInfo.expressionVec.end()) - { - TupleInfo ti = getTupleInfo(key, jobInfo); - oidsAgg.push_back(ti.oid); - keysAgg.push_back(key); - scaleAgg.push_back(ti.scale); - precisionAgg.push_back(ti.precision); - typeAgg.push_back(ti.dtype); - csNumAgg.push_back(ti.csNum); - widthAgg.push_back(ti.width); - ++outIdx; - continue; - } - else if (jobInfo.groupConcatInfo.columns().find(key) != - jobInfo.groupConcatInfo.columns().end()) - { - // TODO: columns only for group_concat do not needed in result set. - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(width[colProj]); - ++outIdx; - continue; - } - else if (jobInfo.windowSet.find(key) != jobInfo.windowSet.end()) - { - // skip window columns/expression, which are computed later - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(width[colProj]); - ++outIdx; - continue; - } - else - { - Message::Args args; - args.add(keyName(i, key, jobInfo)); - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - cerr << "prep1PhaseAggregate: " << emsg << " oid=" - << (int) jobInfo.keyInfo->tupleKeyVec[key].fId << ", alias=" - << jobInfo.keyInfo->tupleKeyVec[key].fTable << ", view=" - << jobInfo.keyInfo->tupleKeyVec[key].fView << ", function=" - << (int) aggOp << endl; - throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); - } - } - - SP_ROWAGG_FUNC_t funct; - - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - for (; it != jobInfo.projectionCols.end(); it++) - { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); - } - } - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, outIdx)); - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(1)prep1PhaseAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } + colAgg = it->second; } else { - funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, outIdx)); + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[retKey] << "' isn't in tuple."; + cerr << "prep1PhaseDistinctAggregate: distinct " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[retKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[retKey].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[retKey].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[retKey].fView; + + cerr << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); } + } - functionVec.push_back(funct); - - switch (aggOp) + if (aggOp == ROWAGG_UDAF) + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) { - case ROWAGG_MIN: - case ROWAGG_MAX: + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(width[colProj]); + for (uint64_t k = i + 1; + k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); + } } break; - - case ROWAGG_AVG: - avgFuncMap.insert(make_pair(key, funct)); - /* fall through */ - case ROWAGG_SUM: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("sum/average"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = - IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep1PhaseAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - wideDecimalOrLongDouble(colProj, typeProj[colProj], - precisionProj, scaleProj, width, - typeAgg, scaleAgg, precisionAgg, widthAgg); - - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - csNumAgg.push_back(csNumProj[colProj]); - } - break; - - case ROWAGG_COUNT_COL_NAME: - case ROWAGG_COUNT_ASTERISK: - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(0); - // work around count() in select subquery - precisionAgg.push_back(9999); - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(bigIntWidth); - } - break; - - case ROWAGG_STATS: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("variance/standard deviation"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = - IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep1PhaseAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(0); - typeAgg.push_back(CalpontSystemCatalog::DOUBLE); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(sizeof(double)); - } - break; - - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(0); - precisionAgg.push_back(-16); // for connector to skip null check - - if (isUnsigned(typeProj[colProj])) - { - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - typeAgg.push_back(CalpontSystemCatalog::BIGINT); - } - - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(bigIntWidth); - } - break; - - case ROWAGG_UDAF: - { - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); - - if (!udafFuncCol) - { - throw logic_error("(2)prep1PhaseAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - // Return column - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(udafFuncCol->fUDAFContext.getScale()); - precisionAgg.push_back(udafFuncCol->fUDAFContext.getPrecision()); - typeAgg.push_back(udafFuncCol->fUDAFContext.getResultType()); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(udafFuncCol->fUDAFContext.getColWidth()); - break; - } - - case ROWAGG_MULTI_PARM: - { - } - break; - - default: - { - ostringstream emsg; - emsg << "aggregate function (" << (uint64_t) aggOp << ") isn't supported"; - cerr << "prep1PhaseAggregate: " << emsg.str() << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); - } + } } - - // find if this func is a duplicate - AGG_MAP::iterator iter = aggFuncMap.find(boost::make_tuple(key, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && iter != aggFuncMap.end()) + if (it == jobInfo.projectionCols.end()) { - if (funct->fAggFunction == ROWAGG_AVG) - funct->fAggFunction = ROWAGG_DUP_AVG; - else if (funct->fAggFunction == ROWAGG_STATS) - funct->fAggFunction = ROWAGG_DUP_STATS; - else if (funct->fAggFunction == ROWAGG_UDAF) - funct->fAggFunction = ROWAGG_DUP_UDAF; - else - funct->fAggFunction = ROWAGG_DUP_FUNCT; - - funct->fAuxColumnIndex = iter->second; + throw logic_error( + "(1)prep1PhaseDistinctAggregate: A UDAF function is called but there\'s not enough " + "UDAFColumns"); } - else + } + + switch (aggOp) + { + case ROWAGG_DISTINCT_AVG: + case ROWAGG_DISTINCT_SUM: { - aggFuncMap.insert(make_pair(boost::make_tuple(key, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), funct->fOutputColumnIndex)); + if (typeAgg[colAgg] == CalpontSystemCatalog::CHAR || + typeAgg[colAgg] == CalpontSystemCatalog::VARCHAR || + typeAgg[colAgg] == CalpontSystemCatalog::BLOB || + typeAgg[colAgg] == CalpontSystemCatalog::TEXT || + typeAgg[colAgg] == CalpontSystemCatalog::DATE || + typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || + typeAgg[colAgg] == CalpontSystemCatalog::TIMESTAMP || + typeAgg[colAgg] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("sum/average"); + args.add(colTypeIdString(typeAgg[colAgg])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep1PhaseDistinctAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + oidsAggDist.push_back(oidsAgg[colAgg]); + keysAggDist.push_back(retKey); + wideDecimalOrLongDouble(colAgg, typeAgg[colAgg], precisionAgg, scaleAgg, widthAgg, typeAggDist, + scaleAggDist, precisionAggDist, widthAggDist); + csNumAggDist.push_back(8); } + break; - if (aggOp != ROWAGG_MULTI_PARM) + case ROWAGG_COUNT_DISTINCT_COL_NAME: { - ++outIdx; + oidsAggDist.push_back(oidsAgg[colAgg]); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(0); + // work around count() in select subquery + precisionAggDist.push_back(9999); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); } - } + break; - // now fix the AVG function, locate the count(column) position - for (uint64_t i = 0; i < functionVec.size(); i++) - { - if (functionVec[i]->fAggFunction != ROWAGG_COUNT_COL_NAME) - continue; - - // if the count(k) can be associated with an avg(k) - map::iterator k = - avgFuncMap.find(keysAgg[functionVec[i]->fOutputColumnIndex]); - - if (k != avgFuncMap.end()) + case ROWAGG_MIN: + case ROWAGG_MAX: + case ROWAGG_SUM: + case ROWAGG_AVG: + case ROWAGG_COUNT_ASTERISK: + case ROWAGG_COUNT_COL_NAME: + case ROWAGG_STATS: + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + default: { - k->second->fAuxColumnIndex = functionVec[i]->fOutputColumnIndex; - functionVec[i]->fAggFunction = ROWAGG_COUNT_NO_OP; - } - } + AGG_MAP::iterator it = aggFuncMap.find( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - // there is avg(k), but no count(k) in the select list - uint64_t lastCol = outIdx; + if (it != aggFuncMap.end()) + { + colAgg = it->second; + oidsAggDist.push_back(oidsAgg[colAgg]); + keysAggDist.push_back(keysAgg[colAgg]); + scaleAggDist.push_back(scaleAgg[colAgg]); + precisionAggDist.push_back(precisionAgg[colAgg]); + typeAggDist.push_back(typeAgg[colAgg]); + csNumAggDist.push_back(csNumAgg[colAgg]); + uint32_t width = widthAgg[colAgg]; - for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) - { - if (k->second->fAuxColumnIndex == (uint32_t) - 1) - { - k->second->fAuxColumnIndex = lastCol++; - oidsAgg.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); - keysAgg.push_back(k->first); - scaleAgg.push_back(0); - precisionAgg.push_back(19); - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - widthAgg.push_back(bigIntWidth); - } - } - - // add auxiliary fields for UDAF and statistics functions - for (uint64_t i = 0; i < functionVec.size(); i++) - { - uint64_t j = functionVec[i]->fInputColumnIndex; - - if (functionVec[i]->fAggFunction == ROWAGG_UDAF) - { - // Column for index of UDAF UserData struct - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVec[i].get()); - - if (!udafFuncCol) - { - throw logic_error("(3)prep1PhaseAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - functionVec[i]->fAuxColumnIndex = lastCol++; - oidsAgg.push_back(oidsProj[j]); - keysAgg.push_back(keysProj[j]); - scaleAgg.push_back(0); - precisionAgg.push_back(0); - precisionAgg.push_back(0); - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - csNumAgg.push_back(8); - widthAgg.push_back(bigUintWidth); - continue; - } - - if (functionVec[i]->fAggFunction != ROWAGG_STATS) - continue; - - functionVec[i]->fAuxColumnIndex = lastCol; - - // sum(x) - oidsAgg.push_back(oidsProj[j]); - keysAgg.push_back(keysProj[j]); - scaleAgg.push_back(0); - precisionAgg.push_back(-1); - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAgg.push_back(8); - widthAgg.push_back(sizeof(long double)); - ++lastCol; - - // sum(x**2) - oidsAgg.push_back(oidsProj[j]); - keysAgg.push_back(keysProj[j]); - scaleAgg.push_back(0); - precisionAgg.push_back(-1); - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAgg.push_back(8); - widthAgg.push_back(sizeof(long double)); - ++lastCol; - } - - // calculate the offset and create the rowaggregation, rowgroup - posAgg.push_back(2); - - for (uint64_t i = 0; i < oidsAgg.size(); i++) - posAgg.push_back(posAgg[i] + widthAgg[i]); - - RowGroup aggRG(oidsAgg.size(), posAgg, oidsAgg, keysAgg, typeAgg, csNumAgg, scaleAgg, precisionAgg, - jobInfo.stringTableThreshold); - SP_ROWAGG_UM_t rowAgg(new RowAggregationUM(groupBy, functionVec, jobInfo.rm, jobInfo.umMemLimit)); - rowAgg->timeZone(jobInfo.timeZone); - rowgroups.push_back(aggRG); - aggregators.push_back(rowAgg); - - // mapping the group_concat columns, if any. - if (jobInfo.groupConcatInfo.groupConcat().size() > 0) - { - jobInfo.groupConcatInfo.mapColumns(projRG); - rowAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); - } - - if (jobInfo.trace) - cout << "\n====== Aggregation RowGroups ======" << endl - << "projected RG: " << projRG.toString() << endl - << "aggregated RG: " << aggRG.toString() << endl; -} - - -void TupleAggregateStep::prep1PhaseDistinctAggregate( - JobInfo& jobInfo, - vector& rowgroups, - vector& aggregators) -{ - // check if there are any aggregate columns - vector > aggColVec; - vector >& returnedColVec = jobInfo.returnedColVec; - - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - if (returnedColVec[i].second != 0) - aggColVec.push_back(returnedColVec[i]); - } - - // populate the aggregate rowgroup: projectedRG -> aggregateRG - // - // Aggregate preparation by joblist factory: - // 1. get projected rowgroup (done by doAggProject) -- passed in - // 2. construct aggregate rowgroup -- output of UM - const RowGroup projRG = rowgroups[0]; - const vector& oidsProj = projRG.getOIDs(); - const vector& keysProj = projRG.getKeys(); - const vector& scaleProj = projRG.getScale(); - const vector& precisionProj = projRG.getPrecision(); - const vector& typeProj = projRG.getColTypes(); - const vector& csNumProj = projRG.getCharsetNumbers(); - - vector posAgg, posAggDist; - vector oidsAgg, oidsAggDist; - vector keysAgg, keysAggDist; - vector scaleAgg, scaleAggDist; - vector precisionAgg, precisionAggDist; - vector typeAgg, typeAggDist; - vector csNumAgg, csNumAggDist; - vector widthProj, widthAgg, widthAggDist; - vector groupBy, groupByNoDist; - vector functionVec1, functionVec2, functionNoDistVec; - uint32_t bigIntWidth = sizeof(int64_t); - // map key = column key, operation (enum), and UDAF pointer if UDAF. - AGG_MAP aggFuncMap; -// set avgSet; - list multiParmIndexes; - - // fOR udaf - UDAFColumn* udafc = NULL; - mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; - uint32_t projColsUDAFIdx = 0; - uint32_t udafcParamIdx = 0; - - // for count column of average function - map avgFuncMap, avgDistFuncMap; - - // collect the projected column info, prepare for aggregation - vector width; - for (uint64_t i = 0; i < keysProj.size(); i++) - { - width.push_back(projRG.getColumnWidth(i)); - } - - - // associate the columns between projected RG and aggregate RG on UM - // populated the aggregate columns - // the groupby columns are put in front, even not a returned column - // sum and count(column name) are omitted, if avg present - { - // project only unique oids, but they may be repeated in aggregation - // collect the projected column info, prepare for aggregation - map projColPosMap; - - for (uint64_t i = 0; i < keysProj.size(); i++) - { - projColPosMap.insert(make_pair(keysProj[i], i)); - widthProj.push_back(projRG.getColumnWidth(i)); - } - - // column index for aggregate rowgroup - uint64_t colAgg = 0; - - // for groupby column - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - uint32_t key = jobInfo.groupByColVec[i]; - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep1PhaseDistinctAggregate: groupby " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - uint64_t colProj = projColPosMap[key]; - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAgg)); - groupBy.push_back(groupby); - - // copy down to aggregation rowgroup - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(widthProj[colProj]); - - aggFuncMap.insert(make_pair(boost::make_tuple(keysAgg[colAgg], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAgg)); - colAgg++; - } - - // for distinct column - for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) - { - uint32_t key = jobInfo.distinctColVec[i]; - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep1PhaseDistinctAggregate: distinct " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - // check for dup distinct column -- @bug6126 - if (find(keysAgg.begin(), keysAgg.end(), key) != keysAgg.end()) - continue; - - uint64_t colProj = projColPosMap[key]; - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAgg)); - groupBy.push_back(groupby); - - // copy down to aggregation rowgroup - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(widthProj[colProj]); - - aggFuncMap.insert(make_pair(boost::make_tuple(keysAgg[colAgg], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAgg)); - colAgg++; - } - - // vectors for aggregate functions - RowAggFunctionType aggOp = ROWAGG_FUNCT_UNDEFINE; - RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; - for (uint64_t i = 0; i < aggColVec.size(); i++) - { - pUDAFFunc = NULL; - uint32_t aggKey = aggColVec[i].first; - aggOp = functionIdMap(aggColVec[i].second); - RowAggFunctionType stats = statsFuncIdMap(aggColVec[i].second); - - // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) - if (aggOp != ROWAGG_MULTI_PARM) - prevAggOp = aggOp; - - // skip if this is a constant - if (aggOp == ROWAGG_CONSTANT) - continue; - - // skip if this is a group_concat if (aggOp == ROWAGG_GROUP_CONCAT) { - TupleInfo ti = getTupleInfo(aggKey, jobInfo); - uint32_t width = sizeof(GroupConcatAg*); - oidsAgg.push_back(ti.oid); - keysAgg.push_back(aggKey); - scaleAgg.push_back(ti.scale); - precisionAgg.push_back(ti.precision); - typeAgg.push_back(ti.dtype); - csNumAgg.push_back(ti.csNum); - widthAgg.push_back(width); - SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol( - aggOp, stats, colAgg, colAgg, -1)); - functionVec1.push_back(funct); - aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAgg)); - colAgg++; + TupleInfo ti = getTupleInfo(retKey, jobInfo); - continue; + if (ti.width > width) + width = ti.width; } - if (projColPosMap.find(aggKey) == projColPosMap.end()) + widthAggDist.push_back(width); + } + + // not a direct hit -- a returned column is not already in the RG from PMs + else + { + bool returnColMissing = true; + + // check if a SUM or COUNT covered by AVG + if (aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[aggKey] << "' isn't in tuple."; - cerr << "prep1PhaseDistinctAggregate: aggregate " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[aggKey].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fTable; + it = aggFuncMap.find(boost::make_tuple(returnedColVec[i].first, ROWAGG_AVG, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL)); - if (jobInfo.keyInfo->tupleKeyVec[aggKey].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fView; + if (it != aggFuncMap.end()) + { + // false alarm + returnColMissing = false; - cerr << endl; - throw logic_error(emsg.str()); - } + colAgg = it->second; - // We skip distinct aggs, including extra parms. These are handled by adding them to group by list above. - if (aggOp == ROWAGG_DISTINCT_SUM || - aggOp == ROWAGG_DISTINCT_AVG || - aggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) - continue; - if (aggOp == ROWAGG_MULTI_PARM && prevAggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) - continue; - - uint64_t colProj = projColPosMap[aggKey]; - - SP_ROWAGG_FUNC_t funct; - - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - - for (; it != jobInfo.projectionCols.end(); it++) + if (aggOp == ROWAGG_SUM) { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < aggColVec.size() && aggColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(aggColVec[k].first); - } - } - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(1)prep1PhaseDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } - } - else - { - funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAgg)); - } - - // skip if this is a duplicate - if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM - && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) - { - // skip if this is a duplicate - continue; - } - - functionVec1.push_back(funct); - aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAgg)); - - switch (aggOp) - { - case ROWAGG_MIN: - case ROWAGG_MAX: - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(widthProj[colProj]); - colAgg++; - } - break; - - case ROWAGG_SUM: - case ROWAGG_AVG: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("sum/average"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep1PhaseDistinctAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - csNumAgg.push_back(8); - wideDecimalOrLongDouble(colProj, typeProj[colProj], - precisionProj, scaleProj, width, - typeAgg, scaleAgg, precisionAgg, widthAgg); - colAgg++; - - // has distinct step, put the count column for avg next to the sum - // let fall through to add a count column for average function - if (aggOp == ROWAGG_AVG) - funct->fAuxColumnIndex = colAgg; - else - break; - } - /* fall through */ - - case ROWAGG_COUNT_ASTERISK: - case ROWAGG_COUNT_COL_NAME: - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(0); - // work around count() in select subquery - precisionAgg.push_back(9999); - - if (isUnsigned(typeProj[colProj])) - { - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - typeAgg.push_back(CalpontSystemCatalog::BIGINT); - } - - csNumAgg.push_back(8); - widthAgg.push_back(bigIntWidth); - colAgg++; - } - break; - - case ROWAGG_STATS: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("variance/standard deviation"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep1PhaseDistinctAggregate:: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - // count(x) - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(0); - typeAgg.push_back(CalpontSystemCatalog::DOUBLE); - csNumAgg.push_back(8); - widthAgg.push_back(sizeof(double)); - funct->fAuxColumnIndex = ++colAgg; - - // sum(x) - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(0); - precisionAgg.push_back(-1); - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAgg.push_back(8); - widthAgg.push_back(sizeof(long double)); - ++colAgg; - - // sum(x**2) - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(0); - precisionAgg.push_back(-1); - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAgg.push_back(8); - widthAgg.push_back(sizeof(long double)); - ++colAgg; - } - break; - - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(0); - precisionAgg.push_back(-16); // for connector to skip null check - - if (isUnsigned(typeProj[colProj])) - { - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - typeAgg.push_back(CalpontSystemCatalog::BIGINT); - } - - csNumAgg.push_back(8); - widthAgg.push_back(bigIntWidth); - colAgg++; - } - break; - - case ROWAGG_UDAF: - { - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); - - if (!udafFuncCol) - { - throw logic_error("(2)prep1PhaseDistinctAggregate A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - // Return column - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(udafFuncCol->fUDAFContext.getScale()); - precisionAgg.push_back(udafFuncCol->fUDAFContext.getPrecision()); - typeAgg.push_back(udafFuncCol->fUDAFContext.getResultType()); - csNumAgg.push_back(udafFuncCol->fUDAFContext.getCharsetNumber()); - widthAgg.push_back(udafFuncCol->fUDAFContext.getColWidth()); - ++colAgg; - // Column for index of UDAF UserData struct - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(0); - precisionAgg.push_back(0); - typeAgg.push_back(CalpontSystemCatalog::UBIGINT); - csNumAgg.push_back(8); - widthAgg.push_back(sizeof(uint64_t)); - funct->fAuxColumnIndex = colAgg++; - // If the first param is const - udafcParamIdx = 0; - ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); - if (cc) - { - funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; - } - ++udafcParamIdx; - break; - } - - case ROWAGG_MULTI_PARM: - { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); - scaleAgg.push_back(scaleProj[colProj]); - precisionAgg.push_back(precisionProj[colProj]); - typeAgg.push_back(typeProj[colProj]); - csNumAgg.push_back(csNumProj[colProj]); - widthAgg.push_back(widthProj[colProj]); - multiParmIndexes.push_back(colAgg); - ++colAgg; - // If the param is const - if (udafc) - { - if (udafcParamIdx > udafc->aggParms().size() - 1) - { - throw QueryDataExcept("prep1PhaseDistinctAggregate: UDAF multi function with too many parms", aggregateFuncErr); - } - ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); - if (cc) - { - funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; - } - } - else if (prevAggOp != ROWAGG_COUNT_DISTINCT_COL_NAME) - { - throw QueryDataExcept("prep1PhaseDistinctAggregate: UDAF multi function with no parms", aggregateFuncErr); - } - ++udafcParamIdx; - } - break; - - default: - { - ostringstream emsg; - emsg << "aggregate function (" << (uint64_t) aggOp << ") isn't supported"; - cerr << "prep1PhaseDistinctAggregate: " << emsg.str() << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); - } - } - } - } - - // populated the functionNoDistVec - { -// for (uint32_t idx = 0; idx < functionVec1.size(); idx++) -// { -// SP_ROWAGG_FUNC_t func1 = functionVec1[idx]; -// SP_ROWAGG_FUNC_t funct( -// new RowAggFunctionCol(func1->fAggFunction, -// func1->fStatsFunction, -// func1->fOutputColumnIndex, -// func1->fOutputColumnIndex, -// func1->fAuxColumnIndex)); -// functionNoDistVec.push_back(funct); -// } - functionNoDistVec = functionVec1; - } - - // associate the columns between the non-distinct aggregator and distinct aggregator - // populated the returned columns - // remove not returned groupby column - // add back sum or count(column name) if omitted due to avg column - // put count(column name) column to the end, if it is for avg only - { - // check if the count column for AVG is also a returned column, - // if so, replace the "-1" to actual position in returned vec. - AGG_MAP aggDupFuncMap; - projColsUDAFIdx = 0; - int64_t multiParms = 0; - - // copy over the groupby vector - // update the outputColumnIndex if returned - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(i, -1)); - groupByNoDist.push_back(groupby); - aggFuncMap.insert(make_pair(boost::make_tuple(keysAgg[i], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), i)); - } - - // locate the return column position in aggregated rowgroup - uint64_t outIdx = 0; - RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; - uint32_t prevRetKey = 0; - - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - udafc = NULL; - pUDAFFunc = NULL; - uint32_t retKey = returnedColVec[i].first; - RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); - RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); - int colAgg = -1; - - if (aggOp == ROWAGG_MULTI_PARM) - { - // Duplicate detection doesn't work for multi-parm` - - // If this function was earlier detected as a duplicate, unduplicate it. - SP_ROWAGG_FUNC_t funct = functionVec2.back(); - if (funct->fAggFunction == ROWAGG_DUP_FUNCT) - funct->fAggFunction = prevAggOp; - - // Remove it from aggDupFuncMap if it's in there. - funct->hasMultiParm = true; - AGG_MAP::iterator it = aggDupFuncMap.find(boost::make_tuple(prevRetKey, prevAggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - if (it != aggDupFuncMap.end()) - { - aggDupFuncMap.erase(it); - } - - // Skip on final agg.: Extra parms for an aggregate have no work there. - ++multiParms; - continue; - } - else - { - // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) - prevAggOp = aggOp; - prevRetKey = returnedColVec[i].first; - } - - if (find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), retKey) != - jobInfo.distinctColVec.end() ) - { - AGG_MAP::iterator it = aggFuncMap.find(boost::make_tuple(retKey, 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - colAgg = it->second; + oidsAggDist.push_back(oidsAgg[colAgg]); + keysAggDist.push_back(retKey); + csNumAggDist.push_back(8); + wideDecimalOrLongDouble(colAgg, typeAgg[colAgg], precisionAgg, scaleAgg, widthAgg, + typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); } else { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[retKey] << "' isn't in tuple."; - cerr << "prep1PhaseDistinctAggregate: distinct " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[retKey].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[retKey].fTable; + // leave the count() to avg + aggOp = ROWAGG_COUNT_NO_OP; - if (jobInfo.keyInfo->tupleKeyVec[retKey].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[retKey].fView; + oidsAggDist.push_back(oidsAgg[colAgg]); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(0); - cerr << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); - } - } - - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - for (; it != jobInfo.projectionCols.end(); it++) - { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); - } - } - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(1)prep1PhaseDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } - } - - switch (aggOp) - { - case ROWAGG_DISTINCT_AVG: - case ROWAGG_DISTINCT_SUM: - { - if (typeAgg[colAgg] == CalpontSystemCatalog::CHAR || - typeAgg[colAgg] == CalpontSystemCatalog::VARCHAR || - typeAgg[colAgg] == CalpontSystemCatalog::BLOB || - typeAgg[colAgg] == CalpontSystemCatalog::TEXT || - typeAgg[colAgg] == CalpontSystemCatalog::DATE || - typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || - typeAgg[colAgg] == CalpontSystemCatalog::TIMESTAMP || - typeAgg[colAgg] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("sum/average"); - args.add(colTypeIdString(typeAgg[colAgg])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep1PhaseDistinctAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - oidsAggDist.push_back(oidsAgg[colAgg]); - keysAggDist.push_back(retKey); - wideDecimalOrLongDouble(colAgg, typeAgg[colAgg], - precisionAgg, scaleAgg, widthAgg, - typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); - csNumAggDist.push_back(8); - } - break; - - case ROWAGG_COUNT_DISTINCT_COL_NAME: - { - oidsAggDist.push_back(oidsAgg[colAgg]); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(0); - // work around count() in select subquery - precisionAggDist.push_back(9999); + if (isUnsigned(typeAgg[colAgg])) + { typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); + precisionAggDist.push_back(20); + } + else + { + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + precisionAggDist.push_back(19); + } + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); } - break; + } + } + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), retKey) != + jobInfo.expressionVec.end()) + { + // a function on aggregation + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggDist.push_back(ti.oid); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(ti.scale); + precisionAggDist.push_back(ti.precision); + typeAggDist.push_back(ti.dtype); + csNumAggDist.push_back(ti.csNum); + widthAggDist.push_back(ti.width); - case ROWAGG_MIN: - case ROWAGG_MAX: - case ROWAGG_SUM: - case ROWAGG_AVG: - case ROWAGG_COUNT_ASTERISK: - case ROWAGG_COUNT_COL_NAME: - case ROWAGG_STATS: - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - default: - { - AGG_MAP::iterator it = aggFuncMap.find(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + returnColMissing = false; + } + else if (aggOp == ROWAGG_CONSTANT) + { + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggDist.push_back(ti.oid); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(ti.scale); + precisionAggDist.push_back(ti.precision); + typeAggDist.push_back(ti.dtype); + csNumAggDist.push_back(ti.csNum); + widthAggDist.push_back(ti.width); - if (it != aggFuncMap.end()) - { - colAgg = it->second; - oidsAggDist.push_back(oidsAgg[colAgg]); - keysAggDist.push_back(keysAgg[colAgg]); - scaleAggDist.push_back(scaleAgg[colAgg]); - precisionAggDist.push_back(precisionAgg[colAgg]); - typeAggDist.push_back(typeAgg[colAgg]); - csNumAggDist.push_back(csNumAgg[colAgg]); - uint32_t width = widthAgg[colAgg]; - - if (aggOp == ROWAGG_GROUP_CONCAT) - { - TupleInfo ti = getTupleInfo(retKey, jobInfo); - - if (ti.width > width) - width = ti.width; - } - - widthAggDist.push_back(width); - } - - // not a direct hit -- a returned column is not already in the RG from PMs - else - { - bool returnColMissing = true; - - // check if a SUM or COUNT covered by AVG - if (aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) - { - it = aggFuncMap.find(boost::make_tuple(returnedColVec[i].first, ROWAGG_AVG, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - // false alarm - returnColMissing = false; - - colAgg = it->second; - - if (aggOp == ROWAGG_SUM) - { - oidsAggDist.push_back(oidsAgg[colAgg]); - keysAggDist.push_back(retKey); - csNumAggDist.push_back(8); - wideDecimalOrLongDouble(colAgg, typeAgg[colAgg], - precisionAgg, scaleAgg, widthAgg, - typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); - } - else - { - // leave the count() to avg - aggOp = ROWAGG_COUNT_NO_OP; - - oidsAggDist.push_back(oidsAgg[colAgg]); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(0); - - if (isUnsigned(typeAgg[colAgg])) - { - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - precisionAggDist.push_back(20); - } - else - { - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); - precisionAggDist.push_back(19); - } - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - } - } - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), - retKey) != jobInfo.expressionVec.end()) - { - // a function on aggregation - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggDist.push_back(ti.oid); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(ti.scale); - precisionAggDist.push_back(ti.precision); - typeAggDist.push_back(ti.dtype); - csNumAggDist.push_back(ti.csNum); - widthAggDist.push_back(ti.width); - - returnColMissing = false; - } - else if (aggOp == ROWAGG_CONSTANT) - { - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggDist.push_back(ti.oid); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(ti.scale); - precisionAggDist.push_back(ti.precision); - typeAggDist.push_back(ti.dtype); - csNumAggDist.push_back(ti.csNum); - widthAggDist.push_back(ti.width); - - returnColMissing = false; - } + returnColMissing = false; + } #if 0 else if (aggOp == ROWAGG_GROUP_CONCAT) @@ -2454,2991 +2352,2960 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( } #endif - else if (jobInfo.groupConcatInfo.columns().find(retKey) != - jobInfo.groupConcatInfo.columns().end()) - { - // TODO: columns only for group_concat do not needed in result set. - for (uint64_t k = 0; k < keysProj.size(); k++) - { - if (retKey == keysProj[k]) - { - oidsAggDist.push_back(oidsProj[k]); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(scaleProj[k] >> 8); - precisionAggDist.push_back(precisionProj[k]); - typeAggDist.push_back(typeProj[k]); - csNumAggDist.push_back(csNumProj[k]); - widthAggDist.push_back(widthProj[k]); - - returnColMissing = false; - break; - } - } - } - - else if (jobInfo.windowSet.find(retKey) != jobInfo.windowSet.end()) - { - // skip window columns/expression, which are computed later - for (uint64_t k = 0; k < keysProj.size(); k++) - { - if (retKey == keysProj[k]) - { - oidsAggDist.push_back(oidsProj[k]); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(scaleProj[k] >> 8); - precisionAggDist.push_back(precisionProj[k]); - typeAggDist.push_back(typeProj[k]); - csNumAggDist.push_back(csNumProj[k]); - widthAggDist.push_back(widthProj[k]); - - returnColMissing = false; - break; - } - } - } - - if (returnColMissing) - { - Message::Args args; - args.add(keyName(outIdx, retKey, jobInfo)); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - cerr << "prep1PhaseDistinctAggregate: " << emsg << " oid=" - << (int) jobInfo.keyInfo->tupleKeyVec[retKey].fId << ", alias=" - << jobInfo.keyInfo->tupleKeyVec[retKey].fTable << ", view=" - << jobInfo.keyInfo->tupleKeyVec[retKey].fView << ", function=" - << (int) aggOp << endl; - throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); - } - } //else - } // switch - } - - // update groupby vector if the groupby column is a returned column - if (returnedColVec[i].second == 0) + else if (jobInfo.groupConcatInfo.columns().find(retKey) != + jobInfo.groupConcatInfo.columns().end()) { - int dupGroupbyIndex = -1; - - for (uint64_t j = 0; j < jobInfo.groupByColVec.size(); j++) + // TODO: columns only for group_concat do not needed in result set. + for (uint64_t k = 0; k < keysProj.size(); k++) + { + if (retKey == keysProj[k]) { - if (jobInfo.groupByColVec[j] == retKey) - { - if (groupByNoDist[j]->fOutputColumnIndex == (uint32_t) - 1) - groupByNoDist[j]->fOutputColumnIndex = outIdx; - else - dupGroupbyIndex = groupByNoDist[j]->fOutputColumnIndex; - } - } + oidsAggDist.push_back(oidsProj[k]); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(scaleProj[k] >> 8); + precisionAggDist.push_back(precisionProj[k]); + typeAggDist.push_back(typeProj[k]); + csNumAggDist.push_back(csNumProj[k]); + widthAggDist.push_back(widthProj[k]); - // a duplicate group by column - if (dupGroupbyIndex != -1) - functionVec2.push_back(SP_ROWAGG_FUNC_t( - new RowAggFunctionCol( - ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, dupGroupbyIndex))); + returnColMissing = false; + break; + } + } } + + else if (jobInfo.windowSet.find(retKey) != jobInfo.windowSet.end()) + { + // skip window columns/expression, which are computed later + for (uint64_t k = 0; k < keysProj.size(); k++) + { + if (retKey == keysProj[k]) + { + oidsAggDist.push_back(oidsProj[k]); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(scaleProj[k] >> 8); + precisionAggDist.push_back(precisionProj[k]); + typeAggDist.push_back(typeProj[k]); + csNumAggDist.push_back(csNumProj[k]); + widthAggDist.push_back(widthProj[k]); + + returnColMissing = false; + break; + } + } + } + + if (returnColMissing) + { + Message::Args args; + args.add(keyName(outIdx, retKey, jobInfo)); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + cerr << "prep1PhaseDistinctAggregate: " << emsg + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[retKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[retKey].fTable + << ", view=" << jobInfo.keyInfo->tupleKeyVec[retKey].fView << ", function=" << (int)aggOp + << endl; + throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); + } + } // else + } // switch + } + + // update groupby vector if the groupby column is a returned column + if (returnedColVec[i].second == 0) + { + int dupGroupbyIndex = -1; + + for (uint64_t j = 0; j < jobInfo.groupByColVec.size(); j++) + { + if (jobInfo.groupByColVec[j] == retKey) + { + if (groupByNoDist[j]->fOutputColumnIndex == (uint32_t)-1) + groupByNoDist[j]->fOutputColumnIndex = outIdx; else - { - // update the aggregate function vector - SP_ROWAGG_FUNC_t funct; - if (aggOp == ROWAGG_UDAF) - { - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colAgg, outIdx)); - } - else - { - funct.reset(new RowAggFunctionCol(aggOp, stats, colAgg, outIdx)); - } - - if (aggOp == ROWAGG_COUNT_NO_OP) - funct->fAuxColumnIndex = colAgg; - else if (aggOp == ROWAGG_CONSTANT) - funct->fAuxColumnIndex = jobInfo.cntStarPos; - - functionVec2.push_back(funct); - - // find if this func is a duplicate - AGG_MAP::iterator iter = aggDupFuncMap.find(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (iter != aggDupFuncMap.end()) - { - if (funct->fAggFunction == ROWAGG_AVG) - funct->fAggFunction = ROWAGG_DUP_AVG; - else if (funct->fAggFunction == ROWAGG_STATS) - funct->fAggFunction = ROWAGG_DUP_STATS; - else if (funct->fAggFunction == ROWAGG_UDAF) - funct->fAggFunction = ROWAGG_DUP_UDAF; - else - funct->fAggFunction = ROWAGG_DUP_FUNCT; - - funct->fAuxColumnIndex = iter->second; - } - else - { - aggDupFuncMap.insert(make_pair(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), - funct->fOutputColumnIndex)); - } - - if (returnedColVec[i].second == AggregateColumn::AVG) - avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); - else if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) - avgDistFuncMap.insert(make_pair(returnedColVec[i].first, funct)); - } - ++outIdx; - } // for (i - - // now fix the AVG function, locate the count(column) position - for (uint64_t i = 0; i < functionVec2.size(); i++) - { - if (functionVec2[i]->fAggFunction == ROWAGG_COUNT_NO_OP) - { - // if the count(k) can be associated with an avg(k) - map::iterator k = - avgFuncMap.find(keysAggDist[functionVec2[i]->fOutputColumnIndex]); - - if (k != avgFuncMap.end()) - k->second->fAuxColumnIndex = functionVec2[i]->fOutputColumnIndex; - } + dupGroupbyIndex = groupByNoDist[j]->fOutputColumnIndex; + } } - // there is avg(k), but no count(k) in the select list - uint64_t lastCol = outIdx; - - for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) + // a duplicate group by column + if (dupGroupbyIndex != -1) + functionVec2.push_back(SP_ROWAGG_FUNC_t( + new RowAggFunctionCol(ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, dupGroupbyIndex))); + } + else + { + // update the aggregate function vector + SP_ROWAGG_FUNC_t funct; + if (aggOp == ROWAGG_UDAF) { - if (k->second->fAuxColumnIndex == (uint32_t) - 1) - { - k->second->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); - keysAggDist.push_back(k->first); - scaleAggDist.push_back(0); - precisionAggDist.push_back(19); - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - } - - // now fix the AVG distinct function, locate the count(distinct column) position - for (uint64_t i = 0; i < functionVec2.size(); i++) - { - if (functionVec2[i]->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME && !functionVec2[i]->hasMultiParm) - { - // if the count(distinct k) can be associated with an avg(distinct k) - map::iterator k = - avgDistFuncMap.find(keysAggDist[functionVec2[i]->fOutputColumnIndex]); - - if (k != avgDistFuncMap.end()) - { - k->second->fAuxColumnIndex = functionVec2[i]->fOutputColumnIndex; - functionVec2[i]->fAggFunction = ROWAGG_COUNT_NO_OP; - } - } - } - - // there is avg(distinct k), but no count(distinct k) in the select list - for (map::iterator k = avgDistFuncMap.begin(); k != avgDistFuncMap.end(); k++) - { - if (k->second->fAuxColumnIndex == (uint32_t) - 1) - { - k->second->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); - keysAggDist.push_back(k->first); - scaleAggDist.push_back(0); - precisionAggDist.push_back(19); - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - } - - // add auxiliary fields for UDAF and statistics functions - for (uint64_t i = 0; i < functionVec2.size(); i++) - { - uint64_t j = functionVec2[i]->fInputColumnIndex; - - if (functionVec2[i]->fAggFunction == ROWAGG_UDAF) - { - // Column for index of UDAF UserData struct - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVec2[i].get()); - - if (!udafFuncCol) - { - throw logic_error("(4)prep1PhaseDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - functionVec2[i]->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(oidsAgg[j]); // Dummy? - keysAggDist.push_back(keysAgg[j]); // Dummy? - scaleAggDist.push_back(0); - precisionAggDist.push_back(0); - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(sizeof(uint64_t)); - continue; - } - - if (functionVec2[i]->fAggFunction != ROWAGG_STATS) - continue; - - functionVec2[i]->fAuxColumnIndex = lastCol; - - // sum(x) - oidsAggDist.push_back(oidsAgg[j]); - keysAggDist.push_back(keysAgg[j]); - scaleAggDist.push_back(0); - precisionAggDist.push_back(0); - typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggDist.push_back(8); - widthAggDist.push_back(sizeof(long double)); - ++lastCol; - - // sum(x**2) - oidsAggDist.push_back(oidsAgg[j]); - keysAggDist.push_back(keysAgg[j]); - scaleAggDist.push_back(0); - precisionAggDist.push_back(-1); - typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggDist.push_back(8); - widthAggDist.push_back(sizeof(long double)); - ++lastCol; - } - } - - // calculate the offset and create the rowaggregation, rowgroup - posAgg.push_back(2); - - for (uint64_t i = 0; i < oidsAgg.size(); i++) - posAgg.push_back(posAgg[i] + widthAgg[i]); - - RowGroup aggRG(oidsAgg.size(), posAgg, oidsAgg, keysAgg, typeAgg, csNumAgg, scaleAgg, precisionAgg, - jobInfo.stringTableThreshold); - SP_ROWAGG_UM_t rowAgg(new RowAggregationUM(groupBy, functionVec1, jobInfo.rm, jobInfo.umMemLimit)); - rowAgg->timeZone(jobInfo.timeZone); - - posAggDist.push_back(2); // rid - - for (uint64_t i = 0; i < oidsAggDist.size(); i++) - posAggDist.push_back(posAggDist[i] + widthAggDist[i]); - - RowGroup aggRgDist(oidsAggDist.size(), posAggDist, oidsAggDist, keysAggDist, typeAggDist, - csNumAggDist, scaleAggDist, precisionAggDist, jobInfo.stringTableThreshold); - SP_ROWAGG_DIST rowAggDist(new RowAggregationDistinct(groupByNoDist, functionVec2, jobInfo.rm, jobInfo.umMemLimit)); - rowAggDist->timeZone(jobInfo.timeZone); - - // mapping the group_concat columns, if any. - if (jobInfo.groupConcatInfo.groupConcat().size() > 0) - { - jobInfo.groupConcatInfo.mapColumns(projRG); - rowAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); - rowAggDist->groupConcat(jobInfo.groupConcatInfo.groupConcat()); - } - - // if distinct key word applied to more than one aggregate column, reset rowAggDist - vector subRgVec; - - if (jobInfo.distinctColVec.size() > 1) - { - RowAggregationMultiDistinct* multiDistinctAggregator = - new RowAggregationMultiDistinct(groupByNoDist, functionVec2, jobInfo.rm, jobInfo.umMemLimit); - multiDistinctAggregator->timeZone(jobInfo.timeZone); - rowAggDist.reset(multiDistinctAggregator); - rowAggDist->groupConcat(jobInfo.groupConcatInfo.groupConcat()); - - // construct and add sub-aggregators to rowAggDist - vector posAggGb, posAggSub; - vector oidsAggGb, oidsAggSub; - vector keysAggGb, keysAggSub; - vector scaleAggGb, scaleAggSub; - vector precisionAggGb, precisionAggSub; - vector typeAggGb, typeAggSub; - vector csNumAggGb, csNumAggSub; - vector widthAggGb, widthAggSub; - - // populate groupby column info - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - oidsAggGb.push_back(oidsProj[i]); - keysAggGb.push_back(keysProj[i]); - scaleAggGb.push_back(scaleProj[i]); - precisionAggGb.push_back(precisionProj[i]); - typeAggGb.push_back(typeProj[i]); - csNumAggGb.push_back(csNumProj[i]); - widthAggGb.push_back(widthProj[i]); - } - - // for distinct, each column requires seperate rowgroup - vector rowAggSubDistVec; - - uint32_t distinctColKey; - int64_t j; - uint64_t k; - uint64_t outIdx = 0; - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - if (returnedColVec[i].second == 0) - { - ++outIdx; - continue; - } - - j = -1; - - distinctColKey = -1; - // Find the entry in distinctColVec, if any - for (k = 0; k < jobInfo.distinctColVec.size(); k++) - { - distinctColKey = jobInfo.distinctColVec[k]; - if (returnedColVec[i].first == distinctColKey) - break; - - } - if (distinctColKey == (uint32_t)-1) - { - ++outIdx; - continue; - } - // locate the distinct key in the row group - for (k = 0; k < keysAgg.size(); k++) - { - if (keysProj[k] == distinctColKey) - { - j = k; - break; - } - } - - idbassert(j != -1); - - oidsAggSub = oidsAggGb; - keysAggSub = keysAggGb; - scaleAggSub = scaleAggGb; - precisionAggSub = precisionAggGb; - typeAggSub = typeAggGb; - csNumAggSub = csNumAggGb; - widthAggSub = widthAggGb; - - oidsAggSub.push_back(oidsProj[j]); - keysAggSub.push_back(keysProj[j]); - scaleAggSub.push_back(scaleProj[j]); - precisionAggSub.push_back(precisionProj[j]); - typeAggSub.push_back(typeProj[j]); - csNumAggSub.push_back(csNumProj[j]); - widthAggSub.push_back(widthProj[j]); - - // construct groupby vector - vector groupBySub; - k = 0; - - while (k < jobInfo.groupByColVec.size()) - { - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(k, k)); - groupBySub.push_back(groupby); - k++; - } - // add the distinct column as groupby - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); - groupBySub.push_back(groupby); - - // Add multi parm distinct - while ((i+1) < returnedColVec.size() && functionIdMap(returnedColVec[i+1].second) == ROWAGG_MULTI_PARM) - { - ++i; - uint32_t dColKey = -1; - j = -1; - - // Find the entry in distinctColVec, if any - for (k = 0; k < jobInfo.distinctColVec.size(); k++) - { - dColKey = jobInfo.distinctColVec[k]; - if (returnedColVec[i].first == dColKey) - break; - - } - idbassert(dColKey != (uint32_t)-1); - // locate the distinct key in the row group - for (k = 0; k < keysAgg.size(); k++) - { - if (keysProj[k] == dColKey) - { - j = k; - break; - } - } - idbassert(j != -1); - - oidsAggSub.push_back(oidsProj[j]); - keysAggSub.push_back(keysProj[j]); - scaleAggSub.push_back(scaleProj[j]); - precisionAggSub.push_back(precisionProj[j]); - typeAggSub.push_back(typeProj[j]); - csNumAggSub.push_back(csNumProj[j]); - widthAggSub.push_back(widthProj[j]); - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); - groupBySub.push_back(groupby); - } - - // construct sub-rowgroup - posAggSub.clear(); - posAggSub.push_back(2); // rid - - for ( k = 0; k < oidsAggSub.size(); k++) - posAggSub.push_back(posAggSub[k] + widthAggSub[k]); - - RowGroup subRg(oidsAggSub.size(), posAggSub, oidsAggSub, keysAggSub, typeAggSub, - csNumAggSub, scaleAggSub, precisionAggSub, jobInfo.stringTableThreshold); - subRgVec.push_back(subRg); - - // Keep a count of the parms after the first for any aggregate. - // These will be skipped and the count needs to be subtracted - // from where the aux column will be. - int64_t multiParms = 0; - - // tricky part : 2 function vectors - // -- dummy function vector for sub-aggregator, which does distinct only - // -- aggregate function on this distinct column for rowAggDist - vector functionSub1, functionSub2; - // search the function in functionVec - vector::iterator it = functionVec2.begin(); - - while (it != functionVec2.end()) - { - SP_ROWAGG_FUNC_t f = *it++; - - if ((f->fOutputColumnIndex == outIdx) && - (f->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME || - f->fAggFunction == ROWAGG_DISTINCT_SUM || - f->fAggFunction == ROWAGG_DISTINCT_AVG)) - { - SP_ROWAGG_FUNC_t funct( - new RowAggFunctionCol( - f->fAggFunction, - f->fStatsFunction, - groupBySub.size() - 1, - f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); - functionSub2.push_back(funct); - } - } - - // construct sub-aggregator - SP_ROWAGG_UM_t subAgg( - new RowAggregationSubDistinct(groupBySub, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); - subAgg->timeZone(jobInfo.timeZone); - subAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); - - // add to rowAggDist - multiDistinctAggregator->addSubAggregator(subAgg, subRg, functionSub2); - - ++outIdx; - } - - // cover any non-distinct column functions - { - vector functionSub1 = functionNoDistVec; - vector functionSub2; - int64_t multiParms = 0; - - for (uint64_t k = 0; k < returnedColVec.size(); k++) - { - // search non-distinct functions in functionVec - vector::iterator it = functionVec2.begin(); - - while (it != functionVec2.end()) - { - SP_ROWAGG_FUNC_t funct; - SP_ROWAGG_FUNC_t f = *it++; - - if (f->fAggFunction == ROWAGG_UDAF) - { - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(f.get()); - funct.reset(new RowUDAFFunctionCol( - udafFuncCol->fUDAFContext, - udafFuncCol->fInputColumnIndex, - udafFuncCol->fOutputColumnIndex, - udafFuncCol->fAuxColumnIndex-multiParms)); - functionSub2.push_back(funct); - } - else if ((f->fOutputColumnIndex == k) && - (f->fAggFunction == ROWAGG_COUNT_ASTERISK || - f->fAggFunction == ROWAGG_COUNT_COL_NAME || - f->fAggFunction == ROWAGG_SUM || - f->fAggFunction == ROWAGG_AVG || - f->fAggFunction == ROWAGG_MIN || - f->fAggFunction == ROWAGG_MAX || - f->fAggFunction == ROWAGG_STATS || - f->fAggFunction == ROWAGG_BIT_AND || - f->fAggFunction == ROWAGG_BIT_OR || - f->fAggFunction == ROWAGG_BIT_XOR || - f->fAggFunction == ROWAGG_CONSTANT || - f->fAggFunction == ROWAGG_GROUP_CONCAT)) - { - funct.reset(new RowAggFunctionCol( - f->fAggFunction, - f->fStatsFunction, - f->fInputColumnIndex, - f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); - functionSub2.push_back(funct); - } - } - } - - - if (functionSub1.size() > 0) - { - // make sure the group by columns are available for next aggregate phase. - vector groupBySubNoDist; - - for (uint64_t i = 0; i < groupByNoDist.size(); i++) - groupBySubNoDist.push_back(SP_ROWAGG_GRPBY_t( - new RowAggGroupByCol(groupByNoDist[i]->fInputColumnIndex, i))); - - // construct sub-aggregator - SP_ROWAGG_UM_t subAgg( - new RowAggregationUM(groupBySubNoDist, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); - subAgg->timeZone(jobInfo.timeZone); - subAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); - - // add to rowAggDist - multiDistinctAggregator->addSubAggregator(subAgg, aggRG, functionSub2); - subRgVec.push_back(aggRG); - } - } - } - - rowAggDist->addAggregator(rowAgg, aggRG); - rowgroups.push_back(aggRgDist); - aggregators.push_back(rowAggDist); - - if (jobInfo.trace) - { - cout << "projected RG: " << projRG.toString() << endl - << "aggregated RG: " << aggRG.toString() << endl; - - for (uint64_t i = 0; i < subRgVec.size(); i++) - cout << "aggregatedSub RG: " << i << " " << subRgVec[i].toString() << endl; - - cout << "aggregatedDist RG: " << aggRgDist.toString() << endl; - } -} - - -void TupleAggregateStep::prep2PhasesAggregate( - JobInfo& jobInfo, vector& rowgroups, vector& aggregators) -{ - // check if there are any aggregate columns - // a vector that has the aggregate function to be done by PM - vector > aggColVec; - set avgSet; - vector >& returnedColVec = jobInfo.returnedColVec; - // For UDAF - uint32_t projColsUDAFIdx = 0; - uint32_t udafcParamIdx = 0; - UDAFColumn* udafc = NULL; - mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; - - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - // skip if not an aggregation column - if (returnedColVec[i].second == 0) - continue; - - aggColVec.push_back(returnedColVec[i]); - - // remember if a column has an average function, - // with avg function, no need for separate sum or count_column_name - if (returnedColVec[i].second == AggregateColumn::AVG) - avgSet.insert(returnedColVec[i].first); - } - - // populate the aggregate rowgroup on PM and UM - // PM: projectedRG -> aggregateRGPM - // UM: aggregateRGPM -> aggregateRGUM - // - // Aggregate preparation by joblist factory: - // 1. get projected rowgroup (done by doAggProject) -- input to PM AGG - // 2. construct aggregate rowgroup -- output of PM, input of UM - // 3. construct aggregate rowgroup -- output of UM - const RowGroup projRG = rowgroups[0]; - const vector& oidsProj = projRG.getOIDs(); - const vector& keysProj = projRG.getKeys(); - const vector& scaleProj = projRG.getScale(); - const vector& precisionProj = projRG.getPrecision(); - const vector& typeProj = projRG.getColTypes(); - const vector& csNumProj = projRG.getCharsetNumbers(); - - vector posAggPm, posAggUm; - vector oidsAggPm, oidsAggUm; - vector keysAggPm, keysAggUm; - vector scaleAggPm, scaleAggUm; - vector precisionAggPm, precisionAggUm; - vector typeAggPm, typeAggUm; - vector csNumAggPm, csNumAggUm; - vector widthAggPm, widthAggUm; - vector groupByPm, groupByUm; - vector functionVecPm, functionVecUm; - uint32_t bigIntWidth = sizeof(int64_t); - uint32_t bigUintWidth = sizeof(uint64_t); - AGG_MAP aggFuncMap; - - // associate the columns between projected RG and aggregate RG on PM - // populated the aggregate columns - // the groupby columns are put in front, even not a returned column - // sum and count(column name) are omitted, if avg present - { - // project only unique oids, but they may be repeated in aggregation - // collect the projected column info, prepare for aggregation - vector width; - map projColPosMap; - - for (uint64_t i = 0; i < keysProj.size(); i++) - { - projColPosMap.insert(make_pair(keysProj[i], i)); - width.push_back(projRG.getColumnWidth(i)); - } - - // column index for PM aggregate rowgroup - uint64_t colAggPm = 0; - - // for groupby column - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - uint32_t key = jobInfo.groupByColVec[i]; - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep2PhasesAggregate: groupby " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - uint64_t colProj = projColPosMap[key]; - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); - groupByPm.push_back(groupby); - - // PM: just copy down to aggregation rowgroup - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(key); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - - aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); - colAggPm++; - } - - // for distinct column - for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) - { - uint32_t key = jobInfo.distinctColVec[i]; - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep2PhasesAggregate: distinct " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - uint64_t colProj = projColPosMap[key]; - - // check for dup distinct column -- @bug6126 - if (find(keysAggPm.begin(), keysAggPm.end(), key) != keysAggPm.end()) - continue; - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); - groupByPm.push_back(groupby); - - // PM: just copy down to aggregation rowgroup - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(key); - scaleAggPm.push_back(scaleProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - - aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); - colAggPm++; - } - - // vectors for aggregate functions - for (uint64_t i = 0; i < aggColVec.size(); i++) - { - pUDAFFunc = NULL; - uint32_t aggKey = aggColVec[i].first; - RowAggFunctionType aggOp = functionIdMap(aggColVec[i].second); - RowAggFunctionType stats = statsFuncIdMap(aggColVec[i].second); - - // skip on PM if this is a constant - if (aggOp == ROWAGG_CONSTANT) - continue; - - if (projColPosMap.find(aggKey) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[aggKey] << "' isn't in tuple."; - cerr << "prep2PhasesAggregate: aggregate " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[aggKey].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[aggKey].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - if ((aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) && - (avgSet.find(aggKey) != avgSet.end())) - // skip sum / count(column) if avg is also selected - continue; - - uint64_t colProj = projColPosMap[aggKey]; - SP_ROWAGG_FUNC_t funct; - - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - for (; it != jobInfo.projectionCols.end(); it++) - { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < aggColVec.size() && aggColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(aggColVec[k].first); - } - } - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(1)prep2PhasesAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } - } - else - { - funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); - } - - // skip if this is a duplicate - if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM - && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) - { - // skip if this is a duplicate - continue; - } - - functionVecPm.push_back(funct); - aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); - - switch (aggOp) - { - case ROWAGG_MIN: - case ROWAGG_MAX: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - colAggPm++; - } - break; - - case ROWAGG_SUM: - case ROWAGG_AVG: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("sum/average"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep2PhasesAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - wideDecimalOrLongDouble(colProj, typeProj[colProj], - precisionProj, scaleProj, width, - typeAggPm, scaleAggPm, precisionAggPm, widthAggPm); - - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - csNumAggPm.push_back(8); - colAggPm++; - } - - // PM: put the count column for avg next to the sum - // let fall through to add a count column for average function - if (aggOp != ROWAGG_AVG) - break; - /* fall through */ - - case ROWAGG_COUNT_ASTERISK: - case ROWAGG_COUNT_COL_NAME: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - // work around count() in select subquery - precisionAggPm.push_back(9999); - typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggPm.push_back(8); - widthAggPm.push_back(bigIntWidth); - colAggPm++; - } - break; - - case ROWAGG_STATS: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("variance/standard deviation"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep2PhaseAggregate:: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - // counts(x) - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(0); - typeAggPm.push_back(CalpontSystemCatalog::DOUBLE); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(double)); - funct->fAuxColumnIndex = ++colAggPm; - - // sum(x) - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-1); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(long double)); - ++colAggPm; - - // sum(x**2) - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-1); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(long double)); - ++colAggPm; - } - break; - - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-16); // for connector to skip null check - - if (isUnsigned(typeProj[colProj])) - { - typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); - } - - csNumAggPm.push_back(8); - widthAggPm.push_back(bigIntWidth); - colAggPm++; - } - break; - - case ROWAGG_UDAF: - { - // Return column - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); - - if (!udafFuncCol) - { - throw logic_error("(2)prep2PhasesAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(udafFuncCol->fUDAFContext.getScale()); - precisionAggPm.push_back(udafFuncCol->fUDAFContext.getPrecision()); - typeAggPm.push_back(udafFuncCol->fUDAFContext.getResultType()); - csNumAggPm.push_back(udafFuncCol->fUDAFContext.getCharsetNumber()); - widthAggPm.push_back(udafFuncCol->fUDAFContext.getColWidth()); - ++colAggPm; - // Column for index of UDAF UserData struct - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(0); - typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggPm.push_back(8); - widthAggPm.push_back(bigUintWidth); - funct->fAuxColumnIndex = colAggPm++; - // If the first param is const - udafcParamIdx = 0; - ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); - if (cc) - { - funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; - } - ++udafcParamIdx; - break; - } - - case ROWAGG_MULTI_PARM: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - colAggPm++; - // If the param is const - if (udafc) - { - if (udafcParamIdx > udafc->aggParms().size() - 1) - { - throw QueryDataExcept("prep2PhasesAggregate: UDAF multi function with too many parms", aggregateFuncErr); - } - ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); - if (cc) - { - funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; - } - } - else - { - throw QueryDataExcept("prep2PhasesAggregate: UDAF multi function with no parms", aggregateFuncErr); - } - ++udafcParamIdx; - } - break; - - default: - { - ostringstream emsg; - emsg << "aggregate function (" << (uint64_t) aggOp << ") isn't supported"; - cerr << "prep2PhasesAggregate: " << emsg.str() << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); - } - } - } - } - - // associate the columns between the aggregate RGs on PM and UM - // populated the returned columns - // remove not returned groupby column - // add back sum or count(column name) if omitted due to avg column - // put count(column name) column to the end, if it is for avg only - { - // check if the count column for AVG is also a returned column, - // if so, replace the "-1" to actual position in returned vec. - map avgFuncMap; - AGG_MAP aggDupFuncMap; - - projColsUDAFIdx = 0; - // copy over the groupby vector - // update the outputColumnIndex if returned - for (uint64_t i = 0; i < groupByPm.size(); i++) - { - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(groupByPm[i]->fOutputColumnIndex, -1)); - groupByUm.push_back(groupby); - } - - // locate the return column position in aggregated rowgroup from PM - // outIdx is i without the multi-columns, - uint64_t outIdx = 0; - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - uint32_t retKey = returnedColVec[i].first; - RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); - RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); - int colPm = -1; - - if (aggOp == ROWAGG_MULTI_PARM) - { - // Skip on UM: Extra parms for an aggregate have no work on the UM - continue; - } - - // Is this a UDAF? use the function as part of the key. - pUDAFFunc = NULL; - udafc = NULL; - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - - for (; it != jobInfo.projectionCols.end(); it++) - { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); - } - } - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(3)prep2PhasesAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } - } - - AGG_MAP::iterator it = aggFuncMap.find(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - colPm = it->second; - oidsAggUm.push_back(oidsAggPm[colPm]); - keysAggUm.push_back(retKey); - scaleAggUm.push_back(scaleAggPm[colPm]); - precisionAggUm.push_back(precisionAggPm[colPm]); - typeAggUm.push_back(typeAggPm[colPm]); - csNumAggUm.push_back(csNumAggPm[colPm]); - widthAggUm.push_back(widthAggPm[colPm]); - } - - // not a direct hit -- a returned column is not already in the RG from PMs - else - { - bool returnColMissing = true; - - // check if a SUM or COUNT covered by AVG - if (aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) - { - it = aggFuncMap.find(boost::make_tuple(returnedColVec[i].first, ROWAGG_AVG, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - // false alarm - returnColMissing = false; - - colPm = it->second; - - if (aggOp == ROWAGG_SUM) - { - wideDecimalOrLongDouble(colPm, typeAggPm[colPm], - precisionAggPm, scaleAggPm, widthAggPm, - typeAggUm, scaleAggUm, precisionAggUm, widthAggUm); - - oidsAggUm.push_back(oidsAggPm[colPm]); - keysAggUm.push_back(retKey); - csNumAggUm.push_back(8); - } - else - { - // leave the count() to avg - aggOp = ROWAGG_COUNT_NO_OP; - - colPm++; - oidsAggUm.push_back(oidsAggPm[colPm]); - keysAggUm.push_back(retKey); - scaleAggUm.push_back(0); - precisionAggUm.push_back(19); - typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggUm.push_back(8); - widthAggUm.push_back(bigIntWidth); - } - } - } - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), - retKey) != jobInfo.expressionVec.end()) - { - // a function on aggregation - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggUm.push_back(ti.oid); - keysAggUm.push_back(retKey); - scaleAggUm.push_back(ti.scale); - precisionAggUm.push_back(ti.precision); - typeAggUm.push_back(ti.dtype); - csNumAggUm.push_back(ti.csNum); - widthAggUm.push_back(ti.width); - - returnColMissing = false; - } - else if (jobInfo.windowSet.find(retKey) != jobInfo.windowSet.end()) - { - // an window function - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggUm.push_back(ti.oid); - keysAggUm.push_back(retKey); - scaleAggUm.push_back(ti.scale); - precisionAggUm.push_back(ti.precision); - typeAggUm.push_back(ti.dtype); - csNumAggUm.push_back(ti.csNum); - widthAggUm.push_back(ti.width); - - returnColMissing = false; - } - else if (aggOp == ROWAGG_CONSTANT) - { - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggUm.push_back(ti.oid); - keysAggUm.push_back(retKey); - scaleAggUm.push_back(ti.scale); - precisionAggUm.push_back(ti.precision); - typeAggUm.push_back(ti.dtype); - csNumAggUm.push_back(ti.csNum); - widthAggUm.push_back(ti.width); - - returnColMissing = false; - } - - - if (returnColMissing) - { - Message::Args args; - args.add(keyName(outIdx, retKey, jobInfo)); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - cerr << "prep2PhasesAggregate: " << emsg << " oid=" - << (int) jobInfo.keyInfo->tupleKeyVec[retKey].fId << ", alias=" - << jobInfo.keyInfo->tupleKeyVec[retKey].fTable << ", view=" - << jobInfo.keyInfo->tupleKeyVec[retKey].fView << ", function=" - << (int) aggOp << endl; - throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); - } - } - - // update groupby vector if the groupby column is a returned column - if (returnedColVec[i].second == 0) - { - int dupGroupbyIndex = -1; - - for (uint64_t j = 0; j < jobInfo.groupByColVec.size(); j++) - { - if (jobInfo.groupByColVec[j] == retKey) - { - if (groupByUm[j]->fOutputColumnIndex == (uint32_t) - 1) - groupByUm[j]->fOutputColumnIndex = outIdx; - else - dupGroupbyIndex = groupByUm[j]->fOutputColumnIndex; - } - } - - for (uint64_t j = 0; j < jobInfo.distinctColVec.size(); j++) - { - if (jobInfo.distinctColVec[j] == retKey) - { - if (groupByUm[j]->fOutputColumnIndex == (uint32_t) - 1) - groupByUm[j]->fOutputColumnIndex = outIdx; - else - dupGroupbyIndex = groupByUm[j]->fOutputColumnIndex; - } - } - - // a duplicate group by column - if (dupGroupbyIndex != -1) - functionVecUm.push_back(SP_ROWAGG_FUNC_t(new RowAggFunctionCol( - ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, dupGroupbyIndex))); - } - else - { - // update the aggregate function vector - SP_ROWAGG_FUNC_t funct; - if (aggOp == ROWAGG_UDAF) - { - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colPm, outIdx)); - } - else - { - funct.reset(new RowAggFunctionCol(aggOp, stats, colPm, outIdx)); - } - - if (aggOp == ROWAGG_COUNT_NO_OP) - funct->fAuxColumnIndex = colPm; - else if (aggOp == ROWAGG_CONSTANT) - funct->fAuxColumnIndex = jobInfo.cntStarPos; - - functionVecUm.push_back(funct); - - // find if this func is a duplicate - AGG_MAP::iterator iter = aggDupFuncMap.find(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (iter != aggDupFuncMap.end()) - { - if (funct->fAggFunction == ROWAGG_AVG) - funct->fAggFunction = ROWAGG_DUP_AVG; - else if (funct->fAggFunction == ROWAGG_STATS) - funct->fAggFunction = ROWAGG_DUP_STATS; - else if (funct->fAggFunction == ROWAGG_UDAF) - funct->fAggFunction = ROWAGG_DUP_UDAF; - else - funct->fAggFunction = ROWAGG_DUP_FUNCT; - - funct->fAuxColumnIndex = iter->second; - } - else - { - aggDupFuncMap.insert(make_pair(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), - funct->fOutputColumnIndex)); - } - - if (returnedColVec[i].second == AggregateColumn::AVG) - avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); - } - ++outIdx; - } - - // now fix the AVG function, locate the count(column) position - for (uint64_t i = 0; i < functionVecUm.size(); i++) - { - if (functionVecUm[i]->fAggFunction != ROWAGG_COUNT_NO_OP) - continue; - - // if the count(k) can be associated with an avg(k) - map::iterator k = - avgFuncMap.find(keysAggUm[functionVecUm[i]->fOutputColumnIndex]); - - if (k != avgFuncMap.end()) - k->second->fAuxColumnIndex = functionVecUm[i]->fOutputColumnIndex; - } - - // there is avg(k), but no count(k) in the select list - uint64_t lastCol = outIdx; - - for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) - { - if (k->second->fAuxColumnIndex == (uint32_t) - 1) - { - k->second->fAuxColumnIndex = lastCol++; - oidsAggUm.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); - keysAggUm.push_back(k->first); - scaleAggUm.push_back(0); - precisionAggUm.push_back(19); - typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggUm.push_back(8); - widthAggUm.push_back(bigIntWidth); - } - } - - // add auxiliary fields for UDAF and statistics functions - for (uint64_t i = 0; i < functionVecUm.size(); i++) - { - uint64_t j = functionVecUm[i]->fInputColumnIndex; - - if (functionVecUm[i]->fAggFunction == ROWAGG_UDAF) - { - // Column for index of UDAF UserData struct - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVecUm[i].get()); - - if (!udafFuncCol) - { - throw logic_error("(4)prep2PhasesAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - functionVecUm[i]->fAuxColumnIndex = lastCol++; - oidsAggUm.push_back(oidsAggPm[j]); // Dummy? - keysAggUm.push_back(keysAggPm[j]); // Dummy? - scaleAggUm.push_back(0); - precisionAggUm.push_back(0); - typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggUm.push_back(8); - widthAggUm.push_back(bigUintWidth); - continue; - } - - if (functionVecUm[i]->fAggFunction != ROWAGG_STATS) - continue; - - functionVecUm[i]->fAuxColumnIndex = lastCol; - - // sum(x) - oidsAggUm.push_back(oidsAggPm[j]); - keysAggUm.push_back(keysAggPm[j]); - scaleAggUm.push_back(0); - precisionAggUm.push_back(-1); - typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggUm.push_back(8); - widthAggUm.push_back(sizeof(long double)); - ++lastCol; - - // sum(x**2) - oidsAggUm.push_back(oidsAggPm[j]); - keysAggUm.push_back(keysAggPm[j]); - scaleAggUm.push_back(0); - precisionAggUm.push_back(-1); - typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggUm.push_back(8); - widthAggUm.push_back(sizeof(long double)); - ++lastCol; - } - } - - // calculate the offset and create the rowaggregations, rowgroups - posAggUm.push_back(2); // rid - - for (uint64_t i = 0; i < oidsAggUm.size(); i++) - posAggUm.push_back(posAggUm[i] + widthAggUm[i]); - - RowGroup aggRgUm(oidsAggUm.size(), posAggUm, oidsAggUm, keysAggUm, typeAggUm, - csNumAggUm, scaleAggUm, precisionAggUm, jobInfo.stringTableThreshold); - SP_ROWAGG_UM_t rowAggUm(new RowAggregationUMP2(groupByUm, functionVecUm, jobInfo.rm, jobInfo.umMemLimit)); - rowAggUm->timeZone(jobInfo.timeZone); - rowgroups.push_back(aggRgUm); - aggregators.push_back(rowAggUm); - - posAggPm.push_back(2); // rid - - for (uint64_t i = 0; i < oidsAggPm.size(); i++) - posAggPm.push_back(posAggPm[i] + widthAggPm[i]); - - RowGroup aggRgPm(oidsAggPm.size(), posAggPm, oidsAggPm, keysAggPm, typeAggPm, - csNumAggPm, scaleAggPm, precisionAggPm, jobInfo.stringTableThreshold); - SP_ROWAGG_PM_t rowAggPm(new RowAggregation(groupByPm, functionVecPm)); - rowAggPm->timeZone(jobInfo.timeZone); - rowgroups.push_back(aggRgPm); - aggregators.push_back(rowAggPm); - - if (jobInfo.trace) - cout << "\n====== Aggregation RowGroups ======" << endl - << "projected RG: " << projRG.toString() << endl - << "aggregated1 RG: " << aggRgPm.toString() << endl - << "aggregated2 RG: " << aggRgUm.toString() << endl; -} - - -void TupleAggregateStep::prep2PhasesDistinctAggregate( - JobInfo& jobInfo, - vector& rowgroups, - vector& aggregators) -{ - // check if there are any aggregate columns - // a vector that has the aggregate function to be done by PM - vector > aggColVec, aggNoDistColVec; - set avgSet, avgDistSet; - vector >& returnedColVec = jobInfo.returnedColVec; - // For UDAF - uint32_t projColsUDAFIdx = 0; - uint32_t udafcParamIdx = 0; - UDAFColumn* udafc = NULL; - mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; - - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - // col should be an aggregate or groupBy or window function - uint32_t rtcKey = returnedColVec[i].first; - uint32_t rtcOp = returnedColVec[i].second; - - if (rtcOp == 0 && - find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), rtcKey) != - jobInfo.distinctColVec.end() && - find(jobInfo.groupByColVec.begin(), jobInfo.groupByColVec.end(), rtcKey ) == - jobInfo.groupByColVec.end() && - jobInfo.windowSet.find(rtcKey) != jobInfo.windowSet.end()) - { - Message::Args args; - args.add(keyName(i, rtcKey, jobInfo)); - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - cerr << "prep2PhasesDistinctAggregate: " << emsg << " oid=" - << (int) jobInfo.keyInfo->tupleKeyVec[rtcKey].fId << ", alias=" - << jobInfo.keyInfo->tupleKeyVec[rtcKey].fTable << ", view=" - << jobInfo.keyInfo->tupleKeyVec[rtcKey].fView << endl; - throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); - } - - // skip if not an aggregation column - if (returnedColVec[i].second == 0) - continue; - - aggColVec.push_back(returnedColVec[i]); - - // remember if a column has an average function, - // with avg function, no need for separate sum or count_column_name - if (returnedColVec[i].second == AggregateColumn::AVG) - avgSet.insert(returnedColVec[i].first); - - if ( returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) - avgDistSet.insert(returnedColVec[i].first); - } - - // populate the aggregate rowgroup on PM and UM - // PM: projectedRG -> aggregateRGPM - // UM: aggregateRGPM -> aggregateRGUM - // - // Aggregate preparation by joblist factory: - // 1. get projected rowgroup (done by doAggProject) -- input to PM AGG - // 2. construct aggregate rowgroup -- output of PM, input of UM - // 3. construct aggregate rowgroup -- output of UM - // 4. construct aggregate rowgroup -- output of distinct aggregates - - const RowGroup projRG = rowgroups[0]; - const vector& oidsProj = projRG.getOIDs(); - const vector& keysProj = projRG.getKeys(); - const vector& scaleProj = projRG.getScale(); - const vector& precisionProj = projRG.getPrecision(); - const vector& typeProj = projRG.getColTypes(); - const vector& csNumProj = projRG.getCharsetNumbers(); - - vector posAggPm, posAggUm, posAggDist; - vector oidsAggPm, oidsAggUm, oidsAggDist; - vector keysAggPm, keysAggUm, keysAggDist; - vector scaleAggPm, scaleAggUm, scaleAggDist; - vector precisionAggPm, precisionAggUm, precisionAggDist; - vector typeAggPm, typeAggUm, typeAggDist; - vector csNumAggPm, csNumAggUm, csNumAggDist; - vector widthAggPm, widthAggUm, widthAggDist; - - vector groupByPm, groupByUm, groupByNoDist; - vector functionVecPm, functionNoDistVec, functionVecUm; - list multiParmIndexes; - - uint32_t bigIntWidth = sizeof(int64_t); - map, uint64_t> avgFuncDistMap; - AGG_MAP aggFuncMap; - - // associate the columns between projected RG and aggregate RG on PM - // populated the aggregate columns - // the groupby columns are put in front, even not a returned column - // sum and count(column name) are omitted, if avg present - { - // project only unique oids, but they may be repeated in aggregation - // collect the projected column info, prepare for aggregation - vector width; - map projColPosMap; - - for (uint64_t i = 0; i < keysProj.size(); i++) - { - projColPosMap.insert(make_pair(keysProj[i], i)); - width.push_back(projRG.getColumnWidth(i)); - } - - // column index for PM aggregate rowgroup - uint64_t colAggPm = 0; - uint64_t multiParm = 0; - - // for groupby column - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - uint32_t key = jobInfo.groupByColVec[i]; - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep2PhasesDistinctAggregate: group " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - uint64_t colProj = projColPosMap[key]; - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); - groupByPm.push_back(groupby); - - // PM: just copy down to aggregation rowgroup - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(key); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - - aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); - colAggPm++; - } - - // for distinct column - for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) - { - uint32_t key = jobInfo.distinctColVec[i]; - - if (projColPosMap.find(key) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; - cerr << "prep2PhasesDistinctAggregate: distinct " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[key].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - // check for dup distinct column -- @bug6126 - if (find(keysAggPm.begin(), keysAggPm.end(), key) != keysAggPm.end()) - continue; - - uint64_t colProj = projColPosMap[key]; - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); - groupByPm.push_back(groupby); - - // PM: just copy down to aggregation rowgroup - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(key); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - - aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); - colAggPm++; - } - - // vectors for aggregate functions - RowAggFunctionType aggOp = ROWAGG_FUNCT_UNDEFINE; - RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; - for (uint64_t i = 0; i < aggColVec.size(); i++) - { - aggOp = functionIdMap(aggColVec[i].second); - - // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) - if (aggOp != ROWAGG_MULTI_PARM) - prevAggOp = aggOp; - - // skip on PM if this is a constant - if (aggOp == ROWAGG_CONSTANT) - continue; - - pUDAFFunc = NULL; - uint32_t aggKey = aggColVec[i].first; - - if (projColPosMap.find(aggKey) == projColPosMap.end()) - { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[aggKey] << "' isn't in tuple."; - cerr << "prep2PhasesDistinctAggregate: aggregate " << emsg.str() - << " oid=" << (int) jobInfo.keyInfo->tupleKeyVec[aggKey].fId - << ", alias=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fTable; - - if (jobInfo.keyInfo->tupleKeyVec[aggKey].fView.length() > 0) - cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fView; - - cerr << endl; - throw logic_error(emsg.str()); - } - - RowAggFunctionType stats = statsFuncIdMap(aggOp); - - // skip sum / count(column) if avg is also selected - if ((aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) && - (avgSet.find(aggKey) != avgSet.end())) - continue; - - // We skip distinct aggs, including extra parms. These are handled by adding them to group by list above. - if (aggOp == ROWAGG_DISTINCT_SUM || - aggOp == ROWAGG_DISTINCT_AVG || - aggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) - continue; - if (aggOp == ROWAGG_MULTI_PARM && prevAggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) - continue; - - - uint64_t colProj = projColPosMap[aggKey]; - SP_ROWAGG_FUNC_t funct; - - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - for (; it != jobInfo.projectionCols.end(); it++) - { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < aggColVec.size() && aggColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(aggColVec[k].first); - } - } - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(1)prep2PhasesDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } - } - else - { - funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); - } - - // skip if this is a duplicate - if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM - && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) - { - // skip if this is a duplicate - continue; - } - - functionVecPm.push_back(funct); - aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm-multiParm)); - - switch (aggOp) - { - case ROWAGG_MIN: - case ROWAGG_MAX: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - colAggPm++; - } - break; - - case ROWAGG_SUM: - case ROWAGG_AVG: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("sum/average"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep2PhasesDistinctAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - csNumAggPm.push_back(8); - wideDecimalOrLongDouble(colProj, typeProj[colProj], - precisionProj, scaleProj, width, - typeAggPm, scaleAggPm, precisionAggPm, widthAggPm); - colAggPm++; - } - - // PM: put the count column for avg next to the sum - // let fall through to add a count column for average function - if (aggOp == ROWAGG_AVG) - funct->fAuxColumnIndex = colAggPm; - else - break; - /* fall through */ - - case ROWAGG_COUNT_ASTERISK: - case ROWAGG_COUNT_COL_NAME: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - // work around count() in select subquery - precisionAggPm.push_back(9999); - - if (isUnsigned(typeProj[colProj])) - { - typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); - } - - csNumAggPm.push_back(8); - widthAggPm.push_back(bigIntWidth); - colAggPm++; - } - break; - - case ROWAGG_STATS: - { - if (typeProj[colProj] == CalpontSystemCatalog::CHAR || - typeProj[colProj] == CalpontSystemCatalog::VARCHAR || - typeProj[colProj] == CalpontSystemCatalog::BLOB || - typeProj[colProj] == CalpontSystemCatalog::TEXT || - typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME || - typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || - typeProj[colProj] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("variance/standard deviation"); - args.add(colTypeIdString(typeProj[colProj])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep2PhasesDistinctAggregate:: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - // count(x) - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(0); - typeAggPm.push_back(CalpontSystemCatalog::DOUBLE); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(double)); - funct->fAuxColumnIndex = ++colAggPm; - - // sum(x) - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-1); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(long double)); - ++colAggPm; - - // sum(x**2) - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-1); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(long double)); - ++colAggPm; - } - break; - - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-16); // for connector to skip null check - - if (isUnsigned(typeProj[colProj])) - { - typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); - } - - csNumAggPm.push_back(8); - widthAggPm.push_back(bigIntWidth); - ++colAggPm; - } - break; - - case ROWAGG_UDAF: - { - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); - - if (!udafFuncCol) - { - throw logic_error("(2)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - // Return column - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(udafFuncCol->fUDAFContext.getScale()); - precisionAggPm.push_back(udafFuncCol->fUDAFContext.getPrecision()); - typeAggPm.push_back(udafFuncCol->fUDAFContext.getResultType()); - csNumAggPm.push_back(udafFuncCol->fUDAFContext.getCharsetNumber()); - widthAggPm.push_back(udafFuncCol->fUDAFContext.getColWidth()); - ++colAggPm; - // Column for index of UDAF UserData struct - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(0); - precisionAggPm.push_back(0); - typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggPm.push_back(8); - widthAggPm.push_back(sizeof(uint64_t)); - funct->fAuxColumnIndex = colAggPm++; - // If the first param is const - udafcParamIdx = 0; - ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); - if (cc) - { - funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; - } - ++udafcParamIdx; - break; - } - - case ROWAGG_MULTI_PARM: - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - scaleAggPm.push_back(scaleProj[colProj]); - precisionAggPm.push_back(precisionProj[colProj]); - typeAggPm.push_back(typeProj[colProj]); - csNumAggPm.push_back(csNumProj[colProj]); - widthAggPm.push_back(width[colProj]); - multiParmIndexes.push_back(colAggPm); - ++colAggPm; - ++multiParm; - // If the param is const - if (udafc) - { - if (udafcParamIdx > udafc->aggParms().size() - 1) - { - throw QueryDataExcept("prep2PhasesDistinctAggregate: UDAF multi function with too many parms", aggregateFuncErr); - } - ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); - if (cc) - { - funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; - } - } - else if (prevAggOp != ROWAGG_COUNT_DISTINCT_COL_NAME) - { - throw QueryDataExcept("prep2PhasesDistinctAggregate: UDAF multi function with no parms", aggregateFuncErr); - } - ++udafcParamIdx; - } - break; - - default: - { - ostringstream emsg; - emsg << "aggregate function (" << (uint64_t) aggOp << ") isn't supported"; - cerr << "prep2PhasesDistinctAggregate: " << emsg.str() << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); - } - } - } - } - - - // associate the columns between the aggregate RGs on PM and UM without distinct aggregator - // populated the returned columns - { - int64_t multiParms = 0; - - for (uint32_t idx = 0; idx < groupByPm.size(); idx++) - { - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(idx, idx)); - groupByUm.push_back(groupby); - } - - for (uint32_t idx = 0; idx < functionVecPm.size(); idx++) - { - SP_ROWAGG_FUNC_t funct; - SP_ROWAGG_FUNC_t funcPm = functionVecPm[idx]; - - if (funcPm->fAggFunction == ROWAGG_MULTI_PARM) - { - // Skip on UM: Extra parms for an aggregate have no work on the UM - ++multiParms; - continue; - } - - if (funcPm->fAggFunction == ROWAGG_UDAF) - { - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funcPm.get()); - if (!udafFuncCol) - { - throw logic_error("(3)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - funct.reset(new RowUDAFFunctionCol( - udafFuncCol->fUDAFContext, - udafFuncCol->fOutputColumnIndex, - udafFuncCol->fOutputColumnIndex-multiParms, - udafFuncCol->fAuxColumnIndex-multiParms)); - functionNoDistVec.push_back(funct); - pUDAFFunc = udafFuncCol->fUDAFContext.getFunction(); - } - else - { - funct.reset(new RowAggFunctionCol( - funcPm->fAggFunction, - funcPm->fStatsFunction, - funcPm->fOutputColumnIndex, - funcPm->fOutputColumnIndex-multiParms, - funcPm->fAuxColumnIndex-multiParms)); - functionNoDistVec.push_back(funct); - pUDAFFunc = NULL; - } - } - - // Copy over the PM arrays to the UM. Skip any that are a multi-parm entry. - for (uint32_t idx = 0; idx < oidsAggPm.size(); ++idx) - { - if (find (multiParmIndexes.begin(), multiParmIndexes.end(), idx ) != multiParmIndexes.end()) - { - continue; - } - oidsAggUm.push_back(oidsAggPm[idx]); - keysAggUm.push_back(keysAggPm[idx]); - scaleAggUm.push_back(scaleAggPm[idx]); - precisionAggUm.push_back(precisionAggPm[idx]); - widthAggUm.push_back(widthAggPm[idx]); - typeAggUm.push_back(typeAggPm[idx]); - csNumAggUm.push_back(csNumAggPm[idx]); - } - } - - - // associate the columns between the aggregate RGs on UM and Distinct - // populated the returned columns - // remove not returned groupby column - // add back sum or count(column name) if omitted due to avg column - // put count(column name) column to the end, if it is for avg only - { - // Keep a count of the parms after the first for any aggregate. - // These will be skipped and the count needs to be subtracted - // from where the aux column will be. - int64_t multiParms = 0; - projColsUDAFIdx = 0; - // check if the count column for AVG is also a returned column, - // if so, replace the "-1" to actual position in returned vec. - map avgFuncMap, avgDistFuncMap; - AGG_MAP aggDupFuncMap; - - // copy over the groupby vector - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(i, -1)); - groupByNoDist.push_back(groupby); - } - - // locate the return column position in aggregated rowgroup from PM - // outIdx is i without the multi-columns, - uint64_t outIdx = 0; - RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; - uint32_t prevRetKey = 0; - - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - pUDAFFunc = NULL; - udafc = NULL; - uint32_t retKey = returnedColVec[i].first; - - RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); - RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); - int colUm = -1; - - if (aggOp == ROWAGG_MULTI_PARM) - { - // Duplicate detection doesn't work for multi-parm` - - // If this function was earlier detected as a duplicate, unduplicate it. - SP_ROWAGG_FUNC_t funct = functionVecUm.back(); - if (funct->fAggFunction == ROWAGG_DUP_FUNCT) - funct->fAggFunction = prevAggOp; - - // Remove it from aggDupFuncMap if it's in there. - funct->hasMultiParm = true; - AGG_MAP::iterator it = aggDupFuncMap.find(boost::make_tuple(prevRetKey, prevAggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - if (it != aggDupFuncMap.end()) - { - aggDupFuncMap.erase(it); - } - // Skip further UM porocessing of the multi-parm: Extra parms for an aggregate have no work on the UM - ++multiParms; - continue; - } - else - { - // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) - prevAggOp = aggOp; - prevRetKey = returnedColVec[i].first; - } - - if (aggOp == ROWAGG_UDAF) - { - std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; - for (; it != jobInfo.projectionCols.end(); it++) - { - udafc = dynamic_cast((*it).get()); - projColsUDAFIdx++; - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Save the multi-parm keys for dup-detection. - if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) - { - for (uint64_t k = i+1; - k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; - ++k) - { - udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); - } - } - break; - } - } - if (it == jobInfo.projectionCols.end()) - { - throw logic_error("(4)prep2PhasesDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); - } - } - - if (find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), retKey) != - jobInfo.distinctColVec.end() ) - { - AGG_MAP::iterator it = aggFuncMap.find(boost::make_tuple(retKey, 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - colUm = it->second; - } - } - - if (colUm > -1) // Means we found a DISTINCT and have a column number - { - switch (aggOp) - { - case ROWAGG_DISTINCT_AVG: - - //avgFuncMap.insert(make_pair(key, funct)); - case ROWAGG_DISTINCT_SUM: - { - if (typeAggUm[colUm] == CalpontSystemCatalog::CHAR || - typeAggUm[colUm] == CalpontSystemCatalog::VARCHAR || - typeAggUm[colUm] == CalpontSystemCatalog::BLOB || - typeAggUm[colUm] == CalpontSystemCatalog::TEXT || - typeAggUm[colUm] == CalpontSystemCatalog::DATE || - typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || - typeAggUm[colUm] == CalpontSystemCatalog::TIMESTAMP || - typeAggUm[colUm] == CalpontSystemCatalog::TIME) - { - Message::Args args; - args.add("sum/average"); - args.add(colTypeIdString(typeAggUm[colUm])); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); - cerr << "prep2PhasesDistinctAggregate: " << emsg << endl; - throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); - } - - oidsAggDist.push_back(oidsAggUm[colUm]); - keysAggDist.push_back(retKey); - csNumAggDist.push_back(8); - wideDecimalOrLongDouble(colUm, typeAggPm[colUm], - precisionAggPm, scaleAggPm, widthAggPm, - typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); - } - // PM: put the count column for avg next to the sum - // let fall through to add a count column for average function - //if (aggOp != ROWAGG_DISTINCT_AVG) - break; - - case ROWAGG_COUNT_DISTINCT_COL_NAME: - { - oidsAggDist.push_back(oidsAggUm[colUm]); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(0); - // work around count() in select subquery - precisionAggDist.push_back(9999); - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - break; - - default: - // could happen if agg and agg distinct use same column. - colUm = -1; - break; - } // switch - } - // For non distinct aggregates - if (colUm == -1) - { - AGG_MAP::iterator it = aggFuncMap.find(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - colUm = it->second; - oidsAggDist.push_back(oidsAggUm[colUm]); - keysAggDist.push_back(keysAggUm[colUm]); - scaleAggDist.push_back(scaleAggUm[colUm]); - precisionAggDist.push_back(precisionAggUm[colUm]); - typeAggDist.push_back(typeAggUm[colUm]); - csNumAggDist.push_back(csNumAggUm[colUm]); - widthAggDist.push_back(widthAggUm[colUm]); - } - - // not a direct hit -- a returned column is not already in the RG from PMs - else - { - bool returnColMissing = true; - - // check if a SUM or COUNT covered by AVG - if (aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) - { - it = aggFuncMap.find(boost::make_tuple(returnedColVec[i].first, ROWAGG_AVG, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (it != aggFuncMap.end()) - { - // false alarm - returnColMissing = false; - - colUm = it->second; - - if (aggOp == ROWAGG_SUM) - { - oidsAggDist.push_back(oidsAggUm[colUm]); - keysAggDist.push_back(retKey); - csNumAggDist.push_back(8); - wideDecimalOrLongDouble(colUm, typeAggUm[colUm], - precisionAggUm, scaleAggUm, widthAggUm, - typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); - } - else - { - // leave the count() to avg - aggOp = ROWAGG_COUNT_NO_OP; - - oidsAggDist.push_back(oidsAggUm[colUm]); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(0); - if (isUnsigned(typeAggUm[colUm])) - { - precisionAggDist.push_back(20); - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - } - else - { - precisionAggDist.push_back(19); - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); - } - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - } - } - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), - retKey) != jobInfo.expressionVec.end()) - { - // a function on aggregation - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggDist.push_back(ti.oid); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(ti.scale); - precisionAggDist.push_back(ti.precision); - typeAggDist.push_back(ti.dtype); - csNumAggDist.push_back(ti.csNum); - widthAggDist.push_back(ti.width); - - returnColMissing = false; - } - else if (jobInfo.windowSet.find(retKey) != jobInfo.windowSet.end()) - { - // a window function - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggDist.push_back(ti.oid); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(ti.scale); - precisionAggDist.push_back(ti.precision); - typeAggDist.push_back(ti.dtype); - csNumAggDist.push_back(ti.csNum); - widthAggDist.push_back(ti.width); - - returnColMissing = false; - } - else if (aggOp == ROWAGG_CONSTANT) - { - TupleInfo ti = getTupleInfo(retKey, jobInfo); - oidsAggDist.push_back(ti.oid); - keysAggDist.push_back(retKey); - scaleAggDist.push_back(ti.scale); - precisionAggDist.push_back(ti.precision); - typeAggDist.push_back(ti.dtype); - csNumAggDist.push_back(ti.csNum); - widthAggDist.push_back(ti.width); - - returnColMissing = false; - } - - if (returnColMissing) - { - Message::Args args; - args.add(keyName(outIdx, retKey, jobInfo)); - string emsg = IDBErrorInfo::instance()-> - errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - cerr << "prep2PhasesDistinctAggregate: " << emsg << " oid=" - << (int) jobInfo.keyInfo->tupleKeyVec[retKey].fId << ", alias=" - << jobInfo.keyInfo->tupleKeyVec[retKey].fTable << ", view=" - << jobInfo.keyInfo->tupleKeyVec[retKey].fView << ", function=" - << (int) aggOp << endl; - throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); - } - } //else not a direct hit - } // else not a DISTINCT - - // update groupby vector if the groupby column is a returned column - if (returnedColVec[i].second == 0) - { - int dupGroupbyIndex = -1; - - for (uint64_t j = 0; j < jobInfo.groupByColVec.size(); j++) - { - if (jobInfo.groupByColVec[j] == retKey) - { - if (groupByNoDist[j]->fOutputColumnIndex == (uint32_t) - 1) - groupByNoDist[j]->fOutputColumnIndex = outIdx; - else - dupGroupbyIndex = groupByNoDist[j]->fOutputColumnIndex; - } - } - - // a duplicate group by column - if (dupGroupbyIndex != -1) - functionVecUm.push_back(SP_ROWAGG_FUNC_t(new RowAggFunctionCol( - ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, dupGroupbyIndex))); - } - else - { - // update the aggregate function vector - SP_ROWAGG_FUNC_t funct; - if (aggOp == ROWAGG_UDAF) - { - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colUm, outIdx)); - } - else - { - funct.reset(new RowAggFunctionCol(aggOp, stats, colUm, outIdx)); - } - - if (aggOp == ROWAGG_COUNT_NO_OP) - funct->fAuxColumnIndex = colUm; - else if (aggOp == ROWAGG_CONSTANT) - funct->fAuxColumnIndex = jobInfo.cntStarPos; - - functionVecUm.push_back(funct); - - // find if this func is a duplicate - AGG_MAP::iterator iter = aggDupFuncMap.find(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - - if (iter != aggDupFuncMap.end()) - { - if (funct->fAggFunction == ROWAGG_AVG) - funct->fAggFunction = ROWAGG_DUP_AVG; - else if (funct->fAggFunction == ROWAGG_STATS) - funct->fAggFunction = ROWAGG_DUP_STATS; - else if (funct->fAggFunction == ROWAGG_UDAF) - funct->fAggFunction = ROWAGG_DUP_UDAF; - else - funct->fAggFunction = ROWAGG_DUP_FUNCT; - - funct->fAuxColumnIndex = iter->second; - } - else - { - aggDupFuncMap.insert(make_pair(boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), - funct->fOutputColumnIndex)); - } - - if (returnedColVec[i].second == AggregateColumn::AVG) - avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); - else if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) - avgDistFuncMap.insert(make_pair(returnedColVec[i].first, funct)); - } - ++outIdx; - } // for (i - - // now fix the AVG function, locate the count(column) position - for (uint64_t i = 0; i < functionVecUm.size(); i++) - { - // if the count(k) can be associated with an avg(k) - if (functionVecUm[i]->fAggFunction == ROWAGG_COUNT_NO_OP) - { - map::iterator k = - avgFuncMap.find(keysAggDist[functionVecUm[i]->fOutputColumnIndex]); - - if (k != avgFuncMap.end()) - k->second->fAuxColumnIndex = functionVecUm[i]->fOutputColumnIndex; - } - } - - // there is avg(k), but no count(k) in the select list - uint64_t lastCol = outIdx; - - for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) - { - if (k->second->fAuxColumnIndex == (uint32_t) - 1) - { - k->second->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); - keysAggDist.push_back(k->first); - scaleAggDist.push_back(0); - precisionAggDist.push_back(19); - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - } - - //distinct avg - for (uint64_t i = 0; i < functionVecUm.size(); i++) - { - if (functionVecUm[i]->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME && !functionVecUm[i]->hasMultiParm) - { - map::iterator k = - avgDistFuncMap.find(keysAggDist[functionVecUm[i]->fOutputColumnIndex]); - - if (k != avgDistFuncMap.end()) - { - k->second->fAuxColumnIndex = functionVecUm[i]->fOutputColumnIndex; - functionVecUm[i]->fAggFunction = ROWAGG_COUNT_NO_OP; - } - } - } - - // there is avg(distinct k), but no count(distinct k) in the select list - for (map::iterator k = avgDistFuncMap.begin(); k != avgDistFuncMap.end(); k++) - { - // find count(distinct k) or add it - if (k->second->fAuxColumnIndex == (uint32_t) - 1) - { - k->second->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); - keysAggDist.push_back(k->first); - scaleAggDist.push_back(0); - precisionAggDist.push_back(19); - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(bigIntWidth); - } - } - - // add auxiliary fields for UDAF and statistics functions - for (uint64_t i = 0; i < functionVecUm.size(); i++) - { - uint64_t j = functionVecUm[i]->fInputColumnIndex; - - if (functionVecUm[i]->fAggFunction == ROWAGG_UDAF) - { - // Column for index of UDAF UserData struct - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVecUm[i].get()); - - if (!udafFuncCol) - { - throw logic_error("(5)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); - } - - functionVecUm[i]->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(oidsAggPm[j]); // Dummy? - keysAggDist.push_back(keysAggPm[j]); // Dummy? - scaleAggDist.push_back(0); - precisionAggDist.push_back(0); - typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); - csNumAggDist.push_back(8); - widthAggDist.push_back(sizeof(uint64_t)); - continue; - } - - if (functionVecUm[i]->fAggFunction != ROWAGG_STATS) - continue; - - functionVecUm[i]->fAuxColumnIndex = lastCol; - - // sum(x) - oidsAggDist.push_back(oidsAggPm[j]); - keysAggDist.push_back(keysAggPm[j]); - scaleAggDist.push_back(0); - precisionAggDist.push_back(-1); - typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggDist.push_back(8); - widthAggDist.push_back(sizeof(long double)); - ++lastCol; - - // sum(x**2) - oidsAggDist.push_back(oidsAggPm[j]); - keysAggDist.push_back(keysAggPm[j]); - scaleAggDist.push_back(0); - precisionAggDist.push_back(-1); - typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggDist.push_back(8); - widthAggDist.push_back(sizeof(long double)); - ++lastCol; - } - } - - - // calculate the offset and create the rowaggregations, rowgroups - posAggUm.push_back(2); // rid - - for (uint64_t i = 0; i < oidsAggUm.size(); i++) - posAggUm.push_back(posAggUm[i] + widthAggUm[i]); - - RowGroup aggRgUm(oidsAggUm.size(), posAggUm, oidsAggUm, keysAggUm, typeAggUm, - csNumAggUm, scaleAggUm, precisionAggUm, jobInfo.stringTableThreshold); - SP_ROWAGG_UM_t rowAggUm(new RowAggregationUMP2(groupByUm, functionNoDistVec, jobInfo.rm, jobInfo.umMemLimit)); - rowAggUm->timeZone(jobInfo.timeZone); - - posAggDist.push_back(2); // rid - - for (uint64_t i = 0; i < oidsAggDist.size(); i++) - posAggDist.push_back(posAggDist[i] + widthAggDist[i]); - - RowGroup aggRgDist(oidsAggDist.size(), posAggDist, oidsAggDist, keysAggDist, - typeAggDist, csNumAggDist, scaleAggDist, - precisionAggDist, jobInfo.stringTableThreshold); - SP_ROWAGG_DIST rowAggDist(new RowAggregationDistinct(groupByNoDist, functionVecUm, jobInfo.rm, jobInfo.umMemLimit)); - rowAggDist->timeZone(jobInfo.timeZone); - - // if distinct key word applied to more than one aggregate column, reset rowAggDist - vector subRgVec; - - if (jobInfo.distinctColVec.size() > 1) - { - RowAggregationMultiDistinct* multiDistinctAggregator = - new RowAggregationMultiDistinct(groupByNoDist, functionVecUm, jobInfo.rm, jobInfo.umMemLimit); - multiDistinctAggregator->timeZone(jobInfo.timeZone); - rowAggDist.reset(multiDistinctAggregator); - - // construct and add sub-aggregators to rowAggDist - vector posAggGb, posAggSub; - vector oidsAggGb, oidsAggSub; - vector keysAggGb, keysAggSub; - vector scaleAggGb, scaleAggSub; - vector precisionAggGb, precisionAggSub; - vector typeAggGb, typeAggSub; - vector csNumAggGb, csNumAggSub; - vector widthAggGb, widthAggSub; - - // populate groupby column info - for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) - { - oidsAggGb.push_back(oidsAggUm[i]); - keysAggGb.push_back(keysAggUm[i]); - scaleAggGb.push_back(scaleAggUm[i]); - precisionAggGb.push_back(precisionAggUm[i]); - typeAggGb.push_back(typeAggUm[i]); - csNumAggGb.push_back(csNumAggUm[i]); - widthAggGb.push_back(widthAggUm[i]); - } - - // for distinct, each column requires a seperate rowgroup - vector rowAggSubDistVec; - - uint32_t distinctColKey; - int64_t j; - uint64_t k; - uint64_t outIdx = 0; - for (uint64_t i = 0; i < returnedColVec.size(); i++) - { - if (returnedColVec[i].second == 0) - { - ++outIdx; - continue; - } - - j = -1; - - distinctColKey = -1; - // Find the entry in distinctColVec, if any - for (k = 0; k < jobInfo.distinctColVec.size(); k++) - { - distinctColKey = jobInfo.distinctColVec[k]; - if (returnedColVec[i].first == distinctColKey) - break; - - } - if (distinctColKey == (uint32_t)-1) - { - ++outIdx; - continue; - } - // locate the distinct key in the row group - for (k = 0; k < keysAggUm.size(); k++) - { - if (keysAggUm[k] == distinctColKey) - { - j = k; - break; - } - } - - idbassert(j != -1); - - oidsAggSub = oidsAggGb; - keysAggSub = keysAggGb; - scaleAggSub = scaleAggGb; - precisionAggSub = precisionAggGb; - typeAggSub = typeAggGb; - csNumAggSub = csNumAggGb; - widthAggSub = widthAggGb; - - oidsAggSub.push_back(oidsAggUm[j]); - keysAggSub.push_back(keysAggUm[j]); - scaleAggSub.push_back(scaleAggUm[j]); - precisionAggSub.push_back(precisionAggUm[j]); - typeAggSub.push_back(typeAggUm[j]); - csNumAggSub.push_back(csNumAggUm[j]); - widthAggSub.push_back(widthAggUm[j]); - - // construct groupby vector - vector groupBySub; - k = 0; - - while (k < jobInfo.groupByColVec.size()) - { - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(k, k)); - groupBySub.push_back(groupby); - k++; - } - // add the distinct column as groupby - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); - groupBySub.push_back(groupby); - - // Add multi parm distinct - while ((i+1) < returnedColVec.size() && functionIdMap(returnedColVec[i+1].second) == ROWAGG_MULTI_PARM) - { - ++i; - uint32_t dColKey = -1; - j = -1; - - // Find the entry in distinctColVec, if any - for (k = 0; k < jobInfo.distinctColVec.size(); k++) - { - dColKey = jobInfo.distinctColVec[k]; - if (returnedColVec[i].first == dColKey) - break; - - } - idbassert(dColKey != (uint32_t)-1); - // locate the distinct key in the row group - for (k = 0; k < keysAggUm.size(); k++) - { - if (keysAggUm[k] == dColKey) - { - j = k; - break; - } - } - idbassert(j != -1); - - oidsAggSub.push_back(oidsAggUm[j]); - keysAggSub.push_back(keysAggUm[j]); - scaleAggSub.push_back(scaleAggUm[j]); - precisionAggSub.push_back(precisionAggUm[j]); - typeAggSub.push_back(typeAggUm[j]); - csNumAggSub.push_back(csNumAggUm[j]); - widthAggSub.push_back(widthAggUm[j]); - - SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); - groupBySub.push_back(groupby); - } - - // construct sub-rowgroup - posAggSub.clear(); - posAggSub.push_back(2); // rid - - for ( k = 0; k < oidsAggSub.size(); k++) - posAggSub.push_back(posAggSub[k] + widthAggSub[k]); - - RowGroup subRg(oidsAggSub.size(), posAggSub, oidsAggSub, keysAggSub, typeAggSub, - csNumAggSub, scaleAggSub, precisionAggSub, jobInfo.stringTableThreshold); - subRgVec.push_back(subRg); - - // Keep a count of the parms after the first for any aggregate. - // These will be skipped and the count needs to be subtracted - // from where the aux column will be. - int64_t multiParms = 0; - - // tricky part : 2 function vectors - // -- dummy function vector for sub-aggregator, which does distinct only - // -- aggregate function on this distinct column for rowAggDist - vector functionSub1, functionSub2; - // search the function in functionVec - vector::iterator it = functionVecUm.begin(); - - while (it != functionVecUm.end()) - { - SP_ROWAGG_FUNC_t f = *it++; - - if ((f->fOutputColumnIndex == outIdx) && - (f->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME || - f->fAggFunction == ROWAGG_DISTINCT_SUM || - f->fAggFunction == ROWAGG_DISTINCT_AVG)) - { - SP_ROWAGG_FUNC_t funct( - new RowAggFunctionCol( - f->fAggFunction, - f->fStatsFunction, - groupBySub.size() - 1, - f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); - functionSub2.push_back(funct); - } - } - - // construct sub-aggregator - SP_ROWAGG_UM_t subAgg(new RowAggregationSubDistinct(groupBySub, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); - subAgg->timeZone(jobInfo.timeZone); - - // add to rowAggDist - multiDistinctAggregator->addSubAggregator(subAgg, subRg, functionSub2); - - ++outIdx; - } - - // cover any non-distinct column functions - { - vector functionSub1 = functionNoDistVec; - vector functionSub2; - int64_t multiParms = 0; - - for (uint64_t k = 0; k < returnedColVec.size(); k++) - { - // search non-distinct functions in functionVec - vector::iterator it = functionVecUm.begin(); - - while (it != functionVecUm.end()) - { - SP_ROWAGG_FUNC_t funct; - SP_ROWAGG_FUNC_t f = *it++; - - if (f->fOutputColumnIndex == k) - { - if (f->fAggFunction == ROWAGG_UDAF) - { - RowUDAFFunctionCol* udafFuncCol = dynamic_cast(f.get()); - funct.reset(new RowUDAFFunctionCol( - udafFuncCol->fUDAFContext, - udafFuncCol->fInputColumnIndex, - udafFuncCol->fOutputColumnIndex, - udafFuncCol->fAuxColumnIndex-multiParms)); - functionSub2.push_back(funct); - } - else if (f->fAggFunction == ROWAGG_COUNT_ASTERISK || - f->fAggFunction == ROWAGG_COUNT_COL_NAME || - f->fAggFunction == ROWAGG_SUM || - f->fAggFunction == ROWAGG_AVG || - f->fAggFunction == ROWAGG_MIN || - f->fAggFunction == ROWAGG_MAX || - f->fAggFunction == ROWAGG_STATS || - f->fAggFunction == ROWAGG_BIT_AND || - f->fAggFunction == ROWAGG_BIT_OR || - f->fAggFunction == ROWAGG_BIT_XOR || - f->fAggFunction == ROWAGG_CONSTANT) - { - funct.reset( - new RowAggFunctionCol( - f->fAggFunction, - f->fStatsFunction, - f->fInputColumnIndex, - f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); - functionSub2.push_back(funct); - } - } - } - } - - if (functionSub1.size() > 0) - { - // make sure the group by columns are available for next aggregate phase. - vector groupBySubNoDist; - - for (uint64_t i = 0; i < groupByNoDist.size(); i++) - groupBySubNoDist.push_back(SP_ROWAGG_GRPBY_t( - new RowAggGroupByCol(groupByNoDist[i]->fInputColumnIndex, i))); - - // construct sub-aggregator - SP_ROWAGG_UM_t subAgg( - new RowAggregationUMP2(groupBySubNoDist, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); - subAgg->timeZone(jobInfo.timeZone); - - // add to rowAggDist - multiDistinctAggregator->addSubAggregator(subAgg, aggRgUm, functionSub2); - subRgVec.push_back(aggRgUm); - } - } - } - - rowAggDist->addAggregator(rowAggUm, aggRgUm); - rowgroups.push_back(aggRgDist); - aggregators.push_back(rowAggDist); - - posAggPm.push_back(2); // rid - - for (uint64_t i = 0; i < oidsAggPm.size(); i++) - posAggPm.push_back(posAggPm[i] + widthAggPm[i]); - - RowGroup aggRgPm(oidsAggPm.size(), posAggPm, oidsAggPm, keysAggPm, typeAggPm, - csNumAggPm, scaleAggPm, precisionAggPm, jobInfo.stringTableThreshold); - SP_ROWAGG_PM_t rowAggPm(new RowAggregation(groupByPm, functionVecPm)); - rowAggPm->timeZone(jobInfo.timeZone); - rowgroups.push_back(aggRgPm); - aggregators.push_back(rowAggPm); - - if (jobInfo.trace) - { - cout << "projected RG: " << projRG.toString() << endl - << "aggregated1 RG: " << aggRgPm.toString() << endl - << "aggregated2 RG: " << aggRgUm.toString() << endl; - - for (uint64_t i = 0; i < subRgVec.size(); i++) - cout << "aggregatedSub RG: " << i << " " << subRgVec[i].toString() << endl; - - cout << "aggregatedDist RG: " << aggRgDist.toString() << endl; - } -} - - -void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInfo& jobInfo) -{ - map keyToIndexMap; - - for (uint64_t i = 0; i < fRowGroupOut.getKeys().size(); ++i) - { - if (keyToIndexMap.find(fRowGroupOut.getKeys()[i]) == keyToIndexMap.end()) - keyToIndexMap.insert(make_pair(fRowGroupOut.getKeys()[i], i)); - } - - RetColsVector expressionVec; - ArithmeticColumn* ac = NULL; - FunctionColumn* fc = NULL; - RetColsVector& cols = jobInfo.nonConstCols; - vector simpleColumns; - - for (RetColsVector::iterator it = cols.begin(); it != cols.end(); ++it) - { - uint64_t eid = -1; - - if (((ac = dynamic_cast(it->get())) != NULL) && - (ac->aggColumnList().size() > 0) && - (ac->windowfunctionColumnList().size() == 0)) - { - const vector& scols = ac->simpleColumnList(); - simpleColumns.insert(simpleColumns.end(), scols.begin(), scols.end()); - - eid = ac->expressionId(); - expressionVec.push_back(*it); - } - else if (((fc = dynamic_cast(it->get())) != NULL) && - (fc->aggColumnList().size() > 0) && - (fc->windowfunctionColumnList().size() == 0)) - { - const vector& sCols = fc->simpleColumnList(); - simpleColumns.insert(simpleColumns.end(), sCols.begin(), sCols.end()); - - eid = fc->expressionId(); - expressionVec.push_back(*it); - } - - // update the output index - if (eid != (uint64_t) - 1) - { - map::iterator mit = keyToIndexMap.find(getExpTupleKey(jobInfo, eid)); - - if (mit != keyToIndexMap.end()) - { - it->get()->outputIndex(mit->second); - } - else - { - ostringstream emsg; - emsg << "expression " << eid << " cannot be found in tuple."; - cerr << "prepExpressionOnAggregate: " << emsg.str() << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); - } - } - } - - // map the input indices - for (vector::iterator i = simpleColumns.begin(); i != simpleColumns.end(); i++) - { - CalpontSystemCatalog::OID oid = (*i)->oid(); - uint32_t key = getTupleKey(jobInfo, *i); - CalpontSystemCatalog::OID dictOid = joblist::isDictCol((*i)->colType()); - - if (dictOid > 0) - { - oid = dictOid; - key = jobInfo.keyInfo->dictKeyMap[key]; - } - - map::iterator mit = keyToIndexMap.find(key); - - if (mit != keyToIndexMap.end()) - { - (*i)->inputIndex(mit->second); + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colAgg, outIdx)); } else { - ostringstream emsg; - emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' cannot be found in tuple."; - cerr << "prepExpressionOnAggregate: " << emsg.str() << " simple column: oid(" - << oid << "), alias(" << extractTableAlias(*i) << ")." << endl; - throw QueryDataExcept(emsg.str(), aggregateFuncErr); + funct.reset(new RowAggFunctionCol(aggOp, stats, colAgg, outIdx)); } + + if (aggOp == ROWAGG_COUNT_NO_OP) + funct->fAuxColumnIndex = colAgg; + else if (aggOp == ROWAGG_CONSTANT) + funct->fAuxColumnIndex = jobInfo.cntStarPos; + + functionVec2.push_back(funct); + + // find if this func is a duplicate + AGG_MAP::iterator iter = aggDupFuncMap.find( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (iter != aggDupFuncMap.end()) + { + if (funct->fAggFunction == ROWAGG_AVG) + funct->fAggFunction = ROWAGG_DUP_AVG; + else if (funct->fAggFunction == ROWAGG_STATS) + funct->fAggFunction = ROWAGG_DUP_STATS; + else if (funct->fAggFunction == ROWAGG_UDAF) + funct->fAggFunction = ROWAGG_DUP_UDAF; + else + funct->fAggFunction = ROWAGG_DUP_FUNCT; + + funct->fAuxColumnIndex = iter->second; + } + else + { + aggDupFuncMap.insert(make_pair( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + funct->fOutputColumnIndex)); + } + + if (returnedColVec[i].second == AggregateColumn::AVG) + avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); + else if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) + avgDistFuncMap.insert(make_pair(returnedColVec[i].first, funct)); + } + ++outIdx; + } // for (i + + // now fix the AVG function, locate the count(column) position + for (uint64_t i = 0; i < functionVec2.size(); i++) + { + if (functionVec2[i]->fAggFunction == ROWAGG_COUNT_NO_OP) + { + // if the count(k) can be associated with an avg(k) + map::iterator k = + avgFuncMap.find(keysAggDist[functionVec2[i]->fOutputColumnIndex]); + + if (k != avgFuncMap.end()) + k->second->fAuxColumnIndex = functionVec2[i]->fOutputColumnIndex; + } } - // add expression to UM aggregator - aggUM->expression(expressionVec); + // there is avg(k), but no count(k) in the select list + uint64_t lastCol = outIdx; + + for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) + { + if (k->second->fAuxColumnIndex == (uint32_t)-1) + { + k->second->fAuxColumnIndex = lastCol++; + oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); + keysAggDist.push_back(k->first); + scaleAggDist.push_back(0); + precisionAggDist.push_back(19); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); + } + } + + // now fix the AVG distinct function, locate the count(distinct column) position + for (uint64_t i = 0; i < functionVec2.size(); i++) + { + if (functionVec2[i]->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME && !functionVec2[i]->hasMultiParm) + { + // if the count(distinct k) can be associated with an avg(distinct k) + map::iterator k = + avgDistFuncMap.find(keysAggDist[functionVec2[i]->fOutputColumnIndex]); + + if (k != avgDistFuncMap.end()) + { + k->second->fAuxColumnIndex = functionVec2[i]->fOutputColumnIndex; + functionVec2[i]->fAggFunction = ROWAGG_COUNT_NO_OP; + } + } + } + + // there is avg(distinct k), but no count(distinct k) in the select list + for (map::iterator k = avgDistFuncMap.begin(); k != avgDistFuncMap.end(); k++) + { + if (k->second->fAuxColumnIndex == (uint32_t)-1) + { + k->second->fAuxColumnIndex = lastCol++; + oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); + keysAggDist.push_back(k->first); + scaleAggDist.push_back(0); + precisionAggDist.push_back(19); + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); + } + } + + // add auxiliary fields for UDAF and statistics functions + for (uint64_t i = 0; i < functionVec2.size(); i++) + { + uint64_t j = functionVec2[i]->fInputColumnIndex; + + if (functionVec2[i]->fAggFunction == ROWAGG_UDAF) + { + // Column for index of UDAF UserData struct + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVec2[i].get()); + + if (!udafFuncCol) + { + throw logic_error( + "(4)prep1PhaseDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + functionVec2[i]->fAuxColumnIndex = lastCol++; + oidsAggDist.push_back(oidsAgg[j]); // Dummy? + keysAggDist.push_back(keysAgg[j]); // Dummy? + scaleAggDist.push_back(0); + precisionAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(sizeof(uint64_t)); + continue; + } + + if (functionVec2[i]->fAggFunction != ROWAGG_STATS) + continue; + + functionVec2[i]->fAuxColumnIndex = lastCol; + + // sum(x) + oidsAggDist.push_back(oidsAgg[j]); + keysAggDist.push_back(keysAgg[j]); + scaleAggDist.push_back(0); + precisionAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggDist.push_back(8); + widthAggDist.push_back(sizeof(long double)); + ++lastCol; + + // sum(x**2) + oidsAggDist.push_back(oidsAgg[j]); + keysAggDist.push_back(keysAgg[j]); + scaleAggDist.push_back(0); + precisionAggDist.push_back(-1); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggDist.push_back(8); + widthAggDist.push_back(sizeof(long double)); + ++lastCol; + } + } + + // calculate the offset and create the rowaggregation, rowgroup + posAgg.push_back(2); + + for (uint64_t i = 0; i < oidsAgg.size(); i++) + posAgg.push_back(posAgg[i] + widthAgg[i]); + + RowGroup aggRG(oidsAgg.size(), posAgg, oidsAgg, keysAgg, typeAgg, csNumAgg, scaleAgg, precisionAgg, + jobInfo.stringTableThreshold); + SP_ROWAGG_UM_t rowAgg(new RowAggregationUM(groupBy, functionVec1, jobInfo.rm, jobInfo.umMemLimit)); + rowAgg->timeZone(jobInfo.timeZone); + + posAggDist.push_back(2); // rid + + for (uint64_t i = 0; i < oidsAggDist.size(); i++) + posAggDist.push_back(posAggDist[i] + widthAggDist[i]); + + RowGroup aggRgDist(oidsAggDist.size(), posAggDist, oidsAggDist, keysAggDist, typeAggDist, csNumAggDist, + scaleAggDist, precisionAggDist, jobInfo.stringTableThreshold); + SP_ROWAGG_DIST rowAggDist( + new RowAggregationDistinct(groupByNoDist, functionVec2, jobInfo.rm, jobInfo.umMemLimit)); + rowAggDist->timeZone(jobInfo.timeZone); + + // mapping the group_concat columns, if any. + if (jobInfo.groupConcatInfo.groupConcat().size() > 0) + { + jobInfo.groupConcatInfo.mapColumns(projRG); + rowAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); + rowAggDist->groupConcat(jobInfo.groupConcatInfo.groupConcat()); + } + + // if distinct key word applied to more than one aggregate column, reset rowAggDist + vector subRgVec; + + if (jobInfo.distinctColVec.size() > 1) + { + RowAggregationMultiDistinct* multiDistinctAggregator = + new RowAggregationMultiDistinct(groupByNoDist, functionVec2, jobInfo.rm, jobInfo.umMemLimit); + multiDistinctAggregator->timeZone(jobInfo.timeZone); + rowAggDist.reset(multiDistinctAggregator); + rowAggDist->groupConcat(jobInfo.groupConcatInfo.groupConcat()); + + // construct and add sub-aggregators to rowAggDist + vector posAggGb, posAggSub; + vector oidsAggGb, oidsAggSub; + vector keysAggGb, keysAggSub; + vector scaleAggGb, scaleAggSub; + vector precisionAggGb, precisionAggSub; + vector typeAggGb, typeAggSub; + vector csNumAggGb, csNumAggSub; + vector widthAggGb, widthAggSub; + + // populate groupby column info + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + oidsAggGb.push_back(oidsProj[i]); + keysAggGb.push_back(keysProj[i]); + scaleAggGb.push_back(scaleProj[i]); + precisionAggGb.push_back(precisionProj[i]); + typeAggGb.push_back(typeProj[i]); + csNumAggGb.push_back(csNumProj[i]); + widthAggGb.push_back(widthProj[i]); + } + + // for distinct, each column requires seperate rowgroup + vector rowAggSubDistVec; + + uint32_t distinctColKey; + int64_t j; + uint64_t k; + uint64_t outIdx = 0; + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + if (returnedColVec[i].second == 0) + { + ++outIdx; + continue; + } + + j = -1; + + distinctColKey = -1; + // Find the entry in distinctColVec, if any + for (k = 0; k < jobInfo.distinctColVec.size(); k++) + { + distinctColKey = jobInfo.distinctColVec[k]; + if (returnedColVec[i].first == distinctColKey) + break; + } + if (distinctColKey == (uint32_t)-1) + { + ++outIdx; + continue; + } + // locate the distinct key in the row group + for (k = 0; k < keysAgg.size(); k++) + { + if (keysProj[k] == distinctColKey) + { + j = k; + break; + } + } + + idbassert(j != -1); + + oidsAggSub = oidsAggGb; + keysAggSub = keysAggGb; + scaleAggSub = scaleAggGb; + precisionAggSub = precisionAggGb; + typeAggSub = typeAggGb; + csNumAggSub = csNumAggGb; + widthAggSub = widthAggGb; + + oidsAggSub.push_back(oidsProj[j]); + keysAggSub.push_back(keysProj[j]); + scaleAggSub.push_back(scaleProj[j]); + precisionAggSub.push_back(precisionProj[j]); + typeAggSub.push_back(typeProj[j]); + csNumAggSub.push_back(csNumProj[j]); + widthAggSub.push_back(widthProj[j]); + + // construct groupby vector + vector groupBySub; + k = 0; + + while (k < jobInfo.groupByColVec.size()) + { + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(k, k)); + groupBySub.push_back(groupby); + k++; + } + // add the distinct column as groupby + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); + groupBySub.push_back(groupby); + + // Add multi parm distinct + while ((i + 1) < returnedColVec.size() && + functionIdMap(returnedColVec[i + 1].second) == ROWAGG_MULTI_PARM) + { + ++i; + uint32_t dColKey = -1; + j = -1; + + // Find the entry in distinctColVec, if any + for (k = 0; k < jobInfo.distinctColVec.size(); k++) + { + dColKey = jobInfo.distinctColVec[k]; + if (returnedColVec[i].first == dColKey) + break; + } + idbassert(dColKey != (uint32_t)-1); + // locate the distinct key in the row group + for (k = 0; k < keysAgg.size(); k++) + { + if (keysProj[k] == dColKey) + { + j = k; + break; + } + } + idbassert(j != -1); + + oidsAggSub.push_back(oidsProj[j]); + keysAggSub.push_back(keysProj[j]); + scaleAggSub.push_back(scaleProj[j]); + precisionAggSub.push_back(precisionProj[j]); + typeAggSub.push_back(typeProj[j]); + csNumAggSub.push_back(csNumProj[j]); + widthAggSub.push_back(widthProj[j]); + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); + groupBySub.push_back(groupby); + } + + // construct sub-rowgroup + posAggSub.clear(); + posAggSub.push_back(2); // rid + + for (k = 0; k < oidsAggSub.size(); k++) + posAggSub.push_back(posAggSub[k] + widthAggSub[k]); + + RowGroup subRg(oidsAggSub.size(), posAggSub, oidsAggSub, keysAggSub, typeAggSub, csNumAggSub, + scaleAggSub, precisionAggSub, jobInfo.stringTableThreshold); + subRgVec.push_back(subRg); + + // Keep a count of the parms after the first for any aggregate. + // These will be skipped and the count needs to be subtracted + // from where the aux column will be. + int64_t multiParms = 0; + + // tricky part : 2 function vectors + // -- dummy function vector for sub-aggregator, which does distinct only + // -- aggregate function on this distinct column for rowAggDist + vector functionSub1, functionSub2; + // search the function in functionVec + vector::iterator it = functionVec2.begin(); + + while (it != functionVec2.end()) + { + SP_ROWAGG_FUNC_t f = *it++; + + if ((f->fOutputColumnIndex == outIdx) && + (f->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME || f->fAggFunction == ROWAGG_DISTINCT_SUM || + f->fAggFunction == ROWAGG_DISTINCT_AVG)) + { + SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol(f->fAggFunction, f->fStatsFunction, + groupBySub.size() - 1, f->fOutputColumnIndex, + f->fAuxColumnIndex - multiParms)); + functionSub2.push_back(funct); + } + } + + // construct sub-aggregator + SP_ROWAGG_UM_t subAgg( + new RowAggregationSubDistinct(groupBySub, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); + subAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); + + // add to rowAggDist + multiDistinctAggregator->addSubAggregator(subAgg, subRg, functionSub2); + + ++outIdx; + } + + // cover any non-distinct column functions + { + vector functionSub1 = functionNoDistVec; + vector functionSub2; + int64_t multiParms = 0; + + for (uint64_t k = 0; k < returnedColVec.size(); k++) + { + // search non-distinct functions in functionVec + vector::iterator it = functionVec2.begin(); + + while (it != functionVec2.end()) + { + SP_ROWAGG_FUNC_t funct; + SP_ROWAGG_FUNC_t f = *it++; + + if (f->fAggFunction == ROWAGG_UDAF) + { + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(f.get()); + funct.reset(new RowUDAFFunctionCol(udafFuncCol->fUDAFContext, udafFuncCol->fInputColumnIndex, + udafFuncCol->fOutputColumnIndex, + udafFuncCol->fAuxColumnIndex - multiParms)); + functionSub2.push_back(funct); + } + else if ((f->fOutputColumnIndex == k) && + (f->fAggFunction == ROWAGG_COUNT_ASTERISK || f->fAggFunction == ROWAGG_COUNT_COL_NAME || + f->fAggFunction == ROWAGG_SUM || f->fAggFunction == ROWAGG_AVG || + f->fAggFunction == ROWAGG_MIN || f->fAggFunction == ROWAGG_MAX || + f->fAggFunction == ROWAGG_STATS || f->fAggFunction == ROWAGG_BIT_AND || + f->fAggFunction == ROWAGG_BIT_OR || f->fAggFunction == ROWAGG_BIT_XOR || + f->fAggFunction == ROWAGG_CONSTANT || f->fAggFunction == ROWAGG_GROUP_CONCAT)) + { + funct.reset(new RowAggFunctionCol(f->fAggFunction, f->fStatsFunction, f->fInputColumnIndex, + f->fOutputColumnIndex, f->fAuxColumnIndex - multiParms)); + functionSub2.push_back(funct); + } + } + } + + if (functionSub1.size() > 0) + { + // make sure the group by columns are available for next aggregate phase. + vector groupBySubNoDist; + + for (uint64_t i = 0; i < groupByNoDist.size(); i++) + groupBySubNoDist.push_back( + SP_ROWAGG_GRPBY_t(new RowAggGroupByCol(groupByNoDist[i]->fInputColumnIndex, i))); + + // construct sub-aggregator + SP_ROWAGG_UM_t subAgg( + new RowAggregationUM(groupBySubNoDist, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); + subAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); + + // add to rowAggDist + multiDistinctAggregator->addSubAggregator(subAgg, aggRG, functionSub2); + subRgVec.push_back(aggRG); + } + } + } + + rowAggDist->addAggregator(rowAgg, aggRG); + rowgroups.push_back(aggRgDist); + aggregators.push_back(rowAggDist); + + if (jobInfo.trace) + { + cout << "projected RG: " << projRG.toString() << endl << "aggregated RG: " << aggRG.toString() << endl; + + for (uint64_t i = 0; i < subRgVec.size(); i++) + cout << "aggregatedSub RG: " << i << " " << subRgVec[i].toString() << endl; + + cout << "aggregatedDist RG: " << aggRgDist.toString() << endl; + } } +void TupleAggregateStep::prep2PhasesAggregate(JobInfo& jobInfo, vector& rowgroups, + vector& aggregators) +{ + // check if there are any aggregate columns + // a vector that has the aggregate function to be done by PM + vector> aggColVec; + set avgSet; + vector>& returnedColVec = jobInfo.returnedColVec; + // For UDAF + uint32_t projColsUDAFIdx = 0; + uint32_t udafcParamIdx = 0; + UDAFColumn* udafc = NULL; + mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; + + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + // skip if not an aggregation column + if (returnedColVec[i].second == 0) + continue; + + aggColVec.push_back(returnedColVec[i]); + + // remember if a column has an average function, + // with avg function, no need for separate sum or count_column_name + if (returnedColVec[i].second == AggregateColumn::AVG) + avgSet.insert(returnedColVec[i].first); + } + + // populate the aggregate rowgroup on PM and UM + // PM: projectedRG -> aggregateRGPM + // UM: aggregateRGPM -> aggregateRGUM + // + // Aggregate preparation by joblist factory: + // 1. get projected rowgroup (done by doAggProject) -- input to PM AGG + // 2. construct aggregate rowgroup -- output of PM, input of UM + // 3. construct aggregate rowgroup -- output of UM + const RowGroup projRG = rowgroups[0]; + const vector& oidsProj = projRG.getOIDs(); + const vector& keysProj = projRG.getKeys(); + const vector& scaleProj = projRG.getScale(); + const vector& precisionProj = projRG.getPrecision(); + const vector& typeProj = projRG.getColTypes(); + const vector& csNumProj = projRG.getCharsetNumbers(); + + vector posAggPm, posAggUm; + vector oidsAggPm, oidsAggUm; + vector keysAggPm, keysAggUm; + vector scaleAggPm, scaleAggUm; + vector precisionAggPm, precisionAggUm; + vector typeAggPm, typeAggUm; + vector csNumAggPm, csNumAggUm; + vector widthAggPm, widthAggUm; + vector groupByPm, groupByUm; + vector functionVecPm, functionVecUm; + uint32_t bigIntWidth = sizeof(int64_t); + uint32_t bigUintWidth = sizeof(uint64_t); + AGG_MAP aggFuncMap; + + // associate the columns between projected RG and aggregate RG on PM + // populated the aggregate columns + // the groupby columns are put in front, even not a returned column + // sum and count(column name) are omitted, if avg present + { + // project only unique oids, but they may be repeated in aggregation + // collect the projected column info, prepare for aggregation + vector width; + map projColPosMap; + + for (uint64_t i = 0; i < keysProj.size(); i++) + { + projColPosMap.insert(make_pair(keysProj[i], i)); + width.push_back(projRG.getColumnWidth(i)); + } + + // column index for PM aggregate rowgroup + uint64_t colAggPm = 0; + + // for groupby column + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + uint32_t key = jobInfo.groupByColVec[i]; + + if (projColPosMap.find(key) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep2PhasesAggregate: groupby " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + uint64_t colProj = projColPosMap[key]; + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); + groupByPm.push_back(groupby); + + // PM: just copy down to aggregation rowgroup + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(key); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + + aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL), + colAggPm)); + colAggPm++; + } + + // for distinct column + for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) + { + uint32_t key = jobInfo.distinctColVec[i]; + + if (projColPosMap.find(key) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep2PhasesAggregate: distinct " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + uint64_t colProj = projColPosMap[key]; + + // check for dup distinct column -- @bug6126 + if (find(keysAggPm.begin(), keysAggPm.end(), key) != keysAggPm.end()) + continue; + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); + groupByPm.push_back(groupby); + + // PM: just copy down to aggregation rowgroup + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(key); + scaleAggPm.push_back(scaleProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + + aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL), + colAggPm)); + colAggPm++; + } + + // vectors for aggregate functions + for (uint64_t i = 0; i < aggColVec.size(); i++) + { + pUDAFFunc = NULL; + uint32_t aggKey = aggColVec[i].first; + RowAggFunctionType aggOp = functionIdMap(aggColVec[i].second); + RowAggFunctionType stats = statsFuncIdMap(aggColVec[i].second); + + // skip on PM if this is a constant + if (aggOp == ROWAGG_CONSTANT) + continue; + + if (projColPosMap.find(aggKey) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[aggKey] << "' isn't in tuple."; + cerr << "prep2PhasesAggregate: aggregate " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[aggKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[aggKey].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + if ((aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) && (avgSet.find(aggKey) != avgSet.end())) + // skip sum / count(column) if avg is also selected + continue; + + uint64_t colProj = projColPosMap[aggKey]; + SP_ROWAGG_FUNC_t funct; + + if (aggOp == ROWAGG_UDAF) + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) + { + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) + { + for (uint64_t k = i + 1; + k < aggColVec.size() && aggColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(aggColVec[k].first); + } + } + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + } + if (it == jobInfo.projectionCols.end()) + { + throw logic_error( + "(1)prep2PhasesAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); + } + } + else + { + funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); + } + + // skip if this is a duplicate + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && + aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL)) != + aggFuncMap.end()) + { + // skip if this is a duplicate + continue; + } + + functionVecPm.push_back(funct); + aggFuncMap.insert(make_pair( + boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + colAggPm)); + + switch (aggOp) + { + case ROWAGG_MIN: + case ROWAGG_MAX: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + colAggPm++; + } + break; + + case ROWAGG_SUM: + case ROWAGG_AVG: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("sum/average"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep2PhasesAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + wideDecimalOrLongDouble(colProj, typeProj[colProj], precisionProj, scaleProj, width, typeAggPm, + scaleAggPm, precisionAggPm, widthAggPm); + + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + csNumAggPm.push_back(8); + colAggPm++; + } + + // PM: put the count column for avg next to the sum + // let fall through to add a count column for average function + if (aggOp != ROWAGG_AVG) + break; + /* fall through */ + + case ROWAGG_COUNT_ASTERISK: + case ROWAGG_COUNT_COL_NAME: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + // work around count() in select subquery + precisionAggPm.push_back(9999); + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggPm.push_back(8); + widthAggPm.push_back(bigIntWidth); + colAggPm++; + } + break; + + case ROWAGG_STATS: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("variance/standard deviation"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep2PhaseAggregate:: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + // counts(x) + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(0); + typeAggPm.push_back(CalpontSystemCatalog::DOUBLE); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(double)); + funct->fAuxColumnIndex = ++colAggPm; + + // sum(x) + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(-1); + typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(long double)); + ++colAggPm; + + // sum(x**2) + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(-1); + typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(long double)); + ++colAggPm; + } + break; + + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(-16); // for connector to skip null check + + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } + + csNumAggPm.push_back(8); + widthAggPm.push_back(bigIntWidth); + colAggPm++; + } + break; + + case ROWAGG_UDAF: + { + // Return column + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); + + if (!udafFuncCol) + { + throw logic_error( + "(2)prep2PhasesAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(udafFuncCol->fUDAFContext.getScale()); + precisionAggPm.push_back(udafFuncCol->fUDAFContext.getPrecision()); + typeAggPm.push_back(udafFuncCol->fUDAFContext.getResultType()); + csNumAggPm.push_back(udafFuncCol->fUDAFContext.getCharsetNumber()); + widthAggPm.push_back(udafFuncCol->fUDAFContext.getColWidth()); + ++colAggPm; + // Column for index of UDAF UserData struct + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(0); + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggPm.push_back(8); + widthAggPm.push_back(bigUintWidth); + funct->fAuxColumnIndex = colAggPm++; + // If the first param is const + udafcParamIdx = 0; + ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) + { + funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; + } + ++udafcParamIdx; + break; + } + + case ROWAGG_MULTI_PARM: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + colAggPm++; + // If the param is const + if (udafc) + { + if (udafcParamIdx > udafc->aggParms().size() - 1) + { + throw QueryDataExcept("prep2PhasesAggregate: UDAF multi function with too many parms", + aggregateFuncErr); + } + ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) + { + funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; + } + } + else + { + throw QueryDataExcept("prep2PhasesAggregate: UDAF multi function with no parms", + aggregateFuncErr); + } + ++udafcParamIdx; + } + break; + + default: + { + ostringstream emsg; + emsg << "aggregate function (" << (uint64_t)aggOp << ") isn't supported"; + cerr << "prep2PhasesAggregate: " << emsg.str() << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); + } + } + } + } + + // associate the columns between the aggregate RGs on PM and UM + // populated the returned columns + // remove not returned groupby column + // add back sum or count(column name) if omitted due to avg column + // put count(column name) column to the end, if it is for avg only + { + // check if the count column for AVG is also a returned column, + // if so, replace the "-1" to actual position in returned vec. + map avgFuncMap; + AGG_MAP aggDupFuncMap; + + projColsUDAFIdx = 0; + // copy over the groupby vector + // update the outputColumnIndex if returned + for (uint64_t i = 0; i < groupByPm.size(); i++) + { + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(groupByPm[i]->fOutputColumnIndex, -1)); + groupByUm.push_back(groupby); + } + + // locate the return column position in aggregated rowgroup from PM + // outIdx is i without the multi-columns, + uint64_t outIdx = 0; + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + uint32_t retKey = returnedColVec[i].first; + RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); + RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); + int colPm = -1; + + if (aggOp == ROWAGG_MULTI_PARM) + { + // Skip on UM: Extra parms for an aggregate have no work on the UM + continue; + } + + // Is this a UDAF? use the function as part of the key. + pUDAFFunc = NULL; + udafc = NULL; + if (aggOp == ROWAGG_UDAF) + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + + for (; it != jobInfo.projectionCols.end(); it++) + { + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) + { + for (uint64_t k = i + 1; + k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); + } + } + break; + } + } + if (it == jobInfo.projectionCols.end()) + { + throw logic_error( + "(3)prep2PhasesAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); + } + } + + AGG_MAP::iterator it = aggFuncMap.find( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (it != aggFuncMap.end()) + { + colPm = it->second; + oidsAggUm.push_back(oidsAggPm[colPm]); + keysAggUm.push_back(retKey); + scaleAggUm.push_back(scaleAggPm[colPm]); + precisionAggUm.push_back(precisionAggPm[colPm]); + typeAggUm.push_back(typeAggPm[colPm]); + csNumAggUm.push_back(csNumAggPm[colPm]); + widthAggUm.push_back(widthAggPm[colPm]); + } + + // not a direct hit -- a returned column is not already in the RG from PMs + else + { + bool returnColMissing = true; + + // check if a SUM or COUNT covered by AVG + if (aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) + { + it = aggFuncMap.find(boost::make_tuple(returnedColVec[i].first, ROWAGG_AVG, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (it != aggFuncMap.end()) + { + // false alarm + returnColMissing = false; + + colPm = it->second; + + if (aggOp == ROWAGG_SUM) + { + wideDecimalOrLongDouble(colPm, typeAggPm[colPm], precisionAggPm, scaleAggPm, widthAggPm, + typeAggUm, scaleAggUm, precisionAggUm, widthAggUm); + + oidsAggUm.push_back(oidsAggPm[colPm]); + keysAggUm.push_back(retKey); + csNumAggUm.push_back(8); + } + else + { + // leave the count() to avg + aggOp = ROWAGG_COUNT_NO_OP; + + colPm++; + oidsAggUm.push_back(oidsAggPm[colPm]); + keysAggUm.push_back(retKey); + scaleAggUm.push_back(0); + precisionAggUm.push_back(19); + typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggUm.push_back(8); + widthAggUm.push_back(bigIntWidth); + } + } + } + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), retKey) != + jobInfo.expressionVec.end()) + { + // a function on aggregation + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggUm.push_back(ti.oid); + keysAggUm.push_back(retKey); + scaleAggUm.push_back(ti.scale); + precisionAggUm.push_back(ti.precision); + typeAggUm.push_back(ti.dtype); + csNumAggUm.push_back(ti.csNum); + widthAggUm.push_back(ti.width); + + returnColMissing = false; + } + else if (jobInfo.windowSet.find(retKey) != jobInfo.windowSet.end()) + { + // an window function + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggUm.push_back(ti.oid); + keysAggUm.push_back(retKey); + scaleAggUm.push_back(ti.scale); + precisionAggUm.push_back(ti.precision); + typeAggUm.push_back(ti.dtype); + csNumAggUm.push_back(ti.csNum); + widthAggUm.push_back(ti.width); + + returnColMissing = false; + } + else if (aggOp == ROWAGG_CONSTANT) + { + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggUm.push_back(ti.oid); + keysAggUm.push_back(retKey); + scaleAggUm.push_back(ti.scale); + precisionAggUm.push_back(ti.precision); + typeAggUm.push_back(ti.dtype); + csNumAggUm.push_back(ti.csNum); + widthAggUm.push_back(ti.width); + + returnColMissing = false; + } + + if (returnColMissing) + { + Message::Args args; + args.add(keyName(outIdx, retKey, jobInfo)); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + cerr << "prep2PhasesAggregate: " << emsg << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[retKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[retKey].fTable + << ", view=" << jobInfo.keyInfo->tupleKeyVec[retKey].fView << ", function=" << (int)aggOp + << endl; + throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); + } + } + + // update groupby vector if the groupby column is a returned column + if (returnedColVec[i].second == 0) + { + int dupGroupbyIndex = -1; + + for (uint64_t j = 0; j < jobInfo.groupByColVec.size(); j++) + { + if (jobInfo.groupByColVec[j] == retKey) + { + if (groupByUm[j]->fOutputColumnIndex == (uint32_t)-1) + groupByUm[j]->fOutputColumnIndex = outIdx; + else + dupGroupbyIndex = groupByUm[j]->fOutputColumnIndex; + } + } + + for (uint64_t j = 0; j < jobInfo.distinctColVec.size(); j++) + { + if (jobInfo.distinctColVec[j] == retKey) + { + if (groupByUm[j]->fOutputColumnIndex == (uint32_t)-1) + groupByUm[j]->fOutputColumnIndex = outIdx; + else + dupGroupbyIndex = groupByUm[j]->fOutputColumnIndex; + } + } + + // a duplicate group by column + if (dupGroupbyIndex != -1) + functionVecUm.push_back(SP_ROWAGG_FUNC_t( + new RowAggFunctionCol(ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, dupGroupbyIndex))); + } + else + { + // update the aggregate function vector + SP_ROWAGG_FUNC_t funct; + if (aggOp == ROWAGG_UDAF) + { + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colPm, outIdx)); + } + else + { + funct.reset(new RowAggFunctionCol(aggOp, stats, colPm, outIdx)); + } + + if (aggOp == ROWAGG_COUNT_NO_OP) + funct->fAuxColumnIndex = colPm; + else if (aggOp == ROWAGG_CONSTANT) + funct->fAuxColumnIndex = jobInfo.cntStarPos; + + functionVecUm.push_back(funct); + + // find if this func is a duplicate + AGG_MAP::iterator iter = aggDupFuncMap.find( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (iter != aggDupFuncMap.end()) + { + if (funct->fAggFunction == ROWAGG_AVG) + funct->fAggFunction = ROWAGG_DUP_AVG; + else if (funct->fAggFunction == ROWAGG_STATS) + funct->fAggFunction = ROWAGG_DUP_STATS; + else if (funct->fAggFunction == ROWAGG_UDAF) + funct->fAggFunction = ROWAGG_DUP_UDAF; + else + funct->fAggFunction = ROWAGG_DUP_FUNCT; + + funct->fAuxColumnIndex = iter->second; + } + else + { + aggDupFuncMap.insert(make_pair( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + funct->fOutputColumnIndex)); + } + + if (returnedColVec[i].second == AggregateColumn::AVG) + avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); + } + ++outIdx; + } + + // now fix the AVG function, locate the count(column) position + for (uint64_t i = 0; i < functionVecUm.size(); i++) + { + if (functionVecUm[i]->fAggFunction != ROWAGG_COUNT_NO_OP) + continue; + + // if the count(k) can be associated with an avg(k) + map::iterator k = + avgFuncMap.find(keysAggUm[functionVecUm[i]->fOutputColumnIndex]); + + if (k != avgFuncMap.end()) + k->second->fAuxColumnIndex = functionVecUm[i]->fOutputColumnIndex; + } + + // there is avg(k), but no count(k) in the select list + uint64_t lastCol = outIdx; + + for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) + { + if (k->second->fAuxColumnIndex == (uint32_t)-1) + { + k->second->fAuxColumnIndex = lastCol++; + oidsAggUm.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); + keysAggUm.push_back(k->first); + scaleAggUm.push_back(0); + precisionAggUm.push_back(19); + typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggUm.push_back(8); + widthAggUm.push_back(bigIntWidth); + } + } + + // add auxiliary fields for UDAF and statistics functions + for (uint64_t i = 0; i < functionVecUm.size(); i++) + { + uint64_t j = functionVecUm[i]->fInputColumnIndex; + + if (functionVecUm[i]->fAggFunction == ROWAGG_UDAF) + { + // Column for index of UDAF UserData struct + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVecUm[i].get()); + + if (!udafFuncCol) + { + throw logic_error( + "(4)prep2PhasesAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + functionVecUm[i]->fAuxColumnIndex = lastCol++; + oidsAggUm.push_back(oidsAggPm[j]); // Dummy? + keysAggUm.push_back(keysAggPm[j]); // Dummy? + scaleAggUm.push_back(0); + precisionAggUm.push_back(0); + typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggUm.push_back(8); + widthAggUm.push_back(bigUintWidth); + continue; + } + + if (functionVecUm[i]->fAggFunction != ROWAGG_STATS) + continue; + + functionVecUm[i]->fAuxColumnIndex = lastCol; + + // sum(x) + oidsAggUm.push_back(oidsAggPm[j]); + keysAggUm.push_back(keysAggPm[j]); + scaleAggUm.push_back(0); + precisionAggUm.push_back(-1); + typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggUm.push_back(8); + widthAggUm.push_back(sizeof(long double)); + ++lastCol; + + // sum(x**2) + oidsAggUm.push_back(oidsAggPm[j]); + keysAggUm.push_back(keysAggPm[j]); + scaleAggUm.push_back(0); + precisionAggUm.push_back(-1); + typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggUm.push_back(8); + widthAggUm.push_back(sizeof(long double)); + ++lastCol; + } + } + + // calculate the offset and create the rowaggregations, rowgroups + posAggUm.push_back(2); // rid + + for (uint64_t i = 0; i < oidsAggUm.size(); i++) + posAggUm.push_back(posAggUm[i] + widthAggUm[i]); + + RowGroup aggRgUm(oidsAggUm.size(), posAggUm, oidsAggUm, keysAggUm, typeAggUm, csNumAggUm, scaleAggUm, + precisionAggUm, jobInfo.stringTableThreshold); + SP_ROWAGG_UM_t rowAggUm(new RowAggregationUMP2(groupByUm, functionVecUm, jobInfo.rm, jobInfo.umMemLimit)); + rowAggUm->timeZone(jobInfo.timeZone); + rowgroups.push_back(aggRgUm); + aggregators.push_back(rowAggUm); + + posAggPm.push_back(2); // rid + + for (uint64_t i = 0; i < oidsAggPm.size(); i++) + posAggPm.push_back(posAggPm[i] + widthAggPm[i]); + + RowGroup aggRgPm(oidsAggPm.size(), posAggPm, oidsAggPm, keysAggPm, typeAggPm, csNumAggPm, scaleAggPm, + precisionAggPm, jobInfo.stringTableThreshold); + SP_ROWAGG_PM_t rowAggPm(new RowAggregation(groupByPm, functionVecPm)); + rowAggPm->timeZone(jobInfo.timeZone); + rowgroups.push_back(aggRgPm); + aggregators.push_back(rowAggPm); + + if (jobInfo.trace) + cout << "\n====== Aggregation RowGroups ======" << endl + << "projected RG: " << projRG.toString() << endl + << "aggregated1 RG: " << aggRgPm.toString() << endl + << "aggregated2 RG: " << aggRgUm.toString() << endl; +} + +void TupleAggregateStep::prep2PhasesDistinctAggregate(JobInfo& jobInfo, vector& rowgroups, + vector& aggregators) +{ + // check if there are any aggregate columns + // a vector that has the aggregate function to be done by PM + vector> aggColVec, aggNoDistColVec; + set avgSet, avgDistSet; + vector>& returnedColVec = jobInfo.returnedColVec; + // For UDAF + uint32_t projColsUDAFIdx = 0; + uint32_t udafcParamIdx = 0; + UDAFColumn* udafc = NULL; + mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; + + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + // col should be an aggregate or groupBy or window function + uint32_t rtcKey = returnedColVec[i].first; + uint32_t rtcOp = returnedColVec[i].second; + + if (rtcOp == 0 && + find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), rtcKey) != + jobInfo.distinctColVec.end() && + find(jobInfo.groupByColVec.begin(), jobInfo.groupByColVec.end(), rtcKey) == + jobInfo.groupByColVec.end() && + jobInfo.windowSet.find(rtcKey) != jobInfo.windowSet.end()) + { + Message::Args args; + args.add(keyName(i, rtcKey, jobInfo)); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + cerr << "prep2PhasesDistinctAggregate: " << emsg + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[rtcKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[rtcKey].fTable + << ", view=" << jobInfo.keyInfo->tupleKeyVec[rtcKey].fView << endl; + throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); + } + + // skip if not an aggregation column + if (returnedColVec[i].second == 0) + continue; + + aggColVec.push_back(returnedColVec[i]); + + // remember if a column has an average function, + // with avg function, no need for separate sum or count_column_name + if (returnedColVec[i].second == AggregateColumn::AVG) + avgSet.insert(returnedColVec[i].first); + + if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) + avgDistSet.insert(returnedColVec[i].first); + } + + // populate the aggregate rowgroup on PM and UM + // PM: projectedRG -> aggregateRGPM + // UM: aggregateRGPM -> aggregateRGUM + // + // Aggregate preparation by joblist factory: + // 1. get projected rowgroup (done by doAggProject) -- input to PM AGG + // 2. construct aggregate rowgroup -- output of PM, input of UM + // 3. construct aggregate rowgroup -- output of UM + // 4. construct aggregate rowgroup -- output of distinct aggregates + + const RowGroup projRG = rowgroups[0]; + const vector& oidsProj = projRG.getOIDs(); + const vector& keysProj = projRG.getKeys(); + const vector& scaleProj = projRG.getScale(); + const vector& precisionProj = projRG.getPrecision(); + const vector& typeProj = projRG.getColTypes(); + const vector& csNumProj = projRG.getCharsetNumbers(); + + vector posAggPm, posAggUm, posAggDist; + vector oidsAggPm, oidsAggUm, oidsAggDist; + vector keysAggPm, keysAggUm, keysAggDist; + vector scaleAggPm, scaleAggUm, scaleAggDist; + vector precisionAggPm, precisionAggUm, precisionAggDist; + vector typeAggPm, typeAggUm, typeAggDist; + vector csNumAggPm, csNumAggUm, csNumAggDist; + vector widthAggPm, widthAggUm, widthAggDist; + + vector groupByPm, groupByUm, groupByNoDist; + vector functionVecPm, functionNoDistVec, functionVecUm; + list multiParmIndexes; + + uint32_t bigIntWidth = sizeof(int64_t); + map, uint64_t> avgFuncDistMap; + AGG_MAP aggFuncMap; + + // associate the columns between projected RG and aggregate RG on PM + // populated the aggregate columns + // the groupby columns are put in front, even not a returned column + // sum and count(column name) are omitted, if avg present + { + // project only unique oids, but they may be repeated in aggregation + // collect the projected column info, prepare for aggregation + vector width; + map projColPosMap; + + for (uint64_t i = 0; i < keysProj.size(); i++) + { + projColPosMap.insert(make_pair(keysProj[i], i)); + width.push_back(projRG.getColumnWidth(i)); + } + + // column index for PM aggregate rowgroup + uint64_t colAggPm = 0; + uint64_t multiParm = 0; + + // for groupby column + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + uint32_t key = jobInfo.groupByColVec[i]; + + if (projColPosMap.find(key) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep2PhasesDistinctAggregate: group " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + uint64_t colProj = projColPosMap[key]; + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); + groupByPm.push_back(groupby); + + // PM: just copy down to aggregation rowgroup + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(key); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + + aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL), + colAggPm)); + colAggPm++; + } + + // for distinct column + for (uint64_t i = 0; i < jobInfo.distinctColVec.size(); i++) + { + uint32_t key = jobInfo.distinctColVec[i]; + + if (projColPosMap.find(key) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' isn't in tuple."; + cerr << "prep2PhasesDistinctAggregate: distinct " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[key].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[key].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[key].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[key].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + // check for dup distinct column -- @bug6126 + if (find(keysAggPm.begin(), keysAggPm.end(), key) != keysAggPm.end()) + continue; + + uint64_t colProj = projColPosMap[key]; + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(colProj, colAggPm)); + groupByPm.push_back(groupby); + + // PM: just copy down to aggregation rowgroup + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(key); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + + aggFuncMap.insert(make_pair(boost::make_tuple(keysAggPm[colAggPm], 0, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL), + colAggPm)); + colAggPm++; + } + + // vectors for aggregate functions + RowAggFunctionType aggOp = ROWAGG_FUNCT_UNDEFINE; + RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; + for (uint64_t i = 0; i < aggColVec.size(); i++) + { + aggOp = functionIdMap(aggColVec[i].second); + + // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) + if (aggOp != ROWAGG_MULTI_PARM) + prevAggOp = aggOp; + + // skip on PM if this is a constant + if (aggOp == ROWAGG_CONSTANT) + continue; + + pUDAFFunc = NULL; + uint32_t aggKey = aggColVec[i].first; + + if (projColPosMap.find(aggKey) == projColPosMap.end()) + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[aggKey] << "' isn't in tuple."; + cerr << "prep2PhasesDistinctAggregate: aggregate " << emsg.str() + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[aggKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fTable; + + if (jobInfo.keyInfo->tupleKeyVec[aggKey].fView.length() > 0) + cerr << ", view=" << jobInfo.keyInfo->tupleKeyVec[aggKey].fView; + + cerr << endl; + throw logic_error(emsg.str()); + } + + RowAggFunctionType stats = statsFuncIdMap(aggOp); + + // skip sum / count(column) if avg is also selected + if ((aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) && (avgSet.find(aggKey) != avgSet.end())) + continue; + + // We skip distinct aggs, including extra parms. These are handled by adding them to group by list + // above. + if (aggOp == ROWAGG_DISTINCT_SUM || aggOp == ROWAGG_DISTINCT_AVG || + aggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) + continue; + if (aggOp == ROWAGG_MULTI_PARM && prevAggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) + continue; + + uint64_t colProj = projColPosMap[aggKey]; + SP_ROWAGG_FUNC_t funct; + + if (aggOp == ROWAGG_UDAF) + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) + { + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) + { + for (uint64_t k = i + 1; + k < aggColVec.size() && aggColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(aggColVec[k].first); + } + } + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + } + if (it == jobInfo.projectionCols.end()) + { + throw logic_error( + "(1)prep2PhasesDistinctAggregate: A UDAF function is called but there\'s not enough " + "UDAFColumns"); + } + } + else + { + funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); + } + + // skip if this is a duplicate + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && + aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL)) != + aggFuncMap.end()) + { + // skip if this is a duplicate + continue; + } + + functionVecPm.push_back(funct); + aggFuncMap.insert(make_pair( + boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + colAggPm - multiParm)); + + switch (aggOp) + { + case ROWAGG_MIN: + case ROWAGG_MAX: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + colAggPm++; + } + break; + + case ROWAGG_SUM: + case ROWAGG_AVG: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("sum/average"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep2PhasesDistinctAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + csNumAggPm.push_back(8); + wideDecimalOrLongDouble(colProj, typeProj[colProj], precisionProj, scaleProj, width, typeAggPm, + scaleAggPm, precisionAggPm, widthAggPm); + colAggPm++; + } + + // PM: put the count column for avg next to the sum + // let fall through to add a count column for average function + if (aggOp == ROWAGG_AVG) + funct->fAuxColumnIndex = colAggPm; + else + break; + /* fall through */ + + case ROWAGG_COUNT_ASTERISK: + case ROWAGG_COUNT_COL_NAME: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + // work around count() in select subquery + precisionAggPm.push_back(9999); + + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } + + csNumAggPm.push_back(8); + widthAggPm.push_back(bigIntWidth); + colAggPm++; + } + break; + + case ROWAGG_STATS: + { + if (typeProj[colProj] == CalpontSystemCatalog::CHAR || + typeProj[colProj] == CalpontSystemCatalog::VARCHAR || + typeProj[colProj] == CalpontSystemCatalog::BLOB || + typeProj[colProj] == CalpontSystemCatalog::TEXT || + typeProj[colProj] == CalpontSystemCatalog::DATE || + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || + typeProj[colProj] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("variance/standard deviation"); + args.add(colTypeIdString(typeProj[colProj])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep2PhasesDistinctAggregate:: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + // count(x) + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(0); + typeAggPm.push_back(CalpontSystemCatalog::DOUBLE); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(double)); + funct->fAuxColumnIndex = ++colAggPm; + + // sum(x) + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(-1); + typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(long double)); + ++colAggPm; + + // sum(x**2) + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(-1); + typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(long double)); + ++colAggPm; + } + break; + + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(-16); // for connector to skip null check + + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } + + csNumAggPm.push_back(8); + widthAggPm.push_back(bigIntWidth); + ++colAggPm; + } + break; + + case ROWAGG_UDAF: + { + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funct.get()); + + if (!udafFuncCol) + { + throw logic_error( + "(2)prep2PhasesDistinctAggregate: A UDAF function is called but there's no " + "RowUDAFFunctionCol"); + } + + // Return column + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(udafFuncCol->fUDAFContext.getScale()); + precisionAggPm.push_back(udafFuncCol->fUDAFContext.getPrecision()); + typeAggPm.push_back(udafFuncCol->fUDAFContext.getResultType()); + csNumAggPm.push_back(udafFuncCol->fUDAFContext.getCharsetNumber()); + widthAggPm.push_back(udafFuncCol->fUDAFContext.getColWidth()); + ++colAggPm; + // Column for index of UDAF UserData struct + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(0); + precisionAggPm.push_back(0); + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggPm.push_back(8); + widthAggPm.push_back(sizeof(uint64_t)); + funct->fAuxColumnIndex = colAggPm++; + // If the first param is const + udafcParamIdx = 0; + ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) + { + funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; + } + ++udafcParamIdx; + break; + } + + case ROWAGG_MULTI_PARM: + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + scaleAggPm.push_back(scaleProj[colProj]); + precisionAggPm.push_back(precisionProj[colProj]); + typeAggPm.push_back(typeProj[colProj]); + csNumAggPm.push_back(csNumProj[colProj]); + widthAggPm.push_back(width[colProj]); + multiParmIndexes.push_back(colAggPm); + ++colAggPm; + ++multiParm; + // If the param is const + if (udafc) + { + if (udafcParamIdx > udafc->aggParms().size() - 1) + { + throw QueryDataExcept("prep2PhasesDistinctAggregate: UDAF multi function with too many parms", + aggregateFuncErr); + } + ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) + { + funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; + } + } + else if (prevAggOp != ROWAGG_COUNT_DISTINCT_COL_NAME) + { + throw QueryDataExcept("prep2PhasesDistinctAggregate: UDAF multi function with no parms", + aggregateFuncErr); + } + ++udafcParamIdx; + } + break; + + default: + { + ostringstream emsg; + emsg << "aggregate function (" << (uint64_t)aggOp << ") isn't supported"; + cerr << "prep2PhasesDistinctAggregate: " << emsg.str() << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); + } + } + } + } + + // associate the columns between the aggregate RGs on PM and UM without distinct aggregator + // populated the returned columns + { + int64_t multiParms = 0; + + for (uint32_t idx = 0; idx < groupByPm.size(); idx++) + { + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(idx, idx)); + groupByUm.push_back(groupby); + } + + for (uint32_t idx = 0; idx < functionVecPm.size(); idx++) + { + SP_ROWAGG_FUNC_t funct; + SP_ROWAGG_FUNC_t funcPm = functionVecPm[idx]; + + if (funcPm->fAggFunction == ROWAGG_MULTI_PARM) + { + // Skip on UM: Extra parms for an aggregate have no work on the UM + ++multiParms; + continue; + } + + if (funcPm->fAggFunction == ROWAGG_UDAF) + { + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funcPm.get()); + if (!udafFuncCol) + { + throw logic_error( + "(3)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + funct.reset(new RowUDAFFunctionCol(udafFuncCol->fUDAFContext, udafFuncCol->fOutputColumnIndex, + udafFuncCol->fOutputColumnIndex - multiParms, + udafFuncCol->fAuxColumnIndex - multiParms)); + functionNoDistVec.push_back(funct); + pUDAFFunc = udafFuncCol->fUDAFContext.getFunction(); + } + else + { + funct.reset(new RowAggFunctionCol(funcPm->fAggFunction, funcPm->fStatsFunction, + funcPm->fOutputColumnIndex, funcPm->fOutputColumnIndex - multiParms, + funcPm->fAuxColumnIndex - multiParms)); + functionNoDistVec.push_back(funct); + pUDAFFunc = NULL; + } + } + + // Copy over the PM arrays to the UM. Skip any that are a multi-parm entry. + for (uint32_t idx = 0; idx < oidsAggPm.size(); ++idx) + { + if (find(multiParmIndexes.begin(), multiParmIndexes.end(), idx) != multiParmIndexes.end()) + { + continue; + } + oidsAggUm.push_back(oidsAggPm[idx]); + keysAggUm.push_back(keysAggPm[idx]); + scaleAggUm.push_back(scaleAggPm[idx]); + precisionAggUm.push_back(precisionAggPm[idx]); + widthAggUm.push_back(widthAggPm[idx]); + typeAggUm.push_back(typeAggPm[idx]); + csNumAggUm.push_back(csNumAggPm[idx]); + } + } + + // associate the columns between the aggregate RGs on UM and Distinct + // populated the returned columns + // remove not returned groupby column + // add back sum or count(column name) if omitted due to avg column + // put count(column name) column to the end, if it is for avg only + { + // Keep a count of the parms after the first for any aggregate. + // These will be skipped and the count needs to be subtracted + // from where the aux column will be. + int64_t multiParms = 0; + projColsUDAFIdx = 0; + // check if the count column for AVG is also a returned column, + // if so, replace the "-1" to actual position in returned vec. + map avgFuncMap, avgDistFuncMap; + AGG_MAP aggDupFuncMap; + + // copy over the groupby vector + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(i, -1)); + groupByNoDist.push_back(groupby); + } + + // locate the return column position in aggregated rowgroup from PM + // outIdx is i without the multi-columns, + uint64_t outIdx = 0; + RowAggFunctionType prevAggOp = ROWAGG_FUNCT_UNDEFINE; + uint32_t prevRetKey = 0; + + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + pUDAFFunc = NULL; + udafc = NULL; + uint32_t retKey = returnedColVec[i].first; + + RowAggFunctionType aggOp = functionIdMap(returnedColVec[i].second); + RowAggFunctionType stats = statsFuncIdMap(returnedColVec[i].second); + int colUm = -1; + + if (aggOp == ROWAGG_MULTI_PARM) + { + // Duplicate detection doesn't work for multi-parm` + + // If this function was earlier detected as a duplicate, unduplicate it. + SP_ROWAGG_FUNC_t funct = functionVecUm.back(); + if (funct->fAggFunction == ROWAGG_DUP_FUNCT) + funct->fAggFunction = prevAggOp; + + // Remove it from aggDupFuncMap if it's in there. + funct->hasMultiParm = true; + AGG_MAP::iterator it = aggDupFuncMap.find(boost::make_tuple( + prevRetKey, prevAggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + if (it != aggDupFuncMap.end()) + { + aggDupFuncMap.erase(it); + } + // Skip further UM porocessing of the multi-parm: Extra parms for an aggregate have no work on the UM + ++multiParms; + continue; + } + else + { + // Save the op for MULTI_PARM exclusion when COUNT(DISTINCT) + prevAggOp = aggOp; + prevRetKey = returnedColVec[i].first; + } + + if (aggOp == ROWAGG_UDAF) + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) + { + udafc = dynamic_cast((*it).get()); + projColsUDAFIdx++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Save the multi-parm keys for dup-detection. + if (pUDAFFunc && udafc->getContext().getParamKeys()->size() == 0) + { + for (uint64_t k = i + 1; + k < returnedColVec.size() && returnedColVec[k].second == AggregateColumn::MULTI_PARM; ++k) + { + udafc->getContext().getParamKeys()->push_back(returnedColVec[k].first); + } + } + break; + } + } + if (it == jobInfo.projectionCols.end()) + { + throw logic_error( + "(4)prep2PhasesDistinctAggregate: A UDAF function is called but there\'s not enough " + "UDAFColumns"); + } + } + + if (find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), retKey) != + jobInfo.distinctColVec.end()) + { + AGG_MAP::iterator it = aggFuncMap.find( + boost::make_tuple(retKey, 0, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (it != aggFuncMap.end()) + { + colUm = it->second; + } + } + + if (colUm > -1) // Means we found a DISTINCT and have a column number + { + switch (aggOp) + { + case ROWAGG_DISTINCT_AVG: + + // avgFuncMap.insert(make_pair(key, funct)); + case ROWAGG_DISTINCT_SUM: + { + if (typeAggUm[colUm] == CalpontSystemCatalog::CHAR || + typeAggUm[colUm] == CalpontSystemCatalog::VARCHAR || + typeAggUm[colUm] == CalpontSystemCatalog::BLOB || + typeAggUm[colUm] == CalpontSystemCatalog::TEXT || + typeAggUm[colUm] == CalpontSystemCatalog::DATE || + typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || + typeAggUm[colUm] == CalpontSystemCatalog::TIMESTAMP || + typeAggUm[colUm] == CalpontSystemCatalog::TIME) + { + Message::Args args; + args.add("sum/average"); + args.add(colTypeIdString(typeAggUm[colUm])); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATE_TYPE_NOT_SUPPORT, args); + cerr << "prep2PhasesDistinctAggregate: " << emsg << endl; + throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); + } + + oidsAggDist.push_back(oidsAggUm[colUm]); + keysAggDist.push_back(retKey); + csNumAggDist.push_back(8); + wideDecimalOrLongDouble(colUm, typeAggPm[colUm], precisionAggPm, scaleAggPm, widthAggPm, + typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); + } + // PM: put the count column for avg next to the sum + // let fall through to add a count column for average function + // if (aggOp != ROWAGG_DISTINCT_AVG) + break; + + case ROWAGG_COUNT_DISTINCT_COL_NAME: + { + oidsAggDist.push_back(oidsAggUm[colUm]); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(0); + // work around count() in select subquery + precisionAggDist.push_back(9999); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); + } + break; + + default: + // could happen if agg and agg distinct use same column. + colUm = -1; + break; + } // switch + } + // For non distinct aggregates + if (colUm == -1) + { + AGG_MAP::iterator it = aggFuncMap.find( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (it != aggFuncMap.end()) + { + colUm = it->second; + oidsAggDist.push_back(oidsAggUm[colUm]); + keysAggDist.push_back(keysAggUm[colUm]); + scaleAggDist.push_back(scaleAggUm[colUm]); + precisionAggDist.push_back(precisionAggUm[colUm]); + typeAggDist.push_back(typeAggUm[colUm]); + csNumAggDist.push_back(csNumAggUm[colUm]); + widthAggDist.push_back(widthAggUm[colUm]); + } + + // not a direct hit -- a returned column is not already in the RG from PMs + else + { + bool returnColMissing = true; + + // check if a SUM or COUNT covered by AVG + if (aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) + { + it = aggFuncMap.find(boost::make_tuple(returnedColVec[i].first, ROWAGG_AVG, pUDAFFunc, + udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (it != aggFuncMap.end()) + { + // false alarm + returnColMissing = false; + + colUm = it->second; + + if (aggOp == ROWAGG_SUM) + { + oidsAggDist.push_back(oidsAggUm[colUm]); + keysAggDist.push_back(retKey); + csNumAggDist.push_back(8); + wideDecimalOrLongDouble(colUm, typeAggUm[colUm], precisionAggUm, scaleAggUm, widthAggUm, + typeAggDist, scaleAggDist, precisionAggDist, widthAggDist); + } + else + { + // leave the count() to avg + aggOp = ROWAGG_COUNT_NO_OP; + + oidsAggDist.push_back(oidsAggUm[colUm]); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(0); + if (isUnsigned(typeAggUm[colUm])) + { + precisionAggDist.push_back(20); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + precisionAggDist.push_back(19); + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + } + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); + } + } + } + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), retKey) != + jobInfo.expressionVec.end()) + { + // a function on aggregation + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggDist.push_back(ti.oid); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(ti.scale); + precisionAggDist.push_back(ti.precision); + typeAggDist.push_back(ti.dtype); + csNumAggDist.push_back(ti.csNum); + widthAggDist.push_back(ti.width); + + returnColMissing = false; + } + else if (jobInfo.windowSet.find(retKey) != jobInfo.windowSet.end()) + { + // a window function + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggDist.push_back(ti.oid); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(ti.scale); + precisionAggDist.push_back(ti.precision); + typeAggDist.push_back(ti.dtype); + csNumAggDist.push_back(ti.csNum); + widthAggDist.push_back(ti.width); + + returnColMissing = false; + } + else if (aggOp == ROWAGG_CONSTANT) + { + TupleInfo ti = getTupleInfo(retKey, jobInfo); + oidsAggDist.push_back(ti.oid); + keysAggDist.push_back(retKey); + scaleAggDist.push_back(ti.scale); + precisionAggDist.push_back(ti.precision); + typeAggDist.push_back(ti.dtype); + csNumAggDist.push_back(ti.csNum); + widthAggDist.push_back(ti.width); + + returnColMissing = false; + } + + if (returnColMissing) + { + Message::Args args; + args.add(keyName(outIdx, retKey, jobInfo)); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + cerr << "prep2PhasesDistinctAggregate: " << emsg + << " oid=" << (int)jobInfo.keyInfo->tupleKeyVec[retKey].fId + << ", alias=" << jobInfo.keyInfo->tupleKeyVec[retKey].fTable + << ", view=" << jobInfo.keyInfo->tupleKeyVec[retKey].fView << ", function=" << (int)aggOp + << endl; + throw IDBExcept(emsg, ERR_NOT_GROUPBY_EXPRESSION); + } + } // else not a direct hit + } // else not a DISTINCT + + // update groupby vector if the groupby column is a returned column + if (returnedColVec[i].second == 0) + { + int dupGroupbyIndex = -1; + + for (uint64_t j = 0; j < jobInfo.groupByColVec.size(); j++) + { + if (jobInfo.groupByColVec[j] == retKey) + { + if (groupByNoDist[j]->fOutputColumnIndex == (uint32_t)-1) + groupByNoDist[j]->fOutputColumnIndex = outIdx; + else + dupGroupbyIndex = groupByNoDist[j]->fOutputColumnIndex; + } + } + + // a duplicate group by column + if (dupGroupbyIndex != -1) + functionVecUm.push_back(SP_ROWAGG_FUNC_t( + new RowAggFunctionCol(ROWAGG_DUP_FUNCT, ROWAGG_FUNCT_UNDEFINE, -1, outIdx, dupGroupbyIndex))); + } + else + { + // update the aggregate function vector + SP_ROWAGG_FUNC_t funct; + if (aggOp == ROWAGG_UDAF) + { + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colUm, outIdx)); + } + else + { + funct.reset(new RowAggFunctionCol(aggOp, stats, colUm, outIdx)); + } + + if (aggOp == ROWAGG_COUNT_NO_OP) + funct->fAuxColumnIndex = colUm; + else if (aggOp == ROWAGG_CONSTANT) + funct->fAuxColumnIndex = jobInfo.cntStarPos; + + functionVecUm.push_back(funct); + + // find if this func is a duplicate + AGG_MAP::iterator iter = aggDupFuncMap.find( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); + + if (iter != aggDupFuncMap.end()) + { + if (funct->fAggFunction == ROWAGG_AVG) + funct->fAggFunction = ROWAGG_DUP_AVG; + else if (funct->fAggFunction == ROWAGG_STATS) + funct->fAggFunction = ROWAGG_DUP_STATS; + else if (funct->fAggFunction == ROWAGG_UDAF) + funct->fAggFunction = ROWAGG_DUP_UDAF; + else + funct->fAggFunction = ROWAGG_DUP_FUNCT; + + funct->fAuxColumnIndex = iter->second; + } + else + { + aggDupFuncMap.insert(make_pair( + boost::make_tuple(retKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), + funct->fOutputColumnIndex)); + } + + if (returnedColVec[i].second == AggregateColumn::AVG) + avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); + else if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) + avgDistFuncMap.insert(make_pair(returnedColVec[i].first, funct)); + } + ++outIdx; + } // for (i + + // now fix the AVG function, locate the count(column) position + for (uint64_t i = 0; i < functionVecUm.size(); i++) + { + // if the count(k) can be associated with an avg(k) + if (functionVecUm[i]->fAggFunction == ROWAGG_COUNT_NO_OP) + { + map::iterator k = + avgFuncMap.find(keysAggDist[functionVecUm[i]->fOutputColumnIndex]); + + if (k != avgFuncMap.end()) + k->second->fAuxColumnIndex = functionVecUm[i]->fOutputColumnIndex; + } + } + + // there is avg(k), but no count(k) in the select list + uint64_t lastCol = outIdx; + + for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) + { + if (k->second->fAuxColumnIndex == (uint32_t)-1) + { + k->second->fAuxColumnIndex = lastCol++; + oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); + keysAggDist.push_back(k->first); + scaleAggDist.push_back(0); + precisionAggDist.push_back(19); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); + } + } + + // distinct avg + for (uint64_t i = 0; i < functionVecUm.size(); i++) + { + if (functionVecUm[i]->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME && !functionVecUm[i]->hasMultiParm) + { + map::iterator k = + avgDistFuncMap.find(keysAggDist[functionVecUm[i]->fOutputColumnIndex]); + + if (k != avgDistFuncMap.end()) + { + k->second->fAuxColumnIndex = functionVecUm[i]->fOutputColumnIndex; + functionVecUm[i]->fAggFunction = ROWAGG_COUNT_NO_OP; + } + } + } + + // there is avg(distinct k), but no count(distinct k) in the select list + for (map::iterator k = avgDistFuncMap.begin(); k != avgDistFuncMap.end(); k++) + { + // find count(distinct k) or add it + if (k->second->fAuxColumnIndex == (uint32_t)-1) + { + k->second->fAuxColumnIndex = lastCol++; + oidsAggDist.push_back(jobInfo.keyInfo->tupleKeyVec[k->first].fId); + keysAggDist.push_back(k->first); + scaleAggDist.push_back(0); + precisionAggDist.push_back(19); + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(bigIntWidth); + } + } + + // add auxiliary fields for UDAF and statistics functions + for (uint64_t i = 0; i < functionVecUm.size(); i++) + { + uint64_t j = functionVecUm[i]->fInputColumnIndex; + + if (functionVecUm[i]->fAggFunction == ROWAGG_UDAF) + { + // Column for index of UDAF UserData struct + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(functionVecUm[i].get()); + + if (!udafFuncCol) + { + throw logic_error( + "(5)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + } + + functionVecUm[i]->fAuxColumnIndex = lastCol++; + oidsAggDist.push_back(oidsAggPm[j]); // Dummy? + keysAggDist.push_back(keysAggPm[j]); // Dummy? + scaleAggDist.push_back(0); + precisionAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + csNumAggDist.push_back(8); + widthAggDist.push_back(sizeof(uint64_t)); + continue; + } + + if (functionVecUm[i]->fAggFunction != ROWAGG_STATS) + continue; + + functionVecUm[i]->fAuxColumnIndex = lastCol; + + // sum(x) + oidsAggDist.push_back(oidsAggPm[j]); + keysAggDist.push_back(keysAggPm[j]); + scaleAggDist.push_back(0); + precisionAggDist.push_back(-1); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggDist.push_back(8); + widthAggDist.push_back(sizeof(long double)); + ++lastCol; + + // sum(x**2) + oidsAggDist.push_back(oidsAggPm[j]); + keysAggDist.push_back(keysAggPm[j]); + scaleAggDist.push_back(0); + precisionAggDist.push_back(-1); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + csNumAggDist.push_back(8); + widthAggDist.push_back(sizeof(long double)); + ++lastCol; + } + } + + // calculate the offset and create the rowaggregations, rowgroups + posAggUm.push_back(2); // rid + + for (uint64_t i = 0; i < oidsAggUm.size(); i++) + posAggUm.push_back(posAggUm[i] + widthAggUm[i]); + + RowGroup aggRgUm(oidsAggUm.size(), posAggUm, oidsAggUm, keysAggUm, typeAggUm, csNumAggUm, scaleAggUm, + precisionAggUm, jobInfo.stringTableThreshold); + SP_ROWAGG_UM_t rowAggUm( + new RowAggregationUMP2(groupByUm, functionNoDistVec, jobInfo.rm, jobInfo.umMemLimit)); + rowAggUm->timeZone(jobInfo.timeZone); + + posAggDist.push_back(2); // rid + + for (uint64_t i = 0; i < oidsAggDist.size(); i++) + posAggDist.push_back(posAggDist[i] + widthAggDist[i]); + + RowGroup aggRgDist(oidsAggDist.size(), posAggDist, oidsAggDist, keysAggDist, typeAggDist, csNumAggDist, + scaleAggDist, precisionAggDist, jobInfo.stringTableThreshold); + SP_ROWAGG_DIST rowAggDist( + new RowAggregationDistinct(groupByNoDist, functionVecUm, jobInfo.rm, jobInfo.umMemLimit)); + rowAggDist->timeZone(jobInfo.timeZone); + + // if distinct key word applied to more than one aggregate column, reset rowAggDist + vector subRgVec; + + if (jobInfo.distinctColVec.size() > 1) + { + RowAggregationMultiDistinct* multiDistinctAggregator = + new RowAggregationMultiDistinct(groupByNoDist, functionVecUm, jobInfo.rm, jobInfo.umMemLimit); + multiDistinctAggregator->timeZone(jobInfo.timeZone); + rowAggDist.reset(multiDistinctAggregator); + + // construct and add sub-aggregators to rowAggDist + vector posAggGb, posAggSub; + vector oidsAggGb, oidsAggSub; + vector keysAggGb, keysAggSub; + vector scaleAggGb, scaleAggSub; + vector precisionAggGb, precisionAggSub; + vector typeAggGb, typeAggSub; + vector csNumAggGb, csNumAggSub; + vector widthAggGb, widthAggSub; + + // populate groupby column info + for (uint64_t i = 0; i < jobInfo.groupByColVec.size(); i++) + { + oidsAggGb.push_back(oidsAggUm[i]); + keysAggGb.push_back(keysAggUm[i]); + scaleAggGb.push_back(scaleAggUm[i]); + precisionAggGb.push_back(precisionAggUm[i]); + typeAggGb.push_back(typeAggUm[i]); + csNumAggGb.push_back(csNumAggUm[i]); + widthAggGb.push_back(widthAggUm[i]); + } + + // for distinct, each column requires a seperate rowgroup + vector rowAggSubDistVec; + + uint32_t distinctColKey; + int64_t j; + uint64_t k; + uint64_t outIdx = 0; + for (uint64_t i = 0; i < returnedColVec.size(); i++) + { + if (returnedColVec[i].second == 0) + { + ++outIdx; + continue; + } + + j = -1; + + distinctColKey = -1; + // Find the entry in distinctColVec, if any + for (k = 0; k < jobInfo.distinctColVec.size(); k++) + { + distinctColKey = jobInfo.distinctColVec[k]; + if (returnedColVec[i].first == distinctColKey) + break; + } + if (distinctColKey == (uint32_t)-1) + { + ++outIdx; + continue; + } + // locate the distinct key in the row group + for (k = 0; k < keysAggUm.size(); k++) + { + if (keysAggUm[k] == distinctColKey) + { + j = k; + break; + } + } + + idbassert(j != -1); + + oidsAggSub = oidsAggGb; + keysAggSub = keysAggGb; + scaleAggSub = scaleAggGb; + precisionAggSub = precisionAggGb; + typeAggSub = typeAggGb; + csNumAggSub = csNumAggGb; + widthAggSub = widthAggGb; + + oidsAggSub.push_back(oidsAggUm[j]); + keysAggSub.push_back(keysAggUm[j]); + scaleAggSub.push_back(scaleAggUm[j]); + precisionAggSub.push_back(precisionAggUm[j]); + typeAggSub.push_back(typeAggUm[j]); + csNumAggSub.push_back(csNumAggUm[j]); + widthAggSub.push_back(widthAggUm[j]); + + // construct groupby vector + vector groupBySub; + k = 0; + + while (k < jobInfo.groupByColVec.size()) + { + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(k, k)); + groupBySub.push_back(groupby); + k++; + } + // add the distinct column as groupby + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); + groupBySub.push_back(groupby); + + // Add multi parm distinct + while ((i + 1) < returnedColVec.size() && + functionIdMap(returnedColVec[i + 1].second) == ROWAGG_MULTI_PARM) + { + ++i; + uint32_t dColKey = -1; + j = -1; + + // Find the entry in distinctColVec, if any + for (k = 0; k < jobInfo.distinctColVec.size(); k++) + { + dColKey = jobInfo.distinctColVec[k]; + if (returnedColVec[i].first == dColKey) + break; + } + idbassert(dColKey != (uint32_t)-1); + // locate the distinct key in the row group + for (k = 0; k < keysAggUm.size(); k++) + { + if (keysAggUm[k] == dColKey) + { + j = k; + break; + } + } + idbassert(j != -1); + + oidsAggSub.push_back(oidsAggUm[j]); + keysAggSub.push_back(keysAggUm[j]); + scaleAggSub.push_back(scaleAggUm[j]); + precisionAggSub.push_back(precisionAggUm[j]); + typeAggSub.push_back(typeAggUm[j]); + csNumAggSub.push_back(csNumAggUm[j]); + widthAggSub.push_back(widthAggUm[j]); + + SP_ROWAGG_GRPBY_t groupby(new RowAggGroupByCol(j, k)); + groupBySub.push_back(groupby); + } + + // construct sub-rowgroup + posAggSub.clear(); + posAggSub.push_back(2); // rid + + for (k = 0; k < oidsAggSub.size(); k++) + posAggSub.push_back(posAggSub[k] + widthAggSub[k]); + + RowGroup subRg(oidsAggSub.size(), posAggSub, oidsAggSub, keysAggSub, typeAggSub, csNumAggSub, + scaleAggSub, precisionAggSub, jobInfo.stringTableThreshold); + subRgVec.push_back(subRg); + + // Keep a count of the parms after the first for any aggregate. + // These will be skipped and the count needs to be subtracted + // from where the aux column will be. + int64_t multiParms = 0; + + // tricky part : 2 function vectors + // -- dummy function vector for sub-aggregator, which does distinct only + // -- aggregate function on this distinct column for rowAggDist + vector functionSub1, functionSub2; + // search the function in functionVec + vector::iterator it = functionVecUm.begin(); + + while (it != functionVecUm.end()) + { + SP_ROWAGG_FUNC_t f = *it++; + + if ((f->fOutputColumnIndex == outIdx) && + (f->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME || f->fAggFunction == ROWAGG_DISTINCT_SUM || + f->fAggFunction == ROWAGG_DISTINCT_AVG)) + { + SP_ROWAGG_FUNC_t funct(new RowAggFunctionCol(f->fAggFunction, f->fStatsFunction, + groupBySub.size() - 1, f->fOutputColumnIndex, + f->fAuxColumnIndex - multiParms)); + functionSub2.push_back(funct); + } + } + + // construct sub-aggregator + SP_ROWAGG_UM_t subAgg( + new RowAggregationSubDistinct(groupBySub, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); + + // add to rowAggDist + multiDistinctAggregator->addSubAggregator(subAgg, subRg, functionSub2); + + ++outIdx; + } + + // cover any non-distinct column functions + { + vector functionSub1 = functionNoDistVec; + vector functionSub2; + int64_t multiParms = 0; + + for (uint64_t k = 0; k < returnedColVec.size(); k++) + { + // search non-distinct functions in functionVec + vector::iterator it = functionVecUm.begin(); + + while (it != functionVecUm.end()) + { + SP_ROWAGG_FUNC_t funct; + SP_ROWAGG_FUNC_t f = *it++; + + if (f->fOutputColumnIndex == k) + { + if (f->fAggFunction == ROWAGG_UDAF) + { + RowUDAFFunctionCol* udafFuncCol = dynamic_cast(f.get()); + funct.reset(new RowUDAFFunctionCol(udafFuncCol->fUDAFContext, udafFuncCol->fInputColumnIndex, + udafFuncCol->fOutputColumnIndex, + udafFuncCol->fAuxColumnIndex - multiParms)); + functionSub2.push_back(funct); + } + else if (f->fAggFunction == ROWAGG_COUNT_ASTERISK || f->fAggFunction == ROWAGG_COUNT_COL_NAME || + f->fAggFunction == ROWAGG_SUM || f->fAggFunction == ROWAGG_AVG || + f->fAggFunction == ROWAGG_MIN || f->fAggFunction == ROWAGG_MAX || + f->fAggFunction == ROWAGG_STATS || f->fAggFunction == ROWAGG_BIT_AND || + f->fAggFunction == ROWAGG_BIT_OR || f->fAggFunction == ROWAGG_BIT_XOR || + f->fAggFunction == ROWAGG_CONSTANT) + { + funct.reset(new RowAggFunctionCol(f->fAggFunction, f->fStatsFunction, f->fInputColumnIndex, + f->fOutputColumnIndex, f->fAuxColumnIndex - multiParms)); + functionSub2.push_back(funct); + } + } + } + } + + if (functionSub1.size() > 0) + { + // make sure the group by columns are available for next aggregate phase. + vector groupBySubNoDist; + + for (uint64_t i = 0; i < groupByNoDist.size(); i++) + groupBySubNoDist.push_back( + SP_ROWAGG_GRPBY_t(new RowAggGroupByCol(groupByNoDist[i]->fInputColumnIndex, i))); + + // construct sub-aggregator + SP_ROWAGG_UM_t subAgg( + new RowAggregationUMP2(groupBySubNoDist, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); + + // add to rowAggDist + multiDistinctAggregator->addSubAggregator(subAgg, aggRgUm, functionSub2); + subRgVec.push_back(aggRgUm); + } + } + } + + rowAggDist->addAggregator(rowAggUm, aggRgUm); + rowgroups.push_back(aggRgDist); + aggregators.push_back(rowAggDist); + + posAggPm.push_back(2); // rid + + for (uint64_t i = 0; i < oidsAggPm.size(); i++) + posAggPm.push_back(posAggPm[i] + widthAggPm[i]); + + RowGroup aggRgPm(oidsAggPm.size(), posAggPm, oidsAggPm, keysAggPm, typeAggPm, csNumAggPm, scaleAggPm, + precisionAggPm, jobInfo.stringTableThreshold); + SP_ROWAGG_PM_t rowAggPm(new RowAggregation(groupByPm, functionVecPm)); + rowAggPm->timeZone(jobInfo.timeZone); + rowgroups.push_back(aggRgPm); + aggregators.push_back(rowAggPm); + + if (jobInfo.trace) + { + cout << "projected RG: " << projRG.toString() << endl + << "aggregated1 RG: " << aggRgPm.toString() << endl + << "aggregated2 RG: " << aggRgUm.toString() << endl; + + for (uint64_t i = 0; i < subRgVec.size(); i++) + cout << "aggregatedSub RG: " << i << " " << subRgVec[i].toString() << endl; + + cout << "aggregatedDist RG: " << aggRgDist.toString() << endl; + } +} + +void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInfo& jobInfo) +{ + map keyToIndexMap; + + for (uint64_t i = 0; i < fRowGroupOut.getKeys().size(); ++i) + { + if (keyToIndexMap.find(fRowGroupOut.getKeys()[i]) == keyToIndexMap.end()) + keyToIndexMap.insert(make_pair(fRowGroupOut.getKeys()[i], i)); + } + + RetColsVector expressionVec; + ArithmeticColumn* ac = NULL; + FunctionColumn* fc = NULL; + RetColsVector& cols = jobInfo.nonConstCols; + vector simpleColumns; + + for (RetColsVector::iterator it = cols.begin(); it != cols.end(); ++it) + { + uint64_t eid = -1; + + if (((ac = dynamic_cast(it->get())) != NULL) && (ac->aggColumnList().size() > 0) && + (ac->windowfunctionColumnList().size() == 0)) + { + const vector& scols = ac->simpleColumnList(); + simpleColumns.insert(simpleColumns.end(), scols.begin(), scols.end()); + + eid = ac->expressionId(); + expressionVec.push_back(*it); + } + else if (((fc = dynamic_cast(it->get())) != NULL) && (fc->aggColumnList().size() > 0) && + (fc->windowfunctionColumnList().size() == 0)) + { + const vector& sCols = fc->simpleColumnList(); + simpleColumns.insert(simpleColumns.end(), sCols.begin(), sCols.end()); + + eid = fc->expressionId(); + expressionVec.push_back(*it); + } + + // update the output index + if (eid != (uint64_t)-1) + { + map::iterator mit = keyToIndexMap.find(getExpTupleKey(jobInfo, eid)); + + if (mit != keyToIndexMap.end()) + { + it->get()->outputIndex(mit->second); + } + else + { + ostringstream emsg; + emsg << "expression " << eid << " cannot be found in tuple."; + cerr << "prepExpressionOnAggregate: " << emsg.str() << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); + } + } + } + + // map the input indices + for (vector::iterator i = simpleColumns.begin(); i != simpleColumns.end(); i++) + { + CalpontSystemCatalog::OID oid = (*i)->oid(); + uint32_t key = getTupleKey(jobInfo, *i); + CalpontSystemCatalog::OID dictOid = joblist::isDictCol((*i)->colType()); + + if (dictOid > 0) + { + oid = dictOid; + key = jobInfo.keyInfo->dictKeyMap[key]; + } + + map::iterator mit = keyToIndexMap.find(key); + + if (mit != keyToIndexMap.end()) + { + (*i)->inputIndex(mit->second); + } + else + { + ostringstream emsg; + emsg << "'" << jobInfo.keyInfo->tupleKeyToName[key] << "' cannot be found in tuple."; + cerr << "prepExpressionOnAggregate: " << emsg.str() << " simple column: oid(" << oid << "), alias(" + << extractTableAlias(*i) << ")." << endl; + throw QueryDataExcept(emsg.str(), aggregateFuncErr); + } + } + + // add expression to UM aggregator + aggUM->expression(expressionVec); +} void TupleAggregateStep::addConstangAggregate(vector& constAggDataVec) { - fAggregator->constantAggregate(constAggDataVec); + fAggregator->constantAggregate(constAggDataVec); } - void TupleAggregateStep::aggregateRowGroups() { - RGData rgData; - bool more = true; - RowGroupDL* dlIn = NULL; + RGData rgData; + bool more = true; + RowGroupDL* dlIn = NULL; - if (!fDoneAggregate) - { - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for TupleAggregate step."); + if (!fDoneAggregate) + { + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for TupleAggregate step."); - dlIn = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + dlIn = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - if (dlIn == NULL) - throw logic_error("Input is not RowGroup data list in TupleAggregate step."); + if (dlIn == NULL) + throw logic_error("Input is not RowGroup data list in TupleAggregate step."); - if (fInputIter < 0) - fInputIter = dlIn->getIterator(); + if (fInputIter < 0) + fInputIter = dlIn->getIterator(); - more = dlIn->next(fInputIter, &rgData); - - if (traceOn()) dlTimes.setFirstReadTime(); - - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - - try - { - // this check covers the no row case - if (!more && cancelled()) - { - fDoneAggregate = true; - fEndOfResult = true; - } - - while (more && !fEndOfResult) - { - fRowGroupIn.setData(&rgData); - fAggregator->addRowGroup(&fRowGroupIn); - more = dlIn->next(fInputIter, &rgData); - - // error checking - if (cancelled()) - { - fEndOfResult = true; - - while (more) - more = dlIn->next(fInputIter, &rgData); - } - } - } // try - catch (...) - { - handleException(std::current_exception(), - logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, - "TupleAggregateStep::aggregateRowGroups()"); - fEndOfResult = true; - } - } - - fDoneAggregate = true; - - while (more) - more = dlIn->next(fInputIter, &rgData); + more = dlIn->next(fInputIter, &rgData); if (traceOn()) + dlTimes.setFirstReadTime(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + try { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); + // this check covers the no row case + if (!more && cancelled()) + { + fDoneAggregate = true; + fEndOfResult = true; + } + + while (more && !fEndOfResult) + { + fRowGroupIn.setData(&rgData); + fAggregator->addRowGroup(&fRowGroupIn); + more = dlIn->next(fInputIter, &rgData); + + // error checking + if (cancelled()) + { + fEndOfResult = true; + + while (more) + more = dlIn->next(fInputIter, &rgData); + } + } + } // try + catch (...) + { + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::aggregateRowGroups()"); + fEndOfResult = true; } + } + + fDoneAggregate = true; + + while (more) + more = dlIn->next(fInputIter, &rgData); + + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + } } void TupleAggregateStep::threadedAggregateFinalize(uint32_t threadID) @@ -5464,343 +5331,553 @@ void TupleAggregateStep::threadedAggregateFinalize(uint32_t threadID) void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) { - RGData rgData; - scoped_array rowBucketVecs(new RowBucketVec[fNumOfBuckets]); - scoped_array distRow; - scoped_array > distRowData; - uint32_t bucketID; - scoped_array bucketDone(new bool[fNumOfBuckets]); - vector hashLens; - bool locked = false; - bool more = true; - RowGroupDL* dlIn = nullptr; - uint32_t rgVecShift = float(fNumOfBuckets) / fNumOfThreads * threadID; + RGData rgData; + scoped_array rowBucketVecs(new RowBucketVec[fNumOfBuckets]); + scoped_array distRow; + scoped_array> distRowData; + uint32_t bucketID; + scoped_array bucketDone(new bool[fNumOfBuckets]); + vector hashLens; + bool locked = false; + bool more = true; + RowGroupDL* dlIn = nullptr; + uint32_t rgVecShift = float(fNumOfBuckets) / fNumOfThreads * threadID; - RowAggregationMultiDistinct* multiDist = nullptr; + RowAggregationMultiDistinct* multiDist = nullptr; - if (!fDoneAggregate) - { - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for delivery."); + if (!fDoneAggregate) + { + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for delivery."); - dlIn = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + dlIn = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - if (dlIn == nullptr) - throw logic_error("Input is not RowGroup data list in delivery step."); + if (dlIn == nullptr) + throw logic_error("Input is not RowGroup data list in delivery step."); - vector rgDatas; - - try - { - // this check covers the no row case - if (!more && cancelled()) - { - fDoneAggregate = true; - fEndOfResult = true; - } - - bool firstRead = true; - Row rowIn; - - while (more && !fEndOfResult) - { - fMutex.lock(); - locked = true; - - for (uint32_t c = 0; c < fNumOfRowGroups && !cancelled(); c++) - { - more = dlIn->next(fInputIter, &rgData); - - if (firstRead) - { - if (threadID == 0) - { - if (traceOn()) - dlTimes.setFirstReadTime(); - - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - } - - multiDist = dynamic_cast(fAggregator.get()); - - if (multiDist) - { - for (uint32_t i = 0; i < fNumOfBuckets; i++) - rowBucketVecs[i].resize(multiDist->subAggregators().size()); - - distRow.reset(new Row[multiDist->subAggregators().size()]); - distRowData.reset(new shared_array[ - multiDist->subAggregators().size()]); - - for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) - { - multiDist->subAggregators()[j]->getOutputRowGroup()->initRow( - &distRow[j], true); - distRowData[j].reset(new uint8_t[distRow[j].getSize()]); - distRow[j].setData(distRowData[j].get()); - hashLens.push_back(multiDist->subAggregators()[j]->aggMapKeyLength()); - } - } - else - { - for (uint32_t i = 0; i < fNumOfBuckets; i++) - rowBucketVecs[i].resize(1); - - if (dynamic_cast(fAggregator.get())) - hashLens.push_back(dynamic_cast(fAggregator.get())->aggregator()->aggMapKeyLength()); - else - hashLens.push_back(fAggregator->aggMapKeyLength()); - } - - fRowGroupIns[threadID] = fRowGroupIn; - fRowGroupIns[threadID].initRow(&rowIn); - firstRead = false; - } - - if (more) - { - fRowGroupIns[threadID].setData(&rgData); - - bool diskAggAllowed = fRm->getAllowDiskAggregation(); - int64_t memSize = fRowGroupIns[threadID].getSizeWithStrings(); - if (!fRm->getMemory(memSize, fSessionMemLimit, !diskAggAllowed)) - { - if (!diskAggAllowed) - { - rgDatas.clear(); // to short-cut the rest of processing - more = false; - fEndOfResult = true; - - if (status() == 0) - { - errorMessage(IDBErrorInfo::instance()->errorMsg( - ERR_AGGREGATION_TOO_BIG)); - status(ERR_AGGREGATION_TOO_BIG); - } - } - else - { - rgDatas.push_back(rgData); - } - break; - } - fMemUsage[threadID] += memSize; - rgDatas.push_back(rgData); - } - else - { - break; - } - } - - // input rowgroup and aggregator is finalized only right before hashjoin starts - // if there is. - if (fAggregators.empty()) - { - fAggregators.resize(fNumOfBuckets); - - for (uint32_t i = 0; i < fNumOfBuckets; i++) - { - fAggregators[i].reset(fAggregator->clone()); - fAggregators[i]->setInputOutput(fRowGroupIn, &fRowGroupOuts[i]); - } - } - - fMutex.unlock(); - locked = false; - - multiDist = dynamic_cast(fAggregator.get()); - - // dispatch rows to row buckets - if (multiDist) - { - for (uint32_t c = 0; c < rgDatas.size(); c++) - { - fRowGroupIns[threadID].setData(&rgDatas[c]); - - for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) - { - fRowGroupIns[threadID].getRow(0, &rowIn); - rowIn.setUserDataStore(rgDatas[c].getUserDataStore()); - - for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) - { - for (uint64_t k = 0; - k < multiDist->subAggregators()[j]->getGroupByCols().size(); - ++k) - { - rowIn.copyField(distRow[j], k, multiDist->subAggregators()[j]->getGroupByCols()[k].get()->fInputColumnIndex); - } - - // TBD This approach could potentiall - // put all values in on bucket. - uint64_t hash = rowgroup::hashRow(distRow[j], hashLens[j] - 1); - bucketID = hash % fNumOfBuckets; - rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); - rowIn.nextRow(); - } - } - } - } - else - { - for (uint32_t c = 0; c < rgDatas.size(); c++) - { - fRowGroupIns[threadID].setData(&rgDatas[c]); - fRowGroupIns[threadID].getRow(0, &rowIn); - rowIn.setUserDataStore(rgDatas[c].getUserDataStore()); - - for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) - { - // The key is the groupby columns, which are the leading columns. - // TBD This approach could potential - // put all values in on bucket. - uint64_t hash = rowgroup::hashRow(rowIn, hashLens[0] - 1); - int bucketID = hash% fNumOfBuckets; - rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); - rowIn.nextRow(); - } - } - } - - // insert to the hashmaps owned by each aggregator - bool done = false; - fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); - - while (!fEndOfResult && !done && !cancelled()) - { - bool didWork = false; - done = true; - - // each thread starts from its own bucket for better distribution - uint32_t shift = (rgVecShift++) % fNumOfBuckets; - for (uint32_t ci = 0; ci < fNumOfBuckets && !cancelled(); ci++) - { - uint32_t c = (ci + shift) % fNumOfBuckets; - if (!fEndOfResult && !bucketDone[c] && fAgg_mutex[c]->try_lock()) - { - try - { - didWork = true; - - if (multiDist) - dynamic_cast(fAggregators[c].get())->addRowGroup(&fRowGroupIns[threadID], rowBucketVecs[c]); - else - fAggregators[c]->addRowGroup(&fRowGroupIns[threadID], rowBucketVecs[c][0]); - } - catch (...) - { - fAgg_mutex[c]->unlock(); - throw; - } - - rowBucketVecs[c][0].clear(); - bucketDone[c] = true; - fAgg_mutex[c]->unlock(); - } - else if (!bucketDone[c]) - { - done = false; - } - } - - if (!didWork) - usleep(1000); // avoid using all CPU during busy wait - } - - rgDatas.clear(); - fRm->returnMemory(fMemUsage[threadID], fSessionMemLimit); - fMemUsage[threadID] = 0; - - if (cancelled()) - { - fEndOfResult = true; - fMutex.lock(); - - while (more) - more = dlIn->next(fInputIter, &rgData); - - fMutex.unlock(); - } - } - } // try - catch (...) - { - handleException(std::current_exception(), - logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, - "TupleAggregateStep::threadedAggregateRowGroups()[" + std::to_string(threadID) + "]"); - fEndOfResult = true; - fDoneAggregate = true; - } - } - - if (!locked) fMutex.lock(); - - while (more) - more = dlIn->next(fInputIter, &rgData); - - fMutex.unlock(); - locked = false; - - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - } -} - - -void TupleAggregateStep::doAggregate_singleThread() -{ - AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - RowGroupDL* dlp = dl->rowGroupDL(); - RGData rgData; + vector rgDatas; try { - if (!fDoneAggregate) - aggregateRowGroups(); + // this check covers the no row case + if (!more && cancelled()) + { + fDoneAggregate = true; + fEndOfResult = true; + } - if (fEndOfResult == false) + bool firstRead = true; + Row rowIn; + + while (more && !fEndOfResult) + { + fMutex.lock(); + locked = true; + + for (uint32_t c = 0; c < fNumOfRowGroups && !cancelled(); c++) { - // do the final aggregtion and deliver the results - // at least one RowGroup for aggregate results - if (dynamic_cast(fAggregator.get()) != NULL) + more = dlIn->next(fInputIter, &rgData); + + if (firstRead) + { + if (threadID == 0) { - dynamic_cast(fAggregator.get())->doDistinctAggregation(); + if (traceOn()) + dlTimes.setFirstReadTime(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); } - while (fAggregator->nextRowGroup()) + multiDist = dynamic_cast(fAggregator.get()); + + if (multiDist) { - fAggregator->finalize(); - fRowsReturned += fRowGroupOut.getRowCount(); - rgData = fRowGroupOut.duplicate(); - fRowGroupDelivered.setData(&rgData); + for (uint32_t i = 0; i < fNumOfBuckets; i++) + rowBucketVecs[i].resize(multiDist->subAggregators().size()); - if (fRowGroupOut.getColumnCount() > fRowGroupDelivered.getColumnCount()) - pruneAuxColumns(); + distRow.reset(new Row[multiDist->subAggregators().size()]); + distRowData.reset(new shared_array[multiDist->subAggregators().size()]); - dlp->insert(rgData); + for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) + { + multiDist->subAggregators()[j]->getOutputRowGroup()->initRow(&distRow[j], true); + distRowData[j].reset(new uint8_t[distRow[j].getSize()]); + distRow[j].setData(distRowData[j].get()); + hashLens.push_back(multiDist->subAggregators()[j]->aggMapKeyLength()); + } } + else + { + for (uint32_t i = 0; i < fNumOfBuckets; i++) + rowBucketVecs[i].resize(1); + + if (dynamic_cast(fAggregator.get())) + hashLens.push_back(dynamic_cast(fAggregator.get()) + ->aggregator() + ->aggMapKeyLength()); + else + hashLens.push_back(fAggregator->aggMapKeyLength()); + } + + fRowGroupIns[threadID] = fRowGroupIn; + fRowGroupIns[threadID].initRow(&rowIn); + firstRead = false; + } + + if (more) + { + fRowGroupIns[threadID].setData(&rgData); + + bool diskAggAllowed = fRm->getAllowDiskAggregation(); + int64_t memSize = fRowGroupIns[threadID].getSizeWithStrings(); + if (!fRm->getMemory(memSize, fSessionMemLimit, !diskAggAllowed)) + { + if (!diskAggAllowed) + { + rgDatas.clear(); // to short-cut the rest of processing + more = false; + fEndOfResult = true; + + if (status() == 0) + { + errorMessage(IDBErrorInfo::instance()->errorMsg(ERR_AGGREGATION_TOO_BIG)); + status(ERR_AGGREGATION_TOO_BIG); + } + } + else + { + rgDatas.push_back(rgData); + } + break; + } + fMemUsage[threadID] += memSize; + rgDatas.push_back(rgData); + } + else + { + break; + } } - } // try + + // input rowgroup and aggregator is finalized only right before hashjoin starts + // if there is. + if (fAggregators.empty()) + { + fAggregators.resize(fNumOfBuckets); + + for (uint32_t i = 0; i < fNumOfBuckets; i++) + { + fAggregators[i].reset(fAggregator->clone()); + fAggregators[i]->setInputOutput(fRowGroupIn, &fRowGroupOuts[i]); + } + } + + fMutex.unlock(); + locked = false; + + multiDist = dynamic_cast(fAggregator.get()); + + // dispatch rows to row buckets + if (multiDist) + { + for (uint32_t c = 0; c < rgDatas.size(); c++) + { + fRowGroupIns[threadID].setData(&rgDatas[c]); + + for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) + { + fRowGroupIns[threadID].getRow(0, &rowIn); + rowIn.setUserDataStore(rgDatas[c].getUserDataStore()); + + for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) + { + for (uint64_t k = 0; k < multiDist->subAggregators()[j]->getGroupByCols().size(); ++k) + { + rowIn.copyField( + distRow[j], k, + multiDist->subAggregators()[j]->getGroupByCols()[k].get()->fInputColumnIndex); + } + + // TBD This approach could potentiall + // put all values in on bucket. + uint64_t hash = rowgroup::hashRow(distRow[j], hashLens[j] - 1); + bucketID = hash % fNumOfBuckets; + rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); + rowIn.nextRow(); + } + } + } + } + else + { + for (uint32_t c = 0; c < rgDatas.size(); c++) + { + fRowGroupIns[threadID].setData(&rgDatas[c]); + fRowGroupIns[threadID].getRow(0, &rowIn); + rowIn.setUserDataStore(rgDatas[c].getUserDataStore()); + + for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) + { + // The key is the groupby columns, which are the leading columns. + // TBD This approach could potential + // put all values in on bucket. + uint64_t hash = rowgroup::hashRow(rowIn, hashLens[0] - 1); + int bucketID = hash % fNumOfBuckets; + rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); + rowIn.nextRow(); + } + } + } + + // insert to the hashmaps owned by each aggregator + bool done = false; + fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); + + while (!fEndOfResult && !done && !cancelled()) + { + bool didWork = false; + done = true; + + // each thread starts from its own bucket for better distribution + uint32_t shift = (rgVecShift++) % fNumOfBuckets; + for (uint32_t ci = 0; ci < fNumOfBuckets && !cancelled(); ci++) + { + uint32_t c = (ci + shift) % fNumOfBuckets; + if (!fEndOfResult && !bucketDone[c] && fAgg_mutex[c]->try_lock()) + { + try + { + didWork = true; + + if (multiDist) + dynamic_cast(fAggregators[c].get()) + ->addRowGroup(&fRowGroupIns[threadID], rowBucketVecs[c]); + else + fAggregators[c]->addRowGroup(&fRowGroupIns[threadID], rowBucketVecs[c][0]); + } + catch (...) + { + fAgg_mutex[c]->unlock(); + throw; + } + + rowBucketVecs[c][0].clear(); + bucketDone[c] = true; + fAgg_mutex[c]->unlock(); + } + else if (!bucketDone[c]) + { + done = false; + } + } + + if (!didWork) + usleep(1000); // avoid using all CPU during busy wait + } + + rgDatas.clear(); + fRm->returnMemory(fMemUsage[threadID], fSessionMemLimit); + fMemUsage[threadID] = 0; + + if (cancelled()) + { + fEndOfResult = true; + fMutex.lock(); + + while (more) + more = dlIn->next(fInputIter, &rgData); + + fMutex.unlock(); + } + } + } // try catch (...) { - handleException(std::current_exception(), - logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, - "TupleAggregateStep::doAggregate_singleThread()"); + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, + "TupleAggregateStep::threadedAggregateRowGroups()[" + std::to_string(threadID) + "]"); + fEndOfResult = true; + fDoneAggregate = true; + } + } + + if (!locked) + fMutex.lock(); + + while (more) + more = dlIn->next(fInputIter, &rgData); + + fMutex.unlock(); + locked = false; + + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + } +} + +void TupleAggregateStep::doAggregate_singleThread() +{ + AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); + RowGroupDL* dlp = dl->rowGroupDL(); + RGData rgData; + + try + { + if (!fDoneAggregate) + aggregateRowGroups(); + + if (fEndOfResult == false) + { + // do the final aggregtion and deliver the results + // at least one RowGroup for aggregate results + if (dynamic_cast(fAggregator.get()) != NULL) + { + dynamic_cast(fAggregator.get())->doDistinctAggregation(); + } + + while (fAggregator->nextRowGroup()) + { + fAggregator->finalize(); + fRowsReturned += fRowGroupOut.getRowCount(); + rgData = fRowGroupOut.duplicate(); + fRowGroupDelivered.setData(&rgData); + + if (fRowGroupOut.getColumnCount() > fRowGroupDelivered.getColumnCount()) + pruneAuxColumns(); + + dlp->insert(rgData); + } + } + } // try + catch (...) + { + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doAggregate_singleThread()"); + } + + if (traceOn()) + printCalTrace(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + // Bug 3136, let mini stats to be formatted if traceOn. + fEndOfResult = true; + dlp->endOfInput(); +} + +void TupleAggregateStep::doAggregate() +{ + // @bug4314. DO NOT access fAggregtor before the first read of input, + // because hashjoin may not have finalized fAggregator. + if (!fIsMultiThread) + return doAggregate_singleThread(); + + AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); + RowGroupDL* dlp = dl->rowGroupDL(); + ByteStream bs; + doThreadedAggregate(bs, dlp); + return; +} + +uint64_t TupleAggregateStep::doThreadedAggregate(ByteStream& bs, RowGroupDL* dlp) +{ + uint32_t i; + RGData rgData; + uint64_t rowCount = 0; + + try + { + if (!fDoneAggregate) + { + initializeMultiThread(); + + vector runners; // thread pool handles + runners.reserve(fNumOfThreads); // to prevent a resize during use + + // Start the aggregator threads + for (i = 0; i < fNumOfThreads; i++) + { + runners.push_back(jobstepThreadPool.invoke(ThreadedAggregator(this, i))); + } + + // Now wait for all those threads + jobstepThreadPool.join(runners); } - if (traceOn()) - printCalTrace(); + if (!cancelled()) + { + vector runners; + // use half of the threads because finalizing requires twice as + // much memory on average + uint32_t threads = std::max(1U, fNumOfThreads / 2); + runners.reserve(threads); + for (i = 0; i < threads; ++i) + { + runners.push_back(jobstepThreadPool.invoke(ThreadedAggregateFinalizer(this, i))); + } + jobstepThreadPool.join(runners); + } + if (dynamic_cast(fAggregator.get()) && fAggregator->aggMapKeyLength() > 0) + { + // 2nd phase multi-threaded aggregate + if (!fEndOfResult) + { + if (!fDoneAggregate) + { + vector runners; // thread pool handles + fRowGroupsDeliveredData.resize(fNumOfBuckets); + + uint32_t bucketsPerThread = fNumOfBuckets / fNumOfThreads; + uint32_t numThreads = ((fNumOfBuckets % fNumOfThreads) == 0 ? fNumOfThreads : fNumOfThreads + 1); + // uint32_t bucketsPerThread = 1; + // uint32_t numThreads = fNumOfBuckets; + + runners.reserve(numThreads); + + for (i = 0; i < numThreads; i++) + { + runners.push_back(jobstepThreadPool.invoke( + ThreadedSecondPhaseAggregator(this, i * bucketsPerThread, bucketsPerThread))); + } + + jobstepThreadPool.join(runners); + } + + fDoneAggregate = true; + bool done = true; + + while (nextDeliveredRowGroup()) + { + done = false; + rowCount = fRowGroupOut.getRowCount(); + + if (rowCount != 0) + { + if (fRowGroupOut.getColumnCount() != fRowGroupDelivered.getColumnCount()) + pruneAuxColumns(); + + if (dlp) + { + rgData = fRowGroupDelivered.duplicate(); + dlp->insert(rgData); + } + else + { + bs.restart(); + fRowGroupDelivered.serializeRGData(bs); + break; + } + } + + done = true; + } + + if (done) + fEndOfResult = true; + } + } + else + { + auto* agg = dynamic_cast(fAggregator.get()); + + if (!fEndOfResult) + { + if (!fDoneAggregate) + { + for (i = 0; i < fNumOfBuckets; i++) + { + if (fEndOfResult == false) + { + // do the final aggregtion and deliver the results + // at least one RowGroup for aggregate results + // for "distinct without group by" case + if (agg != nullptr) + { + auto* aggMultiDist = dynamic_cast(fAggregators[i].get()); + auto* aggDist = dynamic_cast(fAggregators[i].get()); + agg->aggregator(aggDist->aggregator()); + + if (aggMultiDist) + { + (dynamic_cast(agg)) + ->subAggregators(aggMultiDist->subAggregators()); + } + + agg->doDistinctAggregation(); + } + // for "group by without distinct" case + else + { + fAggregator->append(fAggregators[i].get()); + } + } + } + } + + fDoneAggregate = true; + } + + bool done = true; + + //@bug4459 + while (fAggregator->nextRowGroup() && !cancelled()) + { + done = false; + fAggregator->finalize(); + rowCount = fRowGroupOut.getRowCount(); + fRowsReturned += rowCount; + fRowGroupDelivered.setData(fRowGroupOut.getRGData()); + + if (rowCount != 0) + { + if (fRowGroupOut.getColumnCount() != fRowGroupDelivered.getColumnCount()) + pruneAuxColumns(); + + if (dlp) + { + rgData = fRowGroupDelivered.duplicate(); + dlp->insert(rgData); + } + else + { + bs.restart(); + fRowGroupDelivered.serializeRGData(bs); + break; + } + } + + done = true; + } + + if (done) + { + fEndOfResult = true; + } + } + } // try + catch (...) + { + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doThreadedAggregate()"); + fEndOfResult = true; + } + + if (fEndOfResult) + { StepTeleStats sts; sts.query_uuid = fQueryUuid; sts.step_uuid = fStepUuid; @@ -5809,306 +5886,80 @@ void TupleAggregateStep::doAggregate_singleThread() sts.rows = fRowsReturned; postStepSummaryTele(sts); - // Bug 3136, let mini stats to be formatted if traceOn. - fEndOfResult = true; - dlp->endOfInput(); -} - - -void TupleAggregateStep::doAggregate() -{ - // @bug4314. DO NOT access fAggregtor before the first read of input, - // because hashjoin may not have finalized fAggregator. - if (!fIsMultiThread) - return doAggregate_singleThread(); - - AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - RowGroupDL* dlp = dl->rowGroupDL(); - ByteStream bs; - doThreadedAggregate(bs, dlp); - return; -} - - -uint64_t TupleAggregateStep::doThreadedAggregate(ByteStream& bs, RowGroupDL* dlp) -{ - uint32_t i; - RGData rgData; - uint64_t rowCount = 0; - - try + if (dlp) { - if (!fDoneAggregate) - { - initializeMultiThread(); - - vector runners; // thread pool handles - runners.reserve(fNumOfThreads); // to prevent a resize during use - - // Start the aggregator threads - for (i = 0; i < fNumOfThreads; i++) - { - runners.push_back(jobstepThreadPool.invoke(ThreadedAggregator(this, i))); - } - - // Now wait for all those threads - jobstepThreadPool.join(runners); - } - - if (!cancelled()) - { - vector runners; - // use half of the threads because finalizing requires twice as - // much memory on average - uint32_t threads = std::max(1U, fNumOfThreads / 2); - runners.reserve(threads); - for (i = 0; i < threads; ++i) - { - runners.push_back(jobstepThreadPool.invoke(ThreadedAggregateFinalizer(this, i))); - } - jobstepThreadPool.join(runners); - } - - if (dynamic_cast(fAggregator.get()) && fAggregator->aggMapKeyLength() > 0) - { - // 2nd phase multi-threaded aggregate - if (!fEndOfResult) - { - if (!fDoneAggregate) - { - vector runners; // thread pool handles - fRowGroupsDeliveredData.resize(fNumOfBuckets); - - uint32_t bucketsPerThread = fNumOfBuckets / fNumOfThreads; - uint32_t numThreads = ((fNumOfBuckets % fNumOfThreads) == 0 ? - fNumOfThreads : fNumOfThreads + 1); - //uint32_t bucketsPerThread = 1; - //uint32_t numThreads = fNumOfBuckets; - - runners.reserve(numThreads); - - for (i = 0; i < numThreads; i++) - { - runners.push_back(jobstepThreadPool.invoke(ThreadedSecondPhaseAggregator(this, i * bucketsPerThread, bucketsPerThread))); - } - - jobstepThreadPool.join(runners); - } - - fDoneAggregate = true; - bool done = true; - - while (nextDeliveredRowGroup()) - { - done = false; - rowCount = fRowGroupOut.getRowCount(); - - if ( rowCount != 0 ) - { - if (fRowGroupOut.getColumnCount() != fRowGroupDelivered.getColumnCount()) - pruneAuxColumns(); - - if (dlp) - { - rgData = fRowGroupDelivered.duplicate(); - dlp->insert(rgData); - } - else - { - bs.restart(); - fRowGroupDelivered.serializeRGData(bs); - break; - } - } - - done = true; - } - - if (done) - fEndOfResult = true; - } - } - else - { - auto* agg = dynamic_cast(fAggregator.get()); - - if (!fEndOfResult) - { - if (!fDoneAggregate) - { - for (i = 0; i < fNumOfBuckets; i++) - { - if (fEndOfResult == false) - { - // do the final aggregtion and deliver the results - // at least one RowGroup for aggregate results - // for "distinct without group by" case - if (agg != nullptr) - { - auto* aggMultiDist = - dynamic_cast(fAggregators[i].get()); - auto* aggDist = - dynamic_cast(fAggregators[i].get()); - agg->aggregator(aggDist->aggregator()); - - if (aggMultiDist) - { - (dynamic_cast(agg)) - ->subAggregators(aggMultiDist->subAggregators()); - } - - agg->doDistinctAggregation(); - } - // for "group by without distinct" case - else - { - fAggregator->append(fAggregators[i].get()); - } - } - } - } - - fDoneAggregate = true; - } - - bool done = true; - - //@bug4459 - while (fAggregator->nextRowGroup() && !cancelled()) - { - done = false; - fAggregator->finalize(); - rowCount = fRowGroupOut.getRowCount(); - fRowsReturned += rowCount; - fRowGroupDelivered.setData(fRowGroupOut.getRGData()); - - if (rowCount != 0) - { - if (fRowGroupOut.getColumnCount() != fRowGroupDelivered.getColumnCount()) - pruneAuxColumns(); - - if (dlp) - { - rgData = fRowGroupDelivered.duplicate(); - dlp->insert(rgData); - } - else - { - bs.restart(); - fRowGroupDelivered.serializeRGData(bs); - break; - } - } - - done = true; - } - - if (done) - { - fEndOfResult = true; - } - } - } //try - catch (...) + dlp->endOfInput(); + } + else { - handleException(std::current_exception(), - logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, - "TupleAggregateStep::doThreadedAggregate()"); - fEndOfResult = true; + // send an empty / error band + RGData rgData(fRowGroupOut, 0); + fRowGroupOut.setData(&rgData); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setStatus(status()); + fRowGroupOut.serializeRGData(bs); + rowCount = 0; } - if (fEndOfResult) - { - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); + if (traceOn()) + printCalTrace(); + } - if (dlp) - { - dlp->endOfInput(); - } - else - { - // send an empty / error band - RGData rgData(fRowGroupOut, 0); - fRowGroupOut.setData(&rgData); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setStatus(status()); - fRowGroupOut.serializeRGData(bs); - rowCount = 0; - } - - if (traceOn()) - printCalTrace(); - } - - return rowCount; + return rowCount; } - void TupleAggregateStep::pruneAuxColumns() { - uint64_t rowCount = fRowGroupOut.getRowCount(); - Row row1, row2; - fRowGroupOut.initRow(&row1); - fRowGroupOut.getRow(0, &row1); - fRowGroupDelivered.initRow(&row2); - fRowGroupDelivered.getRow(0, &row2); + uint64_t rowCount = fRowGroupOut.getRowCount(); + Row row1, row2; + fRowGroupOut.initRow(&row1); + fRowGroupOut.getRow(0, &row1); + fRowGroupDelivered.initRow(&row2); + fRowGroupDelivered.getRow(0, &row2); - for (uint64_t i = 1; i < rowCount; i++) - { - // skip the first row - row1.nextRow(); - row2.nextRow(); + for (uint64_t i = 1; i < rowCount; i++) + { + // skip the first row + row1.nextRow(); + row2.nextRow(); - // bug4463, memmove for src, dest overlap - memmove(row2.getData(), row1.getData(), row2.getSize()); - } + // bug4463, memmove for src, dest overlap + memmove(row2.getData(), row1.getData(), row2.getSize()); + } } - void TupleAggregateStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } - void TupleAggregateStep::formatMiniStats() { - ostringstream oss; - oss << "TAS " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "TAS " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); } - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/tupleaggregatestep.h b/dbcon/joblist/tupleaggregatestep.h index 07884dcb3..95d6ee424 100644 --- a/dbcon/joblist/tupleaggregatestep.h +++ b/dbcon/joblist/tupleaggregatestep.h @@ -18,7 +18,6 @@ // $Id: tupleaggregatestep.h 9732 2013-08-02 15:56:15Z pleblanc $ - #ifndef JOBLIST_TUPLEAGGREGATESTEP_H #define JOBLIST_TUPLEAGGREGATESTEP_H @@ -28,10 +27,8 @@ #include - namespace joblist { - // forward reference struct JobInfo; @@ -40,199 +37,193 @@ struct JobInfo; */ class TupleAggregateStep : public JobStep, public TupleDeliveryStep { -public: - /** @brief TupleAggregateStep constructor - */ - TupleAggregateStep( - const rowgroup::SP_ROWAGG_UM_t&, - const rowgroup::RowGroup&, - const rowgroup::RowGroup&, - const JobInfo&); + public: + /** @brief TupleAggregateStep constructor + */ + TupleAggregateStep(const rowgroup::SP_ROWAGG_UM_t&, const rowgroup::RowGroup&, const rowgroup::RowGroup&, + const JobInfo&); - /** @brief TupleAggregateStep destructor - */ - ~TupleAggregateStep(); + /** @brief TupleAggregateStep destructor + */ + ~TupleAggregateStep(); - /** @brief virtual void Run method - */ - void run(); - void join(); + /** @brief virtual void Run method + */ + void run(); + void join(); - const std::string toString() const; + const std::string toString() const; - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); - uint32_t nextBand_singleThread(messageqcpp::ByteStream& bs); - bool setPmHJAggregation(JobStep* step); - void savePmHJData(rowgroup::SP_ROWAGG_t&, rowgroup::SP_ROWAGG_t&, rowgroup::RowGroup&); + void setOutputRowGroup(const rowgroup::RowGroup&); + const rowgroup::RowGroup& getOutputRowGroup() const; + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; + uint32_t nextBand(messageqcpp::ByteStream& bs); + uint32_t nextBand_singleThread(messageqcpp::ByteStream& bs); + bool setPmHJAggregation(JobStep* step); + void savePmHJData(rowgroup::SP_ROWAGG_t&, rowgroup::SP_ROWAGG_t&, rowgroup::RowGroup&); - bool umOnly() const + bool umOnly() const + { + return fUmOnly; + } + void umOnly(bool b) + { + fUmOnly = b; + } + + void configDeliveredRowGroup(const JobInfo&); + // void setEidMap(std::map& m) { fIndexEidMap = m; } + + static SJSTEP prepAggregate(SJSTEP&, JobInfo&); + + // for multi-thread variables + void initializeMultiThread(); + + private: + static void prep1PhaseDistinctAggregate(JobInfo&, std::vector&, + std::vector&); + static void prep1PhaseAggregate(JobInfo&, std::vector&, + std::vector&); + static void prep2PhasesAggregate(JobInfo&, std::vector&, + std::vector&); + static void prep2PhasesDistinctAggregate(JobInfo&, std::vector&, + std::vector&); + + void prepExpressionOnAggregate(rowgroup::SP_ROWAGG_UM_t&, JobInfo&); + void addConstangAggregate(std::vector&); + + void doAggregate(); + void doAggregate_singleThread(); + uint64_t doThreadedAggregate(messageqcpp::ByteStream& bs, RowGroupDL* dlp); + void aggregateRowGroups(); + void threadedAggregateRowGroups(uint32_t threadID); + void threadedAggregateFinalize(uint32_t threadID); + void doThreadedSecondPhaseAggregate(uint32_t threadID); + bool nextDeliveredRowGroup(); + void pruneAuxColumns(); + void formatMiniStats(); + void printCalTrace(); + + boost::shared_ptr fCatalog; + uint64_t fRowsReturned; + bool fDoneAggregate; + bool fEndOfResult; + + rowgroup::SP_ROWAGG_UM_t fAggregator; + rowgroup::RowGroup fRowGroupOut; + rowgroup::RowGroup fRowGroupDelivered; + rowgroup::RGData fRowGroupData; + + // for setting aggregate column eid in delivered rowgroup + // std::map fIndexEidMap; + + // data from RowGroupDL + rowgroup::RowGroup fRowGroupIn; + + // for PM HashJoin + // PM hashjoin is selected at runtime, prepare for it anyway. + rowgroup::SP_ROWAGG_UM_t fAggregatorUM; + rowgroup::SP_ROWAGG_PM_t fAggregatorPM; + rowgroup::RowGroup fRowGroupPMHJ; + + // for run thread (first added for union) + class Aggregator + { + public: + Aggregator(TupleAggregateStep* step) : fStep(step) { - return fUmOnly; } - void umOnly(bool b) + void operator()() { - fUmOnly = b; + utils::setThreadName("TASAggr"); + fStep->doAggregate(); } - void configDeliveredRowGroup(const JobInfo&); - //void setEidMap(std::map& m) { fIndexEidMap = m; } + TupleAggregateStep* fStep; + }; - static SJSTEP prepAggregate(SJSTEP&, JobInfo&); - - // for multi-thread variables - void initializeMultiThread(); - -private: - static void prep1PhaseDistinctAggregate( - JobInfo&, std::vector&, std::vector&); - static void prep1PhaseAggregate( - JobInfo&, std::vector&, std::vector&); - static void prep2PhasesAggregate( - JobInfo&, std::vector&, std::vector&); - static void prep2PhasesDistinctAggregate( - JobInfo&, std::vector&, std::vector&); - - void prepExpressionOnAggregate(rowgroup::SP_ROWAGG_UM_t&, JobInfo&); - void addConstangAggregate(std::vector&); - - void doAggregate(); - void doAggregate_singleThread(); - uint64_t doThreadedAggregate(messageqcpp::ByteStream& bs, RowGroupDL* dlp); - void aggregateRowGroups(); - void threadedAggregateRowGroups(uint32_t threadID); - void threadedAggregateFinalize(uint32_t threadID); - void doThreadedSecondPhaseAggregate(uint32_t threadID); - bool nextDeliveredRowGroup(); - void pruneAuxColumns(); - void formatMiniStats(); - void printCalTrace(); - - boost::shared_ptrfCatalog; - uint64_t fRowsReturned; - bool fDoneAggregate; - bool fEndOfResult; - - rowgroup::SP_ROWAGG_UM_t fAggregator; - rowgroup::RowGroup fRowGroupOut; - rowgroup::RowGroup fRowGroupDelivered; - rowgroup::RGData fRowGroupData; - - // for setting aggregate column eid in delivered rowgroup - //std::map fIndexEidMap; - - // data from RowGroupDL - rowgroup::RowGroup fRowGroupIn; - - // for PM HashJoin - // PM hashjoin is selected at runtime, prepare for it anyway. - rowgroup::SP_ROWAGG_UM_t fAggregatorUM; - rowgroup::SP_ROWAGG_PM_t fAggregatorPM; - rowgroup::RowGroup fRowGroupPMHJ; - - // for run thread (first added for union) - class Aggregator + class ThreadedAggregator + { + public: + ThreadedAggregator(TupleAggregateStep* step, uint32_t threadID) : fStep(step), fThreadID(threadID) { - public: - Aggregator(TupleAggregateStep* step) : fStep(step) { } - void operator()() - { - utils::setThreadName("TASAggr"); - fStep->doAggregate(); - } - - TupleAggregateStep* fStep; - }; - - class ThreadedAggregator + } + void operator()() { - public: - ThreadedAggregator(TupleAggregateStep* step, uint32_t threadID) : - fStep(step), - fThreadID(threadID) - {} - void operator()() - { - std::string t{"TASThrAggr"}; - t.append(std::to_string(fThreadID)); - utils::setThreadName(t.c_str()); - fStep->threadedAggregateRowGroups(fThreadID); - } + std::string t{"TASThrAggr"}; + t.append(std::to_string(fThreadID)); + utils::setThreadName(t.c_str()); + fStep->threadedAggregateRowGroups(fThreadID); + } - TupleAggregateStep* fStep; - uint32_t fThreadID; - }; + TupleAggregateStep* fStep; + uint32_t fThreadID; + }; - class ThreadedAggregateFinalizer + class ThreadedAggregateFinalizer + { + public: + ThreadedAggregateFinalizer(TupleAggregateStep* step, uint32_t threadID) : fStep(step), fThreadID(threadID) { - public: - ThreadedAggregateFinalizer(TupleAggregateStep* step, uint32_t threadID) : - fStep(step), - fThreadID(threadID) - {} + } - void operator()() - { - std::string t{"TASThrFin"}; - t.append(std::to_string(fThreadID)); - utils::setThreadName(t.c_str()); - fStep->threadedAggregateFinalize(fThreadID); - } - - TupleAggregateStep* fStep; - uint32_t fThreadID; - }; - - class ThreadedSecondPhaseAggregator + void operator()() { - public: - ThreadedSecondPhaseAggregator(TupleAggregateStep* step, uint32_t threadID, uint32_t bucketsPerThread) : - fStep(step), - fThreadID(threadID), - bucketCount(bucketsPerThread) - { - } - void operator()() - { - utils::setThreadName("TASThr2ndPAggr"); - for (uint32_t i = 0; i < bucketCount; i++) - fStep->doThreadedSecondPhaseAggregate(fThreadID + i); - } - TupleAggregateStep* fStep; - uint32_t fThreadID; - uint32_t bucketCount; - }; + std::string t{"TASThrFin"}; + t.append(std::to_string(fThreadID)); + utils::setThreadName(t.c_str()); + fStep->threadedAggregateFinalize(fThreadID); + } - uint64_t fRunner; // thread pool handle - bool fUmOnly; - ResourceManager* fRm; + TupleAggregateStep* fStep; + uint32_t fThreadID; + }; - // multi-threaded - uint32_t fNumOfThreads; - uint32_t fNumOfBuckets; - uint32_t fNumOfRowGroups; - uint32_t fBucketNum; + class ThreadedSecondPhaseAggregator + { + public: + ThreadedSecondPhaseAggregator(TupleAggregateStep* step, uint32_t threadID, uint32_t bucketsPerThread) + : fStep(step), fThreadID(threadID), bucketCount(bucketsPerThread) + { + } + void operator()() + { + utils::setThreadName("TASThr2ndPAggr"); + for (uint32_t i = 0; i < bucketCount; i++) + fStep->doThreadedSecondPhaseAggregate(fThreadID + i); + } + TupleAggregateStep* fStep; + uint32_t fThreadID; + uint32_t bucketCount; + }; - boost::mutex fMutex; - std::vector fAgg_mutex; - std::vector fRowGroupDatas; - std::vector fAggregators; - std::vector fRowGroupIns; - std::vector fRowGroupOuts; - std::vector > fRowGroupsDeliveredData; - bool fIsMultiThread; - int fInputIter; // iterator - boost::scoped_array fMemUsage; + uint64_t fRunner; // thread pool handle + bool fUmOnly; + ResourceManager* fRm; - boost::shared_ptr fSessionMemLimit; + // multi-threaded + uint32_t fNumOfThreads; + uint32_t fNumOfBuckets; + uint32_t fNumOfRowGroups; + uint32_t fBucketNum; + + boost::mutex fMutex; + std::vector fAgg_mutex; + std::vector fRowGroupDatas; + std::vector fAggregators; + std::vector fRowGroupIns; + std::vector fRowGroupOuts; + std::vector > fRowGroupsDeliveredData; + bool fIsMultiThread; + int fInputIter; // iterator + boost::scoped_array fMemUsage; + + boost::shared_ptr fSessionMemLimit; }; - -} // namespace +} // namespace joblist #endif // JOBLIST_TUPLEAGGREGATESTEP_H diff --git a/dbcon/joblist/tupleannexstep.cpp b/dbcon/joblist/tupleannexstep.cpp index 7b7c5b0cd..f6a7d70c2 100644 --- a/dbcon/joblist/tupleannexstep.cpp +++ b/dbcon/joblist/tupleannexstep.cpp @@ -18,7 +18,6 @@ // $Id: tupleannexstep.cpp 9661 2013-07-01 20:33:05Z pleblanc $ - //#define NDEBUG #include #include @@ -72,637 +71,626 @@ namespace { struct TAHasher { - joblist::TupleAnnexStep* ts; - utils::Hasher_r h; - TAHasher(joblist::TupleAnnexStep* t) : ts(t) { } - uint64_t operator()(const rowgroup::Row::Pointer&) const; + joblist::TupleAnnexStep* ts; + utils::Hasher_r h; + TAHasher(joblist::TupleAnnexStep* t) : ts(t) + { + } + uint64_t operator()(const rowgroup::Row::Pointer&) const; }; struct TAEq { - joblist::TupleAnnexStep* ts; - TAEq(joblist::TupleAnnexStep* t) : ts(t) { } - bool operator()(const rowgroup::Row::Pointer&, const rowgroup::Row::Pointer&) const; -}; -//TODO: Generalize these and put them back in utils/common/hasher.h -typedef tr1::unordered_set > DistinctMap_t; + joblist::TupleAnnexStep* ts; + TAEq(joblist::TupleAnnexStep* t) : ts(t) + { + } + bool operator()(const rowgroup::Row::Pointer&, const rowgroup::Row::Pointer&) const; }; +// TODO: Generalize these and put them back in utils/common/hasher.h +typedef tr1::unordered_set > + DistinctMap_t; +}; // namespace inline uint64_t TAHasher::operator()(const Row::Pointer& p) const { - Row& row = ts->row1; - row.setPointer(p); - return row.hash(); + Row& row = ts->row1; + row.setPointer(p); + return row.hash(); } inline bool TAEq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) const { - Row& r1 = ts->row1, &r2 = ts->row2; - r1.setPointer(d1); - r2.setPointer(d2); - return r1.equals(r2); + Row &r1 = ts->row1, &r2 = ts->row2; + r1.setPointer(d1); + r2.setPointer(d2); + return r1.equals(r2); } namespace joblist { - -TupleAnnexStep::TupleAnnexStep(const JobInfo& jobInfo) : - JobStep(jobInfo), - fInputDL(NULL), - fOutputDL(NULL), - fInputIterator(0), - fOutputIterator(0), - fRunner(0), - fRowsProcessed(0), - fRowsReturned(0), - fLimitStart(0), - fLimitCount(-1), - fLimitHit(false), - fEndOfResult(false), - fDistinct(false), - fParallelOp(false), - fOrderBy(NULL), - fConstant(NULL), - fFeInstance(funcexp::FuncExp::instance()), - fJobList(jobInfo.jobListPtr), - fFinishedThreads(0) +TupleAnnexStep::TupleAnnexStep(const JobInfo& jobInfo) + : JobStep(jobInfo) + , fInputDL(NULL) + , fOutputDL(NULL) + , fInputIterator(0) + , fOutputIterator(0) + , fRunner(0) + , fRowsProcessed(0) + , fRowsReturned(0) + , fLimitStart(0) + , fLimitCount(-1) + , fLimitHit(false) + , fEndOfResult(false) + , fDistinct(false) + , fParallelOp(false) + , fOrderBy(NULL) + , fConstant(NULL) + , fFeInstance(funcexp::FuncExp::instance()) + , fJobList(jobInfo.jobListPtr) + , fFinishedThreads(0) { - fExtendedInfo = "TNS: "; - fQtc.stepParms().stepType = StepTeleStats::T_TNS; + fExtendedInfo = "TNS: "; + fQtc.stepParms().stepType = StepTeleStats::T_TNS; } - TupleAnnexStep::~TupleAnnexStep() { - if(fParallelOp) + if (fParallelOp) + { + if (fOrderByList.size() > 0) { - if(fOrderByList.size() > 0) - { - for(uint64_t id = 0; id < fOrderByList.size(); id++) - { - delete fOrderByList[id]; - } + for (uint64_t id = 0; id < fOrderByList.size(); id++) + { + delete fOrderByList[id]; + } - fOrderByList.clear(); - } - - fInputIteratorsList.clear(); - fRunnersList.clear(); + fOrderByList.clear(); } - if (fOrderBy) - delete fOrderBy; + fInputIteratorsList.clear(); + fRunnersList.clear(); + } - fOrderBy = NULL; + if (fOrderBy) + delete fOrderBy; - if (fConstant) - delete fConstant; + fOrderBy = NULL; - fConstant = NULL; + if (fConstant) + delete fConstant; + + fConstant = NULL; } - void TupleAnnexStep::setOutputRowGroup(const rowgroup::RowGroup& rg) { - throw runtime_error("Disabled, use initialize() to set output RowGroup."); + throw runtime_error("Disabled, use initialize() to set output RowGroup."); } - void TupleAnnexStep::initialize(const RowGroup& rgIn, const JobInfo& jobInfo) { - // Initialize structures used by separate workers - uint64_t id = 1; - fRowGroupIn = rgIn; - fRowGroupIn.initRow(&fRowIn); - if(fParallelOp && fOrderBy) + // Initialize structures used by separate workers + uint64_t id = 1; + fRowGroupIn = rgIn; + fRowGroupIn.initRow(&fRowIn); + if (fParallelOp && fOrderBy) + { + fOrderByList.resize(fMaxThreads + 1); + for (id = 0; id <= fMaxThreads; id++) { - fOrderByList.resize(fMaxThreads+1); - for(id = 0; id <= fMaxThreads; id++) - { - // *DRRTUY use SP here? - fOrderByList[id] = new LimitedOrderBy(); - fOrderByList[id]->distinct(fDistinct); - fOrderByList[id]->initialize(rgIn, jobInfo, false, true); - } + // *DRRTUY use SP here? + fOrderByList[id] = new LimitedOrderBy(); + fOrderByList[id]->distinct(fDistinct); + fOrderByList[id]->initialize(rgIn, jobInfo, false, true); } - else + } + else + { + if (fOrderBy) { - if (fOrderBy) - { - fOrderBy->distinct(fDistinct); - fOrderBy->initialize(rgIn, jobInfo); - } + fOrderBy->distinct(fDistinct); + fOrderBy->initialize(rgIn, jobInfo); } + } - if (fConstant == NULL) - { - vector oids, oidsIn = rgIn.getOIDs(); - vector keys, keysIn = rgIn.getKeys(); - vector scale, scaleIn = rgIn.getScale(); - vector precision, precisionIn = rgIn.getPrecision(); - vector types, typesIn = rgIn.getColTypes(); - vector csNums, csNumsIn = rgIn.getCharsetNumbers(); - vector pos, posIn = rgIn.getOffsets(); - size_t n = jobInfo.nonConstDelCols.size(); + if (fConstant == NULL) + { + vector oids, oidsIn = rgIn.getOIDs(); + vector keys, keysIn = rgIn.getKeys(); + vector scale, scaleIn = rgIn.getScale(); + vector precision, precisionIn = rgIn.getPrecision(); + vector types, typesIn = rgIn.getColTypes(); + vector csNums, csNumsIn = rgIn.getCharsetNumbers(); + vector pos, posIn = rgIn.getOffsets(); + size_t n = jobInfo.nonConstDelCols.size(); - // Add all columns into output RG as keys. Can we put only keys? - oids.insert(oids.end(), oidsIn.begin(), oidsIn.begin() + n); - keys.insert(keys.end(), keysIn.begin(), keysIn.begin() + n); - scale.insert(scale.end(), scaleIn.begin(), scaleIn.begin() + n); - precision.insert(precision.end(), precisionIn.begin(), precisionIn.begin() + n); - types.insert(types.end(), typesIn.begin(), typesIn.begin() + n); - csNums.insert(csNums.end(), csNumsIn.begin(), csNumsIn.begin() + n); - pos.insert(pos.end(), posIn.begin(), posIn.begin() + n + 1); + // Add all columns into output RG as keys. Can we put only keys? + oids.insert(oids.end(), oidsIn.begin(), oidsIn.begin() + n); + keys.insert(keys.end(), keysIn.begin(), keysIn.begin() + n); + scale.insert(scale.end(), scaleIn.begin(), scaleIn.begin() + n); + precision.insert(precision.end(), precisionIn.begin(), precisionIn.begin() + n); + types.insert(types.end(), typesIn.begin(), typesIn.begin() + n); + csNums.insert(csNums.end(), csNumsIn.begin(), csNumsIn.begin() + n); + pos.insert(pos.end(), posIn.begin(), posIn.begin() + n + 1); - fRowGroupOut = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - } - else - { - fConstant->initialize(jobInfo, &rgIn); - fRowGroupOut = fConstant->getOutputRowGroup(); - } + fRowGroupOut = + RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + } + else + { + fConstant->initialize(jobInfo, &rgIn); + fRowGroupOut = fConstant->getOutputRowGroup(); + } - fRowGroupOut.initRow(&fRowOut); - fRowGroupDeliver = fRowGroupOut; + fRowGroupOut.initRow(&fRowOut); + fRowGroupDeliver = fRowGroupOut; } - void TupleAnnexStep::run() { - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for annex step."); + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for annex step."); + fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + + if (fInputDL == NULL) + throw logic_error("Input is not a RowGroup data list."); + + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for annex step."); + + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); + + if (fDelivery == true) + { + fOutputIterator = fOutputDL->getIterator(); + } + + if (fParallelOp) + { + // Indexing begins with 1 + fRunnersList.resize(fMaxThreads); + fInputIteratorsList.resize(fMaxThreads + 1); + + // Activate stats collecting before CS spawns threads. + if (traceOn()) + dlTimes.setFirstReadTime(); + + // *DRRTUY Make this block conditional + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + for (uint32_t id = 1; id <= fMaxThreads; id++) + { + fInputIteratorsList[id] = fInputDL->getIterator(); + fRunnersList[id - 1] = jobstepThreadPool.invoke(Runner(this, id)); + } + } + else + { fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); if (fInputDL == NULL) - throw logic_error("Input is not a RowGroup data list."); + throw logic_error("Input is not a RowGroup data list."); - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for annex step."); - - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); - - if (fDelivery == true) - { - fOutputIterator = fOutputDL->getIterator(); - } - - if(fParallelOp) - { - // Indexing begins with 1 - fRunnersList.resize(fMaxThreads); - fInputIteratorsList.resize(fMaxThreads+1); - - // Activate stats collecting before CS spawns threads. - if (traceOn()) dlTimes.setFirstReadTime(); - - // *DRRTUY Make this block conditional - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - - for(uint32_t id = 1; id <= fMaxThreads; id++) - { - fInputIteratorsList[id] = fInputDL->getIterator(); - fRunnersList[id-1] = jobstepThreadPool.invoke(Runner(this, id)); - } - } - else - { - fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - - if (fInputDL == NULL) - throw logic_error("Input is not a RowGroup data list."); - - fInputIterator = fInputDL->getIterator(); - fRunner = jobstepThreadPool.invoke(Runner(this)); - } - + fInputIterator = fInputDL->getIterator(); + fRunner = jobstepThreadPool.invoke(Runner(this)); + } } void TupleAnnexStep::join() { - if(fParallelOp) + if (fParallelOp) + { + jobstepThreadPool.join(fRunnersList); + } + else + { + if (fRunner) { - jobstepThreadPool.join(fRunnersList); + jobstepThreadPool.join(fRunner); } - else - { - if (fRunner) - { - jobstepThreadPool.join(fRunner); - } - } - + } } uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs) { - RGData rgDataOut; - bool more = false; - uint32_t rowCount = 0; + RGData rgDataOut; + bool more = false; + uint32_t rowCount = 0; - try + try + { + bs.restart(); + + more = fOutputDL->next(fOutputIterator, &rgDataOut); + + if (more && !cancelled()) { - bs.restart(); - + fRowGroupDeliver.setData(&rgDataOut); + fRowGroupDeliver.serializeRGData(bs); + rowCount = fRowGroupDeliver.getRowCount(); + } + else + { + while (more) more = fOutputDL->next(fOutputIterator, &rgDataOut); - if (more && !cancelled()) - { - fRowGroupDeliver.setData(&rgDataOut); - fRowGroupDeliver.serializeRGData(bs); - rowCount = fRowGroupDeliver.getRowCount(); - } - else - { - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - - fEndOfResult = true; - } + fEndOfResult = true; } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_IN_DELIVERY, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::nextBand()"); - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_DELIVERY, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::nextBand()"); + while (more) + more = fOutputDL->next(fOutputIterator, &rgDataOut); - fEndOfResult = true; - } + fEndOfResult = true; + } - if (fEndOfResult) - { - // send an empty / error band - rgDataOut.reinit(fRowGroupDeliver, 0); - fRowGroupDeliver.setData(&rgDataOut); - fRowGroupDeliver.resetRowGroup(0); - fRowGroupDeliver.setStatus(status()); - fRowGroupDeliver.serializeRGData(bs); - } + if (fEndOfResult) + { + // send an empty / error band + rgDataOut.reinit(fRowGroupDeliver, 0); + fRowGroupDeliver.setData(&rgDataOut); + fRowGroupDeliver.resetRowGroup(0); + fRowGroupDeliver.setStatus(status()); + fRowGroupDeliver.serializeRGData(bs); + } - return rowCount; + return rowCount; } - void TupleAnnexStep::execute() { - if (fOrderBy) - executeWithOrderBy(); - else if (fDistinct) - executeNoOrderByWithDistinct(); - else - executeNoOrderBy(); + if (fOrderBy) + executeWithOrderBy(); + else if (fDistinct) + executeNoOrderByWithDistinct(); + else + executeNoOrderBy(); - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); - if (traceOn()) - { - if (dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); + if (traceOn()) + { + if (dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } } void TupleAnnexStep::execute(uint32_t id) { - if(fOrderByList[id]) - executeParallelOrderBy(id); - + if (fOrderByList[id]) + executeParallelOrderBy(id); } void TupleAnnexStep::executeNoOrderBy() { - utils::setThreadName("TASwoOrd"); - RGData rgDataIn; - RGData rgDataOut; - bool more = false; + utils::setThreadName("TASwoOrd"); + RGData rgDataIn; + RGData rgDataOut; + bool more = false; - try + try + { + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (traceOn()) + dlTimes.setFirstReadTime(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + while (more && !cancelled() && !fLimitHit) { - more = fInputDL->next(fInputIterator, &rgDataIn); + fRowGroupIn.setData(&rgDataIn); + fRowGroupIn.getRow(0, &fRowIn); + // Get a new output rowgroup for each input rowgroup to preserve the rids + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); + fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot()); + fRowGroupOut.getRow(0, &fRowOut); - if (traceOn()) dlTimes.setFirstReadTime(); - - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - - while (more && !cancelled() && !fLimitHit) + for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i) + { + // skip first limit-start rows + if (fRowsProcessed++ < fLimitStart) { - fRowGroupIn.setData(&rgDataIn); - fRowGroupIn.getRow(0, &fRowIn); - // Get a new output rowgroup for each input rowgroup to preserve the rids - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot()); - fRowGroupOut.getRow(0, &fRowOut); - - for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i) - { - // skip first limit-start rows - if (fRowsProcessed++ < fLimitStart) - { - fRowIn.nextRow(); - continue; - } - - if (UNLIKELY(fRowsReturned >= fLimitCount)) - { - fLimitHit = true; - fJobList->abortOnLimit((JobStep*) this); - continue; - } - - if (fConstant) - fConstant->fillInConstants(fRowIn, fRowOut); - else - copyRow(fRowIn, &fRowOut); - - fRowGroupOut.incRowCount(); - - if (++fRowsReturned < fLimitCount) - { - fRowOut.nextRow(); - fRowIn.nextRow(); - } - } - - if (fRowGroupOut.getRowCount() > 0) - { - fOutputDL->insert(rgDataOut); - } - - more = fInputDL->next(fInputIterator, &rgDataIn); + fRowIn.nextRow(); + continue; } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_IN_PROCESS, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::executeNoOrderBy()"); - } - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); + if (UNLIKELY(fRowsReturned >= fLimitCount)) + { + fLimitHit = true; + fJobList->abortOnLimit((JobStep*)this); + continue; + } - // Bug 3136, let mini stats to be formatted if traceOn. - fOutputDL->endOfInput(); + if (fConstant) + fConstant->fillInConstants(fRowIn, fRowOut); + else + copyRow(fRowIn, &fRowOut); + + fRowGroupOut.incRowCount(); + + if (++fRowsReturned < fLimitCount) + { + fRowOut.nextRow(); + fRowIn.nextRow(); + } + } + + if (fRowGroupOut.getRowCount() > 0) + { + fOutputDL->insert(rgDataOut); + } + + more = fInputDL->next(fInputIterator, &rgDataIn); + } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_PROCESS, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::executeNoOrderBy()"); + } + + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); + + // Bug 3136, let mini stats to be formatted if traceOn. + fOutputDL->endOfInput(); } - void TupleAnnexStep::executeNoOrderByWithDistinct() { - utils::setThreadName("TASwoOrdDist"); - scoped_ptr distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this))); - vector dataVec; - vector dataVecSkip; - RGData rgDataIn; - RGData rgDataOut; - RGData rgDataSkip; - RowGroup rowGroupSkip; - Row rowSkip; - bool more = false; + utils::setThreadName("TASwoOrdDist"); + scoped_ptr distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this))); + vector dataVec; + vector dataVecSkip; + RGData rgDataIn; + RGData rgDataOut; + RGData rgDataSkip; + RowGroup rowGroupSkip; + Row rowSkip; + bool more = false; - rgDataOut.reinit(fRowGroupOut); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.getRow(0, &fRowOut); + rgDataOut.reinit(fRowGroupOut); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.getRow(0, &fRowOut); - fRowGroupOut.initRow(&row1); - fRowGroupOut.initRow(&row2); + fRowGroupOut.initRow(&row1); + fRowGroupOut.initRow(&row2); - rowGroupSkip = fRowGroupOut; - rgDataSkip.reinit(rowGroupSkip); - rowGroupSkip.setData(&rgDataSkip); - rowGroupSkip.resetRowGroup(0); - rowGroupSkip.initRow(&rowSkip); - rowGroupSkip.getRow(0, &rowSkip); + rowGroupSkip = fRowGroupOut; + rgDataSkip.reinit(rowGroupSkip); + rowGroupSkip.setData(&rgDataSkip); + rowGroupSkip.resetRowGroup(0); + rowGroupSkip.initRow(&rowSkip); + rowGroupSkip.getRow(0, &rowSkip); - try + try + { + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (traceOn()) + dlTimes.setFirstReadTime(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + while (more && !cancelled() && !fLimitHit) { - more = fInputDL->next(fInputIterator, &rgDataIn); + fRowGroupIn.setData(&rgDataIn); + fRowGroupIn.getRow(0, &fRowIn); - if (traceOn()) dlTimes.setFirstReadTime(); + for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i) + { + pair inserted; + Row* rowPtr; - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); + if (distinctMap->size() < fLimitStart) + rowPtr = &rowSkip; + else + rowPtr = &fRowOut; - while (more && !cancelled() && !fLimitHit) + if (fConstant) + fConstant->fillInConstants(fRowIn, *rowPtr); + else + copyRow(fRowIn, rowPtr); + + fRowIn.nextRow(); + inserted = distinctMap->insert(rowPtr->getPointer()); + ++fRowsProcessed; + + if (inserted.second) { - fRowGroupIn.setData(&rgDataIn); - fRowGroupIn.getRow(0, &fRowIn); + if (UNLIKELY(fRowsReturned >= fLimitCount)) + { + fLimitHit = true; + fJobList->abortOnLimit((JobStep*)this); + break; + } - for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled() && !fLimitHit; ++i) + // skip first limit-start rows + if (distinctMap->size() <= fLimitStart) + { + rowGroupSkip.incRowCount(); + rowSkip.nextRow(); + if (UNLIKELY(rowGroupSkip.getRowCount() >= rowgroup::rgCommonSize)) { - pair inserted; - Row* rowPtr; - - if (distinctMap->size() < fLimitStart) - rowPtr = &rowSkip; - else - rowPtr = &fRowOut; - - if (fConstant) - fConstant->fillInConstants(fRowIn, *rowPtr); - else - copyRow(fRowIn, rowPtr); - - fRowIn.nextRow(); - inserted = distinctMap->insert(rowPtr->getPointer()); - ++fRowsProcessed; - - if (inserted.second) - { - if (UNLIKELY(fRowsReturned >= fLimitCount)) - { - fLimitHit = true; - fJobList->abortOnLimit((JobStep*) this); - break; - } - - // skip first limit-start rows - if (distinctMap->size() <= fLimitStart) - { - rowGroupSkip.incRowCount(); - rowSkip.nextRow(); - if (UNLIKELY(rowGroupSkip.getRowCount() >= rowgroup::rgCommonSize)) - { - // allocate new RGData for skipped rows below the fLimitStart - // offset (do not take it into account in RM assuming there - // are few skipped rows - dataVecSkip.push_back(rgDataSkip); - rgDataSkip.reinit(rowGroupSkip); - rowGroupSkip.setData(&rgDataSkip); - rowGroupSkip.resetRowGroup(0); - rowGroupSkip.getRow(0, &rowSkip); - } - continue; - } - - ++fRowsReturned; - - fRowGroupOut.incRowCount(); - fRowOut.nextRow(); - - if (UNLIKELY(fRowGroupOut.getRowCount() >= rowgroup::rgCommonSize)) - { - dataVec.push_back(rgDataOut); - rgDataOut.reinit(fRowGroupOut); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.getRow(0, &fRowOut); - } - } + // allocate new RGData for skipped rows below the fLimitStart + // offset (do not take it into account in RM assuming there + // are few skipped rows + dataVecSkip.push_back(rgDataSkip); + rgDataSkip.reinit(rowGroupSkip); + rowGroupSkip.setData(&rgDataSkip); + rowGroupSkip.resetRowGroup(0); + rowGroupSkip.getRow(0, &rowSkip); } + continue; + } - more = fInputDL->next(fInputIterator, &rgDataIn); - } + ++fRowsReturned; - if (fRowGroupOut.getRowCount() > 0) + fRowGroupOut.incRowCount(); + fRowOut.nextRow(); + + if (UNLIKELY(fRowGroupOut.getRowCount() >= rowgroup::rgCommonSize)) + { dataVec.push_back(rgDataOut); - - for (vector::iterator i = dataVec.begin(); i != dataVec.end(); i++) - { - rgDataOut = *i; + rgDataOut.reinit(fRowGroupOut); fRowGroupOut.setData(&rgDataOut); - fOutputDL->insert(rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.getRow(0, &fRowOut); + } } + } + + more = fInputDL->next(fInputIterator, &rgDataIn); } - catch (...) + + if (fRowGroupOut.getRowCount() > 0) + dataVec.push_back(rgDataOut); + + for (vector::iterator i = dataVec.begin(); i != dataVec.end(); i++) { - handleException(std::current_exception(), - logging::ERR_IN_PROCESS, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::executeNoOrderByWithDistinct()"); + rgDataOut = *i; + fRowGroupOut.setData(&rgDataOut); + fOutputDL->insert(rgDataOut); } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_PROCESS, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::executeNoOrderByWithDistinct()"); + } - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); - // Bug 3136, let mini stats to be formatted if traceOn. - fOutputDL->endOfInput(); + // Bug 3136, let mini stats to be formatted if traceOn. + fOutputDL->endOfInput(); } - void TupleAnnexStep::executeWithOrderBy() { - utils::setThreadName("TASwOrd"); - RGData rgDataIn; - RGData rgDataOut; - bool more = false; + utils::setThreadName("TASwOrd"); + RGData rgDataIn; + RGData rgDataOut; + bool more = false; - try + try + { + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (traceOn()) + dlTimes.setFirstReadTime(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + while (more && !cancelled()) { - more = fInputDL->next(fInputIterator, &rgDataIn); + fRowGroupIn.setData(&rgDataIn); + fRowGroupIn.getRow(0, &fRowIn); - if (traceOn()) dlTimes.setFirstReadTime(); + for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled(); ++i) + { + fOrderBy->processRow(fRowIn); + fRowIn.nextRow(); + } - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - - while (more && !cancelled()) - { - fRowGroupIn.setData(&rgDataIn); - fRowGroupIn.getRow(0, &fRowIn); - - for (uint64_t i = 0; i < fRowGroupIn.getRowCount() && !cancelled(); ++i) - { - fOrderBy->processRow(fRowIn); - fRowIn.nextRow(); - } - - more = fInputDL->next(fInputIterator, &rgDataIn); - } - - fOrderBy->finalize(); - - if (!cancelled()) - { - while (fOrderBy->getData(rgDataIn)) - { - if (fConstant == NULL && - fRowGroupOut.getColumnCount() == fRowGroupIn.getColumnCount()) - { - rgDataOut = rgDataIn; - fRowGroupOut.setData(&rgDataOut); - } - else - { - fRowGroupIn.setData(&rgDataIn); - fRowGroupIn.getRow(0, &fRowIn); - - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot()); - fRowGroupOut.getRow(0, &fRowOut); - - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) - { - if (fConstant) - fConstant->fillInConstants(fRowIn, fRowOut); - else - copyRow(fRowIn, &fRowOut); - - fRowGroupOut.incRowCount(); - fRowOut.nextRow(); - fRowIn.nextRow(); - } - } - - if (fRowGroupOut.getRowCount() > 0) - { - fRowsReturned += fRowGroupOut.getRowCount(); - fOutputDL->insert(rgDataOut); - } - } - } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_IN_PROCESS, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::executeWithOrderBy()"); + more = fInputDL->next(fInputIterator, &rgDataIn); } - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); + fOrderBy->finalize(); - // Bug 3136, let mini stats to be formatted if traceOn. - fOutputDL->endOfInput(); + if (!cancelled()) + { + while (fOrderBy->getData(rgDataIn)) + { + if (fConstant == NULL && fRowGroupOut.getColumnCount() == fRowGroupIn.getColumnCount()) + { + rgDataOut = rgDataIn; + fRowGroupOut.setData(&rgDataOut); + } + else + { + fRowGroupIn.setData(&rgDataIn); + fRowGroupIn.getRow(0, &fRowIn); + + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); + fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot()); + fRowGroupOut.getRow(0, &fRowOut); + + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) + { + if (fConstant) + fConstant->fillInConstants(fRowIn, fRowOut); + else + copyRow(fRowIn, &fRowOut); + + fRowGroupOut.incRowCount(); + fRowOut.nextRow(); + fRowIn.nextRow(); + } + } + + if (fRowGroupOut.getRowCount() > 0) + { + fRowsReturned += fRowGroupOut.getRowCount(); + fOutputDL->insert(rgDataOut); + } + } + } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_PROCESS, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::executeWithOrderBy()"); + } + + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); + + // Bug 3136, let mini stats to be formatted if traceOn. + fOutputDL->endOfInput(); } /* @@ -719,197 +707,189 @@ void TupleAnnexStep::executeWithOrderBy() */ void TupleAnnexStep::finalizeParallelOrderByDistinct() { - utils::setThreadName("TASwParOrdDistM"); - uint64_t count = 0; - uint64_t offset = 0; - uint32_t rowSize = 0; + utils::setThreadName("TASwParOrdDistM"); + uint64_t count = 0; + uint64_t offset = 0; + uint32_t rowSize = 0; - rowgroup::RGData rgDataOut; - rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - // Calculate offset here - fRowGroupOut.getRow(0, &fRowOut); - ordering::SortingPQ finalPQ; - scoped_ptr distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this))); - fRowGroupIn.initRow(&row1); - fRowGroupIn.initRow(&row2); + rowgroup::RGData rgDataOut; + rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + // Calculate offset here + fRowGroupOut.getRow(0, &fRowOut); + ordering::SortingPQ finalPQ; + scoped_ptr distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this))); + fRowGroupIn.initRow(&row1); + fRowGroupIn.initRow(&row2); - try + try + { + for (uint64_t id = 1; id <= fMaxThreads; id++) { - for(uint64_t id = 1; id <= fMaxThreads; id++) + if (cancelled()) + { + break; + } + // Revert the ordering rules before we + // add rows into the final PQ. + fOrderByList[id]->getRule().revertRules(); + ordering::SortingPQ& currentPQ = fOrderByList[id]->getQueue(); + finalPQ.reserve(finalPQ.size() + currentPQ.size()); + pair inserted; + while (currentPQ.size()) + { + ordering::OrderByRow& topOBRow = const_cast(currentPQ.top()); + inserted = distinctMap->insert(topOBRow.fData); + if (inserted.second) { - if (cancelled()) - { - break; - } - // Revert the ordering rules before we - // add rows into the final PQ. - fOrderByList[id]->getRule().revertRules(); - ordering::SortingPQ ¤tPQ = fOrderByList[id]->getQueue(); - finalPQ.reserve(finalPQ.size()+currentPQ.size()); - pair inserted; - while (currentPQ.size()) - { - ordering::OrderByRow &topOBRow = - const_cast(currentPQ.top()); - inserted = distinctMap->insert(topOBRow.fData); - if (inserted.second) - { - finalPQ.push(topOBRow); - } - currentPQ.pop(); - } + finalPQ.push(topOBRow); } + currentPQ.pop(); + } } - catch (...) + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_PROCESS, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::finalizeParallelOrderByDistinct()"); + } + + // OFFSET processing + while (finalPQ.size() && offset < fLimitStart) + { + offset++; + finalPQ.pop(); + } + + // Calculate rowSize only once + if (finalPQ.size()) + { + ordering::OrderByRow& topOBRow = const_cast(finalPQ.top()); + fRowIn.setData(topOBRow.fData); + if (!fConstant) { - handleException(std::current_exception(), - logging::ERR_IN_PROCESS, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::finalizeParallelOrderByDistinct()"); + copyRow(fRowIn, &fRowOut); } - - // OFFSET processing - while (finalPQ.size() && offset < fLimitStart) + else { - offset++; - finalPQ.pop(); - } + fConstant->fillInConstants(fRowIn, fRowOut); + } + rowSize = fRowOut.getSize(); + fRowGroupOut.incRowCount(); + fRowOut.nextRow(rowSize); + finalPQ.pop(); + count++; + } - // Calculate rowSize only once - if (finalPQ.size()) + if (!fConstant) + { + while (finalPQ.size()) { - ordering::OrderByRow& topOBRow = - const_cast(finalPQ.top()); + if (cancelled()) + { + break; + } + + while (count < fLimitCount && finalPQ.size() && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) + { + ordering::OrderByRow& topOBRow = const_cast(finalPQ.top()); + fRowIn.setData(topOBRow.fData); - if (!fConstant) - { - copyRow(fRowIn, &fRowOut); - } - else - { - fConstant->fillInConstants(fRowIn, fRowOut); - } - rowSize = fRowOut.getSize(); + copyRow(fRowIn, &fRowOut); fRowGroupOut.incRowCount(); fRowOut.nextRow(rowSize); + finalPQ.pop(); count++; - } - - if (!fConstant) - { - while(finalPQ.size()) + if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize) { - if (cancelled()) - { - break; - } + break; + } + } - while (count < fLimitCount && finalPQ.size() - && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) - { - ordering::OrderByRow &topOBRow = - const_cast(finalPQ.top()); - - fRowIn.setData(topOBRow.fData); - copyRow(fRowIn, &fRowOut); - fRowGroupOut.incRowCount(); - fRowOut.nextRow(rowSize); - - finalPQ.pop(); - count++; - if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize) - { - break; - } - } - - if (fRowGroupOut.getRowCount() > 0) - { - fRowsReturned += fRowGroupOut.getRowCount(); - fOutputDL->insert(rgDataOut); - rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.getRow(0, &fRowOut); - } - else - { - break; - } - } // end of limit bound while loop - } - else // Add ConstantColumns striped earlier - { - while(finalPQ.size()) - { - if (cancelled()) - { - break; - } - - while (count < fLimitCount && finalPQ.size() - && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) - { - ordering::OrderByRow &topOBRow = - const_cast(finalPQ.top()); - - fRowIn.setData(topOBRow.fData); - fConstant->fillInConstants(fRowIn, fRowOut); - fRowGroupOut.incRowCount(); - fRowOut.nextRow(rowSize); - - finalPQ.pop(); - count++; - if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize) - { - break; - } - } - - if (fRowGroupOut.getRowCount() > 0) - { - fRowsReturned += fRowGroupOut.getRowCount(); - fOutputDL->insert(rgDataOut); - rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.getRow(0, &fRowOut); - } - else - { - break; - } - } // end of limit bound while loop - } // end of if-else - - if (fRowGroupOut.getRowCount() > 0) - { + if (fRowGroupOut.getRowCount() > 0) + { fRowsReturned += fRowGroupOut.getRowCount(); fOutputDL->insert(rgDataOut); - } - - fOutputDL->endOfInput(); - - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - if (traceOn()) + rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.getRow(0, &fRowOut); + } + else + { + break; + } + } // end of limit bound while loop + } + else // Add ConstantColumns striped earlier + { + while (finalPQ.size()) { - if (dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); + if (cancelled()) + { + break; + } - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } + while (count < fLimitCount && finalPQ.size() && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) + { + ordering::OrderByRow& topOBRow = const_cast(finalPQ.top()); + + fRowIn.setData(topOBRow.fData); + fConstant->fillInConstants(fRowIn, fRowOut); + fRowGroupOut.incRowCount(); + fRowOut.nextRow(rowSize); + + finalPQ.pop(); + count++; + if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize) + { + break; + } + } + + if (fRowGroupOut.getRowCount() > 0) + { + fRowsReturned += fRowGroupOut.getRowCount(); + fOutputDL->insert(rgDataOut); + rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.getRow(0, &fRowOut); + } + else + { + break; + } + } // end of limit bound while loop + } // end of if-else + + if (fRowGroupOut.getRowCount() > 0) + { + fRowsReturned += fRowGroupOut.getRowCount(); + fOutputDL->insert(rgDataOut); + } + + fOutputDL->endOfInput(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + if (traceOn()) + { + if (dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } } /* @@ -926,374 +906,356 @@ void TupleAnnexStep::finalizeParallelOrderByDistinct() */ void TupleAnnexStep::finalizeParallelOrderBy() { - utils::setThreadName("TASwParOrdMerge"); - uint64_t count = 0; - uint64_t offset = 0; - uint32_t rowSize = 0; + utils::setThreadName("TASwParOrdMerge"); + uint64_t count = 0; + uint64_t offset = 0; + uint32_t rowSize = 0; - rowgroup::RGData rgDataOut; - ordering::SortingPQ finalPQ; - rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - // Calculate offset here - fRowGroupOut.getRow(0, &fRowOut); + rowgroup::RGData rgDataOut; + ordering::SortingPQ finalPQ; + rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + // Calculate offset here + fRowGroupOut.getRow(0, &fRowOut); - try + try + { + for (uint64_t id = 1; id <= fMaxThreads; id++) { - for(uint64_t id = 1; id <= fMaxThreads; id++) - { - if (cancelled()) - { - break; - } - // Revert the ordering rules before we - // add rows into the final PQ. - fOrderByList[id]->getRule().revertRules(); - ordering::SortingPQ ¤tPQ = fOrderByList[id]->getQueue(); - finalPQ.reserve(currentPQ.size()); - while (currentPQ.size()) - { - ordering::OrderByRow &topOBRow = - const_cast(currentPQ.top()); - finalPQ.push(topOBRow); - currentPQ.pop(); - } - } + if (cancelled()) + { + break; + } + // Revert the ordering rules before we + // add rows into the final PQ. + fOrderByList[id]->getRule().revertRules(); + ordering::SortingPQ& currentPQ = fOrderByList[id]->getQueue(); + finalPQ.reserve(currentPQ.size()); + while (currentPQ.size()) + { + ordering::OrderByRow& topOBRow = const_cast(currentPQ.top()); + finalPQ.push(topOBRow); + currentPQ.pop(); + } } - catch (...) + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_PROCESS, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::finalizeParallelOrderBy()"); + } + + // OFFSET processing + while (finalPQ.size() && offset < fLimitStart) + { + offset++; + finalPQ.pop(); + } + + // Calculate rowSize only once + if (finalPQ.size()) + { + ordering::OrderByRow& topOBRow = const_cast(finalPQ.top()); + fRowIn.setData(topOBRow.fData); + if (!fConstant) { - handleException(std::current_exception(), - logging::ERR_IN_PROCESS, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::finalizeParallelOrderBy()"); + copyRow(fRowIn, &fRowOut); } - - // OFFSET processing - while (finalPQ.size() && offset < fLimitStart) + else { - offset++; - finalPQ.pop(); - } + fConstant->fillInConstants(fRowIn, fRowOut); + } + rowSize = fRowOut.getSize(); + fRowGroupOut.incRowCount(); + fRowOut.nextRow(rowSize); + finalPQ.pop(); + count++; + } - // Calculate rowSize only once - if (finalPQ.size()) + if (!fConstant) + { + while (finalPQ.size()) { - ordering::OrderByRow& topOBRow = - const_cast(finalPQ.top()); + if (cancelled()) + { + break; + } + + while (count < fLimitCount && finalPQ.size() && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) + { + ordering::OrderByRow& topOBRow = const_cast(finalPQ.top()); + fRowIn.setData(topOBRow.fData); - if (!fConstant) - { - copyRow(fRowIn, &fRowOut); - } - else - { - fConstant->fillInConstants(fRowIn, fRowOut); - } - rowSize = fRowOut.getSize(); + copyRow(fRowIn, &fRowOut); fRowGroupOut.incRowCount(); fRowOut.nextRow(rowSize); + finalPQ.pop(); count++; - } + } - if (!fConstant) - { - while(finalPQ.size()) - { - if (cancelled()) - { - break; - } - - while (count < fLimitCount && finalPQ.size() - && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) - { - ordering::OrderByRow &topOBRow = - const_cast(finalPQ.top()); - - fRowIn.setData(topOBRow.fData); - copyRow(fRowIn, &fRowOut); - fRowGroupOut.incRowCount(); - fRowOut.nextRow(rowSize); - - finalPQ.pop(); - count++; - } - - if (fRowGroupOut.getRowCount() > 0) - { - fRowsReturned += fRowGroupOut.getRowCount(); - fOutputDL->insert(rgDataOut); - rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.getRow(0, &fRowOut); - } - else - { - break; - } - } // end of limit bound while loop - } - else // Add ConstantColumns striped earlier - { - while(finalPQ.size()) - { - if (cancelled()) - { - break; - } - - while (count < fLimitCount && finalPQ.size() - && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) - { - ordering::OrderByRow &topOBRow = - const_cast(finalPQ.top()); - - fRowIn.setData(topOBRow.fData); - fConstant->fillInConstants(fRowIn, fRowOut); - fRowGroupOut.incRowCount(); - fRowOut.nextRow(rowSize); - - finalPQ.pop(); - count++; - if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize) - { - break; - } - } - - if (fRowGroupOut.getRowCount() > 0) - { - fRowsReturned += fRowGroupOut.getRowCount(); - fOutputDL->insert(rgDataOut); - rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.getRow(0, &fRowOut); - } - else - { - break; - } - } // end of limit bound while loop - } // end of if-else - - if (fRowGroupOut.getRowCount() > 0) - { + if (fRowGroupOut.getRowCount() > 0) + { fRowsReturned += fRowGroupOut.getRowCount(); fOutputDL->insert(rgDataOut); - } - - fOutputDL->endOfInput(); - - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - if (traceOn()) + rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.getRow(0, &fRowOut); + } + else + { + break; + } + } // end of limit bound while loop + } + else // Add ConstantColumns striped earlier + { + while (finalPQ.size()) { - if (dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); + if (cancelled()) + { + break; + } - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } + while (count < fLimitCount && finalPQ.size() && fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) + { + ordering::OrderByRow& topOBRow = const_cast(finalPQ.top()); + + fRowIn.setData(topOBRow.fData); + fConstant->fillInConstants(fRowIn, fRowOut); + fRowGroupOut.incRowCount(); + fRowOut.nextRow(rowSize); + + finalPQ.pop(); + count++; + if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize) + { + break; + } + } + + if (fRowGroupOut.getRowCount() > 0) + { + fRowsReturned += fRowGroupOut.getRowCount(); + fOutputDL->insert(rgDataOut); + rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.getRow(0, &fRowOut); + } + else + { + break; + } + } // end of limit bound while loop + } // end of if-else + + if (fRowGroupOut.getRowCount() > 0) + { + fRowsReturned += fRowGroupOut.getRowCount(); + fOutputDL->insert(rgDataOut); + } + + fOutputDL->endOfInput(); + + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + if (traceOn()) + { + if (dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } } void TupleAnnexStep::executeParallelOrderBy(uint64_t id) { - utils::setThreadName("TASwParOrd"); - RGData rgDataIn; - RGData rgDataOut; - bool more = false; - uint64_t dlOffset = 0; - uint32_t rowSize = 0; + utils::setThreadName("TASwParOrd"); + RGData rgDataIn; + RGData rgDataOut; + bool more = false; + uint64_t dlOffset = 0; + uint32_t rowSize = 0; - uint64_t rowCount = 0; - uint64_t doubleRGSize = 2*rowgroup::rgCommonSize; - rowgroup::Row r = fRowIn; - rowgroup::RowGroup rg = fRowGroupIn; - rg.initRow(&r); - LimitedOrderBy *limOrderBy = fOrderByList[id]; - ordering::SortingPQ ¤tPQ = limOrderBy->getQueue(); - if (limOrderBy->getLimitCount() < QUEUE_RESERVE_SIZE) + uint64_t rowCount = 0; + uint64_t doubleRGSize = 2 * rowgroup::rgCommonSize; + rowgroup::Row r = fRowIn; + rowgroup::RowGroup rg = fRowGroupIn; + rg.initRow(&r); + LimitedOrderBy* limOrderBy = fOrderByList[id]; + ordering::SortingPQ& currentPQ = limOrderBy->getQueue(); + if (limOrderBy->getLimitCount() < QUEUE_RESERVE_SIZE) + { + currentPQ.reserve(limOrderBy->getLimitCount()); + } + else + { + currentPQ.reserve(QUEUE_RESERVE_SIZE); + } + + try + { + more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); + if (more) + dlOffset++; + + while (more && !cancelled()) { - currentPQ.reserve(limOrderBy->getLimitCount()); + if (dlOffset % fMaxThreads == id - 1) + { + if (cancelled()) + break; + + if (currentPQ.capacity() - currentPQ.size() < doubleRGSize) + { + currentPQ.reserve(QUEUE_RESERVE_SIZE); + } + + rg.setData(&rgDataIn); + rg.getRow(0, &r); + if (!rowSize) + { + rowSize = r.getSize(); + } + rowCount = rg.getRowCount(); + + for (uint64_t i = 0; i < rowCount; ++i) + { + limOrderBy->processRow(r); + r.nextRow(rowSize); + } + } + + // *DRRTUY Implement a method to skip elements in FIFO + more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); + if (more) + dlOffset++; + } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_PROCESS, logging::ERR_ALWAYS_CRITICAL, + "TupleAnnexStep::executeParallelOrderBy()"); + } + + // read out the input DL + while (more) + more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); + + // Count finished sorting threads under mutex and run final + // sort step when the last thread converges + fParallelFinalizeMutex.lock(); + fFinishedThreads++; + if (fFinishedThreads == fMaxThreads) + { + fParallelFinalizeMutex.unlock(); + if (fDistinct) + { + finalizeParallelOrderByDistinct(); } else { - currentPQ.reserve(QUEUE_RESERVE_SIZE); - } - - try - { - more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); - if (more) dlOffset++; - - while (more && !cancelled()) - { - if (dlOffset%fMaxThreads == id-1) - { - if (cancelled()) - break; - - if (currentPQ.capacity()-currentPQ.size() < doubleRGSize) - { - currentPQ.reserve(QUEUE_RESERVE_SIZE); - } - - rg.setData(&rgDataIn); - rg.getRow(0, &r); - if (!rowSize) - { - rowSize = r.getSize(); - } - rowCount = rg.getRowCount(); - - for (uint64_t i = 0; i < rowCount; ++i) - { - limOrderBy->processRow(r); - r.nextRow(rowSize); - } - } - - // *DRRTUY Implement a method to skip elements in FIFO - more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); - if(more) dlOffset++; - } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_IN_PROCESS, - logging::ERR_ALWAYS_CRITICAL, - "TupleAnnexStep::executeParallelOrderBy()"); - } - - // read out the input DL - while (more) - more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); - - // Count finished sorting threads under mutex and run final - // sort step when the last thread converges - fParallelFinalizeMutex.lock(); - fFinishedThreads++; - if (fFinishedThreads == fMaxThreads) - { - fParallelFinalizeMutex.unlock(); - if(fDistinct) - { - finalizeParallelOrderByDistinct(); - } - else - { - finalizeParallelOrderBy(); - } - } - else - { - fParallelFinalizeMutex.unlock(); + finalizeParallelOrderBy(); } + } + else + { + fParallelFinalizeMutex.unlock(); + } } const RowGroup& TupleAnnexStep::getOutputRowGroup() const { - return fRowGroupOut; + return fRowGroupOut; } - const RowGroup& TupleAnnexStep::getDeliveredRowGroup() const { - return fRowGroupDeliver; + return fRowGroupDeliver; } - void TupleAnnexStep::deliverStringTableRowGroup(bool b) { - fRowGroupOut.setUseStringTable(b); - fRowGroupDeliver.setUseStringTable(b); + fRowGroupOut.setUseStringTable(b); + fRowGroupDeliver.setUseStringTable(b); } - bool TupleAnnexStep::deliverStringTableRowGroup() const { - idbassert(fRowGroupOut.usesStringTable() == fRowGroupDeliver.usesStringTable()); - return fRowGroupDeliver.usesStringTable(); + idbassert(fRowGroupOut.usesStringTable() == fRowGroupDeliver.usesStringTable()); + return fRowGroupDeliver.usesStringTable(); } - const string TupleAnnexStep::toString() const { - ostringstream oss; - oss << "AnnexStep "; - oss << " ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "AnnexStep "; + oss << " ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " in:"; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); - oss << " out:"; + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); - if (fOrderBy) - oss << " " << fOrderBy->toString(); + if (fOrderBy) + oss << " " << fOrderBy->toString(); - if (fConstant) - oss << " " << fConstant->toString(); + if (fConstant) + oss << " " << fConstant->toString(); - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } - void TupleAnnexStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } - void TupleAnnexStep::formatMiniStats() { - ostringstream oss; - oss << "TNS "; - oss << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "TNS "; + oss << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); } - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/tupleannexstep.h b/dbcon/joblist/tupleannexstep.h index f9efd459b..96cc00642 100644 --- a/dbcon/joblist/tupleannexstep.h +++ b/dbcon/joblist/tupleannexstep.h @@ -18,7 +18,6 @@ // $Id: tupleannexstep.h 9596 2013-06-04 19:59:04Z xlou $ - #ifndef JOBLIST_TUPLEANNEXSTEP_H #define JOBLIST_TUPLEANNEXSTEP_H @@ -32,8 +31,7 @@ namespace joblist { class TupleConstantStep; class LimitedOrderBy; -} - +} // namespace joblist namespace joblist { @@ -42,148 +40,159 @@ namespace joblist */ class TupleAnnexStep : public JobStep, public TupleDeliveryStep { -public: - /** @brief TupleAnnexStep constructor - */ - TupleAnnexStep(const JobInfo& jobInfo); - // Copy ctor to have a class mutex - TupleAnnexStep(const TupleAnnexStep ©); + public: + /** @brief TupleAnnexStep constructor + */ + TupleAnnexStep(const JobInfo& jobInfo); + // Copy ctor to have a class mutex + TupleAnnexStep(const TupleAnnexStep& copy); - /** @brief TupleAnnexStep destructor - */ - ~TupleAnnexStep(); + /** @brief TupleAnnexStep destructor + */ + ~TupleAnnexStep(); - // inherited methods - void run(); - void join(); - const std::string toString() const; + // inherited methods + void run(); + void join(); + const std::string toString() const; - /** @brief TupleJobStep's pure virtual methods - */ - const rowgroup::RowGroup& getOutputRowGroup() const; - void setOutputRowGroup(const rowgroup::RowGroup&); + /** @brief TupleJobStep's pure virtual methods + */ + const rowgroup::RowGroup& getOutputRowGroup() const; + void setOutputRowGroup(const rowgroup::RowGroup&); - /** @brief TupleDeliveryStep's pure virtual methods - */ - uint32_t nextBand(messageqcpp::ByteStream& bs); - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + /** @brief TupleDeliveryStep's pure virtual methods + */ + uint32_t nextBand(messageqcpp::ByteStream& bs); + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; - void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); + void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); - void addOrderBy(LimitedOrderBy* lob) + void addOrderBy(LimitedOrderBy* lob) + { + fOrderBy = lob; + } + void addConstant(TupleConstantStep* tcs) + { + fConstant = tcs; + } + void setDistinct() + { + fDistinct = true; + } + void setLimit(uint64_t s, uint64_t c) + { + fLimitStart = s; + fLimitCount = c; + } + void setParallelOp() + { + fParallelOp = true; + } + void setMaxThreads(uint32_t number) + { + fMaxThreads = number; + } + + virtual bool stringTableFriendly() + { + return true; + } + + rowgroup::Row row1, row2; // scratch space for distinct comparisons todo: make them private + + protected: + void execute(); + void execute(uint32_t); + void executeNoOrderBy(); + void executeWithOrderBy(); + void executeParallelOrderBy(uint64_t id); + void executeNoOrderByWithDistinct(); + void formatMiniStats(); + void printCalTrace(); + void finalizeParallelOrderBy(); + void finalizeParallelOrderByDistinct(); + + // input/output rowgroup and row + rowgroup::RowGroup fRowGroupIn; + rowgroup::RowGroup fRowGroupOut; + rowgroup::RowGroup fRowGroupDeliver; + rowgroup::Row fRowIn; + rowgroup::Row fRowOut; + + // for datalist + RowGroupDL* fInputDL; + RowGroupDL* fOutputDL; + uint64_t fInputIterator; + std::vector fInputIteratorsList; + uint64_t fOutputIterator; + + class Runner + { + public: + Runner(TupleAnnexStep* step) : fStep(step), id(0) { - fOrderBy = lob; } - void addConstant(TupleConstantStep* tcs) + Runner(TupleAnnexStep* step, uint32_t id) : fStep(step), id(id) { - fConstant = tcs; } - void setDistinct() + void operator()() { - fDistinct = true; - } - void setLimit(uint64_t s, uint64_t c) - { - fLimitStart = s; - fLimitCount = c; - } - void setParallelOp() - { - fParallelOp = true; - } - void setMaxThreads(uint32_t number) - { - fMaxThreads = number; + if (id) + fStep->execute(id); + else + fStep->execute(); } - virtual bool stringTableFriendly() - { - return true; - } + TupleAnnexStep* fStep; + uint16_t id; + }; + uint64_t fRunner; // thread pool handle - rowgroup::Row row1, row2; // scratch space for distinct comparisons todo: make them private + uint64_t fRowsProcessed; + uint64_t fRowsReturned; + uint64_t fLimitStart; + uint64_t fLimitCount; + uint64_t fMaxThreads; + bool fLimitHit; + bool fEndOfResult; + bool fDistinct; + bool fParallelOp; -protected: - void execute(); - void execute(uint32_t); - void executeNoOrderBy(); - void executeWithOrderBy(); - void executeParallelOrderBy(uint64_t id); - void executeNoOrderByWithDistinct(); - void formatMiniStats(); - void printCalTrace(); - void finalizeParallelOrderBy(); - void finalizeParallelOrderByDistinct(); + LimitedOrderBy* fOrderBy; + TupleConstantStep* fConstant; - // input/output rowgroup and row - rowgroup::RowGroup fRowGroupIn; - rowgroup::RowGroup fRowGroupOut; - rowgroup::RowGroup fRowGroupDeliver; - rowgroup::Row fRowIn; - rowgroup::Row fRowOut; + funcexp::FuncExp* fFeInstance; + JobList* fJobList; - // for datalist - RowGroupDL* fInputDL; - RowGroupDL* fOutputDL; - uint64_t fInputIterator; - std::vector fInputIteratorsList; - uint64_t fOutputIterator; - - class Runner - { - public: - Runner(TupleAnnexStep* step) : - fStep(step), id(0) { } - Runner(TupleAnnexStep* step, uint32_t id) : - fStep(step), id(id) { } - void operator()() - { - if(id) - fStep->execute(id); - else - fStep->execute(); - } - - TupleAnnexStep* fStep; - uint16_t id; - }; - uint64_t fRunner; // thread pool handle - - uint64_t fRowsProcessed; - uint64_t fRowsReturned; - uint64_t fLimitStart; - uint64_t fLimitCount; - uint64_t fMaxThreads; - bool fLimitHit; - bool fEndOfResult; - bool fDistinct; - bool fParallelOp; - - LimitedOrderBy* fOrderBy; - TupleConstantStep* fConstant; - - funcexp::FuncExp* fFeInstance; - JobList* fJobList; - - std::vector fOrderByList; - std::vector fRunnersList; - uint16_t fFinishedThreads; - boost::mutex fParallelFinalizeMutex; + std::vector fOrderByList; + std::vector fRunnersList; + uint16_t fFinishedThreads; + boost::mutex fParallelFinalizeMutex; }; template -class reservablePQ: private std::priority_queue +class reservablePQ : private std::priority_queue { -public: - typedef typename std::priority_queue::size_type size_type; - reservablePQ(size_type capacity = 0) { reserve(capacity); }; - void reserve(size_type capacity) { this->c.reserve(capacity); } - size_type capacity() const { return this->c.capacity(); } + public: + typedef typename std::priority_queue::size_type size_type; + reservablePQ(size_type capacity = 0) + { + reserve(capacity); + }; + void reserve(size_type capacity) + { + this->c.reserve(capacity); + } + size_type capacity() const + { + return this->c.capacity(); + } }; -} // namespace +} // namespace joblist #endif // JOBLIST_TUPLEANNEXSTEP_H diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index bf58e4da5..ee252ec0f 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -18,7 +18,6 @@ // $Id: tupleconstantstep.cpp 9649 2013-06-25 16:08:05Z xlou $ - //#define NDEBUG #include #include @@ -53,806 +52,316 @@ using namespace querytele; #include "jlf_common.h" #include "tupleconstantstep.h" - namespace joblist { // static utility method SJSTEP TupleConstantStep::addConstantStep(const JobInfo& jobInfo, const rowgroup::RowGroup* rg) { - TupleConstantStep* tcs = NULL; + TupleConstantStep* tcs = NULL; - if (jobInfo.constantCol != CONST_COL_ONLY) - { - tcs = new TupleConstantStep(jobInfo); - } - else - { - tcs = new TupleConstantOnlyStep(jobInfo); - } + if (jobInfo.constantCol != CONST_COL_ONLY) + { + tcs = new TupleConstantStep(jobInfo); + } + else + { + tcs = new TupleConstantOnlyStep(jobInfo); + } - tcs->initialize(jobInfo, rg); - SJSTEP spcs(tcs); - return spcs; + tcs->initialize(jobInfo, rg); + SJSTEP spcs(tcs); + return spcs; } - -TupleConstantStep::TupleConstantStep(const JobInfo& jobInfo) : - JobStep(jobInfo), - fRowsReturned(0), - fInputDL(NULL), - fOutputDL(NULL), - fInputIterator(0), - fRunner(0), - fEndOfResult(false) +TupleConstantStep::TupleConstantStep(const JobInfo& jobInfo) + : JobStep(jobInfo) + , fRowsReturned(0) + , fInputDL(NULL) + , fOutputDL(NULL) + , fInputIterator(0) + , fRunner(0) + , fEndOfResult(false) { - fExtendedInfo = "TCS: "; - fQtc.stepParms().stepType = StepTeleStats::T_TCS; + fExtendedInfo = "TCS: "; + fQtc.stepParms().stepType = StepTeleStats::T_TCS; } - TupleConstantStep::~TupleConstantStep() { } - void TupleConstantStep::setOutputRowGroup(const rowgroup::RowGroup& rg) { - throw runtime_error("Disabled, use initialize() to set output RowGroup."); + throw runtime_error("Disabled, use initialize() to set output RowGroup."); } - void TupleConstantStep::initialize(const JobInfo& jobInfo, const RowGroup* rgIn) { - vector oids, oidsIn = fRowGroupIn.getOIDs(); - vector keys, keysIn = fRowGroupIn.getKeys(); - vector scale, scaleIn = fRowGroupIn.getScale(); - vector precision, precisionIn = fRowGroupIn.getPrecision(); - vector types, typesIn = fRowGroupIn.getColTypes(); - vector csNums, csNumsIn = fRowGroupIn.getCharsetNumbers(); - vector pos; - pos.push_back(2); + vector oids, oidsIn = fRowGroupIn.getOIDs(); + vector keys, keysIn = fRowGroupIn.getKeys(); + vector scale, scaleIn = fRowGroupIn.getScale(); + vector precision, precisionIn = fRowGroupIn.getPrecision(); + vector types, typesIn = fRowGroupIn.getColTypes(); + vector csNums, csNumsIn = fRowGroupIn.getCharsetNumbers(); + vector pos; + pos.push_back(2); - if (rgIn) + if (rgIn) + { + fRowGroupIn = *rgIn; + fRowGroupIn.initRow(&fRowIn); + oidsIn = fRowGroupIn.getOIDs(); + keysIn = fRowGroupIn.getKeys(); + scaleIn = fRowGroupIn.getScale(); + precisionIn = fRowGroupIn.getPrecision(); + typesIn = fRowGroupIn.getColTypes(); + csNumsIn = fRowGroupIn.getCharsetNumbers(); + } + + for (uint64_t i = 0, j = 0; i < jobInfo.deliveredCols.size(); i++) + { + const ConstantColumn* cc = dynamic_cast(jobInfo.deliveredCols[i].get()); + + if (cc != NULL) { - fRowGroupIn = *rgIn; - fRowGroupIn.initRow(&fRowIn); - oidsIn = fRowGroupIn.getOIDs(); - keysIn = fRowGroupIn.getKeys(); - scaleIn = fRowGroupIn.getScale(); - precisionIn = fRowGroupIn.getPrecision(); - typesIn = fRowGroupIn.getColTypes(); - csNumsIn = fRowGroupIn.getCharsetNumbers(); - } - - for (uint64_t i = 0, j = 0; i < jobInfo.deliveredCols.size(); i++) - { - const ConstantColumn* cc = - dynamic_cast(jobInfo.deliveredCols[i].get()); - - if (cc != NULL) - { - CalpontSystemCatalog::ColType ct = cc->resultType(); - - if (ct.colDataType == CalpontSystemCatalog::VARCHAR) - ct.colWidth++; - - //Round colWidth up - if (ct.colWidth == 3) - ct.colWidth = 4; - else if (ct.colWidth == 5 || ct.colWidth == 6 || ct.colWidth == 7) - ct.colWidth = 8; - - oids.push_back(-1); - keys.push_back(-1); - scale.push_back(ct.scale); - precision.push_back(ct.precision); - types.push_back(ct.colDataType); - csNums.push_back(ct.charsetNumber); - pos.push_back(pos.back() + ct.colWidth); - - fIndexConst.push_back(i); - } - else - { - // select (select a) from region; - if (j >= oidsIn.size() && jobInfo.tableList.empty()) - { - throw IDBExcept(ERR_NO_FROM); - } - - idbassert(j < oidsIn.size()); - - oids.push_back(oidsIn[j]); - keys.push_back(keysIn[j]); - scale.push_back(scaleIn[j]); - precision.push_back(precisionIn[j]); - types.push_back(typesIn[j]); - csNums.push_back(csNumsIn[j]); - pos.push_back(pos.back() + fRowGroupIn.getColumnWidth(j)); - j++; - - fIndexMapping.push_back(i); - } - } - - fRowGroupOut = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, - jobInfo.stringTableThreshold); - fRowGroupOut.initRow(&fRowOut); - fRowGroupOut.initRow(&fRowConst, true); - - constructContanstRow(jobInfo); -} - - -void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) -{ - // construct a row with only the constant values - fConstRowData.reset(new uint8_t[fRowConst.getSize()]); - fRowConst.setData(fConstRowData.get()); - fRowConst.initToNull(); // make sure every col is init'd to something, because later we copy the whole row - const vector& types = fRowGroupOut.getColTypes(); - - for (vector::iterator i = fIndexConst.begin(); i != fIndexConst.end(); i++) - { - const ConstantColumn* cc = - dynamic_cast(jobInfo.deliveredCols[*i].get()); - const execplan::Result c = cc->result(); - - if (cc->type() == ConstantColumn::NULLDATA) - { - if (types[*i] == CalpontSystemCatalog::CHAR || - types[*i] == CalpontSystemCatalog::VARCHAR || - types[*i] == CalpontSystemCatalog::TEXT) - { - fRowConst.setStringField("", *i); - } - else if (isUnsigned(types[*i])) - { - fRowConst.setUintField(fRowConst.getNullValue(*i), *i); - } - else - { - fRowConst.setIntField(fRowConst.getSignedNullValue(*i), *i); - } - - continue; - } - - - switch (types[*i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - { - fRowConst.setIntField(c.intVal, *i); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fRowGroupOut.getColWidths()[*i] > datatypes::MAXLEGACYWIDTH) - fRowConst.setInt128Field(c.decimalVal.TSInt128::getValue(), *i); - else - fRowConst.setIntField(c.decimalVal.value, *i); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - fRowConst.setFloatField(c.floatVal, *i); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - fRowConst.setDoubleField(c.doubleVal, *i); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - fRowConst.setLongDoubleField(c.longDoubleVal, *i); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - fRowConst.setStringField(c.strVal, *i); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - { - fRowConst.setUintField(c.uintVal, *i); - break; - } - - default: - { - throw runtime_error("un-supported constant column type."); - break; - } - } // switch - } // for constant columns -} - - -void TupleConstantStep::run() -{ - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for constant step."); - - fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - - if (fInputDL == NULL) - throw logic_error("Input is not a RowGroup data list."); - - fInputIterator = fInputDL->getIterator(); - - if (fDelivery == false) - { - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for non-delivery constant step."); - - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); - - fRunner = jobstepThreadPool.invoke(Runner(this)); - } -} - - -void TupleConstantStep::join() -{ - if (fRunner) - jobstepThreadPool.join(fRunner); -} - - -uint32_t TupleConstantStep::nextBand(messageqcpp::ByteStream& bs) -{ - RGData rgDataIn; - RGData rgDataOut; - bool more = false; - uint32_t rowCount = 0; - - try - { - bs.restart(); - - more = fInputDL->next(fInputIterator, &rgDataIn); - - if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - if (!more && cancelled()) - { - fEndOfResult = true; - } - - if (more && !fEndOfResult) - { - fRowGroupIn.setData(&rgDataIn); - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - - fillInConstants(); - fRowGroupOut.serializeRGData(bs); - rowCount = fRowGroupOut.getRowCount(); - } - else - { - fEndOfResult = true; - } - } - catch (...) - { - handleException(std::current_exception(), - logging::tupleConstantStepErr, - logging::ERR_ALWAYS_CRITICAL, - "TupleConstantStep::nextBand()"); - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); - - fEndOfResult = true; - } - - if (fEndOfResult) - { - // send an empty / error band - RGData rgData(fRowGroupOut, 0); - fRowGroupOut.setData(&rgData); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setStatus(status()); - fRowGroupOut.serializeRGData(bs); - - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } - } - - return rowCount; -} - - -void TupleConstantStep::execute() -{ - RGData rgDataIn; - RGData rgDataOut; - bool more = false; - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - - try - { - more = fInputDL->next(fInputIterator, &rgDataIn); - - if (traceOn()) dlTimes.setFirstReadTime(); - - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - - if (!more && cancelled()) - { - fEndOfResult = true; - } - - while (more && !fEndOfResult) - { - fRowGroupIn.setData(&rgDataIn); - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - - fillInConstants(); - - more = fInputDL->next(fInputIterator, &rgDataIn); - - if (cancelled()) - { - fEndOfResult = true; - } - else - { - fOutputDL->insert(rgDataOut); - } - } - } - catch (...) - { - handleException(std::current_exception(), - logging::tupleConstantStepErr, - logging::ERR_ALWAYS_CRITICAL, - "TupleConstantStep::execute()"); - } - - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); - - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - // Bug 3136, let mini stats to be formatted if traceOn. - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } - - fEndOfResult = true; - fOutputDL->endOfInput(); -} - -// *DRRTUY Copy row at once not one field at a time -void TupleConstantStep::fillInConstants() -{ - fRowGroupIn.getRow(0, &fRowIn); - fRowGroupOut.getRow(0, &fRowOut); - - if (fIndexConst.size() > 1 || fIndexConst[0] != 0) - { - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) - { - copyRow(fRowConst, &fRowOut); - - fRowOut.setRid(fRowIn.getRelRid()); - - for (uint64_t j = 0; j < fIndexMapping.size(); ++j) - fRowIn.copyField(fRowOut, fIndexMapping[j], j); - - fRowIn.nextRow(); - fRowOut.nextRow(); - } - } - else // only first column is constant - { - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) - { - fRowOut.setRid(fRowIn.getRelRid()); - fRowConst.copyField(fRowOut, 0, 0); - - for (uint32_t i = 1; i < fRowOut.getColumnCount(); i++) - fRowIn.copyField(fRowOut, i, i - 1); - - fRowIn.nextRow(); - fRowOut.nextRow(); - } - } - - fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - fRowGroupOut.setRowCount(fRowGroupIn.getRowCount()); - fRowsReturned += fRowGroupOut.getRowCount(); -} - - -void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Row& rowOut) -{ - if (fIndexConst.size() > 1 || fIndexConst[0] != 0) - { - copyRow(fRowConst, &rowOut); - rowOut.setRid(rowIn.getRelRid()); - - for (uint64_t j = 0; j < fIndexMapping.size(); ++j) - rowIn.copyField(rowOut, fIndexMapping[j], j); - } - else // only first column is constant - { - rowOut.setRid(rowIn.getRelRid()); - fRowConst.copyField(rowOut, 0, 0); - - for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) - rowIn.copyField(rowOut, i, i - 1); - } -} - - -const RowGroup& TupleConstantStep::getOutputRowGroup() const -{ - return fRowGroupOut; -} - - -const RowGroup& TupleConstantStep::getDeliveredRowGroup() const -{ - return fRowGroupOut; -} - - -void TupleConstantStep::deliverStringTableRowGroup(bool b) -{ - fRowGroupOut.setUseStringTable(b); -} - - -bool TupleConstantStep::deliverStringTableRowGroup() const -{ - return fRowGroupOut.usesStringTable(); -} - - -const string TupleConstantStep::toString() const -{ - ostringstream oss; - oss << "ConstantStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - - oss << " in:"; - - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); - - oss << " out:"; - - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); - - oss << endl; - - return oss.str(); -} - - -void TupleConstantStep::printCalTrace() -{ - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); -} - - -void TupleConstantStep::formatMiniStats() -{ - ostringstream oss; - oss << "TCS " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); -} - - -// class TupleConstantOnlyStep -TupleConstantOnlyStep::TupleConstantOnlyStep(const JobInfo& jobInfo) : TupleConstantStep(jobInfo) -{ -// fExtendedInfo = "TCOS: "; -} - - -TupleConstantOnlyStep::~TupleConstantOnlyStep() -{ -} - - -//void TupleConstantOnlyStep::initialize(const RowGroup& rgIn, const JobInfo& jobInfo) -void TupleConstantOnlyStep::initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn) -{ - vector oids; - vector keys; - vector scale; - vector precision; - vector types; - vector csNums; - vector pos; - pos.push_back(2); - - deliverStringTableRowGroup(false); - - for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) - { - const ConstantColumn* cc = - dynamic_cast(jobInfo.deliveredCols[i].get()); - - if (cc == NULL) - throw runtime_error("none constant column found."); - - CalpontSystemCatalog::ColType ct = cc->resultType(); - - if (ct.colDataType == CalpontSystemCatalog::VARCHAR) - ct.colWidth++; - - //Round colWidth up - if (ct.colWidth == 3) - ct.colWidth = 4; - else if (ct.colWidth == 5 || ct.colWidth == 6 || ct.colWidth == 7) - ct.colWidth = 8; - - oids.push_back(-1); - keys.push_back(-1); - scale.push_back(ct.scale); - precision.push_back(ct.precision); - types.push_back(ct.colDataType); - csNums.push_back(ct.charsetNumber); - pos.push_back(pos.back() + ct.colWidth); - - fIndexConst.push_back(i); - } - - fRowGroupOut = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold, false); - fRowGroupOut.initRow(&fRowOut); - fRowGroupOut.initRow(&fRowConst, true); - - constructContanstRow(jobInfo); -} - - -void TupleConstantOnlyStep::run() -{ - if (fDelivery == false) - { - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for non-delivery constant step."); - - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); - - try - { - RGData rgDataOut(fRowGroupOut, 1); - fRowGroupOut.setData(&rgDataOut); - - if (traceOn()) dlTimes.setFirstReadTime(); - - fillInConstants(); - - fOutputDL->insert(rgDataOut); - } - catch (...) - { - handleException(std::current_exception(), - logging::tupleConstantStepErr, - logging::ERR_ALWAYS_CRITICAL, - "TupleConstantOnlyStep::run()"); - } - - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } - - // Bug 3136, let mini stats to be formatted if traceOn. - fEndOfResult = true; - fOutputDL->endOfInput(); - } -} - - -uint32_t TupleConstantOnlyStep::nextBand(messageqcpp::ByteStream& bs) -{ - RGData rgDataOut; - uint32_t rowCount = 0; - - if (!fEndOfResult) - { - try - { - bs.restart(); - - if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - rgDataOut.reinit(fRowGroupOut, 1); - fRowGroupOut.setData(&rgDataOut); - - fillInConstants(); - fRowGroupOut.serializeRGData(bs); - rowCount = fRowGroupOut.getRowCount(); - } - catch (...) - { - handleException(std::current_exception(), - logging::tupleConstantStepErr, - logging::ERR_ALWAYS_CRITICAL, - "TupleConstantOnlyStep::nextBand()"); - } - - fEndOfResult = true; + CalpontSystemCatalog::ColType ct = cc->resultType(); + + if (ct.colDataType == CalpontSystemCatalog::VARCHAR) + ct.colWidth++; + + // Round colWidth up + if (ct.colWidth == 3) + ct.colWidth = 4; + else if (ct.colWidth == 5 || ct.colWidth == 6 || ct.colWidth == 7) + ct.colWidth = 8; + + oids.push_back(-1); + keys.push_back(-1); + scale.push_back(ct.scale); + precision.push_back(ct.precision); + types.push_back(ct.colDataType); + csNums.push_back(ct.charsetNumber); + pos.push_back(pos.back() + ct.colWidth); + + fIndexConst.push_back(i); } else { - // send an empty / error band - RGData rgData(fRowGroupOut, 0); - fRowGroupOut.setData(&rgData); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setStatus(status()); - fRowGroupOut.serializeRGData(bs); + // select (select a) from region; + if (j >= oidsIn.size() && jobInfo.tableList.empty()) + { + throw IDBExcept(ERR_NO_FROM); + } - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } + idbassert(j < oidsIn.size()); + + oids.push_back(oidsIn[j]); + keys.push_back(keysIn[j]); + scale.push_back(scaleIn[j]); + precision.push_back(precisionIn[j]); + types.push_back(typesIn[j]); + csNums.push_back(csNumsIn[j]); + pos.push_back(pos.back() + fRowGroupIn.getColumnWidth(j)); + j++; + + fIndexMapping.push_back(i); } + } - return rowCount; + fRowGroupOut = + RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + fRowGroupOut.initRow(&fRowOut); + fRowGroupOut.initRow(&fRowConst, true); + + constructContanstRow(jobInfo); } - -void TupleConstantOnlyStep::fillInConstants() +void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) { - fRowGroupOut.getRow(0, &fRowOut); - idbassert(fRowConst.getColumnCount() == fRowOut.getColumnCount()); - fRowOut.usesStringTable(fRowConst.usesStringTable()); - copyRow(fRowConst, &fRowOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setRowCount(1); - fRowsReturned = 1; -} + // construct a row with only the constant values + fConstRowData.reset(new uint8_t[fRowConst.getSize()]); + fRowConst.setData(fConstRowData.get()); + fRowConst.initToNull(); // make sure every col is init'd to something, because later we copy the whole row + const vector& types = fRowGroupOut.getColTypes(); + for (vector::iterator i = fIndexConst.begin(); i != fIndexConst.end(); i++) + { + const ConstantColumn* cc = dynamic_cast(jobInfo.deliveredCols[*i].get()); + const execplan::Result c = cc->result(); -const string TupleConstantOnlyStep::toString() const -{ - ostringstream oss; - oss << "ConstantOnlyStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - - oss << " out:"; - - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); - - oss << endl; - - return oss.str(); -} - - -// class TupleConstantBooleanStep -TupleConstantBooleanStep::TupleConstantBooleanStep(const JobInfo& jobInfo, bool value) : - TupleConstantStep(jobInfo), - fValue(value) -{ -// fExtendedInfo = "TCBS: "; -} - - -TupleConstantBooleanStep::~TupleConstantBooleanStep() -{ -} - - -void TupleConstantBooleanStep::initialize(const RowGroup& rgIn, const JobInfo&) -{ - fRowGroupOut = rgIn; - fRowGroupOut.initRow(&fRowOut); - fRowGroupOut.initRow(&fRowConst, true); -} - - -void TupleConstantBooleanStep::run() -{ - if (fDelivery == false) + if (cc->type() == ConstantColumn::NULLDATA) { - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for non-delivery constant step."); + if (types[*i] == CalpontSystemCatalog::CHAR || types[*i] == CalpontSystemCatalog::VARCHAR || + types[*i] == CalpontSystemCatalog::TEXT) + { + fRowConst.setStringField("", *i); + } + else if (isUnsigned(types[*i])) + { + fRowConst.setUintField(fRowConst.getNullValue(*i), *i); + } + else + { + fRowConst.setIntField(fRowConst.getSignedNullValue(*i), *i); + } - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); - - if (traceOn()) - { - dlTimes.setFirstReadTime(); - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); - } - - // Bug 3136, let mini stats to be formatted if traceOn. - fOutputDL->endOfInput(); + continue; } + + switch (types[*i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + { + fRowConst.setIntField(c.intVal, *i); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (fRowGroupOut.getColWidths()[*i] > datatypes::MAXLEGACYWIDTH) + fRowConst.setInt128Field(c.decimalVal.TSInt128::getValue(), *i); + else + fRowConst.setIntField(c.decimalVal.value, *i); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + fRowConst.setFloatField(c.floatVal, *i); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + fRowConst.setDoubleField(c.doubleVal, *i); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + fRowConst.setLongDoubleField(c.longDoubleVal, *i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + { + fRowConst.setStringField(c.strVal, *i); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + fRowConst.setUintField(c.uintVal, *i); + break; + } + + default: + { + throw runtime_error("un-supported constant column type."); + break; + } + } // switch + } // for constant columns } - -uint32_t TupleConstantBooleanStep::nextBand(messageqcpp::ByteStream& bs) +void TupleConstantStep::run() { - // send an empty band + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for constant step."); + + fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + + if (fInputDL == NULL) + throw logic_error("Input is not a RowGroup data list."); + + fInputIterator = fInputDL->getIterator(); + + if (fDelivery == false) + { + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for non-delivery constant step."); + + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); + + fRunner = jobstepThreadPool.invoke(Runner(this)); + } +} + +void TupleConstantStep::join() +{ + if (fRunner) + jobstepThreadPool.join(fRunner); +} + +uint32_t TupleConstantStep::nextBand(messageqcpp::ByteStream& bs) +{ + RGData rgDataIn; + RGData rgDataOut; + bool more = false; + uint32_t rowCount = 0; + + try + { + bs.restart(); + + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + if (!more && cancelled()) + { + fEndOfResult = true; + } + + if (more && !fEndOfResult) + { + fRowGroupIn.setData(&rgDataIn); + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + + fillInConstants(); + fRowGroupOut.serializeRGData(bs); + rowCount = fRowGroupOut.getRowCount(); + } + else + { + fEndOfResult = true; + } + } + catch (...) + { + handleException(std::current_exception(), logging::tupleConstantStepErr, logging::ERR_ALWAYS_CRITICAL, + "TupleConstantStep::nextBand()"); + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); + + fEndOfResult = true; + } + + if (fEndOfResult) + { + // send an empty / error band RGData rgData(fRowGroupOut, 0); fRowGroupOut.setData(&rgData); fRowGroupOut.resetRowGroup(0); @@ -861,32 +370,476 @@ uint32_t TupleConstantBooleanStep::nextBand(messageqcpp::ByteStream& bs) if (traceOn()) { - dlTimes.setFirstReadTime(); - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - printCalTrace(); + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); } + } - return 0; + return rowCount; } +void TupleConstantStep::execute() +{ + RGData rgDataIn; + RGData rgDataOut; + bool more = false; + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + + try + { + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (traceOn()) + dlTimes.setFirstReadTime(); + + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + if (!more && cancelled()) + { + fEndOfResult = true; + } + + while (more && !fEndOfResult) + { + fRowGroupIn.setData(&rgDataIn); + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + + fillInConstants(); + + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (cancelled()) + { + fEndOfResult = true; + } + else + { + fOutputDL->insert(rgDataOut); + } + } + } + catch (...) + { + handleException(std::current_exception(), logging::tupleConstantStepErr, logging::ERR_ALWAYS_CRITICAL, + "TupleConstantStep::execute()"); + } + + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); + + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + // Bug 3136, let mini stats to be formatted if traceOn. + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + + fEndOfResult = true; + fOutputDL->endOfInput(); +} + +// *DRRTUY Copy row at once not one field at a time +void TupleConstantStep::fillInConstants() +{ + fRowGroupIn.getRow(0, &fRowIn); + fRowGroupOut.getRow(0, &fRowOut); + + if (fIndexConst.size() > 1 || fIndexConst[0] != 0) + { + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) + { + copyRow(fRowConst, &fRowOut); + + fRowOut.setRid(fRowIn.getRelRid()); + + for (uint64_t j = 0; j < fIndexMapping.size(); ++j) + fRowIn.copyField(fRowOut, fIndexMapping[j], j); + + fRowIn.nextRow(); + fRowOut.nextRow(); + } + } + else // only first column is constant + { + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) + { + fRowOut.setRid(fRowIn.getRelRid()); + fRowConst.copyField(fRowOut, 0, 0); + + for (uint32_t i = 1; i < fRowOut.getColumnCount(); i++) + fRowIn.copyField(fRowOut, i, i - 1); + + fRowIn.nextRow(); + fRowOut.nextRow(); + } + } + + fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); + fRowGroupOut.setRowCount(fRowGroupIn.getRowCount()); + fRowsReturned += fRowGroupOut.getRowCount(); +} + +void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Row& rowOut) +{ + if (fIndexConst.size() > 1 || fIndexConst[0] != 0) + { + copyRow(fRowConst, &rowOut); + rowOut.setRid(rowIn.getRelRid()); + + for (uint64_t j = 0; j < fIndexMapping.size(); ++j) + rowIn.copyField(rowOut, fIndexMapping[j], j); + } + else // only first column is constant + { + rowOut.setRid(rowIn.getRelRid()); + fRowConst.copyField(rowOut, 0, 0); + + for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) + rowIn.copyField(rowOut, i, i - 1); + } +} + +const RowGroup& TupleConstantStep::getOutputRowGroup() const +{ + return fRowGroupOut; +} + +const RowGroup& TupleConstantStep::getDeliveredRowGroup() const +{ + return fRowGroupOut; +} + +void TupleConstantStep::deliverStringTableRowGroup(bool b) +{ + fRowGroupOut.setUseStringTable(b); +} + +bool TupleConstantStep::deliverStringTableRowGroup() const +{ + return fRowGroupOut.usesStringTable(); +} + +const string TupleConstantStep::toString() const +{ + ostringstream oss; + oss << "ConstantStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + + oss << " in:"; + + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); + + oss << " out:"; + + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); + + oss << endl; + + return oss.str(); +} + +void TupleConstantStep::printCalTrace() +{ + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); +} + +void TupleConstantStep::formatMiniStats() +{ + ostringstream oss; + oss << "TCS " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); +} + +// class TupleConstantOnlyStep +TupleConstantOnlyStep::TupleConstantOnlyStep(const JobInfo& jobInfo) : TupleConstantStep(jobInfo) +{ + // fExtendedInfo = "TCOS: "; +} + +TupleConstantOnlyStep::~TupleConstantOnlyStep() +{ +} + +// void TupleConstantOnlyStep::initialize(const RowGroup& rgIn, const JobInfo& jobInfo) +void TupleConstantOnlyStep::initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn) +{ + vector oids; + vector keys; + vector scale; + vector precision; + vector types; + vector csNums; + vector pos; + pos.push_back(2); + + deliverStringTableRowGroup(false); + + for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) + { + const ConstantColumn* cc = dynamic_cast(jobInfo.deliveredCols[i].get()); + + if (cc == NULL) + throw runtime_error("none constant column found."); + + CalpontSystemCatalog::ColType ct = cc->resultType(); + + if (ct.colDataType == CalpontSystemCatalog::VARCHAR) + ct.colWidth++; + + // Round colWidth up + if (ct.colWidth == 3) + ct.colWidth = 4; + else if (ct.colWidth == 5 || ct.colWidth == 6 || ct.colWidth == 7) + ct.colWidth = 8; + + oids.push_back(-1); + keys.push_back(-1); + scale.push_back(ct.scale); + precision.push_back(ct.precision); + types.push_back(ct.colDataType); + csNums.push_back(ct.charsetNumber); + pos.push_back(pos.back() + ct.colWidth); + + fIndexConst.push_back(i); + } + + fRowGroupOut = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, + jobInfo.stringTableThreshold, false); + fRowGroupOut.initRow(&fRowOut); + fRowGroupOut.initRow(&fRowConst, true); + + constructContanstRow(jobInfo); +} + +void TupleConstantOnlyStep::run() +{ + if (fDelivery == false) + { + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for non-delivery constant step."); + + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); + + try + { + RGData rgDataOut(fRowGroupOut, 1); + fRowGroupOut.setData(&rgDataOut); + + if (traceOn()) + dlTimes.setFirstReadTime(); + + fillInConstants(); + + fOutputDL->insert(rgDataOut); + } + catch (...) + { + handleException(std::current_exception(), logging::tupleConstantStepErr, logging::ERR_ALWAYS_CRITICAL, + "TupleConstantOnlyStep::run()"); + } + + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + + // Bug 3136, let mini stats to be formatted if traceOn. + fEndOfResult = true; + fOutputDL->endOfInput(); + } +} + +uint32_t TupleConstantOnlyStep::nextBand(messageqcpp::ByteStream& bs) +{ + RGData rgDataOut; + uint32_t rowCount = 0; + + if (!fEndOfResult) + { + try + { + bs.restart(); + + if (traceOn() && dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + rgDataOut.reinit(fRowGroupOut, 1); + fRowGroupOut.setData(&rgDataOut); + + fillInConstants(); + fRowGroupOut.serializeRGData(bs); + rowCount = fRowGroupOut.getRowCount(); + } + catch (...) + { + handleException(std::current_exception(), logging::tupleConstantStepErr, logging::ERR_ALWAYS_CRITICAL, + "TupleConstantOnlyStep::nextBand()"); + } + + fEndOfResult = true; + } + else + { + // send an empty / error band + RGData rgData(fRowGroupOut, 0); + fRowGroupOut.setData(&rgData); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setStatus(status()); + fRowGroupOut.serializeRGData(bs); + + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + } + + return rowCount; +} + +void TupleConstantOnlyStep::fillInConstants() +{ + fRowGroupOut.getRow(0, &fRowOut); + idbassert(fRowConst.getColumnCount() == fRowOut.getColumnCount()); + fRowOut.usesStringTable(fRowConst.usesStringTable()); + copyRow(fRowConst, &fRowOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setRowCount(1); + fRowsReturned = 1; +} + +const string TupleConstantOnlyStep::toString() const +{ + ostringstream oss; + oss << "ConstantOnlyStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + + oss << " out:"; + + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); + + oss << endl; + + return oss.str(); +} + +// class TupleConstantBooleanStep +TupleConstantBooleanStep::TupleConstantBooleanStep(const JobInfo& jobInfo, bool value) + : TupleConstantStep(jobInfo), fValue(value) +{ + // fExtendedInfo = "TCBS: "; +} + +TupleConstantBooleanStep::~TupleConstantBooleanStep() +{ +} + +void TupleConstantBooleanStep::initialize(const RowGroup& rgIn, const JobInfo&) +{ + fRowGroupOut = rgIn; + fRowGroupOut.initRow(&fRowOut); + fRowGroupOut.initRow(&fRowConst, true); +} + +void TupleConstantBooleanStep::run() +{ + if (fDelivery == false) + { + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for non-delivery constant step."); + + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); + + if (traceOn()) + { + dlTimes.setFirstReadTime(); + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + + // Bug 3136, let mini stats to be formatted if traceOn. + fOutputDL->endOfInput(); + } +} + +uint32_t TupleConstantBooleanStep::nextBand(messageqcpp::ByteStream& bs) +{ + // send an empty band + RGData rgData(fRowGroupOut, 0); + fRowGroupOut.setData(&rgData); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setStatus(status()); + fRowGroupOut.serializeRGData(bs); + + if (traceOn()) + { + dlTimes.setFirstReadTime(); + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + + return 0; +} const string TupleConstantBooleanStep::toString() const { - ostringstream oss; - oss << "ConstantBooleanStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "ConstantBooleanStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " out:"; + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/tupleconstantstep.h b/dbcon/joblist/tupleconstantstep.h index ee38dad1e..0aca68ff7 100644 --- a/dbcon/joblist/tupleconstantstep.h +++ b/dbcon/joblist/tupleconstantstep.h @@ -17,7 +17,6 @@ // $Id: tupleconstantstep.h 9596 2013-06-04 19:59:04Z xlou $ - #ifndef JOBLIST_TUPLECONSTANTSTEP_H #define JOBLIST_TUPLECONSTANTSTEP_H @@ -26,167 +25,170 @@ namespace joblist { - /** @brief class TupleConstantStep * */ class TupleConstantStep : public JobStep, public TupleDeliveryStep { -public: - /** @brief TupleConstantStep constructor - */ - TupleConstantStep(const JobInfo& jobInfo); + public: + /** @brief TupleConstantStep constructor + */ + TupleConstantStep(const JobInfo& jobInfo); - /** @brief TupleConstantStep destructor - */ - ~TupleConstantStep(); + /** @brief TupleConstantStep destructor + */ + ~TupleConstantStep(); - /** @brief virtual void Run method - */ - void run(); + /** @brief virtual void Run method + */ + void run(); - /** @brief virtual void join method - */ - void join(); + /** @brief virtual void join method + */ + void join(); - /** @brief virtual string toString method - */ - const std::string toString() const; + /** @brief virtual string toString method + */ + const std::string toString() const; - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + void setOutputRowGroup(const rowgroup::RowGroup&); + const rowgroup::RowGroup& getOutputRowGroup() const; + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; + uint32_t nextBand(messageqcpp::ByteStream& bs); - virtual void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn); - virtual void fillInConstants(const rowgroup::Row& rowIn, rowgroup::Row& rowOut); - static SJSTEP addConstantStep(const JobInfo& jobInfo, const rowgroup::RowGroup* rg = NULL); + virtual void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn); + virtual void fillInConstants(const rowgroup::Row& rowIn, rowgroup::Row& rowOut); + static SJSTEP addConstantStep(const JobInfo& jobInfo, const rowgroup::RowGroup* rg = NULL); -protected: - virtual void execute(); - virtual void fillInConstants(); - virtual void formatMiniStats(); - virtual void printCalTrace(); - virtual void constructContanstRow(const JobInfo& jobInfo); + protected: + virtual void execute(); + virtual void fillInConstants(); + virtual void formatMiniStats(); + virtual void printCalTrace(); + virtual void constructContanstRow(const JobInfo& jobInfo); - // for base - uint64_t fRowsReturned; + // for base + uint64_t fRowsReturned; - // input/output rowgroup and row - rowgroup::RowGroup fRowGroupIn; - rowgroup::RowGroup fRowGroupOut; - rowgroup::Row fRowIn; - rowgroup::Row fRowOut; + // input/output rowgroup and row + rowgroup::RowGroup fRowGroupIn; + rowgroup::RowGroup fRowGroupOut; + rowgroup::Row fRowIn; + rowgroup::Row fRowOut; - // mapping - std::vector fIndexConst; // consts in output row - std::vector fIndexMapping; // from input row to output row + // mapping + std::vector fIndexConst; // consts in output row + std::vector fIndexMapping; // from input row to output row - // store constants - rowgroup::Row fRowConst; - boost::scoped_array fConstRowData; + // store constants + rowgroup::Row fRowConst; + boost::scoped_array fConstRowData; - // for datalist - RowGroupDL* fInputDL; - RowGroupDL* fOutputDL; - uint64_t fInputIterator; + // for datalist + RowGroupDL* fInputDL; + RowGroupDL* fOutputDL; + uint64_t fInputIterator; - class Runner + class Runner + { + public: + Runner(TupleConstantStep* step) : fStep(step) { - public: - Runner(TupleConstantStep* step) : fStep(step) { } - void operator()() - { - utils::setThreadName("TCSRunner"); - fStep->execute(); - } + } + void operator()() + { + utils::setThreadName("TCSRunner"); + fStep->execute(); + } - TupleConstantStep* fStep; - }; + TupleConstantStep* fStep; + }; - uint64_t fRunner; // thread pool handle - bool fEndOfResult; + uint64_t fRunner; // thread pool handle + bool fEndOfResult; }; - class TupleConstantOnlyStep : public TupleConstantStep { -public: - /** @brief TupleConstantOnlyStep constructor - */ - TupleConstantOnlyStep(const JobInfo& jobInfo); + public: + /** @brief TupleConstantOnlyStep constructor + */ + TupleConstantOnlyStep(const JobInfo& jobInfo); - /** @brief TupleConstantOnlyStep destructor - */ - ~TupleConstantOnlyStep(); + /** @brief TupleConstantOnlyStep destructor + */ + ~TupleConstantOnlyStep(); - /** @brief virtual void Run method - */ - void run(); + /** @brief virtual void Run method + */ + void run(); - /** @brief virtual void initialize method - */ - virtual void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn); + /** @brief virtual void initialize method + */ + virtual void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn); - const std::string toString() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); - -protected: - using TupleConstantStep::fillInConstants; - void fillInConstants(); + const std::string toString() const; + uint32_t nextBand(messageqcpp::ByteStream& bs); + protected: + using TupleConstantStep::fillInConstants; + void fillInConstants(); }; - class TupleConstantBooleanStep : public TupleConstantStep { -public: - /** @brief TupleConstantBooleanStep constructor - */ - TupleConstantBooleanStep(const JobInfo& jobInfo, bool value); + public: + /** @brief TupleConstantBooleanStep constructor + */ + TupleConstantBooleanStep(const JobInfo& jobInfo, bool value); - /** @brief TupleConstantBooleanStep destructor - */ - ~TupleConstantBooleanStep(); + /** @brief TupleConstantBooleanStep destructor + */ + ~TupleConstantBooleanStep(); - /** @brief virtual void Run method - */ - void run(); + /** @brief virtual void Run method + */ + void run(); - /** @brief virtual void initialize method - For some reason, this doesn't match the base class's virtual signature. - However (for now), it's ok, because it's only called in one place and - doesn't need to be virtual there. - */ - using TupleConstantStep::initialize; - void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); + /** @brief virtual void initialize method + For some reason, this doesn't match the base class's virtual signature. + However (for now), it's ok, because it's only called in one place and + doesn't need to be virtual there. + */ + using TupleConstantStep::initialize; + void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); - const std::string toString() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + const std::string toString() const; + uint32_t nextBand(messageqcpp::ByteStream& bs); - virtual void boolValue(bool b) - { - fValue = b; - } - virtual bool boolValue() const - { - return fValue; - } + virtual void boolValue(bool b) + { + fValue = b; + } + virtual bool boolValue() const + { + return fValue; + } -protected: - void execute() {} - using TupleConstantStep::fillInConstants; - void fillInConstants() {} - void constructContanstRow(const JobInfo& jobInfo) {} + protected: + void execute() + { + } + using TupleConstantStep::fillInConstants; + void fillInConstants() + { + } + void constructContanstRow(const JobInfo& jobInfo) + { + } - // boolean value - bool fValue; + // boolean value + bool fValue; }; - -} // namespace +} // namespace joblist #endif // JOBLIST_TUPLECONSTANTSTEP_H diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index b18f73b61..8625723eb 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -18,7 +18,6 @@ // $Id: tuplehashjoin.cpp 9709 2013-07-20 06:08:46Z xlou $ - #include #include #include @@ -59,155 +58,155 @@ using namespace querytele; namespace joblist { - -TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) : - JobStep(jobInfo), - joinType(INIT), - fTableOID1(0), - fTableOID2(0), - fOid1(0), - fOid2(0), - fDictOid1(0), - fDictOid2(0), - fSequence1(-1), - fSequence2(-1), - fTupleId1(-1), - fTupleId2(-1), - fCorrelatedSide(0), - resourceManager(jobInfo.rm), - fMemSizeForOutputRG(0), - runRan(false), - joinRan(false), - largeSideIndex(1), - joinIsTooBig(false), - isExeMgr(jobInfo.isExeMgr), - lastSmallOuterJoiner(-1), - fTokenJoin(-1), - fStatsMutexPtr(new boost::mutex()), - fFunctionJoinKeys(jobInfo.keyInfo->functionJoinKeys), - sessionMemLimit(jobInfo.umMemLimit), - rgdLock(false) +TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) + : JobStep(jobInfo) + , joinType(INIT) + , fTableOID1(0) + , fTableOID2(0) + , fOid1(0) + , fOid2(0) + , fDictOid1(0) + , fDictOid2(0) + , fSequence1(-1) + , fSequence2(-1) + , fTupleId1(-1) + , fTupleId2(-1) + , fCorrelatedSide(0) + , resourceManager(jobInfo.rm) + , fMemSizeForOutputRG(0) + , runRan(false) + , joinRan(false) + , largeSideIndex(1) + , joinIsTooBig(false) + , isExeMgr(jobInfo.isExeMgr) + , lastSmallOuterJoiner(-1) + , fTokenJoin(-1) + , fStatsMutexPtr(new boost::mutex()) + , fFunctionJoinKeys(jobInfo.keyInfo->functionJoinKeys) + , sessionMemLimit(jobInfo.umMemLimit) + , rgdLock(false) { - /* Need to figure out how much memory these use... - Overhead storing 16 byte elements is about 32 bytes. That - should stay the same for other element sizes. - */ + /* Need to figure out how much memory these use... + Overhead storing 16 byte elements is about 32 bytes. That + should stay the same for other element sizes. + */ - pmMemLimit = resourceManager->getHjPmMaxMemorySmallSide(fSessionId); - uniqueLimit = resourceManager->getHjCPUniqueLimit(); + pmMemLimit = resourceManager->getHjPmMaxMemorySmallSide(fSessionId); + uniqueLimit = resourceManager->getHjCPUniqueLimit(); - fExtendedInfo = "THJS: "; - joinType = INIT; - joinThreadCount = resourceManager->getJlNumScanReceiveThreads(); - largeBPS = NULL; - moreInput = true; - fQtc.stepParms().stepType = StepTeleStats::T_HJS; - outputDL = NULL; - ownsOutputDL = false; - djsSmallUsage = jobInfo.smallSideUsage; - djsSmallLimit = jobInfo.smallSideLimit; - djsLargeLimit = jobInfo.largeSideLimit; - djsPartitionSize = jobInfo.partitionSize; - isDML = jobInfo.isDML; + fExtendedInfo = "THJS: "; + joinType = INIT; + joinThreadCount = resourceManager->getJlNumScanReceiveThreads(); + largeBPS = NULL; + moreInput = true; + fQtc.stepParms().stepType = StepTeleStats::T_HJS; + outputDL = NULL; + ownsOutputDL = false; + djsSmallUsage = jobInfo.smallSideUsage; + djsSmallLimit = jobInfo.smallSideLimit; + djsLargeLimit = jobInfo.largeSideLimit; + djsPartitionSize = jobInfo.partitionSize; + isDML = jobInfo.isDML; - config::Config* config = config::Config::makeConfig(); - string str = config->getConfig("HashJoin", "AllowDiskBasedJoin"); + config::Config* config = config::Config::makeConfig(); + string str = config->getConfig("HashJoin", "AllowDiskBasedJoin"); - if (str.empty() || str == "y" || str == "Y") - allowDJS = true; - else - allowDJS = false; + if (str.empty() || str == "y" || str == "Y") + allowDJS = true; + else + allowDJS = false; - numCores = resourceManager->numCores(); - if (numCores <= 0) - numCores = 8; - /* Debugging, rand() is used to simulate failures - time_t t = time(NULL); - srand(t); - */ + numCores = resourceManager->numCores(); + if (numCores <= 0) + numCores = 8; + /* Debugging, rand() is used to simulate failures + time_t t = time(NULL); + srand(t); + */ } TupleHashJoinStep::~TupleHashJoinStep() { - delete fStatsMutexPtr; + delete fStatsMutexPtr; - if (ownsOutputDL) - delete outputDL; + if (ownsOutputDL) + delete outputDL; - if (memUsedByEachJoin) + if (memUsedByEachJoin) + { + for (uint i = 0; i < smallDLs.size(); i++) { - for (uint i = 0 ; i < smallDLs.size(); i++) - { - if (memUsedByEachJoin[i]) - resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); - } + if (memUsedByEachJoin[i]) + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); } - returnMemory(); - //cout << "deallocated THJS, UM memory available: " << resourceManager.availableMemory() << endl; + } + returnMemory(); + // cout << "deallocated THJS, UM memory available: " << resourceManager.availableMemory() << endl; } void TupleHashJoinStep::run() { - uint32_t i; + uint32_t i; - boost::mutex::scoped_lock lk(jlLock); + boost::mutex::scoped_lock lk(jlLock); - if (runRan) - return; + if (runRan) + return; - runRan = true; + runRan = true; - deliverMutex.lock(); + deliverMutex.lock(); -// cout << "TupleHashJoinStep::run(): fOutputJobStepAssociation.outSize = " << fOutputJobStepAssociation.outSize() << ", fDelivery = " << boolalpha << fDelivery << endl; - idbassert((fOutputJobStepAssociation.outSize() == 1 && !fDelivery) || - (fOutputJobStepAssociation.outSize() == 0 && fDelivery)); - idbassert(fInputJobStepAssociation.outSize() >= 2); + // cout << "TupleHashJoinStep::run(): fOutputJobStepAssociation.outSize = " << + // fOutputJobStepAssociation.outSize() << ", fDelivery = " << boolalpha << fDelivery << endl; + idbassert((fOutputJobStepAssociation.outSize() == 1 && !fDelivery) || + (fOutputJobStepAssociation.outSize() == 0 && fDelivery)); + idbassert(fInputJobStepAssociation.outSize() >= 2); - largeDL = fInputJobStepAssociation.outAt(largeSideIndex)->rowGroupDL(); - largeIt = largeDL->getIterator(); + largeDL = fInputJobStepAssociation.outAt(largeSideIndex)->rowGroupDL(); + largeIt = largeDL->getIterator(); - for (i = 0; i < fInputJobStepAssociation.outSize(); i++) + for (i = 0; i < fInputJobStepAssociation.outSize(); i++) + { + if (i != largeSideIndex) { - if (i != largeSideIndex) - { - smallDLs.push_back(fInputJobStepAssociation.outAt(i)->rowGroupDL()); - smallIts.push_back(smallDLs.back()->getIterator()); - } + smallDLs.push_back(fInputJobStepAssociation.outAt(i)->rowGroupDL()); + smallIts.push_back(smallDLs.back()->getIterator()); } + } - if (!fDelivery) - outputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - else if (!largeBPS) - { - ownsOutputDL = true; - outputDL = new RowGroupDL(1, 5); - outputIt = outputDL->getIterator(); - } + if (!fDelivery) + outputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + else if (!largeBPS) + { + ownsOutputDL = true; + outputDL = new RowGroupDL(1, 5); + outputIt = outputDL->getIterator(); + } - joiners.resize(smallDLs.size()); - mainRunner = jobstepThreadPool.invoke(HJRunner(this)); + joiners.resize(smallDLs.size()); + mainRunner = jobstepThreadPool.invoke(HJRunner(this)); } void TupleHashJoinStep::join() { - boost::mutex::scoped_lock lk(jlLock); + boost::mutex::scoped_lock lk(jlLock); - if (joinRan) - return; + if (joinRan) + return; - joinRan = true; - jobstepThreadPool.join(mainRunner); + joinRan = true; + jobstepThreadPool.join(mainRunner); - if (djs) - { - for (int i = 0; i < (int) djsJoiners.size(); i++) - djs[i].join(); + if (djs) + { + for (int i = 0; i < (int)djsJoiners.size(); i++) + djs[i].join(); - jobstepThreadPool.join(djsReader); - jobstepThreadPool.join(djsRelay); - //cout << "THJS: joined all DJS threads, shared usage = " << *djsSmallUsage << endl; - } + jobstepThreadPool.join(djsReader); + jobstepThreadPool.join(djsRelay); + // cout << "THJS: joined all DJS threads, shared usage = " << *djsSmallUsage << endl; + } } // simple sol'n. Poll mem usage of Joiner once per second. Request mem @@ -215,921 +214,923 @@ void TupleHashJoinStep::join() // the threads inserting into Joiner. void TupleHashJoinStep::trackMem(uint index) { - boost::shared_ptr joiner = joiners[index]; - ssize_t memBefore = 0, memAfter = 0; - bool gotMem; + boost::shared_ptr joiner = joiners[index]; + ssize_t memBefore = 0, memAfter = 0; + bool gotMem; - boost::unique_lock scoped(memTrackMutex); - while (!stopMemTracking) - { - memAfter = joiner->getMemUsage(); - if (memAfter != memBefore) - { - gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, true); - if (gotMem) - atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); - else - return; - - memBefore = memAfter; - } - memTrackDone.timed_wait(scoped, boost::posix_time::seconds(1)); - } - - // one more iteration to capture mem usage since last poll, for this one - // raise an error if mem went over the limit + boost::unique_lock scoped(memTrackMutex); + while (!stopMemTracking) + { memAfter = joiner->getMemUsage(); - if (memAfter == memBefore) - return; - gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, true); - if (gotMem) + if (memAfter != memBefore) { + gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, true); + if (gotMem) atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); + else + return; + + memBefore = memAfter; } - else + memTrackDone.timed_wait(scoped, boost::posix_time::seconds(1)); + } + + // one more iteration to capture mem usage since last poll, for this one + // raise an error if mem went over the limit + memAfter = joiner->getMemUsage(); + if (memAfter == memBefore) + return; + gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, true); + if (gotMem) + { + atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); + } + else + { + if (!joinIsTooBig && + (isDML || !allowDJS || (fSessionId & 0x80000000) || (tableOid() < 3000 && tableOid() >= 1000))) { - if (!joinIsTooBig && (isDML || !allowDJS || (fSessionId & 0x80000000) || - (tableOid() < 3000 && tableOid() >= 1000))) - { - joinIsTooBig = true; - ostringstream oss; - oss << "(" << __LINE__ << ") " << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG); - fLogger->logMessage(logging::LOG_TYPE_INFO, oss.str()); - errorMessage(oss.str()); - status(logging::ERR_JOIN_TOO_BIG); - cout << "Join is too big, raise the UM join limit for now (monitor thread)" << endl; - abort(); - } + joinIsTooBig = true; + ostringstream oss; + oss << "(" << __LINE__ << ") " + << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG); + fLogger->logMessage(logging::LOG_TYPE_INFO, oss.str()); + errorMessage(oss.str()); + status(logging::ERR_JOIN_TOO_BIG); + cout << "Join is too big, raise the UM join limit for now (monitor thread)" << endl; + abort(); } + } } void TupleHashJoinStep::startSmallRunners(uint index) { - utils::setThreadName("HJSStartSmall"); - string extendedInfo; - JoinType jt; - boost::shared_ptr joiner; + utils::setThreadName("HJSStartSmall"); + string extendedInfo; + JoinType jt; + boost::shared_ptr joiner; - jt = joinTypes[index]; - extendedInfo += toString(); + jt = joinTypes[index]; + extendedInfo += toString(); - if (typelessJoin[index]) + if (typelessJoin[index]) + { + joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index], largeSideKeys[index], jt, + &jobstepThreadPool)); + } + else + { + joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index][0], largeSideKeys[index][0], + jt, &jobstepThreadPool)); + } + + joiner->setUniqueLimit(uniqueLimit); + joiner->setTableName(smallTableNames[index]); + joiners[index] = joiner; + + /* check for join types unsupported on the PM. */ + if (!largeBPS || !isExeMgr) + joiner->setInUM(rgData[index]); + + /* + start the small runners + join them + check status + handle abort, out of memory, etc + */ + + /* To measure wall-time spent constructing the small-side tables... + boost::posix_time::ptime end_time, start_time = + boost::posix_time::microsec_clock::universal_time(); + */ + + stopMemTracking = false; + utils::VLArray jobs(numCores); + uint64_t memMonitor = jobstepThreadPool.invoke([this, index] { this->trackMem(index); }); + // starting 1 thread when in PM mode, since it's only inserting into a + // vector of rows. The rest will be started when converted to UM mode. + if (joiner->inUM()) + for (int i = 0; i < numCores; i++) + jobs[i] = jobstepThreadPool.invoke([this, i, index, &jobs] { this->smallRunnerFcn(index, i, jobs); }); + else + jobs[0] = jobstepThreadPool.invoke([this, index, &jobs] { this->smallRunnerFcn(index, 0, jobs); }); + + // wait for the first thread to join, then decide whether the others exist and need joining + jobstepThreadPool.join(jobs[0]); + if (joiner->inUM()) + for (int i = 1; i < numCores; i++) + jobstepThreadPool.join(jobs[i]); + + // stop the monitor thread + memTrackMutex.lock(); + stopMemTracking = true; + memTrackDone.notify_one(); + memTrackMutex.unlock(); + jobstepThreadPool.join(memMonitor); + + /* If there was an error or an abort, drain the input DL, + do endOfInput on the output */ + if (cancelled()) + { + // cout << "HJ stopping... status is " << status() << endl; + if (largeBPS) + largeBPS->abort(); + + bool more = true; + RGData oneRG; + while (more) + more = smallDLs[index]->next(smallIts[index], &oneRG); + } + + /* To measure wall-time spent constructing the small-side tables... + end_time = boost::posix_time::microsec_clock::universal_time(); + if (!(fSessionId & 0x80000000)) + cout << "hash table construction time = " << end_time - start_time << + " size = " << joiner->size() << endl; + */ + + extendedInfo += "\n"; + + ostringstream oss; + if (!joiner->onDisk()) + { + // add extended info, and if not aborted then tell joiner + // we're done reading the small side. + if (joiner->inPM()) { - joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index], - largeSideKeys[index], jt, &jobstepThreadPool)); + oss << "PM join (" << index << ")" << endl; +#ifdef JLF_DEBUG + cout << oss.str(); +#endif + extendedInfo += oss.str(); } - else + else if (joiner->inUM()) { - joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index][0], - largeSideKeys[index][0], jt, &jobstepThreadPool)); + oss << "UM join (" << index << ")" << endl; +#ifdef JLF_DEBUG + cout << oss.str(); +#endif + extendedInfo += oss.str(); } + if (!cancelled()) + joiner->doneInserting(); + } - joiner->setUniqueLimit(uniqueLimit); - joiner->setTableName(smallTableNames[index]); - joiners[index] = joiner; - - /* check for join types unsupported on the PM. */ - if (!largeBPS || !isExeMgr) - joiner->setInUM(rgData[index]); - - /* - start the small runners - join them - check status - handle abort, out of memory, etc - */ - - /* To measure wall-time spent constructing the small-side tables... - boost::posix_time::ptime end_time, start_time = - boost::posix_time::microsec_clock::universal_time(); - */ - - stopMemTracking = false; - utils::VLArray jobs(numCores); - uint64_t memMonitor = jobstepThreadPool.invoke([this, index] { this->trackMem(index); }); - // starting 1 thread when in PM mode, since it's only inserting into a - // vector of rows. The rest will be started when converted to UM mode. - if (joiner->inUM()) - for (int i = 0; i < numCores; i++) - jobs[i] = jobstepThreadPool.invoke([this, i, index, &jobs] { this->smallRunnerFcn(index, i, jobs); }); - else - jobs[0] = jobstepThreadPool.invoke([this, index, &jobs] { this->smallRunnerFcn(index, 0, jobs); }); - - // wait for the first thread to join, then decide whether the others exist and need joining - jobstepThreadPool.join(jobs[0]); - if (joiner->inUM()) - for (int i = 1; i < numCores; i++) - jobstepThreadPool.join(jobs[i]); - - // stop the monitor thread - memTrackMutex.lock(); - stopMemTracking = true; - memTrackDone.notify_one(); - memTrackMutex.unlock(); - jobstepThreadPool.join(memMonitor); - - /* If there was an error or an abort, drain the input DL, - do endOfInput on the output */ - if (cancelled()) - { -// cout << "HJ stopping... status is " << status() << endl; - if (largeBPS) - largeBPS->abort(); - - bool more = true; - RGData oneRG; - while (more) - more = smallDLs[index]->next(smallIts[index], &oneRG); - } - - /* To measure wall-time spent constructing the small-side tables... - end_time = boost::posix_time::microsec_clock::universal_time(); - if (!(fSessionId & 0x80000000)) - cout << "hash table construction time = " << end_time - start_time << - " size = " << joiner->size() << endl; - */ - - extendedInfo += "\n"; - - ostringstream oss; - if (!joiner->onDisk()) - { - // add extended info, and if not aborted then tell joiner - // we're done reading the small side. - if (joiner->inPM()) - { - oss << "PM join (" << index << ")" << endl; - #ifdef JLF_DEBUG - cout << oss.str(); - #endif - extendedInfo += oss.str(); - } - else if (joiner->inUM()) - { - oss << "UM join (" << index << ")" << endl; - #ifdef JLF_DEBUG - cout << oss.str(); - #endif - extendedInfo += oss.str(); - } - if (!cancelled()) - joiner->doneInserting(); - } - - boost::mutex::scoped_lock lk(*fStatsMutexPtr); - fExtendedInfo += extendedInfo; - formatMiniStats(index); + boost::mutex::scoped_lock lk(*fStatsMutexPtr); + fExtendedInfo += extendedInfo; + formatMiniStats(index); } /* Index is which small input to read. */ -void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t *jobs) +void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t* jobs) { - utils::setThreadName("HJSmallRunner"); - bool more = true; - RGData oneRG; - Row r; - RowGroupDL* smallDL; - uint32_t smallIt; - RowGroup smallRG; - boost::shared_ptr joiner = joiners[index]; + utils::setThreadName("HJSmallRunner"); + bool more = true; + RGData oneRG; + Row r; + RowGroupDL* smallDL; + uint32_t smallIt; + RowGroup smallRG; + boost::shared_ptr joiner = joiners[index]; - smallDL = smallDLs[index]; - smallIt = smallIts[index]; - smallRG = smallRGs[index]; + smallDL = smallDLs[index]; + smallIt = smallIts[index]; + smallRG = smallRGs[index]; - smallRG.initRow(&r); - try + smallRG.initRow(&r); + try + { + ssize_t rgSize; + bool gotMem; + goto next; + while (more && !cancelled()) { - ssize_t rgSize; - bool gotMem; + smallRG.setData(&oneRG); + if (smallRG.getRowCount() == 0) goto next; - while (more && !cancelled()) + + // TupleHJ owns the row memory + utils::getSpinlock(rgdLock); + rgData[index].push_back(oneRG); + utils::releaseSpinlock(rgdLock); + + rgSize = smallRG.getSizeWithStrings(); + gotMem = resourceManager->getMemory(rgSize, sessionMemLimit, true); + if (gotMem) + { + atomicops::atomicAdd(&memUsedByEachJoin[index], rgSize); + } + else + { + /* Mem went over the limit. + If DML or a syscat query, abort. + if disk join is enabled, use it. + else abort. + */ + boost::unique_lock sl(saneErrMsg); + if (cancelled()) + return; + if (!allowDJS || isDML || (fSessionId & 0x80000000) || (tableOid() < 3000 && tableOid() >= 1000)) { - smallRG.setData(&oneRG); - if (smallRG.getRowCount() == 0) - goto next; - - // TupleHJ owns the row memory - utils::getSpinlock(rgdLock); - rgData[index].push_back(oneRG); - utils::releaseSpinlock(rgdLock); - - rgSize = smallRG.getSizeWithStrings(); - gotMem = resourceManager->getMemory(rgSize, sessionMemLimit, true); - if (gotMem) - { - atomicops::atomicAdd(&memUsedByEachJoin[index], rgSize); - } - else - { - /* Mem went over the limit. - If DML or a syscat query, abort. - if disk join is enabled, use it. - else abort. - */ - boost::unique_lock sl(saneErrMsg); - if (cancelled()) - return; - if (!allowDJS || isDML || (fSessionId & 0x80000000) || - (tableOid() < 3000 && tableOid() >= 1000)) - { - joinIsTooBig = true; - ostringstream oss; - oss << "(" << __LINE__ << ") " << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG); - fLogger->logMessage(logging::LOG_TYPE_INFO, oss.str()); - errorMessage(oss.str()); - status(logging::ERR_JOIN_TOO_BIG); - cout << "Join is too big, raise the UM join limit for now (small runner)" << endl; - abort(); - } - else if (allowDJS) - joiner->setConvertToDiskJoin(); - - return; - } - joiner->insertRGData(smallRG, threadID); - if (!joiner->inUM() && (memUsedByEachJoin[index] > pmMemLimit)) - { - joiner->setInUM(rgData[index]); - for (int i = 1; i < numCores; i++) - jobs[i] = jobstepThreadPool.invoke([this, i, index, jobs] - { this->smallRunnerFcn(index, i, jobs); }); - } -next: - dlMutex.lock(); - more = smallDL->next(smallIt, &oneRG); - dlMutex.unlock(); + joinIsTooBig = true; + ostringstream oss; + oss << "(" << __LINE__ << ") " + << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG); + fLogger->logMessage(logging::LOG_TYPE_INFO, oss.str()); + errorMessage(oss.str()); + status(logging::ERR_JOIN_TOO_BIG); + cout << "Join is too big, raise the UM join limit for now (small runner)" << endl; + abort(); } + else if (allowDJS) + joiner->setConvertToDiskJoin(); + + return; + } + joiner->insertRGData(smallRG, threadID); + if (!joiner->inUM() && (memUsedByEachJoin[index] > pmMemLimit)) + { + joiner->setInUM(rgData[index]); + for (int i = 1; i < numCores; i++) + jobs[i] = + jobstepThreadPool.invoke([this, i, index, jobs] { this->smallRunnerFcn(index, i, jobs); }); + } + next: + dlMutex.lock(); + more = smallDL->next(smallIt, &oneRG); + dlMutex.unlock(); } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_JOIN_TOO_BIG, - "TupleHashJoinStep::smallRunnerFcn()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - } - if (!joiner->inUM()) - joiner->setInPM(); + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_JOIN_TOO_BIG, + "TupleHashJoinStep::smallRunnerFcn()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + } + if (!joiner->inUM()) + joiner->setInPM(); } void TupleHashJoinStep::forwardCPData() { - uint32_t i, col; + uint32_t i, col; - if (largeBPS == NULL) - return; + if (largeBPS == NULL) + return; - for (i = 0; i < joiners.size(); i++) + for (i = 0; i < joiners.size(); i++) + { + if (joiners[i]->antiJoin() || joiners[i]->largeOuterJoin()) + continue; + + for (col = 0; col < joiners[i]->getSmallKeyColumns().size(); col++) { - if (joiners[i]->antiJoin() || joiners[i]->largeOuterJoin()) - continue; + uint32_t idx = joiners[i]->getSmallKeyColumns()[col]; - for (col = 0; col < joiners[i]->getSmallKeyColumns().size(); col++) - { - uint32_t idx = joiners[i]->getSmallKeyColumns()[col]; + if (smallRGs[i].isLongString(idx)) + continue; - if (smallRGs[i].isLongString(idx)) - continue; + // @bug3683, not to add CP predicates if large side is not simple column + if (fFunctionJoinKeys.find(largeRG.getKeys()[joiners[i]->getLargeKeyColumns()[col]]) != + fFunctionJoinKeys.end()) + continue; - // @bug3683, not to add CP predicates if large side is not simple column - if (fFunctionJoinKeys.find(largeRG.getKeys()[joiners[i]->getLargeKeyColumns()[col]]) != - fFunctionJoinKeys.end()) - continue; + bool isSmallSideWideDecimal = + datatypes::isWideDecimalType(smallRGs[i].getColType(idx), smallRGs[i].getColumnWidth(idx)); - bool isSmallSideWideDecimal = - datatypes::isWideDecimalType(smallRGs[i].getColType(idx), smallRGs[i].getColumnWidth(idx)); - - largeBPS->addCPPredicates(largeRG.getOIDs()[joiners[i]->getLargeKeyColumns()[col]], - joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col], - isSmallSideWideDecimal); - } + largeBPS->addCPPredicates(largeRG.getOIDs()[joiners[i]->getLargeKeyColumns()[col]], + joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col], + isSmallSideWideDecimal); } + } } void TupleHashJoinStep::djsRelayFcn() { - /* - read from largeDL - map to largeRG + outputRG format - insert into fifos[0] - */ + /* + read from largeDL + map to largeRG + outputRG format + insert into fifos[0] + */ - RowGroup djsInputRG = largeRG + outputRG; - RowGroup l_largeRG = (tbpsJoiners.empty() ? largeRG : largeRG + outputRG); - boost::shared_array relayMapping = makeMapping(l_largeRG, djsInputRG); - bool more; - RGData inData, outData; - Row l_largeRow, djsInputRow; - int i; + RowGroup djsInputRG = largeRG + outputRG; + RowGroup l_largeRG = (tbpsJoiners.empty() ? largeRG : largeRG + outputRG); + boost::shared_array relayMapping = makeMapping(l_largeRG, djsInputRG); + bool more; + RGData inData, outData; + Row l_largeRow, djsInputRow; + int i; - l_largeRG.initRow(&l_largeRow); - djsInputRG.initRow(&djsInputRow); + l_largeRG.initRow(&l_largeRow); + djsInputRG.initRow(&djsInputRow); - //cout << "Relay started" << endl; + // cout << "Relay started" << endl; - more = largeDL->next(largeIt, &inData); + more = largeDL->next(largeIt, &inData); - while (more && !cancelled()) + while (more && !cancelled()) + { + l_largeRG.setData(&inData); + + // if (fSessionId < 0x80000000) + // cout << "got largeside data = " << l_largeRG.toString() << endl; + if (l_largeRG.getRowCount() == 0) + goto next; + + outData.reinit(djsInputRG, l_largeRG.getRowCount()); + djsInputRG.setData(&outData); + djsInputRG.resetRowGroup(0); + l_largeRG.getRow(0, &l_largeRow); + djsInputRG.getRow(0, &djsInputRow); + + for (i = 0; i < (int)l_largeRG.getRowCount(); i++, l_largeRow.nextRow(), djsInputRow.nextRow()) { - l_largeRG.setData(&inData); - - //if (fSessionId < 0x80000000) - // cout << "got largeside data = " << l_largeRG.toString() << endl; - if (l_largeRG.getRowCount() == 0) - goto next; - - outData.reinit(djsInputRG, l_largeRG.getRowCount()); - djsInputRG.setData(&outData); - djsInputRG.resetRowGroup(0); - l_largeRG.getRow(0, &l_largeRow); - djsInputRG.getRow(0, &djsInputRow); - - for (i = 0; i < (int) l_largeRG.getRowCount(); i++, l_largeRow.nextRow(), djsInputRow.nextRow()) - { - applyMapping(relayMapping, l_largeRow, &djsInputRow); - djsInputRG.incRowCount(); - } - - fifos[0]->insert(outData); -next: - more = largeDL->next(largeIt, &inData); + applyMapping(relayMapping, l_largeRow, &djsInputRow); + djsInputRG.incRowCount(); } - while (more) - more = largeDL->next(largeIt, &inData); + fifos[0]->insert(outData); + next: + more = largeDL->next(largeIt, &inData); + } - fifos[0]->endOfInput(); + while (more) + more = largeDL->next(largeIt, &inData); + + fifos[0]->endOfInput(); } void TupleHashJoinStep::djsReaderFcn(int index) { - /* - read from fifos[index] - - incoming rgdata's have outputRG format - do FE2 processing - put into outputDL, to be picked up by the next JS or nextBand() - */ + /* + read from fifos[index] + - incoming rgdata's have outputRG format + do FE2 processing + put into outputDL, to be picked up by the next JS or nextBand() + */ - int it = fifos[index]->getIterator(); - bool more = true; - RowGroup l_outputRG = outputRG; - RGData rgData; - vector v_rgData; + int it = fifos[index]->getIterator(); + bool more = true; + RowGroup l_outputRG = outputRG; + RGData rgData; + vector v_rgData; - RowGroup l_fe2RG; - Row fe2InRow, fe2OutRow; - FuncExpWrapper l_fe; + RowGroup l_fe2RG; + Row fe2InRow, fe2OutRow; + FuncExpWrapper l_fe; + + if (fe2) + { + l_fe2RG = fe2Output; + l_outputRG.initRow(&fe2InRow); + l_fe2RG.initRow(&fe2OutRow); + l_fe = *fe2; + } + + makeDupList(fe2 ? l_fe2RG : l_outputRG); + + while (!cancelled()) + { + more = fifos[index]->next(it, &rgData); + + if (!more) + break; + + l_outputRG.setData(&rgData); + + if (l_outputRG.getRowCount() == 0) + continue; + + v_rgData.clear(); + v_rgData.push_back(rgData); if (fe2) - { - l_fe2RG = fe2Output; - l_outputRG.initRow(&fe2InRow); - l_fe2RG.initRow(&fe2OutRow); - l_fe = *fe2; - } + processFE2(l_outputRG, l_fe2RG, fe2InRow, fe2OutRow, &v_rgData, &l_fe); - makeDupList(fe2 ? l_fe2RG : l_outputRG); + processDupList(0, (fe2 ? l_fe2RG : l_outputRG), &v_rgData); + sendResult(v_rgData); + } - while (!cancelled()) - { - more = fifos[index]->next(it, &rgData); + while (more) + more = fifos[index]->next(it, &rgData); - if (!more) - break; + for (int i = 0; i < (int)djsJoiners.size(); i++) + { + fExtendedInfo += djs[i].extendedInfo(); + fMiniInfo += djs[i].miniInfo(); + } - l_outputRG.setData(&rgData); - - if (l_outputRG.getRowCount() == 0) - continue; - - v_rgData.clear(); - v_rgData.push_back(rgData); - - if (fe2) - processFE2(l_outputRG, l_fe2RG, fe2InRow, fe2OutRow, &v_rgData, &l_fe); - - processDupList(0, (fe2 ? l_fe2RG : l_outputRG), &v_rgData); - sendResult(v_rgData); - } - - while (more) - more = fifos[index]->next(it, &rgData); - - for (int i = 0; i < (int) djsJoiners.size(); i++) - { - fExtendedInfo += djs[i].extendedInfo(); - fMiniInfo += djs[i].miniInfo(); - } - - outputDL->endOfInput(); + outputDL->endOfInput(); } void TupleHashJoinStep::hjRunner() { - uint32_t i; - std::vector smallRunners; // thread handles from thread pool + uint32_t i; + std::vector smallRunners; // thread handles from thread pool - if (cancelled()) + if (cancelled()) + { + if (fOutputJobStepAssociation.outSize() > 0) + fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); + + startAdjoiningSteps(); + deliverMutex.unlock(); + return; + } + + StepTeleStats sts; + + if (fTableOID1 >= 3000) + { + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.start_time = QueryTeleClient::timeNowms(); + sts.total_units_of_work = 1; + postStepStartTele(sts); + } + + idbassert(joinTypes.size() == smallDLs.size()); + idbassert(joinTypes.size() == joiners.size()); + + /* Start the small-side runners */ + rgData.reset(new vector[smallDLs.size()]); + memUsedByEachJoin.reset(new ssize_t[smallDLs.size()]); + + for (i = 0; i < smallDLs.size(); i++) + atomicops::atomicZero(&memUsedByEachJoin[i]); + + try + { + /* Note: the only join that can have a useful small outer table is the last small outer, + * the others get clobbered by the join after it. Turn off small outer for 'the others'. + * The last small outer can be: + * the last small side; or followed by large outer small sides */ + bool turnOffSmallouter = false; + + for (int j = smallDLs.size() - 1; j >= 0; j--) { - if (fOutputJobStepAssociation.outSize() > 0) - fOutputJobStepAssociation.outAt(0)->rowGroupDL()->endOfInput(); - - startAdjoiningSteps(); - deliverMutex.unlock(); - return; + if (joinTypes[j] & SMALLOUTER) + { + if (turnOffSmallouter) + { + joinTypes[j] &= ~SMALLOUTER; + } + else // turnOffSmallouter == false, keep this one, but turn off any one in front + { + lastSmallOuterJoiner = j; + turnOffSmallouter = true; + } + } + else if (joinTypes[j] & INNER && turnOffSmallouter == false) + { + turnOffSmallouter = true; + } } - StepTeleStats sts; - - if (fTableOID1 >= 3000) - { - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.start_time = QueryTeleClient::timeNowms(); - sts.total_units_of_work = 1; - postStepStartTele(sts); - } - - idbassert(joinTypes.size() == smallDLs.size()); - idbassert(joinTypes.size() == joiners.size()); - - /* Start the small-side runners */ - rgData.reset(new vector[smallDLs.size()]); - memUsedByEachJoin.reset(new ssize_t[smallDLs.size()]); + smallRunners.clear(); + smallRunners.reserve(smallDLs.size()); for (i = 0; i < smallDLs.size(); i++) - atomicops::atomicZero(&memUsedByEachJoin[i]); + smallRunners.push_back(jobstepThreadPool.invoke(SmallRunner(this, i))); + } + catch (thread_resource_error&) + { + string emsg = "TupleHashJoin caught a thread resource error, aborting...\n"; + errorMessage("too many threads"); + status(logging::threadResourceErr); + errorLogging(emsg, logging::threadResourceErr); + fDie = true; + deliverMutex.unlock(); + } + + jobstepThreadPool.join(smallRunners); + smallRunners.clear(); + + for (i = 0; i < feIndexes.size() && joiners.size() > 0; i++) + joiners[feIndexes[i]]->setFcnExpFilter(fe[i]); + + /* segregate the Joiners into ones for TBPS and ones for DJS */ + segregateJoiners(); + + /* Need to clean this stuff up. If the query was cancelled before this, and this would have had + a disk join, it's still necessary to construct the DJS objects to finish the abort. + Update: Is this more complicated than scanning joiners for either ondisk() or (not isFinished()) + and draining the corresponding inputs & telling downstream EOF? todo, think about it */ + if (!djsJoiners.empty()) + { + joinIsTooBig = false; + + if (!cancelled()) + fLogger->logMessage(logging::LOG_TYPE_INFO, logging::INFO_SWITCHING_TO_DJS); + + uint32_t smallSideCount = djsJoiners.size(); + + if (!outputDL) + { + ownsOutputDL = true; + outputDL = new RowGroupDL(1, 5); + outputIt = outputDL->getIterator(); + } + + djs.reset(new DiskJoinStep[smallSideCount]); + fifos.reset(new boost::shared_ptr[smallSideCount + 1]); + + for (i = 0; i <= smallSideCount; i++) + fifos[i].reset(new RowGroupDL(1, 5)); + + boost::mutex::scoped_lock sl(djsLock); + + for (i = 0; i < smallSideCount; i++) + { + // these link themselves fifos[0]->DSJ[0]->fifos[1]->DSJ[1] ... ->fifos[smallSideCount], + // THJS puts data into fifos[0], reads it from fifos[smallSideCount] + djs[i] = DiskJoinStep(this, i, djsJoinerMap[i], (i == smallSideCount - 1)); + } + + sl.unlock(); try { - /* Note: the only join that can have a useful small outer table is the last small outer, - * the others get clobbered by the join after it. Turn off small outer for 'the others'. - * The last small outer can be: - * the last small side; or followed by large outer small sides */ - bool turnOffSmallouter = false; + for (i = 0; !cancelled() && i < smallSideCount; i++) + { + vector empty; + resourceManager->returnMemory(memUsedByEachJoin[djsJoinerMap[i]], sessionMemLimit); + atomicops::atomicZero(&memUsedByEachJoin[i]); + djs[i].loadExistingData(rgData[djsJoinerMap[i]]); + rgData[djsJoinerMap[i]].swap(empty); + } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXEMGR_MALFUNCTION, logging::ERR_JOIN_TOO_BIG, + "TupleHashJoinStep::hjRunner()"); + status(logging::ERR_EXEMGR_MALFUNCTION); + abort(); + } - for (int j = smallDLs.size() - 1; j >= 0; j--) - { - if (joinTypes[j] & SMALLOUTER) - { - if (turnOffSmallouter) - { - joinTypes[j] &= ~SMALLOUTER; - } - else // turnOffSmallouter == false, keep this one, but turn off any one in front - { - lastSmallOuterJoiner = j; - turnOffSmallouter = true; - } - } - else if (joinTypes[j] & INNER && turnOffSmallouter == false) - { - turnOffSmallouter = true; - } - } + if (fe2) + fe2Mapping = makeMapping(outputRG, fe2Output); - smallRunners.clear(); - smallRunners.reserve(smallDLs.size()); + bool relay = false, reader = false; - for (i = 0; i < smallDLs.size(); i++) - smallRunners.push_back(jobstepThreadPool.invoke(SmallRunner(this, i))); + /* If an error happened loading the existing data, these threads are necessary + to finish the abort */ + try + { + djsRelay = jobstepThreadPool.invoke(DJSRelay(this)); + relay = true; + djsReader = jobstepThreadPool.invoke(DJSReader(this, smallSideCount)); + reader = true; + + for (i = 0; i < smallSideCount; i++) + djs[i].run(); } catch (thread_resource_error&) { - string emsg = "TupleHashJoin caught a thread resource error, aborting...\n"; - errorMessage("too many threads"); - status(logging::threadResourceErr); - errorLogging(emsg, logging::threadResourceErr); - fDie = true; - deliverMutex.unlock(); + /* This means there is a gap somewhere in the chain, need to identify + where the gap is, drain the input, and close the output. */ + + string emsg = "TupleHashJoin caught a thread resource error, aborting...\n"; + errorMessage("too many threads"); + status(logging::threadResourceErr); + errorLogging(emsg, logging::threadResourceErr); + abort(); + + if (reader && relay) // must have been thrown from the djs::run() loop + { + // fill the gap in the chain: drain input of the failed DJS (i), close the last fifo + if (largeBPS) + largeDL->endOfInput(); + + int it = fifos[i]->getIterator(); + RGData rg; + + while (fifos[i]->next(it, &rg)) + ; + + fifos[smallSideCount]->endOfInput(); + } + else // no DJS's have been started + { + if (relay) + { + // drain Relay's output + if (largeBPS) + largeDL->endOfInput(); + + int it = fifos[0]->getIterator(); + RGData rg; + + while (fifos[0]->next(it, &rg)) + ; + } + + if (reader) + // close Reader's input + fifos[smallSideCount]->endOfInput(); + else // close the next JobStep's input + outputDL->endOfInput(); + } } + } - jobstepThreadPool.join(smallRunners); - smallRunners.clear(); + /* Final THJS configuration is settled here at the moment */ + deliverMutex.unlock(); - for (i = 0; i < feIndexes.size() && joiners.size() > 0; i++) - joiners[feIndexes[i]]->setFcnExpFilter(fe[i]); - - /* segregate the Joiners into ones for TBPS and ones for DJS */ - segregateJoiners(); - - /* Need to clean this stuff up. If the query was cancelled before this, and this would have had - a disk join, it's still necessary to construct the DJS objects to finish the abort. - Update: Is this more complicated than scanning joiners for either ondisk() or (not isFinished()) - and draining the corresponding inputs & telling downstream EOF? todo, think about it */ - if (!djsJoiners.empty()) + if (cancelled()) + { + if (joinIsTooBig && !status()) { - joinIsTooBig = false; + ostringstream oss; + oss << "(" << __LINE__ << ") " + << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG); + fLogger->logMessage(logging::LOG_TYPE_INFO, oss.str()); + errorMessage(oss.str()); + status(logging::ERR_JOIN_TOO_BIG); + cout << "Join is too big, raise the UM join limit for now" << endl; - if (!cancelled()) - fLogger->logMessage(logging::LOG_TYPE_INFO, logging::INFO_SWITCHING_TO_DJS); - - uint32_t smallSideCount = djsJoiners.size(); - - if (!outputDL) + /* Drop memory */ + if (!fDelivery) + { + joiners.clear(); + tbpsJoiners.clear(); + rgData.reset(); + for (uint i = 0; i < smallDLs.size(); i++) { - ownsOutputDL = true; - outputDL = new RowGroupDL(1, 5); - outputIt = outputDL->getIterator(); - } - - djs.reset(new DiskJoinStep[smallSideCount]); - fifos.reset(new boost::shared_ptr[smallSideCount + 1]); - - for (i = 0; i <= smallSideCount; i++) - fifos[i].reset(new RowGroupDL(1, 5)); - - boost::mutex::scoped_lock sl(djsLock); - - for (i = 0; i < smallSideCount; i++) - { - // these link themselves fifos[0]->DSJ[0]->fifos[1]->DSJ[1] ... ->fifos[smallSideCount], - // THJS puts data into fifos[0], reads it from fifos[smallSideCount] - djs[i] = DiskJoinStep(this, i, djsJoinerMap[i], (i == smallSideCount - 1)); - } - - sl.unlock(); - - try - { - for (i = 0; !cancelled() && i < smallSideCount; i++) - { - vector empty; - resourceManager->returnMemory(memUsedByEachJoin[djsJoinerMap[i]], sessionMemLimit); - atomicops::atomicZero(&memUsedByEachJoin[i]); - djs[i].loadExistingData(rgData[djsJoinerMap[i]]); - rgData[djsJoinerMap[i]].swap(empty); - } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXEMGR_MALFUNCTION, - logging::ERR_JOIN_TOO_BIG, - "TupleHashJoinStep::hjRunner()"); - status(logging::ERR_EXEMGR_MALFUNCTION); - abort(); - } - - if (fe2) - fe2Mapping = makeMapping(outputRG, fe2Output); - - bool relay = false, reader = false; - - /* If an error happened loading the existing data, these threads are necessary - to finish the abort */ - try - { - djsRelay = jobstepThreadPool.invoke(DJSRelay(this)); - relay = true; - djsReader = jobstepThreadPool.invoke(DJSReader(this, smallSideCount)); - reader = true; - - for (i = 0; i < smallSideCount; i++) - djs[i].run(); - } - catch (thread_resource_error&) - { - /* This means there is a gap somewhere in the chain, need to identify - where the gap is, drain the input, and close the output. */ - - string emsg = "TupleHashJoin caught a thread resource error, aborting...\n"; - errorMessage("too many threads"); - status(logging::threadResourceErr); - errorLogging(emsg, logging::threadResourceErr); - abort(); - - if (reader && relay) // must have been thrown from the djs::run() loop - { - // fill the gap in the chain: drain input of the failed DJS (i), close the last fifo - if (largeBPS) - largeDL->endOfInput(); - - int it = fifos[i]->getIterator(); - RGData rg; - - while (fifos[i]->next(it, &rg)); - - fifos[smallSideCount]->endOfInput(); - } - else // no DJS's have been started - { - if (relay) - { - // drain Relay's output - if (largeBPS) - largeDL->endOfInput(); - - int it = fifos[0]->getIterator(); - RGData rg; - - while (fifos[0]->next(it, &rg)); - } - - if (reader) - // close Reader's input - fifos[smallSideCount]->endOfInput(); - else // close the next JobStep's input - outputDL->endOfInput(); - } + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + atomicops::atomicZero(&memUsedByEachJoin[i]); } + } } + } - /* Final THJS configuration is settled here at the moment */ - deliverMutex.unlock(); + // todo: forwardCPData needs to grab data from djs + if (!djs) + forwardCPData(); // this fcn has its own exclusion list - if (cancelled()) + // decide if perform aggregation on PM + if (dynamic_cast(fDeliveryStep.get()) != NULL && largeBPS) + { + bool pmAggregation = !(dynamic_cast(fDeliveryStep.get())->umOnly()); + + for (i = 0; i < joiners.size() && pmAggregation; ++i) + pmAggregation = pmAggregation && (joiners[i]->inPM() && !joiners[i]->smallOuterJoin()); + + if (pmAggregation) + dynamic_cast(fDeliveryStep.get())->setPmHJAggregation(largeBPS); + } + + // can we sort the joiners? Currently they all have to be inner joins. + // Note, any vars that used to parallel the joiners list will be invalidated + // (ie. smallTableNames) + + for (i = 0; i < tbpsJoiners.size(); i++) + if (!tbpsJoiners[i]->innerJoin()) + break; + + if (i == tbpsJoiners.size()) + sort(tbpsJoiners.begin(), tbpsJoiners.end(), JoinerSorter()); + + /* Each thread independently decides whether a given join can execute on the PM. + * A PM join can't follow a UM join, so we fix that here. + */ + bool doUM; + + for (i = 0, doUM = false; i < tbpsJoiners.size(); i++) + { + if (tbpsJoiners[i]->inUM()) + doUM = true; + + if (tbpsJoiners[i]->inPM() && doUM) { - if (joinIsTooBig && !status()) - { - ostringstream oss; - oss << "(" << __LINE__ << ") " << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG); - fLogger->logMessage(logging::LOG_TYPE_INFO, oss.str()); - errorMessage(oss.str()); - status(logging::ERR_JOIN_TOO_BIG); - cout << "Join is too big, raise the UM join limit for now" << endl; - - /* Drop memory */ - if (!fDelivery) - { - joiners.clear(); - tbpsJoiners.clear(); - rgData.reset(); - for (uint i = 0; i < smallDLs.size(); i++) - { - resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); - atomicops::atomicZero(&memUsedByEachJoin[i]); - } - } - } - } - - // todo: forwardCPData needs to grab data from djs - if (!djs) - forwardCPData(); // this fcn has its own exclusion list - - // decide if perform aggregation on PM - if (dynamic_cast(fDeliveryStep.get()) != NULL && largeBPS) - { - bool pmAggregation = !(dynamic_cast(fDeliveryStep.get())->umOnly()); - - for (i = 0; i < joiners.size() && pmAggregation; ++i) - pmAggregation = pmAggregation && (joiners[i]->inPM() && !joiners[i]->smallOuterJoin()); - - if (pmAggregation) - dynamic_cast(fDeliveryStep.get())->setPmHJAggregation(largeBPS); - } - - // can we sort the joiners? Currently they all have to be inner joins. - // Note, any vars that used to parallel the joiners list will be invalidated - // (ie. smallTableNames) - - for (i = 0; i < tbpsJoiners.size(); i++) - if (!tbpsJoiners[i]->innerJoin()) - break; - - if (i == tbpsJoiners.size()) - sort(tbpsJoiners.begin(), tbpsJoiners.end(), JoinerSorter()); - - /* Each thread independently decides whether a given join can execute on the PM. - * A PM join can't follow a UM join, so we fix that here. - */ - bool doUM; - - for (i = 0, doUM = false; i < tbpsJoiners.size(); i++) - { - if (tbpsJoiners[i]->inUM()) - doUM = true; - - if (tbpsJoiners[i]->inPM() && doUM) - { #ifdef JLF_DEBUG - cout << "moving join " << i << " to UM (PM join can't follow a UM join)\n"; + cout << "moving join " << i << " to UM (PM join can't follow a UM join)\n"; #endif - tbpsJoiners[i]->setInUM(rgData[i]); - } + tbpsJoiners[i]->setInUM(rgData[i]); } + } - // there is an in-mem UM or PM join - if (largeBPS && !tbpsJoiners.empty()) + // there is an in-mem UM or PM join + if (largeBPS && !tbpsJoiners.empty()) + { + largeBPS->useJoiners(tbpsJoiners); + + if (djs) + largeBPS->setJoinedResultRG(largeRG + outputRG); + else + largeBPS->setJoinedResultRG(outputRG); + + if (!feIndexes.empty()) + largeBPS->setJoinFERG(joinFilterRG); + + // cout << "join UM memory available is " << totalUMMemoryUsage << endl; + + /* Figure out whether fe2 can run with the tables joined on the PM. If so, + fe2 -> PM, otherwise fe2 -> UM. + For now, the alg is "assume if any joins are done on the UM, fe2 has to go on + the UM." The structs and logic aren't in place yet to track all of the tables + through a joblist. */ + if (fe2 && !djs) { - largeBPS->useJoiners(tbpsJoiners); + /* Can't do a small outer join when the PM sends back joined rows */ + runFE2onPM = true; - if (djs) - largeBPS->setJoinedResultRG(largeRG + outputRG); - else - largeBPS->setJoinedResultRG(outputRG); + if (joinTypes[joiners.size() - 1] == SMALLOUTER) + runFE2onPM = false; - if (!feIndexes.empty()) - largeBPS->setJoinFERG(joinFilterRG); - -// cout << "join UM memory available is " << totalUMMemoryUsage << endl; - - /* Figure out whether fe2 can run with the tables joined on the PM. If so, - fe2 -> PM, otherwise fe2 -> UM. - For now, the alg is "assume if any joins are done on the UM, fe2 has to go on - the UM." The structs and logic aren't in place yet to track all of the tables - through a joblist. */ - if (fe2 && !djs) + for (i = 0; i < joiners.size(); i++) + if (joiners[i]->inUM()) { - /* Can't do a small outer join when the PM sends back joined rows */ - runFE2onPM = true; - - if (joinTypes[joiners.size() - 1] == SMALLOUTER) - runFE2onPM = false; - - for (i = 0; i < joiners.size(); i++) - if (joiners[i]->inUM()) - { - runFE2onPM = false; - break; - } + runFE2onPM = false; + break; + } #ifdef JLF_DEBUG - if (runFE2onPM) - cout << "PM runs FE2\n"; - else - cout << "UM runs FE2\n"; + if (runFE2onPM) + cout << "PM runs FE2\n"; + else + cout << "UM runs FE2\n"; #endif - largeBPS->setFcnExpGroup2(fe2, fe2Output, runFE2onPM); - } - else if (fe2) - runFE2onPM = false; - - if (!fDelivery && !djs) - { - /* connect the largeBPS directly to the next step */ - JobStepAssociation newJsa; - newJsa.outAdd(fOutputJobStepAssociation.outAt(0)); - - for (unsigned i = 1; i < largeBPS->outputAssociation().outSize(); i++) - newJsa.outAdd(largeBPS->outputAssociation().outAt(i)); - - largeBPS->outputAssociation(newJsa); - } - - startAdjoiningSteps(); + largeBPS->setFcnExpGroup2(fe2, fe2Output, runFE2onPM); } - else if (largeBPS) + else if (fe2) + runFE2onPM = false; + + if (!fDelivery && !djs) { - // there are no in-mem UM or PM joins, only disk-joins - startAdjoiningSteps(); - } - else if (!djs) - // if there's no largeBPS, all joins are either done by DJS or join threads, - // this clause starts the THJS join threads. - startJoinThreads(); + /* connect the largeBPS directly to the next step */ + JobStepAssociation newJsa; + newJsa.outAdd(fOutputJobStepAssociation.outAt(0)); - if (fTableOID1 >= 3000) - { - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.end_time = QueryTeleClient::timeNowms(); - sts.total_units_of_work = sts.units_of_work_completed = 1; - postStepSummaryTele(sts); + for (unsigned i = 1; i < largeBPS->outputAssociation().outSize(); i++) + newJsa.outAdd(largeBPS->outputAssociation().outAt(i)); + + largeBPS->outputAssociation(newJsa); } + + startAdjoiningSteps(); + } + else if (largeBPS) + { + // there are no in-mem UM or PM joins, only disk-joins + startAdjoiningSteps(); + } + else if (!djs) + // if there's no largeBPS, all joins are either done by DJS or join threads, + // this clause starts the THJS join threads. + startJoinThreads(); + + if (fTableOID1 >= 3000) + { + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.end_time = QueryTeleClient::timeNowms(); + sts.total_units_of_work = sts.units_of_work_completed = 1; + postStepSummaryTele(sts); + } } uint32_t TupleHashJoinStep::nextBand(messageqcpp::ByteStream& bs) { - RGData oneRG; - bool more; - uint32_t ret = 0; - RowGroupDL* dl; - uint64_t it; + RGData oneRG; + bool more; + uint32_t ret = 0; + RowGroupDL* dl; + uint64_t it; - idbassert(fDelivery); + idbassert(fDelivery); - boost::mutex::scoped_lock lk(deliverMutex); + boost::mutex::scoped_lock lk(deliverMutex); - RowGroup* deliveredRG; + RowGroup* deliveredRG; - if (fe2) - deliveredRG = &fe2Output; - else - deliveredRG = &outputRG; + if (fe2) + deliveredRG = &fe2Output; + else + deliveredRG = &outputRG; - if (largeBPS && !djs) + if (largeBPS && !djs) + { + dl = largeDL; + it = largeIt; + } + else + { + dl = outputDL; + it = outputIt; + } + + while (ret == 0) + { + if (cancelled()) { - dl = largeDL; - it = largeIt; - } - else - { - dl = outputDL; - it = outputIt; - } - - while (ret == 0) - { - if (cancelled()) - { - oneRG.reinit(*deliveredRG, 0); - deliveredRG->setData(&oneRG); - deliveredRG->resetRowGroup(0); - deliveredRG->setStatus(status()); - deliveredRG->serializeRGData(bs); - more = true; - - while (more) - more = dl->next(it, &oneRG); - - joiners.clear(); - rgData.reset(); - for (uint i = 0; i < smallDLs.size(); i++) - { - resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); - atomicops::atomicZero(&memUsedByEachJoin[i]); - } - return 0; - } + oneRG.reinit(*deliveredRG, 0); + deliveredRG->setData(&oneRG); + deliveredRG->resetRowGroup(0); + deliveredRG->setStatus(status()); + deliveredRG->serializeRGData(bs); + more = true; + while (more) more = dl->next(it, &oneRG); - if (!more) - { - joiners.clear(); - tbpsJoiners.clear(); - rgData.reset(); - oneRG.reinit(*deliveredRG, 0); - deliveredRG->setData(&oneRG); - deliveredRG->resetRowGroup(0); - deliveredRG->setStatus(status()); - - if (status() != 0) - cout << " -- returning error status " << deliveredRG->getStatus() << endl; - - deliveredRG->serializeRGData(bs); - for (uint i = 0; i < smallDLs.size(); i++) - { - resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); - atomicops::atomicZero(&memUsedByEachJoin[i]); - } - return 0; - } - - deliveredRG->setData(&oneRG); - ret = deliveredRG->getRowCount(); + joiners.clear(); + rgData.reset(); + for (uint i = 0; i < smallDLs.size(); i++) + { + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + atomicops::atomicZero(&memUsedByEachJoin[i]); + } + return 0; } - deliveredRG->serializeRGData(bs); - return ret; + more = dl->next(it, &oneRG); + + if (!more) + { + joiners.clear(); + tbpsJoiners.clear(); + rgData.reset(); + oneRG.reinit(*deliveredRG, 0); + deliveredRG->setData(&oneRG); + deliveredRG->resetRowGroup(0); + deliveredRG->setStatus(status()); + + if (status() != 0) + cout << " -- returning error status " << deliveredRG->getStatus() << endl; + + deliveredRG->serializeRGData(bs); + for (uint i = 0; i < smallDLs.size(); i++) + { + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + atomicops::atomicZero(&memUsedByEachJoin[i]); + } + return 0; + } + + deliveredRG->setData(&oneRG); + ret = deliveredRG->getRowCount(); + } + + deliveredRG->serializeRGData(bs); + return ret; } void TupleHashJoinStep::setLargeSideBPS(BatchPrimitive* b) { - largeBPS = dynamic_cast(b); + largeBPS = dynamic_cast(b); } void TupleHashJoinStep::startAdjoiningSteps() { - if (largeBPS) - largeBPS->run(); + if (largeBPS) + largeBPS->run(); } /* TODO: update toString() with the multiple table join info */ const string TupleHashJoinStep::toString() const { - ostringstream oss; - size_t idlsz = fInputJobStepAssociation.outSize(); - idbassert(idlsz > 1); - oss << "TupleHashJoinStep ses:" << fSessionId << " st:" << fStepId; - oss << omitOidInDL; + ostringstream oss; + size_t idlsz = fInputJobStepAssociation.outSize(); + idbassert(idlsz > 1); + oss << "TupleHashJoinStep ses:" << fSessionId << " st:" << fStepId; + oss << omitOidInDL; - for (size_t i = 0; i < idlsz; ++i) - { - RowGroupDL* idl = fInputJobStepAssociation.outAt(i)->rowGroupDL(); - CalpontSystemCatalog::OID oidi = 0; + for (size_t i = 0; i < idlsz; ++i) + { + RowGroupDL* idl = fInputJobStepAssociation.outAt(i)->rowGroupDL(); + CalpontSystemCatalog::OID oidi = 0; - if (idl) oidi = idl->OID(); + if (idl) + oidi = idl->OID(); - oss << " in "; + oss << " in "; - if (largeSideIndex == i) - oss << "*"; + if (largeSideIndex == i) + oss << "*"; - oss << "tb/col:" << fTableOID1 << "/" << oidi; - oss << " " << fInputJobStepAssociation.outAt(i); - } + oss << "tb/col:" << fTableOID1 << "/" << oidi; + oss << " " << fInputJobStepAssociation.outAt(i); + } - idlsz = fOutputJobStepAssociation.outSize(); + idlsz = fOutputJobStepAssociation.outSize(); - if (idlsz > 0) - { - oss << endl << " "; - RowGroupDL* dlo = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - CalpontSystemCatalog::OID oido = 0; + if (idlsz > 0) + { + oss << endl << " "; + RowGroupDL* dlo = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + CalpontSystemCatalog::OID oido = 0; - if (dlo) oido = dlo->OID(); + if (dlo) + oido = dlo->OID(); - oss << " out tb/col:" << fTableOID1 << "/" << oido; - oss << " " << fOutputJobStepAssociation.outAt(0); - } + oss << " out tb/col:" << fTableOID1 << "/" << oido; + oss << " " << fOutputJobStepAssociation.outAt(0); + } - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } //------------------------------------------------------------------------------ @@ -1137,795 +1138,787 @@ const string TupleHashJoinStep::toString() const //------------------------------------------------------------------------------ void TupleHashJoinStep::errorLogging(const string& msg, int err) const { - ostringstream errMsg; - errMsg << "Step " << stepId() << "; " << msg; - cerr << errMsg.str() << endl; - SErrorInfo errorInfo(new ErrorInfo); // dummy, error info already set by caller. - catchHandler(msg, err, errorInfo, fSessionId); + ostringstream errMsg; + errMsg << "Step " << stepId() << "; " << msg; + cerr << errMsg.str() << endl; + SErrorInfo errorInfo(new ErrorInfo); // dummy, error info already set by caller. + catchHandler(msg, err, errorInfo, fSessionId); } -void TupleHashJoinStep::addSmallSideRG(const vector& rgs, - const vector& tnames) +void TupleHashJoinStep::addSmallSideRG(const vector& rgs, const vector& tnames) { - smallTableNames.insert(smallTableNames.end(), tnames.begin(), tnames.end()); - smallRGs.insert(smallRGs.end(), rgs.begin(), rgs.end()); + smallTableNames.insert(smallTableNames.end(), tnames.begin(), tnames.end()); + smallRGs.insert(smallRGs.end(), rgs.begin(), rgs.end()); } -void TupleHashJoinStep::addJoinKeyIndex(const vector& jt, - const vector& typeless, +void TupleHashJoinStep::addJoinKeyIndex(const vector& jt, const vector& typeless, const vector >& smallkey, const vector >& largekey) { - joinTypes.insert(joinTypes.end(), jt.begin(), jt.end()); - typelessJoin.insert(typelessJoin.end(), typeless.begin(), typeless.end()); - smallSideKeys.insert(smallSideKeys.end(), smallkey.begin(), smallkey.end()); - largeSideKeys.insert(largeSideKeys.end(), largekey.begin(), largekey.end()); + joinTypes.insert(joinTypes.end(), jt.begin(), jt.end()); + typelessJoin.insert(typelessJoin.end(), typeless.begin(), typeless.end()); + smallSideKeys.insert(smallSideKeys.end(), smallkey.begin(), smallkey.end()); + largeSideKeys.insert(largeSideKeys.end(), largekey.begin(), largekey.end()); #ifdef JLF_DEBUG - for (uint32_t i = 0; i < joinTypes.size(); i++) - cout << "jointype[" << i << "] = 0x" << hex << joinTypes[i] << dec << endl; + for (uint32_t i = 0; i < joinTypes.size(); i++) + cout << "jointype[" << i << "] = 0x" << hex << joinTypes[i] << dec << endl; #endif } void TupleHashJoinStep::configSmallSideRG(const vector& rgs, const vector& tnames) { - smallTableNames.insert(smallTableNames.begin(), tnames.begin(), tnames.end()); - smallRGs.insert(smallRGs.begin(), rgs.begin(), rgs.end()); + smallTableNames.insert(smallTableNames.begin(), tnames.begin(), tnames.end()); + smallRGs.insert(smallRGs.begin(), rgs.begin(), rgs.end()); } void TupleHashJoinStep::configLargeSideRG(const RowGroup& rg) { - largeRG = rg; + largeRG = rg; } -void TupleHashJoinStep::configJoinKeyIndex(const vector& jt, - const vector& typeless, - const vector >& smallkey, - const vector >& largekey) +void TupleHashJoinStep::configJoinKeyIndex(const vector& jt, const vector& typeless, + const vector >& smallkey, + const vector >& largekey) { - joinTypes.insert(joinTypes.begin(), jt.begin(), jt.end()); - typelessJoin.insert(typelessJoin.begin(), typeless.begin(), typeless.end()); - smallSideKeys.insert(smallSideKeys.begin(), smallkey.begin(), smallkey.end()); - largeSideKeys.insert(largeSideKeys.begin(), largekey.begin(), largekey.end()); + joinTypes.insert(joinTypes.begin(), jt.begin(), jt.end()); + typelessJoin.insert(typelessJoin.begin(), typeless.begin(), typeless.end()); + smallSideKeys.insert(smallSideKeys.begin(), smallkey.begin(), smallkey.end()); + largeSideKeys.insert(largeSideKeys.begin(), largekey.begin(), largekey.end()); #ifdef JLF_DEBUG - for (uint32_t i = 0; i < joinTypes.size(); i++) - cout << "jointype[" << i << "] = 0x" << hex << joinTypes[i] << dec << endl; + for (uint32_t i = 0; i < joinTypes.size(); i++) + cout << "jointype[" << i << "] = 0x" << hex << joinTypes[i] << dec << endl; #endif } void TupleHashJoinStep::setOutputRowGroup(const RowGroup& rg) { - outputRG = rg; + outputRG = rg; } execplan::CalpontSystemCatalog::OID TupleHashJoinStep::smallSideKeyOID(uint32_t s, uint32_t k) const { - return smallRGs[s].getOIDs()[smallSideKeys[s][k]]; + return smallRGs[s].getOIDs()[smallSideKeys[s][k]]; } execplan::CalpontSystemCatalog::OID TupleHashJoinStep::largeSideKeyOID(uint32_t s, uint32_t k) const { - return largeRG.getOIDs()[largeSideKeys[s][k]]; + return largeRG.getOIDs()[largeSideKeys[s][k]]; } void TupleHashJoinStep::addFcnExpGroup2(const boost::shared_ptr& fe) { - if (!fe2) - fe2.reset(new funcexp::FuncExpWrapper()); + if (!fe2) + fe2.reset(new funcexp::FuncExpWrapper()); - fe2->addFilter(fe); + fe2->addFilter(fe); } void TupleHashJoinStep::setFcnExpGroup3(const vector >& v) { - if (!fe2) - fe2.reset(new funcexp::FuncExpWrapper()); + if (!fe2) + fe2.reset(new funcexp::FuncExpWrapper()); - for (uint32_t i = 0; i < v.size(); i++) - fe2->addReturnedColumn(v[i]); + for (uint32_t i = 0; i < v.size(); i++) + fe2->addReturnedColumn(v[i]); } void TupleHashJoinStep::setFE23Output(const rowgroup::RowGroup& rg) { - fe2Output = rg; + fe2Output = rg; } const rowgroup::RowGroup& TupleHashJoinStep::getDeliveredRowGroup() const { - if (fe2) - return fe2Output; + if (fe2) + return fe2Output; - return outputRG; + return outputRG; } void TupleHashJoinStep::deliverStringTableRowGroup(bool b) { - if (fe2) - fe2Output.setUseStringTable(b); + if (fe2) + fe2Output.setUseStringTable(b); - outputRG.setUseStringTable(b); + outputRG.setUseStringTable(b); } bool TupleHashJoinStep::deliverStringTableRowGroup() const { - if (fe2) - return fe2Output.usesStringTable(); + if (fe2) + return fe2Output.usesStringTable(); - return outputRG.usesStringTable(); + return outputRG.usesStringTable(); } -//Must hold the stats lock when calling this! +// Must hold the stats lock when calling this! void TupleHashJoinStep::formatMiniStats(uint32_t index) { - ostringstream oss; - oss << "HJS "; + ostringstream oss; + oss << "HJS "; - if (joiners[index]->inUM()) - oss << "UM "; - else - oss << "PM "; + if (joiners[index]->inUM()) + oss << "UM "; + else + oss << "PM "; - oss << alias() << "-" << joiners[index]->getTableName() << " "; + oss << alias() << "-" << joiners[index]->getTableName() << " "; - if (fTableOID2 >= 3000) - oss << fTableOID2; - else - oss << "- "; + if (fTableOID2 >= 3000) + oss << fTableOID2; + else + oss << "- "; - oss << " " - << "- " - << "- " - << "- " - << "- " -// << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " -// dlTimes are not timed in this step, using '--------' instead. - << "-------- " - << "-\n"; - fMiniInfo += oss.str(); + oss << " " + << "- " + << "- " + << "- " + << "- " + // << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + // dlTimes are not timed in this step, using '--------' instead. + << "-------- " + << "-\n"; + fMiniInfo += oss.str(); } void TupleHashJoinStep::addJoinFilter(boost::shared_ptr pt, uint32_t index) { - boost::shared_ptr newfe(new funcexp::FuncExpWrapper()); + boost::shared_ptr newfe(new funcexp::FuncExpWrapper()); - newfe->addFilter(pt); - fe.push_back(newfe); - feIndexes.push_back(index); + newfe->addFilter(pt); + fe.push_back(newfe); + feIndexes.push_back(index); } bool TupleHashJoinStep::hasJoinFilter(uint32_t index) const { - for (uint32_t i = 0; i < feIndexes.size(); i++) - if (feIndexes[i] == static_cast(index)) - return true; + for (uint32_t i = 0; i < feIndexes.size(); i++) + if (feIndexes[i] == static_cast(index)) + return true; - return false; + return false; } boost::shared_ptr TupleHashJoinStep::getJoinFilter(uint32_t index) const { - for (uint32_t i = 0; i < feIndexes.size(); i++) - if (feIndexes[i] == static_cast(index)) - return fe[i]; + for (uint32_t i = 0; i < feIndexes.size(); i++) + if (feIndexes[i] == static_cast(index)) + return fe[i]; - return boost::shared_ptr(); + return boost::shared_ptr(); } void TupleHashJoinStep::setJoinFilterInputRG(const rowgroup::RowGroup& rg) { - joinFilterRG = rg; + joinFilterRG = rg; } void TupleHashJoinStep::startJoinThreads() { - uint32_t i; - uint32_t smallSideCount = smallDLs.size(); - bool more = true; - RGData oneRG; + uint32_t i; + uint32_t smallSideCount = smallDLs.size(); + bool more = true; + RGData oneRG; - if (joinRunners.size() > 0) - return; - - //@bug4836, in error case, stop process, and unblock the next step. - if (cancelled()) - { - outputDL->endOfInput(); - - //@bug5785, memory leak on canceling complex queries - while (more) - more = largeDL->next(largeIt, &oneRG); - - return; - } - - /* Init class-scope vars. - * - * Get a list of small RGs consistent with the joiners. - * Generate small & large mappings for joinFERG and outputRG. - * If fDelivery, create outputDL. - */ - for (i = 0; i < smallSideCount; i++) - smallRGs[i] = joiners[i]->getSmallRG(); - - columnMappings.reset(new shared_array[smallSideCount + 1]); - - for (i = 0; i < smallSideCount; i++) - columnMappings[i] = makeMapping(smallRGs[i], outputRG); - - columnMappings[smallSideCount] = makeMapping(largeRG, outputRG); - - if (!feIndexes.empty()) - { - fergMappings.reset(new shared_array[smallSideCount + 1]); - - for (i = 0; i < smallSideCount; i++) - fergMappings[i] = makeMapping(smallRGs[i], joinFilterRG); - - fergMappings[smallSideCount] = makeMapping(largeRG, joinFilterRG); - } - - if (fe2) - fe2Mapping = makeMapping(outputRG, fe2Output); - - smallNullMemory.reset(new scoped_array[smallSideCount]); - - for (i = 0; i < smallSideCount; i++) - { - Row smallRow; - smallRGs[i].initRow(&smallRow, true); - smallNullMemory[i].reset(new uint8_t[smallRow.getSize()]); - smallRow.setData(smallNullMemory[i].get()); - smallRow.initToNull(); - } - - for (i = 0; i < smallSideCount; i++) - joiners[i]->setThreadCount(joinThreadCount); - - makeDupList(fe2 ? fe2Output : outputRG); - - /* Start join runners */ - joinRunners.reserve(joinThreadCount); - - for (i = 0; i < joinThreadCount; i++) - joinRunners.push_back(jobstepThreadPool.invoke(JoinRunner(this, i))); - - /* Join them and call endOfInput */ - jobstepThreadPool.join(joinRunners); - - if (lastSmallOuterJoiner != (uint32_t) - 1) - finishSmallOuterJoin(); + if (joinRunners.size() > 0) + return; + //@bug4836, in error case, stop process, and unblock the next step. + if (cancelled()) + { outputDL->endOfInput(); + + //@bug5785, memory leak on canceling complex queries + while (more) + more = largeDL->next(largeIt, &oneRG); + + return; + } + + /* Init class-scope vars. + * + * Get a list of small RGs consistent with the joiners. + * Generate small & large mappings for joinFERG and outputRG. + * If fDelivery, create outputDL. + */ + for (i = 0; i < smallSideCount; i++) + smallRGs[i] = joiners[i]->getSmallRG(); + + columnMappings.reset(new shared_array[smallSideCount + 1]); + + for (i = 0; i < smallSideCount; i++) + columnMappings[i] = makeMapping(smallRGs[i], outputRG); + + columnMappings[smallSideCount] = makeMapping(largeRG, outputRG); + + if (!feIndexes.empty()) + { + fergMappings.reset(new shared_array[smallSideCount + 1]); + + for (i = 0; i < smallSideCount; i++) + fergMappings[i] = makeMapping(smallRGs[i], joinFilterRG); + + fergMappings[smallSideCount] = makeMapping(largeRG, joinFilterRG); + } + + if (fe2) + fe2Mapping = makeMapping(outputRG, fe2Output); + + smallNullMemory.reset(new scoped_array[smallSideCount]); + + for (i = 0; i < smallSideCount; i++) + { + Row smallRow; + smallRGs[i].initRow(&smallRow, true); + smallNullMemory[i].reset(new uint8_t[smallRow.getSize()]); + smallRow.setData(smallNullMemory[i].get()); + smallRow.initToNull(); + } + + for (i = 0; i < smallSideCount; i++) + joiners[i]->setThreadCount(joinThreadCount); + + makeDupList(fe2 ? fe2Output : outputRG); + + /* Start join runners */ + joinRunners.reserve(joinThreadCount); + + for (i = 0; i < joinThreadCount; i++) + joinRunners.push_back(jobstepThreadPool.invoke(JoinRunner(this, i))); + + /* Join them and call endOfInput */ + jobstepThreadPool.join(joinRunners); + + if (lastSmallOuterJoiner != (uint32_t)-1) + finishSmallOuterJoin(); + + outputDL->endOfInput(); } void TupleHashJoinStep::finishSmallOuterJoin() { - vector unmatched; - uint32_t smallSideCount = smallDLs.size(); - uint32_t i, j, k; - shared_array largeNullMemory; - RGData joinedData; - Row joinedBaseRow, fe2InRow, fe2OutRow; - shared_array smallRowTemplates; - shared_array smallNullRows; - Row largeNullRow; - RowGroup l_outputRG = outputRG; - RowGroup l_fe2Output = fe2Output; + vector unmatched; + uint32_t smallSideCount = smallDLs.size(); + uint32_t i, j, k; + shared_array largeNullMemory; + RGData joinedData; + Row joinedBaseRow, fe2InRow, fe2OutRow; + shared_array smallRowTemplates; + shared_array smallNullRows; + Row largeNullRow; + RowGroup l_outputRG = outputRG; + RowGroup l_fe2Output = fe2Output; - joiners[lastSmallOuterJoiner]->getUnmarkedRows(&unmatched); + joiners[lastSmallOuterJoiner]->getUnmarkedRows(&unmatched); - if (unmatched.empty()) - return; + if (unmatched.empty()) + return; - smallRowTemplates.reset(new Row[smallSideCount]); - smallNullRows.reset(new Row[smallSideCount]); + smallRowTemplates.reset(new Row[smallSideCount]); + smallNullRows.reset(new Row[smallSideCount]); - for (i = 0; i < smallSideCount; i++) + for (i = 0; i < smallSideCount; i++) + { + smallRGs[i].initRow(&smallRowTemplates[i]); + smallRGs[i].initRow(&smallNullRows[i], true); + smallNullRows[i].setData(smallNullMemory[i].get()); + } + + largeRG.initRow(&largeNullRow, true); + largeNullMemory.reset(new uint8_t[largeNullRow.getSize()]); + largeNullRow.setData(largeNullMemory.get()); + largeNullRow.initToNull(); + + joinedData.reinit(l_outputRG); + l_outputRG.setData(&joinedData); + l_outputRG.resetRowGroup(0); + l_outputRG.initRow(&joinedBaseRow); + l_outputRG.getRow(0, &joinedBaseRow); + + if (fe2) + { + l_outputRG.initRow(&fe2InRow); + fe2Output.initRow(&fe2OutRow); + } + + for (j = 0; j < unmatched.size(); j++) + { + smallRowTemplates[lastSmallOuterJoiner].setPointer(unmatched[j]); + + for (k = 0; k < smallSideCount; k++) { - smallRGs[i].initRow(&smallRowTemplates[i]); - smallRGs[i].initRow(&smallNullRows[i], true); - smallNullRows[i].setData(smallNullMemory[i].get()); + if (k == lastSmallOuterJoiner) + applyMapping(columnMappings[lastSmallOuterJoiner], smallRowTemplates[lastSmallOuterJoiner], + &joinedBaseRow); + else + applyMapping(columnMappings[k], smallNullRows[k], &joinedBaseRow); } - largeRG.initRow(&largeNullRow, true); - largeNullMemory.reset(new uint8_t[largeNullRow.getSize()]); - largeNullRow.setData(largeNullMemory.get()); - largeNullRow.initToNull(); + applyMapping(columnMappings[smallSideCount], largeNullRow, &joinedBaseRow); + joinedBaseRow.setRid(0); + joinedBaseRow.nextRow(); + l_outputRG.incRowCount(); - joinedData.reinit(l_outputRG); - l_outputRG.setData(&joinedData); - l_outputRG.resetRowGroup(0); - l_outputRG.initRow(&joinedBaseRow); - l_outputRG.getRow(0, &joinedBaseRow); + if (l_outputRG.getRowCount() == 8192) + { + if (fe2) + { + vector rgDatav; + rgDatav.push_back(joinedData); + processFE2(l_outputRG, l_fe2Output, fe2InRow, fe2OutRow, &rgDatav, fe2.get()); + outputDL->insert(rgDatav[0]); + } + else + { + outputDL->insert(joinedData); + } + joinedData.reinit(l_outputRG); + l_outputRG.setData(&joinedData); + l_outputRG.resetRowGroup(0); + l_outputRG.getRow(0, &joinedBaseRow); + } + } + + if (l_outputRG.getRowCount() > 0) + { if (fe2) { - l_outputRG.initRow(&fe2InRow); - fe2Output.initRow(&fe2OutRow); + vector rgDatav; + rgDatav.push_back(joinedData); + processFE2(l_outputRG, l_fe2Output, fe2InRow, fe2OutRow, &rgDatav, fe2.get()); + outputDL->insert(rgDatav[0]); } - - for (j = 0; j < unmatched.size(); j++) + else { - smallRowTemplates[lastSmallOuterJoiner].setPointer(unmatched[j]); - - for (k = 0; k < smallSideCount; k++) - { - if (k == lastSmallOuterJoiner) - applyMapping(columnMappings[lastSmallOuterJoiner], smallRowTemplates[lastSmallOuterJoiner], &joinedBaseRow); - else - applyMapping(columnMappings[k], smallNullRows[k], &joinedBaseRow); - } - - applyMapping(columnMappings[smallSideCount], largeNullRow, &joinedBaseRow); - joinedBaseRow.setRid(0); - joinedBaseRow.nextRow(); - l_outputRG.incRowCount(); - - if (l_outputRG.getRowCount() == 8192) - { - if (fe2) - { - vector rgDatav; - rgDatav.push_back(joinedData); - processFE2(l_outputRG, l_fe2Output, fe2InRow, fe2OutRow, &rgDatav, fe2.get()); - outputDL->insert(rgDatav[0]); - } - else - { - outputDL->insert(joinedData); - } - - joinedData.reinit(l_outputRG); - l_outputRG.setData(&joinedData); - l_outputRG.resetRowGroup(0); - l_outputRG.getRow(0, &joinedBaseRow); - } - } - - if (l_outputRG.getRowCount() > 0) - { - if (fe2) - { - vector rgDatav; - rgDatav.push_back(joinedData); - processFE2(l_outputRG, l_fe2Output, fe2InRow, fe2OutRow, &rgDatav, fe2.get()); - outputDL->insert(rgDatav[0]); - } - else - { - outputDL->insert(joinedData); - } + outputDL->insert(joinedData); } + } } void TupleHashJoinStep::joinRunnerFcn(uint32_t threadID) { - RowGroup local_inputRG, local_outputRG, local_joinFERG; - uint32_t smallSideCount = smallDLs.size(); - vector inputData, joinedRowData; - bool hasJoinFE = !fe.empty(); - uint32_t i; + RowGroup local_inputRG, local_outputRG, local_joinFERG; + uint32_t smallSideCount = smallDLs.size(); + vector inputData, joinedRowData; + bool hasJoinFE = !fe.empty(); + uint32_t i; - /* thread-local scratch space for join processing */ - shared_array joinFERowData; - Row largeRow, joinFERow, joinedRow, baseRow; - shared_array baseRowData; - vector > joinMatches; - shared_array smallRowTemplates; + /* thread-local scratch space for join processing */ + shared_array joinFERowData; + Row largeRow, joinFERow, joinedRow, baseRow; + shared_array baseRowData; + vector > joinMatches; + shared_array smallRowTemplates; - /* F & E vars */ - FuncExpWrapper local_fe; - RowGroup local_fe2RG; - Row fe2InRow, fe2OutRow; + /* F & E vars */ + FuncExpWrapper local_fe; + RowGroup local_fe2RG; + Row fe2InRow, fe2OutRow; - joinMatches.resize(smallSideCount); - local_inputRG = largeRG; - local_outputRG = outputRG; - local_inputRG.initRow(&largeRow); - local_outputRG.initRow(&joinedRow); - local_outputRG.initRow(&baseRow, true); - baseRowData.reset(new uint8_t[baseRow.getSize()]); - baseRow.setData(baseRowData.get()); + joinMatches.resize(smallSideCount); + local_inputRG = largeRG; + local_outputRG = outputRG; + local_inputRG.initRow(&largeRow); + local_outputRG.initRow(&joinedRow); + local_outputRG.initRow(&baseRow, true); + baseRowData.reset(new uint8_t[baseRow.getSize()]); + baseRow.setData(baseRowData.get()); - if (hasJoinFE) + if (hasJoinFE) + { + local_joinFERG = joinFilterRG; + local_joinFERG.initRow(&joinFERow, true); + joinFERowData.reset(new uint8_t[joinFERow.getSize()]); + joinFERow.setData(joinFERowData.get()); + } + + if (fe2) + { + local_fe2RG = fe2Output; + local_outputRG.initRow(&fe2InRow); + local_fe2RG.initRow(&fe2OutRow); + local_fe = *fe2; + } + + smallRowTemplates.reset(new Row[smallSideCount]); + + for (i = 0; i < smallSideCount; i++) + smallRGs[i].initRow(&smallRowTemplates[i]); + + grabSomeWork(&inputData); + + while (!inputData.empty() && !cancelled()) + { + for (i = 0; i < inputData.size() && !cancelled(); i++) { - local_joinFERG = joinFilterRG; - local_joinFERG.initRow(&joinFERow, true); - joinFERowData.reset(new uint8_t[joinFERow.getSize()]); - joinFERow.setData(joinFERowData.get()); + local_inputRG.setData(&inputData[i]); + + if (local_inputRG.getRowCount() == 0) + continue; + + joinOneRG(threadID, joinedRowData, local_inputRG, local_outputRG, largeRow, joinFERow, joinedRow, + baseRow, joinMatches, smallRowTemplates, outputDL); } if (fe2) - { - local_fe2RG = fe2Output; - local_outputRG.initRow(&fe2InRow); - local_fe2RG.initRow(&fe2OutRow); - local_fe = *fe2; - } - - smallRowTemplates.reset(new Row[smallSideCount]); - - for (i = 0; i < smallSideCount; i++) - smallRGs[i].initRow(&smallRowTemplates[i]); + processFE2(local_outputRG, local_fe2RG, fe2InRow, fe2OutRow, &joinedRowData, &local_fe); + processDupList(threadID, (fe2 ? local_fe2RG : local_outputRG), &joinedRowData); + sendResult(joinedRowData); + returnMemory(); + joinedRowData.clear(); grabSomeWork(&inputData); + } - while (!inputData.empty() && !cancelled()) - { - for (i = 0; i < inputData.size() && !cancelled(); i++) - { - local_inputRG.setData(&inputData[i]); - - if (local_inputRG.getRowCount() == 0) - continue; - - joinOneRG(threadID, joinedRowData, local_inputRG, local_outputRG, largeRow, - joinFERow, joinedRow, baseRow, joinMatches, smallRowTemplates, outputDL); - } - - if (fe2) - processFE2(local_outputRG, local_fe2RG, fe2InRow, fe2OutRow, &joinedRowData, &local_fe); - - processDupList(threadID, (fe2 ? local_fe2RG : local_outputRG), &joinedRowData); - sendResult(joinedRowData); - returnMemory(); - joinedRowData.clear(); - grabSomeWork(&inputData); - } - - while (!inputData.empty()) - grabSomeWork(&inputData); + while (!inputData.empty()) + grabSomeWork(&inputData); } void TupleHashJoinStep::makeDupList(const RowGroup& rg) { - uint32_t i, j, cols = rg.getColumnCount(); + uint32_t i, j, cols = rg.getColumnCount(); - for (i = 0; i < cols; i++) - for (j = i + 1; j < cols; j++) - if (rg.getKeys()[i] == rg.getKeys()[j]) - dupList.push_back(make_pair(j, i)); + for (i = 0; i < cols; i++) + for (j = i + 1; j < cols; j++) + if (rg.getKeys()[i] == rg.getKeys()[j]) + dupList.push_back(make_pair(j, i)); - dupRows.reset(new Row[joinThreadCount]); + dupRows.reset(new Row[joinThreadCount]); - for (i = 0; i < joinThreadCount; i++) - rg.initRow(&dupRows[i]); + for (i = 0; i < joinThreadCount; i++) + rg.initRow(&dupRows[i]); } -void TupleHashJoinStep::processDupList(uint32_t threadID, RowGroup& rg, - vector* rowData) +void TupleHashJoinStep::processDupList(uint32_t threadID, RowGroup& rg, vector* rowData) { - uint32_t i, j, k; + uint32_t i, j, k; - if (dupList.empty()) - return; + if (dupList.empty()) + return; - for (i = 0; i < rowData->size(); i++) - { - rg.setData(&(*rowData)[i]); - rg.getRow(0, &dupRows[threadID]); + for (i = 0; i < rowData->size(); i++) + { + rg.setData(&(*rowData)[i]); + rg.getRow(0, &dupRows[threadID]); - for (j = 0; j < rg.getRowCount(); j++, dupRows[threadID].nextRow()) - for (k = 0; k < dupList.size(); k++) - dupRows[threadID].copyField(dupList[k].first, dupList[k].second); - } + for (j = 0; j < rg.getRowCount(); j++, dupRows[threadID].nextRow()) + for (k = 0; k < dupList.size(); k++) + dupRows[threadID].copyField(dupList[k].first, dupList[k].second); + } } void TupleHashJoinStep::processFE2(RowGroup& input, RowGroup& output, Row& inRow, Row& outRow, vector* rgData, funcexp::FuncExpWrapper* local_fe) { - vector results; - RGData result; - uint32_t i, j; - bool ret; + vector results; + RGData result; + uint32_t i, j; + bool ret; - result.reinit(output); - output.setData(&result); - output.resetRowGroup(0); - output.getRow(0, &outRow); + result.reinit(output); + output.setData(&result); + output.resetRowGroup(0); + output.getRow(0, &outRow); - for (i = 0; i < rgData->size(); i++) + for (i = 0; i < rgData->size(); i++) + { + input.setData(&(*rgData)[i]); + + if (output.getRowCount() == 0) { - input.setData(&(*rgData)[i]); - - if (output.getRowCount() == 0) - { - output.resetRowGroup(input.getBaseRid()); - output.setDBRoot(input.getDBRoot()); - } - - input.getRow(0, &inRow); - - for (j = 0; j < input.getRowCount(); j++, inRow.nextRow()) - { - ret = local_fe->evaluate(&inRow); - - if (ret) - { - applyMapping(fe2Mapping, inRow, &outRow); - output.incRowCount(); - outRow.nextRow(); - - if (output.getRowCount() == 8192) - { - results.push_back(result); - result.reinit(output); - output.setData(&result); - output.resetRowGroup(input.getBaseRid()); - output.setDBRoot(input.getDBRoot()); - output.getRow(0, &outRow); - } - } - } + output.resetRowGroup(input.getBaseRid()); + output.setDBRoot(input.getDBRoot()); } - if (output.getRowCount() > 0) - { - results.push_back(result); - } + input.getRow(0, &inRow); - rgData->swap(results); + for (j = 0; j < input.getRowCount(); j++, inRow.nextRow()) + { + ret = local_fe->evaluate(&inRow); + + if (ret) + { + applyMapping(fe2Mapping, inRow, &outRow); + output.incRowCount(); + outRow.nextRow(); + + if (output.getRowCount() == 8192) + { + results.push_back(result); + result.reinit(output); + output.setData(&result); + output.resetRowGroup(input.getBaseRid()); + output.setDBRoot(input.getDBRoot()); + output.getRow(0, &outRow); + } + } + } + } + + if (output.getRowCount() > 0) + { + results.push_back(result); + } + + rgData->swap(results); } void TupleHashJoinStep::sendResult(const vector& res) { - boost::mutex::scoped_lock lock(outputDLLock); + boost::mutex::scoped_lock lock(outputDLLock); - for (uint32_t i = 0; i < res.size(); i++) - //INSERT_ADAPTER(outputDL, res[i]); - outputDL->insert(res[i]); + for (uint32_t i = 0; i < res.size(); i++) + // INSERT_ADAPTER(outputDL, res[i]); + outputDL->insert(res[i]); } void TupleHashJoinStep::grabSomeWork(vector* work) { - boost::mutex::scoped_lock lock(inputDLLock); - work->clear(); + boost::mutex::scoped_lock lock(inputDLLock); + work->clear(); - if (!moreInput) - return; + if (!moreInput) + return; - RGData e; + RGData e; + moreInput = largeDL->next(largeIt, &e); + + /* Tunable number here, but it probably won't change things much */ + for (uint32_t i = 0; i < 10 && moreInput; i++) + { + work->push_back(e); moreInput = largeDL->next(largeIt, &e); + } - /* Tunable number here, but it probably won't change things much */ - for (uint32_t i = 0; i < 10 && moreInput; i++) - { - work->push_back(e); - moreInput = largeDL->next(largeIt, &e); - } - - if (moreInput) - work->push_back(e); + if (moreInput) + work->push_back(e); } /* This function is a port of the main join loop in TupleBPS::receiveMultiPrimitiveMessages(). Any * changes made here should also be made there and vice versa. */ -void TupleHashJoinStep::joinOneRG(uint32_t threadID, vector& out, - RowGroup& inputRG, RowGroup& joinOutput, Row& largeSideRow, Row& joinFERow, - Row& joinedRow, Row& baseRow, vector >& joinMatches, - shared_array& smallRowTemplates, - RowGroupDL* outputDL, - // disk-join support vars. This param list is insane; refactor attempt would be nice at some point. - vector >* tjoiners, - boost::shared_array >* rgMappings, - boost::shared_array >* feMappings, - boost::scoped_array >* smallNullMem - ) +void TupleHashJoinStep::joinOneRG( + uint32_t threadID, vector& out, RowGroup& inputRG, RowGroup& joinOutput, Row& largeSideRow, + Row& joinFERow, Row& joinedRow, Row& baseRow, vector >& joinMatches, + shared_array& smallRowTemplates, RowGroupDL* outputDL, + // disk-join support vars. This param list is insane; refactor attempt would be nice at some point. + vector >* tjoiners, + boost::shared_array >* rgMappings, + boost::shared_array >* feMappings, + boost::scoped_array >* smallNullMem) { - /* Disk-join support. - These dissociate the fcn from THJS's members & allow this fcn to be called from DiskJoinStep - */ - if (!tjoiners) - tjoiners = &joiners; + /* Disk-join support. + These dissociate the fcn from THJS's members & allow this fcn to be called from DiskJoinStep + */ + if (!tjoiners) + tjoiners = &joiners; - if (!rgMappings) - rgMappings = &columnMappings; + if (!rgMappings) + rgMappings = &columnMappings; - if (!feMappings) - feMappings = &fergMappings; + if (!feMappings) + feMappings = &fergMappings; - if (!smallNullMem) - smallNullMem = &smallNullMemory; + if (!smallNullMem) + smallNullMem = &smallNullMemory; - RGData joinedData; - uint32_t matchCount, smallSideCount = tjoiners->size(); - uint32_t j, k; + RGData joinedData; + uint32_t matchCount, smallSideCount = tjoiners->size(); + uint32_t j, k; - joinedData.reinit(joinOutput); - joinOutput.setData(&joinedData); - joinOutput.resetRowGroup(inputRG.getBaseRid()); - joinOutput.setDBRoot(inputRG.getDBRoot()); - inputRG.getRow(0, &largeSideRow); + joinedData.reinit(joinOutput); + joinOutput.setData(&joinedData); + joinOutput.resetRowGroup(inputRG.getBaseRid()); + joinOutput.setDBRoot(inputRG.getDBRoot()); + inputRG.getRow(0, &largeSideRow); - //cout << "jointype = " << (*tjoiners)[0]->getJoinType() << endl; - for (k = 0; k < inputRG.getRowCount() && !cancelled(); k++, largeSideRow.nextRow()) + // cout << "jointype = " << (*tjoiners)[0]->getJoinType() << endl; + for (k = 0; k < inputRG.getRowCount() && !cancelled(); k++, largeSideRow.nextRow()) + { + // cout << "THJS: Large side row: " << largeSideRow.toString() << endl; + matchCount = 0; + + for (j = 0; j < smallSideCount; j++) { - //cout << "THJS: Large side row: " << largeSideRow.toString() << endl; - matchCount = 0; + (*tjoiners)[j]->match(largeSideRow, k, threadID, &joinMatches[j]); + /* Debugging code to print the matches + Row r; + smallRGs[j].initRow(&r); + cout << joinMatches[j].size() << " matches: \n"; + for (uint32_t z = 0; z < joinMatches[j].size(); z++) { + r.setData(joinMatches[j][z]); + cout << " " << r.toString() << endl; + } + */ + matchCount = joinMatches[j].size(); - for (j = 0; j < smallSideCount; j++) + if ((*tjoiners)[j]->hasFEFilter() && matchCount > 0) + { + // cout << "doing FE filter" << endl; + vector newJoinMatches; + applyMapping((*feMappings)[smallSideCount], largeSideRow, &joinFERow); + + for (uint32_t z = 0; z < joinMatches[j].size(); z++) { - (*tjoiners)[j]->match(largeSideRow, k, threadID, &joinMatches[j]); - /* Debugging code to print the matches - Row r; - smallRGs[j].initRow(&r); - cout << joinMatches[j].size() << " matches: \n"; - for (uint32_t z = 0; z < joinMatches[j].size(); z++) { - r.setData(joinMatches[j][z]); - cout << " " << r.toString() << endl; - } - */ - matchCount = joinMatches[j].size(); + smallRowTemplates[j].setPointer(joinMatches[j][z]); + applyMapping((*feMappings)[j], smallRowTemplates[j], &joinFERow); - if ((*tjoiners)[j]->hasFEFilter() && matchCount > 0) - { - //cout << "doing FE filter" << endl; - vector newJoinMatches; - applyMapping((*feMappings)[smallSideCount], largeSideRow, &joinFERow); - - for (uint32_t z = 0; z < joinMatches[j].size(); z++) - { - smallRowTemplates[j].setPointer(joinMatches[j][z]); - applyMapping((*feMappings)[j], smallRowTemplates[j], &joinFERow); - - if (!(*tjoiners)[j]->evaluateFilter(joinFERow, threadID)) - matchCount--; - else - { - /* The first match includes it in a SEMI join result and excludes it from an ANTI join - * result. If it's SEMI & SCALAR however, it needs to continue. - */ - newJoinMatches.push_back(joinMatches[j][z]); - - if ((*tjoiners)[j]->antiJoin() || ((*tjoiners)[j]->semiJoin() && !(*tjoiners)[j]->scalar())) - break; - } - } - - // the filter eliminated all matches, need to join with the NULL row - if (matchCount == 0 && (*tjoiners)[j]->largeOuterJoin()) - { - newJoinMatches.clear(); - newJoinMatches.push_back(Row::Pointer((*smallNullMem)[j].get())); - matchCount = 1; - } - - joinMatches[j].swap(newJoinMatches); - } - - /* If anti-join, reverse the result */ - if ((*tjoiners)[j]->antiJoin()) - { - matchCount = (matchCount ? 0 : 1); - } - - if (matchCount == 0) - { - joinMatches[j].clear(); - break; - } - else if (!(*tjoiners)[j]->scalar() && ((*tjoiners)[j]->semiJoin() || (*tjoiners)[j]->antiJoin())) - { - joinMatches[j].clear(); - joinMatches[j].push_back(Row::Pointer((*smallNullMem)[j].get())); - matchCount = 1; - } - - if (matchCount == 0 && (*tjoiners)[j]->innerJoin()) - break; - - if ((*tjoiners)[j]->scalar() && matchCount > 1) - { - errorMessage(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_MORE_THAN_1_ROW)); - status(logging::ERR_MORE_THAN_1_ROW); - abort(); - } - - if ((*tjoiners)[j]->smallOuterJoin()) - (*tjoiners)[j]->markMatches(threadID, joinMatches[j]); + if (!(*tjoiners)[j]->evaluateFilter(joinFERow, threadID)) + matchCount--; + else + { + /* The first match includes it in a SEMI join result and excludes it from an ANTI join + * result. If it's SEMI & SCALAR however, it needs to continue. + */ + newJoinMatches.push_back(joinMatches[j][z]); + if ((*tjoiners)[j]->antiJoin() || ((*tjoiners)[j]->semiJoin() && !(*tjoiners)[j]->scalar())) + break; + } } - if (matchCount > 0) + // the filter eliminated all matches, need to join with the NULL row + if (matchCount == 0 && (*tjoiners)[j]->largeOuterJoin()) { - /* TODO!!! See TupleBPS for the fix for bug 3510! */ - applyMapping((*rgMappings)[smallSideCount], largeSideRow, &baseRow); - baseRow.setRid(largeSideRow.getRelRid()); - generateJoinResultSet(joinMatches, baseRow, *rgMappings, - 0, joinOutput, joinedData, out, smallRowTemplates, joinedRow, outputDL); + newJoinMatches.clear(); + newJoinMatches.push_back(Row::Pointer((*smallNullMem)[j].get())); + matchCount = 1; } + + joinMatches[j].swap(newJoinMatches); + } + + /* If anti-join, reverse the result */ + if ((*tjoiners)[j]->antiJoin()) + { + matchCount = (matchCount ? 0 : 1); + } + + if (matchCount == 0) + { + joinMatches[j].clear(); + break; + } + else if (!(*tjoiners)[j]->scalar() && ((*tjoiners)[j]->semiJoin() || (*tjoiners)[j]->antiJoin())) + { + joinMatches[j].clear(); + joinMatches[j].push_back(Row::Pointer((*smallNullMem)[j].get())); + matchCount = 1; + } + + if (matchCount == 0 && (*tjoiners)[j]->innerJoin()) + break; + + if ((*tjoiners)[j]->scalar() && matchCount > 1) + { + errorMessage(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_MORE_THAN_1_ROW)); + status(logging::ERR_MORE_THAN_1_ROW); + abort(); + } + + if ((*tjoiners)[j]->smallOuterJoin()) + (*tjoiners)[j]->markMatches(threadID, joinMatches[j]); } - if (joinOutput.getRowCount() > 0) - out.push_back(joinedData); + if (matchCount > 0) + { + /* TODO!!! See TupleBPS for the fix for bug 3510! */ + applyMapping((*rgMappings)[smallSideCount], largeSideRow, &baseRow); + baseRow.setRid(largeSideRow.getRelRid()); + generateJoinResultSet(joinMatches, baseRow, *rgMappings, 0, joinOutput, joinedData, out, + smallRowTemplates, joinedRow, outputDL); + } + } + + if (joinOutput.getRowCount() > 0) + out.push_back(joinedData); } -void TupleHashJoinStep::generateJoinResultSet(const vector >& joinerOutput, - Row& baseRow, const shared_array >& mappings, - const uint32_t depth, RowGroup& l_outputRG, RGData& rgData, - vector& outputData, const shared_array& smallRows, - Row& joinedRow, RowGroupDL* dlp) +void TupleHashJoinStep::generateJoinResultSet(const vector >& joinerOutput, Row& baseRow, + const shared_array >& mappings, + const uint32_t depth, RowGroup& l_outputRG, RGData& rgData, + vector& outputData, const shared_array& smallRows, + Row& joinedRow, RowGroupDL* dlp) { - uint32_t i; - Row& smallRow = smallRows[depth]; - uint32_t smallSideCount = joinerOutput.size(); + uint32_t i; + Row& smallRow = smallRows[depth]; + uint32_t smallSideCount = joinerOutput.size(); - if (depth < smallSideCount - 1) + if (depth < smallSideCount - 1) + { + for (i = 0; i < joinerOutput[depth].size(); i++) { - for (i = 0; i < joinerOutput[depth].size(); i++) - { - smallRow.setPointer(joinerOutput[depth][i]); - applyMapping(mappings[depth], smallRow, &baseRow); - generateJoinResultSet(joinerOutput, baseRow, mappings, depth + 1, - l_outputRG, rgData, outputData, smallRows, joinedRow, dlp); - } + smallRow.setPointer(joinerOutput[depth][i]); + applyMapping(mappings[depth], smallRow, &baseRow); + generateJoinResultSet(joinerOutput, baseRow, mappings, depth + 1, l_outputRG, rgData, outputData, + smallRows, joinedRow, dlp); } - else + } + else + { + l_outputRG.getRow(l_outputRG.getRowCount(), &joinedRow); + + for (i = 0; i < joinerOutput[depth].size(); i++, joinedRow.nextRow(), l_outputRG.incRowCount()) { - l_outputRG.getRow(l_outputRG.getRowCount(), &joinedRow); + smallRow.setPointer(joinerOutput[depth][i]); - for (i = 0; i < joinerOutput[depth].size(); i++, joinedRow.nextRow(), - l_outputRG.incRowCount()) + if (UNLIKELY(l_outputRG.getRowCount() == 8192)) + { + uint32_t dbRoot = l_outputRG.getDBRoot(); + uint64_t baseRid = l_outputRG.getBaseRid(); + outputData.push_back(rgData); + // Count the memory + if (UNLIKELY(!getMemory(l_outputRG.getMaxDataSize()))) { - smallRow.setPointer(joinerOutput[depth][i]); - - if (UNLIKELY(l_outputRG.getRowCount() == 8192)) - { - uint32_t dbRoot = l_outputRG.getDBRoot(); - uint64_t baseRid = l_outputRG.getBaseRid(); - outputData.push_back(rgData); - // Count the memory - if (UNLIKELY(!getMemory(l_outputRG.getMaxDataSize()))) - { - // Don't let the join results buffer get out of control. - sendResult(outputData); - outputData.clear(); - returnMemory(); - } - rgData.reinit(l_outputRG); - l_outputRG.setData(&rgData); - l_outputRG.resetRowGroup(baseRid); - l_outputRG.setDBRoot(dbRoot); - l_outputRG.getRow(0, &joinedRow); - } - - applyMapping(mappings[depth], smallRow, &baseRow); - copyRow(baseRow, &joinedRow); + // Don't let the join results buffer get out of control. + sendResult(outputData); + outputData.clear(); + returnMemory(); } + rgData.reinit(l_outputRG); + l_outputRG.setData(&rgData); + l_outputRG.resetRowGroup(baseRid); + l_outputRG.setDBRoot(dbRoot); + l_outputRG.getRow(0, &joinedRow); + } + + applyMapping(mappings[depth], smallRow, &baseRow); + copyRow(baseRow, &joinedRow); } + } } void TupleHashJoinStep::segregateJoiners() { - uint32_t i; - bool allInnerJoins = true; - bool anyTooLarge = false; - uint32_t smallSideCount = smallDLs.size(); + uint32_t i; + bool allInnerJoins = true; + bool anyTooLarge = false; + uint32_t smallSideCount = smallDLs.size(); - for (i = 0; i < smallSideCount; i++) + for (i = 0; i < smallSideCount; i++) + { + allInnerJoins &= (joinTypes[i] == INNER); + anyTooLarge |= !joiners[i]->isFinished(); + } + + /* When DDL updates syscat, the syscat checks here are necessary */ + if (isDML || !allowDJS || (fSessionId & 0x80000000) || (tableOid() < 3000 && tableOid() >= 1000)) + { + if (anyTooLarge) { - allInnerJoins &= (joinTypes[i] == INNER); - anyTooLarge |= !joiners[i]->isFinished(); + joinIsTooBig = true; + abort(); } - /* When DDL updates syscat, the syscat checks here are necessary */ - if (isDML || !allowDJS || (fSessionId & 0x80000000) || - (tableOid() < 3000 && tableOid() >= 1000)) - { - if (anyTooLarge) - { - joinIsTooBig = true; - abort(); - } + tbpsJoiners = joiners; + return; + } - tbpsJoiners = joiners; - return; - } - - #if 0 +#if 0 // Debugging code, this makes all eligible joins disk-based. else { cout << "making all joins disk-based" << endl; @@ -1937,91 +1930,91 @@ void TupleHashJoinStep::segregateJoiners() } return; } - #endif +#endif - boost::mutex::scoped_lock sl(djsLock); - /* For now if there is no largeBPS all joins need to either be DJS or not, not mixed */ - if (!largeBPS) + boost::mutex::scoped_lock sl(djsLock); + /* For now if there is no largeBPS all joins need to either be DJS or not, not mixed */ + if (!largeBPS) + { + if (anyTooLarge) { - if (anyTooLarge) - { - joinIsTooBig = true; + joinIsTooBig = true; - for (i = 0; i < smallSideCount; i++) - { - joiners[i]->setConvertToDiskJoin(); - djsJoiners.push_back(joiners[i]); - djsJoinerMap.push_back(i); - } - } - else - tbpsJoiners = joiners; - - return; - } - - /* If they are all inner joins they can be segregated w/o respect to - ordering; if they're not, the ordering has to stay consistent therefore - the first joiner that isn't finished and everything after has to be - done by DJS. */ - - if (allInnerJoins) - { - for (i = 0; i < smallSideCount; i++) - { - //if (joiners[i]->isFinished() && (rand() % 2)) { // for debugging - if (joiners[i]->isFinished()) - { - //cout << "1joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> TBPS" << endl; - tbpsJoiners.push_back(joiners[i]); - } - else - { - joinIsTooBig = true; - joiners[i]->setConvertToDiskJoin(); - //cout << "1joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> DJS" << endl; - djsJoiners.push_back(joiners[i]); - djsJoinerMap.push_back(i); - } - } + for (i = 0; i < smallSideCount; i++) + { + joiners[i]->setConvertToDiskJoin(); + djsJoiners.push_back(joiners[i]); + djsJoinerMap.push_back(i); + } } else - { - //uint limit = rand() % smallSideCount; - for (i = 0; i < smallSideCount; i++) - { - //if (joiners[i]->isFinished() && i < limit) { // debugging - if (joiners[i]->isFinished()) - { - //cout << "2joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> TBPS" << endl; - tbpsJoiners.push_back(joiners[i]); - } - else - break; - } + tbpsJoiners = joiners; - for (; i < smallSideCount; i++) - { - joinIsTooBig = true; - joiners[i]->setConvertToDiskJoin(); - //cout << "2joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> DJS" << endl; - djsJoiners.push_back(joiners[i]); - djsJoinerMap.push_back(i); - } + return; + } + + /* If they are all inner joins they can be segregated w/o respect to + ordering; if they're not, the ordering has to stay consistent therefore + the first joiner that isn't finished and everything after has to be + done by DJS. */ + + if (allInnerJoins) + { + for (i = 0; i < smallSideCount; i++) + { + // if (joiners[i]->isFinished() && (rand() % 2)) { // for debugging + if (joiners[i]->isFinished()) + { + // cout << "1joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> TBPS" << endl; + tbpsJoiners.push_back(joiners[i]); + } + else + { + joinIsTooBig = true; + joiners[i]->setConvertToDiskJoin(); + // cout << "1joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> DJS" << endl; + djsJoiners.push_back(joiners[i]); + djsJoinerMap.push_back(i); + } } + } + else + { + // uint limit = rand() % smallSideCount; + for (i = 0; i < smallSideCount; i++) + { + // if (joiners[i]->isFinished() && i < limit) { // debugging + if (joiners[i]->isFinished()) + { + // cout << "2joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> TBPS" << endl; + tbpsJoiners.push_back(joiners[i]); + } + else + break; + } + + for (; i < smallSideCount; i++) + { + joinIsTooBig = true; + joiners[i]->setConvertToDiskJoin(); + // cout << "2joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> DJS" << endl; + djsJoiners.push_back(joiners[i]); + djsJoinerMap.push_back(i); + } + } } void TupleHashJoinStep::abort() { - JobStep::abort(); - boost::mutex::scoped_lock sl(djsLock); + JobStep::abort(); + boost::mutex::scoped_lock sl(djsLock); - if (djs) - { - for (uint32_t i = 0; i < djsJoiners.size(); i++) - djs[i].abort(); - } + if (djs) + { + for (uint32_t i = 0; i < djsJoiners.size(); i++) + djs[i].abort(); + } } -} +} // namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehashjoin.h b/dbcon/joblist/tuplehashjoin.h index d6107138f..8531cfd75 100644 --- a/dbcon/joblist/tuplehashjoin.h +++ b/dbcon/joblist/tuplehashjoin.h @@ -18,7 +18,6 @@ // $Id: tuplehashjoin.h 9655 2013-06-25 23:08:13Z xlou $ - #ifndef TUPLEHASHJOIN_H_ #define TUPLEHASHJOIN_H_ @@ -44,611 +43,615 @@ class DiskJoinStep; class TupleHashJoinStep : public JobStep, public TupleDeliveryStep { -public: - /** - * @param - */ - TupleHashJoinStep(const JobInfo& jobInfo); - virtual ~TupleHashJoinStep(); + public: + /** + * @param + */ + TupleHashJoinStep(const JobInfo& jobInfo); + virtual ~TupleHashJoinStep(); - void setLargeSideBPS(BatchPrimitive*); - void setLargeSideStepsOut(const std::vector& largeSideSteps); - void setSmallSideStepsOut(const std::vector& smallSideSteps); + void setLargeSideBPS(BatchPrimitive*); + void setLargeSideStepsOut(const std::vector& largeSideSteps); + void setSmallSideStepsOut(const std::vector& smallSideSteps); - /* mandatory JobStep interface */ - void run(); - void join(); - const std::string toString() const; + /* mandatory JobStep interface */ + void run(); + void join(); + const std::string toString() const; - /* These tableOID accessors can go away soon */ - execplan::CalpontSystemCatalog::OID tableOid() const + /* These tableOID accessors can go away soon */ + execplan::CalpontSystemCatalog::OID tableOid() const + { + return fTableOID2; + } + execplan::CalpontSystemCatalog::OID tableOid1() const + { + return fTableOID1; + } + execplan::CalpontSystemCatalog::OID tableOid2() const + { + return fTableOID2; + } + void tableOid1(execplan::CalpontSystemCatalog::OID tableOid1) + { + fTableOID1 = tableOid1; + if (fTableOID1 < 3000) + numCores = 1; // syscat query, no need for more than 1 thread + } + void tableOid2(execplan::CalpontSystemCatalog::OID tableOid2) + { + fTableOID2 = tableOid2; + } + + std::string alias1() const + { + return fAlias1; + } + void alias1(const std::string& alias) + { + fAlias1 = alias; + } + std::string alias2() const + { + return fAlias2; + } + void alias2(const std::string& alias) + { + fAlias = fAlias2 = alias; + } + + std::string view1() const + { + return fView1; + } + void view1(const std::string& vw) + { + fView1 = vw; + } + std::string view2() const + { + return fView2; + } + void view2(const std::string& vw) + { + fView = fView2 = vw; + } + + std::string schema1() const + { + return fSchema1; + } + void schema1(const std::string& s) + { + fSchema1 = s; + } + std::string schema2() const + { + return fSchema2; + } + void schema2(const std::string& s) + { + fSchema = fSchema2 = s; + } + + int32_t sequence1() const + { + return fSequence1; + } + void sequence1(int32_t seq) + { + fSequence1 = seq; + } + int32_t sequence2() const + { + return fSequence2; + } + void sequence2(int32_t seq) + { + fSequence2 = seq; + } + + const execplan::ReturnedColumn* column1() const + { + return fColumn1; + } + void column1(const execplan::ReturnedColumn* pos) + { + fColumn1 = pos; + } + const execplan::ReturnedColumn* column2() const + { + return fColumn2; + } + void column2(const execplan::ReturnedColumn* pos) + { + fColumn2 = pos; + } + + int correlatedSide() const + { + return fCorrelatedSide; + } + void correlatedSide(int c) + { + fCorrelatedSide = c; + } + using JobStep::tupleId; + uint64_t tupleId() const + { + return fTupleId2; + } + uint64_t tupleId1() const + { + return fTupleId1; + } + uint64_t tupleId2() const + { + return fTupleId2; + } + void tupleId1(uint64_t id) + { + fTupleId1 = id; + } + void tupleId2(uint64_t id) + { + fTupleId2 = id; + } + + void addSmallSideRG(const std::vector& rgs, const std::vector& tableNames); + void addJoinKeyIndex(const std::vector& jt, const std::vector& typeless, + const std::vector >& smallkeys, + const std::vector >& largekeys); + + void configSmallSideRG(const std::vector& rgs, + const std::vector& tableNames); + void configLargeSideRG(const rowgroup::RowGroup& rg); + + void configJoinKeyIndex(const std::vector& jt, const std::vector& typeless, + const std::vector >& smallkeys, + const std::vector >& largekeys); + + void setOutputRowGroup(const rowgroup::RowGroup& rg); + + uint32_t nextBand(messageqcpp::ByteStream& bs); + + const rowgroup::RowGroup& getOutputRowGroup() const + { + return outputRG; + } + const rowgroup::RowGroup& getSmallRowGroup() const + { + return smallRGs[0]; + } + const std::vector& getSmallRowGroups() const + { + return smallRGs; + } + const rowgroup::RowGroup& getLargeRowGroup() const + { + return largeRG; + } + uint32_t getSmallKey() const + { + return smallSideKeys[0][0]; + } + const std::vector >& getSmallKeys() const + { + return smallSideKeys; + } + const std::vector >& getLargeKeys() const + { + return largeSideKeys; + } + + /* Some compat fcns to get rid of later */ + void oid1(execplan::CalpontSystemCatalog::OID oid) + { + fOid1 = oid; + } + execplan::CalpontSystemCatalog::OID oid1() const + { + return fOid1; + } + void oid2(execplan::CalpontSystemCatalog::OID oid) + { + fOid2 = oid; + } + execplan::CalpontSystemCatalog::OID oid2() const + { + return fOid2; + } + void dictOid1(execplan::CalpontSystemCatalog::OID oid) + { + fDictOid1 = oid; + } + execplan::CalpontSystemCatalog::OID dictOid1() const + { + return fDictOid1; + } + void dictOid2(execplan::CalpontSystemCatalog::OID oid) + { + fDictOid2 = oid; + } + execplan::CalpontSystemCatalog::OID dictOid2() const + { + return fDictOid2; + } + + /* The replacements. I don't think there's a need for setters or vars. + OIDs are already in the rowgroups. */ + // s - sth table pair; k - kth key in compound join, 0 for non-compand join + execplan::CalpontSystemCatalog::OID smallSideKeyOID(uint32_t s, uint32_t k) const; + execplan::CalpontSystemCatalog::OID largeSideKeyOID(uint32_t s, uint32_t k) const; + + void deliveryStep(const SJSTEP& ds) + { + fDeliveryStep = ds; + } + + /* Iteration 18 mods */ + void setLargeSideDLIndex(uint32_t i) + { + largeSideIndex = i; + } + + /* obsolete, need to update JLF */ + void setJoinType(JoinType jt) + { + joinType = jt; + } + JoinType getJoinType() const + { + return joinType; + } + + /* Functions & Expressions interface */ + /* Cross-table Functions & Expressions in where clause */ + void addFcnExpGroup2(const boost::shared_ptr& fe); + bool hasFcnExpGroup2() + { + return (fe2 != NULL); + } + + /* Functions & Expressions in select and groupby clause */ + void setFcnExpGroup3(const std::vector& fe); + void setFE23Output(const rowgroup::RowGroup& rg); + + /* result rowgroup */ + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; + + // joinId + void joinId(int64_t id) + { + fJoinId = id; + } + int64_t joinId() const + { + return fJoinId; + } + + /* semi-join support */ + void addJoinFilter(boost::shared_ptr, uint32_t index); + bool hasJoinFilter() const + { + return (fe.size() > 0); + } + bool hasJoinFilter(uint32_t index) const; + boost::shared_ptr getJoinFilter(uint32_t index) const; + void setJoinFilterInputRG(const rowgroup::RowGroup& rg); + + virtual bool stringTableFriendly() + { + return true; + } + + uint32_t tokenJoin() const + { + return fTokenJoin; + } + void tokenJoin(uint32_t k) + { + fTokenJoin = k; + } + + //@bug3683 function join + boost::shared_ptr& funcJoinInfo() + { + return fFunctionJoinInfo; + } + void funcJoinInfo(const boost::shared_ptr& fji) + { + fFunctionJoinInfo = fji; + } + + void abort(); + void returnMemory() + { + if (fMemSizeForOutputRG > 0) { - return fTableOID2; + resourceManager->returnMemory(fMemSizeForOutputRG); + fMemSizeForOutputRG = 0; } - execplan::CalpontSystemCatalog::OID tableOid1() const + } + bool getMemory(uint64_t memSize) + { + bool gotMem = resourceManager->getMemory(memSize); + if (gotMem) + fMemSizeForOutputRG += memSize; + return gotMem; + } + + private: + TupleHashJoinStep(); + TupleHashJoinStep(const TupleHashJoinStep&); + TupleHashJoinStep& operator=(const TupleHashJoinStep&); + + void errorLogging(const std::string& msg, int err) const; + void startAdjoiningSteps(); + + void formatMiniStats(uint32_t index); + + RowGroupDL *largeDL, *outputDL; + std::vector smallDLs; + std::vector smallIts; + uint32_t largeIt; + + JoinType joinType; // deprecated + std::vector joinTypes; + execplan::CalpontSystemCatalog::OID fTableOID1; + execplan::CalpontSystemCatalog::OID fTableOID2; + execplan::CalpontSystemCatalog::OID fOid1; + execplan::CalpontSystemCatalog::OID fOid2; + + // v-table string join + execplan::CalpontSystemCatalog::OID fDictOid1; + execplan::CalpontSystemCatalog::OID fDictOid2; + + std::string fAlias1; + std::string fAlias2; + + std::string fView1; + std::string fView2; + + std::string fSchema1; + std::string fSchema2; + + int32_t fSequence1; + int32_t fSequence2; + + // @bug3398, add tuple id to steps + uint64_t fTupleId1; + uint64_t fTupleId2; + + // @bug3524 + // for NOT IN subquery where correlate join in subquery is treated as additional comparison. + // These simple columns are for converting join to expression. + // DON'T delete, they owned by exec plan. + const execplan::ReturnedColumn* fColumn1; + const execplan::ReturnedColumn* fColumn2; + + int fCorrelatedSide; + + std::vector typelessJoin; // the size of the vector is # of small side + std::vector > largeSideKeys; + std::vector > smallSideKeys; + + ResourceManager* resourceManager; + uint64_t fMemSizeForOutputRG; + + struct JoinerSorter + { + inline bool operator()(const boost::shared_ptr& j1, + const boost::shared_ptr& j2) const { - return fTableOID1; + return *j1 < *j2; } - execplan::CalpontSystemCatalog::OID tableOid2() const + }; + std::vector > joiners; + boost::scoped_array > rgData; + TupleBPS* largeBPS; + rowgroup::RowGroup largeRG, outputRG; + std::vector smallRGs; + ssize_t pmMemLimit; + + void hjRunner(); + void smallRunnerFcn(uint32_t index, uint threadID, uint64_t* threads); + + struct HJRunner + { + HJRunner(TupleHashJoinStep* hj) : HJ(hj) { - return fTableOID2; } - void tableOid1(execplan::CalpontSystemCatalog::OID tableOid1) + void operator()() { - fTableOID1 = tableOid1; - if (fTableOID1 < 3000) - numCores = 1; // syscat query, no need for more than 1 thread + utils::setThreadName("HJSBigSide"); + HJ->hjRunner(); } - void tableOid2(execplan::CalpontSystemCatalog::OID tableOid2) + TupleHashJoinStep* HJ; + }; + struct SmallRunner + { + SmallRunner(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { - fTableOID2 = tableOid2; } - - std::string alias1() const + void operator()() { - return fAlias1; + HJ->startSmallRunners(index); } - void alias1(const std::string& alias) + TupleHashJoinStep* HJ; + uint32_t index; + }; + + int64_t mainRunner; // thread handle from thread pool + + // for notify TupleAggregateStep PM hashjoin + // Ideally, hashjoin and delivery communicate with RowGroupDL, + // they don't need to know each other. + // Due to dynamic PM/UM hashjoin selection and support PM aggregation, + // delivery step need to know if raw or partially aggregated to process. + SJSTEP fDeliveryStep; + + // temporary hack to make sure JobList only calls run, join once + boost::mutex jlLock; + bool runRan, joinRan; + + /* Iteration 18 mods */ + uint32_t largeSideIndex; + bool joinIsTooBig; + + /* Functions & Expressions support */ + boost::shared_ptr fe2; + std::vector fe2TableDeps; + rowgroup::RowGroup fe2Output; + bool runFE2onPM; + + // Support Mixed Join Type + int64_t fJoinId; + + /* Semi-join support */ + std::vector feIndexes; + std::vector > fe; + rowgroup::RowGroup joinFilterRG; + + /* Casual Partitioning forwarding */ + void forwardCPData(); + uint32_t uniqueLimit; + + /* UM Join support. Most of this code is ported from the UM join code in tuple-bps.cpp. + * They should be kept in sync as much as possible. */ + struct JoinRunner + { + JoinRunner(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { - fAlias1 = alias; } - std::string alias2() const + void operator()() { - return fAlias2; + HJ->joinRunnerFcn(index); } - void alias2(const std::string& alias) + TupleHashJoinStep* HJ; + uint32_t index; + }; + void joinRunnerFcn(uint32_t index); + void startJoinThreads(); + void generateJoinResultSet(const std::vector >& joinerOutput, + rowgroup::Row& baseRow, + const boost::shared_array >& mappings, + const uint32_t depth, rowgroup::RowGroup& outputRG, rowgroup::RGData& rgData, + std::vector& outputData, + const boost::shared_array& smallRows, rowgroup::Row& joinedRow, + RowGroupDL* outputDL); + void grabSomeWork(std::vector* work); + void sendResult(const std::vector& res); + void processFE2(rowgroup::RowGroup& input, rowgroup::RowGroup& output, rowgroup::Row& inRow, + rowgroup::Row& outRow, std::vector* rgData, + funcexp::FuncExpWrapper* local_fe); + void joinOneRG(uint32_t threadID, std::vector& out, rowgroup::RowGroup& inputRG, + rowgroup::RowGroup& joinOutput, rowgroup::Row& largeSideRow, rowgroup::Row& joinFERow, + rowgroup::Row& joinedRow, rowgroup::Row& baseRow, + std::vector >& joinMatches, + boost::shared_array& smallRowTemplates, RowGroupDL* outputDL, + std::vector >* joiners = NULL, + boost::shared_array >* rgMappings = NULL, + boost::shared_array >* feMappings = NULL, + boost::scoped_array >* smallNullMem = NULL); + void finishSmallOuterJoin(); + void makeDupList(const rowgroup::RowGroup& rg); + void processDupList(uint32_t threadID, rowgroup::RowGroup& ingrp, std::vector* rowData); + + std::vector joinRunners; // thread handles from thread pool + boost::mutex inputDLLock, outputDLLock; + boost::shared_array > columnMappings, fergMappings; + boost::shared_array fe2Mapping; + uint32_t joinThreadCount; + boost::scoped_array > smallNullMemory; + uint64_t outputIt; + bool moreInput; + std::vector > dupList; + boost::scoped_array dupRows; + std::vector smallTableNames; + bool isExeMgr; + uint32_t lastSmallOuterJoiner; + + //@bug5958 & 6117, stores the table key for identify token join + uint32_t fTokenJoin; + + // moved from base class JobStep + boost::mutex* fStatsMutexPtr; + + //@bug3683 function join + boost::shared_ptr fFunctionJoinInfo; + std::set fFunctionJoinKeys; // for skipping CP forward + + /* Disk-based join support */ + boost::scoped_array djs; + boost::scoped_array > fifos; + void djsReaderFcn(int index); + uint64_t djsReader; // thread handle from thread pool + struct DJSReader + { + DJSReader(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { - fAlias = fAlias2 = alias; } - - std::string view1() const + void operator()() { - return fView1; + utils::setThreadName("DJSReader"); + HJ->djsReaderFcn(index); } - void view1(const std::string& vw) + TupleHashJoinStep* HJ; + uint32_t index; + }; + + uint64_t djsRelay; // thread handle from thread pool + void djsRelayFcn(); + struct DJSRelay + { + DJSRelay(TupleHashJoinStep* hj) : HJ(hj) { - fView1 = vw; } - std::string view2() const + void operator()() { - return fView2; - } - void view2(const std::string& vw) - { - fView = fView2 = vw; + HJ->djsRelayFcn(); } + TupleHashJoinStep* HJ; + }; - std::string schema1() const - { - return fSchema1; - } - void schema1(const std::string& s) - { - fSchema1 = s; - } - std::string schema2() const - { - return fSchema2; - } - void schema2(const std::string& s) - { - fSchema = fSchema2 = s; - } + boost::shared_ptr djsSmallUsage; + int64_t djsSmallLimit; + int64_t djsLargeLimit; + uint64_t djsPartitionSize; + bool isDML; + bool allowDJS; - int32_t sequence1() const - { - return fSequence1; - } - void sequence1(int32_t seq) - { - fSequence1 = seq; - } - int32_t sequence2() const - { - return fSequence2; - } - void sequence2(int32_t seq) - { - fSequence2 = seq; - } + // hacky mechanism to prevent nextBand from starting before the final + // THJS configuration is settled. Debatable whether to use a bool and poll instead; + // once the config is settled it stays settled, technically no need to + // keep grabbing locks after that. + boost::mutex deliverMutex; + bool ownsOutputDL; - const execplan::ReturnedColumn* column1() const - { - return fColumn1; - } - void column1(const execplan::ReturnedColumn* pos) - { - fColumn1 = pos; - } - const execplan::ReturnedColumn* column2() const - { - return fColumn2; - } - void column2(const execplan::ReturnedColumn* pos) - { - fColumn2 = pos; - } + void segregateJoiners(); + std::vector > tbpsJoiners; + std::vector > djsJoiners; + std::vector djsJoinerMap; + boost::scoped_array memUsedByEachJoin; + boost::mutex djsLock; + boost::shared_ptr sessionMemLimit; - int correlatedSide() const - { - return fCorrelatedSide; - } - void correlatedSide(int c) - { - fCorrelatedSide = c; - } - using JobStep::tupleId; - uint64_t tupleId() const - { - return fTupleId2; - } - uint64_t tupleId1() const - { - return fTupleId1; - } - uint64_t tupleId2() const - { - return fTupleId2; - } - void tupleId1(uint64_t id) - { - fTupleId1 = id; - } - void tupleId2(uint64_t id) - { - fTupleId2 = id; - } + /* Threaded UM join support */ + int numCores; + boost::mutex dlMutex, memTrackMutex, saneErrMsg; + boost::condition memTrackDone; + std::atomic rgdLock; + bool stopMemTracking; + void trackMem(uint index); + void startSmallRunners(uint index); - void addSmallSideRG(const std::vector& rgs, - const std::vector& tableNames); - void addJoinKeyIndex(const std::vector& jt, - const std::vector& typeless, - const std::vector >& smallkeys, - const std::vector >& largekeys); - - void configSmallSideRG(const std::vector& rgs, - const std::vector& tableNames); - void configLargeSideRG(const rowgroup::RowGroup& rg); - - void configJoinKeyIndex(const std::vector& jt, - const std::vector& typeless, - const std::vector >& smallkeys, - const std::vector >& largekeys); - - void setOutputRowGroup(const rowgroup::RowGroup& rg); - - uint32_t nextBand(messageqcpp::ByteStream& bs); - - const rowgroup::RowGroup& getOutputRowGroup() const - { - return outputRG; - } - const rowgroup::RowGroup& getSmallRowGroup() const - { - return smallRGs[0]; - } - const std::vector& getSmallRowGroups() const - { - return smallRGs; - } - const rowgroup::RowGroup& getLargeRowGroup() const - { - return largeRG; - } - uint32_t getSmallKey() const - { - return smallSideKeys[0][0]; - } - const std::vector >& getSmallKeys() const - { - return smallSideKeys; - } - const std::vector >& getLargeKeys() const - { - return largeSideKeys; - } - - /* Some compat fcns to get rid of later */ - void oid1(execplan::CalpontSystemCatalog::OID oid) - { - fOid1 = oid; - } - execplan::CalpontSystemCatalog::OID oid1() const - { - return fOid1; - } - void oid2(execplan::CalpontSystemCatalog::OID oid) - { - fOid2 = oid; - } - execplan::CalpontSystemCatalog::OID oid2() const - { - return fOid2; - } - void dictOid1(execplan::CalpontSystemCatalog::OID oid) - { - fDictOid1 = oid; - } - execplan::CalpontSystemCatalog::OID dictOid1() const - { - return fDictOid1; - } - void dictOid2(execplan::CalpontSystemCatalog::OID oid) - { - fDictOid2 = oid; - } - execplan::CalpontSystemCatalog::OID dictOid2() const - { - return fDictOid2; - } - - /* The replacements. I don't think there's a need for setters or vars. - OIDs are already in the rowgroups. */ - // s - sth table pair; k - kth key in compound join, 0 for non-compand join - execplan::CalpontSystemCatalog::OID smallSideKeyOID(uint32_t s, uint32_t k) const; - execplan::CalpontSystemCatalog::OID largeSideKeyOID(uint32_t s, uint32_t k) const; - - void deliveryStep(const SJSTEP& ds) - { - fDeliveryStep = ds; - } - - /* Iteration 18 mods */ - void setLargeSideDLIndex(uint32_t i) - { - largeSideIndex = i; - } - - /* obsolete, need to update JLF */ - void setJoinType(JoinType jt) - { - joinType = jt; - } - JoinType getJoinType() const - { - return joinType; - } - - - /* Functions & Expressions interface */ - /* Cross-table Functions & Expressions in where clause */ - void addFcnExpGroup2(const boost::shared_ptr& fe); - bool hasFcnExpGroup2() - { - return (fe2 != NULL); - } - - /* Functions & Expressions in select and groupby clause */ - void setFcnExpGroup3(const std::vector& fe); - void setFE23Output(const rowgroup::RowGroup& rg); - - /* result rowgroup */ - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - - // joinId - void joinId(int64_t id) - { - fJoinId = id; - } - int64_t joinId() const - { - return fJoinId; - } - - /* semi-join support */ - void addJoinFilter(boost::shared_ptr, uint32_t index); - bool hasJoinFilter() const - { - return (fe.size() > 0); - } - bool hasJoinFilter(uint32_t index) const; - boost::shared_ptr getJoinFilter(uint32_t index) const; - void setJoinFilterInputRG(const rowgroup::RowGroup& rg); - - virtual bool stringTableFriendly() - { - return true; - } - - uint32_t tokenJoin() const - { - return fTokenJoin; - } - void tokenJoin(uint32_t k) - { - fTokenJoin = k; - } - - //@bug3683 function join - boost::shared_ptr& funcJoinInfo() - { - return fFunctionJoinInfo; - } - void funcJoinInfo(const boost::shared_ptr& fji) - { - fFunctionJoinInfo = fji; - } - - void abort(); - void returnMemory() - { - if (fMemSizeForOutputRG > 0) - { - resourceManager->returnMemory(fMemSizeForOutputRG); - fMemSizeForOutputRG = 0; - } - } - bool getMemory(uint64_t memSize) - { - bool gotMem = resourceManager->getMemory(memSize); - if (gotMem) - fMemSizeForOutputRG += memSize; - return gotMem; - } -private: - TupleHashJoinStep(); - TupleHashJoinStep(const TupleHashJoinStep&); - TupleHashJoinStep& operator=(const TupleHashJoinStep&); - - void errorLogging(const std::string& msg, int err) const; - void startAdjoiningSteps(); - - void formatMiniStats(uint32_t index); - - RowGroupDL* largeDL, *outputDL; - std::vector smallDLs; - std::vector smallIts; - uint32_t largeIt; - - JoinType joinType; // deprecated - std::vector joinTypes; - execplan::CalpontSystemCatalog::OID fTableOID1; - execplan::CalpontSystemCatalog::OID fTableOID2; - execplan::CalpontSystemCatalog::OID fOid1; - execplan::CalpontSystemCatalog::OID fOid2; - - // v-table string join - execplan::CalpontSystemCatalog::OID fDictOid1; - execplan::CalpontSystemCatalog::OID fDictOid2; - - std::string fAlias1; - std::string fAlias2; - - std::string fView1; - std::string fView2; - - std::string fSchema1; - std::string fSchema2; - - int32_t fSequence1; - int32_t fSequence2; - - // @bug3398, add tuple id to steps - uint64_t fTupleId1; - uint64_t fTupleId2; - - // @bug3524 - // for NOT IN subquery where correlate join in subquery is treated as additional comparison. - // These simple columns are for converting join to expression. - // DON'T delete, they owned by exec plan. - const execplan::ReturnedColumn* fColumn1; - const execplan::ReturnedColumn* fColumn2; - - int fCorrelatedSide; - - std::vector typelessJoin; // the size of the vector is # of small side - std::vector > largeSideKeys; - std::vector > smallSideKeys; - - ResourceManager* resourceManager; - uint64_t fMemSizeForOutputRG; - - struct JoinerSorter - { - inline bool operator()(const boost::shared_ptr& j1, - const boost::shared_ptr& j2) const - { - return *j1 < *j2; - } - }; - std::vector > joiners; - boost::scoped_array > rgData; - TupleBPS* largeBPS; - rowgroup::RowGroup largeRG, outputRG; - std::vector smallRGs; - ssize_t pmMemLimit; - - void hjRunner(); - void smallRunnerFcn(uint32_t index, uint threadID, uint64_t *threads); - - struct HJRunner - { - HJRunner(TupleHashJoinStep* hj) : HJ(hj) { } - void operator()() - { - utils::setThreadName("HJSBigSide"); - HJ->hjRunner(); - } - TupleHashJoinStep* HJ; - }; - struct SmallRunner - { - SmallRunner(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { } - void operator()() - { - HJ->startSmallRunners(index); - } - TupleHashJoinStep* HJ; - uint32_t index; - }; - - int64_t mainRunner; // thread handle from thread pool - - // for notify TupleAggregateStep PM hashjoin - // Ideally, hashjoin and delivery communicate with RowGroupDL, - // they don't need to know each other. - // Due to dynamic PM/UM hashjoin selection and support PM aggregation, - // delivery step need to know if raw or partially aggregated to process. - SJSTEP fDeliveryStep; - - // temporary hack to make sure JobList only calls run, join once - boost::mutex jlLock; - bool runRan, joinRan; - - /* Iteration 18 mods */ - uint32_t largeSideIndex; - bool joinIsTooBig; - - /* Functions & Expressions support */ - boost::shared_ptr fe2; - std::vector fe2TableDeps; - rowgroup::RowGroup fe2Output; - bool runFE2onPM; - - // Support Mixed Join Type - int64_t fJoinId; - - /* Semi-join support */ - std::vector feIndexes; - std::vector > fe; - rowgroup::RowGroup joinFilterRG; - - /* Casual Partitioning forwarding */ - void forwardCPData(); - uint32_t uniqueLimit; - - /* UM Join support. Most of this code is ported from the UM join code in tuple-bps.cpp. - * They should be kept in sync as much as possible. */ - struct JoinRunner - { - JoinRunner(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { } - void operator()() - { - HJ->joinRunnerFcn(index); - } - TupleHashJoinStep* HJ; - uint32_t index; - }; - void joinRunnerFcn(uint32_t index); - void startJoinThreads(); - void generateJoinResultSet(const std::vector >& joinerOutput, - rowgroup::Row& baseRow, const boost::shared_array >& mappings, - const uint32_t depth, rowgroup::RowGroup& outputRG, rowgroup::RGData& rgData, - std::vector& outputData, - const boost::shared_array& smallRows, rowgroup::Row& joinedRow, - RowGroupDL* outputDL); - void grabSomeWork(std::vector* work); - void sendResult(const std::vector& res); - void processFE2(rowgroup::RowGroup& input, rowgroup::RowGroup& output, rowgroup::Row& inRow, - rowgroup::Row& outRow, std::vector* rgData, - funcexp::FuncExpWrapper* local_fe); - void joinOneRG(uint32_t threadID, std::vector& out, - rowgroup::RowGroup& inputRG, rowgroup::RowGroup& joinOutput, rowgroup::Row& largeSideRow, - rowgroup::Row& joinFERow, rowgroup::Row& joinedRow, rowgroup::Row& baseRow, - std::vector >& joinMatches, - boost::shared_array& smallRowTemplates, - RowGroupDL* outputDL, - std::vector >* joiners = NULL, - boost::shared_array >* rgMappings = NULL, - boost::shared_array >* feMappings = NULL, - boost::scoped_array >* smallNullMem = NULL - ); - void finishSmallOuterJoin(); - void makeDupList(const rowgroup::RowGroup& rg); - void processDupList(uint32_t threadID, rowgroup::RowGroup& ingrp, - std::vector* rowData); - - std::vector joinRunners; // thread handles from thread pool - boost::mutex inputDLLock, outputDLLock; - boost::shared_array > columnMappings, fergMappings; - boost::shared_array fe2Mapping; - uint32_t joinThreadCount; - boost::scoped_array > smallNullMemory; - uint64_t outputIt; - bool moreInput; - std::vector > dupList; - boost::scoped_array dupRows; - std::vector smallTableNames; - bool isExeMgr; - uint32_t lastSmallOuterJoiner; - - //@bug5958 & 6117, stores the table key for identify token join - uint32_t fTokenJoin; - - // moved from base class JobStep - boost::mutex* fStatsMutexPtr; - - //@bug3683 function join - boost::shared_ptr fFunctionJoinInfo; - std::set fFunctionJoinKeys; // for skipping CP forward - - /* Disk-based join support */ - boost::scoped_array djs; - boost::scoped_array > fifos; - void djsReaderFcn(int index); - uint64_t djsReader; // thread handle from thread pool - struct DJSReader - { - DJSReader(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { } - void operator()() - { - utils::setThreadName("DJSReader"); - HJ->djsReaderFcn(index); - } - TupleHashJoinStep* HJ; - uint32_t index; - }; - - uint64_t djsRelay; // thread handle from thread pool - void djsRelayFcn(); - struct DJSRelay - { - DJSRelay(TupleHashJoinStep* hj) : HJ(hj) { } - void operator()() - { - HJ->djsRelayFcn(); - } - TupleHashJoinStep* HJ; - }; - - boost::shared_ptr djsSmallUsage; - int64_t djsSmallLimit; - int64_t djsLargeLimit; - uint64_t djsPartitionSize; - bool isDML; - bool allowDJS; - - // hacky mechanism to prevent nextBand from starting before the final - // THJS configuration is settled. Debatable whether to use a bool and poll instead; - // once the config is settled it stays settled, technically no need to - // keep grabbing locks after that. - boost::mutex deliverMutex; - bool ownsOutputDL; - - void segregateJoiners(); - std::vector > tbpsJoiners; - std::vector > djsJoiners; - std::vector djsJoinerMap; - boost::scoped_array memUsedByEachJoin; - boost::mutex djsLock; - boost::shared_ptr sessionMemLimit; - - /* Threaded UM join support */ - int numCores; - boost::mutex dlMutex, memTrackMutex, saneErrMsg; - boost::condition memTrackDone; - std::atomic rgdLock; - bool stopMemTracking; - void trackMem(uint index); - void startSmallRunners(uint index); - - friend class DiskJoinStep; + friend class DiskJoinStep; }; -} - +} // namespace joblist #endif // vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehavingstep.cpp b/dbcon/joblist/tuplehavingstep.cpp index 853e12342..701e77b39 100644 --- a/dbcon/joblist/tuplehavingstep.cpp +++ b/dbcon/joblist/tuplehavingstep.cpp @@ -17,7 +17,6 @@ // $Id: tuplehavingstep.cpp 9709 2013-07-20 06:08:46Z xlou $ - //#define NDEBUG #include #include @@ -54,383 +53,359 @@ using namespace querytele; #include "jlf_common.h" #include "tuplehavingstep.h" - namespace joblist { - -TupleHavingStep::TupleHavingStep(const JobInfo& jobInfo) : - ExpressionStep(jobInfo), - fInputDL(NULL), - fOutputDL(NULL), - fInputIterator(0), - fRunner(0), - fRowsReturned(0), - fEndOfResult(false), - fFeInstance(funcexp::FuncExp::instance()) +TupleHavingStep::TupleHavingStep(const JobInfo& jobInfo) + : ExpressionStep(jobInfo) + , fInputDL(NULL) + , fOutputDL(NULL) + , fInputIterator(0) + , fRunner(0) + , fRowsReturned(0) + , fEndOfResult(false) + , fFeInstance(funcexp::FuncExp::instance()) { - fExtendedInfo = "HVS: "; - fQtc.stepParms().stepType = StepTeleStats::T_HVS; + fExtendedInfo = "HVS: "; + fQtc.stepParms().stepType = StepTeleStats::T_HVS; } - TupleHavingStep::~TupleHavingStep() { } - void TupleHavingStep::setOutputRowGroup(const rowgroup::RowGroup& rg) { - throw runtime_error("Disabled, use initialize() to set output RowGroup."); + throw runtime_error("Disabled, use initialize() to set output RowGroup."); } - void TupleHavingStep::initialize(const RowGroup& rgIn, const JobInfo& jobInfo) { - fRowGroupIn = rgIn; - fRowGroupIn.initRow(&fRowIn); + fRowGroupIn = rgIn; + fRowGroupIn.initRow(&fRowIn); - map keyToIndexMap; + map keyToIndexMap; - for (uint64_t i = 0; i < fRowGroupIn.getKeys().size(); ++i) - if (keyToIndexMap.find(fRowGroupIn.getKeys()[i]) == keyToIndexMap.end()) - keyToIndexMap.insert(make_pair(fRowGroupIn.getKeys()[i], i)); + for (uint64_t i = 0; i < fRowGroupIn.getKeys().size(); ++i) + if (keyToIndexMap.find(fRowGroupIn.getKeys()[i]) == keyToIndexMap.end()) + keyToIndexMap.insert(make_pair(fRowGroupIn.getKeys()[i], i)); - updateInputIndex(keyToIndexMap, jobInfo); + updateInputIndex(keyToIndexMap, jobInfo); - vector oids, oidsIn = fRowGroupIn.getOIDs(); - vector keys, keysIn = fRowGroupIn.getKeys(); - vector scale, scaleIn = fRowGroupIn.getScale(); - vector precision, precisionIn = fRowGroupIn.getPrecision(); - vector types, typesIn = fRowGroupIn.getColTypes(); - vector csNums, csNumsIn = fRowGroupIn.getCharsetNumbers(); - vector pos, posIn = fRowGroupIn.getOffsets(); + vector oids, oidsIn = fRowGroupIn.getOIDs(); + vector keys, keysIn = fRowGroupIn.getKeys(); + vector scale, scaleIn = fRowGroupIn.getScale(); + vector precision, precisionIn = fRowGroupIn.getPrecision(); + vector types, typesIn = fRowGroupIn.getColTypes(); + vector csNums, csNumsIn = fRowGroupIn.getCharsetNumbers(); + vector pos, posIn = fRowGroupIn.getOffsets(); - size_t n = 0; - RetColsVector::const_iterator i = jobInfo.deliveredCols.begin(); + size_t n = 0; + RetColsVector::const_iterator i = jobInfo.deliveredCols.begin(); - while (i != jobInfo.deliveredCols.end()) - if (NULL == dynamic_cast(i++->get())) - n++; + while (i != jobInfo.deliveredCols.end()) + if (NULL == dynamic_cast(i++->get())) + n++; - oids.insert(oids.end(), oidsIn.begin(), oidsIn.begin() + n); - keys.insert(keys.end(), keysIn.begin(), keysIn.begin() + n); - scale.insert(scale.end(), scaleIn.begin(), scaleIn.begin() + n); - precision.insert(precision.end(), precisionIn.begin(), precisionIn.begin() + n); - types.insert(types.end(), typesIn.begin(), typesIn.begin() + n); - csNums.insert(csNums.end(), csNumsIn.begin(), csNumsIn.begin() + n); - pos.insert(pos.end(), posIn.begin(), posIn.begin() + n + 1); + oids.insert(oids.end(), oidsIn.begin(), oidsIn.begin() + n); + keys.insert(keys.end(), keysIn.begin(), keysIn.begin() + n); + scale.insert(scale.end(), scaleIn.begin(), scaleIn.begin() + n); + precision.insert(precision.end(), precisionIn.begin(), precisionIn.begin() + n); + types.insert(types.end(), typesIn.begin(), typesIn.begin() + n); + csNums.insert(csNums.end(), csNumsIn.begin(), csNumsIn.begin() + n); + pos.insert(pos.end(), posIn.begin(), posIn.begin() + n + 1); - fRowGroupOut = RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); - fRowGroupOut.initRow(&fRowOut); + fRowGroupOut = + RowGroup(oids.size(), pos, oids, keys, types, csNums, scale, precision, jobInfo.stringTableThreshold); + fRowGroupOut.initRow(&fRowOut); } - void TupleHavingStep::expressionFilter(const ParseTree* filter, JobInfo& jobInfo) { - // let base class handle the simple columns - ExpressionStep::expressionFilter(filter, jobInfo); + // let base class handle the simple columns + ExpressionStep::expressionFilter(filter, jobInfo); - // extract simple columns from parse tree - vector acv; - fExpressionFilter->walk(getAggCols, &acv); - fColumns.insert(fColumns.end(), acv.begin(), acv.end()); + // extract simple columns from parse tree + vector acv; + fExpressionFilter->walk(getAggCols, &acv); + fColumns.insert(fColumns.end(), acv.begin(), acv.end()); } - void TupleHavingStep::run() { - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for having step."); + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for having step."); - fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fInputDL == NULL) - throw logic_error("Input is not a RowGroup data list."); + if (fInputDL == NULL) + throw logic_error("Input is not a RowGroup data list."); - fInputIterator = fInputDL->getIterator(); + fInputIterator = fInputDL->getIterator(); - if (fDelivery == false) - { - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for non-delivery having step."); + if (fDelivery == false) + { + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for non-delivery having step."); - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fOutputDL == NULL) - throw logic_error("Output is not a RowGroup data list."); + if (fOutputDL == NULL) + throw logic_error("Output is not a RowGroup data list."); - fRunner = jobstepThreadPool.invoke(Runner(this)); - } + fRunner = jobstepThreadPool.invoke(Runner(this)); + } } - void TupleHavingStep::join() { - if (fRunner) - jobstepThreadPool.join(fRunner); + if (fRunner) + jobstepThreadPool.join(fRunner); } - uint32_t TupleHavingStep::nextBand(messageqcpp::ByteStream& bs) { - RGData rgDataIn; - RGData rgDataOut; - bool more = false; - uint32_t rowCount = 0; + RGData rgDataIn; + RGData rgDataOut; + bool more = false; + uint32_t rowCount = 0; - try + try + { + bs.restart(); + + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + if (!more || cancelled()) { - bs.restart(); - - more = fInputDL->next(fInputIterator, &rgDataIn); - - if (dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - if (!more || cancelled()) - { - fEndOfResult = true; - } - - bool emptyRowGroup = true; - - while (more && !fEndOfResult && emptyRowGroup) - { - if (cancelled()) - { - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); - - break; - } - - fRowGroupIn.setData(&rgDataIn); - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - - doHavingFilters(); - - if (fRowGroupOut.getRowCount() > 0) - { - emptyRowGroup = false; - fRowGroupOut.serializeRGData(bs); - rowCount = fRowGroupOut.getRowCount(); - } - else - { - more = fInputDL->next(fInputIterator, &rgDataIn); - } - } - - if (!more) - { - fEndOfResult = true; - } - } - catch (...) - { - handleException(std::current_exception(), - logging::tupleHavingStepErr, - logging::ERR_ALWAYS_CRITICAL, - "TupleHavingStep::nextBand()"); - while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); - - fEndOfResult = true; + fEndOfResult = true; } - if (fEndOfResult) + bool emptyRowGroup = true; + + while (more && !fEndOfResult && emptyRowGroup) { - // send an empty / error band - rgDataOut.reinit(fRowGroupOut, 0); - fRowGroupOut.setData(&rgDataOut); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setStatus(status()); + if (cancelled()) + { + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); + + break; + } + + fRowGroupIn.setData(&rgDataIn); + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + + doHavingFilters(); + + if (fRowGroupOut.getRowCount() > 0) + { + emptyRowGroup = false; fRowGroupOut.serializeRGData(bs); - - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - - if (traceOn()) - printCalTrace(); - } - - return rowCount; -} - - -void TupleHavingStep::execute() -{ - RGData rgDataIn; - RGData rgDataOut; - bool more = false; - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - - try - { + rowCount = fRowGroupOut.getRowCount(); + } + else + { more = fInputDL->next(fInputIterator, &rgDataIn); - dlTimes.setFirstReadTime(); - - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - - if (!more && cancelled()) - { - fEndOfResult = true; - } - - while (more && !fEndOfResult) - { - fRowGroupIn.setData(&rgDataIn); - rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgDataOut); - - doHavingFilters(); - - more = fInputDL->next(fInputIterator, &rgDataIn); - - if (cancelled()) - { - fEndOfResult = true; - } - else - { - fOutputDL->insert(rgDataOut); - } - } + } } - catch (...) + + if (!more) { - handleException(std::current_exception(), - logging::tupleHavingStepErr, - logging::ERR_ALWAYS_CRITICAL, - "TupleHavingStep::nextBand()"); + fEndOfResult = true; } - + } + catch (...) + { + handleException(std::current_exception(), logging::tupleHavingStepErr, logging::ERR_ALWAYS_CRITICAL, + "TupleHavingStep::nextBand()"); while (more) - more = fInputDL->next(fInputIterator, &rgDataIn); + more = fInputDL->next(fInputIterator, &rgDataIn); fEndOfResult = true; - fOutputDL->endOfInput(); + } - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); + if (fEndOfResult) + { + // send an empty / error band + rgDataOut.reinit(fRowGroupOut, 0); + fRowGroupOut.setData(&rgDataOut); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setStatus(status()); + fRowGroupOut.serializeRGData(bs); dlTimes.setLastReadTime(); dlTimes.setEndOfInputTime(); if (traceOn()) - printCalTrace(); + printCalTrace(); + } + + return rowCount; } +void TupleHavingStep::execute() +{ + RGData rgDataIn; + RGData rgDataOut; + bool more = false; + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + + try + { + more = fInputDL->next(fInputIterator, &rgDataIn); + dlTimes.setFirstReadTime(); + + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + + if (!more && cancelled()) + { + fEndOfResult = true; + } + + while (more && !fEndOfResult) + { + fRowGroupIn.setData(&rgDataIn); + rgDataOut.reinit(fRowGroupOut, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgDataOut); + + doHavingFilters(); + + more = fInputDL->next(fInputIterator, &rgDataIn); + + if (cancelled()) + { + fEndOfResult = true; + } + else + { + fOutputDL->insert(rgDataOut); + } + } + } + catch (...) + { + handleException(std::current_exception(), logging::tupleHavingStepErr, logging::ERR_ALWAYS_CRITICAL, + "TupleHavingStep::nextBand()"); + } + + while (more) + more = fInputDL->next(fInputIterator, &rgDataIn); + + fEndOfResult = true; + fOutputDL->endOfInput(); + + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + + if (traceOn()) + printCalTrace(); +} void TupleHavingStep::doHavingFilters() { - fRowGroupIn.getRow(0, &fRowIn); - fRowGroupOut.getRow(0, &fRowOut); - fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); + fRowGroupIn.getRow(0, &fRowIn); + fRowGroupOut.getRow(0, &fRowOut); + fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) + { + if (fFeInstance->evaluate(fRowIn, fExpressionFilter)) { - if (fFeInstance->evaluate(fRowIn, fExpressionFilter)) - { - copyRow(fRowIn, &fRowOut); - fRowGroupOut.incRowCount(); - fRowOut.nextRow(); - } - - fRowIn.nextRow(); + copyRow(fRowIn, &fRowOut); + fRowGroupOut.incRowCount(); + fRowOut.nextRow(); } - fRowsReturned += fRowGroupOut.getRowCount(); -} + fRowIn.nextRow(); + } + fRowsReturned += fRowGroupOut.getRowCount(); +} const RowGroup& TupleHavingStep::getOutputRowGroup() const { - return fRowGroupOut; + return fRowGroupOut; } - const RowGroup& TupleHavingStep::getDeliveredRowGroup() const { - return fRowGroupOut; + return fRowGroupOut; } - void TupleHavingStep::deliverStringTableRowGroup(bool b) { - fRowGroupOut.setUseStringTable(b); + fRowGroupOut.setUseStringTable(b); } - bool TupleHavingStep::deliverStringTableRowGroup() const { - return fRowGroupOut.usesStringTable(); + return fRowGroupOut.usesStringTable(); } - const string TupleHavingStep::toString() const { - ostringstream oss; - oss << "HavingStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "HavingStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " in:"; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); - oss << " out:"; + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } - void TupleHavingStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } - void TupleHavingStep::formatMiniStats() { - fMiniInfo += "THS "; - fMiniInfo += "UM "; - fMiniInfo += "- "; - fMiniInfo += "- "; - fMiniInfo += "- "; - fMiniInfo += "- "; - fMiniInfo += "- "; - fMiniInfo += "- "; - fMiniInfo += JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + " "; - fMiniInfo += "- "; + fMiniInfo += "THS "; + fMiniInfo += "UM "; + fMiniInfo += "- "; + fMiniInfo += "- "; + fMiniInfo += "- "; + fMiniInfo += "- "; + fMiniInfo += "- "; + fMiniInfo += "- "; + fMiniInfo += JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + " "; + fMiniInfo += "- "; } - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/tuplehavingstep.h b/dbcon/joblist/tuplehavingstep.h index 7b950d4a9..068f3d35c 100644 --- a/dbcon/joblist/tuplehavingstep.h +++ b/dbcon/joblist/tuplehavingstep.h @@ -17,7 +17,6 @@ // $Id: tuplehavingstep.h 9596 2013-06-04 19:59:04Z xlou $ - #ifndef JOBLIST_TUPLEHAVINGSTEP_H #define JOBLIST_TUPLEHAVINGSTEP_H @@ -31,7 +30,6 @@ namespace fucexp class FuncExp; } - namespace joblist { /** @brief class TupleHavingStep @@ -39,83 +37,84 @@ namespace joblist */ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep { -public: - /** @brief TupleHavingStep constructor - */ - TupleHavingStep(const JobInfo& jobInfo); + public: + /** @brief TupleHavingStep constructor + */ + TupleHavingStep(const JobInfo& jobInfo); - /** @brief TupleHavingStep destructor - */ - ~TupleHavingStep(); + /** @brief TupleHavingStep destructor + */ + ~TupleHavingStep(); - /** @brief virtual void Run method - */ - void run(); - void join(); + /** @brief virtual void Run method + */ + void run(); + void join(); - const std::string toString() const; + const std::string toString() const; - /** @brief TupleJobStep's pure virtual methods - */ - const rowgroup::RowGroup& getOutputRowGroup() const; - void setOutputRowGroup(const rowgroup::RowGroup&); + /** @brief TupleJobStep's pure virtual methods + */ + const rowgroup::RowGroup& getOutputRowGroup() const; + void setOutputRowGroup(const rowgroup::RowGroup&); - /** @brief TupleDeliveryStep's pure virtual methods - */ - uint32_t nextBand(messageqcpp::ByteStream& bs); - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + /** @brief TupleDeliveryStep's pure virtual methods + */ + uint32_t nextBand(messageqcpp::ByteStream& bs); + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; - void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); - using ExpressionStep::expressionFilter; - void expressionFilter(const execplan::ParseTree* filter, JobInfo& jobInfo); + void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); + using ExpressionStep::expressionFilter; + void expressionFilter(const execplan::ParseTree* filter, JobInfo& jobInfo); - virtual bool stringTableFriendly() + virtual bool stringTableFriendly() + { + return true; + } + + protected: + void execute(); + void doHavingFilters(); + void formatMiniStats(); + void printCalTrace(); + + // input/output rowgroup and row + rowgroup::RowGroup fRowGroupIn; + rowgroup::RowGroup fRowGroupOut; + rowgroup::Row fRowIn; + rowgroup::Row fRowOut; + + // for datalist + RowGroupDL* fInputDL; + RowGroupDL* fOutputDL; + uint64_t fInputIterator; + + class Runner + { + public: + Runner(TupleHavingStep* step) : fStep(step) { - return true; + } + void operator()() + { + utils::setThreadName("HVSRunner"); + fStep->execute(); } -protected: - void execute(); - void doHavingFilters(); - void formatMiniStats(); - void printCalTrace(); + TupleHavingStep* fStep; + }; - // input/output rowgroup and row - rowgroup::RowGroup fRowGroupIn; - rowgroup::RowGroup fRowGroupOut; - rowgroup::Row fRowIn; - rowgroup::Row fRowOut; + uint64_t fRunner; // thread pool handle - // for datalist - RowGroupDL* fInputDL; - RowGroupDL* fOutputDL; - uint64_t fInputIterator; + uint64_t fRowsReturned; + bool fEndOfResult; - class Runner - { - public: - Runner(TupleHavingStep* step) : fStep(step) { } - void operator()() - { - utils::setThreadName("HVSRunner"); - fStep->execute(); - } - - TupleHavingStep* fStep; - }; - - uint64_t fRunner; // thread pool handle - - uint64_t fRowsReturned; - bool fEndOfResult; - - funcexp::FuncExp* fFeInstance; + funcexp::FuncExp* fFeInstance; }; - -} // namespace +} // namespace joblist #endif // JOBLIST_TUPLEHAVINGSTEP_H diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 73fe936ba..291dd325a 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -44,1409 +44,1320 @@ using namespace dataconvert; #ifndef __linux__ #ifndef M_LN10 -#define M_LN10 2.30258509299404568402 /* log_e 10 */ +#define M_LN10 2.30258509299404568402 /* log_e 10 */ #endif namespace { -//returns the value of 10 raised to the power x. +// returns the value of 10 raised to the power x. inline double exp10(double x) { - return exp(x * M_LN10); -} + return exp(x * M_LN10); } +} // namespace #endif namespace joblist { inline uint64_t TupleUnion::Hasher::operator()(const RowPosition& p) const { - Row& row = ts->row; + Row& row = ts->row; - if (p.group & RowPosition::normalizedFlag) - ts->normalizedData[p.group & ~RowPosition::normalizedFlag].getRow(p.row, &row); - else - ts->rowMemory[p.group].getRow(p.row, &row); + if (p.group & RowPosition::normalizedFlag) + ts->normalizedData[p.group & ~RowPosition::normalizedFlag].getRow(p.row, &row); + else + ts->rowMemory[p.group].getRow(p.row, &row); - return row.hash(); + return row.hash(); } inline bool TupleUnion::Eq::operator()(const RowPosition& d1, const RowPosition& d2) const { - Row& r1 = ts->row, &r2 = ts->row2; + Row &r1 = ts->row, &r2 = ts->row2; - if (d1.group & RowPosition::normalizedFlag) - ts->normalizedData[d1.group & ~RowPosition::normalizedFlag].getRow(d1.row, &r1); - else - ts->rowMemory[d1.group].getRow(d1.row, &r1); + if (d1.group & RowPosition::normalizedFlag) + ts->normalizedData[d1.group & ~RowPosition::normalizedFlag].getRow(d1.row, &r1); + else + ts->rowMemory[d1.group].getRow(d1.row, &r1); - if (d2.group & RowPosition::normalizedFlag) - ts->normalizedData[d2.group & ~RowPosition::normalizedFlag].getRow(d2.row, &r2); - else - ts->rowMemory[d2.group].getRow(d2.row, &r2); + if (d2.group & RowPosition::normalizedFlag) + ts->normalizedData[d2.group & ~RowPosition::normalizedFlag].getRow(d2.row, &r2); + else + ts->rowMemory[d2.group].getRow(d2.row, &r2); - return r1.equals(r2); + return r1.equals(r2); } -TupleUnion::TupleUnion(CalpontSystemCatalog::OID tableOID, const JobInfo& jobInfo) : - JobStep(jobInfo), - fTableOID(tableOID), - output(NULL), - outputIt(-1), - memUsage(0), - rm(jobInfo.rm), - runnersDone(0), - distinctCount(0), - distinctDone(0), - fRowsReturned(0), - runRan(false), - joinRan(false), - sessionMemLimit(jobInfo.umMemLimit), - fTimeZone(jobInfo.timeZone) +TupleUnion::TupleUnion(CalpontSystemCatalog::OID tableOID, const JobInfo& jobInfo) + : JobStep(jobInfo) + , fTableOID(tableOID) + , output(NULL) + , outputIt(-1) + , memUsage(0) + , rm(jobInfo.rm) + , runnersDone(0) + , distinctCount(0) + , distinctDone(0) + , fRowsReturned(0) + , runRan(false) + , joinRan(false) + , sessionMemLimit(jobInfo.umMemLimit) + , fTimeZone(jobInfo.timeZone) { - uniquer.reset(new Uniquer_t(10, Hasher(this), Eq(this), allocator)); - fExtendedInfo = "TUN: "; - fQtc.stepParms().stepType = StepTeleStats::T_TUN; + uniquer.reset(new Uniquer_t(10, Hasher(this), Eq(this), allocator)); + fExtendedInfo = "TUN: "; + fQtc.stepParms().stepType = StepTeleStats::T_TUN; } TupleUnion::~TupleUnion() { - rm->returnMemory(memUsage, sessionMemLimit); + rm->returnMemory(memUsage, sessionMemLimit); - if (!runRan && output) - output->endOfInput(); + if (!runRan && output) + output->endOfInput(); } CalpontSystemCatalog::OID TupleUnion::tableOid() const { - return fTableOID; + return fTableOID; } void TupleUnion::setInputRowGroups(const vector& in) { - inputRGs = in; + inputRGs = in; } void TupleUnion::setOutputRowGroup(const rowgroup::RowGroup& out) { - outputRG = out; - rowLength = outputRG.getRowSizeWithStrings(); + outputRG = out; + rowLength = outputRG.getRowSizeWithStrings(); } void TupleUnion::setDistinctFlags(const vector& v) { - distinctFlags = v; + distinctFlags = v; } void TupleUnion::readInput(uint32_t which) { - /* The handling of the output got a little kludgey with the string table enhancement. - * When there is no distinct check, the outputs are all generated independently of - * each other locally in this fcn. When there is a distinct check, threads - * share the output, which is built in the 'rowMemory' vector rather than in - * thread-local memory. Building the result in a common space allows us to - * store 8-byte offsets in rowMemory rather than 16-bytes for absolute pointers. - */ + /* The handling of the output got a little kludgey with the string table enhancement. + * When there is no distinct check, the outputs are all generated independently of + * each other locally in this fcn. When there is a distinct check, threads + * share the output, which is built in the 'rowMemory' vector rather than in + * thread-local memory. Building the result in a common space allows us to + * store 8-byte offsets in rowMemory rather than 16-bytes for absolute pointers. + */ - RowGroupDL* dl = NULL; - bool more = true; - RGData inRGData, outRGData, *tmpRGData; - uint32_t it = numeric_limits::max(); - RowGroup l_inputRG, l_outputRG, l_tmpRG; - Row inRow, outRow, tmpRow; - bool distinct; - uint64_t memUsageBefore, memUsageAfter, memDiff; - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; + RowGroupDL* dl = NULL; + bool more = true; + RGData inRGData, outRGData, *tmpRGData; + uint32_t it = numeric_limits::max(); + RowGroup l_inputRG, l_outputRG, l_tmpRG; + Row inRow, outRow, tmpRow; + bool distinct; + uint64_t memUsageBefore, memUsageAfter, memDiff; + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; - l_outputRG = outputRG; - dl = inputs[which]; - l_inputRG = inputRGs[which]; - l_inputRG.initRow(&inRow); - l_outputRG.initRow(&outRow); - distinct = distinctFlags[which]; + l_outputRG = outputRG; + dl = inputs[which]; + l_inputRG = inputRGs[which]; + l_inputRG.initRow(&inRow); + l_outputRG.initRow(&outRow); + distinct = distinctFlags[which]; + + if (distinct) + { + l_tmpRG = outputRG; + tmpRGData = &normalizedData[which]; + l_tmpRG.initRow(&tmpRow); + l_tmpRG.setData(tmpRGData); + l_tmpRG.resetRowGroup(0); + l_tmpRG.getRow(0, &tmpRow); + } + else + { + outRGData = RGData(l_outputRG); + l_outputRG.setData(&outRGData); + l_outputRG.resetRowGroup(0); + l_outputRG.getRow(0, &outRow); + } + + try + { + it = dl->getIterator(); + more = dl->next(it, &inRGData); + + if (dlTimes.FirstReadTime().tv_sec == 0) + dlTimes.setFirstReadTime(); + + if (fStartTime == -1) + { + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); + } + + while (more && !cancelled()) + { + /* + normalize each row + if distinct flag is set + copy the row into the output and test for uniqueness + if unique, increment the row count + else + copy the row into the output & inc row count + */ + l_inputRG.setData(&inRGData); + l_inputRG.getRow(0, &inRow); + + if (distinct) + { + memDiff = 0; + l_tmpRG.resetRowGroup(0); + l_tmpRG.getRow(0, &tmpRow); + l_tmpRG.setRowCount(l_inputRG.getRowCount()); + + for (uint32_t i = 0; i < l_inputRG.getRowCount(); i++, inRow.nextRow(), tmpRow.nextRow()) + normalize(inRow, &tmpRow); + + l_tmpRG.getRow(0, &tmpRow); + { + boost::mutex::scoped_lock lk(uniquerMutex); + getOutput(&l_outputRG, &outRow, &outRGData); + memUsageBefore = allocator.getMemUsage(); + + for (uint32_t i = 0; i < l_tmpRG.getRowCount(); i++, tmpRow.nextRow()) + { + pair inserted; + inserted = uniquer->insert(RowPosition(which | RowPosition::normalizedFlag, i)); + + if (inserted.second) + { + copyRow(tmpRow, &outRow); + const_cast(*(inserted.first)) = + RowPosition(rowMemory.size() - 1, l_outputRG.getRowCount()); + memDiff += outRow.getRealSize(); + addToOutput(&outRow, &l_outputRG, true, outRGData); + } + } + + memUsageAfter = allocator.getMemUsage(); + memDiff += (memUsageAfter - memUsageBefore); + } + + if (rm->getMemory(memDiff, sessionMemLimit)) + { + memUsage += memDiff; + } + else + { + fLogger->logMessage(logging::LOG_TYPE_INFO, logging::ERR_UNION_TOO_BIG); + + if (status() == 0) // preserve existing error code + { + errorMessage(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_UNION_TOO_BIG)); + status(logging::ERR_UNION_TOO_BIG); + } + + abort(); + } + } + else + { + for (uint32_t i = 0; i < l_inputRG.getRowCount(); i++, inRow.nextRow()) + { + normalize(inRow, &outRow); + addToOutput(&outRow, &l_outputRG, false, outRGData); + } + } + + more = dl->next(it, &inRGData); + } + } + catch (...) + { + handleException(std::current_exception(), logging::unionStepErr, logging::ERR_UNION_TOO_BIG, + "TupleUnion::readInput()"); + status(logging::unionStepErr); + abort(); + } + + /* make sure that the input was drained before exiting. This can happen if the + query was aborted */ + if (dl && it != numeric_limits::max()) + while (more) + more = dl->next(it, &inRGData); + + { + boost::mutex::scoped_lock lock1(uniquerMutex); + boost::mutex::scoped_lock lock2(sMutex); + + if (!distinct && l_outputRG.getRowCount() > 0) + output->insert(outRGData); if (distinct) { - l_tmpRG = outputRG; - tmpRGData = &normalizedData[which]; - l_tmpRG.initRow(&tmpRow); - l_tmpRG.setData(tmpRGData); - l_tmpRG.resetRowGroup(0); - l_tmpRG.getRow(0, &tmpRow); + getOutput(&l_outputRG, &outRow, &outRGData); + + if (++distinctDone == distinctCount && l_outputRG.getRowCount() > 0) + output->insert(outRGData); } - else + + if (++runnersDone == fInputJobStepAssociation.outSize()) { - outRGData = RGData(l_outputRG); - l_outputRG.setData(&outRGData); - l_outputRG.resetRowGroup(0); - l_outputRG.getRow(0, &outRow); - } - - try - { - - it = dl->getIterator(); - more = dl->next(it, &inRGData); - - if (dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - - if (fStartTime == -1) - { - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); - } - - while (more && !cancelled()) - { - /* - normalize each row - if distinct flag is set - copy the row into the output and test for uniqueness - if unique, increment the row count - else - copy the row into the output & inc row count - */ - l_inputRG.setData(&inRGData); - l_inputRG.getRow(0, &inRow); - - if (distinct) - { - memDiff = 0; - l_tmpRG.resetRowGroup(0); - l_tmpRG.getRow(0, &tmpRow); - l_tmpRG.setRowCount(l_inputRG.getRowCount()); - - for (uint32_t i = 0; i < l_inputRG.getRowCount(); i++, inRow.nextRow(), - tmpRow.nextRow()) - normalize(inRow, &tmpRow); - - l_tmpRG.getRow(0, &tmpRow); - { - boost::mutex::scoped_lock lk(uniquerMutex); - getOutput(&l_outputRG, &outRow, &outRGData); - memUsageBefore = allocator.getMemUsage(); - - for (uint32_t i = 0; i < l_tmpRG.getRowCount(); i++, tmpRow.nextRow()) - { - pair inserted; - inserted = uniquer->insert(RowPosition(which | RowPosition::normalizedFlag, i)); - - if (inserted.second) - { - copyRow(tmpRow, &outRow); - const_cast(*(inserted.first)) = RowPosition(rowMemory.size() - 1, l_outputRG.getRowCount()); - memDiff += outRow.getRealSize(); - addToOutput(&outRow, &l_outputRG, true, outRGData); - } - } - - memUsageAfter = allocator.getMemUsage(); - memDiff += (memUsageAfter - memUsageBefore); - } - - if (rm->getMemory(memDiff, sessionMemLimit)) - { - memUsage += memDiff; - } - else - { - fLogger->logMessage(logging::LOG_TYPE_INFO, logging::ERR_UNION_TOO_BIG); - - if (status() == 0) // preserve existing error code - { - errorMessage(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_UNION_TOO_BIG)); - status(logging::ERR_UNION_TOO_BIG); - } - - abort(); - } - } - else - { - for (uint32_t i = 0; i < l_inputRG.getRowCount(); i++, inRow.nextRow()) - { - normalize(inRow, &outRow); - addToOutput(&outRow, &l_outputRG, false, outRGData); - } - } - - more = dl->next(it, &inRGData); - } - } - catch (...) - { - handleException(std::current_exception(), - logging::unionStepErr, - logging::ERR_UNION_TOO_BIG, - "TupleUnion::readInput()"); - status(logging::unionStepErr); - abort(); - } - - /* make sure that the input was drained before exiting. This can happen if the - query was aborted */ - if (dl && it != numeric_limits::max()) - while (more) - more = dl->next(it, &inRGData); - - { - boost::mutex::scoped_lock lock1(uniquerMutex); - boost::mutex::scoped_lock lock2(sMutex); - - if (!distinct && l_outputRG.getRowCount() > 0) - output->insert(outRGData); - - if (distinct) - { - getOutput(&l_outputRG, &outRow, &outRGData); - - if (++distinctDone == distinctCount && l_outputRG.getRowCount() > 0) - output->insert(outRGData); - } - - if (++runnersDone == fInputJobStepAssociation.outSize()) - { - output->endOfInput(); - - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - if (traceOn()) - { - dlTimes.setLastReadTime(); - dlTimes.setEndOfInputTime(); - - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " - << timeString << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); - } - } + output->endOfInput(); + + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + if (traceOn()) + { + dlTimes.setLastReadTime(); + dlTimes.setEndOfInputTime(); + + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); + } } + } } uint32_t TupleUnion::nextBand(messageqcpp::ByteStream& bs) { - RGData mem; - bool more; - uint32_t ret = 0; + RGData mem; + bool more; + uint32_t ret = 0; - bs.restart(); - more = output->next(outputIt, &mem); + bs.restart(); + more = output->next(outputIt, &mem); - if (more) - outputRG.setData(&mem); - else - { - mem = RGData(outputRG, 0); - outputRG.setData(&mem); - outputRG.resetRowGroup(0); - outputRG.setStatus(status()); - } + if (more) + outputRG.setData(&mem); + else + { + mem = RGData(outputRG, 0); + outputRG.setData(&mem); + outputRG.resetRowGroup(0); + outputRG.setStatus(status()); + } - outputRG.serializeRGData(bs); - ret = outputRG.getRowCount(); + outputRG.serializeRGData(bs); + ret = outputRG.getRowCount(); - return ret; + return ret; } void TupleUnion::getOutput(RowGroup* rg, Row* row, RGData* data) { - if (UNLIKELY(rowMemory.empty())) - { - *data = RGData(*rg); - rg->setData(data); - rg->resetRowGroup(0); - rowMemory.push_back(*data); - } - else - { - *data = rowMemory.back(); - rg->setData(data); - } + if (UNLIKELY(rowMemory.empty())) + { + *data = RGData(*rg); + rg->setData(data); + rg->resetRowGroup(0); + rowMemory.push_back(*data); + } + else + { + *data = rowMemory.back(); + rg->setData(data); + } - rg->getRow(rg->getRowCount(), row); + rg->getRow(rg->getRowCount(), row); } -void TupleUnion::addToOutput(Row* r, RowGroup* rg, bool keepit, - RGData& data) +void TupleUnion::addToOutput(Row* r, RowGroup* rg, bool keepit, RGData& data) { - r->nextRow(); - rg->incRowCount(); - fRowsReturned++; + r->nextRow(); + rg->incRowCount(); + fRowsReturned++; - if (rg->getRowCount() == 8192) + if (rg->getRowCount() == 8192) + { { - { - boost::mutex::scoped_lock lock(sMutex); - output->insert(data); - } - data = RGData(*rg); - rg->setData(&data); - rg->resetRowGroup(0); - rg->getRow(0, r); - - if (keepit) - rowMemory.push_back(data); + boost::mutex::scoped_lock lock(sMutex); + output->insert(data); } + data = RGData(*rg); + rg->setData(&data); + rg->resetRowGroup(0); + rg->getRow(0, r); + + if (keepit) + rowMemory.push_back(data); + } } void TupleUnion::normalize(const Row& in, Row* out) { - uint32_t i; + uint32_t i; - out->setRid(0); + out->setRid(0); - for (i = 0; i < out->getColumnCount(); i++) + for (i = 0; i < out->getColumnCount(); i++) + { + if (in.isNullValue(i)) { - if (in.isNullValue(i)) - { - writeNull(out, i); - continue; - } - - switch (in.getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - if (out->getScale(i) || in.getScale(i)) - goto dec1; - - out->setIntField(in.getIntField(i), i); - break; - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - if (in.getScale(i)) - goto dec1; - - out->setUintField(in.getUintField(i), i); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - ostringstream os; - - if (in.getScale(i)) - { - double d = in.getIntField(i); - d /= exp10(in.getScale(i)); - os.precision(15); - os << d; - } - else - os << in.getIntField(i); - - out->setStringField(os.str(), i); - break; - } - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime"); - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - auto d = in.getScaledSInt64FieldAsXFloat(i); - out->setFloatField((float) d, i); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - auto d = in.getScaledSInt64FieldAsXFloat(i); - out->setDoubleField(d, i); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - auto d = in.getScaledSInt64FieldAsXFloat(i); - out->setLongDoubleField(d, i); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { -dec1: - int diff = out->getScale(i) - in.getScale(i); - - idbassert(diff >= 0); - /* - Signed INT to XDecimal - TODO: - - This code does not handle overflow that may happen on - scale multiplication. Instead of returning a garbage value - we should probably apply saturation here. In long terms we - should implement DECIMAL(65,x) to avoid overflow completely - (so the UNION between DECIMAL and integer can choose a proper - DECIMAL(M,N) result data type to guarantee that any incoming - integer value can fit into it). - */ - if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) - { - int128_t val = datatypes::applySignedScale(in.getIntField(i), diff); - out->setInt128Field(val, i); - } - else - { - int64_t val = datatypes::applySignedScale(in.getIntField(i), diff); - out->setIntField(val, i); - } - - break; - } - - default: - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: integer to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - - break; - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - if (out->getScale(i)) - goto dec2; - - out->setIntField(in.getUintField(i), i); - break; - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - out->setUintField(in.getUintField(i), i); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - ostringstream os; - - if (in.getScale(i)) - { - double d = in.getUintField(i); - d /= exp10(in.getScale(i)); - os.precision(15); - os << d; - } - else - os << in.getUintField(i); - - out->setStringField(os.str(), i); - break; - } - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime"); - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - auto d = in.getScaledUInt64FieldAsXFloat(i); - out->setFloatField((float) d, i); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - auto d = in.getScaledUInt64FieldAsXFloat(i); - out->setDoubleField(d, i); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - auto d = in.getScaledUInt64FieldAsXFloat(i); - out->setLongDoubleField(d, i); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { -dec2: - int diff = out->getScale(i) - in.getScale(i); - - idbassert(diff >= 0); - /* - Unsigned INT to XDecimal - TODO: - - The overflow problem mentioned in the code under label "dec1:" is - also applicable here. - */ - if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) - { - int128_t val = datatypes::applySignedScale(in.getUintField(i), diff); - out->setInt128Field(val, i); - } - else - { - uint64_t val = datatypes::applySignedScale(in.getUintField(i), diff); - out->setUintField(val, i); - } - - break; - } - - default: - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: integer to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - out->setStringField(in.getStringField(i), i); - break; - - default: - { - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: string to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - } - - break; - - case CalpontSystemCatalog::DATE: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::DATE: - out->setIntField(in.getIntField(i), i); - break; - - case CalpontSystemCatalog::DATETIME: - { - uint64_t date = in.getUintField(i); - date &= ~0x3f; // zero the 'spare' field - date <<= 32; - out->setUintField(date, i); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::Date date(in.getUintField(i)); - dataconvert::MySQLTime m_time; - m_time.year = date.year; - m_time.month = date.month; - m_time.day = date.day; - m_time.hour = 0; - m_time.minute = 0; - m_time.second = 0; - m_time.second_part = 0; - - dataconvert::TimeStamp timeStamp; - bool isValid = true; - int64_t seconds = dataconvert::mySQLTimeToGmtSec(m_time, fTimeZone, isValid); - - if (!isValid) - { - timeStamp.reset(); - } - else - { - timeStamp.second = seconds; - timeStamp.msecond = m_time.second_part; - } - - uint64_t outValue = (uint64_t) *(reinterpret_cast(&timeStamp)); - out->setUintField(outValue, i); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - string d = DataConvert::dateToString(in.getUintField(i)); - out->setStringField(d, i); - break; - } - - default: - { - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: date to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - } - - break; - - case CalpontSystemCatalog::DATETIME: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::DATETIME: - out->setIntField(in.getIntField(i), i); - break; - - case CalpontSystemCatalog::DATE: - { - uint64_t val = in.getUintField(i); - val >>= 32; - out->setUintField(val, i); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - uint64_t val = in.getUintField(i); - dataconvert::DateTime dtime(val); - dataconvert::MySQLTime m_time; - dataconvert::TimeStamp timeStamp; - - m_time.year = dtime.year; - m_time.month = dtime.month; - m_time.day = dtime.day; - m_time.hour = dtime.hour; - m_time.minute = dtime.minute; - m_time.second = dtime.second; - m_time.second_part = dtime.msecond; - - bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid); - - if (!isValid) - { - timeStamp.reset(); - } - else - { - timeStamp.second = seconds; - timeStamp.msecond = m_time.second_part; - } - - uint64_t outValue = (uint64_t) *(reinterpret_cast(&timeStamp)); - out->setUintField(outValue, i); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - string d = DataConvert::datetimeToString(in.getUintField(i)); - out->setStringField(d, i); - break; - } - - default: - { - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: datetime to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - } - - break; - - case CalpontSystemCatalog::TIMESTAMP: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TIMESTAMP: - out->setIntField(in.getIntField(i), i); - break; - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - { - uint64_t val = in.getUintField(i); - dataconvert::TimeStamp timestamp(val); - int64_t seconds = timestamp.second; - uint64_t outValue; - - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); - - if (out->getColTypes()[i] == CalpontSystemCatalog::DATE) - { - dataconvert::Date date; - date.year = time.year; - date.month = time.month; - date.day = time.day; - date.spare = 0; - outValue = (uint32_t) *(reinterpret_cast(&date)); - } - else - { - dataconvert::DateTime datetime; - datetime.year = time.year; - datetime.month = time.month; - datetime.day = time.day; - datetime.hour = time.hour; - datetime.minute = time.minute; - datetime.second = time.second; - datetime.msecond = timestamp.msecond; - outValue = (uint64_t) *(reinterpret_cast(&datetime)); - } - - out->setUintField(outValue, i); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - string d = DataConvert::timestampToString(in.getUintField(i), fTimeZone); - out->setStringField(d, i); - break; - } - - default: - { - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: timestamp to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - } - - break; - - case CalpontSystemCatalog::TIME: - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TIME: - out->setIntField(in.getIntField(i), i); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - string d = DataConvert::timeToString(in.getIntField(i)); - out->setStringField(d, i); - break; - } - - default: - { - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: time to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - } - - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double val = (in.getColTypes()[i] == CalpontSystemCatalog::FLOAT ? - in.getFloatField(i) : in.getDoubleField(i)); - - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - if (out->getScale(i)) - goto dec3; - - out->setIntField((int64_t) val, i); - break; - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - out->setUintField((uint64_t) val, i); - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - out->setFloatField(val, i); - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - out->setDoubleField(val, i); - break; - - case CalpontSystemCatalog::LONGDOUBLE: - out->setLongDoubleField(val, i); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - ostringstream os; - os.precision(15); // to match mysql's output - os << val; - out->setStringField(os.str(), i); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { -dec3: /* have to pick a scale to use for the double. using 5... */ - uint32_t scale = 5; - uint64_t ival = (uint64_t) (double) (val * datatypes::scaleDivisor(scale)); - int diff = out->getScale(i) - scale; - // xFLOAT or xDOUBLE to xDECIMAL conversion. Is it really possible? - // TODO: - // Perhaps we should add an assert here that this combination is not possible - // In the current reduction all problems mentioned in the code under - // label "dec1:" are also applicable here. - // TODO: isn't overflow possible below? - ival = datatypes::applySignedScale(ival, diff); - - if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) - out->setInt128Field(ival, i); - else - out->setIntField(ival, i); - - break; - } - - default: - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: floating point to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double val = in.getLongDoubleField(i); - - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - if (out->getScale(i)) - goto dec4; - - out->setIntField((int64_t) val, i); - break; - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - out->setUintField((uint64_t) val, i); - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - out->setFloatField(val, i); - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - out->setDoubleField(val, i); - break; - - case CalpontSystemCatalog::LONGDOUBLE: - out->setLongDoubleField(val, i); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - ostringstream os; - os.precision(15); // to match mysql's output - os << val; - out->setStringField(os.str(), i); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { -dec4: /* have to pick a scale to use for the double. using 5... */ - uint32_t scale = 5; - uint64_t ival = (uint64_t) (double) (val * datatypes::scaleDivisor(scale)); - int diff = out->getScale(i) - scale; - - // LONGDOUBLE to xDECIMAL conversions: is it really possible? - // TODO: - // Perhaps we should add an assert here that this combination is not possible - // In the current reduction all problems mentioned in the code under - // label "dec1:" are also applicable here. - ival = datatypes::applySignedScale(ival, diff); - - if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) - out->setInt128Field(ival, i); - else - out->setIntField(ival, i); - - break; - } - - default: - ostringstream os; - os << "TupleUnion::normalize(): tried an illegal conversion: floating point to " - << out->getColTypes()[i]; - throw logic_error(os.str()); - } - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - int64_t val = 0; - int128_t val128 = 0; - bool isInputWide = false; - - if (in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) - { - in.getInt128Field(i, val128); - isInputWide = true; - } - else - val = in.getIntField(i); - - uint32_t scale = in.getScale(i); - - switch (out->getColTypes()[i]) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (datatypes::isWideDecimalType(out->getColTypes()[i], - out->getColumnWidth(i))) - { - if (out->getScale(i) == scale) - out->setInt128Field(isInputWide ? val128 : val, i); - else if (out->getScale(i) > scale) - { - int128_t temp = datatypes::applySignedScale( - isInputWide ? val128 : val, out->getScale(i) - scale); - out->setInt128Field(temp, i); - } - else // should not happen, the output's scale is the largest - throw logic_error("TupleUnion::normalize(): incorrect scale setting"); - } - // If output type is narrow decimal, input type - // has to be narrow decimal as well. - else - { - if (out->getScale(i) == scale) - out->setIntField(val, i); - else if (out->getScale(i) > scale) - { - int64_t temp = datatypes::applySignedScale( - val, out->getScale(i) - scale); - out->setIntField(temp, i); - } - else // should not happen, the output's scale is the largest - throw logic_error("TupleUnion::normalize(): incorrect scale setting"); - } - - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float fval = ((float) val) / IDB_pow[scale]; - out->setFloatField(fval, i); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dval = ((double) val) / IDB_pow[scale]; - out->setDoubleField(dval, i); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dval = ((long double) val) / IDB_pow[scale]; - out->setLongDoubleField(dval, i); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - default: - { - if (LIKELY(isInputWide)) - { - datatypes::Decimal dec(0, - in.getScale(i), - in.getPrecision(i), - val128); - out->setStringField(dec.toString(), i); - } - else - { - datatypes::Decimal dec(val, - in.getScale(i), - in.getPrecision(i)); - out->setStringField(dec.toString(), i); - } - break; - } - } - - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::VARBINARY: - { - // out->setVarBinaryField(in.getVarBinaryStringField(i), i); // not efficient - out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), i); - break; - } - - default: - { - ostringstream os; - os << "TupleUnion::normalize(): unknown input type (" << in.getColTypes()[i] - << ")"; - cout << os.str() << endl; - throw logic_error(os.str()); - } - } + writeNull(out, i); + continue; } + + switch (in.getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + if (out->getScale(i) || in.getScale(i)) + goto dec1; + + out->setIntField(in.getIntField(i), i); + break; + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + if (in.getScale(i)) + goto dec1; + + out->setUintField(in.getUintField(i), i); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + ostringstream os; + + if (in.getScale(i)) + { + double d = in.getIntField(i); + d /= exp10(in.getScale(i)); + os.precision(15); + os << d; + } + else + os << in.getIntField(i); + + out->setStringField(os.str(), i); + break; + } + + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + throw logic_error( + "TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime"); + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + auto d = in.getScaledSInt64FieldAsXFloat(i); + out->setFloatField((float)d, i); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + auto d = in.getScaledSInt64FieldAsXFloat(i); + out->setDoubleField(d, i); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + auto d = in.getScaledSInt64FieldAsXFloat(i); + out->setLongDoubleField(d, i); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + dec1: + int diff = out->getScale(i) - in.getScale(i); + + idbassert(diff >= 0); + /* + Signed INT to XDecimal + TODO: + - This code does not handle overflow that may happen on + scale multiplication. Instead of returning a garbage value + we should probably apply saturation here. In long terms we + should implement DECIMAL(65,x) to avoid overflow completely + (so the UNION between DECIMAL and integer can choose a proper + DECIMAL(M,N) result data type to guarantee that any incoming + integer value can fit into it). + */ + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + int128_t val = datatypes::applySignedScale(in.getIntField(i), diff); + out->setInt128Field(val, i); + } + else + { + int64_t val = datatypes::applySignedScale(in.getIntField(i), diff); + out->setIntField(val, i); + } + + break; + } + + default: + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: integer to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + + break; + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + if (out->getScale(i)) + goto dec2; + + out->setIntField(in.getUintField(i), i); + break; + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: out->setUintField(in.getUintField(i), i); break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + ostringstream os; + + if (in.getScale(i)) + { + double d = in.getUintField(i); + d /= exp10(in.getScale(i)); + os.precision(15); + os << d; + } + else + os << in.getUintField(i); + + out->setStringField(os.str(), i); + break; + } + + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + throw logic_error( + "TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime"); + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + auto d = in.getScaledUInt64FieldAsXFloat(i); + out->setFloatField((float)d, i); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + auto d = in.getScaledUInt64FieldAsXFloat(i); + out->setDoubleField(d, i); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + auto d = in.getScaledUInt64FieldAsXFloat(i); + out->setLongDoubleField(d, i); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + dec2: + int diff = out->getScale(i) - in.getScale(i); + + idbassert(diff >= 0); + /* + Unsigned INT to XDecimal + TODO: + - The overflow problem mentioned in the code under label "dec1:" is + also applicable here. + */ + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + int128_t val = datatypes::applySignedScale(in.getUintField(i), diff); + out->setInt128Field(val, i); + } + else + { + uint64_t val = datatypes::applySignedScale(in.getUintField(i), diff); + out->setUintField(val, i); + } + + break; + } + + default: + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: integer to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: out->setStringField(in.getStringField(i), i); break; + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: string to " << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + + case CalpontSystemCatalog::DATE: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::DATE: out->setIntField(in.getIntField(i), i); break; + + case CalpontSystemCatalog::DATETIME: + { + uint64_t date = in.getUintField(i); + date &= ~0x3f; // zero the 'spare' field + date <<= 32; + out->setUintField(date, i); + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::Date date(in.getUintField(i)); + dataconvert::MySQLTime m_time; + m_time.year = date.year; + m_time.month = date.month; + m_time.day = date.day; + m_time.hour = 0; + m_time.minute = 0; + m_time.second = 0; + m_time.second_part = 0; + + dataconvert::TimeStamp timeStamp; + bool isValid = true; + int64_t seconds = dataconvert::mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + + if (!isValid) + { + timeStamp.reset(); + } + else + { + timeStamp.second = seconds; + timeStamp.msecond = m_time.second_part; + } + + uint64_t outValue = (uint64_t) * (reinterpret_cast(&timeStamp)); + out->setUintField(outValue, i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::dateToString(in.getUintField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: date to " << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + + case CalpontSystemCatalog::DATETIME: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::DATETIME: out->setIntField(in.getIntField(i), i); break; + + case CalpontSystemCatalog::DATE: + { + uint64_t val = in.getUintField(i); + val >>= 32; + out->setUintField(val, i); + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + uint64_t val = in.getUintField(i); + dataconvert::DateTime dtime(val); + dataconvert::MySQLTime m_time; + dataconvert::TimeStamp timeStamp; + + m_time.year = dtime.year; + m_time.month = dtime.month; + m_time.day = dtime.day; + m_time.hour = dtime.hour; + m_time.minute = dtime.minute; + m_time.second = dtime.second; + m_time.second_part = dtime.msecond; + + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + + if (!isValid) + { + timeStamp.reset(); + } + else + { + timeStamp.second = seconds; + timeStamp.msecond = m_time.second_part; + } + + uint64_t outValue = (uint64_t) * (reinterpret_cast(&timeStamp)); + out->setUintField(outValue, i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::datetimeToString(in.getUintField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: datetime to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + + case CalpontSystemCatalog::TIMESTAMP: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIMESTAMP: out->setIntField(in.getIntField(i), i); break; + + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + { + uint64_t val = in.getUintField(i); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + uint64_t outValue; + + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + + if (out->getColTypes()[i] == CalpontSystemCatalog::DATE) + { + dataconvert::Date date; + date.year = time.year; + date.month = time.month; + date.day = time.day; + date.spare = 0; + outValue = (uint32_t) * (reinterpret_cast(&date)); + } + else + { + dataconvert::DateTime datetime; + datetime.year = time.year; + datetime.month = time.month; + datetime.day = time.day; + datetime.hour = time.hour; + datetime.minute = time.minute; + datetime.second = time.second; + datetime.msecond = timestamp.msecond; + outValue = (uint64_t) * (reinterpret_cast(&datetime)); + } + + out->setUintField(outValue, i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timestampToString(in.getUintField(i), fTimeZone); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: timestamp to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + + case CalpontSystemCatalog::TIME: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIME: out->setIntField(in.getIntField(i), i); break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timeToString(in.getIntField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: time to " << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double val = + (in.getColTypes()[i] == CalpontSystemCatalog::FLOAT ? in.getFloatField(i) : in.getDoubleField(i)); + + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + if (out->getScale(i)) + goto dec3; + + out->setIntField((int64_t)val, i); + break; + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: out->setUintField((uint64_t)val, i); break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: out->setFloatField(val, i); break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: out->setDoubleField(val, i); break; + + case CalpontSystemCatalog::LONGDOUBLE: out->setLongDoubleField(val, i); break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + ostringstream os; + os.precision(15); // to match mysql's output + os << val; + out->setStringField(os.str(), i); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + dec3: /* have to pick a scale to use for the double. using 5... */ + uint32_t scale = 5; + uint64_t ival = (uint64_t)(double)(val * datatypes::scaleDivisor(scale)); + int diff = out->getScale(i) - scale; + // xFLOAT or xDOUBLE to xDECIMAL conversion. Is it really possible? + // TODO: + // Perhaps we should add an assert here that this combination is not possible + // In the current reduction all problems mentioned in the code under + // label "dec1:" are also applicable here. + // TODO: isn't overflow possible below? + ival = datatypes::applySignedScale(ival, diff); + + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(ival, i); + else + out->setIntField(ival, i); + + break; + } + + default: + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: floating point to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double val = in.getLongDoubleField(i); + + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + if (out->getScale(i)) + goto dec4; + + out->setIntField((int64_t)val, i); + break; + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: out->setUintField((uint64_t)val, i); break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: out->setFloatField(val, i); break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: out->setDoubleField(val, i); break; + + case CalpontSystemCatalog::LONGDOUBLE: out->setLongDoubleField(val, i); break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + ostringstream os; + os.precision(15); // to match mysql's output + os << val; + out->setStringField(os.str(), i); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + dec4: /* have to pick a scale to use for the double. using 5... */ + uint32_t scale = 5; + uint64_t ival = (uint64_t)(double)(val * datatypes::scaleDivisor(scale)); + int diff = out->getScale(i) - scale; + + // LONGDOUBLE to xDECIMAL conversions: is it really possible? + // TODO: + // Perhaps we should add an assert here that this combination is not possible + // In the current reduction all problems mentioned in the code under + // label "dec1:" are also applicable here. + ival = datatypes::applySignedScale(ival, diff); + + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(ival, i); + else + out->setIntField(ival, i); + + break; + } + + default: + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: floating point to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + int64_t val = 0; + int128_t val128 = 0; + bool isInputWide = false; + + if (in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + in.getInt128Field(i, val128); + isInputWide = true; + } + else + val = in.getIntField(i); + + uint32_t scale = in.getScale(i); + + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (datatypes::isWideDecimalType(out->getColTypes()[i], out->getColumnWidth(i))) + { + if (out->getScale(i) == scale) + out->setInt128Field(isInputWide ? val128 : val, i); + else if (out->getScale(i) > scale) + { + int128_t temp = datatypes::applySignedScale(isInputWide ? val128 : val, + out->getScale(i) - scale); + out->setInt128Field(temp, i); + } + else // should not happen, the output's scale is the largest + throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + } + // If output type is narrow decimal, input type + // has to be narrow decimal as well. + else + { + if (out->getScale(i) == scale) + out->setIntField(val, i); + else if (out->getScale(i) > scale) + { + int64_t temp = datatypes::applySignedScale(val, out->getScale(i) - scale); + out->setIntField(temp, i); + } + else // should not happen, the output's scale is the largest + throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + } + + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float fval = ((float)val) / IDB_pow[scale]; + out->setFloatField(fval, i); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dval = ((double)val) / IDB_pow[scale]; + out->setDoubleField(dval, i); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double dval = ((long double)val) / IDB_pow[scale]; + out->setLongDoubleField(dval, i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + default: + { + if (LIKELY(isInputWide)) + { + datatypes::Decimal dec(0, in.getScale(i), in.getPrecision(i), val128); + out->setStringField(dec.toString(), i); + } + else + { + datatypes::Decimal dec(val, in.getScale(i), in.getPrecision(i)); + out->setStringField(dec.toString(), i); + } + break; + } + } + + break; + } + + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::VARBINARY: + { + // out->setVarBinaryField(in.getVarBinaryStringField(i), i); // not efficient + out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): unknown input type (" << in.getColTypes()[i] << ")"; + cout << os.str() << endl; + throw logic_error(os.str()); + } + } + } } void TupleUnion::run() { - uint32_t i; + uint32_t i; - boost::mutex::scoped_lock lk(jlLock); + boost::mutex::scoped_lock lk(jlLock); - if (runRan) - return; + if (runRan) + return; - runRan = true; - lk.unlock(); + runRan = true; + lk.unlock(); - for (i = 0; i < fInputJobStepAssociation.outSize(); i++) - inputs.push_back(fInputJobStepAssociation.outAt(i)->rowGroupDL()); + for (i = 0; i < fInputJobStepAssociation.outSize(); i++) + inputs.push_back(fInputJobStepAssociation.outAt(i)->rowGroupDL()); - output = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + output = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fDelivery) + if (fDelivery) + { + outputIt = output->getIterator(); + } + + outputRG.initRow(&row); + outputRG.initRow(&row2); + + distinctCount = 0; + normalizedData.reset(new RGData[inputs.size()]); + + for (i = 0; i < inputs.size(); i++) + { + if (distinctFlags[i]) { - outputIt = output->getIterator(); + distinctCount++; + normalizedData[i].reinit(outputRG); } + } - outputRG.initRow(&row); - outputRG.initRow(&row2); + runners.reserve(inputs.size()); - distinctCount = 0; - normalizedData.reset(new RGData[inputs.size()]); - - for (i = 0; i < inputs.size(); i++) - { - if (distinctFlags[i]) - { - distinctCount++; - normalizedData[i].reinit(outputRG); - } - } - - runners.reserve(inputs.size()); - - for (i = 0; i < inputs.size(); i++) - { - runners.push_back(jobstepThreadPool.invoke(Runner(this, i))); - } + for (i = 0; i < inputs.size(); i++) + { + runners.push_back(jobstepThreadPool.invoke(Runner(this, i))); + } } void TupleUnion::join() { - boost::mutex::scoped_lock lk(jlLock); + boost::mutex::scoped_lock lk(jlLock); - if (joinRan) - return; + if (joinRan) + return; - joinRan = true; - lk.unlock(); + joinRan = true; + lk.unlock(); - jobstepThreadPool.join(runners); - runners.clear(); - uniquer->clear(); - rowMemory.clear(); - rm->returnMemory(memUsage, sessionMemLimit); - memUsage = 0; + jobstepThreadPool.join(runners); + runners.clear(); + uniquer->clear(); + rowMemory.clear(); + rm->returnMemory(memUsage, sessionMemLimit); + memUsage = 0; } const string TupleUnion::toString() const { - ostringstream oss; - oss << "TupleUnion ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId; - oss << " st:" << fStepId; - oss << " in:"; + ostringstream oss; + oss << "TupleUnion ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId; + oss << " st:" << fStepId; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << ((i == 0) ? " " : ", ") << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << ((i == 0) ? " " : ", ") << fInputJobStepAssociation.outAt(i); - oss << " out:"; + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << ((i == 0) ? " " : ", ") << fOutputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << ((i == 0) ? " " : ", ") << fOutputJobStepAssociation.outAt(i); - oss << endl; - - return oss.str(); + oss << endl; + return oss.str(); } void TupleUnion::writeNull(Row* out, uint32_t col) { - switch (out->getColTypes()[col]) + switch (out->getColTypes()[col]) + { + case CalpontSystemCatalog::TINYINT: out->setUintField<1>(joblist::TINYINTNULL, col); break; + + case CalpontSystemCatalog::SMALLINT: out->setUintField<1>(joblist::SMALLINTNULL, col); break; + + case CalpontSystemCatalog::UTINYINT: out->setUintField<1>(joblist::UTINYINTNULL, col); break; + + case CalpontSystemCatalog::USMALLINT: out->setUintField<1>(joblist::USMALLINTNULL, col); break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - case CalpontSystemCatalog::TINYINT: - out->setUintField<1>(joblist::TINYINTNULL, col); - break; + uint32_t len = out->getColumnWidth(col); - case CalpontSystemCatalog::SMALLINT: - out->setUintField<1>(joblist::SMALLINTNULL, col); - break; + switch (len) + { + case 1: out->setUintField<1>(joblist::TINYINTNULL, col); break; - case CalpontSystemCatalog::UTINYINT: - out->setUintField<1>(joblist::UTINYINTNULL, col); - break; + case 2: out->setUintField<1>(joblist::SMALLINTNULL, col); break; - case CalpontSystemCatalog::USMALLINT: - out->setUintField<1>(joblist::USMALLINTNULL, col); - break; + case 4: out->setUintField<4>(joblist::INTNULL, col); break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - uint32_t len = out->getColumnWidth(col); - - switch (len) - { - case 1: - out->setUintField<1>(joblist::TINYINTNULL, col); - break; - - case 2: - out->setUintField<1>(joblist::SMALLINTNULL, col); - break; - - case 4: - out->setUintField<4>(joblist::INTNULL, col); - break; - - case 8: - out->setUintField<8>(joblist::BIGINTNULL, col); - break; - - default: - {} - } - - break; - } - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - out->setUintField<4>(joblist::INTNULL, col); - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - out->setUintField<4>(joblist::UINTNULL, col); - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - out->setUintField<4>(joblist::FLOATNULL, col); - break; - - case CalpontSystemCatalog::DATE: - out->setUintField<4>(joblist::DATENULL, col); - break; - - case CalpontSystemCatalog::BIGINT: - out->setUintField<8>(joblist::BIGINTNULL, col); - break; - - case CalpontSystemCatalog::UBIGINT: - out->setUintField<8>(joblist::UBIGINTNULL, col); - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - out->setUintField<8>(joblist::DOUBLENULL, col); - break; - - case CalpontSystemCatalog::DATETIME: - out->setUintField<8>(joblist::DATETIMENULL, col); - break; - - case CalpontSystemCatalog::TIMESTAMP: - out->setUintField<8>(joblist::TIMESTAMPNULL, col); - break; - - case CalpontSystemCatalog::TIME: - out->setUintField<8>(joblist::TIMENULL, col); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - uint32_t len = out->getColumnWidth(col); - - switch (len) - { - case 1: - out->setUintField<1>(joblist::CHAR1NULL, col); - break; - - case 2: - out->setUintField<2>(joblist::CHAR2NULL, col); - break; - - case 3: - case 4: - out->setUintField<4>(joblist::CHAR4NULL, col); - break; - - case 5: - case 6: - case 7: - case 8: - out->setUintField<8>(joblist::CHAR8NULL, col); - break; - - default: - out->setStringField(joblist::CPNULLSTRMARK, col); - break; - } - - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::VARBINARY: - // could use below if zero length and NULL are treated the same - // out->setVarBinaryField("", col); break; - out->setVarBinaryField(joblist::CPNULLSTRMARK, col); - break; + case 8: out->setUintField<8>(joblist::BIGINTNULL, col); break; default: - { } + { + } + } + + break; } + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: out->setUintField<4>(joblist::INTNULL, col); break; + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: out->setUintField<4>(joblist::UINTNULL, col); break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: out->setUintField<4>(joblist::FLOATNULL, col); break; + + case CalpontSystemCatalog::DATE: out->setUintField<4>(joblist::DATENULL, col); break; + + case CalpontSystemCatalog::BIGINT: out->setUintField<8>(joblist::BIGINTNULL, col); break; + + case CalpontSystemCatalog::UBIGINT: out->setUintField<8>(joblist::UBIGINTNULL, col); break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: out->setUintField<8>(joblist::DOUBLENULL, col); break; + + case CalpontSystemCatalog::DATETIME: out->setUintField<8>(joblist::DATETIMENULL, col); break; + + case CalpontSystemCatalog::TIMESTAMP: out->setUintField<8>(joblist::TIMESTAMPNULL, col); break; + + case CalpontSystemCatalog::TIME: out->setUintField<8>(joblist::TIMENULL, col); break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + uint32_t len = out->getColumnWidth(col); + + switch (len) + { + case 1: out->setUintField<1>(joblist::CHAR1NULL, col); break; + + case 2: out->setUintField<2>(joblist::CHAR2NULL, col); break; + + case 3: + case 4: out->setUintField<4>(joblist::CHAR4NULL, col); break; + + case 5: + case 6: + case 7: + case 8: out->setUintField<8>(joblist::CHAR8NULL, col); break; + + default: out->setStringField(joblist::CPNULLSTRMARK, col); break; + } + + break; + } + + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::VARBINARY: + // could use below if zero length and NULL are treated the same + // out->setVarBinaryField("", col); break; + out->setVarBinaryField(joblist::CPNULLSTRMARK, col); + break; + + default: + { + } + } } void TupleUnion::formatMiniStats() { - ostringstream oss; - oss << "TUS " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "TUS " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); } -} +} // namespace joblist diff --git a/dbcon/joblist/tupleunion.h b/dbcon/joblist/tupleunion.h index c3c226672..50ae6d510 100644 --- a/dbcon/joblist/tupleunion.h +++ b/dbcon/joblist/tupleunion.h @@ -42,168 +42,167 @@ namespace joblist { - - - class TupleUnion : public JobStep, public TupleDeliveryStep { -public: - TupleUnion(execplan::CalpontSystemCatalog::OID tableOID, const JobInfo& jobInfo); - ~TupleUnion(); + public: + TupleUnion(execplan::CalpontSystemCatalog::OID tableOID, const JobInfo& jobInfo); + ~TupleUnion(); - void run(); - void join(); + void run(); + void join(); - const std::string toString() const; - execplan::CalpontSystemCatalog::OID tableOid() const; + const std::string toString() const; + execplan::CalpontSystemCatalog::OID tableOid() const; - void setInputRowGroups(const std::vector&); - void setOutputRowGroup(const rowgroup::RowGroup&); - void setDistinctFlags(const std::vector&); + void setInputRowGroups(const std::vector&); + void setOutputRowGroup(const rowgroup::RowGroup&); + void setDistinctFlags(const std::vector&); - const rowgroup::RowGroup& getOutputRowGroup() const + const rowgroup::RowGroup& getOutputRowGroup() const + { + return outputRG; + } + const rowgroup::RowGroup& getDeliveredRowGroup() const + { + return outputRG; + } + void deliverStringTableRowGroup(bool b) + { + outputRG.setUseStringTable(b); + } + bool deliverStringTableRowGroup() const + { + return outputRG.usesStringTable(); + } + + // @bug 598 for self-join + std::string alias1() const + { + return fAlias1; + } + void alias1(const std::string& alias) + { + fAlias = fAlias1 = alias; + } + std::string alias2() const + { + return fAlias2; + } + void alias2(const std::string& alias) + { + fAlias2 = alias; + } + + std::string view1() const + { + return fView1; + } + void view1(const std::string& vw) + { + fView = fView1 = vw; + } + std::string view2() const + { + return fView2; + } + void view2(const std::string& vw) + { + fView2 = vw; + } + + uint32_t nextBand(messageqcpp::ByteStream& bs); + + private: + struct RowPosition + { + uint64_t group : 48; + uint64_t row : 16; + + inline RowPosition(uint64_t i = 0, uint64_t j = 0) : group(i), row(j){}; + static const uint64_t normalizedFlag = 0x800000000000ULL; // 48th bit is set + }; + + void getOutput(rowgroup::RowGroup* rg, rowgroup::Row* row, rowgroup::RGData* data); + void addToOutput(rowgroup::Row* r, rowgroup::RowGroup* rg, bool keepit, rowgroup::RGData& data); + void normalize(const rowgroup::Row& in, rowgroup::Row* out); + void writeNull(rowgroup::Row* out, uint32_t col); + void readInput(uint32_t); + void formatMiniStats(); + + execplan::CalpontSystemCatalog::OID fTableOID; + // @bug 598 for self-join + std::string fAlias1; + std::string fAlias2; + + std::string fView1; + std::string fView2; + + rowgroup::RowGroup outputRG; + std::vector inputRGs; + std::vector inputs; + RowGroupDL* output; + uint32_t outputIt; + + struct Runner + { + TupleUnion* tu; + uint32_t index; + Runner(TupleUnion* t, uint32_t in) : tu(t), index(in) { - return outputRG; } - const rowgroup::RowGroup& getDeliveredRowGroup() const + void operator()() { - return outputRG; + utils::setThreadName("TUSRunner"); + tu->readInput(index); } - void deliverStringTableRowGroup(bool b) + }; + std::vector runners; // thread pool handles + + struct Hasher + { + TupleUnion* ts; + utils::Hasher_r h; + Hasher(TupleUnion* t) : ts(t) { - outputRG.setUseStringTable(b); } - bool deliverStringTableRowGroup() const + uint64_t operator()(const RowPosition&) const; + }; + struct Eq + { + TupleUnion* ts; + Eq(TupleUnion* t) : ts(t) { - return outputRG.usesStringTable(); } + bool operator()(const RowPosition&, const RowPosition&) const; + }; - // @bug 598 for self-join - std::string alias1() const - { - return fAlias1; - } - void alias1(const std::string& alias) - { - fAlias = fAlias1 = alias; - } - std::string alias2() const - { - return fAlias2; - } - void alias2(const std::string& alias) - { - fAlias2 = alias; - } + typedef std::tr1::unordered_set > Uniquer_t; - std::string view1() const - { - return fView1; - } - void view1(const std::string& vw) - { - fView = fView1 = vw; - } - std::string view2() const - { - return fView2; - } - void view2(const std::string& vw) - { - fView2 = vw; - } + boost::scoped_ptr uniquer; + std::vector rowMemory; + boost::mutex sMutex, uniquerMutex; + uint64_t memUsage; + uint32_t rowLength; + rowgroup::Row row, row2; + std::vector distinctFlags; + ResourceManager* rm; + utils::STLPoolAllocator allocator; + boost::scoped_array normalizedData; - uint32_t nextBand(messageqcpp::ByteStream& bs); + uint32_t runnersDone; + uint32_t distinctCount; + uint32_t distinctDone; + uint64_t fRowsReturned; -private: + // temporary hack to make sure JobList only calls run, join once + boost::mutex jlLock; + bool runRan, joinRan; - struct RowPosition - { - uint64_t group : 48; - uint64_t row : 16; - - inline RowPosition(uint64_t i = 0, uint64_t j = 0) : group(i), row(j) {}; - static const uint64_t normalizedFlag = 0x800000000000ULL; // 48th bit is set - }; - - void getOutput(rowgroup::RowGroup* rg, rowgroup::Row* row, rowgroup::RGData* data); - void addToOutput(rowgroup::Row* r, rowgroup::RowGroup* rg, bool keepit, - rowgroup::RGData& data); - void normalize(const rowgroup::Row& in, rowgroup::Row* out); - void writeNull(rowgroup::Row* out, uint32_t col); - void readInput(uint32_t); - void formatMiniStats(); - - execplan::CalpontSystemCatalog::OID fTableOID; - // @bug 598 for self-join - std::string fAlias1; - std::string fAlias2; - - std::string fView1; - std::string fView2; - - rowgroup::RowGroup outputRG; - std::vector inputRGs; - std::vector inputs; - RowGroupDL* output; - uint32_t outputIt; - - struct Runner - { - TupleUnion* tu; - uint32_t index; - Runner(TupleUnion* t, uint32_t in) : tu(t), index(in) { } - void operator()() - { - utils::setThreadName("TUSRunner"); - tu->readInput(index); - } - }; - std::vector runners; //thread pool handles - - struct Hasher - { - TupleUnion* ts; - utils::Hasher_r h; - Hasher(TupleUnion* t) : ts(t) { } - uint64_t operator()(const RowPosition&) const; - }; - struct Eq - { - TupleUnion* ts; - Eq(TupleUnion* t) : ts(t) { } - bool operator()(const RowPosition&, const RowPosition&) const; - }; - - typedef std::tr1::unordered_set > Uniquer_t; - - boost::scoped_ptr uniquer; - std::vector rowMemory; - boost::mutex sMutex, uniquerMutex; - uint64_t memUsage; - uint32_t rowLength; - rowgroup::Row row, row2; - std::vector distinctFlags; - ResourceManager* rm; - utils::STLPoolAllocator allocator; - boost::scoped_array normalizedData; - - uint32_t runnersDone; - uint32_t distinctCount; - uint32_t distinctDone; - - uint64_t fRowsReturned; - - // temporary hack to make sure JobList only calls run, join once - boost::mutex jlLock; - bool runRan, joinRan; - - boost::shared_ptr sessionMemLimit; - std::string fTimeZone; + boost::shared_ptr sessionMemLimit; + std::string fTimeZone; }; -} +} // namespace joblist #endif diff --git a/dbcon/joblist/unique32generator.cpp b/dbcon/joblist/unique32generator.cpp index 67f07c2ac..1dfe1b717 100644 --- a/dbcon/joblist/unique32generator.cpp +++ b/dbcon/joblist/unique32generator.cpp @@ -33,9 +33,8 @@ namespace joblist { - /* static */ UniqueNumberGenerator* UniqueNumberGenerator::fUnique32Generator = 0; -/* static */ boost::mutex UniqueNumberGenerator::fLock; +/* static */ boost::mutex UniqueNumberGenerator::fLock; //------------------------------------------------------------------------------ // Accessor to singleton handle @@ -43,14 +42,14 @@ namespace joblist /* static */ UniqueNumberGenerator* UniqueNumberGenerator::instance() { - boost::mutex::scoped_lock lk(fLock); + boost::mutex::scoped_lock lk(fLock); - if ( !fUnique32Generator ) - { - fUnique32Generator = new UniqueNumberGenerator(); - } + if (!fUnique32Generator) + { + fUnique32Generator = new UniqueNumberGenerator(); + } - return fUnique32Generator; + return fUnique32Generator; } //------------------------------------------------------------------------------ @@ -61,13 +60,13 @@ UniqueNumberGenerator* UniqueNumberGenerator::instance() /* static */ void UniqueNumberGenerator::deleteInstance() { - boost::mutex::scoped_lock lk(fLock); + boost::mutex::scoped_lock lk(fLock); - if ( fUnique32Generator ) - { - delete fUnique32Generator; - fUnique32Generator = 0; - } + if (fUnique32Generator) + { + delete fUnique32Generator; + fUnique32Generator = 0; + } } //------------------------------------------------------------------------------ @@ -75,12 +74,12 @@ void UniqueNumberGenerator::deleteInstance() //------------------------------------------------------------------------------ uint32_t UniqueNumberGenerator::getUnique32() { - return fDbrm.getUnique32(); + return fDbrm.getUnique32(); } uint64_t UniqueNumberGenerator::getUnique64() { - return fDbrm.getUnique64(); + return fDbrm.getUnique64(); } -} // namespace +} // namespace joblist diff --git a/dbcon/joblist/unique32generator.h b/dbcon/joblist/unique32generator.h index a5ae06453..e741eec6f 100644 --- a/dbcon/joblist/unique32generator.h +++ b/dbcon/joblist/unique32generator.h @@ -34,7 +34,6 @@ namespace joblist { - /** @brief Controls unique 32-bit generation for joblist * * Maintains a single DBRM connection that is used for the generation of all @@ -44,21 +43,25 @@ namespace joblist */ class UniqueNumberGenerator { -public: - static UniqueNumberGenerator* instance(); // singleton accessor - static void deleteInstance();// singleton cleanup - uint32_t getUnique32(); // generate unique 32-bit int - uint64_t getUnique64(); // generate unique 64-bit int + public: + static UniqueNumberGenerator* instance(); // singleton accessor + static void deleteInstance(); // singleton cleanup + uint32_t getUnique32(); // generate unique 32-bit int + uint64_t getUnique64(); // generate unique 64-bit int -private: - UniqueNumberGenerator() { } - ~UniqueNumberGenerator() { } + private: + UniqueNumberGenerator() + { + } + ~UniqueNumberGenerator() + { + } - static UniqueNumberGenerator* fUnique32Generator; - static boost::mutex fLock; - BRM::DBRM fDbrm; + static UniqueNumberGenerator* fUnique32Generator; + static boost::mutex fLock; + BRM::DBRM fDbrm; }; -} // namespace +} // namespace joblist #endif diff --git a/dbcon/joblist/virtualtable.cpp b/dbcon/joblist/virtualtable.cpp index bb6426bb7..dc9bf275c 100644 --- a/dbcon/joblist/virtualtable.cpp +++ b/dbcon/joblist/virtualtable.cpp @@ -17,7 +17,6 @@ // $Id: virtualtable.cpp 6412 2010-03-29 04:58:09Z xlou $ - #include //#define NDEBUG #include @@ -44,127 +43,119 @@ using namespace execplan; #include "jlf_tuplejoblist.h" #include "virtualtable.h" - namespace joblist { - VirtualTable::VirtualTable() : fTableOid(CNX_VTABLE_ID), fVarBinOK(false) { } - void VirtualTable::initialize() { } - void VirtualTable::addColumn(const SRCP& column) { - // As of bug3695, make sure varbinary is not used in subquery. - if (column->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK) - throw runtime_error ("VARBINARY in subquery is not supported."); + // As of bug3695, make sure varbinary is not used in subquery. + if (column->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK) + throw runtime_error("VARBINARY in subquery is not supported."); - AggregateColumn* agc = NULL; - ArithmeticColumn* arc = NULL; - ConstantColumn* cc = NULL; - FunctionColumn* fc = NULL; - SimpleColumn* sc = NULL; - WindowFunctionColumn* wc = NULL; + AggregateColumn* agc = NULL; + ArithmeticColumn* arc = NULL; + ConstantColumn* cc = NULL; + FunctionColumn* fc = NULL; + SimpleColumn* sc = NULL; + WindowFunctionColumn* wc = NULL; - string columnName; - ostringstream oss; - UniqId colId; + string columnName; + ostringstream oss; + UniqId colId; - if ((sc = dynamic_cast(column.get())) != NULL) - { - if (sc->schemaName().empty()) - sc->oid(fTableOid + sc->colPosition() + 1); + if ((sc = dynamic_cast(column.get())) != NULL) + { + if (sc->schemaName().empty()) + sc->oid(fTableOid + sc->colPosition() + 1); - columnName = sc->columnName(); - colId = UniqId(sc); - } - else if ((agc = dynamic_cast(column.get())) != NULL) - { -// oss << agc->functionName() << "_" << agc->expressionId(); -// oss << "Aggregate_" << agc->expressionId(); - columnName = agc->data(); - colId = UniqId(agc->expressionId(), "", "", ""); - } - else if ((wc = dynamic_cast(column.get())) != NULL) - { -// oss << wc->functionName() << "_" << wc->expressionId(); -// oss << "Window_" << wc->expressionId(); - columnName = wc->data(); - colId = UniqId(wc->expressionId(), "", "", ""); - } - else if ((arc = dynamic_cast(column.get())) != NULL) - { -// oss << "Arithmetic_" << arc->expressionId(); - columnName = arc->data(); - colId = UniqId(arc->expressionId(), "", "", ""); - } - else if ((fc = dynamic_cast(column.get())) != NULL) - { -// oss << fc->functionName() << "_" << fc->expressionId(); - columnName = fc->data(); - colId = UniqId(fc->expressionId(), "", "", ""); - } - else if ((cc = dynamic_cast(column.get())) != NULL) - { -// oss << "Constant_" << cc->expressionId(); - columnName = cc->data(); - colId = UniqId(cc->expressionId(), cc->alias(), "", fView); - } - else // new column type has added, but this code is not updated. - { - auto & columnType = *(column.get()); - oss << "not supported column type: " << typeid(columnType).name(); - throw runtime_error(oss.str()); - } + columnName = sc->columnName(); + colId = UniqId(sc); + } + else if ((agc = dynamic_cast(column.get())) != NULL) + { + // oss << agc->functionName() << "_" << agc->expressionId(); + // oss << "Aggregate_" << agc->expressionId(); + columnName = agc->data(); + colId = UniqId(agc->expressionId(), "", "", ""); + } + else if ((wc = dynamic_cast(column.get())) != NULL) + { + // oss << wc->functionName() << "_" << wc->expressionId(); + // oss << "Window_" << wc->expressionId(); + columnName = wc->data(); + colId = UniqId(wc->expressionId(), "", "", ""); + } + else if ((arc = dynamic_cast(column.get())) != NULL) + { + // oss << "Arithmetic_" << arc->expressionId(); + columnName = arc->data(); + colId = UniqId(arc->expressionId(), "", "", ""); + } + else if ((fc = dynamic_cast(column.get())) != NULL) + { + // oss << fc->functionName() << "_" << fc->expressionId(); + columnName = fc->data(); + colId = UniqId(fc->expressionId(), "", "", ""); + } + else if ((cc = dynamic_cast(column.get())) != NULL) + { + // oss << "Constant_" << cc->expressionId(); + columnName = cc->data(); + colId = UniqId(cc->expressionId(), cc->alias(), "", fView); + } + else // new column type has added, but this code is not updated. + { + auto& columnType = *(column.get()); + oss << "not supported column type: " << typeid(columnType).name(); + throw runtime_error(oss.str()); + } - if (columnName.empty()) - columnName = column->alias(); + if (columnName.empty()) + columnName = column->alias(); - SimpleColumn* vc = new SimpleColumn(); - vc->tableName(fName); - vc->tableAlias(fAlias); - vc->columnName(columnName); - vc->alias(column->alias()); - vc->viewName(fView); + SimpleColumn* vc = new SimpleColumn(); + vc->tableName(fName); + vc->tableAlias(fAlias); + vc->columnName(columnName); + vc->alias(column->alias()); + vc->viewName(fView); - uint32_t index = fColumns.size(); - vc->colPosition(index); - vc->oid(fTableOid + index + 1); - vc->resultType(column->resultType()); + uint32_t index = fColumns.size(); + vc->colPosition(index); + vc->oid(fTableOid + index + 1); + vc->resultType(column->resultType()); - SSC ssc(vc); - fColumns.push_back(ssc); - fColumnTypes.push_back(column->resultType()); - fColumnMap.insert(make_pair(colId, index)); + SSC ssc(vc); + fColumns.push_back(ssc); + fColumnTypes.push_back(column->resultType()); + fColumnMap.insert(make_pair(colId, index)); } - const CalpontSystemCatalog::OID& VirtualTable::columnOid(uint32_t i) const { - idbassert(i < fColumns.size()); - return fColumns[i]->oid(); + idbassert(i < fColumns.size()); + return fColumns[i]->oid(); } void VirtualTable::columnType(CalpontSystemCatalog::ColType& type, uint32_t i) { - idbassert(i < fColumnTypes.size()); - fColumnTypes[i] = type; - fColumns[i]->resultType(type); + idbassert(i < fColumnTypes.size()); + fColumnTypes[i] = type; + fColumns[i]->resultType(type); } - const CalpontSystemCatalog::ColType& VirtualTable::columnType(uint32_t i) const { - idbassert(i < fColumnTypes.size()); - return fColumnTypes[i]; + idbassert(i < fColumnTypes.size()); + return fColumnTypes[i]; } - -} +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/virtualtable.h b/dbcon/joblist/virtualtable.h index 3d3495e78..7606cd7bc 100644 --- a/dbcon/joblist/virtualtable.h +++ b/dbcon/joblist/virtualtable.h @@ -17,7 +17,6 @@ // $Id: virtualtable.h 6370 2010-03-18 02:58:09Z xlou $ - /** @file */ #ifndef VIRTUAL_TABLE_H @@ -29,93 +28,93 @@ namespace joblist { - class VirtualTable { -public: - VirtualTable(); - virtual ~VirtualTable() {} + public: + VirtualTable(); + virtual ~VirtualTable() + { + } - virtual void initialize(); - void addColumn(const execplan::SRCP& column); + virtual void initialize(); + void addColumn(const execplan::SRCP& column); - void tableOid(const execplan::CalpontSystemCatalog::OID& oid) - { - fTableOid = oid; - } - const execplan::CalpontSystemCatalog::OID& tableOid() const - { - return fTableOid; - } + void tableOid(const execplan::CalpontSystemCatalog::OID& oid) + { + fTableOid = oid; + } + const execplan::CalpontSystemCatalog::OID& tableOid() const + { + return fTableOid; + } - void name(const std::string& s) - { - fName = s; - } - const std::string& name() const - { - return fName; - } + void name(const std::string& s) + { + fName = s; + } + const std::string& name() const + { + return fName; + } - void alias(const std::string& s) - { - fAlias = s; - } - const std::string& alias() const - { - return fAlias; - } + void alias(const std::string& s) + { + fAlias = s; + } + const std::string& alias() const + { + return fAlias; + } - void view(const std::string& v) - { - fView = v; - } - const std::string& view() const - { - return fView; - } + void view(const std::string& v) + { + fView = v; + } + const std::string& view() const + { + return fView; + } - const std::vector& columns() const - { - return fColumns; - } - const execplan::CalpontSystemCatalog::OID& columnOid(uint32_t i) const; + const std::vector& columns() const + { + return fColumns; + } + const execplan::CalpontSystemCatalog::OID& columnOid(uint32_t i) const; - const std::vector& columnTypes() const - { - return fColumnTypes; - } - void columnType(execplan::CalpontSystemCatalog::ColType& type, uint32_t i); - const execplan::CalpontSystemCatalog::ColType& columnType(uint32_t i) const; + const std::vector& columnTypes() const + { + return fColumnTypes; + } + void columnType(execplan::CalpontSystemCatalog::ColType& type, uint32_t i); + const execplan::CalpontSystemCatalog::ColType& columnType(uint32_t i) const; - const std::map& columnMap() const - { - return fColumnMap; - } + const std::map& columnMap() const + { + return fColumnMap; + } - void varbinaryOK(bool b) - { - fVarBinOK = b; - } - bool varbinaryOK() const - { - return fVarBinOK; - } + void varbinaryOK(bool b) + { + fVarBinOK = b; + } + bool varbinaryOK() const + { + return fVarBinOK; + } -protected: - execplan::CalpontSystemCatalog::OID fTableOid; - std::string fName; - std::string fAlias; - std::string fView; + protected: + execplan::CalpontSystemCatalog::OID fTableOid; + std::string fName; + std::string fAlias; + std::string fView; - std::vector fColumns; - std::vector fColumnTypes; - std::map fColumnMap; + std::vector fColumns; + std::vector fColumnTypes; + std::map fColumnMap; - bool fVarBinOK; + bool fVarBinOK; }; -} +} // namespace joblist #endif // VIRTUAL_TABLE_H - diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp old mode 100755 new mode 100644 index 3aabe6663..1ab8d3984 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -18,7 +18,6 @@ // $Id: windowfunctionstep.cpp 9681 2013-07-11 22:58:05Z xlou $ - //#define NDEBUG #include #include @@ -80,945 +79,854 @@ using namespace joblist; #include "checks.h" - namespace { - - uint64_t getColumnIndex(const SRCP& c, const map& m, JobInfo& jobInfo) { - uint64_t key = getTupleKey(jobInfo, c, true); - const SimpleColumn* sc = dynamic_cast(c.get()); + uint64_t key = getTupleKey(jobInfo, c, true); + const SimpleColumn* sc = dynamic_cast(c.get()); - if (sc != NULL && !sc->schemaName().empty()) + if (sc != NULL && !sc->schemaName().empty()) + { + // special handling for dictionary + CalpontSystemCatalog::ColType ct = sc->colType(); + + // XXX use this before connector sets colType in sc correctly. + // type of pseudo column is set by connector + if (!(dynamic_cast(sc))) { - // special handling for dictionary - CalpontSystemCatalog::ColType ct = sc->colType(); - -//XXX use this before connector sets colType in sc correctly. -// type of pseudo column is set by connector - if (!(dynamic_cast(sc))) - { - ct = jobInfo.csc->colType(sc->oid()); - ct.charsetNumber =sc->colType().charsetNumber; - } - -//X - CalpontSystemCatalog::OID dictOid = isDictCol(ct); - string alias(extractTableAlias(sc)); - - if (dictOid > 0) - { - TupleInfo ti = - setTupleInfo(ct, dictOid, jobInfo, tableOid(sc, jobInfo.csc), sc, alias); - key = ti.key; - } + ct = jobInfo.csc->colType(sc->oid()); + ct.charsetNumber = sc->colType().charsetNumber; } - map::const_iterator j = m.find(key); + // X + CalpontSystemCatalog::OID dictOid = isDictCol(ct); + string alias(extractTableAlias(sc)); - if (j == m.end()) + if (dictOid > 0) { - string name = jobInfo.keyInfo->tupleKeyToName[key]; - cerr << name << " is not in tuple, key=" << key << endl; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), - ERR_WF_COLUMN_MISSING); + TupleInfo ti = setTupleInfo(ct, dictOid, jobInfo, tableOid(sc, jobInfo.csc), sc, alias); + key = ti.key; } + } - return j->second; -} - - + map::const_iterator j = m.find(key); + + if (j == m.end()) + { + string name = jobInfo.keyInfo->tupleKeyToName[key]; + cerr << name << " is not in tuple, key=" << key << endl; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), ERR_WF_COLUMN_MISSING); + } + + return j->second; } +} // namespace namespace joblist { - - -WindowFunctionStep::WindowFunctionStep(const JobInfo& jobInfo) : - JobStep(jobInfo), - fRunner(0), - fCatalog(jobInfo.csc), - fRowsReturned(0), - fEndOfResult(false), - fIsSelect(true), - fUseSSMutex(false), - fUseUFMutex(false), - fInputDL(NULL), - fOutputDL(NULL), - fInputIterator(-1), - fOutputIterator(-1), - fFunctionCount(0), - fTotalThreads(1), - fNextIndex(0), - fMemUsage(0), - fRm(jobInfo.rm), - fSessionMemLimit(jobInfo.umMemLimit) +WindowFunctionStep::WindowFunctionStep(const JobInfo& jobInfo) + : JobStep(jobInfo) + , fRunner(0) + , fCatalog(jobInfo.csc) + , fRowsReturned(0) + , fEndOfResult(false) + , fIsSelect(true) + , fUseSSMutex(false) + , fUseUFMutex(false) + , fInputDL(NULL) + , fOutputDL(NULL) + , fInputIterator(-1) + , fOutputIterator(-1) + , fFunctionCount(0) + , fTotalThreads(1) + , fNextIndex(0) + , fMemUsage(0) + , fRm(jobInfo.rm) + , fSessionMemLimit(jobInfo.umMemLimit) { - fTotalThreads = fRm->windowFunctionThreads(); - fExtendedInfo = "WFS: "; - fQtc.stepParms().stepType = StepTeleStats::T_WFS; + fTotalThreads = fRm->windowFunctionThreads(); + fExtendedInfo = "WFS: "; + fQtc.stepParms().stepType = StepTeleStats::T_WFS; } - WindowFunctionStep::~WindowFunctionStep() { - if (fMemUsage > 0) - fRm->returnMemory(fMemUsage, fSessionMemLimit); + if (fMemUsage > 0) + fRm->returnMemory(fMemUsage, fSessionMemLimit); } - void WindowFunctionStep::run() { - if (fInputJobStepAssociation.outSize() == 0) - throw logic_error("No input data list for window function step."); + if (fInputJobStepAssociation.outSize() == 0) + throw logic_error("No input data list for window function step."); - fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); + fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fInputDL == NULL) - throw logic_error("Input is not a RowGroup data list in window function step."); + if (fInputDL == NULL) + throw logic_error("Input is not a RowGroup data list in window function step."); - fInputIterator = fInputDL->getIterator(); + fInputIterator = fInputDL->getIterator(); - if (fOutputJobStepAssociation.outSize() == 0) - throw logic_error("No output data list for window function step."); + if (fOutputJobStepAssociation.outSize() == 0) + throw logic_error("No output data list for window function step."); - fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); + fOutputDL = fOutputJobStepAssociation.outAt(0)->rowGroupDL(); - if (fOutputDL == NULL) - throw logic_error("Output of window function step is not a RowGroup data list."); + if (fOutputDL == NULL) + throw logic_error("Output of window function step is not a RowGroup data list."); - if (fDelivery == true) - { - fOutputIterator = fOutputDL->getIterator(); - } + if (fDelivery == true) + { + fOutputIterator = fOutputDL->getIterator(); + } - fRunner = jobstepThreadPool.invoke(Runner(this)); + fRunner = jobstepThreadPool.invoke(Runner(this)); } - void WindowFunctionStep::join() { - if (fRunner) - jobstepThreadPool.join(fRunner); + if (fRunner) + jobstepThreadPool.join(fRunner); } - uint32_t WindowFunctionStep::nextBand(messageqcpp::ByteStream& bs) { - RGData rgDataOut; - bool more = false; - uint32_t rowCount = 0; + RGData rgDataOut; + bool more = false; + uint32_t rowCount = 0; - try + try + { + bs.restart(); + + more = fOutputDL->next(fOutputIterator, &rgDataOut); + + if (more && !cancelled()) { - bs.restart(); - + fRowGroupDelivered.setData(&rgDataOut); + fRowGroupDelivered.serializeRGData(bs); + rowCount = fRowGroupDelivered.getRowCount(); + } + else + { + while (more) more = fOutputDL->next(fOutputIterator, &rgDataOut); - if (more && !cancelled()) - { - fRowGroupDelivered.setData(&rgDataOut); - fRowGroupDelivered.serializeRGData(bs); - rowCount = fRowGroupDelivered.getRowCount(); - } - else - { - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - - fEndOfResult = true; - } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_IN_DELIVERY, - logging::ERR_WF_DATA_SET_TOO_BIG, - "WindowFunctionStep::nextBand()"); - while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); - fEndOfResult = true; + fEndOfResult = true; } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_IN_DELIVERY, logging::ERR_WF_DATA_SET_TOO_BIG, + "WindowFunctionStep::nextBand()"); + while (more) + more = fOutputDL->next(fOutputIterator, &rgDataOut); + fEndOfResult = true; + } - if (fEndOfResult) - { - // send an empty / error band - rgDataOut.reinit(fRowGroupDelivered, 0); - fRowGroupDelivered.setData(&rgDataOut); - fRowGroupDelivered.resetRowGroup(0); - fRowGroupDelivered.setStatus(status()); - fRowGroupDelivered.serializeRGData(bs); - } + if (fEndOfResult) + { + // send an empty / error band + rgDataOut.reinit(fRowGroupDelivered, 0); + fRowGroupDelivered.setData(&rgDataOut); + fRowGroupDelivered.resetRowGroup(0); + fRowGroupDelivered.setStatus(status()); + fRowGroupDelivered.serializeRGData(bs); + } - return rowCount; + return rowCount; } - void WindowFunctionStep::setOutputRowGroup(const RowGroup& rg) { - idbassert(0); + idbassert(0); } - const RowGroup& WindowFunctionStep::getOutputRowGroup() const { - return fRowGroupOut; + return fRowGroupOut; } - const RowGroup& WindowFunctionStep::getDeliveredRowGroup() const { - return fRowGroupDelivered; + return fRowGroupDelivered; } - void WindowFunctionStep::deliverStringTableRowGroup(bool b) { - fRowGroupOut.setUseStringTable(b); - fRowGroupDelivered.setUseStringTable(b); + fRowGroupOut.setUseStringTable(b); + fRowGroupDelivered.setUseStringTable(b); } - bool WindowFunctionStep::deliverStringTableRowGroup() const { - idbassert(fRowGroupOut.usesStringTable() == fRowGroupDelivered.usesStringTable()); - return fRowGroupDelivered.usesStringTable(); + idbassert(fRowGroupOut.usesStringTable() == fRowGroupDelivered.usesStringTable()); + return fRowGroupDelivered.usesStringTable(); } - const string WindowFunctionStep::toString() const { - ostringstream oss; - oss << "WindowFunctionStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; + ostringstream oss; + oss << "WindowFunctionStep ses:" << fSessionId << " txn:" << fTxnId << " st:" << fStepId; - oss << " in:"; + oss << " in:"; - for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) - oss << fInputJobStepAssociation.outAt(i); + for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++) + oss << fInputJobStepAssociation.outAt(i); - if (fOutputJobStepAssociation.outSize() > 0) - { - oss << " out:"; + if (fOutputJobStepAssociation.outSize() > 0) + { + oss << " out:"; - for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) - oss << fOutputJobStepAssociation.outAt(i); - } + for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++) + oss << fOutputJobStepAssociation.outAt(i); + } - return oss.str(); + return oss.str(); } -void WindowFunctionStep::AddSimplColumn(const vector& scs, - JobInfo& jobInfo) +void WindowFunctionStep::AddSimplColumn(const vector& scs, JobInfo& jobInfo) { - // append the simple columns if not already projected - set scProjected; + // append the simple columns if not already projected + set scProjected; - for (RetColsVector::iterator i = jobInfo.projectionCols.begin(); - i != jobInfo.projectionCols.end(); - i++) + for (RetColsVector::iterator i = jobInfo.projectionCols.begin(); i != jobInfo.projectionCols.end(); i++) + { + SimpleColumn* sc = dynamic_cast(i->get()); + + if (sc != NULL) { - SimpleColumn* sc = dynamic_cast(i->get()); + if (sc->schemaName().empty()) + sc->oid(joblist::tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - if (sc != NULL) - { - if (sc->schemaName().empty()) - sc->oid(joblist::tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); - - scProjected.insert(UniqId(sc)); - } + scProjected.insert(UniqId(sc)); } + } - for (vector::const_iterator i = scs.begin(); i != scs.end(); i++) + for (vector::const_iterator i = scs.begin(); i != scs.end(); i++) + { + if (scProjected.find(UniqId(*i)) == scProjected.end()) { - if (scProjected.find(UniqId(*i)) == scProjected.end()) - { - jobInfo.windowDels.push_back(SRCP((*i)->clone())); -// MCOL-3343 Enable this if we decide to allow Window Functions to run with -// aggregates with no group by. MariaDB allows this. Nobody else in the world does. -// There will be more work to get it to function if we try this. -// jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); - scProjected.insert(UniqId(*i)); - } + jobInfo.windowDels.push_back(SRCP((*i)->clone())); + // MCOL-3343 Enable this if we decide to allow Window Functions to run with + // aggregates with no group by. MariaDB allows this. Nobody else in the world does. + // There will be more work to get it to function if we try this. + // jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); + scProjected.insert(UniqId(*i)); } + } } void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { - // window functions in select clause, selected or in expression - jobInfo.windowDels = jobInfo.deliveredCols; + // window functions in select clause, selected or in expression + jobInfo.windowDels = jobInfo.deliveredCols; - for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + { + const vector& wcl = (*i)->windowfunctionColumnList(); + RetColsVector wcList; + + for (vector::const_iterator j = wcl.begin(); j != wcl.end(); j++) + wcList.push_back(SRCP((*j)->clone())); + + if (!wcList.empty()) { - const vector& wcl = (*i)->windowfunctionColumnList(); - RetColsVector wcList; - - for (vector::const_iterator j = wcl.begin(); j != wcl.end(); j++) - wcList.push_back(SRCP((*j)->clone())); - - if (!wcList.empty()) - { - jobInfo.windowExps.push_back(*i); - jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); - } - - if (dynamic_cast(i->get()) != NULL) - { - jobInfo.windowCols.push_back(*i); - jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); - } - else if (!wcList.empty()) - { - jobInfo.windowCols.insert(jobInfo.windowCols.end(), wcList.begin(), wcList.end()); - - for (RetColsVector::const_iterator k = wcList.begin(); k < wcList.end(); k++) - { - jobInfo.windowSet.insert(getTupleKey(jobInfo, *k, true)); - } - } + jobInfo.windowExps.push_back(*i); + jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); } - // window functions in order by clause - const CalpontSelectExecutionPlan::OrderByColumnList& orderByCols = csep->orderByCols(); - RetColsVector wcInOrderby; - - for (uint64_t i = 0; i < orderByCols.size(); i++) + if (dynamic_cast(i->get()) != NULL) { - if (orderByCols[i]->orderPos() == (uint64_t)(-1)) - { - WindowFunctionColumn* wc = dynamic_cast(orderByCols[i].get()); - const vector& wcl = orderByCols[i]->windowfunctionColumnList(); - RetColsVector wcList; - - for (vector::const_iterator j = wcl.begin(); j != wcl.end(); j++) - wcList.push_back(SRCP((*j)->clone())); - - if (wc == NULL && wcList.empty()) - continue; - - // an window function or expression of window functions - wcInOrderby.push_back(orderByCols[i]); - - if (!wcList.empty()) - { - jobInfo.windowExps.push_back(orderByCols[i]); - jobInfo.windowSet.insert(getTupleKey(jobInfo, orderByCols[i], true)); - } - - if (dynamic_cast(orderByCols[i].get()) != NULL) - { - jobInfo.windowCols.push_back(orderByCols[i]); - jobInfo.windowSet.insert(getTupleKey(jobInfo, orderByCols[i], true)); - } - else if (!wcList.empty()) - { - jobInfo.windowCols.insert( - jobInfo.windowCols.end(), wcList.begin(), wcList.end()); - - for (RetColsVector::const_iterator k = wcList.begin(); k < wcList.end(); k++) - { - jobInfo.windowSet.insert(getTupleKey(jobInfo, *k, true)); - } - } - } + jobInfo.windowCols.push_back(*i); + jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); } - - // no window function involved in the query - if (jobInfo.windowCols.empty()) - return; - - // Add in the non-window side of arithmetic columns and functions - for (uint64_t i = 0; i < jobInfo.windowExps.size(); i++) + else if (!wcList.empty()) { - const ArithmeticColumn* ac = - dynamic_cast(jobInfo.windowExps[i].get()); - const FunctionColumn* fc = - dynamic_cast(jobInfo.windowExps[i].get()); + jobInfo.windowCols.insert(jobInfo.windowCols.end(), wcList.begin(), wcList.end()); - if (ac != NULL && ac->windowfunctionColumnList().size() > 0) - { - AddSimplColumn(ac->simpleColumnList(), jobInfo); - } - else if (fc != NULL && fc->windowfunctionColumnList().size() > 0) - { - AddSimplColumn(fc->simpleColumnList(), jobInfo); - } + for (RetColsVector::const_iterator k = wcList.begin(); k < wcList.end(); k++) + { + jobInfo.windowSet.insert(getTupleKey(jobInfo, *k, true)); + } } - // reconstruct the delivered column list with auxiliary columns - set colSet; - jobInfo.deliveredCols.resize(0); + } - for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + // window functions in order by clause + const CalpontSelectExecutionPlan::OrderByColumnList& orderByCols = csep->orderByCols(); + RetColsVector wcInOrderby; + + for (uint64_t i = 0; i < orderByCols.size(); i++) + { + if (orderByCols[i]->orderPos() == (uint64_t)(-1)) { + WindowFunctionColumn* wc = dynamic_cast(orderByCols[i].get()); + const vector& wcl = orderByCols[i]->windowfunctionColumnList(); + RetColsVector wcList; + + for (vector::const_iterator j = wcl.begin(); j != wcl.end(); j++) + wcList.push_back(SRCP((*j)->clone())); + + if (wc == NULL && wcList.empty()) + continue; + + // an window function or expression of window functions + wcInOrderby.push_back(orderByCols[i]); + + if (!wcList.empty()) + { + jobInfo.windowExps.push_back(orderByCols[i]); + jobInfo.windowSet.insert(getTupleKey(jobInfo, orderByCols[i], true)); + } + + if (dynamic_cast(orderByCols[i].get()) != NULL) + { + jobInfo.windowCols.push_back(orderByCols[i]); + jobInfo.windowSet.insert(getTupleKey(jobInfo, orderByCols[i], true)); + } + else if (!wcList.empty()) + { + jobInfo.windowCols.insert(jobInfo.windowCols.end(), wcList.begin(), wcList.end()); + + for (RetColsVector::const_iterator k = wcList.begin(); k < wcList.end(); k++) + { + jobInfo.windowSet.insert(getTupleKey(jobInfo, *k, true)); + } + } + } + } + + // no window function involved in the query + if (jobInfo.windowCols.empty()) + return; + + // Add in the non-window side of arithmetic columns and functions + for (uint64_t i = 0; i < jobInfo.windowExps.size(); i++) + { + const ArithmeticColumn* ac = dynamic_cast(jobInfo.windowExps[i].get()); + const FunctionColumn* fc = dynamic_cast(jobInfo.windowExps[i].get()); + + if (ac != NULL && ac->windowfunctionColumnList().size() > 0) + { + AddSimplColumn(ac->simpleColumnList(), jobInfo); + } + else if (fc != NULL && fc->windowfunctionColumnList().size() > 0) + { + AddSimplColumn(fc->simpleColumnList(), jobInfo); + } + } + // reconstruct the delivered column list with auxiliary columns + set colSet; + jobInfo.deliveredCols.resize(0); + + for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + { + jobInfo.deliveredCols.push_back(*i); + uint64_t key = getTupleKey(jobInfo, *i, true); + + // TODO: remove duplicates in select clause + colSet.insert(key); + } + + // add window columns in orderby + for (RetColsVector::iterator i = wcInOrderby.begin(); i < wcInOrderby.end(); i++) + { + jobInfo.deliveredCols.push_back(*i); + uint64_t key = getTupleKey(jobInfo, *i, true); + colSet.insert(key); + } + + // MCOL-3435 We haven't yet checked for aggregate, but we need to know + bool hasAggregation = false; + for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) + { + if (dynamic_cast(jobInfo.deliveredCols[i].get()) != NULL) + { + hasAggregation = true; + break; + } + } + + // add non-duplicate auxiliary columns + RetColsVector colsInAf; + + for (RetColsVector::iterator i = jobInfo.windowCols.begin(); i < jobInfo.windowCols.end(); i++) + { + uint64_t key = getTupleKey(jobInfo, *i, true); + + if (colSet.find(key) == colSet.end()) + jobInfo.deliveredCols.push_back(*i); + + RetColsVector columns = dynamic_cast(i->get())->getColumnList(); + + for (RetColsVector::iterator j = columns.begin(); j < columns.end(); j++) + { + if (dynamic_cast(j->get()) != NULL) + continue; + + key = getTupleKey(jobInfo, *j, true); + + if (colSet.find(key) == colSet.end()) + { + jobInfo.deliveredCols.push_back(*j); + // MCOL-3435 Allow Window Functions to run with aggregates with + // no group by by inserting a group by for window parameters. + if (hasAggregation) + { + // If an argument is an AggregateColumn, don't group by it. + if (dynamic_cast(j->get()) == NULL) + { + bool bFound = false; + for (std::vector::iterator igpc = csep->groupByCols().begin(); + igpc < csep->groupByCols().end(); ++igpc) + { + if ((*igpc)->alias() == (*j)->alias()) + { + bFound = true; + break; + } + } + if (!bFound) + { + csep->groupByCols().push_back(*j); + } + } + } + } + + colSet.insert(key); + } + } + + // for handling order by and limit in outer query + jobInfo.wfqLimitStart = csep->limitStart(); + jobInfo.wfqLimitCount = csep->limitNum(); + csep->limitStart(0); + csep->limitNum(-1); + + if (csep->orderByCols().size() > 0) + { + jobInfo.wfqOrderby = csep->orderByCols(); + csep->orderByCols().clear(); + + // add order by columns + for (RetColsVector::iterator i = jobInfo.wfqOrderby.begin(); i < jobInfo.wfqOrderby.end(); i++) + { + if (dynamic_cast(i->get()) != NULL) + continue; + + uint64_t key = getTupleKey(jobInfo, *i, true); + + if (colSet.find(key) == colSet.end()) jobInfo.deliveredCols.push_back(*i); - uint64_t key = getTupleKey(jobInfo, *i, true); - // TODO: remove duplicates in select clause - colSet.insert(key); - } - - // add window columns in orderby - for (RetColsVector::iterator i = wcInOrderby.begin(); i < wcInOrderby.end(); i++) - { - jobInfo.deliveredCols.push_back(*i); - uint64_t key = getTupleKey(jobInfo, *i, true); - colSet.insert(key); - } - - // MCOL-3435 We haven't yet checked for aggregate, but we need to know - bool hasAggregation = false; - for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) - { - if (dynamic_cast(jobInfo.deliveredCols[i].get()) != NULL) - { - hasAggregation = true; - break; - } - } - - // add non-duplicate auxiliary columns - RetColsVector colsInAf; - - for (RetColsVector::iterator i = jobInfo.windowCols.begin(); i < jobInfo.windowCols.end(); i++) - { - uint64_t key = getTupleKey(jobInfo, *i, true); - - if (colSet.find(key) == colSet.end()) - jobInfo.deliveredCols.push_back(*i); - - RetColsVector columns = dynamic_cast(i->get())->getColumnList(); - - for (RetColsVector::iterator j = columns.begin(); j < columns.end(); j++) - { - if (dynamic_cast(j->get()) != NULL) - continue; - - key = getTupleKey(jobInfo, *j, true); - - if (colSet.find(key) == colSet.end()) - { - jobInfo.deliveredCols.push_back(*j); - // MCOL-3435 Allow Window Functions to run with aggregates with - // no group by by inserting a group by for window parameters. - if (hasAggregation) - { - // If an argument is an AggregateColumn, don't group by it. - if (dynamic_cast(j->get()) == NULL) - { - bool bFound = false; - for (std::vector::iterator igpc = csep->groupByCols().begin(); - igpc < csep->groupByCols().end(); - ++igpc) - { - if ((*igpc)->alias() == (*j)->alias()) - { - bFound = true; - break; - } - } - if (!bFound) - { - csep->groupByCols().push_back(*j); - } - } - } - } - - colSet.insert(key); - } - } - - // for handling order by and limit in outer query - jobInfo.wfqLimitStart = csep->limitStart(); - jobInfo.wfqLimitCount = csep->limitNum(); - csep->limitStart(0); - csep->limitNum(-1); - - if (csep->orderByCols().size() > 0) - { - jobInfo.wfqOrderby = csep->orderByCols(); - csep->orderByCols().clear(); - - // add order by columns - for (RetColsVector::iterator i = jobInfo.wfqOrderby.begin(); i < jobInfo.wfqOrderby.end(); i++) - { - if (dynamic_cast(i->get()) != NULL) - continue; - - uint64_t key = getTupleKey(jobInfo, *i, true); - - if (colSet.find(key) == colSet.end()) - jobInfo.deliveredCols.push_back(*i); - - colSet.insert(key); - } + colSet.insert(key); } + } } - SJSTEP WindowFunctionStep::makeWindowFunctionStep(SJSTEP& step, JobInfo& jobInfo) { - // create a window function step - WindowFunctionStep* ws = new WindowFunctionStep(jobInfo); + // create a window function step + WindowFunctionStep* ws = new WindowFunctionStep(jobInfo); - // connect to the feeding step - JobStepAssociation jsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(execplan::CNX_VTABLE_ID); - spdl->rowGroupDL(dl); - jsa.outAdd(spdl); - ws->inputAssociation(jsa); - ws->stepId(step->stepId() + 1); - step->outputAssociation(jsa); + // connect to the feeding step + JobStepAssociation jsa; + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + dl->OID(execplan::CNX_VTABLE_ID); + spdl->rowGroupDL(dl); + jsa.outAdd(spdl); + ws->inputAssociation(jsa); + ws->stepId(step->stepId() + 1); + step->outputAssociation(jsa); - AnyDataListSPtr spdlOut(new AnyDataList()); - RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); - dlOut->OID(CNX_VTABLE_ID); - spdlOut->rowGroupDL(dlOut); - JobStepAssociation jsaOut; - jsaOut.outAdd(spdlOut); - ws->outputAssociation(jsaOut); + AnyDataListSPtr spdlOut(new AnyDataList()); + RowGroupDL* dlOut = new RowGroupDL(1, jobInfo.fifoSize); + dlOut->OID(CNX_VTABLE_ID); + spdlOut->rowGroupDL(dlOut); + JobStepAssociation jsaOut; + jsaOut.outAdd(spdlOut); + ws->outputAssociation(jsaOut); - // configure the rowgroups and index mapping - TupleDeliveryStep* ds = dynamic_cast(step.get()); - idbassert(ds != NULL); - ws->initialize(ds->getDeliveredRowGroup(), jobInfo); + // configure the rowgroups and index mapping + TupleDeliveryStep* ds = dynamic_cast(step.get()); + idbassert(ds != NULL); + ws->initialize(ds->getDeliveredRowGroup(), jobInfo); - // restore the original delivery coloumns - jobInfo.deliveredCols = jobInfo.windowDels; - jobInfo.nonConstDelCols.clear(); + // restore the original delivery coloumns + jobInfo.deliveredCols = jobInfo.windowDels; + jobInfo.nonConstDelCols.clear(); - for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) - { - if (NULL == dynamic_cast(i->get())) - jobInfo.nonConstDelCols.push_back(*i); - } + for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + { + if (NULL == dynamic_cast(i->get())) + jobInfo.nonConstDelCols.push_back(*i); + } - return SJSTEP(ws); + return SJSTEP(ws); } - void WindowFunctionStep::initialize(const RowGroup& rg, JobInfo& jobInfo) { - if (jobInfo.trace) cout << "Input to WindowFunctionStep: " << rg.toString() << endl; + if (jobInfo.trace) + cout << "Input to WindowFunctionStep: " << rg.toString() << endl; - // query type decides the output by dbroot or partition - // @bug 5631. Insert select should be treated as select - fIsSelect = (jobInfo.queryType == "SELECT" || - jobInfo.queryType == "INSERT_SELECT"); + // query type decides the output by dbroot or partition + // @bug 5631. Insert select should be treated as select + fIsSelect = (jobInfo.queryType == "SELECT" || jobInfo.queryType == "INSERT_SELECT"); - // input row meta data - fRowGroupIn = rg; - fRowGroupIn.initRow(&fRowIn); + // input row meta data + fRowGroupIn = rg; + fRowGroupIn.initRow(&fRowIn); - // make an input map(id, index) - map colIndexMap; - uint64_t colCntIn = rg.getColumnCount(); - const vector& pos = rg.getOffsets(); - const vector& oids = rg.getOIDs(); - const vector& keys = rg.getKeys(); - const vector& types = rg.getColTypes(); - const vector& csNums = rg.getCharsetNumbers(); - const vector& scales = rg.getScale(); - const vector& precisions = rg.getPrecision(); + // make an input map(id, index) + map colIndexMap; + uint64_t colCntIn = rg.getColumnCount(); + const vector& pos = rg.getOffsets(); + const vector& oids = rg.getOIDs(); + const vector& keys = rg.getKeys(); + const vector& types = rg.getColTypes(); + const vector& csNums = rg.getCharsetNumbers(); + const vector& scales = rg.getScale(); + const vector& precisions = rg.getPrecision(); - for (uint64_t i = 0; i < colCntIn; i++) - colIndexMap.insert(make_pair(keys[i], i)); + for (uint64_t i = 0; i < colCntIn; i++) + colIndexMap.insert(make_pair(keys[i], i)); + // @bug6065, window functions that will update string table + int64_t wfsUpdateStringTable = 0; + int64_t wfsUserFunctionCount = 0; + + for (RetColsVector::iterator i = jobInfo.windowCols.begin(); i < jobInfo.windowCols.end(); i++) + { + bool isUDAF = false; + // window function type + WindowFunctionColumn* wc = dynamic_cast(i->get()); + uint64_t ridx = getColumnIndex(*i, colIndexMap, jobInfo); // result index // @bug6065, window functions that will update string table - int64_t wfsUpdateStringTable = 0; - int64_t wfsUserFunctionCount = 0; - - for (RetColsVector::iterator i = jobInfo.windowCols.begin(); i < jobInfo.windowCols.end(); i++) { - bool isUDAF = false; - // window function type - WindowFunctionColumn* wc = dynamic_cast(i->get()); - uint64_t ridx = getColumnIndex(*i, colIndexMap, jobInfo); // result index - // @bug6065, window functions that will update string table - { - CalpontSystemCatalog::ColType rt = wc->resultType(); + CalpontSystemCatalog::ColType rt = wc->resultType(); - if ((types[ridx] == CalpontSystemCatalog::CHAR || - types[ridx] == CalpontSystemCatalog::VARCHAR || - types[ridx] == CalpontSystemCatalog::TEXT || - types[ridx] == CalpontSystemCatalog::VARBINARY || - types[ridx] == CalpontSystemCatalog::BLOB) && - rg.getColumnWidth(ridx) >= jobInfo.stringTableThreshold) - { - ++wfsUpdateStringTable; - } - } - -// if (boost::iequals(wc->functionName(),"UDAF_FUNC") - if (wc->functionName() == "UDAF_FUNC") - { - isUDAF = true; - ++wfsUserFunctionCount; - } - - vector fields; - fields.push_back(ridx); // result - const RetColsVector& parms = wc->functionParms(); - - for (uint64_t i = 0; i < parms.size(); i++) // arguments - { - // skip constant column - if (dynamic_cast(parms[i].get()) == NULL) - fields.push_back(getColumnIndex(parms[i], colIndexMap, jobInfo)); - else - fields.push_back(-1); - } - - // partition & order by - const RetColsVector& partitions = wc->partitions(); - vector eqIdx; - vector peerIdx; - vector sorts; - - for (uint64_t i = 0; i < partitions.size(); i++) - { - // skip constant column - if (dynamic_cast(partitions[i].get()) != NULL) - continue; - - // get column index - uint64_t idx = getColumnIndex(partitions[i], colIndexMap, jobInfo); - eqIdx.push_back(idx); - sorts.push_back(IdbSortSpec(idx, partitions[i]->asc(), partitions[i]->nullsFirst())); - } - - const RetColsVector& orders = wc->orderBy().fOrders; - - for (uint64_t i = 0; i < orders.size(); i++) - { - // skip constant column - if (dynamic_cast(orders[i].get()) != NULL) - continue; - - // get column index - uint64_t idx = getColumnIndex(orders[i], colIndexMap, jobInfo); - peerIdx.push_back(idx); - sorts.push_back(IdbSortSpec(idx, orders[i]->asc(), orders[i]->nullsFirst())); - } - - // functors for sorting - boost::shared_ptr parts(new EqualCompData(eqIdx, rg)); - boost::shared_ptr orderbys(new OrderByData(sorts, rg)); - boost::shared_ptr peers(new EqualCompData(peerIdx, rg)); - - // column type for functor templates - int ct = 0; - - if (isUDAF) - { - ct = wc->getUDAFContext().getResultType(); - } - // make sure index is in range - else if (fields.size() > 1 && fields[1] >= 0 && static_cast(fields[1]) < types.size()) - ct = types[fields[1]]; - - // workaround for functions using "within group (order by)" syntax - string fn = boost::to_upper_copy(wc->functionName()); - - if ( (fn == "MEDIAN" || fn == "PERCENTILE_CONT" || fn == "PERCENTILE_DISC") && - utils::is_nonnegative(peerIdx[0]) && peerIdx[0] < types.size() ) - ct = types[peerIdx[0]]; - - // create the functor based on function name - boost::shared_ptr func = - WindowFunctionType::makeWindowFunction(fn, ct, wc); - - // parse parms after peer and fields are set - // functions may need to set order column index - func->peer(peers); - func->fieldIndex(fields); - func->parseParms(parms); - - // window frame - const WF_Frame& frame = wc->orderBy().fFrame; - int frameUnit = (frame.fIsRange) ? WF__FRAME_RANGE : WF__FRAME_ROWS; - - if (frame.fStart.fFrame == WF_UNBOUNDED_PRECEDING && - frame.fEnd.fFrame == WF_UNBOUNDED_FOLLOWING) - frameUnit = WF__FRAME_ROWS; - - boost::shared_ptr upper = parseFrameBound( - frame.fStart, colIndexMap, orders, peers, jobInfo, !frame.fIsRange, true); - boost::shared_ptr lower = parseFrameBound( - frame.fEnd, colIndexMap, orders, peers, jobInfo, !frame.fIsRange, false); - boost::shared_ptr windows(new WindowFrame(frameUnit, upper, lower)); - func->frameUnit(frameUnit); - - // add to the function list - fFunctions.push_back(boost::shared_ptr( - new WindowFunction(func, parts, orderbys, windows, rg, fRowIn))); - fFunctionCount++; + if ((types[ridx] == CalpontSystemCatalog::CHAR || types[ridx] == CalpontSystemCatalog::VARCHAR || + types[ridx] == CalpontSystemCatalog::TEXT || types[ridx] == CalpontSystemCatalog::VARBINARY || + types[ridx] == CalpontSystemCatalog::BLOB) && + rg.getColumnWidth(ridx) >= jobInfo.stringTableThreshold) + { + ++wfsUpdateStringTable; + } } - // initialize window function expresssions - fExpression = jobInfo.windowExps; - - for (RetColsVector::iterator i = fExpression.begin(); i < fExpression.end(); i++) + // if (boost::iequals(wc->functionName(),"UDAF_FUNC") + if (wc->functionName() == "UDAF_FUNC") { - // output index - (*i)->outputIndex(getColumnIndex(*i, colIndexMap, jobInfo)); - - // map the input indices - const vector& scols = (*i)->simpleColumnList(); - - for (vector::const_iterator j = scols.begin(); j != scols.end(); j++) - { - uint64_t key = getTupleKey(jobInfo, *j); - CalpontSystemCatalog::OID dictOid = joblist::isDictCol((*j)->colType()); - - if (dictOid > 0) - { - key = jobInfo.keyInfo->dictKeyMap[key]; - } - - map::iterator k = colIndexMap.find(key); - - if (k != colIndexMap.end()) - { - (*j)->inputIndex(k->second); - } - else - { - string name = jobInfo.keyInfo->tupleKeyToName[key]; - cerr << name << " is not in tuple, key=" << key << endl; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), - ERR_WF_COLUMN_MISSING); - } - } - - ArithmeticColumn* ac = dynamic_cast((*i).get()); - FunctionColumn* fc = dynamic_cast((*i).get()); - - if (ac != NULL) - { - updateWindowCols(ac->expression(), colIndexMap, jobInfo); - } - else if (fc != NULL) - { -// RetColsVector wcList = fc->windowfunctionColumnList(); -// for (RetColsVector::iterator j = wcList.begin(); j != wcList.end(); j++) -// (*j)->inputIndex(getColumnIndex(*j, colIndexMap, jobInfo)); - funcexp::FunctionParm parms = fc->functionParms(); - - for (vector::iterator j = parms.begin(); j < parms.end(); j++) - updateWindowCols(j->get(), colIndexMap, jobInfo); - } + isUDAF = true; + ++wfsUserFunctionCount; } - // order by part - if (jobInfo.wfqOrderby.size() > 0) + vector fields; + fields.push_back(ridx); // result + const RetColsVector& parms = wc->functionParms(); + + for (uint64_t i = 0; i < parms.size(); i++) // arguments { - // query order by - vector eqIdx; - vector sorts; - const RetColsVector& orderby = jobInfo.wfqOrderby; - - for (uint64_t i = 0; i < orderby.size(); i++) - { - // skip constant column - if (dynamic_cast(orderby[i].get()) != NULL) - continue; - - // get column index - uint64_t idx = getColumnIndex(orderby[i], colIndexMap, jobInfo); - sorts.push_back(IdbSortSpec(idx, orderby[i]->asc(), orderby[i]->nullsFirst())); - } - - fQueryOrderBy.reset(new OrderByData(sorts, rg)); + // skip constant column + if (dynamic_cast(parms[i].get()) == NULL) + fields.push_back(getColumnIndex(parms[i], colIndexMap, jobInfo)); + else + fields.push_back(-1); } - // limit part - fQueryLimitStart = jobInfo.wfqLimitStart; - fQueryLimitCount = jobInfo.wfqLimitCount; + // partition & order by + const RetColsVector& partitions = wc->partitions(); + vector eqIdx; + vector peerIdx; + vector sorts; - // fix the delivered rowgroup data - vector delColIdx; - - for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + for (uint64_t i = 0; i < partitions.size(); i++) { - // find the none constantant columns in the deliver - // leave constants to annexstep for now. - if (dynamic_cast((*i).get()) != NULL) - continue; + // skip constant column + if (dynamic_cast(partitions[i].get()) != NULL) + continue; - delColIdx.push_back(getColumnIndex(*i, colIndexMap, jobInfo)); + // get column index + uint64_t idx = getColumnIndex(partitions[i], colIndexMap, jobInfo); + eqIdx.push_back(idx); + sorts.push_back(IdbSortSpec(idx, partitions[i]->asc(), partitions[i]->nullsFirst())); } - size_t retColCount = delColIdx.size(); - vector pos1; - vector oids1; - vector keys1; - vector scales1; - vector precisions1; - vector types1; - vector csNums1; - pos1.push_back(2); + const RetColsVector& orders = wc->orderBy().fOrders; - for (size_t i = 0; i < retColCount; i++) + for (uint64_t i = 0; i < orders.size(); i++) { - size_t j = delColIdx[i]; - pos1.push_back(pos1[i] + (pos[j + 1] - pos[j])); - oids1.push_back(oids[j]); - keys1.push_back(keys[j]); - scales1.push_back(scales[j]); - precisions1.push_back(precisions[j]); - types1.push_back(types[j]); - csNums1.push_back(csNums[j]); + // skip constant column + if (dynamic_cast(orders[i].get()) != NULL) + continue; + + // get column index + uint64_t idx = getColumnIndex(orders[i], colIndexMap, jobInfo); + peerIdx.push_back(idx); + sorts.push_back(IdbSortSpec(idx, orders[i]->asc(), orders[i]->nullsFirst())); } - fRowGroupDelivered = RowGroup( - retColCount, pos1, oids1, keys1, types1, csNums1, scales1, precisions1, jobInfo.stringTableThreshold); + // functors for sorting + boost::shared_ptr parts(new EqualCompData(eqIdx, rg)); + boost::shared_ptr orderbys(new OrderByData(sorts, rg)); + boost::shared_ptr peers(new EqualCompData(peerIdx, rg)); - if (jobInfo.trace) - cout << "delivered RG: " << fRowGroupDelivered.toString() << endl << endl; + // column type for functor templates + int ct = 0; - if (wfsUpdateStringTable > 1) - fUseSSMutex = true; + if (isUDAF) + { + ct = wc->getUDAFContext().getResultType(); + } + // make sure index is in range + else if (fields.size() > 1 && fields[1] >= 0 && static_cast(fields[1]) < types.size()) + ct = types[fields[1]]; - if (wfsUserFunctionCount > 1) - fUseUFMutex = true; + // workaround for functions using "within group (order by)" syntax + string fn = boost::to_upper_copy(wc->functionName()); - fRowGroupOut = fRowGroupDelivered; + if ((fn == "MEDIAN" || fn == "PERCENTILE_CONT" || fn == "PERCENTILE_DISC") && + utils::is_nonnegative(peerIdx[0]) && peerIdx[0] < types.size()) + ct = types[peerIdx[0]]; + + // create the functor based on function name + boost::shared_ptr func = WindowFunctionType::makeWindowFunction(fn, ct, wc); + + // parse parms after peer and fields are set + // functions may need to set order column index + func->peer(peers); + func->fieldIndex(fields); + func->parseParms(parms); + + // window frame + const WF_Frame& frame = wc->orderBy().fFrame; + int frameUnit = (frame.fIsRange) ? WF__FRAME_RANGE : WF__FRAME_ROWS; + + if (frame.fStart.fFrame == WF_UNBOUNDED_PRECEDING && frame.fEnd.fFrame == WF_UNBOUNDED_FOLLOWING) + frameUnit = WF__FRAME_ROWS; + + boost::shared_ptr upper = + parseFrameBound(frame.fStart, colIndexMap, orders, peers, jobInfo, !frame.fIsRange, true); + boost::shared_ptr lower = + parseFrameBound(frame.fEnd, colIndexMap, orders, peers, jobInfo, !frame.fIsRange, false); + boost::shared_ptr windows(new WindowFrame(frameUnit, upper, lower)); + func->frameUnit(frameUnit); + + // add to the function list + fFunctions.push_back( + boost::shared_ptr(new WindowFunction(func, parts, orderbys, windows, rg, fRowIn))); + fFunctionCount++; + } + + // initialize window function expresssions + fExpression = jobInfo.windowExps; + + for (RetColsVector::iterator i = fExpression.begin(); i < fExpression.end(); i++) + { + // output index + (*i)->outputIndex(getColumnIndex(*i, colIndexMap, jobInfo)); + + // map the input indices + const vector& scols = (*i)->simpleColumnList(); + + for (vector::const_iterator j = scols.begin(); j != scols.end(); j++) + { + uint64_t key = getTupleKey(jobInfo, *j); + CalpontSystemCatalog::OID dictOid = joblist::isDictCol((*j)->colType()); + + if (dictOid > 0) + { + key = jobInfo.keyInfo->dictKeyMap[key]; + } + + map::iterator k = colIndexMap.find(key); + + if (k != colIndexMap.end()) + { + (*j)->inputIndex(k->second); + } + else + { + string name = jobInfo.keyInfo->tupleKeyToName[key]; + cerr << name << " is not in tuple, key=" << key << endl; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), + ERR_WF_COLUMN_MISSING); + } + } + + ArithmeticColumn* ac = dynamic_cast((*i).get()); + FunctionColumn* fc = dynamic_cast((*i).get()); + + if (ac != NULL) + { + updateWindowCols(ac->expression(), colIndexMap, jobInfo); + } + else if (fc != NULL) + { + // RetColsVector wcList = fc->windowfunctionColumnList(); + // for (RetColsVector::iterator j = wcList.begin(); j != wcList.end(); j++) + // (*j)->inputIndex(getColumnIndex(*j, colIndexMap, jobInfo)); + funcexp::FunctionParm parms = fc->functionParms(); + + for (vector::iterator j = parms.begin(); j < parms.end(); j++) + updateWindowCols(j->get(), colIndexMap, jobInfo); + } + } + + // order by part + if (jobInfo.wfqOrderby.size() > 0) + { + // query order by + vector eqIdx; + vector sorts; + const RetColsVector& orderby = jobInfo.wfqOrderby; + + for (uint64_t i = 0; i < orderby.size(); i++) + { + // skip constant column + if (dynamic_cast(orderby[i].get()) != NULL) + continue; + + // get column index + uint64_t idx = getColumnIndex(orderby[i], colIndexMap, jobInfo); + sorts.push_back(IdbSortSpec(idx, orderby[i]->asc(), orderby[i]->nullsFirst())); + } + + fQueryOrderBy.reset(new OrderByData(sorts, rg)); + } + + // limit part + fQueryLimitStart = jobInfo.wfqLimitStart; + fQueryLimitCount = jobInfo.wfqLimitCount; + + // fix the delivered rowgroup data + vector delColIdx; + + for (RetColsVector::iterator i = jobInfo.windowDels.begin(); i < jobInfo.windowDels.end(); i++) + { + // find the none constantant columns in the deliver + // leave constants to annexstep for now. + if (dynamic_cast((*i).get()) != NULL) + continue; + + delColIdx.push_back(getColumnIndex(*i, colIndexMap, jobInfo)); + } + + size_t retColCount = delColIdx.size(); + vector pos1; + vector oids1; + vector keys1; + vector scales1; + vector precisions1; + vector types1; + vector csNums1; + pos1.push_back(2); + + for (size_t i = 0; i < retColCount; i++) + { + size_t j = delColIdx[i]; + pos1.push_back(pos1[i] + (pos[j + 1] - pos[j])); + oids1.push_back(oids[j]); + keys1.push_back(keys[j]); + scales1.push_back(scales[j]); + precisions1.push_back(precisions[j]); + types1.push_back(types[j]); + csNums1.push_back(csNums[j]); + } + + fRowGroupDelivered = RowGroup(retColCount, pos1, oids1, keys1, types1, csNums1, scales1, precisions1, + jobInfo.stringTableThreshold); + + if (jobInfo.trace) + cout << "delivered RG: " << fRowGroupDelivered.toString() << endl << endl; + + if (wfsUpdateStringTable > 1) + fUseSSMutex = true; + + if (wfsUserFunctionCount > 1) + fUseUFMutex = true; + + fRowGroupOut = fRowGroupDelivered; } - void WindowFunctionStep::execute() { - RGData rgData; - Row row; - fRowGroupIn.initRow(&row); - bool more = fInputDL->next(fInputIterator, &rgData); - uint64_t i = 0; // for RowGroup index in the fInRowGroupData + RGData rgData; + Row row; + fRowGroupIn.initRow(&row); + bool more = fInputDL->next(fInputIterator, &rgData); + uint64_t i = 0; // for RowGroup index in the fInRowGroupData - if (traceOn()) dlTimes.setFirstReadTime(); + if (traceOn()) + dlTimes.setFirstReadTime(); - StepTeleStats sts; - sts.query_uuid = fQueryUuid; - sts.step_uuid = fStepUuid; - sts.msg_type = StepTeleStats::ST_START; - sts.total_units_of_work = 1; - postStepStartTele(sts); + StepTeleStats sts; + sts.query_uuid = fQueryUuid; + sts.step_uuid = fStepUuid; + sts.msg_type = StepTeleStats::ST_START; + sts.total_units_of_work = 1; + postStepStartTele(sts); - try + try + { + while (more && !cancelled()) { - while (more && !cancelled()) + fRowGroupIn.setData(&rgData); + fRowGroupIn.getRow(0, &row); + uint64_t rowCnt = fRowGroupIn.getRowCount(); + + if (rowCnt > 0) + { + fInRowGroupData.push_back(rgData); + uint64_t memAdd = fRowGroupIn.getSizeWithStrings() + rowCnt * sizeof(RowPosition); + + if (fRm->getMemory(memAdd, fSessionMemLimit) == false) + throw IDBExcept(ERR_WF_DATA_SET_TOO_BIG); + + fMemUsage += memAdd; + + for (uint64_t j = 0; j < rowCnt; ++j) { - fRowGroupIn.setData(&rgData); - fRowGroupIn.getRow(0, &row); - uint64_t rowCnt = fRowGroupIn.getRowCount(); + if (i > 0x0000FFFFFFFFFFFFULL || j > 0x000000000000FFFFULL) + throw IDBExcept(ERR_WF_DATA_SET_TOO_BIG); - if (rowCnt > 0) - { - fInRowGroupData.push_back(rgData); - uint64_t memAdd = fRowGroupIn.getSizeWithStrings() + rowCnt * sizeof(RowPosition); - - if (fRm->getMemory(memAdd, fSessionMemLimit) == false) - throw IDBExcept(ERR_WF_DATA_SET_TOO_BIG); - - fMemUsage += memAdd; - - for (uint64_t j = 0; j < rowCnt; ++j) - { - if (i > 0x0000FFFFFFFFFFFFULL || j > 0x000000000000FFFFULL) - throw IDBExcept(ERR_WF_DATA_SET_TOO_BIG); - - fRows.push_back(RowPosition(i, j)); - row.nextRow(); - } - - //@bug6065, make StringStore::storeString() thread safe, default to false. - rgData.useStoreStringMutex(fUseSSMutex); - // For the User Data of UDAnF - rgData.useUserDataMutex(fUseUFMutex); - - // window function does not change row count - fRowsReturned += rowCnt; - - i++; - } - - more = fInputDL->next(fInputIterator, &rgData); + fRows.push_back(RowPosition(i, j)); + row.nextRow(); } - } // try - catch (...) - { - handleException(std::current_exception(), - logging::ERR_READ_INPUT_DATALIST, - logging::ERR_WF_DATA_SET_TOO_BIG, - "WindowFunctionStep::execute()"); + + //@bug6065, make StringStore::storeString() thread safe, default to false. + rgData.useStoreStringMutex(fUseSSMutex); + // For the User Data of UDAnF + rgData.useUserDataMutex(fUseUFMutex); + + // window function does not change row count + fRowsReturned += rowCnt; + + i++; + } + + more = fInputDL->next(fInputIterator, &rgData); } + } // try + catch (...) + { + handleException(std::current_exception(), logging::ERR_READ_INPUT_DATALIST, + logging::ERR_WF_DATA_SET_TOO_BIG, "WindowFunctionStep::execute()"); + } - if (traceOn()) - dlTimes.setLastReadTime(); + if (traceOn()) + dlTimes.setLastReadTime(); - // no need for the window function if aborted or result set is empty. - if (cancelled() || fRows.size() == 0) - { - while (more) - more = fInputDL->next(fInputIterator, &rgData); - - fOutputDL->endOfInput(); - - sts.msg_type = StepTeleStats::ST_SUMMARY; - sts.total_units_of_work = sts.units_of_work_completed = 1; - sts.rows = fRowsReturned; - postStepSummaryTele(sts); - - if (traceOn()) - { - dlTimes.setEndOfInputTime(); - printCalTrace(); - } - - return; - } - - // got something to work on - try - { - if (fFunctionCount == 1) - { - doFunction(); - } - else - { - if (fTotalThreads > fFunctionCount) - fTotalThreads = fFunctionCount; - - fFunctionThreads.clear(); - fFunctionThreads.reserve(fTotalThreads); - - for (uint64_t i = 0; i < fTotalThreads && !cancelled(); i++) - fFunctionThreads.push_back(jobstepThreadPool.invoke(WFunction(this))); - - // If cancelled, not all threads are started. - jobstepThreadPool.join(fFunctionThreads); - } - - if (!(cancelled())) - { - if (fIsSelect) - doPostProcessForSelect(); - else - doPostProcessForDml(); - } - - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXECUTE_WINDOW_FUNCTION, - logging::ERR_WF_DATA_SET_TOO_BIG, - "WindowFunctionStep::execute()"); - } + // no need for the window function if aborted or result set is empty. + if (cancelled() || fRows.size() == 0) + { + while (more) + more = fInputDL->next(fInputIterator, &rgData); fOutputDL->endOfInput(); @@ -1029,640 +937,666 @@ void WindowFunctionStep::execute() if (traceOn()) { - dlTimes.setEndOfInputTime(); - printCalTrace(); + dlTimes.setEndOfInputTime(); + printCalTrace(); } return; -} + } + // got something to work on + try + { + if (fFunctionCount == 1) + { + doFunction(); + } + else + { + if (fTotalThreads > fFunctionCount) + fTotalThreads = fFunctionCount; + + fFunctionThreads.clear(); + fFunctionThreads.reserve(fTotalThreads); + + for (uint64_t i = 0; i < fTotalThreads && !cancelled(); i++) + fFunctionThreads.push_back(jobstepThreadPool.invoke(WFunction(this))); + + // If cancelled, not all threads are started. + jobstepThreadPool.join(fFunctionThreads); + } + + if (!(cancelled())) + { + if (fIsSelect) + doPostProcessForSelect(); + else + doPostProcessForDml(); + } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXECUTE_WINDOW_FUNCTION, + logging::ERR_WF_DATA_SET_TOO_BIG, "WindowFunctionStep::execute()"); + } + + fOutputDL->endOfInput(); + + sts.msg_type = StepTeleStats::ST_SUMMARY; + sts.total_units_of_work = sts.units_of_work_completed = 1; + sts.rows = fRowsReturned; + postStepSummaryTele(sts); + + if (traceOn()) + { + dlTimes.setEndOfInputTime(); + printCalTrace(); + } + + return; +} uint64_t WindowFunctionStep::nextFunctionIndex() { - uint64_t idx = atomicInc(&fNextIndex); + uint64_t idx = atomicInc(&fNextIndex); - // return index in the function array - return --idx; + // return index in the function array + return --idx; } - void WindowFunctionStep::doFunction() { - uint64_t i = 0; + uint64_t i = 0; - try + try + { + while (((i = nextFunctionIndex()) < fFunctionCount) && !cancelled()) { - while (((i = nextFunctionIndex()) < fFunctionCount) && !cancelled()) - { - uint64_t memAdd = fRows.size() * sizeof(RowPosition); + uint64_t memAdd = fRows.size() * sizeof(RowPosition); - if (fRm->getMemory(memAdd, fSessionMemLimit) == false) - throw IDBExcept(ERR_WF_DATA_SET_TOO_BIG); + if (fRm->getMemory(memAdd, fSessionMemLimit) == false) + throw IDBExcept(ERR_WF_DATA_SET_TOO_BIG); - fMemUsage += memAdd; + fMemUsage += memAdd; - fFunctions[i]->setCallback(this, i); - (*fFunctions[i].get())(); - } - } - catch (...) - { - handleException(std::current_exception(), - logging::ERR_EXECUTE_WINDOW_FUNCTION, - logging::ERR_WF_DATA_SET_TOO_BIG, - "WindowFunctionStep::doFunction()"); + fFunctions[i]->setCallback(this, i); + (*fFunctions[i].get())(); } + } + catch (...) + { + handleException(std::current_exception(), logging::ERR_EXECUTE_WINDOW_FUNCTION, + logging::ERR_WF_DATA_SET_TOO_BIG, "WindowFunctionStep::doFunction()"); + } } void WindowFunctionStep::doPostProcessForSelect() { - FuncExp* fe = funcexp::FuncExp::instance(); - boost::shared_array mapping = makeMapping(fRowGroupIn, fRowGroupOut); - Row rowIn, rowOut; - fRowGroupIn.initRow(&rowIn); - fRowGroupOut.initRow(&rowOut); - RGData rgData; - vector& rowData = *(fFunctions.back()->fRowData.get()); - int64_t rowsLeft = rowData.size(); - int64_t rowsInRg = 0; - int64_t rgCapacity = 0; + FuncExp* fe = funcexp::FuncExp::instance(); + boost::shared_array mapping = makeMapping(fRowGroupIn, fRowGroupOut); + Row rowIn, rowOut; + fRowGroupIn.initRow(&rowIn); + fRowGroupOut.initRow(&rowOut); + RGData rgData; + vector& rowData = *(fFunctions.back()->fRowData.get()); + int64_t rowsLeft = rowData.size(); + int64_t rowsInRg = 0; + int64_t rgCapacity = 0; - int64_t begin = fQueryLimitStart; - int64_t count = (fQueryLimitCount == (uint64_t) - 1) ? rowsLeft : fQueryLimitCount; - int64_t end = begin + count; - end = (end < rowsLeft) ? end : rowsLeft; - rowsLeft = (end > begin) ? (end - begin) : 0; + int64_t begin = fQueryLimitStart; + int64_t count = (fQueryLimitCount == (uint64_t)-1) ? rowsLeft : fQueryLimitCount; + int64_t end = begin + count; + end = (end < rowsLeft) ? end : rowsLeft; + rowsLeft = (end > begin) ? (end - begin) : 0; - if (fQueryOrderBy.get() != NULL) - sort(rowData.begin(), rowData.size()); + if (fQueryOrderBy.get() != NULL) + sort(rowData.begin(), rowData.size()); - for (int64_t i = begin; i < end; i++) + for (int64_t i = begin; i < end; i++) + { + if (rgData.rowData.get() == NULL) { - if (rgData.rowData.get() == NULL) - { - rgCapacity = ((rowsLeft > 8192) ? 8192 : rowsLeft); - rowsLeft -= rgCapacity; - rgData.reinit(fRowGroupOut, rgCapacity); + rgCapacity = ((rowsLeft > 8192) ? 8192 : rowsLeft); + rowsLeft -= rgCapacity; + rgData.reinit(fRowGroupOut, rgCapacity); - fRowGroupOut.setData(&rgData); - fRowGroupOut.resetRowGroup(0); - fRowGroupOut.setDBRoot(0); // not valid dbroot - fRowGroupOut.getRow(0, &rowOut); - rowsInRg = 0; - } - - rowIn.setData(getPointer(rowData[i], fRowGroupIn, rowIn)); - - // evaluate the window function expressions before apply mapping - if (fExpression.size() > 0) - fe->evaluate(rowIn, fExpression); - - applyMapping(mapping, rowIn, &rowOut); - rowOut.nextRow(); - rowsInRg++; - - if (rowsInRg == rgCapacity) - { - fRowGroupOut.setRowCount(rowsInRg); - fOutputDL->insert(rgData); - rgData.clear(); - } + fRowGroupOut.setData(&rgData); + fRowGroupOut.resetRowGroup(0); + fRowGroupOut.setDBRoot(0); // not valid dbroot + fRowGroupOut.getRow(0, &rowOut); + rowsInRg = 0; } -} + rowIn.setData(getPointer(rowData[i], fRowGroupIn, rowIn)); + + // evaluate the window function expressions before apply mapping + if (fExpression.size() > 0) + fe->evaluate(rowIn, fExpression); + + applyMapping(mapping, rowIn, &rowOut); + rowOut.nextRow(); + rowsInRg++; + + if (rowsInRg == rgCapacity) + { + fRowGroupOut.setRowCount(rowsInRg); + fOutputDL->insert(rgData); + rgData.clear(); + } + } +} void WindowFunctionStep::doPostProcessForDml() { - FuncExp* fe = funcexp::FuncExp::instance(); - boost::shared_array mapping = makeMapping(fRowGroupIn, fRowGroupOut); - Row rowIn, rowOut; - fRowGroupIn.initRow(&rowIn); - fRowGroupOut.initRow(&rowOut); + FuncExp* fe = funcexp::FuncExp::instance(); + boost::shared_array mapping = makeMapping(fRowGroupIn, fRowGroupOut); + Row rowIn, rowOut; + fRowGroupIn.initRow(&rowIn); + fRowGroupOut.initRow(&rowOut); - for (vector::iterator i = fInRowGroupData.begin(); - i < fInRowGroupData.end(); i++) + for (vector::iterator i = fInRowGroupData.begin(); i < fInRowGroupData.end(); i++) + { + fRowGroupIn.setData(&(*i)); + RGData rgData = RGData(fRowGroupIn, fRowGroupIn.getRowCount()); + fRowGroupOut.setData(&rgData); + // @bug 5631. reset rowgroup before the data is populated. + fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); + fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot()); + fRowGroupOut.setRowCount(fRowGroupIn.getRowCount()); + + fRowGroupIn.getRow(0, &rowIn); + fRowGroupOut.getRow(0, &rowOut); + + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) { - fRowGroupIn.setData(&(*i)); - RGData rgData = RGData(fRowGroupIn, fRowGroupIn.getRowCount()); - fRowGroupOut.setData(&rgData); - // @bug 5631. reset rowgroup before the data is populated. - fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - fRowGroupOut.setDBRoot(fRowGroupIn.getDBRoot()); - fRowGroupOut.setRowCount(fRowGroupIn.getRowCount()); + // evaluate the window function expressions before apply mapping + if (fExpression.size() > 0) + fe->evaluate(rowIn, fExpression); - fRowGroupIn.getRow(0, &rowIn); - fRowGroupOut.getRow(0, &rowOut); - - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) - { - // evaluate the window function expressions before apply mapping - if (fExpression.size() > 0) - fe->evaluate(rowIn, fExpression); - - applyMapping(mapping, rowIn, &rowOut); - rowIn.nextRow(); - rowOut.nextRow(); - } - - //fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); - //fRowGroupOut.setRowCount(fRowGroupIn.getRowCount()); - - fOutputDL->insert(rgData); + applyMapping(mapping, rowIn, &rowOut); + rowIn.nextRow(); + rowOut.nextRow(); } + + // fRowGroupOut.resetRowGroup(fRowGroupIn.getBaseRid()); + // fRowGroupOut.setRowCount(fRowGroupIn.getRowCount()); + + fOutputDL->insert(rgData); + } } - -boost::shared_ptr WindowFunctionStep::parseFrameBoundRows( - const execplan::WF_Boundary& b, - const map& m, - JobInfo& jobInfo) +boost::shared_ptr WindowFunctionStep::parseFrameBoundRows(const execplan::WF_Boundary& b, + const map& m, + JobInfo& jobInfo) { - boost::shared_ptr fb; - - if (b.fFrame == WF_CURRENT_ROW) - { - fb.reset(new FrameBoundRow(WF__CURRENT_ROW)); - return fb; - } - - ConstantColumn* cc = dynamic_cast(b.fVal.get()); - - if (cc != NULL) - { - Row dummy; - bool isNull = false; - int val = cc->getIntVal(dummy, isNull); - - if (val >= 0 && isNull == false) - { - int type = (b.fFrame == WF_PRECEDING) ? WF__CONSTANT_PRECEDING : WF__CONSTANT_FOLLOWING; - fb.reset(new FrameBoundConstantRow(type, val)); - } - else - { - string str("NULL"); - - if (!isNull) - { - ostringstream oss; - oss << val; - str = oss.str(); - } - - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, str), - ERR_WF_BOUND_OUT_OF_RANGE); - } - } - else - { - int type = (b.fFrame == WF_PRECEDING) ? WF__EXPRESSION_PRECEDING : WF__EXPRESSION_FOLLOWING; - uint64_t id = getTupleKey(jobInfo, b.fVal); - uint64_t idx = getColumnIndex(b.fVal, m, jobInfo); - TupleInfo ti = getTupleInfo(id, jobInfo); - - switch (ti.dtype) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - { - fb.reset(new FrameBoundExpressionRow(type, id, idx)); - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - fb.reset(new FrameBoundExpressionRow(type, id, idx)); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - fb.reset(new FrameBoundExpressionRow(type, id, idx)); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - fb.reset(new FrameBoundExpressionRow(type, id, idx)); - break; - } - - default: - { - string str = windowfunction::colType2String[ti.dtype]; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_BOUND_TYPE, str), - ERR_WF_INVALID_BOUND_TYPE); - break; - } - } - } + boost::shared_ptr fb; + if (b.fFrame == WF_CURRENT_ROW) + { + fb.reset(new FrameBoundRow(WF__CURRENT_ROW)); return fb; -} + } + ConstantColumn* cc = dynamic_cast(b.fVal.get()); -boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const execplan::WF_Boundary& b, - const map& m, - const vector& o, - JobInfo& jobInfo) -{ - boost::shared_ptr fb; - - if (b.fFrame == WF_CURRENT_ROW) - { - fb.reset(new FrameBoundRange(WF__CURRENT_ROW)); - return fb; - } - - bool isConstant = false; + if (cc != NULL) + { + Row dummy; bool isNull = false; - Row dummy; - ConstantColumn* cc = dynamic_cast(b.fVal.get()); + int val = cc->getIntVal(dummy, isNull); - if (cc != NULL) + if (val >= 0 && isNull == false) { - isConstant = true; - double val = cc->getDoubleVal(dummy, isNull); - - if (val < 0 || isNull) - { - string str("NULL"); - - if (!isNull) - { - ostringstream oss; - oss << val; - str = oss.str(); - } - - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, str), - ERR_WF_BOUND_OUT_OF_RANGE); - } - } - - int type = 0; - vector ids; - vector index; - ids.push_back(getTupleKey(jobInfo, o[0])); - index.push_back(getColumnIndex(o[0], m, jobInfo)); - - if (isConstant) - { - type = (b.fFrame == WF_PRECEDING) ? WF__CONSTANT_PRECEDING : WF__CONSTANT_FOLLOWING; - ids.push_back(-1); // dummy, n/a for constant - index.push_back(-1); // dummy, n/a for constant + int type = (b.fFrame == WF_PRECEDING) ? WF__CONSTANT_PRECEDING : WF__CONSTANT_FOLLOWING; + fb.reset(new FrameBoundConstantRow(type, val)); } else { - type = (b.fFrame == WF_PRECEDING) ? WF__EXPRESSION_PRECEDING : WF__EXPRESSION_FOLLOWING; - ids.push_back(getTupleKey(jobInfo, b.fVal)); - index.push_back(getColumnIndex(b.fVal, m, jobInfo)); + string str("NULL"); + + if (!isNull) + { + ostringstream oss; + oss << val; + str = oss.str(); + } + + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, str), + ERR_WF_BOUND_OUT_OF_RANGE); } - - ids.push_back(getTupleKey(jobInfo, b.fBound)); - index.push_back(getColumnIndex(b.fBound, m, jobInfo)); - - FrameBoundRange* fbr = NULL; - TupleInfo ti = getTupleInfo(ids[0], jobInfo); - bool asc = o[0]->asc(); - bool nlf = o[0]->nullsFirst(); + } + else + { + int type = (b.fFrame == WF_PRECEDING) ? WF__EXPRESSION_PRECEDING : WF__EXPRESSION_FOLLOWING; + uint64_t id = getTupleKey(jobInfo, b.fVal); + uint64_t idx = getColumnIndex(b.fVal, m, jobInfo); + TupleInfo ti = getTupleInfo(id, jobInfo); switch (ti.dtype) { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - { - if (isConstant) - { - int64_t v = cc->getIntVal(dummy, isNull); - fbr = new FrameBoundConstantRange(type, asc, nlf, &v); - fbr->isZero((v == 0)); - } - else - { - fbr = new FrameBoundExpressionRange(type, asc, nlf); - } + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + { + fb.reset(new FrameBoundExpressionRow(type, id, idx)); + break; + } - break; - } + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + fb.reset(new FrameBoundExpressionRow(type, id, idx)); + break; + } - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - if (isConstant) - { - double v = cc->getDoubleVal(dummy, isNull); - fbr = new FrameBoundConstantRange(type, asc, nlf, &v); - fbr->isZero((v == 0.0)); - } - else - { - fbr = new FrameBoundExpressionRange(type, asc, nlf); - } + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + fb.reset(new FrameBoundExpressionRow(type, id, idx)); + break; + } - break; - } + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + fb.reset(new FrameBoundExpressionRow(type, id, idx)); + break; + } - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - if (isConstant) - { - float v = cc->getFloatVal(dummy, isNull); - fbr = new FrameBoundConstantRange(type, asc, nlf, &v); - fbr->isZero((v == 0.0)); - } - else - { - fbr = new FrameBoundExpressionRange(type, asc, nlf); - } - - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - if (isConstant) - { - uint64_t v = cc->getUintVal(dummy, isNull); - fbr = new FrameBoundConstantRange(type, asc, nlf, &v); - fbr->isZero((v == 0)); - } - else - { - fbr = new FrameBoundExpressionRange(type, asc, nlf); - } - - break; - } - - default: - { - string str = windowfunction::colType2String[ti.dtype]; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_BOUND_TYPE, str), - ERR_WF_INVALID_BOUND_TYPE); - break; - } + default: + { + string str = windowfunction::colType2String[ti.dtype]; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_BOUND_TYPE, str), + ERR_WF_INVALID_BOUND_TYPE); + break; + } } + } - fbr->setTupleId(ids); - fbr->setIndex(index); - fb.reset(fbr); - - return fb; + return fb; } +boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const execplan::WF_Boundary& b, + const map& m, + const vector& o, + JobInfo& jobInfo) +{ + boost::shared_ptr fb; + + if (b.fFrame == WF_CURRENT_ROW) + { + fb.reset(new FrameBoundRange(WF__CURRENT_ROW)); + return fb; + } + + bool isConstant = false; + bool isNull = false; + Row dummy; + ConstantColumn* cc = dynamic_cast(b.fVal.get()); + + if (cc != NULL) + { + isConstant = true; + double val = cc->getDoubleVal(dummy, isNull); + + if (val < 0 || isNull) + { + string str("NULL"); + + if (!isNull) + { + ostringstream oss; + oss << val; + str = oss.str(); + } + + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, str), + ERR_WF_BOUND_OUT_OF_RANGE); + } + } + + int type = 0; + vector ids; + vector index; + ids.push_back(getTupleKey(jobInfo, o[0])); + index.push_back(getColumnIndex(o[0], m, jobInfo)); + + if (isConstant) + { + type = (b.fFrame == WF_PRECEDING) ? WF__CONSTANT_PRECEDING : WF__CONSTANT_FOLLOWING; + ids.push_back(-1); // dummy, n/a for constant + index.push_back(-1); // dummy, n/a for constant + } + else + { + type = (b.fFrame == WF_PRECEDING) ? WF__EXPRESSION_PRECEDING : WF__EXPRESSION_FOLLOWING; + ids.push_back(getTupleKey(jobInfo, b.fVal)); + index.push_back(getColumnIndex(b.fVal, m, jobInfo)); + } + + ids.push_back(getTupleKey(jobInfo, b.fBound)); + index.push_back(getColumnIndex(b.fBound, m, jobInfo)); + + FrameBoundRange* fbr = NULL; + TupleInfo ti = getTupleInfo(ids[0], jobInfo); + bool asc = o[0]->asc(); + bool nlf = o[0]->nullsFirst(); + + switch (ti.dtype) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + { + if (isConstant) + { + int64_t v = cc->getIntVal(dummy, isNull); + fbr = new FrameBoundConstantRange(type, asc, nlf, &v); + fbr->isZero((v == 0)); + } + else + { + fbr = new FrameBoundExpressionRange(type, asc, nlf); + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + if (isConstant) + { + double v = cc->getDoubleVal(dummy, isNull); + fbr = new FrameBoundConstantRange(type, asc, nlf, &v); + fbr->isZero((v == 0.0)); + } + else + { + fbr = new FrameBoundExpressionRange(type, asc, nlf); + } + + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + if (isConstant) + { + float v = cc->getFloatVal(dummy, isNull); + fbr = new FrameBoundConstantRange(type, asc, nlf, &v); + fbr->isZero((v == 0.0)); + } + else + { + fbr = new FrameBoundExpressionRange(type, asc, nlf); + } + + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if (isConstant) + { + uint64_t v = cc->getUintVal(dummy, isNull); + fbr = new FrameBoundConstantRange(type, asc, nlf, &v); + fbr->isZero((v == 0)); + } + else + { + fbr = new FrameBoundExpressionRange(type, asc, nlf); + } + + break; + } + + default: + { + string str = windowfunction::colType2String[ti.dtype]; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_BOUND_TYPE, str), + ERR_WF_INVALID_BOUND_TYPE); + break; + } + } + + fbr->setTupleId(ids); + fbr->setIndex(index); + fb.reset(fbr); + + return fb; +} boost::shared_ptr WindowFunctionStep::parseFrameBound(const execplan::WF_Boundary& b, - const map& m, - const vector& o, - const boost::shared_ptr& p, - JobInfo& j, - bool rows, - bool s) + const map& m, + const vector& o, + const boost::shared_ptr& p, + JobInfo& j, bool rows, bool s) { - boost::shared_ptr fb; + boost::shared_ptr fb; - switch (b.fFrame) + switch (b.fFrame) + { + case WF_UNBOUNDED_PRECEDING: { - case WF_UNBOUNDED_PRECEDING: - { - fb.reset(new FrameBound(WF__UNBOUNDED_PRECEDING)); - break; - } - - case WF_UNBOUNDED_FOLLOWING: - { - fb.reset(new FrameBound(WF__UNBOUNDED_FOLLOWING)); - break; - } - - case WF_CURRENT_ROW: - case WF_PRECEDING: - case WF_FOLLOWING: - { - if (rows) - { - fb = parseFrameBoundRows(b, m, j); - } - else - { - fb = parseFrameBoundRange(b, m, o, j); - } - - break; - } - - default: // unknown - { - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_UNKNOWN_BOUND, b.fFrame), - ERR_WF_UNKNOWN_BOUND); - break; - } + fb.reset(new FrameBound(WF__UNBOUNDED_PRECEDING)); + break; } - fb->peer(p); - fb->start(s); + case WF_UNBOUNDED_FOLLOWING: + { + fb.reset(new FrameBound(WF__UNBOUNDED_FOLLOWING)); + break; + } - return fb; + case WF_CURRENT_ROW: + case WF_PRECEDING: + case WF_FOLLOWING: + { + if (rows) + { + fb = parseFrameBoundRows(b, m, j); + } + else + { + fb = parseFrameBoundRange(b, m, o, j); + } + + break; + } + + default: // unknown + { + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_UNKNOWN_BOUND, b.fFrame), + ERR_WF_UNKNOWN_BOUND); + break; + } + } + + fb->peer(p); + fb->start(s); + + return fb; } - -void WindowFunctionStep::updateWindowCols(ReturnedColumn* rc, - const map& m, - JobInfo& jobInfo) +void WindowFunctionStep::updateWindowCols(ReturnedColumn* rc, const map& m, + JobInfo& jobInfo) { - if (rc == NULL) - return; + if (rc == NULL) + return; - ArithmeticColumn* ac = dynamic_cast(rc); - FunctionColumn* fc = dynamic_cast(rc); - SimpleFilter* sf = dynamic_cast(rc); - WindowFunctionColumn* wc = dynamic_cast(rc); + ArithmeticColumn* ac = dynamic_cast(rc); + FunctionColumn* fc = dynamic_cast(rc); + SimpleFilter* sf = dynamic_cast(rc); + WindowFunctionColumn* wc = dynamic_cast(rc); - if (wc) + if (wc) + { + uint64_t key = getExpTupleKey(jobInfo, wc->expressionId()); + map::const_iterator j = m.find(key); + + if (j == m.end()) { - uint64_t key = getExpTupleKey(jobInfo, wc->expressionId()); - map::const_iterator j = m.find(key); + string name = jobInfo.keyInfo->tupleKeyToName[key]; + cerr << name << " is not in tuple, key=" << key << endl; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), ERR_WF_COLUMN_MISSING); + } - if (j == m.end()) - { - string name = jobInfo.keyInfo->tupleKeyToName[key]; - cerr << name << " is not in tuple, key=" << key << endl; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), - ERR_WF_COLUMN_MISSING); - } + wc->inputIndex(j->second); + } + else if (ac) + { + updateWindowCols(ac->expression(), m, jobInfo); + } + else if (fc) + { + funcexp::FunctionParm parms = fc->functionParms(); - wc->inputIndex(j->second); - } - else if (ac) - { - updateWindowCols(ac->expression(), m, jobInfo); - } - else if (fc) - { - funcexp::FunctionParm parms = fc->functionParms(); - - for (vector::iterator i = parms.begin(); i < parms.end(); i++) - updateWindowCols(i->get(), m, jobInfo); - } - else if (sf) - { - updateWindowCols(sf->lhs(), m, jobInfo); - updateWindowCols(sf->rhs(), m, jobInfo); - } + for (vector::iterator i = parms.begin(); i < parms.end(); i++) + updateWindowCols(i->get(), m, jobInfo); + } + else if (sf) + { + updateWindowCols(sf->lhs(), m, jobInfo); + updateWindowCols(sf->rhs(), m, jobInfo); + } } - -void WindowFunctionStep::updateWindowCols(ParseTree* pt, - const map& m, - JobInfo& jobInfo) +void WindowFunctionStep::updateWindowCols(ParseTree* pt, const map& m, JobInfo& jobInfo) { - if (pt == NULL) - return; + if (pt == NULL) + return; - updateWindowCols(pt->left(), m, jobInfo); - updateWindowCols(pt->right(), m, jobInfo); + updateWindowCols(pt->left(), m, jobInfo); + updateWindowCols(pt->right(), m, jobInfo); - TreeNode* tn = pt->data(); - ArithmeticColumn* ac = dynamic_cast(tn); - FunctionColumn* fc = dynamic_cast(tn); - SimpleFilter* sf = dynamic_cast(tn); - WindowFunctionColumn* wc = dynamic_cast(tn); + TreeNode* tn = pt->data(); + ArithmeticColumn* ac = dynamic_cast(tn); + FunctionColumn* fc = dynamic_cast(tn); + SimpleFilter* sf = dynamic_cast(tn); + WindowFunctionColumn* wc = dynamic_cast(tn); - if (wc) + if (wc) + { + uint64_t key = getExpTupleKey(jobInfo, wc->expressionId()); + map::const_iterator j = m.find(key); + + if (j == m.end()) { - uint64_t key = getExpTupleKey(jobInfo, wc->expressionId()); - map::const_iterator j = m.find(key); + string name = jobInfo.keyInfo->tupleKeyToName[key]; + cerr << name << " is not in tuple, key=" << key << endl; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), ERR_WF_COLUMN_MISSING); + } - if (j == m.end()) - { - string name = jobInfo.keyInfo->tupleKeyToName[key]; - cerr << name << " is not in tuple, key=" << key << endl; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_COLUMN_MISSING, name), - ERR_WF_COLUMN_MISSING); - } + wc->inputIndex(j->second); + } + else if (ac) + { + updateWindowCols(ac->expression(), m, jobInfo); + } + else if (fc) + { + funcexp::FunctionParm parms = fc->functionParms(); - wc->inputIndex(j->second); - } - else if (ac) - { - updateWindowCols(ac->expression(), m, jobInfo); - } - else if (fc) - { - funcexp::FunctionParm parms = fc->functionParms(); - - for (vector::iterator i = parms.begin(); i < parms.end(); i++) - updateWindowCols(i->get(), m, jobInfo); - } - else if (sf) - { - updateWindowCols(sf->lhs(), m, jobInfo); - updateWindowCols(sf->rhs(), m, jobInfo); - } + for (vector::iterator i = parms.begin(); i < parms.end(); i++) + updateWindowCols(i->get(), m, jobInfo); + } + else if (sf) + { + updateWindowCols(sf->lhs(), m, jobInfo); + updateWindowCols(sf->rhs(), m, jobInfo); + } } - void WindowFunctionStep::sort(std::vector::iterator v, uint64_t n) { - // recursive function termination condition. - if (n < 2 || cancelled()) - return; + // recursive function termination condition. + if (n < 2 || cancelled()) + return; - RowPosition p = *(v + n / 2); // pivot value - vector::iterator l = v; // low address - vector::iterator h = v + (n - 1); // high address + RowPosition p = *(v + n / 2); // pivot value + vector::iterator l = v; // low address + vector::iterator h = v + (n - 1); // high address - while (l <= h && !cancelled()) + while (l <= h && !cancelled()) + { + // Can use while here, but need check boundary and cancel status. + if (fQueryOrderBy->operator()(getPointer(*l), getPointer(p))) { - // Can use while here, but need check boundary and cancel status. - if (fQueryOrderBy->operator()(getPointer(*l), getPointer(p))) - { - l++; - } - else if (fQueryOrderBy->operator()(getPointer(p), getPointer(*h))) - { - h--; - } - else - { - RowPosition t = *l; // temp value for swap - *l++ = *h; - *h-- = t; - } + l++; } + else if (fQueryOrderBy->operator()(getPointer(p), getPointer(*h))) + { + h--; + } + else + { + RowPosition t = *l; // temp value for swap + *l++ = *h; + *h-- = t; + } + } - sort(v, std::distance(v, h) + 1); - sort(l, std::distance(l, v) + n); + sort(v, std::distance(v, h) + 1); + sort(l, std::distance(l, v) + n); } - void WindowFunctionStep::printCalTrace() { - time_t t = time (0); - char timeString[50]; - ctime_r (&t, timeString); - timeString[strlen (timeString ) - 1] = '\0'; - ostringstream logStr; - logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString - << "; total rows returned-" << fRowsReturned << endl - << "\t1st read " << dlTimes.FirstReadTimeString() - << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) - << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl - << "\tJob completion status " << status() << endl; - logEnd(logStr.str().c_str()); - fExtendedInfo += logStr.str(); - formatMiniStats(); + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream logStr; + logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at " << timeString + << "; total rows returned-" << fRowsReturned << endl + << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() + << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) + << "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl + << "\tJob completion status " << status() << endl; + logEnd(logStr.str().c_str()); + fExtendedInfo += logStr.str(); + formatMiniStats(); } - void WindowFunctionStep::formatMiniStats() { - ostringstream oss; - oss << "WFS " - << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " - << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; - fMiniInfo += oss.str(); + ostringstream oss; + oss << "WFS " + << "UM " + << "- " + << "- " + << "- " + << "- " + << "- " + << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " + << fRowsReturned << " "; + fMiniInfo += oss.str(); } - -} //namespace +} // namespace joblist // vim:ts=4 sw=4: - diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index 6cb51cb57..e2f03eb4e 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -18,7 +18,6 @@ // $Id: windowfunctionstep.h 9722 2013-07-25 21:29:10Z pleblanc $ - #ifndef JOBLIST_WINDOWFUNCTIONSTEP_H #define JOBLIST_WINDOWFUNCTIONSTEP_H @@ -34,200 +33,198 @@ namespace execplan class CalpontSelectExecutionPlan; class ParseTree; class ReturnedColumn; -}; - +}; // namespace execplan namespace windowfunction { // forward reference class WindowFunction; class FrameBound; -}; - +}; // namespace windowfunction namespace ordering { // forward reference class EqualCompData; -}; - +}; // namespace ordering namespace joblist { - // forward reference struct JobInfo; class ResourceManager; - struct RowPosition { - uint64_t fGroupId : 48; - uint64_t fRowId : 16; + uint64_t fGroupId : 48; + uint64_t fRowId : 16; - inline RowPosition(uint64_t i = 0, uint64_t j = 0) : fGroupId(i), fRowId(j) {}; + inline RowPosition(uint64_t i = 0, uint64_t j = 0) : fGroupId(i), fRowId(j){}; }; - /** @brief class WindowFunctionStep * */ class WindowFunctionStep : public JobStep, public TupleDeliveryStep { -public: - /** @brief WindowFunctionStep constructor - * @param - * @param - */ - WindowFunctionStep(const JobInfo&); + public: + /** @brief WindowFunctionStep constructor + * @param + * @param + */ + WindowFunctionStep(const JobInfo&); - /** @brief WindowFunctionStep destructor - */ - virtual ~WindowFunctionStep(); + /** @brief WindowFunctionStep destructor + */ + virtual ~WindowFunctionStep(); - /** @brief virtual methods - */ - void run(); - void join(); - const std::string toString() const; - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + /** @brief virtual methods + */ + void run(); + void join(); + const std::string toString() const; + void setOutputRowGroup(const rowgroup::RowGroup&); + const rowgroup::RowGroup& getOutputRowGroup() const; + const rowgroup::RowGroup& getDeliveredRowGroup() const; + void deliverStringTableRowGroup(bool b); + bool deliverStringTableRowGroup() const; + uint32_t nextBand(messageqcpp::ByteStream& bs); - /** @brief initialize methods - */ - void initialize(const rowgroup::RowGroup& rg, JobInfo& jobInfo); + /** @brief initialize methods + */ + void initialize(const rowgroup::RowGroup& rg, JobInfo& jobInfo); - static void checkWindowFunction(execplan::CalpontSelectExecutionPlan*, JobInfo&); - static SJSTEP makeWindowFunctionStep(SJSTEP&, JobInfo&); + static void checkWindowFunction(execplan::CalpontSelectExecutionPlan*, JobInfo&); + static SJSTEP makeWindowFunctionStep(SJSTEP&, JobInfo&); - // for WindowFunction and WindowFunctionWrapper callback - const std::vector& getRowData() const + // for WindowFunction and WindowFunctionWrapper callback + const std::vector& getRowData() const + { + return fRows; + } + // for string table + rowgroup::Row::Pointer getPointer(RowPosition& pos) + { + fRowGroupIn.setData(&(fInRowGroupData[pos.fGroupId])); + fRowGroupIn.getRow(pos.fRowId, &fRowIn); + return fRowIn.getPointer(); + } + + rowgroup::Row::Pointer getPointer(RowPosition& pos, rowgroup::RowGroup& rg, rowgroup::Row& row) + { + rg.setData(&(fInRowGroupData[pos.fGroupId])); + rg.getRow(pos.fRowId, &row); + return row.getPointer(); + } + + private: + void execute(); + void doFunction(); + void doPostProcessForSelect(); + void doPostProcessForDml(); + + uint64_t nextFunctionIndex(); + + boost::shared_ptr parseFrameBound( + const execplan::WF_Boundary&, const std::map&, const std::vector&, + const boost::shared_ptr&, JobInfo&, bool, bool); + boost::shared_ptr parseFrameBoundRows(const execplan::WF_Boundary&, + const std::map&, + JobInfo&); + boost::shared_ptr parseFrameBoundRange(const execplan::WF_Boundary&, + const std::map&, + const std::vector&, + JobInfo&); + void updateWindowCols(execplan::ParseTree*, const std::map&, JobInfo&); + void updateWindowCols(execplan::ReturnedColumn*, const std::map&, JobInfo&); + void sort(std::vector::iterator, uint64_t); + + void formatMiniStats(); + void printCalTrace(); + + static void AddSimplColumn(const std::vector& scs, JobInfo& jobInfo); + + class Runner + { + public: + Runner(WindowFunctionStep* step) : fStep(step) { - return fRows; } - // for string table - rowgroup::Row::Pointer getPointer(RowPosition& pos) + void operator()() { - fRowGroupIn.setData(&(fInRowGroupData[pos.fGroupId])); - fRowGroupIn.getRow(pos.fRowId, &fRowIn); - return fRowIn.getPointer(); + utils::setThreadName("WFSRunner"); + fStep->execute(); } - rowgroup::Row::Pointer getPointer(RowPosition& pos, rowgroup::RowGroup& rg, rowgroup::Row& row) + WindowFunctionStep* fStep; + }; + + uint64_t fRunner; // thread pool handle + + boost::shared_ptr fCatalog; + uint64_t fRowsReturned; + bool fEndOfResult; + bool fIsSelect; + bool fUseSSMutex; //@bug6065, mutex for setStringField + bool fUseUFMutex; // To ensure thread safety of User Data (UDAnF) + // for input/output datalist + RowGroupDL* fInputDL; + RowGroupDL* fOutputDL; + int fInputIterator; + int fOutputIterator; + + // rowgroups + rowgroup::RowGroup fRowGroupIn; + rowgroup::RowGroup fRowGroupOut; + rowgroup::RowGroup fRowGroupDelivered; + rowgroup::Row fRowIn; + + // data storage + std::vector fInRowGroupData; + + // for funciton/expression taking window function as parameter + std::vector fExpression; + + // for threads computing window functions and partitions + class WFunction + { + public: + WFunction(WindowFunctionStep* step) : fStep(step) { - rg.setData(&(fInRowGroupData[pos.fGroupId])); - rg.getRow(pos.fRowId, &row); - return row.getPointer(); + } + void operator()() + { + fStep->doFunction(); } -private: - void execute(); - void doFunction(); - void doPostProcessForSelect(); - void doPostProcessForDml(); + WindowFunctionStep* fStep; + }; + std::vector fFunctionThreads; - uint64_t nextFunctionIndex(); - - boost::shared_ptr parseFrameBound(const execplan::WF_Boundary&, - const std::map&, const std::vector&, - const boost::shared_ptr&, JobInfo&, bool, bool); - boost::shared_ptr parseFrameBoundRows( - const execplan::WF_Boundary&, const std::map&, JobInfo&); - boost::shared_ptr parseFrameBoundRange( - const execplan::WF_Boundary&, const std::map&, const std::vector&, - JobInfo&); - void updateWindowCols(execplan::ParseTree*, const std::map&, JobInfo&); - void updateWindowCols(execplan::ReturnedColumn*, const std::map&, JobInfo&); - void sort(std::vector::iterator, uint64_t); - - void formatMiniStats(); - void printCalTrace(); - - static void AddSimplColumn(const std::vector& scs, JobInfo& jobInfo); - - class Runner - { - public: - Runner(WindowFunctionStep* step) : fStep(step) { } - void operator()() - { - utils::setThreadName("WFSRunner"); - fStep->execute(); - } - - WindowFunctionStep* fStep; - }; - - uint64_t fRunner; // thread pool handle - - boost::shared_ptr fCatalog; - uint64_t fRowsReturned; - bool fEndOfResult; - bool fIsSelect; - bool fUseSSMutex; //@bug6065, mutex for setStringField - bool fUseUFMutex; // To ensure thread safety of User Data (UDAnF) - // for input/output datalist - RowGroupDL* fInputDL; - RowGroupDL* fOutputDL; - int fInputIterator; - int fOutputIterator; - - // rowgroups - rowgroup::RowGroup fRowGroupIn; - rowgroup::RowGroup fRowGroupOut; - rowgroup::RowGroup fRowGroupDelivered; - rowgroup::Row fRowIn; - - // data storage - std::vector fInRowGroupData; - - // for funciton/expression taking window function as parameter - std::vector fExpression; - - // for threads computing window functions and partitions - class WFunction - { - public: - WFunction(WindowFunctionStep* step) : fStep(step) { } - void operator()() - { - fStep->doFunction(); - } - - WindowFunctionStep* fStep; - }; - std::vector fFunctionThreads; - - std::vector fRows; - std::vector > fFunctions; - uint64_t fFunctionCount; - uint64_t fTotalThreads; + std::vector fRows; + std::vector > fFunctions; + uint64_t fFunctionCount; + uint64_t fTotalThreads; #ifdef _MSC_VER - volatile LONG fNextIndex; + volatile LONG fNextIndex; #else - int fNextIndex; + int fNextIndex; #endif + // query order by + boost::shared_ptr fQueryOrderBy; + uint64_t fQueryLimitStart; + uint64_t fQueryLimitCount; - // query order by - boost::shared_ptr fQueryOrderBy; - uint64_t fQueryLimitStart; - uint64_t fQueryLimitCount; + // for resource management + uint64_t fMemUsage; + ResourceManager* fRm; + boost::shared_ptr fSessionMemLimit; - // for resource management - uint64_t fMemUsage; - ResourceManager* fRm; - boost::shared_ptr fSessionMemLimit; - - friend class windowfunction::WindowFunction; + friend class windowfunction::WindowFunction; }; - -} // namespace +} // namespace joblist #endif // JOBLIST_WINDOWFUNCTIONSTEP_H diff --git a/dbcon/mysql/ha_autoi.cpp b/dbcon/mysql/ha_autoi.cpp index 4dc4b79e6..ddfbcdc22 100644 --- a/dbcon/mysql/ha_autoi.cpp +++ b/dbcon/mysql/ha_autoi.cpp @@ -19,161 +19,161 @@ * $Id$ */ -bool parseAutoincrementTableComment ( std::string comment, uint64_t& startValue, std::string& columnName ) +bool parseAutoincrementTableComment(std::string comment, uint64_t& startValue, std::string& columnName) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", regex_constants::extended); - bool autoincrement = false; - columnName = ""; - boost::match_results what; - std::string::const_iterator start, end; - start = comment.begin(); - end = comment.end(); - boost::match_flag_type flags = boost::match_default; + algorithm::to_upper(comment); + regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", regex_constants::extended); + bool autoincrement = false; + columnName = ""; + boost::match_results what; + std::string::const_iterator start, end; + start = comment.begin(); + end = comment.end(); + boost::match_flag_type flags = boost::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (boost::regex_search(start, end, what, compat, flags)) + { + if (what[0].matched) { - if (what[0].matched) + // string params (what[0].first, what[0].second); + string params(&(*(what[0].second))); + unsigned i = params.find_first_of(","); + + if (i <= params.length()) + { + // check whether there is more autoincrement column + string restComment = params.substr(i + 1, params.length()); + start = restComment.begin(); + end = restComment.end(); + + if (boost::regex_search(start, end, what, compat, flags)) { - //string params (what[0].first, what[0].second); - string params (&(*(what[0].second))); - unsigned i = params.find_first_of(","); - - if ( i <= params.length() ) - { - //check whether there is more autoincrement column - string restComment = params.substr(i + 1, params.length()); - start = restComment.begin(); - end = restComment.end(); - - if (boost::regex_search(start, end, what, compat, flags)) - { - if (what[0].matched) - throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)); - } - - columnName = params.substr(0, i); - string startVal = params.substr(i + 1, params.length()); - //get rid of possible empty space - i = startVal.find_first_not_of(" "); - - if ( i <= startVal.length() ) - { - startVal = startVal.substr( i, startVal.length()); - //; is the seperator between compression and autoincrement comments. - i = startVal.find_first_of(";"); - - if ( i <= startVal.length() ) - { - startVal = startVal.substr( 0, i); - } - - i = startVal.find_last_not_of(" "); - - if ( i <= startVal.length() ) - { - startVal = startVal.substr( 0, i + 1); - } - - errno = 0; - char* ep = NULL; - const char* str = startVal.c_str(); - startValue = strtoull(str, &ep, 10); - - // (no digits) || (more chars) || (other errors & value = 0) - if ((ep == str) || (*ep != '\0') || (errno != 0)) - { - throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); - } - } - } - else - { - columnName = params; - } - - autoincrement = true; + if (what[0].matched) + throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)); } - } - if (columnName.compare("") != 0) + columnName = params.substr(0, i); + string startVal = params.substr(i + 1, params.length()); + // get rid of possible empty space + i = startVal.find_first_not_of(" "); + + if (i <= startVal.length()) + { + startVal = startVal.substr(i, startVal.length()); + //; is the seperator between compression and autoincrement comments. + i = startVal.find_first_of(";"); + + if (i <= startVal.length()) + { + startVal = startVal.substr(0, i); + } + + i = startVal.find_last_not_of(" "); + + if (i <= startVal.length()) + { + startVal = startVal.substr(0, i + 1); + } + + errno = 0; + char* ep = NULL; + const char* str = startVal.c_str(); + startValue = strtoull(str, &ep, 10); + + // (no digits) || (more chars) || (other errors & value = 0) + if ((ep == str) || (*ep != '\0') || (errno != 0)) + { + throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); + } + } + } + else + { + columnName = params; + } + + autoincrement = true; + } + } + + if (columnName.compare("") != 0) + { + // get rid of possible empty space + unsigned i = columnName.find_last_not_of(" "); + + if (i <= columnName.length()) { - //get rid of possible empty space - unsigned i = columnName.find_last_not_of(" "); - - if ( i <= columnName.length() ) - { - columnName = columnName.substr( 0, i + 1); - } + columnName = columnName.substr(0, i + 1); } + } - return autoincrement; + return autoincrement; } -bool parseAutoincrementColumnComment ( std::string comment, uint64_t& startValue ) +bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*", regex_constants::extended); - bool autoincrement = false; - boost::match_results what; - std::string::const_iterator start, end; - start = comment.begin(); - end = comment.end(); - boost::match_flag_type flags = boost::match_default; + algorithm::to_upper(comment); + regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*", regex_constants::extended); + bool autoincrement = false; + boost::match_results what; + std::string::const_iterator start, end; + start = comment.begin(); + end = comment.end(); + boost::match_flag_type flags = boost::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (boost::regex_search(start, end, what, compat, flags)) + { + if (what[0].matched) { - if (what[0].matched) + string params(&(*(what[0].second))); + unsigned i = params.find_first_of(","); + + if (i <= params.length()) + { + string startVal = params.substr(i + 1, params.length()); + // get rid of possible empty space + i = startVal.find_first_not_of(" "); + + if (i <= startVal.length()) { - string params (&(*(what[0].second))); - unsigned i = params.find_first_of(","); + startVal = startVal.substr(i, startVal.length()); + //; is the seperator between compression and autoincrement comments. + i = startVal.find_first_of(";"); - if ( i <= params.length() ) - { - string startVal = params.substr(i + 1, params.length()); - //get rid of possible empty space - i = startVal.find_first_not_of(" "); + if (i <= startVal.length()) + { + startVal = startVal.substr(0, i); + } - if ( i <= startVal.length() ) - { - startVal = startVal.substr( i, startVal.length()); - //; is the seperator between compression and autoincrement comments. - i = startVal.find_first_of(";"); + i = startVal.find_last_not_of(" "); - if ( i <= startVal.length() ) - { - startVal = startVal.substr( 0, i); - } + if (i <= startVal.length()) + { + startVal = startVal.substr(0, i + 1); + } - i = startVal.find_last_not_of(" "); + errno = 0; + char* ep = NULL; + const char* str = startVal.c_str(); + startValue = strtoll(str, &ep, 10); - if ( i <= startVal.length() ) - { - startVal = startVal.substr( 0, i + 1); - } - - errno = 0; - char* ep = NULL; - const char* str = startVal.c_str(); - startValue = strtoll(str, &ep, 10); - - // (no digits) || (more chars) || (other errors & value = 0) - if ((ep == str) || (*ep != '\0') || (errno != 0)) - { - throw runtime_error (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); - } - } - } - else - { - startValue = 1; - } - - autoincrement = true; + // (no digits) || (more chars) || (other errors & value = 0) + if ((ep == str) || (*ep != '\0') || (errno != 0)) + { + throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); + } } - } + } + else + { + startValue = 1; + } - return autoincrement; + autoincrement = true; + } + } + + return autoincrement; } // vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_exists_sub.cpp b/dbcon/mysql/ha_exists_sub.cpp index 9b80fbe28..36c44f13d 100644 --- a/dbcon/mysql/ha_exists_sub.cpp +++ b/dbcon/mysql/ha_exists_sub.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_exists_sub.cpp 6426 2010-03-30 18:46:11Z zzhu $ -* -* -***********************************************************************/ + * $Id: ha_exists_sub.cpp 6426 2010-03-30 18:46:11Z zzhu $ + * + * + ***********************************************************************/ /** @file */ /** class ExistsSub definition */ @@ -47,108 +47,111 @@ extern void makeAntiJoin(const ParseTree* n); void checkCorrelation(const ParseTree* n, void* obj) { - ExistsFilter* ef = reinterpret_cast(obj); - TreeNode* tn = n->data(); - SimpleFilter* sf = dynamic_cast(tn); + ExistsFilter* ef = reinterpret_cast(obj); + TreeNode* tn = n->data(); + SimpleFilter* sf = dynamic_cast(tn); - if (!sf) - return; + if (!sf) + return; - uint64_t lJoinInfo = sf->lhs()->joinInfo(); - uint64_t rJoinInfo = sf->rhs()->joinInfo(); + uint64_t lJoinInfo = sf->lhs()->joinInfo(); + uint64_t rJoinInfo = sf->rhs()->joinInfo(); - if (lJoinInfo & JOIN_CORRELATED) - { - ConstantColumn* cc = dynamic_cast(sf->rhs()); + if (lJoinInfo & JOIN_CORRELATED) + { + ConstantColumn* cc = dynamic_cast(sf->rhs()); - if ((!cc || (cc && sf->op()->op() == OP_EQ)) && !(rJoinInfo & JOIN_CORRELATED)) - ef->correlated(true); - } + if ((!cc || (cc && sf->op()->op() == OP_EQ)) && !(rJoinInfo & JOIN_CORRELATED)) + ef->correlated(true); + } - if (rJoinInfo & JOIN_CORRELATED) - { - ConstantColumn* cc = dynamic_cast(sf->lhs()); + if (rJoinInfo & JOIN_CORRELATED) + { + ConstantColumn* cc = dynamic_cast(sf->lhs()); - if ((!cc || (cc && sf->op()->op() == OP_EQ)) && !(lJoinInfo & JOIN_CORRELATED)) - ef->correlated(true); - } + if ((!cc || (cc && sf->op()->op() == OP_EQ)) && !(lJoinInfo & JOIN_CORRELATED)) + ef->correlated(true); + } } ExistsSub::ExistsSub(gp_walk_info& gwip) : WhereSubQuery(gwip) -{} +{ +} -ExistsSub::ExistsSub(gp_walk_info& gwip, Item_subselect* sub) : - WhereSubQuery(gwip, sub) -{} +ExistsSub::ExistsSub(gp_walk_info& gwip, Item_subselect* sub) : WhereSubQuery(gwip, sub) +{ +} ExistsSub::~ExistsSub() -{} +{ +} execplan::ParseTree* ExistsSub::transform() { - idbassert(fSub); + idbassert(fSub); - SCSEP csep(new CalpontSelectExecutionPlan()); - csep->sessionID(fGwip.sessionid); - csep->location(CalpontSelectExecutionPlan::WHERE); - csep->subType (CalpontSelectExecutionPlan::EXISTS_SUBS); + SCSEP csep(new CalpontSelectExecutionPlan()); + csep->sessionID(fGwip.sessionid); + csep->location(CalpontSelectExecutionPlan::WHERE); + csep->subType(CalpontSelectExecutionPlan::EXISTS_SUBS); - // gwi for the sub query - gp_walk_info gwi; - gwi.thd = fGwip.thd; - gwi.subQuery = this; + // gwi for the sub query + gp_walk_info gwi; + gwi.thd = fGwip.thd; + gwi.subQuery = this; - // @4827 merge table list to gwi in case there is FROM sub to be referenced - // in the FROM sub - gwi.derivedTbCnt = fGwip.derivedTbList.size(); - uint32_t tbCnt = fGwip.tbList.size(); + // @4827 merge table list to gwi in case there is FROM sub to be referenced + // in the FROM sub + gwi.derivedTbCnt = fGwip.derivedTbList.size(); + uint32_t tbCnt = fGwip.tbList.size(); - gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); - gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); + gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); + gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (fSub->get_select_lex()->with_sum_func) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGG_EXISTS); - return NULL; - } + if (fSub->get_select_lex()->with_sum_func) + { + fGwip.fatalParseError = true; + fGwip.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGG_EXISTS); + return NULL; + } - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) - { - fGwip.fatalParseError = true; + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) + { + fGwip.fatalParseError = true; - if (gwi.fatalParseError && !gwi.parseErrorText.empty()) - fGwip.parseErrorText = gwi.parseErrorText; - else - fGwip.parseErrorText = "Error occured in ExistsSub::transform()"; + if (gwi.fatalParseError && !gwi.parseErrorText.empty()) + fGwip.parseErrorText = gwi.parseErrorText; + else + fGwip.parseErrorText = "Error occured in ExistsSub::transform()"; - return NULL; - } + return NULL; + } - // remove outer query tables - CalpontSelectExecutionPlan::TableList tblist; + // remove outer query tables + CalpontSelectExecutionPlan::TableList tblist; - if (csep->tableList().size() >= tbCnt) - tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); + if (csep->tableList().size() >= tbCnt) + tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); - CalpontSelectExecutionPlan::SelectList derivedTbList; + CalpontSelectExecutionPlan::SelectList derivedTbList; - if (csep->derivedTableList().size() >= gwi.derivedTbCnt) - derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, csep->derivedTableList().end()); + if (csep->derivedTableList().size() >= gwi.derivedTbCnt) + derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, + csep->derivedTableList().end()); - csep->tableList(tblist); - csep->derivedTableList(derivedTbList); + csep->tableList(tblist); + csep->derivedTableList(derivedTbList); - ExistsFilter* subFilter = new ExistsFilter(); - subFilter->correlated(false); - subFilter->sub(csep); - const ParseTree* pt = csep->filters(); + ExistsFilter* subFilter = new ExistsFilter(); + subFilter->correlated(false); + subFilter->sub(csep); + const ParseTree* pt = csep->filters(); - if (pt) - pt->walk(checkCorrelation, subFilter); + if (pt) + pt->walk(checkCorrelation, subFilter); - fGwip.subselectList.push_back(csep); - return new ParseTree(subFilter); + fGwip.subselectList.push_back(csep); + return new ParseTree(subFilter); } /** @@ -158,20 +161,20 @@ execplan::ParseTree* ExistsSub::transform() */ void ExistsSub::handleNot() { - ParseTree* pt = fGwip.ptWorkStack.top(); - ExistsFilter* subFilter = dynamic_cast(pt->data()); - idbassert(subFilter); - subFilter->notExists(true); - SCSEP csep = subFilter->sub(); - const ParseTree* ptsub = csep->filters(); + ParseTree* pt = fGwip.ptWorkStack.top(); + ExistsFilter* subFilter = dynamic_cast(pt->data()); + idbassert(subFilter); + subFilter->notExists(true); + SCSEP csep = subFilter->sub(); + const ParseTree* ptsub = csep->filters(); - if (ptsub) - ptsub->walk(makeAntiJoin); + if (ptsub) + ptsub->walk(makeAntiJoin); - ptsub = csep->having(); + ptsub = csep->having(); - if (ptsub) - ptsub->walk(makeAntiJoin); + if (ptsub) + ptsub->walk(makeAntiJoin); } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 19c96e1d0..a3065d6c9 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_from_sub.cpp 6377 2010-03-22 20:18:47Z zzhu $ -* -* -***********************************************************************/ + * $Id: ha_from_sub.cpp 6377 2010-03-22 20:18:47Z zzhu $ + * + * + ***********************************************************************/ /** @file */ /** class FromSubSelect definition */ @@ -44,409 +44,412 @@ using namespace execplan; namespace cal_impl_if { - void derivedTableOptimization(THD* thd, SCSEP& csep) { - // @bug5634. replace the unused column with ConstantColumn from derived table column list, - // ExeMgr will not project ConstantColumn. Only count for local derived column. - // subquery may carry main query derived table list for column reference, those - // derived tables are not checked for optimization in this scope. - CalpontSelectExecutionPlan::SelectList derivedTbList = csep->derivedTableList(); + // @bug5634. replace the unused column with ConstantColumn from derived table column list, + // ExeMgr will not project ConstantColumn. Only count for local derived column. + // subquery may carry main query derived table list for column reference, those + // derived tables are not checked for optimization in this scope. + CalpontSelectExecutionPlan::SelectList derivedTbList = csep->derivedTableList(); - // @bug6156. Skip horizontal optimization for no table union. - bool horizontalOptimization = true; + // @bug6156. Skip horizontal optimization for no table union. + bool horizontalOptimization = true; - for (uint i = 0; i < derivedTbList.size(); i++) + for (uint i = 0; i < derivedTbList.size(); i++) + { + CalpontSelectExecutionPlan* plan = reinterpret_cast(derivedTbList[i].get()); + CalpontSelectExecutionPlan::ReturnedColumnList cols = plan->returnedCols(); + vector unionColVec; + + // only do vertical optimization for union all + // @bug6134. Also skip the vertical optimization for select distinct + // because all columns need to be projected to check the distinctness. + bool verticalOptimization = false; + + if (plan->distinctUnionNum() == 0 && !plan->distinct()) { - CalpontSelectExecutionPlan* plan = reinterpret_cast(derivedTbList[i].get()); - CalpontSelectExecutionPlan::ReturnedColumnList cols = plan->returnedCols(); - vector unionColVec; + verticalOptimization = true; - // only do vertical optimization for union all - // @bug6134. Also skip the vertical optimization for select distinct - // because all columns need to be projected to check the distinctness. - bool verticalOptimization = false; + for (uint j = 0; j < plan->unionVec().size(); j++) + { + unionColVec.push_back( + reinterpret_cast(plan->unionVec()[j].get())->returnedCols()); + } + } - if (plan->distinctUnionNum() == 0 && !plan->distinct()) + if (plan->tableList().empty()) + horizontalOptimization = false; + + for (uint j = 0; j < plan->unionVec().size(); j++) + { + if (reinterpret_cast(plan->unionVec()[j].get())->tableList().empty()) + { + horizontalOptimization = false; + break; + } + } + + if (verticalOptimization) + { + int64_t val = 1; + + // TODO MCOL-4543 Only project those columns from the subquery + // which are referenced in the outer select. So for example, + // if a table t contains 10 columns c1 ... c10 : + // "select count(c2) from (select * from t) q;" + // with p being the subquery execution plan, p->columnMap() + // and p->returnedCols() should both be of size 1, instead + // of 10, with entries for c2 in each. + // + // We are currently performing a dumb optimization: + // Instead of just referencing c2, we are referencing (c1,c2) + // for the above query. This is due to complexity associated + // with modifying ReturnedColumn::colPosition() + // (from a value of 1 to a value of 0) of the outer query + // which references c2. So essentially, if c2 is replaced by c10 + // in the above query, we fallback to projecting all 10 columns + // of the subquery in ExeMgr. + // This will be addressed in future. + CalpontSelectExecutionPlan::ReturnedColumnList nonConstCols; + vector nonConstUnionColVec(unionColVec.size()); + + int64_t lastNonConstIndex = -1; + + for (int64_t i = cols.size() - 1; i >= 0; i--) + { + // if (cols[i]->derivedTable().empty()) + if (cols[i]->refCount() == 0) { - verticalOptimization = true; + if (cols[i]->derivedRefCol()) + cols[i]->derivedRefCol()->decRefCount(); - for (uint j = 0; j < plan->unionVec().size(); j++) + if (lastNonConstIndex == -1) + { + SimpleColumn* sc = dynamic_cast(cols[i].get()); + + if (sc && (plan->columnMap().count(sc->columnName()) == 1)) { - unionColVec.push_back( - reinterpret_cast(plan->unionVec()[j].get())->returnedCols()); - } - } - - if (plan->tableList().empty()) - horizontalOptimization = false; - - for (uint j = 0; j < plan->unionVec().size(); j++) - { - if (reinterpret_cast(plan->unionVec()[j].get())->tableList().empty()) - { - horizontalOptimization = false; - break; - } - } - - if (verticalOptimization) - { - int64_t val = 1; - - // TODO MCOL-4543 Only project those columns from the subquery - // which are referenced in the outer select. So for example, - // if a table t contains 10 columns c1 ... c10 : - // "select count(c2) from (select * from t) q;" - // with p being the subquery execution plan, p->columnMap() - // and p->returnedCols() should both be of size 1, instead - // of 10, with entries for c2 in each. - // - // We are currently performing a dumb optimization: - // Instead of just referencing c2, we are referencing (c1,c2) - // for the above query. This is due to complexity associated - // with modifying ReturnedColumn::colPosition() - // (from a value of 1 to a value of 0) of the outer query - // which references c2. So essentially, if c2 is replaced by c10 - // in the above query, we fallback to projecting all 10 columns - // of the subquery in ExeMgr. - // This will be addressed in future. - CalpontSelectExecutionPlan::ReturnedColumnList nonConstCols; - vector nonConstUnionColVec(unionColVec.size()); - - int64_t lastNonConstIndex = -1; - - for (int64_t i = cols.size() - 1; i >= 0; i--) - { - //if (cols[i]->derivedTable().empty()) - if (cols[i]->refCount() == 0) - { - if (cols[i]->derivedRefCol()) - cols[i]->derivedRefCol()->decRefCount(); - - if (lastNonConstIndex == -1) - { - SimpleColumn* sc = dynamic_cast(cols[i].get()); - - if (sc && (plan->columnMap().count(sc->columnName()) == 1)) - { - plan->columnMap().erase(sc->columnName()); - } - } - else - { - cols[i].reset(new ConstantColumn(val)); - (reinterpret_cast(cols[i].get()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - } - - for (uint j = 0; j < unionColVec.size(); j++) - { - if (lastNonConstIndex == -1) - { - CalpontSelectExecutionPlan* unionSubPlan = - reinterpret_cast(plan->unionVec()[j].get()); - - SimpleColumn* sc = dynamic_cast(unionSubPlan->returnedCols()[i].get()); - - if (sc && (unionSubPlan->columnMap().count(sc->columnName()) == 1)) - { - unionSubPlan->columnMap().erase(sc->columnName()); - } - } - else - { - unionColVec[j][i].reset(new ConstantColumn(val)); - (reinterpret_cast(unionColVec[j][i].get()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - } - } - } - else if (lastNonConstIndex == -1) - { - lastNonConstIndex = i; - } + plan->columnMap().erase(sc->columnName()); } + } + else + { + cols[i].reset(new ConstantColumn(val)); + (reinterpret_cast(cols[i].get())) + ->timeZone(thd->variables.time_zone->get_name()->ptr()); + } + for (uint j = 0; j < unionColVec.size(); j++) + { if (lastNonConstIndex == -1) { - // None of the subquery columns are referenced, just use the first one - if (!cols.empty()) - { - cols[0].reset(new ConstantColumn(val)); - (reinterpret_cast(cols[0].get()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - nonConstCols.push_back(cols[0]); + CalpontSelectExecutionPlan* unionSubPlan = + reinterpret_cast(plan->unionVec()[j].get()); - for (uint j = 0; j < unionColVec.size(); j++) - { - unionColVec[j][0].reset(new ConstantColumn(val)); - (reinterpret_cast(unionColVec[j][0].get()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - nonConstUnionColVec[j].push_back(unionColVec[j][0]); - } - } + SimpleColumn* sc = dynamic_cast(unionSubPlan->returnedCols()[i].get()); + + if (sc && (unionSubPlan->columnMap().count(sc->columnName()) == 1)) + { + unionSubPlan->columnMap().erase(sc->columnName()); + } } else { - nonConstCols.assign(cols.begin(), cols.begin() + lastNonConstIndex + 1); - - for (uint j = 0; j < unionColVec.size(); j++) - { - nonConstUnionColVec[j].assign(unionColVec[j].begin(), unionColVec[j].begin() + lastNonConstIndex + 1); - } - } - - // set back - plan->returnedCols(nonConstCols); - - for (uint j = 0; j < unionColVec.size(); j++) - { - CalpontSelectExecutionPlan* unionSubPlan = - reinterpret_cast(plan->unionVec()[j].get()); - unionSubPlan->returnedCols(nonConstUnionColVec[j]); + unionColVec[j][i].reset(new ConstantColumn(val)); + (reinterpret_cast(unionColVec[j][i].get())) + ->timeZone(thd->variables.time_zone->get_name()->ptr()); } + } } - } - - /* - * @bug5635. Move filters that only belongs to a derived table to inside the derived table. - * 1. parse tree walk to populate derivedTableFilterMap and set null candidate on the tree. - * 2. remove the null filters - * 3. and the filters of derivedTableFilterMap and append to the WHERE filter of the derived table - * - * Note: - * 1. Subquery filters is ignored because derived table can not be in subquery - * 2. While walking tree, whenever a single derive simplefilter is encountered, - * this filter is pushed to the corresponding stack - * 2. Whenever an OR operator is encountered, all the filter stack of - * that OR involving derived table are emptied and null candidate of each - * stacked filter needs to be reset (not null) - */ - ParseTree* pt = csep->filters(); - map derivedTbFilterMap; - - if (horizontalOptimization && pt) - { - pt->walk(setDerivedTable); - setDerivedFilter(thd, pt, derivedTbFilterMap, derivedTbList); - csep->filters(pt); - } - - // AND the filters of individual stack to the derived table filter tree - // @todo union filters. - // @todo outer join complication - map::iterator mapIt; - - for (uint i = 0; i < derivedTbList.size(); i++) - { - CalpontSelectExecutionPlan* plan = reinterpret_cast(derivedTbList[i].get()); - CalpontSelectExecutionPlan::ReturnedColumnList derivedColList = plan->returnedCols(); - mapIt = derivedTbFilterMap.find(plan->derivedTbAlias()); - - if (mapIt != derivedTbFilterMap.end()) + else if (lastNonConstIndex == -1) { - // replace all derived column of this filter with real column from - // derived table projection list. - ParseTree* mainFilter = new ParseTree(); - mainFilter->copyTree(*(mapIt->second)); - replaceRefCol(mainFilter, derivedColList); - ParseTree* derivedFilter = plan->filters(); - - if (derivedFilter) - { - LogicOperator* op = new LogicOperator("and"); - ParseTree* filter = new ParseTree(op); - filter->left(derivedFilter); - filter->right(mainFilter); - plan->filters(filter); - } - else - { - plan->filters(mainFilter); - } - - // union filter handling - for (uint j = 0; j < plan->unionVec().size(); j++) - { - CalpontSelectExecutionPlan* unionPlan = - reinterpret_cast(plan->unionVec()[j].get()); - CalpontSelectExecutionPlan::ReturnedColumnList unionColList = unionPlan->returnedCols(); - ParseTree* mainFilterForUnion = new ParseTree(); - mainFilterForUnion->copyTree(*(mapIt->second)); - replaceRefCol(mainFilterForUnion, unionColList); - ParseTree* unionFilter = unionPlan->filters(); - - if (unionFilter) - { - LogicOperator* op = new LogicOperator("and"); - ParseTree* filter = new ParseTree(op); - filter->left(unionFilter); - filter->right(mainFilterForUnion); - unionPlan->filters(filter); - } - else - { - unionPlan->filters(mainFilterForUnion); - } - } + lastNonConstIndex = i; } + } + + if (lastNonConstIndex == -1) + { + // None of the subquery columns are referenced, just use the first one + if (!cols.empty()) + { + cols[0].reset(new ConstantColumn(val)); + (reinterpret_cast(cols[0].get())) + ->timeZone(thd->variables.time_zone->get_name()->ptr()); + nonConstCols.push_back(cols[0]); + + for (uint j = 0; j < unionColVec.size(); j++) + { + unionColVec[j][0].reset(new ConstantColumn(val)); + (reinterpret_cast(unionColVec[j][0].get())) + ->timeZone(thd->variables.time_zone->get_name()->ptr()); + nonConstUnionColVec[j].push_back(unionColVec[j][0]); + } + } + } + else + { + nonConstCols.assign(cols.begin(), cols.begin() + lastNonConstIndex + 1); + + for (uint j = 0; j < unionColVec.size(); j++) + { + nonConstUnionColVec[j].assign(unionColVec[j].begin(), + unionColVec[j].begin() + lastNonConstIndex + 1); + } + } + + // set back + plan->returnedCols(nonConstCols); + + for (uint j = 0; j < unionColVec.size(); j++) + { + CalpontSelectExecutionPlan* unionSubPlan = + reinterpret_cast(plan->unionVec()[j].get()); + unionSubPlan->returnedCols(nonConstUnionColVec[j]); + } } + } - // clean derivedTbFilterMap because all the filters are copied - for ( mapIt = derivedTbFilterMap.begin(); mapIt != derivedTbFilterMap.end(); ++mapIt ) - delete (*mapIt).second; + /* + * @bug5635. Move filters that only belongs to a derived table to inside the derived table. + * 1. parse tree walk to populate derivedTableFilterMap and set null candidate on the tree. + * 2. remove the null filters + * 3. and the filters of derivedTableFilterMap and append to the WHERE filter of the derived table + * + * Note: + * 1. Subquery filters is ignored because derived table can not be in subquery + * 2. While walking tree, whenever a single derive simplefilter is encountered, + * this filter is pushed to the corresponding stack + * 2. Whenever an OR operator is encountered, all the filter stack of + * that OR involving derived table are emptied and null candidate of each + * stacked filter needs to be reset (not null) + */ + ParseTree* pt = csep->filters(); + map derivedTbFilterMap; - // recursively process the nested derived table - for (uint i = 0; i < csep->subSelectList().size(); i++) + if (horizontalOptimization && pt) + { + pt->walk(setDerivedTable); + setDerivedFilter(thd, pt, derivedTbFilterMap, derivedTbList); + csep->filters(pt); + } + + // AND the filters of individual stack to the derived table filter tree + // @todo union filters. + // @todo outer join complication + map::iterator mapIt; + + for (uint i = 0; i < derivedTbList.size(); i++) + { + CalpontSelectExecutionPlan* plan = reinterpret_cast(derivedTbList[i].get()); + CalpontSelectExecutionPlan::ReturnedColumnList derivedColList = plan->returnedCols(); + mapIt = derivedTbFilterMap.find(plan->derivedTbAlias()); + + if (mapIt != derivedTbFilterMap.end()) { - SCSEP subselect(boost::dynamic_pointer_cast(csep->subSelectList()[i])); - derivedTableOptimization(thd, subselect); + // replace all derived column of this filter with real column from + // derived table projection list. + ParseTree* mainFilter = new ParseTree(); + mainFilter->copyTree(*(mapIt->second)); + replaceRefCol(mainFilter, derivedColList); + ParseTree* derivedFilter = plan->filters(); + + if (derivedFilter) + { + LogicOperator* op = new LogicOperator("and"); + ParseTree* filter = new ParseTree(op); + filter->left(derivedFilter); + filter->right(mainFilter); + plan->filters(filter); + } + else + { + plan->filters(mainFilter); + } + + // union filter handling + for (uint j = 0; j < plan->unionVec().size(); j++) + { + CalpontSelectExecutionPlan* unionPlan = + reinterpret_cast(plan->unionVec()[j].get()); + CalpontSelectExecutionPlan::ReturnedColumnList unionColList = unionPlan->returnedCols(); + ParseTree* mainFilterForUnion = new ParseTree(); + mainFilterForUnion->copyTree(*(mapIt->second)); + replaceRefCol(mainFilterForUnion, unionColList); + ParseTree* unionFilter = unionPlan->filters(); + + if (unionFilter) + { + LogicOperator* op = new LogicOperator("and"); + ParseTree* filter = new ParseTree(op); + filter->left(unionFilter); + filter->right(mainFilterForUnion); + unionPlan->filters(filter); + } + else + { + unionPlan->filters(mainFilterForUnion); + } + } } + } + + // clean derivedTbFilterMap because all the filters are copied + for (mapIt = derivedTbFilterMap.begin(); mapIt != derivedTbFilterMap.end(); ++mapIt) + delete (*mapIt).second; + + // recursively process the nested derived table + for (uint i = 0; i < csep->subSelectList().size(); i++) + { + SCSEP subselect(boost::dynamic_pointer_cast(csep->subSelectList()[i])); + derivedTableOptimization(thd, subselect); + } } void setDerivedTable(execplan::ParseTree* n) { - ParseTree* lhs = n->left(); - ParseTree* rhs = n->right(); + ParseTree* lhs = n->left(); + ParseTree* rhs = n->right(); - Operator* op = dynamic_cast(n->data()); + Operator* op = dynamic_cast(n->data()); - // if logic operator then lhs and rhs can't be both null - if (op) + // if logic operator then lhs and rhs can't be both null + if (op) + { + if (!lhs || lhs->derivedTable() == "*") { - if (!lhs || lhs->derivedTable() == "*") - { - n->derivedTable(rhs ? rhs->derivedTable() : "*"); - } - else if (!rhs || rhs->derivedTable() == "*") - { - n->derivedTable(lhs->derivedTable()); - } - else if (lhs->derivedTable() == rhs->derivedTable()) - { - n->derivedTable(lhs->derivedTable()); - } - else - { - n->derivedTable(""); - } + n->derivedTable(rhs ? rhs->derivedTable() : "*"); + } + else if (!rhs || rhs->derivedTable() == "*") + { + n->derivedTable(lhs->derivedTable()); + } + else if (lhs->derivedTable() == rhs->derivedTable()) + { + n->derivedTable(lhs->derivedTable()); } else { - n->data()->setDerivedTable(); - n->derivedTable(n->data()->derivedTable()); + n->derivedTable(""); } + } + else + { + n->data()->setDerivedTable(); + n->derivedTable(n->data()->derivedTable()); + } } -ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, - map& filterMap, +ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& filterMap, CalpontSelectExecutionPlan::SelectList& derivedTbList) { - if (!(n->derivedTable().empty())) + if (!(n->derivedTable().empty())) + { + // @todo replace virtual column of n to real column + // all simple columns should belong to the same derived table + CalpontSelectExecutionPlan* csep = NULL; + + for (uint i = 0; i < derivedTbList.size(); i++) { - // @todo replace virtual column of n to real column - // all simple columns should belong to the same derived table - CalpontSelectExecutionPlan* csep = NULL; + CalpontSelectExecutionPlan* plan = dynamic_cast(derivedTbList[i].get()); - for (uint i = 0; i < derivedTbList.size(); i++) - { - CalpontSelectExecutionPlan* plan = dynamic_cast(derivedTbList[i].get()); + if (plan->derivedTbAlias() == n->derivedTable()) + { + csep = plan; + break; + } + } - if (plan->derivedTbAlias() == n->derivedTable()) - { - csep = plan; - break; - } - } + // should never be null; if null then give up optimization. + if (!csep) + return n; - // should never be null; if null then give up optimization. - if (!csep) - return n; + // 2. push the filter to the derived table filter stack, or 'and' with + // the filters in the stack + map::iterator mapIter = filterMap.find(n->derivedTable()); - // 2. push the filter to the derived table filter stack, or 'and' with - // the filters in the stack - map::iterator mapIter = filterMap.find(n->derivedTable()); - - if ( mapIter == filterMap.end()) - { - filterMap.insert(pair(n->derivedTable(), n)); - } - else - { - ParseTree* pt = new ParseTree(new LogicOperator("and")); - pt->left(mapIter->second); - pt->right(n); - mapIter->second = pt; - } - - int64_t val = 1; - n = new ParseTree(new ConstantColumn(val)); - (dynamic_cast(n->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + if (mapIter == filterMap.end()) + { + filterMap.insert(pair(n->derivedTable(), n)); } else { - Operator* op = dynamic_cast(n->data()); - - if (op && (op->op() == OP_OR || op->op() == OP_XOR)) - { - return n; - } - else - { - ParseTree* lhs = n->left(); - ParseTree* rhs = n->right(); - - if (lhs) - n->left(setDerivedFilter(thd, lhs, filterMap, derivedTbList)); - - if (rhs) - n->right(setDerivedFilter(thd, rhs, filterMap, derivedTbList)); - } + ParseTree* pt = new ParseTree(new LogicOperator("and")); + pt->left(mapIter->second); + pt->right(n); + mapIter->second = pt; } - return n; + int64_t val = 1; + n = new ParseTree(new ConstantColumn(val)); + (dynamic_cast(n->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + } + else + { + Operator* op = dynamic_cast(n->data()); + + if (op && (op->op() == OP_OR || op->op() == OP_XOR)) + { + return n; + } + else + { + ParseTree* lhs = n->left(); + ParseTree* rhs = n->right(); + + if (lhs) + n->left(setDerivedFilter(thd, lhs, filterMap, derivedTbList)); + + if (rhs) + n->right(setDerivedFilter(thd, rhs, filterMap, derivedTbList)); + } + } + + return n; } FromSubQuery::FromSubQuery(gp_walk_info& gwip) : SubQuery(gwip) -{} +{ +} -FromSubQuery::FromSubQuery(gp_walk_info& gwip, - SELECT_LEX* sub) : - SubQuery(gwip), - fFromSub(sub) -{} +FromSubQuery::FromSubQuery(gp_walk_info& gwip, SELECT_LEX* sub) : SubQuery(gwip), fFromSub(sub) +{ +} FromSubQuery::~FromSubQuery() -{} +{ +} SCSEP FromSubQuery::transform() { - assert (fFromSub); - SCSEP csep(new CalpontSelectExecutionPlan()); - csep->sessionID(fGwip.sessionid); - csep->location(CalpontSelectExecutionPlan::FROM); - csep->subType (CalpontSelectExecutionPlan::FROM_SUBS); + assert(fFromSub); + SCSEP csep(new CalpontSelectExecutionPlan()); + csep->sessionID(fGwip.sessionid); + csep->location(CalpontSelectExecutionPlan::FROM); + csep->subType(CalpontSelectExecutionPlan::FROM_SUBS); - // gwi for the sub query - gp_walk_info gwi; - gwi.thd = fGwip.thd; - gwi.subQuery = this; - gwi.viewName = fGwip.viewName; - csep->derivedTbAlias(fAlias); // always lower case - csep->derivedTbView(fGwip.viewName.alias, lower_case_table_names); + // gwi for the sub query + gp_walk_info gwi; + gwi.thd = fGwip.thd; + gwi.subQuery = this; + gwi.viewName = fGwip.viewName; + csep->derivedTbAlias(fAlias); // always lower case + csep->derivedTbView(fGwip.viewName.alias, lower_case_table_names); - if (getSelectPlan(gwi, *fFromSub, csep, false) != 0) - { - fGwip.fatalParseError = true; + if (getSelectPlan(gwi, *fFromSub, csep, false) != 0) + { + fGwip.fatalParseError = true; - if (!gwi.parseErrorText.empty()) - fGwip.parseErrorText = gwi.parseErrorText; - else - fGwip.parseErrorText = "Error occured in FromSubQuery::transform()"; + if (!gwi.parseErrorText.empty()) + fGwip.parseErrorText = gwi.parseErrorText; + else + fGwip.parseErrorText = "Error occured in FromSubQuery::transform()"; - csep.reset(); - return csep; - } - - fGwip.subselectList.push_back(csep); + csep.reset(); return csep; + } + + fGwip.subselectList.push_back(csep); + return csep; } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_in_sub.cpp b/dbcon/mysql/ha_in_sub.cpp index c18088aee..cfb4fb3be 100644 --- a/dbcon/mysql/ha_in_sub.cpp +++ b/dbcon/mysql/ha_in_sub.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_in_sub.cpp 6407 2010-03-26 19:36:56Z zzhu $ -* -* -***********************************************************************/ + * $Id: ha_in_sub.cpp 6407 2010-03-26 19:36:56Z zzhu $ + * + * + ***********************************************************************/ /** @file */ /** class InSub definition */ @@ -52,66 +52,69 @@ using namespace logging; namespace cal_impl_if { -extern void parse_item (Item* item, vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo); +extern void parse_item(Item* item, vector& field_vec, bool& hasNonSupportItem, + uint16& parseInfo); void makeAntiJoin(const ParseTree* n) { - TreeNode* tn = n->data(); - SimpleFilter* sf = dynamic_cast(tn); + TreeNode* tn = n->data(); + SimpleFilter* sf = dynamic_cast(tn); - if (!sf) - return; + if (!sf) + return; - uint64_t lJoinInfo = sf->lhs()->joinInfo(); + uint64_t lJoinInfo = sf->lhs()->joinInfo(); - if (lJoinInfo & JOIN_SEMI) - { - lJoinInfo &= ~JOIN_SEMI; - lJoinInfo |= JOIN_ANTI; + if (lJoinInfo & JOIN_SEMI) + { + lJoinInfo &= ~JOIN_SEMI; + lJoinInfo |= JOIN_ANTI; - if (lJoinInfo & JOIN_NULLMATCH_CANDIDATE) - lJoinInfo |= JOIN_NULL_MATCH; + if (lJoinInfo & JOIN_NULLMATCH_CANDIDATE) + lJoinInfo |= JOIN_NULL_MATCH; - sf->lhs()->joinInfo(lJoinInfo); - } + sf->lhs()->joinInfo(lJoinInfo); + } - uint64_t rJoinInfo = sf->rhs()->joinInfo(); + uint64_t rJoinInfo = sf->rhs()->joinInfo(); - if (rJoinInfo & JOIN_SEMI) - { - rJoinInfo &= ~JOIN_SEMI; - rJoinInfo |= JOIN_ANTI; + if (rJoinInfo & JOIN_SEMI) + { + rJoinInfo &= ~JOIN_SEMI; + rJoinInfo |= JOIN_ANTI; - if (rJoinInfo & JOIN_NULLMATCH_CANDIDATE) - rJoinInfo |= JOIN_NULL_MATCH; + if (rJoinInfo & JOIN_NULLMATCH_CANDIDATE) + rJoinInfo |= JOIN_NULL_MATCH; - sf->rhs()->joinInfo(rJoinInfo); - } + sf->rhs()->joinInfo(rJoinInfo); + } } InSub::InSub(gp_walk_info& gwip) : WhereSubQuery(gwip) -{} +{ +} -InSub::InSub(gp_walk_info& gwip, Item_func* func) : - WhereSubQuery(gwip, func) -{} +InSub::InSub(gp_walk_info& gwip, Item_func* func) : WhereSubQuery(gwip, func) +{ +} -InSub::InSub(const InSub& rhs) : - WhereSubQuery(rhs.gwip(), rhs.fColumn, rhs.fSub, rhs.fFunc) -{} +InSub::InSub(const InSub& rhs) : WhereSubQuery(rhs.gwip(), rhs.fColumn, rhs.fSub, rhs.fFunc) +{ +} InSub::~InSub() -{} +{ +} inline void setCorrelatedFlag(execplan::ReturnedColumn* col, gp_walk_info* gwip) { - ConstantColumn* cc = dynamic_cast(col); + ConstantColumn* cc = dynamic_cast(col); - if (!cc) - { - col->joinInfo(col->joinInfo() | JOIN_CORRELATED | JOIN_NULLMATCH_CANDIDATE); - gwip->subQuery->correlated(true); - } + if (!cc) + { + col->joinInfo(col->joinInfo() | JOIN_CORRELATED | JOIN_NULLMATCH_CANDIDATE); + gwip->subQuery->correlated(true); + } } /** MySQL transform (NOT) IN subquery to (NOT) EXIST @@ -119,109 +122,108 @@ inline void setCorrelatedFlag(execplan::ReturnedColumn* col, gp_walk_info* gwip) */ execplan::ParseTree* InSub::transform() { - if (!fFunc) - return NULL; + if (!fFunc) + return NULL; - // @todo need to handle scalar IN and BETWEEN specially - // this blocks handles only one subselect scalar - // arg[0]: column | arg[1]: subselect - //assert (fFunc->argument_count() == 2 && fGwip.rcWorkStack.size() >= 2); - if (fFunc->argument_count() != 2 || fGwip.rcWorkStack.size() < 2) + // @todo need to handle scalar IN and BETWEEN specially + // this blocks handles only one subselect scalar + // arg[0]: column | arg[1]: subselect + // assert (fFunc->argument_count() == 2 && fGwip.rcWorkStack.size() >= 2); + if (fFunc->argument_count() != 2 || fGwip.rcWorkStack.size() < 2) + { + fGwip.fatalParseError = true; + fGwip.parseErrorText = "Unsupported item in IN subquery"; + return NULL; + } + + ReturnedColumn* rhs = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + delete rhs; + ReturnedColumn* lhs = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + + fSub = (Item_subselect*)(fFunc->arguments()[1]); + idbassert(fSub); + + SCSEP csep(new CalpontSelectExecutionPlan()); + csep->sessionID(fGwip.sessionid); + csep->location(CalpontSelectExecutionPlan::WHERE); + csep->subType(CalpontSelectExecutionPlan::IN_SUBS); + + // gwi for the sub query + gp_walk_info gwi; + gwi.thd = fGwip.thd; + gwi.subQuery = this; + + // The below 2 fields are used later on in buildInToExistsFilter() + gwi.inSubQueryLHS = lhs; + gwi.inSubQueryLHSItem = fFunc->arguments()[0]; + + RowColumn* rlhs = dynamic_cast(lhs); + + if (rlhs) + { + for (auto& col : rlhs->columnVec()) { - fGwip.fatalParseError = true; - fGwip.parseErrorText = "Unsupported item in IN subquery"; - return NULL; + setCorrelatedFlag(col.get(), &gwi); } + } + else + { + setCorrelatedFlag(lhs, &gwi); + } - ReturnedColumn* rhs = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); - delete rhs; - ReturnedColumn* lhs = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); + // @4827 merge table list to gwi in case there is FROM sub to be referenced + // in the FROM sub + gwi.derivedTbCnt = fGwip.derivedTbList.size(); + uint32_t tbCnt = fGwip.tbList.size(); - fSub = (Item_subselect*)(fFunc->arguments()[1]); - idbassert(fSub); + gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); + gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - SCSEP csep (new CalpontSelectExecutionPlan()); - csep->sessionID(fGwip.sessionid); - csep->location(CalpontSelectExecutionPlan::WHERE); - csep->subType (CalpontSelectExecutionPlan::IN_SUBS); + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) + { + fGwip.fatalParseError = true; - // gwi for the sub query - gp_walk_info gwi; - gwi.thd = fGwip.thd; - gwi.subQuery = this; - - // The below 2 fields are used later on in buildInToExistsFilter() - gwi.inSubQueryLHS = lhs; - gwi.inSubQueryLHSItem = fFunc->arguments()[0]; - - RowColumn* rlhs = dynamic_cast(lhs); - - if (rlhs) - { - for (auto& col : rlhs->columnVec()) - { - setCorrelatedFlag(col.get(), &gwi); - } - } + if (gwi.fatalParseError && !gwi.parseErrorText.empty()) + fGwip.parseErrorText = gwi.parseErrorText; else - { - setCorrelatedFlag(lhs, &gwi); - } + fGwip.parseErrorText = "Error occured in InSub::transform()"; - // @4827 merge table list to gwi in case there is FROM sub to be referenced - // in the FROM sub - gwi.derivedTbCnt = fGwip.derivedTbList.size(); - uint32_t tbCnt = fGwip.tbList.size(); + return NULL; + } - gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); - gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); + // remove outer query tables + CalpontSelectExecutionPlan::TableList tblist; - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) - { - fGwip.fatalParseError = true; + if (csep->tableList().size() >= tbCnt) + tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); - if (gwi.fatalParseError && !gwi.parseErrorText.empty()) - fGwip.parseErrorText = gwi.parseErrorText; - else - fGwip.parseErrorText = "Error occured in InSub::transform()"; + CalpontSelectExecutionPlan::SelectList derivedTbList; - return NULL; - } + if (csep->derivedTableList().size() >= gwi.derivedTbCnt) + derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, + csep->derivedTableList().end()); - // remove outer query tables - CalpontSelectExecutionPlan::TableList tblist; + csep->tableList(tblist); + csep->derivedTableList(derivedTbList); - if (csep->tableList().size() >= tbCnt) - tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); + ExistsFilter* subFilter = new ExistsFilter(); + subFilter->sub(csep); - CalpontSelectExecutionPlan::SelectList derivedTbList; + if (gwi.subQuery->correlated()) + subFilter->correlated(true); + else + subFilter->correlated(false); - if (csep->derivedTableList().size() >= gwi.derivedTbCnt) - derivedTbList.insert(derivedTbList.begin(), - csep->derivedTableList().begin() + gwi.derivedTbCnt, - csep->derivedTableList().end()); + if (fGwip.clauseType == HAVING && subFilter->correlated()) + { + fGwip.fatalParseError = true; + fGwip.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NON_SUPPORT_HAVING); + } - csep->tableList(tblist); - csep->derivedTableList(derivedTbList); - - ExistsFilter* subFilter = new ExistsFilter(); - subFilter->sub(csep); - - if (gwi.subQuery->correlated()) - subFilter->correlated(true); - else - subFilter->correlated(false); - - if (fGwip.clauseType == HAVING && subFilter->correlated()) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NON_SUPPORT_HAVING); - } - - fGwip.subselectList.push_back(csep); - return new ParseTree(subFilter); + fGwip.subselectList.push_back(csep); + return new ParseTree(subFilter); } /** @@ -230,90 +232,89 @@ execplan::ParseTree* InSub::transform() */ void InSub::handleFunc(gp_walk_info* gwip, Item_func* func) { - if (func->functype() == Item_func::TRIG_COND_FUNC || func->functype() == Item_func::COND_OR_FUNC) + if (func->functype() == Item_func::TRIG_COND_FUNC || func->functype() == Item_func::COND_OR_FUNC) + { + // purpose: remove the isnull() function from the parsetree in ptWorkStack. + // IDB handles the null semantics in the join operation + // trigcond(or_cond) is the only form we recognize for now + if (func->argument_count() > 2) { - // purpose: remove the isnull() function from the parsetree in ptWorkStack. - // IDB handles the null semantics in the join operation - // trigcond(or_cond) is the only form we recognize for now - if (func->argument_count() > 2) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = "Unsupported item in IN subquery"; - return; - } - - Item_cond* cond; - - if (func->functype() == Item_func::TRIG_COND_FUNC) - { - Item* item; - - if (func->arguments()[0]->type() == Item::REF_ITEM) - item = (Item_ref*)(func->arguments()[0])->real_item(); - else - item = func->arguments()[0]; - - cond = (Item_cond*)(item); - } - else - { - cond = (Item_cond*)(func); - } - - if (cond->functype() == Item_func::COND_OR_FUNC) - { - // (cache=item) case. do nothing. ignore trigcond()? - if (cond->argument_list()->elements == 1) - return; - - if (cond->argument_list()->elements == 2) - { - // don't know how to deal with this. don't think it's a fatal error either. - if (gwip->ptWorkStack.empty()) - return; - - ParseTree* pt = gwip->ptWorkStack.top(); - - if (!pt->left() || !pt->right()) - return; - - SimpleFilter* lsf = dynamic_cast(pt->left()->data()); - SimpleFilter* rsf = dynamic_cast(pt->right()->data()); - - if (!lsf || !rsf) - return; - - // (a=b or isnull(item))/(a=b or isnotnull(item)) case. - // swap the lhs and rhs operands of the OR operator. - if ((lsf->op()->op() == execplan::OP_ISNULL || - lsf->op()->op() == execplan::OP_ISNOTNULL) && - rsf->op()->op() == execplan::OP_EQ) - { - ParseTree* temp = pt->left(); - pt->left(pt->right()); - pt->right(temp); - } - } - } - else if (cond->functype() == Item_func::EQ_FUNC) - { - // not in (select const ...) - if (gwip->ptWorkStack.empty()) - return; - - ParseTree* pt = gwip->ptWorkStack.top(); - SimpleFilter* sf = dynamic_cast(pt->data()); - - if (!sf || sf->op()->op() != execplan::OP_EQ) - return; - - if (sf->lhs()->joinInfo() & JOIN_CORRELATED) - sf->lhs()->joinInfo(sf->lhs()->joinInfo() | JOIN_NULLMATCH_CANDIDATE); - - if (sf->rhs()->joinInfo() & JOIN_CORRELATED) - sf->rhs()->joinInfo(sf->rhs()->joinInfo() | JOIN_NULLMATCH_CANDIDATE); - } + fGwip.fatalParseError = true; + fGwip.parseErrorText = "Unsupported item in IN subquery"; + return; } + + Item_cond* cond; + + if (func->functype() == Item_func::TRIG_COND_FUNC) + { + Item* item; + + if (func->arguments()[0]->type() == Item::REF_ITEM) + item = (Item_ref*)(func->arguments()[0])->real_item(); + else + item = func->arguments()[0]; + + cond = (Item_cond*)(item); + } + else + { + cond = (Item_cond*)(func); + } + + if (cond->functype() == Item_func::COND_OR_FUNC) + { + // (cache=item) case. do nothing. ignore trigcond()? + if (cond->argument_list()->elements == 1) + return; + + if (cond->argument_list()->elements == 2) + { + // don't know how to deal with this. don't think it's a fatal error either. + if (gwip->ptWorkStack.empty()) + return; + + ParseTree* pt = gwip->ptWorkStack.top(); + + if (!pt->left() || !pt->right()) + return; + + SimpleFilter* lsf = dynamic_cast(pt->left()->data()); + SimpleFilter* rsf = dynamic_cast(pt->right()->data()); + + if (!lsf || !rsf) + return; + + // (a=b or isnull(item))/(a=b or isnotnull(item)) case. + // swap the lhs and rhs operands of the OR operator. + if ((lsf->op()->op() == execplan::OP_ISNULL || lsf->op()->op() == execplan::OP_ISNOTNULL) && + rsf->op()->op() == execplan::OP_EQ) + { + ParseTree* temp = pt->left(); + pt->left(pt->right()); + pt->right(temp); + } + } + } + else if (cond->functype() == Item_func::EQ_FUNC) + { + // not in (select const ...) + if (gwip->ptWorkStack.empty()) + return; + + ParseTree* pt = gwip->ptWorkStack.top(); + SimpleFilter* sf = dynamic_cast(pt->data()); + + if (!sf || sf->op()->op() != execplan::OP_EQ) + return; + + if (sf->lhs()->joinInfo() & JOIN_CORRELATED) + sf->lhs()->joinInfo(sf->lhs()->joinInfo() | JOIN_NULLMATCH_CANDIDATE); + + if (sf->rhs()->joinInfo() & JOIN_CORRELATED) + sf->rhs()->joinInfo(sf->rhs()->joinInfo() | JOIN_NULLMATCH_CANDIDATE); + } + } } /** @@ -323,20 +324,20 @@ void InSub::handleFunc(gp_walk_info* gwip, Item_func* func) */ void InSub::handleNot() { - ParseTree* pt = fGwip.ptWorkStack.top(); - ExistsFilter* subFilter = dynamic_cast(pt->data()); - idbassert(subFilter); - subFilter->notExists(true); - SCSEP csep = subFilter->sub(); - const ParseTree* ptsub = csep->filters(); + ParseTree* pt = fGwip.ptWorkStack.top(); + ExistsFilter* subFilter = dynamic_cast(pt->data()); + idbassert(subFilter); + subFilter->notExists(true); + SCSEP csep = subFilter->sub(); + const ParseTree* ptsub = csep->filters(); - if (ptsub) - ptsub->walk(makeAntiJoin); + if (ptsub) + ptsub->walk(makeAntiJoin); - ptsub = csep->having(); + ptsub = csep->having(); - if (ptsub) - ptsub->walk(makeAntiJoin); + if (ptsub) + ptsub->walk(makeAntiJoin); } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index d0218b524..dd10c9f36 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -35,20 +35,17 @@ handlerton* mcs_hton = NULL; // This is the maria handlerton that we need for the cache -static handlerton *mcs_maria_hton = NULL; +static handlerton* mcs_maria_hton = NULL; char cs_version[25]; -char cs_commit_hash[41]; // a commit hash is 40 characters +char cs_commit_hash[41]; // a commit hash is 40 characters // handlers creation function for hton. // Look into ha_mcs_pushdown.* for more details. -group_by_handler* -create_columnstore_group_by_handler(THD* thd, Query* query); +group_by_handler* create_columnstore_group_by_handler(THD* thd, Query* query); -derived_handler* -create_columnstore_derived_handler(THD* thd, TABLE_LIST *derived); +derived_handler* create_columnstore_derived_handler(THD* thd, TABLE_LIST* derived); -select_handler* -create_columnstore_select_handler(THD* thd, SELECT_LEX* sel); +select_handler* create_columnstore_select_handler(THD* thd, SELECT_LEX* sel); /* Variables for example share methods */ @@ -76,101 +73,97 @@ pthread_mutex_t mcs_mutex; Function we use in the creation of our hash to get key. */ -static uchar* mcs_get_key(COLUMNSTORE_SHARE* share, size_t* length, - my_bool not_used __attribute__((unused))) +static uchar* mcs_get_key(COLUMNSTORE_SHARE* share, size_t* length, my_bool not_used __attribute__((unused))) { - *length = share->table_name_length; - return (uchar*) share->table_name; + *length = share->table_name_length; + return (uchar*)share->table_name; } // This one is unused int mcs_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) { - DBUG_ENTER("mcs_discover"); - DBUG_PRINT("mcs_discover", ("db: '%s' name: '%s'", share->db.str, share->table_name.str)); + DBUG_ENTER("mcs_discover"); + DBUG_PRINT("mcs_discover", ("db: '%s' name: '%s'", share->db.str, share->table_name.str)); #ifdef INFINIDB_DEBUG - fprintf(stderr, "mcs_discover()\n"); + fprintf(stderr, "mcs_discover()\n"); #endif - uchar* frm_data = NULL; - size_t frm_len = 0; - int error = 0; + uchar* frm_data = NULL; + size_t frm_len = 0; + int error = 0; - if (!ha_mcs_impl_discover_existence(share->db.str, share->table_name.str)) - DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); + if (!ha_mcs_impl_discover_existence(share->db.str, share->table_name.str)) + DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); - error = share->read_frm_image((const uchar**)&frm_data, &frm_len); + error = share->read_frm_image((const uchar**)&frm_data, &frm_len); - if (error) - DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); + if (error) + DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); - my_errno = share->init_from_binary_frm_image(thd, 1, frm_data, frm_len); + my_errno = share->init_from_binary_frm_image(thd, 1, frm_data, frm_len); - my_free(frm_data); - DBUG_RETURN(my_errno); + my_free(frm_data); + DBUG_RETURN(my_errno); } // This f() is also unused -int mcs_discover_existence(handlerton* hton, const char* db, - const char* table_name) +int mcs_discover_existence(handlerton* hton, const char* db, const char* table_name) { - return ha_mcs_impl_discover_existence(db, table_name); + return ha_mcs_impl_discover_existence(db, table_name); } static int mcs_commit(handlerton* hton, THD* thd, bool all) { - int rc; - try - { - rc = ha_mcs_impl_commit(hton, thd, all); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - return rc; + int rc; + try + { + rc = ha_mcs_impl_commit(hton, thd, all); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + return rc; } static int mcs_rollback(handlerton* hton, THD* thd, bool all) { - int rc; - try - { - rc = ha_mcs_impl_rollback(hton, thd, all); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - return rc; + int rc; + try + { + rc = ha_mcs_impl_rollback(hton, thd, all); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + return rc; } static int mcs_close_connection(handlerton* hton, THD* thd) { - int rc; - try - { - rc = ha_mcs_impl_close_connection(hton, thd); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - return rc; + int rc; + try + { + rc = ha_mcs_impl_close_connection(hton, thd); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + return rc; } -ha_mcs::ha_mcs(handlerton* hton, TABLE_SHARE* table_arg) : - handler(hton, table_arg), - int_table_flags(HA_BINLOG_STMT_CAPABLE | HA_BINLOG_ROW_CAPABLE | - HA_TABLE_SCAN_ON_INDEX | - HA_CAN_TABLE_CONDITION_PUSHDOWN | - HA_CAN_DIRECT_UPDATE_AND_DELETE), - m_lock_type(F_UNLCK) -{ } - +ha_mcs::ha_mcs(handlerton* hton, TABLE_SHARE* table_arg) + : handler(hton, table_arg) + , int_table_flags(HA_BINLOG_STMT_CAPABLE | HA_BINLOG_ROW_CAPABLE | HA_TABLE_SCAN_ON_INDEX | + HA_CAN_TABLE_CONDITION_PUSHDOWN | HA_CAN_DIRECT_UPDATE_AND_DELETE) + , m_lock_type(F_UNLCK) +{ +} /** @brief @@ -190,32 +183,29 @@ ha_mcs::ha_mcs(handlerton* hton, TABLE_SHARE* table_arg) : delete_table method in handler.cc */ -static const char* ha_mcs_exts[] = -{ - NullS -}; +static const char* ha_mcs_exts[] = {NullS}; const char** ha_mcs::bas_ext() const { - return ha_mcs_exts; + return ha_mcs_exts; } int ha_mcs::analyze(THD* thd, HA_CHECK_OPT* check_opt) { - DBUG_ENTER("ha_mcs::analyze"); + DBUG_ENTER("ha_mcs::analyze"); - int rc; - try - { - rc = ha_mcs_impl_analyze(thd, table); - } - catch (std::runtime_error& e) - { - thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + int rc; + try + { + rc = ha_mcs_impl_analyze(thd, table); + } + catch (std::runtime_error& e) + { + thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /** @@ -236,46 +226,44 @@ int ha_mcs::analyze(THD* thd, HA_CHECK_OPT* check_opt) int ha_mcs::open(const char* name, int mode, uint32_t test_if_locked) { - DBUG_ENTER("ha_mcs::open"); + DBUG_ENTER("ha_mcs::open"); - bool isPS = current_thd->stmt_arena && - (current_thd->stmt_arena->is_stmt_prepare() || - current_thd->stmt_arena->is_stmt_execute()); + bool isPS = current_thd->stmt_arena && + (current_thd->stmt_arena->is_stmt_prepare() || current_thd->stmt_arena->is_stmt_execute()); - // MCOL-4282 See the description for discover_check_version() in ha_mcs.h - // for why we need to mutate optimizer flags here. Sequence of SQL - // statements that will lead to this execution path for prepared - // statements: - // CREATE TABLE t1 (a int, b int) engine=columnstore; - // PREPARE stmt1 FROM "SELECT * FROM t1"; - // EXECUTE stmt1; - if (isPS) - mutate_optimizer_flags(current_thd); + // MCOL-4282 See the description for discover_check_version() in ha_mcs.h + // for why we need to mutate optimizer flags here. Sequence of SQL + // statements that will lead to this execution path for prepared + // statements: + // CREATE TABLE t1 (a int, b int) engine=columnstore; + // PREPARE stmt1 FROM "SELECT * FROM t1"; + // EXECUTE stmt1; + if (isPS) + mutate_optimizer_flags(current_thd); - int rc; - try - { - rc = ha_mcs_impl_open(name, mode, test_if_locked); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + int rc; + try + { + rc = ha_mcs_impl_open(name, mode, test_if_locked); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } int ha_mcs::discover_check_version() { - bool isPS = current_thd->stmt_arena && - (current_thd->stmt_arena->is_stmt_prepare() || - current_thd->stmt_arena->is_stmt_execute()); + bool isPS = current_thd->stmt_arena && + (current_thd->stmt_arena->is_stmt_prepare() || current_thd->stmt_arena->is_stmt_execute()); - if (isPS) - mutate_optimizer_flags(current_thd); + if (isPS) + mutate_optimizer_flags(current_thd); - return 0; + return 0; } /** @@ -296,23 +284,22 @@ int ha_mcs::discover_check_version() int ha_mcs::close(void) { - DBUG_ENTER("ha_mcs::close"); + DBUG_ENTER("ha_mcs::close"); - int rc; - try - { - rc = ha_mcs_impl_close(); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + int rc; + try + { + rc = ha_mcs_impl_close(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } - /** @brief write_row() inserts a row. No extra() hint is given currently if a bulk load @@ -327,63 +314,63 @@ int ha_mcs::close(void) int ha_mcs::write_row(const uchar* buf) { - DBUG_ENTER("ha_mcs::write_row"); - int rc; - try - { - rc = ha_mcs_impl_write_row(buf, table, rows_changed); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + DBUG_ENTER("ha_mcs::write_row"); + int rc; + try + { + rc = ha_mcs_impl_write_row(buf, table, rows_changed); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } void ha_mcs::start_bulk_insert(ha_rows rows, uint flags) { - DBUG_ENTER("ha_mcs::start_bulk_insert"); - try - { - ha_mcs_impl_start_bulk_insert(rows, table); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - } - DBUG_VOID_RETURN; + DBUG_ENTER("ha_mcs::start_bulk_insert"); + try + { + ha_mcs_impl_start_bulk_insert(rows, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } + DBUG_VOID_RETURN; } void ha_mcs::start_bulk_insert_from_cache(ha_rows rows, uint flags) { - DBUG_ENTER("ha_mcs::start_bulk_insert_from_cache"); - try - { - ha_mcs_impl_start_bulk_insert(rows, table, true); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - } - DBUG_VOID_RETURN; + DBUG_ENTER("ha_mcs::start_bulk_insert_from_cache"); + try + { + ha_mcs_impl_start_bulk_insert(rows, table, true); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } + DBUG_VOID_RETURN; } int ha_mcs::end_bulk_insert() { - DBUG_ENTER("ha_mcs::end_bulk_insert"); - int rc; - try - { - rc = ha_mcs_impl_end_bulk_insert(false, table); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::end_bulk_insert"); + int rc; + try + { + rc = ha_mcs_impl_end_bulk_insert(false, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } /** @@ -401,19 +388,18 @@ int ha_mcs::end_bulk_insert() */ int ha_mcs::update_row(const uchar* old_data, const uchar* new_data) { - - DBUG_ENTER("ha_mcs::update_row"); - int rc; - try - { - rc = ha_mcs_impl_update_row(); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::update_row"); + int rc; + try + { + rc = ha_mcs_impl_update_row(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } /** @@ -429,65 +415,65 @@ int ha_mcs::update_row(const uchar* old_data, const uchar* new_data) @see mysql_update()/mysql_delete */ -int ha_mcs::direct_update_rows_init(List *update_fields) +int ha_mcs::direct_update_rows_init(List* update_fields) { - DBUG_ENTER("ha_mcs::direct_update_rows_init"); - DBUG_RETURN(0); + DBUG_ENTER("ha_mcs::direct_update_rows_init"); + DBUG_RETURN(0); } -int ha_mcs::direct_update_rows(ha_rows *update_rows) +int ha_mcs::direct_update_rows(ha_rows* update_rows) { - DBUG_ENTER("ha_mcs::direct_update_rows"); - int rc; - try - { - rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows, condStack); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::direct_update_rows"); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows, condStack); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } -int ha_mcs::direct_update_rows(ha_rows *update_rows, ha_rows *found_rows) +int ha_mcs::direct_update_rows(ha_rows* update_rows, ha_rows* found_rows) { - DBUG_ENTER("ha_mcs::direct_update_rows"); - int rc; - try - { - rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows, condStack); - *found_rows = *update_rows; - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::direct_update_rows"); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows, condStack); + *found_rows = *update_rows; + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } int ha_mcs::direct_delete_rows_init() { - DBUG_ENTER("ha_mcs::direct_delete_rows_init"); - DBUG_RETURN(0); + DBUG_ENTER("ha_mcs::direct_delete_rows_init"); + DBUG_RETURN(0); } -int ha_mcs::direct_delete_rows(ha_rows *deleted_rows) +int ha_mcs::direct_delete_rows(ha_rows* deleted_rows) { - DBUG_ENTER("ha_mcs::direct_delete_rows"); - int rc; - try - { - rc = ha_mcs_impl_direct_update_delete_rows(true, deleted_rows, condStack); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::direct_delete_rows"); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(true, deleted_rows, condStack); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } /** @brief @@ -511,18 +497,18 @@ int ha_mcs::direct_delete_rows(ha_rows *deleted_rows) int ha_mcs::delete_row(const uchar* buf) { - DBUG_ENTER("ha_mcs::delete_row"); - int rc; - try - { - rc = ha_mcs_impl_delete_row(); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::delete_row"); + int rc; + try + { + rc = ha_mcs_impl_delete_row(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } /** @@ -532,16 +518,13 @@ int ha_mcs::delete_row(const uchar* buf) index. */ -int ha_mcs::index_read_map(uchar* buf, const uchar* key, - key_part_map keypart_map __attribute__((unused)), - enum ha_rkey_function find_flag - __attribute__((unused))) +int ha_mcs::index_read_map(uchar* buf, const uchar* key, key_part_map keypart_map __attribute__((unused)), + enum ha_rkey_function find_flag __attribute__((unused))) { - DBUG_ENTER("ha_mcs::index_read"); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); + DBUG_ENTER("ha_mcs::index_read"); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } - /** @brief Used to read forward through the index. @@ -549,11 +532,10 @@ int ha_mcs::index_read_map(uchar* buf, const uchar* key, int ha_mcs::index_next(uchar* buf) { - DBUG_ENTER("ha_mcs::index_next"); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); + DBUG_ENTER("ha_mcs::index_next"); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } - /** @brief Used to read backwards through the index. @@ -561,11 +543,10 @@ int ha_mcs::index_next(uchar* buf) int ha_mcs::index_prev(uchar* buf) { - DBUG_ENTER("ha_mcs::index_prev"); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); + DBUG_ENTER("ha_mcs::index_prev"); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } - /** @brief index_first() asks for the first key in the index. @@ -578,11 +559,10 @@ int ha_mcs::index_prev(uchar* buf) */ int ha_mcs::index_first(uchar* buf) { - DBUG_ENTER("ha_mcs::index_first"); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); + DBUG_ENTER("ha_mcs::index_first"); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } - /** @brief index_last() asks for the last key in the index. @@ -595,11 +575,10 @@ int ha_mcs::index_first(uchar* buf) */ int ha_mcs::index_last(uchar* buf) { - DBUG_ENTER("ha_mcs::index_last"); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); + DBUG_ENTER("ha_mcs::index_last"); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } - /** @brief rnd_init() is called when the system wants the storage engine to do a table @@ -615,44 +594,43 @@ int ha_mcs::index_last(uchar* buf) */ int ha_mcs::rnd_init(bool scan) { - DBUG_ENTER("ha_mcs::rnd_init"); + DBUG_ENTER("ha_mcs::rnd_init"); - int rc = 0; - if(scan) + int rc = 0; + if (scan) + { + try { - try - { - rc = impl_rnd_init(table, condStack); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + rc = impl_rnd_init(table, condStack); } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } int ha_mcs::rnd_end() { - DBUG_ENTER("ha_mcs::rnd_end"); + DBUG_ENTER("ha_mcs::rnd_end"); - int rc; - try - { - rc = ha_mcs_impl_rnd_end(table); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + int rc; + try + { + rc = ha_mcs_impl_rnd_end(table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } - /** @brief This is called for each row of the table scan. When you run out of records @@ -669,23 +647,22 @@ int ha_mcs::rnd_end() */ int ha_mcs::rnd_next(uchar* buf) { - DBUG_ENTER("ha_mcs::rnd_next"); + DBUG_ENTER("ha_mcs::rnd_next"); - int rc; - try - { - rc = ha_mcs_impl_rnd_next(buf, table); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + int rc; + try + { + rc = ha_mcs_impl_rnd_next(buf, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } - /** @brief position() is called after each call to rnd_next() if the data needs @@ -715,11 +692,10 @@ int ha_mcs::rnd_next(uchar* buf) // to retrieve records quickly by this "key" void ha_mcs::position(const uchar* record) { - DBUG_ENTER("ha_mcs::position"); - DBUG_VOID_RETURN; + DBUG_ENTER("ha_mcs::position"); + DBUG_VOID_RETURN; } - /** @brief This is like rnd_next, but you are given a position to use @@ -735,21 +711,20 @@ void ha_mcs::position(const uchar* record) */ int ha_mcs::rnd_pos(uchar* buf, uchar* pos) { - DBUG_ENTER("ha_mcs::rnd_pos"); - int rc; - try - { - rc = ha_mcs_impl_rnd_pos(buf, pos); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::rnd_pos"); + int rc; + try + { + rc = ha_mcs_impl_rnd_pos(buf, pos); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } - /** @brief ::info() is used to return information to the optimizer. See my_base.h for @@ -790,17 +765,16 @@ int ha_mcs::rnd_pos(uchar* buf, uchar* pos) */ int ha_mcs::info(uint32_t flag) { - DBUG_ENTER("ha_mcs::info"); - // @bug 1635. Raise this number magically fix the filesort crash issue. May need to twist - // the number again if the issue re-occurs - stats.records = 2000; + DBUG_ENTER("ha_mcs::info"); + // @bug 1635. Raise this number magically fix the filesort crash issue. May need to twist + // the number again if the issue re-occurs + stats.records = 2000; #ifdef INFINIDB_DEBUG - puts("info"); + puts("info"); #endif - DBUG_RETURN(0); + DBUG_RETURN(0); } - /** @brief extra() is called whenever the server wishes to send a hint to @@ -812,44 +786,31 @@ int ha_mcs::info(uint32_t flag) */ int ha_mcs::extra(enum ha_extra_function operation) { - DBUG_ENTER("ha_mcs::extra"); + DBUG_ENTER("ha_mcs::extra"); #ifdef INFINIDB_DEBUG + { + const char* hefs; + + switch (operation) { - const char* hefs; + case HA_EXTRA_NO_READCHECK: hefs = "HA_EXTRA_NO_READCHECK"; break; - switch (operation) - { - case HA_EXTRA_NO_READCHECK: - hefs = "HA_EXTRA_NO_READCHECK"; - break; + case HA_EXTRA_CACHE: hefs = "HA_EXTRA_CACHE"; break; - case HA_EXTRA_CACHE: - hefs = "HA_EXTRA_CACHE"; - break; + case HA_EXTRA_NO_CACHE: hefs = "HA_EXTRA_NO_CACHE"; break; - case HA_EXTRA_NO_CACHE: - hefs = "HA_EXTRA_NO_CACHE"; - break; + case HA_EXTRA_NO_IGNORE_DUP_KEY: hefs = "HA_EXTRA_NO_IGNORE_DUP_KEY"; break; - case HA_EXTRA_NO_IGNORE_DUP_KEY: - hefs = "HA_EXTRA_NO_IGNORE_DUP_KEY"; - break; + case HA_EXTRA_PREPARE_FOR_RENAME: hefs = "HA_EXTRA_PREPARE_FOR_RENAME"; break; - case HA_EXTRA_PREPARE_FOR_RENAME: - hefs = "HA_EXTRA_PREPARE_FOR_RENAME"; - break; - - default: - hefs = "UNKNOWN ENUM!"; - break; - } - - fprintf(stderr, "ha_mcs::extra(\"%s\", %d: %s)\n", table->s->table_name.str, operation, hefs); + default: hefs = "UNKNOWN ENUM!"; break; } -#endif - DBUG_RETURN(0); -} + fprintf(stderr, "ha_mcs::extra(\"%s\", %d: %s)\n", table->s->table_name.str, operation, hefs); + } +#endif + DBUG_RETURN(0); +} /** @brief @@ -872,11 +833,10 @@ int ha_mcs::extra(enum ha_extra_function operation) */ int ha_mcs::delete_all_rows() { - DBUG_ENTER("ha_mcs::delete_all_rows"); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); + DBUG_ENTER("ha_mcs::delete_all_rows"); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } - /** @brief This create a lock on the table. If you are implementing a storage engine @@ -896,26 +856,25 @@ int ha_mcs::delete_all_rows() */ int ha_mcs::external_lock(THD* thd, int lock_type) { - DBUG_ENTER("ha_mcs::external_lock"); + DBUG_ENTER("ha_mcs::external_lock"); - int rc; - try - { - //@Bug 2526 Only register the transaction when autocommit is off - if ((thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) - trans_register_ha( thd, true, mcs_hton, 0); + int rc; + try + { + //@Bug 2526 Only register the transaction when autocommit is off + if ((thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) + trans_register_ha(thd, true, mcs_hton, 0); - rc = impl_external_lock(thd, table, lock_type); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + rc = impl_external_lock(thd, table, lock_type); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } - /** @brief The idea with handler::store_lock() is: The statement decides which locks @@ -954,20 +913,17 @@ int ha_mcs::external_lock(THD* thd, int lock_type) get_lock_data() in lock.cc */ -THR_LOCK_DATA** ha_mcs::store_lock(THD* thd, - THR_LOCK_DATA** to, - enum thr_lock_type lock_type) +THR_LOCK_DATA** ha_mcs::store_lock(THD* thd, THR_LOCK_DATA** to, enum thr_lock_type lock_type) { - //if (lock_type != TL_IGNORE && lock.type == TL_UNLOCK) - // lock.type=lock_type; - //*to++= &lock; + // if (lock_type != TL_IGNORE && lock.type == TL_UNLOCK) + // lock.type=lock_type; + //*to++= &lock; #ifdef INFINIDB_DEBUG - puts("store_lock"); + puts("store_lock"); #endif - return to; + return to; } - /** @brief Used to delete a table. By the time delete_table() has been called all @@ -989,25 +945,24 @@ THR_LOCK_DATA** ha_mcs::store_lock(THD* thd, */ int ha_mcs::delete_table(const char* name) { - DBUG_ENTER("ha_mcs::delete_table"); - /* This is not implemented but we want someone to be able that it works. */ + DBUG_ENTER("ha_mcs::delete_table"); + /* This is not implemented but we want someone to be able that it works. */ - int rc; + int rc; - try - { - rc = ha_mcs_impl_delete_table(name); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } + try + { + rc = ha_mcs_impl_delete_table(name); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } - DBUG_RETURN(rc); + DBUG_RETURN(rc); } - /** @brief Renames a table from one name to another via an alter table call. @@ -1024,21 +979,20 @@ int ha_mcs::delete_table(const char* name) */ int ha_mcs::rename_table(const char* from, const char* to) { - DBUG_ENTER("ha_mcs::rename_table "); - int rc; - try - { - rc = ha_mcs_impl_rename_table(from, to); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + DBUG_ENTER("ha_mcs::rename_table "); + int rc; + try + { + rc = ha_mcs_impl_rename_table(from, to); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } - /** @brief Given a starting key and an ending key, estimate the number of rows that @@ -1052,14 +1006,13 @@ int ha_mcs::rename_table(const char* from, const char* to) @see check_quick_keys() in opt_range.cc */ -ha_rows ha_mcs::records_in_range(uint32_t inx, const key_range* min_key, - const key_range* max_key, page_range* res) +ha_rows ha_mcs::records_in_range(uint32_t inx, const key_range* min_key, const key_range* max_key, + page_range* res) { - DBUG_ENTER("ha_mcs::records_in_range"); - DBUG_RETURN(10); // low number to force index usage + DBUG_ENTER("ha_mcs::records_in_range"); + DBUG_RETURN(10); // low number to force index usage } - /** @brief create() is called to create a database. The variable name will have the name @@ -1079,86 +1032,81 @@ ha_rows ha_mcs::records_in_range(uint32_t inx, const key_range* min_key, ha_create_table() in handle.cc */ -int ha_mcs::create(const char* name, TABLE* table_arg, - HA_CREATE_INFO* create_info) +int ha_mcs::create(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info) { - DBUG_ENTER("ha_mcs::create"); + DBUG_ENTER("ha_mcs::create"); - int rc; - try - { - rc = ha_mcs_impl_create(name, table_arg, create_info); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - DBUG_RETURN(rc); + int rc; + try + { + rc = ha_mcs_impl_create(name, table_arg, create_info); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } const COND* ha_mcs::cond_push(const COND* cond) { - DBUG_ENTER("ha_mcs::cond_push"); - COND* ret_cond = NULL; - try - { - ret_cond = ha_mcs_impl_cond_push(const_cast(cond), table, condStack); - } - catch (std::runtime_error& e) - { - current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - } - DBUG_RETURN(ret_cond); + DBUG_ENTER("ha_mcs::cond_push"); + COND* ret_cond = NULL; + try + { + ret_cond = ha_mcs_impl_cond_push(const_cast(cond), table, condStack); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } + DBUG_RETURN(ret_cond); } void ha_mcs::cond_pop() { - DBUG_ENTER("ha_mcs::cond_pop"); + DBUG_ENTER("ha_mcs::cond_pop"); - THD* thd = current_thd; + THD* thd = current_thd; - if ((((thd->lex)->sql_command == SQLCOM_UPDATE) || - ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI) || - ((thd->lex)->sql_command == SQLCOM_DELETE) || - ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI)) && - !condStack.empty()) - { - condStack.pop_back(); - } + if ((((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI) || + ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI)) && + !condStack.empty()) + { + condStack.pop_back(); + } - DBUG_VOID_RETURN; + DBUG_VOID_RETURN; } int ha_mcs::reset() { - DBUG_ENTER("ha_mcs::reset"); + DBUG_ENTER("ha_mcs::reset"); - if (!condStack.empty()) - { - condStack.clear(); - } + if (!condStack.empty()) + { + condStack.clear(); + } - // Restore the optimizer flags which were mutated earlier in - // ha_mcs::open/ha_mcs::discover_check_version - restore_optimizer_flags(current_thd); + // Restore the optimizer flags which were mutated earlier in + // ha_mcs::open/ha_mcs::discover_check_version + restore_optimizer_flags(current_thd); - DBUG_RETURN(0); + DBUG_RETURN(0); } - - int ha_mcs::repair(THD* thd, HA_CHECK_OPT* check_opt) { - DBUG_ENTER("ha_mcs::repair"); - DBUG_ASSERT(!(int_table_flags & HA_CAN_REPAIR)); - DBUG_RETURN(HA_ADMIN_NOT_IMPLEMENTED); + DBUG_ENTER("ha_mcs::repair"); + DBUG_ASSERT(!(int_table_flags & HA_CAN_REPAIR)); + DBUG_RETURN(HA_ADMIN_NOT_IMPLEMENTED); } bool ha_mcs::is_crashed() const { - DBUG_ENTER("ha_mcs::is_crashed"); - DBUG_RETURN(0); + DBUG_ENTER("ha_mcs::is_crashed"); + DBUG_RETURN(0); } /* @@ -1201,19 +1149,17 @@ bool ha_mcs::is_crashed() const lock. */ -my_bool get_status_and_flush_cache(void *param, - my_bool concurrent_insert); - +my_bool get_status_and_flush_cache(void* param, my_bool concurrent_insert); /* Create a name for the cache table */ -static void create_cache_name(char *to, const char *name) +static void create_cache_name(char* to, const char* name) { - uint dir_length= dirname_length(name); - to= strnmov(to, name, dir_length); - strxmov(to, CACHE_PREFIX, name+ dir_length, NullS); + uint dir_length = dirname_length(name); + to = strnmov(to, name, dir_length); + strxmov(to, CACHE_PREFIX, name + dir_length, NullS); } /***************************************************************************** @@ -1230,46 +1176,40 @@ static void create_cache_name(char *to, const char *name) insert */ -my_bool get_status_and_flush_cache(void *param, - my_bool concurrent_insert) +my_bool get_status_and_flush_cache(void* param, my_bool concurrent_insert) { if (current_thd->slave_thread && !get_replication_slave(current_thd)) return (0); - ha_mcs_cache *cache= (ha_mcs_cache*) param; + ha_mcs_cache* cache = (ha_mcs_cache*)param; int error; - enum_sql_command sql_command= cache->table->in_use->lex->sql_command; + enum_sql_command sql_command = cache->table->in_use->lex->sql_command; - cache->sql_command= sql_command; + cache->sql_command = sql_command; - cache->insert_command= (sql_command == SQLCOM_INSERT || - sql_command == SQLCOM_LOAD); + cache->insert_command = (sql_command == SQLCOM_INSERT || sql_command == SQLCOM_LOAD); /* Call first the original Aria get_status function All Aria get_status functions takes Maria handler as the parameter */ if (cache->share->org_lock.get_status) - (*cache->share->org_lock.get_status)(cache->cache_handler->file, - concurrent_insert); + (*cache->share->org_lock.get_status)(cache->cache_handler->file, concurrent_insert); /* If first get_status() call for this table, flush cache if needed */ if (!cache->lock_counter++) { ha_rows num_rows = cache->num_rows_cached(); - if (((!cache->insert_command && num_rows != 0) || - num_rows >= get_cache_flush_threshold(current_thd)) && + if (((!cache->insert_command && num_rows != 0) || num_rows >= get_cache_flush_threshold(current_thd)) && // In replication, LDI on a master comes as sql_command = SQLCOM_END // on the slave, if binlog_format != STATEMENT. See mysql_load // function in sql/sql_load.cc to know why. If this is the case, // make sure we don't flush the cache. (!current_thd->slave_thread || sql_command != SQLCOM_END)) { - if ((error= cache->flush_insert_cache())) + if ((error = cache->flush_insert_cache())) { - my_error(error, MYF(MY_WME | ME_FATAL), - "Got error while trying to flush insert cache: %d", - my_errno); - return(1); + my_error(error, MYF(MY_WME | ME_FATAL), "Got error while trying to flush insert cache: %d", my_errno); + return (1); } } } @@ -1286,7 +1226,7 @@ my_bool get_status_and_flush_cache(void *param, my_bool cache_start_trans(void* param) { - ha_mcs_cache *cache= (ha_mcs_cache*) param; + ha_mcs_cache* cache = (ha_mcs_cache*)param; if (!cache->insert_command) { @@ -1299,42 +1239,39 @@ my_bool cache_start_trans(void* param) /* Pass through functions for all the THR_LOCK virtual functions */ -static void cache_copy_status(void* to, void *from) +static void cache_copy_status(void* to, void* from) { - ha_mcs_cache *to_cache= (ha_mcs_cache*) to, *from_cache= (ha_mcs_cache*) from; - (*to_cache->share->org_lock.copy_status)(to_cache->cache_handler->file, - from_cache->cache_handler->file); + ha_mcs_cache *to_cache = (ha_mcs_cache*)to, *from_cache = (ha_mcs_cache*)from; + (*to_cache->share->org_lock.copy_status)(to_cache->cache_handler->file, from_cache->cache_handler->file); } static void cache_update_status(void* param) { - ha_mcs_cache *cache= (ha_mcs_cache*) param; + ha_mcs_cache* cache = (ha_mcs_cache*)param; (*cache->share->org_lock.update_status)(cache->cache_handler->file); } -static void cache_restore_status(void *param) +static void cache_restore_status(void* param) { - ha_mcs_cache *cache= (ha_mcs_cache*) param; + ha_mcs_cache* cache = (ha_mcs_cache*)param; (*cache->share->org_lock.restore_status)(cache->cache_handler->file); } -static my_bool cache_check_status(void *param) +static my_bool cache_check_status(void* param) { - ha_mcs_cache *cache= (ha_mcs_cache*) param; + ha_mcs_cache* cache = (ha_mcs_cache*)param; return (*cache->share->org_lock.check_status)(cache->cache_handler->file); } - /***************************************************************************** ha_mcs_cache_share functions (Common storage for an open cache file) *****************************************************************************/ -static ha_mcs_cache_share *cache_share_list= 0; +static ha_mcs_cache_share* cache_share_list = 0; static PSI_mutex_key key_LOCK_cache_share; #ifdef HAVE_PSI_INTERFACE -static PSI_mutex_info all_mutexes[]= -{ - { &key_LOCK_cache_share, "LOCK_cache_share", PSI_FLAG_GLOBAL}, +static PSI_mutex_info all_mutexes[] = { + {&key_LOCK_cache_share, "LOCK_cache_share", PSI_FLAG_GLOBAL}, }; #endif static mysql_mutex_t LOCK_cache_share; @@ -1343,68 +1280,64 @@ static mysql_mutex_t LOCK_cache_share; Find or create a share */ -ha_mcs_cache_share *find_cache_share(const char *name, ulonglong cached_rows) +ha_mcs_cache_share* find_cache_share(const char* name, ulonglong cached_rows) { ha_mcs_cache_share *pos, *share; mysql_mutex_lock(&LOCK_cache_share); - for (pos= cache_share_list; pos; pos= pos->next) + for (pos = cache_share_list; pos; pos = pos->next) { if (!strcmp(pos->name, name)) { pos->open_count++; mysql_mutex_unlock(&LOCK_cache_share); - return(pos); + return (pos); } } - if (!(share= (ha_mcs_cache_share*) my_malloc(PSI_NOT_INSTRUMENTED, - sizeof(*share) + strlen(name)+1, - MYF(MY_FAE)))) + if (!(share = (ha_mcs_cache_share*)my_malloc(PSI_NOT_INSTRUMENTED, sizeof(*share) + strlen(name) + 1, + MYF(MY_FAE)))) { mysql_mutex_unlock(&LOCK_cache_share); return 0; } - share->name= (char*) (share+1); - share->open_count= 1; - share->cached_rows= cached_rows; - strmov((char*) share->name, name); - share->next= cache_share_list; - cache_share_list= share; + share->name = (char*)(share + 1); + share->open_count = 1; + share->cached_rows = cached_rows; + strmov((char*)share->name, name); + share->next = cache_share_list; + cache_share_list = share; mysql_mutex_unlock(&LOCK_cache_share); return share; } - /* Decrement open counter and free share if there is no more users */ void ha_mcs_cache_share::close() { - ha_mcs_cache_share *pos; + ha_mcs_cache_share* pos; mysql_mutex_lock(&LOCK_cache_share); if (!--open_count) { - ha_mcs_cache_share **prev= &cache_share_list; - for ( ; (pos= *prev) != this; prev= &pos->next) + ha_mcs_cache_share** prev = &cache_share_list; + for (; (pos = *prev) != this; prev = &pos->next) ; - *prev= next; + *prev = next; my_free(this); } mysql_mutex_unlock(&LOCK_cache_share); } - /***************************************************************************** ha_mcs_cache handler functions *****************************************************************************/ static plugin_ref plugin_maria = NULL; -ha_mcs_cache::ha_mcs_cache(handlerton *hton, TABLE_SHARE *table_arg, MEM_ROOT *mem_root) - :ha_mcs(mcs_hton, table_arg), isSysCatTable(false), isCacheDisabled(false) +ha_mcs_cache::ha_mcs_cache(handlerton* hton, TABLE_SHARE* table_arg, MEM_ROOT* mem_root) + : ha_mcs(mcs_hton, table_arg), isSysCatTable(false), isCacheDisabled(false) { - if (table_arg && table_arg->db.str && - !strcasecmp(table_arg->db.str, "calpontsys") && + if (table_arg && table_arg->db.str && !strcasecmp(table_arg->db.str, "calpontsys") && table_arg->table_name.str && (!strcasecmp(table_arg->table_name.str, "syscolumn") || !strcasecmp(table_arg->table_name.str, "systable"))) @@ -1414,31 +1347,29 @@ ha_mcs_cache::ha_mcs_cache(handlerton *hton, TABLE_SHARE *table_arg, MEM_ROOT *m { if (!plugin_maria) { - LEX_CSTRING name = { STRING_WITH_LEN("Aria") }; - plugin_maria = ha_resolve_by_name(0, &name, 0); - mcs_maria_hton = plugin_hton(plugin_maria); - int error = mcs_maria_hton == NULL; // Engine must exists! - if (error) - my_error(HA_ERR_INITIALIZATION, MYF(0), - "Could not find storage engine %s", name.str); + LEX_CSTRING name = {STRING_WITH_LEN("Aria")}; + plugin_maria = ha_resolve_by_name(0, &name, 0); + mcs_maria_hton = plugin_hton(plugin_maria); + int error = mcs_maria_hton == NULL; // Engine must exists! + if (error) + my_error(HA_ERR_INITIALIZATION, MYF(0), "Could not find storage engine %s", name.str); } assert(mcs_maria_hton); - cache_handler= (ha_maria*) mcs_maria_hton->create(mcs_maria_hton, table_arg, mem_root); - share= 0; - lock_counter= 0; - cache_locked= 0; + cache_handler = (ha_maria*)mcs_maria_hton->create(mcs_maria_hton, table_arg, mem_root); + share = 0; + lock_counter = 0; + cache_locked = 0; } } - ha_mcs_cache::~ha_mcs_cache() { if (get_cache_inserts(current_thd) && !isSysCatTable && cache_handler) { delete cache_handler; - cache_handler= NULL; + cache_handler = NULL; } } @@ -1447,30 +1378,29 @@ ha_mcs_cache::~ha_mcs_cache() cache handler */ -int ha_mcs_cache::create(const char *name, TABLE *table_arg, - HA_CREATE_INFO *ha_create_info) +int ha_mcs_cache::create(const char* name, TABLE* table_arg, HA_CREATE_INFO* ha_create_info) { int error; - char cache_name[FN_REFLEN+8]; + char cache_name[FN_REFLEN + 8]; DBUG_ENTER("ha_mcs_cache::create"); if (isCacheEnabled()) { create_cache_name(cache_name, name); /* Create a cached table */ - ha_choice save_transactional= ha_create_info->transactional; - row_type save_row_type= ha_create_info->row_type; - ha_create_info->transactional= HA_CHOICE_NO; - ha_create_info->row_type= ROW_TYPE_DYNAMIC; + ha_choice save_transactional = ha_create_info->transactional; + row_type save_row_type = ha_create_info->row_type; + ha_create_info->transactional = HA_CHOICE_NO; + ha_create_info->row_type = ROW_TYPE_DYNAMIC; - if ((error= cache_handler->create(cache_name, table_arg, ha_create_info))) + if ((error = cache_handler->create(cache_name, table_arg, ha_create_info))) DBUG_RETURN(error); - ha_create_info->transactional= save_transactional; - ha_create_info->row_type= save_row_type; + ha_create_info->transactional = save_transactional; + ha_create_info->row_type = save_row_type; } /* Create the real table in ColumnStore */ - if ((error= parent::create(name, table_arg, ha_create_info))) + if ((error = parent::create(name, table_arg, ha_create_info))) { if (get_cache_inserts(current_thd) && !isSysCatTable) cache_handler->delete_table(cache_name); @@ -1480,8 +1410,7 @@ int ha_mcs_cache::create(const char *name, TABLE *table_arg, DBUG_RETURN(0); } - -int ha_mcs_cache::open(const char *name, int mode, uint open_flags) +int ha_mcs_cache::open(const char* name, int mode, uint open_flags) { int error, cache_error; DBUG_ENTER("ha_mcs_cache::open"); @@ -1491,19 +1420,19 @@ int ha_mcs_cache::open(const char *name, int mode, uint open_flags) /* Copy table object to cache_handler */ cache_handler->change_table_ptr(table, table->s); - char cache_name[FN_REFLEN+8]; + char cache_name[FN_REFLEN + 8]; create_cache_name(cache_name, name); - if (!(cache_error= cache_handler->open(cache_name, mode, open_flags))) + if (!(cache_error = cache_handler->open(cache_name, mode, open_flags))) { - if (!(share= find_cache_share(name, cache_handler->file->state->records))) + if (!(share = find_cache_share(name, cache_handler->file->state->records))) { cache_handler->close(); DBUG_RETURN(ER_OUTOFMEMORY); } /* Fix lock so that it goes through get_status_and_flush() */ - THR_LOCK *lock= &cache_handler->file->s->lock; + THR_LOCK* lock = &cache_handler->file->s->lock; if (lock->get_status != &get_status_and_flush_cache) { mysql_mutex_lock(&cache_handler->file->s->intern_lock); @@ -1515,28 +1444,28 @@ int ha_mcs_cache::open(const char *name, int mode, uint open_flags) if (lock->get_status != &get_status_and_flush_cache) { /* Remember original lock. Used by the THR_lock cache functions */ - share->org_lock= lock[0]; + share->org_lock = lock[0]; if (lock->start_trans) - lock->start_trans= &cache_start_trans; + lock->start_trans = &cache_start_trans; if (lock->copy_status) - lock->copy_status= &cache_copy_status; + lock->copy_status = &cache_copy_status; if (lock->update_status) - lock->update_status= &cache_update_status; + lock->update_status = &cache_update_status; if (lock->restore_status) - lock->restore_status= &cache_restore_status; + lock->restore_status = &cache_restore_status; if (lock->check_status) - lock->check_status= &cache_check_status; + lock->check_status = &cache_check_status; if (lock->restore_status) - lock->restore_status= &cache_restore_status; - lock->get_status= &get_status_and_flush_cache; + lock->restore_status = &cache_restore_status; + lock->get_status = &get_status_and_flush_cache; } mysql_mutex_unlock(&cache_handler->file->s->intern_lock); } - cache_handler->file->lock.status_param= (void*) this; + cache_handler->file->lock.status_param = (void*)this; } else if (cache_error == ENOENT) { - if (!(error= parent::open(name, mode, open_flags))) + if (!(error = parent::open(name, mode, open_flags))) { isCacheDisabled = true; DBUG_RETURN(0); @@ -1552,7 +1481,7 @@ int ha_mcs_cache::open(const char *name, int mode, uint open_flags) } } - if ((error= parent::open(name, mode, open_flags))) + if ((error = parent::open(name, mode, open_flags))) { if (get_cache_inserts(current_thd) && !isSysCatTable) cache_handler->close(); @@ -1562,7 +1491,6 @@ int ha_mcs_cache::open(const char *name, int mode, uint open_flags) DBUG_RETURN(0); } - int ha_mcs_cache::close() { int error, error2; @@ -1571,22 +1499,21 @@ int ha_mcs_cache::close() if (isCacheEnabled()) { - error= cache_handler->close(); - if ((error2= parent::close())) - error= error2; - ha_mcs_cache_share *org_share= share; + error = cache_handler->close(); + if ((error2 = parent::close())) + error = error2; + ha_mcs_cache_share* org_share = share; if (org_share) org_share->close(); } else { - error= parent::close(); + error = parent::close(); } DBUG_RETURN(error); } - /* Handling locking of the tables. In case of INSERT we have to lock both the cache handler and main table. If not, we only lock the main table @@ -1608,23 +1535,20 @@ uint ha_mcs_cache::lock_count(void) const Store locks for the Aria table and ColumnStore table */ -THR_LOCK_DATA **ha_mcs_cache::store_lock(THD *thd, - THR_LOCK_DATA **to, - enum thr_lock_type lock_type) +THR_LOCK_DATA** ha_mcs_cache::store_lock(THD* thd, THR_LOCK_DATA** to, enum thr_lock_type lock_type) { if (isCacheEnabled()) - to= cache_handler->store_lock(thd, to, TL_WRITE); + to = cache_handler->store_lock(thd, to, TL_WRITE); return parent::store_lock(thd, to, lock_type); } - /** Do external locking of the tables */ -int ha_mcs_cache::external_lock(THD *thd, int lock_type) +int ha_mcs_cache::external_lock(THD* thd, int lock_type) { - int error= 0; + int error = 0; DBUG_ENTER("ha_mcs_cache::external_lock"); if (isCacheEnabled()) @@ -1633,83 +1557,81 @@ int ha_mcs_cache::external_lock(THD *thd, int lock_type) Reset lock_counter. This is ok as external_lock() is guaranteed to be called before first get_status() */ - lock_counter= 0; + lock_counter = 0; if (lock_type == F_UNLCK) { int error2; - error= 0; + error = 0; if (cache_locked) { - error= cache_handler->external_lock(thd, lock_type); - cache_locked= 0; + error = cache_handler->external_lock(thd, lock_type); + cache_locked = 0; } - if ((error2= parent::external_lock(thd, lock_type))) - error= error2; + if ((error2 = parent::external_lock(thd, lock_type))) + error = error2; DBUG_RETURN(error); } /* Lock first with write lock to be able to do insert or flush table */ - original_lock_type= lock_type; - lock_type= F_WRLCK; - if ((error= cache_handler->external_lock(thd, lock_type))) + original_lock_type = lock_type; + lock_type = F_WRLCK; + if ((error = cache_handler->external_lock(thd, lock_type))) DBUG_RETURN(error); - if ((error= parent::external_lock(thd, lock_type))) + if ((error = parent::external_lock(thd, lock_type))) { - error= cache_handler->external_lock(thd, F_UNLCK); + error = cache_handler->external_lock(thd, F_UNLCK); DBUG_RETURN(error); } - cache_locked= 1; + cache_locked = 1; } else { - error= parent::external_lock(thd, lock_type); + error = parent::external_lock(thd, lock_type); } DBUG_RETURN(error); } - -int ha_mcs_cache::delete_table(const char *name) +int ha_mcs_cache::delete_table(const char* name) { - int cache_error= 0, error; + int cache_error = 0, error; DBUG_ENTER("ha_mcs_cache::delete_table"); if (isCacheEnabled()) { - char cache_name[FN_REFLEN+8]; + char cache_name[FN_REFLEN + 8]; create_cache_name(cache_name, name); - cache_error= cache_handler->delete_table(cache_name); + cache_error = cache_handler->delete_table(cache_name); if (cache_error && (cache_error != ENOENT)) DBUG_RETURN(cache_error); else if (cache_error == ENOENT) - cache_error= 0; + cache_error = 0; } - if ((error= parent::delete_table(name))) - cache_error= error; + if ((error = parent::delete_table(name))) + cache_error = error; DBUG_RETURN(cache_error); } - -int ha_mcs_cache::rename_table(const char *from, const char *to) +int ha_mcs_cache::rename_table(const char* from, const char* to) { - int error= 0; + int error = 0; DBUG_ENTER("ha_mcs_cache::rename_table"); if (isCacheEnabled()) { - char cache_from[FN_REFLEN+8], cache_to[FN_REFLEN+8]; + char cache_from[FN_REFLEN + 8], cache_to[FN_REFLEN + 8]; create_cache_name(cache_from, from); create_cache_name(cache_to, to); - if (!(error= cache_handler->rename_table(cache_from, cache_to))) + if (!(error = cache_handler->rename_table(cache_from, cache_to))) { - if ((error= parent::rename_table(from, to))) + if ((error = parent::rename_table(from, to))) { cache_handler->rename_table(cache_to, cache_from); DBUG_RETURN(error); @@ -1717,7 +1639,7 @@ int ha_mcs_cache::rename_table(const char *from, const char *to) } else if (error == ENOENT) { - if ((error= parent::rename_table(from, to))) + if ((error = parent::rename_table(from, to))) { DBUG_RETURN(error); } @@ -1725,34 +1647,32 @@ int ha_mcs_cache::rename_table(const char *from, const char *to) } else { - error= parent::rename_table(from, to); + error = parent::rename_table(from, to); } DBUG_RETURN(error); } - int ha_mcs_cache::delete_all_rows(void) { - int error= 0, error2; + int error = 0, error2; DBUG_ENTER("ha_mcs_cache::delete_all_rows"); if (isCacheEnabled()) { - error= cache_handler->delete_all_rows(); - share->cached_rows= 0; + error = cache_handler->delete_all_rows(); + share->cached_rows = 0; } - if ((error2= parent::delete_all_rows())) - error= error2; + if ((error2 = parent::delete_all_rows())) + error = error2; DBUG_RETURN(error); } bool ha_mcs_cache::is_crashed() const { if (isCacheEnabled()) - return (cache_handler->is_crashed() || - parent::is_crashed()); + return (cache_handler->is_crashed() || parent::is_crashed()); else return parent::is_crashed(); } @@ -1776,10 +1696,10 @@ bool ha_mcs_cache::is_crashed() const When these are updated, we threat the cache as committed */ -int ha_mcs_cache::repair(THD *thd, HA_CHECK_OPT *check_opt) +int ha_mcs_cache::repair(THD* thd, HA_CHECK_OPT* check_opt) { - int error= 0, error2; - int something_crashed= is_crashed(); + int error = 0, error2; + int something_crashed = is_crashed(); DBUG_ENTER("ha_mcs_cache::repair"); if (isCacheEnabled()) @@ -1787,30 +1707,27 @@ int ha_mcs_cache::repair(THD *thd, HA_CHECK_OPT *check_opt) if (cache_handler->is_crashed() || !something_crashed) { /* Delete everything that was not already committed */ - mysql_file_chsize(cache_handler->file->dfile.file, - cache_handler->file->s->state.state.data_file_length, + mysql_file_chsize(cache_handler->file->dfile.file, cache_handler->file->s->state.state.data_file_length, 0, MYF(MY_WME)); mysql_file_chsize(cache_handler->file->s->kfile.file, - cache_handler->file->s->state.state.key_file_length, - 0, MYF(MY_WME)); - check_opt->flags|= T_AUTO_REPAIR; - error= cache_handler->repair(thd, check_opt); - share->cached_rows= cache_handler->file->state->records; + cache_handler->file->s->state.state.key_file_length, 0, MYF(MY_WME)); + check_opt->flags |= T_AUTO_REPAIR; + error = cache_handler->repair(thd, check_opt); + share->cached_rows = cache_handler->file->state->records; } } if (parent::is_crashed() || !something_crashed) - if ((error2= parent::repair(thd, check_opt))) - error= error2; + if ((error2 = parent::repair(thd, check_opt))) + error = error2; DBUG_RETURN(error); } - /** Write to cache handler or main table */ -int ha_mcs_cache::write_row(const uchar *buf) +int ha_mcs_cache::write_row(const uchar* buf) { if (current_thd->slave_thread && !get_replication_slave(current_thd)) return (0); @@ -1825,7 +1742,6 @@ int ha_mcs_cache::write_row(const uchar *buf) return parent::write_row(buf); } - void ha_mcs_cache::start_bulk_insert(ha_rows rows, uint flags) { if (current_thd->slave_thread && !get_replication_slave(current_thd)) @@ -1855,7 +1771,6 @@ void ha_mcs_cache::start_bulk_insert(ha_rows rows, uint flags) } } - int ha_mcs_cache::end_bulk_insert() { if (current_thd->slave_thread && !get_replication_slave(current_thd)) @@ -1870,182 +1785,125 @@ int ha_mcs_cache::end_bulk_insert() ha_mcs_cache Plugin code ******************************************************************************/ -static handler *ha_mcs_cache_create_handler(handlerton *hton, - TABLE_SHARE *table, - MEM_ROOT *mem_root) +static handler* ha_mcs_cache_create_handler(handlerton* hton, TABLE_SHARE* table, MEM_ROOT* mem_root) { return new (mem_root) ha_mcs_cache(hton, table, mem_root); } - /****************************************************************************** ha_mcs Plugin code ******************************************************************************/ static int columnstore_init_func(void* p) { - DBUG_ENTER("columnstore_init_func"); + DBUG_ENTER("columnstore_init_func"); - struct tm tm; - time_t t; - time(&t); - localtime_r(&t, &tm); - fprintf(stderr, "%02d%02d%02d %2d:%02d:%02d ", - tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec); + struct tm tm; + time_t t; + time(&t); + localtime_r(&t, &tm); + fprintf(stderr, "%02d%02d%02d %2d:%02d:%02d ", tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec); - fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), columnstore_release.c_str()); + fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), + columnstore_release.c_str()); - strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version)); - cs_version[sizeof(cs_version) - 1] = 0; + strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version)); + cs_version[sizeof(cs_version) - 1] = 0; - strncpy(cs_commit_hash, columnstore_commit_hash.c_str(), sizeof(cs_commit_hash)); - cs_commit_hash[sizeof(cs_commit_hash) - 1] = 0; + strncpy(cs_commit_hash, columnstore_commit_hash.c_str(), sizeof(cs_commit_hash)); + cs_commit_hash[sizeof(cs_commit_hash) - 1] = 0; - mcs_hton = (handlerton*)p; + mcs_hton = (handlerton*)p; #ifndef _MSC_VER - (void) pthread_mutex_init(&mcs_mutex, MY_MUTEX_INIT_FAST); + (void)pthread_mutex_init(&mcs_mutex, MY_MUTEX_INIT_FAST); #endif - (void) my_hash_init(PSI_NOT_INSTRUMENTED, &mcs_open_tables, system_charset_info, 32, 0, 0, - (my_hash_get_key) mcs_get_key, 0, 0); + (void)my_hash_init(PSI_NOT_INSTRUMENTED, &mcs_open_tables, system_charset_info, 32, 0, 0, + (my_hash_get_key)mcs_get_key, 0, 0); - mcs_hton->create = ha_mcs_cache_create_handler; - mcs_hton->panic = 0; - mcs_hton->flags = HTON_CAN_RECREATE | HTON_NO_PARTITION; -// mcs_hton->discover_table = mcs_discover; -// mcs_hton->discover_table_existence = mcs_discover_existence; - mcs_hton->commit = mcs_commit; - mcs_hton->rollback = mcs_rollback; - mcs_hton->close_connection = mcs_close_connection; - mcs_hton->create_group_by = create_columnstore_group_by_handler; - mcs_hton->create_derived = create_columnstore_derived_handler; - mcs_hton->create_select = create_columnstore_select_handler; - mcs_hton->db_type = DB_TYPE_AUTOASSIGN; + mcs_hton->create = ha_mcs_cache_create_handler; + mcs_hton->panic = 0; + mcs_hton->flags = HTON_CAN_RECREATE | HTON_NO_PARTITION; + // mcs_hton->discover_table = mcs_discover; + // mcs_hton->discover_table_existence = mcs_discover_existence; + mcs_hton->commit = mcs_commit; + mcs_hton->rollback = mcs_rollback; + mcs_hton->close_connection = mcs_close_connection; + mcs_hton->create_group_by = create_columnstore_group_by_handler; + mcs_hton->create_derived = create_columnstore_derived_handler; + mcs_hton->create_select = create_columnstore_select_handler; + mcs_hton->db_type = DB_TYPE_AUTOASSIGN; #ifdef HAVE_PSI_INTERFACE - uint count = sizeof(all_mutexes)/sizeof(all_mutexes[0]); - mysql_mutex_register("ha_mcs_cache", all_mutexes, count); + uint count = sizeof(all_mutexes) / sizeof(all_mutexes[0]); + mysql_mutex_register("ha_mcs_cache", all_mutexes, count); #else - (void)key_LOCK_cache_share; + (void)key_LOCK_cache_share; #endif - mysql_mutex_init(key_LOCK_cache_share, &LOCK_cache_share, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_LOCK_cache_share, &LOCK_cache_share, MY_MUTEX_INIT_FAST); - DBUG_RETURN(0); + DBUG_RETURN(0); } static int columnstore_done_func(void* p) { - DBUG_ENTER("columnstore_done_func"); + DBUG_ENTER("columnstore_done_func"); - config::Config::deleteInstanceMap(); - my_hash_free(&mcs_open_tables); + config::Config::deleteInstanceMap(); + my_hash_free(&mcs_open_tables); #ifndef _MSC_VER - pthread_mutex_destroy(&mcs_mutex); + pthread_mutex_destroy(&mcs_mutex); #endif - if (plugin_maria) - { - plugin_unlock(0, plugin_maria); - plugin_maria = NULL; - } + if (plugin_maria) + { + plugin_unlock(0, plugin_maria); + plugin_maria = NULL; + } - mysql_mutex_destroy(&LOCK_cache_share); + mysql_mutex_destroy(&LOCK_cache_share); - DBUG_RETURN(0); + DBUG_RETURN(0); } -struct st_mysql_storage_engine ha_mcs_cache_storage_engine= -{ MYSQL_HANDLERTON_INTERFACE_VERSION }; +struct st_mysql_storage_engine ha_mcs_cache_storage_engine = {MYSQL_HANDLERTON_INTERFACE_VERSION}; -struct st_mysql_storage_engine columnstore_storage_engine = -{ MYSQL_HANDLERTON_INTERFACE_VERSION }; +struct st_mysql_storage_engine columnstore_storage_engine = {MYSQL_HANDLERTON_INTERFACE_VERSION}; -static struct st_mysql_information_schema is_columnstore_plugin_version = -{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; +static struct st_mysql_information_schema is_columnstore_plugin_version = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION}; - -maria_declare_plugin(columnstore) -{ - MYSQL_STORAGE_ENGINE_PLUGIN, - &columnstore_storage_engine, - "Columnstore", - "MariaDB Corporation", - "ColumnStore storage engine", - PLUGIN_LICENSE_GPL, - columnstore_init_func, - columnstore_done_func, - MCSVERSIONHEX, - mcs_status_variables, /* status variables */ - mcs_system_variables, /* system variables */ - PLUGIN_COLUMNSTORE_VERSION, /* string version */ - COLUMNSTORE_MATURITY /* maturity */ +maria_declare_plugin(columnstore){ + MYSQL_STORAGE_ENGINE_PLUGIN, &columnstore_storage_engine, "Columnstore", + "MariaDB Corporation", "ColumnStore storage engine", PLUGIN_LICENSE_GPL, + columnstore_init_func, columnstore_done_func, MCSVERSIONHEX, + mcs_status_variables, /* status variables */ + mcs_system_variables, /* system variables */ + PLUGIN_COLUMNSTORE_VERSION, /* string version */ + COLUMNSTORE_MATURITY /* maturity */ }, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_plugin_version, - "COLUMNSTORE_COLUMNS", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore columns", - PLUGIN_LICENSE_GPL, - is_columnstore_columns_plugin_init, - //is_columnstore_tables_plugin_deinit, - NULL, - MCSVERSIONHEX, - NULL, - NULL, - PLUGIN_COLUMNSTORE_VERSION, - COLUMNSTORE_MATURITY -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_plugin_version, - "COLUMNSTORE_TABLES", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore tables", - PLUGIN_LICENSE_GPL, - is_columnstore_tables_plugin_init, - //is_columnstore_tables_plugin_deinit, - NULL, - MCSVERSIONHEX, - NULL, - NULL, - PLUGIN_COLUMNSTORE_VERSION, - COLUMNSTORE_MATURITY -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_plugin_version, - "COLUMNSTORE_FILES", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore files", - PLUGIN_LICENSE_GPL, - is_columnstore_files_plugin_init, - //is_columnstore_files_plugin_deinit, - NULL, - MCSVERSIONHEX, - NULL, - NULL, - PLUGIN_COLUMNSTORE_VERSION, - COLUMNSTORE_MATURITY -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_plugin_version, - "COLUMNSTORE_EXTENTS", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore extents", - PLUGIN_LICENSE_GPL, - is_columnstore_extents_plugin_init, - //is_columnstore_extents_plugin_deinit, - NULL, - MCSVERSIONHEX, - NULL, - NULL, - PLUGIN_COLUMNSTORE_VERSION, - COLUMNSTORE_MATURITY -} -maria_declare_plugin_end; + {MYSQL_INFORMATION_SCHEMA_PLUGIN, &is_columnstore_plugin_version, "COLUMNSTORE_COLUMNS", + "MariaDB Corporation", "An information schema plugin to list ColumnStore columns", PLUGIN_LICENSE_GPL, + is_columnstore_columns_plugin_init, + // is_columnstore_tables_plugin_deinit, + NULL, MCSVERSIONHEX, NULL, NULL, PLUGIN_COLUMNSTORE_VERSION, COLUMNSTORE_MATURITY}, + {MYSQL_INFORMATION_SCHEMA_PLUGIN, &is_columnstore_plugin_version, "COLUMNSTORE_TABLES", + "MariaDB Corporation", "An information schema plugin to list ColumnStore tables", PLUGIN_LICENSE_GPL, + is_columnstore_tables_plugin_init, + // is_columnstore_tables_plugin_deinit, + NULL, MCSVERSIONHEX, NULL, NULL, PLUGIN_COLUMNSTORE_VERSION, COLUMNSTORE_MATURITY}, + {MYSQL_INFORMATION_SCHEMA_PLUGIN, &is_columnstore_plugin_version, "COLUMNSTORE_FILES", + "MariaDB Corporation", "An information schema plugin to list ColumnStore files", PLUGIN_LICENSE_GPL, + is_columnstore_files_plugin_init, + // is_columnstore_files_plugin_deinit, + NULL, MCSVERSIONHEX, NULL, NULL, PLUGIN_COLUMNSTORE_VERSION, COLUMNSTORE_MATURITY}, + {MYSQL_INFORMATION_SCHEMA_PLUGIN, &is_columnstore_plugin_version, "COLUMNSTORE_EXTENTS", + "MariaDB Corporation", "An information schema plugin to list ColumnStore extents", PLUGIN_LICENSE_GPL, + is_columnstore_extents_plugin_init, + // is_columnstore_extents_plugin_deinit, + NULL, MCSVERSIONHEX, NULL, NULL, PLUGIN_COLUMNSTORE_VERSION, + COLUMNSTORE_MATURITY} maria_declare_plugin_end; /****************************************************************************** Implementation of write cache @@ -2056,7 +1914,6 @@ ha_rows ha_mcs_cache::num_rows_cached() return cache_handler->file->state->records; } - /* Free write locks if this was not an insert */ void ha_mcs_cache::free_locks() @@ -2071,10 +1928,9 @@ void ha_mcs_cache::free_locks() /* We don't need to lock cache_handler anymore as it's already flushed */ cache_handler->external_lock(table->in_use, F_UNLCK); thr_unlock(&cache_handler->file->lock, 0); - cache_locked= false; + cache_locked = false; } - /** Copy data from cache to ColumnStore @@ -2085,35 +1941,35 @@ void ha_mcs_cache::free_locks() int ha_mcs_cache::flush_insert_cache() { int error, error2; - ha_maria *from= cache_handler; - uchar *record= table->record[0]; - ulonglong copied_rows= 0; + ha_maria* from = cache_handler; + uchar* record = table->record[0]; + ulonglong copied_rows = 0; DBUG_ENTER("flush_insert_cache"); DBUG_ASSERT(from->file->state->records == share->cached_rows); parent::start_bulk_insert_from_cache(from->file->state->records, 0); from->rnd_init(1); - while (!(error= from->rnd_next(record))) + while (!(error = from->rnd_next(record))) { copied_rows++; - if ((error= parent::write_row(record))) + if ((error = parent::write_row(record))) goto end; rows_changed++; } if (error == HA_ERR_END_OF_FILE) - error= 0; + error = 0; DBUG_ASSERT(copied_rows == share->cached_rows); end: from->rnd_end(); - if ((error2= parent::end_bulk_insert()) && !error) - error= error2; + if ((error2 = parent::end_bulk_insert()) && !error) + error = error2; if (!error) { if (parent::ht->commit) - error= parent::ht->commit(parent::ht, table->in_use, 1); + error = parent::ht->commit(parent::ht, table->in_use, 1); } else { @@ -2137,7 +1993,7 @@ end: */ mysql_mutex_unlock(&from->file->s->lock.mutex); from->delete_all_rows(); - share->cached_rows= 0; + share->cached_rows = 0; mysql_mutex_lock(&from->file->s->lock.mutex); } } diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index 609acb056..37cc669d1 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -32,260 +32,256 @@ extern handlerton* mcs_hton; */ typedef struct st_mcs_share { - char* table_name; - uint32_t table_name_length, use_count; - pthread_mutex_t mutex; - THR_LOCK lock; + char* table_name; + uint32_t table_name_length, use_count; + pthread_mutex_t mutex; + THR_LOCK lock; } COLUMNSTORE_SHARE; /** @brief Class definition for the storage engine */ -class ha_mcs: public handler +class ha_mcs : public handler { - THR_LOCK_DATA lock; ///< MySQL lock - COLUMNSTORE_SHARE* share; ///< Shared lock info - ulonglong int_table_flags; - // We are using a vector here to mimick the stack functionality - // using push_back() and pop_back() - // as apparently there is a linker error on the std::stack::pop() - // call on Ubuntu18. - std::vector condStack; - int m_lock_type; + THR_LOCK_DATA lock; ///< MySQL lock + COLUMNSTORE_SHARE* share; ///< Shared lock info + ulonglong int_table_flags; + // We are using a vector here to mimick the stack functionality + // using push_back() and pop_back() + // as apparently there is a linker error on the std::stack::pop() + // call on Ubuntu18. + std::vector condStack; + int m_lock_type; - int impl_external_lock(THD* thd, TABLE* table, int lock_type); - int impl_rnd_init(TABLE* table, const std::vector& condStack); + int impl_external_lock(THD* thd, TABLE* table, int lock_type); + int impl_rnd_init(TABLE* table, const std::vector& condStack); -public: - ha_mcs(handlerton* hton, TABLE_SHARE* table_arg); - virtual ~ha_mcs() - { - } + public: + ha_mcs(handlerton* hton, TABLE_SHARE* table_arg); + virtual ~ha_mcs() + { + } - /** @brief - The name that will be used for display purposes. - */ - const char* table_type() const override - { - return "ColumnStore"; - } + /** @brief + The name that will be used for display purposes. + */ + const char* table_type() const override + { + return "ColumnStore"; + } - /** @brief - The name of the index type that will be used for display. - Don't implement this method unless you really have indexes. - */ - //const char *index_type(uint32_t inx) { return "HASH"; } + /** @brief + The name of the index type that will be used for display. + Don't implement this method unless you really have indexes. + */ + // const char *index_type(uint32_t inx) { return "HASH"; } - /** @brief - The file extensions. - */ - const char** bas_ext() const; + /** @brief + The file extensions. + */ + const char** bas_ext() const; - /** @brief - This is a list of flags that indicate what functionality the storage engine - implements. The current table flags are documented in handler.h - */ - ulonglong table_flags() const override - { - return int_table_flags; - } + /** @brief + This is a list of flags that indicate what functionality the storage engine + implements. The current table flags are documented in handler.h + */ + ulonglong table_flags() const override + { + return int_table_flags; + } - /** @brief - This is a bitmap of flags that indicates how the storage engine - implements indexes. The current index flags are documented in - handler.h. If you do not implement indexes, just return zero here. + /** @brief + This is a bitmap of flags that indicates how the storage engine + implements indexes. The current index flags are documented in + handler.h. If you do not implement indexes, just return zero here. - @details - part is the key part to check. First key part is 0. - If all_parts is set, MySQL wants to know the flags for the combined - index, up to and including 'part'. - */ - ulong index_flags(uint32_t inx, uint32_t part, bool all_parts) const override - { - return 0; - } + @details + part is the key part to check. First key part is 0. + If all_parts is set, MySQL wants to know the flags for the combined + index, up to and including 'part'. + */ + ulong index_flags(uint32_t inx, uint32_t part, bool all_parts) const override + { + return 0; + } + /** @brief + unireg.cc will call max_supported_record_length(), max_supported_keys(), + max_supported_key_parts(), uint32_t max_supported_key_length() + to make sure that the storage engine can handle the data it is about to + send. Return *real* limits of your storage engine here; MySQL will do + min(your_limits, MySQL_limits) automatically. + */ + uint32_t max_supported_record_length() const override + { + return HA_MAX_REC_LENGTH; + } - /** @brief - unireg.cc will call max_supported_record_length(), max_supported_keys(), - max_supported_key_parts(), uint32_t max_supported_key_length() - to make sure that the storage engine can handle the data it is about to - send. Return *real* limits of your storage engine here; MySQL will do - min(your_limits, MySQL_limits) automatically. - */ - uint32_t max_supported_record_length() const override - { - return HA_MAX_REC_LENGTH; - } + /** @brief + Called in test_quick_select to determine if indexes should be used. + */ + virtual double scan_time() override + { + return (double)(stats.records + stats.deleted) / 20.0 + 10; + } - /** @brief - Called in test_quick_select to determine if indexes should be used. - */ - virtual double scan_time() override - { - return (double) (stats.records + stats.deleted) / 20.0 + 10; - } + /** @brief + Analyze table command. + */ + int analyze(THD* thd, HA_CHECK_OPT* check_opt) override; - /** @brief - Analyze table command. - */ - int analyze(THD* thd, HA_CHECK_OPT* check_opt) override; + /* + Everything below are methods that we implement in ha_example.cc. - /* - Everything below are methods that we implement in ha_example.cc. + Most of these methods are not obligatory, skip them and + MySQL will treat them as not implemented + */ - Most of these methods are not obligatory, skip them and - MySQL will treat them as not implemented - */ + /** @brief + We implement this in ha_example.cc; it's a required method. + */ + int open(const char* name, int mode, uint32_t test_if_locked) override; // required - /** @brief - We implement this in ha_example.cc; it's a required method. - */ - int open(const char* name, int mode, uint32_t test_if_locked) override; // required + // MCOL-4282 This function is called by open_tables in sql_base.cc. + // We mutate the optimizer flags here for prepared statements as this + // handler function is called before JOIN::prepare, and we need to + // disable the default optimizer flags before JOIN::prepare (which is + // called during "PREPARE stmt FROM ..." SQL) is called. + // Sequence of SQL statements that will lead to this execution path + // for prepared statements: + // CREATE TABLE t1 (a int, b int) engine=columnstore; + // INSERT INTO t1 VALUES (1, 2), (2, 4), (3, 1); + // PREPARE stmt1 FROM "SELECT * FROM t1"; + // EXECUTE stmt1; - // MCOL-4282 This function is called by open_tables in sql_base.cc. - // We mutate the optimizer flags here for prepared statements as this - // handler function is called before JOIN::prepare, and we need to - // disable the default optimizer flags before JOIN::prepare (which is - // called during "PREPARE stmt FROM ..." SQL) is called. - // Sequence of SQL statements that will lead to this execution path - // for prepared statements: - // CREATE TABLE t1 (a int, b int) engine=columnstore; - // INSERT INTO t1 VALUES (1, 2), (2, 4), (3, 1); - // PREPARE stmt1 FROM "SELECT * FROM t1"; - // EXECUTE stmt1; + int discover_check_version() override; - int discover_check_version() override; + /** @brief + We implement this in ha_example.cc; it's a required method. + */ + int close(void) override; // required - /** @brief - We implement this in ha_example.cc; it's a required method. - */ - int close(void) override; // required + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int write_row(const uchar* buf) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int write_row(const uchar* buf) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + void start_bulk_insert(ha_rows rows, uint flags = 0) override; + void start_bulk_insert_from_cache(ha_rows rows, uint flags = 0); - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - void start_bulk_insert(ha_rows rows, uint flags = 0) override; - void start_bulk_insert_from_cache(ha_rows rows, uint flags = 0); + /**@bug 2461 - Overloaded end_bulk_insert. MariaDB uses the abort bool, mysql does not. */ + int end_bulk_insert() override; - /**@bug 2461 - Overloaded end_bulk_insert. MariaDB uses the abort bool, mysql does not. */ - int end_bulk_insert() override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int update_row(const uchar* old_data, const uchar* new_data) override; + int direct_update_rows_init(List* update_fields) override; + int direct_update_rows(ha_rows* update_rows); + int direct_update_rows(ha_rows* update_rows, ha_rows* found_rows) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int update_row(const uchar* old_data, const uchar* new_data) override; - int direct_update_rows_init(List *update_fields) override; - int direct_update_rows(ha_rows *update_rows); - int direct_update_rows(ha_rows *update_rows, ha_rows *found_rows) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int delete_row(const uchar* buf) override; + int direct_delete_rows_init() override; + int direct_delete_rows(ha_rows* deleted_rows) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int delete_row(const uchar* buf) override; - int direct_delete_rows_init() override; - int direct_delete_rows(ha_rows *deleted_rows) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int index_read_map(uchar* buf, const uchar* key, key_part_map keypart_map, + enum ha_rkey_function find_flag) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int index_read_map(uchar* buf, const uchar* key, - key_part_map keypart_map, enum ha_rkey_function find_flag) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int index_next(uchar* buf) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int index_next(uchar* buf) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int index_prev(uchar* buf) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int index_prev(uchar* buf) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int index_first(uchar* buf) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int index_first(uchar* buf) override; + /** @brief + We implement this in ha_example.cc. It's not an obligatory method; + skip it and and MySQL will treat it as not implemented. + */ + int index_last(uchar* buf) override; - /** @brief - We implement this in ha_example.cc. It's not an obligatory method; - skip it and and MySQL will treat it as not implemented. - */ - int index_last(uchar* buf) override; + /** @brief + Unlike index_init(), rnd_init() can be called two consecutive times + without rnd_end() in between (it only makes sense if scan=1). In this + case, the second call should prepare for the new table scan (e.g if + rnd_init() allocates the cursor, the second call should position the + cursor to the start of the table; no need to deallocate and allocate + it again. This is a required method. + */ + int rnd_init(bool scan) override; // required + int rnd_end() override; + int rnd_next(uchar* buf) override; ///< required + int rnd_pos(uchar* buf, uchar* pos) override; ///< required + int reset() override; + void position(const uchar* record) override; ///< required + int info(uint32_t) override; ///< required + int extra(enum ha_extra_function operation) override; + int external_lock(THD* thd, int lock_type) override; ///< required + int delete_all_rows(void) override; + ha_rows records_in_range(uint32_t inx, const key_range* min_key, const key_range* max_key, + page_range* res) override; + int delete_table(const char* from) override; + int rename_table(const char* from, const char* to) override; + int create(const char* name, TABLE* form, + HA_CREATE_INFO* create_info) override; ///< required - /** @brief - Unlike index_init(), rnd_init() can be called two consecutive times - without rnd_end() in between (it only makes sense if scan=1). In this - case, the second call should prepare for the new table scan (e.g if - rnd_init() allocates the cursor, the second call should position the - cursor to the start of the table; no need to deallocate and allocate - it again. This is a required method. - */ - int rnd_init(bool scan) override; //required - int rnd_end() override; - int rnd_next(uchar* buf) override; ///< required - int rnd_pos(uchar* buf, uchar* pos) override; ///< required - int reset() override; - void position(const uchar* record) override; ///< required - int info(uint32_t) override; ///< required - int extra(enum ha_extra_function operation) override; - int external_lock(THD* thd, int lock_type) override; ///< required - int delete_all_rows(void) override; - ha_rows records_in_range(uint32_t inx, const key_range* min_key, - const key_range* max_key, page_range* res) override; - int delete_table(const char* from) override; - int rename_table(const char* from, const char* to) override; - int create(const char* name, TABLE* form, - HA_CREATE_INFO* create_info) override; ///< required + THR_LOCK_DATA** store_lock(THD* thd, THR_LOCK_DATA** to, + enum thr_lock_type lock_type) override; ///< required + const COND* cond_push(const COND* cond) override; + void cond_pop() override; + uint8 table_cache_type() override + { + return HA_CACHE_TBL_NOCACHE; + } - THR_LOCK_DATA** store_lock(THD* thd, THR_LOCK_DATA** to, - enum thr_lock_type lock_type) override; ///< required - const COND* cond_push(const COND* cond) override; - void cond_pop() override; - uint8 table_cache_type() override - { - return HA_CACHE_TBL_NOCACHE; - } + int repair(THD* thd, HA_CHECK_OPT* check_opt) override; + bool is_crashed() const override; - int repair(THD* thd, HA_CHECK_OPT* check_opt) override; - bool is_crashed() const override; - - bool isReadOnly() const - { - return m_lock_type == F_RDLCK; - } + bool isReadOnly() const + { + return m_lock_type == F_RDLCK; + } }; - class ha_mcs_cache_share { - ha_mcs_cache_share *next; /* Next open share */ - const char *name; + ha_mcs_cache_share* next; /* Next open share */ + const char* name; uint open_count; -public: + + public: ulonglong cached_rows; THR_LOCK org_lock; - friend ha_mcs_cache_share *find_cache_share(const char *name, - ulonglong cached_rows); + friend ha_mcs_cache_share* find_cache_share(const char* name, ulonglong cached_rows); void close(); }; - - -class ha_mcs_cache :public ha_mcs +class ha_mcs_cache : public ha_mcs { typedef ha_mcs parent; int original_lock_type; @@ -302,16 +298,15 @@ class ha_mcs_cache :public ha_mcs bool isCacheEnabled() const { - return (get_cache_inserts(current_thd) && !isSysCatTable && - !isCacheDisabled); + return (get_cache_inserts(current_thd) && !isSysCatTable && !isCacheDisabled); } -public: + public: uint lock_counter; - ha_maria *cache_handler; - ha_mcs_cache_share *share; + ha_maria* cache_handler; + ha_mcs_cache_share* share; - ha_mcs_cache(handlerton *hton, TABLE_SHARE *table_arg, MEM_ROOT *mem_root); + ha_mcs_cache(handlerton* hton, TABLE_SHARE* table_arg, MEM_ROOT* mem_root); ~ha_mcs_cache(); /* @@ -319,27 +314,24 @@ public: cache handler */ - int create(const char *name, TABLE *table_arg, - HA_CREATE_INFO *ha_create_info); - int open(const char *name, int mode, uint open_flags); - int delete_table(const char *name); - int rename_table(const char *from, const char *to); + int create(const char* name, TABLE* table_arg, HA_CREATE_INFO* ha_create_info); + int open(const char* name, int mode, uint open_flags); + int delete_table(const char* name); + int rename_table(const char* from, const char* to); int delete_all_rows(void); int close(void); uint lock_count(void) const; - THR_LOCK_DATA **store_lock(THD *thd, - THR_LOCK_DATA **to, - enum thr_lock_type lock_type); - int external_lock(THD *thd, int lock_type); - int repair(THD *thd, HA_CHECK_OPT *check_opt); + THR_LOCK_DATA** store_lock(THD* thd, THR_LOCK_DATA** to, enum thr_lock_type lock_type); + int external_lock(THD* thd, int lock_type); + int repair(THD* thd, HA_CHECK_OPT* check_opt); bool is_crashed() const; /* Write row uses cache_handler, for normal inserts, otherwise derived handler */ - int write_row(const uchar *buf); + int write_row(const uchar* buf); void start_bulk_insert(ha_rows rows, uint flags); int end_bulk_insert(); @@ -347,9 +339,8 @@ public: void free_locks(); ha_rows num_rows_cached(); int flush_insert_cache(); - friend my_bool get_status_and_flush_cache(void *param, - my_bool concurrent_insert); - friend my_bool cache_start_trans(void *param); + friend my_bool get_status_and_flush_cache(void* param, my_bool concurrent_insert); + friend my_bool cache_start_trans(void* param); }; -#endif //HA_MCS_H__ +#endif // HA_MCS_H__ diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 5670d1313..2ea1a68a2 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -47,715 +47,705 @@ extern "C" { #define MAXSTRINGLENGTH 50 - const char* PmSmallSideMaxMemory = "pmmaxmemorysmallside"; + const char* PmSmallSideMaxMemory = "pmmaxmemorysmallside"; - const char* SetParmsPrelude = "Updated "; - const char* SetParmsError = "Invalid parameter: "; - const char* InvalidParmSize = "Invalid parameter size: Input value cannot be larger than "; + const char* SetParmsPrelude = "Updated "; + const char* SetParmsError = "Invalid parameter: "; + const char* InvalidParmSize = "Invalid parameter size: Input value cannot be larger than "; - const size_t Plen = strlen(SetParmsPrelude); - const size_t Elen = strlen(SetParmsError); + const size_t Plen = strlen(SetParmsPrelude); + const size_t Elen = strlen(SetParmsError); - const char* invalidParmSizeMessage(uint64_t size, size_t& len) - { - static char str[sizeof(InvalidParmSize) + 12] = {0}; - std::ostringstream os; - os << InvalidParmSize << size; - len = os.str().length(); - strcpy(str, os.str().c_str()); - return str; - } + const char* invalidParmSizeMessage(uint64_t size, size_t& len) + { + static char str[sizeof(InvalidParmSize) + 12] = {0}; + std::ostringstream os; + os << InvalidParmSize << size; + len = os.str().length(); + strcpy(str, os.str().c_str()); + return str; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - char parameter[MAXSTRINGLENGTH]; - char valuestr[MAXSTRINGLENGTH]; - size_t plen = args->lengths[0]; - size_t vlen = args->lengths[1]; - - memcpy(parameter, args->args[0], plen); - memcpy(valuestr, args->args[1], vlen); - - parameter[plen] = '\0'; - valuestr[vlen] = '\0'; - - uint64_t value = Config::uFromText(valuestr); - - THD* thd = current_thd; - uint32_t sessionID = execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id); - - const char* msg = SetParmsError; - size_t mlen = Elen; - bool includeInput = true; - - std::string pstr(parameter); - boost::algorithm::to_lower(pstr); - - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - idbassert(ci != 0); - - if (pstr == PmSmallSideMaxMemory) - { - joblist::ResourceManager* rm = joblist::ResourceManager::instance(); - - if (rm->getHjTotalUmMaxMemorySmallSide() >= value) - { - ci->rmParms.push_back(execplan::RMParam(sessionID, execplan::PMSMALLSIDEMEMORY, value)); - - msg = SetParmsPrelude; - mlen = Plen; - } - else - { - msg = invalidParmSizeMessage(rm->getHjTotalUmMaxMemorySmallSide(), mlen); - includeInput = false; - } - } - - memcpy(result, msg, mlen); - - if (includeInput) - { - memcpy(result + mlen, parameter, plen); - mlen += plen; - memcpy(result + mlen++, " ", 1); - memcpy(result + mlen, valuestr, vlen); - *length = mlen + vlen; - } - else - *length = mlen; - - return result; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calsetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) - { - strcpy(message, "CALSETPARMS() requires two string arguments"); - return 1; - } - - initid->max_length = MAXSTRINGLENGTH; - - char valuestr[MAXSTRINGLENGTH]; - size_t vlen = args->lengths[1]; - - memcpy(valuestr, args->args[1], vlen--); - - for (size_t i = 0; i < vlen; ++i) - if (!isdigit(valuestr[i])) - { - strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - - if (!isdigit(valuestr[vlen])) - { - switch (valuestr[vlen]) - { - case 'G': - case 'g': - case 'M': - case 'm': - case 'K': - case 'k': - case '\0': - break; - - default: - strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - } - - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calsetparms_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - unsigned long l = ci->queryStats.size(); - - if (l == 0) - { - *is_null = 1; - return 0; - } - - if (l > 255) l = 255; - - memcpy(result, ci->queryStats.c_str(), l); - *length = l; - return result; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 0) - { - strcpy(message, "CALGETSTATS() takes no arguments"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calgetstats_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) - { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - long long oldTrace = ci->traceFlags; - ci->traceFlags = (uint32_t)(*((long long*)args->args[0])); - // keep the vtablemode bit - ci->traceFlags |= (oldTrace & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); - ci->traceFlags |= (oldTrace & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH); - return oldTrace; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calsettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) - { - strcpy(message, "CALSETTRACE() requires one INTEGER argument"); - return 1; - } - - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calsettrace_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif -// Return 1 if system is ready for reads or 0 if not. - long long mcssystemready(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) - { - long long rtn = 0; - Oam oam; - DBRM dbrm(true); - - try - { - if (dbrm.getSystemReady() - && dbrm.getSystemQueryReady()) - { - return 1; - } - } - catch (...) - { - *error = 1; - } - - return rtn; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcssystemready_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void mcssystemready_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif -// Return non-zero if system is read only; 0 if writeable - long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) - { - long long rtn = 0; - DBRM dbrm(true); - - try - { - if (dbrm.getSystemSuspended()) - { - rtn = 1; - } - - if (dbrm.isReadWrite() > 0) // Returns 0 for writable, 5 for read only - { - rtn = 2; - } - } - catch (...) - { - *error = 1; - rtn = 1; - } - - return rtn; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcssystemreadonly_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void mcssystemreadonly_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif -// Return non-zero if this is the primary UM; 0 if not primary - long long mcssystemprimary(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) - { - long long rtn = 0; - Oam oam; - std::string PrimaryUMModuleName; - std::string localModule; - oamModuleInfo_t st; - - try - { - st = oam.getModuleInfo(); - localModule = boost::get<0>(st); - PrimaryUMModuleName = config::Config::makeConfig()->getConfig("SystemConfig", "PrimaryUMModuleName"); - - if (boost::iequals(localModule, PrimaryUMModuleName)) - rtn = 1; - if (PrimaryUMModuleName == "unassigned") - rtn = 1; - } - catch (std::runtime_error& e) - { - // It's difficult to return an error message from a numerical UDF - //string msg = string("ERROR: Problem getting Primary UM Module Name. ") + e.what(); - *error = 1; - } - catch (...) - { - *error = 1; - } - return rtn; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcssystemprimary_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void mcssystemprimary_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) - { - strcpy(message, "CALVIEWTABLELOCK() requires two string arguments"); - return 1; - } - else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT ) ) - { - strcpy(message, "CALVIEWTABLELOCK() requires one string argument"); - return 1; - } - else if (args->arg_count > 2 ) - { - strcpy(message, "CALVIEWTABLELOCK() takes one or two arguments only"); - return 1; - } - else if (args->arg_count == 0 ) - { - strcpy(message, "CALVIEWTABLELOCK() requires at least one argument"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - THD* thd = current_thd; - - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - execplan::CalpontSystemCatalog::TableName tableName; - - if ( args->arg_count == 2 ) - { - tableName.schema = args->args[0]; - tableName.table = args->args[1]; - } - else if ( args->arg_count == 1 ) - { - tableName.table = args->args[0]; - - if (thd->db.length) - { - tableName.schema = thd->db.str; - } - else - { - std::string msg("No schema information provided"); - memcpy(result, msg.c_str(), msg.length()); - *length = msg.length(); - return result; - } - } - if (lower_case_table_names) - { - boost::algorithm::to_lower(tableName.schema); - boost::algorithm::to_lower(tableName.table); - } - - if ( !ci->dmlProc ) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "viewtablelock starts a new client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } - - std::string lockinfo = ha_mcs_impl_viewtablelock(*ci, tableName); - - memcpy(result, lockinfo.c_str(), lockinfo.length()); - *length = lockinfo.length(); - return result; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calviewtablelock_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) - { - strcpy(message, - "CALCLEARTABLELOCK() requires one integer argument (the lockID)"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast( - get_fe_conn_info_ptr()); - long long lockID = *reinterpret_cast(args->args[0]); - - if ( !ci->dmlProc ) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "cleartablelock starts a new client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } - - unsigned long long uLockID = lockID; - std::string lockinfo = ha_mcs_impl_cleartablelock(*ci, uLockID); - - memcpy(result, lockinfo.c_str(), lockinfo.length()); - *length = lockinfo.length(); - return result; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calcleartablelock_deinit(UDF_INIT* initid) - { - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) - { - strcpy(message, "CALLASTINSRTID() requires two string arguments"); - return 1; - } - else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT ) ) - { - strcpy(message, "CALLASTINSERTID() requires one string argument"); - return 1; - } - else if (args->arg_count > 2 ) - { - strcpy(message, "CALLASTINSERTID() takes one or two arguments only"); - return 1; - } - else if (args->arg_count == 0 ) - { - strcpy(message, "CALLASTINSERTID() requires at least one argument"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, + const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) + { + char parameter[MAXSTRINGLENGTH]; + char valuestr[MAXSTRINGLENGTH]; + size_t plen = args->lengths[0]; + size_t vlen = args->lengths[1]; + + memcpy(parameter, args->args[0], plen); + memcpy(valuestr, args->args[1], vlen); + + parameter[plen] = '\0'; + valuestr[vlen] = '\0'; + + uint64_t value = Config::uFromText(valuestr); + + THD* thd = current_thd; + uint32_t sessionID = execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id); + + const char* msg = SetParmsError; + size_t mlen = Elen; + bool includeInput = true; + + std::string pstr(parameter); + boost::algorithm::to_lower(pstr); + + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + idbassert(ci != 0); + + if (pstr == PmSmallSideMaxMemory) { - THD* thd = current_thd; + joblist::ResourceManager* rm = joblist::ResourceManager::instance(); - execplan::CalpontSystemCatalog::TableName tableName; - uint64_t nextVal = 0; + if (rm->getHjTotalUmMaxMemorySmallSide() >= value) + { + ci->rmParms.push_back(execplan::RMParam(sessionID, execplan::PMSMALLSIDEMEMORY, value)); - if ( args->arg_count == 2 ) - { - tableName.schema = args->args[0]; - tableName.table = args->args[1]; - } - else if ( args->arg_count == 1 ) - { - tableName.table = args->args[0]; - - if (thd->db.length) - tableName.schema = thd->db.str; - else - { - return -1; - } - } - if (lower_case_table_names) - { - boost::algorithm::to_lower(tableName.schema); - boost::algorithm::to_lower(tableName.table); - } - - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); - csc->identity(execplan::CalpontSystemCatalog::FE); - - try - { - nextVal = csc->nextAutoIncrValue(tableName); - } - catch (std::exception&) - { - std::string msg("No such table found during autincrement"); - setError(thd, ER_INTERNAL_ERROR, msg); - return nextVal; - } - - if (nextVal == AUTOINCR_SATURATED) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT)); - return nextVal; - } - - //@Bug 3559. Return a message for table without autoincrement column. - if (nextVal == 0) - { - std::string msg("Autoincrement does not exist for this table."); - setError(thd, ER_INTERNAL_ERROR, msg); - return nextVal; - } - - return (nextVal - 1); + msg = SetParmsPrelude; + mlen = Plen; + } + else + { + msg = invalidParmSizeMessage(rm->getHjTotalUmMaxMemorySmallSide(), mlen); + includeInput = false; + } } + memcpy(result, msg, mlen); + + if (includeInput) + { + memcpy(result + mlen, parameter, plen); + mlen += plen; + memcpy(result + mlen++, " ", 1); + memcpy(result + mlen, valuestr, vlen); + *length = mlen + vlen; + } + else + *length = mlen; + + return result; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void callastinsertid_deinit(UDF_INIT* initid) + my_bool calsetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) { + strcpy(message, "CALSETPARMS() requires two string arguments"); + return 1; } + initid->max_length = MAXSTRINGLENGTH; + + char valuestr[MAXSTRINGLENGTH]; + size_t vlen = args->lengths[1]; + + memcpy(valuestr, args->args[1], vlen--); + + for (size_t i = 0; i < vlen; ++i) + if (!isdigit(valuestr[i])) + { + strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); + return 1; + } + + if (!isdigit(valuestr[vlen])) + { + switch (valuestr[vlen]) + { + case 'G': + case 'g': + case 'M': + case 'm': + case 'K': + case 'k': + case '\0': break; + + default: + strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); + return 1; + } + } + + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void calflushcache_deinit(UDF_INIT* initid) - { - } + void calsetparms_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long calflushcache(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) + const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + unsigned long l = ci->queryStats.size(); + + if (l == 0) { - return static_cast(cacheutils::flushPrimProcCache()); + *is_null = 1; + return 0; } + if (l > 255) + l = 255; + + memcpy(result, ci->queryStats.c_str(), l); + *length = l; + return result; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool calflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) { - if (args->arg_count != 0) - { - strcpy(message, "CALFLUSHCACHE() takes no arguments"); - return 1; - } - - return 0; + strcpy(message, "CALGETSTATS() takes no arguments"); + return 1; } - static const unsigned long TraceSize = 16 * 1024; + initid->maybe_null = 1; + initid->max_length = 255; -//mysqld will call this with only 766 bytes available in result no matter what we asked for in calgettrace_init() + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calgetstats_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + long long oldTrace = ci->traceFlags; + ci->traceFlags = (uint32_t)(*((long long*)args->args[0])); + // keep the vtablemode bit + ci->traceFlags |= (oldTrace & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); + ci->traceFlags |= (oldTrace & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH); + return oldTrace; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calsettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) + { + strcpy(message, "CALSETTRACE() requires one INTEGER argument"); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calsettrace_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + // Return 1 if system is ready for reads or 0 if not. + long long mcssystemready(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + long long rtn = 0; + Oam oam; + DBRM dbrm(true); + + try + { + if (dbrm.getSystemReady() && dbrm.getSystemQueryReady()) + { + return 1; + } + } + catch (...) + { + *error = 1; + } + + return rtn; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssystemready_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssystemready_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + // Return non-zero if system is read only; 0 if writeable + long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + long long rtn = 0; + DBRM dbrm(true); + + try + { + if (dbrm.getSystemSuspended()) + { + rtn = 1; + } + + if (dbrm.isReadWrite() > 0) // Returns 0 for writable, 5 for read only + { + rtn = 2; + } + } + catch (...) + { + *error = 1; + rtn = 1; + } + + return rtn; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssystemreadonly_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssystemreadonly_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + // Return non-zero if this is the primary UM; 0 if not primary + long long mcssystemprimary(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + long long rtn = 0; + Oam oam; + std::string PrimaryUMModuleName; + std::string localModule; + oamModuleInfo_t st; + + try + { + st = oam.getModuleInfo(); + localModule = boost::get<0>(st); + PrimaryUMModuleName = config::Config::makeConfig()->getConfig("SystemConfig", "PrimaryUMModuleName"); + + if (boost::iequals(localModule, PrimaryUMModuleName)) + rtn = 1; + if (PrimaryUMModuleName == "unassigned") + rtn = 1; + } + catch (std::runtime_error& e) + { + // It's difficult to return an error message from a numerical UDF + // string msg = string("ERROR: Problem getting Primary UM Module Name. ") + e.what(); + *error = 1; + } + catch (...) + { + *error = 1; + } + return rtn; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssystemprimary_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssystemprimary_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) + { + strcpy(message, "CALVIEWTABLELOCK() requires two string arguments"); + return 1; + } + else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) + { + strcpy(message, "CALVIEWTABLELOCK() requires one string argument"); + return 1; + } + else if (args->arg_count > 2) + { + strcpy(message, "CALVIEWTABLELOCK() takes one or two arguments only"); + return 1; + } + else if (args->arg_count == 0) + { + strcpy(message, "CALVIEWTABLELOCK() requires at least one argument"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + THD* thd = current_thd; + + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + execplan::CalpontSystemCatalog::TableName tableName; + + if (args->arg_count == 2) + { + tableName.schema = args->args[0]; + tableName.table = args->args[1]; + } + else if (args->arg_count == 1) + { + tableName.table = args->args[0]; + + if (thd->db.length) + { + tableName.schema = thd->db.str; + } + else + { + std::string msg("No schema information provided"); + memcpy(result, msg.c_str(), msg.length()); + *length = msg.length(); + return result; + } + } + if (lower_case_table_names) + { + boost::algorithm::to_lower(tableName.schema); + boost::algorithm::to_lower(tableName.table); + } + + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + // cout << "viewtablelock starts a new client " << ci->dmlProc << " for session " << thd->thread_id << + // endl; + } + + std::string lockinfo = ha_mcs_impl_viewtablelock(*ci, tableName); + + memcpy(result, lockinfo.c_str(), lockinfo.length()); + *length = lockinfo.length(); + return result; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calviewtablelock_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) + { + strcpy(message, "CALCLEARTABLELOCK() requires one integer argument (the lockID)"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + long long lockID = *reinterpret_cast(args->args[0]); + + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + // cout << "cleartablelock starts a new client " << ci->dmlProc << " for session " << thd->thread_id << + // endl; + } + + unsigned long long uLockID = lockID; + std::string lockinfo = ha_mcs_impl_cleartablelock(*ci, uLockID); + + memcpy(result, lockinfo.c_str(), lockinfo.length()); + *length = lockinfo.length(); + return result; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calcleartablelock_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) + { + strcpy(message, "CALLASTINSRTID() requires two string arguments"); + return 1; + } + else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) + { + strcpy(message, "CALLASTINSERTID() requires one string argument"); + return 1; + } + else if (args->arg_count > 2) + { + strcpy(message, "CALLASTINSERTID() takes one or two arguments only"); + return 1; + } + else if (args->arg_count == 0) + { + strcpy(message, "CALLASTINSERTID() requires at least one argument"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + THD* thd = current_thd; + + execplan::CalpontSystemCatalog::TableName tableName; + uint64_t nextVal = 0; + + if (args->arg_count == 2) + { + tableName.schema = args->args[0]; + tableName.table = args->args[1]; + } + else if (args->arg_count == 1) + { + tableName.table = args->args[0]; + + if (thd->db.length) + tableName.schema = thd->db.str; + else + { + return -1; + } + } + if (lower_case_table_names) + { + boost::algorithm::to_lower(tableName.schema); + boost::algorithm::to_lower(tableName.table); + } + + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( + execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); + csc->identity(execplan::CalpontSystemCatalog::FE); + + try + { + nextVal = csc->nextAutoIncrValue(tableName); + } + catch (std::exception&) + { + std::string msg("No such table found during autincrement"); + setError(thd, ER_INTERNAL_ERROR, msg); + return nextVal; + } + + if (nextVal == AUTOINCR_SATURATED) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT)); + return nextVal; + } + + //@Bug 3559. Return a message for table without autoincrement column. + if (nextVal == 0) + { + std::string msg("Autoincrement does not exist for this table."); + setError(thd, ER_INTERNAL_ERROR, msg); + return nextVal; + } + + return (nextVal - 1); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void callastinsertid_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calflushcache_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long calflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return static_cast(cacheutils::flushPrimProcCache()); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, "CALFLUSHCACHE() takes no arguments"); + return 1; + } + + return 0; + } + + static const unsigned long TraceSize = 16 * 1024; + +// mysqld will call this with only 766 bytes available in result no matter what we asked for in +// calgettrace_init() // if we return a pointer that is not result, mysqld will take our pointer and use it, freeing up result #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) + const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + const std::string* msgp; + int flags = 0; + + if (args->arg_count > 0) { - const std::string* msgp; - int flags = 0; - - if (args->arg_count > 0) - { - if (args->arg_type[0] == INT_RESULT) - { - flags = *reinterpret_cast(args->args[0]); - } - } - - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - if (flags > 0) - //msgp = &connMap[sessionID].extendedStats; - msgp = &ci->extendedStats; - else - //msgp = &connMap[sessionID].miniStats; - msgp = &ci->miniStats; - - unsigned long l = msgp->size(); - - if (l == 0) - { - *is_null = 1; - return 0; - } - - if (l > TraceSize) l = TraceSize; - - *length = l; - return msgp->c_str(); + if (args->arg_type[0] == INT_RESULT) + { + flags = *reinterpret_cast(args->args[0]); + } } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + if (flags > 0) + // msgp = &connMap[sessionID].extendedStats; + msgp = &ci->extendedStats; + else + // msgp = &connMap[sessionID].miniStats; + msgp = &ci->miniStats; + + unsigned long l = msgp->size(); + + if (l == 0) { + *is_null = 1; + return 0; + } + + if (l > TraceSize) + l = TraceSize; + + *length = l; + return msgp->c_str(); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { #if 0 if (args->arg_count != 0) @@ -765,116 +755,113 @@ extern "C" } #endif - initid->maybe_null = 1; - initid->max_length = TraceSize; + initid->maybe_null = 1; + initid->max_length = TraceSize; - return 0; - } + return 0; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void calgettrace_deinit(UDF_INIT* initid) - { - } + void calgettrace_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - std::string version(columnstore_version); - *length = version.size(); - memcpy(result, version.c_str(), *length); - return result; - } + const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + std::string version(columnstore_version); + *length = version.size(); + memcpy(result, version.c_str(), *length); + return result; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool calgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETVERSION() takes no arguments"); - return 1; - } - - return 0; + strcpy(message, "CALGETVERSION() takes no arguments"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void calgetversion_deinit(UDF_INIT* initid) - { - } + void calgetversion_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) + const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + idbassert(ci != 0); + + MessageQueueClient* mqc = 0; + mqc = new MessageQueueClient("ExeMgr1"); + + ByteStream msg; + ByteStream::quadbyte runningSql, waitingSql; + ByteStream::quadbyte qb = 5; + msg << qb; + mqc->write(msg); + + // get ExeMgr response + msg.restart(); + msg = mqc->read(); + + if (msg.length() == 0) { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - idbassert(ci != 0); - - MessageQueueClient* mqc = 0; - mqc = new MessageQueueClient("ExeMgr1"); - - ByteStream msg; - ByteStream::quadbyte runningSql, waitingSql; - ByteStream::quadbyte qb = 5; - msg << qb; - mqc->write(msg); - - //get ExeMgr response - msg.restart(); - msg = mqc->read(); - - if (msg.length() == 0) - { - memcpy(result, "Lost connection to ExeMgr", *length); - return result; - } - - msg >> runningSql; - msg >> waitingSql; - delete mqc; - - char ans[128]; - sprintf(ans, "Running SQL statements %d, Waiting SQL statments %d", runningSql, waitingSql); - *length = strlen(ans); - memcpy(result, ans, *length); - return result; + memcpy(result, "Lost connection to ExeMgr", *length); + return result; } + msg >> runningSql; + msg >> waitingSql; + delete mqc; + + char ans[128]; + sprintf(ans, "Running SQL statements %d, Waiting SQL statments %d", runningSql, waitingSql); + *length = strlen(ans); + memcpy(result, ans, *length); + return result; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool calgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETSQLCOUNT() takes no arguments"); - return 1; - } - - return 0; + strcpy(message, "CALGETSQLCOUNT() takes no arguments"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void calgetsqlcount_deinit(UDF_INIT* initid) - { - } + void calgetsqlcount_deinit(UDF_INIT* initid) + { + } - -} //extern "C" +} // extern "C" diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 830befd4c..127f478d1 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -24,58 +24,69 @@ MariaDB header files. */ - -namespace datatypes { - -class StoreFieldMariaDB: public StoreField +namespace datatypes { - Field *m_field; - const CalpontSystemCatalog::ColType &m_type; -public: - StoreFieldMariaDB(Field *f, const CalpontSystemCatalog::ColType &type) - :m_field(f), m_type(type) - { } +class StoreFieldMariaDB : public StoreField +{ + Field* m_field; + const CalpontSystemCatalog::ColType& m_type; - const CalpontSystemCatalog::ColType &type() const { return m_type; } - int32_t colWidth() const override { return m_type.colWidth; } - int32_t precision() const override { return m_type.precision; } - int32_t scale() const override { return m_type.scale; } + public: + StoreFieldMariaDB(Field* f, const CalpontSystemCatalog::ColType& type) : m_field(f), m_type(type) + { + } + + const CalpontSystemCatalog::ColType& type() const + { + return m_type; + } + int32_t colWidth() const override + { + return m_type.colWidth; + } + int32_t precision() const override + { + return m_type.precision; + } + int32_t scale() const override + { + return m_type.scale; + } int store_date(int64_t val) override { char tmp[256]; - DataConvert::dateToString(val, tmp, sizeof(tmp)-1); + DataConvert::dateToString(val, tmp, sizeof(tmp) - 1); return store_string(tmp, strlen(tmp)); } int store_datetime(int64_t val) override { char tmp[256]; - DataConvert::datetimeToString(val, tmp, sizeof(tmp)-1, m_type.precision); + DataConvert::datetimeToString(val, tmp, sizeof(tmp) - 1, m_type.precision); return store_string(tmp, strlen(tmp)); } int store_time(int64_t val) override { char tmp[256]; - DataConvert::timeToString(val, tmp, sizeof(tmp)-1, m_type.precision); + DataConvert::timeToString(val, tmp, sizeof(tmp) - 1, m_type.precision); return store_string(tmp, strlen(tmp)); } int store_timestamp(int64_t val) override { char tmp[256]; - DataConvert::timestampToString(val, tmp, sizeof(tmp), - current_thd->variables.time_zone->get_name()->ptr(), + DataConvert::timestampToString(val, tmp, sizeof(tmp), current_thd->variables.time_zone->get_name()->ptr(), m_type.precision); return store_string(tmp, strlen(tmp)); } - int store_string(const char *str, size_t length) override + int store_string(const char* str, size_t length) override { return m_field->store(str, length, m_field->charset()); } - int store_varbinary(const char *str, size_t length) override + int store_varbinary(const char* str, size_t length) override { if (get_varbin_always_hex(current_thd)) { @@ -134,7 +145,7 @@ public: m_field->field_length = 310; if (dl == 0) { - float8store(m_field->ptr,dl); + float8store(m_field->ptr, dl); return 0; } return m_field->store(dl); @@ -174,44 +185,39 @@ public: return m_field->store(decAsAStr.c_str(), decAsAStr.length(), m_field->charset()); } - int store_lob(const char *str, size_t length) override + int store_lob(const char* str, size_t length) override { idbassert(dynamic_cast(m_field)); Field_blob* f2 = static_cast(m_field); - f2->set_ptr(length, (uchar*) str); + f2->set_ptr(length, (uchar*)str); return 0; } - }; - /*******************************************************************************/ -class WriteBatchFieldMariaDB: public WriteBatchField +class WriteBatchFieldMariaDB : public WriteBatchField { // Maximum number of decimal digits that can be represented in 4 bytes static const int DIG_PER_DEC = 9; // See strings/decimal.c - const int dig2bytes[DIG_PER_DEC+1]={0, 1, 1, 2, 2, 3, 3, 4, 4, 4}; - + const int dig2bytes[DIG_PER_DEC + 1] = {0, 1, 1, 2, 2, 3, 3, 4, 4, 4}; // Returns the number of bytes required to store a given number // of decimal digits int numDecimalBytes(int digits) { - return (((digits/DIG_PER_DEC) * 4) + dig2bytes[digits % DIG_PER_DEC]); + return (((digits / DIG_PER_DEC) * 4) + dig2bytes[digits % DIG_PER_DEC]); } - -public: - Field * m_field; - const CalpontSystemCatalog::ColType & m_type; + public: + Field* m_field; + const CalpontSystemCatalog::ColType& m_type; uint32_t m_mbmaxlen; - WriteBatchFieldMariaDB(Field *field, - const CalpontSystemCatalog::ColType & type, - uint32_t mbmaxlen) - :m_field(field), m_type(type), m_mbmaxlen(mbmaxlen) - { } - size_t ColWriteBatchDate(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + WriteBatchFieldMariaDB(Field* field, const CalpontSystemCatalog::ColType& type, uint32_t mbmaxlen) + : m_field(field), m_type(type), m_mbmaxlen(mbmaxlen) + { + } + size_t ColWriteBatchDate(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { // QQ: OLD DATE is not handled if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) @@ -229,7 +235,7 @@ public: } return 3; } - size_t ColWriteBatchDatetime(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchDatetime(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { @@ -250,37 +256,34 @@ public: if (!ltime.second_part) { - fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", ltime.year, ltime.month, ltime.day, ltime.hour, ltime.minute, ltime.second, ci.delimiter()); } else { - fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, - ltime.second_part, ci.delimiter()); + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ltime.second_part, ci.delimiter()); } return m_field->pack_length(); } // Old DATETIME - long long value = *((long long*) buf); - long datePart = (long) (value / 1000000ll); + long long value = *((long long*)buf); + long datePart = (long)(value / 1000000ll); int day = datePart % 100; int month = (datePart / 100) % 100; int year = datePart / 10000; fprintf(ci.filePtr(), "%04d-%02d-%02d ", year, month, day); - long timePart = (long) (value - (long long) datePart * 1000000ll); + long timePart = (long)(value - (long long)datePart * 1000000ll); int second = timePart % 100; int min = (timePart / 100) % 100; int hour = timePart / 10000; fprintf(ci.filePtr(), "%02d:%02d:%02d%c", hour, min, second, ci.delimiter()); return 8; } - size_t ColWriteBatchTime(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchTime(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { // QQ: why old TIME is not handled? if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) @@ -298,20 +301,18 @@ public: if (!ltime.second_part) { - fprintf(ci.filePtr(), "%02d:%02d:%02d%c", - ltime.hour, ltime.minute, ltime.second, ci.delimiter()); + fprintf(ci.filePtr(), "%02d:%02d:%02d%c", ltime.hour, ltime.minute, ltime.second, ci.delimiter()); } else { - fprintf(ci.filePtr(), "%02d:%02d:%02d.%ld%c", - ltime.hour, ltime.minute, ltime.second, - ltime.second_part, ci.delimiter()); + fprintf(ci.filePtr(), "%02d:%02d:%02d.%ld%c", ltime.hour, ltime.minute, ltime.second, ltime.second_part, + ci.delimiter()); } return m_field->pack_length(); } - size_t ColWriteBatchTimestamp(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchTimestamp(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { // QQ: old TIMESTAMP is not handled @@ -329,30 +330,25 @@ public: if (!tm.tv_usec) { - fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", - time.year, time.month, time.day, - time.hour, time.minute, time.second, ci.delimiter()); + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", time.year, time.month, time.day, time.hour, + time.minute, time.second, ci.delimiter()); } else { - fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", - time.year, time.month, time.day, - time.hour, time.minute, time.second, - tm.tv_usec, ci.delimiter()); + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", time.year, time.month, time.day, time.hour, + time.minute, time.second, tm.tv_usec, ci.delimiter()); } return m_field->pack_length(); } - static inline void ColWriteBatchTextStringPrintout(std::string& escape, - const ColBatchWriter &ci) + static inline void ColWriteBatchTextStringPrintout(std::string& escape, const ColBatchWriter& ci) { boost::replace_all(escape, "\\", "\\\\"); - fprintf(ci.filePtr(), "%c%.*s%c%c", ci.enclosed_by(), (int)escape.length(), - escape.c_str(), ci.enclosed_by(), ci.delimiter()); + fprintf(ci.filePtr(), "%c%.*s%c%c", ci.enclosed_by(), (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); } - static void ColWriteBatchTextString(const String &value, - const ColBatchWriter &ci, + static void ColWriteBatchTextString(const String& value, const ColBatchWriter& ci, const size_t colWidthInBytes) { std::string escape; @@ -360,8 +356,7 @@ public: ColWriteBatchTextStringPrintout(escape, ci); } - static void ColWriteBatchPaddedTextString(const String &value, - const ColBatchWriter &ci, + static void ColWriteBatchPaddedTextString(const String& value, const ColBatchWriter& ci, const size_t colWidthInBytes) { std::string escape; @@ -369,23 +364,19 @@ public: ColWriteBatchTextStringPrintout(escape, ci); } - static void ColWriteBatchBlobString(const String &value, - const ColBatchWriter &ci, + static void ColWriteBatchBlobString(const String& value, const ColBatchWriter& ci, const size_t colWidthInBytes) { const char* ptr = value.ptr(); for (uint32_t i = 0; i < value.length(); i++) { - fprintf(ci.filePtr(), "%02x", *(uint8_t*)(ptr+i)); + fprintf(ci.filePtr(), "%02x", *(uint8_t*)(ptr + i)); } fprintf(ci.filePtr(), "%c", ci.delimiter()); } - size_t ColWriteBatchString(const uchar *buf, - bool nullVal, - ColBatchWriter &ci, - void (*printFuncPtr)(const String&, - const ColBatchWriter &, + size_t ColWriteBatchString(const uchar* buf, bool nullVal, ColBatchWriter& ci, + void (*printFuncPtr)(const String&, const ColBatchWriter&, const size_t colWidthInBytes)) const { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) @@ -403,19 +394,19 @@ public: return m_field->pack_length(); } - size_t ColWriteBatchChar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchChar(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { - return (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) ? - ColWriteBatchString(buf, nullVal, ci, &ColWriteBatchPaddedTextString) : - ColWriteBatchString(buf, nullVal, ci, &ColWriteBatchTextString); + return (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) + ? ColWriteBatchString(buf, nullVal, ci, &ColWriteBatchPaddedTextString) + : ColWriteBatchString(buf, nullVal, ci, &ColWriteBatchTextString); } - size_t ColWriteBatchVarchar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchVarchar(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { return ColWriteBatchString(buf, nullVal, ci, &ColWriteBatchTextString); } - size_t ColWriteBatchSInt64(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchSInt64(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -424,7 +415,7 @@ public: return 8; } - size_t ColWriteBatchUInt64(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchUInt64(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -433,8 +424,7 @@ public: return 8; } - - size_t ColWriteBatchSInt32(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchSInt32(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -443,7 +433,7 @@ public: return 4; } - size_t ColWriteBatchUInt32(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchUInt32(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -452,7 +442,7 @@ public: return 4; } - size_t ColWriteBatchSInt16(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchSInt16(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -461,7 +451,7 @@ public: return 2; } - size_t ColWriteBatchUInt16(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchUInt16(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -470,7 +460,7 @@ public: return 2; } - size_t ColWriteBatchSInt8(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchSInt8(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -479,7 +469,7 @@ public: return 1; } - size_t ColWriteBatchUInt8(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchUInt8(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -488,8 +478,7 @@ public: return 1; } - - size_t ColWriteBatchXFloat(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchXFloat(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -497,7 +486,7 @@ public: { float val = *((float*)buf); - if ((fabs(val) > (1.0 / IDB_pow[4])) && (fabs(val) < (float) IDB_pow[6])) + if ((fabs(val) > (1.0 / IDB_pow[4])) && (fabs(val) < (float)IDB_pow[6])) { fprintf(ci.filePtr(), "%.7f%c", val, ci.delimiter()); } @@ -506,29 +495,27 @@ public: fprintf(ci.filePtr(), "%e%c", val, ci.delimiter()); } - //fprintf(ci.filePtr(), "%.7g|", *((float*)buf2)); - //printf("%.7f|", *((float*)buf2)); + // fprintf(ci.filePtr(), "%.7g|", *((float*)buf2)); + // printf("%.7f|", *((float*)buf2)); } return 4; } - - size_t ColWriteBatchXDouble(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchXDouble(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); else { fprintf(ci.filePtr(), "%.15g%c", *((double*)buf), ci.delimiter()); - //printf("%.15g|", *((double*)buf)); + // printf("%.15g|", *((double*)buf)); } return 8; } - - size_t ColWriteBatchSLongDouble(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchSLongDouble(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -537,8 +524,7 @@ public: return sizeof(long double); } - - size_t ColWriteBatchXDecimal(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchXDecimal(const uchar* buf, bool nullVal, ColBatchWriter& ci) override { uint bytesBefore = numDecimalBytes(m_type.precision - m_type.scale); uint totalBytes = bytesBefore + numDecimalBytes(m_type.scale); @@ -546,7 +532,7 @@ public: if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { fprintf(ci.filePtr(), "%c", ci.delimiter()); - //printf("|"); + // printf("|"); } else if (m_type.precision > datatypes::INT64MAXPRECISION) { @@ -564,16 +550,16 @@ public: } else { - uint32_t mask [5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF}; + uint32_t mask[5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF}; char neg = '-'; if (m_type.scale == 0) { const uchar* tmpBuf = buf; - //test flag bit for sign - bool posNum = tmpBuf[0] & (0x80); + // test flag bit for sign + bool posNum = tmpBuf[0] & (0x80); uchar tmpChr = tmpBuf[0]; - tmpChr ^= 0x80; //flip the bit + tmpChr ^= 0x80; // flip the bit int32_t tmp1 = tmpChr; if (totalBytes > 4) @@ -583,7 +569,7 @@ public: tmp1 = (tmp1 << 8) + tmpBuf[i]; } - if (( tmp1 != 0 ) && (tmp1 != -1)) + if ((tmp1 != 0) && (tmp1 != -1)) { if (!posNum) { @@ -592,7 +578,7 @@ public: if (tmp1 != 0) { fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } } @@ -610,7 +596,7 @@ public: tmp2 = (tmp2 << 8) + tmpBuf[i]; } - if ( tmp1 != 0 ) + if (tmp1 != 0) { if (!posNum) { @@ -632,7 +618,7 @@ public: else { fprintf(ci.filePtr(), "%09u%c", tmp2, ci.delimiter()); - //printf("%09u|", tmp2); + // printf("%09u|", tmp2); } } else @@ -641,11 +627,11 @@ public: { tmp2 = mask[4] - tmp2; fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } fprintf(ci.filePtr(), "%d%c", tmp2, ci.delimiter()); - //printf("%d|", tmp2); + // printf("%d|", tmp2); } } else @@ -659,33 +645,33 @@ public: { tmp1 = mask[totalBytes] - tmp1; fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } fprintf(ci.filePtr(), "%d%c", tmp1, ci.delimiter()); - //printf("%d|", tmp1); + // printf("%d|", tmp1); } } else { const uchar* tmpBuf = buf; - //test flag bit for sign - bool posNum = tmpBuf[0] & (0x80); + // test flag bit for sign + bool posNum = tmpBuf[0] & (0x80); uchar tmpChr = tmpBuf[0]; - tmpChr ^= 0x80; //flip the bit + tmpChr ^= 0x80; // flip the bit int32_t tmp1 = tmpChr; - //fetch the digits before decimal point + // fetch the digits before decimal point if (bytesBefore == 0) { if (!posNum) { fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } fprintf(ci.filePtr(), "0."); - //printf("0."); + // printf("0."); } else if (bytesBefore > 4) { @@ -699,23 +685,23 @@ public: tmp1 = mask[bytesBefore - 4] - tmp1; } - if (( tmp1 != 0 ) && (tmp1 != -1)) + if ((tmp1 != 0) && (tmp1 != -1)) { if (!posNum) { fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } fprintf(ci.filePtr(), "%d", tmp1); - //printf("%d", tmp1); + // printf("%d", tmp1); } tmpBuf += (bytesBefore - 4); int32_t tmp2 = *((int32_t*)tmpBuf); tmp2 = ntohl(tmp2); - if ( tmp1 != 0 ) + if (tmp1 != 0) { if (!posNum) { @@ -726,13 +712,13 @@ public: { fprintf(ci.filePtr(), "%c", neg); fprintf(ci.filePtr(), "%d.", tmp2); - //printf("%c", neg); - //printf("%d.", tmp2); + // printf("%c", neg); + // printf("%d.", tmp2); } else { fprintf(ci.filePtr(), "%09u.", tmp2); - //printf("%09u.", tmp2); + // printf("%09u.", tmp2); } } else @@ -741,11 +727,11 @@ public: { tmp2 = mask[4] - tmp2; fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } fprintf(ci.filePtr(), "%d.", tmp2); - //printf("%d.", tmp2); + // printf("%d.", tmp2); } } else @@ -759,14 +745,14 @@ public: { tmp1 = mask[bytesBefore] - tmp1; fprintf(ci.filePtr(), "%c", neg); - //printf("%c", neg); + // printf("%c", neg); } fprintf(ci.filePtr(), "%d.", tmp1); - //printf("%d.", tmp1); + // printf("%d.", tmp1); } - //fetch the digits after decimal point + // fetch the digits after decimal point int32_t tmp2 = 0; if (bytesBefore > 4) @@ -783,9 +769,9 @@ public: tmp2 = (tmp2 << 8) + tmpBuf[j]; } - int8_t digits = m_type.scale - 9; //9 digits is a 4 bytes chunk + int8_t digits = m_type.scale - 9; // 9 digits is a 4 bytes chunk - if ( digits <= 0 ) + if (digits <= 0) digits = m_type.scale; if (!posNum) @@ -794,13 +780,13 @@ public: } fprintf(ci.filePtr(), "%0*u%c", digits, tmp2, ci.delimiter()); - //printf("%0*u|", digits, tmp2); + // printf("%0*u|", digits, tmp2); } else { for (uint j = 1; j < 4; j++) { - tmp2 = (tmp2 << 8) + tmpBuf[j]; + tmp2 = (tmp2 << 8) + tmpBuf[j]; } if (!posNum) @@ -809,19 +795,19 @@ public: } fprintf(ci.filePtr(), "%09u", tmp2); - //printf("%09u", tmp2); + // printf("%09u", tmp2); tmpBuf += 4; int32_t tmp3 = tmpBuf[0]; for (uint j = 1; j < (totalBytes - bytesBefore - 4); j++) { - tmp3 = (tmp3 << 8) + tmpBuf[j]; + tmp3 = (tmp3 << 8) + tmpBuf[j]; } - int8_t digits = m_type.scale - 9; //9 digits is a 4 bytes chunk + int8_t digits = m_type.scale - 9; // 9 digits is a 4 bytes chunk - if ( digits < 0 ) + if (digits < 0) digits = m_type.scale; if (!posNum) @@ -830,7 +816,7 @@ public: } fprintf(ci.filePtr(), "%0*u%c", digits, tmp3, ci.delimiter()); - //printf("%0*u|", digits, tmp3); + // printf("%0*u|", digits, tmp3); } } } @@ -838,23 +824,20 @@ public: return totalBytes; } - - size_t ColWriteBatchVarbinary(const uchar *buf0, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchVarbinary(const uchar* buf0, bool nullVal, ColBatchWriter& ci) override { return ColWriteBatchString(buf0, nullVal, ci, &ColWriteBatchBlobString); } - - size_t ColWriteBatchBlob(const uchar *buf0, bool nullVal, ColBatchWriter &ci) override + size_t ColWriteBatchBlob(const uchar* buf0, bool nullVal, ColBatchWriter& ci) override { - return (UNLIKELY(m_type.colDataType == CalpontSystemCatalog::BLOB)) ? - ColWriteBatchString(buf0, nullVal, ci, &ColWriteBatchBlobString) : - ColWriteBatchString(buf0, nullVal, ci, &ColWriteBatchTextString); + return (UNLIKELY(m_type.colDataType == CalpontSystemCatalog::BLOB)) + ? ColWriteBatchString(buf0, nullVal, ci, &ColWriteBatchBlobString) + : ColWriteBatchString(buf0, nullVal, ci, &ColWriteBatchTextString); } - }; -} // end of namespace datatypes +} // end of namespace datatypes #endif diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 06c215084..465fdf47d 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -92,49 +92,46 @@ using namespace joblist; namespace { - typedef CalpontSelectExecutionPlan::ColumnMap::value_type CMVT_; // HDFS is never used nowadays, so don't bother -bool useHdfs = false; // ResourceManager::instance()->useHdfs(); +bool useHdfs = false; // ResourceManager::instance()->useHdfs(); #include "ha_autoi.cpp" -//convenience fcn +// convenience fcn inline uint32_t tid2sid(const uint32_t tid) { - return CalpontSystemCatalog::idb_tid2sid(tid); + return CalpontSystemCatalog::idb_tid2sid(tid); } -static void decode_objectname(char *buf, const char *path, size_t buf_size) +static void decode_objectname(char* buf, const char* path, size_t buf_size) { - size_t new_path_len = filename_to_tablename(path, buf, buf_size); - buf[new_path_len] = '\0'; + size_t new_path_len = filename_to_tablename(path, buf, buf_size); + buf[new_path_len] = '\0'; } -static void decode_file_path(const char *path, char *decoded_dbname, - char *decoded_tbname) +static void decode_file_path(const char* path, char* decoded_dbname, char* decoded_tbname) { - // The format cont ains './' in the beginning of a path. - char *dbname_start = (char*) path + 2; - char *dbname_end = dbname_start; - while (*dbname_end != '/') + // The format cont ains './' in the beginning of a path. + char* dbname_start = (char*)path + 2; + char* dbname_end = dbname_start; + while (*dbname_end != '/') dbname_end++; - int cnt = dbname_end - dbname_start; - char *dbname = (char *)my_alloca(cnt + 1); - memcpy(dbname, dbname_start, cnt); - dbname[cnt] = '\0'; - decode_objectname(decoded_dbname, dbname, FN_REFLEN); - my_afree(dbname); + int cnt = dbname_end - dbname_start; + char* dbname = (char*)my_alloca(cnt + 1); + memcpy(dbname, dbname_start, cnt); + dbname[cnt] = '\0'; + decode_objectname(decoded_dbname, dbname, FN_REFLEN); + my_afree(dbname); - char *tbname_start = dbname_end + 1; - decode_objectname(decoded_tbname, tbname_start, FN_REFLEN); + char* tbname_start = dbname_end + 1; + decode_objectname(decoded_tbname, tbname_start, FN_REFLEN); } - -CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType &ct) +CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType& ct) { - const datatypes::TypeHandler *h= datatypes::TypeHandler::find_by_ddltype(ct); + const datatypes::TypeHandler* h = datatypes::TypeHandler::find_by_ddltype(ct); if (!h) { throw runtime_error("Unsupported datatype!"); @@ -143,1981 +140,2077 @@ CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType & return h->code(); } - -int parseCompressionComment ( std::string comment ) +int parseCompressionComment(std::string comment) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*COMPRESSION[[:space:]]*=[[:space:]]*", regex_constants::extended); - int compressiontype = 0; - boost::match_results what; - std::string::const_iterator start, end; - start = comment.begin(); - end = comment.end(); - boost::match_flag_type flags = boost::match_default; + algorithm::to_upper(comment); + regex compat("[[:space:]]*COMPRESSION[[:space:]]*=[[:space:]]*", regex_constants::extended); + int compressiontype = 0; + boost::match_results what; + std::string::const_iterator start, end; + start = comment.begin(); + end = comment.end(); + boost::match_flag_type flags = boost::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (boost::regex_search(start, end, what, compat, flags)) + { + // Find the pattern, now get the compression type + string compType(&(*(what[0].second))); + //; is the separator between compression and autoincrement comments. + unsigned i = compType.find_first_of(";"); + + if (i <= compType.length()) { - //Find the pattern, now get the compression type - string compType (&(*(what[0].second))); - //; is the separator between compression and autoincrement comments. - unsigned i = compType.find_first_of(";"); - - if ( i <= compType.length() ) - { - compType = compType.substr( 0, i); - } - - i = compType.find_last_not_of(" "); - - if ( i <= compType.length() ) - { - compType = compType.substr( 0, i + 1); - } - - errno = 0; - char* ep = NULL; - const char* str = compType.c_str(); - compressiontype = strtoll(str, &ep, 10); - - // (no digits) || (more chars) || (other errors & value = 0) - if ((ep == str) || (*ep != '\0') || (errno != 0 && compressiontype == 0)) - { - compressiontype = -1; - } - + compType = compType.substr(0, i); } - else - compressiontype = MAX_INT; - // MCOL-4685: ignore [COMMENT '[compression=0] option at table or column level (no error - // messages, just disregard); - if (compressiontype == 0) - compressiontype = 2; + i = compType.find_last_not_of(" "); - return compressiontype; + if (i <= compType.length()) + { + compType = compType.substr(0, i + 1); + } + + errno = 0; + char* ep = NULL; + const char* str = compType.c_str(); + compressiontype = strtoll(str, &ep, 10); + + // (no digits) || (more chars) || (other errors & value = 0) + if ((ep == str) || (*ep != '\0') || (errno != 0 && compressiontype == 0)) + { + compressiontype = -1; + } + } + else + compressiontype = MAX_INT; + + // MCOL-4685: ignore [COMMENT '[compression=0] option at table or column level (no error + // messages, just disregard); + if (compressiontype == 0) + compressiontype = 2; + + return compressiontype; } - -bool validateAutoincrementDatatype ( int type ) +bool validateAutoincrementDatatype(int type) { - bool validAutoType = false; + bool validAutoType = false; - switch (type) - { - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: - case ddlpackage::DDL_BIGINT: - case ddlpackage::DDL_MEDINT: - case ddlpackage::DDL_SMALLINT: - case ddlpackage::DDL_TINYINT: - case ddlpackage::DDL_UNSIGNED_INT: - case ddlpackage::DDL_UNSIGNED_BIGINT: - case ddlpackage::DDL_UNSIGNED_MEDINT: - case ddlpackage::DDL_UNSIGNED_SMALLINT: - case ddlpackage::DDL_UNSIGNED_TINYINT: - validAutoType = true; - break; - } + switch (type) + { + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: + case ddlpackage::DDL_BIGINT: + case ddlpackage::DDL_MEDINT: + case ddlpackage::DDL_SMALLINT: + case ddlpackage::DDL_TINYINT: + case ddlpackage::DDL_UNSIGNED_INT: + case ddlpackage::DDL_UNSIGNED_BIGINT: + case ddlpackage::DDL_UNSIGNED_MEDINT: + case ddlpackage::DDL_UNSIGNED_SMALLINT: + case ddlpackage::DDL_UNSIGNED_TINYINT: validAutoType = true; break; + } - return validAutoType; + return validAutoType; } -bool validateNextValue( int type, int64_t value ) +bool validateNextValue(int type, int64_t value) { - bool validValue = true; + bool validValue = true; - switch (type) + switch (type) + { + case ddlpackage::DDL_BIGINT: { - case ddlpackage::DDL_BIGINT: - { - if (value > MAX_BIGINT) - validValue = false; - } - break; - - case ddlpackage::DDL_UNSIGNED_BIGINT: - { - if (static_cast(value) > MAX_UBIGINT) - validValue = false; - } - break; - - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: - { - if (value > MAX_INT) - validValue = false; - } - break; - - case ddlpackage::DDL_MEDINT: - { - if (value > MAX_MEDINT) - validValue = false; - } - break; - - case ddlpackage::DDL_UNSIGNED_INT: - { - if (static_cast(value) > MAX_UINT) - validValue = false; - } - break; - - case ddlpackage::DDL_UNSIGNED_MEDINT: - { - if (static_cast(value) > MAX_UMEDINT) - validValue = false; - } - break; - - case ddlpackage::DDL_SMALLINT: - { - if (value > MAX_SMALLINT) - validValue = false; - } - break; - - case ddlpackage::DDL_UNSIGNED_SMALLINT: - { - if (static_cast(value) > MAX_USMALLINT) - validValue = false; - } - break; - - case ddlpackage::DDL_TINYINT: - { - if (value > MAX_TINYINT) - validValue = false; - } - break; - - case ddlpackage::DDL_UNSIGNED_TINYINT: - { - if (static_cast(value) > MAX_UTINYINT) - validValue = false; - } - break; + if (value > MAX_BIGINT) + validValue = false; } + break; - return validValue; + case ddlpackage::DDL_UNSIGNED_BIGINT: + { + if (static_cast(value) > MAX_UBIGINT) + validValue = false; + } + break; + + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: + { + if (value > MAX_INT) + validValue = false; + } + break; + + case ddlpackage::DDL_MEDINT: + { + if (value > MAX_MEDINT) + validValue = false; + } + break; + + case ddlpackage::DDL_UNSIGNED_INT: + { + if (static_cast(value) > MAX_UINT) + validValue = false; + } + break; + + case ddlpackage::DDL_UNSIGNED_MEDINT: + { + if (static_cast(value) > MAX_UMEDINT) + validValue = false; + } + break; + + case ddlpackage::DDL_SMALLINT: + { + if (value > MAX_SMALLINT) + validValue = false; + } + break; + + case ddlpackage::DDL_UNSIGNED_SMALLINT: + { + if (static_cast(value) > MAX_USMALLINT) + validValue = false; + } + break; + + case ddlpackage::DDL_TINYINT: + { + if (value > MAX_TINYINT) + validValue = false; + } + break; + + case ddlpackage::DDL_UNSIGNED_TINYINT: + { + if (static_cast(value) > MAX_UTINYINT) + validValue = false; + } + break; + } + + return validValue; } bool anyRowInTable(string& schema, string& tableName, int sessionID) { - //find a column in the table - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableName aTableName; - if (lower_case_table_names) + // find a column in the table + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName aTableName; + if (lower_case_table_names) + { + algorithm::to_lower(schema); + algorithm::to_lower(tableName); + } + aTableName.schema = schema; + aTableName.table = tableName; + + CalpontSystemCatalog::RIDList ridList = csc->columnRIDs(aTableName, true); + CalpontSystemCatalog::TableColName tableColName = csc->colName(ridList[0].objnum); + + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + csep.txnID(txnID.id); + csep.verID(verID); + csep.sessionID(sessionID); + + string firstCol = tableColName.schema + "." + tableColName.table + "." + tableColName.column; + SimpleColumn* col[1]; + col[0] = new SimpleColumn(firstCol, sessionID); + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(firstCol, srcp)); + csep.columnMapNonStatic(colMap); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable(schema, tableName, "")); + csep.tableList(tablelist); + + boost::shared_ptr exemgrClient( + new messageqcpp::MessageQueueClient("ExeMgr1")); + ByteStream msg, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + rowgroup::RowGroup* rowGroup = 0; + bool anyRow = false; + + exemgrClient->write(msg); + ByteStream msgPlan; + csep.serialize(msgPlan); + exemgrClient->write(msgPlan); + msg.restart(); + msg = exemgrClient->read(); // error handling + emsgBs = exemgrClient->read(); + ByteStream::quadbyte qb1; + + if (emsgBs.length() == 0) + { + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb1; + + if (qb1 != 0) { - algorithm::to_lower(schema); - algorithm::to_lower(tableName); + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); } - aTableName.schema = schema; - aTableName.table = tableName; + } - CalpontSystemCatalog::RIDList ridList = csc->columnRIDs(aTableName, true); - CalpontSystemCatalog::TableColName tableColName = csc->colName(ridList[0].objnum); - - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - csep.txnID(txnID.id); - csep.verID(verID); - csep.sessionID(sessionID); - - string firstCol = tableColName.schema + "." + tableColName.table + "." + tableColName.column; - SimpleColumn* col[1]; - col[0] = new SimpleColumn(firstCol, sessionID); - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(firstCol, srcp)); - csep.columnMapNonStatic(colMap); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - CalpontSelectExecutionPlan::TableList tablelist; - tablelist.push_back(make_aliastable(schema, tableName, "")); - csep.tableList(tablelist); - - boost::shared_ptr exemgrClient (new messageqcpp::MessageQueueClient("ExeMgr1")); - ByteStream msg, emsgBs; - rowgroup::RGData rgData; - ByteStream::quadbyte qb = 4; - msg << qb; - rowgroup::RowGroup* rowGroup = 0; - bool anyRow = false; - - exemgrClient->write(msg); - ByteStream msgPlan; - csep.serialize(msgPlan); - exemgrClient->write(msgPlan); + while (true) + { msg.restart(); - msg = exemgrClient->read(); //error handling - emsgBs = exemgrClient->read(); - ByteStream::quadbyte qb1; + msg = exemgrClient->read(); - if (emsgBs.length() == 0) + if (msg.length() == 0) { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); } - - string emsgStr; - emsgBs >> emsgStr; - - if (msg.length() == 4) - { - msg >> qb1; - - if (qb1 != 0) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - } - - while (true) + else { + if (!rowGroup) + { + // This is mete data + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(msg); + qb = 100; msg.restart(); - msg = exemgrClient->read(); + msg << qb; + exemgrClient->write(msg); + continue; + } - if ( msg.length() == 0 ) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); - } - else - { - if (!rowGroup) - { - //This is mete data - rowGroup = new rowgroup::RowGroup(); - rowGroup->deserialize(msg); - qb = 100; - msg.restart(); - msg << qb; - exemgrClient->write(msg); - continue; - } + rgData.deserialize(msg); + rowGroup->setData(&rgData); - rgData.deserialize(msg); - rowGroup->setData(&rgData); + if (rowGroup->getStatus() != 0) + { + // msg.advance(rowGroup->getDataSize()); + msg >> emsgStr; + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); + } - if (rowGroup->getStatus() != 0) - { - //msg.advance(rowGroup->getDataSize()); - msg >> emsgStr; - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } + if (rowGroup->getRowCount() > 0) + anyRow = true; - if (rowGroup->getRowCount() > 0) - anyRow = true; - - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - return anyRow; - } + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + return anyRow; } + } } bool anyTimestampColumn(string& schema, string& tableName, int sessionID) { - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableName aTableName; - if (lower_case_table_names) + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName aTableName; + if (lower_case_table_names) + { + algorithm::to_lower(schema); + algorithm::to_lower(tableName); + } + + // select columnname from calpontsys.syscolumn + // where schema = schema and tablename = tableName + // and datatype = 'timestamp' + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + csep.txnID(txnID.id); + csep.verID(verID); + csep.sessionID(sessionID); + + string sysTable = "calpontsys.syscolumn."; + string firstCol = sysTable + "columnname"; + SimpleColumn* c1 = new SimpleColumn(firstCol, sessionID); + string secondCol = sysTable + "schema"; + SimpleColumn* c2 = new SimpleColumn(secondCol, sessionID); + string thirdCol = sysTable + "tablename"; + SimpleColumn* c3 = new SimpleColumn(thirdCol, sessionID); + string fourthCol = sysTable + "datatype"; + SimpleColumn* c4 = new SimpleColumn(fourthCol, sessionID); + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(firstCol, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(secondCol, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(thirdCol, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(fourthCol, srcp)); + csep.columnMapNonStatic(colMap); + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + // Filters + const SOP opeq(new Operator("=")); + SimpleFilter* f1 = new SimpleFilter(opeq, c2->clone(), new ConstantColumn(schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, c3->clone(), new ConstantColumn(tableName, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = new SimpleFilter( + opeq, c4->clone(), + new ConstantColumn((uint64_t)execplan::CalpontSystemCatalog::TIMESTAMP, ConstantColumn::NUM)); + filterTokenList.push_back(f3); + csep.filterTokenList(filterTokenList); + + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable("calpontsys", "syscolumn", "")); + csep.tableList(tablelist); + + boost::shared_ptr exemgrClient( + new messageqcpp::MessageQueueClient("ExeMgr1")); + ByteStream msg, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + rowgroup::RowGroup* rowGroup = 0; + bool anyRow = false; + + exemgrClient->write(msg); + ByteStream msgPlan; + csep.serialize(msgPlan); + exemgrClient->write(msgPlan); + msg.restart(); + msg = exemgrClient->read(); // error handling + emsgBs = exemgrClient->read(); + ByteStream::quadbyte qb1; + + if (emsgBs.length() == 0) + { + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb1; + + if (qb1 != 0) { - algorithm::to_lower(schema); - algorithm::to_lower(tableName); + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); } + } - // select columnname from calpontsys.syscolumn - // where schema = schema and tablename = tableName - // and datatype = 'timestamp' - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - csep.txnID(txnID.id); - csep.verID(verID); - csep.sessionID(sessionID); - - string sysTable = "calpontsys.syscolumn."; - string firstCol = sysTable + "columnname"; - SimpleColumn* c1 = new SimpleColumn(firstCol, sessionID); - string secondCol = sysTable + "schema"; - SimpleColumn* c2 = new SimpleColumn(secondCol, sessionID); - string thirdCol = sysTable + "tablename"; - SimpleColumn* c3 = new SimpleColumn(thirdCol, sessionID); - string fourthCol = sysTable + "datatype"; - SimpleColumn* c4 = new SimpleColumn(fourthCol, sessionID); - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(firstCol, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(secondCol, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(thirdCol, srcp)); - srcp.reset(c4); - colMap.insert(CMVT_(fourthCol, srcp)); - csep.columnMapNonStatic(colMap); - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - // Filters - const SOP opeq(new Operator("=")); - SimpleFilter* f1 = new SimpleFilter (opeq, - c2->clone(), - new ConstantColumn(schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - c3->clone(), - new ConstantColumn(tableName, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f3 = new SimpleFilter (opeq, - c4->clone(), - new ConstantColumn((uint64_t) execplan::CalpontSystemCatalog::TIMESTAMP, ConstantColumn::NUM)); - filterTokenList.push_back(f3); - csep.filterTokenList(filterTokenList); - - CalpontSelectExecutionPlan::TableList tablelist; - tablelist.push_back(make_aliastable("calpontsys", "syscolumn", "")); - csep.tableList(tablelist); - - boost::shared_ptr exemgrClient (new messageqcpp::MessageQueueClient("ExeMgr1")); - ByteStream msg, emsgBs; - rowgroup::RGData rgData; - ByteStream::quadbyte qb = 4; - msg << qb; - rowgroup::RowGroup* rowGroup = 0; - bool anyRow = false; - - exemgrClient->write(msg); - ByteStream msgPlan; - csep.serialize(msgPlan); - exemgrClient->write(msgPlan); + while (true) + { msg.restart(); - msg = exemgrClient->read(); //error handling - emsgBs = exemgrClient->read(); - ByteStream::quadbyte qb1; + msg = exemgrClient->read(); - if (emsgBs.length() == 0) + if (msg.length() == 0) { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); - } - - string emsgStr; - emsgBs >> emsgStr; - - if (msg.length() == 4) - { - msg >> qb1; - - if (qb1 != 0) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - } - - while (true) - { - msg.restart(); - msg = exemgrClient->read(); - - if ( msg.length() == 0 ) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); - } - else - { - if (!rowGroup) - { - //This is mete data - rowGroup = new rowgroup::RowGroup(); - rowGroup->deserialize(msg); - qb = 100; - msg.restart(); - msg << qb; - exemgrClient->write(msg); - continue; - } - - rgData.deserialize(msg); - rowGroup->setData(&rgData); - - if (rowGroup->getStatus() != 0) - { - //msg.advance(rowGroup->getDataSize()); - msg >> emsgStr; - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - - if (rowGroup->getRowCount() > 0) - anyRow = true; - - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - return anyRow; - } - } -} - -bool anyNullInTheColumn (THD* thd, string& schema, string& table, string& columnName, int sessionID) -{ - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; - if (lower_case_table_names) - { - algorithm::to_lower(schema); - algorithm::to_lower(table); - } - algorithm::to_lower(columnName); - - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - csep.txnID(txnID.id); - csep.verID(verID); - csep.sessionID(sessionID); - - string firstCol = schema + "." + table + "." + columnName; - SimpleColumn* col[1]; - col[0] = new SimpleColumn(firstCol, sessionID); - SRCP srcp; - srcp.reset(col[0]); - colMap.insert(CMVT_(firstCol, srcp)); - csep.columnMapNonStatic(colMap); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); - boost::shared_ptr sop(new PredicateOperator("isnull")); - sf->op(sop); - ConstantColumn* rhs = new ConstantColumn("", ConstantColumn::NULLDATA); - rhs->timeZone(thd->variables.time_zone->get_name()->ptr()); - sf->lhs(col[0]->clone()); - sf->rhs(rhs); - - filterTokenList.push_back(sf); - csep.filterTokenList(filterTokenList); - - CalpontSelectExecutionPlan::TableList tablelist; - tablelist.push_back(make_aliastable(schema, table, "")); - csep.tableList(tablelist); - - boost::shared_ptr exemgrClient (new messageqcpp::MessageQueueClient("ExeMgr1")); - ByteStream msg, emsgBs; - rowgroup::RGData rgData; - ByteStream::quadbyte qb = 4; - msg << qb; - rowgroup::RowGroup* rowGroup = 0; - bool anyRow = false; - - exemgrClient->write(msg); - ByteStream msgPlan; - csep.serialize(msgPlan); - exemgrClient->write(msgPlan); - msg.restart(); - msg = exemgrClient->read(); //error handling - emsgBs = exemgrClient->read(); - ByteStream::quadbyte qb1; - - if (emsgBs.length() == 0) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); - } - - string emsgStr; - emsgBs >> emsgStr; - - if (msg.length() == 4) - { - msg >> qb1; - - if (qb1 != 0) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - } - - while (true) - { - msg.restart(); - msg = exemgrClient->read(); - - if ( msg.length() == 0 ) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); - } - else - { - if (!rowGroup) - { - //This is mete data - rowGroup = new rowgroup::RowGroup(); - rowGroup->deserialize(msg); - qb = 100; - msg.restart(); - msg << qb; - exemgrClient->write(msg); - continue; - } - - rgData.deserialize(msg); - rowGroup->setData(&rgData); - - if (rowGroup->getStatus() != 0) - { - //msg.advance(amount); - msg >> emsgStr; - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - - if (rowGroup->getRowCount() > 0) - anyRow = true; - - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - return anyRow; - } - } -} - -int ProcessDDLStatement(string& ddlStatement, string& schema, const string& table, int sessionID, - string& emsg, int compressionTypeIn = 2, bool isAnyAutoincreCol = false, int64_t nextvalue = 1, std::string autoiColName = "", - const CHARSET_INFO* default_table_charset = NULL) -{ - SqlParser parser; - THD* thd = current_thd; -#ifdef MCS_DEBUG - cout << "ProcessDDLStatement: " << schema << "." << table << ":" << ddlStatement << endl; -#endif - - parser.setDefaultSchema(schema); - parser.setDefaultCharset(default_table_charset); - int rc = 0; - parser.Parse(ddlStatement.c_str()); - - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - if (parser.Good()) - { - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - const ddlpackage::ParseTree& ptree = parser.GetParseTree(); - - if (UNLIKELY(ptree.fList.size() == 0)) - { - // TODO: Once the crash bug is found, this should convert to "return 0" - cout << "***** ProcessDDLStatement has no stmt *****" << endl; - setError(thd, ER_CHECK_NOT_IMPLEMENTED, "DDL processed without statement"); - return 1; - } - - SqlStatement& stmt = *ptree.fList[0]; - bool isVarbinaryAllowed = false; - std::string valConfig = config::Config::makeConfig()->getConfig( - "WriteEngine", "AllowVarbinary" ); - algorithm::to_upper(valConfig); - - if (valConfig.compare("YES") == 0) - isVarbinaryAllowed = true; - - //@Bug 1771. error out for not supported feature. - if ( typeid ( stmt ) == typeid ( CreateTableStatement ) ) - { - CreateTableStatement* createTable = dynamic_cast ( &stmt ); - if (lower_case_table_names) - { - algorithm::to_lower(createTable->fTableDef->fQualifiedName->fSchema); - algorithm::to_lower(createTable->fTableDef->fQualifiedName->fName); - } - - bool matchedCol = false; - bool isFirstTimestamp = true; - - for ( unsigned i = 0; i < createTable->fTableDef->fColumns.size(); i++ ) - { - // if there are any constraints other than 'DEFAULT NULL' (which is the default in IDB), kill - // the statement - bool autoIncre = false; - uint64_t startValue = 1; - - if (createTable->fTableDef->fColumns[i]->fConstraints.size() > 0 ) - { - //support default value and NOT NULL constraint - for (uint32_t j = 0; j < createTable->fTableDef->fColumns[i]->fConstraints.size(); j++) - { - if (createTable->fTableDef->fColumns[i]->fConstraints[j]->fConstraintType != DDL_NOT_NULL) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - } - - //check varbinary data type - if ((createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY) && !isVarbinaryAllowed) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary is currently not supported by Columnstore."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY) && - ((createTable->fTableDef->fColumns[i]->fType->fLength > 8000) || (createTable->fTableDef->fColumns[i]->fType->fLength < 8))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary length has to be between 8 and 8000."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - // For TIMESTAMP, if no constraint is given, default to NOT NULL - if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP && - createTable->fTableDef->fColumns[i]->fConstraints.empty()) - { - createTable->fTableDef->fColumns[i]->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); - } - - if (createTable->fTableDef->fColumns[i]->fDefaultValue) - { - if ((!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) && (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary column cannot have default value."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) - { - //validate the default value, if out of range, just error out - CalpontSystemCatalog::ColDataType dataType; - dataType = convertDataType(createTable->fTableDef->fColumns[i]->fType->fType); - CalpontSystemCatalog::ColType colType; - colType.colDataType = dataType; - colType.colWidth = createTable->fTableDef->fColumns[i]->fType->fLength; - colType.precision = createTable->fTableDef->fColumns[i]->fType->fPrecision; - colType.scale = createTable->fTableDef->fColumns[i]->fType->fScale; - boost::any convertedVal; - bool pushWarning = false; - - try - { - convertedVal = colType.convertColumnData(createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); - } - catch (std::exception&) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The default value is out of range for the specified data type."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (pushWarning) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The default value is out of range for the specified data type."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP) - { - if (createTable->fTableDef->fColumns[i]->fDefaultValue->fValue == "0") - { - createTable->fTableDef->fColumns[i]->fDefaultValue->fValue = "0000-00-00 00:00:00"; - } - if (isFirstTimestamp) - { - isFirstTimestamp = false; - } - } - } - } - // If no default value exists for TIMESTAMP, we apply - // automatic TIMESTAMP properties. - // TODO: If no default value exists but the constraint is NULL, - // default value should be set to NULL. But this is currently - // not supported since columnstore does not track whether user - // specified a NULL or not - else if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP) - { - if (isFirstTimestamp) - { - isFirstTimestamp = false; - createTable->fTableDef->fColumns[i]->fDefaultValue = new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); - } - else - { - createTable->fTableDef->fColumns[i]->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); - } - } - - //Parse the column comment - string comment = createTable->fTableDef->fColumns[i]->fComment; - int compressionType = compressionTypeIn; - - if ( comment.length() > 0 ) - { - compressionType = parseCompressionComment( comment); - - if ( compressionType == MAX_INT ) - { - compressionType = compressionTypeIn; - } - else if ( compressionType < 0 ) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (compressionType == 1) compressionType = 2; - - if ((compressionType > 0) && - !(compress::CompressInterface::isCompressionAvail( - compressionType))) - { - rc = 1; - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; -#ifdef SKIP_IDB_COMPRESSION - Message::Args args; - thd->get_stmt_da()->set_overwrite_status(true); - args.add("The compression type"); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_ENTERPRISE_ONLY, args)).c_str()); -#else - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); -#endif - return rc; - } - - try - { - autoIncre = parseAutoincrementColumnComment(comment, startValue); - - if (autoIncre) - { - //Check whether there is a column with autoincrement already - if ((isAnyAutoincreCol) && !(boost::iequals(autoiColName, createTable->fTableDef->fColumns[i]->fName))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - else - { - isAnyAutoincreCol = true; - autoiColName = createTable->fTableDef->fColumns[i]->fName; - matchedCol = true; - } - - //Check whether the column has default value. If there is, error out - if (createTable->fTableDef->fColumns[i]->fDefaultValue) - { - if (!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Autoincrement column cannot have a default value."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - } - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - } - - if (!autoIncre && isAnyAutoincreCol && (boost::iequals(autoiColName, createTable->fTableDef->fColumns[i]->fName))) - { - autoIncre = true; - matchedCol = true; - startValue = nextvalue; - } - - if (startValue <= 0) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (autoIncre) - { - if (!validateAutoincrementDatatype(createTable->fTableDef->fColumns[i]->fType->fType)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - - if (!validateNextValue(createTable->fTableDef->fColumns[i]->fType->fType, startValue)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //hdfs - if ((compressionType == 0) && (useHdfs)) - { - compressionType = 2; - string errmsg ("The table is created with Columnstore compression type 2 under HDFS." ); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); - } - - (createTable->fTableDef->fColumns[i]->fType)->fCompressiontype = compressionType; - - if (autoIncre) - (createTable->fTableDef->fColumns[i]->fType)->fAutoincrement = "y"; - else - (createTable->fTableDef->fColumns[i]->fType)->fAutoincrement = "n"; - - (createTable->fTableDef->fColumns[i]->fType)->fNextvalue = startValue; - } - - if (isAnyAutoincreCol && !matchedCol) //@Bug 3555 error out on invalid column - { - rc = 1; - Message::Args args; - thd->get_stmt_da()->set_overwrite_status(true); - args.add(autoiColName); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL, args)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - else if ( typeid ( stmt ) == typeid ( AlterTableStatement ) ) - { - AlterTableStatement* alterTable = dynamic_cast ( &stmt ); - if (lower_case_table_names) - { - algorithm::to_lower(alterTable->fTableName->fSchema); - algorithm::to_lower(alterTable->fTableName->fName); - } - - alterTable->fTimeZone.assign(thd->variables.time_zone->get_name()->ptr()); - - if ( schema.length() == 0 ) - { - schema = alterTable->fTableName->fSchema; - - if ( schema.length() == 0 ) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "No database selected."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - - ddlpackage::AlterTableActionList actionList = alterTable->fActions; - - if (actionList.size() > 1) //@bug 3753 we don't support multiple actions in alter table statement - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Multiple actions in alter table statement is currently not supported by Columnstore."); - - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - for ( unsigned i = 0; i < actionList.size(); i++ ) - { - if ( ddlpackage::AtaAddColumn* addColumnPtr = dynamic_cast (actionList[i]) ) - { - //check varbinary data type - if ((addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_VARBINARY) && !isVarbinaryAllowed) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary is currently not supported by Columnstore."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_VARBINARY) && - ((addColumnPtr->fColumnDef->fType->fLength > 8000) || (addColumnPtr->fColumnDef->fType->fLength < 8))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary length has to be between 8 and 8000."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - uint64_t startValue = 1; - bool autoIncre = false; - - if ( (addColumnPtr->fColumnDef->fConstraints.size() > 0 ) || addColumnPtr->fColumnDef->fDefaultValue ) - { - //support default value and NOT NULL constraint - for (uint32_t j = 0; j < addColumnPtr->fColumnDef->fConstraints.size(); j++) - { - if (addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType != DDL_NOT_NULL) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //if not null constraint, user has to provide a default value - if ((addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (!addColumnPtr->fColumnDef->fDefaultValue) && (addColumnPtr->fColumnDef->fType->fType != ddlpackage::DDL_TIMESTAMP)) - { - - //do select count(*) from the table to check whether there are existing rows. if there is, error out. - bool anyRow = false; - - try - { - anyRow = anyRowInTable(alterTable->fTableName->fSchema, alterTable->fTableName->fName, sessionID); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - catch (...) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown exception caught when checking any rows in the table."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (anyRow) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Table is not empty. New column has to have a default value if NOT NULL required."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - else if ((addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (addColumnPtr->fColumnDef->fDefaultValue)) - { - if (addColumnPtr->fColumnDef->fDefaultValue->fValue.length() == 0) //empty string is NULL in infinidb - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "New column has to have a default value if NOT NULL required."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - } - - if (addColumnPtr->fColumnDef->fDefaultValue) - { - if ((!addColumnPtr->fColumnDef->fDefaultValue->fNull) && (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_VARBINARY)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary column cannot have default value."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //validate the default value, if out of range, just error out - CalpontSystemCatalog::ColDataType dataType; - dataType = convertDataType(addColumnPtr->fColumnDef->fType->fType); - CalpontSystemCatalog::ColType colType; - colType.colDataType = dataType; - colType.colWidth = addColumnPtr->fColumnDef->fType->fLength; - colType.precision = addColumnPtr->fColumnDef->fType->fPrecision; - colType.scale = addColumnPtr->fColumnDef->fType->fScale; - boost::any convertedVal; - bool pushWarning = false; - - try - { - convertedVal = colType.convertColumnData(addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); - } - catch (std::exception&) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The default value is out of range for the specified data type."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (pushWarning) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The default value is out of range for the specified data type."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && - addColumnPtr->fColumnDef->fDefaultValue->fValue == "0") - { - addColumnPtr->fColumnDef->fDefaultValue->fValue = "0000-00-00 00:00:00"; - } - } - } - - // For TIMESTAMP, if no constraint is given, default to NOT NULL - if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && - addColumnPtr->fColumnDef->fConstraints.empty()) - { - addColumnPtr->fColumnDef->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); - } - - // If no default value exists for TIMESTAMP, we apply - // automatic TIMESTAMP properties. - // TODO: If no default value exists but the constraint is NULL, - // default value should be set to NULL. But this is currently - // not supported since columnstore does not track whether user - // specified a NULL or not - if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && - !addColumnPtr->fColumnDef->fDefaultValue) - { - // Query calpontsys.syscolumn to see - // if a timestamp column already exists in this table - if (!anyTimestampColumn(alterTable->fTableName->fSchema, alterTable->fTableName->fName, sessionID)) - { - addColumnPtr->fColumnDef->fDefaultValue = new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); - } - else - { - addColumnPtr->fColumnDef->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); - } - } - - //Handle compression type - string comment = addColumnPtr->fColumnDef->fComment; - int compressionType = compressionTypeIn; - - if ( comment.length() > 0 ) - { - //@Bug 3782 This is for synchronization after calonlinealter to use - algorithm::to_upper(comment); - regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); - - if (regex_search(comment, pat)) - { - return 0; - } - - compressionType = parseCompressionComment( comment); - - if ( compressionType == MAX_INT ) - { - compressionType = compressionTypeIn; - } - else if ( compressionType < 0 ) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((compressionType > 0) && - !(compress::CompressInterface::isCompressionAvail( - compressionType))) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((compressionType == 0) && (useHdfs)) - { - compressionType = 2; - string errmsg ("The column is created with Columnstore compression type 2 under HDFS." ); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); - } - - try - { - autoIncre = parseAutoincrementColumnComment(comment, startValue); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (autoIncre) - { - //Check if the table already has autoincrement column - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableName tableName; - tableName.schema = alterTable->fTableName->fSchema; - tableName.table = alterTable->fTableName->fName; - CalpontSystemCatalog::TableInfo tblInfo; - - try - { - tblInfo = csc->tableInfo(tableName); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (tblInfo.tablewithautoincr == 1) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (!validateAutoincrementDatatype(addColumnPtr->fColumnDef->fType->fType)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (!validateNextValue(addColumnPtr->fColumnDef->fType->fType, startValue)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (startValue <= 0) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - } - - addColumnPtr->fColumnDef->fType->fCompressiontype = compressionType; - - if (autoIncre) - addColumnPtr->fColumnDef->fType->fAutoincrement = "y"; - else - addColumnPtr->fColumnDef->fType->fAutoincrement = "n"; - - addColumnPtr->fColumnDef->fType->fNextvalue = startValue; - - } - else if ( dynamic_cast (actionList[i])) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - /* else if ( dynamic_cast (actionList[i]) ) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - else if ( dynamic_cast (actionList[i]) ) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - */ - else if ( ddlpackage::AtaAddColumns* addColumnsPtr = dynamic_cast(actionList[i])) - { - if ((addColumnsPtr->fColumns).size() > 1) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Multiple actions in alter table statement is currently not supported by Columnstore."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //check varbinary data type - if ((addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_VARBINARY) && !isVarbinaryAllowed) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary is currently not supported by Columnstore."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - - if ((addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_VARBINARY) && - ((addColumnsPtr->fColumns[0]->fType->fLength > 8000) || (addColumnsPtr->fColumns[0]->fType->fLength < 8))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary length has to be between 8 and 8000."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - uint64_t startValue = 1; - bool autoIncre = false; - - if ( (addColumnsPtr->fColumns[0]->fConstraints.size() > 0 ) || addColumnsPtr->fColumns[0]->fDefaultValue ) - { - //@Bug 5274. support default value and NOT NULL constraint - for (uint32_t j = 0; j < addColumnsPtr->fColumns[0]->fConstraints.size(); j++) - { - if (addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType != DDL_NOT_NULL) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //if not null constraint, user has to provide a default value - if ((addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (!addColumnsPtr->fColumns[0]->fDefaultValue) && (addColumnsPtr->fColumns[0]->fType->fType != ddlpackage::DDL_TIMESTAMP)) - { - - //do select count(*) from the table to check whether there are existing rows. if there is, error out. - bool anyRow = false; - - try - { - anyRow = anyRowInTable(alterTable->fTableName->fSchema, alterTable->fTableName->fName, sessionID); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - catch (...) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown exception caught when checking any rows in the table."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (anyRow) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Table is not empty. New column has to have a default value if NOT NULL required."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - else if ((addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (addColumnsPtr->fColumns[0]->fDefaultValue)) - { - if (addColumnsPtr->fColumns[0]->fDefaultValue->fValue.length() == 0) //empty string is NULL in infinidb - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "New column has to have a default value if NOT NULL required."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - } - - if (addColumnsPtr->fColumns[0]->fDefaultValue) - { - if ((!addColumnsPtr->fColumns[0]->fDefaultValue->fNull) && (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_VARBINARY)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary column cannot have default value."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //validate the default value, if out of range, just error out - CalpontSystemCatalog::ColDataType dataType; - dataType = convertDataType(addColumnsPtr->fColumns[0]->fType->fType); - CalpontSystemCatalog::ColType colType; - colType.colDataType = dataType; - colType.colWidth = addColumnsPtr->fColumns[0]->fType->fLength; - colType.precision = addColumnsPtr->fColumns[0]->fType->fPrecision; - colType.scale = addColumnsPtr->fColumns[0]->fType->fScale; - boost::any convertedVal; - bool pushWarning = false; - - try - { - convertedVal = colType.convertColumnData(addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); - } - catch (std::exception&) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The default value is out of range for the specified data type."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (pushWarning) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The default value is out of range for the specified data type."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && - addColumnsPtr->fColumns[0]->fDefaultValue->fValue == "0") - { - addColumnsPtr->fColumns[0]->fDefaultValue->fValue = "0000-00-00 00:00:00"; - } - } - } - - // For TIMESTAMP, if no constraint is given, default to NOT NULL - if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && - addColumnsPtr->fColumns[0]->fConstraints.empty()) - { - addColumnsPtr->fColumns[0]->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); - } - - // If no default value exists for TIMESTAMP, we apply - // automatic TIMESTAMP properties. - // TODO: If no default value exists but the constraint is NULL, - // default value should be set to NULL. But this is currently - // not supported since columnstore does not track whether user - // specified a NULL or not - if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && - !addColumnsPtr->fColumns[0]->fDefaultValue) - { - // Query calpontsys.syscolumn to see - // if a timestamp column already exists in this table - if (!anyTimestampColumn(alterTable->fTableName->fSchema, alterTable->fTableName->fName, sessionID)) - { - addColumnsPtr->fColumns[0]->fDefaultValue = new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); - } - else - { - addColumnsPtr->fColumns[0]->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); - } - } - - //Handle compression type - string comment = addColumnsPtr->fColumns[0]->fComment; - int compressionType = compressionTypeIn; - - if ( comment.length() > 0 ) - { - compressionType = parseCompressionComment( comment); - - if ( compressionType == MAX_INT ) - { - compressionType = compressionTypeIn; - } - else if ( compressionType < 0 ) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((compressionType > 0) && - !(compress::CompressInterface::isCompressionAvail( - compressionType))) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((compressionType == 0) && (useHdfs)) - { - compressionType = 2; - string errmsg ("The column is created with Columnstore compression type 2 under HDFS." ); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); - } - - - try - { - autoIncre = parseAutoincrementColumnComment(comment, startValue); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (autoIncre) - { - //Check if the table already has autoincrement column - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableName tableName; - tableName.schema = alterTable->fTableName->fSchema; - tableName.table = alterTable->fTableName->fName; - CalpontSystemCatalog::TableInfo tblInfo; - - try - { - tblInfo = csc->tableInfo(tableName); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (tblInfo.tablewithautoincr == 1) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (!validateAutoincrementDatatype(addColumnsPtr->fColumns[0]->fType->fType)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (!validateNextValue(addColumnsPtr->fColumns[0]->fType->fType, startValue)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (startValue <= 0) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)).c_str()); - return rc; - } - } - } - - addColumnsPtr->fColumns[0]->fType->fCompressiontype = compressionType; - - if (autoIncre) - addColumnsPtr->fColumns[0]->fType->fAutoincrement = "y"; - else - addColumnsPtr->fColumns[0]->fType->fAutoincrement = "n"; - - addColumnsPtr->fColumns[0]->fType->fNextvalue = startValue; - } - else if (ddlpackage::AtaRenameColumn* renameColumnsPtr = dynamic_cast(actionList[i])) - { - uint64_t startValue = 1; - bool autoIncre = false; - //@Bug 3746 Handle compression type - string comment = renameColumnsPtr->fComment; - int compressionType = compressionTypeIn; - - if ( comment.length() > 0 ) - { - compressionType = parseCompressionComment( comment); - - if ( compressionType == MAX_INT ) - { - compressionType = compressionTypeIn; - } - else if ( compressionType < 0 ) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((compressionType > 0) && - !(compress::CompressInterface::isCompressionAvail( - compressionType))) - { - rc = 1; - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if ((compressionType == 0) && (useHdfs)) - { - compressionType = 2; - string errmsg ("The column is created with Columnstore compression type 2 under HDFS." ); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); - } - - } - - //Handle autoincrement - if ( comment.length() > 0 ) - { - try - { - autoIncre = parseAutoincrementColumnComment(comment, startValue); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (autoIncre) - { - //Check if the table already has autoincrement column - CalpontSystemCatalog::TableName tableName; - tableName.schema = alterTable->fTableName->fSchema; - tableName.table = alterTable->fTableName->fName; - //CalpontSystemCatalog::TableInfo tblInfo = csc->tableInfo(tableName); - - //@Bug 5444. rename column doen't need to check this. - /* if (tblInfo.tablewithautoincr == 1) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } */ - - if (!validateAutoincrementDatatype(renameColumnsPtr->fNewType->fType)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (!validateNextValue(renameColumnsPtr->fNewType->fType, startValue)) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - } - } - - if (autoIncre) - renameColumnsPtr->fNewType->fAutoincrement = "y"; - else - renameColumnsPtr->fNewType->fAutoincrement = "n"; - - renameColumnsPtr->fNewType->fNextvalue = startValue; - renameColumnsPtr->fNewType->fCompressiontype = compressionType; - - if (renameColumnsPtr->fConstraints.size() > 0) - { - for (uint32_t j = 0; j < renameColumnsPtr->fConstraints.size(); j++) - { - if (renameColumnsPtr->fConstraints[j]->fConstraintType == DDL_NOT_NULL) - { - //If NOT NULL constraint is added, check whether the existing rows has null vlues. If there is, error out the query. - bool anyNullVal = false; - - try - { - anyNullVal = anyNullInTheColumn (thd, alterTable->fTableName->fSchema, alterTable->fTableName->fName, renameColumnsPtr->fName, sessionID); - } - catch (runtime_error& ex) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - catch (...) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown exception caught when checking any existing null values in the column."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - if (anyNullVal) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "The existing rows in this column has null value already."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - else - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - } - } - } - else - { - } - } - } - //@Bug 5923 error out on unsupported statements. - else if (( typeid ( stmt ) != typeid ( DropTableStatement )) && ( typeid ( stmt ) != typeid ( TruncTableStatement )) - && ( typeid ( stmt ) != typeid ( MarkPartitionStatement )) && ( typeid ( stmt ) != typeid ( RestorePartitionStatement )) - && ( typeid ( stmt ) != typeid ( DropPartitionStatement ))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SYNTAX)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - //@Bug 4387 - scoped_ptr dbrmp(new DBRM()); - int rc = dbrmp->isReadWrite(); - - if (rc != 0 ) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Cannot execute the statement. DBRM is read only!"); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - - stmt.fSessionID = sessionID; - stmt.fSql = ddlStatement; - stmt.fOwner = schema; - stmt.fTableWithAutoi = isAnyAutoincreCol; - ByteStream bytestream; - bytestream << stmt.fSessionID; - stmt.serialize(bytestream); - MessageQueueClient mq("DDLProc"); - ByteStream::byte b = 0; - - try - { - mq.write(bytestream); - bytestream = mq.read(); - - if ( bytestream.length() == 0 ) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - } - else - { - bytestream >> b; - bytestream >> emsg; - rc = b; - } - } - catch (runtime_error& e) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - } - catch (...) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown error caught"); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - } - - if (b == ddlpackageprocessor::DDLPackageProcessor::DROP_TABLE_NOT_IN_CATALOG_ERROR) - { - return ER_NO_SUCH_TABLE_IN_ENGINE; - } - - if ((b != 0) && (b != ddlpackageprocessor::DDLPackageProcessor::WARNING)) - { - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); - } - - if (b == ddlpackageprocessor::DDLPackageProcessor::WARNING) - { - rc = 0; - string errmsg ("Error occured during file deletion. Restart DDLProc or use command tool ddlcleanup to clean up. " ); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); - } - - return rc; - + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); } else { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - //@Bug 3602. Error message for MySql syntax for autoincrement - algorithm::to_upper(ddlStatement); + if (!rowGroup) + { + // This is mete data + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(msg); + qb = 100; + msg.restart(); + msg << qb; + exemgrClient->write(msg); + continue; + } - if (ddlStatement.find("AUTO_INCREMENT") != string::npos) + rgData.deserialize(msg); + rowGroup->setData(&rgData); + + if (rowGroup->getStatus() != 0) + { + // msg.advance(rowGroup->getDataSize()); + msg >> emsgStr; + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); + } + + if (rowGroup->getRowCount() > 0) + anyRow = true; + + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + return anyRow; + } + } +} + +bool anyNullInTheColumn(THD* thd, string& schema, string& table, string& columnName, int sessionID) +{ + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + if (lower_case_table_names) + { + algorithm::to_lower(schema); + algorithm::to_lower(table); + } + algorithm::to_lower(columnName); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + csep.txnID(txnID.id); + csep.verID(verID); + csep.sessionID(sessionID); + + string firstCol = schema + "." + table + "." + columnName; + SimpleColumn* col[1]; + col[0] = new SimpleColumn(firstCol, sessionID); + SRCP srcp; + srcp.reset(col[0]); + colMap.insert(CMVT_(firstCol, srcp)); + csep.columnMapNonStatic(colMap); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + SimpleFilter* sf = new SimpleFilter(); + sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + boost::shared_ptr sop(new PredicateOperator("isnull")); + sf->op(sop); + ConstantColumn* rhs = new ConstantColumn("", ConstantColumn::NULLDATA); + rhs->timeZone(thd->variables.time_zone->get_name()->ptr()); + sf->lhs(col[0]->clone()); + sf->rhs(rhs); + + filterTokenList.push_back(sf); + csep.filterTokenList(filterTokenList); + + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable(schema, table, "")); + csep.tableList(tablelist); + + boost::shared_ptr exemgrClient( + new messageqcpp::MessageQueueClient("ExeMgr1")); + ByteStream msg, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + rowgroup::RowGroup* rowGroup = 0; + bool anyRow = false; + + exemgrClient->write(msg); + ByteStream msgPlan; + csep.serialize(msgPlan); + exemgrClient->write(msgPlan); + msg.restart(); + msg = exemgrClient->read(); // error handling + emsgBs = exemgrClient->read(); + ByteStream::quadbyte qb1; + + if (emsgBs.length() == 0) + { + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb1; + + if (qb1 != 0) + { + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); + } + } + + while (true) + { + msg.restart(); + msg = exemgrClient->read(); + + if (msg.length() == 0) + { + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + else + { + if (!rowGroup) + { + // This is mete data + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(msg); + qb = 100; + msg.restart(); + msg << qb; + exemgrClient->write(msg); + continue; + } + + rgData.deserialize(msg); + rowGroup->setData(&rgData); + + if (rowGroup->getStatus() != 0) + { + // msg.advance(amount); + msg >> emsgStr; + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); + } + + if (rowGroup->getRowCount() > 0) + anyRow = true; + + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + return anyRow; + } + } +} + +int ProcessDDLStatement(string& ddlStatement, string& schema, const string& table, int sessionID, + string& emsg, int compressionTypeIn = 2, bool isAnyAutoincreCol = false, + int64_t nextvalue = 1, std::string autoiColName = "", + const CHARSET_INFO* default_table_charset = NULL) +{ + SqlParser parser; + THD* thd = current_thd; +#ifdef MCS_DEBUG + cout << "ProcessDDLStatement: " << schema << "." << table << ":" << ddlStatement << endl; +#endif + + parser.setDefaultSchema(schema); + parser.setDefaultCharset(default_table_charset); + int rc = 0; + parser.Parse(ddlStatement.c_str()); + + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + if (parser.Good()) + { + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + const ddlpackage::ParseTree& ptree = parser.GetParseTree(); + + if (UNLIKELY(ptree.fList.size() == 0)) + { + // TODO: Once the crash bug is found, this should convert to "return 0" + cout << "***** ProcessDDLStatement has no stmt *****" << endl; + setError(thd, ER_CHECK_NOT_IMPLEMENTED, "DDL processed without statement"); + return 1; + } + + SqlStatement& stmt = *ptree.fList[0]; + bool isVarbinaryAllowed = false; + std::string valConfig = config::Config::makeConfig()->getConfig("WriteEngine", "AllowVarbinary"); + algorithm::to_upper(valConfig); + + if (valConfig.compare("YES") == 0) + isVarbinaryAllowed = true; + + //@Bug 1771. error out for not supported feature. + if (typeid(stmt) == typeid(CreateTableStatement)) + { + CreateTableStatement* createTable = dynamic_cast(&stmt); + if (lower_case_table_names) + { + algorithm::to_lower(createTable->fTableDef->fQualifiedName->fSchema); + algorithm::to_lower(createTable->fTableDef->fQualifiedName->fName); + } + + bool matchedCol = false; + bool isFirstTimestamp = true; + + for (unsigned i = 0; i < createTable->fTableDef->fColumns.size(); i++) + { + // if there are any constraints other than 'DEFAULT NULL' (which is the default in IDB), kill + // the statement + bool autoIncre = false; + uint64_t startValue = 1; + + if (createTable->fTableDef->fColumns[i]->fConstraints.size() > 0) { - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Use of the MySQL auto_increment syntax is not supported in Columnstore. If you wish to create an auto increment column in Columnstore, please consult the Columnstore SQL Syntax Guide for the correct usage."); + // support default value and NOT NULL constraint + for (uint32_t j = 0; j < createTable->fTableDef->fColumns[i]->fConstraints.size(); j++) + { + if (createTable->fTableDef->fColumns[i]->fConstraints[j]->fConstraintType != DDL_NOT_NULL) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } + + // check varbinary data type + if ((createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY) && + !isVarbinaryAllowed) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + "Varbinary is currently not supported by Columnstore."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY) && + ((createTable->fTableDef->fColumns[i]->fType->fLength > 8000) || + (createTable->fTableDef->fColumns[i]->fType->fLength < 8))) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary length has to be between 8 and 8000."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // For TIMESTAMP, if no constraint is given, default to NOT NULL + if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP && + createTable->fTableDef->fColumns[i]->fConstraints.empty()) + { + createTable->fTableDef->fColumns[i]->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); + } + + if (createTable->fTableDef->fColumns[i]->fDefaultValue) + { + if ((!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) && + (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Varbinary column cannot have default value."); ci->alterTableState = cal_connection_info::NOT_ALTER; ci->isAlter = false; + return rc; + } + + if (!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) + { + // validate the default value, if out of range, just error out + CalpontSystemCatalog::ColDataType dataType; + dataType = convertDataType(createTable->fTableDef->fColumns[i]->fType->fType); + CalpontSystemCatalog::ColType colType; + colType.colDataType = dataType; + colType.colWidth = createTable->fTableDef->fColumns[i]->fType->fLength; + colType.precision = createTable->fTableDef->fColumns[i]->fType->fPrecision; + colType.scale = createTable->fTableDef->fColumns[i]->fType->fScale; + boost::any convertedVal; + bool pushWarning = false; + + try + { + convertedVal = colType.convertColumnData( + createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, + thd->variables.time_zone->get_name()->ptr(), false, false, false); + } + catch (std::exception&) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The default value is out of range for the specified data type."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (pushWarning) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The default value is out of range for the specified data type."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP) + { + if (createTable->fTableDef->fColumns[i]->fDefaultValue->fValue == "0") + { + createTable->fTableDef->fColumns[i]->fDefaultValue->fValue = "0000-00-00 00:00:00"; + } + if (isFirstTimestamp) + { + isFirstTimestamp = false; + } + } + } + } + // If no default value exists for TIMESTAMP, we apply + // automatic TIMESTAMP properties. + // TODO: If no default value exists but the constraint is NULL, + // default value should be set to NULL. But this is currently + // not supported since columnstore does not track whether user + // specified a NULL or not + else if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP) + { + if (isFirstTimestamp) + { + isFirstTimestamp = false; + createTable->fTableDef->fColumns[i]->fDefaultValue = + new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); + } + else + { + createTable->fTableDef->fColumns[i]->fDefaultValue = + new ColumnDefaultValue("0000-00-00 00:00:00"); + } + } + + // Parse the column comment + string comment = createTable->fTableDef->fColumns[i]->fComment; + int compressionType = compressionTypeIn; + + if (comment.length() > 0) + { + compressionType = parseCompressionComment(comment); + + if (compressionType == MAX_INT) + { + compressionType = compressionTypeIn; + } + else if (compressionType < 0) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (compressionType == 1) + compressionType = 2; + + if ((compressionType > 0) && !(compress::CompressInterface::isCompressionAvail(compressionType))) + { + rc = 1; + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; +#ifdef SKIP_IDB_COMPRESSION + Message::Args args; + thd->get_stmt_da()->set_overwrite_status(true); + args.add("The compression type"); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_ENTERPRISE_ONLY, args)).c_str()); +#else + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); +#endif + return rc; + } + + try + { + autoIncre = parseAutoincrementColumnComment(comment, startValue); + + if (autoIncre) + { + // Check whether there is a column with autoincrement already + if ((isAnyAutoincreCol) && + !(boost::iequals(autoiColName, createTable->fTableDef->fColumns[i]->fName))) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + else + { + isAnyAutoincreCol = true; + autoiColName = createTable->fTableDef->fColumns[i]->fName; + matchedCol = true; + } + + // Check whether the column has default value. If there is, error out + if (createTable->fTableDef->fColumns[i]->fDefaultValue) + { + if (!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "Autoincrement column cannot have a default value."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + + if (!autoIncre && isAnyAutoincreCol && + (boost::iequals(autoiColName, createTable->fTableDef->fColumns[i]->fName))) + { + autoIncre = true; + matchedCol = true; + startValue = nextvalue; + } + + if (startValue <= 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (autoIncre) + { + if (!validateAutoincrementDatatype(createTable->fTableDef->fColumns[i]->fType->fType)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + + if (!validateNextValue(createTable->fTableDef->fColumns[i]->fType->fType, startValue)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // hdfs + if ((compressionType == 0) && (useHdfs)) + { + compressionType = 2; + string errmsg("The table is created with Columnstore compression type 2 under HDFS."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); + } + + (createTable->fTableDef->fColumns[i]->fType)->fCompressiontype = compressionType; + + if (autoIncre) + (createTable->fTableDef->fColumns[i]->fType)->fAutoincrement = "y"; + else + (createTable->fTableDef->fColumns[i]->fType)->fAutoincrement = "n"; + + (createTable->fTableDef->fColumns[i]->fType)->fNextvalue = startValue; + } + + if (isAnyAutoincreCol && !matchedCol) //@Bug 3555 error out on invalid column + { + rc = 1; + Message::Args args; + thd->get_stmt_da()->set_overwrite_status(true); + args.add(autoiColName); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL, args)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + else if (typeid(stmt) == typeid(AlterTableStatement)) + { + AlterTableStatement* alterTable = dynamic_cast(&stmt); + if (lower_case_table_names) + { + algorithm::to_lower(alterTable->fTableName->fSchema); + algorithm::to_lower(alterTable->fTableName->fName); + } + + alterTable->fTimeZone.assign(thd->variables.time_zone->get_name()->ptr()); + + if (schema.length() == 0) + { + schema = alterTable->fTableName->fSchema; + + if (schema.length() == 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "No database selected."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + + ddlpackage::AlterTableActionList actionList = alterTable->fActions; + + if (actionList.size() > 1) //@bug 3753 we don't support multiple actions in alter table statement + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf( + ER_CHECK_NOT_IMPLEMENTED, + "Multiple actions in alter table statement is currently not supported by Columnstore."); + + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + for (unsigned i = 0; i < actionList.size(); i++) + { + if (ddlpackage::AtaAddColumn* addColumnPtr = dynamic_cast(actionList[i])) + { + // check varbinary data type + if ((addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_VARBINARY) && !isVarbinaryAllowed) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + "Varbinary is currently not supported by Columnstore."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_VARBINARY) && + ((addColumnPtr->fColumnDef->fType->fLength > 8000) || + (addColumnPtr->fColumnDef->fType->fLength < 8))) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary length has to be between 8 and 8000."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + uint64_t startValue = 1; + bool autoIncre = false; + + if ((addColumnPtr->fColumnDef->fConstraints.size() > 0) || addColumnPtr->fColumnDef->fDefaultValue) + { + // support default value and NOT NULL constraint + for (uint32_t j = 0; j < addColumnPtr->fColumnDef->fConstraints.size(); j++) + { + if (addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType != DDL_NOT_NULL) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // if not null constraint, user has to provide a default value + if ((addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && + (!addColumnPtr->fColumnDef->fDefaultValue) && + (addColumnPtr->fColumnDef->fType->fType != ddlpackage::DDL_TIMESTAMP)) + { + // do select count(*) from the table to check whether there are existing rows. if there is, + // error out. + bool anyRow = false; + + try + { + anyRow = anyRowInTable(alterTable->fTableName->fSchema, alterTable->fTableName->fName, + sessionID); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + catch (...) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "Unknown exception caught when checking any rows in the table."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (anyRow) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + "Table is not empty. New column has to have a default value if NOT NULL required."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + else if ((addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && + (addColumnPtr->fColumnDef->fDefaultValue)) + { + if (addColumnPtr->fColumnDef->fDefaultValue->fValue.length() == + 0) // empty string is NULL in infinidb + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "New column has to have a default value if NOT NULL required."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } + + if (addColumnPtr->fColumnDef->fDefaultValue) + { + if ((!addColumnPtr->fColumnDef->fDefaultValue->fNull) && + (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_VARBINARY)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary column cannot have default value."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // validate the default value, if out of range, just error out + CalpontSystemCatalog::ColDataType dataType; + dataType = convertDataType(addColumnPtr->fColumnDef->fType->fType); + CalpontSystemCatalog::ColType colType; + colType.colDataType = dataType; + colType.colWidth = addColumnPtr->fColumnDef->fType->fLength; + colType.precision = addColumnPtr->fColumnDef->fType->fPrecision; + colType.scale = addColumnPtr->fColumnDef->fType->fScale; + boost::any convertedVal; + bool pushWarning = false; + + try + { + convertedVal = colType.convertColumnData( + addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, + thd->variables.time_zone->get_name()->ptr(), false, false, false); + } + catch (std::exception&) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The default value is out of range for the specified data type."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (pushWarning) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The default value is out of range for the specified data type."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnPtr->fColumnDef->fDefaultValue->fValue == "0") + { + addColumnPtr->fColumnDef->fDefaultValue->fValue = "0000-00-00 00:00:00"; + } + } + } + + // For TIMESTAMP, if no constraint is given, default to NOT NULL + if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnPtr->fColumnDef->fConstraints.empty()) + { + addColumnPtr->fColumnDef->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); + } + + // If no default value exists for TIMESTAMP, we apply + // automatic TIMESTAMP properties. + // TODO: If no default value exists but the constraint is NULL, + // default value should be set to NULL. But this is currently + // not supported since columnstore does not track whether user + // specified a NULL or not + if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && + !addColumnPtr->fColumnDef->fDefaultValue) + { + // Query calpontsys.syscolumn to see + // if a timestamp column already exists in this table + if (!anyTimestampColumn(alterTable->fTableName->fSchema, alterTable->fTableName->fName, + sessionID)) + { + addColumnPtr->fColumnDef->fDefaultValue = + new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); + } + else + { + addColumnPtr->fColumnDef->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); + } + } + + // Handle compression type + string comment = addColumnPtr->fColumnDef->fComment; + int compressionType = compressionTypeIn; + + if (comment.length() > 0) + { + //@Bug 3782 This is for synchronization after calonlinealter to use + algorithm::to_upper(comment); + regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); + + if (regex_search(comment, pat)) + { + return 0; + } + + compressionType = parseCompressionComment(comment); + + if (compressionType == MAX_INT) + { + compressionType = compressionTypeIn; + } + else if (compressionType < 0) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((compressionType > 0) && !(compress::CompressInterface::isCompressionAvail(compressionType))) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((compressionType == 0) && (useHdfs)) + { + compressionType = 2; + string errmsg("The column is created with Columnstore compression type 2 under HDFS."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); + } + + try + { + autoIncre = parseAutoincrementColumnComment(comment, startValue); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (autoIncre) + { + // Check if the table already has autoincrement column + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName tableName; + tableName.schema = alterTable->fTableName->fSchema; + tableName.table = alterTable->fTableName->fName; + CalpontSystemCatalog::TableInfo tblInfo; + + try + { + tblInfo = csc->tableInfo(tableName); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (tblInfo.tablewithautoincr == 1) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (!validateAutoincrementDatatype(addColumnPtr->fColumnDef->fType->fType)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (!validateNextValue(addColumnPtr->fColumnDef->fType->fType, startValue)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (startValue <= 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } + + addColumnPtr->fColumnDef->fType->fCompressiontype = compressionType; + + if (autoIncre) + addColumnPtr->fColumnDef->fType->fAutoincrement = "y"; + else + addColumnPtr->fColumnDef->fType->fAutoincrement = "n"; + + addColumnPtr->fColumnDef->fType->fNextvalue = startValue; + } + else if (dynamic_cast(actionList[i])) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + /* else if ( dynamic_cast (actionList[i]) ) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); ci->alterTableState = + cal_connection_info::NOT_ALTER; ci->isAlter = false; return rc; + } + else if ( dynamic_cast (actionList[i]) ) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); ci->alterTableState = + cal_connection_info::NOT_ALTER; ci->isAlter = false; return rc; + } + */ + else if (ddlpackage::AtaAddColumns* addColumnsPtr = dynamic_cast(actionList[i])) + { + if ((addColumnsPtr->fColumns).size() > 1) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_CHECK_NOT_IMPLEMENTED, + "Multiple actions in alter table statement is currently not supported by Columnstore."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // check varbinary data type + if ((addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_VARBINARY) && !isVarbinaryAllowed) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + "Varbinary is currently not supported by Columnstore."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_VARBINARY) && + ((addColumnsPtr->fColumns[0]->fType->fLength > 8000) || + (addColumnsPtr->fColumns[0]->fType->fLength < 8))) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary length has to be between 8 and 8000."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + uint64_t startValue = 1; + bool autoIncre = false; + + if ((addColumnsPtr->fColumns[0]->fConstraints.size() > 0) || + addColumnsPtr->fColumns[0]->fDefaultValue) + { + //@Bug 5274. support default value and NOT NULL constraint + for (uint32_t j = 0; j < addColumnsPtr->fColumns[0]->fConstraints.size(); j++) + { + if (addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType != DDL_NOT_NULL) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // if not null constraint, user has to provide a default value + if ((addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && + (!addColumnsPtr->fColumns[0]->fDefaultValue) && + (addColumnsPtr->fColumns[0]->fType->fType != ddlpackage::DDL_TIMESTAMP)) + { + // do select count(*) from the table to check whether there are existing rows. if there is, + // error out. + bool anyRow = false; + + try + { + anyRow = anyRowInTable(alterTable->fTableName->fSchema, alterTable->fTableName->fName, + sessionID); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + catch (...) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "Unknown exception caught when checking any rows in the table."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (anyRow) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + "Table is not empty. New column has to have a default value if NOT NULL required."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + else if ((addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && + (addColumnsPtr->fColumns[0]->fDefaultValue)) + { + if (addColumnsPtr->fColumns[0]->fDefaultValue->fValue.length() == + 0) // empty string is NULL in infinidb + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "New column has to have a default value if NOT NULL required."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } + + if (addColumnsPtr->fColumns[0]->fDefaultValue) + { + if ((!addColumnsPtr->fColumns[0]->fDefaultValue->fNull) && + (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_VARBINARY)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Varbinary column cannot have default value."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + // validate the default value, if out of range, just error out + CalpontSystemCatalog::ColDataType dataType; + dataType = convertDataType(addColumnsPtr->fColumns[0]->fType->fType); + CalpontSystemCatalog::ColType colType; + colType.colDataType = dataType; + colType.colWidth = addColumnsPtr->fColumns[0]->fType->fLength; + colType.precision = addColumnsPtr->fColumns[0]->fType->fPrecision; + colType.scale = addColumnsPtr->fColumns[0]->fType->fScale; + boost::any convertedVal; + bool pushWarning = false; + + try + { + convertedVal = colType.convertColumnData( + addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, + thd->variables.time_zone->get_name()->ptr(), false, false, false); + } + catch (std::exception&) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The default value is out of range for the specified data type."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (pushWarning) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The default value is out of range for the specified data type."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnsPtr->fColumns[0]->fDefaultValue->fValue == "0") + { + addColumnsPtr->fColumns[0]->fDefaultValue->fValue = "0000-00-00 00:00:00"; + } + } + } + + // For TIMESTAMP, if no constraint is given, default to NOT NULL + if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnsPtr->fColumns[0]->fConstraints.empty()) + { + addColumnsPtr->fColumns[0]->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); + } + + // If no default value exists for TIMESTAMP, we apply + // automatic TIMESTAMP properties. + // TODO: If no default value exists but the constraint is NULL, + // default value should be set to NULL. But this is currently + // not supported since columnstore does not track whether user + // specified a NULL or not + if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && + !addColumnsPtr->fColumns[0]->fDefaultValue) + { + // Query calpontsys.syscolumn to see + // if a timestamp column already exists in this table + if (!anyTimestampColumn(alterTable->fTableName->fSchema, alterTable->fTableName->fName, + sessionID)) + { + addColumnsPtr->fColumns[0]->fDefaultValue = + new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); + } + else + { + addColumnsPtr->fColumns[0]->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); + } + } + + // Handle compression type + string comment = addColumnsPtr->fColumns[0]->fComment; + int compressionType = compressionTypeIn; + + if (comment.length() > 0) + { + compressionType = parseCompressionComment(comment); + + if (compressionType == MAX_INT) + { + compressionType = compressionTypeIn; + } + else if (compressionType < 0) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((compressionType > 0) && !(compress::CompressInterface::isCompressionAvail(compressionType))) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((compressionType == 0) && (useHdfs)) + { + compressionType = 2; + string errmsg("The column is created with Columnstore compression type 2 under HDFS."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); + } + + try + { + autoIncre = parseAutoincrementColumnComment(comment, startValue); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (autoIncre) + { + // Check if the table already has autoincrement column + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName tableName; + tableName.schema = alterTable->fTableName->fSchema; + tableName.table = alterTable->fTableName->fName; + CalpontSystemCatalog::TableInfo tblInfo; + + try + { + tblInfo = csc->tableInfo(tableName); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (tblInfo.tablewithautoincr == 1) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (!validateAutoincrementDatatype(addColumnsPtr->fColumns[0]->fType->fType)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (!validateNextValue(addColumnsPtr->fColumns[0]->fType->fType, startValue)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (startValue <= 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, (IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)).c_str()); + return rc; + } + } + } + + addColumnsPtr->fColumns[0]->fType->fCompressiontype = compressionType; + + if (autoIncre) + addColumnsPtr->fColumns[0]->fType->fAutoincrement = "y"; + else + addColumnsPtr->fColumns[0]->fType->fAutoincrement = "n"; + + addColumnsPtr->fColumns[0]->fType->fNextvalue = startValue; + } + else if (ddlpackage::AtaRenameColumn* renameColumnsPtr = + dynamic_cast(actionList[i])) + { + uint64_t startValue = 1; + bool autoIncre = false; + //@Bug 3746 Handle compression type + string comment = renameColumnsPtr->fComment; + int compressionType = compressionTypeIn; + + if (comment.length() > 0) + { + compressionType = parseCompressionComment(comment); + + if (compressionType == MAX_INT) + { + compressionType = compressionTypeIn; + } + else if (compressionType < 0) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((compressionType > 0) && !(compress::CompressInterface::isCompressionAvail(compressionType))) + { + rc = 1; + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if ((compressionType == 0) && (useHdfs)) + { + compressionType = 2; + string errmsg("The column is created with Columnstore compression type 2 under HDFS."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); + } + } + + // Handle autoincrement + if (comment.length() > 0) + { + try + { + autoIncre = parseAutoincrementColumnComment(comment, startValue); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (autoIncre) + { + // Check if the table already has autoincrement column + CalpontSystemCatalog::TableName tableName; + tableName.schema = alterTable->fTableName->fSchema; + tableName.table = alterTable->fTableName->fName; + // CalpontSystemCatalog::TableInfo tblInfo = csc->tableInfo(tableName); + + //@Bug 5444. rename column doen't need to check this. + /* if (tblInfo.tablewithautoincr == 1) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } */ + + if (!validateAutoincrementDatatype(renameColumnsPtr->fNewType->fType)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_AUTOINCREMENT_TYPE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (!validateNextValue(renameColumnsPtr->fNewType->fType, startValue)) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_INVALID_STARTVALUE)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } + + if (autoIncre) + renameColumnsPtr->fNewType->fAutoincrement = "y"; + else + renameColumnsPtr->fNewType->fAutoincrement = "n"; + + renameColumnsPtr->fNewType->fNextvalue = startValue; + renameColumnsPtr->fNewType->fCompressiontype = compressionType; + + if (renameColumnsPtr->fConstraints.size() > 0) + { + for (uint32_t j = 0; j < renameColumnsPtr->fConstraints.size(); j++) + { + if (renameColumnsPtr->fConstraints[j]->fConstraintType == DDL_NOT_NULL) + { + // If NOT NULL constraint is added, check whether the existing rows has null vlues. If there + // is, error out the query. + bool anyNullVal = false; + + try + { + anyNullVal = + anyNullInTheColumn(thd, alterTable->fTableName->fSchema, alterTable->fTableName->fName, + renameColumnsPtr->fName, sessionID); + } + catch (runtime_error& ex) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + catch (...) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf( + ER_INTERNAL_ERROR, + "Unknown exception caught when checking any existing null values in the column."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + if (anyNullVal) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + "The existing rows in this column has null value already."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + else + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + (IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + } + } } else { - //@Bug 1888,1885. update error message - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; } + } + } + //@Bug 5923 error out on unsupported statements. + else if ((typeid(stmt) != typeid(DropTableStatement)) && (typeid(stmt) != typeid(TruncTableStatement)) && + (typeid(stmt) != typeid(MarkPartitionStatement)) && + (typeid(stmt) != typeid(RestorePartitionStatement)) && + (typeid(stmt) != typeid(DropPartitionStatement))) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, + (IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SYNTAX)).c_str()); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + //@Bug 4387 + scoped_ptr dbrmp(new DBRM()); + int rc = dbrmp->isReadWrite(); + + if (rc != 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Cannot execute the statement. DBRM is read only!"); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + return rc; + } + + stmt.fSessionID = sessionID; + stmt.fSql = ddlStatement; + stmt.fOwner = schema; + stmt.fTableWithAutoi = isAnyAutoincreCol; + ByteStream bytestream; + bytestream << stmt.fSessionID; + stmt.serialize(bytestream); + MessageQueueClient mq("DDLProc"); + ByteStream::byte b = 0; + + try + { + mq.write(bytestream); + bytestream = mq.read(); + + if (bytestream.length() == 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + } + else + { + bytestream >> b; + bytestream >> emsg; + rc = b; + } + } + catch (runtime_error& e) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + } + catch (...) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown error caught"); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + } + + if (b == ddlpackageprocessor::DDLPackageProcessor::DROP_TABLE_NOT_IN_CATALOG_ERROR) + { + return ER_NO_SUCH_TABLE_IN_ENGINE; + } + + if ((b != 0) && (b != ddlpackageprocessor::DDLPackageProcessor::WARNING)) + { + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + } + + if (b == ddlpackageprocessor::DDLPackageProcessor::WARNING) + { + rc = 0; + string errmsg( + "Error occured during file deletion. Restart DDLProc or use command tool ddlcleanup to clean up. "); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, errmsg.c_str()); } return rc; + } + else + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + //@Bug 3602. Error message for MySql syntax for autoincrement + algorithm::to_upper(ddlStatement); + + if (ddlStatement.find("AUTO_INCREMENT") != string::npos) + { + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + "Use of the MySQL auto_increment syntax is not supported in Columnstore. If " + "you wish to create an auto increment column in Columnstore, please consult " + "the Columnstore SQL Syntax Guide for the correct usage."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + } + else + { + //@Bug 1888,1885. update error message + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + "The syntax or the data type(s) is not supported by Columnstore. Please check " + "the Columnstore syntax guide for supported syntax or data types."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; + } + } + + return rc; } -} +} // namespace // // get_field_default_value: Returns the default value as a string value @@ -2125,594 +2218,588 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl // been created. // -static bool get_field_default_value(THD *thd, Field *field, String *def_value, - bool quoted) +static bool get_field_default_value(THD* thd, Field* field, String* def_value, bool quoted) { - bool has_default; - enum enum_field_types field_type= field->type(); + bool has_default; + enum enum_field_types field_type = field->type(); - has_default= (field->default_value || - (!(field->flags & NO_DEFAULT_VALUE_FLAG) && - field->unireg_check != Field::NEXT_NUMBER)); + has_default = (field->default_value || + (!(field->flags & NO_DEFAULT_VALUE_FLAG) && field->unireg_check != Field::NEXT_NUMBER)); - def_value->length(0); - if (has_default) + def_value->length(0); + if (has_default) + { + StringBuffer str(field->charset()); + if (field->default_value) { - StringBuffer str(field->charset()); - if (field->default_value) - { - field->default_value->print(&str); - if (field->default_value->expr->need_parentheses_in_default()) - { - def_value->set_charset(&my_charset_utf8mb4_general_ci); - def_value->append('('); - def_value->append(str); - def_value->append(')'); - } - else - def_value->append(str); - } - else if (!field->is_null()) - { // Not null by default - if (field_type == MYSQL_TYPE_BIT) - { - str.qs_append('b'); - str.qs_append('\''); - str.qs_append(field->val_int(), 2); - str.qs_append('\''); - quoted= 0; - } - else - { - field->val_str(&str); - if (!field->str_needs_quotes()) - quoted= 0; - } - if (str.length()) - { - StringBuffer def_val; - uint dummy_errors; - /* convert to system_charset_info == utf8 */ - def_val.copy(str.ptr(), str.length(), field->charset(), - system_charset_info, &dummy_errors); - if (quoted) - append_unescaped(def_value, def_val.ptr(), def_val.length()); - else - def_value->append(def_val); - } - else if (quoted) - def_value->set(STRING_WITH_LEN("''"), system_charset_info); - } - else if (field->maybe_null() && quoted) - def_value->set(STRING_WITH_LEN("NULL"), system_charset_info); // Null as default - else - return 0; - + field->default_value->print(&str); + if (field->default_value->expr->need_parentheses_in_default()) + { + def_value->set_charset(&my_charset_utf8mb4_general_ci); + def_value->append('('); + def_value->append(str); + def_value->append(')'); + } + else + def_value->append(str); } - return has_default; + else if (!field->is_null()) + { // Not null by default + if (field_type == MYSQL_TYPE_BIT) + { + str.qs_append('b'); + str.qs_append('\''); + str.qs_append(field->val_int(), 2); + str.qs_append('\''); + quoted = 0; + } + else + { + field->val_str(&str); + if (!field->str_needs_quotes()) + quoted = 0; + } + if (str.length()) + { + StringBuffer def_val; + uint dummy_errors; + /* convert to system_charset_info == utf8 */ + def_val.copy(str.ptr(), str.length(), field->charset(), system_charset_info, &dummy_errors); + if (quoted) + append_unescaped(def_value, def_val.ptr(), def_val.length()); + else + def_value->append(def_val); + } + else if (quoted) + def_value->set(STRING_WITH_LEN("''"), system_charset_info); + } + else if (field->maybe_null() && quoted) + def_value->set(STRING_WITH_LEN("NULL"), system_charset_info); // Null as default + else + return 0; + } + return has_default; } /* - Utility function search for ZEROFILL + Utility function search for ZEROFILL */ -bool hasZerofillDecimal(TABLE *table_arg) +bool hasZerofillDecimal(TABLE* table_arg) { - for (Field **field= table_arg->field; *field; field++) - { - if (((*field)->flags & ZEROFILL_FLAG) - && typeid (**field) == typeid(Field_new_decimal)) - return true; - } + for (Field** field = table_arg->field; *field; field++) + { + if (((*field)->flags & ZEROFILL_FLAG) && typeid(**field) == typeid(Field_new_decimal)) + return true; + } - return false; + return false; } -int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info, cal_connection_info& ci) +int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info, + cal_connection_info& ci) { #ifdef MCS_DEBUG - cout << "ha_mcs_impl_create_: " << name << endl; + cout << "ha_mcs_impl_create_: " << name << endl; #endif - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd)) + return 0; - char* query = thd->query(); + char* query = thd->query(); - if (!query) + if (!query) + { + setError(thd, ER_INTERNAL_ERROR, "Attempt to create table, but query is NULL"); + return 1; + } + + string stmt(query); + stmt += ";"; + algorithm::to_upper(stmt); + + string db, tbl; + db = table_arg->s->db.str; + tbl = table_arg->s->table_name.str; + + string tablecomment; + bool isAnyAutoincreCol = false; + std::string columnName(""); + uint64_t startValue = 1; + + if (table_arg->s->comment.length > 0) + { + tablecomment = table_arg->s->comment.str; + + try { - setError(thd, ER_INTERNAL_ERROR, "Attempt to create table, but query is NULL"); - return 1; + isAnyAutoincreCol = parseAutoincrementTableComment(tablecomment, startValue, columnName); + } + catch (runtime_error& ex) + { + setError(thd, ER_INTERNAL_ERROR, ex.what()); + return 1; } - string stmt(query); - stmt += ";"; - algorithm::to_upper(stmt); + algorithm::to_upper(tablecomment); + } - string db, tbl; - db = table_arg->s->db.str; - tbl = table_arg->s->table_name.str; + //@Bug 2553 Add a parenthesis around option to group them together + string alterpatstr( + "ALTER[[:space:]]+TABLE[[:space:]]+.*[[:space:]]+(((ADD)|(DROP)|(CHANGE)|(ALTER)|(COMMENT))[[:space:]]+" + "|(COMMENT=))"); + string createpatstr("CREATE[[:space:]]+TABLE[[:space:]]"); + bool schemaSyncOnly = false; + bool isCreate = true; - string tablecomment; - bool isAnyAutoincreCol = false; - std::string columnName(""); - uint64_t startValue = 1; + regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); - if (table_arg->s->comment.length > 0 ) + if (regex_search(tablecomment, pat)) + { + schemaSyncOnly = true; + pat = createpatstr; + + if (!regex_search(stmt, pat)) { - tablecomment = table_arg->s->comment.str; - - try - { - isAnyAutoincreCol = parseAutoincrementTableComment(tablecomment, startValue, columnName); - } - catch (runtime_error& ex) - { - setError(thd, ER_INTERNAL_ERROR, ex.what()); - return 1; - } - - algorithm::to_upper(tablecomment); + isCreate = false; } - //@Bug 2553 Add a parenthesis around option to group them together - string alterpatstr("ALTER[[:space:]]+TABLE[[:space:]]+.*[[:space:]]+(((ADD)|(DROP)|(CHANGE)|(ALTER)|(COMMENT))[[:space:]]+|(COMMENT=))"); - string createpatstr("CREATE[[:space:]]+TABLE[[:space:]]"); - bool schemaSyncOnly = false; - bool isCreate = true; - - regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); - - if (regex_search(tablecomment, pat)) + if (isCreate) { - schemaSyncOnly = true; - pat = createpatstr; - - if (!regex_search(stmt, pat)) - { - isCreate = false; - } - - if (isCreate) - { #ifdef MCS_DEBUG - cout << "ha_mcs_impl_create_: SCHEMA SYNC ONLY found, returning" << endl; + cout << "ha_mcs_impl_create_: SCHEMA SYNC ONLY found, returning" << endl; #endif - return 0; - } + return 0; } - else + } + else + { + if (db == "calpontsys") { - if (db == "calpontsys") - { - setError(thd, ER_INTERNAL_ERROR, "Calpont system tables can only be created with 'SCHEMA SYNC ONLY'"); - return 1; - } - else if ( db == "infinidb_vtable") //@bug 3540. table created in infinidb_vtable schema could be dropped when select statement happen to have same tablename. - { - setError(thd, ER_INTERNAL_ERROR, "Table creation is not allowed in infinidb_vtable schema."); - return 1; - } + setError(thd, ER_INTERNAL_ERROR, "Calpont system tables can only be created with 'SCHEMA SYNC ONLY'"); + return 1; } - - pat = alterpatstr; - - if (regex_search(stmt, pat)) + else if (db == "infinidb_vtable") //@bug 3540. table created in infinidb_vtable schema could be dropped + //when select statement happen to have same tablename. { - ci.isAlter = true; - ci.alterTableState = cal_connection_info::ALTER_FIRST_RENAME; + setError(thd, ER_INTERNAL_ERROR, "Table creation is not allowed in infinidb_vtable schema."); + return 1; + } + } + + pat = alterpatstr; + + if (regex_search(stmt, pat)) + { + ci.isAlter = true; + ci.alterTableState = cal_connection_info::ALTER_FIRST_RENAME; #ifdef MCS_DEBUG - cout << "ha_mcs_impl_create_: now in state ALTER_FIRST_RENAME" << endl; + cout << "ha_mcs_impl_create_: now in state ALTER_FIRST_RENAME" << endl; #endif - } + } - string emsg; - stmt = thd->query(); - stmt += ";"; - int rc = 0; + string emsg; + stmt = thd->query(); + stmt += ";"; + int rc = 0; - // Don't do the DDL (only for create table if this is SSO. Should only get here on ATAC w/SSO. - if ( schemaSyncOnly && isCreate) - return rc; + // Don't do the DDL (only for create table if this is SSO. Should only get here on ATAC w/SSO. + if (schemaSyncOnly && isCreate) + return rc; - //@bug 5660. Error out REAL DDL/DML on slave node. - // When the statement gets here, it's NOT SSO or RESTRICT - if (ci.isSlaveNode) - { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); - setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return 1; - } + //@bug 5660. Error out REAL DDL/DML on slave node. + // When the statement gets here, it's NOT SSO or RESTRICT + if (ci.isSlaveNode) + { + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); + setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return 1; + } - // Send notice if primary key specified that it is not supported - if (table_arg->key_info && table_arg->key_info->name.length && string(table_arg->key_info->name.str) == "PRIMARY") - { - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "INDEXES"); - } + // Send notice if primary key specified that it is not supported + if (table_arg->key_info && table_arg->key_info->name.length && + string(table_arg->key_info->name.str) == "PRIMARY") + { + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "INDEXES"); + } - int compressiontype = get_compression_type(thd); + int compressiontype = get_compression_type(thd); + // MCOL-4685: + // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). + if (compressiontype == 1 || compressiontype == 0) + compressiontype = 2; + + // string tablecomment; + if (table_arg->s->comment.length > 0) + { + tablecomment = table_arg->s->comment.str; + compressiontype = parseCompressionComment(tablecomment); + } + + if (compressiontype == MAX_INT) + { + compressiontype = get_compression_type(thd); // MCOL-4685: // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). - if (compressiontype == 1 || compressiontype == 0) - compressiontype = 2; + if (compressiontype == 0) + compressiontype = 2; + } + else if (compressiontype < 0) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE); + setError(thd, ER_INTERNAL_ERROR, emsg); + ci.alterTableState = cal_connection_info::NOT_ALTER; + ci.isAlter = false; + return 1; + } - //string tablecomment; - if (table_arg->s->comment.length > 0 ) + // hdfs + if ((compressiontype == 0) && (useHdfs)) + { + compressiontype = 2; + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, + "The table is created with Columnstore compression type 2 under HDFS."); + } + + if (compressiontype == 1) + compressiontype = 2; + + if ((compressiontype > 0) && !(compress::CompressInterface::isCompressionAvail(compressiontype))) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE); + setError(thd, ER_INTERNAL_ERROR, emsg); + ci.alterTableState = cal_connection_info::NOT_ALTER; + ci.isAlter = false; + return 1; + } + + // Check if this is one of + // * "CREATE TABLE ... LIKE " + // * "ALTER TABLE ... ENGINE=Columnstore" + // * "CREATE TABLE ... AS ..." + // If so generate a full create table statement using the properties of + // the source table. Note that source table has to be a columnstore table and + // we only check for currently supported options. + // + + if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && thd->lex->used_tables) || + (thd->lex->sql_command == SQLCOM_ALTER_TABLE && create_info->used_fields & HA_CREATE_USED_ENGINE) || + (thd->lex->create_info.like())) + { + TABLE_SHARE* share = table_arg->s; + MY_BITMAP* old_map; // To save the read_set + char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH]; + String datatype, def_value; + ostringstream oss; + string tbl_name = "`" + string(share->db.str) + "`.`" + string(share->table_name.str) + "`"; + + // Save the current read_set map and mark it for read + old_map = tmp_use_all_columns(table_arg, &table_arg->read_set); + + oss << "CREATE TABLE " << tbl_name << " ("; + + restore_record(table_arg, s->default_values); + for (Field** field = table_arg->field; *field; field++) { - tablecomment = table_arg->s->comment.str; - compressiontype = parseCompressionComment( tablecomment ); - } + uint flags = (*field)->flags; + datatype.set(datatype_buf, sizeof(datatype_buf), system_charset_info); + (*field)->sql_type(datatype); + if (field != table_arg->field) + oss << ", "; + oss << "`" << (*field)->field_name.str << "` " << datatype.ptr(); - if (compressiontype == MAX_INT) - { - compressiontype = get_compression_type(thd); - // MCOL-4685: - // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). - if (compressiontype == 0) - compressiontype = 2; - } - else if ( compressiontype < 0 ) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE); - setError(thd, ER_INTERNAL_ERROR, emsg); - ci.alterTableState = cal_connection_info::NOT_ALTER; - ci.isAlter = false; - return 1; - } - - //hdfs - if ((compressiontype == 0) && (useHdfs)) - { - compressiontype = 2; - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, "The table is created with Columnstore compression type 2 under HDFS."); - } - - if (compressiontype == 1) compressiontype = 2; - - if ((compressiontype > 0) && - !(compress::CompressInterface::isCompressionAvail(compressiontype))) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE); - setError(thd, ER_INTERNAL_ERROR, emsg); - ci.alterTableState = cal_connection_info::NOT_ALTER; - ci.isAlter = false; - return 1; - } - - // Check if this is one of - // * "CREATE TABLE ... LIKE " - // * "ALTER TABLE ... ENGINE=Columnstore" - // * "CREATE TABLE ... AS ..." - // If so generate a full create table statement using the properties of - // the source table. Note that source table has to be a columnstore table and - // we only check for currently supported options. - // - - if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && thd->lex->used_tables) || - (thd->lex->sql_command == SQLCOM_ALTER_TABLE && create_info->used_fields & HA_CREATE_USED_ENGINE) || - (thd->lex->create_info.like())) - { - TABLE_SHARE *share = table_arg->s; - MY_BITMAP *old_map; // To save the read_set - char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH]; - String datatype, def_value; - ostringstream oss; - string tbl_name = "`" + string(share->db.str) + "`.`" + string(share->table_name.str) + "`"; - - // Save the current read_set map and mark it for read - old_map= tmp_use_all_columns(table_arg, &table_arg->read_set); - - oss << "CREATE TABLE " << tbl_name << " ("; - - restore_record(table_arg, s->default_values); - for (Field **field= table_arg->field; *field; field++) + if ((*field)->has_charset()) + { + const CHARSET_INFO* field_cs = (*field)->charset(); + if (field_cs && (!share->table_charset || field_cs->number != share->table_charset->number)) { - uint flags = (*field)->flags; - datatype.set(datatype_buf, sizeof(datatype_buf), system_charset_info); - (*field)->sql_type(datatype); - if (field != table_arg->field) - oss << ", "; - oss << "`" << (*field)->field_name.str << "` " << datatype.ptr(); + oss << " CHARACTER SET " << field_cs->cs_name.str; + } + } + if (flags & NOT_NULL_FLAG) + oss << " NOT NULL"; - if ((*field)->has_charset()) + def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info); + if (get_field_default_value(thd, *field, &def_value, true)) + { + oss << " DEFAULT " << def_value.c_ptr(); + } + + if ((*field)->comment.length) + { + String comment; + append_unescaped(&comment, (*field)->comment.str, (*field)->comment.length); + oss << " COMMENT "; + oss << comment.c_ptr(); + } + } + // End the list of columns + oss << ") ENGINE=columnstore "; + + // Process table level options + + /* TODO: uncomment when we support AUTO_INCREMENT + if (create_info->auto_increment_value > 1) { - const CHARSET_INFO* field_cs = (*field)->charset(); - if (field_cs && - (!share->table_charset || - field_cs->number != share->table_charset->number)) - { - oss << " CHARACTER SET " << field_cs->cs_name.str; - } + oss << " AUTO_INCREMENT=" << create_info->auto_increment_value; } + */ - if (flags & NOT_NULL_FLAG) - oss << " NOT NULL"; - - def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info); - if (get_field_default_value(thd, *field, &def_value, true)) - { - oss << " DEFAULT " << def_value.c_ptr(); - } - - if ((*field)->comment.length) - { - String comment; - append_unescaped(&comment, (*field)->comment.str, (*field)->comment.length); - oss << " COMMENT "; - oss << comment.c_ptr(); - } - - } - // End the list of columns - oss << ") ENGINE=columnstore "; - - // Process table level options - - /* TODO: uncomment when we support AUTO_INCREMENT - if (create_info->auto_increment_value > 1) - { - oss << " AUTO_INCREMENT=" << create_info->auto_increment_value; - } - */ - - if (create_info->auto_increment_value > 1) - { - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "AUTO INCREMENT"); - } - - - if (share->table_charset) - { - oss << " DEFAULT CHARSET=" << share->table_charset->cs_name.str; - } - - // Process table level options such as MIN_ROWS, MAX_ROWS, COMMENT - - if (share->min_rows) - { - char buff[80]; - longlong10_to_str(share->min_rows, buff, 10); - oss << " MIN_ROWS=" << buff; - } - - if (share->max_rows) - { - char buff[80]; - longlong10_to_str(share->max_rows, buff, 10); - oss << " MAX_ROWS=" << buff; - } - - if (share->comment.length) - { - String comment; - append_unescaped(&comment, share->comment.str, share->comment.length); - oss << " COMMENT "; - oss << comment.c_ptr(); - } - - oss << ";"; - stmt = oss.str(); - - tmp_restore_column_map(&table_arg->read_set, old_map); + if (create_info->auto_increment_value > 1) + { + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "AUTO INCREMENT"); } - rc = ProcessDDLStatement(stmt, db, tbl, tid2sid(thd->thread_id), emsg, compressiontype, isAnyAutoincreCol, startValue, columnName, create_info->default_table_charset); - - if (rc != 0) + if (share->table_charset) { - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); - //Bug 1705 reset the flag if error occurs - ci.alterTableState = cal_connection_info::NOT_ALTER; - ci.isAlter = false; + oss << " DEFAULT CHARSET=" << share->table_charset->cs_name.str; + } + + // Process table level options such as MIN_ROWS, MAX_ROWS, COMMENT + + if (share->min_rows) + { + char buff[80]; + longlong10_to_str(share->min_rows, buff, 10); + oss << " MIN_ROWS=" << buff; + } + + if (share->max_rows) + { + char buff[80]; + longlong10_to_str(share->max_rows, buff, 10); + oss << " MAX_ROWS=" << buff; + } + + if (share->comment.length) + { + String comment; + append_unescaped(&comment, share->comment.str, share->comment.length); + oss << " COMMENT "; + oss << comment.c_ptr(); + } + + oss << ";"; + stmt = oss.str(); + + tmp_restore_column_map(&table_arg->read_set, old_map); + } + + rc = ProcessDDLStatement(stmt, db, tbl, tid2sid(thd->thread_id), emsg, compressiontype, isAnyAutoincreCol, + startValue, columnName, create_info->default_table_charset); + + if (rc != 0) + { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + // Bug 1705 reset the flag if error occurs + ci.alterTableState = cal_connection_info::NOT_ALTER; + ci.isAlter = false; #ifdef MCS_DEBUG - cout << "ha_mcs_impl_create_: ProcessDDL error, now in state NOT_ALTER" << endl; + cout << "ha_mcs_impl_create_: ProcessDDL error, now in state NOT_ALTER" << endl; #endif - } - else + } + else + { + if (hasZerofillDecimal(table_arg)) { - if (hasZerofillDecimal(table_arg)) - { - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - WARN_OPTION_IGNORED, "ZEROFILL is ignored in ColumnStore"); - } + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, WARN_OPTION_IGNORED, + "ZEROFILL is ignored in ColumnStore"); } + } - return rc; + return rc; } int ha_mcs_impl_delete_table_(const char* db, const char* name, cal_connection_info& ci) { #ifdef MCS_DEBUG - cout << "ha_mcs_impl_delete_table: " << db << name << endl; + cout << "ha_mcs_impl_delete_table: " << db << name << endl; #endif - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd)) + return 0; - char* query = thd->query(); + char* query = thd->query(); - if (!query) - { - setError(thd, ER_INTERNAL_ERROR, "Attempt to drop table, but query is NULL"); - return 1; - } + if (!query) + { + setError(thd, ER_INTERNAL_ERROR, "Attempt to drop table, but query is NULL"); + return 1; + } - std::string stmt(query); - algorithm::to_upper(stmt); - // @bug 4158 allow table name with 'restrict' in it (but not by itself) - std::string::size_type fpos; - fpos = stmt.rfind(" RESTRICT"); + std::string stmt(query); + algorithm::to_upper(stmt); + // @bug 4158 allow table name with 'restrict' in it (but not by itself) + std::string::size_type fpos; + fpos = stmt.rfind(" RESTRICT"); - if ((fpos != std::string::npos) && ((stmt.size() - fpos) == 9)) //last 9 chars of stmt are " RESTRICT" - { - return 0; - } + if ((fpos != std::string::npos) && ((stmt.size() - fpos) == 9)) // last 9 chars of stmt are " RESTRICT" + { + return 0; + } - //@bug 5660. Error out REAL DDL/DML on slave node. - // When the statement gets here, it's NOT SSO or RESTRICT - if (ci.isSlaveNode) - { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); - setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return 1; - } + //@bug 5660. Error out REAL DDL/DML on slave node. + // When the statement gets here, it's NOT SSO or RESTRICT + if (ci.isSlaveNode) + { + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); + setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return 1; + } - std::string emsg; + std::string emsg; - char decodedSchema[FN_REFLEN]; - char decodedTbl[FN_REFLEN]; - decode_file_path(name, decodedSchema, decodedTbl); - std::string schema(decodedSchema); - std::string tbl(decodedTbl); + char decodedSchema[FN_REFLEN]; + char decodedTbl[FN_REFLEN]; + decode_file_path(name, decodedSchema, decodedTbl); + std::string schema(decodedSchema); + std::string tbl(decodedTbl); - stmt.clear(); - stmt.assign("DROP TABLE `"); - stmt.append(decodedSchema); - stmt.append("`.`"); - stmt.append(decodedTbl); - stmt.append("`;"); + stmt.clear(); + stmt.assign("DROP TABLE `"); + stmt.append(decodedSchema); + stmt.append("`.`"); + stmt.append(decodedTbl); + stmt.append("`;"); - int rc = ProcessDDLStatement(stmt, schema, tbl, tid2sid(thd->thread_id), emsg); + int rc = ProcessDDLStatement(stmt, schema, tbl, tid2sid(thd->thread_id), emsg); - if (rc != 0 && rc != ER_NO_SUCH_TABLE_IN_ENGINE) - { - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); - } + if (rc != 0 && rc != ER_NO_SUCH_TABLE_IN_ENGINE) + { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + } - return rc; + return rc; } int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_info& ci) { - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd)) + return 0; - string emsg; + string emsg; - string tblFrom (from+2); - size_t pos = tblFrom.find("/"); - std::string dbFrom = tblFrom.substr(0, pos); - tblFrom = tblFrom.erase(0, pos + 1); + string tblFrom(from + 2); + size_t pos = tblFrom.find("/"); + std::string dbFrom = tblFrom.substr(0, pos); + tblFrom = tblFrom.erase(0, pos + 1); - string tblTo (to+2); - pos = tblTo.find("/"); - std::string dbTo = tblTo.substr(0, pos); - tblTo = tblTo.erase(0, pos + 1); + string tblTo(to + 2); + pos = tblTo.find("/"); + std::string dbTo = tblTo.substr(0, pos); + tblTo = tblTo.erase(0, pos + 1); - string stmt; + string stmt; - // This is a temporary table rename, we don't use the temporary table name - // so this is a NULL op - if (tblFrom.compare(0, 4, "#sql") == 0) - { - return 0; - } + // This is a temporary table rename, we don't use the temporary table name + // so this is a NULL op + if (tblFrom.compare(0, 4, "#sql") == 0) + { + return 0; + } - //@bug 5660. Error out REAL DDL/DML on slave node. - // When the statement gets here, it's NOT SSO or RESTRICT - if (ci.isSlaveNode) - { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return 1; - } + //@bug 5660. Error out REAL DDL/DML on slave node. + // When the statement gets here, it's NOT SSO or RESTRICT + if (ci.isSlaveNode) + { + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return 1; + } - stmt.assign("alter table `"); - stmt.append(dbFrom); - stmt.append("`.`"); - stmt.append(tblFrom); - stmt.append("` rename to `"); - stmt.append(dbTo); - stmt.append("`.`"); - stmt.append(tblTo); - stmt.append("`;"); + stmt.assign("alter table `"); + stmt.append(dbFrom); + stmt.append("`.`"); + stmt.append(tblFrom); + stmt.append("` rename to `"); + stmt.append(dbTo); + stmt.append("`.`"); + stmt.append(tblTo); + stmt.append("`;"); - string db; - if (thd->db.length) - db = thd->db.str; - else - db.assign(dbFrom); + string db; + if (thd->db.length) + db = thd->db.str; + else + db.assign(dbFrom); - int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); + int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); - if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + if (rc != 0) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); - return rc; + return rc; } - extern "C" { - #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long calonlinealter(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) + long long calonlinealter(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + string stmt(args->args[0], args->lengths[0]); + + string emsg; + THD* thd = current_thd; + string db(""); + + if (thd->db.length) + db = thd->db.str; + + // MCOL-4685: + // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). + int compressiontype = get_compression_type(thd); + + if (compressiontype == 1 || compressiontype == 0) + compressiontype = 2; + + if (compressiontype == MAX_INT) { - string stmt(args->args[0], args->lengths[0]); - - string emsg; - THD* thd = current_thd; - string db(""); - - if ( thd->db.length ) - db = thd->db.str; - - // MCOL-4685: - // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). - int compressiontype = get_compression_type(thd); - - if (compressiontype == 1 || compressiontype == 0) - compressiontype = 2; - - if (compressiontype == MAX_INT) - { - // MCOL-4685: remove the option to declare uncompressed columns (set - // infinidb_compression_type = 0). - compressiontype = get_compression_type(thd); - if (compressiontype == 0) - compressiontype = 2; - } - - if (compressiontype == 1) compressiontype = 2; - - int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype); - - if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); - - return rc; + // MCOL-4685: remove the option to declare uncompressed columns (set + // infinidb_compression_type = 0). + compressiontype = get_compression_type(thd); + if (compressiontype == 0) + compressiontype = 2; } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calonlinealter_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1 || args->arg_type[0] != STRING_RESULT) - { - strcpy(message, "CALONLINEALTER() requires one string argument"); - return 1; - } + if (compressiontype == 1) + compressiontype = 2; - return 0; - } + int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype); + + if (rc != 0) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + + return rc; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void calonlinealter_deinit(UDF_INIT* initid) + my_bool calonlinealter_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1 || args->arg_type[0] != STRING_RESULT) { + strcpy(message, "CALONLINEALTER() requires one string argument"); + return 1; } + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calonlinealter_deinit(UDF_INIT* initid) + { + } } // vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 0524447c6..3c7bd5573 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -72,7 +72,7 @@ using namespace execplan; #include "resourcemanager.h" using namespace joblist; //#include "stopwatch.h" -//using namespace logging; +// using namespace logging; #include "dbrm.h" @@ -81,752 +81,750 @@ using namespace joblist; namespace { #define BATCH_INSERT_GROUP_ROWS_FOR_CACHE 100000 -uint64_t fBatchInsertGroupRows = 0; // ResourceManager::instance()->getRowsPerBatch(); +uint64_t fBatchInsertGroupRows = 0; // ResourceManager::instance()->getRowsPerBatch(); // HDFS is never used nowadays, so don't bother -bool useHdfs = false; // ResourceManager::instance()->useHdfs(); +bool useHdfs = false; // ResourceManager::instance()->useHdfs(); -//convenience fcn +// convenience fcn inline uint32_t tid2sid(const uint32_t tid) { - return execplan::CalpontSystemCatalog::idb_tid2sid(tid); + return execplan::CalpontSystemCatalog::idb_tid2sid(tid); } -//StopWatch timer; -uint32_t buildValueList (TABLE* table, cal_connection_info& ci ) +// StopWatch timer; +uint32_t buildValueList(TABLE* table, cal_connection_info& ci) { - char attribute_buffer[1024]; - String attribute(attribute_buffer, sizeof(attribute_buffer), - &my_charset_bin); - uint32_t size = 0; - int columnPos = 0; - double dbval; - ci.nullValuesBitset.reset(); + char attribute_buffer[1024]; + String attribute(attribute_buffer, sizeof(attribute_buffer), &my_charset_bin); + uint32_t size = 0; + int columnPos = 0; + double dbval; + ci.nullValuesBitset.reset(); - for (Field** field = table->field; *field; field++) + for (Field** field = table->field; *field; field++) + { + if ((*field)->is_null()) { - if ((*field)->is_null()) - { - ci.tableValuesMap[columnPos].push_back (""); //currently, empty string is treated as null. - ci.nullValuesBitset[columnPos] = true; - } - else - { - bitmap_set_bit(table->read_set, (*field)->field_index); - ci.nullValuesBitset[columnPos] = false; - - // @bug 3798 get real value for float/double type - if ((*field)->result_type() == REAL_RESULT) - { - dbval = (*field)->val_real(); - //int maxlen = (*field)->max_display_length(); - const unsigned maxlen = 1024 + 1 + 1 + 1; //1 for leading zero, 1 for dp, 1 for null - char buf[maxlen]; - memset(buf, 0, maxlen); - snprintf(buf, maxlen, "%.1024f", dbval); - ci.tableValuesMap[columnPos].push_back(buf); - } - else - { - //fetch different data type - (*field)->val_str(&attribute, &attribute); - - if (attribute.length() == 0) - { - ci.tableValuesMap[columnPos].push_back (""); //currently, empty string is treated as null. - } - else - { - string val(attribute.ptr(), attribute.length()); - ci.tableValuesMap[columnPos].push_back(val); - } - } - } - - ci.colNameList.push_back((*field)->field_name.str); - - columnPos++; - } - - size = ci.tableValuesMap[0].size(); - return size; -} - - -int ProcessCommandStatement(THD* thd, string& dmlStatement, cal_connection_info& ci, std::string schema = "") -{ - int rc = 0; - - ulong sessionID = tid2sid(thd->thread_id); - - CalpontDMLPackage* pDMLPackage; - - //@Bug 2721 and 2722. Log the statement before issuing commit/rollback - if ( dmlStatement == "LOGGING" ) - { - char* query_char = idb_mysql_query_str(thd); - std::string query_str; - if (!query_char) - { - query_str = ""; - } - else - { - query_str = query_char; - } - VendorDMLStatement cmdStmt(query_str, DML_COMMAND, sessionID); - cmdStmt.set_Logging( false ); - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_Logging( false ); - pDMLPackage->set_SchemaName( schema ); + ci.tableValuesMap[columnPos].push_back(""); // currently, empty string is treated as null. + ci.nullValuesBitset[columnPos] = true; } else { - VendorDMLStatement cmdStmt(dmlStatement, DML_COMMAND, sessionID); - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - } + bitmap_set_bit(table->read_set, (*field)->field_index); + ci.nullValuesBitset[columnPos] = false; - pDMLPackage->setTableOid (ci.tableOid); + // @bug 3798 get real value for float/double type + if ((*field)->result_type() == REAL_RESULT) + { + dbval = (*field)->val_real(); + // int maxlen = (*field)->max_display_length(); + const unsigned maxlen = 1024 + 1 + 1 + 1; // 1 for leading zero, 1 for dp, 1 for null + char buf[maxlen]; + memset(buf, 0, maxlen); + snprintf(buf, maxlen, "%.1024f", dbval); + ci.tableValuesMap[columnPos].push_back(buf); + } + else + { + // fetch different data type + (*field)->val_str(&attribute, &attribute); - if (!ci.singleInsert) - { - pDMLPackage->set_isBatchInsert(true); - } - - if (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) - pDMLPackage->set_isAutocommitOn(true); - - if (useHdfs) - pDMLPackage->set_isAutocommitOn(true); - - ByteStream bytestream; - bytestream << static_cast(sessionID); - - pDMLPackage->write(bytestream); - delete pDMLPackage; - - ByteStream::byte b = 0; - string errormsg; - ByteStream::octbyte rows; - - try - { - ci.dmlProc->write(bytestream); - bytestream = ci.dmlProc->read(); - - if ( bytestream.length() == 0 ) + if (attribute.length() == 0) { - rc = 1; - thd->killed = KILL_QUERY; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [1]"); + ci.tableValuesMap[columnPos].push_back(""); // currently, empty string is treated as null. } else { - bytestream >> b; - bytestream >> rows; - bytestream >> errormsg; + string val(attribute.ptr(), attribute.length()); + ci.tableValuesMap[columnPos].push_back(val); } - } - catch (runtime_error&) - { - rc = 1 ; - thd->killed = KILL_QUERY; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [2]"); - } - catch (...) - { - rc = 1; - thd->killed = KILL_QUERY; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_INTERNAL_ERROR, "Caught unknown error"); + } } - if (( b != 0 ) && (!thd->get_stmt_da()->is_set())) - { - rc = 1; - thd->killed = KILL_QUERY; - thd->raise_error_printf(ER_INTERNAL_ERROR, errormsg.c_str()); - } + ci.colNameList.push_back((*field)->field_name.str); - delete ci.dmlProc; - ci.dmlProc = NULL; - return rc; + columnPos++; + } + + size = ci.tableValuesMap[0].size(); + return size; } -int doProcessInsertValues ( TABLE* table, uint32_t size, cal_connection_info& ci, bool lastBatch = false ) +int ProcessCommandStatement(THD* thd, string& dmlStatement, cal_connection_info& ci, std::string schema = "") { - THD* thd = current_thd; - uint32_t sessionID = tid2sid(thd->thread_id); + int rc = 0; - int rc = 0; + ulong sessionID = tid2sid(thd->thread_id); + CalpontDMLPackage* pDMLPackage; + + //@Bug 2721 and 2722. Log the statement before issuing commit/rollback + if (dmlStatement == "LOGGING") + { char* query_char = idb_mysql_query_str(thd); std::string query_str; if (!query_char) { - query_str = ""; + query_str = ""; } else { - query_str = query_char; + query_str = query_char; } + VendorDMLStatement cmdStmt(query_str, DML_COMMAND, sessionID); + cmdStmt.set_Logging(false); + pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + pDMLPackage->set_Logging(false); + pDMLPackage->set_SchemaName(schema); + } + else + { + VendorDMLStatement cmdStmt(dmlStatement, DML_COMMAND, sessionID); + pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + } - VendorDMLStatement dmlStmts(query_str, DML_INSERT, table->s->table_name.str, - table->s->db.str, size, ci.colNameList.size(), ci.colNameList, - ci.tableValuesMap, ci.nullValuesBitset, sessionID); + pDMLPackage->setTableOid(ci.tableOid); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStmts); - //@Bug 2466 Move the clean up earlier to avoid the second insert in another session to get the data - ci.tableValuesMap.clear(); - ci.colNameList.clear(); + if (!ci.singleInsert) + { + pDMLPackage->set_isBatchInsert(true); + } - pDMLPackage->set_isCacheInsert(ci.isCacheInsert); + if (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) + pDMLPackage->set_isAutocommitOn(true); - if (!pDMLPackage) + if (useHdfs) + pDMLPackage->set_isAutocommitOn(true); + + ByteStream bytestream; + bytestream << static_cast(sessionID); + + pDMLPackage->write(bytestream); + delete pDMLPackage; + + ByteStream::byte b = 0; + string errormsg; + ByteStream::octbyte rows; + + try + { + ci.dmlProc->write(bytestream); + bytestream = ci.dmlProc->read(); + + if (bytestream.length() == 0) { - rc = -1; - string emsg("Calpont DML package cannot build. "); - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); - return rc; + rc = 1; + thd->killed = KILL_QUERY; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [1]"); } - - //@Bug 2721 and 2722 log batch insert statement only once in the bebug file. - if (( ( ci.isLoaddataInfile ) || ((ci.rowsHaveInserted + size) < ci.bulkInsertRows ) ) && (ci.rowsHaveInserted > 0)) + else { - pDMLPackage->set_Logging( false ); - pDMLPackage->set_Logending( false ); + bytestream >> b; + bytestream >> rows; + bytestream >> errormsg; } - else if (( ( ci.isLoaddataInfile ) || ((ci.rowsHaveInserted + size) < ci.bulkInsertRows ) ) && (ci.rowsHaveInserted == 0)) + } + catch (runtime_error&) + { + rc = 1; + thd->killed = KILL_QUERY; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [2]"); + } + catch (...) + { + rc = 1; + thd->killed = KILL_QUERY; + thd->get_stmt_da()->set_overwrite_status(true); + + thd->raise_error_printf(ER_INTERNAL_ERROR, "Caught unknown error"); + } + + if ((b != 0) && (!thd->get_stmt_da()->is_set())) + { + rc = 1; + thd->killed = KILL_QUERY; + thd->raise_error_printf(ER_INTERNAL_ERROR, errormsg.c_str()); + } + + delete ci.dmlProc; + ci.dmlProc = NULL; + return rc; +} + +int doProcessInsertValues(TABLE* table, uint32_t size, cal_connection_info& ci, bool lastBatch = false) +{ + THD* thd = current_thd; + uint32_t sessionID = tid2sid(thd->thread_id); + + int rc = 0; + + char* query_char = idb_mysql_query_str(thd); + std::string query_str; + if (!query_char) + { + query_str = ""; + } + else + { + query_str = query_char; + } + + VendorDMLStatement dmlStmts(query_str, DML_INSERT, table->s->table_name.str, table->s->db.str, size, + ci.colNameList.size(), ci.colNameList, ci.tableValuesMap, ci.nullValuesBitset, + sessionID); + + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStmts); + //@Bug 2466 Move the clean up earlier to avoid the second insert in another session to get the data + ci.tableValuesMap.clear(); + ci.colNameList.clear(); + + pDMLPackage->set_isCacheInsert(ci.isCacheInsert); + + if (!pDMLPackage) + { + rc = -1; + string emsg("Calpont DML package cannot build. "); + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + return rc; + } + + //@Bug 2721 and 2722 log batch insert statement only once in the bebug file. + if (((ci.isLoaddataInfile) || ((ci.rowsHaveInserted + size) < ci.bulkInsertRows)) && + (ci.rowsHaveInserted > 0)) + { + pDMLPackage->set_Logging(false); + pDMLPackage->set_Logending(false); + } + else if (((ci.isLoaddataInfile) || ((ci.rowsHaveInserted + size) < ci.bulkInsertRows)) && + (ci.rowsHaveInserted == 0)) + { + pDMLPackage->set_Logging(true); + pDMLPackage->set_Logending(false); + } + + if (ci.singleInsert) + { + pDMLPackage->set_Logging(true); + pDMLPackage->set_Logending(true); + } + + if (!ci.singleInsert) + { + pDMLPackage->set_isBatchInsert(true); + } + + if (thd->is_strict_mode()) + { + pDMLPackage->set_isWarnToError(true); + } + + pDMLPackage->setTableOid(ci.tableOid); + + if (lastBatch) + { + pDMLPackage->set_Logending(true); + } + + if (lastBatch && (ci.rowsHaveInserted > 0)) + pDMLPackage->set_Logging(false); + + std::string name = table->s->table_name.str; + pDMLPackage->set_TableName(name); + name = table->s->db.str; + pDMLPackage->set_SchemaName(name); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + + if (thd->lex->sql_command == SQLCOM_INSERT_SELECT) + pDMLPackage->set_isInsertSelect(true); + + // Carry session autocommit info in the pkg to use in DMLProc + // cout << "Thread options = " << thd->variables.option_bits << " and OPTION_NOT_AUTOCOMMIT:OPTION_BEGIN = + // " << OPTION_NOT_AUTOCOMMIT << ":" << OPTION_BEGIN << endl; + if (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) + { + // cout << "autocommit is on" << endl; + pDMLPackage->set_isAutocommitOn(true); + } + else if (useHdfs) + { + pDMLPackage->set_isAutocommitOn(true); + } + + ByteStream bytestream, bytestreamRcv; + bytestream << sessionID; + + pDMLPackage->write(bytestream); + delete pDMLPackage; + + ByteStream::byte b = 0; + string errormsg; + ByteStream::octbyte rows; + + try + { + ci.dmlProc->write(bytestream); + bytestreamRcv = ci.dmlProc->read(); + + if (bytestreamRcv.length() == 0) { - pDMLPackage->set_Logging( true ); - pDMLPackage->set_Logending( false ); - } + // check if it is first batch and DMLProc restarted. Only this case, get a new client and resend + if (ci.rowsHaveInserted == 0) + { + delete ci.dmlProc; + ci.dmlProc = new MessageQueueClient("DMLProc"); - if ( ci.singleInsert ) - { - pDMLPackage->set_Logging( true ); - pDMLPackage->set_Logending( true ); - } - - if ( !ci.singleInsert ) - { - pDMLPackage->set_isBatchInsert( true ); - } - - if (thd->is_strict_mode()) - { - pDMLPackage->set_isWarnToError( true ); - } - - pDMLPackage->setTableOid (ci.tableOid); - - if (lastBatch) - { - pDMLPackage->set_Logending( true ); - - } - - if (lastBatch && (ci.rowsHaveInserted > 0)) - pDMLPackage->set_Logging( false ); - - std::string name = table->s->table_name.str; - pDMLPackage->set_TableName(name); - name = table->s->db.str; - pDMLPackage->set_SchemaName(name); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); - - if (thd->lex->sql_command == SQLCOM_INSERT_SELECT) - pDMLPackage->set_isInsertSelect(true); - - //Carry session autocommit info in the pkg to use in DMLProc - //cout << "Thread options = " << thd->variables.option_bits << " and OPTION_NOT_AUTOCOMMIT:OPTION_BEGIN = " << OPTION_NOT_AUTOCOMMIT << ":" << OPTION_BEGIN << endl; - if (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) - { - //cout << "autocommit is on" << endl; - pDMLPackage->set_isAutocommitOn(true); - } - else if (useHdfs) - { - pDMLPackage->set_isAutocommitOn(true); - } - - ByteStream bytestream, bytestreamRcv; - bytestream << sessionID; - - pDMLPackage->write(bytestream); - delete pDMLPackage; - - ByteStream::byte b = 0; - string errormsg; - ByteStream::octbyte rows; - - try - { - ci.dmlProc->write(bytestream); - bytestreamRcv = ci.dmlProc->read(); - - if ( bytestreamRcv.length() == 0 ) + // cout << "doProcessInsertValues starts a client " << ci.dmlProc << " for session " << thd->thread_id + // << endl; + try { - //check if it is first batch and DMLProc restarted. Only this case, get a new client and resend - if (ci.rowsHaveInserted == 0) - { - delete ci.dmlProc; - ci.dmlProc = new MessageQueueClient("DMLProc"); + ci.dmlProc->write(bytestream); + bytestreamRcv = ci.dmlProc->read(); - //cout << "doProcessInsertValues starts a client " << ci.dmlProc << " for session " << thd->thread_id << endl; - try - { - ci.dmlProc->write(bytestream); - bytestreamRcv = ci.dmlProc->read(); - - if ( bytestreamRcv.length() == 0 ) - { - rc = -1; - b = 1; - errormsg = "Lost connection to DMLProc [3]"; - } - else - { - bytestreamRcv >> b; - bytestreamRcv >> rows; - bytestreamRcv >> errormsg; - rc = b; - } - } - catch (runtime_error&) - { - rc = -1; - thd->get_stmt_da()->set_overwrite_status(true); - errormsg = "Lost connection to DMLProc [4]"; - b = 1; - } - } - } - else - { + if (bytestreamRcv.length() == 0) + { + rc = -1; + b = 1; + errormsg = "Lost connection to DMLProc [3]"; + } + else + { bytestreamRcv >> b; bytestreamRcv >> rows; bytestreamRcv >> errormsg; rc = b; + } } - } - catch (std::exception& rex) - { - //check if it is first batch and DMLProc restarted. Only this case, get a new client and resend - if (ci.rowsHaveInserted == 0) + catch (runtime_error&) { - delete ci.dmlProc; - ci.dmlProc = new MessageQueueClient("DMLProc"); - - //cout << "doProcessInsertValues exception starts a client " << ci.dmlProc << " for session " << thd->thread_id << endl; - try - { - ci.dmlProc->write(bytestream); - bytestreamRcv = ci.dmlProc->read(); - - if ( bytestreamRcv.length() == 0 ) - { - rc = -1; - b = 1; - errormsg = string("Lost connection to DMLProc after getting a new client [1:") + rex.what() + "]"; - } - else - { - bytestreamRcv >> b; - bytestreamRcv >> rows; - bytestreamRcv >> errormsg; - rc = b; - } - } - catch (std::exception& rrex) - { - rc = -1; - thd->get_stmt_da()->set_overwrite_status(true); - errormsg = string("Lost connection to DMLProc after getting a new client [2:") + rex.what() + " then " + rrex.what() + "]"; - b = 1; - } + rc = -1; + thd->get_stmt_da()->set_overwrite_status(true); + errormsg = "Lost connection to DMLProc [4]"; + b = 1; } - else //really lost connection - { - rc = -1; - thd->get_stmt_da()->set_overwrite_status(true); - errormsg = string("Lost connection to DMLProc really [1:") + rex.what() + "]"; - b = 1; - } - } - catch (...) - { - rc = -1; - thd->get_stmt_da()->set_overwrite_status(true); - errormsg = "Unknown error caught"; - b = 1; - } - - - if ((b != 0) && (b != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, errormsg.c_str()); - - } - - if ( b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING ) - { - if (!thd->is_strict_mode()) - { - rc = 0; - } - - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errormsg.c_str()); - } - - if ( rc != 0 ) - ci.rc = rc; - - if ( b == dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR ) - rc = b; - - return rc; -} - -} - -int ha_mcs_impl_write_last_batch(TABLE* table, cal_connection_info& ci, bool abort) -{ - int rc = 0; - THD* thd = current_thd; - std::string command; - uint32_t size = ci.tableValuesMap[0].size(); - //@Bug 2468. Add a logging statement command - command = "COMMIT"; - std::string schema; - schema = table->s->db.str; - - //@Bug 6112. if no row to be insert and no rows have been inserted, no need to send to DMLProc - if ((size == 0) && (ci.rowsHaveInserted == 0)) - return rc; - - //@Bug 2715 Check the saved error code. - //@Bug 4516 always send the last package to allow DMLProc receive all messages from WES - if (( ci.rc != 0 ) || abort ) - { - if (abort) //@Bug 5285. abort is different from error, dmlproc only clean up when erroring out - rc = doProcessInsertValues( table, size, ci, true); - - //@Bug 2722 Log the statement into datamod log - //@Bug 4605 if error, rollback and no need to check whether the session is autocommit off - - command = "ROLLBACK"; - rc = ProcessCommandStatement ( thd, command, ci, schema ); - rc = ci.rc; - ci.rc = 0; - - if (size > 0 ) - { - ci.tableValuesMap.clear(); - ci.colNameList.clear(); - } - - return rc; + } } else { - rc = doProcessInsertValues( table, size, ci, true); + bytestreamRcv >> b; + bytestreamRcv >> rows; + bytestreamRcv >> errormsg; + rc = b; } - - if ( abort ) + } + catch (std::exception& rex) + { + // check if it is first batch and DMLProc restarted. Only this case, get a new client and resend + if (ci.rowsHaveInserted == 0) { - rc = 1; + delete ci.dmlProc; + ci.dmlProc = new MessageQueueClient("DMLProc"); + + // cout << "doProcessInsertValues exception starts a client " << ci.dmlProc << " for session " << + // thd->thread_id << endl; + try + { + ci.dmlProc->write(bytestream); + bytestreamRcv = ci.dmlProc->read(); + + if (bytestreamRcv.length() == 0) + { + rc = -1; + b = 1; + errormsg = string("Lost connection to DMLProc after getting a new client [1:") + rex.what() + "]"; + } + else + { + bytestreamRcv >> b; + bytestreamRcv >> rows; + bytestreamRcv >> errormsg; + rc = b; + } + } + catch (std::exception& rrex) + { + rc = -1; thd->get_stmt_da()->set_overwrite_status(true); - std::string errormsg = "statement is aborted."; - thd->raise_error_printf(ER_INTERNAL_ERROR, errormsg.c_str()); + errormsg = string("Lost connection to DMLProc after getting a new client [2:") + rex.what() + + " then " + rrex.what() + "]"; + b = 1; + } + } + else // really lost connection + { + rc = -1; + thd->get_stmt_da()->set_overwrite_status(true); + errormsg = string("Lost connection to DMLProc really [1:") + rex.what() + "]"; + b = 1; + } + } + catch (...) + { + rc = -1; + thd->get_stmt_da()->set_overwrite_status(true); + errormsg = "Unknown error caught"; + b = 1; + } + + if ((b != 0) && (b != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, errormsg.c_str()); + } + + if (b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + { + if (!thd->is_strict_mode()) + { + rc = 0; } - if ( rc == dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR ) - return rc; + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errormsg.c_str()); + } - //@Bug 4605 - int rc1 = 0; - if ( (rc == 0) && !abort && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) - { - ci.rowsHaveInserted += size; - command = "COMMIT"; - rc1 = ProcessCommandStatement ( thd, command, ci, schema ); - } - else if (useHdfs) - { - ci.rowsHaveInserted += size; - command = "COMMIT"; - rc1 = ProcessCommandStatement ( thd, command, ci, schema ); - } - else if (( rc != 0) || abort ) - { - command = "ROLLBACK"; - rc1 = ProcessCommandStatement ( thd, command, ci, schema ); - } - rc = max(rc, rc1); - + if (rc != 0) + ci.rc = rc; + + if (b == dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) + rc = b; + + return rc; +} + +} // namespace + +int ha_mcs_impl_write_last_batch(TABLE* table, cal_connection_info& ci, bool abort) +{ + int rc = 0; + THD* thd = current_thd; + std::string command; + uint32_t size = ci.tableValuesMap[0].size(); + //@Bug 2468. Add a logging statement command + command = "COMMIT"; + std::string schema; + schema = table->s->db.str; + + //@Bug 6112. if no row to be insert and no rows have been inserted, no need to send to DMLProc + if ((size == 0) && (ci.rowsHaveInserted == 0)) return rc; + //@Bug 2715 Check the saved error code. + //@Bug 4516 always send the last package to allow DMLProc receive all messages from WES + if ((ci.rc != 0) || abort) + { + if (abort) //@Bug 5285. abort is different from error, dmlproc only clean up when erroring out + rc = doProcessInsertValues(table, size, ci, true); + + //@Bug 2722 Log the statement into datamod log + //@Bug 4605 if error, rollback and no need to check whether the session is autocommit off + + command = "ROLLBACK"; + rc = ProcessCommandStatement(thd, command, ci, schema); + rc = ci.rc; + ci.rc = 0; + + if (size > 0) + { + ci.tableValuesMap.clear(); + ci.colNameList.clear(); + } + + return rc; + } + else + { + rc = doProcessInsertValues(table, size, ci, true); + } + + if (abort) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + std::string errormsg = "statement is aborted."; + thd->raise_error_printf(ER_INTERNAL_ERROR, errormsg.c_str()); + } + + if (rc == dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) + return rc; + + //@Bug 4605 + int rc1 = 0; + if ((rc == 0) && !abort && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) + { + ci.rowsHaveInserted += size; + command = "COMMIT"; + rc1 = ProcessCommandStatement(thd, command, ci, schema); + } + else if (useHdfs) + { + ci.rowsHaveInserted += size; + command = "COMMIT"; + rc1 = ProcessCommandStatement(thd, command, ci, schema); + } + else if ((rc != 0) || abort) + { + command = "ROLLBACK"; + rc1 = ProcessCommandStatement(thd, command, ci, schema); + } + rc = max(rc, rc1); + + return rc; } int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_connection_info& ci, ha_rows& rowsInserted) { - int rc = 0; - //timer.start( "buildValueList"); - ci.colNameList.clear(); - THD* thd = current_thd; - uint32_t size = 0; - std::string schema; - schema = table->s->db.str; + int rc = 0; + // timer.start( "buildValueList"); + ci.colNameList.clear(); + THD* thd = current_thd; + uint32_t size = 0; + std::string schema; + schema = table->s->db.str; - //@Bug 2086 Added syntax check for '\0' - try + //@Bug 2086 Added syntax check for '\0' + try + { + size = buildValueList(table, ci); + } + catch (runtime_error& rex) + { + rc = 1; + ci.rc = rc; //@Bug 2790 Save the error infomation. + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, rex.what()); + return rc; + } + + if (fBatchInsertGroupRows == 0) + { + fBatchInsertGroupRows = ResourceManager::instance()->getRowsPerBatch(); + } + + // timer.stop( "buildValueList"); + if (ci.singleInsert // Single insert + || ((ci.bulkInsertRows > 0) && (((ci.rowsHaveInserted + size) >= ci.bulkInsertRows) || + ((!ci.isCacheInsert && size >= fBatchInsertGroupRows) || + (ci.isCacheInsert && size >= BATCH_INSERT_GROUP_ROWS_FOR_CACHE)))) + // Insert with mutilple value case: processed batch by batch. Last batch is sent also. + || ((ci.bulkInsertRows == 0) && + ((!ci.isCacheInsert && size >= fBatchInsertGroupRows) || + (ci.isCacheInsert && + size >= BATCH_INSERT_GROUP_ROWS_FOR_CACHE)))) // Load data in file is processed batch by batch + { + // timer.start( "DMLProc takes"); + // cout <<" sending a batch to DMLProc ... The size is " << size << " the current bulkInsertRows = " << + // ci.bulkInsertRows << endl; Build dmlpackage + if ((ci.bulkInsertRows > 0) && ((ci.rowsHaveInserted + size) >= ci.bulkInsertRows)) { - size = buildValueList ( table, ci ); - } - catch (runtime_error& rex) - { - rc = 1; - ci.rc = rc; //@Bug 2790 Save the error infomation. - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, rex.what()); - return rc; - } - - if (fBatchInsertGroupRows == 0) - { - fBatchInsertGroupRows = ResourceManager::instance()->getRowsPerBatch(); - } - - //timer.stop( "buildValueList"); - if ( ci.singleInsert // Single insert - || (( ci.bulkInsertRows > 0 ) && (( ( ci.rowsHaveInserted + size) >= ci.bulkInsertRows ) - || ( (!ci.isCacheInsert && size >= fBatchInsertGroupRows) || (ci.isCacheInsert && size >= BATCH_INSERT_GROUP_ROWS_FOR_CACHE) )) ) - //Insert with mutilple value case: processed batch by batch. Last batch is sent also. - || (( ci.bulkInsertRows == 0 ) && ( (!ci.isCacheInsert && size >= fBatchInsertGroupRows) - || (ci.isCacheInsert && size >= BATCH_INSERT_GROUP_ROWS_FOR_CACHE) ) ) ) // Load data in file is processed batch by batch - { - //timer.start( "DMLProc takes"); - //cout <<" sending a batch to DMLProc ... The size is " << size << " the current bulkInsertRows = " << ci.bulkInsertRows << endl; - //Build dmlpackage - if (( ci.bulkInsertRows > 0 ) && ( ( ci.rowsHaveInserted + size) >= ci.bulkInsertRows )) - { - rc = doProcessInsertValues( table, size, ci, true ); - } - else - { - rc = doProcessInsertValues( table, size, ci ); - } - - if ( rc == 0 ) - rowsInserted = size; - else - ci.rc = rc; - - //@Bug 2481. The current active transaction needs to be released if autocommit is on - //@Bug 2438 Added a check for batchinsert's last batch to send commit if autocommit is on - std::string command; - - if ( ci.singleInsert || ( (ci.bulkInsertRows > 0 ) && (( ci.rowsHaveInserted + size) >= ci.bulkInsertRows ) ) ) - { - if ( thd->killed > 0 ) - { - command = "ROLLBACK"; - rc = ProcessCommandStatement ( thd, command, ci, schema ); - } - else if (rc != dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) - { - //@Bug 4605 - int rc1 = 0; - if ( rc != 0 ) - { - command = "ROLLBACK"; - rc1 = ProcessCommandStatement ( thd, command, ci, schema ); - } - else if (( rc == 0 ) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) - { - command = "COMMIT"; - rc1 = ProcessCommandStatement ( thd, command, ci, schema ); - } - else if (useHdfs) - { - command = "COMMIT"; - rc1 = ProcessCommandStatement ( thd, command, ci, schema ); - } - rc = max(rc, rc1); - } - } - - //timer.stop( "DMLProc takes"); - //timer.finish(); - return rc; - + rc = doProcessInsertValues(table, size, ci, true); } else { - return rc; + rc = doProcessInsertValues(table, size, ci); } -} + if (rc == 0) + rowsInserted = size; + else + ci.rc = rc; + + //@Bug 2481. The current active transaction needs to be released if autocommit is on + //@Bug 2438 Added a check for batchinsert's last batch to send commit if autocommit is on + std::string command; + + if (ci.singleInsert || ((ci.bulkInsertRows > 0) && ((ci.rowsHaveInserted + size) >= ci.bulkInsertRows))) + { + if (thd->killed > 0) + { + command = "ROLLBACK"; + rc = ProcessCommandStatement(thd, command, ci, schema); + } + else if (rc != dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) + { + //@Bug 4605 + int rc1 = 0; + if (rc != 0) + { + command = "ROLLBACK"; + rc1 = ProcessCommandStatement(thd, command, ci, schema); + } + else if ((rc == 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) + { + command = "COMMIT"; + rc1 = ProcessCommandStatement(thd, command, ci, schema); + } + else if (useHdfs) + { + command = "COMMIT"; + rc1 = ProcessCommandStatement(thd, command, ci, schema); + } + rc = max(rc, rc1); + } + } + + // timer.stop( "DMLProc takes"); + // timer.finish(); + return rc; + } + else + { + return rc; + } +} int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci) { - ByteStream rowData; - int rc = 0; - //std::ostringstream data; - bool nullVal = false; - const uchar* bufHdr = buf; // bit flag indicating a field is null. Only those fields that are nullable are represented. - int32_t headerByte = 0; // Current byte in the bufHdr - int32_t headerBit = 0; // current bit in the bufHdr current byte. - uint16_t colpos = 0; - buf = buf + ci.headerLength; // Number of bytes used for null bits. - //@Bug 6122 if all columns have not null constraint, there is no information in the header - char nullBits = *bufHdr++; + ByteStream rowData; + int rc = 0; + // std::ostringstream data; + bool nullVal = false; + const uchar* bufHdr = + buf; // bit flag indicating a field is null. Only those fields that are nullable are represented. + int32_t headerByte = 0; // Current byte in the bufHdr + int32_t headerBit = 0; // current bit in the bufHdr current byte. + uint16_t colpos = 0; + buf = buf + ci.headerLength; // Number of bytes used for null bits. + //@Bug 6122 if all columns have not null constraint, there is no information in the header + char nullBits = *bufHdr++; - if (!ci.useXbit) + if (!ci.useXbit) + { + // Skip the first bit. For some reason, mysql reserves the first bit of the first byte, unless there's a + // varchar column in the table. + nullBits = nullBits >> 1; + ++headerBit; + } + + while (colpos < ci.columnTypes.size()) // test bitmap for null values + { + uint8_t numLoop = 7; + + if ((ci.useXbit) || (colpos > 6)) + numLoop++; + + for (uint8_t i = 0; i < numLoop; i++) { - // Skip the first bit. For some reason, mysql reserves the first bit of the first byte, unless there's a varchar column in the table. + if (colpos == ci.columnTypes.size()) + break; + + // if a column has not null constraint, it will not be in the bit map + if (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (ci.headerLength > 0 && headerByte >= ci.headerLength) + { + // We've used more null bits than allowed. Something is seriously wrong. + std::string errormsg = "Null bit header is wrong size"; + setError(current_thd, ER_INTERNAL_ERROR, errormsg); + return -1; + } + + nullVal = nullBits & 0x01; nullBits = nullBits >> 1; ++headerBit; - } - while (colpos < ci.columnTypes.size()) //test bitmap for null values - { - uint8_t numLoop = 7; - - if ((ci.useXbit) || (colpos > 6)) - numLoop++; - - for (uint8_t i = 0; i < numLoop; i++) + if (headerBit == 8) { - if (colpos == ci.columnTypes.size()) - break; - - //if a column has not null constraint, it will not be in the bit map - if (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (ci.headerLength > 0 && headerByte >= ci.headerLength) - { - // We've used more null bits than allowed. Something is seriously wrong. - std::string errormsg = "Null bit header is wrong size"; - setError(current_thd, ER_INTERNAL_ERROR, errormsg); - return -1; - } - - nullVal = nullBits & 0x01; - nullBits = nullBits >> 1; - ++headerBit; - - if (headerBit == 8) - { - nullBits = *bufHdr++; - headerBit = 0; - ++headerByte; - } - } - else - { - nullVal = false; - } - - const CalpontSystemCatalog::ColType &colType= ci.columnTypes[colpos]; - const datatypes::TypeHandler *h= colType.typeHandler(); - if (h) // QQ: error reporting - { - datatypes::ColBatchWriter writer(ci.filePtr, - ci.delimiter, - ci.enclosed_by); - Field* fieldPtr= table->field[colpos]; - uint32_t mbmaxlen = (fieldPtr->charset() && fieldPtr->charset()->mbmaxlen) - ? fieldPtr->charset()->mbmaxlen : 0; - datatypes::WriteBatchFieldMariaDB field(fieldPtr, - colType, - mbmaxlen); - idbassert(table == table->field[colpos]->table); - buf+= h->ColWriteBatch(&field, buf, nullVal, writer); - } - colpos++; + nullBits = *bufHdr++; + headerBit = 0; + ++headerByte; } + } + else + { + nullVal = false; + } + + const CalpontSystemCatalog::ColType& colType = ci.columnTypes[colpos]; + const datatypes::TypeHandler* h = colType.typeHandler(); + if (h) // QQ: error reporting + { + datatypes::ColBatchWriter writer(ci.filePtr, ci.delimiter, ci.enclosed_by); + Field* fieldPtr = table->field[colpos]; + uint32_t mbmaxlen = + (fieldPtr->charset() && fieldPtr->charset()->mbmaxlen) ? fieldPtr->charset()->mbmaxlen : 0; + datatypes::WriteBatchFieldMariaDB field(fieldPtr, colType, mbmaxlen); + idbassert(table == table->field[colpos]->table); + buf += h->ColWriteBatch(&field, buf, nullVal, writer); + } + colpos++; } + } - rc = fprintf(ci.filePtr, "\n"); //@bug 6077 check whether the pipe is still open + rc = fprintf(ci.filePtr, "\n"); //@bug 6077 check whether the pipe is still open - if ( rc < 0) - rc = -1; - else - rc = 0; + if (rc < 0) + rc = -1; + else + rc = 0; - return rc; + return rc; } -std::string ha_mcs_impl_viewtablelock( cal_impl_if::cal_connection_info& ci, execplan::CalpontSystemCatalog::TableName& tablename) +std::string ha_mcs_impl_viewtablelock(cal_impl_if::cal_connection_info& ci, + execplan::CalpontSystemCatalog::TableName& tablename) { - THD* thd = current_thd; - ulong sessionID = tid2sid(thd->thread_id); - CalpontDMLPackage* pDMLPackage; - std::string dmlStatement( "VIEWTABLELOCK" ); - VendorDMLStatement cmdStmt(dmlStatement, DML_COMMAND, sessionID); - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - if (lower_case_table_names) + THD* thd = current_thd; + ulong sessionID = tid2sid(thd->thread_id); + CalpontDMLPackage* pDMLPackage; + std::string dmlStatement("VIEWTABLELOCK"); + VendorDMLStatement cmdStmt(dmlStatement, DML_COMMAND, sessionID); + pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + if (lower_case_table_names) + { + boost::algorithm::to_lower(tablename.schema); + boost::algorithm::to_lower(tablename.table); + } + pDMLPackage->set_SchemaName(tablename.schema); + pDMLPackage->set_TableName(tablename.table); + + ByteStream bytestream; + bytestream << static_cast(sessionID); + pDMLPackage->write(bytestream); + delete pDMLPackage; + + ByteStream::byte b = 0; + ByteStream::octbyte rows; + std::string errorMsg; + std::string tableLockInfo; + // int dmlRowCount = 0; + + try + { + ci.dmlProc->write(bytestream); + bytestream = ci.dmlProc->read(); + + if (bytestream.length() == 0) { - boost::algorithm::to_lower(tablename.schema); - boost::algorithm::to_lower(tablename.table); + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [5]"); } - pDMLPackage->set_SchemaName (tablename.schema); - pDMLPackage->set_TableName (tablename.table); - - ByteStream bytestream; - bytestream << static_cast(sessionID); - pDMLPackage->write(bytestream); - delete pDMLPackage; - - ByteStream::byte b = 0; - ByteStream::octbyte rows; - std::string errorMsg; - std::string tableLockInfo; - //int dmlRowCount = 0; - - try + else { - ci.dmlProc->write(bytestream); - bytestream = ci.dmlProc->read(); - - if ( bytestream.length() == 0 ) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [5]"); - } - else - { - bytestream >> b; - bytestream >> rows; - bytestream >> errorMsg; - bytestream >> tableLockInfo; - } - - } - catch (runtime_error&) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [6]"); - } - catch (...) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Caught unknown error"); + bytestream >> b; + bytestream >> rows; + bytestream >> errorMsg; + bytestream >> tableLockInfo; } + } + catch (runtime_error&) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [6]"); + } + catch (...) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Caught unknown error"); + } - if ( b != 0 ) - tableLockInfo = errorMsg; - - return tableLockInfo; + if (b != 0) + tableLockInfo = errorMsg; + return tableLockInfo; } //------------------------------------------------------------------------------ @@ -834,175 +832,169 @@ std::string ha_mcs_impl_viewtablelock( cal_impl_if::cal_connection_info& ci, ex // Any bulk rollback that is pending will be applied before the table // lock is released. //------------------------------------------------------------------------------ -std::string ha_mcs_impl_cleartablelock( - cal_impl_if::cal_connection_info& ci, - uint64_t tableLockID) +std::string ha_mcs_impl_cleartablelock(cal_impl_if::cal_connection_info& ci, uint64_t tableLockID) { - execplan::CalpontSystemCatalog::TableName tblName; - THD* thd = current_thd; - ulong sessionID = tid2sid(thd->thread_id); - std::string tableLockInfo; - BRM::TableLockInfo lockInfo; + execplan::CalpontSystemCatalog::TableName tblName; + THD* thd = current_thd; + ulong sessionID = tid2sid(thd->thread_id); + std::string tableLockInfo; + BRM::TableLockInfo lockInfo; - // Perform preliminary setup. CalpontDMLPackage expects schema and table - // name to be provided, so we get the table OID for the specified table - // lock, and then get the table name for the applicable table OID. - std::string prelimTask; + // Perform preliminary setup. CalpontDMLPackage expects schema and table + // name to be provided, so we get the table OID for the specified table + // lock, and then get the table name for the applicable table OID. + std::string prelimTask; - try + try + { + BRM::DBRM brm; + prelimTask = "getting table locks from BRM."; + bool getLockInfo = brm.getTableLockInfo(tableLockID, &lockInfo); + + if (!getLockInfo) { - BRM::DBRM brm; - prelimTask = "getting table locks from BRM."; - bool getLockInfo = brm.getTableLockInfo(tableLockID, &lockInfo); - - if (!getLockInfo) - { - tableLockInfo = "No table lock found for specified table lock ID"; - return tableLockInfo; - } - - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - - prelimTask = "getting table name from system catalog."; - tblName = csc->tableName( lockInfo.tableOID ); - } - catch (std::exception& ex) - { - std::string eMsg(ex.what()); - eMsg += " Error "; - eMsg += prelimTask; - - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, eMsg.c_str()); - return tableLockInfo; - } - catch (...) - { - std::string eMsg(" Error "); - eMsg += prelimTask; - - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, eMsg.c_str()); - return tableLockInfo; + tableLockInfo = "No table lock found for specified table lock ID"; + return tableLockInfo; } - std::string dmlStatement( "CLEARTABLELOCK" ); - VendorDMLStatement cmdStmt(dmlStatement, DML_COMMAND, sessionID); - CalpontDMLPackage* pDMLPackage = - CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer( cmdStmt ); - pDMLPackage->set_SchemaName(tblName.schema); - pDMLPackage->set_TableName (tblName.table ); + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); - // Table lock ID is passed in the SQL statement attribute - std::ostringstream lockIDString; - lockIDString << tableLockID; - pDMLPackage->set_SQLStatement( lockIDString.str() ); - - ByteStream bytestream; - bytestream << static_cast(sessionID); - pDMLPackage->write(bytestream); - delete pDMLPackage; - - ByteStream::byte b = 0; - ByteStream::octbyte rows; - std::string errorMsg; - - try - { - ci.dmlProc->write(bytestream); - bytestream = ci.dmlProc->read(); - - if ( bytestream.length() == 0 ) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [7]"); - } - else - { - bytestream >> b; - bytestream >> rows; - bytestream >> errorMsg; - bytestream >> tableLockInfo; - } - - } - catch (runtime_error&) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [8]"); - } - catch (...) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Caught unknown error"); - } - - //@Bug 2606. Send error message back to sql session - if ( b != 0 ) - tableLockInfo = errorMsg; + prelimTask = "getting table name from system catalog."; + tblName = csc->tableName(lockInfo.tableOID); + } + catch (std::exception& ex) + { + std::string eMsg(ex.what()); + eMsg += " Error "; + eMsg += prelimTask; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, eMsg.c_str()); return tableLockInfo; -} + } + catch (...) + { + std::string eMsg(" Error "); + eMsg += prelimTask; -int ha_mcs_impl_commit_ (handlerton* hton, THD* thd, bool all, cal_connection_info& ci ) -{ - if (thd->slave_thread && !get_replication_slave(thd)) - return 0; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, eMsg.c_str()); + return tableLockInfo; + } - int rc = 0; + std::string dmlStatement("CLEARTABLELOCK"); + VendorDMLStatement cmdStmt(dmlStatement, DML_COMMAND, sessionID); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + pDMLPackage->set_SchemaName(tblName.schema); + pDMLPackage->set_TableName(tblName.table); - std::string command("COMMIT"); -#ifdef INFINIDB_DEBUG - cout << "COMMIT" << endl; -#endif - rc = ProcessCommandStatement(thd, command, ci); - return rc; -} + // Table lock ID is passed in the SQL statement attribute + std::ostringstream lockIDString; + lockIDString << tableLockID; + pDMLPackage->set_SQLStatement(lockIDString.str()); -int ha_mcs_impl_rollback_ (handlerton* hton, THD* thd, bool all, cal_connection_info& ci) -{ - int rc = 0; -#ifdef INFINIDB_DEBUG - cout << "ROLLBACK" << endl; -#endif + ByteStream bytestream; + bytestream << static_cast(sessionID); + pDMLPackage->write(bytestream); + delete pDMLPackage; - if (useHdfs) + ByteStream::byte b = 0; + ByteStream::octbyte rows; + std::string errorMsg; + + try + { + ci.dmlProc->write(bytestream); + bytestream = ci.dmlProc->read(); + + if (bytestream.length() == 0) { - string msg = string("Some non-transactional changed tables couldn't be rolled back"); - // cout << "Some non-transactional changed tables couldn't be rolled back" << endl; - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 1196, msg.c_str()); - return rc; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [7]"); } + else + { + bytestream >> b; + bytestream >> rows; + bytestream >> errorMsg; + bytestream >> tableLockInfo; + } + } + catch (runtime_error&) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DMLProc [8]"); + } + catch (...) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Caught unknown error"); + } - std::string command("ROLLBACK"); - rc = ProcessCommandStatement(thd, command, ci); - return rc; + //@Bug 2606. Send error message back to sql session + if (b != 0) + tableLockInfo = errorMsg; + + return tableLockInfo; } -int ha_mcs_impl_close_connection_ (handlerton* hton, THD* thd, cal_connection_info& ci ) +int ha_mcs_impl_commit_(handlerton* hton, THD* thd, bool all, cal_connection_info& ci) { - int rc = 0; + if (thd->slave_thread && !get_replication_slave(thd)) + return 0; + + int rc = 0; + + std::string command("COMMIT"); #ifdef INFINIDB_DEBUG - cout << "Close connection session ID " << thd->thread_id << endl; + cout << "COMMIT" << endl; +#endif + rc = ProcessCommandStatement(thd, command, ci); + return rc; +} + +int ha_mcs_impl_rollback_(handlerton* hton, THD* thd, bool all, cal_connection_info& ci) +{ + int rc = 0; +#ifdef INFINIDB_DEBUG + cout << "ROLLBACK" << endl; #endif - if ( !ci.dmlProc ) - { - return rc; - } - - std::string command("CLEANUP"); - rc = ProcessCommandStatement(thd, command, ci); - // @bug 1622. remove calpontsystemcatalog and close the socket when session quit. - // @info when Calpont process a select query, an alter table phase is involved in - // the vtable design, which will auto start a transaction. when autocommit on (by default), rollback is automically called - // when session quit. rollback can also be called by user explicitly to rollback - // a transaction. Under either situation, system catalog cache for this session should - // be removed + if (useHdfs) + { + string msg = string("Some non-transactional changed tables couldn't be rolled back"); + // cout << "Some non-transactional changed tables couldn't be rolled back" << endl; + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 1196, msg.c_str()); return rc; + } + + std::string command("ROLLBACK"); + rc = ProcessCommandStatement(thd, command, ci); + return rc; +} + +int ha_mcs_impl_close_connection_(handlerton* hton, THD* thd, cal_connection_info& ci) +{ + int rc = 0; +#ifdef INFINIDB_DEBUG + cout << "Close connection session ID " << thd->thread_id << endl; +#endif + + if (!ci.dmlProc) + { + return rc; + } + + std::string command("CLEANUP"); + rc = ProcessCommandStatement(thd, command, ci); + // @bug 1622. remove calpontsystemcatalog and close the socket when session quit. + // @info when Calpont process a select query, an alter table phase is involved in + // the vtable design, which will auto start a transaction. when autocommit on (by default), rollback is + // automically called when session quit. rollback can also be called by user explicitly to rollback a + // transaction. Under either situation, system catalog cache for this session should be removed + return rc; } // vim:ts=4 sw=4: - diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp old mode 100755 new mode 100644 index 9f6087ad5..7a7a8978b --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -107,82 +107,81 @@ const uint64_t CORRELATED = 0x08; // HWM => HighWaterMark class RecursionCounter { -private: - RecursionCounter() {} -public: - RecursionCounter(gp_walk_info* gwip) : fgwip(gwip) + private: + RecursionCounter() + { + } + + public: + RecursionCounter(gp_walk_info* gwip) : fgwip(gwip) + { + ++fgwip->recursionLevel; + + if (fgwip->recursionLevel > fgwip->recursionHWM) { - ++fgwip->recursionLevel; - - if (fgwip->recursionLevel > fgwip->recursionHWM) - { - fgwip->rcBookMarkStack.push(fgwip->rcWorkStack.size()); - fgwip->recursionHWM = fgwip->recursionLevel; - } + fgwip->rcBookMarkStack.push(fgwip->rcWorkStack.size()); + fgwip->recursionHWM = fgwip->recursionLevel; } - ~RecursionCounter() + } + ~RecursionCounter() + { + --fgwip->recursionLevel; + + if (fgwip->recursionLevel < fgwip->recursionHWM - 1) { - --fgwip->recursionLevel; - - if (fgwip->recursionLevel < fgwip->recursionHWM - 1) - { - fgwip->rcBookMarkStack.pop(); - --fgwip->recursionHWM; - } + fgwip->rcBookMarkStack.pop(); + --fgwip->recursionHWM; } + } - gp_walk_info* fgwip; + gp_walk_info* fgwip; }; #include "ha_view.h" namespace cal_impl_if { - // This is taken from Item_cond::fix_fields in sql/item_cmpfunc.cc. -void calculateNotNullTables(const std::vector& condList, - table_map& not_null_tables) +void calculateNotNullTables(const std::vector& condList, table_map& not_null_tables) { - for (Item* item : condList) + for (Item* item : condList) + { + if (item->can_eval_in_optimize() && !item->with_sp_var() && !cond_has_datetime_is_null(item)) { - if (item->can_eval_in_optimize() && !item->with_sp_var() && - !cond_has_datetime_is_null(item)) - { - if (item->eval_const_cond()) - { - } - else - { - not_null_tables = (table_map) 0; - } - } - else - { - not_null_tables |= item->not_null_tables(); - } + if (item->eval_const_cond()) + { + } + else + { + not_null_tables = (table_map)0; + } } + else + { + not_null_tables |= item->not_null_tables(); + } + } } // Recursively iterate through the join_list and store all non-null // TABLE_LIST::on_expr items to a hash map keyed by the TABLE_LIST ptr. // This is then used by convertOuterJoinToInnerJoin(). -void buildTableOnExprList(List* join_list, - TableOnExprList& tableOnExprList) +void buildTableOnExprList(List* join_list, TableOnExprList& tableOnExprList) { - TABLE_LIST *table; - NESTED_JOIN *nested_join; - List_iterator li(*join_list); + TABLE_LIST* table; + NESTED_JOIN* nested_join; + List_iterator li(*join_list); - while ((table = li++)) + while ((table = li++)) + { + if ((nested_join = table->nested_join)) { - if ((nested_join = table->nested_join)) - { - buildTableOnExprList(&nested_join->join_list, tableOnExprList); - } - - if (table->on_expr) - tableOnExprList[table].push_back(table->on_expr); + buildTableOnExprList(&nested_join->join_list, tableOnExprList); } + + if (table->on_expr) + tableOnExprList[table].push_back(table->on_expr); + } } // This is a trimmed down version of simplify_joins() in sql/sql_select.cc. @@ -193,140 +192,131 @@ void buildTableOnExprList(List* join_list, // the SELECT_LEX in a scenario where the select_handler execution has failed // and we want to fallback to the server execution (MCOL-4525). Here, we mimick // the creation of Item_cond_and using tableOnExprList and condList. -void convertOuterJoinToInnerJoin(List* join_list, - TableOnExprList& tableOnExprList, - std::vector& condList, - TableOuterJoinMap& tableOuterJoinMap) +void convertOuterJoinToInnerJoin(List* join_list, TableOnExprList& tableOnExprList, + std::vector& condList, TableOuterJoinMap& tableOuterJoinMap) { - TABLE_LIST *table; - NESTED_JOIN *nested_join; - List_iterator li(*join_list); + TABLE_LIST* table; + NESTED_JOIN* nested_join; + List_iterator li(*join_list); - while ((table = li++)) + while ((table = li++)) + { + table_map used_tables; + table_map not_null_tables = (table_map)0; + + if ((nested_join = table->nested_join)) { - table_map used_tables; - table_map not_null_tables = (table_map) 0; + auto iter = tableOnExprList.find(table); - if ((nested_join = table->nested_join)) + if ((iter != tableOnExprList.end()) && !iter->second.empty()) + { + convertOuterJoinToInnerJoin(&nested_join->join_list, tableOnExprList, tableOnExprList[table], + tableOuterJoinMap); + } + + nested_join->used_tables = (table_map)0; + nested_join->not_null_tables = (table_map)0; + + convertOuterJoinToInnerJoin(&nested_join->join_list, tableOnExprList, condList, tableOuterJoinMap); + + used_tables = nested_join->used_tables; + not_null_tables = nested_join->not_null_tables; + } + else + { + used_tables = table->get_map(); + + if (!condList.empty()) + { + if (condList.size() == 1) { - auto iter = tableOnExprList.find(table); - - if ((iter != tableOnExprList.end()) && !iter->second.empty()) - { - convertOuterJoinToInnerJoin(&nested_join->join_list, - tableOnExprList, tableOnExprList[table], tableOuterJoinMap); - } - - nested_join->used_tables = (table_map) 0; - nested_join->not_null_tables = (table_map) 0; - - convertOuterJoinToInnerJoin(&nested_join->join_list, - tableOnExprList, condList, tableOuterJoinMap); - - used_tables = nested_join->used_tables; - not_null_tables = nested_join->not_null_tables; + not_null_tables = condList[0]->not_null_tables(); } else { - used_tables = table->get_map(); - - if (!condList.empty()) - { - if (condList.size() == 1) - { - not_null_tables = condList[0]->not_null_tables(); - } - else - { - calculateNotNullTables(condList, not_null_tables); - } - } - } - - if (table->embedding) - { - table->embedding->nested_join->used_tables |= used_tables; - table->embedding->nested_join->not_null_tables |= not_null_tables; - } - - if (!(table->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) || - (used_tables & not_null_tables)) - { - if (table->outer_join) - { - tableOuterJoinMap[table] = table->outer_join; - } - - table->outer_join = 0; - - auto iter = tableOnExprList.find(table); - - if (iter != tableOnExprList.end() && !iter->second.empty()) - { - // The original implementation in simplify_joins() creates - // an Item_cond_and object here. Instead of doing so, we - // append the table->on_expr to the condList and hence avoid - // making any permanent changes to SELECT_LEX::JOIN::conds. - condList.insert(condList.end(), - tableOnExprList[table].begin(), tableOnExprList[table].end()); - iter->second.clear(); - } + calculateNotNullTables(condList, not_null_tables); } + } } + + if (table->embedding) + { + table->embedding->nested_join->used_tables |= used_tables; + table->embedding->nested_join->not_null_tables |= not_null_tables; + } + + if (!(table->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) || (used_tables & not_null_tables)) + { + if (table->outer_join) + { + tableOuterJoinMap[table] = table->outer_join; + } + + table->outer_join = 0; + + auto iter = tableOnExprList.find(table); + + if (iter != tableOnExprList.end() && !iter->second.empty()) + { + // The original implementation in simplify_joins() creates + // an Item_cond_and object here. Instead of doing so, we + // append the table->on_expr to the condList and hence avoid + // making any permanent changes to SELECT_LEX::JOIN::conds. + condList.insert(condList.end(), tableOnExprList[table].begin(), tableOnExprList[table].end()); + iter->second.clear(); + } + } + } } CalpontSystemCatalog::TableAliasName makeTableAliasName(TABLE_LIST* table) { - return make_aliasview( - (table->db.length ? table->db.str : ""), - (table->table_name.length ? table->table_name.str : ""), - (table->alias.length ? table->alias.str : ""), - getViewName(table), true, lower_case_table_names); + return make_aliasview( + (table->db.length ? table->db.str : ""), (table->table_name.length ? table->table_name.str : ""), + (table->alias.length ? table->alias.str : ""), getViewName(table), true, lower_case_table_names); } //@bug5228. need to escape backtick ` string escapeBackTick(const char* str) { - if (!str) - return ""; + if (!str) + return ""; - string ret; + string ret; - for (uint32_t i = 0; str[i] != 0; i++) - { - if (str[i] == '`') - ret.append("``"); - else - ret.append(1, str[i]); - } + for (uint32_t i = 0; str[i] != 0; i++) + { + if (str[i] == '`') + ret.append("``"); + else + ret.append(1, str[i]); + } - return ret; + return ret; } void clearStacks(gp_walk_info& gwi) { - while (!gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); + while (!gwi.rcWorkStack.empty()) + gwi.rcWorkStack.pop(); - while (!gwi.ptWorkStack.empty()) - gwi.ptWorkStack.pop(); + while (!gwi.ptWorkStack.empty()) + gwi.ptWorkStack.pop(); } bool nonConstFunc(Item_func* ifp) { - if (strcasecmp(ifp->func_name(), "rand") == 0 || - strcasecmp(ifp->func_name(), "sysdate") == 0 || - strcasecmp(ifp->func_name(), "idblocalpm") == 0) - return true; + if (strcasecmp(ifp->func_name(), "rand") == 0 || strcasecmp(ifp->func_name(), "sysdate") == 0 || + strcasecmp(ifp->func_name(), "idblocalpm") == 0) + return true; - for (uint32_t i = 0; i < ifp->argument_count(); i++) - { - if (ifp->arguments()[i]->type() == Item::FUNC_ITEM && - nonConstFunc(((Item_func*)ifp->arguments()[i]))) - return true; - } + for (uint32_t i = 0; i < ifp->argument_count(); i++) + { + if (ifp->arguments()[i]->type() == Item::FUNC_ITEM && nonConstFunc(((Item_func*)ifp->arguments()[i]))) + return true; + } - return false; + return false; } /*@brief getColNameFromItem - builds a name from an Item */ @@ -344,39 +334,39 @@ bool nonConstFunc(Item_func* ifp) ***********************************************************/ void getColNameFromItem(std::ostringstream& ostream, Item* item) { -// Item_func doesn't have proper db.table.field values -// inherited from Item_ident. TBD what is the valid output. -// !!!dynamic_cast fails compilation - ostream << "'"; + // Item_func doesn't have proper db.table.field values + // inherited from Item_ident. TBD what is the valid output. + // !!!dynamic_cast fails compilation + ostream << "'"; - if (item->type() != Item::FIELD_ITEM) - { - ostream << "unknown db" << '.'; - ostream << "unknown table" << '.'; - ostream << "unknown field"; - } + if (item->type() != Item::FIELD_ITEM) + { + ostream << "unknown db" << '.'; + ostream << "unknown table" << '.'; + ostream << "unknown field"; + } + else + { + Item_ident* iip = reinterpret_cast(item); + + if (iip->db_name.str) + ostream << iip->db_name.str << '.'; else - { - Item_ident* iip = reinterpret_cast(item); + ostream << "unknown db" << '.'; - if (iip->db_name.str) - ostream << iip->db_name.str << '.'; - else - ostream << "unknown db" << '.'; + if (iip->table_name.str) + ostream << iip->table_name.str << '.'; + else + ostream << "unknown table" << '.'; - if (iip->table_name.str) - ostream << iip->table_name.str << '.'; - else - ostream << "unknown table" << '.'; + if (iip->field_name.length) + ostream << iip->field_name.str; + else + ostream << "unknown field"; + } - if (iip->field_name.length) - ostream << iip->field_name.str; - else - ostream << "unknown field"; - } - - ostream << "'"; - return; + ostream << "'"; + return; } /*@brf sortItemIsInGroupRec - seeks for an item in grouping*/ @@ -393,79 +383,77 @@ void getColNameFromItem(std::ostringstream& ostream, Item* item) ***********************************************************/ bool sortItemIsInGroupRec(Item* sort_item, Item* group_item) { - bool found = false; - // If ITEM_REF::ref is NULL - if (sort_item == NULL) - { - return found; - } - - Item_func* ifp_sort = reinterpret_cast(sort_item); - - // base cases for Item_field and Item_ref. The second arg is binary cmp switch - found = group_item->eq(sort_item, false); - if (!found && sort_item->type() == Item::REF_ITEM) - { - Item_ref* ifp_sort_ref = reinterpret_cast(sort_item); - found = sortItemIsInGroupRec(*ifp_sort_ref->ref, group_item); - } - else if (!found && sort_item->type() == Item::FIELD_ITEM) - { - return found; - } - - // seeking for a group_item match - for (uint32_t i = 0; !found && i < ifp_sort->argument_count(); i++) - { - Item* ifp_sort_arg = ifp_sort->arguments()[i]; - if (ifp_sort_arg->type() == Item::FUNC_ITEM - || ifp_sort_arg->type() == Item::FIELD_ITEM) - { - Item* ifp_sort_arg = ifp_sort->arguments()[i]; - found = sortItemIsInGroupRec(ifp_sort_arg, group_item); - } - else if (ifp_sort_arg->type() == Item::REF_ITEM) - { - // dereference the Item - Item_ref* ifp_sort_ref = reinterpret_cast(ifp_sort_arg); - found = sortItemIsInGroupRec(*ifp_sort_ref->ref, group_item); - } - } - + bool found = false; + // If ITEM_REF::ref is NULL + if (sort_item == NULL) + { return found; + } + + Item_func* ifp_sort = reinterpret_cast(sort_item); + + // base cases for Item_field and Item_ref. The second arg is binary cmp switch + found = group_item->eq(sort_item, false); + if (!found && sort_item->type() == Item::REF_ITEM) + { + Item_ref* ifp_sort_ref = reinterpret_cast(sort_item); + found = sortItemIsInGroupRec(*ifp_sort_ref->ref, group_item); + } + else if (!found && sort_item->type() == Item::FIELD_ITEM) + { + return found; + } + + // seeking for a group_item match + for (uint32_t i = 0; !found && i < ifp_sort->argument_count(); i++) + { + Item* ifp_sort_arg = ifp_sort->arguments()[i]; + if (ifp_sort_arg->type() == Item::FUNC_ITEM || ifp_sort_arg->type() == Item::FIELD_ITEM) + { + Item* ifp_sort_arg = ifp_sort->arguments()[i]; + found = sortItemIsInGroupRec(ifp_sort_arg, group_item); + } + else if (ifp_sort_arg->type() == Item::REF_ITEM) + { + // dereference the Item + Item_ref* ifp_sort_ref = reinterpret_cast(ifp_sort_arg); + found = sortItemIsInGroupRec(*ifp_sort_ref->ref, group_item); + } + } + + return found; } /*@brief check_sum_func_item - This traverses Item */ /********************************************************** -* DESCRIPTION: -* This f() walks Item looking for the existence of -* a Item::REF_ITEM, which references an item of -* type Item::SUM_FUNC_ITEM -* PARAMETERS: -* Item * Item to traverse -* RETURN: -*********************************************************/ + * DESCRIPTION: + * This f() walks Item looking for the existence of + * a Item::REF_ITEM, which references an item of + * type Item::SUM_FUNC_ITEM + * PARAMETERS: + * Item * Item to traverse + * RETURN: + *********************************************************/ void check_sum_func_item(const Item* item, void* arg) { - bool* found = reinterpret_cast(arg); + bool* found = reinterpret_cast(arg); - if (*found) - return; + if (*found) + return; - if (item->type() == Item::REF_ITEM) + if (item->type() == Item::REF_ITEM) + { + const Item_ref* ref_item = reinterpret_cast(item); + Item* ref_item_item = (Item*)*ref_item->ref; + if (ref_item_item->type() == Item::SUM_FUNC_ITEM) { - const Item_ref* ref_item = reinterpret_cast(item); - Item* ref_item_item = (Item*) *ref_item->ref; - if (ref_item_item->type() == Item::SUM_FUNC_ITEM) - { - *found = true; - } + *found = true; } - else if (item->type() == Item::CONST_ITEM) - { - *found= true; - } - + } + else if (item->type() == Item::CONST_ITEM) + { + *found = true; + } } /*@brief sortItemIsInGrouping- seeks for an item in grouping*/ @@ -483,44 +471,42 @@ void check_sum_func_item(const Item* item, void* arg) ***********************************************************/ bool sortItemIsInGrouping(Item* sort_item, ORDER* groupcol) { - bool found = false; + bool found = false; - if(sort_item->type() == Item::SUM_FUNC_ITEM) - { - found = true; - } + if (sort_item->type() == Item::SUM_FUNC_ITEM) + { + found = true; + } - // A function that contains an aggregate function - // can be included in the ORDER BY clause - // e.g. select a, if (sum(b) > 1, 2, 1) from t1 group by 1 order by 2; - if (sort_item->type() == Item::FUNC_ITEM) - { - Item_func *ifp = reinterpret_cast(sort_item); - ifp->traverse_cond(check_sum_func_item, &found, Item::POSTFIX); - } - else if (sort_item->type() == Item::CONST_ITEM || - sort_item->type() == Item::WINDOW_FUNC_ITEM) - { - found = true; - } + // A function that contains an aggregate function + // can be included in the ORDER BY clause + // e.g. select a, if (sum(b) > 1, 2, 1) from t1 group by 1 order by 2; + if (sort_item->type() == Item::FUNC_ITEM) + { + Item_func* ifp = reinterpret_cast(sort_item); + ifp->traverse_cond(check_sum_func_item, &found, Item::POSTFIX); + } + else if (sort_item->type() == Item::CONST_ITEM || sort_item->type() == Item::WINDOW_FUNC_ITEM) + { + found = true; + } - for (; !found && groupcol; groupcol = groupcol->next) + for (; !found && groupcol; groupcol = groupcol->next) + { + Item* group_item = *(groupcol->item); + found = (group_item->eq(sort_item, false)) ? true : false; + // Detect aggregation functions first then traverse + // if sort field is a Func and group field + // is either Field or Func + // Consider nonConstFunc() check here + if (!found && sort_item->type() == Item::FUNC_ITEM && + (group_item->type() == Item::FUNC_ITEM || group_item->type() == Item::FIELD_ITEM)) { - Item* group_item = *(groupcol->item); - found = (group_item->eq(sort_item, false)) ? true : false; - // Detect aggregation functions first then traverse - // if sort field is a Func and group field - // is either Field or Func - // Consider nonConstFunc() check here - if(!found && sort_item->type() == Item::FUNC_ITEM - && (group_item->type() == Item::FUNC_ITEM - || group_item->type() == Item::FIELD_ITEM)) - { - found = sortItemIsInGroupRec(sort_item, group_item); - } + found = sortItemIsInGroupRec(sort_item, group_item); } + } - return found; + return found; } /*@brief buildAggFrmTempField- build aggr func from extSELECT list item*/ @@ -539,44 +525,41 @@ bool sortItemIsInGrouping(Item* sort_item, ORDER* groupcol) ***********************************************************/ ReturnedColumn* buildAggFrmTempField(Item* item, gp_walk_info& gwi) { - ReturnedColumn* result = NULL; - Item_field* ifip = NULL; - Item_ref* irip; - Item_func_or_sum* isfp; + ReturnedColumn* result = NULL; + Item_field* ifip = NULL; + Item_ref* irip; + Item_func_or_sum* isfp; - switch ( item->type() ) + switch (item->type()) + { + case Item::FIELD_ITEM: ifip = reinterpret_cast(item); break; + default: + irip = reinterpret_cast(item); + if (irip) + ifip = reinterpret_cast(irip->ref[0]); + break; + } + + if (ifip && ifip->field) + { + std::vector::iterator iter = gwi.extSelAggColsItems.begin(); + for (; iter != gwi.extSelAggColsItems.end(); iter++) { - case Item::FIELD_ITEM: - ifip = reinterpret_cast(item); - break; - default: - irip = reinterpret_cast(item); - if ( irip ) - ifip = reinterpret_cast(irip->ref[0]); - break; + isfp = reinterpret_cast(*iter); + + if (isfp->type() == Item::SUM_FUNC_ITEM && isfp->result_field == ifip->field) + { + ReturnedColumn* rc = buildAggregateColumn(isfp, gwi); + + if (rc) + result = rc; + + break; + } } + } - if (ifip && ifip->field) - { - std::vector::iterator iter = gwi.extSelAggColsItems.begin(); - for ( ; iter != gwi.extSelAggColsItems.end(); iter++ ) - { - isfp = reinterpret_cast(*iter); - - if ( isfp->type() == Item::SUM_FUNC_ITEM && - isfp->result_field == ifip->field ) - { - ReturnedColumn* rc = buildAggregateColumn(isfp, gwi); - - if (rc) - result = rc; - - break; - } - } - } - - return result; + return result; } /*@brief isDuplicateSF - search for a duplicate SimpleFilter*/ @@ -598,19 +581,19 @@ ReturnedColumn* buildAggFrmTempField(Item* item, gp_walk_info& gwi) * USED * buildPredicateItem() ***********************************************************/ -bool isDuplicateSF(gp_walk_info *gwip, execplan::SimpleFilter *sfp) +bool isDuplicateSF(gp_walk_info* gwip, execplan::SimpleFilter* sfp) { - List_iterator it(gwip->equiCondSFList); - execplan::SimpleFilter *isfp; - while((isfp = it++)) + List_iterator it(gwip->equiCondSFList); + execplan::SimpleFilter* isfp; + while ((isfp = it++)) + { + if (sfp->semanticEq(*isfp)) { - if (sfp->semanticEq(*isfp)) - { - return true; - } + return true; } + } - return false; + return false; } // DESCRIPTION: @@ -621,609 +604,617 @@ bool isDuplicateSF(gp_walk_info *gwip, execplan::SimpleFilter *sfp) // bool inline bool isUDFSumItem(const Item_sum* isp) { - return typeid(*isp) == typeid(Item_sum_udf_int) || - typeid(*isp) == typeid(Item_sum_udf_float) || - typeid(*isp) == typeid(Item_sum_udf_decimal) || - typeid(*isp) == typeid(Item_sum_udf_str); + return typeid(*isp) == typeid(Item_sum_udf_int) || typeid(*isp) == typeid(Item_sum_udf_float) || + typeid(*isp) == typeid(Item_sum_udf_decimal) || typeid(*isp) == typeid(Item_sum_udf_str); } string getViewName(TABLE_LIST* table_ptr) { - string viewName = ""; - - if (!table_ptr) - return viewName; - - TABLE_LIST* view = table_ptr->referencing_view; - - if (view) - { - if (!view->derived) - viewName = view->alias.str; - - while ((view = view->referencing_view)) - { - if (view->derived) continue; - - viewName = view->alias.str + string(".") + viewName; - } - } + string viewName = ""; + if (!table_ptr) return viewName; + + TABLE_LIST* view = table_ptr->referencing_view; + + if (view) + { + if (!view->derived) + viewName = view->alias.str; + + while ((view = view->referencing_view)) + { + if (view->derived) + continue; + + viewName = view->alias.str + string(".") + viewName; + } + } + + return viewName; } #ifdef DEBUG_WALK_COND void debug_walk(const Item* item, void* arg) { - switch (item->type()) + switch (item->type()) + { + case Item::FIELD_ITEM: { - case Item::FIELD_ITEM: + Item_field* ifp = (Item_field*)item; + cerr << "FIELD_ITEM: " << (ifp->db_name.str ? ifp->db_name.str : "") << '.' << bestTableName(ifp) << '.' + << ifp->field_name.str << endl; + break; + } + case Item::CONST_ITEM: + { + switch (item->cmp_type()) + { + case INT_RESULT: { - Item_field* ifp = (Item_field*)item; - cerr << "FIELD_ITEM: " << (ifp->db_name.str ? ifp->db_name.str : "") << '.' << bestTableName(ifp) << - '.' << ifp->field_name.str << endl; - break; + Item_int* iip = (Item_int*)item; + cerr << "INT_ITEM: "; + + if (iip->name.length) + cerr << iip->name.str << " (from name string)" << endl; + else + cerr << iip->val_int() << endl; + + break; } - case Item::CONST_ITEM: + case STRING_RESULT: { - switch (item->cmp_type()) - { - case INT_RESULT: - { - Item_int* iip = (Item_int*)item; - cerr << "INT_ITEM: "; - - if (iip->name.length) cerr << iip->name.str << " (from name string)" << endl; - else cerr << iip->val_int() << endl; - - break; - } - case STRING_RESULT: - { - Item_string* isp = (Item_string*)item; - String val, *str = isp->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - cerr << "STRING_ITEM: >" << valStr << '<' << endl; - break; - } - case REAL_RESULT: - { - cerr << "REAL_ITEM" << endl; - break; - } - case DECIMAL_RESULT: - { - cerr << "DECIMAL_ITEM" << endl; - break; - } - case TIME_RESULT: - { - String val, *str = NULL; - Item_temporal_literal* itp = (Item_temporal_literal*)item; - str = itp->val_str(&val); - cerr << "DATE ITEM: "; - - if (str) - cerr << ": (" << str->ptr() << ')' << endl; - else - cerr << ": " << endl; - - break; - } - default: - { - cerr << ": Unknown cmp_type" << endl; - break; - } - } - break; + Item_string* isp = (Item_string*)item; + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + cerr << "STRING_ITEM: >" << valStr << '<' << endl; + break; } - case Item::FUNC_ITEM: + case REAL_RESULT: { - Item_func* ifp = (Item_func*)item; - Item_func_opt_neg* inp; - cerr << "FUNC_ITEM: "; + cerr << "REAL_ITEM" << endl; + break; + } + case DECIMAL_RESULT: + { + cerr << "DECIMAL_ITEM" << endl; + break; + } + case TIME_RESULT: + { + String val, *str = NULL; + Item_temporal_literal* itp = (Item_temporal_literal*)item; + str = itp->val_str(&val); + cerr << "DATE ITEM: "; - switch (ifp->functype()) - { - case Item_func::UNKNOWN_FUNC: // 0 - cerr << ifp->func_name() << " (" << ifp->functype() << ")" << endl; - break; + if (str) + cerr << ": (" << str->ptr() << ')' << endl; + else + cerr << ": " << endl; - case Item_func::GT_FUNC: // 7 - cerr << '>' << " (" << ifp->functype() << ")" << endl; - break; + break; + } + default: + { + cerr << ": Unknown cmp_type" << endl; + break; + } + } + break; + } + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)item; + Item_func_opt_neg* inp; + cerr << "FUNC_ITEM: "; - case Item_func::EQ_FUNC: // 1 - cerr << '=' << " (" << ifp->functype() << ")" << endl; - break; + switch (ifp->functype()) + { + case Item_func::UNKNOWN_FUNC: // 0 + cerr << ifp->func_name() << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::GE_FUNC: - cerr << ">=" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::GT_FUNC: // 7 + cerr << '>' << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::LE_FUNC: - cerr << "<=" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::EQ_FUNC: // 1 + cerr << '=' << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::LT_FUNC: - cerr << '<' << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::GE_FUNC: + cerr << ">=" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::NE_FUNC: - cerr << "<>" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::LE_FUNC: + cerr << "<=" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::NEG_FUNC: // 45 - cerr << "unary minus" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::LT_FUNC: cerr << '<' << " (" << ifp->functype() << ")" << endl; break; - case Item_func::IN_FUNC: // 16 - inp = (Item_func_opt_neg*)ifp; + case Item_func::NE_FUNC: + cerr << "<>" + << " (" << ifp->functype() << ")" << endl; + break; - if (inp->negated) cerr << "not "; + case Item_func::NEG_FUNC: // 45 + cerr << "unary minus" + << " (" << ifp->functype() << ")" << endl; + break; - cerr << "in" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::IN_FUNC: // 16 + inp = (Item_func_opt_neg*)ifp; - case Item_func::BETWEEN: - inp = (Item_func_opt_neg*)ifp; + if (inp->negated) + cerr << "not "; - if (inp->negated) cerr << "not "; + cerr << "in" + << " (" << ifp->functype() << ")" << endl; + break; - cerr << "between" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::BETWEEN: + inp = (Item_func_opt_neg*)ifp; - case Item_func::ISNULL_FUNC: // 10 - cerr << "is null" << " (" << ifp->functype() << ")" << endl; - break; + if (inp->negated) + cerr << "not "; - case Item_func::ISNOTNULL_FUNC: // 11 - cerr << "is not null" << " (" << ifp->functype() << ")" << endl; - break; + cerr << "between" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::NOT_ALL_FUNC: - cerr << "not_all" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::ISNULL_FUNC: // 10 + cerr << "is null" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::NOT_FUNC: - cerr << "not_func" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::ISNOTNULL_FUNC: // 11 + cerr << "is not null" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::TRIG_COND_FUNC: - cerr << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::NOT_ALL_FUNC: + cerr << "not_all" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::ISNOTNULLTEST_FUNC: - cerr << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; - break; + case Item_func::NOT_FUNC: + cerr << "not_func" + << " (" << ifp->functype() << ")" << endl; + break; - case Item_func::MULT_EQUAL_FUNC: - { - cerr << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; - Item_equal* item_eq = (Item_equal*)ifp; - Item_equal_fields_iterator it(*item_eq); - Item* item; + case Item_func::TRIG_COND_FUNC: + cerr << "trig_cond_func" + << " (" << ifp->functype() << ")" << endl; + break; - while ((item = it++)) - { - Field* equal_field = it.get_curr_field(); - cerr << equal_field->field_name.str << endl; - } + case Item_func::ISNOTNULLTEST_FUNC: + cerr << "isnotnulltest_func" + << " (" << ifp->functype() << ")" << endl; + break; - break; - } + case Item_func::MULT_EQUAL_FUNC: + { + cerr << "mult_equal_func:" + << " (" << ifp->functype() << ")" << endl; + Item_equal* item_eq = (Item_equal*)ifp; + Item_equal_fields_iterator it(*item_eq); + Item* item; - case Item_func::EQUAL_FUNC: - cerr << "equal func" << " (" << ifp->functype() << ")" << endl; - break; + while ((item = it++)) + { + Field* equal_field = it.get_curr_field(); + cerr << equal_field->field_name.str << endl; + } - case Item_func::FT_FUNC: - cerr << "ft func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::LIKE_FUNC: - cerr << "like func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::COND_AND_FUNC: - cerr << "cond and func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::COND_OR_FUNC: - cerr << "cond or func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::XOR_FUNC: - cerr << "xor func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::INTERVAL_FUNC: - cerr << "interval func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_EQUALS_FUNC: - cerr << "sp equals func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_DISJOINT_FUNC: - cerr << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_INTERSECTS_FUNC: - cerr << "sp intersects func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_TOUCHES_FUNC: - cerr << "sp touches func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_CROSSES_FUNC: - cerr << "sp crosses func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_WITHIN_FUNC: - cerr << "sp within func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_CONTAINS_FUNC: - cerr << "sp contains func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_OVERLAPS_FUNC: - cerr << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_STARTPOINT: - cerr << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_ENDPOINT: - cerr << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_EXTERIORRING: - cerr << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_POINTN: - cerr << "sp pointn func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_GEOMETRYN: - cerr << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_INTERIORRINGN: - cerr << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SP_RELATE_FUNC: - cerr << "sp relate func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::NOW_FUNC: - cerr << "now func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::SUSERVAR_FUNC: - cerr << "suservar func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::GUSERVAR_FUNC: - cerr << "guservar func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::COLLATE_FUNC: - cerr << "collate func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::EXTRACT_FUNC: - cerr << "extract func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::CHAR_TYPECAST_FUNC: - cerr << "char typecast func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::FUNC_SP: - cerr << "func sp func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::UDF_FUNC: - cerr << "udf func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::GSYSVAR_FUNC: - cerr << "gsysvar func" << " (" << ifp->functype() << ")" << endl; - break; - - case Item_func::DYNCOL_FUNC: - cerr << "dyncol func" << " (" << ifp->functype() << ")" << endl; - break; - - default: - cerr << "type=" << ifp->functype() << endl; - break; - } - - break; + break; } - case Item::COND_ITEM: + case Item_func::EQUAL_FUNC: + cerr << "equal func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::FT_FUNC: + cerr << "ft func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::LIKE_FUNC: + cerr << "like func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::COND_AND_FUNC: + cerr << "cond and func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::COND_OR_FUNC: + cerr << "cond or func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::XOR_FUNC: + cerr << "xor func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::INTERVAL_FUNC: + cerr << "interval func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_EQUALS_FUNC: + cerr << "sp equals func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_DISJOINT_FUNC: + cerr << "sp disjoint func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_INTERSECTS_FUNC: + cerr << "sp intersects func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_TOUCHES_FUNC: + cerr << "sp touches func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_CROSSES_FUNC: + cerr << "sp crosses func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_WITHIN_FUNC: + cerr << "sp within func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_CONTAINS_FUNC: + cerr << "sp contains func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_OVERLAPS_FUNC: + cerr << "sp overlaps func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_STARTPOINT: + cerr << "sp startpoint func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_ENDPOINT: + cerr << "sp endpoint func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_EXTERIORRING: + cerr << "sp exteriorring func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_POINTN: + cerr << "sp pointn func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_GEOMETRYN: + cerr << "sp geometryn func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_INTERIORRINGN: + cerr << "sp exteriorringn func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SP_RELATE_FUNC: + cerr << "sp relate func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::NOW_FUNC: + cerr << "now func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::SUSERVAR_FUNC: + cerr << "suservar func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::GUSERVAR_FUNC: + cerr << "guservar func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::COLLATE_FUNC: + cerr << "collate func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::EXTRACT_FUNC: + cerr << "extract func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::CHAR_TYPECAST_FUNC: + cerr << "char typecast func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::FUNC_SP: + cerr << "func sp func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::UDF_FUNC: + cerr << "udf func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::GSYSVAR_FUNC: + cerr << "gsysvar func" + << " (" << ifp->functype() << ")" << endl; + break; + + case Item_func::DYNCOL_FUNC: + cerr << "dyncol func" + << " (" << ifp->functype() << ")" << endl; + break; + + default: cerr << "type=" << ifp->functype() << endl; break; + } + + break; + } + + case Item::COND_ITEM: + { + Item_cond* icp = (Item_cond*)item; + cerr << "COND_ITEM: " << icp->func_name() << endl; + break; + } + + case Item::SUM_FUNC_ITEM: + { + Item_sum* isp = (Item_sum*)item; + char* item_name = const_cast(item->name.str); + + // MCOL-1052 This is an extended SELECT list item + if (!item_name && isp->get_arg_count() && isp->get_arg(0)->name.length) + { + item_name = const_cast(isp->get_arg(0)->name.str); + } + else if (!item_name && isp->get_arg_count() && isp->get_arg(0)->type() == Item::CONST_ITEM && + isp->get_arg(0)->cmp_type() == INT_RESULT) + { + item_name = (char*)"INT||*"; + } + else if (!item_name) + { + item_name = (char*)""; + } + + switch (isp->sum_func()) + { + case Item_sum::SUM_FUNC: cerr << "SUM_FUNC: " << item_name << endl; break; + + case Item_sum::SUM_DISTINCT_FUNC: cerr << "SUM_DISTINCT_FUNC: " << item_name << endl; break; + + case Item_sum::AVG_FUNC: cerr << "AVG_FUNC: " << item_name << endl; break; + + case Item_sum::COUNT_FUNC: cerr << "COUNT_FUNC: " << item_name << endl; break; + + case Item_sum::COUNT_DISTINCT_FUNC: cerr << "COUNT_DISTINCT_FUNC: " << item_name << endl; break; + + case Item_sum::MIN_FUNC: cerr << "MIN_FUNC: " << item_name << endl; break; + + case Item_sum::MAX_FUNC: cerr << "MAX_FUNC: " << item_name << endl; break; + + case Item_sum::UDF_SUM_FUNC: cerr << "UDAF_FUNC: " << item_name << endl; break; + + default: cerr << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; break; + } + + break; + } + + case Item::SUBSELECT_ITEM: + { + Item_subselect* sub = (Item_subselect*)item; + cerr << "SUBSELECT Item: "; + + switch (sub->substype()) + { + case Item_subselect::EXISTS_SUBS: cerr << "EXISTS"; break; + + case Item_subselect::IN_SUBS: cerr << "IN"; break; + + default: cerr << sub->substype(); break; + } + + cerr << endl; + JOIN* join = sub->get_select_lex()->join; + + if (join) + { + Item_cond* cond = reinterpret_cast(join->conds); + + if (cond) + cond->traverse_cond(debug_walk, arg, Item::POSTFIX); + } + + cerr << "Finish subselect item traversing" << endl; + break; + } + + case Item::REF_ITEM: + { + Item_ref* ref = (Item_ref*)item; + + if (ref->real_item()->type() == Item::CACHE_ITEM) + { + Item* field = ((Item_cache*)ref->real_item())->get_example(); + + if (field->type() == Item::FIELD_ITEM) { - Item_cond* icp = (Item_cond*)item; - cerr << "COND_ITEM: " << icp->func_name() << endl; - break; + Item_field* ifp = (Item_field*)field; + // ifp->cached_table->select_lex->select_number gives the select level. + // could be used on alias. + // could also be used to tell correlated join (equal level). + cerr << "CACHED REF FIELD_ITEM: " << ifp->db_name.str << '.' << bestTableName(ifp) << '.' + << ifp->field_name.str << endl; + break; + } + else if (field->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)field; + cerr << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + } + else if (field->type() == Item::REF_ITEM) + { + Item_ref* ifr = (Item_ref*)field; + string refType; + string realType; + + switch (ifr->ref_type()) + { + case Item_ref::REF: refType = "REF"; break; + + case Item_ref::DIRECT_REF: refType = "DIRECT_REF"; break; + + case Item_ref::VIEW_REF: refType = "VIEW_REF"; break; + + case Item_ref::OUTER_REF: refType = "OUTER_REF"; break; + + case Item_ref::AGGREGATE_REF: refType = "AGGREGATE_REF"; break; + + default: refType = "UNKNOWN"; break; + } + + switch (ifr->real_type()) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)(*(ifr->ref)); + realType = "FIELD_ITEM "; + realType += ifp->db_name.str; + realType += '.'; + realType += bestTableName(ifp); + realType += '.'; + realType += ifp->field_name.str; + break; + } + + case Item::SUM_FUNC_ITEM: + { + Item_sum* isp = (Item_sum*)(*(ifr->ref)); + + if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) + realType = "GROUP_CONCAT_FUNC"; + else + realType = "SUM_FUNC_ITEM"; + + break; + } + + case Item::REF_ITEM: + // Need recursion here + realType = "REF_ITEM"; + break; + + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)(*(ifr->ref)); + realType = "FUNC_ITEM "; + realType += ifp->func_name(); + break; + } + + default: + { + realType = "UNKNOWN"; + } + } + + cerr << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() + << endl; + break; + } + else + { + cerr << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; + } + } + else if (ref->real_item()->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)ref->real_item(); + + // MCOL-1052 The field referenced presumable came from + // extended SELECT list. + if (!ifp->field_name.str) + { + cerr << "REF extra FIELD_ITEM: " << ifp->name.str << endl; + } + else + { + cerr << "REF FIELD_ITEM: " << ifp->db_name.str << '.' << bestTableName(ifp) << '.' + << ifp->field_name.str << endl; } - case Item::SUM_FUNC_ITEM: - { - Item_sum* isp = (Item_sum*)item; - char* item_name = const_cast(item->name.str); + break; + } + else if (ref->real_item()->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)ref->real_item(); + cerr << "REF FUNC_ITEM " << ifp->func_name() << endl; + } + else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) + { + Item_window_func* ifp = (Item_window_func*)ref->real_item(); + cerr << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + } + else + { + cerr << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + } - // MCOL-1052 This is an extended SELECT list item - if (!item_name && isp->get_arg_count() && isp->get_arg(0)->name.length) - { - item_name = const_cast(isp->get_arg(0)->name.str); - } - else if (!item_name && isp->get_arg_count() - && isp->get_arg(0)->type() == Item::CONST_ITEM - && isp->get_arg(0)->cmp_type() == INT_RESULT) - { - item_name = (char*)"INT||*"; - } - else if (!item_name) - { - item_name = (char*)""; - } + break; + } - switch (isp->sum_func()) - { - case Item_sum::SUM_FUNC: - cerr << "SUM_FUNC: " << item_name << endl; - break; + case Item::ROW_ITEM: + { + Item_row* row = (Item_row*)item; + cerr << "ROW_ITEM: " << endl; - case Item_sum::SUM_DISTINCT_FUNC: - cerr << "SUM_DISTINCT_FUNC: " << item_name << endl; - break; + for (uint32_t i = 0; i < row->cols(); i++) + debug_walk(row->element_index(i), 0); - case Item_sum::AVG_FUNC: - cerr << "AVG_FUNC: " << item_name << endl; - break; + break; + } - case Item_sum::COUNT_FUNC: - cerr << "COUNT_FUNC: " << item_name << endl; - break; + case Item::EXPR_CACHE_ITEM: + { + cerr << "Expr Cache Item" << endl; + ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(debug_walk, arg, Item::POSTFIX); + break; + } - case Item_sum::COUNT_DISTINCT_FUNC: - cerr << "COUNT_DISTINCT_FUNC: " << item_name << endl; - break; - - case Item_sum::MIN_FUNC: - cerr << "MIN_FUNC: " << item_name << endl; - break; - - case Item_sum::MAX_FUNC: - cerr << "MAX_FUNC: " << item_name << endl; - break; - - case Item_sum::UDF_SUM_FUNC: - cerr << "UDAF_FUNC: " << item_name << endl; - break; - - default: - cerr << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; - break; - } - - break; - } - - case Item::SUBSELECT_ITEM: - { - Item_subselect* sub = (Item_subselect*)item; - cerr << "SUBSELECT Item: "; - - switch (sub->substype()) - { - case Item_subselect::EXISTS_SUBS: - cerr << "EXISTS"; - break; - - case Item_subselect::IN_SUBS: - cerr << "IN"; - break; - - default: - cerr << sub->substype(); - break; - } - - cerr << endl; - JOIN* join = sub->get_select_lex()->join; - - if (join) - { - Item_cond* cond = reinterpret_cast(join->conds); - - if (cond) - cond->traverse_cond(debug_walk, arg, Item::POSTFIX); - } - - cerr << "Finish subselect item traversing" << endl; - break; - } - - case Item::REF_ITEM: - { - Item_ref* ref = (Item_ref*)item; - - if (ref->real_item()->type() == Item::CACHE_ITEM) - { - Item* field = ((Item_cache*)ref->real_item())->get_example(); - - if (field->type() == Item::FIELD_ITEM) - { - Item_field* ifp = (Item_field*)field; - //ifp->cached_table->select_lex->select_number gives the select level. - // could be used on alias. - // could also be used to tell correlated join (equal level). - cerr << "CACHED REF FIELD_ITEM: " << ifp->db_name.str << '.' << bestTableName(ifp) << - '.' << ifp->field_name.str << endl; - break; - } - else if (field->type() == Item::FUNC_ITEM) - { - Item_func* ifp = (Item_func*)field; - cerr << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; - } - else if (field->type() == Item::REF_ITEM) - { - Item_ref* ifr = (Item_ref*)field; - string refType; - string realType; - - switch (ifr->ref_type()) - { - case Item_ref::REF: - refType = "REF"; - break; - - case Item_ref::DIRECT_REF: - refType = "DIRECT_REF"; - break; - - case Item_ref::VIEW_REF: - refType = "VIEW_REF"; - break; - - case Item_ref::OUTER_REF: - refType = "OUTER_REF"; - break; - - case Item_ref::AGGREGATE_REF: - refType = "AGGREGATE_REF"; - break; - - default: - refType = "UNKNOWN"; - break; - } - - switch (ifr->real_type()) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = (Item_field*)(*(ifr->ref)); - realType = "FIELD_ITEM "; - realType += ifp->db_name.str; - realType += '.'; - realType += bestTableName(ifp); - realType += '.'; - realType += ifp->field_name.str; - break; - } - - case Item::SUM_FUNC_ITEM: - { - Item_sum* isp = (Item_sum*)(*(ifr->ref)); - - if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) - realType = "GROUP_CONCAT_FUNC"; - else - realType = "SUM_FUNC_ITEM"; - - break; - } - - case Item::REF_ITEM: - // Need recursion here - realType = "REF_ITEM"; - break; - - case Item::FUNC_ITEM: - { - Item_func* ifp = (Item_func*)(*(ifr->ref)); - realType = "FUNC_ITEM "; - realType += ifp->func_name(); - break; - } - - default: - { - realType = "UNKNOWN"; - } - } - - cerr << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; - break; - } - else - { - cerr << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; - } - } - else if (ref->real_item()->type() == Item::FIELD_ITEM) - { - Item_field* ifp = (Item_field*)ref->real_item(); - - // MCOL-1052 The field referenced presumable came from - // extended SELECT list. - if ( !ifp->field_name.str ) - { - cerr << "REF extra FIELD_ITEM: " << ifp->name.str << endl; - } - else - { - cerr << "REF FIELD_ITEM: " << ifp->db_name.str << '.' << bestTableName(ifp) << '.' << - ifp->field_name.str << endl; - } - - break; - } - else if (ref->real_item()->type() == Item::FUNC_ITEM) - { - Item_func* ifp = (Item_func*)ref->real_item(); - cerr << "REF FUNC_ITEM " << ifp->func_name() << endl; - } - else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) - { - Item_window_func* ifp = (Item_window_func*)ref->real_item(); - cerr << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; - } - else - { - cerr << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; - } - - break; - } - - case Item::ROW_ITEM: - { - Item_row* row = (Item_row*)item; - cerr << "ROW_ITEM: " << endl; - - for (uint32_t i = 0; i < row->cols(); i++) - debug_walk(row->element_index(i), 0); - - break; - } - - case Item::EXPR_CACHE_ITEM: - { - cerr << "Expr Cache Item" << endl; - ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(debug_walk, arg, Item::POSTFIX); - break; - } - - case Item::CACHE_ITEM: - { - Item_cache* isp = (Item_cache*)item; - // MCOL-46 isp->val_str() can cause a call to execute a subquery. We're not set up - // to execute yet. + case Item::CACHE_ITEM: + { + Item_cache* isp = (Item_cache*)item; + // MCOL-46 isp->val_str() can cause a call to execute a subquery. We're not set up + // to execute yet. #if 0 switch (item->result_type()) @@ -1254,872 +1245,855 @@ void debug_walk(const Item* item, void* arg) } #endif - Item* field = isp->get_example(); + Item* field = isp->get_example(); - if (field->type() == Item::FIELD_ITEM) - { - Item_field* ifp = (Item_field*)field; - //ifp->cached_table->select_lex->select_number gives the select level. - // could be used on alias. - // could also be used to tell correlated join (equal level). - cerr << "CACHED FIELD_ITEM: " << ifp->db_name.str << '.' << bestTableName(ifp) << - '.' << ifp->field_name.str << endl; - break; - } - else if (field->type() == Item::REF_ITEM) - { - Item_ref* ifr = (Item_ref*)field; - string refType; - string realType; + if (field->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)field; + // ifp->cached_table->select_lex->select_number gives the select level. + // could be used on alias. + // could also be used to tell correlated join (equal level). + cerr << "CACHED FIELD_ITEM: " << ifp->db_name.str << '.' << bestTableName(ifp) << '.' + << ifp->field_name.str << endl; + break; + } + else if (field->type() == Item::REF_ITEM) + { + Item_ref* ifr = (Item_ref*)field; + string refType; + string realType; - switch (ifr->ref_type()) - { - case Item_ref::REF: - refType = "REF"; - break; + switch (ifr->ref_type()) + { + case Item_ref::REF: refType = "REF"; break; - case Item_ref::DIRECT_REF: - refType = "DIRECT_REF"; - break; + case Item_ref::DIRECT_REF: refType = "DIRECT_REF"; break; - case Item_ref::VIEW_REF: - refType = "VIEW_REF"; - break; + case Item_ref::VIEW_REF: refType = "VIEW_REF"; break; - case Item_ref::OUTER_REF: - refType = "OUTER_REF"; - break; + case Item_ref::OUTER_REF: refType = "OUTER_REF"; break; - case Item_ref::AGGREGATE_REF: - refType = "AGGREGATE_REF"; - break; + case Item_ref::AGGREGATE_REF: refType = "AGGREGATE_REF"; break; - default: - refType = "UNKNOWN"; - break; - } + default: refType = "UNKNOWN"; break; + } - switch (ifr->real_type()) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = (Item_field*)(*(ifr->ref)); - realType = "FIELD_ITEM "; - realType += ifp->db_name.str; - realType += '.'; - realType += bestTableName(ifp); - realType += '.'; - realType += ifp->field_name.str; - break; - } + switch (ifr->real_type()) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)(*(ifr->ref)); + realType = "FIELD_ITEM "; + realType += ifp->db_name.str; + realType += '.'; + realType += bestTableName(ifp); + realType += '.'; + realType += ifp->field_name.str; + break; + } - case Item::SUM_FUNC_ITEM: - { - Item_sum* isp = (Item_sum*)(*(ifr->ref)); + case Item::SUM_FUNC_ITEM: + { + Item_sum* isp = (Item_sum*)(*(ifr->ref)); - if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) - realType = "GROUP_CONCAT_FUNC"; - else - realType = "SUM_FUNC_ITEM"; - - break; - } - - case Item::REF_ITEM: - // Need recursion here - realType = "REF_ITEM"; - break; - - case Item::FUNC_ITEM: - { - Item_func* ifp = (Item_func*)(*(ifr->ref)); - realType = "FUNC_ITEM "; - realType += ifp->func_name(); - break; - } - - default: - { - realType = "UNKNOWN"; - } - } - - cerr << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; - break; - } - else if (field->type() == Item::FUNC_ITEM) - { - Item_func* ifp = (Item_func*)field; - cerr << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; - break; - } + if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) + realType = "GROUP_CONCAT_FUNC"; else - { - cerr << "CACHE_ITEM type unknown " << field->type() << endl; - } + realType = "SUM_FUNC_ITEM"; break; + } + + case Item::REF_ITEM: + // Need recursion here + realType = "REF_ITEM"; + break; + + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)(*(ifr->ref)); + realType = "FUNC_ITEM "; + realType += ifp->func_name(); + break; + } + + default: + { + realType = "UNKNOWN"; + } } - case Item::WINDOW_FUNC_ITEM: - { - Item_window_func* ifp = (Item_window_func*)item; - cerr << "Window Function Item " << ifp->window_func()->func_name() << endl; - break; - } + cerr << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + break; + } + else if (field->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)field; + cerr << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; + break; + } + else + { + cerr << "CACHE_ITEM type unknown " << field->type() << endl; + } - case Item::NULL_ITEM: - { - cerr << "NULL item" << endl; - break; - } - - case Item::TYPE_HOLDER: - { - cerr << "TYPE_HOLDER item with cmp_type " << item->cmp_type() << endl; - break; - } - - default: - { - cerr << "UNKNOWN_ITEM type " << item->type() << endl; - break; - } + break; } + + case Item::WINDOW_FUNC_ITEM: + { + Item_window_func* ifp = (Item_window_func*)item; + cerr << "Window Function Item " << ifp->window_func()->func_name() << endl; + break; + } + + case Item::NULL_ITEM: + { + cerr << "NULL item" << endl; + break; + } + + case Item::TYPE_HOLDER: + { + cerr << "TYPE_HOLDER item with cmp_type " << item->cmp_type() << endl; + break; + } + + default: + { + cerr << "UNKNOWN_ITEM type " << item->type() << endl; + break; + } + } } #endif void buildNestedJoinLeafTables(List& join_list, - std::set& leafTables) + std::set& leafTables) { - TABLE_LIST *table; - List_iterator li(join_list); + TABLE_LIST* table; + List_iterator li(join_list); - while ((table = li++)) + while ((table = li++)) + { + if (table->nested_join) + buildNestedJoinLeafTables(table->nested_join->join_list, leafTables); + else { - if (table->nested_join) - buildNestedJoinLeafTables(table->nested_join->join_list, leafTables); - else - { - CalpontSystemCatalog::TableAliasName tan = makeTableAliasName(table); - leafTables.insert(tan); - } + CalpontSystemCatalog::TableAliasName tan = makeTableAliasName(table); + leafTables.insert(tan); } + } } uint32_t buildJoin(gp_walk_info& gwi, List& join_list, - std::stack& outerJoinStack) + std::stack& outerJoinStack) { - TABLE_LIST *table; - List_iterator li(join_list); + TABLE_LIST* table; + List_iterator li(join_list); - while ((table = li++)) + while ((table = li++)) + { + // Make sure we don't process the derived table nests again, + // they were already handled by FromSubQuery::transform() + if (table->nested_join && !table->derived) + buildJoin(gwi, table->nested_join->join_list, outerJoinStack); + + std::vector tableOnExprList; + + auto iter = gwi.tableOnExprList.find(table); + + // Check if this table's on_expr is available in the hash map + // built/updated during convertOuterJoinToInnerJoin(). + if ((iter != gwi.tableOnExprList.end()) && !iter->second.empty()) { - // Make sure we don't process the derived table nests again, - // they were already handled by FromSubQuery::transform() - if (table->nested_join && !table->derived) - buildJoin(gwi, table->nested_join->join_list, outerJoinStack); - - std::vector tableOnExprList; - - auto iter = gwi.tableOnExprList.find(table); - - // Check if this table's on_expr is available in the hash map - // built/updated during convertOuterJoinToInnerJoin(). - if ((iter != gwi.tableOnExprList.end()) && !iter->second.empty()) - { - tableOnExprList = iter->second; - } - // This table's on_expr has not been seen/processed before. - else if ((iter == gwi.tableOnExprList.end()) && table->on_expr) - { - tableOnExprList.push_back(table->on_expr); - } - - if (!tableOnExprList.empty()) - { - if (table->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) - { - // inner tables block - gp_walk_info gwi_outer = gwi; - gwi_outer.subQuery = NULL; - gwi_outer.hasSubSelect = false; - gwi_outer.innerTables.clear(); - clearStacks(gwi_outer); - - // recursively build the leaf tables for this nested join node - if (table->nested_join) - buildNestedJoinLeafTables(table->nested_join->join_list, - gwi_outer.innerTables); - else // this is a leaf table - { - CalpontSystemCatalog::TableAliasName tan = makeTableAliasName(table); - gwi_outer.innerTables.insert(tan); - } - -#ifdef DEBUG_WALK_COND - cerr << "inner tables: " << endl; - set::const_iterator it; - for (it = gwi_outer.innerTables.begin(); it != gwi_outer.innerTables.end(); ++it) - cerr << (*it) << " "; - cerr << endl; - - cerr << " outer table expression: " << endl; - - for (Item* expr : tableOnExprList) - expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); -#endif - - for (Item* expr : tableOnExprList) - { - expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); - - // Error out subquery in outer join on filter for now - if (gwi_outer.hasSubSelect) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_OUTER_JOIN_SUBSELECT); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); - return -1; - } - } - - // build outerjoinon filter - ParseTree* filters = NULL, *ptp = NULL, *rhs = NULL; - - while (!gwi_outer.ptWorkStack.empty()) - { - filters = gwi_outer.ptWorkStack.top(); - gwi_outer.ptWorkStack.pop(); - - if (gwi_outer.ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - rhs = gwi_outer.ptWorkStack.top(); - gwi_outer.ptWorkStack.pop(); - ptp->right(rhs); - gwi_outer.ptWorkStack.push(ptp); - } - - // should have only 1 pt left in stack. - if (filters) - { - SPTP on_sp(filters); - OuterJoinOnFilter* onFilter = new OuterJoinOnFilter(on_sp); - ParseTree* pt = new ParseTree(onFilter); - outerJoinStack.push(pt); - } - } - else // inner join - { - for (Item* expr : tableOnExprList) - { - expr->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - } - -#ifdef DEBUG_WALK_COND - cerr << " inner join expression: " << endl; - for (Item* expr : tableOnExprList) - expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); -#endif - } - } + tableOnExprList = iter->second; + } + // This table's on_expr has not been seen/processed before. + else if ((iter == gwi.tableOnExprList.end()) && table->on_expr) + { + tableOnExprList.push_back(table->on_expr); } - return 0; + if (!tableOnExprList.empty()) + { + if (table->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) + { + // inner tables block + gp_walk_info gwi_outer = gwi; + gwi_outer.subQuery = NULL; + gwi_outer.hasSubSelect = false; + gwi_outer.innerTables.clear(); + clearStacks(gwi_outer); + + // recursively build the leaf tables for this nested join node + if (table->nested_join) + buildNestedJoinLeafTables(table->nested_join->join_list, gwi_outer.innerTables); + else // this is a leaf table + { + CalpontSystemCatalog::TableAliasName tan = makeTableAliasName(table); + gwi_outer.innerTables.insert(tan); + } + +#ifdef DEBUG_WALK_COND + cerr << "inner tables: " << endl; + set::const_iterator it; + for (it = gwi_outer.innerTables.begin(); it != gwi_outer.innerTables.end(); ++it) + cerr << (*it) << " "; + cerr << endl; + + cerr << " outer table expression: " << endl; + + for (Item* expr : tableOnExprList) + expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); +#endif + + for (Item* expr : tableOnExprList) + { + expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); + + // Error out subquery in outer join on filter for now + if (gwi_outer.hasSubSelect) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_OUTER_JOIN_SUBSELECT); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); + return -1; + } + } + + // build outerjoinon filter + ParseTree *filters = NULL, *ptp = NULL, *rhs = NULL; + + while (!gwi_outer.ptWorkStack.empty()) + { + filters = gwi_outer.ptWorkStack.top(); + gwi_outer.ptWorkStack.pop(); + + if (gwi_outer.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + rhs = gwi_outer.ptWorkStack.top(); + gwi_outer.ptWorkStack.pop(); + ptp->right(rhs); + gwi_outer.ptWorkStack.push(ptp); + } + + // should have only 1 pt left in stack. + if (filters) + { + SPTP on_sp(filters); + OuterJoinOnFilter* onFilter = new OuterJoinOnFilter(on_sp); + ParseTree* pt = new ParseTree(onFilter); + outerJoinStack.push(pt); + } + } + else // inner join + { + for (Item* expr : tableOnExprList) + { + expr->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + } + +#ifdef DEBUG_WALK_COND + cerr << " inner join expression: " << endl; + for (Item* expr : tableOnExprList) + expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); +#endif + } + } + } + + return 0; } ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string predicateOp) { - PredicateOperator* po = new PredicateOperator(predicateOp); - boost::shared_ptr sop(po); - LogicOperator* lo = NULL; + PredicateOperator* po = new PredicateOperator(predicateOp); + boost::shared_ptr sop(po); + LogicOperator* lo = NULL; - if (predicateOp == "=") - lo = new LogicOperator("and"); - else - lo = new LogicOperator("or"); + if (predicateOp == "=") + lo = new LogicOperator("and"); + else + lo = new LogicOperator("or"); - ParseTree* pt = new ParseTree(lo); - sop->setOpType(lhs->columnVec()[0]->resultType(), rhs->columnVec()[0]->resultType()); - SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0].get(), rhs->columnVec()[0].get()); + ParseTree* pt = new ParseTree(lo); + sop->setOpType(lhs->columnVec()[0]->resultType(), rhs->columnVec()[0]->resultType()); + SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0].get(), rhs->columnVec()[0].get()); + sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + pt->left(new ParseTree(sf)); + + for (uint32_t i = 1; i < lhs->columnVec().size(); i++) + { + sop.reset(po->clone()); + sop->setOpType(lhs->columnVec()[i]->resultType(), rhs->columnVec()[i]->resultType()); + SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i].get(), rhs->columnVec()[i].get()); sf->timeZone(thd->variables.time_zone->get_name()->ptr()); - pt->left(new ParseTree(sf)); + pt->right(new ParseTree(sf)); - for (uint32_t i = 1; i < lhs->columnVec().size(); i++) + if (i + 1 < lhs->columnVec().size()) { - sop.reset(po->clone()); - sop->setOpType(lhs->columnVec()[i]->resultType(), rhs->columnVec()[i]->resultType()); - SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i].get(), rhs->columnVec()[i].get()); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); - pt->right(new ParseTree(sf)); - - if (i + 1 < lhs->columnVec().size()) - { - ParseTree* lpt = pt; - pt = new ParseTree(lo->clone()); - pt->left(lpt); - } + ParseTree* lpt = pt; + pt = new ParseTree(lo->clone()); + pt->left(lpt); } + } - return pt; + return pt; } bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, Item_func* ifp) { - if (ifp->functype() == Item_func::EQ_FUNC || ifp->functype() == Item_func::NE_FUNC) + if (ifp->functype() == Item_func::EQ_FUNC || ifp->functype() == Item_func::NE_FUNC) + { + // (c1,c2,..) = (v1,v2,...) transform to: c1=v1 and c2=v2 and ... + assert(!lhs->columnVec().empty() && lhs->columnVec().size() == rhs->columnVec().size()); + gwip->ptWorkStack.push(buildRowPredicate(gwip->thd, rhs, lhs, ifp->func_name())); + } + else if (ifp->functype() == Item_func::IN_FUNC) + { + // (c1,c2,...) in ((v11,v12,...),(v21,v22,...)...) transform to: + // ((c1 = v11 and c2 = v12 and ...) or (c1 = v21 and c2 = v22 and ...) or ...) + // and c1 in (v11,v21,...) and c2 in (v12,v22,...) => to favor CP + Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; + string predicateOp, logicOp; + + if (inp->negated) { - // (c1,c2,..) = (v1,v2,...) transform to: c1=v1 and c2=v2 and ... - assert (!lhs->columnVec().empty() && lhs->columnVec().size() == rhs->columnVec().size()); - gwip->ptWorkStack.push(buildRowPredicate(gwip->thd, rhs, lhs, ifp->func_name())); - } - else if (ifp->functype() == Item_func::IN_FUNC) - { - // (c1,c2,...) in ((v11,v12,...),(v21,v22,...)...) transform to: - // ((c1 = v11 and c2 = v12 and ...) or (c1 = v21 and c2 = v22 and ...) or ...) - // and c1 in (v11,v21,...) and c2 in (v12,v22,...) => to favor CP - Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; - string predicateOp, logicOp; - - if (inp->negated) - { - predicateOp = "<>"; - logicOp = "and"; - } - else - { - predicateOp = "="; - logicOp = "or"; - } - - boost::scoped_ptr lo(new LogicOperator(logicOp)); - - // 1st round. build the equivalent filters - // two entries have been popped from the stack already: lhs and rhs - stack tmpStack; - vector valVec; - tmpStack.push(rhs); - tmpStack.push(lhs); - assert (gwip->rcWorkStack.size() >= ifp->argument_count() - 2); - - for (uint32_t i = 2; i < ifp->argument_count(); i++) - { - tmpStack.push(gwip->rcWorkStack.top()); - - if (!gwip->rcWorkStack.empty()) - gwip->rcWorkStack.pop(); - } - - RowColumn* columns = dynamic_cast(tmpStack.top()); - tmpStack.pop(); - RowColumn* vals = dynamic_cast(tmpStack.top()); - valVec.push_back(vals); - tmpStack.pop(); - ParseTree* pt = buildRowPredicate(gwip->thd, columns, vals, predicateOp); - - while (!tmpStack.empty()) - { - ParseTree* pt1 = new ParseTree(lo->clone()); - pt1->left(pt); - vals = dynamic_cast(tmpStack.top()); - valVec.push_back(vals); - tmpStack.pop(); - pt1->right(buildRowPredicate(gwip->thd, columns->clone(), vals, predicateOp)); - pt = pt1; - } - - gwip->ptWorkStack.push(pt); - - // done for NOTIN clause - if (predicateOp == "<>") - return true; - - // 2nd round. add the filter to favor casual partition for IN clause - boost::shared_ptr sop; - boost::shared_ptr ssc; - stack tmpPtStack; - - for (uint32_t i = 0; i < columns->columnVec().size(); i++) - { - ConstantFilter* cf = new ConstantFilter(); - - sop.reset(lo->clone()); - cf->op(sop); - SimpleColumn* sc = dynamic_cast(columns->columnVec()[i].get()); - - // no optimization for non-simple column because CP won't apply - if (!sc) - continue; - - ssc.reset(sc->clone()); - cf->col(ssc); - - uint32_t j = 0; - - for (; j < valVec.size(); j++) - { - sop.reset(new PredicateOperator(predicateOp)); - ConstantColumn* cc = dynamic_cast(valVec[j]->columnVec()[i].get()); - - // no optimization for non-constant value because CP won't apply - if (!cc) - break; - - sop->setOpType(sc->resultType(), valVec[j]->columnVec()[i]->resultType()); - cf->pushFilter(new SimpleFilter(sop, sc->clone(), - valVec[j]->columnVec()[i]->clone(), - gwip->thd->variables.time_zone->get_name()->ptr())); - } - - if (j < valVec.size()) - continue; - - tmpPtStack.push(new ParseTree(cf)); - } - - // "and" all the filters together - ParseTree* ptp = NULL; - pt = NULL; - - while (!tmpPtStack.empty()) - { - pt = tmpPtStack.top(); - tmpPtStack.pop(); - - if (tmpPtStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(pt); - pt = tmpPtStack.top(); - tmpPtStack.pop(); - ptp->right(pt); - tmpPtStack.push(ptp); - } - - if (pt) - { - tmpPtStack.push(pt); - - // AND with the pt built from the first round - if (!gwip->ptWorkStack.empty() && !tmpPtStack.empty()) - { - pt = new ParseTree(new LogicOperator("and")); - pt->left(gwip->ptWorkStack.top()); - gwip->ptWorkStack.pop(); - pt->right(tmpPtStack.top()); - tmpPtStack.pop(); - } - - // Push the final pt tree for this IN clause - gwip->ptWorkStack.push(pt); - } + predicateOp = "<>"; + logicOp = "and"; } else { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FUNC_MULTI_COL); - return false; + predicateOp = "="; + logicOp = "or"; } - return true; + boost::scoped_ptr lo(new LogicOperator(logicOp)); + + // 1st round. build the equivalent filters + // two entries have been popped from the stack already: lhs and rhs + stack tmpStack; + vector valVec; + tmpStack.push(rhs); + tmpStack.push(lhs); + assert(gwip->rcWorkStack.size() >= ifp->argument_count() - 2); + + for (uint32_t i = 2; i < ifp->argument_count(); i++) + { + tmpStack.push(gwip->rcWorkStack.top()); + + if (!gwip->rcWorkStack.empty()) + gwip->rcWorkStack.pop(); + } + + RowColumn* columns = dynamic_cast(tmpStack.top()); + tmpStack.pop(); + RowColumn* vals = dynamic_cast(tmpStack.top()); + valVec.push_back(vals); + tmpStack.pop(); + ParseTree* pt = buildRowPredicate(gwip->thd, columns, vals, predicateOp); + + while (!tmpStack.empty()) + { + ParseTree* pt1 = new ParseTree(lo->clone()); + pt1->left(pt); + vals = dynamic_cast(tmpStack.top()); + valVec.push_back(vals); + tmpStack.pop(); + pt1->right(buildRowPredicate(gwip->thd, columns->clone(), vals, predicateOp)); + pt = pt1; + } + + gwip->ptWorkStack.push(pt); + + // done for NOTIN clause + if (predicateOp == "<>") + return true; + + // 2nd round. add the filter to favor casual partition for IN clause + boost::shared_ptr sop; + boost::shared_ptr ssc; + stack tmpPtStack; + + for (uint32_t i = 0; i < columns->columnVec().size(); i++) + { + ConstantFilter* cf = new ConstantFilter(); + + sop.reset(lo->clone()); + cf->op(sop); + SimpleColumn* sc = dynamic_cast(columns->columnVec()[i].get()); + + // no optimization for non-simple column because CP won't apply + if (!sc) + continue; + + ssc.reset(sc->clone()); + cf->col(ssc); + + uint32_t j = 0; + + for (; j < valVec.size(); j++) + { + sop.reset(new PredicateOperator(predicateOp)); + ConstantColumn* cc = dynamic_cast(valVec[j]->columnVec()[i].get()); + + // no optimization for non-constant value because CP won't apply + if (!cc) + break; + + sop->setOpType(sc->resultType(), valVec[j]->columnVec()[i]->resultType()); + cf->pushFilter(new SimpleFilter(sop, sc->clone(), valVec[j]->columnVec()[i]->clone(), + gwip->thd->variables.time_zone->get_name()->ptr())); + } + + if (j < valVec.size()) + continue; + + tmpPtStack.push(new ParseTree(cf)); + } + + // "and" all the filters together + ParseTree* ptp = NULL; + pt = NULL; + + while (!tmpPtStack.empty()) + { + pt = tmpPtStack.top(); + tmpPtStack.pop(); + + if (tmpPtStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(pt); + pt = tmpPtStack.top(); + tmpPtStack.pop(); + ptp->right(pt); + tmpPtStack.push(ptp); + } + + if (pt) + { + tmpPtStack.push(pt); + + // AND with the pt built from the first round + if (!gwip->ptWorkStack.empty() && !tmpPtStack.empty()) + { + pt = new ParseTree(new LogicOperator("and")); + pt->left(gwip->ptWorkStack.top()); + gwip->ptWorkStack.pop(); + pt->right(tmpPtStack.top()); + tmpPtStack.pop(); + } + + // Push the final pt tree for this IN clause + gwip->ptWorkStack.push(pt); + } + } + else + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FUNC_MULTI_COL); + return false; + } + + return true; } -void checkOuterTableColumn(gp_walk_info* gwip, - const CalpontSystemCatalog::TableAliasName& tan, - execplan::ReturnedColumn* col) +void checkOuterTableColumn(gp_walk_info* gwip, const CalpontSystemCatalog::TableAliasName& tan, + execplan::ReturnedColumn* col) { - set::const_iterator it; + set::const_iterator it; - bool notInner = true; + bool notInner = true; - for (it = gwip->innerTables.begin(); it != gwip->innerTables.end(); ++it) - { - if (tan.alias == it->alias && tan.view == it->view) - notInner = false; - } + for (it = gwip->innerTables.begin(); it != gwip->innerTables.end(); ++it) + { + if (tan.alias == it->alias && tan.view == it->view) + notInner = false; + } - if (notInner) - { - col->returnAll(true); - IDEBUG( cerr << "setting returnAll on " << tan << endl); - } + if (notInner) + { + col->returnAll(true); + IDEBUG(cerr << "setting returnAll on " << tan << endl); + } } -bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, - execplan::ReturnedColumn* rhs, - gp_walk_info* gwip, - boost::shared_ptr& sop, - const Item_func::Functype& funcType, - const vector& itemList, - bool isInSubs) +bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedColumn* rhs, gp_walk_info* gwip, + boost::shared_ptr& sop, const Item_func::Functype& funcType, + const vector& itemList, bool isInSubs) { - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - // push the column that is associated with the correlated column to the returned - // column list, so the materialized view have the complete projection list. - // e.g. tout.c1 in (select tin.c1 from tin where tin.c2=tout.c2); - // the projetion list of subquery will have tin.c1, tin.c2. - ReturnedColumn* correlatedCol = nullptr; - ReturnedColumn* localCol = nullptr; + // push the column that is associated with the correlated column to the returned + // column list, so the materialized view have the complete projection list. + // e.g. tout.c1 in (select tin.c1 from tin where tin.c2=tout.c2); + // the projetion list of subquery will have tin.c1, tin.c2. + ReturnedColumn* correlatedCol = nullptr; + ReturnedColumn* localCol = nullptr; - if (rhs->joinInfo() & JOIN_CORRELATED) + if (rhs->joinInfo() & JOIN_CORRELATED) + { + correlatedCol = rhs; + localCol = lhs; + } + else if (lhs->joinInfo() & JOIN_CORRELATED) + { + correlatedCol = lhs; + localCol = rhs; + } + + if (correlatedCol && localCol) + { + ConstantColumn* cc = dynamic_cast(localCol); + + if ((!cc || (cc && funcType == Item_func::EQ_FUNC)) && !(localCol->joinInfo() & JOIN_CORRELATED)) { - correlatedCol = rhs; - localCol = lhs; - } - else if (lhs->joinInfo() & JOIN_CORRELATED) - { - correlatedCol = lhs; - localCol = rhs; + if (isInSubs) + localCol->sequence(0); + else + localCol->sequence(gwip->returnedCols.size()); + + localCol->expressionId(ci->expressionId++); + ReturnedColumn* rc = localCol->clone(); + rc->colSource(rc->colSource() | CORRELATED_JOIN); + gwip->additionalRetCols.push_back(SRCP(rc)); + gwip->localCols.push_back(localCol); + + if (rc->hasWindowFunc() && !isInSubs) + gwip->windowFuncList.push_back(rc); } - if (correlatedCol && localCol) + // push the correlated join partner to the group by list only when there's aggregate + // and we don't push aggregate column to the group by + // @bug4756. mysql does not always give correct information about whether there is + // aggregate on the SELECT list. Need to figure that by ourselves and then decide + // to add the group by or not. + if (gwip->subQuery) { - ConstantColumn* cc = dynamic_cast(localCol); - - if ((!cc || (cc && funcType == Item_func::EQ_FUNC)) && - !(localCol->joinInfo() & JOIN_CORRELATED)) - { - if (isInSubs) - localCol->sequence(0); - else - localCol->sequence(gwip->returnedCols.size()); - - localCol->expressionId(ci->expressionId++); - ReturnedColumn* rc = localCol->clone(); - rc->colSource(rc->colSource() | CORRELATED_JOIN); - gwip->additionalRetCols.push_back(SRCP(rc)); - gwip->localCols.push_back(localCol); - - if (rc->hasWindowFunc() && !isInSubs) - gwip->windowFuncList.push_back(rc); - } - - // push the correlated join partner to the group by list only when there's aggregate - // and we don't push aggregate column to the group by - // @bug4756. mysql does not always give correct information about whether there is - // aggregate on the SELECT list. Need to figure that by ourselves and then decide - // to add the group by or not. - if (gwip->subQuery) - { - if (!localCol->hasAggregate() && !localCol->hasWindowFunc()) - gwip->subGroupByCols.push_back(SRCP(localCol->clone())); - } - - if (sop->op() == OP_EQ) - { - if (gwip->subSelectType == CalpontSelectExecutionPlan::IN_SUBS || - gwip->subSelectType == CalpontSelectExecutionPlan::EXISTS_SUBS) - correlatedCol->joinInfo(correlatedCol->joinInfo() | JOIN_SEMI); - else if (gwip->subSelectType == CalpontSelectExecutionPlan::NOT_IN_SUBS || - gwip->subSelectType == CalpontSelectExecutionPlan::NOT_EXISTS_SUBS) - correlatedCol->joinInfo(correlatedCol->joinInfo() | JOIN_ANTI); - } + if (!localCol->hasAggregate() && !localCol->hasWindowFunc()) + gwip->subGroupByCols.push_back(SRCP(localCol->clone())); } - SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - - //@bug 2101 for when there are only constants in a delete or update where clause (eg "where 5 < 6"). - //There will be no field column and it will get here only if the comparison is true. - if (gwip->columnMap.empty() && - ((current_thd->lex->sql_command == SQLCOM_UPDATE) || - (current_thd->lex->sql_command == SQLCOM_UPDATE_MULTI) || - (current_thd->lex->sql_command == SQLCOM_DELETE) || - (current_thd->lex->sql_command == SQLCOM_DELETE_MULTI))) - { - IDEBUG( cerr << "deleted func with 2 const columns" << endl ); - delete rhs; - delete lhs; - return false; - } - - // handle noop (only for table mode) - if (rhs->data() == "noop" || lhs->data() == "noop") - { - sop.reset(new Operator("noop")); - } - else - { - for (uint32_t i = 0; i < itemList.size(); i++) - { - if (isPredicateFunction(itemList[i], gwip)) - { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SUB_EXPRESSION); - } - } - } - - sf->op(sop); - sf->lhs(lhs); - sf->rhs(rhs); - sop->setOpType(lhs->resultType(), rhs->resultType()); - sop->resultType(sop->operationType()); - if (sop->op() == OP_EQ) { - CalpontSystemCatalog::TableAliasName tan_lhs; - CalpontSystemCatalog::TableAliasName tan_rhs; - bool outerjoin = (rhs->singleTable(tan_rhs) && lhs->singleTable(tan_lhs)); + if (gwip->subSelectType == CalpontSelectExecutionPlan::IN_SUBS || + gwip->subSelectType == CalpontSelectExecutionPlan::EXISTS_SUBS) + correlatedCol->joinInfo(correlatedCol->joinInfo() | JOIN_SEMI); + else if (gwip->subSelectType == CalpontSelectExecutionPlan::NOT_IN_SUBS || + gwip->subSelectType == CalpontSelectExecutionPlan::NOT_EXISTS_SUBS) + correlatedCol->joinInfo(correlatedCol->joinInfo() | JOIN_ANTI); + } + } - // @bug 1632. Alias should be taken account to the identity of tables for selfjoin to work - if (outerjoin && tan_lhs != tan_rhs) // join + SimpleFilter* sf = new SimpleFilter(); + sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + + //@bug 2101 for when there are only constants in a delete or update where clause (eg "where 5 < 6"). + // There will be no field column and it will get here only if the comparison is true. + if (gwip->columnMap.empty() && ((current_thd->lex->sql_command == SQLCOM_UPDATE) || + (current_thd->lex->sql_command == SQLCOM_UPDATE_MULTI) || + (current_thd->lex->sql_command == SQLCOM_DELETE) || + (current_thd->lex->sql_command == SQLCOM_DELETE_MULTI))) + { + IDEBUG(cerr << "deleted func with 2 const columns" << endl); + delete rhs; + delete lhs; + return false; + } + + // handle noop (only for table mode) + if (rhs->data() == "noop" || lhs->data() == "noop") + { + sop.reset(new Operator("noop")); + } + else + { + for (uint32_t i = 0; i < itemList.size(); i++) + { + if (isPredicateFunction(itemList[i], gwip)) + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SUB_EXPRESSION); + } + } + } + + sf->op(sop); + sf->lhs(lhs); + sf->rhs(rhs); + sop->setOpType(lhs->resultType(), rhs->resultType()); + sop->resultType(sop->operationType()); + + if (sop->op() == OP_EQ) + { + CalpontSystemCatalog::TableAliasName tan_lhs; + CalpontSystemCatalog::TableAliasName tan_rhs; + bool outerjoin = (rhs->singleTable(tan_rhs) && lhs->singleTable(tan_lhs)); + + // @bug 1632. Alias should be taken account to the identity of tables for selfjoin to work + if (outerjoin && tan_lhs != tan_rhs) // join + { + if (!gwip->condPush) // vtable + { + if (!gwip->innerTables.empty()) { - if (!gwip->condPush) // vtable - { - if (!gwip->innerTables.empty()) - { - checkOuterTableColumn(gwip, tan_lhs, lhs); - checkOuterTableColumn(gwip, tan_rhs, rhs); - } - - if (funcType == Item_func::EQ_FUNC) - { - gwip->equiCondSFList.push_back(sf); - } - - ParseTree* ptp = new ParseTree(sf); - gwip->ptWorkStack.push(ptp); - } + checkOuterTableColumn(gwip, tan_lhs, lhs); + checkOuterTableColumn(gwip, tan_rhs, rhs); } - else + + if (funcType == Item_func::EQ_FUNC) { - ParseTree* ptp = new ParseTree(sf); - gwip->ptWorkStack.push(ptp); + gwip->equiCondSFList.push_back(sf); } + + ParseTree* ptp = new ParseTree(sf); + gwip->ptWorkStack.push(ptp); + } } else { - ParseTree* ptp = new ParseTree(sf); - gwip->ptWorkStack.push(ptp); + ParseTree* ptp = new ParseTree(sf); + gwip->ptWorkStack.push(ptp); } + } + else + { + ParseTree* ptp = new ParseTree(sf); + gwip->ptWorkStack.push(ptp); + } - return true; + return true; } bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { - boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); + boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); - if (ifp->functype() == Item_func::LIKE_FUNC) + if (ifp->functype() == Item_func::LIKE_FUNC) + { + // Starting with MariaDB 10.2, LIKE uses a negated flag instead of FUNC_NOT + // Further processing is done below as before for LIKE + if (((Item_func_like*)ifp)->get_negated()) { - // Starting with MariaDB 10.2, LIKE uses a negated flag instead of FUNC_NOT - // Further processing is done below as before for LIKE - if (((Item_func_like*)ifp)->get_negated()) - { - sop->reverseOp(); - } + sop->reverseOp(); + } + } + + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + if (ifp->functype() == Item_func::BETWEEN) + { + idbassert(gwip->rcWorkStack.size() >= 3); + ReturnedColumn* rhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + ReturnedColumn* lhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + ReturnedColumn* filterCol = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); // pop gwip->scsp; + Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; + SimpleFilter* sfr = 0; + SimpleFilter* sfl = 0; + + if (inp->negated) + { + sop.reset(new PredicateOperator(">")); + sop->setOpType(filterCol->resultType(), rhs->resultType()); + sfr = new SimpleFilter(sop, filterCol, rhs); + sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sop.reset(new PredicateOperator("<")); + sop->setOpType(filterCol->resultType(), lhs->resultType()); + sfl = new SimpleFilter(sop, filterCol->clone(), lhs); + sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + ParseTree* ptp = new ParseTree(new LogicOperator("or")); + ptp->left(sfr); + ptp->right(sfl); + gwip->ptWorkStack.push(ptp); + } + else + { + sop.reset(new PredicateOperator("<=")); + sop->setOpType(filterCol->resultType(), rhs->resultType()); + sfr = new SimpleFilter(sop, filterCol, rhs); + sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sop.reset(new PredicateOperator(">=")); + sop->setOpType(filterCol->resultType(), lhs->resultType()); + sfl = new SimpleFilter(sop, filterCol->clone(), lhs); + sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + ParseTree* ptp = new ParseTree(new LogicOperator("and")); + ptp->left(sfr); + ptp->right(sfl); + gwip->ptWorkStack.push(ptp); } - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + return true; + } + else if (ifp->functype() == Item_func::IN_FUNC) + { + idbassert(gwip->rcWorkStack.size() >= 2); + ReturnedColumn* rhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + ReturnedColumn* lhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); - if (ifp->functype() == Item_func::BETWEEN) + // @bug3038 + RowColumn* rrhs = dynamic_cast(rhs); + RowColumn* rlhs = dynamic_cast(lhs); + + if (rrhs && rlhs) { - idbassert (gwip->rcWorkStack.size() >= 3); - ReturnedColumn* rhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - ReturnedColumn* lhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - ReturnedColumn* filterCol = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); // pop gwip->scsp; - Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; - SimpleFilter* sfr = 0; - SimpleFilter* sfl = 0; - - if (inp->negated) - { - sop.reset(new PredicateOperator(">")); - sop->setOpType(filterCol->resultType(), rhs->resultType()); - sfr = new SimpleFilter(sop, filterCol, rhs); - sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - sop.reset(new PredicateOperator("<")); - sop->setOpType(filterCol->resultType(), lhs->resultType()); - sfl = new SimpleFilter(sop, filterCol->clone(), lhs); - sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - ParseTree* ptp = new ParseTree(new LogicOperator("or")); - ptp->left(sfr); - ptp->right(sfl); - gwip->ptWorkStack.push(ptp); - } - else - { - sop.reset(new PredicateOperator("<=")); - sop->setOpType(filterCol->resultType(), rhs->resultType()); - sfr = new SimpleFilter(sop, filterCol, rhs); - sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - sop.reset(new PredicateOperator(">=")); - sop->setOpType(filterCol->resultType(), lhs->resultType()); - sfl = new SimpleFilter(sop, filterCol->clone(), lhs); - sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - ParseTree* ptp = new ParseTree(new LogicOperator("and")); - ptp->left(sfr); - ptp->right(sfl); - gwip->ptWorkStack.push(ptp); - } - - return true; - } - else if (ifp->functype() == Item_func::IN_FUNC) - { - idbassert(gwip->rcWorkStack.size() >= 2); - ReturnedColumn* rhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - ReturnedColumn* lhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - - // @bug3038 - RowColumn* rrhs = dynamic_cast(rhs); - RowColumn* rlhs = dynamic_cast(lhs); - - if (rrhs && rlhs) - { - return buildRowColumnFilter(gwip, rrhs, rlhs, ifp); - } - - ConstantColumn* crhs = dynamic_cast(rhs); - ConstantColumn* clhs = dynamic_cast(lhs); - - if (!crhs || !clhs) - { - gwip->fatalParseError = true; - gwip->parseErrorText = "non constant value in IN clause"; - return false; - } - - string eqop; - string cmbop; - Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; - - if (inp->negated) - { - eqop = "<>"; - cmbop = "and"; - } - else - { - eqop = "="; - cmbop = "or"; - } - - sop.reset(new PredicateOperator(eqop)); - sop->setOpType(gwip->scsp->resultType(), rhs->resultType()); - ConstantFilter* cf = 0; - - cf = new ConstantFilter(sop, gwip->scsp->clone(), rhs); - sop.reset(new LogicOperator(cmbop)); - cf->op(sop); - sop.reset(new PredicateOperator(eqop)); - sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); - - while (!gwip->rcWorkStack.empty()) - { - lhs = gwip->rcWorkStack.top(); - - if (dynamic_cast(lhs) == 0) break; - - gwip->rcWorkStack.pop(); - sop.reset(new PredicateOperator(eqop)); - sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); - } - - if (!gwip->rcWorkStack.empty()) - gwip->rcWorkStack.pop(); // pop gwip->scsp - - if (cf->filterList().size() < inp->argument_count() - 1) - { - gwip->fatalParseError = true; - gwip->parseErrorText = "non constant value in IN clause"; - delete cf; - return false; - } - - cf->functionName(gwip->funcName); - String str; - // @bug5811. This filter string is for cross engine to use. - // Use real table name. - ifp->print(&str, QT_ORDINARY); - IDEBUG(cerr << str.ptr() << endl); - - if (str.ptr()) - cf->data(str.c_ptr()); - - ParseTree* ptp = new ParseTree(cf); - gwip->ptWorkStack.push(ptp); + return buildRowColumnFilter(gwip, rrhs, rlhs, ifp); } - else if (ifp->functype() == Item_func::ISNULL_FUNC || - ifp->functype() == Item_func::ISNOTNULL_FUNC) - { - ReturnedColumn* rhs = NULL; - if (!gwip->rcWorkStack.empty() && !gwip->inCaseStmt) - { - rhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - } - else - { - rhs = buildReturnedColumn(ifp->arguments()[0], *gwip, gwip->fatalParseError); - } + ConstantColumn* crhs = dynamic_cast(rhs); + ConstantColumn* clhs = dynamic_cast(lhs); - if (rhs && !gwip->fatalParseError) - buildConstPredicate(ifp, rhs, gwip); - else if (!rhs) // @bug 3802 - { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - return false; - } + if (!crhs || !clhs) + { + gwip->fatalParseError = true; + gwip->parseErrorText = "non constant value in IN clause"; + return false; } - else if (ifp->functype() == Item_func::GUSERVAR_FUNC) + string eqop; + string cmbop; + Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; + + if (inp->negated) { - Item_func_get_user_var* udf = reinterpret_cast(ifp); - String buf; - - if (udf->result_type() == INT_RESULT) - { - if (udf->unsigned_flag) - { - gwip->rcWorkStack.push(new ConstantColumn((uint64_t)udf->val_uint())); - } - else - { - gwip->rcWorkStack.push(new ConstantColumn((int64_t)udf->val_int())); - } - (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - } - else - { - //const String* str = udf->val_str(&buf); - udf->val_str(&buf); - - if (!buf.ptr()) - { - ostringstream oss; - oss << "Unknown user variable: " << udf->name.str; - gwip->parseErrorText = oss.str(); - gwip->fatalParseError = true; - return false; - } - - if (udf->result_type() == STRING_RESULT) - gwip->rcWorkStack.push(new ConstantColumn(buf.ptr())); - else - { - gwip->rcWorkStack.push(new ConstantColumn(buf.ptr(), ConstantColumn::NUM)); - } - (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - - return false; - } + eqop = "<>"; + cmbop = "and"; } + else + { + eqop = "="; + cmbop = "or"; + } + + sop.reset(new PredicateOperator(eqop)); + sop->setOpType(gwip->scsp->resultType(), rhs->resultType()); + ConstantFilter* cf = 0; + + cf = new ConstantFilter(sop, gwip->scsp->clone(), rhs); + sop.reset(new LogicOperator(cmbop)); + cf->op(sop); + sop.reset(new PredicateOperator(eqop)); + sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); + cf->pushFilter( + new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); + + while (!gwip->rcWorkStack.empty()) + { + lhs = gwip->rcWorkStack.top(); + + if (dynamic_cast(lhs) == 0) + break; + + gwip->rcWorkStack.pop(); + sop.reset(new PredicateOperator(eqop)); + sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); + cf->pushFilter( + new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); + } + + if (!gwip->rcWorkStack.empty()) + gwip->rcWorkStack.pop(); // pop gwip->scsp + + if (cf->filterList().size() < inp->argument_count() - 1) + { + gwip->fatalParseError = true; + gwip->parseErrorText = "non constant value in IN clause"; + delete cf; + return false; + } + + cf->functionName(gwip->funcName); + String str; + // @bug5811. This filter string is for cross engine to use. + // Use real table name. + ifp->print(&str, QT_ORDINARY); + IDEBUG(cerr << str.ptr() << endl); + + if (str.ptr()) + cf->data(str.c_ptr()); + + ParseTree* ptp = new ParseTree(cf); + gwip->ptWorkStack.push(ptp); + } + + else if (ifp->functype() == Item_func::ISNULL_FUNC || ifp->functype() == Item_func::ISNOTNULL_FUNC) + { + ReturnedColumn* rhs = NULL; + if (!gwip->rcWorkStack.empty() && !gwip->inCaseStmt) + { + rhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + } + else + { + rhs = buildReturnedColumn(ifp->arguments()[0], *gwip, gwip->fatalParseError); + } + + if (rhs && !gwip->fatalParseError) + buildConstPredicate(ifp, rhs, gwip); + else if (!rhs) // @bug 3802 + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + return false; + } + } + + else if (ifp->functype() == Item_func::GUSERVAR_FUNC) + { + Item_func_get_user_var* udf = reinterpret_cast(ifp); + String buf; + + if (udf->result_type() == INT_RESULT) + { + if (udf->unsigned_flag) + { + gwip->rcWorkStack.push(new ConstantColumn((uint64_t)udf->val_uint())); + } + else + { + gwip->rcWorkStack.push(new ConstantColumn((int64_t)udf->val_int())); + } + (dynamic_cast(gwip->rcWorkStack.top())) + ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + } + else + { + // const String* str = udf->val_str(&buf); + udf->val_str(&buf); + + if (!buf.ptr()) + { + ostringstream oss; + oss << "Unknown user variable: " << udf->name.str; + gwip->parseErrorText = oss.str(); + gwip->fatalParseError = true; + return false; + } + + if (udf->result_type() == STRING_RESULT) + gwip->rcWorkStack.push(new ConstantColumn(buf.ptr())); + else + { + gwip->rcWorkStack.push(new ConstantColumn(buf.ptr(), ConstantColumn::NUM)); + } + (dynamic_cast(gwip->rcWorkStack.top())) + ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + + return false; + } + } #if 0 else if (ifp->functype() == Item_func::NEG_FUNC) @@ -2148,640 +2122,638 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) } #endif - else if (ifp->functype() == Item_func::NOT_FUNC) + else if (ifp->functype() == Item_func::NOT_FUNC) + { + if (gwip->condPush && ifp->next->type() == Item::SUBSELECT_ITEM) + return false; + + if (ifp->next && ifp->next->type() == Item::SUBSELECT_ITEM && gwip->lastSub) { - if (gwip->condPush && ifp->next->type() == Item::SUBSELECT_ITEM) - return false; + gwip->lastSub->handleNot(); + return false; + } - if (ifp->next && ifp->next->type() == Item::SUBSELECT_ITEM && gwip->lastSub) + idbassert(ifp->argument_count() == 1); + ParseTree* ptp = 0; + if (((Item_func*)(ifp->arguments()[0]))->functype() == Item_func::EQUAL_FUNC) + { + // negate it in place + // Note that an EQUAL_FUNC ( a <=> b) was converted to + // ( a = b OR ( a is null AND b is null) ) + // NOT of the above expression is: ( a != b AND (a is not null OR b is not null ) + + if (!gwip->ptWorkStack.empty()) + ptp = gwip->ptWorkStack.top(); + + if (ptp) + { + ParseTree* or_ptp = ptp; + ParseTree* and_ptp = or_ptp->right(); + ParseTree* equal_ptp = or_ptp->left(); + ParseTree* nullck_left_ptp = and_ptp->left(); + ParseTree* nullck_right_ptp = and_ptp->right(); + SimpleFilter* sf_left_nullck = dynamic_cast(nullck_left_ptp->data()); + SimpleFilter* sf_right_nullck = dynamic_cast(nullck_right_ptp->data()); + SimpleFilter* sf_equal = dynamic_cast(equal_ptp->data()); + + if (sf_left_nullck && sf_right_nullck && sf_equal) { - gwip->lastSub->handleNot(); - return false; - } - - idbassert(ifp->argument_count() == 1); - ParseTree* ptp = 0; - if (((Item_func*)(ifp->arguments()[0]))->functype() == Item_func::EQUAL_FUNC) - { - // negate it in place - // Note that an EQUAL_FUNC ( a <=> b) was converted to - // ( a = b OR ( a is null AND b is null) ) - // NOT of the above expression is: ( a != b AND (a is not null OR b is not null ) - - if (!gwip->ptWorkStack.empty()) - ptp = gwip->ptWorkStack.top(); - - if (ptp) - { - ParseTree* or_ptp = ptp; - ParseTree* and_ptp = or_ptp->right(); - ParseTree* equal_ptp = or_ptp->left(); - ParseTree* nullck_left_ptp = and_ptp->left(); - ParseTree* nullck_right_ptp = and_ptp->right(); - SimpleFilter *sf_left_nullck = dynamic_cast(nullck_left_ptp->data()); - SimpleFilter *sf_right_nullck = dynamic_cast(nullck_right_ptp->data()); - SimpleFilter *sf_equal = dynamic_cast(equal_ptp->data()); - - if (sf_left_nullck && sf_right_nullck && sf_equal) { - // Negate the null checks - sf_left_nullck->op()->reverseOp(); - sf_right_nullck->op()->reverseOp(); - sf_equal->op()->reverseOp(); - // Rehook the nodes - ptp = and_ptp; - ptp->left(equal_ptp); - ptp->right(or_ptp); - or_ptp->left(nullck_left_ptp); - or_ptp->right(nullck_right_ptp); - gwip->ptWorkStack.pop(); - gwip->ptWorkStack.push(ptp); - } - else { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ASSERTION_FAILURE); - return false; - } - } - } - else if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM) - { - // negate it in place - if (!gwip->ptWorkStack.empty()) - ptp = gwip->ptWorkStack.top(); - - SimpleFilter* sf = 0; - - if (ptp) - { - sf = dynamic_cast(ptp->data()); - - if (sf) - sf->op()->reverseOp(); - } + // Negate the null checks + sf_left_nullck->op()->reverseOp(); + sf_right_nullck->op()->reverseOp(); + sf_equal->op()->reverseOp(); + // Rehook the nodes + ptp = and_ptp; + ptp->left(equal_ptp); + ptp->right(or_ptp); + or_ptp->left(nullck_left_ptp); + or_ptp->right(nullck_right_ptp); + gwip->ptWorkStack.pop(); + gwip->ptWorkStack.push(ptp); } else { - // transfrom the not item to item = 0 - ReturnedColumn* rhs = 0; - - if (!gwip->rcWorkStack.empty()) - { - rhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - } - else - { - rhs = buildReturnedColumn(ifp->arguments()[0], *gwip, gwip->fatalParseError); - } - - if (rhs && !gwip->fatalParseError) - return buildConstPredicate(ifp, rhs, gwip); - else if (!rhs) // @bug3802 - { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - return false; - } + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ASSERTION_FAILURE); + return false; } + } } - /*else if (ifp->functype() == Item_func::MULT_EQUAL_FUNC) + else if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM) { - Item_equal *cur_item_eq = (Item_equal*)ifp; - Item *lhs_item, *rhs_item; - // There must be at least two items - Item_equal_fields_iterator lhs_it(*cur_item_eq); - Item_equal_fields_iterator rhs_it(*cur_item_eq); rhs_it++; - while ((lhs_item = lhs_it++) && (rhs_item = rhs_it++)) - { - ReturnedColumn* lhs = buildReturnedColumn(lhs_item, *gwip, gwip->fatalParseError); - ReturnedColumn* rhs = buildReturnedColumn(rhs_item, *gwip, gwip->fatalParseError); - if (!rhs || !lhs) - { - gwip->fatalParseError = true; - gwip->parseErrorText = "Unsupport elements in MULT_EQUAL item"; - delete rhs; - delete lhs; - return false; - } - PredicateOperator* po = new PredicateOperator("="); - boost::shared_ptr sop(po); - sop->setOpType(lhs->resultType(), rhs->resultType()); - SimpleFilter* sf = new SimpleFilter(sop, lhs, rhs); - // Search in ptWorkStack for duplicates of the SF - // before we push the SF - if (!isDuplicateSF(gwip, sf)) - { - ParseTree* pt = new ParseTree(sf); - gwip->ptWorkStack.push(pt); - } - } - }*/ - else if (ifp->functype() == Item_func::EQUAL_FUNC) - { - // Convert "a <=> b" to (a = b OR (a IS NULL AND b IS NULL))" - idbassert (gwip->rcWorkStack.size() >= 2); - ReturnedColumn* rhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - ReturnedColumn* lhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - SimpleFilter* sfn1 = 0; - SimpleFilter* sfn2 = 0; - SimpleFilter* sfo = 0; - // b IS NULL - ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); - nlhs1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - sop.reset(new PredicateOperator("isnull")); - sop->setOpType(lhs->resultType(), rhs->resultType()); - sfn1 = new SimpleFilter(sop, rhs, nlhs1); - sfn1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - ParseTree* ptpl = new ParseTree(sfn1); - // a IS NULL - ConstantColumn* nlhs2 = new ConstantColumn("", ConstantColumn::NULLDATA); - nlhs2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - sop.reset(new PredicateOperator("isnull")); - sop->setOpType(lhs->resultType(), rhs->resultType()); - sfn2 = new SimpleFilter(sop, lhs, nlhs2); - sfn2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - ParseTree* ptpr = new ParseTree(sfn2); - // AND them both - ParseTree* ptpn = new ParseTree(new LogicOperator("and")); - ptpn->left(ptpl); - ptpn->right(ptpr); - // a = b - sop.reset(new PredicateOperator("=")); - sop->setOpType(lhs->resultType(), rhs->resultType()); - sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); - sfo->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - // OR with the NULL comparison tree - ParseTree* ptp = new ParseTree(new LogicOperator("or")); - ptp->left(sfo); - ptp->right(ptpn); - gwip->ptWorkStack.push(ptp); - return true; + // negate it in place + if (!gwip->ptWorkStack.empty()) + ptp = gwip->ptWorkStack.top(); + + SimpleFilter* sf = 0; + + if (ptp) + { + sf = dynamic_cast(ptp->data()); + + if (sf) + sf->op()->reverseOp(); + } } - else //std rel ops (incl "like") + else { - if (gwip->rcWorkStack.size() < 2) - { - idbassert(ifp->argument_count() == 2); + // transfrom the not item to item = 0 + ReturnedColumn* rhs = 0; - if (isPredicateFunction(ifp->arguments()[0], gwip) || - ifp->arguments()[0]->type() == Item::COND_ITEM || - isPredicateFunction(ifp->arguments()[1], gwip) || - ifp->arguments()[1]->type() == Item::COND_ITEM) - { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - } - - return false; - } - - ReturnedColumn* rhs = gwip->rcWorkStack.top(); - gwip->rcWorkStack.pop(); - ReturnedColumn* lhs = gwip->rcWorkStack.top(); + if (!gwip->rcWorkStack.empty()) + { + rhs = gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); + } + else + { + rhs = buildReturnedColumn(ifp->arguments()[0], *gwip, gwip->fatalParseError); + } - // @bug3038. rowcolumn filter - RowColumn* rrhs = dynamic_cast(rhs); - RowColumn* rlhs = dynamic_cast(lhs); - - if (rrhs && rlhs) - return buildRowColumnFilter(gwip, rrhs, rlhs, ifp); - - vector itemList; - - for (uint32_t i = 0; i < ifp->argument_count(); i++) - itemList.push_back(ifp->arguments()[i]); - - return buildEqualityPredicate(lhs, rhs, gwip, sop, ifp->functype(), itemList); + if (rhs && !gwip->fatalParseError) + return buildConstPredicate(ifp, rhs, gwip); + else if (!rhs) // @bug3802 + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + return false; + } } - + } + /*else if (ifp->functype() == Item_func::MULT_EQUAL_FUNC) + { + Item_equal *cur_item_eq = (Item_equal*)ifp; + Item *lhs_item, *rhs_item; + // There must be at least two items + Item_equal_fields_iterator lhs_it(*cur_item_eq); + Item_equal_fields_iterator rhs_it(*cur_item_eq); rhs_it++; + while ((lhs_item = lhs_it++) && (rhs_item = rhs_it++)) + { + ReturnedColumn* lhs = buildReturnedColumn(lhs_item, *gwip, gwip->fatalParseError); + ReturnedColumn* rhs = buildReturnedColumn(rhs_item, *gwip, gwip->fatalParseError); + if (!rhs || !lhs) + { + gwip->fatalParseError = true; + gwip->parseErrorText = "Unsupport elements in MULT_EQUAL item"; + delete rhs; + delete lhs; + return false; + } + PredicateOperator* po = new PredicateOperator("="); + boost::shared_ptr sop(po); + sop->setOpType(lhs->resultType(), rhs->resultType()); + SimpleFilter* sf = new SimpleFilter(sop, lhs, rhs); + // Search in ptWorkStack for duplicates of the SF + // before we push the SF + if (!isDuplicateSF(gwip, sf)) + { + ParseTree* pt = new ParseTree(sf); + gwip->ptWorkStack.push(pt); + } + } + }*/ + else if (ifp->functype() == Item_func::EQUAL_FUNC) + { + // Convert "a <=> b" to (a = b OR (a IS NULL AND b IS NULL))" + idbassert(gwip->rcWorkStack.size() >= 2); + ReturnedColumn* rhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + ReturnedColumn* lhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + SimpleFilter* sfn1 = 0; + SimpleFilter* sfn2 = 0; + SimpleFilter* sfo = 0; + // b IS NULL + ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); + nlhs1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sop.reset(new PredicateOperator("isnull")); + sop->setOpType(lhs->resultType(), rhs->resultType()); + sfn1 = new SimpleFilter(sop, rhs, nlhs1); + sfn1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + ParseTree* ptpl = new ParseTree(sfn1); + // a IS NULL + ConstantColumn* nlhs2 = new ConstantColumn("", ConstantColumn::NULLDATA); + nlhs2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sop.reset(new PredicateOperator("isnull")); + sop->setOpType(lhs->resultType(), rhs->resultType()); + sfn2 = new SimpleFilter(sop, lhs, nlhs2); + sfn2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + ParseTree* ptpr = new ParseTree(sfn2); + // AND them both + ParseTree* ptpn = new ParseTree(new LogicOperator("and")); + ptpn->left(ptpl); + ptpn->right(ptpr); + // a = b + sop.reset(new PredicateOperator("=")); + sop->setOpType(lhs->resultType(), rhs->resultType()); + sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); + sfo->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + // OR with the NULL comparison tree + ParseTree* ptp = new ParseTree(new LogicOperator("or")); + ptp->left(sfo); + ptp->right(ptpn); + gwip->ptWorkStack.push(ptp); return true; + } + else // std rel ops (incl "like") + { + if (gwip->rcWorkStack.size() < 2) + { + idbassert(ifp->argument_count() == 2); + + if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM || + isPredicateFunction(ifp->arguments()[1], gwip) || ifp->arguments()[1]->type() == Item::COND_ITEM) + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + } + + return false; + } + + ReturnedColumn* rhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + ReturnedColumn* lhs = gwip->rcWorkStack.top(); + gwip->rcWorkStack.pop(); + + // @bug3038. rowcolumn filter + RowColumn* rrhs = dynamic_cast(rhs); + RowColumn* rlhs = dynamic_cast(lhs); + + if (rrhs && rlhs) + return buildRowColumnFilter(gwip, rrhs, rlhs, ifp); + + vector itemList; + + for (uint32_t i = 0; i < ifp->argument_count(); i++) + itemList.push_back(ifp->arguments()[i]); + + return buildEqualityPredicate(lhs, rhs, gwip, sop, ifp->functype(), itemList); + } + + return true; } bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip) { - SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); - ConstantColumn* lhs = 0; + SimpleFilter* sf = new SimpleFilter(); + sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); + ConstantColumn* lhs = 0; - if (ifp->functype() == Item_func::ISNULL_FUNC) - { - lhs = new ConstantColumn("", ConstantColumn::NULLDATA); - sop.reset(new PredicateOperator("isnull")); - } - else if (ifp->functype() == Item_func::ISNOTNULL_FUNC) - { - lhs = new ConstantColumn("", ConstantColumn::NULLDATA); - sop.reset(new PredicateOperator("isnotnull")); - } - else //if (ifp->functype() == Item_func::NOT_FUNC) - { - lhs = new ConstantColumn((int64_t)0, ConstantColumn::NUM); - sop.reset(new PredicateOperator("=")); - } - lhs->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + if (ifp->functype() == Item_func::ISNULL_FUNC) + { + lhs = new ConstantColumn("", ConstantColumn::NULLDATA); + sop.reset(new PredicateOperator("isnull")); + } + else if (ifp->functype() == Item_func::ISNOTNULL_FUNC) + { + lhs = new ConstantColumn("", ConstantColumn::NULLDATA); + sop.reset(new PredicateOperator("isnotnull")); + } + else // if (ifp->functype() == Item_func::NOT_FUNC) + { + lhs = new ConstantColumn((int64_t)0, ConstantColumn::NUM); + sop.reset(new PredicateOperator("=")); + } + lhs->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - CalpontSystemCatalog::ColType opType = rhs->resultType(); + CalpontSystemCatalog::ColType opType = rhs->resultType(); - if ( (opType.colDataType == CalpontSystemCatalog::CHAR && opType.colWidth <= 8) || - (opType.colDataType == CalpontSystemCatalog::VARCHAR && opType.colWidth < 8) || - (opType.colDataType == CalpontSystemCatalog::VARBINARY && opType.colWidth < 8) ) - { - opType.colDataType = execplan::CalpontSystemCatalog::BIGINT; - opType.colWidth = 8; - } + if ((opType.colDataType == CalpontSystemCatalog::CHAR && opType.colWidth <= 8) || + (opType.colDataType == CalpontSystemCatalog::VARCHAR && opType.colWidth < 8) || + (opType.colDataType == CalpontSystemCatalog::VARBINARY && opType.colWidth < 8)) + { + opType.colDataType = execplan::CalpontSystemCatalog::BIGINT; + opType.colWidth = 8; + } - sop->operationType(opType); - sf->op(sop); + sop->operationType(opType); + sf->op(sop); - //yes, these are backwards - assert (lhs); - sf->lhs(rhs); - sf->rhs(lhs); - ParseTree* ptp = new ParseTree(sf); - gwip->ptWorkStack.push(ptp); - return true; + // yes, these are backwards + assert(lhs); + sf->lhs(rhs); + sf->rhs(lhs); + ParseTree* ptp = new ParseTree(sf); + gwip->ptWorkStack.push(ptp); + return true; } SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) { - SimpleColumn* sc = NULL; + SimpleColumn* sc = NULL; - // view name - string viewName = getViewName(ifp->cached_table); - - // Check from the end because local scope resolve is preferred - for (int32_t i = gwi.tbList.size() - 1; i >= 0; i--) - { - if (sc) break; - - if (!gwi.tbList[i].schema.empty() && !gwi.tbList[i].table.empty() && - (!ifp->table_name.str || strcasecmp(ifp->table_name.str, gwi.tbList[i].alias.c_str()) == 0)) - { - CalpontSystemCatalog::TableName tn(gwi.tbList[i].schema, gwi.tbList[i].table); - CalpontSystemCatalog::RIDList oidlist = gwi.csc->columnRIDs(tn, true); - - for (unsigned int j = 0; j < oidlist.size(); j++) - { - CalpontSystemCatalog::TableColName tcn = gwi.csc->colName(oidlist[j].objnum); - CalpontSystemCatalog::ColType ct = gwi.csc->colType(oidlist[j].objnum); - - if (strcasecmp(ifp->field_name.str, tcn.column.c_str()) == 0) - { - // @bug4827. Remove the checking because outside tables could be the same - // name as inner tables. This function is to identify column from a table, - // as long as it matches the next step in predicate parsing will tell the scope - // of the column. - /*if (sc) - { - gwi.fatalParseError = true; - Message::Args args; - args.add(ifp->name); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_AMBIGUOUS_COL, args); - return NULL; - }*/ - - sc = new SimpleColumn(); - sc->columnName(tcn.column); - sc->tableName(tcn.table); - sc->schemaName(tcn.schema); - sc->oid(oidlist[j].objnum); - - // @bug 3003. Keep column alias if it has. - sc->alias(!ifp->is_explicit_name() ? tcn.column : ifp->name.str); - - sc->tableAlias(gwi.tbList[i].alias); - sc->viewName(viewName, lower_case_table_names); - sc->resultType(ct); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - break; - } - } - } - } + // view name + string viewName = getViewName(ifp->cached_table); + // Check from the end because local scope resolve is preferred + for (int32_t i = gwi.tbList.size() - 1; i >= 0; i--) + { if (sc) - return sc; + break; - // Check from the end because local scope resolve is preferred - for (int32_t i = gwi.derivedTbList.size() - 1; i >= 0; i--) + if (!gwi.tbList[i].schema.empty() && !gwi.tbList[i].table.empty() && + (!ifp->table_name.str || strcasecmp(ifp->table_name.str, gwi.tbList[i].alias.c_str()) == 0)) { - if (sc) break; + CalpontSystemCatalog::TableName tn(gwi.tbList[i].schema, gwi.tbList[i].table); + CalpontSystemCatalog::RIDList oidlist = gwi.csc->columnRIDs(tn, true); - CalpontSelectExecutionPlan* csep = dynamic_cast(gwi.derivedTbList[i].get()); - string derivedName = csep->derivedTbAlias(); + for (unsigned int j = 0; j < oidlist.size(); j++) + { + CalpontSystemCatalog::TableColName tcn = gwi.csc->colName(oidlist[j].objnum); + CalpontSystemCatalog::ColType ct = gwi.csc->colType(oidlist[j].objnum); - if (!ifp->table_name.str || strcasecmp(ifp->table_name.str, derivedName.c_str()) == 0) + if (strcasecmp(ifp->field_name.str, tcn.column.c_str()) == 0) { - CalpontSelectExecutionPlan::ReturnedColumnList cols = csep->returnedCols(); + // @bug4827. Remove the checking because outside tables could be the same + // name as inner tables. This function is to identify column from a table, + // as long as it matches the next step in predicate parsing will tell the scope + // of the column. + /*if (sc) + { + gwi.fatalParseError = true; + Message::Args args; + args.add(ifp->name); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_AMBIGUOUS_COL, args); + return NULL; + }*/ - for (uint32_t j = 0; j < cols.size(); j++) - { - // @bug 3167 duplicate column alias is full name - SimpleColumn* col = dynamic_cast(cols[j].get()); - string alias = cols[j]->alias(); + sc = new SimpleColumn(); + sc->columnName(tcn.column); + sc->tableName(tcn.table); + sc->schemaName(tcn.schema); + sc->oid(oidlist[j].objnum); - if (strcasecmp(ifp->field_name.str, alias.c_str()) == 0 || - (col && alias.find(".") != string::npos && - (strcasecmp(ifp->field_name.str, col->columnName().c_str()) == 0 || - strcasecmp(ifp->field_name.str, (alias.substr(alias.find_last_of(".") + 1)).c_str()) == 0))) //@bug6066 - { - // @bug4827. Remove the checking because outside tables could be the same - // name as inner tables. This function is to identify column from a table, - // as long as it matches the next step in predicate parsing will tell the scope - // of the column. - /*if (sc) - { - gwi.fatalParseError = true; - Message::Args args; - args.add(ifp->name); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_AMBIGUOUS_COL, args); - return NULL; - }*/ - sc = new SimpleColumn(); + // @bug 3003. Keep column alias if it has. + sc->alias(!ifp->is_explicit_name() ? tcn.column : ifp->name.str); - if (!col) - sc->columnName(cols[j]->alias()); - else - sc->columnName(col->columnName()); - - // @bug 3003. Keep column alias if it has. - sc->alias(!ifp->is_explicit_name() ? cols[j]->alias() : ifp->name.str); - sc->tableName(csep->derivedTbAlias()); - sc->colPosition(j); - sc->tableAlias(csep->derivedTbAlias()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - if (!viewName.empty()) - { - sc->viewName(viewName, lower_case_table_names); - } - else - { - sc->viewName(csep->derivedTbView()); - } - sc->resultType(cols[j]->resultType()); - sc->hasAggregate(cols[j]->hasAggregate()); - - if (col) - sc->isColumnStore(col->isColumnStore()); - - // @bug5634, @bug5635. mark used derived col on derived table. - // outer join inner table filter can not be moved in - // MariaDB 10.1: cached_table is never available for derived tables. - // Find the uncached object in table_list - TABLE_LIST* tblList = ifp->context ? ifp->context->table_list : NULL; - - while (tblList) - { - if (strcasecmp(tblList->alias.str, ifp->table_name.str) == 0) - { - if (!tblList->outer_join) - { - sc->derivedTable(derivedName); - sc->derivedRefCol(cols[j].get()); - } - - break; - } - - tblList = tblList->next_local; - } - - cols[j]->incRefCount(); - break; - } - } + sc->tableAlias(gwi.tbList[i].alias); + sc->viewName(viewName, lower_case_table_names); + sc->resultType(ct); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + break; } + } } + } - if (!sc) - { - gwi.fatalParseError = true; - Message::Args args; - string name; - - if (ifp->db_name.str) - name += string(ifp->db_name.str) + "."; - - if (ifp->table_name.str) - name += string(ifp->table_name.str) + "."; - - if (ifp->name.length) - name += ifp->name.str; - args.add(name); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL, args); - } - - if (ifp->depended_from) - { - sc->joinInfo(sc->joinInfo() | JOIN_CORRELATED); - - if (gwi.subQuery) - gwi.subQuery->correlated(true); - - // for error out non-support select filter case (comparison outside semi join tables) - gwi.correlatedTbNameVec.push_back(make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias())); - - // imply semi for scalar for now. - if (gwi.subSelectType == CalpontSelectExecutionPlan::SINGLEROW_SUBS) - sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_SEMI); - - if (gwi.subSelectType == CalpontSelectExecutionPlan::SELECT_SUBS) - sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_OUTER_SELECT); - } - + if (sc) return sc; + + // Check from the end because local scope resolve is preferred + for (int32_t i = gwi.derivedTbList.size() - 1; i >= 0; i--) + { + if (sc) + break; + + CalpontSelectExecutionPlan* csep = dynamic_cast(gwi.derivedTbList[i].get()); + string derivedName = csep->derivedTbAlias(); + + if (!ifp->table_name.str || strcasecmp(ifp->table_name.str, derivedName.c_str()) == 0) + { + CalpontSelectExecutionPlan::ReturnedColumnList cols = csep->returnedCols(); + + for (uint32_t j = 0; j < cols.size(); j++) + { + // @bug 3167 duplicate column alias is full name + SimpleColumn* col = dynamic_cast(cols[j].get()); + string alias = cols[j]->alias(); + + if (strcasecmp(ifp->field_name.str, alias.c_str()) == 0 || + (col && alias.find(".") != string::npos && + (strcasecmp(ifp->field_name.str, col->columnName().c_str()) == 0 || + strcasecmp(ifp->field_name.str, (alias.substr(alias.find_last_of(".") + 1)).c_str()) == + 0))) //@bug6066 + { + // @bug4827. Remove the checking because outside tables could be the same + // name as inner tables. This function is to identify column from a table, + // as long as it matches the next step in predicate parsing will tell the scope + // of the column. + /*if (sc) + { + gwi.fatalParseError = true; + Message::Args args; + args.add(ifp->name); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_AMBIGUOUS_COL, args); + return NULL; + }*/ + sc = new SimpleColumn(); + + if (!col) + sc->columnName(cols[j]->alias()); + else + sc->columnName(col->columnName()); + + // @bug 3003. Keep column alias if it has. + sc->alias(!ifp->is_explicit_name() ? cols[j]->alias() : ifp->name.str); + sc->tableName(csep->derivedTbAlias()); + sc->colPosition(j); + sc->tableAlias(csep->derivedTbAlias()); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + if (!viewName.empty()) + { + sc->viewName(viewName, lower_case_table_names); + } + else + { + sc->viewName(csep->derivedTbView()); + } + sc->resultType(cols[j]->resultType()); + sc->hasAggregate(cols[j]->hasAggregate()); + + if (col) + sc->isColumnStore(col->isColumnStore()); + + // @bug5634, @bug5635. mark used derived col on derived table. + // outer join inner table filter can not be moved in + // MariaDB 10.1: cached_table is never available for derived tables. + // Find the uncached object in table_list + TABLE_LIST* tblList = ifp->context ? ifp->context->table_list : NULL; + + while (tblList) + { + if (strcasecmp(tblList->alias.str, ifp->table_name.str) == 0) + { + if (!tblList->outer_join) + { + sc->derivedTable(derivedName); + sc->derivedRefCol(cols[j].get()); + } + + break; + } + + tblList = tblList->next_local; + } + + cols[j]->incRefCount(); + break; + } + } + } + } + + if (!sc) + { + gwi.fatalParseError = true; + Message::Args args; + string name; + + if (ifp->db_name.str) + name += string(ifp->db_name.str) + "."; + + if (ifp->table_name.str) + name += string(ifp->table_name.str) + "."; + + if (ifp->name.length) + name += ifp->name.str; + args.add(name); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL, args); + } + + if (ifp->depended_from) + { + sc->joinInfo(sc->joinInfo() | JOIN_CORRELATED); + + if (gwi.subQuery) + gwi.subQuery->correlated(true); + + // for error out non-support select filter case (comparison outside semi join tables) + gwi.correlatedTbNameVec.push_back(make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias())); + + // imply semi for scalar for now. + if (gwi.subSelectType == CalpontSelectExecutionPlan::SINGLEROW_SUBS) + sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_SEMI); + + if (gwi.subSelectType == CalpontSelectExecutionPlan::SELECT_SUBS) + sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_OUTER_SELECT); + } + + return sc; } // for FROM clause subquery. get all the columns from real tables and derived tables. void collectAllCols(gp_walk_info& gwi, Item_field* ifp) { - // view name - string viewName = getViewName(ifp->cached_table); + // view name + string viewName = getViewName(ifp->cached_table); - if (gwi.derivedTbList.empty()) + if (gwi.derivedTbList.empty()) + { + gwi.fatalParseError = true; + Message::Args args; + args.add("*"); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL, args); + } + + string tableName = (ifp->table_name.str ? string(ifp->table_name.str) : ""); + CalpontSelectExecutionPlan::SelectList::const_iterator it = gwi.derivedTbList.begin(); + + for (uint32_t i = 0; i < gwi.tbList.size(); i++) + { + SRCP srcp; + + // derived table + if (gwi.tbList[i].schema.empty()) { - gwi.fatalParseError = true; - Message::Args args; - args.add("*"); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL, args); + CalpontSelectExecutionPlan* csep = dynamic_cast((*it).get()); + ++it; + if (!tableName.empty() && strcasecmp(tableName.c_str(), csep->derivedTbAlias().c_str()) != 0) + continue; + + CalpontSelectExecutionPlan::ReturnedColumnList cols = csep->returnedCols(); + + for (uint32_t j = 0; j < cols.size(); j++) + { + SimpleColumn* sc = new SimpleColumn(); + sc->columnName(cols[j]->alias()); + sc->colPosition(j); + sc->tableAlias(csep->derivedTbAlias()); + sc->viewName(gwi.tbList[i].view); + sc->resultType(cols[j]->resultType()); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + + // @bug5634 derived table optimization + cols[j]->incRefCount(); + sc->derivedTable(sc->tableAlias()); + sc->derivedRefCol(cols[j].get()); + srcp.reset(sc); + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + } } - - string tableName = (ifp->table_name.str ? string(ifp->table_name.str) : ""); - CalpontSelectExecutionPlan::SelectList::const_iterator it = gwi.derivedTbList.begin(); - - for (uint32_t i = 0; i < gwi.tbList.size(); i++) + // real tables + else { - SRCP srcp; + CalpontSystemCatalog::TableName tn(gwi.tbList[i].schema, gwi.tbList[i].table); - // derived table - if (gwi.tbList[i].schema.empty()) - { - CalpontSelectExecutionPlan* csep = dynamic_cast((*it).get()); - ++it; + if (lower_case_table_names) + { + if (!tableName.empty() && (strcasecmp(tableName.c_str(), tn.table.c_str()) != 0 && + strcasecmp(tableName.c_str(), gwi.tbList[i].alias.c_str()) != 0)) + continue; + } + else + { + if (!tableName.empty() && (strcmp(tableName.c_str(), tn.table.c_str()) != 0 && + strcmp(tableName.c_str(), gwi.tbList[i].alias.c_str()) != 0)) + continue; + } + if (lower_case_table_names) + { + boost::algorithm::to_lower(tn.schema); + boost::algorithm::to_lower(tn.table); + } + CalpontSystemCatalog::RIDList oidlist = gwi.csc->columnRIDs(tn, true); - if (!tableName.empty() && strcasecmp(tableName.c_str(), csep->derivedTbAlias().c_str()) != 0) - continue; - - CalpontSelectExecutionPlan::ReturnedColumnList cols = csep->returnedCols(); - - for (uint32_t j = 0; j < cols.size(); j++) - { - SimpleColumn* sc = new SimpleColumn(); - sc->columnName(cols[j]->alias()); - sc->colPosition(j); - sc->tableAlias(csep->derivedTbAlias()); - sc->viewName(gwi.tbList[i].view); - sc->resultType(cols[j]->resultType()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - - // @bug5634 derived table optimization - cols[j]->incRefCount(); - sc->derivedTable(sc->tableAlias()); - sc->derivedRefCol(cols[j].get()); - srcp.reset(sc); - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - } - } - // real tables - else - { - CalpontSystemCatalog::TableName tn(gwi.tbList[i].schema, gwi.tbList[i].table); - - if (lower_case_table_names) - { - if (!tableName.empty() && (strcasecmp(tableName.c_str(), tn.table.c_str()) != 0 && - strcasecmp(tableName.c_str(), gwi.tbList[i].alias.c_str()) != 0 )) - continue; - } - else - { - if (!tableName.empty() && (strcmp(tableName.c_str(), tn.table.c_str()) != 0 && - strcmp(tableName.c_str(), gwi.tbList[i].alias.c_str()) != 0 )) - continue; - } - if (lower_case_table_names) - { - boost::algorithm::to_lower(tn.schema); - boost::algorithm::to_lower(tn.table); - } - CalpontSystemCatalog::RIDList oidlist = gwi.csc->columnRIDs(tn, true); - - for (unsigned int j = 0; j < oidlist.size(); j++) - { - SimpleColumn* sc = new SimpleColumn(); - CalpontSystemCatalog::TableColName tcn = gwi.csc->colName(oidlist[j].objnum); - CalpontSystemCatalog::ColType ct = gwi.csc->colType(oidlist[j].objnum); - sc->columnName(tcn.column); - sc->tableName(tcn.table, lower_case_table_names); - sc->schemaName(tcn.schema, lower_case_table_names); - sc->oid(oidlist[j].objnum); - sc->alias(tcn.column); - sc->resultType(ct); - sc->tableAlias(gwi.tbList[i].alias, lower_case_table_names); - sc->viewName(viewName, lower_case_table_names); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - srcp.reset(sc); - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - } - } + for (unsigned int j = 0; j < oidlist.size(); j++) + { + SimpleColumn* sc = new SimpleColumn(); + CalpontSystemCatalog::TableColName tcn = gwi.csc->colName(oidlist[j].objnum); + CalpontSystemCatalog::ColType ct = gwi.csc->colType(oidlist[j].objnum); + sc->columnName(tcn.column); + sc->tableName(tcn.table, lower_case_table_names); + sc->schemaName(tcn.schema, lower_case_table_names); + sc->oid(oidlist[j].objnum); + sc->alias(tcn.column); + sc->resultType(ct); + sc->tableAlias(gwi.tbList[i].alias, lower_case_table_names); + sc->viewName(viewName, lower_case_table_names); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + srcp.reset(sc); + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + } } + } } void buildSubselectFunc(Item_func* ifp, gp_walk_info* gwip) { - // @bug 3035 - if (!isPredicateFunction(ifp, gwip)) - { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_FUNC_SUB); - return; - } + // @bug 3035 + if (!isPredicateFunction(ifp, gwip)) + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_FUNC_SUB); + return; + } - WhereSubQuery* subquery = NULL; + WhereSubQuery* subquery = NULL; - for (uint32_t i = 0; i < ifp->argument_count(); i++) - { + for (uint32_t i = 0; i < ifp->argument_count(); i++) + { #if MYSQL_VERSION_ID >= 50172 - // changes comply with mysql 5.1.72 - if (ifp->arguments()[i]->type() == Item::FUNC_ITEM && - string(((Item_func*)ifp->arguments()[i])->func_name()) == "") - { - if (ifp->functype() == Item_func::NOT_FUNC) - { - if (gwip->lastSub) - gwip->lastSub->handleNot(); - } - } + // changes comply with mysql 5.1.72 + if (ifp->arguments()[i]->type() == Item::FUNC_ITEM && + string(((Item_func*)ifp->arguments()[i])->func_name()) == "") + { + if (ifp->functype() == Item_func::NOT_FUNC) + { + if (gwip->lastSub) + gwip->lastSub->handleNot(); + } + } #endif - if (ifp->arguments()[i]->type() == Item::SUBSELECT_ITEM) - { - Item_subselect* sub = (Item_subselect*)ifp->arguments()[i]; - - switch (sub->substype()) - { - case Item_subselect::SINGLEROW_SUBS: - subquery = new ScalarSub(*gwip, ifp); - break; - - case Item_subselect::IN_SUBS: - subquery = new InSub(*gwip, ifp); - break; - - case Item_subselect::EXISTS_SUBS: - - // exists sub has been handled earlier. here is for not function - if (ifp->functype() == Item_func::NOT_FUNC) - { - if (gwip->lastSub) - gwip->lastSub->handleNot(); - } - - break; - - default: - Message::Args args; - gwip->fatalParseError = true; - gwip->parseErrorText = "non supported subquery"; - return; - } - } - } - - if (subquery) + if (ifp->arguments()[i]->type() == Item::SUBSELECT_ITEM) { - gwip->hasSubSelect = true; - SubQuery* orig = gwip->subQuery; - gwip->subQuery = subquery; - // no need to check NULL for now. error will be handled in gp_walk - gwip->ptWorkStack.push(subquery->transform()); - // recover original sub. Save current sub for Not handling. - gwip->lastSub = subquery; - gwip->subQuery = orig; - } + Item_subselect* sub = (Item_subselect*)ifp->arguments()[i]; - return; + switch (sub->substype()) + { + case Item_subselect::SINGLEROW_SUBS: subquery = new ScalarSub(*gwip, ifp); break; + + case Item_subselect::IN_SUBS: subquery = new InSub(*gwip, ifp); break; + + case Item_subselect::EXISTS_SUBS: + + // exists sub has been handled earlier. here is for not function + if (ifp->functype() == Item_func::NOT_FUNC) + { + if (gwip->lastSub) + gwip->lastSub->handleNot(); + } + + break; + + default: + Message::Args args; + gwip->fatalParseError = true; + gwip->parseErrorText = "non supported subquery"; + return; + } + } + } + + if (subquery) + { + gwip->hasSubSelect = true; + SubQuery* orig = gwip->subQuery; + gwip->subQuery = subquery; + // no need to check NULL for now. error will be handled in gp_walk + gwip->ptWorkStack.push(subquery->transform()); + // recover original sub. Save current sub for Not handling. + gwip->lastSub = subquery; + gwip->subQuery = orig; + } + + return; } bool isPredicateFunction(Item* item, gp_walk_info* gwip) { - if (item->type() == Item::COND_ITEM) - return true; + if (item->type() == Item::COND_ITEM) + return true; - if (item->type() != Item::FUNC_ITEM) - return false; + if (item->type() != Item::FUNC_ITEM) + return false; - Item_func* ifp = (Item_func*)item; - return ( ifp->functype() == Item_func::EQ_FUNC || + Item_func* ifp = (Item_func*)item; + return ( ifp->functype() == Item_func::EQ_FUNC || ifp->functype() == Item_func::NE_FUNC || ifp->functype() == Item_func::LT_FUNC || ifp->functype() == Item_func::LE_FUNC || @@ -2803,482 +2775,463 @@ bool isPredicateFunction(Item* item, gp_walk_info* gwip) void setError(THD* thd, uint32_t errcode, string errmsg) { - thd->get_stmt_da()->set_overwrite_status(true); + thd->get_stmt_da()->set_overwrite_status(true); - if (errmsg.empty()) - errmsg = "Unknown error"; + if (errmsg.empty()) + errmsg = "Unknown error"; - if (errcode < ER_ERROR_FIRST || errcode > ER_ERROR_LAST) - { - errcode = ER_UNKNOWN_ERROR; - } + if (errcode < ER_ERROR_FIRST || errcode > ER_ERROR_LAST) + { + errcode = ER_UNKNOWN_ERROR; + } - thd->raise_error_printf(errcode, errmsg.c_str()); + thd->raise_error_printf(errcode, errmsg.c_str()); - // reset expressionID - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + // reset expressionID + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - ci->expressionId = 0; + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + ci->expressionId = 0; } void setError(THD* thd, uint32_t errcode, string errmsg, gp_walk_info& gwi) { - setError(thd, errcode, errmsg); - clearStacks(gwi); + setError(thd, errcode, errmsg); + clearStacks(gwi); } -int setErrorAndReturn(gp_walk_info &gwi) +int setErrorAndReturn(gp_walk_info& gwi) { - // if this is dervied table process phase, mysql may have not developed the plan - // completely. Do not error and eventually mysql will call JOIN::exec() again. - // related to bug 2922. Need to find a way to skip calling rnd_init for derived table - // processing. - if (gwi.thd->derived_tables_processing) - { - gwi.cs_vtable_is_update_with_derive = true; - return -1; - } + // if this is dervied table process phase, mysql may have not developed the plan + // completely. Do not error and eventually mysql will call JOIN::exec() again. + // related to bug 2922. Need to find a way to skip calling rnd_init for derived table + // processing. + if (gwi.thd->derived_tables_processing) + { + gwi.cs_vtable_is_update_with_derive = true; + return -1; + } - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; } const string bestTableName(const Item_field* ifp) { - idbassert(ifp); + idbassert(ifp); - if (!ifp->table_name.str) - return ""; + if (!ifp->table_name.str) + return ""; - if (!ifp->field) - return ifp->table_name.str; + if (!ifp->field) + return ifp->table_name.str; - string table_name(ifp->table_name.str); - string field_table_table_name; + string table_name(ifp->table_name.str); + string field_table_table_name; - if (ifp->cached_table) - field_table_table_name = ifp->cached_table->table_name.str; - else if (ifp->field->table && ifp->field->table->s && ifp->field->table->s->table_name.str) - field_table_table_name = ifp->field->table->s->table_name.str; + if (ifp->cached_table) + field_table_table_name = ifp->cached_table->table_name.str; + else if (ifp->field->table && ifp->field->table->s && ifp->field->table->s->table_name.str) + field_table_table_name = ifp->field->table->s->table_name.str; - string tn; + string tn; - if (!field_table_table_name.empty()) - tn = field_table_table_name; - else - tn = table_name; + if (!field_table_table_name.empty()) + tn = field_table_table_name; + else + tn = table_name; - return tn; + return tn; } uint32_t setAggOp(AggregateColumn* ac, Item_sum* isp) { - Item_sum::Sumfunctype agg_type = isp->sum_func(); - uint32_t rc = 0; + Item_sum::Sumfunctype agg_type = isp->sum_func(); + uint32_t rc = 0; - switch (agg_type) + switch (agg_type) + { + case Item_sum::COUNT_FUNC: ac->aggOp(AggregateColumn::COUNT); return rc; + + case Item_sum::SUM_FUNC: ac->aggOp(AggregateColumn::SUM); return rc; + + case Item_sum::AVG_FUNC: ac->aggOp(AggregateColumn::AVG); return rc; + + case Item_sum::MIN_FUNC: ac->aggOp(AggregateColumn::MIN); return rc; + + case Item_sum::MAX_FUNC: ac->aggOp(AggregateColumn::MAX); return rc; + + case Item_sum::COUNT_DISTINCT_FUNC: + ac->aggOp(AggregateColumn::DISTINCT_COUNT); + ac->distinct(true); + return rc; + + case Item_sum::SUM_DISTINCT_FUNC: + ac->aggOp(AggregateColumn::DISTINCT_SUM); + ac->distinct(true); + return rc; + + case Item_sum::AVG_DISTINCT_FUNC: + ac->aggOp(AggregateColumn::DISTINCT_AVG); + ac->distinct(true); + return rc; + + case Item_sum::STD_FUNC: { - case Item_sum::COUNT_FUNC: - ac->aggOp(AggregateColumn::COUNT); - return rc; + Item_sum_variance* var = (Item_sum_variance*)isp; - case Item_sum::SUM_FUNC: - ac->aggOp(AggregateColumn::SUM); - return rc; + if (var->sample) + ac->aggOp(AggregateColumn::STDDEV_SAMP); + else + ac->aggOp(AggregateColumn::STDDEV_POP); - case Item_sum::AVG_FUNC: - ac->aggOp(AggregateColumn::AVG); - return rc; - - case Item_sum::MIN_FUNC: - ac->aggOp(AggregateColumn::MIN); - return rc; - - case Item_sum::MAX_FUNC: - ac->aggOp(AggregateColumn::MAX); - return rc; - - case Item_sum::COUNT_DISTINCT_FUNC: - ac->aggOp(AggregateColumn::DISTINCT_COUNT); - ac->distinct(true); - return rc; - - case Item_sum::SUM_DISTINCT_FUNC: - ac->aggOp(AggregateColumn::DISTINCT_SUM); - ac->distinct(true); - return rc; - - case Item_sum::AVG_DISTINCT_FUNC: - ac->aggOp(AggregateColumn::DISTINCT_AVG); - ac->distinct(true); - return rc; - - case Item_sum::STD_FUNC: - { - Item_sum_variance* var = (Item_sum_variance*)isp; - - if (var->sample) - ac->aggOp(AggregateColumn::STDDEV_SAMP); - else - ac->aggOp(AggregateColumn::STDDEV_POP); - - return rc; - } - - case Item_sum::VARIANCE_FUNC: - { - Item_sum_variance* var = (Item_sum_variance*)isp; - - if (var->sample) - ac->aggOp(AggregateColumn::VAR_SAMP); - else - ac->aggOp(AggregateColumn::VAR_POP); - - return rc; - } - - case Item_sum::GROUP_CONCAT_FUNC: - { - Item_func_group_concat* gc = (Item_func_group_concat*)isp; - ac->aggOp(AggregateColumn::GROUP_CONCAT); - ac->distinct(gc->get_distinct()); - return rc; - } - - case Item_sum::SUM_BIT_FUNC: - { - string funcName = isp->func_name(); - - if (funcName.compare("bit_and(") == 0) - ac->aggOp(AggregateColumn::BIT_AND); - else if (funcName.compare("bit_or(") == 0) - ac->aggOp(AggregateColumn::BIT_OR); - else if (funcName.compare("bit_xor(") == 0) - ac->aggOp(AggregateColumn::BIT_XOR); - else - return ER_CHECK_NOT_IMPLEMENTED; - - return rc; - } - - case Item_sum::UDF_SUM_FUNC: - ac->aggOp(AggregateColumn::UDAF); - return rc; - - default: - return ER_CHECK_NOT_IMPLEMENTED; + return rc; } + + case Item_sum::VARIANCE_FUNC: + { + Item_sum_variance* var = (Item_sum_variance*)isp; + + if (var->sample) + ac->aggOp(AggregateColumn::VAR_SAMP); + else + ac->aggOp(AggregateColumn::VAR_POP); + + return rc; + } + + case Item_sum::GROUP_CONCAT_FUNC: + { + Item_func_group_concat* gc = (Item_func_group_concat*)isp; + ac->aggOp(AggregateColumn::GROUP_CONCAT); + ac->distinct(gc->get_distinct()); + return rc; + } + + case Item_sum::SUM_BIT_FUNC: + { + string funcName = isp->func_name(); + + if (funcName.compare("bit_and(") == 0) + ac->aggOp(AggregateColumn::BIT_AND); + else if (funcName.compare("bit_or(") == 0) + ac->aggOp(AggregateColumn::BIT_OR); + else if (funcName.compare("bit_xor(") == 0) + ac->aggOp(AggregateColumn::BIT_XOR); + else + return ER_CHECK_NOT_IMPLEMENTED; + + return rc; + } + + case Item_sum::UDF_SUM_FUNC: ac->aggOp(AggregateColumn::UDAF); return rc; + + default: return ER_CHECK_NOT_IMPLEMENTED; + } } /* get the smallest column of a table. Used for filling columnmap */ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, CalpontSystemCatalog::TableName& tn, - CalpontSystemCatalog::TableAliasName& tan, - TABLE* table, - gp_walk_info& gwi) + CalpontSystemCatalog::TableAliasName& tan, TABLE* table, gp_walk_info& gwi) { - if (lower_case_table_names) + if (lower_case_table_names) + { + boost::algorithm::to_lower(tn.schema); + boost::algorithm::to_lower(tn.table); + boost::algorithm::to_lower(tan.schema); + boost::algorithm::to_lower(tan.table); + boost::algorithm::to_lower(tan.alias); + boost::algorithm::to_lower(tan.view); + } + // derived table + if (tan.schema.empty()) + { + for (uint32_t i = 0; i < gwi.derivedTbList.size(); i++) { - boost::algorithm::to_lower(tn.schema); - boost::algorithm::to_lower(tn.table); - boost::algorithm::to_lower(tan.schema); - boost::algorithm::to_lower(tan.table); - boost::algorithm::to_lower(tan.alias); - boost::algorithm::to_lower(tan.view); - } - // derived table - if (tan.schema.empty()) - { - for (uint32_t i = 0; i < gwi.derivedTbList.size(); i++) + CalpontSelectExecutionPlan* csep = + dynamic_cast(gwi.derivedTbList[i].get()); + + if (tan.alias == csep->derivedTbAlias()) + { + const CalpontSelectExecutionPlan::ReturnedColumnList& cols = csep->returnedCols(); + + CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator iter; + + ReturnedColumn* rc; + + for (iter = cols.begin(); iter != cols.end(); iter++) { - CalpontSelectExecutionPlan* csep = dynamic_cast(gwi.derivedTbList[i].get()); - - if (tan.alias == csep->derivedTbAlias()) - { - const CalpontSelectExecutionPlan::ReturnedColumnList& cols = csep->returnedCols(); - - CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator iter; - - ReturnedColumn* rc; - - for (iter = cols.begin(); iter != cols.end(); iter++) - { - if ((*iter)->refCount() != 0) - { - rc = dynamic_cast(iter->get()); - break; - } - } - - if (iter == cols.end()) - { - assert (!cols.empty()); - - // We take cols[0] here due to the optimization happening in - // derivedTableOptimization. All cols with refCount 0 from - // the end of the cols list are optimized out, until the - // first column with non-zero refCount is encountered. So - // here, if instead of cols[0], we take cols[1] (based on - // some logic) and increment it's refCount, then cols[0] is - // not optimized out in derivedTableOptimization and is - // added as a ConstantColumn to the derived table's returned - // column list. This later causes an ineffective row group - // with row of the form (1, cols[1]_value1) to be created in ExeMgr. - rc = dynamic_cast(cols[0].get()); - - // @bug5634 derived table optimization. - rc->incRefCount(); - } - - SimpleColumn* sc = new SimpleColumn(); - sc->columnName(rc->alias()); - sc->sequence(0); - sc->tableAlias(tan.alias); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - sc->derivedTable(csep->derivedTbAlias()); - sc->derivedRefCol(rc); - return sc; - } + if ((*iter)->refCount() != 0) + { + rc = dynamic_cast(iter->get()); + break; + } } - throw runtime_error ("getSmallestColumn: Internal error."); - } + if (iter == cols.end()) + { + assert(!cols.empty()); - // check engine type - if (!tan.fisColumnStore) - { - // get the first column to project. @todo optimization to get the smallest one for foreign engine. - Field* field = *(table->field); - SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name.str, tan.fisColumnStore, gwi.sessionid, lower_case_table_names); - sc->tableAlias(table->alias.ptr(), lower_case_table_names); - sc->isColumnStore(false); + // We take cols[0] here due to the optimization happening in + // derivedTableOptimization. All cols with refCount 0 from + // the end of the cols list are optimized out, until the + // first column with non-zero refCount is encountered. So + // here, if instead of cols[0], we take cols[1] (based on + // some logic) and increment it's refCount, then cols[0] is + // not optimized out in derivedTableOptimization and is + // added as a ConstantColumn to the derived table's returned + // column list. This later causes an ineffective row group + // with row of the form (1, cols[1]_value1) to be created in ExeMgr. + rc = dynamic_cast(cols[0].get()); + + // @bug5634 derived table optimization. + rc->incRefCount(); + } + + SimpleColumn* sc = new SimpleColumn(); + sc->columnName(rc->alias()); + sc->sequence(0); + sc->tableAlias(tan.alias); sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - sc->resultType(fieldType_MysqlToIDB(field)); - sc->oid(field->field_index + 1); + sc->derivedTable(csep->derivedTbAlias()); + sc->derivedRefCol(rc); return sc; + } } - CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(tn, true); - idbassert(oidlist.size() == table->s->fields); - CalpontSystemCatalog::TableColName tcn; - int minColWidth = -1; - int minWidthColOffset = 0; + throw runtime_error("getSmallestColumn: Internal error."); + } - for (unsigned int j = 0; j < oidlist.size(); j++) - { - CalpontSystemCatalog::ColType ct = csc->colType(oidlist[j].objnum); - - if (ct.colDataType == CalpontSystemCatalog::VARBINARY) - continue; - - if (minColWidth == -1 || ct.colWidth < minColWidth) - { - minColWidth = ct.colWidth; - minWidthColOffset = j; - } - } - - tcn = csc->colName(oidlist[minWidthColOffset].objnum); - SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); - sc->tableAlias(tan.alias); - sc->viewName(tan.view); + // check engine type + if (!tan.fisColumnStore) + { + // get the first column to project. @todo optimization to get the smallest one for foreign engine. + Field* field = *(table->field); + SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name.str, + tan.fisColumnStore, gwi.sessionid, lower_case_table_names); + sc->tableAlias(table->alias.ptr(), lower_case_table_names); + sc->isColumnStore(false); sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - sc->resultType(csc->colType(oidlist[minWidthColOffset].objnum)); - sc->charsetNumber(table->field[minWidthColOffset]->charset()->number); + sc->resultType(fieldType_MysqlToIDB(field)); + sc->oid(field->field_index + 1); return sc; + } + + CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(tn, true); + idbassert(oidlist.size() == table->s->fields); + CalpontSystemCatalog::TableColName tcn; + int minColWidth = -1; + int minWidthColOffset = 0; + + for (unsigned int j = 0; j < oidlist.size(); j++) + { + CalpontSystemCatalog::ColType ct = csc->colType(oidlist[j].objnum); + + if (ct.colDataType == CalpontSystemCatalog::VARBINARY) + continue; + + if (minColWidth == -1 || ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minWidthColOffset = j; + } + } + + tcn = csc->colName(oidlist[minWidthColOffset].objnum); + SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); + sc->tableAlias(tan.alias); + sc->viewName(tan.view); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->resultType(csc->colType(oidlist[minWidthColOffset].objnum)); + sc->charsetNumber(table->field[minWidthColOffset]->charset()->number); + return sc; } -CalpontSystemCatalog::ColType fieldType_MysqlToIDB (const Field* field) +CalpontSystemCatalog::ColType fieldType_MysqlToIDB(const Field* field) { - CalpontSystemCatalog::ColType ct; - ct.precision = 4; + CalpontSystemCatalog::ColType ct; + ct.precision = 4; - switch (field->result_type()) + switch (field->result_type()) + { + case INT_RESULT: + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + break; + + case STRING_RESULT: + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = field->field_length; + break; + + case DECIMAL_RESULT: { - case INT_RESULT: - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - break; + Field_decimal* idp = (Field_decimal*)field; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + ct.colWidth = 8; + ct.scale = idp->dec; - case STRING_RESULT: - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = field->field_length; - break; + if (ct.scale == 0) + ct.precision = idp->field_length - 1; + else + ct.precision = idp->field_length - idp->dec; - case DECIMAL_RESULT: + break; + } + + case REAL_RESULT: + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.colWidth = 8; + break; + + default: + IDEBUG(cerr << "fieldType_MysqlToIDB:: Unknown result type of MySQL " << field->result_type() << endl); + break; + } + + return ct; +} + +CalpontSystemCatalog::ColType colType_MysqlToIDB(const Item* item) +{ + CalpontSystemCatalog::ColType ct; + ct.precision = 4; + + switch (item->result_type()) + { + case INT_RESULT: + if (item->unsigned_flag) + { + ct.colDataType = CalpontSystemCatalog::UBIGINT; + } + else + { + ct.colDataType = CalpontSystemCatalog::BIGINT; + } + + ct.colWidth = 8; + break; + + case STRING_RESULT: + ct.colDataType = CalpontSystemCatalog::VARCHAR; + + // MCOL-4758 the longest TEXT we deal with is 16777215 so + // limit to that. + if (item->max_length < 16777215) + ct.colWidth = item->max_length; + else + ct.colWidth = 16777215; + + // force token + if (item->type() == Item::FUNC_ITEM) + { + if (ct.colWidth < 20) + ct.colWidth = 20; // for infinidb date length + } + + // @bug5083. MySQL gives string type for date/datetime column. + // need to adjust here. + if (item->type() == Item::FIELD_ITEM) + { + if (item->field_type() == MYSQL_TYPE_DATE) { - Field_decimal* idp = (Field_decimal*)field; - ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = 8; - ct.scale = idp->dec; - - if (ct.scale == 0) - ct.precision = idp->field_length - 1; - else - ct.precision = idp->field_length - idp->dec; - - break; + ct.colDataType = CalpontSystemCatalog::DATE; + ct.colWidth = 4; + } + else if (item->field_type() == MYSQL_TYPE_DATETIME || item->field_type() == MYSQL_TYPE_DATETIME2) + { + ct.colDataType = CalpontSystemCatalog::DATETIME; + ct.colWidth = 8; + } + else if (item->field_type() == MYSQL_TYPE_TIMESTAMP || item->field_type() == MYSQL_TYPE_TIMESTAMP2) + { + ct.colDataType = CalpontSystemCatalog::TIMESTAMP; + ct.colWidth = 8; + } + else if (item->field_type() == MYSQL_TYPE_TIME) + { + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; } - case REAL_RESULT: - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.colWidth = 8; - break; - - default: - IDEBUG( cerr << "fieldType_MysqlToIDB:: Unknown result type of MySQL " - << field->result_type() << endl ); - break; - } - - return ct; -} - -CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) -{ - CalpontSystemCatalog::ColType ct; - ct.precision = 4; - - switch (item->result_type()) - { - case INT_RESULT: - if (item->unsigned_flag) - { - ct.colDataType = CalpontSystemCatalog::UBIGINT; - } - else - { - ct.colDataType = CalpontSystemCatalog::BIGINT; - } - - ct.colWidth = 8; - break; - - case STRING_RESULT: - ct.colDataType = CalpontSystemCatalog::VARCHAR; - - // MCOL-4758 the longest TEXT we deal with is 16777215 so - // limit to that. - if (item->max_length < 16777215) - ct.colWidth = item->max_length; - else - ct.colWidth = 16777215; - - // force token - if (item->type() == Item::FUNC_ITEM) - { - if (ct.colWidth < 20) - ct.colWidth = 20; // for infinidb date length - } - - // @bug5083. MySQL gives string type for date/datetime column. - // need to adjust here. - if (item->type() == Item::FIELD_ITEM) - { - if (item->field_type() == MYSQL_TYPE_DATE) - { - ct.colDataType = CalpontSystemCatalog::DATE; - ct.colWidth = 4; - } - else if (item->field_type() == MYSQL_TYPE_DATETIME || - item->field_type() == MYSQL_TYPE_DATETIME2 - ) - { - ct.colDataType = CalpontSystemCatalog::DATETIME; - ct.colWidth = 8; - } - else if (item->field_type() == MYSQL_TYPE_TIMESTAMP || - item->field_type() == MYSQL_TYPE_TIMESTAMP2 - ) - { - ct.colDataType = CalpontSystemCatalog::TIMESTAMP; - ct.colWidth = 8; - } - else if (item->field_type() == MYSQL_TYPE_TIME) - { - ct.colDataType = CalpontSystemCatalog::TIME; - ct.colWidth = 8; - } - - if (item->field_type() == MYSQL_TYPE_BLOB) - { - ct.colDataType = CalpontSystemCatalog::BLOB; - } - } - - break; - - /* FIXME: - case xxxBINARY_RESULT: - ct.colDataType = CalpontSystemCatalog::VARBINARY; - ct.colWidth = item->max_length; - // force token - if (item->type() == Item::FUNC_ITEM) - { - if (ct.colWidth < 20) - ct.colWidth = 20; // for infinidb date length - } - break; - */ - case DECIMAL_RESULT: + if (item->field_type() == MYSQL_TYPE_BLOB) { - Item_decimal* idp = (Item_decimal*)item; - - ct.colDataType = CalpontSystemCatalog::DECIMAL; - - unsigned int precision = idp->decimal_precision(); - unsigned int scale = idp->decimal_scale(); - - ct.setDecimalScalePrecision(precision, scale); - - break; + ct.colDataType = CalpontSystemCatalog::BLOB; } + } - case REAL_RESULT: - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.colWidth = 8; - break; + break; - default: - IDEBUG( cerr << "colType_MysqlToIDB:: Unknown result type of MySQL " - << item->result_type() << endl ); - break; - } - ct.charsetNumber = item->collation.collation->number; - return ct; -} - - -ReturnedColumn * -buildReturnedColumnNull(gp_walk_info& gwi) -{ - if (gwi.condPush) - return new SimpleColumn("noop"); - ConstantColumn *rc = new ConstantColumnNull(); - if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - return rc; -} - - -class ValStrStdString: public string -{ - bool mIsNull; -public: - ValStrStdString(Item *item) + /* FIXME: + case xxxBINARY_RESULT: + ct.colDataType = CalpontSystemCatalog::VARBINARY; + ct.colWidth = item->max_length; + // force token + if (item->type() == Item::FUNC_ITEM) + { + if (ct.colWidth < 20) + ct.colWidth = 20; // for infinidb date length + } + break; + */ + case DECIMAL_RESULT: { - String val, *str = item->val_str(&val); - mIsNull = (str == nullptr); - DBUG_ASSERT(mIsNull == item->null_value); - if (!mIsNull) - assign(str->ptr(), str->length()); + Item_decimal* idp = (Item_decimal*)item; + + ct.colDataType = CalpontSystemCatalog::DECIMAL; + + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); + + ct.setDecimalScalePrecision(precision, scale); + + break; } - bool isNull() const { return mIsNull; } + + case REAL_RESULT: + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.colWidth = 8; + break; + + default: + IDEBUG(cerr << "colType_MysqlToIDB:: Unknown result type of MySQL " << item->result_type() << endl); + break; + } + ct.charsetNumber = item->collation.collation->number; + return ct; +} + +ReturnedColumn* buildReturnedColumnNull(gp_walk_info& gwi) +{ + if (gwi.condPush) + return new SimpleColumn("noop"); + ConstantColumn* rc = new ConstantColumnNull(); + if (rc) + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + return rc; +} + +class ValStrStdString : public string +{ + bool mIsNull; + + public: + ValStrStdString(Item* item) + { + String val, *str = item->val_str(&val); + mIsNull = (str == nullptr); + DBUG_ASSERT(mIsNull == item->null_value); + if (!mIsNull) + assign(str->ptr(), str->length()); + } + bool isNull() const + { + return mIsNull; + } }; - /* Create a ConstantColumn according to cmp_type(). But do not set the time zone yet. @@ -3296,65 +3249,59 @@ public: - N3 performs double evalation of the value, which may cause various negative effects (double side effects or double warnings). */ -static ConstantColumn * -newConstantColumnNotNullUsingValNativeNoTz(Item *item, gp_walk_info& gwi) +static ConstantColumn* newConstantColumnNotNullUsingValNativeNoTz(Item* item, gp_walk_info& gwi) { - DBUG_ASSERT(item->const_item()); + DBUG_ASSERT(item->const_item()); - switch (item->cmp_type()) + switch (item->cmp_type()) + { + case INT_RESULT: { - case INT_RESULT: - { - if (item->unsigned_flag) - return new ConstantColumnUInt((uint64_t) item->val_uint(), - (int8_t) item->decimal_scale(), - (uint8_t) item->decimal_precision()); - ValStrStdString str(item); - DBUG_ASSERT(!str.isNull()); - return new ConstantColumnSInt(colType_MysqlToIDB(item), str, - (int64_t) item->val_int()); - } - case STRING_RESULT: - { - // Special handling for 0xHHHH literals - if (item->type_handler() == &type_handler_hex_hybrid) - return new ConstantColumn((int64_t)item->val_int(), - ConstantColumn::NUM); - ValStrStdString str(item); - DBUG_ASSERT(!str.isNull()); - return new ConstantColumnString(str); - } - case REAL_RESULT: - { - ValStrStdString str(item); - DBUG_ASSERT(!str.isNull()); - return new ConstantColumnReal(colType_MysqlToIDB(item), - str, item->val_real()); - } - case DECIMAL_RESULT: - { - ValStrStdString str(item); - DBUG_ASSERT(!str.isNull()); - return buildDecimalColumn(item, str, gwi); - } - case TIME_RESULT: - { - ValStrStdString str(item); - DBUG_ASSERT(!str.isNull()); - return new ConstantColumnTemporal(colType_MysqlToIDB(item), str); - } - default: - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Unknown item type"; - break; - } + if (item->unsigned_flag) + return new ConstantColumnUInt((uint64_t)item->val_uint(), (int8_t)item->decimal_scale(), + (uint8_t)item->decimal_precision()); + ValStrStdString str(item); + DBUG_ASSERT(!str.isNull()); + return new ConstantColumnSInt(colType_MysqlToIDB(item), str, (int64_t)item->val_int()); } + case STRING_RESULT: + { + // Special handling for 0xHHHH literals + if (item->type_handler() == &type_handler_hex_hybrid) + return new ConstantColumn((int64_t)item->val_int(), ConstantColumn::NUM); + ValStrStdString str(item); + DBUG_ASSERT(!str.isNull()); + return new ConstantColumnString(str); + } + case REAL_RESULT: + { + ValStrStdString str(item); + DBUG_ASSERT(!str.isNull()); + return new ConstantColumnReal(colType_MysqlToIDB(item), str, item->val_real()); + } + case DECIMAL_RESULT: + { + ValStrStdString str(item); + DBUG_ASSERT(!str.isNull()); + return buildDecimalColumn(item, str, gwi); + } + case TIME_RESULT: + { + ValStrStdString str(item); + DBUG_ASSERT(!str.isNull()); + return new ConstantColumnTemporal(colType_MysqlToIDB(item), str); + } + default: + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Unknown item type"; + break; + } + } - return nullptr; + return nullptr; } - /* Create a ConstantColumn according to cmp_type(). But do not set the time zone yet. @@ -3371,1023 +3318,994 @@ newConstantColumnNotNullUsingValNativeNoTz(Item *item, gp_walk_info& gwi) to avoid possible negative effects mentioned in the comments to newConstantColumnNotNullUsingValNativeNoTz(). */ -static ConstantColumn * -newConstantColumnMaybeNullFromValStrNoTz(const Item *item, - const ValStrStdString &valStr, - gp_walk_info& gwi) +static ConstantColumn* newConstantColumnMaybeNullFromValStrNoTz(const Item* item, + const ValStrStdString& valStr, + gp_walk_info& gwi) { - if (valStr.isNull()) - return new ConstantColumnNull(); + if (valStr.isNull()) + return new ConstantColumnNull(); - switch (item->result_type()) - { - case STRING_RESULT: - return new ConstantColumnString(valStr); - case DECIMAL_RESULT: - return buildDecimalColumn(item, valStr, gwi); - case TIME_RESULT: - case INT_RESULT: - case REAL_RESULT: - case ROW_RESULT: - return new ConstantColumnNum(colType_MysqlToIDB(item), valStr); - } - return nullptr; + switch (item->result_type()) + { + case STRING_RESULT: return new ConstantColumnString(valStr); + case DECIMAL_RESULT: return buildDecimalColumn(item, valStr, gwi); + case TIME_RESULT: + case INT_RESULT: + case REAL_RESULT: + case ROW_RESULT: return new ConstantColumnNum(colType_MysqlToIDB(item), valStr); + } + return nullptr; } - // Create a ConstantColumn from a previously evaluated val_str() result, // Supports both NULL and NOT NULL values. // Sets the time zone according to gwi. -static ConstantColumn * -buildConstantColumnMaybeNullFromValStr(const Item *item, - const ValStrStdString &valStr, - gp_walk_info& gwi) +static ConstantColumn* buildConstantColumnMaybeNullFromValStr(const Item* item, const ValStrStdString& valStr, + gp_walk_info& gwi) { - ConstantColumn *rc= newConstantColumnMaybeNullFromValStrNoTz(item, valStr, gwi); - if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - return rc; + ConstantColumn* rc = newConstantColumnMaybeNullFromValStrNoTz(item, valStr, gwi); + if (rc) + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + return rc; } - // Create a ConstantColumn by calling val_str(). // Supports both NULL and NOT NULL values. // Sets the time zone according to gwi. -static ConstantColumn * -buildConstantColumnMaybeNullUsingValStr(Item *item, gp_walk_info& gwi) +static ConstantColumn* buildConstantColumnMaybeNullUsingValStr(Item* item, gp_walk_info& gwi) { - return buildConstantColumnMaybeNullFromValStr(item, ValStrStdString(item), gwi); + return buildConstantColumnMaybeNullFromValStr(item, ValStrStdString(item), gwi); } - // Create a ConstantColumn for a NOT NULL expression. // Sets the time zone according to gwi. -static ConstantColumn * -buildConstantColumnNotNullUsingValNative(Item *item, gp_walk_info& gwi) +static ConstantColumn* buildConstantColumnNotNullUsingValNative(Item* item, gp_walk_info& gwi) { - ConstantColumn *rc = newConstantColumnNotNullUsingValNativeNoTz(item, gwi); - if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - return rc; + ConstantColumn* rc = newConstantColumnNotNullUsingValNativeNoTz(item, gwi); + if (rc) + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + return rc; } - -ReturnedColumn* buildReturnedColumn( - Item* item, gp_walk_info& gwi, - bool& nonSupport, - bool isRefItem) +ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem) { - ReturnedColumn* rc = NULL; + ReturnedColumn* rc = NULL; - if (gwi.thd) + if (gwi.thd) + { + // if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == + // SQLCOM_UPDATE_MULTI )) { - //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) - { - if ( !item->fixed()) - { - item->fix_fields(gwi.thd, (Item**)&item); - } - } + if (!item->fixed()) + { + item->fix_fields(gwi.thd, (Item**)&item); + } + } + } + + switch (item->type()) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + + if (isRefItem && gwi.isGroupByHandler && !gwi.extSelAggColsItems.empty()) + { + return buildAggFrmTempField(ifp, gwi); + } + + return buildSimpleColumn(ifp, gwi); + } + case Item::NULL_ITEM: return buildReturnedColumnNull(gwi); + case Item::CONST_ITEM: + { + rc = buildConstantColumnNotNullUsingValNative(item, gwi); + break; + } + case Item::FUNC_ITEM: + { + if (item->const_item()) + { + rc = buildConstantColumnMaybeNullUsingValStr(item, gwi); + break; + } + Item_func* ifp = (Item_func*)item; + string func_name = ifp->func_name(); + + // try to evaluate const F&E. only for select clause + vector tmpVec; + // bool hasAggColumn = false; + uint16_t parseInfo = 0; + parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo, &gwi); + + if (parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + break; + } + + if (!gwi.fatalParseError && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && (tmpVec.size() == 0)) + { + rc = buildConstantColumnMaybeNullUsingValStr(item, gwi); + break; + } + + if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/") + return buildArithmeticColumn(ifp, gwi, nonSupport); + else + return buildFunctionColumn(ifp, gwi, nonSupport); } - switch (item->type()) + case Item::SUM_FUNC_ITEM: { - case Item::FIELD_ITEM: + return buildAggregateColumn(item, gwi); + } + + case Item::REF_ITEM: + { + Item_ref* ref = (Item_ref*)item; + + switch ((*(ref->ref))->type()) + { + case Item::SUM_FUNC_ITEM: return buildAggregateColumn(*(ref->ref), gwi); + + case Item::FIELD_ITEM: return buildReturnedColumn(*(ref->ref), gwi, nonSupport); + + case Item::REF_ITEM: return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); + + case Item::FUNC_ITEM: return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); + + case Item::WINDOW_FUNC_ITEM: return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + + case Item::SUBSELECT_ITEM: + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + break; + + default: + gwi.fatalParseError = true; + gwi.parseErrorText = "Unknown REF item"; + break; + } + return buildReturnedColumnNull(gwi); + } + case Item::CACHE_ITEM: + { + Item* col = ((Item_cache*)item)->get_example(); + rc = buildReturnedColumn(col, gwi, nonSupport); + + if (rc) + { + ConstantColumn* cc = dynamic_cast(rc); + + if (!cc) { - Item_field* ifp = (Item_field*)item; + rc->joinInfo(rc->joinInfo() | JOIN_CORRELATED); - if (isRefItem && gwi.isGroupByHandler && !gwi.extSelAggColsItems.empty()) - { - return buildAggFrmTempField(ifp, gwi); - } - - return buildSimpleColumn(ifp, gwi); + if (gwi.subQuery) + gwi.subQuery->correlated(true); } - case Item::NULL_ITEM: - return buildReturnedColumnNull(gwi); - case Item::CONST_ITEM: - { - rc = buildConstantColumnNotNullUsingValNative(item, gwi); - break; - } - case Item::FUNC_ITEM: - { - if (item->const_item()) - { - rc = buildConstantColumnMaybeNullUsingValStr(item, gwi); - break; - } - Item_func* ifp = (Item_func*)item; - string func_name = ifp->func_name(); + } - // try to evaluate const F&E. only for select clause - vector tmpVec; - //bool hasAggColumn = false; - uint16_t parseInfo = 0; - parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo, &gwi); + break; + } - if (parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - break; - } + case Item::EXPR_CACHE_ITEM: + { + // TODO: item is a Item_cache_wrapper + printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); + cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; + break; + } - if (!gwi.fatalParseError && - !nonConstFunc(ifp) && - !(parseInfo & AF_BIT) && - (tmpVec.size() == 0)) - { - rc = buildConstantColumnMaybeNullUsingValStr(item, gwi); - break; - } - - if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/" ) - return buildArithmeticColumn(ifp, gwi, nonSupport); - else - return buildFunctionColumn(ifp, gwi, nonSupport); - } - - case Item::SUM_FUNC_ITEM: - { - return buildAggregateColumn(item, gwi); - } - - case Item::REF_ITEM: - { - Item_ref* ref = (Item_ref*)item; - - switch ((*(ref->ref))->type()) - { - case Item::SUM_FUNC_ITEM: - return buildAggregateColumn(*(ref->ref), gwi); - - case Item::FIELD_ITEM: - return buildReturnedColumn(*(ref->ref), gwi, nonSupport); - - case Item::REF_ITEM: - return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); - - case Item::FUNC_ITEM: - return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); - - case Item::WINDOW_FUNC_ITEM: - return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); - - case Item::SUBSELECT_ITEM: - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - break; - - default: - gwi.fatalParseError = true; - gwi.parseErrorText = "Unknown REF item"; - break; - } - return buildReturnedColumnNull(gwi); - } - case Item::CACHE_ITEM: - { - Item* col = ((Item_cache*)item)->get_example(); - rc = buildReturnedColumn(col, gwi, nonSupport); - - if (rc) - { - ConstantColumn* cc = dynamic_cast(rc); - - if (!cc) - { - rc->joinInfo(rc->joinInfo() | JOIN_CORRELATED); - - if (gwi.subQuery) - gwi.subQuery->correlated(true); - } - } - - break; - } - - case Item::EXPR_CACHE_ITEM: - { - // TODO: item is a Item_cache_wrapper - printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); - cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; - break; - } - - case Item::WINDOW_FUNC_ITEM: - { - return buildWindowFunctionColumn(item, gwi, nonSupport); - } + case Item::WINDOW_FUNC_ITEM: + { + return buildWindowFunctionColumn(item, gwi, nonSupport); + } #if INTERVAL_ITEM - case Item::INTERVAL_ITEM: - { - Item_interval* interval = (Item_interval*)item; - SRCP srcp; - srcp.reset(buildReturnedColumn(interval->item, gwi, nonSupport)); + case Item::INTERVAL_ITEM: + { + Item_interval* interval = (Item_interval*)item; + SRCP srcp; + srcp.reset(buildReturnedColumn(interval->item, gwi, nonSupport)); - if (!srcp) - return NULL; + if (!srcp) + return NULL; - rc = new IntervalColumn(srcp, (int)interval->interval); - rc->resultType(srcp->resultType()); - break; - } + rc = new IntervalColumn(srcp, (int)interval->interval); + rc->resultType(srcp->resultType()); + break; + } #endif - case Item::SUBSELECT_ITEM: - { - gwi.hasSubSelect = true; - break; - } - - case Item::COND_ITEM: - { - // MCOL-1196: Allow COND_ITEM thru. They will be picked up - // by further logic. It may become desirable to add code here. - break; - } - - default: - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Unknown item type"; - break; - } + case Item::SUBSELECT_ITEM: + { + gwi.hasSubSelect = true; + break; } - if (rc && item->name.length) - rc->alias(item->name.str); + case Item::COND_ITEM: + { + // MCOL-1196: Allow COND_ITEM thru. They will be picked up + // by further logic. It may become desirable to add code here. + break; + } - if (rc) - rc->charsetNumber(item->collation.collation->number); + default: + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Unknown item type"; + break; + } + } - return rc; + if (rc && item->name.length) + rc->alias(item->name.str); + + if (rc) + rc->charsetNumber(item->collation.collation->number); + + return rc; } -ArithmeticColumn* buildArithmeticColumn( - Item_func* item, - gp_walk_info& gwi, - bool& nonSupport) +ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - ArithmeticColumn* ac = new ArithmeticColumn(); - Item** sfitempp = item->arguments(); - ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); - aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - aop->setOverflowCheck(get_decimal_overflow_check(gwi.thd)); - ParseTree* pt = new ParseTree(aop); - //ReturnedColumn *lhs = 0, *rhs = 0; - ParseTree* lhs = 0, *rhs = 0; - SRCP srcp; + ArithmeticColumn* ac = new ArithmeticColumn(); + Item** sfitempp = item->arguments(); + ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); + aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->setOverflowCheck(get_decimal_overflow_check(gwi.thd)); + ParseTree* pt = new ParseTree(aop); + // ReturnedColumn *lhs = 0, *rhs = 0; + ParseTree *lhs = 0, *rhs = 0; + SRCP srcp; - if (item->name.length) - ac->alias(item->name.str); + if (item->name.length) + ac->alias(item->name.str); - // argument_count() should generally be 2, except negate expression - if (item->argument_count() == 2) + // argument_count() should generally be 2, except negate expression + if (item->argument_count() == 2) + { + if (gwi.clauseType == SELECT || /*gwi.clauseType == HAVING || */ gwi.clauseType == GROUP_BY || + gwi.clauseType == FROM) // select list { - if (gwi.clauseType == SELECT || /*gwi.clauseType == HAVING || */gwi.clauseType == GROUP_BY || gwi.clauseType == FROM) // select list - { - lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); + lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); - if (!lhs->data() && (sfitempp[0]->type() == Item::FUNC_ITEM)) - { - delete lhs; - Item_func* ifp = (Item_func*)sfitempp[0]; - lhs = buildParseTree(ifp, gwi, nonSupport); - } - else if(!lhs->data() && (sfitempp[0]->type() == Item::REF_ITEM)) - { - // There must be an aggregation column in extended SELECT - // list so find the corresponding column. - // Could have it set if there are aggregation funcs as this function arguments. - gwi.fatalParseError = false; + if (!lhs->data() && (sfitempp[0]->type() == Item::FUNC_ITEM)) + { + delete lhs; + Item_func* ifp = (Item_func*)sfitempp[0]; + lhs = buildParseTree(ifp, gwi, nonSupport); + } + else if (!lhs->data() && (sfitempp[0]->type() == Item::REF_ITEM)) + { + // There must be an aggregation column in extended SELECT + // list so find the corresponding column. + // Could have it set if there are aggregation funcs as this function arguments. + gwi.fatalParseError = false; - ReturnedColumn* rc = buildAggFrmTempField(sfitempp[0], gwi); - if(rc) - lhs = new ParseTree(rc); - } - rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); + ReturnedColumn* rc = buildAggFrmTempField(sfitempp[0], gwi); + if (rc) + lhs = new ParseTree(rc); + } + rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); - if (!rhs->data() && (sfitempp[1]->type() == Item::FUNC_ITEM)) - { - delete rhs; - Item_func* ifp = (Item_func*)sfitempp[1]; - rhs = buildParseTree(ifp, gwi, nonSupport); - } - else if(!rhs->data() && (sfitempp[1]->type() == Item::REF_ITEM)) - { - // There must be an aggregation column in extended SELECT - // list so find the corresponding column. - // Could have it set if there are aggregation funcs as this function arguments. - gwi.fatalParseError = false; + if (!rhs->data() && (sfitempp[1]->type() == Item::FUNC_ITEM)) + { + delete rhs; + Item_func* ifp = (Item_func*)sfitempp[1]; + rhs = buildParseTree(ifp, gwi, nonSupport); + } + else if (!rhs->data() && (sfitempp[1]->type() == Item::REF_ITEM)) + { + // There must be an aggregation column in extended SELECT + // list so find the corresponding column. + // Could have it set if there are aggregation funcs as this function arguments. + gwi.fatalParseError = false; - ReturnedColumn* rc = buildAggFrmTempField(sfitempp[1], gwi); - if(rc) - rhs = new ParseTree(rc); - } - } - else // where clause - { - if (isPredicateFunction(sfitempp[1], &gwi)) - { - if (gwi.ptWorkStack.empty()) - { - rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); - } - else - { - rhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - } - } - else - { - if (gwi.rcWorkStack.empty()) - { - rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); - } - else - { - rhs = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } - } - - if (isPredicateFunction(sfitempp[0], &gwi)) - { - if (gwi.ptWorkStack.empty()) - { - lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); - } - else - { - lhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - } - } - else - { - if (gwi.rcWorkStack.empty()) - { - lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); - } - else - { - lhs = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } - } - } - - if (nonSupport || !lhs->data() || !rhs->data()) - { - gwi.fatalParseError = true; - - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Un-recognized Arithmetic Operand"; - - delete lhs; - delete rhs; - return NULL; - } - - //aop->operationType(lhs->resultType(), rhs->resultType()); - pt->left(lhs); - pt->right(rhs); + ReturnedColumn* rc = buildAggFrmTempField(sfitempp[1], gwi); + if (rc) + rhs = new ParseTree(rc); + } } - else + else // where clause { - ConstantColumn* cc = new ConstantColumn(string("0"), (int64_t)0); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - - if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause + if (isPredicateFunction(sfitempp[1], &gwi)) + { + if (gwi.ptWorkStack.empty()) { - rhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); + rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); } else { - if (gwi.rcWorkStack.empty()) - { - rhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); - } - else - { - rhs = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); } - - if (nonSupport || !rhs->data()) + } + else + { + if (gwi.rcWorkStack.empty()) { - gwi.fatalParseError = true; - - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Un-recognized Arithmetic Operand"; - - delete rhs; - return NULL; + rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); } + else + { + rhs = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + } - pt->left(cc); - pt->right(rhs); + if (isPredicateFunction(sfitempp[0], &gwi)) + { + if (gwi.ptWorkStack.empty()) + { + lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); + } + else + { + lhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + } + } + else + { + if (gwi.rcWorkStack.empty()) + { + lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); + } + else + { + lhs = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + } } - // @bug5715. Use InfiniDB adjusted coltype for result type. - // decimal arithmetic operation gives double result when the session variable is set. - CalpontSystemCatalog::ColType mysqlType = colType_MysqlToIDB(item); - - const CalpontSystemCatalog::ColType& leftColType = pt->left()->data()->resultType(); - const CalpontSystemCatalog::ColType& rightColType = pt->right()->data()->resultType(); - - if (datatypes::isDecimal(leftColType.colDataType) || datatypes::isDecimal(rightColType.colDataType)) + if (nonSupport || !lhs->data() || !rhs->data()) { - int32_t leftColWidth = leftColType.colWidth; - int32_t rightColWidth = rightColType.colWidth; + gwi.fatalParseError = true; - if (leftColWidth == datatypes::MAXDECIMALWIDTH || - rightColWidth == datatypes::MAXDECIMALWIDTH) - { - mysqlType.colWidth = datatypes::MAXDECIMALWIDTH; + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Un-recognized Arithmetic Operand"; - string funcName = item->func_name(); - - int32_t scale1 = leftColType.scale; - int32_t scale2 = rightColType.scale; - - mysqlType.precision = datatypes::INT128MAXPRECISION; - - if (funcName == "/" && - (mysqlType.scale - (scale1 - scale2)) > datatypes::INT128MAXPRECISION) - { - Item_decimal* idp = (Item_decimal*)item; - - unsigned int precision = idp->decimal_precision(); - unsigned int scale = idp->decimal_scale(); - - mysqlType.setDecimalScalePrecisionHeuristic(precision, scale); - - if (mysqlType.scale < scale1) - mysqlType.scale = scale1; - - if (mysqlType.precision < mysqlType.scale) - mysqlType.precision = mysqlType.scale; - } - } + delete lhs; + delete rhs; + return NULL; } - if (get_double_for_decimal_math(current_thd) == true) - aop->adjustResultType(mysqlType); + // aop->operationType(lhs->resultType(), rhs->resultType()); + pt->left(lhs); + pt->right(rhs); + } + else + { + ConstantColumn* cc = new ConstantColumn(string("0"), (int64_t)0); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + + if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause + { + rhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); + } else - aop->resultType(mysqlType); - - // adjust decimal result type according to internalDecimalScale - if (gwi.internalDecimalScale >= 0 && aop->resultType().colDataType == CalpontSystemCatalog::DECIMAL) { - CalpontSystemCatalog::ColType ct = aop->resultType(); - ct.scale = gwi.internalDecimalScale; - aop->resultType(ct); + if (gwi.rcWorkStack.empty()) + { + rhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); + } + else + { + rhs = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } } - aop->operationType(aop->resultType()); - ac->expression(pt); - ac->resultType(aop->resultType()); - ac->operationType(aop->operationType()); - ac->expressionId(ci->expressionId++); - - // @3391. optimization. try to associate expression ID to the expression on the select list - if (gwi.clauseType != SELECT) + if (nonSupport || !rhs->data()) { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if ((!ac->alias().empty()) && strcasecmp(ac->alias().c_str(), gwi.returnedCols[i]->alias().c_str()) == 0) - { - ac->expressionId(gwi.returnedCols[i]->expressionId()); - break; - } - } + gwi.fatalParseError = true; + + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Un-recognized Arithmetic Operand"; + + delete rhs; + return NULL; } - // For function join. If any argument has non-zero joininfo, set it to the function. - ac->setSimpleColumnList(); - std::vector simpleColList = ac->simpleColumnList(); + pt->left(cc); + pt->right(rhs); + } - for (uint i = 0; i < simpleColList.size(); i++) + // @bug5715. Use InfiniDB adjusted coltype for result type. + // decimal arithmetic operation gives double result when the session variable is set. + CalpontSystemCatalog::ColType mysqlType = colType_MysqlToIDB(item); + + const CalpontSystemCatalog::ColType& leftColType = pt->left()->data()->resultType(); + const CalpontSystemCatalog::ColType& rightColType = pt->right()->data()->resultType(); + + if (datatypes::isDecimal(leftColType.colDataType) || datatypes::isDecimal(rightColType.colDataType)) + { + int32_t leftColWidth = leftColType.colWidth; + int32_t rightColWidth = rightColType.colWidth; + + if (leftColWidth == datatypes::MAXDECIMALWIDTH || rightColWidth == datatypes::MAXDECIMALWIDTH) { - if (simpleColList[i]->joinInfo() != 0) - { - ac->joinInfo(simpleColList[i]->joinInfo()); - break; - } - } + mysqlType.colWidth = datatypes::MAXDECIMALWIDTH; - return ac; + string funcName = item->func_name(); + + int32_t scale1 = leftColType.scale; + int32_t scale2 = rightColType.scale; + + mysqlType.precision = datatypes::INT128MAXPRECISION; + + if (funcName == "/" && (mysqlType.scale - (scale1 - scale2)) > datatypes::INT128MAXPRECISION) + { + Item_decimal* idp = (Item_decimal*)item; + + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); + + mysqlType.setDecimalScalePrecisionHeuristic(precision, scale); + + if (mysqlType.scale < scale1) + mysqlType.scale = scale1; + + if (mysqlType.precision < mysqlType.scale) + mysqlType.precision = mysqlType.scale; + } + } + } + + if (get_double_for_decimal_math(current_thd) == true) + aop->adjustResultType(mysqlType); + else + aop->resultType(mysqlType); + + // adjust decimal result type according to internalDecimalScale + if (gwi.internalDecimalScale >= 0 && aop->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + { + CalpontSystemCatalog::ColType ct = aop->resultType(); + ct.scale = gwi.internalDecimalScale; + aop->resultType(ct); + } + + aop->operationType(aop->resultType()); + ac->expression(pt); + ac->resultType(aop->resultType()); + ac->operationType(aop->operationType()); + ac->expressionId(ci->expressionId++); + + // @3391. optimization. try to associate expression ID to the expression on the select list + if (gwi.clauseType != SELECT) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if ((!ac->alias().empty()) && + strcasecmp(ac->alias().c_str(), gwi.returnedCols[i]->alias().c_str()) == 0) + { + ac->expressionId(gwi.returnedCols[i]->expressionId()); + break; + } + } + } + + // For function join. If any argument has non-zero joininfo, set it to the function. + ac->setSimpleColumnList(); + std::vector simpleColList = ac->simpleColumnList(); + + for (uint i = 0; i < simpleColList.size(); i++) + { + if (simpleColList[i]->joinInfo() != 0) + { + ac->joinInfo(simpleColList[i]->joinInfo()); + break; + } + } + + return ac; } -ReturnedColumn* buildFunctionColumn( - Item_func* ifp, - gp_walk_info& gwi, - bool& nonSupport, - bool selectBetweenIn) +ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn) { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - string funcName = ifp->func_name(); - FuncExp* funcExp = FuncExp::instance(); - Func* functor; - FunctionColumn* fc = NULL; + string funcName = ifp->func_name(); + FuncExp* funcExp = FuncExp::instance(); + Func* functor; + FunctionColumn* fc = NULL; - // Pseudocolumn - uint32_t pseudoType = PSEUDO_UNKNOWN; + // Pseudocolumn + uint32_t pseudoType = PSEUDO_UNKNOWN; - if (ifp->functype() == Item_func::UDF_FUNC) - pseudoType = PseudoColumn::pseudoNameToType(funcName); + if (ifp->functype() == Item_func::UDF_FUNC) + pseudoType = PseudoColumn::pseudoNameToType(funcName); - if (pseudoType != PSEUDO_UNKNOWN) + if (pseudoType != PSEUDO_UNKNOWN) + { + ReturnedColumn* rc = buildPseudoColumn(ifp, gwi, gwi.fatalParseError, pseudoType); + + if (!rc || gwi.fatalParseError) { - ReturnedColumn* rc = buildPseudoColumn(ifp, gwi, gwi.fatalParseError, pseudoType); + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported function."; - if (!rc || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported function."; - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return NULL; - } - - return rc; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return NULL; } - // Item_equal is supported by buildPredicateItem() - if (funcName == "multiple equal") + return rc; + } + + // Item_equal is supported by buildPredicateItem() + if (funcName == "multiple equal") + return NULL; + + // Arithmetic exp + if (funcName == "+" || funcName == "-" || funcName == "*" || funcName == "/") + { + ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport); + return ac; + } + + else if (funcName == "case") + { + fc = buildCaseFunction(ifp, gwi, nonSupport); + } + + else if ((funcName == "charset" || funcName == "collation") && ifp->argument_count() == 1 && + ifp->arguments()[0]->type() == Item::FIELD_ITEM) + { + Item_field* item = reinterpret_cast(ifp->arguments()[0]); + CHARSET_INFO* info = item->charset_for_protocol(); + ReturnedColumn* rc; + string val; + + if (funcName == "charset") + { + val = info->cs_name.str; + } + else // collation + { + val = info->coll_name.str; + } + + rc = new ConstantColumn(val, ConstantColumn::LITERAL); + + return rc; + } + + else if ((functor = funcExp->getFunctor(funcName))) + { + // where clause isnull still treated as predicate operator + // MCOL-686: between also a predicate operator so that extent elimination can happen + if ((funcName == "isnull" || funcName == "isnotnull" || funcName == "between") && + (gwi.clauseType == WHERE || gwi.clauseType == HAVING)) + return NULL; + + if (funcName == "in" || funcName == " IN " || funcName == "between") + { + // if F&E involved, build function. otherwise, fall to the old path. + // @todo need more checks here + if (ifp->arguments()[0]->type() == Item::ROW_ITEM) + { return NULL; + } - // Arithmetic exp - if (funcName == "+" || funcName == "-" || funcName == "*" || funcName == "/" ) - { - ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport); - return ac; - } + if (!selectBetweenIn && (ifp->arguments()[0]->type() == Item::FIELD_ITEM || + (ifp->arguments()[0]->type() == Item::REF_ITEM && + (*(((Item_ref*)ifp->arguments()[0])->ref))->type() == Item::FIELD_ITEM))) + { + bool fe = false; - else if (funcName == "case") - { - fc = buildCaseFunction(ifp, gwi, nonSupport); - } - - else if ((funcName == "charset" || funcName == "collation") && - ifp->argument_count() == 1 && - ifp->arguments()[0]->type() == Item::FIELD_ITEM) - { - Item_field *item = reinterpret_cast(ifp->arguments()[0]); - CHARSET_INFO* info = item->charset_for_protocol(); - ReturnedColumn* rc; - string val; - - if (funcName == "charset") + for (uint32_t i = 1; i < ifp->argument_count(); i++) { - val = info->cs_name.str; - } - else // collation - { - val = info->coll_name.str; + if (!(ifp->arguments()[i]->type() == Item::NULL_ITEM || + (ifp->arguments()[i]->type() == Item::CONST_ITEM && + (ifp->arguments()[i]->cmp_type() == INT_RESULT || + ifp->arguments()[i]->cmp_type() == STRING_RESULT || + ifp->arguments()[i]->cmp_type() == REAL_RESULT || + ifp->arguments()[i]->cmp_type() == DECIMAL_RESULT)))) + { + if (ifp->arguments()[i]->type() == Item::FUNC_ITEM) + { + // try to identify const F&E. fall to primitive if parms are constant F&E. + vector tmpVec; + uint16_t parseInfo = 0; + parse_item(ifp->arguments()[i], tmpVec, gwi.fatalParseError, parseInfo, &gwi); + + if (!gwi.fatalParseError && !(parseInfo & AF_BIT) && tmpVec.size() == 0) + continue; + } + + fe = true; + break; + } } - rc = new ConstantColumn(val, ConstantColumn::LITERAL); + if (!fe) + return NULL; + } - return rc; + Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; + + if (inp->negated) + funcName = "not" + funcName; } - else if ((functor = funcExp->getFunctor(funcName))) + // @todo non-support function as argument. need to do post process. Assume all support for now + fc = new FunctionColumn(); + fc->data(funcName); + FunctionParm funcParms; + SPTP sptp; + ClauseType clauseType = gwi.clauseType; + + if (gwi.clauseType == SELECT || + /*gwi.clauseType == HAVING || */ gwi.clauseType == GROUP_BY) // select clause { - // where clause isnull still treated as predicate operator - // MCOL-686: between also a predicate operator so that extent elimination can happen - if ((funcName == "isnull" || funcName == "isnotnull" || funcName == "between") && - (gwi.clauseType == WHERE || gwi.clauseType == HAVING)) + for (uint32_t i = 0; i < ifp->argument_count(); i++) + { + // group by clause try to see if the arguments are alias + if (gwi.clauseType == GROUP_BY && ifp->arguments()[i]->name.length) + { + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (string(ifp->arguments()[i]->name.str) == gwi.returnedCols[j]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[j]->clone(); + rc->orderPos(j); + sptp.reset(new ParseTree(rc)); + funcParms.push_back(sptp); + break; + } + } + + if (j != gwi.returnedCols.size()) + continue; + } + + // special handling for function that takes a filter arguments, like if(). + // @todo. merge this logic to buildParseTree(). + if ((funcName == "if" && i == 0) || funcName == "xor") + { + // make sure the rcWorkStack is cleaned. + gwi.clauseType = WHERE; + sptp.reset(buildParseTree((Item_func*)(ifp->arguments()[i]), gwi, nonSupport)); + gwi.clauseType = clauseType; + + if (!sptp) + { + nonSupport = true; return NULL; + } - if (funcName == "in" || funcName == " IN " || funcName == "between") - { - // if F&E involved, build function. otherwise, fall to the old path. - // @todo need more checks here - if (ifp->arguments()[0]->type() == Item::ROW_ITEM) - { - return NULL; - } - - if (!selectBetweenIn && (ifp->arguments()[0]->type() == Item::FIELD_ITEM || - (ifp->arguments()[0]->type() == Item::REF_ITEM && - (*(((Item_ref*)ifp->arguments()[0])->ref))->type() == Item::FIELD_ITEM))) - { - bool fe = false; - - for (uint32_t i = 1; i < ifp->argument_count(); i++) - { - if (!(ifp->arguments()[i]->type() == Item::NULL_ITEM || - (ifp->arguments()[i]->type() == Item::CONST_ITEM && - (ifp->arguments()[i]->cmp_type() == INT_RESULT || - ifp->arguments()[i]->cmp_type() == STRING_RESULT || - ifp->arguments()[i]->cmp_type() == REAL_RESULT || - ifp->arguments()[i]->cmp_type() == DECIMAL_RESULT - ) - ) - ) - ) - { - if (ifp->arguments()[i]->type() == Item::FUNC_ITEM) - { - // try to identify const F&E. fall to primitive if parms are constant F&E. - vector tmpVec; - uint16_t parseInfo = 0; - parse_item(ifp->arguments()[i], tmpVec, gwi.fatalParseError, parseInfo, &gwi); - - if (!gwi.fatalParseError && !(parseInfo & AF_BIT) && tmpVec.size() == 0) - continue; - } - - fe = true; - break; - } - } - - if (!fe) return NULL; - } - - Item_func_opt_neg* inp = (Item_func_opt_neg*)ifp; - - if (inp->negated) - funcName = "not" + funcName; + funcParms.push_back(sptp); + continue; } - // @todo non-support function as argument. need to do post process. Assume all support for now - fc = new FunctionColumn(); - fc->data(funcName); - FunctionParm funcParms; - SPTP sptp; - ClauseType clauseType = gwi.clauseType; - - if (gwi.clauseType == SELECT || /*gwi.clauseType == HAVING || */gwi.clauseType == GROUP_BY) // select clause + // @bug 3039 + // if (isPredicateFunction(ifp->arguments()[i], &gwi) || ifp->arguments()[i]->with_subquery()) + if (ifp->arguments()[i]->with_subquery()) { - for (uint32_t i = 0; i < ifp->argument_count(); i++) - { - // group by clause try to see if the arguments are alias - if (gwi.clauseType == GROUP_BY && ifp->arguments()[i]->name.length) - { - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (string (ifp->arguments()[i]->name.str) == gwi.returnedCols[j]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[j]->clone(); - rc->orderPos(j); - sptp.reset(new ParseTree(rc)); - funcParms.push_back(sptp); - break; - } - } - - if (j != gwi.returnedCols.size()) - continue; - } - - // special handling for function that takes a filter arguments, like if(). - // @todo. merge this logic to buildParseTree(). - if ((funcName == "if" && i == 0) || funcName == "xor") - { - // make sure the rcWorkStack is cleaned. - gwi.clauseType = WHERE; - sptp.reset(buildParseTree((Item_func*)(ifp->arguments()[i]), gwi, nonSupport)); - gwi.clauseType = clauseType; - - if (!sptp) - { - nonSupport = true; - return NULL; - } - - funcParms.push_back(sptp); - continue; - } - - // @bug 3039 - //if (isPredicateFunction(ifp->arguments()[i], &gwi) || ifp->arguments()[i]->with_subquery()) - if (ifp->arguments()[i]->with_subquery()) - { - nonSupport = true; - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SUB_EXPRESSION); - return NULL; - } - - ReturnedColumn* rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport); - - // MCOL-1510 It must be a temp table field, so find the corresponding column. - if (!rc && ifp->arguments()[i]->type() == Item::REF_ITEM) - { - gwi.fatalParseError = false; - rc = buildAggFrmTempField(ifp->arguments()[i], gwi); - } - - if (!rc || nonSupport) - { - nonSupport = true; - return NULL; - } - - sptp.reset(new ParseTree(rc)); - funcParms.push_back(sptp); - } - } - else // where clause - { - stack tmpPtStack; - - for (int32_t i = ifp->argument_count() - 1; i >= 0; i--) - { - if (isPredicateFunction((ifp->arguments()[i]), &gwi) && !gwi.ptWorkStack.empty()) - { - sptp.reset(gwi.ptWorkStack.top()); - tmpPtStack.push(sptp); - gwi.ptWorkStack.pop(); - } - else if (!isPredicateFunction((ifp->arguments()[i]), &gwi) && !gwi.rcWorkStack.empty()) - { - sptp.reset(new ParseTree(gwi.rcWorkStack.top())); - tmpPtStack.push(sptp); - gwi.rcWorkStack.pop(); - } - else - { - nonSupport = true; - return NULL; - } - } - - while (!tmpPtStack.empty()) - { - funcParms.push_back(tmpPtStack.top()); - tmpPtStack.pop(); - } + nonSupport = true; + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SUB_EXPRESSION); + return NULL; } - // the followings are special treatment of some functions - if (funcName == "week" && funcParms.size() == 1) + ReturnedColumn* rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport); + + // MCOL-1510 It must be a temp table field, so find the corresponding column. + if (!rc && ifp->arguments()[i]->type() == Item::REF_ITEM) { - THD* thd = current_thd; - sptp.reset(new ParseTree(new ConstantColumn(static_cast(thd->variables.default_week_format)))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); + gwi.fatalParseError = false; + rc = buildAggFrmTempField(ifp->arguments()[i], gwi); } - // add the keyword unit argument for interval function - if (funcName == "date_add_interval" || funcName == "extract" || funcName == "timestampdiff") + if (!rc || nonSupport) { - - addIntervalArgs(gwi.thd, ifp, funcParms); + nonSupport = true; + return NULL; } - // check for unsupported arguments add the keyword unit argument for extract functions - if (funcName == "extract") + sptp.reset(new ParseTree(rc)); + funcParms.push_back(sptp); + } + } + else // where clause + { + stack tmpPtStack; + + for (int32_t i = ifp->argument_count() - 1; i >= 0; i--) + { + if (isPredicateFunction((ifp->arguments()[i]), &gwi) && !gwi.ptWorkStack.empty()) { - Item_date_add_interval* idai = (Item_date_add_interval*)ifp; + sptp.reset(gwi.ptWorkStack.top()); + tmpPtStack.push(sptp); + gwi.ptWorkStack.pop(); + } + else if (!isPredicateFunction((ifp->arguments()[i]), &gwi) && !gwi.rcWorkStack.empty()) + { + sptp.reset(new ParseTree(gwi.rcWorkStack.top())); + tmpPtStack.push(sptp); + gwi.rcWorkStack.pop(); + } + else + { + nonSupport = true; + return NULL; + } + } - switch (idai->int_type) - { - case INTERVAL_DAY_MICROSECOND: - { - nonSupport = true; - gwi.fatalParseError = true; - Message::Args args; - string info = funcName + " with DAY_MICROSECOND parameter"; - args.add(info); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - return NULL; - } + while (!tmpPtStack.empty()) + { + funcParms.push_back(tmpPtStack.top()); + tmpPtStack.pop(); + } + } - case INTERVAL_HOUR_MICROSECOND: - { - nonSupport = true; - gwi.fatalParseError = true; - Message::Args args; - string info = funcName + " with HOUR_MICROSECOND parameter"; - args.add(info); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - return NULL; - } + // the followings are special treatment of some functions + if (funcName == "week" && funcParms.size() == 1) + { + THD* thd = current_thd; + sptp.reset( + new ParseTree(new ConstantColumn(static_cast(thd->variables.default_week_format)))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + } - case INTERVAL_MINUTE_MICROSECOND: - { - nonSupport = true; - gwi.fatalParseError = true; - Message::Args args; - string info = funcName + " with MINUTE_MICROSECOND parameter"; - args.add(info); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - return NULL; - } + // add the keyword unit argument for interval function + if (funcName == "date_add_interval" || funcName == "extract" || funcName == "timestampdiff") + { + addIntervalArgs(gwi.thd, ifp, funcParms); + } - default: - break; - } + // check for unsupported arguments add the keyword unit argument for extract functions + if (funcName == "extract") + { + Item_date_add_interval* idai = (Item_date_add_interval*)ifp; + + switch (idai->int_type) + { + case INTERVAL_DAY_MICROSECOND: + { + nonSupport = true; + gwi.fatalParseError = true; + Message::Args args; + string info = funcName + " with DAY_MICROSECOND parameter"; + args.add(info); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + return NULL; } - // add the keyword unit argument and char length for cast functions - if (funcName == "cast_as_char" ) + case INTERVAL_HOUR_MICROSECOND: { - castCharArgs(gwi.thd, ifp, funcParms); + nonSupport = true; + gwi.fatalParseError = true; + Message::Args args; + string info = funcName + " with HOUR_MICROSECOND parameter"; + args.add(info); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + return NULL; } - // add the length and scale arguments - if (funcName == "decimal_typecast" ) + case INTERVAL_MINUTE_MICROSECOND: { - castDecimalArgs(gwi.thd, ifp, funcParms); + nonSupport = true; + gwi.fatalParseError = true; + Message::Args args; + string info = funcName + " with MINUTE_MICROSECOND parameter"; + args.add(info); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + return NULL; } - // add the type argument - if (funcName == "get_format") - { - castTypeArgs(gwi.thd, ifp, funcParms); - } + default: break; + } + } - // add my_time_zone - if (funcName == "unix_timestamp") - { + // add the keyword unit argument and char length for cast functions + if (funcName == "cast_as_char") + { + castCharArgs(gwi.thd, ifp, funcParms); + } + + // add the length and scale arguments + if (funcName == "decimal_typecast") + { + castDecimalArgs(gwi.thd, ifp, funcParms); + } + + // add the type argument + if (funcName == "get_format") + { + castTypeArgs(gwi.thd, ifp, funcParms); + } + + // add my_time_zone + if (funcName == "unix_timestamp") + { #ifndef _MSC_VER - time_t tmp_t = 1; - struct tm tmp; - localtime_r(&tmp_t, &tmp); - sptp.reset(new ParseTree(new ConstantColumn(static_cast(tmp.tm_gmtoff), ConstantColumn::NUM))); + time_t tmp_t = 1; + struct tm tmp; + localtime_r(&tmp_t, &tmp); + sptp.reset(new ParseTree(new ConstantColumn(static_cast(tmp.tm_gmtoff), ConstantColumn::NUM))); #else - //FIXME: Get GMT offset (in seconds east of GMT) in Windows... - sptp.reset(new ParseTree(new ConstantColumn(static_cast(0), ConstantColumn::NUM))); + // FIXME: Get GMT offset (in seconds east of GMT) in Windows... + sptp.reset(new ParseTree(new ConstantColumn(static_cast(0), ConstantColumn::NUM))); #endif - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - } + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + } - // add the default seed to rand function without arguments - if (funcName == "rand") - { - if (funcParms.size() == 0) - { - sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed1, ConstantColumn::NUM))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed2, ConstantColumn::NUM))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - gwi.no_parm_func_list.push_back(fc); - } - else - { - ConstantColumn* cc = dynamic_cast(funcParms[0]->data()); + // add the default seed to rand function without arguments + if (funcName == "rand") + { + if (funcParms.size() == 0) + { + sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed1, ConstantColumn::NUM))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed2, ConstantColumn::NUM))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + gwi.no_parm_func_list.push_back(fc); + } + else + { + ConstantColumn* cc = dynamic_cast(funcParms[0]->data()); - if (cc) - gwi.no_parm_func_list.push_back(fc); - } - } + if (cc) + gwi.no_parm_func_list.push_back(fc); + } + } - // Take the value of to/from TZ data and check if its a description or offset via - // my_tzinfo_find. Offset value will leave the corresponding tzinfo funcParms empty. - // while descriptions will lookup the time_zone_info structure and serialize for use - // in primproc func_convert_tz - if (funcName == "convert_tz") - { - messageqcpp::ByteStream bs; - string tzinfo; - SimpleColumn* scCheck; - // MCOL-XXXX There is no way currently to perform this lookup when the timezone description - // comes from another table of timezone descriptions. - // 1. Move proc code into plugin where it will have access to this table data - // 2. Create a library that primproc can use to access the time zone data tables. - // for now throw a message that this is not supported - scCheck = dynamic_cast(funcParms[1]->data()); - if (scCheck) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "convert_tz with parameter column_name in a Columnstore table"; - setError(gwi.thd, ER_NOT_SUPPORTED_YET, gwi.parseErrorText, gwi); - return NULL; - } - scCheck = dynamic_cast(funcParms[2]->data()); - if (scCheck) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "convert_tz with parameter column_name in a Columnstore table"; - setError(gwi.thd, ER_NOT_SUPPORTED_YET, gwi.parseErrorText, gwi); - return NULL; - } - dataconvert::TIME_ZONE_INFO* from_tzinfo = my_tzinfo_find(gwi.thd, ifp->arguments()[1]->val_str()); - dataconvert::TIME_ZONE_INFO* to_tzinfo = my_tzinfo_find(gwi.thd, ifp->arguments()[2]->val_str()); - if (from_tzinfo) - { - serializeTimezoneInfo(bs,from_tzinfo); - uint32_t length = bs.length(); - uint8_t *buf = new uint8_t[length]; - bs >> buf; - tzinfo = string((char*)buf, length); - } - sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - tzinfo.clear(); - if (to_tzinfo) - { - serializeTimezoneInfo(bs,to_tzinfo); - uint32_t length = bs.length(); - uint8_t *buf = new uint8_t[length]; - bs >> buf; - tzinfo = string((char*)buf, length); - } - sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - tzinfo.clear(); - } - if (funcName == "sysdate") - { - gwi.no_parm_func_list.push_back(fc); - } + // Take the value of to/from TZ data and check if its a description or offset via + // my_tzinfo_find. Offset value will leave the corresponding tzinfo funcParms empty. + // while descriptions will lookup the time_zone_info structure and serialize for use + // in primproc func_convert_tz + if (funcName == "convert_tz") + { + messageqcpp::ByteStream bs; + string tzinfo; + SimpleColumn* scCheck; + // MCOL-XXXX There is no way currently to perform this lookup when the timezone description + // comes from another table of timezone descriptions. + // 1. Move proc code into plugin where it will have access to this table data + // 2. Create a library that primproc can use to access the time zone data tables. + // for now throw a message that this is not supported + scCheck = dynamic_cast(funcParms[1]->data()); + if (scCheck) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "convert_tz with parameter column_name in a Columnstore table"; + setError(gwi.thd, ER_NOT_SUPPORTED_YET, gwi.parseErrorText, gwi); + return NULL; + } + scCheck = dynamic_cast(funcParms[2]->data()); + if (scCheck) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "convert_tz with parameter column_name in a Columnstore table"; + setError(gwi.thd, ER_NOT_SUPPORTED_YET, gwi.parseErrorText, gwi); + return NULL; + } + dataconvert::TIME_ZONE_INFO* from_tzinfo = my_tzinfo_find(gwi.thd, ifp->arguments()[1]->val_str()); + dataconvert::TIME_ZONE_INFO* to_tzinfo = my_tzinfo_find(gwi.thd, ifp->arguments()[2]->val_str()); + if (from_tzinfo) + { + serializeTimezoneInfo(bs, from_tzinfo); + uint32_t length = bs.length(); + uint8_t* buf = new uint8_t[length]; + bs >> buf; + tzinfo = string((char*)buf, length); + } + sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + tzinfo.clear(); + if (to_tzinfo) + { + serializeTimezoneInfo(bs, to_tzinfo); + uint32_t length = bs.length(); + uint8_t* buf = new uint8_t[length]; + bs >> buf; + tzinfo = string((char*)buf, length); + } + sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + tzinfo.clear(); + } + if (funcName == "sysdate") + { + gwi.no_parm_func_list.push_back(fc); + } - // func name is addtime/subtime in 10.3.9 - // note: this means get_time() can now go away in our server fork - if ((funcName == "addtime") || (funcName == "subtime")) - { - int64_t sign = 1; - if (funcName == "subtime") - { - sign = -1; - } - sptp.reset(new ParseTree(new ConstantColumn(sign))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - } + // func name is addtime/subtime in 10.3.9 + // note: this means get_time() can now go away in our server fork + if ((funcName == "addtime") || (funcName == "subtime")) + { + int64_t sign = 1; + if (funcName == "subtime") + { + sign = -1; + } + sptp.reset(new ParseTree(new ConstantColumn(sign))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + } - fc->functionName(funcName); - fc->functionParms(funcParms); - fc->resultType(colType_MysqlToIDB(ifp)); + fc->functionName(funcName); + fc->functionParms(funcParms); + fc->resultType(colType_MysqlToIDB(ifp)); - // if the result type is DECIMAL and any function parameter is a wide decimal - // column, set the result colwidth to wide - if (fc->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + // if the result type is DECIMAL and any function parameter is a wide decimal + // column, set the result colwidth to wide + if (fc->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + { + for (size_t i = 0; i < funcParms.size(); i++) + { + if (funcParms[i]->data()->resultType().isWideDecimalType()) { - for (size_t i = 0; i < funcParms.size(); i++) - { - if (funcParms[i]->data()->resultType().isWideDecimalType()) - { - fc->resultType().colWidth = datatypes::MAXDECIMALWIDTH; - break; - } - } + fc->resultType().colWidth = datatypes::MAXDECIMALWIDTH; + break; } + } + } - // MySQL give string result type for date function, but has the flag set. - // we should set the result type to be datetime for comparision. - if (ifp->field_type() == MYSQL_TYPE_DATETIME || - ifp->field_type() == MYSQL_TYPE_DATETIME2) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DATETIME; - ct.colWidth = 8; - fc->resultType(ct); - } - if (ifp->field_type() == MYSQL_TYPE_TIMESTAMP || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::TIMESTAMP; - ct.colWidth = 8; - fc->resultType(ct); - } - else if (ifp->field_type() == MYSQL_TYPE_DATE) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DATE; - ct.colWidth = 4; - fc->resultType(ct); - } - else if (ifp->field_type() == MYSQL_TYPE_TIME) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::TIME; - ct.colWidth = 8; - fc->resultType(ct); - } + // MySQL give string result type for date function, but has the flag set. + // we should set the result type to be datetime for comparision. + if (ifp->field_type() == MYSQL_TYPE_DATETIME || ifp->field_type() == MYSQL_TYPE_DATETIME2) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DATETIME; + ct.colWidth = 8; + fc->resultType(ct); + } + if (ifp->field_type() == MYSQL_TYPE_TIMESTAMP || ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIMESTAMP; + ct.colWidth = 8; + fc->resultType(ct); + } + else if (ifp->field_type() == MYSQL_TYPE_DATE) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DATE; + ct.colWidth = 4; + fc->resultType(ct); + } + else if (ifp->field_type() == MYSQL_TYPE_TIME) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + fc->resultType(ct); + } #if 0 @@ -4409,2190 +4327,2118 @@ ReturnedColumn* buildFunctionColumn( #endif - fc->operationType(functor->operationType(funcParms, fc->resultType())); - // For some reason, MDB has MYSQL_TYPE_DATETIME2 for functions on a TIMESTAMP - if (fc->operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) - { - CalpontSystemCatalog::ColType ct = fc->resultType(); - ct.colDataType = CalpontSystemCatalog::TIMESTAMP; - ct.colWidth = 8; - fc->resultType(ct); - } - fc->expressionId(ci->expressionId++); - // A few functions use a different collation than that found in - // the base ifp class - if (funcName == "locate" || - funcName == "find_in_set" || - funcName == "strcmp") - { - DTCollation dt; - ifp->Type_std_attributes::agg_arg_charsets_for_comparison(dt, ifp->func_name_cstring(), ifp->arguments(), 1, 1); - fc->charsetNumber(dt.collation->number); - } - else - { - fc->charsetNumber(ifp->collation.collation->number); - } - } - else if (ifp->type() == Item::COND_ITEM || - ifp->functype() == Item_func::EQ_FUNC || - ifp->functype() == Item_func::NE_FUNC || - ifp->functype() == Item_func::LT_FUNC || - ifp->functype() == Item_func::LE_FUNC || - ifp->functype() == Item_func::GE_FUNC || - ifp->functype() == Item_func::GT_FUNC || - ifp->functype() == Item_func::LIKE_FUNC || - ifp->functype() == Item_func::BETWEEN || - ifp->functype() == Item_func::IN_FUNC || - ifp->functype() == Item_func::ISNULL_FUNC || - ifp->functype() == Item_func::ISNOTNULL_FUNC || - ifp->functype() == Item_func::NOT_FUNC || - ifp->functype() == Item_func::EQUAL_FUNC) + fc->operationType(functor->operationType(funcParms, fc->resultType())); + // For some reason, MDB has MYSQL_TYPE_DATETIME2 for functions on a TIMESTAMP + if (fc->operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) { - return NULL; + CalpontSystemCatalog::ColType ct = fc->resultType(); + ct.colDataType = CalpontSystemCatalog::TIMESTAMP; + ct.colWidth = 8; + fc->resultType(ct); + } + fc->expressionId(ci->expressionId++); + // A few functions use a different collation than that found in + // the base ifp class + if (funcName == "locate" || funcName == "find_in_set" || funcName == "strcmp") + { + DTCollation dt; + ifp->Type_std_attributes::agg_arg_charsets_for_comparison(dt, ifp->func_name_cstring(), + ifp->arguments(), 1, 1); + fc->charsetNumber(dt.collation->number); } else { - nonSupport = true; - gwi.fatalParseError = true; - Message::Args args; - args.add(funcName); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - return NULL; + fc->charsetNumber(ifp->collation.collation->number); } + } + else if (ifp->type() == Item::COND_ITEM || ifp->functype() == Item_func::EQ_FUNC || + ifp->functype() == Item_func::NE_FUNC || ifp->functype() == Item_func::LT_FUNC || + ifp->functype() == Item_func::LE_FUNC || ifp->functype() == Item_func::GE_FUNC || + ifp->functype() == Item_func::GT_FUNC || ifp->functype() == Item_func::LIKE_FUNC || + ifp->functype() == Item_func::BETWEEN || ifp->functype() == Item_func::IN_FUNC || + ifp->functype() == Item_func::ISNULL_FUNC || ifp->functype() == Item_func::ISNOTNULL_FUNC || + ifp->functype() == Item_func::NOT_FUNC || ifp->functype() == Item_func::EQUAL_FUNC) + { + return NULL; + } + else + { + nonSupport = true; + gwi.fatalParseError = true; + Message::Args args; + args.add(funcName); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + return NULL; + } - // adjust decimal result type according to internalDecimalScale - if (!fc) - return NULL; + // adjust decimal result type according to internalDecimalScale + if (!fc) + return NULL; - if (gwi.internalDecimalScale >= 0 && fc->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + if (gwi.internalDecimalScale >= 0 && fc->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + { + CalpontSystemCatalog::ColType ct = fc->resultType(); + ct.scale = gwi.internalDecimalScale; + fc->resultType(ct); + } + + if (ifp->name.length) + fc->alias(ifp->name.str); + + // @3391. optimization. try to associate expression ID to the expression on the select list + if (gwi.clauseType != SELECT) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) { - CalpontSystemCatalog::ColType ct = fc->resultType(); - ct.scale = gwi.internalDecimalScale; - fc->resultType(ct); + if ((!fc->alias().empty()) && + strcasecmp(fc->alias().c_str(), gwi.returnedCols[i]->alias().c_str()) == 0) + fc->expressionId(gwi.returnedCols[i]->expressionId()); } + } - if (ifp->name.length) - fc->alias(ifp->name.str); + // For function join. If any argument has non-zero joininfo, set it to the function. + fc->setSimpleColumnList(); + std::vector simpleColList = fc->simpleColumnList(); - // @3391. optimization. try to associate expression ID to the expression on the select list - if (gwi.clauseType != SELECT) + for (uint i = 0; i < simpleColList.size(); i++) + { + if (simpleColList[i]->joinInfo() != 0) { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if ((!fc->alias().empty()) && strcasecmp(fc->alias().c_str(), gwi.returnedCols[i]->alias().c_str()) == 0) - fc->expressionId(gwi.returnedCols[i]->expressionId()); - } + fc->joinInfo(simpleColList[i]->joinInfo()); + break; } + } - // For function join. If any argument has non-zero joininfo, set it to the function. - fc->setSimpleColumnList(); - std::vector simpleColList = fc->simpleColumnList(); + fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - for (uint i = 0; i < simpleColList.size(); i++) - { - if (simpleColList[i]->joinInfo() != 0) - { - fc->joinInfo(simpleColList[i]->joinInfo()); - break; - } - } - - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - - return fc; + return fc; } FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - FunctionColumn* fc = new FunctionColumn(); - FunctionParm funcParms; - SPTP sptp; - stack tmpPtStack; - FuncExp* funcexp = FuncExp::instance(); - string funcName = "case_simple"; + FunctionColumn* fc = new FunctionColumn(); + FunctionParm funcParms; + SPTP sptp; + stack tmpPtStack; + FuncExp* funcexp = FuncExp::instance(); + string funcName = "case_simple"; - if (item->functype() == Item_func::CASE_SEARCHED_FUNC) + if (item->functype() == Item_func::CASE_SEARCHED_FUNC) + { + funcName = "case_searched"; + } + /* if (dynamic_cast(item)) + { + funcName = "case_searched"; + }*/ + + funcParms.reserve(item->argument_count()); + // so buildXXXcolumn function will not pop stack. + ClauseType realClauseType = gwi.clauseType; + gwi.clauseType = SELECT; + + // We ought to be able to just build from the stack, and would + // be able to if there were any way to know which stack had the + // next case item. Unfortunately, parameters may have been pushed + // onto the ptWorkStack or rcWorkStack or neither, depending on type + // and position. We can't tell which at this point, so we + // rebuild the item from the arguments directly and then try to + // figure what to pop, if anything, in order to sync the stacks. + // + // MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order: + // [case,]when1,when2,...,then1,then2,...[,else] + // See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more + // information. + int32_t arg_offset = 0; + + if ((item->argument_count() - 1) % 2) + { + arg_offset = (item->argument_count() - 1) / 2; + } + else + { + arg_offset = item->argument_count() / 2; + } + + for (int32_t i = item->argument_count() - 1; i >= 0; i--) + { + // For case_searched, we know the items for the WHEN clause will + // not be ReturnedColumns. We do this separately just to save + // some cpu cycles trying to build a ReturnedColumn as below. + // Every even numbered arg is a WHEN. In between are the THEN. + // An odd number of args indicates an ELSE residing in the last spot. + if ((item->functype() == Item_func::CASE_SEARCHED_FUNC) && (i < arg_offset)) { - funcName = "case_searched"; - } -/* if (dynamic_cast(item)) - { - funcName = "case_searched"; - }*/ - - funcParms.reserve(item->argument_count()); - // so buildXXXcolumn function will not pop stack. - ClauseType realClauseType = gwi.clauseType; - gwi.clauseType = SELECT; - - // We ought to be able to just build from the stack, and would - // be able to if there were any way to know which stack had the - // next case item. Unfortunately, parameters may have been pushed - // onto the ptWorkStack or rcWorkStack or neither, depending on type - // and position. We can't tell which at this point, so we - // rebuild the item from the arguments directly and then try to - // figure what to pop, if anything, in order to sync the stacks. - // - // MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order: - // [case,]when1,when2,...,then1,then2,...[,else] - // See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more - // information. - int32_t arg_offset = 0; - - if ((item->argument_count() - 1) % 2) - { - arg_offset = (item->argument_count() - 1) / 2; + // MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate + // to pull off of rcWorkStack, so we set this inCaseStmt flag to tell it + // not to. + gwi.inCaseStmt = true; + sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); + gwi.inCaseStmt = false; + if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top() == *sptp.get()) + { + gwi.ptWorkStack.pop(); + } } else { - arg_offset = item->argument_count() / 2; - } + // First try building a ReturnedColumn. It may or may not succeed + // depending on the types involved. There's also little correlation + // between buildReturnedColumn and the existance of the item on + // rwWorkStack or ptWorkStack. + // For example, simple predicates, such as 1=1 or 1=0, land in the + // ptWorkStack but other stuff might land in the rwWorkStack + ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - for (int32_t i = item->argument_count() - 1; i >= 0; i--) - { - // For case_searched, we know the items for the WHEN clause will - // not be ReturnedColumns. We do this separately just to save - // some cpu cycles trying to build a ReturnedColumn as below. - // Every even numbered arg is a WHEN. In between are the THEN. - // An odd number of args indicates an ELSE residing in the last spot. - if ((item->functype() == Item_func::CASE_SEARCHED_FUNC) && - (i < arg_offset)) + if (parm) + { + sptp.reset(new ParseTree(parm)); + + // We need to pop whichever stack is holding it, if any. + if ((!gwi.rcWorkStack.empty()) && *gwi.rcWorkStack.top() == parm) { - // MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate - // to pull off of rcWorkStack, so we set this inCaseStmt flag to tell it - // not to. - gwi.inCaseStmt = true; - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - gwi.inCaseStmt = false; - if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top() == *sptp.get()) - { - gwi.ptWorkStack.pop(); - } + gwi.rcWorkStack.pop(); } - else + else if (!gwi.ptWorkStack.empty()) { - // First try building a ReturnedColumn. It may or may not succeed - // depending on the types involved. There's also little correlation - // between buildReturnedColumn and the existance of the item on - // rwWorkStack or ptWorkStack. - // For example, simple predicates, such as 1=1 or 1=0, land in the - // ptWorkStack but other stuff might land in the rwWorkStack - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); + ReturnedColumn* ptrc = dynamic_cast(gwi.ptWorkStack.top()->data()); - if (parm) - { - sptp.reset(new ParseTree(parm)); - - // We need to pop whichever stack is holding it, if any. - if ((!gwi.rcWorkStack.empty()) && - *gwi.rcWorkStack.top() == parm) - { - gwi.rcWorkStack.pop(); - } - else if (!gwi.ptWorkStack.empty()) - { - ReturnedColumn* ptrc = dynamic_cast(gwi.ptWorkStack.top()->data()); - - if (ptrc && *ptrc == *parm) - gwi.ptWorkStack.pop(); - } - } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - - // We need to pop whichever stack is holding it, if any. - if ((!gwi.ptWorkStack.empty()) && - *gwi.ptWorkStack.top()->data() == sptp->data()) - { - gwi.ptWorkStack.pop(); - } - else if (!gwi.rcWorkStack.empty()) - { - // Probably won't happen, but it might have been on the - // rcWorkStack all along. - ReturnedColumn* ptrc = dynamic_cast(sptp->data()); - - if (ptrc && *ptrc == *gwi.rcWorkStack.top()) - { - gwi.rcWorkStack.pop(); - } - } - } + if (ptrc && *ptrc == *parm) + gwi.ptWorkStack.pop(); } + } + else + { + sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - funcParms.insert(funcParms.begin(), sptp); - } - - // recover clause type - gwi.clauseType = realClauseType; - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return NULL; - } - - Func* functor = funcexp->getFunctor(funcName); - fc->resultType(colType_MysqlToIDB(item)); - fc->operationType(functor->operationType(funcParms, fc->resultType())); - fc->functionName(funcName); - fc->functionParms(funcParms); - fc->expressionId(ci->expressionId++); - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - - // For function join. If any argument has non-zero joininfo, set it to the function. - fc->setSimpleColumnList(); - std::vector simpleColList = fc->simpleColumnList(); - - for (uint i = 0; i < simpleColList.size(); i++) - { - if (simpleColList[i]->joinInfo() != 0) + // We need to pop whichever stack is holding it, if any. + if ((!gwi.ptWorkStack.empty()) && *gwi.ptWorkStack.top()->data() == sptp->data()) { - fc->joinInfo(simpleColList[i]->joinInfo()); - break; + gwi.ptWorkStack.pop(); } + else if (!gwi.rcWorkStack.empty()) + { + // Probably won't happen, but it might have been on the + // rcWorkStack all along. + ReturnedColumn* ptrc = dynamic_cast(sptp->data()); + + if (ptrc && *ptrc == *gwi.rcWorkStack.top()) + { + gwi.rcWorkStack.pop(); + } + } + } } - return fc; + funcParms.insert(funcParms.begin(), sptp); + } + + // recover clause type + gwi.clauseType = realClauseType; + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return NULL; + } + + Func* functor = funcexp->getFunctor(funcName); + fc->resultType(colType_MysqlToIDB(item)); + fc->operationType(functor->operationType(funcParms, fc->resultType())); + fc->functionName(funcName); + fc->functionParms(funcParms); + fc->expressionId(ci->expressionId++); + fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + + // For function join. If any argument has non-zero joininfo, set it to the function. + fc->setSimpleColumnList(); + std::vector simpleColList = fc->simpleColumnList(); + + for (uint i = 0; i < simpleColList.size(); i++) + { + if (simpleColList[i]->joinInfo() != 0) + { + fc->joinInfo(simpleColList[i]->joinInfo()); + break; + } + } + + return fc; } -ConstantColumn* buildDecimalColumn(const Item *idp, - const std::string &valStr, - gp_walk_info& gwi) +ConstantColumn* buildDecimalColumn(const Item* idp, const std::string& valStr, gp_walk_info& gwi) { - IDB_Decimal columnstore_decimal; - ostringstream columnstore_decimal_val; - uint32_t i = 0; + IDB_Decimal columnstore_decimal; + ostringstream columnstore_decimal_val; + uint32_t i = 0; - if (valStr[0] == '+' || valStr[0] == '-') - { - columnstore_decimal_val << valStr[0]; - i = 1; - } + if (valStr[0] == '+' || valStr[0] == '-') + { + columnstore_decimal_val << valStr[0]; + i = 1; + } - bool specialPrecision = false; + bool specialPrecision = false; - // handle the case when the constant value is 0.12345678901234567890123456789012345678 - // In this case idp->decimal_precision() = 39, but we can - if (((i + 1) < valStr.length()) && - valStr[i] == '0' && valStr[i + 1] == '.') - specialPrecision = true; + // handle the case when the constant value is 0.12345678901234567890123456789012345678 + // In this case idp->decimal_precision() = 39, but we can + if (((i + 1) < valStr.length()) && valStr[i] == '0' && valStr[i + 1] == '.') + specialPrecision = true; - for (; i < valStr.length(); i++) - { - if (valStr[i] == '.') - continue; + for (; i < valStr.length(); i++) + { + if (valStr[i] == '.') + continue; - columnstore_decimal_val << valStr[i]; - } + columnstore_decimal_val << valStr[i]; + } - if (idp->decimal_precision() <= datatypes::INT64MAXPRECISION) - columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); - else if (idp->decimal_precision() <= datatypes::INT128MAXPRECISION || - (idp->decimal_precision() <= datatypes::INT128MAXPRECISION + 1 && - specialPrecision)) - { - bool dummy = false; - columnstore_decimal.s128Value = - dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); - } + if (idp->decimal_precision() <= datatypes::INT64MAXPRECISION) + columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + else if (idp->decimal_precision() <= datatypes::INT128MAXPRECISION || + (idp->decimal_precision() <= datatypes::INT128MAXPRECISION + 1 && specialPrecision)) + { + bool dummy = false; + columnstore_decimal.s128Value = dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); + } - // TODO MCOL-641 Add support here - if (gwi.internalDecimalScale >= 0 && idp->decimals > (uint)gwi.internalDecimalScale) - { - columnstore_decimal.scale = gwi.internalDecimalScale; - uint32_t diff = (uint32_t) (idp->decimals - gwi.internalDecimalScale); - columnstore_decimal.value= columnstore_decimal.TDecimal64::toSInt64Round(diff); - } - else - columnstore_decimal.scale = idp->decimal_scale(); + // TODO MCOL-641 Add support here + if (gwi.internalDecimalScale >= 0 && idp->decimals > (uint)gwi.internalDecimalScale) + { + columnstore_decimal.scale = gwi.internalDecimalScale; + uint32_t diff = (uint32_t)(idp->decimals - gwi.internalDecimalScale); + columnstore_decimal.value = columnstore_decimal.TDecimal64::toSInt64Round(diff); + } + else + columnstore_decimal.scale = idp->decimal_scale(); - columnstore_decimal.precision = (idp->decimal_precision() > datatypes::INT128MAXPRECISION) ? - datatypes::INT128MAXPRECISION : idp->decimal_precision(); - ConstantColumn* cc = new ConstantColumn(valStr, columnstore_decimal); - cc->charsetNumber(idp->collation.collation->number); - return cc; + columnstore_decimal.precision = (idp->decimal_precision() > datatypes::INT128MAXPRECISION) + ? datatypes::INT128MAXPRECISION + : idp->decimal_precision(); + ConstantColumn* cc = new ConstantColumn(valStr, columnstore_decimal); + cc->charsetNumber(idp->collation.collation->number); + return cc; } SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) { - if (!gwi.csc) + if (!gwi.csc) + { + gwi.csc = CalpontSystemCatalog::makeCalpontSystemCatalog(gwi.sessionid); + gwi.csc->identity(CalpontSystemCatalog::FE); + } + + bool isInformationSchema = false; + + // @bug5523 + if (ifp->cached_table && ifp->cached_table->db.length > 0 && + strcmp(ifp->cached_table->db.str, "information_schema") == 0) + isInformationSchema = true; + + // support FRPM subquery. columns from the derived table has no definition + if ((!ifp->field || !ifp->db_name.str || strlen(ifp->db_name.str) == 0) && !isInformationSchema) + return buildSimpleColFromDerivedTable(gwi, ifp); + + CalpontSystemCatalog::ColType ct; + datatypes::SimpleColumnParam prm(gwi.sessionid, true); + + try + { + // check foreign engine + if (ifp->cached_table && ifp->cached_table->table) + prm.columnStore(isMCSTable(ifp->cached_table->table)); + // @bug4509. ifp->cached_table could be null for myisam sometimes + else if (ifp->field && ifp->field->table) + prm.columnStore(isMCSTable(ifp->field->table)); + + if (prm.columnStore()) { - gwi.csc = CalpontSystemCatalog::makeCalpontSystemCatalog(gwi.sessionid); - gwi.csc->identity(CalpontSystemCatalog::FE); + ct = gwi.csc->colType( + gwi.csc->lookupOID(make_tcn(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str))); } - - bool isInformationSchema = false; - - // @bug5523 - if (ifp->cached_table && ifp->cached_table->db.length > 0 - && strcmp(ifp->cached_table->db.str, "information_schema") == 0) - isInformationSchema = true; - - // support FRPM subquery. columns from the derived table has no definition - if ((!ifp->field || !ifp->db_name.str || strlen(ifp->db_name.str) == 0) && !isInformationSchema) - return buildSimpleColFromDerivedTable(gwi, ifp); - - CalpontSystemCatalog::ColType ct; - datatypes::SimpleColumnParam prm(gwi.sessionid, true); - - try + else { - // check foreign engine - if (ifp->cached_table && ifp->cached_table->table) - prm.columnStore(isMCSTable(ifp->cached_table->table)); - // @bug4509. ifp->cached_table could be null for myisam sometimes - else if (ifp->field && ifp->field->table) - prm.columnStore(isMCSTable(ifp->field->table)); - - if (prm.columnStore()) - { - ct = gwi.csc->colType( - gwi.csc->lookupOID(make_tcn(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str))); - } - else - { - ct = colType_MysqlToIDB(ifp); - } - } - catch (std::exception& ex ) - { - gwi.fatalParseError = true; - gwi.parseErrorText = ex.what(); - return NULL; + ct = colType_MysqlToIDB(ifp); } + } + catch (std::exception& ex) + { + gwi.fatalParseError = true; + gwi.parseErrorText = ex.what(); + return NULL; + } - const datatypes::DatabaseQualifiedColumnName name(ifp->db_name.str, - bestTableName(ifp), - ifp->field_name.str); - const datatypes::TypeHandler *h= ct.typeHandler(); - SimpleColumn *sc = h->newSimpleColumn(name, ct, prm); + const datatypes::DatabaseQualifiedColumnName name(ifp->db_name.str, bestTableName(ifp), + ifp->field_name.str); + const datatypes::TypeHandler* h = ct.typeHandler(); + SimpleColumn* sc = h->newSimpleColumn(name, ct, prm); - sc->resultType(ct); - sc->charsetNumber(ifp->collation.collation->number); - string tbname(ifp->table_name.str); + sc->resultType(ct); + sc->charsetNumber(ifp->collation.collation->number); + string tbname(ifp->table_name.str); - if (isInformationSchema) - { - sc->schemaName("information_schema"); - sc->tableName(tbname, lower_case_table_names); - } + if (isInformationSchema) + { + sc->schemaName("information_schema"); + sc->tableName(tbname, lower_case_table_names); + } - sc->tableAlias(tbname, lower_case_table_names); + sc->tableAlias(tbname, lower_case_table_names); - // view name - sc->viewName(getViewName(ifp->cached_table), lower_case_table_names); - sc->alias(ifp->name.str); + // view name + sc->viewName(getViewName(ifp->cached_table), lower_case_table_names); + sc->alias(ifp->name.str); - sc->isColumnStore(prm.columnStore()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->isColumnStore(prm.columnStore()); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - if (!prm.columnStore() && ifp->field) - sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 + if (!prm.columnStore() && ifp->field) + sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 - if (ifp->depended_from) - { - sc->joinInfo(sc->joinInfo() | JOIN_CORRELATED); + if (ifp->depended_from) + { + sc->joinInfo(sc->joinInfo() | JOIN_CORRELATED); - if (gwi.subQuery) - gwi.subQuery->correlated(true); + if (gwi.subQuery) + gwi.subQuery->correlated(true); - // for error out non-support select filter case (comparison outside semi join tables) - gwi.correlatedTbNameVec.push_back(make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias())); + // for error out non-support select filter case (comparison outside semi join tables) + gwi.correlatedTbNameVec.push_back(make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias())); - // imply semi for scalar for now. - if (gwi.subSelectType == CalpontSelectExecutionPlan::SINGLEROW_SUBS) - sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_SEMI); + // imply semi for scalar for now. + if (gwi.subSelectType == CalpontSelectExecutionPlan::SINGLEROW_SUBS) + sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_SEMI); - if (gwi.subSelectType == CalpontSelectExecutionPlan::SELECT_SUBS) - sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_OUTER_SELECT); - } + if (gwi.subSelectType == CalpontSelectExecutionPlan::SELECT_SUBS) + sc->joinInfo(sc->joinInfo() | JOIN_SCALAR | JOIN_OUTER_SELECT); + } - return sc; + return sc; } ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { - ParseTree* pt = 0; - Item_cond* icp = (Item_cond*)item; + ParseTree* pt = 0; + Item_cond* icp = (Item_cond*)item; #ifdef DEBUG_WALK_COND - // debug - cerr << "Build Parsetree: " << endl; - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + // debug + cerr << "Build Parsetree: " << endl; + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); #endif - //@bug5044. PPSTFIX walking should always be treated as WHERE clause filter - ClauseType clauseType = gwi.clauseType; - gwi.clauseType = WHERE; - icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - gwi.clauseType = clauseType; + //@bug5044. PPSTFIX walking should always be treated as WHERE clause filter + ClauseType clauseType = gwi.clauseType; + gwi.clauseType = WHERE; + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + gwi.clauseType = clauseType; - if (gwi.fatalParseError) - return NULL; + if (gwi.fatalParseError) + return NULL; - // bug 2840. if the filter/function is constant, result is in rcWorkStack - if (!gwi.ptWorkStack.empty()) - { - pt = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - } - else if (!gwi.rcWorkStack.empty()) - { - pt = new ParseTree (gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } + // bug 2840. if the filter/function is constant, result is in rcWorkStack + if (!gwi.ptWorkStack.empty()) + { + pt = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + } + else if (!gwi.rcWorkStack.empty()) + { + pt = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } - return pt; + return pt; } class ConstArgParam { -public: - unsigned int precision; - unsigned int scale; - bool bIsConst; - bool hasDecimalConst; - ConstArgParam() - :precision(0), - scale(0), - bIsConst(false), - hasDecimalConst(false) - { } + public: + unsigned int precision; + unsigned int scale; + bool bIsConst; + bool hasDecimalConst; + ConstArgParam() : precision(0), scale(0), bIsConst(false), hasDecimalConst(false) + { + } }; - -static bool -isSupportedAggregateWithOneConstArg(const Item_sum *item, Item ** orig_args) +static bool isSupportedAggregateWithOneConstArg(const Item_sum* item, Item** orig_args) { - if (item->argument_count() != 1 || !orig_args[0]->const_item()) - return false; - switch(orig_args[0]->cmp_type()) - { - case INT_RESULT: - case STRING_RESULT: - case REAL_RESULT: - case DECIMAL_RESULT: - return true; - default: - break; - } + if (item->argument_count() != 1 || !orig_args[0]->const_item()) return false; + switch (orig_args[0]->cmp_type()) + { + case INT_RESULT: + case STRING_RESULT: + case REAL_RESULT: + case DECIMAL_RESULT: return true; + default: break; + } + return false; } - -static void -processAggregateColumnConstArg(gp_walk_info& gwi, SRCP &parm, - AggregateColumn* ac, Item *sfitemp, - ConstArgParam & constParam) +static void processAggregateColumnConstArg(gp_walk_info& gwi, SRCP& parm, AggregateColumn* ac, Item* sfitemp, + ConstArgParam& constParam) { - DBUG_ASSERT(sfitemp->const_item()); - switch(sfitemp->cmp_type()) + DBUG_ASSERT(sfitemp->const_item()); + switch (sfitemp->cmp_type()) + { + case INT_RESULT: + case STRING_RESULT: + case REAL_RESULT: + case DECIMAL_RESULT: { - case INT_RESULT: - case STRING_RESULT: - case REAL_RESULT: - case DECIMAL_RESULT: - { - ReturnedColumn *rt = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); - if (!rt) - { - gwi.fatalParseError = true; - return; - } - ConstantColumn *cc; - if ((cc= dynamic_cast(rt)) && - cc->type() == ConstantColumn::NULLDATA) - { - // Explicit NULL or a const function that evaluated to NULL - cc = new ConstantColumnNull(); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - parm.reset(cc); - ac->constCol(SRCP(rt)); - return; - } + ReturnedColumn* rt = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); + if (!rt) + { + gwi.fatalParseError = true; + return; + } + ConstantColumn* cc; + if ((cc = dynamic_cast(rt)) && cc->type() == ConstantColumn::NULLDATA) + { + // Explicit NULL or a const function that evaluated to NULL + cc = new ConstantColumnNull(); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + parm.reset(cc); + ac->constCol(SRCP(rt)); + return; + } - // treat as count(*) - if (ac->aggOp() == AggregateColumn::COUNT) - ac->aggOp(AggregateColumn::COUNT_ASTERISK); + // treat as count(*) + if (ac->aggOp() == AggregateColumn::COUNT) + ac->aggOp(AggregateColumn::COUNT_ASTERISK); - parm.reset(rt); - ac->constCol(parm); - constParam.bIsConst = true; - if (sfitemp->cmp_type() == DECIMAL_RESULT) - { - constParam.hasDecimalConst = true; - constParam.precision = sfitemp->decimal_precision(); - constParam.scale = sfitemp->decimal_scale(); - } - break; - } - case TIME_RESULT: - // QQ: why temporal constants are not handled? - case ROW_RESULT: - { - gwi.fatalParseError = true; - } + parm.reset(rt); + ac->constCol(parm); + constParam.bIsConst = true; + if (sfitemp->cmp_type() == DECIMAL_RESULT) + { + constParam.hasDecimalConst = true; + constParam.precision = sfitemp->decimal_precision(); + constParam.scale = sfitemp->decimal_scale(); + } + break; } + case TIME_RESULT: + // QQ: why temporal constants are not handled? + case ROW_RESULT: + { + gwi.fatalParseError = true; + } + } } - ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) { - // MCOL-1201 For UDAnF multiple parameters - vector selCols; - vector orderCols; - ConstArgParam constArgParam; + // MCOL-1201 For UDAnF multiple parameters + vector selCols; + vector orderCols; + ConstArgParam constArgParam; - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - Item_sum* isp = reinterpret_cast(item); - Item** sfitempp = isp->get_orig_args(); - SRCP parm; + Item_sum* isp = reinterpret_cast(item); + Item** sfitempp = isp->get_orig_args(); + SRCP parm; - // @bug4756 - if (gwi.clauseType == SELECT) - gwi.aggOnSelect = true; + // @bug4756 + if (gwi.clauseType == SELECT) + gwi.aggOnSelect = true; - // Argument_count() is the # of formal parms to the agg fcn. Columnstore - // only supports 1 argument except UDAnF, COUNT(DISTINC) and GROUP_CONCAT - if (isp->argument_count() != 1 - && isp->sum_func() != Item_sum::COUNT_DISTINCT_FUNC - && isp->sum_func() != Item_sum::GROUP_CONCAT_FUNC - && isp->sum_func() != Item_sum::UDF_SUM_FUNC) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_MUL_ARG_AGG); - return NULL; - } + // Argument_count() is the # of formal parms to the agg fcn. Columnstore + // only supports 1 argument except UDAnF, COUNT(DISTINC) and GROUP_CONCAT + if (isp->argument_count() != 1 && isp->sum_func() != Item_sum::COUNT_DISTINCT_FUNC && + isp->sum_func() != Item_sum::GROUP_CONCAT_FUNC && isp->sum_func() != Item_sum::UDF_SUM_FUNC) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_MUL_ARG_AGG); + return NULL; + } - AggregateColumn* ac = NULL; + AggregateColumn* ac = NULL; + if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) + { + ac = new GroupConcatColumn(gwi.sessionid); + } + else if (isp->sum_func() == Item_sum::UDF_SUM_FUNC) + { + ac = new UDAFColumn(gwi.sessionid); + } + else + { + ac = new AggregateColumn(gwi.sessionid); + } + + ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + + if (isp->name.length) + ac->alias(isp->name.str); + + if ((setAggOp(ac, isp))) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Non supported aggregate type on the select clause"; + + if (ac) + delete ac; + + return NULL; + } + + try + { + // special parsing for group_concat if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) { - ac = new GroupConcatColumn(gwi.sessionid); + Item_func_group_concat* gc = (Item_func_group_concat*)isp; + vector orderCols; + RowColumn* rowCol = new RowColumn(); + vector selCols; + + uint32_t select_ctn = gc->get_count_field(); + ReturnedColumn* rc = NULL; + + for (uint32_t i = 0; i < select_ctn; i++) + { + rc = buildReturnedColumn(sfitempp[i], gwi, gwi.fatalParseError); + + if (!rc || gwi.fatalParseError) + { + if (ac) + delete ac; + + return NULL; + } + + selCols.push_back(SRCP(rc)); + } + + ORDER **order_item, **end; + + for (order_item = gc->get_order(), end = order_item + gc->get_order_field(); order_item < end; + order_item++) + { + Item* ord_col = *(*order_item)->item; + + if (ord_col->type() == Item::CONST_ITEM && ord_col->cmp_type() == INT_RESULT) + { + Item_int* id = (Item_int*)ord_col; + + if (id->val_int() > (int)selCols.size()) + { + gwi.fatalParseError = true; + + if (ac) + delete ac; + + return NULL; + } + + rc = selCols[id->val_int() - 1]->clone(); + rc->orderPos(id->val_int() - 1); + } + else + { + rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); + + if (!rc || gwi.fatalParseError) + { + if (ac) + delete ac; + + return NULL; + } + } + + // 10.2 TODO: direction is now a tri-state flag + rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); + orderCols.push_back(SRCP(rc)); + } + + rowCol->columnVec(selCols); + (dynamic_cast(ac))->orderCols(orderCols); + parm.reset(rowCol); + ac->aggParms().push_back(parm); + + if (gc->get_separator()) + { + string separator; + separator.assign(gc->get_separator()->ptr(), gc->get_separator()->length()); + (dynamic_cast(ac))->separator(separator); + } } - else if (isp->sum_func() == Item_sum::UDF_SUM_FUNC) + else if (isSupportedAggregateWithOneConstArg(isp, sfitempp)) { - ac = new UDAFColumn(gwi.sessionid); + processAggregateColumnConstArg(gwi, parm, ac, sfitempp[0], constArgParam); } else { - ac = new AggregateColumn(gwi.sessionid); - } + for (uint32_t i = 0; i < isp->argument_count(); i++) + { + Item* sfitemp = sfitempp[i]; + Item::Type sfitype = sfitemp->type(); - ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + switch (sfitype) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = reinterpret_cast(sfitemp); + SimpleColumn* sc = buildSimpleColumn(ifp, gwi); - if (isp->name.length) - ac->alias(isp->name.str); + if (!sc) + { + gwi.fatalParseError = true; + break; + } - if ((setAggOp(ac, isp))) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Non supported aggregate type on the select clause"; + parm.reset(sc); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), parm)); + TABLE_LIST* tmp = (ifp->cached_table ? ifp->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), + sc->isColumnStore())] = make_pair(1, tmp); + break; + } - if (ac) + case Item::CONST_ITEM: + case Item::NULL_ITEM: + { + processAggregateColumnConstArg(gwi, parm, ac, sfitemp, constArgParam); + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)sfitemp; + ReturnedColumn* rc = 0; + + // check count(1+1) case + vector tmpVec; + uint16_t parseInfo = 0; + parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo, &gwi); + + if (parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + break; + } + else if (!gwi.fatalParseError && !(parseInfo & AGG_BIT) && !(parseInfo & AF_BIT) && + tmpVec.size() == 0) + { + rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + FunctionColumn* fc = dynamic_cast(rc); + + if ((fc && fc->functionParms().empty()) || !fc) + { + ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); + + if (dynamic_cast(rc)) + { + //@bug5229. handle constant function on aggregate argument + ac->constCol(SRCP(rc)); + break; + } + } + } + + // MySQL carelessly allows correlated aggregate function on the WHERE clause. + // Here is the work around to deal with that inconsistence. + // e.g., SELECT (SELECT t.c FROM t1 AS t WHERE t.b=MAX(t1.b + 0)) FROM t1; + ClauseType clauseType = gwi.clauseType; + + if (gwi.clauseType == WHERE) + gwi.clauseType = HAVING; + + // @bug 3603. for cases like max(rand()). try to build function first. + if (!rc) + rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + parm.reset(rc); + gwi.clauseType = clauseType; + + if (gwi.fatalParseError) + break; + + break; + } + + case Item::REF_ITEM: + { + ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); + + if (rc) + { + parm.reset(rc); + break; + } + } + /* fall through */ + + default: + { + gwi.fatalParseError = true; + } + } + + if (gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + { + Message::Args args; + + if (item->name.length) + args.add(item->name.str); + else + args.add(""); + + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_AGG_ARGS, args); + } + + if (ac) delete ac; - return NULL; + return NULL; + } + + if (parm) + { + // MCOL-1201 multi-argument aggregate + ac->aggParms().push_back(parm); + } + } } - try + bool isAvg = (isp->sum_func() == Item_sum::AVG_FUNC || isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC); + + // Get result type + // Modified for MCOL-1201 multi-argument aggregate + if (!constArgParam.bIsConst && ac->aggParms().size() > 0) { + // These are all one parm functions, so we can safely + // use the first parm for result type. + parm = ac->aggParms()[0]; - // special parsing for group_concat - if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) + if (isAvg || isp->sum_func() == Item_sum::SUM_FUNC || isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) + { + CalpontSystemCatalog::ColType ct = parm->resultType(); + if (ct.isWideDecimalType()) { - Item_func_group_concat* gc = (Item_func_group_concat*)isp; - vector orderCols; - RowColumn* rowCol = new RowColumn(); - vector selCols; - - uint32_t select_ctn = gc->get_count_field(); - ReturnedColumn* rc = NULL; - - for (uint32_t i = 0; i < select_ctn; i++) - { - rc = buildReturnedColumn(sfitempp[i], gwi, gwi.fatalParseError); - - if (!rc || gwi.fatalParseError) - { - if (ac) - delete ac; - - return NULL; - } - - selCols.push_back(SRCP(rc)); - } - - ORDER** order_item, **end; - - for (order_item = gc->get_order(), - end = order_item + gc->get_order_field(); order_item < end; - order_item++) - { - Item* ord_col = *(*order_item)->item; - - if (ord_col->type() == Item::CONST_ITEM - && ord_col->cmp_type() == INT_RESULT) - { - Item_int* id = (Item_int*)ord_col; - - if (id->val_int() > (int)selCols.size()) - { - gwi.fatalParseError = true; - - if (ac) - delete ac; - - return NULL; - } - - rc = selCols[id->val_int() - 1]->clone(); - rc->orderPos(id->val_int() - 1); - } - else - { - rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); - - if (!rc || gwi.fatalParseError) - { - if (ac) - delete ac; - - return NULL; - } - } - - // 10.2 TODO: direction is now a tri-state flag - rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); - orderCols.push_back(SRCP(rc)); - } - - rowCol->columnVec(selCols); - (dynamic_cast(ac))->orderCols(orderCols); - parm.reset(rowCol); - ac->aggParms().push_back(parm); - - if (gc->get_separator()) - { - string separator; - separator.assign(gc->get_separator()->ptr(), gc->get_separator()->length()); - (dynamic_cast(ac))->separator(separator); - } + uint32_t precision = ct.precision; + uint32_t scale = ct.scale; + if (isAvg) + { + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + } + ct.precision = precision; + ct.scale = scale; } - else if (isSupportedAggregateWithOneConstArg(isp, sfitempp)) + else if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(ct.colDataType)) { - processAggregateColumnConstArg(gwi, parm, ac, sfitempp[0], constArgParam); + uint32_t precision = datatypes::INT128MAXPRECISION; + uint32_t scale = ct.scale; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + ct.colWidth = datatypes::MAXDECIMALWIDTH; + if (isAvg) + { + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + } + ct.scale = scale; + ct.precision = precision; } else { - for (uint32_t i = 0; i < isp->argument_count(); i++) - { - Item* sfitemp = sfitempp[i]; - Item::Type sfitype = sfitemp->type(); - - switch (sfitype) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = reinterpret_cast(sfitemp); - SimpleColumn* sc = buildSimpleColumn(ifp, gwi); - - if (!sc) - { - gwi.fatalParseError = true; - break; - } - - parm.reset(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), parm)); - TABLE_LIST* tmp = (ifp->cached_table ? ifp->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = make_pair(1, tmp); - break; - } - - case Item::CONST_ITEM: - case Item::NULL_ITEM: - { - processAggregateColumnConstArg(gwi, parm, - ac, sfitemp, - constArgParam); - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = (Item_func*)sfitemp; - ReturnedColumn* rc = 0; - - // check count(1+1) case - vector tmpVec; - uint16_t parseInfo = 0; - parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo, &gwi); - - if (parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; - break; - } - else if (!gwi.fatalParseError && - !(parseInfo & AGG_BIT) && - !(parseInfo & AF_BIT) && - tmpVec.size() == 0) - { - rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - FunctionColumn* fc = dynamic_cast(rc); - - if ((fc && fc->functionParms().empty()) || !fc) - { - ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); - - if (dynamic_cast(rc)) - { - //@bug5229. handle constant function on aggregate argument - ac->constCol(SRCP(rc)); - break; - } - } - } - - // MySQL carelessly allows correlated aggregate function on the WHERE clause. - // Here is the work around to deal with that inconsistence. - // e.g., SELECT (SELECT t.c FROM t1 AS t WHERE t.b=MAX(t1.b + 0)) FROM t1; - ClauseType clauseType = gwi.clauseType; - - if (gwi.clauseType == WHERE) - gwi.clauseType = HAVING; - - // @bug 3603. for cases like max(rand()). try to build function first. - if (!rc) - rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - parm.reset(rc); - gwi.clauseType = clauseType; - - if (gwi.fatalParseError) - break; - - break; - } - - case Item::REF_ITEM: - { - ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); - - if (rc) - { - parm.reset(rc); - break; - } - } - /* fall through */ - - default: - { - gwi.fatalParseError = true; - } - } - - if (gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - { - Message::Args args; - - if (item->name.length) - args.add(item->name.str); - else - args.add(""); - - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_AGG_ARGS, args); - } - - if (ac) - delete ac; - - return NULL; - } - - if (parm) - { - // MCOL-1201 multi-argument aggregate - ac->aggParms().push_back(parm); - } - } + ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; + ct.colWidth = sizeof(long double); + if (isAvg) + { + ct.scale += datatypes::MAXSCALEINC4AVG; + } + ct.precision = datatypes::INT64MAXPRECISION; } - - bool isAvg = (isp->sum_func() == Item_sum::AVG_FUNC || - isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC); - - // Get result type - // Modified for MCOL-1201 multi-argument aggregate - if (!constArgParam.bIsConst && ac->aggParms().size() > 0) - { - // These are all one parm functions, so we can safely - // use the first parm for result type. - parm = ac->aggParms()[0]; - - if (isAvg || isp->sum_func() == Item_sum::SUM_FUNC || - isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) - { - CalpontSystemCatalog::ColType ct = parm->resultType(); - if (ct.isWideDecimalType()) - { - uint32_t precision = ct.precision; - uint32_t scale = ct.scale; - if (isAvg) - { - datatypes::Decimal::setScalePrecision4Avg(precision, scale); - } - ct.precision = precision; - ct.scale = scale; - } - else if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(ct.colDataType)) - { - uint32_t precision = datatypes::INT128MAXPRECISION; - uint32_t scale = ct.scale; - ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = datatypes::MAXDECIMALWIDTH; - if (isAvg) - { - datatypes::Decimal::setScalePrecision4Avg(precision, scale); - } - ct.scale = scale; - ct.precision = precision; - } - else - { - ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; - ct.colWidth = sizeof(long double); - if (isAvg) - { - ct.scale += datatypes::MAXSCALEINC4AVG; - } - ct.precision = datatypes::INT64MAXPRECISION; - } - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::COUNT_FUNC || - isp->sum_func() == Item_sum::COUNT_DISTINCT_FUNC) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - ct.scale = 0; - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::STD_FUNC || - isp->sum_func() == Item_sum::VARIANCE_FUNC) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.colWidth = 8; - ct.scale = 0; - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::SUM_BIT_FUNC) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - ct.scale = 0; - ct.precision = -16; // borrowed to indicate skip null value check on connector - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) - { - //Item_func_group_concat* gc = (Item_func_group_concat*)isp; - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = isp->max_length; - ct.precision = 0; - ac->resultType(ct); - } - // Setting the ColType in the resulting RowGroup - // according with what MDB expects. - // Internal processing UDAF result type will be set below. - else if (isUDFSumItem(isp)) - { - ac->resultType(colType_MysqlToIDB(isp)); - } - // Using the first param to deduce ac data type - else if (ac->aggParms().size() == 1) - { - ac->resultType(parm->resultType()); - } - else - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Can not deduce Aggregate Column resulting type \ + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::COUNT_FUNC || isp->sum_func() == Item_sum::COUNT_DISTINCT_FUNC) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + ct.scale = 0; + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::STD_FUNC || isp->sum_func() == Item_sum::VARIANCE_FUNC) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.colWidth = 8; + ct.scale = 0; + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::SUM_BIT_FUNC) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + ct.scale = 0; + ct.precision = -16; // borrowed to indicate skip null value check on connector + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) + { + // Item_func_group_concat* gc = (Item_func_group_concat*)isp; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = isp->max_length; + ct.precision = 0; + ac->resultType(ct); + } + // Setting the ColType in the resulting RowGroup + // according with what MDB expects. + // Internal processing UDAF result type will be set below. + else if (isUDFSumItem(isp)) + { + ac->resultType(colType_MysqlToIDB(isp)); + } + // Using the first param to deduce ac data type + else if (ac->aggParms().size() == 1) + { + ac->resultType(parm->resultType()); + } + else + { + gwi.fatalParseError = true; + gwi.parseErrorText = + "Can not deduce Aggregate Column resulting type \ because it has multiple arguments."; - if (ac) - delete ac; + if (ac) + delete ac; - return nullptr; - } - } - else if (constArgParam.bIsConst && constArgParam.hasDecimalConst && isAvg) - { - CalpontSystemCatalog::ColType ct = parm->resultType(); - if (datatypes::Decimal::isWideDecimalTypeByPrecision(constArgParam.precision)) - { - ct.precision = constArgParam.precision; - ct.scale = constArgParam.scale; - ct.colWidth = datatypes::MAXDECIMALWIDTH; - } - ac->resultType(ct); - } - else - { - ac->resultType(colType_MysqlToIDB(isp)); - } - - // adjust decimal result type according to internalDecimalScale - bool isWideDecimal = ac->resultType().isWideDecimalType(); - if (!isWideDecimal && gwi.internalDecimalScale >= 0 && - (ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL || - ac->resultType().colDataType == CalpontSystemCatalog::UDECIMAL)) - { - CalpontSystemCatalog::ColType ct = ac->resultType(); - ct.scale = gwi.internalDecimalScale; - ac->resultType(ct); - } - - // check for same aggregate on the select list - ac->expressionId(ci->expressionId++); - - if (gwi.clauseType != SELECT) - { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (*ac == gwi.returnedCols[i].get() - && ac->alias() == gwi.returnedCols[i].get()->alias()) - ac->expressionId(gwi.returnedCols[i]->expressionId()); - } - } - - // @bug5977 @note Temporary fix to avoid mysqld crash. The permanent fix will - // be applied in ExeMgr. When the ExeMgr fix is available, this checking - // will be taken out. - if (isp->sum_func() != Item_sum::UDF_SUM_FUNC) - { - if (ac->constCol() && gwi.tbList.empty() && gwi.derivedTbList.empty()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "No project column found for aggregate function"; - - if (ac) - delete ac; - - return NULL; - } - else if (ac->constCol()) - { - gwi.count_asterisk_list.push_back(ac); - } - } - - // For UDAF, populate the context and call the UDAF init() function. - // The return type is (should be) set in context by init(). - if (isp->sum_func() == Item_sum::UDF_SUM_FUNC) - { - UDAFColumn* udafc = dynamic_cast(ac); - - if (udafc) - { - mcsv1sdk::mcsv1Context& context = udafc->getContext(); - context.setName(isp->func_name()); - - // Get the return type as defined by CREATE AGGREGATE FUNCTION - // Most functions don't care, but some may. - context.setMariaDBReturnType((mcsv1sdk::enum_mariadb_return_type)isp->field_type()); - - // Set up the return type defaults for the call to init() - context.setResultType(udafc->resultType().colDataType); - context.setColWidth(udafc->resultType().colWidth); - context.setScale(udafc->resultType().scale); - context.setPrecision(udafc->resultType().precision); - - context.setParamCount(udafc->aggParms().size()); - utils::VLArray colTypes(udafc->aggParms().size()); - - // Build the column type vector. - // Modified for MCOL-1201 multi-argument aggregate - for (uint32_t i = 0; i < udafc->aggParms().size(); ++i) - { - const execplan::CalpontSystemCatalog::ColType& resultType - = udafc->aggParms()[i]->resultType(); - mcsv1sdk::ColumnDatum& colType = colTypes[i]; - colType.dataType = resultType.colDataType; - colType.precision = resultType.precision; - colType.scale = resultType.scale; - colType.charsetNumber = resultType.charsetNumber; - } - - // Call the user supplied init() - mcsv1sdk::mcsv1_UDAF* udaf = context.getFunction(); - - if (!udaf) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Aggregate Function " + context.getName() + " doesn't exist in the ColumnStore engine"; - - if (ac) - delete ac; - - return NULL; - } - - if (udaf->init(&context, colTypes) == mcsv1sdk::mcsv1_UDAF::ERROR) - { - gwi.fatalParseError = true; - gwi.parseErrorText = udafc->getContext().getErrorMessage(); - - if (ac) - delete ac; - - return NULL; - } - - // UDAF_OVER_REQUIRED means that this function is for Window - // Function only. Reject it here in aggregate land. - if (udafc->getContext().getRunFlag(mcsv1sdk::UDAF_OVER_REQUIRED)) - { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WINDOW_FUNC_ONLY, - context.getName()); - - if (ac) - delete ac; - - return NULL; - } - - // Set the return type as set in init() - CalpontSystemCatalog::ColType ct; - ct.colDataType = context.getResultType(); - ct.colWidth = context.getColWidth(); - ct.scale = context.getScale(); - ct.precision = context.getPrecision(); - udafc->resultType(ct); - } - } + return nullptr; + } } - catch (std::logic_error &e) + else if (constArgParam.bIsConst && constArgParam.hasDecimalConst && isAvg) { + CalpontSystemCatalog::ColType ct = parm->resultType(); + if (datatypes::Decimal::isWideDecimalTypeByPrecision(constArgParam.precision)) + { + ct.precision = constArgParam.precision; + ct.scale = constArgParam.scale; + ct.colWidth = datatypes::MAXDECIMALWIDTH; + } + ac->resultType(ct); + } + else + { + ac->resultType(colType_MysqlToIDB(isp)); + } + + // adjust decimal result type according to internalDecimalScale + bool isWideDecimal = ac->resultType().isWideDecimalType(); + if (!isWideDecimal && gwi.internalDecimalScale >= 0 && + (ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL || + ac->resultType().colDataType == CalpontSystemCatalog::UDECIMAL)) + { + CalpontSystemCatalog::ColType ct = ac->resultType(); + ct.scale = gwi.internalDecimalScale; + ac->resultType(ct); + } + + // check for same aggregate on the select list + ac->expressionId(ci->expressionId++); + + if (gwi.clauseType != SELECT) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (*ac == gwi.returnedCols[i].get() && ac->alias() == gwi.returnedCols[i].get()->alias()) + ac->expressionId(gwi.returnedCols[i]->expressionId()); + } + } + + // @bug5977 @note Temporary fix to avoid mysqld crash. The permanent fix will + // be applied in ExeMgr. When the ExeMgr fix is available, this checking + // will be taken out. + if (isp->sum_func() != Item_sum::UDF_SUM_FUNC) + { + if (ac->constCol() && gwi.tbList.empty() && gwi.derivedTbList.empty()) + { gwi.fatalParseError = true; - gwi.parseErrorText = "error building Aggregate Function: "; - gwi.parseErrorText += e.what(); + gwi.parseErrorText = "No project column found for aggregate function"; if (ac) - delete ac; + delete ac; return NULL; + } + else if (ac->constCol()) + { + gwi.count_asterisk_list.push_back(ac); + } } - catch (...) + + // For UDAF, populate the context and call the UDAF init() function. + // The return type is (should be) set in context by init(). + if (isp->sum_func() == Item_sum::UDF_SUM_FUNC) { - gwi.fatalParseError = true; - gwi.parseErrorText = "error building Aggregate Function: Unspecified exception"; + UDAFColumn* udafc = dynamic_cast(ac); - if (ac) + if (udafc) + { + mcsv1sdk::mcsv1Context& context = udafc->getContext(); + context.setName(isp->func_name()); + + // Get the return type as defined by CREATE AGGREGATE FUNCTION + // Most functions don't care, but some may. + context.setMariaDBReturnType((mcsv1sdk::enum_mariadb_return_type)isp->field_type()); + + // Set up the return type defaults for the call to init() + context.setResultType(udafc->resultType().colDataType); + context.setColWidth(udafc->resultType().colWidth); + context.setScale(udafc->resultType().scale); + context.setPrecision(udafc->resultType().precision); + + context.setParamCount(udafc->aggParms().size()); + utils::VLArray colTypes(udafc->aggParms().size()); + + // Build the column type vector. + // Modified for MCOL-1201 multi-argument aggregate + for (uint32_t i = 0; i < udafc->aggParms().size(); ++i) + { + const execplan::CalpontSystemCatalog::ColType& resultType = udafc->aggParms()[i]->resultType(); + mcsv1sdk::ColumnDatum& colType = colTypes[i]; + colType.dataType = resultType.colDataType; + colType.precision = resultType.precision; + colType.scale = resultType.scale; + colType.charsetNumber = resultType.charsetNumber; + } + + // Call the user supplied init() + mcsv1sdk::mcsv1_UDAF* udaf = context.getFunction(); + + if (!udaf) + { + gwi.fatalParseError = true; + gwi.parseErrorText = + "Aggregate Function " + context.getName() + " doesn't exist in the ColumnStore engine"; + + if (ac) delete ac; - return NULL; - } + return NULL; + } - ac->charsetNumber(item->collation.collation->number); - return ac; + if (udaf->init(&context, colTypes) == mcsv1sdk::mcsv1_UDAF::ERROR) + { + gwi.fatalParseError = true; + gwi.parseErrorText = udafc->getContext().getErrorMessage(); + + if (ac) + delete ac; + + return NULL; + } + + // UDAF_OVER_REQUIRED means that this function is for Window + // Function only. Reject it here in aggregate land. + if (udafc->getContext().getRunFlag(mcsv1sdk::UDAF_OVER_REQUIRED)) + { + gwi.fatalParseError = true; + gwi.parseErrorText = + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WINDOW_FUNC_ONLY, context.getName()); + + if (ac) + delete ac; + + return NULL; + } + + // Set the return type as set in init() + CalpontSystemCatalog::ColType ct; + ct.colDataType = context.getResultType(); + ct.colWidth = context.getColWidth(); + ct.scale = context.getScale(); + ct.precision = context.getPrecision(); + udafc->resultType(ct); + } + } + } + catch (std::logic_error& e) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "error building Aggregate Function: "; + gwi.parseErrorText += e.what(); + + if (ac) + delete ac; + + return NULL; + } + catch (...) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "error building Aggregate Function: Unspecified exception"; + + if (ac) + delete ac; + + return NULL; + } + + ac->charsetNumber(item->collation.collation->number); + return ac; } void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { - string funcName = ifp->func_name(); - int interval_type = -1; + string funcName = ifp->func_name(); + int interval_type = -1; - if (funcName == "date_add_interval") - interval_type = ((Item_date_add_interval*)ifp)->int_type; - else if (funcName == "timestampdiff") - interval_type = ((Item_func_timestamp_diff*)ifp)->get_int_type(); - else if (funcName == "extract") - interval_type = ((Item_extract*)ifp)->int_type; + if (funcName == "date_add_interval") + interval_type = ((Item_date_add_interval*)ifp)->int_type; + else if (funcName == "timestampdiff") + interval_type = ((Item_func_timestamp_diff*)ifp)->get_int_type(); + else if (funcName == "extract") + interval_type = ((Item_extract*)ifp)->int_type; - functionParms.push_back(getIntervalType(thd, interval_type)); - SPTP sptp; + functionParms.push_back(getIntervalType(thd, interval_type)); + SPTP sptp; - if (funcName == "date_add_interval") + if (funcName == "date_add_interval") + { + if (((Item_date_add_interval*)ifp)->date_sub_interval) { - if (((Item_date_add_interval*)ifp)->date_sub_interval) - { - sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_SUB))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - functionParms.push_back(sptp); - } - else - { - sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_ADD))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - functionParms.push_back(sptp); - } + sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_SUB))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + functionParms.push_back(sptp); } + else + { + sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_ADD))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + functionParms.push_back(sptp); + } + } } SPTP getIntervalType(THD* thd, int interval_type) { - SPTP sptp; - sptp.reset(new ParseTree(new ConstantColumn((int64_t)interval_type))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - return sptp; + SPTP sptp; + sptp.reset(new ParseTree(new ConstantColumn((int64_t)interval_type))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + return sptp; } void castCharArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { - Item_char_typecast* idai = (Item_char_typecast*)ifp; + Item_char_typecast* idai = (Item_char_typecast*)ifp; - SPTP sptp; - sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->get_cast_length()))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - functionParms.push_back(sptp); + SPTP sptp; + sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->get_cast_length()))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + functionParms.push_back(sptp); } void castDecimalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { - Item_decimal_typecast* idai = (Item_decimal_typecast*)ifp; - SPTP sptp; - sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->decimals))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - functionParms.push_back(sptp); + Item_decimal_typecast* idai = (Item_decimal_typecast*)ifp; + SPTP sptp; + sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->decimals))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + functionParms.push_back(sptp); - // max length including sign and/or decimal points - if (idai->decimals == 0) - sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 1))); - else - sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 2))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + // max length including sign and/or decimal points + if (idai->decimals == 0) + sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 1))); + else + sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 2))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - functionParms.push_back(sptp); + functionParms.push_back(sptp); } void castTypeArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { - Item_func_get_format* get_format = (Item_func_get_format*)ifp; - SPTP sptp; + Item_func_get_format* get_format = (Item_func_get_format*)ifp; + SPTP sptp; - if (get_format->type == MYSQL_TIMESTAMP_DATE) - sptp.reset(new ParseTree(new ConstantColumn("DATE"))); - else - sptp.reset(new ParseTree(new ConstantColumn("DATETIME"))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + if (get_format->type == MYSQL_TIMESTAMP_DATE) + sptp.reset(new ParseTree(new ConstantColumn("DATE"))); + else + sptp.reset(new ParseTree(new ConstantColumn("DATETIME"))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); - functionParms.push_back(sptp); + functionParms.push_back(sptp); } bool isSecondArgumentConstItem(Item_func* ifp) { - return (ifp->argument_count() == 2 && - ifp->arguments()[1]->type() == Item::CONST_ITEM); + return (ifp->argument_count() == 2 && ifp->arguments()[1]->type() == Item::CONST_ITEM); } // SELECT ... WHERE NOT IN (SELECT ); bool isNotFuncAndConstScalarSubSelect(Item_func* ifp, const std::string& funcName) { - return (ifp->with_subquery() && funcName == "not" && - ifp->argument_count() == 1 && - ifp->arguments()[0]->type() == Item::FUNC_ITEM && - std::string(((Item_func*)ifp->arguments()[0])->func_name()) == "=" && - isSecondArgumentConstItem((Item_func*)ifp->arguments()[0])); + return (ifp->with_subquery() && funcName == "not" && ifp->argument_count() == 1 && + ifp->arguments()[0]->type() == Item::FUNC_ITEM && + std::string(((Item_func*)ifp->arguments()[0])->func_name()) == "=" && + isSecondArgumentConstItem((Item_func*)ifp->arguments()[0])); } void gp_walk(const Item* item, void* arg) { - gp_walk_info* gwip = reinterpret_cast(arg); - idbassert(gwip); + gp_walk_info* gwip = reinterpret_cast(arg); + idbassert(gwip); - //Bailout... - if (gwip->fatalParseError) return; + // Bailout... + if (gwip->fatalParseError) + return; - RecursionCounter r(gwip); // Increments and auto-decrements upon exit. + RecursionCounter r(gwip); // Increments and auto-decrements upon exit. - Item::Type itype = item->type(); + Item::Type itype = item->type(); - // Allow to process XOR(which is Item_func) like other logical operators (which are Item_cond) - if (itype == Item::FUNC_ITEM && ((Item_func*)item)->functype() == Item_func::XOR_FUNC ) - itype = Item::COND_ITEM; + // Allow to process XOR(which is Item_func) like other logical operators (which are Item_cond) + if (itype == Item::FUNC_ITEM && ((Item_func*)item)->functype() == Item_func::XOR_FUNC) + itype = Item::COND_ITEM; - switch (itype) + switch (itype) + { + case Item::CACHE_ITEM: { - case Item::CACHE_ITEM: - { - // The item or condition is cached as per MariaDB server view but - // for InfiniDB it need to be parsed and executed. - // MCOL-1188 and MCOL-1029 - Item* orig_item = ((Item_cache*)item)->get_example(); - orig_item->traverse_cond(gp_walk, gwip, Item::POSTFIX); - break; - } - case Item::FIELD_ITEM: + // The item or condition is cached as per MariaDB server view but + // for InfiniDB it need to be parsed and executed. + // MCOL-1188 and MCOL-1029 + Item* orig_item = ((Item_cache*)item)->get_example(); + orig_item->traverse_cond(gp_walk, gwip, Item::POSTFIX); + break; + } + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + + if (ifp) + { + SimpleColumn* scp = buildSimpleColumn(ifp, *gwip); + + if (!scp) + break; + + string aliasTableName(scp->tableAlias()); + scp->tableAlias(aliasTableName); + gwip->rcWorkStack.push(scp->clone()); + boost::shared_ptr scsp(scp); + gwip->scsp = scsp; + + gwip->funcName.clear(); + gwip->columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), scsp)); + + //@bug4636 take where clause column as dummy projection column, but only on local column. + // varbinary aggregate is not supported yet, so rule it out + if (!((scp->joinInfo() & JOIN_CORRELATED) || + scp->colType().colDataType == CalpontSystemCatalog::VARBINARY)) { - Item_field* ifp = (Item_field*)item; - - if (ifp) - { - SimpleColumn* scp = buildSimpleColumn(ifp, *gwip); - - if (!scp) - break; - - string aliasTableName(scp->tableAlias()); - scp->tableAlias(aliasTableName); - gwip->rcWorkStack.push(scp->clone()); - boost::shared_ptr scsp(scp); - gwip->scsp = scsp; - - gwip->funcName.clear(); - gwip->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), scsp)); - - //@bug4636 take where clause column as dummy projection column, but only on local column. - // varbinary aggregate is not supported yet, so rule it out - if (!((scp->joinInfo() & JOIN_CORRELATED) || scp->colType().colDataType == CalpontSystemCatalog::VARBINARY)) - { - TABLE_LIST* tmp = (ifp->cached_table ? ifp->cached_table : 0); - gwip->tableMap[make_aliastable(scp->schemaName(), scp->tableName(), scp->tableAlias(), scp->isColumnStore())] = - make_pair(1, tmp); - } - } - - break; + TABLE_LIST* tmp = (ifp->cached_table ? ifp->cached_table : 0); + gwip->tableMap[make_aliastable(scp->schemaName(), scp->tableName(), scp->tableAlias(), + scp->isColumnStore())] = make_pair(1, tmp); } + } - case Item::CONST_ITEM: - { - switch(item->cmp_type()) - { - case INT_RESULT: - { - Item_int* iip = (Item_int*)item; - gwip->rcWorkStack.push(buildReturnedColumn(iip, *gwip, gwip->fatalParseError)); - break; - } - - case STRING_RESULT: - { - // Special handling for 0xHHHH literals - if (item->type_handler() == &type_handler_hex_hybrid) - { - Item_hex_hybrid *hip = reinterpret_cast(const_cast(item)); - gwip->rcWorkStack.push(new ConstantColumn((int64_t)hip->val_int(), ConstantColumn::NUM)); - ConstantColumn *cc = dynamic_cast(gwip->rcWorkStack.top()); - cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - break; - } - - Item_string* isp = (Item_string*)item; - - if (isp) - { - if (isp->result_type() == STRING_RESULT) - { - String val, *str = isp->val_str(&val); - string cval; - - if (str->ptr()) - { - cval.assign(str->ptr(), str->length()); - } - - gwip->rcWorkStack.push(new ConstantColumn(cval)); - (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - break; - } - - gwip->rcWorkStack.push(buildReturnedColumn(isp, *gwip, gwip->fatalParseError)); - } - break; - } - - case REAL_RESULT: - { - Item_float* ifp = (Item_float*)item; - gwip->rcWorkStack.push(buildReturnedColumn(ifp, *gwip, gwip->fatalParseError)); - break; - } - - case DECIMAL_RESULT: - { - Item_decimal* idp = (Item_decimal*)item; - gwip->rcWorkStack.push(buildReturnedColumn(idp, *gwip, gwip->fatalParseError)); - break; - } - - case TIME_RESULT: - { - Item_temporal_literal* itp = (Item_temporal_literal*)item; - gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); - break; - } - default: - { - if (gwip->condPush) - { - // push noop for unhandled item - SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - gwip->rcWorkStack.push(rc); - break; - } - - ostringstream oss; - oss << "Unhandled Item type: " << item->type(); - gwip->parseErrorText = oss.str(); - gwip->fatalParseError = true; - break; - } - } - break; - } - case Item::NULL_ITEM: - { - if (gwip->condPush) - { - // push noop for unhandled item - SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - gwip->rcWorkStack.push(rc); - break; - } - - gwip->rcWorkStack.push(new ConstantColumn("", ConstantColumn::NULLDATA)); - (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = (Item_func*)item; - string funcName = ifp->func_name(); - - if (!gwip->condPush) - { - if (!ifp->fixed()) - { - ifp->fix_fields(gwip->thd, reinterpret_cast(&ifp)); - } - - // Special handling for queries of the form: - // SELECT ... WHERE col1 NOT IN (SELECT ); - if (isNotFuncAndConstScalarSubSelect(ifp, funcName)) - { - idbassert(!gwip->ptWorkStack.empty()); - ParseTree* pt = gwip->ptWorkStack.top(); - SimpleFilter* sf = dynamic_cast(pt->data()); - - if (sf) - { - boost::shared_ptr sop(new PredicateOperator("<>")); - sf->op(sop); - return; - } - } - - // Do not call buildSubselectFunc() if the subquery is a const scalar - // subselect of the form: - // (SELECT ) - // As an example: SELECT col1 FROM t1 WHERE col2 = (SELECT 2); - if ((ifp->with_subquery() && !isSecondArgumentConstItem(ifp)) || - funcName == "") - { - buildSubselectFunc(ifp, gwip); - return; - } - - if (ifp->argument_count() > 0 && ifp->arguments()) - { - for (uint32_t i = 0; i < ifp->argument_count(); i++) - { - if (ifp->arguments()[i]->type() == Item::SUBSELECT_ITEM) - { - // This is probably NOT IN subquery with derived table in it. - // for some reason, MySQL has not fully optimized the plan at this point. - // noop here, and eventually MySQL will continue its optimization and get - // to rnd_init again. - if (ifp->functype() == Item_func::NOT_FUNC) - return; - - buildSubselectFunc(ifp, gwip); - return; - } - } - } - - if (ifp->functype() == Item_func::TRIG_COND_FUNC && gwip->subQuery) - { - gwip->subQuery->handleFunc(gwip, ifp); - break; - } - - // having clause null function added by MySQL - if (ifp->functype() == Item_func::ISNOTNULLTEST_FUNC) - { - // @bug 4215. remove the argument in rcWorkStack. - if (!gwip->rcWorkStack.empty()) - gwip->rcWorkStack.pop(); - - break; - } - } - - // try to evaluate const F&E - vector tmpVec; - uint16_t parseInfo = 0; - parse_item(ifp, tmpVec, gwip->fatalParseError, parseInfo, gwip); - - // table mode takes only one table filter - if (gwip->condPush) - { - set tableSet; - - for (uint32_t i = 0; i < tmpVec.size(); i++) - { - if (tmpVec[i]->table_name.str) - tableSet.insert(tmpVec[i]->table_name.str); - } - - if (tableSet.size() > 1) - break; - } - - if (!gwip->fatalParseError && !(parseInfo & AGG_BIT) && - !(parseInfo & SUB_BIT) && - !nonConstFunc(ifp) && - !(parseInfo & AF_BIT) && - tmpVec.size() == 0 && - ifp->functype() != Item_func::MULT_EQUAL_FUNC) - { - ValStrStdString valStr(ifp); - - ConstantColumn* cc = buildConstantColumnMaybeNullFromValStr(ifp, valStr, *gwip); - - for (uint32_t i = 0; i < ifp->argument_count() && !gwip->rcWorkStack.empty(); i++) - { - gwip->rcWorkStack.pop(); - } - - // bug 3137. If filter constant like 1=0, put it to ptWorkStack - // MariaDB bug 750. Breaks if compare is an argument to a function. -// if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top()) -// && isPredicateFunction(ifp, gwip)) - if (isPredicateFunction(ifp, gwip)) - gwip->ptWorkStack.push(new ParseTree(cc)); - else - gwip->rcWorkStack.push(cc); - - if (!valStr.isNull()) - IDEBUG( cerr << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); - - break; - } - - ReturnedColumn* rc = NULL; - - // @bug4488. Process function for table mode also, not just vtable mode. - rc = buildFunctionColumn(ifp, *gwip, gwip->fatalParseError); - - if (gwip->fatalParseError) - { - if (gwip->clauseType == SELECT) - return; - - // @bug 2585 - if (gwip->parseErrorText.empty()) - { - Message::Args args; - args.add(funcName); - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - } - - return; - } - - // predicate operators fall in the old path - if (rc) - { - // @bug 2383. For some reason func_name() for "in" gives " IN " always - if (funcName == "between" || funcName == "in" || funcName == " IN ") - gwip->ptWorkStack.push(new ParseTree(rc)); - else - gwip->rcWorkStack.push(rc); - } - else - { - // push to pt or rc stack is handled inside the function - buildPredicateItem(ifp, gwip); - } - - break; - } - - case Item::SUM_FUNC_ITEM: - { - Item_sum* isp = (Item_sum*)item; - ReturnedColumn* rc = buildAggregateColumn(isp, *gwip); - - if (rc) - gwip->rcWorkStack.push(rc); - - break; - } - - case Item::COND_ITEM: - { - // All logical functions are handled here, most of them are Item_cond, - // but XOR (it is Item_func_boolean2) - Item_func* func = (Item_func*)item; - - enum Item_func::Functype ftype = func->functype(); - bool isOr = (ftype == Item_func::COND_OR_FUNC); - bool isXor = (ftype == Item_func::XOR_FUNC); - - List* argumentList; - List xorArgumentList; - - if (isXor) - { - for (unsigned i = 0; i < func->argument_count(); i++) - { - xorArgumentList.push_back(func->arguments()[i]); - } - - argumentList = &xorArgumentList; - } - else - { - argumentList = ((Item_cond*)item)->argument_list(); - } - - - // @bug2932. if ptWorkStack contains less items than the condition's arguments, - // the missing one should be in the rcWorkStack, unless the it's subselect. - // @todo need to figure out a way to combine these two stacks while walking. - //if (gwip->ptWorkStack.size() < icp->argument_list()->elements) - { - List_iterator_fast li(*argumentList); - - while (Item* it = li++) - { - //@bug3495, @bug5865 error out non-supported OR with correlated subquery - if (isOr) - { - vector fieldVec; - uint16_t parseInfo = 0; - parse_item(it, fieldVec, gwip->fatalParseError, parseInfo, gwip); - - if (parseInfo & CORRELATED) - { - gwip->fatalParseError = true; - gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_CORRELATED_SUB_OR); - return; - } - } - - if ((it->type() == Item::FIELD_ITEM - || ( it->type() == Item::CONST_ITEM - && ( it->cmp_type() == INT_RESULT - || it->cmp_type() == DECIMAL_RESULT - || it->cmp_type() == STRING_RESULT - || it->cmp_type() == REAL_RESULT - ) - ) - || it->type() == Item::NULL_ITEM - || (it->type() == Item::FUNC_ITEM - && !isPredicateFunction(it, gwip))) && !gwip->rcWorkStack.empty() - ) - { - gwip->ptWorkStack.push(new ParseTree(gwip->rcWorkStack.top())); - gwip->rcWorkStack.pop(); - } - } - } - - // @bug1603. MySQL's filter tree is a multi-tree grouped by operator. So more than - // two filters saved on the stack so far might belong to this operator. - uint32_t leftInStack = gwip->ptWorkStack.size() - argumentList->elements + 1; - - while (true) - { - if (gwip->ptWorkStack.size() < 2) - break; - - ParseTree* lhs = gwip->ptWorkStack.top(); - gwip->ptWorkStack.pop(); - SimpleFilter* lsf = dynamic_cast(lhs->data()); - - if (lsf && lsf->op()->data() == "noop") - { - if (isOr) - { - gwip->parseErrorText = "Unhandled item in WHERE or HAVING clause"; - gwip->fatalParseError = true; - break; - } - else - continue; - } - - ParseTree* rhs = gwip->ptWorkStack.top(); - gwip->ptWorkStack.pop(); - SimpleFilter* rsf = dynamic_cast(rhs->data()); - - if (rsf && rsf->op()->data() == "noop") - { - if (isOr) - { - gwip->parseErrorText = "Unhandled item in WHERE or HAVING clause"; - gwip->fatalParseError = true; - break; - } - else - { - gwip->ptWorkStack.push(lhs); - continue; - } - } - - Operator* op = new LogicOperator(func->func_name()); - ParseTree* ptp = new ParseTree(op); - ptp->left(lhs); - ptp->right(rhs); - gwip->ptWorkStack.push(ptp); - - if (gwip->ptWorkStack.size() == leftInStack) - break; - } - - // special handling for subquery with aggregate. MySQL adds isnull function to the selected - // column. InfiniDB will remove it and set nullmatch flag if it's NOT_IN sub. - // @todo need more checking here to make sure it's not a user input OR operator - if (isOr && gwip->subQuery) - gwip->subQuery->handleFunc(gwip, func); - - break; - } - - case Item::REF_ITEM: - { - Item* col = *(((Item_ref*)item)->ref); - ReturnedColumn* rc = NULL; - // ref item is not pre-walked. force clause type to SELECT - ClauseType clauseType = gwip->clauseType; - gwip->clauseType = SELECT; - - if (col->type() != Item::COND_ITEM) - { - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, true); - - if ( col->type() == Item::FIELD_ITEM ) - gwip->fatalParseError = false; - } - - SimpleColumn* sc = dynamic_cast(rc); - - if (sc) - { - boost::shared_ptr scsp(sc->clone()); - gwip->scsp = scsp; - - if (col->type() == Item::FIELD_ITEM) - { - const auto &field_name = string(((Item_field*)item)->field_name.str); - auto colMap = CalpontSelectExecutionPlan::ColumnMap::value_type(field_name, scsp); - gwip->columnMap.insert(colMap); - } - } - - bool cando = true; - gwip->clauseType = clauseType; - - if (rc) - { - if (((Item_ref*)item)->depended_from) - { - rc->joinInfo(rc->joinInfo() | JOIN_CORRELATED); - - if (gwip->subQuery) - gwip->subQuery->correlated(true); - - SimpleColumn* scp = dynamic_cast(rc); - - if (scp) - gwip->correlatedTbNameVec.push_back(make_aliastable(scp->schemaName(), scp->tableName(), scp->tableAlias())); - - if (gwip->subSelectType == CalpontSelectExecutionPlan::SINGLEROW_SUBS) - rc->joinInfo(rc->joinInfo() | JOIN_SCALAR | JOIN_SEMI); - - if (gwip->subSelectType == CalpontSelectExecutionPlan::SELECT_SUBS) - rc->joinInfo(rc->joinInfo() | JOIN_SCALAR | JOIN_OUTER_SELECT); - } - - gwip->rcWorkStack.push(rc); - } - else if (col->type() == Item::FUNC_ITEM) - { - // sometimes mysql treat having filter items inconsistently. In such cases, - // which are always predicate operator, the function (gp_key>3) comes in as - // one item. - Item_func* ifp = (Item_func*)col; - - for (uint32_t i = 0; i < ifp->argument_count(); i++) - { - ReturnedColumn* operand = NULL; - - if (ifp->arguments()[i]->type() == Item::REF_ITEM) - { - Item* op = *(((Item_ref*)ifp->arguments()[i])->ref); - operand = buildReturnedColumn(op, *gwip, gwip->fatalParseError); - } - else - operand = buildReturnedColumn(ifp->arguments()[i], *gwip, gwip->fatalParseError); - - if (operand) - gwip->rcWorkStack.push(operand); - else - { - cando = false; - break; - } - } - - if (cando) - buildPredicateItem(ifp, gwip); - } - else if (col->type() == Item::COND_ITEM) - { - Item_func* ifp = (Item_func*)col; - gwip->ptWorkStack.push(buildParseTree(ifp, *gwip, gwip->fatalParseError)); - } - else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) - { - ReturnedColumn* rc = buildAggFrmTempField(const_cast(item), *gwip); - if (rc) - gwip->rcWorkStack.push(rc); - - break; - } - else - cando = false; - - if (!cando) - { - ostringstream oss; - oss << "Unhandled Item type: " << item->type(); - gwip->parseErrorText = oss.str(); - gwip->fatalParseError = true; - } - - break; - } - - case Item::SUBSELECT_ITEM: - { - if (gwip->condPush) // table mode - break; - - Item_subselect* sub = (Item_subselect*)item; - - if (sub->substype() == Item_subselect::EXISTS_SUBS) - { - SubQuery* orig = gwip->subQuery; - ExistsSub* existsSub = new ExistsSub(*gwip, sub); - gwip->hasSubSelect = true; - gwip->subQuery = existsSub; - gwip->ptWorkStack.push(existsSub->transform()); - // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = true; // only temp. bypass the 2nd phase. - // recover original - gwip->subQuery = orig; - gwip->lastSub = existsSub; - } - else if (sub->substype() == Item_subselect::IN_SUBS) - { - if (!((Item_in_subselect*)sub)->optimizer && gwip->thd->derived_tables_processing) - { - ostringstream oss; - oss << "Invalid In_optimizer: " << item->type(); - gwip->parseErrorText = oss.str(); - gwip->fatalParseError = true; - break; - } - } - - // store a dummy subselect object. the transform is handled in item_func. - SubSelect* subselect = new SubSelect(); - gwip->rcWorkStack.push(subselect); - break; - } - - case Item::ROW_ITEM: - { - Item_row* row = (Item_row*)item; - RowColumn* rowCol = new RowColumn(); - vector cols; - // temp change clause type because the elements of row column are not walked yet - gwip->clauseType = SELECT; - - for (uint32_t i = 0; i < row->cols(); i++) - cols.push_back(SRCP(buildReturnedColumn(row->element_index(i), *gwip, gwip->fatalParseError))); - - gwip->clauseType = WHERE; - rowCol->columnVec(cols); - gwip->rcWorkStack.push(rowCol); - break; - } - - case Item::EXPR_CACHE_ITEM: - { - ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(gp_walk, arg, Item::POSTFIX); - break; - } - - - case Item::WINDOW_FUNC_ITEM: - { - gwip->hasWindowFunc = true; - Item_window_func* ifa = (Item_window_func*)item; - ReturnedColumn* af = buildWindowFunctionColumn(ifa, *gwip, gwip->fatalParseError); - - if (af) - gwip->rcWorkStack.push(af); - - break; - } - - case Item::COPY_STR_ITEM: - printf("********** received COPY_STR_ITEM *********\n"); - break; - - case Item::FIELD_AVG_ITEM: - printf("********** received FIELD_AVG_ITEM *********\n"); - break; - - case Item::DEFAULT_VALUE_ITEM: - printf("********** received DEFAULT_VALUE_ITEM *********\n"); - break; - - case Item::PROC_ITEM: - printf("********** received PROC_ITEM *********\n"); - break; - - case Item::FIELD_STD_ITEM: - printf("********** received FIELD_STD_ITEM *********\n"); - break; - - case Item::FIELD_VARIANCE_ITEM: - printf("********** received FIELD_VARIANCE_ITEM *********\n"); - break; - - case Item::INSERT_VALUE_ITEM: - printf("********** received INSERT_VALUE_ITEM *********\n"); - break; - - case Item::PARAM_ITEM: - printf("********** received PARAM_ITEM *********\n"); - break; - - case Item::TRIGGER_FIELD_ITEM: - printf("********** received TRIGGER_FIELD_ITEM *********\n"); - break; - - case Item::TYPE_HOLDER: - std::cerr << "********** received TYPE_HOLDER *********" << std::endl; - break; - default: - { - if (gwip->condPush) - { - // push noop for unhandled item - SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); - gwip->rcWorkStack.push(rc); - break; - } - - ostringstream oss; - oss << "Unhandled Item type: " << item->type(); - gwip->parseErrorText = oss.str(); - gwip->fatalParseError = true; - break; - } + break; } - return; + case Item::CONST_ITEM: + { + switch (item->cmp_type()) + { + case INT_RESULT: + { + Item_int* iip = (Item_int*)item; + gwip->rcWorkStack.push(buildReturnedColumn(iip, *gwip, gwip->fatalParseError)); + break; + } + + case STRING_RESULT: + { + // Special handling for 0xHHHH literals + if (item->type_handler() == &type_handler_hex_hybrid) + { + Item_hex_hybrid* hip = reinterpret_cast(const_cast(item)); + gwip->rcWorkStack.push(new ConstantColumn((int64_t)hip->val_int(), ConstantColumn::NUM)); + ConstantColumn* cc = dynamic_cast(gwip->rcWorkStack.top()); + cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + break; + } + + Item_string* isp = (Item_string*)item; + + if (isp) + { + if (isp->result_type() == STRING_RESULT) + { + String val, *str = isp->val_str(&val); + string cval; + + if (str->ptr()) + { + cval.assign(str->ptr(), str->length()); + } + + gwip->rcWorkStack.push(new ConstantColumn(cval)); + (dynamic_cast(gwip->rcWorkStack.top())) + ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + break; + } + + gwip->rcWorkStack.push(buildReturnedColumn(isp, *gwip, gwip->fatalParseError)); + } + break; + } + + case REAL_RESULT: + { + Item_float* ifp = (Item_float*)item; + gwip->rcWorkStack.push(buildReturnedColumn(ifp, *gwip, gwip->fatalParseError)); + break; + } + + case DECIMAL_RESULT: + { + Item_decimal* idp = (Item_decimal*)item; + gwip->rcWorkStack.push(buildReturnedColumn(idp, *gwip, gwip->fatalParseError)); + break; + } + + case TIME_RESULT: + { + Item_temporal_literal* itp = (Item_temporal_literal*)item; + gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); + break; + } + default: + { + if (gwip->condPush) + { + // push noop for unhandled item + SimpleColumn* rc = new SimpleColumn("noop"); + rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + gwip->rcWorkStack.push(rc); + break; + } + + ostringstream oss; + oss << "Unhandled Item type: " << item->type(); + gwip->parseErrorText = oss.str(); + gwip->fatalParseError = true; + break; + } + } + break; + } + case Item::NULL_ITEM: + { + if (gwip->condPush) + { + // push noop for unhandled item + SimpleColumn* rc = new SimpleColumn("noop"); + rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + gwip->rcWorkStack.push(rc); + break; + } + + gwip->rcWorkStack.push(new ConstantColumn("", ConstantColumn::NULLDATA)); + (dynamic_cast(gwip->rcWorkStack.top())) + ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)item; + string funcName = ifp->func_name(); + + if (!gwip->condPush) + { + if (!ifp->fixed()) + { + ifp->fix_fields(gwip->thd, reinterpret_cast(&ifp)); + } + + // Special handling for queries of the form: + // SELECT ... WHERE col1 NOT IN (SELECT ); + if (isNotFuncAndConstScalarSubSelect(ifp, funcName)) + { + idbassert(!gwip->ptWorkStack.empty()); + ParseTree* pt = gwip->ptWorkStack.top(); + SimpleFilter* sf = dynamic_cast(pt->data()); + + if (sf) + { + boost::shared_ptr sop(new PredicateOperator("<>")); + sf->op(sop); + return; + } + } + + // Do not call buildSubselectFunc() if the subquery is a const scalar + // subselect of the form: + // (SELECT ) + // As an example: SELECT col1 FROM t1 WHERE col2 = (SELECT 2); + if ((ifp->with_subquery() && !isSecondArgumentConstItem(ifp)) || funcName == "") + { + buildSubselectFunc(ifp, gwip); + return; + } + + if (ifp->argument_count() > 0 && ifp->arguments()) + { + for (uint32_t i = 0; i < ifp->argument_count(); i++) + { + if (ifp->arguments()[i]->type() == Item::SUBSELECT_ITEM) + { + // This is probably NOT IN subquery with derived table in it. + // for some reason, MySQL has not fully optimized the plan at this point. + // noop here, and eventually MySQL will continue its optimization and get + // to rnd_init again. + if (ifp->functype() == Item_func::NOT_FUNC) + return; + + buildSubselectFunc(ifp, gwip); + return; + } + } + } + + if (ifp->functype() == Item_func::TRIG_COND_FUNC && gwip->subQuery) + { + gwip->subQuery->handleFunc(gwip, ifp); + break; + } + + // having clause null function added by MySQL + if (ifp->functype() == Item_func::ISNOTNULLTEST_FUNC) + { + // @bug 4215. remove the argument in rcWorkStack. + if (!gwip->rcWorkStack.empty()) + gwip->rcWorkStack.pop(); + + break; + } + } + + // try to evaluate const F&E + vector tmpVec; + uint16_t parseInfo = 0; + parse_item(ifp, tmpVec, gwip->fatalParseError, parseInfo, gwip); + + // table mode takes only one table filter + if (gwip->condPush) + { + set tableSet; + + for (uint32_t i = 0; i < tmpVec.size(); i++) + { + if (tmpVec[i]->table_name.str) + tableSet.insert(tmpVec[i]->table_name.str); + } + + if (tableSet.size() > 1) + break; + } + + if (!gwip->fatalParseError && !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) && !nonConstFunc(ifp) && + !(parseInfo & AF_BIT) && tmpVec.size() == 0 && ifp->functype() != Item_func::MULT_EQUAL_FUNC) + { + ValStrStdString valStr(ifp); + + ConstantColumn* cc = buildConstantColumnMaybeNullFromValStr(ifp, valStr, *gwip); + + for (uint32_t i = 0; i < ifp->argument_count() && !gwip->rcWorkStack.empty(); i++) + { + gwip->rcWorkStack.pop(); + } + + // bug 3137. If filter constant like 1=0, put it to ptWorkStack + // MariaDB bug 750. Breaks if compare is an argument to a function. + // if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 + //: gwip->rcBookMarkStack.top()) + // && isPredicateFunction(ifp, gwip)) + if (isPredicateFunction(ifp, gwip)) + gwip->ptWorkStack.push(new ParseTree(cc)); + else + gwip->rcWorkStack.push(cc); + + if (!valStr.isNull()) + IDEBUG(cerr << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl); + + break; + } + + ReturnedColumn* rc = NULL; + + // @bug4488. Process function for table mode also, not just vtable mode. + rc = buildFunctionColumn(ifp, *gwip, gwip->fatalParseError); + + if (gwip->fatalParseError) + { + if (gwip->clauseType == SELECT) + return; + + // @bug 2585 + if (gwip->parseErrorText.empty()) + { + Message::Args args; + args.add(funcName); + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + } + + return; + } + + // predicate operators fall in the old path + if (rc) + { + // @bug 2383. For some reason func_name() for "in" gives " IN " always + if (funcName == "between" || funcName == "in" || funcName == " IN ") + gwip->ptWorkStack.push(new ParseTree(rc)); + else + gwip->rcWorkStack.push(rc); + } + else + { + // push to pt or rc stack is handled inside the function + buildPredicateItem(ifp, gwip); + } + + break; + } + + case Item::SUM_FUNC_ITEM: + { + Item_sum* isp = (Item_sum*)item; + ReturnedColumn* rc = buildAggregateColumn(isp, *gwip); + + if (rc) + gwip->rcWorkStack.push(rc); + + break; + } + + case Item::COND_ITEM: + { + // All logical functions are handled here, most of them are Item_cond, + // but XOR (it is Item_func_boolean2) + Item_func* func = (Item_func*)item; + + enum Item_func::Functype ftype = func->functype(); + bool isOr = (ftype == Item_func::COND_OR_FUNC); + bool isXor = (ftype == Item_func::XOR_FUNC); + + List* argumentList; + List xorArgumentList; + + if (isXor) + { + for (unsigned i = 0; i < func->argument_count(); i++) + { + xorArgumentList.push_back(func->arguments()[i]); + } + + argumentList = &xorArgumentList; + } + else + { + argumentList = ((Item_cond*)item)->argument_list(); + } + + // @bug2932. if ptWorkStack contains less items than the condition's arguments, + // the missing one should be in the rcWorkStack, unless the it's subselect. + // @todo need to figure out a way to combine these two stacks while walking. + // if (gwip->ptWorkStack.size() < icp->argument_list()->elements) + { + List_iterator_fast li(*argumentList); + + while (Item* it = li++) + { + //@bug3495, @bug5865 error out non-supported OR with correlated subquery + if (isOr) + { + vector fieldVec; + uint16_t parseInfo = 0; + parse_item(it, fieldVec, gwip->fatalParseError, parseInfo, gwip); + + if (parseInfo & CORRELATED) + { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_CORRELATED_SUB_OR); + return; + } + } + + if ((it->type() == Item::FIELD_ITEM || + (it->type() == Item::CONST_ITEM && + (it->cmp_type() == INT_RESULT || it->cmp_type() == DECIMAL_RESULT || + it->cmp_type() == STRING_RESULT || it->cmp_type() == REAL_RESULT)) || + it->type() == Item::NULL_ITEM || + (it->type() == Item::FUNC_ITEM && !isPredicateFunction(it, gwip))) && + !gwip->rcWorkStack.empty()) + { + gwip->ptWorkStack.push(new ParseTree(gwip->rcWorkStack.top())); + gwip->rcWorkStack.pop(); + } + } + } + + // @bug1603. MySQL's filter tree is a multi-tree grouped by operator. So more than + // two filters saved on the stack so far might belong to this operator. + uint32_t leftInStack = gwip->ptWorkStack.size() - argumentList->elements + 1; + + while (true) + { + if (gwip->ptWorkStack.size() < 2) + break; + + ParseTree* lhs = gwip->ptWorkStack.top(); + gwip->ptWorkStack.pop(); + SimpleFilter* lsf = dynamic_cast(lhs->data()); + + if (lsf && lsf->op()->data() == "noop") + { + if (isOr) + { + gwip->parseErrorText = "Unhandled item in WHERE or HAVING clause"; + gwip->fatalParseError = true; + break; + } + else + continue; + } + + ParseTree* rhs = gwip->ptWorkStack.top(); + gwip->ptWorkStack.pop(); + SimpleFilter* rsf = dynamic_cast(rhs->data()); + + if (rsf && rsf->op()->data() == "noop") + { + if (isOr) + { + gwip->parseErrorText = "Unhandled item in WHERE or HAVING clause"; + gwip->fatalParseError = true; + break; + } + else + { + gwip->ptWorkStack.push(lhs); + continue; + } + } + + Operator* op = new LogicOperator(func->func_name()); + ParseTree* ptp = new ParseTree(op); + ptp->left(lhs); + ptp->right(rhs); + gwip->ptWorkStack.push(ptp); + + if (gwip->ptWorkStack.size() == leftInStack) + break; + } + + // special handling for subquery with aggregate. MySQL adds isnull function to the selected + // column. InfiniDB will remove it and set nullmatch flag if it's NOT_IN sub. + // @todo need more checking here to make sure it's not a user input OR operator + if (isOr && gwip->subQuery) + gwip->subQuery->handleFunc(gwip, func); + + break; + } + + case Item::REF_ITEM: + { + Item* col = *(((Item_ref*)item)->ref); + ReturnedColumn* rc = NULL; + // ref item is not pre-walked. force clause type to SELECT + ClauseType clauseType = gwip->clauseType; + gwip->clauseType = SELECT; + + if (col->type() != Item::COND_ITEM) + { + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, true); + + if (col->type() == Item::FIELD_ITEM) + gwip->fatalParseError = false; + } + + SimpleColumn* sc = dynamic_cast(rc); + + if (sc) + { + boost::shared_ptr scsp(sc->clone()); + gwip->scsp = scsp; + + if (col->type() == Item::FIELD_ITEM) + { + const auto& field_name = string(((Item_field*)item)->field_name.str); + auto colMap = CalpontSelectExecutionPlan::ColumnMap::value_type(field_name, scsp); + gwip->columnMap.insert(colMap); + } + } + + bool cando = true; + gwip->clauseType = clauseType; + + if (rc) + { + if (((Item_ref*)item)->depended_from) + { + rc->joinInfo(rc->joinInfo() | JOIN_CORRELATED); + + if (gwip->subQuery) + gwip->subQuery->correlated(true); + + SimpleColumn* scp = dynamic_cast(rc); + + if (scp) + gwip->correlatedTbNameVec.push_back( + make_aliastable(scp->schemaName(), scp->tableName(), scp->tableAlias())); + + if (gwip->subSelectType == CalpontSelectExecutionPlan::SINGLEROW_SUBS) + rc->joinInfo(rc->joinInfo() | JOIN_SCALAR | JOIN_SEMI); + + if (gwip->subSelectType == CalpontSelectExecutionPlan::SELECT_SUBS) + rc->joinInfo(rc->joinInfo() | JOIN_SCALAR | JOIN_OUTER_SELECT); + } + + gwip->rcWorkStack.push(rc); + } + else if (col->type() == Item::FUNC_ITEM) + { + // sometimes mysql treat having filter items inconsistently. In such cases, + // which are always predicate operator, the function (gp_key>3) comes in as + // one item. + Item_func* ifp = (Item_func*)col; + + for (uint32_t i = 0; i < ifp->argument_count(); i++) + { + ReturnedColumn* operand = NULL; + + if (ifp->arguments()[i]->type() == Item::REF_ITEM) + { + Item* op = *(((Item_ref*)ifp->arguments()[i])->ref); + operand = buildReturnedColumn(op, *gwip, gwip->fatalParseError); + } + else + operand = buildReturnedColumn(ifp->arguments()[i], *gwip, gwip->fatalParseError); + + if (operand) + gwip->rcWorkStack.push(operand); + else + { + cando = false; + break; + } + } + + if (cando) + buildPredicateItem(ifp, gwip); + } + else if (col->type() == Item::COND_ITEM) + { + Item_func* ifp = (Item_func*)col; + gwip->ptWorkStack.push(buildParseTree(ifp, *gwip, gwip->fatalParseError)); + } + else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) + { + ReturnedColumn* rc = buildAggFrmTempField(const_cast(item), *gwip); + if (rc) + gwip->rcWorkStack.push(rc); + + break; + } + else + cando = false; + + if (!cando) + { + ostringstream oss; + oss << "Unhandled Item type: " << item->type(); + gwip->parseErrorText = oss.str(); + gwip->fatalParseError = true; + } + + break; + } + + case Item::SUBSELECT_ITEM: + { + if (gwip->condPush) // table mode + break; + + Item_subselect* sub = (Item_subselect*)item; + + if (sub->substype() == Item_subselect::EXISTS_SUBS) + { + SubQuery* orig = gwip->subQuery; + ExistsSub* existsSub = new ExistsSub(*gwip, sub); + gwip->hasSubSelect = true; + gwip->subQuery = existsSub; + gwip->ptWorkStack.push(existsSub->transform()); + // MCOL-2178 isUnion member only assigned, never used + // MIGR::infinidb_vtable.isUnion = true; // only temp. bypass the 2nd phase. + // recover original + gwip->subQuery = orig; + gwip->lastSub = existsSub; + } + else if (sub->substype() == Item_subselect::IN_SUBS) + { + if (!((Item_in_subselect*)sub)->optimizer && gwip->thd->derived_tables_processing) + { + ostringstream oss; + oss << "Invalid In_optimizer: " << item->type(); + gwip->parseErrorText = oss.str(); + gwip->fatalParseError = true; + break; + } + } + + // store a dummy subselect object. the transform is handled in item_func. + SubSelect* subselect = new SubSelect(); + gwip->rcWorkStack.push(subselect); + break; + } + + case Item::ROW_ITEM: + { + Item_row* row = (Item_row*)item; + RowColumn* rowCol = new RowColumn(); + vector cols; + // temp change clause type because the elements of row column are not walked yet + gwip->clauseType = SELECT; + + for (uint32_t i = 0; i < row->cols(); i++) + cols.push_back(SRCP(buildReturnedColumn(row->element_index(i), *gwip, gwip->fatalParseError))); + + gwip->clauseType = WHERE; + rowCol->columnVec(cols); + gwip->rcWorkStack.push(rowCol); + break; + } + + case Item::EXPR_CACHE_ITEM: + { + ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(gp_walk, arg, Item::POSTFIX); + break; + } + + case Item::WINDOW_FUNC_ITEM: + { + gwip->hasWindowFunc = true; + Item_window_func* ifa = (Item_window_func*)item; + ReturnedColumn* af = buildWindowFunctionColumn(ifa, *gwip, gwip->fatalParseError); + + if (af) + gwip->rcWorkStack.push(af); + + break; + } + + case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; + + case Item::FIELD_AVG_ITEM: printf("********** received FIELD_AVG_ITEM *********\n"); break; + + case Item::DEFAULT_VALUE_ITEM: printf("********** received DEFAULT_VALUE_ITEM *********\n"); break; + + case Item::PROC_ITEM: printf("********** received PROC_ITEM *********\n"); break; + + case Item::FIELD_STD_ITEM: printf("********** received FIELD_STD_ITEM *********\n"); break; + + case Item::FIELD_VARIANCE_ITEM: printf("********** received FIELD_VARIANCE_ITEM *********\n"); break; + + case Item::INSERT_VALUE_ITEM: printf("********** received INSERT_VALUE_ITEM *********\n"); break; + + case Item::PARAM_ITEM: printf("********** received PARAM_ITEM *********\n"); break; + + case Item::TRIGGER_FIELD_ITEM: printf("********** received TRIGGER_FIELD_ITEM *********\n"); break; + + case Item::TYPE_HOLDER: std::cerr << "********** received TYPE_HOLDER *********" << std::endl; break; + default: + { + if (gwip->condPush) + { + // push noop for unhandled item + SimpleColumn* rc = new SimpleColumn("noop"); + rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + gwip->rcWorkStack.push(rc); + break; + } + + ostringstream oss; + oss << "Unhandled Item type: " << item->type(); + gwip->parseErrorText = oss.str(); + gwip->fatalParseError = true; + break; + } + } + + return; } /** @info this function recursivly walks an item's arguments and push all * the involved item_fields to the passed in vector. It's used in parsing * functions or arithmetic expressions for vtable post process. */ -void parse_item (Item* item, vector& field_vec, - bool& hasNonSupportItem, - uint16_t& parseInfo, - gp_walk_info* gwi) +void parse_item(Item* item, vector& field_vec, bool& hasNonSupportItem, uint16_t& parseInfo, + gp_walk_info* gwi) { - Item::Type itype = item->type(); + Item::Type itype = item->type(); - switch (itype) + switch (itype) + { + case Item::FIELD_ITEM: { - case Item::FIELD_ITEM: - { - Item_field* ifp = reinterpret_cast(item); - field_vec.push_back(ifp); - return; - } - - case Item::SUM_FUNC_ITEM: - { - //hasAggColumn = true; - parseInfo |= AGG_BIT; - Item_sum* isp = reinterpret_cast(item); - Item** sfitempp = isp->arguments(); - - for (uint32_t i = 0; i < isp->argument_count(); i++) - parse_item(sfitempp[i], field_vec, hasNonSupportItem, parseInfo, gwi); - - break; - } - - case Item::FUNC_ITEM: - { - Item_func* isp = reinterpret_cast(item); - - if (string(isp->func_name()) == "") - { - parseInfo |= SUB_BIT; - parseInfo |= CORRELATED; - break; - } - - for (uint32_t i = 0; i < isp->argument_count(); i++) - parse_item(isp->arguments()[i], field_vec, hasNonSupportItem, parseInfo, gwi); - - break; - } - - case Item::COND_ITEM: - { - Item_cond* icp = reinterpret_cast(item); - List_iterator_fast it(*(icp->argument_list())); - Item* cond_item; - - while ((cond_item = it++)) - parse_item(cond_item, field_vec, hasNonSupportItem, parseInfo, gwi); - - break; - } - - case Item::REF_ITEM: - { - while (true) - { - Item_ref* ref = (Item_ref*)item; - - if ((*(ref->ref))->type() == Item::SUM_FUNC_ITEM) - { - parseInfo |= AGG_BIT; - Item_sum* isp = reinterpret_cast(*(ref->ref)); - Item** sfitempp = isp->arguments(); - - // special handling for count(*). This should not be treated as constant. - if (isSupportedAggregateWithOneConstArg(isp, sfitempp)) - { - field_vec.push_back((Item_field*)item); //dummy - } - - for (uint32_t i = 0; i < isp->argument_count(); i++) - parse_item(sfitempp[i], field_vec, hasNonSupportItem, parseInfo, gwi); - - break; - } - else if ((*(ref->ref))->type() == Item::FIELD_ITEM) - { - // MCOL-1510. This could be a non-supported function - // argument in form of a temp_table_field, so check - // and set hasNonSupportItem if it is so. - ReturnedColumn* rc = NULL; - if (gwi) - rc = buildAggFrmTempField(ref, *gwi); - - if (!rc) - { - Item_field* ifp = reinterpret_cast(*(ref->ref)); - field_vec.push_back(ifp); - } - break; - } - else if ((*(ref->ref))->type() == Item::FUNC_ITEM) - { - Item_func* isp = reinterpret_cast(*(ref->ref)); - Item** sfitempp = isp->arguments(); - - for (uint32_t i = 0; i < isp->argument_count(); i++) - parse_item(sfitempp[i], field_vec, hasNonSupportItem, parseInfo, gwi); - - break; - } - else if ((*(ref->ref))->type() == Item::CACHE_ITEM) - { - Item_cache* isp = reinterpret_cast(*(ref->ref)); - parse_item(isp->get_example(), field_vec, hasNonSupportItem, parseInfo, gwi); - break; - } - else if ((*(ref->ref))->type() == Item::REF_ITEM) - { - item = (*(ref->ref)); - continue; - } - else if ((*(ref->ref))->type() == Item::WINDOW_FUNC_ITEM) - { - parseInfo |= AF_BIT; - break; - } - else - { - cerr << "UNKNOWN REF Item" << endl; - break; - } - } - - break; - } - - case Item::SUBSELECT_ITEM: - { - parseInfo |= SUB_BIT; - Item_subselect* sub = (Item_subselect*)item; - - if (sub->is_correlated) - parseInfo |= CORRELATED; - - break; - } - - case Item::ROW_ITEM: - { - Item_row* row = (Item_row*)item; - - for (uint32_t i = 0; i < row->cols(); i++) - parse_item(row->element_index(i), field_vec, hasNonSupportItem, parseInfo, gwi); - - break; - } - - case Item::EXPR_CACHE_ITEM: - { - // item is a Item_cache_wrapper. Shouldn't get here. - // DRRTUY TODO Why - IDEBUG(std::cerr << "EXPR_CACHE_ITEM in parse_item\n" << std::endl); - gwi->fatalParseError = true; - // DRRTUY The questionable error text. I've seen - // ERR_CORRELATED_SUB_OR - string parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SUB_QUERY_TYPE); - setError(gwi->thd, ER_CHECK_NOT_IMPLEMENTED, parseErrorText); - break; - } - - case Item::WINDOW_FUNC_ITEM: - parseInfo |= AF_BIT; - break; - - default: - break; + Item_field* ifp = reinterpret_cast(item); + field_vec.push_back(ifp); + return; } + + case Item::SUM_FUNC_ITEM: + { + // hasAggColumn = true; + parseInfo |= AGG_BIT; + Item_sum* isp = reinterpret_cast(item); + Item** sfitempp = isp->arguments(); + + for (uint32_t i = 0; i < isp->argument_count(); i++) + parse_item(sfitempp[i], field_vec, hasNonSupportItem, parseInfo, gwi); + + break; + } + + case Item::FUNC_ITEM: + { + Item_func* isp = reinterpret_cast(item); + + if (string(isp->func_name()) == "") + { + parseInfo |= SUB_BIT; + parseInfo |= CORRELATED; + break; + } + + for (uint32_t i = 0; i < isp->argument_count(); i++) + parse_item(isp->arguments()[i], field_vec, hasNonSupportItem, parseInfo, gwi); + + break; + } + + case Item::COND_ITEM: + { + Item_cond* icp = reinterpret_cast(item); + List_iterator_fast it(*(icp->argument_list())); + Item* cond_item; + + while ((cond_item = it++)) + parse_item(cond_item, field_vec, hasNonSupportItem, parseInfo, gwi); + + break; + } + + case Item::REF_ITEM: + { + while (true) + { + Item_ref* ref = (Item_ref*)item; + + if ((*(ref->ref))->type() == Item::SUM_FUNC_ITEM) + { + parseInfo |= AGG_BIT; + Item_sum* isp = reinterpret_cast(*(ref->ref)); + Item** sfitempp = isp->arguments(); + + // special handling for count(*). This should not be treated as constant. + if (isSupportedAggregateWithOneConstArg(isp, sfitempp)) + { + field_vec.push_back((Item_field*)item); // dummy + } + + for (uint32_t i = 0; i < isp->argument_count(); i++) + parse_item(sfitempp[i], field_vec, hasNonSupportItem, parseInfo, gwi); + + break; + } + else if ((*(ref->ref))->type() == Item::FIELD_ITEM) + { + // MCOL-1510. This could be a non-supported function + // argument in form of a temp_table_field, so check + // and set hasNonSupportItem if it is so. + ReturnedColumn* rc = NULL; + if (gwi) + rc = buildAggFrmTempField(ref, *gwi); + + if (!rc) + { + Item_field* ifp = reinterpret_cast(*(ref->ref)); + field_vec.push_back(ifp); + } + break; + } + else if ((*(ref->ref))->type() == Item::FUNC_ITEM) + { + Item_func* isp = reinterpret_cast(*(ref->ref)); + Item** sfitempp = isp->arguments(); + + for (uint32_t i = 0; i < isp->argument_count(); i++) + parse_item(sfitempp[i], field_vec, hasNonSupportItem, parseInfo, gwi); + + break; + } + else if ((*(ref->ref))->type() == Item::CACHE_ITEM) + { + Item_cache* isp = reinterpret_cast(*(ref->ref)); + parse_item(isp->get_example(), field_vec, hasNonSupportItem, parseInfo, gwi); + break; + } + else if ((*(ref->ref))->type() == Item::REF_ITEM) + { + item = (*(ref->ref)); + continue; + } + else if ((*(ref->ref))->type() == Item::WINDOW_FUNC_ITEM) + { + parseInfo |= AF_BIT; + break; + } + else + { + cerr << "UNKNOWN REF Item" << endl; + break; + } + } + + break; + } + + case Item::SUBSELECT_ITEM: + { + parseInfo |= SUB_BIT; + Item_subselect* sub = (Item_subselect*)item; + + if (sub->is_correlated) + parseInfo |= CORRELATED; + + break; + } + + case Item::ROW_ITEM: + { + Item_row* row = (Item_row*)item; + + for (uint32_t i = 0; i < row->cols(); i++) + parse_item(row->element_index(i), field_vec, hasNonSupportItem, parseInfo, gwi); + + break; + } + + case Item::EXPR_CACHE_ITEM: + { + // item is a Item_cache_wrapper. Shouldn't get here. + // DRRTUY TODO Why + IDEBUG(std::cerr << "EXPR_CACHE_ITEM in parse_item\n" << std::endl); + gwi->fatalParseError = true; + // DRRTUY The questionable error text. I've seen + // ERR_CORRELATED_SUB_OR + string parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SUB_QUERY_TYPE); + setError(gwi->thd, ER_CHECK_NOT_IMPLEMENTED, parseErrorText); + break; + } + + case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; + + default: break; + } } bool isMCSTable(TABLE* table_ptr) { #if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX) - if (!(table_ptr->s && (*table_ptr->s->db_plugin)->name.str)) + if (!(table_ptr->s && (*table_ptr->s->db_plugin)->name.str)) #else - if (!(table_ptr->s && (table_ptr->s->db_plugin)->name.str)) + if (!(table_ptr->s && (table_ptr->s->db_plugin)->name.str)) #endif - return true; + return true; #if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX) - string engineName = (*table_ptr->s->db_plugin)->name.str; + string engineName = (*table_ptr->s->db_plugin)->name.str; #else - string engineName = table_ptr->s->db_plugin->name.str; + string engineName = table_ptr->s->db_plugin->name.str; #endif - if (engineName == "Columnstore" || - engineName == "Columnstore_cache") - return true; - else - return false; + if (engineName == "Columnstore" || engineName == "Columnstore_cache") + return true; + else + return false; } bool isForeignTableUpdate(THD* thd) { - LEX* lex = thd->lex; - - if (!isUpdateStatement(lex->sql_command)) - return false; - - Item_field* item; - List_iterator_fast field_it(lex->first_select_lex()->item_list); - - while ((item = (Item_field*) field_it++)) - { - if (item->field && item->field->table && !isMCSTable(item->field->table)) - return true; - } + LEX* lex = thd->lex; + if (!isUpdateStatement(lex->sql_command)) return false; + + Item_field* item; + List_iterator_fast field_it(lex->first_select_lex()->item_list); + + while ((item = (Item_field*)field_it++)) + { + if (item->field && item->field->table && !isMCSTable(item->field->table)) + return true; + } + + return false; } bool isMCSTableUpdate(THD* thd) { - LEX* lex = thd->lex; - - if (!isUpdateStatement(lex->sql_command)) - return false; - - Item_field* item; - List_iterator_fast field_it(lex->first_select_lex()->item_list); - - while ((item = (Item_field*) field_it++)) - { - if (item->field && item->field->table && isMCSTable(item->field->table)) - return true; - } + LEX* lex = thd->lex; + if (!isUpdateStatement(lex->sql_command)) return false; + + Item_field* item; + List_iterator_fast field_it(lex->first_select_lex()->item_list); + + while ((item = (Item_field*)field_it++)) + { + if (item->field && item->field->table && isMCSTable(item->field->table)) + return true; + } + + return false; } bool isMCSTableDelete(THD* thd) { - LEX* lex = thd->lex; - - if (!isDeleteStatement(lex->sql_command)) - return false; - - TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list(); - - if (table_ptr && table_ptr->table && isMCSTable(table_ptr->table)) - return true; + LEX* lex = thd->lex; + if (!isDeleteStatement(lex->sql_command)) return false; + + TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list(); + + if (table_ptr && table_ptr->table && isMCSTable(table_ptr->table)) + return true; + + return false; } // This function is different from isForeignTableUpdate() @@ -6602,20 +6448,20 @@ bool isMCSTableDelete(THD* thd) // foreign table or not, as in isForeignTableUpdate(). bool isUpdateHasForeignTable(THD* thd) { - LEX* lex = thd->lex; - - if (!isUpdateStatement(lex->sql_command)) - return false; - - TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list(); - - for (; table_ptr; table_ptr = table_ptr->next_local) - { - if (table_ptr->table && !isMCSTable(table_ptr->table)) - return true; - } + LEX* lex = thd->lex; + if (!isUpdateStatement(lex->sql_command)) return false; + + TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list(); + + for (; table_ptr; table_ptr = table_ptr->next_local) + { + if (table_ptr->table && !isMCSTable(table_ptr->table)) + return true; + } + + return false; } /*@brief set some runtime params to run the query */ @@ -6624,30 +6470,30 @@ bool isUpdateHasForeignTable(THD* thd) * This function just sets a number of runtime params that * limits resource consumed. ***********************************************************/ -void setExecutionParams(gp_walk_info &gwi, SCSEP &csep) +void setExecutionParams(gp_walk_info& gwi, SCSEP& csep) { - gwi.internalDecimalScale = (get_use_decimal_scale(gwi.thd) ? get_decimal_scale(gwi.thd) : -1); - // @bug 2123. Override large table estimate if infinidb_ordered hint was used. - // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. - if (get_ordered_only(gwi.thd)) - csep->overrideLargeSideEstimate(true); + gwi.internalDecimalScale = (get_use_decimal_scale(gwi.thd) ? get_decimal_scale(gwi.thd) : -1); + // @bug 2123. Override large table estimate if infinidb_ordered hint was used. + // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. + if (get_ordered_only(gwi.thd)) + csep->overrideLargeSideEstimate(true); - // @bug 5741. Set a flag when in Local PM only query mode - csep->localQuery(get_local_query(gwi.thd)); + // @bug 5741. Set a flag when in Local PM only query mode + csep->localQuery(get_local_query(gwi.thd)); - // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering - csep->stringScanThreshold(get_string_scan_threshold(gwi.thd)); + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(get_string_scan_threshold(gwi.thd)); - csep->stringTableThreshold(get_stringtable_threshold(gwi.thd)); + csep->stringTableThreshold(get_stringtable_threshold(gwi.thd)); - csep->djsSmallSideLimit(get_diskjoin_smallsidelimit(gwi.thd) * 1024ULL * 1024); - csep->djsLargeSideLimit(get_diskjoin_largesidelimit(gwi.thd) * 1024ULL * 1024); - csep->djsPartitionSize(get_diskjoin_bucketsize(gwi.thd) * 1024ULL * 1024); + csep->djsSmallSideLimit(get_diskjoin_smallsidelimit(gwi.thd) * 1024ULL * 1024); + csep->djsLargeSideLimit(get_diskjoin_largesidelimit(gwi.thd) * 1024ULL * 1024); + csep->djsPartitionSize(get_diskjoin_bucketsize(gwi.thd) * 1024ULL * 1024); - if (get_um_mem_limit(gwi.thd) == 0) - csep->umMemLimit(numeric_limits::max()); - else - csep->umMemLimit(get_um_mem_limit(gwi.thd) * 1024ULL * 1024); + if (get_um_mem_limit(gwi.thd) == 0) + csep->umMemLimit(numeric_limits::max()); + else + csep->umMemLimit(get_um_mem_limit(gwi.thd) * 1024ULL * 1024); } /*@brief Process FROM part of the query or sub-query */ @@ -6660,180 +6506,183 @@ void setExecutionParams(gp_walk_info &gwi, SCSEP &csep) * RETURNS * error id as an int ***********************************************************/ -int processFrom(bool &isUnion, - SELECT_LEX &select_lex, - gp_walk_info &gwi, - SCSEP &csep) +int processFrom(bool& isUnion, SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep) { - // populate table map and trigger syscolumn cache for all the tables (@bug 1637). - // all tables on FROM list must have at least one col in colmap - TABLE_LIST* table_ptr = select_lex.get_table_list(); + // populate table map and trigger syscolumn cache for all the tables (@bug 1637). + // all tables on FROM list must have at least one col in colmap + TABLE_LIST* table_ptr = select_lex.get_table_list(); #ifdef DEBUG_WALK_COND - List_iterator sj_list_it(select_lex.sj_nests); - TABLE_LIST* sj_nest; + List_iterator sj_list_it(select_lex.sj_nests); + TABLE_LIST* sj_nest; - while ((sj_nest = sj_list_it++)) - { - cerr << sj_nest->db.str << "." << sj_nest->table_name.str << endl; - } + while ((sj_nest = sj_list_it++)) + { + cerr << sj_nest->db.str << "." << sj_nest->table_name.str << endl; + } #endif - try + try + { + for (; table_ptr; table_ptr = table_ptr->next_local) { - for (; table_ptr; table_ptr = table_ptr->next_local) + // Until we handle recursive cte: + // Checking here ensures we catch all with clauses in the query. + if (table_ptr->is_recursive_with_table()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Recursive CTE"; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + string viewName = getViewName(table_ptr); + if (lower_case_table_names) + { + boost::algorithm::to_lower(viewName); + } + + // @todo process from subquery + if (table_ptr->derived) + { + SELECT_LEX* select_cursor = table_ptr->derived->first_select(); + FromSubQuery fromSub(gwi, select_cursor); + string alias(table_ptr->alias.str); + if (lower_case_table_names) { - // Until we handle recursive cte: - // Checking here ensures we catch all with clauses in the query. - if (table_ptr->is_recursive_with_table()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Recursive CTE"; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } + boost::algorithm::to_lower(alias); + } + fromSub.alias(alias); - string viewName = getViewName(table_ptr); - if (lower_case_table_names) - { - boost::algorithm::to_lower(viewName); - } + CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); + // @bug 3852. check return execplan + SCSEP plan = fromSub.transform(); - // @todo process from subquery - if (table_ptr->derived) - { - SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor); - string alias(table_ptr->alias.str); - if (lower_case_table_names) - { - boost::algorithm::to_lower(alias); - } - fromSub.alias(alias); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); - // @bug 3852. check return execplan - SCSEP plan = fromSub.transform(); - - if (!plan) - { - setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); - return ER_INTERNAL_ERROR; - } - - gwi.derivedTbList.push_back(plan); - gwi.tbList.push_back(tn); - CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); - gwi.tableMap[tan] = make_pair(0, table_ptr); - // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init - } - else if (table_ptr->view) - { - View* view = new View(*table_ptr->view->first_select_lex(), &gwi); - CalpontSystemCatalog::TableAliasName tn = make_aliastable(table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, true, lower_case_table_names); - view->viewName(tn); - gwi.viewList.push_back(view); - view->transform(); - } - else - { - // check foreign engine tables - bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true); - - // trigger system catalog cache - if (columnStore) - gwi.csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str, lower_case_table_names), true); - - string table_name = table_ptr->table_name.str; - - // @bug5523 - if (table_ptr->db.length && strcmp(table_ptr->db.str, "information_schema") == 0) - table_name = (table_ptr->schema_table_name.length ? table_ptr->schema_table_name.str : table_ptr->alias.str); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db.str, table_name, table_ptr->alias.str, viewName, columnStore, lower_case_table_names); - gwi.tbList.push_back(tn); - CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db.str, table_name, table_ptr->alias.str, columnStore, lower_case_table_names); - gwi.tableMap[tan] = make_pair(0, table_ptr); -#ifdef DEBUG_WALK_COND - cerr << tn << endl; -#endif - } + if (!plan) + { + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); + return ER_INTERNAL_ERROR; } - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - } - catch (IDBExcept& ie) - { - setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); - // @bug 3852. set error status for gwi. - gwi.fatalParseError = true; - gwi.parseErrorText = ie.what(); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - // @bug3852 set error status for gwi. - gwi.fatalParseError = true; - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); - return ER_INTERNAL_ERROR; - } - - csep->tableList(gwi.tbList); - - // Send this recursively to getSelectPlan - bool unionSel = false; - // UNION master unit check - // Existed pushdown handlers won't get in this scope - // except UNION pushdown that is to come. - // is_unit_op() give a segv for derived_handler's SELECT_LEX - if (!isUnion && select_lex.master_unit()->is_unit_op()) - { + gwi.derivedTbList.push_back(plan); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); + gwi.tableMap[tan] = make_pair(0, table_ptr); // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = true; - CalpontSelectExecutionPlan::SelectList unionVec; - SELECT_LEX* select_cursor = select_lex.master_unit()->first_select(); - unionSel = true; - uint8_t distUnionNum = 0; + // MIGR::infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + } + else if (table_ptr->view) + { + View* view = new View(*table_ptr->view->first_select_lex(), &gwi); + CalpontSystemCatalog::TableAliasName tn = make_aliastable( + table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, true, lower_case_table_names); + view->viewName(tn); + gwi.viewList.push_back(view); + view->transform(); + } + else + { + // check foreign engine tables + bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true); - for (SELECT_LEX* sl = select_cursor; sl; sl = sl->next_select()) - { - SCSEP plan(new CalpontSelectExecutionPlan()); - plan->txnID(csep->txnID()); - plan->verID(csep->verID()); - plan->sessionID(csep->sessionID()); - plan->traceFlags(csep->traceFlags()); - plan->data(csep->data()); + // trigger system catalog cache + if (columnStore) + gwi.csc->columnRIDs( + make_table(table_ptr->db.str, table_ptr->table_name.str, lower_case_table_names), true); - // gwi for the union unit - gp_walk_info union_gwi; - union_gwi.thd = gwi.thd; - uint32_t err = 0; + string table_name = table_ptr->table_name.str; - if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel)) != 0) - return err; + // @bug5523 + if (table_ptr->db.length && strcmp(table_ptr->db.str, "information_schema") == 0) + table_name = + (table_ptr->schema_table_name.length ? table_ptr->schema_table_name.str : table_ptr->alias.str); - unionVec.push_back(SCEP(plan)); - - // distinct union num - if (sl == select_lex.master_unit()->union_distinct) - distUnionNum = unionVec.size(); - } - - csep->unionVec(unionVec); - csep->distinctUnionNum(distUnionNum); + CalpontSystemCatalog::TableAliasName tn = + make_aliasview(table_ptr->db.str, table_name, table_ptr->alias.str, viewName, columnStore, + lower_case_table_names); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable( + table_ptr->db.str, table_name, table_ptr->alias.str, columnStore, lower_case_table_names); + gwi.tableMap[tan] = make_pair(0, table_ptr); +#ifdef DEBUG_WALK_COND + cerr << tn << endl; +#endif + } } - return 0; + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + catch (IDBExcept& ie) + { + setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); + // @bug 3852. set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = ie.what(); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + // @bug3852 set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); + return ER_INTERNAL_ERROR; + } + + csep->tableList(gwi.tbList); + + // Send this recursively to getSelectPlan + bool unionSel = false; + // UNION master unit check + // Existed pushdown handlers won't get in this scope + // except UNION pushdown that is to come. + // is_unit_op() give a segv for derived_handler's SELECT_LEX + if (!isUnion && select_lex.master_unit()->is_unit_op()) + { + // MCOL-2178 isUnion member only assigned, never used + // MIGR::infinidb_vtable.isUnion = true; + CalpontSelectExecutionPlan::SelectList unionVec; + SELECT_LEX* select_cursor = select_lex.master_unit()->first_select(); + unionSel = true; + uint8_t distUnionNum = 0; + + for (SELECT_LEX* sl = select_cursor; sl; sl = sl->next_select()) + { + SCSEP plan(new CalpontSelectExecutionPlan()); + plan->txnID(csep->txnID()); + plan->verID(csep->verID()); + plan->sessionID(csep->sessionID()); + plan->traceFlags(csep->traceFlags()); + plan->data(csep->data()); + + // gwi for the union unit + gp_walk_info union_gwi; + union_gwi.thd = gwi.thd; + uint32_t err = 0; + + if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel)) != 0) + return err; + + unionVec.push_back(SCEP(plan)); + + // distinct union num + if (sl == select_lex.master_unit()->union_distinct) + distUnionNum = unionVec.size(); + } + + csep->unionVec(unionVec); + csep->distinctUnionNum(distUnionNum); + } + + return 0; } /*@brief Process WHERE part of the query or sub-query */ @@ -6844,266 +6693,264 @@ int processFrom(bool &isUnion, * RETURNS * error id as an int ***********************************************************/ -int processWhere(SELECT_LEX &select_lex, - gp_walk_info &gwi, - SCSEP &csep, - const std::vector& condStack) +int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const std::vector& condStack) { - JOIN* join = select_lex.join; - Item* icp = 0; - bool isUpdateDelete = false; + JOIN* join = select_lex.join; + Item* icp = 0; + bool isUpdateDelete = false; - // Flag to indicate if this is a prepared statement - bool isPS = gwi.thd->stmt_arena && gwi.thd->stmt_arena->is_stmt_execute(); + // Flag to indicate if this is a prepared statement + bool isPS = gwi.thd->stmt_arena && gwi.thd->stmt_arena->is_stmt_execute(); - if (join != 0 && !isPS) - icp = join->conds; - else if (isPS && select_lex.prep_where) - icp = select_lex.prep_where; + if (join != 0 && !isPS) + icp = join->conds; + else if (isPS && select_lex.prep_where) + icp = select_lex.prep_where; - // if icp is null, try to find the where clause other where - if (!join && gwi.thd->lex->derived_tables) + // if icp is null, try to find the where clause other where + if (!join && gwi.thd->lex->derived_tables) + { + if (select_lex.prep_where) + icp = select_lex.prep_where; + else if (select_lex.where) + icp = select_lex.where; + } + else if (!join && isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) + { + isUpdateDelete = true; + } + + if (icp) + { + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". + //@bug 3039. fix fields for constants + if (!icp->fixed()) { - if (select_lex.prep_where) - icp = select_lex.prep_where; - else if (select_lex.where) - icp = select_lex.where; - } - else if (!join && isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { - isUpdateDelete = true; + icp->fix_fields(gwi.thd, (Item**)&icp); } - if (icp) - { - // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". - //@bug 3039. fix fields for constants - if (!icp->fixed()) - { - icp->fix_fields(gwi.thd, (Item**)&icp); - } - - gwi.fatalParseError = false; + gwi.fatalParseError = false; #ifdef DEBUG_WALK_COND - std::cerr << "------------------ WHERE -----------------------" << std::endl; - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - std::cerr << "------------------------------------------------\n" << std::endl; + std::cerr << "------------------ WHERE -----------------------" << std::endl; + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + std::cerr << "------------------------------------------------\n" << std::endl; #endif - icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + return setErrorAndReturn(gwi); + } + } + else if (isUpdateDelete) + { + // MCOL-4023 For updates/deletes, we iterate over the pushed down condStack + if (!condStack.empty()) + { + std::vector::const_iterator condStackIter = condStack.begin(); + + while (condStackIter != condStack.end()) + { + COND* cond = *condStackIter++; + + cond->traverse_cond(gp_walk, &gwi, Item::POSTFIX); if (gwi.fatalParseError) { - return setErrorAndReturn(gwi); + return setErrorAndReturn(gwi); } + } } - else if (isUpdateDelete) + // if condStack is empty(), check the select_lex for where conditions + // as a last resort + else if ((icp = select_lex.where) != 0) { - // MCOL-4023 For updates/deletes, we iterate over the pushed down condStack - if (!condStack.empty()) - { - std::vector::const_iterator condStackIter = condStack.begin(); + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - while (condStackIter != condStack.end()) - { - COND* cond = *condStackIter++; - - cond->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - return setErrorAndReturn(gwi); - } - } - } - // if condStack is empty(), check the select_lex for where conditions - // as a last resort - else if ((icp = select_lex.where) != 0) - { - icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - return setErrorAndReturn(gwi); - } - } + if (gwi.fatalParseError) + { + return setErrorAndReturn(gwi); + } } - else if (join && join->zero_result_cause) + } + else if (join && join->zero_result_cause) + { + gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + (dynamic_cast(gwi.rcWorkStack.top())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + } + + for (Item* item : gwi.condList) + { + if (item && (item != icp)) { - gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + item->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + return setErrorAndReturn(gwi); + } } + } - for (Item* item : gwi.condList) - { - if (item && (item != icp)) - { - item->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - return setErrorAndReturn(gwi); - } - } - } - - // ZZ - the followinig debug shows the structure of nested outer join. should - // use a recursive function. + // ZZ - the followinig debug shows the structure of nested outer join. should + // use a recursive function. #ifdef OUTER_JOIN_DEBUG - List* tables = &(select_lex.top_join_list); - List_iterator_fast ti(*tables); + List* tables = &(select_lex.top_join_list); + List_iterator_fast ti(*tables); - TABLE_LIST **table= (TABLE_LIST **)gwi.thd->alloc(sizeof(TABLE_LIST*) * tables->elements); - for (TABLE_LIST **t= table + (tables->elements - 1); t >= table; t--) - *t= ti++; + TABLE_LIST** table = (TABLE_LIST**)gwi.thd->alloc(sizeof(TABLE_LIST*) * tables->elements); + for (TABLE_LIST** t = table + (tables->elements - 1); t >= table; t--) + *t = ti++; - DBUG_ASSERT(tables->elements >= 1); + DBUG_ASSERT(tables->elements >= 1); - TABLE_LIST **end= table + tables->elements; - for (TABLE_LIST **tbl= table; tbl < end; tbl++) + TABLE_LIST** end = table + tables->elements; + for (TABLE_LIST** tbl = table; tbl < end; tbl++) + { + TABLE_LIST* curr; + + while ((curr = ti++)) { - TABLE_LIST* curr; + TABLE_LIST* curr = *tbl; - while ((curr = ti++)) + if (curr->table_name.length) + cerr << curr->table_name.str << " "; + else + cerr << curr->alias.str << endl; + + if (curr->outer_join) + cerr << " is inner table" << endl; + else if (curr->straight) + cerr << "straight_join" << endl; + else + cerr << "join" << endl; + + if (curr->nested_join) + { + List* inners = &(curr->nested_join->join_list); + List_iterator_fast li(*inners); + TABLE_LIST** inner = (TABLE_LIST**)gwi.thd->alloc(sizeof(TABLE_LIST*) * inners->elements); + + for (TABLE_LIST** t = inner + (inners->elements - 1); t >= inner; t--) + *t = li++; + + TABLE_LIST** end1 = inner + inners->elements; + + for (TABLE_LIST** tb = inner; tb < end1; tb++) { - TABLE_LIST* curr = *tbl; + TABLE_LIST* curr1 = *tb; + cerr << curr1->alias.str << endl; - if (curr->table_name.length) - cerr << curr->table_name.str << " "; - else - cerr << curr->alias.str << endl; - - if (curr->outer_join) - cerr << " is inner table" << endl; - else if (curr->straight) - cerr << "straight_join" << endl; - else - cerr << "join" << endl; - - if (curr->nested_join) - { - List* inners = &(curr->nested_join->join_list); - List_iterator_fast li(*inners); - TABLE_LIST** inner = (TABLE_LIST**)gwi.thd->alloc(sizeof(TABLE_LIST*) * inners->elements); - - for (TABLE_LIST** t = inner + (inners->elements - 1); t >= inner; t--) - *t = li++; - - TABLE_LIST** end1 = inner + inners->elements; - - for (TABLE_LIST** tb = inner; tb < end1; tb++) - { - TABLE_LIST* curr1 = *tb; - cerr << curr1->alias.str << endl; - - if (curr1->sj_on_expr) - { - curr1->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } - } - } - - if (curr->sj_on_expr) - { - curr->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } + if (curr1->sj_on_expr) + { + curr1->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } } + } + + if (curr->sj_on_expr) + { + curr->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } } + } #endif - uint32_t failed = 0; + uint32_t failed = 0; - // InfiniDB bug5764 requires outer joins to be appended to the - // end of the filter list. This causes outer join filters to - // have a higher join id than inner join filters. - // TODO MCOL-4680 Figure out why this is the case, and possibly - // eliminate this requirement. - std::stack outerJoinStack; + // InfiniDB bug5764 requires outer joins to be appended to the + // end of the filter list. This causes outer join filters to + // have a higher join id than inner join filters. + // TODO MCOL-4680 Figure out why this is the case, and possibly + // eliminate this requirement. + std::stack outerJoinStack; - if ((failed = buildJoin(gwi, select_lex.top_join_list, outerJoinStack))) + if ((failed = buildJoin(gwi, select_lex.top_join_list, outerJoinStack))) + return failed; + + if (gwi.subQuery) + { + for (uint i = 0; i < gwi.viewList.size(); i++) + { + if ((failed = gwi.viewList[i]->processJoin(gwi, outerJoinStack))) return failed; - - if (gwi.subQuery) - { - for (uint i = 0; i < gwi.viewList.size(); i++) - { - if ((failed = gwi.viewList[i]->processJoin(gwi, outerJoinStack))) - return failed; - } } + } - ParseTree* filters = NULL; - ParseTree* outerJoinFilters = NULL; - ParseTree* ptp = NULL; - ParseTree* rhs = NULL; + ParseTree* filters = NULL; + ParseTree* outerJoinFilters = NULL; + ParseTree* ptp = NULL; + ParseTree* rhs = NULL; - // @bug 2932. for "select * from region where r_name" case. if icp not null and - // ptWorkStack empty, the item is in rcWorkStack. - // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 - // is pushed to rcWorkStack. - if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) - { - filters = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } + // @bug 2932. for "select * from region where r_name" case. if icp not null and + // ptWorkStack empty, the item is in rcWorkStack. + // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 + // is pushed to rcWorkStack. + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + filters = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } - while (!gwi.ptWorkStack.empty()) - { - filters = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); + while (!gwi.ptWorkStack.empty()) + { + filters = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); - if (gwi.ptWorkStack.empty()) - break; + if (gwi.ptWorkStack.empty()) + break; - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - rhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - ptp->right(rhs); - gwi.ptWorkStack.push(ptp); - } + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } - while (!outerJoinStack.empty()) - { - outerJoinFilters = outerJoinStack.top(); - outerJoinStack.pop(); + while (!outerJoinStack.empty()) + { + outerJoinFilters = outerJoinStack.top(); + outerJoinStack.pop(); - if (outerJoinStack.empty()) - break; + if (outerJoinStack.empty()) + break; - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(outerJoinFilters); - rhs = outerJoinStack.top(); - outerJoinStack.pop(); - ptp->right(rhs); - outerJoinStack.push(ptp); - } + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(outerJoinFilters); + rhs = outerJoinStack.top(); + outerJoinStack.pop(); + ptp->right(rhs); + outerJoinStack.push(ptp); + } - // Append outer join filters at the end of inner join filters. - // JLF_ExecPlanToJobList::walkTree processes ParseTree::left - // before ParseTree::right which is what we intend to do in the - // below. - if (filters && outerJoinFilters) - { - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - ptp->right(outerJoinFilters); - filters = ptp; - } - else if (outerJoinFilters) - { - filters = outerJoinFilters; - } + // Append outer join filters at the end of inner join filters. + // JLF_ExecPlanToJobList::walkTree processes ParseTree::left + // before ParseTree::right which is what we intend to do in the + // below. + if (filters && outerJoinFilters) + { + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + ptp->right(outerJoinFilters); + filters = ptp; + } + else if (outerJoinFilters) + { + filters = outerJoinFilters; + } - if (filters) - { - csep->filters(filters); - std::string aTmpDir(startup::StartUp::tmpDir()); - aTmpDir = aTmpDir + "/filter1.dot"; - filters->drawTree(aTmpDir); - } + if (filters) + { + csep->filters(filters); + std::string aTmpDir(startup::StartUp::tmpDir()); + aTmpDir = aTmpDir + "/filter1.dot"; + filters->drawTree(aTmpDir); + } - return 0; + return 0; } /*@brief Process LIMIT part of a query or sub-query */ @@ -7113,126 +6960,121 @@ int processWhere(SELECT_LEX &select_lex, * RETURNS * error id as an int ***********************************************************/ -int processLimitAndOffset( - SELECT_LEX& select_lex, - gp_walk_info& gwi, - SCSEP& csep, - bool unionSel, - bool isUnion, - bool isSelectHandlerTop -) +int processLimitAndOffset(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, bool unionSel, bool isUnion, + bool isSelectHandlerTop) { - // LIMIT processing part - uint64_t limitNum = std::numeric_limits::max(); + // LIMIT processing part + uint64_t limitNum = std::numeric_limits::max(); - // non-MAIN union branch - if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + // non-MAIN union branch + if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + /* Consider the following query: + "select a from t1 where exists (select b from t2 where a=b);" + CS first builds a hash table for t2, then pushes down the hash to + PrimProc for a distributed hash join execution, with t1 being the + large-side table. However, the server applies an optimization in + Item_exists_subselect::fix_length_and_dec in sql/item_subselect.cc + (see server commit ae476868a5394041a00e75a29c7d45917e8dfae8) + where it sets explicit_limit to true, which causes csep->limitNum set to 1. + This causes the hash table for t2 to only contain a single record for the + hash join, giving less number of rows in the output result set than expected. + We therefore do not allow limit set to 1 here for such queries. + */ + if (gwi.subSelectType != CalpontSelectExecutionPlan::IN_SUBS && + gwi.subSelectType != CalpontSelectExecutionPlan::EXISTS_SUBS && + select_lex.master_unit()->global_parameters()->limit_params.explicit_limit) { - /* Consider the following query: - "select a from t1 where exists (select b from t2 where a=b);" - CS first builds a hash table for t2, then pushes down the hash to - PrimProc for a distributed hash join execution, with t1 being the - large-side table. However, the server applies an optimization in - Item_exists_subselect::fix_length_and_dec in sql/item_subselect.cc - (see server commit ae476868a5394041a00e75a29c7d45917e8dfae8) - where it sets explicit_limit to true, which causes csep->limitNum set to 1. - This causes the hash table for t2 to only contain a single record for the - hash join, giving less number of rows in the output result set than expected. - We therefore do not allow limit set to 1 here for such queries. - */ - if (gwi.subSelectType != CalpontSelectExecutionPlan::IN_SUBS - && gwi.subSelectType != CalpontSelectExecutionPlan::EXISTS_SUBS - && select_lex.master_unit()->global_parameters()->limit_params.explicit_limit) - { - if (select_lex.master_unit()->global_parameters()->limit_params.offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.offset_limit; - csep->limitStart(offset->val_int()); - } + if (select_lex.master_unit()->global_parameters()->limit_params.offset_limit) + { + Item_int* offset = + (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.offset_limit; + csep->limitStart(offset->val_int()); + } - if (select_lex.master_unit()->global_parameters()->limit_params.select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.select_limit; - csep->limitNum(select->val_int()); - // MCOL-894 Activate parallel ORDER BY - csep->orderByThreads(get_orderby_threads(gwi.thd)); - } - } + if (select_lex.master_unit()->global_parameters()->limit_params.select_limit) + { + Item_int* select = + (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.select_limit; + csep->limitNum(select->val_int()); + // MCOL-894 Activate parallel ORDER BY + csep->orderByThreads(get_orderby_threads(gwi.thd)); + } } - // union with explicit select at the top level - else if (isUnion && select_lex.limit_params.explicit_limit) + } + // union with explicit select at the top level + else if (isUnion && select_lex.limit_params.explicit_limit) + { + if (select_lex.braces) { - if (select_lex.braces) - { - if (select_lex.limit_params.offset_limit) - csep->limitStart(((Item_int*)select_lex.limit_params.offset_limit)->val_int()); + if (select_lex.limit_params.offset_limit) + csep->limitStart(((Item_int*)select_lex.limit_params.offset_limit)->val_int()); - if (select_lex.limit_params.select_limit) - csep->limitNum(((Item_int*)select_lex.limit_params.select_limit)->val_int()); - } + if (select_lex.limit_params.select_limit) + csep->limitNum(((Item_int*)select_lex.limit_params.select_limit)->val_int()); } - // other types of queries that have explicit LIMIT - else if (select_lex.limit_params.explicit_limit) + } + // other types of queries that have explicit LIMIT + else if (select_lex.limit_params.explicit_limit) + { + uint32_t limitOffset = 0; + + if (select_lex.join) { - uint32_t limitOffset = 0; + JOIN* join = select_lex.join; - if (select_lex.join) - { - JOIN* join = select_lex.join; - - // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer - // (not null though) and will cause seg fault. Prefer checking - // select_lex->limit_params.offset_limit if not null. - if (join->select_lex && - join->select_lex->limit_params.offset_limit && - join->select_lex->limit_params.offset_limit->fixed() && - join->select_lex->limit_params.select_limit && - join->select_lex->limit_params.select_limit->fixed()) - { - limitOffset = join->select_lex->limit_params.offset_limit->val_int(); - limitNum = join->select_lex->limit_params.select_limit->val_int(); - } - else if (join->unit) - { - limitOffset = join->unit->lim.get_offset_limit(); - limitNum = join->unit->lim.get_select_limit() - limitOffset; - } - } - else - { - if (select_lex.master_unit()->global_parameters()->limit_params.offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.offset_limit; - limitOffset = offset->val_int(); - } - - if (select_lex.master_unit()->global_parameters()->limit_params.select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.select_limit; - limitNum = select->val_int(); - } - } - - csep->limitStart(limitOffset); - csep->limitNum(limitNum); + // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer + // (not null though) and will cause seg fault. Prefer checking + // select_lex->limit_params.offset_limit if not null. + if (join->select_lex && join->select_lex->limit_params.offset_limit && + join->select_lex->limit_params.offset_limit->fixed() && + join->select_lex->limit_params.select_limit && join->select_lex->limit_params.select_limit->fixed()) + { + limitOffset = join->select_lex->limit_params.offset_limit->val_int(); + limitNum = join->select_lex->limit_params.select_limit->val_int(); + } + else if (join->unit) + { + limitOffset = join->unit->lim.get_offset_limit(); + limitNum = join->unit->lim.get_select_limit() - limitOffset; + } } - // If an explicit limit is not specified, use the system variable value else { - csep->limitNum(gwi.thd->variables.select_limit); + if (select_lex.master_unit()->global_parameters()->limit_params.offset_limit) + { + Item_int* offset = + (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.offset_limit; + limitOffset = offset->val_int(); + } + + if (select_lex.master_unit()->global_parameters()->limit_params.select_limit) + { + Item_int* select = + (Item_int*)select_lex.master_unit()->global_parameters()->limit_params.select_limit; + limitNum = select->val_int(); + } } - // We don't currently support limit with correlated subquery - if (csep->limitNum() != (uint64_t) - 1 && - gwi.subQuery && !gwi.correlatedTbNameVec.empty()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } + csep->limitStart(limitOffset); + csep->limitNum(limitNum); + } + // If an explicit limit is not specified, use the system variable value + else + { + csep->limitNum(gwi.thd->variables.select_limit); + } - return 0; + // We don't currently support limit with correlated subquery + if (csep->limitNum() != (uint64_t)-1 && gwi.subQuery && !gwi.correlatedTbNameVec.empty()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + return 0; } /*@brief Create in-to-exists predicate for an IN subquery */ @@ -7252,106 +7094,98 @@ int processLimitAndOffset( ***********************************************************/ void buildInToExistsFilter(gp_walk_info& gwi, SELECT_LEX& select_lex) { - RowColumn* rlhs = dynamic_cast(gwi.inSubQueryLHS); + RowColumn* rlhs = dynamic_cast(gwi.inSubQueryLHS); - size_t additionalRetColsBefore = gwi.additionalRetCols.size(); + size_t additionalRetColsBefore = gwi.additionalRetCols.size(); - if (rlhs) + if (rlhs) + { + idbassert(gwi.inSubQueryLHSItem->type() == Item::ROW_ITEM); + + Item_row* row = (Item_row*)gwi.inSubQueryLHSItem; + + idbassert(!rlhs->columnVec().empty() && (rlhs->columnVec().size() == gwi.returnedCols.size()) && + row->cols() && (row->cols() == select_lex.item_list.elements) && + (row->cols() == gwi.returnedCols.size())); + + List_iterator_fast it(select_lex.item_list); + Item* item; + + int i = 0; + + ParseTree* rowFilter = nullptr; + + while ((item = it++)) { - idbassert(gwi.inSubQueryLHSItem->type() == Item::ROW_ITEM); + boost::shared_ptr sop(new PredicateOperator("=")); + vector itemList = {row->element_index(i), item}; + ReturnedColumn* rhs = gwi.returnedCols[i]->clone(); - Item_row* row = (Item_row*)gwi.inSubQueryLHSItem; - - idbassert(!rlhs->columnVec().empty() && - (rlhs->columnVec().size() == gwi.returnedCols.size()) && - row->cols() && (row->cols() == select_lex.item_list.elements) && - (row->cols() == gwi.returnedCols.size())); - - List_iterator_fast it(select_lex.item_list); - Item* item; - - int i = 0; - - ParseTree* rowFilter = nullptr; - - while ((item = it++)) - { - boost::shared_ptr sop(new PredicateOperator("=")); - vector itemList = {row->element_index(i), item}; - ReturnedColumn* rhs = gwi.returnedCols[i]->clone(); - - buildEqualityPredicate(rlhs->columnVec()[i]->clone(), rhs, &gwi, sop, - Item_func::EQ_FUNC, itemList, true); - - if (gwi.fatalParseError) - { - delete rlhs; - return; - } - - ParseTree* tmpFilter = nullptr; - - if (!gwi.ptWorkStack.empty()) - { - tmpFilter = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - } - - if (i == 0 && tmpFilter) - { - rowFilter = tmpFilter; - } - else if (i != 0 && tmpFilter && rowFilter) - { - ParseTree* ptp = new ParseTree(new LogicOperator("and")); - ptp->left(rowFilter); - ptp->right(tmpFilter); - rowFilter = ptp; - } - - i++; - } + buildEqualityPredicate(rlhs->columnVec()[i]->clone(), rhs, &gwi, sop, Item_func::EQ_FUNC, itemList, + true); + if (gwi.fatalParseError) + { delete rlhs; + return; + } - if (rowFilter) - gwi.ptWorkStack.push(rowFilter); + ParseTree* tmpFilter = nullptr; + + if (!gwi.ptWorkStack.empty()) + { + tmpFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + } + + if (i == 0 && tmpFilter) + { + rowFilter = tmpFilter; + } + else if (i != 0 && tmpFilter && rowFilter) + { + ParseTree* ptp = new ParseTree(new LogicOperator("and")); + ptp->left(rowFilter); + ptp->right(tmpFilter); + rowFilter = ptp; + } + + i++; } - else + + delete rlhs; + + if (rowFilter) + gwi.ptWorkStack.push(rowFilter); + } + else + { + idbassert((gwi.returnedCols.size() == 1) && (select_lex.item_list.elements == 1)); + + boost::shared_ptr sop(new PredicateOperator("=")); + vector itemList = {gwi.inSubQueryLHSItem, select_lex.item_list.head()}; + ReturnedColumn* rhs = gwi.returnedCols[0]->clone(); + buildEqualityPredicate(gwi.inSubQueryLHS, rhs, &gwi, sop, Item_func::EQ_FUNC, itemList, true); + + if (gwi.fatalParseError) + return; + } + + size_t additionalRetColsAdded = gwi.additionalRetCols.size() - additionalRetColsBefore; + + if (gwi.returnedCols.size() && (gwi.returnedCols.size() == additionalRetColsAdded)) + { + for (size_t i = 0; i < gwi.returnedCols.size(); i++) { - idbassert((gwi.returnedCols.size() == 1) && - (select_lex.item_list.elements == 1)); - - boost::shared_ptr sop(new PredicateOperator("=")); - vector itemList = {gwi.inSubQueryLHSItem, - select_lex.item_list.head()}; - ReturnedColumn* rhs = gwi.returnedCols[0]->clone(); - buildEqualityPredicate(gwi.inSubQueryLHS, rhs, &gwi, sop, - Item_func::EQ_FUNC, itemList, true); - - if (gwi.fatalParseError) - return; + gwi.returnedCols[i]->expressionId(gwi.additionalRetCols[additionalRetColsBefore + i]->expressionId()); + gwi.returnedCols[i]->colSource(gwi.additionalRetCols[additionalRetColsBefore + i]->colSource()); } - size_t additionalRetColsAdded = gwi.additionalRetCols.size() - - additionalRetColsBefore; - - if (gwi.returnedCols.size() && - (gwi.returnedCols.size() == additionalRetColsAdded)) - { - for (size_t i = 0; i < gwi.returnedCols.size(); i++) - { - gwi.returnedCols[i]->expressionId( - gwi.additionalRetCols[additionalRetColsBefore + i]->expressionId()); - gwi.returnedCols[i]->colSource( - gwi.additionalRetCols[additionalRetColsBefore + i]->colSource()); - } - - // Delete the duplicate copy of the returned cols - auto iter = gwi.additionalRetCols.begin(); - std::advance(iter, additionalRetColsBefore); - gwi.additionalRetCols.erase(iter, gwi.additionalRetCols.end()); - } + // Delete the duplicate copy of the returned cols + auto iter = gwi.additionalRetCols.begin(); + std::advance(iter, additionalRetColsBefore); + gwi.additionalRetCols.erase(iter, gwi.additionalRetCols.end()); + } } /*@brief Translates SELECT_LEX into CSEP */ @@ -7369,1466 +7203,1457 @@ void buildInToExistsFilter(gp_walk_info& gwi, SELECT_LEX& select_lex) * RETURNS * error id as an int ***********************************************************/ -int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, - SCSEP& csep, - bool isUnion, - bool isSelectHandlerTop, - const std::vector& condStack) +int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool isUnion, + bool isSelectHandlerTop, const std::vector& condStack) { #ifdef DEBUG_WALK_COND - cerr << "getSelectPlan()" << endl; + cerr << "getSelectPlan()" << endl; #endif - int rc = 0; - // rollup is currently not supported - if (select_lex.olap == ROLLUP_TYPE) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } + int rc = 0; + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } - setExecutionParams(gwi, csep); + setExecutionParams(gwi, csep); - gwi.subSelectType = csep->subType(); - uint32_t sessionID = csep->sessionID(); - gwi.sessionid = sessionID; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - gwi.csc = csc; + gwi.subSelectType = csep->subType(); + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + gwi.csc = csc; - CalpontSelectExecutionPlan::SelectList derivedTbList; - // @bug 1796. Remember table order on the FROM list. - gwi.clauseType = FROM; - if ((rc = processFrom(isUnion, select_lex, gwi, csep))) - { - return rc; - } + CalpontSelectExecutionPlan::SelectList derivedTbList; + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; + if ((rc = processFrom(isUnion, select_lex, gwi, csep))) + { + return rc; + } - bool unionSel = (!isUnion && select_lex.master_unit()->is_unit_op()) ? true : false; + bool unionSel = (!isUnion && select_lex.master_unit()->is_unit_op()) ? true : false; - gwi.clauseType = WHERE; - if ((rc = processWhere(select_lex, gwi, csep, condStack))) - { - return rc; - } + gwi.clauseType = WHERE; + if ((rc = processWhere(select_lex, gwi, csep, condStack))) + { + return rc; + } - gwi.clauseType = SELECT; + gwi.clauseType = SELECT; #ifdef DEBUG_WALK_COND - { - cerr << "------------------- SELECT --------------------" << endl; - List_iterator_fast it(select_lex.item_list); - Item* item; - - while ((item = it++)) - { - debug_walk(item, 0); - } - - cerr << "-----------------------------------------------\n" << endl; - } -#endif - - // populate returnedcolumnlist and columnmap + { + cerr << "------------------- SELECT --------------------" << endl; List_iterator_fast it(select_lex.item_list); Item* item; - vector funcFieldVec; - - // empty rcWorkStack and ptWorkStack. They should all be empty by now. - clearStacks(gwi); - - // indicate the starting pos of scalar returned column, because some join column - // has been inserted to the returned column list. - if (gwi.subQuery) - { - ScalarSub* scalar = dynamic_cast(gwi.subQuery); - - if (scalar) - scalar->returnedColPos(gwi.additionalRetCols.size()); - } - - CalpontSelectExecutionPlan::SelectList selectSubList; while ((item = it++)) { - string itemAlias = (item->name.length ? item->name.str : ""); + debug_walk(item, 0); + } - // @bug 5916. Need to keep checking until getting concret item in case - // of nested view. - while (item->type() == Item::REF_ITEM) - { - Item_ref* ref = (Item_ref*)item; - item = (*(ref->ref)); - } - - Item::Type itype = item->type(); - - switch (itype) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = (Item_field*)item; - SimpleColumn* sc = NULL; - - if (ifp->field_name.length && string(ifp->field_name.str) == "*") - { - collectAllCols(gwi, ifp); - break; - } - - sc = buildSimpleColumn(ifp, gwi); - - if (sc) - { - boost::shared_ptr spsc(sc); - - string fullname; - String str; - ifp->print(&str, QT_ORDINARY); - fullname = str.c_ptr(); - - if (!ifp->is_explicit_name()) // no alias - { - sc->alias(fullname); - } - else // alias - { - if (!itemAlias.empty()) - sc->alias(itemAlias); - - } - - gwi.returnedCols.push_back(spsc); - - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), spsc)); - TABLE_LIST* tmp = 0; - - if (ifp->cached_table) - tmp = ifp->cached_table; - - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, tmp); - } - else - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - delete sc; - return ER_INTERNAL_ERROR; - } - - break; - } - - //aggregate column - case Item::SUM_FUNC_ITEM: - { - ReturnedColumn* ac = buildAggregateColumn(item, gwi); - - if (gwi.fatalParseError) - { - // e.g., non-support ref column - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - delete ac; - return ER_CHECK_NOT_IMPLEMENTED; - } - - // add this agg col to returnedColumnList - boost::shared_ptr spac(ac); - gwi.returnedCols.push_back(spac); - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = reinterpret_cast(item); - - // @bug4383. error out non-support stored function - if (ifp->functype() == Item_func::FUNC_SP) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - if (string(ifp->func_name()) == "xor") - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - uint16_t parseInfo = 0; - vector tmpVec; - bool hasNonSupportItem = false; - parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo, &gwi); - - if (ifp->with_subquery() || - string(ifp->func_name()) == string("") || - ifp->functype() == Item_func::NOT_ALL_FUNC || - parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // if "IN" or "BETWEEN" are in the SELECT clause, build function column - string funcName = ifp->func_name(); - ReturnedColumn* rc; - if (funcName == "in" || funcName == " IN " || funcName == "between") - { - rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, true); - } - else - { - rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem); - } - - SRCP srcp(rc); - - if (rc) - { - // MCOL-2178 CS has to process determenistic functions with constant arguments. - if (!hasNonSupportItem && ifp->const_item() && !(parseInfo & AF_BIT) && tmpVec.size() == 0) - { - srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (ifp->name.length) - srcp->alias(ifp->name.str); - - continue; - } - - gwi.returnedCols.push_back(srcp); - } - else // This was a vtable post-process block - { - hasNonSupportItem = false; - uint32_t before_size = funcFieldVec.size(); - parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo, &gwi); - uint32_t after_size = funcFieldVec.size(); - - // pushdown handler projection functions - // @bug3881. set_user_var can not be treated as constant function - // @bug5716. Try to avoid post process function for union query. - if (!hasNonSupportItem && (after_size - before_size) == 0 && - !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT)) - { - ConstantColumn* cc = buildConstantColumnMaybeNullUsingValStr(ifp, gwi); - - SRCP srcp(cc); - - if (ifp->name.length) - cc->alias(ifp->name.str); - - gwi.returnedCols.push_back(srcp); - - // clear the error set by buildFunctionColumn - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - break; - } - else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || - (gwi.fatalParseError && gwi.subQuery)) - { - if (gwi.parseErrorText.empty()) - { - Message::Args args; - args.add(ifp->func_name()); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - } - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else if ( gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM )) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - //@Bug 3030 Add error check for dml statement - if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { - if ( after_size - before_size != 0 ) - { - gwi.parseErrorText = ifp->func_name(); - return -1; - } - } - else - { - Message::Args args; - args.add(ifp->func_name()); - gwi.parseErrorText = - IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - break; - } // End of FUNC_ITEM - - // DRRTUY Replace the whole section with typeid() checks or use - // reinterpret_cast here - case Item::CONST_ITEM: - { - switch(item->cmp_type()) - { - case INT_RESULT: - case STRING_RESULT: - case DECIMAL_RESULT: - case REAL_RESULT: - case TIME_RESULT: - { - if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { } - else - { - // do not push the dummy column (mysql added) to returnedCol - if (item->name.length && string(item->name.str) == "Not_used") - continue; - - // @bug3509. Constant column is sent to ExeMgr now. - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - - if (item->name.length) - srcp->alias(item->name.str); - - gwi.returnedCols.push_back(srcp); - } - - break; - } - // MCOL-2178 This switch doesn't handl - // ROW_ - default: - { - IDEBUG(cerr << "Warning unsupported cmp_type() in projection" << endl); - //noop - } - } - break; - } // CONST_ITEM ends here - - case Item::NULL_ITEM: - { - if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { } - else - { - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (item->name.length) - srcp->alias(item->name.str); - } - - break; - } - - case Item::SUBSELECT_ITEM: - { - Item_subselect* sub = (Item_subselect*)item; - - if (sub->substype() != Item_subselect::SINGLEROW_SUBS) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - -#ifdef DEBUG_WALK_COND - cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; - JOIN* join = sub->get_select_lex()->join; - - if (join) - { - Item_cond* cond = reinterpret_cast(join->conds); - - if (cond) - cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } - - cerr << "Finish SELECT clause subselect item traversing" << endl; + cerr << "-----------------------------------------------\n" << endl; + } #endif - SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); - //selectSub->gwip(&gwi); - SCSEP ssub = selectSub->transform(); - if (!ssub || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + // populate returnedcolumnlist and columnmap + List_iterator_fast it(select_lex.item_list); + Item* item; + vector funcFieldVec; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } + // empty rcWorkStack and ptWorkStack. They should all be empty by now. + clearStacks(gwi); - selectSubList.push_back(ssub); - SimpleColumn* rc = new SimpleColumn(); - rc->colSource(rc->colSource() | SELECT_SUB); - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + // indicate the starting pos of scalar returned column, because some join column + // has been inserted to the returned column list. + if (gwi.subQuery) + { + ScalarSub* scalar = dynamic_cast(gwi.subQuery); - if (sub->get_select_lex()->get_table_list()) - { - rc->viewName(getViewName(sub->get_select_lex()->get_table_list()), lower_case_table_names); - } - if (sub->name.length) - rc->alias(sub->name.str); + if (scalar) + scalar->returnedColPos(gwi.additionalRetCols.size()); + } - gwi.returnedCols.push_back(SRCP(rc)); + CalpontSelectExecutionPlan::SelectList selectSubList; - break; - } + while ((item = it++)) + { + string itemAlias = (item->name.length ? item->name.str : ""); - case Item::COND_ITEM: - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - case Item::EXPR_CACHE_ITEM: - { - printf("EXPR_CACHE_ITEM in getSelectPlan\n"); - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - case Item::WINDOW_FUNC_ITEM: - { - SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); - - if (!srcp || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported Item in SELECT subquery."; - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.returnedCols.push_back(srcp); - break; - } - case Item::TYPE_HOLDER: - { - if(!gwi.tbList.size()) - { - gwi.parseErrorText = "subquery with VALUES"; - gwi.fatalParseError = true; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else - { - std::cerr << "********** received TYPE_HOLDER *********" << std::endl; - - } - break; - } - - - default: - { - break; - } - } + // @bug 5916. Need to keep checking until getting concret item in case + // of nested view. + while (item->type() == Item::REF_ITEM) + { + Item_ref* ref = (Item_ref*)item; + item = (*(ref->ref)); } - // @bug4388 normalize the project coltypes for union main select list - if (!csep->unionVec().empty()) + Item::Type itype = item->type(); + + switch (itype) { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + SimpleColumn* sc = NULL; + + if (ifp->field_name.length && string(ifp->field_name.str) == "*") { - vector coltypes; - - for (uint32_t j = 0; j < csep->unionVec().size(); j++) - { - coltypes.push_back( - dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->resultType()); - - // @bug5976. set hasAggregate true for the main column if - // one corresponding union column has aggregate - if (dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->hasAggregate()) - gwi.returnedCols[i]->hasAggregate(true); - } - - gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); - } - } - - // Having clause handling - gwi.clauseType = HAVING; - clearStacks(gwi); - ParseTree* havingFilter = 0; - // clear fatalParseError that may be left from post process functions - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - - if (select_lex.having != 0) - { - Item_cond* having = reinterpret_cast(select_lex.having); -#ifdef DEBUG_WALK_COND - cerr << "------------------- HAVING ---------------------" << endl; - having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cerr << "------------------------------------------------\n" << endl; -#endif - having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; + collectAllCols(gwi, ifp); + break; } - ParseTree* ptp = 0; - ParseTree* rhs = 0; + sc = buildSimpleColumn(ifp, gwi); - // @bug 4215. some function filter will be in the rcWorkStack. - if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + if (sc) { - havingFilter = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } - - while (!gwi.ptWorkStack.empty()) - { - havingFilter = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - - if (gwi.ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(havingFilter); - rhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - ptp->right(rhs); - gwi.ptWorkStack.push(ptp); - } - } - - // MCOL-4617 If this is an IN subquery, then create the in-to-exists - // predicate and inject it into the csep - if (gwi.subQuery && - gwi.subSelectType == CalpontSelectExecutionPlan::IN_SUBS && - gwi.inSubQueryLHS && gwi.inSubQueryLHSItem) - { - // create the predicate - buildInToExistsFilter(gwi, select_lex); - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - - // now inject the created predicate - if (!gwi.ptWorkStack.empty()) - { - ParseTree* inToExistsFilter = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - - if (havingFilter) - { - ParseTree* ptp = new ParseTree(new LogicOperator("and")); - ptp->left(havingFilter); - ptp->right(inToExistsFilter); - havingFilter = ptp; - } - else - { - if (csep->filters()) - { - ParseTree* ptp = new ParseTree(new LogicOperator("and")); - ptp->left(csep->filters()); - ptp->right(inToExistsFilter); - csep->filters(ptp); - } - else - { - csep->filters(inToExistsFilter); - } - } - } - } - - // for post process expressions on the select list - // error out post process for union and sub select unit - if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - { - if (funcFieldVec.size() != 0 && !gwi.fatalParseError) - { - string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - for (uint32_t i = 0; i < funcFieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); - - if (!sc || gwi.fatalParseError) - { - string emsg; - - if (gwi.parseErrorText.empty()) - { - emsg = "un-recognized column"; - - if (funcFieldVec[i]->name.length) - emsg += string(funcFieldVec[i]->name.str); - } - else - { - emsg = gwi.parseErrorText; - } - - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ER_INTERNAL_ERROR; - } - - String str; - funcFieldVec[i]->print(&str, QT_ORDINARY); - sc->alias(string(str.c_ptr())); - sc->tableAlias(sc->tableAlias()); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; - } - } - - if (j == gwi.returnedCols.size()) - { - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name.str), srcp)); - - string fullname; - fullname = str.c_ptr(); - TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, tmp); - } - } - - SRCP minSc; // min width projected column. for count(*) use - - // Group by list. not valid for union main query - if (!unionSel) - { - gwi.clauseType = GROUP_BY; - Item* nonSupportItem = NULL; - ORDER* groupcol = reinterpret_cast(select_lex.group_list.first); - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - bool hasWindowFunc = gwi.hasWindowFunc; - gwi.hasWindowFunc = false; - - for (; groupcol; groupcol = groupcol->next) - { - if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; - } - - if (gwi.hasWindowFunc) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.hasWindowFunc = hasWindowFunc; - groupcol = reinterpret_cast(select_lex.group_list.first); - - for (; groupcol; groupcol = groupcol->next) - { - Item* groupItem = *(groupcol->item); - - // @bug5993. Could be nested ref. - while (groupItem->type() == Item::REF_ITEM) - groupItem = (*((Item_ref*)groupItem)->ref); - - if (groupItem->type() == Item::FUNC_ITEM) - { - Item_func* ifp = (Item_func*)groupItem; - - // call buildFunctionColumn here mostly for finding out - // non-support column on GB list. Should be simplified. - ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (!fc || gwi.fatalParseError) - { - nonSupportItem = ifp; - break; - } - - if (groupcol->in_field_list && groupcol->counter_used) - { - delete fc; - fc = gwi.returnedCols[groupcol->counter - 1].get(); - SRCP srcp(fc->clone()); - - // check if no column parm - for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) - { - if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) - { - gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); - break; - } - } - - srcp->orderPos(groupcol->counter - 1); - gwi.groupByCols.push_back(srcp); - continue; - } - else if (groupItem->is_explicit_name()) // alias - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - delete fc; - break; - } - } - - if (i == gwi.returnedCols.size()) - { - nonSupportItem = groupItem; - break; - } - } - else - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (fc->operator==(gwi.returnedCols[i].get())) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - delete fc; - break; - } - } - - if (i == gwi.returnedCols.size()) - { - gwi.groupByCols.push_back(SRCP(fc)); - break; - } - } - } - else if (groupItem->type() == Item::FIELD_ITEM) - { - Item_field* ifp = (Item_field*)groupItem; - // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. - ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); - SimpleColumn* sc = dynamic_cast(rc); - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (sc) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - sc->orderPos(j); - break; - } - else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - else - { - if (ifp->name.length && string(ifp->name.str) == gwi.returnedCols[j].get()->alias()) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - } - - if (!rc) - { - nonSupportItem = ifp; - break; - } - - SRCP srcp(rc); - - // bug 3151 - AggregateColumn* ac = dynamic_cast(rc); - - if (ac) - { - nonSupportItem = ifp; - break; - } - - gwi.groupByCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), srcp)); - } - // @bug5638. The group by column is constant but not counter, alias has to match a column - // on the select list - else if (!groupcol->counter_used && - (groupItem->type() == Item::CONST_ITEM && - (groupItem->cmp_type() == INT_RESULT || - groupItem->cmp_type() == STRING_RESULT || - groupItem->cmp_type() == REAL_RESULT || - groupItem->cmp_type() == DECIMAL_RESULT) - ) - ) - { - ReturnedColumn* rc = 0; - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (groupItem->name.length && string(groupItem->name.str) == gwi.returnedCols[j].get()->alias()) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - - if (!rc) - { - nonSupportItem = groupItem; - break; - } - - gwi.groupByCols.push_back(SRCP(rc)); - } - else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) - { - if (!groupcol->in_field_list || !groupItem->name.length) - { - nonSupportItem = groupItem; - } - else - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - break; - } - } - - if (i == gwi.returnedCols.size()) - { - nonSupportItem = groupItem; - } - } - } - // @bug 3761. - else if (groupcol->counter_used) - { - if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) - { - nonSupportItem = groupItem; - } - else - { - gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); - } - } - else - { - nonSupportItem = groupItem; - } - - } - - // @bug 4756. Add internal groupby column for correlated join to the groupby list - if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) - gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); - - // this is window func on SELECT becuase ORDER BY has not been processed - if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) - { - for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) - { - if (gwi.windowFuncList[i]->hasWindowFunc()) - { - vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); - - for (uint32_t j = 0; j < windowFunctions.size(); j++) - windowFunctions[j]->addToPartition(gwi.subGroupByCols); - } - } - } - - if (nonSupportItem) - { - if (gwi.parseErrorText.length() == 0) - { - Message::Args args; - if (nonSupportItem->name.length) - args.add("'" + string(nonSupportItem->name.str) + "'"); - else - args.add(""); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); - } - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - // ORDER BY processing - { - SQL_I_List order_list = select_lex.order_list; - ORDER* ordercol = reinterpret_cast(order_list.first); - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - for (; ordercol; ordercol = ordercol->next) - { - if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; - // MCOL-2166 Looking for this sorting item in GROUP_BY items list. - // Shouldn't look into this if query doesn't have GROUP BY or - // aggregations - if(select_lex.agg_func_used() && select_lex.group_list.first - && !sortItemIsInGrouping(*ordercol->item, - select_lex.group_list.first)) - { - std::ostringstream ostream; - std::ostringstream& osr = ostream; - getColNameFromItem(osr, *ordercol->item); - Message::Args args; - args.add(ostream.str()); - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ERR_NOT_GROUPBY_EXPRESSION; - } - } - - // re-visit the first of ordercol list - ordercol = reinterpret_cast(order_list.first); - - { - for (; ordercol; ordercol = ordercol->next) - { - ReturnedColumn* rc = NULL; - - if (ordercol->in_field_list && ordercol->counter_used) - { - rc = gwi.returnedCols[ordercol->counter - 1]->clone(); - rc->orderPos(ordercol->counter - 1); - // can not be optimized off if used in order by with counter. - // set with self derived table alias if it's derived table - gwi.returnedCols[ordercol->counter - 1]->incRefCount(); - } - else - { - Item* ord_item = *(ordercol->item); - - // ignore not_used column on order by. - if ((ord_item->type() == Item::CONST_ITEM - && ord_item->cmp_type() == INT_RESULT) - && ord_item->full_name() - && !strcmp(ord_item->full_name(), "Not_used")) - { - continue; - } - else if (ord_item->type() == Item::CONST_ITEM - && ord_item->cmp_type() == INT_RESULT) - { - // DRRTUY This section looks useless b/c there is no - // way to put constant INT into an ORDER BY list - rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - gwi.fatalParseError = true; - } - else if ((ord_item->type() == Item::FUNC_ITEM) && (((Item_func*)ord_item)->functype() == Item_func::COLLATE_FUNC)) - { - push_warning(gwi.thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "COLLATE is ignored in ColumnStore"); - continue; - } - else - { - rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - } - // @bug5501 try item_ptr if item can not be fixed. For some - // weird dml statement state, item can not be fixed but the - // infomation is available in item_ptr. - if (!rc || gwi.fatalParseError) - { - Item* item_ptr = ordercol->item_ptr; - - while (item_ptr->type() == Item::REF_ITEM) - item_ptr = *(((Item_ref*)item_ptr)->ref); - - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - - if (!rc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - if (ordercol->direction == ORDER::ORDER_ASC) - rc->asc(true); - else - rc->asc(false); - - gwi.orderByCols.push_back(SRCP(rc)); - } - } - // make sure columnmap, returnedcols and count(*) arg_list are not empty - TableMap::iterator tb_iter = gwi.tableMap.begin(); - - try - { - for (; tb_iter != gwi.tableMap.end(); tb_iter++) - { - if ((*tb_iter).second.first == 1) continue; - - CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; - CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); - SRCP srcp(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - (*tb_iter).second.first = 1; - } - } - catch (runtime_error& e) - { - setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || - gwi.returnedCols.empty()) - { - // get the smallest column from colmap - CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; - int minColWidth = 0; - CalpontSystemCatalog::ColType ct; - - try - { - for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) - { - // should always not null - SimpleColumn* sc = dynamic_cast(iter->second.get()); - - if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) - { - ct = csc->colType(sc->oid()); - - if (minColWidth == 0) - { - minColWidth = ct.colWidth; - minSc = iter->second; - } - else if (ct.colWidth < minColWidth) - { - minColWidth = ct.colWidth; - minSc = iter->second; - } - } - } - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty() && minSc) - gwi.returnedCols.push_back(minSc); - } - - // ORDER BY translation part - if (!isUnion && !gwi.hasWindowFunc - && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT ) - { - { - if (unionSel) - order_list = select_lex.master_unit()->global_parameters()->order_list; - - ordercol = reinterpret_cast(order_list.first); - - for (; ordercol; ordercol = ordercol->next) - { - Item* ord_item = *(ordercol->item); - - if (ord_item->name.length) - { - // for union order by 1 case. For unknown reason, it doesn't show in_field_list - if (ord_item->type() == Item::CONST_ITEM - && ord_item->cmp_type() == INT_RESULT) - { - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - } - else - { - } - } - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug5636. check if this order by column is on the select list - ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - break; - } - } - } - } - } - - if ( gwi.orderByCols.size() ) // has order by - { - csep->hasOrderBy(true); - // To activate LimitedOrderBy - csep->orderByThreads(get_orderby_threads(gwi.thd)); - csep->specHandlerProcessed(true); - } - } - - // We don't currently support limit with correlated subquery - if ((rc = processLimitAndOffset(select_lex, gwi, csep, unionSel, isUnion, isSelectHandlerTop))) - { - return rc; - } - } // ORDER BY end - - if (select_lex.options & SELECT_DISTINCT) - csep->distinct(true); - - // add the smallest column to count(*) parm. - // select constant in subquery case - std::vector::iterator coliter; - - if (!minSc) - { - if (!gwi.returnedCols.empty()) - minSc = gwi.returnedCols[0]; - else if (!gwi.additionalRetCols.empty()) - minSc = gwi.additionalRetCols[0]; - } - - // @bug3523, count(*) on subquery always pick column[0]. - SimpleColumn* sc = dynamic_cast(minSc.get()); - - if (sc && sc->schemaName().empty()) - { - if (gwi.derivedTbList.size() >= 1) - { - SimpleColumn* sc1 = new SimpleColumn(); - sc1->columnName(sc->columnName()); - sc1->tableName(sc->tableName()); - sc1->tableAlias(sc->tableAlias()); - sc1->viewName(sc->viewName()); - sc1->colPosition(0); - sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - minSc.reset(sc1); - } - } - - for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) - { - // @bug5977 @note should never throw this, but checking just in case. - // When ExeMgr fix is ready, this should not error out... - if (dynamic_cast(minSc.get())) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "No project column found for aggregate function"; - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // Replace the last (presumably constant) object with minSc - if ((*coliter)->aggParms().empty()) - { - (*coliter)->aggParms().push_back(minSc); + boost::shared_ptr spsc(sc); + + string fullname; + String str; + ifp->print(&str, QT_ORDINARY); + fullname = str.c_ptr(); + + if (!ifp->is_explicit_name()) // no alias + { + sc->alias(fullname); + } + else // alias + { + if (!itemAlias.empty()) + sc->alias(itemAlias); + } + + gwi.returnedCols.push_back(spsc); + + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), spsc)); + TABLE_LIST* tmp = 0; + + if (ifp->cached_table) + tmp = ifp->cached_table; + + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), + sc->isColumnStore())] = make_pair(1, tmp); } else { - (*coliter)->aggParms()[0] = minSc; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + delete sc; + return ER_INTERNAL_ERROR; } + + break; + } + + // aggregate column + case Item::SUM_FUNC_ITEM: + { + ReturnedColumn* ac = buildAggregateColumn(item, gwi); + + if (gwi.fatalParseError) + { + // e.g., non-support ref column + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + delete ac; + return ER_CHECK_NOT_IMPLEMENTED; + } + + // add this agg col to returnedColumnList + boost::shared_ptr spac(ac); + gwi.returnedCols.push_back(spac); + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = reinterpret_cast(item); + + // @bug4383. error out non-support stored function + if (ifp->functype() == Item_func::FUNC_SP) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + if (string(ifp->func_name()) == "xor") + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + uint16_t parseInfo = 0; + vector tmpVec; + bool hasNonSupportItem = false; + parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo, &gwi); + + if (ifp->with_subquery() || string(ifp->func_name()) == string("") || + ifp->functype() == Item_func::NOT_ALL_FUNC || parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // if "IN" or "BETWEEN" are in the SELECT clause, build function column + string funcName = ifp->func_name(); + ReturnedColumn* rc; + if (funcName == "in" || funcName == " IN " || funcName == "between") + { + rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, true); + } + else + { + rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem); + } + + SRCP srcp(rc); + + if (rc) + { + // MCOL-2178 CS has to process determenistic functions with constant arguments. + if (!hasNonSupportItem && ifp->const_item() && !(parseInfo & AF_BIT) && tmpVec.size() == 0) + { + srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (ifp->name.length) + srcp->alias(ifp->name.str); + + continue; + } + + gwi.returnedCols.push_back(srcp); + } + else // This was a vtable post-process block + { + hasNonSupportItem = false; + uint32_t before_size = funcFieldVec.size(); + parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo, &gwi); + uint32_t after_size = funcFieldVec.size(); + + // pushdown handler projection functions + // @bug3881. set_user_var can not be treated as constant function + // @bug5716. Try to avoid post process function for union query. + if (!hasNonSupportItem && (after_size - before_size) == 0 && !(parseInfo & AGG_BIT) && + !(parseInfo & SUB_BIT)) + { + ConstantColumn* cc = buildConstantColumnMaybeNullUsingValStr(ifp, gwi); + + SRCP srcp(cc); + + if (ifp->name.length) + cc->alias(ifp->name.str); + + gwi.returnedCols.push_back(srcp); + + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + break; + } + else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || + (gwi.fatalParseError && gwi.subQuery)) + { + if (gwi.parseErrorText.empty()) + { + Message::Args args; + args.add(ifp->func_name()); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + } + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else if (gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM)) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + //@Bug 3030 Add error check for dml statement + if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) + { + if (after_size - before_size != 0) + { + gwi.parseErrorText = ifp->func_name(); + return -1; + } + } + else + { + Message::Args args; + args.add(ifp->func_name()); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + break; + } // End of FUNC_ITEM + + // DRRTUY Replace the whole section with typeid() checks or use + // reinterpret_cast here + case Item::CONST_ITEM: + { + switch (item->cmp_type()) + { + case INT_RESULT: + case STRING_RESULT: + case DECIMAL_RESULT: + case REAL_RESULT: + case TIME_RESULT: + { + if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) + { + } + else + { + // do not push the dummy column (mysql added) to returnedCol + if (item->name.length && string(item->name.str) == "Not_used") + continue; + + // @bug3509. Constant column is sent to ExeMgr now. + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + + if (item->name.length) + srcp->alias(item->name.str); + + gwi.returnedCols.push_back(srcp); + } + + break; + } + // MCOL-2178 This switch doesn't handl + // ROW_ + default: + { + IDEBUG(cerr << "Warning unsupported cmp_type() in projection" << endl); + // noop + } + } + break; + } // CONST_ITEM ends here + + case Item::NULL_ITEM: + { + if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) + { + } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name.length) + srcp->alias(item->name.str); + } + + break; + } + + case Item::SUBSELECT_ITEM: + { + Item_subselect* sub = (Item_subselect*)item; + + if (sub->substype() != Item_subselect::SINGLEROW_SUBS) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + +#ifdef DEBUG_WALK_COND + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + JOIN* join = sub->get_select_lex()->join; + + if (join) + { + Item_cond* cond = reinterpret_cast(join->conds); + + if (cond) + cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + + cerr << "Finish SELECT clause subselect item traversing" << endl; +#endif + SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); + // selectSub->gwip(&gwi); + SCSEP ssub = selectSub->transform(); + + if (!ssub || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + selectSubList.push_back(ssub); + SimpleColumn* rc = new SimpleColumn(); + rc->colSource(rc->colSource() | SELECT_SUB); + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + + if (sub->get_select_lex()->get_table_list()) + { + rc->viewName(getViewName(sub->get_select_lex()->get_table_list()), lower_case_table_names); + } + if (sub->name.length) + rc->alias(sub->name.str); + + gwi.returnedCols.push_back(SRCP(rc)); + + break; + } + + case Item::COND_ITEM: + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::EXPR_CACHE_ITEM: + { + printf("EXPR_CACHE_ITEM in getSelectPlan\n"); + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::WINDOW_FUNC_ITEM: + { + SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); + + if (!srcp || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.returnedCols.push_back(srcp); + break; + } + case Item::TYPE_HOLDER: + { + if (!gwi.tbList.size()) + { + gwi.parseErrorText = "subquery with VALUES"; + gwi.fatalParseError = true; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + std::cerr << "********** received TYPE_HOLDER *********" << std::endl; + } + break; + } + + default: + { + break; + } } + } - std::vector::iterator funciter; - - SPTP sptp(new ParseTree(minSc.get()->clone())); - - for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + // @bug4388 normalize the project coltypes for union main select list + if (!csep->unionVec().empty()) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) { - FunctionParm funcParms = (*funciter)->functionParms(); - funcParms.push_back(sptp); - (*funciter)->functionParms(funcParms); + vector coltypes; + + for (uint32_t j = 0; j < csep->unionVec().size(); j++) + { + coltypes.push_back(dynamic_cast(csep->unionVec()[j].get()) + ->returnedCols()[i] + ->resultType()); + + // @bug5976. set hasAggregate true for the main column if + // one corresponding union column has aggregate + if (dynamic_cast(csep->unionVec()[j].get()) + ->returnedCols()[i] + ->hasAggregate()) + gwi.returnedCols[i]->hasAggregate(true); + } + + gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); + } + } + + // Having clause handling + gwi.clauseType = HAVING; + clearStacks(gwi); + ParseTree* havingFilter = 0; + // clear fatalParseError that may be left from post process functions + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + + if (select_lex.having != 0) + { + Item_cond* having = reinterpret_cast(select_lex.having); +#ifdef DEBUG_WALK_COND + cerr << "------------------- HAVING ---------------------" << endl; + having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; } - // set sequence# for subquery localCols - for (uint32_t i = 0; i < gwi.localCols.size(); i++) - gwi.localCols[i]->sequence(i); + ParseTree* ptp = 0; + ParseTree* rhs = 0; - // append additionalRetCols to returnedCols - gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), - gwi.additionalRetCols.end()); + // @bug 4215. some function filter will be in the rcWorkStack. + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + havingFilter = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } - csep->groupByCols(gwi.groupByCols); - csep->orderByCols(gwi.orderByCols); - csep->returnedCols(gwi.returnedCols); - csep->columnMap(gwi.columnMap); - csep->having(havingFilter); - csep->derivedTableList(gwi.derivedTbList); - csep->selectSubList(selectSubList); - csep->subSelectList(gwi.subselectList); - clearStacks(gwi); - return 0; + while (!gwi.ptWorkStack.empty()) + { + havingFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(havingFilter); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } + } + + // MCOL-4617 If this is an IN subquery, then create the in-to-exists + // predicate and inject it into the csep + if (gwi.subQuery && gwi.subSelectType == CalpontSelectExecutionPlan::IN_SUBS && gwi.inSubQueryLHS && + gwi.inSubQueryLHSItem) + { + // create the predicate + buildInToExistsFilter(gwi, select_lex); + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + + // now inject the created predicate + if (!gwi.ptWorkStack.empty()) + { + ParseTree* inToExistsFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (havingFilter) + { + ParseTree* ptp = new ParseTree(new LogicOperator("and")); + ptp->left(havingFilter); + ptp->right(inToExistsFilter); + havingFilter = ptp; + } + else + { + if (csep->filters()) + { + ParseTree* ptp = new ParseTree(new LogicOperator("and")); + ptp->left(csep->filters()); + ptp->right(inToExistsFilter); + csep->filters(ptp); + } + else + { + csep->filters(inToExistsFilter); + } + } + } + } + + // for post process expressions on the select list + // error out post process for union and sub select unit + if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (funcFieldVec.size() != 0 && !gwi.fatalParseError) + { + string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + for (uint32_t i = 0; i < funcFieldVec.size(); i++) + { + SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); + + if (!sc || gwi.fatalParseError) + { + string emsg; + + if (gwi.parseErrorText.empty()) + { + emsg = "un-recognized column"; + + if (funcFieldVec[i]->name.length) + emsg += string(funcFieldVec[i]->name.str); + } + else + { + emsg = gwi.parseErrorText; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ER_INTERNAL_ERROR; + } + + String str; + funcFieldVec[i]->print(&str, QT_ORDINARY); + sc->alias(string(str.c_ptr())); + sc->tableAlias(sc->tableAlias()); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); + + if (field && field->alias() == sc->alias()) + break; + } + } + + if (j == gwi.returnedCols.size()) + { + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name.str), srcp)); + + string fullname; + fullname = str.c_ptr(); + TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), + sc->isColumnStore())] = make_pair(1, tmp); + } + } + + SRCP minSc; // min width projected column. for count(*) use + + // Group by list. not valid for union main query + if (!unionSel) + { + gwi.clauseType = GROUP_BY; + Item* nonSupportItem = NULL; + ORDER* groupcol = reinterpret_cast(select_lex.group_list.first); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + bool hasWindowFunc = gwi.hasWindowFunc; + gwi.hasWindowFunc = false; + + for (; groupcol; groupcol = groupcol->next) + { + if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + if (gwi.hasWindowFunc) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.hasWindowFunc = hasWindowFunc; + groupcol = reinterpret_cast(select_lex.group_list.first); + + for (; groupcol; groupcol = groupcol->next) + { + Item* groupItem = *(groupcol->item); + + // @bug5993. Could be nested ref. + while (groupItem->type() == Item::REF_ITEM) + groupItem = (*((Item_ref*)groupItem)->ref); + + if (groupItem->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)groupItem; + + // call buildFunctionColumn here mostly for finding out + // non-support column on GB list. Should be simplified. + ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (!fc || gwi.fatalParseError) + { + nonSupportItem = ifp; + break; + } + + if (groupcol->in_field_list && groupcol->counter_used) + { + delete fc; + fc = gwi.returnedCols[groupcol->counter - 1].get(); + SRCP srcp(fc->clone()); + + // check if no column parm + for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) + { + if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) + { + gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); + break; + } + } + + srcp->orderPos(groupcol->counter - 1); + gwi.groupByCols.push_back(srcp); + continue; + } + else if (groupItem->is_explicit_name()) // alias + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + break; + } + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + gwi.groupByCols.push_back(SRCP(fc)); + break; + } + } + } + else if (groupItem->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)groupItem; + // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. + ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); + SimpleColumn* sc = dynamic_cast(rc); + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (sc) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + sc->orderPos(j); + break; + } + else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + else + { + if (ifp->name.length && string(ifp->name.str) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + } + + if (!rc) + { + nonSupportItem = ifp; + break; + } + + SRCP srcp(rc); + + // bug 3151 + AggregateColumn* ac = dynamic_cast(rc); + + if (ac) + { + nonSupportItem = ifp; + break; + } + + gwi.groupByCols.push_back(srcp); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), srcp)); + } + // @bug5638. The group by column is constant but not counter, alias has to match a column + // on the select list + else if (!groupcol->counter_used && + (groupItem->type() == Item::CONST_ITEM && + (groupItem->cmp_type() == INT_RESULT || groupItem->cmp_type() == STRING_RESULT || + groupItem->cmp_type() == REAL_RESULT || groupItem->cmp_type() == DECIMAL_RESULT))) + { + ReturnedColumn* rc = 0; + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (groupItem->name.length && string(groupItem->name.str) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + + if (!rc) + { + nonSupportItem = groupItem; + break; + } + + gwi.groupByCols.push_back(SRCP(rc)); + } + else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) + { + if (!groupcol->in_field_list || !groupItem->name.length) + { + nonSupportItem = groupItem; + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + } + } + } + // @bug 3761. + else if (groupcol->counter_used) + { + if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) + { + nonSupportItem = groupItem; + } + else + { + gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); + } + } + else + { + nonSupportItem = groupItem; + } + } + + // @bug 4756. Add internal groupby column for correlated join to the groupby list + if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) + gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); + + // this is window func on SELECT becuase ORDER BY has not been processed + if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) + { + for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) + { + if (gwi.windowFuncList[i]->hasWindowFunc()) + { + vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); + + for (uint32_t j = 0; j < windowFunctions.size(); j++) + windowFunctions[j]->addToPartition(gwi.subGroupByCols); + } + } + } + + if (nonSupportItem) + { + if (gwi.parseErrorText.length() == 0) + { + Message::Args args; + if (nonSupportItem->name.length) + args.add("'" + string(nonSupportItem->name.str) + "'"); + else + args.add(""); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + } + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + // ORDER BY processing + { + SQL_I_List order_list = select_lex.order_list; + ORDER* ordercol = reinterpret_cast(order_list.first); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + for (; ordercol; ordercol = ordercol->next) + { + if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + // MCOL-2166 Looking for this sorting item in GROUP_BY items list. + // Shouldn't look into this if query doesn't have GROUP BY or + // aggregations + if (select_lex.agg_func_used() && select_lex.group_list.first && + !sortItemIsInGrouping(*ordercol->item, select_lex.group_list.first)) + { + std::ostringstream ostream; + std::ostringstream& osr = ostream; + getColNameFromItem(osr, *ordercol->item); + Message::Args args; + args.add(ostream.str()); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ERR_NOT_GROUPBY_EXPRESSION; + } + } + + // re-visit the first of ordercol list + ordercol = reinterpret_cast(order_list.first); + + { + for (; ordercol; ordercol = ordercol->next) + { + ReturnedColumn* rc = NULL; + + if (ordercol->in_field_list && ordercol->counter_used) + { + rc = gwi.returnedCols[ordercol->counter - 1]->clone(); + rc->orderPos(ordercol->counter - 1); + // can not be optimized off if used in order by with counter. + // set with self derived table alias if it's derived table + gwi.returnedCols[ordercol->counter - 1]->incRefCount(); + } + else + { + Item* ord_item = *(ordercol->item); + + // ignore not_used column on order by. + if ((ord_item->type() == Item::CONST_ITEM && ord_item->cmp_type() == INT_RESULT) && + ord_item->full_name() && !strcmp(ord_item->full_name(), "Not_used")) + { + continue; + } + else if (ord_item->type() == Item::CONST_ITEM && ord_item->cmp_type() == INT_RESULT) + { + // DRRTUY This section looks useless b/c there is no + // way to put constant INT into an ORDER BY list + rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + gwi.fatalParseError = true; + } + else if ((ord_item->type() == Item::FUNC_ITEM) && + (((Item_func*)ord_item)->functype() == Item_func::COLLATE_FUNC)) + { + push_warning(gwi.thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, + "COLLATE is ignored in ColumnStore"); + continue; + } + else + { + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + } + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) + { + Item* item_ptr = ordercol->item_ptr; + + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); + + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + if (!rc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + if (ordercol->direction == ORDER::ORDER_ASC) + rc->asc(true); + else + rc->asc(false); + + gwi.orderByCols.push_back(SRCP(rc)); + } + } + // make sure columnmap, returnedcols and count(*) arg_list are not empty + TableMap::iterator tb_iter = gwi.tableMap.begin(); + + try + { + for (; tb_iter != gwi.tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) + continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || gwi.returnedCols.empty()) + { + // get the smallest column from colmap + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + int minColWidth = 0; + CalpontSystemCatalog::ColType ct; + + try + { + for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) + { + // should always not null + SimpleColumn* sc = dynamic_cast(iter->second.get()); + + if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + { + ct = csc->colType(sc->oid()); + + if (minColWidth == 0) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + else if (ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + } + } + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty() && minSc) + gwi.returnedCols.push_back(minSc); + } + + // ORDER BY translation part + if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + { + if (unionSel) + order_list = select_lex.master_unit()->global_parameters()->order_list; + + ordercol = reinterpret_cast(order_list.first); + + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + if (ord_item->name.length) + { + // for union order by 1 case. For unknown reason, it doesn't show in_field_list + if (ord_item->type() == Item::CONST_ITEM && ord_item->cmp_type() == INT_RESULT) + { + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + } + else + { + } + } + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug5636. check if this order by column is on the select list + ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + break; + } + } + } + } + } + + if (gwi.orderByCols.size()) // has order by + { + csep->hasOrderBy(true); + // To activate LimitedOrderBy + csep->orderByThreads(get_orderby_threads(gwi.thd)); + csep->specHandlerProcessed(true); + } + } + + // We don't currently support limit with correlated subquery + if ((rc = processLimitAndOffset(select_lex, gwi, csep, unionSel, isUnion, isSelectHandlerTop))) + { + return rc; + } + } // ORDER BY end + + if (select_lex.options & SELECT_DISTINCT) + csep->distinct(true); + + // add the smallest column to count(*) parm. + // select constant in subquery case + std::vector::iterator coliter; + + if (!minSc) + { + if (!gwi.returnedCols.empty()) + minSc = gwi.returnedCols[0]; + else if (!gwi.additionalRetCols.empty()) + minSc = gwi.additionalRetCols[0]; + } + + // @bug3523, count(*) on subquery always pick column[0]. + SimpleColumn* sc = dynamic_cast(minSc.get()); + + if (sc && sc->schemaName().empty()) + { + if (gwi.derivedTbList.size() >= 1) + { + SimpleColumn* sc1 = new SimpleColumn(); + sc1->columnName(sc->columnName()); + sc1->tableName(sc->tableName()); + sc1->tableAlias(sc->tableAlias()); + sc1->viewName(sc->viewName()); + sc1->colPosition(0); + sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + minSc.reset(sc1); + } + } + + for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) + { + // @bug5977 @note should never throw this, but checking just in case. + // When ExeMgr fix is ready, this should not error out... + if (dynamic_cast(minSc.get())) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // Replace the last (presumably constant) object with minSc + if ((*coliter)->aggParms().empty()) + { + (*coliter)->aggParms().push_back(minSc); + } + else + { + (*coliter)->aggParms()[0] = minSc; + } + } + + std::vector::iterator funciter; + + SPTP sptp(new ParseTree(minSc.get()->clone())); + + for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + { + FunctionParm funcParms = (*funciter)->functionParms(); + funcParms.push_back(sptp); + (*funciter)->functionParms(funcParms); + } + + // set sequence# for subquery localCols + for (uint32_t i = 0; i < gwi.localCols.size(); i++) + gwi.localCols[i]->sequence(i); + + // append additionalRetCols to returnedCols + gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), + gwi.additionalRetCols.end()); + + csep->groupByCols(gwi.groupByCols); + csep->orderByCols(gwi.orderByCols); + csep->returnedCols(gwi.returnedCols); + csep->columnMap(gwi.columnMap); + csep->having(havingFilter); + csep->derivedTableList(gwi.derivedTbList); + csep->selectSubList(selectSubList); + csep->subSelectList(gwi.subselectList); + clearStacks(gwi); + return 0; } int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) { + gp_walk_info* gwi = ti.condInfo; + + if (!gwi) + gwi = new gp_walk_info(); + + gwi->thd = thd; + LEX* lex = thd->lex; + idbassert(lex != 0); + uint32_t sessionID = csep->sessionID(); + gwi->sessionid = sessionID; + TABLE* table = ti.msTablePtr; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + // get all columns that mysql needs to fetch + MY_BITMAP* read_set = table->read_set; + Field **f_ptr, *field; + gwi->columnMap.clear(); + + for (f_ptr = table->field; (field = *f_ptr); f_ptr++) + { + if (bitmap_is_set(read_set, field->field_index)) + { + SimpleColumn* sc = + new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name.str, sessionID); + string alias(table->alias.c_ptr()); + if (lower_case_table_names) + { + boost::algorithm::to_lower(alias); + } + sc->tableAlias(alias); + sc->timeZone(gwi->thd->variables.time_zone->get_name()->ptr()); + assert(sc); + boost::shared_ptr spsc(sc); + gwi->returnedCols.push_back(spsc); + gwi->columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name.str), spsc)); + } + } + + if (gwi->columnMap.empty()) + { + CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); + CalpontSystemCatalog::TableAliasName tan = + make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); + SRCP srcp(sc); + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + gwi->returnedCols.push_back(srcp); + } + + // get filter + if (ti.condInfo) + { gp_walk_info* gwi = ti.condInfo; + ParseTree* filters = 0; + ParseTree* ptp = 0; + ParseTree* rhs = 0; - if (!gwi) - gwi = new gp_walk_info(); - - gwi->thd = thd; - LEX* lex = thd->lex; - idbassert(lex != 0); - uint32_t sessionID = csep->sessionID(); - gwi->sessionid = sessionID; - TABLE* table = ti.msTablePtr; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - - // get all columns that mysql needs to fetch - MY_BITMAP* read_set = table->read_set; - Field** f_ptr, *field; - gwi->columnMap.clear(); - - for (f_ptr = table->field ; (field = *f_ptr) ; f_ptr++) + while (!gwi->ptWorkStack.empty()) { - if (bitmap_is_set(read_set, field->field_index)) - { - SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name.str, sessionID); - string alias(table->alias.c_ptr()); - if (lower_case_table_names) - { - boost::algorithm::to_lower(alias); - } - sc->tableAlias(alias); - sc->timeZone(gwi->thd->variables.time_zone->get_name()->ptr()); - assert (sc); - boost::shared_ptr spsc(sc); - gwi->returnedCols.push_back(spsc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name.str), spsc)); - } + filters = gwi->ptWorkStack.top(); + gwi->ptWorkStack.pop(); + SimpleFilter* sf = dynamic_cast(filters->data()); + + if (sf && sf->op()->data() == "noop") + { + delete filters; + filters = 0; + + if (gwi->ptWorkStack.empty()) + break; + + continue; + } + + if (gwi->ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + rhs = gwi->ptWorkStack.top(); + gwi->ptWorkStack.pop(); + ptp->right(rhs); + gwi->ptWorkStack.push(ptp); } - if (gwi->columnMap.empty()) - { - CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); - CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); - SRCP srcp(sc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - gwi->returnedCols.push_back(srcp); - } + csep->filters(filters); + } - // get filter - if (ti.condInfo) - { - gp_walk_info* gwi = ti.condInfo; - ParseTree* filters = 0; - ParseTree* ptp = 0; - ParseTree* rhs = 0; + csep->returnedCols(gwi->returnedCols); + csep->columnMap(gwi->columnMap); + CalpontSelectExecutionPlan::TableList tblist; + tblist.push_back(make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr(), true, + lower_case_table_names)); + csep->tableList(tblist); - while (!gwi->ptWorkStack.empty()) - { - filters = gwi->ptWorkStack.top(); - gwi->ptWorkStack.pop(); - SimpleFilter* sf = dynamic_cast(filters->data()); + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(get_string_scan_threshold(gwi->thd)); - if (sf && sf->op()->data() == "noop") - { - delete filters; - filters = 0; - - if (gwi->ptWorkStack.empty()) - break; - - continue; - } - - if (gwi->ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - rhs = gwi->ptWorkStack.top(); - gwi->ptWorkStack.pop(); - ptp->right(rhs); - gwi->ptWorkStack.push(ptp); - } - - csep->filters(filters); - } - - csep->returnedCols(gwi->returnedCols); - csep->columnMap(gwi->columnMap); - CalpontSelectExecutionPlan::TableList tblist; - tblist.push_back(make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr(), true, lower_case_table_names)); - csep->tableList(tblist); - - // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering - csep->stringScanThreshold(get_string_scan_threshold(gwi->thd)); - - return 0; + return 0; } int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) { - - SELECT_LEX *select_lex = gi.groupByTables->select_lex; - gp_walk_info gwi; - gwi.thd = thd; - gwi.isGroupByHandler = true; - int status = getGroupPlan(gwi, *select_lex, csep, gi); + SELECT_LEX* select_lex = gi.groupByTables->select_lex; + gp_walk_info gwi; + gwi.thd = thd; + gwi.isGroupByHandler = true; + int status = getGroupPlan(gwi, *select_lex, csep, gi); #ifdef DEBUG_WALK_COND - cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif - if (status > 0) - return ER_INTERNAL_ERROR; - else if (status < 0) - return status; - // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; + // Derived table projection and filter optimization. + derivedTableOptimization(thd, csep); - return 0; + return 0; } int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, SCSEP& csep, gp_walk_info& gwi) { - SELECT_LEX& select_lex = *handler->select; - int status = getSelectPlan(gwi, select_lex, csep, false); + SELECT_LEX& select_lex = *handler->select; + int status = getSelectPlan(gwi, select_lex, csep, false); - if (status > 0) - return ER_INTERNAL_ERROR; - else if (status < 0) - return status; + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; #ifdef DEBUG_WALK_COND - cerr << "---------------- cs_get_derived_plan EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + cerr << "---------------- cs_get_derived_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif - // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); - return 0; + // Derived table projection and filter optimization. + derivedTableOptimization(thd, csep); + return 0; } int cs_get_select_plan(ha_columnstore_select_handler* handler, THD* thd, SCSEP& csep, gp_walk_info& gwi) { - SELECT_LEX& select_lex = *handler->select; + SELECT_LEX& select_lex = *handler->select; - if (select_lex.where) - { - gwi.condList.push_back(select_lex.where); - } + if (select_lex.where) + { + gwi.condList.push_back(select_lex.where); + } - buildTableOnExprList(&select_lex.top_join_list, gwi.tableOnExprList); + buildTableOnExprList(&select_lex.top_join_list, gwi.tableOnExprList); - convertOuterJoinToInnerJoin(&select_lex.top_join_list, - gwi.tableOnExprList, gwi.condList, handler->tableOuterJoinMap); + convertOuterJoinToInnerJoin(&select_lex.top_join_list, gwi.tableOnExprList, gwi.condList, + handler->tableOuterJoinMap); - int status = getSelectPlan(gwi, select_lex, csep, false, true); + int status = getSelectPlan(gwi, select_lex, csep, false, true); - if (status > 0) - return ER_INTERNAL_ERROR; - else if (status < 0) - return status; + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; #ifdef DEBUG_WALK_COND - cerr << "---------------- cs_get_select_plan EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + cerr << "---------------- cs_get_select_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif - // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + // Derived table projection and filter optimization. + derivedTableOptimization(thd, csep); - return 0; + return 0; } - /*@brief buildConstColFromFilter- change SimpleColumn into ConstColumn*/ /*********************************************************** * DESCRIPTION: @@ -8846,1799 +8671,1795 @@ int cs_get_select_plan(ha_columnstore_select_handler* handler, THD* thd, SCSEP& * ConstantColumn* if originalSC equals with cond_pushed columns. * NULL otherwise ***********************************************************/ -ConstantColumn* buildConstColFromFilter(SimpleColumn* originalSC, - gp_walk_info& gwi, cal_group_info& gi) +ConstantColumn* buildConstColFromFilter(SimpleColumn* originalSC, gp_walk_info& gwi, cal_group_info& gi) { - execplan::SimpleColumn* simpleCol; - execplan::ConstantColumn* constCol; - execplan::SOP op; - execplan::SimpleFilter* simpFilter; - execplan::ConstantColumn* result = NULL; - std::vector::iterator ptIt = gi.pushedPts.begin(); + execplan::SimpleColumn* simpleCol; + execplan::ConstantColumn* constCol; + execplan::SOP op; + execplan::SimpleFilter* simpFilter; + execplan::ConstantColumn* result = NULL; + std::vector::iterator ptIt = gi.pushedPts.begin(); - for (; ptIt != gi.pushedPts.end(); ptIt++) + for (; ptIt != gi.pushedPts.end(); ptIt++) + { + simpFilter = dynamic_cast((*ptIt)->data()); + + if (simpFilter == NULL) + continue; + + simpleCol = dynamic_cast(simpFilter->lhs()); + constCol = dynamic_cast(simpFilter->rhs()); + + if (simpleCol == NULL || constCol == NULL) + continue; + + op = simpFilter->op(); + execplan::ReturnedColumn* rc = dynamic_cast(simpleCol); + + // The filter could have any kind of op + if (originalSC->sameColumn(rc)) { - simpFilter = dynamic_cast((*ptIt)->data()); - - if (simpFilter == NULL) - continue; - - simpleCol = dynamic_cast(simpFilter->lhs()); - constCol = dynamic_cast(simpFilter->rhs()); - - if (simpleCol == NULL || constCol == NULL) - continue; - - op = simpFilter->op(); - execplan::ReturnedColumn* rc = dynamic_cast(simpleCol); - - // The filter could have any kind of op - if ( originalSC->sameColumn(rc) ) - { #ifdef DEBUG_WALK_COND - cerr << "buildConstColFromFilter() replaced " << endl; - cerr << simpleCol->toString() << endl; - cerr << " with " << endl; - cerr << constCol << endl; + cerr << "buildConstColFromFilter() replaced " << endl; + cerr << simpleCol->toString() << endl; + cerr << " with " << endl; + cerr << constCol << endl; #endif - result = constCol; - } + result = constCol; } + } - return result; + return result; } int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_group_info& gi, bool isUnion) { #ifdef DEBUG_WALK_COND - cerr << "getGroupPlan()" << endl; + cerr << "getGroupPlan()" << endl; #endif - // rollup is currently not supported - if (select_lex.olap == ROLLUP_TYPE) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } - gwi.internalDecimalScale = (get_use_decimal_scale(gwi.thd) ? get_decimal_scale(gwi.thd) : -1); - gwi.subSelectType = csep->subType(); + gwi.internalDecimalScale = (get_use_decimal_scale(gwi.thd) ? get_decimal_scale(gwi.thd) : -1); + gwi.subSelectType = csep->subType(); - JOIN* join = select_lex.join; - Item_cond* icp = 0; + JOIN* join = select_lex.join; + Item_cond* icp = 0; - if ( gi.groupByWhere ) - icp = reinterpret_cast(gi.groupByWhere); + if (gi.groupByWhere) + icp = reinterpret_cast(gi.groupByWhere); - uint32_t sessionID = csep->sessionID(); - gwi.sessionid = sessionID; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - gwi.csc = csc; + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + gwi.csc = csc; - // @bug 2123. Override large table estimate if infinidb_ordered hint was used. - // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. - if (get_ordered_only(gwi.thd)) - csep->overrideLargeSideEstimate(true); + // @bug 2123. Override large table estimate if infinidb_ordered hint was used. + // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. + if (get_ordered_only(gwi.thd)) + csep->overrideLargeSideEstimate(true); - // @bug 5741. Set a flag when in Local PM only query mode - csep->localQuery(get_local_query(gwi.thd)); + // @bug 5741. Set a flag when in Local PM only query mode + csep->localQuery(get_local_query(gwi.thd)); - // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering - csep->stringScanThreshold(get_string_scan_threshold(gwi.thd)); + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(get_string_scan_threshold(gwi.thd)); - csep->stringTableThreshold(get_stringtable_threshold(gwi.thd)); + csep->stringTableThreshold(get_stringtable_threshold(gwi.thd)); - csep->djsSmallSideLimit(get_diskjoin_smallsidelimit(gwi.thd) * 1024ULL * 1024); - csep->djsLargeSideLimit(get_diskjoin_largesidelimit(gwi.thd) * 1024ULL * 1024); - csep->djsPartitionSize(get_diskjoin_bucketsize(gwi.thd) * 1024ULL * 1024); + csep->djsSmallSideLimit(get_diskjoin_smallsidelimit(gwi.thd) * 1024ULL * 1024); + csep->djsLargeSideLimit(get_diskjoin_largesidelimit(gwi.thd) * 1024ULL * 1024); + csep->djsPartitionSize(get_diskjoin_bucketsize(gwi.thd) * 1024ULL * 1024); - if (get_um_mem_limit(gwi.thd) == 0) - csep->umMemLimit(numeric_limits::max()); - else - csep->umMemLimit(get_um_mem_limit(gwi.thd) * 1024ULL * 1024); + if (get_um_mem_limit(gwi.thd) == 0) + csep->umMemLimit(numeric_limits::max()); + else + csep->umMemLimit(get_um_mem_limit(gwi.thd) * 1024ULL * 1024); - // populate table map and trigger syscolumn cache for all the tables (@bug 1637). - // all tables on FROM list must have at least one col in colmap - TABLE_LIST* table_ptr = gi.groupByTables; - CalpontSelectExecutionPlan::SelectList derivedTbList; + // populate table map and trigger syscolumn cache for all the tables (@bug 1637). + // all tables on FROM list must have at least one col in colmap + TABLE_LIST* table_ptr = gi.groupByTables; + CalpontSelectExecutionPlan::SelectList derivedTbList; // DEBUG #ifdef DEBUG_WALK_COND - List_iterator sj_list_it(select_lex.sj_nests); - TABLE_LIST* sj_nest; + List_iterator sj_list_it(select_lex.sj_nests); + TABLE_LIST* sj_nest; - while ((sj_nest = sj_list_it++)) - { - cerr << sj_nest->db.str << "." << sj_nest->table_name.str << endl; - } + while ((sj_nest = sj_list_it++)) + { + cerr << sj_nest->db.str << "." << sj_nest->table_name.str << endl; + } #endif - // @bug 1796. Remember table order on the FROM list. - gwi.clauseType = FROM; + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; - try + try + { + for (; table_ptr; table_ptr = table_ptr->next_local) { - for (; table_ptr; table_ptr = table_ptr->next_local) + // mysql put vtable here for from sub. we ignore it + // if (string(table_ptr->table_name).find("$vtable") != string::npos) + // continue; + + // Until we handle recursive cte: + // Checking here ensures we catch all with clauses in the query. + if (table_ptr->is_recursive_with_table()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Recursive CTE"; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + string viewName = getViewName(table_ptr); + if (lower_case_table_names) + { + boost::algorithm::to_lower(viewName); + } + + // @todo process from subquery + if (table_ptr->derived) + { + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_ORDINARY); + + SELECT_LEX* select_cursor = table_ptr->derived->first_select(); + // Use Pushdown handler for subquery processing + FromSubQuery fromSub(gwi, select_cursor); + string alias(table_ptr->alias.str); + if (lower_case_table_names) { - // mysql put vtable here for from sub. we ignore it - //if (string(table_ptr->table_name).find("$vtable") != string::npos) - // continue; + boost::algorithm::to_lower(alias); + } + fromSub.alias(alias); - // Until we handle recursive cte: - // Checking here ensures we catch all with clauses in the query. - if (table_ptr->is_recursive_with_table()) + CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); + // @bug 3852. check return execplan + SCSEP plan = fromSub.transform(); + + if (!plan) + { + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + gwi.derivedTbList.push_back(plan); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); + gwi.tableMap[tan] = make_pair(0, table_ptr); + // MCOL-2178 isUnion member only assigned, never used + // MIGR::infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + } + else if (table_ptr->view) + { + View* view = new View(*table_ptr->view->first_select_lex(), &gwi); + CalpontSystemCatalog::TableAliasName tn = make_aliastable( + table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, true, lower_case_table_names); + view->viewName(tn); + gwi.viewList.push_back(view); + view->transform(); + } + else + { + // check foreign engine tables + bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true); + + // trigger system catalog cache + if (columnStore) + csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str, lower_case_table_names), + true); + + string table_name = table_ptr->table_name.str; + + // @bug5523 + if (table_ptr->db.length && strcmp(table_ptr->db.str, "information_schema") == 0) + table_name = + (table_ptr->schema_table_name.length ? table_ptr->schema_table_name.str : table_ptr->alias.str); + + CalpontSystemCatalog::TableAliasName tn = + make_aliasview(table_ptr->db.str, table_name, table_ptr->alias.str, viewName, columnStore, + lower_case_table_names); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable( + table_ptr->db.str, table_name, table_ptr->alias.str, columnStore, lower_case_table_names); + gwi.tableMap[tan] = make_pair(0, table_ptr); +#ifdef DEBUG_WALK_COND + cerr << tn << endl; +#endif + } + } + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + catch (IDBExcept& ie) + { + setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + // @bug 3852. set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = ie.what(); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + // @bug3852 set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + csep->tableList(gwi.tbList); + + bool unionSel = false; + + gwi.clauseType = WHERE; + + if (icp) + { + // MCOL-1052 The condition could be useless. + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". + //#if MYSQL_VERSION_ID < 50172 + //@bug 3039. fix fields for constants + if (!icp->fixed()) + { + icp->fix_fields(gwi.thd, (Item**)&icp); + } + + //#endif + gwi.fatalParseError = false; +#ifdef DEBUG_WALK_COND + cerr << "------------------ WHERE -----------------------" << endl; + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + // if this is dervied table process phase, mysql may have not developed the plan + // completely. Do not error and eventually mysql will call JOIN::exec() again. + // related to bug 2922. Need to find a way to skip calling rnd_init for derived table + // processing. + if (gwi.thd->derived_tables_processing) + { + // MCOL-2178 isUnion member only assigned, never used + // MIGR::infinidb_vtable.isUnion = false; + return -1; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + else if (join && join->zero_result_cause) + { + gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + (dynamic_cast(gwi.rcWorkStack.top())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + } + + SELECT_LEX tmp_select_lex; + tmp_select_lex.table_list.first = gi.groupByTables; + + // InfiniDB bug5764 requires outer joins to be appended to the + // end of the filter list. This causes outer join filters to + // have a higher join id than inner join filters. + // TODO MCOL-4680 Figure out why this is the case, and possibly + // eliminate this requirement. + std::stack outerJoinStack; + + uint32_t failed = buildJoin(gwi, tmp_select_lex.top_join_list, outerJoinStack); + + if (failed) + return failed; + + if (gwi.subQuery) + { + for (uint i = 0; i < gwi.viewList.size(); i++) + { + failed = gwi.viewList[i]->processJoin(gwi, outerJoinStack); + + if (failed) + break; + } + } + + if (failed != 0) + return failed; + + ParseTree* filters = NULL; + ParseTree* outerJoinFilters = NULL; + ParseTree* ptp = NULL; + ParseTree* rhs = NULL; + + // @bug 2932. for "select * from region where r_name" case. if icp not null and + // ptWorkStack empty, the item is in rcWorkStack. + // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 + // is pushed to rcWorkStack. + if (/*icp && */ gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + filters = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + filters = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } + + while (!outerJoinStack.empty()) + { + outerJoinFilters = outerJoinStack.top(); + outerJoinStack.pop(); + + if (outerJoinStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(outerJoinFilters); + rhs = outerJoinStack.top(); + outerJoinStack.pop(); + ptp->right(rhs); + outerJoinStack.push(ptp); + } + + // Append outer join filters at the end of inner join filters. + // JLF_ExecPlanToJobList::walkTree processes ParseTree::left + // before ParseTree::right which is what we intend to do in the + // below. + if (filters && outerJoinFilters) + { + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + ptp->right(outerJoinFilters); + filters = ptp; + } + else if (outerJoinFilters) + { + filters = outerJoinFilters; + } + + if (filters) + { + csep->filters(filters); +#ifdef DEBUG_WALK_COND + std::string aTmpDir(startup::StartUp::tmpDir()); + aTmpDir = aTmpDir + "/filter1.dot"; + filters->drawTree(aTmpDir); +#endif + } + + gwi.clauseType = SELECT; +#ifdef DEBUG_WALK_COND + { + cerr << "------------------- SELECT --------------------" << endl; + List_iterator_fast it(*gi.groupByFields); + Item* item; + + while ((item = it++)) + { + debug_walk(item, 0); + } + + cerr << "-----------------------------------------------\n" << endl; + } +#endif + + // populate returnedcolumnlist and columnmap + List_iterator_fast it(*gi.groupByFields); + Item* item; + vector funcFieldVec; + bool redo = false; + + // empty rcWorkStack and ptWorkStack. They should all be empty by now. + clearStacks(gwi); + + // indicate the starting pos of scalar returned column, because some join column + // has been inserted to the returned column list. + if (gwi.subQuery) + { + ScalarSub* scalar = dynamic_cast(gwi.subQuery); + + if (scalar) + scalar->returnedColPos(gwi.additionalRetCols.size()); + } + + CalpontSelectExecutionPlan::SelectList selectSubList; + + while ((item = it++)) + { + string itemAlias; + if (item->name.length) + itemAlias = (item->name.str); + else + { + itemAlias = ""; + } + + // @bug 5916. Need to keep checking until getting concret item in case + // of nested view. + while (item->type() == Item::REF_ITEM) + { + Item_ref* ref = (Item_ref*)item; + item = (*(ref->ref)); + } + + Item::Type itype = item->type(); + + switch (itype) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + SimpleColumn* sc = NULL; + ConstantColumn* constCol = NULL; + + if (ifp->field_name.length && string(ifp->field_name.str) == "*") + { + collectAllCols(gwi, ifp); + break; + } + + sc = buildSimpleColumn(ifp, gwi); + + if (sc) + { + constCol = buildConstColFromFilter(sc, gwi, gi); + boost::shared_ptr spcc(constCol); + boost::shared_ptr spsc(sc); + + string fullname; + String str; + ifp->print(&str, QT_ORDINARY); + fullname = str.c_ptr(); + + if (!ifp->is_explicit_name()) // no alias + { + sc->alias(fullname); + } + else // alias + { + if (!itemAlias.empty()) + sc->alias(itemAlias); + } + + // MCOL-1052 Replace SimpleColumn with ConstantColumn, + // since it must have a single value only. + if (constCol) + { + gwi.returnedCols.push_back(spcc); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), spcc)); + } + else + { + gwi.returnedCols.push_back(spsc); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), spsc)); + } + + TABLE_LIST* tmp = 0; + + if (ifp->cached_table) + tmp = ifp->cached_table; + + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), + sc->isColumnStore())] = make_pair(1, tmp); + } + else + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + delete sc; + return ER_INTERNAL_ERROR; + } + + break; + } + + // aggregate column + case Item::SUM_FUNC_ITEM: + { + ReturnedColumn* ac = buildAggregateColumn(item, gwi); + + if (gwi.fatalParseError) + { + // e.g., non-support ref column + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + delete ac; + return ER_CHECK_NOT_IMPLEMENTED; + } + + // add this agg col to returnedColumnList + boost::shared_ptr spac(ac); + gwi.returnedCols.push_back(spac); + // This item could be used in projection or HAVING later. + gwi.extSelAggColsItems.push_back(item); + + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = reinterpret_cast(item); + + // @bug4383. error out non-support stored function + if (ifp->functype() == Item_func::FUNC_SP) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + if (string(ifp->func_name()) == "xor") + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + uint16_t parseInfo = 0; + vector tmpVec; + bool hasNonSupportItem = false; + parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo, &gwi); + + if (ifp->with_subquery() || string(ifp->func_name()) == string("") || + ifp->functype() == Item_func::NOT_ALL_FUNC || parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + ReturnedColumn* rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, true); + SRCP srcp(rc); + + if (rc) + { + if (!hasNonSupportItem && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && tmpVec.size() == 0) + { + if (isUnion || unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT || + parseInfo & SUB_BIT) //|| select_lex.group_list.elements != 0) { - gwi.fatalParseError = true; - gwi.parseErrorText = "Recursive CTE"; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; + srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (ifp->name.length) + srcp->alias(ifp->name.str); + + continue; } - string viewName = getViewName(table_ptr); - if (lower_case_table_names) + break; + } + + gwi.returnedCols.push_back(srcp); + } + else // InfiniDB Non support functions still go through post process for now + { + hasNonSupportItem = false; + uint32_t before_size = funcFieldVec.size(); + // MCOL-1510 Use gwi pointer here to catch funcs with + // not supported aggregate args in projections, + // e.g. NOT(SUM(i)). + parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo, &gwi); + uint32_t after_size = funcFieldVec.size(); + + // group by func and func in subquery can not be post processed + // @bug3881. set_user_var can not be treated as constant function + // @bug5716. Try to avoid post process function for union query. + if ((gwi.subQuery /*|| select_lex.group_list.elements != 0 */ || !csep->unionVec().empty() || + isUnion) && + !hasNonSupportItem && (after_size - before_size) == 0 && !(parseInfo & AGG_BIT) && + !(parseInfo & SUB_BIT)) + { + ConstantColumn* cc = buildConstantColumnMaybeNullUsingValStr(ifp, gwi); + + SRCP srcp(cc); + + if (ifp->name.length) + cc->alias(ifp->name.str); + + gwi.returnedCols.push_back(srcp); + + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + break; + } + else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || + (gwi.fatalParseError && gwi.subQuery)) + { + if (gwi.parseErrorText.empty()) { - boost::algorithm::to_lower(viewName); + Message::Args args; + args.add(ifp->func_name()); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); } - // @todo process from subquery - if (table_ptr->derived) + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else if (gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM)) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + //@Bug 3030 Add error check for dml statement + if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) + { + if (after_size - before_size != 0) { - String str; - (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_ORDINARY); - - SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - // Use Pushdown handler for subquery processing - FromSubQuery fromSub(gwi, select_cursor); - string alias(table_ptr->alias.str); - if (lower_case_table_names) - { - boost::algorithm::to_lower(alias); - } - fromSub.alias(alias); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); - // @bug 3852. check return execplan - SCSEP plan = fromSub.transform(); - - if (!plan) - { - setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - gwi.derivedTbList.push_back(plan); - gwi.tbList.push_back(tn); - CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); - gwi.tableMap[tan] = make_pair(0, table_ptr); - // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + gwi.parseErrorText = ifp->func_name(); + return -1; } - else if (table_ptr->view) + } + else + { + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + } + } + + break; + } + + // DRRTUY Replace the whole section with typeid() checks or use + // reinterpret_cast here + case Item::CONST_ITEM: + { + switch (item->cmp_type()) + { + case INT_RESULT: + case STRING_RESULT: + case DECIMAL_RESULT: + case REAL_RESULT: + case TIME_RESULT: + { + if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) { - View* view = new View(*table_ptr->view->first_select_lex(), &gwi); - CalpontSystemCatalog::TableAliasName tn = make_aliastable(table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, true, lower_case_table_names); - view->viewName(tn); - gwi.viewList.push_back(view); - view->transform(); } else { - // check foreign engine tables - bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true); + // do not push the dummy column (mysql added) to returnedCol + if (item->name.length && string(item->name.str) == "Not_used") + continue; - // trigger system catalog cache - if (columnStore) - csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str, lower_case_table_names), true); + // @bug3509. Constant column is sent to ExeMgr now. + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - string table_name = table_ptr->table_name.str; + if (item->name.length) + srcp->alias(item->name.str); - // @bug5523 - if (table_ptr->db.length && strcmp(table_ptr->db.str, "information_schema") == 0) - table_name = (table_ptr->schema_table_name.length ? table_ptr->schema_table_name.str : table_ptr->alias.str); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db.str, table_name, table_ptr->alias.str, viewName, columnStore, lower_case_table_names); - gwi.tbList.push_back(tn); - CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db.str, table_name, table_ptr->alias.str, columnStore, lower_case_table_names); - gwi.tableMap[tan] = make_pair(0, table_ptr); -#ifdef DEBUG_WALK_COND - cerr << tn << endl; -#endif - } - } - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - } - catch (IDBExcept& ie) - { - setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - // @bug 3852. set error status for gwi. - gwi.fatalParseError = true; - gwi.parseErrorText = ie.what(); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - // @bug3852 set error status for gwi. - gwi.fatalParseError = true; - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - csep->tableList(gwi.tbList); - - bool unionSel = false; - - gwi.clauseType = WHERE; - - - if (icp) - { - // MCOL-1052 The condition could be useless. - // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". - //#if MYSQL_VERSION_ID < 50172 - //@bug 3039. fix fields for constants - if (!icp->fixed()) - { - icp->fix_fields(gwi.thd, (Item**)&icp); - } - -//#endif - gwi.fatalParseError = false; -#ifdef DEBUG_WALK_COND - cerr << "------------------ WHERE -----------------------" << endl; - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cerr << "------------------------------------------------\n" << endl; -#endif - - icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - // if this is dervied table process phase, mysql may have not developed the plan - // completely. Do not error and eventually mysql will call JOIN::exec() again. - // related to bug 2922. Need to find a way to skip calling rnd_init for derived table - // processing. - if (gwi.thd->derived_tables_processing) - { - // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = false; - return -1; + gwi.returnedCols.push_back(srcp); } - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; + break; + } + + // MCOL-2178 This switch doesn't handl + // ROW_ + default: + { + IDEBUG(cerr << "Warning unsupported cmp_type() in projection" << endl); + } } - } - else if (join && join->zero_result_cause) - { - gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - } + break; + } // CONST_ITEM ends here - SELECT_LEX tmp_select_lex; - tmp_select_lex.table_list.first = gi.groupByTables; - - // InfiniDB bug5764 requires outer joins to be appended to the - // end of the filter list. This causes outer join filters to - // have a higher join id than inner join filters. - // TODO MCOL-4680 Figure out why this is the case, and possibly - // eliminate this requirement. - std::stack outerJoinStack; - - uint32_t failed = buildJoin(gwi, tmp_select_lex.top_join_list, outerJoinStack); - - if (failed) return failed; - - if (gwi.subQuery) - { - for (uint i = 0; i < gwi.viewList.size(); i++) + case Item::NULL_ITEM: + { + if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) { - failed = gwi.viewList[i]->processJoin(gwi, outerJoinStack); - - if (failed) - break; } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name.length) + srcp->alias(item->name.str); + } + + break; + } + + case Item::SUBSELECT_ITEM: + { + Item_subselect* sub = (Item_subselect*)item; + + if (sub->substype() != Item_subselect::SINGLEROW_SUBS) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + +#ifdef DEBUG_WALK_COND + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + JOIN* join = sub->get_select_lex()->join; + + if (join) + { + Item_cond* cond = reinterpret_cast(join->conds); + + if (cond) + cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + + cerr << "Finish SELECT clause subselect item traversing" << endl; +#endif + SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); + // selectSub->gwip(&gwi); + SCSEP ssub = selectSub->transform(); + + if (!ssub || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + selectSubList.push_back(ssub); + SimpleColumn* rc = new SimpleColumn(); + rc->colSource(rc->colSource() | SELECT_SUB); + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + + if (sub->get_select_lex()->get_table_list()) + { + rc->viewName(getViewName(sub->get_select_lex()->get_table_list()), lower_case_table_names); + } + if (sub->name.length) + rc->alias(sub->name.str); + + gwi.returnedCols.push_back(SRCP(rc)); + + break; + } + + case Item::COND_ITEM: + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::EXPR_CACHE_ITEM: + { + printf("EXPR_CACHE_ITEM in getSelectPlan\n"); + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::WINDOW_FUNC_ITEM: + { + SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); + + if (!srcp || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.returnedCols.push_back(srcp); + break; + } + + default: + { + break; + } + } + } + + // @bug4388 normalize the project coltypes for union main select list + if (!csep->unionVec().empty()) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + vector coltypes; + + for (uint32_t j = 0; j < csep->unionVec().size(); j++) + { + coltypes.push_back(dynamic_cast(csep->unionVec()[j].get()) + ->returnedCols()[i] + ->resultType()); + + // @bug5976. set hasAggregate true for the main column if + // one corresponding union column has aggregate + if (dynamic_cast(csep->unionVec()[j].get()) + ->returnedCols()[i] + ->hasAggregate()) + gwi.returnedCols[i]->hasAggregate(true); + } + + gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); + } + } + + // Having clause handling + gwi.clauseType = HAVING; + clearStacks(gwi); + ParseTree* havingFilter = 0; + // clear fatalParseError that may be left from post process functions + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + + if (gi.groupByHaving != 0) + { + Item_cond* having = reinterpret_cast(gi.groupByHaving); +#ifdef DEBUG_WALK_COND + cerr << "------------------- HAVING ---------------------" << endl; + having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; } - if (failed != 0) - return failed; + ParseTree* ptp = 0; + ParseTree* rhs = 0; - ParseTree* filters = NULL; - ParseTree* outerJoinFilters = NULL; - ParseTree* ptp = NULL; - ParseTree* rhs = NULL; - - // @bug 2932. for "select * from region where r_name" case. if icp not null and - // ptWorkStack empty, the item is in rcWorkStack. - // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 - // is pushed to rcWorkStack. - if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + // @bug 4215. some function filter will be in the rcWorkStack. + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) { - filters = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); + havingFilter = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); } while (!gwi.ptWorkStack.empty()) { - filters = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); + havingFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); - if (gwi.ptWorkStack.empty()) - break; + if (gwi.ptWorkStack.empty()) + break; - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - rhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - ptp->right(rhs); - gwi.ptWorkStack.push(ptp); + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(havingFilter); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } + } + + // for post process expressions on the select list + // error out post process for union and sub select unit + if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (funcFieldVec.size() != 0 && !gwi.fatalParseError) + { + string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + for (uint32_t i = 0; i < funcFieldVec.size(); i++) + { + SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); + + if (!sc || gwi.fatalParseError) + { + string emsg; + + if (gwi.parseErrorText.empty()) + { + emsg = "un-recognized column"; + + if (funcFieldVec[i]->name.length) + emsg += string(funcFieldVec[i]->name.str); + } + else + { + emsg = gwi.parseErrorText; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ER_INTERNAL_ERROR; } - while (!outerJoinStack.empty()) + String str; + funcFieldVec[i]->print(&str, QT_ORDINARY); + sc->alias(string(str.c_ptr())); + // sc->tableAlias(funcFieldVec[i]->table_name); + sc->tableAlias(sc->alias()); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) { - outerJoinFilters = outerJoinStack.top(); - outerJoinStack.pop(); + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - if (outerJoinStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(outerJoinFilters); - rhs = outerJoinStack.top(); - outerJoinStack.pop(); - ptp->right(rhs); - outerJoinStack.push(ptp); + if (field && field->alias() == sc->alias()) + break; + } } - // Append outer join filters at the end of inner join filters. - // JLF_ExecPlanToJobList::walkTree processes ParseTree::left - // before ParseTree::right which is what we intend to do in the - // below. - if (filters && outerJoinFilters) + if (j == gwi.returnedCols.size()) { - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - ptp->right(outerJoinFilters); - filters = ptp; + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name.str), srcp)); + + string fullname; + fullname = str.c_ptr(); + TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), + sc->isColumnStore())] = make_pair(1, tmp); } - else if (outerJoinFilters) + } + + // post-process Order by list and expressions on select by redo phase1. only for vtable + // ignore ORDER BY clause for union select unit + string ord_cols = ""; // for normal select phase + SRCP minSc; // min width projected column. for count(*) use + + // Group by list. not valid for union main query + if (!unionSel) + { + gwi.clauseType = GROUP_BY; + Item* nonSupportItem = NULL; + ORDER* groupcol = reinterpret_cast(gi.groupByGroup); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + bool hasWindowFunc = gwi.hasWindowFunc; + gwi.hasWindowFunc = false; + + for (; groupcol; groupcol = groupcol->next) { - filters = outerJoinFilters; + if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; } - if (filters) + if (gwi.hasWindowFunc) { - csep->filters(filters); -#ifdef DEBUG_WALK_COND - std::string aTmpDir(startup::StartUp::tmpDir()); - aTmpDir = aTmpDir + "/filter1.dot"; - filters->drawTree(aTmpDir); -#endif + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; } - gwi.clauseType = SELECT; -#ifdef DEBUG_WALK_COND - { - cerr << "------------------- SELECT --------------------" << endl; - List_iterator_fast it(*gi.groupByFields); - Item* item; + gwi.hasWindowFunc = hasWindowFunc; + groupcol = reinterpret_cast(gi.groupByGroup); - while ((item = it++)) + for (; groupcol; groupcol = groupcol->next) + { + Item* groupItem = *(groupcol->item); + + // @bug5993. Could be nested ref. + while (groupItem->type() == Item::REF_ITEM) + groupItem = (*((Item_ref*)groupItem)->ref); + + if (groupItem->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)groupItem; + + // call buildFunctionColumn here mostly for finding out + // non-support column on GB list. Should be simplified. + ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (!fc || gwi.fatalParseError) { - debug_walk(item, 0); + nonSupportItem = ifp; + break; } - cerr << "-----------------------------------------------\n" << endl; - } -#endif + if (groupcol->in_field_list && groupcol->counter_used) + { + delete fc; + fc = gwi.returnedCols[groupcol->counter - 1].get(); + SRCP srcp(fc->clone()); - // populate returnedcolumnlist and columnmap - List_iterator_fast it(*gi.groupByFields); - Item* item; - vector funcFieldVec; - bool redo = false; + // check if no column parm + for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) + { + if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) + { + gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); + break; + } + } - // empty rcWorkStack and ptWorkStack. They should all be empty by now. - clearStacks(gwi); + srcp->orderPos(groupcol->counter - 1); + gwi.groupByCols.push_back(srcp); + continue; + } + else if (groupItem->is_explicit_name()) // alias + { + uint32_t i = 0; - // indicate the starting pos of scalar returned column, because some join column - // has been inserted to the returned column list. - if (gwi.subQuery) - { - ScalarSub* scalar = dynamic_cast(gwi.subQuery); + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } - if (scalar) - scalar->returnedColPos(gwi.additionalRetCols.size()); - } - - CalpontSelectExecutionPlan::SelectList selectSubList; - - while ((item = it++)) - { - string itemAlias; - if(item->name.length) - itemAlias = (item->name.str); + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + break; + } + } else { - itemAlias = ""; + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + gwi.groupByCols.push_back(SRCP(fc)); + break; + } } + } + else if (groupItem->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)groupItem; + // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. + ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); + SimpleColumn* sc = dynamic_cast(rc); - // @bug 5916. Need to keep checking until getting concret item in case - // of nested view. - while (item->type() == Item::REF_ITEM) - { - Item_ref* ref = (Item_ref*)item; - item = (*(ref->ref)); - } - - Item::Type itype = item->type(); - - switch (itype) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = (Item_field*)item; - SimpleColumn* sc = NULL; - ConstantColumn* constCol = NULL; - - if (ifp->field_name.length && string(ifp->field_name.str) == "*") - { - collectAllCols(gwi, ifp); - break; - } - - sc = buildSimpleColumn(ifp, gwi); - - if (sc) - { - constCol = buildConstColFromFilter(sc, gwi, gi); - boost::shared_ptr spcc(constCol); - boost::shared_ptr spsc(sc); - - string fullname; - String str; - ifp->print(&str, QT_ORDINARY); - fullname = str.c_ptr(); - - if (!ifp->is_explicit_name()) // no alias - { - sc->alias(fullname); - } - else // alias - { - if (!itemAlias.empty()) - sc->alias(itemAlias); - - } - - // MCOL-1052 Replace SimpleColumn with ConstantColumn, - // since it must have a single value only. - if (constCol) - { - gwi.returnedCols.push_back(spcc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), spcc)); - } - else - { - gwi.returnedCols.push_back(spsc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), spsc)); - } - - TABLE_LIST* tmp = 0; - - if (ifp->cached_table) - tmp = ifp->cached_table; - - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, tmp); - } - else - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - delete sc; - return ER_INTERNAL_ERROR; - } - - break; - } - - //aggregate column - case Item::SUM_FUNC_ITEM: - { - ReturnedColumn* ac = buildAggregateColumn(item, gwi); - - if (gwi.fatalParseError) - { - // e.g., non-support ref column - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - delete ac; - return ER_CHECK_NOT_IMPLEMENTED; - } - - // add this agg col to returnedColumnList - boost::shared_ptr spac(ac); - gwi.returnedCols.push_back(spac); - // This item could be used in projection or HAVING later. - gwi.extSelAggColsItems.push_back(item); - - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = reinterpret_cast(item); - - // @bug4383. error out non-support stored function - if (ifp->functype() == Item_func::FUNC_SP) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - if (string(ifp->func_name()) == "xor") - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - uint16_t parseInfo = 0; - vector tmpVec; - bool hasNonSupportItem = false; - parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo, &gwi); - - if (ifp->with_subquery() || - string(ifp->func_name()) == string("") || - ifp->functype() == Item_func::NOT_ALL_FUNC || - parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - ReturnedColumn* rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, true); - SRCP srcp(rc); - - if (rc) - { - if (!hasNonSupportItem && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && tmpVec.size() == 0) - { - if (isUnion || unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT || - parseInfo & SUB_BIT ) //|| select_lex.group_list.elements != 0) - { - srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (ifp->name.length) - srcp->alias(ifp->name.str); - - continue; - } - - break; - } - - gwi.returnedCols.push_back(srcp); - - } - else // InfiniDB Non support functions still go through post process for now - { - hasNonSupportItem = false; - uint32_t before_size = funcFieldVec.size(); - // MCOL-1510 Use gwi pointer here to catch funcs with - // not supported aggregate args in projections, - // e.g. NOT(SUM(i)). - parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo, &gwi); - uint32_t after_size = funcFieldVec.size(); - - // group by func and func in subquery can not be post processed - // @bug3881. set_user_var can not be treated as constant function - // @bug5716. Try to avoid post process function for union query. - if ((gwi.subQuery /*|| select_lex.group_list.elements != 0 */ || - !csep->unionVec().empty() || isUnion) && - !hasNonSupportItem && (after_size - before_size) == 0 && - !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) - ) - { - ConstantColumn* cc = buildConstantColumnMaybeNullUsingValStr(ifp, gwi); - - SRCP srcp(cc); - - if (ifp->name.length) - cc->alias(ifp->name.str); - - gwi.returnedCols.push_back(srcp); - - // clear the error set by buildFunctionColumn - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - break; - } - else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || - (gwi.fatalParseError && gwi.subQuery)) - { - if (gwi.parseErrorText.empty()) - { - Message::Args args; - args.add(ifp->func_name()); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - } - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else if ( gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM )) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - //@Bug 3030 Add error check for dml statement - if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { - if ( after_size - before_size != 0 ) - { - gwi.parseErrorText = ifp->func_name(); - return -1; - } - } - else - { - // clear the error set by buildFunctionColumn - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - } - } - - break; - } - - // DRRTUY Replace the whole section with typeid() checks or use - // reinterpret_cast here - case Item::CONST_ITEM: - { - switch(item->cmp_type()) - { - case INT_RESULT: - case STRING_RESULT: - case DECIMAL_RESULT: - case REAL_RESULT: - case TIME_RESULT: - { - if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { } - else - { - // do not push the dummy column (mysql added) to returnedCol - if (item->name.length && string(item->name.str) == "Not_used") - continue; - - // @bug3509. Constant column is sent to ExeMgr now. - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - - if (item->name.length) - srcp->alias(item->name.str); - - gwi.returnedCols.push_back(srcp); - } - - break; - } - - // MCOL-2178 This switch doesn't handl - // ROW_ - default: - { - IDEBUG(cerr << "Warning unsupported cmp_type() in projection" << endl); - } - } - break; - } // CONST_ITEM ends here - - case Item::NULL_ITEM: - { - if (isUpdateOrDeleteStatement(gwi.thd->lex->sql_command)) - { } - else - { - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (item->name.length) - srcp->alias(item->name.str); - } - - break; - } - - case Item::SUBSELECT_ITEM: - { - Item_subselect* sub = (Item_subselect*)item; - - if (sub->substype() != Item_subselect::SINGLEROW_SUBS) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - -#ifdef DEBUG_WALK_COND - cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; - JOIN* join = sub->get_select_lex()->join; - - if (join) - { - Item_cond* cond = reinterpret_cast(join->conds); - - if (cond) - cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } - - cerr << "Finish SELECT clause subselect item traversing" << endl; -#endif - SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); - //selectSub->gwip(&gwi); - SCSEP ssub = selectSub->transform(); - - if (!ssub || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported Item in SELECT subquery."; - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - selectSubList.push_back(ssub); - SimpleColumn* rc = new SimpleColumn(); - rc->colSource(rc->colSource() | SELECT_SUB); - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - - if (sub->get_select_lex()->get_table_list()) - { - rc->viewName(getViewName(sub->get_select_lex()->get_table_list()), lower_case_table_names); - } - if (sub->name.length) - rc->alias(sub->name.str); - - gwi.returnedCols.push_back(SRCP(rc)); - - break; - } - - case Item::COND_ITEM: - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - case Item::EXPR_CACHE_ITEM: - { - printf("EXPR_CACHE_ITEM in getSelectPlan\n"); - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - case Item::WINDOW_FUNC_ITEM: - { - SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); - - if (!srcp || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported Item in SELECT subquery."; - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.returnedCols.push_back(srcp); - break; - } - - default: - { - break; - } - } - } - - // @bug4388 normalize the project coltypes for union main select list - if (!csep->unionVec().empty()) - { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - vector coltypes; - - for (uint32_t j = 0; j < csep->unionVec().size(); j++) - { - coltypes.push_back( - dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->resultType()); - - // @bug5976. set hasAggregate true for the main column if - // one corresponding union column has aggregate - if (dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->hasAggregate()) - gwi.returnedCols[i]->hasAggregate(true); - } - - gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); - } - } - - // Having clause handling - gwi.clauseType = HAVING; - clearStacks(gwi); - ParseTree* havingFilter = 0; - // clear fatalParseError that may be left from post process functions - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - - if (gi.groupByHaving != 0) - { - Item_cond* having = reinterpret_cast(gi.groupByHaving); -#ifdef DEBUG_WALK_COND - cerr << "------------------- HAVING ---------------------" << endl; - having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cerr << "------------------------------------------------\n" << endl; -#endif - having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - - ParseTree* ptp = 0; - ParseTree* rhs = 0; - - // @bug 4215. some function filter will be in the rcWorkStack. - if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) - { - havingFilter = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } - - while (!gwi.ptWorkStack.empty()) - { - havingFilter = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - - if (gwi.ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(havingFilter); - rhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - ptp->right(rhs); - gwi.ptWorkStack.push(ptp); - } - } - - // for post process expressions on the select list - // error out post process for union and sub select unit - if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - { - if (funcFieldVec.size() != 0 && !gwi.fatalParseError) - { - string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - for (uint32_t i = 0; i < funcFieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); - - if (!sc || gwi.fatalParseError) - { - string emsg; - - if (gwi.parseErrorText.empty()) - { - emsg = "un-recognized column"; - - if (funcFieldVec[i]->name.length) - emsg += string(funcFieldVec[i]->name.str); - } - else - { - emsg = gwi.parseErrorText; - } - - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ER_INTERNAL_ERROR; - } - - String str; - funcFieldVec[i]->print(&str, QT_ORDINARY); - sc->alias(string(str.c_ptr())); - //sc->tableAlias(funcFieldVec[i]->table_name); - sc->tableAlias(sc->alias()); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) { + if (sc) + { if (sc->sameColumn(gwi.returnedCols[j].get())) { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; + sc->orderPos(j); + break; } - } - - if (j == gwi.returnedCols.size()) - { - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name.str), srcp)); - - string fullname; - fullname = str.c_ptr(); - TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, tmp); - } - } - - // post-process Order by list and expressions on select by redo phase1. only for vtable - // ignore ORDER BY clause for union select unit - string ord_cols = ""; // for normal select phase - SRCP minSc; // min width projected column. for count(*) use - - // Group by list. not valid for union main query - if (!unionSel) - { - gwi.clauseType = GROUP_BY; - Item* nonSupportItem = NULL; - ORDER* groupcol = reinterpret_cast(gi.groupByGroup); - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - bool hasWindowFunc = gwi.hasWindowFunc; - gwi.hasWindowFunc = false; - - for (; groupcol; groupcol = groupcol->next) - { - if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; - } - - if (gwi.hasWindowFunc) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.hasWindowFunc = hasWindowFunc; - groupcol = reinterpret_cast(gi.groupByGroup); - - for (; groupcol; groupcol = groupcol->next) - { - Item* groupItem = *(groupcol->item); - - // @bug5993. Could be nested ref. - while (groupItem->type() == Item::REF_ITEM) - groupItem = (*((Item_ref*)groupItem)->ref); - - if (groupItem->type() == Item::FUNC_ITEM) + else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) { - Item_func* ifp = (Item_func*)groupItem; - - // call buildFunctionColumn here mostly for finding out - // non-support column on GB list. Should be simplified. - ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (!fc || gwi.fatalParseError) - { - nonSupportItem = ifp; - break; - } - - if (groupcol->in_field_list && groupcol->counter_used) - { - delete fc; - fc = gwi.returnedCols[groupcol->counter - 1].get(); - SRCP srcp(fc->clone()); - - // check if no column parm - for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) - { - if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) - { - gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); - break; - } - } - - srcp->orderPos(groupcol->counter - 1); - gwi.groupByCols.push_back(srcp); - continue; - } - else if (groupItem->is_explicit_name()) // alias - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - delete fc; - break; - } - } - - if (i == gwi.returnedCols.size()) - { - nonSupportItem = groupItem; - break; - } - } - else - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (fc->operator==(gwi.returnedCols[i].get())) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - delete fc; - break; - } - } - - if (i == gwi.returnedCols.size()) - { - gwi.groupByCols.push_back(SRCP(fc)); - break; - } - } + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; } - else if (groupItem->type() == Item::FIELD_ITEM) + } + else + { + if (ifp->name.length && string(ifp->name.str) == gwi.returnedCols[j].get()->alias()) { - Item_field* ifp = (Item_field*)groupItem; - // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. - ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); - SimpleColumn* sc = dynamic_cast(rc); - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (sc) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - sc->orderPos(j); - break; - } - else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - else - { - if (ifp->name.length && string(ifp->name.str) == gwi.returnedCols[j].get()->alias()) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - } - - if (!rc) - { - nonSupportItem = ifp; - break; - } - - SRCP srcp(rc); - - // bug 3151 - AggregateColumn* ac = dynamic_cast(rc); - - if (ac) - { - nonSupportItem = ifp; - break; - } - - gwi.groupByCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), srcp)); + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; } - // @bug5638. The group by column is constant but not counter, alias has to match a column - // on the select list - else if (!groupcol->counter_used && - (groupItem->type() == Item::CONST_ITEM && - (groupItem->cmp_type() == INT_RESULT || - groupItem->cmp_type() == STRING_RESULT || - groupItem->cmp_type() == REAL_RESULT || - groupItem->cmp_type() == DECIMAL_RESULT) - ) - ) - - { - ReturnedColumn* rc = 0; - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (groupItem->name.length && string(groupItem->name.str) == gwi.returnedCols[j].get()->alias()) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - - if (!rc) - { - nonSupportItem = groupItem; - break; - } - - gwi.groupByCols.push_back(SRCP(rc)); - } - else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) - { - if (!groupcol->in_field_list || !groupItem->name.length) - { - nonSupportItem = groupItem; - } - else - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - break; - } - } - - if (i == gwi.returnedCols.size()) - { - nonSupportItem = groupItem; - } - } - } - // @bug 3761. - else if (groupcol->counter_used) - { - if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) - { - nonSupportItem = groupItem; - } - else - { - gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); - } - } - else - { - nonSupportItem = groupItem; - } - + } } - // @bug 4756. Add internal groupby column for correlated join to the groupby list - if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) - gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); - - // this is window func on SELECT becuase ORDER BY has not been processed - if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) + if (!rc) { - for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) - { - if (gwi.windowFuncList[i]->hasWindowFunc()) - { - vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); - - for (uint32_t j = 0; j < windowFunctions.size(); j++) - windowFunctions[j]->addToPartition(gwi.subGroupByCols); - } - } + nonSupportItem = ifp; + break; } - if (nonSupportItem) + SRCP srcp(rc); + + // bug 3151 + AggregateColumn* ac = dynamic_cast(rc); + + if (ac) { - Message::Args args; - - if (nonSupportItem->name.length) - args.add("'" + string(nonSupportItem->name.str) + "'"); - else - args.add(""); - - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } // GROUP processing ends here - - - // ORDER BY processing starts here - { - ORDER* ordercol = reinterpret_cast(gi.groupByOrder); - - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - for (; ordercol; ordercol = ordercol->next) - { - if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; + nonSupportItem = ifp; + break; } - // re-visit the first of ordercol list - ordercol = reinterpret_cast(gi.groupByOrder); + gwi.groupByCols.push_back(srcp); + gwi.columnMap.insert( + CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), srcp)); + } + // @bug5638. The group by column is constant but not counter, alias has to match a column + // on the select list + else if (!groupcol->counter_used && + (groupItem->type() == Item::CONST_ITEM && + (groupItem->cmp_type() == INT_RESULT || groupItem->cmp_type() == STRING_RESULT || + groupItem->cmp_type() == REAL_RESULT || groupItem->cmp_type() == DECIMAL_RESULT))) - // for subquery, order+limit by will be supported in infinidb. build order by columns - // @todo union order by and limit support - //if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + ReturnedColumn* rc = 0; - for (; ordercol; ordercol = ordercol->next) + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) { - ReturnedColumn* rc = NULL; - - if (ordercol->in_field_list && ordercol->counter_used) - { - rc = gwi.returnedCols[ordercol->counter - 1]->clone(); - rc->orderPos(ordercol->counter - 1); - // can not be optimized off if used in order by with counter. - // set with self derived table alias if it's derived table - gwi.returnedCols[ordercol->counter - 1]->incRefCount(); - } - else - { - Item* ord_item = *(ordercol->item); - bool nonAggField = true; - - // ignore not_used column on order by. - if ((ord_item->type() == Item::CONST_ITEM - && ord_item->cmp_type() == INT_RESULT) - && ord_item->full_name() - && !strcmp(ord_item->full_name(), "Not_used")) - { - continue; - } - else if (ord_item->type() == Item::CONST_ITEM - && ord_item->cmp_type() == INT_RESULT) - { - rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - gwi.fatalParseError = true; - } - else if (ordercol->in_field_list && ord_item->type() == Item::FIELD_ITEM) - { - rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - Item_field* ifp = static_cast(ord_item); - - // The item must be an alias for a projected column - // and extended SELECT list must contain a proper rc - // either aggregation or a field. - if (!rc && ifp->name.length) - { - gwi.fatalParseError = false; - execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.returnedCols.begin(); - - for ( ; iter != gwi.returnedCols.end(); iter++ ) - { - if ( (*iter).get()->alias() == ord_item->name.str ) - { - rc = (*iter).get()->clone(); - nonAggField = rc->hasAggregate() ? false : true; - break; - } - } - } - } - else - rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - - // Looking for a match for this item in GROUP BY list. - if ( rc && ord_item->type() == Item::FIELD_ITEM && nonAggField) - { - execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); - - for ( ; iter != gwi.groupByCols.end(); iter++ ) - { - if ( rc->sameColumn((*iter).get()) ) - break; - } - - // MCOL-1052 Find and remove the optimized field - // from ORDER using cond_pushed filters. - if (buildConstColFromFilter( - dynamic_cast(rc), gwi, gi)) - { - break; - } - - // MCOL-1052 GROUP BY items list doesn't contain - // this ORDER BY item. - if ( iter == gwi.groupByCols.end() ) - { - std::ostringstream ostream; - std::ostringstream& osr = ostream; - getColNameFromItem(osr, *ordercol->item); - Message::Args args; - args.add(ostream.str()); - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ERR_NOT_GROUPBY_EXPRESSION; - } - } - - // @bug5501 try item_ptr if item can not be fixed. For some - // weird dml statement state, item can not be fixed but the - // infomation is available in item_ptr. - if (!rc || gwi.fatalParseError) - { - gwi.fatalParseError = false; - Item* item_ptr = ordercol->item_ptr; - - while (item_ptr->type() == Item::REF_ITEM) - item_ptr = *(((Item_ref*)item_ptr)->ref); - - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - - // This ORDER BY item must be an agg function - - // the ordercol->item_ptr and exteded SELECT list - // must contain the corresponding item. - if (!rc) - { - Item* item_ptr = ordercol->item_ptr; - - if (item_ptr) - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - - if (!rc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - if (ordercol->direction == ORDER::ORDER_ASC) - rc->asc(true); - else - rc->asc(false); - - gwi.orderByCols.push_back(SRCP(rc)); + if (groupItem->name.length && string(groupItem->name.str) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } } - - // make sure columnmap, returnedcols and count(*) arg_list are not empty - TableMap::iterator tb_iter = gwi.tableMap.begin(); - - try + if (!rc) { - for (; tb_iter != gwi.tableMap.end(); tb_iter++) - { - if ((*tb_iter).second.first == 1) continue; - - CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; - CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); - SRCP srcp(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - (*tb_iter).second.first = 1; - } - } - catch (runtime_error& e) - { - setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; + nonSupportItem = groupItem; + break; } - if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || - gwi.returnedCols.empty()) + gwi.groupByCols.push_back(SRCP(rc)); + } + else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) + { + if (!groupcol->in_field_list || !groupItem->name.length) { - // get the smallest column from colmap - CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; - int minColWidth = 0; - CalpontSystemCatalog::ColType ct; - - try - { - for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) - { - // should always not null - SimpleColumn* sc = dynamic_cast(iter->second.get()); - - if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) - { - ct = csc->colType(sc->oid()); - - if (minColWidth == 0) - { - minColWidth = ct.colWidth; - minSc = iter->second; - } - else if (ct.colWidth < minColWidth) - { - minColWidth = ct.colWidth; - minSc = iter->second; - } - } - } - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) - gwi.returnedCols.push_back(minSc); - } - - if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) - { - // re-construct the select query and redo phase 1 - if (redo) - { - TABLE_LIST* table_ptr = gi.groupByTables; - - // put all tables, derived tables and views on the list - //TABLE_LIST* table_ptr = select_lex.get_table_list(); - set aliasSet; // to avoid duplicate table alias - - for (; table_ptr; table_ptr = table_ptr->next_local) - { - if (string(table_ptr->table_name.str).find("$vtable") != string::npos) - continue; - - if (table_ptr->derived) - { - if (aliasSet.find(table_ptr->alias.str) != aliasSet.end()) - continue; - - aliasSet.insert(table_ptr->alias.str); - } - else if (table_ptr->view) - { - if (aliasSet.find(table_ptr->alias.str) != aliasSet.end()) - continue; - - aliasSet.insert(table_ptr->alias.str); - } - else - { - // table referenced by view is represented by viewAlias_tableAlias. - // consistent with item.cc field print. - if (table_ptr->referencing_view) - { - if (aliasSet.find(string(table_ptr->referencing_view->alias.str) + "_" + - string(table_ptr->alias.str)) != aliasSet.end()) - continue; - - aliasSet.insert(string(table_ptr->referencing_view->alias.str) + "_" + - string(table_ptr->alias.str)); - } - else - { - if (aliasSet.find(table_ptr->alias.str) != aliasSet.end()) - continue; - - aliasSet.insert(table_ptr->alias.str); - } - - } - } - - } - else - { - // remove order by clause in case this phase has been executed before. - // need a better fix later, like skip all the other non-optimized phase. - - // MCOL-1052 - if (unionSel) - { - ordercol = reinterpret_cast(gi.groupByOrder); - } - else - ordercol = 0; - - for (; ordercol; ordercol = ordercol->next) - { - Item* ord_item = *(ordercol->item); - - if (ord_item->type() == Item::NULL_ITEM) - { - // MCOL-793 Do nothing for an ORDER BY NULL - } - else if (ord_item->type() == Item::SUM_FUNC_ITEM) - { - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (fc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - - //continue; - } - // @bug 3518. if order by clause = selected column, use position. - else if (ord_item->name.length && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - string fullname; - - if (field->db_name.str) - fullname += string(field->db_name.str) + "."; - - if (field->table_name.str) - fullname += string(field->table_name.str) + "."; - - if (field->field_name.length) - fullname += string(field->field_name.str); - - uint32_t i = 0; - - for (i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - continue; - - if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || - strcasecmp(ord_item->name.str, gwi.returnedCols[i]->alias().c_str()) == 0) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - - if (i == gwi.returnedCols.size()) - ord_cols += string(" `") + escapeBackTick(ord_item->name.str) + '`'; - } - - else if (ord_item->name.length) - { - // for union order by 1 case. For unknown reason, it doesn't show in_field_list - if (ord_item->type() == Item::CONST_ITEM - && ord_item->cmp_type() == INT_RESULT) - { - ord_cols += ord_item->name.str; - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else - { - ord_cols += string(" `") + escapeBackTick(ord_item->name.str) + '`'; - } - } - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug5636. check if this order by column is on the select list - ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - } - else - { - String str; - ord_item->print(&str, QT_ORDINARY); - ord_cols += string(str.c_ptr()); - } - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - } - } - - if ( gwi.orderByCols.size() ) // has order by - { - csep->hasOrderBy(true); - csep->specHandlerProcessed(true); - csep->orderByThreads(get_orderby_threads(gwi.thd)); - } - } - - // LIMIT and OFFSET are extracted from TABLE_LIST elements. - // All of JOIN-ed tables contain relevant limit and offset. - uint64_t limit = (uint64_t)-1; - if (gi.groupByTables->select_lex->limit_params.select_limit && - ( limit = static_cast(gi.groupByTables->select_lex->limit_params.select_limit)->val_int() ) && - limit != (uint64_t)-1 ) - { - csep->limitNum(limit); - } - else if (csep->hasOrderBy()) - { - // We use LimitedOrderBy so set the limit to - // go through the check in addOrderByAndLimit - csep->limitNum((uint64_t) - 2); - } - - if (gi.groupByTables->select_lex->limit_params.offset_limit) - { - csep->limitStart(((Item_int*)gi.groupByTables->select_lex->limit_params.offset_limit)->val_int()); - } - - // We don't currently support limit with correlated subquery - if (csep->limitNum() != (uint64_t) - 1 && - gwi.subQuery && !gwi.correlatedTbNameVec.empty()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - } // ORDER BY processing ends here - - if ( gi.groupByDistinct ) - csep->distinct(true); - - // add the smallest column to count(*) parm. - // select constant in subquery case - std::vector::iterator coliter; - - if (!minSc) - { - if (!gwi.returnedCols.empty()) - minSc = gwi.returnedCols[0]; - else if (!gwi.additionalRetCols.empty()) - minSc = gwi.additionalRetCols[0]; - } - - // @bug3523, count(*) on subquery always pick column[0]. - SimpleColumn* sc = dynamic_cast(minSc.get()); - - if (sc && sc->schemaName().empty()) - { - if (gwi.derivedTbList.size() >= 1) - { - SimpleColumn* sc1 = new SimpleColumn(); - sc1->columnName(sc->columnName()); - sc1->tableName(sc->tableName()); - sc1->tableAlias(sc->tableAlias()); - sc1->viewName(sc->viewName()); - sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - sc1->colPosition(0); - minSc.reset(sc1); - } - } - - for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) - { - // @bug5977 @note should never throw this, but checking just in case. - // When ExeMgr fix is ready, this should not error out... - if (dynamic_cast(minSc.get())) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "No project column found for aggregate function"; - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // Replace the last (presumably constant) object with minSc - if ((*coliter)->aggParms().empty()) - { - (*coliter)->aggParms().push_back(minSc); + nonSupportItem = groupItem; } else { - (*coliter)->aggParms()[0] = minSc; + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name.str) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + } } + } + // @bug 3761. + else if (groupcol->counter_used) + { + if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) + { + nonSupportItem = groupItem; + } + else + { + gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); + } + } + else + { + nonSupportItem = groupItem; + } } - std::vector::iterator funciter; + // @bug 4756. Add internal groupby column for correlated join to the groupby list + if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) + gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); - SPTP sptp(new ParseTree(minSc.get()->clone())); - - for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + // this is window func on SELECT becuase ORDER BY has not been processed + if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) { - FunctionParm funcParms = (*funciter)->functionParms(); - funcParms.push_back(sptp); - (*funciter)->functionParms(funcParms); + for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) + { + if (gwi.windowFuncList[i]->hasWindowFunc()) + { + vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); + + for (uint32_t j = 0; j < windowFunctions.size(); j++) + windowFunctions[j]->addToPartition(gwi.subGroupByCols); + } + } } - // set sequence# for subquery localCols - for (uint32_t i = 0; i < gwi.localCols.size(); i++) - gwi.localCols[i]->sequence(i); + if (nonSupportItem) + { + Message::Args args; - // append additionalRetCols to returnedCols - gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), - gwi.additionalRetCols.end()); + if (nonSupportItem->name.length) + args.add("'" + string(nonSupportItem->name.str) + "'"); + else + args.add(""); - csep->groupByCols(gwi.groupByCols); - csep->orderByCols(gwi.orderByCols); - csep->returnedCols(gwi.returnedCols); - csep->columnMap(gwi.columnMap); - csep->having(havingFilter); - csep->derivedTableList(gwi.derivedTbList); - csep->selectSubList(selectSubList); - csep->subSelectList(gwi.subselectList); - clearStacks(gwi); - return 0; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } // GROUP processing ends here + + // ORDER BY processing starts here + { + ORDER* ordercol = reinterpret_cast(gi.groupByOrder); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + for (; ordercol; ordercol = ordercol->next) + { + if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + // re-visit the first of ordercol list + ordercol = reinterpret_cast(gi.groupByOrder); + + // for subquery, order+limit by will be supported in infinidb. build order by columns + // @todo union order by and limit support + // if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + + for (; ordercol; ordercol = ordercol->next) + { + ReturnedColumn* rc = NULL; + + if (ordercol->in_field_list && ordercol->counter_used) + { + rc = gwi.returnedCols[ordercol->counter - 1]->clone(); + rc->orderPos(ordercol->counter - 1); + // can not be optimized off if used in order by with counter. + // set with self derived table alias if it's derived table + gwi.returnedCols[ordercol->counter - 1]->incRefCount(); + } + else + { + Item* ord_item = *(ordercol->item); + bool nonAggField = true; + + // ignore not_used column on order by. + if ((ord_item->type() == Item::CONST_ITEM && ord_item->cmp_type() == INT_RESULT) && + ord_item->full_name() && !strcmp(ord_item->full_name(), "Not_used")) + { + continue; + } + else if (ord_item->type() == Item::CONST_ITEM && ord_item->cmp_type() == INT_RESULT) + { + rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + gwi.fatalParseError = true; + } + else if (ordercol->in_field_list && ord_item->type() == Item::FIELD_ITEM) + { + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + Item_field* ifp = static_cast(ord_item); + + // The item must be an alias for a projected column + // and extended SELECT list must contain a proper rc + // either aggregation or a field. + if (!rc && ifp->name.length) + { + gwi.fatalParseError = false; + execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = + gwi.returnedCols.begin(); + + for (; iter != gwi.returnedCols.end(); iter++) + { + if ((*iter).get()->alias() == ord_item->name.str) + { + rc = (*iter).get()->clone(); + nonAggField = rc->hasAggregate() ? false : true; + break; + } + } + } + } + else + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + + // Looking for a match for this item in GROUP BY list. + if (rc && ord_item->type() == Item::FIELD_ITEM && nonAggField) + { + execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); + + for (; iter != gwi.groupByCols.end(); iter++) + { + if (rc->sameColumn((*iter).get())) + break; + } + + // MCOL-1052 Find and remove the optimized field + // from ORDER using cond_pushed filters. + if (buildConstColFromFilter(dynamic_cast(rc), gwi, gi)) + { + break; + } + + // MCOL-1052 GROUP BY items list doesn't contain + // this ORDER BY item. + if (iter == gwi.groupByCols.end()) + { + std::ostringstream ostream; + std::ostringstream& osr = ostream; + getColNameFromItem(osr, *ordercol->item); + Message::Args args; + args.add(ostream.str()); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ERR_NOT_GROUPBY_EXPRESSION; + } + } + + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) + { + gwi.fatalParseError = false; + Item* item_ptr = ordercol->item_ptr; + + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); + + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + // This ORDER BY item must be an agg function - + // the ordercol->item_ptr and exteded SELECT list + // must contain the corresponding item. + if (!rc) + { + Item* item_ptr = ordercol->item_ptr; + + if (item_ptr) + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + if (!rc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + if (ordercol->direction == ORDER::ORDER_ASC) + rc->asc(true); + else + rc->asc(false); + + gwi.orderByCols.push_back(SRCP(rc)); + } + + // make sure columnmap, returnedcols and count(*) arg_list are not empty + TableMap::iterator tb_iter = gwi.tableMap.begin(); + + try + { + for (; tb_iter != gwi.tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) + continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || gwi.returnedCols.empty()) + { + // get the smallest column from colmap + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + int minColWidth = 0; + CalpontSystemCatalog::ColType ct; + + try + { + for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) + { + // should always not null + SimpleColumn* sc = dynamic_cast(iter->second.get()); + + if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + { + ct = csc->colType(sc->oid()); + + if (minColWidth == 0) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + else if (ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + } + } + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) + gwi.returnedCols.push_back(minSc); + } + + if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + // re-construct the select query and redo phase 1 + if (redo) + { + TABLE_LIST* table_ptr = gi.groupByTables; + + // put all tables, derived tables and views on the list + // TABLE_LIST* table_ptr = select_lex.get_table_list(); + set aliasSet; // to avoid duplicate table alias + + for (; table_ptr; table_ptr = table_ptr->next_local) + { + if (string(table_ptr->table_name.str).find("$vtable") != string::npos) + continue; + + if (table_ptr->derived) + { + if (aliasSet.find(table_ptr->alias.str) != aliasSet.end()) + continue; + + aliasSet.insert(table_ptr->alias.str); + } + else if (table_ptr->view) + { + if (aliasSet.find(table_ptr->alias.str) != aliasSet.end()) + continue; + + aliasSet.insert(table_ptr->alias.str); + } + else + { + // table referenced by view is represented by viewAlias_tableAlias. + // consistent with item.cc field print. + if (table_ptr->referencing_view) + { + if (aliasSet.find(string(table_ptr->referencing_view->alias.str) + "_" + + string(table_ptr->alias.str)) != aliasSet.end()) + continue; + + aliasSet.insert(string(table_ptr->referencing_view->alias.str) + "_" + + string(table_ptr->alias.str)); + } + else + { + if (aliasSet.find(table_ptr->alias.str) != aliasSet.end()) + continue; + + aliasSet.insert(table_ptr->alias.str); + } + } + } + } + else + { + // remove order by clause in case this phase has been executed before. + // need a better fix later, like skip all the other non-optimized phase. + + // MCOL-1052 + if (unionSel) + { + ordercol = reinterpret_cast(gi.groupByOrder); + } + else + ordercol = 0; + + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + if (ord_item->type() == Item::NULL_ITEM) + { + // MCOL-793 Do nothing for an ORDER BY NULL + } + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + // continue; + } + // @bug 3518. if order by clause = selected column, use position. + else if (ord_item->name.length && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + string fullname; + + if (field->db_name.str) + fullname += string(field->db_name.str) + "."; + + if (field->table_name.str) + fullname += string(field->table_name.str) + "."; + + if (field->field_name.length) + fullname += string(field->field_name.str); + + uint32_t i = 0; + + for (i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), + sc->viewName().c_str()) != 0)) + continue; + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name.str, gwi.returnedCols[i]->alias().c_str()) == 0) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + if (i == gwi.returnedCols.size()) + ord_cols += string(" `") + escapeBackTick(ord_item->name.str) + '`'; + } + + else if (ord_item->name.length) + { + // for union order by 1 case. For unknown reason, it doesn't show in_field_list + if (ord_item->type() == Item::CONST_ITEM && ord_item->cmp_type() == INT_RESULT) + { + ord_cols += ord_item->name.str; + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + ord_cols += string(" `") + escapeBackTick(ord_item->name.str) + '`'; + } + } + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug5636. check if this order by column is on the select list + ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + } + else + { + String str; + ord_item->print(&str, QT_ORDINARY); + ord_cols += string(str.c_ptr()); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + } + } + + if (gwi.orderByCols.size()) // has order by + { + csep->hasOrderBy(true); + csep->specHandlerProcessed(true); + csep->orderByThreads(get_orderby_threads(gwi.thd)); + } + } + + // LIMIT and OFFSET are extracted from TABLE_LIST elements. + // All of JOIN-ed tables contain relevant limit and offset. + uint64_t limit = (uint64_t)-1; + if (gi.groupByTables->select_lex->limit_params.select_limit && + (limit = + static_cast(gi.groupByTables->select_lex->limit_params.select_limit)->val_int()) && + limit != (uint64_t)-1) + { + csep->limitNum(limit); + } + else if (csep->hasOrderBy()) + { + // We use LimitedOrderBy so set the limit to + // go through the check in addOrderByAndLimit + csep->limitNum((uint64_t)-2); + } + + if (gi.groupByTables->select_lex->limit_params.offset_limit) + { + csep->limitStart(((Item_int*)gi.groupByTables->select_lex->limit_params.offset_limit)->val_int()); + } + + // We don't currently support limit with correlated subquery + if (csep->limitNum() != (uint64_t)-1 && gwi.subQuery && !gwi.correlatedTbNameVec.empty()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + } // ORDER BY processing ends here + + if (gi.groupByDistinct) + csep->distinct(true); + + // add the smallest column to count(*) parm. + // select constant in subquery case + std::vector::iterator coliter; + + if (!minSc) + { + if (!gwi.returnedCols.empty()) + minSc = gwi.returnedCols[0]; + else if (!gwi.additionalRetCols.empty()) + minSc = gwi.additionalRetCols[0]; + } + + // @bug3523, count(*) on subquery always pick column[0]. + SimpleColumn* sc = dynamic_cast(minSc.get()); + + if (sc && sc->schemaName().empty()) + { + if (gwi.derivedTbList.size() >= 1) + { + SimpleColumn* sc1 = new SimpleColumn(); + sc1->columnName(sc->columnName()); + sc1->tableName(sc->tableName()); + sc1->tableAlias(sc->tableAlias()); + sc1->viewName(sc->viewName()); + sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc1->colPosition(0); + minSc.reset(sc1); + } + } + + for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) + { + // @bug5977 @note should never throw this, but checking just in case. + // When ExeMgr fix is ready, this should not error out... + if (dynamic_cast(minSc.get())) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // Replace the last (presumably constant) object with minSc + if ((*coliter)->aggParms().empty()) + { + (*coliter)->aggParms().push_back(minSc); + } + else + { + (*coliter)->aggParms()[0] = minSc; + } + } + + std::vector::iterator funciter; + + SPTP sptp(new ParseTree(minSc.get()->clone())); + + for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + { + FunctionParm funcParms = (*funciter)->functionParms(); + funcParms.push_back(sptp); + (*funciter)->functionParms(funcParms); + } + + // set sequence# for subquery localCols + for (uint32_t i = 0; i < gwi.localCols.size(); i++) + gwi.localCols[i]->sequence(i); + + // append additionalRetCols to returnedCols + gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), + gwi.additionalRetCols.end()); + + csep->groupByCols(gwi.groupByCols); + csep->orderByCols(gwi.orderByCols); + csep->returnedCols(gwi.returnedCols); + csep->columnMap(gwi.columnMap); + csep->having(havingFilter); + csep->derivedTableList(gwi.derivedTbList); + csep->selectSubList(selectSubList); + csep->subSelectList(gwi.subselectList); + clearStacks(gwi); + return 0; } - -} +} // namespace cal_impl_if // vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 5b7de4c22..5ef57cf43 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -42,10 +42,10 @@ #include #include #if defined(__linux__) -#include //wait() +#include //wait() #elif defined(__FreeBSD__) #include -#include // For stat(). +#include // For stat(). #include #include #include @@ -151,31 +151,42 @@ namespace cal_impl_if extern bool nonConstFunc(Item_func* ifp); } - namespace { // Calpont vtable non-support error message -const string infinidb_autoswitch_warning = "The query includes syntax that is not supported by MariaDB Columnstore distributed mode. The execution was switched to standard mode with downgraded performance."; +const string infinidb_autoswitch_warning = + "The query includes syntax that is not supported by MariaDB Columnstore distributed mode. The execution " + "was switched to standard mode with downgraded performance."; // copied from item_timefunc.cc -static const string interval_names[] = -{ - "year", "quarter", "month", "week", "day", - "hour", "minute", "second", "microsecond", - "year_month", "day_hour", "day_minute", - "day_second", "hour_minute", "hour_second", - "minute_second", "day_microsecond", - "hour_microsecond", "minute_microsecond", - "second_microsecond" -}; +static const string interval_names[] = {"year", + "quarter", + "month", + "week", + "day", + "hour", + "minute", + "second", + "microsecond", + "year_month", + "day_hour", + "day_minute", + "day_second", + "hour_minute", + "hour_second", + "minute_second", + "day_microsecond", + "hour_microsecond", + "minute_microsecond", + "second_microsecond"}; // HDFS is never used nowadays, so don't bother -bool useHdfs = false; // ResourceManager::instance()->useHdfs(); +bool useHdfs = false; // ResourceManager::instance()->useHdfs(); -//convenience fcn +// convenience fcn inline uint32_t tid2sid(const uint32_t tid) { - return CalpontSystemCatalog::idb_tid2sid(tid); + return CalpontSystemCatalog::idb_tid2sid(tid); } /** @@ -195,77 +206,76 @@ inline uint32_t tid2sid(const uint32_t tid) Called from many places in ha_mcs_impl.cpp(). */ -void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc = false) +void force_close_fep_conn(THD* thd, cal_connection_info* ci, bool check_prev_rc = false) { - if (!ci->cal_conn_hndl) - { - return; - } + if (!ci->cal_conn_hndl) + { + return; + } - if(check_prev_rc && !ci->rc) - { - return; - } + if (check_prev_rc && !ci->rc) + { + return; + } - // send ExeMgr an unknown signal to force him to close - // the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; + // send ExeMgr an unknown signal to force him to close + // the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // Add details into the message. - ha_mcs_impl::log_this(thd, "Exception in force_close_fep_conn().", - logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - } + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // Add details into the message. + ha_mcs_impl::log_this(thd, "Exception in force_close_fep_conn().", logging::LOG_TYPE_DEBUG, + tid2sid(thd->thread_id)); + } - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; } // // @bug 2244. Log exception related to lost connection to ExeMgr. // Log exception error from calls to sm::tpl_scan_fetch in fetchNextRow() // -void tpl_scan_fetch_LogException( cal_table_info& ti, cal_connection_info* ci, std::exception* ex) +void tpl_scan_fetch_LogException(cal_table_info& ti, cal_connection_info* ci, std::exception* ex) { - time_t t = time(0); - char datestr[50]; - ctime_r(&t, datestr); - datestr[ strlen(datestr) - 1 ] = '\0'; // strip off trailing newline + time_t t = time(0); + char datestr[50]; + ctime_r(&t, datestr); + datestr[strlen(datestr) - 1] = '\0'; // strip off trailing newline - uint32_t sesID = 0; - string connHndl("No connection handle to use"); + uint32_t sesID = 0; + string connHndl("No connection handle to use"); - if (ti.conn_hndl) - { - connHndl = "ti connection used"; - sesID = ti.conn_hndl->sessionID; - } + if (ti.conn_hndl) + { + connHndl = "ti connection used"; + sesID = ti.conn_hndl->sessionID; + } - else if (ci->cal_conn_hndl) - { - connHndl = "ci connection used"; - sesID = ci->cal_conn_hndl->sessionID; - } + else if (ci->cal_conn_hndl) + { + connHndl = "ci connection used"; + sesID = ci->cal_conn_hndl->sessionID; + } - int64_t rowsRet = -1; + int64_t rowsRet = -1; - if (ti.tpl_scan_ctx) - rowsRet = ti.tpl_scan_ctx->rowsreturned; + if (ti.tpl_scan_ctx) + rowsRet = ti.tpl_scan_ctx->rowsreturned; - if (ex) - cerr << datestr << ": sm::tpl_scan_fetch error getting rows for sessionID: " << - sesID << "; " << connHndl << "; rowsReturned: " << rowsRet << - "; reason-" << ex->what() << endl; - else - cerr << datestr << ": sm::tpl_scan_fetch unknown error getting rows for sessionID: " << - sesID << "; " << connHndl << "; rowsReturned: " << rowsRet << endl; + if (ex) + cerr << datestr << ": sm::tpl_scan_fetch error getting rows for sessionID: " << sesID << "; " << connHndl + << "; rowsReturned: " << rowsRet << "; reason-" << ex->what() << endl; + else + cerr << datestr << ": sm::tpl_scan_fetch unknown error getting rows for sessionID: " << sesID << "; " + << connHndl << "; rowsReturned: " << rowsRet << endl; } // Table Map is used by both cond_push and table mode processing @@ -273,894 +283,894 @@ void tpl_scan_fetch_LogException( cal_table_info& ti, cal_connection_info* ci, s // When bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) { - size_t counter = 0; - for (auto &tableMapEntry: ci->tableMap) - { - if (tableMapEntry.second.csep) - counter++; - if (counter >= 1) - return false; - } + size_t counter = 0; + for (auto& tableMapEntry : ci->tableMap) + { + if (tableMapEntry.second.csep) + counter++; + if (counter >= 1) + return false; + } - return true; + return true; } - int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) { - int rc = HA_ERR_END_OF_FILE; - int num_attr = ti.msTablePtr->s->fields; - sm::status_t sm_stat; + int rc = HA_ERR_END_OF_FILE; + int num_attr = ti.msTablePtr->s->fields; + sm::status_t sm_stat; - try + try + { + if (ti.conn_hndl) { - if (ti.conn_hndl) - { - sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ti.conn_hndl); - } - else if (ci->cal_conn_hndl) - { - sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ci->cal_conn_hndl, (int*)(¤t_thd->killed)); - } - else - throw runtime_error("internal error"); + sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ti.conn_hndl); } - catch (std::exception& ex) + else if (ci->cal_conn_hndl) { -// @bug 2244. Always log this msg for now, as we try to track down when/why we are -// losing socket connection with ExeMgr -//#ifdef INFINIDB_DEBUG - tpl_scan_fetch_LogException( ti, ci, &ex); -//#endif - sm_stat = sm::CALPONT_INTERNAL_ERROR; - } - catch (...) - { -// @bug 2244. Always log this msg for now, as we try to track down when/why we are -// losing socket connection with ExeMgr -//#ifdef INFINIDB_DEBUG - tpl_scan_fetch_LogException( ti, ci, 0 ); -//#endif - sm_stat = sm::CALPONT_INTERNAL_ERROR; - } - - if (sm_stat == sm::STATUS_OK) - { - Field** f; - f = ti.msTablePtr->field; - - //set all fields to null in null col bitmap - if (!handler_flag) - memset(buf, -1, ti.msTablePtr->s->null_bytes); - else - { - memset(ti.msTablePtr->null_flags, -1, ti.msTablePtr->s->null_bytes); - } - - std::vector& colTypes = ti.tpl_scan_ctx->ctp; - - RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; - - // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup - // set coltype.position to be the position in rowgroup. only set once. - if (ti.tpl_scan_ctx->rowsreturned == 0 && - (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)) - { - for (uint32_t i = 0; i < rowGroup->getColumnCount(); i++) - { - int oid = rowGroup->getOIDs()[i]; - int j = 0; - - for (; j < num_attr; j++) - { - // mysql should haved eliminated duplicate projection columns - if (oid == colTypes[j].columnOID || oid == colTypes[j].ddn.dictOID) - { - colTypes[j].colPosition = i; - break; - } - } - } - } - - rowgroup::Row row; - rowGroup->initRow(&row); - rowGroup->getRow(ti.tpl_scan_ctx->rowsreturned, &row); - int s; - - for (int p = 0; p < num_attr; p++, f++) - { - //This col is going to be written - bitmap_set_bit(ti.msTablePtr->write_set, (*f)->field_index); - - // get coltype if not there yet - if (colTypes[0].colWidth == 0) - { - for (short c = 0; c < num_attr; c++) - { - colTypes[c].colPosition = c; - colTypes[c].colWidth = rowGroup->getColumnWidth(c); - colTypes[c].colDataType = rowGroup->getColTypes()[c]; - colTypes[c].columnOID = rowGroup->getOIDs()[c]; - colTypes[c].scale = rowGroup->getScale()[c]; - colTypes[c].precision = rowGroup->getPrecision()[c]; - } - } - - CalpontSystemCatalog::ColType colType(colTypes[p]); - - // table mode handling - if (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) - { - if (colType.colPosition == -1) // not projected by tuplejoblist - continue; - else - s = colType.colPosition; - } - else - { - s = p; - } - - // precision == -16 is borrowed as skip null check indicator for bit ops. - if (row.isNullValue(s) && colType.precision != -16) - { - // @2835. Handle empty string and null confusion. store empty string for string column - if (colType.colDataType == CalpontSystemCatalog::CHAR || - colType.colDataType == CalpontSystemCatalog::VARCHAR || - colType.colDataType == CalpontSystemCatalog::VARBINARY) - { - (*f)->store("", 0, (*f)->charset()); - } - - continue; - } - - const datatypes::TypeHandler *h= colType.typeHandler(); - if (!h) - { - idbassert(0); - (*f)->reset(); - (*f)->set_null(); - } - else - { - // fetch and store data - (*f)->set_notnull(); - datatypes::StoreFieldMariaDB mf(*f, colType); - h->storeValueToField(row, s, &mf); - } - } - - ti.tpl_scan_ctx->rowsreturned++; - ti.c++; -#ifdef INFINIDB_DEBUG - - if ((ti.c % 1000000) == 0) - cerr << "fetchNextRow so far table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl; - -#endif - ti.moreRows = true; - rc = 0; - } - else if (sm_stat == sm::SQL_NOT_FOUND) - { - IDEBUG( cerr << "fetchNextRow done for table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl ); - ti.c = 0; - ti.moreRows = false; - rc = HA_ERR_END_OF_FILE; - } - else if (sm_stat == sm::CALPONT_INTERNAL_ERROR) - { - ti.moreRows = false; - rc = ER_INTERNAL_ERROR; - ci->rc = rc; - } - else if ((uint32_t)sm_stat == logging::ERR_LOST_CONN_EXEMGR) - { - ti.moreRows = false; - rc = logging::ERR_LOST_CONN_EXEMGR; - sm::sm_init(tid2sid(current_thd->thread_id), &ci->cal_conn_hndl, - get_local_query(current_thd)); - idbassert(ci->cal_conn_hndl != 0); - ci->rc = rc; - } - else if (sm_stat == sm::SQL_KILLED) - { - // query was aborted by the user. treat it the same as limit query. close - // connection after rnd_close. - ti.c = 0; - ti.moreRows = false; - rc = HA_ERR_END_OF_FILE; - ci->rc = rc; + sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ci->cal_conn_hndl, (int*)(¤t_thd->killed)); } + else + throw runtime_error("internal error"); + } + catch (std::exception& ex) + { + // @bug 2244. Always log this msg for now, as we try to track down when/why we are + // losing socket connection with ExeMgr + //#ifdef INFINIDB_DEBUG + tpl_scan_fetch_LogException(ti, ci, &ex); + //#endif + sm_stat = sm::CALPONT_INTERNAL_ERROR; + } + catch (...) + { + // @bug 2244. Always log this msg for now, as we try to track down when/why we are + // losing socket connection with ExeMgr + //#ifdef INFINIDB_DEBUG + tpl_scan_fetch_LogException(ti, ci, 0); + //#endif + sm_stat = sm::CALPONT_INTERNAL_ERROR; + } + + if (sm_stat == sm::STATUS_OK) + { + Field** f; + f = ti.msTablePtr->field; + + // set all fields to null in null col bitmap + if (!handler_flag) + memset(buf, -1, ti.msTablePtr->s->null_bytes); else { - ti.moreRows = false; - rc = sm_stat; - ci->rc = rc; + memset(ti.msTablePtr->null_flags, -1, ti.msTablePtr->s->null_bytes); } - return rc; + std::vector& colTypes = ti.tpl_scan_ctx->ctp; + + RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; + + // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup + // set coltype.position to be the position in rowgroup. only set once. + if (ti.tpl_scan_ctx->rowsreturned == 0 && + (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)) + { + for (uint32_t i = 0; i < rowGroup->getColumnCount(); i++) + { + int oid = rowGroup->getOIDs()[i]; + int j = 0; + + for (; j < num_attr; j++) + { + // mysql should haved eliminated duplicate projection columns + if (oid == colTypes[j].columnOID || oid == colTypes[j].ddn.dictOID) + { + colTypes[j].colPosition = i; + break; + } + } + } + } + + rowgroup::Row row; + rowGroup->initRow(&row); + rowGroup->getRow(ti.tpl_scan_ctx->rowsreturned, &row); + int s; + + for (int p = 0; p < num_attr; p++, f++) + { + // This col is going to be written + bitmap_set_bit(ti.msTablePtr->write_set, (*f)->field_index); + + // get coltype if not there yet + if (colTypes[0].colWidth == 0) + { + for (short c = 0; c < num_attr; c++) + { + colTypes[c].colPosition = c; + colTypes[c].colWidth = rowGroup->getColumnWidth(c); + colTypes[c].colDataType = rowGroup->getColTypes()[c]; + colTypes[c].columnOID = rowGroup->getOIDs()[c]; + colTypes[c].scale = rowGroup->getScale()[c]; + colTypes[c].precision = rowGroup->getPrecision()[c]; + } + } + + CalpontSystemCatalog::ColType colType(colTypes[p]); + + // table mode handling + if (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) + { + if (colType.colPosition == -1) // not projected by tuplejoblist + continue; + else + s = colType.colPosition; + } + else + { + s = p; + } + + // precision == -16 is borrowed as skip null check indicator for bit ops. + if (row.isNullValue(s) && colType.precision != -16) + { + // @2835. Handle empty string and null confusion. store empty string for string column + if (colType.colDataType == CalpontSystemCatalog::CHAR || + colType.colDataType == CalpontSystemCatalog::VARCHAR || + colType.colDataType == CalpontSystemCatalog::VARBINARY) + { + (*f)->store("", 0, (*f)->charset()); + } + + continue; + } + + const datatypes::TypeHandler* h = colType.typeHandler(); + if (!h) + { + idbassert(0); + (*f)->reset(); + (*f)->set_null(); + } + else + { + // fetch and store data + (*f)->set_notnull(); + datatypes::StoreFieldMariaDB mf(*f, colType); + h->storeValueToField(row, s, &mf); + } + } + + ti.tpl_scan_ctx->rowsreturned++; + ti.c++; +#ifdef INFINIDB_DEBUG + + if ((ti.c % 1000000) == 0) + cerr << "fetchNextRow so far table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl; + +#endif + ti.moreRows = true; + rc = 0; + } + else if (sm_stat == sm::SQL_NOT_FOUND) + { + IDEBUG(cerr << "fetchNextRow done for table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c + << endl); + ti.c = 0; + ti.moreRows = false; + rc = HA_ERR_END_OF_FILE; + } + else if (sm_stat == sm::CALPONT_INTERNAL_ERROR) + { + ti.moreRows = false; + rc = ER_INTERNAL_ERROR; + ci->rc = rc; + } + else if ((uint32_t)sm_stat == logging::ERR_LOST_CONN_EXEMGR) + { + ti.moreRows = false; + rc = logging::ERR_LOST_CONN_EXEMGR; + sm::sm_init(tid2sid(current_thd->thread_id), &ci->cal_conn_hndl, get_local_query(current_thd)); + idbassert(ci->cal_conn_hndl != 0); + ci->rc = rc; + } + else if (sm_stat == sm::SQL_KILLED) + { + // query was aborted by the user. treat it the same as limit query. close + // connection after rnd_close. + ti.c = 0; + ti.moreRows = false; + rc = HA_ERR_END_OF_FILE; + ci->rc = rc; + } + else + { + ti.moreRows = false; + rc = sm_stat; + ci->rc = rc; + } + + return rc; } void makeUpdateScalarJoin(const ParseTree* n, void* obj) { - TreeNode* tn = n->data(); - SimpleFilter* sf = dynamic_cast(tn); + TreeNode* tn = n->data(); + SimpleFilter* sf = dynamic_cast(tn); - if (!sf) - return; + if (!sf) + return; - SimpleColumn* scLeft = dynamic_cast(sf->lhs()); - SimpleColumn* scRight = dynamic_cast(sf->rhs()); - CalpontSystemCatalog::TableAliasName* updatedTables = reinterpret_cast(obj); + SimpleColumn* scLeft = dynamic_cast(sf->lhs()); + SimpleColumn* scRight = dynamic_cast(sf->rhs()); + CalpontSystemCatalog::TableAliasName* updatedTables = + reinterpret_cast(obj); - if ( scLeft && scRight ) + if (scLeft && scRight) + { + if ((strcasecmp(scLeft->tableName().c_str(), updatedTables->table.c_str()) == 0) && + (strcasecmp(scLeft->schemaName().c_str(), updatedTables->schema.c_str()) == 0) && + (strcasecmp(scLeft->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) { - if ( (strcasecmp(scLeft->tableName().c_str(), updatedTables->table.c_str()) == 0 ) && (strcasecmp(scLeft->schemaName().c_str(), updatedTables->schema.c_str()) == 0) - && (strcasecmp(scLeft->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) - { - uint64_t lJoinInfo = sf->lhs()->joinInfo(); - lJoinInfo |= JOIN_SCALAR; - //lJoinInfo |= JOIN_OUTER_SELECT; - //lJoinInfo |= JOIN_CORRELATED; - sf->lhs()->joinInfo(lJoinInfo); - } - else if ( (strcasecmp(scRight->tableName().c_str(), updatedTables->table.c_str()) == 0) && (strcasecmp(scRight->schemaName().c_str(), updatedTables->schema.c_str()) == 0) - && (strcasecmp(scRight->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) - { - uint64_t rJoinInfo = sf->rhs()->joinInfo(); - rJoinInfo |= JOIN_SCALAR; - //rJoinInfo |= JOIN_OUTER_SELECT; - //rJoinInfo |= JOIN_CORRELATED; - sf->rhs()->joinInfo(rJoinInfo); - } - else - return; + uint64_t lJoinInfo = sf->lhs()->joinInfo(); + lJoinInfo |= JOIN_SCALAR; + // lJoinInfo |= JOIN_OUTER_SELECT; + // lJoinInfo |= JOIN_CORRELATED; + sf->lhs()->joinInfo(lJoinInfo); + } + else if ((strcasecmp(scRight->tableName().c_str(), updatedTables->table.c_str()) == 0) && + (strcasecmp(scRight->schemaName().c_str(), updatedTables->schema.c_str()) == 0) && + (strcasecmp(scRight->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) + { + uint64_t rJoinInfo = sf->rhs()->joinInfo(); + rJoinInfo |= JOIN_SCALAR; + // rJoinInfo |= JOIN_OUTER_SELECT; + // rJoinInfo |= JOIN_CORRELATED; + sf->rhs()->joinInfo(rJoinInfo); } else - return; + return; + } + else + return; } void makeUpdateSemiJoin(const ParseTree* n, void* obj) { - TreeNode* tn = n->data(); - SimpleFilter* sf = dynamic_cast(tn); + TreeNode* tn = n->data(); + SimpleFilter* sf = dynamic_cast(tn); - if (!sf) - return; + if (!sf) + return; - SimpleColumn* scLeft = dynamic_cast(sf->lhs()); - SimpleColumn* scRight = dynamic_cast(sf->rhs()); - CalpontSystemCatalog::TableAliasName* updatedTables = reinterpret_cast(obj); + SimpleColumn* scLeft = dynamic_cast(sf->lhs()); + SimpleColumn* scRight = dynamic_cast(sf->rhs()); + CalpontSystemCatalog::TableAliasName* updatedTables = + reinterpret_cast(obj); - //@Bug 3279. Added a check for column filters. - if ( scLeft && scRight && (strcasecmp(scRight->tableAlias().c_str(), scLeft->tableAlias().c_str()) != 0)) + //@Bug 3279. Added a check for column filters. + if (scLeft && scRight && (strcasecmp(scRight->tableAlias().c_str(), scLeft->tableAlias().c_str()) != 0)) + { + if ((strcasecmp(scLeft->tableName().c_str(), updatedTables->table.c_str()) == 0) && + (strcasecmp(scLeft->schemaName().c_str(), updatedTables->schema.c_str()) == 0) && + (strcasecmp(scLeft->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) { - if ( (strcasecmp(scLeft->tableName().c_str(), updatedTables->table.c_str()) == 0 ) && (strcasecmp(scLeft->schemaName().c_str(), updatedTables->schema.c_str()) == 0) - && (strcasecmp(scLeft->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) - { - uint64_t lJoinInfo = sf->lhs()->joinInfo(); - lJoinInfo |= JOIN_SEMI; - //lJoinInfo |= JOIN_OUTER_SELECT; - //lJoinInfo |= JOIN_CORRELATED; - sf->lhs()->joinInfo(lJoinInfo); - } - else if ( (strcasecmp(scRight->tableName().c_str(), updatedTables->table.c_str()) == 0) && (strcasecmp(scRight->schemaName().c_str(), updatedTables->schema.c_str()) == 0) - && (strcasecmp(scRight->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) - { - uint64_t rJoinInfo = sf->rhs()->joinInfo(); - rJoinInfo |= JOIN_SEMI; - //rJoinInfo |= JOIN_OUTER_SELECT; - //rJoinInfo |= JOIN_CORRELATED; - sf->rhs()->joinInfo(rJoinInfo); - } - else - return; + uint64_t lJoinInfo = sf->lhs()->joinInfo(); + lJoinInfo |= JOIN_SEMI; + // lJoinInfo |= JOIN_OUTER_SELECT; + // lJoinInfo |= JOIN_CORRELATED; + sf->lhs()->joinInfo(lJoinInfo); + } + else if ((strcasecmp(scRight->tableName().c_str(), updatedTables->table.c_str()) == 0) && + (strcasecmp(scRight->schemaName().c_str(), updatedTables->schema.c_str()) == 0) && + (strcasecmp(scRight->tableAlias().c_str(), updatedTables->alias.c_str()) == 0)) + { + uint64_t rJoinInfo = sf->rhs()->joinInfo(); + rJoinInfo |= JOIN_SEMI; + // rJoinInfo |= JOIN_OUTER_SELECT; + // rJoinInfo |= JOIN_CORRELATED; + sf->rhs()->joinInfo(rJoinInfo); } else - return; + return; + } + else + return; } vector getOnUpdateTimestampColumns(string& schema, string& tableName, int sessionID) { - vector returnVal; - typedef CalpontSelectExecutionPlan::ColumnMap::value_type CMVT_; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableName aTableName; + vector returnVal; + typedef CalpontSelectExecutionPlan::ColumnMap::value_type CMVT_; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName aTableName; - // select columnname from calpontsys.syscolumn - // where schema = schema and tablename = tableName - // and datatype = 'timestamp' - // and defaultvalue = 'current_timestamp() ON UPDATE current_timestamp()' - CalpontSelectExecutionPlan csep; - CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; - CalpontSelectExecutionPlan::FilterTokenList filterTokenList; - CalpontSelectExecutionPlan::ColumnMap colMap; + // select columnname from calpontsys.syscolumn + // where schema = schema and tablename = tableName + // and datatype = 'timestamp' + // and defaultvalue = 'current_timestamp() ON UPDATE current_timestamp()' + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); - if (!txnID.valid) + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + csep.txnID(txnID.id); + csep.verID(verID); + csep.sessionID(sessionID); + + string sysTable = "calpontsys.syscolumn."; + string firstCol = sysTable + "columnname"; + SimpleColumn* c1 = new SimpleColumn(firstCol, sessionID); + string secondCol = sysTable + "schema"; + SimpleColumn* c2 = new SimpleColumn(secondCol, sessionID); + string thirdCol = sysTable + "tablename"; + SimpleColumn* c3 = new SimpleColumn(thirdCol, sessionID); + string fourthCol = sysTable + "datatype"; + SimpleColumn* c4 = new SimpleColumn(fourthCol, sessionID); + string fifthCol = sysTable + "defaultvalue"; + SimpleColumn* c5 = new SimpleColumn(fifthCol, sessionID); + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(firstCol, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(secondCol, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(thirdCol, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(fourthCol, srcp)); + srcp.reset(c5); + colMap.insert(CMVT_(fifthCol, srcp)); + csep.columnMapNonStatic(colMap); + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + // Filters + const SOP opeq(new Operator("=")); + SimpleFilter* f1 = new SimpleFilter(opeq, c2->clone(), new ConstantColumn(schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = + new SimpleFilter(opeq, c3->clone(), new ConstantColumn(tableName, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = new SimpleFilter( + opeq, c4->clone(), + new ConstantColumn((uint64_t)execplan::CalpontSystemCatalog::TIMESTAMP, ConstantColumn::NUM)); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); + + string defaultValue = "current_timestamp() ON UPDATE current_timestamp()"; + SimpleFilter* f4 = + new SimpleFilter(opeq, c5->clone(), new ConstantColumn(defaultValue, ConstantColumn::LITERAL)); + filterTokenList.push_back(f4); + csep.filterTokenList(filterTokenList); + + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable("calpontsys", "syscolumn", "")); + csep.tableList(tablelist); + + boost::shared_ptr exemgrClient( + new messageqcpp::MessageQueueClient("ExeMgr1")); + ByteStream msg, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + rowgroup::RowGroup* rowGroup = 0; + uint32_t rowCount; + + exemgrClient->write(msg); + ByteStream msgPlan; + csep.serialize(msgPlan); + exemgrClient->write(msgPlan); + msg.restart(); + msg = exemgrClient->read(); // error handling + emsgBs = exemgrClient->read(); + ByteStream::quadbyte qb1; + + if (emsgBs.length() == 0) + { + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb1; + + if (qb1 != 0) { - txnID.id = 0; - txnID.valid = true; + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); } + } - QueryContext verID; - verID = sm.verID(); - csep.txnID(txnID.id); - csep.verID(verID); - csep.sessionID(sessionID); - - string sysTable = "calpontsys.syscolumn."; - string firstCol = sysTable + "columnname"; - SimpleColumn* c1 = new SimpleColumn(firstCol, sessionID); - string secondCol = sysTable + "schema"; - SimpleColumn* c2 = new SimpleColumn(secondCol, sessionID); - string thirdCol = sysTable + "tablename"; - SimpleColumn* c3 = new SimpleColumn(thirdCol, sessionID); - string fourthCol = sysTable + "datatype"; - SimpleColumn* c4 = new SimpleColumn(fourthCol, sessionID); - string fifthCol = sysTable + "defaultvalue"; - SimpleColumn* c5 = new SimpleColumn(fifthCol, sessionID); - SRCP srcp; - srcp.reset(c1); - colMap.insert(CMVT_(firstCol, srcp)); - srcp.reset(c2); - colMap.insert(CMVT_(secondCol, srcp)); - srcp.reset(c3); - colMap.insert(CMVT_(thirdCol, srcp)); - srcp.reset(c4); - colMap.insert(CMVT_(fourthCol, srcp)); - srcp.reset(c5); - colMap.insert(CMVT_(fifthCol, srcp)); - csep.columnMapNonStatic(colMap); - srcp.reset(c1->clone()); - returnedColumnList.push_back(srcp); - csep.returnedCols(returnedColumnList); - - // Filters - const SOP opeq(new Operator("=")); - SimpleFilter* f1 = new SimpleFilter (opeq, - c2->clone(), - new ConstantColumn(schema, ConstantColumn::LITERAL)); - filterTokenList.push_back(f1); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f2 = new SimpleFilter (opeq, - c3->clone(), - new ConstantColumn(tableName, ConstantColumn::LITERAL)); - filterTokenList.push_back(f2); - filterTokenList.push_back(new Operator("and")); - - SimpleFilter* f3 = new SimpleFilter (opeq, - c4->clone(), - new ConstantColumn((uint64_t) execplan::CalpontSystemCatalog::TIMESTAMP, ConstantColumn::NUM)); - filterTokenList.push_back(f3); - filterTokenList.push_back(new Operator("and")); - - string defaultValue = "current_timestamp() ON UPDATE current_timestamp()"; - SimpleFilter* f4 = new SimpleFilter (opeq, - c5->clone(), - new ConstantColumn(defaultValue, ConstantColumn::LITERAL)); - filterTokenList.push_back(f4); - csep.filterTokenList(filterTokenList); - - CalpontSelectExecutionPlan::TableList tablelist; - tablelist.push_back(make_aliastable("calpontsys", "syscolumn", "")); - csep.tableList(tablelist); - - boost::shared_ptr exemgrClient (new messageqcpp::MessageQueueClient("ExeMgr1")); - ByteStream msg, emsgBs; - rowgroup::RGData rgData; - ByteStream::quadbyte qb = 4; - msg << qb; - rowgroup::RowGroup* rowGroup = 0; - uint32_t rowCount; - - exemgrClient->write(msg); - ByteStream msgPlan; - csep.serialize(msgPlan); - exemgrClient->write(msgPlan); + while (true) + { msg.restart(); - msg = exemgrClient->read(); //error handling - emsgBs = exemgrClient->read(); - ByteStream::quadbyte qb1; + msg = exemgrClient->read(); - if (emsgBs.length() == 0) + if (msg.length() == 0) { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); } - - string emsgStr; - emsgBs >> emsgStr; - - if (msg.length() == 4) - { - msg >> qb1; - - if (qb1 != 0) - { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - } - - while (true) + else { + if (!rowGroup) + { + // This is mete data + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(msg); + qb = 100; msg.restart(); - msg = exemgrClient->read(); + msg << qb; + exemgrClient->write(msg); + continue; + } - if ( msg.length() == 0 ) + rgData.deserialize(msg); + rowGroup->setData(&rgData); + + if (rowGroup->getStatus() != 0) + { + // msg.advance(rowGroup->getDataSize()); + msg >> emsgStr; + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; + throw runtime_error(emsgStr); + } + + rowCount = rowGroup->getRowCount(); + if (rowCount > 0) + { + rowgroup::Row row; + rowGroup->initRow(&row); + for (uint32_t i = 0; i < rowCount; i++) { - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error("Lost conection to ExeMgr."); + rowGroup->getRow(i, &row); + // we are only fetching a single column + returnVal.push_back(row.getStringField(0)); } - else - { - if (!rowGroup) - { - //This is mete data - rowGroup = new rowgroup::RowGroup(); - rowGroup->deserialize(msg); - qb = 100; - msg.restart(); - msg << qb; - exemgrClient->write(msg); - continue; - } + } + else + { + break; + } - rgData.deserialize(msg); - rowGroup->setData(&rgData); - - if (rowGroup->getStatus() != 0) - { - //msg.advance(rowGroup->getDataSize()); - msg >> emsgStr; - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - throw runtime_error(emsgStr); - } - - rowCount = rowGroup->getRowCount(); - if (rowCount > 0) - { - rowgroup::Row row; - rowGroup->initRow(&row); - for (uint32_t i = 0; i < rowCount; i++) - { - rowGroup->getRow(i, &row); - // we are only fetching a single column - returnVal.push_back(row.getStringField(0)); - } - } - else - { - break; - } - - //exemgrClient->shutdown(); - //delete exemgrClient; - //exemgrClient = 0; - } + // exemgrClient->shutdown(); + // delete exemgrClient; + // exemgrClient = 0; } + } - return returnVal; + return returnVal; } uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& condStack) { - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (ci->isSlaveNode && !thd->slave_thread) + if (ci->isSlaveNode && !thd->slave_thread) + { + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); + setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + + //@Bug 4387. Check BRM status before start statement. + boost::scoped_ptr dbrmp(new DBRM()); + int rc = dbrmp->isReadWrite(); + + if (rc != 0) + { + setError(current_thd, ER_READ_ONLY_MODE, "Cannot execute the statement. DBRM is read only!"); + return ER_READ_ONLY_MODE; + } + + // stats start + ci->stats.reset(); + ci->stats.setStartTime(); + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + try + { + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + + ci->stats.fSessionID = tid2sid(thd->thread_id); + + LEX* lex = thd->lex; + idbassert(lex != 0); + + // Error out DELETE on VIEW. It's currently not supported. + // @note DELETE on VIEW works natually (for simple cases at least), but we choose to turn it off + // for now - ZZ. + TABLE_LIST* tables = thd->lex->query_tables; + + for (; tables; tables = tables->next_local) + { + if (tables->view) { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); - setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; - } + Message::Args args; - //@Bug 4387. Check BRM status before start statement. - boost::scoped_ptr dbrmp(new DBRM()); - int rc = dbrmp->isReadWrite(); - - if (rc != 0 ) - { - setError(current_thd, ER_READ_ONLY_MODE, "Cannot execute the statement. DBRM is read only!"); - return ER_READ_ONLY_MODE; - } - - // stats start - ci->stats.reset(); - ci->stats.setStartTime(); - if (thd->main_security_ctx.user) - { - ci->stats.fUser = thd->main_security_ctx.user; - } - else - { - ci->stats.fUser = ""; - } - - if (thd->main_security_ctx.host) - ci->stats.fHost = thd->main_security_ctx.host; - else if (thd->main_security_ctx.host_or_ip) - ci->stats.fHost = thd->main_security_ctx.host_or_ip; - else - ci->stats.fHost = "unknown"; - - try - { - ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - - ci->stats.fSessionID = tid2sid(thd->thread_id); - - LEX* lex = thd->lex; - idbassert(lex != 0); - - // Error out DELETE on VIEW. It's currently not supported. - // @note DELETE on VIEW works natually (for simple cases at least), but we choose to turn it off - // for now - ZZ. - TABLE_LIST* tables = thd->lex->query_tables; - - for (; tables; tables = tables->next_local) - { - if (tables->view) - { - Message::Args args; - - if (isUpdateStatement(thd->lex->sql_command)) - args.add("Update"); + if (isUpdateStatement(thd->lex->sql_command)) + args.add("Update"); #if 0 else if (thd->rgi_slave && thd->rgi_slave->m_table_map.count() != 0) args.add("Row based replication event"); #endif - else - args.add("Delete"); + else + args.add("Delete"); - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_VIEW, args); - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; - } - - /* - #if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX) - if ((strcmp((*tables->table->s->db_plugin)->name.str, "InfiniDB") != 0) && (strcmp((*tables->table->s->db_plugin)->name.str, "MEMORY") != 0) && - (tables->table->s->table_category != TABLE_CATEGORY_TEMPORARY) ) - #else - if ((strcmp(tables->table->s->db_plugin->name.str, "InfiniDB") != 0) && (strcmp(tables->table->s->db_plugin->name.str, "MEMORY") != 0) && - (tables->table->s->table_category != TABLE_CATEGORY_TEMPORARY) ) - #endif - { - Message::Args args; - args.add("Non Calpont table(s)"); - string emsg(IDBErrorInfo::instance()->errorMsg(ERR_DML_NOT_SUPPORT_FEATURE, args)); - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; - } - */ + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_VIEW, args); + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; } - // @bug 1127. Re-construct update stmt using lex instead of using the original query. - char* query_char = idb_mysql_query_str(thd); - std::string dmlStmt; - if (!query_char) - { - dmlStmt = ""; - } - else - { - dmlStmt = query_char; - } - string schemaName; - string tableName(""); - string aliasName(""); - UpdateSqlStatement updateSqlStmt; - ColumnAssignmentList* colAssignmentListPtr = new ColumnAssignmentList(); - bool isFromCol = false; - bool isFromSameTable = true; - execplan::SCSEP updateCP(new execplan::CalpontSelectExecutionPlan()); + /* + #if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX) + if ((strcmp((*tables->table->s->db_plugin)->name.str, "InfiniDB") != 0) && + (strcmp((*tables->table->s->db_plugin)->name.str, "MEMORY") != 0) && (tables->table->s->table_category != + TABLE_CATEGORY_TEMPORARY) ) #else if ((strcmp(tables->table->s->db_plugin->name.str, "InfiniDB") != 0) && + (strcmp(tables->table->s->db_plugin->name.str, "MEMORY") != 0) && (tables->table->s->table_category != + TABLE_CATEGORY_TEMPORARY) ) #endif + { + Message::Args args; + args.add("Non Calpont table(s)"); + string emsg(IDBErrorInfo::instance()->errorMsg(ERR_DML_NOT_SUPPORT_FEATURE, + args)); setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); return ER_CHECK_NOT_IMPLEMENTED; + } + */ + } - updateCP->isDML(true); + // @bug 1127. Re-construct update stmt using lex instead of using the original query. + char* query_char = idb_mysql_query_str(thd); + std::string dmlStmt; + if (!query_char) + { + dmlStmt = ""; + } + else + { + dmlStmt = query_char; + } + string schemaName; + string tableName(""); + string aliasName(""); + UpdateSqlStatement updateSqlStmt; + ColumnAssignmentList* colAssignmentListPtr = new ColumnAssignmentList(); + bool isFromCol = false; + bool isFromSameTable = true; + execplan::SCSEP updateCP(new execplan::CalpontSelectExecutionPlan()); - //@Bug 2753. the memory already freed by destructor of UpdateSqlStatement - if (isUpdateStatement(thd->lex->sql_command)) + updateCP->isDML(true); + + //@Bug 2753. the memory already freed by destructor of UpdateSqlStatement + if (isUpdateStatement(thd->lex->sql_command)) + { + ColumnAssignment* columnAssignmentPtr; + Item_field* item; + List_iterator_fast field_it(thd->lex->first_select_lex()->item_list); + List_iterator_fast value_it(thd->lex->value_list); + updateCP->queryType(CalpontSelectExecutionPlan::UPDATE); + ci->stats.fQueryType = updateCP->queryType(); + uint32_t cnt = 0; + tr1::unordered_set timeStampColumnNames; + + while ((item = (Item_field*)field_it++)) { - ColumnAssignment* columnAssignmentPtr; - Item_field* item; - List_iterator_fast field_it(thd->lex->first_select_lex()->item_list); - List_iterator_fast value_it(thd->lex->value_list); - updateCP->queryType(CalpontSelectExecutionPlan::UPDATE); - ci->stats.fQueryType = updateCP->queryType(); - uint32_t cnt = 0; - tr1::unordered_set timeStampColumnNames; + cnt++; - while ((item = (Item_field*) field_it++)) + string tmpTableName = bestTableName(item); + + //@Bug 5312 populate aliasname with tablename if it is empty + if (!item->table_name.str) + aliasName = tmpTableName; + else + aliasName = item->table_name.str; + + if (lower_case_table_names) + { + boost::algorithm::to_lower(aliasName); + boost::algorithm::to_lower(tableName); + boost::algorithm::to_lower(tmpTableName); + } + + if (strcasecmp(tableName.c_str(), "") == 0) + { + tableName = tmpTableName; + } + else if (strcmp(tableName.c_str(), tmpTableName.c_str()) != 0) + { + //@ Bug3326 error out for multi table update + string emsg(IDBErrorInfo::instance()->errorMsg(ERR_UPDATE_NOT_SUPPORT_FEATURE)); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, emsg.c_str()); + ci->rc = -1; + thd->set_row_count_func(0); + return -1; + } + + if (!item->db_name.str) + { + //@Bug 5312. if subselect, wait until the schema info is available. + if (thd->derived_tables_processing) + return 0; + else { - cnt++; - - string tmpTableName = bestTableName(item); - - //@Bug 5312 populate aliasname with tablename if it is empty - if (!item->table_name.str) - aliasName = tmpTableName; - else - aliasName = item->table_name.str; - - if (lower_case_table_names) - { - boost::algorithm::to_lower(aliasName); - boost::algorithm::to_lower(tableName); - boost::algorithm::to_lower(tmpTableName); - } - - if (strcasecmp(tableName.c_str(), "") == 0) - { - tableName = tmpTableName; - } - else if (strcmp(tableName.c_str(), tmpTableName.c_str()) != 0) - { - //@ Bug3326 error out for multi table update - string emsg(IDBErrorInfo::instance()->errorMsg(ERR_UPDATE_NOT_SUPPORT_FEATURE)); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, emsg.c_str()); - ci->rc = -1; - thd->set_row_count_func(0); - return -1; - } - - if (!item->db_name.str) - { - //@Bug 5312. if subselect, wait until the schema info is available. - if (thd->derived_tables_processing) - return 0; - else - { - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The statement cannot be processed without schema."); - ci->rc = -1; - thd->set_row_count_func(0); - return -1; - } - } - else - { - schemaName = string(item->db_name.str); - if (lower_case_table_names) - { - boost::algorithm::to_lower(schemaName); - } - } - columnAssignmentPtr = new ColumnAssignment(item->name.str, "=", ""); - if (item->field_type() == MYSQL_TYPE_TIMESTAMP || - item->field_type() == MYSQL_TYPE_TIMESTAMP2) - { - timeStampColumnNames.insert(string(item->name.str)); - } - - Item* value = value_it++; - - if (value->type() == Item::CONST_ITEM) - { - if (value->cmp_type() == STRING_RESULT || - value->cmp_type() == DECIMAL_RESULT || - value->cmp_type() == REAL_RESULT) - { - //@Bug 2587 use val_str to replace value->name to get rid of 255 limit - String val, *str; - str = value->val_str(&val); - columnAssignmentPtr->fScalarExpression.assign(str->ptr(), str->length()); - columnAssignmentPtr->fFromCol = false; - } - else if (value->cmp_type() == INT_RESULT) - { - std::ostringstream oss; - - if (value->unsigned_flag) - { - oss << value->val_uint(); - } - else - { - oss << value->val_int(); - } - - columnAssignmentPtr->fScalarExpression = oss.str(); - columnAssignmentPtr->fFromCol = false; - } - } - else if ( value->type() == Item::FUNC_ITEM ) - { - //Bug 2092 handle negative values - Item_func* ifp = (Item_func*)value; - - if (ifp->result_type() == DECIMAL_RESULT) - columnAssignmentPtr->fFuncScale = ifp->decimals; //decimal scale - - vector tmpVec; - bool hasNonSupportItem = false; - uint16_t parseInfo = 0; - parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo); - - // const f&e evaluate here. @bug3513. Rule out special functions that takes - // no argument but needs to be sent to the back end to process. Like rand(), - // sysdate() etc. - if (!hasNonSupportItem && !cal_impl_if::nonConstFunc(ifp) && tmpVec.size() == 0) - { - gp_walk_info gwi; - gwi.thd = thd; - SRCP srcp(buildReturnedColumn(value, gwi, gwi.fatalParseError)); - ConstantColumn* constCol = dynamic_cast(srcp.get()); - - if (constCol ) - { - columnAssignmentPtr->fScalarExpression = constCol->constval(); - isFromCol = false; - columnAssignmentPtr->fFromCol = false; - } - else - { - isFromCol = true; - columnAssignmentPtr->fFromCol = true; - - } - } - else - { - isFromCol = true; - columnAssignmentPtr->fFromCol = true; - } - - if ( isFromCol ) - { - string sectableName(""); - string secschemaName (""); - - for ( unsigned i = 0; i < tmpVec.size(); i++ ) - { - sectableName = bestTableName(tmpVec[i]); - - if ( tmpVec[i]->db_name.str ) - { - secschemaName = string(tmpVec[i]->db_name.str); - } - - if ( (strcasecmp(tableName.c_str(), sectableName.c_str()) != 0) || - (strcasecmp(schemaName.c_str(), secschemaName.c_str()) != 0)) - { - isFromSameTable = false; - break; - } - } - } - } - else if ( value->type() == Item::FIELD_ITEM) - { - isFromCol = true; - columnAssignmentPtr->fFromCol = true; - Item_field* setIt = reinterpret_cast (value); - - // Minor optimization: - // do not perform updates of the form "update t1 set a = a;" - if (!strcmp(item->name.str, setIt->name.str) - && item->table_name.str && setIt->table_name.str && !strcmp(item->table_name.str, setIt->table_name.str) - && item->db_name.str && setIt->db_name.str && !strcmp(item->db_name.str, setIt->db_name.str)) - { - delete columnAssignmentPtr; - continue; - } - - string sectableName = string(setIt->table_name.str); - - if ( setIt->db_name.str ) //derived table - { - string secschemaName = string(setIt->db_name.str); - - if ( (strcasecmp(tableName.c_str(), sectableName.c_str()) != 0) || (strcasecmp(schemaName.c_str(), secschemaName.c_str()) != 0)) - { - isFromSameTable = false; - } - } - else - { - isFromSameTable = false; - } - } - else if ( value->type() == Item::NULL_ITEM ) - { - columnAssignmentPtr->fScalarExpression = ""; - columnAssignmentPtr->fFromCol = false; - columnAssignmentPtr->fIsNull = true; - } - else if ( value->type() == Item::SUBSELECT_ITEM ) - { - isFromCol = true; - columnAssignmentPtr->fFromCol = true; - isFromSameTable = false; - } - //@Bug 4449 handle default value - else if (value->type() == Item::DEFAULT_VALUE_ITEM) - { - Item_field* tmp = (Item_field*)value; - - if (!tmp->field_name.length) //null - { - columnAssignmentPtr->fScalarExpression = "NULL"; - columnAssignmentPtr->fFromCol = false; - } - else - { - String val, *str; - str = value->val_str(&val); - columnAssignmentPtr->fScalarExpression.assign(str->ptr(), str->length()); - columnAssignmentPtr->fFromCol = false; - } - } - else if (value->type() == Item::WINDOW_FUNC_ITEM) - { - setError(thd, ER_INTERNAL_ERROR, - logging::IDBErrorInfo::instance()->errorMsg(ERR_WF_UPDATE)); - return ER_CHECK_NOT_IMPLEMENTED; - } - else - { - String val, *str; - str = value->val_str(&val); - - if (str) - { - columnAssignmentPtr->fScalarExpression.assign(str->ptr(), str->length()); - columnAssignmentPtr->fFromCol = false; - } - else - { - columnAssignmentPtr->fScalarExpression = "NULL"; - columnAssignmentPtr->fFromCol = false; - } - } - - colAssignmentListPtr->push_back ( columnAssignmentPtr ); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, + "The statement cannot be processed without schema."); + ci->rc = -1; + thd->set_row_count_func(0); + return -1; } - - // Support for on update current_timestamp() for timestamp fields - // Query calpontsys.syscolumn to get all timestamp columns with - // ON UPDATE current_timestamp() property - vector onUpdateTimeStampColumns = getOnUpdateTimestampColumns(schemaName, tableName, tid2sid(thd->thread_id)); - for (size_t i = 0; i < onUpdateTimeStampColumns.size(); i++) + } + else + { + schemaName = string(item->db_name.str); + if (lower_case_table_names) { - if (timeStampColumnNames.find(onUpdateTimeStampColumns[i]) == timeStampColumnNames.end()) - { - // DRRTUY * That is far from ideal. - columnAssignmentPtr = new ColumnAssignment(string(onUpdateTimeStampColumns[i]), "=", ""); - struct timeval tv; - char buf[64]; - gettimeofday(&tv, 0); - MySQLTime time; - gmtSecToMySQLTime(tv.tv_sec, time, thd->variables.time_zone->get_name()->ptr()); - sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, time.minute, time.second, tv.tv_usec); - columnAssignmentPtr->fScalarExpression = buf; - colAssignmentListPtr->push_back ( columnAssignmentPtr ); - } + boost::algorithm::to_lower(schemaName); } + } + columnAssignmentPtr = new ColumnAssignment(item->name.str, "=", ""); + if (item->field_type() == MYSQL_TYPE_TIMESTAMP || item->field_type() == MYSQL_TYPE_TIMESTAMP2) + { + timeStampColumnNames.insert(string(item->name.str)); + } + + Item* value = value_it++; + + if (value->type() == Item::CONST_ITEM) + { + if (value->cmp_type() == STRING_RESULT || value->cmp_type() == DECIMAL_RESULT || + value->cmp_type() == REAL_RESULT) + { + //@Bug 2587 use val_str to replace value->name to get rid of 255 limit + String val, *str; + str = value->val_str(&val); + columnAssignmentPtr->fScalarExpression.assign(str->ptr(), str->length()); + columnAssignmentPtr->fFromCol = false; + } + else if (value->cmp_type() == INT_RESULT) + { + std::ostringstream oss; + + if (value->unsigned_flag) + { + oss << value->val_uint(); + } + else + { + oss << value->val_int(); + } + + columnAssignmentPtr->fScalarExpression = oss.str(); + columnAssignmentPtr->fFromCol = false; + } + } + else if (value->type() == Item::FUNC_ITEM) + { + // Bug 2092 handle negative values + Item_func* ifp = (Item_func*)value; + + if (ifp->result_type() == DECIMAL_RESULT) + columnAssignmentPtr->fFuncScale = ifp->decimals; // decimal scale + + vector tmpVec; + bool hasNonSupportItem = false; + uint16_t parseInfo = 0; + parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo); + + // const f&e evaluate here. @bug3513. Rule out special functions that takes + // no argument but needs to be sent to the back end to process. Like rand(), + // sysdate() etc. + if (!hasNonSupportItem && !cal_impl_if::nonConstFunc(ifp) && tmpVec.size() == 0) + { + gp_walk_info gwi; + gwi.thd = thd; + SRCP srcp(buildReturnedColumn(value, gwi, gwi.fatalParseError)); + ConstantColumn* constCol = dynamic_cast(srcp.get()); + + if (constCol) + { + columnAssignmentPtr->fScalarExpression = constCol->constval(); + isFromCol = false; + columnAssignmentPtr->fFromCol = false; + } + else + { + isFromCol = true; + columnAssignmentPtr->fFromCol = true; + } + } + else + { + isFromCol = true; + columnAssignmentPtr->fFromCol = true; + } + + if (isFromCol) + { + string sectableName(""); + string secschemaName(""); + + for (unsigned i = 0; i < tmpVec.size(); i++) + { + sectableName = bestTableName(tmpVec[i]); + + if (tmpVec[i]->db_name.str) + { + secschemaName = string(tmpVec[i]->db_name.str); + } + + if ((strcasecmp(tableName.c_str(), sectableName.c_str()) != 0) || + (strcasecmp(schemaName.c_str(), secschemaName.c_str()) != 0)) + { + isFromSameTable = false; + break; + } + } + } + } + else if (value->type() == Item::FIELD_ITEM) + { + isFromCol = true; + columnAssignmentPtr->fFromCol = true; + Item_field* setIt = reinterpret_cast(value); + + // Minor optimization: + // do not perform updates of the form "update t1 set a = a;" + if (!strcmp(item->name.str, setIt->name.str) && item->table_name.str && setIt->table_name.str && + !strcmp(item->table_name.str, setIt->table_name.str) && item->db_name.str && setIt->db_name.str && + !strcmp(item->db_name.str, setIt->db_name.str)) + { + delete columnAssignmentPtr; + continue; + } + + string sectableName = string(setIt->table_name.str); + + if (setIt->db_name.str) // derived table + { + string secschemaName = string(setIt->db_name.str); + + if ((strcasecmp(tableName.c_str(), sectableName.c_str()) != 0) || + (strcasecmp(schemaName.c_str(), secschemaName.c_str()) != 0)) + { + isFromSameTable = false; + } + } + else + { + isFromSameTable = false; + } + } + else if (value->type() == Item::NULL_ITEM) + { + columnAssignmentPtr->fScalarExpression = ""; + columnAssignmentPtr->fFromCol = false; + columnAssignmentPtr->fIsNull = true; + } + else if (value->type() == Item::SUBSELECT_ITEM) + { + isFromCol = true; + columnAssignmentPtr->fFromCol = true; + isFromSameTable = false; + } + //@Bug 4449 handle default value + else if (value->type() == Item::DEFAULT_VALUE_ITEM) + { + Item_field* tmp = (Item_field*)value; + + if (!tmp->field_name.length) // null + { + columnAssignmentPtr->fScalarExpression = "NULL"; + columnAssignmentPtr->fFromCol = false; + } + else + { + String val, *str; + str = value->val_str(&val); + columnAssignmentPtr->fScalarExpression.assign(str->ptr(), str->length()); + columnAssignmentPtr->fFromCol = false; + } + } + else if (value->type() == Item::WINDOW_FUNC_ITEM) + { + setError(thd, ER_INTERNAL_ERROR, logging::IDBErrorInfo::instance()->errorMsg(ERR_WF_UPDATE)); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + String val, *str; + str = value->val_str(&val); + + if (str) + { + columnAssignmentPtr->fScalarExpression.assign(str->ptr(), str->length()); + columnAssignmentPtr->fFromCol = false; + } + else + { + columnAssignmentPtr->fScalarExpression = "NULL"; + columnAssignmentPtr->fFromCol = false; + } + } + + colAssignmentListPtr->push_back(columnAssignmentPtr); } + + // Support for on update current_timestamp() for timestamp fields + // Query calpontsys.syscolumn to get all timestamp columns with + // ON UPDATE current_timestamp() property + vector onUpdateTimeStampColumns = + getOnUpdateTimestampColumns(schemaName, tableName, tid2sid(thd->thread_id)); + for (size_t i = 0; i < onUpdateTimeStampColumns.size(); i++) + { + if (timeStampColumnNames.find(onUpdateTimeStampColumns[i]) == timeStampColumnNames.end()) + { + // DRRTUY * That is far from ideal. + columnAssignmentPtr = new ColumnAssignment(string(onUpdateTimeStampColumns[i]), "=", ""); + struct timeval tv; + char buf[64]; + gettimeofday(&tv, 0); + MySQLTime time; + gmtSecToMySQLTime(tv.tv_sec, time, thd->variables.time_zone->get_name()->ptr()); + sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, + time.minute, time.second, tv.tv_usec); + columnAssignmentPtr->fScalarExpression = buf; + colAssignmentListPtr->push_back(columnAssignmentPtr); + } + } + } #if 0 else if (thd->rgi_slave && thd->rgi_slave->m_table_map.count() != 0) { @@ -1169,1048 +1179,1048 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c return ER_CHECK_NOT_IMPLEMENTED; } #endif - else + else + { + updateCP->queryType(CalpontSelectExecutionPlan::DELETE); + ci->stats.fQueryType = updateCP->queryType(); + } + + // Exit early if there is nothing to update + if (colAssignmentListPtr->empty() && isUpdateStatement(thd->lex->sql_command)) + { + ci->affectedRows = 0; + return 0; + } + + // save table oid for commit/rollback to use + uint32_t sessionID = tid2sid(thd->thread_id); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName aTableName; + TABLE_LIST* first_table = 0; + + if (isUpdateStatement(thd->lex->sql_command)) + { + aTableName.schema = schemaName; + aTableName.table = tableName; + } + else + { + first_table = (TABLE_LIST*)thd->lex->first_select_lex()->table_list.first; + aTableName.schema = first_table->table->s->db.str; + aTableName.table = first_table->table->s->table_name.str; + } + if (lower_case_table_names) + { + boost::algorithm::to_lower(aTableName.schema); + boost::algorithm::to_lower(aTableName.table); + } + + CalpontDMLPackage* pDMLPackage = 0; + // dmlStmt += ";"; + IDEBUG(cout << "STMT: " << dmlStmt << " and sessionID " << thd->thread_id << endl); + VendorDMLStatement dmlStatement(dmlStmt, sessionID); + + if (isUpdateStatement(thd->lex->sql_command)) + dmlStatement.set_DMLStatementType(DML_UPDATE); + else + dmlStatement.set_DMLStatementType(DML_DELETE); + + TableName* qualifiedTablName = new TableName(); + + UpdateSqlStatement updateStmt; + //@Bug 2753. To make sure the momory is freed. + updateStmt.fColAssignmentListPtr = colAssignmentListPtr; + + if (isUpdateStatement(thd->lex->sql_command)) + { + qualifiedTablName->fName = tableName; + qualifiedTablName->fSchema = schemaName; + updateStmt.fNamePtr = qualifiedTablName; + pDMLPackage = CalpontDMLFactory::makeCalpontUpdatePackageFromMysqlBuffer(dmlStatement, updateStmt); + } + else if ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) //@Bug 6121 error out on multi tables delete. + { + if ((thd->lex->first_select_lex()->join) != 0) { - updateCP->queryType(CalpontSelectExecutionPlan::DELETE); - ci->stats.fQueryType = updateCP->queryType(); - } + multi_delete* deleteTable = (multi_delete*)((thd->lex->first_select_lex()->join)->result); + first_table = (TABLE_LIST*)deleteTable->get_tables(); - // Exit early if there is nothing to update - if (colAssignmentListPtr->empty() && - isUpdateStatement(thd->lex->sql_command)) - { - ci->affectedRows = 0; - return 0; - } - - //save table oid for commit/rollback to use - uint32_t sessionID = tid2sid(thd->thread_id); - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableName aTableName; - TABLE_LIST* first_table = 0; - - if (isUpdateStatement(thd->lex->sql_command)) - { - aTableName.schema = schemaName; - aTableName.table = tableName; - } - else - { - first_table = (TABLE_LIST*) thd->lex->first_select_lex()->table_list.first; - aTableName.schema = first_table->table->s->db.str; - aTableName.table = first_table->table->s->table_name.str; - } - if (lower_case_table_names) - { - boost::algorithm::to_lower(aTableName.schema); - boost::algorithm::to_lower(aTableName.table); - } - - CalpontDMLPackage* pDMLPackage = 0; -// dmlStmt += ";"; - IDEBUG( cout << "STMT: " << dmlStmt << " and sessionID " << thd->thread_id << endl ); - VendorDMLStatement dmlStatement(dmlStmt, sessionID); - - if (isUpdateStatement(thd->lex->sql_command)) - dmlStatement.set_DMLStatementType(DML_UPDATE); - else - dmlStatement.set_DMLStatementType(DML_DELETE); - - TableName* qualifiedTablName = new TableName(); - - UpdateSqlStatement updateStmt; - //@Bug 2753. To make sure the momory is freed. - updateStmt.fColAssignmentListPtr = colAssignmentListPtr; - - if (isUpdateStatement(thd->lex->sql_command)) - { - qualifiedTablName->fName = tableName; - qualifiedTablName->fSchema = schemaName; - updateStmt.fNamePtr = qualifiedTablName; - pDMLPackage = CalpontDMLFactory::makeCalpontUpdatePackageFromMysqlBuffer( dmlStatement, updateStmt ); - } - else if ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) //@Bug 6121 error out on multi tables delete. - { - if ( (thd->lex->first_select_lex()->join) != 0) - { - multi_delete* deleteTable = (multi_delete*)((thd->lex->first_select_lex()->join)->result); - first_table = (TABLE_LIST*) deleteTable->get_tables(); - - if (deleteTable->get_num_of_tables() == 1) - { - schemaName = first_table->db.str; - tableName = first_table->table_name.str; - aliasName = first_table->alias.str; - if (lower_case_table_names) - { - boost::algorithm::to_lower(schemaName); - boost::algorithm::to_lower(tableName); - boost::algorithm::to_lower(aliasName); - } - qualifiedTablName->fName = tableName; - qualifiedTablName->fSchema = schemaName; - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); - } - else - { - string emsg("Deleting rows from multiple tables in a single statement is currently not supported."); - thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); - ci->rc = 1; - thd->set_row_count_func(0); - return 1; - } - } - else - { - first_table = (TABLE_LIST*) thd->lex->first_select_lex()->table_list.first; - schemaName = first_table->table->s->db.str; - tableName = first_table->table->s->table_name.str; - aliasName = first_table->alias.str; - if (lower_case_table_names) - { - boost::algorithm::to_lower(schemaName); - boost::algorithm::to_lower(tableName); - boost::algorithm::to_lower(aliasName); - } - qualifiedTablName->fName = tableName; - qualifiedTablName->fSchema = schemaName; - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); - } - } - else - { - first_table = (TABLE_LIST*) thd->lex->first_select_lex()->table_list.first; - schemaName = first_table->table->s->db.str; - tableName = first_table->table->s->table_name.str; + if (deleteTable->get_num_of_tables() == 1) + { + schemaName = first_table->db.str; + tableName = first_table->table_name.str; aliasName = first_table->alias.str; if (lower_case_table_names) { - boost::algorithm::to_lower(schemaName); - boost::algorithm::to_lower(tableName); - boost::algorithm::to_lower(aliasName); + boost::algorithm::to_lower(schemaName); + boost::algorithm::to_lower(tableName); + boost::algorithm::to_lower(aliasName); } qualifiedTablName->fName = tableName; qualifiedTablName->fSchema = schemaName; pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); - } - - if (!pDMLPackage) - { - string emsg("Fatal parse error in vtable mode in DMLParser "); - setError(thd, ER_INTERNAL_ERROR, emsg); - return ER_INTERNAL_ERROR; - } - - pDMLPackage->set_TableName(tableName); - - pDMLPackage->set_SchemaName(schemaName); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); - - pDMLPackage->set_IsFromCol( true ); - //cout << " setting isFromCol to " << isFromCol << endl; - std::string origStmt = dmlStmt; - origStmt += ";"; - pDMLPackage->set_SQLStatement( origStmt ); - - //Save the item list - List items; - SELECT_LEX select_lex; - - if (isUpdateStatement(thd->lex->sql_command)) - { - items = (thd->lex->first_select_lex()->item_list); - thd->lex->first_select_lex()->item_list = thd->lex->value_list; - } - - select_lex = *lex->first_select_lex(); - - - //@Bug 2808 Error out on order by or limit clause - //@bug5096. support dml limit. - if (( thd->lex->first_select_lex()->order_list.elements != 0 ) ) - { - string emsg("DML Statement with order by clause is not currently supported."); + } + else + { + string emsg("Deleting rows from multiple tables in a single statement is currently not supported."); thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); ci->rc = 1; thd->set_row_count_func(0); return 1; - } - - { - updateCP->subType (CalpontSelectExecutionPlan::SELECT_SUBS); - //@Bug 2975. - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - - updateCP->txnID(txnID.id); - updateCP->verID(verID); - updateCP->sessionID(sessionID); - char* query_char = idb_mysql_query_str(thd); - std::string query_str; - if (!query_char) - { - query_str = ""; - } - else - { - query_str = query_char; - } - updateCP->data(query_str); - - try - { - updateCP->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - - gwi.clauseType = WHERE; - - if (getSelectPlan(gwi, select_lex, updateCP, false, false, condStack) != 0) //@Bug 3030 Modify the error message for unsupported functions - { - if (gwi.cs_vtable_is_update_with_derive) - { - // @bug 4457. MySQL inconsistence! for some queries, some structures are only available - // in the derived_tables_processing phase. So by pass the phase for DML only when the - // execution plan can not be successfully generated. recover lex before returning; - thd->lex->first_select_lex()->item_list = items; - return 0; - } - - //check different error code - // avoid double set IDB error - string emsg; - - if (gwi.parseErrorText.find("MCS-") == string::npos) - { - Message::Args args; - args.add(gwi.parseErrorText); - emsg = IDBErrorInfo::instance()->errorMsg(ER_INTERNAL_ERROR, args); - } - else - { - emsg = gwi.parseErrorText; - } - - thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); - ci->rc = -1; - thd->set_row_count_func(0); - return -1; - - } - -//cout<< "Plan before is " << endl << *updateCP << endl; - //set the large side by putting the updated table at the first - CalpontSelectExecutionPlan::TableList tbList = updateCP->tableList(); - - CalpontSelectExecutionPlan::TableList::iterator iter = tbList.begin(); - bool notFirst = false; - - while ( iter != tbList.end() ) - { - if ( ( iter != tbList.begin() ) && (iter->schema == schemaName) && ( iter->table == tableName ) && ( iter->alias == aliasName ) ) - { - notFirst = true; - tbList.erase(iter); - break; - } - - iter++; - } - - if ( notFirst ) - { - CalpontSystemCatalog::TableAliasName tn = make_aliastable(schemaName, tableName, aliasName); - iter = tbList.begin(); - tbList.insert( iter, 1, tn ); - } - - updateCP->tableList( tbList ); - // DRRTUY * this is very optimisic assumption - updateCP->overrideLargeSideEstimate( true ); - //loop through returnedcols to find out constant columns - CalpontSelectExecutionPlan::ReturnedColumnList returnedCols = updateCP->returnedCols(); - CalpontSelectExecutionPlan::ReturnedColumnList::iterator coliter = returnedCols.begin(); - - while ( coliter != returnedCols.end() ) - { - ConstantColumn* returnCol = dynamic_cast((*coliter).get()); - - if (returnCol ) - { - returnedCols.erase(coliter); - coliter = returnedCols.begin(); - } - else - coliter++; - } - - if ((updateCP->columnMap()).empty()) - throw runtime_error ("column map is empty!"); - - if (returnedCols.empty()) - returnedCols.push_back((updateCP->columnMap()).begin()->second); - - //@Bug 6123. get the correct returned columnlist - if (isDeleteStatement(thd->lex->sql_command)) - { - returnedCols.clear(); - //choose the smallest column to project - CalpontSystemCatalog::TableName deleteTableName; - deleteTableName.schema = schemaName; - deleteTableName.table = tableName; - CalpontSystemCatalog::RIDList colrids; - - try - { - colrids = csc->columnRIDs(deleteTableName, false, lower_case_table_names); - } - catch (IDBExcept& ie) - { - thd->raise_error_printf(ER_INTERNAL_ERROR, ie.what()); - ci->rc = -1; - thd->set_row_count_func(0); - return -1; - } - - int minColWidth = -1; - int minWidthColOffset = 0; - - for (unsigned int j = 0; j < colrids.size(); j++) - { - CalpontSystemCatalog::ColType ct = csc->colType(colrids[j].objnum); - - if (ct.colDataType == CalpontSystemCatalog::VARBINARY) - continue; - - if (minColWidth == -1 || ct.colWidth < minColWidth) - { - minColWidth = ct.colWidth; - minWidthColOffset = j; - } - } - - CalpontSystemCatalog::TableColName tcn = csc->colName(colrids[minWidthColOffset].objnum); - SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); - sc->tableAlias(aliasName); - sc->timeZone(thd->variables.time_zone->get_name()->ptr()); - sc->resultType(csc->colType(colrids[minWidthColOffset].objnum)); - SRCP srcp; - srcp.reset(sc); - returnedCols.push_back(srcp); - //cout << "tablename:alias = " << tcn.table<<":"<returnedCols( returnedCols ); - - if (isUpdateStatement(thd->lex->sql_command)) - { - const ParseTree* ptsub = updateCP->filters(); - - if ( !isFromSameTable ) - { - //cout << "set scalar" << endl; - //walk tree to set scalar - if (ptsub) - ptsub->walk(makeUpdateScalarJoin, &tbList[0] ); - } - else - { - //cout << "set semi" << endl; - if (ptsub) - ptsub->walk(makeUpdateSemiJoin, &tbList[0] ); - } - - thd->lex->first_select_lex()->item_list = items; - } - - } - - //cout<< "Plan is " << endl << *updateCP << endl; - //updateCP->traceFlags(1); - pDMLPackage->HasFilter(true); - pDMLPackage->uuid(updateCP->uuid()); - - ByteStream bytestream, bytestream1; - bytestream << sessionID; - boost::shared_ptr plan = pDMLPackage->get_ExecutionPlan(); - updateCP->rmParms(ci->rmParms); - updateCP->serialize(*plan); - pDMLPackage->write(bytestream); - - delete pDMLPackage; - - ByteStream::byte b = 0; - ByteStream::octbyte rows = 0; - std::string errorMsg; - long long dmlRowCount = 0; - - if ( thd->killed > 0 ) - { - return 0; - } - - //querystats::QueryStats stats; - string tableLockInfo; - - // Save the tableOid for the COMMIT | ROLLBACK - CalpontSystemCatalog::ROPair roPair; - try - { - roPair = csc->tableRID(aTableName); - } - catch (IDBExcept& ie) - { - setError(thd, ER_INTERNAL_ERROR, ie.what()); - return ER_INTERNAL_ERROR; - } - catch (std::exception& ex) - { - setError(thd, ER_INTERNAL_ERROR, - logging::IDBErrorInfo::instance()->errorMsg(ERR_SYSTEM_CATALOG) + ex.what()); - return ER_INTERNAL_ERROR; - } - ci->tableOid = roPair.objnum; - - // Send the request to DMLProc and wait for a response. - try - { - timespec* tsp = 0; -#ifndef _MSC_VER - timespec ts; - ts.tv_sec = 3L; - ts.tv_nsec = 0L; - tsp = &ts; -#else - //FIXME: @#$%^&! mysql has buggered up timespec! - // The definition in my_pthread.h isn't the same as in winport/unistd.h... - struct timespec_foo - { - long tv_sec; - long tv_nsec; - } ts_foo; - ts_foo.tv_sec = 3; - ts_foo.tv_nsec = 0; - //This is only to get the compiler to not carp below at the read() call. - // The messagequeue lib uses the correct struct - tsp = reinterpret_cast(&ts_foo); -#endif - bool isTimeOut = true; - int maxRetries = 2; - std::string exMsg; - - // We try twice to get a response from dmlproc. - // Every (3) seconds, check for ctrl+c - for (int retry = 0; bytestream1.length() == 0 && retry < maxRetries; ++ retry) - { - try - { - if (!ci->dmlProc) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "doUpdateDelete start new DMLProc client " << ci->dmlProc << " for session " << sessionID << endl; - } - else - { - delete ci->dmlProc; - ci->dmlProc = nullptr; - ci->dmlProc = new MessageQueueClient("DMLProc"); - } - - // Send the request to DMLProc - ci->dmlProc->write(bytestream); - - // Get an answer from DMLProc - while (isTimeOut) - { - isTimeOut = false; - bytestream1 = ci->dmlProc->read(tsp, &isTimeOut); - - if (b == 0 && thd->killed > 0 && isTimeOut) - { - // We send the CTRL+C command to DMLProc out of band - // (on a different connection) This will cause - // DMLProc to stop processing and return an error on the - // original connection which will cause a rollback. - messageqcpp::MessageQueueClient ctrlCProc("DMLProc"); - //cout << "doUpdateDelete start new DMLProc client for ctrl-c " << " for session " << sessionID << endl; - VendorDMLStatement cmdStmt( "CTRL+C", DML_COMMAND, sessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); - ByteStream bytestream; - bytestream << static_cast(sessionID); - pDMLPackage->write(bytestream); - delete pDMLPackage; - b = 1; - retry = maxRetries; - errorMsg = "Command canceled by user"; - - try - { - ctrlCProc.write(bytestream); - } - catch (runtime_error&) - { - errorMsg = "Lost connection to DMLProc while doing ctrl+c"; - } - catch (...) - { - errorMsg = "Unknown error caught while doing ctrl+c"; - } - -// break; - } - } - } - catch (runtime_error& ex) - { - // An exception causes a retry, so fall thru - exMsg = ex.what(); - } - - if (bytestream1.length() == 0 && thd->killed <= 0) - { - //cout << "line 1442. received 0 byte from DMLProc and retry = "<< retry << endl; - // Seems dmlProc isn't playing. Reset it and try again. - delete ci->dmlProc; - ci->dmlProc = nullptr; - isTimeOut = true; //@Bug 4742 - } - } - - if (bytestream1.length() == 0) - { - // If we didn't get anything, error - b = 1; - - if (exMsg.length() > 0) - { - errorMsg = exMsg; - } - else - { - errorMsg = "Lost connection to DMLProc"; - } - } - else - { - bytestream1 >> b; - bytestream1 >> rows; - bytestream1 >> errorMsg; - - if (b == 0) - { - bytestream1 >> tableLockInfo; - bytestream1 >> ci->queryStats; - bytestream1 >> ci->extendedStats; - bytestream1 >> ci->miniStats; - ci->stats.unserialize(bytestream1); - } - } - - dmlRowCount = rows; - - if (thd->killed && b == 0) - { - b = dmlpackageprocessor::DMLPackageProcessor::JOB_CANCELED; - errorMsg = "Command canceled by user"; - } - } - catch (runtime_error& ex) - { - cout << ex.what() << endl; - b = 1; - delete ci->dmlProc; - ci->dmlProc = nullptr; - errorMsg = ex.what(); - } - catch ( ... ) - { - //cout << "... exception while writing to DMLProc" << endl; - b = 1; - delete ci->dmlProc; - ci->dmlProc = nullptr; - errorMsg = "Unknown error caught"; - } - - // If autocommit is on then go ahead and tell the engine to commit the transaction - //@Bug 1960 If error occurs, the commit is just to release the active transaction. - //@Bug 2241. Rollback transaction when it failed - //@Bug 4605. If error, always rollback. - if (b != dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) - { - std::string command; - - if ((useHdfs) && (b == 0)) - command = "COMMIT"; - else if ((useHdfs) && (b != 0)) - command = "ROLLBACK"; - else if ((b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) && thd->is_strict_mode()) - command = "ROLLBACK"; - else if ((!(current_thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) && (( b == 0 ) || (b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) ) - command = "COMMIT"; - else if (( b != 0 ) && (b != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) ) - command = "ROLLBACK"; - else - command = ""; - - if ( command != "") - { - VendorDMLStatement cmdStmt(command, DML_COMMAND, sessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); - pDMLPackage->setTableOid (ci->tableOid); - ByteStream bytestream; - bytestream << static_cast(sessionID); - pDMLPackage->write(bytestream); - delete pDMLPackage; - - ByteStream::byte bc; - std::string errMsg; - - try - { - if (!ci->dmlProc) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << " doupdateDelete command use a new dml client " << ci->dmlProc << endl; - } - - ci->dmlProc->write(bytestream); - bytestream1 = ci->dmlProc->read(); - bytestream1 >> bc; - bytestream1 >> rows; - bytestream1 >> errMsg; - - if ( b == 0 ) - { - b = bc; - errorMsg = errMsg; - } - } - catch (runtime_error&) - { - errorMsg = "Lost connection to DMLProc"; - b = 1; - delete ci->dmlProc; - ci->dmlProc = nullptr; - } - catch (...) - { - errorMsg = "Unknown error caught"; - b = 1; - } - - // Clear tableOid for the next SQL statement - ci->tableOid = 0; - } - } - - //@Bug 2241 Display an error message to user - - if ( ( b != 0 ) && (b != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) - { - //@Bug 2540. Set error status instead of warning - thd->raise_error_printf(ER_INTERNAL_ERROR, errorMsg.c_str()); - ci->rc = b; - rc = ER_INTERNAL_ERROR; - } - - if (b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - { - if (thd->is_strict_mode()) - { - thd->set_row_count_func(0); - ci->rc = b; - // Turn this on as MariaDB doesn't do it until the next phase - thd->abort_on_warning = thd->is_strict_mode(); - rc = ER_INTERNAL_ERROR; - } - else - { - ci->affectedRows = dmlRowCount; - } - - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errorMsg.c_str()); + } } else { -// if (dmlRowCount != 0) //Bug 5117. Handling self join. - ci->affectedRows = dmlRowCount; - //cout << " error status " << ci->rc << " and rowcount = " << dmlRowCount << endl; + first_table = (TABLE_LIST*)thd->lex->first_select_lex()->table_list.first; + schemaName = first_table->table->s->db.str; + tableName = first_table->table->s->table_name.str; + aliasName = first_table->alias.str; + if (lower_case_table_names) + { + boost::algorithm::to_lower(schemaName); + boost::algorithm::to_lower(tableName); + boost::algorithm::to_lower(aliasName); + } + qualifiedTablName->fName = tableName; + qualifiedTablName->fSchema = schemaName; + pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); } - - // insert query stats - ci->stats.setEndTime(); - - try + } + else + { + first_table = (TABLE_LIST*)thd->lex->first_select_lex()->table_list.first; + schemaName = first_table->table->s->db.str; + tableName = first_table->table->s->table_name.str; + aliasName = first_table->alias.str; + if (lower_case_table_names) { - ci->stats.insert(); - } - catch (std::exception& e) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + boost::algorithm::to_lower(schemaName); + boost::algorithm::to_lower(tableName); + boost::algorithm::to_lower(aliasName); } + qualifiedTablName->fName = tableName; + qualifiedTablName->fSchema = schemaName; + pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); + } - delete ci->dmlProc; - ci->dmlProc = nullptr; - return rc; -} + if (!pDMLPackage) + { + string emsg("Fatal parse error in vtable mode in DMLParser "); + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } -inline bool isSupportedToAnalyze(const execplan::CalpontSystemCatalog::ColType& colType) -{ - return colType.isUnsignedInteger() || colType.isSignedInteger(); -} + pDMLPackage->set_TableName(tableName); -// Initializes `cal_connection_info` using given `thd` and `sessionID`. -bool initializeCalConnectionInfo(cal_connection_info* ci, THD* thd, - boost::shared_ptr csc, - uint32_t sessionID, bool localQuery) -{ - ci->stats.reset(); - ci->stats.setStartTime(); + pDMLPackage->set_SchemaName(schemaName); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); - if (thd->main_security_ctx.user) - { - ci->stats.fUser = thd->main_security_ctx.user; - } - else - { - ci->stats.fUser = ""; - } + pDMLPackage->set_IsFromCol(true); + // cout << " setting isFromCol to " << isFromCol << endl; + std::string origStmt = dmlStmt; + origStmt += ";"; + pDMLPackage->set_SQLStatement(origStmt); - if (thd->main_security_ctx.host) - ci->stats.fHost = thd->main_security_ctx.host; - else if (thd->main_security_ctx.host_or_ip) - ci->stats.fHost = thd->main_security_ctx.host_or_ip; - else - ci->stats.fHost = "unknown"; + // Save the item list + List items; + SELECT_LEX select_lex; - try - { - ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - ci->warningMsg = msg; - } + if (isUpdateStatement(thd->lex->sql_command)) + { + items = (thd->lex->first_select_lex()->item_list); + thd->lex->first_select_lex()->item_list = thd->lex->value_list; + } - if (ci->queryState != 0) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + select_lex = *lex->first_select_lex(); - sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); - idbassert(ci->cal_conn_hndl != 0); - ci->cal_conn_hndl->csc = csc; - idbassert(ci->cal_conn_hndl->exeMgr != 0); - - try - { - ci->cal_conn_hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return false; - } - - return true; -} - -bool sendExecutionPlanToExeMgr(sm::cpsm_conhdl_t* hndl, ByteStream::quadbyte qb, - std::shared_ptr caep, - cal_connection_info* ci, THD* thd) -{ - ByteStream msg; - try - { - msg << qb; - hndl->exeMgr->write(msg); - msg.restart(); - caep->rmParms(ci->rmParms); - - // Send the execution plan. - caep->serialize(msg); - hndl->exeMgr->write(msg); - - // Get the status from ExeMgr. - msg.restart(); - msg = hndl->exeMgr->read(); - - // Any return code is ok for now. - if (msg.length() == 0) - { - auto emsg = "Lost connection to ExeMgr. Please contact your administrator"; - setError(thd, ER_INTERNAL_ERROR, emsg); - return false; - } - } - catch (...) - { - return false; - } - - return true; -} - -} //anon namespace - -int ha_mcs_impl_analyze(THD* thd, TABLE* table) -{ - uint32_t sessionID = execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id); - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - - csc->identity(execplan::CalpontSystemCatalog::FE); - - auto table_name = - execplan::make_table(table->s->db.str, table->s->table_name.str, lower_case_table_names); - - // Skip for now. - if (table->s->db.length && strcmp(table->s->db.str, "information_schema") == 0) - return 0; - - bool columnStore = (table ? isMCSTable(table) : true); - // Skip non columnstore tables. - if (!columnStore) - return 0; - - execplan::CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(table_name, true); - execplan::MCSAnalyzeTableExecutionPlan::ReturnedColumnList returnedColumnList; - execplan::MCSAnalyzeTableExecutionPlan::ColumnMap columnMap; - - // Iterate over table oid list and create a `SimpleColumn` for every column with supported type. - for (uint32_t i = 0, e = oidlist.size(); i < e; ++i) - { - execplan::SRCP returnedColumn; - const auto objNum = oidlist[i].objnum; - auto tableColName = csc->colName(objNum); - auto colType = csc->colType(objNum); - - if (!isSupportedToAnalyze(colType)) - continue; - - execplan::SimpleColumn* simpleColumn = new execplan::SimpleColumn(); - simpleColumn->columnName(tableColName.column); - simpleColumn->tableName(tableColName.table, lower_case_table_names); - simpleColumn->schemaName(tableColName.schema, lower_case_table_names); - simpleColumn->oid(objNum); - simpleColumn->alias(tableColName.column); - simpleColumn->resultType(colType); - simpleColumn->timeZone(thd->variables.time_zone->get_name()->ptr()); - - returnedColumn.reset(simpleColumn); - returnedColumnList.push_back(returnedColumn); - columnMap.insert(execplan::MCSAnalyzeTableExecutionPlan::ColumnMap::value_type( - simpleColumn->columnName(), returnedColumn)); - } - - // Create execution plan and initialize it with `returned columns` and `column map`. - std::shared_ptr caep( - new execplan::MCSAnalyzeTableExecutionPlan(returnedColumnList, columnMap)); - - caep->schemaName(table->s->db.str, lower_case_table_names); - caep->tableName(table->s->table_name.str, lower_case_table_names); - caep->timeZone(thd->variables.time_zone->get_name()->ptr()); + //@Bug 2808 Error out on order by or limit clause + //@bug5096. support dml limit. + if ((thd->lex->first_select_lex()->order_list.elements != 0)) + { + string emsg("DML Statement with order by clause is not currently supported."); + thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + ci->rc = 1; + thd->set_row_count_func(0); + return 1; + } + { + updateCP->subType(CalpontSelectExecutionPlan::SELECT_SUBS); + //@Bug 2975. SessionManager sm; BRM::TxnID txnID; txnID = sm.getTxnID(sessionID); if (!txnID.valid) { - txnID.id = 0; - txnID.valid = true; + txnID.id = 0; + txnID.valid = true; } QueryContext verID; verID = sm.verID(); - caep->txnID(txnID.id); - caep->verID(verID); - caep->sessionID(sessionID); - - string query; - query.assign(idb_mysql_query_str(thd)); - caep->data(query); - - if (!get_fe_conn_info_ptr()) - set_fe_conn_info_ptr(reinterpret_cast(new cal_connection_info(), thd)); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - idbassert(ci != 0); + updateCP->txnID(txnID.id); + updateCP->verID(verID); + updateCP->sessionID(sessionID); + char* query_char = idb_mysql_query_str(thd); + std::string query_str; + if (!query_char) + { + query_str = ""; + } + else + { + query_str = query_char; + } + updateCP->data(query_str); try { - caep->priority(ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + updateCP->priority(ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); } catch (std::exception& e) { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); } - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + gwi.clauseType = WHERE; + + if (getSelectPlan(gwi, select_lex, updateCP, false, false, condStack) != + 0) //@Bug 3030 Modify the error message for unsupported functions { - force_close_fep_conn(thd, ci); + if (gwi.cs_vtable_is_update_with_derive) + { + // @bug 4457. MySQL inconsistence! for some queries, some structures are only available + // in the derived_tables_processing phase. So by pass the phase for DML only when the + // execution plan can not be successfully generated. recover lex before returning; + thd->lex->first_select_lex()->item_list = items; return 0; + } + + // check different error code + // avoid double set IDB error + string emsg; + + if (gwi.parseErrorText.find("MCS-") == string::npos) + { + Message::Args args; + args.add(gwi.parseErrorText); + emsg = IDBErrorInfo::instance()->errorMsg(ER_INTERNAL_ERROR, args); + } + else + { + emsg = gwi.parseErrorText; + } + + thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + ci->rc = -1; + thd->set_row_count_func(0); + return -1; } - caep->traceFlags(ci->traceFlags); + // cout<< "Plan before is " << endl << *updateCP << endl; + // set the large side by putting the updated table at the first + CalpontSelectExecutionPlan::TableList tbList = updateCP->tableList(); - cal_table_info ti; - sm::cpsm_conhdl_t* hndl; + CalpontSelectExecutionPlan::TableList::iterator iter = tbList.begin(); + bool notFirst = false; - bool localQuery = (get_local_query(thd) > 0 ? true : false); - caep->localQuery(localQuery); - - // Try to initialize connection. - if (!initializeCalConnectionInfo(ci, thd, csc, sessionID, localQuery)) - goto error; - - hndl = ci->cal_conn_hndl; - - if (caep->traceOn()) - std::cout << caep->toString() << std::endl; + while (iter != tbList.end()) { - ByteStream::quadbyte qb = ANALYZE_TABLE_EXECUTE; - // Serialize and the send the `anlyze table` execution plan. - if (!sendExecutionPlanToExeMgr(hndl, qb, caep, ci, thd)) - goto error; + if ((iter != tbList.begin()) && (iter->schema == schemaName) && (iter->table == tableName) && + (iter->alias == aliasName)) + { + notFirst = true; + tbList.erase(iter); + break; + } + + iter++; } - ci->rmParms.clear(); - ci->tableMap[table] = ti; + if (notFirst) + { + CalpontSystemCatalog::TableAliasName tn = make_aliastable(schemaName, tableName, aliasName); + iter = tbList.begin(); + tbList.insert(iter, 1, tn); + } + updateCP->tableList(tbList); + // DRRTUY * this is very optimisic assumption + updateCP->overrideLargeSideEstimate(true); + // loop through returnedcols to find out constant columns + CalpontSelectExecutionPlan::ReturnedColumnList returnedCols = updateCP->returnedCols(); + CalpontSelectExecutionPlan::ReturnedColumnList::iterator coliter = returnedCols.begin(); + + while (coliter != returnedCols.end()) + { + ConstantColumn* returnCol = dynamic_cast((*coliter).get()); + + if (returnCol) + { + returnedCols.erase(coliter); + coliter = returnedCols.begin(); + } + else + coliter++; + } + + if ((updateCP->columnMap()).empty()) + throw runtime_error("column map is empty!"); + + if (returnedCols.empty()) + returnedCols.push_back((updateCP->columnMap()).begin()->second); + + //@Bug 6123. get the correct returned columnlist + if (isDeleteStatement(thd->lex->sql_command)) + { + returnedCols.clear(); + // choose the smallest column to project + CalpontSystemCatalog::TableName deleteTableName; + deleteTableName.schema = schemaName; + deleteTableName.table = tableName; + CalpontSystemCatalog::RIDList colrids; + + try + { + colrids = csc->columnRIDs(deleteTableName, false, lower_case_table_names); + } + catch (IDBExcept& ie) + { + thd->raise_error_printf(ER_INTERNAL_ERROR, ie.what()); + ci->rc = -1; + thd->set_row_count_func(0); + return -1; + } + + int minColWidth = -1; + int minWidthColOffset = 0; + + for (unsigned int j = 0; j < colrids.size(); j++) + { + CalpontSystemCatalog::ColType ct = csc->colType(colrids[j].objnum); + + if (ct.colDataType == CalpontSystemCatalog::VARBINARY) + continue; + + if (minColWidth == -1 || ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minWidthColOffset = j; + } + } + + CalpontSystemCatalog::TableColName tcn = csc->colName(colrids[minWidthColOffset].objnum); + SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); + sc->tableAlias(aliasName); + sc->timeZone(thd->variables.time_zone->get_name()->ptr()); + sc->resultType(csc->colType(colrids[minWidthColOffset].objnum)); + SRCP srcp; + srcp.reset(sc); + returnedCols.push_back(srcp); + // cout << "tablename:alias = " << tcn.table<<":"<returnedCols(returnedCols); + + if (isUpdateStatement(thd->lex->sql_command)) + { + const ParseTree* ptsub = updateCP->filters(); + + if (!isFromSameTable) + { + // cout << "set scalar" << endl; + // walk tree to set scalar + if (ptsub) + ptsub->walk(makeUpdateScalarJoin, &tbList[0]); + } + else + { + // cout << "set semi" << endl; + if (ptsub) + ptsub->walk(makeUpdateSemiJoin, &tbList[0]); + } + + thd->lex->first_select_lex()->item_list = items; + } + } + + // cout<< "Plan is " << endl << *updateCP << endl; + // updateCP->traceFlags(1); + pDMLPackage->HasFilter(true); + pDMLPackage->uuid(updateCP->uuid()); + + ByteStream bytestream, bytestream1; + bytestream << sessionID; + boost::shared_ptr plan = pDMLPackage->get_ExecutionPlan(); + updateCP->rmParms(ci->rmParms); + updateCP->serialize(*plan); + pDMLPackage->write(bytestream); + + delete pDMLPackage; + + ByteStream::byte b = 0; + ByteStream::octbyte rows = 0; + std::string errorMsg; + long long dmlRowCount = 0; + + if (thd->killed > 0) + { return 0; + } + + // querystats::QueryStats stats; + string tableLockInfo; + + // Save the tableOid for the COMMIT | ROLLBACK + CalpontSystemCatalog::ROPair roPair; + try + { + roPair = csc->tableRID(aTableName); + } + catch (IDBExcept& ie) + { + setError(thd, ER_INTERNAL_ERROR, ie.what()); + return ER_INTERNAL_ERROR; + } + catch (std::exception& ex) + { + setError(thd, ER_INTERNAL_ERROR, + logging::IDBErrorInfo::instance()->errorMsg(ERR_SYSTEM_CATALOG) + ex.what()); + return ER_INTERNAL_ERROR; + } + ci->tableOid = roPair.objnum; + + // Send the request to DMLProc and wait for a response. + try + { + timespec* tsp = 0; +#ifndef _MSC_VER + timespec ts; + ts.tv_sec = 3L; + ts.tv_nsec = 0L; + tsp = &ts; +#else + // FIXME: @#$%^&! mysql has buggered up timespec! + // The definition in my_pthread.h isn't the same as in winport/unistd.h... + struct timespec_foo + { + long tv_sec; + long tv_nsec; + } ts_foo; + ts_foo.tv_sec = 3; + ts_foo.tv_nsec = 0; + // This is only to get the compiler to not carp below at the read() call. + // The messagequeue lib uses the correct struct + tsp = reinterpret_cast(&ts_foo); +#endif + bool isTimeOut = true; + int maxRetries = 2; + std::string exMsg; + + // We try twice to get a response from dmlproc. + // Every (3) seconds, check for ctrl+c + for (int retry = 0; bytestream1.length() == 0 && retry < maxRetries; ++retry) + { + try + { + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + // cout << "doUpdateDelete start new DMLProc client " << ci->dmlProc << " for session " << sessionID + // << endl; + } + else + { + delete ci->dmlProc; + ci->dmlProc = nullptr; + ci->dmlProc = new MessageQueueClient("DMLProc"); + } + + // Send the request to DMLProc + ci->dmlProc->write(bytestream); + + // Get an answer from DMLProc + while (isTimeOut) + { + isTimeOut = false; + bytestream1 = ci->dmlProc->read(tsp, &isTimeOut); + + if (b == 0 && thd->killed > 0 && isTimeOut) + { + // We send the CTRL+C command to DMLProc out of band + // (on a different connection) This will cause + // DMLProc to stop processing and return an error on the + // original connection which will cause a rollback. + messageqcpp::MessageQueueClient ctrlCProc("DMLProc"); + // cout << "doUpdateDelete start new DMLProc client for ctrl-c " << " for session " << sessionID + // << endl; + VendorDMLStatement cmdStmt("CTRL+C", DML_COMMAND, sessionID); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + ByteStream bytestream; + bytestream << static_cast(sessionID); + pDMLPackage->write(bytestream); + delete pDMLPackage; + b = 1; + retry = maxRetries; + errorMsg = "Command canceled by user"; + + try + { + ctrlCProc.write(bytestream); + } + catch (runtime_error&) + { + errorMsg = "Lost connection to DMLProc while doing ctrl+c"; + } + catch (...) + { + errorMsg = "Unknown error caught while doing ctrl+c"; + } + + // break; + } + } + } + catch (runtime_error& ex) + { + // An exception causes a retry, so fall thru + exMsg = ex.what(); + } + + if (bytestream1.length() == 0 && thd->killed <= 0) + { + // cout << "line 1442. received 0 byte from DMLProc and retry = "<< retry << endl; + // Seems dmlProc isn't playing. Reset it and try again. + delete ci->dmlProc; + ci->dmlProc = nullptr; + isTimeOut = true; //@Bug 4742 + } + } + + if (bytestream1.length() == 0) + { + // If we didn't get anything, error + b = 1; + + if (exMsg.length() > 0) + { + errorMsg = exMsg; + } + else + { + errorMsg = "Lost connection to DMLProc"; + } + } + else + { + bytestream1 >> b; + bytestream1 >> rows; + bytestream1 >> errorMsg; + + if (b == 0) + { + bytestream1 >> tableLockInfo; + bytestream1 >> ci->queryStats; + bytestream1 >> ci->extendedStats; + bytestream1 >> ci->miniStats; + ci->stats.unserialize(bytestream1); + } + } + + dmlRowCount = rows; + + if (thd->killed && b == 0) + { + b = dmlpackageprocessor::DMLPackageProcessor::JOB_CANCELED; + errorMsg = "Command canceled by user"; + } + } + catch (runtime_error& ex) + { + cout << ex.what() << endl; + b = 1; + delete ci->dmlProc; + ci->dmlProc = nullptr; + errorMsg = ex.what(); + } + catch (...) + { + // cout << "... exception while writing to DMLProc" << endl; + b = 1; + delete ci->dmlProc; + ci->dmlProc = nullptr; + errorMsg = "Unknown error caught"; + } + + // If autocommit is on then go ahead and tell the engine to commit the transaction + //@Bug 1960 If error occurs, the commit is just to release the active transaction. + //@Bug 2241. Rollback transaction when it failed + //@Bug 4605. If error, always rollback. + if (b != dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) + { + std::string command; + + if ((useHdfs) && (b == 0)) + command = "COMMIT"; + else if ((useHdfs) && (b != 0)) + command = "ROLLBACK"; + else if ((b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) && thd->is_strict_mode()) + command = "ROLLBACK"; + else if ((!(current_thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) && + ((b == 0) || (b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING))) + command = "COMMIT"; + else if ((b != 0) && (b != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + command = "ROLLBACK"; + else + command = ""; + + if (command != "") + { + VendorDMLStatement cmdStmt(command, DML_COMMAND, sessionID); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->setTableOid(ci->tableOid); + ByteStream bytestream; + bytestream << static_cast(sessionID); + pDMLPackage->write(bytestream); + delete pDMLPackage; + + ByteStream::byte bc; + std::string errMsg; + + try + { + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + // cout << " doupdateDelete command use a new dml client " << ci->dmlProc << endl; + } + + ci->dmlProc->write(bytestream); + bytestream1 = ci->dmlProc->read(); + bytestream1 >> bc; + bytestream1 >> rows; + bytestream1 >> errMsg; + + if (b == 0) + { + b = bc; + errorMsg = errMsg; + } + } + catch (runtime_error&) + { + errorMsg = "Lost connection to DMLProc"; + b = 1; + delete ci->dmlProc; + ci->dmlProc = nullptr; + } + catch (...) + { + errorMsg = "Unknown error caught"; + b = 1; + } + + // Clear tableOid for the next SQL statement + ci->tableOid = 0; + } + } + + //@Bug 2241 Display an error message to user + + if ((b != 0) && (b != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + { + //@Bug 2540. Set error status instead of warning + thd->raise_error_printf(ER_INTERNAL_ERROR, errorMsg.c_str()); + ci->rc = b; + rc = ER_INTERNAL_ERROR; + } + + if (b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + { + if (thd->is_strict_mode()) + { + thd->set_row_count_func(0); + ci->rc = b; + // Turn this on as MariaDB doesn't do it until the next phase + thd->abort_on_warning = thd->is_strict_mode(); + rc = ER_INTERNAL_ERROR; + } + else + { + ci->affectedRows = dmlRowCount; + } + + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errorMsg.c_str()); + } + else + { + // if (dmlRowCount != 0) //Bug 5117. Handling self join. + ci->affectedRows = dmlRowCount; + // cout << " error status " << ci->rc << " and rowcount = " << dmlRowCount << endl; + } + + // insert query stats + ci->stats.setEndTime(); + + try + { + ci->stats.insert(); + } + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + + delete ci->dmlProc; + ci->dmlProc = nullptr; + return rc; +} + +inline bool isSupportedToAnalyze(const execplan::CalpontSystemCatalog::ColType& colType) +{ + return colType.isUnsignedInteger() || colType.isSignedInteger(); +} + +// Initializes `cal_connection_info` using given `thd` and `sessionID`. +bool initializeCalConnectionInfo(cal_connection_info* ci, THD* thd, + boost::shared_ptr csc, uint32_t sessionID, + bool localQuery) +{ + ci->stats.reset(); + ci->stats.setStartTime(); + + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + try + { + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + ci->warningMsg = msg; + } + + if (ci->queryState != 0) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); + idbassert(ci->cal_conn_hndl != 0); + ci->cal_conn_hndl->csc = csc; + idbassert(ci->cal_conn_hndl->exeMgr != 0); + + try + { + ci->cal_conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return false; + } + + return true; +} + +bool sendExecutionPlanToExeMgr(sm::cpsm_conhdl_t* hndl, ByteStream::quadbyte qb, + std::shared_ptr caep, + cal_connection_info* ci, THD* thd) +{ + ByteStream msg; + try + { + msg << qb; + hndl->exeMgr->write(msg); + msg.restart(); + caep->rmParms(ci->rmParms); + + // Send the execution plan. + caep->serialize(msg); + hndl->exeMgr->write(msg); + + // Get the status from ExeMgr. + msg.restart(); + msg = hndl->exeMgr->read(); + + // Any return code is ok for now. + if (msg.length() == 0) + { + auto emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return false; + } + } + catch (...) + { + return false; + } + + return true; +} + +} // namespace + +int ha_mcs_impl_analyze(THD* thd, TABLE* table) +{ + uint32_t sessionID = execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id); + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + + csc->identity(execplan::CalpontSystemCatalog::FE); + + auto table_name = execplan::make_table(table->s->db.str, table->s->table_name.str, lower_case_table_names); + + // Skip for now. + if (table->s->db.length && strcmp(table->s->db.str, "information_schema") == 0) + return 0; + + bool columnStore = (table ? isMCSTable(table) : true); + // Skip non columnstore tables. + if (!columnStore) + return 0; + + execplan::CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(table_name, true); + execplan::MCSAnalyzeTableExecutionPlan::ReturnedColumnList returnedColumnList; + execplan::MCSAnalyzeTableExecutionPlan::ColumnMap columnMap; + + // Iterate over table oid list and create a `SimpleColumn` for every column with supported type. + for (uint32_t i = 0, e = oidlist.size(); i < e; ++i) + { + execplan::SRCP returnedColumn; + const auto objNum = oidlist[i].objnum; + auto tableColName = csc->colName(objNum); + auto colType = csc->colType(objNum); + + if (!isSupportedToAnalyze(colType)) + continue; + + execplan::SimpleColumn* simpleColumn = new execplan::SimpleColumn(); + simpleColumn->columnName(tableColName.column); + simpleColumn->tableName(tableColName.table, lower_case_table_names); + simpleColumn->schemaName(tableColName.schema, lower_case_table_names); + simpleColumn->oid(objNum); + simpleColumn->alias(tableColName.column); + simpleColumn->resultType(colType); + simpleColumn->timeZone(thd->variables.time_zone->get_name()->ptr()); + + returnedColumn.reset(simpleColumn); + returnedColumnList.push_back(returnedColumn); + columnMap.insert(execplan::MCSAnalyzeTableExecutionPlan::ColumnMap::value_type(simpleColumn->columnName(), + returnedColumn)); + } + + // Create execution plan and initialize it with `returned columns` and `column map`. + std::shared_ptr caep( + new execplan::MCSAnalyzeTableExecutionPlan(returnedColumnList, columnMap)); + + caep->schemaName(table->s->db.str, lower_case_table_names); + caep->tableName(table->s->table_name.str, lower_case_table_names); + caep->timeZone(thd->variables.time_zone->get_name()->ptr()); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + caep->txnID(txnID.id); + caep->verID(verID); + caep->sessionID(sessionID); + + string query; + query.assign(idb_mysql_query_str(thd)); + caep->data(query); + + if (!get_fe_conn_info_ptr()) + set_fe_conn_info_ptr(reinterpret_cast(new cal_connection_info(), thd)); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + idbassert(ci != 0); + + try + { + caep->priority(ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + return 0; + } + + caep->traceFlags(ci->traceFlags); + + cal_table_info ti; + sm::cpsm_conhdl_t* hndl; + + bool localQuery = (get_local_query(thd) > 0 ? true : false); + caep->localQuery(localQuery); + + // Try to initialize connection. + if (!initializeCalConnectionInfo(ci, thd, csc, sessionID, localQuery)) + goto error; + + hndl = ci->cal_conn_hndl; + + if (caep->traceOn()) + std::cout << caep->toString() << std::endl; + { + ByteStream::quadbyte qb = ANALYZE_TABLE_EXECUTE; + // Serialize and the send the `anlyze table` execution plan. + if (!sendExecutionPlanToExeMgr(hndl, qb, caep, ci, thd)) + goto error; + } + + ci->rmParms.clear(); + ci->tableMap[table] = ti; + + return 0; error: - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - return ER_INTERNAL_ERROR; + return ER_INTERNAL_ERROR; } int ha_mcs_impl_open(const char* name, int mode, uint32_t test_if_locked) { - IDEBUG ( cout << "ha_mcs_impl_open: " << name << ", " << mode << ", " << test_if_locked << endl ); - Config::makeConfig(); - return 0; + IDEBUG(cout << "ha_mcs_impl_open: " << name << ", " << mode << ", " << test_if_locked << endl); + Config::makeConfig(); + return 0; } int ha_mcs_impl_close(void) { - IDEBUG( cout << "ha_mcs_impl_close" << endl ); - return 0; + IDEBUG(cout << "ha_mcs_impl_close" << endl); + return 0; } int ha_mcs_impl_discover_existence(const char* schema, const char* name) { - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); - try - { - const CalpontSystemCatalog::OID oid = csc->lookupTableOID(make_table(schema, name, lower_case_table_names)); + try + { + const CalpontSystemCatalog::OID oid = + csc->lookupTableOID(make_table(schema, name, lower_case_table_names)); - if (oid) - return 1; - } - catch ( ... ) - { - } + if (oid) + return 1; + } + catch (...) + { + } - return 0; + return 0; } -int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows, const std::vector& condStack) +int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows, + const std::vector& condStack) { - THD* thd = current_thd; - cal_impl_if::gp_walk_info gwi; - gwi.thd = thd; - int rc = 0; + THD* thd = current_thd; + cal_impl_if::gp_walk_info gwi; + gwi.thd = thd; + int rc = 0; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - { - if (affected_rows) - *affected_rows = 0; - return 0; - } + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + { + if (affected_rows) + *affected_rows = 0; + return 0; + } - if (execute) - { - rc = doUpdateDelete(thd, gwi, condStack); - } + if (execute) + { + rc = doUpdateDelete(thd, gwi, condStack); + } - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (ci) - { - *affected_rows = ci->affectedRows; - } + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + if (ci) + { + *affected_rows = ci->affectedRows; + } - return rc; + return rc; } int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) { - IDEBUG( cout << "rnd_init for table " << table->s->table_name.str << endl ); - THD* thd = current_thd; + IDEBUG(cout << "rnd_init for table " << table->s->table_name.str << endl); + THD* thd = current_thd; - gp_walk_info gwi; - gwi.thd = thd; + gp_walk_info gwi; + gwi.thd = thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return 0; - //check whether the system is ready to process statement. + // check whether the system is ready to process statement. #ifndef _MSC_VER - static DBRM dbrm(true); - int bSystemQueryReady = dbrm.getSystemQueryReady(); + static DBRM dbrm(true); + int bSystemQueryReady = dbrm.getSystemQueryReady(); - if (bSystemQueryReady == 0) - { - // Still not ready - setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); - return ER_INTERNAL_ERROR; - } - else if (bSystemQueryReady < 0) - { - // Still not ready - setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); - return ER_INTERNAL_ERROR; - } + if (bSystemQueryReady == 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); + return ER_INTERNAL_ERROR; + } + else if (bSystemQueryReady < 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); + return ER_INTERNAL_ERROR; + } #endif - // Set this to close all outstanding FEP connections on - // client disconnect in handlerton::closecon_handlerton(). - if ( !thd_get_ha_data(thd, mcs_hton)) - { - thd_set_ha_data(thd, mcs_hton, reinterpret_cast(0x42)); - } + // Set this to close all outstanding FEP connections on + // client disconnect in handlerton::closecon_handlerton(). + if (!thd_get_ha_data(thd, mcs_hton)) + { + thd_set_ha_data(thd, mcs_hton, reinterpret_cast(0x42)); + } #if 0 if (thd->rgi_slave && thd->rgi_slave->m_table_map.count() != 0) @@ -2221,1860 +2231,1877 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) } #endif - // If ALTER TABLE and not ENGINE= we don't need rnd_init (gets us in a bad state) - if ((thd->lex->sql_command == SQLCOM_ALTER_TABLE) && !(thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE)) + // If ALTER TABLE and not ENGINE= we don't need rnd_init (gets us in a bad state) + if ((thd->lex->sql_command == SQLCOM_ALTER_TABLE) && + !(thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE)) + { + return 0; + } + + /* + Update and delete code. + Note, we may be updating/deleting a different table, + and the current one is only needed for reading, + e.g. cstab1 is needed for reading in this example: + + UPDATE innotab1 SET a=100 WHERE a NOT IN (SELECT a FROM cstab1 WHERE a=1); + */ + if (!isReadOnly() && // make sure the current table is being modified + isUpdateOrDeleteStatement(thd->lex->sql_command)) + return doUpdateDelete(thd, gwi, condStack); + + uint32_t sessionID = tid2sid(thd->thread_id); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + idbassert(ci != 0); + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + return 0; + } + + sm::tableid_t tableid = 0; + cal_table_info ti; + sm::cpsm_conhdl_t* hndl; + SCSEP csep; + + // update traceFlags according to the autoswitch state. + ci->traceFlags |= CalpontSelectExecutionPlan::TRACE_TUPLE_OFF; + + bool localQuery = get_local_query(thd); + + // table mode + { + ti = ci->tableMap[table]; + + // get connection handle for this table handler + // re-establish table handle + if (ti.conn_hndl) { - return 0; + sm::sm_cleanup(ti.conn_hndl); + ti.conn_hndl = 0; } - /* - Update and delete code. - Note, we may be updating/deleting a different table, - and the current one is only needed for reading, - e.g. cstab1 is needed for reading in this example: + sm::sm_init(sessionID, &ti.conn_hndl, localQuery); + ti.conn_hndl->csc = csc; + hndl = ti.conn_hndl; - UPDATE innotab1 SET a=100 WHERE a NOT IN (SELECT a FROM cstab1 WHERE a=1); - */ - if (!isReadOnly() && // make sure the current table is being modified - isUpdateOrDeleteStatement(thd->lex->sql_command)) - return doUpdateDelete(thd, gwi, condStack); - - uint32_t sessionID = tid2sid(thd->thread_id); - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - idbassert(ci != 0); - - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + try { - force_close_fep_conn(thd, ci); - return 0; + ti.conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; } - sm::tableid_t tableid = 0; - cal_table_info ti; - sm::cpsm_conhdl_t* hndl; - SCSEP csep; - - // update traceFlags according to the autoswitch state. - ci->traceFlags |= CalpontSelectExecutionPlan::TRACE_TUPLE_OFF; - - bool localQuery = get_local_query(thd); - - // table mode + // get filter plan for table + if (ti.csep.get() == 0) { - ti = ci->tableMap[table]; + ti.csep.reset(new CalpontSelectExecutionPlan()); - // get connection handle for this table handler - // re-establish table handle - if (ti.conn_hndl) + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + ti.csep->txnID(txnID.id); + ti.csep->verID(verID); + ti.csep->sessionID(sessionID); + + if (thd->db.length) + ti.csep->schemaName(thd->db.str, lower_case_table_names); + + ti.csep->traceFlags(ci->traceFlags); + ti.msTablePtr = table; + + // send plan whenever rnd_init is called + cp_get_table_plan(thd, ti.csep, ti); + } + + IDEBUG(cerr << table->s->table_name.str << " send plan:" << endl); + IDEBUG(cerr << *ti.csep << endl); + csep = ti.csep; + + // for ExeMgr logging sqltext. only log once for the query although multi plans may be sent + // CS adds the ti into TM in the end of rnd_init thus we log the SQL + // only once when there is no ti with csep. + if (onlyOneTableinTM(ci)) + { + ti.csep->data(idb_mysql_query_str(thd)); + } + else + { + ti.csep->data(""); + } + } + + { + ByteStream msg; + ByteStream emsgBs; + + while (true) + { + try + { + ByteStream::quadbyte qb = 4; + msg << qb; + hndl->exeMgr->write(msg); + msg.restart(); + csep->rmParms(ci->rmParms); + + // send plan + csep->serialize(msg); + hndl->exeMgr->write(msg); + + // get ExeMgr status back to indicate a vtable joblist success or not + msg.restart(); + emsgBs.restart(); + msg = hndl->exeMgr->read(); + emsgBs = hndl->exeMgr->read(); + string emsg; + + if (msg.length() == 0 || emsgBs.length() == 0) { - sm::sm_cleanup(ti.conn_hndl); - ti.conn_hndl = 0; + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; } - sm::sm_init(sessionID, &ti.conn_hndl, localQuery); - ti.conn_hndl->csc = csc; - hndl = ti.conn_hndl; + string emsgStr; + emsgBs >> emsgStr; + bool err = false; - try + if (msg.length() == 4) { - ti.conn_hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } + msg >> qb; - // get filter plan for table - if (ti.csep.get() == 0) - { - ti.csep.reset(new CalpontSelectExecutionPlan()); + if (qb != 0) + { + err = true; + // for makejoblist error, stats contains only error code and insert from here + // because table fetch is not started + ci->stats.setEndTime(); + ci->stats.fQuery = csep->data(); + ci->stats.fQueryType = csep->queryType(); + ci->stats.fErrorNo = qb; - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) + try { - txnID.id = 0; - txnID.valid = true; + ci->stats.insert(); } - - QueryContext verID; - verID = sm.verID(); - - ti.csep->txnID(txnID.id); - ti.csep->verID(verID); - ti.csep->sessionID(sessionID); - - if (thd->db.length) - ti.csep->schemaName(thd->db.str, lower_case_table_names); - - ti.csep->traceFlags(ci->traceFlags); - ti.msTablePtr = table; - - // send plan whenever rnd_init is called - cp_get_table_plan(thd, ti.csep, ti); - } - - IDEBUG( cerr << table->s->table_name.str << " send plan:" << endl ); - IDEBUG( cerr << *ti.csep << endl ); - csep = ti.csep; - - // for ExeMgr logging sqltext. only log once for the query although multi plans may be sent - // CS adds the ti into TM in the end of rnd_init thus we log the SQL - // only once when there is no ti with csep. - if (onlyOneTableinTM(ci)) - { - ti.csep->data(idb_mysql_query_str(thd)); + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + } } else { - ti.csep->data(""); + err = true; } - } - { - ByteStream msg; - ByteStream emsgBs; - - while (true) + if (err) { - try - { - ByteStream::quadbyte qb = 4; - msg << qb; - hndl->exeMgr->write(msg); - msg.restart(); - csep->rmParms(ci->rmParms); - - //send plan - csep->serialize(msg); - hndl->exeMgr->write(msg); - - //get ExeMgr status back to indicate a vtable joblist success or not - msg.restart(); - emsgBs.restart(); - msg = hndl->exeMgr->read(); - emsgBs = hndl->exeMgr->read(); - string emsg; - - if (msg.length() == 0 || emsgBs.length() == 0) - { - emsg = "Lost connection to ExeMgr. Please contact your administrator"; - setError(thd, ER_INTERNAL_ERROR, emsg); - return ER_INTERNAL_ERROR; - } - - string emsgStr; - emsgBs >> emsgStr; - bool err = false; - - if (msg.length() == 4) - { - msg >> qb; - - if (qb != 0) - { - err = true; - // for makejoblist error, stats contains only error code and insert from here - // because table fetch is not started - ci->stats.setEndTime(); - ci->stats.fQuery = csep->data(); - ci->stats.fQueryType = csep->queryType(); - ci->stats.fErrorNo = qb; - - try - { - ci->stats.insert(); - } - catch (std::exception& e) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - } - } - else - { - err = true; - } - - if (err) - { - setError(thd, ER_INTERNAL_ERROR, emsgStr); - return ER_INTERNAL_ERROR; - } - - ci->rmParms.clear(); - - ci->tableMap[table] = ti; - - break; - } - catch (...) - { - sm::sm_cleanup(hndl); - hndl = 0; - - sm::sm_init(sessionID, &hndl, localQuery); - idbassert(hndl != 0); - hndl->csc = csc; - - ti.conn_hndl = hndl; - - try - { - hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } - - msg.restart(); - } + setError(thd, ER_INTERNAL_ERROR, emsgStr); + return ER_INTERNAL_ERROR; } + + ci->rmParms.clear(); + + ci->tableMap[table] = ti; + + break; + } + catch (...) + { + sm::sm_cleanup(hndl); + hndl = 0; + + sm::sm_init(sessionID, &hndl, localQuery); + idbassert(hndl != 0); + hndl->csc = csc; + + ti.conn_hndl = hndl; + + try + { + hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + msg.restart(); + } + } + } + + // common path for both vtable select phase and table mode -- open scan handle + ti = ci->tableMap[table]; + ti.msTablePtr = table; + + if (ti.tpl_ctx == nullptr) + { + ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); + } + + // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql + // call rnd_init for a table more than once. + ti.tpl_scan_ctx->rowGroup = nullptr; + + try + { + tableid = execplan::IDB_VTABLE_ID; + } + catch (...) + { + string emsg = "No table ID found for table " + string(table->s->table_name.str); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + try + { + sm::tpl_open(tableid, ti.tpl_ctx, hndl); + sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); + } + catch (std::exception& e) + { + string emsg = "table can not be opened: " + string(e.what()); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + catch (...) + { + string emsg = "table can not be opened"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + + if ((ti.tpl_scan_ctx->ctp).size() == 0) + { + uint32_t num_attr = table->s->fields; + + for (uint32_t i = 0; i < num_attr; i++) + { + CalpontSystemCatalog::ColType ctype; + ti.tpl_scan_ctx->ctp.push_back(ctype); } - // common path for both vtable select phase and table mode -- open scan handle - ti = ci->tableMap[table]; - ti.msTablePtr = table; - - if (ti.tpl_ctx == nullptr) + // populate coltypes here for table mode because tableband gives treeoid for dictionary column { - ti.tpl_ctx = new sm::cpsm_tplh_t(); - ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); - } + CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs( + make_table(table->s->db.str, table->s->table_name.str, lower_case_table_names), true); - // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql - // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = nullptr; - - try - { - tableid = execplan::IDB_VTABLE_ID; - } - catch (...) - { - string emsg = "No table ID found for table " + string(table->s->table_name.str); + if (oidlist.size() != num_attr) + { + string emsg = "Size mismatch probably caused by front end out of sync"; setError(thd, ER_INTERNAL_ERROR, emsg); CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); goto internal_error; + } + + for (unsigned int j = 0; j < oidlist.size(); j++) + { + CalpontSystemCatalog::ColType ctype = csc->colType(oidlist[j].objnum); + ti.tpl_scan_ctx->ctp[ctype.colPosition] = ctype; + ti.tpl_scan_ctx->ctp[ctype.colPosition].colPosition = -1; + } } + } - try - { - sm::tpl_open(tableid, ti.tpl_ctx, hndl); - sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); - } - catch (std::exception& e) - { - string emsg = "table can not be opened: " + string(e.what()); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - catch (...) - { - string emsg = "table can not be opened"; - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - - ti.tpl_scan_ctx->traceFlags = ci->traceFlags; - - if ((ti.tpl_scan_ctx->ctp).size() == 0) - { - uint32_t num_attr = table->s->fields; - - for (uint32_t i = 0; i < num_attr; i++) - { - CalpontSystemCatalog::ColType ctype; - ti.tpl_scan_ctx->ctp.push_back(ctype); - } - - // populate coltypes here for table mode because tableband gives treeoid for dictionary column - { - CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(make_table(table->s->db.str, table->s->table_name.str, lower_case_table_names), true); - - if (oidlist.size() != num_attr) - { - string emsg = "Size mismatch probably caused by front end out of sync"; - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - - for (unsigned int j = 0; j < oidlist.size(); j++) - { - CalpontSystemCatalog::ColType ctype = csc->colType(oidlist[j].objnum); - ti.tpl_scan_ctx->ctp[ctype.colPosition] = ctype; - ti.tpl_scan_ctx->ctp[ctype.colPosition].colPosition = -1; - } - } - } - - ci->tableMap[table] = ti; - return 0; + ci->tableMap[table] = ti; + return 0; error: - // CS doesn't need to close the actual sockets - // b/c it tries to reuse it running next query. - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + // CS doesn't need to close the actual sockets + // b/c it tries to reuse it running next query. + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - return ER_INTERNAL_ERROR; + return ER_INTERNAL_ERROR; internal_error: - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - return ER_INTERNAL_ERROR; + return ER_INTERNAL_ERROR; } int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) { - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return HA_ERR_END_OF_FILE; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return HA_ERR_END_OF_FILE; - if (isMCSTableUpdate(thd) || isMCSTableDelete(thd)) - return HA_ERR_END_OF_FILE; + if (isMCSTableUpdate(thd) || isMCSTableDelete(thd)) + return HA_ERR_END_OF_FILE; - // @bug 2547 - // MCOL-2178 This variable can never be true in the scope of this function - // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) - // return HA_ERR_END_OF_FILE; + // @bug 2547 + // MCOL-2178 This variable can never be true in the scope of this function + // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) + // return HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + return 0; + } + + if (ci->alterTableState > 0) + return HA_ERR_END_OF_FILE; + + cal_table_info ti; + ti = ci->tableMap[table]; + int rc = HA_ERR_END_OF_FILE; + + if (!ti.tpl_ctx || !ti.tpl_scan_ctx) + { + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + idbassert(ti.msTablePtr == table); + + try + { + rc = fetchNextRow(buf, ti, ci); + } + catch (std::exception& e) + { + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + ci->tableMap[table] = ti; + + if (rc != 0 && rc != HA_ERR_END_OF_FILE) + { + string emsg; + + // remove this check when all error handling migrated to the new framework. + if (rc >= 1000) + emsg = ti.tpl_scan_ctx->errMsg; + else { - force_close_fep_conn(thd, ci); - return 0; + logging::ErrorCodes errorcodes; + emsg = errorcodes.errorString(rc); } - if (ci->alterTableState > 0) return HA_ERR_END_OF_FILE; + setError(thd, ER_INTERNAL_ERROR, emsg); + // setError(thd, ER_INTERNAL_ERROR, "testing"); + ci->stats.fErrorNo = rc; + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + rc = ER_INTERNAL_ERROR; + } - cal_table_info ti; - ti = ci->tableMap[table]; - int rc = HA_ERR_END_OF_FILE; - - if (!ti.tpl_ctx || !ti.tpl_scan_ctx) - { - CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - return ER_INTERNAL_ERROR; - } - - idbassert(ti.msTablePtr == table); - - try - { - rc = fetchNextRow(buf, ti, ci); - } - catch (std::exception& e) - { - string emsg = string("Error while fetching from ExeMgr: ") + e.what(); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - return ER_INTERNAL_ERROR; - } - - ci->tableMap[table] = ti; - - if (rc != 0 && rc != HA_ERR_END_OF_FILE) - { - string emsg; - - // remove this check when all error handling migrated to the new framework. - if (rc >= 1000) - emsg = ti.tpl_scan_ctx->errMsg; - else - { - logging::ErrorCodes errorcodes; - emsg = errorcodes.errorString(rc); - } - - setError(thd, ER_INTERNAL_ERROR, emsg); - //setError(thd, ER_INTERNAL_ERROR, "testing"); - ci->stats.fErrorNo = rc; - CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - rc = ER_INTERNAL_ERROR; - } - - return rc; + return rc; } int ha_mcs_impl_rnd_end(TABLE* table, bool is_pushdown_hand) { - int rc = 0; - THD* thd = current_thd; + int rc = 0; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return 0; - cal_connection_info* ci = nullptr; + cal_connection_info* ci = nullptr; - if (get_fe_conn_info_ptr() != NULL) - ci = reinterpret_cast(get_fe_conn_info_ptr()); - - if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) - return rc; - - if (isMCSTableUpdate(thd) || isMCSTableDelete(thd)) - return rc; - - if (!ci) - { - set_fe_conn_info_ptr((void*)new cal_connection_info()); - ci = reinterpret_cast(get_fe_conn_info_ptr()); - } - - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) - { - force_close_fep_conn(thd, ci); - // clear querystats because no query stats available for cancelled query - ci->queryStats = ""; - return 0; - } - - IDEBUG( cerr << "rnd_end for table " << table->s->table_name.str << endl ); - - cal_table_info ti = ci->tableMap[table]; - sm::cpsm_conhdl_t* hndl; - - if (!is_pushdown_hand) - hndl = ti.conn_hndl; - else - hndl = ci->cal_conn_hndl; - - if (ti.tpl_ctx) - { - if (ti.tpl_scan_ctx.get()) - { - try - { - sm::tpl_scan_close(ti.tpl_scan_ctx); - } - catch (...) - { - rc = ER_INTERNAL_ERROR; - } - } - - ti.tpl_scan_ctx.reset(); - - try - { - { - bool ask_4_stats = (ci->traceFlags) ? true : false; - sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats, ask_4_stats); - } - - // set conn hndl back. could be changed in tpl_close - if (!is_pushdown_hand) - ti.conn_hndl = hndl; - else - ci->cal_conn_hndl = hndl; - - ti.tpl_ctx = 0; - } - catch (IDBExcept& e) - { - if (e.errorCode() == ERR_CROSS_ENGINE_CONNECT || e.errorCode() == ERR_CROSS_ENGINE_CONFIG) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - else - { - setError(thd, ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - } - catch (std::exception& e) - { - setError(thd, ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in rnd_end"); - rc = ER_INTERNAL_ERROR; - } - } - - ti.tpl_ctx = 0; - - ci->tableMap[table] = ti; - - // push warnings from CREATE phase - if (!ci->warningMsg.empty()) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, ci->warningMsg.c_str()); - - ci->warningMsg.clear(); - // reset expressionId just in case - ci->expressionId = 0; - - thd_set_ha_data(thd, mcs_hton, reinterpret_cast(ci)); + if (get_fe_conn_info_ptr() != NULL) + ci = reinterpret_cast(get_fe_conn_info_ptr()); + if ((thd->lex)->sql_command == SQLCOM_ALTER_TABLE) return rc; + + if (isMCSTableUpdate(thd) || isMCSTableDelete(thd)) + return rc; + + if (!ci) + { + set_fe_conn_info_ptr((void*)new cal_connection_info()); + ci = reinterpret_cast(get_fe_conn_info_ptr()); + } + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; + return 0; + } + + IDEBUG(cerr << "rnd_end for table " << table->s->table_name.str << endl); + + cal_table_info ti = ci->tableMap[table]; + sm::cpsm_conhdl_t* hndl; + + if (!is_pushdown_hand) + hndl = ti.conn_hndl; + else + hndl = ci->cal_conn_hndl; + + if (ti.tpl_ctx) + { + if (ti.tpl_scan_ctx.get()) + { + try + { + sm::tpl_scan_close(ti.tpl_scan_ctx); + } + catch (...) + { + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_scan_ctx.reset(); + + try + { + { + bool ask_4_stats = (ci->traceFlags) ? true : false; + sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats, ask_4_stats); + } + + // set conn hndl back. could be changed in tpl_close + if (!is_pushdown_hand) + ti.conn_hndl = hndl; + else + ci->cal_conn_hndl = hndl; + + ti.tpl_ctx = 0; + } + catch (IDBExcept& e) + { + if (e.errorCode() == ERR_CROSS_ENGINE_CONNECT || e.errorCode() == ERR_CROSS_ENGINE_CONFIG) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + else + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + } + catch (std::exception& e) + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in rnd_end"); + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_ctx = 0; + + ci->tableMap[table] = ti; + + // push warnings from CREATE phase + if (!ci->warningMsg.empty()) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, ci->warningMsg.c_str()); + + ci->warningMsg.clear(); + // reset expressionId just in case + ci->expressionId = 0; + + thd_set_ha_data(thd, mcs_hton, reinterpret_cast(ci)); + + return rc; } int ha_mcs_impl_create(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info) { - THD* thd = current_thd; + THD* thd = current_thd; - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - //@Bug 1948. Mysql calls create table to create a new table with new signature. - if (ci->alterTableState > 0) return 0; + //@Bug 1948. Mysql calls create table to create a new table with new signature. + if (ci->alterTableState > 0) + return 0; - // Just to be sure - if (!table_arg) - { - setError(thd, ER_INTERNAL_ERROR, "ha_mcs_impl_create_: table_arg is NULL"); - return 1; - } + // Just to be sure + if (!table_arg) + { + setError(thd, ER_INTERNAL_ERROR, "ha_mcs_impl_create_: table_arg is NULL"); + return 1; + } - if (!table_arg->s) - { - setError(thd, ER_INTERNAL_ERROR, "ha_mcs_impl_create_: table_arg->s is NULL"); - return 1; - } + if (!table_arg->s) + { + setError(thd, ER_INTERNAL_ERROR, "ha_mcs_impl_create_: table_arg->s is NULL"); + return 1; + } - int rc = ha_mcs_impl_create_(name, table_arg, create_info, *ci); + int rc = ha_mcs_impl_create_(name, table_arg, create_info, *ci); - return rc; + return rc; } int ha_mcs_impl_delete_table(const char* name) { - THD* thd = current_thd; - char* dbName = nullptr; + THD* thd = current_thd; + char* dbName = nullptr; - if (!name) - { - setError(thd, ER_INTERNAL_ERROR, "Drop Table with NULL name not permitted"); - return 1; - } + if (!name) + { + setError(thd, ER_INTERNAL_ERROR, "Drop Table with NULL name not permitted"); + return 1; + } - //if this is an InfiniDB tmp table ('#sql*.frm') just leave... - if (!memcmp((uchar*)name, tmp_file_prefix, tmp_file_prefix_length)) return 0; + // if this is an InfiniDB tmp table ('#sql*.frm') just leave... + if (!memcmp((uchar*)name, tmp_file_prefix, tmp_file_prefix_length)) + return 0; - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (!thd) return 0; + if (!thd) + return 0; - if (!thd->lex) return 0; + if (!thd->lex) + return 0; - if (!idb_mysql_query_str(thd)) return 0; + if (!idb_mysql_query_str(thd)) + return 0; - if (thd->lex->sql_command == SQLCOM_DROP_DB) - { - dbName = const_cast(thd->lex->name.str); - } - else - { - TABLE_LIST* first_table = (TABLE_LIST*) thd->lex->first_select_lex()->table_list.first; - dbName = const_cast(first_table->db.str); - } + if (thd->lex->sql_command == SQLCOM_DROP_DB) + { + dbName = const_cast(thd->lex->name.str); + } + else + { + TABLE_LIST* first_table = (TABLE_LIST*)thd->lex->first_select_lex()->table_list.first; + dbName = const_cast(first_table->db.str); + } - if (!dbName) - { - setError(thd, ER_INTERNAL_ERROR, "Drop Table with NULL schema not permitted"); - return 1; - } + if (!dbName) + { + setError(thd, ER_INTERNAL_ERROR, "Drop Table with NULL schema not permitted"); + return 1; + } - if (!ci) return 0; + if (!ci) + return 0; - //@Bug 1948,2306. if alter table want to drop the old table, InfiniDB does not need to drop. - if ( ci->isAlter ) - { - ci->isAlter = false; - return 0; - } + //@Bug 1948,2306. if alter table want to drop the old table, InfiniDB does not need to drop. + if (ci->isAlter) + { + ci->isAlter = false; + return 0; + } - int rc = ha_mcs_impl_delete_table_(dbName, name, *ci); - return rc; + int rc = ha_mcs_impl_delete_table_(dbName, name, *ci); + return rc; } int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) { - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd)) + return 0; - // Error out INSERT on VIEW. It's currently not supported. - // @note INSERT on VIEW works natually (for simple cases at least), but we choose to turn it off - // for now - ZZ. + // Error out INSERT on VIEW. It's currently not supported. + // @note INSERT on VIEW works natually (for simple cases at least), but we choose to turn it off + // for now - ZZ. - if (thd->lex->query_tables->view) + if (thd->lex->query_tables->view) + { + Message::Args args; + args.add("Insert"); + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_VIEW, args); + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + // At the beginning of insert, make sure there are no + // left-over values from a previously possibly failed insert. + if (rows_changed == 0) + ci->tableValuesMap.clear(); + + if (ci->alterTableState > 0) + return 0; + + ha_rows rowsInserted = 0; + int rc = 0; + + // ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ALWAYS means ALWAYS use + // cpimport, whether it's in a transaction or not. User should use this option + // very carefully since cpimport currently does not support rollbacks + if (((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ALWAYS) || + ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ON) && + (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))))) && + (!ci->singleInsert) && + ((ci->isLoaddataInfile) || ((thd->lex)->sql_command == SQLCOM_INSERT) || + ((thd->lex)->sql_command == SQLCOM_LOAD) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || + ci->isCacheInsert)) + { + rc = ha_mcs_impl_write_batch_row_(buf, table, *ci); + } + else + { + if (!ci->dmlProc) { - Message::Args args; - args.add("Insert"); - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_VIEW, args); - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; + ci->dmlProc = new MessageQueueClient("DMLProc"); + // cout << "write_row starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; } - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + rc = ha_mcs_impl_write_row_(buf, table, *ci, rowsInserted); + } - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + //@Bug 2438 Added a variable rowsHaveInserted to keep track of how many rows have been inserted already. + if (!ci->singleInsert && (rc == 0) && (rowsInserted > 0)) + { + ci->rowsHaveInserted += rowsInserted; + } - // At the beginning of insert, make sure there are no - // left-over values from a previously possibly failed insert. - if (rows_changed == 0) - ci->tableValuesMap.clear(); - - if (ci->alterTableState > 0) - return 0; - - ha_rows rowsInserted = 0; - int rc = 0; - - // ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ALWAYS means ALWAYS use - // cpimport, whether it's in a transaction or not. User should use this option - // very carefully since cpimport currently does not support rollbacks - if (((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ALWAYS) || - ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ON) && - (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))))) && - (!ci->singleInsert) && - ((ci->isLoaddataInfile) || - ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || - ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ci->isCacheInsert) ) - { - rc = ha_mcs_impl_write_batch_row_(buf, table, *ci); - } - else - { - if ( !ci->dmlProc ) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "write_row starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } - - rc = ha_mcs_impl_write_row_(buf, table, *ci, rowsInserted); - - } - - //@Bug 2438 Added a variable rowsHaveInserted to keep track of how many rows have been inserted already. - if ( !ci->singleInsert && ( rc == 0 ) && ( rowsInserted > 0 )) - { - ci->rowsHaveInserted += rowsInserted; - } - - return rc; + return rc; } int ha_mcs_impl_update_row() { - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - int rc = ci->rc; + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + int rc = ci->rc; - if ( rc != 0 ) - ci->rc = 0; + if (rc != 0) + ci->rc = 0; - return ( rc ); + return (rc); } int ha_mcs_impl_delete_row() { - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - int rc = ci->rc; + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + int rc = ci->rc; - if ( rc != 0 ) - ci->rc = 0; + if (rc != 0) + ci->rc = 0; - return ( rc ); + return (rc); } void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table, bool is_cache_insert) { - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd)) - return; + if (thd->slave_thread && !get_replication_slave(thd)) + return; - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - // clear rows variable - ci->rowsHaveInserted = 0; + // clear rows variable + ci->rowsHaveInserted = 0; - if (ci->alterTableState > 0) - return; + if (ci->alterTableState > 0) + return; - //@bug 5660. Error out DDL/DML on slave node, or on local query node - if (ci->isSlaveNode) + //@bug 5660. Error out DDL/DML on slave node, or on local query node + if (ci->isSlaveNode) + { + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return; + } + + //@bug 4771. reject REPLACE key word + if ((thd->lex)->sql_command == SQLCOM_REPLACE_SELECT) + { + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, "REPLACE statement is not supported in Columnstore."); + } + + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(thd->thread_id)); + csc->identity(execplan::CalpontSystemCatalog::FE); + + //@Bug 2515. + // Check command instead of vtable state + if ((thd->lex)->sql_command == SQLCOM_INSERT) + { + string insertStmt = idb_mysql_query_str(thd); + boost::algorithm::to_lower(insertStmt); + string intoStr("into"); + size_t found = insertStmt.find(intoStr); + + if (found != string::npos) + insertStmt.erase(found); + + found = insertStmt.find("ignore"); + + if (found != string::npos) { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return; + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, + "IGNORE option in insert statement is not supported in Columnstore."); } - //@bug 4771. reject REPLACE key word - if ((thd->lex)->sql_command == SQLCOM_REPLACE_SELECT) + if (rows > 1) { - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, "REPLACE statement is not supported in Columnstore."); + ci->singleInsert = false; + } + } + else if ((thd->lex)->sql_command == SQLCOM_LOAD || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) + { + ci->singleInsert = false; + ci->isLoaddataInfile = true; + } + + if (is_cache_insert && (thd->lex)->sql_command != SQLCOM_INSERT_SELECT) + { + ci->isCacheInsert = true; + + if (rows > 1) + ci->singleInsert = false; + } + + ci->bulkInsertRows = rows; + + if ((((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || + (thd->lex)->sql_command == SQLCOM_INSERT_SELECT || ci->isCacheInsert) && + !ci->singleInsert) + { + ci->useCpimport = get_use_import_for_batchinsert_mode(thd); + + if (((thd->lex)->sql_command == SQLCOM_INSERT) && (rows > 0)) + ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::OFF; + + if (ci->isCacheInsert) + { + if (get_cache_use_import(thd)) + ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ALWAYS; + else + ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::OFF; } - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(thd->thread_id)); - csc->identity(execplan::CalpontSystemCatalog::FE); + // ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ALWAYS means ALWAYS use + // cpimport, whether it's in a transaction or not. User should use this option + // very carefully since cpimport currently does not support rollbacks + if ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ALWAYS) || + ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ON) && + (!(thd->variables.option_bits & + (OPTION_NOT_AUTOCOMMIT | + OPTION_BEGIN))))) // If autocommit on batch insert will use cpimport to load data + { + // store table info to connection info + CalpontSystemCatalog::TableName tableName; + tableName.schema = table->s->db.str; + tableName.table = table->s->table_name.str; + ci->useXbit = false; + CalpontSystemCatalog::RIDList colrids; + + try + { + colrids = csc->columnRIDs(tableName, false, lower_case_table_names); + } + catch (IDBExcept& ie) + { + // TODO Can't use ERR_UNKNOWN_TABLE because it needs two + // arguments to format. Update setError to take vararg. + // setError(thd, ER_UNKNOWN_TABLE, ie.what()); + setError(thd, ER_INTERNAL_ERROR, ie.what()); + ci->rc = 5; + ci->singleInsert = true; + return; + } + + ci->useXbit = table->s->db_options_in_use & HA_OPTION_PACK_RECORD; + + // TODO: This needs a proper fix. + if (is_cache_insert) + ci->useXbit = false; + + //@bug 6122 Check how many columns have not null constraint. columnn with not null constraint will not + //show up in header. + unsigned int numberNotNull = 0; + + for (unsigned int j = 0; j < colrids.size(); j++) + { + CalpontSystemCatalog::ColType ctype = csc->colType(colrids[j].objnum); + ci->columnTypes.push_back(ctype); + + if (((ctype.colDataType == CalpontSystemCatalog::VARCHAR) || + (ctype.colDataType == CalpontSystemCatalog::VARBINARY)) && + !ci->useXbit) + ci->useXbit = true; + + if (ctype.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + numberNotNull++; + } + + // The length of the record header is:(1 + number of columns + 7) / 8 bytes + if (ci->useXbit) + ci->headerLength = (1 + colrids.size() + 7 - 1 - numberNotNull) / 8; // xbit is used + else + ci->headerLength = (1 + colrids.size() + 7 - numberNotNull) / 8; + + // Log the statement to debug.log + { + ostringstream oss; + oss << "Start SQL statement: " << idb_mysql_query_str(thd) << "; |" << table->s->db.str << "|"; + ha_mcs_impl::log_this(thd, oss.str().c_str(), logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } + + // start process cpimport mode 1 + ci->mysqld_pid = getpid(); + + // get delimiter + if (char(get_import_for_batchinsert_delimiter(thd)) != '\007') + ci->delimiter = char(get_import_for_batchinsert_delimiter(thd)); + else + ci->delimiter = '\007'; + + // get enclosed by + if (char(get_import_for_batchinsert_enclosed_by(thd)) != 8) + ci->enclosed_by = char(get_import_for_batchinsert_enclosed_by(thd)); + else + ci->enclosed_by = 8; + + // cout << "current set up is usecpimport:delimiter = " << (int)ci->useCpimport<<":"<< ci->delimiter + // < Cmds; + std::string aCmdLine; + std::string aTmpDir(startup::StartUp::tmpDir()); + + // If local module type is not PM and Local PM query is set, error out + char escapechar[2] = ""; + + if (ci->enclosed_by == 34) // Double quotes + strcat(escapechar, "\\"); + + if (get_local_query(thd)) + { + const auto oamcache = oam::OamCache::makeOamCache(); + int localModuleId = oamcache->getLocalPMId(); + + if (localModuleId == 0) + { + setError(current_thd, ER_INTERNAL_ERROR, + logging::IDBErrorInfo::instance()->errorMsg(ERR_LOCAL_QUERY_UM)); + ci->singleInsert = true; + ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + return; + } + else + { +#ifdef _MSC_VER + aCmdLine = "cpimport.exe -N -P " + to_string(localModuleId) + " -s " + ci->delimiter + " -e 0" + + " -E " + escapechar + ci->enclosed_by + " "; +#else + aCmdLine = "cpimport -m 1 -N -P " + boost::to_string(localModuleId) + " -s " + ci->delimiter + + " -e 0" + " -T " + thd->variables.time_zone->get_name()->ptr() + " -E " + escapechar + + ci->enclosed_by + " "; +#endif + } + } + else + { +#ifdef _MSC_VER + aCmdLine = + "cpimport.exe -N -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; +#else + aCmdLine = std::string("cpimport -m 1 -N -s ") + ci->delimiter + " -e 0" + " -T " + + thd->variables.time_zone->get_name()->ptr() + " -E " + escapechar + ci->enclosed_by + " "; +#endif + } + + aCmdLine = aCmdLine + table->s->db.str + " " + table->s->table_name.str; + + std::istringstream ss(aCmdLine); + std::string arg; + std::vector v2(20, ""); + unsigned int i = 0; + + while (ss >> arg) + { + v2[i++] = arg; + } + + for (unsigned int j = 0; j < i; ++j) + { + Cmds.push_back(const_cast(v2[j].c_str())); + } + + Cmds.push_back(0); // null terminate + +#ifdef _MSC_VER + BOOL bSuccess = false; + BOOL bInitialized = false; + SECURITY_ATTRIBUTES saAttr; + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = nullptr; + HANDLE handleList[2]; + const char* pSectionMsg; + bSuccess = true; + + // Create a pipe for the child process's STDIN. + if (bSuccess) + { + pSectionMsg = "Create Stdin"; + bSuccess = CreatePipe(&ci->cpimport_stdin_Rd, &ci->cpimport_stdin_Wr, &saAttr, 65536); + + // Ensure the write handle to the pipe for STDIN is not inherited. + if (bSuccess) + { + pSectionMsg = "SetHandleInformation(stdin)"; + bSuccess = SetHandleInformation(ci->cpimport_stdin_Wr, HANDLE_FLAG_INHERIT, 0); + } + } + + // Launch cpimport + LPPROC_THREAD_ATTRIBUTE_LIST lpAttributeList = nullptr; + SIZE_T attrSize = 0; + STARTUPINFOEX siStartInfo; + + // To ensure the child only inherits the STDIN and STDOUT Handles, we add a list of + // Handles that can be inherited to the call to CreateProcess + if (bSuccess) + { + pSectionMsg = "InitializeProcThreadAttributeList(NULL)"; + bSuccess = InitializeProcThreadAttributeList(NULL, 1, 0, &attrSize) || + GetLastError() == ERROR_INSUFFICIENT_BUFFER; // Asks how much buffer to alloc + } + + if (bSuccess) + { + pSectionMsg = "HeapAlloc for AttrList"; + lpAttributeList = + reinterpret_cast(HeapAlloc(GetProcessHeap(), 0, attrSize)); + bSuccess = lpAttributeList != nullptr; + } + + if (bSuccess) + { + pSectionMsg = "InitializeProcThreadAttributeList"; + bSuccess = InitializeProcThreadAttributeList(lpAttributeList, 1, 0, &attrSize); + } + + if (bSuccess) + { + pSectionMsg = "UpdateProcThreadAttribute"; + bInitialized = true; + handleList[0] = ci->cpimport_stdin_Rd; + bSuccess = UpdateProcThreadAttribute(lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + handleList, sizeof(HANDLE), NULL, NULL); + } + + if (bSuccess) + { + pSectionMsg = "CreateProcess"; + // In order for GenerateConsoleCtrlEvent (used when job is canceled) to work, + // this process must have a Console, which Services don't have. We create this + // when we create the child process. Once created, we leave it around for next time. + // AllocConsole will silently fail if it already exists, so no pain. + AllocConsole(); + // Set up members of the PROCESS_INFORMATION structure. + memset(&ci->cpimportProcInfo, 0, sizeof(PROCESS_INFORMATION)); + + // Set up members of the STARTUPINFOEX structure. + // This structure specifies the STDIN and STDOUT handles for redirection. + memset(&siStartInfo, 0, sizeof(STARTUPINFOEX)); + siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEX); + siStartInfo.lpAttributeList = lpAttributeList; + siStartInfo.StartupInfo.hStdError = nullptr; + siStartInfo.StartupInfo.hStdOutput = nullptr; + siStartInfo.StartupInfo.hStdInput = ci->cpimport_stdin_Rd; + siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; + // Create the child process. + bSuccess = CreateProcess(NULL, // program. NULL means use command line + const_cast(aCmdLine.c_str()), // command line + NULL, // process security attributes + NULL, // primary thread security attributes + TRUE, // handles are inherited + EXTENDED_STARTUPINFO_PRESENT | CREATE_NEW_PROCESS_GROUP, // creation flags + NULL, // use parent's environment + NULL, // use parent's current directory + &siStartInfo.StartupInfo, // STARTUPINFO pointer + &ci->cpimportProcInfo); // receives PROCESS_INFORMATION + } + + // We need to clean up the memory created by InitializeProcThreadAttributeList + // and HeapAlloc + if (bInitialized) + DeleteProcThreadAttributeList(lpAttributeList); + + if (lpAttributeList) + HeapFree(GetProcessHeap(), 0, lpAttributeList); + + if (!bSuccess) + { + // If an error occurs, Log and return. + int errnum = GetLastError(); + char errmsg[512]; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, errnum, 0, errmsg, 512, NULL); + ostringstream oss; + oss << " : Error in " << pSectionMsg << " (errno-" << errnum << "); " << errmsg; + setError(current_thd, ER_INTERNAL_ERROR, oss.str()); + ci->singleInsert = true; + ha_mcs_impl::log_this(thd, oss.str(), logging::LOG_TYPE_ERROR, tid2sid(thd->thread_id)); + ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + return; + } + + // Close the read handle that the child is using. We won't be needing this. + CloseHandle(ci->cpimport_stdin_Rd); + // The write functions all want a FILE* + ci->fdt[1] = _open_osfhandle((intptr_t)ci->cpimport_stdin_Wr, _O_APPEND); + ci->filePtr = _fdopen(ci->fdt[1], "w"); +#else + long maxFD = -1; + maxFD = sysconf(_SC_OPEN_MAX); + + if (pipe(ci->fdt) == -1) + { + int errnum = errno; + ostringstream oss; + oss << " : Error in creating pipe (errno-" << errnum << "); " << strerror(errnum); + setError(current_thd, ER_INTERNAL_ERROR, oss.str()); + ci->singleInsert = true; + ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + return; + } + + // cout << "maxFD = " << maxFD <singleInsert = true; + ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + return; + } + else if (aChPid == 0) // we are in child + { + for (int i = 0; i < maxFD; i++) + { + if (i != ci->fdt[0]) + close(i); + } + + errno = 0; + + if (dup2(ci->fdt[0], 0) < 0) // make stdin be the reading end of the pipe + { + setError(current_thd, ER_INTERNAL_ERROR, "dup2 failed"); + ci->singleInsert = true; + exit(1); + } + + close(ci->fdt[0]); // will trigger an EOF on stdin + ci->fdt[0] = -1; + open("/dev/null", O_WRONLY); + open("/dev/null", O_WRONLY); + errno = 0; + execvp(Cmds[0], &Cmds[0]); // NOTE - works with full Path + + int execvErrno = errno; + + ostringstream oss; + oss << " : execvp error: cpimport.bin invocation failed; " + << "(errno-" << errno << "); " << strerror(execvErrno) + << "; Check file and try invoking locally."; + cout << oss.str(); + + setError(current_thd, ER_INTERNAL_ERROR, "Forking process cpimport failed."); + ci->singleInsert = true; + ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + exit(1); + } + else // parent + { + ci->filePtr = fdopen(ci->fdt[1], "w"); + ci->cpimport_pid = aChPid; // This is the child PID + close(ci->fdt[0]); // close the READER of PARENT + ci->fdt[0] = -1; + // now we can send all the data thru FIFO[1], writer of PARENT + } + // Set read_set used for bulk insertion of Fields inheriting + // from Field_blob|Field_varstring. Used in ColWriteBatchString() + bitmap_set_all(table->read_set); + +#endif + } + else + { + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + } + } + } + + // Save table oid for commit to use + if ((((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || + (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || + ci->isCacheInsert) + { + // query stats. only collect execution time and rows inserted for insert/load_data_infile + ci->stats.reset(); + ci->stats.setStartTime(); + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + ci->stats.fSessionID = thd->thread_id; + ci->stats.fQuery = idb_mysql_query_str(thd); + + try + { + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } - //@Bug 2515. - //Check command instead of vtable state if ((thd->lex)->sql_command == SQLCOM_INSERT) + ci->stats.fQueryType = + CalpontSelectExecutionPlan::queryTypeToString(CalpontSelectExecutionPlan::INSERT); + else if ((thd->lex)->sql_command == SQLCOM_LOAD) + ci->stats.fQueryType = + CalpontSelectExecutionPlan::queryTypeToString(CalpontSelectExecutionPlan::LOAD_DATA_INFILE); + + //@Bug 4387. Check BRM status before start statement. + boost::scoped_ptr dbrmp(new DBRM()); + int rc = dbrmp->isReadWrite(); + + if (rc != 0) { - string insertStmt = idb_mysql_query_str(thd); - boost::algorithm::to_lower(insertStmt); - string intoStr("into"); - size_t found = insertStmt.find(intoStr); - - if (found != string::npos) - insertStmt.erase(found); - - found = insertStmt.find("ignore"); - - if (found != string::npos) - { - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, "IGNORE option in insert statement is not supported in Columnstore."); - } - - if ( rows > 1 ) - { - ci->singleInsert = false; - } - } - else if ( (thd->lex)->sql_command == SQLCOM_LOAD || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) - { - ci->singleInsert = false; - ci->isLoaddataInfile = true; + setError(current_thd, ER_READ_ONLY_MODE, "Cannot execute the statement. DBRM is read only!"); + ci->rc = rc; + ci->singleInsert = true; + bitmap_clear_all(table->read_set); + return; } - if (is_cache_insert && (thd->lex)->sql_command != SQLCOM_INSERT_SELECT) - { - ci->isCacheInsert = true; + uint32_t stateFlags; + dbrmp->getSystemState(stateFlags); - if (rows > 1) - ci->singleInsert = false; + if (stateFlags & SessionManagerServer::SS_SUSPENDED) + { + setError(current_thd, ER_INTERNAL_ERROR, "Writing to the database is disabled."); + bitmap_clear_all(table->read_set); + return; } - ci->bulkInsertRows = rows; + CalpontSystemCatalog::TableName tableName; + tableName.schema = table->s->db.str; + tableName.table = table->s->table_name.str; - if ((((thd->lex)->sql_command == SQLCOM_INSERT) || - ((thd->lex)->sql_command == SQLCOM_LOAD) || - (thd->lex)->sql_command == SQLCOM_INSERT_SELECT || - ci->isCacheInsert) && !ci->singleInsert ) + try { - ci->useCpimport = get_use_import_for_batchinsert_mode(thd); - - if (((thd->lex)->sql_command == SQLCOM_INSERT) && (rows > 0)) - ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::OFF; - - if (ci->isCacheInsert) - { - if (get_cache_use_import(thd)) - ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ALWAYS; - else - ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::OFF; - } - - // ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ALWAYS means ALWAYS use - // cpimport, whether it's in a transaction or not. User should use this option - // very carefully since cpimport currently does not support rollbacks - if ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ALWAYS) || - ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ON) && - (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))))) //If autocommit on batch insert will use cpimport to load data - { - //store table info to connection info - CalpontSystemCatalog::TableName tableName; - tableName.schema = table->s->db.str; - tableName.table = table->s->table_name.str; - ci->useXbit = false; - CalpontSystemCatalog::RIDList colrids; - - try - { - colrids = csc->columnRIDs(tableName, false, lower_case_table_names); - } - catch (IDBExcept& ie) - { - // TODO Can't use ERR_UNKNOWN_TABLE because it needs two - // arguments to format. Update setError to take vararg. -// setError(thd, ER_UNKNOWN_TABLE, ie.what()); - setError(thd, ER_INTERNAL_ERROR, ie.what()); - ci->rc = 5; - ci->singleInsert = true; - return; - } - - ci->useXbit = table->s->db_options_in_use & HA_OPTION_PACK_RECORD; - - // TODO: This needs a proper fix. - if (is_cache_insert) - ci->useXbit = false; - - //@bug 6122 Check how many columns have not null constraint. columnn with not null constraint will not show up in header. - unsigned int numberNotNull = 0; - - for (unsigned int j = 0; j < colrids.size(); j++) - { - CalpontSystemCatalog::ColType ctype = csc->colType(colrids[j].objnum); - ci->columnTypes.push_back(ctype); - - if ((( ctype.colDataType == CalpontSystemCatalog::VARCHAR ) || ( ctype.colDataType == CalpontSystemCatalog::VARBINARY )) && !ci->useXbit ) - ci->useXbit = true; - - if (ctype.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - numberNotNull++; - } - - // The length of the record header is:(1 + number of columns + 7) / 8 bytes - if (ci->useXbit) - ci->headerLength = (1 + colrids.size() + 7 - 1 - numberNotNull) / 8; //xbit is used - else - ci->headerLength = (1 + colrids.size() + 7 - numberNotNull) / 8; - - //Log the statement to debug.log - { - ostringstream oss; - oss << "Start SQL statement: " << idb_mysql_query_str(thd) << "; |" << table->s->db.str << "|"; - ha_mcs_impl::log_this(thd, oss.str().c_str(), logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - } - - //start process cpimport mode 1 - ci->mysqld_pid = getpid(); - - //get delimiter - if (char(get_import_for_batchinsert_delimiter(thd)) != '\007') - ci->delimiter = char(get_import_for_batchinsert_delimiter(thd)); - else - ci->delimiter = '\007'; - - //get enclosed by - if (char(get_import_for_batchinsert_enclosed_by(thd)) != 8) - ci->enclosed_by = char(get_import_for_batchinsert_enclosed_by(thd)); - else - ci->enclosed_by = 8; - - //cout << "current set up is usecpimport:delimiter = " << (int)ci->useCpimport<<":"<< ci->delimiter < Cmds; - std::string aCmdLine; - std::string aTmpDir(startup::StartUp::tmpDir()); - - //If local module type is not PM and Local PM query is set, error out - char escapechar[2] = ""; - - if (ci->enclosed_by == 34) // Double quotes - strcat(escapechar, "\\"); - - if (get_local_query(thd)) - { - const auto oamcache = oam::OamCache::makeOamCache(); - int localModuleId = oamcache->getLocalPMId(); - - if (localModuleId == 0) - { - setError(current_thd, ER_INTERNAL_ERROR, logging::IDBErrorInfo::instance()->errorMsg(ERR_LOCAL_QUERY_UM)); - ci->singleInsert = true; - ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, - tid2sid(thd->thread_id)); - return; - } - else - { -#ifdef _MSC_VER - aCmdLine = "cpimport.exe -N -P " + to_string(localModuleId) + " -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; -#else - aCmdLine = "cpimport -m 1 -N -P " + boost::to_string(localModuleId) + " -s " + ci->delimiter + " -e 0" + " -T " + thd->variables.time_zone->get_name()->ptr() + " -E " + escapechar + ci->enclosed_by + " "; -#endif - } - } - else - { -#ifdef _MSC_VER - aCmdLine = "cpimport.exe -N -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; -#else - aCmdLine = std::string("cpimport -m 1 -N -s ") + ci->delimiter + " -e 0" + " -T " + thd->variables.time_zone->get_name()->ptr() + " -E " + escapechar + ci->enclosed_by + " "; -#endif - } - - aCmdLine = aCmdLine + table->s->db.str + " " + table->s->table_name.str ; - - std::istringstream ss(aCmdLine); - std::string arg; - std::vector v2(20, ""); - unsigned int i = 0; - - while (ss >> arg) - { - v2[i++] = arg; - } - - for (unsigned int j = 0; j < i; ++j) - { - Cmds.push_back(const_cast(v2[j].c_str())); - } - - Cmds.push_back(0); //null terminate - -#ifdef _MSC_VER - BOOL bSuccess = false; - BOOL bInitialized = false; - SECURITY_ATTRIBUTES saAttr; - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - saAttr.lpSecurityDescriptor = nullptr; - HANDLE handleList[2]; - const char* pSectionMsg; - bSuccess = true; - - // Create a pipe for the child process's STDIN. - if (bSuccess) - { - pSectionMsg = "Create Stdin"; - bSuccess = CreatePipe(&ci->cpimport_stdin_Rd, &ci->cpimport_stdin_Wr, &saAttr, 65536); - - // Ensure the write handle to the pipe for STDIN is not inherited. - if (bSuccess) - { - pSectionMsg = "SetHandleInformation(stdin)"; - bSuccess = SetHandleInformation(ci->cpimport_stdin_Wr, HANDLE_FLAG_INHERIT, 0); - } - } - - // Launch cpimport - LPPROC_THREAD_ATTRIBUTE_LIST lpAttributeList = nullptr; - SIZE_T attrSize = 0; - STARTUPINFOEX siStartInfo; - - // To ensure the child only inherits the STDIN and STDOUT Handles, we add a list of - // Handles that can be inherited to the call to CreateProcess - if (bSuccess) - { - pSectionMsg = "InitializeProcThreadAttributeList(NULL)"; - bSuccess = InitializeProcThreadAttributeList(NULL, 1, 0, &attrSize) || - GetLastError() == ERROR_INSUFFICIENT_BUFFER; // Asks how much buffer to alloc - } - - if (bSuccess) - { - pSectionMsg = "HeapAlloc for AttrList"; - lpAttributeList = reinterpret_cast - (HeapAlloc(GetProcessHeap(), 0, attrSize)); - bSuccess = lpAttributeList != nullptr; - } - - if (bSuccess) - { - pSectionMsg = "InitializeProcThreadAttributeList"; - bSuccess = InitializeProcThreadAttributeList(lpAttributeList, 1, 0, &attrSize); - } - - if (bSuccess) - { - pSectionMsg = "UpdateProcThreadAttribute"; - bInitialized = true; - handleList[0] = ci->cpimport_stdin_Rd; - bSuccess = UpdateProcThreadAttribute(lpAttributeList, - 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, - handleList, sizeof(HANDLE), NULL, NULL); - } - - if (bSuccess) - { - pSectionMsg = "CreateProcess"; - // In order for GenerateConsoleCtrlEvent (used when job is canceled) to work, - // this process must have a Console, which Services don't have. We create this - // when we create the child process. Once created, we leave it around for next time. - // AllocConsole will silently fail if it already exists, so no pain. - AllocConsole(); - // Set up members of the PROCESS_INFORMATION structure. - memset(&ci->cpimportProcInfo, 0, sizeof(PROCESS_INFORMATION)); - - // Set up members of the STARTUPINFOEX structure. - // This structure specifies the STDIN and STDOUT handles for redirection. - memset(&siStartInfo, 0, sizeof(STARTUPINFOEX)); - siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEX); - siStartInfo.lpAttributeList = lpAttributeList; - siStartInfo.StartupInfo.hStdError = nullptr; - siStartInfo.StartupInfo.hStdOutput = nullptr; - siStartInfo.StartupInfo.hStdInput = ci->cpimport_stdin_Rd; - siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; - // Create the child process. - bSuccess = CreateProcess(NULL, // program. NULL means use command line - const_cast(aCmdLine.c_str()), // command line - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - EXTENDED_STARTUPINFO_PRESENT | CREATE_NEW_PROCESS_GROUP, // creation flags - NULL, // use parent's environment - NULL, // use parent's current directory - &siStartInfo.StartupInfo, // STARTUPINFO pointer - &ci->cpimportProcInfo); // receives PROCESS_INFORMATION - - } - - // We need to clean up the memory created by InitializeProcThreadAttributeList - // and HeapAlloc - if (bInitialized) - DeleteProcThreadAttributeList(lpAttributeList); - - if (lpAttributeList) - HeapFree(GetProcessHeap(), 0, lpAttributeList); - - if (!bSuccess) - { - // If an error occurs, Log and return. - int errnum = GetLastError(); - char errmsg[512]; - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, errnum, 0, errmsg, 512, NULL); - ostringstream oss; - oss << " : Error in " << pSectionMsg << " (errno-" << - errnum << "); " << errmsg; - setError(current_thd, ER_INTERNAL_ERROR, oss.str()); - ci->singleInsert = true; - ha_mcs_impl::log_this(thd, oss.str(), logging::LOG_TYPE_ERROR, tid2sid(thd->thread_id)); - ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - return; - } - - // Close the read handle that the child is using. We won't be needing this. - CloseHandle(ci->cpimport_stdin_Rd); - // The write functions all want a FILE* - ci->fdt[1] = _open_osfhandle((intptr_t)ci->cpimport_stdin_Wr, _O_APPEND); - ci->filePtr = _fdopen(ci->fdt[1], "w"); -#else - long maxFD = -1; - maxFD = sysconf(_SC_OPEN_MAX); - - if (pipe(ci->fdt) == -1) - { - int errnum = errno; - ostringstream oss; - oss << " : Error in creating pipe (errno-" << - errnum << "); " << strerror(errnum); - setError(current_thd, ER_INTERNAL_ERROR, oss.str()); - ci->singleInsert = true; - ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - return; - } - - //cout << "maxFD = " << maxFD <singleInsert = true; - ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - return; - } - else if (aChPid == 0) // we are in child - { - for (int i = 0; i < maxFD; i++) - { - if (i != ci->fdt[0]) - close(i); - } - - errno = 0; - - if (dup2(ci->fdt[0], 0) < 0) //make stdin be the reading end of the pipe - { - setError(current_thd, ER_INTERNAL_ERROR, "dup2 failed"); - ci->singleInsert = true; - exit (1); - } - - close(ci->fdt[0]); // will trigger an EOF on stdin - ci->fdt[0] = -1; - open("/dev/null", O_WRONLY); - open("/dev/null", O_WRONLY); - errno = 0; - execvp(Cmds[0], &Cmds[0]); //NOTE - works with full Path - - int execvErrno = errno; - - ostringstream oss; - oss << " : execvp error: cpimport.bin invocation failed; " - << "(errno-" << errno << "); " << strerror(execvErrno) << - "; Check file and try invoking locally."; - cout << oss.str(); - - setError(current_thd, ER_INTERNAL_ERROR, "Forking process cpimport failed."); - ci->singleInsert = true; - ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, - tid2sid(thd->thread_id)); - exit(1); - } - else // parent - { - ci->filePtr = fdopen(ci->fdt[1], "w"); - ci->cpimport_pid = aChPid; // This is the child PID - close(ci->fdt[0]); //close the READER of PARENT - ci->fdt[0] = -1; - // now we can send all the data thru FIFO[1], writer of PARENT - } - // Set read_set used for bulk insertion of Fields inheriting - //from Field_blob|Field_varstring. Used in ColWriteBatchString() - bitmap_set_all(table->read_set); - -#endif - } - else - { - if (!ci->dmlProc) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - } - } + CalpontSystemCatalog::ROPair roPair = csc->tableRID(tableName, lower_case_table_names); + ci->tableOid = roPair.objnum; } - - //Save table oid for commit to use - if ( ( ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ci->isCacheInsert) + catch (IDBExcept& ie) { - // query stats. only collect execution time and rows inserted for insert/load_data_infile - ci->stats.reset(); - ci->stats.setStartTime(); - if (thd->main_security_ctx.user) - { - ci->stats.fUser = thd->main_security_ctx.user; - } - else - { - ci->stats.fUser = ""; - } - - if (thd->main_security_ctx.host) - ci->stats.fHost = thd->main_security_ctx.host; - else if (thd->main_security_ctx.host_or_ip) - ci->stats.fHost = thd->main_security_ctx.host_or_ip; - else - ci->stats.fHost = "unknown"; - - ci->stats.fSessionID = thd->thread_id; - ci->stats.fQuery = idb_mysql_query_str(thd); - - try - { - ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - - if ((thd->lex)->sql_command == SQLCOM_INSERT) - ci->stats.fQueryType = CalpontSelectExecutionPlan::queryTypeToString(CalpontSelectExecutionPlan::INSERT); - else if ((thd->lex)->sql_command == SQLCOM_LOAD) - ci->stats.fQueryType = CalpontSelectExecutionPlan::queryTypeToString(CalpontSelectExecutionPlan::LOAD_DATA_INFILE); - - //@Bug 4387. Check BRM status before start statement. - boost::scoped_ptr dbrmp(new DBRM()); - int rc = dbrmp->isReadWrite(); - - if (rc != 0 ) - { - setError(current_thd, ER_READ_ONLY_MODE, "Cannot execute the statement. DBRM is read only!"); - ci->rc = rc; - ci->singleInsert = true; - bitmap_clear_all(table->read_set); - return; - } - - uint32_t stateFlags; - dbrmp->getSystemState(stateFlags); - - if (stateFlags & SessionManagerServer::SS_SUSPENDED) - { - setError(current_thd, ER_INTERNAL_ERROR, "Writing to the database is disabled."); - bitmap_clear_all(table->read_set); - return; - } - - CalpontSystemCatalog::TableName tableName; - tableName.schema = table->s->db.str; - tableName.table = table->s->table_name.str; - - try - { - CalpontSystemCatalog::ROPair roPair = csc->tableRID(tableName, lower_case_table_names); - ci->tableOid = roPair.objnum; - } - catch (IDBExcept& ie) - { - bitmap_clear_all(table->read_set); - setError(thd, ER_INTERNAL_ERROR, ie.what()); - } - catch (std::exception& ex) - { - bitmap_clear_all(table->read_set); - setError(thd, ER_INTERNAL_ERROR, - logging::IDBErrorInfo::instance()->errorMsg(ERR_SYSTEM_CATALOG) + ex.what()); - } + bitmap_clear_all(table->read_set); + setError(thd, ER_INTERNAL_ERROR, ie.what()); } + catch (std::exception& ex) + { + bitmap_clear_all(table->read_set); + setError(thd, ER_INTERNAL_ERROR, + logging::IDBErrorInfo::instance()->errorMsg(ERR_SYSTEM_CATALOG) + ex.what()); + } + } - if ( ci->rc != 0 ) - ci->rc = 0; + if (ci->rc != 0) + ci->rc = 0; } - - int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) { - // Clear read_set used for bulk insertion of Fields inheriting - //from Field_blob|Field_varstring - bitmap_clear_all(table->read_set); - THD* thd = current_thd; + // Clear read_set used for bulk insertion of Fields inheriting + // from Field_blob|Field_varstring + bitmap_clear_all(table->read_set); + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd)) + return 0; - std::string aTmpDir(startup::StartUp::tmpDir()); + std::string aTmpDir(startup::StartUp::tmpDir()); - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - int rc = 0; + int rc = 0; - if (ci->rc == 5) //read only dbrm - return rc; + if (ci->rc == 5) // read only dbrm + return rc; - // @bug 2378. do not enter for select, reset singleInsert flag after multiple insert. - // @bug 2515. Check command intead of vtable state - if ( ( ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT || ci->isCacheInsert) && !ci->singleInsert ) + // @bug 2378. do not enter for select, reset singleInsert flag after multiple insert. + // @bug 2515. Check command intead of vtable state + if ((((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || + (thd->lex)->sql_command == SQLCOM_INSERT_SELECT || ci->isCacheInsert) && + !ci->singleInsert) + { + if (((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ALWAYS) || + ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ON) && + (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))))) && + (!ci->singleInsert) && + ((ci->isLoaddataInfile) || ((thd->lex)->sql_command == SQLCOM_INSERT) || + ((thd->lex)->sql_command == SQLCOM_LOAD) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || + ci->isCacheInsert)) { - if (((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ALWAYS) || - ((ci->useCpimport == mcs_use_import_for_batchinsert_mode_t::ON) && - (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))))) && - (!ci->singleInsert) && - ((ci->isLoaddataInfile) || - ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || - ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ci->isCacheInsert) ) - { #ifdef _MSC_VER - if (thd->killed > 0) - { - errno = 0; - // GenerateConsoleCtrlEvent sends a signal to cpimport - BOOL brtn = GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, ci->cpimportProcInfo.dwProcessId); + if (thd->killed > 0) + { + errno = 0; + // GenerateConsoleCtrlEvent sends a signal to cpimport + BOOL brtn = GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, ci->cpimportProcInfo.dwProcessId); - if (!brtn) - { - int errnum = GetLastError(); - char errmsg[512]; - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, errnum, 0, errmsg, 512, NULL); - ostringstream oss; - oss << "GenerateConsoleCtrlEvent: (errno-" << errnum << "); " << errmsg; - ha_mcs_impl::log_this(thd, oss.str(), logging::LOG_TYPE_DEBUG,0); - } + if (!brtn) + { + int errnum = GetLastError(); + char errmsg[512]; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, errnum, 0, errmsg, 512, NULL); + ostringstream oss; + oss << "GenerateConsoleCtrlEvent: (errno-" << errnum << "); " << errmsg; + ha_mcs_impl::log_this(thd, oss.str(), logging::LOG_TYPE_DEBUG, 0); + } - // Close handles to the cpimport process and its primary thread. - fclose (ci->filePtr); - ci->filePtr = 0; - ci->fdt[1] = -1; - CloseHandle(ci->cpimportProcInfo.hProcess); - CloseHandle(ci->cpimportProcInfo.hThread); - WaitForSingleObject(ci->cpimportProcInfo.hProcess, INFINITE); - } + // Close handles to the cpimport process and its primary thread. + fclose(ci->filePtr); + ci->filePtr = 0; + ci->fdt[1] = -1; + CloseHandle(ci->cpimportProcInfo.hProcess); + CloseHandle(ci->cpimportProcInfo.hThread); + WaitForSingleObject(ci->cpimportProcInfo.hProcess, INFINITE); + } #else - if ( (thd->killed > 0) && (ci->cpimport_pid > 0) ) //handle CTRL-C - { - //cout << "sending ctrl-c to cpimport" << endl; - errno = 0; - kill( ci->cpimport_pid, SIGUSR1 ); - fclose (ci->filePtr); - ci->filePtr = 0; - ci->fdt[1] = -1; - int aStatus; - waitpid(ci->cpimport_pid, &aStatus, 0); // wait until cpimport finishs - } + if ((thd->killed > 0) && (ci->cpimport_pid > 0)) // handle CTRL-C + { + // cout << "sending ctrl-c to cpimport" << endl; + errno = 0; + kill(ci->cpimport_pid, SIGUSR1); + fclose(ci->filePtr); + ci->filePtr = 0; + ci->fdt[1] = -1; + int aStatus; + waitpid(ci->cpimport_pid, &aStatus, 0); // wait until cpimport finishs + } #endif + else + { + // tear down cpimport +#ifdef _MSC_VER + fclose(ci->filePtr); + ci->filePtr = 0; + ci->fdt[1] = -1; + DWORD exitCode; + WaitForSingleObject(ci->cpimportProcInfo.hProcess, INFINITE); + GetExitCodeProcess(ci->cpimportProcInfo.hProcess, &exitCode); + + if (exitCode != 0) + { + rc = 1; + setError(thd, ER_INTERNAL_ERROR, + "load failed. The detailed error information is listed in InfiniDBLog.txt."); + } + + // Close handles to the cpimport process and its primary thread. + CloseHandle(ci->cpimportProcInfo.hProcess); + CloseHandle(ci->cpimportProcInfo.hThread); +#else + fclose(ci->filePtr); + ci->filePtr = 0; + ci->fdt[1] = -1; + int aStatus; + pid_t aPid = waitpid(ci->cpimport_pid, &aStatus, 0); // wait until cpimport finishs + + if ((aPid == ci->cpimport_pid) || (aPid == -1)) + { + ci->cpimport_pid = 0; + + if ((WIFEXITED(aStatus)) && (WEXITSTATUS(aStatus) == 0)) + { + // cout << "\tCpimport exit on success" << endl; + } + else + { + if (WEXITSTATUS(aStatus) == 2) + { + rc = 1; + ifstream dmlFile; + ostringstream oss; + oss << aTmpDir << ci->tableOid << ".txt"; + dmlFile.open(oss.str().c_str()); + + if (dmlFile.is_open()) + { + string line; + getline(dmlFile, line); + setError(thd, ER_INTERNAL_ERROR, line); + dmlFile.close(); + remove(oss.str().c_str()); + } + } else { - //tear down cpimport -#ifdef _MSC_VER - fclose (ci->filePtr); - ci->filePtr = 0; - ci->fdt[1] = -1; - DWORD exitCode; - WaitForSingleObject(ci->cpimportProcInfo.hProcess, INFINITE); - GetExitCodeProcess(ci->cpimportProcInfo.hProcess, &exitCode); + rc = 1; + ifstream dmlFile; + ostringstream oss; + oss << aTmpDir << ci->tableOid << ".txt"; + dmlFile.open(oss.str().c_str()); - if (exitCode != 0) - { - rc = 1; - setError(thd, ER_INTERNAL_ERROR, "load failed. The detailed error information is listed in InfiniDBLog.txt."); - } - - // Close handles to the cpimport process and its primary thread. - CloseHandle(ci->cpimportProcInfo.hProcess); - CloseHandle(ci->cpimportProcInfo.hThread); -#else - fclose (ci->filePtr); - ci->filePtr = 0; - ci->fdt[1] = -1; - int aStatus; - pid_t aPid = waitpid(ci->cpimport_pid, &aStatus, 0); // wait until cpimport finishs - - if ((aPid == ci->cpimport_pid) || (aPid == -1)) - { - ci->cpimport_pid = 0; - - if ((WIFEXITED(aStatus)) && (WEXITSTATUS(aStatus) == 0)) - { - //cout << "\tCpimport exit on success" << endl; - } - else - { - if (WEXITSTATUS(aStatus) == 2) - { - rc = 1; - ifstream dmlFile; - ostringstream oss; - oss << aTmpDir << ci->tableOid << ".txt"; - dmlFile.open(oss.str().c_str()); - - if (dmlFile.is_open()) - { - string line; - getline(dmlFile, line); - setError(thd, ER_INTERNAL_ERROR, line); - dmlFile.close(); - remove (oss.str().c_str()); - } - } - else - { - rc = 1; - ifstream dmlFile; - ostringstream oss; - oss << aTmpDir << ci->tableOid << ".txt"; - dmlFile.open(oss.str().c_str()); - - if (dmlFile.is_open()) - { - string line; - getline(dmlFile, line); - setError(thd, ER_INTERNAL_ERROR, line); - dmlFile.close(); - remove (oss.str().c_str()); - } - else - setError(thd, ER_INTERNAL_ERROR, "load failed. The detailed error information is listed in err.log."); - } - } - } + if (dmlFile.is_open()) + { + string line; + getline(dmlFile, line); + setError(thd, ER_INTERNAL_ERROR, line); + dmlFile.close(); + remove(oss.str().c_str()); + } + else + setError(thd, ER_INTERNAL_ERROR, + "load failed. The detailed error information is listed in err.log."); + } + } + } #endif - if ( rc == 0) - { - ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - } - else - { - ha_mcs_impl::log_this(thd, "End SQL statement with error", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); - } - - ci->columnTypes.clear(); - //get extra warning count if any - ifstream dmlFile; - ostringstream oss; - oss << aTmpDir << ci->tableOid << ".txt"; - dmlFile.open(oss.str().c_str()); - int totalWarnCount = 0; - int colWarns = 0; - string line; - - if (dmlFile.is_open()) - { - while (getline(dmlFile, line)) - { - colWarns = atoi(line.c_str()); - totalWarnCount += colWarns; - } - - dmlFile.close(); - remove (oss.str().c_str()); - - for (int i = 0; i < totalWarnCount; i++) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, "Values saturated"); - } - } + if (rc == 0) + { + ha_mcs_impl::log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); } else { - if ( thd->killed > 0 ) - abort = true; - - if ( !ci->dmlProc ) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "end_bulk_insert starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } - - if (((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ((thd->lex)->sql_command == SQLCOM_LOAD)) - rc = ha_mcs_impl_write_last_batch(table, *ci, abort); + ha_mcs_impl::log_this(thd, "End SQL statement with error", logging::LOG_TYPE_DEBUG, + tid2sid(thd->thread_id)); } - } - // populate query stats for insert and load data infile. insert select has - // stats entered in sm already - if (((thd->lex)->sql_command == SQLCOM_INSERT) || - ((thd->lex)->sql_command == SQLCOM_LOAD) || - ci->isCacheInsert) - { - ci->stats.setEndTime(); - ci->stats.fErrorNo = rc; + ci->columnTypes.clear(); + // get extra warning count if any + ifstream dmlFile; + ostringstream oss; + oss << aTmpDir << ci->tableOid << ".txt"; + dmlFile.open(oss.str().c_str()); + int totalWarnCount = 0; + int colWarns = 0; + string line; - if (ci->singleInsert) - ci->stats.fRows = 1; - else - ci->stats.fRows = ci->rowsHaveInserted; - - try + if (dmlFile.is_open()) { - ci->stats.insert(); - } - catch (std::exception& e) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + while (getline(dmlFile, line)) + { + colWarns = atoi(line.c_str()); + totalWarnCount += colWarns; + } + + dmlFile.close(); + remove(oss.str().c_str()); + + for (int i = 0; i < totalWarnCount; i++) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, "Values saturated"); } + } } - - // MCOL-4002 We earlier had these re-initializations set only for - // non-transactions, i.e.: - // !(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) - // However, we should be resetting these members anyways. - ci->singleInsert = true; // reset the flag - ci->isLoaddataInfile = false; - ci->isCacheInsert = false; - ci->tableOid = 0; - ci->rowsHaveInserted = 0; - ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ON; - - return rc; -} - -int ha_mcs_impl_commit (handlerton* hton, THD* thd, bool all) -{ - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - if (ci->isAlter) - return 0; - - //@Bug 5823 check if any active transaction for this session - boost::scoped_ptr dbrmp(new DBRM()); - BRM::TxnID txnId = dbrmp->getTxnID(tid2sid(thd->thread_id)); - - if (!txnId.valid) - return 0; - - if ( !ci->dmlProc ) + else { + if (thd->killed > 0) + abort = true; + + if (!ci->dmlProc) + { ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "commit starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } + // cout << "end_bulk_insert starts a client " << ci->dmlProc << " for session " << thd->thread_id << + // endl; + } - int rc = ha_mcs_impl_commit_(hton, thd, all, *ci); - thd->server_status &= ~SERVER_STATUS_IN_TRANS; - ci->singleInsert = true; // reset the flag - ci->isLoaddataInfile = false; - ci->isCacheInsert = false; - ci->tableOid = 0; - ci->rowsHaveInserted = 0; - return rc; + if (((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ((thd->lex)->sql_command == SQLCOM_LOAD)) + rc = ha_mcs_impl_write_last_batch(table, *ci, abort); + } + } + + // populate query stats for insert and load data infile. insert select has + // stats entered in sm already + if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || + ci->isCacheInsert) + { + ci->stats.setEndTime(); + ci->stats.fErrorNo = rc; + + if (ci->singleInsert) + ci->stats.fRows = 1; + else + ci->stats.fRows = ci->rowsHaveInserted; + + try + { + ci->stats.insert(); + } + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + } + + // MCOL-4002 We earlier had these re-initializations set only for + // non-transactions, i.e.: + // !(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) + // However, we should be resetting these members anyways. + ci->singleInsert = true; // reset the flag + ci->isLoaddataInfile = false; + ci->isCacheInsert = false; + ci->tableOid = 0; + ci->rowsHaveInserted = 0; + ci->useCpimport = mcs_use_import_for_batchinsert_mode_t::ON; + + return rc; } -int ha_mcs_impl_rollback (handlerton* hton, THD* thd, bool all) +int ha_mcs_impl_commit(handlerton* hton, THD* thd, bool all) { - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if ( !ci->dmlProc ) - { + if (ci->isAlter) + return 0; - ci->dmlProc = new MessageQueueClient("DMLProc"); - } + //@Bug 5823 check if any active transaction for this session + boost::scoped_ptr dbrmp(new DBRM()); + BRM::TxnID txnId = dbrmp->getTxnID(tid2sid(thd->thread_id)); - int rc = ha_mcs_impl_rollback_(hton, thd, all, *ci); - ci->singleInsert = true; // reset the flag - ci->isLoaddataInfile = false; - ci->isCacheInsert = false; - ci->tableOid = 0; - ci->rowsHaveInserted = 0; - thd->server_status &= ~SERVER_STATUS_IN_TRANS; - return rc; + if (!txnId.valid) + return 0; + + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + // cout << "commit starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; + } + + int rc = ha_mcs_impl_commit_(hton, thd, all, *ci); + thd->server_status &= ~SERVER_STATUS_IN_TRANS; + ci->singleInsert = true; // reset the flag + ci->isLoaddataInfile = false; + ci->isCacheInsert = false; + ci->tableOid = 0; + ci->rowsHaveInserted = 0; + return rc; } -int ha_mcs_impl_close_connection (handlerton* hton, THD* thd) +int ha_mcs_impl_rollback(handlerton* hton, THD* thd, bool all) { - if (!thd) return 0; + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - if (thd->thread_id == 0) - return 0; + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + if (!ci->dmlProc) + { + ci->dmlProc = new MessageQueueClient("DMLProc"); + } - // MCOL-3247 Use THD::ha_data as a per-plugin per-session - // storage. Filled in external_lock when we remove a lock - // from vtable(lock_type = 2) - // An ugly way. I will use ha_data w/o external_lock. - // This in MCOL-2178 - cal_connection_info* ci = nullptr; - if(thd_get_ha_data(thd, hton) != (void*)0x42) // 0x42 is the magic CS sets when setup hton - { - ci = reinterpret_cast(thd_get_ha_data(thd, hton)); - } + int rc = ha_mcs_impl_rollback_(hton, thd, all, *ci); + ci->singleInsert = true; // reset the flag + ci->isLoaddataInfile = false; + ci->isCacheInsert = false; + ci->tableOid = 0; + ci->rowsHaveInserted = 0; + thd->server_status &= ~SERVER_STATUS_IN_TRANS; + return rc; +} - if (!ci) return 0; +int ha_mcs_impl_close_connection(handlerton* hton, THD* thd) +{ + if (!thd) + return 0; - int rc = 0; + if (thd->thread_id == 0) + return 0; - if ( ci->dmlProc ) - { - rc = ha_mcs_impl_close_connection_(hton, thd, *ci); - delete ci->dmlProc; - ci->dmlProc = nullptr; - } + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + // MCOL-3247 Use THD::ha_data as a per-plugin per-session + // storage. Filled in external_lock when we remove a lock + // from vtable(lock_type = 2) + // An ugly way. I will use ha_data w/o external_lock. + // This in MCOL-2178 + cal_connection_info* ci = nullptr; + if (thd_get_ha_data(thd, hton) != (void*)0x42) // 0x42 is the magic CS sets when setup hton + { + ci = reinterpret_cast(thd_get_ha_data(thd, hton)); + } - return rc; + if (!ci) + return 0; + + int rc = 0; + + if (ci->dmlProc) + { + rc = ha_mcs_impl_close_connection_(hton, thd, *ci); + delete ci->dmlProc; + ci->dmlProc = nullptr; + } + + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return rc; } int ha_mcs_impl_rename_table(const char* from, const char* to) { - IDEBUG( cout << "ha_mcs_impl_rename_table: " << from << " => " << to << endl ); + IDEBUG(cout << "ha_mcs_impl_rename_table: " << from << " => " << to << endl); - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - //@Bug 1948. Alter table call rename table twice - if ( ci->alterTableState == cal_connection_info::ALTER_FIRST_RENAME ) - { - ci->alterTableState = cal_connection_info::ALTER_SECOND_RENAME; - IDEBUG( cout << "ha_mcs_impl_rename_table: was in state ALTER_FIRST_RENAME, now in ALTER_SECOND_RENAME" << endl ); - return 0; - } - else if (ci->alterTableState == cal_connection_info::ALTER_SECOND_RENAME) - { - ci->alterTableState = cal_connection_info::NOT_ALTER; - IDEBUG( cout << "ha_mcs_impl_rename_table: was in state ALTER_SECOND_RENAME, now in NOT_ALTER" << endl ); - return 0; - } + //@Bug 1948. Alter table call rename table twice + if (ci->alterTableState == cal_connection_info::ALTER_FIRST_RENAME) + { + ci->alterTableState = cal_connection_info::ALTER_SECOND_RENAME; + IDEBUG(cout << "ha_mcs_impl_rename_table: was in state ALTER_FIRST_RENAME, now in ALTER_SECOND_RENAME" + << endl); + return 0; + } + else if (ci->alterTableState == cal_connection_info::ALTER_SECOND_RENAME) + { + ci->alterTableState = cal_connection_info::NOT_ALTER; + IDEBUG(cout << "ha_mcs_impl_rename_table: was in state ALTER_SECOND_RENAME, now in NOT_ALTER" << endl); + return 0; + } - int rc = ha_mcs_impl_rename_table_(from, to, *ci); - return rc; + int rc = ha_mcs_impl_rename_table_(from, to, *ci); + return rc; } int ha_mcs_impl_delete_row(const uchar* buf) { - IDEBUG( cout << "ha_mcs_impl_delete_row" << endl ); - return 0; + IDEBUG(cout << "ha_mcs_impl_delete_row" << endl); + return 0; } COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condStack) { - THD* thd = current_thd; + THD* thd = current_thd; - if (isUpdateOrDeleteStatement(thd->lex->sql_command)) - { - condStack.push_back(cond); - return nullptr; - } + if (isUpdateOrDeleteStatement(thd->lex->sql_command)) + { + condStack.push_back(cond); + return nullptr; + } - string alias; - alias.assign(table->alias.ptr(), table->alias.length()); - IDEBUG( cout << "ha_mcs_impl_cond_push: " << alias << endl ); + string alias; + alias.assign(table->alias.ptr(), table->alias.length()); + IDEBUG(cout << "ha_mcs_impl_cond_push: " << alias << endl); - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - cal_table_info ti = ci->tableMap[table]; + cal_table_info ti = ci->tableMap[table]; #ifdef DEBUG_WALK_COND - { - gp_walk_info gwi; - gwi.condPush = true; - gwi.sessionid = tid2sid(thd->thread_id); - cout << "------------------ cond push -----------------------" << endl; - cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; - } + { + gp_walk_info gwi; + gwi.condPush = true; + gwi.sessionid = tid2sid(thd->thread_id); + cout << "------------------ cond push -----------------------" << endl; + cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cout << "------------------------------------------------\n" << endl; + } #endif - if (!ti.csep) + if (!ti.csep) + { + if (!ti.condInfo) + ti.condInfo = new gp_walk_info(); + + gp_walk_info* gwi = ti.condInfo; + gwi->dropCond = false; + gwi->fatalParseError = false; + gwi->condPush = true; + gwi->thd = thd; + gwi->sessionid = tid2sid(thd->thread_id); + cond->traverse_cond(gp_walk, gwi, Item::POSTFIX); + ci->tableMap[table] = ti; + + if (gwi->fatalParseError) { - if (!ti.condInfo) - ti.condInfo = new gp_walk_info(); + IDEBUG(cout << gwi->parseErrorText << endl); - gp_walk_info* gwi = ti.condInfo; - gwi->dropCond = false; - gwi->fatalParseError = false; - gwi->condPush = true; - gwi->thd = thd; - gwi->sessionid = tid2sid(thd->thread_id); - cond->traverse_cond(gp_walk, gwi, Item::POSTFIX); + if (ti.condInfo) + { + delete ti.condInfo; + ti.condInfo = nullptr; ci->tableMap[table] = ti; + } - if (gwi->fatalParseError) - { - IDEBUG( cout << gwi->parseErrorText << endl ); - - if (ti.condInfo) - { - delete ti.condInfo; - ti.condInfo = nullptr; - ci->tableMap[table] = ti; - } - - return cond; - } - - if (gwi->dropCond) - { - return cond; - } - else - { - return nullptr; - } + return cond; } - return cond; + if (gwi->dropCond) + { + return cond; + } + else + { + return nullptr; + } + } + + return cond; } inline void disableBinlogForDML(THD* thd) { - if (isDMLStatement(thd->lex->sql_command) && - (thd->variables.option_bits & OPTION_BIN_LOG)) - { - set_original_option_bits(thd->variables.option_bits, thd); - thd->variables.option_bits &= ~OPTION_BIN_LOG; - thd->variables.option_bits |= OPTION_BIN_TMP_LOG_OFF; - } + if (isDMLStatement(thd->lex->sql_command) && (thd->variables.option_bits & OPTION_BIN_LOG)) + { + set_original_option_bits(thd->variables.option_bits, thd); + thd->variables.option_bits &= ~OPTION_BIN_LOG; + thd->variables.option_bits |= OPTION_BIN_TMP_LOG_OFF; + } } inline void restoreBinlogForDML(THD* thd) { - if (isDMLStatement(thd->lex->sql_command)) - { - ulonglong orig_option_bits = get_original_option_bits(thd); + if (isDMLStatement(thd->lex->sql_command)) + { + ulonglong orig_option_bits = get_original_option_bits(thd); - if (orig_option_bits) - { - thd->variables.option_bits = orig_option_bits; - set_original_option_bits(0, thd); - } + if (orig_option_bits) + { + thd->variables.option_bits = orig_option_bits; + set_original_option_bits(0, thd); } + } } int ha_mcs::impl_external_lock(THD* thd, TABLE* table, int lock_type) { - // @bug 3014. Error out locking table command. IDB does not support it now. - if (thd->lex->sql_command == SQLCOM_LOCK_TABLES) + // @bug 3014. Error out locking table command. IDB does not support it now. + if (thd->lex->sql_command == SQLCOM_LOCK_TABLES) + { + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, + logging::IDBErrorInfo::instance()->errorMsg(ERR_LOCK_TABLE)); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // @info called for every table at the beginning and at the end of a query. + // used for cleaning up the tableinfo. + string alias; + alias.assign(table->alias.ptr(), table->alias.length()); + IDEBUG(cout << "external_lock for " << alias << endl); + + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + ci->physTablesList.clear(); + ci->tableMap.clear(); + force_close_fep_conn(thd, ci); + return 0; + } + + m_lock_type = lock_type; + + CalTableMap::iterator mapiter = ci->tableMap.find(table); + // make sure this is a release lock (2nd) call called in + // the table mode. + if (mapiter != ci->tableMap.end() && (mapiter->second.condInfo || mapiter->second.csep) && lock_type == 2) + { + // CS ends up processing query with handlers + // table mode + if (mapiter->second.conn_hndl) { - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, - logging::IDBErrorInfo::instance()->errorMsg(ERR_LOCK_TABLE)); - return ER_CHECK_NOT_IMPLEMENTED; + if (ci->traceFlags & 1) + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, + mapiter->second.conn_hndl->queryStats.c_str()); + + ci->queryStats = mapiter->second.conn_hndl->queryStats; + ci->extendedStats = mapiter->second.conn_hndl->extendedStats; + ci->miniStats = mapiter->second.conn_hndl->miniStats; + sm::sm_cleanup(mapiter->second.conn_hndl); + mapiter->second.conn_hndl = nullptr; } - // @info called for every table at the beginning and at the end of a query. - // used for cleaning up the tableinfo. - string alias; - alias.assign(table->alias.ptr(), table->alias.length()); - IDEBUG( cout << "external_lock for " << alias << endl ); - - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + if (mapiter->second.condInfo) { - ci->physTablesList.clear(); - ci->tableMap.clear(); - force_close_fep_conn(thd, ci); - return 0; + delete mapiter->second.condInfo; + mapiter->second.condInfo = nullptr; } - m_lock_type= lock_type; - - CalTableMap::iterator mapiter = ci->tableMap.find(table); - // make sure this is a release lock (2nd) call called in - // the table mode. - if (mapiter != ci->tableMap.end() - && (mapiter->second.condInfo || mapiter->second.csep) - && lock_type == 2) + // MCOL-2178 Check for tableMap size to set this only once. + ci->queryState = 0; + // Clean up the tableMap and physTablesList + ci->tableMap.erase(table); + ci->physTablesList.erase(table); + thd->variables.in_subquery_conversion_threshold = IN_SUBQUERY_CONVERSION_THRESHOLD; + restore_optimizer_flags(thd); + restoreBinlogForDML(thd); + } + else + { + if ((lock_type == 0) || (lock_type == 1)) { - // CS ends up processing query with handlers - // table mode - if (mapiter->second.conn_hndl) - { - if (ci->traceFlags & 1) - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); + ci->physTablesList.insert(table); + // MCOL-2178 Disable Conversion of Big IN Predicates Into Subqueries + thd->variables.in_subquery_conversion_threshold = ~0; + // Early optimizer_switch changes to avoid unsupported opt-s. + mutate_optimizer_flags(thd); - ci->queryStats = mapiter->second.conn_hndl->queryStats; - ci->extendedStats = mapiter->second.conn_hndl->extendedStats; - ci->miniStats = mapiter->second.conn_hndl->miniStats; - sm::sm_cleanup(mapiter->second.conn_hndl); - mapiter->second.conn_hndl = nullptr; - } - - if (mapiter->second.condInfo) - { - delete mapiter->second.condInfo; - mapiter->second.condInfo = nullptr; - } - - // MCOL-2178 Check for tableMap size to set this only once. - ci->queryState = 0; - // Clean up the tableMap and physTablesList - ci->tableMap.erase(table); + // MCOL-4936 Disable binlog for DMLs + if (lock_type == 1) + { + disableBinlogForDML(thd); + } + } + else if (lock_type == 2) + { + restoreBinlogForDML(thd); + std::set::iterator iter = ci->physTablesList.find(table); + if (iter != ci->physTablesList.end()) + { ci->physTablesList.erase(table); + } + + // CS ends up processing query with handlers + if (iter != ci->physTablesList.end() && ci->physTablesList.empty()) + { + if (!ci->cal_conn_hndl) + return 0; + + if (ci->traceFlags & 1) + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, ci->cal_conn_hndl->queryStats.c_str()); + + ci->queryStats = ci->cal_conn_hndl->queryStats; + ci->extendedStats = ci->cal_conn_hndl->extendedStats; + ci->miniStats = ci->cal_conn_hndl->miniStats; + ci->queryState = 0; + // MCOL-3247 Use THD::ha_data as a per-plugin per-session + // storage for cal_conn_hndl to use it later in close_connection + thd_set_ha_data(thd, mcs_hton, get_fe_conn_info_ptr()); + // Clean up all tableMap entries made by cond_push + for (auto& tme : ci->tableMap) + { + if (tme.second.condInfo) + { + delete tme.second.condInfo; + tme.second.condInfo = nullptr; + } + } + ci->tableMap.clear(); + // MCOL-2178 Enable Conversion of Big IN Predicates Into Subqueries thd->variables.in_subquery_conversion_threshold = IN_SUBQUERY_CONVERSION_THRESHOLD; restore_optimizer_flags(thd); - restoreBinlogForDML(thd); + } } - else - { - if ((lock_type == 0) || (lock_type == 1)) - { - ci->physTablesList.insert(table); - // MCOL-2178 Disable Conversion of Big IN Predicates Into Subqueries - thd->variables.in_subquery_conversion_threshold=~ 0; - // Early optimizer_switch changes to avoid unsupported opt-s. - mutate_optimizer_flags(thd); + } - // MCOL-4936 Disable binlog for DMLs - if (lock_type == 1) - { - disableBinlogForDML(thd); - } - } - else if (lock_type == 2) - { - restoreBinlogForDML(thd); - std::set::iterator iter = ci->physTablesList.find(table); - if (iter != ci->physTablesList.end()) - { - ci->physTablesList.erase(table); - } - - // CS ends up processing query with handlers - if (iter != ci->physTablesList.end() && ci->physTablesList.empty()) - { - if (!ci->cal_conn_hndl) - return 0; - - if (ci->traceFlags & 1) - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, ci->cal_conn_hndl->queryStats.c_str()); - - ci->queryStats = ci->cal_conn_hndl->queryStats; - ci->extendedStats = ci->cal_conn_hndl->extendedStats; - ci->miniStats = ci->cal_conn_hndl->miniStats; - ci->queryState = 0; - // MCOL-3247 Use THD::ha_data as a per-plugin per-session - // storage for cal_conn_hndl to use it later in close_connection - thd_set_ha_data(thd, mcs_hton, get_fe_conn_info_ptr()); - // Clean up all tableMap entries made by cond_push - for (auto &tme: ci->tableMap) - { - if (tme.second.condInfo) - { - delete tme.second.condInfo; - tme.second.condInfo= nullptr; - } - } - ci->tableMap.clear(); - // MCOL-2178 Enable Conversion of Big IN Predicates Into Subqueries - thd->variables.in_subquery_conversion_threshold = IN_SUBQUERY_CONVERSION_THRESHOLD; - restore_optimizer_flags(thd); - } - - } - } - - return 0; + return 0; } // for sorting length exceeds blob limit. Just error out for now. int ha_mcs_impl_rnd_pos(uchar* buf, uchar* pos) { - IDEBUG( cout << "ha_mcs_impl_rnd_pos" << endl); - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_TOO_BIG); - setError(current_thd, ER_INTERNAL_ERROR, emsg); - return ER_INTERNAL_ERROR; + IDEBUG(cout << "ha_mcs_impl_rnd_pos" << endl); + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_TOO_BIG); + setError(current_thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; } /*@brief ha_mcs_impl_group_by_init - Get data for MariaDB group_by @@ -4089,429 +4116,428 @@ int ha_mcs_impl_rnd_pos(uchar* buf, uchar* pos) * 0 if success * others if something went wrong whilst getting the result set ***********************************************************/ -int ha_mcs_impl_group_by_init(mcs_handler_info *handler_info, TABLE* table) +int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table) { - ha_mcs_group_by_handler *group_hand= - reinterpret_cast(handler_info->hndl_ptr); - string tableName = group_hand->table_list->table->s->table_name.str; - IDEBUG( cout << "group_by_init for table " << tableName << endl ); - THD* thd = current_thd; + ha_mcs_group_by_handler* group_hand = reinterpret_cast(handler_info->hndl_ptr); + string tableName = group_hand->table_list->table->s->table_name.str; + IDEBUG(cout << "group_by_init for table " << tableName << endl); + THD* thd = current_thd; - //check whether the system is ready to process statement. + // check whether the system is ready to process statement. #ifndef _MSC_VER - static DBRM dbrm(true); - int bSystemQueryReady = dbrm.getSystemQueryReady(); + static DBRM dbrm(true); + int bSystemQueryReady = dbrm.getSystemQueryReady(); - if (bSystemQueryReady == 0) - { - // Still not ready - setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); - return ER_INTERNAL_ERROR; - } - else if (bSystemQueryReady < 0) - { - // Still not ready - setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); - return ER_INTERNAL_ERROR; - } + if (bSystemQueryReady == 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); + return ER_INTERNAL_ERROR; + } + else if (bSystemQueryReady < 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); + return ER_INTERNAL_ERROR; + } #endif - uint32_t sessionID = tid2sid(thd->thread_id); - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); + uint32_t sessionID = tid2sid(thd->thread_id); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - idbassert(ci != 0); + idbassert(ci != 0); - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + return 0; + } + + sm::tableid_t tableid = 0; + cal_table_info ti; + cal_group_info gi; + sm::cpsm_conhdl_t* hndl; + SCSEP csep; + + bool localQuery = get_local_query(thd); + + { + ci->stats.reset(); // reset query stats + ci->stats.setStartTime(); + if (thd->main_security_ctx.user) { - force_close_fep_conn(thd, ci); - return 0; + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; } - sm::tableid_t tableid = 0; - cal_table_info ti; - cal_group_info gi; - sm::cpsm_conhdl_t* hndl; - SCSEP csep; - - bool localQuery = get_local_query(thd); + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + try { - ci->stats.reset(); // reset query stats - ci->stats.setStartTime(); - if (thd->main_security_ctx.user) + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + ci->warningMsg = msg; + } + + // If the previous query has error and + // this is not a subquery run by the server(MCOL-1601) + // re-establish the connection + if (ci->queryState != 0) + { + if (ci->cal_conn_hndl_st.size() == 0) + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); + idbassert(ci->cal_conn_hndl != 0); + ci->cal_conn_hndl->csc = csc; + idbassert(ci->cal_conn_hndl->exeMgr != 0); + + try + { + ci->cal_conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + hndl = ci->cal_conn_hndl; + + ci->cal_conn_hndl_st.push(ci->cal_conn_hndl); + if (!csep) + csep.reset(new CalpontSelectExecutionPlan()); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + csep->txnID(txnID.id); + csep->verID(verID); + csep->sessionID(sessionID); + + if (group_hand->table_list->db.length) + csep->schemaName(group_hand->table_list->db.str, lower_case_table_names); + + csep->traceFlags(ci->traceFlags); + + // MCOL-1052 Send Items lists down to the optimizer. + gi.groupByTables = group_hand->table_list; + gi.groupByFields = group_hand->select; + gi.groupByWhere = group_hand->where; + gi.groupByGroup = group_hand->group_by; + gi.groupByOrder = group_hand->order_by; + gi.groupByHaving = group_hand->having; + gi.groupByDistinct = group_hand->distinct; + + // MCOL-1052 Send pushed conditions here, since server could omit GROUP BY + // items in case of = or IN functions used on GROUP BY columns. + { + CalTableMap::iterator mapiter; + execplan::CalpontSelectExecutionPlan::ColumnMap::iterator colMapIter; + execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter; + execplan::ParseTree* ptIt; + + for (TABLE_LIST* tl = gi.groupByTables; tl; tl = tl->next_local) + { + mapiter = ci->tableMap.find(tl->table); + + if (mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL && + mapiter->second.condInfo->condPush) { - ci->stats.fUser = thd->main_security_ctx.user; - } - else - { - ci->stats.fUser = ""; + while (!mapiter->second.condInfo->ptWorkStack.empty()) + { + ptIt = mapiter->second.condInfo->ptWorkStack.top(); + mapiter->second.condInfo->ptWorkStack.pop(); + gi.pushedPts.push_back(ptIt); + } } + } + } + // send plan whenever group_init is called + int status = cp_get_group_plan(thd, csep, gi); - if (thd->main_security_ctx.host) - ci->stats.fHost = thd->main_security_ctx.host; - else if (thd->main_security_ctx.host_or_ip) - ci->stats.fHost = thd->main_security_ctx.host_or_ip; - else - ci->stats.fHost = "unknown"; + // Never proceed if status != 0 to avoid empty DA + // crashes on later stages + if (status != 0) + goto internal_error; - try - { - ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - ci->warningMsg = msg; - } + // @bug 2547. don't need to send the plan if it's impossible where for all unions. + // MCOL-2178 commenting the below out since cp_get_group_plan does not modify this variable + // which has a default value of false + // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) + // return 0; - // If the previous query has error and - // this is not a subquery run by the server(MCOL-1601) - // re-establish the connection - if (ci->queryState != 0) - { - if( ci->cal_conn_hndl_st.size() == 0 ) - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + string query; + // Set the query text only once if the server executes + // subqueries separately. + if (ci->queryState) + query.assign(""); + else + query.assign(thd->query_string.str(), thd->query_string.length()); + csep->data(query); - sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); - idbassert(ci->cal_conn_hndl != 0); - ci->cal_conn_hndl->csc = csc; - idbassert(ci->cal_conn_hndl->exeMgr != 0); - - try - { - ci->cal_conn_hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } - - hndl = ci->cal_conn_hndl; - - ci->cal_conn_hndl_st.push(ci->cal_conn_hndl); - if (!csep) - csep.reset(new CalpontSelectExecutionPlan()); - - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - - csep->txnID(txnID.id); - csep->verID(verID); - csep->sessionID(sessionID); - - if (group_hand->table_list->db.length) - csep->schemaName(group_hand->table_list->db.str, lower_case_table_names); - - csep->traceFlags(ci->traceFlags); - - // MCOL-1052 Send Items lists down to the optimizer. - gi.groupByTables = group_hand->table_list; - gi.groupByFields = group_hand->select; - gi.groupByWhere = group_hand->where; - gi.groupByGroup = group_hand->group_by; - gi.groupByOrder = group_hand->order_by; - gi.groupByHaving = group_hand->having; - gi.groupByDistinct = group_hand->distinct; - - // MCOL-1052 Send pushed conditions here, since server could omit GROUP BY - // items in case of = or IN functions used on GROUP BY columns. - { - CalTableMap::iterator mapiter; - execplan::CalpontSelectExecutionPlan::ColumnMap::iterator colMapIter; - execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter; - execplan::ParseTree* ptIt; - - for (TABLE_LIST* tl = gi.groupByTables; tl; tl = tl->next_local) - { - mapiter = ci->tableMap.find(tl->table); - - if (mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL - && mapiter->second.condInfo->condPush) - { - while (!mapiter->second.condInfo->ptWorkStack.empty()) - { - ptIt = mapiter->second.condInfo->ptWorkStack.top(); - mapiter->second.condInfo->ptWorkStack.pop(); - gi.pushedPts.push_back(ptIt); - } - } - } - } - // send plan whenever group_init is called - int status = cp_get_group_plan(thd, csep, gi); - - // Never proceed if status != 0 to avoid empty DA - // crashes on later stages - if (status != 0) - goto internal_error; - - // @bug 2547. don't need to send the plan if it's impossible where for all unions. - // MCOL-2178 commenting the below out since cp_get_group_plan does not modify this variable - // which has a default value of false - //if (MIGR::infinidb_vtable.impossibleWhereOnUnion) - // return 0; - - string query; - // Set the query text only once if the server executes - // subqueries separately. - if(ci->queryState) - query.assign(""); - else - query.assign(thd->query_string.str(), thd->query_string.length()); - csep->data(query); - - try - { - csep->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } + try + { + csep->priority(ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } #ifdef PLAN_HEX_FILE - // plan serialization - string tmpDir = aTmpDir + "/li1-plan.hex"; + // plan serialization + string tmpDir = aTmpDir + "/li1-plan.hex"; - ifstream ifs(tmpDir); - ByteStream bs1; - ifs >> bs1; - ifs.close(); - csep->unserialize(bs1); + ifstream ifs(tmpDir); + ByteStream bs1; + ifs >> bs1; + ifs.close(); + csep->unserialize(bs1); #endif - if (ci->traceFlags & 1) + if (ci->traceFlags & 1) + { + cerr << "---------------- EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + } + else + { + IDEBUG(cout << "---------------- EXECUTION PLAN ----------------" << endl); + IDEBUG(cerr << *csep << endl); + IDEBUG(cout << "-------------- EXECUTION PLAN END --------------\n" << endl); + } + } // end of execution plan generation + + { + ByteStream msg; + ByteStream emsgBs; + + while (true) + { + try + { + ByteStream::quadbyte qb = 4; + msg << qb; + hndl->exeMgr->write(msg); + msg.restart(); + csep->rmParms(ci->rmParms); + + // send plan + csep->serialize(msg); + hndl->exeMgr->write(msg); + + // get ExeMgr status back to indicate a vtable joblist success or not + msg.restart(); + emsgBs.restart(); + msg = hndl->exeMgr->read(); + emsgBs = hndl->exeMgr->read(); + string emsg; + + if (msg.length() == 0 || emsgBs.length() == 0) { - cerr << "---------------- EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } + + string emsgStr; + emsgBs >> emsgStr; + bool err = false; + + if (msg.length() == 4) + { + msg >> qb; + + if (qb != 0) + { + err = true; + // for makejoblist error, stats contains only error code and insert from here + // because table fetch is not started + ci->stats.setEndTime(); + ci->stats.fQuery = csep->data(); + ci->stats.fQueryType = csep->queryType(); + ci->stats.fErrorNo = qb; + + try + { + ci->stats.insert(); + } + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + } } else { - IDEBUG( cout << "---------------- EXECUTION PLAN ----------------" << endl ); - IDEBUG( cerr << *csep << endl ); - IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); + err = true; } - }// end of execution plan generation - { - ByteStream msg; - ByteStream emsgBs; - - while (true) + if (err) { - try - { - ByteStream::quadbyte qb = 4; - msg << qb; - hndl->exeMgr->write(msg); - msg.restart(); - csep->rmParms(ci->rmParms); - - //send plan - csep->serialize(msg); - hndl->exeMgr->write(msg); - - //get ExeMgr status back to indicate a vtable joblist success or not - msg.restart(); - emsgBs.restart(); - msg = hndl->exeMgr->read(); - emsgBs = hndl->exeMgr->read(); - string emsg; - - if (msg.length() == 0 || emsgBs.length() == 0) - { - emsg = "Lost connection to ExeMgr. Please contact your administrator"; - setError(thd, ER_INTERNAL_ERROR, emsg); - return ER_INTERNAL_ERROR; - } - - string emsgStr; - emsgBs >> emsgStr; - bool err = false; - - if (msg.length() == 4) - { - msg >> qb; - - if (qb != 0) - { - err = true; - // for makejoblist error, stats contains only error code and insert from here - // because table fetch is not started - ci->stats.setEndTime(); - ci->stats.fQuery = csep->data(); - ci->stats.fQueryType = csep->queryType(); - ci->stats.fErrorNo = qb; - - try - { - ci->stats.insert(); - } - catch (std::exception& e) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - } - } - else - { - err = true; - } - - if (err) - { - setError(thd, ER_INTERNAL_ERROR, emsgStr); - return ER_INTERNAL_ERROR; - } - - ci->rmParms.clear(); - - ci->queryState = 1; - - break; - } - catch (...) - { - sm::sm_cleanup(hndl); - hndl = 0; - - sm::sm_init(sessionID, &hndl, localQuery); - idbassert(hndl != 0); - hndl->csc = csc; - - ci->cal_conn_hndl = hndl; - ci->cal_conn_hndl_st.pop(); - ci->cal_conn_hndl_st.push(ci->cal_conn_hndl); - try - { - hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } - - msg.restart(); - } + setError(thd, ER_INTERNAL_ERROR, emsgStr); + return ER_INTERNAL_ERROR; } - } - // set query state to be in_process. Sometimes mysql calls rnd_init multiple - // times, this makes sure plan only being generated and sent once. It will be - // reset when query finishes in sm::end_query + ci->rmParms.clear(); - // common path for both vtable select phase and table mode -- open scan handle - ti = ci->tableMap[table]; - ti.msTablePtr = table; + ci->queryState = 1; - { - // MCOL-1601 Using stacks of ExeMgr conn hndls, table and scan contexts. - ti.tpl_ctx = new sm::cpsm_tplh_t(); - ti.tpl_ctx_st.push(ti.tpl_ctx); - ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); - ti.tpl_scan_ctx_st.push(ti.tpl_scan_ctx); + break; + } + catch (...) + { + sm::sm_cleanup(hndl); + hndl = 0; - // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql - // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = nullptr; + sm::sm_init(sessionID, &hndl, localQuery); + idbassert(hndl != 0); + hndl->csc = csc; + ci->cal_conn_hndl = hndl; + ci->cal_conn_hndl_st.pop(); + ci->cal_conn_hndl_st.push(ci->cal_conn_hndl); try { - tableid = execplan::IDB_VTABLE_ID; + hndl->connect(); } catch (...) { - string emsg = "No table ID found for table " + string(table->s->table_name.str); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; } - try - { - sm::tpl_open(tableid, ti.tpl_ctx, hndl); - sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); - } - catch (std::exception& e) - { - string emsg = "table can not be opened: " + string(e.what()); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - catch (...) - { - string emsg = "table can not be opened"; - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } + msg.restart(); + } + } + } - ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + // set query state to be in_process. Sometimes mysql calls rnd_init multiple + // times, this makes sure plan only being generated and sent once. It will be + // reset when query finishes in sm::end_query - if ((ti.tpl_scan_ctx->ctp).size() == 0) - { - uint32_t num_attr = table->s->fields; + // common path for both vtable select phase and table mode -- open scan handle + ti = ci->tableMap[table]; + ti.msTablePtr = table; - for (uint32_t i = 0; i < num_attr; i++) - { - CalpontSystemCatalog::ColType ctype; - ti.tpl_scan_ctx->ctp.push_back(ctype); - } - } + { + // MCOL-1601 Using stacks of ExeMgr conn hndls, table and scan contexts. + ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_ctx_st.push(ti.tpl_ctx); + ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); + ti.tpl_scan_ctx_st.push(ti.tpl_scan_ctx); + + // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql + // call rnd_init for a table more than once. + ti.tpl_scan_ctx->rowGroup = nullptr; + + try + { + tableid = execplan::IDB_VTABLE_ID; + } + catch (...) + { + string emsg = "No table ID found for table " + string(table->s->table_name.str); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; } - ci->tableMap[table] = ti; - return 0; + try + { + sm::tpl_open(tableid, ti.tpl_ctx, hndl); + sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); + } + catch (std::exception& e) + { + string emsg = "table can not be opened: " + string(e.what()); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + catch (...) + { + string emsg = "table can not be opened"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + + if ((ti.tpl_scan_ctx->ctp).size() == 0) + { + uint32_t num_attr = table->s->fields; + + for (uint32_t i = 0; i < num_attr; i++) + { + CalpontSystemCatalog::ColType ctype; + ti.tpl_scan_ctx->ctp.push_back(ctype); + } + } + } + + ci->tableMap[table] = ti; + return 0; error: - if (ci->cal_conn_hndl) - { - // end_query() should be called here. - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + if (ci->cal_conn_hndl) + { + // end_query() should be called here. + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - // do we need to close all connection handle of the table map? - return ER_INTERNAL_ERROR; + // do we need to close all connection handle of the table map? + return ER_INTERNAL_ERROR; internal_error: - if (ci->cal_conn_hndl) - { - // end_query() should be called here. - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + if (ci->cal_conn_hndl) + { + // end_query() should be called here. + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - return ER_INTERNAL_ERROR; + return ER_INTERNAL_ERROR; } /*@brief ha_mcs_impl_group_by_next - Return result set for MariaDB group_by @@ -4530,232 +4556,229 @@ internal_error: ***********************************************************/ int ha_mcs_impl_group_by_next(TABLE* table) { - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return HA_ERR_END_OF_FILE; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return HA_ERR_END_OF_FILE; - if (isMCSTableUpdate(thd) || isMCSTableDelete(thd)) - return HA_ERR_END_OF_FILE; + if (isMCSTableUpdate(thd) || isMCSTableDelete(thd)) + return HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + return 0; + } + + if (ci->alterTableState > 0) + return HA_ERR_END_OF_FILE; + + cal_table_info ti; + ti = ci->tableMap[table]; + int rc = HA_ERR_END_OF_FILE; + + if (!ti.tpl_ctx || !ti.tpl_scan_ctx) + { + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + idbassert(ti.msTablePtr == table); + + try + { + // fetchNextRow interface forces to use buf. + unsigned char buf; + rc = fetchNextRow(&buf, ti, ci, true); + } + catch (std::exception& e) + { + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + ci->tableMap[table] = ti; + + if (rc != 0 && rc != HA_ERR_END_OF_FILE) + { + string emsg; + + // remove this check when all error handling migrated to the new framework. + if (rc >= 1000) + emsg = ti.tpl_scan_ctx->errMsg; + else { - force_close_fep_conn(thd, ci); - return 0; + logging::ErrorCodes errorcodes; + emsg = errorcodes.errorString(rc); } - if (ci->alterTableState > 0) return HA_ERR_END_OF_FILE; + setError(thd, ER_INTERNAL_ERROR, emsg); + ci->stats.fErrorNo = rc; + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + rc = ER_INTERNAL_ERROR; + } - cal_table_info ti; - ti = ci->tableMap[table]; - int rc = HA_ERR_END_OF_FILE; - - if (!ti.tpl_ctx || !ti.tpl_scan_ctx) - { - CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - return ER_INTERNAL_ERROR; - } - - idbassert(ti.msTablePtr == table); - - try - { - // fetchNextRow interface forces to use buf. - unsigned char buf; - rc = fetchNextRow(&buf, ti, ci, true); - } - catch (std::exception& e) - { - string emsg = string("Error while fetching from ExeMgr: ") + e.what(); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - return ER_INTERNAL_ERROR; - } - - ci->tableMap[table] = ti; - - if (rc != 0 && rc != HA_ERR_END_OF_FILE) - { - string emsg; - - // remove this check when all error handling migrated to the new framework. - if (rc >= 1000) - emsg = ti.tpl_scan_ctx->errMsg; - else - { - logging::ErrorCodes errorcodes; - emsg = errorcodes.errorString(rc); - } - - setError(thd, ER_INTERNAL_ERROR, emsg); - ci->stats.fErrorNo = rc; - CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - rc = ER_INTERNAL_ERROR; - } - - return rc; + return rc; } int ha_mcs_impl_group_by_end(TABLE* table) { - int rc = 0; - THD* thd = current_thd; + int rc = 0; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return 0; - cal_connection_info* ci = nullptr; + cal_connection_info* ci = nullptr; - if (get_fe_conn_info_ptr() != NULL) - ci = reinterpret_cast(get_fe_conn_info_ptr()); + if (get_fe_conn_info_ptr() != NULL) + ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (!ci) - { - set_fe_conn_info_ptr((void*)new cal_connection_info()); - ci = reinterpret_cast(get_fe_conn_info_ptr()); - } + if (!ci) + { + set_fe_conn_info_ptr((void*)new cal_connection_info()); + ci = reinterpret_cast(get_fe_conn_info_ptr()); + } - if (((thd->lex)->sql_command == SQLCOM_INSERT) || - ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) - { - force_close_fep_conn(thd, ci, true); // with checking prev command rc - return rc; - } - - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) - { - force_close_fep_conn(thd, ci); - // clear querystats because no query stats available for cancelled query - ci->queryStats = ""; - // Poping next ExeMgr connection out of the stack - if ( ci->cal_conn_hndl_st.size() ) - { - ci->cal_conn_hndl_st.pop(); - if ( ci->cal_conn_hndl_st.size() ) - ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); - } - - return 0; - } - - IDEBUG( cerr << "group_by_end for table " << table->s->table_name.str << endl ); - - cal_table_info ti = ci->tableMap[table]; - sm::cpsm_conhdl_t* hndl; - bool clearScanCtx = false; - - hndl = ci->cal_conn_hndl; - - if (ti.tpl_ctx) - { - if (ti.tpl_scan_ctx.get()) - { - clearScanCtx = ( (ti.tpl_scan_ctx.get()->rowsreturned) && - ti.tpl_scan_ctx.get()->rowsreturned == ti.tpl_scan_ctx.get()->getRowCount() ); - try - { - sm::tpl_scan_close(ti.tpl_scan_ctx); - } - catch (...) - { - rc = ER_INTERNAL_ERROR; - } - } - - ti.tpl_scan_ctx.reset(); - if ( ti.tpl_scan_ctx_st.size() ) - { - ti.tpl_scan_ctx_st.pop(); - if ( ti.tpl_scan_ctx_st.size() ) - ti.tpl_scan_ctx = ti.tpl_scan_ctx_st.top(); - } - try - { - if(hndl) - { - { - bool ask_4_stats = (ci->traceFlags) ? true : false; - sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats, ask_4_stats, clearScanCtx); - } -// Normaly stats variables are set in external_lock method but we set it here -// since they we pretend we are in vtable_disabled mode and the stats vars won't be set. -// We sum the stats up here since server could run a number of -// queries e.g. each for a subquery in a filter. - if(hndl) - { - if (hndl->queryStats.length()) - ci->queryStats += hndl->queryStats; - if (hndl->extendedStats.length()) - ci->extendedStats += hndl->extendedStats; - if (hndl->miniStats.length()) - ci->miniStats += hndl->miniStats; - } - } - - ci->cal_conn_hndl = hndl; - - ti.tpl_ctx = 0; - } - catch (IDBExcept& e) - { - if (e.errorCode() == ERR_CROSS_ENGINE_CONNECT || e.errorCode() == ERR_CROSS_ENGINE_CONFIG) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - else - { - setError(thd, ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - } - catch (std::exception& e) - { - setError(thd, ER_INTERNAL_ERROR, e.what()); - rc = ER_INTERNAL_ERROR; - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in group_by_end"); - rc = ER_INTERNAL_ERROR; - } - } - - ti.tpl_ctx = 0; - - if ( ti.tpl_ctx_st.size() ) - { - ti.tpl_ctx_st.pop(); - if ( ti.tpl_ctx_st.size() ) - ti.tpl_ctx = ti.tpl_ctx_st.top(); - } - - if ( ci->cal_conn_hndl_st.size() ) - { - ci->cal_conn_hndl_st.pop(); - if ( ci->cal_conn_hndl_st.size() ) - ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); - } - - ci->tableMap[table] = ti; - - // push warnings from CREATE phase - if (!ci->warningMsg.empty()) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, ci->warningMsg.c_str()); - - ci->warningMsg.clear(); - // reset expressionId just in case - ci->expressionId = 0; + if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT)) + { + force_close_fep_conn(thd, ci, true); // with checking prev command rc return rc; -} + } + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; + // Poping next ExeMgr connection out of the stack + if (ci->cal_conn_hndl_st.size()) + { + ci->cal_conn_hndl_st.pop(); + if (ci->cal_conn_hndl_st.size()) + ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); + } + + return 0; + } + + IDEBUG(cerr << "group_by_end for table " << table->s->table_name.str << endl); + + cal_table_info ti = ci->tableMap[table]; + sm::cpsm_conhdl_t* hndl; + bool clearScanCtx = false; + + hndl = ci->cal_conn_hndl; + + if (ti.tpl_ctx) + { + if (ti.tpl_scan_ctx.get()) + { + clearScanCtx = ((ti.tpl_scan_ctx.get()->rowsreturned) && + ti.tpl_scan_ctx.get()->rowsreturned == ti.tpl_scan_ctx.get()->getRowCount()); + try + { + sm::tpl_scan_close(ti.tpl_scan_ctx); + } + catch (...) + { + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_scan_ctx.reset(); + if (ti.tpl_scan_ctx_st.size()) + { + ti.tpl_scan_ctx_st.pop(); + if (ti.tpl_scan_ctx_st.size()) + ti.tpl_scan_ctx = ti.tpl_scan_ctx_st.top(); + } + try + { + if (hndl) + { + { + bool ask_4_stats = (ci->traceFlags) ? true : false; + sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats, ask_4_stats, clearScanCtx); + } + // Normaly stats variables are set in external_lock method but we set it here + // since they we pretend we are in vtable_disabled mode and the stats vars won't be set. + // We sum the stats up here since server could run a number of + // queries e.g. each for a subquery in a filter. + if (hndl) + { + if (hndl->queryStats.length()) + ci->queryStats += hndl->queryStats; + if (hndl->extendedStats.length()) + ci->extendedStats += hndl->extendedStats; + if (hndl->miniStats.length()) + ci->miniStats += hndl->miniStats; + } + } + + ci->cal_conn_hndl = hndl; + + ti.tpl_ctx = 0; + } + catch (IDBExcept& e) + { + if (e.errorCode() == ERR_CROSS_ENGINE_CONNECT || e.errorCode() == ERR_CROSS_ENGINE_CONFIG) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + else + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + } + catch (std::exception& e) + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in group_by_end"); + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_ctx = 0; + + if (ti.tpl_ctx_st.size()) + { + ti.tpl_ctx_st.pop(); + if (ti.tpl_ctx_st.size()) + ti.tpl_ctx = ti.tpl_ctx_st.top(); + } + + if (ci->cal_conn_hndl_st.size()) + { + ci->cal_conn_hndl_st.pop(); + if (ci->cal_conn_hndl_st.size()) + ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); + } + + ci->tableMap[table] = ti; + + // push warnings from CREATE phase + if (!ci->warningMsg.empty()) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, ci->warningMsg.c_str()); + + ci->warningMsg.clear(); + // reset expressionId just in case + ci->expressionId = 0; + return rc; +} /*@brief Initiate the query for derived_handler */ /*********************************************************** @@ -4772,593 +4795,591 @@ int ha_mcs_impl_group_by_end(TABLE* table) ***********************************************************/ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) { - IDEBUG( cout << "pushdown_init for table " << endl ); - THD* thd = current_thd; + IDEBUG(cout << "pushdown_init for table " << endl); + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return 0; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return 0; - gp_walk_info gwi; - gwi.thd = thd; - bool err = false; + gp_walk_info gwi; + gwi.thd = thd; + bool err = false; - //check whether the system is ready to process statement. + // check whether the system is ready to process statement. #ifndef _MSC_VER - static DBRM dbrm(true); - int bSystemQueryReady = dbrm.getSystemQueryReady(); + static DBRM dbrm(true); + int bSystemQueryReady = dbrm.getSystemQueryReady(); - if (bSystemQueryReady == 0) - { - // Still not ready - setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); - return ER_INTERNAL_ERROR; - } - else if (bSystemQueryReady < 0) - { - // Still not ready - setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); - return ER_INTERNAL_ERROR; - } + if (bSystemQueryReady == 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); + return ER_INTERNAL_ERROR; + } + else if (bSystemQueryReady < 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); + return ER_INTERNAL_ERROR; + } #endif - // Set this to close all outstanding FEP connections on - // client disconnect in handlerton::closecon_handlerton(). - if ( !thd_get_ha_data(thd, mcs_hton)) + // Set this to close all outstanding FEP connections on + // client disconnect in handlerton::closecon_handlerton(). + if (!thd_get_ha_data(thd, mcs_hton)) + { + thd_set_ha_data(thd, mcs_hton, reinterpret_cast(0x42)); + } + + if ((thd->lex)->sql_command == SQLCOM_ALTER_TABLE) + { + return 0; + } + + // MCOL-4023 We need to test this code path. + // Update and delete code + if (isUpdateOrDeleteStatement(thd->lex->sql_command)) + return doUpdateDelete(thd, gwi, std::vector()); + + uint32_t sessionID = tid2sid(thd->thread_id); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + if (!get_fe_conn_info_ptr()) + set_fe_conn_info_ptr(reinterpret_cast(new cal_connection_info(), thd)); + + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + idbassert(ci != 0); + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + if (ci->cal_conn_hndl) { - thd_set_ha_data(thd, mcs_hton, reinterpret_cast(0x42)); - } + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; - if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) - { - return 0; - } + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // canceling query. ignore connection failure. + } - // MCOL-4023 We need to test this code path. - // Update and delete code - if (isUpdateOrDeleteStatement(thd->lex->sql_command)) - return doUpdateDelete(thd, gwi, std::vector()); - - uint32_t sessionID = tid2sid(thd->thread_id); - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - - if (!get_fe_conn_info_ptr()) - set_fe_conn_info_ptr(reinterpret_cast(new cal_connection_info(), thd)); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - - idbassert(ci != 0); - - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) - { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // canceling query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - - return 0; - } - - sm::tableid_t tableid = 0; - cal_table_info ti; - sm::cpsm_conhdl_t* hndl; - SCSEP csep; - // Declare handlers ptrs in this scope for future use. - ha_columnstore_select_handler* sh = nullptr; - ha_columnstore_derived_handler* dh = nullptr; - - // update traceFlags according to the autoswitch state. - ci->traceFlags = (ci->traceFlags | CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)^ - CalpontSelectExecutionPlan::TRACE_TUPLE_OFF; - - bool localQuery = (get_local_query(thd) > 0 ? true : false); - - { - ci->stats.reset(); // reset query stats - ci->stats.setStartTime(); - if (thd->main_security_ctx.user) - { - ci->stats.fUser = thd->main_security_ctx.user; - } - else - { - ci->stats.fUser = ""; - } - - if (thd->main_security_ctx.host) - ci->stats.fHost = thd->main_security_ctx.host; - else if (thd->main_security_ctx.host_or_ip) - ci->stats.fHost = thd->main_security_ctx.host_or_ip; - else - ci->stats.fHost = "unknown"; - - try - { - ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - ci->warningMsg = msg; - } - - // if the previous query has error, re-establish the connection - if (ci->queryState != 0) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - - sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); - idbassert(ci->cal_conn_hndl != 0); - ci->cal_conn_hndl->csc = csc; - idbassert(ci->cal_conn_hndl->exeMgr != 0); - - try - { - ci->cal_conn_hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } - - hndl = ci->cal_conn_hndl; - - IDEBUG( std::cout << idb_mysql_query_str(thd) << std::endl ); - - { - if (!csep) - csep.reset(new CalpontSelectExecutionPlan()); - - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - - csep->txnID(txnID.id); - csep->verID(verID); - csep->sessionID(sessionID); - - if (thd->db.length) - csep->schemaName(thd->db.str, lower_case_table_names); - - csep->traceFlags(ci->traceFlags); - - // cast the handler and get a plan. - int status = 42; - if (handler_info->hndl_type == mcs_handler_types_t::SELECT) - { - sh = reinterpret_cast(handler_info->hndl_ptr); - status = cs_get_select_plan(sh, thd, csep, gwi); - } - else if (handler_info->hndl_type == DERIVED) - { - dh = reinterpret_cast(handler_info->hndl_ptr); - status = cs_get_derived_plan(dh, thd, csep, gwi); - } - - // Return an error to avoid MDB crash later in end_statement - if (status != 0) - goto internal_error; - - string query; - query.assign(idb_mysql_query_str(thd)); - csep->data(query); - - try - { - csep->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - -// DRRTUY Make this runtime configureable -#ifdef PLAN_HEX_FILE - // plan serialization - ifstream ifs("/tmp/li1-plan.hex"); - ByteStream bs1; - ifs >> bs1; - ifs.close(); - csep->unserialize(bs1); -#endif - - if (ci->traceFlags & 1) - { - cerr << "---------------- EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; - } - else - { - IDEBUG( cout << "---------------- EXECUTION PLAN ----------------" << endl ); - IDEBUG( cerr << *csep << endl ); - IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); - } - } - }// end of execution plan generation - - { - ByteStream msg; - ByteStream emsgBs; - - while (true) - { - try - { - ByteStream::quadbyte qb = 4; - msg << qb; - hndl->exeMgr->write(msg); - msg.restart(); - csep->rmParms(ci->rmParms); - - //send plan - csep->serialize(msg); - hndl->exeMgr->write(msg); - - //get ExeMgr status back to indicate a vtable joblist success or not - msg.restart(); - emsgBs.restart(); - msg = hndl->exeMgr->read(); - emsgBs = hndl->exeMgr->read(); - string emsg; - - if (msg.length() == 0 || emsgBs.length() == 0) - { - emsg = "Lost connection to ExeMgr. Please contact your administrator"; - setError(thd, ER_INTERNAL_ERROR, emsg); - return ER_INTERNAL_ERROR; - } - - string emsgStr; - emsgBs >> emsgStr; - - if (msg.length() == 4) - { - msg >> qb; - - if (qb != 0) - { - err = true; - // for makejoblist error, stats contains only error code and insert from here - // because table fetch is not started - ci->stats.setEndTime(); - ci->stats.fQuery = csep->data(); - ci->stats.fQueryType = csep->queryType(); - ci->stats.fErrorNo = qb; - - try - { - ci->stats.insert(); - } - catch (std::exception& e) - { - string msg = string("Columnstore Query Stats - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - } - } - else - { - err = true; - } - - if (err) - { - // CS resets error in create_SH() if fallback is enabled - setError(thd, ER_INTERNAL_ERROR, emsgStr); - goto internal_error; - } - - ci->rmParms.clear(); - - // SH will initiate SM in select_next() only - if (!sh) - ci->queryState= sm::QUERY_IN_PROCESS; - - break; - } - catch (...) - { - sm::sm_cleanup(hndl); - hndl = 0; - - sm::sm_init(sessionID, &hndl, localQuery); - idbassert(hndl != 0); - hndl->csc = csc; - - ci->cal_conn_hndl = hndl; - - try - { - hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } - - msg.restart(); - } - } - } - - // set query state to be in_process. Sometimes mysql calls rnd_init multiple - // times, this makes sure plan only being generated and sent once. It will be - // reset when query finishes in sm::end_query - - // common path for both vtable select phase and table mode -- open scan handle - ti = ci->tableMap[table]; - // This is the server's temp table for the result. - if(sh) - { - ti.msTablePtr = sh->table; - } - else - { - ti.msTablePtr = dh->table; - } - - // For SH CS creates SM environment inside select_next(). - // This allows us to try and fail with SH. - if (!sh) - { - if (ti.tpl_ctx == 0) - { - ti.tpl_ctx = new sm::cpsm_tplh_t(); - ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); - } - - // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql - // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = nullptr; - - try - { - tableid = execplan::IDB_VTABLE_ID; - } - catch (...) - { - string emsg = "No table ID found for table " + string(table->s->table_name.str); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - - try - { - sm::tpl_open(tableid, ti.tpl_ctx, hndl); - sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); - } - catch (std::exception& e) - { - string emsg = "table can not be opened: " + string(e.what()); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - catch (...) - { - string emsg = "table can not be opened"; - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - - ti.tpl_scan_ctx->traceFlags = ci->traceFlags; - - if ((ti.tpl_scan_ctx->ctp).size() == 0) - { - uint32_t num_attr = table->s->fields; - - for (uint32_t i = 0; i < num_attr; i++) - { - CalpontSystemCatalog::ColType ctype; - ti.tpl_scan_ctx->ctp.push_back(ctype); - } - } - ci->tableMap[table] = ti; + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; } return 0; + } + + sm::tableid_t tableid = 0; + cal_table_info ti; + sm::cpsm_conhdl_t* hndl; + SCSEP csep; + // Declare handlers ptrs in this scope for future use. + ha_columnstore_select_handler* sh = nullptr; + ha_columnstore_derived_handler* dh = nullptr; + + // update traceFlags according to the autoswitch state. + ci->traceFlags = (ci->traceFlags | CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) ^ + CalpontSelectExecutionPlan::TRACE_TUPLE_OFF; + + bool localQuery = (get_local_query(thd) > 0 ? true : false); + + { + ci->stats.reset(); // reset query stats + ci->stats.setStartTime(); + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + try + { + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + ci->warningMsg = msg; + } + + // if the previous query has error, re-establish the connection + if (ci->queryState != 0) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); + idbassert(ci->cal_conn_hndl != 0); + ci->cal_conn_hndl->csc = csc; + idbassert(ci->cal_conn_hndl->exeMgr != 0); + + try + { + ci->cal_conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + hndl = ci->cal_conn_hndl; + + IDEBUG(std::cout << idb_mysql_query_str(thd) << std::endl); + + { + if (!csep) + csep.reset(new CalpontSelectExecutionPlan()); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + csep->txnID(txnID.id); + csep->verID(verID); + csep->sessionID(sessionID); + + if (thd->db.length) + csep->schemaName(thd->db.str, lower_case_table_names); + + csep->traceFlags(ci->traceFlags); + + // cast the handler and get a plan. + int status = 42; + if (handler_info->hndl_type == mcs_handler_types_t::SELECT) + { + sh = reinterpret_cast(handler_info->hndl_ptr); + status = cs_get_select_plan(sh, thd, csep, gwi); + } + else if (handler_info->hndl_type == DERIVED) + { + dh = reinterpret_cast(handler_info->hndl_ptr); + status = cs_get_derived_plan(dh, thd, csep, gwi); + } + + // Return an error to avoid MDB crash later in end_statement + if (status != 0) + goto internal_error; + + string query; + query.assign(idb_mysql_query_str(thd)); + csep->data(query); + + try + { + csep->priority(ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + +// DRRTUY Make this runtime configureable +#ifdef PLAN_HEX_FILE + // plan serialization + ifstream ifs("/tmp/li1-plan.hex"); + ByteStream bs1; + ifs >> bs1; + ifs.close(); + csep->unserialize(bs1); +#endif + + if (ci->traceFlags & 1) + { + cerr << "---------------- EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + } + else + { + IDEBUG(cout << "---------------- EXECUTION PLAN ----------------" << endl); + IDEBUG(cerr << *csep << endl); + IDEBUG(cout << "-------------- EXECUTION PLAN END --------------\n" << endl); + } + } + } // end of execution plan generation + + { + ByteStream msg; + ByteStream emsgBs; + + while (true) + { + try + { + ByteStream::quadbyte qb = 4; + msg << qb; + hndl->exeMgr->write(msg); + msg.restart(); + csep->rmParms(ci->rmParms); + + // send plan + csep->serialize(msg); + hndl->exeMgr->write(msg); + + // get ExeMgr status back to indicate a vtable joblist success or not + msg.restart(); + emsgBs.restart(); + msg = hndl->exeMgr->read(); + emsgBs = hndl->exeMgr->read(); + string emsg; + + if (msg.length() == 0 || emsgBs.length() == 0) + { + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb; + + if (qb != 0) + { + err = true; + // for makejoblist error, stats contains only error code and insert from here + // because table fetch is not started + ci->stats.setEndTime(); + ci->stats.fQuery = csep->data(); + ci->stats.fQueryType = csep->queryType(); + ci->stats.fErrorNo = qb; + + try + { + ci->stats.insert(); + } + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + } + } + else + { + err = true; + } + + if (err) + { + // CS resets error in create_SH() if fallback is enabled + setError(thd, ER_INTERNAL_ERROR, emsgStr); + goto internal_error; + } + + ci->rmParms.clear(); + + // SH will initiate SM in select_next() only + if (!sh) + ci->queryState = sm::QUERY_IN_PROCESS; + + break; + } + catch (...) + { + sm::sm_cleanup(hndl); + hndl = 0; + + sm::sm_init(sessionID, &hndl, localQuery); + idbassert(hndl != 0); + hndl->csc = csc; + + ci->cal_conn_hndl = hndl; + + try + { + hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + msg.restart(); + } + } + } + + // set query state to be in_process. Sometimes mysql calls rnd_init multiple + // times, this makes sure plan only being generated and sent once. It will be + // reset when query finishes in sm::end_query + + // common path for both vtable select phase and table mode -- open scan handle + ti = ci->tableMap[table]; + // This is the server's temp table for the result. + if (sh) + { + ti.msTablePtr = sh->table; + } + else + { + ti.msTablePtr = dh->table; + } + + // For SH CS creates SM environment inside select_next(). + // This allows us to try and fail with SH. + if (!sh) + { + if (ti.tpl_ctx == 0) + { + ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); + } + + // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql + // call rnd_init for a table more than once. + ti.tpl_scan_ctx->rowGroup = nullptr; + + try + { + tableid = execplan::IDB_VTABLE_ID; + } + catch (...) + { + string emsg = "No table ID found for table " + string(table->s->table_name.str); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + try + { + sm::tpl_open(tableid, ti.tpl_ctx, hndl); + sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); + } + catch (std::exception& e) + { + string emsg = "table can not be opened: " + string(e.what()); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + catch (...) + { + string emsg = "table can not be opened"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + + if ((ti.tpl_scan_ctx->ctp).size() == 0) + { + uint32_t num_attr = table->s->fields; + + for (uint32_t i = 0; i < num_attr; i++) + { + CalpontSystemCatalog::ColType ctype; + ti.tpl_scan_ctx->ctp.push_back(ctype); + } + } + ci->tableMap[table] = ti; + } + + return 0; error: - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - // do we need to close all connection handle of the table map - return ER_INTERNAL_ERROR; + // do we need to close all connection handle of the table map + return ER_INTERNAL_ERROR; internal_error: - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } - return ER_INTERNAL_ERROR; + return ER_INTERNAL_ERROR; } int ha_mcs_impl_select_next(uchar* buf, TABLE* table) { - THD* thd = current_thd; + THD* thd = current_thd; - if (thd->slave_thread && !get_replication_slave(thd) && - isDMLStatement(thd->lex->sql_command)) - return HA_ERR_END_OF_FILE; + if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) + return HA_ERR_END_OF_FILE; - int rc = HA_ERR_END_OF_FILE; + int rc = HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == nullptr) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == nullptr) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (isUpdateOrDeleteStatement(thd->lex->sql_command)) - return rc; + if (isUpdateOrDeleteStatement(thd->lex->sql_command)) + return rc; - // @bug 2547 - // MCOL-2178 This variable can never be true in the scope of this function - // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) - // return HA_ERR_END_OF_FILE; + // @bug 2547 + // MCOL-2178 This variable can never be true in the scope of this function + // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) + // return HA_ERR_END_OF_FILE; - if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + force_close_fep_conn(thd, ci); + return 0; + } + + if (ci->alterTableState > 0) + return rc; + + cal_table_info ti; + ti = ci->tableMap[table]; + // This is the server's temp table for the result. + ti.msTablePtr = table; + sm::tableid_t tableid = execplan::IDB_VTABLE_ID; + sm::cpsm_conhdl_t* hndl = ci->cal_conn_hndl; + + if (!ti.tpl_ctx || !ti.tpl_scan_ctx || (hndl && hndl->queryState == sm::NO_QUERY)) + { + if (ti.tpl_ctx == 0) { - force_close_fep_conn(thd, ci); - return 0; + ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); } - if (ci->alterTableState > 0) return rc; - - cal_table_info ti; - ti= ci->tableMap[table]; - // This is the server's temp table for the result. - ti.msTablePtr= table; - sm::tableid_t tableid= execplan::IDB_VTABLE_ID; - sm::cpsm_conhdl_t* hndl= ci->cal_conn_hndl; - - if (!ti.tpl_ctx || !ti.tpl_scan_ctx || (hndl && hndl->queryState == sm::NO_QUERY)) - { - if (ti.tpl_ctx == 0) - { - ti.tpl_ctx = new sm::cpsm_tplh_t(); - ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); - } - - // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql - // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = nullptr; - - try - { - sm::tpl_open(tableid, ti.tpl_ctx, hndl); - sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); - } - catch (std::exception& e) - { - uint32_t sessionID = tid2sid(thd->thread_id); - string emsg = "table can not be opened: " + string(e.what()); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - catch (...) - { - uint32_t sessionID = tid2sid(thd->thread_id); - string emsg = "table can not be opened"; - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto internal_error; - } - - ti.tpl_scan_ctx->traceFlags = ci->traceFlags; - - if ((ti.tpl_scan_ctx->ctp).size() == 0) - { - uint32_t num_attr = table->s->fields; - - for (uint32_t i = 0; i < num_attr; i++) - { - CalpontSystemCatalog::ColType ctype; - ti.tpl_scan_ctx->ctp.push_back(ctype); - } - } - ci->tableMap[table] = ti; - hndl->queryState= sm::QUERY_IN_PROCESS; - } - - if (!ti.tpl_ctx || !ti.tpl_scan_ctx) - { - uint32_t sessionID = tid2sid(thd->thread_id); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - idbassert(ti.msTablePtr == table); + // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql + // call rnd_init for a table more than once. + ti.tpl_scan_ctx->rowGroup = nullptr; try { - rc = fetchNextRow(buf, ti, ci); + sm::tpl_open(tableid, ti.tpl_ctx, hndl); + sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); } catch (std::exception& e) { - uint32_t sessionID = tid2sid(thd->thread_id); - string emsg = string("Error while fetching from ExeMgr: ") + e.what(); - setError(thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; + uint32_t sessionID = tid2sid(thd->thread_id); + string emsg = "table can not be opened: " + string(e.what()); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; } - - ci->tableMap[table]= ti; - - if (rc != 0 && rc != HA_ERR_END_OF_FILE) + catch (...) { - string emsg; - - // remove this check when all error handling migrated to the new framework. - if (rc >= 1000) - emsg = ti.tpl_scan_ctx->errMsg; - else - { - logging::ErrorCodes errorcodes; - emsg = errorcodes.errorString(rc); - } - - uint32_t sessionID = tid2sid(thd->thread_id); - setError(thd, ER_INTERNAL_ERROR, emsg); - ci->stats.fErrorNo = rc; - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - rc = ER_INTERNAL_ERROR; + uint32_t sessionID = tid2sid(thd->thread_id); + string emsg = "table can not be opened"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; } - return rc; + ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + + if ((ti.tpl_scan_ctx->ctp).size() == 0) + { + uint32_t num_attr = table->s->fields; + + for (uint32_t i = 0; i < num_attr; i++) + { + CalpontSystemCatalog::ColType ctype; + ti.tpl_scan_ctx->ctp.push_back(ctype); + } + } + ci->tableMap[table] = ti; + hndl->queryState = sm::QUERY_IN_PROCESS; + } + + if (!ti.tpl_ctx || !ti.tpl_scan_ctx) + { + uint32_t sessionID = tid2sid(thd->thread_id); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + idbassert(ti.msTablePtr == table); + + try + { + rc = fetchNextRow(buf, ti, ci); + } + catch (std::exception& e) + { + uint32_t sessionID = tid2sid(thd->thread_id); + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + ci->tableMap[table] = ti; + + if (rc != 0 && rc != HA_ERR_END_OF_FILE) + { + string emsg; + + // remove this check when all error handling migrated to the new framework. + if (rc >= 1000) + emsg = ti.tpl_scan_ctx->errMsg; + else + { + logging::ErrorCodes errorcodes; + emsg = errorcodes.errorString(rc); + } + + uint32_t sessionID = tid2sid(thd->thread_id); + setError(thd, ER_INTERNAL_ERROR, emsg); + ci->stats.fErrorNo = rc; + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + rc = ER_INTERNAL_ERROR; + } + + return rc; internal_error: - if (ci->cal_conn_hndl) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - - return ER_INTERNAL_ERROR; + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + return ER_INTERNAL_ERROR; } // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_mcs_impl.h b/dbcon/mysql/ha_mcs_impl.h index ff21737c5..1b9988fda 100644 --- a/dbcon/mysql/ha_mcs_impl.h +++ b/dbcon/mysql/ha_mcs_impl.h @@ -36,17 +36,18 @@ extern int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_c extern void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table, bool is_cache_insert = false); extern int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table); extern int ha_mcs_impl_rename_table(const char* from, const char* to); -extern int ha_mcs_impl_commit (handlerton* hton, THD* thd, bool all); -extern int ha_mcs_impl_rollback (handlerton* hton, THD* thd, bool all); -extern int ha_mcs_impl_close_connection (handlerton* hton, THD* thd); +extern int ha_mcs_impl_commit(handlerton* hton, THD* thd, bool all); +extern int ha_mcs_impl_rollback(handlerton* hton, THD* thd, bool all); +extern int ha_mcs_impl_close_connection(handlerton* hton, THD* thd); extern COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector&); extern int ha_mcs_impl_update_row(); -extern int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows, const std::vector& condStack); +extern int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows, + const std::vector& condStack); extern int ha_mcs_impl_delete_row(); extern int ha_mcs_impl_rnd_pos(uchar* buf, uchar* pos); extern int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table); -extern int ha_mcs_impl_select_next(uchar *buf, TABLE *table); -extern int ha_mcs_impl_group_by_init(mcs_handler_info *handler_info, TABLE* table); +extern int ha_mcs_impl_select_next(uchar* buf, TABLE* table); +extern int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table); extern int ha_mcs_impl_group_by_next(TABLE* table); extern int ha_mcs_impl_group_by_end(TABLE* table); @@ -57,22 +58,26 @@ extern int ha_mcs_impl_group_by_end(TABLE* table); #include "calpontsystemcatalog.h" #include "ha_mcs.h" extern int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_impl_if::cal_connection_info& ci); -extern int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, ha_rows& rowsInserted); +extern int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, + ha_rows& rowsInserted); extern int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_write_last_batch(TABLE* table, cal_impl_if::cal_connection_info& ci, bool abort); -extern int ha_mcs_impl_commit_ (handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); -extern int ha_mcs_impl_rollback_ (handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); -extern int ha_mcs_impl_close_connection_ (handlerton* hton, THD* thd, cal_impl_if::cal_connection_info& ci); +extern int ha_mcs_impl_commit_(handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); +extern int ha_mcs_impl_rollback_(handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); +extern int ha_mcs_impl_close_connection_(handlerton* hton, THD* thd, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_delete_table_(const char* db, const char* name, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info, - cal_impl_if::cal_connection_info& ci); -extern std::string ha_mcs_impl_markpartition_ (execplan::CalpontSystemCatalog::TableName tableName, uint32_t partition); -extern std::string ha_mcs_impl_restorepartition_ (execplan::CalpontSystemCatalog::TableName tableName, uint32_t partition); -extern std::string ha_mcs_impl_droppartition_ (execplan::CalpontSystemCatalog::TableName tableName, uint32_t partition); + cal_impl_if::cal_connection_info& ci); +extern std::string ha_mcs_impl_markpartition_(execplan::CalpontSystemCatalog::TableName tableName, + uint32_t partition); +extern std::string ha_mcs_impl_restorepartition_(execplan::CalpontSystemCatalog::TableName tableName, + uint32_t partition); +extern std::string ha_mcs_impl_droppartition_(execplan::CalpontSystemCatalog::TableName tableName, + uint32_t partition); -extern std::string ha_mcs_impl_viewtablelock( cal_impl_if::cal_connection_info& ci, execplan::CalpontSystemCatalog::TableName& tablename); -extern std::string ha_mcs_impl_cleartablelock( cal_impl_if::cal_connection_info& ci, uint64_t tableLockID); +extern std::string ha_mcs_impl_viewtablelock(cal_impl_if::cal_connection_info& ci, + execplan::CalpontSystemCatalog::TableName& tablename); +extern std::string ha_mcs_impl_cleartablelock(cal_impl_if::cal_connection_info& ci, uint64_t tableLockID); #endif #endif - diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 9d0a9c4e1..d842d1612 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -49,9 +49,14 @@ class ha_columnstore_derived_handler; /** Debug macro */ #ifdef INFINIDB_DEBUG -#define IDEBUG(x) {x;} +#define IDEBUG(x) \ + { \ + x; \ + } #else -#define IDEBUG(x) {} +#define IDEBUG(x) \ + { \ + } #endif namespace execplan @@ -65,7 +70,7 @@ class ArithmeticColumn; class ParseTree; class ConstantColumn; class RowColumn; -} +} // namespace execplan namespace cal_impl_if { @@ -74,175 +79,185 @@ class View; struct JoinInfo { - execplan::CalpontSystemCatalog::TableAliasName tn; - uint32_t joinTimes; - std::vector IDs; + execplan::CalpontSystemCatalog::TableAliasName tn; + uint32_t joinTimes; + std::vector IDs; }; enum ClauseType { - INIT = 0, - SELECT, - FROM, - WHERE, - HAVING, - GROUP_BY, - ORDER_BY + INIT = 0, + SELECT, + FROM, + WHERE, + HAVING, + GROUP_BY, + ORDER_BY }; typedef std::vector JoinInfoVec; -typedef std::map > TableMap; +typedef std::map> TableMap; typedef std::tr1::unordered_map> TableOnExprList; typedef std::tr1::unordered_map TableOuterJoinMap; struct gp_walk_info { - // MCOL-2178 Marked for removal after 1.4 - std::vector selectCols; - execplan::CalpontSelectExecutionPlan::ReturnedColumnList returnedCols; - execplan::CalpontSelectExecutionPlan::ReturnedColumnList groupByCols; - execplan::CalpontSelectExecutionPlan::ReturnedColumnList subGroupByCols; - execplan::CalpontSelectExecutionPlan::ReturnedColumnList orderByCols; - std::vector extSelAggColsItems; - execplan::CalpontSelectExecutionPlan::ColumnMap columnMap; - // This vector temporarily hold the projection columns to be added - // to the returnedCols vector for subquery processing. It will be appended - // to the end of returnedCols when the processing is finished. - execplan::CalpontSelectExecutionPlan::ReturnedColumnList additionalRetCols; - // the sequence # of the subselect local columns need to be set after - // the additionRetCols are merged to the returnedCols. - std::vector localCols; - std::stack rcWorkStack; - std::stack ptWorkStack; - boost::shared_ptr scsp; - uint32_t sessionid; - bool fatalParseError; - std::string parseErrorText; - // for outer join walk. the column that is not of the outerTable has the returnAll flag set. - std::set innerTables; - // the followinig members are used for table mode - bool condPush; - bool dropCond; - std::string funcName; - std::vector count_asterisk_list; - std::vector no_parm_func_list; - std::vector windowFuncList; - TableMap tableMap; - boost::shared_ptr csc; - int8_t internalDecimalScale; - THD* thd; - uint64_t subSelectType; // the type of sub select filter that owns the gwi - SubQuery* subQuery; - execplan::CalpontSelectExecutionPlan::SelectList derivedTbList; - execplan::CalpontSelectExecutionPlan::TableList tbList; - std::vector correlatedTbNameVec; - ClauseType clauseType; - execplan::CalpontSystemCatalog::TableAliasName viewName; - bool aggOnSelect; - bool hasWindowFunc; - bool hasSubSelect; - SubQuery* lastSub; - std::vector viewList; - std::map derivedTbFilterMap; - uint32_t derivedTbCnt; - std::vector subselectList; - // Workaround for duplicate equi-JOIN predicates - // See isDuplicateSF() for more info. - List equiCondSFList; + // MCOL-2178 Marked for removal after 1.4 + std::vector selectCols; + execplan::CalpontSelectExecutionPlan::ReturnedColumnList returnedCols; + execplan::CalpontSelectExecutionPlan::ReturnedColumnList groupByCols; + execplan::CalpontSelectExecutionPlan::ReturnedColumnList subGroupByCols; + execplan::CalpontSelectExecutionPlan::ReturnedColumnList orderByCols; + std::vector extSelAggColsItems; + execplan::CalpontSelectExecutionPlan::ColumnMap columnMap; + // This vector temporarily hold the projection columns to be added + // to the returnedCols vector for subquery processing. It will be appended + // to the end of returnedCols when the processing is finished. + execplan::CalpontSelectExecutionPlan::ReturnedColumnList additionalRetCols; + // the sequence # of the subselect local columns need to be set after + // the additionRetCols are merged to the returnedCols. + std::vector localCols; + std::stack rcWorkStack; + std::stack ptWorkStack; + boost::shared_ptr scsp; + uint32_t sessionid; + bool fatalParseError; + std::string parseErrorText; + // for outer join walk. the column that is not of the outerTable has the returnAll flag set. + std::set innerTables; + // the followinig members are used for table mode + bool condPush; + bool dropCond; + std::string funcName; + std::vector count_asterisk_list; + std::vector no_parm_func_list; + std::vector windowFuncList; + TableMap tableMap; + boost::shared_ptr csc; + int8_t internalDecimalScale; + THD* thd; + uint64_t subSelectType; // the type of sub select filter that owns the gwi + SubQuery* subQuery; + execplan::CalpontSelectExecutionPlan::SelectList derivedTbList; + execplan::CalpontSelectExecutionPlan::TableList tbList; + std::vector correlatedTbNameVec; + ClauseType clauseType; + execplan::CalpontSystemCatalog::TableAliasName viewName; + bool aggOnSelect; + bool hasWindowFunc; + bool hasSubSelect; + SubQuery* lastSub; + std::vector viewList; + std::map derivedTbFilterMap; + uint32_t derivedTbCnt; + std::vector subselectList; + // Workaround for duplicate equi-JOIN predicates + // See isDuplicateSF() for more info. + List equiCondSFList; - // Kludge for Bug 750 - int32_t recursionLevel; - int32_t recursionHWM; - std::stack rcBookMarkStack; + // Kludge for Bug 750 + int32_t recursionLevel; + int32_t recursionHWM; + std::stack rcBookMarkStack; - // Kludge for MCOL-1472 - bool inCaseStmt; - bool cs_vtable_is_update_with_derive; - bool cs_vtable_impossible_where_on_union; + // Kludge for MCOL-1472 + bool inCaseStmt; + bool cs_vtable_is_update_with_derive; + bool cs_vtable_impossible_where_on_union; - bool isGroupByHandler; + bool isGroupByHandler; - // MCOL-4617 The below 2 fields are used for in-to-exists - // predicate creation and injection. See usage in InSub::transform() - // and buildInToExistsFilter() - execplan::ReturnedColumn* inSubQueryLHS; - Item* inSubQueryLHSItem; + // MCOL-4617 The below 2 fields are used for in-to-exists + // predicate creation and injection. See usage in InSub::transform() + // and buildInToExistsFilter() + execplan::ReturnedColumn* inSubQueryLHS; + Item* inSubQueryLHSItem; - // The below 2 fields are required for MCOL-4525. - TableOnExprList tableOnExprList; - std::vector condList; + // The below 2 fields are required for MCOL-4525. + TableOnExprList tableOnExprList; + std::vector condList; - gp_walk_info() : sessionid(0), - fatalParseError(false), - condPush(false), - dropCond(false), - internalDecimalScale(4), - thd(0), - subSelectType(uint64_t(-1)), - subQuery(0), - clauseType(INIT), - aggOnSelect(false), - hasWindowFunc(false), - hasSubSelect(false), - lastSub(0), - derivedTbCnt(0), - recursionLevel(-1), - recursionHWM(0), - inCaseStmt(false), - cs_vtable_is_update_with_derive(false), - cs_vtable_impossible_where_on_union(false), - isGroupByHandler(false), - inSubQueryLHS(nullptr), - inSubQueryLHSItem(nullptr) - {} + gp_walk_info() + : sessionid(0) + , fatalParseError(false) + , condPush(false) + , dropCond(false) + , internalDecimalScale(4) + , thd(0) + , subSelectType(uint64_t(-1)) + , subQuery(0) + , clauseType(INIT) + , aggOnSelect(false) + , hasWindowFunc(false) + , hasSubSelect(false) + , lastSub(0) + , derivedTbCnt(0) + , recursionLevel(-1) + , recursionHWM(0) + , inCaseStmt(false) + , cs_vtable_is_update_with_derive(false) + , cs_vtable_impossible_where_on_union(false) + , isGroupByHandler(false) + , inSubQueryLHS(nullptr) + , inSubQueryLHSItem(nullptr) + { + } - ~gp_walk_info() {} + ~gp_walk_info() + { + } }; struct cal_table_info { - enum RowSources { FROM_ENGINE, FROM_FILE }; + enum RowSources + { + FROM_ENGINE, + FROM_FILE + }; - cal_table_info() : tpl_ctx(0), - c(0), - msTablePtr(0), - conn_hndl(0), - condInfo(0), - moreRows(false) - { } - ~cal_table_info() {} - sm::cpsm_tplh_t* tpl_ctx; - std::stack tpl_ctx_st; - sm::sp_cpsm_tplsch_t tpl_scan_ctx; - std::stack tpl_scan_ctx_st; - unsigned c; // for debug purpose - TABLE* msTablePtr; // no ownership - sm::cpsm_conhdl_t* conn_hndl; - gp_walk_info* condInfo; - execplan::SCSEP csep; - bool moreRows; //are there more rows to consume (b/c of limit) + cal_table_info() : tpl_ctx(0), c(0), msTablePtr(0), conn_hndl(0), condInfo(0), moreRows(false) + { + } + ~cal_table_info() + { + } + sm::cpsm_tplh_t* tpl_ctx; + std::stack tpl_ctx_st; + sm::sp_cpsm_tplsch_t tpl_scan_ctx; + std::stack tpl_scan_ctx_st; + unsigned c; // for debug purpose + TABLE* msTablePtr; // no ownership + sm::cpsm_conhdl_t* conn_hndl; + gp_walk_info* condInfo; + execplan::SCSEP csep; + bool moreRows; // are there more rows to consume (b/c of limit) }; struct cal_group_info { - cal_group_info() : groupByFields(0), - groupByTables(0), - groupByWhere(0), - groupByGroup(0), - groupByOrder(0), - groupByHaving(0), - groupByDistinct(false) - { } - ~cal_group_info() { } + cal_group_info() + : groupByFields(0) + , groupByTables(0) + , groupByWhere(0) + , groupByGroup(0) + , groupByOrder(0) + , groupByHaving(0) + , groupByDistinct(false) + { + } + ~cal_group_info() + { + } - List* groupByFields; // MCOL-1052 SELECT - TABLE_LIST* groupByTables; // MCOL-1052 FROM - Item* groupByWhere; // MCOL-1052 WHERE - ORDER* groupByGroup; // MCOL-1052 GROUP BY - ORDER* groupByOrder; // MCOL-1052 ORDER BY - Item* groupByHaving; // MCOL-1052 HAVING - bool groupByDistinct; //MCOL-1052 DISTINCT - std::vector pushedPts; + List* groupByFields; // MCOL-1052 SELECT + TABLE_LIST* groupByTables; // MCOL-1052 FROM + Item* groupByWhere; // MCOL-1052 WHERE + ORDER* groupByGroup; // MCOL-1052 GROUP BY + ORDER* groupByOrder; // MCOL-1052 ORDER BY + Item* groupByHaving; // MCOL-1052 HAVING + bool groupByDistinct; // MCOL-1052 DISTINCT + std::vector pushedPts; }; typedef std::tr1::unordered_map CalTableMap; @@ -252,131 +267,147 @@ typedef std::map TableValuesMap; typedef std::bitset<4096> NullValuesBitset; struct cal_connection_info { - enum AlterTableState { NOT_ALTER, ALTER_SECOND_RENAME, ALTER_FIRST_RENAME }; - cal_connection_info() : cal_conn_hndl(0), - queryState(0), - currentTable(0), - traceFlags(0), - alterTableState(NOT_ALTER), - isAlter(false), - bulkInsertRows(0), - singleInsert(true), - isLoaddataInfile(false), - isCacheInsert(false), - dmlProc(0), - rowsHaveInserted(0), - rc(0), - tableOid(0), - localPm(-1), - isSlaveNode(false), - expressionId(0), - mysqld_pid(getpid()), - cpimport_pid(0), - filePtr(0), - headerLength(0), - useXbit(false), - useCpimport(mcs_use_import_for_batchinsert_mode_t::ON), - delimiter('\7'), - affectedRows(0) - { - auto* cf = config::Config::makeConfig(); - if (checkQueryStats(cf)) - traceFlags |= execplan::CalpontSelectExecutionPlan::TRACE_LOG; - // check if this is a slave mysql daemon - isSlaveNode = checkSlave(cf); - } + enum AlterTableState + { + NOT_ALTER, + ALTER_SECOND_RENAME, + ALTER_FIRST_RENAME + }; + cal_connection_info() + : cal_conn_hndl(0) + , queryState(0) + , currentTable(0) + , traceFlags(0) + , alterTableState(NOT_ALTER) + , isAlter(false) + , bulkInsertRows(0) + , singleInsert(true) + , isLoaddataInfile(false) + , isCacheInsert(false) + , dmlProc(0) + , rowsHaveInserted(0) + , rc(0) + , tableOid(0) + , localPm(-1) + , isSlaveNode(false) + , expressionId(0) + , mysqld_pid(getpid()) + , cpimport_pid(0) + , filePtr(0) + , headerLength(0) + , useXbit(false) + , useCpimport(mcs_use_import_for_batchinsert_mode_t::ON) + , delimiter('\7') + , affectedRows(0) + { + auto* cf = config::Config::makeConfig(); + if (checkQueryStats(cf)) + traceFlags |= execplan::CalpontSelectExecutionPlan::TRACE_LOG; + // check if this is a slave mysql daemon + isSlaveNode = checkSlave(cf); + } - bool checkQueryStats(config::Config* cfg) - { - std::string qsVal = cfg->getConfig("QueryStats", "Enabled"); - if (qsVal == "Y" || qsVal == "Y") - return true; + bool checkQueryStats(config::Config* cfg) + { + std::string qsVal = cfg->getConfig("QueryStats", "Enabled"); + if (qsVal == "Y" || qsVal == "Y") + return true; - return false; - } + return false; + } - static bool checkSlave(config::Config* cf = nullptr) - { - if (!cf) - cf = config::Config::makeConfig(); - std::string configVal = cf->getConfig("Installation", "MySQLRep"); - bool isMysqlRep = (configVal == "y" || configVal == "Y"); + static bool checkSlave(config::Config* cf = nullptr) + { + if (!cf) + cf = config::Config::makeConfig(); + std::string configVal = cf->getConfig("Installation", "MySQLRep"); + bool isMysqlRep = (configVal == "y" || configVal == "Y"); - if (!isMysqlRep) return false; + if (!isMysqlRep) + return false; - configVal = cf->getConfig("SystemConfig", "PrimaryUMModuleName"); - std::string module = execplan::ClientRotator::getModule(); + configVal = cf->getConfig("SystemConfig", "PrimaryUMModuleName"); + std::string module = execplan::ClientRotator::getModule(); - if (boost::iequals(configVal, module)) - return false; + if (boost::iequals(configVal, module)) + return false; - return true; - } + return true; + } - sm::cpsm_conhdl_t* cal_conn_hndl; - std::stack cal_conn_hndl_st; - int queryState; - CalTableMap tableMap; - std::set physTablesList; - sm::tableid_t currentTable; - uint32_t traceFlags; - std::string queryStats; - AlterTableState alterTableState; - bool isAlter; - ha_rows bulkInsertRows; - bool singleInsert; - bool isLoaddataInfile; - bool isCacheInsert; - std::string extendedStats; - std::string miniStats; - messageqcpp::MessageQueueClient* dmlProc; - ha_rows rowsHaveInserted; - ColNameList colNameList; - TableValuesMap tableValuesMap; - NullValuesBitset nullValuesBitset; - int rc; - uint32_t tableOid; - querystats::QueryStats stats; - std::string warningMsg; - int64_t localPm; - bool isSlaveNode; - uint32_t expressionId; // for F&E - pid_t mysqld_pid; - pid_t cpimport_pid; - int fdt[2]; + sm::cpsm_conhdl_t* cal_conn_hndl; + std::stack cal_conn_hndl_st; + int queryState; + CalTableMap tableMap; + std::set physTablesList; + sm::tableid_t currentTable; + uint32_t traceFlags; + std::string queryStats; + AlterTableState alterTableState; + bool isAlter; + ha_rows bulkInsertRows; + bool singleInsert; + bool isLoaddataInfile; + bool isCacheInsert; + std::string extendedStats; + std::string miniStats; + messageqcpp::MessageQueueClient* dmlProc; + ha_rows rowsHaveInserted; + ColNameList colNameList; + TableValuesMap tableValuesMap; + NullValuesBitset nullValuesBitset; + int rc; + uint32_t tableOid; + querystats::QueryStats stats; + std::string warningMsg; + int64_t localPm; + bool isSlaveNode; + uint32_t expressionId; // for F&E + pid_t mysqld_pid; + pid_t cpimport_pid; + int fdt[2]; #ifdef _MSC_VER - // Used for launching cpimport for Load Data Infile - HANDLE cpimport_stdin_Rd; - HANDLE cpimport_stdin_Wr; - HANDLE cpimport_stdout_Rd; - HANDLE cpimport_stdout_Wr; - PROCESS_INFORMATION cpimportProcInfo; + // Used for launching cpimport for Load Data Infile + HANDLE cpimport_stdin_Rd; + HANDLE cpimport_stdin_Wr; + HANDLE cpimport_stdout_Rd; + HANDLE cpimport_stdout_Wr; + PROCESS_INFORMATION cpimportProcInfo; #endif - FILE* filePtr; - uint8_t headerLength; - bool useXbit; - mcs_use_import_for_batchinsert_mode_t useCpimport; - char delimiter; - char enclosed_by; - std::vector columnTypes; - // MCOL-1101 remove compilation unit variable rmParms - std::vector rmParms; - long long affectedRows; + FILE* filePtr; + uint8_t headerLength; + bool useXbit; + mcs_use_import_for_batchinsert_mode_t useCpimport; + char delimiter; + char enclosed_by; + std::vector columnTypes; + // MCOL-1101 remove compilation unit variable rmParms + std::vector rmParms; + long long affectedRows; }; -const std::string infinidb_err_msg = "\nThe query includes syntax that is not supported by MariaDB Columnstore. Use 'show warnings;' to get more information. Review the MariaDB Columnstore Syntax guide for additional information on supported distributed syntax or consider changing the MariaDB Columnstore Operating Mode (infinidb_vtable_mode)."; +const std::string infinidb_err_msg = + "\nThe query includes syntax that is not supported by MariaDB Columnstore. Use 'show warnings;' to get " + "more information. Review the MariaDB Columnstore Syntax guide for additional information on supported " + "distributed syntax or consider changing the MariaDB Columnstore Operating Mode (infinidb_vtable_mode)."; int cp_get_plan(THD* thd, execplan::SCSEP& csep); int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); -int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); -int cs_get_select_plan(ha_columnstore_select_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); -int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false, bool isSelectHandlerTop = false, const std::vector& condStack = std::vector()); -int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, cal_group_info& gi, bool isUnion = false); +int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, execplan::SCSEP& csep, + gp_walk_info& gwi); +int cs_get_select_plan(ha_columnstore_select_handler* handler, THD* thd, execplan::SCSEP& csep, + gp_walk_info& gwi); +int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false, + bool isSelectHandlerTop = false, + const std::vector& condStack = std::vector()); +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, cal_group_info& gi, + bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); void gp_walk(const Item* item, void* arg); -void parse_item (Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo, gp_walk_info* gwip = NULL); +void parse_item(Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo, + gp_walk_info* gwip = NULL); const std::string bestTableName(const Item_field* ifp); bool isMCSTable(TABLE* table_ptr); bool isForeignTableUpdate(THD* thd); @@ -385,32 +416,39 @@ bool isMCSTableUpdate(THD* thd); bool isMCSTableDelete(THD* thd); // execution plan util functions prototypes -execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem = false); -execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn = false); +execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, + bool isRefItem = false); +execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, + bool selectBetweenIn = false); execplan::ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport); -execplan::ConstantColumn* buildDecimalColumn(const Item *item, const std::string &str, gp_walk_info& gwi); +execplan::ConstantColumn* buildDecimalColumn(const Item* item, const std::string& str, gp_walk_info& gwi); execplan::SimpleColumn* buildSimpleColumn(Item_field* item, gp_walk_info& gwi); execplan::FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonSupport); execplan::ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& nonSupport); execplan::ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi); execplan::ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport); -execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, uint32_t pseudoType); +execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, + uint32_t pseudoType); void addIntervalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); void castCharArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); void castDecimalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); void castTypeArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -//void parse_item (Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo); +// void parse_item (Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& +// parseInfo); bool isPredicateFunction(Item* item, gp_walk_info* gwip); -execplan::ParseTree* buildRowPredicate(THD* thd, execplan::RowColumn* lhs, execplan::RowColumn* rhs, std::string predicateOp); -bool buildRowColumnFilter(gp_walk_info* gwip, execplan::RowColumn* rhs, execplan::RowColumn* lhs, Item_func* ifp); +execplan::ParseTree* buildRowPredicate(THD* thd, execplan::RowColumn* lhs, execplan::RowColumn* rhs, + std::string predicateOp); +bool buildRowColumnFilter(gp_walk_info* gwip, execplan::RowColumn* rhs, execplan::RowColumn* lhs, + Item_func* ifp); bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip); void collectAllCols(gp_walk_info& gwi, Item_field* ifp); void buildSubselectFunc(Item_func* ifp, gp_walk_info* gwip); -uint32_t buildJoin(gp_walk_info& gwi, List& join_list, std::stack& outerJoinStack); +uint32_t buildJoin(gp_walk_info& gwi, List& join_list, + std::stack& outerJoinStack); std::string getViewName(TABLE_LIST* table_ptr); bool buildConstPredicate(Item_func* ifp, execplan::ReturnedColumn* rhs, gp_walk_info* gwip); -execplan::CalpontSystemCatalog::ColType fieldType_MysqlToIDB (const Field* field); -execplan::CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item); +execplan::CalpontSystemCatalog::ColType fieldType_MysqlToIDB(const Field* field); +execplan::CalpontSystemCatalog::ColType colType_MysqlToIDB(const Item* item); execplan::SPTP getIntervalType(THD* thd, int interval_type); uint32_t isPseudoColumn(std::string funcName); void setDerivedTable(execplan::ParseTree* n); @@ -418,45 +456,34 @@ execplan::ParseTree* setDerivedFilter(THD* thd, execplan::ParseTree*& n, std::map& obj, execplan::CalpontSelectExecutionPlan::SelectList& derivedTbList); void derivedTableOptimization(THD* thd, execplan::SCSEP& csep); -bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, - execplan::ReturnedColumn* rhs, - gp_walk_info* gwip, - boost::shared_ptr& sop, - const Item_func::Functype& funcType, - const std::vector& itemList, - bool isInSubs = false); +bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedColumn* rhs, gp_walk_info* gwip, + boost::shared_ptr& sop, const Item_func::Functype& funcType, + const std::vector& itemList, bool isInSubs = false); inline bool isUpdateStatement(const enum_sql_command& command) { - return ((command == SQLCOM_UPDATE) || - (command == SQLCOM_UPDATE_MULTI)); + return ((command == SQLCOM_UPDATE) || (command == SQLCOM_UPDATE_MULTI)); } inline bool isDeleteStatement(const enum_sql_command& command) { - return (command == SQLCOM_DELETE) || - (command == SQLCOM_DELETE_MULTI); + return (command == SQLCOM_DELETE) || (command == SQLCOM_DELETE_MULTI); } inline bool isUpdateOrDeleteStatement(const enum_sql_command& command) { - return isUpdateStatement(command) || - isDeleteStatement(command); + return isUpdateStatement(command) || isDeleteStatement(command); } inline bool isDMLStatement(const enum_sql_command& command) { - return (command == SQLCOM_INSERT || - command == SQLCOM_INSERT_SELECT || - command == SQLCOM_TRUNCATE || - command == SQLCOM_LOAD || - isUpdateOrDeleteStatement(command)); + return (command == SQLCOM_INSERT || command == SQLCOM_INSERT_SELECT || command == SQLCOM_TRUNCATE || + command == SQLCOM_LOAD || isUpdateOrDeleteStatement(command)); } #ifdef DEBUG_WALK_COND void debug_walk(const Item* item, void* arg); #endif -} +} // namespace cal_impl_if #endif - diff --git a/dbcon/mysql/ha_mcs_logging.h b/dbcon/mysql/ha_mcs_logging.h index f3bcf108d..9293ad32c 100644 --- a/dbcon/mysql/ha_mcs_logging.h +++ b/dbcon/mysql/ha_mcs_logging.h @@ -33,21 +33,20 @@ */ namespace ha_mcs_impl { -inline void log_this(THD *thd, const char *message, - logging::LOG_TYPE log_type, unsigned sid) +inline void log_this(THD* thd, const char* message, logging::LOG_TYPE log_type, unsigned sid) { - // corresponds with dbcon in SubsystemID vector - // in messagelog.cpp - unsigned int subSystemId = 24; - logging::LoggingID logid( subSystemId, sid, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add(message); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(log_type, msg, logid); + // corresponds with dbcon in SubsystemID vector + // in messagelog.cpp + unsigned int subSystemId = 24; + logging::LoggingID logid(subSystemId, sid, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add(message); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(log_type, msg, logid); } -}; +}; // namespace ha_mcs_impl #endif diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 5f9b6009b..c6bea2401 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -21,135 +21,130 @@ /*@brief in_subselect_rewrite_walk - Rewrites Item_in_subselect*/ /************************************************************ -* DESCRIPTION: -* It traverses filter predicates searching for -* Item_in_subselect and rewrites it adding equi-join predicate -* to finalise IN_2_EXISTS rewrite. -* PARAMETERS: -* item_arg - Item to check. -* arg - bool to early return if predicate injection fails. -* RETURN: -***********************************************************/ + * DESCRIPTION: + * It traverses filter predicates searching for + * Item_in_subselect and rewrites it adding equi-join predicate + * to finalise IN_2_EXISTS rewrite. + * PARAMETERS: + * item_arg - Item to check. + * arg - bool to early return if predicate injection fails. + * RETURN: + ***********************************************************/ void in_subselect_rewrite_walk(const Item* item_arg, void* arg) { - bool* result= reinterpret_cast(arg); - if (*result) return; + bool* result = reinterpret_cast(arg); + if (*result) + return; - Item* item= const_cast(item_arg); + Item* item = const_cast(item_arg); - JOIN* join= nullptr; - if (typeid(*item) == typeid(Item_in_subselect)) - { - Item_in_subselect* sub= reinterpret_cast(item); - // MCS 1.4.3 doesn't support IN + subquery with UNION so - // we safe to take this JOIN. - join= sub->unit->first_select()->join; - // Inject equi-JOIN predicates if needed. - *result= sub->create_in_to_exists_cond(join); - *result= (*result) ? *result : - sub->inject_in_to_exists_cond(join); - sub->unit->first_select()->prep_where= - join->conds ? join->conds->copy_andor_structure(current_thd) : 0; - } - else if (typeid(*item) == typeid(Item_singlerow_subselect)) - { - Item_singlerow_subselect* sub= - reinterpret_cast(item); - // MCS 1.4.3 doesn't support IN + subquery with UNION so - // we safe to take this JOIN. - join= sub->unit->first_select()->join; - } - else - { - // Exit for any but dedicated Items. - return; - } + JOIN* join = nullptr; + if (typeid(*item) == typeid(Item_in_subselect)) + { + Item_in_subselect* sub = reinterpret_cast(item); + // MCS 1.4.3 doesn't support IN + subquery with UNION so + // we safe to take this JOIN. + join = sub->unit->first_select()->join; + // Inject equi-JOIN predicates if needed. + *result = sub->create_in_to_exists_cond(join); + *result = (*result) ? *result : sub->inject_in_to_exists_cond(join); + sub->unit->first_select()->prep_where = join->conds ? join->conds->copy_andor_structure(current_thd) : 0; + } + else if (typeid(*item) == typeid(Item_singlerow_subselect)) + { + Item_singlerow_subselect* sub = reinterpret_cast(item); + // MCS 1.4.3 doesn't support IN + subquery with UNION so + // we safe to take this JOIN. + join = sub->unit->first_select()->join; + } + else + { + // Exit for any but dedicated Items. + return; + } - // Walk recursively to process nested IN ops. - if (join->conds) - { - join->conds->traverse_cond(in_subselect_rewrite_walk, - arg, Item::POSTFIX); - } + // Walk recursively to process nested IN ops. + if (join->conds) + { + join->conds->traverse_cond(in_subselect_rewrite_walk, arg, Item::POSTFIX); + } } // Sets SELECT_LEX::first_cond_optimization void first_cond_optimization_flag_set(SELECT_LEX* select_lex) { - select_lex->first_cond_optimization = true; + select_lex->first_cond_optimization = true; } // Unsets SELECT_LEX::first_cond_optimization void first_cond_optimization_flag_unset(SELECT_LEX* select_lex) { - select_lex->first_cond_optimization = false; + select_lex->first_cond_optimization = false; } /* @brief first_cond_optimization_flag_toggle() - Sets/Unsets first_cond_optimization */ /************************************************************ -* DESCRIPTION: -* This function traverses SELECT_LEX::table_list recursively -* to set/unset SELECT_LEX::first_cond_optimization: a marker -* to control optimizations executing PS. If set it allows to -* apply optimizations. If unset, it disables optimizations. -* PARAMETERS: -* select_lex - SELECT_LEX* that describes the query. -* func - Pointer to a function which either sets/unsets -* SELECT_LEX::first_cond_optimization -***********************************************************/ -void first_cond_optimization_flag_toggle(SELECT_LEX *select_lex, void (*func)(SELECT_LEX*)) + * DESCRIPTION: + * This function traverses SELECT_LEX::table_list recursively + * to set/unset SELECT_LEX::first_cond_optimization: a marker + * to control optimizations executing PS. If set it allows to + * apply optimizations. If unset, it disables optimizations. + * PARAMETERS: + * select_lex - SELECT_LEX* that describes the query. + * func - Pointer to a function which either sets/unsets + * SELECT_LEX::first_cond_optimization + ***********************************************************/ +void first_cond_optimization_flag_toggle(SELECT_LEX* select_lex, void (*func)(SELECT_LEX*)) { - for (TABLE_LIST *tl= select_lex->get_table_list(); tl; tl= tl->next_local) + for (TABLE_LIST* tl = select_lex->get_table_list(); tl; tl = tl->next_local) + { + if (tl->is_view_or_derived()) { - if (tl->is_view_or_derived()) + SELECT_LEX_UNIT* unit = tl->get_unit(); + + if (unit) + { + for (SELECT_LEX* sl = unit->first_select(); sl; sl = sl->next_select()) { - SELECT_LEX_UNIT *unit= tl->get_unit(); - - if (unit) - { - for (SELECT_LEX *sl= unit->first_select(); sl; sl= sl->next_select()) - { - first_cond_optimization_flag_toggle(sl, func); - } - } + first_cond_optimization_flag_toggle(sl, func); } + } } + } - (*func)(select_lex); + (*func)(select_lex); } - /* @brief in_subselect_rewrite - Rewrites Item_in_subselect */ /************************************************************ -* DESCRIPTION: -* This function traverses TABLE_LISTs running in_subselect_rewrite_walk -* PARAMETERS: -* select_lex - SELECT_LEX* that describes the query. -* RETURN: -* bool to indicate predicate injection failures. -***********************************************************/ -bool in_subselect_rewrite(SELECT_LEX *select_lex) + * DESCRIPTION: + * This function traverses TABLE_LISTs running in_subselect_rewrite_walk + * PARAMETERS: + * select_lex - SELECT_LEX* that describes the query. + * RETURN: + * bool to indicate predicate injection failures. + ***********************************************************/ +bool in_subselect_rewrite(SELECT_LEX* select_lex) { - bool result = false; - TABLE_LIST *tbl; - List_iterator_fast li(select_lex->leaf_tables); + bool result = false; + TABLE_LIST* tbl; + List_iterator_fast li(select_lex->leaf_tables); - while (!result && (tbl = li++)) + while (!result && (tbl = li++)) + { + if (tbl->is_view_or_derived()) { - if (tbl->is_view_or_derived()) - { - SELECT_LEX_UNIT *unit= tbl->get_unit(); + SELECT_LEX_UNIT* unit = tbl->get_unit(); - for (SELECT_LEX *sl= unit->first_select(); sl; sl= sl->next_select()) - result = in_subselect_rewrite(sl); - } + for (SELECT_LEX* sl = unit->first_select(); sl; sl = sl->next_select()) + result = in_subselect_rewrite(sl); } + } - if (select_lex->join && select_lex->join->conds) - { - select_lex->join->conds->traverse_cond(in_subselect_rewrite_walk, &result, - Item::POSTFIX); - } + if (select_lex->join && select_lex->join->conds) + { + select_lex->join->conds->traverse_cond(in_subselect_rewrite_walk, &result, Item::POSTFIX); + } - return result; + return result; } diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index 56ec49a06..4b3a2808c 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -20,8 +20,8 @@ #include "idb_mysql.h" -bool in_subselect_rewrite(SELECT_LEX *select_lex); -void first_cond_optimization_flag_toggle(SELECT_LEX *select_lex, void (*func)(SELECT_LEX*)); +bool in_subselect_rewrite(SELECT_LEX* select_lex); +void first_cond_optimization_flag_toggle(SELECT_LEX* select_lex, void (*func)(SELECT_LEX*)); void first_cond_optimization_flag_unset(SELECT_LEX*); void first_cond_optimization_flag_set(SELECT_LEX*); diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index c8cacf0fb..8ed8c59f3 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /* -* $Id: ha_mcs_partition.cpp 9642 2013-06-24 14:57:42Z rdempsey $ -*/ + * $Id: ha_mcs_partition.cpp 9642 2013-06-24 14:57:42Z rdempsey $ + */ #define PREFER_MY_CONFIG_H #include @@ -33,7 +33,6 @@ using namespace std; #include - #include "idb_mysql.h" #include "blocksize.h" @@ -69,1385 +68,1351 @@ using namespace boost; namespace { - -datatypes::SimpleValue getStartVal(const datatypes::SessionParam &sp, - const CalpontSystemCatalog::ColType &ct, - const char *val, - datatypes::round_style_t & rfMin) +datatypes::SimpleValue getStartVal(const datatypes::SessionParam& sp, const CalpontSystemCatalog::ColType& ct, + const char* val, datatypes::round_style_t& rfMin) { - const datatypes::TypeHandler *h= ct.typeHandler(); - return val ? h->toSimpleValue(sp, ct, val, rfMin) : - h->getMinValueSimple(); + const datatypes::TypeHandler* h = ct.typeHandler(); + return val ? h->toSimpleValue(sp, ct, val, rfMin) : h->getMinValueSimple(); } - -datatypes::SimpleValue getEndVal(const datatypes::SessionParam &sp, - const CalpontSystemCatalog::ColType &ct, - const char *val, - datatypes::round_style_t & rfMax) +datatypes::SimpleValue getEndVal(const datatypes::SessionParam& sp, const CalpontSystemCatalog::ColType& ct, + const char* val, datatypes::round_style_t& rfMax) { - const datatypes::TypeHandler *h= ct.typeHandler(); - return val ? h->toSimpleValue(sp, ct, val, rfMax) : - h->getMaxValueSimple(); + const datatypes::TypeHandler* h = ct.typeHandler(); + return val ? h->toSimpleValue(sp, ct, val, rfMax) : h->getMaxValueSimple(); } - -//convenience fcn +// convenience fcn inline uint32_t tid2sid(const uint32_t tid) { - return CalpontSystemCatalog::idb_tid2sid(tid); + return CalpontSystemCatalog::idb_tid2sid(tid); } -void CHECK( int rc) +void CHECK(int rc) { - if (rc != 0) - { - ostringstream oss; - oss << "Error in DBRM call " << rc << endl; - throw runtime_error(oss.str()); - } + if (rc != 0) + { + ostringstream oss; + oss << "Error in DBRM call " << rc << endl; + throw runtime_error(oss.str()); + } } // partition warnings are delimited by '\n' void push_warnings(THD* thd, string& warnings) { - typedef boost::tokenizer > tokenizer; - boost::char_separator sep("\n"); - tokenizer tokens(warnings, sep); + typedef boost::tokenizer > tokenizer; + boost::char_separator sep("\n"); + tokenizer tokens(warnings, sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) - { - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, (*tok_iter).c_str()); - } + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, (*tok_iter).c_str()); + } } - string name(CalpontSystemCatalog::ColType& ct) { - const datatypes::TypeHandler *h= ct.typeHandler(); - if (!h) - return "Unknown Type"; - return h->print(ct); + const datatypes::TypeHandler* h = ct.typeHandler(); + if (!h) + return "Unknown Type"; + return h->print(ct); } - bool CP_type(CalpontSystemCatalog::ColType& ct) { - const datatypes::TypeHandler *h= ct.typeHandler(); - if (!h) - return false; - return h->CP_type(ct); + const datatypes::TypeHandler* h = ct.typeHandler(); + if (!h) + return false; + return h->CP_type(ct); } - typedef map PartitionMap; - - -void parsePartitionString(UDF_ARGS* args, - int offset, - set& partitionNums, - string& errMsg, +void parsePartitionString(UDF_ARGS* args, int offset, set& partitionNums, string& errMsg, execplan::CalpontSystemCatalog::TableName tableName) { - if (lower_case_table_names) - { - boost::algorithm::to_lower(tableName.schema); - } - - if (tableName.schema == "calpontsys") - { - errMsg = IDBErrorInfo::instance()->errorMsg(SYSTABLE_PARTITION); - return; - } - - partitionNums.clear(); - - string partStr = args->args[offset]; - char* partStrNoSpace = (char*)alloca(partStr.length() + 1); - char* tmp = partStrNoSpace; - - // trim off space - for (uint32_t i = 0; i < partStr.length(); i++) - { - if (partStr[i] == ' ' || partStr[i] == '\t') - continue; - - *tmp = partStr[i]; - ++tmp; - } - - *tmp = 0; - string parts(partStrNoSpace); - - typedef boost::tokenizer > tokenizer; - boost::char_separator sep(","); - boost::char_separator sep1("."); - tokenizer tokens(parts, sep); - stringstream ss (stringstream::in | stringstream::out); - - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) - { - //cout << "token " << (*tok_iter) << endl; - LogicalPartition lp; - - tokenizer tokens1((*tok_iter), sep1); - uint32_t ctn = 0; - - for (tokenizer::iterator tok_iter1 = tokens1.begin(); tok_iter1 != tokens1.end(); ++tok_iter1, ctn++) - { - ss << *tok_iter1; - - switch (ctn) - { - case 0: - if (ss >> lp.pp && ss.eof()) - break; - - goto error; - - case 1: - if (ss >> lp.seg && ss.eof()) - break; - - goto error; - - case 2: - if (ss >> lp.dbroot && ss.eof()) - break; - - goto error; - - default: - goto error; - } - - ss.clear(); - ss.str(""); - } - - partitionNums.insert(lp); - } - - if (partitionNums.empty()) - { - goto error; - } + if (lower_case_table_names) + { + boost::algorithm::to_lower(tableName.schema); + } + if (tableName.schema == "calpontsys") + { + errMsg = IDBErrorInfo::instance()->errorMsg(SYSTABLE_PARTITION); return; + } + + partitionNums.clear(); + + string partStr = args->args[offset]; + char* partStrNoSpace = (char*)alloca(partStr.length() + 1); + char* tmp = partStrNoSpace; + + // trim off space + for (uint32_t i = 0; i < partStr.length(); i++) + { + if (partStr[i] == ' ' || partStr[i] == '\t') + continue; + + *tmp = partStr[i]; + ++tmp; + } + + *tmp = 0; + string parts(partStrNoSpace); + + typedef boost::tokenizer > tokenizer; + boost::char_separator sep(","); + boost::char_separator sep1("."); + tokenizer tokens(parts, sep); + stringstream ss(stringstream::in | stringstream::out); + + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + // cout << "token " << (*tok_iter) << endl; + LogicalPartition lp; + + tokenizer tokens1((*tok_iter), sep1); + uint32_t ctn = 0; + + for (tokenizer::iterator tok_iter1 = tokens1.begin(); tok_iter1 != tokens1.end(); ++tok_iter1, ctn++) + { + ss << *tok_iter1; + + switch (ctn) + { + case 0: + if (ss >> lp.pp && ss.eof()) + break; + + goto error; + + case 1: + if (ss >> lp.seg && ss.eof()) + break; + + goto error; + + case 2: + if (ss >> lp.dbroot && ss.eof()) + break; + + goto error; + + default: goto error; + } + + ss.clear(); + ss.str(""); + } + + partitionNums.insert(lp); + } + + if (partitionNums.empty()) + { + goto error; + } + + return; error: - errMsg = "Invalid partition identifier(s)"; + errMsg = "Invalid partition identifier(s)"; + return; + + // debug + set::const_iterator it; + cout << "Partition: "; + + for (it = partitionNums.begin(); it != partitionNums.end(); ++it) + cout << (*it) << endl; +} + +int processPartition(SqlStatement* stmt) +{ + // cout << "Sending to DDLProc" << endl; + ByteStream bytestream; + bytestream << stmt->fSessionID; + stmt->serialize(bytestream); + MessageQueueClient mq("DDLProc"); + ByteStream::byte b = 0; + int rc = 0; + THD* thd = current_thd; + string emsg; + + try + { + mq.write(bytestream); + bytestream = mq.read(); + + if (bytestream.length() == 0) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); + } + else + { + bytestream >> b; + bytestream >> emsg; + rc = b; + } + } + catch (runtime_error&) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); + } + catch (...) + { + rc = 1; + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown error caught"); + } + + if (b == ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) + { + rc = b; + push_warnings(thd, emsg); + } + else if (b == ddlpackageprocessor::DDLPackageProcessor::PARTITION_WARNING) + { + rc = 0; + push_warnings(thd, emsg); + } + else if (b == ddlpackageprocessor::DDLPackageProcessor::WARNING) + { + rc = 0; + string errmsg( + "Error occured during partitioning operation. Restart DMLProc or use command tool ddlcleanup to " + "clean up. "); + push_warnings(thd, errmsg); + } + else if (b != 0 && b != ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + } + + return rc; +} + +static void addPartition(const CalpontSystemCatalog::ColType& ct, DBRM& em, const BRM::EMEntry& entry, + PartitionMap& partMap, const LogicalPartition& logicalPartNum) +{ + const datatypes::TypeHandler* h = ct.typeHandler(); + int state; + datatypes::MinMaxPartitionInfo partInfo = h->getExtentPartitionInfo(ct, em, entry, &state); + + PartitionMap::iterator mapit = partMap.find(logicalPartNum); + if (mapit == partMap.end()) + { + if (state != CP_VALID) + partInfo.set_invalid(); + + partMap[logicalPartNum] = partInfo; + } + else + { + if (mapit->second.is_invalid()) + return; + + mapit->second.MinMaxInfo::operator=(h->widenMinMaxInfo(ct, mapit->second, partInfo)); + } +} + +void partitionByValue_common(UDF_ARGS* args, // input + string& errMsg, // output + CalpontSystemCatalog::TableName& tableName, // output + set& partSet, // output + string functionName) // input +{ + // identify partitions by the range + BRM::DBRM::refreshShm(); + DBRM em; + vector entries; + vector::iterator iter; + PartitionMap partMap; + string schema, table, column; + + if (args->arg_count == 5) + { + schema = (char*)(args->args[0]); + table = (char*)(args->args[1]); + column = (char*)(args->args[2]); + } + else + { + if (current_thd->db.length) + { + schema = current_thd->db.str; + } + else + { + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NO_SCHEMA); + return; + } + + table = (char*)(args->args[0]); + column = (char*)(args->args[1]); + } + + if (lower_case_table_names) + { + boost::algorithm::to_lower(schema); + boost::algorithm::to_lower(table); + } + boost::algorithm::to_lower(column); + + tableName.schema = schema; + tableName.table = table; + + //@Bug 4695 + if (tableName.schema == "calpontsys") + { + errMsg = IDBErrorInfo::instance()->errorMsg(SYSTABLE_PARTITION); return; + } - // debug - set::const_iterator it; - cout << "Partition: "; + try + { + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(current_thd->thread_id)); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); + csc->identity(CalpontSystemCatalog::FE); + OID_t oid = csc->lookupOID(tcn); + CalpontSystemCatalog::ColType ct = csc->colType(oid); + datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + datatypes::SimpleValue startVal; + datatypes::SimpleValue endVal; + datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; + datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; - for (it = partitionNums.begin(); it != partitionNums.end(); ++it) - cout << (*it) << endl; -} - -int processPartition ( SqlStatement* stmt) -{ - //cout << "Sending to DDLProc" << endl; - ByteStream bytestream; - bytestream << stmt->fSessionID; - stmt->serialize(bytestream); - MessageQueueClient mq("DDLProc"); - ByteStream::byte b = 0; - int rc = 0; - THD* thd = current_thd; - string emsg; - - try + if (oid == -1) { - mq.write(bytestream); - bytestream = mq.read(); - - if ( bytestream.length() == 0 ) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); - } - else - { - bytestream >> b; - bytestream >> emsg; - rc = b; - } - } - catch (runtime_error&) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); - } - catch (...) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown error caught"); + Message::Args args; + args.add("'" + schema + string(".") + table + string(".") + column + "'"); + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_TABLE_NOT_IN_CATALOG, args); + return; } - if (b == ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) + // check casual partition data type + if (!CP_type(ct)) { - rc = b; - push_warnings(thd, emsg); - } - else if (b == ddlpackageprocessor::DDLPackageProcessor::PARTITION_WARNING) - { - rc = 0; - push_warnings(thd, emsg); - } - else if (b == ddlpackageprocessor::DDLPackageProcessor::WARNING) - { - rc = 0; - string errmsg ("Error occured during partitioning operation. Restart DMLProc or use command tool ddlcleanup to clean up. " ); - push_warnings(thd, errmsg); - } - else if (b != 0 && b != ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + Message::Args args; + args.add(name(ct)); + args.add(functionName); + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_BY_RANGE, args); + return; } - return rc; -} - - -static void addPartition(const CalpontSystemCatalog::ColType& ct, - DBRM &em, - const BRM::EMEntry &entry, - PartitionMap &partMap, - const LogicalPartition &logicalPartNum) -{ - const datatypes::TypeHandler *h= ct.typeHandler(); - int state; - datatypes::MinMaxPartitionInfo partInfo= h->getExtentPartitionInfo(ct, em, entry, &state); - - PartitionMap::iterator mapit = partMap.find(logicalPartNum); - if (mapit == partMap.end()) + if (args->arg_count == 4) { - if (state != CP_VALID) - partInfo.set_invalid(); - - partMap[logicalPartNum] = partInfo; + startVal = getStartVal(sp, ct, args->args[2], rfMin); + endVal = getEndVal(sp, ct, args->args[3], rfMax); } else { + startVal = getStartVal(sp, ct, args->args[3], rfMin); + endVal = getEndVal(sp, ct, args->args[4], rfMax); + } + + CHECK(em.getExtents(oid, entries, false, false, true)); + + if (entries.size() > 0) + { + for (iter = entries.begin(); iter != entries.end(); ++iter) + { + LogicalPartition logicalPartNum; + logicalPartNum.dbroot = (*iter).dbRoot; + logicalPartNum.pp = (*iter).partitionNum; + logicalPartNum.seg = (*iter).segmentNum; + addPartition(ct, em, *iter, partMap, logicalPartNum); + } + + // check col value range + for (PartitionMap::iterator mapit = partMap.begin(); mapit != partMap.end(); ++mapit) + { if (mapit->second.is_invalid()) - return; + continue; - mapit->second.MinMaxInfo::operator=(h->widenMinMaxInfo(ct, mapit->second, partInfo)); + const datatypes::TypeHandler* h = ct.typeHandler(); + // @bug 4595. check empty/null case + if (h->isSuitablePartition(ct, mapit->second, startVal, rfMin, endVal, rfMax)) + partSet.insert(mapit->first); + } } + } + catch (QueryDataExcept& ex) + { + Message::Args args; + args.add(ex.what()); + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_FUNC_ARGUMENT, args); + return; + } + catch (IDBExcept& ex) + { + errMsg = ex.what(); + return; + } + catch (...) + { + errMsg = string("Error occured when calling ") + functionName; + return; + } + + if (partSet.empty()) + { + errMsg = IDBErrorInfo::instance()->errorMsg(WARN_NO_PARTITION_FOUND); + return; + } } - -void partitionByValue_common(UDF_ARGS* args, // input - string& errMsg, // output - CalpontSystemCatalog::TableName& tableName, // output - set& partSet, // output - string functionName) // input +std::string ha_mcs_impl_markpartitions_(execplan::CalpontSystemCatalog::TableName tableName, + set& partitionNums) { - // identify partitions by the range - BRM::DBRM::refreshShm(); - DBRM em; - vector entries; - vector::iterator iter; - PartitionMap partMap; - string schema, table, column; + ddlpackage::QualifiedName* qualifiedName = new QualifiedName(); + qualifiedName->fSchema = tableName.schema; + qualifiedName->fName = tableName.table; + MarkPartitionStatement* stmt = new MarkPartitionStatement(qualifiedName); + stmt->fSessionID = tid2sid(current_thd->thread_id); + stmt->fSql = "caldisablepartitions"; + stmt->fOwner = tableName.schema; + stmt->fPartitions = partitionNums; + string msg = "Partitions are disabled successfully"; - if (args->arg_count == 5) - { - schema = (char*)(args->args[0]); - table = (char*)(args->args[1]); - column = (char*)(args->args[2]); - } - else - { - if (current_thd->db.length) - { - schema = current_thd->db.str; - } - else - { - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NO_SCHEMA); - return; - } + int rc = processPartition(stmt); // warnings will be pushed in the function - table = (char*)(args->args[0]); - column = (char*)(args->args[1]); - } + if (rc == ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) + msg = "No partitions are disabled"; - if (lower_case_table_names) - { - boost::algorithm::to_lower(schema); - boost::algorithm::to_lower(table); - } - boost::algorithm::to_lower(column); - - tableName.schema = schema; - tableName.table = table; - - //@Bug 4695 - if (tableName.schema == "calpontsys") - { - errMsg = IDBErrorInfo::instance()->errorMsg(SYSTABLE_PARTITION); - return; - } - - try - { - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(current_thd->thread_id)); - csc->identity(execplan::CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); - csc->identity(CalpontSystemCatalog::FE); - OID_t oid = csc->lookupOID(tcn); - CalpontSystemCatalog::ColType ct = csc->colType(oid); - datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); - datatypes::SimpleValue startVal; - datatypes::SimpleValue endVal; - datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; - datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; - - if (oid == -1) - { - Message::Args args; - args.add("'" + schema + string(".") + table + string(".") + column + "'"); - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_TABLE_NOT_IN_CATALOG, args); - return; - } - - // check casual partition data type - if (!CP_type(ct)) - { - Message::Args args; - args.add(name(ct)); - args.add(functionName); - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_BY_RANGE, args); - return; - } - - if (args->arg_count == 4) - { - startVal = getStartVal(sp, ct, args->args[2], rfMin); - endVal = getEndVal(sp, ct, args->args[3], rfMax); - } - else - { - startVal = getStartVal(sp, ct, args->args[3], rfMin); - endVal = getEndVal(sp, ct, args->args[4], rfMax); - } - - CHECK(em.getExtents(oid, entries, false, false, true)); - - if (entries.size() > 0) - { - - for (iter = entries.begin(); iter != entries.end(); ++iter) - { - LogicalPartition logicalPartNum; - logicalPartNum.dbroot = (*iter).dbRoot; - logicalPartNum.pp = (*iter).partitionNum; - logicalPartNum.seg = (*iter).segmentNum; - addPartition(ct, em, *iter, partMap, logicalPartNum); - } - - // check col value range - for (PartitionMap::iterator mapit = partMap.begin(); mapit != partMap.end(); ++mapit) - { - if (mapit->second.is_invalid()) - continue; - - const datatypes::TypeHandler *h= ct.typeHandler(); - // @bug 4595. check empty/null case - if (h->isSuitablePartition(ct, mapit->second, startVal, rfMin, endVal, rfMax)) - partSet.insert(mapit->first); - } - } - } - catch (QueryDataExcept& ex) - { - Message::Args args; - args.add(ex.what()); - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_FUNC_ARGUMENT, args); - return; - } - catch (IDBExcept& ex) - { - errMsg = ex.what(); - return; - } - catch (...) - { - errMsg = string("Error occured when calling ") + functionName; - return; - } - - if (partSet.empty()) - { - errMsg = IDBErrorInfo::instance()->errorMsg(WARN_NO_PARTITION_FOUND); - return; - } + delete stmt; + return msg; } -std::string ha_mcs_impl_markpartitions_( - execplan::CalpontSystemCatalog::TableName tableName, set& partitionNums) +std::string ha_mcs_impl_restorepartitions_(execplan::CalpontSystemCatalog::TableName tableName, + set& partitionNums) { - ddlpackage::QualifiedName* qualifiedName = new QualifiedName(); - qualifiedName->fSchema = tableName.schema; - qualifiedName->fName = tableName.table; - MarkPartitionStatement* stmt = new MarkPartitionStatement(qualifiedName); - stmt->fSessionID = tid2sid(current_thd->thread_id); - stmt->fSql = "caldisablepartitions"; - stmt->fOwner = tableName.schema; - stmt->fPartitions = partitionNums; - string msg = "Partitions are disabled successfully" ; + ddlpackage::QualifiedName* qualifiedName = new QualifiedName(); + qualifiedName->fSchema = tableName.schema; + qualifiedName->fName = tableName.table; + RestorePartitionStatement* stmt = new RestorePartitionStatement(qualifiedName); + stmt->fSessionID = tid2sid(current_thd->thread_id); + stmt->fSql = "calenablepartitions"; + stmt->fOwner = tableName.schema; + stmt->fPartitions = partitionNums; + string msg; + int rc = processPartition(stmt); - int rc = processPartition( stmt); // warnings will be pushed in the function - - if (rc == ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) - msg = "No partitions are disabled"; - - delete stmt; + if (rc != 0) return msg; + + msg = "Partitions are enabled successfully."; + + delete stmt; + return msg; } -std::string ha_mcs_impl_restorepartitions_( - execplan::CalpontSystemCatalog::TableName tableName, set& partitionNums) +std::string ha_mcs_impl_droppartitions_(execplan::CalpontSystemCatalog::TableName tableName, + set& partitionNums) { - ddlpackage::QualifiedName* qualifiedName = new QualifiedName(); - qualifiedName->fSchema = tableName.schema; - qualifiedName->fName = tableName.table; - RestorePartitionStatement* stmt = new RestorePartitionStatement(qualifiedName); - stmt->fSessionID = tid2sid(current_thd->thread_id); - stmt->fSql = "calenablepartitions"; - stmt->fOwner = tableName.schema; - stmt->fPartitions = partitionNums; - string msg; - int rc = processPartition( stmt); + ddlpackage::QualifiedName* qualifiedName = new QualifiedName(); + qualifiedName->fSchema = tableName.schema; + qualifiedName->fName = tableName.table; + DropPartitionStatement* stmt = new DropPartitionStatement(qualifiedName); + stmt->fSessionID = tid2sid(current_thd->thread_id); + stmt->fSql = "caldroppartitions"; + stmt->fOwner = tableName.schema; + stmt->fPartitions = partitionNums; + string msg = "Partitions are dropped successfully"; - if ( rc != 0 ) - return msg; + int rc = processPartition(stmt); - msg = "Partitions are enabled successfully." ; + if (rc == ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) + msg = "No partitions are dropped"; - delete stmt; - return msg; -} - -std::string ha_mcs_impl_droppartitions_( - execplan::CalpontSystemCatalog::TableName tableName, set& partitionNums) -{ - ddlpackage::QualifiedName* qualifiedName = new QualifiedName(); - qualifiedName->fSchema = tableName.schema; - qualifiedName->fName = tableName.table; - DropPartitionStatement* stmt = new DropPartitionStatement(qualifiedName); - stmt->fSessionID = tid2sid(current_thd->thread_id); - stmt->fSql = "caldroppartitions"; - stmt->fOwner = tableName.schema; - stmt->fPartitions = partitionNums; - string msg = "Partitions are dropped successfully" ; - - int rc = processPartition( stmt); - - if (rc == ddlpackageprocessor::DDLPackageProcessor::WARN_NO_PARTITION) - msg = "No partitions are dropped"; - - delete stmt; - return msg; + delete stmt; + return msg; } extern "C" { - - /** - * CalShowPartitions - */ + /** + * CalShowPartitions + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool calshowpartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calshowpartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count < 2 || args->arg_count > 3 || args->arg_type[0] != STRING_RESULT || + args->arg_type[1] != STRING_RESULT || (args->arg_count == 3 && args->arg_type[2] != STRING_RESULT)) { - if (args->arg_count < 2 || - args->arg_count > 3 || - args->arg_type[0] != STRING_RESULT || - args->arg_type[1] != STRING_RESULT || - (args->arg_count == 3 && args->arg_type[2] != STRING_RESULT)) - { - strcpy(message, "usage: CALSHOWPARTITIONS ([schema], table, column)"); - return 1; - } - - for (uint32_t i = 0; i < args->arg_count; i++) - { - if (!args->args[i]) - { - strcpy(message, "usage: CALSHOWPARTITIONS ([schema], table, column)"); - return 1; - } - } - - return 0; + strcpy(message, "usage: CALSHOWPARTITIONS ([schema], table, column)"); + return 1; } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calshowpartitions_deinit(UDF_INIT* initid) + for (uint32_t i = 0; i < args->arg_count; i++) { - delete initid->ptr; + if (!args->args[i]) + { + strcpy(message, "usage: CALSHOWPARTITIONS ([schema], table, column)"); + return 1; + } } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - BRM::DBRM::refreshShm(); - DBRM em; - vector entries; - vector::iterator iter; - vector::iterator end; - PartitionMap partMap; - string schema, table, column; - CalpontSystemCatalog::ColType ct; - string errMsg; - - try - { - PartitionMap::iterator mapit; - - if (args->arg_count == 3) - { - schema = (char*)(args->args[0]); - table = (char*)(args->args[1]); - column = (char*)(args->args[2]); - } - else - { - if (current_thd->db.length) - { - schema = current_thd->db.str; - } - else - { - throw IDBExcept(ERR_PARTITION_NO_SCHEMA); - } - - table = (char*)(args->args[0]); - column = (char*)(args->args[1]); - } - if (lower_case_table_names) - { - boost::algorithm::to_lower(schema); - boost::algorithm::to_lower(table); - } - boost::algorithm::to_lower(column); - - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(current_thd->thread_id)); - csc->identity(CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); - OID_t oid = csc->lookupOID(tcn); - ct = csc->colType(oid); - - if (oid == -1) - { - Message::Args args; - args.add("'" + schema + string(".") + table + string(".") + column + "'"); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); - } - - CHECK(em.getExtents(oid, entries, false, false, true)); - - if (entries.size() > 0) - { - iter = entries.begin(); - end = entries.end(); - LogicalPartition logicalPartNum; - - for (; iter != end; ++iter) - { - logicalPartNum.dbroot = (*iter).dbRoot; - logicalPartNum.pp = (*iter).partitionNum; - logicalPartNum.seg = (*iter).segmentNum; - addPartition(ct, em, *iter, partMap, logicalPartNum); - } - } - } - catch (IDBExcept& ex) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - return result; - } - catch (...) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, "Error occured when calling CALSHOWPARTITIONS"); - return result; - } - - const datatypes::TypeHandler *h= ct.typeHandler(); - uint8_t valueCharLength= h->PartitionValueCharLength(ct); - ostringstream output; - output.setf(ios::left, ios::adjustfield); - - output << setw(10) << "Part#" - << setw(valueCharLength) << "Min" - << setw(valueCharLength) << "Max" << "Status"; - - PartitionMap::const_iterator partIt; - - for (partIt = partMap.begin(); partIt != partMap.end(); ++partIt) - { - ostringstream oss; - oss << partIt->first; - output << "\n " << setw(10) << oss.str(); - - if (partIt->second.is_invalid()) - { - output << setw(valueCharLength) << "N/A" - << setw(valueCharLength) << "N/A"; - } - else - { - const datatypes::TypeHandler *h= ct.typeHandler(); - output << h->formatPartitionInfo(ct, partIt->second); - } - - if (partIt->second.is_disabled()) - output << "Disabled"; - else - output << "Enabled"; - } - - // use our own buffer to make sure it fits. - initid->ptr = new char[output.str().length() + 1]; - memcpy(initid->ptr, output.str().c_str(), output.str().length()); - *length = output.str().length(); - return initid->ptr; - } - - - /** - * CalDisablePartitions - */ + return 0; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool caldisablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - bool err = false; - - if (args->arg_count < 2 || args->arg_count > 3) - err = true; - else if (args->arg_count == 3 && ((args->arg_type[0] != STRING_RESULT || - args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT))) - err = true; - else if (args->arg_count == 2 && ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT))) - err = true; - - for (uint32_t i = 0; i < args->arg_count; i++) - { - if (!args->args[i]) - { - err = true; - break; - } - } - - if (err) - { - strcpy(message, "\nusage: CALDISABLEPARTITIONS (['schemaName'], 'tableName', 'partitionList')"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; - } + void calshowpartitions_deinit(UDF_INIT* initid) + { + delete initid->ptr; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* caldisablepartitions(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - CalpontSystemCatalog::TableName tableName; - set partitionNums; - string errMsg; - - if ( args->arg_count == 3 ) - { - tableName.schema = args->args[0]; - tableName.table = args->args[1]; - parsePartitionString(args, 2, partitionNums, errMsg, tableName); - } - else - { - tableName.table = args->args[0]; - - if (!current_thd->db.length) - { - errMsg = "No schema name indicated."; - memcpy(result, errMsg.c_str(), errMsg.length()); - *length = errMsg.length(); - return result; - } - - tableName.schema = current_thd->db.str; - parsePartitionString(args, 1, partitionNums, errMsg, tableName); - } - - if (errMsg.empty()) - errMsg = ha_mcs_impl_markpartitions_(tableName, partitionNums ); - - memcpy(result, errMsg.c_str(), errMsg.length()); - *length = errMsg.length(); - return result; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void caldisablepartitions_deinit(UDF_INIT* initid) - { - } - - /** - * CalEnablePartitions - */ - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calenablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - bool err = false; - - if (args->arg_count < 2 || args->arg_count > 3) - err = true; - else if (args->arg_count == 3 && ((args->arg_type[0] != STRING_RESULT || - args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT))) - err = true; - else if (args->arg_count == 2 && ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT))) - err = true; - - for (uint32_t i = 0; i < args->arg_count; i++) - { - if (!args->args[i]) - { - err = true; - break; - } - } - - if (err) - { - strcpy(message, "\nusage: CALENABLEPARTITIONS (['schemaName'], 'tableName', 'partitionList')"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calenablepartitions(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, + const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) - { - CalpontSystemCatalog::TableName tableName; - string errMsg; - set partitionNums; + { + BRM::DBRM::refreshShm(); + DBRM em; + vector entries; + vector::iterator iter; + vector::iterator end; + PartitionMap partMap; + string schema, table, column; + CalpontSystemCatalog::ColType ct; + string errMsg; - if ( args->arg_count == 3 ) + try + { + PartitionMap::iterator mapit; + + if (args->arg_count == 3) + { + schema = (char*)(args->args[0]); + table = (char*)(args->args[1]); + column = (char*)(args->args[2]); + } + else + { + if (current_thd->db.length) { - tableName.schema = args->args[0]; - tableName.table = args->args[1]; - parsePartitionString(args, 2, partitionNums, errMsg, tableName); + schema = current_thd->db.str; } else { - tableName.table = args->args[0]; - - if (!current_thd->db.length) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NO_SCHEMA).c_str()); - return result; - } - - tableName.schema = current_thd->db.str; - parsePartitionString(args, 1, partitionNums, errMsg, tableName); + throw IDBExcept(ERR_PARTITION_NO_SCHEMA); } - if (errMsg.empty()) - errMsg = ha_mcs_impl_restorepartitions_(tableName, partitionNums ); + table = (char*)(args->args[0]); + column = (char*)(args->args[1]); + } + if (lower_case_table_names) + { + boost::algorithm::to_lower(schema); + boost::algorithm::to_lower(table); + } + boost::algorithm::to_lower(column); + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(current_thd->thread_id)); + csc->identity(CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); + OID_t oid = csc->lookupOID(tcn); + ct = csc->colType(oid); + + if (oid == -1) + { + Message::Args args; + args.add("'" + schema + string(".") + table + string(".") + column + "'"); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); + } + + CHECK(em.getExtents(oid, entries, false, false, true)); + + if (entries.size() > 0) + { + iter = entries.begin(); + end = entries.end(); + LogicalPartition logicalPartNum; + + for (; iter != end; ++iter) + { + logicalPartNum.dbroot = (*iter).dbRoot; + logicalPartNum.pp = (*iter).partitionNum; + logicalPartNum.seg = (*iter).segmentNum; + addPartition(ct, em, *iter, partMap, logicalPartNum); + } + } + } + catch (IDBExcept& ex) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + return result; + } + catch (...) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, "Error occured when calling CALSHOWPARTITIONS"); + return result; + } + + const datatypes::TypeHandler* h = ct.typeHandler(); + uint8_t valueCharLength = h->PartitionValueCharLength(ct); + ostringstream output; + output.setf(ios::left, ios::adjustfield); + + output << setw(10) << "Part#" << setw(valueCharLength) << "Min" << setw(valueCharLength) << "Max" + << "Status"; + + PartitionMap::const_iterator partIt; + + for (partIt = partMap.begin(); partIt != partMap.end(); ++partIt) + { + ostringstream oss; + oss << partIt->first; + output << "\n " << setw(10) << oss.str(); + + if (partIt->second.is_invalid()) + { + output << setw(valueCharLength) << "N/A" << setw(valueCharLength) << "N/A"; + } + else + { + const datatypes::TypeHandler* h = ct.typeHandler(); + output << h->formatPartitionInfo(ct, partIt->second); + } + + if (partIt->second.is_disabled()) + output << "Disabled"; + else + output << "Enabled"; + } + + // use our own buffer to make sure it fits. + initid->ptr = new char[output.str().length() + 1]; + memcpy(initid->ptr, output.str().c_str(), output.str().length()); + *length = output.str().length(); + return initid->ptr; + } + + /** + * CalDisablePartitions + */ + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool caldisablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 2 || args->arg_count > 3) + err = true; + else if (args->arg_count == 3 && + ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT))) + err = true; + else if (args->arg_count == 2 && + ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT))) + err = true; + + for (uint32_t i = 0; i < args->arg_count; i++) + { + if (!args->args[i]) + { + err = true; + break; + } + } + + if (err) + { + strcpy(message, "\nusage: CALDISABLEPARTITIONS (['schemaName'], 'tableName', 'partitionList')"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* caldisablepartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + CalpontSystemCatalog::TableName tableName; + set partitionNums; + string errMsg; + + if (args->arg_count == 3) + { + tableName.schema = args->args[0]; + tableName.table = args->args[1]; + parsePartitionString(args, 2, partitionNums, errMsg, tableName); + } + else + { + tableName.table = args->args[0]; + + if (!current_thd->db.length) + { + errMsg = "No schema name indicated."; memcpy(result, errMsg.c_str(), errMsg.length()); *length = errMsg.length(); return result; + } + + tableName.schema = current_thd->db.str; + parsePartitionString(args, 1, partitionNums, errMsg, tableName); } + if (errMsg.empty()) + errMsg = ha_mcs_impl_markpartitions_(tableName, partitionNums); + + memcpy(result, errMsg.c_str(), errMsg.length()); + *length = errMsg.length(); + return result; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void calenablepartitions_deinit(UDF_INIT* initid) + void caldisablepartitions_deinit(UDF_INIT* initid) + { + } + + /** + * CalEnablePartitions + */ + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calenablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 2 || args->arg_count > 3) + err = true; + else if (args->arg_count == 3 && + ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT))) + err = true; + else if (args->arg_count == 2 && + ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT))) + err = true; + + for (uint32_t i = 0; i < args->arg_count; i++) { + if (!args->args[i]) + { + err = true; + break; + } } - /** - * CalDropPartitions - */ - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool caldroppartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + if (err) { - bool err = false; - - if (args->arg_count < 2 || args->arg_count > 3) - err = true; - else if (args->arg_count == 3 && ((args->arg_type[0] != STRING_RESULT || - args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT))) - err = true; - else if (args->arg_count == 2 && ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT))) - err = true; - - for (uint32_t i = 0; i < args->arg_count; i++) - { - if (!args->args[i]) - { - err = true; - break; - } - } - - if (err) - { - strcpy(message, "\nusage: CALDROPPARTITIONS (['schemaName'], 'tableName', 'partitionList')"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; + strcpy(message, "\nusage: CALENABLEPARTITIONS (['schemaName'], 'tableName', 'partitionList')"); + return 1; } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* caldroppartitions(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - CalpontSystemCatalog::TableName tableName; - string errMsg; - set partSet; + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } - if ( args->arg_count == 3 ) +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calenablepartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + CalpontSystemCatalog::TableName tableName; + string errMsg; + set partitionNums; + + if (args->arg_count == 3) + { + tableName.schema = args->args[0]; + tableName.table = args->args[1]; + parsePartitionString(args, 2, partitionNums, errMsg, tableName); + } + else + { + tableName.table = args->args[0]; + + if (!current_thd->db.length) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, + IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NO_SCHEMA).c_str()); + return result; + } + + tableName.schema = current_thd->db.str; + parsePartitionString(args, 1, partitionNums, errMsg, tableName); + } + + if (errMsg.empty()) + errMsg = ha_mcs_impl_restorepartitions_(tableName, partitionNums); + + memcpy(result, errMsg.c_str(), errMsg.length()); + *length = errMsg.length(); + return result; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calenablepartitions_deinit(UDF_INIT* initid) + { + } + + /** + * CalDropPartitions + */ + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool caldroppartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 2 || args->arg_count > 3) + err = true; + else if (args->arg_count == 3 && + ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT))) + err = true; + else if (args->arg_count == 2 && + ((args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT))) + err = true; + + for (uint32_t i = 0; i < args->arg_count; i++) + { + if (!args->args[i]) + { + err = true; + break; + } + } + + if (err) + { + strcpy(message, "\nusage: CALDROPPARTITIONS (['schemaName'], 'tableName', 'partitionList')"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* caldroppartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + CalpontSystemCatalog::TableName tableName; + string errMsg; + set partSet; + + if (args->arg_count == 3) + { + tableName.schema = args->args[0]; + tableName.table = args->args[1]; + parsePartitionString(args, 2, partSet, errMsg, tableName); + } + else + { + tableName.table = args->args[0]; + + if (!current_thd->db.length) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, + IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NO_SCHEMA).c_str()); + return result; + } + + tableName.schema = current_thd->db.str; + parsePartitionString(args, 1, partSet, errMsg, tableName); + } + + if (errMsg.empty()) + errMsg = ha_mcs_impl_droppartitions_(tableName, partSet); + + memcpy(result, errMsg.c_str(), errMsg.length()); + *length = errMsg.length(); + return result; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void caldroppartitions_deinit(UDF_INIT* initid) + { + } + + /** + * CalDropPartitionsByValue + */ + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void caldroppartitionsbyvalue_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool caldroppartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 4 || args->arg_count > 5) + { + err = true; + } + else if (args->arg_count == 4) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT) + err = true; + } + else if (args->arg_count == 5) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || + args->arg_type[4] != STRING_RESULT) + err = true; + } + + if (err) + { + string msg = "\nusage: CALDROPPARTITIONSBYVALUE (['schema'], 'table', 'column', 'min', 'max')"; + // message = new char[msg.length()+1]; + strcpy(message, msg.c_str()); + message[msg.length()] = 0; + //*length = msg.length(); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* caldroppartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long* length, char* is_null, char* error) + { + string msg; + CalpontSystemCatalog::TableName tableName; + set partSet; + partitionByValue_common(args, msg, tableName, partSet, "calDropPartitionsByValue"); + + if (!msg.empty()) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, msg.c_str()); + return result; + } + + msg = ha_mcs_impl_droppartitions_(tableName, partSet); + + memcpy(result, msg.c_str(), msg.length()); + *length = msg.length(); + return result; + } + + /** + * CalDisablePartitionsByValue + */ + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void caldisablepartitionsbyvalue_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool caldisablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 4 || args->arg_count > 5) + { + err = true; + } + else if (args->arg_count == 4) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT) + err = true; + } + else if (args->arg_count == 5) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || + args->arg_type[4] != STRING_RESULT) + err = true; + } + + if (err) + { + strcpy(message, "\nusage: CALDISABLEPARTITIONS (['schema'], 'table', 'column', 'min', 'max')"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* caldisablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long* length, char* is_null, char* error) + { + string msg; + set partSet; + CalpontSystemCatalog::TableName tableName; + partitionByValue_common(args, msg, tableName, partSet, "calDisablePartitionsByValue"); + + if (!msg.empty()) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, msg.c_str()); + return result; + } + + msg = ha_mcs_impl_markpartitions_(tableName, partSet); + + memcpy(result, msg.c_str(), msg.length()); + *length = msg.length(); + return result; + } + + /** + * CalEnablePartitionsByValue + */ +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calenablepartitionsbyvalue_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calenablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 4 || args->arg_count > 5) + { + err = true; + } + else if (args->arg_count == 4) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT) + err = true; + } + else if (args->arg_count == 5) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || + args->arg_type[4] != STRING_RESULT) + err = true; + } + + if (err) + { + strcpy(message, "\nusage: CALENABLEPARTITIONSBYVALUE (['schema'], 'table', 'column', 'min', 'max')"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calenablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long* length, char* is_null, char* error) + { + string msg; + set partSet; + CalpontSystemCatalog::TableName tableName; + partitionByValue_common(args, msg, tableName, partSet, "calEnablePartitionsByValue"); + + if (!msg.empty()) + { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, msg.c_str()); + return result; + } + + msg = ha_mcs_impl_restorepartitions_(tableName, partSet); + + memcpy(result, msg.c_str(), msg.length()); + *length = msg.length(); + return result; + } + + /** + * CalShowPartitionsByValue + */ +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool calshowpartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + bool err = false; + + if (args->arg_count < 4 || args->arg_count > 5) + { + err = true; + } + else if (args->arg_count == 4) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT) + err = true; + } + else if (args->arg_count == 5) + { + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || + args->arg_type[4] != STRING_RESULT) + err = true; + } + + if (err) + { + strcpy(message, "\nusage: CALSHOWPARTITIONSBYVALUE (['schema'], 'table', 'column', 'min', 'max')"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calshowpartitionsbyvalue_deinit(UDF_INIT* initid) + { + delete initid->ptr; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long* length, char* is_null, char* error) + { + BRM::DBRM::refreshShm(); + DBRM em; + vector entries; + vector::iterator iter; + vector::iterator end; + PartitionMap partMap; + PartitionMap::iterator mapit; + string schema, table, column; + CalpontSystemCatalog::ColType ct; + string errMsg; + datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + datatypes::SimpleValue startVal; + datatypes::SimpleValue endVal; + datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; + datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; + + try + { + if (args->arg_count == 5) + { + schema = (char*)(args->args[0]); + table = (char*)(args->args[1]); + column = (char*)(args->args[2]); + } + else + { + if (current_thd->db.length) { - tableName.schema = args->args[0]; - tableName.table = args->args[1]; - parsePartitionString(args, 2, partSet, errMsg, tableName); + schema = current_thd->db.str; } else { - tableName.table = args->args[0]; - - if (!current_thd->db.length) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NO_SCHEMA).c_str()); - return result; - } - - tableName.schema = current_thd->db.str; - parsePartitionString(args, 1, partSet, errMsg, tableName); + throw IDBExcept(ERR_PARTITION_NO_SCHEMA); } - if (errMsg.empty()) - errMsg = ha_mcs_impl_droppartitions_(tableName, partSet); + table = (char*)(args->args[0]); + column = (char*)(args->args[1]); + } - memcpy(result, errMsg.c_str(), errMsg.length()); - *length = errMsg.length(); - return result; + if (lower_case_table_names) + { + boost::algorithm::to_lower(schema); + boost::algorithm::to_lower(table); + } + boost::algorithm::to_lower(column); + + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(current_thd->thread_id)); + csc->identity(CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); + OID_t oid = csc->lookupOID(tcn); + ct = csc->colType(oid); + + if (oid == -1) + { + Message::Args args; + args.add("'" + schema + string(".") + table + string(".") + column + "'"); + throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); + } + + // check casual partition data type + if (!CP_type(ct)) + { + Message::Args args; + args.add(name(ct)); + args.add("calShowPartitionsByValue"); + throw IDBExcept(ERR_PARTITION_BY_RANGE, args); + } + + if (args->arg_count == 4) + { + startVal = getStartVal(sp, ct, args->args[2], rfMin); + endVal = getEndVal(sp, ct, args->args[3], rfMax); + } + else + { + startVal = getStartVal(sp, ct, args->args[3], rfMin); + endVal = getEndVal(sp, ct, args->args[4], rfMax); + } + + CHECK(em.getExtents(oid, entries, false, false, true)); + + if (entries.size() > 0) + { + iter = entries.begin(); + end = entries.end(); + LogicalPartition logicalPartNum; + + for (; iter != end; ++iter) + { + logicalPartNum.dbroot = (*iter).dbRoot; + logicalPartNum.pp = (*iter).partitionNum; + logicalPartNum.seg = (*iter).segmentNum; + addPartition(ct, em, *iter, partMap, logicalPartNum); + } + } } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void caldroppartitions_deinit(UDF_INIT* initid) + catch (logging::QueryDataExcept& ex) { + Message::Args args; + args.add(ex.what()); + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_FUNC_ARGUMENT, args); + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, (char*)errMsg.c_str()); + return result; } - - /** - * CalDropPartitionsByValue - */ - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void caldroppartitionsbyvalue_deinit(UDF_INIT* initid) + catch (IDBExcept& ex) { + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); + return result; } - - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool caldroppartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + catch (...) { - bool err = false; - - if (args->arg_count < 4 || args->arg_count > 5) - { - err = true; - } - else if (args->arg_count == 4) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || args->arg_type[2] != STRING_RESULT) - err = true; - } - else if (args->arg_count == 5) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || args->arg_type[4] != STRING_RESULT) - err = true; - } - - if (err) - { - string msg = "\nusage: CALDROPPARTITIONSBYVALUE (['schema'], 'table', 'column', 'min', 'max')"; - //message = new char[msg.length()+1]; - strcpy(message, msg.c_str()); - message[msg.length()] = 0; - //*length = msg.length(); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, "Error occured when calling CALSHOWPARTITIONS"); + return result; } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* caldroppartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) + // create a set of partInfo for sorting. + bool noPartFound = true; + ostringstream output; + + for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { - string msg; - CalpontSystemCatalog::TableName tableName; - set partSet; - partitionByValue_common(args, msg, tableName, partSet, "calDropPartitionsByValue"); + const datatypes::TypeHandler* h = ct.typeHandler(); + uint8_t valueCharLength = h->PartitionValueCharLength(ct); + if (mapit->second.is_invalid()) + { + output << setw(valueCharLength) << "N/A" << setw(valueCharLength) << "N/A"; + continue; + } + // @bug 4595. check empty/null case + string tmp = h->PrintPartitionValue(ct, mapit->second, startVal, rfMin, endVal, rfMax); + if (tmp == "") + continue; - if (!msg.empty()) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, msg.c_str()); - return result; - } + // print header + if (noPartFound) + { + output.setf(ios::left, ios::adjustfield); + output << setw(10) << "Part#" << setw(valueCharLength) << "Min" << setw(valueCharLength) << "Max" + << "Status"; + noPartFound = false; + } - msg = ha_mcs_impl_droppartitions_(tableName, partSet); + // print part info + ostringstream oss; + oss << mapit->first; + output << "\n " << setw(10) << oss.str() << tmp; - memcpy(result, msg.c_str(), msg.length()); - *length = msg.length(); - return result; + if (mapit->second.is_disabled()) + output << "Disabled"; + else + output << "Enabled"; } - /** - * CalDisablePartitionsByValue - */ - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void caldisablepartitionsbyvalue_deinit(UDF_INIT* initid) + if (noPartFound) { + errMsg = IDBErrorInfo::instance()->errorMsg(WARN_NO_PARTITION_FOUND); + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, errMsg.c_str()); + return result; } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool caldisablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - bool err = false; - - if (args->arg_count < 4 || args->arg_count > 5) - { - err = true; - } - else if (args->arg_count == 4) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || args->arg_type[2] != STRING_RESULT) - err = true; - } - else if (args->arg_count == 5) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || args->arg_type[4] != STRING_RESULT) - err = true; - } - - if (err) - { - strcpy(message, "\nusage: CALDISABLEPARTITIONS (['schema'], 'table', 'column', 'min', 'max')"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* caldisablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - string msg; - set partSet; - CalpontSystemCatalog::TableName tableName; - partitionByValue_common(args, msg, tableName, partSet, "calDisablePartitionsByValue"); - - if (!msg.empty()) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, msg.c_str()); - return result; - } - - msg = ha_mcs_impl_markpartitions_(tableName, partSet); - - memcpy(result, msg.c_str(), msg.length()); - *length = msg.length(); - return result; - } - - - /** - * CalEnablePartitionsByValue - */ -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calenablepartitionsbyvalue_deinit(UDF_INIT* initid) - { - } - - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calenablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - bool err = false; - - if (args->arg_count < 4 || args->arg_count > 5) - { - err = true; - } - else if (args->arg_count == 4) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || args->arg_type[2] != STRING_RESULT) - err = true; - } - else if (args->arg_count == 5) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || args->arg_type[4] != STRING_RESULT) - err = true; - } - - if (err) - { - strcpy(message, "\nusage: CALENABLEPARTITIONSBYVALUE (['schema'], 'table', 'column', 'min', 'max')"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calenablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - string msg; - set partSet; - CalpontSystemCatalog::TableName tableName; - partitionByValue_common(args, msg, tableName, partSet, "calEnablePartitionsByValue"); - - if (!msg.empty()) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, msg.c_str()); - return result; - } - - msg = ha_mcs_impl_restorepartitions_(tableName, partSet); - - memcpy(result, msg.c_str(), msg.length()); - *length = msg.length(); - return result; - } - - /** - * CalShowPartitionsByValue - */ -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool calshowpartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - bool err = false; - - if (args->arg_count < 4 || args->arg_count > 5) - { - err = true; - } - else if (args->arg_count == 4) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || args->arg_type[2] != STRING_RESULT) - err = true; - } - else if (args->arg_count == 5) - { - if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || - args->arg_type[2] != STRING_RESULT || args->arg_type[3] != STRING_RESULT || args->arg_type[4] != STRING_RESULT) - err = true; - } - - if (err) - { - strcpy(message, "\nusage: CALSHOWPARTITIONSBYVALUE (['schema'], 'table', 'column', 'min', 'max')"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void calshowpartitionsbyvalue_deinit(UDF_INIT* initid) - { - delete initid->ptr; - } - - - - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - BRM::DBRM::refreshShm(); - DBRM em; - vector entries; - vector::iterator iter; - vector::iterator end; - PartitionMap partMap; - PartitionMap::iterator mapit; - string schema, table, column; - CalpontSystemCatalog::ColType ct; - string errMsg; - datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); - datatypes::SimpleValue startVal; - datatypes::SimpleValue endVal; - datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; - datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; - - try - { - if (args->arg_count == 5) - { - schema = (char*)(args->args[0]); - table = (char*)(args->args[1]); - column = (char*)(args->args[2]); - } - else - { - if (current_thd->db.length) - { - schema = current_thd->db.str; - } - else - { - throw IDBExcept(ERR_PARTITION_NO_SCHEMA); - } - - table = (char*)(args->args[0]); - column = (char*)(args->args[1]); - } - - if (lower_case_table_names) - { - boost::algorithm::to_lower(schema); - boost::algorithm::to_lower(table); - } - boost::algorithm::to_lower(column); - - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(tid2sid(current_thd->thread_id)); - csc->identity(CalpontSystemCatalog::FE); - CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); - OID_t oid = csc->lookupOID(tcn); - ct = csc->colType(oid); - - if (oid == -1) - { - Message::Args args; - args.add("'" + schema + string(".") + table + string(".") + column + "'"); - throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args); - } - - // check casual partition data type - if (!CP_type(ct)) - { - Message::Args args; - args.add(name(ct)); - args.add("calShowPartitionsByValue"); - throw IDBExcept(ERR_PARTITION_BY_RANGE, args); - } - - if (args->arg_count == 4) - { - startVal= getStartVal(sp, ct, args->args[2], rfMin); - endVal= getEndVal(sp, ct, args->args[3], rfMax); - } - else - { - startVal= getStartVal(sp, ct, args->args[3], rfMin); - endVal= getEndVal(sp, ct, args->args[4], rfMax); - } - - CHECK(em.getExtents(oid, entries, false, false, true)); - - if (entries.size() > 0) - { - iter = entries.begin(); - end = entries.end(); - LogicalPartition logicalPartNum; - - for (; iter != end; ++iter) - { - logicalPartNum.dbroot = (*iter).dbRoot; - logicalPartNum.pp = (*iter).partitionNum; - logicalPartNum.seg = (*iter).segmentNum; - addPartition(ct, em, *iter, partMap, logicalPartNum); - } - } - } - catch (logging::QueryDataExcept& ex) - { - Message::Args args; - args.add(ex.what()); - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_FUNC_ARGUMENT, args); - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, (char*)errMsg.c_str()); - return result; - } - catch (IDBExcept& ex) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, ex.what()); - return result; - } - catch (...) - { - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, "Error occured when calling CALSHOWPARTITIONS"); - return result; - } - - // create a set of partInfo for sorting. - bool noPartFound = true; - ostringstream output; - - for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) - { - const datatypes::TypeHandler *h= ct.typeHandler(); - uint8_t valueCharLength= h->PartitionValueCharLength(ct); - if (mapit->second.is_invalid()) - { - output << setw(valueCharLength) << "N/A" - << setw(valueCharLength) << "N/A"; - continue; - } - // @bug 4595. check empty/null case - string tmp= h->PrintPartitionValue(ct, mapit->second, - startVal, rfMin, - endVal, rfMax); - if (tmp == "") - continue; - - // print header - if (noPartFound) - { - output.setf(ios::left, ios::adjustfield); - output << setw(10) << "Part#" - << setw(valueCharLength) << "Min" - << setw(valueCharLength) << "Max" << "Status"; - noPartFound = false; - } - - // print part info - ostringstream oss; - oss << mapit->first; - output << "\n " << setw(10) << oss.str() << tmp; - - if (mapit->second.is_disabled()) - output << "Disabled"; - else - output << "Enabled"; - - } - - if (noPartFound) - { - errMsg = IDBErrorInfo::instance()->errorMsg(WARN_NO_PARTITION_FOUND); - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, errMsg.c_str()); - return result; - } - - // use our own buffer to make sure it fits. - initid->ptr = new char[output.str().length() + 1]; - memcpy(initid->ptr, output.str().c_str(), output.str().length()); - *length = output.str().length(); - return initid->ptr; - } - + // use our own buffer to make sure it fits. + initid->ptr = new char[output.str().length() + 1]; + memcpy(initid->ptr, output.str().c_str(), output.str().length()); + *length = output.str().length(); + return initid->ptr; + } } -} +} // namespace diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index c4bac1fe7..cefe23986 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -23,344 +23,341 @@ void check_walk(const Item* item, void* arg); void restore_query_state(ha_columnstore_select_handler* handler) { - for (auto iter = handler->tableOuterJoinMap.begin(); - iter != handler->tableOuterJoinMap.end(); iter++) - { - iter->first->outer_join = iter->second; - } + for (auto iter = handler->tableOuterJoinMap.begin(); iter != handler->tableOuterJoinMap.end(); iter++) + { + iter->first->outer_join = iter->second; + } } -void mutate_optimizer_flags(THD *thd_) +void mutate_optimizer_flags(THD* thd_) { - // MCOL-2178 Disable all optimizer flags as it was in the fork. - // CS restores it later in SH::scan_end() and in case of an error - // in SH::scan_init() + // MCOL-2178 Disable all optimizer flags as it was in the fork. + // CS restores it later in SH::scan_end() and in case of an error + // in SH::scan_init() - ulonglong flags_to_set = OPTIMIZER_SWITCH_MATERIALIZATION | - OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_DERIVED | - OPTIMIZER_SWITCH_COND_PUSHDOWN_FROM_HAVING; + ulonglong flags_to_set = OPTIMIZER_SWITCH_MATERIALIZATION | OPTIMIZER_SWITCH_COND_PUSHDOWN_FOR_DERIVED | + OPTIMIZER_SWITCH_COND_PUSHDOWN_FROM_HAVING; - if (thd_->variables.optimizer_switch == flags_to_set) - return; + if (thd_->variables.optimizer_switch == flags_to_set) + return; - set_original_optimizer_flags(thd_->variables.optimizer_switch, thd_); - thd_->variables.optimizer_switch = flags_to_set; + set_original_optimizer_flags(thd_->variables.optimizer_switch, thd_); + thd_->variables.optimizer_switch = flags_to_set; } -void restore_optimizer_flags(THD *thd_) +void restore_optimizer_flags(THD* thd_) { - // MCOL-2178 restore original optimizer flags after SH, DH - ulonglong orig_flags = get_original_optimizer_flags(thd_); - if (orig_flags) - { - thd_->variables.optimizer_switch = orig_flags; - set_original_optimizer_flags(0, thd_); - } + // MCOL-2178 restore original optimizer flags after SH, DH + ulonglong orig_flags = get_original_optimizer_flags(thd_); + if (orig_flags) + { + thd_->variables.optimizer_switch = orig_flags; + set_original_optimizer_flags(0, thd_); + } } /*@brief find_tables - This traverses Item */ /********************************************************** -* DESCRIPTION: -* This f() pushes TABLE of an Item_field to a list -* provided. The list is used for JOIN predicate check in -* is_joinkeys_predicate(). -* PARAMETERS: -* Item * Item to check -* RETURN: -***********************************************************/ + * DESCRIPTION: + * This f() pushes TABLE of an Item_field to a list + * provided. The list is used for JOIN predicate check in + * is_joinkeys_predicate(). + * PARAMETERS: + * Item * Item to check + * RETURN: + ***********************************************************/ void find_tables(const Item* item, void* arg) { - if (typeid(*item) == typeid(Item_field)) - { - Item_field *ifp = (Item_field*)item; - List *tables_list = (List
*)arg; - tables_list->push_back(ifp->field->table); - } + if (typeid(*item) == typeid(Item_field)) + { + Item_field* ifp = (Item_field*)item; + List
* tables_list = (List
*)arg; + tables_list->push_back(ifp->field->table); + } } /*@brief is_joinkeys_predicate - This traverses Item_func*/ /*********************************************************** -* DESCRIPTION: -* This f() walks Item_func and checks whether it contains -* JOIN predicate -* PARAMETERS: -* Item_func * Item to walk -* RETURN: -* BOOL false if Item_func isn't a JOIN predicate -* BOOL true otherwise -***********************************************************/ -bool is_joinkeys_predicate(const Item_func *ifp) + * DESCRIPTION: + * This f() walks Item_func and checks whether it contains + * JOIN predicate + * PARAMETERS: + * Item_func * Item to walk + * RETURN: + * BOOL false if Item_func isn't a JOIN predicate + * BOOL true otherwise + ***********************************************************/ +bool is_joinkeys_predicate(const Item_func* ifp) { - bool result = false; - if(ifp->argument_count() == 2) + bool result = false; + if (ifp->argument_count() == 2) + { + if (ifp->arguments()[0]->type() == Item::FIELD_ITEM && ifp->arguments()[1]->type() == Item::FIELD_ITEM) { - if (ifp->arguments()[0]->type() == Item::FIELD_ITEM && - ifp->arguments()[1]->type() == Item::FIELD_ITEM) - { - Item_field* left = reinterpret_cast(ifp->arguments()[0]); - Item_field* right = reinterpret_cast(ifp->arguments()[1]); + Item_field* left = reinterpret_cast(ifp->arguments()[0]); + Item_field* right = reinterpret_cast(ifp->arguments()[1]); - // If MDB crashes here with non-fixed Item_field and field == NULL - // there must be a check over on_expr for a different SELECT_LEX. - // e.g. checking subquery with ON from upper query. - if (left->field->table != right->field->table) - { - result= true; - } - } - else - { - List
llt; List
rlt; - Item *left = ifp->arguments()[0]; - Item *right = ifp->arguments()[1]; - // Search for tables inside left and right expressions - // and compare them - left->traverse_cond(find_tables, (void*)&llt, Item::POSTFIX); - right->traverse_cond(find_tables, (void*)&rlt, Item::POSTFIX); - // TODO Find the way to have more then one element or prove - // the idea is useless. - if (llt.elements && rlt.elements && (llt.elem(0) != rlt.elem(0))) - { - result = true; - } - } + // If MDB crashes here with non-fixed Item_field and field == NULL + // there must be a check over on_expr for a different SELECT_LEX. + // e.g. checking subquery with ON from upper query. + if (left->field->table != right->field->table) + { + result = true; + } } - return result; + else + { + List
llt; + List
rlt; + Item* left = ifp->arguments()[0]; + Item* right = ifp->arguments()[1]; + // Search for tables inside left and right expressions + // and compare them + left->traverse_cond(find_tables, (void*)&llt, Item::POSTFIX); + right->traverse_cond(find_tables, (void*)&rlt, Item::POSTFIX); + // TODO Find the way to have more then one element or prove + // the idea is useless. + if (llt.elements && rlt.elements && (llt.elem(0) != rlt.elem(0))) + { + result = true; + } + } + } + return result; } /*@brief find_nonequi_join - This traverses Item */ /************************************************************ -* DESCRIPTION: -* This f() walks Item and looks for a non-equi join -* predicates -* PARAMETERS: -* Item * Item to walk -* RETURN: -***********************************************************/ -void find_nonequi_join(const Item* item, void *arg) + * DESCRIPTION: + * This f() walks Item and looks for a non-equi join + * predicates + * PARAMETERS: + * Item * Item to walk + * RETURN: + ***********************************************************/ +void find_nonequi_join(const Item* item, void* arg) { - bool *unsupported_feature = reinterpret_cast(arg); - if ( *unsupported_feature ) - return; + bool* unsupported_feature = reinterpret_cast(arg); + if (*unsupported_feature) + return; - if (item->type() == Item::FUNC_ITEM) + if (item->type() == Item::FUNC_ITEM) + { + const Item_func* ifp = reinterpret_cast(item); + // TODO Check for IN + // NOT IN + correlated subquery + if (ifp->functype() != Item_func::EQ_FUNC) { - const Item_func* ifp = reinterpret_cast(item); - //TODO Check for IN - //NOT IN + correlated subquery - if (ifp->functype() != Item_func::EQ_FUNC) - { - if (is_joinkeys_predicate(ifp)) - *unsupported_feature = true; - else if (ifp->functype() == Item_func::NOT_FUNC - && ifp->arguments()[0]->type() == Item::EXPR_CACHE_ITEM) - { - check_walk(ifp->arguments()[0], arg); - } - } + if (is_joinkeys_predicate(ifp)) + *unsupported_feature = true; + else if (ifp->functype() == Item_func::NOT_FUNC && ifp->arguments()[0]->type() == Item::EXPR_CACHE_ITEM) + { + check_walk(ifp->arguments()[0], arg); + } } + } } /*@brief find_join - This traverses Item */ /************************************************************ -* DESCRIPTION: -* This f() walks traverses Item looking for JOIN, SEMI-JOIN -* predicates. -* PARAMETERS: -* Item * Item to traverse -* RETURN: -***********************************************************/ + * DESCRIPTION: + * This f() walks traverses Item looking for JOIN, SEMI-JOIN + * predicates. + * PARAMETERS: + * Item * Item to traverse + * RETURN: + ***********************************************************/ void find_join(const Item* item, void* arg) { - bool *unsupported_feature = reinterpret_cast(arg); - if ( *unsupported_feature ) - return; + bool* unsupported_feature = reinterpret_cast(arg); + if (*unsupported_feature) + return; - if (item->type() == Item::FUNC_ITEM) + if (item->type() == Item::FUNC_ITEM) + { + const Item_func* ifp = reinterpret_cast(item); + // TODO Check for IN + // NOT IN + correlated subquery { - const Item_func* ifp = reinterpret_cast(item); - //TODO Check for IN - //NOT IN + correlated subquery - { - if (is_joinkeys_predicate(ifp)) - *unsupported_feature = true; - else if (ifp->functype() == Item_func::NOT_FUNC - && ifp->arguments()[0]->type() == Item::EXPR_CACHE_ITEM) - { - check_walk(ifp->arguments()[0], arg); - } - } + if (is_joinkeys_predicate(ifp)) + *unsupported_feature = true; + else if (ifp->functype() == Item_func::NOT_FUNC && ifp->arguments()[0]->type() == Item::EXPR_CACHE_ITEM) + { + check_walk(ifp->arguments()[0], arg); + } } + } } /*@brief save_join_predicate - This traverses Item */ /************************************************************ -* DESCRIPTION: -* This f() walks Item and saves found JOIN predicates into -* a List. The list will be used for a simple CROSS JOIN -* check in create_DH. -* PARAMETERS: -* Item * Item to walk -* RETURN: -***********************************************************/ + * DESCRIPTION: + * This f() walks Item and saves found JOIN predicates into + * a List. The list will be used for a simple CROSS JOIN + * check in create_DH. + * PARAMETERS: + * Item * Item to walk + * RETURN: + ***********************************************************/ void save_join_predicates(const Item* item, void* arg) { - if (item->type() == Item::FUNC_ITEM) + if (item->type() == Item::FUNC_ITEM) + { + const Item_func* ifp = reinterpret_cast(item); + if (is_joinkeys_predicate(ifp)) { - const Item_func* ifp = reinterpret_cast(item); - if (is_joinkeys_predicate(ifp)) - { - List *join_preds_list = (List*)arg; - join_preds_list->push_back(const_cast(item)); - } + List* join_preds_list = (List*)arg; + join_preds_list->push_back(const_cast(item)); } + } } /*@brief check_walk - It traverses filter conditions */ /************************************************************ -* DESCRIPTION: -* It traverses filter predicates looking for unsupported -* JOIN types: non-equi JOIN, e.g t1.c1 > t2.c2; -* logical OR. -* PARAMETERS: -* thd - THD pointer. -* derived - TABLE_LIST* to work with. -* RETURN: -***********************************************************/ + * DESCRIPTION: + * It traverses filter predicates looking for unsupported + * JOIN types: non-equi JOIN, e.g t1.c1 > t2.c2; + * logical OR. + * PARAMETERS: + * thd - THD pointer. + * derived - TABLE_LIST* to work with. + * RETURN: + ***********************************************************/ void check_walk(const Item* item, void* arg) { - bool *unsupported_feature = reinterpret_cast(arg); - if ( *unsupported_feature ) - return; + bool* unsupported_feature = reinterpret_cast(arg); + if (*unsupported_feature) + return; - switch (item->type()) + switch (item->type()) + { + case Item::FUNC_ITEM: { - case Item::FUNC_ITEM: - { - find_nonequi_join(item, arg); - break; - } - case Item::EXPR_CACHE_ITEM: // IN + correlated subquery - { - const Item_cache_wrapper* icw = reinterpret_cast(item); - if ( icw->get_orig_item()->type() == Item::FUNC_ITEM ) - { - const Item_func *ifp = reinterpret_cast(icw->get_orig_item()); - if ( ifp->argument_count() == 2 && - ( ifp->arguments()[0]->type() == Item::Item::SUBSELECT_ITEM - || ifp->arguments()[1]->type() == Item::Item::SUBSELECT_ITEM )) - { - *unsupported_feature = true; - return; - } - } - break; - } - case Item::COND_ITEM: // OR contains JOIN conds thats is unsupported yet - { - Item_cond* icp = (Item_cond*)item; - if ( is_cond_or(icp) ) - { - bool left_flag= false, right_flag= false; - if (icp->argument_list()->elements >= 2) - { - Item *left; Item *right; - List_iterator li(*icp->argument_list()); - left = li++; right = li++; - left->traverse_cond(find_join, (void*)&left_flag, Item::POSTFIX); - right->traverse_cond(find_join, (void*)&right_flag, Item::POSTFIX); - if (left_flag && right_flag) - { - *unsupported_feature = true; - } - } - } - break; - } - default: - { - break; - } + find_nonequi_join(item, arg); + break; } + case Item::EXPR_CACHE_ITEM: // IN + correlated subquery + { + const Item_cache_wrapper* icw = reinterpret_cast(item); + if (icw->get_orig_item()->type() == Item::FUNC_ITEM) + { + const Item_func* ifp = reinterpret_cast(icw->get_orig_item()); + if (ifp->argument_count() == 2 && (ifp->arguments()[0]->type() == Item::Item::SUBSELECT_ITEM || + ifp->arguments()[1]->type() == Item::Item::SUBSELECT_ITEM)) + { + *unsupported_feature = true; + return; + } + } + break; + } + case Item::COND_ITEM: // OR contains JOIN conds thats is unsupported yet + { + Item_cond* icp = (Item_cond*)item; + if (is_cond_or(icp)) + { + bool left_flag = false, right_flag = false; + if (icp->argument_list()->elements >= 2) + { + Item* left; + Item* right; + List_iterator li(*icp->argument_list()); + left = li++; + right = li++; + left->traverse_cond(find_join, (void*)&left_flag, Item::POSTFIX); + right->traverse_cond(find_join, (void*)&right_flag, Item::POSTFIX); + if (left_flag && right_flag) + { + *unsupported_feature = true; + } + } + } + break; + } + default: + { + break; + } + } } /*@brief check_user_var_func - This traverses Item */ /************************************************************ -* DESCRIPTION: -* This f() walks Item looking for the existence of -* "set_user_var" or "get_user_var" functions. -* PARAMETERS: -* Item * Item to traverse -* RETURN: -***********************************************************/ + * DESCRIPTION: + * This f() walks Item looking for the existence of + * "set_user_var" or "get_user_var" functions. + * PARAMETERS: + * Item * Item to traverse + * RETURN: + ***********************************************************/ void check_user_var_func(const Item* item, void* arg) { - bool* unsupported_feature = reinterpret_cast(arg); + bool* unsupported_feature = reinterpret_cast(arg); - if (*unsupported_feature) - return; + if (*unsupported_feature) + return; - if (item->type() == Item::FUNC_ITEM) + if (item->type() == Item::FUNC_ITEM) + { + const Item_func* ifp = reinterpret_cast(item); + std::string funcname = ifp->func_name(); + if (funcname == "set_user_var") { - const Item_func* ifp = reinterpret_cast(item); - std::string funcname = ifp->func_name(); - if (funcname == "set_user_var") - { - *unsupported_feature = true; - } + *unsupported_feature = true; } + } } void item_check(Item* item, bool* unsupported_feature) { - switch (item->type()) + switch (item->type()) + { + case Item::COND_ITEM: { - case Item::COND_ITEM: - { - Item_cond *icp = reinterpret_cast(item); - icp->traverse_cond(check_user_var_func, unsupported_feature, Item::POSTFIX); - break; - } - case Item::FUNC_ITEM: - { - Item_func *ifp = reinterpret_cast(item); - ifp->traverse_cond(check_user_var_func, unsupported_feature, Item::POSTFIX); - break; - } - default: - { - item->traverse_cond(check_user_var_func, unsupported_feature, Item::POSTFIX); - break; - } + Item_cond* icp = reinterpret_cast(item); + icp->traverse_cond(check_user_var_func, unsupported_feature, Item::POSTFIX); + break; } + case Item::FUNC_ITEM: + { + Item_func* ifp = reinterpret_cast(item); + ifp->traverse_cond(check_user_var_func, unsupported_feature, Item::POSTFIX); + break; + } + default: + { + item->traverse_cond(check_user_var_func, unsupported_feature, Item::POSTFIX); + break; + } + } } bool check_user_var(SELECT_LEX* select_lex) { - List_iterator_fast it(select_lex->item_list); - Item* item; - bool is_user_var_func = false; + List_iterator_fast it(select_lex->item_list); + Item* item; + bool is_user_var_func = false; - while ((item = it++)) + while ((item = it++)) + { + item_check(item, &is_user_var_func); + + if (is_user_var_func) { - item_check(item, &is_user_var_func); - - if (is_user_var_func) - { - return true; - } + return true; } + } - JOIN *join = select_lex->join; + JOIN* join = select_lex->join; - if (join->conds) - { - Item_cond* icp = reinterpret_cast(join->conds); + if (join->conds) + { + Item_cond* icp = reinterpret_cast(join->conds); - icp->traverse_cond(check_user_var_func, &is_user_var_func, Item::POSTFIX); - } + icp->traverse_cond(check_user_var_func, &is_user_var_func, Item::POSTFIX); + } - return is_user_var_func; + return is_user_var_func; } /*@brief create_columnstore_group_by_handler- Creates handler*/ @@ -383,89 +380,87 @@ bool check_user_var(SELECT_LEX* select_lex) * group_by_handler if success * NULL in other case ***********************************************************/ -group_by_handler* -create_columnstore_group_by_handler(THD* thd, Query* query) +group_by_handler* create_columnstore_group_by_handler(THD* thd, Query* query) { - ha_mcs_group_by_handler* handler = NULL; - // Disable GBH. + ha_mcs_group_by_handler* handler = NULL; + // Disable GBH. + return handler; + + // same as thd->lex->current_select + SELECT_LEX* select_lex = query->from->select_lex; + + // MCOL-2178 Disable SP support in the group_by_handler for now + // Check the session variable value to enable/disable use of + // group_by_handler. There is no GBH if SH works for the query. + if ((get_select_handler_mode(thd) == mcs_select_handler_mode_t::ON) || !get_group_by_handler(thd) || + (thd->lex)->sphead) + { return handler; + } - // same as thd->lex->current_select - SELECT_LEX *select_lex = query->from->select_lex; - - // MCOL-2178 Disable SP support in the group_by_handler for now - // Check the session variable value to enable/disable use of - // group_by_handler. There is no GBH if SH works for the query. - if ((get_select_handler_mode(thd) == mcs_select_handler_mode_t::ON) || - !get_group_by_handler(thd) || (thd->lex)->sphead) + // Create a handler if query is valid. See comments for details. + if (query->group_by || select_lex->with_sum_func) + { + bool unsupported_feature = false; + // revisit SELECT_LEX for all units + for (TABLE_LIST* tl = query->from; !unsupported_feature && tl; tl = tl->next_global) { - return handler; - } + select_lex = tl->select_lex; + // Correlation subquery. Comming soon so fail on this yet. + unsupported_feature = select_lex->is_correlated; - // Create a handler if query is valid. See comments for details. - if ( query->group_by || select_lex->with_sum_func ) - { - bool unsupported_feature = false; - // revisit SELECT_LEX for all units - for(TABLE_LIST* tl = query->from; !unsupported_feature && tl; tl = tl->next_global) + // Impossible HAVING or WHERE + if (!unsupported_feature && + (select_lex->having_value == Item::COND_FALSE || select_lex->cond_value == Item::COND_FALSE)) + { + unsupported_feature = true; + } + + // Unsupported JOIN conditions + if (!unsupported_feature) + { + JOIN* join = select_lex->join; + Item_cond* icp = 0; + + if (join != 0) + icp = reinterpret_cast(join->conds); + + if (unsupported_feature == false && icp) { - select_lex = tl->select_lex; - // Correlation subquery. Comming soon so fail on this yet. - unsupported_feature = select_lex->is_correlated; - - // Impossible HAVING or WHERE - if (!unsupported_feature && - (select_lex->having_value == Item::COND_FALSE - || select_lex->cond_value == Item::COND_FALSE )) - { - unsupported_feature = true; - } - - // Unsupported JOIN conditions - if ( !unsupported_feature ) - { - JOIN *join = select_lex->join; - Item_cond *icp = 0; - - if (join != 0) - icp = reinterpret_cast(join->conds); - - if (unsupported_feature == false && icp) - { - icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); - } - - // Optimizer could move some join conditions into where - if (select_lex->where != 0) - icp = reinterpret_cast(select_lex->where); - - if (unsupported_feature == false && icp) - { - icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); - } - } - - // Iterate and traverse through the item list and the JOIN cond - // and do not create GBH if the unsupported (set_user_var) - // function is present. - if (check_user_var(select_lex)) - { - return handler; - } - } // unsupported features check ends here - - if (!unsupported_feature) - { - handler = new ha_mcs_group_by_handler(thd, query); - - // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. - query->group_by = NULL; - query->order_by = NULL; - query->having = NULL; + icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); } - } - return handler; + // Optimizer could move some join conditions into where + if (select_lex->where != 0) + icp = reinterpret_cast(select_lex->where); + + if (unsupported_feature == false && icp) + { + icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); + } + } + + // Iterate and traverse through the item list and the JOIN cond + // and do not create GBH if the unsupported (set_user_var) + // function is present. + if (check_user_var(select_lex)) + { + return handler; + } + } // unsupported features check ends here + + if (!unsupported_feature) + { + handler = new ha_mcs_group_by_handler(thd, query); + + // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. + query->group_by = NULL; + query->order_by = NULL; + query->having = NULL; + } + } + + return handler; } /*@brief create_columnstore_derived_handler- Creates handler*/ @@ -481,102 +476,96 @@ create_columnstore_group_by_handler(THD* thd, Query* query) * derived_handler if possible * NULL in other case ***********************************************************/ -derived_handler* -create_columnstore_derived_handler(THD* thd, TABLE_LIST *table_ptr) +derived_handler* create_columnstore_derived_handler(THD* thd, TABLE_LIST* table_ptr) { - ha_columnstore_derived_handler* handler = NULL; + ha_columnstore_derived_handler* handler = NULL; - // MCOL-2178 Disable SP support in the derived_handler for now - // Check the session variable value to enable/disable use of - // derived_handler - if (!get_derived_handler(thd) || (thd->lex)->sphead) + // MCOL-2178 Disable SP support in the derived_handler for now + // Check the session variable value to enable/disable use of + // derived_handler + if (!get_derived_handler(thd) || (thd->lex)->sphead) + { + return handler; + } + + // Disable derived handler for prepared statements + if (thd->stmt_arena && thd->stmt_arena->is_stmt_execute()) + return handler; + + // MCOL-1482 Disable derived_handler if the multi-table update + // statement contains a non-columnstore table. + if (cal_impl_if::isUpdateHasForeignTable(thd)) + { + return handler; + } + + SELECT_LEX_UNIT* unit = table_ptr->derived; + SELECT_LEX* sl = unit->first_select(); + + bool unsupported_feature = false; + + // Impossible HAVING or WHERE + if (unsupported_feature || sl->having_value == Item::COND_FALSE || sl->cond_value == Item::COND_FALSE) + { + unsupported_feature = true; + } + + // JOIN expression from WHERE, ON expressions + JOIN* join = sl->join; + // TODO DRRTUY Make a proper tree traverse + // To search for CROSS JOIN-s we use tree invariant + // G(V,E) where [V] = [E]+1 + List join_preds_list; + TABLE_LIST* tl; + for (tl = sl->get_table_list(); !unsupported_feature && tl; tl = tl->next_local) + { + if (tl->where) { - return handler; + Item_cond* where_icp = reinterpret_cast(tl->where); + where_icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); + where_icp->traverse_cond(save_join_predicates, &join_preds_list, Item::POSTFIX); } - // Disable derived handler for prepared statements - if (thd->stmt_arena && thd->stmt_arena->is_stmt_execute()) - return handler; - - // MCOL-1482 Disable derived_handler if the multi-table update - // statement contains a non-columnstore table. - if (cal_impl_if::isUpdateHasForeignTable(thd)) + // Looking for JOIN with ON expression through + // TABLE_LIST in FROM until CS meets unsupported feature + if (tl->on_expr) { - return handler; + Item_cond* on_icp = reinterpret_cast(tl->on_expr); + on_icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); + on_icp->traverse_cond(save_join_predicates, &join_preds_list, Item::POSTFIX); } - SELECT_LEX_UNIT *unit = table_ptr->derived; - SELECT_LEX *sl = unit->first_select(); - - bool unsupported_feature = false; - - // Impossible HAVING or WHERE - if ( unsupported_feature - || sl->having_value == Item::COND_FALSE - || sl->cond_value == Item::COND_FALSE ) + // Iterate and traverse through the item list and the JOIN cond + // and do not create DH if the unsupported (set_user_var) + // function is present. + if (check_user_var(tl->select_lex)) { - unsupported_feature = true; + return handler; } + } - // JOIN expression from WHERE, ON expressions - JOIN* join = sl->join; - //TODO DRRTUY Make a proper tree traverse - //To search for CROSS JOIN-s we use tree invariant - //G(V,E) where [V] = [E]+1 - List join_preds_list; - TABLE_LIST *tl; - for (tl = sl->get_table_list(); !unsupported_feature && tl; tl = tl->next_local) - { - if (tl->where) - { - Item_cond* where_icp = reinterpret_cast(tl->where); - where_icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); - where_icp->traverse_cond(save_join_predicates, &join_preds_list, Item::POSTFIX); - } + if (!unsupported_feature && !join_preds_list.elements && join && join->conds) + { + Item_cond* conds = reinterpret_cast(join->conds); + conds->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); + conds->traverse_cond(save_join_predicates, &join_preds_list, Item::POSTFIX); + } - // Looking for JOIN with ON expression through - // TABLE_LIST in FROM until CS meets unsupported feature - if (tl->on_expr) - { - Item_cond* on_icp = reinterpret_cast(tl->on_expr); - on_icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); - on_icp->traverse_cond(save_join_predicates, &join_preds_list, Item::POSTFIX); - } + // CROSS JOIN w/o conditions isn't supported until MCOL-301 + // is ready. + if (!unsupported_feature && join && join->table_count >= 2 && !join_preds_list.elements) + { + unsupported_feature = true; + } - // Iterate and traverse through the item list and the JOIN cond - // and do not create DH if the unsupported (set_user_var) - // function is present. - if (check_user_var(tl->select_lex)) - { - return handler; - } - } + // CROSS JOIN with not enough JOIN predicates + if (!unsupported_feature && join && join_preds_list.elements < join->table_count - 1) + { + unsupported_feature = true; + } - if (!unsupported_feature && !join_preds_list.elements - && join && join->conds) - { - Item_cond* conds = reinterpret_cast(join->conds); - conds->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); - conds->traverse_cond(save_join_predicates, &join_preds_list, Item::POSTFIX); - } - - // CROSS JOIN w/o conditions isn't supported until MCOL-301 - // is ready. - if (!unsupported_feature && join - && join->table_count >= 2 && !join_preds_list.elements) - { - unsupported_feature = true; - } - - // CROSS JOIN with not enough JOIN predicates - if(!unsupported_feature && join - && join_preds_list.elements < join->table_count-1) - { - unsupported_feature = true; - } - - if (!unsupported_feature) - handler = new ha_columnstore_derived_handler(thd, table_ptr); + if (!unsupported_feature) + handler = new ha_columnstore_derived_handler(thd, table_ptr); return handler; } @@ -588,9 +577,8 @@ create_columnstore_derived_handler(THD* thd, TABLE_LIST *table_ptr) * thd - THD pointer. * tbl - tables involved. ***********************************************************/ -ha_columnstore_derived_handler::ha_columnstore_derived_handler(THD *thd, - TABLE_LIST *dt) - : derived_handler(thd, mcs_hton) +ha_columnstore_derived_handler::ha_columnstore_derived_handler(THD* thd, TABLE_LIST* dt) + : derived_handler(thd, mcs_hton) { derived = dt; } @@ -600,7 +588,8 @@ ha_columnstore_derived_handler::ha_columnstore_derived_handler(THD *thd, * derived_handler destructor ***********************************************************/ ha_columnstore_derived_handler::~ha_columnstore_derived_handler() -{} +{ +} /*@brief Initiate the query for derived_handler */ /*********************************************************** @@ -613,13 +602,13 @@ ha_columnstore_derived_handler::~ha_columnstore_derived_handler() ***********************************************************/ int ha_columnstore_derived_handler::init_scan() { - DBUG_ENTER("ha_columnstore_derived_handler::init_scan"); + DBUG_ENTER("ha_columnstore_derived_handler::init_scan"); - mcs_handler_info mhi(reinterpret_cast(this), DERIVED); - // this::table is the place for the result set - int rc = ha_mcs_impl_pushdown_init(&mhi, table); + mcs_handler_info mhi(reinterpret_cast(this), DERIVED); + // this::table is the place for the result set + int rc = ha_mcs_impl_pushdown_init(&mhi, table); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /*@brief Fetch next row for derived_handler */ @@ -634,11 +623,11 @@ int ha_columnstore_derived_handler::init_scan() ***********************************************************/ int ha_columnstore_derived_handler::next_row() { - DBUG_ENTER("ha_columnstore_derived_handler::next_row"); + DBUG_ENTER("ha_columnstore_derived_handler::next_row"); - int rc = ha_mcs_impl_rnd_next(table->record[0], table); + int rc = ha_mcs_impl_rnd_next(table->record[0], table); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /*@brief Finishes the scan and clean it up */ @@ -653,11 +642,11 @@ int ha_columnstore_derived_handler::next_row() ***********************************************************/ int ha_columnstore_derived_handler::end_scan() { - DBUG_ENTER("ha_columnstore_derived_handler::end_scan"); + DBUG_ENTER("ha_columnstore_derived_handler::end_scan"); - int rc = ha_mcs_impl_rnd_end(table, true); + int rc = ha_mcs_impl_rnd_end(table, true); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } void ha_columnstore_derived_handler::print_error(int, unsigned long) @@ -672,14 +661,14 @@ void ha_columnstore_derived_handler::print_error(int, unsigned long) * query - Query describing structure ***********************************************************/ ha_mcs_group_by_handler::ha_mcs_group_by_handler(THD* thd_arg, Query* query) - : group_by_handler(thd_arg, mcs_hton), - select(query->select), - table_list(query->from), - distinct(query->distinct), - where(query->where), - group_by(query->group_by), - order_by(query->order_by), - having(query->having) + : group_by_handler(thd_arg, mcs_hton) + , select(query->select) + , table_list(query->from) + , distinct(query->distinct) + , where(query->where) + , group_by(query->group_by) + , order_by(query->order_by) + , having(query->having) { } @@ -699,12 +688,12 @@ ha_mcs_group_by_handler::~ha_mcs_group_by_handler() ***********************************************************/ int ha_mcs_group_by_handler::init_scan() { - DBUG_ENTER("ha_mcs_group_by_handler::init_scan"); + DBUG_ENTER("ha_mcs_group_by_handler::init_scan"); - mcs_handler_info mhi = mcs_handler_info(reinterpret_cast(this), GROUP_BY); - int rc = ha_mcs_impl_group_by_init(&mhi, table); + mcs_handler_info mhi = mcs_handler_info(reinterpret_cast(this), GROUP_BY); + int rc = ha_mcs_impl_group_by_init(&mhi, table); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /*********************************************************** @@ -715,10 +704,10 @@ int ha_mcs_group_by_handler::init_scan() ***********************************************************/ int ha_mcs_group_by_handler::next_row() { - DBUG_ENTER("ha_mcs_group_by_handler::next_row"); - int rc = ha_mcs_impl_group_by_next(table); + DBUG_ENTER("ha_mcs_group_by_handler::next_row"); + int rc = ha_mcs_impl_group_by_next(table); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /*********************************************************** @@ -729,10 +718,10 @@ int ha_mcs_group_by_handler::next_row() ***********************************************************/ int ha_mcs_group_by_handler::end_scan() { - DBUG_ENTER("ha_mcs_group_by_handler::end_scan"); - int rc = ha_mcs_impl_group_by_end(table); + DBUG_ENTER("ha_mcs_group_by_handler::end_scan"); + int rc = ha_mcs_impl_group_by_end(table); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /*@brief create_columnstore_select_handler- Creates handler*/ @@ -748,248 +737,237 @@ int ha_mcs_group_by_handler::end_scan() * select_handler if possible * NULL in other case ***********************************************************/ -select_handler* -create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) +select_handler* create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { - mcs_select_handler_mode_t select_handler_mode = get_select_handler_mode(thd); + mcs_select_handler_mode_t select_handler_mode = get_select_handler_mode(thd); - // Check the session variable value to enable/disable use of - // select_handler - if ((select_handler_mode == mcs_select_handler_mode_t::OFF) || - ((thd->lex)->sphead && !get_select_handler_in_stored_procedures(thd))) + // Check the session variable value to enable/disable use of + // select_handler + if ((select_handler_mode == mcs_select_handler_mode_t::OFF) || + ((thd->lex)->sphead && !get_select_handler_in_stored_procedures(thd))) + { + return nullptr; + } + + // MCOL-1482 Disable select_handler for a multi-table update + // with a non-columnstore table as the target table of the update + // operation. + if (cal_impl_if::isForeignTableUpdate(thd)) + { + return nullptr; + } + + // Flag to indicate if this is a prepared statement + bool isPS = thd->stmt_arena && thd->stmt_arena->is_stmt_execute(); + + // Disable processing of select_result_interceptor classes + // which intercept and transform result set rows. E.g.: + // select a,b into @a1, @a2 from t1; + if (((thd->lex)->result && !((select_dumpvar*)(thd->lex)->result)->var_list.is_empty()) && (!isPS)) + { + return nullptr; + } + + // Select_handler couldn't properly process UPSERT..SELECT + if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT && thd->lex->duplicates == DUP_UPDATE) + { + return nullptr; + } + + // Iterate and traverse through the item list and the JOIN cond + // and do not create SH if the unsupported (set_user_var) + // function is present. + TABLE_LIST* table_ptr = select_lex->get_table_list(); + for (; table_ptr; table_ptr = table_ptr->next_global) + { + if (check_user_var(table_ptr->select_lex)) { - return nullptr; + return nullptr; } + } - // MCOL-1482 Disable select_handler for a multi-table update - // with a non-columnstore table as the target table of the update - // operation. - if (cal_impl_if::isForeignTableUpdate(thd)) + // We apply dedicated rewrites from MDB here so MDB's data structures + // becomes dirty and CS has to raise an error in case of any problem + // or unsupported feature. + ha_columnstore_select_handler* handler = new ha_columnstore_select_handler(thd, select_lex); + + JOIN* join = select_lex->join; + + if (select_lex->first_cond_optimization && select_lex->handle_derived(thd->lex, DT_MERGE)) + { + if (!thd->is_error()) { - return nullptr; - } - - // Flag to indicate if this is a prepared statement - bool isPS = thd->stmt_arena && thd->stmt_arena->is_stmt_execute(); - - // Disable processing of select_result_interceptor classes - // which intercept and transform result set rows. E.g.: - // select a,b into @a1, @a2 from t1; - if (((thd->lex)->result && - !((select_dumpvar *)(thd->lex)->result)->var_list.is_empty()) && - (!isPS)) - { - return nullptr; - } - - // Select_handler couldn't properly process UPSERT..SELECT - if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT - && thd->lex->duplicates == DUP_UPDATE) - { - return nullptr; - } - - // Iterate and traverse through the item list and the JOIN cond - // and do not create SH if the unsupported (set_user_var) - // function is present. - TABLE_LIST* table_ptr = select_lex->get_table_list(); - for (; table_ptr; table_ptr = table_ptr->next_global) - { - if (check_user_var(table_ptr->select_lex)) - { - return nullptr; - } - } - - // We apply dedicated rewrites from MDB here so MDB's data structures - // becomes dirty and CS has to raise an error in case of any problem - // or unsupported feature. - ha_columnstore_select_handler* handler = - new ha_columnstore_select_handler(thd, select_lex); - - JOIN *join = select_lex->join; - - if (select_lex->first_cond_optimization && - select_lex->handle_derived(thd->lex, DT_MERGE)) - { - if (!thd->is_error()) - { - my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), - "Error occured in select_lex::handle_derived()"); - } - - return handler; - } - - // This is partially taken from JOIN::optimize_inner() in sql/sql_select.cc - if (select_lex->first_cond_optimization) - { - create_explain_query_if_not_exists(thd->lex, thd->mem_root); - Query_arena *arena, backup; - arena = thd->activate_stmt_arena_if_needed(&backup); - COND* conds = join->conds; - select_lex->where = conds; - - if (isPS) - { - select_lex->prep_where = conds ? conds->copy_andor_structure(thd) : 0; - } - - select_lex->update_used_tables(); - - if (arena) - thd->restore_active_arena(arena, &backup); - -#ifdef DEBUG_WALK_COND - if (conds) - { - conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); - } -#endif - } - - // Attempt to execute the query using the select handler. - // If query execution fails and columnstore_select_handler=AUTO, - // fallback to table mode. - // Skip execution for EXPLAIN queries - if (!thd->lex->describe) - { - // This is taken from JOIN::optimize() - join->fields= &select_lex->item_list; - - // Instantiate handler::table, which is the place for the result set. - if (handler->prepare()) - { - // check fallback - if (select_handler_mode == mcs_select_handler_mode_t::AUTO) // columnstore_select_handler=AUTO - { - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, - "MCS select_handler execution failed. Falling back to server execution"); - restore_query_state(handler); - delete handler; - return nullptr; - } - - // error out - if (!thd->is_error()) - { - my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), - "Error occured in handler->prepare()"); - } - - return handler; - } - - // Prepare query execution - // This is taken from JOIN::exec_inner() - if (!select_lex->outer_select() && // (1) - select_lex != select_lex->master_unit()->fake_select_lex) // (2) - thd->lex->set_limit_rows_examined(); - - if (!join->tables_list && (join->table_count || !select_lex->with_sum_func) && - !select_lex->have_window_funcs()) - { - if (!thd->is_error()) - { - restore_query_state(handler); - delete handler; - return nullptr; - } - - return handler; - } - - if (!join->zero_result_cause && - join->exec_const_cond && !join->exec_const_cond->val_int()) - join->zero_result_cause = "Impossible WHERE noticed after reading const tables"; - - // We've called exec_const_cond->val_int(). This may have caused an error. - if (unlikely(thd->is_error())) - { - // error out - handler->pushdown_init_rc = 1; - return handler; - } - - if (join->zero_result_cause) - { - if (join->select_lex->have_window_funcs() && join->send_row_on_empty_set()) - { - join->const_tables = join->table_count; - join->first_select = sub_select_postjoin_aggr; - } - else - { - if (!thd->is_error()) - { - restore_query_state(handler); - delete handler; - return nullptr; - } - - return handler; - } - } - - if ((join->select_lex->options & OPTION_SCHEMA_TABLE) && - get_schema_tables_result(join, PROCESSED_BY_JOIN_EXEC)) - { - if (!thd->is_error()) - { - my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), - "Error occured in get_schema_tables_result()"); - } - - return handler; - } - - handler->scan_initialized = true; - mcs_handler_info mhi(reinterpret_cast(handler), SELECT); - - if ((handler->pushdown_init_rc = ha_mcs_impl_pushdown_init(&mhi, handler->table))) - { - // check fallback - if (select_handler_mode == mcs_select_handler_mode_t::AUTO) - { - restore_query_state(handler); - std::ostringstream oss; - oss << "MCS select_handler execution failed"; - - if (thd->is_error()) - { - oss << " due to: "; - oss << thd->get_stmt_da()->sql_errno() << ": " ; - oss << thd->get_stmt_da()->message(); - oss << " F"; - thd->clear_error(); - } - else - { - oss << ", f"; - } - - oss << "alling back to server execution"; - thd->get_stmt_da()->clear_warning_info(thd->query_id); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, oss.str().c_str()); - delete handler; - return nullptr; - } - - if (!thd->is_error()) - { - my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), - "Error occured in ha_mcs_impl_pushdown_init()"); - } - } - - // Unset select_lex::first_cond_optimization - if (select_lex->first_cond_optimization) - { - first_cond_optimization_flag_toggle(select_lex, &first_cond_optimization_flag_unset); - } + my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), "Error occured in select_lex::handle_derived()"); } return handler; + } + + // This is partially taken from JOIN::optimize_inner() in sql/sql_select.cc + if (select_lex->first_cond_optimization) + { + create_explain_query_if_not_exists(thd->lex, thd->mem_root); + Query_arena *arena, backup; + arena = thd->activate_stmt_arena_if_needed(&backup); + COND* conds = join->conds; + select_lex->where = conds; + + if (isPS) + { + select_lex->prep_where = conds ? conds->copy_andor_structure(thd) : 0; + } + + select_lex->update_used_tables(); + + if (arena) + thd->restore_active_arena(arena, &backup); + +#ifdef DEBUG_WALK_COND + if (conds) + { + conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); + } +#endif + } + + // Attempt to execute the query using the select handler. + // If query execution fails and columnstore_select_handler=AUTO, + // fallback to table mode. + // Skip execution for EXPLAIN queries + if (!thd->lex->describe) + { + // This is taken from JOIN::optimize() + join->fields = &select_lex->item_list; + + // Instantiate handler::table, which is the place for the result set. + if (handler->prepare()) + { + // check fallback + if (select_handler_mode == mcs_select_handler_mode_t::AUTO) // columnstore_select_handler=AUTO + { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, + "MCS select_handler execution failed. Falling back to server execution"); + restore_query_state(handler); + delete handler; + return nullptr; + } + + // error out + if (!thd->is_error()) + { + my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), "Error occured in handler->prepare()"); + } + + return handler; + } + + // Prepare query execution + // This is taken from JOIN::exec_inner() + if (!select_lex->outer_select() && // (1) + select_lex != select_lex->master_unit()->fake_select_lex) // (2) + thd->lex->set_limit_rows_examined(); + + if (!join->tables_list && (join->table_count || !select_lex->with_sum_func) && + !select_lex->have_window_funcs()) + { + if (!thd->is_error()) + { + restore_query_state(handler); + delete handler; + return nullptr; + } + + return handler; + } + + if (!join->zero_result_cause && join->exec_const_cond && !join->exec_const_cond->val_int()) + join->zero_result_cause = "Impossible WHERE noticed after reading const tables"; + + // We've called exec_const_cond->val_int(). This may have caused an error. + if (unlikely(thd->is_error())) + { + // error out + handler->pushdown_init_rc = 1; + return handler; + } + + if (join->zero_result_cause) + { + if (join->select_lex->have_window_funcs() && join->send_row_on_empty_set()) + { + join->const_tables = join->table_count; + join->first_select = sub_select_postjoin_aggr; + } + else + { + if (!thd->is_error()) + { + restore_query_state(handler); + delete handler; + return nullptr; + } + + return handler; + } + } + + if ((join->select_lex->options & OPTION_SCHEMA_TABLE) && + get_schema_tables_result(join, PROCESSED_BY_JOIN_EXEC)) + { + if (!thd->is_error()) + { + my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), "Error occured in get_schema_tables_result()"); + } + + return handler; + } + + handler->scan_initialized = true; + mcs_handler_info mhi(reinterpret_cast(handler), SELECT); + + if ((handler->pushdown_init_rc = ha_mcs_impl_pushdown_init(&mhi, handler->table))) + { + // check fallback + if (select_handler_mode == mcs_select_handler_mode_t::AUTO) + { + restore_query_state(handler); + std::ostringstream oss; + oss << "MCS select_handler execution failed"; + + if (thd->is_error()) + { + oss << " due to: "; + oss << thd->get_stmt_da()->sql_errno() << ": "; + oss << thd->get_stmt_da()->message(); + oss << " F"; + thd->clear_error(); + } + else + { + oss << ", f"; + } + + oss << "alling back to server execution"; + thd->get_stmt_da()->clear_warning_info(thd->query_id); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, oss.str().c_str()); + delete handler; + return nullptr; + } + + if (!thd->is_error()) + { + my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), "Error occured in ha_mcs_impl_pushdown_init()"); + } + } + + // Unset select_lex::first_cond_optimization + if (select_lex->first_cond_optimization) + { + first_cond_optimization_flag_toggle(select_lex, &first_cond_optimization_flag_unset); + } + } + + return handler; } /*********************************************************** @@ -999,15 +977,14 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) * thd - THD pointer. * select_lex - sematic tree for the query. ***********************************************************/ -ha_columnstore_select_handler::ha_columnstore_select_handler(THD *thd, - SELECT_LEX* select_lex) - : select_handler(thd, mcs_hton), - prepared(false), - scan_ended(false), - scan_initialized(false), - pushdown_init_rc(0) +ha_columnstore_select_handler::ha_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) + : select_handler(thd, mcs_hton) + , prepared(false) + , scan_ended(false) + , scan_initialized(false) + , pushdown_init_rc(0) { - select = select_lex; + select = select_lex; } /*********************************************************** @@ -1016,10 +993,10 @@ ha_columnstore_select_handler::ha_columnstore_select_handler(THD *thd, ***********************************************************/ ha_columnstore_select_handler::~ha_columnstore_select_handler() { - if (scan_initialized && !scan_ended) - { - end_scan(); - } + if (scan_initialized && !scan_ended) + { + end_scan(); + } } /*@brief Initiate the query for select_handler */ @@ -1033,8 +1010,8 @@ ha_columnstore_select_handler::~ha_columnstore_select_handler() ***********************************************************/ int ha_columnstore_select_handler::init_scan() { - DBUG_ENTER("ha_columnstore_select_handler::init_scan"); - DBUG_RETURN(pushdown_init_rc); + DBUG_ENTER("ha_columnstore_select_handler::init_scan"); + DBUG_RETURN(pushdown_init_rc); } /*@brief Fetch next row for select_handler */ @@ -1049,11 +1026,11 @@ int ha_columnstore_select_handler::init_scan() ***********************************************************/ int ha_columnstore_select_handler::next_row() { - DBUG_ENTER("ha_columnstore_select_handler::next_row"); + DBUG_ENTER("ha_columnstore_select_handler::next_row"); - int rc = ha_mcs_impl_select_next(table->record[0], table); + int rc = ha_mcs_impl_select_next(table->record[0], table); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } /*@brief Finishes the scan and clean it up */ @@ -1068,32 +1045,31 @@ int ha_columnstore_select_handler::next_row() ***********************************************************/ int ha_columnstore_select_handler::end_scan() { - DBUG_ENTER("ha_columnstore_select_handler::end_scan"); + DBUG_ENTER("ha_columnstore_select_handler::end_scan"); - scan_ended = true; + scan_ended = true; - int rc = ha_mcs_impl_rnd_end(table, true); + int rc = ha_mcs_impl_rnd_end(table, true); - DBUG_RETURN(rc); + DBUG_RETURN(rc); } // Copy of select_handler::prepare (see sql/select_handler.cc), // with an added if guard bool ha_columnstore_select_handler::prepare() { - DBUG_ENTER("ha_columnstore_select_handler::prepare"); + DBUG_ENTER("ha_columnstore_select_handler::prepare"); - if (prepared) - DBUG_RETURN(pushdown_init_rc ? true : false); + if (prepared) + DBUG_RETURN(pushdown_init_rc ? true : false); - prepared = true; + prepared = true; - if ((!table && !(table = create_tmp_table(thd, select))) || - table->fill_item_list(&result_columns)) - { - pushdown_init_rc = 1; - DBUG_RETURN(true); - } + if ((!table && !(table = create_tmp_table(thd, select))) || table->fill_item_list(&result_columns)) + { + pushdown_init_rc = 1; + DBUG_RETURN(true); + } - DBUG_RETURN(false); + DBUG_RETURN(false); } diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index 01806f0d7..0d6dd0422 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -14,7 +14,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - #ifndef HA_MCS_PUSH #define HA_MCS_PUSH @@ -27,25 +26,25 @@ #include "ha_mcs_impl_if.h" #include "ha_mcs_opt_rewrites.h" -void mutate_optimizer_flags(THD *thd_); -void restore_optimizer_flags(THD *thd_); +void mutate_optimizer_flags(THD* thd_); +void restore_optimizer_flags(THD* thd_); enum mcs_handler_types_t { - SELECT, - DERIVED, - GROUP_BY, - LEGACY + SELECT, + DERIVED, + GROUP_BY, + LEGACY }; struct mcs_handler_info { - mcs_handler_info() : hndl_ptr(NULL), hndl_type(LEGACY) { }; - mcs_handler_info(mcs_handler_types_t type) : hndl_ptr(NULL), hndl_type(type) { }; - mcs_handler_info(void* ptr, mcs_handler_types_t type) : hndl_ptr(ptr), hndl_type(type) { }; - ~mcs_handler_info() { }; - void* hndl_ptr; - mcs_handler_types_t hndl_type; + mcs_handler_info() : hndl_ptr(NULL), hndl_type(LEGACY){}; + mcs_handler_info(mcs_handler_types_t type) : hndl_ptr(NULL), hndl_type(type){}; + mcs_handler_info(void* ptr, mcs_handler_types_t type) : hndl_ptr(ptr), hndl_type(type){}; + ~mcs_handler_info(){}; + void* hndl_ptr; + mcs_handler_types_t hndl_type; }; /*@brief group_by_handler class*/ @@ -76,22 +75,22 @@ struct mcs_handler_info * next_row - get a row back from sm. * end_scan - finish and clean the things up. ***********************************************************/ -class ha_mcs_group_by_handler: public group_by_handler +class ha_mcs_group_by_handler : public group_by_handler { -public: - ha_mcs_group_by_handler(THD* thd_arg, Query* query); - ~ha_mcs_group_by_handler(); - int init_scan() override; - int next_row() override; - int end_scan() override; + public: + ha_mcs_group_by_handler(THD* thd_arg, Query* query); + ~ha_mcs_group_by_handler(); + int init_scan() override; + int next_row() override; + int end_scan() override; - List* select; - TABLE_LIST* table_list; - bool distinct; - Item* where; - ORDER* group_by; - ORDER* order_by; - Item* having; + List* select; + TABLE_LIST* table_list; + bool distinct; + Item* where; + ORDER* group_by; + ORDER* order_by; + Item* having; }; /*@brief derived_handler class*/ @@ -107,13 +106,13 @@ public: * next_row - get a row back from sm. * end_scan - finish and clean the things up. ***********************************************************/ -class ha_columnstore_derived_handler: public derived_handler +class ha_columnstore_derived_handler : public derived_handler { -private: - COLUMNSTORE_SHARE *share; + private: + COLUMNSTORE_SHARE* share; -public: - ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST *tbl); + public: + ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST* tbl); ~ha_columnstore_derived_handler(); int init_scan() override; int next_row() override; @@ -134,14 +133,14 @@ public: * next_row - get a row back from sm. * end_scan - finish and clean the things up. ***********************************************************/ -class ha_columnstore_select_handler: public select_handler +class ha_columnstore_select_handler : public select_handler { -private: - COLUMNSTORE_SHARE *share; + private: + COLUMNSTORE_SHARE* share; bool prepared; bool scan_ended; -public: + public: bool scan_initialized; int pushdown_init_rc; // MCOL-4525 Store the original TABLE_LIST::outer_join value in a hash map. diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index 65d43bd20..1a129819e 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -21,692 +21,488 @@ #include "ha_mcs_sysvars.h" #include "mcsconfig.h" -const char* mcs_compression_type_names[] = { - "SNAPPY", // 0 - "SNAPPY", // 1 - "SNAPPY", // 2 +const char* mcs_compression_type_names[] = {"SNAPPY", // 0 + "SNAPPY", // 1 + "SNAPPY", // 2 #ifdef HAVE_LZ4 - "LZ4", // 3 + "LZ4", // 3 #endif - NullS -}; + NullS}; -static TYPELIB mcs_compression_type_names_lib = { - array_elements(mcs_compression_type_names) - 1, - "mcs_compression_type_names", - mcs_compression_type_names, - NULL -}; +static TYPELIB mcs_compression_type_names_lib = {array_elements(mcs_compression_type_names) - 1, + "mcs_compression_type_names", mcs_compression_type_names, + NULL}; // compression type -static MYSQL_THDVAR_ENUM( - compression_type, - PLUGIN_VAR_RQCMDARG, - "Controls compression algorithm for create tables. Possible values are: " - "SNAPPY segment files are Snappy compressed (default);" +static MYSQL_THDVAR_ENUM(compression_type, PLUGIN_VAR_RQCMDARG, + "Controls compression algorithm for create tables. Possible values are: " + "SNAPPY segment files are Snappy compressed (default);" #ifdef HAVE_LZ4 - "LZ4 segment files are LZ4 compressed;", + "LZ4 segment files are LZ4 compressed;", #endif - NULL, // check - NULL, // update - 1, //default - &mcs_compression_type_names_lib); // values lib + NULL, // check + NULL, // update + 1, // default + &mcs_compression_type_names_lib); // values lib // fe_conn_info pointer -static MYSQL_THDVAR_ULONGLONG( - fe_conn_info_ptr, - PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT, - "FrontEnd connection structure pointer. For internal usage.", - NULL, - NULL, - 0, - 0, - ~0U, - 1 -); +static MYSQL_THDVAR_ULONGLONG(fe_conn_info_ptr, PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT, + "FrontEnd connection structure pointer. For internal usage.", NULL, NULL, 0, 0, + ~0U, 1); // optimizer flags vault -static MYSQL_THDVAR_ULONGLONG( - original_optimizer_flags, - PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT, - "Vault for original optimizer flags. For internal usage.", - NULL, - NULL, - 0, - 0, - ~0U, - 1 +static MYSQL_THDVAR_ULONGLONG(original_optimizer_flags, PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT, + "Vault for original optimizer flags. For internal usage.", NULL, NULL, 0, 0, + ~0U, 1); + +static MYSQL_THDVAR_ULONGLONG(original_option_bits, PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT, + "Storage for thd->variables.option_bits. For internal usage.", NULL, NULL, 0, 0, + ~0U, 1); + +const char* mcs_select_handler_mode_values[] = {"OFF", "ON", "AUTO", NullS}; + +static TYPELIB mcs_select_handler_mode_values_lib = {array_elements(mcs_select_handler_mode_values) - 1, + "mcs_select_handler_mode_values", + mcs_select_handler_mode_values, NULL}; + +static MYSQL_THDVAR_ENUM(select_handler, PLUGIN_VAR_RQCMDARG, + "Set the MCS select_handler to Disabled, Enabled, or Automatic", + NULL, // check + NULL, // update + 1, // default + &mcs_select_handler_mode_values_lib // values lib ); -static MYSQL_THDVAR_ULONGLONG( - original_option_bits, - PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT, - "Storage for thd->variables.option_bits. For internal usage.", - NULL, - NULL, - 0, - 0, - ~0U, - 1 -); +static MYSQL_THDVAR_BOOL(derived_handler, PLUGIN_VAR_NOCMDARG, "Enable/Disable the MCS derived_handler", NULL, + NULL, 1); -const char* mcs_select_handler_mode_values[] = { - "OFF", - "ON", - "AUTO", - NullS -}; +static MYSQL_THDVAR_BOOL(group_by_handler, PLUGIN_VAR_NOCMDARG, "Enable/Disable the MCS group_by_handler", + NULL, NULL, 1); -static TYPELIB mcs_select_handler_mode_values_lib = { - array_elements(mcs_select_handler_mode_values) - 1, - "mcs_select_handler_mode_values", - mcs_select_handler_mode_values, - NULL -}; +static MYSQL_THDVAR_BOOL(select_handler_in_stored_procedures, PLUGIN_VAR_NOCMDARG, + "Enable/Disable the MCS select_handler for Stored Procedures", NULL, NULL, 1); -static MYSQL_THDVAR_ENUM( - select_handler, - PLUGIN_VAR_RQCMDARG, - "Set the MCS select_handler to Disabled, Enabled, or Automatic", - NULL, // check - NULL, // update - 1, // default - &mcs_select_handler_mode_values_lib // values lib -); - -static MYSQL_THDVAR_BOOL( - derived_handler, - PLUGIN_VAR_NOCMDARG, - "Enable/Disable the MCS derived_handler", - NULL, - NULL, - 1 -); - -static MYSQL_THDVAR_BOOL( - group_by_handler, - PLUGIN_VAR_NOCMDARG, - "Enable/Disable the MCS group_by_handler", - NULL, - NULL, - 1 -); - -static MYSQL_THDVAR_BOOL( - select_handler_in_stored_procedures, - PLUGIN_VAR_NOCMDARG, - "Enable/Disable the MCS select_handler for Stored Procedures", - NULL, - NULL, - 1 -); - -static MYSQL_THDVAR_UINT( - orderby_threads, - PLUGIN_VAR_RQCMDARG, - "Number of parallel threads used by ORDER BY. (default to 16)", - NULL, - NULL, - 16, - 0, - 2048, - 1 -); +static MYSQL_THDVAR_UINT(orderby_threads, PLUGIN_VAR_RQCMDARG, + "Number of parallel threads used by ORDER BY. (default to 16)", NULL, NULL, 16, 0, + 2048, 1); // legacy system variables -static MYSQL_THDVAR_ULONG( - decimal_scale, - PLUGIN_VAR_RQCMDARG, - "The default decimal precision for calculated column sub-operations ", - NULL, - NULL, - 8, - 0, - 18, - 1 -); +static MYSQL_THDVAR_ULONG(decimal_scale, PLUGIN_VAR_RQCMDARG, + "The default decimal precision for calculated column sub-operations ", NULL, NULL, + 8, 0, 18, 1); + +static MYSQL_THDVAR_BOOL(varbin_always_hex, PLUGIN_VAR_NOCMDARG, + "Always display/process varbinary columns as if they have been hexified.", NULL, + NULL, 0); + +static MYSQL_THDVAR_BOOL(use_decimal_scale, PLUGIN_VAR_NOCMDARG, + "Enable/disable the MCS decimal scale to be used internally", NULL, NULL, 0); static MYSQL_THDVAR_BOOL( - varbin_always_hex, - PLUGIN_VAR_NOCMDARG, - "Always display/process varbinary columns as if they have been hexified.", - NULL, - NULL, - 0 + double_for_decimal_math, PLUGIN_VAR_NOCMDARG, + "Enable/disable for ColumnStore to replace DECIMAL with DOUBLE in arithmetic operation.", NULL, NULL, 0); + +static MYSQL_THDVAR_BOOL(decimal_overflow_check, PLUGIN_VAR_NOCMDARG, + "Enable/disable for ColumnStore to check for overflow in arithmetic operation.", + NULL, NULL, 0); + +static MYSQL_THDVAR_BOOL(ordered_only, PLUGIN_VAR_NOCMDARG, + "Always use the first table in the from clause as the large side " + "table for joins", + NULL, NULL, 0); + +static MYSQL_THDVAR_ULONG(string_scan_threshold, PLUGIN_VAR_RQCMDARG, + "Max number of blocks in a dictionary file to be scanned for filtering", NULL, NULL, + 10, 1, ~0U, 1); + +static MYSQL_THDVAR_ULONG(stringtable_threshold, PLUGIN_VAR_RQCMDARG, + "The minimum width of a string column to be stored in a string table", NULL, NULL, + 20, 9, ~0U, 1); + +static MYSQL_THDVAR_ULONG(diskjoin_smallsidelimit, PLUGIN_VAR_RQCMDARG, + "The maximum amount of disk space in MB to use per query for storing " + "'small side' tables for a disk-based join. (0 = unlimited)", + NULL, NULL, 0, 0, ~0U, 1); + +static MYSQL_THDVAR_ULONG(diskjoin_largesidelimit, PLUGIN_VAR_RQCMDARG, + "The maximum amount of disk space in MB to use per join for storing " + "'large side' table data for a disk-based join. (0 = unlimited)", + NULL, NULL, 0, 0, ~0U, 1); + +static MYSQL_THDVAR_ULONG(diskjoin_bucketsize, PLUGIN_VAR_RQCMDARG, + "The maximum size in MB of each 'small side' table in memory.", NULL, NULL, 100, 1, + ~0U, 1); + +static MYSQL_THDVAR_ULONG(um_mem_limit, PLUGIN_VAR_RQCMDARG, + "Per user Memory limit(MB). Switch to disk-based JOIN when limit is reached", NULL, + NULL, 0, 0, ~0U, 1); + +static MYSQL_THDVAR_ULONG(local_query, PLUGIN_VAR_RQCMDARG, + "Enable/disable the ColumnStore local PM query only feature.", NULL, NULL, 0, 0, 2, + 1); + +static MYSQL_THDVAR_ULONG(import_for_batchinsert_delimiter, PLUGIN_VAR_RQCMDARG, + "ASCII value of the delimiter used by LDI and INSERT..SELECT", + NULL, // check + NULL, // update + 7, // default + 0, // min + 127, // max + 1 // block size ); -static MYSQL_THDVAR_BOOL( - use_decimal_scale, - PLUGIN_VAR_NOCMDARG, - "Enable/disable the MCS decimal scale to be used internally", - NULL, - NULL, - 0 +static MYSQL_THDVAR_ULONG(import_for_batchinsert_enclosed_by, PLUGIN_VAR_RQCMDARG, + "ASCII value of the quote symbol used by batch data ingestion", + NULL, // check + NULL, // update + 17, // default + 17, // min + 127, // max + 1 // block size ); -static MYSQL_THDVAR_BOOL( - double_for_decimal_math, - PLUGIN_VAR_NOCMDARG, - "Enable/disable for ColumnStore to replace DECIMAL with DOUBLE in arithmetic operation.", - NULL, - NULL, - 0 -); - -static MYSQL_THDVAR_BOOL( - decimal_overflow_check, - PLUGIN_VAR_NOCMDARG, - "Enable/disable for ColumnStore to check for overflow in arithmetic operation.", - NULL, - NULL, - 0 -); - -static MYSQL_THDVAR_BOOL( - ordered_only, - PLUGIN_VAR_NOCMDARG, - "Always use the first table in the from clause as the large side " - "table for joins", - NULL, - NULL, - 0 -); - -static MYSQL_THDVAR_ULONG( - string_scan_threshold, - PLUGIN_VAR_RQCMDARG, - "Max number of blocks in a dictionary file to be scanned for filtering", - NULL, - NULL, - 10, - 1, - ~0U, - 1 -); - -static MYSQL_THDVAR_ULONG( - stringtable_threshold, - PLUGIN_VAR_RQCMDARG, - "The minimum width of a string column to be stored in a string table", - NULL, - NULL, - 20, - 9, - ~0U, - 1 -); - -static MYSQL_THDVAR_ULONG( - diskjoin_smallsidelimit, - PLUGIN_VAR_RQCMDARG, - "The maximum amount of disk space in MB to use per query for storing " - "'small side' tables for a disk-based join. (0 = unlimited)", - NULL, - NULL, - 0, - 0, - ~0U, - 1 -); - -static MYSQL_THDVAR_ULONG( - diskjoin_largesidelimit, - PLUGIN_VAR_RQCMDARG, - "The maximum amount of disk space in MB to use per join for storing " - "'large side' table data for a disk-based join. (0 = unlimited)", - NULL, - NULL, - 0, - 0, - ~0U, - 1 -); - -static MYSQL_THDVAR_ULONG( - diskjoin_bucketsize, - PLUGIN_VAR_RQCMDARG, - "The maximum size in MB of each 'small side' table in memory.", - NULL, - NULL, - 100, - 1, - ~0U, - 1 -); - -static MYSQL_THDVAR_ULONG( - um_mem_limit, - PLUGIN_VAR_RQCMDARG, - "Per user Memory limit(MB). Switch to disk-based JOIN when limit is reached", - NULL, - NULL, - 0, - 0, - ~0U, - 1 -); - -static MYSQL_THDVAR_ULONG( - local_query, - PLUGIN_VAR_RQCMDARG, - "Enable/disable the ColumnStore local PM query only feature.", - NULL, - NULL, - 0, - 0, - 2, - 1 -); - -static MYSQL_THDVAR_ULONG( - import_for_batchinsert_delimiter, - PLUGIN_VAR_RQCMDARG, - "ASCII value of the delimiter used by LDI and INSERT..SELECT", - NULL, // check - NULL, // update - 7, // default - 0, // min - 127, // max - 1 // block size -); - -static MYSQL_THDVAR_ULONG( - import_for_batchinsert_enclosed_by, - PLUGIN_VAR_RQCMDARG, - "ASCII value of the quote symbol used by batch data ingestion", - NULL, // check - NULL, // update - 17, // default - 17, // min - 127, // max - 1 // block size -); - -const char* mcs_use_import_for_batchinsert_mode_values[] = { - "OFF", - "ON", - "ALWAYS", - NullS -}; +const char* mcs_use_import_for_batchinsert_mode_values[] = {"OFF", "ON", "ALWAYS", NullS}; static TYPELIB mcs_use_import_for_batchinsert_mode_values_lib = { array_elements(mcs_use_import_for_batchinsert_mode_values) - 1, - "mcs_use_import_for_batchinsert_mode_values", - mcs_use_import_for_batchinsert_mode_values, - NULL -}; + "mcs_use_import_for_batchinsert_mode_values", mcs_use_import_for_batchinsert_mode_values, NULL}; -static MYSQL_THDVAR_ENUM( - use_import_for_batchinsert, - PLUGIN_VAR_RQCMDARG, - "LOAD DATA INFILE and INSERT..SELECT will use cpimport internally", - NULL, // check - NULL, // update - 1, // default - &mcs_use_import_for_batchinsert_mode_values_lib // values lib +static MYSQL_THDVAR_ENUM(use_import_for_batchinsert, PLUGIN_VAR_RQCMDARG, + "LOAD DATA INFILE and INSERT..SELECT will use cpimport internally", + NULL, // check + NULL, // update + 1, // default + &mcs_use_import_for_batchinsert_mode_values_lib // values lib ); -static MYSQL_THDVAR_BOOL( - replication_slave, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Allow this MariaDB server to apply replication changes to ColumnStore", - NULL, - NULL, - 0 -); +static MYSQL_THDVAR_BOOL(replication_slave, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, + "Allow this MariaDB server to apply replication changes to ColumnStore", NULL, NULL, + 0); -static MYSQL_THDVAR_BOOL( - cache_inserts, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Perform cache-based inserts to ColumnStore", - NULL, - NULL, - 0 -); +static MYSQL_THDVAR_BOOL(cache_inserts, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, + "Perform cache-based inserts to ColumnStore", NULL, NULL, 0); -static MYSQL_THDVAR_BOOL( - cache_use_import, - PLUGIN_VAR_RQCMDARG, - "Use cpimport for the cache flush into ColumnStore", - NULL, - NULL, - 0 -); +static MYSQL_THDVAR_BOOL(cache_use_import, PLUGIN_VAR_RQCMDARG, + "Use cpimport for the cache flush into ColumnStore", NULL, NULL, 0); -static MYSQL_THDVAR_ULONGLONG( - cache_flush_threshold, - PLUGIN_VAR_RQCMDARG, - "Threshold on the number of rows in the cache to trigger a flush", - NULL, - NULL, - 500000, - 1, - 1000000000, - 1 -); +static MYSQL_THDVAR_ULONGLONG(cache_flush_threshold, PLUGIN_VAR_RQCMDARG, + "Threshold on the number of rows in the cache to trigger a flush", NULL, NULL, + 500000, 1, 1000000000, 1); -st_mysql_sys_var* mcs_system_variables[] = -{ - MYSQL_SYSVAR(compression_type), - MYSQL_SYSVAR(fe_conn_info_ptr), - MYSQL_SYSVAR(original_optimizer_flags), - MYSQL_SYSVAR(original_option_bits), - MYSQL_SYSVAR(select_handler), - MYSQL_SYSVAR(derived_handler), - MYSQL_SYSVAR(group_by_handler), - MYSQL_SYSVAR(select_handler_in_stored_procedures), - MYSQL_SYSVAR(orderby_threads), - MYSQL_SYSVAR(decimal_scale), - MYSQL_SYSVAR(use_decimal_scale), - MYSQL_SYSVAR(ordered_only), - MYSQL_SYSVAR(string_scan_threshold), - MYSQL_SYSVAR(stringtable_threshold), - MYSQL_SYSVAR(diskjoin_smallsidelimit), - MYSQL_SYSVAR(diskjoin_largesidelimit), - MYSQL_SYSVAR(diskjoin_bucketsize), - MYSQL_SYSVAR(um_mem_limit), - MYSQL_SYSVAR(double_for_decimal_math), - MYSQL_SYSVAR(decimal_overflow_check), - MYSQL_SYSVAR(local_query), - MYSQL_SYSVAR(use_import_for_batchinsert), - MYSQL_SYSVAR(import_for_batchinsert_delimiter), - MYSQL_SYSVAR(import_for_batchinsert_enclosed_by), - MYSQL_SYSVAR(varbin_always_hex), - MYSQL_SYSVAR(replication_slave), - MYSQL_SYSVAR(cache_inserts), - MYSQL_SYSVAR(cache_use_import), - MYSQL_SYSVAR(cache_flush_threshold), - NULL -}; +st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type), + MYSQL_SYSVAR(fe_conn_info_ptr), + MYSQL_SYSVAR(original_optimizer_flags), + MYSQL_SYSVAR(original_option_bits), + MYSQL_SYSVAR(select_handler), + MYSQL_SYSVAR(derived_handler), + MYSQL_SYSVAR(group_by_handler), + MYSQL_SYSVAR(select_handler_in_stored_procedures), + MYSQL_SYSVAR(orderby_threads), + MYSQL_SYSVAR(decimal_scale), + MYSQL_SYSVAR(use_decimal_scale), + MYSQL_SYSVAR(ordered_only), + MYSQL_SYSVAR(string_scan_threshold), + MYSQL_SYSVAR(stringtable_threshold), + MYSQL_SYSVAR(diskjoin_smallsidelimit), + MYSQL_SYSVAR(diskjoin_largesidelimit), + MYSQL_SYSVAR(diskjoin_bucketsize), + MYSQL_SYSVAR(um_mem_limit), + MYSQL_SYSVAR(double_for_decimal_math), + MYSQL_SYSVAR(decimal_overflow_check), + MYSQL_SYSVAR(local_query), + MYSQL_SYSVAR(use_import_for_batchinsert), + MYSQL_SYSVAR(import_for_batchinsert_delimiter), + MYSQL_SYSVAR(import_for_batchinsert_enclosed_by), + MYSQL_SYSVAR(varbin_always_hex), + MYSQL_SYSVAR(replication_slave), + MYSQL_SYSVAR(cache_inserts), + MYSQL_SYSVAR(cache_use_import), + MYSQL_SYSVAR(cache_flush_threshold), + NULL}; -st_mysql_show_var mcs_status_variables[] = -{ - {"columnstore_version", (char*)&cs_version, SHOW_CHAR}, - {"columnstore_commit_hash", (char*)&cs_commit_hash, SHOW_CHAR}, - {0, 0, (enum_mysql_show_type)0} -}; +st_mysql_show_var mcs_status_variables[] = {{"columnstore_version", (char*)&cs_version, SHOW_CHAR}, + {"columnstore_commit_hash", (char*)&cs_commit_hash, SHOW_CHAR}, + {0, 0, (enum_mysql_show_type)0}}; void* get_fe_conn_info_ptr(THD* thd) { - return ( current_thd == NULL && thd == NULL ) ? NULL : - (void*)THDVAR(current_thd, fe_conn_info_ptr); + return (current_thd == NULL && thd == NULL) ? NULL : (void*)THDVAR(current_thd, fe_conn_info_ptr); } void set_fe_conn_info_ptr(void* ptr, THD* thd) { - if ( current_thd == NULL && thd == NULL) - { - return; - } + if (current_thd == NULL && thd == NULL) + { + return; + } - THDVAR(current_thd, fe_conn_info_ptr) = (uint64_t)(ptr); + THDVAR(current_thd, fe_conn_info_ptr) = (uint64_t)(ptr); } ulonglong get_original_optimizer_flags(THD* thd) { - return THDVAR(current_thd, original_optimizer_flags); + return THDVAR(current_thd, original_optimizer_flags); } void set_original_optimizer_flags(ulonglong ptr, THD* thd) { - if ( current_thd == NULL && thd == NULL) - { - return; - } + if (current_thd == NULL && thd == NULL) + { + return; + } - THDVAR(current_thd, original_optimizer_flags) = (uint64_t)(ptr); + THDVAR(current_thd, original_optimizer_flags) = (uint64_t)(ptr); } ulonglong get_original_option_bits(THD* thd) { - return (thd == NULL) ? 0 : THDVAR(thd, original_option_bits); + return (thd == NULL) ? 0 : THDVAR(thd, original_option_bits); } void set_original_option_bits(ulonglong value, THD* thd) { - if (thd == NULL) - { - return; - } + if (thd == NULL) + { + return; + } - THDVAR(thd, original_option_bits) = (uint64_t)(value); + THDVAR(thd, original_option_bits) = (uint64_t)(value); } mcs_select_handler_mode_t get_select_handler_mode(THD* thd) { - return ( thd == NULL ) ? mcs_select_handler_mode_t::ON : - (mcs_select_handler_mode_t) THDVAR(thd, select_handler); + return (thd == NULL) ? mcs_select_handler_mode_t::ON + : (mcs_select_handler_mode_t)THDVAR(thd, select_handler); } void set_select_handler_mode(THD* thd, ulong value) { - THDVAR(thd, select_handler) = value; + THDVAR(thd, select_handler) = value; } bool get_derived_handler(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, derived_handler); + return (thd == NULL) ? false : THDVAR(thd, derived_handler); } void set_derived_handler(THD* thd, bool value) { - THDVAR(thd, derived_handler) = value; + THDVAR(thd, derived_handler) = value; } bool get_group_by_handler(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, group_by_handler); + return (thd == NULL) ? false : THDVAR(thd, group_by_handler); } void set_group_by_handler(THD* thd, bool value) { - THDVAR(thd, group_by_handler) = value; + THDVAR(thd, group_by_handler) = value; } bool get_select_handler_in_stored_procedures(THD* thd) { - return ( thd == NULL ) ? true : - THDVAR(thd, select_handler_in_stored_procedures); + return (thd == NULL) ? true : THDVAR(thd, select_handler_in_stored_procedures); } void set_select_handler_in_stored_procedures(THD* thd, bool value) { - THDVAR(thd, select_handler_in_stored_procedures) = value; + THDVAR(thd, select_handler_in_stored_procedures) = value; } void set_compression_type(THD* thd, ulong value) { - THDVAR(thd, compression_type) = value; + THDVAR(thd, compression_type) = value; } -mcs_compression_type_t get_compression_type(THD* thd) { - return (mcs_compression_type_t) THDVAR(thd, compression_type); +mcs_compression_type_t get_compression_type(THD* thd) +{ + return (mcs_compression_type_t)THDVAR(thd, compression_type); } uint get_orderby_threads(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, orderby_threads); + return (thd == NULL) ? 0 : THDVAR(thd, orderby_threads); } void set_orderby_threads(THD* thd, uint value) { - THDVAR(thd, orderby_threads) = value; + THDVAR(thd, orderby_threads) = value; } - bool get_use_decimal_scale(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, use_decimal_scale); + return (thd == NULL) ? false : THDVAR(thd, use_decimal_scale); } void set_use_decimal_scale(THD* thd, bool value) { - THDVAR(thd, use_decimal_scale) = value; + THDVAR(thd, use_decimal_scale) = value; } ulong get_decimal_scale(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, decimal_scale); + return (thd == NULL) ? 0 : THDVAR(thd, decimal_scale); } void set_decimal_scale(THD* thd, ulong value) { - THDVAR(thd, decimal_scale) = value; + THDVAR(thd, decimal_scale) = value; } bool get_ordered_only(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, ordered_only); + return (thd == NULL) ? false : THDVAR(thd, ordered_only); } void set_ordered_only(THD* thd, bool value) { - THDVAR(thd, ordered_only) = value; + THDVAR(thd, ordered_only) = value; } ulong get_string_scan_threshold(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, string_scan_threshold); + return (thd == NULL) ? 0 : THDVAR(thd, string_scan_threshold); } void set_string_scan_threshold(THD* thd, ulong value) { - THDVAR(thd, string_scan_threshold) = value; + THDVAR(thd, string_scan_threshold) = value; } ulong get_stringtable_threshold(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, stringtable_threshold); + return (thd == NULL) ? 0 : THDVAR(thd, stringtable_threshold); } void set_stringtable_threshold(THD* thd, ulong value) { - THDVAR(thd, stringtable_threshold) = value; + THDVAR(thd, stringtable_threshold) = value; } ulong get_diskjoin_smallsidelimit(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, diskjoin_smallsidelimit); + return (thd == NULL) ? 0 : THDVAR(thd, diskjoin_smallsidelimit); } void set_diskjoin_smallsidelimit(THD* thd, ulong value) { - THDVAR(thd, diskjoin_smallsidelimit) = value; + THDVAR(thd, diskjoin_smallsidelimit) = value; } ulong get_diskjoin_largesidelimit(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, diskjoin_largesidelimit); + return (thd == NULL) ? 0 : THDVAR(thd, diskjoin_largesidelimit); } void set_diskjoin_largesidelimit(THD* thd, ulong value) { - THDVAR(thd, diskjoin_largesidelimit) = value; + THDVAR(thd, diskjoin_largesidelimit) = value; } ulong get_diskjoin_bucketsize(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, diskjoin_bucketsize); + return (thd == NULL) ? 0 : THDVAR(thd, diskjoin_bucketsize); } void set_diskjoin_bucketsize(THD* thd, ulong value) { - THDVAR(thd, diskjoin_bucketsize) = value; + THDVAR(thd, diskjoin_bucketsize) = value; } ulong get_um_mem_limit(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, um_mem_limit); + return (thd == NULL) ? 0 : THDVAR(thd, um_mem_limit); } void set_um_mem_limit(THD* thd, ulong value) { - THDVAR(thd, um_mem_limit) = value; + THDVAR(thd, um_mem_limit) = value; } bool get_varbin_always_hex(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, varbin_always_hex); + return (thd == NULL) ? false : THDVAR(thd, varbin_always_hex); } void set_varbin_always_hex(THD* thd, bool value) { - THDVAR(thd, varbin_always_hex) = value; + THDVAR(thd, varbin_always_hex) = value; } bool get_double_for_decimal_math(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, double_for_decimal_math); + return (thd == NULL) ? false : THDVAR(thd, double_for_decimal_math); } void set_double_for_decimal_math(THD* thd, bool value) { - THDVAR(thd, double_for_decimal_math) = value; + THDVAR(thd, double_for_decimal_math) = value; } bool get_decimal_overflow_check(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, decimal_overflow_check); + return (thd == NULL) ? false : THDVAR(thd, decimal_overflow_check); } void set_decimal_overflow_check(THD* thd, bool value) { - THDVAR(thd, decimal_overflow_check) = value; + THDVAR(thd, decimal_overflow_check) = value; } ulong get_local_query(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, local_query); + return (thd == NULL) ? 0 : THDVAR(thd, local_query); } void set_local_query(THD* thd, ulong value) { - THDVAR(thd, local_query) = value; + THDVAR(thd, local_query) = value; } mcs_use_import_for_batchinsert_mode_t get_use_import_for_batchinsert_mode(THD* thd) { - return ( thd == NULL ) ? mcs_use_import_for_batchinsert_mode_t::ON : - (mcs_use_import_for_batchinsert_mode_t) THDVAR(thd, use_import_for_batchinsert); + return (thd == NULL) ? mcs_use_import_for_batchinsert_mode_t::ON + : (mcs_use_import_for_batchinsert_mode_t)THDVAR(thd, use_import_for_batchinsert); } void set_use_import_for_batchinsert_mode(THD* thd, ulong value) { - THDVAR(thd, use_import_for_batchinsert) = value; + THDVAR(thd, use_import_for_batchinsert) = value; } ulong get_import_for_batchinsert_delimiter(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, import_for_batchinsert_delimiter); + return (thd == NULL) ? 0 : THDVAR(thd, import_for_batchinsert_delimiter); } void set_import_for_batchinsert_delimiter(THD* thd, ulong value) { - THDVAR(thd, import_for_batchinsert_delimiter) = value; + THDVAR(thd, import_for_batchinsert_delimiter) = value; } ulong get_import_for_batchinsert_enclosed_by(THD* thd) { - return ( thd == NULL ) ? 0 : THDVAR(thd, import_for_batchinsert_enclosed_by); + return (thd == NULL) ? 0 : THDVAR(thd, import_for_batchinsert_enclosed_by); } void set_import_for_batchinsert_enclosed_by(THD* thd, ulong value) { - THDVAR(thd, import_for_batchinsert_enclosed_by) = value; + THDVAR(thd, import_for_batchinsert_enclosed_by) = value; } bool get_replication_slave(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, replication_slave); + return (thd == NULL) ? false : THDVAR(thd, replication_slave); } void set_replication_slave(THD* thd, bool value) { - THDVAR(thd, replication_slave) = value; + THDVAR(thd, replication_slave) = value; } bool get_cache_inserts(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, cache_inserts); + return (thd == NULL) ? false : THDVAR(thd, cache_inserts); } void set_cache_inserts(THD* thd, bool value) { - THDVAR(thd, cache_inserts) = value; + THDVAR(thd, cache_inserts) = value; } bool get_cache_use_import(THD* thd) { - return ( thd == NULL ) ? false : THDVAR(thd, cache_use_import); + return (thd == NULL) ? false : THDVAR(thd, cache_use_import); } void set_cache_use_import(THD* thd, bool value) { - THDVAR(thd, cache_use_import) = value; + THDVAR(thd, cache_use_import) = value; } ulonglong get_cache_flush_threshold(THD* thd) { - return ( thd == NULL ) ? 500000 : THDVAR(thd, cache_flush_threshold); + return (thd == NULL) ? 500000 : THDVAR(thd, cache_flush_threshold); } void set_cache_flush_threshold(THD* thd, ulonglong value) { - THDVAR(thd, cache_flush_threshold) = value; + THDVAR(thd, cache_flush_threshold) = value; } diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index f0d216860..4597b5a20 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -29,26 +29,29 @@ extern char cs_version[]; extern char cs_commit_hash[]; // compression_type -enum mcs_compression_type_t { - NO_COMPRESSION = 0, - SNAPPY = 2, +enum mcs_compression_type_t +{ + NO_COMPRESSION = 0, + SNAPPY = 2, #ifdef HAVE_LZ4 - LZ4 = 3 + LZ4 = 3 #endif }; // use_import_for_batchinsert mode -enum class mcs_use_import_for_batchinsert_mode_t { - OFF = 0, - ON = 1, - ALWAYS = 2 +enum class mcs_use_import_for_batchinsert_mode_t +{ + OFF = 0, + ON = 1, + ALWAYS = 2 }; // select_handler mode -enum class mcs_select_handler_mode_t { - OFF = 0, - ON = 1, - AUTO = 2 +enum class mcs_select_handler_mode_t +{ + OFF = 0, + ON = 1, + AUTO = 2 }; // simple setters/getters diff --git a/dbcon/mysql/ha_mcs_udf.cpp b/dbcon/mysql/ha_mcs_udf.cpp index 52746f396..d28fd7cbd 100644 --- a/dbcon/mysql/ha_mcs_udf.cpp +++ b/dbcon/mysql/ha_mcs_udf.cpp @@ -34,197 +34,194 @@ using namespace std; extern "C" { + // Connector function stub -- CPFUNC1 + long long cpfunc1(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } -// Connector function stub -- CPFUNC1 - long long cpfunc1(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + my_bool cpfunc1_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { - return 0; + strcpy(message, "CALFUNC1() requires one COLUMN argument"); + return 1; } - my_bool cpfunc1_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC1() requires one COLUMN argument"); - return 1; - } + return 0; + } - return 0; + void cpfunc1_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC2 + long long cpfunc2(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc2_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) + { + strcpy(message, "CALFUNC2() requires one COLUMN argument"); + return 1; } - void cpfunc1_deinit(UDF_INIT* initid) + return 0; + } + + void cpfunc2_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC3 + long long cpfunc3(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc3_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "CALFUNC3() requires one COLUMN argument"); + return 1; } -// Connector function stub -- CPFUNC2 - long long cpfunc2(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + return 0; + } + + void cpfunc3_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC4 + long long cpfunc4(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc4_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { - return 0; + strcpy(message, "CALFUNC4() requires one COLUMN argument"); + return 1; } - my_bool cpfunc2_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC2() requires one COLUMN argument"); - return 1; - } + return 0; + } - return 0; + void cpfunc4_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC5 + long long cpfunc5(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc5_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) + { + strcpy(message, "CALFUNC5() requires one COLUMN argument"); + return 1; } - void cpfunc2_deinit(UDF_INIT* initid) + return 0; + } + + void cpfunc5_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC6 + long long cpfunc6(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc6_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "CALFUNC6() requires one COLUMN argument"); + return 1; } -// Connector function stub -- CPFUNC3 - long long cpfunc3(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + return 0; + } + + void cpfunc6_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC7 + long long cpfunc7(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc7_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { - return 0; + strcpy(message, "CALFUNC7() requires one COLUMN argument"); + return 1; } - my_bool cpfunc3_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC3() requires one COLUMN argument"); - return 1; - } + return 0; + } - return 0; + void cpfunc7_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC8 + long long cpfunc8(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc8_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) + { + strcpy(message, "CALFUNC8() requires one COLUMN argument"); + return 1; } - void cpfunc3_deinit(UDF_INIT* initid) + return 0; + } + + void cpfunc8_deinit(UDF_INIT* initid) + { + } + + // Connector function stub -- CPFUNC9 + long long cpfunc9(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } + + my_bool cpfunc9_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "CALFUNC9() requires one COLUMN argument"); + return 1; } -// Connector function stub -- CPFUNC4 - long long cpfunc4(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } - - my_bool cpfunc4_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC4() requires one COLUMN argument"); - return 1; - } - - return 0; - } - - void cpfunc4_deinit(UDF_INIT* initid) - { - } - -// Connector function stub -- CPFUNC5 - long long cpfunc5(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } - - my_bool cpfunc5_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC5() requires one COLUMN argument"); - return 1; - } - - return 0; - } - - void cpfunc5_deinit(UDF_INIT* initid) - { - } - -// Connector function stub -- CPFUNC6 - long long cpfunc6(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } - - my_bool cpfunc6_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC6() requires one COLUMN argument"); - return 1; - } - - return 0; - } - - void cpfunc6_deinit(UDF_INIT* initid) - { - } - -// Connector function stub -- CPFUNC7 - long long cpfunc7(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } - - my_bool cpfunc7_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC7() requires one COLUMN argument"); - return 1; - } - - return 0; - } - - void cpfunc7_deinit(UDF_INIT* initid) - { - } - -// Connector function stub -- CPFUNC8 - long long cpfunc8(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } - - my_bool cpfunc8_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC8() requires one COLUMN argument"); - return 1; - } - - return 0; - } - - void cpfunc8_deinit(UDF_INIT* initid) - { - } - -// Connector function stub -- CPFUNC9 - long long cpfunc9(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } - - my_bool cpfunc9_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "CALFUNC9() requires one COLUMN argument"); - return 1; - } - - return 0; - } - - void cpfunc9_deinit(UDF_INIT* initid) - { - } + return 0; + } + void cpfunc9_deinit(UDF_INIT* initid) + { + } } // vim:sw=4 ts=4: - diff --git a/dbcon/mysql/ha_pseudocolumn.cpp b/dbcon/mysql/ha_pseudocolumn.cpp index ae1ddc01e..fb87a76cc 100644 --- a/dbcon/mysql/ha_pseudocolumn.cpp +++ b/dbcon/mysql/ha_pseudocolumn.cpp @@ -26,7 +26,6 @@ using namespace cal_impl_if; namespace { - /******************************************************************************* * Pseudo column connector interface * @@ -46,548 +45,538 @@ namespace void bailout(char* error, const string& funcName) { - string errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PSEUDOCOL_IDB_ONLY, funcName); - current_thd->get_stmt_da()->set_overwrite_status(true); - current_thd->raise_error_printf(ER_INTERNAL_ERROR, errMsg.c_str()); - *error = 1; + string errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PSEUDOCOL_IDB_ONLY, funcName); + current_thd->get_stmt_da()->set_overwrite_status(true); + current_thd->raise_error_printf(ER_INTERNAL_ERROR, errMsg.c_str()); + *error = 1; } int64_t idblocalpm() { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (ci->localPm == -1) - { - string module = ClientRotator::getModule(); + if (ci->localPm == -1) + { + string module = ClientRotator::getModule(); - if (module.size() >= 3 && (module[0] == 'p' || module[0] == 'P')) - ci->localPm = atol(module.c_str() + 2); - else - ci->localPm = 0; - } + if (module.size() >= 3 && (module[0] == 'p' || module[0] == 'P')) + ci->localPm = atol(module.c_str() + 2); + else + ci->localPm = 0; + } - return ci->localPm; + return ci->localPm; } extern "C" { - /** - * IDBDBROOT - */ + /** + * IDBDBROOT + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbdbroot_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool idbdbroot_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { - if (args->arg_count != 1) - { - strcpy(message, "idbdbroot() requires one argument"); - return 1; - } - - return 0; + strcpy(message, "idbdbroot() requires one argument"); + return 1; } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void idbdbroot_deinit(UDF_INIT* initid) - { - } + return 0; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbdbroot(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbdbroot"); - return 0; - } - - /** - * IDBPM - */ + void idbdbroot_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbpm_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbpm() requires one argument"); - return 1; - } + long long idbdbroot(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbdbroot"); + return 0; + } - return 0; - } + /** + * IDBPM + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbpm_deinit(UDF_INIT* initid) + my_bool idbpm_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbpm() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbpm(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbpm"); - return 0; - } - - /** - * IDBEXTENTRELATIVERID - */ + void idbpm_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbextentrelativerid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbextentrelativerid() requires one argument"); - return 1; - } + long long idbpm(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbpm"); + return 0; + } - return 0; - } + /** + * IDBEXTENTRELATIVERID + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbextentrelativerid_deinit(UDF_INIT* initid) + my_bool idbextentrelativerid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbextentrelativerid() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbextentrelativerid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbextentrelativerid"); - return 0; - } - - /** - * IDBBLOCKID - */ + void idbextentrelativerid_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbblockid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbblockid() requires one argument"); - return 1; - } + long long idbextentrelativerid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbextentrelativerid"); + return 0; + } - return 0; - } + /** + * IDBBLOCKID + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbblockid_deinit(UDF_INIT* initid) + my_bool idbblockid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbblockid() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbblockid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbblockid"); - return 0; - } - - /** - * IDBEXTENTID - */ + void idbblockid_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbextentid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbextentid() requires one argument"); - return 1; - } + long long idbblockid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbblockid"); + return 0; + } - return 0; - } + /** + * IDBEXTENTID + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbextentid_deinit(UDF_INIT* initid) + my_bool idbextentid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbextentid() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbextentid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbextentid"); - return 0; - } - - /** - * IDBSEGMENT - */ + void idbextentid_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbsegment_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbsegment() requires one argument"); - return 1; - } + long long idbextentid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbextentid"); + return 0; + } - return 0; - } + /** + * IDBSEGMENT + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbsegment_deinit(UDF_INIT* initid) + my_bool idbsegment_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbsegment() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbsegment(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbsegment"); - return 0; - } - - /** - * IDBSEGMENTDIR - */ + void idbsegment_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbsegmentdir_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbsegmentdir() requires one argument"); - return 1; - } + long long idbsegment(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbsegment"); + return 0; + } - return 0; - } + /** + * IDBSEGMENTDIR + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbsegmentdir_deinit(UDF_INIT* initid) + my_bool idbsegmentdir_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbsegmentdir() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idbsegmentdir(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - bailout(error, "idbsegmentdir"); - return 0; - } - - /** - * IDBPARTITION - */ + void idbsegmentdir_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbpartition_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbpartition() requires one argument"); - return 1; - } + long long idbsegmentdir(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + bailout(error, "idbsegmentdir"); + return 0; + } - return 0; - } + /** + * IDBPARTITION + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbpartition_deinit(UDF_INIT* initid) + my_bool idbpartition_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbpartition() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* idbpartition(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - bailout(error, "idbpartition"); - return result; - } - - /** - * IDBEXTENTMIN - */ + void idbpartition_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbextentmin_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbpm() requires one argument"); - return 1; - } + const char* idbpartition(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + bailout(error, "idbpartition"); + return result; + } - initid->maybe_null = 1; - return 0; - } + /** + * IDBEXTENTMIN + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbextentmin_deinit(UDF_INIT* initid) + my_bool idbextentmin_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbpm() requires one argument"); + return 1; } + initid->maybe_null = 1; + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* idbextentmin(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - bailout(error, "idbextentmin"); - return result; - } - - /** - * IDBEXTENTMAX - */ + void idbextentmin_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idbextentmax_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "idbextentmax() requires one argument"); - return 1; - } + const char* idbextentmin(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + bailout(error, "idbextentmin"); + return result; + } - initid->maybe_null = 1; - return 0; - } + /** + * IDBEXTENTMAX + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idbextentmax_deinit(UDF_INIT* initid) + my_bool idbextentmax_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { + strcpy(message, "idbextentmax() requires one argument"); + return 1; } + initid->maybe_null = 1; + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - const char* idbextentmax(UDF_INIT* initid, UDF_ARGS* args, - char* result, unsigned long* length, - char* is_null, char* error) - { - bailout(error, "idbextentmax"); - return result; - } - - /** - * IDBLOCALPM - */ + void idbextentmax_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool idblocalpm_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 0) - { - strcpy(message, "idblocalpm() should take no argument"); - return 1; - } + const char* idbextentmax(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + bailout(error, "idbextentmax"); + return result; + } - initid->maybe_null = 1; - return 0; - } + /** + * IDBLOCALPM + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void idblocalpm_deinit(UDF_INIT* initid) + my_bool idblocalpm_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) { + strcpy(message, "idblocalpm() should take no argument"); + return 1; } + initid->maybe_null = 1; + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long idblocalpm(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - longlong localpm = idblocalpm(); + void idblocalpm_deinit(UDF_INIT* initid) + { + } - if (localpm == 0) - *is_null = 1; +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long idblocalpm(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + longlong localpm = idblocalpm(); - return localpm; - } + if (localpm == 0) + *is_null = 1; + return localpm; + } } -} +} // namespace namespace cal_impl_if { - ReturnedColumn* nullOnError(gp_walk_info& gwi, string& funcName) { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_PSEUDOCOL_WRONG_ARG, funcName); - return NULL; + gwi.fatalParseError = true; + gwi.parseErrorText = + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_PSEUDOCOL_WRONG_ARG, funcName); + return NULL; } uint32_t isPseudoColumn(string funcName) { - return execplan::PseudoColumn::pseudoNameToType(funcName); + return execplan::PseudoColumn::pseudoNameToType(funcName); } -execplan::ReturnedColumn* buildPseudoColumn(Item* item, - gp_walk_info& gwi, - bool& nonSupport, - uint32_t pseudoType) +execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, + uint32_t pseudoType) { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - Item_func* ifp = (Item_func*)item; + Item_func* ifp = (Item_func*)item; - // idblocalpm is replaced by constant - if (pseudoType == PSEUDO_LOCALPM) - { - int64_t localPm = idblocalpm(); - ConstantColumn* cc; + // idblocalpm is replaced by constant + if (pseudoType == PSEUDO_LOCALPM) + { + int64_t localPm = idblocalpm(); + ConstantColumn* cc; - if (localPm) - cc = new ConstantColumn(localPm); - else - cc = new ConstantColumn("", ConstantColumn::NULLDATA); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + if (localPm) + cc = new ConstantColumn(localPm); + else + cc = new ConstantColumn("", ConstantColumn::NULLDATA); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - cc->alias(ifp->full_name() ? ifp->full_name() : ""); - return cc; - } + cc->alias(ifp->full_name() ? ifp->full_name() : ""); + return cc; + } - // convert udf item to pseudocolumn item. - // adjust result type - // put arg col to column map - string funcName = ifp->func_name(); + // convert udf item to pseudocolumn item. + // adjust result type + // put arg col to column map + string funcName = ifp->func_name(); - if (ifp->argument_count() != 1 || - !(ifp->arguments()) || - !(ifp->arguments()[0]) || - ifp->arguments()[0]->type() != Item::FIELD_ITEM) - return nullOnError(gwi, funcName); + if (ifp->argument_count() != 1 || !(ifp->arguments()) || !(ifp->arguments()[0]) || + ifp->arguments()[0]->type() != Item::FIELD_ITEM) + return nullOnError(gwi, funcName); - Item_field* field = (Item_field*)(ifp->arguments()[0]); + Item_field* field = (Item_field*)(ifp->arguments()[0]); - // @todo rule out derive table - if (!field->field || !field->db_name.str || strlen(field->db_name.str) == 0) - return nullOnError(gwi, funcName); + // @todo rule out derive table + if (!field->field || !field->db_name.str || strlen(field->db_name.str) == 0) + return nullOnError(gwi, funcName); - SimpleColumn* sc = buildSimpleColumn(field, gwi); + SimpleColumn* sc = buildSimpleColumn(field, gwi); - if (!sc) - return nullOnError(gwi, funcName); + if (!sc) + return nullOnError(gwi, funcName); - if ((pseudoType == PSEUDO_EXTENTMIN || pseudoType == PSEUDO_EXTENTMAX) && - (sc->colType().colDataType == CalpontSystemCatalog::VARBINARY || - (sc->colType().colDataType == CalpontSystemCatalog::VARCHAR && sc->colType().colWidth > 7) || - (sc->colType().colDataType == CalpontSystemCatalog::CHAR && sc->colType().colWidth > 8))) - return nullOnError(gwi, funcName); + if ((pseudoType == PSEUDO_EXTENTMIN || pseudoType == PSEUDO_EXTENTMAX) && + (sc->colType().colDataType == CalpontSystemCatalog::VARBINARY || + (sc->colType().colDataType == CalpontSystemCatalog::VARCHAR && sc->colType().colWidth > 7) || + (sc->colType().colDataType == CalpontSystemCatalog::CHAR && sc->colType().colWidth > 8))) + return nullOnError(gwi, funcName); - // put arg col to column map - if (gwi.clauseType == SELECT || gwi.clauseType == GROUP_BY) // select clause - { - SRCP srcp(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, field->cached_table); - } - else if (!gwi.rcWorkStack.empty()) - { - gwi.rcWorkStack.pop(); - } + // put arg col to column map + if (gwi.clauseType == SELECT || gwi.clauseType == GROUP_BY) // select clause + { + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = + make_pair(1, field->cached_table); + } + else if (!gwi.rcWorkStack.empty()) + { + gwi.rcWorkStack.pop(); + } - if (pseudoType == PSEUDO_PARTITION) - { - // parms: psueducolumn dbroot, segmentdir, segment - SPTP sptp; - FunctionColumn* fc = new FunctionColumn(funcName); - funcexp::FunctionParm parms; - PseudoColumn* dbroot = new PseudoColumn(*sc, PSEUDO_DBROOT); - sptp.reset(new ParseTree(dbroot)); - parms.push_back(sptp); - PseudoColumn* pp = new PseudoColumn(*sc, PSEUDO_SEGMENTDIR); - sptp.reset(new ParseTree(pp)); - parms.push_back(sptp); - PseudoColumn* seg = new PseudoColumn(*sc, PSEUDO_SEGMENT); - sptp.reset(new ParseTree(seg)); - parms.push_back(sptp); - fc->functionParms(parms); - fc->expressionId(ci->expressionId++); - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + if (pseudoType == PSEUDO_PARTITION) + { + // parms: psueducolumn dbroot, segmentdir, segment + SPTP sptp; + FunctionColumn* fc = new FunctionColumn(funcName); + funcexp::FunctionParm parms; + PseudoColumn* dbroot = new PseudoColumn(*sc, PSEUDO_DBROOT); + sptp.reset(new ParseTree(dbroot)); + parms.push_back(sptp); + PseudoColumn* pp = new PseudoColumn(*sc, PSEUDO_SEGMENTDIR); + sptp.reset(new ParseTree(pp)); + parms.push_back(sptp); + PseudoColumn* seg = new PseudoColumn(*sc, PSEUDO_SEGMENT); + sptp.reset(new ParseTree(seg)); + parms.push_back(sptp); + fc->functionParms(parms); + fc->expressionId(ci->expressionId++); + fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - // string result type - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 256; - fc->resultType(ct); + // string result type + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 256; + fc->resultType(ct); - // operation type integer - funcexp::Func_idbpartition* idbpartition = new funcexp::Func_idbpartition(); - fc->operationType(idbpartition->operationType(parms, fc->resultType())); - fc->alias(ifp->full_name() ? ifp->full_name() : ""); - return fc; - } + // operation type integer + funcexp::Func_idbpartition* idbpartition = new funcexp::Func_idbpartition(); + fc->operationType(idbpartition->operationType(parms, fc->resultType())); + fc->alias(ifp->full_name() ? ifp->full_name() : ""); + return fc; + } - PseudoColumn* pc = new PseudoColumn(*sc, pseudoType); + PseudoColumn* pc = new PseudoColumn(*sc, pseudoType); - // @bug5892. set alias for derived table column matching. - pc->alias(ifp->name.length ? ifp->name.str : ""); - return pc; + // @bug5892. set alias for derived table column matching. + pc->alias(ifp->name.length ? ifp->name.str : ""); + return pc; } -} +} // namespace cal_impl_if // vim:ts=4 sw=4: - diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index b0170e2fd..4a4f08fca 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_scalar_sub.cpp 6418 2010-03-29 21:55:08Z zzhu $ -* -* -***********************************************************************/ + * $Id: ha_scalar_sub.cpp 6418 2010-03-29 21:55:08Z zzhu $ + * + * + ***********************************************************************/ /** @file */ /** class ScalarSub definition */ @@ -52,290 +52,294 @@ using namespace logging; namespace cal_impl_if { ScalarSub::ScalarSub(gp_walk_info& gwip) : WhereSubQuery(gwip), fReturnedColPos(0) -{} +{ +} -ScalarSub::ScalarSub(gp_walk_info& gwip, Item_func* func) : - WhereSubQuery(gwip, func), fReturnedColPos(0) -{} +ScalarSub::ScalarSub(gp_walk_info& gwip, Item_func* func) : WhereSubQuery(gwip, func), fReturnedColPos(0) +{ +} -ScalarSub::ScalarSub(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func): - WhereSubQuery(gwip, column, sub, func), fReturnedColPos(0) -{} +ScalarSub::ScalarSub(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func) + : WhereSubQuery(gwip, column, sub, func), fReturnedColPos(0) +{ +} -ScalarSub::ScalarSub(const ScalarSub& rhs) : - WhereSubQuery(rhs.gwip(), rhs.fColumn, rhs.fSub, rhs.fFunc), - fReturnedColPos(rhs.fReturnedColPos) -{} +ScalarSub::ScalarSub(const ScalarSub& rhs) + : WhereSubQuery(rhs.gwip(), rhs.fColumn, rhs.fSub, rhs.fFunc), fReturnedColPos(rhs.fReturnedColPos) +{ +} ScalarSub::~ScalarSub() -{} +{ +} execplan::ParseTree* ScalarSub::transform() { - if (!fFunc) - return NULL; + if (!fFunc) + return NULL; - // @todo need to handle scalar IN and BETWEEN specially - // this blocks handles only one subselect scalar - // arg[0]: column | arg[1]: subselect - //idbassert(fGwip.rcWorkStack.size() >= 2); - if (fFunc->functype() == Item_func::BETWEEN) - return transform_between(); + // @todo need to handle scalar IN and BETWEEN specially + // this blocks handles only one subselect scalar + // arg[0]: column | arg[1]: subselect + // idbassert(fGwip.rcWorkStack.size() >= 2); + if (fFunc->functype() == Item_func::BETWEEN) + return transform_between(); - if (fFunc->functype() == Item_func::IN_FUNC) - return transform_in(); + if (fFunc->functype() == Item_func::IN_FUNC) + return transform_in(); - ReturnedColumn* rhs = NULL; - ReturnedColumn* lhs = NULL; + ReturnedColumn* rhs = NULL; + ReturnedColumn* lhs = NULL; - if (!fGwip.rcWorkStack.empty()) - { - rhs = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); - } + if (!fGwip.rcWorkStack.empty()) + { + rhs = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + } - if (!fGwip.rcWorkStack.empty()) - { - lhs = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); - } + if (!fGwip.rcWorkStack.empty()) + { + lhs = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + } - PredicateOperator* op = new PredicateOperator(fFunc->func_name()); - - if (!lhs && (fFunc->functype() == Item_func::ISNULL_FUNC || - fFunc->functype() == Item_func::ISNOTNULL_FUNC)) - { - fSub = (Item_subselect*)(fFunc->arguments()[0]); - fColumn.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); - (dynamic_cast(fColumn.get()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); - delete rhs; - return buildParseTree(op); - } - - bool reverseOp = false; - SubSelect* sub = dynamic_cast(rhs); - - if (!sub) - { - reverseOp = true; - delete lhs; - lhs = rhs; - fSub = (Item_subselect*)(fFunc->arguments()[0]); - } - else - { - delete rhs; - fSub = (Item_subselect*)(fFunc->arguments()[1]); - } - - fColumn.reset(lhs); // column should be in the stack already. in, between may be different - - //PredicateOperator *op = new PredicateOperator(fFunc->func_name()); - if (reverseOp) - op->reverseOp(); + PredicateOperator* op = new PredicateOperator(fFunc->func_name()); + if (!lhs && (fFunc->functype() == Item_func::ISNULL_FUNC || fFunc->functype() == Item_func::ISNOTNULL_FUNC)) + { + fSub = (Item_subselect*)(fFunc->arguments()[0]); + fColumn.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); + (dynamic_cast(fColumn.get())) + ->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + delete rhs; return buildParseTree(op); + } + + bool reverseOp = false; + SubSelect* sub = dynamic_cast(rhs); + + if (!sub) + { + reverseOp = true; + delete lhs; + lhs = rhs; + fSub = (Item_subselect*)(fFunc->arguments()[0]); + } + else + { + delete rhs; + fSub = (Item_subselect*)(fFunc->arguments()[1]); + } + + fColumn.reset(lhs); // column should be in the stack already. in, between may be different + + // PredicateOperator *op = new PredicateOperator(fFunc->func_name()); + if (reverseOp) + op->reverseOp(); + + return buildParseTree(op); } execplan::ParseTree* ScalarSub::transform_between() { - //idbassert(fGwip.rcWorkStack.size() >= 3); - if (fGwip.rcWorkStack.size() < 3) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); - return NULL; - } + // idbassert(fGwip.rcWorkStack.size() >= 3); + if (fGwip.rcWorkStack.size() < 3) + { + fGwip.fatalParseError = true; + fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); + return NULL; + } - ReturnedColumn* op3 = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); - ReturnedColumn* op2 = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); - ReturnedColumn* op1 = fGwip.rcWorkStack.top(); - fGwip.rcWorkStack.pop(); - fColumn.reset(op1); + ReturnedColumn* op3 = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + ReturnedColumn* op2 = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + ReturnedColumn* op1 = fGwip.rcWorkStack.top(); + fGwip.rcWorkStack.pop(); + fColumn.reset(op1); - ParseTree* lhs = NULL; - ParseTree* rhs = NULL; - PredicateOperator* op_LE = new PredicateOperator("<="); - PredicateOperator* op_GE = new PredicateOperator(">="); + ParseTree* lhs = NULL; + ParseTree* rhs = NULL; + PredicateOperator* op_LE = new PredicateOperator("<="); + PredicateOperator* op_GE = new PredicateOperator(">="); - SubSelect* sub2 = dynamic_cast(op3); - fSub = (Item_subselect*)(fFunc->arguments()[2]); + SubSelect* sub2 = dynamic_cast(op3); + fSub = (Item_subselect*)(fFunc->arguments()[2]); - if (sub2) - { - rhs = buildParseTree(op_LE); - delete sub2; - } - else - { - SOP sop; - sop.reset(op_LE); - rhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op3)); - (dynamic_cast(rhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); - } + if (sub2) + { + rhs = buildParseTree(op_LE); + delete sub2; + } + else + { + SOP sop; + sop.reset(op_LE); + rhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op3)); + (dynamic_cast(rhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + } - SubSelect* sub1 = dynamic_cast(op2); - fSub = (Item_subselect*)(fFunc->arguments()[1]); + SubSelect* sub1 = dynamic_cast(op2); + fSub = (Item_subselect*)(fFunc->arguments()[1]); - if (sub1) - { - lhs = buildParseTree(op_GE); - delete sub1; - } - else - { - SOP sop; - sop.reset(op_GE); - lhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op2)); - (dynamic_cast(lhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); - } + if (sub1) + { + lhs = buildParseTree(op_GE); + delete sub1; + } + else + { + SOP sop; + sop.reset(op_GE); + lhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op2)); + (dynamic_cast(lhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + } - if (!rhs || !lhs) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = "non-supported scalar subquery"; - fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); - return NULL; - } + if (!rhs || !lhs) + { + fGwip.fatalParseError = true; + fGwip.parseErrorText = "non-supported scalar subquery"; + fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); + return NULL; + } - ParseTree* pt = new ParseTree (new LogicOperator("and")); - pt->left(lhs); - pt->right(rhs); - return pt; + ParseTree* pt = new ParseTree(new LogicOperator("and")); + pt->left(lhs); + pt->right(rhs); + return pt; } execplan::ParseTree* ScalarSub::transform_in() { - fGwip.fatalParseError = true; - fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); - return NULL; + fGwip.fatalParseError = true; + fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); + return NULL; } execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) { - idbassert(fColumn.get() && fSub && fFunc); + idbassert(fColumn.get() && fSub && fFunc); - vector cols; - Filter* filter; - RowColumn* rcol = dynamic_cast(fColumn.get()); + vector cols; + Filter* filter; + RowColumn* rcol = dynamic_cast(fColumn.get()); - if (rcol) + if (rcol) + { + // IDB only supports (c1,c2..) =/!= (subquery) + if (fFunc->functype() != Item_func::EQ_FUNC && fFunc->functype() != Item_func::NE_FUNC) { - // IDB only supports (c1,c2..) =/!= (subquery) - if (fFunc->functype() != Item_func::EQ_FUNC && fFunc->functype() != Item_func::NE_FUNC) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_OPERATOR_WITH_LIST); - return NULL; - } - - cols = rcol->columnVec(); - } - else - cols.push_back(fColumn); - - SCSEP csep(new CalpontSelectExecutionPlan()); - csep->sessionID(fGwip.sessionid); - csep->location(CalpontSelectExecutionPlan::WHERE); - csep->subType (CalpontSelectExecutionPlan::SINGLEROW_SUBS); - - // gwi for the sub query - gp_walk_info gwi; - gwi.thd = fGwip.thd; - gwi.subQuery = this; - - // @4827 merge table list to gwi in case there is FROM sub to be referenced - // in the FROM sub - gwi.derivedTbCnt = fGwip.derivedTbList.size(); - uint32_t tbCnt = fGwip.tbList.size(); - - gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); - gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) - { - //@todo more in error handling - if (!gwi.fatalParseError) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = "Error occured in ScalarSub::transform()"; - } - else - { - fGwip.fatalParseError = gwi.fatalParseError; - fGwip.parseErrorText = gwi.parseErrorText; - } - - return NULL; + fGwip.fatalParseError = true; + fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_OPERATOR_WITH_LIST); + return NULL; } - fGwip.subselectList.push_back(csep); + cols = rcol->columnVec(); + } + else + cols.push_back(fColumn); - // error out non-support case for now: comparison out of semi join tables. - // only check for simplecolumn - if (!gwi.correlatedTbNameVec.empty()) + SCSEP csep(new CalpontSelectExecutionPlan()); + csep->sessionID(fGwip.sessionid); + csep->location(CalpontSelectExecutionPlan::WHERE); + csep->subType(CalpontSelectExecutionPlan::SINGLEROW_SUBS); + + // gwi for the sub query + gp_walk_info gwi; + gwi.thd = fGwip.thd; + gwi.subQuery = this; + + // @4827 merge table list to gwi in case there is FROM sub to be referenced + // in the FROM sub + gwi.derivedTbCnt = fGwip.derivedTbList.size(); + uint32_t tbCnt = fGwip.tbList.size(); + + gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); + gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); + + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) + { + //@todo more in error handling + if (!gwi.fatalParseError) { - for (uint32_t i = 0; i < cols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(cols[i].get()); - - if (sc) - { - CalpontSystemCatalog::TableAliasName tan = make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias()); - uint32_t j = 0; - - for (; j < gwi.correlatedTbNameVec.size(); j++) - if (tan == gwi.correlatedTbNameVec[j]) - break; - - if (j == gwi.correlatedTbNameVec.size()) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); - return NULL; - } - } - } - } - - // remove outer query tables - CalpontSelectExecutionPlan::TableList tblist; - - if (csep->tableList().size() >= tbCnt) - tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); - - CalpontSelectExecutionPlan::SelectList derivedTbList; - - if (csep->derivedTableList().size() >= gwi.derivedTbCnt) - derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, csep->derivedTableList().end()); - - csep->tableList(tblist); - csep->derivedTableList(derivedTbList); - -// if (fSub->is_correlated) - if (fSub->unit->first_select()->master_unit()->uncacheable) - { - SelectFilter* subFilter = new SelectFilter(); - subFilter->correlated(true); - subFilter->cols(cols); - subFilter->sub(csep); - subFilter->op(SOP(op)); - subFilter->returnedColPos(fReturnedColPos); - filter = subFilter; + fGwip.fatalParseError = true; + fGwip.parseErrorText = "Error occured in ScalarSub::transform()"; } else { - SimpleScalarFilter* subFilter = new SimpleScalarFilter(); - subFilter->cols(cols); - subFilter->sub(csep); - subFilter->op(SOP(op)); - filter = subFilter; + fGwip.fatalParseError = gwi.fatalParseError; + fGwip.parseErrorText = gwi.parseErrorText; } - return new ParseTree(filter); + return NULL; + } + fGwip.subselectList.push_back(csep); + + // error out non-support case for now: comparison out of semi join tables. + // only check for simplecolumn + if (!gwi.correlatedTbNameVec.empty()) + { + for (uint32_t i = 0; i < cols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(cols[i].get()); + + if (sc) + { + CalpontSystemCatalog::TableAliasName tan = + make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias()); + uint32_t j = 0; + + for (; j < gwi.correlatedTbNameVec.size(); j++) + if (tan == gwi.correlatedTbNameVec[j]) + break; + + if (j == gwi.correlatedTbNameVec.size()) + { + fGwip.fatalParseError = true; + fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SCALAR); + return NULL; + } + } + } + } + + // remove outer query tables + CalpontSelectExecutionPlan::TableList tblist; + + if (csep->tableList().size() >= tbCnt) + tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); + + CalpontSelectExecutionPlan::SelectList derivedTbList; + + if (csep->derivedTableList().size() >= gwi.derivedTbCnt) + derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, + csep->derivedTableList().end()); + + csep->tableList(tblist); + csep->derivedTableList(derivedTbList); + + // if (fSub->is_correlated) + if (fSub->unit->first_select()->master_unit()->uncacheable) + { + SelectFilter* subFilter = new SelectFilter(); + subFilter->correlated(true); + subFilter->cols(cols); + subFilter->sub(csep); + subFilter->op(SOP(op)); + subFilter->returnedColPos(fReturnedColPos); + filter = subFilter; + } + else + { + SimpleScalarFilter* subFilter = new SimpleScalarFilter(); + subFilter->cols(cols); + subFilter->sub(csep); + subFilter->op(SOP(op)); + filter = subFilter; + } + + return new ParseTree(filter); } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_select_sub.cpp b/dbcon/mysql/ha_select_sub.cpp index 000c61720..897da428e 100644 --- a/dbcon/mysql/ha_select_sub.cpp +++ b/dbcon/mysql/ha_select_sub.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_select_sub.cpp 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ha_select_sub.cpp 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** class SelectSubQuery definition */ //#define NDEBUG @@ -47,70 +47,72 @@ using namespace logging; namespace cal_impl_if { -SelectSubQuery::SelectSubQuery(gp_walk_info& gwip) : SubQuery(gwip), fSelSub (NULL) -{} +SelectSubQuery::SelectSubQuery(gp_walk_info& gwip) : SubQuery(gwip), fSelSub(NULL) +{ +} -SelectSubQuery::SelectSubQuery(gp_walk_info& gwip, Item_subselect* selSub) : - SubQuery(gwip), - fSelSub(selSub) -{} +SelectSubQuery::SelectSubQuery(gp_walk_info& gwip, Item_subselect* selSub) : SubQuery(gwip), fSelSub(selSub) +{ +} SelectSubQuery::~SelectSubQuery() -{} +{ +} SCSEP SelectSubQuery::transform() { - idbassert(fSelSub); - SCSEP csep(new CalpontSelectExecutionPlan()); - csep->sessionID(fGwip.sessionid); - csep->subType (CalpontSelectExecutionPlan::SELECT_SUBS); + idbassert(fSelSub); + SCSEP csep(new CalpontSelectExecutionPlan()); + csep->sessionID(fGwip.sessionid); + csep->subType(CalpontSelectExecutionPlan::SELECT_SUBS); - // gwi for the sub query - gp_walk_info gwi; - gwi.thd = fGwip.thd; - gwi.subQuery = this; + // gwi for the sub query + gp_walk_info gwi; + gwi.thd = fGwip.thd; + gwi.subQuery = this; - // @4632 merge table list to gwi in case there is FROM sub to be referenced - // in the SELECT sub - gwi.derivedTbCnt = fGwip.derivedTbList.size(); - uint32_t tbCnt = fGwip.tbList.size(); + // @4632 merge table list to gwi in case there is FROM sub to be referenced + // in the SELECT sub + gwi.derivedTbCnt = fGwip.derivedTbList.size(); + uint32_t tbCnt = fGwip.tbList.size(); - gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); - gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); + gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); + gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSelSub->get_select_lex()), csep, false) != 0) + if (getSelectPlan(gwi, *(fSelSub->get_select_lex()), csep, false) != 0) + { + if (!gwi.fatalParseError) { - if (!gwi.fatalParseError) - { - fGwip.fatalParseError = true; - fGwip.parseErrorText = "Error occured in SelectSubQuery::transform()"; - } - else - { - fGwip.fatalParseError = gwi.fatalParseError; - fGwip.parseErrorText = gwi.parseErrorText; - } - - csep.reset(); - return csep; + fGwip.fatalParseError = true; + fGwip.parseErrorText = "Error occured in SelectSubQuery::transform()"; + } + else + { + fGwip.fatalParseError = gwi.fatalParseError; + fGwip.parseErrorText = gwi.parseErrorText; } - fGwip.subselectList.push_back(csep); - - // remove outer query tables - CalpontSelectExecutionPlan::TableList tblist; - - if (csep->tableList().size() >= tbCnt) - tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); - - CalpontSelectExecutionPlan::SelectList derivedTbList; - - if (csep->derivedTableList().size() >= gwi.derivedTbCnt) - derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, csep->derivedTableList().end()); - - csep->tableList(tblist); - csep->derivedTableList(derivedTbList); + csep.reset(); return csep; + } + + fGwip.subselectList.push_back(csep); + + // remove outer query tables + CalpontSelectExecutionPlan::TableList tblist; + + if (csep->tableList().size() >= tbCnt) + tblist.insert(tblist.begin(), csep->tableList().begin() + tbCnt, csep->tableList().end()); + + CalpontSelectExecutionPlan::SelectList derivedTbList; + + if (csep->derivedTableList().size() >= gwi.derivedTbCnt) + derivedTbList.insert(derivedTbList.begin(), csep->derivedTableList().begin() + gwi.derivedTbCnt, + csep->derivedTableList().end()); + + csep->tableList(tblist); + csep->derivedTableList(derivedTbList); + return csep; } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index 03cd48033..977de979d 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_subquery.h 6418 2010-03-29 21:55:08Z zzhu $ -* -* -***********************************************************************/ + * $Id: ha_subquery.h 6418 2010-03-29 21:55:08Z zzhu $ + * + * + ***********************************************************************/ /** @file */ /** class subquery series interface */ @@ -39,34 +39,39 @@ class PredicateOperator; namespace cal_impl_if { - /** @file */ class SubQuery { -public: - SubQuery(gp_walk_info& gwip) : - fGwip(gwip), - fCorrelated(false) - {} - virtual ~SubQuery() {} - virtual gp_walk_info& gwip() const - { - return fGwip; - } - bool correlated() const - { - return fCorrelated; - } - void correlated (const bool correlated) - { - fCorrelated = correlated; - } - virtual void handleFunc (gp_walk_info* gwip, Item_func* func) {} - virtual void handleNot () {} -protected: - gp_walk_info& fGwip; - bool fCorrelated; + public: + SubQuery(gp_walk_info& gwip) : fGwip(gwip), fCorrelated(false) + { + } + virtual ~SubQuery() + { + } + virtual gp_walk_info& gwip() const + { + return fGwip; + } + bool correlated() const + { + return fCorrelated; + } + void correlated(const bool correlated) + { + fCorrelated = correlated; + } + virtual void handleFunc(gp_walk_info* gwip, Item_func* func) + { + } + virtual void handleNot() + { + } + + protected: + gp_walk_info& fGwip; + bool fCorrelated; }; /** @@ -74,41 +79,47 @@ protected: */ class WhereSubQuery : public SubQuery { -public: - WhereSubQuery(gp_walk_info& gwip) : SubQuery(gwip), - fSub(NULL), - fFunc(NULL) {} - WhereSubQuery(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func) : - SubQuery(gwip), - fColumn(column), - fSub(sub), - fFunc(func) {} - WhereSubQuery(gp_walk_info& gwip, Item_func* func) : SubQuery(gwip), fFunc(func) {} - WhereSubQuery(gp_walk_info& gwip, Item_subselect* sub) : SubQuery(gwip), fSub(sub) {} // for exists - virtual ~WhereSubQuery() {} + public: + WhereSubQuery(gp_walk_info& gwip) : SubQuery(gwip), fSub(NULL), fFunc(NULL) + { + } + WhereSubQuery(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func) + : SubQuery(gwip), fColumn(column), fSub(sub), fFunc(func) + { + } + WhereSubQuery(gp_walk_info& gwip, Item_func* func) : SubQuery(gwip), fFunc(func) + { + } + WhereSubQuery(gp_walk_info& gwip, Item_subselect* sub) : SubQuery(gwip), fSub(sub) + { + } // for exists + virtual ~WhereSubQuery() + { + } - /** Accessors and mutators */ - virtual Item_subselect* sub() const - { - return fSub; - } - virtual void sub(Item_subselect* sub) - { - fSub = sub; - } - virtual Item_func* func() const - { - return fFunc; - } - virtual void func(Item_func* func) - { - fFunc = func; - } - virtual execplan::ParseTree* transform() = 0; -protected: - execplan::SRCP fColumn; - Item_subselect* fSub; - Item_func* fFunc; + /** Accessors and mutators */ + virtual Item_subselect* sub() const + { + return fSub; + } + virtual void sub(Item_subselect* sub) + { + fSub = sub; + } + virtual Item_func* func() const + { + return fFunc; + } + virtual void func(Item_func* func) + { + fFunc = func; + } + virtual execplan::ParseTree* transform() = 0; + + protected: + execplan::SRCP fColumn; + Item_subselect* fSub; + Item_func* fFunc; }; /** @@ -116,27 +127,27 @@ protected: */ class ScalarSub : public WhereSubQuery { -public: - ScalarSub(gp_walk_info& gwip); - ScalarSub(gp_walk_info& gwip, Item_func* func); - ScalarSub(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func); - ScalarSub(const ScalarSub& rhs); - ~ScalarSub(); - execplan::ParseTree* transform(); - execplan::ParseTree* transform_between(); - execplan::ParseTree* transform_in(); - execplan::ParseTree* buildParseTree(execplan::PredicateOperator* op); - uint64_t returnedColPos() const - { - return fReturnedColPos; - } - void returnedColPos(const uint64_t returnedColPos) - { - fReturnedColPos = returnedColPos; - } + public: + ScalarSub(gp_walk_info& gwip); + ScalarSub(gp_walk_info& gwip, Item_func* func); + ScalarSub(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func); + ScalarSub(const ScalarSub& rhs); + ~ScalarSub(); + execplan::ParseTree* transform(); + execplan::ParseTree* transform_between(); + execplan::ParseTree* transform_in(); + execplan::ParseTree* buildParseTree(execplan::PredicateOperator* op); + uint64_t returnedColPos() const + { + return fReturnedColPos; + } + void returnedColPos(const uint64_t returnedColPos) + { + fReturnedColPos = returnedColPos; + } -private: - uint64_t fReturnedColPos; + private: + uint64_t fReturnedColPos; }; /** @@ -144,14 +155,14 @@ private: */ class InSub : public WhereSubQuery { -public: - InSub(gp_walk_info& gwip); - InSub(gp_walk_info& gwip, Item_func* func); - InSub(const InSub& rhs); - ~InSub(); - execplan::ParseTree* transform(); - void handleFunc(gp_walk_info* gwip, Item_func* func); - void handleNot(); + public: + InSub(gp_walk_info& gwip); + InSub(gp_walk_info& gwip, Item_func* func); + InSub(const InSub& rhs); + ~InSub(); + execplan::ParseTree* transform(); + void handleFunc(gp_walk_info* gwip, Item_func* func); + void handleNot(); }; /** @@ -159,12 +170,12 @@ public: */ class ExistsSub : public WhereSubQuery { -public: - ExistsSub(gp_walk_info&); // not complete. just for compile - ExistsSub(gp_walk_info&, Item_subselect* sub); - ~ExistsSub(); - execplan::ParseTree* transform(); - void handleNot(); + public: + ExistsSub(gp_walk_info&); // not complete. just for compile + ExistsSub(gp_walk_info&, Item_subselect* sub); + ~ExistsSub(); + execplan::ParseTree* transform(); + void handleNot(); }; /** @@ -179,51 +190,53 @@ class AggregateSub : public WhereSubQuery */ class FromSubQuery : public SubQuery { -public: - FromSubQuery(gp_walk_info&); - FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub); - ~FromSubQuery(); - const SELECT_LEX* fromSub() const - { - return fFromSub; - } - void fromSub(SELECT_LEX* fromSub) - { - fFromSub = fromSub; - } - const std::string alias () const - { - return fAlias; - } - void alias (const std::string alias) - { - fAlias = alias; - } - execplan::SCSEP transform(); -private: - SELECT_LEX* fFromSub; - std::string fAlias; + public: + FromSubQuery(gp_walk_info&); + FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub); + ~FromSubQuery(); + const SELECT_LEX* fromSub() const + { + return fFromSub; + } + void fromSub(SELECT_LEX* fromSub) + { + fFromSub = fromSub; + } + const std::string alias() const + { + return fAlias; + } + void alias(const std::string alias) + { + fAlias = alias; + } + execplan::SCSEP transform(); + + private: + SELECT_LEX* fFromSub; + std::string fAlias; }; -class SelectSubQuery : public SubQuery +class SelectSubQuery : public SubQuery { -public: - SelectSubQuery(gp_walk_info&); - SelectSubQuery(gp_walk_info&, Item_subselect* sel); - ~SelectSubQuery(); - execplan::SCSEP transform(); - Item_subselect* selSub() - { - return fSelSub; - } - void selSub( Item_subselect* selSub ) - { - fSelSub = selSub; - } -private: - Item_subselect* fSelSub; + public: + SelectSubQuery(gp_walk_info&); + SelectSubQuery(gp_walk_info&, Item_subselect* sel); + ~SelectSubQuery(); + execplan::SCSEP transform(); + Item_subselect* selSub() + { + return fSelSub; + } + void selSub(Item_subselect* selSub) + { + fSelSub = selSub; + } + + private: + Item_subselect* fSelSub; }; -} +} // namespace cal_impl_if #endif diff --git a/dbcon/mysql/ha_tzinfo.cpp b/dbcon/mysql/ha_tzinfo.cpp index be9a70a41..b866e9db6 100644 --- a/dbcon/mysql/ha_tzinfo.cpp +++ b/dbcon/mysql/ha_tzinfo.cpp @@ -21,55 +21,49 @@ /* needed for sql_base.h which contains open_system_tables_for_read*/ #ifndef TRUE -#define TRUE (1) /* Logical true */ -#define FALSE (0) /* Logical false */ +#define TRUE (1) /* Logical true */ +#define FALSE (0) /* Logical false */ #endif #include -//static HASH tz_names; -//static HASH offset_tzs; +// static HASH tz_names; +// static HASH offset_tzs; static MEM_ROOT tz_storage; -static uint tz_leapcnt= 0; -static LS_INFO *tz_lsis= 0; -static bool time_zone_tables_exist= 1; +static uint tz_leapcnt = 0; +static LS_INFO* tz_lsis = 0; +static bool time_zone_tables_exist = 1; namespace cal_impl_if { - /* Names of tables (with their lengths) that are needed for dynamical loading of time zone descriptions. */ -static const LEX_CSTRING tz_tables_names[MY_TZ_TABLES_COUNT]= -{ - { STRING_WITH_LEN("time_zone_name")}, - { STRING_WITH_LEN("time_zone")}, - { STRING_WITH_LEN("time_zone_transition_type")}, - { STRING_WITH_LEN("time_zone_transition")} -}; +static const LEX_CSTRING tz_tables_names[MY_TZ_TABLES_COUNT] = { + {STRING_WITH_LEN("time_zone_name")}, + {STRING_WITH_LEN("time_zone")}, + {STRING_WITH_LEN("time_zone_transition_type")}, + {STRING_WITH_LEN("time_zone_transition")}}; -static void -tz_init_table_list(TABLE_LIST *tz_tabs) +static void tz_init_table_list(TABLE_LIST* tz_tabs) { - for (int i= 0; i < MY_TZ_TABLES_COUNT; i++) + for (int i = 0; i < MY_TZ_TABLES_COUNT; i++) { - tz_tabs[i].init_one_table(&MYSQL_SCHEMA_NAME, tz_tables_names + i, - NULL, TL_READ); + tz_tabs[i].init_one_table(&MYSQL_SCHEMA_NAME, tz_tables_names + i, NULL, TL_READ); if (i != MY_TZ_TABLES_COUNT - 1) - tz_tabs[i].next_global= tz_tabs[i].next_local= &tz_tabs[i+1]; + tz_tabs[i].next_global = tz_tabs[i].next_local = &tz_tabs[i + 1]; if (i != 0) - tz_tabs[i].prev_global= &tz_tabs[i-1].next_global; + tz_tabs[i].prev_global = &tz_tabs[i - 1].next_global; } } -static my_bool -prepare_tz_info(TIME_ZONE_INFO *sp, MEM_ROOT *storage) +static my_bool prepare_tz_info(TIME_ZONE_INFO* sp, MEM_ROOT* storage) { - my_time_t cur_t= MY_TIME_T_MIN; + my_time_t cur_t = MY_TIME_T_MIN; my_time_t cur_l, end_t; my_time_t end_l = 0; - my_time_t cur_max_seen_l= MY_TIME_T_MIN; + my_time_t cur_max_seen_l = MY_TIME_T_MIN; long cur_offset, cur_corr, cur_off_and_corr; uint next_trans_idx, next_leap_idx; uint i; @@ -87,17 +81,16 @@ prepare_tz_info(TIME_ZONE_INFO *sp, MEM_ROOT *storage) We will find first non-DST local time type and use it (or use first local time type if all of them are DST types). */ - for (i= 0; i < sp->typecnt && sp->ttis[i].tt_isdst; i++) - /* no-op */ ; + for (i = 0; i < sp->typecnt && sp->ttis[i].tt_isdst; i++) + /* no-op */; if (i == sp->typecnt) - i= 0; - sp->fallback_tti= &(sp->ttis[i]); - + i = 0; + sp->fallback_tti = &(sp->ttis[i]); /* Let us build shifted my_time_t -> my_time_t map. */ - sp->revcnt= 0; + sp->revcnt = 0; /* Let us find initial offset */ if (sp->timecnt == 0 || cur_t < sp->ats[0]) @@ -106,105 +99,96 @@ prepare_tz_info(TIME_ZONE_INFO *sp, MEM_ROOT *storage) If we have not any transitions or t is before first transition we are using already found fallback time type which index is already in i. */ - next_trans_idx= 0; + next_trans_idx = 0; } else { /* cur_t == sp->ats[0] so we found transition */ - i= sp->types[0]; - next_trans_idx= 1; + i = sp->types[0]; + next_trans_idx = 1; } - cur_offset= sp->ttis[i].tt_gmtoff; - + cur_offset = sp->ttis[i].tt_gmtoff; /* let us find leap correction... unprobable, but... */ - for (next_leap_idx= 0; next_leap_idx < sp->leapcnt && - cur_t >= sp->lsis[next_leap_idx].ls_trans; - ++next_leap_idx) + for (next_leap_idx = 0; next_leap_idx < sp->leapcnt && cur_t >= sp->lsis[next_leap_idx].ls_trans; + ++next_leap_idx) continue; if (next_leap_idx > 0) - cur_corr= sp->lsis[next_leap_idx - 1].ls_corr; + cur_corr = sp->lsis[next_leap_idx - 1].ls_corr; else - cur_corr= 0; + cur_corr = 0; /* Iterate trough t space */ while (sp->revcnt < TZ_MAX_REV_RANGES - 1) { - cur_off_and_corr= cur_offset - cur_corr; + cur_off_and_corr = cur_offset - cur_corr; /* We assuming that cur_t could be only overflowed downwards, we also assume that end_t won't be overflowed in this case. */ - if (cur_off_and_corr < 0 && - cur_t < MY_TIME_T_MIN - cur_off_and_corr) - cur_t= MY_TIME_T_MIN - cur_off_and_corr; + if (cur_off_and_corr < 0 && cur_t < MY_TIME_T_MIN - cur_off_and_corr) + cur_t = MY_TIME_T_MIN - cur_off_and_corr; - cur_l= cur_t + cur_off_and_corr; + cur_l = cur_t + cur_off_and_corr; /* Let us choose end_t as point before next time type change or leap second correction. */ - end_t= MY_MIN((next_trans_idx < sp->timecnt) ? sp->ats[next_trans_idx] - 1: - MY_TIME_T_MAX, - (next_leap_idx < sp->leapcnt) ? - sp->lsis[next_leap_idx].ls_trans - 1: MY_TIME_T_MAX); + end_t = MY_MIN((next_trans_idx < sp->timecnt) ? sp->ats[next_trans_idx] - 1 : MY_TIME_T_MAX, + (next_leap_idx < sp->leapcnt) ? sp->lsis[next_leap_idx].ls_trans - 1 : MY_TIME_T_MAX); /* again assuming that end_t can be overlowed only in positive side we also assume that end_t won't be overflowed in this case. */ - if (cur_off_and_corr > 0 && - end_t > MY_TIME_T_MAX - cur_off_and_corr) - end_t= MY_TIME_T_MAX - cur_off_and_corr; - - end_l= end_t + cur_off_and_corr; + if (cur_off_and_corr > 0 && end_t > MY_TIME_T_MAX - cur_off_and_corr) + end_t = MY_TIME_T_MAX - cur_off_and_corr; + end_l = end_t + cur_off_and_corr; if (end_l > cur_max_seen_l) { /* We want special handling in the case of first range */ if (cur_max_seen_l == MY_TIME_T_MIN) { - revts[sp->revcnt]= cur_l; - revtis[sp->revcnt].rt_offset= cur_off_and_corr; - revtis[sp->revcnt].rt_type= 0; + revts[sp->revcnt] = cur_l; + revtis[sp->revcnt].rt_offset = cur_off_and_corr; + revtis[sp->revcnt].rt_type = 0; sp->revcnt++; - cur_max_seen_l= end_l; + cur_max_seen_l = end_l; } else { if (cur_l > cur_max_seen_l + 1) { /* We have a spring time-gap and we are not at the first range */ - revts[sp->revcnt]= cur_max_seen_l + 1; - revtis[sp->revcnt].rt_offset= revtis[sp->revcnt-1].rt_offset; - revtis[sp->revcnt].rt_type= 1; + revts[sp->revcnt] = cur_max_seen_l + 1; + revtis[sp->revcnt].rt_offset = revtis[sp->revcnt - 1].rt_offset; + revtis[sp->revcnt].rt_type = 1; sp->revcnt++; if (sp->revcnt == TZ_MAX_TIMES + TZ_MAX_LEAPS + 1) break; /* That was too much */ - cur_max_seen_l= cur_l - 1; + cur_max_seen_l = cur_l - 1; } /* Assume here end_l > cur_max_seen_l (because end_l>=cur_l) */ - revts[sp->revcnt]= cur_max_seen_l + 1; - revtis[sp->revcnt].rt_offset= cur_off_and_corr; - revtis[sp->revcnt].rt_type= 0; + revts[sp->revcnt] = cur_max_seen_l + 1; + revtis[sp->revcnt].rt_offset = cur_off_and_corr; + revtis[sp->revcnt].rt_type = 0; sp->revcnt++; - cur_max_seen_l= end_l; + cur_max_seen_l = end_l; } } - if (end_t == MY_TIME_T_MAX || - ((cur_off_and_corr > 0) && - (end_t >= MY_TIME_T_MAX - cur_off_and_corr))) + if (end_t == MY_TIME_T_MAX || ((cur_off_and_corr > 0) && (end_t >= MY_TIME_T_MAX - cur_off_and_corr))) /* end of t space */ break; - cur_t= end_t + 1; + cur_t = end_t + 1; /* Let us find new offset and correction. Because of our choice of end_t @@ -212,19 +196,17 @@ prepare_tz_info(TIME_ZONE_INFO *sp, MEM_ROOT *storage) correction is performed. */ if (sp->timecnt != 0 && cur_t >= sp->ats[0]) /* else reuse old offset */ - if (next_trans_idx < sp->timecnt && - cur_t == sp->ats[next_trans_idx]) + if (next_trans_idx < sp->timecnt && cur_t == sp->ats[next_trans_idx]) { /* We are at offset point */ - cur_offset= sp->ttis[sp->types[next_trans_idx]].tt_gmtoff; + cur_offset = sp->ttis[sp->types[next_trans_idx]].tt_gmtoff; ++next_trans_idx; } - if (next_leap_idx < sp->leapcnt && - cur_t == sp->lsis[next_leap_idx].ls_trans) + if (next_leap_idx < sp->leapcnt && cur_t == sp->lsis[next_leap_idx].ls_trans) { /* we are at leap point */ - cur_corr= sp->lsis[next_leap_idx].ls_corr; + cur_corr = sp->lsis[next_leap_idx].ls_corr; ++next_leap_idx; } } @@ -234,13 +216,11 @@ prepare_tz_info(TIME_ZONE_INFO *sp, MEM_ROOT *storage) return 1; /* set maximum end_l as finisher */ - revts[sp->revcnt]= end_l; + revts[sp->revcnt] = end_l; /* Allocate arrays of proper size in sp and copy result there */ - if (!(sp->revts= (my_time_t *)alloc_root(storage, - sizeof(my_time_t) * (sp->revcnt + 1))) || - !(sp->revtis= (REVT_INFO *)alloc_root(storage, - sizeof(REVT_INFO) * sp->revcnt))) + if (!(sp->revts = (my_time_t*)alloc_root(storage, sizeof(my_time_t) * (sp->revcnt + 1))) || + !(sp->revtis = (REVT_INFO*)alloc_root(storage, sizeof(REVT_INFO) * sp->revcnt))) return 1; memcpy(sp->revts, revts, sizeof(my_time_t) * (sp->revcnt + 1)); @@ -249,21 +229,21 @@ prepare_tz_info(TIME_ZONE_INFO *sp, MEM_ROOT *storage) return 0; } -static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables) +static TIME_ZONE_INFO* tz_load_from_open_tables(const String* tz_name, TABLE_LIST* tz_tables) { - TABLE *table= 0; - TIME_ZONE_INFO *tz_info= NULL; - //Tz_names_entry *tmp_tzname; - TIME_ZONE_INFO *return_val= 0; + TABLE* table = 0; + TIME_ZONE_INFO* tz_info = NULL; + // Tz_names_entry *tmp_tzname; + TIME_ZONE_INFO* return_val = 0; int res; uint tzid, ttid; my_time_t ttime; char buff[MAX_FIELD_WIDTH]; uchar keybuff[32]; - Field *field; + Field* field; String abbr(buff, sizeof(buff), &my_charset_latin1); - char *alloc_buff= NULL; - char *tz_name_buff= NULL; + char* alloc_buff = NULL; + char* tz_name_buff = NULL; /* Temporary arrays that are used for loading of data for filling TIME_ZONE_INFO structure @@ -287,28 +267,25 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS Let us find out time zone id by its name (there is only one index and it is specifically for this purpose). */ - table= tz_tables->table; - tz_tables= tz_tables->next_local; - table->field[0]->store(tz_name->ptr(), tz_name->length(), - &my_charset_latin1); + table = tz_tables->table; + tz_tables = tz_tables->next_local; + table->field[0]->store(tz_name->ptr(), tz_name->length(), &my_charset_latin1); if (table->file->ha_index_init(0, 1)) goto end; - if (table->file->ha_index_read_map(table->record[0], table->field[0]->ptr, - HA_WHOLE_KEY, HA_READ_KEY_EXACT)) + if (table->file->ha_index_read_map(table->record[0], table->field[0]->ptr, HA_WHOLE_KEY, HA_READ_KEY_EXACT)) { #ifdef EXTRA_DEBUG /* Most probably user has mistyped time zone name, so no need to bark here unless we need it for debugging. */ - sql_print_error("Can't find description of time zone '%.*b'", - tz_name->length(), tz_name->ptr()); + sql_print_error("Can't find description of time zone '%.*b'", tz_name->length(), tz_name->ptr()); #endif goto end; } - tzid= (uint)table->field[1]->val_int(); + tzid = (uint)table->field[1]->val_int(); (void)table->file->ha_index_end(); @@ -317,19 +294,16 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS understand whenever this timezone uses leap seconds (again we are using the only index in this table). */ - table= tz_tables->table; - tz_tables= tz_tables->next_local; - field= table->field[0]; - field->store((longlong) tzid, true); + table = tz_tables->table; + tz_tables = tz_tables->next_local; + field = table->field[0]; + field->store((longlong)tzid, true); DBUG_ASSERT(field->key_length() <= sizeof(keybuff)); - field->get_key_image(keybuff, - MY_MIN(field->key_length(), sizeof(keybuff)), - Field::itRAW); + field->get_key_image(keybuff, MY_MIN(field->key_length(), sizeof(keybuff)), Field::itRAW); if (table->file->ha_index_init(0, 1)) goto end; - if (table->file->ha_index_read_map(table->record[0], keybuff, - HA_WHOLE_KEY, HA_READ_KEY_EXACT)) + if (table->file->ha_index_read_map(table->record[0], keybuff, HA_WHOLE_KEY, HA_READ_KEY_EXACT)) { sql_print_error("Can't find description of time zone '%u'", tzid); goto end; @@ -338,8 +312,8 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS /* If Uses_leap_seconds == 'Y' */ if (table->field[1]->val_int() == 1) { - tmp_tz_info.leapcnt= tz_leapcnt; - tmp_tz_info.lsis= tz_lsis; + tmp_tz_info.leapcnt = tz_leapcnt; + tmp_tz_info.lsis = tz_lsis; } (void)table->file->ha_index_end(); @@ -350,116 +324,118 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS only for our time zone guess what are we doing? Right - using special index. */ - table= tz_tables->table; - tz_tables= tz_tables->next_local; - field= table->field[0]; - field->store((longlong) tzid, true); + table = tz_tables->table; + tz_tables = tz_tables->next_local; + field = table->field[0]; + field->store((longlong)tzid, true); DBUG_ASSERT(field->key_length() <= sizeof(keybuff)); - field->get_key_image(keybuff, - MY_MIN(field->key_length(), sizeof(keybuff)), - Field::itRAW); + field->get_key_image(keybuff, MY_MIN(field->key_length(), sizeof(keybuff)), Field::itRAW); if (table->file->ha_index_init(0, 1)) goto end; - res= table->file->ha_index_read_map(table->record[0], keybuff, - (key_part_map)1, HA_READ_KEY_EXACT); + res = table->file->ha_index_read_map(table->record[0], keybuff, (key_part_map)1, HA_READ_KEY_EXACT); while (!res) { - ttid= (uint)table->field[1]->val_int(); + ttid = (uint)table->field[1]->val_int(); if (ttid >= TZ_MAX_TYPES) { - sql_print_error("Error while loading time zone description from " - "mysql.time_zone_transition_type table: too big " - "transition type id"); + sql_print_error( + "Error while loading time zone description from " + "mysql.time_zone_transition_type table: too big " + "transition type id"); goto end; } - ttis[ttid].tt_gmtoff= (long)table->field[2]->val_int(); - ttis[ttid].tt_isdst= (table->field[3]->val_int() > 0); + ttis[ttid].tt_gmtoff = (long)table->field[2]->val_int(); + ttis[ttid].tt_isdst = (table->field[3]->val_int() > 0); #ifdef ABBR_ARE_USED // FIXME should we do something with duplicates here ? table->field[4]->val_str(&abbr, &abbr); if (tmp_tz_info.charcnt + abbr.length() + 1 > sizeof(chars)) { - sql_print_error("Error while loading time zone description from " - "mysql.time_zone_transition_type table: not enough " - "room for abbreviations"); + sql_print_error( + "Error while loading time zone description from " + "mysql.time_zone_transition_type table: not enough " + "room for abbreviations"); goto end; } - ttis[ttid].tt_abbrind= tmp_tz_info.charcnt; + ttis[ttid].tt_abbrind = tmp_tz_info.charcnt; memcpy(chars + tmp_tz_info.charcnt, abbr.ptr(), abbr.length()); - tmp_tz_info.charcnt+= abbr.length(); - chars[tmp_tz_info.charcnt]= 0; + tmp_tz_info.charcnt += abbr.length(); + chars[tmp_tz_info.charcnt] = 0; tmp_tz_info.charcnt++; - DBUG_PRINT("info",("time_zone_transition_type table: tz_id=%u tt_id=%u tt_gmtoff=%ld " - "abbr='%s' tt_isdst=%u", tzid, ttid, ttis[ttid].tt_gmtoff, - chars + ttis[ttid].tt_abbrind, ttis[ttid].tt_isdst)); + DBUG_PRINT("info", + ("time_zone_transition_type table: tz_id=%u tt_id=%u tt_gmtoff=%ld " + "abbr='%s' tt_isdst=%u", + tzid, ttid, ttis[ttid].tt_gmtoff, chars + ttis[ttid].tt_abbrind, ttis[ttid].tt_isdst)); #else - DBUG_PRINT("info",("time_zone_transition_type table: tz_id=%u tt_id=%u tt_gmtoff=%ld " - "tt_isdst=%u", tzid, ttid, ttis[ttid].tt_gmtoff, ttis[ttid].tt_isdst)); + DBUG_PRINT("info", ("time_zone_transition_type table: tz_id=%u tt_id=%u tt_gmtoff=%ld " + "tt_isdst=%u", + tzid, ttid, ttis[ttid].tt_gmtoff, ttis[ttid].tt_isdst)); #endif /* ttid is increasing because we are reading using index */ DBUG_ASSERT(ttid >= tmp_tz_info.typecnt); - tmp_tz_info.typecnt= ttid + 1; + tmp_tz_info.typecnt = ttid + 1; - res= table->file->ha_index_next_same(table->record[0], keybuff, 4); + res = table->file->ha_index_next_same(table->record[0], keybuff, 4); } if (res != HA_ERR_END_OF_FILE) { - sql_print_error("Error while loading time zone description from " - "mysql.time_zone_transition_type table"); + sql_print_error( + "Error while loading time zone description from " + "mysql.time_zone_transition_type table"); goto end; } (void)table->file->ha_index_end(); - /* At last we are doing the same thing for records in mysql.time_zone_transition table. Here we additionally need records in ascending order by index scan also satisfies us. */ - table= tz_tables->table; - table->field[0]->store((longlong) tzid, true); + table = tz_tables->table; + table->field[0]->store((longlong)tzid, true); if (table->file->ha_index_init(0, 1)) goto end; - res= table->file->ha_index_read_map(table->record[0], keybuff, - (key_part_map)1, HA_READ_KEY_EXACT); + res = table->file->ha_index_read_map(table->record[0], keybuff, (key_part_map)1, HA_READ_KEY_EXACT); while (!res) { - ttime= (my_time_t)table->field[1]->val_int(); - ttid= (uint)table->field[2]->val_int(); + ttime = (my_time_t)table->field[1]->val_int(); + ttid = (uint)table->field[2]->val_int(); if (tmp_tz_info.timecnt + 1 > TZ_MAX_TIMES) { - sql_print_error("Error while loading time zone description from " - "mysql.time_zone_transition table: " - "too much transitions"); + sql_print_error( + "Error while loading time zone description from " + "mysql.time_zone_transition table: " + "too much transitions"); goto end; } if (ttid + 1 > tmp_tz_info.typecnt) { - sql_print_error("Error while loading time zone description from " - "mysql.time_zone_transition table: " - "bad transition type id"); + sql_print_error( + "Error while loading time zone description from " + "mysql.time_zone_transition table: " + "bad transition type id"); goto end; } - ats[tmp_tz_info.timecnt]= ttime; - types[tmp_tz_info.timecnt]= ttid; + ats[tmp_tz_info.timecnt] = ttime; + types[tmp_tz_info.timecnt] = ttid; tmp_tz_info.timecnt++; - DBUG_PRINT("info", ("time_zone_transition table: tz_id: %u tt_time: %lu tt_id: %u", - tzid, (ulong) ttime, ttid)); + DBUG_PRINT("info", + ("time_zone_transition table: tz_id: %u tt_time: %lu tt_id: %u", tzid, (ulong)ttime, ttid)); - res= table->file->ha_index_next_same(table->record[0], keybuff, 4); + res = table->file->ha_index_next_same(table->record[0], keybuff, 4); } /* @@ -468,13 +444,14 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS */ if (res != HA_ERR_END_OF_FILE && res != HA_ERR_KEY_NOT_FOUND) { - sql_print_error("Error while loading time zone description from " - "mysql.time_zone_transition table"); + sql_print_error( + "Error while loading time zone description from " + "mysql.time_zone_transition table"); goto end; } (void)table->file->ha_index_end(); - table= 0; + table = 0; /* Let us check how correct our time zone description is. We don't check for @@ -487,16 +464,16 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS } /* Allocate memory for the timezone info and timezone name in tz_storage. */ - if (!(alloc_buff= (char*) alloc_root(&tz_storage, sizeof(TIME_ZONE_INFO) + tz_name->length() + 1))) + if (!(alloc_buff = (char*)alloc_root(&tz_storage, sizeof(TIME_ZONE_INFO) + tz_name->length() + 1))) { sql_print_error("Out of memory while loading time zone description"); return 0; } /* Move the temporary tz_info into the allocated area */ - tz_info= (TIME_ZONE_INFO *)alloc_buff; + tz_info = (TIME_ZONE_INFO*)alloc_buff; memcpy(tz_info, &tmp_tz_info, sizeof(TIME_ZONE_INFO)); - tz_name_buff= alloc_buff + sizeof(TIME_ZONE_INFO); + tz_name_buff = alloc_buff + sizeof(TIME_ZONE_INFO); /* By writing zero to the end we guarantee that we can call ptr() instead of c_ptr() for time zone name. @@ -506,32 +483,29 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS /* Now we will allocate memory and init TIME_ZONE_INFO structure. */ - if (!(alloc_buff= (char*) alloc_root(&tz_storage, - ALIGN_SIZE(sizeof(my_time_t) * - tz_info->timecnt) + - ALIGN_SIZE(tz_info->timecnt) + + if (!(alloc_buff = (char*)alloc_root(&tz_storage, ALIGN_SIZE(sizeof(my_time_t) * tz_info->timecnt) + + ALIGN_SIZE(tz_info->timecnt) + #ifdef ABBR_ARE_USED - ALIGN_SIZE(tz_info->charcnt) + + ALIGN_SIZE(tz_info->charcnt) + #endif - sizeof(TRAN_TYPE_INFO) * - tz_info->typecnt))) + sizeof(TRAN_TYPE_INFO) * tz_info->typecnt))) { sql_print_error("Out of memory while loading time zone description"); goto end; } - tz_info->ats= (my_time_t *) alloc_buff; + tz_info->ats = (my_time_t*)alloc_buff; memcpy(tz_info->ats, ats, tz_info->timecnt * sizeof(my_time_t)); - alloc_buff+= ALIGN_SIZE(sizeof(my_time_t) * tz_info->timecnt); - tz_info->types= (uchar *)alloc_buff; + alloc_buff += ALIGN_SIZE(sizeof(my_time_t) * tz_info->timecnt); + tz_info->types = (uchar*)alloc_buff; memcpy(tz_info->types, types, tz_info->timecnt); - alloc_buff+= ALIGN_SIZE(tz_info->timecnt); + alloc_buff += ALIGN_SIZE(tz_info->timecnt); #ifdef ABBR_ARE_USED - tz_info->chars= alloc_buff; + tz_info->chars = alloc_buff; memcpy(tz_info->chars, chars, tz_info->charcnt); - alloc_buff+= ALIGN_SIZE(tz_info->charcnt); + alloc_buff += ALIGN_SIZE(tz_info->charcnt); #endif - tz_info->ttis= (TRAN_TYPE_INFO *)alloc_buff; + tz_info->ttis = (TRAN_TYPE_INFO*)alloc_buff; memcpy(tz_info->ttis, ttis, tz_info->typecnt * sizeof(TRAN_TYPE_INFO)); /* Build reversed map. */ @@ -544,35 +518,34 @@ static TIME_ZONE_INFO* tz_load_from_open_tables(const String *tz_name, TABLE_LIS /* Loading of time zone succeeded */ - return_val= tz_info; + return_val = tz_info; end: if (table && table->file->inited) - (void) table->file->ha_index_end(); + (void)table->file->ha_index_end(); DBUG_RETURN(return_val); } -TIME_ZONE_INFO* my_tzinfo_find(THD *thd, const String *name) +TIME_ZONE_INFO* my_tzinfo_find(THD* thd, const String* name) { - TIME_ZONE_INFO* result_tzinfo= 0; + TIME_ZONE_INFO* result_tzinfo = 0; long offset; DBUG_ENTER("my_tz_find"); - DBUG_PRINT("enter", ("time zone name='%s'\n", - name ? ((String *)name)->c_ptr_safe() : "NULL")); + DBUG_PRINT("enter", ("time zone name='%s'\n", name ? ((String*)name)->c_ptr_safe() : "NULL")); if (!name || name->is_empty()) DBUG_RETURN(0); if (!timeZoneToOffset(name->ptr(), name->length(), &offset)) { - return NULL; + return NULL; } else { - result_tzinfo= 0; - my_tz_init(thd,NULL, 0); + result_tzinfo = 0; + my_tz_init(thd, NULL, 0); if (time_zone_tables_exist) { TABLE_LIST tz_tables[MY_TZ_TABLES_COUNT]; @@ -581,12 +554,12 @@ TIME_ZONE_INFO* my_tzinfo_find(THD *thd, const String *name) Allocate start_new_trans with malloc as it's > 4000 bytes and this function can be called deep inside a stored procedure */ - start_new_trans *new_trans= new start_new_trans(thd); + start_new_trans* new_trans = new start_new_trans(thd); tz_init_table_list(tz_tables); init_mdl_requests(tz_tables); if (!open_system_tables_for_read(thd, tz_tables)) { - result_tzinfo= tz_load_from_open_tables(name, tz_tables); + result_tzinfo = tz_load_from_open_tables(name, tz_tables); thd->commit_whole_transaction_and_close_tables(); } new_trans->restore_old_transaction(); @@ -597,4 +570,4 @@ TIME_ZONE_INFO* my_tzinfo_find(THD *thd, const String *name) DBUG_RETURN(result_tzinfo); } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_tzinfo.h b/dbcon/mysql/ha_tzinfo.h index 4f428651a..129c1fe26 100644 --- a/dbcon/mysql/ha_tzinfo.h +++ b/dbcon/mysql/ha_tzinfo.h @@ -26,8 +26,7 @@ using namespace dataconvert; namespace cal_impl_if { - -TIME_ZONE_INFO* my_tzinfo_find(THD *thd, const String *name); +TIME_ZONE_INFO* my_tzinfo_find(THD* thd, const String* name); } diff --git a/dbcon/mysql/ha_view.cpp b/dbcon/mysql/ha_view.cpp index e3b82b141..c37907e98 100644 --- a/dbcon/mysql/ha_view.cpp +++ b/dbcon/mysql/ha_view.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_view.cpp 9642 2013-06-24 14:57:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ha_view.cpp 9642 2013-06-24 14:57:42Z rdempsey $ + * + * + ***********************************************************************/ #define PREFER_MY_CONFIG_H #include @@ -46,147 +46,156 @@ using namespace execplan; namespace cal_impl_if { extern uint32_t buildJoin(gp_walk_info& gwi, List& join_list, - std::stack& outerJoinStack); + std::stack& outerJoinStack); extern string getViewName(TABLE_LIST* table_ptr); CalpontSystemCatalog::TableAliasName& View::viewName() { - return fViewName; + return fViewName; } void View::viewName(execplan::CalpontSystemCatalog::TableAliasName& viewName) { - fViewName = viewName; + fViewName = viewName; } void View::transform() { - CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); - csep->sessionID(fParentGwip->sessionid); + CalpontSelectExecutionPlan* csep = new CalpontSelectExecutionPlan(); + csep->sessionID(fParentGwip->sessionid); - // gwi for the sub query - gp_walk_info gwi; - gwi.thd = fParentGwip->thd; + // gwi for the sub query + gp_walk_info gwi; + gwi.thd = fParentGwip->thd; - uint32_t sessionID = csep->sessionID(); - gwi.sessionid = sessionID; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - gwi.csc = csc; + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + gwi.csc = csc; - // traverse the table list of the view - TABLE_LIST* table_ptr = fSelect.get_table_list(); - CalpontSelectExecutionPlan::SelectList derivedTbList; + // traverse the table list of the view + TABLE_LIST* table_ptr = fSelect.get_table_list(); + CalpontSelectExecutionPlan::SelectList derivedTbList; - // @bug 1796. Remember table order on the FROM list. - gwi.clauseType = FROM; + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; - try + try + { + for (; table_ptr; table_ptr = table_ptr->next_local) { - for (; table_ptr; table_ptr = table_ptr->next_local) + // mysql put vtable here for from sub. we ignore it + if (string(table_ptr->table_name.str).find("$vtable") != string::npos) + continue; + + string viewName = getViewName(table_ptr); + if (lower_case_table_names) + { + boost::algorithm::to_lower(viewName); + } + + if (table_ptr->derived) + { + SELECT_LEX* select_cursor = table_ptr->derived->first_select(); + FromSubQuery* fromSub = new FromSubQuery(gwi, select_cursor); + string alias(table_ptr->alias.str); + if (lower_case_table_names) { - // mysql put vtable here for from sub. we ignore it - if (string(table_ptr->table_name.str).find("$vtable") != string::npos) - continue; - - string viewName = getViewName(table_ptr); - if (lower_case_table_names) - { - boost::algorithm::to_lower(viewName); - } - - if (table_ptr->derived) - { - SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery* fromSub = new FromSubQuery(gwi, select_cursor); - string alias(table_ptr->alias.str); - if (lower_case_table_names) - { - boost::algorithm::to_lower(alias); - } - gwi.viewName = make_aliasview("", alias, table_ptr->belong_to_view->alias.str, "", true, lower_case_table_names); - fromSub->alias(alias); - gwi.derivedTbList.push_back(SCSEP(fromSub->transform())); - // set alias to both table name and alias name of the derived table - CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); - gwi.tbList.push_back(tn); - gwi.tableMap[tn] = make_pair(0, table_ptr); - // TODO MCOL-2178 isUnion member only assigned, never used - // MIGR::infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init - } - else if (table_ptr->view) - { - // for nested view, the view name is vout.vin... format - CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, viewName, true, lower_case_table_names); - gwi.viewName = make_aliastable(table_ptr->db.str, table_ptr->table_name.str, viewName, true, lower_case_table_names); - View* view = new View(*table_ptr->view->first_select_lex(), &gwi); - view->viewName(gwi.viewName); - gwi.viewList.push_back(view); - view->transform(); - } - else - { - // check foreign engine tables - bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true); - - // trigger system catalog cache - if (columnStore) - csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str, lower_case_table_names), true); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, viewName, columnStore, lower_case_table_names); - gwi.tbList.push_back(tn); - gwi.tableMap[tn] = make_pair(0, table_ptr); - fParentGwip->tableMap[tn] = make_pair(0, table_ptr); - } + boost::algorithm::to_lower(alias); } + gwi.viewName = + make_aliasview("", alias, table_ptr->belong_to_view->alias.str, "", true, lower_case_table_names); + fromSub->alias(alias); + gwi.derivedTbList.push_back(SCSEP(fromSub->transform())); + // set alias to both table name and alias name of the derived table + CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); + gwi.tbList.push_back(tn); + gwi.tableMap[tn] = make_pair(0, table_ptr); + // TODO MCOL-2178 isUnion member only assigned, never used + // MIGR::infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + } + else if (table_ptr->view) + { + // for nested view, the view name is vout.vin... format + CalpontSystemCatalog::TableAliasName tn = + make_aliasview(table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, viewName, true, + lower_case_table_names); + gwi.viewName = make_aliastable(table_ptr->db.str, table_ptr->table_name.str, viewName, true, + lower_case_table_names); + View* view = new View(*table_ptr->view->first_select_lex(), &gwi); + view->viewName(gwi.viewName); + gwi.viewList.push_back(view); + view->transform(); + } + else + { + // check foreign engine tables + bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true); - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); - return; - } + // trigger system catalog cache + if (columnStore) + csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str, lower_case_table_names), + true); + + CalpontSystemCatalog::TableAliasName tn = + make_aliasview(table_ptr->db.str, table_ptr->table_name.str, table_ptr->alias.str, viewName, + columnStore, lower_case_table_names); + gwi.tbList.push_back(tn); + gwi.tableMap[tn] = make_pair(0, table_ptr); + fParentGwip->tableMap[tn] = make_pair(0, table_ptr); + } } - catch (IDBExcept& ie) + + if (gwi.fatalParseError) { - setError(gwi.thd, ER_INTERNAL_ERROR, ie.what()); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); + return; } + } + catch (IDBExcept& ie) + { + setError(gwi.thd, ER_INTERNAL_ERROR, ie.what()); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return; + } - // merge table list to parent select - fParentGwip->tbList.insert(fParentGwip->tbList.end(), gwi.tbList.begin(), gwi.tbList.end()); - fParentGwip->derivedTbList.insert(fParentGwip->derivedTbList.end(), gwi.derivedTbList.begin(), gwi.derivedTbList.end()); - fParentGwip->correlatedTbNameVec.insert(fParentGwip->correlatedTbNameVec.end(), gwi.correlatedTbNameVec.begin(), gwi.correlatedTbNameVec.end()); + // merge table list to parent select + fParentGwip->tbList.insert(fParentGwip->tbList.end(), gwi.tbList.begin(), gwi.tbList.end()); + fParentGwip->derivedTbList.insert(fParentGwip->derivedTbList.end(), gwi.derivedTbList.begin(), + gwi.derivedTbList.end()); + fParentGwip->correlatedTbNameVec.insert(fParentGwip->correlatedTbNameVec.end(), + gwi.correlatedTbNameVec.begin(), gwi.correlatedTbNameVec.end()); - // merge view list to parent - fParentGwip->viewList.insert(fParentGwip->viewList.end(), gwi.viewList.begin(), gwi.viewList.end()); + // merge view list to parent + fParentGwip->viewList.insert(fParentGwip->viewList.end(), gwi.viewList.begin(), gwi.viewList.end()); - // merge non-collapsed outer join to parent select - stack tmpstack; + // merge non-collapsed outer join to parent select + stack tmpstack; - while (!gwi.ptWorkStack.empty()) - { - tmpstack.push(gwi.ptWorkStack.top()); - gwi.ptWorkStack.pop(); - } + while (!gwi.ptWorkStack.empty()) + { + tmpstack.push(gwi.ptWorkStack.top()); + gwi.ptWorkStack.pop(); + } - while (!tmpstack.empty()) - { - fParentGwip->ptWorkStack.push(tmpstack.top()); - tmpstack.pop(); - } + while (!tmpstack.empty()) + { + fParentGwip->ptWorkStack.push(tmpstack.top()); + tmpstack.pop(); + } } uint32_t View::processJoin(gp_walk_info& gwi, std::stack& outerJoinStack) { - return buildJoin(gwi, fSelect.top_join_list, outerJoinStack); + return buildJoin(gwi, fSelect.top_join_list, outerJoinStack); } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/ha_view.h b/dbcon/mysql/ha_view.h index 033a3aa51..e1c009b1d 100644 --- a/dbcon/mysql/ha_view.h +++ b/dbcon/mysql/ha_view.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_view.h 9210 2013-01-21 14:10:42Z rdempsey $ -* -* -***********************************************************************/ + * $Id: ha_view.h 9210 2013-01-21 14:10:42Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ /** class View interface */ @@ -37,30 +37,31 @@ class CalpontSystemCatalog; namespace cal_impl_if { - class View { -public: - View(SELECT_LEX& select, gp_walk_info* parentGwip) : - fSelect(select), - fParentGwip(parentGwip) {} - ~View() {} + public: + View(SELECT_LEX& select, gp_walk_info* parentGwip) : fSelect(select), fParentGwip(parentGwip) + { + } + ~View() + { + } - execplan::CalpontSystemCatalog::TableAliasName& viewName(); - void viewName(execplan::CalpontSystemCatalog::TableAliasName& viewName); + execplan::CalpontSystemCatalog::TableAliasName& viewName(); + void viewName(execplan::CalpontSystemCatalog::TableAliasName& viewName); - /** get execution plan for this view. merge the table list and join list to the - parent select. - */ - void transform(); - uint32_t processJoin(gp_walk_info& gwi, std::stack&); + /** get execution plan for this view. merge the table list and join list to the + parent select. + */ + void transform(); + uint32_t processJoin(gp_walk_info& gwi, std::stack&); -private: - SELECT_LEX fSelect; - gp_walk_info* fParentGwip; - execplan::CalpontSystemCatalog::TableAliasName fViewName; + private: + SELECT_LEX fSelect; + gp_walk_info* fParentGwip; + execplan::CalpontSystemCatalog::TableAliasName fViewName; }; -} +} // namespace cal_impl_if #endif diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 2f8025e3f..8111e14b0 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ha_window_function.cpp 9586 2013-05-31 22:02:06Z zzhu $ -* -* -***********************************************************************/ + * $Id: ha_window_function.cpp 9586 2013-05-31 22:02:06Z zzhu $ + * + * + ***********************************************************************/ #define PREFER_MY_CONFIG_H #include #include @@ -54,153 +54,153 @@ using namespace mcsv1sdk; namespace cal_impl_if { - ReturnedColumn* nullOnError(gp_walk_info& gwi) { - if (gwi.hasSubSelect) - { - gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText); - } + if (gwi.hasSubSelect) + { + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText); + } - if (gwi.parseErrorText.empty()) - { - gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_NON_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText); - } + if (gwi.parseErrorText.empty()) + { + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_NON_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText); + } - return NULL; + return NULL; } WF_FRAME frame(Window_frame_bound::Bound_precedence_type bound, Item* offset) { - switch (bound) - { - case Window_frame_bound::PRECEDING: - if (offset) - return WF_PRECEDING; - else - return WF_UNBOUNDED_PRECEDING; + switch (bound) + { + case Window_frame_bound::PRECEDING: + if (offset) + return WF_PRECEDING; + else + return WF_UNBOUNDED_PRECEDING; - case Window_frame_bound::FOLLOWING: - if (offset) - return WF_FOLLOWING; - else - return WF_UNBOUNDED_FOLLOWING; + case Window_frame_bound::FOLLOWING: + if (offset) + return WF_FOLLOWING; + else + return WF_UNBOUNDED_FOLLOWING; - case Window_frame_bound::CURRENT: // Offset is meaningless - return WF_CURRENT_ROW; + case Window_frame_bound::CURRENT: // Offset is meaningless + return WF_CURRENT_ROW; - default: - return WF_UNKNOWN; - } + default: return WF_UNKNOWN; + } } ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - bool addOp = true; - ReturnedColumn* rc = NULL; + bool addOp = true; + ReturnedColumn* rc = NULL; - if (bound.fFrame == execplan::WF_PRECEDING) + if (bound.fFrame == execplan::WF_PRECEDING) + { + if (order->asc()) + addOp = false; + } + else if (!order->asc()) // must be WF_FOLLOWING + addOp = false; + + funcexp::FunctionParm funcParms; + SPTP sptp; + IntervalColumn* intervalCol = dynamic_cast(bound.fVal.get()); + + // @todo error out non constant. only support literal interval for now. + if (!intervalCol && order->resultType().colDataType == CalpontSystemCatalog::DATE) + { + intervalCol = new IntervalColumn(bound.fVal, (int)IntervalColumn::INTERVAL_DAY); + bound.fVal.reset(intervalCol); + } + + if (intervalCol) + { + // date_add + rc = new FunctionColumn(); + string funcName = "date_add_interval"; + + // @bug6061 . YEAR, QUARTER, MONTH, WEEK, DAY type + CalpontSystemCatalog::ColType ct; + + if (order->resultType().colDataType == CalpontSystemCatalog::DATE && + intervalCol->intervalType() <= IntervalColumn::INTERVAL_DAY) { - if (order->asc()) - addOp = false; + ct.colDataType = CalpontSystemCatalog::DATE; + ct.colWidth = 4; } - else if (!order->asc()) // must be WF_FOLLOWING - addOp = false; - - funcexp::FunctionParm funcParms; - SPTP sptp; - IntervalColumn* intervalCol = dynamic_cast(bound.fVal.get()); - - // @todo error out non constant. only support literal interval for now. - if (!intervalCol && order->resultType().colDataType == CalpontSystemCatalog::DATE) + else { - intervalCol = new IntervalColumn(bound.fVal, (int)IntervalColumn::INTERVAL_DAY); - bound.fVal.reset(intervalCol); + ct.colDataType = CalpontSystemCatalog::DATETIME; + ct.colWidth = 8; } - if (intervalCol) - { - // date_add - rc = new FunctionColumn(); - string funcName = "date_add_interval"; - - // @bug6061 . YEAR, QUARTER, MONTH, WEEK, DAY type - CalpontSystemCatalog::ColType ct; - - if (order->resultType().colDataType == CalpontSystemCatalog::DATE && - intervalCol->intervalType() <= IntervalColumn::INTERVAL_DAY) - { - ct.colDataType = CalpontSystemCatalog::DATE; - ct.colWidth = 4; - } - else - { - ct.colDataType = CalpontSystemCatalog::DATETIME; - ct.colWidth = 8; - } - - // put interval val column to bound - (dynamic_cast(rc))->functionName(funcName); - (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - sptp.reset(new ParseTree(order->clone())); - funcParms.push_back(sptp); - sptp.reset(new ParseTree(intervalCol->val()->clone())); - funcParms.push_back(sptp); - funcParms.push_back(getIntervalType(gwi.thd, intervalCol->intervalType())); - SRCP srcp(intervalCol->val()); - bound.fVal = srcp; - - if (addOp) - { - sptp.reset(new ParseTree(new ConstantColumn("ADD"))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - } - else - { - sptp.reset(new ParseTree(new ConstantColumn("SUB"))); - (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(sptp); - } - - (dynamic_cast(rc))->functionParms(funcParms); - - rc->resultType(ct); - // @bug6061. Use result type as operation type for WF bound expression - rc->operationType(ct); - rc->expressionId(ci->expressionId++); - return rc; - } - - // arithmetic - rc = new ArithmeticColumn(); - ArithmeticOperator* aop; + // put interval val column to bound + (dynamic_cast(rc))->functionName(funcName); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sptp.reset(new ParseTree(order->clone())); + funcParms.push_back(sptp); + sptp.reset(new ParseTree(intervalCol->val()->clone())); + funcParms.push_back(sptp); + funcParms.push_back(getIntervalType(gwi.thd, intervalCol->intervalType())); + SRCP srcp(intervalCol->val()); + bound.fVal = srcp; if (addOp) - aop = new ArithmeticOperator("+"); + { + sptp.reset(new ParseTree(new ConstantColumn("ADD"))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + } else - aop = new ArithmeticOperator("-"); + { + sptp.reset(new ParseTree(new ConstantColumn("SUB"))); + (dynamic_cast(sptp->data())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(sptp); + } - aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - ParseTree* pt = new ParseTree(aop); - ParseTree* lhs = 0, *rhs = 0; - lhs = new ParseTree(order->clone()); - rhs = new ParseTree(bound.fVal->clone()); - pt->left(lhs); - pt->right(rhs); - aop->resultType(order->resultType()); - aop->operationType(aop->resultType()); - (dynamic_cast(rc))->expression(pt); - rc->resultType(aop->resultType()); - rc->operationType(aop->operationType()); + (dynamic_cast(rc))->functionParms(funcParms); + + rc->resultType(ct); + // @bug6061. Use result type as operation type for WF bound expression + rc->operationType(ct); rc->expressionId(ci->expressionId++); return rc; + } + + // arithmetic + rc = new ArithmeticColumn(); + ArithmeticOperator* aop; + + if (addOp) + aop = new ArithmeticOperator("+"); + else + aop = new ArithmeticOperator("-"); + + aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + ParseTree* pt = new ParseTree(aop); + ParseTree *lhs = 0, *rhs = 0; + lhs = new ParseTree(order->clone()); + rhs = new ParseTree(bound.fVal->clone()); + pt->left(lhs); + pt->right(rhs); + aop->resultType(order->resultType()); + aop->operationType(aop->resultType()); + (dynamic_cast(rc))->expression(pt); + rc->resultType(aop->resultType()); + rc->operationType(aop->operationType()); + rc->expressionId(ci->expressionId++); + return rc; } // Since columnstore implemented Windows Functions before MariaDB, we need @@ -208,719 +208,676 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi // identify the function type. string ConvertFuncName(Item_sum* item) { - switch (item->sum_func()) - { - case Item_sum::COUNT_FUNC: - if (!item->arguments()[0]->name.str) - return "COUNT(*)"; + switch (item->sum_func()) + { + case Item_sum::COUNT_FUNC: + if (!item->arguments()[0]->name.str) + return "COUNT(*)"; - return "COUNT"; - break; + return "COUNT"; + break; - case Item_sum::COUNT_DISTINCT_FUNC: - return "COUNT_DISTINCT"; - break; + case Item_sum::COUNT_DISTINCT_FUNC: return "COUNT_DISTINCT"; break; - case Item_sum::SUM_FUNC: - return "SUM"; - break; + case Item_sum::SUM_FUNC: return "SUM"; break; - case Item_sum::SUM_DISTINCT_FUNC: - return "SUM_DISTINCT"; - break; + case Item_sum::SUM_DISTINCT_FUNC: return "SUM_DISTINCT"; break; - case Item_sum::AVG_FUNC: - return "AVG"; - break; + case Item_sum::AVG_FUNC: return "AVG"; break; - case Item_sum::AVG_DISTINCT_FUNC: - return "AVG_DISTINCT"; - break; + case Item_sum::AVG_DISTINCT_FUNC: return "AVG_DISTINCT"; break; - case Item_sum::MIN_FUNC: - return "MIN"; - break; + case Item_sum::MIN_FUNC: return "MIN"; break; - case Item_sum::MAX_FUNC: - return "MAX"; - break; + case Item_sum::MAX_FUNC: return "MAX"; break; - case Item_sum::STD_FUNC: - if (((Item_sum_variance*)item)->sample) - return "STDDEV_SAMP"; - else - return "STDDEV_POP"; + case Item_sum::STD_FUNC: + if (((Item_sum_variance*)item)->sample) + return "STDDEV_SAMP"; + else + return "STDDEV_POP"; - break; + break; - case Item_sum::VARIANCE_FUNC: - if (((Item_sum_variance*)item)->sample) - return "VAR_SAMP"; - else - return "VAR_POP"; + case Item_sum::VARIANCE_FUNC: + if (((Item_sum_variance*)item)->sample) + return "VAR_SAMP"; + else + return "VAR_POP"; - break; + break; - case Item_sum::SUM_BIT_FUNC: - if (strcmp(item->func_name(), "bit_or(") == 0) - return "BIT_OR"; + case Item_sum::SUM_BIT_FUNC: + if (strcmp(item->func_name(), "bit_or(") == 0) + return "BIT_OR"; - if (strcmp(item->func_name(), "bit_and(") == 0) - return "BIT_AND"; + if (strcmp(item->func_name(), "bit_and(") == 0) + return "BIT_AND"; - if (strcmp(item->func_name(), "bit_xor(") == 0) - return "BIT_XOR"; + if (strcmp(item->func_name(), "bit_xor(") == 0) + return "BIT_XOR"; - break; + break; - case Item_sum::UDF_SUM_FUNC: - return "UDAF_FUNC"; - break; + case Item_sum::UDF_SUM_FUNC: return "UDAF_FUNC"; break; - case Item_sum::GROUP_CONCAT_FUNC: - return "GROUP_CONCAT"; // Not supported - break; + case Item_sum::GROUP_CONCAT_FUNC: + return "GROUP_CONCAT"; // Not supported + break; - case Item_sum::ROW_NUMBER_FUNC: - return "ROW_NUMBER"; - break; + case Item_sum::ROW_NUMBER_FUNC: return "ROW_NUMBER"; break; - case Item_sum::RANK_FUNC: - return "RANK"; - break; + case Item_sum::RANK_FUNC: return "RANK"; break; - case Item_sum::DENSE_RANK_FUNC: - return "DENSE_RANK"; - break; + case Item_sum::DENSE_RANK_FUNC: return "DENSE_RANK"; break; - case Item_sum::PERCENT_RANK_FUNC: - return "PERCENT_RANK"; - break; + case Item_sum::PERCENT_RANK_FUNC: return "PERCENT_RANK"; break; - case Item_sum::PERCENTILE_CONT_FUNC: - return "PERCENTILE_CONT"; - break; + case Item_sum::PERCENTILE_CONT_FUNC: return "PERCENTILE_CONT"; break; - case Item_sum::PERCENTILE_DISC_FUNC: - return "PERCENTILE_DISC"; + case Item_sum::PERCENTILE_DISC_FUNC: return "PERCENTILE_DISC"; - case Item_sum::CUME_DIST_FUNC: - return "CUME_DIST"; - break; + case Item_sum::CUME_DIST_FUNC: return "CUME_DIST"; break; - case Item_sum::NTILE_FUNC: - return "NTILE"; - break; + case Item_sum::NTILE_FUNC: return "NTILE"; break; - case Item_sum::FIRST_VALUE_FUNC: - return "FIRST_VALUE"; - break; + case Item_sum::FIRST_VALUE_FUNC: return "FIRST_VALUE"; break; - case Item_sum::LAST_VALUE_FUNC: - return "LAST_VALUE"; - break; + case Item_sum::LAST_VALUE_FUNC: return "LAST_VALUE"; break; - case Item_sum::NTH_VALUE_FUNC: - return "NTH_VALUE"; - break; + case Item_sum::NTH_VALUE_FUNC: return "NTH_VALUE"; break; - case Item_sum::LEAD_FUNC: - return "LEAD"; - break; + case Item_sum::LEAD_FUNC: return "LEAD"; break; - case Item_sum::LAG_FUNC: - return "LAG"; - break; - default: - // We just don't handle it. - break; - }; + case Item_sum::LAG_FUNC: return "LAG"; break; + default: + // We just don't handle it. + break; + }; - return ""; + return ""; } ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport) { - //@todo fix print for create view - //String str; - //item->print(&str, QT_INFINIDB_NO_QUOTE); - //cout << str.c_ptr() << endl; - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); + //@todo fix print for create view + // String str; + // item->print(&str, QT_INFINIDB_NO_QUOTE); + // cout << str.c_ptr() << endl; + if (get_fe_conn_info_ptr() == NULL) + set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - gwi.hasWindowFunc = true; - Item_window_func* wf = (Item_window_func*)item; - Item_sum* item_sum = wf->window_func(); - string funcName = ConvertFuncName(item_sum); - WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); - ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - ac->distinct(item_sum->has_with_distinct()); - Window_spec* win_spec = wf->window_spec; - SRCP srcp; - CalpontSystemCatalog::ColType ct; // For return type - // arguments - vector funcParms; + gwi.hasWindowFunc = true; + Item_window_func* wf = (Item_window_func*)item; + Item_sum* item_sum = wf->window_func(); + string funcName = ConvertFuncName(item_sum); + WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); + ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + ac->distinct(item_sum->has_with_distinct()); + Window_spec* win_spec = wf->window_spec; + SRCP srcp; + CalpontSystemCatalog::ColType ct; // For return type + // arguments + vector funcParms; - for (uint32_t i = 0; i < item_sum->argument_count(); i++) + for (uint32_t i = 0; i < item_sum->argument_count(); i++) + { + srcp.reset(buildReturnedColumn((item_sum->arguments()[i]), gwi, nonSupport)); + + if (!srcp) + return nullOnError(gwi); + + funcParms.push_back(srcp); + + if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) + gwi.rcWorkStack.pop(); + } + + // Setup UDAnF functions + if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) + { + Item_udf_sum* udfsum = (Item_udf_sum*)item_sum; + + mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); + context.setName(udfsum->func_name()); + + // Set up the return type defaults for the call to init() + execplan::CalpontSystemCatalog::ColType& rt = ac->resultType(); + context.setResultType(rt.colDataType); + context.setColWidth(rt.colWidth); + context.setScale(rt.scale); + context.setPrecision(rt.precision); + context.setParamCount(funcParms.size()); + + utils::VLArray colTypes(funcParms.size()); + + // Turn on the Analytic flag so the function is aware it is being called + // as a Window Function. + context.setContextFlag(CONTEXT_IS_ANALYTIC); + + // Build the column type vector. + // Modified for MCOL-1201 multi-argument aggregate + for (size_t i = 0; i < funcParms.size(); ++i) { - srcp.reset(buildReturnedColumn((item_sum->arguments()[i]), gwi, nonSupport)); + const execplan::CalpontSystemCatalog::ColType& resultType = funcParms[i]->resultType(); + mcsv1sdk::ColumnDatum& colType = colTypes[i]; + colType.dataType = resultType.colDataType; + colType.precision = resultType.precision; + colType.scale = resultType.scale; + colType.charsetNumber = resultType.charsetNumber; + } + + // Call the user supplied init() + if (context.getFunction()->init(&context, colTypes) == mcsv1_UDAF::ERROR) + { + gwi.fatalParseError = true; + gwi.parseErrorText = context.getErrorMessage(); + return NULL; + } + + if (!context.getRunFlag(UDAF_OVER_REQUIRED) && !context.getRunFlag(UDAF_OVER_ALLOWED)) + { + gwi.parseErrorText = + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_NOT_ALLOWED, context.getName()); + return nullOnError(gwi); + } + + // Set the return type as set in init() + ct.colDataType = context.getResultType(); + ct.colWidth = context.getColWidth(); + ct.scale = context.getScale(); + ct.precision = context.getPrecision(); + ac->resultType(ct); + } + + // Some functions, such as LEAD/LAG don't have all parameters implemented in the + // front end. Add dummies here to make the backend use defaults. + // Some of these will be temporary until they are implemented in the front end. + // Others need to stay because the back end expects them, but the front end + // no longer sends them. + // This case is kept in enum order in hopes the compiler can optimize + switch (item_sum->sum_func()) + { + case Item_sum::UDF_SUM_FUNC: + { + unsigned long bRespectNulls = (ac->getUDAFContext().getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) ? 0 : 1; + char sRespectNulls[18]; + sprintf(sRespectNulls, "%lu", bRespectNulls); + srcp.reset(new ConstantColumn(sRespectNulls, (uint64_t)bRespectNulls, + ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + break; + } + + case Item_sum::FIRST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + srcp.reset( + new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + break; + + case Item_sum::LAST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + srcp.reset( + new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + break; + + case Item_sum::NTH_VALUE_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + srcp.reset( + new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + break; + + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + srcp.reset( + new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + funcParms.push_back(srcp); + break; + + default: break; + }; + + ac->functionParms(funcParms); + + // Partition by + if (win_spec) + { + vector partitions; + + for (ORDER* ord = win_spec->partition_list->first; ord; ord = ord->next) + { + srcp.reset(buildReturnedColumn(*ord->item, gwi, nonSupport)); + + if (!srcp) + return nullOnError(gwi); + + partitions.push_back(srcp); + } + + ac->partitions(partitions); + + // Order by + WF_OrderBy orderBy; + + // order columns + if (win_spec->order_list) + { + // It is an error to have an order by clause if a UDAnF says it shouldn't + if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) + { + mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); + + if (!context.getRunFlag(UDAF_ORDER_ALLOWED)) + { + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg( + logging::ERR_WF_UDANF_ORDER_NOT_ALLOWED, context.getName()); + return nullOnError(gwi); + } + } + + vector orders; + ORDER* orderCol = reinterpret_cast(win_spec->order_list->first); + + for (; orderCol; orderCol = orderCol->next) + { + Item* orderItem = *(orderCol->item); + srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); + + // MCOL-1052 GROUP BY handler has all of query's agg Items + // as field and correlates them with its extended SELECT Items. + if (!srcp) + { + orderItem = orderCol->item_ptr; + + if (orderItem) + { + gwi.fatalParseError = false; + srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); + } + } if (!srcp) - return nullOnError(gwi); + return nullOnError(gwi); - funcParms.push_back(srcp); + srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); + // srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, + //0-nulls last + srcp->nullsFirst(orderCol->direction == ORDER::ORDER_ASC + ? 1 + : 0); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + orders.push_back(srcp); + } - if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); + orderBy.fOrders = orders; } - - // Setup UDAnF functions - if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) + else { - Item_udf_sum* udfsum = (Item_udf_sum*)item_sum; - + if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) + { mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); - context.setName(udfsum->func_name()); - // Set up the return type defaults for the call to init() - execplan::CalpontSystemCatalog::ColType& rt = ac->resultType(); - context.setResultType(rt.colDataType); - context.setColWidth(rt.colWidth); - context.setScale(rt.scale); - context.setPrecision(rt.precision); - context.setParamCount(funcParms.size()); - - utils::VLArray colTypes(funcParms.size()); - - // Turn on the Analytic flag so the function is aware it is being called - // as a Window Function. - context.setContextFlag(CONTEXT_IS_ANALYTIC); - - // Build the column type vector. - // Modified for MCOL-1201 multi-argument aggregate - for (size_t i = 0; i < funcParms.size(); ++i) + if (context.getRunFlag(UDAF_ORDER_REQUIRED)) { - const execplan::CalpontSystemCatalog::ColType& resultType - = funcParms[i]->resultType(); - mcsv1sdk::ColumnDatum& colType = colTypes[i]; - colType.dataType = resultType.colDataType; - colType.precision = resultType.precision; - colType.scale = resultType.scale; - colType.charsetNumber = resultType.charsetNumber; + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_NOT_ALLOWED, + context.getName()); + return nullOnError(gwi); } - - // Call the user supplied init() - if (context.getFunction()->init(&context, colTypes) == mcsv1_UDAF::ERROR) - { - gwi.fatalParseError = true; - gwi.parseErrorText = context.getErrorMessage(); - return NULL; - } - - if (!context.getRunFlag(UDAF_OVER_REQUIRED) && !context.getRunFlag(UDAF_OVER_ALLOWED)) - { - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_NOT_ALLOWED, - context.getName()); - return nullOnError(gwi); - } - - // Set the return type as set in init() - ct.colDataType = context.getResultType(); - ct.colWidth = context.getColWidth(); - ct.scale = context.getScale(); - ct.precision = context.getPrecision(); - ac->resultType(ct); + } } - // Some functions, such as LEAD/LAG don't have all parameters implemented in the - // front end. Add dummies here to make the backend use defaults. - // Some of these will be temporary until they are implemented in the front end. - // Others need to stay because the back end expects them, but the front end - // no longer sends them. - // This case is kept in enum order in hopes the compiler can optimize - switch (item_sum->sum_func()) + // window frame + WF_Frame frm; + + if (win_spec->window_frame) { + // It is an error to have a frame clause if a UDAnF says it shouldn't + if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) + { + mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); + + if (!context.getRunFlag(UDAF_WINDOWFRAME_ALLOWED)) + { + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg( + logging::ERR_WF_UDANF_FRAME_NOT_ALLOWED, context.getName()); + return nullOnError(gwi); + } + } + + frm.fIsRange = win_spec->window_frame->units == Window_frame::UNITS_RANGE; + + // start + if (win_spec->window_frame->top_bound) + { + frm.fStart.fFrame = frame(win_spec->window_frame->top_bound->precedence_type, + win_spec->window_frame->top_bound->offset); // offset NULL means UNBOUNDED + + if (win_spec->window_frame->top_bound->offset) + { + frm.fStart.fVal.reset( + buildReturnedColumn(win_spec->window_frame->top_bound->offset, gwi, nonSupport)); + + if (!frm.fStart.fVal) + return nullOnError(gwi); + + // 1. check expr is numeric type (rows) or interval (range) + bool boundTypeErr = false; + + switch (frm.fStart.fVal->resultType().colDataType) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: boundTypeErr = true; break; + + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + if (!frm.fIsRange) + boundTypeErr = true; + else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) + boundTypeErr = true; + + break; + + default: // okay + break; + } + + if (boundTypeErr) + { + gwi.fatalParseError = true; + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg( + logging::ERR_WF_INVALID_BOUND_TYPE, + colDataTypeToString(frm.fStart.fVal->resultType().colDataType)); + return nullOnError(gwi); + } + } + } + + // end + if (win_spec->window_frame->bottom_bound) + { + frm.fEnd.fFrame = frame(win_spec->window_frame->bottom_bound->precedence_type, + win_spec->window_frame->bottom_bound->offset); + + if (win_spec->window_frame->bottom_bound->offset) + { + frm.fEnd.fVal.reset( + buildReturnedColumn(win_spec->window_frame->bottom_bound->offset, gwi, nonSupport)); + + if (!frm.fEnd.fVal) + return nullOnError(gwi); + + // check expr is numeric type (rows) or interval (range) + bool boundTypeErr = false; + + switch (frm.fEnd.fVal->resultType().colDataType) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: boundTypeErr = true; break; + + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + if (!frm.fIsRange) + boundTypeErr = true; + else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) + boundTypeErr = true; + + break; + + default: // okay + break; + } + + if (boundTypeErr) + { + gwi.fatalParseError = true; + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg( + logging::ERR_WF_INVALID_BOUND_TYPE, + colDataTypeToString(frm.fStart.fVal->resultType().colDataType)); + return nullOnError(gwi); + } + } + } + else // no end specified. default end to current row + { + frm.fEnd.fFrame = WF_CURRENT_ROW; + } + + if (frm.fStart.fVal || frm.fEnd.fVal) + { + // check order by key only 1 (should be error-ed out in parser. double check here) + if (frm.fIsRange && orderBy.fOrders.size() > 1) + { + gwi.fatalParseError = true; + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_INVALID_ORDER_KEY); + return nullOnError(gwi); + } + + // check order by key type is numeric or date/datetime + bool orderTypeErr = false; + + if (frm.fIsRange && orderBy.fOrders.size() == 1) + { + switch (orderBy.fOrders[0]->resultType().colDataType) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::CLOB: orderTypeErr = true; break; + + default: // okay + + // interval bound has to have date/datetime order key + if ((dynamic_cast(frm.fStart.fVal.get()) != NULL || + dynamic_cast(frm.fEnd.fVal.get()) != NULL)) + { + if (orderBy.fOrders[0]->resultType().colDataType != CalpontSystemCatalog::DATE && + orderBy.fOrders[0]->resultType().colDataType != CalpontSystemCatalog::DATETIME) + orderTypeErr = true; + } + else + { + if (orderBy.fOrders[0]->resultType().colDataType == CalpontSystemCatalog::DATETIME) + orderTypeErr = true; + } + + break; + } + + if (orderTypeErr) + { + gwi.fatalParseError = true; + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg( + logging::ERR_WF_INVALID_ORDER_TYPE, + colDataTypeToString(orderBy.fOrders[0]->resultType().colDataType)); + return nullOnError(gwi); + } + } + } + + // construct +,- or interval function for boundary + if (frm.fIsRange && frm.fStart.fVal) + { + frm.fStart.fBound.reset(buildBoundExp(frm.fStart, orderBy.fOrders[0], gwi)); + + if (!frm.fStart.fBound) + return nullOnError(gwi); + } + + if (frm.fIsRange && frm.fEnd.fVal) + { + frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, orderBy.fOrders[0], gwi)); + + if (!frm.fEnd.fVal) + return nullOnError(gwi); + } + } + else + { + // Certain function types have different default boundaries + // This case is kept in enum order in hopes the compiler can optimize + switch (item_sum->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + case Item_sum::SUM_FUNC: + case Item_sum::SUM_DISTINCT_FUNC: + case Item_sum::AVG_FUNC: + case Item_sum::AVG_DISTINCT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + + case Item_sum::MIN_FUNC: + case Item_sum::MAX_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + // frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + + case Item_sum::STD_FUNC: + case Item_sum::VARIANCE_FUNC: + case Item_sum::SUM_BIT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::UDF_SUM_FUNC: { - unsigned long bRespectNulls = (ac->getUDAFContext().getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) ? 0 : 1; - char sRespectNulls[18]; - sprintf(sRespectNulls, "%lu", bRespectNulls); - srcp.reset(new ConstantColumn(sRespectNulls, (uint64_t)bRespectNulls, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - break; + // UDAnF functions each have their own default set in context. + mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); + + if (context.getRunFlag(UDAF_WINDOWFRAME_REQUIRED)) + { + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg( + logging::ERR_WF_UDANF_FRAME_REQUIRED, context.getName()); + return nullOnError(gwi); + } + + int32_t bound; + context.getStartFrame(frm.fStart.fFrame, bound); + + if (frm.fStart.fFrame == execplan::WF_PRECEDING) + { + if (bound == 0) + bound = 1; + + srcp.reset(new ConstantColumn((int64_t)bound)); + (dynamic_cast(srcp.get())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + frm.fStart.fVal = srcp; + frm.fStart.fBound.reset(buildBoundExp(frm.fStart, srcp, gwi)); + + if (!frm.fStart.fBound) + return nullOnError(gwi); + } + + context.getEndFrame(frm.fEnd.fFrame, bound); + + if (frm.fEnd.fFrame == execplan::WF_FOLLOWING) + { + if (bound == 0) + bound = 1; + + srcp.reset(new ConstantColumn((int64_t)bound)); + (dynamic_cast(srcp.get())) + ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + frm.fEnd.fVal = srcp; + frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, srcp, gwi)); + + if (!frm.fEnd.fBound) + return nullOnError(gwi); + } + + break; } + case Item_sum::GROUP_CONCAT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + + case Item_sum::ROW_NUMBER_FUNC: + case Item_sum::RANK_FUNC: + case Item_sum::DENSE_RANK_FUNC: + case Item_sum::PERCENT_RANK_FUNC: + case Item_sum::CUME_DIST_FUNC: + case Item_sum::NTILE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::FIRST_VALUE_FUNC: - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - break; - case Item_sum::LAST_VALUE_FUNC: - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - break; - case Item_sum::NTH_VALUE_FUNC: - // When the front end supports these paramters, this needs modification - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - break; + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::LEAD_FUNC: case Item_sum::LAG_FUNC: - // When the front end supports these paramters, this needs modification - srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - funcParms.push_back(srcp); - break; + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; default: - break; - }; - - ac->functionParms(funcParms); - - // Partition by - if (win_spec) - { - vector partitions; - - for (ORDER* ord = win_spec->partition_list->first; ord; ord = ord->next) - { - srcp.reset(buildReturnedColumn(*ord->item, gwi, nonSupport)); - - if (!srcp) - return nullOnError(gwi); - - partitions.push_back(srcp); - } - - ac->partitions(partitions); - - // Order by - WF_OrderBy orderBy; - - // order columns - if (win_spec->order_list) - { - // It is an error to have an order by clause if a UDAnF says it shouldn't - if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) - { - mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); - - if (!context.getRunFlag(UDAF_ORDER_ALLOWED)) - { - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_ORDER_NOT_ALLOWED, - context.getName()); - return nullOnError(gwi); - } - } - - vector orders; - ORDER* orderCol = reinterpret_cast(win_spec->order_list->first); - - for (; orderCol; orderCol = orderCol->next) - { - Item* orderItem = *(orderCol->item); - srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); - - // MCOL-1052 GROUP BY handler has all of query's agg Items - // as field and correlates them with its extended SELECT Items. - if (!srcp) - { - orderItem = orderCol->item_ptr; - - if (orderItem) - { - gwi.fatalParseError = false; - srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); - } - } - - if (!srcp) - return nullOnError(gwi); - - srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); -// srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last - srcp->nullsFirst(orderCol->direction == ORDER::ORDER_ASC ? 1 : 0); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end - orders.push_back(srcp); - } - - orderBy.fOrders = orders; - } - else - { - if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) - { - mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); - - if (context.getRunFlag(UDAF_ORDER_REQUIRED)) - { - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_NOT_ALLOWED, - context.getName()); - return nullOnError(gwi); - } - } - } - - // window frame - WF_Frame frm; - - if (win_spec->window_frame) - { - // It is an error to have a frame clause if a UDAnF says it shouldn't - if (item_sum->sum_func() == Item_sum::UDF_SUM_FUNC) - { - mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); - - if (!context.getRunFlag(UDAF_WINDOWFRAME_ALLOWED)) - { - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_FRAME_NOT_ALLOWED, - context.getName()); - return nullOnError(gwi); - } - } - - frm.fIsRange = win_spec->window_frame->units == Window_frame::UNITS_RANGE; - - // start - if (win_spec->window_frame->top_bound) - { - frm.fStart.fFrame = frame(win_spec->window_frame->top_bound->precedence_type, - win_spec->window_frame->top_bound->offset); // offset NULL means UNBOUNDED - - if (win_spec->window_frame->top_bound->offset) - { - frm.fStart.fVal.reset(buildReturnedColumn(win_spec->window_frame->top_bound->offset, gwi, nonSupport)); - - if (!frm.fStart.fVal) - return nullOnError(gwi); - - // 1. check expr is numeric type (rows) or interval (range) - bool boundTypeErr = false; - - switch (frm.fStart.fVal->resultType().colDataType) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - boundTypeErr = true; - break; - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - if (!frm.fIsRange) - boundTypeErr = true; - else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) - boundTypeErr = true; - - break; - - default: //okay - break; - } - - if (boundTypeErr) - { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_INVALID_BOUND_TYPE, - colDataTypeToString(frm.fStart.fVal->resultType().colDataType)); - return nullOnError(gwi); - } - } - } - - // end - if (win_spec->window_frame->bottom_bound) - { - frm.fEnd.fFrame = frame(win_spec->window_frame->bottom_bound->precedence_type, - win_spec->window_frame->bottom_bound->offset); - - if (win_spec->window_frame->bottom_bound->offset) - { - frm.fEnd.fVal.reset(buildReturnedColumn(win_spec->window_frame->bottom_bound->offset, gwi, nonSupport)); - - if (!frm.fEnd.fVal) - return nullOnError(gwi); - - // check expr is numeric type (rows) or interval (range) - bool boundTypeErr = false; - - switch (frm.fEnd.fVal->resultType().colDataType) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - boundTypeErr = true; - break; - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - if (!frm.fIsRange) - boundTypeErr = true; - else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) - boundTypeErr = true; - - break; - - default: //okay - break; - } - - if (boundTypeErr) - { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_INVALID_BOUND_TYPE, - colDataTypeToString(frm.fStart.fVal->resultType().colDataType)); - return nullOnError(gwi); - } - } - } - else // no end specified. default end to current row - { - frm.fEnd.fFrame = WF_CURRENT_ROW; - } - - if (frm.fStart.fVal || frm.fEnd.fVal) - { - // check order by key only 1 (should be error-ed out in parser. double check here) - if (frm.fIsRange && orderBy.fOrders.size() > 1) - { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_INVALID_ORDER_KEY); - return nullOnError(gwi); - } - - // check order by key type is numeric or date/datetime - bool orderTypeErr = false; - - if (frm.fIsRange && orderBy.fOrders.size() == 1) - { - switch (orderBy.fOrders[0]->resultType().colDataType) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - orderTypeErr = true; - break; - - default: //okay - - // interval bound has to have date/datetime order key - if ((dynamic_cast(frm.fStart.fVal.get()) != NULL || - dynamic_cast(frm.fEnd.fVal.get()) != NULL)) - { - if (orderBy.fOrders[0]->resultType().colDataType != CalpontSystemCatalog::DATE && - orderBy.fOrders[0]->resultType().colDataType != CalpontSystemCatalog::DATETIME) - orderTypeErr = true; - } - else - { - if (orderBy.fOrders[0]->resultType().colDataType == CalpontSystemCatalog::DATETIME) - orderTypeErr = true; - } - - break; - } - - if (orderTypeErr) - { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_INVALID_ORDER_TYPE, - colDataTypeToString(orderBy.fOrders[0]->resultType().colDataType)); - return nullOnError(gwi); - } - } - } - - // construct +,- or interval function for boundary - if (frm.fIsRange && frm.fStart.fVal) - { - frm.fStart.fBound.reset(buildBoundExp(frm.fStart, orderBy.fOrders[0], gwi)); - - if (!frm.fStart.fBound) - return nullOnError(gwi); - } - - if (frm.fIsRange && frm.fEnd.fVal) - { - frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, orderBy.fOrders[0], gwi)); - - if (!frm.fEnd.fVal) - return nullOnError(gwi); - } - } - else - { - // Certain function types have different default boundaries - // This case is kept in enum order in hopes the compiler can optimize - switch (item_sum->sum_func()) - { - case Item_sum::COUNT_FUNC: - case Item_sum::COUNT_DISTINCT_FUNC: - case Item_sum::SUM_FUNC: - case Item_sum::SUM_DISTINCT_FUNC: - case Item_sum::AVG_FUNC: - case Item_sum::AVG_DISTINCT_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; - - case Item_sum::MIN_FUNC: - case Item_sum::MAX_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; -// frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; - - case Item_sum::STD_FUNC: - case Item_sum::VARIANCE_FUNC: - case Item_sum::SUM_BIT_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; - - case Item_sum::UDF_SUM_FUNC: - { - // UDAnF functions each have their own default set in context. - mcsv1sdk::mcsv1Context& context = ac->getUDAFContext(); - - if (context.getRunFlag(UDAF_WINDOWFRAME_REQUIRED)) - { - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_UDANF_FRAME_REQUIRED, - context.getName()); - return nullOnError(gwi); - } - - int32_t bound; - context.getStartFrame(frm.fStart.fFrame, bound); - - if (frm.fStart.fFrame == execplan::WF_PRECEDING) - { - if (bound == 0) - bound = 1; - - srcp.reset(new ConstantColumn((int64_t)bound)); - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - frm.fStart.fVal = srcp; - frm.fStart.fBound.reset(buildBoundExp(frm.fStart, srcp, gwi)); - - if (!frm.fStart.fBound) - return nullOnError(gwi); - } - - context.getEndFrame(frm.fEnd.fFrame, bound); - - if (frm.fEnd.fFrame == execplan::WF_FOLLOWING) - { - if (bound == 0) - bound = 1; - - srcp.reset(new ConstantColumn((int64_t)bound)); - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - frm.fEnd.fVal = srcp; - frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, srcp, gwi)); - - if (!frm.fEnd.fBound) - return nullOnError(gwi); - } - - break; - } - - case Item_sum::GROUP_CONCAT_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; - - case Item_sum::ROW_NUMBER_FUNC: - case Item_sum::RANK_FUNC: - case Item_sum::DENSE_RANK_FUNC: - case Item_sum::PERCENT_RANK_FUNC: - case Item_sum::CUME_DIST_FUNC: - case Item_sum::NTILE_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - break; - - case Item_sum::FIRST_VALUE_FUNC: - case Item_sum::LAST_VALUE_FUNC: - case Item_sum::NTH_VALUE_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; - - case Item_sum::LEAD_FUNC: - case Item_sum::LAG_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - break; - - default: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; - }; - } - - orderBy.fFrame = frm; - ac->orderBy(orderBy); + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + }; } - if (gwi.fatalParseError || nonSupport) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_NON_SUPPORT); + orderBy.fFrame = frm; + ac->orderBy(orderBy); + } - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText); - return NULL; - } + if (gwi.fatalParseError || nonSupport) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WF_NON_SUPPORT); + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText); + return NULL; + } #if 0 if (item_sum->sum_func() != Item_sum::UDF_SUM_FUNC && item_sum->sum_func() != Item_sum::SUM_FUNC && @@ -928,25 +885,24 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n item_sum->sum_func() != Item_sum::AVG_FUNC && item_sum->sum_func() != Item_sum::AVG_DISTINCT_FUNC) #endif - if (item_sum->sum_func() != Item_sum::UDF_SUM_FUNC) - { - ac->resultType(colType_MysqlToIDB(item_sum)); - // bug5736. Make the result type double for some window functions when - // plugin variable double_for_decimal_math is set. - ac->adjustResultType(); - } + if (item_sum->sum_func() != Item_sum::UDF_SUM_FUNC) + { + ac->resultType(colType_MysqlToIDB(item_sum)); + // bug5736. Make the result type double for some window functions when + // plugin variable double_for_decimal_math is set. + ac->adjustResultType(); + } - ac->expressionId(ci->expressionId++); + ac->expressionId(ci->expressionId++); - if (item->full_name()) - ac->alias(item->full_name()); + if (item->full_name()) + ac->alias(item->full_name()); - ac->charsetNumber(item->collation.collation->number); - - // put ac on windowFuncList - gwi.windowFuncList.push_back(ac); - return ac; + ac->charsetNumber(item->collation.collation->number); + // put ac on windowFuncList + gwi.windowFuncList.push_back(ac); + return ac; } -} +} // namespace cal_impl_if diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 0ee6e857f..bc966ffc4 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -14,7 +14,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -//One include file to deal with all the MySQL pollution of the +// One include file to deal with all the MySQL pollution of the // global namespace // // Don't include ANY mysql headers anywhere except here! @@ -28,14 +28,15 @@ #ifdef _MSC_VER #include #if _MSC_VER >= 1800 -template bool isnan(T); +template +bool isnan(T); #endif #endif //#define INFINIDB_DEBUG //#define DEBUG_WALK_COND -#define MYSQL_SERVER 1 //needed for definition of struct THD in mysql_priv.h +#define MYSQL_SERVER 1 // needed for definition of struct THD in mysql_priv.h #define USE_CALPONT_REGEX #undef LOG_INFO @@ -54,7 +55,7 @@ template bool isnan(T); #endif #define DBUG_ON 1 -#undef DBUG_OFF +#undef DBUG_OFF #else #undef SAFE_MUTEX #undef SAFEMALLOC @@ -111,12 +112,12 @@ namespace inline char* idb_mysql_query_str(THD* thd) { #if MYSQL_VERSION_ID >= 50172 - return thd->query(); + return thd->query(); #else - return thd->query; + return thd->query; #endif } -} +} // namespace #endif // vim:ts=4 sw=4: diff --git a/dbcon/mysql/is_columnstore.h b/dbcon/mysql/is_columnstore.h index 487f98d1a..d5bf4aaa3 100644 --- a/dbcon/mysql/is_columnstore.h +++ b/dbcon/mysql/is_columnstore.h @@ -22,65 +22,72 @@ int is_columnstore_files_plugin_init(void* p); int is_columnstore_tables_plugin_init(void* p); int is_columnstore_columns_plugin_init(void* p); - class InformationSchemaCond { - StringBuffer mBufferTableName; - StringBuffer mBufferTableSchema; - String *mTableName; - String *mTableSchema; -public: - InformationSchemaCond() - :mBufferTableName(system_charset_info), - mBufferTableSchema(system_charset_info), - mTableName(nullptr), - mTableSchema(nullptr) - { } - const String *tableName() const { return mTableName; } - const String *tableSchema() const { return mTableSchema; } - static bool eqName(const String &a, const std::string &b) - { - return a.length() == b.length() && !memcmp(a.ptr(), b.data(), a.length()); - } - static bool eqName(const String *a, const std::string &b) - { - return !a || eqName(*a, b); - } - bool match(const std::string &schema, const std::string &table) const - { - return eqName(mTableName, table) && eqName(mTableSchema, schema); - } + StringBuffer mBufferTableName; + StringBuffer mBufferTableSchema; + String* mTableName; + String* mTableSchema; - void getCondItem(Item_field *item_field, Item *item_const) - { - if (strcasecmp(item_field->field_name.str, "table_name") == 0) - { - mTableName = item_const->val_str(&mBufferTableName); - } - else if (strcasecmp(item_field->field_name.str, "table_schema") == 0) - { - mTableSchema = item_const->val_str(&mBufferTableSchema); - } - } + public: + InformationSchemaCond() + : mBufferTableName(system_charset_info) + , mBufferTableSchema(system_charset_info) + , mTableName(nullptr) + , mTableSchema(nullptr) + { + } + const String* tableName() const + { + return mTableName; + } + const String* tableSchema() const + { + return mTableSchema; + } + static bool eqName(const String& a, const std::string& b) + { + return a.length() == b.length() && !memcmp(a.ptr(), b.data(), a.length()); + } + static bool eqName(const String* a, const std::string& b) + { + return !a || eqName(*a, b); + } + bool match(const std::string& schema, const std::string& table) const + { + return eqName(mTableName, table) && eqName(mTableSchema, schema); + } - void getCondItemBoolFunc2(Item_bool_func2 *func) + void getCondItem(Item_field* item_field, Item* item_const) + { + if (strcasecmp(item_field->field_name.str, "table_name") == 0) { - Item_field *field = dynamic_cast(func->arguments()[0]->real_item()); - if (field && func->arguments()[1]->const_item()) - getCondItem(field, func->arguments()[1]); + mTableName = item_const->val_str(&mBufferTableName); } + else if (strcasecmp(item_field->field_name.str, "table_schema") == 0) + { + mTableSchema = item_const->val_str(&mBufferTableSchema); + } + } - void getCondItems(COND* cond) + void getCondItemBoolFunc2(Item_bool_func2* func) + { + Item_field* field = dynamic_cast(func->arguments()[0]->real_item()); + if (field && func->arguments()[1]->const_item()) + getCondItem(field, func->arguments()[1]); + } + + void getCondItems(COND* cond) + { + if (Item_bool_func2* func = dynamic_cast(cond)) { - if (Item_bool_func2 *func = dynamic_cast(cond)) - { - getCondItemBoolFunc2(func); - } - else if (Item_cond_and *subcond = dynamic_cast(cond)) - { - List_iterator li(*(subcond)->argument_list()); - for (Item *item = li++; item ; item = li++ ) - getCondItems(item); - } + getCondItemBoolFunc2(func); } + else if (Item_cond_and* subcond = dynamic_cast(cond)) + { + List_iterator li(*(subcond)->argument_list()); + for (Item* item = li++; item; item = li++) + getCondItems(item); + } + } }; diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 43ff15d08..4076bbd1d 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -32,12 +32,10 @@ #include "is_columnstore.h" using namespace logging; - // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); -ST_FIELD_INFO is_columnstore_columns_fields[] = -{ +ST_FIELD_INFO is_columnstore_columns_fields[] = { Show::Column("TABLE_SCHEMA", Show::Varchar(64), NOT_NULL), Show::Column("TABLE_NAME", Show::Varchar(64), NOT_NULL), Show::Column("COLUMN_NAME", Show::Varchar(64), NOT_NULL), @@ -54,159 +52,153 @@ ST_FIELD_INFO is_columnstore_columns_fields[] = Show::Column("NUMERIC_SCALE", Show::ULong(0), NOT_NULL), Show::Column("IS_AUTOINCREMENT", Show::STiny(0), NOT_NULL), Show::Column("COMPRESSION_TYPE", Show::Varchar(64), NOT_NULL), - Show::CEnd() -}; - + Show::CEnd()}; static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) { - CHARSET_INFO* cs = system_charset_info; - TABLE* table = tables->table; - InformationSchemaCond isCond; + CHARSET_INFO* cs = system_charset_info; + TABLE* table = tables->table; + InformationSchemaCond isCond; - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( + execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); - const std::vector< std::pair > catalog_tables - = systemCatalogPtr->getTables(); + const std::vector< + std::pair > + catalog_tables = systemCatalogPtr->getTables(); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE); - if (cond) + if (cond) + { + isCond.getCondItems(cond); + } + + for (std::vector >::const_iterator it = + catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + if (!isCond.match((*it).second.schema, (*it).second.table)) + continue; + + execplan::CalpontSystemCatalog::RIDList column_rid_list; + + // Note a table may get dropped as you iterate over the list of tables. + // So simply ignore the dropped table. + try { - isCond.getCondItems(cond); + column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true, lower_case_table_names); + } + catch (IDBExcept& ex) + { + if (ex.errorCode() == ERR_TABLE_NOT_IN_CATALOG) + { + continue; + } + else + { + return 1; + } } - for (std::vector >::const_iterator it = catalog_tables.begin(); - it != catalog_tables.end(); ++it) + for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) { - if (!isCond.match((*it).second.schema, (*it).second.table)) - continue; + execplan::CalpontSystemCatalog::TableColName tcn = + systemCatalogPtr->colName(column_rid_list[col_num].objnum); + execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum); - execplan::CalpontSystemCatalog::RIDList column_rid_list; + table->field[0]->store(tcn.schema.c_str(), tcn.schema.length(), cs); + table->field[1]->store(tcn.table.c_str(), tcn.table.length(), cs); + table->field[2]->store(tcn.column.c_str(), tcn.column.length(), cs); + table->field[3]->store(column_rid_list[col_num].objnum); - // Note a table may get dropped as you iterate over the list of tables. - // So simply ignore the dropped table. - try - { - column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true, lower_case_table_names); - } - catch (IDBExcept& ex) - { - if (ex.errorCode() == ERR_TABLE_NOT_IN_CATALOG) - { - continue; - } - else - { - return 1; - } - } + if (ct.ddn.dictOID == std::numeric_limits::min()) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(ct.ddn.dictOID); + } - for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) - { - execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum); - execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum); + if (ct.ddn.listOID == std::numeric_limits::min()) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(ct.ddn.listOID); + } - table->field[0]->store(tcn.schema.c_str(), tcn.schema.length(), cs); - table->field[1]->store(tcn.table.c_str(), tcn.table.length(), cs); - table->field[2]->store(tcn.column.c_str(), tcn.column.length(), cs); - table->field[3]->store(column_rid_list[col_num].objnum); + if (ct.ddn.treeOID == std::numeric_limits::min()) + { + table->field[6]->set_null(); + } + else + { + table->field[6]->set_notnull(); + table->field[6]->store(ct.ddn.treeOID); + } - if (ct.ddn.dictOID == std::numeric_limits::min()) - { - table->field[4]->set_null(); - } - else - { - table->field[4]->set_notnull(); - table->field[4]->store(ct.ddn.dictOID); - } + std::string data_type = execplan::colDataTypeToString(ct.colDataType); + table->field[7]->store(data_type.c_str(), data_type.length(), cs); + table->field[8]->store(ct.colWidth); + table->field[9]->store(ct.colPosition); - if (ct.ddn.listOID == std::numeric_limits::min()) - { - table->field[5]->set_null(); - } - else - { - table->field[5]->set_notnull(); - table->field[5]->store(ct.ddn.listOID); - } + if (ct.defaultValue.empty()) + { + table->field[10]->set_null(); + } + else + { + table->field[10]->set_notnull(); + table->field[10]->store(ct.defaultValue.c_str(), ct.defaultValue.length(), cs); + } - if (ct.ddn.treeOID == std::numeric_limits::min()) - { - table->field[6]->set_null(); - } - else - { - table->field[6]->set_notnull(); - table->field[6]->store(ct.ddn.treeOID); - } + table->field[11]->store(ct.autoincrement); + table->field[12]->store(ct.precision); + table->field[13]->store(ct.scale); - std::string data_type = execplan::colDataTypeToString(ct.colDataType); - table->field[7]->store(data_type.c_str(), data_type.length(), cs); - table->field[8]->store(ct.colWidth); - table->field[9]->store(ct.colPosition); + if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + table->field[14]->store(true); + } + else + { + table->field[14]->store(false); + } - if (ct.defaultValue.empty()) - { - table->field[10]->set_null(); - } - else - { - table->field[10]->set_notnull(); - table->field[10]->store(ct.defaultValue.c_str(), ct.defaultValue.length(), cs); - } + std::string compression_type; - table->field[11]->store(ct.autoincrement); - table->field[12]->store(ct.precision); - table->field[13]->store(ct.scale); + switch (ct.compressionType) + { + case 0: compression_type = "None"; break; - if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - table->field[14]->store(true); - } - else - { - table->field[14]->store(false); - } + case 2: compression_type = "Snappy"; break; - std::string compression_type; + case 3: compression_type = "LZ4"; break; - switch (ct.compressionType) - { - case 0: - compression_type = "None"; - break; + default: compression_type = "Unknown"; break; + } - case 2: - compression_type = "Snappy"; - break; + table->field[15]->store(compression_type.c_str(), compression_type.length(), cs); - case 3: - compression_type = "LZ4"; - break; - - default: - compression_type = "Unknown"; - break; - } - - table->field[15]->store(compression_type.c_str(), compression_type.length(), cs); - - if (schema_table_store_record(thd, table)) - return 1; - } + if (schema_table_store_record(thd, table)) + return 1; } + } - return 0; + return 0; } int is_columnstore_columns_plugin_init(void* p) { - ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; - schema->fields_info = is_columnstore_columns_fields; - schema->fill_table = is_columnstore_columns_fill; - return 0; + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = is_columnstore_columns_fields; + schema->fill_table = is_columnstore_columns_fill; + return 0; } - diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 6589c6b37..0d6b00c3d 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -34,292 +34,269 @@ // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); -ST_FIELD_INFO is_columnstore_extents_fields[] = -{ - Show::Column("OBJECT_ID", Show::ULong(0), NOT_NULL), // 0 - Show::Column("OBJECT_TYPE", Show::Varchar(64), NOT_NULL), // 1 - Show::Column("LOGICAL_BLOCK_START", Show::SLonglong(0), NOT_NULL), // 2 - Show::Column("LOGICAL_BLOCK_END", Show::SLonglong(0), NOT_NULL), // 3 +ST_FIELD_INFO is_columnstore_extents_fields[] = { + Show::Column("OBJECT_ID", Show::ULong(0), NOT_NULL), // 0 + Show::Column("OBJECT_TYPE", Show::Varchar(64), NOT_NULL), // 1 + Show::Column("LOGICAL_BLOCK_START", Show::SLonglong(0), NOT_NULL), // 2 + Show::Column("LOGICAL_BLOCK_END", Show::SLonglong(0), NOT_NULL), // 3 // length=3800 here because sql/sql_i_s.h sets // decimal_precision() as (length / 100) % 100). Not sure why. - Show::Column("MIN_VALUE", Show::Decimal(3800), NULLABLE), // 4 - Show::Column("MAX_VALUE", Show::Decimal(3800), NULLABLE), // 5 - Show::Column("WIDTH", Show::ULong(0), NOT_NULL), // 6 - Show::Column("DBROOT", Show::ULong(0), NOT_NULL), // 7 - Show::Column("PARTITION_ID", Show::ULong(0), NOT_NULL), // 8 - Show::Column("SEGMENT_ID", Show::ULong(0), NOT_NULL), // 9 - Show::Column("BLOCK_OFFSET", Show::ULong(0), NOT_NULL), // 10 - Show::Column("MAX_BLOCKS", Show::ULong(0), NOT_NULL), // 11 - Show::Column("HIGH_WATER_MARK", Show::ULong(0), NOT_NULL), // 12 - Show::Column("STATE", Show::Varchar(64), NOT_NULL), // 13 - Show::Column("STATUS", Show::Varchar(64), NOT_NULL), // 14 - Show::Column("DATA_SIZE", Show::ULonglong(0), NOT_NULL), // 15 - Show::CEnd() -}; + Show::Column("MIN_VALUE", Show::Decimal(3800), NULLABLE), // 4 + Show::Column("MAX_VALUE", Show::Decimal(3800), NULLABLE), // 5 + Show::Column("WIDTH", Show::ULong(0), NOT_NULL), // 6 + Show::Column("DBROOT", Show::ULong(0), NOT_NULL), // 7 + Show::Column("PARTITION_ID", Show::ULong(0), NOT_NULL), // 8 + Show::Column("SEGMENT_ID", Show::ULong(0), NOT_NULL), // 9 + Show::Column("BLOCK_OFFSET", Show::ULong(0), NOT_NULL), // 10 + Show::Column("MAX_BLOCKS", Show::ULong(0), NOT_NULL), // 11 + Show::Column("HIGH_WATER_MARK", Show::ULong(0), NOT_NULL), // 12 + Show::Column("STATE", Show::Varchar(64), NOT_NULL), // 13 + Show::Column("STATUS", Show::Varchar(64), NOT_NULL), // 14 + Show::Column("DATA_SIZE", Show::ULonglong(0), NOT_NULL), // 15 + Show::CEnd()}; static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* thd) { - CHARSET_INFO* cs = system_charset_info; - std::vector entries; - std::vector::iterator iter; - std::vector::iterator end; + CHARSET_INFO* cs = system_charset_info; + std::vector entries; + std::vector::iterator iter; + std::vector::iterator end; - emp->getExtents(oid, entries, false, false, true); + emp->getExtents(oid, entries, false, false, true); - if (entries.size() == 0) - return 0; + if (entries.size() == 0) + return 0; - iter = entries.begin(); - end = entries.end(); + iter = entries.begin(); + end = entries.end(); - while (iter != end) + while (iter != end) + { + table->field[0]->store(oid); + + if (iter->colWid > 0) { - table->field[0]->store(oid); + table->field[1]->store("Column", strlen("Column"), cs); - if (iter->colWid > 0) + if (iter->colWid != datatypes::MAXDECIMALWIDTH) + { + if (iter->partition.cprange.loVal == std::numeric_limits::max() || + iter->partition.cprange.loVal <= (std::numeric_limits::min() + 1)) { - table->field[1]->store("Column", strlen("Column"), cs); - - if (iter->colWid != datatypes::MAXDECIMALWIDTH) - { - if (iter->partition.cprange.loVal == std::numeric_limits::max() || - iter->partition.cprange.loVal <= (std::numeric_limits::min() + 1)) - { - table->field[4]->set_null(); - } - else - { - table->field[4]->set_notnull(); - table->field[4]->store((longlong) iter->partition.cprange.loVal, false); - } - - if (iter->partition.cprange.hiVal <= (std::numeric_limits::min() + 1)) - { - table->field[5]->set_null(); - } - else - { - table->field[5]->set_notnull(); - table->field[5]->store((longlong) iter->partition.cprange.hiVal, false); - } - } - else - { - if (iter->partition.cprange.bigLoVal == utils::maxInt128 || - iter->partition.cprange.bigLoVal <= (utils::minInt128 + 1)) - { - table->field[4]->set_null(); - } - else - { - table->field[4]->set_notnull(); - std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigLoVal) - .toString(); - table->field[4]->store(decAsAStr.c_str(), - decAsAStr.length(), - table->field[4]->charset()); - } - - if (iter->partition.cprange.bigHiVal <= (utils::minInt128 + 1)) - { - table->field[5]->set_null(); - } - else - { - table->field[5]->set_notnull(); - std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigHiVal) - .toString(); - table->field[5]->store(decAsAStr.c_str(), - decAsAStr.length(), - table->field[5]->charset()); - } - } - - table->field[6]->store(iter->colWid); - + table->field[4]->set_null(); } else { - table->field[1]->store("Dictionary", strlen("Dictionary"), cs); - table->field[4]->set_null(); - table->field[5]->set_null(); - table->field[6]->store(8192); + table->field[4]->set_notnull(); + table->field[4]->store((longlong)iter->partition.cprange.loVal, false); } - table->field[2]->store(iter->range.start); - table->field[3]->store(iter->range.start + (iter->range.size * 1024) - 1); - - table->field[7]->store(iter->dbRoot); - table->field[8]->store(iter->partitionNum); - table->field[9]->store(iter->segmentNum); - table->field[10]->store(iter->blockOffset); - table->field[11]->store(iter->range.size * 1024); - table->field[12]->store(iter->HWM); - - switch (iter->partition.cprange.isValid) + if (iter->partition.cprange.hiVal <= (std::numeric_limits::min() + 1)) { - case 0: - table->field[13]->store("Invalid", strlen("Invalid"), cs); - break; - - case 1: - table->field[13]->store("Updating", strlen("Updating"), cs); - break; - - case 2: - table->field[13]->store("Valid", strlen("Valid"), cs); - break; - - default: - table->field[13]->store("Unknown", strlen("Unknown"), cs); - break; - } - - switch (iter->status) - { - case BRM::EXTENTAVAILABLE: - table->field[14]->store("Available", strlen("Available"), cs); - break; - - case BRM::EXTENTUNAVAILABLE: - table->field[14]->store("Unavailable", strlen("Unavailable"), cs); - break; - - case BRM::EXTENTOUTOFSERVICE: - table->field[14]->store("Out of service", strlen("Out of service"), cs); - break; - - default: - table->field[14]->store("Unknown", strlen("Unknown"), cs); - } - - // MCOL-1016: on multiple segments HWM is set to 0 on the lower - // segments, we don't want these to show as 8KB. The down side is - // if the column has less than 1 block it will show as 0 bytes. - // We have no lookahead without it getting messy so this is the - // best compromise. - if (iter->HWM == 0) - { - table->field[15]->store(0); + table->field[5]->set_null(); } else { - table->field[15]->store((iter->HWM + 1) * 8192); + table->field[5]->set_notnull(); + table->field[5]->store((longlong)iter->partition.cprange.hiVal, false); } - - if (schema_table_store_record(thd, table)) + } + else + { + if (iter->partition.cprange.bigLoVal == utils::maxInt128 || + iter->partition.cprange.bigLoVal <= (utils::minInt128 + 1)) { - delete emp; - return 1; + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigLoVal).toString(); + table->field[4]->store(decAsAStr.c_str(), decAsAStr.length(), table->field[4]->charset()); } - iter++; + if (iter->partition.cprange.bigHiVal <= (utils::minInt128 + 1)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigHiVal).toString(); + table->field[5]->store(decAsAStr.c_str(), decAsAStr.length(), table->field[5]->charset()); + } + } + table->field[6]->store(iter->colWid); + } + else + { + table->field[1]->store("Dictionary", strlen("Dictionary"), cs); + table->field[4]->set_null(); + table->field[5]->set_null(); + table->field[6]->store(8192); } - return 0; + table->field[2]->store(iter->range.start); + table->field[3]->store(iter->range.start + (iter->range.size * 1024) - 1); + + table->field[7]->store(iter->dbRoot); + table->field[8]->store(iter->partitionNum); + table->field[9]->store(iter->segmentNum); + table->field[10]->store(iter->blockOffset); + table->field[11]->store(iter->range.size * 1024); + table->field[12]->store(iter->HWM); + + switch (iter->partition.cprange.isValid) + { + case 0: table->field[13]->store("Invalid", strlen("Invalid"), cs); break; + + case 1: table->field[13]->store("Updating", strlen("Updating"), cs); break; + + case 2: table->field[13]->store("Valid", strlen("Valid"), cs); break; + + default: table->field[13]->store("Unknown", strlen("Unknown"), cs); break; + } + + switch (iter->status) + { + case BRM::EXTENTAVAILABLE: table->field[14]->store("Available", strlen("Available"), cs); break; + + case BRM::EXTENTUNAVAILABLE: table->field[14]->store("Unavailable", strlen("Unavailable"), cs); break; + + case BRM::EXTENTOUTOFSERVICE: + table->field[14]->store("Out of service", strlen("Out of service"), cs); + break; + + default: table->field[14]->store("Unknown", strlen("Unknown"), cs); + } + + // MCOL-1016: on multiple segments HWM is set to 0 on the lower + // segments, we don't want these to show as 8KB. The down side is + // if the column has less than 1 block it will show as 0 bytes. + // We have no lookahead without it getting messy so this is the + // best compromise. + if (iter->HWM == 0) + { + table->field[15]->store(0); + } + else + { + table->field[15]->store((iter->HWM + 1) * 8192); + } + + if (schema_table_store_record(thd, table)) + { + delete emp; + return 1; + } + + iter++; + } + + return 0; } static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) { - BRM::OID_t cond_oid = 0; - TABLE* table = tables->table; + BRM::OID_t cond_oid = 0; + TABLE* table = tables->table; - BRM::DBRM::refreshShm(); - BRM::DBRM* emp = new BRM::DBRM(); + BRM::DBRM::refreshShm(); + BRM::DBRM* emp = new BRM::DBRM(); - if (!emp || !emp->isDBRMReady()) + if (!emp || !emp->isDBRMReady()) + { + return 1; + } + + if (cond && cond->type() == Item::FUNC_ITEM) + { + Item_func* fitem = (Item_func*)cond; + + if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2)) { - return 1; + if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && + fitem->arguments()[1]->const_item()) + { + // WHERE object_id = value + Item_field* item_field = (Item_field*)fitem->arguments()[0]->real_item(); + + if (strcasecmp(item_field->field_name.str, "object_id") == 0) + { + cond_oid = fitem->arguments()[1]->val_int(); + return generate_result(cond_oid, emp, table, thd); + } + } + else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && + fitem->arguments()[0]->const_item()) + { + // WHERE value = object_id + Item_field* item_field = (Item_field*)fitem->arguments()[1]->real_item(); + + if (strcasecmp(item_field->field_name.str, "object_id") == 0) + { + cond_oid = fitem->arguments()[0]->val_int(); + return generate_result(cond_oid, emp, table, thd); + } + } } - - if (cond && cond->type() == Item::FUNC_ITEM) + else if (fitem->functype() == Item_func::IN_FUNC) { - Item_func* fitem = (Item_func*) cond; + // WHERE object_id in (value1, value2) + Item_field* item_field = (Item_field*)fitem->arguments()[0]->real_item(); - if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2)) + if (strcasecmp(item_field->field_name.str, "object_id") == 0) + { + for (unsigned int i = 1; i < fitem->argument_count(); i++) { - if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[1]->const_item()) - { - // WHERE object_id = value - Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); + cond_oid = fitem->arguments()[i]->val_int(); + int result = generate_result(cond_oid, emp, table, thd); - if (strcasecmp(item_field->field_name.str, "object_id") == 0) - { - cond_oid = fitem->arguments()[1]->val_int(); - return generate_result(cond_oid, emp, table, thd); - } - } - else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[0]->const_item()) - { - // WHERE value = object_id - Item_field* item_field = (Item_field*) fitem->arguments()[1]->real_item(); - - if (strcasecmp(item_field->field_name.str, "object_id") == 0) - { - cond_oid = fitem->arguments()[0]->val_int(); - return generate_result(cond_oid, emp, table, thd); - } - } - } - else if (fitem->functype() == Item_func::IN_FUNC) - { - // WHERE object_id in (value1, value2) - Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); - - if (strcasecmp(item_field->field_name.str, "object_id") == 0) - { - for (unsigned int i = 1; i < fitem->argument_count(); i++) - { - cond_oid = fitem->arguments()[i]->val_int(); - int result = generate_result(cond_oid, emp, table, thd); - - if (result) - return 1; - } - } - } - else if (fitem->functype() == Item_func::UNKNOWN_FUNC && - strcasecmp(fitem->func_name(), "find_in_set") == 0) - { - // WHERE FIND_IN_SET(object_id, values) - String* tmp_var = fitem->arguments()[1]->val_str(); - std::stringstream ss(tmp_var->ptr()); - - while (ss >> cond_oid) - { - int ret = generate_result(cond_oid, emp, table, thd); - - if (ret) - return 1; - - if (ss.peek() == ',') - ss.ignore(); - } - } - } - - execplan::ObjectIDManager oidm; - BRM::OID_t MaxOID = oidm.size(); - - for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) - { - int result = generate_result(oid, emp, table, thd); - - if (result) + if (result) return 1; + } + } } + else if (fitem->functype() == Item_func::UNKNOWN_FUNC && + strcasecmp(fitem->func_name(), "find_in_set") == 0) + { + // WHERE FIND_IN_SET(object_id, values) + String* tmp_var = fitem->arguments()[1]->val_str(); + std::stringstream ss(tmp_var->ptr()); - delete emp; - return 0; + while (ss >> cond_oid) + { + int ret = generate_result(cond_oid, emp, table, thd); + + if (ret) + return 1; + + if (ss.peek() == ',') + ss.ignore(); + } + } + } + + execplan::ObjectIDManager oidm; + BRM::OID_t MaxOID = oidm.size(); + + for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) + { + int result = generate_result(oid, emp, table, thd); + + if (result) + return 1; + } + + delete emp; + return 0; } int is_columnstore_extents_plugin_init(void* p) { - ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; - schema->fields_info = is_columnstore_extents_fields; - schema->fill_table = is_columnstore_extents_fill; - return 0; + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = is_columnstore_extents_fields; + schema->fill_table = is_columnstore_extents_fill; + return 0; } diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 91cb4e83f..5c3ea2d12 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -42,281 +42,279 @@ // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); -ST_FIELD_INFO is_columnstore_files_fields[] = -{ +ST_FIELD_INFO is_columnstore_files_fields[] = { Show::Column("OBJECT_ID", Show::ULong(0), NOT_NULL), Show::Column("SEGMENT_ID", Show::ULong(0), NOT_NULL), Show::Column("PARTITION_ID", Show::ULong(0), NOT_NULL), Show::Column("FILENAME", Show::Varchar(1024), NOT_NULL), Show::Column("FILE_SIZE", Show::ULonglong(0), NULLABLE), Show::Column("COMPRESSED_DATA_SIZE", Show::ULonglong(0), NULLABLE), - Show::CEnd() -}; + Show::CEnd()}; -static bool get_file_sizes(THD* thd, messageqcpp::MessageQueueClient* msgQueueClient, const char* fileName, off_t* fileSize, off_t* compressedFileSize) +static bool get_file_sizes(THD* thd, messageqcpp::MessageQueueClient* msgQueueClient, const char* fileName, + off_t* fileSize, off_t* compressedFileSize) { - messageqcpp::ByteStream bs; - messageqcpp::ByteStream::byte rc; - std::string errMsg; + messageqcpp::ByteStream bs; + messageqcpp::ByteStream::byte rc; + std::string errMsg; - try + try + { + bs << (messageqcpp::ByteStream::byte)WriteEngine::WE_SVR_GET_FILESIZE; + // header?? + bs << fileName; + msgQueueClient->write(bs); + // namespace?? + messageqcpp::SBS sbs; + sbs = msgQueueClient->read(); + + if (sbs->length() == 0) { - bs << (messageqcpp::ByteStream::byte) WriteEngine::WE_SVR_GET_FILESIZE; - // header?? - bs << fileName; - msgQueueClient->write(bs); - // namespace?? - messageqcpp::SBS sbs; - sbs = msgQueueClient->read(); - - if (sbs->length() == 0) - { - delete msgQueueClient; - return false; - } - - *sbs >> rc; - *sbs >> errMsg; - if (rc) - { - ha_mcs_impl::log_this(thd, - "I_S::COLUMNSTORE_FILE::get_file_sizes(): WriteEngineServer returns an error().", - logging::LOG_TYPE_ERROR, thd->thread_id); - return false; - } - *sbs >> *fileSize; - *sbs >> *compressedFileSize; - return true; + delete msgQueueClient; + return false; } - catch (...) + + *sbs >> rc; + *sbs >> errMsg; + if (rc) { - return false; + ha_mcs_impl::log_this(thd, + "I_S::COLUMNSTORE_FILE::get_file_sizes(): WriteEngineServer returns an error().", + logging::LOG_TYPE_ERROR, thd->thread_id); + return false; } + *sbs >> *fileSize; + *sbs >> *compressedFileSize; + return true; + } + catch (...) + { + return false; + } } static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* thd) { - std::vector entries; - CHARSET_INFO* cs = system_charset_info; + std::vector entries; + CHARSET_INFO* cs = system_charset_info; - char oidDirName[WriteEngine::FILE_NAME_SIZE]; - char fullFileName[WriteEngine::FILE_NAME_SIZE]; - char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; + char oidDirName[WriteEngine::FILE_NAME_SIZE]; + char fullFileName[WriteEngine::FILE_NAME_SIZE]; + char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; - config::Config* config; + config::Config* config; + + try + { + config = config::Config::makeConfig(); + } + catch (std::runtime_error& e) + { + thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + return ER_INTERNAL_ERROR; + } + + WriteEngine::Config we_config; + off_t fileSize = 0; + off_t compressedFileSize = 0; + we_config.initConfigCache(); + messageqcpp::MessageQueueClient* msgQueueClient; + oam::Oam oam_instance; + int pmId = 0; + int rc; + + emp->getExtents(oid, entries, false, false, true); + + if (entries.size() == 0) + return 0; + + std::vector::const_iterator iter = entries.begin(); + + while (iter != entries.end()) // organize extents into files + { + // Don't include files more than once at different block offsets + if (iter->blockOffset > 0) + { + iter++; + continue; + } try { - config = config::Config::makeConfig(); + oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); } - catch (std::runtime_error& e) + catch (std::runtime_error&) { - thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); - return ER_INTERNAL_ERROR; + // MCOL-1116: If we are here a DBRoot is offline/missing + iter++; + return 0; } - WriteEngine::Config we_config; - off_t fileSize = 0; - off_t compressedFileSize = 0; - we_config.initConfigCache(); - messageqcpp::MessageQueueClient* msgQueueClient; - oam::Oam oam_instance; - int pmId = 0; - int rc; + table->field[0]->store(oid); + table->field[1]->store(iter->segmentNum); + table->field[2]->store(iter->partitionNum); - emp->getExtents(oid, entries, false, false, true); + WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum); + std::stringstream DbRootName; + DbRootName << "DBRoot" << iter->dbRoot; + std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str()); + fileSize = compressedFileSize = 0; + rc = snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName); - if (entries.size() == 0) - return 0; + std::ostringstream oss; + oss << "pm" << pmId << "_WriteEngineServer"; + std::string client = oss.str(); + msgQueueClient = messageqcpp::MessageQueueClientPool::getInstance(oss.str()); - std::vector::const_iterator iter = entries.begin(); - - while ( iter != entries.end() ) //organize extents into files + // snprintf output truncation check + if (rc == WriteEngine::FILE_NAME_SIZE || + !get_file_sizes(thd, msgQueueClient, fullFileName, &fileSize, &compressedFileSize)) { - // Don't include files more than once at different block offsets - if (iter->blockOffset > 0) - { - iter++; - continue; - } - - try - { - oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); - } - catch (std::runtime_error&) - { - // MCOL-1116: If we are here a DBRoot is offline/missing - iter++; - return 0; - } - - table->field[0]->store(oid); - table->field[1]->store(iter->segmentNum); - table->field[2]->store(iter->partitionNum); - - WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum); - std::stringstream DbRootName; - DbRootName << "DBRoot" << iter->dbRoot; - std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str()); - fileSize = compressedFileSize = 0; - rc = snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName); - - std::ostringstream oss; - oss << "pm" << pmId << "_WriteEngineServer"; - std::string client = oss.str(); - msgQueueClient = messageqcpp::MessageQueueClientPool::getInstance(oss.str()); - - // snprintf output truncation check - if (rc == WriteEngine::FILE_NAME_SIZE || - !get_file_sizes(thd, msgQueueClient, fullFileName, &fileSize, &compressedFileSize)) - { - messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); - delete emp; - return 1; - } - - table->field[3]->store(fullFileName, strlen(fullFileName), cs); - - if (fileSize > 0) - { - table->field[4]->set_notnull(); - table->field[4]->store(fileSize); - - if (compressedFileSize > 0) - { - table->field[5]->set_notnull(); - table->field[5]->store(compressedFileSize); - } - else - { - table->field[5]->set_null(); - } - } - else - { - table->field[4]->set_null(); - table->field[5]->set_null(); - } - - if (schema_table_store_record(thd, table)) - { - messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); - delete emp; - return 1; - } - - iter++; - messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); - msgQueueClient = NULL; + messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); + delete emp; + return 1; } - return 0; + table->field[3]->store(fullFileName, strlen(fullFileName), cs); + + if (fileSize > 0) + { + table->field[4]->set_notnull(); + table->field[4]->store(fileSize); + + if (compressedFileSize > 0) + { + table->field[5]->set_notnull(); + table->field[5]->store(compressedFileSize); + } + else + { + table->field[5]->set_null(); + } + } + else + { + table->field[4]->set_null(); + table->field[5]->set_null(); + } + + if (schema_table_store_record(thd, table)) + { + messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); + delete emp; + return 1; + } + + iter++; + messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); + msgQueueClient = NULL; + } + + return 0; } static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) { - BRM::DBRM::refreshShm(); - BRM::DBRM* emp = new BRM::DBRM(); - BRM::OID_t cond_oid = 0; - TABLE* table = tables->table; + BRM::DBRM::refreshShm(); + BRM::DBRM* emp = new BRM::DBRM(); + BRM::OID_t cond_oid = 0; + TABLE* table = tables->table; - if (!emp || !emp->isDBRMReady()) + if (!emp || !emp->isDBRMReady()) + { + return 1; + } + + if (cond && cond->type() == Item::FUNC_ITEM) + { + Item_func* fitem = (Item_func*)cond; + + if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2)) { + if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && + fitem->arguments()[1]->const_item()) + { + // WHERE object_id = value + Item_field* item_field = (Item_field*)fitem->arguments()[0]->real_item(); + + if (strcasecmp(item_field->field_name.str, "object_id") == 0) + { + cond_oid = fitem->arguments()[1]->val_int(); + return generate_result(cond_oid, emp, table, thd); + } + } + else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && + fitem->arguments()[0]->const_item()) + { + // WHERE value = object_id + Item_field* item_field = (Item_field*)fitem->arguments()[1]->real_item(); + + if (strcasecmp(item_field->field_name.str, "object_id") == 0) + { + cond_oid = fitem->arguments()[0]->val_int(); + return generate_result(cond_oid, emp, table, thd); + } + } + } + else if (fitem->functype() == Item_func::IN_FUNC) + { + // WHERE object_id in (value1, value2) + Item_field* item_field = (Item_field*)fitem->arguments()[0]->real_item(); + + if (strcasecmp(item_field->field_name.str, "object_id") == 0) + { + for (unsigned int i = 1; i < fitem->argument_count(); i++) + { + cond_oid = fitem->arguments()[i]->val_int(); + int result = generate_result(cond_oid, emp, table, thd); + + if (result) + return 1; + } + } + } + else if (fitem->functype() == Item_func::UNKNOWN_FUNC && + strcasecmp(fitem->func_name(), "find_in_set") == 0) + { + // WHERE FIND_IN_SET(object_id, values) + String* tmp_var = fitem->arguments()[1]->val_str(); + std::stringstream ss(tmp_var->ptr()); + + while (ss >> cond_oid) + { + int ret = generate_result(cond_oid, emp, table, thd); + + if (ret) + return 1; + + if (ss.peek() == ',') + ss.ignore(); + } + } + } + + execplan::ObjectIDManager oidm; + BRM::OID_t MaxOID = oidm.size(); + + if (!cond_oid) + { + for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) + { + int result = generate_result(oid, emp, table, thd); + + if (result) return 1; } + } - if (cond && cond->type() == Item::FUNC_ITEM) - { - Item_func* fitem = (Item_func*) cond; - - if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2)) - { - if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[1]->const_item()) - { - // WHERE object_id = value - Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); - - if (strcasecmp(item_field->field_name.str, "object_id") == 0) - { - cond_oid = fitem->arguments()[1]->val_int(); - return generate_result(cond_oid, emp, table, thd); - } - } - else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[0]->const_item()) - { - // WHERE value = object_id - Item_field* item_field = (Item_field*) fitem->arguments()[1]->real_item(); - - if (strcasecmp(item_field->field_name.str, "object_id") == 0) - { - cond_oid = fitem->arguments()[0]->val_int(); - return generate_result(cond_oid, emp, table, thd); - } - } - } - else if (fitem->functype() == Item_func::IN_FUNC) - { - // WHERE object_id in (value1, value2) - Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); - - if (strcasecmp(item_field->field_name.str, "object_id") == 0) - { - for (unsigned int i = 1; i < fitem->argument_count(); i++) - { - cond_oid = fitem->arguments()[i]->val_int(); - int result = generate_result(cond_oid, emp, table, thd); - - if (result) - return 1; - } - } - } - else if (fitem->functype() == Item_func::UNKNOWN_FUNC && - strcasecmp(fitem->func_name(), "find_in_set") == 0) - { - // WHERE FIND_IN_SET(object_id, values) - String* tmp_var = fitem->arguments()[1]->val_str(); - std::stringstream ss(tmp_var->ptr()); - - while (ss >> cond_oid) - { - int ret = generate_result(cond_oid, emp, table, thd); - - if (ret) - return 1; - - if (ss.peek() == ',') - ss.ignore(); - } - } - } - - execplan::ObjectIDManager oidm; - BRM::OID_t MaxOID = oidm.size(); - - if (!cond_oid) - { - for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) - { - int result = generate_result(oid, emp, table, thd); - - if (result) - return 1; - } - } - - delete emp; - return 0; + delete emp; + return 0; } int is_columnstore_files_plugin_init(void* p) { - ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; - schema->fields_info = is_columnstore_files_fields; - schema->fill_table = is_columnstore_files_fill; - return 0; + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = is_columnstore_files_fields; + schema->fill_table = is_columnstore_files_fill; + return 0; } - diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp index 5ba5e95a3..63b8517cb 100644 --- a/dbcon/mysql/is_columnstore_tables.cpp +++ b/dbcon/mysql/is_columnstore_tables.cpp @@ -32,82 +32,82 @@ // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); -ST_FIELD_INFO is_columnstore_tables_fields[] = -{ +ST_FIELD_INFO is_columnstore_tables_fields[] = { Show::Column("TABLE_SCHEMA", Show::Varchar(64), NOT_NULL), Show::Column("TABLE_NAME", Show::Varchar(64), NOT_NULL), Show::Column("OBJECT_ID", Show::SLong(0), NOT_NULL), - Show::Column("CREATION_DATE", Show::Datetime(0), NOT_NULL), // TODO: Make a date if possible + Show::Column("CREATION_DATE", Show::Datetime(0), NOT_NULL), // TODO: Make a date if possible Show::Column("COLUMN_COUNT", Show::SLong(0), NOT_NULL), Show::Column("AUTOINCREMENT", Show::SLong(0), NULLABLE), - Show::CEnd() -}; - + Show::CEnd()}; static int is_columnstore_tables_fill(THD* thd, TABLE_LIST* tables, COND* cond) { - CHARSET_INFO* cs = system_charset_info; - TABLE* table = tables->table; - InformationSchemaCond isCond; + CHARSET_INFO* cs = system_charset_info; + TABLE* table = tables->table; + InformationSchemaCond isCond; - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( + execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE); - if (cond) + if (cond) + { + isCond.getCondItems(cond); + } + + const std::vector< + std::pair > + catalog_tables = systemCatalogPtr->getTables(); + + for (std::vector >::const_iterator it = + catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + if (!isCond.match((*it).second.schema, (*it).second.table)) + continue; + + try { - isCond.getCondItems(cond); + execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second); + std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date); + table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs); + table->field[1]->store((*it).second.table.c_str(), (*it).second.table.length(), cs); + table->field[2]->store((*it).first); + table->field[3]->store(create_date.c_str(), create_date.length(), cs); + table->field[4]->store(tb_info.numOfCols); + + if (tb_info.tablewithautoincr) + { + table->field[5]->set_notnull(); + table->field[5]->store(systemCatalogPtr->nextAutoIncrValue((*it).second)); + } + else + { + table->field[5]->set_null(); + } + + table->field[5]->store(tb_info.tablewithautoincr); + + if (schema_table_store_record(thd, table)) + return 1; } - - const std::vector< std::pair > catalog_tables - = systemCatalogPtr->getTables(); - - for (std::vector >::const_iterator it = catalog_tables.begin(); - it != catalog_tables.end(); ++it) + catch (std::runtime_error& e) { - if (!isCond.match((*it).second.schema, (*it).second.table)) - continue; - - try - { - execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second); - std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date); - table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs); - table->field[1]->store((*it).second.table.c_str(), (*it).second.table.length(), cs); - table->field[2]->store((*it).first); - table->field[3]->store(create_date.c_str(), create_date.length(), cs); - table->field[4]->store(tb_info.numOfCols); - - if (tb_info.tablewithautoincr) - { - table->field[5]->set_notnull(); - table->field[5]->store(systemCatalogPtr->nextAutoIncrValue((*it).second)); - } - else - { - table->field[5]->set_null(); - } - - table->field[5]->store(tb_info.tablewithautoincr); - - if (schema_table_store_record(thd, table)) - return 1; - } - catch (std::runtime_error& e) - { - std::cerr << e.what() << std::endl; - } + std::cerr << e.what() << std::endl; } + } - return 0; + return 0; } int is_columnstore_tables_plugin_init(void* p) { - ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; - schema->fields_info = is_columnstore_tables_fields; - schema->fill_table = is_columnstore_tables_fill; - return 0; + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = is_columnstore_tables_fields; + schema->fill_table = is_columnstore_tables_fill; + return 0; } - diff --git a/dbcon/mysql/sm.cpp b/dbcon/mysql/sm.cpp index a4894c543..7ca9116b0 100644 --- a/dbcon/mysql/sm.cpp +++ b/dbcon/mysql/sm.cpp @@ -20,7 +20,6 @@ * ***********************************************************************/ - #define PREFER_MY_CONFIG_H #include #include @@ -65,214 +64,217 @@ using namespace sm; // @bug 159 fix. clean up routine when error happened void cleanup(cpsm_conhdl_t* hndl) { - // remove system catalog instance for this statement. - CalpontSystemCatalog::removeCalpontSystemCatalog(hndl->sessionID); - hndl->queryState = NO_QUERY; - hndl->resultSet.erase(hndl->resultSet.begin(), hndl->resultSet.end()); + // remove system catalog instance for this statement. + CalpontSystemCatalog::removeCalpontSystemCatalog(hndl->sessionID); + hndl->queryState = NO_QUERY; + hndl->resultSet.erase(hndl->resultSet.begin(), hndl->resultSet.end()); } status_t tpl_scan_fetch_getband(cpsm_conhdl_t* hndl, sp_cpsm_tplsch_t& ntplsch, int* killed) { - // @bug 649 check keybandmap first - map::iterator keyBandMapIter = hndl->keyBandMap.find(ntplsch->key); + // @bug 649 check keybandmap first + map::iterator keyBandMapIter = hndl->keyBandMap.find(ntplsch->key); - try + try + { + if (keyBandMapIter != hndl->keyBandMap.end()) { - if (keyBandMapIter != hndl->keyBandMap.end()) - { - ByteStream bs; - ostringstream oss; - oss << DEFAULT_SAVE_PATH << '/' << hndl->sessionID << '/' << ntplsch->key << '_' << ntplsch->bandID << ".band"; - ifstream bandFile (oss.str().c_str(), ios::in); - bandFile >> bs; - unlink (oss.str().c_str()); + ByteStream bs; + ostringstream oss; + oss << DEFAULT_SAVE_PATH << '/' << hndl->sessionID << '/' << ntplsch->key << '_' << ntplsch->bandID + << ".band"; + ifstream bandFile(oss.str().c_str(), ios::in); + bandFile >> bs; + unlink(oss.str().c_str()); - // not possible for vtable - ntplsch->deserializeTable(bs); - ntplsch->bandID++; + // not possible for vtable + ntplsch->deserializeTable(bs); + ntplsch->bandID++; - // end of result set - if (ntplsch->bandID == keyBandMapIter->second) - { - hndl->keyBandMap.erase(keyBandMapIter); - return SQL_NOT_FOUND; - } - } - else - { - ByteStream bs; - - // @bug 626. check saveFlag. If SAVING, read band from socket and save to disk - if (ntplsch->saveFlag == SAVING) - { - ByteStream bs; - // @bug 2244. Bypass ClientRotator::read() because if I/O error occurs, it tries - // to reestablish a connection with ExeMgr which ends up causing mysql - // session to hang. - bs = hndl->exeMgr->read(); - ostringstream oss; - oss << DEFAULT_SAVE_PATH << '/' << hndl->sessionID << '/' << ntplsch->tableid << '_' << ntplsch->bandsReturned << ".band"; - ofstream saveFile (oss.str().c_str(), ios::out); - saveFile << bs; - saveFile.close(); - ntplsch->bandsReturned++; - // not possible for vtable - ntplsch->deserializeTable(bs); - } - // if SAVED, read from saved file. not possible for vtable - else if (ntplsch->saveFlag == SAVED) - { - ostringstream oss; - oss << DEFAULT_SAVE_PATH << '/' << hndl->sessionID << '/' << ntplsch->tableid << '_' << ntplsch->bandsReturned << ".band"; - ifstream saveFile (oss.str().c_str(), ios::in); - saveFile >> bs; - saveFile.close(); - ntplsch->bandsReturned++; - ntplsch->deserializeTable(bs); - } - // most normal path. also the path for vtable - else - { - ntplsch->bs.restart(); - // @bug 2244. Bypass ClientRotator::read() because if I/O error occurs, it tries - // to reestablish a connection with ExeMgr which ends up causing mysql - // session to hang. - // @bug 3386. need to abort the query when user does ctrl+c - timespec t; - t.tv_sec = 5L; - t.tv_nsec = 0L; - - if (killed && *killed) - return SQL_KILLED; - - ntplsch->bs = hndl->exeMgr->read(); - - if (ntplsch->bs.length() != 0) - { - ntplsch->deserializeTable(ntplsch->bs); - - if (ntplsch->rowGroup && ntplsch->rowGroup->getRGData() == NULL) - { - ntplsch->bs.restart(); - // @bug 2244. Bypass ClientRotator::read() because if I/O error occurs, it tries - // to reestablish a connection with ExeMgr which ends up causing mysql - // session to hang. - bool timeout = true; - - while (timeout) - { - timeout = false; - ntplsch->bs = hndl->exeMgr->getClient()->read(&t, &timeout); - - if (killed && *killed) - return SQL_KILLED; - } - - if (ntplsch->bs.length() == 0) - { - hndl->curFetchTb = 0; - return logging::ERR_LOST_CONN_EXEMGR; - } - - ntplsch->deserializeTable(ntplsch->bs); - } - - uint16_t error = ntplsch->getStatus(); - - if (0 != error) - { - ntplsch->setErrMsg(); - return error; - } - } - else // @todo error handling - { - hndl->curFetchTb = 0; - - if (ntplsch->saveFlag == NO_SAVE) - hndl->tidScanMap[ntplsch->tableid] = ntplsch; - - ntplsch->errMsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - return logging::ERR_LOST_CONN_EXEMGR; - } - } - - // All done with this table. reset curFetchTb when finish SOCKET reading - if (ntplsch->getRowCount() == 0) - { - hndl->curFetchTb = 0; - - if (ntplsch->saveFlag == NO_SAVE) - hndl->tidScanMap[ntplsch->tableid] = ntplsch; - - return SQL_NOT_FOUND; - } - } + // end of result set + if (ntplsch->bandID == keyBandMapIter->second) + { + hndl->keyBandMap.erase(keyBandMapIter); + return SQL_NOT_FOUND; + } } - catch (std::exception& e) + else { + ByteStream bs; + + // @bug 626. check saveFlag. If SAVING, read band from socket and save to disk + if (ntplsch->saveFlag == SAVING) + { + ByteStream bs; + // @bug 2244. Bypass ClientRotator::read() because if I/O error occurs, it tries + // to reestablish a connection with ExeMgr which ends up causing mysql + // session to hang. + bs = hndl->exeMgr->read(); + ostringstream oss; + oss << DEFAULT_SAVE_PATH << '/' << hndl->sessionID << '/' << ntplsch->tableid << '_' + << ntplsch->bandsReturned << ".band"; + ofstream saveFile(oss.str().c_str(), ios::out); + saveFile << bs; + saveFile.close(); + ntplsch->bandsReturned++; + // not possible for vtable + ntplsch->deserializeTable(bs); + } + // if SAVED, read from saved file. not possible for vtable + else if (ntplsch->saveFlag == SAVED) + { + ostringstream oss; + oss << DEFAULT_SAVE_PATH << '/' << hndl->sessionID << '/' << ntplsch->tableid << '_' + << ntplsch->bandsReturned << ".band"; + ifstream saveFile(oss.str().c_str(), ios::in); + saveFile >> bs; + saveFile.close(); + ntplsch->bandsReturned++; + ntplsch->deserializeTable(bs); + } + // most normal path. also the path for vtable + else + { + ntplsch->bs.restart(); + // @bug 2244. Bypass ClientRotator::read() because if I/O error occurs, it tries + // to reestablish a connection with ExeMgr which ends up causing mysql + // session to hang. + // @bug 3386. need to abort the query when user does ctrl+c + timespec t; + t.tv_sec = 5L; + t.tv_nsec = 0L; + + if (killed && *killed) + return SQL_KILLED; + + ntplsch->bs = hndl->exeMgr->read(); + + if (ntplsch->bs.length() != 0) + { + ntplsch->deserializeTable(ntplsch->bs); + + if (ntplsch->rowGroup && ntplsch->rowGroup->getRGData() == NULL) + { + ntplsch->bs.restart(); + // @bug 2244. Bypass ClientRotator::read() because if I/O error occurs, it tries + // to reestablish a connection with ExeMgr which ends up causing mysql + // session to hang. + bool timeout = true; + + while (timeout) + { + timeout = false; + ntplsch->bs = hndl->exeMgr->getClient()->read(&t, &timeout); + + if (killed && *killed) + return SQL_KILLED; + } + + if (ntplsch->bs.length() == 0) + { + hndl->curFetchTb = 0; + return logging::ERR_LOST_CONN_EXEMGR; + } + + ntplsch->deserializeTable(ntplsch->bs); + } + + uint16_t error = ntplsch->getStatus(); + + if (0 != error) + { + ntplsch->setErrMsg(); + return error; + } + } + else // @todo error handling + { + hndl->curFetchTb = 0; + + if (ntplsch->saveFlag == NO_SAVE) + hndl->tidScanMap[ntplsch->tableid] = ntplsch; + + ntplsch->errMsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + return logging::ERR_LOST_CONN_EXEMGR; + } + } + + // All done with this table. reset curFetchTb when finish SOCKET reading + if (ntplsch->getRowCount() == 0) + { hndl->curFetchTb = 0; if (ntplsch->saveFlag == NO_SAVE) - hndl->tidScanMap[ntplsch->tableid] = ntplsch; + hndl->tidScanMap[ntplsch->tableid] = ntplsch; - ntplsch->errMsg = e.what(); - return logging::ERR_LOST_CONN_EXEMGR; + return SQL_NOT_FOUND; + } } + } + catch (std::exception& e) + { + hndl->curFetchTb = 0; - ntplsch->rowsreturned = 0; - return STATUS_OK; + if (ntplsch->saveFlag == NO_SAVE) + hndl->tidScanMap[ntplsch->tableid] = ntplsch; + + ntplsch->errMsg = e.what(); + return logging::ERR_LOST_CONN_EXEMGR; + } + + ntplsch->rowsreturned = 0; + return STATUS_OK; } void end_query(cpsm_conhdl_t* hndl) { - // remove system catalog instance for this statement. - // @bug 695. turn on system catalog session cache for FE - // CalpontSystemCatalog::removeCalpontSystemCatalog(hndl->sessionID); - hndl->queryState = NO_QUERY; - // reset at the end of query - hndl->curFetchTb = 0; - // @bug 626 clear up - hndl->tidMap.clear(); - hndl->tidScanMap.clear(); - hndl->keyBandMap.clear(); + // remove system catalog instance for this statement. + // @bug 695. turn on system catalog session cache for FE + // CalpontSystemCatalog::removeCalpontSystemCatalog(hndl->sessionID); + hndl->queryState = NO_QUERY; + // reset at the end of query + hndl->curFetchTb = 0; + // @bug 626 clear up + hndl->tidMap.clear(); + hndl->tidScanMap.clear(); + hndl->keyBandMap.clear(); - // Tell ExeMgr we are done with this query - try - { - ByteStream bs; - ByteStream::quadbyte qb = 0; - bs << qb; - hndl->write(bs); - } - catch (...) - { - throw; - } + // Tell ExeMgr we are done with this query + try + { + ByteStream bs; + ByteStream::quadbyte qb = 0; + bs << qb; + hndl->write(bs); + } + catch (...) + { + throw; + } } // @bug 1054, 863 - SIGPIPE handler bool sigFlag = false; void sighandler(int sig_num) { - FILE* p; - char buf[1024]; + FILE* p; + char buf[1024]; - string tmpDir = startup::StartUp::tmpDir() + "/f1.dat"; - const char *cstr = tmpDir.c_str(); - - if ((p = fopen(cstr, "a")) != NULL) - { - snprintf(buf, 1024, "sighandler() hit with %d\n", sig_num); - fwrite(buf, 1, strlen(buf), p); - fclose(p); - } + string tmpDir = startup::StartUp::tmpDir() + "/f1.dat"; + const char* cstr = tmpDir.c_str(); - sigFlag = true; - throw runtime_error("zerror"); + if ((p = fopen(cstr, "a")) != NULL) + { + snprintf(buf, 1024, "sighandler() hit with %d\n", sig_num); + fwrite(buf, 1, strlen(buf), p); + fclose(p); + } + + sigFlag = true; + throw runtime_error("zerror"); } -} +} // namespace namespace sm { @@ -282,246 +284,229 @@ const std::string DEFAULT_SAVE_PATH = "C:\\Calpont\\tmp"; const std::string DEFAULT_SAVE_PATH = "/var/tmp"; #endif -status_t -tpl_open ( tableid_t tableid, - cpsm_tplh_t* ntplh, - cpsm_conhdl_t* conn_hdl) +status_t tpl_open(tableid_t tableid, cpsm_tplh_t* ntplh, cpsm_conhdl_t* conn_hdl) { - SMDEBUGLOG << "tpl_open: ntplh: " << ntplh << " conn_hdl: " << conn_hdl << " tableid: " << tableid << endl; + SMDEBUGLOG << "tpl_open: ntplh: " << ntplh << " conn_hdl: " << conn_hdl << " tableid: " << tableid << endl; - // if first time enter this function for a statement, set - // queryState to QUERY_IN_PROCESS and get execution plan. - if (conn_hdl->queryState == NO_QUERY) + // if first time enter this function for a statement, set + // queryState to QUERY_IN_PROCESS and get execution plan. + if (conn_hdl->queryState == NO_QUERY) + { + conn_hdl->queryState = QUERY_IN_PROCESS; + } + + try + { + // @bug 626. check saveFlag, if SAVED, do not project + if (ntplh->saveFlag != SAVED) { - conn_hdl->queryState = QUERY_IN_PROCESS; + // Request ExeMgr to start projecting table tableid + CalpontSystemCatalog::OID tableOID = static_cast(tableid); + ByteStream::quadbyte qb = static_cast(tableOID); + ByteStream bs; + bs << qb; + conn_hdl->write(bs); + } + } + catch (std::exception& ex) + { + SMDEBUGLOG << "Exception caught in tpl_open: " << ex.what() << endl; + cleanup(conn_hdl); + return CALPONT_INTERNAL_ERROR; + } + + ntplh->tableid = tableid; + + return STATUS_OK; +} + +status_t tpl_scan_open(tableid_t tableid, sp_cpsm_tplsch_t& ntplsch, cpsm_conhdl_t* conn_hdl) +{ +#if IDB_SM_DEBUG + SMDEBUGLOG << "tpl_scan_open: " << conn_hdl << " tableid: " << tableid << endl; +#endif + + // @bug 649. No initialization here. take passed in reference + ntplsch->tableid = tableid; + + ntplsch->rowsreturned = 0; + return STATUS_OK; +} + +status_t tpl_scan_fetch(sp_cpsm_tplsch_t& ntplsch, cpsm_conhdl_t* conn_hdl, int* killed) +{ + // @770. force end of result set when this is not the first table to be fetched. + if (ntplsch->traceFlags & CalpontSelectExecutionPlan::TRACE_NO_ROWS2) + if (conn_hdl->tidScanMap.size() >= 1) + return SQL_NOT_FOUND; + + // need another band + status_t status = STATUS_OK; + + if (ntplsch->rowsreturned == ntplsch->getRowCount()) + status = tpl_scan_fetch_getband(conn_hdl, ntplsch, killed); + + return status; +} + +status_t tpl_scan_close(sp_cpsm_tplsch_t& ntplsch) +{ +#if IDB_SM_DEBUG + SMDEBUGLOG << "tpl_scan_close: "; + + if (ntplsch) + SMDEBUGLOG << "tpl_scan_close: ntplsch " << ntplsch; + SMDEBUGLOG << "tpl_scan_close: tableid: " << ntplsch->tableid << endl; +#endif + ntplsch.reset(); + + return STATUS_OK; +} + +status_t tpl_close(cpsm_tplh_t* ntplh, cpsm_conhdl_t** conn_hdl, QueryStats& stats, bool ask_4_stats, + bool clear_scan_ctx) +{ + cpsm_conhdl_t* hndl = *conn_hdl; + SMDEBUGLOG << "tpl_close: hndl" << hndl << " ntplh " << ntplh; + + if (ntplh) + SMDEBUGLOG << " tableid: " << ntplh->tableid; + + SMDEBUGLOG << endl; + delete ntplh; + + // determine end of result set and end of statement execution + if (hndl->queryState == QUERY_IN_PROCESS) + { + // Get the query stats + ByteStream bs; + // Ask for a stats only if a user explicitly asks + if (ask_4_stats) + { + ByteStream::quadbyte qb = 3; + bs << qb; + hndl->write(bs); + } + // MCOL-1601 Dispose of unused empty RowGroup + if (clear_scan_ctx) + { + SMDEBUGLOG << "tpl_close() clear_scan_ctx read" << std::endl; + bs = hndl->exeMgr->read(); } - try + SMDEBUGLOG << "tpl_close hndl->exeMgr: " << hndl->exeMgr << endl; + // keep reading until we get a string + // Ask for a stats only if a user explicitly asks + if (ask_4_stats) { - // @bug 626. check saveFlag, if SAVED, do not project - if (ntplh->saveFlag != SAVED) + for (int tries = 0; tries < 10; tries++) + { + bs = hndl->exeMgr->read(); + + if (bs.length() == 0) + break; + + try { - // Request ExeMgr to start projecting table tableid - CalpontSystemCatalog::OID tableOID = static_cast(tableid); - ByteStream::quadbyte qb = static_cast(tableOID); - ByteStream bs; - bs << qb; - conn_hdl->write(bs); + bs >> hndl->queryStats; + bs >> hndl->extendedStats; + bs >> hndl->miniStats; + stats.unserialize(bs); + stats.setEndTime(); + stats.insert(); + break; } - } - catch (std::exception& ex) - { - SMDEBUGLOG << "Exception caught in tpl_open: " << ex.what() << endl; - cleanup(conn_hdl); - return CALPONT_INTERNAL_ERROR; - } - - ntplh->tableid = tableid; - - return STATUS_OK; -} - -status_t -tpl_scan_open ( tableid_t tableid, - sp_cpsm_tplsch_t& ntplsch, - cpsm_conhdl_t* conn_hdl ) -{ -#if IDB_SM_DEBUG - SMDEBUGLOG << "tpl_scan_open: " << conn_hdl << " tableid: " << tableid << endl; -#endif - - // @bug 649. No initialization here. take passed in reference - ntplsch->tableid = tableid; - - ntplsch->rowsreturned = 0; - return STATUS_OK; -} - -status_t -tpl_scan_fetch ( sp_cpsm_tplsch_t& ntplsch, - cpsm_conhdl_t* conn_hdl, - int* killed ) -{ - // @770. force end of result set when this is not the first table to be fetched. - if (ntplsch->traceFlags & CalpontSelectExecutionPlan::TRACE_NO_ROWS2) - if (conn_hdl->tidScanMap.size() >= 1) - return SQL_NOT_FOUND; - - // need another band - status_t status = STATUS_OK; - - if (ntplsch->rowsreturned == ntplsch->getRowCount()) - status = tpl_scan_fetch_getband(conn_hdl, ntplsch, killed); - - return status; -} - -status_t -tpl_scan_close ( sp_cpsm_tplsch_t& ntplsch ) -{ -#if IDB_SM_DEBUG - SMDEBUGLOG << "tpl_scan_close: "; - - if (ntplsch) - SMDEBUGLOG << "tpl_scan_close: ntplsch " << ntplsch; - SMDEBUGLOG << "tpl_scan_close: tableid: " << ntplsch->tableid << endl; -#endif - ntplsch.reset(); - - return STATUS_OK; -} - -status_t -tpl_close ( cpsm_tplh_t* ntplh, - cpsm_conhdl_t** conn_hdl, - QueryStats& stats, - bool ask_4_stats, - bool clear_scan_ctx) -{ - cpsm_conhdl_t* hndl = *conn_hdl; - SMDEBUGLOG << "tpl_close: hndl" << hndl << " ntplh " << ntplh; - - if (ntplh) - SMDEBUGLOG << " tableid: " << ntplh->tableid; - - SMDEBUGLOG << endl; - delete ntplh; - - // determine end of result set and end of statement execution - if (hndl->queryState == QUERY_IN_PROCESS) - { - // Get the query stats - ByteStream bs; - // Ask for a stats only if a user explicitly asks - if(ask_4_stats) + catch (IDBExcept&) { - ByteStream::quadbyte qb = 3; - bs << qb; - hndl->write(bs); + // @bug4732 + end_query(hndl); + throw; } - // MCOL-1601 Dispose of unused empty RowGroup - if (clear_scan_ctx) + catch (...) { - SMDEBUGLOG << "tpl_close() clear_scan_ctx read" << std::endl; - bs = hndl->exeMgr->read(); + // querystats messed up. close connection. + // no need to throw for querystats protocol error, like for tablemode. + SMDEBUGLOG << "tpl_close() exception whilst getting stats" << endl; + end_query(hndl); + sm_cleanup(hndl); + *conn_hdl = 0; + return STATUS_OK; + // throw runtime_error(string("tbl_close catch exception: ") + e.what()); } + } + } // stats - SMDEBUGLOG << "tpl_close hndl->exeMgr: " << hndl->exeMgr << endl; - //keep reading until we get a string - // Ask for a stats only if a user explicitly asks - if(ask_4_stats) - { - for (int tries = 0; tries < 10; tries++) - { - bs = hndl->exeMgr->read(); + end_query(hndl); + } - if (bs.length() == 0) break; - - try - { - bs >> hndl->queryStats; - bs >> hndl->extendedStats; - bs >> hndl->miniStats; - stats.unserialize(bs); - stats.setEndTime(); - stats.insert(); - break; - } - catch (IDBExcept&) - { - // @bug4732 - end_query(hndl); - throw; - } - catch (...) - { - // querystats messed up. close connection. - // no need to throw for querystats protocol error, like for tablemode. - SMDEBUGLOG << "tpl_close() exception whilst getting stats" << endl; - end_query(hndl); - sm_cleanup(hndl); - *conn_hdl = 0; - return STATUS_OK; - //throw runtime_error(string("tbl_close catch exception: ") + e.what()); - } - } - } //stats - - end_query(hndl); - } - - return STATUS_OK; + return STATUS_OK; } -status_t -sm_init ( uint32_t sid, - cpsm_conhdl_t** conn_hdl, - uint32_t columnstore_local_query) +status_t sm_init(uint32_t sid, cpsm_conhdl_t** conn_hdl, uint32_t columnstore_local_query) { - // clear file content + // clear file content #if IDB_SM_DEBUG - //smlog.close(); - //smlog.open("/tmp/sm.log"); - SMDEBUGLOG << "sm_init: " << endl; + // smlog.close(); + // smlog.open("/tmp/sm.log"); + SMDEBUGLOG << "sm_init: " << endl; #endif - // @bug5660 Connection changes related to the local pm setting - /** - * when local PM is detected, or columnstore_local_query is set: - * 1. SELECT query connect to local ExeMgr 127.0.0.1:8601; - * 2. DML/DDL is disallowed. - * once local connection is determined, no need to check - * again because it will not switch back. - **/ - if (*conn_hdl) - { - // existing connection is local, ok. - if ((*conn_hdl)->exeMgr->localQuery() || ! columnstore_local_query) - return STATUS_OK; - // if session variable changes to local, re-establish the connection to loopback. - else - sm_cleanup(*conn_hdl); - } + // @bug5660 Connection changes related to the local pm setting + /** + * when local PM is detected, or columnstore_local_query is set: + * 1. SELECT query connect to local ExeMgr 127.0.0.1:8601; + * 2. DML/DDL is disallowed. + * once local connection is determined, no need to check + * again because it will not switch back. + **/ + if (*conn_hdl) + { + // existing connection is local, ok. + if ((*conn_hdl)->exeMgr->localQuery() || !columnstore_local_query) + return STATUS_OK; + // if session variable changes to local, re-establish the connection to loopback. + else + sm_cleanup(*conn_hdl); + } - cpsm_conhdl_t* hndl = new cpsm_conhdl_t(time(0), sid, columnstore_local_query); - *conn_hdl = hndl; - hndl->sessionID = sid; + cpsm_conhdl_t* hndl = new cpsm_conhdl_t(time(0), sid, columnstore_local_query); + *conn_hdl = hndl; + hndl->sessionID = sid; - // profiling statistics - GET_PF_TIME(hndl->pf.login); + // profiling statistics + GET_PF_TIME(hndl->pf.login); - return STATUS_OK; + return STATUS_OK; } -status_t -sm_cleanup ( cpsm_conhdl_t* conn_hdl ) +status_t sm_cleanup(cpsm_conhdl_t* conn_hdl) { #if IDB_SM_DEBUG - SMDEBUGLOG << "sm_cleanup: " << conn_hdl << endl; + SMDEBUGLOG << "sm_cleanup: " << conn_hdl << endl; #endif - delete conn_hdl; + delete conn_hdl; - return STATUS_OK; + return STATUS_OK; } void cpsm_conhdl_t::write(ByteStream bs) { #ifdef _MSC_VER - exeMgr->write(bs); + exeMgr->write(bs); #else - sighandler_t old_handler = signal(SIGPIPE, sighandler); - sigFlag = false; - exeMgr->write(bs); - signal(SIGPIPE, old_handler); + sighandler_t old_handler = signal(SIGPIPE, sighandler); + sigFlag = false; + exeMgr->write(bs); + signal(SIGPIPE, old_handler); - if (sigFlag) - throw runtime_error("Broken Pipe Error"); + if (sigFlag) + throw runtime_error("Broken Pipe Error"); #endif } -} +} // namespace sm /** * The following functions exist because QueryStats is used by sm. @@ -533,59 +518,57 @@ void cpsm_conhdl_t::write(ByteStream bs) * these functions aren't needed. */ -unsigned long mysql_real_escape_string(MYSQL* mysql, - char* to, const char* from, - unsigned long length) +unsigned long mysql_real_escape_string(MYSQL* mysql, char* to, const char* from, unsigned long length) { - my_bool overflow; - return escape_string_for_mysql(mysql->charset, to, length * 2 + 1, from, length, &overflow); + my_bool overflow; + return escape_string_for_mysql(mysql->charset, to, length * 2 + 1, from, length, &overflow); } // Clone of sql-common/client.c cli_use_result MYSQL_RES* mysql_use_result(MYSQL* mysql) { - MYSQL_RES* result; - DBUG_ENTER("mysql_use_result (clone)"); + MYSQL_RES* result; + DBUG_ENTER("mysql_use_result (clone)"); - if (!mysql->fields) - DBUG_RETURN(0); + if (!mysql->fields) + DBUG_RETURN(0); - if (mysql->status != MYSQL_STATUS_GET_RESULT) - { - set_mysql_error(mysql, CR_COMMANDS_OUT_OF_SYNC, unknown_sqlstate); - DBUG_RETURN(0); - } + if (mysql->status != MYSQL_STATUS_GET_RESULT) + { + set_mysql_error(mysql, CR_COMMANDS_OUT_OF_SYNC, unknown_sqlstate); + DBUG_RETURN(0); + } - if (!(result = (MYSQL_RES*) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(*result) + - sizeof(ulong) * mysql->field_count, - MYF(MY_WME | MY_ZEROFILL)))) - DBUG_RETURN(0); + if (!(result = + (MYSQL_RES*)my_malloc(PSI_NOT_INSTRUMENTED, sizeof(*result) + sizeof(ulong) * mysql->field_count, + MYF(MY_WME | MY_ZEROFILL)))) + DBUG_RETURN(0); - result->lengths = (ulong*) (result + 1); - result->methods = mysql->methods; + result->lengths = (ulong*)(result + 1); + result->methods = mysql->methods; - if (!(result->row = (MYSQL_ROW) - my_malloc(PSI_NOT_INSTRUMENTED, sizeof(result->row[0]) * (mysql->field_count + 1), MYF(MY_WME)))) - { - /* Ptrs: to one row */ - my_free(result); - DBUG_RETURN(0); - } + if (!(result->row = (MYSQL_ROW)my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(result->row[0]) * (mysql->field_count + 1), MYF(MY_WME)))) + { + /* Ptrs: to one row */ + my_free(result); + DBUG_RETURN(0); + } - result->fields = mysql->fields; - result->field_alloc = mysql->field_alloc; - result->field_count = mysql->field_count; - result->current_field = 0; - result->handle = mysql; - result->current_row = 0; - mysql->fields = 0; /* fields is now in result */ - clear_alloc_root(&mysql->field_alloc); - mysql->status = MYSQL_STATUS_USE_RESULT; - mysql->unbuffered_fetch_owner = &result->unbuffered_fetch_cancelled; - DBUG_RETURN(result); /* Data is read to be fetched */ + result->fields = mysql->fields; + result->field_alloc = mysql->field_alloc; + result->field_count = mysql->field_count; + result->current_field = 0; + result->handle = mysql; + result->current_row = 0; + mysql->fields = 0; /* fields is now in result */ + clear_alloc_root(&mysql->field_alloc); + mysql->status = MYSQL_STATUS_USE_RESULT; + mysql->unbuffered_fetch_owner = &result->unbuffered_fetch_cancelled; + DBUG_RETURN(result); /* Data is read to be fetched */ } MYSQL_FIELD* mysql_fetch_fields(MYSQL_RES* res) { - return res->fields; + return res->fields; } diff --git a/dbcon/mysql/sm.h b/dbcon/mysql/sm.h index 43da05ee1..6796f8f66 100644 --- a/dbcon/mysql/sm.h +++ b/dbcon/mysql/sm.h @@ -48,9 +48,14 @@ #if IDB_SM_PROFILE #include -#define GET_PF_TIME(n) {gettimeofday(&n, NULL);} +#define GET_PF_TIME(n) \ + { \ + gettimeofday(&n, NULL); \ + } #else -#define GET_PF_TIME(n) {} +#define GET_PF_TIME(n) \ + { \ + } #endif namespace sm @@ -61,10 +66,12 @@ const int SQL_KILLED = -1001; const int CALPONT_INTERNAL_ERROR = -1007; //#if IDB_SM_DEBUG -//extern std::ofstream smlog; +// extern std::ofstream smlog; //#define SMDEBUGLOG smlog //#else -#define SMDEBUGLOG if (false) std::cout +#define SMDEBUGLOG \ + if (false) \ + std::cout //#endif extern const std::string DEFAULT_SAVE_PATH; @@ -73,206 +80,219 @@ typedef int32_t status_t; enum QueryState { - NO_QUERY = 0, - QUERY_IN_PROCESS + NO_QUERY = 0, + QUERY_IN_PROCESS }; typedef struct Column { - Column(): tableID(-1) {} - ~Column() {} - int tableID; - int colPos; - int dataType; - std::vector data; + Column() : tableID(-1) + { + } + ~Column() + { + } + int tableID; + int colPos; + int dataType; + std::vector data; } Column; -typedef std::map ResultMap; +typedef std::map ResultMap; struct Profiler { - struct timeval login; - struct timeval beforePlan; - struct timeval afterPlan; - struct timeval resultArrival; - struct timeval resultReady; - struct timeval endProcess; - long prePlan() - { - return (beforePlan.tv_sec - login.tv_sec) * 1000 + - (beforePlan.tv_usec - login.tv_usec) / 1000; - } - long buildPlan() - { - return (afterPlan.tv_sec - beforePlan.tv_sec) * 1000 + - (afterPlan.tv_usec - beforePlan.tv_usec) / 1000; - } - long jobProcess() - { - return (resultArrival.tv_sec - afterPlan.tv_sec) * 1000 + - (resultArrival.tv_usec - afterPlan.tv_usec) / 1000; - } - long buildResult() - { - return (resultReady.tv_sec - resultArrival.tv_sec) * 1000 + - (resultReady.tv_usec - resultArrival.tv_usec) / 1000; - } - long tableFetch () - { - return (endProcess.tv_sec - resultReady.tv_sec) * 1000 + - (endProcess.tv_usec - resultReady.tv_usec) / 1000; - } + struct timeval login; + struct timeval beforePlan; + struct timeval afterPlan; + struct timeval resultArrival; + struct timeval resultReady; + struct timeval endProcess; + long prePlan() + { + return (beforePlan.tv_sec - login.tv_sec) * 1000 + (beforePlan.tv_usec - login.tv_usec) / 1000; + } + long buildPlan() + { + return (afterPlan.tv_sec - beforePlan.tv_sec) * 1000 + (afterPlan.tv_usec - beforePlan.tv_usec) / 1000; + } + long jobProcess() + { + return (resultArrival.tv_sec - afterPlan.tv_sec) * 1000 + + (resultArrival.tv_usec - afterPlan.tv_usec) / 1000; + } + long buildResult() + { + return (resultReady.tv_sec - resultArrival.tv_sec) * 1000 + + (resultReady.tv_usec - resultArrival.tv_usec) / 1000; + } + long tableFetch() + { + return (endProcess.tv_sec - resultReady.tv_sec) * 1000 + + (endProcess.tv_usec - resultReady.tv_usec) / 1000; + } }; /** @brief Calpont table scan handle */ struct cpsm_tplsch_t { - cpsm_tplsch_t() : tableid(0), rowsreturned(0), rowGroup(0), traceFlags(0), bandID(0), saveFlag(0), bandsReturned(0), - ctp(0) {} - ~cpsm_tplsch_t() - { - delete rowGroup; - } + cpsm_tplsch_t() + : tableid(0), rowsreturned(0), rowGroup(0), traceFlags(0), bandID(0), saveFlag(0), bandsReturned(0), ctp(0) + { + } + ~cpsm_tplsch_t() + { + delete rowGroup; + } - tableid_t tableid; - uint64_t rowsreturned; - rowgroup::RowGroup* rowGroup; - messageqcpp::ByteStream bs; // rowgroup bytestream. need to stay with the life span of rowgroup - uint32_t traceFlags; - // @bug 649 - int bandID; // the band that being read from the disk - int key; // unique key for the table's scan context - // @bug 626 - uint16_t saveFlag; - uint32_t bandsReturned; - std::vector ctp; - std::string errMsg; - rowgroup::RGData rgData; - void deserializeTable(messageqcpp::ByteStream& bs) + tableid_t tableid; + uint64_t rowsreturned; + rowgroup::RowGroup* rowGroup; + messageqcpp::ByteStream bs; // rowgroup bytestream. need to stay with the life span of rowgroup + uint32_t traceFlags; + // @bug 649 + int bandID; // the band that being read from the disk + int key; // unique key for the table's scan context + // @bug 626 + uint16_t saveFlag; + uint32_t bandsReturned; + std::vector ctp; + std::string errMsg; + rowgroup::RGData rgData; + void deserializeTable(messageqcpp::ByteStream& bs) + { + if (!rowGroup) { - if (!rowGroup) - { - rowGroup = new rowgroup::RowGroup(); - rowGroup->deserialize(bs); - } - else - { - // XXXST: the 'true' is to ease the transition to RGDatas. Take it out when the - // transition is done. - rgData.deserialize(bs, true); - rowGroup->setData(&rgData); - //rowGroup->setData(const_cast(bs.buf())); - } + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(bs); } + else + { + // XXXST: the 'true' is to ease the transition to RGDatas. Take it out when the + // transition is done. + rgData.deserialize(bs, true); + rowGroup->setData(&rgData); + // rowGroup->setData(const_cast(bs.buf())); + } + } - uint16_t getStatus() - { - idbassert(rowGroup != 0); - return rowGroup->getStatus(); - } + uint16_t getStatus() + { + idbassert(rowGroup != 0); + return rowGroup->getStatus(); + } - uint64_t getRowCount() - { - if (rowGroup) - return rowGroup->getRowCount(); - else - return 0; - } + uint64_t getRowCount() + { + if (rowGroup) + return rowGroup->getRowCount(); + else + return 0; + } - void setErrMsg() + void setErrMsg() + { + if (rowGroup && getStatus()) { - if (rowGroup && getStatus()) - { - //bs.advance(rowGroup->getDataSize()); - bs >> errMsg; - } - else - { - errMsg = "NOERROR"; - } + // bs.advance(rowGroup->getDataSize()); + bs >> errMsg; } + else + { + errMsg = "NOERROR"; + } + } }; typedef boost::shared_ptr sp_cpsm_tplsch_t; /** @brief Calpont connection handle structure */ class cpsm_conhdl_t { -public: - cpsm_conhdl_t(time_t v, const uint32_t sid, bool columnstore_local_query) : - value(v), sessionID(sid), queryState (NO_QUERY), - exeMgr( new execplan::ClientRotator(sid, "ExeMgr", columnstore_local_query)), - tblinfo_idx(0), idxinfo_idx(0), curFetchTb (0) - { } + public: + cpsm_conhdl_t(time_t v, const uint32_t sid, bool columnstore_local_query) + : value(v) + , sessionID(sid) + , queryState(NO_QUERY) + , exeMgr(new execplan::ClientRotator(sid, "ExeMgr", columnstore_local_query)) + , tblinfo_idx(0) + , idxinfo_idx(0) + , curFetchTb(0) + { + } + /** @brief connnect ExeMgr + * + * Try connecting to ExeMgr. If no connection, try ExeMgr1, + * ExeMgr2... until timeout lapses. Then throw exception. + */ + void connect(double timeout = 0.005) + { + exeMgr->connect(timeout); + } + EXPORT void write(messageqcpp::ByteStream bs); - /** @brief connnect ExeMgr - * - * Try connecting to ExeMgr. If no connection, try ExeMgr1, - * ExeMgr2... until timeout lapses. Then throw exception. - */ - void connect(double timeout = 0.005) - { - exeMgr->connect(timeout); - } - EXPORT void write(messageqcpp::ByteStream bs); + ~cpsm_conhdl_t() + { + delete exeMgr; + } + EXPORT const std::string toString() const; + time_t value; + uint32_t sessionID; + short queryState; // 0 -- NO_QUERY; 1 -- QUERY_IN_PROCESS + execplan::ClientRotator* exeMgr; + ResultMap resultSet; + Profiler pf; + int tblinfo_idx; + int idxinfo_idx; + std::string schemaname; + std::string tablename; + int tboid; + short requestType; // 0 -- ID2NAME; 1 -- NAME2ID + boost::shared_ptr csc; + // @bug 649; @bug 626 + std::map tidMap; // tableid-tableStartCount map + std::map tidScanMap; + std::map keyBandMap; // key-savedBandCount map + int curFetchTb; // current fetching table key + std::string queryStats; + std::string extendedStats; + std::string miniStats; - ~cpsm_conhdl_t() - { - delete exeMgr; - } - EXPORT const std::string toString() const; - time_t value; - uint32_t sessionID; - short queryState; // 0 -- NO_QUERY; 1 -- QUERY_IN_PROCESS - execplan::ClientRotator* exeMgr; - ResultMap resultSet; - Profiler pf; - int tblinfo_idx; - int idxinfo_idx; - std::string schemaname; - std::string tablename; - int tboid; - short requestType; // 0 -- ID2NAME; 1 -- NAME2ID - boost::shared_ptr csc; - // @bug 649; @bug 626 - std::map tidMap; // tableid-tableStartCount map - std::map tidScanMap; - std::map keyBandMap; // key-savedBandCount map - int curFetchTb; // current fetching table key - std::string queryStats; - std::string extendedStats; - std::string miniStats; -private: + private: }; std::ostream& operator<<(std::ostream& output, const cpsm_conhdl_t& rhs); // @bug 626 save table bands to avoid sending plan too many times enum SavingFlag { - NO_SAVE = 0, - SAVING, - SAVED + NO_SAVE = 0, + SAVING, + SAVED }; /** @brief Calpont table handle */ struct cpsm_tplh_t { - cpsm_tplh_t() : tableid(0), rowsintable(0), bandID(0), saveFlag(NO_SAVE), bandsInTable(0) {} - tableid_t tableid; - int rowsintable; - // @bug 649 - int bandID; // the band that being read from the disk - int key; // unique key for the table's scan context - // @bug 626 - uint16_t saveFlag; - int bandsInTable; + cpsm_tplh_t() : tableid(0), rowsintable(0), bandID(0), saveFlag(NO_SAVE), bandsInTable(0) + { + } + tableid_t tableid; + int rowsintable; + // @bug 649 + int bandID; // the band that being read from the disk + int key; // unique key for the table's scan context + // @bug 626 + uint16_t saveFlag; + int bandsInTable; }; struct cpsm_tid_t { - cpsm_tid_t() : valid(false), value(0) {} - bool valid; - int value; + cpsm_tid_t() : valid(false), value(0) + { + } + bool valid; + int value; }; extern status_t sm_init(uint32_t, cpsm_conhdl_t**, uint32_t columnstore_local_query = false); @@ -282,11 +302,11 @@ extern status_t tpl_open(tableid_t, cpsm_tplh_t*, cpsm_conhdl_t*); extern status_t tpl_scan_open(tableid_t, sp_cpsm_tplsch_t&, cpsm_conhdl_t*); extern status_t tpl_scan_fetch(sp_cpsm_tplsch_t&, cpsm_conhdl_t*, int* k = 0); extern status_t tpl_scan_close(sp_cpsm_tplsch_t&); -extern status_t tpl_close(cpsm_tplh_t*, cpsm_conhdl_t**, querystats::QueryStats& stats, bool ask_4_stats, bool clear_scan_ctx = false); +extern status_t tpl_close(cpsm_tplh_t*, cpsm_conhdl_t**, querystats::QueryStats& stats, bool ask_4_stats, + bool clear_scan_ctx = false); -} +} // namespace sm #undef EXPORT #endif - diff --git a/dbcon/mysql/tdriver1.cpp b/dbcon/mysql/tdriver1.cpp index d1285b490..fa256904e 100644 --- a/dbcon/mysql/tdriver1.cpp +++ b/dbcon/mysql/tdriver1.cpp @@ -29,85 +29,84 @@ typedef int64_t longlong; string toString(int64_t t1, unsigned scale) { - longlong int_val = (longlong)t1; - // MySQL seems to round off values unless we use the string store method. Groan. - // Taken from tablefuncs.cpp + longlong int_val = (longlong)t1; + // MySQL seems to round off values unless we use the string store method. Groan. + // Taken from tablefuncs.cpp - //biggest Calpont supports is DECIMAL(18,x), or 18 total digits+dp+sign - const int ctmp_size = 18 + 1 + 1 + 1; - char ctmp[ctmp_size]; - snprintf(ctmp, ctmp_size, + // biggest Calpont supports is DECIMAL(18,x), or 18 total digits+dp+sign + const int ctmp_size = 18 + 1 + 1 + 1; + char ctmp[ctmp_size]; + snprintf(ctmp, ctmp_size, #if __WORDSIZE <= 32 - "%lld", + "%lld", #else - "%ld", + "%ld", #endif - int_val); - //we want to move the last dt_scale chars right by one spot to insert the dp - //we want to move the trailing null as well, so it's really dt_scale+1 chars - size_t l1 = strlen(ctmp); - //need to make sure we have enough leading zeros for this to work... - //at this point scale is always > 0 - char* ptr = &ctmp[0]; + int_val); + // we want to move the last dt_scale chars right by one spot to insert the dp + // we want to move the trailing null as well, so it's really dt_scale+1 chars + size_t l1 = strlen(ctmp); + // need to make sure we have enough leading zeros for this to work... + // at this point scale is always > 0 + char* ptr = &ctmp[0]; - if (int_val < 0) - { - ptr++; - idbassert(l1 >= 2); - l1--; - } + if (int_val < 0) + { + ptr++; + idbassert(l1 >= 2); + l1--; + } - if (scale > l1) - { - const char* zeros = "000000000000000000"; //18 0's - size_t diff = scale - l1; //this will always be > 0 - memmove((ptr + diff), ptr, l1 + 1); //also move null - memcpy(ptr, zeros, diff); - l1 = 0; - } - else - l1 -= scale; + if (scale > l1) + { + const char* zeros = "000000000000000000"; // 18 0's + size_t diff = scale - l1; // this will always be > 0 + memmove((ptr + diff), ptr, l1 + 1); // also move null + memcpy(ptr, zeros, diff); + l1 = 0; + } + else + l1 -= scale; - memmove((ptr + l1 + 1), (ptr + l1), scale + 1); //also move null - *(ptr + l1) = '.'; + memmove((ptr + l1 + 1), (ptr + l1), scale + 1); // also move null + *(ptr + l1) = '.'; - return string(ctmp); -} + return string(ctmp); } +} // namespace int main(int argc, char** argv) { - int64_t x; - string xstr; + int64_t x; + string xstr; - x = 10001LL; - xstr = toString(x, 2); - idbassert(xstr == "100.01"); + x = 10001LL; + xstr = toString(x, 2); + idbassert(xstr == "100.01"); - x = -10001LL; - xstr = toString(x, 2); - idbassert(xstr == "-100.01"); + x = -10001LL; + xstr = toString(x, 2); + idbassert(xstr == "-100.01"); - x = 999999999999999999LL; - xstr = toString(x, 2); - idbassert(xstr == "9999999999999999.99"); + x = 999999999999999999LL; + xstr = toString(x, 2); + idbassert(xstr == "9999999999999999.99"); - x = -999999999999999999LL; - xstr = toString(x, 2); - idbassert(xstr == "-9999999999999999.99"); + x = -999999999999999999LL; + xstr = toString(x, 2); + idbassert(xstr == "-9999999999999999.99"); - x = 1LL; - xstr = toString(x, 5); - idbassert(xstr == ".00001"); + x = 1LL; + xstr = toString(x, 5); + idbassert(xstr == ".00001"); - x = -1LL; - xstr = toString(x, 5); - idbassert(xstr == "-.00001"); + x = -1LL; + xstr = toString(x, 5); + idbassert(xstr == "-.00001"); - x = -1LL; - xstr = toString(x, 16); - idbassert(xstr == "-.0000000000000001"); + x = -1LL; + xstr = toString(x, 16); + idbassert(xstr == "-.0000000000000001"); - return 0; + return 0; } - diff --git a/dbcon/mysql/versionnumber.h b/dbcon/mysql/versionnumber.h index edb73a534..d452c750c 100644 --- a/dbcon/mysql/versionnumber.h +++ b/dbcon/mysql/versionnumber.h @@ -5,4 +5,3 @@ const std::string idb_version("1.0.2"); const std::string idb_release("1"); #endif - diff --git a/ddlproc/ddlproc.cpp b/ddlproc/ddlproc.cpp index 86594ce73..e479fbf49 100644 --- a/ddlproc/ddlproc.cpp +++ b/ddlproc/ddlproc.cpp @@ -68,197 +68,184 @@ using namespace execplan; namespace { - class Opt { -public: - int m_debug; - bool m_fg; - Opt(int argc, char *argv[]) - :m_debug(0), - m_fg(false) + public: + int m_debug; + bool m_fg; + Opt(int argc, char* argv[]) : m_debug(0), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "df")) != EOF) { - int c; - while ((c = getopt(argc, argv, "df")) != EOF) - { - switch(c) - { - case 'd': - m_debug++; // TODO: not really used yes - break; - case 'f': - m_fg= true; - break; - case '?': - default: - break; - } - } + switch (c) + { + case 'd': + m_debug++; // TODO: not really used yes + break; + case 'f': m_fg = true; break; + case '?': + default: break; + } + } } }; - -class ServiceDDLProc: public Service, public Opt +class ServiceDDLProc : public Service, public Opt { -protected: - void setupChildSignalHandlers(); + protected: + void setupChildSignalHandlers(); - void log(logging::LOG_TYPE type, const std::string &str) - { - LoggingID logid(23, 0, 0); - Message::Args args; - Message message(8); - args.add(str); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, message, logid); - } + void log(logging::LOG_TYPE type, const std::string& str) + { + LoggingID logid(23, 0, 0); + Message::Args args; + Message message(8); + args.add(str); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, message, logid); + } -public: - ServiceDDLProc(const Opt &opt) - :Service("DDLProc"), Opt(opt) - { } - void LogErrno() override - { - log(LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string &str) override - { - log(LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServiceDDLProc(const Opt& opt) : Service("DDLProc"), Opt(opt) + { + } + void LogErrno() override + { + log(LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - DistributedEngineComm* Dec; int8_t setupCwd() { - string workdir = startup::StartUp::tmpDir(); + string workdir = startup::StartUp::tmpDir(); - if (workdir.length() == 0) - workdir = "."; + if (workdir.length() == 0) + workdir = "."; - int8_t rc = chdir(workdir.c_str()); - return rc; + int8_t rc = chdir(workdir.c_str()); + return rc; } void added_a_pm(int) { - LoggingID logid(23, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("DDLProc caught SIGHUP. Resetting connections"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - Dec->Setup(); + LoggingID logid(23, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("DDLProc caught SIGHUP. Resetting connections"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + Dec->Setup(); } -} - +} // namespace void ServiceDDLProc::setupChildSignalHandlers() { #ifndef _MSC_VER - /* set up some signal handlers */ - struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = added_a_pm; - sigaction(SIGHUP, &ign, 0); - ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); + /* set up some signal handlers */ + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = added_a_pm; + sigaction(SIGHUP, &ign, 0); + ign.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif } - int ServiceDDLProc::Child() { - if ( setupCwd() < 0 ) - { - LoggingID logid(23, 0, 0); - logging::Message::Args args1; - logging::Message msg(9); - args1.add("DDLProc could not set working directory "); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, msg, logid); - NotifyServiceInitializationFailed(); - return 1; - } + if (setupCwd() < 0) + { + LoggingID logid(23, 0, 0); + logging::Message::Args args1; + logging::Message msg(9); + args1.add("DDLProc could not set working directory "); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, msg, logid); + NotifyServiceInitializationFailed(); + return 1; + } - - WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DDLPROC ); + WriteEngine::WriteEngineWrapper::init(WriteEngine::SUBSYSTEM_ID_DDLPROC); #ifdef _MSC_VER - // In windows, initializing the wrapper (A dll) does not set the static variables - // in the main program - idbdatafile::IDBPolicy::configIDBPolicy(); + // In windows, initializing the wrapper (A dll) does not set the static variables + // in the main program + idbdatafile::IDBPolicy::configIDBPolicy(); #endif - ResourceManager* rm = ResourceManager::instance(); - Dec = DistributedEngineComm::instance(rm); + ResourceManager* rm = ResourceManager::instance(); + Dec = DistributedEngineComm::instance(rm); - setupChildSignalHandlers(); + setupChildSignalHandlers(); - ddlprocessor::DDLProcessor ddlprocessor(1, 20); + ddlprocessor::DDLProcessor ddlprocessor(1, 20); - NotifyServiceStarted(); + NotifyServiceStarted(); - try - { - ddlprocessor.process(); - } - catch (std::exception& ex) - { - cerr << ex.what() << endl; - LoggingID logid(23, 0, 0); - Message::Args args; - Message message(8); - args.add("DDLProc failed on: "); - args.add(ex.what()); - message.format( args ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, message, logid); - return 1; - } - catch (...) - { - cerr << "Caught unknown exception!" << endl; - LoggingID logid(23, 0, 0); - Message::Args args; - Message message(8); - args.add("DDLProc failed on: "); - args.add("receiving DDLPackage (unknown exception)"); - message.format( args ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, message, logid); - return 1; - } + try + { + ddlprocessor.process(); + } + catch (std::exception& ex) + { + cerr << ex.what() << endl; + LoggingID logid(23, 0, 0); + Message::Args args; + Message message(8); + args.add("DDLProc failed on: "); + args.add(ex.what()); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, message, logid); + return 1; + } + catch (...) + { + cerr << "Caught unknown exception!" << endl; + LoggingID logid(23, 0, 0); + Message::Args args; + Message message(8); + args.add("DDLProc failed on: "); + args.add("receiving DDLPackage (unknown exception)"); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, message, logid); + return 1; + } - return 0; + return 0; } - int main(int argc, char** argv) { - Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - // This is unset due to the way we start it - program_invocation_short_name = const_cast("DDLProc"); - // Initialize the charset library - MY_INIT(argv[0]); + Opt opt(argc, argv); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DDLProc"); + // Initialize the charset library + MY_INIT(argv[0]); - return ServiceDDLProc(opt).Run(); + return ServiceDDLProc(opt).Run(); } // vim:ts=4 sw=4: - diff --git a/ddlproc/ddlproc.h b/ddlproc/ddlproc.h index 519a95d22..a88b1ad03 100644 --- a/ddlproc/ddlproc.h +++ b/ddlproc/ddlproc.h @@ -44,5 +44,4 @@ #include "messagelog.h" - -#endif //DDLPROC_H +#endif // DDLPROC_H diff --git a/ddlproc/ddlprocessor.cpp b/ddlproc/ddlprocessor.cpp index 1872d4f30..de9f0fc1e 100644 --- a/ddlproc/ddlprocessor.cpp +++ b/ddlproc/ddlprocessor.cpp @@ -66,815 +66,814 @@ const quadbyte UNRECOGNIZED_PACKAGE_TYPE = 100; const std::string DDLProcName = "DDLProc"; - void cleanPMSysCache() { - vector oidList = getAllSysCatOIDs(); - cacheutils::flushOIDsFromCache ( oidList ); + vector oidList = getAllSysCatOIDs(); + cacheutils::flushOIDsFromCache(oidList); } class PackageHandler { -public: - PackageHandler(QueryTeleClient qtc, DBRM* dbrm, messageqcpp::IOSocket& ios, bool concurrentSupport) - : fIos(ios) - , fDbrm(dbrm) - , fQtc(qtc) - , fConcurrentSupport(concurrentSupport) - {} - - void operator()() - { - try - { - fByteStream = fIos.read(); - if (fByteStream.empty()) - { - logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "Empty package", true); - return; - } - fByteStream >> fSessionID; - fByteStream >> fPackageType; - - uint32_t stateFlags; - if (fDbrm->getSystemState(stateFlags) > 0) // > 0 implies successful retrieval. It doesn't imply anything about the contents - { - messageqcpp::ByteStream results; - const char* responseMsg = nullptr; - messageqcpp::ByteStream::byte status; - bool bReject = false; - - // Check to see if we're in write suspended or shutdown mode - // If so, we can't process the request. - if (stateFlags & SessionManagerServer::SS_SUSPENDED || - stateFlags & SessionManagerServer::SS_SUSPEND_PENDING || - stateFlags & SessionManagerServer::SS_SHUTDOWN_PENDING) - { - if (stateFlags & SessionManagerServer::SS_SUSPENDED || - stateFlags & SessionManagerServer::SS_SUSPEND_PENDING) - { - responseMsg = "Writing to the database is disabled."; - } - else - { - responseMsg = "The database is being shut down."; - } - - status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; - bReject = true; - } - - if (bReject) - { - results << status; - //@bug 266 - MessageLog logger(LoggingID(27)); - logging::Message::Args args; - logging::Message message(2); - args.add(responseMsg); - message.format( args ); - results << message.msg(); - fIos.write(results); - std::cout << responseMsg << endl; - std::cout << "Command rejected. Status " << (int)status << message.msg() << endl; - return; - } - } - - //check whether the system is ready to process statement. - if (fDbrm->getSystemReady() < 1) - { - logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "System is not ready yet. Please try again.", true); - return; - } - - int rc = 0; - - if (!fConcurrentSupport) - { - //Check if any other active transaction - bool bIsDbrmUp = true; - bool anyOtherActiveTransaction = true; - execplan::SessionManager sessionManager; - BRM::SIDTIDEntry blockingsid; - int i = 0; - int waitPeriod = 10; - //@Bug 2487 Check transaction map every 1/10 second - - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - - string waitPeriodStr = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); - - if ( waitPeriodStr.length() != 0 ) - waitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); - - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - //cout << "starting i = " << i << endl; - //anyOtherActiveTransaction = sessionManager.checkActiveTransaction(fSessionID, bIsDbrmUp); - while (anyOtherActiveTransaction) - { - anyOtherActiveTransaction = sessionManager.checkActiveTransaction(fSessionID, bIsDbrmUp, - blockingsid); - - if (anyOtherActiveTransaction) - { - for ( ; i < numTries; i++ ) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - //cout << "session " << fSessionID << " nanosleep on package type " << (int)packageType << endl; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - anyOtherActiveTransaction = sessionManager.checkActiveTransaction(fSessionID, bIsDbrmUp, - blockingsid); - - if (!anyOtherActiveTransaction) - { - //cout << "Ready to process type " << (int)packageType << endl; - fTxnid = sessionManager.getTxnID(fSessionID); - - if (!fTxnid.valid) - { - fTxnid = sessionManager.newTxnID(fSessionID, true, true); - - if (fTxnid.valid) - { - //cout << "Ready to process type " << (int)packageType << " with txnid " << txnid.id << endl; - anyOtherActiveTransaction = false; - break; - } - else - { - anyOtherActiveTransaction = true; - } - } - else - { - string errorMsg; - rc = commitTransaction(fTxnid.id, errorMsg); - - if ( rc != 0) - throw std::runtime_error(errorMsg); - - //need unlock the table. - std::vector tableLocks = fDbrm->getAllTableLocks(); - bool lockReleased = true; - - for (unsigned k = 0; k < tableLocks.size(); k++) - { - if (tableLocks[k].ownerTxnID == fTxnid.id) - { - lockReleased = fDbrm->releaseTableLock(tableLocks[k].id); - - if (!lockReleased) - { - ostringstream os; - os << "tablelock id " << tableLocks[k].id << " is not found"; - throw std::runtime_error(os.str()); - } - } - } - - fDbrm->committed(fTxnid); - fTxnid = fDbrm->newTxnID(fSessionID, true, true); - - if (fTxnid.valid) - { - //cout << "Ready to process type " << (int)packageType << " with txnid " << txnid.id << endl; - anyOtherActiveTransaction = false; - break; - } - else - { - anyOtherActiveTransaction = true; - } - } - } - } - - //cout << "ending i = " << i << endl; - } - else - { - //cout << "Ready to process type " << (int)packageType << endl; - fTxnid = sessionManager.getTxnID(fSessionID); - - if ( !fTxnid.valid ) - { - fTxnid = sessionManager.newTxnID(fSessionID, true, true); - - if (!fTxnid.valid) - { - //cout << "cannot get txnid " << (int)packageType << " for session " << sessionID << endl; - anyOtherActiveTransaction = true; - } - else - { - anyOtherActiveTransaction = false; - } - } - else - { - string errorMsg; - rc = commitTransaction(fTxnid.id, errorMsg); - - if ( rc != 0) - throw std::runtime_error(errorMsg); - - //need unlock the table. - std::vector tableLocks = fDbrm->getAllTableLocks(); - bool lockReleased = true; - - for (unsigned k = 0; k < tableLocks.size(); k++) - { - if (tableLocks[k].ownerTxnID == fTxnid.id) - { - lockReleased = fDbrm->releaseTableLock(tableLocks[k].id); - - if (!lockReleased) - { - ostringstream os; - os << "tablelock id " << tableLocks[k].id << " is not found"; - throw std::runtime_error(os.str()); - } - } - } - - sessionManager.committed(fTxnid); - fTxnid = sessionManager.newTxnID(fSessionID, true, true); - - if (!fTxnid.valid) - { - //cout << "cannot get txnid " << (int)packageType << " for session " << sessionID << endl; - anyOtherActiveTransaction = true; - } - else - { - anyOtherActiveTransaction = false; - } - } - } - - if ((anyOtherActiveTransaction) && (i >= numTries)) - { - //cout << " Erroring out on package type " << (int)packageType << endl; - break; - } - } - - if ((anyOtherActiveTransaction) && (i >= numTries)) - { - messageqcpp::ByteStream::byte status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; - - Message::Args args; - args.add(static_cast(blockingsid.sessionid)); - - //@Bug 3854 Log to debug.log - LoggingID logid(15, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add(IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args)); - msg.format(args1); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - - logError(status, IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args)); - } - else - { - processStatement(); - } - } - else - { - fTxnid = fDbrm->getTxnID(fSessionID); - - if ( !fTxnid.valid ) - { - fTxnid = fDbrm->newTxnID(fSessionID, true, true); - - if (!fTxnid.valid) - { - throw std::runtime_error( std::string("Unable to start a transaction. Check critical log.") ); - } - } - else - { - string errorMsg; - rc = commitTransaction(fTxnid.id, errorMsg); - - if ( rc != 0) - throw std::runtime_error(errorMsg); - - //need unlock the table. - std::vector tableLocks = fDbrm->getAllTableLocks(); - bool lockReleased = true; - - for (unsigned k = 0; k < tableLocks.size(); k++) - { - if (tableLocks[k].ownerTxnID == fTxnid.id) - { - lockReleased = fDbrm->releaseTableLock(tableLocks[k].id); - - if (!lockReleased) - { - ostringstream os; - os << "tablelock id " << tableLocks[k].id << " is not found"; - throw std::runtime_error(os.str()); - } - } - } - - fDbrm->committed(fTxnid); - fTxnid = fDbrm->newTxnID(fSessionID, true, true); - - if (!fTxnid.valid) - { - throw std::runtime_error( std::string("Unable to start a transaction. Check critical log.") ); - } - } - - processStatement(); - } - } - catch (const std::exception &ex) - { - logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, ex.what(), true); - throw; - } - } - -private: - int commitTransaction(uint32_t txnID, std::string& errorMsg) - { - auto WEClient = std::unique_ptr(new WriteEngine::WEClients(WriteEngine::WEClients::DDLPROC)); - auto PMCount = WEClient->getPmCount(); - ByteStream bytestream; - uint64_t uniqueId = fDbrm->getUnique64(); - WEClient->addQueue(uniqueId); - bytestream << (ByteStream::byte)WE_SVR_COMMIT_VERSION; - bytestream << uniqueId; - bytestream << txnID; - uint32_t msgRecived = 0; - WEClient->write_to_all(bytestream); - boost::shared_ptr bsIn; - bsIn.reset(new ByteStream()); - int rc = 0; - ByteStream::byte tmp8; - - while (true) - { - if (msgRecived == PMCount) - break; - - WEClient->read(uniqueId, bsIn); - - if (bsIn->length() == 0) //read error - { - rc = 1; - errorMsg = "DDL cannot communicate with WES"; - WEClient->removeQueue(uniqueId); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - WEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } - } - - return rc; - } - - void processStatement() - { - DDLPackageProcessor::DDLResult result; - result.result = DDLPackageProcessor::NO_ERROR; - //boost::shared_ptr systemCatalogPtr; - - try - { - if (!fOamCache) - fOamCache = oam::OamCache::makeOamCache(); - //cout << "DDLProc received package " << fPackageType << endl; - switch ( fPackageType ) - { - case ddlpackage::DDL_CREATE_TABLE_STATEMENT: - { - CreateTableStatement createTableStmt; - createTableStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(createTableStmt.fSessionID ); - boost::scoped_ptr processor(new CreateTableProcessor(fDbrm)); - processor->fTxnid.id = fTxnid.id; - processor->fTxnid.valid = true; - //cout << "create table using txnid " << fTxnid.id << endl; - - QueryTeleStats qts; - qts.query_uuid = QueryTeleClient::genUUID(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = createTableStmt.fSessionID; - qts.query_type = "CREATE"; - qts.query = createTableStmt.fSql; - qts.system_name = fOamCache->getSystemName(); - qts.module_name = fOamCache->getModuleName(); - qts.schema_name = createTableStmt.schemaName(); - fQtc.postQueryTele(qts); - - result = processor->processPackage(createTableStmt); - - systemCatalogPtr->removeCalpontSystemCatalog( createTableStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog( createTableStmt.fSessionID | 0x80000000); - - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.end_time = QueryTeleClient::timeNowms(); - fQtc.postQueryTele(qts); - } - break; - - case ddlpackage::DDL_ALTER_TABLE_STATEMENT: - { - AlterTableStatement alterTableStmt; - alterTableStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(alterTableStmt.fSessionID ); - boost::scoped_ptr processor(new AlterTableProcessor(fDbrm)); - processor->fTxnid.id = fTxnid.id; - processor->fTxnid.valid = true; - - QueryTeleStats qts; - qts.query_uuid = QueryTeleClient::genUUID(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = alterTableStmt.fSessionID; - qts.query_type = "ALTER"; - qts.query = alterTableStmt.fSql; - qts.system_name = fOamCache->getSystemName(); - qts.module_name = fOamCache->getModuleName(); - qts.schema_name = alterTableStmt.schemaName(); - fQtc.postQueryTele(qts); - - processor->fTimeZone = alterTableStmt.fTimeZone; - - result = processor->processPackage(alterTableStmt); - - systemCatalogPtr->removeCalpontSystemCatalog( alterTableStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog( alterTableStmt.fSessionID | 0x80000000); - - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.end_time = QueryTeleClient::timeNowms(); - fQtc.postQueryTele(qts); - } - break; - - case ddlpackage::DDL_DROP_TABLE_STATEMENT: - { - DropTableStatement dropTableStmt; - dropTableStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(dropTableStmt.fSessionID ); - boost::scoped_ptr processor(new DropTableProcessor(fDbrm)); - - processor->fTxnid.id = fTxnid.id; - processor->fTxnid.valid = true; - - QueryTeleStats qts; - qts.query_uuid = QueryTeleClient::genUUID(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = dropTableStmt.fSessionID; - qts.query_type = "DROP"; - qts.query = dropTableStmt.fSql; - qts.system_name = fOamCache->getSystemName(); - qts.module_name = fOamCache->getModuleName(); - qts.schema_name = dropTableStmt.schemaName(); - fQtc.postQueryTele(qts); - - //cout << "Drop table using txnid " << fTxnid.id << endl; - result = processor->processPackage(dropTableStmt); - - systemCatalogPtr->removeCalpontSystemCatalog( dropTableStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog( dropTableStmt.fSessionID | 0x80000000); - - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.end_time = QueryTeleClient::timeNowms(); - fQtc.postQueryTele(qts); - } - break; - - case ddlpackage::DDL_TRUNC_TABLE_STATEMENT: - { - TruncTableStatement truncTableStmt; - truncTableStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(truncTableStmt.fSessionID); - boost::scoped_ptr processor(new TruncTableProcessor(fDbrm)); - - processor->fTxnid.id = fTxnid.id; - processor->fTxnid.valid = true; - - QueryTeleStats qts; - qts.query_uuid = QueryTeleClient::genUUID(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = truncTableStmt.fSessionID; - qts.query_type = "TRUNCATE"; - qts.query = truncTableStmt.fSql; - qts.system_name = fOamCache->getSystemName(); - qts.module_name = fOamCache->getModuleName(); - qts.schema_name = truncTableStmt.schemaName(); - fQtc.postQueryTele(qts); - - result = processor->processPackage(truncTableStmt); - - systemCatalogPtr->removeCalpontSystemCatalog(truncTableStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog(truncTableStmt.fSessionID | 0x80000000); - - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.end_time = QueryTeleClient::timeNowms(); - fQtc.postQueryTele(qts); - } - break; - - case ddlpackage::DDL_MARK_PARTITION_STATEMENT: - { - MarkPartitionStatement markPartitionStmt; - markPartitionStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(markPartitionStmt.fSessionID); - boost::scoped_ptr processor(new MarkPartitionProcessor(fDbrm)); - (processor->fTxnid).id = fTxnid.id; - (processor->fTxnid).valid = true; - result = processor->processPackage(markPartitionStmt); - systemCatalogPtr->removeCalpontSystemCatalog(markPartitionStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog(markPartitionStmt.fSessionID | 0x80000000); - } - break; - - case ddlpackage::DDL_RESTORE_PARTITION_STATEMENT: - { - RestorePartitionStatement restorePartitionStmt; - restorePartitionStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(restorePartitionStmt.fSessionID); - boost::scoped_ptr processor(new RestorePartitionProcessor(fDbrm)); - (processor->fTxnid).id = fTxnid.id; - (processor->fTxnid).valid = true; - result = processor->processPackage(restorePartitionStmt); - systemCatalogPtr->removeCalpontSystemCatalog(restorePartitionStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog(restorePartitionStmt.fSessionID | 0x80000000); - } - break; - - case ddlpackage::DDL_DROP_PARTITION_STATEMENT: - { - DropPartitionStatement dropPartitionStmt; - dropPartitionStmt.unserialize(fByteStream); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(dropPartitionStmt.fSessionID); - boost::scoped_ptr processor(new DropPartitionProcessor(fDbrm)); - (processor->fTxnid).id = fTxnid.id; - (processor->fTxnid).valid = true; - result = processor->processPackage(dropPartitionStmt); - systemCatalogPtr->removeCalpontSystemCatalog(dropPartitionStmt.fSessionID ); - systemCatalogPtr->removeCalpontSystemCatalog(dropPartitionStmt.fSessionID | 0x80000000); - } - break; - - default: - throw UNRECOGNIZED_PACKAGE_TYPE; - break; - } - - //@Bug 3427. No need to log user rror, just return the message to user. - //Log errors - if ((result.result != DDLPackageProcessor::NO_ERROR) && (result.result != DDLPackageProcessor::USER_ERROR)) - { - logging::LoggingID lid(23); - logging::MessageLog ml(lid); - - ml.logErrorMessage( result.message ); - } - - string hdfstest = config::Config::makeConfig()->getConfig("Installation", "DBRootStorageType"); - - if (hdfstest == "hdfs" || hdfstest == "HDFS") - cleanPMSysCache(); - - messageqcpp::ByteStream results; - messageqcpp::ByteStream::byte status = result.result; - results << status; - results << result.message.msg(); - - fIos.write(results); - - fIos.close(); - } - catch (quadbyte& /*foo*/) - { - logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "Unrecognized package type", true); - } - catch (logging::IDBExcept& idbEx) - { - cleanPMSysCache(); - logError(DDLPackageProcessor::CREATE_ERROR, idbEx.what(), true); - } - catch (...) - { - logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "Unknown error", true); - } - } - - void logError(messageqcpp::ByteStream::byte status, const std::string& msg, bool closeSocket = false) - { - messageqcpp::ByteStream res; - res << status; - res << msg; - fIos.write(res); - cerr << "DDLProc error: " << msg << endl; - if (closeSocket) - fIos.close(); - } - -private: - messageqcpp::IOSocket fIos; - messageqcpp::ByteStream fByteStream; - messageqcpp::ByteStream::quadbyte fPackageType; - uint32_t fSessionID; - BRM::TxnID fTxnid; - BRM::DBRM* fDbrm; - QueryTeleClient fQtc; - oam::OamCache* fOamCache = nullptr; - bool fConcurrentSupport; -}; - -} - -namespace ddlprocessor -{ - -DDLProcessor::DDLProcessor( int packageMaxThreads, int packageWorkQueueSize ) - : fPackageMaxThreads(packageMaxThreads), fPackageWorkQueueSize(packageWorkQueueSize), - fMqServer(DDLProcName) -{ - fDdlPackagepool.setMaxThreads(fPackageMaxThreads); - fDdlPackagepool.setQueueSize(fPackageWorkQueueSize); - fDdlPackagepool.setName("DdlPackagepool"); - csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); - csc->identity(CalpontSystemCatalog::EC); - string teleServerHost(config::Config::makeConfig()->getConfig("QueryTele", "Host")); - - if (!teleServerHost.empty()) - { - int teleServerPort = config::Config::fromText(config::Config::makeConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - fQtc.serverParms(QueryTeleServerParms(teleServerHost, teleServerPort)); - } - } -} - -DDLProcessor::~DDLProcessor() -{ - -} -void DDLProcessor::process() -{ - DBRM dbrm; - messageqcpp::IOSocket ios; - bool concurrentSupport = true; - string concurrentTranStr = config::Config::makeConfig()->getConfig("SystemConfig", "ConcurrentTransactions"); - - if ( concurrentTranStr.length() != 0 ) - { - if ((concurrentTranStr.compare("N") == 0) || (concurrentTranStr.compare("n") == 0)) - concurrentSupport = false; - } - - cout << "DDLProc is ready..." << endl; - + public: + PackageHandler(QueryTeleClient qtc, DBRM* dbrm, messageqcpp::IOSocket& ios, bool concurrentSupport) + : fIos(ios), fDbrm(dbrm), fQtc(qtc), fConcurrentSupport(concurrentSupport) + { + } + + void operator()() + { try { - for (;;) + fByteStream = fIos.read(); + if (fByteStream.empty()) + { + logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "Empty package", true); + return; + } + fByteStream >> fSessionID; + fByteStream >> fPackageType; + + uint32_t stateFlags; + if (fDbrm->getSystemState(stateFlags) > + 0) // > 0 implies successful retrieval. It doesn't imply anything about the contents + { + messageqcpp::ByteStream results; + const char* responseMsg = nullptr; + messageqcpp::ByteStream::byte status; + bool bReject = false; + + // Check to see if we're in write suspended or shutdown mode + // If so, we can't process the request. + if (stateFlags & SessionManagerServer::SS_SUSPENDED || + stateFlags & SessionManagerServer::SS_SUSPEND_PENDING || + stateFlags & SessionManagerServer::SS_SHUTDOWN_PENDING) { - try - { - ios = fMqServer.accept(); - fDdlPackagepool.invoke(PackageHandler(fQtc, &dbrm, ios, concurrentSupport)); - } - catch (...) - { - throw; - } + if (stateFlags & SessionManagerServer::SS_SUSPENDED || + stateFlags & SessionManagerServer::SS_SUSPEND_PENDING) + { + responseMsg = "Writing to the database is disabled."; + } + else + { + responseMsg = "The database is being shut down."; + } + + status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; + bReject = true; } + + if (bReject) + { + results << status; + //@bug 266 + MessageLog logger(LoggingID(27)); + logging::Message::Args args; + logging::Message message(2); + args.add(responseMsg); + message.format(args); + results << message.msg(); + fIos.write(results); + std::cout << responseMsg << endl; + std::cout << "Command rejected. Status " << (int)status << message.msg() << endl; + return; + } + } + + // check whether the system is ready to process statement. + if (fDbrm->getSystemReady() < 1) + { + logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "System is not ready yet. Please try again.", + true); + return; + } + + int rc = 0; + + if (!fConcurrentSupport) + { + // Check if any other active transaction + bool bIsDbrmUp = true; + bool anyOtherActiveTransaction = true; + execplan::SessionManager sessionManager; + BRM::SIDTIDEntry blockingsid; + int i = 0; + int waitPeriod = 10; + //@Bug 2487 Check transaction map every 1/10 second + + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + + string waitPeriodStr = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); + + if (waitPeriodStr.length() != 0) + waitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); + + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + // cout << "starting i = " << i << endl; + // anyOtherActiveTransaction = sessionManager.checkActiveTransaction(fSessionID, bIsDbrmUp); + while (anyOtherActiveTransaction) + { + anyOtherActiveTransaction = + sessionManager.checkActiveTransaction(fSessionID, bIsDbrmUp, blockingsid); + + if (anyOtherActiveTransaction) + { + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + // cout << "session " << fSessionID << " nanosleep on package type " << (int)packageType << + // endl; + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + anyOtherActiveTransaction = + sessionManager.checkActiveTransaction(fSessionID, bIsDbrmUp, blockingsid); + + if (!anyOtherActiveTransaction) + { + // cout << "Ready to process type " << (int)packageType << endl; + fTxnid = sessionManager.getTxnID(fSessionID); + + if (!fTxnid.valid) + { + fTxnid = sessionManager.newTxnID(fSessionID, true, true); + + if (fTxnid.valid) + { + // cout << "Ready to process type " << (int)packageType << " with txnid " << txnid.id << + // endl; + anyOtherActiveTransaction = false; + break; + } + else + { + anyOtherActiveTransaction = true; + } + } + else + { + string errorMsg; + rc = commitTransaction(fTxnid.id, errorMsg); + + if (rc != 0) + throw std::runtime_error(errorMsg); + + // need unlock the table. + std::vector tableLocks = fDbrm->getAllTableLocks(); + bool lockReleased = true; + + for (unsigned k = 0; k < tableLocks.size(); k++) + { + if (tableLocks[k].ownerTxnID == fTxnid.id) + { + lockReleased = fDbrm->releaseTableLock(tableLocks[k].id); + + if (!lockReleased) + { + ostringstream os; + os << "tablelock id " << tableLocks[k].id << " is not found"; + throw std::runtime_error(os.str()); + } + } + } + + fDbrm->committed(fTxnid); + fTxnid = fDbrm->newTxnID(fSessionID, true, true); + + if (fTxnid.valid) + { + // cout << "Ready to process type " << (int)packageType << " with txnid " << txnid.id << + // endl; + anyOtherActiveTransaction = false; + break; + } + else + { + anyOtherActiveTransaction = true; + } + } + } + } + + // cout << "ending i = " << i << endl; + } + else + { + // cout << "Ready to process type " << (int)packageType << endl; + fTxnid = sessionManager.getTxnID(fSessionID); + + if (!fTxnid.valid) + { + fTxnid = sessionManager.newTxnID(fSessionID, true, true); + + if (!fTxnid.valid) + { + // cout << "cannot get txnid " << (int)packageType << " for session " << sessionID << endl; + anyOtherActiveTransaction = true; + } + else + { + anyOtherActiveTransaction = false; + } + } + else + { + string errorMsg; + rc = commitTransaction(fTxnid.id, errorMsg); + + if (rc != 0) + throw std::runtime_error(errorMsg); + + // need unlock the table. + std::vector tableLocks = fDbrm->getAllTableLocks(); + bool lockReleased = true; + + for (unsigned k = 0; k < tableLocks.size(); k++) + { + if (tableLocks[k].ownerTxnID == fTxnid.id) + { + lockReleased = fDbrm->releaseTableLock(tableLocks[k].id); + + if (!lockReleased) + { + ostringstream os; + os << "tablelock id " << tableLocks[k].id << " is not found"; + throw std::runtime_error(os.str()); + } + } + } + + sessionManager.committed(fTxnid); + fTxnid = sessionManager.newTxnID(fSessionID, true, true); + + if (!fTxnid.valid) + { + // cout << "cannot get txnid " << (int)packageType << " for session " << sessionID << endl; + anyOtherActiveTransaction = true; + } + else + { + anyOtherActiveTransaction = false; + } + } + } + + if ((anyOtherActiveTransaction) && (i >= numTries)) + { + // cout << " Erroring out on package type " << (int)packageType << endl; + break; + } + } + + if ((anyOtherActiveTransaction) && (i >= numTries)) + { + messageqcpp::ByteStream::byte status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; + + Message::Args args; + args.add(static_cast(blockingsid.sessionid)); + + //@Bug 3854 Log to debug.log + LoggingID logid(15, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add(IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args)); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + + logError(status, IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args)); + } + else + { + processStatement(); + } + } + else + { + fTxnid = fDbrm->getTxnID(fSessionID); + + if (!fTxnid.valid) + { + fTxnid = fDbrm->newTxnID(fSessionID, true, true); + + if (!fTxnid.valid) + { + throw std::runtime_error(std::string("Unable to start a transaction. Check critical log.")); + } + } + else + { + string errorMsg; + rc = commitTransaction(fTxnid.id, errorMsg); + + if (rc != 0) + throw std::runtime_error(errorMsg); + + // need unlock the table. + std::vector tableLocks = fDbrm->getAllTableLocks(); + bool lockReleased = true; + + for (unsigned k = 0; k < tableLocks.size(); k++) + { + if (tableLocks[k].ownerTxnID == fTxnid.id) + { + lockReleased = fDbrm->releaseTableLock(tableLocks[k].id); + + if (!lockReleased) + { + ostringstream os; + os << "tablelock id " << tableLocks[k].id << " is not found"; + throw std::runtime_error(os.str()); + } + } + } + + fDbrm->committed(fTxnid); + fTxnid = fDbrm->newTxnID(fSessionID, true, true); + + if (!fTxnid.valid) + { + throw std::runtime_error(std::string("Unable to start a transaction. Check critical log.")); + } + } + + processStatement(); + } } - catch (exception& ex) + catch (const std::exception& ex) { - cerr << ex.what() << endl; - messageqcpp::ByteStream results; - messageqcpp::ByteStream::byte status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; - - results << status; - results << ex.what(); - ios.write(results); - - ios.close(); + logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, ex.what(), true); + throw; } - catch (...) - { - cerr << "Caught unknown exception!" << endl; - messageqcpp::ByteStream results; - messageqcpp::ByteStream::byte status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; + } - results << status; - results << "Caught unknown exception!"; - ios.write(results); - - ios.close(); - } - - // wait for all the threads to exit - fDdlPackagepool.wait(); -} - -int DDLProcessor::commitTransaction(uint32_t txnID, std::string& errorMsg) -{ - fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DDLPROC); - fPMCount = fWEClient->getPmCount(); + private: + int commitTransaction(uint32_t txnID, std::string& errorMsg) + { + auto WEClient = + std::unique_ptr(new WriteEngine::WEClients(WriteEngine::WEClients::DDLPROC)); + auto PMCount = WEClient->getPmCount(); ByteStream bytestream; - DBRM dbrm; - uint64_t uniqueId = dbrm.getUnique64(); - fWEClient->addQueue(uniqueId); + uint64_t uniqueId = fDbrm->getUnique64(); + WEClient->addQueue(uniqueId); bytestream << (ByteStream::byte)WE_SVR_COMMIT_VERSION; bytestream << uniqueId; bytestream << txnID; uint32_t msgRecived = 0; - fWEClient->write_to_all(bytestream); + WEClient->write_to_all(bytestream); boost::shared_ptr bsIn; bsIn.reset(new ByteStream()); int rc = 0; ByteStream::byte tmp8; - while (1) + while (true) { - if (msgRecived == fPMCount) - break; + if (msgRecived == PMCount) + break; - fWEClient->read(uniqueId, bsIn); + WEClient->read(uniqueId, bsIn); - if ( bsIn->length() == 0 ) //read error + if (bsIn->length() == 0) // read error + { + rc = 1; + errorMsg = "DDL cannot communicate with WES"; + WEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) { - rc = 1; - errorMsg = "DDL cannot communicate with WES"; - fWEClient->removeQueue(uniqueId); - break; + *bsIn >> errorMsg; + WEClient->removeQueue(uniqueId); + break; } else - { - *bsIn >> tmp8; - rc = tmp8; - - if (rc != 0) - { - *bsIn >> errorMsg; - fWEClient->removeQueue(uniqueId); - break; - } - else - msgRecived++; - } + msgRecived++; + } } - delete fWEClient; - fWEClient = 0; return rc; + } + + void processStatement() + { + DDLPackageProcessor::DDLResult result; + result.result = DDLPackageProcessor::NO_ERROR; + // boost::shared_ptr systemCatalogPtr; + + try + { + if (!fOamCache) + fOamCache = oam::OamCache::makeOamCache(); + // cout << "DDLProc received package " << fPackageType << endl; + switch (fPackageType) + { + case ddlpackage::DDL_CREATE_TABLE_STATEMENT: + { + CreateTableStatement createTableStmt; + createTableStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(createTableStmt.fSessionID); + boost::scoped_ptr processor(new CreateTableProcessor(fDbrm)); + processor->fTxnid.id = fTxnid.id; + processor->fTxnid.valid = true; + // cout << "create table using txnid " << fTxnid.id << endl; + + QueryTeleStats qts; + qts.query_uuid = QueryTeleClient::genUUID(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = createTableStmt.fSessionID; + qts.query_type = "CREATE"; + qts.query = createTableStmt.fSql; + qts.system_name = fOamCache->getSystemName(); + qts.module_name = fOamCache->getModuleName(); + qts.schema_name = createTableStmt.schemaName(); + fQtc.postQueryTele(qts); + + result = processor->processPackage(createTableStmt); + + systemCatalogPtr->removeCalpontSystemCatalog(createTableStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(createTableStmt.fSessionID | 0x80000000); + + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.end_time = QueryTeleClient::timeNowms(); + fQtc.postQueryTele(qts); + } + break; + + case ddlpackage::DDL_ALTER_TABLE_STATEMENT: + { + AlterTableStatement alterTableStmt; + alterTableStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(alterTableStmt.fSessionID); + boost::scoped_ptr processor(new AlterTableProcessor(fDbrm)); + processor->fTxnid.id = fTxnid.id; + processor->fTxnid.valid = true; + + QueryTeleStats qts; + qts.query_uuid = QueryTeleClient::genUUID(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = alterTableStmt.fSessionID; + qts.query_type = "ALTER"; + qts.query = alterTableStmt.fSql; + qts.system_name = fOamCache->getSystemName(); + qts.module_name = fOamCache->getModuleName(); + qts.schema_name = alterTableStmt.schemaName(); + fQtc.postQueryTele(qts); + + processor->fTimeZone = alterTableStmt.fTimeZone; + + result = processor->processPackage(alterTableStmt); + + systemCatalogPtr->removeCalpontSystemCatalog(alterTableStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(alterTableStmt.fSessionID | 0x80000000); + + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.end_time = QueryTeleClient::timeNowms(); + fQtc.postQueryTele(qts); + } + break; + + case ddlpackage::DDL_DROP_TABLE_STATEMENT: + { + DropTableStatement dropTableStmt; + dropTableStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(dropTableStmt.fSessionID); + boost::scoped_ptr processor(new DropTableProcessor(fDbrm)); + + processor->fTxnid.id = fTxnid.id; + processor->fTxnid.valid = true; + + QueryTeleStats qts; + qts.query_uuid = QueryTeleClient::genUUID(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = dropTableStmt.fSessionID; + qts.query_type = "DROP"; + qts.query = dropTableStmt.fSql; + qts.system_name = fOamCache->getSystemName(); + qts.module_name = fOamCache->getModuleName(); + qts.schema_name = dropTableStmt.schemaName(); + fQtc.postQueryTele(qts); + + // cout << "Drop table using txnid " << fTxnid.id << endl; + result = processor->processPackage(dropTableStmt); + + systemCatalogPtr->removeCalpontSystemCatalog(dropTableStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(dropTableStmt.fSessionID | 0x80000000); + + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.end_time = QueryTeleClient::timeNowms(); + fQtc.postQueryTele(qts); + } + break; + + case ddlpackage::DDL_TRUNC_TABLE_STATEMENT: + { + TruncTableStatement truncTableStmt; + truncTableStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(truncTableStmt.fSessionID); + boost::scoped_ptr processor(new TruncTableProcessor(fDbrm)); + + processor->fTxnid.id = fTxnid.id; + processor->fTxnid.valid = true; + + QueryTeleStats qts; + qts.query_uuid = QueryTeleClient::genUUID(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = truncTableStmt.fSessionID; + qts.query_type = "TRUNCATE"; + qts.query = truncTableStmt.fSql; + qts.system_name = fOamCache->getSystemName(); + qts.module_name = fOamCache->getModuleName(); + qts.schema_name = truncTableStmt.schemaName(); + fQtc.postQueryTele(qts); + + result = processor->processPackage(truncTableStmt); + + systemCatalogPtr->removeCalpontSystemCatalog(truncTableStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(truncTableStmt.fSessionID | 0x80000000); + + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.end_time = QueryTeleClient::timeNowms(); + fQtc.postQueryTele(qts); + } + break; + + case ddlpackage::DDL_MARK_PARTITION_STATEMENT: + { + MarkPartitionStatement markPartitionStmt; + markPartitionStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(markPartitionStmt.fSessionID); + boost::scoped_ptr processor(new MarkPartitionProcessor(fDbrm)); + (processor->fTxnid).id = fTxnid.id; + (processor->fTxnid).valid = true; + result = processor->processPackage(markPartitionStmt); + systemCatalogPtr->removeCalpontSystemCatalog(markPartitionStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(markPartitionStmt.fSessionID | 0x80000000); + } + break; + + case ddlpackage::DDL_RESTORE_PARTITION_STATEMENT: + { + RestorePartitionStatement restorePartitionStmt; + restorePartitionStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(restorePartitionStmt.fSessionID); + boost::scoped_ptr processor(new RestorePartitionProcessor(fDbrm)); + (processor->fTxnid).id = fTxnid.id; + (processor->fTxnid).valid = true; + result = processor->processPackage(restorePartitionStmt); + systemCatalogPtr->removeCalpontSystemCatalog(restorePartitionStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(restorePartitionStmt.fSessionID | 0x80000000); + } + break; + + case ddlpackage::DDL_DROP_PARTITION_STATEMENT: + { + DropPartitionStatement dropPartitionStmt; + dropPartitionStmt.unserialize(fByteStream); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(dropPartitionStmt.fSessionID); + boost::scoped_ptr processor(new DropPartitionProcessor(fDbrm)); + (processor->fTxnid).id = fTxnid.id; + (processor->fTxnid).valid = true; + result = processor->processPackage(dropPartitionStmt); + systemCatalogPtr->removeCalpontSystemCatalog(dropPartitionStmt.fSessionID); + systemCatalogPtr->removeCalpontSystemCatalog(dropPartitionStmt.fSessionID | 0x80000000); + } + break; + + default: throw UNRECOGNIZED_PACKAGE_TYPE; break; + } + + //@Bug 3427. No need to log user rror, just return the message to user. + // Log errors + if ((result.result != DDLPackageProcessor::NO_ERROR) && + (result.result != DDLPackageProcessor::USER_ERROR)) + { + logging::LoggingID lid(23); + logging::MessageLog ml(lid); + + ml.logErrorMessage(result.message); + } + + string hdfstest = config::Config::makeConfig()->getConfig("Installation", "DBRootStorageType"); + + if (hdfstest == "hdfs" || hdfstest == "HDFS") + cleanPMSysCache(); + + messageqcpp::ByteStream results; + messageqcpp::ByteStream::byte status = result.result; + results << status; + results << result.message.msg(); + + fIos.write(results); + + fIos.close(); + } + catch (quadbyte& /*foo*/) + { + logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "Unrecognized package type", true); + } + catch (logging::IDBExcept& idbEx) + { + cleanPMSysCache(); + logError(DDLPackageProcessor::CREATE_ERROR, idbEx.what(), true); + } + catch (...) + { + logError(DDLPackageProcessor::NOT_ACCEPTING_PACKAGES, "Unknown error", true); + } + } + + void logError(messageqcpp::ByteStream::byte status, const std::string& msg, bool closeSocket = false) + { + messageqcpp::ByteStream res; + res << status; + res << msg; + fIos.write(res); + cerr << "DDLProc error: " << msg << endl; + if (closeSocket) + fIos.close(); + } + + private: + messageqcpp::IOSocket fIos; + messageqcpp::ByteStream fByteStream; + messageqcpp::ByteStream::quadbyte fPackageType; + uint32_t fSessionID; + BRM::TxnID fTxnid; + BRM::DBRM* fDbrm; + QueryTeleClient fQtc; + oam::OamCache* fOamCache = nullptr; + bool fConcurrentSupport; +}; + +} // namespace + +namespace ddlprocessor +{ +DDLProcessor::DDLProcessor(int packageMaxThreads, int packageWorkQueueSize) + : fPackageMaxThreads(packageMaxThreads), fPackageWorkQueueSize(packageWorkQueueSize), fMqServer(DDLProcName) +{ + fDdlPackagepool.setMaxThreads(fPackageMaxThreads); + fDdlPackagepool.setQueueSize(fPackageWorkQueueSize); + fDdlPackagepool.setName("DdlPackagepool"); + csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); + csc->identity(CalpontSystemCatalog::EC); + string teleServerHost(config::Config::makeConfig()->getConfig("QueryTele", "Host")); + + if (!teleServerHost.empty()) + { + int teleServerPort = + config::Config::fromText(config::Config::makeConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) + { + fQtc.serverParms(QueryTeleServerParms(teleServerHost, teleServerPort)); + } + } +} + +DDLProcessor::~DDLProcessor() +{ +} +void DDLProcessor::process() +{ + DBRM dbrm; + messageqcpp::IOSocket ios; + bool concurrentSupport = true; + string concurrentTranStr = + config::Config::makeConfig()->getConfig("SystemConfig", "ConcurrentTransactions"); + + if (concurrentTranStr.length() != 0) + { + if ((concurrentTranStr.compare("N") == 0) || (concurrentTranStr.compare("n") == 0)) + concurrentSupport = false; + } + + cout << "DDLProc is ready..." << endl; + + try + { + for (;;) + { + try + { + ios = fMqServer.accept(); + fDdlPackagepool.invoke(PackageHandler(fQtc, &dbrm, ios, concurrentSupport)); + } + catch (...) + { + throw; + } + } + } + catch (exception& ex) + { + cerr << ex.what() << endl; + messageqcpp::ByteStream results; + messageqcpp::ByteStream::byte status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; + + results << status; + results << ex.what(); + ios.write(results); + + ios.close(); + } + catch (...) + { + cerr << "Caught unknown exception!" << endl; + messageqcpp::ByteStream results; + messageqcpp::ByteStream::byte status = DDLPackageProcessor::NOT_ACCEPTING_PACKAGES; + + results << status; + results << "Caught unknown exception!"; + ios.write(results); + + ios.close(); + } + + // wait for all the threads to exit + fDdlPackagepool.wait(); +} + +int DDLProcessor::commitTransaction(uint32_t txnID, std::string& errorMsg) +{ + fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DDLPROC); + fPMCount = fWEClient->getPmCount(); + ByteStream bytestream; + DBRM dbrm; + uint64_t uniqueId = dbrm.getUnique64(); + fWEClient->addQueue(uniqueId); + bytestream << (ByteStream::byte)WE_SVR_COMMIT_VERSION; + bytestream << uniqueId; + bytestream << txnID; + uint32_t msgRecived = 0; + fWEClient->write_to_all(bytestream); + boost::shared_ptr bsIn; + bsIn.reset(new ByteStream()); + int rc = 0; + ByteStream::byte tmp8; + + while (1) + { + if (msgRecived == fPMCount) + break; + + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + rc = 1; + errorMsg = "DDL cannot communicate with WES"; + fWEClient->removeQueue(uniqueId); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + + if (rc != 0) + { + *bsIn >> errorMsg; + fWEClient->removeQueue(uniqueId); + break; + } + else + msgRecived++; + } + } + + delete fWEClient; + fWEClient = 0; + return rc; } } // namespace ddlprocessor // vim:ts=4 sw=4: - diff --git a/ddlproc/ddlprocessor.h b/ddlproc/ddlprocessor.h index 525927b21..c9313b0dd 100644 --- a/ddlproc/ddlprocessor.h +++ b/ddlproc/ddlprocessor.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: ddlprocessor.h 6 2006-06-23 17:58:51Z rcraighead $ -* -* -***********************************************************************/ + * $Id: ddlprocessor.h 6 2006-06-23 17:58:51Z rcraighead $ + * + * + ***********************************************************************/ /** @file */ #ifndef DDLPROCESSOR_H #define DDLPROCESSOR_H @@ -32,74 +32,70 @@ namespace ddlprocessor { /** @brief process ddl packages as they arrive - */ + */ class DDLProcessor { -public: - /** @brief ctor - * - * @param packageMaxThreads the maximum number of threads to process ddl packages - * @param packageWorkQueueSize the maximum number of ddl packages in the work queue - */ - DDLProcessor( int packageMaxThreads, int packageWorkQueueSize ); - ~DDLProcessor(); - /** @brief entry point for the DDLProcessor - */ - void process(); - int commitTransaction(uint32_t txnID, std::string& errorMsg); + public: + /** @brief ctor + * + * @param packageMaxThreads the maximum number of threads to process ddl packages + * @param packageWorkQueueSize the maximum number of ddl packages in the work queue + */ + DDLProcessor(int packageMaxThreads, int packageWorkQueueSize); + ~DDLProcessor(); + /** @brief entry point for the DDLProcessor + */ + void process(); + int commitTransaction(uint32_t txnID, std::string& errorMsg); - /** @brief get the ddl package thread pool size - */ - inline int getPackageThreadPoolSize() const - { - return fPackageMaxThreads; - } + /** @brief get the ddl package thread pool size + */ + inline int getPackageThreadPoolSize() const + { + return fPackageMaxThreads; + } - /** @brief set the ddl package thread pool size - */ - inline void setPackageThreadPoolSize( int threadPoolSize ) - { - fPackageMaxThreads = threadPoolSize; - } + /** @brief set the ddl package thread pool size + */ + inline void setPackageThreadPoolSize(int threadPoolSize) + { + fPackageMaxThreads = threadPoolSize; + } - /** @brief get the maximum number of ddl packages allowed in the work queue - */ - inline int getPackageWorkQueueSize() const - { - return fPackageWorkQueueSize; - } + /** @brief get the maximum number of ddl packages allowed in the work queue + */ + inline int getPackageWorkQueueSize() const + { + return fPackageWorkQueueSize; + } - /** @brief set the maximum number of ddl packages allowed in the work queue - */ - inline void setPackageWorkQueueSize( int workQueueSize ) - { - fPackageWorkQueueSize = workQueueSize; - } + /** @brief set the maximum number of ddl packages allowed in the work queue + */ + inline void setPackageWorkQueueSize(int workQueueSize) + { + fPackageWorkQueueSize = workQueueSize; + } + protected: + private: + /** @brief ctor + */ + DDLProcessor(); -protected: + /** @brief the thread pool for processing ddl packages + */ + threadpool::ThreadPool fDdlPackagepool; -private: - /** @brief ctor - */ - DDLProcessor(); + int fPackageMaxThreads; /** @brief max number of threads to process ddl packages */ + int fPackageWorkQueueSize; /** @brief max number of ddl packages waiting in the work queue */ - /** @brief the thread pool for processing ddl packages - */ - threadpool::ThreadPool fDdlPackagepool; - - int fPackageMaxThreads; /** @brief max number of threads to process ddl packages */ - int fPackageWorkQueueSize; /** @brief max number of ddl packages waiting in the work queue */ - - messageqcpp::MessageQueueServer fMqServer; - boost::shared_ptr csc; - WriteEngine::WEClients* fWEClient; - uint32_t fPMCount; - querytele::QueryTeleClient fQtc; + messageqcpp::MessageQueueServer fMqServer; + boost::shared_ptr csc; + WriteEngine::WEClients* fWEClient; + uint32_t fPMCount; + querytele::QueryTeleClient fQtc; }; +} // namespace ddlprocessor -} //namespace ddlprocessor - -#endif //DDLPROCESSOR_H - +#endif // DDLPROCESSOR_H diff --git a/ddlproc/resource.h b/ddlproc/resource.h index f8e774b7b..fc7b96244 100644 --- a/ddlproc/resource.h +++ b/ddlproc/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/ddlproc/tdriver.cpp b/ddlproc/tdriver.cpp index edb2a5b08..55ec7e10e 100644 --- a/ddlproc/tdriver.cpp +++ b/ddlproc/tdriver.cpp @@ -43,33 +43,32 @@ using namespace messageqcpp; void sendOne(string sqlfile) { - MessageQueueClient ddlproc("DDLProc"); - ByteStream obs, ibs; - ByteStream::byte status; + MessageQueueClient ddlproc("DDLProc"); + ByteStream obs, ibs; + ByteStream::byte status; + SqlFileParser p; + p.Parse(sqlfile); + assert(p.Good()); - SqlFileParser p; - p.Parse(sqlfile); - assert(p.Good()); + const ParseTree& stmts = p.GetParseTree(); + SqlStatement* stmt = stmts[0]; + assert(stmt); - const ParseTree& stmts = p.GetParseTree(); - SqlStatement* stmt = stmts[0]; - assert(stmt); + stmt->serialize(obs); + ddlproc.write(obs); - stmt->serialize(obs); - ddlproc.write(obs); + ibs = ddlproc.read(); - ibs = ddlproc.read(); + cout << "bs: " << ibs.length() << endl; - cout << "bs: " << ibs.length() << endl; + if (ibs.length() > 0) + { + ibs >> status; + cout << "status: " << (int)status << endl; + } - if (ibs.length() > 0) - { - ibs >> status; - cout << "status: " << (int)status << endl; - } - - cout << "Bye" << endl; + cout << "Bye" << endl; } // ../dbcon/ddlpackage/sql/atac01.sql @@ -106,25 +105,20 @@ void sendOne(string sqlfile) // ../dbcon/ddlpackage/sql/dt01.sql // ../dbcon/ddlpackage/sql/dt02.sql - int main(int argc, char* argv[]) { - string sqlfile; + string sqlfile; - po::options_description desc ("Allowed options"); - desc.add_options () - ("help", "produce help message") - ("sql", po::value < string > (), "sql file"); - po::variables_map vm; - po::store (po::parse_command_line (argc, argv, desc), vm); - po::notify (vm); - - if (vm.count ("sql")) - { - sqlfile = vm["sql"].as (); - sendOne(sqlfile); - return 0; - } + po::options_description desc("Allowed options"); + desc.add_options()("help", "produce help message")("sql", po::value(), "sql file"); + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); + if (vm.count("sql")) + { + sqlfile = vm["sql"].as(); + sendOne(sqlfile); + return 0; + } } - diff --git a/dmlproc/batchinsertprocessor.cpp b/dmlproc/batchinsertprocessor.cpp index 1f7fbcda2..1a61e1eb6 100644 --- a/dmlproc/batchinsertprocessor.cpp +++ b/dmlproc/batchinsertprocessor.cpp @@ -51,499 +51,500 @@ boost::mutex fLock; namespace dmlprocessor { - -BatchInsertProc::BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm) : - fTxnid(txnId), fIsAutocommitOn(isAutocommitOn), fTableOid(tableOid), fDbrm(aDbrm) +BatchInsertProc::BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, + execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm) + : fTxnid(txnId), fIsAutocommitOn(isAutocommitOn), fTableOid(tableOid), fDbrm(aDbrm) { - fErrorCode = 0; - fErrMsg = ""; - fLastpkg = false; - fUniqueId = fDbrm->getUnique64(); - fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DMLPROC); - fWEClient->addQueue(fUniqueId); - fOamcache = OamCache::makeOamCache(); - std::vector moduleIds = fOamcache->getModuleIds(); + fErrorCode = 0; + fErrMsg = ""; + fLastpkg = false; + fUniqueId = fDbrm->getUnique64(); + fWEClient = new WriteEngine::WEClients(WriteEngine::WEClients::DMLPROC); + fWEClient->addQueue(fUniqueId); + fOamcache = OamCache::makeOamCache(); + std::vector moduleIds = fOamcache->getModuleIds(); - //cout << "moduleIds size is " << moduleIds.size() << endl; - for (unsigned i = 0; i < moduleIds.size(); i++) - { - fPMs.push_back((uint32_t)moduleIds[i]); - //cout << "got module id " << (uint32_t)moduleIds[i] << endl; - //cout << "fPMs[0] is " << fPMs[0] << endl; - } + // cout << "moduleIds size is " << moduleIds.size() << endl; + for (unsigned i = 0; i < moduleIds.size(); i++) + { + fPMs.push_back((uint32_t)moduleIds[i]); + // cout << "got module id " << (uint32_t)moduleIds[i] << endl; + // cout << "fPMs[0] is " << fPMs[0] << endl; + } - fBatchLoader = new BatchLoader(fTableOid, fTxnid, fPMs); + fBatchLoader = new BatchLoader(fTableOid, fTxnid, fPMs); - for (unsigned i = 0; i < fPMs.size(); i++) - { - fPmState[fPMs[i]] = true; - } + for (unsigned i = 0; i < fPMs.size(); i++) + { + fPmState[fPMs[i]] = true; + } - //cout << "Constructor: There are " << (int)fPMs.size() << " pms" << endl; - fInsertPkgQueue.reset(new pkg_type); + // cout << "Constructor: There are " << (int)fPMs.size() << " pms" << endl; + fInsertPkgQueue.reset(new pkg_type); } BatchInsertProc::~BatchInsertProc() { - fWEClient->removeQueue(fUniqueId); - delete fWEClient; + fWEClient->removeQueue(fUniqueId); + delete fWEClient; } uint64_t BatchInsertProc::grabTableLock(int32_t sessionId) { - uint32_t processID = ::getpid(); - int32_t txnId = fTxnid; - std::string processName("DMLProc batchinsert"); - int32_t tmpSessionId = sessionId; - uint32_t tableOid = fTableOid; - int i = 0; + uint32_t processID = ::getpid(); + int32_t txnId = fTxnid; + std::string processName("DMLProc batchinsert"); + int32_t tmpSessionId = sessionId; + uint32_t tableOid = fTableOid; + int i = 0; - try + try + { + // cout << "In grabTableLock, fPMs[0] is "<< fPMs[0] << endl; + fTableLockid = + fDbrm->getTableLock(fPMs, tableOid, &processName, &processID, &tmpSessionId, &txnId, BRM::LOADING); + } + catch (std::exception&) + { + setError(dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR, + IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + return 0; + } + + if (fTableLockid == 0) + { + int waitPeriod = 10; + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + string waitPeriodStr = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); + + if (waitPeriodStr.length() != 0) + waitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); + + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + for (; i < numTries; i++) { - //cout << "In grabTableLock, fPMs[0] is "<< fPMs[0] << endl; - fTableLockid = fDbrm->getTableLock(fPMs, tableOid, &processName, &processID, &tmpSessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - setError(dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - return 0; - } - - if ( fTableLockid == 0 ) - { - int waitPeriod = 10; - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - string waitPeriodStr = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); - - if ( waitPeriodStr.length() != 0 ) - waitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); - - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - for (; i < numTries; i++) - { #ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); + Sleep(rm_ts.tv_sec * 1000); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif - try - { - processID = ::getpid(); - txnId = fTxnid; - tmpSessionId = sessionId; - processName = "DMLProc batchinsert"; - fTableLockid = fDbrm->getTableLock(fPMs, tableOid, &processName, &processID, &tmpSessionId, &txnId, BRM::LOADING ); - } - catch (std::exception&) - { - setError(dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - return 0; - } + try + { + processID = ::getpid(); + txnId = fTxnid; + tmpSessionId = sessionId; + processName = "DMLProc batchinsert"; + fTableLockid = fDbrm->getTableLock(fPMs, tableOid, &processName, &processID, &tmpSessionId, &txnId, + BRM::LOADING); + } + catch (std::exception&) + { + setError(dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR, + IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + return 0; + } - if (fTableLockid > 0) - return fTableLockid; - } - - if (i >= numTries) //error out - { - logging::Message::Args args; - string strOp("batch insert"); - args.add(strOp); - args.add(processName); - args.add((uint64_t)processID); - args.add(tmpSessionId); - setError(dmlpackageprocessor::DMLPackageProcessor::TABLE_LOCK_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); - return 0; - } + if (fTableLockid > 0) + return fTableLockid; } - return fTableLockid; + if (i >= numTries) // error out + { + logging::Message::Args args; + string strOp("batch insert"); + args.add(strOp); + args.add(processName); + args.add((uint64_t)processID); + args.add(tmpSessionId); + setError(dmlpackageprocessor::DMLPackageProcessor::TABLE_LOCK_ERROR, + IDBErrorInfo::instance()->errorMsg(ERR_TABLE_LOCKED, args)); + return 0; + } + } + + return fTableLockid; } -BatchInsertProc::SP_PKG BatchInsertProc::getInsertQueue() +BatchInsertProc::SP_PKG BatchInsertProc::getInsertQueue() { - boost::mutex::scoped_lock lk(fLock); - return fInsertPkgQueue; + boost::mutex::scoped_lock lk(fLock); + return fInsertPkgQueue; } -void BatchInsertProc::setLastPkg (bool lastPkg) +void BatchInsertProc::setLastPkg(bool lastPkg) { - fLastpkg = lastPkg; + fLastpkg = lastPkg; } void BatchInsertProc::addPkg(messageqcpp::ByteStream& insertBs) { - boost::mutex::scoped_lock lk(fLock); - fInsertPkgQueue->push(insertBs); - + boost::mutex::scoped_lock lk(fLock); + fInsertPkgQueue->push(insertBs); } messageqcpp::ByteStream BatchInsertProc::getPkg() { - messageqcpp::ByteStream bs; - boost::mutex::scoped_lock lk(fLock); - bs = fInsertPkgQueue->front(); - fInsertPkgQueue->pop(); - return bs; + messageqcpp::ByteStream bs; + boost::mutex::scoped_lock lk(fLock); + bs = fInsertPkgQueue->front(); + fInsertPkgQueue->pop(); + return bs; } void BatchInsertProc::buildPkg(messageqcpp::ByteStream& bs) { - bs.reset(); - bs << (ByteStream::byte) WE_SVR_BATCH_INSERT; - bs << fUniqueId; - bs << (uint32_t) fTxnid; - bs << fCurrentPMid; //to keep track of PMs - bs += getPkg(); + bs.reset(); + bs << (ByteStream::byte)WE_SVR_BATCH_INSERT; + bs << fUniqueId; + bs << (uint32_t)fTxnid; + bs << fCurrentPMid; // to keep track of PMs + bs += getPkg(); } void BatchInsertProc::buildLastPkg(messageqcpp::ByteStream& bs) { - ByteStream::byte rt; + ByteStream::byte rt; - // Bug 757. WriteEngineServer deletes metadata if ErrorCode != 0. With range warning, - // we still inserted data, just truncated. Kludge to keep WriteEngineServer on track. - if (fErrorCode == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - rt = 0; - else - rt = fErrorCode; + // Bug 757. WriteEngineServer deletes metadata if ErrorCode != 0. With range warning, + // we still inserted data, just truncated. Kludge to keep WriteEngineServer on track. + if (fErrorCode == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + rt = 0; + else + rt = fErrorCode; - bs.reset(); - bs << (ByteStream::byte) WE_SVR_BATCH_INSERT_END; - bs << fUniqueId; - bs << (ByteStream::quadbyte) fTxnid; - bs << (ByteStream::byte)fIsAutocommitOn; - bs << fTableOid; - bs << rt; + bs.reset(); + bs << (ByteStream::byte)WE_SVR_BATCH_INSERT_END; + bs << fUniqueId; + bs << (ByteStream::quadbyte)fTxnid; + bs << (ByteStream::byte)fIsAutocommitOn; + bs << fTableOid; + bs << rt; } void BatchInsertProc::sendFirstBatch() { - uint32_t firstPmId = 0; - int rc = 0; + uint32_t firstPmId = 0; + int rc = 0; - try - { - firstPmId = fBatchLoader->selectNextPM(); - } - catch (std::exception& ex) - { - rc = 1; - setError(rc, ex.what()); - return; - } + try + { + firstPmId = fBatchLoader->selectNextPM(); + } + catch (std::exception& ex) + { + rc = 1; + setError(rc, ex.what()); + return; + } - if (firstPmId == 0) - { - rc = 1; - setError(rc, "Request cannot be sent to PM 0."); - return; - } + if (firstPmId == 0) + { + rc = 1; + setError(rc, "Request cannot be sent to PM 0."); + return; + } - fCurrentPMid = firstPmId; - messageqcpp::ByteStream bs; - buildPkg(bs); + fCurrentPMid = firstPmId; + messageqcpp::ByteStream bs; + buildPkg(bs); - fWEClient->write(bs, fCurrentPMid); - //cout << "in sendFirstBatch: batchinsertprocessor sent pkg to pmnum " << fCurrentPMid << endl; - fPmState[fCurrentPMid] = false; + fWEClient->write(bs, fCurrentPMid); + // cout << "in sendFirstBatch: batchinsertprocessor sent pkg to pmnum " << fCurrentPMid << endl; + fPmState[fCurrentPMid] = false; } void BatchInsertProc::sendNextBatch() { - int rc = 0; + int rc = 0; - try - { - fCurrentPMid = fBatchLoader->selectNextPM(); - } - catch (std::exception& ex) - { - rc = 1; - setError(rc, ex.what()); - return; - } + try + { + fCurrentPMid = fBatchLoader->selectNextPM(); + } + catch (std::exception& ex) + { + rc = 1; + setError(rc, ex.what()); + return; + } - messageqcpp::ByteStream bs; - buildPkg(bs); - string errorMsg; + messageqcpp::ByteStream bs; + buildPkg(bs); + string errorMsg; - //cout << "in sendNextBatch: fCurrentPMid changed to " << fCurrentPMid << " this = " << this << endl; - if (fPmState[fCurrentPMid]) + // cout << "in sendNextBatch: fCurrentPMid changed to " << fCurrentPMid << " this = " << this << endl; + if (fPmState[fCurrentPMid]) + { + // cout << "current pm state for pm is true for pm " << fCurrentPMid << " this = " << this<< endl; + fWEClient->write(bs, fCurrentPMid); + // cout << "batchinsertprocessor sent pkg to pmnum " << fCurrentPMid << endl; + fPmState[fCurrentPMid] = false; + // cout << "set pm state to false for pm " << fCurrentPMid << " this = " << this << endl; + } + else + { + // cout << "current pm state for pm is false for pm " << fCurrentPMid<< endl; + while (1) { - //cout << "current pm state for pm is true for pm " << fCurrentPMid << " this = " << this<< endl; - fWEClient->write(bs, fCurrentPMid); - //cout << "batchinsertprocessor sent pkg to pmnum " << fCurrentPMid << endl; - fPmState[fCurrentPMid] = false; - //cout << "set pm state to false for pm " << fCurrentPMid << " this = " << this << endl; - } - else - { - //cout << "current pm state for pm is false for pm " << fCurrentPMid<< endl; - while (1) + // cout << "Read from WES for pm id " << (uint32_t) tmp32 << endl; + bsIn.reset(new ByteStream()); + fWEClient->read(fUniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + errorMsg = "Lost connection to WES."; + setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); + break; + } + else + { + *bsIn >> tmp8; + rc = tmp8; + *bsIn >> errorMsg; + *bsIn >> tmp32; + + // cout << "got response from WES for pm id " << (uint32_t) tmp32 << endl; + if (rc != 0) { - //cout << "Read from WES for pm id " << (uint32_t) tmp32 << endl; - bsIn.reset(new ByteStream()); - fWEClient->read(fUniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - errorMsg = "Lost connection to WES."; - setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); - break; - } - else - { - *bsIn >> tmp8; - rc = tmp8; - *bsIn >> errorMsg; - *bsIn >> tmp32; - - //cout << "got response from WES for pm id " << (uint32_t) tmp32 << endl; - if (rc != 0) - { - //cout << "Batch insert got error code:errormsg = " << (uint32_t)tmp8<<":"<write(bs, fCurrentPMid); - //cout << "batchinsertprocessor sent pkg to pm " << fCurrentPMid << endl; - fPmState[fCurrentPMid] = false; - //cout << "set pm state to false for pm " << fCurrentPMid << " this = " << this << endl; + fPmState[tmp32] = true; + + // cout << "set pm state to true for pm " << (uint32_t) tmp32 << " and current PM id is " << + // fCurrentPMid<< " this = " << this << endl; + if (tmp32 == fCurrentPMid) + break; } + // cout << "before batchinsertprocessor sent pkg to pm " << fCurrentPMid << endl; + fWEClient->write(bs, fCurrentPMid); + // cout << "batchinsertprocessor sent pkg to pm " << fCurrentPMid << endl; + fPmState[fCurrentPMid] = false; + // cout << "set pm state to false for pm " << fCurrentPMid << " this = " << this << endl; + } } void BatchInsertProc::sendlastBatch() { - messageqcpp::ByteStream bs; - buildLastPkg(bs); + messageqcpp::ByteStream bs; + buildLastPkg(bs); - try - { - fWEClient->write_to_all(bs); - //cout << "sent the last pkg" << endl; - } - catch (std::exception& ex) - { - ostringstream oss; - oss << "Exception on communicating to WES "; - oss << ex.what(); - setError(1, oss.str()); - //cout << oss.str() << endl; - } + try + { + fWEClient->write_to_all(bs); + // cout << "sent the last pkg" << endl; + } + catch (std::exception& ex) + { + ostringstream oss; + oss << "Exception on communicating to WES "; + oss << ex.what(); + setError(1, oss.str()); + // cout << oss.str() << endl; + } } void BatchInsertProc::receiveOutstandingMsg() { - //check how many message we need to receive - uint32_t messagesNotReceived = 0; - int rc = 0; + // check how many message we need to receive + uint32_t messagesNotReceived = 0; + int rc = 0; - for (unsigned i = 0; i < fPMs.size(); i++) + for (unsigned i = 0; i < fPMs.size(); i++) + { + if (!fPmState[fPMs[i]]) + messagesNotReceived++; + } + + // cout << "receiveOutstandingMsg: Need to receive " << messagesNotReceived << " messages. this = " << this + // << endl; + if ((messagesNotReceived > 0) && (messagesNotReceived <= fWEClient->getPmCount())) + { + string errorMsg; + uint32_t msgReceived = 0; + + while (1) { - if (!fPmState[fPMs[i]]) - messagesNotReceived++; - } + if (msgReceived == messagesNotReceived) + break; - //cout << "receiveOutstandingMsg: Need to receive " << messagesNotReceived << " messages. this = " << this << endl; - if ((messagesNotReceived > 0) && (messagesNotReceived <= fWEClient->getPmCount())) - { - string errorMsg; - uint32_t msgReceived = 0; + bsIn.reset(new ByteStream()); - while (1) + try + { + fWEClient->read(fUniqueId, bsIn); + + if (bsIn->length() == 0) // read error { - if (msgReceived == messagesNotReceived) - break; - - bsIn.reset(new ByteStream()); - - try - { - fWEClient->read(fUniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - rc = 1; - setError(rc, errorMsg); - } - else - { - *bsIn >> tmp8; - *bsIn >> errorMsg; - *bsIn >> tmp32; - - fPmState[tmp32] = true; - msgReceived++; - - if ( tmp8 != 0 ) - setError(tmp8, errorMsg); - } - } - catch (runtime_error& ex) //write error - { - errorMsg = ex.what(); - setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); - break; - } - catch (...) - { - errorMsg = "Lost connection to WES."; - setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); - break; - } + rc = 1; + setError(rc, errorMsg); } - } + else + { + *bsIn >> tmp8; + *bsIn >> errorMsg; + *bsIn >> tmp32; + fPmState[tmp32] = true; + msgReceived++; + + if (tmp8 != 0) + setError(tmp8, errorMsg); + } + } + catch (runtime_error& ex) // write error + { + errorMsg = ex.what(); + setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); + break; + } + catch (...) + { + errorMsg = "Lost connection to WES."; + setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); + break; + } + } + } } void BatchInsertProc::receiveAllMsg() { - uint32_t msgRevd = 0; - int rc = 0; - string errorMsg; + uint32_t msgRevd = 0; + int rc = 0; + string errorMsg; - try + try + { + while (1) { - while (1) + if (msgRevd == fWEClient->getPmCount()) + break; + + // cout << "Read last from WES bytestream" << endl; + fWEClient->read(fUniqueId, bsIn); + + if (bsIn->length() == 0) // read error + { + errorMsg = "Lost connection to WES."; + setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); + msgRevd++; + + if (!fIsAutocommitOn) { - if (msgRevd == fWEClient->getPmCount()) - break; - - //cout << "Read last from WES bytestream" << endl; - fWEClient->read(fUniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - errorMsg = "Lost connection to WES."; - setError(dmlpackageprocessor::DMLPackageProcessor::NETWORK_ERROR, errorMsg); - msgRevd++; - - if (!fIsAutocommitOn) - { - BulkSetHWMArgs setHWMArgs; - fHwmArgsAllPms.push_back(setHWMArgs); - } - } - else - { - *bsIn >> tmp8; - rc = tmp8; - *bsIn >> errorMsg; - msgRevd++; - - if (!fIsAutocommitOn) //collect Hwm - { - BulkSetHWMArgs setHWMArgs; - deserializeInlineVector(*(bsIn.get()), setHWMArgs); - fHwmArgsAllPms.push_back(setHWMArgs); - - } - - if (rc == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - { - setError(rc, errorMsg); - } - else if (rc != 0) - { - //cout << "Batch insert lastpkg got error code:errormsg = " << (uint32_t)tmp8<<":"<> tmp8; + rc = tmp8; + *bsIn >> errorMsg; + msgRevd++; - if (!fIsAutocommitOn && (fHwmArgsAllPms.size() == fWEClient->getPmCount())) - setHwm(); + if (!fIsAutocommitOn) // collect Hwm + { + BulkSetHWMArgs setHWMArgs; + deserializeInlineVector(*(bsIn.get()), setHWMArgs); + fHwmArgsAllPms.push_back(setHWMArgs); + } + if (rc == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + { + setError(rc, errorMsg); + } + else if (rc != 0) + { + // cout << "Batch insert lastpkg got error code:errormsg = " << (uint32_t)tmp8<<":"<getPmCount())) + setHwm(); } void BatchInsertProc::collectHwm() { - BulkSetHWMArgs setHWMArgs; - //cout << "received from WES bytestream length = " << bsIn->length() << endl; - deserializeInlineVector(*(bsIn.get()), setHWMArgs); - //cout << "get hwm info from WES size " << setHWMArgs.size() << endl; - fHwmArgsAllPms.push_back(setHWMArgs); + BulkSetHWMArgs setHWMArgs; + // cout << "received from WES bytestream length = " << bsIn->length() << endl; + deserializeInlineVector(*(bsIn.get()), setHWMArgs); + // cout << "get hwm info from WES size " << setHWMArgs.size() << endl; + fHwmArgsAllPms.push_back(setHWMArgs); } void BatchInsertProc::setHwm() { - std::vector allHwm; - BulkSetHWMArgs::const_iterator itor; + std::vector allHwm; + BulkSetHWMArgs::const_iterator itor; - //cout << "total hwmArgsAllPms size " << hwmArgsAllPms.size() << endl; - for (unsigned i = 0; i < fWEClient->getPmCount(); i++) + // cout << "total hwmArgsAllPms size " << hwmArgsAllPms.size() << endl; + for (unsigned i = 0; i < fWEClient->getPmCount(); i++) + { + itor = fHwmArgsAllPms[i].begin(); + + while (itor != fHwmArgsAllPms[i].end()) { - itor = fHwmArgsAllPms[i].begin(); - - while (itor != fHwmArgsAllPms[i].end()) - { - allHwm.push_back(*itor); - //cout << "received hwm info: " << itor->oid << ":" << itor->hwm << endl; - itor++; - } + allHwm.push_back(*itor); + // cout << "received hwm info: " << itor->oid << ":" << itor->hwm << endl; + itor++; } + } - if (allHwm.size() > 0) + if (allHwm.size() > 0) + { + // cout << "setting hwm allHwm size " << allHwm.size() << endl; + int rc = fDbrm->bulkSetHWM(allHwm, 0); + + if (rc != 0) { - //cout << "setting hwm allHwm size " << allHwm.size() << endl; - int rc = fDbrm->bulkSetHWM(allHwm, 0); - - if ( rc != 0 ) - { - string errorMsg; - BRM::errString(rc, errorMsg); - setError(rc, errorMsg); - } + string errorMsg; + BRM::errString(rc, errorMsg); + setError(rc, errorMsg); } + } } void BatchInsertProc::setError(int errorCode, std::string errMsg) { - boost::mutex::scoped_lock lk(fLock); - fErrorCode = errorCode; - fErrMsg = errMsg; + boost::mutex::scoped_lock lk(fLock); + fErrorCode = errorCode; + fErrMsg = errMsg; } void BatchInsertProc::getError(int& errorCode, std::string& errMsg) { - boost::mutex::scoped_lock lk(fLock); - errorCode = fErrorCode; - errMsg = fErrMsg; -} + boost::mutex::scoped_lock lk(fLock); + errorCode = fErrorCode; + errMsg = fErrMsg; } +} // namespace dmlprocessor // vim:ts=4 sw=4: diff --git a/dmlproc/batchinsertprocessor.h b/dmlproc/batchinsertprocessor.h index 2c8b3a32d..c7cccb1db 100644 --- a/dmlproc/batchinsertprocessor.h +++ b/dmlproc/batchinsertprocessor.h @@ -42,57 +42,57 @@ namespace dmlprocessor { class BatchInsertProc { -public: - typedef std::queue pkg_type; - typedef boost::shared_ptr SP_PKG; - typedef std::vector BulkSetHWMArgs; - BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm); - BatchInsertProc(const BatchInsertProc& rhs); - ~BatchInsertProc(); - uint64_t grabTableLock(int32_t sessionId); - SP_PKG getInsertQueue (); - uint32_t getNumDBRoots(); - void setLastPkg (bool lastPkg); - void addPkg(messageqcpp::ByteStream& insertBs); - messageqcpp::ByteStream getPkg(); - void setError(int errorCode, std::string errMsg); - void getError(int& errorCode, std::string& errMsg); - int sendPkg(int pmId); - void buildPkg(messageqcpp::ByteStream& bs); - void buildLastPkg(messageqcpp::ByteStream& bs); - void sendFirstBatch(); - void sendNextBatch(); - void sendlastBatch(); - void collectHwm(); - void setHwm(); - void receiveAllMsg(); - void receiveOutstandingMsg(); -private: - SP_PKG fInsertPkgQueue; - boost::condition condvar; - execplan::CalpontSystemCatalog::SCN fTxnid; - int fErrorCode; - std::string fErrMsg; - bool fLastpkg; - bool fIsAutocommitOn; - uint32_t fTableOid; - uint64_t fUniqueId; - BRM::DBRM* fDbrm; - WriteEngine::WEClients* fWEClient; - oam::OamCache* fOamcache; - std::vector fPMs; //active PMs - batchloader::BatchLoader* fBatchLoader; - std::map fPmState; - uint32_t fCurrentPMid; - boost::shared_ptr bsIn; - messageqcpp::ByteStream::byte tmp8; - messageqcpp::ByteStream::quadbyte tmp32; - std::vector fHwmArgsAllPms; - uint64_t fTableLockid; - + public: + typedef std::queue pkg_type; + typedef boost::shared_ptr SP_PKG; + typedef std::vector BulkSetHWMArgs; + BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, execplan::CalpontSystemCatalog::SCN txnId, + BRM::DBRM* aDbrm); + BatchInsertProc(const BatchInsertProc& rhs); + ~BatchInsertProc(); + uint64_t grabTableLock(int32_t sessionId); + SP_PKG getInsertQueue(); + uint32_t getNumDBRoots(); + void setLastPkg(bool lastPkg); + void addPkg(messageqcpp::ByteStream& insertBs); + messageqcpp::ByteStream getPkg(); + void setError(int errorCode, std::string errMsg); + void getError(int& errorCode, std::string& errMsg); + int sendPkg(int pmId); + void buildPkg(messageqcpp::ByteStream& bs); + void buildLastPkg(messageqcpp::ByteStream& bs); + void sendFirstBatch(); + void sendNextBatch(); + void sendlastBatch(); + void collectHwm(); + void setHwm(); + void receiveAllMsg(); + void receiveOutstandingMsg(); + private: + SP_PKG fInsertPkgQueue; + boost::condition condvar; + execplan::CalpontSystemCatalog::SCN fTxnid; + int fErrorCode; + std::string fErrMsg; + bool fLastpkg; + bool fIsAutocommitOn; + uint32_t fTableOid; + uint64_t fUniqueId; + BRM::DBRM* fDbrm; + WriteEngine::WEClients* fWEClient; + oam::OamCache* fOamcache; + std::vector fPMs; // active PMs + batchloader::BatchLoader* fBatchLoader; + std::map fPmState; + uint32_t fCurrentPMid; + boost::shared_ptr bsIn; + messageqcpp::ByteStream::byte tmp8; + messageqcpp::ByteStream::quadbyte tmp32; + std::vector fHwmArgsAllPms; + uint64_t fTableLockid; }; -} // namespace dmlprocessor +} // namespace dmlprocessor #endif // vim:ts=4 sw=4: diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index 3a6ca82c0..f1fa9cc4c 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlproc.cpp 1029 2013-07-31 19:17:42Z dhall $ -* -* -***********************************************************************/ + * $Id: dmlproc.cpp 1029 2013-07-31 19:17:42Z dhall $ + * + * + ***********************************************************************/ #include #include #include @@ -62,7 +62,7 @@ using namespace oam; #include "writeengine.h" #include "IDBPolicy.h" -#undef READ //someone has polluted the namespace! +#undef READ // someone has polluted the namespace! #include "vss.h" #include "brmtypes.h" using namespace BRM; @@ -76,7 +76,6 @@ using namespace messageqcpp; #include "distributedenginecomm.h" using namespace joblist; - #include "crashtrace.h" #include "installdir.h" @@ -84,626 +83,619 @@ using namespace joblist; #include "service.h" - threadpool::ThreadPool DMLServer::fDmlPackagepool(10, 0); namespace { - - class Opt { -public: - int m_debug; - bool m_fg; - Opt(int argc, char *argv[]) - :m_debug(0), - m_fg(false) + public: + int m_debug; + bool m_fg; + Opt(int argc, char* argv[]) : m_debug(0), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "df")) != EOF) { - int c; - while ((c = getopt(argc, argv, "df")) != EOF) - { - switch(c) - { - case 'd': - m_debug++; // TODO: not really used yes - break; - case 'f': - m_fg= true; - break; - case '?': - default: - break; - } - } + switch (c) + { + case 'd': + m_debug++; // TODO: not really used yes + break; + case 'f': m_fg = true; break; + case '?': + default: break; + } + } } }; - -class ServiceDMLProc: public Service, public Opt +class ServiceDMLProc : public Service, public Opt { -protected: - int Parent() override - { - /* - Need to shutdown TheadPool, - otherwise it would get stuck when trying to join fPruneThread. - */ - joblist::JobStep::jobstepThreadPool.stop(); - return Service::Parent(); - } + protected: + int Parent() override + { + /* + Need to shutdown TheadPool, + otherwise it would get stuck when trying to join fPruneThread. + */ + joblist::JobStep::jobstepThreadPool.stop(); + return Service::Parent(); + } - void log(logging::LOG_TYPE type, const std::string &str) - { - LoggingID logid(23, 0, 0); - Message::Args args; - Message message(8); - args.add(str); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, message, logid); - } + void log(logging::LOG_TYPE type, const std::string& str) + { + LoggingID logid(23, 0, 0); + Message::Args args; + Message message(8); + args.add(str); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, message, logid); + } - void setupChildSignalHandlers(); + void setupChildSignalHandlers(); -public: - ServiceDMLProc(const Opt &opt) - :Service("DMLProc"), Opt(opt) - { } - void LogErrno() override - { - log(LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string &str) override - { - log(LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServiceDMLProc(const Opt& opt) : Service("DMLProc"), Opt(opt) + { + } + void LogErrno() override + { + log(LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - DistributedEngineComm* Dec; void added_a_pm(int) { - LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("DMLProc caught SIGHUP. Resetting connections"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("DMLProc caught SIGHUP. Resetting connections"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - Dec->Setup(); - // MCOL-140 clear the waiting queue as all transactions are probably going to fail - PackageHandler::clearTableAccess(); + Dec->Setup(); + // MCOL-140 clear the waiting queue as all transactions are probably going to fail + PackageHandler::clearTableAccess(); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - //WriteEngine::WEClients::instance(WriteEngine::WEClients::DMLPROC)->Setup(); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + // WriteEngine::WEClients::instance(WriteEngine::WEClients::DMLPROC)->Setup(); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); } int toInt(const string& val) { - if (val.length() == 0) - return -1; + if (val.length() == 0) + return -1; - return static_cast(Config::fromText(val)); + return static_cast(Config::fromText(val)); } class CleanUpThread { -public: - CleanUpThread() {} - ~CleanUpThread() {} - void operator()() - { - ddlcleanuputil::ddl_cleanup(); - return; - } + public: + CleanUpThread() + { + } + ~CleanUpThread() + { + } + void operator()() + { + ddlcleanuputil::ddl_cleanup(); + return; + } }; // This function rolls back any active transactions in case of an abnormal shutdown. void rollbackAll(DBRM* dbrm) { - Oam oam; + Oam oam; - //Log a message in info.log - logging::Message::Args args; - logging::Message message(2); - args.add("DMLProc starts rollbackAll."); - message.format( args ); - logging::LoggingID lid(20); - logging::MessageLog ml(lid); - ml.logInfoMessage( message ); + // Log a message in info.log + logging::Message::Args args; + logging::Message message(2); + args.add("DMLProc starts rollbackAll."); + message.format(args); + logging::LoggingID lid(20); + logging::MessageLog ml(lid); + ml.logInfoMessage(message); - boost::shared_array activeTxns; - BRM::TxnID txnID; - SessionManager sessionManager; - int rc = 0; - rc = dbrm->isReadWrite(); + boost::shared_array activeTxns; + BRM::TxnID txnID; + SessionManager sessionManager; + int rc = 0; + rc = dbrm->isReadWrite(); - if (rc != 0 ) - throw std::runtime_error("Rollback will be deferred due to DBRM is in read only state."); + if (rc != 0) + throw std::runtime_error("Rollback will be deferred due to DBRM is in read only state."); - dbrm->setSystemReady(false); - //DDL clean up thread - thread_group tg; - tg.create_thread(CleanUpThread()); + dbrm->setSystemReady(false); + // DDL clean up thread + thread_group tg; + tg.create_thread(CleanUpThread()); - std::vector tableLocks; + std::vector tableLocks; - try + try + { + tableLocks = dbrm->getAllTableLocks(); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + uint64_t uniqueId = dbrm->getUnique64(); + RollbackTransactionProcessor rollbackProcessor(dbrm); + std::string errorMsg; + unsigned int i = 0; + BRM::TxnID txnId; + ostringstream oss; + + oss << "DMLProc will rollback " << tableLocks.size() << " tables."; + // cout << oss.str() << endl; + logging::Message::Args args5; + logging::Message message5(2); + args5.add(oss.str()); + message5.format(args5); + ml.logInfoMessage(message5); + OamCache* oamcache = OamCache::makeOamCache(); + OamCache::dbRootPMMap_t dbRootPMMap = oamcache->getDBRootToPMMap(); + int errorTxn = 0; + + for (i = 0; i < tableLocks.size(); i++) + { + if (tableLocks[i].ownerTxnID > 0) // transaction rollback { - tableLocks = dbrm->getAllTableLocks(); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } + ostringstream oss; + oss << "DMLProc is rolling back transaction " << tableLocks[i].ownerTxnID; + // cout << oss.str() << endl; + logging::Message::Args args1; + logging::Message message2(2); + args1.add(oss.str()); + message2.format(args1); + ml.logInfoMessage(message2); + dbrm->invalidateUncommittedExtentLBIDs(tableLocks[i].ownerTxnID, false); + uint32_t sessionid = 0; + txnId.id = tableLocks[i].ownerTxnID; + txnId.valid = true; + rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg); - uint64_t uniqueId = dbrm->getUnique64(); - RollbackTransactionProcessor rollbackProcessor(dbrm); - std::string errorMsg; - unsigned int i = 0; - BRM::TxnID txnId; - ostringstream oss; - - oss << "DMLProc will rollback " << tableLocks.size() << " tables."; - //cout << oss.str() << endl; - logging::Message::Args args5; - logging::Message message5(2); - args5.add(oss.str()); - message5.format( args5 ); - ml.logInfoMessage( message5 ); - OamCache* oamcache = OamCache::makeOamCache(); - OamCache::dbRootPMMap_t dbRootPMMap = oamcache->getDBRootToPMMap(); - int errorTxn = 0; - - for ( i = 0; i < tableLocks.size(); i++) - { - if (tableLocks[i].ownerTxnID > 0) //transaction rollback - { - ostringstream oss; - oss << "DMLProc is rolling back transaction " << tableLocks[i].ownerTxnID; - //cout << oss.str() << endl; - logging::Message::Args args1; - logging::Message message2(2); - args1.add(oss.str()); - message2.format( args1 ); - ml.logInfoMessage( message2 ); - dbrm->invalidateUncommittedExtentLBIDs(tableLocks[i].ownerTxnID, false); - uint32_t sessionid = 0; - txnId.id = tableLocks[i].ownerTxnID; - txnId.valid = true; - rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg); - - if ( rc == 0 ) - { - ostringstream oss; - oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and is releasing table lock id " << tableLocks[i].id; - logging::Message::Args args3; - logging::Message message3(2); - args3.add(oss.str()); - message3.format( args3 ); - ml.logInfoMessage( message3 ); - - //@Bug 5285 get the process name to see whether calling bulkrollback is necessary. - string::size_type namelen = tableLocks[i].ownerName.find_first_of(" "); - - if (namelen != string::npos) - { - rollbackProcessor.rollBackBatchAutoOnTransaction(uniqueId, txnId, sessionid, tableLocks[i].tableOID, errorMsg); - } - - logging::logCommand(0, tableLocks[i].ownerTxnID, "ROLLBACK;"); - - //release table lock if not DDLProc - if (tableLocks[i].ownerName == "DDLProc") - continue; - - bool lockReleased = true; - - try - { - lockReleased = dbrm->releaseTableLock(tableLocks[i].id); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (lockReleased) - { - sessionManager.rolledback(txnId); - ostringstream oss; - oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is released."; - //cout << oss.str() << endl; - logging::Message::Args args2; - logging::Message message2(2); - args2.add(oss.str()); - message2.format( args2 ); - ml.logInfoMessage( message2 ); - } - else - { - ostringstream oss; - oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is not released."; - //cout << oss.str() << endl; - logging::Message::Args args2; - logging::Message message2(2); - args2.add(oss.str()); - message2.format( args2 ); - ml.logInfoMessage( message2 ); - } - } - else - { - //@Bug 4524 still need to set readonly as transaction information is lost during system restart. - ostringstream oss; - oss << " problem with rollback transaction " << tableLocks[i].ownerTxnID << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; - rc = dbrm->setReadOnly(true); - - //Log to critical log - logging::Message::Args args6; - logging::Message message6(2); - args6.add(oss.str()); - message6.format( args6 ); - ml.logCriticalMessage( message6 ); - errorTxn = tableLocks[i].ownerTxnID; - } - } - else if (tableLocks[i].ownerName == "WriteEngineServer") //redistribution - { - //Just release the table lock - bool lockReleased = true; - - try - { - lockReleased = dbrm->releaseTableLock(tableLocks[i].id); - } - catch (std::exception& ex) - { - ostringstream oss; - oss << "DMLProc didn't release table lock id " << tableLocks[i].id << " for redistribution due to " << ex.what(); - logging::Message::Args args2; - logging::Message message2(2); - args2.add(oss.str()); - message2.format( args2 ); - ml.logInfoMessage( message2 ); - } - - if (lockReleased) - { - ostringstream oss; - oss << "DMLProc released table lock id " << tableLocks[i].id << " for redistribution."; - logging::Message::Args args2; - logging::Message message2(2); - args2.add(oss.str()); - message2.format( args2 ); - ml.logInfoMessage( message2 ); - } - - } - else //bulkrollback - { - //change owner, still keep transaction id to -1, and session id to -1. - bool lockReleased = true; - - try - { - rollbackProcessor.processBulkRollback(tableLocks[i], dbrm, uniqueId, dbRootPMMap, lockReleased); - ostringstream oss; - oss << "DMLProc started bulkrollback on table OID " << tableLocks[i].tableOID << " and table lock id " << tableLocks[i].id - << " finished and tablelock is released."; - //cout << oss.str() << endl; - logging::Message::Args args4; - logging::Message message4(2); - args4.add(oss.str()); - message4.format( args4 ); - ml.logInfoMessage( message4 ); - } - catch (std::exception& ex) - { - ostringstream oss; - oss << "DMLProc started bulkrollback on table OID " << tableLocks[i].tableOID << " and table lock id " << tableLocks[i].id - << " failed:" << ex.what(); - - if (lockReleased) - oss << " but the tablelock is released due to it is in CLEANUP state"; - else - oss << " and tablelock is not released."; - - //cout << oss.str() << endl; - logging::Message::Args args3; - logging::Message message3(2); - args3.add(oss.str()); - message3.format( args3 ); - ml.logWarningMessage( message3 ); - } - } - } - - // Clear out the session manager session list of sessions / transactions. - //Take care of any transaction left from create table as there is no table lock information - set txnList; - rc = dbrm->getCurrentTxnIDs(txnList); - - if (txnList.size() > 0) - { + if (rc == 0) + { ostringstream oss; - oss << "DMLProc will rollback " << txnList.size() << " transactions."; + oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID + << " and is releasing table lock id " << tableLocks[i].id; + logging::Message::Args args3; + logging::Message message3(2); + args3.add(oss.str()); + message3.format(args3); + ml.logInfoMessage(message3); + + //@Bug 5285 get the process name to see whether calling bulkrollback is necessary. + string::size_type namelen = tableLocks[i].ownerName.find_first_of(" "); + + if (namelen != string::npos) + { + rollbackProcessor.rollBackBatchAutoOnTransaction(uniqueId, txnId, sessionid, tableLocks[i].tableOID, + errorMsg); + } + + logging::logCommand(0, tableLocks[i].ownerTxnID, "ROLLBACK;"); + + // release table lock if not DDLProc + if (tableLocks[i].ownerName == "DDLProc") + continue; + + bool lockReleased = true; + + try + { + lockReleased = dbrm->releaseTableLock(tableLocks[i].id); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + if (lockReleased) + { + sessionManager.rolledback(txnId); + ostringstream oss; + oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and table lock id " + << tableLocks[i].id << " is released."; + // cout << oss.str() << endl; + logging::Message::Args args2; + logging::Message message2(2); + args2.add(oss.str()); + message2.format(args2); + ml.logInfoMessage(message2); + } + else + { + ostringstream oss; + oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and table lock id " + << tableLocks[i].id << " is not released."; + // cout << oss.str() << endl; + logging::Message::Args args2; + logging::Message message2(2); + args2.add(oss.str()); + message2.format(args2); + ml.logInfoMessage(message2); + } + } + else + { + //@Bug 4524 still need to set readonly as transaction information is lost during system restart. + ostringstream oss; + oss << " problem with rollback transaction " << tableLocks[i].ownerTxnID + << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; + rc = dbrm->setReadOnly(true); + + // Log to critical log + logging::Message::Args args6; + logging::Message message6(2); + args6.add(oss.str()); + message6.format(args6); + ml.logCriticalMessage(message6); + errorTxn = tableLocks[i].ownerTxnID; + } + } + else if (tableLocks[i].ownerName == "WriteEngineServer") // redistribution + { + // Just release the table lock + bool lockReleased = true; + + try + { + lockReleased = dbrm->releaseTableLock(tableLocks[i].id); + } + catch (std::exception& ex) + { + ostringstream oss; + oss << "DMLProc didn't release table lock id " << tableLocks[i].id << " for redistribution due to " + << ex.what(); logging::Message::Args args2; logging::Message message2(2); args2.add(oss.str()); - message2.format( args2 ); - ml.logInfoMessage( message2 ); - set::const_iterator curTxnID; + message2.format(args2); + ml.logInfoMessage(message2); + } - for (curTxnID = txnList.begin(); curTxnID != txnList.end(); ++curTxnID) - { - dbrm->invalidateUncommittedExtentLBIDs(*curTxnID, false); - txnId.id = *curTxnID; - txnId.valid = true; - uint32_t sessionid = 0; - ostringstream oss; - oss << "DMLProc will roll back transaction " << *curTxnID; - logging::Message::Args args2; - logging::Message message2(2); - args2.add(oss.str()); - message2.format( args2 ); - ml.logInfoMessage( message2 ); - rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg); - - if ( rc == 0 ) - { - logging::logCommand(0, *curTxnID, "ROLLBACK;"); - ostringstream oss; - - oss << " DMLProc rolled back transaction " << *curTxnID; - //Log to warning log - logging::Message::Args args6; - logging::Message message6(2); - args6.add(oss.str()); - message6.format( args6 ); - ml.logInfoMessage( message6 ); - } - else - { - //@Bug 4524 still need to set readonly as transaction information is lost during system restart. - ostringstream oss; - oss << " problem with rollback transaction " << txnId.id << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; - rc = dbrm->setReadOnly(true); - - //Log to critical log - logging::Message::Args args6; - logging::Message message6(2); - args6.add(oss.str()); - message6.format( args6 ); - ml.logCriticalMessage( message6 ); - errorTxn = *curTxnID; - } - } + if (lockReleased) + { + ostringstream oss; + oss << "DMLProc released table lock id " << tableLocks[i].id << " for redistribution."; + logging::Message::Args args2; + logging::Message message2(2); + args2.add(oss.str()); + message2.format(args2); + ml.logInfoMessage(message2); + } } - - int len; - activeTxns = sessionManager.SIDTIDMap(len); - - //@Bug 1627 Don't start DMLProc if either controller or worker node is not up - if ( activeTxns == NULL ) + else // bulkrollback { - std::string err = "DBRM problem is encountered."; - throw std::runtime_error(err); - } + // change owner, still keep transaction id to -1, and session id to -1. + bool lockReleased = true; - for (int i = 0; i < len; i++) + try + { + rollbackProcessor.processBulkRollback(tableLocks[i], dbrm, uniqueId, dbRootPMMap, lockReleased); + ostringstream oss; + oss << "DMLProc started bulkrollback on table OID " << tableLocks[i].tableOID << " and table lock id " + << tableLocks[i].id << " finished and tablelock is released."; + // cout << oss.str() << endl; + logging::Message::Args args4; + logging::Message message4(2); + args4.add(oss.str()); + message4.format(args4); + ml.logInfoMessage(message4); + } + catch (std::exception& ex) + { + ostringstream oss; + oss << "DMLProc started bulkrollback on table OID " << tableLocks[i].tableOID << " and table lock id " + << tableLocks[i].id << " failed:" << ex.what(); + + if (lockReleased) + oss << " but the tablelock is released due to it is in CLEANUP state"; + else + oss << " and tablelock is not released."; + + // cout << oss.str() << endl; + logging::Message::Args args3; + logging::Message message3(2); + args3.add(oss.str()); + message3.format(args3); + ml.logWarningMessage(message3); + } + } + } + + // Clear out the session manager session list of sessions / transactions. + // Take care of any transaction left from create table as there is no table lock information + set txnList; + rc = dbrm->getCurrentTxnIDs(txnList); + + if (txnList.size() > 0) + { + ostringstream oss; + oss << "DMLProc will rollback " << txnList.size() << " transactions."; + logging::Message::Args args2; + logging::Message message2(2); + args2.add(oss.str()); + message2.format(args2); + ml.logInfoMessage(message2); + set::const_iterator curTxnID; + + for (curTxnID = txnList.begin(); curTxnID != txnList.end(); ++curTxnID) { - txnID = activeTxns[i].txnid; + dbrm->invalidateUncommittedExtentLBIDs(*curTxnID, false); + txnId.id = *curTxnID; + txnId.valid = true; + uint32_t sessionid = 0; + ostringstream oss; + oss << "DMLProc will roll back transaction " << *curTxnID; + logging::Message::Args args2; + logging::Message message2(2); + args2.add(oss.str()); + message2.format(args2); + ml.logInfoMessage(message2); + rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg); - if (txnID.id != errorTxn) - { - sessionManager.rolledback(txnID); - oss << " DMLProc released transaction " << txnID.id; - logging::Message::Args args6; - logging::Message message6(2); - args6.add(oss.str()); - message6.format( args6 ); - ml.logInfoMessage( message6 ); - } + if (rc == 0) + { + logging::logCommand(0, *curTxnID, "ROLLBACK;"); + ostringstream oss; + + oss << " DMLProc rolled back transaction " << *curTxnID; + // Log to warning log + logging::Message::Args args6; + logging::Message message6(2); + args6.add(oss.str()); + message6.format(args6); + ml.logInfoMessage(message6); + } + else + { + //@Bug 4524 still need to set readonly as transaction information is lost during system restart. + ostringstream oss; + oss << " problem with rollback transaction " << txnId.id + << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; + rc = dbrm->setReadOnly(true); + + // Log to critical log + logging::Message::Args args6; + logging::Message message6(2); + args6.add(oss.str()); + message6.format(args6); + ml.logCriticalMessage(message6); + errorTxn = *curTxnID; + } } + } - // Clear out the DBRM. + int len; + activeTxns = sessionManager.SIDTIDMap(len); - dbrm->clear(); + //@Bug 1627 Don't start DMLProc if either controller or worker node is not up + if (activeTxns == NULL) + { + std::string err = "DBRM problem is encountered."; + throw std::runtime_error(err); + } - //Flush the cache - cacheutils::flushPrimProcCache(); + for (int i = 0; i < len; i++) + { + txnID = activeTxns[i].txnid; - //Log a message in info.log - logging::Message::Args args1; - logging::Message message1(2); - args1.add("DMLProc finished rollbackAll."); - message1.format( args1 ); - ml.logInfoMessage( message1 ); + if (txnID.id != errorTxn) + { + sessionManager.rolledback(txnID); + oss << " DMLProc released transaction " << txnID.id; + logging::Message::Args args6; + logging::Message message6(2); + args6.add(oss.str()); + message6.format(args6); + ml.logInfoMessage(message6); + } + } - dbrm->setSystemReady(true); + // Clear out the DBRM. + + dbrm->clear(); + + // Flush the cache + cacheutils::flushPrimProcCache(); + + // Log a message in info.log + logging::Message::Args args1; + logging::Message message1(2); + args1.add("DMLProc finished rollbackAll."); + message1.format(args1); + ml.logInfoMessage(message1); + + dbrm->setSystemReady(true); } int8_t setupCwd() { - string workdir = startup::StartUp::tmpDir(); + string workdir = startup::StartUp::tmpDir(); - if (workdir.length() == 0) - workdir = "."; + if (workdir.length() == 0) + workdir = "."; - int8_t rc = chdir(workdir.c_str()); + int8_t rc = chdir(workdir.c_str()); - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); - return rc; + return rc; } -} // Namewspace - +} // namespace void ServiceDMLProc::setupChildSignalHandlers() { #ifndef _MSC_VER - /* set up some signal handlers */ - struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = added_a_pm; - sigaction(SIGHUP, &ign, 0); - ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); + /* set up some signal handlers */ + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = added_a_pm; + sigaction(SIGHUP, &ign, 0); + ign.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif } - int ServiceDMLProc::Child() { - BRM::DBRM dbrm; - Oam oam; + BRM::DBRM dbrm; + Oam oam; - Config* cf = Config::makeConfig(); + Config* cf = Config::makeConfig(); - if ( setupCwd() ) - { - LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("DMLProc couldn't cwd."); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, msg, logid); - NotifyServiceInitializationFailed(); - return 1; - } + if (setupCwd()) + { + LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("DMLProc couldn't cwd."); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, msg, logid); + NotifyServiceInitializationFailed(); + return 1; + } - - WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DMLPROC ); + WriteEngine::WriteEngineWrapper::init(WriteEngine::SUBSYSTEM_ID_DMLPROC); #ifdef _MSC_VER - // In windows, initializing the wrapper (A dll) does not set the static variables - // in the main program - idbdatafile::IDBPolicy::configIDBPolicy(); + // In windows, initializing the wrapper (A dll) does not set the static variables + // in the main program + idbdatafile::IDBPolicy::configIDBPolicy(); #endif - //@Bug 1627 - try - { - rollbackAll(&dbrm); // Rollback any - } - catch ( std::exception& e ) - { - logging::Message::Args args; - logging::Message message(2); - args.add("DMLProc failed to start due to :"); - args.add(e.what()); - message.format( args ); - logging::LoggingID lid(20); - logging::MessageLog ml(lid); + //@Bug 1627 + try + { + rollbackAll(&dbrm); // Rollback any + } + catch (std::exception& e) + { + logging::Message::Args args; + logging::Message message(2); + args.add("DMLProc failed to start due to :"); + args.add(e.what()); + message.format(args); + logging::LoggingID lid(20); + logging::MessageLog ml(lid); - ml.logCriticalMessage( message ); + ml.logCriticalMessage(message); - cerr << "DMLProc failed to start due to : " << e.what() << endl; - NotifyServiceInitializationFailed(); - return 1; - } + cerr << "DMLProc failed to start due to : " << e.what() << endl; + NotifyServiceInitializationFailed(); + return 1; + } - int temp; - int serverThreads = 10; - int serverQueueSize = 0; - const string DMLProc("DMLProc"); + int temp; + int serverThreads = 10; + int serverQueueSize = 0; + const string DMLProc("DMLProc"); - temp = toInt(cf->getConfig(DMLProc, "ServerThreads")); + temp = toInt(cf->getConfig(DMLProc, "ServerThreads")); - if (temp > 0) - serverThreads = temp; + if (temp > 0) + serverThreads = temp; -// temp = toInt(cf->getConfig(DMLProc, "ServerQueueSize")); -// if (temp > 0) -// serverQueueSize = temp; + // temp = toInt(cf->getConfig(DMLProc, "ServerQueueSize")); + // if (temp > 0) + // serverQueueSize = temp; - //read and cleanup port before trying to use - try - { - string port = cf->getConfig(DMLProc, "Port"); - string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; + // read and cleanup port before trying to use + try + { + string port = cf->getConfig(DMLProc, "Port"); + string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; // Couldn't check the return code b/c // fuser returns 1 for unused port. #if defined(__GNUC__) && __GNUC__ >= 5 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - (void)::system(cmd.c_str()); + (void)::system(cmd.c_str()); #pragma GCC diagnostic pop #else - (void)::system(cmd.c_str()); + (void)::system(cmd.c_str()); #endif + } + catch (...) + { + } + DMLServer dmlserver(serverThreads, serverQueueSize, &dbrm); - } - catch (...) - { - } + NotifyServiceStarted(); - DMLServer dmlserver(serverThreads, serverQueueSize, &dbrm); + ResourceManager* rm = ResourceManager::instance(); - NotifyServiceStarted(); + // jobstepThreadPool is used by other processes. We can't call + // resourcemanaager (rm) functions during the static creation of threadpool + // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). + // From the pools perspective, it has no idea if it is ExeMgr doing the + // creation, so it has no idea which way to set the flag. So we set the max here. + JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); + JobStep::jobstepThreadPool.setName("DMLProcJobList"); - ResourceManager* rm = ResourceManager::instance(); + if (rm->getDMLJlThreadPoolDebug() == "Y" || rm->getDMLJlThreadPoolDebug() == "y") + { + JobStep::jobstepThreadPool.setDebug(true); + JobStep::jobstepThreadPool.invoke(threadpool::ThreadPoolMonitor(&JobStep::jobstepThreadPool)); + DMLServer::fDmlPackagepool.setDebug(true); + DMLServer::fDmlPackagepool.invoke(threadpool::ThreadPoolMonitor(&DMLServer::fDmlPackagepool)); + } - // jobstepThreadPool is used by other processes. We can't call - // resourcemanaager (rm) functions during the static creation of threadpool - // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). - // From the pools perspective, it has no idea if it is ExeMgr doing the - // creation, so it has no idea which way to set the flag. So we set the max here. - JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); - JobStep::jobstepThreadPool.setName("DMLProcJobList"); + Dec = DistributedEngineComm::instance(rm); - if (rm->getDMLJlThreadPoolDebug() == "Y" || rm->getDMLJlThreadPoolDebug() == "y") - { - JobStep::jobstepThreadPool.setDebug(true); - JobStep::jobstepThreadPool.invoke(threadpool::ThreadPoolMonitor(&JobStep::jobstepThreadPool)); - DMLServer::fDmlPackagepool.setDebug(true); - DMLServer::fDmlPackagepool.invoke(threadpool::ThreadPoolMonitor(&DMLServer::fDmlPackagepool)); - } + setupChildSignalHandlers(); - Dec = DistributedEngineComm::instance(rm); + _exit(dmlserver.start()); - setupChildSignalHandlers(); - - _exit(dmlserver.start()); - - return 1; + return 1; } - int main(int argc, char** argv) { - Opt opt(argc, argv); + Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - // This is unset due to the way we start it - program_invocation_short_name = const_cast("DMLProc"); - // Initialize the charset library - MY_INIT(argv[0]); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DMLProc"); + // Initialize the charset library + MY_INIT(argv[0]); - return ServiceDMLProc(opt).Run(); + return ServiceDMLProc(opt).Run(); } // vim:ts=4 sw=4: - diff --git a/dmlproc/dmlproc.h b/dmlproc/dmlproc.h index fa76dc700..433a9d319 100644 --- a/dmlproc/dmlproc.h +++ b/dmlproc/dmlproc.h @@ -16,22 +16,22 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* -* $Id: dmlproc.h 927 2013-01-21 14:11:25Z rdempsey $ -* -******************************************************************************************/ + * + * $Id: dmlproc.h 927 2013-01-21 14:11:25Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ #ifndef DMLPROC_H #define DMLPROC_H -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include "messagelog.h" -#endif //DMLPROC_H +#endif // DMLPROC_H diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 4875a15d8..33e9b3d14 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlprocessor.cpp 1024 2013-07-26 16:23:59Z chao $ -* -* -***********************************************************************/ + * $Id: dmlprocessor.cpp 1024 2013-07-26 16:23:59Z chao $ + * + * + ***********************************************************************/ /** @file */ #include "configcpp.h" #include @@ -63,7 +63,7 @@ using namespace querytele; extern boost::mutex mute; extern boost::condition_variable cond; -#define MCOL_140 // Undefine to test VSS for out of order transactions +#define MCOL_140 // Undefine to test VSS for out of order transactions namespace { @@ -77,7 +77,7 @@ namespace dmlprocessor DMLProcessor::PackageHandlerMap_t DMLProcessor::packageHandlerMap; boost::mutex DMLProcessor::packageHandlerMapLock; -//Map to store the BatchInsertProc object +// Map to store the BatchInsertProc object std::map DMLProcessor::batchinsertProcessorMap; boost::mutex DMLProcessor::batchinsertProcessorMapLock; @@ -98,240 +98,243 @@ boost::mutex PackageHandler::tableOidMutex; // If FORCE is set, we can't rollback. struct CancellationThread { - CancellationThread(DBRM* aDbrm, DMLServer& aServer) : fDbrm(aDbrm), fServer(aServer) - {} - void operator()() + CancellationThread(DBRM* aDbrm, DMLServer& aServer) : fDbrm(aDbrm), fServer(aServer) + { + } + void operator()() + { + bool bDoingRollback = false; + bool bRollback = false; + bool bForce = false; + ostringstream oss; + std::vector tableLocks; + BRM::TxnID txnId; + DMLProcessor::PackageHandlerMap_t::iterator phIter; + uint32_t sessionID; + int rc = 0; + + while (true) { - bool bDoingRollback = false; - bool bRollback = false; - bool bForce = false; + usleep(1000000); // 1 seconds + // Check to see if someone has ordered a shutdown or suspend with rollback. + (void)fDbrm->getSystemShutdownPending(bRollback, bForce); + + if (bForce) + break; + + if (bDoingRollback && bRollback) + { + continue; + // We've already started the rollbacks. Don't start again. + } + + bDoingRollback = false; + + if (bRollback) + { + RollbackTransactionProcessor rollbackProcessor(fDbrm); + SessionManager sessionManager; + uint64_t uniqueId = fDbrm->getUnique64(); + std::string errorMsg; + int activeTransCount = 0; + int idleTransCount = 0; + bDoingRollback = true; ostringstream oss; - std::vector tableLocks; - BRM::TxnID txnId; - DMLProcessor::PackageHandlerMap_t::iterator phIter; - uint32_t sessionID; - int rc = 0; + oss << "DMLProc has been told to rollback all DML transactions."; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); + // Tell any active processors to stop working and return an error + // The front end will respond with a ROLLBACK command. + // Mark all active processors to rollback + boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock); - while (true) + for (phIter = DMLProcessor::packageHandlerMap.begin(); + phIter != DMLProcessor::packageHandlerMap.end(); ++phIter) { - usleep(1000000); // 1 seconds - // Check to see if someone has ordered a shutdown or suspend with rollback. - (void)fDbrm->getSystemShutdownPending(bRollback, bForce); + ostringstream oss; + oss << "DMLProc will rollback active session " << phIter->second->getSessionID() << " Transaction " + << phIter->second->getTxnid(); + DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); - if (bForce) - break; - - if (bDoingRollback && bRollback) - { - continue; - // We've already started the rollbacks. Don't start again. - } - - bDoingRollback = false; - - if (bRollback) - { - RollbackTransactionProcessor rollbackProcessor(fDbrm); - SessionManager sessionManager; - uint64_t uniqueId = fDbrm->getUnique64(); - std::string errorMsg; - int activeTransCount = 0; - int idleTransCount = 0; - bDoingRollback = true; - ostringstream oss; - oss << "DMLProc has been told to rollback all DML transactions."; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); - // Tell any active processors to stop working and return an error - // The front end will respond with a ROLLBACK command. - // Mark all active processors to rollback - boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock); - - for (phIter = DMLProcessor::packageHandlerMap.begin(); - phIter != DMLProcessor::packageHandlerMap.end(); - ++phIter) - { - ostringstream oss; - oss << "DMLProc will rollback active session " << phIter->second->getSessionID() << " Transaction " << phIter->second->getTxnid(); - DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); - - ++activeTransCount; - phIter->second->rollbackPending(); - } - - if (activeTransCount > 0) - { - ostringstream oss1; - oss1 << "DMLProc is rolling back back " << activeTransCount << " active transactions."; - DMLProcessor::log(oss1.str(), logging::LOG_TYPE_INFO); - } - - // WIP Need to set cluster to read-only via CMAPI before shutting the cluster down. - if (fDbrm->isReadWrite()) - { - continue; - } - - // Check for any open DML transactions that don't currently have - // a processor - tableLocks = fDbrm->getAllTableLocks(); - - if (tableLocks.size() > 0) - { - for (uint32_t i = 0; i < tableLocks.size(); ++i) - { - sessionID = tableLocks[i].ownerSessionID; - phIter = DMLProcessor::packageHandlerMap.find(sessionID); - - if (phIter == DMLProcessor::packageHandlerMap.end()) - { - // We have found an active transaction without a packagehandler. - // This means that a transaction is open with autocommit turned - // off, but there's no current activity on the transaction. We - // need to roll it back if it's a DML transaction. - // If ownerName == "DMLProc" then it's a DML transaction. - if (tableLocks[i].ownerName == "DMLProc") - { - // OK, we know this is an idle DML transaction, so roll it back. - ++idleTransCount; - txnId.id = tableLocks[i].ownerTxnID; - txnId.valid = true; - rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionID, errorMsg); - - if ( rc == 0 ) - { - fDbrm->invalidateUncommittedExtentLBIDs(txnId.id, false); - - //@Bug 4524. In case it is batchinsert, call bulkrollback. - rc = rollbackProcessor.rollBackBatchAutoOnTransaction(uniqueId, txnId, sessionID, tableLocks[i].tableOID, errorMsg); - - if (rc == 0) - { - logging::logCommand(0, tableLocks[i].ownerTxnID, "ROLLBACK;"); - - bool lockReleased = true; - - try - { - lockReleased = fDbrm->releaseTableLock(tableLocks[i].id); - TablelockData::removeTablelockData(sessionID); - } - catch (std::exception&) - { - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } - - if (lockReleased) - { - sessionManager.rolledback(txnId); - ostringstream oss; - oss << "DMLProc rolled back idle transaction " << tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is released."; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); - } - else - { - ostringstream oss; - oss << "DMLProc rolled back idle transaction " << tableLocks[i].ownerTxnID << " and tble lock id " << tableLocks[i].id << " is not released."; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); - } - } - else - { - ostringstream oss; - oss << " problem with bulk rollback of idle transaction " << tableLocks[i].ownerTxnID << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_CRITICAL); - rc = fDbrm->setReadOnly(true); - } - } - else - { - ostringstream oss; - oss << " problem with rollback of idle transaction " << tableLocks[i].ownerTxnID << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_CRITICAL); - rc = fDbrm->setReadOnly(true); - } - } - } - } - } - - // If there are any abandonded transactions without locks - // release them. - int len; - boost::shared_array activeTxns = sessionManager.SIDTIDMap(len); - - for (int i = 0; i < len; i++) - { - // If there isn't a table lock for this transaction, roll it back. Otherwise, assume - // it has an active processor or is not DML initiated and leave it alone. It's someone - // else's concern. - bool bFoundit = false; - - for (uint32_t j = 0; j < tableLocks.size(); ++j) - { - if (tableLocks[j].ownerTxnID == activeTxns[i].txnid.id) - { - bFoundit = true; - break; - } - } - - if (!bFoundit && activeTxns[i].txnid.valid) - { - rollbackProcessor.rollBackTransaction(uniqueId, activeTxns[i].txnid, activeTxns[i].sessionid, errorMsg); - sessionManager.rolledback(activeTxns[i].txnid); - ++idleTransCount; - ostringstream oss; - oss << "DMLProc rolled back idle transaction with no tablelock" << tableLocks[i].ownerTxnID; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); - } - } - - if (idleTransCount > 0) - { - ostringstream oss2; - oss2 << "DMLProc has rolled back " << idleTransCount << " idle transactions."; - DMLProcessor::log(oss2.str(), logging::LOG_TYPE_INFO); - } - // Here is the end of the rollback if so DMLProc rollbacks what it can. - break; - } + ++activeTransCount; + phIter->second->rollbackPending(); } - // Setting the flag to tell DMLServer to exit. - fServer.startShutdown(); + + if (activeTransCount > 0) + { + ostringstream oss1; + oss1 << "DMLProc is rolling back back " << activeTransCount << " active transactions."; + DMLProcessor::log(oss1.str(), logging::LOG_TYPE_INFO); + } + + // WIP Need to set cluster to read-only via CMAPI before shutting the cluster down. + if (fDbrm->isReadWrite()) + { + continue; + } + + // Check for any open DML transactions that don't currently have + // a processor + tableLocks = fDbrm->getAllTableLocks(); + + if (tableLocks.size() > 0) + { + for (uint32_t i = 0; i < tableLocks.size(); ++i) + { + sessionID = tableLocks[i].ownerSessionID; + phIter = DMLProcessor::packageHandlerMap.find(sessionID); + + if (phIter == DMLProcessor::packageHandlerMap.end()) + { + // We have found an active transaction without a packagehandler. + // This means that a transaction is open with autocommit turned + // off, but there's no current activity on the transaction. We + // need to roll it back if it's a DML transaction. + // If ownerName == "DMLProc" then it's a DML transaction. + if (tableLocks[i].ownerName == "DMLProc") + { + // OK, we know this is an idle DML transaction, so roll it back. + ++idleTransCount; + txnId.id = tableLocks[i].ownerTxnID; + txnId.valid = true; + rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionID, errorMsg); + + if (rc == 0) + { + fDbrm->invalidateUncommittedExtentLBIDs(txnId.id, false); + + //@Bug 4524. In case it is batchinsert, call bulkrollback. + rc = rollbackProcessor.rollBackBatchAutoOnTransaction(uniqueId, txnId, sessionID, + tableLocks[i].tableOID, errorMsg); + + if (rc == 0) + { + logging::logCommand(0, tableLocks[i].ownerTxnID, "ROLLBACK;"); + + bool lockReleased = true; + + try + { + lockReleased = fDbrm->releaseTableLock(tableLocks[i].id); + TablelockData::removeTablelockData(sessionID); + } + catch (std::exception&) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } + + if (lockReleased) + { + sessionManager.rolledback(txnId); + ostringstream oss; + oss << "DMLProc rolled back idle transaction " << tableLocks[i].ownerTxnID + << " and table lock id " << tableLocks[i].id << " is released."; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); + } + else + { + ostringstream oss; + oss << "DMLProc rolled back idle transaction " << tableLocks[i].ownerTxnID + << " and tble lock id " << tableLocks[i].id << " is not released."; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); + } + } + else + { + ostringstream oss; + oss << " problem with bulk rollback of idle transaction " << tableLocks[i].ownerTxnID + << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_CRITICAL); + rc = fDbrm->setReadOnly(true); + } + } + else + { + ostringstream oss; + oss << " problem with rollback of idle transaction " << tableLocks[i].ownerTxnID + << "and DBRM is setting to readonly and table lock is not released: " << errorMsg; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_CRITICAL); + rc = fDbrm->setReadOnly(true); + } + } + } + } + } + + // If there are any abandonded transactions without locks + // release them. + int len; + boost::shared_array activeTxns = sessionManager.SIDTIDMap(len); + + for (int i = 0; i < len; i++) + { + // If there isn't a table lock for this transaction, roll it back. Otherwise, assume + // it has an active processor or is not DML initiated and leave it alone. It's someone + // else's concern. + bool bFoundit = false; + + for (uint32_t j = 0; j < tableLocks.size(); ++j) + { + if (tableLocks[j].ownerTxnID == activeTxns[i].txnid.id) + { + bFoundit = true; + break; + } + } + + if (!bFoundit && activeTxns[i].txnid.valid) + { + rollbackProcessor.rollBackTransaction(uniqueId, activeTxns[i].txnid, activeTxns[i].sessionid, + errorMsg); + sessionManager.rolledback(activeTxns[i].txnid); + ++idleTransCount; + ostringstream oss; + oss << "DMLProc rolled back idle transaction with no tablelock" << tableLocks[i].ownerTxnID; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_INFO); + } + } + + if (idleTransCount > 0) + { + ostringstream oss2; + oss2 << "DMLProc has rolled back " << idleTransCount << " idle transactions."; + DMLProcessor::log(oss2.str(), logging::LOG_TYPE_INFO); + } + // Here is the end of the rollback if so DMLProc rollbacks what it can. + break; + } } - DBRM* fDbrm; - DMLServer& fServer; + // Setting the flag to tell DMLServer to exit. + fServer.startShutdown(); + } + DBRM* fDbrm; + DMLServer& fServer; }; PackageHandler::PackageHandler(const messageqcpp::IOSocket& ios, - boost::shared_ptr bs, - uint8_t packageType, - joblist::DistributedEngineComm* ec, - bool concurrentSupport, - uint64_t maxDeleteRows, - uint32_t sessionID, - execplan::CalpontSystemCatalog::SCN txnId, - DBRM* aDbrm, + boost::shared_ptr bs, uint8_t packageType, + joblist::DistributedEngineComm* ec, bool concurrentSupport, + uint64_t maxDeleteRows, uint32_t sessionID, + execplan::CalpontSystemCatalog::SCN txnId, DBRM* aDbrm, const QueryTeleClient& qtc, - boost::shared_ptr csc) : - fIos(ios), - fByteStream(bs), - fPackageType(packageType), - fEC(ec), - fConcurrentSupport(concurrentSupport), - fMaxDeleteRows(maxDeleteRows), - fSessionID(sessionID), - fTableOid(0), - fTxnid(txnId), - fDbrm(aDbrm), - fQtc(qtc), - fcsc(csc) + boost::shared_ptr csc) + : fIos(ios) + , fByteStream(bs) + , fPackageType(packageType) + , fEC(ec) + , fConcurrentSupport(concurrentSupport) + , fMaxDeleteRows(maxDeleteRows) + , fSessionID(sessionID) + , fTableOid(0) + , fTxnid(txnId) + , fDbrm(aDbrm) + , fQtc(qtc) + , fcsc(csc) { } PackageHandler::~PackageHandler() { - //cout << "In destructor" << endl; + // cout << "In destructor" << endl; } // MCOL-140 @@ -360,1382 +363,1405 @@ PackageHandler::~PackageHandler() // A tranasaction for one fTableOid is not blocked by a txn for a different fTableOid. int PackageHandler::synchTableAccess(dmlpackage::CalpontDMLPackage* dmlPackage) { - // MCOL-140 Wait for any other DML using this table. - std::map::iterator it; - boost::unique_lock lock(tableOidMutex); - BRM::TxnID txnid; + // MCOL-140 Wait for any other DML using this table. + std::map::iterator it; + boost::unique_lock lock(tableOidMutex); + BRM::TxnID txnid; - if (fPackageType != dmlpackage::DML_COMMAND) + if (fPackageType != dmlpackage::DML_COMMAND) + { + txnid = sessionManager.getTxnID(fSessionID); + + if (!txnid.valid) { - txnid = sessionManager.getTxnID(fSessionID); + txnid = sessionManager.newTxnID(fSessionID, true); - if ( !txnid.valid ) - { - txnid = sessionManager.newTxnID(fSessionID, true); - - if (!txnid.valid) - { - throw std::runtime_error( std::string("Unable to start a transaction. Check critical log.") ); - } - } + if (!txnid.valid) + { + throw std::runtime_error(std::string("Unable to start a transaction. Check critical log.")); + } } - else + } + else + { + txnid = sessionManager.getTxnID(fSessionID); + } + + fTxnid = txnid.id; + + if ((it = tableOidMap.find(fTableOid)) != tableOidMap.end()) + { + PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; + + // There's at least one working txn on this table. We may be the same txn. + if (fTxnid == tableOidQueue.front()) { - txnid = sessionManager.getTxnID(fSessionID); + return 1; // We're next in line or the same as the last. Keep working } - fTxnid = txnid.id; + tableOidQueue.push(fTxnid); // Get on the waiting list. - if ((it = tableOidMap.find(fTableOid)) != tableOidMap.end()) + // We need to wait + // tableOidQueue here is the queue holding the waitng transactions for this fTableOid + while (true) { - PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; + // Log something that we're waiting + LoggingID logid(21, fSessionID, fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + ostringstream oss; + oss << "Txn is waiting for" << tableOidQueue.front() << " " << dmlPackage->get_SQLStatement() << "; |" + << dmlPackage->get_SchemaName() << "|"; + args1.add(oss.str()); + args1.add((uint64_t)fTableOid); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - // There's at least one working txn on this table. We may be the same txn. - if (fTxnid == tableOidQueue.front()) - { - return 1; // We're next in line or the same as the last. Keep working - } + tableOidCond.wait(lock); + // In case of CTRL+C, the tableOidQueue could be invalidated + if ((tableOidMap.find(fTableOid))->second != tableOidQueue) + { + break; + } + if (tableOidQueue.front() == fTxnid) + { + // We're up next. Let's go do stuff. + break; + } - tableOidQueue.push(fTxnid); // Get on the waiting list. + if (tableOidQueue.empty()) + { + // If we had been the last txn waiting and CTRL+C was hit, then the queue is empty now. + // Empty queues must be erased from the map. + tableOidMap.erase(fTableOid); + break; + } - // We need to wait - // tableOidQueue here is the queue holding the waitng transactions for this fTableOid - while (true) - { - // Log something that we're waiting - LoggingID logid(21, fSessionID, fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - ostringstream oss; - oss << "Txn is waiting for" << tableOidQueue.front() << " " << dmlPackage->get_SQLStatement() << "; |" << dmlPackage->get_SchemaName() <<"|"; - args1.add(oss.str()); - args1.add((uint64_t)fTableOid); - msg.format(args1); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + // If we're not in the queue at all, then continue. CTRL+C was probably hit. + PackageHandler::tableAccessQueue_t::container_type::iterator c_it = tableOidQueue.find(fTxnid); - tableOidCond.wait(lock); - // In case of CTRL+C, the tableOidQueue could be invalidated - if ((tableOidMap.find(fTableOid))->second != tableOidQueue) - { - break; - } - if (tableOidQueue.front() == fTxnid) - { - // We're up next. Let's go do stuff. - break; - } + if (c_it == tableOidQueue.end()) + { + break; + } - if (tableOidQueue.empty()) - { - // If we had been the last txn waiting and CTRL+C was hit, then the queue is empty now. - // Empty queues must be erased from the map. - tableOidMap.erase(fTableOid); - break; - } - - // If we're not in the queue at all, then continue. CTRL+C was probably hit. - PackageHandler::tableAccessQueue_t::container_type::iterator c_it = tableOidQueue.find(fTxnid); - - if (c_it == tableOidQueue.end()) - { - break; - } - - // We're still in the queue and not on top. Go back and wait some more. - } - } - else - { - // We're the first for this tableoid. Start a new queue. - tableAccessQueue_t tableOidQueue; - tableOidQueue.push(fTxnid); - tableOidMap[fTableOid] = tableOidQueue; + // We're still in the queue and not on top. Go back and wait some more. } + } + else + { + // We're the first for this tableoid. Start a new queue. + tableAccessQueue_t tableOidQueue; + tableOidQueue.push(fTxnid); + tableOidMap[fTableOid] = tableOidQueue; + } - return 1; + return 1; } // MCOL-140 Called when it's time to release the next thread for this tablOid int PackageHandler::releaseTableAccess() { - // take us out of the queue - std::map::iterator it; - boost::lock_guard lock(tableOidMutex); + // take us out of the queue + std::map::iterator it; + boost::lock_guard lock(tableOidMutex); - if (fTableOid == 0 || (it = tableOidMap.find(fTableOid)) == tableOidMap.end()) + if (fTableOid == 0 || (it = tableOidMap.find(fTableOid)) == tableOidMap.end()) + { + return 2; // For now, return codes are not used + } + + PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; + + if (tableOidQueue.front() != fTxnid) + { + // This is a severe error. The front should be the working thread. If we're here, + // we're the working thread and should be front(). + cout << fTxnid << " " << fTableOid << " We got to release and we're not on top " << tableOidQueue.front() + << endl; + LoggingID logid(21, fSessionID, fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add( + "ReleaseTableAccess: Txn being released is not the current txn in the tablOidQueue for tableid"); + args1.add((uint64_t)fTableOid); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_ERROR, msg, logid); + } + else + { + if (!tableOidQueue.empty()) + tableOidQueue.pop(); // Get off the waiting list. + + if (tableOidQueue.empty()) { - return 2; // For now, return codes are not used + // remove the queue from the map. + tableOidMap.erase(fTableOid); } + } - PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; - - if (tableOidQueue.front() != fTxnid) - { - // This is a severe error. The front should be the working thread. If we're here, - // we're the working thread and should be front(). - cout << fTxnid << " " << fTableOid << " We got to release and we're not on top " << tableOidQueue.front() << endl; - LoggingID logid(21, fSessionID, fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("ReleaseTableAccess: Txn being released is not the current txn in the tablOidQueue for tableid"); - args1.add((uint64_t)fTableOid); - msg.format(args1); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_ERROR, msg, logid); - } - else - { - if (!tableOidQueue.empty()) - tableOidQueue.pop(); // Get off the waiting list. - - if (tableOidQueue.empty()) - { - // remove the queue from the map. - tableOidMap.erase(fTableOid); - } - } - - // release the condition - tableOidCond.notify_all(); - return 1; + // release the condition + tableOidCond.notify_all(); + return 1; } int PackageHandler::forceReleaseTableAccess() { - // By removing the txnid from the queue, the logic after the wait in - // synchTableAccess() will release the thread and clean up if needed. - std::map::iterator it; - boost::lock_guard lock(tableOidMutex); + // By removing the txnid from the queue, the logic after the wait in + // synchTableAccess() will release the thread and clean up if needed. + std::map::iterator it; + boost::lock_guard lock(tableOidMutex); - if (fTableOid == 0 || (it = tableOidMap.find(fTableOid)) == tableOidMap.end()) - { - // This will happen for DML_COMMAND, as we never got the tableoid or called synchTableAccess - return 2; - } + if (fTableOid == 0 || (it = tableOidMap.find(fTableOid)) == tableOidMap.end()) + { + // This will happen for DML_COMMAND, as we never got the tableoid or called synchTableAccess + return 2; + } - PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; - tableOidQueue.erase(fTxnid); - if (tableOidQueue.empty()) - { - // remove the queue from the map. - tableOidMap.erase(fTableOid); - } - // release the condition - tableOidCond.notify_all(); - return 1; + PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; + tableOidQueue.erase(fTxnid); + if (tableOidQueue.empty()) + { + // remove the queue from the map. + tableOidMap.erase(fTableOid); + } + // release the condition + tableOidCond.notify_all(); + return 1; } -//static +// static // Called upon sighup, often because PrimProc crashed. We don't want to leave all the transactions hung, // though some may be because they never returned from PrimProc and will leave the table lock on. int PackageHandler::clearTableAccess() { - tableOidMap.clear(); - return 1; + tableOidMap.clear(); + return 1; } void PackageHandler::run() { - ResourceManager* frm = ResourceManager::instance(); - dmlpackageprocessor::DMLPackageProcessor::DMLResult result; - result.result = dmlpackageprocessor::DMLPackageProcessor::NO_ERROR; - //cout << "PackageHandler handling "; - std::string stmt; - unsigned DMLLoggingId = 21; - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - SynchTable synchTable; + ResourceManager* frm = ResourceManager::instance(); + dmlpackageprocessor::DMLPackageProcessor::DMLResult result; + result.result = dmlpackageprocessor::DMLPackageProcessor::NO_ERROR; + // cout << "PackageHandler handling "; + std::string stmt; + unsigned DMLLoggingId = 21; + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + SynchTable synchTable; - try + try + { + switch (fPackageType) { - switch ( fPackageType ) - { - case dmlpackage::DML_INSERT: - { - // build an InsertDMLPackage from the bytestream - //cout << "an INSERT package" << endl; - dmlpackage::InsertDMLPackage insertPkg; - //boost::shared_ptr insertBs (new messageqcpp::ByteStream); - messageqcpp::ByteStream bsSave = *(fByteStream.get()); - insertPkg.readMetaData(*(fByteStream.get())); + case dmlpackage::DML_INSERT: + { + // build an InsertDMLPackage from the bytestream + // cout << "an INSERT package" << endl; + dmlpackage::InsertDMLPackage insertPkg; + // boost::shared_ptr insertBs (new messageqcpp::ByteStream); + messageqcpp::ByteStream bsSave = *(fByteStream.get()); + insertPkg.readMetaData(*(fByteStream.get())); #ifdef MCOL_140 - if (fConcurrentSupport) - { - fTableOid = insertPkg.getTableOid(); + if (fConcurrentSupport) + { + fTableOid = insertPkg.getTableOid(); - // Single Insert has no start like bulk does, so insertPkg.getTableOid() - // isn't set. Go get it now. - if (fTableOid == 0) - { - CalpontSystemCatalog::TableName tableName; - tableName.schema = insertPkg.get_Table()->get_SchemaName(); - tableName.table = insertPkg.get_Table()->get_TableName(); - CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); - fTableOid = roPair.objnum; - } - synchTable.setPackage(this, &insertPkg); // Blocks if another DML thread is using this fTableOid - } + // Single Insert has no start like bulk does, so insertPkg.getTableOid() + // isn't set. Go get it now. + if (fTableOid == 0) + { + CalpontSystemCatalog::TableName tableName; + tableName.schema = insertPkg.get_Table()->get_SchemaName(); + tableName.table = insertPkg.get_Table()->get_TableName(); + CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); + fTableOid = roPair.objnum; + } + synchTable.setPackage(this, &insertPkg); // Blocks if another DML thread is using this fTableOid + } #endif - QueryTeleStats qts; - qts.query_uuid = QueryTeleClient::genUUID(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = fSessionID; - qts.query_type = "INSERT"; - qts.query = insertPkg.get_SQLStatement(); - qts.system_name = oamCache->getSystemName(); - qts.module_name = oamCache->getModuleName(); - qts.schema_name = insertPkg.get_SchemaName(); - fQtc.postQueryTele(qts); + QueryTeleStats qts; + qts.query_uuid = QueryTeleClient::genUUID(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = fSessionID; + qts.query_type = "INSERT"; + qts.query = insertPkg.get_SQLStatement(); + qts.system_name = oamCache->getSystemName(); + qts.module_name = oamCache->getModuleName(); + qts.schema_name = insertPkg.get_SchemaName(); + fQtc.postQueryTele(qts); - //cout << "This is batch insert " << insertPkg->get_isBatchInsert() << endl; - if (insertPkg.get_isBatchInsert()) - { - fByteStream->reset(); - //cout << "This is batch insert " << endl; - BatchInsertProc* batchProcessor = NULL; - { - boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); + // cout << "This is batch insert " << insertPkg->get_isBatchInsert() << endl; + if (insertPkg.get_isBatchInsert()) + { + fByteStream->reset(); + // cout << "This is batch insert " << endl; + BatchInsertProc* batchProcessor = NULL; + { + boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); - std::map::iterator batchIter = DMLProcessor::batchinsertProcessorMap.find(fSessionID); + std::map::iterator batchIter = + DMLProcessor::batchinsertProcessorMap.find(fSessionID); - if (batchIter == DMLProcessor::batchinsertProcessorMap.end()) - { - batchProcessor = new BatchInsertProc(insertPkg.get_isAutocommitOn(), insertPkg.getTableOid(), fTxnid, fDbrm); - DMLProcessor::batchinsertProcessorMap[fSessionID] = batchProcessor; - //cout << "batchProcessor is created " << batchProcessor << endl; - } - else - { - batchProcessor = batchIter->second; - //cout << "Found batchProcessor " << batchProcessor << endl; - } - } - - if ( insertPkg.get_Logging() ) - { - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Start SQL statement: "); - - if (!insertPkg.get_isCacheInsert()) - { - ostringstream oss; - oss << insertPkg.get_SQLStatement() << "; |" << insertPkg.get_SchemaName() << "|"; - args1.add(oss.str()); - } - - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - TablelockData* tablelockData = TablelockData::makeTablelockData(insertPkg.get_SessionID()); - uint64_t tableLockId = tablelockData->getTablelockId(insertPkg.getTableOid()); - - //cout << "Processing table oid " << insertPkg.getTableOid() << " for transaction "<< (int)fTxnid << endl; - if (tableLockId == 0) - { - //cout << "Grabing tablelock for batchProcessor " << batchProcessor << endl; - tableLockId = batchProcessor->grabTableLock(insertPkg.get_SessionID()); - - if (tableLockId == 0) - { - BRM::TxnID brmTxnID; - brmTxnID.id = fTxnid; - brmTxnID.valid = true; - sessionManager.rolledback(brmTxnID); - string errMsg; - int rc = 0; - batchProcessor->getError(rc, errMsg); - result.result = DMLPackageProcessor::TABLE_LOCK_ERROR; - logging::Message::Args args; - logging::Message message(1); - args.add("Insert Failed: "); - args.add(errMsg); - args.add(""); - args.add(""); - message.format(args); - result.message = message; - break; - } - - if (tableLockId > 0) - tablelockData->setTablelock(insertPkg.getTableOid(), tableLockId); - } - } - - if (insertPkg.get_Logending() && insertPkg.get_Logging()) //only one batch need to be processed. - { - //cout << "dmlprocessor add last pkg" << endl; - //need to add error handling. - batchProcessor->addPkg(bsSave); - batchProcessor->sendFirstBatch(); - batchProcessor->receiveOutstandingMsg(); - //@Bug 5162. Get the correct error message before the last message. - string errMsg; - int rc = 0; - batchProcessor->getError(rc, errMsg); - batchProcessor->sendlastBatch(); - batchProcessor->receiveAllMsg(); - - if (rc == DMLPackageProcessor::IDBRANGE_WARNING) - { - result.result = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with warnings"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - logging::Message::Args args; - logging::Message message(1); - args.add(errMsg); - args.add(""); - args.add(""); - message.format(args); - result.message = message; - } - else if ( rc != 0) - { - result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - logging::Message::Args args; - logging::Message message(1); - cout << "Got error in the end of one batchinsert." << endl; - args.add("Insert Failed: "); - args.add(errMsg); - args.add(""); - args.add(""); - message.format(args); - result.message = message; - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with error"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - } - else - { - // if (!insertPkg.get_isAutocommitOn()) - // { - // batchProcessor->setHwm(); - // } - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - } - - //remove the batch insert object - { - boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); - - std::map::iterator batchIter = DMLProcessor::batchinsertProcessorMap.find(fSessionID); - - if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) - { - delete batchIter->second; - DMLProcessor::batchinsertProcessorMap.erase(fSessionID); - } - } - } - else if (insertPkg.get_Logending()) //Last batch - { - int rc = 0; - string errMsg; - batchProcessor->getError(rc, errMsg); - - //cout <<"dmlprocessor received last pkg from mysql rc == " << rc << endl; - if (( rc == 0) || (rc == DMLPackageProcessor::IDBRANGE_WARNING)) - { - //cout << " rc = " << rc << endl; - batchProcessor->addPkg(bsSave); - batchProcessor->sendNextBatch(); - batchProcessor->receiveOutstandingMsg(); - //@Bug 5162. Get the correct error message before the last message. - batchProcessor->getError(rc, errMsg); - batchProcessor->sendlastBatch(); - batchProcessor->receiveAllMsg(); - - if (rc == DMLPackageProcessor::IDBRANGE_WARNING) - { - result.result = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with warnings"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - logging::Message::Args args; - logging::Message message(1); - args.add(errMsg); - args.add(""); - args.add(""); - message.format(args); - result.message = message; - } - else if ( rc != 0) - { - //cout << "Got error in the end of last batchinsert. error message is " << errMsg << endl; - result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - logging::Message::Args args; - logging::Message message(1); - args.add("Insert Failed: "); - args.add(errMsg); - args.add(""); - args.add(""); - message.format(args); - result.message = message; - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with error"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - } - else - { - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - } - - //cout << "finished batch insert" << endl; - } - else - { - //error occured. Receive all outstanding messages before erroring out. - batchProcessor->receiveOutstandingMsg(); - batchProcessor->sendlastBatch(); //needs to flush files - batchProcessor->receiveAllMsg(); - result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - //cout << "Got error in the end of batchinsert2. error msg is " << errMsg<< endl; - logging::Message::Args args; - logging::Message message(1); - args.add("Insert Failed: "); - args.add(errMsg); - args.add(""); - args.add(""); - message.format(args); - result.message = message; - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with error"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", insertPkg.get_SchemaName()); - } - - //remove from map - { - boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); - std::map::iterator batchIter = DMLProcessor::batchinsertProcessorMap.find(fSessionID); - - if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) - { - //cout << "Batchinsertprcessor is deleted. " << batchIter->second << endl; - delete batchIter->second; - DMLProcessor::batchinsertProcessorMap.erase(fSessionID); - } - } - - } - else - { - int rc = 0; - string errMsg; - batchProcessor->getError(rc, errMsg); - - if (rc == DMLPackageProcessor::IDBRANGE_WARNING) - { - result.result = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - } - else if ( rc != 0) - { - result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - //@Bug - //cout << "Got error during batchinsert. with message " << errMsg << endl; - logging::Message::Args args; - logging::Message message(6); - args.add( errMsg ); - message.format( args ); - result.message = message; - batchProcessor->receiveOutstandingMsg(); - batchProcessor->sendlastBatch(); //needs to flush files - //cout << "Last batch is sent to WES." << endl; - batchProcessor->receiveAllMsg(); - LoggingID logid( DMLLoggingId, insertPkg.get_SessionID(), fTxnid); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement with error"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - //remove from map - { - boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); - std::map::iterator batchIter = DMLProcessor::batchinsertProcessorMap.find(fSessionID); - - if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) - { - //cout << "Batchinsertprcessor is deleted. " << batchIter->second << endl; - delete batchIter->second; - DMLProcessor::batchinsertProcessorMap.erase(fSessionID); - } - } - break; - } - - batchProcessor->addPkg(bsSave); - batchProcessor->sendNextBatch(); - break; - } - } - else // Single Insert - { - // make sure insertPkg.readMetaData() is called before - // this on fByteStream! - // TODO: Similar to batch inserts, don't - // deserialize the row data here for single inserts. - insertPkg.readRowData(*(fByteStream.get())); - insertPkg.set_TxnID(fTxnid); - fProcessor.reset(new dmlpackageprocessor::InsertPackageProcessor(fDbrm, insertPkg.get_SessionID())); - result = fProcessor->processPackage(insertPkg); - } - - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = result.stats.fMaxMemPct; - qts.num_files = result.stats.fNumFiles; - qts.phy_io = result.stats.fPhyIO; - qts.cache_io = result.stats.fCacheIO; - qts.msg_rcv_cnt = result.stats.fMsgRcvCnt; - qts.cp_blocks_skipped = result.stats.fCPBlocksSkipped; - qts.msg_bytes_in = result.stats.fMsgBytesIn; - qts.msg_bytes_out = result.stats.fMsgBytesOut; - qts.rows = result.stats.fRows; - qts.end_time = QueryTeleClient::timeNowms(); - qts.blocks_changed = result.stats.fBlocksChanged; - fQtc.postQueryTele(qts); - } - break; - - case dmlpackage::DML_UPDATE: + if (batchIter == DMLProcessor::batchinsertProcessorMap.end()) { - // build an UpdateDMLPackage from the bytestream - //cout << "an UPDATE package" << endl; - boost::scoped_ptr updatePkg(new dmlpackage::UpdateDMLPackage()); - updatePkg->read(*(fByteStream.get())); + batchProcessor = + new BatchInsertProc(insertPkg.get_isAutocommitOn(), insertPkg.getTableOid(), fTxnid, fDbrm); + DMLProcessor::batchinsertProcessorMap[fSessionID] = batchProcessor; + // cout << "batchProcessor is created " << batchProcessor << endl; + } + else + { + batchProcessor = batchIter->second; + // cout << "Found batchProcessor " << batchProcessor << endl; + } + } + + if (insertPkg.get_Logging()) + { + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("Start SQL statement: "); + + if (!insertPkg.get_isCacheInsert()) + { + ostringstream oss; + oss << insertPkg.get_SQLStatement() << "; |" << insertPkg.get_SchemaName() << "|"; + args1.add(oss.str()); + } + + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + TablelockData* tablelockData = TablelockData::makeTablelockData(insertPkg.get_SessionID()); + uint64_t tableLockId = tablelockData->getTablelockId(insertPkg.getTableOid()); + + // cout << "Processing table oid " << insertPkg.getTableOid() << " for transaction "<< (int)fTxnid + // << endl; + if (tableLockId == 0) + { + // cout << "Grabing tablelock for batchProcessor " << batchProcessor << endl; + tableLockId = batchProcessor->grabTableLock(insertPkg.get_SessionID()); + + if (tableLockId == 0) + { + BRM::TxnID brmTxnID; + brmTxnID.id = fTxnid; + brmTxnID.valid = true; + sessionManager.rolledback(brmTxnID); + string errMsg; + int rc = 0; + batchProcessor->getError(rc, errMsg); + result.result = DMLPackageProcessor::TABLE_LOCK_ERROR; + logging::Message::Args args; + logging::Message message(1); + args.add("Insert Failed: "); + args.add(errMsg); + args.add(""); + args.add(""); + message.format(args); + result.message = message; + break; + } + + if (tableLockId > 0) + tablelockData->setTablelock(insertPkg.getTableOid(), tableLockId); + } + } + + if (insertPkg.get_Logending() && insertPkg.get_Logging()) // only one batch need to be processed. + { + // cout << "dmlprocessor add last pkg" << endl; + // need to add error handling. + batchProcessor->addPkg(bsSave); + batchProcessor->sendFirstBatch(); + batchProcessor->receiveOutstandingMsg(); + //@Bug 5162. Get the correct error message before the last message. + string errMsg; + int rc = 0; + batchProcessor->getError(rc, errMsg); + batchProcessor->sendlastBatch(); + batchProcessor->receiveAllMsg(); + + if (rc == DMLPackageProcessor::IDBRANGE_WARNING) + { + result.result = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with warnings"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + logging::Message::Args args; + logging::Message message(1); + args.add(errMsg); + args.add(""); + args.add(""); + message.format(args); + result.message = message; + } + else if (rc != 0) + { + result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + logging::Message::Args args; + logging::Message message(1); + cout << "Got error in the end of one batchinsert." << endl; + args.add("Insert Failed: "); + args.add(errMsg); + args.add(""); + args.add(""); + message.format(args); + result.message = message; + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with error"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + } + else + { + // if (!insertPkg.get_isAutocommitOn()) + // { + // batchProcessor->setHwm(); + // } + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + } + + // remove the batch insert object + { + boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); + + std::map::iterator batchIter = + DMLProcessor::batchinsertProcessorMap.find(fSessionID); + + if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) + { + delete batchIter->second; + DMLProcessor::batchinsertProcessorMap.erase(fSessionID); + } + } + } + else if (insertPkg.get_Logending()) // Last batch + { + int rc = 0; + string errMsg; + batchProcessor->getError(rc, errMsg); + + // cout <<"dmlprocessor received last pkg from mysql rc == " << rc << endl; + if ((rc == 0) || (rc == DMLPackageProcessor::IDBRANGE_WARNING)) + { + // cout << " rc = " << rc << endl; + batchProcessor->addPkg(bsSave); + batchProcessor->sendNextBatch(); + batchProcessor->receiveOutstandingMsg(); + //@Bug 5162. Get the correct error message before the last message. + batchProcessor->getError(rc, errMsg); + batchProcessor->sendlastBatch(); + batchProcessor->receiveAllMsg(); + + if (rc == DMLPackageProcessor::IDBRANGE_WARNING) + { + result.result = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with warnings"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + logging::Message::Args args; + logging::Message message(1); + args.add(errMsg); + args.add(""); + args.add(""); + message.format(args); + result.message = message; + } + else if (rc != 0) + { + // cout << "Got error in the end of last batchinsert. error message is " << errMsg << endl; + result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + logging::Message::Args args; + logging::Message message(1); + args.add("Insert Failed: "); + args.add(errMsg); + args.add(""); + args.add(""); + message.format(args); + result.message = message; + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with error"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + } + else + { + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + } + + // cout << "finished batch insert" << endl; + } + else + { + // error occured. Receive all outstanding messages before erroring out. + batchProcessor->receiveOutstandingMsg(); + batchProcessor->sendlastBatch(); // needs to flush files + batchProcessor->receiveAllMsg(); + result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + // cout << "Got error in the end of batchinsert2. error msg is " << errMsg<< endl; + logging::Message::Args args; + logging::Message message(1); + args.add("Insert Failed: "); + args.add(errMsg); + args.add(""); + args.add(""); + message.format(args); + result.message = message; + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with error"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + logging::logDML(insertPkg.get_SessionID(), fTxnid, insertPkg.get_SQLStatement() + ";", + insertPkg.get_SchemaName()); + } + + // remove from map + { + boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); + std::map::iterator batchIter = + DMLProcessor::batchinsertProcessorMap.find(fSessionID); + + if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) + { + // cout << "Batchinsertprcessor is deleted. " << batchIter->second << endl; + delete batchIter->second; + DMLProcessor::batchinsertProcessorMap.erase(fSessionID); + } + } + } + else + { + int rc = 0; + string errMsg; + batchProcessor->getError(rc, errMsg); + + if (rc == DMLPackageProcessor::IDBRANGE_WARNING) + { + result.result = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + } + else if (rc != 0) + { + result.result = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + //@Bug + // cout << "Got error during batchinsert. with message " << errMsg << endl; + logging::Message::Args args; + logging::Message message(6); + args.add(errMsg); + message.format(args); + result.message = message; + batchProcessor->receiveOutstandingMsg(); + batchProcessor->sendlastBatch(); // needs to flush files + // cout << "Last batch is sent to WES." << endl; + batchProcessor->receiveAllMsg(); + LoggingID logid(DMLLoggingId, insertPkg.get_SessionID(), fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("End SQL statement with error"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + // remove from map + { + boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); + std::map::iterator batchIter = + DMLProcessor::batchinsertProcessorMap.find(fSessionID); + + if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) + { + // cout << "Batchinsertprcessor is deleted. " << batchIter->second << endl; + delete batchIter->second; + DMLProcessor::batchinsertProcessorMap.erase(fSessionID); + } + } + break; + } + + batchProcessor->addPkg(bsSave); + batchProcessor->sendNextBatch(); + break; + } + } + else // Single Insert + { + // make sure insertPkg.readMetaData() is called before + // this on fByteStream! + // TODO: Similar to batch inserts, don't + // deserialize the row data here for single inserts. + insertPkg.readRowData(*(fByteStream.get())); + insertPkg.set_TxnID(fTxnid); + fProcessor.reset(new dmlpackageprocessor::InsertPackageProcessor(fDbrm, insertPkg.get_SessionID())); + result = fProcessor->processPackage(insertPkg); + } + + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = result.stats.fMaxMemPct; + qts.num_files = result.stats.fNumFiles; + qts.phy_io = result.stats.fPhyIO; + qts.cache_io = result.stats.fCacheIO; + qts.msg_rcv_cnt = result.stats.fMsgRcvCnt; + qts.cp_blocks_skipped = result.stats.fCPBlocksSkipped; + qts.msg_bytes_in = result.stats.fMsgBytesIn; + qts.msg_bytes_out = result.stats.fMsgBytesOut; + qts.rows = result.stats.fRows; + qts.end_time = QueryTeleClient::timeNowms(); + qts.blocks_changed = result.stats.fBlocksChanged; + fQtc.postQueryTele(qts); + } + break; + + case dmlpackage::DML_UPDATE: + { + // build an UpdateDMLPackage from the bytestream + // cout << "an UPDATE package" << endl; + boost::scoped_ptr updatePkg(new dmlpackage::UpdateDMLPackage()); + updatePkg->read(*(fByteStream.get())); #ifdef MCOL_140 - if (fConcurrentSupport) - { - fTableOid = updatePkg->getTableOid(); + if (fConcurrentSupport) + { + fTableOid = updatePkg->getTableOid(); - // Update generally doesn't set fTableOid in updatePkg. Go get it now. - if (fTableOid == 0) - { - CalpontSystemCatalog::TableName tableName; - tableName.schema = updatePkg->get_Table()->get_SchemaName(); - tableName.table = updatePkg->get_Table()->get_TableName(); - CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); - fTableOid = roPair.objnum; - } - synchTable.setPackage(this, updatePkg.get()); // Blocks if another DML thread is using this fTableOid - } + // Update generally doesn't set fTableOid in updatePkg. Go get it now. + if (fTableOid == 0) + { + CalpontSystemCatalog::TableName tableName; + tableName.schema = updatePkg->get_Table()->get_SchemaName(); + tableName.table = updatePkg->get_Table()->get_TableName(); + CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); + fTableOid = roPair.objnum; + } + synchTable.setPackage(this, + updatePkg.get()); // Blocks if another DML thread is using this fTableOid + } #endif - updatePkg->set_TxnID(fTxnid); - QueryTeleStats qts; - qts.query_uuid = updatePkg->uuid(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = fSessionID; - qts.query_type = "UPDATE"; - qts.query = updatePkg->get_SQLStatement(); - qts.system_name = oamCache->getSystemName(); - qts.module_name = oamCache->getModuleName(); - qts.schema_name = updatePkg->get_SchemaName(); - fQtc.postQueryTele(qts); - // process it - //@Bug 1341. Don't remove calpontsystemcatalog from this - //session to take advantage of cache. - fProcessor.reset(new dmlpackageprocessor::UpdatePackageProcessor(fDbrm, updatePkg->get_SessionID())); - fProcessor->setEngineComm(fEC); - fProcessor->setRM( frm); - idbassert( fTxnid != 0); - result = fProcessor->processPackage(*(updatePkg.get())) ; - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = result.stats.fMaxMemPct; - qts.num_files = result.stats.fNumFiles; - qts.phy_io = result.stats.fPhyIO; - qts.cache_io = result.stats.fCacheIO; - qts.msg_rcv_cnt = result.stats.fMsgRcvCnt; - qts.cp_blocks_skipped = result.stats.fCPBlocksSkipped; - qts.msg_bytes_in = result.stats.fMsgBytesIn; - qts.msg_bytes_out = result.stats.fMsgBytesOut; - qts.rows = result.stats.fRows; - qts.end_time = QueryTeleClient::timeNowms(); - qts.blocks_changed = result.stats.fBlocksChanged; - fQtc.postQueryTele(qts); - } - break; + updatePkg->set_TxnID(fTxnid); + QueryTeleStats qts; + qts.query_uuid = updatePkg->uuid(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = fSessionID; + qts.query_type = "UPDATE"; + qts.query = updatePkg->get_SQLStatement(); + qts.system_name = oamCache->getSystemName(); + qts.module_name = oamCache->getModuleName(); + qts.schema_name = updatePkg->get_SchemaName(); + fQtc.postQueryTele(qts); + // process it + //@Bug 1341. Don't remove calpontsystemcatalog from this + // session to take advantage of cache. + fProcessor.reset(new dmlpackageprocessor::UpdatePackageProcessor(fDbrm, updatePkg->get_SessionID())); + fProcessor->setEngineComm(fEC); + fProcessor->setRM(frm); + idbassert(fTxnid != 0); + result = fProcessor->processPackage(*(updatePkg.get())); + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = result.stats.fMaxMemPct; + qts.num_files = result.stats.fNumFiles; + qts.phy_io = result.stats.fPhyIO; + qts.cache_io = result.stats.fCacheIO; + qts.msg_rcv_cnt = result.stats.fMsgRcvCnt; + qts.cp_blocks_skipped = result.stats.fCPBlocksSkipped; + qts.msg_bytes_in = result.stats.fMsgBytesIn; + qts.msg_bytes_out = result.stats.fMsgBytesOut; + qts.rows = result.stats.fRows; + qts.end_time = QueryTeleClient::timeNowms(); + qts.blocks_changed = result.stats.fBlocksChanged; + fQtc.postQueryTele(qts); + } + break; - case dmlpackage::DML_DELETE: - { - boost::scoped_ptr deletePkg(new dmlpackage::DeleteDMLPackage()); - deletePkg->read(*(fByteStream.get())); + case dmlpackage::DML_DELETE: + { + boost::scoped_ptr deletePkg(new dmlpackage::DeleteDMLPackage()); + deletePkg->read(*(fByteStream.get())); #ifdef MCOL_140 - if (fConcurrentSupport) - { - fTableOid = deletePkg->getTableOid(); + if (fConcurrentSupport) + { + fTableOid = deletePkg->getTableOid(); - // Delete generally doesn't set fTableOid in updatePkg. Go get it now. - if (fTableOid == 0) - { - CalpontSystemCatalog::TableName tableName; - tableName.schema = deletePkg->get_Table()->get_SchemaName(); - tableName.table = deletePkg->get_Table()->get_TableName(); - CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); - fTableOid = roPair.objnum; - } - synchTable.setPackage(this, deletePkg.get()); // Blocks if another DML thread is using this fTableOid - } + // Delete generally doesn't set fTableOid in updatePkg. Go get it now. + if (fTableOid == 0) + { + CalpontSystemCatalog::TableName tableName; + tableName.schema = deletePkg->get_Table()->get_SchemaName(); + tableName.table = deletePkg->get_Table()->get_TableName(); + CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); + fTableOid = roPair.objnum; + } + synchTable.setPackage(this, + deletePkg.get()); // Blocks if another DML thread is using this fTableOid + } #endif - deletePkg->set_TxnID(fTxnid); - QueryTeleStats qts; - qts.query_uuid = deletePkg->uuid(); - qts.msg_type = QueryTeleStats::QT_START; - qts.start_time = QueryTeleClient::timeNowms(); - qts.session_id = fSessionID; - qts.query_type = "DELETE"; - qts.query = deletePkg->get_SQLStatement(); - qts.system_name = oamCache->getSystemName(); - qts.module_name = oamCache->getModuleName(); - qts.schema_name = deletePkg->get_SchemaName(); - fQtc.postQueryTele(qts); - // process it - //@Bug 1341. Don't remove calpontsystemcatalog from this session to take advantage of cache. - fProcessor.reset(new dmlpackageprocessor::DeletePackageProcessor(fDbrm, deletePkg->get_SessionID())); - fProcessor->setEngineComm(fEC); - fProcessor->setRM( frm); - idbassert( fTxnid != 0); - result = fProcessor->processPackage(*(deletePkg.get())) ; - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = result.stats.fMaxMemPct; - qts.num_files = result.stats.fNumFiles; - qts.phy_io = result.stats.fPhyIO; - qts.cache_io = result.stats.fCacheIO; - qts.msg_rcv_cnt = result.stats.fMsgRcvCnt; - qts.cp_blocks_skipped = result.stats.fCPBlocksSkipped; - qts.msg_bytes_in = result.stats.fMsgBytesIn; - qts.msg_bytes_out = result.stats.fMsgBytesOut; - qts.rows = result.stats.fRows; - qts.end_time = QueryTeleClient::timeNowms(); - qts.blocks_changed = result.stats.fBlocksChanged; - fQtc.postQueryTele(qts); - } - break; + deletePkg->set_TxnID(fTxnid); + QueryTeleStats qts; + qts.query_uuid = deletePkg->uuid(); + qts.msg_type = QueryTeleStats::QT_START; + qts.start_time = QueryTeleClient::timeNowms(); + qts.session_id = fSessionID; + qts.query_type = "DELETE"; + qts.query = deletePkg->get_SQLStatement(); + qts.system_name = oamCache->getSystemName(); + qts.module_name = oamCache->getModuleName(); + qts.schema_name = deletePkg->get_SchemaName(); + fQtc.postQueryTele(qts); + // process it + //@Bug 1341. Don't remove calpontsystemcatalog from this session to take advantage of cache. + fProcessor.reset(new dmlpackageprocessor::DeletePackageProcessor(fDbrm, deletePkg->get_SessionID())); + fProcessor->setEngineComm(fEC); + fProcessor->setRM(frm); + idbassert(fTxnid != 0); + result = fProcessor->processPackage(*(deletePkg.get())); + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = result.stats.fMaxMemPct; + qts.num_files = result.stats.fNumFiles; + qts.phy_io = result.stats.fPhyIO; + qts.cache_io = result.stats.fCacheIO; + qts.msg_rcv_cnt = result.stats.fMsgRcvCnt; + qts.cp_blocks_skipped = result.stats.fCPBlocksSkipped; + qts.msg_bytes_in = result.stats.fMsgBytesIn; + qts.msg_bytes_out = result.stats.fMsgBytesOut; + qts.rows = result.stats.fRows; + qts.end_time = QueryTeleClient::timeNowms(); + qts.blocks_changed = result.stats.fBlocksChanged; + fQtc.postQueryTele(qts); + } + break; - case dmlpackage::DML_COMMAND: - { - // build a CommandDMLPackage from the bytestream - //cout << "a COMMAND package" << endl; - dmlpackage::CommandDMLPackage commandPkg; - commandPkg.read(*(fByteStream.get())); - stmt = commandPkg.get_DMLStatement(); - boost::algorithm::to_upper(stmt); - trim(stmt); + case dmlpackage::DML_COMMAND: + { + // build a CommandDMLPackage from the bytestream + // cout << "a COMMAND package" << endl; + dmlpackage::CommandDMLPackage commandPkg; + commandPkg.read(*(fByteStream.get())); + stmt = commandPkg.get_DMLStatement(); + boost::algorithm::to_upper(stmt); + trim(stmt); - if (stmt == "CLEANUP") - { - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(commandPkg.get_SessionID()); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(commandPkg.get_SessionID() | 0x80000000); - } - else - { - // process it - //@Bug 1341. Don't remove calpontsystemcatalog from this session to take advantage of cache. - fProcessor.reset(new dmlpackageprocessor::CommandPackageProcessor(fDbrm, commandPkg.get_SessionID())); - - //cout << "got command " << stmt << " for session " << commandPkg.get_SessionID() << endl; - result = fProcessor->processPackage(commandPkg); - } - } - break; - } - - - - - //Log errors - if ( (result.result != dmlpackageprocessor::DMLPackageProcessor::NO_ERROR) - && (result.result != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - && (result.result != dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) - && (result.result != dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR) ) + if (stmt == "CLEANUP") { - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - - ml.logErrorMessage( result.message ); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(commandPkg.get_SessionID()); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(commandPkg.get_SessionID() | 0x80000000); } - else if (result.result == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + else { - logging::LoggingID lid(21); - logging::MessageLog ml(lid); + // process it + //@Bug 1341. Don't remove calpontsystemcatalog from this session to take advantage of cache. + fProcessor.reset( + new dmlpackageprocessor::CommandPackageProcessor(fDbrm, commandPkg.get_SessionID())); - ml.logWarningMessage( result.message ); + // cout << "got command " << stmt << " for session " << commandPkg.get_SessionID() << endl; + result = fProcessor->processPackage(commandPkg); } + } + break; } - catch (std::exception& e) + + // Log errors + if ((result.result != dmlpackageprocessor::DMLPackageProcessor::NO_ERROR) && + (result.result != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) && + (result.result != dmlpackageprocessor::DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR) && + (result.result != dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR)) { + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - - cout << "dmlprocessor.cpp PackageHandler::run() package type(" - << fPackageType << ") exception: " << e.what() << endl; - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - logging::Message::Args args; - logging::Message message(1); - args.add("dmlprocessor.cpp PackageHandler::run() package type"); - args.add((uint64_t)fPackageType); - args.add(e.what()); - message.format(args); - ml.logErrorMessage(message); - result.result = DMLPackageProcessor::COMMAND_ERROR; - result.message = message; + ml.logErrorMessage(result.message); } - catch (...) + else if (result.result == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) { + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - - logging::LoggingID lid(21); - logging::MessageLog ml(lid); - logging::Message::Args args; - logging::Message message(1); - args.add("dmlprocessor.cpp PackageHandler::run() ... exception package type"); - args.add((uint64_t)fPackageType); - message.format(args); - ml.logErrorMessage(message); - result.result = DMLPackageProcessor::COMMAND_ERROR; - result.message = message; + ml.logWarningMessage(result.message); } + } + catch (std::exception& e) + { + cout << "dmlprocessor.cpp PackageHandler::run() package type(" << fPackageType + << ") exception: " << e.what() << endl; + logging::LoggingID lid(21); + logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message message(1); + args.add("dmlprocessor.cpp PackageHandler::run() package type"); + args.add((uint64_t)fPackageType); + args.add(e.what()); + message.format(args); + ml.logErrorMessage(message); + result.result = DMLPackageProcessor::COMMAND_ERROR; + result.message = message; + } + catch (...) + { + logging::LoggingID lid(21); + logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message message(1); + args.add("dmlprocessor.cpp PackageHandler::run() ... exception package type"); + args.add((uint64_t)fPackageType); + message.format(args); + ml.logErrorMessage(message); + result.result = DMLPackageProcessor::COMMAND_ERROR; + result.message = message; + } - // We put the packageHandler into a map so that if we receive a - // message to affect the previous command, we can find it. - // We need to remove it from the list before sending the response back. - // If we remove it after sending the results, it's possible for a commit - // or rollback be sent and get processed before it is removed, and that - // will fail. - boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock); - DMLProcessor::packageHandlerMap.erase(getSessionID()); - lk2.unlock(); + // We put the packageHandler into a map so that if we receive a + // message to affect the previous command, we can find it. + // We need to remove it from the list before sending the response back. + // If we remove it after sending the results, it's possible for a commit + // or rollback be sent and get processed before it is removed, and that + // will fail. + boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock); + DMLProcessor::packageHandlerMap.erase(getSessionID()); + lk2.unlock(); - // send back the results - messageqcpp::ByteStream results; - messageqcpp::ByteStream::octbyte rowCount = result.rowCount; - messageqcpp::ByteStream::byte retval = result.result; - results << retval; - results << rowCount; - results << result.message.msg(); - results << result.tableLockInfo; // ? connector does not get - // query stats - results << result.queryStats; - results << result.extendedStats; - results << result.miniStats; - result.stats.serialize(results); - fIos.write(results); - //Bug 5226. dmlprocessor thread will close the socket to mysqld. - //if (stmt == "CLEANUP") - // fIos.close(); + // send back the results + messageqcpp::ByteStream results; + messageqcpp::ByteStream::octbyte rowCount = result.rowCount; + messageqcpp::ByteStream::byte retval = result.result; + results << retval; + results << rowCount; + results << result.message.msg(); + results << result.tableLockInfo; // ? connector does not get + // query stats + results << result.queryStats; + results << result.extendedStats; + results << result.miniStats; + result.stats.serialize(results); + fIos.write(results); + // Bug 5226. dmlprocessor thread will close the socket to mysqld. + // if (stmt == "CLEANUP") + // fIos.close(); } void PackageHandler::rollbackPending() { - if (fProcessor.get() == NULL) - { - // This happens when batch insert - return; - } + if (fProcessor.get() == NULL) + { + // This happens when batch insert + return; + } - fProcessor->setRollbackPending(true); + fProcessor->setRollbackPending(true); - // Force a release of the processing from MCOL-140 + // Force a release of the processing from MCOL-140 #ifdef MCOL_140 - if (fConcurrentSupport) - { - // MCOL-140 We're not necessarily the next in line. - // This forces this thread to be released anyway. - forceReleaseTableAccess(); - } + if (fConcurrentSupport) + { + // MCOL-140 We're not necessarily the next in line. + // This forces this thread to be released anyway. + forceReleaseTableAccess(); + } #endif - ostringstream oss; - oss << "PackageHandler::rollbackPending: Processing DMLPackage."; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); + ostringstream oss; + oss << "PackageHandler::rollbackPending: Processing DMLPackage."; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); } void added_a_pm(int) { - DistributedEngineComm* dec; - ResourceManager* rm = ResourceManager::instance(); - dec = DistributedEngineComm::instance(rm); - dec->Setup(); - // MCOL-140 clear the waiting queue as all transactions are probably going to fail - PackageHandler::clearTableAccess(); + DistributedEngineComm* dec; + ResourceManager* rm = ResourceManager::instance(); + dec = DistributedEngineComm::instance(rm); + dec->Setup(); + // MCOL-140 clear the waiting queue as all transactions are probably going to fail + PackageHandler::clearTableAccess(); } -DMLServer::DMLServer(int packageMaxThreads, int packageWorkQueueSize, DBRM* dbrm) : - fPackageMaxThreads(packageMaxThreads), fPackageWorkQueueSize(packageWorkQueueSize), fDbrm(dbrm), fShutdownFlag(false) +DMLServer::DMLServer(int packageMaxThreads, int packageWorkQueueSize, DBRM* dbrm) + : fPackageMaxThreads(packageMaxThreads) + , fPackageWorkQueueSize(packageWorkQueueSize) + , fDbrm(dbrm) + , fShutdownFlag(false) { - fMqServer.reset(new MessageQueueServer("DMLProc")); + fMqServer.reset(new MessageQueueServer("DMLProc")); - fDmlPackagepool.setMaxThreads(fPackageMaxThreads); - fDmlPackagepool.setQueueSize(fPackageWorkQueueSize); - fDmlPackagepool.setName("DmlPackagepool"); + fDmlPackagepool.setMaxThreads(fPackageMaxThreads); + fDmlPackagepool.setQueueSize(fPackageWorkQueueSize); + fDmlPackagepool.setName("DmlPackagepool"); } int DMLServer::start() { - messageqcpp::IOSocket ios; - uint32_t nextID = 1; + messageqcpp::IOSocket ios; + uint32_t nextID = 1; - try + try + { + // CancellationThread is for telling all active transactions + // to quit working because the system is either going down + // or going into write suspend mode + CancellationThread cancelObject(fDbrm, *this); + boost::thread cancelThread(cancelObject); + + cout << "DMLProc is ready..." << endl; + + const static struct timespec timeout = {1, 100}; // roughly 1 second TO + for (;;) { - // CancellationThread is for telling all active transactions - // to quit working because the system is either going down - // or going into write suspend mode - CancellationThread cancelObject(fDbrm, *this); - boost::thread cancelThread(cancelObject); + ios = fMqServer->accept(&timeout); + // MCS polls in a loop watching for a pending shutdown + // that is signalled via fShutdownFlag set in a + // CancellationThread. CT sets the flag if a cluster state + // has SS_SHUTDOWNPENDING value set. + while (!ios.hasSocketDescriptor() && !pendingShutdown()) + ios = fMqServer->accept(&timeout); - cout << "DMLProc is ready..." << endl; - - const static struct timespec timeout = {1, 100}; // roughly 1 second TO - for (;;) - { - ios = fMqServer->accept(&timeout); - // MCS polls in a loop watching for a pending shutdown - // that is signalled via fShutdownFlag set in a - // CancellationThread. CT sets the flag if a cluster state - // has SS_SHUTDOWNPENDING value set. - while (!ios.hasSocketDescriptor() && !pendingShutdown()) - ios = fMqServer->accept(&timeout); - - if (pendingShutdown()) - break; - ios.setSockID(nextID++); - fDmlPackagepool.invoke(DMLProcessor(ios, fDbrm)); - } - - cancelThread.join(); - return EXIT_SUCCESS; - } - catch (std::exception& ex) - { - cerr << ex.what() << endl; - logging::LoggingID lid(21); - Message::Args args; - Message message(8); - args.add("DMLProc init caught exception: "); - args.add(ex.what()); - message.format(args); - logging::Logger logger(lid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_CRITICAL, message, lid); - return EXIT_FAILURE; - } - catch (...) - { - cerr << "Caught unknown exception!" << endl; - logging::LoggingID lid(21); - Message::Args args; - Message message(8); - args.add("DMLProc init caught unknown exception"); - message.format(args); - logging::Logger logger(lid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_CRITICAL, message, lid); - return EXIT_FAILURE; + if (pendingShutdown()) + break; + ios.setSockID(nextID++); + fDmlPackagepool.invoke(DMLProcessor(ios, fDbrm)); } + + cancelThread.join(); + return EXIT_SUCCESS; + } + catch (std::exception& ex) + { + cerr << ex.what() << endl; + logging::LoggingID lid(21); + Message::Args args; + Message message(8); + args.add("DMLProc init caught exception: "); + args.add(ex.what()); + message.format(args); + logging::Logger logger(lid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_CRITICAL, message, lid); + return EXIT_FAILURE; + } + catch (...) + { + cerr << "Caught unknown exception!" << endl; + logging::LoggingID lid(21); + Message::Args args; + Message message(8); + args.add("DMLProc init caught unknown exception"); + message.format(args); + logging::Logger logger(lid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_CRITICAL, message, lid); + return EXIT_FAILURE; + } } -DMLProcessor::DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm) : - fIos(ios), fDbrm(aDbrm) +DMLProcessor::DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm) : fIos(ios), fDbrm(aDbrm) { - csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); - csc->identity(CalpontSystemCatalog::EC); - string teleServerHost(config::Config::makeConfig()->getConfig("QueryTele", "Host")); + csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); + csc->identity(CalpontSystemCatalog::EC); + string teleServerHost(config::Config::makeConfig()->getConfig("QueryTele", "Host")); - if (!teleServerHost.empty()) + if (!teleServerHost.empty()) + { + int teleServerPort = + config::Config::fromText(config::Config::makeConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) { - int teleServerPort = config::Config::fromText(config::Config::makeConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - fQtc.serverParms(QueryTeleServerParms(teleServerHost, teleServerPort)); - } + fQtc.serverParms(QueryTeleServerParms(teleServerHost, teleServerPort)); } + } } void DMLProcessor::operator()() { - bool bIsDbrmUp = true; + bool bIsDbrmUp = true; - try + try + { + boost::shared_ptr bs1(new messageqcpp::ByteStream()); + // messageqcpp::ByteStream bs; + uint8_t packageType; + + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* fEC = DistributedEngineComm::instance(rm); + + uint64_t maxDeleteRows = rm->getDMLMaxDeleteRows(); + + fConcurrentSupport = true; + string concurrentTranStr = + config::Config::makeConfig()->getConfig("SystemConfig", "ConcurrentTransactions"); + + if (concurrentTranStr.length() != 0) { - boost::shared_ptr bs1 (new messageqcpp::ByteStream()); - //messageqcpp::ByteStream bs; - uint8_t packageType; - - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* fEC = DistributedEngineComm::instance(rm); - - uint64_t maxDeleteRows = rm->getDMLMaxDeleteRows(); - - fConcurrentSupport = true; - string concurrentTranStr = config::Config::makeConfig()->getConfig("SystemConfig", "ConcurrentTransactions"); - - if ( concurrentTranStr.length() != 0 ) - { - if ((concurrentTranStr.compare("N") == 0) || (concurrentTranStr.compare("n") == 0)) - fConcurrentSupport = false; - } + if ((concurrentTranStr.compare("N") == 0) || (concurrentTranStr.compare("n") == 0)) + fConcurrentSupport = false; + } #ifndef _MSC_VER - struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = added_a_pm; - sigaction(SIGHUP, &ign, 0); + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = added_a_pm; + sigaction(SIGHUP, &ign, 0); #endif - fEC->Open(); + fEC->Open(); - for (;;) + for (;;) + { + // cout << "DMLProc is waiting for a Calpont DML Package on " << fIos.getSockID() << endl; + try + { + bs1.reset(new messageqcpp::ByteStream(fIos.read())); + // cout << "received from mysql socket " << fIos.getSockID() << endl; + } + catch (std::exception& ex) + { + // This is an I/O error from InetStreamSocket::read(), just close and move on... + cout << "runtime error during read on " << fIos.getSockID() << " " << ex.what() << endl; + bs1->reset(); + } + catch (...) + { + cout << "... error during read " << fIos.getSockID() << endl; + // all this throw does is cause this thread to silently go away. I doubt this is the right + // thing to do... + throw; + } + + if (!bs1 || bs1->length() == 0) + { + cout << "Read 0 bytes. Closing connection " << fIos.getSockID() << endl; + fIos.close(); + break; + } + + uint32_t sessionID; + *bs1 >> sessionID; + *bs1 >> packageType; + // cout << "DMLProc received pkg. sessionid:type = " << sessionID <<":"<<(int)packageType << endl; + uint32_t stateFlags; + messageqcpp::ByteStream::byte status = 255; + messageqcpp::ByteStream::octbyte rowCount = 0; + + if (fDbrm->getSystemState(stateFlags) > + 0) // > 0 implies succesful retrieval. It doesn't imply anything about the contents + { + messageqcpp::ByteStream results; + const char* responseMsg = 0; + bool bReject = false; + + // Check to see if we're in write suspended mode + // If so, we can't process the request. + if (stateFlags & SessionManagerServer::SS_SUSPENDED) { - //cout << "DMLProc is waiting for a Calpont DML Package on " << fIos.getSockID() << endl; - try + status = DMLPackageProcessor::NOT_ACCEPTING_PACKAGES; + responseMsg = "Writing to the database is disabled."; + bReject = true; + } + + // Check to see if we're in write suspend or shutdown pending mode + if (packageType != dmlpackage::DML_COMMAND) // Not a commit or rollback + { + if (stateFlags & SessionManagerServer::SS_SUSPEND_PENDING || + stateFlags & SessionManagerServer::SS_SHUTDOWN_PENDING) + { + if (stateFlags & SessionManagerServer::SS_SUSPEND_PENDING) { - bs1.reset(new messageqcpp::ByteStream(fIos.read())); - //cout << "received from mysql socket " << fIos.getSockID() << endl; - } - catch (std::exception& ex) - { - //This is an I/O error from InetStreamSocket::read(), just close and move on... - cout << "runtime error during read on " << fIos.getSockID() << " " << ex.what() << endl; - bs1->reset(); - } - catch (...) - { - cout << "... error during read " << fIos.getSockID() << endl; - // all this throw does is cause this thread to silently go away. I doubt this is the right - // thing to do... - throw; - } - - if (!bs1 || bs1->length() == 0) - { - cout << "Read 0 bytes. Closing connection " << fIos.getSockID() << endl; - fIos.close(); - break; - } - - uint32_t sessionID; - *bs1 >> sessionID; - *bs1 >> packageType; -//cout << "DMLProc received pkg. sessionid:type = " << sessionID <<":"<<(int)packageType << endl; - uint32_t stateFlags; - messageqcpp::ByteStream::byte status = 255; - messageqcpp::ByteStream::octbyte rowCount = 0; - - if (fDbrm->getSystemState(stateFlags) > 0) // > 0 implies succesful retrieval. It doesn't imply anything about the contents - { - messageqcpp::ByteStream results; - const char* responseMsg = 0; - bool bReject = false; - - // Check to see if we're in write suspended mode - // If so, we can't process the request. - if (stateFlags & SessionManagerServer::SS_SUSPENDED) - { - status = DMLPackageProcessor::NOT_ACCEPTING_PACKAGES; - responseMsg = "Writing to the database is disabled."; - bReject = true; - } - - // Check to see if we're in write suspend or shutdown pending mode - if (packageType != dmlpackage::DML_COMMAND) // Not a commit or rollback - { - if (stateFlags & SessionManagerServer::SS_SUSPEND_PENDING - || stateFlags & SessionManagerServer::SS_SHUTDOWN_PENDING) - { - if (stateFlags & SessionManagerServer::SS_SUSPEND_PENDING) - { - responseMsg = "Writing to the database is disabled."; - } - else - { - responseMsg = "The database is being shut down."; - } - - // Refuse all non active tranasactions - // Check the rollback flag - // -- Set: Rollback active transactions. - // -- Not set: Allow active transactions. - if (sessionManager.isTransactionActive(sessionID, bIsDbrmUp)) - { - if (stateFlags & SessionManagerServer::SS_ROLLBACK) - { - status = DMLPackageProcessor::JOB_CANCELED; - bReject = true; - } - } - else - { - status = DMLPackageProcessor::NOT_ACCEPTING_PACKAGES; - bReject = true; - } - } - - if (bReject) - { - // For batch insert, we need to send a lastpkg message - // to batchInsertProcessor so it can clean things up. - if (packageType == dmlpackage::DML_INSERT) - { - // build an InsertDMLPackage from the bytestream - // We need the flags from the package to know what - // type of package we're dealing with before we can - // take special action for the last package of a - // batch insert. - dmlpackage::InsertDMLPackage insertPkg; - messageqcpp::ByteStream bsSave = *(bs1.get()); - insertPkg.read(*(bs1.get())); - BatchInsertProc* batchInsertProcessor = NULL; - - if (insertPkg.get_isBatchInsert() && insertPkg.get_Logending()) - { - { - boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); - std::map::iterator batchIter = DMLProcessor::batchinsertProcessorMap.find(sessionID); - - if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) //The first batch, no need to do anything - { - - batchInsertProcessor = batchIter->second; - batchInsertProcessor->addPkg(bsSave); - - batchInsertProcessor->sendlastBatch(); - batchInsertProcessor->receiveAllMsg(); - - - if (!insertPkg.get_isAutocommitOn()) - { - batchInsertProcessor->setHwm(); - } - - batchIter = DMLProcessor::batchinsertProcessorMap.find(sessionID); - - if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) - { - DMLProcessor::batchinsertProcessorMap.erase(sessionID); - } - } - } - } - } - - results << status; - results << rowCount; - logging::Message::Args args; - logging::Message message(2); - args.add(responseMsg); - message.format( args ); - results << message.msg(); - fIos.write(results); - continue; - } - } - } - - // This section is to check to see if the user hit CTRL+C while the - // DML was processing If so, the sessionID will be found in - // packageHandlerMap and we can set rollbackPending in the - // associated packageHandler. Other than CTRL+C, we should never - // find our own sessionID in the map. - // This mechanism may prove useful for other things, so the above - // comment may change. - { - boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock); - DMLProcessor::PackageHandlerMap_t::iterator phIter = packageHandlerMap.find(sessionID); - - if (phIter != packageHandlerMap.end()) - { - if (packageType == dmlpackage::DML_COMMAND) - { - // MCOL-66 It's possible for a commit or rollback to get here if - // the timing is just right. Don't destroy its data - messageqcpp::ByteStream bsctrlc(bs1); - dmlpackage::CommandDMLPackage commandPkg; - commandPkg.read(bsctrlc); - std::string stmt = commandPkg.get_DMLStatement(); - boost::algorithm::to_upper(stmt); - trim(stmt); - - if (stmt == "CTRL+C") - { - phIter->second->rollbackPending(); - fIos.close(); - break; - } - } - else - { - // If there's a PackageHandler already working for this - // sessionID, we have a problem. Reject this package - messageqcpp::ByteStream results; - ostringstream oss; - oss << "Received a DML command for session " << sessionID << - " while still processing a command for the same sessionID"; - results << static_cast(DMLPackageProcessor::DEAD_LOCK_ERROR); - results << static_cast(0); // rowcount - logging::Message::Args args; - logging::Message message(2); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(20); - logging::MessageLog ml(lid); - ml.logErrorMessage(message); - results << message.msg(); - fIos.write(results); - continue; - } - } - } - - //cout << " got a "; - switch (packageType) - { - case dmlpackage::DML_INSERT: - //cout << "DML_INSERT"; - break; - - case dmlpackage::DML_UPDATE: - //cout << "DML_UPDATE"; - break; - - case dmlpackage::DML_DELETE: - //cout << "DML_DELETE"; - break; - - case dmlpackage::DML_COMMAND: - //cout << "DML_COMMAND"; - break; - - case dmlpackage::DML_INVALID_TYPE: - //cout << "DML_INVALID_TYPE"; - break; - - default: - //cout << "UNKNOWN"; - break; - } - - //cout << " package" << endl; - - BRM::TxnID txnid; - - if (!fConcurrentSupport) - { - //Check if any other active transaction - bool anyOtherActiveTransaction = true; - BRM::SIDTIDEntry blockingsid; - - //For logout commit trigger - if ( packageType == dmlpackage::DML_COMMAND ) - { - anyOtherActiveTransaction = false; - } - - int i = 0; - int waitPeriod = 10; - //@Bug 2487 Check transaction map every 1/10 second - - int sleepTime = 100; // sleep 100 milliseconds between checks - int numTries = 10; // try 10 times per second - - - string waitPeriodStr = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); - - if ( waitPeriodStr.length() != 0 ) - waitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); - - numTries = waitPeriod * 10; - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - - //cout << "starting i = " << i << endl; - //txnid = sessionManager.getTxnID(sessionID); - while (anyOtherActiveTransaction) - { - anyOtherActiveTransaction = sessionManager.checkActiveTransaction( sessionID, bIsDbrmUp, - blockingsid ); - - //cout << "session " << sessionID << " with package type " << (int)packageType << " got anyOtherActiveTransaction " << anyOtherActiveTransaction << endl; - if (anyOtherActiveTransaction) - { - for ( ; i < numTries; i++ ) - { -#ifdef _MSC_VER - Sleep(rm_ts.tv_sec * 1000); -#else - struct timespec abs_ts; - - //cout << "session " << sessionID << " nanosleep on package type " << (int)packageType << endl; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); - -#endif - anyOtherActiveTransaction = sessionManager.checkActiveTransaction( sessionID, bIsDbrmUp, - blockingsid ); - - if ( !anyOtherActiveTransaction ) - { - txnid = sessionManager.getTxnID(sessionID); - - //cout << "Ready to process type " << (int)packageType << " with txd " << txnid << endl; - if ( !txnid.valid ) - { - txnid = sessionManager.newTxnID(sessionID, true); - - if (txnid.valid) - { - //cout << "Ready to process type " << (int)packageType << " for session "<< sessionID << " with new txnid " << txnid.id << endl; - anyOtherActiveTransaction = false; - break; - } - else - { - anyOtherActiveTransaction = true; - } - } - else - { - anyOtherActiveTransaction = false; - //cout << "already have transaction to process type " << (int)packageType << " for session "<< sessionID <<" with existing txnid " << txnid.id << endl; - break; - } - } - } - - //cout << "ending i = " << i << endl; - } - else - { - //cout << "Ready to process type " << (int)packageType << endl; - txnid = sessionManager.getTxnID(sessionID); - - if ( !txnid.valid ) - { - txnid = sessionManager.newTxnID(sessionID, true); - - if (txnid.valid) - { - //cout << "later Ready to process type " << (int)packageType << " for session "<< sessionID << " with new txnid " << txnid.id << endl; - anyOtherActiveTransaction = false; - } - else - { - anyOtherActiveTransaction = true; - //cout << "Cannot get txnid for process type " << (int)packageType << " for session "<< sessionID << endl; - } - } - else - { - anyOtherActiveTransaction = false; - //cout << "already have transaction to process type " << (int)packageType << " for session "<< sessionID <<" with txnid " << txnid.id << endl; - break; - } - } - - if ((anyOtherActiveTransaction) && (i >= numTries)) - { - //cout << " Erroring out on package type " << (int)packageType << " for session " << sessionID << endl; - break; - } - } - - if (anyOtherActiveTransaction && (i >= numTries)) - { - //cout << " again Erroring out on package type " << (int)packageType << endl; - messageqcpp::ByteStream results; - //@Bug 2681 set error code for active transaction - status = DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR; - rowCount = 0; - results << status; - results << rowCount; - Message::Args args; - args.add(static_cast(blockingsid.sessionid)); - results << IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args); - //@Bug 3854 Log to debug.log - LoggingID logid(20, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add(IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args)); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); - - fIos.write(results); - } - else - { - //cout << "starting processing package type " << (int) packageType << " for session " << sessionID << " with id " << txnid.id << endl; - boost::shared_ptr php(new PackageHandler(fIos, bs1, packageType, fEC, - fConcurrentSupport, maxDeleteRows, sessionID, txnid.id, fDbrm, fQtc, csc)); - // We put the packageHandler into a map so that if we receive a - // message to affect the previous command, we can find it. - boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock, boost::defer_lock); - - lk2.lock(); - packageHandlerMap[sessionID] = php; - lk2.unlock(); - - php->run(); // Operates in this thread. - -// Move this to the end of PackageHandler so it is removed from the map before the response is sent -// lk2.lock(); -// packageHandlerMap.erase(sessionID); -// lk2.unlock(); - } + responseMsg = "Writing to the database is disabled."; } else { + responseMsg = "The database is being shut down."; + } + + // Refuse all non active tranasactions + // Check the rollback flag + // -- Set: Rollback active transactions. + // -- Not set: Allow active transactions. + if (sessionManager.isTransactionActive(sessionID, bIsDbrmUp)) + { + if (stateFlags & SessionManagerServer::SS_ROLLBACK) + { + status = DMLPackageProcessor::JOB_CANCELED; + bReject = true; + } + } + else + { + status = DMLPackageProcessor::NOT_ACCEPTING_PACKAGES; + bReject = true; + } + } + + if (bReject) + { + // For batch insert, we need to send a lastpkg message + // to batchInsertProcessor so it can clean things up. + if (packageType == dmlpackage::DML_INSERT) + { + // build an InsertDMLPackage from the bytestream + // We need the flags from the package to know what + // type of package we're dealing with before we can + // take special action for the last package of a + // batch insert. + dmlpackage::InsertDMLPackage insertPkg; + messageqcpp::ByteStream bsSave = *(bs1.get()); + insertPkg.read(*(bs1.get())); + BatchInsertProc* batchInsertProcessor = NULL; + + if (insertPkg.get_isBatchInsert() && insertPkg.get_Logending()) + { + { + boost::mutex::scoped_lock lk(DMLProcessor::batchinsertProcessorMapLock); + std::map::iterator batchIter = + DMLProcessor::batchinsertProcessorMap.find(sessionID); + + if (batchIter != + DMLProcessor::batchinsertProcessorMap.end()) // The first batch, no need to do anything + { + batchInsertProcessor = batchIter->second; + batchInsertProcessor->addPkg(bsSave); + + batchInsertProcessor->sendlastBatch(); + batchInsertProcessor->receiveAllMsg(); + + if (!insertPkg.get_isAutocommitOn()) + { + batchInsertProcessor->setHwm(); + } + + batchIter = DMLProcessor::batchinsertProcessorMap.find(sessionID); + + if (batchIter != DMLProcessor::batchinsertProcessorMap.end()) + { + DMLProcessor::batchinsertProcessorMap.erase(sessionID); + } + } + } + } + } + + results << status; + results << rowCount; + logging::Message::Args args; + logging::Message message(2); + args.add(responseMsg); + message.format(args); + results << message.msg(); + fIos.write(results); + continue; + } + } + } + + // This section is to check to see if the user hit CTRL+C while the + // DML was processing If so, the sessionID will be found in + // packageHandlerMap and we can set rollbackPending in the + // associated packageHandler. Other than CTRL+C, we should never + // find our own sessionID in the map. + // This mechanism may prove useful for other things, so the above + // comment may change. + { + boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock); + DMLProcessor::PackageHandlerMap_t::iterator phIter = packageHandlerMap.find(sessionID); + + if (phIter != packageHandlerMap.end()) + { + if (packageType == dmlpackage::DML_COMMAND) + { + // MCOL-66 It's possible for a commit or rollback to get here if + // the timing is just right. Don't destroy its data + messageqcpp::ByteStream bsctrlc(bs1); + dmlpackage::CommandDMLPackage commandPkg; + commandPkg.read(bsctrlc); + std::string stmt = commandPkg.get_DMLStatement(); + boost::algorithm::to_upper(stmt); + trim(stmt); + + if (stmt == "CTRL+C") + { + phIter->second->rollbackPending(); + fIos.close(); + break; + } + } + else + { + // If there's a PackageHandler already working for this + // sessionID, we have a problem. Reject this package + messageqcpp::ByteStream results; + ostringstream oss; + oss << "Received a DML command for session " << sessionID + << " while still processing a command for the same sessionID"; + results << static_cast(DMLPackageProcessor::DEAD_LOCK_ERROR); + results << static_cast(0); // rowcount + logging::Message::Args args; + logging::Message message(2); + args.add(oss.str()); + message.format(args); + logging::LoggingID lid(20); + logging::MessageLog ml(lid); + ml.logErrorMessage(message); + results << message.msg(); + fIos.write(results); + continue; + } + } + } + + // cout << " got a "; + switch (packageType) + { + case dmlpackage::DML_INSERT: + // cout << "DML_INSERT"; + break; + + case dmlpackage::DML_UPDATE: + // cout << "DML_UPDATE"; + break; + + case dmlpackage::DML_DELETE: + // cout << "DML_DELETE"; + break; + + case dmlpackage::DML_COMMAND: + // cout << "DML_COMMAND"; + break; + + case dmlpackage::DML_INVALID_TYPE: + // cout << "DML_INVALID_TYPE"; + break; + + default: + // cout << "UNKNOWN"; + break; + } + + // cout << " package" << endl; + + BRM::TxnID txnid; + + if (!fConcurrentSupport) + { + // Check if any other active transaction + bool anyOtherActiveTransaction = true; + BRM::SIDTIDEntry blockingsid; + + // For logout commit trigger + if (packageType == dmlpackage::DML_COMMAND) + { + anyOtherActiveTransaction = false; + } + + int i = 0; + int waitPeriod = 10; + //@Bug 2487 Check transaction map every 1/10 second + + int sleepTime = 100; // sleep 100 milliseconds between checks + int numTries = 10; // try 10 times per second + + string waitPeriodStr = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); + + if (waitPeriodStr.length() != 0) + waitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); + + numTries = waitPeriod * 10; + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + + // cout << "starting i = " << i << endl; + // txnid = sessionManager.getTxnID(sessionID); + while (anyOtherActiveTransaction) + { + anyOtherActiveTransaction = + sessionManager.checkActiveTransaction(sessionID, bIsDbrmUp, blockingsid); + + // cout << "session " << sessionID << " with package type " << (int)packageType << " got + // anyOtherActiveTransaction " << anyOtherActiveTransaction << endl; + if (anyOtherActiveTransaction) + { + for (; i < numTries; i++) + { +#ifdef _MSC_VER + Sleep(rm_ts.tv_sec * 1000); +#else + struct timespec abs_ts; + + // cout << "session " << sessionID << " nanosleep on package type " << (int)packageType << endl; + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); + +#endif + anyOtherActiveTransaction = + sessionManager.checkActiveTransaction(sessionID, bIsDbrmUp, blockingsid); + + if (!anyOtherActiveTransaction) + { + txnid = sessionManager.getTxnID(sessionID); + + // cout << "Ready to process type " << (int)packageType << " with txd " << txnid << endl; + if (!txnid.valid) + { + txnid = sessionManager.newTxnID(sessionID, true); + + if (txnid.valid) + { + // cout << "Ready to process type " << (int)packageType << " for session "<< sessionID << + // " with new txnid " << txnid.id << endl; + anyOtherActiveTransaction = false; + break; + } + else + { + anyOtherActiveTransaction = true; + } + } + else + { + anyOtherActiveTransaction = false; + // cout << "already have transaction to process type " << (int)packageType << " for session + // "<< sessionID <<" with existing txnid " << txnid.id << endl; + break; + } + } + } + + // cout << "ending i = " << i << endl; + } + else + { + // cout << "Ready to process type " << (int)packageType << endl; + txnid = sessionManager.getTxnID(sessionID); + + if (!txnid.valid) + { + txnid = sessionManager.newTxnID(sessionID, true); + + if (txnid.valid) + { + // cout << "later Ready to process type " << (int)packageType << " for session "<< sessionID + // << " with new txnid " << txnid.id << endl; + anyOtherActiveTransaction = false; + } + else + { + anyOtherActiveTransaction = true; + // cout << "Cannot get txnid for process type " << (int)packageType << " for session "<< + // sessionID << endl; + } + } + else + { + anyOtherActiveTransaction = false; + // cout << "already have transaction to process type " << (int)packageType << " for session "<< + // sessionID <<" with txnid " << txnid.id << endl; + break; + } + } + + if ((anyOtherActiveTransaction) && (i >= numTries)) + { + // cout << " Erroring out on package type " << (int)packageType << " for session " << sessionID << + // endl; + break; + } + } + + if (anyOtherActiveTransaction && (i >= numTries)) + { + // cout << " again Erroring out on package type " << (int)packageType << endl; + messageqcpp::ByteStream results; + //@Bug 2681 set error code for active transaction + status = DMLPackageProcessor::ACTIVE_TRANSACTION_ERROR; + rowCount = 0; + results << status; + results << rowCount; + Message::Args args; + args.add(static_cast(blockingsid.sessionid)); + results << IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args); + //@Bug 3854 Log to debug.log + LoggingID logid(20, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add(IDBErrorInfo::instance()->errorMsg(ERR_ACTIVE_TRANSACTION, args)); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + + fIos.write(results); + } + else + { + // cout << "starting processing package type " << (int) packageType << " for session " << sessionID + // << " with id " << txnid.id << endl; + boost::shared_ptr php(new PackageHandler(fIos, bs1, packageType, fEC, + fConcurrentSupport, maxDeleteRows, + sessionID, txnid.id, fDbrm, fQtc, csc)); + // We put the packageHandler into a map so that if we receive a + // message to affect the previous command, we can find it. + boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock, boost::defer_lock); + + lk2.lock(); + packageHandlerMap[sessionID] = php; + lk2.unlock(); + + php->run(); // Operates in this thread. + + // Move this to the end of PackageHandler so it is removed from the map before the response is sent + // lk2.lock(); + // packageHandlerMap.erase(sessionID); + // lk2.unlock(); + } + } + else + { #if 0 if (packageType != dmlpackage::DML_COMMAND) @@ -1758,230 +1784,221 @@ void DMLProcessor::operator()() } #endif - boost::shared_ptr php(new PackageHandler(fIos, bs1, packageType, fEC, - fConcurrentSupport, maxDeleteRows, sessionID, 0, fDbrm, fQtc, csc)); - // We put the packageHandler into a map so that if we receive a - // message to affect the previous command, we can find it. - boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock, boost::defer_lock); + boost::shared_ptr php(new PackageHandler( + fIos, bs1, packageType, fEC, fConcurrentSupport, maxDeleteRows, sessionID, 0, fDbrm, fQtc, csc)); + // We put the packageHandler into a map so that if we receive a + // message to affect the previous command, we can find it. + boost::mutex::scoped_lock lk2(DMLProcessor::packageHandlerMapLock, boost::defer_lock); - lk2.lock(); - packageHandlerMap[sessionID] = php; - lk2.unlock(); + lk2.lock(); + packageHandlerMap[sessionID] = php; + lk2.unlock(); - php->run(); // Operates in this thread. + php->run(); // Operates in this thread. -// Move this to the end of PackageHandler so it is removed from the map before the response is sent -// lk2.lock(); -// packageHandlerMap.erase(sessionID); -// lk2.unlock(); - } - } - } - catch (std::exception& ex) - { - ostringstream oss; - oss << "DMLProcessor failed on: " << ex.what(); - DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); - fIos.close(); - } - catch (...) - { - ostringstream oss; - oss << "DMLProcessor failed on: processing DMLPackage."; - DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); - cerr << "Caught unknown exception! " << oss.str(); - fIos.close(); + // Move this to the end of PackageHandler so it is removed from the map before the response is sent + // lk2.lock(); + // packageHandlerMap.erase(sessionID); + // lk2.unlock(); + } } + } + catch (std::exception& ex) + { + ostringstream oss; + oss << "DMLProcessor failed on: " << ex.what(); + DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); + fIos.close(); + } + catch (...) + { + ostringstream oss; + oss << "DMLProcessor failed on: processing DMLPackage."; + DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); + cerr << "Caught unknown exception! " << oss.str(); + fIos.close(); + } } -void RollbackTransactionProcessor::processBulkRollback (BRM::TableLockInfo lockInfo, BRM::DBRM* dbrm, uint64_t uniqueId, - OamCache::dbRootPMMap_t& dbRootPMMap, bool& lockReleased) +void RollbackTransactionProcessor::processBulkRollback(BRM::TableLockInfo lockInfo, BRM::DBRM* dbrm, + uint64_t uniqueId, + OamCache::dbRootPMMap_t& dbRootPMMap, + bool& lockReleased) { - // Take over ownership of stale lock. - // Use "DMLProc" as process name, session id and transaction id -1 to distinguish from real DMLProc rollback - int32_t sessionID = -1; - int32_t txnid = -1; - std::string processName("DMLProc"); - uint32_t processID = ::getpid(); - bool ownerChanged = true; - lockReleased = true; + // Take over ownership of stale lock. + // Use "DMLProc" as process name, session id and transaction id -1 to distinguish from real DMLProc rollback + int32_t sessionID = -1; + int32_t txnid = -1; + std::string processName("DMLProc"); + uint32_t processID = ::getpid(); + bool ownerChanged = true; + lockReleased = true; - try - { - ownerChanged = dbrm->changeOwner(lockInfo.id, processName, processID, sessionID, txnid); - } - catch (std::exception&) - { - lockReleased = false; - throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); - } + try + { + ownerChanged = dbrm->changeOwner(lockInfo.id, processName, processID, sessionID, txnid); + } + catch (std::exception&) + { + lockReleased = false; + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE)); + } - if (!ownerChanged) - { - lockReleased = false; - throw std::runtime_error( std::string("Unable to grab lock; lock not found or still in use.") ); - } + if (!ownerChanged) + { + lockReleased = false; + throw std::runtime_error(std::string("Unable to grab lock; lock not found or still in use.")); + } - //send to all PMs - boost::shared_ptr bsIn; - messageqcpp::ByteStream bsOut; - string tableName(""); - fWEClient->addQueue(uniqueId); - //find the PMs need to send the message to - std::set pmSet; - int pmId; + // send to all PMs + boost::shared_ptr bsIn; + messageqcpp::ByteStream bsOut; + string tableName(""); + fWEClient->addQueue(uniqueId); + // find the PMs need to send the message to + std::set pmSet; + int pmId; - for (uint32_t i = 0; i < lockInfo.dbrootList.size(); i++) - { - pmId = (*dbRootPMMap)[lockInfo.dbrootList[i]]; - pmSet.insert(pmId); - } + for (uint32_t i = 0; i < lockInfo.dbrootList.size(); i++) + { + pmId = (*dbRootPMMap)[lockInfo.dbrootList[i]]; + pmSet.insert(pmId); + } - if (lockInfo.state == BRM::LOADING) - { - bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK; - bsOut << uniqueId; - bsOut << lockInfo.id; - bsOut << lockInfo.tableOID; - bsOut << tableName; - bsOut << processName; - std::set::const_iterator iter = pmSet.begin(); - - while (iter != pmSet.end()) - { - fWEClient->write(bsOut, *iter); - iter++; - } - - // Wait for "all" the responses, and accumulate any/all errors - unsigned int pmMsgCnt = 0; - - while (pmMsgCnt < pmSet.size()) - { - std::string rollbackErrMsg; - bsIn.reset(new messageqcpp::ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if (bsIn->length() == 0) - { - fWEClient->removeQueue(uniqueId); - lockReleased = false; - throw std::runtime_error("Network error, PM rollback; "); - } - else - { - messageqcpp::ByteStream::byte rc; - uint16_t pmNum; - *bsIn >> rc; - *bsIn >> rollbackErrMsg; - *bsIn >> pmNum; - - if (rc != 0) - { - fWEClient->removeQueue(uniqueId); - lockReleased = false; - throw std::runtime_error(rollbackErrMsg); - } - } - - pmMsgCnt++; - } // end of while loop to process all responses to bulk rollback - - // If no errors so far, then change state to CLEANUP state. - // We ignore the return stateChange flag. - dbrm->changeState( lockInfo.id, BRM::CLEANUP ); - } // end of (lockInfo.state == BRM::LOADING) - - //delete meta data backup rollback files - bsOut.reset(); - - bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK_CLEANUP; + if (lockInfo.state == BRM::LOADING) + { + bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK; bsOut << uniqueId; + bsOut << lockInfo.id; bsOut << lockInfo.tableOID; + bsOut << tableName; + bsOut << processName; std::set::const_iterator iter = pmSet.begin(); while (iter != pmSet.end()) { - fWEClient->write(bsOut, *iter); - iter++; + fWEClient->write(bsOut, *iter); + iter++; } // Wait for "all" the responses, and accumulate any/all errors unsigned int pmMsgCnt = 0; - //@Bug 4517 Release tablelock when it is in CLEANUP state - uint32_t rcCleanup = 0; - std::string fileDeleteErrMsg; while (pmMsgCnt < pmSet.size()) { - bsIn.reset(new messageqcpp::ByteStream()); - fWEClient->read(uniqueId, bsIn); + std::string rollbackErrMsg; + bsIn.reset(new messageqcpp::ByteStream()); + fWEClient->read(uniqueId, bsIn); - if (bsIn->length() == 0) + if (bsIn->length() == 0) + { + fWEClient->removeQueue(uniqueId); + lockReleased = false; + throw std::runtime_error("Network error, PM rollback; "); + } + else + { + messageqcpp::ByteStream::byte rc; + uint16_t pmNum; + *bsIn >> rc; + *bsIn >> rollbackErrMsg; + *bsIn >> pmNum; + + if (rc != 0) { - fWEClient->removeQueue(uniqueId); - rcCleanup = 1; - fileDeleteErrMsg = "Network error, PM clean up; "; + fWEClient->removeQueue(uniqueId); + lockReleased = false; + throw std::runtime_error(rollbackErrMsg); } - else - { - messageqcpp::ByteStream::byte rc; - uint16_t pmNum; - *bsIn >> rc; - *bsIn >> fileDeleteErrMsg; - *bsIn >> pmNum; + } - if ((rc != 0) && (rcCleanup == 0)) - { - fWEClient->removeQueue(uniqueId); - rcCleanup = rc; - } - } + pmMsgCnt++; + } // end of while loop to process all responses to bulk rollback - pmMsgCnt++; - } // end of while loop to process all responses to rollback cleanup + // If no errors so far, then change state to CLEANUP state. + // We ignore the return stateChange flag. + dbrm->changeState(lockInfo.id, BRM::CLEANUP); + } // end of (lockInfo.state == BRM::LOADING) - fWEClient->removeQueue(uniqueId); - // We ignore return release flag from releaseTableLock(). - dbrm->releaseTableLock( lockInfo.id ); + // delete meta data backup rollback files + bsOut.reset(); - if (rcCleanup != 0) - throw std::runtime_error(fileDeleteErrMsg); + bsOut << (messageqcpp::ByteStream::byte)WE_SVR_DML_BULKROLLBACK_CLEANUP; + bsOut << uniqueId; + bsOut << lockInfo.tableOID; + std::set::const_iterator iter = pmSet.begin(); + + while (iter != pmSet.end()) + { + fWEClient->write(bsOut, *iter); + iter++; + } + + // Wait for "all" the responses, and accumulate any/all errors + unsigned int pmMsgCnt = 0; + //@Bug 4517 Release tablelock when it is in CLEANUP state + uint32_t rcCleanup = 0; + std::string fileDeleteErrMsg; + + while (pmMsgCnt < pmSet.size()) + { + bsIn.reset(new messageqcpp::ByteStream()); + fWEClient->read(uniqueId, bsIn); + + if (bsIn->length() == 0) + { + fWEClient->removeQueue(uniqueId); + rcCleanup = 1; + fileDeleteErrMsg = "Network error, PM clean up; "; + } + else + { + messageqcpp::ByteStream::byte rc; + uint16_t pmNum; + *bsIn >> rc; + *bsIn >> fileDeleteErrMsg; + *bsIn >> pmNum; + + if ((rc != 0) && (rcCleanup == 0)) + { + fWEClient->removeQueue(uniqueId); + rcCleanup = rc; + } + } + + pmMsgCnt++; + } // end of while loop to process all responses to rollback cleanup + + fWEClient->removeQueue(uniqueId); + // We ignore return release flag from releaseTableLock(). + dbrm->releaseTableLock(lockInfo.id); + + if (rcCleanup != 0) + throw std::runtime_error(fileDeleteErrMsg); } void DMLProcessor::log(const std::string& msg, logging::LOG_TYPE level) { - logging::Message::Args args; - logging::Message message(2); - args.add(msg); - message.format(args); - logging::LoggingID lid(20); - logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message message(2); + args.add(msg); + message.format(args); + logging::LoggingID lid(20); + logging::MessageLog ml(lid); - switch (level) - { - case LOG_TYPE_DEBUG: - ml.logDebugMessage(message); - break; + switch (level) + { + case LOG_TYPE_DEBUG: ml.logDebugMessage(message); break; - case LOG_TYPE_INFO: - ml.logInfoMessage(message); - break; + case LOG_TYPE_INFO: ml.logInfoMessage(message); break; - case LOG_TYPE_WARNING: - ml.logWarningMessage(message); - break; + case LOG_TYPE_WARNING: ml.logWarningMessage(message); break; - case LOG_TYPE_ERROR: - ml.logErrorMessage(message); - break; + case LOG_TYPE_ERROR: ml.logErrorMessage(message); break; - case LOG_TYPE_CRITICAL: - ml.logCriticalMessage(message); - break; - } + case LOG_TYPE_CRITICAL: ml.logCriticalMessage(message); break; + } } -} //namespace dmlprocessor +} // namespace dmlprocessor // vim:ts=4 sw=4: - diff --git a/dmlproc/dmlprocessor.h b/dmlproc/dmlprocessor.h index 51676943f..b1863600c 100644 --- a/dmlproc/dmlprocessor.h +++ b/dmlproc/dmlprocessor.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlprocessor.h 955 2013-03-20 19:37:27Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlprocessor.h 955 2013-03-20 19:37:27Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DMLPROCESSOR_H #define DMLPROCESSOR_H @@ -45,289 +45,291 @@ #include "batchinsertprocessor.h" #include "querytele.h" -template > +template > class iterable_queue : public std::queue { -public: - typedef typename Container::iterator iterator; - typedef typename Container::const_iterator const_iterator; + public: + typedef typename Container::iterator iterator; + typedef typename Container::const_iterator const_iterator; - iterator begin() - { - return this->c.begin(); - } - iterator end() - { - return this->c.end(); - } - const_iterator begin() const - { - return this->c.begin(); - } - const_iterator end() const - { - return this->c.end(); - } - iterator find(T t) - { - iterator it; + iterator begin() + { + return this->c.begin(); + } + iterator end() + { + return this->c.end(); + } + const_iterator begin() const + { + return this->c.begin(); + } + const_iterator end() const + { + return this->c.end(); + } + iterator find(T t) + { + iterator it; - for (it = begin(); it != end(); ++it) - { - if (*it == t) - { - break; - } - } - - return it; - } - iterator erase(typename Container::iterator it) + for (it = begin(); it != end(); ++it) { - return this->c.erase(it); + if (*it == t) + { + break; + } } - iterator erase(T t) + + return it; + } + iterator erase(typename Container::iterator it) + { + return this->c.erase(it); + } + iterator erase(T t) + { + iterator it = find(t); + + if (it != end()) { - iterator it = find(t); - - if (it != end()) - { - erase(it); - } - - return it; + erase(it); } + + return it; + } }; namespace dmlprocessor { - /** @brief main server thread - */ + */ class DMLServer { -public: - DMLServer(int packageMaxThreads, int packageWorkQueueSize, BRM::DBRM* aDbrm); + public: + DMLServer(int packageMaxThreads, int packageWorkQueueSize, BRM::DBRM* aDbrm); - ~DMLServer() { } + ~DMLServer() + { + } + int start(); - int start(); + /** @brief get the dml package thread pool size + */ + inline int getPackageThreadPoolSize() const + { + return fPackageMaxThreads; + } - /** @brief get the dml package thread pool size - */ - inline int getPackageThreadPoolSize() const - { - return fPackageMaxThreads; - } + /** @brief set the dml package thread pool size + * + * @param threadPoolSize the maximum number of threads to process dml packages + */ + inline void setPackageThreadPoolSize(int threadPoolSize) + { + fPackageMaxThreads = threadPoolSize; + } - /** @brief set the dml package thread pool size - * - * @param threadPoolSize the maximum number of threads to process dml packages - */ - inline void setPackageThreadPoolSize( int threadPoolSize ) - { - fPackageMaxThreads = threadPoolSize; - } + /** @brief get the maximum number of dml packages allowed in the work queue + */ + inline int getPackageWorkQueueSize() const + { + return fPackageWorkQueueSize; + } - /** @brief get the maximum number of dml packages allowed in the work queue - */ - inline int getPackageWorkQueueSize() const - { - return fPackageWorkQueueSize; - } + /** @brief set the dml package work queue size + * + * @param workQueueSize the maximum number of dml packages in the work queue + */ + inline void setPackageWorkQueueSize(int workQueueSize) + { + fPackageWorkQueueSize = workQueueSize; + } - /** @brief set the dml package work queue size - * - * @param workQueueSize the maximum number of dml packages in the work queue - */ - inline void setPackageWorkQueueSize( int workQueueSize ) - { - fPackageWorkQueueSize = workQueueSize; - } + inline bool pendingShutdown() const + { + return fShutdownFlag; + } - inline bool pendingShutdown() const - { - return fShutdownFlag; - } + inline void startShutdown() + { + fShutdownFlag = true; + } - inline void startShutdown() - { - fShutdownFlag = true; - } + private: + // not copyable + DMLServer(const DMLServer& rhs); + DMLServer& operator=(const DMLServer& rhs); -private: - //not copyable - DMLServer(const DMLServer& rhs); - DMLServer& operator=(const DMLServer& rhs); + int fPackageMaxThreads; /** @brief max number of threads to process dml packages */ + int fPackageWorkQueueSize; /** @brief max number of packages waiting in the work queue */ - int fPackageMaxThreads; /** @brief max number of threads to process dml packages */ - int fPackageWorkQueueSize; /** @brief max number of packages waiting in the work queue */ + boost::scoped_ptr fMqServer; + BRM::DBRM* fDbrm; + bool fShutdownFlag; - boost::scoped_ptr fMqServer; - BRM::DBRM* fDbrm; - bool fShutdownFlag; - -public: - /** @brief the thread pool for processing dml packages - */ - static threadpool::ThreadPool fDmlPackagepool; + public: + /** @brief the thread pool for processing dml packages + */ + static threadpool::ThreadPool fDmlPackagepool; }; /** @brief Thread to process a single dml package. * Created and run from DMLProcessor - */ + */ class PackageHandler { -public: - PackageHandler(const messageqcpp::IOSocket& ios, boost::shared_ptr bs, - uint8_t packageType, joblist::DistributedEngineComm* ec, bool concurrentSuport, uint64_t maxDeleteRows, - uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm, - const querytele::QueryTeleClient& qtc, boost::shared_ptr csc); - ~PackageHandler(); + public: + PackageHandler(const messageqcpp::IOSocket& ios, boost::shared_ptr bs, + uint8_t packageType, joblist::DistributedEngineComm* ec, bool concurrentSuport, + uint64_t maxDeleteRows, uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnId, + BRM::DBRM* aDbrm, const querytele::QueryTeleClient& qtc, + boost::shared_ptr csc); + ~PackageHandler(); - void run(); - void rollbackPending(); + void run(); + void rollbackPending(); - execplan::CalpontSystemCatalog::SCN getTxnid() + execplan::CalpontSystemCatalog::SCN getTxnid() + { + return fTxnid; + } + uint32_t getSessionID() + { + return fSessionID; + } + + private: + messageqcpp::IOSocket fIos; + boost::shared_ptr fByteStream; + boost::scoped_ptr fProcessor; + messageqcpp::ByteStream::quadbyte fPackageType; + joblist::DistributedEngineComm* fEC; + bool fConcurrentSupport; + uint64_t fMaxDeleteRows; + uint32_t fSessionID; + uint32_t fTableOid; + execplan::CalpontSystemCatalog::SCN fTxnid; + execplan::SessionManager sessionManager; + BRM::DBRM* fDbrm; + querytele::QueryTeleClient fQtc; + boost::shared_ptr fcsc; + + // MCOL-140 A map to hold table oids so that we can wait for DML on a table. + // This effectively creates a table lock for transactions. + // Used to serialize operations because the VSS can't handle inserts + // or updates on the same block. + // When an Insert, Update or Delete command arrives, we look here + // for the table oid. If found, wait until it is no longer here. + // If this transactionID (SCN) is < the transactionID in the table, don't delay + // and hope for the best, as we're already out of order. + // When the VSS is engineered to handle transactions out of order, all MCOL-140 + // code is to be removed. + int synchTableAccess(dmlpackage::CalpontDMLPackage* dmlPackage); + int releaseTableAccess(); + int forceReleaseTableAccess(); + typedef iterable_queue tableAccessQueue_t; + static std::map tableOidMap; + static boost::condition_variable tableOidCond; + static boost::mutex tableOidMutex; + + public: + static int clearTableAccess(); + + // MCOL-3296 Add a class to call synchTableAccess on creation and + // releaseTableAccess on destuction for exception safeness. + class SynchTable + { + public: + SynchTable() : fphp(NULL){}; + SynchTable(PackageHandler* php, dmlpackage::CalpontDMLPackage* dmlPackage) { - return fTxnid; + setPackage(php, dmlPackage); } - uint32_t getSessionID() + ~SynchTable() { - return fSessionID; + if (fphp) + fphp->releaseTableAccess(); + } + bool setPackage(PackageHandler* php, dmlpackage::CalpontDMLPackage* dmlPackage) + { + if (fphp) + fphp->releaseTableAccess(); + fphp = php; + if (fphp) + fphp->synchTableAccess(dmlPackage); + return true; } -private: - messageqcpp::IOSocket fIos; - boost::shared_ptr fByteStream; - boost::scoped_ptr fProcessor; - messageqcpp::ByteStream::quadbyte fPackageType; - joblist::DistributedEngineComm* fEC; - bool fConcurrentSupport; - uint64_t fMaxDeleteRows; - uint32_t fSessionID; - uint32_t fTableOid; - execplan::CalpontSystemCatalog::SCN fTxnid; - execplan::SessionManager sessionManager; - BRM::DBRM* fDbrm; - querytele::QueryTeleClient fQtc; - boost::shared_ptr fcsc; - - // MCOL-140 A map to hold table oids so that we can wait for DML on a table. - // This effectively creates a table lock for transactions. - // Used to serialize operations because the VSS can't handle inserts - // or updates on the same block. - // When an Insert, Update or Delete command arrives, we look here - // for the table oid. If found, wait until it is no longer here. - // If this transactionID (SCN) is < the transactionID in the table, don't delay - // and hope for the best, as we're already out of order. - // When the VSS is engineered to handle transactions out of order, all MCOL-140 - // code is to be removed. - int synchTableAccess(dmlpackage::CalpontDMLPackage* dmlPackage); - int releaseTableAccess(); - int forceReleaseTableAccess(); - typedef iterable_queue tableAccessQueue_t; - static std::map tableOidMap; - static boost::condition_variable tableOidCond; - static boost::mutex tableOidMutex; -public: - static int clearTableAccess(); - - // MCOL-3296 Add a class to call synchTableAccess on creation and - // releaseTableAccess on destuction for exception safeness. - class SynchTable - { - public: - SynchTable() : fphp(NULL) {}; - SynchTable(PackageHandler* php, dmlpackage::CalpontDMLPackage* dmlPackage) - { - setPackage(php, dmlPackage); - } - ~SynchTable() - { - if (fphp) - fphp->releaseTableAccess(); - } - bool setPackage(PackageHandler* php, dmlpackage::CalpontDMLPackage* dmlPackage) - { - if (fphp) - fphp->releaseTableAccess(); - fphp = php; - if (fphp) - fphp->synchTableAccess(dmlPackage); - return true; - } - private: - PackageHandler* fphp; - }; - + private: + PackageHandler* fphp; + }; }; /** @brief processes dml packages as they arrive - */ + */ class DMLProcessor { -public: - /** @brief ctor - * - * @param packageMaxThreads the maximum number of threads to process dml packages - * @param packageWorkQueueSize the maximum number of dml packages in the work queue - */ - DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm); + public: + /** @brief ctor + * + * @param packageMaxThreads the maximum number of threads to process dml packages + * @param packageWorkQueueSize the maximum number of dml packages in the work queue + */ + DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm); - /** @brief entry point for the DMLProcessor - */ - void operator()(); + /** @brief entry point for the DMLProcessor + */ + void operator()(); - static void log(const std::string& msg, logging::LOG_TYPE level); -private: - messageqcpp::IOSocket fIos; + static void log(const std::string& msg, logging::LOG_TYPE level); - execplan::SessionManager sessionManager; - boost::shared_ptr csc; - BRM::DBRM* fDbrm; - querytele::QueryTeleClient fQtc; - bool fConcurrentSupport; + private: + messageqcpp::IOSocket fIos; - // A map to hold pointers to all active PackageProcessors - typedef std::map > PackageHandlerMap_t; - static PackageHandlerMap_t packageHandlerMap; - static boost::mutex packageHandlerMapLock; + execplan::SessionManager sessionManager; + boost::shared_ptr csc; + BRM::DBRM* fDbrm; + querytele::QueryTeleClient fQtc; + bool fConcurrentSupport; - // A map to hold pointers to all BatchInsertProc object - static std::map batchinsertProcessorMap; - static boost::mutex batchinsertProcessorMapLock; + // A map to hold pointers to all active PackageProcessors + typedef std::map > PackageHandlerMap_t; + static PackageHandlerMap_t packageHandlerMap; + static boost::mutex packageHandlerMapLock; - friend struct CancellationThread; - friend class PackageHandler; + // A map to hold pointers to all BatchInsertProc object + static std::map batchinsertProcessorMap; + static boost::mutex batchinsertProcessorMapLock; + + friend struct CancellationThread; + friend class PackageHandler; }; class RollbackTransactionProcessor : public dmlpackageprocessor::DMLPackageProcessor { + public: + RollbackTransactionProcessor(BRM::DBRM* aDbrm) : DMLPackageProcessor(aDbrm, 1) + { + } + /** @brief process an Rollback transactions + * + * @param cpackage the UpdateDMLPackage to process + */ + inline DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage) + { + DMLResult result; + result.result = NO_ERROR; + return result; + } -public: - RollbackTransactionProcessor(BRM::DBRM* aDbrm) : DMLPackageProcessor(aDbrm, 1) {} - /** @brief process an Rollback transactions - * - * @param cpackage the UpdateDMLPackage to process - */ - inline DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage) - { - DMLResult result; - result.result = NO_ERROR; - return result; - } + void processBulkRollback(BRM::TableLockInfo lockInfo, BRM::DBRM* dbrm, uint64_t uniqueId, + oam::OamCache::dbRootPMMap_t& dbRootPMMap, bool& lockReleased); - void processBulkRollback (BRM::TableLockInfo lockInfo, BRM::DBRM* dbrm, uint64_t uniqueId, - oam::OamCache::dbRootPMMap_t& dbRootPMMap, bool& lockReleased); - -protected: + protected: }; +} // namespace dmlprocessor -} // namespace dmlprocessor - -#endif //DMLPROCESSOR_H +#endif // DMLPROCESSOR_H // vim:ts=4 sw=4: - diff --git a/dmlproc/dmlresultbuffer.cpp b/dmlproc/dmlresultbuffer.cpp index c70df4093..17fdeb093 100644 --- a/dmlproc/dmlresultbuffer.cpp +++ b/dmlproc/dmlresultbuffer.cpp @@ -16,70 +16,63 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlresultbuffer.cpp 927 2013-01-21 14:11:25Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlresultbuffer.cpp 927 2013-01-21 14:11:25Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #include "dmlresultbuffer.h" namespace dmlprocessor { - -DMLResultBuffer::DMLResultBuffer() - : fBufferSize(1), fFull(0) +DMLResultBuffer::DMLResultBuffer() : fBufferSize(1), fFull(0) { - } -DMLResultBuffer::DMLResultBuffer(int bufferSize) - : fBufferSize(bufferSize), fFull(0) +DMLResultBuffer::DMLResultBuffer(int bufferSize) : fBufferSize(bufferSize), fFull(0) { - } DMLResultBuffer::~DMLResultBuffer() { - fResultBuffer.clear(); + fResultBuffer.clear(); } void DMLResultBuffer::put(dmlpackageprocessor::DMLPackageProcessor::DMLResult result, int sessionID) { - scoped_lock lock(fMutex); + scoped_lock lock(fMutex); - if (fFull == fBufferSize) - { - while (fFull == fBufferSize) - fCond.wait(lock); - } + if (fFull == fBufferSize) + { + while (fFull == fBufferSize) + fCond.wait(lock); + } - ResultPair rp; - rp.result = result; - rp.sessionID = sessionID; + ResultPair rp; + rp.result = result; + rp.sessionID = sessionID; - fResultBuffer.push_back(rp); - ++fFull; - fCond.notify_one(); + fResultBuffer.push_back(rp); + ++fFull; + fCond.notify_one(); } DMLResultBuffer::ResultPair DMLResultBuffer::get() { - scoped_lock lk(fMutex); + scoped_lock lk(fMutex); - if (fFull == 0) - { - while (fFull == 0) - fCond.wait(lk); - } + if (fFull == 0) + { + while (fFull == 0) + fCond.wait(lk); + } - ResultPair rp = fResultBuffer[0]; - fResultBuffer.pop_front(); - --fFull; - fCond.notify_one(); - return rp; + ResultPair rp = fResultBuffer[0]; + fResultBuffer.pop_front(); + --fFull; + fCond.notify_one(); + return rp; } -} //namespace dmlprocessor - - +} // namespace dmlprocessor diff --git a/dmlproc/dmlresultbuffer.h b/dmlproc/dmlresultbuffer.h index 89360f906..50a372178 100644 --- a/dmlproc/dmlresultbuffer.h +++ b/dmlproc/dmlresultbuffer.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: dmlresultbuffer.h 927 2013-01-21 14:11:25Z rdempsey $ -* -* -***********************************************************************/ + * $Id: dmlresultbuffer.h 927 2013-01-21 14:11:25Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef DMLRESULTBUFFER_H @@ -32,65 +32,65 @@ namespace dmlprocessor { /** @brief holds dml results - */ + */ class DMLResultBuffer { -public: - typedef boost::mutex::scoped_lock scoped_lock; + public: + typedef boost::mutex::scoped_lock scoped_lock; - /** @brief the type of a pair - * - */ - struct ResultPair - { - dmlpackageprocessor::DMLPackageProcessor::DMLResult result; - int sessionID; - }; + /** @brief the type of a pair + * + */ + struct ResultPair + { + dmlpackageprocessor::DMLPackageProcessor::DMLResult result; + int sessionID; + }; - /** @brief ctor - */ - DMLResultBuffer(); + /** @brief ctor + */ + DMLResultBuffer(); - /** @brief ctor - * - */ - DMLResultBuffer(int bufferSize); + /** @brief ctor + * + */ + DMLResultBuffer(int bufferSize); - /** @brief dtor - * - */ - ~DMLResultBuffer(); + /** @brief dtor + * + */ + ~DMLResultBuffer(); - /** @brief set the size of the buffer - * - * @param bufferSize the size of the buffer - */ - inline void setBufferSize(int bufferSize) - { - fBufferSize = bufferSize; - } + /** @brief set the size of the buffer + * + * @param bufferSize the size of the buffer + */ + inline void setBufferSize(int bufferSize) + { + fBufferSize = bufferSize; + } - /** @brief put results in the buffer - * - */ - void put( dmlpackageprocessor::DMLPackageProcessor::DMLResult result, int sessionID ); + /** @brief put results in the buffer + * + */ + void put(dmlpackageprocessor::DMLPackageProcessor::DMLResult result, int sessionID); - /** @brief get results from the buffer - * - */ - ResultPair get(); + /** @brief get results from the buffer + * + */ + ResultPair get(); -private: - boost::mutex fMutex; - boost::condition fCond; + private: + boost::mutex fMutex; + boost::condition fCond; - typedef std::deque ResultBuffer; - ResultBuffer fResultBuffer; + typedef std::deque ResultBuffer; + ResultBuffer fResultBuffer; - int fBufferSize; - int fFull; + int fBufferSize; + int fFull; }; -} //namespace dmlprocessor +} // namespace dmlprocessor -#endif //DMLRESULTPROCESSOR_H +#endif // DMLRESULTPROCESSOR_H diff --git a/dmlproc/resource.h b/dmlproc/resource.h index cdcd581ec..4145319b9 100644 --- a/dmlproc/resource.h +++ b/dmlproc/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/exemgr/activestatementcounter.cpp b/exemgr/activestatementcounter.cpp index e267ad6b4..27a3df1de 100644 --- a/exemgr/activestatementcounter.cpp +++ b/exemgr/activestatementcounter.cpp @@ -26,36 +26,35 @@ using namespace boost; void ActiveStatementCounter::incr(bool& counted) { - if (counted) - return; + if (counted) + return; - counted = true; - boost::mutex::scoped_lock lk(fMutex); + counted = true; + boost::mutex::scoped_lock lk(fMutex); - if (upperLimit > 0) - while (fStatementCount >= upperLimit) - { - fStatementsWaiting++; - condvar.wait(lk); - --fStatementsWaiting; - } + if (upperLimit > 0) + while (fStatementCount >= upperLimit) + { + fStatementsWaiting++; + condvar.wait(lk); + --fStatementsWaiting; + } - fStatementCount++; + fStatementCount++; } void ActiveStatementCounter::decr(bool& counted) { - if (!counted) - return; + if (!counted) + return; - counted = false; - boost::mutex::scoped_lock lk(fMutex); + counted = false; + boost::mutex::scoped_lock lk(fMutex); - if (fStatementCount == 0) - return; + if (fStatementCount == 0) + return; - --fStatementCount; - condvar.notify_one(); + --fStatementCount; + condvar.notify_one(); } // vim:ts=4 sw=4: - diff --git a/exemgr/activestatementcounter.h b/exemgr/activestatementcounter.h index 9b465b5f3..e8e75c58a 100644 --- a/exemgr/activestatementcounter.h +++ b/exemgr/activestatementcounter.h @@ -31,38 +31,37 @@ class ActiveStatementCounter { -public: - ActiveStatementCounter(uint32_t limit) : - fStatementCount(0), - upperLimit(limit), - fStatementsWaiting(0) - {} + public: + ActiveStatementCounter(uint32_t limit) : fStatementCount(0), upperLimit(limit), fStatementsWaiting(0) + { + } - virtual ~ActiveStatementCounter() {} + virtual ~ActiveStatementCounter() + { + } - void incr(bool& counted); - void decr(bool& counted); - uint32_t cur() const - { - return fStatementCount; - } - uint32_t waiting() const - { - return fStatementsWaiting; - } + void incr(bool& counted); + void decr(bool& counted); + uint32_t cur() const + { + return fStatementCount; + } + uint32_t waiting() const + { + return fStatementsWaiting; + } -private: - ActiveStatementCounter(const ActiveStatementCounter& rhs); - ActiveStatementCounter& operator=(const ActiveStatementCounter& rhs); + private: + ActiveStatementCounter(const ActiveStatementCounter& rhs); + ActiveStatementCounter& operator=(const ActiveStatementCounter& rhs); - uint32_t fStatementCount; - uint32_t upperLimit; - uint32_t fStatementsWaiting; - boost::mutex fMutex; - boost::condition condvar; - BRM::VSS fVss; + uint32_t fStatementCount; + uint32_t upperLimit; + uint32_t fStatementsWaiting; + boost::mutex fMutex; + boost::condition condvar; + BRM::VSS fVss; }; #endif // vim:ts=4 sw=4: - diff --git a/exemgr/femsghandler.cpp b/exemgr/femsghandler.cpp index 89b6fbddf..cdb0cd81d 100644 --- a/exemgr/femsghandler.cpp +++ b/exemgr/femsghandler.cpp @@ -29,62 +29,63 @@ threadpool::ThreadPool FEMsgHandler::threadPool; namespace { - class Runner { -public: - Runner(FEMsgHandler* f) : target(f) { } - void operator()() - { - utils::setThreadName("FEMsgHandler"); - target->threadFcn(); - } - FEMsgHandler* target; + public: + Runner(FEMsgHandler* f) : target(f) + { + } + void operator()() + { + utils::setThreadName("FEMsgHandler"); + target->threadFcn(); + } + FEMsgHandler* target; }; -} +} // namespace FEMsgHandler::FEMsgHandler() : die(false), running(false), sawData(false), sock(NULL) { } -FEMsgHandler::FEMsgHandler(boost::shared_ptr j, IOSocket* s) : - die(false), running(false), sawData(false), jl(j) +FEMsgHandler::FEMsgHandler(boost::shared_ptr j, IOSocket* s) + : die(false), running(false), sawData(false), jl(j) { - sock = s; - assert(sock); + sock = s; + assert(sock); } FEMsgHandler::~FEMsgHandler() { - stop(); - threadPool.join(thr); + stop(); + threadPool.join(thr); } void FEMsgHandler::start() { - if (!running) - { - running = true; - thr = threadPool.invoke(Runner(this)); - } + if (!running) + { + running = true; + thr = threadPool.invoke(Runner(this)); + } } void FEMsgHandler::stop() { - die = true; - jl.reset(); + die = true; + jl.reset(); } void FEMsgHandler::setJobList(boost::shared_ptr j) { - jl = j; + jl = j; } void FEMsgHandler::setSocket(IOSocket* i) { - sock = i; - assert(sock); + sock = i; + assert(sock); } /* Note, the next two fcns strongly depend on ExeMgr's current implementation. There's a @@ -97,49 +98,48 @@ void FEMsgHandler::setSocket(IOSocket* i) */ bool FEMsgHandler::aborted() { - if (sawData) - return true; + if (sawData) + return true; - boost::mutex::scoped_lock sl(mutex); - int err; - int connectionNum = sock->getConnectionNum(); + boost::mutex::scoped_lock sl(mutex); + int err; + int connectionNum = sock->getConnectionNum(); - err = InetStreamSocket::pollConnection(connectionNum, 1000); + err = InetStreamSocket::pollConnection(connectionNum, 1000); - if (err == 1) - { - sawData = true; - return true; - } + if (err == 1) + { + sawData = true; + return true; + } - return false; + return false; } void FEMsgHandler::threadFcn() { - int err = 0; - int connectionNum = sock->getConnectionNum(); + int err = 0; + int connectionNum = sock->getConnectionNum(); - /* This waits for the next readable event on sock. An abort is signaled - * by sending something (anything at the moment), then dropping the connection. - * This fcn exits on all other events. - */ - while (!die && err == 0) - { - boost::mutex::scoped_lock sl(mutex); - err = InetStreamSocket::pollConnection(connectionNum, 1000); - } + /* This waits for the next readable event on sock. An abort is signaled + * by sending something (anything at the moment), then dropping the connection. + * This fcn exits on all other events. + */ + while (!die && err == 0) + { + boost::mutex::scoped_lock sl(mutex); + err = InetStreamSocket::pollConnection(connectionNum, 1000); + } - if (err == 1) - sawData = true; // there's data to read, must be the abort signal + if (err == 1) + sawData = true; // there's data to read, must be the abort signal - if (!die && (err == 2 || err == 1)) - { - die = true; - jl->abort(); - jl.reset(); - } + if (!die && (err == 2 || err == 1)) + { + die = true; + jl->abort(); + jl.reset(); + } - running = false; + running = false; } - diff --git a/exemgr/femsghandler.h b/exemgr/femsghandler.h index 603413d52..89a26e4d5 100644 --- a/exemgr/femsghandler.h +++ b/exemgr/femsghandler.h @@ -24,27 +24,27 @@ class FEMsgHandler { -public: - FEMsgHandler(); - FEMsgHandler(boost::shared_ptr, messageqcpp::IOSocket*); - virtual ~FEMsgHandler(); + public: + FEMsgHandler(); + FEMsgHandler(boost::shared_ptr, messageqcpp::IOSocket*); + virtual ~FEMsgHandler(); - void start(); - void stop(); - void setJobList(boost::shared_ptr); - void setSocket(messageqcpp::IOSocket*); - bool aborted(); + void start(); + void stop(); + void setJobList(boost::shared_ptr); + void setSocket(messageqcpp::IOSocket*); + bool aborted(); - void threadFcn(); + void threadFcn(); - static threadpool::ThreadPool threadPool; + static threadpool::ThreadPool threadPool; -private: - bool die, running, sawData; - messageqcpp::IOSocket* sock; - boost::shared_ptr jl; - boost::mutex mutex; - uint64_t thr; + private: + bool die, running, sawData; + messageqcpp::IOSocket* sock; + boost::shared_ptr jl; + boost::mutex mutex; + uint64_t thr; }; #endif /* FEMSGHANDLER_H_ */ diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 87cb102a4..513cc075e 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /********************************************************************** -* $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ -* -* -***********************************************************************/ + * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ + * + * + ***********************************************************************/ /** * @brief execution plan manager main program * @@ -83,99 +83,84 @@ class Opt { -public: - int m_debug; - bool m_e; - bool m_fg; - Opt(int argc, char *argv[]) - :m_debug(0), - m_e(false), - m_fg(false) + public: + int m_debug; + bool m_e; + bool m_fg; + Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "edf")) != EOF) { - int c; - while ((c = getopt(argc, argv, "edf")) != EOF) - { - switch (c) - { - case 'd': - m_debug++; - break; + switch (c) + { + case 'd': m_debug++; break; - case 'e': - m_e= true; - break; + case 'e': m_e = true; break; - case 'f': - m_fg= true; - break; + case 'f': m_fg = true; break; - case '?': - default: - break; - } - } + case '?': + default: break; + } } + } }; - -class ServiceExeMgr: public Service, public Opt +class ServiceExeMgr : public Service, public Opt { -protected: + protected: + void log(logging::LOG_TYPE type, const std::string& str) + { + logging::LoggingID logid(16); + logging::Message::Args args; + logging::Message message(8); + args.add(strerror(errno)); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(type, message, logid); + } - void log(logging::LOG_TYPE type, const std::string &str) - { - logging::LoggingID logid(16); - logging::Message::Args args; - logging::Message message(8); - args.add(strerror(errno)); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(type, message, logid); - } - -public: - ServiceExeMgr(const Opt &opt) - :Service("ExeMgr"), Opt(opt) - { } - void LogErrno() override - { - log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string &str) override - { - log(logging::LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt) + { + } + void LogErrno() override + { + log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(logging::LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - namespace { - -//If any flags other than the table mode flags are set, produce output to screeen -const uint32_t flagsWantOutput = (0xffffffff & - ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & +// If any flags other than the table mode flags are set, produce output to screeen +const uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); int gDebug; -const unsigned logDefaultMsg = logging::M0000; +const unsigned logDefaultMsg = logging::M0000; const unsigned logDbProfStartStatement = logging::M0028; -const unsigned logDbProfEndStatement = logging::M0029; -const unsigned logStartSql = logging::M0041; -const unsigned logEndSql = logging::M0042; -const unsigned logRssTooBig = logging::M0044; -const unsigned logDbProfQueryStats = logging::M0047; -const unsigned logExeMgrExcpt = logging::M0055; +const unsigned logDbProfEndStatement = logging::M0029; +const unsigned logStartSql = logging::M0041; +const unsigned logEndSql = logging::M0042; +const unsigned logRssTooBig = logging::M0044; +const unsigned logDbProfQueryStats = logging::M0047; +const unsigned logExeMgrExcpt = logging::M0055; logging::Logger msgLog(16); typedef std::map SessionMemMap_t; -SessionMemMap_t sessionMemMap; // track memory% usage during a query +SessionMemMap_t sessionMemMap; // track memory% usage during a query std::mutex sessionMemMapMutex; //...The FrontEnd may establish more than 1 connection (which results in @@ -185,10 +170,10 @@ std::mutex sessionMemMapMutex; // that we know when we can safely delete a CalpontSystemCatalog object // shared by multiple threads per session. typedef std::map ThreadCntPerSessionMap_t; -ThreadCntPerSessionMap_t threadCntPerSessionMap; +ThreadCntPerSessionMap_t threadCntPerSessionMap; std::mutex threadCntPerSessionMapMutex; -//This var is only accessed using thread-safe inc/dec calls +// This var is only accessed using thread-safe inc/dec calls ActiveStatementCounter* statementsRunningCount; joblist::DistributedEngineComm* ec; @@ -197,1399 +182,1371 @@ auto rm = joblist::ResourceManager::instance(true); int toInt(const std::string& val) { - if (val.length() == 0) return -1; + if (val.length() == 0) + return -1; - return static_cast(config::Config::fromText(val)); + return static_cast(config::Config::fromText(val)); } const std::string ExeMgr("ExeMgr1"); const std::string prettyPrintMiniInfo(const std::string& in) { - //1. take the std::string and tok it by '\n' - //2. for each part in each line calc the longest part - //3. padding to each longest value, output a header and the lines - typedef boost::tokenizer > my_tokenizer; - boost::char_separator sep1("\n"); - my_tokenizer tok1(in, sep1); - std::vector lines; - std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; - const int header_parts = 10; - lines.push_back(header); + // 1. take the std::string and tok it by '\n' + // 2. for each part in each line calc the longest part + // 3. padding to each longest value, output a header and the lines + typedef boost::tokenizer > my_tokenizer; + boost::char_separator sep1("\n"); + my_tokenizer tok1(in, sep1); + std::vector lines; + std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; + const int header_parts = 10; + lines.push_back(header); - for (my_tokenizer::const_iterator iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) + for (my_tokenizer::const_iterator iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) + { + if (!iter1->empty()) + lines.push_back(*iter1); + } + + std::vector lens; + + for (int i = 0; i < header_parts; i++) + lens.push_back(0); + + std::vector > lineparts; + std::vector::iterator iter2; + int j; + + for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) + { + boost::char_separator sep2(" "); + my_tokenizer tok2(*iter2, sep2); + int i; + std::vector parts; + my_tokenizer::iterator iter3; + + for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) { - if (!iter1->empty()) - lines.push_back(*iter1); + if (i >= header_parts) + break; + + std::string part(*iter3); + + if (j != 0 && i == 8) + part.resize(part.size() - 3); + + assert(i < header_parts); + + if (part.size() > lens[i]) + lens[i] = part.size(); + + parts.push_back(part); } - std::vector lens; + assert(i == header_parts); + lineparts.push_back(parts); + } - for (int i = 0; i < header_parts; i++) - lens.push_back(0); + std::ostringstream oss; - std::vector > lineparts; - std::vector::iterator iter2; - int j; + std::vector >::iterator iter1 = lineparts.begin(); + std::vector >::iterator end1 = lineparts.end(); - for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) + oss << "\n"; + + while (iter1 != end1) + { + std::vector::iterator iter2 = iter1->begin(); + std::vector::iterator end2 = iter1->end(); + assert(distance(iter2, end2) == header_parts); + int i = 0; + + while (iter2 != end2) { - boost::char_separator sep2(" "); - my_tokenizer tok2(*iter2, sep2); - int i; - std::vector parts; - my_tokenizer::iterator iter3; - - for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) - { - if (i >= header_parts) break; - - std::string part(*iter3); - - if (j != 0 && i == 8) - part.resize(part.size() - 3); - - assert(i < header_parts); - - if (part.size() > lens[i]) - lens[i] = part.size(); - - parts.push_back(part); - } - - assert(i == header_parts); - lineparts.push_back(parts); + assert(i < header_parts); + oss << std::setw(lens[i]) << std::left << *iter2 << " "; + ++iter2; + i++; } - std::ostringstream oss; - - std::vector >::iterator iter1 = lineparts.begin(); - std::vector >::iterator end1 = lineparts.end(); - oss << "\n"; + ++iter1; + } - while (iter1 != end1) - { - std::vector::iterator iter2 = iter1->begin(); - std::vector::iterator end2 = iter1->end(); - assert(distance(iter2, end2) == header_parts); - int i = 0; - - while (iter2 != end2) - { - assert(i < header_parts); - oss << std::setw(lens[i]) << std::left << *iter2 << " "; - ++iter2; - i++; - } - - oss << "\n"; - ++iter1; - } - - return oss.str(); + return oss.str(); } const std::string timeNow() { - time_t outputTime = time(0); - struct tm ltm; - char buf[32]; //ctime(3) says at least 26 - size_t len = 0; + time_t outputTime = time(0); + struct tm ltm; + char buf[32]; // ctime(3) says at least 26 + size_t len = 0; #ifdef _MSC_VER - asctime_s(buf, 32, localtime_r(&outputTime, <m)); + asctime_s(buf, 32, localtime_r(&outputTime, <m)); #else - asctime_r(localtime_r(&outputTime, <m), buf); + asctime_r(localtime_r(&outputTime, <m), buf); #endif - len = strlen(buf); + len = strlen(buf); - if (len > 0) --len; + if (len > 0) + --len; - if (buf[len] == '\n') buf[len] = 0; + if (buf[len] == '\n') + buf[len] = 0; - return buf; + return buf; } querytele::QueryTeleServerParms gTeleServerParms; class SessionThread { -public: + public: + SessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, + joblist::ResourceManager* rm) + : fIos(ios) + , fEc(ec) + , fRm(rm) + , fStatsRetrieved(false) + , fTeleClient(gTeleServerParms) + , fOamCachePtr(oam::OamCache::makeOamCache()) + { + } - SessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, joblist::ResourceManager* rm) : - fIos(ios), fEc(ec), - fRm(rm), - fStatsRetrieved(false), - fTeleClient(gTeleServerParms), - fOamCachePtr(oam::OamCache::makeOamCache()) + private: + messageqcpp::IOSocket fIos; + joblist::DistributedEngineComm* fEc; + joblist::ResourceManager* fRm; + querystats::QueryStats fStats; + + // Variables used to store return stats + bool fStatsRetrieved; + + querytele::QueryTeleClient fTeleClient; + + oam::OamCache* fOamCachePtr; // this ptr is copyable... + + //...Reinitialize stats for start of a new query + void initStats(uint32_t sessionId, std::string& sqlText) + { + initMaxMemPct(sessionId); + + fStats.reset(); + fStats.setStartTime(); + fStats.fSessionID = sessionId; + fStats.fQuery = sqlText; + fStatsRetrieved = false; + } + + //...Get % memory usage during latest query for sesssionId. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static uint64_t getMaxMemPct(uint32_t sessionId) + { + uint64_t maxMemoryPct = 0; + + if (sessionId < 0x80000000) { + std::lock_guard lk(sessionMemMapMutex); + SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); + + if (mapIter != sessionMemMap.end()) + { + maxMemoryPct = (uint64_t)mapIter->second; + } } -private: + return maxMemoryPct; + } - messageqcpp::IOSocket fIos; - joblist::DistributedEngineComm* fEc; - joblist::ResourceManager* fRm; - querystats::QueryStats fStats; - - // Variables used to store return stats - bool fStatsRetrieved; - - querytele::QueryTeleClient fTeleClient; - - oam::OamCache* fOamCachePtr; //this ptr is copyable... - - //...Reinitialize stats for start of a new query - void initStats ( uint32_t sessionId, std::string& sqlText ) + //...Delete sessionMemMap entry for the specified session's memory % use. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void deleteMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) { - initMaxMemPct ( sessionId ); + std::lock_guard lk(sessionMemMapMutex); + SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - fStats.reset(); - fStats.setStartTime(); - fStats.fSessionID = sessionId; - fStats.fQuery = sqlText; - fStatsRetrieved = false; + if (mapIter != sessionMemMap.end()) + { + sessionMemMap.erase(sessionId); + } } + } - //...Get % memory usage during latest query for sesssionId. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static uint64_t getMaxMemPct ( uint32_t sessionId ) + //...Get and log query stats to specified output stream + const std::string formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) + { + std::ostringstream os; + + // Get stats if not already acquired for current query + if (!fStatsRetrieved) { - uint64_t maxMemoryPct = 0; - - if ( sessionId < 0x80000000 ) - { - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = - sessionMemMap.find( sessionId ); - - if ( mapIter != sessionMemMap.end() ) - { - maxMemoryPct = (uint64_t)mapIter->second; - } - } - - return maxMemoryPct; - } - - //...Delete sessionMemMap entry for the specified session's memory % use. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void deleteMaxMemPct ( uint32_t sessionId ) - { - if ( sessionId < 0x80000000 ) - { - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = - sessionMemMap.find( sessionId ); - - if ( mapIter != sessionMemMap.end() ) - { - sessionMemMap.erase( sessionId ); - } - } - } - - //...Get and log query stats to specified output stream - const std::string formatQueryStats ( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine,//include line breaks in query stats std::string - bool vtableModeOn, - bool wantExtendedStats, - uint64_t rowsReturned) - { - std::ostringstream os; - - // Get stats if not already acquired for current query - if ( !fStatsRetrieved ) - { - if (wantExtendedStats) - { - //wait for the ei data to be written by another thread (brain-dead) - struct timespec req = { 0, 250000 }; //250 usec + if (wantExtendedStats) + { + // wait for the ei data to be written by another thread (brain-dead) + struct timespec req = {0, 250000}; // 250 usec #ifdef _MSC_VER - Sleep(20); //20ms on Windows + Sleep(20); // 20ms on Windows #else - nanosleep(&req, 0); + nanosleep(&req, 0); #endif - } + } - // Get % memory usage during current query for sessionId - jl->querySummary( wantExtendedStats ); - fStats = jl->queryStats(); - fStats.fMaxMemPct = getMaxMemPct( fStats.fSessionID ); - fStats.fRows = rowsReturned; - fStatsRetrieved = true; + // Get % memory usage during current query for sessionId + jl->querySummary(wantExtendedStats); + fStats = jl->queryStats(); + fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); + fStats.fRows = rowsReturned; + fStatsRetrieved = true; + } + + std::string queryMode; + queryMode = (vtableModeOn ? "Distributed" : "Standard"); + + // Log stats to specified output stream + os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles + << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO + << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; + + if (includeNewLine) + os << std::endl << " "; // insert line break + else + os << "; "; // continue without line break + + os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" + << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" + << queryMode; + + return os.str(); + } + + //...Increment the number of threads using the specified sessionId + static void incThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex); + ThreadCntPerSessionMap_t::iterator mapIter = threadCntPerSessionMap.find(sessionId); + + if (mapIter == threadCntPerSessionMap.end()) + threadCntPerSessionMap.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); + else + mapIter->second++; + } + + //...Decrement the number of threads using the specified sessionId. + //...When the thread count for a sessionId reaches 0, the corresponding + //...CalpontSystemCatalog objects are deleted. + //...The user query and its associated catalog query have a different + //...session Id where the highest bit is flipped. + //...The object with id(sessionId | 0x80000000) cannot be removed before + //...user query session completes because the sysdata may be used for + //...debugging/stats purpose, such as result graph, etc. + static void decThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex); + ThreadCntPerSessionMap_t::iterator mapIter = threadCntPerSessionMap.find(sessionId); + + if (mapIter != threadCntPerSessionMap.end()) + { + if (--mapIter->second == 0) + { + threadCntPerSessionMap.erase(mapIter); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); + } + } + } + + //...Init sessionMemMap entry for specified session to 0 memory %. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void initMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) + { + // std::cout << "Setting pct to 0 for session " << sessionId << std::endl; + std::lock_guard lk(sessionMemMapMutex); + SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); + + if (mapIter == sessionMemMap.end()) + { + sessionMemMap[sessionId] = 0; + } + else + { + mapIter->second = 0; + } + } + } + + //... Round off to human readable format (KB, MB, or GB). + const std::string roundBytes(uint64_t value) const + { + const char* units[] = {"B", "KB", "MB", "GB", "TB"}; + uint64_t i = 0, up = 0; + uint64_t roundedValue = value; + + while (roundedValue > 1024 && i < 4) + { + up = (roundedValue & 512); + roundedValue /= 1024; + i++; + } + + if (up) + roundedValue++; + + std::ostringstream oss; + oss << roundedValue << units[i]; + return oss.str(); + } + + //...Round off to nearest (1024*1024) MB + uint64_t roundMB(uint64_t value) const + { + uint64_t roundedValue = value >> 20; + + if (value & 524288) + roundedValue++; + + return roundedValue; + } + + void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) + { + for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); + it != parms.end(); ++it) + { + switch (it->id) + { + case execplan::PMSMALLSIDEMEMORY: + { + fRm->addHJPmMaxSmallSideMap(it->sessionId, it->value); + break; } - std::string queryMode; - queryMode = (vtableModeOn ? "Distributed" : "Standard"); - - // Log stats to specified output stream - os << label << - ": MaxMemPct-" << fStats.fMaxMemPct << - "; NumTempFiles-" << fStats.fNumFiles << - "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << - "; ApproxPhyI/O-" << fStats.fPhyIO << - "; CacheI/O-" << fStats.fCacheIO << - "; BlocksTouched-" << fStats.fMsgRcvCnt; - - if ( includeNewLine ) - os << std::endl << " "; // insert line break - else - os << "; "; // continue without line break - - os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << - "; MsgBytesIn-" << roundBytes(fStats.fMsgBytesIn) << - "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << - "; Mode-" << queryMode; - - return os.str(); - } - - //...Increment the number of threads using the specified sessionId - static void incThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex); - ThreadCntPerSessionMap_t::iterator mapIter = - threadCntPerSessionMap.find(sessionId); - - if (mapIter == threadCntPerSessionMap.end()) - threadCntPerSessionMap.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); - else - mapIter->second++; - } - - //...Decrement the number of threads using the specified sessionId. - //...When the thread count for a sessionId reaches 0, the corresponding - //...CalpontSystemCatalog objects are deleted. - //...The user query and its associated catalog query have a different - //...session Id where the highest bit is flipped. - //...The object with id(sessionId | 0x80000000) cannot be removed before - //...user query session completes because the sysdata may be used for - //...debugging/stats purpose, such as result graph, etc. - static void decThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex); - ThreadCntPerSessionMap_t::iterator mapIter = - threadCntPerSessionMap.find(sessionId); - - if (mapIter != threadCntPerSessionMap.end()) + case execplan::UMSMALLSIDEMEMORY: { - if (--mapIter->second == 0) - { - threadCntPerSessionMap.erase(mapIter); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); - } + fRm->addHJUmMaxSmallSideMap(it->sessionId, it->value); + break; } + + default:; + } + } + } + + void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc) + { + const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); + execplan::CalpontSelectExecutionPlan::ColumnMap::const_iterator it; + std::string schemaName; + + for (it = colMap.begin(); it != colMap.end(); ++it) + { + const auto sc = dynamic_cast((it->second).get()); + + if (sc) + { + schemaName = sc->schemaName(); + + // only the first time a schema is got will actually query + // system catalog. System catalog keeps a schema name std::map. + // if a schema exists, the call getSchemaInfo returns without + // doing anything. + if (!schemaName.empty()) + csc->getSchemaInfo(schemaName); + } } - //...Init sessionMemMap entry for specified session to 0 memory %. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void initMaxMemPct ( uint32_t sessionId ) - { - if ( sessionId < 0x80000000 ) - { - // std::cout << "Setting pct to 0 for session " << sessionId << std::endl; - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find( sessionId ); + execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; - if ( mapIter == sessionMemMap.end() ) + for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) + { + buildSysCache(*(dynamic_cast(subIt->get())), csc); + } + } + + void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) + { + messageqcpp::ByteStream emsgBs; + messageqcpp::ByteStream tbs; + tbs << code; + fIos.write(tbs); + emsgBs << emsg; + fIos.write(emsgBs); + } + + void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) + { + messageqcpp::ByteStream::quadbyte qb; + execplan::MCSAnalyzeTableExecutionPlan caep; + + bs = fIos.read(); + caep.unserialize(bs); + + statementsRunningCount->incr(stmtCounted); + jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); + + // Joblist is empty. + if (jl->status() == logging::statisticsJobListEmpty) + { + if (caep.traceOn()) + std::cout << "JobList is empty " << std::endl; + + jl.reset(); + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + return; + } + + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + std::cout << "fEc setup " << std::endl; + fEc->Setup(); + } + + if (jl->status() == 0) + { + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + + // Execute a joblist. + jl->doQuery(); + + FEMsgHandler msgHandler(jl, &fIos); + + msgHandler.start(); + auto rowCount = jl->projectTable(100, bs); + msgHandler.stop(); + + auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); + + if (caep.traceOn()) + std::cout << "Row count " << rowCount << std::endl; + + // Process `RowGroup`, increase an epoch and save statistics to the file. + auto* statisticsManager = statistics::StatisticsManager::instance(); + statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); + statisticsManager->incEpoch(); + statisticsManager->saveToFile(); + + // Distribute statistics across all ExeMgr clients if possible. + statistics::StatisticsDistributor::instance()->distributeStatistics(); + + // Send the signal back to front-end. + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + } + + void analyzeTableHandleStats(messageqcpp::ByteStream& bs) + { + messageqcpp::ByteStream::quadbyte qb; +#ifdef DEBUG_STATISTICS + std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + uint64_t dataHashRec; + bs >> dataHashRec; + + uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); + // The stats are the same. + if (dataHash == dataHashRec) + { +#ifdef DEBUG_STATISTICS + std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + return; + } + + bs.restart(); + qb = ANALYZE_TABLE_NEED_STATS; + bs << qb; + fIos.write(bs); + + bs.restart(); + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + statistics::StatisticsManager::instance()->unserialize(bs); + statistics::StatisticsManager::instance()->saveToFile(); + +#ifdef DEBUG_STATISTICS + std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + } + + public: + void operator()() + { + utils::setThreadName("SessionThread"); + messageqcpp::ByteStream bs, inbs; + execplan::CalpontSelectExecutionPlan csep; + csep.sessionID(0); + joblist::SJLP jl; + bool incSessionThreadCnt = true; + std::mutex jlMutex; + std::condition_variable jlCleanupDone; + int destructing = 0; + + bool selfJoin = false; + bool tryTuples = false; + bool usingTuples = false; + bool stmtCounted = false; + + try + { + for (;;) + { + selfJoin = false; + tryTuples = false; + usingTuples = false; + + if (jl) + { + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, &destructing] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this runs first + bgdtor.detach(); + } + + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; + + // connection closed by client + fIos.close(); + break; + } + else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan + { + if (gDebug) + std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " + << bs.length() << std::endl; + + fIos.close(); + break; + } + else if (bs.length() == 4) // possible tuple flag + { + messageqcpp::ByteStream::quadbyte qb; + bs >> qb; + + if (qb == 4) // UM wants new tuple i/f + { + if (gDebug) + std::cout << "### UM wants tuples" << std::endl; + + tryTuples = true; + // now wait for the CSEP... + bs = fIos.read(); + } + else if (qb == 5) // somebody wants stats + { + bs.restart(); + qb = statementsRunningCount->cur(); + bs << qb; + qb = statementsRunningCount->waiting(); + bs << qb; + fIos.write(bs); + fIos.close(); + break; + } + else if (qb == ANALYZE_TABLE_EXECUTE) + { + analyzeTableExecute(bs, jl, stmtCounted); + continue; + } + else if (qb == ANALYZE_TABLE_REC_STATS) + { + analyzeTableHandleStats(bs); + continue; + } + else + { + if (gDebug) + std::cout << "### Got a not-a-plan value " << qb << std::endl; + + fIos.close(); + break; + } + } + + new_plan: + try + { + csep.unserialize(bs); + } + catch (logging::IDBExcept& ex) + { + // We can get here on illegal function parameter data type, e.g. + // SELECT blob_column|1 FROM t1; + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(ex.errorCode(), std::string(ex.what())); + continue; + } + + querytele::QueryTeleStats qts; + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.query_uuid = csep.uuid(); + qts.msg_type = querytele::QueryTeleStats::QT_START; + qts.start_time = querytele::QueryTeleClient::timeNowms(); + qts.query = csep.data(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + qts.schema_name = csep.schemaName(); + fTeleClient.postQueryTele(qts); + } + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; + + setRMParms(csep.rmParms()); + // Re-establish lost PP connections. + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + fEc->Setup(); + } + // @bug 1021. try to get schema cache for a come in query. + // skip system catalog queries. + if (!csep.isInternal()) + { + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); + buildSysCache(csep, csc); + } + + // As soon as we have a session id for this thread, update the + // thread count per session; only do this once per thread. + // Mask 0x80000000 is for associate user query and csc query + if (incSessionThreadCnt) + { + incThreadCntPerSession(csep.sessionID() | 0x80000000); + incSessionThreadCnt = false; + } + + bool needDbProfEndStatementMsg = false; + logging::Message::Args args; + std::string sqlText = csep.data(); + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + + // Initialize stats for this query, including + // init sessionMemMap entry for this session to 0 memory %. + // We will need this later for traceOn() or if we receive a + // table request with qb=3 (see below). This is also recorded + // as query start time. + initStats(csep.sessionID(), sqlText); + fStats.fQueryType = csep.queryType(); + + // Log start and end statement if tracing is enabled. Keep in + // mind the trace flag won't be set for system catalog queries. + if (csep.traceOn()) + { + args.reset(); + args.add((int)csep.statementID()); + args.add((int)csep.verID().currentScn); + args.add(sqlText); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfStartStatement, args, li); + needDbProfEndStatementMsg = true; + } + + // Don't log subsequent self joins after first. + if (selfJoin) + sqlText = ""; + + std::ostringstream oss; + oss << sqlText << "; |" << csep.schemaName() << "|"; + logging::SQLLogger sqlLog(oss.str(), li); + + statementsRunningCount->incr(stmtCounted); + + if (tryTuples) + { + try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList + { + jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); + // assign query stats + jl->queryStats(fStats); + + if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) { - sessionMemMap[sessionId] = 0; + usingTuples = true; + + // Tell the FE that we're sending tuples back, not TableBands + writeCodeAndError(0, "NOERROR"); + auto tjlp = dynamic_cast(jl.get()); + assert(tjlp); + messageqcpp::ByteStream tbs; + tbs << tjlp->getOutputRowGroup(); + fIos.write(tbs); } else { - mapIter->second = 0; + const std::string emsg = jl->errMsg(); + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(jl->status(), emsg); + std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; + continue; } - } - } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing makeJoblist " + "response; " + << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing makeJoblist " + "response; "; + throw std::runtime_error(errMsg.str()); + } - //... Round off to human readable format (KB, MB, or GB). - const std::string roundBytes(uint64_t value) const - { - const char* units[] = {"B", "KB", "MB", "GB", "TB"}; - uint64_t i = 0, up = 0; - uint64_t roundedValue = value; - - while (roundedValue > 1024 && i < 4) - { - up = (roundedValue & 512); - roundedValue /= 1024; - i++; - } - - if (up) - roundedValue++; - - std::ostringstream oss; - oss << roundedValue << units[i]; - return oss.str(); - } - - //...Round off to nearest (1024*1024) MB - uint64_t roundMB ( uint64_t value ) const - { - uint64_t roundedValue = value >> 20; - - if (value & 524288) - roundedValue++; - - return roundedValue; - } - - void setRMParms ( const execplan::CalpontSelectExecutionPlan::RMParmVec& parms ) - { - for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); - it != parms.end(); ++it) - { - switch (it->id) - { - case execplan::PMSMALLSIDEMEMORY: - { - fRm->addHJPmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - case execplan::UMSMALLSIDEMEMORY: - { - fRm->addHJUmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - default: - ; - } - } - } - - void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, boost::shared_ptr csc) - { - const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); - execplan::CalpontSelectExecutionPlan::ColumnMap::const_iterator it; - std::string schemaName; - - for (it = colMap.begin(); it != colMap.end(); ++it) - { - const auto sc = dynamic_cast((it->second).get()); - - if (sc) - { - schemaName = sc->schemaName(); - - // only the first time a schema is got will actually query - // system catalog. System catalog keeps a schema name std::map. - // if a schema exists, the call getSchemaInfo returns without - // doing anything. - if (!schemaName.empty()) - csc->getSchemaInfo(schemaName); - } - } - - execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; - - for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++ subIt) - { - buildSysCache(*(dynamic_cast(subIt->get())), csc); - } - } - - void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, - const std::string emsg) - { - messageqcpp::ByteStream emsgBs; - messageqcpp::ByteStream tbs; - tbs << code; - fIos.write(tbs); - emsgBs << emsg; - fIos.write(emsgBs); - } - - void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) - { - messageqcpp::ByteStream::quadbyte qb; - execplan::MCSAnalyzeTableExecutionPlan caep; - - bs = fIos.read(); - caep.unserialize(bs); - - statementsRunningCount->incr(stmtCounted); - jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); - - // Joblist is empty. - if (jl->status() == logging::statisticsJobListEmpty) - { - if (caep.traceOn()) - std::cout << "JobList is empty " << std::endl; - - jl.reset(); - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - return; - } - - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - std::cout << "fEc setup " << std::endl; - fEc->Setup(); - } - - if (jl->status() == 0) - { - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); + if (!usingTuples) + { + if (gDebug) + std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; + } + else + { + if (gDebug) + std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; + } } else { + usingTuples = false; + jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { throw std::runtime_error("ExeMgr: could not build a JobList!"); + } } - // Execute a joblist. jl->doQuery(); - FEMsgHandler msgHandler(jl, &fIos); + execplan::CalpontSystemCatalog::OID tableOID; + bool swallowRows = false; + joblist::DeliveredTableMap tm; + uint64_t totalBytesSent = 0; + uint64_t totalRowCount = 0; - msgHandler.start(); - auto rowCount = jl->projectTable(100, bs); - msgHandler.stop(); - - auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); - - if (caep.traceOn()) - std::cout << "Row count " << rowCount << std::endl; - - // Process `RowGroup`, increase an epoch and save statistics to the file. - auto* statisticsManager = statistics::StatisticsManager::instance(); - statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); - statisticsManager->incEpoch(); - statisticsManager->saveToFile(); - - // Distribute statistics across all ExeMgr clients if possible. - statistics::StatisticsDistributor::instance()->distributeStatistics(); - - // Send the signal back to front-end. - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - } - - void analyzeTableHandleStats(messageqcpp::ByteStream& bs) - { - messageqcpp::ByteStream::quadbyte qb; -#ifdef DEBUG_STATISTICS - std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " - << std::endl; -#endif - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " - << std::endl; -#endif - uint64_t dataHashRec; - bs >> dataHashRec; - - uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); - // The stats are the same. - if (dataHash == dataHashRec) + // Project each table as the FE asks for it + for (;;) { -#ifdef DEBUG_STATISTICS - std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; + + break; + } + + if (gDebug && bs.length() > 4) + std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - return; - } - bs.restart(); - qb = ANALYZE_TABLE_NEED_STATS; - bs << qb; - fIos.write(bs); + // TODO: Holy crud! Can this be right? + //@bug 1444 Yes, if there is a self-join + if (bs.length() > 4) + { + selfJoin = true; + statementsRunningCount->decr(stmtCounted); + goto new_plan; + } - bs.restart(); - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - statistics::StatisticsManager::instance()->unserialize(bs); - statistics::StatisticsManager::instance()->saveToFile(); + assert(bs.length() == 4); -#ifdef DEBUG_STATISTICS - std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - } + messageqcpp::ByteStream::quadbyte qb; - public: + try // @bug2244: try/catch around fIos.write() calls responding to qb command + { + bs >> qb; - void operator()() - { - utils::setThreadName("SessionThread"); - messageqcpp::ByteStream bs, inbs; - execplan::CalpontSelectExecutionPlan csep; - csep.sessionID(0); - joblist::SJLP jl; - bool incSessionThreadCnt = true; - std::mutex jlMutex; - std::condition_variable jlCleanupDone; - int destructing = 0; + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb + << std::endl; - bool selfJoin = false; - bool tryTuples = false; - bool usingTuples = false; - bool stmtCounted = false; - - try - { - for (;;) + if (qb == 0) { - selfJoin = false; - tryTuples = false; - usingTuples = false; - - if (jl) - { - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - std::unique_lock scoped(jlMutex); - destructing++; - std::thread bgdtor([jl, &jlMutex, &jlCleanupDone, &destructing] { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this runs first - bgdtor.detach(); - } - - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; - - // connection closed by client - fIos.close(); - break; - } - else if (bs.length() < 4) //Not a CalpontSelectExecutionPlan - { - if (gDebug) - std::cout << "### Got a not-a-plan for session id " << csep.sessionID() - << " with length " << bs.length() << std::endl; - - fIos.close(); - break; - } - else if (bs.length() == 4) //possible tuple flag - { - messageqcpp::ByteStream::quadbyte qb; - bs >> qb; - - if (qb == 4) //UM wants new tuple i/f - { - if (gDebug) - std::cout << "### UM wants tuples" << std::endl; - - tryTuples = true; - // now wait for the CSEP... - bs = fIos.read(); - } - else if (qb == 5) //somebody wants stats - { - bs.restart(); - qb = statementsRunningCount->cur(); - bs << qb; - qb = statementsRunningCount->waiting(); - bs << qb; - fIos.write(bs); - fIos.close(); - break; - } - else if (qb == ANALYZE_TABLE_EXECUTE) - { - analyzeTableExecute(bs, jl, stmtCounted); - continue; - } - else if (qb == ANALYZE_TABLE_REC_STATS) - { - analyzeTableHandleStats(bs); - continue; - } - else - { - if (gDebug) - std::cout << "### Got a not-a-plan value " << qb << std::endl; - - fIos.close(); - break; - } - } - -new_plan: - try - { - csep.unserialize(bs); - } - catch (logging::IDBExcept &ex) - { - // We can get here on illegal function parameter data type, e.g. - // SELECT blob_column|1 FROM t1; - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(ex.errorCode(), std::string(ex.what())); - continue; - } - - querytele::QueryTeleStats qts; - - if ( !csep.isInternal() && - (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT") ) - { - qts.query_uuid = csep.uuid(); - qts.msg_type = querytele::QueryTeleStats::QT_START; - qts.start_time = querytele::QueryTeleClient::timeNowms(); - qts.query = csep.data(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - qts.schema_name = csep.schemaName(); - fTeleClient.postQueryTele(qts); - } + // No more tables, query is done + break; + } + else if (qb == 1) + { + // super-secret flag indicating that the UM is going to scarf down all the rows in the + // query. + swallowRows = true; + tm = jl->deliveredTables(); + continue; + } + else if (qb == 2) + { + // UM just wants any table + assert(swallowRows); + auto iter = tm.begin(); + if (iter == tm.end()) + { if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; + std::cout << "### For session id " << csep.sessionID() << ", returning end flag" + << std::endl; - setRMParms(csep.rmParms()); - // Re-establish lost PP connections. - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - fEc->Setup(); - } - // @bug 1021. try to get schema cache for a come in query. - // skip system catalog queries. - if (!csep.isInternal()) - { - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); - buildSysCache(csep, csc); - } + bs.restart(); + bs << (messageqcpp::ByteStream::byte)1; + fIos.write(bs); + continue; + } - // As soon as we have a session id for this thread, update the - // thread count per session; only do this once per thread. - // Mask 0x80000000 is for associate user query and csc query - if (incSessionThreadCnt) - { - incThreadCntPerSession( csep.sessionID() | 0x80000000 ); - incSessionThreadCnt = false; - } + tableOID = iter->first; + } + else if (qb == 3) // special option-UM wants job stats std::string + { + std::string statsString; - bool needDbProfEndStatementMsg = false; - logging::Message::Args args; - std::string sqlText = csep.data(); - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + // Log stats std::string to be sent back to front end + statsString = formatQueryStats( + jl, "Query Stats", false, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); - // Initialize stats for this query, including - // init sessionMemMap entry for this session to 0 memory %. - // We will need this later for traceOn() or if we receive a - // table request with qb=3 (see below). This is also recorded - // as query start time. - initStats( csep.sessionID(), sqlText ); - fStats.fQueryType = csep.queryType(); + bs.restart(); + bs << statsString; - // Log start and end statement if tracing is enabled. Keep in - // mind the trace flag won't be set for system catalog queries. - if (csep.traceOn()) - { - args.reset(); - args.add((int)csep.statementID()); - args.add((int)csep.verID().currentScn); - args.add(sqlText); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, - logDbProfStartStatement, - args, - li); - needDbProfEndStatementMsg = true; - } + if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) + { + bs << jl->extendedInfo(); + bs << prettyPrintMiniInfo(jl->miniInfo()); + } + else + { + std::string empty; + bs << empty; + bs << empty; + } - //Don't log subsequent self joins after first. - if (selfJoin) - sqlText = ""; + // send stats to connector for inserting to the querystats table + fStats.serialize(bs); + fIos.write(bs); + continue; + } + // for table mode handling + else if (qb == 4) + { + statementsRunningCount->decr(stmtCounted); + bs = fIos.read(); + goto new_plan; + } + else // (qb > 3) + { + // Return table bands for the requested tableOID + tableOID = static_cast(qb); + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing qb response " + "for qb cmd " + << qb << "; " << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing qb response " + "for qb cmd " + << qb; + throw std::runtime_error(errMsg.str()); + } - std::ostringstream oss; - oss << sqlText << "; |" << csep.schemaName() << "|"; - logging::SQLLogger sqlLog(oss.str(), li); + if (swallowRows) + tm.erase(tableOID); - statementsRunningCount->incr(stmtCounted); + FEMsgHandler msgHandler(jl, &fIos); - if (tryTuples) - { - try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList - { - jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); - // assign query stats - jl->queryStats(fStats); + if (tableOID == 100) + msgHandler.start(); + //...Loop serializing table bands projected for the tableOID + for (;;) + { + uint32_t rowCount; - if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) - { - usingTuples = true; + rowCount = jl->projectTable(tableOID, bs); - //Tell the FE that we're sending tuples back, not TableBands - writeCodeAndError(0, "NOERROR"); - auto tjlp = dynamic_cast(jl.get()); - assert(tjlp); - messageqcpp::ByteStream tbs; - tbs << tjlp->getOutputRowGroup(); - fIos.write(tbs); - } - else - { - const std::string emsg = jl->errMsg(); - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(jl->status(), emsg); - std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; - continue; - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing makeJoblist " - "response; " << ex.what(); - throw std::runtime_error( errMsg.str() ); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing makeJoblist " - "response; "; - throw std::runtime_error( errMsg.str() ); - } + msgHandler.stop(); - if (!usingTuples) - { - if (gDebug) - std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; - } - else - { - if (gDebug) - std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; - } - } - else - { - usingTuples = false; - jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); + if (jl->status()) + { + const auto errInfo = logging::IDBErrorInfo::instance(); - if (jl->status() == 0) - { - std::string emsg; - - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - } - - - jl->doQuery(); - - execplan::CalpontSystemCatalog::OID tableOID; - bool swallowRows = false; - joblist::DeliveredTableMap tm; - uint64_t totalBytesSent = 0; - uint64_t totalRowCount = 0; - - // Project each table as the FE asks for it - for (;;) - { - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; - - break; - } - - if (gDebug && bs.length() > 4) - std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << - bs.length() << std::endl; - - //TODO: Holy crud! Can this be right? - //@bug 1444 Yes, if there is a self-join - if (bs.length() > 4) - { - selfJoin = true; - statementsRunningCount->decr(stmtCounted); - goto new_plan; - } - - assert(bs.length() == 4); - - messageqcpp::ByteStream::quadbyte qb; - - try // @bug2244: try/catch around fIos.write() calls responding to qb command - { - bs >> qb; - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb << std::endl; - - if (qb == 0) - { - // No more tables, query is done - break; - } - else if (qb == 1) - { - // super-secret flag indicating that the UM is going to scarf down all the rows in the - // query. - swallowRows = true; - tm = jl->deliveredTables(); - continue; - } - else if (qb == 2) - { - // UM just wants any table - assert(swallowRows); - auto iter = tm.begin(); - - if (iter == tm.end()) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", returning end flag" << std::endl; - - bs.restart(); - bs << (messageqcpp::ByteStream::byte)1; - fIos.write(bs); - continue; - } - - tableOID = iter->first; - } - else if (qb == 3) //special option-UM wants job stats std::string - { - std::string statsString; - - //Log stats std::string to be sent back to front end - statsString = formatQueryStats( - jl, - "Query Stats", - false, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), - totalRowCount - ); - - bs.restart(); - bs << statsString; - - if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) - { - bs << jl->extendedInfo(); - bs << prettyPrintMiniInfo(jl->miniInfo()); - } - else - { - std::string empty; - bs << empty; - bs << empty; - } - - // send stats to connector for inserting to the querystats table - fStats.serialize(bs); - fIos.write(bs); - continue; - } - // for table mode handling - else if (qb == 4) - { - statementsRunningCount->decr(stmtCounted); - bs = fIos.read(); - goto new_plan; - } - else // (qb > 3) - { - //Return table bands for the requested tableOID - tableOID = static_cast(qb); - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing qb response " - "for qb cmd " << qb << - "; " << ex.what(); - throw std::runtime_error( errMsg.str() ); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing qb response " - "for qb cmd " << qb; - throw std::runtime_error( errMsg.str() ); - } - - if (swallowRows) tm.erase(tableOID); - - FEMsgHandler msgHandler(jl, &fIos); - - if (tableOID == 100) - msgHandler.start(); - - //...Loop serializing table bands projected for the tableOID - for (;;) - { - uint32_t rowCount; - - rowCount = jl->projectTable(tableOID, bs); - - msgHandler.stop(); - - if (jl->status()) - { - const auto errInfo = logging::IDBErrorInfo::instance(); - - if (jl->errMsg().length() != 0) - bs << jl->errMsg(); - else - bs << errInfo->errorMsg(jl->status()); - } - - try // @bug2244: try/catch around fIos.write() calls projecting rows - { - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - // Skip the write to the front end until the last empty band. Used to time queries - // through without any front end waiting. - if (tableOID < 3000 || rowCount == 0) - fIos.write(bs); - } - else - { - fIos.write(bs); - } - } - catch (std::exception& ex) - { - msgHandler.stop(); - std::ostringstream errMsg; - errMsg << "ExeMgr: error projecting rows " - "for tableOID: " << tableOID << - "; rowCnt: " << rowCount << - "; prevTotRowCnt: " << totalRowCount << - "; " << ex.what(); - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - if (tableOID == 100 && msgHandler.aborted()) - { - /* TODO: modularize the cleanup code, as well as - * the rest of this fcn */ - - decThreadCntPerSession( csep.sessionID() | 0x80000000 ); - statementsRunningCount->decr(stmtCounted); - fIos.close(); - return; - } - - //std::cout << "connection drop\n"; - throw std::runtime_error( errMsg.str() ); - } - catch (...) - { - std::ostringstream errMsg; - msgHandler.stop(); - errMsg << "ExeMgr: unknown error projecting rows " - "for tableOID: " << - tableOID << "; rowCnt: " << rowCount << - "; prevTotRowCnt: " << totalRowCount; - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - throw std::runtime_error( errMsg.str() ); - } - - totalRowCount += rowCount; - totalBytesSent += bs.length(); - - if (rowCount == 0) - { - msgHandler.stop(); - // No more bands, table is done - bs.reset(); - - // @bug 2083 decr active statement count here for table mode. - if (!usingTuples) - statementsRunningCount->decr(stmtCounted); - - break; - } - else - { - bs.restart(); - } - } // End of loop to project and serialize table bands for a table - } // End of loop to process tables - - // @bug 828 - if (csep.traceOn()) - jl->graph(csep.sessionID()); - - if (needDbProfEndStatementMsg) - { - std::string ss; - std::ostringstream prefix; - prefix << "ses:" << csep.sessionID() << " Query Totals"; - - //Log stats std::string to standard out - ss = formatQueryStats( - jl, - prefix.str(), - true, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), - totalRowCount); - //@Bug 1306. Added timing info for real time tracking. - std::cout << ss << " at " << timeNow() << std::endl; - - // log query stats to debug log file - args.reset(); - args.add((int)csep.statementID()); - args.add(fStats.fMaxMemPct); - args.add(fStats.fNumFiles); - args.add(fStats.fFileBytes); // log raw byte count instead of MB - args.add(fStats.fPhyIO); - args.add(fStats.fCacheIO); - args.add(fStats.fMsgRcvCnt); - args.add(fStats.fMsgBytesIn); - args.add(fStats.fMsgBytesOut); - args.add(fStats.fCPBlocksSkipped); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, - logDbProfQueryStats, - args, - li); - //@bug 1327 - deleteMaxMemPct( csep.sessionID() ); - // Calling reset here, will cause joblist destructor to be - // called, which "joins" the threads. We need to do that - // here to make sure all syslogging from all the threads - // are complete; and that our logDbProfEndStatement will - // appear "last" in the syslog for this SQL statement. - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - int stmtID = csep.statementID(); - std::unique_lock scoped(jlMutex); - // C7's compiler complains about the msgLog capture here - // msgLog is global scope, and passed by copy, so, unclear - // what the warning is about. - destructing++; - std::thread bgdtor([jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing] { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - logging::Message::Args args; - args.add(stmtID); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, - logDbProfEndStatement, - args, - li); - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this happens first - bgdtor.detach(); - } - else - // delete sessionMemMap entry for this session's memory % use - deleteMaxMemPct( csep.sessionID() ); - - std::string endtime(timeNow()); - - if ((csep.traceFlags() & flagsWantOutput) && (csep.sessionID() < 0x80000000)) - { - std::cout << "For session " << csep.sessionID() << ": " << - totalBytesSent << - " bytes sent back at " << endtime << std::endl; - - // @bug 663 - Implemented caltraceon(16) to replace the - // $FIFO_SINK compiler definition in pColStep. - // This option consumes rows in the project steps. - if (csep.traceFlags() & - execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM. Rows consumed " - "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." - " ****" << std::endl; - std::cout << std::endl; - } - else if (csep.traceFlags() & - execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM - caltrace(8) is " - "on (SWALLOW_ROWS_EXEMGR). ****" << std::endl; - std::cout << std::endl; - } - } - - statementsRunningCount->decr(stmtCounted); - - if ( !csep.isInternal() && - (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT") ) - { - qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = fStats.fMaxMemPct; - qts.num_files = fStats.fNumFiles; - qts.phy_io = fStats.fPhyIO; - qts.cache_io = fStats.fCacheIO; - qts.msg_rcv_cnt = fStats.fMsgRcvCnt; - qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; - qts.msg_bytes_in = fStats.fMsgBytesIn; - qts.msg_bytes_out = fStats.fMsgBytesOut; - qts.rows = totalRowCount; - qts.end_time = querytele::QueryTeleClient::timeNowms(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.query = csep.data(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - fTeleClient.postQueryTele(qts); - } + if (jl->errMsg().length() != 0) + bs << jl->errMsg(); + else + bs << errInfo->errorMsg(jl->status()); } - // Release CSC object (for sessionID) that was added by makeJobList() - // Mask 0x80000000 is for associate user query and csc query. - // (actual joblist destruction happens at the top of this loop) - decThreadCntPerSession( csep.sessionID() | 0x80000000 ); - } - catch (std::exception& ex) + try // @bug2244: try/catch around fIos.write() calls projecting rows + { + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + // Skip the write to the front end until the last empty band. Used to time queries + // through without any front end waiting. + if (tableOID < 3000 || rowCount == 0) + fIos.write(bs); + } + else + { + fIos.write(bs); + } + } + catch (std::exception& ex) + { + msgHandler.stop(); + std::ostringstream errMsg; + errMsg << "ExeMgr: error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " + << ex.what(); + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + if (tableOID == 100 && msgHandler.aborted()) + { + /* TODO: modularize the cleanup code, as well as + * the rest of this fcn */ + + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + fIos.close(); + return; + } + + // std::cout << "connection drop\n"; + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + msgHandler.stop(); + errMsg << "ExeMgr: unknown error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + throw std::runtime_error(errMsg.str()); + } + + totalRowCount += rowCount; + totalBytesSent += bs.length(); + + if (rowCount == 0) + { + msgHandler.stop(); + // No more bands, table is done + bs.reset(); + + // @bug 2083 decr active statement count here for table mode. + if (!usingTuples) + statementsRunningCount->decr(stmtCounted); + + break; + } + else + { + bs.restart(); + } + } // End of loop to project and serialize table bands for a table + } // End of loop to process tables + + // @bug 828 + if (csep.traceOn()) + jl->graph(csep.sessionID()); + + if (needDbProfEndStatementMsg) { - decThreadCntPerSession( csep.sessionID() | 0x80000000 ); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add(ex.what()); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); - fIos.close(); + std::string ss; + std::ostringstream prefix; + prefix << "ses:" << csep.sessionID() << " Query Totals"; + + // Log stats std::string to standard out + ss = formatQueryStats(jl, prefix.str(), true, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), + totalRowCount); + //@Bug 1306. Added timing info for real time tracking. + std::cout << ss << " at " << timeNow() << std::endl; + + // log query stats to debug log file + args.reset(); + args.add((int)csep.statementID()); + args.add(fStats.fMaxMemPct); + args.add(fStats.fNumFiles); + args.add(fStats.fFileBytes); // log raw byte count instead of MB + args.add(fStats.fPhyIO); + args.add(fStats.fCacheIO); + args.add(fStats.fMsgRcvCnt); + args.add(fStats.fMsgBytesIn); + args.add(fStats.fMsgBytesOut); + args.add(fStats.fCPBlocksSkipped); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfQueryStats, args, li); + //@bug 1327 + deleteMaxMemPct(csep.sessionID()); + // Calling reset here, will cause joblist destructor to be + // called, which "joins" the threads. We need to do that + // here to make sure all syslogging from all the threads + // are complete; and that our logDbProfEndStatement will + // appear "last" in the syslog for this SQL statement. + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + int stmtID = csep.statementID(); + std::unique_lock scoped(jlMutex); + // C7's compiler complains about the msgLog capture here + // msgLog is global scope, and passed by copy, so, unclear + // what the warning is about. + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + logging::Message::Args args; + args.add(stmtID); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfEndStatement, args, li); + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this happens first + bgdtor.detach(); } - catch (...) + else + // delete sessionMemMap entry for this session's memory % use + deleteMaxMemPct(csep.sessionID()); + + std::string endtime(timeNow()); + + if ((csep.traceFlags() & flagsWantOutput) && (csep.sessionID() < 0x80000000)) { - decThreadCntPerSession( csep.sessionID() | 0x80000000 ); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### Exception caught!" << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add("ExeMgr caught unknown exception"); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); - fIos.close(); + std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " + << endtime << std::endl; + + // @bug 663 - Implemented caltraceon(16) to replace the + // $FIFO_SINK compiler definition in pColStep. + // This option consumes rows in the project steps. + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM. Rows consumed " + "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." + " ****" + << std::endl; + std::cout << std::endl; + } + else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM - caltrace(8) is " + "on (SWALLOW_ROWS_EXEMGR). ****" + << std::endl; + std::cout << std::endl; + } } - // make sure we don't leave scope while joblists are being destroyed - std::unique_lock scoped(jlMutex); - while (destructing > 0) - jlCleanupDone.wait(scoped); + statementsRunningCount->decr(stmtCounted); + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = fStats.fMaxMemPct; + qts.num_files = fStats.fNumFiles; + qts.phy_io = fStats.fPhyIO; + qts.cache_io = fStats.fCacheIO; + qts.msg_rcv_cnt = fStats.fMsgRcvCnt; + qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; + qts.msg_bytes_in = fStats.fMsgBytesIn; + qts.msg_bytes_out = fStats.fMsgBytesOut; + qts.rows = totalRowCount; + qts.end_time = querytele::QueryTeleClient::timeNowms(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.query = csep.data(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + fTeleClient.postQueryTele(qts); + } + } + + // Release CSC object (for sessionID) that was added by makeJobList() + // Mask 0x80000000 is for associate user query and csc query. + // (actual joblist destruction happens at the top of this loop) + decThreadCntPerSession(csep.sessionID() | 0x80000000); + } + catch (std::exception& ex) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add(ex.what()); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); + fIos.close(); + } + catch (...) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### Exception caught!" << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add("ExeMgr caught unknown exception"); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); + fIos.close(); } -}; + // make sure we don't leave scope while joblists are being destroyed + std::unique_lock scoped(jlMutex); + while (destructing > 0) + jlCleanupDone.wait(scoped); + } +}; class RssMonFcn : public utils::MonitorProcMem { -public: - RssMonFcn(size_t maxPct, int pauseSeconds) : - MonitorProcMem(maxPct, 0, 21, pauseSeconds) {} + public: + RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) + { + } - /*virtual*/ - void operator()() const + /*virtual*/ + void operator()() const + { + for (;;) { - for (;;) + size_t rssMb = rss(); + size_t pct = rssMb * 100 / fMemTotal; + + if (pct > fMaxPct) + { + if (fMaxPct >= 95) { - size_t rssMb = rss(); - size_t pct = rssMb * 100 / fMemTotal; - - if (pct > fMaxPct) - { - if (fMaxPct >= 95) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - if (statementsRunningCount->cur() == 0) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_WARNING, logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - std::cerr << "Too much memory allocated, but stmts running" << std::endl; - } - - // Update sessionMemMap entries lower than current mem % use - { - std::lock_guard lk(sessionMemMapMutex); - - for (SessionMemMap_t::iterator mapIter = sessionMemMap.begin(); - mapIter != sessionMemMap.end(); - ++mapIter) - { - if (pct > mapIter->second) - { - mapIter->second = pct; - } - } - - } - - pause_(); + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logRssTooBig, args, logging::LoggingID(16)); + exit(1); } + + if (statementsRunningCount->cur() == 0) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_WARNING, logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + std::cerr << "Too much memory allocated, but stmts running" << std::endl; + } + + // Update sessionMemMap entries lower than current mem % use + { + std::lock_guard lk(sessionMemMapMutex); + + for (SessionMemMap_t::iterator mapIter = sessionMemMap.begin(); mapIter != sessionMemMap.end(); + ++mapIter) + { + if (pct > mapIter->second) + { + mapIter->second = pct; + } + } + } + + pause_(); } + } }; #ifdef _MSC_VER void exit_(int) { - exit(0); + exit(0); } #endif void added_a_pm(int) { - logging::LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("exeMgr caught SIGHUP. Resetting connections"); - msg.format( args1 ); - std::cout << msg.msg().c_str() << std::endl; - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); + logging::LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("exeMgr caught SIGHUP. Resetting connections"); + msg.format(args1); + std::cout << msg.msg().c_str() << std::endl; + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); - if (ec) - { - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - oamCache->forceReload(); - ec->Setup(); - } + if (ec) + { + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + oamCache->forceReload(); + ec->Setup(); + } } void printTotalUmMemory(int sig) { - int64_t num = rm->availableMemory(); - std::cout << "Total UM memory available: " << num << std::endl; + int64_t num = rm->availableMemory(); + std::cout << "Total UM memory available: " << num << std::endl; } void setupSignalHandlers() { #ifdef _MSC_VER - signal(SIGINT, exit_); - signal(SIGTERM, exit_); + signal(SIGINT, exit_); + signal(SIGTERM, exit_); #else - struct sigaction ign; + struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); + sigaction(SIGPIPE, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = added_a_pm; - sigaction(SIGHUP, &ign, 0); - ign.sa_handler = printTotalUmMemory; - sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = added_a_pm; + sigaction(SIGHUP, &ign, 0); + ign.sa_handler = printTotalUmMemory; + sigaction(SIGUSR1, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif } int8_t setupCwd(joblist::ResourceManager* rm) { - std::string workdir = rm->getScWorkingDir(); - int8_t rc = chdir(workdir.c_str()); + std::string workdir = rm->getScWorkingDir(); + int8_t rc = chdir(workdir.c_str()); - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); - return (rc < 0) ? -5 : rc; + return (rc < 0) ? -5 : rc; } void startRssMon(size_t maxPct, int pauseSeconds) { - new boost::thread(RssMonFcn(maxPct, pauseSeconds)); + new boost::thread(RssMonFcn(maxPct, pauseSeconds)); } int setupResources() { #ifdef _MSC_VER - //FIXME: + // FIXME: #else - struct rlimit rlim; + struct rlimit rlim; - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } - rlim.rlim_cur = rlim.rlim_max = 65536; + rlim.rlim_cur = rlim.rlim_max = 65536; - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } - if (rlim.rlim_cur != 65536) - { - return -4; - } + if (rlim.rlim_cur != 65536) + { + return -4; + } #endif - return 0; + return 0; } -} +} // namespace void cleanTempDir() { @@ -1600,18 +1557,8 @@ void cleanTempDir() std::string allowed; TempDirPurpose purpose; }; - std::vector dirs{ - { - "HashJoin", - "AllowDiskBasedJoin", - TempDirPurpose::Joins - }, - { - "RowAggregation", - "AllowDiskBasedAggregation", - TempDirPurpose::Aggregates - } - }; + std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, + {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; const auto config = config::Config::makeConfig(); for (const auto& dir : dirs) @@ -1647,7 +1594,7 @@ void cleanTempDir() } boost::filesystem::create_directories(tmpPrefix); } - catch (const std::exception &ex) + catch (const std::exception& ex) { std::cerr << ex.what() << std::endl; logging::LoggingID logid(16, 0, 0); @@ -1661,8 +1608,7 @@ void cleanTempDir() } catch (...) { - std::cerr << "Caught unknown exception during tmpdir cleanup" - << std::endl; + std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; logging::LoggingID logid(16, 0, 0); logging::Message::Args args; logging::Message message(8); @@ -1674,217 +1620,209 @@ void cleanTempDir() } } - int ServiceExeMgr::Child() { - - gDebug= m_debug; + gDebug = m_debug; #ifdef _MSC_VER - //FIXME: + // FIXME: #else - // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall - if (!m_e) - setenv("CALPONT_CSC_IDENT", "um", 1); + // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall + if (!m_e) + setenv("CALPONT_CSC_IDENT", "um", 1); #endif - setupSignalHandlers(); - int err = 0; - if (!m_debug) - err = setupResources(); - std::string errMsg; + setupSignalHandlers(); + int err = 0; + if (!m_debug) + err = setupResources(); + std::string errMsg; - switch (err) - { - case -1: - case -3: - errMsg = "Error getting file limits, please see non-root install documentation"; - break; + switch (err) + { + case -1: + case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; - case -2: - errMsg = "Error setting file limits, please see non-root install documentation"; - break; + case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; - case -4: - errMsg = "Could not install file limits to required value, please see non-root install documentation"; - break; + case -4: + errMsg = "Could not install file limits to required value, please see non-root install documentation"; + break; - default: - errMsg = "Couldn't change working directory or unknown error"; - break; - } + default: errMsg = "Couldn't change working directory or unknown error"; break; + } - err = setupCwd(rm); + err = setupCwd(rm); - if (err < 0) - { - oam::Oam oam; - logging::Message::Args args; - logging::Message message; - args.add( errMsg ); - message.format(args); - logging::LoggingID lid(16); - logging::MessageLog ml(lid); - ml.logCriticalMessage( message ); - std::cerr << errMsg << std::endl; + if (err < 0) + { + oam::Oam oam; + logging::Message::Args args; + logging::Message message; + args.add(errMsg); + message.format(args); + logging::LoggingID lid(16); + logging::MessageLog ml(lid); + ml.logCriticalMessage(message); + std::cerr << errMsg << std::endl; - NotifyServiceInitializationFailed(); - return 2; - } + NotifyServiceInitializationFailed(); + return 2; + } - cleanTempDir(); + cleanTempDir(); - logging::MsgMap msgMap; - msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); - msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); - msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); - msgMap[logStartSql] = logging::Message(logStartSql); - msgMap[logEndSql] = logging::Message(logEndSql); - msgMap[logRssTooBig] = logging::Message(logRssTooBig); - msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); - msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); - msgLog.msgMap(msgMap); + logging::MsgMap msgMap; + msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); + msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); + msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); + msgMap[logStartSql] = logging::Message(logStartSql); + msgMap[logEndSql] = logging::Message(logEndSql); + msgMap[logRssTooBig] = logging::Message(logRssTooBig); + msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); + msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); + msgLog.msgMap(msgMap); - ec = joblist::DistributedEngineComm::instance(rm, true); - ec->Open(); + ec = joblist::DistributedEngineComm::instance(rm, true); + ec->Open(); - bool tellUser = true; + bool tellUser = true; - messageqcpp::MessageQueueServer* mqs; + messageqcpp::MessageQueueServer* mqs; - statementsRunningCount = new ActiveStatementCounter(rm->getEmExecQueueSize()); + statementsRunningCount = new ActiveStatementCounter(rm->getEmExecQueueSize()); - for (;;) - { - try - { - mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm->getConfig(), messageqcpp::ByteStream::BlockSize, 64); - break; - } - catch (std::runtime_error& re) - { - std::string what = re.what(); - - if (what.find("Address already in use") != std::string::npos) - { - if (tellUser) - { - std::cerr << "Address already in use, retrying..." << std::endl; - tellUser = false; - } - - sleep(5); - } - else - { - throw; - } - } - } - - // class jobstepThreadPool is used by other processes. We can't call - // resourcemanaager (rm) functions during the static creation of threadpool - // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). - // From the pools perspective, it has no idea if it is ExeMgr doing the - // creation, so it has no idea which way to set the flag. So we set the max here. - joblist::JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); - joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); - - if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") - { - joblist::JobStep::jobstepThreadPool.setDebug(true); - joblist::JobStep::jobstepThreadPool.invoke(threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); - } - - int serverThreads = rm->getEmServerThreads(); - int maxPct = rm->getEmMaxPct(); - int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); - int priority = rm->getEmPriority(); - - FEMsgHandler::threadPool.setMaxThreads(serverThreads); - FEMsgHandler::threadPool.setName("FEMsgHandler"); - - if (maxPct > 0) - startRssMon(maxPct, pauseSeconds); - -#ifndef _MSC_VER - setpriority(PRIO_PROCESS, 0, priority); -#endif - - std::string teleServerHost(rm->getConfig()->getConfig("QueryTele", "Host")); - - if (!teleServerHost.empty()) - { - int teleServerPort = toInt(rm->getConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - gTeleServerParms.host = teleServerHost; - gTeleServerParms.port = teleServerPort; - } - } - - NotifyServiceStarted(); - - std::cout << "Starting ExeMgr: st = " << serverThreads << - ", qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << - rm->getConfig()->configFile() << std::endl; - - { - BRM::DBRM *dbrm = new BRM::DBRM(); - dbrm->setSystemQueryReady(true); - delete dbrm; - } - - threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); - exeMgrThreadPool.setName("ExeMgrServer"); - - if (rm->getExeMgrThreadPoolDebug() == "Y" || rm->getExeMgrThreadPoolDebug() == "y") - { - exeMgrThreadPool.setDebug(true); - exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); - } - - // Load statistics. + for (;;) + { try { - statistics::StatisticsManager::instance()->loadFromFile(); + mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm->getConfig(), messageqcpp::ByteStream::BlockSize, + 64); + break; } - catch (...) + catch (std::runtime_error& re) { - std::cerr << "Cannot load statistics from file " << std::endl; - } + std::string what = re.what(); - for (;;) + if (what.find("Address already in use") != std::string::npos) + { + if (tellUser) + { + std::cerr << "Address already in use, retrying..." << std::endl; + tellUser = false; + } + + sleep(5); + } + else + { + throw; + } + } + } + + // class jobstepThreadPool is used by other processes. We can't call + // resourcemanaager (rm) functions during the static creation of threadpool + // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). + // From the pools perspective, it has no idea if it is ExeMgr doing the + // creation, so it has no idea which way to set the flag. So we set the max here. + joblist::JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); + joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); + + if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") + { + joblist::JobStep::jobstepThreadPool.setDebug(true); + joblist::JobStep::jobstepThreadPool.invoke( + threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); + } + + int serverThreads = rm->getEmServerThreads(); + int maxPct = rm->getEmMaxPct(); + int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); + int priority = rm->getEmPriority(); + + FEMsgHandler::threadPool.setMaxThreads(serverThreads); + FEMsgHandler::threadPool.setName("FEMsgHandler"); + + if (maxPct > 0) + startRssMon(maxPct, pauseSeconds); + +#ifndef _MSC_VER + setpriority(PRIO_PROCESS, 0, priority); +#endif + + std::string teleServerHost(rm->getConfig()->getConfig("QueryTele", "Host")); + + if (!teleServerHost.empty()) + { + int teleServerPort = toInt(rm->getConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) { - messageqcpp::IOSocket ios; - ios = mqs->accept(); - exeMgrThreadPool.invoke(SessionThread(ios, ec, rm)); + gTeleServerParms.host = teleServerHost; + gTeleServerParms.port = teleServerPort; } + } - exeMgrThreadPool.wait(); + NotifyServiceStarted(); - return 0; + std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm->getEmExecQueueSize() + << ", mx = " << maxPct << ", cf = " << rm->getConfig()->configFile() << std::endl; + + { + BRM::DBRM* dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; + } + + threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); + exeMgrThreadPool.setName("ExeMgrServer"); + + if (rm->getExeMgrThreadPoolDebug() == "Y" || rm->getExeMgrThreadPoolDebug() == "y") + { + exeMgrThreadPool.setDebug(true); + exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); + } + + // Load statistics. + try + { + statistics::StatisticsManager::instance()->loadFromFile(); + } + catch (...) + { + std::cerr << "Cannot load statistics from file " << std::endl; + } + + for (;;) + { + messageqcpp::IOSocket ios; + ios = mqs->accept(); + exeMgrThreadPool.invoke(SessionThread(ios, ec, rm)); + } + + exeMgrThreadPool.wait(); + + return 0; } - int main(int argc, char* argv[]) { - opterr = 0; - Opt opt(argc, argv); + opterr = 0; + Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); - // This is unset due to the way we start it - program_invocation_short_name = const_cast("ExeMgr"); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ExeMgr"); - // Initialize the charset library - MY_INIT(argv[0]); + // Initialize the charset library + MY_INIT(argv[0]); - return ServiceExeMgr(opt).Run(); + return ServiceExeMgr(opt).Run(); } // vim:ts=4 sw=4: diff --git a/exemgr/resource.h b/exemgr/resource.h index cc861c700..3e2717d59 100644 --- a/exemgr/resource.h +++ b/exemgr/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index a87cf1a4e..cfbd00886 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -33,8 +33,8 @@ #include #include -#elif defined (_MSC_VER) -#elif defined (__FreeBSD__) +#elif defined(_MSC_VER) +#elif defined(__FreeBSD__) #include #include #include @@ -67,11 +67,12 @@ #if defined(__GNUC__) #include -static const std::string optim("Build is " +static const std::string optim( + "Build is " #if !defined(__OPTIMIZE__) - "NOT " + "NOT " #endif - "optimized"); + "optimized"); #endif namespace fs = boost::filesystem; @@ -86,7 +87,7 @@ using namespace BRM; namespace oam { // flag to tell us ctrl-c was hit -uint32_t ctrlc = 0; +uint32_t ctrlc = 0; //------------------------------------------------------------------------------ // Signal handler to catch Control-C signal to terminate the process @@ -94,34 +95,34 @@ uint32_t ctrlc = 0; //------------------------------------------------------------------------------ void handleControlC(int i) { - std::cout << "Received Control-C to terminate the command..." << std::endl; - ctrlc = 1; + std::cout << "Received Control-C to terminate the command..." << std::endl; + ctrlc = 1; } Oam::Oam() { - CalpontConfigFile = std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml"; + CalpontConfigFile = std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml"; - //get user - string USER = "root"; - char* p = getenv("USER"); + // get user + string USER = "root"; + char* p = getenv("USER"); - if (p && *p) - USER = p; + if (p && *p) + USER = p; - userDir = USER; + userDir = USER; - if ( USER != "root") - { - userDir = "home/" + USER; - } - - tmpdir = startup::StartUp::tmpDir(); + if (USER != "root") + { + userDir = "home/" + USER; + } + tmpdir = startup::StartUp::tmpDir(); } Oam::~Oam() -{} +{ +} /******************************************************************** * @@ -131,27 +132,27 @@ Oam::~Oam() void Oam::getSystemConfig(SystemModuleTypeConfig& systemmoduletypeconfig) { - const string Section = "SystemModuleConfig"; - const string MODULE_TYPE = "ModuleType"; - systemmoduletypeconfig.moduletypeconfig.clear(); + const string Section = "SystemModuleConfig"; + const string MODULE_TYPE = "ModuleType"; + systemmoduletypeconfig.moduletypeconfig.clear(); - Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); + Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); - for (int moduleTypeID = 1; moduleTypeID < MAX_MODULE_TYPE + 1; moduleTypeID++) - { - ModuleTypeConfig moduletypeconfig; + for (int moduleTypeID = 1; moduleTypeID < MAX_MODULE_TYPE + 1; moduleTypeID++) + { + ModuleTypeConfig moduletypeconfig; - // get Module info + // get Module info - string moduleType = MODULE_TYPE + itoa(moduleTypeID); + string moduleType = MODULE_TYPE + itoa(moduleTypeID); - Oam::getSystemConfig(sysConfig->getConfig(Section, moduleType), moduletypeconfig ); + Oam::getSystemConfig(sysConfig->getConfig(Section, moduleType), moduletypeconfig); - if (moduletypeconfig.ModuleType.empty()) - continue; + if (moduletypeconfig.ModuleType.empty()) + continue; - systemmoduletypeconfig.moduletypeconfig.push_back(moduletypeconfig); - } + systemmoduletypeconfig.moduletypeconfig.push_back(moduletypeconfig); + } } /******************************************************************** @@ -162,197 +163,211 @@ void Oam::getSystemConfig(SystemModuleTypeConfig& systemmoduletypeconfig) void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& moduletypeconfig) { - Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); - const string Section = "SystemModuleConfig"; - const string MODULE_TYPE = "ModuleType"; - const string MODULE_DESC = "ModuleDesc"; - const string MODULE_RUN_TYPE = "RunType"; - const string MODULE_COUNT = "ModuleCount"; - const string MODULE_DISABLE_STATE = "ModuleDisableState"; - const string MODULE_CPU_CRITICAL = "ModuleCPUCriticalThreshold"; - const string MODULE_CPU_MAJOR = "ModuleCPUMajorThreshold"; - const string MODULE_CPU_MINOR = "ModuleCPUMinorThreshold"; - const string MODULE_CPU_MINOR_CLEAR = "ModuleCPUMinorClearThreshold"; - const string MODULE_DISK_CRITICAL = "ModuleDiskCriticalThreshold"; - const string MODULE_DISK_MAJOR = "ModuleDiskMajorThreshold"; - const string MODULE_DISK_MINOR = "ModuleDiskMinorThreshold"; - const string MODULE_MEM_CRITICAL = "ModuleMemCriticalThreshold"; - const string MODULE_MEM_MAJOR = "ModuleMemMajorThreshold"; - const string MODULE_MEM_MINOR = "ModuleMemMinorThreshold"; - const string MODULE_SWAP_CRITICAL = "ModuleSwapCriticalThreshold"; - const string MODULE_SWAP_MAJOR = "ModuleSwapMajorThreshold"; - const string MODULE_SWAP_MINOR = "ModuleSwapMinorThreshold"; - const string MODULE_IP_ADDR = "ModuleIPAddr"; - const string MODULE_SERVER_NAME = "ModuleHostName"; - const string MODULE_DISK_MONITOR_FS = "ModuleDiskMonitorFileSystem"; - const string MODULE_DBROOT_COUNT = "ModuleDBRootCount"; - const string MODULE_DBROOT_ID = "ModuleDBRootID"; + Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); + const string Section = "SystemModuleConfig"; + const string MODULE_TYPE = "ModuleType"; + const string MODULE_DESC = "ModuleDesc"; + const string MODULE_RUN_TYPE = "RunType"; + const string MODULE_COUNT = "ModuleCount"; + const string MODULE_DISABLE_STATE = "ModuleDisableState"; + const string MODULE_CPU_CRITICAL = "ModuleCPUCriticalThreshold"; + const string MODULE_CPU_MAJOR = "ModuleCPUMajorThreshold"; + const string MODULE_CPU_MINOR = "ModuleCPUMinorThreshold"; + const string MODULE_CPU_MINOR_CLEAR = "ModuleCPUMinorClearThreshold"; + const string MODULE_DISK_CRITICAL = "ModuleDiskCriticalThreshold"; + const string MODULE_DISK_MAJOR = "ModuleDiskMajorThreshold"; + const string MODULE_DISK_MINOR = "ModuleDiskMinorThreshold"; + const string MODULE_MEM_CRITICAL = "ModuleMemCriticalThreshold"; + const string MODULE_MEM_MAJOR = "ModuleMemMajorThreshold"; + const string MODULE_MEM_MINOR = "ModuleMemMinorThreshold"; + const string MODULE_SWAP_CRITICAL = "ModuleSwapCriticalThreshold"; + const string MODULE_SWAP_MAJOR = "ModuleSwapMajorThreshold"; + const string MODULE_SWAP_MINOR = "ModuleSwapMinorThreshold"; + const string MODULE_IP_ADDR = "ModuleIPAddr"; + const string MODULE_SERVER_NAME = "ModuleHostName"; + const string MODULE_DISK_MONITOR_FS = "ModuleDiskMonitorFileSystem"; + const string MODULE_DBROOT_COUNT = "ModuleDBRootCount"; + const string MODULE_DBROOT_ID = "ModuleDBRootID"; - for (int moduleTypeID = 1; moduleTypeID < MAX_MODULE_TYPE + 1; moduleTypeID++) + for (int moduleTypeID = 1; moduleTypeID < MAX_MODULE_TYPE + 1; moduleTypeID++) + { + string moduleType = MODULE_TYPE + itoa(moduleTypeID); + + if (sysConfig->getConfig(Section, moduleType) == moduletype) { - string moduleType = MODULE_TYPE + itoa(moduleTypeID); + string ModuleCount = MODULE_COUNT + itoa(moduleTypeID); + string ModuleType = MODULE_TYPE + itoa(moduleTypeID); + string ModuleDesc = MODULE_DESC + itoa(moduleTypeID); + string ModuleRunType = MODULE_RUN_TYPE + itoa(moduleTypeID); + string ModuleCPUCriticalThreshold = MODULE_CPU_CRITICAL + itoa(moduleTypeID); + string ModuleCPUMajorThreshold = MODULE_CPU_MAJOR + itoa(moduleTypeID); + string ModuleCPUMinorThreshold = MODULE_CPU_MINOR + itoa(moduleTypeID); + string ModuleCPUMinorClearThreshold = MODULE_CPU_MINOR_CLEAR + itoa(moduleTypeID); + string ModuleDiskCriticalThreshold = MODULE_DISK_CRITICAL + itoa(moduleTypeID); + string ModuleDiskMajorThreshold = MODULE_DISK_MAJOR + itoa(moduleTypeID); + string ModuleDiskMinorThreshold = MODULE_DISK_MINOR + itoa(moduleTypeID); + string ModuleMemCriticalThreshold = MODULE_MEM_CRITICAL + itoa(moduleTypeID); + string ModuleMemMajorThreshold = MODULE_MEM_MAJOR + itoa(moduleTypeID); + string ModuleMemMinorThreshold = MODULE_MEM_MINOR + itoa(moduleTypeID); + string ModuleSwapCriticalThreshold = MODULE_SWAP_CRITICAL + itoa(moduleTypeID); + string ModuleSwapMajorThreshold = MODULE_SWAP_MAJOR + itoa(moduleTypeID); + string ModuleSwapMinorThreshold = MODULE_SWAP_MINOR + itoa(moduleTypeID); - if ( sysConfig->getConfig(Section, moduleType) == moduletype) + moduletypeconfig.ModuleCount = strtol(sysConfig->getConfig(Section, ModuleCount).c_str(), 0, 0); + moduletypeconfig.ModuleType = sysConfig->getConfig(Section, ModuleType); + moduletypeconfig.ModuleDesc = sysConfig->getConfig(Section, ModuleDesc); + moduletypeconfig.RunType = sysConfig->getConfig(Section, ModuleRunType); + moduletypeconfig.ModuleCPUCriticalThreshold = + strtol(sysConfig->getConfig(Section, ModuleCPUCriticalThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleCPUMajorThreshold = + strtol(sysConfig->getConfig(Section, ModuleCPUMajorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleCPUMinorThreshold = + strtol(sysConfig->getConfig(Section, ModuleCPUMinorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleCPUMinorClearThreshold = + strtol(sysConfig->getConfig(Section, ModuleCPUMinorClearThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleDiskCriticalThreshold = + strtol(sysConfig->getConfig(Section, ModuleDiskCriticalThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleDiskMajorThreshold = + strtol(sysConfig->getConfig(Section, ModuleDiskMajorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleDiskMinorThreshold = + strtol(sysConfig->getConfig(Section, ModuleDiskMinorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleMemCriticalThreshold = + strtol(sysConfig->getConfig(Section, ModuleMemCriticalThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleMemMajorThreshold = + strtol(sysConfig->getConfig(Section, ModuleMemMajorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleMemMinorThreshold = + strtol(sysConfig->getConfig(Section, ModuleMemMinorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleSwapCriticalThreshold = + strtol(sysConfig->getConfig(Section, ModuleSwapCriticalThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleSwapMajorThreshold = + strtol(sysConfig->getConfig(Section, ModuleSwapMajorThreshold).c_str(), 0, 0); + moduletypeconfig.ModuleSwapMinorThreshold = + strtol(sysConfig->getConfig(Section, ModuleSwapMinorThreshold).c_str(), 0, 0); + + int moduleFound = 0; + + // get NIC IP address/hostnames + for (int moduleID = 1; moduleID <= moduletypeconfig.ModuleCount; moduleID++) + { + DeviceNetworkConfig devicenetworkconfig; + HostConfig hostconfig; + + for (int nicID = 1; nicID < MAX_NIC + 1; nicID++) { - string ModuleCount = MODULE_COUNT + itoa(moduleTypeID); - string ModuleType = MODULE_TYPE + itoa(moduleTypeID); - string ModuleDesc = MODULE_DESC + itoa(moduleTypeID); - string ModuleRunType = MODULE_RUN_TYPE + itoa(moduleTypeID); - string ModuleCPUCriticalThreshold = MODULE_CPU_CRITICAL + itoa(moduleTypeID); - string ModuleCPUMajorThreshold = MODULE_CPU_MAJOR + itoa(moduleTypeID); - string ModuleCPUMinorThreshold = MODULE_CPU_MINOR + itoa(moduleTypeID); - string ModuleCPUMinorClearThreshold = MODULE_CPU_MINOR_CLEAR + itoa(moduleTypeID); - string ModuleDiskCriticalThreshold = MODULE_DISK_CRITICAL + itoa(moduleTypeID); - string ModuleDiskMajorThreshold = MODULE_DISK_MAJOR + itoa(moduleTypeID); - string ModuleDiskMinorThreshold = MODULE_DISK_MINOR + itoa(moduleTypeID); - string ModuleMemCriticalThreshold = MODULE_MEM_CRITICAL + itoa(moduleTypeID); - string ModuleMemMajorThreshold = MODULE_MEM_MAJOR + itoa(moduleTypeID); - string ModuleMemMinorThreshold = MODULE_MEM_MINOR + itoa(moduleTypeID); - string ModuleSwapCriticalThreshold = MODULE_SWAP_CRITICAL + itoa(moduleTypeID); - string ModuleSwapMajorThreshold = MODULE_SWAP_MAJOR + itoa(moduleTypeID); - string ModuleSwapMinorThreshold = MODULE_SWAP_MINOR + itoa(moduleTypeID); + string ModuleIpAddr = + MODULE_IP_ADDR + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); - moduletypeconfig.ModuleCount = strtol(sysConfig->getConfig(Section, ModuleCount).c_str(), 0, 0); - moduletypeconfig.ModuleType = sysConfig->getConfig(Section, ModuleType); - moduletypeconfig.ModuleDesc = sysConfig->getConfig(Section, ModuleDesc); - moduletypeconfig.RunType = sysConfig->getConfig(Section, ModuleRunType); - moduletypeconfig.ModuleCPUCriticalThreshold = strtol(sysConfig->getConfig(Section, ModuleCPUCriticalThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleCPUMajorThreshold = strtol(sysConfig->getConfig(Section, ModuleCPUMajorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleCPUMinorThreshold = strtol(sysConfig->getConfig(Section, ModuleCPUMinorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleCPUMinorClearThreshold = strtol(sysConfig->getConfig(Section, ModuleCPUMinorClearThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleDiskCriticalThreshold = strtol(sysConfig->getConfig(Section, ModuleDiskCriticalThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleDiskMajorThreshold = strtol(sysConfig->getConfig(Section, ModuleDiskMajorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleDiskMinorThreshold = strtol(sysConfig->getConfig(Section, ModuleDiskMinorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleMemCriticalThreshold = strtol(sysConfig->getConfig(Section, ModuleMemCriticalThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleMemMajorThreshold = strtol(sysConfig->getConfig(Section, ModuleMemMajorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleMemMinorThreshold = strtol(sysConfig->getConfig(Section, ModuleMemMinorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleSwapCriticalThreshold = strtol(sysConfig->getConfig(Section, ModuleSwapCriticalThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleSwapMajorThreshold = strtol(sysConfig->getConfig(Section, ModuleSwapMajorThreshold).c_str(), 0, 0); - moduletypeconfig.ModuleSwapMinorThreshold = strtol(sysConfig->getConfig(Section, ModuleSwapMinorThreshold).c_str(), 0, 0); + string ipAddr = sysConfig->getConfig(Section, ModuleIpAddr); - int moduleFound = 0; + if (ipAddr.empty()) + break; + else if (ipAddr == UnassignedIpAddr) + continue; - //get NIC IP address/hostnames - for (int moduleID = 1; moduleID <= moduletypeconfig.ModuleCount ; moduleID++) - { - DeviceNetworkConfig devicenetworkconfig; - HostConfig hostconfig; + string ModuleHostName = + MODULE_SERVER_NAME + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); + string serverName = sysConfig->getConfig(Section, ModuleHostName); - for (int nicID = 1; nicID < MAX_NIC + 1 ; nicID++) - { - string ModuleIpAddr = MODULE_IP_ADDR + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); + hostconfig.IPAddr = ipAddr; + hostconfig.HostName = serverName; + hostconfig.NicID = nicID; - string ipAddr = sysConfig->getConfig(Section, ModuleIpAddr); - - if (ipAddr.empty()) - break; - else if (ipAddr == UnassignedIpAddr ) - continue; - - string ModuleHostName = MODULE_SERVER_NAME + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); - string serverName = sysConfig->getConfig(Section, ModuleHostName); - - hostconfig.IPAddr = ipAddr; - hostconfig.HostName = serverName; - hostconfig.NicID = nicID; - - devicenetworkconfig.hostConfigList.push_back(hostconfig); - } - - if ( !devicenetworkconfig.hostConfigList.empty() ) - { - string ModuleDisableState = MODULE_DISABLE_STATE + itoa(moduleID) + "-" + itoa(moduleTypeID); - devicenetworkconfig.DisableState = sysConfig->getConfig(Section, ModuleDisableState); - - devicenetworkconfig.DeviceName = moduletypeconfig.ModuleType + itoa(moduleID); - moduletypeconfig.ModuleNetworkList.push_back(devicenetworkconfig); - devicenetworkconfig.hostConfigList.clear(); - - moduleFound++; - - if ( moduleFound >= moduletypeconfig.ModuleCount ) - break; - } - } - - // get filesystems - DiskMonitorFileSystems fs; - - for (int fsID = 1;; fsID++) - { - string ModuleDiskMonitorFS = MODULE_DISK_MONITOR_FS + itoa(fsID) + "-" + itoa(moduleTypeID); - - string fsName = sysConfig->getConfig(Section, ModuleDiskMonitorFS); - - if (fsName.empty()) - break; - - fs.push_back(fsName); - } - - moduletypeconfig.FileSystems = fs; - - // get dbroot IDs - moduleFound = 0; - - for (int moduleID = 1; moduleID <= moduletypeconfig.ModuleCount ; moduleID++) - { - string ModuleDBRootCount = MODULE_DBROOT_COUNT + itoa(moduleID) + "-" + itoa(moduleTypeID); - string temp = sysConfig->getConfig(Section, ModuleDBRootCount).c_str(); - - if ( temp.empty() || temp == oam::UnassignedName) - continue; - - int moduledbrootcount = strtol(temp.c_str(), 0, 0); - - DeviceDBRootConfig devicedbrootconfig; - DBRootConfigList dbrootconfiglist; - - if ( moduledbrootcount < 1 ) - { - dbrootconfiglist.clear(); - } - else - { - - int foundIDs = 0; - - for (int dbrootID = 1; dbrootID < moduledbrootcount + 1 ; dbrootID++) - { - string DBRootID = MODULE_DBROOT_ID + itoa(moduleID) + "-" + itoa(dbrootID) + "-" + itoa(moduleTypeID); - - string dbrootid = sysConfig->getConfig(Section, DBRootID); - - if (dbrootid.empty() || dbrootid == oam::UnassignedName - || dbrootid == "0") - continue; - - dbrootconfiglist.push_back(atoi(dbrootid.c_str())); - foundIDs++; - - if ( moduledbrootcount == foundIDs) - break; - } - } - - sort ( dbrootconfiglist.begin(), dbrootconfiglist.end() ); - devicedbrootconfig.DeviceID = moduleID; - devicedbrootconfig.dbrootConfigList = dbrootconfiglist; - moduletypeconfig.ModuleDBRootList.push_back(devicedbrootconfig); - devicedbrootconfig.dbrootConfigList.clear(); - - moduleFound++; - - if ( moduleFound >= moduletypeconfig.ModuleCount ) - break; - } - - return; + devicenetworkconfig.hostConfigList.push_back(hostconfig); } - } - // Module Not found - exceptionControl("getSystemConfig", API_INVALID_PARAMETER); + if (!devicenetworkconfig.hostConfigList.empty()) + { + string ModuleDisableState = MODULE_DISABLE_STATE + itoa(moduleID) + "-" + itoa(moduleTypeID); + devicenetworkconfig.DisableState = sysConfig->getConfig(Section, ModuleDisableState); + + devicenetworkconfig.DeviceName = moduletypeconfig.ModuleType + itoa(moduleID); + moduletypeconfig.ModuleNetworkList.push_back(devicenetworkconfig); + devicenetworkconfig.hostConfigList.clear(); + + moduleFound++; + + if (moduleFound >= moduletypeconfig.ModuleCount) + break; + } + } + + // get filesystems + DiskMonitorFileSystems fs; + + for (int fsID = 1;; fsID++) + { + string ModuleDiskMonitorFS = MODULE_DISK_MONITOR_FS + itoa(fsID) + "-" + itoa(moduleTypeID); + + string fsName = sysConfig->getConfig(Section, ModuleDiskMonitorFS); + + if (fsName.empty()) + break; + + fs.push_back(fsName); + } + + moduletypeconfig.FileSystems = fs; + + // get dbroot IDs + moduleFound = 0; + + for (int moduleID = 1; moduleID <= moduletypeconfig.ModuleCount; moduleID++) + { + string ModuleDBRootCount = MODULE_DBROOT_COUNT + itoa(moduleID) + "-" + itoa(moduleTypeID); + string temp = sysConfig->getConfig(Section, ModuleDBRootCount).c_str(); + + if (temp.empty() || temp == oam::UnassignedName) + continue; + + int moduledbrootcount = strtol(temp.c_str(), 0, 0); + + DeviceDBRootConfig devicedbrootconfig; + DBRootConfigList dbrootconfiglist; + + if (moduledbrootcount < 1) + { + dbrootconfiglist.clear(); + } + else + { + int foundIDs = 0; + + for (int dbrootID = 1; dbrootID < moduledbrootcount + 1; dbrootID++) + { + string DBRootID = + MODULE_DBROOT_ID + itoa(moduleID) + "-" + itoa(dbrootID) + "-" + itoa(moduleTypeID); + + string dbrootid = sysConfig->getConfig(Section, DBRootID); + + if (dbrootid.empty() || dbrootid == oam::UnassignedName || dbrootid == "0") + continue; + + dbrootconfiglist.push_back(atoi(dbrootid.c_str())); + foundIDs++; + + if (moduledbrootcount == foundIDs) + break; + } + } + + sort(dbrootconfiglist.begin(), dbrootconfiglist.end()); + devicedbrootconfig.DeviceID = moduleID; + devicedbrootconfig.dbrootConfigList = dbrootconfiglist; + moduletypeconfig.ModuleDBRootList.push_back(devicedbrootconfig); + devicedbrootconfig.dbrootConfigList.clear(); + + moduleFound++; + + if (moduleFound >= moduletypeconfig.ModuleCount) + break; + } + + return; + } + } + + // Module Not found + exceptionControl("getSystemConfig", API_INVALID_PARAMETER); } /******************************************************************** @@ -363,96 +378,97 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul void Oam::getSystemConfig(const std::string& module, ModuleConfig& moduleconfig) { - Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); - const string Section = "SystemModuleConfig"; - const string MODULE_TYPE = "ModuleType"; - const string MODULE_DESC = "ModuleDesc"; - const string MODULE_COUNT = "ModuleCount"; - const string MODULE_IP_ADDR = "ModuleIPAddr"; - const string MODULE_SERVER_NAME = "ModuleHostName"; - const string MODULE_DISABLE_STATE = "ModuleDisableState"; - const string MODULE_DBROOT_COUNT = "ModuleDBRootCount"; - const string MODULE_DBROOT_ID = "ModuleDBRootID"; + Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); + const string Section = "SystemModuleConfig"; + const string MODULE_TYPE = "ModuleType"; + const string MODULE_DESC = "ModuleDesc"; + const string MODULE_COUNT = "ModuleCount"; + const string MODULE_IP_ADDR = "ModuleIPAddr"; + const string MODULE_SERVER_NAME = "ModuleHostName"; + const string MODULE_DISABLE_STATE = "ModuleDisableState"; + const string MODULE_DBROOT_COUNT = "ModuleDBRootCount"; + const string MODULE_DBROOT_ID = "ModuleDBRootID"; - string moduletype = module.substr(0, MAX_MODULE_TYPE_SIZE); - int moduleID = atoi(module.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); + string moduletype = module.substr(0, MAX_MODULE_TYPE_SIZE); + int moduleID = atoi(module.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - if ( moduleID < 1 ) - //invalid ID - exceptionControl("getSystemConfig", API_INVALID_PARAMETER); - - for (int moduleTypeID = 1; moduleTypeID < MAX_MODULE_TYPE + 1; moduleTypeID++) - { - string moduleType = MODULE_TYPE + itoa(moduleTypeID); - string ModuleCount = MODULE_COUNT + itoa(moduleTypeID); - - if ( sysConfig->getConfig(Section, moduleType) == moduletype ) - { - string ModuleType = MODULE_TYPE + itoa(moduleTypeID); - string ModuleDesc = MODULE_DESC + itoa(moduleTypeID); - string ModuleDisableState = MODULE_DISABLE_STATE + itoa(moduleID) + "-" + itoa(moduleTypeID); - - moduleconfig.ModuleName = module; - moduleconfig.ModuleType = sysConfig->getConfig(Section, ModuleType); - moduleconfig.ModuleDesc = sysConfig->getConfig(Section, ModuleDesc) + " #" + itoa(moduleID); - moduleconfig.DisableState = sysConfig->getConfig(Section, ModuleDisableState); - - string ModuleDBRootCount = MODULE_DBROOT_COUNT + itoa(moduleID) + "-" + itoa(moduleTypeID); - string temp = sysConfig->getConfig(Section, ModuleDBRootCount).c_str(); - - int moduledbrootcount = 0; - - if ( temp.empty() || temp != oam::UnassignedName) - moduledbrootcount = strtol(temp.c_str(), 0, 0); - - HostConfig hostconfig; - - //get NIC IP address/hostnames - moduleconfig.hostConfigList.clear(); - - for (int nicID = 1; nicID < MAX_NIC + 1 ; nicID++) - { - string ModuleIpAddr = MODULE_IP_ADDR + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); - - string ipAddr = sysConfig->getConfig(Section, ModuleIpAddr); - - if (ipAddr.empty() || ipAddr == UnassignedIpAddr ) - continue; - - string ModuleHostName = MODULE_SERVER_NAME + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); - string serverName = sysConfig->getConfig(Section, ModuleHostName); - - hostconfig.IPAddr = ipAddr; - hostconfig.HostName = serverName; - hostconfig.NicID = nicID; - - moduleconfig.hostConfigList.push_back(hostconfig); - } - - //get DBroot IDs - moduleconfig.dbrootConfigList.clear(); - - for (int dbrootID = 1; dbrootID < moduledbrootcount + 1 ; dbrootID++) - { - string ModuleDBRootID = MODULE_DBROOT_ID + itoa(moduleID) + "-" + itoa(dbrootID) + "-" + itoa(moduleTypeID); - - string moduleDBRootID = sysConfig->getConfig(Section, ModuleDBRootID); - - if (moduleDBRootID.empty() || moduleDBRootID == oam::UnassignedName - || moduleDBRootID == "0") - continue; - - moduleconfig.dbrootConfigList.push_back(atoi(moduleDBRootID.c_str())); - } - - sort ( moduleconfig.dbrootConfigList.begin(), moduleconfig.dbrootConfigList.end() ); - - return; - } - } - - // Module Not found + if (moduleID < 1) + // invalid ID exceptionControl("getSystemConfig", API_INVALID_PARAMETER); + + for (int moduleTypeID = 1; moduleTypeID < MAX_MODULE_TYPE + 1; moduleTypeID++) + { + string moduleType = MODULE_TYPE + itoa(moduleTypeID); + string ModuleCount = MODULE_COUNT + itoa(moduleTypeID); + + if (sysConfig->getConfig(Section, moduleType) == moduletype) + { + string ModuleType = MODULE_TYPE + itoa(moduleTypeID); + string ModuleDesc = MODULE_DESC + itoa(moduleTypeID); + string ModuleDisableState = MODULE_DISABLE_STATE + itoa(moduleID) + "-" + itoa(moduleTypeID); + + moduleconfig.ModuleName = module; + moduleconfig.ModuleType = sysConfig->getConfig(Section, ModuleType); + moduleconfig.ModuleDesc = sysConfig->getConfig(Section, ModuleDesc) + " #" + itoa(moduleID); + moduleconfig.DisableState = sysConfig->getConfig(Section, ModuleDisableState); + + string ModuleDBRootCount = MODULE_DBROOT_COUNT + itoa(moduleID) + "-" + itoa(moduleTypeID); + string temp = sysConfig->getConfig(Section, ModuleDBRootCount).c_str(); + + int moduledbrootcount = 0; + + if (temp.empty() || temp != oam::UnassignedName) + moduledbrootcount = strtol(temp.c_str(), 0, 0); + + HostConfig hostconfig; + + // get NIC IP address/hostnames + moduleconfig.hostConfigList.clear(); + + for (int nicID = 1; nicID < MAX_NIC + 1; nicID++) + { + string ModuleIpAddr = MODULE_IP_ADDR + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); + + string ipAddr = sysConfig->getConfig(Section, ModuleIpAddr); + + if (ipAddr.empty() || ipAddr == UnassignedIpAddr) + continue; + + string ModuleHostName = + MODULE_SERVER_NAME + itoa(moduleID) + "-" + itoa(nicID) + "-" + itoa(moduleTypeID); + string serverName = sysConfig->getConfig(Section, ModuleHostName); + + hostconfig.IPAddr = ipAddr; + hostconfig.HostName = serverName; + hostconfig.NicID = nicID; + + moduleconfig.hostConfigList.push_back(hostconfig); + } + + // get DBroot IDs + moduleconfig.dbrootConfigList.clear(); + + for (int dbrootID = 1; dbrootID < moduledbrootcount + 1; dbrootID++) + { + string ModuleDBRootID = + MODULE_DBROOT_ID + itoa(moduleID) + "-" + itoa(dbrootID) + "-" + itoa(moduleTypeID); + + string moduleDBRootID = sysConfig->getConfig(Section, ModuleDBRootID); + + if (moduleDBRootID.empty() || moduleDBRootID == oam::UnassignedName || moduleDBRootID == "0") + continue; + + moduleconfig.dbrootConfigList.push_back(atoi(moduleDBRootID.c_str())); + } + + sort(moduleconfig.dbrootConfigList.begin(), moduleconfig.dbrootConfigList.end()); + + return; + } + } + + // Module Not found + exceptionControl("getSystemConfig", API_INVALID_PARAMETER); } /******************************************************************** @@ -463,28 +479,27 @@ void Oam::getSystemConfig(const std::string& module, ModuleConfig& moduleconfig) void Oam::getSystemConfig(const std::string& name, std::string& value) { + Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); - Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); + // get string variables - // get string variables + for (int i = 0;; i++) + { + if (configSections[i] == "") + // end of section list + break; - for ( int i = 0;; i++) + value = sysConfig->getConfig(configSections[i], name); + + if (!(value.empty())) { - if ( configSections[i] == "" ) - // end of section list - break; - - value = sysConfig->getConfig(configSections[i], name); - - if (!(value.empty())) - { - // match found - return; - } + // match found + return; } + } - // no match found - exceptionControl("getSystemConfig", API_INVALID_PARAMETER); + // no match found + exceptionControl("getSystemConfig", API_INVALID_PARAMETER); } /******************************************************************** @@ -495,15 +510,15 @@ void Oam::getSystemConfig(const std::string& name, std::string& value) void Oam::getSystemConfig(const std::string& name, int& value) { - string returnValue; + string returnValue; - // get string variables + // get string variables - Oam::getSystemConfig(name, returnValue); + Oam::getSystemConfig(name, returnValue); - // covert returned Parameter value to Interger + // covert returned Parameter value to Interger - value = atoi(returnValue.c_str()); + value = atoi(returnValue.c_str()); } /******************************************************************** @@ -517,63 +532,66 @@ void Oam::getSystemConfig(const std::string& name, int& value) oamModuleInfo_t Oam::getModuleInfo() { - string localModule; - string localModuleType; - int localModuleID; + string localModule; + string localModuleType; + int localModuleID; - // Get Module Name from module-file - string fileName = "/var/lib/columnstore/local/module"; + // Get Module Name from module-file + string fileName = "/var/lib/columnstore/local/module"; - ifstream oldFile (fileName.c_str()); + ifstream oldFile(fileName.c_str()); - char line[400]; + char line[400]; - while (oldFile.getline(line, 400)) - { - localModule = line; - break; - } + while (oldFile.getline(line, 400)) + { + localModule = line; + break; + } - oldFile.close(); + oldFile.close(); - if (localModule.empty() ) - { - // not found - //system("touch /var/log/mariadb/columnstore/test8"); - exceptionControl("getModuleInfo", API_FAILURE); - } + if (localModule.empty()) + { + // not found + // system("touch /var/log/mariadb/columnstore/test8"); + exceptionControl("getModuleInfo", API_FAILURE); + } - localModuleType = localModule.substr(0, MAX_MODULE_TYPE_SIZE); - localModuleID = atoi(localModule.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); + localModuleType = localModule.substr(0, MAX_MODULE_TYPE_SIZE); + localModuleID = atoi(localModule.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - // Get Parent OAM Module name + // Get Parent OAM Module name - string ParentOAMModule = oam::UnassignedName; - string StandbyOAMModule = oam::UnassignedName; - bool parentOAMModuleFlag = false; - bool standbyOAMModuleFlag = false; - int serverTypeInstall = 1; + string ParentOAMModule = oam::UnassignedName; + string StandbyOAMModule = oam::UnassignedName; + bool parentOAMModuleFlag = false; + bool standbyOAMModuleFlag = false; + int serverTypeInstall = 1; - try - { - Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); - string Section = "SystemConfig"; - ParentOAMModule = sysConfig->getConfig(Section, "ParentOAMModuleName"); - StandbyOAMModule = sysConfig->getConfig(Section, "StandbyOAMModuleName"); + try + { + Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); + string Section = "SystemConfig"; + ParentOAMModule = sysConfig->getConfig(Section, "ParentOAMModuleName"); + StandbyOAMModule = sysConfig->getConfig(Section, "StandbyOAMModuleName"); - if ( localModule == ParentOAMModule ) - parentOAMModuleFlag = true; + if (localModule == ParentOAMModule) + parentOAMModuleFlag = true; - if ( localModule == StandbyOAMModule ) - standbyOAMModuleFlag = true; + if (localModule == StandbyOAMModule) + standbyOAMModuleFlag = true; - // Get Server Type Install ID + // Get Server Type Install ID - serverTypeInstall = atoi(sysConfig->getConfig("Installation", "ServerTypeInstall").c_str()); - } - catch (...) {} + serverTypeInstall = atoi(sysConfig->getConfig("Installation", "ServerTypeInstall").c_str()); + } + catch (...) + { + } - return boost::make_tuple(localModule, localModuleType, localModuleID, ParentOAMModule, parentOAMModuleFlag, serverTypeInstall, StandbyOAMModule, standbyOAMModuleFlag); + return boost::make_tuple(localModule, localModuleType, localModuleID, ParentOAMModule, parentOAMModuleFlag, + serverTypeInstall, StandbyOAMModule, standbyOAMModuleFlag); } /******************************************************************** @@ -583,66 +601,66 @@ oamModuleInfo_t Oam::getModuleInfo() ********************************************************************/ systemStorageInfo_t Oam::getStorageConfig() { - DeviceDBRootList deviceDBRootList; - std::string storageType = ""; - std::string UMstorageType = ""; - int SystemDBRootCount = 0; + DeviceDBRootList deviceDBRootList; + std::string storageType = ""; + std::string UMstorageType = ""; + int SystemDBRootCount = 0; - try + try + { + getSystemConfig("DBRootStorageType", storageType); + } + catch (...) + { + exceptionControl("getStorageConfig", oam::API_FAILURE); + } + + try + { + getSystemConfig("UMStorageType", UMstorageType); + } + catch (...) + { + exceptionControl("getStorageConfig", oam::API_FAILURE); + } + + try + { + getSystemConfig("DBRootCount", SystemDBRootCount); + } + catch (...) + { + exceptionControl("getStorageConfig", oam::API_FAILURE); + } + + try + { + SystemModuleTypeConfig systemmoduletypeconfig; + getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { - getSystemConfig("DBRootStorageType", storageType); - } - catch (...) - { - exceptionControl("getStorageConfig", oam::API_FAILURE); - } + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; - try - { - getSystemConfig("UMStorageType", UMstorageType); - } - catch (...) - { - exceptionControl("getStorageConfig", oam::API_FAILURE); + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + + if (moduleCount > 0 && moduletype == "pm") + { + deviceDBRootList = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList; + return boost::make_tuple(storageType, SystemDBRootCount, deviceDBRootList, UMstorageType); + } } + } + catch (...) + { + exceptionControl("getStorageConfig", oam::API_FAILURE); + } - try - { - getSystemConfig("DBRootCount", SystemDBRootCount); - } - catch (...) - { - exceptionControl("getStorageConfig", oam::API_FAILURE); - } - - try - { - SystemModuleTypeConfig systemmoduletypeconfig; - getSystemConfig(systemmoduletypeconfig); - - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) - { - if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty() ) - // end of list - break; - - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - - if ( moduleCount > 0 && moduletype == "pm") - { - deviceDBRootList = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList; - return boost::make_tuple(storageType, SystemDBRootCount, deviceDBRootList, UMstorageType); - } - } - } - catch (...) - { - exceptionControl("getStorageConfig", oam::API_FAILURE); - } - - return boost::make_tuple(storageType, SystemDBRootCount, deviceDBRootList, UMstorageType); + return boost::make_tuple(storageType, SystemDBRootCount, deviceDBRootList, UMstorageType); } /******************************************************************** @@ -652,30 +670,30 @@ systemStorageInfo_t Oam::getStorageConfig() ********************************************************************/ void Oam::getPmDbrootConfig(const int pmid, DBRootConfigList& dbrootconfiglist) { - string module = "pm" + itoa(pmid); - // validate Module name - int returnStatus = validateModule(module); + string module = "pm" + itoa(pmid); + // validate Module name + int returnStatus = validateModule(module); - if (returnStatus != API_SUCCESS) - exceptionControl("getPmDbrootConfig", returnStatus); + if (returnStatus != API_SUCCESS) + exceptionControl("getPmDbrootConfig", returnStatus); - try + try + { + ModuleConfig moduleconfig; + getSystemConfig(module, moduleconfig); + + DBRootConfigList::iterator pt1 = moduleconfig.dbrootConfigList.begin(); + + for (; pt1 != moduleconfig.dbrootConfigList.end(); pt1++) { - ModuleConfig moduleconfig; - getSystemConfig(module, moduleconfig); - - DBRootConfigList::iterator pt1 = moduleconfig.dbrootConfigList.begin(); - - for ( ; pt1 != moduleconfig.dbrootConfigList.end() ; pt1++) - { - dbrootconfiglist.push_back((*pt1)); - } - } - catch (...) - { - // dbrootid not found, return with error - exceptionControl("getPmDbrootConfig", API_INVALID_PARAMETER); + dbrootconfiglist.push_back((*pt1)); } + } + catch (...) + { + // dbrootid not found, return with error + exceptionControl("getPmDbrootConfig", API_INVALID_PARAMETER); + } } /******************************************************************** @@ -685,52 +703,53 @@ void Oam::getPmDbrootConfig(const int pmid, DBRootConfigList& dbrootconfiglist) ********************************************************************/ void Oam::getDbrootPmConfig(const int dbrootid, int& pmid) { - SystemModuleTypeConfig systemmoduletypeconfig; - ModuleTypeConfig moduletypeconfig; - ModuleConfig moduleconfig; + SystemModuleTypeConfig systemmoduletypeconfig; + ModuleTypeConfig moduletypeconfig; + ModuleConfig moduleconfig; - try + try + { + getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { - getSystemConfig(systemmoduletypeconfig); + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + + if (moduleCount > 0 && moduletype == "pm") + { + DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++) { - if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty() ) - // end of list - break; + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - - if ( moduleCount > 0 && moduletype == "pm") + for (; pt1 != (*pt).dbrootConfigList.end(); pt1++) + { + if (*pt1 == dbrootid) { - DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end() ; pt++) - { - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for ( ; pt1 != (*pt).dbrootConfigList.end() ; pt1++) - { - if (*pt1 == dbrootid) - { - pmid = (*pt).DeviceID; - return; - } - } - } + pmid = (*pt).DeviceID; + return; } + } } - - // dbrootid not found, return with error - exceptionControl("getDbrootPmConfig", API_INVALID_PARAMETER); + } } - catch (exception& ) - {} // dbrootid not found, return with error exceptionControl("getDbrootPmConfig", API_INVALID_PARAMETER); + } + catch (exception&) + { + } + + // dbrootid not found, return with error + exceptionControl("getDbrootPmConfig", API_INVALID_PARAMETER); } /******************************************************************** @@ -740,49 +759,49 @@ void Oam::getDbrootPmConfig(const int dbrootid, int& pmid) ********************************************************************/ void Oam::getSystemDbrootConfig(DBRootConfigList& dbrootconfiglist) { - SystemModuleTypeConfig systemmoduletypeconfig; - ModuleTypeConfig moduletypeconfig; - ModuleConfig moduleconfig; + SystemModuleTypeConfig systemmoduletypeconfig; + ModuleTypeConfig moduletypeconfig; + ModuleConfig moduleconfig; - try + try + { + getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { - getSystemConfig(systemmoduletypeconfig); + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + + if (moduleCount > 0 && moduletype == "pm") + { + DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++) { - if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty() ) - // end of list - break; + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - - if ( moduleCount > 0 && moduletype == "pm") - { - DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end() ; pt++) - { - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for ( ; pt1 != (*pt).dbrootConfigList.end() ; pt1++) - { - dbrootconfiglist.push_back(*pt1); - } - } - } + for (; pt1 != (*pt).dbrootConfigList.end(); pt1++) + { + dbrootconfiglist.push_back(*pt1); + } } - - sort ( dbrootconfiglist.begin(), dbrootconfiglist.end() ); - } - catch (...) - { - // dbrootid not found, return with error - exceptionControl("getSystemDbrootConfig", API_INVALID_PARAMETER); + } } - return; + sort(dbrootconfiglist.begin(), dbrootconfiglist.end()); + } + catch (...) + { + // dbrootid not found, return with error + exceptionControl("getSystemDbrootConfig", API_INVALID_PARAMETER); + } + + return; } /*************************************************************************** @@ -795,133 +814,132 @@ void Oam::getSystemDbrootConfig(DBRootConfigList& dbrootconfiglist) int Oam::validateModule(const std::string name) { - if ( name.size() < 3 ) - // invalid ID - return API_INVALID_PARAMETER; - - string moduletype = name.substr(0, MAX_MODULE_TYPE_SIZE); - int moduleID = atoi(name.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - - if ( moduleID < 1 ) - // invalid ID - return API_INVALID_PARAMETER; - - SystemModuleTypeConfig systemmoduletypeconfig; - - try - { - getSystemConfig(systemmoduletypeconfig); - } - catch (...) - { - return API_INVALID_PARAMETER; - } - - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) - { - if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType == moduletype ) - { - if (systemmoduletypeconfig.moduletypeconfig[i].ModuleCount == 0 ) - return API_INVALID_PARAMETER; - - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end() ; pt++) - { - if ( name == (*pt).DeviceName ) - return API_SUCCESS; - } - } - } - + if (name.size() < 3) + // invalid ID return API_INVALID_PARAMETER; + + string moduletype = name.substr(0, MAX_MODULE_TYPE_SIZE); + int moduleID = atoi(name.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); + + if (moduleID < 1) + // invalid ID + return API_INVALID_PARAMETER; + + SystemModuleTypeConfig systemmoduletypeconfig; + + try + { + getSystemConfig(systemmoduletypeconfig); + } + catch (...) + { + return API_INVALID_PARAMETER; + } + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + { + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType == moduletype) + { + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleCount == 0) + return API_INVALID_PARAMETER; + + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) + { + if (name == (*pt).DeviceName) + return API_SUCCESS; + } + } + } + + return API_INVALID_PARAMETER; } /****************************************************************************************** -* @brief changeMyCnf -* -* purpose: change my.cnf and add if value is not present to mysqld section -* -******************************************************************************************/ -bool Oam::changeMyCnf( std::string paramater, std::string value ) + * @brief changeMyCnf + * + * purpose: change my.cnf and add if value is not present to mysqld section + * + ******************************************************************************************/ +bool Oam::changeMyCnf(std::string paramater, std::string value) { - string mycnfFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; - ifstream file (mycnfFile.c_str()); + string mycnfFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; + ifstream file(mycnfFile.c_str()); - if (!file) + if (!file) + { + cout << "File not found: " << mycnfFile << endl; + return false; + } + + vector lines; + char line[200]; + string buf; + bool foundIt = false; + + while (file.getline(line, 200)) + { + buf = line; + string::size_type pos = buf.find(paramater, 0); + + if (pos == 0) { - cout << "File not found: " << mycnfFile << endl; - return false; + string::size_type pos = buf.find("=", 0); + + if (pos != string::npos) + { + buf = paramater + " = " + value; + foundIt = true; + } } - vector lines; - char line[200]; - string buf; - bool foundIt = false; + // output to temp file + lines.push_back(buf); + } + + if (!foundIt) + { + // Its not in the file go back to beginning of file + file.clear(); + file.seekg(0, ios::beg); + lines.clear(); while (file.getline(line, 200)) { - buf = line; - string::size_type pos = buf.find(paramater, 0); + buf = line; + string::size_type pos = buf.find("[mysqld]", 0); - if ( pos == 0 ) - { - string::size_type pos = buf.find("=", 0); - - if ( pos != string::npos ) - { - buf = paramater + " = " + value; - foundIt = true; - } - } - - //output to temp file + if (pos != string::npos) + { lines.push_back(buf); + buf = "loose-" + paramater + " = " + value; + } + + // output to temp file + lines.push_back(buf); } + } - if (!foundIt) - { - // Its not in the file go back to beginning of file - file.clear(); - file.seekg(0, ios::beg); - lines.clear(); + file.close(); + unlink(mycnfFile.c_str()); + ofstream newFile(mycnfFile.c_str()); - while (file.getline(line, 200)) - { - buf = line; - string::size_type pos = buf.find("[mysqld]", 0); + // create new file + int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0664); - if ( pos != string::npos ) - { - lines.push_back(buf); - buf = "loose-" + paramater + " = " + value; - } + copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); + newFile.close(); - //output to temp file - lines.push_back(buf); - } - } + close(fd); - file.close(); - unlink (mycnfFile.c_str()); - ofstream newFile (mycnfFile.c_str()); - - //create new file - int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0664); - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - close(fd); - - return true; + return true; } /*************************************************************************** * PRIVATE FUNCTIONS ***************************************************************************/ - /*************************************************************************** * * Function: exceptionControl @@ -932,105 +950,105 @@ bool Oam::changeMyCnf( std::string paramater, std::string value ) void Oam::exceptionControl(std::string function, int returnStatus, const char* extraMsg) { - std::string msg; + std::string msg; - switch (returnStatus) + switch (returnStatus) + { + case API_INVALID_PARAMETER: { - case API_INVALID_PARAMETER: - { - msg = "Invalid Parameter passed in "; - msg.append(function); - msg.append(" API"); - } - break; - - case API_FILE_OPEN_ERROR: - { - msg = "File Open error from "; - msg.append(function); - msg.append(" API"); - } - break; - - case API_TIMEOUT: - { - msg = "Timeout error from "; - msg.append(function); - msg.append(" API"); - } - break; - - case API_DISABLED: - { - msg = "API Disabled: "; - msg.append(function); - } - break; - - case API_FILE_ALREADY_EXIST: - { - msg = "File Already Exist"; - } - break; - - case API_ALREADY_IN_PROGRESS: - { - msg = "Already In Process"; - } - break; - - case API_FAILURE_DB_ERROR: - { - msg = "Database Test Error"; - } - break; - - case API_INVALID_STATE: - { - msg = "Target in an invalid state"; - } - break; - - case API_READONLY_PARAMETER: - { - msg = "Parameter is Read-Only, can't update"; - } - break; - - case API_TRANSACTIONS_COMPLETE: - { - msg = "Finished waiting for transactions"; - } - break; - - case API_CONN_REFUSED: - { - msg = "Connection refused"; - } - break; - - case API_CANCELLED: - { - msg = "Operation Cancelled"; - } - break; - - default: - { - msg = "API Failure return in "; - msg.append(function); - msg.append(" API"); - } - break; - } // end of switch - - if (extraMsg) - { - msg.append(":\n "); - msg.append(extraMsg); + msg = "Invalid Parameter passed in "; + msg.append(function); + msg.append(" API"); } + break; - throw runtime_error(msg); + case API_FILE_OPEN_ERROR: + { + msg = "File Open error from "; + msg.append(function); + msg.append(" API"); + } + break; + + case API_TIMEOUT: + { + msg = "Timeout error from "; + msg.append(function); + msg.append(" API"); + } + break; + + case API_DISABLED: + { + msg = "API Disabled: "; + msg.append(function); + } + break; + + case API_FILE_ALREADY_EXIST: + { + msg = "File Already Exist"; + } + break; + + case API_ALREADY_IN_PROGRESS: + { + msg = "Already In Process"; + } + break; + + case API_FAILURE_DB_ERROR: + { + msg = "Database Test Error"; + } + break; + + case API_INVALID_STATE: + { + msg = "Target in an invalid state"; + } + break; + + case API_READONLY_PARAMETER: + { + msg = "Parameter is Read-Only, can't update"; + } + break; + + case API_TRANSACTIONS_COMPLETE: + { + msg = "Finished waiting for transactions"; + } + break; + + case API_CONN_REFUSED: + { + msg = "Connection refused"; + } + break; + + case API_CANCELLED: + { + msg = "Operation Cancelled"; + } + break; + + default: + { + msg = "API Failure return in "; + msg.append(function); + msg.append(" API"); + } + break; + } // end of switch + + if (extraMsg) + { + msg.append(":\n "); + msg.append(extraMsg); + } + + throw runtime_error(msg); } /*************************************************************************** @@ -1043,9 +1061,9 @@ void Oam::exceptionControl(std::string function, int returnStatus, const char* e std::string Oam::itoa(const int i) { - stringstream ss; - ss << i; - return ss.str(); + stringstream ss; + ss << i; + return ss.str(); } -} //namespace oam +} // namespace oam // vim:ts=4 sw=4: diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 53b26ebbd..54ebc2887 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -56,44 +56,43 @@ namespace oam { - /* * Global OAM parmaters */ /** @brief Maximum Number of Modules within the Calpont System */ -//const int MAX_MODULE = 1024; +// const int MAX_MODULE = 1024; /** @brief Maximum Number of DBRoots within the Calpont System */ const int MAX_DBROOT = 10240; -//const int MAX_DBROOT_AMAZON = 190; //DUE TO DEVICE NAME LIMIT +// const int MAX_DBROOT_AMAZON = 190; //DUE TO DEVICE NAME LIMIT /** @brief Maximum Number of Modules Types within the Calpont System - */ + */ const int MAX_MODULE_TYPE = 3; /** @brief Maximum Number of External Devices within the Calpont System */ -//const int MAX_EXT_DEVICE = 20; +// const int MAX_EXT_DEVICE = 20; /** @brief Maximum Number of Arguments per process */ -//const int MAX_ARGUMENTS = 15; +// const int MAX_ARGUMENTS = 15; /** @brief Maximum Number of Dependancy processes per process */ -//const int MAX_DEPENDANCY = 6; +// const int MAX_DEPENDANCY = 6; /** @brief Maximum Number of processes within the Calpont System */ -//const int MAX_PROCESS_PER_MODULE = 15; -//const int MAX_PROCESS = MAX_MODULE * MAX_PROCESS_PER_MODULE; +// const int MAX_PROCESS_PER_MODULE = 15; +// const int MAX_PROCESS = MAX_MODULE * MAX_PROCESS_PER_MODULE; /** @brief Maximum Number of Parameters per process */ -//const int MAX_PARAMS = 13; +// const int MAX_PARAMS = 13; /** @brief Maximum Module Type Size */ @@ -112,34 +111,32 @@ const int MAX_NIC = 4; const std::string UnassignedIpAddr = "0.0.0.0"; const std::string UnassignedName = "unassigned"; - /** @brief Calpont System Configuration file sections */ -const std::string configSections[] = { "SystemConfig", - "SystemModuleConfig", - "SystemModuleConfig", - "SystemExtDeviceConfig", - "SessionManager", - "VersionBuffer", - "OIDManager", - "PrimitiveServers", - "Installation", - "ExtentMap", - "" - }; +const std::string configSections[] = {"SystemConfig", + "SystemModuleConfig", + "SystemModuleConfig", + "SystemExtDeviceConfig", + "SessionManager", + "VersionBuffer", + "OIDManager", + "PrimitiveServers", + "Installation", + "ExtentMap", + ""}; -//const uint32_t NOTIFICATIONKEY = 0x49444231; +// const uint32_t NOTIFICATIONKEY = 0x49444231; /** @brief Server Type Installs */ enum INSTALLTYPE { - RESERVED, // 0 = not used - INSTALL_NORMAL, // 1 = Normal - dm/um/pm on a seperate servers - INSTALL_COMBINE_DM_UM_PM, // 2 = dm/um/pm on a single server - INSTALL_COMBINE_DM_UM, // 3 = dm/um on a same server - INSTALL_COMBINE_PM_UM // 4 = pm/um on a same server + RESERVED, // 0 = not used + INSTALL_NORMAL, // 1 = Normal - dm/um/pm on a seperate servers + INSTALL_COMBINE_DM_UM_PM, // 2 = dm/um/pm on a single server + INSTALL_COMBINE_DM_UM, // 3 = dm/um on a same server + INSTALL_COMBINE_PM_UM // 4 = pm/um on a same server }; /** @brief OAM API Return values @@ -147,24 +144,24 @@ enum INSTALLTYPE enum API_STATUS { - API_SUCCESS, - API_FAILURE, - API_INVALID_PARAMETER, - API_FILE_OPEN_ERROR, - API_TIMEOUT, - API_DISABLED, - API_FILE_ALREADY_EXIST, - API_ALREADY_IN_PROGRESS, - API_MINOR_FAILURE, - API_FAILURE_DB_ERROR, - API_INVALID_STATE, - API_READONLY_PARAMETER, - API_TRANSACTIONS_COMPLETE, - API_CONN_REFUSED, - API_CANCELLED, - API_STILL_WORKING, - API_DETACH_FAILURE, - API_MAX + API_SUCCESS, + API_FAILURE, + API_INVALID_PARAMETER, + API_FILE_OPEN_ERROR, + API_TIMEOUT, + API_DISABLED, + API_FILE_ALREADY_EXIST, + API_ALREADY_IN_PROGRESS, + API_MINOR_FAILURE, + API_FAILURE_DB_ERROR, + API_INVALID_STATE, + API_READONLY_PARAMETER, + API_TRANSACTIONS_COMPLETE, + API_CONN_REFUSED, + API_CANCELLED, + API_STILL_WORKING, + API_DETACH_FAILURE, + API_MAX }; /** @brief Host/IP Address Config Structure @@ -173,9 +170,9 @@ enum API_STATUS struct HostConfig_s { - std::string HostName; //!< Host Name - std::string IPAddr; //!< IP address - uint16_t NicID; //!< NIC ID + std::string HostName; //!< Host Name + std::string IPAddr; //!< IP address + uint16_t NicID; //!< NIC ID }; typedef struct HostConfig_s HostConfig; @@ -191,10 +188,10 @@ typedef std::vector HostConfigList; struct DeviceNetworkConfig_s { - std::string DeviceName; //!< Device Name - std::string UserTempDeviceName; //!< User Temp Device Name - std::string DisableState; //!< Disabled State - HostConfigList hostConfigList; //!< IP Address and Hostname List + std::string DeviceName; //!< Device Name + std::string UserTempDeviceName; //!< User Temp Device Name + std::string DisableState; //!< Disabled State + HostConfigList hostConfigList; //!< IP Address and Hostname List }; typedef struct DeviceNetworkConfig_s DeviceNetworkConfig; @@ -223,8 +220,8 @@ typedef std::vector DBRootConfigList; struct DeviceDBRootConfig_s { - uint16_t DeviceID; //!< Device ID - DBRootConfigList dbrootConfigList; //!< DBRoot List + uint16_t DeviceID; //!< Device ID + DBRootConfigList dbrootConfigList; //!< DBRoot List }; typedef struct DeviceDBRootConfig_s DeviceDBRootConfig; @@ -237,30 +234,29 @@ typedef std::vector DeviceDBRootList; struct ModuleTypeConfig_s { - std::string ModuleType; //!< Module Type - std::string ModuleDesc; //!< Module Description - std::string RunType; //!< Run Type - uint16_t ModuleCount; //!< Module Equipage Count - uint16_t ModuleCPUCriticalThreshold; //!< CPU Critical Threahold % - uint16_t ModuleCPUMajorThreshold; //!< CPU Major Threahold % - uint16_t ModuleCPUMinorThreshold; //!< CPU Minor Threahold % - uint16_t ModuleCPUMinorClearThreshold; //!< CPU Minor Clear Threahold % - uint16_t ModuleMemCriticalThreshold; //!< Mem Critical Threahold % - uint16_t ModuleMemMajorThreshold; //!< Mem Major Threahold % - uint16_t ModuleMemMinorThreshold; //!< Mem Minor Threahold % - uint16_t ModuleDiskCriticalThreshold; //!< Disk Critical Threahold % - uint16_t ModuleDiskMajorThreshold; //!< Disk Major Threahold % - uint16_t ModuleDiskMinorThreshold; //!< Disk Minor Threahold % - uint16_t ModuleSwapCriticalThreshold; //!< Swap Critical Threahold % - uint16_t ModuleSwapMajorThreshold; //!< Swap Major Threahold % - uint16_t ModuleSwapMinorThreshold; //!< Swap Minor Threahold % - DeviceNetworkList ModuleNetworkList; //!< Module IP Address and Hostname List - DiskMonitorFileSystems FileSystems; //!< Module Disk File System list - DeviceDBRootList ModuleDBRootList; //!< Module DBRoot + std::string ModuleType; //!< Module Type + std::string ModuleDesc; //!< Module Description + std::string RunType; //!< Run Type + uint16_t ModuleCount; //!< Module Equipage Count + uint16_t ModuleCPUCriticalThreshold; //!< CPU Critical Threahold % + uint16_t ModuleCPUMajorThreshold; //!< CPU Major Threahold % + uint16_t ModuleCPUMinorThreshold; //!< CPU Minor Threahold % + uint16_t ModuleCPUMinorClearThreshold; //!< CPU Minor Clear Threahold % + uint16_t ModuleMemCriticalThreshold; //!< Mem Critical Threahold % + uint16_t ModuleMemMajorThreshold; //!< Mem Major Threahold % + uint16_t ModuleMemMinorThreshold; //!< Mem Minor Threahold % + uint16_t ModuleDiskCriticalThreshold; //!< Disk Critical Threahold % + uint16_t ModuleDiskMajorThreshold; //!< Disk Major Threahold % + uint16_t ModuleDiskMinorThreshold; //!< Disk Minor Threahold % + uint16_t ModuleSwapCriticalThreshold; //!< Swap Critical Threahold % + uint16_t ModuleSwapMajorThreshold; //!< Swap Major Threahold % + uint16_t ModuleSwapMinorThreshold; //!< Swap Minor Threahold % + DeviceNetworkList ModuleNetworkList; //!< Module IP Address and Hostname List + DiskMonitorFileSystems FileSystems; //!< Module Disk File System list + DeviceDBRootList ModuleDBRootList; //!< Module DBRoot }; typedef struct ModuleTypeConfig_s ModuleTypeConfig; - /** @brief System Module Type Configuration Structure * * Structure that is returned by the getSystemConfigFile API for the @@ -269,7 +265,7 @@ typedef struct ModuleTypeConfig_s ModuleTypeConfig; struct SystemModuleTypeConfig_s { - std::vector moduletypeconfig; //!< Module Type Configuration Structure + std::vector moduletypeconfig; //!< Module Type Configuration Structure }; typedef struct SystemModuleTypeConfig_s SystemModuleTypeConfig; @@ -281,12 +277,12 @@ typedef struct SystemModuleTypeConfig_s SystemModuleTypeConfig; struct ModuleConfig_s { - std::string ModuleName; //!< Module Name - std::string ModuleType; //!< Module Type - std::string ModuleDesc; //!< Module Description - std::string DisableState; //!< Disabled State - HostConfigList hostConfigList; //!< IP Address and Hostname List - DBRootConfigList dbrootConfigList; //!< DBRoot ID list + std::string ModuleName; //!< Module Name + std::string ModuleType; //!< Module Type + std::string ModuleDesc; //!< Module Description + std::string DisableState; //!< Disabled State + HostConfigList hostConfigList; //!< IP Address and Hostname List + DBRootConfigList dbrootConfigList; //!< DBRoot ID list }; typedef struct ModuleConfig_s ModuleConfig; @@ -301,7 +297,8 @@ typedef struct ModuleConfig_s ModuleConfig; * OAM Standby Parent Flag, */ -typedef boost::tuple oamModuleInfo_t; +typedef boost::tuple + oamModuleInfo_t; /** @brief System Storage Configuration Structure * @@ -309,7 +306,7 @@ typedef boost::tuple systemStorageInfo_t; +typedef boost::tuple systemStorageInfo_t; /** @brief OAM API I/F class * @@ -322,122 +319,120 @@ typedef boost::tuple syst class Oam { -public: - /** @brief OAM C++ API Class constructor - */ - EXPORT Oam(); + public: + /** @brief OAM C++ API Class constructor + */ + EXPORT Oam(); - /** @brief OAM C++ API Class destructor - */ - EXPORT virtual ~Oam(); + /** @brief OAM C++ API Class destructor + */ + EXPORT virtual ~Oam(); - /** @brief get System Module Configuration information - * - * get System Module Configuration information value from the system config file. - * @param systemmoduletypeconfig Returned System Module Configuration Structure - */ - EXPORT void getSystemConfig(SystemModuleTypeConfig& systemmoduletypeconfig); + /** @brief get System Module Configuration information + * + * get System Module Configuration information value from the system config file. + * @param systemmoduletypeconfig Returned System Module Configuration Structure + */ + EXPORT void getSystemConfig(SystemModuleTypeConfig& systemmoduletypeconfig); - /** @brief get System Module Type Configuration information for a Module - * - * get System Module Type Configuration information for a Module from the system config file. - * @param moduletype the Module Type to get information - * @param moduletypeconfig Returned System Module Configuration Structure - */ - EXPORT void getSystemConfig(const std::string& moduletype, ModuleTypeConfig& moduletypeconfig); + /** @brief get System Module Type Configuration information for a Module + * + * get System Module Type Configuration information for a Module from the system config file. + * @param moduletype the Module Type to get information + * @param moduletypeconfig Returned System Module Configuration Structure + */ + EXPORT void getSystemConfig(const std::string& moduletype, ModuleTypeConfig& moduletypeconfig); - /** @brief get System Module Name Configuration information for a Module - * - * get System Module Name Configuration information for a Module from the system config file. - * @param moduleName the Module Name to get information - * @param moduleconfig Returned System Module Configuration Structure - */ - EXPORT void getSystemConfig(const std::string& moduleName, ModuleConfig& moduleconfig); + /** @brief get System Module Name Configuration information for a Module + * + * get System Module Name Configuration information for a Module from the system config file. + * @param moduleName the Module Name to get information + * @param moduleconfig Returned System Module Configuration Structure + */ + EXPORT void getSystemConfig(const std::string& moduleName, ModuleConfig& moduleconfig); - /** @brief get System Configuration String Parameter - * - * get System Configuration String Parameter from the system config file. - * @param name the Parameter Name to get value - * @param value Returned Parameter Value - */ - EXPORT void getSystemConfig(const std::string& name, std::string& value); + /** @brief get System Configuration String Parameter + * + * get System Configuration String Parameter from the system config file. + * @param name the Parameter Name to get value + * @param value Returned Parameter Value + */ + EXPORT void getSystemConfig(const std::string& name, std::string& value); - /** @brief get System Configuration Integer Parameter - * - * get System Configuration Integer Parameter from the system config file. - * @param name the Parameter Name to get value - * @param value Returned Parameter Value - */ - EXPORT void getSystemConfig(const std::string& name, int& value); + /** @brief get System Configuration Integer Parameter + * + * get System Configuration Integer Parameter from the system config file. + * @param name the Parameter Name to get value + * @param value Returned Parameter Value + */ + EXPORT void getSystemConfig(const std::string& name, int& value); - /** @brief get Local Module Configuration Data - * - * get Local Module Name, OAM Parent Flag, and Realtime Linux OS Flag from - * local config file. - * @return oamModuleInfo_t structure, which contains the local Module OAM - * Configuration Data - */ - EXPORT oamModuleInfo_t getModuleInfo(); + /** @brief get Local Module Configuration Data + * + * get Local Module Name, OAM Parent Flag, and Realtime Linux OS Flag from + * local config file. + * @return oamModuleInfo_t structure, which contains the local Module OAM + * Configuration Data + */ + EXPORT oamModuleInfo_t getModuleInfo(); - // Integer to ASCII convertor + // Integer to ASCII convertor - EXPORT std::string itoa(const int); + EXPORT std::string itoa(const int); - /** - *@brief Get Storage Config Data - */ - EXPORT systemStorageInfo_t getStorageConfig(); + /** + *@brief Get Storage Config Data + */ + EXPORT systemStorageInfo_t getStorageConfig(); - /** - *@brief Get PM - DBRoot Config data - */ - EXPORT void getPmDbrootConfig(const int pmid, DBRootConfigList& dbrootconfiglist); + /** + *@brief Get PM - DBRoot Config data + */ + EXPORT void getPmDbrootConfig(const int pmid, DBRootConfigList& dbrootconfiglist); - /** - *@brief Get DBRoot - PM Config data - */ - EXPORT void getDbrootPmConfig(const int dbrootid, int& pmid); + /** + *@brief Get DBRoot - PM Config data + */ + EXPORT void getDbrootPmConfig(const int dbrootid, int& pmid); - /** - *@brief Get System DBRoot Config data - */ - EXPORT void getSystemDbrootConfig(DBRootConfigList& dbrootconfiglist); + /** + *@brief Get System DBRoot Config data + */ + EXPORT void getSystemDbrootConfig(DBRootConfigList& dbrootconfiglist); - /** @brief validate Module name - */ - EXPORT int validateModule(const std::string name); + /** @brief validate Module name + */ + EXPORT int validateModule(const std::string name); - /** - * @brief changeMyCnf - * - * purpose: change my.cnf - * - **/ - EXPORT bool changeMyCnf( std::string paramater, std::string value ); + /** + * @brief changeMyCnf + * + * purpose: change my.cnf + * + **/ + EXPORT bool changeMyCnf(std::string paramater, std::string value); -private: + private: + /** @brief validate Process name + */ + int validateProcess(const std::string moduleName, std::string processName); - /** @brief validate Process name - */ - int validateProcess(const std::string moduleName, std::string processName); + /** @brief local exception control function + * @param function Function throwing the exception + * @param returnStatus + * @param msg A message to be included + */ + EXPORT void exceptionControl(std::string function, int returnStatus, const char* extraMsg = NULL); - /** @brief local exception control function - * @param function Function throwing the exception - * @param returnStatus - * @param msg A message to be included - */ - EXPORT void exceptionControl(std::string function, int returnStatus, const char* extraMsg = NULL); + std::string tmpdir; + std::string CalpontConfigFile; + std::string userDir; - std::string tmpdir; - std::string CalpontConfigFile; - std::string userDir; +}; // end of class -}; // end of class - -} // end of namespace +} // namespace oam #undef EXPORT #endif // vim:ts=4 sw=4: - diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index ba049ed5e..e3ab242ce 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -40,237 +40,240 @@ namespace { oam::OamCache* oamCache = 0; boost::mutex cacheLock; -} +} // namespace namespace oam { - OamCache* OamCache::makeOamCache() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - if (oamCache == 0) - oamCache = new OamCache(); + if (oamCache == 0) + oamCache = new OamCache(); - return oamCache; + return oamCache; } OamCache::OamCache() : mtime(0), mLocalPMId(0) -{} +{ +} OamCache::~OamCache() -{} +{ +} void OamCache::checkReload() { - Oam oam; - config::Config* config = config::Config::makeConfig(); - int temp; + Oam oam; + config::Config* config = config::Config::makeConfig(); + int temp; - if (config->getCurrentMTime() == mtime) - return; + if (config->getCurrentMTime() == mtime) + return; - dbroots.clear(); - oam.getSystemDbrootConfig(dbroots); + dbroots.clear(); + oam.getSystemDbrootConfig(dbroots); - string txt = config->getConfig("SystemConfig", "DBRootCount"); - mtime = config->getLastMTime(); // get 'last' after the first access; quicker than 'current' - idbassert(txt != ""); - numDBRoots = config->fromText(txt); + string txt = config->getConfig("SystemConfig", "DBRootCount"); + mtime = config->getLastMTime(); // get 'last' after the first access; quicker than 'current' + idbassert(txt != ""); + numDBRoots = config->fromText(txt); - dbRootPMMap.reset(new map()); + dbRootPMMap.reset(new map()); - //cout << "reloading oamcache\n"; - for (uint32_t i = 0; i < dbroots.size(); i++) - { - oam.getDbrootPmConfig(dbroots[i], temp); - //cout << " dbroot " << dbroots[i] << " -> PM " << temp << endl; - (*dbRootPMMap)[dbroots[i]] = temp; - } + // cout << "reloading oamcache\n"; + for (uint32_t i = 0; i < dbroots.size(); i++) + { + oam.getDbrootPmConfig(dbroots[i], temp); + // cout << " dbroot " << dbroots[i] << " -> PM " << temp << endl; + (*dbRootPMMap)[dbroots[i]] = temp; + } - ModuleTypeConfig moduletypeconfig; - std::set uniquePids; - oam.getSystemConfig("pm", moduletypeconfig); - int moduleID = 0; + ModuleTypeConfig moduletypeconfig; + std::set uniquePids; + oam.getSystemConfig("pm", moduletypeconfig); + int moduleID = 0; - for (unsigned i = 0; i < moduletypeconfig.ModuleCount; i++) - { - moduleID = atoi((moduletypeconfig.ModuleNetworkList[i]).DeviceName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - uniquePids.insert(moduleID); - } + for (unsigned i = 0; i < moduletypeconfig.ModuleCount; i++) + { + moduleID = atoi((moduletypeconfig.ModuleNetworkList[i]) + .DeviceName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE) + .c_str()); + uniquePids.insert(moduleID); + } - std::set::const_iterator it = uniquePids.begin(); - moduleIds.clear(); - uint32_t i = 0; - map pmToConnectionMap; + std::set::const_iterator it = uniquePids.begin(); + moduleIds.clear(); + uint32_t i = 0; + map pmToConnectionMap; #ifdef _MSC_VER - moduleIds.push_back(*it); - pmToConnectionMap[*it] = i++; + moduleIds.push_back(*it); + pmToConnectionMap[*it] = i++; #else - // Restore for Windows when we support multiple PMs - while (it != uniquePids.end()) - { - pmToConnectionMap[*it] = i++; - moduleIds.push_back(*it); - it++; - } + // Restore for Windows when we support multiple PMs + while (it != uniquePids.end()) + { + pmToConnectionMap[*it] = i++; + moduleIds.push_back(*it); + it++; + } #endif - dbRootConnectionMap.reset(new map()); + dbRootConnectionMap.reset(new map()); - for (i = 0; i < dbroots.size(); i++) + for (i = 0; i < dbroots.size(); i++) + { + map::iterator pmIter = pmToConnectionMap.find((*dbRootPMMap)[dbroots[i]]); + + if (pmIter != pmToConnectionMap.end()) { - map::iterator pmIter = pmToConnectionMap.find((*dbRootPMMap)[dbroots[i]]); - - if (pmIter != pmToConnectionMap.end()) - { - (*dbRootConnectionMap)[dbroots[i]] = (*pmIter).second; - } + (*dbRootConnectionMap)[dbroots[i]] = (*pmIter).second; } + } - pmDbrootsMap.reset(new OamCache::PMDbrootsMap_t::element_type()); - systemStorageInfo_t t; - t = oam.getStorageConfig(); - DeviceDBRootList moduledbrootlist = boost::get<2>(t); - DeviceDBRootList::iterator pt = moduledbrootlist.begin(); + pmDbrootsMap.reset(new OamCache::PMDbrootsMap_t::element_type()); + systemStorageInfo_t t; + t = oam.getStorageConfig(); + DeviceDBRootList moduledbrootlist = boost::get<2>(t); + DeviceDBRootList::iterator pt = moduledbrootlist.begin(); - for ( ; pt != moduledbrootlist.end() ; pt++) + for (; pt != moduledbrootlist.end(); pt++) + { + moduleID = (*pt).DeviceID; + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); + + for (; pt1 != (*pt).dbrootConfigList.end(); pt1++) { - moduleID = (*pt).DeviceID; - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for ( ; pt1 != (*pt).dbrootConfigList.end(); pt1++) - { - (*pmDbrootsMap)[moduleID].push_back(*pt1); - } + (*pmDbrootsMap)[moduleID].push_back(*pt1); } + } - oamModuleInfo_t tm; - tm = oam.getModuleInfo(); - OAMParentModuleName = boost::get<3>(tm); - systemName = config->getConfig("SystemConfig", "SystemName"); + oamModuleInfo_t tm; + tm = oam.getModuleInfo(); + OAMParentModuleName = boost::get<3>(tm); + systemName = config->getConfig("SystemConfig", "SystemName"); } OamCache::dbRootPMMap_t OamCache::getDBRootToPMMap() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return dbRootPMMap; + checkReload(); + return dbRootPMMap; } OamCache::dbRootPMMap_t OamCache::getDBRootToConnectionMap() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return dbRootConnectionMap; + checkReload(); + return dbRootConnectionMap; } OamCache::PMDbrootsMap_t OamCache::getPMToDbrootsMap() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return pmDbrootsMap; + checkReload(); + return pmDbrootsMap; } uint32_t OamCache::getDBRootCount() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return numDBRoots; + checkReload(); + return numDBRoots; } DBRootConfigList& OamCache::getDBRootNums() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return dbroots; + checkReload(); + return dbroots; } std::vector& OamCache::getModuleIds() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return moduleIds; + checkReload(); + return moduleIds; } std::string OamCache::getOAMParentModuleName() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return OAMParentModuleName; + checkReload(); + return OAMParentModuleName; } int OamCache::getLocalPMId() { - boost::mutex::scoped_lock lk(cacheLock); - - // This comes from the file /var/lib/columnstore/local/module, not from the xml. - // Thus, it's not refreshed during checkReload(). - if (mLocalPMId > 0) - { - return mLocalPMId; - } - - string localModule; - string moduleType; - string fileName = "/var/lib/columnstore/local/module"; - ifstream moduleFile (fileName.c_str()); - char line[400]; - - while (moduleFile.getline(line, 400)) - { - localModule = line; - break; - } - - moduleFile.close(); - - if (localModule.empty() ) - { - mLocalPMId = 0; - return mLocalPMId; - } - - moduleType = localModule.substr(0, MAX_MODULE_TYPE_SIZE); - mLocalPMId = atoi(localModule.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - - if (moduleType != "pm") - { - mLocalPMId = 0; - } + boost::mutex::scoped_lock lk(cacheLock); + // This comes from the file /var/lib/columnstore/local/module, not from the xml. + // Thus, it's not refreshed during checkReload(). + if (mLocalPMId > 0) + { return mLocalPMId; + } + + string localModule; + string moduleType; + string fileName = "/var/lib/columnstore/local/module"; + ifstream moduleFile(fileName.c_str()); + char line[400]; + + while (moduleFile.getline(line, 400)) + { + localModule = line; + break; + } + + moduleFile.close(); + + if (localModule.empty()) + { + mLocalPMId = 0; + return mLocalPMId; + } + + moduleType = localModule.substr(0, MAX_MODULE_TYPE_SIZE); + mLocalPMId = atoi(localModule.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); + + if (moduleType != "pm") + { + mLocalPMId = 0; + } + + return mLocalPMId; } string OamCache::getSystemName() { - boost::mutex::scoped_lock lk(cacheLock); + boost::mutex::scoped_lock lk(cacheLock); - checkReload(); - return systemName; + checkReload(); + return systemName; } string OamCache::getModuleName() { - boost::mutex::scoped_lock lk(cacheLock); - - if (!moduleName.empty()) - return moduleName; - - string fileName = "/var/lib/columnstore/local/module"; - ifstream moduleFile(fileName.c_str()); - getline(moduleFile, moduleName); - moduleFile.close(); + boost::mutex::scoped_lock lk(cacheLock); + if (!moduleName.empty()) return moduleName; + + string fileName = "/var/lib/columnstore/local/module"; + ifstream moduleFile(fileName.c_str()); + getline(moduleFile, moduleName); + moduleFile.close(); + + return moduleName; } } /* namespace oam */ diff --git a/oam/oamcpp/oamcache.h b/oam/oamcpp/oamcache.h index 013dcf85a..4c4d99039 100644 --- a/oam/oamcpp/oamcache.h +++ b/oam/oamcpp/oamcache.h @@ -34,52 +34,51 @@ namespace oam { - class OamCache { -public: - typedef boost::shared_ptr > dbRootPMMap_t; - typedef std::vector dbRoots; - typedef boost::shared_ptr > PMDbrootsMap_t; - EXPORT virtual ~OamCache(); + public: + typedef boost::shared_ptr > dbRootPMMap_t; + typedef std::vector dbRoots; + typedef boost::shared_ptr > PMDbrootsMap_t; + EXPORT virtual ~OamCache(); - EXPORT void checkReload(); - EXPORT void forceReload() - { - mtime = 0; - } + EXPORT void checkReload(); + EXPORT void forceReload() + { + mtime = 0; + } - EXPORT dbRootPMMap_t getDBRootToPMMap(); - EXPORT dbRootPMMap_t getDBRootToConnectionMap(); - EXPORT PMDbrootsMap_t getPMToDbrootsMap(); - EXPORT uint32_t getDBRootCount(); - EXPORT DBRootConfigList& getDBRootNums(); - EXPORT std::vector& getModuleIds(); - EXPORT static OamCache* makeOamCache(); - EXPORT std::string getOAMParentModuleName(); - EXPORT int getLocalPMId(); // return the PM id of this machine. - EXPORT std::string getSystemName(); - EXPORT std::string getModuleName(); + EXPORT dbRootPMMap_t getDBRootToPMMap(); + EXPORT dbRootPMMap_t getDBRootToConnectionMap(); + EXPORT PMDbrootsMap_t getPMToDbrootsMap(); + EXPORT uint32_t getDBRootCount(); + EXPORT DBRootConfigList& getDBRootNums(); + EXPORT std::vector& getModuleIds(); + EXPORT static OamCache* makeOamCache(); + EXPORT std::string getOAMParentModuleName(); + EXPORT int getLocalPMId(); // return the PM id of this machine. + EXPORT std::string getSystemName(); + EXPORT std::string getModuleName(); -private: - OamCache(); - OamCache(const OamCache&); - OamCache& operator=(const OamCache&) const; + private: + OamCache(); + OamCache(const OamCache&); + OamCache& operator=(const OamCache&) const; - dbRootPMMap_t dbRootPMMap; - dbRootPMMap_t dbRootConnectionMap; - PMDbrootsMap_t pmDbrootsMap; - uint32_t numDBRoots; - time_t mtime; - DBRootConfigList dbroots; - std::vector moduleIds; - std::string OAMParentModuleName; - int mLocalPMId; // The PM id running on this machine - std::string systemName; - std::string moduleName; + dbRootPMMap_t dbRootPMMap; + dbRootPMMap_t dbRootConnectionMap; + PMDbrootsMap_t pmDbrootsMap; + uint32_t numDBRoots; + time_t mtime; + DBRootConfigList dbroots; + std::vector moduleIds; + std::string OAMParentModuleName; + int mLocalPMId; // The PM id running on this machine + std::string systemName; + std::string moduleName; }; -} /* namespace */ +} // namespace oam #undef EXPORT diff --git a/oam/oamcpp/resource.h b/oam/oamcpp/resource.h index dd31d7c30..b47ef9c17 100644 --- a/oam/oamcpp/resource.h +++ b/oam/oamcpp/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/oam/replaytxnlog/replaytxnlog.cpp b/oam/replaytxnlog/replaytxnlog.cpp index 8bb2d1425..b161feacd 100644 --- a/oam/replaytxnlog/replaytxnlog.cpp +++ b/oam/replaytxnlog/replaytxnlog.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /************************* -* -* $Id: replaytxnlog.cpp 2678 2007-06-14 03:41:50Z wweeks $ -* -*************************/ + * + * $Id: replaytxnlog.cpp 2678 2007-06-14 03:41:50Z wweeks $ + * + *************************/ #include #include #include @@ -44,909 +44,887 @@ namespace fs = boost::filesystem; #include "mcsconfig.h" // Testing definitions. These should never be commented out in checked in code. -#define OAM_AVAILABLE // Comment out to skip the OAM calls to get log file locations and turn on/off logging. -#define LAST_TXN_AVAILABLE // Comment out to use a test start txnId. -#define CALDB_AVAILABLE // Comment out to use against non-Calpont db (commits instead of calcommits). +#define OAM_AVAILABLE // Comment out to skip the OAM calls to get log file locations and turn on/off logging. +#define LAST_TXN_AVAILABLE // Comment out to use a test start txnId. +#define CALDB_AVAILABLE // Comment out to use against non-Calpont db (commits instead of calcommits). ReplayTxnLog::ReplayTxnLog(const string& user, const string& password, const string& stopDateStr, - const bool& ignoreBulkLoad, const bool& reportMode) : - fUser(user), fPassword(password), fStopDateStr(stopDateStr), fIgnoreBulkLoad(ignoreBulkLoad) + const bool& ignoreBulkLoad, const bool& reportMode) + : fUser(user), fPassword(password), fStopDateStr(stopDateStr), fIgnoreBulkLoad(ignoreBulkLoad) { - // Initialize fCurrentYearStr. - // WWW - Check for a better way to do this. - time_t ttNow; - time(&ttNow); - struct tm tmNow; - localtime_r(&ttNow, &tmNow); - int year = tmNow.tm_year + 1900; - ostringstream oss; - oss << setw(4) << year; - fCurrentYearStr = oss.str(); + // Initialize fCurrentYearStr. + // WWW - Check for a better way to do this. + time_t ttNow; + time(&ttNow); + struct tm tmNow; + localtime_r(&ttNow, &tmNow); + int year = tmNow.tm_year + 1900; + ostringstream oss; + oss << setw(4) << year; + fCurrentYearStr = oss.str(); - // Set the program start time. - time(&fProgramStarted); + // Set the program start time. + time(&fProgramStarted); - fDDLCount = 0; - fCommitCount = 0; - fRollbackCount = 0; - fLastSchemaOwner = ""; + fDDLCount = 0; + fCommitCount = 0; + fRollbackCount = 0; + fLastSchemaOwner = ""; }; /*************************************************************************** -* -* Function: validateInput -* -* Purpose: Validates the user id, password, and stop date string. Prompts -* for user id, password, and stop date if they were not passed in -* with the constructor. Attempts to start a database session using -* the user and password. Throws a runtime_error if the session -* can not be started. -* -****************************************************************************/ + * + * Function: validateInput + * + * Purpose: Validates the user id, password, and stop date string. Prompts + * for user id, password, and stop date if they were not passed in + * with the constructor. Attempts to start a database session using + * the user and password. Throws a runtime_error if the session + * can not be started. + * + ****************************************************************************/ void ReplayTxnLog::validateInput() { + bool validStopDate = false; - bool validStopDate = false; + // Prompt for a user if not provided. + /* + while(fUser.length() == 0) { + cout << "Enter User: "; + getline(cin, fUser); + } - // Prompt for a user if not provided. - /* - while(fUser.length() == 0) { - cout << "Enter User: "; - getline(cin, fUser); - } + // Prompt including commits and rollbacks.for a password if not provided. + // WWW - Make the password masked with *s. + while(fPassword.length() == 0 && !fReportMode) { + cout << "Enter Password: "; + getline(cin, fPassword); + } - // Prompt including commits and rollbacks.for a password if not provided. - // WWW - Make the password masked with *s. - while(fPassword.length() == 0 && !fReportMode) { - cout << "Enter Password: "; - getline(cin, fPassword); - } + if(!fReportMode) { + // @bug 2417. Took out the convert user id to uppercase. Leftover from a prior db front end. + // boost::algorithm::to_upper(fUser); + fSession.startSession(fUser, fPassword, fOracleSID); + } - if(!fReportMode) { - // @bug 2417. Took out the convert user id to uppercase. Leftover from a prior db front end. - // boost::algorithm::to_upper(fUser); - fSession.startSession(fUser, fPassword, fOracleSID); - } - - // Validate the stop date. Default to 'Now' if a stop date wasn't passed. - while(!validStopDate) { - */ - if (fStopDateStr == "Now" || fStopDateStr.length() == 0) + // Validate the stop date. Default to 'Now' if a stop date wasn't passed. + while(!validStopDate) { + */ + if (fStopDateStr == "Now" || fStopDateStr.length() == 0) + { + validStopDate = true; + time(&fStopDate); + } + else if (fStopDateStr.length() == 17) + { + if ((fStopDateStr[2] != '/') || (fStopDateStr[5] != '/') || (fStopDateStr[8] != '@') || + (fStopDateStr[11] != ':') || (fStopDateStr[14] != ':')) { - validStopDate = true; - time(&fStopDate); - } - else if (fStopDateStr.length() == 17) - { - if ((fStopDateStr[2] != '/') || (fStopDateStr[5] != '/') || - (fStopDateStr[8] != '@') || (fStopDateStr[11] != ':') || - (fStopDateStr[14] != ':')) - { - validStopDate = false; - } - else - { - tm lts; - - // Build date structure. - lts.tm_isdst = -1; // -1 for unknown whether date is during Daylight Saving Time. - lts.tm_mon = atoi(fStopDateStr.substr(0, 2).c_str()) - 1; // Month is 0..11 - lts.tm_mday = atoi(fStopDateStr.substr(3, 2).c_str()); // Day - lts.tm_year = atoi(fStopDateStr.substr(6, 2).c_str()) + 100; // Year since 1900 - lts.tm_hour = atoi(fStopDateStr.substr(9, 2).c_str()); // Hour - lts.tm_min = atoi(fStopDateStr.substr(12, 2).c_str()); // Minute - lts.tm_sec = atoi(fStopDateStr.substr(15, 2).c_str()); // Second - - // Convert to a time_t object. - fStopDate = mktime(<s); - - validStopDate = true; - } - } - - if (!validStopDate) - { - if (fStopDateStr.length() > 0) - { - /* - cout << endl; - cout << "'" << fStopDateStr << "' is not a valid date and time." << endl; - cout << endl; - */ - throw runtime_error(fStopDateStr + " is not a valid date and time."); - - } - - cout << "Enter Stop Date and Time as 'mm/dd/yy@hh:mm:ss' or 'Now': "; - getline(cin, fStopDateStr); - } - -// } -} - -/*************************************************************************** -* -* Function: split -* -* Purpose: Split a string on a given character into a vector of strings. -* The vector is passed by reference and cleared each time. -* The number of strings split out is returned. -* -****************************************************************************/ -int ReplayTxnLog::split(vector& v, const string& str, const char& c) -{ - v.clear(); - string::const_iterator s = str.begin(); - - while (true) - { - string::const_iterator begin = s; - - while (*s != c && s != str.end()) - { - ++s; - } - - v.push_back(string(begin, s)); - - if (s == str.end()) - { - break; - } - - if (++s == str.end()) - { - v.push_back(""); - break; - } - } - - return v.size(); -} - -/*************************************************************************** -* -* Function: convertLogDateStr -* -* Purpose: Converts a string formatted as "Apr 30 16:10:49" to a time_t. -* The date is passed in as three string parameters, such as -* "Apr", "30", "16:10:49". -* -* Note: There is no year included in the syslog time stamp. This -* function returns the date in the current year if the date is -* at or before the current date and time. If the date on a -* after current date / time, it returns the date in the prior year. -* -****************************************************************************/ -time_t ReplayTxnLog::convertLogDateStr(const string& sMth, const string& sDay, const string& sTime) -{ - time_t tt; - tm ltm = {0, 0, 0, 0, 0, 0, 0, 0, -1}; // -1 is tm_dst unknown. - - // Convert the log time to a time_t using the current year. - string dt = fCurrentYearStr + "-" + sMth + "-" + sDay + "-" + sTime; - strptime(dt.c_str(), "%Y-%b-%d-%H:%M:%S", <m); - tt = mktime(<m); - - // If the log time is in the future, change it to the prior year. - if (tt > fProgramStarted) - { - ltm.tm_year--; - tt = mktime(<m); - } - - return tt; -} - -/*************************************************************************** -* - * Function: getNextCalpontLogEntry -* -* Purpose: Reads the passed filestream until a Calpont log entry is found. -* The log entry is assigned in the pass by reference argument. -* A bool is returned indicating whether a Calpont entry was found. -* -****************************************************************************/ -bool ReplayTxnLog::getNextCalpontLogEntry(ifstream& dataFile, LogEntry& logEntry) -{ - bool found = false; - string text; - string::size_type pos; - vector v; - vector vFirstSection; - /* - Sample log entries: - Mar 11 01:54:04 srvswdev3 Calpont[29720]: 04.372658 |5|1|0| C 00 CAL0018: DDL|WALT|CREATE TABLE TEST (XXX CHAR(8)); - Mar 11 02:40:22 srvswdev3 Calpont[29720]: 04.373627 |7|2|0| C 00 CAL0017: DML|WALT|INSERT INTO TEST (XXX) values ('TEST'); - Mar 11 02:40:24 srvswdev3 Calpont[29720]: 04.373429 |5|1|0| C 00 CAL0019: CMD|COMMIT; - */ - - while (!found && getline(dataFile, text)) - { - split(v, text, '|'); - - if (v.size() >= 5) - { - - // Read the log entry type. - if (v[4].find("CAL0017") != string::npos) - { - logEntry.type = DML; - } - else if (v[4].find("CAL0019") != string::npos) - { - if (v[5].find("COMMIT") != string::npos) - { - logEntry.type = XCOMMIT; // WWW - COMMIT caused compiler error, - } - else if (v[5].find("ROLLBACK") != string::npos) - { - logEntry.type = ROLLBACK; - } - } - else if (v[4].find("CAL0018") != string::npos) - { - logEntry.type = DDL; - } - else if (v[4].find("CAL0008") != string::npos) - { - logEntry.type = BULKLOAD; - logEntry.fSchemaOwner = v[5]; // steal schema owner for job number - logEntry.fSqlStatement = v[6]; - - } - else if (v[4].find("CAL0022") != string::npos) - { - if (!fIgnoreBulkLoad) - { - cout << endl; - cout << "Encountered the following bulk load log entry." << endl; - cout << text << endl; - cout << "Press Enter to continue." << endl; - getline(cin, text); - cout << endl; - continue; - } - } - else - { - continue; - } - - // Get the versionID. - logEntry.fVersionID = atoi(v[2].c_str()); - - // Get the current schema and sql for DML or DDL. - if (logEntry.type == DML or logEntry.type == DDL) - { - logEntry.fSchemaOwner = v[5]; - - // Parse out the sql statement. - if (v.size() == 7) - { - logEntry.fSqlStatement = v[6]; - } - // Sql contains one or more "|" (separator) characters. - else - { - pos = text.find(v[6]); - logEntry.fSqlStatement = text.substr(pos, text.size() - pos); - } - } - - // Parse out the date and time. - split(vFirstSection, v[0], ' '); - - if (vFirstSection.size() < 3) - { - continue; - } - else - { - - // Quick hack for extra spaces showing up in the date such as: - // Mar 7 21:28:36 srvperf6 DDLProc[7964]: 36.179609 |1|86|0| C 15 CAL0018: DDL ||CREATE TABLE Y (K NUMERIC)ENGINE=INFINIDB; - // This can be stripped out when the log is no longer using syslog. - if (vFirstSection[1].size() == 0) - logEntry.fLogTime = convertLogDateStr(vFirstSection[0], vFirstSection[2], vFirstSection[3]); - else - logEntry.fLogTime = convertLogDateStr(vFirstSection[0], vFirstSection[1], vFirstSection[2]); - } - - found = true; - } - } - - return found; -} - - -/*************************************************************************** -* -* Function: getLogFileNames -* -* Purpose: Returns the list of files that contain the transactions that will -* be replayed. -* -* NOTE: Sorting the files by name must put the files in -* date sorted order or this will not work. -* -****************************************************************************/ -list ReplayTxnLog::getLogFileNames() -{ - oam::Oam oam; - string::size_type pos; - string fullLogFileName; - string dir; - string archiveDir; - string logFileName; - - // Get the log file name, directory, and archive file directory. - /* - #ifdef OAM_AVAILABLE - oam.getLogFile("pm1", "data", fullLogFileName); - #else - */ - fullLogFileName = string(MCSLOGDIR) + "/data/data_mods.log"; - /* - #endif - */ - pos = fullLogFileName.rfind("/"); - - if (pos == string::npos) - { - throw runtime_error("Unable to parse log file path - " + fullLogFileName); - } - - dir = fullLogFileName.substr(0, pos); - logFileName = fullLogFileName.substr(pos + 1, fullLogFileName.size() - pos + 1); - archiveDir = dir + "/archive"; - - // Read the archive directory and get a list of archived log file names. - list dbFileNames; - string fileName; - fs::path sourceDir(archiveDir); - fs::directory_iterator iter(sourceDir); - fs::directory_iterator end_iter; - - while (iter != end_iter) - { - fs::path source = *iter; - - if (!fs::is_directory(source) ) - { -#if BOOST_VERSION >= 105200 - fileName = source.filename().c_str(); -#else - fileName = iter->leaf(); -#endif - - if (fileName.find(logFileName, 0) == 0) - { - fileName = archiveDir + "/" + fileName; - dbFileNames.push_back(fileName); - } - } - - ++iter; - } - - // Sort the list by file name. The files are named such that sorting them by name will be the - // same as sorting them by the order they were created. - dbFileNames.sort(); - - // Make sure the current log file exists and add it to the end of the list. - if (!fs::exists(fullLogFileName)) - { - throw runtime_error("Current log file not found: " + fullLogFileName); + validStopDate = false; } else { - dbFileNames.push_back(fullLogFileName); + tm lts; + + // Build date structure. + lts.tm_isdst = -1; // -1 for unknown whether date is during Daylight Saving Time. + lts.tm_mon = atoi(fStopDateStr.substr(0, 2).c_str()) - 1; // Month is 0..11 + lts.tm_mday = atoi(fStopDateStr.substr(3, 2).c_str()); // Day + lts.tm_year = atoi(fStopDateStr.substr(6, 2).c_str()) + 100; // Year since 1900 + lts.tm_hour = atoi(fStopDateStr.substr(9, 2).c_str()); // Hour + lts.tm_min = atoi(fStopDateStr.substr(12, 2).c_str()); // Minute + lts.tm_sec = atoi(fStopDateStr.substr(15, 2).c_str()); // Second + + // Convert to a time_t object. + fStopDate = mktime(<s); + + validStopDate = true; } + } - // WWW - Warn the user and prompt to exit if there are missing file names. - - // If there is more than one log file, Read through the first transaction id in each file until - // we find the one that begins with an id greater than the one we are starting with. Once we've - // found that, we know we can start with the previous file. - list::iterator curFile; - list::iterator emptyFile; - vector v; - - if (dbFileNames.size() > 1) + if (!validStopDate) + { + if (fStopDateStr.length() > 0) { - curFile = dbFileNames.begin(); - curFile++; - bool done = false; - bool validLogEntry = false; - LogEntry logEntry; - - while (!done && curFile != dbFileNames.end()) - { - fileName = *curFile; - ifstream dataFile(fileName.c_str()); - validLogEntry = getNextCalpontLogEntry(dataFile, logEntry); - dataFile.close(); - - if (!validLogEntry) // Empty file - { - emptyFile = curFile; - curFile++; - dbFileNames.erase(emptyFile); // Zap the empty file from the list. - } - else if (logEntry.fVersionID > fStartVersionID) - { - done = true; - } - else - { - - // This file starts with an id less than our start id, so the prior file - // contains only transactions from before the backup was restored. Throw - // the prior file away and keep looking. - dbFileNames.pop_front(); - curFile++; - } - } + /* + cout << endl; + cout << "'" << fStopDateStr << "' is not a valid date and time." << endl; + cout << endl; + */ + throw runtime_error(fStopDateStr + " is not a valid date and time."); } + cout << "Enter Stop Date and Time as 'mm/dd/yy@hh:mm:ss' or 'Now': "; + getline(cin, fStopDateStr); + } - return dbFileNames; + // } } /*************************************************************************** -* -* Function: processStatement -* -* Purpose: Receives a versionID and a statement. -* -* For DML statements, the statement is added to the transaction if -* it's a subsequent statement in an active transaction. If it's for a -* new transaction, the transactioni is first added to the vector of -* active transactions, then the statement is added to the transaction's -* list of statements. -* -* For Rollbacks, the transaction is zapped from the vector of active -* transactions. No database rollback is done because the statements -* within the transaction never hit the database. -* -* For Commits, the transaction's statements are issued and committed -* to the database, then the transaction is removed from the vector. -* -* For DDL statements, the statement is executed immediately. Each -* DDL statement is it's own transaction. -****************************************************************************/ + * + * Function: split + * + * Purpose: Split a string on a given character into a vector of strings. + * The vector is passed by reference and cleared each time. + * The number of strings split out is returned. + * + ****************************************************************************/ +int ReplayTxnLog::split(vector& v, const string& str, const char& c) +{ + v.clear(); + string::const_iterator s = str.begin(); + + while (true) + { + string::const_iterator begin = s; + + while (*s != c && s != str.end()) + { + ++s; + } + + v.push_back(string(begin, s)); + + if (s == str.end()) + { + break; + } + + if (++s == str.end()) + { + v.push_back(""); + break; + } + } + + return v.size(); +} + +/*************************************************************************** + * + * Function: convertLogDateStr + * + * Purpose: Converts a string formatted as "Apr 30 16:10:49" to a time_t. + * The date is passed in as three string parameters, such as + * "Apr", "30", "16:10:49". + * + * Note: There is no year included in the syslog time stamp. This + * function returns the date in the current year if the date is + * at or before the current date and time. If the date on a + * after current date / time, it returns the date in the prior year. + * + ****************************************************************************/ +time_t ReplayTxnLog::convertLogDateStr(const string& sMth, const string& sDay, const string& sTime) +{ + time_t tt; + tm ltm = {0, 0, 0, 0, 0, 0, 0, 0, -1}; // -1 is tm_dst unknown. + + // Convert the log time to a time_t using the current year. + string dt = fCurrentYearStr + "-" + sMth + "-" + sDay + "-" + sTime; + strptime(dt.c_str(), "%Y-%b-%d-%H:%M:%S", <m); + tt = mktime(<m); + + // If the log time is in the future, change it to the prior year. + if (tt > fProgramStarted) + { + ltm.tm_year--; + tt = mktime(<m); + } + + return tt; +} + +/*************************************************************************** + * + * Function: getNextCalpontLogEntry + * + * Purpose: Reads the passed filestream until a Calpont log entry is found. + * The log entry is assigned in the pass by reference argument. + * A bool is returned indicating whether a Calpont entry was found. + * + ****************************************************************************/ +bool ReplayTxnLog::getNextCalpontLogEntry(ifstream& dataFile, LogEntry& logEntry) +{ + bool found = false; + string text; + string::size_type pos; + vector v; + vector vFirstSection; + /* + Sample log entries: + Mar 11 01:54:04 srvswdev3 Calpont[29720]: 04.372658 |5|1|0| C 00 CAL0018: DDL|WALT|CREATE TABLE TEST (XXX + CHAR(8)); Mar 11 02:40:22 srvswdev3 Calpont[29720]: 04.373627 |7|2|0| C 00 CAL0017: DML|WALT|INSERT INTO + TEST (XXX) values ('TEST'); Mar 11 02:40:24 srvswdev3 Calpont[29720]: 04.373429 |5|1|0| C 00 CAL0019: + CMD|COMMIT; + */ + + while (!found && getline(dataFile, text)) + { + split(v, text, '|'); + + if (v.size() >= 5) + { + // Read the log entry type. + if (v[4].find("CAL0017") != string::npos) + { + logEntry.type = DML; + } + else if (v[4].find("CAL0019") != string::npos) + { + if (v[5].find("COMMIT") != string::npos) + { + logEntry.type = XCOMMIT; // WWW - COMMIT caused compiler error, + } + else if (v[5].find("ROLLBACK") != string::npos) + { + logEntry.type = ROLLBACK; + } + } + else if (v[4].find("CAL0018") != string::npos) + { + logEntry.type = DDL; + } + else if (v[4].find("CAL0008") != string::npos) + { + logEntry.type = BULKLOAD; + logEntry.fSchemaOwner = v[5]; // steal schema owner for job number + logEntry.fSqlStatement = v[6]; + } + else if (v[4].find("CAL0022") != string::npos) + { + if (!fIgnoreBulkLoad) + { + cout << endl; + cout << "Encountered the following bulk load log entry." << endl; + cout << text << endl; + cout << "Press Enter to continue." << endl; + getline(cin, text); + cout << endl; + continue; + } + } + else + { + continue; + } + + // Get the versionID. + logEntry.fVersionID = atoi(v[2].c_str()); + + // Get the current schema and sql for DML or DDL. + if (logEntry.type == DML or logEntry.type == DDL) + { + logEntry.fSchemaOwner = v[5]; + + // Parse out the sql statement. + if (v.size() == 7) + { + logEntry.fSqlStatement = v[6]; + } + // Sql contains one or more "|" (separator) characters. + else + { + pos = text.find(v[6]); + logEntry.fSqlStatement = text.substr(pos, text.size() - pos); + } + } + + // Parse out the date and time. + split(vFirstSection, v[0], ' '); + + if (vFirstSection.size() < 3) + { + continue; + } + else + { + // Quick hack for extra spaces showing up in the date such as: + // Mar 7 21:28:36 srvperf6 DDLProc[7964]: 36.179609 |1|86|0| C 15 CAL0018: DDL ||CREATE TABLE Y (K + // NUMERIC)ENGINE=INFINIDB; This can be stripped out when the log is no longer using syslog. + if (vFirstSection[1].size() == 0) + logEntry.fLogTime = convertLogDateStr(vFirstSection[0], vFirstSection[2], vFirstSection[3]); + else + logEntry.fLogTime = convertLogDateStr(vFirstSection[0], vFirstSection[1], vFirstSection[2]); + } + + found = true; + } + } + + return found; +} + +/*************************************************************************** + * + * Function: getLogFileNames + * + * Purpose: Returns the list of files that contain the transactions that will + * be replayed. + * + * NOTE: Sorting the files by name must put the files in + * date sorted order or this will not work. + * + ****************************************************************************/ +list ReplayTxnLog::getLogFileNames() +{ + oam::Oam oam; + string::size_type pos; + string fullLogFileName; + string dir; + string archiveDir; + string logFileName; + + // Get the log file name, directory, and archive file directory. + /* + #ifdef OAM_AVAILABLE + oam.getLogFile("pm1", "data", fullLogFileName); + #else + */ + fullLogFileName = string(MCSLOGDIR) + "/data/data_mods.log"; + /* + #endif + */ + pos = fullLogFileName.rfind("/"); + + if (pos == string::npos) + { + throw runtime_error("Unable to parse log file path - " + fullLogFileName); + } + + dir = fullLogFileName.substr(0, pos); + logFileName = fullLogFileName.substr(pos + 1, fullLogFileName.size() - pos + 1); + archiveDir = dir + "/archive"; + + // Read the archive directory and get a list of archived log file names. + list dbFileNames; + string fileName; + fs::path sourceDir(archiveDir); + fs::directory_iterator iter(sourceDir); + fs::directory_iterator end_iter; + + while (iter != end_iter) + { + fs::path source = *iter; + + if (!fs::is_directory(source)) + { +#if BOOST_VERSION >= 105200 + fileName = source.filename().c_str(); +#else + fileName = iter->leaf(); +#endif + + if (fileName.find(logFileName, 0) == 0) + { + fileName = archiveDir + "/" + fileName; + dbFileNames.push_back(fileName); + } + } + + ++iter; + } + + // Sort the list by file name. The files are named such that sorting them by name will be the + // same as sorting them by the order they were created. + dbFileNames.sort(); + + // Make sure the current log file exists and add it to the end of the list. + if (!fs::exists(fullLogFileName)) + { + throw runtime_error("Current log file not found: " + fullLogFileName); + } + else + { + dbFileNames.push_back(fullLogFileName); + } + + // WWW - Warn the user and prompt to exit if there are missing file names. + + // If there is more than one log file, Read through the first transaction id in each file until + // we find the one that begins with an id greater than the one we are starting with. Once we've + // found that, we know we can start with the previous file. + list::iterator curFile; + list::iterator emptyFile; + vector v; + + if (dbFileNames.size() > 1) + { + curFile = dbFileNames.begin(); + curFile++; + bool done = false; + bool validLogEntry = false; + LogEntry logEntry; + + while (!done && curFile != dbFileNames.end()) + { + fileName = *curFile; + ifstream dataFile(fileName.c_str()); + validLogEntry = getNextCalpontLogEntry(dataFile, logEntry); + dataFile.close(); + + if (!validLogEntry) // Empty file + { + emptyFile = curFile; + curFile++; + dbFileNames.erase(emptyFile); // Zap the empty file from the list. + } + else if (logEntry.fVersionID > fStartVersionID) + { + done = true; + } + else + { + // This file starts with an id less than our start id, so the prior file + // contains only transactions from before the backup was restored. Throw + // the prior file away and keep looking. + dbFileNames.pop_front(); + curFile++; + } + } + } + + return dbFileNames; +} + +/*************************************************************************** + * + * Function: processStatement + * + * Purpose: Receives a versionID and a statement. + * + * For DML statements, the statement is added to the transaction if + * it's a subsequent statement in an active transaction. If it's for a + * new transaction, the transactioni is first added to the vector of + * active transactions, then the statement is added to the transaction's + * list of statements. + * + * For Rollbacks, the transaction is zapped from the vector of active + * transactions. No database rollback is done because the statements + * within the transaction never hit the database. + * + * For Commits, the transaction's statements are issued and committed + * to the database, then the transaction is removed from the vector. + * + * For DDL statements, the statement is executed immediately. Each + * DDL statement is it's own transaction. + ****************************************************************************/ void ReplayTxnLog::processStatement(const LogEntry& logEntry) { + vector::iterator curTxn; + bool found = false; - vector::iterator curTxn; - bool found = false; + // First, see if transaction already active. + curTxn = fActiveTxns.begin(); - // First, see if transaction already active. - curTxn = fActiveTxns.begin(); - - while (!found && (curTxn != fActiveTxns.end())) + while (!found && (curTxn != fActiveTxns.end())) + { + if (curTxn->fVersionID == logEntry.fVersionID) { - if (curTxn->fVersionID == logEntry.fVersionID) - { - found = true; - } - else - { - curTxn++; - } + found = true; + } + else + { + curTxn++; + } + } + + // If it's a commit or DDL statement, issue each statement for the transaction, then + // process the DLL statement or issue the commit. + if (logEntry.type == XCOMMIT || logEntry.type == DDL) + { + replaytxnlog::DBSession dbSession; + + // Track the first committed txn and last committed txn. DDL statements cause + // an auto commit, so consider those to be commits. + if (fLogEntryFirstCommit.fLogTime == 0) + { + fLogEntryFirstCommit = logEntry; } + fLogEntryLastCommit = logEntry; - // If it's a commit or DDL statement, issue each statement for the transaction, then - // process the DLL statement or issue the commit. - if (logEntry.type == XCOMMIT || logEntry.type == DDL) + // Increment commit count or ddl count. + if (logEntry.type == XCOMMIT) { - - replaytxnlog::DBSession dbSession; - - // Track the first committed txn and last committed txn. DDL statements cause - // an auto commit, so consider those to be commits. - if (fLogEntryFirstCommit.fLogTime == 0) - { - fLogEntryFirstCommit = logEntry; - } - - fLogEntryLastCommit = logEntry; - - // Increment commit count or ddl count. - if (logEntry.type == XCOMMIT) - { - fCommitCount++; - } - else - { - fDDLCount++; - } - - cout << "-- Transaction " << logEntry.fVersionID << " committed " << - ctime(&logEntry.fLogTime) ; - dbSession.startSession(logEntry.fSchemaOwner); - - if (found) - { - - // Loop through the statements and execute each one. - vector::iterator curStatement; - - for (curStatement = curTxn->fStatements.begin(); - curStatement != curTxn->fStatements.end(); curStatement++) - { - dbSession.issueStatement(curStatement->fSqlStatement); - } - - // Commit. - curTxn->fStatements.clear(); - - if (logEntry.type == XCOMMIT) - { - dbSession.commit(); - } - - // Clear the transaction from the vector. - fActiveTxns.erase(curTxn); - } - - // If it's DDL, issue the statement. - if (logEntry.type == DDL) - { - dbSession.issueStatement(logEntry.fSqlStatement); - } - - dbSession.endSession(); + fCommitCount++; + } + else + { + fDDLCount++; } - else if (logEntry.type == BULKLOAD) + cout << "-- Transaction " << logEntry.fVersionID << " committed " << ctime(&logEntry.fLogTime); + dbSession.startSession(logEntry.fSchemaOwner); + + if (found) { - cout << "-- Import completed " << ctime(&logEntry.fLogTime); - cout << "-- " << logEntry.fSqlStatement << endl; - cout << "-- " << logEntry.fSchemaOwner << endl << endl; + // Loop through the statements and execute each one. + vector::iterator curStatement; + + for (curStatement = curTxn->fStatements.begin(); curStatement != curTxn->fStatements.end(); + curStatement++) + { + dbSession.issueStatement(curStatement->fSqlStatement); + } + + // Commit. + curTxn->fStatements.clear(); + + if (logEntry.type == XCOMMIT) + { + dbSession.commit(); + } + + // Clear the transaction from the vector. + fActiveTxns.erase(curTxn); } - // If it's a rollback, clear the transaction from the vector. No need to do any database work. - else if (logEntry.type == ROLLBACK) + // If it's DDL, issue the statement. + if (logEntry.type == DDL) { - fRollbackCount++; - - if (found) - { - - // Throw away the transaction. - curTxn->fStatements.clear(); - fActiveTxns.erase(curTxn); - } + dbSession.issueStatement(logEntry.fSqlStatement); } - // If it's a DML statement, add it to the vector of statements for the transaction. - else if (logEntry.type == DML) - { - if (found) - { - curTxn->fStatements.push_back(logEntry); - } - else - { - ActiveTxn newTxn; - newTxn.fVersionID = logEntry.fVersionID; - newTxn.fStatements.push_back(logEntry); - fActiveTxns.push_back(newTxn); - } - } + dbSession.endSession(); + } + else if (logEntry.type == BULKLOAD) + { + cout << "-- Import completed " << ctime(&logEntry.fLogTime); + cout << "-- " << logEntry.fSqlStatement << endl; + cout << "-- " << logEntry.fSchemaOwner << endl << endl; + } + + // If it's a rollback, clear the transaction from the vector. No need to do any database work. + else if (logEntry.type == ROLLBACK) + { + fRollbackCount++; + + if (found) + { + // Throw away the transaction. + curTxn->fStatements.clear(); + fActiveTxns.erase(curTxn); + } + } + + // If it's a DML statement, add it to the vector of statements for the transaction. + else if (logEntry.type == DML) + { + if (found) + { + curTxn->fStatements.push_back(logEntry); + } + else + { + ActiveTxn newTxn; + newTxn.fVersionID = logEntry.fVersionID; + newTxn.fStatements.push_back(logEntry); + fActiveTxns.push_back(newTxn); + } + } } /*************************************************************************** -* -* Function: displaySummaryAndConfirm -* -* Purpose: Displays a message with the date and time of the first transaction -* to be replayed and the date and time of the last transaction. -****************************************************************************/ + * + * Function: displaySummaryAndConfirm + * + * Purpose: Displays a message with the date and time of the first transaction + * to be replayed and the date and time of the last transaction. + ****************************************************************************/ bool ReplayTxnLog::displaySummaryAndConfirm(const LogEntry& logEntry) { - string replyYN; + string replyYN; - /* - // Show summary. - cout << endl; - cout << "--" << endl; - cout << "-- Summary" << endl; - cout << "--" << endl; - cout << "-- First Transaction : " << logEntry.fVersionID << endl; - cout << "-- Start Date and Time : " << ctime(&logEntry.fLogTime); - cout << "-- Stop Date and Time : " << ctime(&fStopDate) << endl; - cout << "-- Transactions that were committed before the stop date and time will be reported." << endl; - cout << "--" << endl; + /* + // Show summary. + cout << endl; + cout << "--" << endl; + cout << "-- Summary" << endl; + cout << "--" << endl; + cout << "-- First Transaction : " << logEntry.fVersionID << endl; + cout << "-- Start Date and Time : " << ctime(&logEntry.fLogTime); + cout << "-- Stop Date and Time : " << ctime(&fStopDate) << endl; + cout << "-- Transactions that were committed before the stop date and time will be reported." << endl; + cout << "--" << endl; - cout << endl; + cout << endl; - // Prompt to continue. - do { - cout << "Would you like to continue (Y/N)?"; - getline(cin, replyYN); - } - while ((replyYN != "Y") && (replyYN != "N")); - cout << endl; - if(replyYN == "Y") { - return true; - } - return false; - */ - return true; + // Prompt to continue. + do { + cout << "Would you like to continue (Y/N)?"; + getline(cin, replyYN); + } + while ((replyYN != "Y") && (replyYN != "N")); + cout << endl; + if(replyYN == "Y") { + return true; + } + return false; + */ + return true; } /*************************************************************************** -* -* Function: displayProgress -* -* Purpose: Outputs progress message. -****************************************************************************/ + * + * Function: displayProgress + * + * Purpose: Outputs progress message. + ****************************************************************************/ void ReplayTxnLog::displayProgress(const string& fileName) { - // WWW - Check into having this stay on the same line like dbBuilder. - /* - if(fLogEntryLastCommit.fLogTime > 0) { - cout << "Last transaction committed " << fLogEntryLastCommit.fVersionID << " from " << - ctime(&fLogEntryLastCommit.fLogTime); - } - cout << endl; - cout << "Now processing file " << fileName << endl; - */ + // WWW - Check into having this stay on the same line like dbBuilder. + /* + if(fLogEntryLastCommit.fLogTime > 0) { + cout << "Last transaction committed " << fLogEntryLastCommit.fVersionID << " from " << + ctime(&fLogEntryLastCommit.fLogTime); + } + cout << endl; + cout << "Now processing file " << fileName << endl; + */ } /*************************************************************************** -* -* Function: replayTransactions -* -* Purpose: Reads the qualifying files and processes the transactions -* starting with the first one with an id greater than the last -* id handed out before the backup and ending with the last -* transaction committed before the stop date. -****************************************************************************/ + * + * Function: replayTransactions + * + * Purpose: Reads the qualifying files and processes the transactions + * starting with the first one with an id greater than the last + * id handed out before the backup and ending with the last + * transaction committed before the stop date. + ****************************************************************************/ void ReplayTxnLog::replayTransactions(list& fileNames) { - list::iterator curFile; - vector v; - curFile = fileNames.begin(); - LogEntry logEntry; - string statement; - bool done = false; - bool confirmed = true; - oam::Oam oam; + list::iterator curFile; + vector v; + curFile = fileNames.begin(); + LogEntry logEntry; + string statement; + bool done = false; + bool confirmed = true; + oam::Oam oam; - curFile = fileNames.begin(); + curFile = fileNames.begin(); -// cout << "Searching files for first transaction to process." << endl << endl; + // cout << "Searching files for first transaction to process." << endl << endl; - while (!done && curFile != fileNames.end()) + while (!done && curFile != fileNames.end()) + { + ifstream dataFile(curFile->c_str()); + + if (confirmed) { - ifstream dataFile(curFile->c_str()); - - if (confirmed) - { - displayProgress(*curFile); - } - - while (!done && getNextCalpontLogEntry(dataFile, logEntry)) - { - if (logEntry.fLogTime >= fStopDate) - { - done = true; - } - else if (logEntry.fVersionID > fStartVersionID) - { - if (confirmed) - { - processStatement(logEntry); - } - - /* - else if(displaySummaryAndConfirm(logEntry)) { - confirmed = true; - if(!fReportMode) { - - // Log start message. - ostringstream oss; - oss << "First transaction ID to be processed is " << logEntry.fVersionID << "."; - string msg = oss.str(); - logging::logEventToDataLog(logging::M0019, msg); - - // Turn off DML/DDL logging to keep from the statements from - // being logged again. - #ifdef OAM_AVAILABLE - cout << endl << "Turning off DML/DDL logging." << endl << endl; - oam.updateLog(oam::MANDISABLEDSTATE, "system", "data"); - #endif - } - displayProgress(*curFile); - processStatement(logEntry); - } - else { - done = true; - } - */ - } - } - - dataFile.close(); - curFile++; + displayProgress(*curFile); } - // Turn DML/DDL logging back on. - /* - if(!fReportMode && confirmed) { - #ifdef OAM_AVAILABLE - cout << endl << "Turning DML/DDL logging back on." << endl << endl; - oam.updateLog(oam::ENABLEDSTATE, "system", "data"); - #endif + while (!done && getNextCalpontLogEntry(dataFile, logEntry)) + { + if (logEntry.fLogTime >= fStopDate) + { + done = true; + } + else if (logEntry.fVersionID > fStartVersionID) + { + if (confirmed) + { + processStatement(logEntry); + } - // Log complete message. - ostringstream oss; - oss << "Last transaction ID processed was " << fLogEntryLastCommit.fVersionID << "."; - string msg = oss.str(); - logging::logEventToDataLog(logging::M0020, msg); - } - */ + /* + else if(displaySummaryAndConfirm(logEntry)) { + confirmed = true; + if(!fReportMode) { + + // Log start message. + ostringstream oss; + oss << "First transaction ID to be processed is " << + logEntry.fVersionID << "."; string msg = oss.str(); logging::logEventToDataLog(logging::M0019, + msg); + + // Turn off DML/DDL logging to keep from the + statements from + // being logged again. + #ifdef OAM_AVAILABLE + cout << endl << "Turning off DML/DDL logging." << endl + << endl; oam.updateLog(oam::MANDISABLEDSTATE, "system", "data"); #endif + } + displayProgress(*curFile); + processStatement(logEntry); + } + else { + done = true; + } + */ + } + } + + dataFile.close(); + curFile++; + } + + // Turn DML/DDL logging back on. + /* + if(!fReportMode && confirmed) { + #ifdef OAM_AVAILABLE + cout << endl << "Turning DML/DDL logging back on." << endl << endl; + oam.updateLog(oam::ENABLEDSTATE, "system", "data"); + #endif + + // Log complete message. + ostringstream oss; + oss << "Last transaction ID processed was " << fLogEntryLastCommit.fVersionID << "."; + string msg = oss.str(); + logging::logEventToDataLog(logging::M0020, msg); + } + */ } /*************************************************************************** -* -* Function: checkDefinitions -* -* Purpose: Reminder to uncomment required #defines. -****************************************************************************/ + * + * Function: checkDefinitions + * + * Purpose: Reminder to uncomment required #defines. + ****************************************************************************/ void ReplayTxnLog::checkDefinitions() { #ifndef OAM_AVAILABLE - cout << "Warning!!! Uncomment #define OAM_AVAILABLE before checking in." << endl; + cout << "Warning!!! Uncomment #define OAM_AVAILABLE before checking in." << endl; #endif #ifndef LAST_TXN_AVAILABLE - cout << "Warning!!! Uncomment #define LAST_TXN_AVAILABLE before checking in." << endl; + cout << "Warning!!! Uncomment #define LAST_TXN_AVAILABLE before checking in." << endl; #endif #ifndef CALDB_AVAILABLE - cout << "Warning!!! Uncomment #define CALDB_AVAILABLE before checking in." << endl; + cout << "Warning!!! Uncomment #define CALDB_AVAILABLE before checking in." << endl; #endif } /*************************************************************************** -* -* Function: displayFinalSummary -* -* Purpose: Shows totals at end of program. -****************************************************************************/ + * + * Function: displayFinalSummary + * + * Purpose: Shows totals at end of program. + ****************************************************************************/ void ReplayTxnLog::displayFinalSummary() { - cout << endl; + cout << endl; - if (fCommitCount == 0 && fRollbackCount == 0 && fDDLCount == 0) - { - cout << "No transactions were processed." << endl; - } - else - { - /* - cout << "---------------------------------------------------------------------------------------" << endl; - cout << "-- Summary of transaction reported." << endl; - cout << "--" << endl; - cout << "-- Commits : " << fCommitCount << endl; - // cout << "-- Rollbacks : " << fRollbackCount << endl; - cout << "-- DDL Statements : " << fDDLCount << endl; - cout << "--" << endl; - cout << "-- First Commit : " << "Transaction " << fLogEntryFirstCommit.fVersionID << " originally " << - "committed " << ctime(&fLogEntryFirstCommit.fLogTime); - cout << "-- Last Commit : " << "Transaction " << fLogEntryLastCommit.fVersionID << " originally " << - "committed " << ctime(&fLogEntryLastCommit.fLogTime) ; - int openCount = fActiveTxns.size(); - if(openCount > 0) { - cout << "-- " << endl; - cout << "-- There were " << openCount << " transactions that were open at the stop time." << endl; - } - cout << "---------------------------------------------------------------------------------------" << endl; - cout << endl; - */ - } + if (fCommitCount == 0 && fRollbackCount == 0 && fDDLCount == 0) + { + cout << "No transactions were processed." << endl; + } + else + { + /* + cout << + "---------------------------------------------------------------------------------------" << endl; cout << + "-- Summary of transaction reported." << endl; cout << "--" << endl; cout << "-- Commits : " << + fCommitCount << endl; + // cout << "-- Rollbacks : " << fRollbackCount << endl; + cout << "-- DDL Statements : " << fDDLCount << endl; + cout << "--" << endl; + cout << "-- First Commit : " << "Transaction " << fLogEntryFirstCommit.fVersionID << " + originally " << "committed " << ctime(&fLogEntryFirstCommit.fLogTime); cout << "-- Last Commit : " << + "Transaction " << fLogEntryLastCommit.fVersionID << " originally " << "committed " << + ctime(&fLogEntryLastCommit.fLogTime) ; int openCount = fActiveTxns.size(); if(openCount > 0) { cout << "-- + " << endl; cout << "-- There were " << openCount << " transactions that were open at the stop time." << + endl; + } + cout << + "---------------------------------------------------------------------------------------" << endl; cout << + endl; + */ + } } /*************************************************************************** -* -* Function: process -* -* Purpose: The main function. Receives and validates the user's input and -* and processes the transactions contained in the log files. -****************************************************************************/ + * + * Function: process + * + * Purpose: The main function. Receives and validates the user's input and + * and processes the transactions contained in the log files. + ****************************************************************************/ void ReplayTxnLog::process() { - list dbFileNames; + list dbFileNames; - // Output a warning to the developer if #defines are commented out. - checkDefinitions(); + // Output a warning to the developer if #defines are commented out. + checkDefinitions(); - // Validate the user id, password, and stop date. - validateInput(); // Starts the session + // Validate the user id, password, and stop date. + validateInput(); // Starts the session - // Get the last transaction id. - execplan::SessionManager mgr; + // Get the last transaction id. + execplan::SessionManager mgr; - // Get the last version from issued before the backup. + // Get the last version from issued before the backup. #ifdef LAST_TXN_AVAILABLE - fStartVersionID = mgr.verID().currentScn; + fStartVersionID = mgr.verID().currentScn; #else - fStartVersionID = 0; + fStartVersionID = 0; #endif - cout << "-- The current database version ID is " << fStartVersionID << "." << endl << endl; + cout << "-- The current database version ID is " << fStartVersionID << "." << endl << endl; - // Get the log file names. - dbFileNames = getLogFileNames(); + // Get the log file names. + dbFileNames = getLogFileNames(); - // Replay the transactions. - replayTransactions(dbFileNames); + // Replay the transactions. + replayTransactions(dbFileNames); - // Display summary of transactions processed. - displayFinalSummary(); + // Display summary of transactions processed. + displayFinalSummary(); - // Warn the developer again if #defines are commented out. - checkDefinitions(); + // Warn the developer again if #defines are commented out. + checkDefinitions(); -// cout << "Program completed successfully." << endl; + // cout << "Program completed successfully." << endl; } namespace replaytxnlog { - DBSession::DBSession() { - fConnectedUser = ""; + fConnectedUser = ""; } // Begins an Oracle session. void DBSession::startSession(const std::string& connectAsUser) { - cout << "use " << connectAsUser << ";" << endl; - fConnectedUser = connectAsUser; + cout << "use " << connectAsUser << ";" << endl; + fConnectedUser = connectAsUser; } void DBSession::issueStatement(const std::string& stmt) { - - try - { - - cout << stmt << endl; - - } - catch (std::runtime_error e) - { - throw std::runtime_error("Error executing Statement: " + stmt + "\n" + e.what()); - } + try + { + cout << stmt << endl; + } + catch (std::runtime_error e) + { + throw std::runtime_error("Error executing Statement: " + stmt + "\n" + e.what()); + } } - void DBSession::endSession() { - cout << endl; + cout << endl; } void DBSession::commit() { - cout << "commit;" << endl << endl; + cout << "commit;" << endl << endl; } void DBSession::rollback() { - cout << "rollback;" << endl << endl; + cout << "rollback;" << endl << endl; } std::string DBSession::getConnectedUser() { - return fConnectedUser; + return fConnectedUser; } -} //namespace replaytxnlog - +} // namespace replaytxnlog diff --git a/oam/replaytxnlog/replaytxnlog.h b/oam/replaytxnlog/replaytxnlog.h index d60578606..3c3645a80 100644 --- a/oam/replaytxnlog/replaytxnlog.h +++ b/oam/replaytxnlog/replaytxnlog.h @@ -35,153 +35,157 @@ * * * This class replays transactions after a full backup has been restored. The class re-issues statements - * from the DML/DDL log files allowing the database to be a state from a point in time after the backup occurred. + * from the DML/DDL log files allowing the database to be a state from a point in time after the backup + * occurred. * - * The user provides an Oracle user id and password as well as a stop date and time. Transactions that were committed - * after the time of the backup through before the stop date and time will be replayed in the order that they were - * committed. + * The user provides an Oracle user id and password as well as a stop date and time. Transactions that were + * committed after the time of the backup through before the stop date and time will be replayed in the order + * that they were committed. * * This program only works under the following conditions: * 1) A backup was restored including the Calpont database files, the file containing the last txnId, and the * corresponding Oracle files. - * 2) The DML/DDL log files include the log entry for the backup and all of the transactions from the backup time - * through the stop date and time provided by the user. - * 3) All of the transaction ids being replayed have higher numbers than the transaction id from the backup. - * The procedure for resetting the txnId had not been defined when this utility was written. We need to take - * this utility into account when defining the procedure. + * 2) The DML/DDL log files include the log entry for the backup and all of the transactions from the backup + * time through the stop date and time provided by the user. 3) All of the transaction ids being replayed have + * higher numbers than the transaction id from the backup. The procedure for resetting the txnId had not been + * defined when this utility was written. We need to take this utility into account when defining the + * procedure. * */ class ReplayTxnLog { -public: + public: + /** @brief Constructor + * user - The database user id. User will be prompted if passed blank. Program will + * abort if invalid user / password combination. + * + * password - The database user's password. User will be prompted if passed blank. + * + * stopDate - The stopdate formatted as "mm/dd/yy@hh:mm:ss" or "Now". User will be + * prompted if passed blank or the format is incorrect. + * + * ignoreBulkLoad - True if bulk load log entries are to be ignored. If false, the user + * will be prompted with each bulk load entry. Bulk loads are not included + * in the log, so the user would have to know that the bulk load will not + * cause any sql errors because of dependent data if bulk load entries + * are skipped. Otherwise, the user would have to re-apply bulk loads + * each time this utility pauses. + */ + ReplayTxnLog(const std::string& user, const std::string& password, const std::string& stopDate, + const bool& ignoreBulkLoad, const bool& reportMode); - /** @brief Constructor - * user - The database user id. User will be prompted if passed blank. Program will - * abort if invalid user / password combination. - * - * password - The database user's password. User will be prompted if passed blank. - * - * stopDate - The stopdate formatted as "mm/dd/yy@hh:mm:ss" or "Now". User will be - * prompted if passed blank or the format is incorrect. - * - * ignoreBulkLoad - True if bulk load log entries are to be ignored. If false, the user - * will be prompted with each bulk load entry. Bulk loads are not included - * in the log, so the user would have to know that the bulk load will not - * cause any sql errors because of dependent data if bulk load entries - * are skipped. Otherwise, the user would have to re-apply bulk loads - * each time this utility pauses. - */ - ReplayTxnLog(const std::string& user, const std::string& password, const std::string& stopDate, - const bool& ignoreBulkLoad, const bool& reportMode); + /** @brief processes the transaction replay. + */ + void process(); - /** @brief processes the transaction replay. - */ - void process(); -private: - /* Passed in constructor. */ - std::string fUser; - std::string fPassword; - std::string fStopDateStr; - bool fIgnoreBulkLoad; + private: + /* Passed in constructor. */ + std::string fUser; + std::string fPassword; + std::string fStopDateStr; + bool fIgnoreBulkLoad; - /* Other private members. */ - execplan::CalpontSystemCatalog::SCN fStartVersionID; - execplan::CalpontSystemCatalog::SCN fDDLCount; - execplan::CalpontSystemCatalog::SCN fCommitCount; - execplan::CalpontSystemCatalog::SCN fRollbackCount; - time_t fProgramStarted; - time_t fStopDate; - std::string fCurrentYearStr; - std::string fLastSchemaOwner; - enum LogEntryType {DML, DDL, ROLLBACK, XCOMMIT, BULKLOAD, - BACKUP_START, BACKUP_END, REPLAY_TXN_START, REPLAY_TXN_END - }; // WWW - COMMIT reserved? + /* Other private members. */ + execplan::CalpontSystemCatalog::SCN fStartVersionID; + execplan::CalpontSystemCatalog::SCN fDDLCount; + execplan::CalpontSystemCatalog::SCN fCommitCount; + execplan::CalpontSystemCatalog::SCN fRollbackCount; + time_t fProgramStarted; + time_t fStopDate; + std::string fCurrentYearStr; + std::string fLastSchemaOwner; + enum LogEntryType + { + DML, + DDL, + ROLLBACK, + XCOMMIT, + BULKLOAD, + BACKUP_START, + BACKUP_END, + REPLAY_TXN_START, + REPLAY_TXN_END + }; // WWW - COMMIT reserved? - struct LogEntry + struct LogEntry + { + execplan::CalpontSystemCatalog::SCN fVersionID; + std::string fSqlStatement; + time_t fLogTime; + std::string fSchemaOwner; + LogEntryType type; + LogEntry() : fVersionID(0), fSqlStatement(""), fLogTime(0), fSchemaOwner("") { - execplan::CalpontSystemCatalog::SCN fVersionID; - std::string fSqlStatement; - time_t fLogTime; - std::string fSchemaOwner; - LogEntryType type; - LogEntry(): fVersionID(0), fSqlStatement(""), fLogTime(0), fSchemaOwner("") {} - }; - LogEntry fLogEntryFirstCommit; - LogEntry fLogEntryLastCommit; - struct ActiveTxn - { - execplan::CalpontSystemCatalog::SCN fVersionID; - std::vector fStatements; - }; - std::vector fActiveTxns; + } + }; + LogEntry fLogEntryFirstCommit; + LogEntry fLogEntryLastCommit; + struct ActiveTxn + { + execplan::CalpontSystemCatalog::SCN fVersionID; + std::vector fStatements; + }; + std::vector fActiveTxns; - /* Private functions */ - void validateInput(); - std::list getLogFileNames(); - void replayTransactions(std::list& files); - void processStatement(const LogEntry& logEntry); - int split(vector& v, const std::string& str, const char& c); - bool getNextCalpontLogEntry(ifstream& dataFile, LogEntry& logEntry); - bool displaySummaryAndConfirm(const LogEntry& logEntry); - void displayProgress(const std::string& fileName); - void displayFinalSummary(); - time_t convertLogDateStr(const std::string& mth, const std::string& day, const std::string& time); - void checkDefinitions(); - - ReplayTxnLog(const ReplayTxnLog& rhs); // no copies - ReplayTxnLog& operator=(const ReplayTxnLog& rhs); // no assignments + /* Private functions */ + void validateInput(); + std::list getLogFileNames(); + void replayTransactions(std::list& files); + void processStatement(const LogEntry& logEntry); + int split(vector& v, const std::string& str, const char& c); + bool getNextCalpontLogEntry(ifstream& dataFile, LogEntry& logEntry); + bool displaySummaryAndConfirm(const LogEntry& logEntry); + void displayProgress(const std::string& fileName); + void displayFinalSummary(); + time_t convertLogDateStr(const std::string& mth, const std::string& day, const std::string& time); + void checkDefinitions(); + ReplayTxnLog(const ReplayTxnLog& rhs); // no copies + ReplayTxnLog& operator=(const ReplayTxnLog& rhs); // no assignments }; namespace replaytxnlog { - /** @brief wraps an Oracle session - * This class provides an Oracle session interface allowing a session to be started, DML statements - * to be executed, and closing the session. - */ + * This class provides an Oracle session interface allowing a session to be started, DML statements + * to be executed, and closing the session. + */ class DBSession { + public: + /** @brief starts the session. + */ + void startSession(const std::string& connectAsUser); -public: + /** @brief issues a DML or DDL statement. + */ + void issueStatement(const std::string& stmt); - /** @brief starts the session. - */ - void startSession(const std::string& connectAsUser); + /** @brief issues a commit. + */ + void commit(); - /** @brief issues a DML or DDL statement. - */ - void issueStatement(const std::string& stmt); + /** @brief issues a rollback. + */ + void rollback(); - /** @brief issues a commit. - */ - void commit(); + /** @brief ends the session. + */ + void endSession(); - /** @brief issues a rollback. - */ - void rollback(); + /** @brief returns the connected user or empty string if a session is not active. + */ + std::string getConnectedUser(); - /** @brief ends the session. - */ - void endSession(); + /** @brief constructor + */ + DBSession(); - - /** @brief returns the connected user or empty string if a session is not active. - */ - std::string getConnectedUser(); - - /** @brief constructor - */ - DBSession(); - -private: - - DBSession(const DBSession& rhs); // private ctor - DBSession& operator=(const DBSession& rhs); - std::string fConnectedUser; + private: + DBSession(const DBSession& rhs); // private ctor + DBSession& operator=(const DBSession& rhs); + std::string fConnectedUser; }; - -} //namespace replaytxnlog - +} // namespace replaytxnlog diff --git a/oamapps/columnstoreDB/columnstoreDB.cpp b/oamapps/columnstoreDB/columnstoreDB.cpp index d50d77429..f716f9bd8 100644 --- a/oamapps/columnstoreDB/columnstoreDB.cpp +++ b/oamapps/columnstoreDB/columnstoreDB.cpp @@ -51,24 +51,22 @@ using namespace execplan; namespace { - void usage(char* prog) { + cout << endl; + cout << "Usage: " << prog << " [options]" << endl; - cout << endl; - cout << "Usage: " << prog << " [options]" << endl; + cout << endl; + cout << "This utility is used to suspend and resume Columnstore Database Writes." << endl; + cout << "Normally this would be done while performing Database Backups and" << endl; + cout << "Restores " << endl; + cout << endl; - cout << endl; - cout << "This utility is used to suspend and resume Columnstore Database Writes." << endl; - cout << "Normally this would be done while performing Database Backups and" << endl; - cout << "Restores " << endl; - cout << endl; - - cout << "Options:" << endl; - cout << "-c Command: suspend or resume" << endl << endl; - cout << "-h Display this help." << endl << endl; -} + cout << "Options:" << endl; + cout << "-c Command: suspend or resume" << endl << endl; + cout << "-h Display this help." << endl << endl; } +} // namespace /****************************************************************************************** * @brief DisplayLockedTables @@ -80,294 +78,281 @@ void usage(char* prog) ******************************************************************************************/ void DisplayLockedTables(std::vector& tableLocks, BRM::DBRM* pDBRM) { - cout << "The following tables are locked:" << endl; + cout << "The following tables are locked:" << endl; - // Initial widths of columns to display. We pass thru the table - // and see if we need to grow any of these. - unsigned int lockIDColumnWidth = 6; // "LockID" - unsigned int tableNameColumnWidth = 12; // "Name" - unsigned int ownerColumnWidth = 7; // "Process" - unsigned int pidColumnWidth = 3; // "PID" - unsigned int sessionIDColumnWidth = 7; // "Session" - unsigned int createTimeColumnWidth = 12; // "CreationTime" - unsigned int dbrootColumnWidth = 7; // "DBRoots" - unsigned int stateColumnWidth = 9; // "State" + // Initial widths of columns to display. We pass thru the table + // and see if we need to grow any of these. + unsigned int lockIDColumnWidth = 6; // "LockID" + unsigned int tableNameColumnWidth = 12; // "Name" + unsigned int ownerColumnWidth = 7; // "Process" + unsigned int pidColumnWidth = 3; // "PID" + unsigned int sessionIDColumnWidth = 7; // "Session" + unsigned int createTimeColumnWidth = 12; // "CreationTime" + unsigned int dbrootColumnWidth = 7; // "DBRoots" + unsigned int stateColumnWidth = 9; // "State" - // Initialize System Catalog object used to get table name - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); + // Initialize System Catalog object used to get table name + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); - std::string fullTblName; - const char* tableState; + std::string fullTblName; + const char* tableState; - // Make preliminary pass through the table locks in order to determine our - // output column widths based on the data. Min column widths are based on - // the width of the column heading (except for the 'state' column). - uint64_t maxLockID = 0; - uint32_t maxPID = 0; - int32_t maxSessionID = 0; - int32_t minSessionID = 0; - std::vector createTimes; - std::vector pms; - char cTimeBuffer[64]; + // Make preliminary pass through the table locks in order to determine our + // output column widths based on the data. Min column widths are based on + // the width of the column heading (except for the 'state' column). + uint64_t maxLockID = 0; + uint32_t maxPID = 0; + int32_t maxSessionID = 0; + int32_t minSessionID = 0; + std::vector createTimes; + std::vector pms; + char cTimeBuffer[64]; - execplan::CalpontSystemCatalog::TableName tblName; + execplan::CalpontSystemCatalog::TableName tblName; - for (unsigned idx = 0; idx < tableLocks.size(); idx++) + for (unsigned idx = 0; idx < tableLocks.size(); idx++) + { + if (tableLocks[idx].id > maxLockID) { - if (tableLocks[idx].id > maxLockID) - { - maxLockID = tableLocks[idx].id; - } - - try - { - tblName = systemCatalogPtr->tableName(tableLocks[idx].tableOID); - } - catch (...) - { - tblName.schema.clear(); - tblName.table.clear(); - } - - fullTblName = tblName.toString(); - - if (fullTblName.size() > tableNameColumnWidth) - { - tableNameColumnWidth = fullTblName.size(); - } - - if (tableLocks[idx].ownerName.length() > ownerColumnWidth) - { - ownerColumnWidth = tableLocks[idx].ownerName.length(); - } - - if (tableLocks[idx].ownerPID > maxPID) - { - maxPID = tableLocks[idx].ownerPID; - } - - if (tableLocks[idx].ownerSessionID > maxSessionID) - { - maxSessionID = tableLocks[idx].ownerSessionID; - } - - if (tableLocks[idx].ownerSessionID < minSessionID) - { - minSessionID = tableLocks[idx].ownerSessionID; - } - - // Creation Time. - // While we're at it, we save the time string off into a vector - // so we can display it later without recalcing it. - struct tm timeTM; - localtime_r(&tableLocks[idx].creationTime, &timeTM); - ctime_r(&tableLocks[idx].creationTime, cTimeBuffer); - strftime(cTimeBuffer, 64, "%F %r:", &timeTM); - cTimeBuffer[strlen(cTimeBuffer) - 1] = '\0'; // strip trailing '\n' - std::string cTimeStr( cTimeBuffer ); - - if (cTimeStr.length() > createTimeColumnWidth) - { - createTimeColumnWidth = cTimeStr.length(); - } - - createTimes.push_back(cTimeStr); + maxLockID = tableLocks[idx].id; } - tableNameColumnWidth += 1; - ownerColumnWidth += 1; - createTimeColumnWidth += 1; - - std::ostringstream idString; - idString << maxLockID; - - if (idString.str().length() > lockIDColumnWidth) - lockIDColumnWidth = idString.str().length(); - - lockIDColumnWidth += 1; - - std::ostringstream pidString; - pidString << maxPID; - - if (pidString.str().length() > pidColumnWidth) - pidColumnWidth = pidString.str().length(); - - pidColumnWidth += 1; - - const std::string sessionNoneStr("BulkLoad"); - std::ostringstream sessionString; - sessionString << maxSessionID; - - if (sessionString.str().length() > sessionIDColumnWidth) - sessionIDColumnWidth = sessionString.str().length(); - - if ((minSessionID < 0) && - (sessionNoneStr.length() > sessionIDColumnWidth)) - sessionIDColumnWidth = sessionNoneStr.length(); - - sessionIDColumnWidth += 1; - - // write the column headers before the first entry - cout.setf(ios::left, ios::adjustfield); - cout << setw(lockIDColumnWidth) << "LockID" << - setw(tableNameColumnWidth) << "Name" << - setw(ownerColumnWidth) << "Process" << - setw(pidColumnWidth) << "PID" << - setw(sessionIDColumnWidth) << "Session" << - setw(createTimeColumnWidth) << "CreationTime" << - setw(stateColumnWidth) << "State" << - setw(dbrootColumnWidth) << "DBRoots" << endl; - - for (unsigned idx = 0; idx < tableLocks.size(); idx++) + try { - try - { + tblName = systemCatalogPtr->tableName(tableLocks[idx].tableOID); + } + catch (...) + { + tblName.schema.clear(); + tblName.table.clear(); + } - tblName = systemCatalogPtr->tableName(tableLocks[idx].tableOID); - } - catch (...) - { - tblName.schema.clear(); - tblName.table.clear(); - } + fullTblName = tblName.toString(); - fullTblName = tblName.toString(); - cout << - setw(lockIDColumnWidth) << tableLocks[idx].id << - setw(tableNameColumnWidth) << fullTblName << - setw(ownerColumnWidth) << tableLocks[idx].ownerName << - setw(pidColumnWidth) << tableLocks[idx].ownerPID; + if (fullTblName.size() > tableNameColumnWidth) + { + tableNameColumnWidth = fullTblName.size(); + } - // Log session ID, or "BulkLoad" if session is -1 - if (tableLocks[idx].ownerSessionID < 0) - cout << setw(sessionIDColumnWidth) << sessionNoneStr; - else - cout << setw(sessionIDColumnWidth) << - tableLocks[idx].ownerSessionID; + if (tableLocks[idx].ownerName.length() > ownerColumnWidth) + { + ownerColumnWidth = tableLocks[idx].ownerName.length(); + } - // Creation Time - cout << setw(createTimeColumnWidth) << createTimes[idx]; + if (tableLocks[idx].ownerPID > maxPID) + { + maxPID = tableLocks[idx].ownerPID; + } - // Processor State - if (pDBRM && !pDBRM->checkOwner(tableLocks[idx].id)) - { - tableState = "Abandoned"; - } - else - { - tableState = ((tableLocks[idx].state == BRM::LOADING) ? - "LOADING" : "CLEANUP"); - } + if (tableLocks[idx].ownerSessionID > maxSessionID) + { + maxSessionID = tableLocks[idx].ownerSessionID; + } - cout << setw(stateColumnWidth) << tableState; + if (tableLocks[idx].ownerSessionID < minSessionID) + { + minSessionID = tableLocks[idx].ownerSessionID; + } - // PM List - cout << setw(dbrootColumnWidth); + // Creation Time. + // While we're at it, we save the time string off into a vector + // so we can display it later without recalcing it. + struct tm timeTM; + localtime_r(&tableLocks[idx].creationTime, &timeTM); + ctime_r(&tableLocks[idx].creationTime, cTimeBuffer); + strftime(cTimeBuffer, 64, "%F %r:", &timeTM); + cTimeBuffer[strlen(cTimeBuffer) - 1] = '\0'; // strip trailing '\n' + std::string cTimeStr(cTimeBuffer); - for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) - { - if (k > 0) - cout << ','; + if (cTimeStr.length() > createTimeColumnWidth) + { + createTimeColumnWidth = cTimeStr.length(); + } - cout << tableLocks[idx].dbrootList[k]; - } + createTimes.push_back(cTimeStr); + } - cout << endl; - } // end of loop through table locks + tableNameColumnWidth += 1; + ownerColumnWidth += 1; + createTimeColumnWidth += 1; + + std::ostringstream idString; + idString << maxLockID; + + if (idString.str().length() > lockIDColumnWidth) + lockIDColumnWidth = idString.str().length(); + + lockIDColumnWidth += 1; + + std::ostringstream pidString; + pidString << maxPID; + + if (pidString.str().length() > pidColumnWidth) + pidColumnWidth = pidString.str().length(); + + pidColumnWidth += 1; + + const std::string sessionNoneStr("BulkLoad"); + std::ostringstream sessionString; + sessionString << maxSessionID; + + if (sessionString.str().length() > sessionIDColumnWidth) + sessionIDColumnWidth = sessionString.str().length(); + + if ((minSessionID < 0) && (sessionNoneStr.length() > sessionIDColumnWidth)) + sessionIDColumnWidth = sessionNoneStr.length(); + + sessionIDColumnWidth += 1; + + // write the column headers before the first entry + cout.setf(ios::left, ios::adjustfield); + cout << setw(lockIDColumnWidth) << "LockID" << setw(tableNameColumnWidth) << "Name" + << setw(ownerColumnWidth) << "Process" << setw(pidColumnWidth) << "PID" << setw(sessionIDColumnWidth) + << "Session" << setw(createTimeColumnWidth) << "CreationTime" << setw(stateColumnWidth) << "State" + << setw(dbrootColumnWidth) << "DBRoots" << endl; + + for (unsigned idx = 0; idx < tableLocks.size(); idx++) + { + try + { + tblName = systemCatalogPtr->tableName(tableLocks[idx].tableOID); + } + catch (...) + { + tblName.schema.clear(); + tblName.table.clear(); + } + + fullTblName = tblName.toString(); + cout << setw(lockIDColumnWidth) << tableLocks[idx].id << setw(tableNameColumnWidth) << fullTblName + << setw(ownerColumnWidth) << tableLocks[idx].ownerName << setw(pidColumnWidth) + << tableLocks[idx].ownerPID; + + // Log session ID, or "BulkLoad" if session is -1 + if (tableLocks[idx].ownerSessionID < 0) + cout << setw(sessionIDColumnWidth) << sessionNoneStr; + else + cout << setw(sessionIDColumnWidth) << tableLocks[idx].ownerSessionID; + + // Creation Time + cout << setw(createTimeColumnWidth) << createTimes[idx]; + + // Processor State + if (pDBRM && !pDBRM->checkOwner(tableLocks[idx].id)) + { + tableState = "Abandoned"; + } + else + { + tableState = ((tableLocks[idx].state == BRM::LOADING) ? "LOADING" : "CLEANUP"); + } + + cout << setw(stateColumnWidth) << tableState; + + // PM List + cout << setw(dbrootColumnWidth); + + for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) + { + if (k > 0) + cout << ','; + + cout << tableLocks[idx].dbrootList[k]; + } + + cout << endl; + } // end of loop through table locks } int main(int argc, char** argv) { - string command; - Oam oam; - BRM::DBRM dbrm; + string command; + Oam oam; + BRM::DBRM dbrm; - char c; + char c; - // Invokes member function `int operator ()(void);' - while ((c = getopt(argc, argv, "c:h")) != -1) + // Invokes member function `int operator ()(void);' + while ((c = getopt(argc, argv, "c:h")) != -1) + { + switch (c) { - switch (c) - { - case 'c': - command = optarg; - break; + case 'c': command = optarg; break; - case 'h': - usage(argv[0]); - exit(-1); - break; + case 'h': + usage(argv[0]); + exit(-1); + break; - default: - usage(argv[0]); - exit(1); - break; - } + default: + usage(argv[0]); + exit(1); + break; } + } - if ( command == "suspend" ) + if (command == "suspend") + { + try { - try - { - std::vector tableLocks = dbrm.getAllTableLocks(); + std::vector tableLocks = dbrm.getAllTableLocks(); - if (!tableLocks.empty()) - { - DisplayLockedTables(tableLocks, &dbrm); - } - else - { - dbrm.setSystemSuspended(true); - sleep(5); - string cmd = "save_brm > " + string(MCSLOGDIR) + "/save_brm.log1 2>&1"; - int rtnCode = system(cmd.c_str()); + if (!tableLocks.empty()) + { + DisplayLockedTables(tableLocks, &dbrm); + } + else + { + dbrm.setSystemSuspended(true); + sleep(5); + string cmd = "save_brm > " + string(MCSLOGDIR) + "/save_brm.log1 2>&1"; + int rtnCode = system(cmd.c_str()); - if (rtnCode == 0) - { - cout << endl << "Suspend Columnstore Database Writes Request successfully completed" << endl; - } - else - { - cout << endl << "Suspend Columnstore Database Writes Failed: save_brm Failed" << endl; - dbrm.setSystemSuspended(false); - } - } - } - catch (exception& e) + if (rtnCode == 0) { - cout << endl << "**** Suspend Columnstore Database Writes Failed: " << e.what() << endl; - } - catch (...) - { - cout << endl << "**** Suspend Columnstore Database Writes Failed" << endl; - } - } - else - { - if ( command == "resume" ) - { - try - { - dbrm.setSystemSuspended(false); - cout << endl << "Resume Columnstore Database Writes Request successfully completed" << endl; - } - catch (exception& e) - { - cout << endl << "**** Resume Columnstore Database Writes Failed: " << e.what() << endl; - } - catch (...) - { - cout << endl << "**** Resume Columnstore Database Writes Failed" << endl; - } + cout << endl << "Suspend Columnstore Database Writes Request successfully completed" << endl; } else { - cout << "Invalid Command Entered, please try again" << endl; - exit(-1); + cout << endl << "Suspend Columnstore Database Writes Failed: save_brm Failed" << endl; + dbrm.setSystemSuspended(false); } + } } + catch (exception& e) + { + cout << endl << "**** Suspend Columnstore Database Writes Failed: " << e.what() << endl; + } + catch (...) + { + cout << endl << "**** Suspend Columnstore Database Writes Failed" << endl; + } + } + else + { + if (command == "resume") + { + try + { + dbrm.setSystemSuspended(false); + cout << endl << "Resume Columnstore Database Writes Request successfully completed" << endl; + } + catch (exception& e) + { + cout << endl << "**** Resume Columnstore Database Writes Failed: " << e.what() << endl; + } + catch (...) + { + cout << endl << "**** Resume Columnstore Database Writes Failed" << endl; + } + } + else + { + cout << "Invalid Command Entered, please try again" << endl; + exit(-1); + } + } - exit(0); + exit(0); } - diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 4ef646b21..0f8abad74 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -2,11 +2,11 @@ /* Copyright (C) 2016 MariaDB Corporation */ /****************************************************************************************** -* $Id: columnstoreSupport.cpp 64 2006-10-12 22:21:51Z dhill $ -* -* -* -******************************************************************************************/ + * $Id: columnstoreSupport.cpp 64 2006-10-12 22:21:51Z dhill $ + * + * + * + ******************************************************************************************/ /** * @file */ @@ -43,9 +43,9 @@ using namespace config; typedef struct Child_Module_struct { - std::string moduleName; - std::string moduleIP; - std::string hostName; + std::string moduleName; + std::string moduleIP; + std::string hostName; } ChildModule; typedef std::vector ChildModuleList; @@ -67,824 +67,853 @@ string ProfileFile; int runningThreads = 0; pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; -typedef boost::tuple threadInfo_t; +typedef boost::tuple threadInfo_t; bool LOCAL = false; void* childReportThread(threadInfo_t* st) { - assert(st); - ChildModuleList::iterator& list = boost::get<0>(*st); - string reportType = boost::get<1>(*st); + assert(st); + ChildModuleList::iterator& list = boost::get<0>(*st); + string reportType = boost::get<1>(*st); - string remoteModuleName = (*list).moduleName; - string remoteModuleIP = (*list).moduleIP; - string remoteHostName = (*list).hostName; + string remoteModuleName = (*list).moduleName; + string remoteModuleIP = (*list).moduleIP; + string remoteHostName = (*list).hostName; - pthread_mutex_lock( &mutex1 ); - runningThreads++; -//cout << "++ " << runningThreads << endl; - pthread_mutex_unlock( &mutex1 ); + pthread_mutex_lock(&mutex1); + runningThreads++; + // cout << "++ " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); - string outputFile; + string outputFile; - if (reportType == "log") + if (reportType == "log") + { + outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz"; + } + else + { + outputFile = remoteModuleName + "_" + reportType + "Report.txt"; + + FILE* pOutputFile = fopen(outputFile.c_str(), "a"); + if (pOutputFile == NULL) { - outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz"; - } - else - { - outputFile = remoteModuleName + "_" + reportType + "Report.txt"; - - FILE *pOutputFile = fopen(outputFile.c_str(), "a"); - if (pOutputFile == NULL) - { - printf("Could not open file: %s",outputFile.c_str()); - exit(1); - } - - fprintf(pOutputFile,"********************************************************************************\n" - "\n" - " System %s\n" - " columnstoreSupportReport script ran from Module %s on %s\n" - " SoftwareVersion = %s-%s" - "\n" - "********************************************************************************\n" - "\n" - " %s report\n" - "\n" - "********************************************************************************\n", - systemName.c_str(),localModule.c_str(),currentDate.c_str(),columnstore_version.c_str(),columnstore_release.c_str(),reportType.c_str()); + printf("Could not open file: %s", outputFile.c_str()); + exit(1); } - cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl; + fprintf(pOutputFile, + "********************************************************************************\n" + "\n" + " System %s\n" + " columnstoreSupportReport script ran from Module %s on %s\n" + " SoftwareVersion = %s-%s" + "\n" + "********************************************************************************\n" + "\n" + " %s report\n" + "\n" + "********************************************************************************\n", + systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), + columnstore_release.c_str(), reportType.c_str()); + } - string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" + - reportType + "Report.sh " + remoteModuleName + - "' " + debug_flag + " - forcetty"; + cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl; - int rtnCode = system(cmd.c_str()); + string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" + + reportType + "Report.sh " + remoteModuleName + "' " + debug_flag + " - forcetty"; - if (WEXITSTATUS(rtnCode) != 0) - { - cout << "Error with running remote_command.sh, exiting..." << endl; - } + int rtnCode = system(cmd.c_str()); - cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + " > /dev/null 2>&1"; - rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) + { + cout << "Error with running remote_command.sh, exiting..." << endl; + } - if (WEXITSTATUS(rtnCode) != 0) - cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl; + cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + + " > /dev/null 2>&1"; + rtnCode = system(cmd.c_str()); - pthread_mutex_lock( &mutex1 ); - runningThreads--; -//cout << "-- " << runningThreads << endl; - pthread_mutex_unlock( &mutex1 ); + if (WEXITSTATUS(rtnCode) != 0) + cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl; - // exit thread - pthread_exit(0); + pthread_mutex_lock(&mutex1); + runningThreads--; + // cout << "-- " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); + + // exit thread + pthread_exit(0); } void* reportThread(string* reporttype) { - assert(reporttype); - string reportType = *reporttype; + assert(reporttype); + string reportType = *reporttype; - Oam oam; + Oam oam; - pthread_mutex_lock( &mutex1 ); - runningThreads++; -//cout << "++ " << runningThreads << endl; - pthread_mutex_unlock( &mutex1 ); + pthread_mutex_lock(&mutex1); + runningThreads++; + // cout << "++ " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); - string outputFile = localModule + "_" + reportType + "Report.txt"; + string outputFile = localModule + "_" + reportType + "Report.txt"; - FILE *pOutputFile = fopen(outputFile.c_str(), "a"); - if (pOutputFile == NULL) + FILE* pOutputFile = fopen(outputFile.c_str(), "a"); + if (pOutputFile == NULL) + { + printf("Could not open file: %s", outputFile.c_str()); + exit(1); + } + // get local report + fprintf(pOutputFile, + "********************************************************************************\n" + "\n" + " System %s\n" + " columnstoreSupportReport script ran from Module %s on %s\n" + " SoftwareVersion = %s-%s" + "\n" + "********************************************************************************\n" + "\n" + " %s report\n" + "\n" + "********************************************************************************\n", + systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), + columnstore_release.c_str(), reportType.c_str()); + + fclose(pOutputFile); + // run on child servers and get report + if (!LOCAL) + { + ChildModuleList::iterator list1 = childmodulelist.begin(); + + for (; list1 != childmodulelist.end(); list1++) { - printf("Could not open file: %s",outputFile.c_str()); + threadInfo_t* st = new threadInfo_t; + *st = boost::make_tuple(list1, reportType); + + pthread_t childreportthread; + int status = pthread_create(&childreportthread, NULL, (void* (*)(void*)) & childReportThread, st); + + if (status != 0) + { + cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status) + << endl; + } + + sleep(1); + } + } + + if (reportType == "log") + { + // run log config on local server + cout << "Get log config data for " + localModule << endl; + + string cmd = "logReport.sh " + localModule + " " + outputFile; + system(cmd.c_str()); + } + else + { + string cmd = reportType + "Report.sh " + localModule + " " + outputFile; + system(cmd.c_str()); + + if (reportType == "config") + { + pOutputFile = fopen(outputFile.c_str(), "a"); + if (pOutputFile == NULL) + { + printf("Could not open file: %s", outputFile.c_str()); exit(1); + } + + fprintf(pOutputFile, + "\n******************** System Network Configuration ******************************\n\n"); + getSystemNetworkConfig(pOutputFile); + + fprintf(pOutputFile, + "\n******************** System Module Configure **********************************\n\n"); + getModuleTypeConfig(pOutputFile); + + fprintf(pOutputFile, + "\n******************** System Storage Configuration *****************************\n\n"); + getStorageConfig(pOutputFile); + + fprintf(pOutputFile, + "\n******************** System Storage Status ************************************\n\n"); + getStorageStatus(pOutputFile); + + // BT: most of this is tedious to collect and can be manually looked up in the debug.log file + // fprintf(pOutputFile,"\n******************** System Status + // ********************************************\n\n"); printSystemStatus(pOutputFile); + // printProcessStatus(pOutputFile); + // printAlarmSummary(pOutputFile); + // + // fprintf(pOutputFile,"\n******************** System Directories + // ***************************************\n\n"); getSystemDirectories(pOutputFile); + + boost::filesystem::path configFile = + std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml"); + boost::filesystem::copy_file(configFile, "./Columnstore.xml", + boost::filesystem::copy_option::overwrite_if_exists); + boost::filesystem::path SMconfigFile = + std::string(MCSSYSCONFDIR) + std::string("/columnstore/storagemanager.cnf"); + boost::filesystem::copy_file(SMconfigFile, "./storagemanager.cnf", + boost::filesystem::copy_option::overwrite_if_exists); + system("sed -i 's/.*aws_access_key_id.*/aws_access_key_id={PRIVATE}/' ./storagemanager.cnf"); + system("sed -i 's/.*aws_secret_access_key.*/aws_secret_access_key={PRIVATE}/' ./storagemanager.cnf"); + fclose(pOutputFile); } - //get local report - fprintf(pOutputFile,"********************************************************************************\n" - "\n" - " System %s\n" - " columnstoreSupportReport script ran from Module %s on %s\n" - " SoftwareVersion = %s-%s" - "\n" - "********************************************************************************\n" - "\n" - " %s report\n" - "\n" - "********************************************************************************\n", - systemName.c_str(),localModule.c_str(),currentDate.c_str(),columnstore_version.c_str(),columnstore_release.c_str(),reportType.c_str()); - fclose(pOutputFile); - // run on child servers and get report - if (!LOCAL) + /* + // TODO: This can be ported from mcsadmin if needed most info included does not seem useful at this time + if (reportType == "resource" ) { - ChildModuleList::iterator list1 = childmodulelist.begin(); - - for (; list1 != childmodulelist.end() ; list1++) + if (LOCAL) { - threadInfo_t* st = new threadInfo_t; - *st = boost::make_tuple(list1, reportType); - - pthread_t childreportthread; - int status = pthread_create (&childreportthread, NULL, (void* (*)(void*)) &childReportThread, st); - - if ( status != 0 ) - { - cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status) << endl; - } - - sleep(1); + fprintf(pOutputFile,"\n******************** mcsadmin getModuleResourceUsage + **************************\n\n"); string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " + + outputFile; system(cmd.c_str()); } - } - - if (reportType == "log") - { - // run log config on local server - cout << "Get log config data for " + localModule << endl; - - string cmd = "logReport.sh " + localModule + " " + outputFile; - system(cmd.c_str()); - } - else - { - string cmd = reportType + "Report.sh " + localModule + " " + outputFile; - system(cmd.c_str()); - - if (reportType == "config" ) + else { - pOutputFile = fopen(outputFile.c_str(), "a"); - if (pOutputFile == NULL) - { - printf("Could not open file: %s",outputFile.c_str()); - exit(1); - } - - fprintf(pOutputFile,"\n******************** System Network Configuration ******************************\n\n"); - getSystemNetworkConfig(pOutputFile); - - fprintf(pOutputFile,"\n******************** System Module Configure **********************************\n\n"); - getModuleTypeConfig(pOutputFile); - - fprintf(pOutputFile,"\n******************** System Storage Configuration *****************************\n\n"); - getStorageConfig(pOutputFile); - - fprintf(pOutputFile,"\n******************** System Storage Status ************************************\n\n"); - getStorageStatus(pOutputFile); - - // BT: most of this is tedious to collect and can be manually looked up in the debug.log file - //fprintf(pOutputFile,"\n******************** System Status ********************************************\n\n"); - //printSystemStatus(pOutputFile); - //printProcessStatus(pOutputFile); - //printAlarmSummary(pOutputFile); - // - //fprintf(pOutputFile,"\n******************** System Directories ***************************************\n\n"); - //getSystemDirectories(pOutputFile); - - boost::filesystem::path configFile = std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml"); - boost::filesystem::copy_file(configFile,"./Columnstore.xml",boost::filesystem::copy_option::overwrite_if_exists); - boost::filesystem::path SMconfigFile = std::string(MCSSYSCONFDIR) + std::string("/columnstore/storagemanager.cnf"); - boost::filesystem::copy_file(SMconfigFile,"./storagemanager.cnf",boost::filesystem::copy_option::overwrite_if_exists); - system("sed -i 's/.*aws_access_key_id.*/aws_access_key_id={PRIVATE}/' ./storagemanager.cnf"); - system("sed -i 's/.*aws_secret_access_key.*/aws_secret_access_key={PRIVATE}/' ./storagemanager.cnf"); - fclose(pOutputFile); + fprintf(pOutputFile,"\n******************** mcsadmin getSystemResourceUsage + **************************\n\n"); string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile; + system(cmd.c_str()); } + }*/ + } - /* - // TODO: This can be ported from mcsadmin if needed most info included does not seem useful at this time - if (reportType == "resource" ) - { - if (LOCAL) - { - fprintf(pOutputFile,"\n******************** mcsadmin getModuleResourceUsage **************************\n\n"); - string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " + outputFile; - system(cmd.c_str()); - } - else - { - fprintf(pOutputFile,"\n******************** mcsadmin getSystemResourceUsage **************************\n\n"); - string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile; - system(cmd.c_str()); - } - }*/ - } + // exit thread + pthread_mutex_lock(&mutex1); + runningThreads--; + // cout << "-- " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); - // exit thread - pthread_mutex_lock( &mutex1 ); - runningThreads--; -//cout << "-- " << runningThreads << endl; - pthread_mutex_unlock( &mutex1 ); - - pthread_exit(0); + pthread_exit(0); } int main(int argc, char* argv[]) { - Oam oam; + Oam oam; - Config* sysConfig = Config::makeConfig(); - string SystemSection = "SystemConfig"; - string InstallSection = "Installation"; + Config* sysConfig = Config::makeConfig(); + string SystemSection = "SystemConfig"; + string InstallSection = "Installation"; - bool HARDWARE = false; - bool CONFIG = false; - bool DBMS = false; - bool RESOURCE = false; - bool LOG = false; - bool BULKLOG = false; - bool HADOOP = false; + bool HARDWARE = false; + bool CONFIG = false; + bool DBMS = false; + bool RESOURCE = false; + bool LOG = false; + bool BULKLOG = false; + bool HADOOP = false; - //get current time and date - time_t now; - now = time(NULL); - struct tm tm; - localtime_r(&now, &tm); - char timestamp[200]; - strftime (timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); - currentDate = timestamp; + // get current time and date + time_t now; + now = time(NULL); + struct tm tm; + localtime_r(&now, &tm); + char timestamp[200]; + strftime(timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); + currentDate = timestamp; - char helpArg[3] = "-h"; + char helpArg[3] = "-h"; - // Get System Name - try + // Get System Name + try + { + oam.getSystemConfig("SystemName", systemName); + } + catch (...) + { + systemName = "unassigned"; + } + + // get Local Module Name and Server Install Indicator + string singleServerInstall; + + oamModuleInfo_t st; + + try + { + st = oam.getModuleInfo(); + localModule = boost::get<0>(st); + } + catch (...) + { + cout << endl << "**** Failed : Failed to read Local Module Name" << endl; + exit(-1); + } + + try + { + oam.getSystemConfig("SingleServerInstall", singleServerInstall); + } + catch (...) + { + singleServerInstall = "y"; + } + + if (argc == 1) + { + argv[1] = &helpArg[0]; + argc = 2; + } + + string DataFilePlugin; + + try + { + DataFilePlugin = sysConfig->getConfig(SystemSection, "DataFilePlugin"); + } + catch (...) + { + cout << "ERROR: Problem accessing Columnstore configuration file" << endl; + exit(-1); + } + + tmpDir = startup::StartUp::tmpDir(); + + for (int i = 1; i < argc; i++) + { + if (string("-h") == argv[i]) { - oam.getSystemConfig("SystemName", systemName); + cout << endl; + cout << "'columnstoreSupport' generates a Set of System Support Report Files in a tar file" << endl; + cout << "called columnstoreSupportReport.'system-name'.tar.gz in the local directory." << endl; + cout << "It should be run on the server with the DBRM front-end." << endl; + cout << "Check the Admin Guide for additional information." << endl; + cout << endl; + cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p " + "'root-password'][-de]"; + + cout << endl; + cout << " -h help" << endl; + cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl; + cout << " -hw Output Hardware Reports only" << endl; + cout << " -c Output Configuration/Status Reports only" << endl; + cout << " -db Output DBMS Reports only" << endl; + cout << " -r Output Resource Reports only" << endl; + cout << " -l Output Columnstore Log/Alarms Reports only" << endl; + cout << " -bl Output Columnstore Bulk Log Reports only" << endl; + cout << " -lc Output Reports for Local Server only" << endl; + cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'" + << endl; + cout << " -de Debug Flag" << endl; + + exit(0); } - catch (...) - { - systemName = "unassigned"; - } - - // get Local Module Name and Server Install Indicator - string singleServerInstall; - - oamModuleInfo_t st; - - try - { - st = oam.getModuleInfo(); - localModule = boost::get<0>(st); - } - catch (...) - { - cout << endl << "**** Failed : Failed to read Local Module Name" << endl; - exit(-1); - } - - try - { - oam.getSystemConfig("SingleServerInstall", singleServerInstall); - } - catch (...) - { - singleServerInstall = "y"; - } - - if (argc == 1) - { - argv[1] = &helpArg[0]; - argc = 2; - } - - string DataFilePlugin; - - try - { - DataFilePlugin = sysConfig->getConfig(SystemSection, "DataFilePlugin"); - } - catch (...) - { - cout << "ERROR: Problem accessing Columnstore configuration file" << endl; - exit(-1); - } - - tmpDir = startup::StartUp::tmpDir(); - - for ( int i = 1; i < argc; i++ ) - { - if ( string("-h") == argv[i] ) - { - cout << endl; - cout << "'columnstoreSupport' generates a Set of System Support Report Files in a tar file" << endl; - cout << "called columnstoreSupportReport.'system-name'.tar.gz in the local directory." << endl; - cout << "It should be run on the server with the DBRM front-end." << endl; - cout << "Check the Admin Guide for additional information." << endl; - cout << endl; - cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p 'root-password'][-de]"; - - cout << endl; - cout << " -h help" << endl; - cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl; - cout << " -hw Output Hardware Reports only" << endl; - cout << " -c Output Configuration/Status Reports only" << endl; - cout << " -db Output DBMS Reports only" << endl; - cout << " -r Output Resource Reports only" << endl; - cout << " -l Output Columnstore Log/Alarms Reports only" << endl; - cout << " -bl Output Columnstore Bulk Log Reports only" << endl; - cout << " -lc Output Reports for Local Server only" << endl; - cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'" << endl; - cout << " -de Debug Flag" << endl; - - exit (0); - } - else - { - if ( string("-a") == argv[i] ) - { - HARDWARE = true; - CONFIG = true; - DBMS = true; - RESOURCE = true; - LOG = true; - HADOOP = (DataFilePlugin.empty() ? false : true); - } - else if ( string("-hw") == argv[i] ) - HARDWARE = true; - else if ( string("-c") == argv[i] ) - CONFIG = true; - else if ( string("-db") == argv[i] ) - DBMS = true; - else if ( string("-r") == argv[i] ) - RESOURCE = true; - else if ( string("-l") == argv[i] ) - LOG = true; - else if ( string("-bl") == argv[i] ) - BULKLOG = true; - else if ( string("-lc") == argv[i] ) - LOCAL = true; - else if ( string("-p") == argv[i] ) - { - i++; - - if ( argc == i ) - { - cout << "ERROR: missing root password argument" << endl; - exit(-1); - } - - rootPassword = argv[i]; - - //add single quote for special characters - if ( rootPassword != "ssh" ) - { - rootPassword = "'" + rootPassword + "'"; - } - } - else if ( string("-mp") == argv[i] ) - { - i++; - - if ( argc == i ) - { - cout << "ERROR: missing MariaDB Columnstore root user password argument" << endl; - exit(-1); - } - - mysqlpw = argv[i]; - mysqlpw = "'" + mysqlpw + "'"; - } - else if ( string("-de") == argv[i] ) - debug_flag = "1"; - else if ( string("-hd") == argv[i] ) - { - HADOOP = (DataFilePlugin.empty() ? false : true); - } - else - { - cout << "Invalid Option of '" << argv[i] << "', run with '-h' for help" << endl; - exit (1); - } - } - } - - //default to -a if nothing is set - if ( !HARDWARE && !CONFIG && !DBMS && !RESOURCE && !LOG && !BULKLOG && !HADOOP) + else { + if (string("-a") == argv[i]) + { HARDWARE = true; CONFIG = true; DBMS = true; RESOURCE = true; LOG = true; HADOOP = (DataFilePlugin.empty() ? false : true); - } + } + else if (string("-hw") == argv[i]) + HARDWARE = true; + else if (string("-c") == argv[i]) + CONFIG = true; + else if (string("-db") == argv[i]) + DBMS = true; + else if (string("-r") == argv[i]) + RESOURCE = true; + else if (string("-l") == argv[i]) + LOG = true; + else if (string("-bl") == argv[i]) + BULKLOG = true; + else if (string("-lc") == argv[i]) + LOCAL = true; + else if (string("-p") == argv[i]) + { + i++; - //get Parent OAM Module Name and setup of it's Custom OS files - string PrimaryUMModuleName; - - try - { - PrimaryUMModuleName = sysConfig->getConfig(SystemSection, "PrimaryUMModuleName"); - } - catch (...) - { - cout << "ERROR: Problem getting Parent OAM Module Name" << endl; - exit(-1); - } - - if ( PrimaryUMModuleName == "unassigned" ) - PrimaryUMModuleName = localModule; - - if ( (localModule != PrimaryUMModuleName) && DBMS ) - { - char* pcommand = 0; - char* p; - string argument = "n"; - - while (true) + if (argc == i) { - cout << endl << "You selected to get the DBMS data." << endl; - cout << "You need to run the columnstoreSupport command on module '" << PrimaryUMModuleName << "' to get that information." << endl; - cout << "Or you can proceed on to get all data except the DBMS." << endl; - - pcommand = readline(" Do you want to proceed: (y or n) [n]: "); - - if (pcommand && *pcommand) - { - p = strtok(pcommand, " "); - argument = p; - free(pcommand); - pcommand = 0; - } - - if (pcommand) - { - free(pcommand); - pcommand = 0; - } - - if ( argument == "y") - { - cout << endl; - break; - } - else if ( argument == "n") - exit (1); - } - } - - //get number of worker-nodes, will tell us if a single server system - //get Parent OAM Module Name and setup of it's Custom OS files - try - { - string NumWorkers = sysConfig->getConfig("DBRM_Controller", "NumWorkers"); - - if ( NumWorkers == "1" ) - singleServerInstall = "y"; - } - catch (...) - {} - - if ( singleServerInstall == "n" && !LOCAL) - if ( HARDWARE || CONFIG || RESOURCE || LOG || HADOOP ) - if ( rootPassword.empty() ) - { - cout << "ERROR: Multi-Module System, Password Argument required or use '-lc' option, check help for more information" << endl; - exit(-1); - } - - //get Parent OAM Module Name and setup of it's Custom OS files - //string parentOAMModuleName; - ChildModule parentOAMModule; - - try - { - parentOAMModule.moduleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName"); - } - catch (...) - { - cout << "ERROR: Problem getting Parent OAM Module Name" << endl; - exit(-1); - } - - //Get list of configured system modules - SystemModuleTypeConfig sysModuleTypeConfig; - - try - { - oam.getSystemConfig(sysModuleTypeConfig); - } - catch (...) - { - cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; - exit(-1); - } - - //Get Profile file - try - { - ProfileFile = sysConfig->getConfig(InstallSection, "ProfileFile"); - } - catch (...) - { - cout << "ERROR: Problem getting ProfileFile" << endl; - exit(-1); - } - - string ModuleSection = "SystemModuleConfig"; - - for ( unsigned int i = 0 ; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) - { - string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType; - int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount; - - if ( moduleCount == 0 ) - //no modules equipped for this Module Type, skip - continue; - - //get IP addresses and Host Names - DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for ( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) - { - string moduleName = (*listPT).DeviceName; - HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); - string moduleIPAddr = (*pt1).IPAddr; - string moduleHostName = (*pt1).HostName; - - if ( moduleName == localModule) - { - localModuleHostName = moduleHostName; - } - - //save Child modules - if ( moduleName != localModule && moduleType != "xm") - { - childmodule.moduleName = moduleName; - childmodule.moduleIP = moduleIPAddr; - childmodule.hostName = moduleHostName; - childmodulelist.push_back(childmodule); - } - - if (moduleName == parentOAMModule.moduleName) - { - parentOAMModule.moduleIP = moduleIPAddr; - parentOAMModule.hostName = moduleHostName; - parentOAMModule.moduleName = moduleName; - } - } - } //end of i for loop - - // create a clean Columnstore Support Report - system("rm -f *_configReport.txt"); - system("rm -f *_dbmsReport.txt"); - system("rm -f *_hardwareReport.txt"); - system("rm -f *_logReport.txt"); - system("rm -f *_bulklogReport.txt"); - system("rm -f *_resourceReport.txt"); - - // - // Configuration - // - if ( CONFIG ) - { - string reportType = "config"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create (&reportthread, NULL, (void* (*)(void*)) &reportThread, &reportType); - if ( status != 0 ) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Alarms and Columnstore Logs - // - if ( LOG ) - { - string reportType = "log"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create (&reportthread, NULL, (void* (*)(void*)) &reportThread, &reportType); - if ( status != 0 ) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Bulk Logs - // - if ( BULKLOG ) - { - string reportType = "bulklog"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create (&reportthread, NULL, (void* (*)(void*)) &reportThread, &reportType); - if ( status != 0 ) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Hardware - // - if ( HARDWARE ) - { - string reportType = "hardware"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create (&reportthread, NULL, (void* (*)(void*)) &reportThread, &reportType); - if ( status != 0 ) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Resources - // - if ( RESOURCE ) - { - string reportType = "resource"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create (&reportthread, NULL, (void* (*)(void*)) &reportThread, &reportType); - if ( status != 0 ) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // DBMS - // - if ( DBMS ) - { - cout << "Get dbms report data for " << localModule << endl; - - string outputFile = localModule + "_dbmsReport.txt"; - - FILE *pOutputFile = fopen(outputFile.c_str(), "w"); - if (pOutputFile == NULL) - { - cout << "Could not open file: " + outputFile << endl; - exit(1); + cout << "ERROR: missing root password argument" << endl; + exit(-1); } - fprintf(pOutputFile,"********************************************************************************\n" - "\n" - " System %s\n" - " columnstoreSupportReport script ran from Module %s on %s\n" - " SoftwareVersion = %s-%s" - "\n" - "********************************************************************************\n" - "\n" - " DBMS report\n" - "\n" - "********************************************************************************\n", - systemName.c_str(),localModule.c_str(),currentDate.c_str(),columnstore_version.c_str(),columnstore_release.c_str()); + rootPassword = argv[i]; - fclose(pOutputFile); - - // run DBMS report on local server - bool FAILED = false; - - if ( localModule != PrimaryUMModuleName ) + // add single quote for special characters + if (rootPassword != "ssh") { - cout << " FAILED: run columnstoreSupport on '" << PrimaryUMModuleName << "' to get the dbrm report" << endl; - FAILED = true; + rootPassword = "'" + rootPassword + "'"; } - else + } + else if (string("-mp") == argv[i]) + { + i++; + + if (argc == i) { - // check if mysql is supported and get info - string logFile = tmpDir + "/idbmysql.log"; - string columnstoreMysql = "mysql -u root "; - string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1"; - system(cmd.c_str()); - - //check for mysql password set - string pwprompt = " "; - - if (checkLogStatus(logFile, "ERROR 1045") ) - { - cout << "NOTE: MariaDB Columnstore root user password is set" << endl; - - //needs a password, was password entered on command line - if ( mysqlpw == " " ) - { - //go check columnstore.cnf - string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; - ifstream oldFile (file.c_str()); - - vector lines; - char line[200]; - string buf; - - while (oldFile.getline(line, 200)) - { - buf = line; - string::size_type pos = buf.find("password", 0); - - if (pos != string::npos) - { - string::size_type pos1 = buf.find("=", 0); - - if (pos1 != string::npos) - { - pos = buf.find("#", 0); - - if (pos == string::npos) - { - //password arg in columnstore.cnf, go get password - cout << "NOTE: Using password from columnstore.cnf" << endl; - mysqlpw = buf.substr(pos1 + 1, 80); - cout << mysqlpw << endl; - break; - } - } - } - } - - oldFile.close(); - - if ( mysqlpw == " " ) - { - cout << "NOTE: No password provide on command line or found uncommented in columnstore.cnf" << endl; - cout << endl; - string prompt = " *** Enter MariaDB Columnstore password > "; - mysqlpw = getpass(prompt.c_str()); - } - } - - //check for mysql password set - pwprompt = "--password=" + mysqlpw; - - string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1"; - system(cmd.c_str()); - - if (checkLogStatus(logFile, "ERROR 1045") ) - { - cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'" << endl; - FAILED = true; - } - } - - if (!FAILED) - { - string cmd = "dbmsReport.sh " + localModule + " " + outputFile + " " + std::string(MCSSUPPORTDIR) + " " + pwprompt ; - system(cmd.c_str()); - } + cout << "ERROR: missing MariaDB Columnstore root user password argument" << endl; + exit(-1); } - /* - BT: This doesn't appear to do anything - fprintf(pOutputFile,"\n******************** Database Size Report *************************************\n\n"); - getStorageStatus(pOutputFile); - - string file = "databaseSizeReport"; - ifstream File (file.c_str()); - - if (File) - { - string cmd = "databaseSizeReport >> " + outputFile; - system(cmd.c_str()); - } - */ - - boost::filesystem::path configFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; - boost::filesystem::copy_file(configFile,"./columnstore.cnf",boost::filesystem::copy_option::overwrite_if_exists); + mysqlpw = argv[i]; + mysqlpw = "'" + mysqlpw + "'"; + } + else if (string("-de") == argv[i]) + debug_flag = "1"; + else if (string("-hd") == argv[i]) + { + HADOOP = (DataFilePlugin.empty() ? false : true); + } + else + { + cout << "Invalid Option of '" << argv[i] << "', run with '-h' for help" << endl; + exit(1); + } } + } - int wait = 0; + // default to -a if nothing is set + if (!HARDWARE && !CONFIG && !DBMS && !RESOURCE && !LOG && !BULKLOG && !HADOOP) + { + HARDWARE = true; + CONFIG = true; + DBMS = true; + RESOURCE = true; + LOG = true; + HADOOP = (DataFilePlugin.empty() ? false : true); + } + + // get Parent OAM Module Name and setup of it's Custom OS files + string PrimaryUMModuleName; + + try + { + PrimaryUMModuleName = sysConfig->getConfig(SystemSection, "PrimaryUMModuleName"); + } + catch (...) + { + cout << "ERROR: Problem getting Parent OAM Module Name" << endl; + exit(-1); + } + + if (PrimaryUMModuleName == "unassigned") + PrimaryUMModuleName = localModule; + + if ((localModule != PrimaryUMModuleName) && DBMS) + { + char* pcommand = 0; + char* p; + string argument = "n"; while (true) { -//cout << "check " << runningThreads << endl; - if (runningThreads < 1) - break; + cout << endl << "You selected to get the DBMS data." << endl; + cout << "You need to run the columnstoreSupport command on module '" << PrimaryUMModuleName + << "' to get that information." << endl; + cout << "Or you can proceed on to get all data except the DBMS." << endl; - sleep(2); - wait++; + pcommand = readline(" Do you want to proceed: (y or n) [n]: "); - // give it 60 minutes to complete - if ( wait >= 3600 * 5) - { - cout << "Timed out (60 minutes) waiting for Requests to complete" << endl; - } + if (pcommand && *pcommand) + { + p = strtok(pcommand, " "); + argument = p; + free(pcommand); + pcommand = 0; + } + + if (pcommand) + { + free(pcommand); + pcommand = 0; + } + + if (argument == "y") + { + cout << endl; + break; + } + else if (argument == "n") + exit(1); + } + } + + // get number of worker-nodes, will tell us if a single server system + // get Parent OAM Module Name and setup of it's Custom OS files + try + { + string NumWorkers = sysConfig->getConfig("DBRM_Controller", "NumWorkers"); + + if (NumWorkers == "1") + singleServerInstall = "y"; + } + catch (...) + { + } + + if (singleServerInstall == "n" && !LOCAL) + if (HARDWARE || CONFIG || RESOURCE || LOG || HADOOP) + if (rootPassword.empty()) + { + cout << "ERROR: Multi-Module System, Password Argument required or use '-lc' option, check help for " + "more information" + << endl; + exit(-1); + } + + // get Parent OAM Module Name and setup of it's Custom OS files + // string parentOAMModuleName; + ChildModule parentOAMModule; + + try + { + parentOAMModule.moduleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName"); + } + catch (...) + { + cout << "ERROR: Problem getting Parent OAM Module Name" << endl; + exit(-1); + } + + // Get list of configured system modules + SystemModuleTypeConfig sysModuleTypeConfig; + + try + { + oam.getSystemConfig(sysModuleTypeConfig); + } + catch (...) + { + cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; + exit(-1); + } + + // Get Profile file + try + { + ProfileFile = sysConfig->getConfig(InstallSection, "ProfileFile"); + } + catch (...) + { + cout << "ERROR: Problem getting ProfileFile" << endl; + exit(-1); + } + + string ModuleSection = "SystemModuleConfig"; + + for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) + { + string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType; + int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount; + + if (moduleCount == 0) + // no modules equipped for this Module Type, skip + continue; + + // get IP addresses and Host Names + DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end(); listPT++) + { + string moduleName = (*listPT).DeviceName; + HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); + string moduleIPAddr = (*pt1).IPAddr; + string moduleHostName = (*pt1).HostName; + + if (moduleName == localModule) + { + localModuleHostName = moduleHostName; + } + + // save Child modules + if (moduleName != localModule && moduleType != "xm") + { + childmodule.moduleName = moduleName; + childmodule.moduleIP = moduleIPAddr; + childmodule.hostName = moduleHostName; + childmodulelist.push_back(childmodule); + } + + if (moduleName == parentOAMModule.moduleName) + { + parentOAMModule.moduleIP = moduleIPAddr; + parentOAMModule.hostName = moduleHostName; + parentOAMModule.moduleName = moduleName; + } + } + } // end of i for loop + + // create a clean Columnstore Support Report + system("rm -f *_configReport.txt"); + system("rm -f *_dbmsReport.txt"); + system("rm -f *_hardwareReport.txt"); + system("rm -f *_logReport.txt"); + system("rm -f *_bulklogReport.txt"); + system("rm -f *_resourceReport.txt"); + + // + // Configuration + // + if (CONFIG) + { + string reportType = "config"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Alarms and Columnstore Logs + // + if (LOG) + { + string reportType = "log"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Bulk Logs + // + if (BULKLOG) + { + string reportType = "bulklog"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Hardware + // + if (HARDWARE) + { + string reportType = "hardware"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Resources + // + if (RESOURCE) + { + string reportType = "resource"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // DBMS + // + if (DBMS) + { + cout << "Get dbms report data for " << localModule << endl; + + string outputFile = localModule + "_dbmsReport.txt"; + + FILE* pOutputFile = fopen(outputFile.c_str(), "w"); + if (pOutputFile == NULL) + { + cout << "Could not open file: " + outputFile << endl; + exit(1); } - system("unix2dos *Report.txt > /dev/null 2>&1"); - system("rm -rf columnstoreSupportReport;" - "mkdir columnstoreSupportReport;" - "mv *Report.txt columnstoreSupportReport/. > /dev/null 2>&1;" - "mv Columnstore.xml columnstoreSupportReport/. > /dev/null 2>&1;" - "mv columnstore.cnf columnstoreSupportReport/. > /dev/null 2>&1;" - "mv storagemanager.cnf columnstoreSupportReport/. > /dev/null 2>&1;" - "mv *Report.tar.gz columnstoreSupportReport/. > /dev/null 2>&1"); - string cmd = "tar -zcf columnstoreSupportReport." + systemName + ".tar.gz columnstoreSupportReport/*"; - system(cmd.c_str()); + fprintf(pOutputFile, + "********************************************************************************\n" + "\n" + " System %s\n" + " columnstoreSupportReport script ran from Module %s on %s\n" + " SoftwareVersion = %s-%s" + "\n" + "********************************************************************************\n" + "\n" + " DBMS report\n" + "\n" + "********************************************************************************\n", + systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), + columnstore_release.c_str()); + fclose(pOutputFile); - cout << endl << "Columnstore Support Script Successfully completed, files located in columnstoreSupportReport." + systemName + ".tar.gz" << endl; + // run DBMS report on local server + bool FAILED = false; + + if (localModule != PrimaryUMModuleName) + { + cout << " FAILED: run columnstoreSupport on '" << PrimaryUMModuleName << "' to get the dbrm report" + << endl; + FAILED = true; + } + else + { + // check if mysql is supported and get info + string logFile = tmpDir + "/idbmysql.log"; + string columnstoreMysql = "mysql -u root "; + string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1"; + system(cmd.c_str()); + + // check for mysql password set + string pwprompt = " "; + + if (checkLogStatus(logFile, "ERROR 1045")) + { + cout << "NOTE: MariaDB Columnstore root user password is set" << endl; + + // needs a password, was password entered on command line + if (mysqlpw == " ") + { + // go check columnstore.cnf + string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; + ifstream oldFile(file.c_str()); + + vector lines; + char line[200]; + string buf; + + while (oldFile.getline(line, 200)) + { + buf = line; + string::size_type pos = buf.find("password", 0); + + if (pos != string::npos) + { + string::size_type pos1 = buf.find("=", 0); + + if (pos1 != string::npos) + { + pos = buf.find("#", 0); + + if (pos == string::npos) + { + // password arg in columnstore.cnf, go get password + cout << "NOTE: Using password from columnstore.cnf" << endl; + mysqlpw = buf.substr(pos1 + 1, 80); + cout << mysqlpw << endl; + break; + } + } + } + } + + oldFile.close(); + + if (mysqlpw == " ") + { + cout << "NOTE: No password provide on command line or found uncommented in columnstore.cnf" + << endl; + cout << endl; + string prompt = " *** Enter MariaDB Columnstore password > "; + mysqlpw = getpass(prompt.c_str()); + } + } + + // check for mysql password set + pwprompt = "--password=" + mysqlpw; + + string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1"; + system(cmd.c_str()); + + if (checkLogStatus(logFile, "ERROR 1045")) + { + cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'" + << endl; + FAILED = true; + } + } + + if (!FAILED) + { + string cmd = "dbmsReport.sh " + localModule + " " + outputFile + " " + std::string(MCSSUPPORTDIR) + + " " + pwprompt; + system(cmd.c_str()); + } + } + + /* + BT: This doesn't appear to do anything + fprintf(pOutputFile,"\n******************** Database Size Report + *************************************\n\n"); getStorageStatus(pOutputFile); + + string file = "databaseSizeReport"; + ifstream File (file.c_str()); + + if (File) + { + string cmd = "databaseSizeReport >> " + outputFile; + system(cmd.c_str()); + } + */ + + boost::filesystem::path configFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; + boost::filesystem::copy_file(configFile, "./columnstore.cnf", + boost::filesystem::copy_option::overwrite_if_exists); + } + + int wait = 0; + + while (true) + { + // cout << "check " << runningThreads << endl; + if (runningThreads < 1) + break; + + sleep(2); + wait++; + + // give it 60 minutes to complete + if (wait >= 3600 * 5) + { + cout << "Timed out (60 minutes) waiting for Requests to complete" << endl; + } + } + + system("unix2dos *Report.txt > /dev/null 2>&1"); + system( + "rm -rf columnstoreSupportReport;" + "mkdir columnstoreSupportReport;" + "mv *Report.txt columnstoreSupportReport/. > /dev/null 2>&1;" + "mv Columnstore.xml columnstoreSupportReport/. > /dev/null 2>&1;" + "mv columnstore.cnf columnstoreSupportReport/. > /dev/null 2>&1;" + "mv storagemanager.cnf columnstoreSupportReport/. > /dev/null 2>&1;" + "mv *Report.tar.gz columnstoreSupportReport/. > /dev/null 2>&1"); + string cmd = "tar -zcf columnstoreSupportReport." + systemName + ".tar.gz columnstoreSupportReport/*"; + system(cmd.c_str()); + + cout << endl + << "Columnstore Support Script Successfully completed, files located in columnstoreSupportReport." + + systemName + ".tar.gz" + << endl; } diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.cpp b/oamapps/columnstoreSupport/mcsSupportUtil.cpp index 376c0881b..4c3e7106e 100644 --- a/oamapps/columnstoreSupport/mcsSupportUtil.cpp +++ b/oamapps/columnstoreSupport/mcsSupportUtil.cpp @@ -21,573 +21,584 @@ using namespace std; using namespace oam; using namespace config; -void getSystemNetworkConfig(FILE * pOutputFile) +void getSystemNetworkConfig(FILE* pOutputFile) { - Oam oam; - // get and display Module Network Config - SystemModuleTypeConfig systemmoduletypeconfig; - systemmoduletypeconfig.moduletypeconfig.clear(); + Oam oam; + // get and display Module Network Config + SystemModuleTypeConfig systemmoduletypeconfig; + systemmoduletypeconfig.moduletypeconfig.clear(); - // get max length of a host name for header formatting + // get max length of a host name for header formatting - int maxSize = 9; + int maxSize = 9; - try + try + { + oam.getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { - oam.getSystemConfig(systemmoduletypeconfig); + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; + + if (moduleCount > 0) + { + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) { - if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty() ) - // end of list - break; + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; - - if ( moduleCount > 0 ) - { - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end() ; pt++) - { - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) - { - if ( maxSize < (int) (*pt1).HostName.size() ) - maxSize = (*pt1).HostName.size(); - } - } - } + for (; pt1 != (*pt).hostConfigList.end(); pt1++) + { + if (maxSize < (int)(*pt1).HostName.size()) + maxSize = (*pt1).HostName.size(); + } } + } } - catch (exception& e) + } + catch (exception& e) + { + fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what()); + } + + fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "Module Name", "Module Description", "NIC ID", + "Host Name", "IP Address"); + fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "-----------", "-------------------------", "------", + "---------", "---------------"); + + try + { + oam.getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { - fprintf(pOutputFile,"**** getNetworkConfig Failed = %s\n\n",e.what()); - } + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; - fprintf(pOutputFile,"%-15s%-30s%-10s%-14s%-20s\n", - "Module Name", - "Module Description", - "NIC ID", - "Host Name", - "IP Address"); - fprintf(pOutputFile,"%-15s%-30s%-10s%-14s%-20s\n", - "-----------", - "-------------------------", - "------", - "---------", - "---------------"); + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; - try - { - oam.getSystemConfig(systemmoduletypeconfig); + if (moduleCount > 0) + { + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) { - if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty() ) - // end of list - break; + string modulename = (*pt).DeviceName; + string moduleID = modulename.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE); + string modulenamedesc = moduletypedesc + " #" + moduleID; - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; + fprintf(pOutputFile, "%-15s%-30s", modulename.c_str(), modulenamedesc.c_str()); - if ( moduleCount > 0 ) + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for (; pt1 != (*pt).hostConfigList.end(); pt1++) + { + /* MCOL-1607. IPAddr may be a host name here b/c it is read straight + from the config file. */ + string tmphost = getIPAddress(pt1->IPAddr); + string ipAddr; + if (tmphost.empty()) + ipAddr = pt1->IPAddr; + else + ipAddr = tmphost; + string hostname = (*pt1).HostName; + string nicID = oam.itoa((*pt1).NicID); + + if (nicID != "1") { - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end() ; pt++) - { - string modulename = (*pt).DeviceName; - string moduleID = modulename.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE); - string modulenamedesc = moduletypedesc + " #" + moduleID; - - fprintf(pOutputFile,"%-15s%-30s", - modulename.c_str(), - modulenamedesc.c_str()); - - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) - { - /* MCOL-1607. IPAddr may be a host name here b/c it is read straight - from the config file. */ - string tmphost = getIPAddress(pt1->IPAddr); - string ipAddr; - if (tmphost.empty()) - ipAddr = pt1->IPAddr; - else - ipAddr = tmphost; - string hostname = (*pt1).HostName; - string nicID = oam.itoa((*pt1).NicID); - - if ( nicID != "1" ) - { - fprintf(pOutputFile,"%-45s",""); - } - fprintf(pOutputFile,"%-13s%-14s%-20s\n", - nicID.c_str(), - hostname.c_str(), - ipAddr.c_str()); - } - } + fprintf(pOutputFile, "%-45s", ""); } + fprintf(pOutputFile, "%-13s%-14s%-20s\n", nicID.c_str(), hostname.c_str(), ipAddr.c_str()); + } } + } } - catch (exception& e) - { - fprintf(pOutputFile,"**** getNetworkConfig Failed = %s\n\n",e.what()); - } - + } + catch (exception& e) + { + fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what()); + } } -void getModuleTypeConfig(FILE * pOutputFile) +void getModuleTypeConfig(FILE* pOutputFile) { - Oam oam; - SystemModuleTypeConfig systemmoduletypeconfig; - ModuleTypeConfig moduletypeconfig; - ModuleConfig moduleconfig; - systemmoduletypeconfig.moduletypeconfig.clear(); + Oam oam; + SystemModuleTypeConfig systemmoduletypeconfig; + ModuleTypeConfig moduletypeconfig; + ModuleConfig moduleconfig; + systemmoduletypeconfig.moduletypeconfig.clear(); - try + try + { + oam.getSystemConfig(systemmoduletypeconfig); + + fprintf(pOutputFile, "Module Type Configuration\n\n"); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { - oam.getSystemConfig(systemmoduletypeconfig); + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; - fprintf(pOutputFile,"Module Type Configuration\n\n"); + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + if (moduleCount < 1) + continue; + + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + + fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str()); + fprintf(pOutputFile, "ModuleDesc = %s\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str()); + fprintf(pOutputFile, "RunType = %s\n", systemmoduletypeconfig.moduletypeconfig[i].RunType.c_str()); + fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount); + + if (moduleCount > 0) + { + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) { - if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty() ) - // end of list - break; + string modulename = (*pt).DeviceName; + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - - if ( moduleCount < 1 ) - continue; - - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - - fprintf(pOutputFile,"ModuleType '%s' Configuration information\n",moduletype.c_str()); - fprintf(pOutputFile,"ModuleDesc = %s\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str()); - fprintf(pOutputFile,"RunType = %s\n",systemmoduletypeconfig.moduletypeconfig[i].RunType.c_str()); - fprintf(pOutputFile,"ModuleCount = %i\n",moduleCount); - - if ( moduleCount > 0 ) - { - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end() ; pt++) - { - string modulename = (*pt).DeviceName; - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) - { - string ipAddr = (*pt1).IPAddr; - string servername = (*pt1).HostName; - fprintf(pOutputFile,"ModuleHostName and ModuleIPAddr for NIC ID %u on module '%s' = %s , %s\n",(*pt1).NicID,modulename.c_str(),servername.c_str(),ipAddr.c_str()); - } - } - } - - DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); - - for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end() ; pt++) - { - if ( (*pt).dbrootConfigList.size() > 0 ) - { - fprintf(pOutputFile,"DBRootIDs assigned to module 'pm%u' = ",(*pt).DeviceID); - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for ( ; pt1 != (*pt).dbrootConfigList.end() ; ) - { - fprintf(pOutputFile,"%u",*pt1); - pt1++; - - if (pt1 != (*pt).dbrootConfigList.end()) - fprintf(pOutputFile,", "); - } - } - fprintf(pOutputFile,"\n"); - } - - fprintf(pOutputFile,"ModuleCPUCriticalThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUCriticalThreshold); - fprintf(pOutputFile,"ModuleCPUMajorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMajorThreshold); - fprintf(pOutputFile,"ModuleCPUMinorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorThreshold); - fprintf(pOutputFile,"ModuleCPUMinorClearThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorClearThreshold); - fprintf(pOutputFile,"ModuleDiskCriticalThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskCriticalThreshold); - fprintf(pOutputFile,"ModuleDiskMajorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMajorThreshold); - fprintf(pOutputFile,"ModuleDiskMinorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMinorThreshold); - fprintf(pOutputFile,"ModuleMemCriticalThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleMemCriticalThreshold); - fprintf(pOutputFile,"ModuleMemMajorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMajorThreshold); - fprintf(pOutputFile,"ModuleMemMinorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMinorThreshold); - fprintf(pOutputFile,"ModuleSwapCriticalThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapCriticalThreshold); - fprintf(pOutputFile,"ModuleSwapMajorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMajorThreshold); - fprintf(pOutputFile,"ModuleSwapMinorThreshold = %u%%\n",systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMinorThreshold); - - DiskMonitorFileSystems::iterator pt2 = systemmoduletypeconfig.moduletypeconfig[i].FileSystems.begin(); - int id = 1; - - for ( ; pt2 != systemmoduletypeconfig.moduletypeconfig[i].FileSystems.end() ; pt2++) - { - string fs = *pt2; - fprintf(pOutputFile,"ModuleDiskMonitorFileSystem#%i = %s\n",id,fs.c_str()); - ++id; - } - fprintf(pOutputFile,"\n"); + for (; pt1 != (*pt).hostConfigList.end(); pt1++) + { + string ipAddr = (*pt1).IPAddr; + string servername = (*pt1).HostName; + fprintf(pOutputFile, "ModuleHostName and ModuleIPAddr for NIC ID %u on module '%s' = %s , %s\n", + (*pt1).NicID, modulename.c_str(), servername.c_str(), ipAddr.c_str()); + } } + } + + DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++) + { + if ((*pt).dbrootConfigList.size() > 0) + { + fprintf(pOutputFile, "DBRootIDs assigned to module 'pm%u' = ", (*pt).DeviceID); + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); + + for (; pt1 != (*pt).dbrootConfigList.end();) + { + fprintf(pOutputFile, "%u", *pt1); + pt1++; + + if (pt1 != (*pt).dbrootConfigList.end()) + fprintf(pOutputFile, ", "); + } + } + fprintf(pOutputFile, "\n"); + } + + fprintf(pOutputFile, "ModuleCPUCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUCriticalThreshold); + fprintf(pOutputFile, "ModuleCPUMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMajorThreshold); + fprintf(pOutputFile, "ModuleCPUMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorThreshold); + fprintf(pOutputFile, "ModuleCPUMinorClearThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorClearThreshold); + fprintf(pOutputFile, "ModuleDiskCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskCriticalThreshold); + fprintf(pOutputFile, "ModuleDiskMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMajorThreshold); + fprintf(pOutputFile, "ModuleDiskMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMinorThreshold); + fprintf(pOutputFile, "ModuleMemCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleMemCriticalThreshold); + fprintf(pOutputFile, "ModuleMemMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMajorThreshold); + fprintf(pOutputFile, "ModuleMemMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMinorThreshold); + fprintf(pOutputFile, "ModuleSwapCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapCriticalThreshold); + fprintf(pOutputFile, "ModuleSwapMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMajorThreshold); + fprintf(pOutputFile, "ModuleSwapMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMinorThreshold); + + DiskMonitorFileSystems::iterator pt2 = systemmoduletypeconfig.moduletypeconfig[i].FileSystems.begin(); + int id = 1; + + for (; pt2 != systemmoduletypeconfig.moduletypeconfig[i].FileSystems.end(); pt2++) + { + string fs = *pt2; + fprintf(pOutputFile, "ModuleDiskMonitorFileSystem#%i = %s\n", id, fs.c_str()); + ++id; + } + fprintf(pOutputFile, "\n"); } - catch (exception& e) - { - cout << endl << "**** getModuleTypeConfig Failed = " << e.what() << endl; - } + } + catch (exception& e) + { + cout << endl << "**** getModuleTypeConfig Failed = " << e.what() << endl; + } } -void getStorageConfig(FILE * pOutputFile) +void getStorageConfig(FILE* pOutputFile) { - Oam oam; + Oam oam; + try + { + systemStorageInfo_t t; + t = oam.getStorageConfig(); + + string cloud; + try { - systemStorageInfo_t t; - t = oam.getStorageConfig(); + oam.getSystemConfig("Cloud", cloud); + } + catch (...) + { + } - string cloud; + string::size_type pos = cloud.find("amazon", 0); - try + if (pos != string::npos) + cloud = "amazon"; + + fprintf(pOutputFile, "System Storage Configuration\n"); + + fprintf(pOutputFile, "Performance Module (DBRoot) Storage Type = %s\n", boost::get<0>(t).c_str()); + + if (cloud == "amazon") + fprintf(pOutputFile, "User Module Storage Type = %s\n", boost::get<3>(t).c_str()); + + fprintf(pOutputFile, "System Assigned DBRoot Count = %i\n", boost::get<1>(t)); + + DeviceDBRootList moduledbrootlist = boost::get<2>(t); + + typedef std::vector dbrootList; + dbrootList dbrootlist; + + DeviceDBRootList::iterator pt = moduledbrootlist.begin(); + + for (; pt != moduledbrootlist.end(); pt++) + { + fprintf(pOutputFile, "DBRoot IDs assigned to 'pm%u' = ", (*pt).DeviceID); + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); + + for (; pt1 != (*pt).dbrootConfigList.end();) + { + fprintf(pOutputFile, "%u", *pt1); + dbrootlist.push_back(*pt1); + pt1++; + + if (pt1 != (*pt).dbrootConfigList.end()) + fprintf(pOutputFile, ", "); + } + + fprintf(pOutputFile, "\n"); + } + + // get any unassigned DBRoots + /*DBRootConfigList undbrootlist; + + try + { + oam.getUnassignedDbroot(undbrootlist); + } + catch (...) {} + + if ( !undbrootlist.empty() ) + { + fprintf(pOutputFile,"DBRoot IDs unassigned = "); + DBRootConfigList::iterator pt1 = undbrootlist.begin(); + + for ( ; pt1 != undbrootlist.end() ;) { - oam.getSystemConfig("Cloud", cloud); + fprintf(pOutputFile,"%u",*pt1); + pt1++; + + if (pt1 != undbrootlist.end()) + fprintf(pOutputFile,", "); } - catch (...) {} - - string::size_type pos = cloud.find("amazon", 0); - - if (pos != string::npos) - cloud = "amazon"; - - fprintf(pOutputFile,"System Storage Configuration\n"); - - fprintf(pOutputFile,"Performance Module (DBRoot) Storage Type = %s\n",boost::get<0>(t).c_str()); - - if ( cloud == "amazon" ) - fprintf(pOutputFile,"User Module Storage Type = %s\n",boost::get<3>(t).c_str()); - - fprintf(pOutputFile,"System Assigned DBRoot Count = %i\n",boost::get<1>(t)); - - DeviceDBRootList moduledbrootlist = boost::get<2>(t); - - typedef std::vector dbrootList; - dbrootList dbrootlist; - - DeviceDBRootList::iterator pt = moduledbrootlist.begin(); - - for ( ; pt != moduledbrootlist.end() ; pt++) - { - fprintf(pOutputFile,"DBRoot IDs assigned to 'pm%u' = ",(*pt).DeviceID); - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for ( ; pt1 != (*pt).dbrootConfigList.end() ;) - { - fprintf(pOutputFile,"%u",*pt1); - dbrootlist.push_back(*pt1); - pt1++; - - if (pt1 != (*pt).dbrootConfigList.end()) - fprintf(pOutputFile,", "); - } - - fprintf(pOutputFile,"\n"); - } - - //get any unassigned DBRoots - /*DBRootConfigList undbrootlist; - - try - { - oam.getUnassignedDbroot(undbrootlist); - } - catch (...) {} - - if ( !undbrootlist.empty() ) - { - fprintf(pOutputFile,"DBRoot IDs unassigned = "); - DBRootConfigList::iterator pt1 = undbrootlist.begin(); - - for ( ; pt1 != undbrootlist.end() ;) - { - fprintf(pOutputFile,"%u",*pt1); - pt1++; - - if (pt1 != undbrootlist.end()) - fprintf(pOutputFile,", "); - } - - fprintf(pOutputFile,"\n"); - }*/ fprintf(pOutputFile,"\n"); - - // um volumes - if (cloud == "amazon" && boost::get<3>(t) == "external") - { - ModuleTypeConfig moduletypeconfig; - oam.getSystemConfig("um", moduletypeconfig); - - for ( int id = 1; id < moduletypeconfig.ModuleCount + 1 ; id++) - { - string volumeNameID = "UMVolumeName" + oam.itoa(id); - string volumeName = oam::UnassignedName; - string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id); - string deviceName = oam::UnassignedName; - - try - { - oam.getSystemConfig( volumeNameID, volumeName); - oam.getSystemConfig( deviceNameID, deviceName); - } - catch (...) - {} - - fprintf(pOutputFile,"Amazon EC2 Volume Name/Device Name for 'um%i': %s, %s",id,volumeName.c_str(),deviceName.c_str()); - } - } - - // pm volumes - if (cloud == "amazon" && boost::get<0>(t) == "external") - { - fprintf(pOutputFile,"\n"); - - DBRootConfigList dbrootConfigList; - - try - { - oam.getSystemDbrootConfig(dbrootConfigList); - - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - - for ( ; pt != dbrootConfigList.end() ; pt++) - { - string volumeNameID = "PMVolumeName" + oam.itoa(*pt); - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt); - string deviceName = oam::UnassignedName; - string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt); - string amazondeviceName = oam::UnassignedName; - - try - { - oam.getSystemConfig( volumeNameID, volumeName); - oam.getSystemConfig( deviceNameID, deviceName); - oam.getSystemConfig( amazonDeviceNameID, amazondeviceName); - } - catch (...) - { - continue; - } - - fprintf(pOutputFile,"Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, %s",*pt,volumeName.c_str(),deviceName.c_str(),amazondeviceName.c_str()); - - } - } - catch (exception& e) - { - cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; - } - - // print un-assigned dbroots - /*DBRootConfigList::iterator pt1 = undbrootlist.begin(); - - for ( ; pt1 != undbrootlist.end() ; pt1++) - { - string volumeNameID = "PMVolumeName" + oam.itoa(*pt1); - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1); - string deviceName = oam::UnassignedName; - string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt1); - string amazondeviceName = oam::UnassignedName; - - try - { - oam.getSystemConfig( volumeNameID, volumeName); - oam.getSystemConfig( deviceNameID, deviceName); - oam.getSystemConfig( amazonDeviceNameID, amazondeviceName); - } - catch (...) - { - continue; - } - - fprintf(pOutputFile,"Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, %s",*pt1,volumeName.c_str(),deviceName.c_str(),amazondeviceName.c_str()); - }*/ - } - - string DataRedundancyConfig; - int DataRedundancyCopies; - string DataRedundancyStorageType; - - try - { - oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); - oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); - oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); - } - catch (...) {} - - if ( DataRedundancyConfig == "y" ) - { - fprintf(pOutputFile,"\nData Redundant Configuration\n\n"); - fprintf(pOutputFile,"Copies Per DBroot = %i",DataRedundancyCopies); - - oamModuleInfo_t st; - string moduleType; - - try - { - st = oam.getModuleInfo(); - moduleType = boost::get<1>(st); - } - catch (...) {} - - if ( moduleType != "pm") - return; - - try - { - DBRootConfigList dbrootConfigList; - oam.getSystemDbrootConfig(dbrootConfigList); - - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - - for ( ; pt != dbrootConfigList.end() ; pt++) - { - fprintf(pOutputFile,"DBRoot #%u has copies on PMs = ", *pt); - - string pmList = ""; - - try - { - string errmsg; - //oam.glusterctl(oam::GLUSTER_WHOHAS, oam.itoa(*pt), pmList, errmsg); - } - catch (...) - {} - - boost::char_separator sep(" "); - boost::tokenizer< boost::char_separator > tokens(pmList, sep); - - for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); - it != tokens.end(); - ++it) - { - fprintf(pOutputFile,"%s ",(*it).c_str()); - } - - fprintf(pOutputFile,"\n"); - } - - fprintf(pOutputFile,"\n"); - } - catch (exception& e) - { - cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; - } - } - } - catch (exception& e) - { - cout << endl << "**** getStorageConfig Failed : " << e.what() << endl; - } -} - -void getStorageStatus(FILE * pOutputFile) -{ - Oam oam; - - fprintf(pOutputFile,"System External DBRoot Storage Statuses\n\n"); - fprintf(pOutputFile,"Component Status Last Status Change\n"); - fprintf(pOutputFile,"------------ -------------------------- ------------------------\n"); - - /*try - { - oam.getSystemStatus(systemstatus, false); - - if ( systemstatus.systemdbrootstatus.dbrootstatus.size() == 0 ) - { - fprintf(pOutputFile," No External DBRoot Storage Configured\n\n"); - return; - } - - for ( unsigned int i = 0 ; i < systemstatus.systemdbrootstatus.dbrootstatus.size(); i++) - { - if ( systemstatus.systemdbrootstatus.dbrootstatus[i].Name.empty() ) - // end of list - break; - - int state = systemstatus.systemdbrootstatus.dbrootstatus[i].OpState; - string stime = systemstatus.systemdbrootstatus.dbrootstatus[i].StateChangeDate ; - stime = stime.substr (0, 24); - fprintf(pOutputFile,"DBRoot%s%-29s%-24s\n", - systemstatus.systemdbrootstatus.dbrootstatus[i].Name.c_str(), - oamState[state].c_str(), - stime.c_str()); - } - fprintf(pOutputFile,"\n"); - } - catch (exception& e) - { - cout << endl << "**** getSystemStatus Failed = " << e.what() << endl; }*/ + fprintf(pOutputFile, "\n"); + + // um volumes + if (cloud == "amazon" && boost::get<3>(t) == "external") + { + ModuleTypeConfig moduletypeconfig; + oam.getSystemConfig("um", moduletypeconfig); + + for (int id = 1; id < moduletypeconfig.ModuleCount + 1; id++) + { + string volumeNameID = "UMVolumeName" + oam.itoa(id); + string volumeName = oam::UnassignedName; + string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id); + string deviceName = oam::UnassignedName; + + try + { + oam.getSystemConfig(volumeNameID, volumeName); + oam.getSystemConfig(deviceNameID, deviceName); + } + catch (...) + { + } + + fprintf(pOutputFile, "Amazon EC2 Volume Name/Device Name for 'um%i': %s, %s", id, volumeName.c_str(), + deviceName.c_str()); + } + } + + // pm volumes + if (cloud == "amazon" && boost::get<0>(t) == "external") + { + fprintf(pOutputFile, "\n"); + + DBRootConfigList dbrootConfigList; + + try + { + oam.getSystemDbrootConfig(dbrootConfigList); + + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + + for (; pt != dbrootConfigList.end(); pt++) + { + string volumeNameID = "PMVolumeName" + oam.itoa(*pt); + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt); + string deviceName = oam::UnassignedName; + string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt); + string amazondeviceName = oam::UnassignedName; + + try + { + oam.getSystemConfig(volumeNameID, volumeName); + oam.getSystemConfig(deviceNameID, deviceName); + oam.getSystemConfig(amazonDeviceNameID, amazondeviceName); + } + catch (...) + { + continue; + } + + fprintf(pOutputFile, + "Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, %s", *pt, + volumeName.c_str(), deviceName.c_str(), amazondeviceName.c_str()); + } + } + catch (exception& e) + { + cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; + } + + // print un-assigned dbroots + /*DBRootConfigList::iterator pt1 = undbrootlist.begin(); + + for ( ; pt1 != undbrootlist.end() ; pt1++) + { + string volumeNameID = "PMVolumeName" + oam.itoa(*pt1); + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1); + string deviceName = oam::UnassignedName; + string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt1); + string amazondeviceName = oam::UnassignedName; + + try + { + oam.getSystemConfig( volumeNameID, volumeName); + oam.getSystemConfig( deviceNameID, deviceName); + oam.getSystemConfig( amazonDeviceNameID, amazondeviceName); + } + catch (...) + { + continue; + } + + fprintf(pOutputFile,"Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, + %s",*pt1,volumeName.c_str(),deviceName.c_str(),amazondeviceName.c_str()); + }*/ + } + string DataRedundancyConfig; int DataRedundancyCopies; string DataRedundancyStorageType; try { - oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); - oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); - oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); + oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); + } + catch (...) + { } - catch (...) {} + if (DataRedundancyConfig == "y") + { + fprintf(pOutputFile, "\nData Redundant Configuration\n\n"); + fprintf(pOutputFile, "Copies Per DBroot = %i", DataRedundancyCopies); + + oamModuleInfo_t st; + string moduleType; + + try + { + st = oam.getModuleInfo(); + moduleType = boost::get<1>(st); + } + catch (...) + { + } + + if (moduleType != "pm") + return; + + try + { + DBRootConfigList dbrootConfigList; + oam.getSystemDbrootConfig(dbrootConfigList); + + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + + for (; pt != dbrootConfigList.end(); pt++) + { + fprintf(pOutputFile, "DBRoot #%u has copies on PMs = ", *pt); + + string pmList = ""; + + try + { + string errmsg; + // oam.glusterctl(oam::GLUSTER_WHOHAS, oam.itoa(*pt), pmList, errmsg); + } + catch (...) + { + } + + boost::char_separator sep(" "); + boost::tokenizer > tokens(pmList, sep); + + for (boost::tokenizer >::iterator it = tokens.begin(); + it != tokens.end(); ++it) + { + fprintf(pOutputFile, "%s ", (*it).c_str()); + } + + fprintf(pOutputFile, "\n"); + } + + fprintf(pOutputFile, "\n"); + } + catch (exception& e) + { + cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; + } + } + } + catch (exception& e) + { + cout << endl << "**** getStorageConfig Failed : " << e.what() << endl; + } } +void getStorageStatus(FILE* pOutputFile) +{ + Oam oam; + + fprintf(pOutputFile, "System External DBRoot Storage Statuses\n\n"); + fprintf(pOutputFile, "Component Status Last Status Change\n"); + fprintf(pOutputFile, "------------ -------------------------- ------------------------\n"); + + /*try + { + oam.getSystemStatus(systemstatus, false); + + if ( systemstatus.systemdbrootstatus.dbrootstatus.size() == 0 ) + { + fprintf(pOutputFile," No External DBRoot Storage Configured\n\n"); + return; + } + + for ( unsigned int i = 0 ; i < systemstatus.systemdbrootstatus.dbrootstatus.size(); i++) + { + if ( systemstatus.systemdbrootstatus.dbrootstatus[i].Name.empty() ) + // end of list + break; + + int state = systemstatus.systemdbrootstatus.dbrootstatus[i].OpState; + string stime = systemstatus.systemdbrootstatus.dbrootstatus[i].StateChangeDate ; + stime = stime.substr (0, 24); + fprintf(pOutputFile,"DBRoot%s%-29s%-24s\n", + systemstatus.systemdbrootstatus.dbrootstatus[i].Name.c_str(), + oamState[state].c_str(), + stime.c_str()); + } + fprintf(pOutputFile,"\n"); + } + catch (exception& e) + { + cout << endl << "**** getSystemStatus Failed = " << e.what() << endl; + }*/ + + string DataRedundancyConfig; + int DataRedundancyCopies; + string DataRedundancyStorageType; + + try + { + oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); + } + catch (...) + { + } +} /******************************************************************** * * checkLogStatus - Check for a phrase in a log file and return status * ********************************************************************/ -bool checkLogStatus(std::string fileName, std::string phrase ) +bool checkLogStatus(std::string fileName, std::string phrase) { - ifstream file (fileName.c_str()); + ifstream file(fileName.c_str()); - if (!file.is_open()) - { - return false; - } - - string buf; - - while (getline(file, buf)) - { - string::size_type pos = buf.find(phrase, 0); - - if (pos != string::npos) - //found phrase - return true; - } - - if (file.bad()) - { - return false; - } - - file.close(); + if (!file.is_open()) + { return false; + } + + string buf; + + while (getline(file, buf)) + { + string::size_type pos = buf.find(phrase, 0); + + if (pos != string::npos) + // found phrase + return true; + } + + if (file.bad()) + { + return false; + } + + file.close(); + return false; } /****************************************************************************************** @@ -598,26 +609,27 @@ bool checkLogStatus(std::string fileName, std::string phrase ) ******************************************************************************************/ string getIPAddress(string hostName) { - static uint32_t my_bind_addr; - struct hostent* ent; - string IPAddr = ""; - Oam oam; + static uint32_t my_bind_addr; + struct hostent* ent; + string IPAddr = ""; + Oam oam; - ent = gethostbyname(hostName.c_str()); + ent = gethostbyname(hostName.c_str()); - if (ent != 0) - { - my_bind_addr = (uint32_t) ((in_addr*)ent->h_addr_list[0])->s_addr; + if (ent != 0) + { + my_bind_addr = (uint32_t)((in_addr*)ent->h_addr_list[0])->s_addr; - uint8_t split[4]; - uint32_t ip = my_bind_addr; - split[0] = (ip & 0xff000000) >> 24; - split[1] = (ip & 0x00ff0000) >> 16; - split[2] = (ip & 0x0000ff00) >> 8; - split[3] = (ip & 0x000000ff); + uint8_t split[4]; + uint32_t ip = my_bind_addr; + split[0] = (ip & 0xff000000) >> 24; + split[1] = (ip & 0x00ff0000) >> 16; + split[2] = (ip & 0x0000ff00) >> 8; + split[3] = (ip & 0x000000ff); - IPAddr = oam.itoa(split[3]) + "." + oam.itoa(split[2]) + "." + oam.itoa(split[1]) + "." + oam.itoa(split[0]); - } + IPAddr = + oam.itoa(split[3]) + "." + oam.itoa(split[2]) + "." + oam.itoa(split[1]) + "." + oam.itoa(split[0]); + } - return IPAddr; + return IPAddr; } diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.h b/oamapps/columnstoreSupport/mcsSupportUtil.h index 9a8374068..86c8f17fc 100644 --- a/oamapps/columnstoreSupport/mcsSupportUtil.h +++ b/oamapps/columnstoreSupport/mcsSupportUtil.h @@ -27,10 +27,10 @@ #include "configcpp.h" #include "liboamcpp.h" -void getSystemNetworkConfig(FILE * pOutputFile); -void getModuleTypeConfig(FILE * pOutputFile); -void getStorageConfig(FILE * pOutputFile); -void getStorageStatus(FILE * pOutputFile); +void getSystemNetworkConfig(FILE* pOutputFile); +void getModuleTypeConfig(FILE* pOutputFile); +void getStorageConfig(FILE* pOutputFile); +void getStorageStatus(FILE* pOutputFile); bool checkLogStatus(std::string filename, std::string phase); std::string getIPAddress(std::string hostName); diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index e35a36d57..d46bd773a 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -17,16 +17,15 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: mycnfUpgrade.cpp 64 2006-10-12 22:21:51Z dhill $ -* -* -* -******************************************************************************************/ + * $Id: mycnfUpgrade.cpp 64 2006-10-12 22:21:51Z dhill $ + * + * + * + ******************************************************************************************/ /** * @file */ - #include #include #include @@ -56,232 +55,233 @@ using namespace std; using namespace oam; - -/* MCOL-1844. On an upgrade, the user may have customized options in their old +/* MCOL-1844. On an upgrade, the user may have customized options in their old * myCnf-include-args.text file. Merge it with the packaged version, and then process as we * have before. */ -string rtrim(const string &in) { - string::const_reverse_iterator rbegin = in.rbegin(); - while (rbegin != in.rend() && isspace(*rbegin)) - ++rbegin; - return string(in.begin(), rbegin.base()); +string rtrim(const string& in) +{ + string::const_reverse_iterator rbegin = in.rbegin(); + while (rbegin != in.rend() && isspace(*rbegin)) + ++rbegin; + return string(in.begin(), rbegin.base()); } -void mergeMycnfIncludeArgs() +void mergeMycnfIncludeArgs() { - string userArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text.rpmsave"; - string packagedArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text"; - ifstream userArgs(userArgsFilename.c_str()); - fstream packagedArgs(packagedArgsFilename.c_str(), ios::in); + string userArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text.rpmsave"; + string packagedArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text"; + ifstream userArgs(userArgsFilename.c_str()); + fstream packagedArgs(packagedArgsFilename.c_str(), ios::in); - if (!userArgs || !packagedArgs) - return; + if (!userArgs || !packagedArgs) + return; - // de-dup the args and comments in both files - set argMerger; - set comments; - string line; - while (getline(packagedArgs, line)) { - line = rtrim(line); - if (line[0] == '#') - comments.insert(line); - else if (line.size() > 0) - argMerger.insert(line); - } - while (getline(userArgs, line)) { - line = rtrim(line); - if (line[0] == '#') - comments.insert(line); - else if (line.size() > 0) - argMerger.insert(line); - } - userArgs.close(); - packagedArgs.close(); + // de-dup the args and comments in both files + set argMerger; + set comments; + string line; + while (getline(packagedArgs, line)) + { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + while (getline(userArgs, line)) + { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + userArgs.close(); + packagedArgs.close(); - // write the merged version, comments first. They'll get ordered - // alphabetically but, meh. - packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc); - for (set::iterator it = comments.begin(); it != comments.end(); it++) - packagedArgs << *it << endl; - for (set::iterator it = argMerger.begin(); it != argMerger.end(); it++) - packagedArgs << *it << endl; - packagedArgs.close(); + // write the merged version, comments first. They'll get ordered + // alphabetically but, meh. + packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc); + for (set::iterator it = comments.begin(); it != comments.end(); it++) + packagedArgs << *it << endl; + for (set::iterator it = argMerger.begin(); it != argMerger.end(); it++) + packagedArgs << *it << endl; + packagedArgs.close(); } int main(int argc, char* argv[]) { - Oam oam; + Oam oam; - //check for port argument - string mysqlPort; + // check for port argument + string mysqlPort; - if (argc > 1) - { - mysqlPort = argv[1]; + if (argc > 1) + { + mysqlPort = argv[1]; - // set mysql password - oam.changeMyCnf( "port", mysqlPort ); - exit (0); - } + // set mysql password + oam.changeMyCnf("port", mysqlPort); + exit(0); + } - //my.cnf file - string mycnfFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; - ifstream mycnffile (mycnfFile.c_str()); + // my.cnf file + string mycnfFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; + ifstream mycnffile(mycnfFile.c_str()); - if (!mycnffile) - { - cerr << "mycnfUpgrade - columnstore.cnf file not found: " << mycnfFile << endl; - exit (1); - } + if (!mycnffile) + { + cerr << "mycnfUpgrade - columnstore.cnf file not found: " << mycnfFile << endl; + exit(1); + } - //my.cnf.rpmsave file - string mycnfsaveFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf.rpmsave"; - ifstream mycnfsavefile (mycnfsaveFile.c_str()); + // my.cnf.rpmsave file + string mycnfsaveFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf.rpmsave"; + ifstream mycnfsavefile(mycnfsaveFile.c_str()); - if (!mycnfsavefile) - { - cerr << "mycnfUpgrade - columnstore.cnf.rpmsave file not found: " << mycnfsaveFile << endl; - exit (1); - } + if (!mycnfsavefile) + { + cerr << "mycnfUpgrade - columnstore.cnf.rpmsave file not found: " << mycnfsaveFile << endl; + exit(1); + } - // MCOL-1844. The user may have added options to their myCnf-include-args file. Merge - // myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text - mergeMycnfIncludeArgs(); + // MCOL-1844. The user may have added options to their myCnf-include-args file. Merge + // myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text + mergeMycnfIncludeArgs(); - //include arguments file - string includeFile = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text"; - ifstream includefile (includeFile.c_str()); + // include arguments file + string includeFile = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text"; + ifstream includefile(includeFile.c_str()); - if (!includefile) - { - cerr << "mycnfUpgrade - columnstore.cnf include argument file not found: " << includeFile << endl; - exit (1); - } + if (!includefile) + { + cerr << "mycnfUpgrade - columnstore.cnf include argument file not found: " << includeFile << endl; + exit(1); + } - //exclude arguments file - string excludeFile = std::string(MCSSUPPORTDIR) + "/myCnf-exclude-args.text"; - ifstream excludefile (excludeFile.c_str()); + // exclude arguments file + string excludeFile = std::string(MCSSUPPORTDIR) + "/myCnf-exclude-args.text"; + ifstream excludefile(excludeFile.c_str()); - if (!excludefile) - { - cerr << "mycnfUpgrade - columnstore.cnf exclude argument file not found: " << endl; - exit (1); - } + if (!excludefile) + { + cerr << "mycnfUpgrade - columnstore.cnf exclude argument file not found: " << endl; + exit(1); + } - //go though include list + // go though include list + char line[200]; + string includeArg; + + while (includefile.getline(line, 200)) + { + includeArg = line; + + boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); + // see if in columnstore.cnf.rpmsave + ifstream mycnfsavefile(mycnfsaveFile.c_str()); char line[200]; - string includeArg; + string oldbuf; - while (includefile.getline(line, 200)) + while (mycnfsavefile.getline(line, 200)) { - includeArg = line; + oldbuf = line; - boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); - //see if in columnstore.cnf.rpmsave - ifstream mycnfsavefile (mycnfsaveFile.c_str()); - char line[200]; - string oldbuf; - - while (mycnfsavefile.getline(line, 200)) + if (boost::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx)) + { + // found in columnstore.cnf.rpmsave, check if this is commented out + if (line[0] != '#') { - oldbuf = line; + // no, check in columnstore.cnf and replace if exist or add if it doesn't - if ( boost::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx) ) + ifstream mycnffile(mycnfFile.c_str()); + vector lines; + char line1[200]; + string newbuf; + bool updated = false; + + while (mycnffile.getline(line1, 200)) + { + newbuf = line1; + + if (boost::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx)) { - //found in columnstore.cnf.rpmsave, check if this is commented out - if ( line[0] != '#' ) - { - // no, check in columnstore.cnf and replace if exist or add if it doesn't - - ifstream mycnffile (mycnfFile.c_str()); - vector lines; - char line1[200]; - string newbuf; - bool updated = false; - - while (mycnffile.getline(line1, 200)) - { - newbuf = line1; - - if ( boost::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx) ) - { - newbuf = oldbuf; - cout << "Updated argument: " << includeArg << endl; - updated = true; - } - - //output to temp file - lines.push_back(newbuf); - } - - //write out a new columnstore.cnf - mycnffile.close(); - unlink (mycnfFile.c_str()); - ofstream newFile (mycnfFile.c_str()); - - //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - close(fd); - - if (!updated) - { - //not found, so add - ifstream mycnffile (mycnfFile.c_str()); - vector lines; - char line1[200]; - string newbuf; - - while (mycnffile.getline(line1, 200)) - { - newbuf = line1; - boost::regex mysqldSectionRegEx("\\[mysqld\\]"); - - if ( boost::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx) ) - { - lines.push_back(newbuf); - newbuf = oldbuf; - cout << "Added argument: " << includeArg << endl; - } - - //output to temp file - lines.push_back(newbuf); - } - - //write out a new columnstore.cnf - mycnffile.close(); - unlink (mycnfFile.c_str()); - ofstream newFile (mycnfFile.c_str()); - - //create new file - int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0666); - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - close(fd); - break; - } - } + newbuf = oldbuf; + cout << "Updated argument: " << includeArg << endl; + updated = true; } + + // output to temp file + lines.push_back(newbuf); + } + + // write out a new columnstore.cnf + mycnffile.close(); + unlink(mycnfFile.c_str()); + ofstream newFile(mycnfFile.c_str()); + + // create new file + int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0644); + + copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); + newFile.close(); + + close(fd); + + if (!updated) + { + // not found, so add + ifstream mycnffile(mycnfFile.c_str()); + vector lines; + char line1[200]; + string newbuf; + + while (mycnffile.getline(line1, 200)) + { + newbuf = line1; + boost::regex mysqldSectionRegEx("\\[mysqld\\]"); + + if (boost::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx)) + { + lines.push_back(newbuf); + newbuf = oldbuf; + cout << "Added argument: " << includeArg << endl; + } + + // output to temp file + lines.push_back(newbuf); + } + + // write out a new columnstore.cnf + mycnffile.close(); + unlink(mycnfFile.c_str()); + ofstream newFile(mycnfFile.c_str()); + + // create new file + int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0666); + + copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); + newFile.close(); + + close(fd); + break; + } } + } } + } - string USER = "mysql"; - - char* p = getenv("USER"); + string USER = "mysql"; - if (p && *p) - USER = p; + char* p = getenv("USER"); - string cmd = "chown " + USER + ":" + USER + " " + mycnfFile; - system(cmd.c_str()); + if (p && *p) + USER = p; - exit (0); + string cmd = "chown " + USER + ":" + USER + " " + mycnfFile; + system(cmd.c_str()); + + exit(0); } // vim:ts=4 sw=4: - diff --git a/oamapps/replayTransactionLog/replaytransactionlog.cpp b/oamapps/replayTransactionLog/replaytransactionlog.cpp index c23aadcb3..0a662510b 100644 --- a/oamapps/replayTransactionLog/replaytransactionlog.cpp +++ b/oamapps/replayTransactionLog/replaytransactionlog.cpp @@ -27,134 +27,130 @@ using namespace oam; #include "replaytxnlog.h" namespace { - void usage(char* prog) { + cout << endl; + cout << "Usage: " << prog << " [options]" << endl; - cout << endl; - cout << "Usage: " << prog << " [options]" << endl; + cout << endl; + cout << "This utility can be used after a backup is restored to report transactions that " << endl; + cout << "occurred after the backup. It begins with the first transaction that was committed " << endl; + cout << "after the backup and reports DDL and DML statements as well as imports." << endl; + cout << endl; - cout << endl; - cout << "This utility can be used after a backup is restored to report transactions that " << endl; - cout << "occurred after the backup. It begins with the first transaction that was committed " << endl; - cout << "after the backup and reports DDL and DML statements as well as imports." << endl; - cout << endl; + cout << "Options:" << endl; + /* + cout << "-u Database user id." << endl << endl; - cout << "Options:" << endl; - /* - cout << "-u Database user id." << endl << endl; + cout << "-p Password." << endl << endl; + */ + cout << "-d Stop date and time as mm/dd/yy@hh:mm:ss or 'Now'." << endl; + cout << " Only transactions committed before this date and time will be reported." << endl; + cout << " The current date and time will be used if 'Now'." << endl << endl; - cout << "-p Password." << endl << endl; - */ - cout << "-d Stop date and time as mm/dd/yy@hh:mm:ss or 'Now'." << endl; - cout << " Only transactions committed before this date and time will be reported." << endl; - cout << " The current date and time will be used if 'Now'." << endl << endl; + /* + cout << "-i Ignore bulk load log entries." << endl; + cout << " The program will pause and prompt at bulk load entries by default." << endl << + endl; - /* - cout << "-i Ignore bulk load log entries." << endl; - cout << " The program will pause and prompt at bulk load entries by default." << endl << endl; + cout << "-e Report mode. The sql statements will be displayed to the console only. No" << + endl; cout << " transactions will be processed. The user and password will be ignored." << + endl << endl; + */ - cout << "-e Report mode. The sql statements will be displayed to the console only. No" << endl; - cout << " transactions will be processed. The user and password will be ignored." << endl << endl; - */ - - cout << "-h Display this help." << endl << endl; + cout << "-h Display this help." << endl << endl; } bool isRunningOnPm() { - Oam oam; - oamModuleInfo_t t; - string moduleType; - int installType = -1; + Oam oam; + oamModuleInfo_t t; + string moduleType; + int installType = -1; - char* csc_ident = getenv("CALPONT_CSC_IDENT"); + char* csc_ident = getenv("CALPONT_CSC_IDENT"); - if (csc_ident == 0 || *csc_ident == 0) + if (csc_ident == 0 || *csc_ident == 0) + { + // get local module info valdiate running on a pm + try { - //get local module info valdiate running on a pm - try - { - t = oam.getModuleInfo(); - moduleType = boost::get<1>(t); - installType = boost::get<5>(t); - } - catch (exception& e) - { - moduleType = "pm"; - } + t = oam.getModuleInfo(); + moduleType = boost::get<1>(t); + installType = boost::get<5>(t); } - else - moduleType = csc_ident; - - if ( installType != oam::INSTALL_COMBINE_DM_UM_PM ) + catch (exception& e) { - if ( moduleType != "pm" ) - { - cerr << "Exiting, ReplayTransactionLog can only be run on a performance module (pm)" << endl; - return false; - } + moduleType = "pm"; } + } + else + moduleType = csc_ident; - return true; -} + if (installType != oam::INSTALL_COMBINE_DM_UM_PM) + { + if (moduleType != "pm") + { + cerr << "Exiting, ReplayTransactionLog can only be run on a performance module (pm)" << endl; + return false; + } + } + + return true; } +} // namespace int main(int argc, char** argv) { + string user; + string password; + string stopDate; + bool ignoreBulk = false; + bool reportMode = false; + char c; - string user; - string password; - string stopDate; - bool ignoreBulk = false; - bool reportMode = false; - char c; - - // Invokes member function `int operator ()(void);' - while ((c = getopt(argc, argv, "u:p:d:ihe")) != -1) + // Invokes member function `int operator ()(void);' + while ((c = getopt(argc, argv, "u:p:d:ihe")) != -1) + { + switch (c) { - switch (c) - { - /* - case 'u': - user = optarg; - break; - case 'p': - password = optarg; - break; - */ - case 'd': - stopDate = optarg; - break; + /* + case 'u': + user = optarg; + break; + case 'p': + password = optarg; + break; + */ + case 'd': stopDate = optarg; break; - /* - case 'i': - ignoreBulk = true; - break; - case 'e': - reportMode = true; - break; - */ - case 'h': - usage(argv[0]); - return 0; - break; - - default: - usage(argv[0]); - return 1; - break; - } - } - - if (!isRunningOnPm()) - { + /* + case 'i': + ignoreBulk = true; + break; + case 'e': + reportMode = true; + break; + */ + case 'h': + usage(argv[0]); return 0; + break; + + default: + usage(argv[0]); + return 1; + break; } + } - ReplayTxnLog replayTxnLog(user, password, stopDate, ignoreBulk, reportMode); - replayTxnLog.process(); - + if (!isRunningOnPm()) + { return 0; -} + } + ReplayTxnLog replayTxnLog(user, password, stopDate, ignoreBulk, reportMode); + replayTxnLog.process(); + + return 0; +} diff --git a/oamapps/replayTransactionLog/tdriver.cpp b/oamapps/replayTransactionLog/tdriver.cpp index 4296d048b..809786d7d 100644 --- a/oamapps/replayTransactionLog/tdriver.cpp +++ b/oamapps/replayTransactionLog/tdriver.cpp @@ -36,34 +36,31 @@ int maxTxns = 1000; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } +}; // test suite - void setUp() - { - } - - void tearDown() - { - } -}; // test suite - -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest); +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include int main(int argc, char* argv[]) { - - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/oamapps/sessionWalker/sessionwalker.cpp b/oamapps/sessionWalker/sessionwalker.cpp index 65458f0fb..5f0960128 100644 --- a/oamapps/sessionWalker/sessionwalker.cpp +++ b/oamapps/sessionWalker/sessionwalker.cpp @@ -39,102 +39,98 @@ using namespace messageqcpp; namespace { - void usage() { - cout << "sessionwalker [-d|-h]" << endl << - " -r rollback all transactions found" << endl << - " -h display this help" << endl; + cout << "sessionwalker [-d|-h]" << endl + << " -r rollback all transactions found" << endl + << " -h display this help" << endl; } void rollback(const SessionMonitor::MonSIDTIDEntry& txn) { - VendorDMLStatement dmlStmt("ROLLBACK;", txn.sessionid); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + VendorDMLStatement dmlStmt("ROLLBACK;", txn.sessionid); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - if (pDMLPackage == 0) - { - return; - } + if (pDMLPackage == 0) + { + return; + } - ByteStream bytestream; - pDMLPackage->write(bytestream); - delete pDMLPackage; - MessageQueueClient mq("DMLProc"); + ByteStream bytestream; + pDMLPackage->write(bytestream); + delete pDMLPackage; + MessageQueueClient mq("DMLProc"); - try - { - cout << "sending ROLLBACK for sessionID " << txn.sessionid << endl; - mq.write(bytestream); - bytestream = mq.read(); - } - catch (...) - { - } + try + { + cout << "sending ROLLBACK for sessionID " << txn.sessionid << endl; + mq.write(bytestream); + bytestream = mq.read(); + } + catch (...) + { + } } -} +} // namespace int main(int argc, char** argv) { - bool rflg = false; - opterr = 0; - int c; + bool rflg = false; + opterr = 0; + int c; - while ((c = getopt(argc, argv, "rh")) != EOF) - switch (c) - { - case 'r': - rflg = true; - break; - - case 'h': - usage(); - return 0; - break; - - default: - usage(); - return 1; - break; - } - - vector toTxns; - SessionMonitor* monitor = new SessionMonitor(); - - toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - - vector::iterator iter = toTxns.begin(); - vector::iterator end = toTxns.end(); - - vector tmp; - - while (iter != end) + while ((c = getopt(argc, argv, "rh")) != EOF) + switch (c) { - if ((*iter)->sessionid > 0) - tmp.push_back(*iter); + case 'r': rflg = true; break; - ++iter; + case 'h': + usage(); + return 0; + break; + + default: + usage(); + return 1; + break; } - toTxns.swap(tmp); + vector toTxns; + SessionMonitor* monitor = new SessionMonitor(); - cout << toTxns.size() << " timed out transactions." << endl; + toTxns.clear(); + toTxns = monitor->timedOutTxns(); // get timed out txns - for (unsigned idx = 0; idx < toTxns.size(); idx++) + vector::iterator iter = toTxns.begin(); + vector::iterator end = toTxns.end(); + + vector tmp; + + while (iter != end) + { + if ((*iter)->sessionid > 0) + tmp.push_back(*iter); + + ++iter; + } + + toTxns.swap(tmp); + + cout << toTxns.size() << " timed out transactions." << endl; + + for (unsigned idx = 0; idx < toTxns.size(); idx++) + { + monitor->printTxns(*toTxns[idx]); + + if (rflg) { - monitor->printTxns(*toTxns[idx]); - - if (rflg) - { - rollback(*toTxns[idx]); - } + rollback(*toTxns[idx]); } + } - delete monitor; + delete monitor; - return 0; + return 0; } // vim:ts=4 sw=4: - diff --git a/oamapps/sessionWalker/tdriver.cpp b/oamapps/sessionWalker/tdriver.cpp index 2ddcac2e1..29e1771fd 100644 --- a/oamapps/sessionWalker/tdriver.cpp +++ b/oamapps/sessionWalker/tdriver.cpp @@ -37,163 +37,153 @@ int maxTxns = 1000; class ExecPlanTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ExecPlanTest); - CPPUNIT_TEST_SUITE( ExecPlanTest ); + CPPUNIT_TEST(MonitorTestPlan_1); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST( MonitorTestPlan_1 ); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: + void tearDown() + { + } - void setUp() + int verifyLen; + SessionManager* manager; + SessionManager::TxnID managerTxns[1000]; + + int createTxns(const int& start, const int& end) + { + int first = start; + int last = end; + int newTxns = 0; + + verifyLen = manager->verifySize(); + + for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++) { + managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000); + CPPUNIT_ASSERT(managerTxns[idx].id > 0); + CPPUNIT_ASSERT(managerTxns[idx].valid == true); + verifyLen = manager->verifySize(); + CPPUNIT_ASSERT(verifyLen > 0); + newTxns++; } - void tearDown() + CPPUNIT_ASSERT(newTxns == last - first); + return newTxns; + } + + int closeTxns(const int& start, const int& end) + { + int first = start; + int last = end; + int totalClosed = 0; + + for (int idx = first; idx < last; idx++) { + try + { + SessionManager::TxnID tmp = manager->getTxnID(idx + 1000); + + if (tmp.valid == true) + { + manager->committed(tmp); + CPPUNIT_ASSERT(tmp.valid == false); + totalClosed++; + } + } + catch (exception& e) + { + cerr << e.what() << endl; + continue; + } } - int verifyLen; - SessionManager* manager; - SessionManager::TxnID managerTxns[1000]; + return totalClosed; - int createTxns(const int& start, const int& end) + } // closeTxns + + void MonitorTestPlan_1() + { + int currStartTxn = 0; + int currEndTxn = 5; + int txnCntIncr = 5; + const int sleepTime = 1; + const int iterMax = 1; + vector toTxns; + + manager = new SessionManager(); + // CPPUNIT_ASSERT(manager->verifySize()==0); + + SessionMonitor* monitor = NULL; + + for (int jdx = 0; jdx < iterMax; jdx++) { + // store the current state of the SessionManager + monitor = new SessionMonitor(); + monitor->AgeLimit(sleepTime); + delete monitor; + int idx = 0; + int grpStart = currStartTxn; - int first = start; - int last = end; - int newTxns = 0; + for (idx = 0; idx < 3; idx++) + { + createTxns(currStartTxn, currEndTxn); + // CPPUNIT_ASSERT(manager->verifySize()==(idx+1)*txnCntIncr); - verifyLen = manager->verifySize(); - - for (int idx = first; idx < last && verifyLen < maxNewTxns ; idx++) - { - managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000); - CPPUNIT_ASSERT(managerTxns[idx].id > 0); - CPPUNIT_ASSERT(managerTxns[idx].valid == true); - verifyLen = manager->verifySize(); - CPPUNIT_ASSERT(verifyLen > 0); - newTxns++; - } - - CPPUNIT_ASSERT(newTxns == last - first); - return newTxns; - } - - int closeTxns(const int& start, const int& end) - { - - int first = start; - int last = end; - int totalClosed = 0; - - for (int idx = first; idx < last ; idx++) - { - try - { - SessionManager::TxnID tmp = manager->getTxnID(idx + 1000); - - if (tmp.valid == true) - { - manager->committed(tmp); - CPPUNIT_ASSERT(tmp.valid == false); - totalClosed++; - } - - } - catch (exception& e) - { - cerr << e.what() << endl; - continue; - } - } - - return totalClosed; - - } //closeTxns - - void MonitorTestPlan_1() - { - - int currStartTxn = 0; - int currEndTxn = 5; - int txnCntIncr = 5; - const int sleepTime = 1; - const int iterMax = 1; - vector toTxns; - - manager = new SessionManager(); - //CPPUNIT_ASSERT(manager->verifySize()==0); - - SessionMonitor* monitor = NULL; - - for (int jdx = 0; jdx < iterMax; jdx++) - { - - // store the current state of the SessionManager - monitor = new SessionMonitor(); - monitor->AgeLimit(sleepTime); - delete monitor; - int idx = 0; - int grpStart = currStartTxn; - - for (idx = 0; idx < 3; idx++ ) - { - - createTxns(currStartTxn, currEndTxn); - //CPPUNIT_ASSERT(manager->verifySize()==(idx+1)*txnCntIncr); - - currStartTxn += txnCntIncr; - currEndTxn += txnCntIncr; - sleep(sleepTime + 1); //make sessions time out - - monitor = new SessionMonitor(); // read Monitor data - monitor->AgeLimit(sleepTime); - toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx); - - delete monitor; - } - - int grpEnd = currEndTxn; - monitor = new SessionMonitor(); - monitor->AgeLimit(sleepTime); - closeTxns(grpStart, grpEnd); // close this iteration of txns - //CPPUNIT_ASSERT(manager->verifySize()==0); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == 0); - - delete monitor; - - } - - monitor = new SessionMonitor(); // readload Monitor data - monitor->AgeLimit(sleepTime - 1); + currStartTxn += txnCntIncr; + currEndTxn += txnCntIncr; + sleep(sleepTime + 1); // make sessions time out + monitor = new SessionMonitor(); // read Monitor data + monitor->AgeLimit(sleepTime); toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == 0); - delete monitor; + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx); - //CPPUNIT_ASSERT(manager->verifySize()==0); - delete manager; + delete monitor; + } + + int grpEnd = currEndTxn; + monitor = new SessionMonitor(); + monitor->AgeLimit(sleepTime); + closeTxns(grpStart, grpEnd); // close this iteration of txns + // CPPUNIT_ASSERT(manager->verifySize()==0); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == 0); + + delete monitor; } -}; // test suite + monitor = new SessionMonitor(); // readload Monitor data + monitor->AgeLimit(sleepTime - 1); -CPPUNIT_TEST_SUITE_REGISTRATION( ExecPlanTest); + toTxns.clear(); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == 0); + delete monitor; + + // CPPUNIT_ASSERT(manager->verifySize()==0); + delete manager; + } + +}; // test suite + +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); #include #include int main(int argc, char* argv[]) { - - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } diff --git a/primitives/blockcache/bcTest.cpp b/primitives/blockcache/bcTest.cpp index 4530de00b..e555f7ef0 100644 --- a/primitives/blockcache/bcTest.cpp +++ b/primitives/blockcache/bcTest.cpp @@ -56,10 +56,10 @@ uint64_t noOpCountTot = 0; struct thr_wait_struct { - int predicate; - pthread_mutex_t fMutex; - pthread_cond_t fCond; - vector range_thr; + int predicate; + pthread_mutex_t fMutex; + pthread_cond_t fCond; + vector range_thr; }; typedef thr_wait_struct thr_wait_t; @@ -71,374 +71,380 @@ u_int64_t totBlocks = 0; void* thr_client(void* clientArgs) { - blockCacheClient bc(*BRP); - uint64_t bfound = 0; - uint64_t bnfound = 0; - uint64_t rfound = 0; - uint64_t rnfound = 0; - uint64_t rangeOpCount = 0; - uint64_t blockOpCount = 0; - uint64_t noOpCount = 0; - thr_wait_t* clientWait = (thr_wait_t*)clientArgs; - struct timeval tv, tv2; - uint32_t randstate = 0; - randstate = static_cast(tv.tv_usec); - pthread_mutex_lock(&clientWait->fMutex); - clientWait->predicate++; - pthread_mutex_unlock(&clientWait->fMutex); - vector& range_thr = clientWait->range_thr; + blockCacheClient bc(*BRP); + uint64_t bfound = 0; + uint64_t bnfound = 0; + uint64_t rfound = 0; + uint64_t rnfound = 0; + uint64_t rangeOpCount = 0; + uint64_t blockOpCount = 0; + uint64_t noOpCount = 0; + thr_wait_t* clientWait = (thr_wait_t*)clientArgs; + struct timeval tv, tv2; + uint32_t randstate = 0; + randstate = static_cast(tv.tv_usec); + pthread_mutex_lock(&clientWait->fMutex); + clientWait->predicate++; + pthread_mutex_unlock(&clientWait->fMutex); + vector& range_thr = clientWait->range_thr; - gettimeofday(&tv, NULL); - uint8_t fbData[8192] = {0}; - FileBuffer* fbPtr = NULL; - int ret = 0; - int idx = 0; - uint32_t jdx = 0; - uint32_t l = 0; - uint32_t m = 0; - uint32_t start = 0, max = 0, size = 0; - LBIDRange_v& r = range_thr[0]; + gettimeofday(&tv, NULL); + uint8_t fbData[8192] = {0}; + FileBuffer* fbPtr = NULL; + int ret = 0; + int idx = 0; + uint32_t jdx = 0; + uint32_t l = 0; + uint32_t m = 0; + uint32_t start = 0, max = 0, size = 0; + LBIDRange_v& r = range_thr[0]; - for (idx = 0; idx < fLoops; idx++) + for (idx = 0; idx < fLoops; idx++) + { + for (jdx = 0; jdx < range_thr.size(); jdx++) { - for (jdx = 0; jdx < range_thr.size(); jdx++) + ret = 0; + r = range_thr[jdx]; + + for (l = 0; l < r.size(); l++) + { + start = r[l].start; + size = r[l].size; + max = r[l].start + r[l].size; + + // cout << "readThr " <getData(), ptr->datLen()); + } + else + { + bnfound++; + // cout << "bc fail:" << m << " " <getData(), ptr->datLen()); - } - else - { - bnfound++; - //cout << "bc fail:" << m << " " < 0) - { - avgTot = (bfound + rfound) / elTime; - rangeAvg = (rfound) / elTime; - blkAvg = (bfound) / elTime; - } - else - { - avgTot = bfound + rfound; - rangeAvg = rfound; - blkAvg = bfound; - } + if (elTime > 0) + { + avgTot = (bfound + rfound) / elTime; + rangeAvg = (rfound) / elTime; + blkAvg = (bfound) / elTime; + } + else + { + avgTot = bfound + rfound; + rangeAvg = rfound; + blkAvg = bfound; + } - cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl << - "\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound << - " Blks/Sec Blk " << blkAvg << endl << endl; + cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl + << "\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound << " Blks/Sec Blk " + << blkAvg << endl + << endl; - pthread_mutex_lock(&clientWait->fMutex); - bfoundTot += bfound; - bnfoundTot += bnfound; - rfoundTot += rfound; - rnfoundTot += rnfound; - rangeOpCountTot += rangeOpCount; - blockOpCountTot += blockOpCount; - noOpCountTot += noOpCount; - clientWait->predicate--; - pthread_cond_signal(&clientWait->fCond); - pthread_mutex_unlock(&clientWait->fMutex); + pthread_mutex_lock(&clientWait->fMutex); + bfoundTot += bfound; + bnfoundTot += bnfound; + rfoundTot += rfound; + rnfoundTot += rnfound; + rangeOpCountTot += rangeOpCount; + blockOpCountTot += blockOpCount; + noOpCountTot += noOpCount; + clientWait->predicate--; + pthread_cond_signal(&clientWait->fCond); + pthread_mutex_unlock(&clientWait->fMutex); - return NULL; + return NULL; -} // end thr_client +} // end thr_client void LoadRange(const LBIDRange_v& v, uint32_t& loadCount) { - blockCacheClient bc(*BRP); + blockCacheClient bc(*BRP); - uint32_t rCount = 0; + uint32_t rCount = 0; - for (uint32_t i = 0; i < v.size() ; i++) + for (uint32_t i = 0; i < v.size(); i++) + { + InlineLBIDRange r = {v[i].start, v[i].size}; + + if (r.size + loadCount > cacheSize) + r.size = (r.size + loadCount) - cacheSize; + + if (r.size <= 1024) { - InlineLBIDRange r = {v[i].start, v[i].size}; - - if (r.size + loadCount > cacheSize) - r.size = (r.size + loadCount) - cacheSize; - - if (r.size <= 1024) - { - //cout << "check() " << r.start << " " << r.size << endl; - bc.check(r, ver, rCount ); - loadCount += rCount; - } - - rCount = 0; - + // cout << "check() " << r.start << " " << r.size << endl; + bc.check(r, ver, rCount); + loadCount += rCount; } + rCount = 0; + } } void ReadRange(const LBIDRange_v& v) { - blockCacheClient bc(*BRP); - int found = 0; - int notfound = 0; - int ret = 0; + blockCacheClient bc(*BRP); + int found = 0; + int notfound = 0; + int ret = 0; - for (uint32_t i = 0; i < v.size(); i++) + for (uint32_t i = 0; i < v.size(); i++) + { + const InlineLBIDRange r = {v[i].start, v[i].size}; + FileBuffer fb(-1, -1); + + // cout << "read() " << r.start << " " << r.size << endl; + for (int j = r.start; j < r.start + r.size; j++) { - const InlineLBIDRange r = {v[i].start, v[i].size}; - FileBuffer fb(-1, -1); + if (r.size > 1024) + continue; - //cout << "read() " << r.start << " " << r.size << endl; - for (int j = r.start; j < r.start + r.size; j++) - { - if (r.size > 1024) - continue; + ret = bc.read(j, ver, fb); - ret = bc.read(j, ver, fb); + if (ret) + found++; + else + { + notfound++; + } - if (ret) - found++; - else - { - notfound++; - } - - ret = 0; - } - - totBlocks += found; - totBlocks += notfound; - found = 0; - notfound = 0; + ret = 0; } + totBlocks += found; + totBlocks += notfound; + found = 0; + notfound = 0; + } } -void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) +void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - blockCacheClient bc(*BRP); - bool b; - bc.check(lbid, ver, false, b); + blockCacheClient bc(*BRP); + bool b; + bc.check(lbid, ver, false, b); } void ReadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - static int found = 0, notfound = 0; - uint8_t** d = NULL; //[8192]; - blockCacheClient bc(*BRP); - //FileBuffer fb(-1, -1); - //bc.read(lbid, ver, fb); - int ret = bc.read(lbid, ver, d); + static int found = 0, notfound = 0; + uint8_t** d = NULL; //[8192]; + blockCacheClient bc(*BRP); + // FileBuffer fb(-1, -1); + // bc.read(lbid, ver, fb); + int ret = bc.read(lbid, ver, d); - if (ret) - found++; - else - notfound++; + if (ret) + found++; + else + notfound++; - if ((found + notfound) % 10000 == 0) - cout << "found " << found << " notfound " << notfound << endl; + if ((found + notfound) % 10000 == 0) + cout << "found " << found << " notfound " << notfound << endl; } // int main(int argc, char* argv[]) { + if (argc >= 2) + thr_cnt = atoi(argv[1]); - if (argc >= 2) thr_cnt = atoi(argv[1]); + if (argc >= 3) + fLoops = atoi(argv[2]); - if (argc >= 3) fLoops = atoi(argv[2]); + if (argc >= 4) + cacheSize = atoi(argv[3]); - if (argc >= 4) cacheSize = atoi(argv[3]); + if (thr_cnt <= 0) + thr_cnt = 1; - if (thr_cnt <= 0) thr_cnt = 1; + if (thr_cnt > 1024) + thr_cnt = 1024; - if (thr_cnt > 1024) thr_cnt = 1024; + if (fLoops <= 0) + fLoops = 1; - if (fLoops <= 0) fLoops = 1; + BlockRequestProcessor brp(cacheSize, 4, 16); + LBIDRange_v r; + vector ranges; + DBRM dbrm; + uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid; + struct timeval tv, tv2; + BRP = &brp; - BlockRequestProcessor brp(cacheSize, 4, 16); - LBIDRange_v r; - vector ranges; - DBRM dbrm; - uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid; - struct timeval tv, tv2; - BRP = &brp; + cout << "Reading Ranges " << endl; + extentSize = dbrm.getExtentSize(); + BRM::OID_t oid = 3000; + uint32_t totalExt = 0; - cout << "Reading Ranges " << endl; - extentSize = dbrm.getExtentSize(); - BRM::OID_t oid = 3000; - uint32_t totalExt = 0; + do + { + int ret = dbrm.lookup(oid, r); - do + if (ret == 0 && r.size() > 0) { - int ret = dbrm.lookup(oid, r); + lowlbid = (r[0].start / extentSize) * extentSize; + dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid + dbrm.getHWM(oid, hwm); + lowfbo = fbo - (r[0].start - lowlbid); + highfbo = lowfbo + extentSize; + r[0].start = lowlbid; - if (ret == 0 && r.size() > 0) - { - lowlbid = (r[0].start / extentSize) * extentSize; - dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid - dbrm.getHWM(oid, hwm); - lowfbo = fbo - (r[0].start - lowlbid); - highfbo = lowfbo + extentSize; - r[0].start = lowlbid; + if (hwm < highfbo) + r[0].size = hwm - lowfbo + 1; + else + r[0].size = extentSize; - if (hwm < highfbo) - r[0].size = hwm - lowfbo + 1; - else - r[0].size = extentSize; + for (uint32_t idx = 0; idx < r.size(); idx++) + totalExt += r[idx].size; - for (uint32_t idx = 0; idx < r.size(); idx++) - totalExt += r[idx].size; + ranges.push_back(r); + cout << "."; - ranges.push_back(r); - cout << "."; - - if (ranges.size() % 50 == 0) - cout << endl; - } - - oid++; - } - while ( (r.size() > 0 || oid < 900000) ); - - cout << endl << ranges.size() << " ranges found" << endl; - - gettimeofday(&tv, NULL); - uint32_t blksLoaded = 0; - uint32_t rangesLoaded = 0; - cout << "Loading Ranges " << endl; - - for (uint32_t i = 0; i < ranges.size() && blksLoaded < (cacheSize - 1024); i++) - { - LoadRange(ranges[i], blksLoaded); - rangesLoaded++; - cout << "."; - - if (i % 50 == 0 && i) - cout << endl; + if (ranges.size() % 50 == 0) + cout << endl; } - cout << endl; + oid++; + } while ((r.size() > 0 || oid < 900000)); - gettimeofday(&tv2, NULL); - cout << endl << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec << endl; + cout << endl << ranges.size() << " ranges found" << endl; - while (ranges.size() >= rangesLoaded) ranges.pop_back(); + gettimeofday(&tv, NULL); + uint32_t blksLoaded = 0; + uint32_t rangesLoaded = 0; + cout << "Loading Ranges " << endl; + for (uint32_t i = 0; i < ranges.size() && blksLoaded < (cacheSize - 1024); i++) + { + LoadRange(ranges[i], blksLoaded); + rangesLoaded++; + cout << "."; + + if (i % 50 == 0 && i) + cout << endl; + } + + cout << endl; + + gettimeofday(&tv2, NULL); + cout << endl + << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec + << endl; + + while (ranges.size() >= rangesLoaded) ranges.pop_back(); + ranges.pop_back(); + #ifdef BLAH - for (uint32_t i = 0; i < ranges; i++) - ReadRange(ranges[i]); + for (uint32_t i = 0; i < ranges; i++) + ReadRange(ranges[i]); - for (uint32_t i = 0; i < ranges.size(); i++) + for (uint32_t i = 0; i < ranges.size(); i++) + { + LBIDRange_v rv = ranges[i]; + + for (uint32_t j = 0; j < rv.size(); j++) { - LBIDRange_v rv = ranges[i]; + const InlineLBIDRange l = {rv[j].start, rv[j].size}; - for (uint32_t j = 0; j < rv.size(); j++) - { - const InlineLBIDRange l = {rv[j].start, rv[j].size}; - - for (uint32_t k = l.start; k < l.start + l.size; k++) - { - LoadLbid(k, ver); - ReadLbid(k, ver); - } - } + for (uint32_t k = l.start; k < l.start + l.size; k++) + { + LoadLbid(k, ver); + ReadLbid(k, ver); + } } + } #endif - pthread_t thr_id[thr_cnt]; - thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges}; + pthread_t thr_id[thr_cnt]; + thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges}; - //start threads running - cout << "Starting reader threads" << endl; - gettimeofday(&tv, NULL); - memset(thr_id, 0, thr_cnt * (sizeof(pthread_t))); + // start threads running + cout << "Starting reader threads" << endl; + gettimeofday(&tv, NULL); + memset(thr_id, 0, thr_cnt * (sizeof(pthread_t))); - for (int i = 0; i < thr_cnt; i++) - { - pthread_create(&thr_id[i], NULL, thr_client, &thr_wait); - } + for (int i = 0; i < thr_cnt; i++) + { + pthread_create(&thr_id[i], NULL, thr_client, &thr_wait); + } - // waiting until all threads have indicated completion - pthread_mutex_lock(&thr_wait.fMutex); + // waiting until all threads have indicated completion + pthread_mutex_lock(&thr_wait.fMutex); - while (thr_wait.predicate > 0) - { - pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex); - } + while (thr_wait.predicate > 0) + { + pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex); + } - pthread_mutex_unlock(&thr_wait.fMutex); + pthread_mutex_unlock(&thr_wait.fMutex); - // join threads back to main - for (int i = 0; i < thr_cnt; i++) - { - pthread_join(thr_id[i], NULL); - } + // join threads back to main + for (int i = 0; i < thr_cnt; i++) + { + pthread_join(thr_id[i], NULL); + } - gettimeofday(&tv2, NULL); - time_t tm = time(0); - char t[50]; - ctime_r(&tm, t); - t[strlen(t) - 1] = 0; - uint32_t elTime = tv2.tv_sec - tv.tv_sec; - //uint64_t total = bfoundTot + rfoundTot; - uint64_t avgTot = 0; - uint64_t rangeAvg = 0; - uint64_t blkAvg = 0; + gettimeofday(&tv2, NULL); + time_t tm = time(0); + char t[50]; + ctime_r(&tm, t); + t[strlen(t) - 1] = 0; + uint32_t elTime = tv2.tv_sec - tv.tv_sec; + // uint64_t total = bfoundTot + rfoundTot; + uint64_t avgTot = 0; + uint64_t rangeAvg = 0; + uint64_t blkAvg = 0; - if (elTime > 0) - { - avgTot = (bfoundTot + rfoundTot) / elTime; - rangeAvg = (rfoundTot) / elTime; - blkAvg = (bfoundTot) / elTime; - } - else - { - avgTot = bfoundTot + rfoundTot; - rangeAvg = rfoundTot; - blkAvg = bfoundTot; - } + if (elTime > 0) + { + avgTot = (bfoundTot + rfoundTot) / elTime; + rangeAvg = (rfoundTot) / elTime; + blkAvg = (bfoundTot) / elTime; + } + else + { + avgTot = bfoundTot + rfoundTot; + rangeAvg = rfoundTot; + blkAvg = bfoundTot; + } - cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl << - "\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot << - //"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl << - //"\tNoOp: c " << noOpCountTot << " Total " << total << endl << - //"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl; - " Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl << endl; + cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl + << "\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot << + //"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl << + //"\tNoOp: c " << noOpCountTot << " Total " << total << endl << + //"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt + //<< endl << endl; + " Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl + << endl; + return 0; - return 0; - -} // end main +} // end main diff --git a/primitives/blockcache/blockcacheclient.cpp b/primitives/blockcache/blockcacheclient.cpp index dce9c840d..ca0bbf118 100644 --- a/primitives/blockcache/blockcacheclient.cpp +++ b/primitives/blockcache/blockcacheclient.cpp @@ -28,5 +28,4 @@ namespace dbbc { - } diff --git a/primitives/blockcache/blockcacheclient.h b/primitives/blockcache/blockcacheclient.h index 7560a9d9f..ccdb5b161 100644 --- a/primitives/blockcache/blockcacheclient.h +++ b/primitives/blockcache/blockcacheclient.h @@ -29,7 +29,7 @@ #include "brmtypes.h" /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** @@ -40,141 +40,142 @@ namespace dbbc { - class blockCacheClient { -public: + public: + /** + * @brief ctor requires reference to BlockRequestProcessor object + **/ + blockCacheClient(BlockRequestProcessor& brp) : fBCCBrp(&brp) + { + } - /** - * @brief ctor requires reference to BlockRequestProcessor object - **/ - blockCacheClient(BlockRequestProcessor& brp) : fBCCBrp(&brp) {} + /** + * @brief dtor + **/ + virtual ~blockCacheClient() + { + } - /** - * @brief dtor - **/ - virtual ~blockCacheClient() {} + /** + * @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache. + **/ + inline void check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, + bool& wasBlockInCache) + { + fBCCBrp->check(lbid, ver, txn, flg, compType, wasBlockInCache); + } - /** - * @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache. - **/ - inline void check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache) - { - fBCCBrp->check(lbid, ver, txn, flg, compType, wasBlockInCache); - } + /** + * @brief verify all Disk Blocks for the LBID range are loaded into the Cache + **/ + inline void check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, + const int compType, uint32_t& rCount) + { + fBCCBrp->check(range, ver, txn, compType, rCount); + } - /** - * @brief verify all Disk Blocks for the LBID range are loaded into the Cache - **/ - inline void check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType, - uint32_t& rCount) - { - fBCCBrp->check(range, ver, txn, compType, rCount); - } + inline FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver, bool flg) + { + return fBCCBrp->getBlockPtr(lbid, ver, flg); + } - inline FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver, bool flg) - { - return fBCCBrp->getBlockPtr(lbid, ver, flg); - } + /** + * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache + **/ + inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) + { + return fBCCBrp->read(lbid, ver, fb); + } - /** - * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache - **/ - inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) - { - return fBCCBrp->read(lbid, ver, fb); - } + /** + * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache + **/ + inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) + { + return fBCCBrp->read(lbid, ver, bufferPtr); + } - /** - * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache - **/ - inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) - { - return fBCCBrp->read(lbid, ver, bufferPtr); - } + inline int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, const BRM::VER_t txn, + const int compType, void* bufferPtr, bool flg, bool& wasCached, + bool* wasVersioned = NULL, bool insertIntoCache = true, bool readFromCache = true) + { + return fBCCBrp->getBlock(lbid, ver, txn, compType, bufferPtr, flg, wasCached, wasVersioned, + insertIntoCache, readFromCache); + } - inline int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType, - void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned = NULL, bool insertIntoCache = true, - bool readFromCache = true) - { - return fBCCBrp->getBlock(lbid, ver, txn, compType, bufferPtr, flg, wasCached, wasVersioned, insertIntoCache, - readFromCache); - } + inline int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** bufferPtrs, + bool* wasCached, uint32_t blockCount) + { + return fBCCBrp->getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount); + } - inline int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** bufferPtrs, - bool* wasCached, uint32_t blockCount) - { - return fBCCBrp->getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount); - } + inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) + { + return fBCCBrp->exists(lbid, ver); + } - inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) - { - return fBCCBrp->exists(lbid, ver); - } + /** + * @brief flush the cache + **/ + inline void flushCache() + { + fBCCBrp->flushCache(); + } - /** - * @brief flush the cache - **/ - inline void flushCache() - { - fBCCBrp->flushCache(); - } + /** + * @brief flush one LBID@Ver from the cache + **/ + inline void flushOne(const BRM::LBID_t& lbid, const BRM::VER_t& ver) + { + fBCCBrp->flushOne(lbid, ver); + } - /** - * @brief flush one LBID@Ver from the cache - **/ - inline void flushOne(const BRM::LBID_t& lbid, const BRM::VER_t& ver) - { - fBCCBrp->flushOne(lbid, ver); - } + /** + * @brief flush specific LBID/version pairs from the cache + **/ + inline void flushMany(const LbidAtVer* laVptr, uint32_t cnt) + { + fBCCBrp->flushMany(laVptr, cnt); + } - /** - * @brief flush specific LBID/version pairs from the cache - **/ - inline void flushMany(const LbidAtVer* laVptr, uint32_t cnt) - { - fBCCBrp->flushMany(laVptr, cnt); - } + /** + * @brief flush all versions of the given lbids from the cache. + **/ + inline void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) + { + fBCCBrp->flushManyAllversion(laVptr, cnt); + } - /** - * @brief flush all versions of the given lbids from the cache. - **/ - inline void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) - { - fBCCBrp->flushManyAllversion(laVptr, cnt); - } + /** + * @brief Flush all versions of all LBIDs belonging to the given OIDs. + */ + inline void flushOIDs(const uint32_t* oids, uint32_t count) + { + fBCCBrp->flushOIDs(oids, count); + } - /** - * @brief Flush all versions of all LBIDs belonging to the given OIDs. - */ - inline void flushOIDs(const uint32_t* oids, uint32_t count) - { - fBCCBrp->flushOIDs(oids, count); - } + /** + * @brief Flush all versions of a partition from the given OIDs. + */ + inline void flushPartition(const std::vector& oids, + const std::set partitions) + { + fBCCBrp->flushPartition(oids, partitions); + } - /** - * @brief Flush all versions of a partition from the given OIDs. - */ - inline void flushPartition(const std::vector& oids, const std::set partitions) - { - fBCCBrp->flushPartition(oids, partitions); - } - -private: - - /** - * @brief pointer to the BlockRequestProcessor object on which the API will operate - **/ - BlockRequestProcessor* fBCCBrp; - - //do not implement - blockCacheClient(const blockCacheClient& bc); - blockCacheClient& operator=(const blockCacheClient& blk); + private: + /** + * @brief pointer to the BlockRequestProcessor object on which the API will operate + **/ + BlockRequestProcessor* fBCCBrp; + // do not implement + blockCacheClient(const blockCacheClient& bc); + blockCacheClient& operator=(const blockCacheClient& blk); }; -} +} // namespace dbbc #endif // vim:ts=4 sw=4: - diff --git a/primitives/blockcache/blockrequestprocessor.cpp b/primitives/blockcache/blockrequestprocessor.cpp index 0f399d3b5..573dcf6c2 100644 --- a/primitives/blockcache/blockrequestprocessor.cpp +++ b/primitives/blockcache/blockrequestprocessor.cpp @@ -38,232 +38,213 @@ using namespace std; namespace dbbc { - -BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks, - int thrCount, - int blocksPerRead, - uint32_t deleteBlocks, - uint32_t blckSz) : - fbMgr(numBlcks, blckSz, deleteBlocks), - fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead) +BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, + uint32_t deleteBlocks, uint32_t blckSz) + : fbMgr(numBlcks, blckSz, deleteBlocks), fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead) { - //pthread_mutex_init(&check_mutex, NULL); - config::Config* fConfig = config::Config::makeConfig(); - string val = fConfig->getConfig("DBBC", "BRPTracing"); - int temp = 0; + // pthread_mutex_init(&check_mutex, NULL); + config::Config* fConfig = config::Config::makeConfig(); + string val = fConfig->getConfig("DBBC", "BRPTracing"); + int temp = 0; #ifdef _MSC_VER - int tid = GetCurrentThreadId(); + int tid = GetCurrentThreadId(); #else - pthread_t tid = pthread_self(); + pthread_t tid = pthread_self(); #endif - if (val.length() > 0) temp = static_cast(config::Config::fromText(val)); + if (val.length() > 0) + temp = static_cast(config::Config::fromText(val)); - if (temp > 0) - fTrace = true; - else - fTrace = false; + if (temp > 0) + fTrace = true; + else + fTrace = false; - if (fTrace) - { - ostringstream brpLogFileName; + if (fTrace) + { + ostringstream brpLogFileName; #ifdef _MSC_VER - brpLogFileName << "C:/Calpont/log/trace/brp." << tid; + brpLogFileName << "C:/Calpont/log/trace/brp." << tid; #else - brpLogFileName << MCSLOGDIR << "/trace/brp." << tid; + brpLogFileName << MCSLOGDIR << "/trace/brp." << tid; #endif - fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate); - } + fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate); + } } - BlockRequestProcessor::~BlockRequestProcessor() { - //pthread_mutex_destroy(&check_mutex); - if (fTrace) - fLogFile.close(); + // pthread_mutex_destroy(&check_mutex); + if (fTrace) + fLogFile.close(); } void BlockRequestProcessor::stop() { - fBRPRequestQueue.stop(); - fIOMgr.stop(); + fBRPRequestQueue.stop(); + fIOMgr.stop(); } -int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType, uint32_t& lbidCount) +int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, + const BRM::VER_t txn, const int compType, uint32_t& lbidCount) { - uint64_t maxLbid = range.start; // highest existent lbid - uint64_t rangeLen = range.size; - uint64_t idx; - uint64_t adjSz; - struct timespec start_tm; - lbidCount = 0; + uint64_t maxLbid = range.start; // highest existent lbid + uint64_t rangeLen = range.size; + uint64_t idx; + uint64_t adjSz; + struct timespec start_tm; + lbidCount = 0; - if (fTrace) - clock_gettime(CLOCK_MONOTONIC, &start_tm); + if (fTrace) + clock_gettime(CLOCK_MONOTONIC, &start_tm); - for (idx = 0; fbMgr.exists(maxLbid, ver.currentScn) == true && idx < rangeLen; maxLbid++, idx++) - (void)0; - - if (idx == rangeLen) // range is already loaded - { - if (fTrace) - { - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t fbo; - BRM::OID_t oid; - fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo); - fLogFile - << oid << " " - << maxLbid << " " - << fbo << " " - << rangeLen << " " - << 0 << " " - << 0 << " " - << 0 << " " - << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) - << endl; - } - - return 0; - } - - adjSz = rangeLen - idx; - BRM::InlineLBIDRange adjRange; - adjRange.start = maxLbid; - adjRange.size = adjSz; - fileRequest rqstBlk(adjRange, ver, txn, compType); - check(rqstBlk); - - if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR) - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP); - else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL) - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT), - logging::ERR_O_DIRECT); - else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT) - throw logging::IDBExcept( - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT), - logging::ERR_ENOENT); - else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) - throw runtime_error(rqstBlk.RequestStatusStr()); - - lbidCount = rqstBlk.BlocksRead(); + for (idx = 0; fbMgr.exists(maxLbid, ver.currentScn) == true && idx < rangeLen; maxLbid++, idx++) + (void)0; + if (idx == rangeLen) // range is already loaded + { if (fTrace) { - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t fbo; - BRM::OID_t oid; - fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo); - fLogFile - << oid << " " - << maxLbid << " " - << fbo << " " - << rangeLen << " " - << adjSz << " " - << rqstBlk.BlocksRead() << " " - << rqstBlk.BlocksLoaded() << " " - << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) - << endl; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t fbo; + BRM::OID_t oid; + fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo); + fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << 0 << " " << 0 << " " << 0 + << " " << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl; } - return rqstBlk.BlocksLoaded(); -} // check + return 0; + } + + adjSz = rangeLen - idx; + BRM::InlineLBIDRange adjRange; + adjRange.start = maxLbid; + adjRange.size = adjSz; + fileRequest rqstBlk(adjRange, ver, txn, compType); + check(rqstBlk); + + if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR) + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), + logging::ERR_BRM_LOOKUP); + else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL) + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT), + logging::ERR_O_DIRECT); + else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT) + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT), + logging::ERR_ENOENT); + else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) + throw runtime_error(rqstBlk.RequestStatusStr()); + + lbidCount = rqstBlk.BlocksRead(); + + if (fTrace) + { + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t fbo; + BRM::OID_t oid; + fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo); + fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << adjSz << " " + << rqstBlk.BlocksRead() << " " << rqstBlk.BlocksLoaded() << " " << right << fixed + << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl; + } + + return rqstBlk.BlocksLoaded(); +} // check int BlockRequestProcessor::check(fileRequest& rqstBlk) { - rqstBlk.frMutex().lock(); - rqstBlk.SetPredicate(fileRequest::SENDING); - sendRequest(rqstBlk); // start file read request + rqstBlk.frMutex().lock(); + rqstBlk.SetPredicate(fileRequest::SENDING); + sendRequest(rqstBlk); // start file read request - while (rqstBlk.frPredicate() < fileRequest::COMPLETE) - rqstBlk.frCond().wait(rqstBlk.frMutex()); + while (rqstBlk.frPredicate() < fileRequest::COMPLETE) + rqstBlk.frCond().wait(rqstBlk.frMutex()); - rqstBlk.frMutex().unlock(); + rqstBlk.frMutex().unlock(); - return 0; + return 0; } // For future use. Not currently used. -int BlockRequestProcessor::check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache) +int BlockRequestProcessor::check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, + int compType, bool& wasBlockInCache) { - if (fbMgr.exists(lbid, ver.currentScn) == true) - { - wasBlockInCache = true; - return 0; - } - else - { - wasBlockInCache = false; - fileRequest rqstBlk(lbid, ver, flg, txn, compType); - int ret = check(rqstBlk); + if (fbMgr.exists(lbid, ver.currentScn) == true) + { + wasBlockInCache = true; + return 0; + } + else + { + wasBlockInCache = false; + fileRequest rqstBlk(lbid, ver, flg, txn, compType); + int ret = check(rqstBlk); - if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) - { - throw runtime_error(rqstBlk.RequestStatusStr()); - } - - return ret; + if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) + { + throw runtime_error(rqstBlk.RequestStatusStr()); } + + return ret; + } } int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn, - int compType, void* bufferPtr, bool vbFlg, bool& wasCached, bool* versioned, bool insertIntoCache, - bool readFromCache) + int compType, void* bufferPtr, bool vbFlg, bool& wasCached, + bool* versioned, bool insertIntoCache, bool readFromCache) { - if (readFromCache) - { - HashObject_t hashObj(lbid, ver.currentScn, 0); - wasCached = fbMgr.find(hashObj, bufferPtr); + if (readFromCache) + { + HashObject_t hashObj(lbid, ver.currentScn, 0); + wasCached = fbMgr.find(hashObj, bufferPtr); - if (wasCached) - return 1; - } + if (wasCached) + return 1; + } - wasCached = false; - fileRequest rqstBlk(lbid, ver, vbFlg, txn, compType, (uint8_t*) bufferPtr, insertIntoCache); - check(rqstBlk); + wasCached = false; + fileRequest rqstBlk(lbid, ver, vbFlg, txn, compType, (uint8_t*)bufferPtr, insertIntoCache); + check(rqstBlk); - if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR) - { - ostringstream os; - os << "BRP::getBlock(): got a BRM lookup error. LBID=" << lbid << " ver=" << ver << " txn=" - << txn << " vbFlg=" << (int) vbFlg; - primitiveprocessor::Logger logger; - logger.logMessage(os.str(), false); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP); - } - else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL) - { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT), - logging::ERR_O_DIRECT); - } - else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT) - { - throw logging::IDBExcept( - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT), - logging::ERR_ENOENT); - } - else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) - { - throw runtime_error(rqstBlk.RequestStatusStr()); - } + if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR) + { + ostringstream os; + os << "BRP::getBlock(): got a BRM lookup error. LBID=" << lbid << " ver=" << ver << " txn=" << txn + << " vbFlg=" << (int)vbFlg; + primitiveprocessor::Logger logger; + logger.logMessage(os.str(), false); + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), + logging::ERR_BRM_LOOKUP); + } + else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL) + { + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT), + logging::ERR_O_DIRECT); + } + else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT) + { + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT), + logging::ERR_ENOENT); + } + else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) + { + throw runtime_error(rqstBlk.RequestStatusStr()); + } - if (versioned) - *versioned = rqstBlk.versioned(); + if (versioned) + *versioned = rqstBlk.versioned(); - return 1; + return 1; } -int BlockRequestProcessor::getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, - uint8_t** ptrs, bool* wasCached, uint32_t count) +int BlockRequestProcessor::getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** ptrs, + bool* wasCached, uint32_t count) { - return fbMgr.bulkFind(lbids, vers, ptrs, wasCached, count); + return fbMgr.bulkFind(lbids, vers, ptrs, wasCached, count); } - -} // namespace dbbc +} // namespace dbbc diff --git a/primitives/blockcache/blockrequestprocessor.h b/primitives/blockcache/blockrequestprocessor.h index d6ee5320a..edb1479f1 100644 --- a/primitives/blockcache/blockrequestprocessor.h +++ b/primitives/blockcache/blockrequestprocessor.h @@ -35,12 +35,11 @@ #include /** - @author Jason Rodriguez + @author Jason Rodriguez */ namespace dbbc { - typedef std::list FileBufferList_t; /** @@ -49,152 +48,148 @@ typedef std::list FileBufferList_t; class BlockRequestProcessor { + public: + /** + * @brief default ctor + **/ + BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0, + uint32_t blckSz = BLOCK_SIZE); -public: + /** + * @brief default dtor + **/ + virtual ~BlockRequestProcessor(); - /** - * @brief default ctor - **/ - BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0, - uint32_t blckSz = BLOCK_SIZE); + /** + * @brief send a request for disk blocks to the IO manager + **/ + int sendRequest(fileRequest& blk) + { + return fBRPRequestQueue.push(blk); + } - /** - * @brief default dtor - **/ - virtual ~BlockRequestProcessor(); + /** + * @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not + **/ + int check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, + bool& wasBlockInCache); - /** - * @brief send a request for disk blocks to the IO manager - **/ - int sendRequest(fileRequest& blk) - { - return fBRPRequestQueue.push(blk); - } + /** + * @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not + **/ + int check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, + const int compType, uint32_t& lbidCount); - /** - * @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not - **/ - int check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache); + /** + * @brief retrieve the lbid@ver disk block from the block cache + **/ + inline FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver, bool flg) + { + return fbMgr.findPtr(HashObject_t(lbid, ver, flg)); + } - /** - * @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not - **/ - int check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType, - uint32_t& lbidCount); + inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) + { + return (fbMgr.find(HashObject_t(lbid, ver, 0), fb) ? 1 : 0); + } - /** - * @brief retrieve the lbid@ver disk block from the block cache - **/ - inline FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver, bool flg) - { - return fbMgr.findPtr(HashObject_t(lbid, ver, flg)); - } + /** + * @brief retrieve the lbid@ver disk block from the block cache + **/ + inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) + { + return (fbMgr.find(HashObject_t(lbid, ver, 0), bufferPtr) ? 1 : 0); + } - inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) - { - return (fbMgr.find(HashObject_t(lbid, ver, 0), fb) ? 1 : 0); - } + int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn, int compType, + void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned, bool insertIntoCache, + bool readFromCache); - /** - * @brief retrieve the lbid@ver disk block from the block cache - **/ - inline int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) - { - return (fbMgr.find(HashObject_t(lbid, ver, 0), bufferPtr) ? 1 : 0); - } + int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** ptrs, bool* wasCached, + uint32_t count); - int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn, int compType, - void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned, bool insertIntoCache, - bool readFromCache); + inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) + { + return fbMgr.exists(HashObject_t(lbid, ver, 0)); + } - int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** ptrs, - bool* wasCached, uint32_t count); + /** + * @brief + **/ + void flushCache() + { + fbMgr.flushCache(); + } - inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) - { - return fbMgr.exists(HashObject_t(lbid, ver, 0)); - } + /** + * @brief + **/ + void flushOne(BRM::LBID_t lbid, BRM::VER_t ver) + { + fbMgr.flushOne(lbid, ver); + } - /** - * @brief - **/ - void flushCache() - { - fbMgr.flushCache(); - } + void flushMany(const LbidAtVer* laVptr, uint32_t cnt) + { + fbMgr.flushMany(laVptr, cnt); + } - /** - * @brief - **/ - void flushOne(BRM::LBID_t lbid, BRM::VER_t ver) - { - fbMgr.flushOne(lbid, ver); - } + void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) + { + fbMgr.flushManyAllversion(laVptr, cnt); + } - void flushMany(const LbidAtVer* laVptr, uint32_t cnt) - { - fbMgr.flushMany(laVptr, cnt); - } + void flushOIDs(const uint32_t* oids, uint32_t count) + { + fbMgr.flushOIDs(oids, count); + } - void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) - { - fbMgr.flushManyAllversion(laVptr, cnt); - } + void flushPartition(const std::vector& oids, const std::set& partitions) + { + fbMgr.flushPartition(oids, partitions); + } - void flushOIDs(const uint32_t* oids, uint32_t count) - { - fbMgr.flushOIDs(oids, count); - } + void setReportingFrequency(const uint32_t d) + { + fbMgr.setReportingFrequency(d); + } - void flushPartition(const std::vector& oids, const std::set& partitions) - { - fbMgr.flushPartition(oids, partitions); - } + uint32_t ReportingFrequency() const + { + return fbMgr.ReportingFrequency(); + } - void setReportingFrequency(const uint32_t d) - { - fbMgr.setReportingFrequency(d); - } + std::ostream& formatLRUList(std::ostream& os) const + { + return fbMgr.formatLRUList(os); + } - uint32_t ReportingFrequency() const - { - return fbMgr.ReportingFrequency(); - } + private: + FileBufferMgr fbMgr; + fileBlockRequestQueue fBRPRequestQueue; + ioManager fIOMgr; + boost::mutex check_mutex; - std::ostream& formatLRUList(std::ostream& os) const - { - return fbMgr.formatLRUList(os); - } + /** + * helper function for public check functions + **/ + int check(fileRequest& rqstBlk); -private: + /** + * send stop requests for IOmanager and request Q + **/ + void stop(); - FileBufferMgr fbMgr; - fileBlockRequestQueue fBRPRequestQueue; - ioManager fIOMgr; - boost::mutex check_mutex; + std::ofstream fLogFile; + bool fTrace; - /** - * helper function for public check functions - **/ - int check(fileRequest& rqstBlk); - - /** - * send stop requests for IOmanager and request Q - **/ - void stop(); - - std::ofstream fLogFile; - bool fTrace; - - BRM::DBRM fdbrm; - - // do not implement - BlockRequestProcessor(const BlockRequestProcessor& brp); - BlockRequestProcessor& operator=(const BlockRequestProcessor& brp); + BRM::DBRM fdbrm; + // do not implement + BlockRequestProcessor(const BlockRequestProcessor& brp); + BlockRequestProcessor& operator=(const BlockRequestProcessor& brp); }; -} +} // namespace dbbc #endif // vim:ts=4 sw=4: - diff --git a/primitives/blockcache/fileblockrequestqueue.cpp b/primitives/blockcache/fileblockrequestqueue.cpp index 82899975a..ce0c4127b 100644 --- a/primitives/blockcache/fileblockrequestqueue.cpp +++ b/primitives/blockcache/fileblockrequestqueue.cpp @@ -32,67 +32,64 @@ using namespace std; namespace dbbc { - - fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0) { - //pthread_mutex_init(&mutex, NULL); - //pthread_cond_init(¬Empty, NULL); + // pthread_mutex_init(&mutex, NULL); + // pthread_cond_init(¬Empty, NULL); } fileBlockRequestQueue::~fileBlockRequestQueue() { - //pthread_cond_destroy(¬Empty); - //pthread_mutex_destroy(&mutex); + // pthread_cond_destroy(¬Empty); + // pthread_mutex_destroy(&mutex); } bool fileBlockRequestQueue::empty() const { - return (queueSize == 0); + return (queueSize == 0); } fileRequest* fileBlockRequestQueue::top() const { - return fbQueue.front(); + return fbQueue.front(); } int fileBlockRequestQueue::push(fileRequest& blk) { - mutex.lock(); //pthread_mutex_lock(&mutex); - fbQueue.push_back(&blk); + mutex.lock(); // pthread_mutex_lock(&mutex); + fbQueue.push_back(&blk); - // @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which - // caused only one i/o thread to be working at a time. - if (++queueSize >= 1 && readersWaiting > 0) - notEmpty.notify_one(); //pthread_cond_signal(¬Empty); + // @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size + // was 1 which caused only one i/o thread to be working at a time. + if (++queueSize >= 1 && readersWaiting > 0) + notEmpty.notify_one(); // pthread_cond_signal(¬Empty); - mutex.unlock(); //pthread_mutex_unlock(&mutex); + mutex.unlock(); // pthread_mutex_unlock(&mutex); - return 0; + return 0; } void fileBlockRequestQueue::stop() { - notEmpty.notify_all(); //pthread_cond_broadcast(¬Empty); + notEmpty.notify_all(); // pthread_cond_broadcast(¬Empty); } - fileRequest* fileBlockRequestQueue::pop(void) { - mutex.lock(); //pthread_mutex_lock(&mutex); + mutex.lock(); // pthread_mutex_lock(&mutex); - while (queueSize == 0) - { - readersWaiting++; - notEmpty.wait(mutex); //pthread_cond_wait(¬Empty, &mutex); - readersWaiting--; - } + while (queueSize == 0) + { + readersWaiting++; + notEmpty.wait(mutex); // pthread_cond_wait(¬Empty, &mutex); + readersWaiting--; + } - fileRequest* blk = fbQueue.front(); - fbQueue.pop_front(); - --queueSize; - mutex.unlock(); //pthread_mutex_unlock(&mutex); - return blk; + fileRequest* blk = fbQueue.front(); + fbQueue.pop_front(); + --queueSize; + mutex.unlock(); // pthread_mutex_unlock(&mutex); + return blk; } -} +} // namespace dbbc diff --git a/primitives/blockcache/fileblockrequestqueue.h b/primitives/blockcache/fileblockrequestqueue.h index 0acec04e6..f9cdc1bf3 100644 --- a/primitives/blockcache/fileblockrequestqueue.h +++ b/primitives/blockcache/fileblockrequestqueue.h @@ -24,8 +24,7 @@ * * jrodriguez@calpont.com * * * -***************************************************************************/ - + ***************************************************************************/ #include #include @@ -33,9 +32,8 @@ #include #include "filerequest.h" - /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** @@ -43,75 +41,71 @@ **/ namespace dbbc { - typedef std::deque fileBlockRequestQueue_t; /** * @brief class to hold requests for disk blocks in a queue. sorted by the size of a request **/ - class fileBlockRequestQueue { + public: + /** + * @brief default ctor + **/ + fileBlockRequestQueue(); -public: + /** + * @brief dtor + **/ + virtual ~fileBlockRequestQueue(); - /** - * @brief default ctor - **/ - fileBlockRequestQueue(); + /** + * @brief add a request to the queue + **/ + int push(fileRequest& blk); - /** - * @brief dtor - **/ - virtual ~fileBlockRequestQueue(); + /** + * @brief get the next request from the queue and delete it from the queue + **/ + fileRequest* pop(void); - /** - * @brief add a request to the queue - **/ - int push(fileRequest& blk); + /** + * @brief true if no reuquests are in the queue. false if there are requests in the queue + **/ + bool empty() const; - /** - * @brief get the next request from the queue and delete it from the queue - **/ - fileRequest* pop(void); + /** + * @brief number of requests in the queue + **/ + uint32_t size() const + { + return queueSize; + } - /** - * @brief true if no reuquests are in the queue. false if there are requests in the queue - **/ - bool empty() const; + /** + * @brief queue will stop accecpting requests in preparation for the dtor + **/ + void stop(); - /** - * @brief number of requests in the queue - **/ - uint32_t size() const - { - return queueSize; - } - - /** - * @brief queue will stop accecpting requests in preparation for the dtor - **/ - void stop(); - -protected: - boost::mutex mutex; - boost::condition notEmpty; - fileBlockRequestQueue_t fbQueue; - uint32_t queueSize; - uint32_t readersWaiting; - -private: - // do not implement - fileBlockRequestQueue(const fileBlockRequestQueue& Q) {} - const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q); - - /** - * @brief pointer to the next request to be popped from the queue - **/ - fileRequest* top() const; + protected: + boost::mutex mutex; + boost::condition notEmpty; + fileBlockRequestQueue_t fbQueue; + uint32_t queueSize; + uint32_t readersWaiting; + private: + // do not implement + fileBlockRequestQueue(const fileBlockRequestQueue& Q) + { + } + const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q); + /** + * @brief pointer to the next request to be popped from the queue + **/ + fileRequest* top() const; }; -} +} // namespace dbbc #endif diff --git a/primitives/blockcache/filebuffer.cpp b/primitives/blockcache/filebuffer.cpp index aa598c008..d38a40243 100644 --- a/primitives/blockcache/filebuffer.cpp +++ b/primitives/blockcache/filebuffer.cpp @@ -32,72 +32,67 @@ using namespace std; namespace dbbc { - FileBuffer::FileBuffer() : fDataLen(0), fLbid(-1), fVerid(0) { } FileBuffer::FileBuffer(const FileBuffer& rhs) { + // Removed the check for gcc 8.2. The latest gcc version we + // use ATM is 4.8.2 for centos 6 and it also doesn't need + // the check + if (this == &rhs) + return; - // Removed the check for gcc 8.2. The latest gcc version we - // use ATM is 4.8.2 for centos 6 and it also doesn't need - // the check - if (this == &rhs) - return; - - fLbid = rhs.fLbid; - fVerid = rhs.fVerid; - setData(rhs.fByteData, rhs.fDataLen); - fListLoc = rhs.listLoc(); - fDataLen = rhs.fDataLen; + fLbid = rhs.fLbid; + fVerid = rhs.fVerid; + setData(rhs.fByteData, rhs.fDataLen); + fListLoc = rhs.listLoc(); + fDataLen = rhs.fDataLen; } - FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len) { - fLbid = lbid; - fVerid = ver; - fDataLen = len; - setData(data, fDataLen); + fLbid = lbid; + fVerid = ver; + fDataLen = len; + setData(data, fDataLen); } - FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver) { - fLbid = lbid; - fVerid = ver; - fDataLen = 0; + fLbid = lbid; + fVerid = ver; + fDataLen = 0; } - -FileBuffer& FileBuffer::operator= (const FileBuffer& rhs) +FileBuffer& FileBuffer::operator=(const FileBuffer& rhs) { - fLbid = rhs.fLbid; - fVerid = rhs.fVerid; - fDataLen = rhs.fDataLen; - setData(rhs.fByteData, fDataLen); - fListLoc = rhs.listLoc(); - return *this; + fLbid = rhs.fLbid; + fVerid = rhs.fVerid; + fDataLen = rhs.fDataLen; + setData(rhs.fByteData, fDataLen); + fListLoc = rhs.listLoc(); + return *this; } void FileBuffer::setData(const uint8_t* d, const int len) { - if (d == NULL || len <= 0) - return; + if (d == NULL || len <= 0) + return; - fDataLen = len; - memcpy(fByteData, d, len); + fDataLen = len; + memcpy(fByteData, d, len); } void FileBuffer::setData(const uint8_t* d) { - fDataLen = 8192; - memcpy(fByteData, d, 8192); + fDataLen = 8192; + memcpy(fByteData, d, 8192); } FileBuffer::~FileBuffer() { } -} +} // namespace dbbc diff --git a/primitives/blockcache/filebuffer.h b/primitives/blockcache/filebuffer.h index 5aee1bd1c..b5b54546b 100644 --- a/primitives/blockcache/filebuffer.h +++ b/primitives/blockcache/filebuffer.h @@ -21,7 +21,7 @@ * * jrodriguez@calpont.com * * * -***************************************************************************/ + ***************************************************************************/ #ifndef FILEBUFFER_H #define FILEBUFFER_H @@ -38,21 +38,19 @@ #include "blocksize.h" /** - @author Jason Rodriguez + @author Jason Rodriguez */ - /** * @brief represents a disk blockrequest **/ namespace dbbc { - typedef struct FBData { - BRM::LBID_t lbid; - BRM::VER_t ver; - uint8_t hits; + BRM::LBID_t lbid; + BRM::VER_t ver; + uint8_t hits; } FBData_t; //@bug 669 Change to list for least recently used cache @@ -61,126 +59,123 @@ typedef std::list::iterator filebuffer_list_iter_t; class FileBuffer { + public: + FileBuffer(); + /** + * @brief copy ctor + **/ + FileBuffer(const FileBuffer& fb); -public: + /** + * @brief the disk block from lbid@ver, and a data block len bytes long + **/ + FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len); - FileBuffer(); - /** - * @brief copy ctor - **/ - FileBuffer(const FileBuffer& fb); + /** + * @brief disk block lbid@ver and empty data + **/ + FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver); - /** - * @brief the disk block from lbid@ver, and a data block len bytes long - **/ - FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len); + /** + * @brief class dtor + **/ + ~FileBuffer(); - /** - * @brief disk block lbid@ver and empty data - **/ - FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver); + /** + * @brief set the data value of this block to d have len bytestream + **/ + void setData(const uint8_t* d, const int len); + void setData(const uint8_t* d); // assumes len = 8192 - /** - * @brief class dtor - **/ - ~FileBuffer(); + /** + * @brief retrieve the data in byte* format from this data block + **/ + inline const uint8_t* getData() const + { + return fByteData; + } + inline uint8_t* getData() + { + return fByteData; + } - /** - * @brief set the data value of this block to d have len bytestream - **/ - void setData(const uint8_t* d, const int len); - void setData(const uint8_t* d); // assumes len = 8192 + inline uint32_t datLen() const + { + return fDataLen; + } - /** - * @brief retrieve the data in byte* format from this data block - **/ - inline const uint8_t* getData() const - { - return fByteData; - } - inline uint8_t* getData() - { - return fByteData; - } + /** + * @brief assignment operator + **/ + FileBuffer& operator=(const FileBuffer& rhs); - inline uint32_t datLen() const - { - return fDataLen; - } + /** + * @brief equality operator is based on lbid@ver + **/ + inline bool operator==(const FileBuffer& rhs) const + { + return (fLbid == rhs.fLbid && fVerid == rhs.fVerid); + } - /** - * @brief assignment operator - **/ - FileBuffer& operator= (const FileBuffer& rhs); + /** + * @brief inequality operator + **/ + inline bool operator!=(const FileBuffer& rhs) const + { + return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid)); + } - /** - * @brief equality operator is based on lbid@ver - **/ - inline bool operator==(const FileBuffer& rhs) const - { - return (fLbid == rhs.fLbid && fVerid == rhs.fVerid); - } + FileBuffer* thisPtr() + { + return this; + } + /** + * @brief return the lbid value of disk bloc + **/ + inline BRM::LBID_t Lbid() const + { + return fLbid; + } + inline void Lbid(const BRM::LBID_t l) + { + fLbid = l; + } - /** - * @brief inequality operator - **/ - inline bool operator!=(const FileBuffer& rhs) const - { - return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid)); - } + /** + * @brief return the version of this disk block. ignored for range retrievals + **/ + inline BRM::VER_t Verid() const + { + return fVerid; + } + inline void Verid(BRM::VER_t v) + { + fVerid = v; + } - FileBuffer* thisPtr() - { - return this; - } - /** - * @brief return the lbid value of disk bloc - **/ - inline BRM::LBID_t Lbid() const - { - return fLbid; - } - inline void Lbid(const BRM::LBID_t l) - { - fLbid = l; - } + /** + * @brief return the number of bytes in this disk blockrequest + **/ - /** - * @brief return the version of this disk block. ignored for range retrievals - **/ - inline BRM::VER_t Verid() const - { - return fVerid; - } - inline void Verid(BRM::VER_t v) - { - fVerid = v; - } + inline void listLoc(const filebuffer_list_iter_t& loc) + { + fListLoc = loc; + } - /** - * @brief return the number of bytes in this disk blockrequest - **/ + inline const filebuffer_list_iter_t& listLoc() const + { + return fListLoc; + } - inline void listLoc(const filebuffer_list_iter_t& loc) - { - fListLoc = loc; - } - - inline const filebuffer_list_iter_t& listLoc() const - { - return fListLoc; - } - -private: - - uint8_t fByteData[BLOCK_SIZE]; - uint32_t fDataLen; - BRM::LBID_t fLbid; - BRM::VER_t fVerid; - filebuffer_list_iter_t fListLoc; + private: + uint8_t fByteData[BLOCK_SIZE]; + uint32_t fDataLen; + BRM::LBID_t fLbid; + BRM::VER_t fVerid; + filebuffer_list_iter_t fListLoc; }; typedef std::vector FileBufferPool_t; -} +} // namespace dbbc #endif diff --git a/primitives/blockcache/filebuffermgr.cpp b/primitives/blockcache/filebuffermgr.cpp index 2dd1be204..fa50a850f 100644 --- a/primitives/blockcache/filebuffermgr.cpp +++ b/primitives/blockcache/filebuffermgr.cpp @@ -25,7 +25,7 @@ ***************************************************************************/ /** * InitialDBBCSize - the starting number of elements the unordered set used to store disk blocks. - This does not instantiate InitialDBBCSize disk blocks but only the initial size of the unordered_set + This does not instantiate InitialDBBCSize disk blocks but only the initial size of the unordered_set **/ @@ -57,520 +57,514 @@ namespace dbbc const uint32_t gReportingFrequencyMin(32768); FileBufferMgr::FileBufferMgr(const uint32_t numBlcks, const uint32_t blkSz, const uint32_t deleteBlocks) - : fMaxNumBlocks(numBlcks), - fBlockSz(blkSz), - fWLock(), - fbSet(), - fbList(), - fCacheSize(0), - fFBPool(), - fDeleteBlocks(deleteBlocks), - fEmptyPoolSlots(), - fReportFrequency(0) + : fMaxNumBlocks(numBlcks) + , fBlockSz(blkSz) + , fWLock() + , fbSet() + , fbList() + , fCacheSize(0) + , fFBPool() + , fDeleteBlocks(deleteBlocks) + , fEmptyPoolSlots() + , fReportFrequency(0) { - fFBPool.reserve(numBlcks); - fConfig = Config::makeConfig(); - setReportingFrequency(0); + fFBPool.reserve(numBlcks); + fConfig = Config::makeConfig(); + setReportingFrequency(0); #ifdef _MSC_VER - fLog.open("C:/Calpont/log/trace/bc", ios_base::app | ios_base::ate); + fLog.open("C:/Calpont/log/trace/bc", ios_base::app | ios_base::ate); #else - fLog.open(string(MCSLOGDIR) + "/trace/bc", ios_base::app | ios_base::ate); + fLog.open(string(MCSLOGDIR) + "/trace/bc", ios_base::app | ios_base::ate); #endif } FileBufferMgr::~FileBufferMgr() { - flushCache(); + flushCache(); } // param d is used as a togle only void FileBufferMgr::setReportingFrequency(const uint32_t d) { - if (d == 0) - { - fReportFrequency = 0; - return; - } + if (d == 0) + { + fReportFrequency = 0; + return; + } - const string val = fConfig->getConfig("DBBC", "ReportFrequency"); - uint32_t temp = 0; + const string val = fConfig->getConfig("DBBC", "ReportFrequency"); + uint32_t temp = 0; - if (val.length() > 0) temp = static_cast(Config::fromText(val)); - - if (temp > 0 && temp <= gReportingFrequencyMin) - fReportFrequency = gReportingFrequencyMin; - else - fReportFrequency = temp; + if (val.length() > 0) + temp = static_cast(Config::fromText(val)); + if (temp > 0 && temp <= gReportingFrequencyMin) + fReportFrequency = gReportingFrequencyMin; + else + fReportFrequency = temp; } void FileBufferMgr::flushCache() { - boost::mutex::scoped_lock lk(fWLock); - { - filebuffer_uset_t sEmpty; - filebuffer_list_t lEmpty; - emptylist_t vEmpty; + boost::mutex::scoped_lock lk(fWLock); + { + filebuffer_uset_t sEmpty; + filebuffer_list_t lEmpty; + emptylist_t vEmpty; - fbList.swap(lEmpty); - fbSet.swap(sEmpty); - fEmptyPoolSlots.swap(vEmpty); - } - fCacheSize = 0; + fbList.swap(lEmpty); + fbSet.swap(sEmpty); + fEmptyPoolSlots.swap(vEmpty); + } + fCacheSize = 0; - // the block pool should not be freed in the above block to allow us - // to continue doing concurrent unprotected-but-"safe" memcpys - // from that memory - if (fReportFrequency) - { - fLog << "Clearing entire cache" << endl; - } - fFBPool.clear(); -// fFBPool.reserve(fMaxNumBlocks); + // the block pool should not be freed in the above block to allow us + // to continue doing concurrent unprotected-but-"safe" memcpys + // from that memory + if (fReportFrequency) + { + fLog << "Clearing entire cache" << endl; + } + fFBPool.clear(); + // fFBPool.reserve(fMaxNumBlocks); } void FileBufferMgr::flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver) { - //similar in function to depleteCache() - boost::mutex::scoped_lock lk(fWLock); + // similar in function to depleteCache() + boost::mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t iter = fbSet.find(HashObject_t(lbid, ver, 0)); - - if (iter != fbSet.end()) - { - //remove it from fbList - uint32_t idx = iter->poolIdx; - fbList.erase(fFBPool[idx].listLoc()); - //add to fEmptyPoolSlots - fEmptyPoolSlots.push_back(idx); - //remove it from fbSet - fbSet.erase(iter); - //adjust fCacheSize - fCacheSize--; - } + filebuffer_uset_iter_t iter = fbSet.find(HashObject_t(lbid, ver, 0)); + if (iter != fbSet.end()) + { + // remove it from fbList + uint32_t idx = iter->poolIdx; + fbList.erase(fFBPool[idx].listLoc()); + // add to fEmptyPoolSlots + fEmptyPoolSlots.push_back(idx); + // remove it from fbSet + fbSet.erase(iter); + // adjust fCacheSize + fCacheSize--; + } } void FileBufferMgr::flushMany(const LbidAtVer* laVptr, uint32_t cnt) { - boost::mutex::scoped_lock lk(fWLock); + boost::mutex::scoped_lock lk(fWLock); - BRM::LBID_t lbid; - BRM::VER_t ver; - filebuffer_uset_iter_t iter; - if (fReportFrequency) - { - fLog << "flushMany " << cnt << " items: "; - for (uint32_t j = 0; j < cnt; j++) - { - fLog << "lbid: " << laVptr[j].LBID << " ver: " << laVptr[j].Ver << ", "; - } - fLog << endl; - } + BRM::LBID_t lbid; + BRM::VER_t ver; + filebuffer_uset_iter_t iter; + if (fReportFrequency) + { + fLog << "flushMany " << cnt << " items: "; for (uint32_t j = 0; j < cnt; j++) { - lbid = static_cast(laVptr->LBID); - ver = static_cast(laVptr->Ver); - iter = fbSet.find(HashObject_t(lbid, ver, 0)); - - if (iter != fbSet.end()) - { - if (fReportFrequency) - { - fLog << "flushMany hit, lbid: " << lbid << " index: " << iter->poolIdx << endl; - } - //remove it from fbList - uint32_t idx = iter->poolIdx; - fbList.erase(fFBPool[idx].listLoc()); - //add to fEmptyPoolSlots - fEmptyPoolSlots.push_back(idx); - //remove it from fbSet - fbSet.erase(iter); - //adjust fCacheSize - fCacheSize--; - } - - ++laVptr; + fLog << "lbid: " << laVptr[j].LBID << " ver: " << laVptr[j].Ver << ", "; } + fLog << endl; + } + for (uint32_t j = 0; j < cnt; j++) + { + lbid = static_cast(laVptr->LBID); + ver = static_cast(laVptr->Ver); + iter = fbSet.find(HashObject_t(lbid, ver, 0)); + + if (iter != fbSet.end()) + { + if (fReportFrequency) + { + fLog << "flushMany hit, lbid: " << lbid << " index: " << iter->poolIdx << endl; + } + // remove it from fbList + uint32_t idx = iter->poolIdx; + fbList.erase(fFBPool[idx].listLoc()); + // add to fEmptyPoolSlots + fEmptyPoolSlots.push_back(idx); + // remove it from fbSet + fbSet.erase(iter); + // adjust fCacheSize + fCacheSize--; + } + + ++laVptr; + } } void FileBufferMgr::flushManyAllversion(const LBID_t* laVptr, uint32_t cnt) { - filebuffer_uset_t::iterator it, tmpIt; - tr1::unordered_set uniquer; - tr1::unordered_set::iterator uit; + filebuffer_uset_t::iterator it, tmpIt; + tr1::unordered_set uniquer; + tr1::unordered_set::iterator uit; - boost::mutex::scoped_lock lk(fWLock); - - if (fReportFrequency) - { - fLog << "flushManyAllversion " << cnt << " items: "; - for (uint32_t i = 0; i < cnt; i++) - { - fLog << laVptr[i] << ", "; - } - fLog << endl; - } - - if (fCacheSize == 0 || cnt == 0) - return; + boost::mutex::scoped_lock lk(fWLock); + if (fReportFrequency) + { + fLog << "flushManyAllversion " << cnt << " items: "; for (uint32_t i = 0; i < cnt; i++) - uniquer.insert(laVptr[i]); - - for (it = fbSet.begin(); it != fbSet.end();) { - if (uniquer.find(it->lbid) != uniquer.end()) - { - if (fReportFrequency) - { - fLog << "flushManyAllversion hit: " << it->lbid << " index: " << it->poolIdx << endl; - } - const uint32_t idx = it->poolIdx; - fbList.erase(fFBPool[idx].listLoc()); - fEmptyPoolSlots.push_back(idx); - tmpIt = it; - ++it; - fbSet.erase(tmpIt); - fCacheSize--; - } - else - ++it; + fLog << laVptr[i] << ", "; } + fLog << endl; + } + + if (fCacheSize == 0 || cnt == 0) + return; + + for (uint32_t i = 0; i < cnt; i++) + uniquer.insert(laVptr[i]); + + for (it = fbSet.begin(); it != fbSet.end();) + { + if (uniquer.find(it->lbid) != uniquer.end()) + { + if (fReportFrequency) + { + fLog << "flushManyAllversion hit: " << it->lbid << " index: " << it->poolIdx << endl; + } + const uint32_t idx = it->poolIdx; + fbList.erase(fFBPool[idx].listLoc()); + fEmptyPoolSlots.push_back(idx); + tmpIt = it; + ++it; + fbSet.erase(tmpIt); + fCacheSize--; + } + else + ++it; + } } void FileBufferMgr::flushOIDs(const uint32_t* oids, uint32_t count) { - DBRM dbrm; - uint32_t i; - vector extents; - int err; - uint32_t currentExtent; - LBID_t currentLBID; - typedef tr1::unordered_multimap byLBID_t; - byLBID_t byLBID; - pair itList; - filebuffer_uset_t::iterator it; + DBRM dbrm; + uint32_t i; + vector extents; + int err; + uint32_t currentExtent; + LBID_t currentLBID; + typedef tr1::unordered_multimap byLBID_t; + byLBID_t byLBID; + pair itList; + filebuffer_uset_t::iterator it; - if (fReportFrequency) + if (fReportFrequency) + { + fLog << "flushOIDs " << count << " items: "; + for (uint32_t i = 0; i < count; i++) { - fLog << "flushOIDs " << count << " items: "; - for (uint32_t i = 0; i < count; i++) - { - fLog << oids[i] << ", "; - } - fLog << endl; + fLog << oids[i] << ", "; + } + fLog << endl; + } + + // If there are more than this # of extents to drop, the whole cache will be cleared + const uint32_t clearThreshold = 50000; + + boost::mutex::scoped_lock lk(fWLock); + + if (fCacheSize == 0 || count == 0) + return; + + /* Index the cache by LBID */ + for (it = fbSet.begin(); it != fbSet.end(); it++) + byLBID.insert(pair(it->lbid, it)); + + for (i = 0; i < count; i++) + { + extents.clear(); + err = dbrm.getExtents(oids[i], extents, true, true, true); // @Bug 3838 Include outofservice extents + + if (err < 0 || (i == 0 && (extents.size() * count) > clearThreshold)) + { + // (The i == 0 should ensure it's not a dictionary column) + lk.unlock(); + flushCache(); + return; } - // If there are more than this # of extents to drop, the whole cache will be cleared - const uint32_t clearThreshold = 50000; - - boost::mutex::scoped_lock lk(fWLock); - - if (fCacheSize == 0 || count == 0) - return; - - /* Index the cache by LBID */ - for (it = fbSet.begin(); it != fbSet.end(); it++) - byLBID.insert(pair(it->lbid, it)); - - for (i = 0; i < count; i++) + for (currentExtent = 0; currentExtent < extents.size(); currentExtent++) { - extents.clear(); - err = dbrm.getExtents(oids[i], extents, true, true, true); // @Bug 3838 Include outofservice extents + EMEntry& range = extents[currentExtent]; + LBID_t lastLBID = range.range.start + (range.range.size * 1024); - if (err < 0 || (i == 0 && (extents.size() * count) > clearThreshold)) + for (currentLBID = range.range.start; currentLBID < lastLBID; currentLBID++) + { + itList = byLBID.equal_range(currentLBID); + + for (byLBID_t::iterator tmpIt = itList.first; tmpIt != itList.second; tmpIt++) { - // (The i == 0 should ensure it's not a dictionary column) - lk.unlock(); - flushCache(); - return; - } - - for (currentExtent = 0; currentExtent < extents.size(); currentExtent++) - { - EMEntry& range = extents[currentExtent]; - LBID_t lastLBID = range.range.start + (range.range.size * 1024); - - for (currentLBID = range.range.start; currentLBID < lastLBID; - currentLBID++) - { - itList = byLBID.equal_range(currentLBID); - - for (byLBID_t::iterator tmpIt = itList.first; tmpIt != itList.second; - tmpIt++) - { - fbList.erase(fFBPool[tmpIt->second->poolIdx].listLoc()); - fEmptyPoolSlots.push_back(tmpIt->second->poolIdx); - fbSet.erase(tmpIt->second); - fCacheSize--; - } - } + fbList.erase(fFBPool[tmpIt->second->poolIdx].listLoc()); + fEmptyPoolSlots.push_back(tmpIt->second->poolIdx); + fbSet.erase(tmpIt->second); + fCacheSize--; } + } } + } } void FileBufferMgr::flushPartition(const vector& oids, const set& partitions) { - DBRM dbrm; - uint32_t i; - vector extents; - int err; - uint32_t currentExtent; - LBID_t currentLBID; - typedef tr1::unordered_multimap byLBID_t; - byLBID_t byLBID; - pair itList; - filebuffer_uset_t::iterator it; - uint32_t count = oids.size(); + DBRM dbrm; + uint32_t i; + vector extents; + int err; + uint32_t currentExtent; + LBID_t currentLBID; + typedef tr1::unordered_multimap byLBID_t; + byLBID_t byLBID; + pair itList; + filebuffer_uset_t::iterator it; + uint32_t count = oids.size(); - boost::mutex::scoped_lock lk(fWLock); + boost::mutex::scoped_lock lk(fWLock); - if (fReportFrequency) + if (fReportFrequency) + { + std::set::iterator sit; + fLog << "flushPartition oids: "; + for (uint32_t i = 0; i < count; i++) { - std::set::iterator sit; - fLog << "flushPartition oids: "; - for (uint32_t i = 0; i < count; i++) - { - fLog << oids[i] << ", "; - } - fLog << "flushPartition partitions: "; - for (sit = partitions.begin(); sit != partitions.end(); ++sit) - { - fLog << (*sit).toString() << ", "; - } - fLog << endl; + fLog << oids[i] << ", "; + } + fLog << "flushPartition partitions: "; + for (sit = partitions.begin(); sit != partitions.end(); ++sit) + { + fLog << (*sit).toString() << ", "; + } + fLog << endl; + } + + if (fCacheSize == 0 || oids.size() == 0 || partitions.size() == 0) + return; + + /* Index the cache by LBID */ + for (it = fbSet.begin(); it != fbSet.end(); it++) + byLBID.insert(pair(it->lbid, it)); + + for (i = 0; i < count; i++) + { + extents.clear(); + err = dbrm.getExtents(oids[i], extents, true, true, true); // @Bug 3838 Include outofservice extents + + if (err < 0) + { + lk.unlock(); + flushCache(); // better than returning an error code to the user + return; } - if (fCacheSize == 0 || oids.size() == 0 || partitions.size() == 0) - return; - - /* Index the cache by LBID */ - for (it = fbSet.begin(); it != fbSet.end(); it++) - byLBID.insert(pair(it->lbid, it)); - - for (i = 0; i < count; i++) + for (currentExtent = 0; currentExtent < extents.size(); currentExtent++) { - extents.clear(); - err = dbrm.getExtents(oids[i], extents, true, true, true); // @Bug 3838 Include outofservice extents + EMEntry& range = extents[currentExtent]; - if (err < 0) + LogicalPartition logicalPartNum(range.dbRoot, range.partitionNum, range.segmentNum); + + if (partitions.find(logicalPartNum) == partitions.end()) + continue; + + LBID_t lastLBID = range.range.start + (range.range.size * 1024); + + for (currentLBID = range.range.start; currentLBID < lastLBID; currentLBID++) + { + itList = byLBID.equal_range(currentLBID); + + for (byLBID_t::iterator tmpIt = itList.first; tmpIt != itList.second; tmpIt++) { - lk.unlock(); - flushCache(); // better than returning an error code to the user - return; - } - - for (currentExtent = 0; currentExtent < extents.size(); currentExtent++) - { - EMEntry& range = extents[currentExtent]; - - LogicalPartition logicalPartNum(range.dbRoot, range.partitionNum, range.segmentNum); - - if (partitions.find(logicalPartNum) == partitions.end()) - continue; - - LBID_t lastLBID = range.range.start + (range.range.size * 1024); - - for (currentLBID = range.range.start; currentLBID < lastLBID; currentLBID++) - { - itList = byLBID.equal_range(currentLBID); - - for (byLBID_t::iterator tmpIt = itList.first; tmpIt != itList.second; - tmpIt++) - { - fbList.erase(fFBPool[tmpIt->second->poolIdx].listLoc()); - fEmptyPoolSlots.push_back(tmpIt->second->poolIdx); - fbSet.erase(tmpIt->second); - fCacheSize--; - } - } + fbList.erase(fFBPool[tmpIt->second->poolIdx].listLoc()); + fEmptyPoolSlots.push_back(tmpIt->second->poolIdx); + fbSet.erase(tmpIt->second); + fCacheSize--; } + } } + } } - bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const { - const HashObject_t fb(lbid, ver, 0); - const bool b = exists(fb); - return b; + const HashObject_t fb(lbid, ver, 0); + const bool b = exists(fb); + return b; } FileBuffer* FileBufferMgr::findPtr(const HashObject_t& keyFb) { - boost::mutex::scoped_lock lk(fWLock); + boost::mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(keyFb); + filebuffer_uset_iter_t it = fbSet.find(keyFb); - if (fbSet.end() != it) - { - FileBuffer* fb = &(fFBPool[it->poolIdx]); - fFBPool[it->poolIdx].listLoc()->hits++; - fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() ); - return fb; - } + if (fbSet.end() != it) + { + FileBuffer* fb = &(fFBPool[it->poolIdx]); + fFBPool[it->poolIdx].listLoc()->hits++; + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + return fb; + } - return NULL; + return NULL; } - bool FileBufferMgr::find(const HashObject_t& keyFb, FileBuffer& fb) { - bool ret = false; + bool ret = false; - boost::mutex::scoped_lock lk(fWLock); + boost::mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(keyFb); + filebuffer_uset_iter_t it = fbSet.find(keyFb); - if (fbSet.end() != it) - { - fFBPool[it->poolIdx].listLoc()->hits++; - fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() ); - fb = fFBPool[it->poolIdx]; - ret = true; - } + if (fbSet.end() != it) + { + fFBPool[it->poolIdx].listLoc()->hits++; + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + fb = fFBPool[it->poolIdx]; + ret = true; + } - return ret; + return ret; } bool FileBufferMgr::find(const HashObject_t& keyFb, void* bufferPtr) { - bool ret = false; + bool ret = false; + + if (gPMProfOn && gPMStatsPtr) +#ifdef _MSC_VER + gPMStatsPtr->markEvent(keyFb.lbid, GetCurrentThreadId(), gSession, 'L'); + +#else + gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L'); +#endif + boost::mutex::scoped_lock lk(fWLock); + + if (gPMProfOn && gPMStatsPtr) +#ifdef _MSC_VER + gPMStatsPtr->markEvent(keyFb.lbid, GetCurrentThreadId(), gSession, 'M'); + +#else + gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M'); +#endif + filebuffer_uset_iter_t it = fbSet.find(keyFb); + + if (fbSet.end() != it) + { + uint32_t idx = it->poolIdx; + + //@bug 669 LRU cache, move block to front of list as last recently used. + fFBPool[idx].listLoc()->hits++; + fbList.splice(fbList.begin(), fbList, (fFBPool[idx]).listLoc()); + lk.unlock(); + memcpy(bufferPtr, (fFBPool[idx]).getData(), 8192); if (gPMProfOn && gPMStatsPtr) #ifdef _MSC_VER - gPMStatsPtr->markEvent(keyFb.lbid, GetCurrentThreadId(), gSession, 'L'); + gPMStatsPtr->markEvent(keyFb.lbid, GetCurrentThreadId(), gSession, 'U'); #else - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L'); + gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U'); #endif - boost::mutex::scoped_lock lk(fWLock); + ret = true; + } - if (gPMProfOn && gPMStatsPtr) -#ifdef _MSC_VER - gPMStatsPtr->markEvent(keyFb.lbid, GetCurrentThreadId(), gSession, 'M'); - -#else - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M'); -#endif - filebuffer_uset_iter_t it = fbSet.find(keyFb); - - if (fbSet.end() != it) - { - uint32_t idx = it->poolIdx; - - //@bug 669 LRU cache, move block to front of list as last recently used. - fFBPool[idx].listLoc()->hits++; - fbList.splice(fbList.begin(), fbList, (fFBPool[idx]).listLoc()); - lk.unlock(); - memcpy(bufferPtr, (fFBPool[idx]).getData(), 8192); - - if (gPMProfOn && gPMStatsPtr) -#ifdef _MSC_VER - gPMStatsPtr->markEvent(keyFb.lbid, GetCurrentThreadId(), gSession, 'U'); - -#else - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U'); -#endif - ret = true; - } - - return ret; + return ret; } uint32_t FileBufferMgr::bulkFind(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** buffers, bool* wasCached, uint32_t count) { - uint32_t i, ret = 0; - filebuffer_uset_iter_t* it = (filebuffer_uset_iter_t*) alloca(count * sizeof(filebuffer_uset_iter_t)); - uint32_t* indexes = (uint32_t*) alloca(count * 4); - - if (gPMProfOn && gPMStatsPtr) - { - for (i = 0; i < count; i++) - { -#ifdef _MSC_VER - gPMStatsPtr->markEvent(lbids[i], GetCurrentThreadId(), gSession, 'L'); -#else - gPMStatsPtr->markEvent(lbids[i], pthread_self(), gSession, 'L'); -#endif - } - } - - boost::mutex::scoped_lock lk(fWLock); - - if (gPMProfOn && gPMStatsPtr) - { - for (i = 0; i < count; i++) - { -#ifdef _MSC_VER - gPMStatsPtr->markEvent(lbids[i], GetCurrentThreadId(), gSession, 'M'); -#else - gPMStatsPtr->markEvent(lbids[i], pthread_self(), gSession, 'M'); -#endif - } - } + uint32_t i, ret = 0; + filebuffer_uset_iter_t* it = (filebuffer_uset_iter_t*)alloca(count * sizeof(filebuffer_uset_iter_t)); + uint32_t* indexes = (uint32_t*)alloca(count * 4); + if (gPMProfOn && gPMStatsPtr) + { for (i = 0; i < count; i++) { - new ((void*) &it[i]) filebuffer_uset_iter_t(); - it[i] = fbSet.find(HashObject_t(lbids[i], vers[i], 0)); - - if (it[i] != fbSet.end()) - { - indexes[i] = it[i]->poolIdx; - wasCached[i] = true; - fFBPool[it[i]->poolIdx].listLoc()->hits++; - fbList.splice(fbList.begin(), fbList, (fFBPool[it[i]->poolIdx]).listLoc()); - } - else - { - wasCached[i] = false; - indexes[i] = 0; - } +#ifdef _MSC_VER + gPMStatsPtr->markEvent(lbids[i], GetCurrentThreadId(), gSession, 'L'); +#else + gPMStatsPtr->markEvent(lbids[i], pthread_self(), gSession, 'L'); +#endif } + } - lk.unlock(); + boost::mutex::scoped_lock lk(fWLock); + if (gPMProfOn && gPMStatsPtr) + { for (i = 0; i < count; i++) { - if (wasCached[i]) - { - memcpy(buffers[i], fFBPool[indexes[i]].getData(), 8192); - ret++; - - if (gPMProfOn && gPMStatsPtr) - { #ifdef _MSC_VER - gPMStatsPtr->markEvent(lbids[i], GetCurrentThreadId(), gSession, 'U'); + gPMStatsPtr->markEvent(lbids[i], GetCurrentThreadId(), gSession, 'M'); #else - gPMStatsPtr->markEvent(lbids[i], pthread_self(), gSession, 'U'); + gPMStatsPtr->markEvent(lbids[i], pthread_self(), gSession, 'M'); #endif - } - } + } + } - it[i].filebuffer_uset_iter_t::~filebuffer_uset_iter_t(); + for (i = 0; i < count; i++) + { + new ((void*)&it[i]) filebuffer_uset_iter_t(); + it[i] = fbSet.find(HashObject_t(lbids[i], vers[i], 0)); + + if (it[i] != fbSet.end()) + { + indexes[i] = it[i]->poolIdx; + wasCached[i] = true; + fFBPool[it[i]->poolIdx].listLoc()->hits++; + fbList.splice(fbList.begin(), fbList, (fFBPool[it[i]->poolIdx]).listLoc()); + } + else + { + wasCached[i] = false; + indexes[i] = 0; + } + } + + lk.unlock(); + + for (i = 0; i < count; i++) + { + if (wasCached[i]) + { + memcpy(buffers[i], fFBPool[indexes[i]].getData(), 8192); + ret++; + + if (gPMProfOn && gPMStatsPtr) + { +#ifdef _MSC_VER + gPMStatsPtr->markEvent(lbids[i], GetCurrentThreadId(), gSession, 'U'); +#else + gPMStatsPtr->markEvent(lbids[i], pthread_self(), gSession, 'U'); +#endif + } } - return ret; + it[i].filebuffer_uset_iter_t::~filebuffer_uset_iter_t(); + } + + return ret; } bool FileBufferMgr::exists(const HashObject_t& fb) const { - bool find_bool = false; - boost::mutex::scoped_lock lk(fWLock); + bool find_bool = false; + boost::mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(fb); + filebuffer_uset_iter_t it = fbSet.find(fb); - if (it != fbSet.end()) - { - find_bool = true; - fFBPool[it->poolIdx].listLoc()->hits++; - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - } + if (it != fbSet.end()) + { + find_bool = true; + fFBPool[it->poolIdx].listLoc()->hits++; + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + } - return find_bool; + return find_bool; } // default insert operation. @@ -581,286 +575,283 @@ bool FileBufferMgr::exists(const HashObject_t& fb) const int FileBufferMgr::insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data) { - int ret = 0; + int ret = 0; + if (gPMProfOn && gPMStatsPtr) +#ifdef _MSC_VER + gPMStatsPtr->markEvent(lbid, GetCurrentThreadId(), gSession, 'I'); + +#else + gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I'); +#endif + + boost::mutex::scoped_lock lk(fWLock); + + HashObject_t fbIndex(lbid, ver, 0); + filebuffer_pair_t pr = fbSet.insert(fbIndex); + + if (pr.second) + { + // It was inserted (it wasn't there before) + // Right now we have an invalid cache: we have inserted an entry with a -1 index. + // We need to fix this quickly... + fCacheSize++; + FBData_t fbdata = {lbid, ver, 0}; + fbList.push_front(fbdata); + fBlksLoaded++; + + if (fReportFrequency && (fBlksLoaded % fReportFrequency) == 0) + { + struct timespec tm; + clock_gettime(CLOCK_MONOTONIC, &tm); + fLog << "insert: " << left << fixed << ((double)(tm.tv_sec + (1.e-9 * tm.tv_nsec))) << " " << right + << setw(12) << fBlksLoaded << " " << right << setw(12) << fBlksNotUsed << endl; + } + } + else + { + // if it's a duplicate there's nothing to do if (gPMProfOn && gPMStatsPtr) #ifdef _MSC_VER - gPMStatsPtr->markEvent(lbid, GetCurrentThreadId(), gSession, 'I'); + gPMStatsPtr->markEvent(lbid, GetCurrentThreadId(), gSession, 'D'); #else - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I'); + gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D'); #endif - - boost::mutex::scoped_lock lk(fWLock); - - HashObject_t fbIndex(lbid, ver, 0); - filebuffer_pair_t pr = fbSet.insert(fbIndex); - - if (pr.second) - { - // It was inserted (it wasn't there before) - // Right now we have an invalid cache: we have inserted an entry with a -1 index. - // We need to fix this quickly... - fCacheSize++; - FBData_t fbdata = {lbid, ver, 0}; - fbList.push_front(fbdata); - fBlksLoaded++; - - if (fReportFrequency && (fBlksLoaded % fReportFrequency) == 0) - { - struct timespec tm; - clock_gettime(CLOCK_MONOTONIC, &tm); - fLog << "insert: " - << left << fixed << ((double)(tm.tv_sec + (1.e-9 * tm.tv_nsec))) << " " - << right << setw(12) << fBlksLoaded << " " - << right << setw(12) << fBlksNotUsed << endl; - } - } - else - { - // if it's a duplicate there's nothing to do - if (gPMProfOn && gPMStatsPtr) -#ifdef _MSC_VER - gPMStatsPtr->markEvent(lbid, GetCurrentThreadId(), gSession, 'D'); - -#else - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D'); -#endif - return ret; - } - - uint32_t pi = numeric_limits::max(); - - if (fCacheSize > maxCacheSize()) - { - // If the insert above caused the cache to exceed its max size, find the lru block in - // the cache and use its pool index to store the block data. - FBData_t& fbdata = fbList.back(); //the lru block - HashObject_t lastFB(fbdata.lbid, fbdata.ver, 0); - filebuffer_uset_iter_t iter = fbSet.find( lastFB ); //should be there - - idbassert(iter != fbSet.end()); - pi = iter->poolIdx; - idbassert(pi < maxCacheSize()); - idbassert(pi < fFBPool.size()); - - // set iters are always const. We are not changing the hash here, and this gets us - // the pointer we need cheaply... - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - - //replace the lru block with this block - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool[pi] = fb; - fFBPool[pi].setData(data, 8192); - fbSet.erase(iter); - - if (fbList.back().hits == 0) - fBlksNotUsed++; - - fbList.pop_back(); - fCacheSize--; - depleteCache(); - ret = 1; - } - else - { - if ( ! fEmptyPoolSlots.empty() ) - { - pi = fEmptyPoolSlots.front(); - fEmptyPoolSlots.pop_front(); - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool[pi] = fb; - fFBPool[pi].setData(data, 8192); - } - else - { - pi = fFBPool.size(); - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool.push_back(fb); - fFBPool[pi].setData(data, 8192); - } - - // See comment above - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - ret = 1; - } - - idbassert(pi < fFBPool.size()); - fFBPool[pi].listLoc(fbList.begin()); - - if (gPMProfOn && gPMStatsPtr) -#ifdef _MSC_VER - gPMStatsPtr->markEvent(lbid, GetCurrentThreadId(), gSession, 'J'); - -#else - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J'); -#endif - - idbassert(fCacheSize <= maxCacheSize()); -// idbassert(fCacheSize == fbSet.size()); -// idbassert(fCacheSize == fbList.size()); return ret; -} + } + uint32_t pi = numeric_limits::max(); + + if (fCacheSize > maxCacheSize()) + { + // If the insert above caused the cache to exceed its max size, find the lru block in + // the cache and use its pool index to store the block data. + FBData_t& fbdata = fbList.back(); // the lru block + HashObject_t lastFB(fbdata.lbid, fbdata.ver, 0); + filebuffer_uset_iter_t iter = fbSet.find(lastFB); // should be there + + idbassert(iter != fbSet.end()); + pi = iter->poolIdx; + idbassert(pi < maxCacheSize()); + idbassert(pi < fFBPool.size()); + + // set iters are always const. We are not changing the hash here, and this gets us + // the pointer we need cheaply... + HashObject_t& ref = const_cast(*pr.first); + ref.poolIdx = pi; + + // replace the lru block with this block + FileBuffer fb(lbid, ver, NULL, 0); + fFBPool[pi] = fb; + fFBPool[pi].setData(data, 8192); + fbSet.erase(iter); + + if (fbList.back().hits == 0) + fBlksNotUsed++; + + fbList.pop_back(); + fCacheSize--; + depleteCache(); + ret = 1; + } + else + { + if (!fEmptyPoolSlots.empty()) + { + pi = fEmptyPoolSlots.front(); + fEmptyPoolSlots.pop_front(); + FileBuffer fb(lbid, ver, NULL, 0); + fFBPool[pi] = fb; + fFBPool[pi].setData(data, 8192); + } + else + { + pi = fFBPool.size(); + FileBuffer fb(lbid, ver, NULL, 0); + fFBPool.push_back(fb); + fFBPool[pi].setData(data, 8192); + } + + // See comment above + HashObject_t& ref = const_cast(*pr.first); + ref.poolIdx = pi; + ret = 1; + } + + idbassert(pi < fFBPool.size()); + fFBPool[pi].listLoc(fbList.begin()); + + if (gPMProfOn && gPMStatsPtr) +#ifdef _MSC_VER + gPMStatsPtr->markEvent(lbid, GetCurrentThreadId(), gSession, 'J'); + +#else + gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J'); +#endif + + idbassert(fCacheSize <= maxCacheSize()); + // idbassert(fCacheSize == fbSet.size()); + // idbassert(fCacheSize == fbList.size()); + return ret; +} void FileBufferMgr::depleteCache() { - for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i) - { - FBData_t fbdata(fbList.back()); //the lru block - HashObject_t lastFB(fbdata.lbid, fbdata.ver, 0); - filebuffer_uset_iter_t iter = fbSet.find( lastFB ); + for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i) + { + FBData_t fbdata(fbList.back()); // the lru block + HashObject_t lastFB(fbdata.lbid, fbdata.ver, 0); + filebuffer_uset_iter_t iter = fbSet.find(lastFB); - idbassert(iter != fbSet.end()); - uint32_t idx = iter->poolIdx; - idbassert(idx < fFBPool.size()); - //Save position in FileBuffer pool for reuse. - fEmptyPoolSlots.push_back(idx); - fbSet.erase(iter); + idbassert(iter != fbSet.end()); + uint32_t idx = iter->poolIdx; + idbassert(idx < fFBPool.size()); + // Save position in FileBuffer pool for reuse. + fEmptyPoolSlots.push_back(idx); + fbSet.erase(iter); - if (fbList.back().hits == 0) - fBlksNotUsed++; + if (fbList.back().hits == 0) + fBlksNotUsed++; - fbList.pop_back(); - fCacheSize--; - } + fbList.pop_back(); + fCacheSize--; + } } ostream& FileBufferMgr::formatLRUList(ostream& os) const { - filebuffer_list_t::const_iterator iter = fbList.begin(); - filebuffer_list_t::const_iterator end = fbList.end(); + filebuffer_list_t::const_iterator iter = fbList.begin(); + filebuffer_list_t::const_iterator end = fbList.end(); - while (iter != end) - { - os << iter->lbid << '\t' << iter->ver << endl; - ++iter; - } + while (iter != end) + { + os << iter->lbid << '\t' << iter->ver << endl; + ++iter; + } - return os; + return os; } // puts the new entry at the front of the list void FileBufferMgr::updateLRU(const FBData_t& f) { - if (fCacheSize > maxCacheSize()) - { - list::iterator last = fbList.end(); - last--; - FBData_t& fbdata = *last; - HashObject_t lastFB(fbdata.lbid, fbdata.ver, 0); - filebuffer_uset_iter_t iter = fbSet.find(lastFB); - fEmptyPoolSlots.push_back(iter->poolIdx); + if (fCacheSize > maxCacheSize()) + { + list::iterator last = fbList.end(); + last--; + FBData_t& fbdata = *last; + HashObject_t lastFB(fbdata.lbid, fbdata.ver, 0); + filebuffer_uset_iter_t iter = fbSet.find(lastFB); + fEmptyPoolSlots.push_back(iter->poolIdx); - if (fbdata.hits == 0) - fBlksNotUsed++; + if (fbdata.hits == 0) + fBlksNotUsed++; - fbSet.erase(iter); - fbList.splice(fbList.begin(), fbList, last); - fbdata = f; - fCacheSize--; - //cout << "booted an entry\n"; - } - else - { - //cout << "new entry\n"; - fbList.push_front(f); - } + fbSet.erase(iter); + fbList.splice(fbList.begin(), fbList, last); + fbdata = f; + fCacheSize--; + // cout << "booted an entry\n"; + } + else + { + // cout << "new entry\n"; + fbList.push_front(f); + } } uint32_t FileBufferMgr::doBlockCopy(const BRM::LBID_t& lbid, const BRM::VER_t& ver, const uint8_t* data) { - uint32_t poolIdx; + uint32_t poolIdx; - if (!fEmptyPoolSlots.empty()) - { - poolIdx = fEmptyPoolSlots.front(); - fEmptyPoolSlots.pop_front(); - } - else - { - poolIdx = fFBPool.size(); - fFBPool.resize(poolIdx + 1); //shouldn't trigger a 'real' resize b/c of the reserve call - } + if (!fEmptyPoolSlots.empty()) + { + poolIdx = fEmptyPoolSlots.front(); + fEmptyPoolSlots.pop_front(); + } + else + { + poolIdx = fFBPool.size(); + fFBPool.resize(poolIdx + 1); // shouldn't trigger a 'real' resize b/c of the reserve call + } - fFBPool[poolIdx].Lbid(lbid); - fFBPool[poolIdx].Verid(ver); - fFBPool[poolIdx].setData(data); - return poolIdx; + fFBPool[poolIdx].Lbid(lbid); + fFBPool[poolIdx].Verid(ver); + fFBPool[poolIdx].setData(data); + return poolIdx; } int FileBufferMgr::bulkInsert(const vector& ops) { - uint32_t i; - int32_t pi; - int ret = 0; + uint32_t i; + int32_t pi; + int ret = 0; - boost::mutex::scoped_lock lk(fWLock); + boost::mutex::scoped_lock lk(fWLock); + + if (fReportFrequency) + { + fLog << "bulkInsert: "; + } + + for (i = 0; i < ops.size(); i++) + { + const CacheInsert_t& op = ops[i]; + + if (gPMProfOn && gPMStatsPtr) +#ifdef _MSC_VER + gPMStatsPtr->markEvent(op.lbid, GetCurrentThreadId(), gSession, 'I'); + +#else + gPMStatsPtr->markEvent(op.lbid, pthread_self(), gSession, 'I'); +#endif + + HashObject_t fbIndex(op.lbid, op.ver, 0); + filebuffer_pair_t pr = fbSet.insert(fbIndex); + + if (!pr.second) + { + if (gPMProfOn && gPMStatsPtr) +#ifdef _MSC_VER + gPMStatsPtr->markEvent(op.lbid, GetCurrentThreadId(), gSession, 'D'); + +#else + gPMStatsPtr->markEvent(op.lbid, pthread_self(), gSession, 'D'); +#endif + continue; + } if (fReportFrequency) { - fLog << "bulkInsert: "; + fLog << op.lbid << " " << op.ver << ", "; } + fCacheSize++; + fBlksLoaded++; + FBData_t fbdata = {op.lbid, op.ver, 0}; + updateLRU(fbdata); + pi = doBlockCopy(op.lbid, op.ver, op.data); - for (i = 0; i < ops.size(); i++) { - const CacheInsert_t &op = ops[i]; + HashObject_t& ref = const_cast(*pr.first); + ref.poolIdx = pi; + fFBPool[pi].listLoc(fbList.begin()); - if (gPMProfOn && gPMStatsPtr) + if (gPMProfOn && gPMStatsPtr) #ifdef _MSC_VER - gPMStatsPtr->markEvent(op.lbid, GetCurrentThreadId(), gSession, 'I'); + gPMStatsPtr->markEvent(op.lbid, GetCurrentThreadId(), gSession, 'J'); #else - gPMStatsPtr->markEvent(op.lbid, pthread_self(), gSession, 'I'); + gPMStatsPtr->markEvent(op.lbid, pthread_self(), gSession, 'J'); #endif + ret++; + } + if (fReportFrequency) + { + fLog << endl; + } + idbassert(fCacheSize <= maxCacheSize()); - HashObject_t fbIndex(op.lbid, op.ver, 0); - filebuffer_pair_t pr = fbSet.insert(fbIndex); - - if (!pr.second) - { - if (gPMProfOn && gPMStatsPtr) -#ifdef _MSC_VER - gPMStatsPtr->markEvent(op.lbid, GetCurrentThreadId(), gSession, 'D'); - -#else - gPMStatsPtr->markEvent(op.lbid, pthread_self(), gSession, 'D'); -#endif - continue; - } - - if (fReportFrequency) - { - fLog << op.lbid << " " << op.ver << ", "; - } - fCacheSize++; - fBlksLoaded++; - FBData_t fbdata = {op.lbid, op.ver, 0}; - updateLRU(fbdata); - pi = doBlockCopy(op.lbid, op.ver, op.data); - - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - fFBPool[pi].listLoc(fbList.begin()); - - if (gPMProfOn && gPMStatsPtr) -#ifdef _MSC_VER - gPMStatsPtr->markEvent(op.lbid, GetCurrentThreadId(), gSession, 'J'); - -#else - gPMStatsPtr->markEvent(op.lbid, pthread_self(), gSession, 'J'); -#endif - ret++; - } - if (fReportFrequency) - { - fLog << endl; - } - idbassert(fCacheSize <= maxCacheSize()); - - return ret; + return ret; } - -} +} // namespace dbbc diff --git a/primitives/blockcache/filebuffermgr.h b/primitives/blockcache/filebuffermgr.h index 7a64ab875..0743195d3 100644 --- a/primitives/blockcache/filebuffermgr.h +++ b/primitives/blockcache/filebuffermgr.h @@ -41,203 +41,204 @@ #include "rwlock_local.h" /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** - * @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and list. + * @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and + *list. * **/ namespace dbbc { - /** * @brief used as the hasher algorithm for the unordered_set used to store the disk blocks **/ struct FileBufferIndex { - FileBufferIndex(BRM::LBID_t l, BRM::VER_t v, uint32_t p) : lbid(l), ver(v), poolIdx(p) {} - BRM::LBID_t lbid; - BRM::VER_t ver; - uint32_t poolIdx; + FileBufferIndex(BRM::LBID_t l, BRM::VER_t v, uint32_t p) : lbid(l), ver(v), poolIdx(p) + { + } + BRM::LBID_t lbid; + BRM::VER_t ver; + uint32_t poolIdx; }; struct CacheInsert_t { - CacheInsert_t(const BRM::LBID_t& l, const BRM::VER_t& v, const uint8_t* d) : - lbid(l), ver(v), data(d) { } - BRM::LBID_t lbid; - BRM::VER_t ver; - const uint8_t* data; + CacheInsert_t(const BRM::LBID_t& l, const BRM::VER_t& v, const uint8_t* d) : lbid(l), ver(v), data(d) + { + } + BRM::LBID_t lbid; + BRM::VER_t ver; + const uint8_t* data; }; typedef FileBufferIndex HashObject_t; class bcHasher { -public: - inline size_t operator()(const HashObject_t& rhs) const - { - return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL)); - } + public: + inline size_t operator()(const HashObject_t& rhs) const + { + return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL)); + } }; class bcEqual { -public: - inline bool operator()(const HashObject_t& f1, const HashObject_t& f2) const - { - return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver)); - } + public: + inline bool operator()(const HashObject_t& f1, const HashObject_t& f2) const + { + return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver)); + } }; inline bool operator<(const HashObject_t& f1, const HashObject_t& f2) { - return ((f1.lbid < f2.lbid) || (f1.lbid == f2.lbid && f1.ver < f2.ver)); + return ((f1.lbid < f2.lbid) || (f1.lbid == f2.lbid && f1.ver < f2.ver)); } - class FileBufferMgr { + public: + typedef std::tr1::unordered_set filebuffer_uset_t; + typedef std::tr1::unordered_set::const_iterator filebuffer_uset_iter_t; + typedef std::pair filebuffer_pair_t; // return type for insert -public: + typedef std::deque emptylist_t; - typedef std::tr1::unordered_set filebuffer_uset_t; - typedef std::tr1::unordered_set::const_iterator filebuffer_uset_iter_t; - typedef std::pair filebuffer_pair_t; // return type for insert + /** + * @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz + **/ - typedef std::deque emptylist_t; + FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0); - /** - * @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz - **/ + /** + * @brief default dtor + **/ + virtual ~FileBufferMgr(); - FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0); + /** + * @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return + *FALSE. + **/ + bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const; - /** - * @brief default dtor - **/ - virtual ~FileBufferMgr(); + /** + * @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache + *otherwise return FALSE. + **/ + bool exists(const HashObject_t& fb) const; - /** - * @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE. - **/ - bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const; + /** + * @brief add the Disk Block reference by fb into the Disk Block Buffer Cache + **/ + int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data); - /** - * @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE. - **/ - bool exists(const HashObject_t& fb) const; + int bulkInsert(const std::vector&); - /** - * @brief add the Disk Block reference by fb into the Disk Block Buffer Cache - **/ - int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data); + /** + * @brief returns the total number of Disk Blocks in the Cache + **/ + uint32_t size() const + { + return fbSet.size(); + } - int bulkInsert(const std::vector&); + /** + * @brief + **/ + void flushCache(); - /** - * @brief returns the total number of Disk Blocks in the Cache - **/ - uint32_t size() const - { - return fbSet.size(); - } + /** + * @brief + **/ + void flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver); - /** - * @brief - **/ - void flushCache(); + /** + * @brief + **/ + void flushMany(const LbidAtVer* laVptr, uint32_t cnt); - /** - * @brief - **/ - void flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver); + /** + * @brief flush all versions + **/ + void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt); - /** - * @brief - **/ - void flushMany(const LbidAtVer* laVptr, uint32_t cnt); + void flushOIDs(const uint32_t* oids, uint32_t count); + void flushPartition(const std::vector& oids, const std::set& partitions); - /** - * @brief flush all versions - **/ - void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt); + /** + * @brief return the disk Block referenced by fb + **/ - void flushOIDs(const uint32_t* oids, uint32_t count); - void flushPartition(const std::vector& oids, const std::set& partitions); + FileBuffer* findPtr(const HashObject_t& keyFb); - /** - * @brief return the disk Block referenced by fb - **/ + bool find(const HashObject_t& keyFb, FileBuffer& fb); - FileBuffer* findPtr(const HashObject_t& keyFb); + /** + * @brief return the disk Block referenced by bufferPtr + **/ - bool find(const HashObject_t& keyFb, FileBuffer& fb); + bool find(const HashObject_t& keyFb, void* bufferPtr); + uint32_t bulkFind(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** buffers, bool* wasCached, + uint32_t blockCount); - /** - * @brief return the disk Block referenced by bufferPtr - **/ + uint32_t maxCacheSize() const + { + return fMaxNumBlocks; + } - bool find(const HashObject_t& keyFb, void* bufferPtr); - uint32_t bulkFind(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** buffers, - bool* wasCached, uint32_t blockCount); + uint32_t listSize() const + { + return fbList.size(); + } - uint32_t maxCacheSize() const - { - return fMaxNumBlocks; - } + const filebuffer_uset_iter_t end() const + { + return fbSet.end(); + } - uint32_t listSize() const - { - return fbList.size(); - } + void setReportingFrequency(const uint32_t d); + uint32_t ReportingFrequency() const + { + return fReportFrequency; + } - const filebuffer_uset_iter_t end() const - { - return fbSet.end(); - } + std::ostream& formatLRUList(std::ostream& os) const; - void setReportingFrequency(const uint32_t d); - uint32_t ReportingFrequency() const - { - return fReportFrequency; - } + private: + uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list + uint32_t fBlockSz; // size in bytes size of a data block - probably 8 - std::ostream& formatLRUList(std::ostream& os) const; + mutable boost::mutex fWLock; + mutable filebuffer_uset_t fbSet; -private: + mutable filebuffer_list_t fbList; // rename this + uint32_t fCacheSize; - uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list - uint32_t fBlockSz; // size in bytes size of a data block - probably 8 + FileBufferPool_t fFBPool; // vector + uint32_t fDeleteBlocks; + emptylist_t fEmptyPoolSlots; // keep track of FBPool slots that can be reused - mutable boost::mutex fWLock; - mutable filebuffer_uset_t fbSet; + void depleteCache(); + uint64_t fBlksLoaded; // number of blocks inserted into cache + uint64_t fBlksNotUsed; // number of blocks inserted and not used + uint64_t fReportFrequency; // how many blocks are read between reports + std::ofstream fLog; + config::Config* fConfig; - mutable filebuffer_list_t fbList; // rename this - uint32_t fCacheSize; + // do not implement + FileBufferMgr(const FileBufferMgr& fbm); + const FileBufferMgr& operator=(const FileBufferMgr& fbm); - FileBufferPool_t fFBPool; // vector - uint32_t fDeleteBlocks; - emptylist_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused - - void depleteCache(); - uint64_t fBlksLoaded; // number of blocks inserted into cache - uint64_t fBlksNotUsed; // number of blocks inserted and not used - uint64_t fReportFrequency; // how many blocks are read between reports - std::ofstream fLog; - config::Config* fConfig; - - // do not implement - FileBufferMgr(const FileBufferMgr& fbm); - const FileBufferMgr& operator =(const FileBufferMgr& fbm); - - // used by bulkInsert - void updateLRU(const FBData_t& f); - uint32_t doBlockCopy(const BRM::LBID_t& lbid, const BRM::VER_t& ver, const uint8_t* data); + // used by bulkInsert + void updateLRU(const FBData_t& f); + uint32_t doBlockCopy(const BRM::LBID_t& lbid, const BRM::VER_t& ver, const uint8_t* data); }; -} +} // namespace dbbc #endif diff --git a/primitives/blockcache/filerequest.cpp b/primitives/blockcache/filerequest.cpp index dc63a246f..3774c4559 100644 --- a/primitives/blockcache/filerequest.cpp +++ b/primitives/blockcache/filerequest.cpp @@ -29,68 +29,82 @@ using namespace std; namespace dbbc { - -fileRequest::fileRequest() : - data(0), fLBID(-1), fVer(-1), fFlg(false), fTxn(-1), fRqstType(LBIDREQUEST), fCompType(0), - cache(true), wasVersioned(false) +fileRequest::fileRequest() + : data(0) + , fLBID(-1) + , fVer(-1) + , fFlg(false) + , fTxn(-1) + , fRqstType(LBIDREQUEST) + , fCompType(0) + , cache(true) + , wasVersioned(false) { - init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus + init(); // resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus } -fileRequest::fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType, - uint8_t* ptr, bool cacheIt) : - data(ptr), fLBID(lbid), fVer(ver), fFlg(flg), fTxn(txn), fRqstType(LBIDREQUEST), fCompType(compType), - cache(cacheIt), wasVersioned(false) +fileRequest::fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, + int compType, uint8_t* ptr, bool cacheIt) + : data(ptr) + , fLBID(lbid) + , fVer(ver) + , fFlg(flg) + , fTxn(txn) + , fRqstType(LBIDREQUEST) + , fCompType(compType) + , cache(cacheIt) + , wasVersioned(false) { - init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus - fLength = 1; + init(); // resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus + fLength = 1; } -fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType) : - data(0), fLBID(range.start), fVer(ver), fFlg(false), fTxn(txn), fLength(range.size), - fRqstType(RANGEREQUEST), fCompType(compType), cache(true), wasVersioned(false) +fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, + int compType) + : data(0) + , fLBID(range.start) + , fVer(ver) + , fFlg(false) + , fTxn(txn) + , fLength(range.size) + , fRqstType(RANGEREQUEST) + , fCompType(compType) + , cache(true) + , wasVersioned(false) { - init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus - fLength = range.size; + init(); // resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus + fLength = range.size; } fileRequest::fileRequest(const fileRequest& blk) { - fLBID = blk.fLBID; - fVer = blk.fVer; - fTxn = blk.fTxn; - fFlg = blk.fFlg; - fRqstType = blk.fRqstType; - fRqstStatusString = blk.fRqstStatusString; - data = blk.data; - fCompType = blk.fCompType; - cache = blk.cache; - wasVersioned = blk.wasVersioned; - init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus + fLBID = blk.fLBID; + fVer = blk.fVer; + fTxn = blk.fTxn; + fFlg = blk.fFlg; + fRqstType = blk.fRqstType; + fRqstStatusString = blk.fRqstStatusString; + data = blk.data; + fCompType = blk.fCompType; + cache = blk.cache; + wasVersioned = blk.wasVersioned; + init(); // resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus } void fileRequest::init() { - fFRPredicate = INIT; - fLength = 0; - fblksRead = 0; - fblksLoaded = 0; - fRqstStatus = SUCCESSFUL; + fFRPredicate = INIT; + fLength = 0; + fblksRead = 0; + fblksLoaded = 0; + fRqstStatus = SUCCESSFUL; } ostream& operator<<(ostream& os, const fileRequest& rhs) { - os - << "LBID: " << rhs.fLBID - << " ver: " << rhs.fVer - << " Txn: " << rhs.fTxn - << " len: " << rhs.fLength - << " read: " << rhs.fblksRead - << " load: " << rhs.fblksLoaded - << " ct: " << rhs.fCompType - ; - return os; -} - + os << "LBID: " << rhs.fLBID << " ver: " << rhs.fVer << " Txn: " << rhs.fTxn << " len: " << rhs.fLength + << " read: " << rhs.fblksRead << " load: " << rhs.fblksLoaded << " ct: " << rhs.fCompType; + return os; } +} // namespace dbbc diff --git a/primitives/blockcache/filerequest.h b/primitives/blockcache/filerequest.h index 3dcf967a9..19bd9256f 100644 --- a/primitives/blockcache/filerequest.h +++ b/primitives/blockcache/filerequest.h @@ -35,7 +35,7 @@ #include "brmtypes.h" /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** @@ -43,287 +43,283 @@ **/ namespace dbbc { - class fileRequest { + public: + /** + * @brief default ctor + **/ + fileRequest(); -public: + /** + * @brief copy constructor + **/ + fileRequest(const fileRequest& blk); - /** - * @brief default ctor - **/ - fileRequest(); + /** + * @brief request for the disk block lbid@ver + * note, useCache tells IOManager to cache the loaded blocks. + **/ + fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType, + uint8_t* ptr = 0, bool useCache = true); - /** - * @brief copy constructor - **/ - fileRequest(const fileRequest& blk); + /** + * @brief request a range of disk blocks + **/ + fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType); - /** - * @brief request for the disk block lbid@ver - * note, useCache tells IOManager to cache the loaded blocks. - **/ - fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType, - uint8_t* ptr = 0, bool useCache = true); + /** + * @brief class dtor + **/ + virtual ~fileRequest(){}; - /** - * @brief request a range of disk blocks - **/ - fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType); + /** + * @brief less-than operator + **/ + bool operator<(const fileRequest& rhs) const + { + return fLength < rhs.fLength; + } - /** - * @brief class dtor - **/ - virtual ~fileRequest() {}; + /** + * @brief greater-than operator + **/ + bool operator>(const fileRequest& rhs) const + { + return fLength > rhs.fLength; + } - /** - * @brief less-than operator - **/ - bool operator< (const fileRequest& rhs) const - { - return fLength < rhs.fLength; - } + /** + * @brief equality operator + **/ + bool operator==(const fileRequest& rhs) const + { + return fLength == rhs.fLength; + } - /** - * @brief greater-than operator - **/ - bool operator> (const fileRequest& rhs) const - { - return fLength > rhs.fLength; - } + enum request_status_enum + { + SUCCESSFUL, + FAILED, + BRM_LOOKUP_ERROR, + FS_EINVAL, + FS_ENOENT + }; - /** - * @brief equality operator - **/ - bool operator== (const fileRequest& rhs) const - { - return fLength == rhs.fLength; - } + enum request_type_enum + { + LBIDREQUEST, + RANGEREQUEST + }; - enum request_status_enum - { - SUCCESSFUL, - FAILED, - BRM_LOOKUP_ERROR, - FS_EINVAL, - FS_ENOENT - }; + /** + * @brief used to manage request processing synchronzation + **/ + enum predicate_status_enum + { + INIT, + SENDING, + READING, + COMPLETE, + STOP + }; - enum request_type_enum - { - LBIDREQUEST, - RANGEREQUEST - }; + /** + * @brief lbid requested + **/ + BRM::LBID_t Lbid() const + { + return fLBID; + } - /** - * @brief used to manage request processing synchronzation - **/ - enum predicate_status_enum - { - INIT, - SENDING, - READING, - COMPLETE, - STOP - }; + /** + * @brief version of the lbid requested + **/ + const BRM::QueryContext Ver() const + { + return fVer; + } - /** - * @brief lbid requested - **/ - BRM::LBID_t Lbid() const - { - return fLBID; - } + /** + * @brief VBBM flag of the LBID/Ver + **/ + bool Flg() const + { + return fFlg; + } - /** - * @brief version of the lbid requested - **/ - const BRM::QueryContext Ver() const - { - return fVer; - } + BRM::VER_t Txn() const + { + return fTxn; + } - /** - * @brief VBBM flag of the LBID/Ver - **/ - bool Flg() const - { - return fFlg; - } + int CompType() const + { + return fCompType; + } - BRM::VER_t Txn() const - { - return fTxn; - } + /** + * @brief number of blocks requested + **/ + uint32_t BlocksRequested() const + { + return fLength; + } - int CompType() const - { - return fCompType; - } + /** + * @brief setter for blocks requested + **/ + void BlocksRequested(const int l) + { + fLength = l; + } - /** - * @brief number of blocks requested - **/ - uint32_t BlocksRequested() const - { - return fLength; - } + /** + * @brief number of blocks read from disk + **/ + uint32_t BlocksRead() const + { + return fblksRead; + } + uint32_t BlocksLoaded() const + { + return fblksLoaded; + } - /** - * @brief setter for blocks requested - **/ - void BlocksRequested(const int l) - { - fLength = l; - } + /** + * @brief setter for blocks read from disk + **/ + void BlocksRead(const int l) + { + fblksRead = l; + } + void BlocksLoaded(const int l) + { + fblksLoaded = l; + } - /** - * @brief number of blocks read from disk - **/ - uint32_t BlocksRead() const - { - return fblksRead; - } - uint32_t BlocksLoaded() const - { - return fblksLoaded; - } + /** + * @brief did the request succeed for fail (0-success; else failure) + **/ + int RequestStatus() const + { + return fRqstStatus; + } - /** - * @brief setter for blocks read from disk - **/ - void BlocksRead(const int l) - { - fblksRead = l; - } - void BlocksLoaded(const int l) - { - fblksLoaded = l; - } + /** + * @brief setter for the request status + **/ + void RequestStatus(int s) + { + fRqstStatus = s; + } - /** - * @brief did the request succeed for fail (0-success; else failure) - **/ - int RequestStatus() const - { - return fRqstStatus; - } + void CompType(int compType) + { + fCompType = compType; + } - /** - * @brief setter for the request status - **/ - void RequestStatus(int s) - { - fRqstStatus = s; - } + /** + * @brief if request failed, this is the error message string + **/ + std::string RequestStatusStr() const + { + return fRqstStatusString; + } - void CompType(int compType) - { - fCompType = compType; - } + /** + * @brief setter for the request status error message string + **/ + void RequestStatusStr(const std::string& s) + { + fRqstStatusString = s; + } - /** - * @brief if request failed, this is the error message string - **/ - std::string RequestStatusStr() const - { - return fRqstStatusString; - } + /** + * @brief return BLOCK or RANGE requested + **/ + int RequestType() const + { + return fRqstType; + } - /** - * @brief setter for the request status error message string - **/ - void RequestStatusStr(const std::string& s) - { - fRqstStatusString = s; - } + /** + * @brief mutex to control synchronzation of request processing + **/ + boost::mutex& frMutex() const + { + return fFRMutex; + } - /** - * @brief return BLOCK or RANGE requested - **/ - int RequestType() const - { - return fRqstType; - } + /** + * @brief condition variable. signal when request is complete + **/ + boost::condition& frCond() const + { + return fFRCond; + } - /** - * @brief mutex to control synchronzation of request processing - **/ - boost::mutex& frMutex() const - { - return fFRMutex; - } + /** + * @brief + **/ + const enum predicate_status_enum& frPredicate() const + { + return fFRPredicate; + } - /** - * @brief condition variable. signal when request is complete - **/ - boost::condition& frCond() const - { - return fFRCond; - } + /** + * @brief setter for the predicate + **/ + void SetPredicate(const enum predicate_status_enum& p) + { + fFRPredicate = p; + } - /** - * @brief - **/ - const enum predicate_status_enum& frPredicate() const - { - return fFRPredicate; - } + uint8_t* data; - /** - * @brief setter for the predicate - **/ - void SetPredicate(const enum predicate_status_enum& p) - { - fFRPredicate = p; - } + friend std::ostream& operator<<(std::ostream& os, const fileRequest& rhs); - uint8_t* data; + // tells IOManager to cache the loaded blocks or not + bool useCache() + { + return cache; + } + void useCache(bool c) + { + cache = c; + } - friend std::ostream& operator<<(std::ostream& os, const fileRequest& rhs); + bool versioned() + { + return wasVersioned; + } + void versioned(bool b) + { + wasVersioned = b; + } - // tells IOManager to cache the loaded blocks or not - bool useCache() - { - return cache; - } - void useCache(bool c) - { - cache = c; - } + private: + void init(); - bool versioned() - { - return wasVersioned; - } - void versioned(bool b) - { - wasVersioned = b; - } - -private: - void init(); - - BRM::LBID_t fLBID; - BRM::QueryContext fVer; - bool fFlg; - BRM::VER_t fTxn; - mutable boost::mutex fFRMutex; - mutable boost::condition fFRCond; - predicate_status_enum fFRPredicate; - uint32_t fLength; // lbids requested - uint32_t fblksRead; // lbids read - uint32_t fblksLoaded; // lbids loaded into cache - int fRqstStatus; - std::string fRqstStatusString; - enum request_type_enum fRqstType; - int fCompType; - bool cache; - bool wasVersioned; + BRM::LBID_t fLBID; + BRM::QueryContext fVer; + bool fFlg; + BRM::VER_t fTxn; + mutable boost::mutex fFRMutex; + mutable boost::condition fFRCond; + predicate_status_enum fFRPredicate; + uint32_t fLength; // lbids requested + uint32_t fblksRead; // lbids read + uint32_t fblksLoaded; // lbids loaded into cache + int fRqstStatus; + std::string fRqstStatusString; + enum request_type_enum fRqstType; + int fCompType; + bool cache; + bool wasVersioned; }; -} +} // namespace dbbc #endif // vim:ts=4 sw=4: - diff --git a/primitives/blockcache/fsutils.cpp b/primitives/blockcache/fsutils.cpp index 1503873c1..5fc3b8052 100644 --- a/primitives/blockcache/fsutils.cpp +++ b/primitives/blockcache/fsutils.cpp @@ -32,91 +32,88 @@ namespace fs = boost::filesystem; namespace { - const string resolveInDir(const string& dir, const string& name) { - idbassert(!dir.empty() && !name.empty()); - string ret; - fs::path path(dir); + idbassert(!dir.empty() && !name.empty()); + string ret; + fs::path path(dir); - if (!fs::exists(path)) - return ret; + if (!fs::exists(path)) + return ret; - idbassert(fs::exists(path)); - path /= name; + idbassert(fs::exists(path)); + path /= name; - if (!fs::exists(path)) - return ret; + if (!fs::exists(path)) + return ret; - idbassert(fs::exists(path)); + idbassert(fs::exists(path)); #ifndef _MSC_VER - if (!fs::is_symlink(path)) - return ret; - - idbassert(fs::is_symlink(path)); - char* realname = (char*)alloca(PATH_MAX + 1); - ssize_t realnamelen = readlink(path.string().c_str(), realname, PATH_MAX); - - if (realnamelen <= 0) - return ret; - - realname[realnamelen] = 0; - fs::path linkname(realname); - fs::path realpath("/dev"); - realpath /= linkname.filename(); - ret = realpath.string(); -#endif + if (!fs::is_symlink(path)) return ret; + + idbassert(fs::is_symlink(path)); + char* realname = (char*)alloca(PATH_MAX + 1); + ssize_t realnamelen = readlink(path.string().c_str(), realname, PATH_MAX); + + if (realnamelen <= 0) + return ret; + + realname[realnamelen] = 0; + fs::path linkname(realname); + fs::path realpath("/dev"); + realpath /= linkname.filename(); + ret = realpath.string(); +#endif + return ret; } inline const string label2dev(const string& name) { - return resolveInDir("/dev/disk/by-label", name); + return resolveInDir("/dev/disk/by-label", name); } inline const string uuid2dev(const string& name) { - return resolveInDir("/dev/disk/by-uuid", name); + return resolveInDir("/dev/disk/by-uuid", name); } -} +} // namespace namespace fsutils { - const string symname2devname(const string& sympath) { - string ret; + string ret; #ifndef _MSC_VER - typedef boost::tokenizer > tokenizer; - boost::char_separator sep("="); - tokenizer tokens(sympath, sep); - tokenizer::iterator tok_iter = tokens.begin(); + typedef boost::tokenizer > tokenizer; + boost::char_separator sep("="); + tokenizer tokens(sympath, sep); + tokenizer::iterator tok_iter = tokens.begin(); - idbassert(tok_iter != tokens.end()); - string symtype = *tok_iter; + idbassert(tok_iter != tokens.end()); + string symtype = *tok_iter; - if (symtype != "LABEL" && symtype != "UUID") - return ret; + if (symtype != "LABEL" && symtype != "UUID") + return ret; - idbassert(symtype == "LABEL" || symtype == "UUID"); + idbassert(symtype == "LABEL" || symtype == "UUID"); - ++tok_iter; - idbassert(tok_iter != tokens.end()); - string symname = *tok_iter; + ++tok_iter; + idbassert(tok_iter != tokens.end()); + string symname = *tok_iter; - ++tok_iter; - idbassert(tok_iter == tokens.end()); + ++tok_iter; + idbassert(tok_iter == tokens.end()); - if (symtype == "LABEL") - ret = label2dev(symname); - else if (symtype == "UUID") - ret = uuid2dev(symname); + if (symtype == "LABEL") + ret = label2dev(symname); + else if (symtype == "UUID") + ret = uuid2dev(symname); #endif - return ret; -} - + return ret; } +} // namespace fsutils diff --git a/primitives/blockcache/fsutils.h b/primitives/blockcache/fsutils.h index 49face7d9..3d16b08cb 100644 --- a/primitives/blockcache/fsutils.h +++ b/primitives/blockcache/fsutils.h @@ -26,4 +26,3 @@ const std::string symname2devname(const std::string& sympath); } #endif - diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index c6d8f445a..81eb787c6 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -113,21 +113,20 @@ namespace primitiveprocessor extern Logger* mlp; extern int directIOFlag; extern int noVB; -} +} // namespace primitiveprocessor #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif namespace { - using namespace dbbc; using namespace std; @@ -137,17 +136,15 @@ const std::string boldStop = "\033[0;39m"; const uint32_t MAX_OPEN_FILES = 16384; const uint32_t DECREASE_OPEN_FILES = 4096; -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } struct IOMThreadArg { - ioManager* iom; - int32_t thdId; + ioManager* iom; + int32_t thdId; }; typedef IOMThreadArg IOMThreadArg_t; @@ -155,89 +152,80 @@ typedef IOMThreadArg IOMThreadArg_t; /* structures shared across all iomanagers */ class FdEntry { -public: - FdEntry() : oid(0), dbroot(0), partNum(0), segNum(0), - fp(0), c(0), inUse(0), compType(0) - { - cmpMTime = 0; - } + public: + FdEntry() : oid(0), dbroot(0), partNum(0), segNum(0), fp(0), c(0), inUse(0), compType(0) + { + cmpMTime = 0; + } - FdEntry(const BRM::OID_t o, const uint16_t d, const uint32_t p, const uint16_t s, const int ct, IDBDataFile* f) : - oid(o), dbroot(d), partNum(p), segNum(s), fp(f), c(0), inUse(0), compType(0) - { - cmpMTime = 0; + FdEntry(const BRM::OID_t o, const uint16_t d, const uint32_t p, const uint16_t s, const int ct, + IDBDataFile* f) + : oid(o), dbroot(d), partNum(p), segNum(s), fp(f), c(0), inUse(0), compType(0) + { + cmpMTime = 0; - if (oid >= 1000) - compType = ct; - } + if (oid >= 1000) + compType = ct; + } - ~FdEntry() - { - delete fp; - fp = 0; - } + ~FdEntry() + { + delete fp; + fp = 0; + } - BRM::OID_t oid; - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - IDBDataFile* fp; - uint32_t c; - int inUse; + BRM::OID_t oid; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + IDBDataFile* fp; + uint32_t c; + int inUse; - CompChunkPtrList ptrList; + CompChunkPtrList ptrList; - int compType; - bool isCompressed() const - { - return (oid >= 1000 && compType != 0); - } - time_t cmpMTime; - friend ostream& operator<<(ostream& out, const FdEntry& o) - { - out << " o: " << o.oid - << " f: " << o.fp - << " d: " << o.dbroot - << " p: " << o.partNum - << " s: " << o.segNum - << " c: " << o.c - << " t: " << o.compType - << " m: " << o.cmpMTime; - return out; - } + int compType; + bool isCompressed() const + { + return (oid >= 1000 && compType != 0); + } + time_t cmpMTime; + friend ostream& operator<<(ostream& out, const FdEntry& o) + { + out << " o: " << o.oid << " f: " << o.fp << " d: " << o.dbroot << " p: " << o.partNum + << " s: " << o.segNum << " c: " << o.c << " t: " << o.compType << " m: " << o.cmpMTime; + return out; + } }; struct fdCacheMapLessThan { - bool operator()(const FdEntry& lhs, const FdEntry& rhs) const - { - if (lhs.oid < rhs.oid) - return true; + bool operator()(const FdEntry& lhs, const FdEntry& rhs) const + { + if (lhs.oid < rhs.oid) + return true; - if (lhs.oid == rhs.oid && lhs.dbroot < rhs.dbroot) - return true; + if (lhs.oid == rhs.oid && lhs.dbroot < rhs.dbroot) + return true; - if (lhs.oid == rhs.oid && lhs.dbroot == rhs.dbroot && lhs.partNum < rhs.partNum) - return true; + if (lhs.oid == rhs.oid && lhs.dbroot == rhs.dbroot && lhs.partNum < rhs.partNum) + return true; - if (lhs.oid == rhs.oid && lhs.dbroot == rhs.dbroot && lhs.partNum == rhs.partNum && lhs.segNum < rhs.segNum) - return true; + if (lhs.oid == rhs.oid && lhs.dbroot == rhs.dbroot && lhs.partNum == rhs.partNum && + lhs.segNum < rhs.segNum) + return true; - return false; - - } + return false; + } }; struct fdMapEqual { - bool operator()(const FdEntry& lhs, const FdEntry& rhs) const - { - return (lhs.oid == rhs.oid && - lhs.dbroot == rhs.dbroot && - lhs.partNum == rhs.partNum && - lhs.segNum == rhs.segNum); - } - + bool operator()(const FdEntry& lhs, const FdEntry& rhs) const + { + return (lhs.oid == rhs.oid && lhs.dbroot == rhs.dbroot && lhs.partNum == rhs.partNum && + lhs.segNum == rhs.segNum); + } }; typedef boost::shared_ptr SPFdEntry_t; @@ -245,27 +233,34 @@ typedef std::map FdCacheType_t; struct FdCountEntry { - FdCountEntry() {} - FdCountEntry(const BRM::OID_t o, const uint16_t d, const uint32_t p, const uint16_t s, const uint32_t c, - const FdCacheType_t::iterator it) : oid(o), dbroot(d), partNum(p), segNum(s), cnt(c), fdit(it) {} - ~FdCountEntry() {} + FdCountEntry() + { + } + FdCountEntry(const BRM::OID_t o, const uint16_t d, const uint32_t p, const uint16_t s, const uint32_t c, + const FdCacheType_t::iterator it) + : oid(o), dbroot(d), partNum(p), segNum(s), cnt(c), fdit(it) + { + } + ~FdCountEntry() + { + } - BRM::OID_t oid; - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t cnt; - FdCacheType_t::iterator fdit; -}; // FdCountEntry + BRM::OID_t oid; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t cnt; + FdCacheType_t::iterator fdit; +}; // FdCountEntry typedef FdCountEntry FdCountEntry_t; struct fdCountCompare { - bool operator() (const FdCountEntry_t& lhs, const FdCountEntry_t& rhs) - { - return lhs.cnt > rhs.cnt; - } + bool operator()(const FdCountEntry_t& lhs, const FdCountEntry_t& rhs) + { + return lhs.cnt > rhs.cnt; + } }; typedef multiset FdCacheCountType_t; @@ -276,873 +271,843 @@ rwlock::RWLock_local localLock; char* alignTo(const char* in, int av) { - ptrdiff_t inx = reinterpret_cast(in); - ptrdiff_t avx = static_cast(av); + ptrdiff_t inx = reinterpret_cast(in); + ptrdiff_t avx = static_cast(av); - if ((inx % avx) != 0) - { - inx &= ~(avx - 1); - inx += avx; - } + if ((inx % avx) != 0) + { + inx &= ~(avx - 1); + inx += avx; + } - char* outx = reinterpret_cast(inx); - return outx; + char* outx = reinterpret_cast(inx); + return outx; } void waitForRetry(long count) { - usleep(5000 * count); - return; + usleep(5000 * count); + return; } - -//Must hold the FD cache lock! +// Must hold the FD cache lock! static int updateptrs(char* ptr, FdCacheType_t::iterator fdit) { - ssize_t i; - uint32_t progress; + ssize_t i; + uint32_t progress; + // ptr is taken from buffer, already been checked: realbuff.get() == 0 + if (ptr == 0) + return -1; - // ptr is taken from buffer, already been checked: realbuff.get() == 0 - if (ptr == 0) - return -1; + // already checked before: fdit->second->isCompressed() + if (fdit->second.get() == 0) + return -2; - // already checked before: fdit->second->isCompressed() - if (fdit->second.get() == 0) - return -2; + IDBDataFile* fp = fdit->second->fp; - IDBDataFile* fp = fdit->second->fp; + if (fp == INVALID_HANDLE_VALUE) + { + Message::Args args; + args.add("updateptrs got invalid fp."); + primitiveprocessor::mlp->logInfoMessage(logging::M0006, args); + return -3; + } - if (fp == INVALID_HANDLE_VALUE) - { - Message::Args args; - args.add("updateptrs got invalid fp."); - primitiveprocessor::mlp->logInfoMessage(logging::M0006, args); - return -3; - } + // We need to read one extra block because we need the first ptr in the 3rd block + // to know if we're done. + // FIXME: re-work all of this so we don't have to re-read the 3rd block. + progress = 0; + + while (progress < 4096 * 3) + { + i = fp->pread(&ptr[progress], progress, (4096 * 3) - progress); + + if (i <= 0) + break; + + progress += i; + } + + if (progress != 4096 * 3) + return -4; // let it retry. Not likely, but ... + + fdit->second->cmpMTime = 0; + time_t mtime = fp->mtime(); + + if (mtime != (time_t)-1) + fdit->second->cmpMTime = mtime; + + int gplRc = 0; + gplRc = compress::CompressInterface::getPtrList(&ptr[4096], 4096, fdit->second->ptrList); + + if (gplRc != 0) + return -5; // go for a retry. + + if (fdit->second->ptrList.size() == 0) + return -6; // go for a retry. + + uint64_t numHdrs = fdit->second->ptrList[0].first / 4096ULL - 2ULL; + + if (numHdrs > 0) + { + boost::scoped_array nextHdrBufsa(new char[numHdrs * 4096 + 4095]); + char* nextHdrBufPtr = 0; + + nextHdrBufPtr = alignTo(nextHdrBufsa.get(), 4096); - //We need to read one extra block because we need the first ptr in the 3rd block - // to know if we're done. - //FIXME: re-work all of this so we don't have to re-read the 3rd block. progress = 0; - while (progress < 4096 * 3) + while (progress < numHdrs * 4096) { - i = fp->pread(&ptr[progress], progress, (4096 * 3) - progress); + i = fp->pread(&nextHdrBufPtr[progress], (4096 * 2) + progress, (numHdrs * 4096) - progress); - if (i <= 0) - break; + if (i <= 0) + break; - progress += i; + progress += i; } - if (progress != 4096 * 3) - return -4; // let it retry. Not likely, but ... + if (progress != numHdrs * 4096) + return -8; - fdit->second->cmpMTime = 0; - time_t mtime = fp->mtime(); - - if ( mtime != (time_t) - 1 ) - fdit->second->cmpMTime = mtime; - - int gplRc = 0; - gplRc = compress::CompressInterface::getPtrList(&ptr[4096], 4096, - fdit->second->ptrList); + CompChunkPtrList nextPtrList; + gplRc = compress::CompressInterface::getPtrList(&nextHdrBufPtr[0], numHdrs * 4096, nextPtrList); if (gplRc != 0) - return -5; // go for a retry. + return -7; // go for a retry. - if (fdit->second->ptrList.size() == 0) - return -6; // go for a retry. + fdit->second->ptrList.insert(fdit->second->ptrList.end(), nextPtrList.begin(), nextPtrList.end()); + } - uint64_t numHdrs = fdit->second->ptrList[0].first / 4096ULL - 2ULL; - - if (numHdrs > 0) - { - boost::scoped_array nextHdrBufsa(new char[numHdrs * 4096 + 4095]); - char* nextHdrBufPtr = 0; - - nextHdrBufPtr = alignTo(nextHdrBufsa.get(), 4096); - - progress = 0; - - while (progress < numHdrs * 4096) - { - i = fp->pread(&nextHdrBufPtr[progress], (4096 * 2) + progress, - (numHdrs * 4096) - progress); - - if (i <= 0) - break; - - progress += i; - } - - if (progress != numHdrs * 4096) - return -8; - - CompChunkPtrList nextPtrList; - gplRc = compress::CompressInterface::getPtrList( - &nextHdrBufPtr[0], numHdrs * 4096, nextPtrList); - - if (gplRc != 0) - return -7; // go for a retry. - - fdit->second->ptrList.insert(fdit->second->ptrList.end(), nextPtrList.begin(), nextPtrList.end()); - } - - return 0; + return 0; } void* thr_popper(ioManager* arg) { - utils::setThreadName("thr_popper"); - ioManager* iom = arg; - FileBufferMgr* fbm; - int totalRqst = 0; - fileRequest* fr = 0; - BRM::LBID_t lbid = 0; - BRM::OID_t oid = 0; - BRM::VER_t ver = 0; - BRM::QueryContext qc; - BRM::VER_t txn = 0; - int compType = 0; - int blocksLoaded = 0; - int blocksRead = 0; - const unsigned pageSize = 4096; - fbm = &iom->fileBufferManager(); - char fileName[WriteEngine::FILE_NAME_SIZE]; - const uint64_t fileBlockSize = BLOCK_SIZE; - bool flg = false; - bool useCache; - uint16_t dbroot = 0; - uint32_t partNum = 0; - uint16_t segNum = 0; - uint32_t offset = 0; - char* fileNamePtr = fileName; - uint64_t longSeekOffset = 0; - int err; - uint32_t dlen = 0, acc, readSize, blocksThisRead, j; - uint32_t blocksRequested = 0; - ssize_t i; - char* alignedbuff = 0; - boost::scoped_array realbuff; - pthread_t threadId = 0; - ostringstream iomLogFileName; - ofstream lFile; - struct timespec rqst1; - struct timespec rqst2; - struct timespec tm; - struct timespec tm2; - double tm3; - double rqst3; - bool locked = false; - SPFdEntry_t fe; - vector cacheInsertOps; - bool copyLocked = false; + utils::setThreadName("thr_popper"); + ioManager* iom = arg; + FileBufferMgr* fbm; + int totalRqst = 0; + fileRequest* fr = 0; + BRM::LBID_t lbid = 0; + BRM::OID_t oid = 0; + BRM::VER_t ver = 0; + BRM::QueryContext qc; + BRM::VER_t txn = 0; + int compType = 0; + int blocksLoaded = 0; + int blocksRead = 0; + const unsigned pageSize = 4096; + fbm = &iom->fileBufferManager(); + char fileName[WriteEngine::FILE_NAME_SIZE]; + const uint64_t fileBlockSize = BLOCK_SIZE; + bool flg = false; + bool useCache; + uint16_t dbroot = 0; + uint32_t partNum = 0; + uint16_t segNum = 0; + uint32_t offset = 0; + char* fileNamePtr = fileName; + uint64_t longSeekOffset = 0; + int err; + uint32_t dlen = 0, acc, readSize, blocksThisRead, j; + uint32_t blocksRequested = 0; + ssize_t i; + char* alignedbuff = 0; + boost::scoped_array realbuff; + pthread_t threadId = 0; + ostringstream iomLogFileName; + ofstream lFile; + struct timespec rqst1; + struct timespec rqst2; + struct timespec tm; + struct timespec tm2; + double tm3; + double rqst3; + bool locked = false; + SPFdEntry_t fe; + vector cacheInsertOps; + bool copyLocked = false; + + if (iom->IOTrace()) + { +#ifdef _MSC_VER + threadId = GetCurrentThreadId(); + iomLogFileName << "C:/Calpont/log/trace/iom." << threadId; +#else + threadId = pthread_self(); + iomLogFileName << MCSLOGDIR << "/trace/iom." << threadId; +#endif + lFile.open(iomLogFileName.str().c_str(), ios_base::app | ios_base::ate); + } + + FdCacheType_t::iterator fdit; + IDBDataFile* fp = 0; + size_t maxCompSz = + compress::CompressInterface::getMaxCompressedSizeGeneric(iom->blocksPerRead * BLOCK_SIZE); + size_t readBufferSz = maxCompSz + pageSize; + + realbuff.reset(new char[readBufferSz]); + + if (realbuff.get() == 0) + { + cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; + return 0; + } + + alignedbuff = alignTo(realbuff.get(), 4096); + + if ((((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) >= (ptrdiff_t)pageSize) || + (((ptrdiff_t)alignedbuff % pageSize) != 0)) + throw runtime_error("aligned buffer size is not matching the page size."); + + uint8_t* uCmpBuf = 0; + uCmpBuf = new uint8_t[4 * 1024 * 1024 + 4]; + + for (;;) + { + if (copyLocked) + { + iom->dbrm()->releaseLBIDRange(lbid, blocksRequested); + copyLocked = false; + } + + if (locked) + { + localLock.read_unlock(); + locked = false; + } + + fr = iom->getNextRequest(); + + localLock.read_lock(); + locked = true; if (iom->IOTrace()) + clock_gettime(CLOCK_REALTIME, &rqst1); + + lbid = fr->Lbid(); + qc = fr->Ver(); + txn = fr->Txn(); + flg = fr->Flg(); + compType = fr->CompType(); + useCache = fr->useCache(); + blocksLoaded = 0; + blocksRead = 0; + dlen = fr->BlocksRequested(); + blocksRequested = fr->BlocksRequested(); + oid = 0; + dbroot = 0; + partNum = 0; + segNum = 0; + offset = 0; + + // special case for getBlock. + iom->dbrm()->lockLBIDRange(lbid, blocksRequested); + copyLocked = true; + + // special case for getBlock. + if (blocksRequested == 1) { -#ifdef _MSC_VER - threadId = GetCurrentThreadId(); - iomLogFileName << "C:/Calpont/log/trace/iom." << threadId; -#else - threadId = pthread_self(); - iomLogFileName << MCSLOGDIR << "/trace/iom." << threadId; -#endif - lFile.open(iomLogFileName.str().c_str(), ios_base::app | ios_base::ate); + BRM::VER_t outVer; + iom->dbrm()->vssLookup((BRM::LBID_t)lbid, qc, txn, &outVer, &flg); + ver = outVer; + fr->versioned(flg); + } + else + { + fr->versioned(false); + ver = qc.currentScn; } - FdCacheType_t::iterator fdit; - IDBDataFile* fp = 0; - size_t maxCompSz = - compress::CompressInterface::getMaxCompressedSizeGeneric( - iom->blocksPerRead * BLOCK_SIZE); - size_t readBufferSz = maxCompSz + pageSize; + err = iom->localLbidLookup(lbid, ver, flg, oid, dbroot, partNum, segNum, offset); - realbuff.reset(new char[readBufferSz]); - - if (realbuff.get() == 0) + if (err == BRM::ERR_SNAPSHOT_TOO_OLD) { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return 0; + ostringstream errMsg; + errMsg << "thr_popper: version " << ver << " of LBID " << lbid << "is too old"; + iom->handleBlockReadError(fr, errMsg.str(), ©Locked); + continue; } - - alignedbuff = alignTo(realbuff.get(), 4096); - - if ((((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) >= (ptrdiff_t)pageSize) || - (((ptrdiff_t)alignedbuff % pageSize) != 0)) - throw runtime_error("aligned buffer size is not matching the page size."); - - uint8_t* uCmpBuf = 0; - uCmpBuf = new uint8_t[4 * 1024 * 1024 + 4]; - - for ( ; ; ) + else if (err < 0) { - if (copyLocked) - { - iom->dbrm()->releaseLBIDRange(lbid, blocksRequested); - copyLocked = false; - } - - if (locked) - { - localLock.read_unlock(); - locked = false; - } - - fr = iom->getNextRequest(); - - localLock.read_lock(); - locked = true; - - if (iom->IOTrace()) - clock_gettime(CLOCK_REALTIME, &rqst1); - - lbid = fr->Lbid(); - qc = fr->Ver(); - txn = fr->Txn(); - flg = fr->Flg(); - compType = fr->CompType(); - useCache = fr->useCache(); - blocksLoaded = 0; - blocksRead = 0; - dlen = fr->BlocksRequested(); - blocksRequested = fr->BlocksRequested(); - oid = 0; - dbroot = 0; - partNum = 0; - segNum = 0; - offset = 0; - - // special case for getBlock. - iom->dbrm()->lockLBIDRange(lbid, blocksRequested); - copyLocked = true; - - // special case for getBlock. - if (blocksRequested == 1) - { - BRM::VER_t outVer; - iom->dbrm()->vssLookup((BRM::LBID_t) lbid, qc, txn, &outVer, &flg); - ver = outVer; - fr->versioned(flg); - } - else - { - fr->versioned(false); - ver = qc.currentScn; - } - - err = iom->localLbidLookup(lbid, - ver, - flg, - oid, - dbroot, - partNum, - segNum, - offset); - - if (err == BRM::ERR_SNAPSHOT_TOO_OLD) - { - ostringstream errMsg; - errMsg << "thr_popper: version " << ver << " of LBID " << lbid << - "is too old"; - iom->handleBlockReadError(fr, errMsg.str(), ©Locked); - continue; - } - else if (err < 0) - { - ostringstream errMsg; - errMsg << "thr_popper: BRM lookup failure; lbid=" << - lbid << "; ver=" << ver << "; flg=" << (flg ? 1 : 0); - iom->handleBlockReadError( fr, errMsg.str(), ©Locked, fileRequest::BRM_LOOKUP_ERROR); - continue; - } + ostringstream errMsg; + errMsg << "thr_popper: BRM lookup failure; lbid=" << lbid << "; ver=" << ver + << "; flg=" << (flg ? 1 : 0); + iom->handleBlockReadError(fr, errMsg.str(), ©Locked, fileRequest::BRM_LOOKUP_ERROR); + continue; + } #ifdef IDB_COMP_POC_DEBUG - { - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + { + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); - if (compType != 0) cout << boldStart; + if (compType != 0) + cout << boldStart; - cout << "fileRequest: " << *fr << endl; + cout << "fileRequest: " << *fr << endl; - if (compType != 0) cout << boldStop; - } + if (compType != 0) + cout << boldStop; + } #endif - const uint32_t extentSize = iom->getExtentRows(); - FdEntry fdKey(oid, dbroot, partNum, segNum, compType, NULL); - //cout << "Looking for " << fdKey << endl - // << "O: " << oid << " D: " << dbroot << " P: " << partNum << " S: " << segNum << endl; + const uint32_t extentSize = iom->getExtentRows(); + FdEntry fdKey(oid, dbroot, partNum, segNum, compType, NULL); + // cout << "Looking for " << fdKey << endl + // << "O: " << oid << " D: " << dbroot << " P: " << partNum << " S: " << segNum << endl; - fdMapMutex.lock(); - fdit = fdcache.find(fdKey); + fdMapMutex.lock(); + fdit = fdcache.find(fdKey); - if (fdit == fdcache.end()) - { - try - { - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - } - catch (exception& exc) - { - fdMapMutex.unlock(); - Message::Args args; - args.add(oid); - args.add(exc.what()); - primitiveprocessor::mlp->logMessage(logging::M0053, args, true); - ostringstream errMsg; - errMsg << "thr_popper: Error building filename for OID " << - oid << "; " << exc.what(); - iom->handleBlockReadError( fr, errMsg.str(), ©Locked ); - continue; - } + if (fdit == fdcache.end()) + { + try + { + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + } + catch (exception& exc) + { + fdMapMutex.unlock(); + Message::Args args; + args.add(oid); + args.add(exc.what()); + primitiveprocessor::mlp->logMessage(logging::M0053, args, true); + ostringstream errMsg; + errMsg << "thr_popper: Error building filename for OID " << oid << "; " << exc.what(); + iom->handleBlockReadError(fr, errMsg.str(), ©Locked); + continue; + } #ifdef IDB_COMP_USE_CMP_SUFFIX - if (compType != 0) - { - char* ptr = strrchr(fileNamePtr, '.'); - idbassert(ptr); - strcpy(ptr, ".cmp"); - } + if (compType != 0) + { + char* ptr = strrchr(fileNamePtr, '.'); + idbassert(ptr); + strcpy(ptr, ".cmp"); + } #endif - if (oid > 3000) - { - //TODO: should syscat columns be considered when reducing open file count - // They are always needed why should they be closed? - if (fdcache.size() >= iom->MaxOpenFiles()) - { - FdCacheCountType_t fdCountSort; - - for (FdCacheType_t::iterator it = fdcache.begin(); it != fdcache.end(); it++) - { - struct FdCountEntry fdc(it->second->oid, - it->second->dbroot, - it->second->partNum, - it->second->segNum, - it->second->c, - it); - - fdCountSort.insert(fdc); - } - - if (iom->FDCacheTrace()) - { - iom->FDTraceFile() - << "Before flushing sz: " << fdcache.size() - << " delCount: " << iom->DecreaseOpenFilesCount() - << endl; - - for (FdCacheType_t::iterator it = fdcache.begin(); it != fdcache.end(); it++) - iom->FDTraceFile() << *(*it).second << endl; - - iom->FDTraceFile() << "==================" << endl << endl; - } - - // TODO: should we consider a minimum number of open files - // currently, there is nothing to prevent all open files - // from being closed by the IOManager. - - uint32_t delCount = 0; - - for (FdCacheCountType_t::reverse_iterator rit = fdCountSort.rbegin(); - rit != fdCountSort.rend() && - fdcache.size() > 0 && - delCount < iom->DecreaseOpenFilesCount(); - rit++) - { - FdEntry oldfdKey(rit->oid, rit->dbroot, rit->partNum, rit->segNum, 0, NULL); - FdCacheType_t::iterator it = fdcache.find(oldfdKey); - - if (it != fdcache.end()) - { - if (iom->FDCacheTrace()) - { - if (!rit->fdit->second->inUse) - iom->FDTraceFile() - << "Removing dc: " << delCount << " sz: " << fdcache.size() - << *(*it).second << " u: " << rit->fdit->second->inUse << endl; - else - iom->FDTraceFile() - << "Skip Remove in use dc: " << delCount << " sz: " << fdcache.size() - << *(*it).second << " u: " << rit->fdit->second->inUse << endl; - } - - if (rit->fdit->second->inUse <= 0) - { - fdcache.erase(it); - delCount++; - } - } - } // for (FdCacheCountType_t... - - if (iom->FDCacheTrace()) - { - iom->FDTraceFile() << "After flushing sz: " << fdcache.size() << endl; - - for (FdCacheType_t::iterator it = fdcache.begin(); it != fdcache.end(); it++) - { - iom->FDTraceFile() - << *(*it).second << endl; - } - - iom->FDTraceFile() << "==================" << endl << endl; - } - - fdCountSort.clear(); - - } // if (fdcache.size()... - } // if (oid > 3000) - - int opts = primitiveprocessor::directIOFlag ? IDBDataFile::USE_ODIRECT : 0; - fp = NULL; - uint32_t openRetries = 0; - int saveErrno = 0; - - while (fp == NULL && openRetries++ < 5) - { - fp = IDBDataFile::open( - IDBPolicy::getType( fileNamePtr, IDBPolicy::PRIMPROC ), - fileNamePtr, - "r", - opts); - saveErrno = errno; - - if (fp == NULL) - sleep(1); - } - - if ( fp == NULL ) - { - Message::Args args; - fdit = fdcache.end(); - fdMapMutex.unlock(); - args.add(oid); - args.add(string(fileNamePtr) + ":" + strerror(saveErrno)); - primitiveprocessor::mlp->logMessage(logging::M0053, args, true); - ostringstream errMsg; - errMsg << "thr_popper: Error opening file for OID " << oid << "; " - << fileNamePtr << "; " << strerror(saveErrno); - int errorCode = fileRequest::FAILED; - - if (saveErrno == EINVAL) - errorCode = fileRequest::FS_EINVAL; - else if (saveErrno == ENOENT) - errorCode = fileRequest::FS_ENOENT; - - iom->handleBlockReadError(fr, errMsg.str(), ©Locked, errorCode); - continue; - } - - fe.reset( new FdEntry(oid, dbroot, partNum, segNum, compType, fp) ); - fe->inUse++; - fdcache[fdKey] = fe; - fdit = fdcache.find(fdKey); - fe.reset(); - } - - else + if (oid > 3000) + { + // TODO: should syscat columns be considered when reducing open file count + // They are always needed why should they be closed? + if (fdcache.size() >= iom->MaxOpenFiles()) { - if (fdit->second.get()) - { - fdit->second->c++; - fdit->second->inUse++; - fp = fdit->second->fp; - } - else - { - Message::Args args; - fdit = fdcache.end(); - fdMapMutex.unlock(); - args.add(oid); - ostringstream errMsg; - errMsg << "Null FD cache entry. (dbroot, partNum, segNum, compType) = (" - << dbroot << ", " << partNum << ", " << segNum << ", " << compType << ")"; - args.add(errMsg.str()); - primitiveprocessor::mlp->logMessage(logging::M0053, args, true); - iom->handleBlockReadError( fr, errMsg.str(), ©Locked ); - continue; - } - } + FdCacheCountType_t fdCountSort; + for (FdCacheType_t::iterator it = fdcache.begin(); it != fdcache.end(); it++) + { + struct FdCountEntry fdc(it->second->oid, it->second->dbroot, it->second->partNum, + it->second->segNum, it->second->c, it); + + fdCountSort.insert(fdc); + } + + if (iom->FDCacheTrace()) + { + iom->FDTraceFile() << "Before flushing sz: " << fdcache.size() + << " delCount: " << iom->DecreaseOpenFilesCount() << endl; + + for (FdCacheType_t::iterator it = fdcache.begin(); it != fdcache.end(); it++) + iom->FDTraceFile() << *(*it).second << endl; + + iom->FDTraceFile() << "==================" << endl << endl; + } + + // TODO: should we consider a minimum number of open files + // currently, there is nothing to prevent all open files + // from being closed by the IOManager. + + uint32_t delCount = 0; + + for (FdCacheCountType_t::reverse_iterator rit = fdCountSort.rbegin(); + rit != fdCountSort.rend() && fdcache.size() > 0 && delCount < iom->DecreaseOpenFilesCount(); + rit++) + { + FdEntry oldfdKey(rit->oid, rit->dbroot, rit->partNum, rit->segNum, 0, NULL); + FdCacheType_t::iterator it = fdcache.find(oldfdKey); + + if (it != fdcache.end()) + { + if (iom->FDCacheTrace()) + { + if (!rit->fdit->second->inUse) + iom->FDTraceFile() << "Removing dc: " << delCount << " sz: " << fdcache.size() + << *(*it).second << " u: " << rit->fdit->second->inUse << endl; + else + iom->FDTraceFile() << "Skip Remove in use dc: " << delCount << " sz: " << fdcache.size() + << *(*it).second << " u: " << rit->fdit->second->inUse << endl; + } + + if (rit->fdit->second->inUse <= 0) + { + fdcache.erase(it); + delCount++; + } + } + } // for (FdCacheCountType_t... + + if (iom->FDCacheTrace()) + { + iom->FDTraceFile() << "After flushing sz: " << fdcache.size() << endl; + + for (FdCacheType_t::iterator it = fdcache.begin(); it != fdcache.end(); it++) + { + iom->FDTraceFile() << *(*it).second << endl; + } + + iom->FDTraceFile() << "==================" << endl << endl; + } + + fdCountSort.clear(); + + } // if (fdcache.size()... + } // if (oid > 3000) + + int opts = primitiveprocessor::directIOFlag ? IDBDataFile::USE_ODIRECT : 0; + fp = NULL; + uint32_t openRetries = 0; + int saveErrno = 0; + + while (fp == NULL && openRetries++ < 5) + { + fp = IDBDataFile::open(IDBPolicy::getType(fileNamePtr, IDBPolicy::PRIMPROC), fileNamePtr, "r", opts); + saveErrno = errno; + + if (fp == NULL) + sleep(1); + } + + if (fp == NULL) + { + Message::Args args; + fdit = fdcache.end(); fdMapMutex.unlock(); + args.add(oid); + args.add(string(fileNamePtr) + ":" + strerror(saveErrno)); + primitiveprocessor::mlp->logMessage(logging::M0053, args, true); + ostringstream errMsg; + errMsg << "thr_popper: Error opening file for OID " << oid << "; " << fileNamePtr << "; " + << strerror(saveErrno); + int errorCode = fileRequest::FAILED; + + if (saveErrno == EINVAL) + errorCode = fileRequest::FS_EINVAL; + else if (saveErrno == ENOENT) + errorCode = fileRequest::FS_ENOENT; + + iom->handleBlockReadError(fr, errMsg.str(), ©Locked, errorCode); + continue; + } + + fe.reset(new FdEntry(oid, dbroot, partNum, segNum, compType, fp)); + fe->inUse++; + fdcache[fdKey] = fe; + fdit = fdcache.find(fdKey); + fe.reset(); + } + + else + { + if (fdit->second.get()) + { + fdit->second->c++; + fdit->second->inUse++; + fp = fdit->second->fp; + } + else + { + Message::Args args; + fdit = fdcache.end(); + fdMapMutex.unlock(); + args.add(oid); + ostringstream errMsg; + errMsg << "Null FD cache entry. (dbroot, partNum, segNum, compType) = (" << dbroot << ", " << partNum + << ", " << segNum << ", " << compType << ")"; + args.add(errMsg.str()); + primitiveprocessor::mlp->logMessage(logging::M0053, args, true); + iom->handleBlockReadError(fr, errMsg.str(), ©Locked); + continue; + } + } + + fdMapMutex.unlock(); #ifdef SHARED_NOTHING_DEMO_2 - // change offset if it's shared nothing - /* Get the extent #, divide by # of PMs, calculate base offset for the new extent #, - add extent offset */ - if (oid >= 10000) - offset = (((offset / extentSize) / iom->pmCount) * extentSize) + (offset % extentSize); + // change offset if it's shared nothing + /* Get the extent #, divide by # of PMs, calculate base offset for the new extent #, + add extent offset */ + if (oid >= 10000) + offset = (((offset / extentSize) / iom->pmCount) * extentSize) + (offset % extentSize); #endif - longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize; - lldiv_t cmpOffFact = lldiv(longSeekOffset, (4LL * 1024LL * 1024LL)); - totalRqst++; + longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize; + lldiv_t cmpOffFact = lldiv(longSeekOffset, (4LL * 1024LL * 1024LL)); + totalRqst++; - uint32_t readCount = 0; - uint32_t bytesRead = 0; - uint32_t compressedBytesRead = 0; // @Bug 3149. IOMTrace was not reporting bytesRead correctly for compressed columns. - uint32_t jend = blocksRequested / iom->blocksPerRead; + uint32_t readCount = 0; + uint32_t bytesRead = 0; + uint32_t compressedBytesRead = + 0; // @Bug 3149. IOMTrace was not reporting bytesRead correctly for compressed columns. + uint32_t jend = blocksRequested / iom->blocksPerRead; - if (iom->IOTrace()) - clock_gettime(CLOCK_REALTIME, &tm); + if (iom->IOTrace()) + clock_gettime(CLOCK_REALTIME, &tm); - ostringstream errMsg; - bool errorOccurred = false; - string errorString; + ostringstream errMsg; + bool errorOccurred = false; + string errorString; #ifdef IDB_COMP_POC_DEBUG - bool debugWrite = false; + bool debugWrite = false; #endif #ifdef EM_AS_A_TABLE_POC__ - dlen = 1; + dlen = 1; #endif - if (blocksRequested % iom->blocksPerRead) - jend++; + if (blocksRequested % iom->blocksPerRead) + jend++; - for (j = 0; j < jend; j++) - { + for (j = 0; j < jend; j++) + { + int decompRetryCount = 0; + int retryReadHeadersCount = 0; - int decompRetryCount = 0; - int retryReadHeadersCount = 0; + decompRetry: + blocksThisRead = std::min(dlen, iom->blocksPerRead); + readSize = blocksThisRead * BLOCK_SIZE; -decompRetry: - blocksThisRead = std::min(dlen, iom->blocksPerRead); - readSize = blocksThisRead * BLOCK_SIZE; + acc = 0; - acc = 0; - - while (acc < readSize) - { + while (acc < readSize) + { #if defined(EM_AS_A_TABLE_POC__) - if (oid == 1084) - { - uint32_t h; - int32_t o = 0; - int32_t* ip; - ip = (int32_t*)(&alignedbuff[acc]); + if (oid == 1084) + { + uint32_t h; + int32_t o = 0; + int32_t* ip; + ip = (int32_t*)(&alignedbuff[acc]); - for (o = 0; o < 2048; o++) - { - if (iom->dbrm()->getHWM(o + 3000, h) == 0) - *ip++ = h; - else - *ip++ = numeric_limits::min() + 1; - } + for (o = 0; o < 2048; o++) + { + if (iom->dbrm()->getHWM(o + 3000, h) == 0) + *ip++ = h; + else + *ip++ = numeric_limits::min() + 1; + } - i = BLOCK_SIZE; - } - else - i = pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset); + i = BLOCK_SIZE; + } + else + i = pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset); #else - if (fdit->second->isCompressed()) - { -retryReadHeaders: - //hdrs may have been modified since we cached them in fdit->second... - time_t cur_mtime = numeric_limits::max(); - int updatePtrsRc = 0; - fdMapMutex.lock(); - time_t fp_mtime = fp->mtime(); + if (fdit->second->isCompressed()) + { + retryReadHeaders: + // hdrs may have been modified since we cached them in fdit->second... + time_t cur_mtime = numeric_limits::max(); + int updatePtrsRc = 0; + fdMapMutex.lock(); + time_t fp_mtime = fp->mtime(); - if ( fp_mtime != (time_t) - 1) - cur_mtime = fp_mtime; + if (fp_mtime != (time_t)-1) + cur_mtime = fp_mtime; - if (decompRetryCount > 0 || retryReadHeadersCount > 0 || cur_mtime > fdit->second->cmpMTime) - updatePtrsRc = updateptrs(&alignedbuff[0], fdit); + if (decompRetryCount > 0 || retryReadHeadersCount > 0 || cur_mtime > fdit->second->cmpMTime) + updatePtrsRc = updateptrs(&alignedbuff[0], fdit); - fdMapMutex.unlock(); + fdMapMutex.unlock(); - int idx = cmpOffFact.quot; + int idx = cmpOffFact.quot; - if (updatePtrsRc != 0 || idx >= (signed)fdit->second->ptrList.size()) - { - // Due to race condition, the header on disk may not upated yet. - // Log an info message and retry. - if (retryReadHeadersCount == 0) - { - Message::Args args; - args.add(oid); - ostringstream infoMsg; - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - infoMsg << "retry updateptrs for " << fileNamePtr - << ". rc=" << updatePtrsRc << ", idx=" - << idx << ", ptr.size=" << fdit->second->ptrList.size(); - args.add(infoMsg.str()); - primitiveprocessor::mlp->logInfoMessage(logging::M0061, args); - } - - if (++retryReadHeadersCount < 30) - { - waitForRetry(retryReadHeadersCount); - fdit->second->cmpMTime = 0; - goto retryReadHeaders; - } - else - { - // still fail after all the retries. - errorOccurred = true; - errMsg << "Error reading compression header. rc=" << updatePtrsRc << ", idx=" - << idx << ", ptr.size=" << fdit->second->ptrList.size(); - errorString = errMsg.str(); - break; - } - } - - //FIXME: make sure alignedbuff can hold fdit->second->ptrList[idx].second bytes - if (fdit->second->ptrList[idx].second > maxCompSz) - { - errorOccurred = true; - errMsg << "aligned buff too small. dataSize=" - << fdit->second->ptrList[idx].second - << ", buffSize=" << maxCompSz; - errorString = errMsg.str(); - break; - } - - i = fp->pread(&alignedbuff[0], fdit->second->ptrList[idx].first, fdit->second->ptrList[idx].second ); -#ifdef IDB_COMP_POC_DEBUG - { - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); - cout << boldStart << "pread1.1(" << fp << ", 0x" << hex << (ptrdiff_t)&alignedbuff[0] << dec << ", " << fdit->second->ptrList[idx].second << - ", " << fdit->second->ptrList[idx].first << ") = " << i << ' ' << cmpOffFact.quot << ' ' << cmpOffFact.rem << boldStop << endl; - } -#endif - - // @bug3407, give it some retries if pread failed. - if (i != (ssize_t)fdit->second->ptrList[idx].second) - { - // log an info message - if (retryReadHeadersCount == 0) - { - Message::Args args; - args.add(oid); - ostringstream infoMsg; - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - infoMsg << " Read from " << fileNamePtr << " failed at chunk " << idx - << ". (offset, size, actuall read) = (" - << fdit->second->ptrList[idx].first << ", " - << fdit->second->ptrList[idx].second << ", " << i << ")"; - args.add(infoMsg.str()); - primitiveprocessor::mlp->logInfoMessage(logging::M0061, args); - } - - if (++retryReadHeadersCount < 30) - { - waitForRetry(retryReadHeadersCount); - fdit->second->cmpMTime = 0; - goto retryReadHeaders; - } - else - { - errorOccurred = true; - errMsg << "Error reading chunk " << idx; - errorString = errMsg.str(); - break; - } - } - - compressedBytesRead += i; // @Bug 3149. - i = readSize; - } - else - { - i = fp->pread(&alignedbuff[acc], longSeekOffset, readSize - acc); -#ifdef IDB_COMP_POC_DEBUG - { - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); - cout << "pread1.2(" << fp << ", 0x" << hex << (ptrdiff_t)&alignedbuff[acc] << dec << ", " << (readSize - acc) << - ", " << longSeekOffset << ") = " << i << ' ' << cmpOffFact.quot << ' ' << cmpOffFact.rem << endl; - } -#endif - } - -#endif - - if (i < 0 && errno == EINTR) - { - continue; - } - else if (i < 0) - { - try - { - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - } - catch (exception& exc) - { - cerr << "FileName Err:" << exc.what() << endl; - strcpy(fileNamePtr, "unknown filename"); - } - - errorString = strerror(errno); - errorOccurred = true; - errMsg << "thr_popper: Error reading file for OID " << oid << "; " - << " fp " << fp << "; offset " << longSeekOffset << "; fileName " - << fileNamePtr << "; " << errorString; - break; // break from "while(acc..." loop - } - else if (i == 0) - { - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - errorString = "early EOF"; - errorOccurred = true; - errMsg << "thr_popper: Early EOF reading file for OID " << - oid << "; " << fileNamePtr; - break; // break from "while(acc..." loop - } - - acc += i; - longSeekOffset += (uint64_t)i; - readCount++; - bytesRead += i; - } // while(acc... - - //..Break out of "for (j..." read loop if error occurred - if (errorOccurred) + if (updatePtrsRc != 0 || idx >= (signed)fdit->second->ptrList.size()) + { + // Due to race condition, the header on disk may not upated yet. + // Log an info message and retry. + if (retryReadHeadersCount == 0) { - Message::Args args; - args.add(oid); - args.add(errorString); - primitiveprocessor::mlp->logMessage(logging::M0061, args, true); - iom->handleBlockReadError( fr, errMsg.str(), ©Locked ); - break; + Message::Args args; + args.add(oid); + ostringstream infoMsg; + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + infoMsg << "retry updateptrs for " << fileNamePtr << ". rc=" << updatePtrsRc << ", idx=" << idx + << ", ptr.size=" << fdit->second->ptrList.size(); + args.add(infoMsg.str()); + primitiveprocessor::mlp->logInfoMessage(logging::M0061, args); } - blocksRead += blocksThisRead; - - if (iom->IOTrace()) - clock_gettime(CLOCK_REALTIME, &tm2); - - /* New bulk VSS lookup code */ + if (++retryReadHeadersCount < 30) { - vector lbids; - vector versions; - vector isLocked; + waitForRetry(retryReadHeadersCount); + fdit->second->cmpMTime = 0; + goto retryReadHeaders; + } + else + { + // still fail after all the retries. + errorOccurred = true; + errMsg << "Error reading compression header. rc=" << updatePtrsRc << ", idx=" << idx + << ", ptr.size=" << fdit->second->ptrList.size(); + errorString = errMsg.str(); + break; + } + } - for (i = 0; (uint32_t) i < blocksThisRead; i++) - lbids.push_back((BRM::LBID_t) (lbid + i) + (j * iom->blocksPerRead)); + // FIXME: make sure alignedbuff can hold fdit->second->ptrList[idx].second bytes + if (fdit->second->ptrList[idx].second > maxCompSz) + { + errorOccurred = true; + errMsg << "aligned buff too small. dataSize=" << fdit->second->ptrList[idx].second + << ", buffSize=" << maxCompSz; + errorString = errMsg.str(); + break; + } - if (blocksRequested > 1 || !flg) // prefetch, or an unversioned single-block read - iom->dbrm()->bulkGetCurrentVersion(lbids, &versions, &isLocked); - else // a single-block read that was versioned - { - versions.push_back(ver); - isLocked.push_back(false); - } + i = fp->pread(&alignedbuff[0], fdit->second->ptrList[idx].first, fdit->second->ptrList[idx].second); +#ifdef IDB_COMP_POC_DEBUG + { + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + cout << boldStart << "pread1.1(" << fp << ", 0x" << hex << (ptrdiff_t)&alignedbuff[0] << dec + << ", " << fdit->second->ptrList[idx].second << ", " << fdit->second->ptrList[idx].first + << ") = " << i << ' ' << cmpOffFact.quot << ' ' << cmpOffFact.rem << boldStop << endl; + } +#endif - uint8_t* ptr = (uint8_t*)&alignedbuff[0]; + // @bug3407, give it some retries if pread failed. + if (i != (ssize_t)fdit->second->ptrList[idx].second) + { + // log an info message + if (retryReadHeadersCount == 0) + { + Message::Args args; + args.add(oid); + ostringstream infoMsg; + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + infoMsg << " Read from " << fileNamePtr << " failed at chunk " << idx + << ". (offset, size, actuall read) = (" << fdit->second->ptrList[idx].first << ", " + << fdit->second->ptrList[idx].second << ", " << i << ")"; + args.add(infoMsg.str()); + primitiveprocessor::mlp->logInfoMessage(logging::M0061, args); + } - if (blocksThisRead > 0 && fdit->second->isCompressed()) - { + if (++retryReadHeadersCount < 30) + { + waitForRetry(retryReadHeadersCount); + fdit->second->cmpMTime = 0; + goto retryReadHeaders; + } + else + { + errorOccurred = true; + errMsg << "Error reading chunk " << idx; + errorString = errMsg.str(); + break; + } + } + + compressedBytesRead += i; // @Bug 3149. + i = readSize; + } + else + { + i = fp->pread(&alignedbuff[acc], longSeekOffset, readSize - acc); +#ifdef IDB_COMP_POC_DEBUG + { + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + cout << "pread1.2(" << fp << ", 0x" << hex << (ptrdiff_t)&alignedbuff[acc] << dec << ", " + << (readSize - acc) << ", " << longSeekOffset << ") = " << i << ' ' << cmpOffFact.quot << ' ' + << cmpOffFact.rem << endl; + } +#endif + } + +#endif + + if (i < 0 && errno == EINTR) + { + continue; + } + else if (i < 0) + { + try + { + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + } + catch (exception& exc) + { + cerr << "FileName Err:" << exc.what() << endl; + strcpy(fileNamePtr, "unknown filename"); + } + + errorString = strerror(errno); + errorOccurred = true; + errMsg << "thr_popper: Error reading file for OID " << oid << "; " + << " fp " << fp << "; offset " << longSeekOffset << "; fileName " << fileNamePtr << "; " + << errorString; + break; // break from "while(acc..." loop + } + else if (i == 0) + { + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + errorString = "early EOF"; + errorOccurred = true; + errMsg << "thr_popper: Early EOF reading file for OID " << oid << "; " << fileNamePtr; + break; // break from "while(acc..." loop + } + + acc += i; + longSeekOffset += (uint64_t)i; + readCount++; + bytesRead += i; + } // while(acc... + + //..Break out of "for (j..." read loop if error occurred + if (errorOccurred) + { + Message::Args args; + args.add(oid); + args.add(errorString); + primitiveprocessor::mlp->logMessage(logging::M0061, args, true); + iom->handleBlockReadError(fr, errMsg.str(), ©Locked); + break; + } + + blocksRead += blocksThisRead; + + if (iom->IOTrace()) + clock_gettime(CLOCK_REALTIME, &tm2); + + /* New bulk VSS lookup code */ + { + vector lbids; + vector versions; + vector isLocked; + + for (i = 0; (uint32_t)i < blocksThisRead; i++) + lbids.push_back((BRM::LBID_t)(lbid + i) + (j * iom->blocksPerRead)); + + if (blocksRequested > 1 || !flg) // prefetch, or an unversioned single-block read + iom->dbrm()->bulkGetCurrentVersion(lbids, &versions, &isLocked); + else // a single-block read that was versioned + { + versions.push_back(ver); + isLocked.push_back(false); + } + + uint8_t* ptr = (uint8_t*)&alignedbuff[0]; + + if (blocksThisRead > 0 && fdit->second->isCompressed()) + { #ifdef _MSC_VER - unsigned int blen = 4 * 1024 * 1024 + 4; + unsigned int blen = 4 * 1024 * 1024 + 4; #else - size_t blen = 4 * 1024 * 1024 + 4; + size_t blen = 4 * 1024 * 1024 + 4; #endif #ifdef IDB_COMP_POC_DEBUG - { - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); - cout << "decompress(0x" << hex << (ptrdiff_t)&alignedbuff[0] << dec << ", " << fdit->second->ptrList[cmpOffFact.quot].second << ", 0x" << hex << (ptrdiff_t)uCmpBuf << dec << ", " << blen << ")" << endl; - } + { + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + cout << "decompress(0x" << hex << (ptrdiff_t)&alignedbuff[0] << dec << ", " + << fdit->second->ptrList[cmpOffFact.quot].second << ", 0x" << hex << (ptrdiff_t)uCmpBuf + << dec << ", " << blen << ")" << endl; + } #endif - std::unique_ptr decompressor( - compress::getCompressInterfaceByType( - static_cast(fdit->second->compType))); - if (!decompressor) - { - // Use default? - decompressor.reset( - new compress::CompressInterfaceSnappy()); - } + std::unique_ptr decompressor( + compress::getCompressInterfaceByType(static_cast(fdit->second->compType))); + if (!decompressor) + { + // Use default? + decompressor.reset(new compress::CompressInterfaceSnappy()); + } - int dcrc = decompressor->uncompressBlock(&alignedbuff[0], - fdit->second->ptrList[cmpOffFact.quot].second, uCmpBuf, blen); + int dcrc = decompressor->uncompressBlock( + &alignedbuff[0], fdit->second->ptrList[cmpOffFact.quot].second, uCmpBuf, blen); - if (dcrc != 0) - { + if (dcrc != 0) + { #ifdef IDB_COMP_POC_DEBUG - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); #endif - if (++decompRetryCount < 30) - { - blocksRead -= blocksThisRead; - waitForRetry(decompRetryCount); + if (++decompRetryCount < 30) + { + blocksRead -= blocksThisRead; + waitForRetry(decompRetryCount); - // log an info message every 10 retries - if (decompRetryCount == 1) - { - Message::Args args; - args.add(oid); - ostringstream infoMsg; - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - infoMsg << "decompress retry for " << fileNamePtr - << " decompRetry chunk " << cmpOffFact.quot - << " code=" << dcrc; - args.add(infoMsg.str()); - primitiveprocessor::mlp->logInfoMessage(logging::M0061, args); - } + // log an info message every 10 retries + if (decompRetryCount == 1) + { + Message::Args args; + args.add(oid); + ostringstream infoMsg; + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + infoMsg << "decompress retry for " << fileNamePtr << " decompRetry chunk " << cmpOffFact.quot + << " code=" << dcrc; + args.add(infoMsg.str()); + primitiveprocessor::mlp->logInfoMessage(logging::M0061, args); + } - goto decompRetry; - } + goto decompRetry; + } - cout << boldStart << "decomp returned " << dcrc << boldStop << endl; + cout << boldStart << "decomp returned " << dcrc << boldStop << endl; - errorOccurred = true; - Message::Args args; - args.add(oid); - errMsg << "Error decompressing block " << cmpOffFact.quot << " code=" << dcrc << " part=" << partNum << " seg=" << segNum; - args.add(errMsg.str()); - primitiveprocessor::mlp->logMessage(logging::M0061, args, true); - iom->handleBlockReadError( fr, errMsg.str(), ©Locked ); - break; - } + errorOccurred = true; + Message::Args args; + args.add(oid); + errMsg << "Error decompressing block " << cmpOffFact.quot << " code=" << dcrc + << " part=" << partNum << " seg=" << segNum; + args.add(errMsg.str()); + primitiveprocessor::mlp->logMessage(logging::M0061, args, true); + iom->handleBlockReadError(fr, errMsg.str(), ©Locked); + break; + } - //FIXME: why doesn't this work??? (See later for why) - //ptr = &uCmpBuf[cmpOffFact.rem]; - memcpy(ptr, &uCmpBuf[cmpOffFact.rem], blocksThisRead * BLOCK_SIZE); + // FIXME: why doesn't this work??? (See later for why) + // ptr = &uCmpBuf[cmpOffFact.rem]; + memcpy(ptr, &uCmpBuf[cmpOffFact.rem], blocksThisRead * BLOCK_SIZE); - // log the retries, if any - if (retryReadHeadersCount > 0 || decompRetryCount > 0) - { - Message::Args args; - args.add(oid); - ostringstream infoMsg; - iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); - infoMsg << "Successfully uncompress " << fileNamePtr << " chunk " - << cmpOffFact.quot << " @"; + // log the retries, if any + if (retryReadHeadersCount > 0 || decompRetryCount > 0) + { + Message::Args args; + args.add(oid); + ostringstream infoMsg; + iom->buildOidFileName(oid, dbroot, partNum, segNum, fileNamePtr); + infoMsg << "Successfully uncompress " << fileNamePtr << " chunk " << cmpOffFact.quot << " @"; - if (retryReadHeadersCount > 0) - infoMsg << " HeaderRetry:" << retryReadHeadersCount; + if (retryReadHeadersCount > 0) + infoMsg << " HeaderRetry:" << retryReadHeadersCount; - if (decompRetryCount > 0) - infoMsg << " UncompressRetry:" << decompRetryCount; + if (decompRetryCount > 0) + infoMsg << " UncompressRetry:" << decompRetryCount; - args.add(infoMsg.str()); - primitiveprocessor::mlp->logInfoMessage(logging::M0006, args); - } - } + args.add(infoMsg.str()); + primitiveprocessor::mlp->logInfoMessage(logging::M0006, args); + } + } - for (i = 0; useCache && (uint32_t) i < lbids.size(); i++) - { - if (!isLocked[i]) - { + for (i = 0; useCache && (uint32_t)i < lbids.size(); i++) + { + if (!isLocked[i]) + { #ifdef IDB_COMP_POC_DEBUG - { - if (debugWrite) - { - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); - cout << boldStart << "i = " << i << ", ptr = 0x" << hex << (ptrdiff_t)&ptr[i * BLOCK_SIZE] << dec << boldStop << endl; - cout << boldStart; + { + if (debugWrite) + { + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + cout << boldStart << "i = " << i << ", ptr = 0x" << hex << (ptrdiff_t)&ptr[i * BLOCK_SIZE] + << dec << boldStop << endl; + cout << boldStart; #if 0 int32_t* i32p; i32p = (int32_t*)&ptr[i * BLOCK_SIZE]; @@ -1156,347 +1121,325 @@ retryReadHeaders: } #else - int64_t* i64p; - i64p = (int64_t*)&ptr[i * BLOCK_SIZE]; + int64_t* i64p; + i64p = (int64_t*)&ptr[i * BLOCK_SIZE]; - for (int iy = 0; iy < 2; iy++) - { - for (int ix = 0; ix < 8; ix++, i64p++) - cout << *i64p << ' '; - - cout << endl; - } - -#endif - cout << boldStop << endl; - } - } -#endif - cacheInsertOps.push_back(CacheInsert_t(lbids[i], versions[i], (uint8_t*) - &alignedbuff[i * BLOCK_SIZE])); - } - } - - if (useCache) + for (int iy = 0; iy < 2; iy++) { - blocksLoaded += fbm->bulkInsert(cacheInsertOps); - cacheInsertOps.clear(); + for (int ix = 0; ix < 8; ix++, i64p++) + cout << *i64p << ' '; + + cout << endl; } + +#endif + cout << boldStop << endl; + } } - - dlen -= blocksThisRead; - - } // for (j... - - fdMapMutex.lock(); - - if (fdit->second.get()) - fdit->second->inUse--; - - fdit = fdcache.end(); - fdMapMutex.unlock(); - - if (errorOccurred) - continue; - - try - { - iom->dbrm()->releaseLBIDRange(lbid, blocksRequested); - copyLocked = false; - } - catch (exception& e) - { - cout << "releaseRange: " << e.what() << endl; +#endif + cacheInsertOps.push_back( + CacheInsert_t(lbids[i], versions[i], (uint8_t*)&alignedbuff[i * BLOCK_SIZE])); + } } - fr->BlocksRead(blocksRead); - fr->BlocksLoaded(blocksLoaded); - - //FIXME: This is why some code above doesn't work... - if (fr->data != 0 && blocksRequested == 1) - memcpy(fr->data, alignedbuff, BLOCK_SIZE); - - fr->frMutex().lock(); - fr->SetPredicate(fileRequest::COMPLETE); - fr->frCond().notify_one(); - fr->frMutex().unlock(); - - if (iom->IOTrace()) + if (useCache) { - clock_gettime(CLOCK_REALTIME, &rqst2); - timespec_sub(tm, tm2, tm3); - timespec_sub(rqst1, rqst2, rqst3); - - // @Bug 3149. IOMTrace was not reporting bytesRead correctly for compressed columns. - uint32_t reportBytesRead = (compressedBytesRead > 0) ? compressedBytesRead : bytesRead; - - lFile - << left << setw(5) << setfill(' ') << oid - << right << setw(5) << setfill(' ') << offset / extentSize << " " - << right << setw(11) << setfill(' ') << lbid << " " - << right << setw(9) << bytesRead / (readCount << 13) << " " - << right << setw(9) << blocksRequested << " " - << right << setw(10) << fixed << tm3 << " " - << right << fixed << ((double)(rqst1.tv_sec + (1.e-9 * rqst1.tv_nsec))) << " " - << right << setw(10) << fixed << rqst3 << " " - << right << setw(10) << fixed << longSeekOffset << " " - << right << setw(10) << fixed << reportBytesRead << " " - << right << setw(3) << fixed << dbroot << " " - << right << setw(3) << fixed << partNum << " " - << right << setw(3) << fixed << segNum << " " - << endl; + blocksLoaded += fbm->bulkInsert(cacheInsertOps); + cacheInsertOps.clear(); } + } - } // for(;;) + dlen -= blocksThisRead; - delete [] uCmpBuf; + } // for (j... - lFile.close(); + fdMapMutex.lock(); - //reaching here is an error... + if (fdit->second.get()) + fdit->second->inUse--; - return 0; -} // end thr_popper + fdit = fdcache.end(); + fdMapMutex.unlock(); -} // anonymous namespace + if (errorOccurred) + continue; + + try + { + iom->dbrm()->releaseLBIDRange(lbid, blocksRequested); + copyLocked = false; + } + catch (exception& e) + { + cout << "releaseRange: " << e.what() << endl; + } + + fr->BlocksRead(blocksRead); + fr->BlocksLoaded(blocksLoaded); + + // FIXME: This is why some code above doesn't work... + if (fr->data != 0 && blocksRequested == 1) + memcpy(fr->data, alignedbuff, BLOCK_SIZE); + + fr->frMutex().lock(); + fr->SetPredicate(fileRequest::COMPLETE); + fr->frCond().notify_one(); + fr->frMutex().unlock(); + + if (iom->IOTrace()) + { + clock_gettime(CLOCK_REALTIME, &rqst2); + timespec_sub(tm, tm2, tm3); + timespec_sub(rqst1, rqst2, rqst3); + + // @Bug 3149. IOMTrace was not reporting bytesRead correctly for compressed columns. + uint32_t reportBytesRead = (compressedBytesRead > 0) ? compressedBytesRead : bytesRead; + + lFile << left << setw(5) << setfill(' ') << oid << right << setw(5) << setfill(' ') + << offset / extentSize << " " << right << setw(11) << setfill(' ') << lbid << " " << right + << setw(9) << bytesRead / (readCount << 13) << " " << right << setw(9) << blocksRequested << " " + << right << setw(10) << fixed << tm3 << " " << right << fixed + << ((double)(rqst1.tv_sec + (1.e-9 * rqst1.tv_nsec))) << " " << right << setw(10) << fixed + << rqst3 << " " << right << setw(10) << fixed << longSeekOffset << " " << right << setw(10) + << fixed << reportBytesRead << " " << right << setw(3) << fixed << dbroot << " " << right + << setw(3) << fixed << partNum << " " << right << setw(3) << fixed << segNum << " " << endl; + } + + } // for(;;) + + delete[] uCmpBuf; + + lFile.close(); + + // reaching here is an error... + + return 0; +} // end thr_popper + +} // anonymous namespace namespace dbbc { - void setReadLock() { - localLock.read_lock(); + localLock.read_lock(); } void releaseReadLock() { - localLock.read_unlock(); + localLock.read_unlock(); } void dropFDCache() { - localLock.write_lock(); - fdcache.clear(); - localLock.write_unlock(); + localLock.write_lock(); + fdcache.clear(); + localLock.write_unlock(); } void purgeFDCache(std::vector& files) { - localLock.write_lock(); + localLock.write_lock(); - FdCacheType_t::iterator fdit; + FdCacheType_t::iterator fdit; - for ( uint32_t i = 0; i < files.size(); i++) - { - FdEntry fdKey(files[i].oid, files[i].dbRoot, files[i].partitionNum, files[i].segmentNum, files[i].compType, NULL); - fdit = fdcache.find(fdKey); + for (uint32_t i = 0; i < files.size(); i++) + { + FdEntry fdKey(files[i].oid, files[i].dbRoot, files[i].partitionNum, files[i].segmentNum, + files[i].compType, NULL); + fdit = fdcache.find(fdKey); - if (fdit != fdcache.end()) - fdcache.erase(fdit); - } + if (fdit != fdcache.end()) + fdcache.erase(fdit); + } - localLock.write_unlock(); + localLock.write_unlock(); } -ioManager::ioManager(FileBufferMgr& fbm, - fileBlockRequestQueue& fbrq, - int thrCount, - int bsPerRead): - blocksPerRead(bsPerRead), - fIOMfbMgr(fbm), - fIOMRequestQueue(fbrq), - fFileOp(false) +ioManager::ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead) + : blocksPerRead(bsPerRead), fIOMfbMgr(fbm), fIOMRequestQueue(fbrq), fFileOp(false) { - if (thrCount <= 0) - thrCount = 1; + if (thrCount <= 0) + thrCount = 1; - if (thrCount > 256) - thrCount = 256; + if (thrCount > 256) + thrCount = 256; - fConfig = Config::makeConfig(); - string val = fConfig->getConfig("DBBC", "IOMTracing"); - int temp = 0; + fConfig = Config::makeConfig(); + string val = fConfig->getConfig("DBBC", "IOMTracing"); + int temp = 0; - if (val.length() > 0) temp = static_cast(Config::fromText(val)); + if (val.length() > 0) + temp = static_cast(Config::fromText(val)); - if (temp > 0) - fIOTrace = true; - else - fIOTrace = false; + if (temp > 0) + fIOTrace = true; + else + fIOTrace = false; - val = fConfig->getConfig("DBBC", "MaxOpenFiles"); - temp = 0; - fMaxOpenFiles = MAX_OPEN_FILES; + val = fConfig->getConfig("DBBC", "MaxOpenFiles"); + temp = 0; + fMaxOpenFiles = MAX_OPEN_FILES; - if (val.length() > 0) temp = static_cast(Config::fromText(val)); + if (val.length() > 0) + temp = static_cast(Config::fromText(val)); - if (temp > 0) - fMaxOpenFiles = temp; + if (temp > 0) + fMaxOpenFiles = temp; - val = fConfig->getConfig("DBBC", "DecreaseOpenFilesCount"); - temp = 0; - fDecreaseOpenFilesCount = DECREASE_OPEN_FILES; + val = fConfig->getConfig("DBBC", "DecreaseOpenFilesCount"); + temp = 0; + fDecreaseOpenFilesCount = DECREASE_OPEN_FILES; - if (val.length() > 0) temp = static_cast(Config::fromText(val)); + if (val.length() > 0) + temp = static_cast(Config::fromText(val)); - if (temp > 0) - fDecreaseOpenFilesCount = temp; + if (temp > 0) + fDecreaseOpenFilesCount = temp; - // limit the number of files closed - if (fDecreaseOpenFilesCount > (uint32_t)(0.75 * fMaxOpenFiles)) - fDecreaseOpenFilesCount = (uint32_t)(0.75 * fMaxOpenFiles); + // limit the number of files closed + if (fDecreaseOpenFilesCount > (uint32_t)(0.75 * fMaxOpenFiles)) + fDecreaseOpenFilesCount = (uint32_t)(0.75 * fMaxOpenFiles); - val = fConfig->getConfig("DBBC", "FDCacheTrace"); - temp = 0; - fFDCacheTrace = false; + val = fConfig->getConfig("DBBC", "FDCacheTrace"); + temp = 0; + fFDCacheTrace = false; - if (val.length() > 0) temp = static_cast(Config::fromText(val)); + if (val.length() > 0) + temp = static_cast(Config::fromText(val)); - if (temp > 0) - { - fFDCacheTrace = true; + if (temp > 0) + { + fFDCacheTrace = true; #ifdef _MSC_VER - FDTraceFile().open("C:/Calpont/log/trace/fdcache", ios_base::ate | ios_base::app); + FDTraceFile().open("C:/Calpont/log/trace/fdcache", ios_base::ate | ios_base::app); #else - FDTraceFile().open(string(MCSLOGDIR) + "/trace/fdcache", ios_base::ate | ios_base::app); + FDTraceFile().open(string(MCSLOGDIR) + "/trace/fdcache", ios_base::ate | ios_base::app); #endif - } + } - fThreadCount = thrCount; - go(); + fThreadCount = thrCount; + go(); } -void ioManager::buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint32_t partNum, const uint16_t segNum, char* file_name) +void ioManager::buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint32_t partNum, + const uint16_t segNum, char* file_name) { - // when it's a request for the version buffer, the dbroot comes in as 0 for legacy reasons - if (dbRoot == 0 && oid < 1000) - dbRoot = fdbrm.getDBRootOfVBOID(oid); + // when it's a request for the version buffer, the dbroot comes in as 0 for legacy reasons + if (dbRoot == 0 && oid < 1000) + dbRoot = fdbrm.getDBRootOfVBOID(oid); - fFileOp.getFileNameForPrimProc(oid, file_name, dbRoot, partNum, segNum); + fFileOp.getFileNameForPrimProc(oid, file_name, dbRoot, partNum, segNum); } -int ioManager::localLbidLookup(BRM::LBID_t lbid, - BRM::VER_t verid, - bool vbFlag, - BRM::OID_t& oid, - uint16_t& dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - uint32_t& fileBlockOffset) +int ioManager::localLbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, + uint16_t& dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + uint32_t& fileBlockOffset) { - if (primitiveprocessor::noVB > 0) - vbFlag = false; + if (primitiveprocessor::noVB > 0) + vbFlag = false; - int rc = fdbrm.lookupLocal(lbid, - verid, - vbFlag, - oid, - dbRoot, - partitionNum, - segmentNum, - fileBlockOffset); + int rc = fdbrm.lookupLocal(lbid, verid, vbFlag, oid, dbRoot, partitionNum, segmentNum, fileBlockOffset); - return rc; + return rc; } struct LambdaKludge { - LambdaKludge(ioManager* i) : iom(i) { } - ~LambdaKludge() - { - iom = NULL; - } - ioManager* iom; - void operator()() - { - thr_popper(iom); - } + LambdaKludge(ioManager* i) : iom(i) + { + } + ~LambdaKludge() + { + iom = NULL; + } + ioManager* iom; + void operator()() + { + thr_popper(iom); + } }; void ioManager::createReaders() { - int idx; + int idx; - for (idx = 0; idx < fThreadCount; idx++) + for (idx = 0; idx < fThreadCount; idx++) + { + try { - try - { - fThreadArr.create_thread(LambdaKludge(this)); - } - catch (exception& e) - { - cerr << "IOM::createReaders() caught " << e.what() << endl; - idx--; - sleep(1); - continue; - } + fThreadArr.create_thread(LambdaKludge(this)); } + catch (exception& e) + { + cerr << "IOM::createReaders() caught " << e.what() << endl; + idx--; + sleep(1); + continue; + } + } } ioManager::~ioManager() { - stop(); + stop(); } void ioManager::go(void) { - createReaders(); + createReaders(); } - -//FIXME: is this right? what does this method do? +// FIXME: is this right? what does this method do? void ioManager::stop() { - for (int idx = 0; idx < fThreadCount; idx++) - { - (void)0; //pthread_detach(fThreadArr[idx]); - } + for (int idx = 0; idx < fThreadCount; idx++) + { + (void)0; // pthread_detach(fThreadArr[idx]); + } } - fileRequest* ioManager::getNextRequest() { - fileRequest* blk = 0; - - try - { - blk = fIOMRequestQueue.pop(); - return blk; - } - catch (exception&) - { - cerr << "ioManager::getNextRequest() ERROR " << endl; - } + fileRequest* blk = 0; + try + { + blk = fIOMRequestQueue.pop(); return blk; + } + catch (exception&) + { + cerr << "ioManager::getNextRequest() ERROR " << endl; + } + return blk; } //------------------------------------------------------------------------------ // Prints stderr msg and updates fileRequest object to reflect an error. // Lastly, notifies waiting thread that fileRequest has been completed. //------------------------------------------------------------------------------ -void ioManager::handleBlockReadError( fileRequest* fr, - const string& errMsg, bool* copyLocked, int errorCode ) +void ioManager::handleBlockReadError(fileRequest* fr, const string& errMsg, bool* copyLocked, int errorCode) { - try - { - dbrm()->releaseLBIDRange(fr->Lbid(), fr->BlocksRequested()); - *copyLocked = false; - } - catch (exception& e) - { - cout << "releaseRange on read error: " << e.what() << endl; - } + try + { + dbrm()->releaseLBIDRange(fr->Lbid(), fr->BlocksRequested()); + *copyLocked = false; + } + catch (exception& e) + { + cout << "releaseRange on read error: " << e.what() << endl; + } - cerr << errMsg << endl; - fr->RequestStatus(errorCode); - fr->RequestStatusStr(errMsg); + cerr << errMsg << endl; + fr->RequestStatus(errorCode); + fr->RequestStatusStr(errMsg); - fr->frMutex().lock(); - fr->SetPredicate(fileRequest::COMPLETE); - fr->frCond().notify_one(); - fr->frMutex().unlock(); + fr->frMutex().lock(); + fr->SetPredicate(fileRequest::COMPLETE); + fr->frCond().notify_one(); + fr->frMutex().unlock(); } -} +} // namespace dbbc // vim:ts=4 sw=4: diff --git a/primitives/blockcache/iomanager.h b/primitives/blockcache/iomanager.h index b7c93fd6f..6cf61569f 100644 --- a/primitives/blockcache/iomanager.h +++ b/primitives/blockcache/iomanager.h @@ -30,7 +30,7 @@ // /** - @author Jason Rodriguez + @author Jason Rodriguez */ #include @@ -47,112 +47,97 @@ namespace dbbc { - class ioManager { + public: + ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead); + // ioManager(FileBufferMgr& fbm, int thrCount); + ~ioManager(); + int readerCount() const + { + return fThreadCount; + } + fileRequest* getNextRequest(); + void go(void); + void stop(); + FileBufferMgr& fileBufferManager() + { + return fIOMfbMgr; + } + config::Config* configPtr() + { + return fConfig; + } -public: + int localLbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, uint16_t& dbRoot, + uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset); - ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, - int bsPerRead); - //ioManager(FileBufferMgr& fbm, int thrCount); - ~ioManager(); - int readerCount() const - { - return fThreadCount; - } - fileRequest* getNextRequest(); - void go(void); - void stop(); - FileBufferMgr& fileBufferManager() - { - return fIOMfbMgr; - } - config::Config* configPtr() - { - return fConfig; - } + void buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint32_t partNum, const uint16_t segNum, + char* file_name); - int localLbidLookup(BRM::LBID_t lbid, - BRM::VER_t verid, - bool vbFlag, - BRM::OID_t& oid, - uint16_t& dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - uint32_t& fileBlockOffset); + uint32_t getExtentRows() + { + return fdbrm.getExtentRows(); + } - void buildOidFileName(const BRM::OID_t oid, - uint16_t dbRoot, - const uint32_t partNum, - const uint16_t segNum, - char* file_name); + uint32_t blocksPerRead; - uint32_t getExtentRows() - { - return fdbrm.getExtentRows(); - } + bool IOTrace() const + { + return fIOTrace; + } - uint32_t blocksPerRead; + uint32_t MaxOpenFiles() const + { + return fMaxOpenFiles; + } - bool IOTrace() const - { - return fIOTrace; - } + uint32_t DecreaseOpenFilesCount() const + { + return fDecreaseOpenFilesCount; + } - uint32_t MaxOpenFiles() const - { - return fMaxOpenFiles; - } + bool FDCacheTrace() const + { + return fFDCacheTrace; + } - uint32_t DecreaseOpenFilesCount() const - { - return fDecreaseOpenFilesCount; - } + void handleBlockReadError(fileRequest* fr, const std::string& errMsg, bool* copyLocked, + int errorCode = fileRequest::FAILED); - bool FDCacheTrace() const - { - return fFDCacheTrace; - } - - void handleBlockReadError ( fileRequest* fr, - const std::string& errMsg, bool* copyLocked, int errorCode = fileRequest::FAILED ); - - std::ofstream& FDTraceFile() - { - return fFDTraceFile; - } - - BRM::DBRM* dbrm() - { - return &fdbrm; - } + std::ofstream& FDTraceFile() + { + return fFDTraceFile; + } + BRM::DBRM* dbrm() + { + return &fdbrm; + } #ifdef SHARED_NOTHING_DEMO_2 - uint32_t pmCount; + uint32_t pmCount; #endif -private: + private: + FileBufferMgr& fIOMfbMgr; + fileBlockRequestQueue& fIOMRequestQueue; + int fThreadCount; + boost::thread_group fThreadArr; + void createReaders(); + config::Config* fConfig; + BRM::DBRM fdbrm; + WriteEngine::FileOp fFileOp; - FileBufferMgr& fIOMfbMgr; - fileBlockRequestQueue& fIOMRequestQueue; - int fThreadCount; - boost::thread_group fThreadArr; - void createReaders(); - config::Config* fConfig; - BRM::DBRM fdbrm; - WriteEngine::FileOp fFileOp; - - // do not implement - ioManager(); - ioManager(const ioManager& iom); - const ioManager& operator=(const ioManager& iom); - bool fIOTrace; - uint32_t fMaxOpenFiles; - uint32_t fDecreaseOpenFilesCount; - bool fFDCacheTrace; - std::ofstream fFDTraceFile; + // do not implement + ioManager(); + ioManager(const ioManager& iom); + const ioManager& operator=(const ioManager& iom); + bool fIOTrace; + uint32_t fMaxOpenFiles; + uint32_t fDecreaseOpenFilesCount; + bool fFDCacheTrace; + std::ofstream fFDTraceFile; }; // @bug2631, for remount filesystem by loadBlock() in primitiveserver @@ -162,6 +147,6 @@ void releaseReadLock(); void dropFDCache(); void purgeFDCache(std::vector& files); -} +} // namespace dbbc #endif // vim:ts=4 sw=4: diff --git a/primitives/blockcache/stats.cpp b/primitives/blockcache/stats.cpp index b451c51de..50372758e 100644 --- a/primitives/blockcache/stats.cpp +++ b/primitives/blockcache/stats.cpp @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /* -* $Id: stats.cpp 2035 2013-01-21 14:12:19Z rdempsey $ -*/ + * $Id: stats.cpp 2035 2013-01-21 14:12:19Z rdempsey $ + */ #include #include @@ -48,247 +48,243 @@ using namespace BRM; namespace { - void pause_(unsigned delay) { - struct timespec req; - struct timespec rem; + struct timespec req; + struct timespec rem; - req.tv_sec = delay; - req.tv_nsec = 0; + req.tv_sec = delay; + req.tv_nsec = 0; - rem.tv_sec = 0; - rem.tv_nsec = 0; + rem.tv_sec = 0; + rem.tv_nsec = 0; #ifdef _MSC_VER - Sleep(req.tv_sec * 1000); + Sleep(req.tv_sec * 1000); #else again: - if (nanosleep(&req, &rem) != 0) - if (rem.tv_sec > 0 || rem.tv_nsec > 0) - { - req = rem; - goto again; - } + if (nanosleep(&req, &rem) != 0) + if (rem.tv_sec > 0 || rem.tv_nsec > 0) + { + req = rem; + goto again; + } #endif } const string timestr() { - // Get a timestamp for output. - struct tm tm; - struct timeval tv; + // Get a timestamp for output. + struct tm tm; + struct timeval tv; - gettimeofday(&tv, 0); - localtime_r(reinterpret_cast(&tv.tv_sec), &tm); + gettimeofday(&tv, 0); + localtime_r(reinterpret_cast(&tv.tv_sec), &tm); - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec << '.' - << setw(4) << tv.tv_usec / 100; + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec + << '.' << setw(4) << tv.tv_usec / 100; - return oss.str(); + return oss.str(); } class TraceFile { -public: - TraceFile(uint32_t sessionID, const char* name) + public: + TraceFile(uint32_t sessionID, const char* name) + { + if (sessionID > 0) { - if (sessionID > 0 ) - { - const char* outName; + const char* outName; - if (name == 0) - outName = "lbids"; - else - outName = name; + if (name == 0) + outName = "lbids"; + else + outName = name; - ostringstream oss; + ostringstream oss; #ifdef _MSC_VER - oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID; + oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID; #else - oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID; + oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID; #endif - oFile.reset(new ofstream()); - oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); - } + oFile.reset(new ofstream()); + oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); } + } - ~TraceFile() + ~TraceFile() + { + } + + void close() + { + if (oFile) { + oFile->close(); } + } - void close() - { - if (oFile) - { - oFile->close(); - } - } + void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') + { + *oFile << oid << ' ' << timestr() << ' ' << lbid << ' ' << thdid; - void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') - { - *oFile << oid << ' ' << timestr() << ' ' << lbid - << ' ' << thdid; + if (event != '\0') + *oFile << ' ' << event; - if (event != '\0') - *oFile << ' ' << event; - - *oFile << endl; - oFile->flush(); - } - -private: - //Compiler defaults okay - //TraceFile(const TraceFile& rhs); - //TraceFile operator=(const TraceFile& rhs); - boost::shared_ptr oFile; + *oFile << endl; + oFile->flush(); + } + private: + // Compiler defaults okay + // TraceFile(const TraceFile& rhs); + // TraceFile operator=(const TraceFile& rhs); + boost::shared_ptr oFile; }; struct TraceFileInfo { - TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) { } - ~TraceFileInfo() { } + TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) + { + } + ~TraceFileInfo() + { + } - void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') - { - traceFile.log(oid, lbid, thdid, event); - lastTouched = time(0); - } + void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') + { + traceFile.log(oid, lbid, thdid, event); + lastTouched = time(0); + } - void close() - { - traceFile.close(); - } + void close() + { + traceFile.close(); + } - TraceFile traceFile; - time_t lastTouched; + TraceFile traceFile; + time_t lastTouched; -private: - //Compiler defaults okay - //TraceFileInfo(const TraceFileInfo& rhs); - //TraceFileInfo operator=(const TraceFileInfo& rhs); + private: + // Compiler defaults okay + // TraceFileInfo(const TraceFileInfo& rhs); + // TraceFileInfo operator=(const TraceFileInfo& rhs); }; -//map a session id to a trace file +// map a session id to a trace file typedef map TraceFileMap_t; TraceFileMap_t traceFileMap; -//map mutex +// map mutex boost::mutex traceFileMapMutex; class StatMon { -public: - StatMon() - { + public: + StatMon() + { #ifndef _MSC_VER - sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGPIPE); - sigaddset(&sigset, SIGUSR1); - sigaddset(&sigset, SIGUSR2); - pthread_sigmask(SIG_BLOCK, &sigset, 0); + sigset_t sigset; + sigemptyset(&sigset); + sigaddset(&sigset, SIGPIPE); + sigaddset(&sigset, SIGUSR1); + sigaddset(&sigset, SIGUSR2); + pthread_sigmask(SIG_BLOCK, &sigset, 0); #endif - } - void operator()() const + } + void operator()() const + { + // struct timespec ts = { 60 * 1, 0 }; + boost::mutex::scoped_lock lk(traceFileMapMutex); + TraceFileMap_t::iterator iter; + TraceFileMap_t::iterator end; + + for (;;) { - //struct timespec ts = { 60 * 1, 0 }; - boost::mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter; - TraceFileMap_t::iterator end; + lk.unlock(); + time_t beforeSleep = time(0); + // nanosleep(&ts, 0); + pause_(60); + lk.lock(); + iter = traceFileMap.begin(); + end = traceFileMap.end(); - for (;;) + while (iter != end) + { + if (iter->second.lastTouched < beforeSleep) { - lk.unlock(); - time_t beforeSleep = time(0); - //nanosleep(&ts, 0); - pause_(60); - lk.lock(); - iter = traceFileMap.begin(); - end = traceFileMap.end(); - - while (iter != end) - { - if (iter->second.lastTouched < beforeSleep) - { - //remove this session trace file - iter->second.close(); - traceFileMap.erase(iter++); - } - else - ++iter; - } + // remove this session trace file + iter->second.close(); + traceFileMap.erase(iter++); } + else + ++iter; + } } -private: - //Compiler defaults okay - //StatMon(const StatMon& rhs); - //StatMon operator=(const StatMon& rhs); + } + + private: + // Compiler defaults okay + // StatMon(const StatMon& rhs); + // StatMon operator=(const StatMon& rhs); }; -} +} // namespace namespace dbbc { - -Stats::Stats() : - fMonitorp(0) +Stats::Stats() : fMonitorp(0) { - - fMonitorp = new boost::thread(StatMon()); + fMonitorp = new boost::thread(StatMon()); } -Stats::Stats(const char* name) : - fMonitorp(0), fName(name) +Stats::Stats(const char* name) : fMonitorp(0), fName(name) { - fMonitorp = new boost::thread(StatMon()); - //fName << name; + fMonitorp = new boost::thread(StatMon()); + // fName << name; } Stats::~Stats() { - delete fMonitorp; + delete fMonitorp; } void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session) { - if (session == 0) return; + if (session == 0) + return; - boost::mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter = traceFileMap.find(session); + boost::mutex::scoped_lock lk(traceFileMapMutex); + TraceFileMap_t::iterator iter = traceFileMap.find(session); - if (iter == traceFileMap.end()) - { - traceFileMap[session] = TraceFileInfo(session); - iter = traceFileMap.find(session); - idbassert(iter != traceFileMap.end()); - } + if (iter == traceFileMap.end()) + { + traceFileMap[session] = TraceFileInfo(session); + iter = traceFileMap.find(session); + idbassert(iter != traceFileMap.end()); + } - iter->second.log(lbid2oid(lbid), lbid, thdid); + iter->second.log(lbid2oid(lbid), lbid, thdid); } void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event) { - if (session == 0) return; + if (session == 0) + return; - boost::mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter = traceFileMap.find(session); + boost::mutex::scoped_lock lk(traceFileMapMutex); + TraceFileMap_t::iterator iter = traceFileMap.find(session); - if (iter == traceFileMap.end()) - { - traceFileMap[session] = TraceFileInfo(session, fName); - iter = traceFileMap.find(session); - idbassert(iter != traceFileMap.end()); - } - - iter->second.log(lbid2oid(lbid), lbid, thdid, event); -} + if (iter == traceFileMap.end()) + { + traceFileMap[session] = TraceFileInfo(session, fName); + iter = traceFileMap.find(session); + idbassert(iter != traceFileMap.end()); + } + iter->second.log(lbid2oid(lbid), lbid, thdid, event); } +} // namespace dbbc diff --git a/primitives/blockcache/stats.h b/primitives/blockcache/stats.h index 1e7e2ac9e..a32c2ecb3 100644 --- a/primitives/blockcache/stats.h +++ b/primitives/blockcache/stats.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: stats.h 2035 2013-01-21 14:12:19Z rdempsey $ -*/ + * $Id: stats.h 2035 2013-01-21 14:12:19Z rdempsey $ + */ /** @file */ @@ -27,7 +27,7 @@ #ifndef _MSC_VER #include #else -//FIXME: find a portable solution +// FIXME: find a portable solution typedef int pthread_t; #endif #include @@ -38,39 +38,37 @@ typedef int pthread_t; namespace dbbc { - class Stats { -public: - Stats(); - Stats(const char* name); - virtual ~Stats(); + public: + Stats(); + Stats(const char* name); + virtual ~Stats(); - void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0); - void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event); + void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0); + void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event); - inline BRM::OID_t lbid2oid(uint64_t lbid) - { - BRM::OID_t oid; - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t fbo; - brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo); - return oid; - } + inline BRM::OID_t lbid2oid(uint64_t lbid) + { + BRM::OID_t oid; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t fbo; + brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo); + return oid; + } -private: - Stats(const Stats& rhs); - Stats& operator=(const Stats& rhs); - - boost::thread* fMonitorp; - BRM::DBRM brm; - //ostringstream fName; - const char* fName; + private: + Stats(const Stats& rhs); + Stats& operator=(const Stats& rhs); + boost::thread* fMonitorp; + BRM::DBRM brm; + // ostringstream fName; + const char* fName; }; -} +} // namespace dbbc -#endif //PRIMPROC_STATS_H +#endif // PRIMPROC_STATS_H diff --git a/primitives/blockcache/tdriver.cpp b/primitives/blockcache/tdriver.cpp index 489c808c5..bdef557e5 100644 --- a/primitives/blockcache/tdriver.cpp +++ b/primitives/blockcache/tdriver.cpp @@ -56,10 +56,10 @@ uint64_t noOpCountTot = 0; struct thr_wait_struct { - int predicate; - pthread_mutex_t fMutex; - pthread_cond_t fCond; - vector range_thr; + int predicate; + pthread_mutex_t fMutex; + pthread_cond_t fCond; + vector range_thr; }; typedef thr_wait_struct thr_wait_t; @@ -70,339 +70,343 @@ u_int64_t totBlocks = 0; void* thr_client(void* clientArgs) { - blockCacheClient bc(BRP); - uint64_t bfound = 0; - uint64_t bnfound = 0; - uint64_t rfound = 0; - uint64_t rnfound = 0; - uint64_t rangeOpCount = 0; - uint64_t blockOpCount = 0; - uint64_t noOpCount = 0; - thr_wait_t* clientWait = (thr_wait_t*)clientArgs; - struct timeval tv, tv2; - uint32_t randstate = 0; - randstate = static_cast(tv.tv_usec); - pthread_mutex_lock(&clientWait->fMutex); - clientWait->predicate++; - pthread_mutex_unlock(&clientWait->fMutex); - vector& range_thr = clientWait->range_thr; + blockCacheClient bc(BRP); + uint64_t bfound = 0; + uint64_t bnfound = 0; + uint64_t rfound = 0; + uint64_t rnfound = 0; + uint64_t rangeOpCount = 0; + uint64_t blockOpCount = 0; + uint64_t noOpCount = 0; + thr_wait_t* clientWait = (thr_wait_t*)clientArgs; + struct timeval tv, tv2; + uint32_t randstate = 0; + randstate = static_cast(tv.tv_usec); + pthread_mutex_lock(&clientWait->fMutex); + clientWait->predicate++; + pthread_mutex_unlock(&clientWait->fMutex); + vector& range_thr = clientWait->range_thr; - gettimeofday(&tv, NULL); - int j = 0; - int s = 0; - uint8_t fb[8192] = {0}; - LBIDRange_v& r = range_thr[0]; + gettimeofday(&tv, NULL); + int j = 0; + int s = 0; + uint8_t fb[8192] = {0}; + LBIDRange_v& r = range_thr[0]; - for (int idx = 0; idx < fLoops; idx++) + for (int idx = 0; idx < fLoops; idx++) + { + for (int jdx = 0; jdx < range_thr.size(); jdx++) { - for (int jdx = 0; jdx < range_thr.size(); jdx++) + uint32_t lbid = 0; + bool b; + int ret = 0; + r = range_thr[jdx]; + + for (int l = 0; l < r.size(); l++) + { + for (int m = r[l].start; m < r[l].start + r[l].size; m++) { - uint32_t lbid = 0; - bool b; - int ret = 0; - r = range_thr[jdx]; + // ret=bc.getBlock(m, ver, &fb, false, b); + ret = bc.read(m, ver, &fb); - for (int l = 0; l < r.size(); l++) - { - for (int m = r[l].start; m < r[l].start + r[l].size; m++) - { - //ret=bc.getBlock(m, ver, &fb, false, b); - ret = bc.read(m, ver, &fb); - - if (ret) - { - bfound++; - } - else - { - bnfound++; - } - } - } + if (ret) + { + bfound++; + } + else + { + bnfound++; + } } + } } + } - gettimeofday(&tv2, NULL); - time_t tm = time(0); - char t[50]; - ctime_r(&tm, t); - t[strlen(t) - 1] = 0; - uint32_t elTime = tv2.tv_sec - tv.tv_sec; - uint64_t avgTot = 0; - uint64_t rangeAvg = 0; - uint64_t blkAvg = 0; + gettimeofday(&tv2, NULL); + time_t tm = time(0); + char t[50]; + ctime_r(&tm, t); + t[strlen(t) - 1] = 0; + uint32_t elTime = tv2.tv_sec - tv.tv_sec; + uint64_t avgTot = 0; + uint64_t rangeAvg = 0; + uint64_t blkAvg = 0; - if (elTime > 0) - { - avgTot = (bfound + rfound) / elTime; - rangeAvg = (rfound) / elTime; - blkAvg = (bfound) / elTime; - } - else - { - avgTot = bfound + rfound; - rangeAvg = rfound; - blkAvg = bfound; - } + if (elTime > 0) + { + avgTot = (bfound + rfound) / elTime; + rangeAvg = (rfound) / elTime; + blkAvg = (bfound) / elTime; + } + else + { + avgTot = bfound + rfound; + rangeAvg = rfound; + blkAvg = bfound; + } - cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl << - "\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound << - " Blks/Sec Blk " << blkAvg << endl << endl; + cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl + << "\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound << " Blks/Sec Blk " + << blkAvg << endl + << endl; - pthread_mutex_lock(&clientWait->fMutex); - bfoundTot += bfound; - bnfoundTot += bnfound; - rfoundTot += rfound; - rnfoundTot += rnfound; - rangeOpCountTot += rangeOpCount; - blockOpCountTot += blockOpCount; - noOpCountTot += noOpCount; - clientWait->predicate--; - pthread_cond_signal(&clientWait->fCond); - pthread_mutex_unlock(&clientWait->fMutex); + pthread_mutex_lock(&clientWait->fMutex); + bfoundTot += bfound; + bnfoundTot += bnfound; + rfoundTot += rfound; + rnfoundTot += rnfound; + rangeOpCountTot += rangeOpCount; + blockOpCountTot += blockOpCount; + noOpCountTot += noOpCount; + clientWait->predicate--; + pthread_cond_signal(&clientWait->fCond); + pthread_mutex_unlock(&clientWait->fMutex); - return NULL; + return NULL; -} // end thr_client +} // end thr_client void LoadRange(const LBIDRange_v& v, uint32_t& loadCount) { - blockCacheClient bc(BRP); + blockCacheClient bc(BRP); - uint32_t rCount = 0; + uint32_t rCount = 0; - for (uint32_t i = 0; i < v.size() ; i++) + for (uint32_t i = 0; i < v.size(); i++) + { + const InlineLBIDRange r = {v[i].start, v[i].size}; + + if (r.size <= 1024) { - const InlineLBIDRange r = {v[i].start, v[i].size}; - - if (r.size <= 1024) - { - bc.check(r, ver, rCount ); - loadCount += rCount; - } - - rCount = 0; - + bc.check(r, ver, rCount); + loadCount += rCount; } + rCount = 0; + } } void ReadRange(const LBIDRange_v& v) { - blockCacheClient bc(BRP); - int found = 0; - int notfound = 0; - int ret = 0; + blockCacheClient bc(BRP); + int found = 0; + int notfound = 0; + int ret = 0; - for (uint32_t i = 0; i < v.size(); i++) + for (uint32_t i = 0; i < v.size(); i++) + { + const InlineLBIDRange r = {v[i].start, v[i].size}; + FileBuffer fb(-1, -1); + + for (int j = r.start; j < r.start + r.size; j++) { - const InlineLBIDRange r = {v[i].start, v[i].size}; - FileBuffer fb(-1, -1); + if (r.size > 1024) + continue; - for (int j = r.start; j < r.start + r.size; j++) - { - if (r.size > 1024) - continue; + ret = bc.read(j, ver, fb); - ret = bc.read(j, ver, fb); + if (ret) + found++; + else + notfound++; - if (ret) - found++; - else - notfound++; - - ret = 0; - } - - totBlocks += found; - totBlocks += notfound; - found = 0; - notfound = 0; + ret = 0; } + totBlocks += found; + totBlocks += notfound; + found = 0; + notfound = 0; + } } -void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) +void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); + blockCacheClient bc(BRP); + bool b; + bc.check(lbid, ver, false, b); } void ReadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - static int found = 0, notfound = 0; - uint8_t d[8192]; - blockCacheClient bc(BRP); - //FileBuffer fb(-1, -1); - //bc.read(lbid, ver, fb); - int ret = bc.read(lbid, ver, d); + static int found = 0, notfound = 0; + uint8_t d[8192]; + blockCacheClient bc(BRP); + // FileBuffer fb(-1, -1); + // bc.read(lbid, ver, fb); + int ret = bc.read(lbid, ver, d); - if (ret) - found++; - else - notfound++; + if (ret) + found++; + else + notfound++; - if ((found + notfound) % 10000 == 0) - cout << "found " << found << " notfound " << notfound << endl; + if ((found + notfound) % 10000 == 0) + cout << "found " << found << " notfound " << notfound << endl; } // int main(int argc, char* argv[]) { + if (argc >= 2) + thr_cnt = atoi(argv[1]); - if (argc >= 2) thr_cnt = atoi(argv[1]); + if (argc >= 3) + fLoops = atoi(argv[2]); - if (argc >= 3) fLoops = atoi(argv[2]); + if (thr_cnt <= 0) + thr_cnt = 1; - if (thr_cnt <= 0) thr_cnt = 1; + if (thr_cnt > 1024) + thr_cnt = 1024; - if (thr_cnt > 1024) thr_cnt = 1024; + if (fLoops <= 0) + fLoops = 1; - if (fLoops <= 0) fLoops = 1; + LBIDRange_v r; + vector ranges; + DBRM dbrm; + uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid; + struct timeval tv, tv2; - LBIDRange_v r; - vector ranges; - DBRM dbrm; - uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid; - struct timeval tv, tv2; + cout << "Starting " << endl; + extentSize = dbrm.getExtentSize(); + BRM::OID_t oid = 3000; + uint32_t totalExt = 0; - cout << "Starting " << endl; - extentSize = dbrm.getExtentSize(); - BRM::OID_t oid = 3000; - uint32_t totalExt = 0; + do + { + int ret = dbrm.lookup(oid, r); - do + if (ret == 0 && r.size() > 0) { - int ret = dbrm.lookup(oid, r); + lowlbid = (r[0].start / extentSize) * extentSize; + dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid + dbrm.getHWM(oid, hwm); + lowfbo = fbo - (r[0].start - lowlbid); + highfbo = lowfbo + extentSize; + r[0].start = lowlbid; - if (ret == 0 && r.size() > 0) - { - lowlbid = (r[0].start / extentSize) * extentSize; - dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid - dbrm.getHWM(oid, hwm); - lowfbo = fbo - (r[0].start - lowlbid); - highfbo = lowfbo + extentSize; - r[0].start = lowlbid; + if (hwm < highfbo) + r[0].size = hwm - lowfbo + 1; + else + r[0].size = extentSize; - if (hwm < highfbo) - r[0].size = hwm - lowfbo + 1; - else - r[0].size = extentSize; + for (uint32_t idx = 0; idx < r.size(); idx++) + totalExt += r[idx].size; - for (uint32_t idx = 0; idx < r.size(); idx++) - totalExt += r[idx].size; - - ranges.push_back(r); - } - - oid++; - } - while ( (r.size() > 0 || oid < 900000) ); - - cout << ranges.size() << " ranges found" << endl; - - gettimeofday(&tv, NULL); - uint32_t blksLoaded = 0; - int rangesLoaded = 0; - - for (uint32_t i = 0; i < ranges.size() && blksLoaded < cacheSize; i++) - { - LoadRange(ranges[i], blksLoaded); - rangesLoaded++; + ranges.push_back(r); } - cout << endl; + oid++; + } while ((r.size() > 0 || oid < 900000)); - gettimeofday(&tv2, NULL); - cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec << endl; + cout << ranges.size() << " ranges found" << endl; - while (ranges.size() > rangesLoaded) ranges.pop_back(); + gettimeofday(&tv, NULL); + uint32_t blksLoaded = 0; + int rangesLoaded = 0; + + for (uint32_t i = 0; i < ranges.size() && blksLoaded < cacheSize; i++) + { + LoadRange(ranges[i], blksLoaded); + rangesLoaded++; + } + + cout << endl; + + gettimeofday(&tv2, NULL); + cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec + << endl; + + while (ranges.size() > rangesLoaded) + ranges.pop_back(); #ifdef BLAH - for (uint32_t i = 0; i < ranges; i++) - ReadRange(ranges[i]); + for (uint32_t i = 0; i < ranges; i++) + ReadRange(ranges[i]); - for (uint32_t i = 0; i < ranges.size(); i++) + for (uint32_t i = 0; i < ranges.size(); i++) + { + LBIDRange_v rv = ranges[i]; + + for (uint32_t j = 0; j < rv.size(); j++) { - LBIDRange_v rv = ranges[i]; + const InlineLBIDRange l = {rv[j].start, rv[j].size}; - for (uint32_t j = 0; j < rv.size(); j++) - { - const InlineLBIDRange l = {rv[j].start, rv[j].size}; - - for (uint32_t k = l.start; k < l.start + l.size; k++) - { - LoadLbid(k, ver); - ReadLbid(k, ver); - } - } + for (uint32_t k = l.start; k < l.start + l.size; k++) + { + LoadLbid(k, ver); + ReadLbid(k, ver); + } } + } #endif - pthread_t thr_id[thr_cnt]; - thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges}; + pthread_t thr_id[thr_cnt]; + thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges}; - //start threads running - cout << "Starting driver threads" << endl; - gettimeofday(&tv, NULL); - memset(thr_id, 0, thr_cnt * (sizeof(pthread_t))); + // start threads running + cout << "Starting driver threads" << endl; + gettimeofday(&tv, NULL); + memset(thr_id, 0, thr_cnt * (sizeof(pthread_t))); - for (int i = 0; i < thr_cnt; i++) - { - pthread_create(&thr_id[i], NULL, thr_client, &thr_wait); - } + for (int i = 0; i < thr_cnt; i++) + { + pthread_create(&thr_id[i], NULL, thr_client, &thr_wait); + } - // waiting until all threads have indicated completion - pthread_mutex_lock(&thr_wait.fMutex); + // waiting until all threads have indicated completion + pthread_mutex_lock(&thr_wait.fMutex); - while (thr_wait.predicate > 0) - { - pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex); - } + while (thr_wait.predicate > 0) + { + pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex); + } - pthread_mutex_unlock(&thr_wait.fMutex); + pthread_mutex_unlock(&thr_wait.fMutex); - // join threads back to main - for (int i = 0; i < thr_cnt; i++) - { - pthread_join(thr_id[i], NULL); - } + // join threads back to main + for (int i = 0; i < thr_cnt; i++) + { + pthread_join(thr_id[i], NULL); + } - gettimeofday(&tv2, NULL); - time_t tm = time(0); - char t[50]; - ctime_r(&tm, t); - t[strlen(t) - 1] = 0; - uint32_t elTime = tv2.tv_sec - tv.tv_sec; - uint64_t total = bfoundTot + rfoundTot; - uint64_t avgTot = 0; - uint64_t rangeAvg = 0; - uint64_t blkAvg = 0; + gettimeofday(&tv2, NULL); + time_t tm = time(0); + char t[50]; + ctime_r(&tm, t); + t[strlen(t) - 1] = 0; + uint32_t elTime = tv2.tv_sec - tv.tv_sec; + uint64_t total = bfoundTot + rfoundTot; + uint64_t avgTot = 0; + uint64_t rangeAvg = 0; + uint64_t blkAvg = 0; - if (elTime > 0) - { - avgTot = (bfoundTot + rfoundTot) / elTime; - rangeAvg = (rfoundTot) / elTime; - blkAvg = (bfoundTot) / elTime; - } - else - { - avgTot = bfoundTot + rfoundTot; - rangeAvg = rfoundTot; - blkAvg = bfoundTot; - } + if (elTime > 0) + { + avgTot = (bfoundTot + rfoundTot) / elTime; + rangeAvg = (rfoundTot) / elTime; + blkAvg = (bfoundTot) / elTime; + } + else + { + avgTot = bfoundTot + rfoundTot; + rangeAvg = rfoundTot; + blkAvg = bfoundTot; + } - cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl << - "\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot << - //"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl << - //"\tNoOp: c " << noOpCountTot << " Total " << total << endl << - //"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl; - " Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl << endl; + cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl + << "\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot << + //"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl << + //"\tNoOp: c " << noOpCountTot << " Total " << total << endl << + //"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt + //<< endl << endl; + " Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl + << endl; + return 0; - return 0; - -} // end main +} // end main diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index f1533fe7e..825ac86b7 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -54,152 +54,116 @@ using MT = uint16_t; inline uint64_t order_swap(uint64_t x) { - uint64_t ret = (x >> 56) | - ((x << 40) & 0x00FF000000000000ULL) | - ((x << 24) & 0x0000FF0000000000ULL) | - ((x << 8) & 0x000000FF00000000ULL) | - ((x >> 8) & 0x00000000FF000000ULL) | - ((x >> 24) & 0x0000000000FF0000ULL) | - ((x >> 40) & 0x000000000000FF00ULL) | - (x << 56); - return ret; + uint64_t ret = (x >> 56) | ((x << 40) & 0x00FF000000000000ULL) | ((x << 24) & 0x0000FF0000000000ULL) | + ((x << 8) & 0x000000FF00000000ULL) | ((x >> 8) & 0x00000000FF000000ULL) | + ((x >> 24) & 0x0000000000FF0000ULL) | ((x >> 40) & 0x000000000000FF00ULL) | (x << 56); + return ret; } template -inline int compareBlock( const void* a, const void* b ) +inline int compareBlock(const void* a, const void* b) { - return ( (*(T*)a) - (*(T*)b) ); + return ((*(T*)a) - (*(T*)b)); } -//this function is out-of-band, we don't need to inline it +// this function is out-of-band, we don't need to inline it void logIt(int mid, int arg1, const string& arg2 = string()) { - MessageLog logger(LoggingID(28)); - logging::Message::Args args; - Message msg(mid); + MessageLog logger(LoggingID(28)); + logging::Message::Args args; + Message msg(mid); - args.add(arg1); + args.add(arg1); - if (arg2.length() > 0) - args.add(arg2); + if (arg2.length() > 0) + args.add(arg2); - msg.format(args); - logger.logErrorMessage(msg); + msg.format(args); + logger.logErrorMessage(msg); } -template +template inline bool colCompare_(const T& val1, const T& val2, uint8_t COP) { - switch (COP) - { - case COMPARE_NIL: - return false; + switch (COP) + { + case COMPARE_NIL: return false; - case COMPARE_LT: - return val1 < val2; + case COMPARE_LT: return val1 < val2; - case COMPARE_EQ: - return val1 == val2; + case COMPARE_EQ: return val1 == val2; - case COMPARE_LE: - return val1 <= val2; + case COMPARE_LE: return val1 <= val2; - case COMPARE_GT: - return val1 > val2; + case COMPARE_GT: return val1 > val2; - case COMPARE_NE: - return val1 != val2; + case COMPARE_NE: return val1 != val2; - case COMPARE_GE: - return val1 >= val2; + case COMPARE_GE: return val1 >= val2; - default: - logIt(34, COP, "colCompare_"); - return false; // throw an exception here? - } + default: logIt(34, COP, "colCompare_"); return false; // throw an exception here? + } } -inline bool colCompareStr(const ColRequestHeaderDataType &type, - uint8_t COP, - const utils::ConstString &val1, - const utils::ConstString &val2) +inline bool colCompareStr(const ColRequestHeaderDataType& type, uint8_t COP, const utils::ConstString& val1, + const utils::ConstString& val2) { - int error = 0; - bool rc = primitives::StringComparator(type).op(&error, COP, val1, val2); - if (error) - { - logIt(34, COP, "colCompareStr"); - return false; // throw an exception here? - } - return rc; + int error = 0; + bool rc = primitives::StringComparator(type).op(&error, COP, val1, val2); + if (error) + { + logIt(34, COP, "colCompareStr"); + return false; // throw an exception here? + } + return rc; } - -template +template inline bool colCompare_(const T& val1, const T& val2, uint8_t COP, uint8_t rf) { - switch (COP) - { - case COMPARE_NIL: - return false; + switch (COP) + { + case COMPARE_NIL: return false; - case COMPARE_LT: - return val1 < val2 || (val1 == val2 && (rf & 0x01)); + case COMPARE_LT: return val1 < val2 || (val1 == val2 && (rf & 0x01)); - case COMPARE_LE: - return val1 < val2 || (val1 == val2 && rf ^ 0x80); + case COMPARE_LE: return val1 < val2 || (val1 == val2 && rf ^ 0x80); - case COMPARE_EQ: - return val1 == val2 && rf == 0; + case COMPARE_EQ: return val1 == val2 && rf == 0; - case COMPARE_NE: - return val1 != val2 || rf != 0; + case COMPARE_NE: return val1 != val2 || rf != 0; - case COMPARE_GE: - return val1 > val2 || (val1 == val2 && rf ^ 0x01); + case COMPARE_GE: return val1 > val2 || (val1 == val2 && rf ^ 0x01); - case COMPARE_GT: - return val1 > val2 || (val1 == val2 && (rf & 0x80)); + case COMPARE_GT: return val1 > val2 || (val1 == val2 && (rf & 0x80)); - default: - logIt(34, COP, "colCompare_"); - return false; // throw an exception here? - } + default: logIt(34, COP, "colCompare_"); return false; // throw an exception here? + } } - //@bug 1828 Like must be a string compare. inline bool colStrCompare_(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf) { - switch (COP) - { - case COMPARE_NIL: - return false; + switch (COP) + { + case COMPARE_NIL: return false; - case COMPARE_LT: - return val1 < val2 || (val1 == val2 && rf != 0); + case COMPARE_LT: return val1 < val2 || (val1 == val2 && rf != 0); - case COMPARE_LE: - return val1 <= val2; + case COMPARE_LE: return val1 <= val2; - case COMPARE_EQ: - return val1 == val2 && rf == 0; + case COMPARE_EQ: return val1 == val2 && rf == 0; - case COMPARE_NE: - return val1 != val2 || rf != 0; + case COMPARE_NE: return val1 != val2 || rf != 0; - case COMPARE_GE: - return val1 > val2 || (val1 == val2 && rf == 0); + case COMPARE_GE: return val1 > val2 || (val1 == val2 && rf == 0); - case COMPARE_GT: - return val1 > val2; + case COMPARE_GT: return val1 > val2; - case COMPARE_LIKE: - case COMPARE_NLIKE: - default: - logIt(34, COP, "colStrCompare_"); - return false; // throw an exception here? - } + case COMPARE_LIKE: + case COMPARE_NLIKE: + default: logIt(34, COP, "colStrCompare_"); return false; // throw an exception here? + } } // Set the minimum and maximum in the return header if we will be doing a block scan and @@ -207,207 +171,167 @@ inline bool colStrCompare_(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf // skip this block if the value being searched is outside of the Min/Max range. inline bool isMinMaxValid(const NewColRequestHeader* in) { - if (in->NVALS != 0) + if (in->NVALS != 0) + { + return false; + } + else + { + switch (in->colType.DataType) { - return false; - } - else - { - switch (in->colType.DataType) - { - case CalpontSystemCatalog::CHAR: - return (in->colType.DataSize < 9); - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - return (in->colType.DataSize < 8); - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - return true; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - return (in->colType.DataSize <= datatypes::MAXDECIMALWIDTH); - - default: - return false; - } + case CalpontSystemCatalog::CHAR: return (in->colType.DataSize < 9); + + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: return (in->colType.DataSize < 8); + + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: return true; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: return (in->colType.DataSize <= datatypes::MAXDECIMALWIDTH); + + default: return false; } + } } -template::type* = nullptr> -inline bool colCompareDispatcherT( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null) +template ::type* = + nullptr> +inline bool colCompareDispatcherT(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null) { - float dVal1 = *((float*) &columnValue); - float dVal2 = *((float*) &filterValue); - return colCompare_(dVal1, dVal2, cop); + float dVal1 = *((float*)&columnValue); + float dVal2 = *((float*)&filterValue); + return colCompare_(dVal1, dVal2, cop); } -template::type* = nullptr> -inline bool colCompareDispatcherT( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null) +template ::type* = + nullptr> +inline bool colCompareDispatcherT(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null) { - double dVal1 = *((double*) &columnValue); - double dVal2 = *((double*) &filterValue); - return colCompare_(dVal1, dVal2, cop); + double dVal1 = *((double*)&columnValue); + double dVal2 = *((double*)&filterValue); + return colCompare_(dVal1, dVal2, cop); } -template::type* = nullptr> -inline bool colCompareDispatcherT( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null) +template ::type* = nullptr> +inline bool colCompareDispatcherT(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null) { - if (cop & COMPARE_LIKE) // LIKE and NOT LIKE - { - utils::ConstString subject{reinterpret_cast(&columnValue), COL_WIDTH}; - utils::ConstString pattern{reinterpret_cast(&filterValue), COL_WIDTH}; - return typeHolder.like(cop & COMPARE_NOT, subject.rtrimZero(), - pattern.rtrimZero()); - } - - if (!rf) - { - // A temporary hack for xxx_nopad_bin collations - // TODO: MCOL-4534 Improve comparison performance in 8bit nopad_bin collations - if ((typeHolder.getCharset().state & (MY_CS_BINSORT|MY_CS_NOPAD)) == - (MY_CS_BINSORT|MY_CS_NOPAD)) - return colCompare_(order_swap(columnValue), order_swap(filterValue), cop); - utils::ConstString s1{reinterpret_cast(&columnValue), COL_WIDTH}; - utils::ConstString s2{reinterpret_cast(&filterValue), COL_WIDTH}; - return colCompareStr(typeHolder, cop, s1.rtrimZero(), s2.rtrimZero()); - } - else - return colStrCompare_(order_swap(columnValue), order_swap(filterValue), cop, rf); + if (cop & COMPARE_LIKE) // LIKE and NOT LIKE + { + utils::ConstString subject{reinterpret_cast(&columnValue), COL_WIDTH}; + utils::ConstString pattern{reinterpret_cast(&filterValue), COL_WIDTH}; + return typeHolder.like(cop & COMPARE_NOT, subject.rtrimZero(), pattern.rtrimZero()); + } + if (!rf) + { + // A temporary hack for xxx_nopad_bin collations + // TODO: MCOL-4534 Improve comparison performance in 8bit nopad_bin collations + if ((typeHolder.getCharset().state & (MY_CS_BINSORT | MY_CS_NOPAD)) == (MY_CS_BINSORT | MY_CS_NOPAD)) + return colCompare_(order_swap(columnValue), order_swap(filterValue), cop); + utils::ConstString s1{reinterpret_cast(&columnValue), COL_WIDTH}; + utils::ConstString s2{reinterpret_cast(&filterValue), COL_WIDTH}; + return colCompareStr(typeHolder, cop, s1.rtrimZero(), s2.rtrimZero()); + } + else + return colStrCompare_(order_swap(columnValue), order_swap(filterValue), cop, rf); } // This template where IS_NULL = true is used only comparing filter predicate // values with column NULL so I left branching here. -template::type* = nullptr> -inline bool colCompareDispatcherT( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null) +template ::type* = nullptr> +inline bool colCompareDispatcherT(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null) { - if (IS_NULL == isVal2Null || (isVal2Null && cop == COMPARE_NE)) + if (IS_NULL == isVal2Null || (isVal2Null && cop == COMPARE_NE)) + { + if (KIND_UNSIGNED == KIND) { - if (KIND_UNSIGNED == KIND) - { - // Ugly hack to convert all to the biggest type b/w T1 and T2. - // I presume that sizeof(T2) AKA a filter predicate type is GEQ sizeof(T1) AKA col type. - using UT2 = typename datatypes::make_unsigned::type; - UT2 ucolumnValue = columnValue; - UT2 ufilterValue = filterValue; - return colCompare_(ucolumnValue, ufilterValue, cop, rf); - } - else - { - // Ugly hack to convert all to the biggest type b/w T1 and T2. - // I presume that sizeof(T2) AKA a filter predicate type is GEQ sizeof(T1) AKA col type. - T2 tempVal1 = columnValue; - return colCompare_(tempVal1, filterValue, cop, rf); - } + // Ugly hack to convert all to the biggest type b/w T1 and T2. + // I presume that sizeof(T2) AKA a filter predicate type is GEQ sizeof(T1) AKA col type. + using UT2 = typename datatypes::make_unsigned::type; + UT2 ucolumnValue = columnValue; + UT2 ufilterValue = filterValue; + return colCompare_(ucolumnValue, ufilterValue, cop, rf); } else - return false; + { + // Ugly hack to convert all to the biggest type b/w T1 and T2. + // I presume that sizeof(T2) AKA a filter predicate type is GEQ sizeof(T1) AKA col type. + T2 tempVal1 = columnValue; + return colCompare_(tempVal1, filterValue, cop, rf); + } + } + else + return false; } -template::type* = nullptr> -inline bool colCompareDispatcherT( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null) +template ::type* = nullptr> +inline bool colCompareDispatcherT(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null) { - if (IS_NULL == isVal2Null || (isVal2Null && cop == COMPARE_NE)) - { - // Ugly hack to convert all to the biggest type b/w T1 and T2. - // I presume that sizeof(T2)(a filter predicate type) is GEQ T1(col type). - using UT2 = typename datatypes::make_unsigned::type; - UT2 ucolumnValue = columnValue; - UT2 ufilterValue = filterValue; - return colCompare_(ucolumnValue, ufilterValue, cop, rf); - } - else - return false; + if (IS_NULL == isVal2Null || (isVal2Null && cop == COMPARE_NE)) + { + // Ugly hack to convert all to the biggest type b/w T1 and T2. + // I presume that sizeof(T2)(a filter predicate type) is GEQ T1(col type). + using UT2 = typename datatypes::make_unsigned::type; + UT2 ucolumnValue = columnValue; + UT2 ufilterValue = filterValue; + return colCompare_(ucolumnValue, ufilterValue, cop, rf); + } + else + return false; } -template::type* = nullptr> -inline bool colCompareDispatcherT( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null) +template ::type* = nullptr> +inline bool colCompareDispatcherT(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null) { - if (IS_NULL == isVal2Null || (isVal2Null && cop == COMPARE_NE)) - { - // Ugly hack to convert all to the biggest type b/w T1 and T2. - // I presume that sizeof(T2)(a filter predicate type) is GEQ T1(col type). - T2 tempVal1 = columnValue; - return colCompare_(tempVal1, filterValue, cop, rf); - } - else - return false; + if (IS_NULL == isVal2Null || (isVal2Null && cop == COMPARE_NE)) + { + // Ugly hack to convert all to the biggest type b/w T1 and T2. + // I presume that sizeof(T2)(a filter predicate type) is GEQ T1(col type). + T2 tempVal1 = columnValue; + return colCompare_(tempVal1, filterValue, cop, rf); + } + else + return false; } // Compare two column values using given comparison operation, // taking into account all rules about NULL values, string trimming and so on -template -inline bool colCompare( - T1 columnValue, - T2 filterValue, - uint8_t cop, - uint8_t rf, - const ColRequestHeaderDataType& typeHolder, - bool isVal2Null = false) +template +inline bool colCompare(T1 columnValue, T2 filterValue, uint8_t cop, uint8_t rf, + const ColRequestHeaderDataType& typeHolder, bool isVal2Null = false) { -// cout << "comparing " << hex << columnValue << " to " << filterValue << endl; - if (COMPARE_NIL == cop) return false; + // cout << "comparing " << hex << columnValue << " to " << filterValue << endl; + if (COMPARE_NIL == cop) + return false; - return colCompareDispatcherT(columnValue, filterValue, - cop, rf, typeHolder, isVal2Null); + return colCompareDispatcherT(columnValue, filterValue, cop, rf, + typeHolder, isVal2Null); } /***************************************************************************** @@ -415,502 +339,440 @@ inline bool colCompare( *****************************************************************************/ // Bit pattern representing EMPTY value for given column type/width // TBD Use typeHandler -template::type* = nullptr> +template ::type* = nullptr> T getEmptyValue(uint8_t type) { - return datatypes::Decimal128Empty; + return datatypes::Decimal128Empty; } -template::type* = nullptr> +template ::type* = nullptr> T getEmptyValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return joblist::DOUBLEEMPTYROW; + switch (type) + { + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLEEMPTYROW; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - return joblist::CHAR8EMPTYROW; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: return joblist::CHAR8EMPTYROW; - case CalpontSystemCatalog::UBIGINT: - return joblist::UBIGINTEMPTYROW; + case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTEMPTYROW; - default: - return joblist::BIGINTEMPTYROW; - } + default: return joblist::BIGINTEMPTYROW; + } } -template::type* = nullptr> +template ::type* = nullptr> T getEmptyValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return joblist::FLOATEMPTYROW; + switch (type) + { + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return joblist::FLOATEMPTYROW; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return joblist::CHAR4EMPTYROW; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return joblist::CHAR4EMPTYROW; - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - return joblist::UINTEMPTYROW; + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: return joblist::UINTEMPTYROW; - default: - return joblist::INTEMPTYROW; - } + default: return joblist::INTEMPTYROW; + } } -template::type* = nullptr> +template ::type* = nullptr> T getEmptyValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return joblist::CHAR2EMPTYROW; + switch (type) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return joblist::CHAR2EMPTYROW; - case CalpontSystemCatalog::USMALLINT: - return joblist::USMALLINTEMPTYROW; + case CalpontSystemCatalog::USMALLINT: return joblist::USMALLINTEMPTYROW; - default: - return joblist::SMALLINTEMPTYROW; - } + default: return joblist::SMALLINTEMPTYROW; + } } -template::type* = nullptr> +template ::type* = nullptr> T getEmptyValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return joblist::CHAR1EMPTYROW; + switch (type) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return joblist::CHAR1EMPTYROW; - case CalpontSystemCatalog::UTINYINT: - return joblist::UTINYINTEMPTYROW; + case CalpontSystemCatalog::UTINYINT: return joblist::UTINYINTEMPTYROW; - default: - return joblist::TINYINTEMPTYROW; - } + default: return joblist::TINYINTEMPTYROW; + } } // Bit pattern representing NULL value for given column type/width // TBD Use TypeHandler -template::type* = nullptr> +template ::type* = nullptr> T getNullValue(uint8_t type) { - return datatypes::Decimal128Null; + return datatypes::Decimal128Null; } -template::type* = nullptr> +template ::type* = nullptr> T getNullValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return joblist::DOUBLENULL; + switch (type) + { + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - return joblist::CHAR8NULL; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: return joblist::CHAR8NULL; - case CalpontSystemCatalog::UBIGINT: - return joblist::UBIGINTNULL; + case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTNULL; - default: - return joblist::BIGINTNULL; - } + default: return joblist::BIGINTNULL; + } } -template::type* = nullptr> +template ::type* = nullptr> T getNullValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return joblist::FLOATNULL; + switch (type) + { + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return joblist::FLOATNULL; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - return joblist::CHAR4NULL; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: return joblist::CHAR4NULL; - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return joblist::DATENULL; + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return joblist::DATENULL; - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - return joblist::UINTNULL; + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: return joblist::UINTNULL; - default: - return joblist::INTNULL; - } + default: return joblist::INTNULL; + } } -template::type* = nullptr> +template ::type* = nullptr> T getNullValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return joblist::CHAR2NULL; + switch (type) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return joblist::CHAR2NULL; - case CalpontSystemCatalog::USMALLINT: - return joblist::USMALLINTNULL; + case CalpontSystemCatalog::USMALLINT: return joblist::USMALLINTNULL; - default: - return joblist::SMALLINTNULL; - } + default: return joblist::SMALLINTNULL; + } } -template::type* = nullptr> +template ::type* = nullptr> T getNullValue(uint8_t type) { - switch (type) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - return joblist::CHAR1NULL; + switch (type) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: return joblist::CHAR1NULL; - case CalpontSystemCatalog::UTINYINT: - return joblist::UTINYINTNULL; + case CalpontSystemCatalog::UTINYINT: return joblist::UTINYINTNULL; - default: - return joblist::TINYINTNULL; - } + default: return joblist::TINYINTNULL; + } } // Check whether val is NULL (or alternative NULL bit pattern for 64-bit string types) -template +template inline bool isNullValue(const T val, const T NULL_VALUE) { - return val == NULL_VALUE; + return val == NULL_VALUE; } -template<> +template <> inline bool isNullValue(const int64_t val, const int64_t NULL_VALUE) { - //@bug 339 might be a token here - //TODO: what's up with the alternative NULL here? - constexpr const int64_t ALT_NULL_VALUE = 0xFFFFFFFFFFFFFFFELL; + //@bug 339 might be a token here + // TODO: what's up with the alternative NULL here? + constexpr const int64_t ALT_NULL_VALUE = 0xFFFFFFFFFFFFFFFELL; - return (val == NULL_VALUE || - val == ALT_NULL_VALUE); + return (val == NULL_VALUE || val == ALT_NULL_VALUE); } // // FILTER A COLUMN VALUE // -template::type* = nullptr> -inline bool noneValuesInArray(const T curValue, - const FT* filterValues, - const uint32_t filterCount) +template ::type* = nullptr> +inline bool noneValuesInArray(const T curValue, const FT* filterValues, const uint32_t filterCount) { - // ignore NULLs in the array and in the column data - return false; + // ignore NULLs in the array and in the column data + return false; } -template::type* = nullptr> -inline bool noneValuesInArray(const T curValue, - const FT* filterValues, - const uint32_t filterCount) +template ::type* = nullptr> +inline bool noneValuesInArray(const T curValue, const FT* filterValues, const uint32_t filterCount) { - for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) - { - if (curValue == static_cast(filterValues[argIndex])) - return false; - } + for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) + { + if (curValue == static_cast(filterValues[argIndex])) + return false; + } - return true; + return true; } -template::type* = nullptr> +template ::type* = nullptr> inline bool noneValuesInSet(const T curValue, const ST* filterSet) { - // bug 1920: ignore NULLs in the set and in the column data - return false; + // bug 1920: ignore NULLs in the set and in the column data + return false; } -template::type* = nullptr> +template ::type* = nullptr> inline bool noneValuesInSet(const T curValue, const ST* filterSet) { - bool found = (filterSet->find(curValue) != filterSet->end()); - return !found; + bool found = (filterSet->find(curValue) != filterSet->end()); + return !found; } // The routine is used to test the value from a block against filters // according with columnFilterMode(see the corresponding enum for details). // Returns true if the curValue matches the filter. -template -inline bool matchingColValue(const T curValue, - const ColumnFilterMode columnFilterMode, - const ST* filterSet, // Set of values for simple filters (any of values / none of them) - const uint32_t filterCount, // Number of filter elements, each described by one entry in the following arrays: - const uint8_t* filterCOPs, // comparison operation - const FT* filterValues, // value to compare to - const uint8_t* filterRFs, // reverse byte order flags - const ColRequestHeaderDataType& typeHolder, - const T NULL_VALUE) // Bit pattern representing NULL value for this column type/width +template +inline bool matchingColValue( + const T curValue, const ColumnFilterMode columnFilterMode, + const ST* filterSet, // Set of values for simple filters (any of values / none of them) + const uint32_t + filterCount, // Number of filter elements, each described by one entry in the following arrays: + const uint8_t* filterCOPs, // comparison operation + const FT* filterValues, // value to compare to + const uint8_t* filterRFs, // reverse byte order flags + const ColRequestHeaderDataType& typeHolder, + const T NULL_VALUE) // Bit pattern representing NULL value for this column type/width { - /* In order to make filtering as fast as possible, we replaced the single generic algorithm - with several algorithms, better tailored for more specific cases: - empty filter, single comparison, and/or/xor comparison results, one/none of small/large set of values - */ - switch (columnFilterMode) + /* In order to make filtering as fast as possible, we replaced the single generic algorithm + with several algorithms, better tailored for more specific cases: + empty filter, single comparison, and/or/xor comparison results, one/none of small/large set of values + */ + switch (columnFilterMode) + { + // Empty filter is always true + case ALWAYS_TRUE: return true; + + // Filter consisting of exactly one comparison operation + case SINGLE_COMPARISON: { - // Empty filter is always true - case ALWAYS_TRUE: - return true; - - - // Filter consisting of exactly one comparison operation - case SINGLE_COMPARISON: - { - auto filterValue = filterValues[0]; - // This can be future optimized checking if a filterValue is NULL or not - bool cmp = colCompare(curValue, filterValue, filterCOPs[0], - filterRFs[0], typeHolder, isNullValue(filterValue, - NULL_VALUE)); - return cmp; - } - - - // Filter is true if ANY comparison is true (BOP_OR) - case ANY_COMPARISON_TRUE: - { - for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) - { - auto filterValue = filterValues[argIndex]; - // This can be future optimized checking if a filterValues are NULLs or not before the higher level loop. - bool cmp = colCompare(curValue, filterValue, filterCOPs[argIndex], - filterRFs[argIndex], typeHolder, isNullValue(filterValue, - NULL_VALUE)); - - // Short-circuit the filter evaluation - true || ... == true - if (cmp == true) - return true; - } - - // We can get here only if all filters returned false - return false; - } - - - // Filter is true only if ALL comparisons are true (BOP_AND) - case ALL_COMPARISONS_TRUE: - { - for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) - { - auto filterValue = filterValues[argIndex]; - // This can be future optimized checking if a filterValues are NULLs or not before the higher level loop. - bool cmp = colCompare(curValue, filterValue, filterCOPs[argIndex], - filterRFs[argIndex], typeHolder, - isNullValue(filterValue, NULL_VALUE)); - - // Short-circuit the filter evaluation - false && ... = false - if (cmp == false) - return false; - } - - // We can get here only if all filters returned true - return true; - } - - - // XORing results of comparisons (BOP_XOR) - case XOR_COMPARISONS: - { - bool result = false; - - for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) - { - auto filterValue = filterValues[argIndex]; - // This can be future optimized checking if a filterValues are NULLs or not before the higher level loop. - bool cmp = colCompare(curValue, filterValue, filterCOPs[argIndex], - filterRFs[argIndex], typeHolder, - isNullValue(filterValue, NULL_VALUE)); - result ^= cmp; - } - - return result; - } - - - // ONE of the values in the small set represented by an array (BOP_OR + all COMPARE_EQ) - case ONE_OF_VALUES_IN_ARRAY: - { - for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) - { - if (curValue == static_cast(filterValues[argIndex])) - return true; - } - - return false; - } - - - // NONE of the values in the small set represented by an array (BOP_AND + all COMPARE_NE) - case NONE_OF_VALUES_IN_ARRAY: - return noneValuesInArray(curValue, filterValues, filterCount); - - - // ONE of the values in the set is equal to the value checked (BOP_OR + all COMPARE_EQ) - case ONE_OF_VALUES_IN_SET: - { - bool found = (filterSet->find(curValue) != filterSet->end()); - return found; - } - - - // NONE of the values in the set is equal to the value checked (BOP_AND + all COMPARE_NE) - case NONE_OF_VALUES_IN_SET: - return noneValuesInSet(curValue, filterSet); - - - default: - idbassert(0); - return true; + auto filterValue = filterValues[0]; + // This can be future optimized checking if a filterValue is NULL or not + bool cmp = + colCompare(curValue, filterValue, filterCOPs[0], filterRFs[0], typeHolder, + isNullValue(filterValue, NULL_VALUE)); + return cmp; } + + // Filter is true if ANY comparison is true (BOP_OR) + case ANY_COMPARISON_TRUE: + { + for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) + { + auto filterValue = filterValues[argIndex]; + // This can be future optimized checking if a filterValues are NULLs or not before the higher level + // loop. + bool cmp = colCompare(curValue, filterValue, filterCOPs[argIndex], + filterRFs[argIndex], typeHolder, + isNullValue(filterValue, NULL_VALUE)); + + // Short-circuit the filter evaluation - true || ... == true + if (cmp == true) + return true; + } + + // We can get here only if all filters returned false + return false; + } + + // Filter is true only if ALL comparisons are true (BOP_AND) + case ALL_COMPARISONS_TRUE: + { + for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) + { + auto filterValue = filterValues[argIndex]; + // This can be future optimized checking if a filterValues are NULLs or not before the higher level + // loop. + bool cmp = colCompare(curValue, filterValue, filterCOPs[argIndex], + filterRFs[argIndex], typeHolder, + isNullValue(filterValue, NULL_VALUE)); + + // Short-circuit the filter evaluation - false && ... = false + if (cmp == false) + return false; + } + + // We can get here only if all filters returned true + return true; + } + + // XORing results of comparisons (BOP_XOR) + case XOR_COMPARISONS: + { + bool result = false; + + for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) + { + auto filterValue = filterValues[argIndex]; + // This can be future optimized checking if a filterValues are NULLs or not before the higher level + // loop. + bool cmp = colCompare(curValue, filterValue, filterCOPs[argIndex], + filterRFs[argIndex], typeHolder, + isNullValue(filterValue, NULL_VALUE)); + result ^= cmp; + } + + return result; + } + + // ONE of the values in the small set represented by an array (BOP_OR + all COMPARE_EQ) + case ONE_OF_VALUES_IN_ARRAY: + { + for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) + { + if (curValue == static_cast(filterValues[argIndex])) + return true; + } + + return false; + } + + // NONE of the values in the small set represented by an array (BOP_AND + all COMPARE_NE) + case NONE_OF_VALUES_IN_ARRAY: + return noneValuesInArray(curValue, filterValues, filterCount); + + // ONE of the values in the set is equal to the value checked (BOP_OR + all COMPARE_EQ) + case ONE_OF_VALUES_IN_SET: + { + bool found = (filterSet->find(curValue) != filterSet->end()); + return found; + } + + // NONE of the values in the set is equal to the value checked (BOP_AND + all COMPARE_NE) + case NONE_OF_VALUES_IN_SET: return noneValuesInSet(curValue, filterSet); + + default: idbassert(0); return true; + } } /***************************************************************************** *** MISC FUNCS ************************************************************** *****************************************************************************/ // These two are templates update min/max values in the loop iterating the values in filterColumnData. -template::type* = nullptr> +template ::type* = nullptr> inline void updateMinMax(T& Min, T& Max, const T curValue, NewColRequestHeader* in) { - constexpr int COL_WIDTH = sizeof(T); - if (colCompare(Min, curValue, COMPARE_GT, false, in->colType)) - Min = curValue; + constexpr int COL_WIDTH = sizeof(T); + if (colCompare(Min, curValue, COMPARE_GT, false, in->colType)) + Min = curValue; - if (colCompare(Max, curValue, COMPARE_LT, false, in->colType)) - Max = curValue; + if (colCompare(Max, curValue, COMPARE_LT, false, in->colType)) + Max = curValue; } -template::type* = nullptr> +template ::type* = nullptr> inline void updateMinMax(T& Min, T& Max, const T curValue, NewColRequestHeader* in) { - if (Min > curValue) - Min = curValue; + if (Min > curValue) + Min = curValue; - if (Max < curValue) - Max = curValue; + if (Max < curValue) + Max = curValue; } // The next templates group sets initial Min/Max values in filterColumnData. -template::type* = nullptr> +template ::type* = nullptr> T getInitialMin(NewColRequestHeader* in) { - const CHARSET_INFO &cs = in->colType.getCharset(); - T Min = 0; - cs.max_str((uchar*) &Min, sizeof(Min), sizeof(Min)); - return Min; + const CHARSET_INFO& cs = in->colType.getCharset(); + T Min = 0; + cs.max_str((uchar*)&Min, sizeof(Min), sizeof(Min)); + return Min; } -template::type* = nullptr> +template ::type* = nullptr> T getInitialMin(NewColRequestHeader* in) { - return datatypes::numeric_limits::max(); + return datatypes::numeric_limits::max(); } -template::type* = nullptr> +template ::type* = nullptr> T getInitialMax(NewColRequestHeader* in) { - return datatypes::numeric_limits::min(); + return datatypes::numeric_limits::min(); } -template::type* = nullptr> +template ::type* = nullptr> T getInitialMax(NewColRequestHeader* in) { - return 0; + return 0; } -template::type* = nullptr> +template ::type* = nullptr> T getInitialMax(NewColRequestHeader* in) { - const CHARSET_INFO &cs = in->colType.getCharset(); - T Max = 0; - cs.min_str((uchar*) &Max, sizeof(Max), sizeof(Max)); - return Max; + const CHARSET_INFO& cs = in->colType.getCharset(); + T Max = 0; + cs.min_str((uchar*)&Max, sizeof(Max), sizeof(Max)); + return Max; } /***************************************************************************** @@ -921,70 +783,71 @@ T getInitialMax(NewColRequestHeader* in) // Return true on success, false on End of Block. // Values are read from srcArray either in natural order or in the order defined by ridArray. // Empty values are skipped, unless ridArray==0 && !(OutputType & OT_RID). -template +template inline bool nextColValue( - T& result, // Place for the value returned - bool* isEmpty, // ... and flag whether it's EMPTY - uint32_t* index, // Successive index either in srcArray (going from 0 to srcSize-1) or ridArray (0..ridSize-1) - uint16_t* rid, // Index in srcArray of the value returned - const T* srcArray, // Input array - const uint32_t srcSize, // ... and its size - const uint16_t* ridArray, // Optional array of indexes into srcArray, that defines the read order - const uint16_t ridSize, // ... and its size - const uint8_t OutputType, // Used to decide whether to skip EMPTY values + T& result, // Place for the value returned + bool* isEmpty, // ... and flag whether it's EMPTY + uint32_t* + index, // Successive index either in srcArray (going from 0 to srcSize-1) or ridArray (0..ridSize-1) + uint16_t* rid, // Index in srcArray of the value returned + const T* srcArray, // Input array + const uint32_t srcSize, // ... and its size + const uint16_t* ridArray, // Optional array of indexes into srcArray, that defines the read order + const uint16_t ridSize, // ... and its size + const uint8_t OutputType, // Used to decide whether to skip EMPTY values T EMPTY_VALUE) { - auto i = *index; // local copy of *index to speed up loops - T value; // value to be written into *result, local for the same reason + auto i = *index; // local copy of *index to speed up loops + T value; // value to be written into *result, local for the same reason - if (ridArray) + if (ridArray) + { + // Read next non-empty value in the order defined by ridArray + for (;; i++) { - // Read next non-empty value in the order defined by ridArray - for( ; ; i++) - { - if (UNLIKELY(i >= ridSize)) - return false; + if (UNLIKELY(i >= ridSize)) + return false; - value = srcArray[ridArray[i]]; + value = srcArray[ridArray[i]]; - if (value != EMPTY_VALUE) - break; - } - - *rid = ridArray[i]; - *isEmpty = false; - } - else if (OutputType & OT_RID) //TODO: check correctness of this condition for SKIP_EMPTY_VALUES - { - // Read next non-empty value in the natural order - for( ; ; i++) - { - if (UNLIKELY(i >= srcSize)) - return false; - - value = srcArray[i]; - - if (value != EMPTY_VALUE) - break; - } - - *rid = i; - *isEmpty = false; - } - else - { - // Read next value in the natural order - if (UNLIKELY(i >= srcSize)) - return false; - - *rid = i; - value = srcArray[i]; - *isEmpty = (value == EMPTY_VALUE); + if (value != EMPTY_VALUE) + break; } - *index = i+1; - result = value; - return true; + *rid = ridArray[i]; + *isEmpty = false; + } + else if (OutputType & OT_RID) // TODO: check correctness of this condition for SKIP_EMPTY_VALUES + { + // Read next non-empty value in the natural order + for (;; i++) + { + if (UNLIKELY(i >= srcSize)) + return false; + + value = srcArray[i]; + + if (value != EMPTY_VALUE) + break; + } + + *rid = i; + *isEmpty = false; + } + else + { + // Read next value in the natural order + if (UNLIKELY(i >= srcSize)) + return false; + + *rid = i; + value = srcArray[i]; + *isEmpty = (value == EMPTY_VALUE); + } + + *index = i + 1; + result = value; + return true; } /// @@ -994,259 +857,262 @@ inline bool nextColValue( // Write the value index in srcArray and/or the value itself, depending on bits in OutputType, // into the output buffer and update the output pointer. // TODO Introduce another dispatching layer based on OutputType. -template -inline void writeColValue( - uint8_t OutputType, - ColResultHeader* out, - uint16_t rid, - const T* srcArray) +template +inline void writeColValue(uint8_t OutputType, ColResultHeader* out, uint16_t rid, const T* srcArray) { - // TODO move base ptr calculation one level up. - uint8_t* outPtr = reinterpret_cast(&out[1]); - auto idx = out->NVALS++; - if (OutputType & OT_RID) - { - auto* outPos = getRIDArrayPosition(outPtr, idx); - *outPos = rid; - out->RidFlags |= (1 << (rid >> 9)); // set the (row/512)'th bit - } + // TODO move base ptr calculation one level up. + uint8_t* outPtr = reinterpret_cast(&out[1]); + auto idx = out->NVALS++; + if (OutputType & OT_RID) + { + auto* outPos = getRIDArrayPosition(outPtr, idx); + *outPos = rid; + out->RidFlags |= (1 << (rid >> 9)); // set the (row/512)'th bit + } - if (OutputType & (OT_TOKEN | OT_DATAVALUE)) - { - // TODO move base ptr calculation one level up. - T* outPos = getValuesArrayPosition(primitives::getFirstValueArrayPosition(out), idx); - // TODO check bytecode for the 16 byte type - *outPos = srcArray[rid]; - } + if (OutputType & (OT_TOKEN | OT_DATAVALUE)) + { + // TODO move base ptr calculation one level up. + T* outPos = getValuesArrayPosition(primitives::getFirstValueArrayPosition(out), idx); + // TODO check bytecode for the 16 byte type + *outPos = srcArray[rid]; + } } -#if defined(__x86_64__ ) -template::type* = nullptr> -inline void vectUpdateMinMax(const bool validMinMax, const bool isNonNullOrEmpty, - T& Min, T& Max, T curValue, NewColRequestHeader* in) +#if defined(__x86_64__) +template ::type* = nullptr> +inline void vectUpdateMinMax(const bool validMinMax, const bool isNonNullOrEmpty, T& Min, T& Max, T curValue, + NewColRequestHeader* in) { - if (validMinMax && isNonNullOrEmpty) - updateMinMax(Min, Max, curValue, in); + if (validMinMax && isNonNullOrEmpty) + updateMinMax(Min, Max, curValue, in); } // MCS won't update Min/Max for a block if it doesn't read all values in a block. // This happens if in->NVALS > 0(HAS_INPUT_RIDS is set). -template::type* = nullptr> -inline void vectUpdateMinMax(const bool validMinMax, const bool isNonNullOrEmpty, - T& Min, T& Max, T curValue, NewColRequestHeader* in) +template ::type* = nullptr> +inline void vectUpdateMinMax(const bool validMinMax, const bool isNonNullOrEmpty, T& Min, T& Max, T curValue, + NewColRequestHeader* in) { - // + // } -template::type* = nullptr> +template ::type* = nullptr> void vectWriteColValuesLoopRIDAsignment(primitives::RIDType* ridDstArray, ColResultHeader* out, const primitives::RIDType calculatedRID, const primitives::RIDType* ridSrcArray, const uint32_t srcRIDIdx) { - *ridDstArray = calculatedRID; - out->RidFlags |= (1 << (calculatedRID >> 9)); // set the (row/512)'th bit + *ridDstArray = calculatedRID; + out->RidFlags |= (1 << (calculatedRID >> 9)); // set the (row/512)'th bit } -template::type* = nullptr> +template ::type* = nullptr> void vectWriteColValuesLoopRIDAsignment(primitives::RIDType* ridDstArray, ColResultHeader* out, const primitives::RIDType calculatedRID, const primitives::RIDType* ridSrcArray, const uint32_t srcRIDIdx) { - *ridDstArray = ridSrcArray[srcRIDIdx]; - out->RidFlags |= (1 << (ridSrcArray[srcRIDIdx] >> 9)); // set the (row/512)'th bit + *ridDstArray = ridSrcArray[srcRIDIdx]; + out->RidFlags |= (1 << (ridSrcArray[srcRIDIdx] >> 9)); // set the (row/512)'th bit } // The set of SFINAE templates are used to write values/RID into the output buffer based on // a number of template parameters // No RIDs only values -template::type* = nullptr> -inline uint16_t vectWriteColValues(VT& simdProcessor, // SIMD processor - const MT writeMask, // SIMD intrinsics bitmask for values to write - const MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values - const bool validMinMax, // The flag to update Min/Max for a block or not - const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr - T* dataVecTPtr, // Typed SIMD vector from the input block - char* dstArray, // the actual char dst array ptr to start writing values - T& Min, T&Max, // Min/Max of the extent - NewColRequestHeader* in, // Proto message - ColResultHeader* out, // Proto message - primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs - primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs +template ::type* = nullptr> +inline uint16_t vectWriteColValues( + VT& simdProcessor, // SIMD processor + const MT writeMask, // SIMD intrinsics bitmask for values to write + const MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values + const bool validMinMax, // The flag to update Min/Max for a block or not + const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr + T* dataVecTPtr, // Typed SIMD vector from the input block + char* dstArray, // the actual char dst array ptr to start writing values + T& Min, T& Max, // Min/Max of the extent + NewColRequestHeader* in, // Proto message + ColResultHeader* out, // Proto message + primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs + primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs { - constexpr const uint16_t FilterMaskStep = VT::FilterMaskStep; - using SimdType = typename VT::SimdType; - SimdType tmpStorageVector; - T* tmpDstVecTPtr = reinterpret_cast(&tmpStorageVector); - // Saving values based on writeMask into tmp vec. - // Min/Max processing. - // The mask is 16 bit long and it describes N elements. - // N = sizeof(vector type) / WIDTH. - uint32_t j = 0; - for (uint32_t it = 0; it < VT::vecByteSize; ++j, it += FilterMaskStep) + constexpr const uint16_t FilterMaskStep = VT::FilterMaskStep; + using SimdType = typename VT::SimdType; + SimdType tmpStorageVector; + T* tmpDstVecTPtr = reinterpret_cast(&tmpStorageVector); + // Saving values based on writeMask into tmp vec. + // Min/Max processing. + // The mask is 16 bit long and it describes N elements. + // N = sizeof(vector type) / WIDTH. + uint32_t j = 0; + for (uint32_t it = 0; it < VT::vecByteSize; ++j, it += FilterMaskStep) + { + MT bitMapPosition = 1 << it; + if (writeMask & bitMapPosition) { - MT bitMapPosition = 1 << it; - if (writeMask & bitMapPosition) - { - *tmpDstVecTPtr = dataVecTPtr[j]; - ++tmpDstVecTPtr; - } - - vectUpdateMinMax(validMinMax, nonNullOrEmptyMask & bitMapPosition, - Min, Max, dataVecTPtr[j], in); + *tmpDstVecTPtr = dataVecTPtr[j]; + ++tmpDstVecTPtr; } - // Store the whole vector however one level up the stack - // vectorizedFiltering() increases the dstArray by a number of - // actual values written that is the result of this function. - simdProcessor.store(dstArray, tmpStorageVector); - return tmpDstVecTPtr - reinterpret_cast(&tmpStorageVector); + vectUpdateMinMax(validMinMax, nonNullOrEmptyMask & bitMapPosition, Min, Max, + dataVecTPtr[j], in); + } + // Store the whole vector however one level up the stack + // vectorizedFiltering() increases the dstArray by a number of + // actual values written that is the result of this function. + simdProcessor.store(dstArray, tmpStorageVector); + + return tmpDstVecTPtr - reinterpret_cast(&tmpStorageVector); } // RIDs no values -template::type* = nullptr> -inline uint16_t vectWriteColValues(VT& simdProcessor, // SIMD processor - const MT writeMask, // SIMD intrinsics bitmask for values to write - const MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values - const bool validMinMax, // The flag to update Min/Max for a block or not - const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr - T* dataVecTPtr, // Typed SIMD vector from the input block - char* dstArray, // the actual char dst array ptr to start writing values - T& Min, T&Max, // Min/Max of the extent - NewColRequestHeader* in, // Proto message - ColResultHeader* out, // Proto message - primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs - primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs +template ::type* = nullptr> +inline uint16_t vectWriteColValues( + VT& simdProcessor, // SIMD processor + const MT writeMask, // SIMD intrinsics bitmask for values to write + const MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values + const bool validMinMax, // The flag to update Min/Max for a block or not + const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr + T* dataVecTPtr, // Typed SIMD vector from the input block + char* dstArray, // the actual char dst array ptr to start writing values + T& Min, T& Max, // Min/Max of the extent + NewColRequestHeader* in, // Proto message + ColResultHeader* out, // Proto message + primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs + primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs { - return 0; + return 0; } // Both RIDs and values -template::type* = nullptr> -inline uint16_t vectWriteColValues(VT& simdProcessor, // SIMD processor - const MT writeMask, // SIMD intrinsics bitmask for values to write - const MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values - const bool validMinMax, // The flag to update Min/Max for a block or not - const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr - T* dataVecTPtr, // Typed SIMD vector from the input block - char* dstArray, // the actual char dst array ptr to start writing values - T& Min, T&Max, // Min/Max of the extent - NewColRequestHeader* in, // Proto message - ColResultHeader* out, // Proto message - primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs - primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs +template ::type* = nullptr> +inline uint16_t vectWriteColValues( + VT& simdProcessor, // SIMD processor + const MT writeMask, // SIMD intrinsics bitmask for values to write + const MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values + const bool validMinMax, // The flag to update Min/Max for a block or not + const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr + T* dataVecTPtr, // Typed SIMD vector from the input block + char* dstArray, // the actual char dst array ptr to start writing values + T& Min, T& Max, // Min/Max of the extent + NewColRequestHeader* in, // Proto message + ColResultHeader* out, // Proto message + primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs + primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs { - constexpr const uint16_t FilterMaskStep = VT::FilterMaskStep; - using SimdType = typename VT::SimdType; - SimdType tmpStorageVector; - T* tmpDstVecTPtr = reinterpret_cast(&tmpStorageVector); - // Saving values based on writeMask into tmp vec. - // Min/Max processing. - // The mask is 16 bit long and it describes N elements. - // N = sizeof(vector type) / WIDTH. - uint32_t j = 0; - for (uint32_t it = 0; it < VT::vecByteSize; ++j, it += FilterMaskStep) + constexpr const uint16_t FilterMaskStep = VT::FilterMaskStep; + using SimdType = typename VT::SimdType; + SimdType tmpStorageVector; + T* tmpDstVecTPtr = reinterpret_cast(&tmpStorageVector); + // Saving values based on writeMask into tmp vec. + // Min/Max processing. + // The mask is 16 bit long and it describes N elements. + // N = sizeof(vector type) / WIDTH. + uint32_t j = 0; + for (uint32_t it = 0; it < VT::vecByteSize; ++j, it += FilterMaskStep) + { + MT bitMapPosition = 1 << it; + if (writeMask & bitMapPosition) { - MT bitMapPosition = 1 << it; - if (writeMask & bitMapPosition) - { - *tmpDstVecTPtr = dataVecTPtr[j]; - ++tmpDstVecTPtr; - vectWriteColValuesLoopRIDAsignment(ridDstArray, out, ridOffset + j, - ridSrcArray, j); - ++ridDstArray; - } - vectUpdateMinMax(validMinMax, nonNullOrEmptyMask & bitMapPosition, - Min, Max, dataVecTPtr[j], in); + *tmpDstVecTPtr = dataVecTPtr[j]; + ++tmpDstVecTPtr; + vectWriteColValuesLoopRIDAsignment(ridDstArray, out, ridOffset + j, ridSrcArray, j); + ++ridDstArray; } - // Store the whole vector however one level up the stack - // vectorizedFiltering() increases the dstArray by a number of - // actual values written that is the result of this function. - simdProcessor.store(dstArray, tmpStorageVector); + vectUpdateMinMax(validMinMax, nonNullOrEmptyMask & bitMapPosition, Min, Max, + dataVecTPtr[j], in); + } + // Store the whole vector however one level up the stack + // vectorizedFiltering() increases the dstArray by a number of + // actual values written that is the result of this function. + simdProcessor.store(dstArray, tmpStorageVector); - return tmpDstVecTPtr - reinterpret_cast(&tmpStorageVector); + return tmpDstVecTPtr - reinterpret_cast(&tmpStorageVector); } // RIDs no values -template::type* = nullptr> -inline uint16_t vectWriteRIDValues(VT& processor, // SIMD processor - const uint16_t valuesWritten, // The number of values written to in certain SFINAE cases - const bool validMinMax, // The flag to update Min/Max for a block or not - const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr - T* dataVecTPtr, // Typed SIMD vector from the input block - primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs - MT writeMask, // SIMD intrinsics bitmask for values to write - T& Min, T&Max, // Min/Max of the extent - NewColRequestHeader* in, // Proto message - ColResultHeader* out, // Proto message - MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values - primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs +template ::type* = nullptr> +inline uint16_t vectWriteRIDValues( + VT& processor, // SIMD processor + const uint16_t valuesWritten, // The number of values written to in certain SFINAE cases + const bool validMinMax, // The flag to update Min/Max for a block or not + const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr + T* dataVecTPtr, // Typed SIMD vector from the input block + primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs + MT writeMask, // SIMD intrinsics bitmask for values to write + T& Min, T& Max, // Min/Max of the extent + NewColRequestHeader* in, // Proto message + ColResultHeader* out, // Proto message + MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values + primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs { - constexpr const uint16_t FilterMaskStep = VT::FilterMaskStep; - primitives::RIDType* origRIDDstArray = ridDstArray; - // Saving values based on writeMask into tmp vec. - // Min/Max processing. - // The mask is 16 bit long and it describes N elements where N = sizeof(vector type) / WIDTH. - uint16_t j = 0; - for (uint32_t it = 0; it < VT::vecByteSize; ++j, it += FilterMaskStep) + constexpr const uint16_t FilterMaskStep = VT::FilterMaskStep; + primitives::RIDType* origRIDDstArray = ridDstArray; + // Saving values based on writeMask into tmp vec. + // Min/Max processing. + // The mask is 16 bit long and it describes N elements where N = sizeof(vector type) / WIDTH. + uint16_t j = 0; + for (uint32_t it = 0; it < VT::vecByteSize; ++j, it += FilterMaskStep) + { + MT bitMapPosition = 1 << it; + if (writeMask & (1 << it)) { - MT bitMapPosition = 1 << it; - if (writeMask & (1 << it)) - { - vectWriteColValuesLoopRIDAsignment(ridDstArray, out, ridOffset + j, - ridSrcArray, j); - ++ridDstArray; - } - vectUpdateMinMax(validMinMax, nonNullOrEmptyMask & bitMapPosition, - Min, Max, dataVecTPtr[j], in); + vectWriteColValuesLoopRIDAsignment(ridDstArray, out, ridOffset + j, ridSrcArray, j); + ++ridDstArray; } - return ridDstArray - origRIDDstArray; + vectUpdateMinMax(validMinMax, nonNullOrEmptyMask & bitMapPosition, Min, Max, + dataVecTPtr[j], in); + } + return ridDstArray - origRIDDstArray; } // Both RIDs and values // vectWriteColValues writes RIDs traversing the writeMask. -template::type* = nullptr> -inline uint16_t vectWriteRIDValues(VT& processor, // SIMD processor - const uint16_t valuesWritten, // The number of values written to in certain SFINAE cases - const bool validMinMax, // The flag to update Min/Max for a block or not - const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr - T* dataVecTPtr, // Typed SIMD vector from the input block - primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs - MT writeMask, // SIMD intrinsics bitmask for values to write - T& Min, T&Max, // Min/Max of the extent - NewColRequestHeader* in, // Proto message - ColResultHeader* out, // Proto message - MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values - primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs +template ::type* = nullptr> +inline uint16_t vectWriteRIDValues( + VT& processor, // SIMD processor + const uint16_t valuesWritten, // The number of values written to in certain SFINAE cases + const bool validMinMax, // The flag to update Min/Max for a block or not + const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr + T* dataVecTPtr, // Typed SIMD vector from the input block + primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs + MT writeMask, // SIMD intrinsics bitmask for values to write + T& Min, T& Max, // Min/Max of the extent + NewColRequestHeader* in, // Proto message + ColResultHeader* out, // Proto message + MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values + primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs { - return valuesWritten; + return valuesWritten; } // No RIDs only values -template::type* = nullptr> -inline uint16_t vectWriteRIDValues(VT& processor, // SIMD processor - const uint16_t valuesWritten, // The number of values written to in certain SFINAE cases - const bool validMinMax, // The flag to update Min/Max for a block or not - const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr - T* dataVecTPtr, // Typed SIMD vector from the input block - primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs - MT writeMask, // SIMD intrinsics bitmask for values to write - T& Min, T&Max, // Min/Max of the extent - NewColRequestHeader* in, // Proto message - ColResultHeader* out, // Proto message - MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values - primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs +template ::type* = nullptr> +inline uint16_t vectWriteRIDValues( + VT& processor, // SIMD processor + const uint16_t valuesWritten, // The number of values written to in certain SFINAE cases + const bool validMinMax, // The flag to update Min/Max for a block or not + const primitives::RIDType ridOffset, // The first RID value of the dataVecTPtr + T* dataVecTPtr, // Typed SIMD vector from the input block + primitives::RIDType* ridDstArray, // The actual dst arrray ptr to start writing RIDs + MT writeMask, // SIMD intrinsics bitmask for values to write + T& Min, T& Max, // Min/Max of the extent + NewColRequestHeader* in, // Proto message + ColResultHeader* out, // Proto message + MT nonNullOrEmptyMask, // SIMD intrinsics inverce bitmask for NULL/EMPTY values + primitives::RIDType* ridSrcArray) // The actual src array ptr to read RIDs { - return valuesWritten; + return valuesWritten; } #endif @@ -1255,17 +1121,18 @@ inline uint16_t vectWriteRIDValues(VT& processor, // SIMD processor *****************************************************************************/ // TODO turn columnFilterMode into template param to use it in matchingColValue // This routine filters values in a columnar block processing one scalar at a time. -template -void scalarFiltering(NewColRequestHeader* in, ColResultHeader* out, - const ColumnFilterMode columnFilterMode, - const ST* filterSet, // Set of values for simple filters (any of values / none of them) - const uint32_t filterCount, // Number of filter elements, each described by one entry in the following arrays: +template +void scalarFiltering( + NewColRequestHeader* in, ColResultHeader* out, const ColumnFilterMode columnFilterMode, + const ST* filterSet, // Set of values for simple filters (any of values / none of them) + const uint32_t + filterCount, // Number of filter elements, each described by one entry in the following arrays: const uint8_t* filterCOPs, // comparison operation const FT* filterValues, // value to compare to const uint8_t* filterRFs, - const ColRequestHeaderDataType& typeHolder, // TypeHolder to use collation-aware ops for char/text. - const T* srcArray, // Input array - const uint32_t srcSize, // ... and its size + const ColRequestHeaderDataType& typeHolder, // TypeHolder to use collation-aware ops for char/text. + const T* srcArray, // Input array + const uint32_t srcSize, // ... and its size const uint16_t* ridArray, // Optional array of indexes into srcArray, that defines the read order const uint16_t ridSize, // ... and its size const uint32_t initialRID, // The input block idx to start scanning/filter at. @@ -1273,82 +1140,77 @@ void scalarFiltering(NewColRequestHeader* in, ColResultHeader* out, const bool validMinMax, // The flag to store min/max T emptyValue, // Deduced empty value magic T nullValue, // Deduced null value magic - T Min, - T Max, - const bool isNullValueMatches) + T Min, T Max, const bool isNullValueMatches) { - constexpr int WIDTH = sizeof(T); - // Loop-local variables - T curValue = 0; - primitives::RIDType rid = 0; - bool isEmpty = false; + constexpr int WIDTH = sizeof(T); + // Loop-local variables + T curValue = 0; + primitives::RIDType rid = 0; + bool isEmpty = false; - // Loop over the column values, storing those matching the filter, and updating the min..max range - for (uint32_t i = initialRID; - nextColValue(curValue, &isEmpty, - &i, &rid, - srcArray, srcSize, ridArray, ridSize, - outputType, emptyValue); ) + // Loop over the column values, storing those matching the filter, and updating the min..max range + for (uint32_t i = initialRID; nextColValue(curValue, &isEmpty, &i, &rid, srcArray, srcSize, + ridArray, ridSize, outputType, emptyValue);) + { + if (isEmpty) + continue; + else if (isNullValue(curValue, nullValue)) { - if (isEmpty) - continue; - else if (isNullValue(curValue, nullValue)) - { - // If NULL values match the filter, write curValue to the output buffer - if (isNullValueMatches) - writeColValue(outputType, out, rid, srcArray); - } - else - { - // If curValue matches the filter, write it to the output buffer - if (matchingColValue(curValue, columnFilterMode, filterSet, filterCount, - filterCOPs, filterValues, filterRFs, in->colType, nullValue)) - { - writeColValue(outputType, out, rid, srcArray); - } - - // Update Min and Max if necessary. EMPTY/NULL values are processed in other branches. - if (validMinMax) - updateMinMax(Min, Max, curValue, in); - } + // If NULL values match the filter, write curValue to the output buffer + if (isNullValueMatches) + writeColValue(outputType, out, rid, srcArray); } - - // Write captured Min/Max values to *out - out->ValidMinMax = validMinMax; - if (validMinMax) + else { - out->Min = Min; - out->Max = Max; + // If curValue matches the filter, write it to the output buffer + if (matchingColValue(curValue, columnFilterMode, filterSet, filterCount, filterCOPs, + filterValues, filterRFs, in->colType, nullValue)) + { + writeColValue(outputType, out, rid, srcArray); + } + + // Update Min and Max if necessary. EMPTY/NULL values are processed in other branches. + if (validMinMax) + updateMinMax(Min, Max, curValue, in); } + } + + // Write captured Min/Max values to *out + out->ValidMinMax = validMinMax; + if (validMinMax) + { + out->Min = Min; + out->Max = Max; + } } -#if defined(__x86_64__ ) +#if defined(__x86_64__) template ::type* = nullptr> -inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, - const T* origSrcArray, const primitives::RIDType* ridArray, const uint16_t iter) +inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, const T* origSrcArray, + const primitives::RIDType* ridArray, const uint16_t iter) { - return {processor.loadFrom(reinterpret_cast(srcArray))}; + return {processor.loadFrom(reinterpret_cast(srcArray))}; } // Scatter-gather implementation // TODO Move the logic into simd namespace class methods and use intrinsics template ::type* = nullptr> -inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, - const T* origSrcArray, const primitives::RIDType* ridArray, const uint16_t iter) +inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, const T* origSrcArray, + const primitives::RIDType* ridArray, const uint16_t iter) { - constexpr const uint16_t WIDTH = sizeof(T); - constexpr const uint16_t VECTOR_SIZE = VT::vecByteSize / WIDTH; - using SimdType = typename VT::SimdType; - SimdType result; - T* resultTypedPtr = reinterpret_cast(&result); - for (uint32_t i = 0; i < VECTOR_SIZE; ++i) - { - resultTypedPtr[i] = origSrcArray[ridArray[i]]; - } + constexpr const uint16_t WIDTH = sizeof(T); + constexpr const uint16_t VECTOR_SIZE = VT::vecByteSize / WIDTH; + using SimdType = typename VT::SimdType; + SimdType result; + T* resultTypedPtr = reinterpret_cast(&result); + for (uint32_t i = 0; i < VECTOR_SIZE; ++i) + { + resultTypedPtr[i] = origSrcArray[ridArray[i]]; + } - return {result}; + return {result}; } // This routine filters input block in a vectorized manner. @@ -1360,297 +1222,259 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, // to glue the masks produced by actual filters. // Then it takes a vector of data, run filters and logical function using pointers. // See the corresponding dispatcher to get more details on vector processing class. -template -void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, - const T* srcArray, const uint32_t srcSize, primitives::RIDType* ridArray, - const uint16_t ridSize, ParsedColumnFilter* parsedColumnFilter, - const bool validMinMax, const T emptyValue, const T nullValue, - T Min, T Max, const bool isNullValueMatches) +template +void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* srcArray, + const uint32_t srcSize, primitives::RIDType* ridArray, const uint16_t ridSize, + ParsedColumnFilter* parsedColumnFilter, const bool validMinMax, const T emptyValue, + const T nullValue, T Min, T Max, const bool isNullValueMatches) { - constexpr const uint16_t WIDTH = sizeof(T); - using SimdType = typename VT::SimdType; - using SimdWrapperType = typename VT::SimdWrapperType; - using FilterType = typename VT::FilterType; - VT simdProcessor; - SimdType dataVec; - SimdType emptyFilterArgVec = simdProcessor.emptyNullLoadValue(emptyValue); - SimdType nullFilterArgVec = simdProcessor.emptyNullLoadValue(nullValue); - MT writeMask, nonEmptyMask, nonNullMask, nonNullOrEmptyMask; - MT initFilterMask = 0xFFFF; - primitives::RIDType rid = 0; - primitives::RIDType* origRidArray = ridArray; - uint16_t totalValuesWritten = 0; - char* dstArray = reinterpret_cast(primitives::getFirstValueArrayPosition(out)); - primitives::RIDType* ridDstArray = reinterpret_cast(getFirstRIDArrayPosition(out)); - const T* origSrcArray = srcArray; - const FT* filterValues = nullptr; - const ParsedColumnFilter::CopsType* filterCOPs = nullptr; - ColumnFilterMode columnFilterMode = ALWAYS_TRUE; - const ST* filterSet = nullptr; - const ParsedColumnFilter::RFsType* filterRFs = nullptr; - uint8_t outputType = in->OutputType; - constexpr uint16_t VECTOR_SIZE = VT::vecByteSize / WIDTH; - // If there are RIDs use its number to get a number of vectorized iterations. - uint16_t iterNumber = HAS_INPUT_RIDS ? ridSize / VECTOR_SIZE : srcSize / VECTOR_SIZE; - uint32_t filterCount = 0; - // These pragmas are to silence GCC warnings - // warning: ignoring attributes on template argument + constexpr const uint16_t WIDTH = sizeof(T); + using SimdType = typename VT::SimdType; + using SimdWrapperType = typename VT::SimdWrapperType; + using FilterType = typename VT::FilterType; + VT simdProcessor; + SimdType dataVec; + SimdType emptyFilterArgVec = simdProcessor.emptyNullLoadValue(emptyValue); + SimdType nullFilterArgVec = simdProcessor.emptyNullLoadValue(nullValue); + MT writeMask, nonEmptyMask, nonNullMask, nonNullOrEmptyMask; + MT initFilterMask = 0xFFFF; + primitives::RIDType rid = 0; + primitives::RIDType* origRidArray = ridArray; + uint16_t totalValuesWritten = 0; + char* dstArray = reinterpret_cast(primitives::getFirstValueArrayPosition(out)); + primitives::RIDType* ridDstArray = reinterpret_cast(getFirstRIDArrayPosition(out)); + const T* origSrcArray = srcArray; + const FT* filterValues = nullptr; + const ParsedColumnFilter::CopsType* filterCOPs = nullptr; + ColumnFilterMode columnFilterMode = ALWAYS_TRUE; + const ST* filterSet = nullptr; + const ParsedColumnFilter::RFsType* filterRFs = nullptr; + uint8_t outputType = in->OutputType; + constexpr uint16_t VECTOR_SIZE = VT::vecByteSize / WIDTH; + // If there are RIDs use its number to get a number of vectorized iterations. + uint16_t iterNumber = HAS_INPUT_RIDS ? ridSize / VECTOR_SIZE : srcSize / VECTOR_SIZE; + uint32_t filterCount = 0; + // These pragmas are to silence GCC warnings + // warning: ignoring attributes on template argument #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" - std::vector filterArgsVectors; - auto ptrA = std::mem_fn(&VT::cmpEq); - using COPType = decltype(ptrA); - std::vector copFunctorVec; + std::vector filterArgsVectors; + auto ptrA = std::mem_fn(&VT::cmpEq); + using COPType = decltype(ptrA); + std::vector copFunctorVec; #pragma GCC diagnostic pop - using BOPType = std::function; - BOPType bopFunctor; - // filter comparators and logical function compilation. - if (parsedColumnFilter != nullptr) + using BOPType = std::function; + BOPType bopFunctor; + // filter comparators and logical function compilation. + if (parsedColumnFilter != nullptr) + { + filterValues = parsedColumnFilter->getFilterVals(); + filterCOPs = parsedColumnFilter->prestored_cops.get(); + columnFilterMode = parsedColumnFilter->columnFilterMode; + filterSet = parsedColumnFilter->getFilterSet(); + filterRFs = parsedColumnFilter->prestored_rfs.get(); + filterCount = parsedColumnFilter->getFilterCount(); + if (iterNumber > 0) { - filterValues = parsedColumnFilter->getFilterVals(); - filterCOPs = parsedColumnFilter->prestored_cops.get(); - columnFilterMode = parsedColumnFilter->columnFilterMode; - filterSet = parsedColumnFilter->getFilterSet(); - filterRFs = parsedColumnFilter->prestored_rfs.get(); - filterCount = parsedColumnFilter->getFilterCount(); - if (iterNumber > 0) + copFunctorVec.reserve(filterCount); + switch (parsedColumnFilter->getBOP()) + { + case BOP_OR: + bopFunctor = std::bit_or(); + initFilterMask = 0; + break; + case BOP_AND: bopFunctor = std::bit_and(); break; + case BOP_XOR: + bopFunctor = std::bit_or(); + initFilterMask = 0; + break; + case BOP_NONE: + // According with the comments in linux-port/primitiveprocessor.h + // there can't be BOP_NONE with filterCount > 0 + bopFunctor = std::bit_and(); + break; + default: idbassert(false); + } + filterArgsVectors.reserve(filterCount); + for (uint32_t j = 0; j < filterCount; ++j) + { + // Preload filter argument values only once. + filterArgsVectors[j] = simdProcessor.loadValue(*((FilterType*)&filterValues[j])); + switch (filterCOPs[j]) { - copFunctorVec.reserve(filterCount); - switch(parsedColumnFilter->getBOP()) - { - case BOP_OR: - bopFunctor = std::bit_or(); - initFilterMask = 0; - break; - case BOP_AND: - bopFunctor = std::bit_and(); - break; - case BOP_XOR: - bopFunctor = std::bit_or(); - initFilterMask = 0; - break; - case BOP_NONE: - // According with the comments in linux-port/primitiveprocessor.h - // there can't be BOP_NONE with filterCount > 0 - bopFunctor = std::bit_and(); - break; - default: - idbassert(false); - } - filterArgsVectors.reserve(filterCount); - for (uint32_t j = 0; j < filterCount; ++j) - { - // Preload filter argument values only once. - filterArgsVectors[j] = simdProcessor.loadValue(*((FilterType*)&filterValues[j])); - switch(filterCOPs[j]) - { - case(COMPARE_EQ): - // Skipping extra filter pass generated by IS NULL - if (memcmp(&filterValues[j], &nullValue, sizeof(nullValue)) == 0) - copFunctorVec.push_back(std::mem_fn(&VT::nullEmptyCmpEq)); - else - copFunctorVec.push_back(std::mem_fn(&VT::cmpEq)); - break; - case(COMPARE_GE): - copFunctorVec.push_back(std::mem_fn(&VT::cmpGe)); - break; + case (COMPARE_EQ): + // Skipping extra filter pass generated by IS NULL + if (memcmp(&filterValues[j], &nullValue, sizeof(nullValue)) == 0) + copFunctorVec.push_back(std::mem_fn(&VT::nullEmptyCmpEq)); + else + copFunctorVec.push_back(std::mem_fn(&VT::cmpEq)); + break; + case (COMPARE_GE): copFunctorVec.push_back(std::mem_fn(&VT::cmpGe)); break; - case(COMPARE_GT): - copFunctorVec.push_back(std::mem_fn(&VT::cmpGt)); - break; - case(COMPARE_LE): - copFunctorVec.push_back(std::mem_fn(&VT::cmpLe)); - break; - case(COMPARE_LT): - copFunctorVec.push_back(std::mem_fn(&VT::cmpLt)); - break; - case(COMPARE_NE): - copFunctorVec.push_back(std::mem_fn(&VT::cmpNe)); - break; - case(COMPARE_NIL): - copFunctorVec.push_back(std::mem_fn(&VT::cmpAlwaysFalse)); - break; - // There are couple other COP, e.g. COMPARE_NOT however they can't be met here - // b/c MCS 6.x uses COMPARE_NOT for strings with OP_LIKE only. See op2num() for - // details. + case (COMPARE_GT): copFunctorVec.push_back(std::mem_fn(&VT::cmpGt)); break; + case (COMPARE_LE): copFunctorVec.push_back(std::mem_fn(&VT::cmpLe)); break; + case (COMPARE_LT): copFunctorVec.push_back(std::mem_fn(&VT::cmpLt)); break; + case (COMPARE_NE): copFunctorVec.push_back(std::mem_fn(&VT::cmpNe)); break; + case (COMPARE_NIL): + copFunctorVec.push_back(std::mem_fn(&VT::cmpAlwaysFalse)); + break; + // There are couple other COP, e.g. COMPARE_NOT however they can't be met here + // b/c MCS 6.x uses COMPARE_NOT for strings with OP_LIKE only. See op2num() for + // details. - default: - idbassert(false); - } - } + default: idbassert(false); } + } } + } - // main loop - // writeMask tells which values must get into the result. Includes values that matches filters. Can have NULLs. - // nonEmptyMask tells which vector coords are not EMPTY magics. - // nonNullMask tells which vector coords are not NULL magics. - for (uint16_t i = 0; i < iterNumber; ++i) + // main loop + // writeMask tells which values must get into the result. Includes values that matches filters. Can have + // NULLs. nonEmptyMask tells which vector coords are not EMPTY magics. nonNullMask tells which vector coords + // are not NULL magics. + for (uint16_t i = 0; i < iterNumber; ++i) + { + primitives::RIDType ridOffset = i * VECTOR_SIZE; + assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset)); + dataVec = simdDataLoadTemplate(simdProcessor, srcArray, + origSrcArray, ridArray, i) + .v; + nonEmptyMask = simdProcessor.nullEmptyCmpNe(dataVec, emptyFilterArgVec); + writeMask = nonEmptyMask; + // NULL check + nonNullMask = simdProcessor.nullEmptyCmpNe(dataVec, nullFilterArgVec); + // Exclude NULLs from the resulting set if NULL doesn't match the filters. + writeMask = isNullValueMatches ? writeMask : writeMask & nonNullMask; + nonNullOrEmptyMask = nonNullMask & nonEmptyMask; + // filters + MT prevFilterMask = initFilterMask; + // TODO name this mask literal + MT filterMask = 0xFFFF; + for (uint32_t j = 0; j < filterCount; ++j) { - primitives::RIDType ridOffset = i * VECTOR_SIZE; - assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset)); - dataVec = simdDataLoadTemplate(simdProcessor, srcArray, origSrcArray, ridArray, i).v; - nonEmptyMask = simdProcessor.nullEmptyCmpNe(dataVec, emptyFilterArgVec); - writeMask = nonEmptyMask; - // NULL check - nonNullMask = simdProcessor.nullEmptyCmpNe(dataVec, nullFilterArgVec); - // Exclude NULLs from the resulting set if NULL doesn't match the filters. - writeMask = isNullValueMatches ? writeMask : writeMask & nonNullMask; - nonNullOrEmptyMask = nonNullMask & nonEmptyMask; - // filters - MT prevFilterMask = initFilterMask; - // TODO name this mask literal - MT filterMask = 0xFFFF; - for (uint32_t j = 0; j < filterCount; ++j) - { - // filter using compiled filter and preloaded filter argument - filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]); - filterMask = bopFunctor(prevFilterMask, filterMask); - prevFilterMask = filterMask; - } - writeMask = writeMask & filterMask; - - T* dataVecTPtr = reinterpret_cast(&dataVec); - - // vectWriteColValues iterates over the values in the source vec - // to store values/RIDs into dstArray/ridDstArray. - // It also sets Min/Max values for the block if eligible. - // !!! vectWriteColValues increases ridDstArray internally but it doesn't go - // outside the scope of the memory allocated to out msg. - // vectWriteColValues is empty if outputMode == OT_RID. - uint16_t valuesWritten = - vectWriteColValues(simdProcessor, - writeMask, - nonNullOrEmptyMask, - validMinMax, - ridOffset, - dataVecTPtr, - dstArray, - Min, Max, - in, out, ridDstArray, - ridArray); - // Some outputType modes saves RIDs also. vectWriteRIDValues is empty for - // OT_DATAVALUE, OT_BOTH(vectWriteColValues takes care about RIDs). - valuesWritten = - vectWriteRIDValues(simdProcessor, - valuesWritten, - validMinMax, - ridOffset, - dataVecTPtr, - ridDstArray, - writeMask, - Min, Max, - in, out, - nonNullOrEmptyMask, - ridArray); - - // Calculate bytes written - uint16_t bytesWritten = valuesWritten * WIDTH; - totalValuesWritten += valuesWritten; - ridDstArray += valuesWritten; - dstArray += bytesWritten; - rid += VECTOR_SIZE; - srcArray += VECTOR_SIZE; - ridArray += VECTOR_SIZE; + // filter using compiled filter and preloaded filter argument + filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]); + filterMask = bopFunctor(prevFilterMask, filterMask); + prevFilterMask = filterMask; } + writeMask = writeMask & filterMask; - // Set the number of output values here b/c tail processing can skip this operation. - out->NVALS = totalValuesWritten; - // WIP Remove this block - // Write captured Min/Max values to *out - out->ValidMinMax = validMinMax; - if (validMinMax) - { - out->Min = Min; - out->Max = Max; - } + T* dataVecTPtr = reinterpret_cast(&dataVec); - // process the tail. scalarFiltering changes out contents, e.g. Min/Max, NVALS, RIDs and values array - // This tail also sets out::Min/Max, out::validMinMax if validMinMax is set. - uint32_t processedSoFar = rid; - scalarFiltering(in, out, columnFilterMode, filterSet, filterCount, filterCOPs, - filterValues, filterRFs, in->colType, origSrcArray, srcSize, origRidArray, - ridSize, processedSoFar, outputType, validMinMax, emptyValue, nullValue, - Min, Max, isNullValueMatches); + // vectWriteColValues iterates over the values in the source vec + // to store values/RIDs into dstArray/ridDstArray. + // It also sets Min/Max values for the block if eligible. + // !!! vectWriteColValues increases ridDstArray internally but it doesn't go + // outside the scope of the memory allocated to out msg. + // vectWriteColValues is empty if outputMode == OT_RID. + uint16_t valuesWritten = vectWriteColValues( + simdProcessor, writeMask, nonNullOrEmptyMask, validMinMax, ridOffset, dataVecTPtr, dstArray, Min, Max, + in, out, ridDstArray, ridArray); + // Some outputType modes saves RIDs also. vectWriteRIDValues is empty for + // OT_DATAVALUE, OT_BOTH(vectWriteColValues takes care about RIDs). + valuesWritten = vectWriteRIDValues( + simdProcessor, valuesWritten, validMinMax, ridOffset, dataVecTPtr, ridDstArray, writeMask, Min, Max, + in, out, nonNullOrEmptyMask, ridArray); + + // Calculate bytes written + uint16_t bytesWritten = valuesWritten * WIDTH; + totalValuesWritten += valuesWritten; + ridDstArray += valuesWritten; + dstArray += bytesWritten; + rid += VECTOR_SIZE; + srcArray += VECTOR_SIZE; + ridArray += VECTOR_SIZE; + } + + // Set the number of output values here b/c tail processing can skip this operation. + out->NVALS = totalValuesWritten; + // WIP Remove this block + // Write captured Min/Max values to *out + out->ValidMinMax = validMinMax; + if (validMinMax) + { + out->Min = Min; + out->Max = Max; + } + + // process the tail. scalarFiltering changes out contents, e.g. Min/Max, NVALS, RIDs and values array + // This tail also sets out::Min/Max, out::validMinMax if validMinMax is set. + uint32_t processedSoFar = rid; + scalarFiltering(in, out, columnFilterMode, filterSet, filterCount, filterCOPs, + filterValues, filterRFs, in->colType, origSrcArray, srcSize, origRidArray, + ridSize, processedSoFar, outputType, validMinMax, emptyValue, nullValue, + Min, Max, isNullValueMatches); } // This routine dispatches template function calls to reduce branching. -template +template void vectorizedFilteringDispatcher(NewColRequestHeader* in, ColResultHeader* out, - const STORAGE_TYPE* srcArray, const uint32_t srcSize, uint16_t* ridArray, - const uint16_t ridSize, ParsedColumnFilter* parsedColumnFilter, - const bool validMinMax, const STORAGE_TYPE emptyValue, const STORAGE_TYPE nullValue, - STORAGE_TYPE Min, STORAGE_TYPE Max, const bool isNullValueMatches) + const STORAGE_TYPE* srcArray, const uint32_t srcSize, uint16_t* ridArray, + const uint16_t ridSize, ParsedColumnFilter* parsedColumnFilter, + const bool validMinMax, const STORAGE_TYPE emptyValue, + const STORAGE_TYPE nullValue, STORAGE_TYPE Min, STORAGE_TYPE Max, + const bool isNullValueMatches) { - // Using struct to dispatch SIMD type based on integral type T. - using SimdType = typename simd::IntegralToSIMD::type; - using FilterType = typename simd::StorageToFiltering::type; - using VT = typename simd::SimdFilterProcessor; - bool hasInputRIDs = (in->NVALS > 0) ? true : false; - if (hasInputRIDs) + // Using struct to dispatch SIMD type based on integral type T. + using SimdType = typename simd::IntegralToSIMD::type; + using FilterType = typename simd::StorageToFiltering::type; + using VT = typename simd::SimdFilterProcessor; + bool hasInputRIDs = (in->NVALS > 0) ? true : false; + if (hasInputRIDs) + { + const bool hasInput = true; + switch (in->OutputType) { - const bool hasInput = true; - switch (in->OutputType) - { - case OT_RID: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - case OT_BOTH: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - case OT_TOKEN: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - case OT_DATAVALUE: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - } + case OT_RID: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; + case OT_BOTH: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; + case OT_TOKEN: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; + case OT_DATAVALUE: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; } - else + } + else + { + const bool hasInput = false; + switch (in->OutputType) { - const bool hasInput = false; - switch (in->OutputType) - { - case OT_RID: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - case OT_BOTH: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - case OT_TOKEN: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - case OT_DATAVALUE: - vectorizedFiltering(in, out, - srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter, - validMinMax, emptyValue, nullValue, Min, Max, isNullValueMatches); - break; - - } + case OT_RID: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; + case OT_BOTH: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; + case OT_TOKEN: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; + case OT_DATAVALUE: + vectorizedFiltering( + in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + break; } + } } #endif @@ -1660,264 +1484,250 @@ void vectorizedFilteringDispatcher(NewColRequestHeader* in, ColResultHeader* out // Copy data matching parsedColumnFilter from input to output. // Input is srcArray[srcSize], optionally accessed in the order defined by ridArray[ridSize]. // Output is buf: ColResponseHeader, RIDType[BLOCK_SIZE], T[BLOCK_SIZE]. -template -void filterColumnData( - NewColRequestHeader* in, - ColResultHeader* out, - uint16_t* ridArray, - const uint16_t ridSize, // Number of values in ridArray - int* srcArray16, - const uint32_t srcSize, - boost::shared_ptr parsedColumnFilter) +template +void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* ridArray, + const uint16_t ridSize, // Number of values in ridArray + int* srcArray16, const uint32_t srcSize, + boost::shared_ptr parsedColumnFilter) { - using FT = typename IntegralTypeToFilterType::type; - using ST = typename IntegralTypeToFilterSetType::type; - constexpr int WIDTH = sizeof(T); - const T* srcArray = reinterpret_cast(srcArray16); + using FT = typename IntegralTypeToFilterType::type; + using ST = typename IntegralTypeToFilterSetType::type; + constexpr int WIDTH = sizeof(T); + const T* srcArray = reinterpret_cast(srcArray16); - // Cache some structure fields in local vars - auto dataType = (CalpontSystemCatalog::ColDataType) in->colType.DataType; // Column datatype - uint32_t filterCount = in->NOPS; // Number of elements in the filter - uint8_t outputType = in->OutputType; + // Cache some structure fields in local vars + auto dataType = (CalpontSystemCatalog::ColDataType)in->colType.DataType; // Column datatype + uint32_t filterCount = in->NOPS; // Number of elements in the filter + uint8_t outputType = in->OutputType; - // If no pre-parsed column filter is set, parse the filter in the message - if (parsedColumnFilter.get() == nullptr && filterCount > 0) - parsedColumnFilter = _parseColumnFilter(in->getFilterStringPtr(), - dataType, filterCount, in->BOP); + // If no pre-parsed column filter is set, parse the filter in the message + if (parsedColumnFilter.get() == nullptr && filterCount > 0) + parsedColumnFilter = _parseColumnFilter(in->getFilterStringPtr(), dataType, filterCount, in->BOP); - // Cache parsedColumnFilter fields in local vars - auto columnFilterMode = filterCount==0 ? ALWAYS_TRUE : parsedColumnFilter->columnFilterMode; - FT* filterValues = filterCount==0 ? nullptr : parsedColumnFilter->getFilterVals(); - auto filterCOPs = filterCount==0 ? nullptr : parsedColumnFilter->prestored_cops.get(); - auto filterRFs = filterCount==0 ? nullptr : parsedColumnFilter->prestored_rfs.get(); - ST* filterSet = filterCount==0 ? nullptr : parsedColumnFilter->getFilterSet(); + // Cache parsedColumnFilter fields in local vars + auto columnFilterMode = filterCount == 0 ? ALWAYS_TRUE : parsedColumnFilter->columnFilterMode; + FT* filterValues = filterCount == 0 ? nullptr : parsedColumnFilter->getFilterVals(); + auto filterCOPs = filterCount == 0 ? nullptr : parsedColumnFilter->prestored_cops.get(); + auto filterRFs = filterCount == 0 ? nullptr : parsedColumnFilter->prestored_rfs.get(); + ST* filterSet = filterCount == 0 ? nullptr : parsedColumnFilter->getFilterSet(); - // Bit patterns in srcArray[i] representing EMPTY and NULL values - T emptyValue = getEmptyValue(dataType); - T nullValue = getNullValue(dataType); + // Bit patterns in srcArray[i] representing EMPTY and NULL values + T emptyValue = getEmptyValue(dataType); + T nullValue = getNullValue(dataType); - // Precompute filter results for NULL values - bool isNullValueMatches = matchingColValue(nullValue, columnFilterMode, - filterSet, filterCount, filterCOPs, filterValues, filterRFs, in->colType, nullValue); + // Precompute filter results for NULL values + bool isNullValueMatches = + matchingColValue(nullValue, columnFilterMode, filterSet, filterCount, filterCOPs, + filterValues, filterRFs, in->colType, nullValue); - // ########################### - // Boolean indicating whether to capture the min and max values - bool validMinMax = isMinMaxValid(in); - T Min = getInitialMin(in); - T Max = getInitialMax(in); + // ########################### + // Boolean indicating whether to capture the min and max values + bool validMinMax = isMinMaxValid(in); + T Min = getInitialMin(in); + T Max = getInitialMax(in); - // Vectorized scanning/filtering for all numerics except float/double types. - // If the total number of input values can't fill a vector the vector path - // applies scalar filtering. - // Syscat queries mustn't follow vectorized processing path b/c PP must return - // all values w/o any filter(even empty values filter) applied. + // Vectorized scanning/filtering for all numerics except float/double types. + // If the total number of input values can't fill a vector the vector path + // applies scalar filtering. + // Syscat queries mustn't follow vectorized processing path b/c PP must return + // all values w/o any filter(even empty values filter) applied. -#if defined(__x86_64__ ) - // Don't use vectorized filtering for text based data types. - if (KIND <= KIND_FLOAT && WIDTH < 16) +#if defined(__x86_64__) + // Don't use vectorized filtering for text based data types. + if (KIND <= KIND_FLOAT && WIDTH < 16) + { + bool canUseFastFiltering = true; + for (uint32_t i = 0; i < filterCount; ++i) + if (filterRFs[i] != 0) + canUseFastFiltering = false; + + if (canUseFastFiltering) { - bool canUseFastFiltering = true; - for (uint32_t i = 0; i < filterCount; ++i) - if (filterRFs[i] != 0) - canUseFastFiltering = false; - - if (canUseFastFiltering) - { - vectorizedFilteringDispatcher(in, out, srcArray, srcSize, ridArray, ridSize, - parsedColumnFilter.get(), validMinMax, emptyValue, - nullValue, Min, Max, isNullValueMatches); - return; - } + vectorizedFilteringDispatcher(in, out, srcArray, srcSize, ridArray, ridSize, + parsedColumnFilter.get(), validMinMax, emptyValue, + nullValue, Min, Max, isNullValueMatches); + return; } + } #endif - uint32_t initialRID = 0; - scalarFiltering(in, out, columnFilterMode, filterSet, filterCount, filterCOPs, - filterValues, filterRFs, in->colType, srcArray, srcSize, ridArray, - ridSize, initialRID, outputType, validMinMax, emptyValue, nullValue, - Min, Max, isNullValueMatches); -} // end of filterColumnData + uint32_t initialRID = 0; + scalarFiltering(in, out, columnFilterMode, filterSet, filterCount, filterCOPs, + filterValues, filterRFs, in->colType, srcArray, srcSize, ridArray, ridSize, + initialRID, outputType, validMinMax, emptyValue, nullValue, Min, Max, + isNullValueMatches); +} // end of filterColumnData -} //namespace anon +} // namespace namespace primitives { - // The routine used to dispatch CHAR|VARCHAR|TEXT|BLOB scan. inline bool isDictTokenScan(NewColRequestHeader* in) { - switch (in->colType.DataType) - { - case CalpontSystemCatalog::CHAR: - return (in->colType.DataSize > 8); + switch (in->colType.DataType) + { + case CalpontSystemCatalog::CHAR: return (in->colType.DataSize > 8); - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - return (in->colType.DataSize > 7); - default: - return false; - } + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: return (in->colType.DataSize > 7); + default: return false; + } } // A set of dispatchers for different column widths/integral types. -template= 5 - typename std::enable_if::type* = nullptr> // gcc >= 5 - #else - typename std::enable_if::type*> // gcc 4.8.5 - #endif +#if ___GNUC__ >= 5 + typename std::enable_if::type* = nullptr> // gcc >= 5 #else - typename std::enable_if::type* = nullptr> + typename std::enable_if::type*> // gcc 4.8.5 #endif -void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, - ColResultHeader* out) -{ - constexpr int W = sizeof(T); - auto dataType = (execplan::CalpontSystemCatalog::ColDataType) in->colType.DataType; - if (dataType == execplan::CalpontSystemCatalog::FLOAT) - { - const uint16_t ridSize = in->NVALS; - uint16_t* ridArray = in->getRIDArrayPtr(W); - const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE - : BLOCK_SIZE / W; - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); - return; - } - _scanAndFilterTypeDispatcher(in, out); -} - -template= 5 - typename std::enable_if::type* = nullptr> // gcc >= 5 - #else - typename std::enable_if::type*> // gcc 4.8.5 - #endif #else - typename std::enable_if::type* = nullptr> + typename std::enable_if::type* = nullptr> #endif -void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, - ColResultHeader* out) +void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out) { - constexpr int W = sizeof(T); - auto dataType = (execplan::CalpontSystemCatalog::ColDataType) in->colType.DataType; - if (dataType == execplan::CalpontSystemCatalog::DOUBLE) - { - const uint16_t ridSize = in->NVALS; - uint16_t* ridArray = in->getRIDArrayPtr(W); - const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE - : BLOCK_SIZE / W; - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); - return; - } - _scanAndFilterTypeDispatcher(in, out); -} - -template= 5 - sizeof(T) == sizeof(int128_t), T>::type* = nullptr> // gcc >= 5 - #else - sizeof(T) == sizeof(int128_t), T>::type*> // gcc 4.8.5 - #endif -#else - sizeof(T) == sizeof(int128_t), T>::type* = nullptr> -#endif -void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, - ColResultHeader* out) -{ - _scanAndFilterTypeDispatcher(in, out); -} - -template= 5 - typename std::enable_if::type* = nullptr> // gcc >= 5 - #else - typename std::enable_if::type*> // gcc 4.8.5 - #endif -#else - typename std::enable_if::type* = nullptr> -#endif -void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, - ColResultHeader* out) -{ - constexpr int W = sizeof(T); + constexpr int W = sizeof(T); + auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType; + if (dataType == execplan::CalpontSystemCatalog::FLOAT) + { const uint16_t ridSize = in->NVALS; uint16_t* ridArray = in->getRIDArrayPtr(W); - const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE - : BLOCK_SIZE / W; - - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + return; + } + _scanAndFilterTypeDispatcher(in, out); } -template= 5 - typename std::enable_if::type* = nullptr> // gcc >= 5 - #else - typename std::enable_if::type*> // gcc 4.8.5 - #endif +#if ___GNUC__ >= 5 + typename std::enable_if::type* = nullptr> // gcc >= 5 #else - typename std::enable_if::type* = nullptr> + typename std::enable_if::type*> // gcc 4.8.5 #endif -void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, - ColResultHeader* out) +#else + typename std::enable_if::type* = nullptr> +#endif +void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out) { - constexpr int W = sizeof(T); + constexpr int W = sizeof(T); + auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType; + if (dataType == execplan::CalpontSystemCatalog::DOUBLE) + { const uint16_t ridSize = in->NVALS; uint16_t* ridArray = in->getRIDArrayPtr(W); - const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE - : BLOCK_SIZE / W; + const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + return; + } + _scanAndFilterTypeDispatcher(in, out); +} - auto dataType = (execplan::CalpontSystemCatalog::ColDataType) in->colType.DataType; - if ((dataType == execplan::CalpontSystemCatalog::CHAR || - dataType == execplan::CalpontSystemCatalog::VARCHAR || - dataType == execplan::CalpontSystemCatalog::TEXT) && - !isDictTokenScan(in)) - { - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); - return; - } +template = 5 + sizeof(T) == sizeof(int128_t), + T>::type* = nullptr> // gcc >= 5 +#else + sizeof(T) == sizeof(int128_t), + T>::type*> // gcc 4.8.5 +#endif +#else + sizeof(T) == sizeof(int128_t), + T>::type* = nullptr> +#endif +void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out) +{ + _scanAndFilterTypeDispatcher(in, out); +} - if (datatypes::isUnsigned(dataType)) - { - using UT = typename std::conditional::value || datatypes::is_uint128_t::value, T, typename datatypes::make_unsigned::type>::type; - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); - return; - } - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); +template = 5 + typename std::enable_if::type* = nullptr> // gcc >= 5 +#else + typename std::enable_if::type*> // gcc 4.8.5 +#endif +#else + typename std::enable_if::type* = nullptr> +#endif +void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out) +{ + constexpr int W = sizeof(T); + const uint16_t ridSize = in->NVALS; + uint16_t* ridArray = in->getRIDArrayPtr(W); + const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; + + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); +} + +template = 5 + typename std::enable_if::type* = nullptr> // gcc >= 5 +#else + typename std::enable_if::type*> // gcc 4.8.5 +#endif +#else + typename std::enable_if::type* = nullptr> +#endif +void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out) +{ + constexpr int W = sizeof(T); + const uint16_t ridSize = in->NVALS; + uint16_t* ridArray = in->getRIDArrayPtr(W); + const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; + + auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType; + if ((dataType == execplan::CalpontSystemCatalog::CHAR || + dataType == execplan::CalpontSystemCatalog::VARCHAR || + dataType == execplan::CalpontSystemCatalog::TEXT) && + !isDictTokenScan(in)) + { + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + return; + } + + if (datatypes::isUnsigned(dataType)) + { + using UT = typename std::conditional::value || datatypes::is_uint128_t::value, T, + typename datatypes::make_unsigned::type>::type; + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + return; + } + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); } // The entrypoint for block scanning and filtering. // The block is in in msg, out msg is used to store values|RIDs matched. -template +template void PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out) { #ifdef PRIM_DEBUG - auto markEvent = [&] (char eventChar) - { - if (fStatsPtr) - fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, eventChar); - }; + auto markEvent = [&](char eventChar) + { + if (fStatsPtr) + fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, eventChar); + }; #endif - constexpr int W = sizeof(T); + constexpr int W = sizeof(T); - void *outp = static_cast(out); - memcpy(outp, in, sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - out->NVALS = 0; - out->LBID = in->LBID; - out->ism.Command = COL_RESULTS; - out->OutputType = in->OutputType; - out->RidFlags = 0; - //...Initialize I/O counts; - out->CacheIO = 0; - out->PhysicalIO = 0; + void* outp = static_cast(out); + memcpy(outp, in, sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + out->NVALS = 0; + out->LBID = in->LBID; + out->ism.Command = COL_RESULTS; + out->OutputType = in->OutputType; + out->RidFlags = 0; + //...Initialize I/O counts; + out->CacheIO = 0; + out->PhysicalIO = 0; #if 0 // short-circuit the actual block scan for testing @@ -1931,27 +1741,27 @@ void PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader* in, ColResultH #endif #ifdef PRIM_DEBUG - markEvent('B'); + markEvent('B'); #endif - // Sort ridArray (the row index array) if there are RIDs with this in msg - in->sortRIDArrayIfNeeded(W); - scanAndFilterTypeDispatcher(in, out); + // Sort ridArray (the row index array) if there are RIDs with this in msg + in->sortRIDArrayIfNeeded(W); + scanAndFilterTypeDispatcher(in, out); #ifdef PRIM_DEBUG - markEvent('C'); + markEvent('C'); #endif } -template -void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, ColResultHeader*); -template -void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, ColResultHeader*); -template -void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, ColResultHeader*); -template -void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, ColResultHeader*); -template -void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, ColResultHeader*); +template void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, + ColResultHeader*); +template void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, + ColResultHeader*); +template void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, + ColResultHeader*); +template void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, + ColResultHeader*); +template void primitives::PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader*, + ColResultHeader*); -} // namespace primitives +} // namespace primitives // vim:ts=4 sw=4: diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 53fbef648..2942d4fda 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -33,7 +33,6 @@ using namespace std; #include "dataconvert.h" #include - using namespace logging; const char* nullString = " "; // this is not NULL to preempt segfaults. @@ -46,601 +45,579 @@ const char* signatureNotFound = joblist::CPSTRNOTFOUND.c_str(); namespace primitives { - -inline bool PrimitiveProcessor::compare(const datatypes::Charset &cs, uint8_t COP, - const char *str1, size_t length1, - const char *str2, size_t length2) throw() +inline bool PrimitiveProcessor::compare(const datatypes::Charset& cs, uint8_t COP, const char* str1, + size_t length1, const char* str2, size_t length2) throw() { - int error = 0; - bool rc = primitives::StringComparator(cs).op(&error, COP, - ConstString(str1, length1), - ConstString(str2, length2)); - if (error) - { - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(34); + int error = 0; + bool rc = primitives::StringComparator(cs).op(&error, COP, ConstString(str1, length1), + ConstString(str2, length2)); + if (error) + { + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(34); - colWidth.add(COP); - colWidth.add("compare"); - msg.format(colWidth); - logger.logErrorMessage(msg); - return false; // throw an exception here? - } - return rc; + colWidth.add(COP); + colWidth.add("compare"); + msg.format(colWidth); + logger.logErrorMessage(msg); + return false; // throw an exception here? + } + return rc; } /* Notes: - - assumes no continuation pointer + - assumes no continuation pointer */ -void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader* h, - TokenByScanResultHeader* ret, unsigned outSize, - boost::shared_ptr eqFilter) +void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader* h, TokenByScanResultHeader* ret, + unsigned outSize, boost::shared_ptr eqFilter) { - const DataValue* args; - const uint8_t* niceBlock; // block cast to a byte-indexed type - const uint8_t* niceInput; // h cast to a byte-indexed type - const uint16_t* offsets; - int offsetIndex, argIndex, argsOffset; - bool cmpResult = false; - int i; - const char* sig; - uint16_t siglen; + const DataValue* args; + const uint8_t* niceBlock; // block cast to a byte-indexed type + const uint8_t* niceInput; // h cast to a byte-indexed type + const uint16_t* offsets; + int offsetIndex, argIndex, argsOffset; + bool cmpResult = false; + int i; + const char* sig; + uint16_t siglen; - PrimToken* retTokens; - DataValue* retDataValues; - int rdvOffset; - uint8_t* niceRet; // ret cast to a byte-indexed type + PrimToken* retTokens; + DataValue* retDataValues; + int rdvOffset; + uint8_t* niceRet; // ret cast to a byte-indexed type - // set up pointers to fields within each structure + // set up pointers to fields within each structure - // either retTokens or retDataValues will be used but not both. - niceRet = reinterpret_cast(ret); - rdvOffset = sizeof(TokenByScanResultHeader); + // either retTokens or retDataValues will be used but not both. + niceRet = reinterpret_cast(ret); + rdvOffset = sizeof(TokenByScanResultHeader); - retTokens = reinterpret_cast(&niceRet[rdvOffset]); - retDataValues = reinterpret_cast(&niceRet[rdvOffset]); + retTokens = reinterpret_cast(&niceRet[rdvOffset]); + retDataValues = reinterpret_cast(&niceRet[rdvOffset]); + { + void* retp = static_cast(ret); + memcpy(retp, h, sizeof(PrimitiveHeader) + sizeof(ISMPacketHeader)); + } + ret->NVALS = 0; + ret->NBYTES = sizeof(TokenByScanResultHeader); + ret->ism.Command = DICT_SCAN_COMPARE_RESULTS; + + //...Initialize I/O counts + ret->CacheIO = 0; + ret->PhysicalIO = 0; + + niceBlock = reinterpret_cast(block); + offsets = reinterpret_cast(&niceBlock[10]); + niceInput = reinterpret_cast(h); + + const datatypes::Charset cs(h->charsetNumber); + + for (offsetIndex = 1; offsets[offsetIndex] != 0xffff; offsetIndex++) + { + siglen = offsets[offsetIndex - 1] - offsets[offsetIndex]; + sig = reinterpret_cast(&niceBlock[offsets[offsetIndex]]); + argsOffset = sizeof(TokenByScanRequestHeader); + argIndex = 0; + args = reinterpret_cast(&niceInput[argsOffset]); + + if (eqFilter) { - void *retp = static_cast(ret); - memcpy(retp, h, sizeof(PrimitiveHeader) + sizeof(ISMPacketHeader)); + if (&cs.getCharset() != &eqFilter->getCharset()) + { + // throw runtime_error("Collations mismatch: TokenByScanRequestHeader and DicEqualityFilter"); + } + string strData(sig, siglen); + bool gotIt = eqFilter->find(strData) != eqFilter->end(); + + if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE && !gotIt)) + goto store; + + goto no_store; } - ret->NVALS = 0; - ret->NBYTES = sizeof(TokenByScanResultHeader); - ret->ism.Command = DICT_SCAN_COMPARE_RESULTS; - //...Initialize I/O counts - ret->CacheIO = 0; - ret->PhysicalIO = 0; + cmpResult = compare(cs, h->COP1, sig, siglen, args->data, args->len); - niceBlock = reinterpret_cast(block); - offsets = reinterpret_cast(&niceBlock[10]); - niceInput = reinterpret_cast(h); - - const datatypes::Charset cs(h->charsetNumber); - - for (offsetIndex = 1; offsets[offsetIndex] != 0xffff; offsetIndex++) + switch (h->NVALS) { + case 1: + { + if (cmpResult) + goto store; - siglen = offsets[offsetIndex - 1] - offsets[offsetIndex]; - sig = reinterpret_cast(&niceBlock[offsets[offsetIndex]]); - argsOffset = sizeof(TokenByScanRequestHeader); - argIndex = 0; - args = reinterpret_cast(&niceInput[argsOffset]); + goto no_store; + } - if (eqFilter) + case 2: + { + if (!cmpResult && h->BOP == BOP_AND) + goto no_store; + + if (cmpResult && h->BOP == BOP_OR) + goto store; + + argsOffset += sizeof(uint16_t) + args->len; + argIndex++; + args = (DataValue*)&niceInput[argsOffset]; + + cmpResult = compare(cs, h->COP2, sig, siglen, args->data, args->len); + + if (cmpResult) + goto store; + + goto no_store; + } + + default: + { + idbassert(0); + for (i = 0, cmpResult = true; i < h->NVALS; i++) { - if (& cs.getCharset() != & eqFilter->getCharset()) - { - //throw runtime_error("Collations mismatch: TokenByScanRequestHeader and DicEqualityFilter"); - } - string strData(sig, siglen); - bool gotIt = eqFilter->find(strData) != eqFilter->end(); - - if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE && - !gotIt)) - goto store; + cmpResult = compare(cs, h->COP1, sig, siglen, args->data, args->len); + if (!cmpResult && h->BOP == BOP_AND) goto no_store; + + if (cmpResult && h->BOP == BOP_OR) + goto store; + + argsOffset += sizeof(uint16_t) + args->len; + argIndex++; + args = (DataValue*)&niceInput[argsOffset]; } - cmpResult = compare(cs, h->COP1, sig, siglen, args->data, args->len); - - switch (h->NVALS) - { - case 1: - { - - if (cmpResult) - goto store; - - goto no_store; - } - - case 2: - { - if (!cmpResult && h->BOP == BOP_AND) - goto no_store; - - if (cmpResult && h->BOP == BOP_OR) - goto store; - - argsOffset += sizeof(uint16_t) + args->len; - argIndex++; - args = (DataValue*) &niceInput[argsOffset]; - - cmpResult = compare(cs, h->COP2, sig, siglen, args->data, args->len); - - if (cmpResult) - goto store; - - goto no_store; - } - - default: - { - idbassert(0); - for (i = 0, cmpResult = true; i < h->NVALS; i++) - { - cmpResult = compare(cs, h->COP1, sig, siglen, args->data, args->len); - - if (!cmpResult && h->BOP == BOP_AND) - goto no_store; - - if (cmpResult && h->BOP == BOP_OR) - goto store; - - argsOffset += sizeof(uint16_t) + args->len; - argIndex++; - args = (DataValue*) &niceInput[argsOffset]; - } - - if (i == h->NVALS && cmpResult) - goto store; - else - goto no_store; - } - } - -store: - - if (h->OutputType == OT_DATAVALUE) - { - if ((ret->NBYTES + sizeof(DataValue) + siglen) > outSize) - { - MessageLog logger(LoggingID(28)); - logging::Message::Args marker; - Message msg(35); - - marker.add(8); - msg.format(marker); - logger.logErrorMessage(msg); - - throw logging::DictionaryBufferOverflow(); - } - - retDataValues->len = siglen; - memcpy(retDataValues->data, sig, siglen); - rdvOffset += sizeof(DataValue) + siglen; - retDataValues = (DataValue*) &niceRet[rdvOffset]; - ret->NVALS++; - ret->NBYTES += sizeof(DataValue) + siglen; - } - else if (h->OutputType == OT_TOKEN) - { - if ((ret->NBYTES + sizeof(PrimToken)) > outSize) - { - MessageLog logger(LoggingID(28)); - logging::Message::Args marker; - Message msg(35); - - marker.add(9); - msg.format(marker); - logger.logErrorMessage(msg); - - throw logging::DictionaryBufferOverflow(); - } - - retTokens[ret->NVALS].LBID = h->LBID; - retTokens[ret->NVALS].offset = offsetIndex; // need index rather than the block offset... rp 12/19/06 - retTokens[ret->NVALS].len = args->len; - ret->NVALS++; - ret->NBYTES += sizeof(PrimToken); - } - /* - * XXXPAT: HACK! Ron requested a special case where the input string - * that matched and the token of the matched string were returned. - * It will not be used in cases where there are multiple input strings. - * We need to rethink the requirements for this primitive after Dec 15. - */ - else if (h->OutputType == OT_BOTH) - { - if (ret->NBYTES + sizeof(PrimToken) + sizeof(DataValue) + args->len > outSize) - { - MessageLog logger(LoggingID(28)); - logging::Message::Args marker; - Message msg(35); - - marker.add(10); - msg.format(marker); - logger.logErrorMessage(msg); - - throw logging::DictionaryBufferOverflow(); - } - - retDataValues->len = args->len; - memcpy(retDataValues->data, args->data, args->len); - rdvOffset += sizeof(DataValue) + args->len; - retTokens = reinterpret_cast(&niceRet[rdvOffset]); - retTokens->LBID = h->LBID; - retTokens->offset = offsetIndex; // need index rather than the block offset... rp 12/19/06 - retTokens->len = args->len; - rdvOffset += sizeof(PrimToken); - retDataValues = reinterpret_cast(&niceRet[rdvOffset]); - ret->NBYTES += sizeof(PrimToken) + sizeof(DataValue) + args->len; - ret->NVALS++; - } - -no_store: - ; //this is intentional + if (i == h->NVALS && cmpResult) + goto store; + else + goto no_store; + } } - return; + store: + + if (h->OutputType == OT_DATAVALUE) + { + if ((ret->NBYTES + sizeof(DataValue) + siglen) > outSize) + { + MessageLog logger(LoggingID(28)); + logging::Message::Args marker; + Message msg(35); + + marker.add(8); + msg.format(marker); + logger.logErrorMessage(msg); + + throw logging::DictionaryBufferOverflow(); + } + + retDataValues->len = siglen; + memcpy(retDataValues->data, sig, siglen); + rdvOffset += sizeof(DataValue) + siglen; + retDataValues = (DataValue*)&niceRet[rdvOffset]; + ret->NVALS++; + ret->NBYTES += sizeof(DataValue) + siglen; + } + else if (h->OutputType == OT_TOKEN) + { + if ((ret->NBYTES + sizeof(PrimToken)) > outSize) + { + MessageLog logger(LoggingID(28)); + logging::Message::Args marker; + Message msg(35); + + marker.add(9); + msg.format(marker); + logger.logErrorMessage(msg); + + throw logging::DictionaryBufferOverflow(); + } + + retTokens[ret->NVALS].LBID = h->LBID; + retTokens[ret->NVALS].offset = offsetIndex; // need index rather than the block offset... rp 12/19/06 + retTokens[ret->NVALS].len = args->len; + ret->NVALS++; + ret->NBYTES += sizeof(PrimToken); + } + /* + * XXXPAT: HACK! Ron requested a special case where the input string + * that matched and the token of the matched string were returned. + * It will not be used in cases where there are multiple input strings. + * We need to rethink the requirements for this primitive after Dec 15. + */ + else if (h->OutputType == OT_BOTH) + { + if (ret->NBYTES + sizeof(PrimToken) + sizeof(DataValue) + args->len > outSize) + { + MessageLog logger(LoggingID(28)); + logging::Message::Args marker; + Message msg(35); + + marker.add(10); + msg.format(marker); + logger.logErrorMessage(msg); + + throw logging::DictionaryBufferOverflow(); + } + + retDataValues->len = args->len; + memcpy(retDataValues->data, args->data, args->len); + rdvOffset += sizeof(DataValue) + args->len; + retTokens = reinterpret_cast(&niceRet[rdvOffset]); + retTokens->LBID = h->LBID; + retTokens->offset = offsetIndex; // need index rather than the block offset... rp 12/19/06 + retTokens->len = args->len; + rdvOffset += sizeof(PrimToken); + retDataValues = reinterpret_cast(&niceRet[rdvOffset]); + ret->NBYTES += sizeof(PrimToken) + sizeof(DataValue) + args->len; + ret->NVALS++; + } + + no_store:; // this is intentional + } + + return; } -void PrimitiveProcessor::nextSig(int NVALS, - const PrimToken* tokens, - p_DataValue* ret, - uint8_t outputFlags, +void PrimitiveProcessor::nextSig(int NVALS, const PrimToken* tokens, p_DataValue* ret, uint8_t outputFlags, bool oldGetSigBehavior, bool skipNulls) throw() { - const uint8_t* niceBlock = reinterpret_cast(block); - const uint16_t* offsets - = reinterpret_cast(&niceBlock[10]); + const uint8_t* niceBlock = reinterpret_cast(block); + const uint16_t* offsets = reinterpret_cast(&niceBlock[10]); - if (NVALS == 0) + if (NVALS == 0) + { + if (offsets[dict_OffsetIndex + 1] == 0xffff) { - if (offsets[dict_OffsetIndex + 1] == 0xffff) - { - ret->len = -1; - return; - } - - ret->len = offsets[dict_OffsetIndex] - offsets[dict_OffsetIndex + 1]; - ret->data = &niceBlock[offsets[dict_OffsetIndex + 1]]; - - if (outputFlags & OT_TOKEN) - currentOffsetIndex = dict_OffsetIndex + 1; - } - else - { - -again: - - if (dict_OffsetIndex >= NVALS) - { - ret->len = -1; - return; - } - - if (oldGetSigBehavior) - { - - const OldGetSigParams* oldParams = - reinterpret_cast(tokens); - - if (oldParams[dict_OffsetIndex].rid & 0x8000000000000000LL) - { - if (skipNulls) - { - /* Bug 3321. For some cases the NULL token should be skipped. The - * isnull filter is handled by token columncommand or by the F & E - * framework. This primitive should only process nulls - * when it's for projection. - */ - dict_OffsetIndex++; - goto again; - } - - ret->len = nullStringLen; - ret->data = (const uint8_t*) nullString; - } - else - { - ret->len = offsets[oldParams[dict_OffsetIndex].offsetIndex - 1] - - offsets[oldParams[dict_OffsetIndex].offsetIndex]; - //Whoa! apparently we have come across a missing signature! That is, the requested ordinal - // is larger than the number of signatures in this block. Return a "special" string so that - // the query keeps going, but that can be recognized as an internal error upon inspection. - //@Bug 2534. Change the length check to 8000 - - // MCOL-267: - // With BLOB support we have had to increase this to 8176 - // because a BLOB can take 8176 bytes of a dictionary block - // instead of the fixed 8000 with CHAR/VARCHAR - if (ret->len < 0 || ret->len > 8176) - { - ret->data = reinterpret_cast(signatureNotFound); - ret->len = strlen(reinterpret_cast(ret->data)); - } - else - ret->data = &niceBlock[offsets[oldParams[dict_OffsetIndex].offsetIndex]]; - } - -// idbassert(ret->len >= 0); - currentOffsetIndex = oldParams[dict_OffsetIndex].offsetIndex; - dict_OffsetIndex++; - return; - } - - /* XXXPAT: Need to check for the NULL token here */ - ret->len = tokens[dict_OffsetIndex].len; - ret->data = &niceBlock[tokens[dict_OffsetIndex].offset]; - - if (outputFlags & OT_TOKEN) - { - //offsets = reinterpret_cast(&niceBlock[10]); - for (currentOffsetIndex = 1; offsets[currentOffsetIndex] != 0xffff; currentOffsetIndex++) - if (tokens[dict_OffsetIndex].offset == offsets[currentOffsetIndex]) - break; - - if (offsets[currentOffsetIndex] == 0xffff) - { - MessageLog logger(LoggingID(28)); - logging::Message::Args offset; - Message msg(38); - - offset.add(tokens[dict_OffsetIndex].offset); - msg.format(offset); - logger.logErrorMessage(msg); - - currentOffsetIndex = -1; - dict_OffsetIndex++; - return; - } - } + ret->len = -1; + return; } - dict_OffsetIndex++; + ret->len = offsets[dict_OffsetIndex] - offsets[dict_OffsetIndex + 1]; + ret->data = &niceBlock[offsets[dict_OffsetIndex + 1]]; + + if (outputFlags & OT_TOKEN) + currentOffsetIndex = dict_OffsetIndex + 1; + } + else + { + again: + + if (dict_OffsetIndex >= NVALS) + { + ret->len = -1; + return; + } + + if (oldGetSigBehavior) + { + const OldGetSigParams* oldParams = reinterpret_cast(tokens); + + if (oldParams[dict_OffsetIndex].rid & 0x8000000000000000LL) + { + if (skipNulls) + { + /* Bug 3321. For some cases the NULL token should be skipped. The + * isnull filter is handled by token columncommand or by the F & E + * framework. This primitive should only process nulls + * when it's for projection. + */ + dict_OffsetIndex++; + goto again; + } + + ret->len = nullStringLen; + ret->data = (const uint8_t*)nullString; + } + else + { + ret->len = offsets[oldParams[dict_OffsetIndex].offsetIndex - 1] - + offsets[oldParams[dict_OffsetIndex].offsetIndex]; + // Whoa! apparently we have come across a missing signature! That is, the requested ordinal + // is larger than the number of signatures in this block. Return a "special" string so that + // the query keeps going, but that can be recognized as an internal error upon inspection. + //@Bug 2534. Change the length check to 8000 + + // MCOL-267: + // With BLOB support we have had to increase this to 8176 + // because a BLOB can take 8176 bytes of a dictionary block + // instead of the fixed 8000 with CHAR/VARCHAR + if (ret->len < 0 || ret->len > 8176) + { + ret->data = reinterpret_cast(signatureNotFound); + ret->len = strlen(reinterpret_cast(ret->data)); + } + else + ret->data = &niceBlock[offsets[oldParams[dict_OffsetIndex].offsetIndex]]; + } + + // idbassert(ret->len >= 0); + currentOffsetIndex = oldParams[dict_OffsetIndex].offsetIndex; + dict_OffsetIndex++; + return; + } + + /* XXXPAT: Need to check for the NULL token here */ + ret->len = tokens[dict_OffsetIndex].len; + ret->data = &niceBlock[tokens[dict_OffsetIndex].offset]; + + if (outputFlags & OT_TOKEN) + { + // offsets = reinterpret_cast(&niceBlock[10]); + for (currentOffsetIndex = 1; offsets[currentOffsetIndex] != 0xffff; currentOffsetIndex++) + if (tokens[dict_OffsetIndex].offset == offsets[currentOffsetIndex]) + break; + + if (offsets[currentOffsetIndex] == 0xffff) + { + MessageLog logger(LoggingID(28)); + logging::Message::Args offset; + Message msg(38); + + offset.add(tokens[dict_OffsetIndex].offset); + msg.format(offset); + logger.logErrorMessage(msg); + + currentOffsetIndex = -1; + dict_OffsetIndex++; + return; + } + } + } + + dict_OffsetIndex++; } -void PrimitiveProcessor::p_Dictionary(const DictInput* in, - vector* out, - bool skipNulls, - uint32_t charsetNumber, - boost::shared_ptr eqFilter, +void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, bool skipNulls, + uint32_t charsetNumber, boost::shared_ptr eqFilter, uint8_t eqOp) { - PrimToken* outToken; - const DictFilterElement* filter = 0; - const uint8_t* in8; - DataValue* outValue; - p_DataValue min = {0, NULL}, max = {0, NULL}, sigptr = {0, NULL}; - int tmp, filterIndex, filterOffset; - uint16_t aggCount; - bool cmpResult; - DictOutput header; - const datatypes::Charset &cs(charsetNumber); + PrimToken* outToken; + const DictFilterElement* filter = 0; + const uint8_t* in8; + DataValue* outValue; + p_DataValue min = {0, NULL}, max = {0, NULL}, sigptr = {0, NULL}; + int tmp, filterIndex, filterOffset; + uint16_t aggCount; + bool cmpResult; + DictOutput header; + const datatypes::Charset& cs(charsetNumber); - // default size of the ouput to something sufficiently large to prevent - // excessive reallocation and copy when resizing - const unsigned DEF_OUTSIZE = 16 * 1024; - // use this factor to scale out size of future resize calls - const int SCALE_FACTOR = 2; - out->resize(DEF_OUTSIZE); + // default size of the ouput to something sufficiently large to prevent + // excessive reallocation and copy when resizing + const unsigned DEF_OUTSIZE = 16 * 1024; + // use this factor to scale out size of future resize calls + const int SCALE_FACTOR = 2; + out->resize(DEF_OUTSIZE); - in8 = reinterpret_cast(in); + in8 = reinterpret_cast(in); - { - void *hp = static_cast(&header); - memcpy(hp, in, sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - } - header.ism.Command = DICT_RESULTS; - header.NVALS = 0; - header.LBID = in->LBID; - dict_OffsetIndex = 0; - filterIndex = 0; - aggCount = 0; - min.len = 0; - max.len = 0; + { + void* hp = static_cast(&header); + memcpy(hp, in, sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + } + header.ism.Command = DICT_RESULTS; + header.NVALS = 0; + header.LBID = in->LBID; + dict_OffsetIndex = 0; + filterIndex = 0; + aggCount = 0; + min.len = 0; + max.len = 0; - //...Initialize I/O counts - header.CacheIO = 0; - header.PhysicalIO = 0; + //...Initialize I/O counts + header.CacheIO = 0; + header.PhysicalIO = 0; - header.NBYTES = sizeof(DictOutput); - - for (nextSig(in->NVALS, in->tokens, &sigptr, in->OutputType, - (in->InputFlags ? true : false), skipNulls); - sigptr.len != -1; - nextSig(in->NVALS, in->tokens, &sigptr, in->OutputType, - (in->InputFlags ? true : false), skipNulls)) - { - // do aggregate processing - if (in->OutputType & OT_AGGREGATE) - { - // len == 0 indicates this is the first pass - if (max.len != 0) - { - tmp = cs.strnncollsp(sigptr.data, sigptr.len, max.data, max.len); - - if (tmp > 0) - max = sigptr; - } - else - max = sigptr; - - if (min.len != 0) - { - tmp = cs.strnncollsp(sigptr.data, sigptr.len, min.data, min.len); - - if (tmp < 0) - min = sigptr; - } - else - min = sigptr; - - aggCount++; - } - - // filter processing - if (in->InputFlags == 1) - filterOffset = sizeof(DictInput) + (in->NVALS * sizeof(OldGetSigParams)); - else - filterOffset = sizeof(DictInput) + (in->NVALS * sizeof(PrimToken)); - - if (eqFilter) - { - // MCOL-4407. - // Support filters: - // `where key = value0 and key = value1 {and key = value2}` - // - // The problem occurs only when HWM > columnstore_string_scan_threshold - 1 - // because in this case: - // CS uses `tryCombineDictionary` which combines `DictStep`s into one as result - // `eqFilter` has more than 1 filter and applies the logic below which is `or` by - // default. - // Note: The case HWM <= columnstore_string_scan_threshold - 1 has the same problem, - // function `p_TokenByScan`, but it does not occur because `tryCombineDictionaryScan` - // was turned off. - if (eqFilter->size() > 1 && in->BOP == BOP_AND && eqOp == COMPARE_EQ) - goto no_store; - - if (eqFilter->size() > 1 && in->BOP == BOP_OR && eqOp == COMPARE_NE) - goto store; - - // MCOL-1246 Trim whitespace before match - string strData((char*)sigptr.data, sigptr.len); - boost::trim_right_if(strData, boost::is_any_of(" ")); - bool gotIt = eqFilter->find(strData) != eqFilter->end(); - - if ((gotIt && eqOp == COMPARE_EQ) || (!gotIt && eqOp == COMPARE_NE)) - goto store; - - goto no_store; - } - - for (filterIndex = 0; filterIndex < in->NOPS; filterIndex++) - { - filter = reinterpret_cast(&in8[filterOffset]); - - cmpResult = compare(cs, filter->COP, - (const char *) sigptr.data, sigptr.len, - (const char *) filter->data, filter->len); - - if (!cmpResult && in->BOP != BOP_OR) - goto no_store; - - if (cmpResult && in->BOP != BOP_AND) - goto store; - - filterOffset += sizeof(DictFilterElement) + filter->len; - } - - if (filterIndex == in->NOPS && in->BOP != BOP_OR) - { -store: - //cout << "storing it, str = " << string((char *)sigptr.data, sigptr.len) << endl; - header.NVALS++; - - if (in->OutputType & OT_RID && in->InputFlags == 1) // hack that indicates old GetSignature behavior - { - const OldGetSigParams* oldParams; - uint64_t* outRid; - oldParams = reinterpret_cast(in->tokens); - uint32_t newlen = header.NBYTES + 8; - - if ( newlen > out->size() ) - { - out->resize( out->size() * SCALE_FACTOR ); - } - - outRid = (uint64_t*) & (*out)[header.NBYTES]; - // mask off the upper bit of the rid; signifies the NULL token was passed in - *outRid = (oldParams[dict_OffsetIndex - 1].rid & 0x7fffffffffffffffLL); - header.NBYTES += 8; - } - - if (in->OutputType & OT_INPUTARG && in->InputFlags == 0) - { - uint32_t newlen = header.NBYTES + sizeof(DataValue) + filter->len; - - if ( newlen > out->size() ) - { - out->resize( out->size() * SCALE_FACTOR ); - } - - outValue = reinterpret_cast(&(*out)[header.NBYTES]); - outValue->len = filter->len; - memcpy(outValue->data, filter->data, filter->len); - header.NBYTES += sizeof(DataValue) + filter->len; - } - - if (in->OutputType & OT_TOKEN) - { - uint32_t newlen = header.NBYTES + sizeof(PrimToken); - - if ( newlen > out->size() ) - { - out->resize( out->size() * SCALE_FACTOR ); - } - - outToken = reinterpret_cast(&(*out)[header.NBYTES]); - outToken->LBID = in->LBID; - outToken->offset = currentOffsetIndex; - outToken->len = filter->len; - header.NBYTES += sizeof(PrimToken); - } - - if (in->OutputType & OT_DATAVALUE) - { - uint32_t newlen = header.NBYTES + sizeof(DataValue) + sigptr.len; - - if ( newlen > out->size() ) - { - out->resize( out->size() * SCALE_FACTOR ); - } - - outValue = reinterpret_cast(&(*out)[header.NBYTES]); - outValue->len = sigptr.len; - memcpy(outValue->data, sigptr.data, sigptr.len); - header.NBYTES += sizeof(DataValue) + sigptr.len; - } - } - -no_store: ; // intentional - } + header.NBYTES = sizeof(DictOutput); + for (nextSig(in->NVALS, in->tokens, &sigptr, in->OutputType, (in->InputFlags ? true : false), skipNulls); + sigptr.len != -1; + nextSig(in->NVALS, in->tokens, &sigptr, in->OutputType, (in->InputFlags ? true : false), skipNulls)) + { + // do aggregate processing if (in->OutputType & OT_AGGREGATE) { - uint32_t newlen = header.NBYTES + 3 * sizeof(uint16_t) + min.len + max.len; + // len == 0 indicates this is the first pass + if (max.len != 0) + { + tmp = cs.strnncollsp(sigptr.data, sigptr.len, max.data, max.len); - if ( newlen > out->size() ) - { - out->resize( out->size() * SCALE_FACTOR ); - } + if (tmp > 0) + max = sigptr; + } + else + max = sigptr; - uint16_t* tmp16 = reinterpret_cast(&(*out)[header.NBYTES]); - DataValue* tmpDV = reinterpret_cast(&(*out)[header.NBYTES + sizeof(uint16_t)]); + if (min.len != 0) + { + tmp = cs.strnncollsp(sigptr.data, sigptr.len, min.data, min.len); - *tmp16 = aggCount; - tmpDV->len = min.len; - memcpy(tmpDV->data, min.data, min.len); - header.NBYTES += 2 * sizeof(uint16_t) + min.len; + if (tmp < 0) + min = sigptr; + } + else + min = sigptr; - tmpDV = reinterpret_cast(&(*out)[header.NBYTES]); - tmpDV->len = max.len; - memcpy(tmpDV->data, max.data, max.len); - header.NBYTES += sizeof(uint16_t) + max.len; + aggCount++; } - out->resize( header.NBYTES ); + // filter processing + if (in->InputFlags == 1) + filterOffset = sizeof(DictInput) + (in->NVALS * sizeof(OldGetSigParams)); + else + filterOffset = sizeof(DictInput) + (in->NVALS * sizeof(PrimToken)); - memcpy(&(*out)[0], &header, sizeof(DictOutput)); + if (eqFilter) + { + // MCOL-4407. + // Support filters: + // `where key = value0 and key = value1 {and key = value2}` + // + // The problem occurs only when HWM > columnstore_string_scan_threshold - 1 + // because in this case: + // CS uses `tryCombineDictionary` which combines `DictStep`s into one as result + // `eqFilter` has more than 1 filter and applies the logic below which is `or` by + // default. + // Note: The case HWM <= columnstore_string_scan_threshold - 1 has the same problem, + // function `p_TokenByScan`, but it does not occur because `tryCombineDictionaryScan` + // was turned off. + if (eqFilter->size() > 1 && in->BOP == BOP_AND && eqOp == COMPARE_EQ) + goto no_store; + + if (eqFilter->size() > 1 && in->BOP == BOP_OR && eqOp == COMPARE_NE) + goto store; + + // MCOL-1246 Trim whitespace before match + string strData((char*)sigptr.data, sigptr.len); + boost::trim_right_if(strData, boost::is_any_of(" ")); + bool gotIt = eqFilter->find(strData) != eqFilter->end(); + + if ((gotIt && eqOp == COMPARE_EQ) || (!gotIt && eqOp == COMPARE_NE)) + goto store; + + goto no_store; + } + + for (filterIndex = 0; filterIndex < in->NOPS; filterIndex++) + { + filter = reinterpret_cast(&in8[filterOffset]); + + cmpResult = compare(cs, filter->COP, (const char*)sigptr.data, sigptr.len, (const char*)filter->data, + filter->len); + + if (!cmpResult && in->BOP != BOP_OR) + goto no_store; + + if (cmpResult && in->BOP != BOP_AND) + goto store; + + filterOffset += sizeof(DictFilterElement) + filter->len; + } + + if (filterIndex == in->NOPS && in->BOP != BOP_OR) + { + store: + // cout << "storing it, str = " << string((char *)sigptr.data, sigptr.len) << endl; + header.NVALS++; + + if (in->OutputType & OT_RID && in->InputFlags == 1) // hack that indicates old GetSignature behavior + { + const OldGetSigParams* oldParams; + uint64_t* outRid; + oldParams = reinterpret_cast(in->tokens); + uint32_t newlen = header.NBYTES + 8; + + if (newlen > out->size()) + { + out->resize(out->size() * SCALE_FACTOR); + } + + outRid = (uint64_t*)&(*out)[header.NBYTES]; + // mask off the upper bit of the rid; signifies the NULL token was passed in + *outRid = (oldParams[dict_OffsetIndex - 1].rid & 0x7fffffffffffffffLL); + header.NBYTES += 8; + } + + if (in->OutputType & OT_INPUTARG && in->InputFlags == 0) + { + uint32_t newlen = header.NBYTES + sizeof(DataValue) + filter->len; + + if (newlen > out->size()) + { + out->resize(out->size() * SCALE_FACTOR); + } + + outValue = reinterpret_cast(&(*out)[header.NBYTES]); + outValue->len = filter->len; + memcpy(outValue->data, filter->data, filter->len); + header.NBYTES += sizeof(DataValue) + filter->len; + } + + if (in->OutputType & OT_TOKEN) + { + uint32_t newlen = header.NBYTES + sizeof(PrimToken); + + if (newlen > out->size()) + { + out->resize(out->size() * SCALE_FACTOR); + } + + outToken = reinterpret_cast(&(*out)[header.NBYTES]); + outToken->LBID = in->LBID; + outToken->offset = currentOffsetIndex; + outToken->len = filter->len; + header.NBYTES += sizeof(PrimToken); + } + + if (in->OutputType & OT_DATAVALUE) + { + uint32_t newlen = header.NBYTES + sizeof(DataValue) + sigptr.len; + + if (newlen > out->size()) + { + out->resize(out->size() * SCALE_FACTOR); + } + + outValue = reinterpret_cast(&(*out)[header.NBYTES]); + outValue->len = sigptr.len; + memcpy(outValue->data, sigptr.data, sigptr.len); + header.NBYTES += sizeof(DataValue) + sigptr.len; + } + } + + no_store:; // intentional + } + + if (in->OutputType & OT_AGGREGATE) + { + uint32_t newlen = header.NBYTES + 3 * sizeof(uint16_t) + min.len + max.len; + + if (newlen > out->size()) + { + out->resize(out->size() * SCALE_FACTOR); + } + + uint16_t* tmp16 = reinterpret_cast(&(*out)[header.NBYTES]); + DataValue* tmpDV = reinterpret_cast(&(*out)[header.NBYTES + sizeof(uint16_t)]); + + *tmp16 = aggCount; + tmpDV->len = min.len; + memcpy(tmpDV->data, min.data, min.len); + header.NBYTES += 2 * sizeof(uint16_t) + min.len; + + tmpDV = reinterpret_cast(&(*out)[header.NBYTES]); + tmpDV->len = max.len; + memcpy(tmpDV->data, max.data, max.len); + header.NBYTES += sizeof(uint16_t) + max.len; + } + + out->resize(header.NBYTES); + + memcpy(&(*out)[0], &header, sizeof(DictOutput)); } -} +} // namespace primitives // vim:ts=4 sw=4: - diff --git a/primitives/linux-port/index.cpp b/primitives/linux-port/index.cpp index f2a4a681d..bbfc949ef 100644 --- a/primitives/linux-port/index.cpp +++ b/primitives/linux-port/index.cpp @@ -37,926 +37,911 @@ using namespace std; using namespace logging; #ifdef VERBOSE -#define GET_BITTEST(test, string, x) \ - if (in->Shift + x >= in->SSlen) { \ - test = (string & masks[in->SSlen - in->Shift]); \ - lastStage = true; \ - cerr << " bittest is 0x" << hex << (int) test << dec << " this is the last iteration" << endl; \ - } \ - else { \ - test = (string >> (in->SSlen - in->Shift - x)) & masks[x]; \ - lastStage = false; \ - cerr << " bittest is 0x" << hex << (int) test << dec << endl; \ - } +#define GET_BITTEST(test, string, x) \ + if (in->Shift + x >= in->SSlen) \ + { \ + test = (string & masks[in->SSlen - in->Shift]); \ + lastStage = true; \ + cerr << " bittest is 0x" << hex << (int)test << dec << " this is the last iteration" << endl; \ + } \ + else \ + { \ + test = (string >> (in->SSlen - in->Shift - x)) & masks[x]; \ + lastStage = false; \ + cerr << " bittest is 0x" << hex << (int)test << dec << endl; \ + } #else -#define GET_BITTEST(test, string, x) \ - if (in->Shift + x >= in->SSlen) { \ - test = (string & masks[in->SSlen - in->Shift]); \ - lastStage = true; \ - } \ - else { \ - test = (string >> (in->SSlen - in->Shift - x)) & masks[x]; \ - lastStage = false; \ - } +#define GET_BITTEST(test, string, x) \ + if (in->Shift + x >= in->SSlen) \ + { \ + test = (string & masks[in->SSlen - in->Shift]); \ + lastStage = true; \ + } \ + else \ + { \ + test = (string >> (in->SSlen - in->Shift - x)) & masks[x]; \ + lastStage = false; \ + } #endif -#define IDXWALK_INIT() \ - niceBlock = reinterpret_cast(block); \ - blockOffset = (in->SubBlock * WriteEngine::SUBBLOCK_TOTAL_BYTES) + \ - (in->SBEntry * 8); \ - treePtr = reinterpret_cast(&niceBlock[blockOffset]); +#define IDXWALK_INIT() \ + niceBlock = reinterpret_cast(block); \ + blockOffset = (in->SubBlock * WriteEngine::SUBBLOCK_TOTAL_BYTES) + (in->SBEntry * 8); \ + treePtr = reinterpret_cast(&niceBlock[blockOffset]); -#define GET_GROUP_SIZE() \ - switch (treePtr->group) { \ - case 0: bitTestGroupSize = 1; break; \ - case 1: bitTestGroupSize = 2; break; \ - case 2: bitTestGroupSize = 4; break; \ - case 3: bitTestGroupSize = 8; break; \ - case 4: bitTestGroupSize = 16; break; \ - case 5: bitTestGroupSize = 32; break; \ - default: \ - cerr << "PrimitiveProcessor::IndexWalk*(): bad group field " << \ - treePtr->group << endl; \ - return; \ - } +#define GET_GROUP_SIZE() \ + switch (treePtr->group) \ + { \ + case 0: bitTestGroupSize = 1; break; \ + case 1: bitTestGroupSize = 2; break; \ + case 2: bitTestGroupSize = 4; break; \ + case 3: bitTestGroupSize = 8; break; \ + case 4: bitTestGroupSize = 16; break; \ + case 5: bitTestGroupSize = 32; break; \ + default: cerr << "PrimitiveProcessor::IndexWalk*(): bad group field " << treePtr->group << endl; return; \ + } #ifdef VERBOSE -#define ADD_ELEMENT(index, shift, state) \ - element = new IndexWalkHeader(); \ - memcpy(element, in, sizeof(IndexWalkHeader)); \ - element->ism.Command = INDEX_WALK_RESULTS; \ - element->Shift += shift; \ - element->LBID = treePtr[index].fbo; \ - element->SubBlock = treePtr[index].sbid; \ - element->SBEntry = treePtr[index].entry; \ - element->State = state; \ - cerr << " (no convert) creating a result from subblock entry " << (int) index << " with Shift=" << (int) element->Shift << \ - " LBID=" << element->LBID << " Subblock=" << (int) element->SubBlock << " Subblock entry=" << \ - (int) element->SBEntry << " State=" << (int) state; \ - if (element->LBID == in->LBID && element->Shift < element->SSlen) { \ - cerr << " recursing..." << endl; \ - p_IdxWalk(element, out); \ - delete element; \ - } \ - else { \ - cerr << " adding this to the result set" << endl; \ - out->push_back(element); \ - } +#define ADD_ELEMENT(index, shift, state) \ + element = new IndexWalkHeader(); \ + memcpy(element, in, sizeof(IndexWalkHeader)); \ + element->ism.Command = INDEX_WALK_RESULTS; \ + element->Shift += shift; \ + element->LBID = treePtr[index].fbo; \ + element->SubBlock = treePtr[index].sbid; \ + element->SBEntry = treePtr[index].entry; \ + element->State = state; \ + cerr << " (no convert) creating a result from subblock entry " << (int)index \ + << " with Shift=" << (int)element->Shift << " LBID=" << element->LBID \ + << " Subblock=" << (int)element->SubBlock << " Subblock entry=" << (int)element->SBEntry \ + << " State=" << (int)state; \ + if (element->LBID == in->LBID && element->Shift < element->SSlen) \ + { \ + cerr << " recursing..." << endl; \ + p_IdxWalk(element, out); \ + delete element; \ + } \ + else \ + { \ + cerr << " adding this to the result set" << endl; \ + out->push_back(element); \ + } #else -#define ADD_ELEMENT(index, shift, state) \ - element = new IndexWalkHeader(); \ - memcpy(element, in, sizeof(IndexWalkHeader)); \ - element->ism.Command = INDEX_WALK_RESULTS; \ - element->Shift += shift; \ - element->LBID = treePtr[index].fbo; \ - element->SubBlock = treePtr[index].sbid; \ - element->SBEntry = treePtr[index].entry; \ - element->State = state; \ - if (element->LBID == in->LBID && element->Shift < element->SSlen) { \ - p_IdxWalk(element, out); \ - delete element; \ - } \ - else \ - out->push_back(element); +#define ADD_ELEMENT(index, shift, state) \ + element = new IndexWalkHeader(); \ + memcpy(element, in, sizeof(IndexWalkHeader)); \ + element->ism.Command = INDEX_WALK_RESULTS; \ + element->Shift += shift; \ + element->LBID = treePtr[index].fbo; \ + element->SubBlock = treePtr[index].sbid; \ + element->SBEntry = treePtr[index].entry; \ + element->State = state; \ + if (element->LBID == in->LBID && element->Shift < element->SSlen) \ + { \ + p_IdxWalk(element, out); \ + delete element; \ + } \ + else \ + out->push_back(element); #endif #ifdef VERBOSE -#define ADD_ELEMENT_WITH_CONVERT(index, shift, state) \ - element = new IndexWalkHeader(); \ - memcpy(element, in, sizeof(IndexWalkHeader)); \ - element->ism.Command = INDEX_WALK_RESULTS; \ - element->Shift += shift; \ - element->LBID = treePtr[index].fbo; \ - element->SubBlock = treePtr[index].sbid; \ - element->SBEntry = treePtr[index].entry; \ - element->State = state; \ - cerr << " (convert) creating a result from subblock entry " << (int) index << " with Shift=" << (int) element->Shift << \ - " LBID=" << element->LBID << " Subblock=" << (int) element->SubBlock << " Subblock entry=" << \ - (int) element->SBEntry << " State=" << (int) state << endl; \ - if (convertToSingleOp != -1) { \ - cerr << " converting it to a single COP filter" << endl; \ - element->NVALS = 1; \ - if (convertToSingleOp == 1) { \ - element->COP1 = element->COP2; \ - element->SearchString[0] = element->SearchString[1]; \ - } \ - } \ - if (element->LBID == in->LBID && element->Shift < element->SSlen) { \ - cerr << " recursing..." << endl; \ - p_IdxWalk(element, out); \ - delete element; \ - } \ - else { \ - cerr << " adding it to the result set" << endl;\ - out->push_back(element); \ - } +#define ADD_ELEMENT_WITH_CONVERT(index, shift, state) \ + element = new IndexWalkHeader(); \ + memcpy(element, in, sizeof(IndexWalkHeader)); \ + element->ism.Command = INDEX_WALK_RESULTS; \ + element->Shift += shift; \ + element->LBID = treePtr[index].fbo; \ + element->SubBlock = treePtr[index].sbid; \ + element->SBEntry = treePtr[index].entry; \ + element->State = state; \ + cerr << " (convert) creating a result from subblock entry " << (int)index \ + << " with Shift=" << (int)element->Shift << " LBID=" << element->LBID \ + << " Subblock=" << (int)element->SubBlock << " Subblock entry=" << (int)element->SBEntry \ + << " State=" << (int)state << endl; \ + if (convertToSingleOp != -1) \ + { \ + cerr << " converting it to a single COP filter" << endl; \ + element->NVALS = 1; \ + if (convertToSingleOp == 1) \ + { \ + element->COP1 = element->COP2; \ + element->SearchString[0] = element->SearchString[1]; \ + } \ + } \ + if (element->LBID == in->LBID && element->Shift < element->SSlen) \ + { \ + cerr << " recursing..." << endl; \ + p_IdxWalk(element, out); \ + delete element; \ + } \ + else \ + { \ + cerr << " adding it to the result set" << endl; \ + out->push_back(element); \ + } #else -#define ADD_ELEMENT_WITH_CONVERT(index, shift, state) \ - element = new IndexWalkHeader(); \ - memcpy(element, in, sizeof(IndexWalkHeader)); \ - element->ism.Command = INDEX_WALK_RESULTS; \ - element->Shift += shift; \ - element->LBID = treePtr[index].fbo; \ - element->SubBlock = treePtr[index].sbid; \ - element->SBEntry = treePtr[index].entry; \ - element->State = state; \ - if (convertToSingleOp != -1) { \ - element->NVALS = 1; \ - if (convertToSingleOp == 1) { \ - element->COP1 = element->COP2; \ - element->SearchString[0] = element->SearchString[1]; \ - } \ - } \ - if (element->LBID == in->LBID && element->Shift < element->SSlen) { \ - p_IdxWalk(element, out); \ - delete element; \ - } \ - else \ - out->push_back(element); +#define ADD_ELEMENT_WITH_CONVERT(index, shift, state) \ + element = new IndexWalkHeader(); \ + memcpy(element, in, sizeof(IndexWalkHeader)); \ + element->ism.Command = INDEX_WALK_RESULTS; \ + element->Shift += shift; \ + element->LBID = treePtr[index].fbo; \ + element->SubBlock = treePtr[index].sbid; \ + element->SBEntry = treePtr[index].entry; \ + element->State = state; \ + if (convertToSingleOp != -1) \ + { \ + element->NVALS = 1; \ + if (convertToSingleOp == 1) \ + { \ + element->COP1 = element->COP2; \ + element->SearchString[0] = element->SearchString[1]; \ + } \ + } \ + if (element->LBID == in->LBID && element->Shift < element->SSlen) \ + { \ + p_IdxWalk(element, out); \ + delete element; \ + } \ + else \ + out->push_back(element); #endif namespace primitives { - -void PrimitiveProcessor::indexWalk_1(const IndexWalkHeader* in, - vector* out) throw() +void PrimitiveProcessor::indexWalk_1(const IndexWalkHeader* in, vector* out) throw() { - uint16_t bitTest; - uint8_t* niceBlock; - int blockOffset, bitTestGroupSize, i; - WriteEngine::IdxBitTestEntry* treePtr; - IndexWalkHeader* element; - int cmp; + uint16_t bitTest; + uint8_t* niceBlock; + int blockOffset, bitTestGroupSize, i; + WriteEngine::IdxBitTestEntry* treePtr; + IndexWalkHeader* element; + int cmp; - IDXWALK_INIT(); + IDXWALK_INIT(); - if (in->SubBlock == 1 && in->Shift == 0) //assume this is the first lookup step - { - //which happens in the direct pointer block - bitTest = in->SearchString[0] >> (in->SSlen - 5); + if (in->SubBlock == 1 && in->Shift == 0) // assume this is the first lookup step + { + // which happens in the direct pointer block + bitTest = in->SearchString[0] >> (in->SSlen - 5); #ifdef VERBOSE - cerr << " first iteration of search for 0x" << hex << in->SearchString[0] << dec << endl; - cerr << " bitTest is 0x" << hex << bitTest << dec << endl; + cerr << " first iteration of search for 0x" << hex << in->SearchString[0] << dec << endl; + cerr << " bitTest is 0x" << hex << bitTest << dec << endl; #endif - switch (in->COP1) + switch (in->COP1) + { + case COMPARE_LT: + case COMPARE_LE: + for (i = 0; i <= bitTest; i++) { - case COMPARE_LT: - case COMPARE_LE: - for (i = 0; i <= bitTest; i++) - { - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; - ADD_ELEMENT(i, 5, (i < bitTest ? 1 : 0)); - } - - break; - - case COMPARE_GT: - case COMPARE_GE: - for (i = bitTest; i < 32; i++) - { - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; - - ADD_ELEMENT(i, 5, (i > bitTest ? 1 : 0)); - } - - break; - - case COMPARE_EQ: - if (treePtr[bitTest].fbo == 0 && treePtr[bitTest].sbid == 0 && treePtr[bitTest].entry == 0) - break; - - ADD_ELEMENT(bitTest, 5, 0); - break; - - case COMPARE_NE: - for (i = 0; i < 32; i++) - { - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; - - ADD_ELEMENT(i, 5, (i != bitTest ? 1 : 0)); - } - - break; - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(34); - - colWidth.add(in->COP1); - colWidth.add("indexWalk_1"); - msg.format(colWidth); - logger.logDebugMessage(msg); - return; + ADD_ELEMENT(i, 5, (i < bitTest ? 1 : 0)); } - } - // This is the general case where we're working within a bit test group - else - { - GET_GROUP_SIZE(); -#ifdef VERBOSE - cerr << " search string is 0x" << hex << in->SearchString[0] << dec << endl; -#endif + break; - for (i = 0; i < bitTestGroupSize; i++) + case COMPARE_GT: + case COMPARE_GE: + for (i = bitTest; i < 32; i++) { - bool lastStage; + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; - // skip holes - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; - - if (treePtr[i].bitCompare == 0) - { - GET_BITTEST(bitTest, in->SearchString[0], 5); - } - else - { - GET_BITTEST(bitTest, in->SearchString[0], 10); - } - - cmp = compare(treePtr[i].bitTest, bitTest, in->COP1, lastStage); - - if (cmp > 0) - { - ADD_ELEMENT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), (cmp == 2 ? 1 : 0)); - } + ADD_ELEMENT(i, 5, (i > bitTest ? 1 : 0)); } - } -} -inline int PrimitiveProcessor::compare(int val1, int val2, uint8_t COP, - bool lastStage) throw() -{ - switch (COP) - { - case COMPARE_LT: - if (val1 < val2) - return 2; + break; - if (val1 == val2 && !lastStage) - return 1; + case COMPARE_EQ: + if (treePtr[bitTest].fbo == 0 && treePtr[bitTest].sbid == 0 && treePtr[bitTest].entry == 0) + break; - return 0; - - case COMPARE_LE: - if (val1 < val2) - return 2; - - if (val1 == val2) - return 1; - - return 0; - - case COMPARE_GT: - if (val1 > val2) - return 2; - - if (val1 == val2 && !lastStage) - return 1; - - return 0; - - case COMPARE_GE: - if (val1 > val2) - return 2; - - if (val1 == val2) - return 1; - - return 0; - - case COMPARE_EQ: - if (val1 == val2) - return 1; - - return 0; - - case COMPARE_NE: - if (val1 != val2) - return 2; - - if (!lastStage) - return 1; - - return 0; - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(34); - - colWidth.add(COP); - colWidth.add("compare"); - msg.format(colWidth); - logger.logDebugMessage(msg); - return false; - } -} - -void PrimitiveProcessor::indexWalk_2(const IndexWalkHeader* in, - vector* out) throw() -{ - uint16_t bitTest1, bitTest2; - uint8_t* niceBlock; - int blockOffset, bitTestGroupSize, cmp[2], i; - WriteEngine::IdxBitTestEntry* treePtr; - IndexWalkHeader* element; - int convertToSingleOp; - bool lastStage, setState; - - IDXWALK_INIT(); - - if (in->SubBlock == 1 && in->Shift == 0) //assume this is the first lookup step - { - //which happens in the direct pointer block - bitTest1 = in->SearchString[0] >> (in->SSlen - 5); - bitTest2 = in->SearchString[1] >> (in->SSlen - 5); -#ifdef VERBOSE - cerr << " first iteration. SearchString[0]=0x" << hex << in->SearchString[0] << - " bittest=0x" << bitTest1 << " SearchString[1]=0x" << in->SearchString[1] << - " bittest=0x" << bitTest2 << dec << endl; -#endif + ADD_ELEMENT(bitTest, 5, 0); + break; + case COMPARE_NE: for (i = 0; i < 32; i++) { - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; - - setState = false; - convertToSingleOp = -1; - cmp[0] = compare(i, bitTest1, in->COP1, false); - cmp[1] = compare(i, bitTest2, in->COP2, false); - - switch (in->BOP) - { - case BOP_OR: - if (cmp[0] == 2 || cmp[1] == 2) - { - setState = true; - goto add2; - } - -// if (cmp[0] == 1 || cmp[1] == 1) -// goto add2; - - /* XXXPAT: clean up this logic */ - - if (cmp[0] == 1 && cmp[1] == 1) - goto add2; - - if (cmp[0] == 1) - { - convertToSingleOp = 0; - ADD_ELEMENT_WITH_CONVERT(i, 5, 0); - goto skip2; - } - else if (cmp[1] == 1) - { - convertToSingleOp = 1; - ADD_ELEMENT_WITH_CONVERT(i, 5, 0); - goto skip2; - } - - break; - - /* XXXPAT: Need to verify the logic for AND. - observations: if control reaches this point, then in the previous iteration the - decision must have been 1, which implies equality for every previous comparison. - - If one of the comparisons returns 0, then there are no entries in this subtree - in the result set and it can stop here. - else, if both of the comparisons return 2, then the whole subtree is in the result set. - else, if only one of the comparisons returns 2, then that comparison will be 2 for - every comparison made down this subtree, and it is equivalent to being - (cmp1 && true). In this case, we repackage the query as a single argument version - (using the comparison operator that was 1) to be processed by indexwalk_1(). - else, the comparisons are both 1, meaning we have equality so far and can't decide on - this subtree yet. - */ - case BOP_AND: - if (cmp[0] == 0 || cmp[1] == 0) - break; - else if (cmp[0] == 2 && cmp[1] == 2) - setState = true; - else if (cmp[0] == 2) - convertToSingleOp = 1; - else if (cmp[1] == 2) - convertToSingleOp = 0; - - goto add2; - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(39); - - colWidth.add(in->BOP); - colWidth.add("indexwalk_2"); - msg.format(colWidth); - logger.logDebugMessage(msg); - return; - } - + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) continue; -add2: - ADD_ELEMENT_WITH_CONVERT(i, 5, (setState ? 1 : 0)); -skip2: - ; + + ADD_ELEMENT(i, 5, (i != bitTest ? 1 : 0)); } + + break; + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(34); + + colWidth.add(in->COP1); + colWidth.add("indexWalk_1"); + msg.format(colWidth); + logger.logDebugMessage(msg); + return; } - else // the general case + } + // This is the general case where we're working within a bit test group + else + { + GET_GROUP_SIZE(); +#ifdef VERBOSE + cerr << " search string is 0x" << hex << in->SearchString[0] << dec << endl; +#endif + + for (i = 0; i < bitTestGroupSize; i++) { - GET_GROUP_SIZE(); + bool lastStage; + + // skip holes + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; + + if (treePtr[i].bitCompare == 0) + { + GET_BITTEST(bitTest, in->SearchString[0], 5); + } + else + { + GET_BITTEST(bitTest, in->SearchString[0], 10); + } + + cmp = compare(treePtr[i].bitTest, bitTest, in->COP1, lastStage); + + if (cmp > 0) + { + ADD_ELEMENT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), (cmp == 2 ? 1 : 0)); + } + } + } +} + +inline int PrimitiveProcessor::compare(int val1, int val2, uint8_t COP, bool lastStage) throw() +{ + switch (COP) + { + case COMPARE_LT: + if (val1 < val2) + return 2; + + if (val1 == val2 && !lastStage) + return 1; + + return 0; + + case COMPARE_LE: + if (val1 < val2) + return 2; + + if (val1 == val2) + return 1; + + return 0; + + case COMPARE_GT: + if (val1 > val2) + return 2; + + if (val1 == val2 && !lastStage) + return 1; + + return 0; + + case COMPARE_GE: + if (val1 > val2) + return 2; + + if (val1 == val2) + return 1; + + return 0; + + case COMPARE_EQ: + if (val1 == val2) + return 1; + + return 0; + + case COMPARE_NE: + if (val1 != val2) + return 2; + + if (!lastStage) + return 1; + + return 0; + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(34); + + colWidth.add(COP); + colWidth.add("compare"); + msg.format(colWidth); + logger.logDebugMessage(msg); + return false; + } +} + +void PrimitiveProcessor::indexWalk_2(const IndexWalkHeader* in, vector* out) throw() +{ + uint16_t bitTest1, bitTest2; + uint8_t* niceBlock; + int blockOffset, bitTestGroupSize, cmp[2], i; + WriteEngine::IdxBitTestEntry* treePtr; + IndexWalkHeader* element; + int convertToSingleOp; + bool lastStage, setState; + + IDXWALK_INIT(); + + if (in->SubBlock == 1 && in->Shift == 0) // assume this is the first lookup step + { + // which happens in the direct pointer block + bitTest1 = in->SearchString[0] >> (in->SSlen - 5); + bitTest2 = in->SearchString[1] >> (in->SSlen - 5); +#ifdef VERBOSE + cerr << " first iteration. SearchString[0]=0x" << hex << in->SearchString[0] << " bittest=0x" + << bitTest1 << " SearchString[1]=0x" << in->SearchString[1] << " bittest=0x" << bitTest2 << dec + << endl; +#endif + + for (i = 0; i < 32; i++) + { + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; + + setState = false; + convertToSingleOp = -1; + cmp[0] = compare(i, bitTest1, in->COP1, false); + cmp[1] = compare(i, bitTest2, in->COP2, false); + + switch (in->BOP) + { + case BOP_OR: + if (cmp[0] == 2 || cmp[1] == 2) + { + setState = true; + goto add2; + } + + // if (cmp[0] == 1 || cmp[1] == 1) + // goto add2; + + /* XXXPAT: clean up this logic */ + + if (cmp[0] == 1 && cmp[1] == 1) + goto add2; + + if (cmp[0] == 1) + { + convertToSingleOp = 0; + ADD_ELEMENT_WITH_CONVERT(i, 5, 0); + goto skip2; + } + else if (cmp[1] == 1) + { + convertToSingleOp = 1; + ADD_ELEMENT_WITH_CONVERT(i, 5, 0); + goto skip2; + } + + break; + + /* XXXPAT: Need to verify the logic for AND. + observations: if control reaches this point, then in the previous iteration the + decision must have been 1, which implies equality for every previous comparison. + + If one of the comparisons returns 0, then there are no entries in this subtree + in the result set and it can stop here. + else, if both of the comparisons return 2, then the whole subtree is in the result set. + else, if only one of the comparisons returns 2, then that comparison will be 2 for + every comparison made down this subtree, and it is equivalent to being + (cmp1 && true). In this case, we repackage the query as a single argument version + (using the comparison operator that was 1) to be processed by indexwalk_1(). + else, the comparisons are both 1, meaning we have equality so far and can't decide on + this subtree yet. + */ + case BOP_AND: + if (cmp[0] == 0 || cmp[1] == 0) + break; + else if (cmp[0] == 2 && cmp[1] == 2) + setState = true; + else if (cmp[0] == 2) + convertToSingleOp = 1; + else if (cmp[1] == 2) + convertToSingleOp = 0; + + goto add2; + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(39); + + colWidth.add(in->BOP); + colWidth.add("indexwalk_2"); + msg.format(colWidth); + logger.logDebugMessage(msg); + return; + } + + continue; + add2: + ADD_ELEMENT_WITH_CONVERT(i, 5, (setState ? 1 : 0)); + skip2:; + } + } + else // the general case + { + GET_GROUP_SIZE(); #ifdef VERBOSE - cerr << " SearchString[0]=0x" << hex << in->SearchString[0] << - " SearchString[1]=0x" << in->SearchString[1] << dec << endl; + cerr << " SearchString[0]=0x" << hex << in->SearchString[0] << " SearchString[1]=0x" + << in->SearchString[1] << dec << endl; #endif + for (i = 0; i < bitTestGroupSize; i++) + { + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; + + setState = false; + lastStage = false; + convertToSingleOp = -1; + + if (treePtr[i].bitCompare == 0) + { + GET_BITTEST(bitTest1, in->SearchString[0], 5); + GET_BITTEST(bitTest2, in->SearchString[1], 5); + } + else + { + GET_BITTEST(bitTest1, in->SearchString[0], 10); + GET_BITTEST(bitTest2, in->SearchString[1], 10); + } + + cmp[0] = compare(treePtr[i].bitTest, bitTest1, in->COP1, lastStage); + cmp[1] = compare(treePtr[i].bitTest, bitTest2, in->COP2, lastStage); + + switch (in->BOP) + { + case BOP_OR: + if (cmp[0] == 2 || cmp[1] == 2) + { + setState = true; + goto add3; + } + + // if (cmp[0] == 1 || cmp[1] == 1) + // goto add3; + + /* XXXPAT: clean up this logic */ + + if (cmp[0] == 1 && cmp[1] == 1) + goto add3; + + if (cmp[0] == 1) + { + convertToSingleOp = 0; + ADD_ELEMENT_WITH_CONVERT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), 0); + goto skip3; + } + else if (cmp[1] == 1) + { + convertToSingleOp = 1; + ADD_ELEMENT_WITH_CONVERT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), 0); + goto skip3; + } + + break; + + /* XXXPAT: Need to verify the logic for AND. + observations: if control reaches this point, then in the previous iteration the + decision must have been 1, which implies equality for every previous comparison. + + If one of the comparisons returns 0, then there are no entries in this subtree + in the result set and it can stop here. + else, if both of the comparisons return 2, then the whole subtree is in the result + set. else, if only one of the comparisons returns 2, then that comparison will be 2 for every + comparison made down this subtree, and it is equivalent to being (cmp1 && true). In this case, we + repackage the query as a single argument version (using the comparison operator that was 1) to be + processed by indexwalk_1(). else, the comparisons are both 1, meaning we have equality so far and + can't decide on this subtree yet. + */ + case BOP_AND: + if (cmp[0] == 0 || cmp[1] == 0) + break; + else if (cmp[0] == 2 && cmp[1] == 2) + setState = true; + else if (cmp[0] == 2) + convertToSingleOp = 1; + else if (cmp[1] == 2) + convertToSingleOp = 0; + + goto add3; + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(39); + + colWidth.add(in->BOP); + colWidth.add("indexWalk_2"); + msg.format(colWidth); + logger.logDebugMessage(msg); + return; + } + + continue; + add3: + ADD_ELEMENT_WITH_CONVERT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), (setState ? 1 : 0)); + skip3:; + } + } +} + +void PrimitiveProcessor::indexWalk_many(const IndexWalkHeader* in, vector* out) throw() +{ + uint16_t bitTest; + uint8_t* niceBlock; + int blockOffset, bitTestGroupSize, i; + WriteEngine::IdxBitTestEntry* treePtr; + IndexWalkHeader* element; + bool lastStage; + vector::const_iterator it; + struct + { + int action; + vector* searchStrings; + } nextIteration[32]; + + IDXWALK_INIT(); + + /* + Here's the high-level algorithm for this function: + 1) Iterate over the bit test tree entries (index is i) + 1a) if it is not a match, set nextIteration[i].action = 0 (searchStrings = NULL) + 1b) if it is a match which determines the whole subtree goes in the result set action = 2 + (searchStrings = NULL) 1c) if it's a match that determines the result is somewhere further down the tree, + set action = 1, stuff the matching search string into nextIteration[i].searchStrings. 2) Iterate over the + nextIteration structures 2a) if action == 0 do nothing 2b) if action == 1, make an intermediate + IndexWalkHeader configured s.t. it makes sense given the # of search strings that match on that subtree + 2c) if action == 2, + make an intermediate IndexWalkHeader with state = 1 so the whole subtree will be + included. 2d) if the current LBID is also the next LBID, recurse at p_IdxWalk() otherwise, add it to the + result set to return. + + An implementation that is definitely simpler and which may be faster + in some circumstances: + 1) if BOP is OR, COP is =, so split the query into one single-op + query for each search string. + 2) if BOP is AND, COP is !=, so grab every indexed value and eliminate + the entries in the argument list. (Need to store the index values somewhere). + *** 3) Generalize the index_many case; get rid of the NVALS=[1,2] cases. + - need to find a way to avoid dynamic mem allocation. + */ + + if (in->SubBlock == 1 && in->Shift == 0) // direct pointer block + { + bitTestGroupSize = 32; +#ifdef VERBOSE + cerr << " first iteration using many search strings" << endl; +#endif + + switch (in->BOP) + { + case BOP_OR: + + // according to the design doc, it's safe to assume COP1 is '=' + for (i = 0; i < bitTestGroupSize; i++) + { + nextIteration[i].action = 0; + nextIteration[i].searchStrings = NULL; + } + + for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) + { + bitTest = *it >> (in->SSlen - 5); +#ifdef VERBOSE + cerr << " search string=0x" << hex << *it << " bittest=0x" << bitTest << dec << endl; +#endif + + if (treePtr[bitTest].fbo == 0 && treePtr[bitTest].sbid == 0 && treePtr[bitTest].entry == 0) + continue; + + if (nextIteration[bitTest].action == 0) + { + nextIteration[bitTest].searchStrings = new vector(); + nextIteration[bitTest].action = 1; + } + + nextIteration[bitTest].searchStrings->push_back(*it); + } + + break; + + case BOP_AND: + + // safe to assume COP1 is '!=' + /* Here's the logic here.... + With a set of 0 search strings, the result set is the entire index. + With a set of 1 search strings, recurse the subtree containing that string, return every other + subtree. So, for every search string, we flag the respective subtree as one that has to be + recursed. + */ + for (i = 0; i < bitTestGroupSize; i++) { + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + nextIteration[i].action = 0; + else + nextIteration[i].action = 2; - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; + nextIteration[i].searchStrings = NULL; + } - setState = false; - lastStage = false; - convertToSingleOp = -1; + for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) + { + bitTest = *it >> (in->SSlen - 5); +#ifdef VERBOSE + cerr << " search string=0x" << hex << *it << " bittest=0x" << bitTest << dec << endl; +#endif + + if (nextIteration[bitTest].action == 2) + { + nextIteration[bitTest].searchStrings = new vector(); + nextIteration[bitTest].action = 1; + } + + nextIteration[bitTest].searchStrings->push_back(*it); + } + + break; + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(39); + + colWidth.add(in->BOP); + msg.format(colWidth); + logger.logDebugMessage(msg); + return; + } + } + else // The general case of being at a node in the middle of the tree + { + GET_GROUP_SIZE(); + + switch (in->BOP) + { + case BOP_OR: + + // COP1 == '=' + for (i = 0; i < bitTestGroupSize; i++) + { + nextIteration[i].action = 0; + nextIteration[i].searchStrings = NULL; + } + + for (i = 0; i < bitTestGroupSize; i++) + { + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; + + for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) + { +#ifdef VERBOSE + cerr << " search string=0x" << hex << *it << dec << endl; +#endif if (treePtr[i].bitCompare == 0) { - GET_BITTEST(bitTest1, in->SearchString[0], 5); - GET_BITTEST(bitTest2, in->SearchString[1], 5); + GET_BITTEST(bitTest, *it, 5); } else { - GET_BITTEST(bitTest1, in->SearchString[0], 10); - GET_BITTEST(bitTest2, in->SearchString[1], 10); + GET_BITTEST(bitTest, *it, 10); } - cmp[0] = compare(treePtr[i].bitTest, bitTest1, in->COP1, lastStage); - cmp[1] = compare(treePtr[i].bitTest, bitTest2, in->COP2, lastStage); - - switch (in->BOP) + if (bitTest == treePtr[i].bitTest) { - case BOP_OR: - if (cmp[0] == 2 || cmp[1] == 2) - { - setState = true; - goto add3; - } + if (nextIteration[i].action == 0) + { + nextIteration[i].searchStrings = new vector(); + nextIteration[i].action = 1; + } -// if (cmp[0] == 1 || cmp[1] == 1) -// goto add3; - - /* XXXPAT: clean up this logic */ - - if (cmp[0] == 1 && cmp[1] == 1) - goto add3; - - if (cmp[0] == 1) - { - convertToSingleOp = 0; - ADD_ELEMENT_WITH_CONVERT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), 0); - goto skip3; - } - else if (cmp[1] == 1) - { - convertToSingleOp = 1; - ADD_ELEMENT_WITH_CONVERT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), 0); - goto skip3; - } - - break; - - /* XXXPAT: Need to verify the logic for AND. - observations: if control reaches this point, then in the previous iteration the - decision must have been 1, which implies equality for every previous comparison. - - If one of the comparisons returns 0, then there are no entries in this subtree - in the result set and it can stop here. - else, if both of the comparisons return 2, then the whole subtree is in the result set. - else, if only one of the comparisons returns 2, then that comparison will be 2 for - every comparison made down this subtree, and it is equivalent to being - (cmp1 && true). In this case, we repackage the query as a single argument version - (using the comparison operator that was 1) to be processed by indexwalk_1(). - else, the comparisons are both 1, meaning we have equality so far and can't decide on - this subtree yet. - */ - case BOP_AND: - if (cmp[0] == 0 || cmp[1] == 0) - break; - else if (cmp[0] == 2 && cmp[1] == 2) - setState = true; - else if (cmp[0] == 2) - convertToSingleOp = 1; - else if (cmp[1] == 2) - convertToSingleOp = 0; - - goto add3; - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(39); - - colWidth.add(in->BOP); - colWidth.add("indexWalk_2"); - msg.format(colWidth); - logger.logDebugMessage(msg); - return; + nextIteration[i].searchStrings->push_back(*it); } + } + } + break; + + case BOP_AND: + + // COP1 == '!=' + for (i = 0; i < bitTestGroupSize; i++) + { + nextIteration[i].action = 2; + nextIteration[i].searchStrings = NULL; + } + + for (i = 0; i < bitTestGroupSize; i++) + { + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + { + nextIteration[i].action = 0; continue; -add3: - ADD_ELEMENT_WITH_CONVERT(i, (treePtr[i].bitCompare == 0 ? 5 : 10), - (setState ? 1 : 0)); -skip3: - ; - } - } -} + } -void PrimitiveProcessor::indexWalk_many(const IndexWalkHeader* in, - vector* out) throw() -{ - uint16_t bitTest; - uint8_t* niceBlock; - int blockOffset, bitTestGroupSize, i; - WriteEngine::IdxBitTestEntry* treePtr; - IndexWalkHeader* element; - bool lastStage; - vector::const_iterator it; - struct - { - int action; - vector* searchStrings; - } nextIteration[32]; - - IDXWALK_INIT(); - - /* - Here's the high-level algorithm for this function: - 1) Iterate over the bit test tree entries (index is i) - 1a) if it is not a match, set nextIteration[i].action = 0 (searchStrings = NULL) - 1b) if it is a match which determines the whole subtree goes in the result set action = 2 (searchStrings = NULL) - 1c) if it's a match that determines the result is somewhere further down the tree, set action = 1, - stuff the matching search string into nextIteration[i].searchStrings. - 2) Iterate over the nextIteration structures - 2a) if action == 0 do nothing - 2b) if action == 1, - make an intermediate IndexWalkHeader configured s.t. it makes sense given the # of search strings - that match on that subtree - 2c) if action == 2, - make an intermediate IndexWalkHeader with state = 1 so the whole subtree will be included. - 2d) if the current LBID is also the next LBID, recurse at p_IdxWalk() - otherwise, add it to the result set to return. - - An implementation that is definitely simpler and which may be faster - in some circumstances: - 1) if BOP is OR, COP is =, so split the query into one single-op - query for each search string. - 2) if BOP is AND, COP is !=, so grab every indexed value and eliminate - the entries in the argument list. (Need to store the index values somewhere). - *** 3) Generalize the index_many case; get rid of the NVALS=[1,2] cases. - - need to find a way to avoid dynamic mem allocation. - */ - - - if (in->SubBlock == 1 && in->Shift == 0) // direct pointer block - { - bitTestGroupSize = 32; + for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) + { #ifdef VERBOSE - cerr << " first iteration using many search strings" << endl; + cerr << " search string=0x" << hex << *it << dec << endl; #endif - switch (in->BOP) - { - case BOP_OR: - - // according to the design doc, it's safe to assume COP1 is '=' - for (i = 0; i < bitTestGroupSize; i++) - { - nextIteration[i].action = 0; - nextIteration[i].searchStrings = NULL; - } - - for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) - { - bitTest = *it >> (in->SSlen - 5); -#ifdef VERBOSE - cerr << " search string=0x" << hex << *it << " bittest=0x" << bitTest << dec << endl; -#endif - - if (treePtr[bitTest].fbo == 0 && treePtr[bitTest].sbid == 0 && treePtr[bitTest].entry == 0) - continue; - - if (nextIteration[bitTest].action == 0) - { - nextIteration[bitTest].searchStrings = new vector(); - nextIteration[bitTest].action = 1; - } - - nextIteration[bitTest].searchStrings->push_back(*it); - } - - break; - - case BOP_AND: - - // safe to assume COP1 is '!=' - /* Here's the logic here.... - With a set of 0 search strings, the result set is the entire index. - With a set of 1 search strings, recurse the subtree containing that string, return every other subtree. - So, for every search string, we flag the respective subtree as one that has to be recursed. - */ - - for (i = 0; i < bitTestGroupSize; i++) - { - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - nextIteration[i].action = 0; - else - nextIteration[i].action = 2; - - nextIteration[i].searchStrings = NULL; - } - - for (it = in->SearchStrings->begin(); - it != in->SearchStrings->end(); - it++) - { - - bitTest = *it >> (in->SSlen - 5); -#ifdef VERBOSE - cerr << " search string=0x" << hex << *it << " bittest=0x" << bitTest << dec << endl; -#endif - - if (nextIteration[bitTest].action == 2) - { - nextIteration[bitTest].searchStrings = new vector(); - nextIteration[bitTest].action = 1; - } - - nextIteration[bitTest].searchStrings->push_back(*it); - } - - break; - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(39); - - colWidth.add(in->BOP); - msg.format(colWidth); - logger.logDebugMessage(msg); - return; - } - } - else // The general case of being at a node in the middle of the tree - { - GET_GROUP_SIZE(); - - switch (in->BOP) - { - case BOP_OR: - - // COP1 == '=' - for (i = 0; i < bitTestGroupSize; i++) - { - nextIteration[i].action = 0; - nextIteration[i].searchStrings = NULL; - } - - for (i = 0; i < bitTestGroupSize; i++) - { - - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; - - for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) - { -#ifdef VERBOSE - cerr << " search string=0x" << hex << *it << dec << endl; -#endif - - if (treePtr[i].bitCompare == 0) - { - GET_BITTEST(bitTest, *it, 5); - } - else - { - GET_BITTEST(bitTest, *it, 10); - } - - if (bitTest == treePtr[i].bitTest) - { - if (nextIteration[i].action == 0) - { - nextIteration[i].searchStrings = new vector(); - nextIteration[i].action = 1; - } - - nextIteration[i].searchStrings->push_back(*it); - } - } - } - - break; - - case BOP_AND: - - // COP1 == '!=' - for (i = 0; i < bitTestGroupSize; i++) - { - nextIteration[i].action = 2; - nextIteration[i].searchStrings = NULL; - } - - for (i = 0; i < bitTestGroupSize; i++) - { - - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - { - nextIteration[i].action = 0; - continue; - } - - for (it = in->SearchStrings->begin(); it != in->SearchStrings->end(); it++) - { -#ifdef VERBOSE - cerr << " search string=0x" << hex << *it << dec << endl; -#endif - - if (treePtr[i].bitCompare == 0) - { - GET_BITTEST(bitTest, *it, 5); - } - else - { - GET_BITTEST(bitTest, *it, 10); - } - - // note: at the last stage, matches are left with action = 2 and NULL searchStrings - if (!lastStage && bitTest == treePtr[i].bitTest) - { - if (nextIteration[i].action == 2) - { - nextIteration[i].searchStrings = new vector(); - nextIteration[i].action = 1; - } - - nextIteration[i].searchStrings->push_back(*it); - } - else if (lastStage && bitTest == treePtr[i].bitTest) - nextIteration[i].action = 0; - } - } - - break; - - default: - MessageLog logger(LoggingID(28)); - logging::Message::Args colWidth; - Message msg(39); - - colWidth.add(in->BOP); - colWidth.add("indexWalk_many"); - msg.format(colWidth); - logger.logDebugMessage(msg); - return; - } - } - - for (i = 0; i < bitTestGroupSize; i++) - { - if (nextIteration[i].action > 0) - { - element = new IndexWalkHeader(); - memcpy(element, in, sizeof(IndexWalkHeader)); - element->ism.Command = INDEX_WALK_RESULTS; - element->Shift += (treePtr[i].bitCompare ? 10 : 5); - element->LBID = treePtr[i].fbo; - element->SubBlock = treePtr[i].sbid; - element->SBEntry = treePtr[i].entry; -#ifdef VERBOSE - cerr << " (inline _many) creating a result from subblock entry " << i << " with Shift=" << (int) element->Shift << - " LBID=" << element->LBID << " Subblock=" << (int)element->SubBlock << " Subblock entry=" << - (int) element->SBEntry << " State=0" << endl; -#endif - - if (nextIteration[i].action == 1) + if (treePtr[i].bitCompare == 0) { - element->NVALS = nextIteration[i].searchStrings->size(); - - if (nextIteration[i].searchStrings->size() > 2) - element->SearchStrings = nextIteration[i].searchStrings; - else if (nextIteration[i].searchStrings->size() == 2) - { - element->SearchString[0] = nextIteration[i].searchStrings->at(0); - element->SearchString[1] = nextIteration[i].searchStrings->at(1); - element->COP2 = element->COP1; - delete nextIteration[i].searchStrings; - } - else // size == 1 - { - element->SearchString[0] = nextIteration[i].searchStrings->at(0); - delete nextIteration[i].searchStrings; - } - } - else // action == 2 - element->State = 1; - - if (element->LBID == in->LBID && element->Shift < element->SSlen) - { -#ifdef VERBOSE - cerr << " recursing..." << endl; -#endif - p_IdxWalk(element, out); - - if (element->State == 0 && element->NVALS > 2) - delete element->SearchStrings; - - delete element; + GET_BITTEST(bitTest, *it, 5); } else - out->push_back(element); + { + GET_BITTEST(bitTest, *it, 10); + } + + // note: at the last stage, matches are left with action = 2 and NULL searchStrings + if (!lastStage && bitTest == treePtr[i].bitTest) + { + if (nextIteration[i].action == 2) + { + nextIteration[i].searchStrings = new vector(); + nextIteration[i].action = 1; + } + + nextIteration[i].searchStrings->push_back(*it); + } + else if (lastStage && bitTest == treePtr[i].bitTest) + nextIteration[i].action = 0; + } } + + break; + + default: + MessageLog logger(LoggingID(28)); + logging::Message::Args colWidth; + Message msg(39); + + colWidth.add(in->BOP); + colWidth.add("indexWalk_many"); + msg.format(colWidth); + logger.logDebugMessage(msg); + return; } -} + } -void PrimitiveProcessor::grabSubTree(const IndexWalkHeader* in, - vector* out) throw() -{ - uint8_t* niceBlock; - int blockOffset, bitTestGroupSize, i; - WriteEngine::IdxBitTestEntry* treePtr; - IndexWalkHeader* element; - - IDXWALK_INIT(); - GET_GROUP_SIZE(); - - for (i = 0; i < bitTestGroupSize; i++) + for (i = 0; i < bitTestGroupSize; i++) + { + if (nextIteration[i].action > 0) { + element = new IndexWalkHeader(); + memcpy(element, in, sizeof(IndexWalkHeader)); + element->ism.Command = INDEX_WALK_RESULTS; + element->Shift += (treePtr[i].bitCompare ? 10 : 5); + element->LBID = treePtr[i].fbo; + element->SubBlock = treePtr[i].sbid; + element->SBEntry = treePtr[i].entry; +#ifdef VERBOSE + cerr << " (inline _many) creating a result from subblock entry " << i + << " with Shift=" << (int)element->Shift << " LBID=" << element->LBID + << " Subblock=" << (int)element->SubBlock << " Subblock entry=" << (int)element->SBEntry + << " State=0" << endl; +#endif - if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) - continue; + if (nextIteration[i].action == 1) + { + element->NVALS = nextIteration[i].searchStrings->size(); - ADD_ELEMENT(i, (treePtr[i].bitCompare ? 10 : 5), 1); + if (nextIteration[i].searchStrings->size() > 2) + element->SearchStrings = nextIteration[i].searchStrings; + else if (nextIteration[i].searchStrings->size() == 2) + { + element->SearchString[0] = nextIteration[i].searchStrings->at(0); + element->SearchString[1] = nextIteration[i].searchStrings->at(1); + element->COP2 = element->COP1; + delete nextIteration[i].searchStrings; + } + else // size == 1 + { + element->SearchString[0] = nextIteration[i].searchStrings->at(0); + delete nextIteration[i].searchStrings; + } + } + else // action == 2 + element->State = 1; + + if (element->LBID == in->LBID && element->Shift < element->SSlen) + { +#ifdef VERBOSE + cerr << " recursing..." << endl; +#endif + p_IdxWalk(element, out); + + if (element->State == 0 && element->NVALS > 2) + delete element->SearchStrings; + + delete element; + } + else + out->push_back(element); } + } } -void PrimitiveProcessor::p_IdxWalk(const IndexWalkHeader* in, - vector* out) throw() +void PrimitiveProcessor::grabSubTree(const IndexWalkHeader* in, vector* out) throw() { + uint8_t* niceBlock; + int blockOffset, bitTestGroupSize, i; + WriteEngine::IdxBitTestEntry* treePtr; + IndexWalkHeader* element; + IDXWALK_INIT(); + GET_GROUP_SIZE(); + + for (i = 0; i < bitTestGroupSize; i++) + { + if (treePtr[i].fbo == 0 && treePtr[i].sbid == 0 && treePtr[i].entry == 0) + continue; + + ADD_ELEMENT(i, (treePtr[i].bitCompare ? 10 : 5), 1); + } +} + +void PrimitiveProcessor::p_IdxWalk(const IndexWalkHeader* in, vector* out) throw() +{ #ifdef VERBOSE - cerr << "p_IdxWalk()" << endl; - cerr << " COP1=" << (int) in->COP1 << endl; - cerr << " COP2=" << (int) in->COP2 << endl; - cerr << " BOP=" << (int) in->BOP << endl; - cerr << " Shift=" << (int) in->Shift << endl; - cerr << " SSlen=" << (int) in->SSlen << endl; - cerr << " LBID=" << (int) in->LBID << endl; - cerr << " Subblock=" << (int) in->SubBlock << endl; - cerr << " SBEntry=" << (int) in->SBEntry << endl; - cerr << " NVALS=" << (int) in->NVALS << endl; + cerr << "p_IdxWalk()" << endl; + cerr << " COP1=" << (int)in->COP1 << endl; + cerr << " COP2=" << (int)in->COP2 << endl; + cerr << " BOP=" << (int)in->BOP << endl; + cerr << " Shift=" << (int)in->Shift << endl; + cerr << " SSlen=" << (int)in->SSlen << endl; + cerr << " LBID=" << (int)in->LBID << endl; + cerr << " Subblock=" << (int)in->SubBlock << endl; + cerr << " SBEntry=" << (int)in->SBEntry << endl; + cerr << " NVALS=" << (int)in->NVALS << endl; #endif #ifdef PRIM_DEBUG - if (in->Shift >= in->SSlen) - throw logic_error("p_IdxWalk: called on a completed search"); + if (in->Shift >= in->SSlen) + throw logic_error("p_IdxWalk: called on a completed search"); #endif - if (in->State == 1) - grabSubTree(in, out); - else - switch (in->NVALS) - { - case 1: - indexWalk_1(in, out); - break; + if (in->State == 1) + grabSubTree(in, out); + else + switch (in->NVALS) + { + case 1: indexWalk_1(in, out); break; - case 2: - indexWalk_2(in, out); - break; + case 2: indexWalk_2(in, out); break; - default: - indexWalk_many(in, out); - break; - } + default: indexWalk_many(in, out); break; + } #ifdef VERBOSE - cerr << "/p_IdxWalk()" << endl; + cerr << "/p_IdxWalk()" << endl; #endif } @@ -966,189 +951,183 @@ void PrimitiveProcessor::p_IdxWalk(const IndexWalkHeader* in, * indicate whether the Index List section is a header (0), sub-block (4), or * a block (5). */ -void PrimitiveProcessor::p_IdxList(const IndexListHeader* rqst, - IndexListHeader* rslt, int mode) +void PrimitiveProcessor::p_IdxList(const IndexListHeader* rqst, IndexListHeader* rslt, int mode) { - uint8_t* listPtr; - IndexListParam* linkList; - IndexListEntry* listEntry, *rsltList, *sizeEntry = 0; - int listOfst, listType, ridCt, i, j, originalRidCt; - unsigned entryNumber, lastEntry = 0; - int subblk_sz = WriteEngine::SUBBLOCK_TOTAL_BYTES; - int entry_sz = WriteEngine::NEXT_PTR_BYTES; + uint8_t* listPtr; + IndexListParam* linkList; + IndexListEntry *listEntry, *rsltList, *sizeEntry = 0; + int listOfst, listType, ridCt, i, j, originalRidCt; + unsigned entryNumber, lastEntry = 0; + int subblk_sz = WriteEngine::SUBBLOCK_TOTAL_BYTES; + int entry_sz = WriteEngine::NEXT_PTR_BYTES; #ifdef VERBOSE - cerr << "p_IdxList()" << endl; + cerr << "p_IdxList()" << endl; #endif - memcpy(rslt, rqst, sizeof(IndexListHeader)); - rslt->ism.Command = INDEX_LIST_RESULTS; - rslt->NVALS = 0; + memcpy(rslt, rqst, sizeof(IndexListHeader)); + rslt->ism.Command = INDEX_LIST_RESULTS; + rslt->NVALS = 0; - listPtr = (uint8_t*)block; - linkList = (IndexListParam*)(rqst + 1); - rsltList = (IndexListEntry*)(rslt + 1); + listPtr = (uint8_t*)block; + linkList = (IndexListParam*)(rqst + 1); + rsltList = (IndexListEntry*)(rslt + 1); - for (i = 0; i < rqst->NVALS; i++) + for (i = 0; i < rqst->NVALS; i++) + { + j = 0; + listOfst = (linkList->sbid * subblk_sz) + (linkList->entry * entry_sz); + entryNumber = linkList->entry; + listEntry = (IndexListEntry*)(listPtr + listOfst); + +#ifdef VERBOSE + cerr << " processing argument number " << i + 1 << endl; + cerr << " type=" << linkList->type << " LBID=" << linkList->fbo << " subblock=" << linkList->sbid + << " entry=" << linkList->entry << endl; +#endif + + listType = linkList->type; + + if (listType == LIST_SIZE) { - j = 0; - listOfst = (linkList->sbid * subblk_sz) + (linkList->entry * entry_sz); - entryNumber = linkList->entry; - listEntry = (IndexListEntry*)(listPtr + listOfst); + if (listEntry->type != LIST_SIZE) + { + MessageLog logger(LoggingID(28)); + Message msg(40); + logger.logDebugMessage(msg); +#ifdef VERBOSE + cerr << "PrimitiveProcessor::p_IdxList: was told to parse a header, but the given pointer does not " + "point to one. sbid=" + << linkList->sbid << " entry=" << linkList->entry << endl; +#endif + throw runtime_error("p_IdxList: not a header"); + } + + ridCt = listEntry->value; + originalRidCt = ridCt; +#ifdef VERBOSE + uint64_t* tmp = (uint64_t*)&listEntry[1]; + cerr << " ridCount=" << ridCt << " key value=0x" << hex << *tmp << dec << endl; +#endif + listEntry += 2; // Skip size and key values + entryNumber += 2; + lastEntry = linkList->entry + 4; + } + + else + { + if (listType == LLP_SUBBLK) + { + sizeEntry = listEntry + WriteEngine::LIST_SUB_LLP_POS; + lastEntry = linkList->entry + 32; + } + else if (listType == LLP_BLK) + { + sizeEntry = listEntry + WriteEngine::LIST_BLOCK_LLP_POS; + lastEntry = linkList->entry + (mode == 0 ? 1024 : 1023); + // ignore the size entry for blocks if mode == 1 + } + else + cerr << "p_IdxList: bad pointer type " << listType << endl; + + ridCt = sizeEntry->ridCt; // XXXPAT: make sure these are the right bits + originalRidCt = ridCt; +#ifdef VERBOSE + cerr << " ridCount for this " << (listType == LLP_SUBBLK ? "subblock: " : "block: ") << ridCt + << endl; +#endif + } + + // while (ridCt > 0 && entryNumber < lastEntry) { + while (entryNumber < lastEntry) + { + switch (listEntry->type) + { + case LLP_SUBBLK: + case LLP_BLK: + + /* the second to last entry of a subblock + can now be a HWM disguised as a continuation ptr. */ + if (mode && entryNumber == static_cast(linkList->entry + 30) && listType == LLP_SUBBLK) + break; #ifdef VERBOSE - cerr << " processing argument number " << i + 1 << endl; - cerr << " type=" << linkList->type << " LBID=" << linkList->fbo << " subblock=" << - linkList->sbid << " entry=" << linkList->entry << endl; + ilptmp = reinterpret_cast(listEntry); + + cerr << " found a continuation pointer: LBID=" << ilptmp->fbo << " subblock=" << ilptmp->sbid + << " subblock entry=" << ilptmp->entry << endl; #endif + *rsltList = *listEntry; + rslt->NVALS++; + rsltList++; + break; - listType = linkList->type; - - if (listType == LIST_SIZE) - { - if (listEntry->type != LIST_SIZE) - { - MessageLog logger(LoggingID(28)); - Message msg(40); - logger.logDebugMessage(msg); -#ifdef VERBOSE - cerr << "PrimitiveProcessor::p_IdxList: was told to parse a header, but the given pointer does not point to one. sbid=" << linkList->sbid << " entry=" << linkList->entry << endl; -#endif - throw runtime_error("p_IdxList: not a header"); - } - - ridCt = listEntry->value; - originalRidCt = ridCt; -#ifdef VERBOSE - uint64_t* tmp = (uint64_t*) &listEntry[1]; - cerr << " ridCount=" << ridCt << " key value=0x" << hex << *tmp << - dec << endl; -#endif - listEntry += 2; // Skip size and key values - entryNumber += 2; - lastEntry = linkList->entry + 4; - } - - else - { - if (listType == LLP_SUBBLK) - { - sizeEntry = listEntry + WriteEngine::LIST_SUB_LLP_POS; - lastEntry = linkList->entry + 32; - } - else if (listType == LLP_BLK) - { - sizeEntry = listEntry + WriteEngine::LIST_BLOCK_LLP_POS; - lastEntry = linkList->entry + (mode == 0 ? 1024 : 1023); - // ignore the size entry for blocks if mode == 1 - } - else - cerr << "p_IdxList: bad pointer type " << listType << endl; - - ridCt = sizeEntry->ridCt; //XXXPAT: make sure these are the right bits - originalRidCt = ridCt; -#ifdef VERBOSE - cerr << " ridCount for this " << (listType == LLP_SUBBLK ? "subblock: " : "block: ") << - ridCt << endl; -#endif - } - -// while (ridCt > 0 && entryNumber < lastEntry) { - while (entryNumber < lastEntry) - { - switch (listEntry->type) - { - case LLP_SUBBLK: - case LLP_BLK: - - /* the second to last entry of a subblock - can now be a HWM disguised as a continuation ptr. */ - if (mode && - entryNumber == static_cast(linkList->entry + 30) && - listType == LLP_SUBBLK) - break; - -#ifdef VERBOSE - ilptmp = reinterpret_cast(listEntry); - - cerr << " found a continuation pointer: LBID=" << ilptmp->fbo << - " subblock=" << ilptmp->sbid << " subblock entry=" << - ilptmp->entry << endl; -#endif - *rsltList = *listEntry; - rslt->NVALS++; - rsltList++; - break; - - /* - #ifdef VERBOSE - cerr << " reached the end of the list" << endl; - if (listEntry != sizeEntry && listType != LIST_SIZE) - cerr << " ERROR: there's a list pointer in the middle of the list" << endl; - else if (listType != LIST_SIZE) - cerr << " Possible error: the pointer clause executed for a subblock or block" << endl; - #endif - endOfList = true; - break; - */ - case RID: + /* + #ifdef VERBOSE + cerr << " reached the end of the list" << endl; + if (listEntry != sizeEntry && listType != LIST_SIZE) + cerr << " ERROR: there's a list pointer in the + middle of the list" << endl; else if (listType != LIST_SIZE) cerr << " Possible error: the pointer + clause executed for a subblock or block" << endl; #endif endOfList = true; break; + */ + case RID: #ifdef VERBOSE // cerr << " returning rid " << listEntry->value << endl; #endif - *rsltList = *listEntry; - rslt->NVALS++; - rsltList++; - ridCt--; - j++; - break; + *rsltList = *listEntry; + rslt->NVALS++; + rsltList++; + ridCt--; + j++; + break; - case LIST_SIZE: - case NOT_IN_USE: - case EMPTY_LIST_PTR: - case EMPTY_PTR: - case PARENT: - break; + case LIST_SIZE: + case NOT_IN_USE: + case EMPTY_LIST_PTR: + case EMPTY_PTR: + case PARENT: break; - default: + default: #ifdef VERBOSE - IndexListParam* tmp = reinterpret_cast(listEntry); - cerr << "PrimitiveProcessor::p_IdxList: invalid list entry type" << endl; - cerr << " Entry contents: type=" << listEntry->type << " value/RID=" << listEntry->value << endl; - cerr << " (if a pointer, fbo=" << tmp->fbo << " subblock=" << tmp->sbid << - " entry=" << tmp->entry << ")" << endl; - cerr << " Location of the entry: lbid=" << linkList->fbo << " subblock=" << - linkList->sbid + entryNumber / 32 << " entry=" << entryNumber % 32 << endl; - cerr << " Input parameter indicates the start of the list is sbid=" << - linkList->sbid << " entry=" << linkList->entry << endl; - cerr << " Type of search: " << - (linkList->type == LIST_SIZE ? "header" : - (linkList->type == LLP_SUBBLK ? "subblock" : - (linkList->type == LLP_BLK ? "block" : "unknown..?"))) << endl; - cerr << " Processed " << entryNumber - linkList->entry << - " entries for this request so far." << endl; - cerr << " Rid count read from the list is " << originalRidCt << endl; + IndexListParam* tmp = reinterpret_cast(listEntry); + cerr << "PrimitiveProcessor::p_IdxList: invalid list entry type" << endl; + cerr << " Entry contents: type=" << listEntry->type << " value/RID=" << listEntry->value << endl; + cerr << " (if a pointer, fbo=" << tmp->fbo << " subblock=" << tmp->sbid << " entry=" << tmp->entry + << ")" << endl; + cerr << " Location of the entry: lbid=" << linkList->fbo + << " subblock=" << linkList->sbid + entryNumber / 32 << " entry=" << entryNumber % 32 << endl; + cerr << " Input parameter indicates the start of the list is sbid=" << linkList->sbid + << " entry=" << linkList->entry << endl; + cerr << " Type of search: " + << (linkList->type == LIST_SIZE ? "header" + : (linkList->type == LLP_SUBBLK + ? "subblock" + : (linkList->type == LLP_BLK ? "block" : "unknown..?"))) + << endl; + cerr << " Processed " << entryNumber - linkList->entry << " entries for this request so far." + << endl; + cerr << " Rid count read from the list is " << originalRidCt << endl; #endif - MessageLog logger(LoggingID(28)); - Message msg(40); - logger.logDebugMessage(msg); - throw runtime_error("Bad index list entry, see stderr"); - return; - } + MessageLog logger(LoggingID(28)); + Message msg(40); + logger.logDebugMessage(msg); + throw runtime_error("Bad index list entry, see stderr"); + return; + } - listEntry++; - entryNumber++; - } - -#ifdef VERBOSE - cerr << " RIDs in result generated from input arg #" << i + 1 << ": " << j << endl; -#endif - - linkList++; + listEntry++; + entryNumber++; } - return; +#ifdef VERBOSE + cerr << " RIDs in result generated from input arg #" << i + 1 << ": " << j << endl; +#endif + + linkList++; + } + + return; } -} +} // namespace primitives // vim:ts=4 sw=4: - diff --git a/primitives/linux-port/pp-scan-unittest.cpp b/primitives/linux-port/pp-scan-unittest.cpp index 58fb53374..0afff0d37 100644 --- a/primitives/linux-port/pp-scan-unittest.cpp +++ b/primitives/linux-port/pp-scan-unittest.cpp @@ -46,5293 +46,5182 @@ int done; void alarm_handler(int sig) { - done = 1; + done = 1; } class PrimTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(PrimTest); - CPPUNIT_TEST_SUITE(PrimTest); + CPPUNIT_TEST(p_IdxWalk_1_eq_1); + CPPUNIT_TEST(p_IdxWalk_1_eq_2); + CPPUNIT_TEST(p_IdxWalk_1_lt_1); + CPPUNIT_TEST(p_IdxWalk_1_lte_1); + CPPUNIT_TEST(p_IdxWalk_1_gt_1); + CPPUNIT_TEST(p_IdxWalk_1_gte_1); + CPPUNIT_TEST(p_IdxWalk_1_neq_1); + CPPUNIT_TEST(p_IdxWalk_2_eq_or_1); + CPPUNIT_TEST(p_IdxWalk_2_range_1); + CPPUNIT_TEST(p_IdxWalk_2_range_2); + CPPUNIT_TEST(p_IdxWalk_2_range_3); + CPPUNIT_TEST(p_IdxWalk_40_eq_or_1); + CPPUNIT_TEST(p_IdxWalk_40_neq_and_1); - CPPUNIT_TEST(p_IdxWalk_1_eq_1); - CPPUNIT_TEST(p_IdxWalk_1_eq_2); - CPPUNIT_TEST(p_IdxWalk_1_lt_1); - CPPUNIT_TEST(p_IdxWalk_1_lte_1); - CPPUNIT_TEST(p_IdxWalk_1_gt_1); - CPPUNIT_TEST(p_IdxWalk_1_gte_1); - CPPUNIT_TEST(p_IdxWalk_1_neq_1); - CPPUNIT_TEST(p_IdxWalk_2_eq_or_1); - CPPUNIT_TEST(p_IdxWalk_2_range_1); - CPPUNIT_TEST(p_IdxWalk_2_range_2); - CPPUNIT_TEST(p_IdxWalk_2_range_3); - CPPUNIT_TEST(p_IdxWalk_40_eq_or_1); - CPPUNIT_TEST(p_IdxWalk_40_neq_and_1); + CPPUNIT_TEST(p_AggregateSignature_1); + CPPUNIT_TEST(p_AggregateSignature_2); - CPPUNIT_TEST(p_AggregateSignature_1); - CPPUNIT_TEST(p_AggregateSignature_2); + CPPUNIT_TEST(p_TokenByScan_1); + CPPUNIT_TEST(p_TokenByScan_2); + CPPUNIT_TEST(p_TokenByScan_3); + CPPUNIT_TEST(p_TokenByScan_gt_1); + CPPUNIT_TEST(p_TokenByScan_gte_1); + CPPUNIT_TEST(p_TokenByScan_lt_1); + CPPUNIT_TEST(p_TokenByScan_lte_1); + CPPUNIT_TEST(p_TokenByScan_neq_1); + CPPUNIT_TEST(p_TokenByScan_range_1); + CPPUNIT_TEST(p_TokenByScan_eq_6); + CPPUNIT_TEST(p_TokenByScan_neq_6); + CPPUNIT_TEST(p_TokenByScan_token_eq_1); + CPPUNIT_TEST(p_TokenByScan_like_1); - CPPUNIT_TEST(p_TokenByScan_1); - CPPUNIT_TEST(p_TokenByScan_2); - CPPUNIT_TEST(p_TokenByScan_3); - CPPUNIT_TEST(p_TokenByScan_gt_1); - CPPUNIT_TEST(p_TokenByScan_gte_1); - CPPUNIT_TEST(p_TokenByScan_lt_1); - CPPUNIT_TEST(p_TokenByScan_lte_1); - CPPUNIT_TEST(p_TokenByScan_neq_1); - CPPUNIT_TEST(p_TokenByScan_range_1); - CPPUNIT_TEST(p_TokenByScan_eq_6); - CPPUNIT_TEST(p_TokenByScan_neq_6); - CPPUNIT_TEST(p_TokenByScan_token_eq_1); - CPPUNIT_TEST(p_TokenByScan_like_1); + CPPUNIT_TEST(p_IdxList_1); + CPPUNIT_TEST(p_IdxList_2); + // whole block tests + CPPUNIT_TEST(p_Col_1); + CPPUNIT_TEST(p_Col_2); + CPPUNIT_TEST(p_Col_3); + CPPUNIT_TEST(p_Col_4); - CPPUNIT_TEST(p_IdxList_1); - CPPUNIT_TEST(p_IdxList_2); -// whole block tests - CPPUNIT_TEST(p_Col_1); - CPPUNIT_TEST(p_Col_2); - CPPUNIT_TEST(p_Col_3); - CPPUNIT_TEST(p_Col_4); + // rid array test + CPPUNIT_TEST(p_Col_5); -// rid array test - CPPUNIT_TEST(p_Col_5); + // whole block range tests + CPPUNIT_TEST(p_Col_6); + CPPUNIT_TEST(p_Col_7); + CPPUNIT_TEST(p_Col_8); -// whole block range tests - CPPUNIT_TEST(p_Col_6); - CPPUNIT_TEST(p_Col_7); - CPPUNIT_TEST(p_Col_8); + // rid array range test + CPPUNIT_TEST(p_Col_9); -// rid array range test - CPPUNIT_TEST(p_Col_9); + // OT_RID output type tests + CPPUNIT_TEST(p_Col_10); -// OT_RID output type tests - CPPUNIT_TEST(p_Col_10); + // OT_BOTH output type tests + CPPUNIT_TEST(p_Col_11); -// OT_BOTH output type tests - CPPUNIT_TEST(p_Col_11); + // 8-bit null value test + CPPUNIT_TEST(p_Col_12); -// 8-bit null value test - CPPUNIT_TEST(p_Col_12); + // alternating NOP, RID pairs on input + CPPUNIT_TEST(p_Col_13); -// alternating NOP, RID pairs on input - CPPUNIT_TEST(p_Col_13); + // double column test + CPPUNIT_TEST(p_Col_double_1); -// double column test - CPPUNIT_TEST(p_Col_double_1); + // float column test + CPPUNIT_TEST(p_Col_float_1); -// float column test - CPPUNIT_TEST(p_Col_float_1); + // negative float column test + CPPUNIT_TEST(p_Col_neg_float_1); -// negative float column test - CPPUNIT_TEST(p_Col_neg_float_1); + // negative double column test + CPPUNIT_TEST(p_Col_neg_double_1); -// negative double column test - CPPUNIT_TEST(p_Col_neg_double_1); + // some ports of TokenByScan tests to validate similar & shared code + CPPUNIT_TEST(p_Dictionary_1); + CPPUNIT_TEST(p_Dictionary_2); + CPPUNIT_TEST(p_Dictionary_3); + CPPUNIT_TEST(p_Dictionary_gt_1); -// some ports of TokenByScan tests to validate similar & shared code - CPPUNIT_TEST(p_Dictionary_1); - CPPUNIT_TEST(p_Dictionary_2); - CPPUNIT_TEST(p_Dictionary_3); - CPPUNIT_TEST(p_Dictionary_gt_1); + // exercise the OT_TOKEN & OT_INPUTARG output code + CPPUNIT_TEST(p_Dictionary_token_1); + CPPUNIT_TEST(p_Dictionary_inputArg_1); -// exercise the OT_TOKEN & OT_INPUTARG output code - CPPUNIT_TEST(p_Dictionary_token_1); - CPPUNIT_TEST(p_Dictionary_inputArg_1); + // add the OT_AGGREGATE output flag + CPPUNIT_TEST(p_Dictionary_token_agg_1); -// add the OT_AGGREGATE output flag - CPPUNIT_TEST(p_Dictionary_token_agg_1); + // restrict the scan to a list of tokens + CPPUNIT_TEST(p_Dictionary_inToken_1); -// restrict the scan to a list of tokens - CPPUNIT_TEST(p_Dictionary_inToken_1); + // test the old GetSignature behavior + // CPPUNIT_TEST(p_Dictionary_oldgetsig_1); -// test the old GetSignature behavior -//CPPUNIT_TEST(p_Dictionary_oldgetsig_1); + // test & benchmark the new LIKE operator + CPPUNIT_TEST(p_Dictionary_like_1); + CPPUNIT_TEST(p_Dictionary_like_2); + CPPUNIT_TEST(p_Dictionary_like_3); + CPPUNIT_TEST(p_Dictionary_like_4); -// test & benchmark the new LIKE operator - CPPUNIT_TEST(p_Dictionary_like_1); - CPPUNIT_TEST(p_Dictionary_like_2); - CPPUNIT_TEST(p_Dictionary_like_3); - CPPUNIT_TEST(p_Dictionary_like_4); + // new LIKE-regexp functionality + CPPUNIT_TEST(p_Dictionary_like_5); // "_NDO%a%" + CPPUNIT_TEST(p_Dictionary_like_6); // "%NIT%ING%D%" + CPPUNIT_TEST(p_Dictionary_like_7); // "UNI%TES" + CPPUNIT_TEST(p_Dictionary_like_8); // "%TH_OP%" -// new LIKE-regexp functionality - CPPUNIT_TEST(p_Dictionary_like_5); // "_NDO%a%" - CPPUNIT_TEST(p_Dictionary_like_6); // "%NIT%ING%D%" - CPPUNIT_TEST(p_Dictionary_like_7); // "UNI%TES" - CPPUNIT_TEST(p_Dictionary_like_8); // "%TH_OP%" + // CPPUNIT_TEST(p_Dictionary_like_prefixbench_1); + // CPPUNIT_TEST(p_Dictionary_like_substrbench_1); -// CPPUNIT_TEST(p_Dictionary_like_prefixbench_1); -// CPPUNIT_TEST(p_Dictionary_like_substrbench_1); - -// binary data type - CPPUNIT_TEST(p_Col_bin_16); -// CPPUNIT_TEST(p_Col_bin_32); - - CPPUNIT_TEST_SUITE_END(); + // binary data type + CPPUNIT_TEST(p_Col_bin_16); + // CPPUNIT_TEST(p_Col_bin_32); -private: -public: + CPPUNIT_TEST_SUITE_END(); - void p_IdxWalk_1_eq_1() + private: + public: + void p_IdxWalk_1_eq_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - // this search should end at FBO 10251 subblock 22 entry 12. - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; - - params = new IndexWalkHeader(); - - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_EQ; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 64; - - pp.setBlockPtr((int*) block); - - while (params->LBID != 10251) - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - CPPUNIT_ASSERT(results.size() == 1); - params = results.at(0); - results.clear(); - } - - delete params; - close(fd); + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_1_lt_1() + // this search should end at FBO 10251 subblock 22 entry 12. + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_EQ; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + while (params->LBID != 10251) { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) - { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; - - params = new IndexWalkHeader(); - - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_LT; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 64; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + pp.p_IdxWalk(params, &results); + delete params; + CPPUNIT_ASSERT(results.size() == 1); + params = results.at(0); + results.clear(); } - void p_IdxWalk_1_lte_1() + delete params; + close(fd); + } + + void p_IdxWalk_1_lt_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; - - params = new IndexWalkHeader(); - - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_LE; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 64; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_1_gt_1() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_LT; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - params = new IndexWalkHeader(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_GT; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 64; + close(fd); + } - pp.setBlockPtr((int*) block); + void p_IdxWalk_1_lte_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); + fd = open(filename.c_str(), O_RDONLY); - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_1_gte_1() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_LE; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - params = new IndexWalkHeader(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_GE; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 64; + close(fd); + } - pp.setBlockPtr((int*) block); + void p_IdxWalk_1_gt_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); + fd = open(filename.c_str(), O_RDONLY); - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_1_neq_1() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_GT; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - params = new IndexWalkHeader(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_NE; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 64; + close(fd); + } - pp.setBlockPtr((int*) block); + void p_IdxWalk_1_gte_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); + fd = open(filename.c_str(), O_RDONLY); - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_1_eq_2() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_GE; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - // this shouldn't match anywhere - searchKey = (24LL << 9) | (3LL << 4) | 1; + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - params = new IndexWalkHeader(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_EQ; - params->NVALS = 1; - params->SubBlock = 1; - params->SSlen = 14; + close(fd); + } - pp.setBlockPtr((int*) block); + void p_IdxWalk_1_neq_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); + fd = open(filename.c_str(), O_RDONLY); - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - CPPUNIT_ASSERT(results.size() == 0); - results.clear(); - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_2_eq_or_1() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_NE; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey[2]; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; - searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 0; + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - params = new IndexWalkHeader(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey[0]; - params->SearchString[1] = searchKey[1]; - params->COP1 = COMPARE_EQ; - params->COP2 = COMPARE_EQ; - params->BOP = BOP_OR; - params->NVALS = 2; - params->SubBlock = 1; - params->SSlen = 64; + close(fd); + } - pp.setBlockPtr((int*) block); + void p_IdxWalk_1_eq_2() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); + fd = open(filename.c_str(), O_RDONLY); - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - CPPUNIT_ASSERT(results.size() == 1); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } -// this test singles out the same entry we've used above - void p_IdxWalk_2_range_1() + // this shouldn't match anywhere + searchKey = (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_EQ; + params->NVALS = 1; + params->SubBlock = 1; + params->SSlen = 14; + + pp.setBlockPtr((int*)block); + + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); + + if (err <= 0) { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey[2]; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 2; - searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 0; - - params = new IndexWalkHeader(); - - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey[0]; - params->SearchString[1] = searchKey[1]; - params->COP1 = COMPARE_LT; - params->COP2 = COMPARE_GT; - params->BOP = BOP_AND; - params->NVALS = 2; - params->SubBlock = 1; - params->SSlen = 64; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - CPPUNIT_ASSERT(results.size() == 1); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); } -// this test singles out a contiguous chunk of 3 entries - void p_IdxWalk_2_range_2() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey[2]; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 3; - searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; - - params = new IndexWalkHeader(); - - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey[0]; - params->SearchString[1] = searchKey[1]; - params->COP1 = COMPARE_LE; - params->COP2 = COMPARE_GE; - params->BOP = BOP_AND; - params->NVALS = 2; - params->SubBlock = 1; - params->SSlen = 64; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); } -// this test excludes all entries. - void p_IdxWalk_2_range_3() + pp.p_IdxWalk(params, &results); + delete params; + CPPUNIT_ASSERT(results.size() == 0); + results.clear(); + + close(fd); + } + + void p_IdxWalk_2_eq_or_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey[2]; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey[2]; - int fd; - uint32_t err; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 3; - searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; - - params = new IndexWalkHeader(); - - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey[0]; - params->SearchString[1] = searchKey[1]; - params->COP1 = COMPARE_GE; - params->COP2 = COMPARE_LE; - params->BOP = BOP_AND; - params->NVALS = 2; - params->SubBlock = 1; - params->SSlen = 64; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - CPPUNIT_ASSERT(results.size() == 0); - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_40_eq_or_1() + searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 0; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey[0]; + params->SearchString[1] = searchKey[1]; + params->COP1 = COMPARE_EQ; + params->COP2 = COMPARE_EQ; + params->BOP = BOP_OR; + params->NVALS = 2; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err, i; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; - vector* vTmp; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + CPPUNIT_ASSERT(results.size() == 1); - params = new IndexWalkHeader(); + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_EQ; - params->BOP = BOP_OR; - params->NVALS = 40; - params->SubBlock = 1; - params->SSlen = 64; - vTmp = new vector(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - for (i = 0; i < 40; i++) - vTmp->push_back(searchKey + i); + close(fd); + } - params->SearchStrings = vTmp; + // this test singles out the same entry we've used above + void p_IdxWalk_2_range_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey[2]; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - pp.setBlockPtr((int*) block); + fd = open(filename.c_str(), O_RDONLY); - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - - if (params->NVALS > 2 && params->State == 0) - delete params->SearchStrings; - - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_IdxWalk_40_neq_and_1() + searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 2; + searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 0; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey[0]; + params->SearchString[1] = searchKey[1]; + params->COP1 = COMPARE_LT; + params->COP2 = COMPARE_GT; + params->BOP = BOP_AND; + params->NVALS = 2; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - char block[BLOCK_SIZE]; - u_int64_t searchKey; - int fd; - uint32_t err, i; - string filename("FILE_990.dat"); - IndexWalkHeader* params; - vector results; - vector::iterator it; - vector* vTmp; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - cerr << "p_IdxWalk: skipping this test; needs the index tree file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | - (6LL << 39) | (29LL << 34) | (10LL << 29) | (19LL << 24) | - (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + CPPUNIT_ASSERT(results.size() == 1); - params = new IndexWalkHeader(); + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - memset(params, 0, sizeof(IndexWalkHeader)); - params->SearchString[0] = searchKey; - params->COP1 = COMPARE_NE; - params->BOP = BOP_AND; - params->NVALS = 40; - params->SubBlock = 1; - params->SSlen = 64; - vTmp = new vector(); + for (it = results.begin(); it != results.end(); it++) + delete *it; - for (i = 0; i < 40; i++) - vTmp->push_back(searchKey + i); + close(fd); + } - params->SearchStrings = vTmp; + // this test singles out a contiguous chunk of 3 entries + void p_IdxWalk_2_range_2() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey[2]; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - pp.setBlockPtr((int*) block); + fd = open(filename.c_str(), O_RDONLY); - do - { - lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxWalk(params, &results); - - if (params->NVALS > 2 && params->State == 0) - delete params->SearchStrings; - - delete params; - params = NULL; - - for (it = results.begin(); it != results.end(); it++) - if ((*it)->Shift < (*it)->SSlen) - { - params = *it; - results.erase(it); - break; - } - } - while (params != NULL); - - // No automatic way to verify the results yet. Need a field that says what - // value was matched so we can verify they're all < the search key. - - for (it = results.begin(); it != results.end(); it++) - delete *it; - - close(fd); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } -// this test scans the entire block - void p_AggregateSignature_1() + searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 3; + searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey[0]; + params->SearchString[1] = searchKey[1]; + params->COP1 = COMPARE_LE; + params->COP2 = COMPARE_GE; + params->BOP = BOP_AND; + params->NVALS = 2; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - AggregateSignatureRequestHeader* cmd; - AggregateSignatureResultHeader* results; - DataValue* dvPtr; - char minmax[BLOCK_SIZE]; - int fd; - uint32_t err; + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - fd = open(filename.c_str(), O_RDONLY); + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } - if (fd < 0) + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - perror("open"); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - err = read(fd, block, BLOCK_SIZE); + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - if (err < 0) - { - perror("read"); - return; - } + for (it = results.begin(); it != results.end(); it++) + delete *it; - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } + close(fd); + } - close(fd); + // this test excludes all entries. + void p_IdxWalk_2_range_3() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey[2]; + int fd; + uint32_t err; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - cmd->NVALS = 0; + fd = open(filename.c_str(), O_RDONLY); - pp.setBlockPtr((int*) block); - pp.p_AggregateSignature(cmd, results, BLOCK_SIZE, &err); - dvPtr = reinterpret_cast(&output[sizeof(AggregateSignatureResultHeader)]); - memcpy(minmax, dvPtr->data, dvPtr->len); - minmax[dvPtr->len] = '\0'; - CPPUNIT_ASSERT(results->Count == 50); - CPPUNIT_ASSERT(strcmp(minmax, "ALGERIA") == 0); -// cerr << "count is " << results->Count << endl; -// cerr << "min is " << minmax << endl; - dvPtr = reinterpret_cast - (&output[sizeof(AggregateSignatureResultHeader) + dvPtr->len + sizeof(DataValue)]); - memcpy(minmax, dvPtr->data, dvPtr->len); - minmax[dvPtr->len] = '\0'; - CPPUNIT_ASSERT(strcmp(minmax, "XUSSIA") == 0); -// cerr << "max is " << minmax << endl; + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } -// this test scans a set of tokens - void p_AggregateSignature_2() + searchKey[0] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 3; + searchKey[1] = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey[0]; + params->SearchString[1] = searchKey[1]; + params->COP1 = COMPARE_GE; + params->COP2 = COMPARE_LE; + params->BOP = BOP_AND; + params->NVALS = 2; + params->SubBlock = 1; + params->SSlen = 64; + + pp.setBlockPtr((int*)block); + + do { + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - AggregateSignatureRequestHeader* cmd; - AggregateSignatureResultHeader* results; - DataValue* dvPtr; - char minmax[BLOCK_SIZE]; - int fd; - uint32_t err; + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - fd = open(filename.c_str(), O_RDONLY); + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } - if (fd < 0) + pp.p_IdxWalk(params, &results); + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - perror("open"); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - err = read(fd, block, BLOCK_SIZE); + CPPUNIT_ASSERT(results.size() == 0); + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - if (err < 0) - { - perror("read"); - return; - } + for (it = results.begin(); it != results.end(); it++) + delete *it; - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } + close(fd); + } - close(fd); + void p_IdxWalk_40_eq_or_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err, i; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; + vector* vTmp; - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - cmd->NVALS = 4; - // JORDAN - cmd->tokens[0].LBID = 0; - cmd->tokens[0].offset = 7928; - cmd->tokens[0].len = 6; - // XHINA - cmd->tokens[1].LBID = 0; - cmd->tokens[1].offset = 8074; - cmd->tokens[1].len = 5; - // ARGENTINA - cmd->tokens[2].LBID = 0; - cmd->tokens[2].offset = 7999; - cmd->tokens[2].len = 9; - // UNITED STATES - cmd->tokens[3].LBID = 0; - cmd->tokens[3].offset = 7838; - cmd->tokens[3].len = 13; + fd = open(filename.c_str(), O_RDONLY); - pp.setBlockPtr((int*) block); - pp.p_AggregateSignature(cmd, results, BLOCK_SIZE, &err); - - dvPtr = reinterpret_cast(&output[sizeof(AggregateSignatureResultHeader)]); - memcpy(minmax, dvPtr->data, dvPtr->len); - minmax[dvPtr->len] = '\0'; - CPPUNIT_ASSERT(results->Count == 4); - CPPUNIT_ASSERT(strcmp(minmax, "ARGENTINA") == 0); -// cerr << "count is " << results->Count << endl; -// cerr << "min is " << minmax << endl; - dvPtr = reinterpret_cast - (&output[sizeof(AggregateSignatureResultHeader) + dvPtr->len + sizeof(DataValue)]); - memcpy(minmax, dvPtr->data, dvPtr->len); - minmax[dvPtr->len] = '\0'; - CPPUNIT_ASSERT(strcmp(minmax, "XHINA") == 0); -// cerr << "max is " << minmax << endl; + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_TokenByScan_1() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_EQ; + params->BOP = BOP_OR; + params->NVALS = 40; + params->SubBlock = 1; + params->SSlen = 64; + vTmp = new vector(); + + for (i = 0; i < 40; i++) + vTmp->push_back(searchKey + i); + + params->SearchStrings = vTmp; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd; - uint32_t err; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + + if (params->NVALS > 2 && params->State == 0) + delete params->SearchStrings; + + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - perror("open"); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - err = read(fd, block, BLOCK_SIZE); + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - if (err < 0) - { - perror("read"); - return; - } + for (it = results.begin(); it != results.end(); it++) + delete *it; - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } + close(fd); + } - close(fd); + void p_IdxWalk_40_neq_and_1() + { + PrimitiveProcessor pp; + char block[BLOCK_SIZE]; + u_int64_t searchKey; + int fd; + uint32_t err, i; + string filename("FILE_990.dat"); + IndexWalkHeader* params; + vector results; + vector::iterator it; + vector* vTmp; - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); + fd = open(filename.c_str(), O_RDONLY); - cmd->NVALS = 1; - cmd->COP1 = COMPARE_EQ; - cmd->OutputType = OT_DATAVALUE; - args->len = 5; - strncpy(args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - - args = reinterpret_cast(&results[1]); - args->data[args->len] = '\0'; // not reusable in tests with multiple matches. - CPPUNIT_ASSERT(results->NVALS == 1); - CPPUNIT_ASSERT(args->len == 5); - CPPUNIT_ASSERT(strncmp(args->data, "XHINA", 5) == 0); + if (fd < 0) + { + cerr << "p_IdxWalk: skipping this test; needs the index tree file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; } - void p_TokenByScan_2() + searchKey = (12LL << 59) | (21LL << 54) | (17LL << 49) | (22LL << 44) | (6LL << 39) | (29LL << 34) | + (10LL << 29) | (19LL << 24) | (6LL << 19) | (0LL << 14) | (24LL << 9) | (3LL << 4) | 1; + + params = new IndexWalkHeader(); + + memset(params, 0, sizeof(IndexWalkHeader)); + params->SearchString[0] = searchKey; + params->COP1 = COMPARE_NE; + params->BOP = BOP_AND; + params->NVALS = 40; + params->SubBlock = 1; + params->SSlen = 64; + vTmp = new vector(); + + for (i = 0; i < 40; i++) + vTmp->push_back(searchKey + i); + + params->SearchStrings = vTmp; + + pp.setBlockPtr((int*)block); + + do { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd; - uint32_t err; + lseek(fd, params->LBID * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); - fd = open(filename.c_str(), O_RDONLY); + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } - if (fd < 0) + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxWalk(params, &results); + + if (params->NVALS > 2 && params->State == 0) + delete params->SearchStrings; + + delete params; + params = NULL; + + for (it = results.begin(); it != results.end(); it++) + if ((*it)->Shift < (*it)->SSlen) { - perror("open"); - return; + params = *it; + results.erase(it); + break; } + } while (params != NULL); - err = read(fd, block, BLOCK_SIZE); + // No automatic way to verify the results yet. Need a field that says what + // value was matched so we can verify they're all < the search key. - if (err < 0) - { - perror("read"); - return; - } + for (it = results.begin(); it != results.end(); it++) + delete *it; - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } + close(fd); + } - close(fd); + // this test scans the entire block + void p_AggregateSignature_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + AggregateSignatureRequestHeader* cmd; + AggregateSignatureResultHeader* results; + DataValue* dvPtr; + char minmax[BLOCK_SIZE]; + int fd; + uint32_t err; - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); + fd = open(filename.c_str(), O_RDONLY); - cmd->NVALS = 2; - cmd->BOP = BOP_OR; - cmd->COP1 = COMPARE_EQ; - cmd->COP2 = COMPARE_EQ; - cmd->OutputType = OT_DATAVALUE; - args->len = 13; - strncpy(args->data, "UNITED STATES", 13); - - args = reinterpret_cast(&input[sizeof(TokenByScanRequestHeader) + args->len + - sizeof(DataValue)]); - args->len = 5; - strncpy(args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - - args = reinterpret_cast(&results[1]); - memcpy(tmp, args->data, args->len); - tmp[args->len] = '\0'; - CPPUNIT_ASSERT(results->NVALS == 2); -// cout << "len is " << args->len << endl; -// cout << "data is " << tmp << endl; - - args = reinterpret_cast(&output[sizeof(TokenByScanResultHeader) + args->len + - sizeof(DataValue)]); - memcpy(tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << "len is " << args->len << endl; -// cout << "data is " << tmp << endl; + if (fd < 0) + { + perror("open"); + return; } - void p_TokenByScan_3() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 0; - cmd->COP1 = COMPARE_EQ; - cmd->OutputType = OT_DATAVALUE; - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - - argsOffset = sizeof(TokenByScanResultHeader); -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 50); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } + perror("read"); + return; } - - void p_TokenByScan_gt_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_GT; - cmd->OutputType = OT_DATAVALUE; - args->len = 7; - strncpy(args->data, "GERMANY", 7); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 41); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_TokenByScan_gte_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + cmd->NVALS = 0; + + pp.setBlockPtr((int*)block); + pp.p_AggregateSignature(cmd, results, BLOCK_SIZE, &err); + dvPtr = reinterpret_cast(&output[sizeof(AggregateSignatureResultHeader)]); + memcpy(minmax, dvPtr->data, dvPtr->len); + minmax[dvPtr->len] = '\0'; + CPPUNIT_ASSERT(results->Count == 50); + CPPUNIT_ASSERT(strcmp(minmax, "ALGERIA") == 0); + // cerr << "count is " << results->Count << endl; + // cerr << "min is " << minmax << endl; + dvPtr = reinterpret_cast( + &output[sizeof(AggregateSignatureResultHeader) + dvPtr->len + sizeof(DataValue)]); + memcpy(minmax, dvPtr->data, dvPtr->len); + minmax[dvPtr->len] = '\0'; + CPPUNIT_ASSERT(strcmp(minmax, "XUSSIA") == 0); + // cerr << "max is " << minmax << endl; + } + + // this test scans a set of tokens + void p_AggregateSignature_2() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + AggregateSignatureRequestHeader* cmd; + AggregateSignatureResultHeader* results; + DataValue* dvPtr; + char minmax[BLOCK_SIZE]; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_GE; - cmd->OutputType = OT_DATAVALUE; - args->len = 7; - strncpy(args->data, "GERMANY", 7); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 42); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + perror("open"); + return; } - void p_TokenByScan_lt_1() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_LT; - cmd->OutputType = OT_DATAVALUE; - args->len = 7; - strncpy(args->data, "GERMANY", 7); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 8); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + perror("read"); + return; } - void p_TokenByScan_lte_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_LE; - cmd->OutputType = OT_DATAVALUE; - args->len = 7; - strncpy(args->data, "GERMANY", 7); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 9); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_TokenByScan_neq_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + cmd->NVALS = 4; + // JORDAN + cmd->tokens[0].LBID = 0; + cmd->tokens[0].offset = 7928; + cmd->tokens[0].len = 6; + // XHINA + cmd->tokens[1].LBID = 0; + cmd->tokens[1].offset = 8074; + cmd->tokens[1].len = 5; + // ARGENTINA + cmd->tokens[2].LBID = 0; + cmd->tokens[2].offset = 7999; + cmd->tokens[2].len = 9; + // UNITED STATES + cmd->tokens[3].LBID = 0; + cmd->tokens[3].offset = 7838; + cmd->tokens[3].len = 13; + + pp.setBlockPtr((int*)block); + pp.p_AggregateSignature(cmd, results, BLOCK_SIZE, &err); + + dvPtr = reinterpret_cast(&output[sizeof(AggregateSignatureResultHeader)]); + memcpy(minmax, dvPtr->data, dvPtr->len); + minmax[dvPtr->len] = '\0'; + CPPUNIT_ASSERT(results->Count == 4); + CPPUNIT_ASSERT(strcmp(minmax, "ARGENTINA") == 0); + // cerr << "count is " << results->Count << endl; + // cerr << "min is " << minmax << endl; + dvPtr = reinterpret_cast( + &output[sizeof(AggregateSignatureResultHeader) + dvPtr->len + sizeof(DataValue)]); + memcpy(minmax, dvPtr->data, dvPtr->len); + minmax[dvPtr->len] = '\0'; + CPPUNIT_ASSERT(strcmp(minmax, "XHINA") == 0); + // cerr << "max is " << minmax << endl; + } + + void p_TokenByScan_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_NE; - cmd->OutputType = OT_DATAVALUE; - args->len = 7; - strncpy(args->data, "GERMANY", 7); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 49); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + perror("open"); + return; } - void p_TokenByScan_range_1() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - - argsOffset = sizeof(TokenByScanRequestHeader); - args = reinterpret_cast(&input[argsOffset]); - - cmd->NVALS = 2; - cmd->COP1 = COMPARE_LT; - cmd->COP2 = COMPARE_GT; - cmd->BOP = BOP_AND; - cmd->OutputType = OT_DATAVALUE; - args->len = 5; - strncpy(args->data, "KENYA", 5); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 6; - strncpy(args->data, "BRAZIL", 6); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 12); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + perror("read"); + return; } - void p_TokenByScan_eq_6() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - - argsOffset = sizeof(TokenByScanRequestHeader); - args = reinterpret_cast(&input[argsOffset]); - - cmd->NVALS = 6; - cmd->COP1 = COMPARE_EQ; - cmd->BOP = BOP_OR; - cmd->OutputType = OT_DATAVALUE; - args->len = 5; - strncpy(args->data, "KENYA", 5); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 6; - strncpy(args->data, "BRAZIL", 6); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 8; - strncpy(args->data, "ETHIOPIA", 8); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 6; - strncpy(args->data, "CANADA", 6); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 13; - strncpy(args->data, "UNITED_STATES", 13); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 13; - strncpy(args->data, "UNITED STATES", 13); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 5); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_TokenByScan_neq_6() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_EQ; + cmd->OutputType = OT_DATAVALUE; + args->len = 5; + strncpy(args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + args = reinterpret_cast(&results[1]); + args->data[args->len] = '\0'; // not reusable in tests with multiple matches. + CPPUNIT_ASSERT(results->NVALS == 1); + CPPUNIT_ASSERT(args->len == 5); + CPPUNIT_ASSERT(strncmp(args->data, "XHINA", 5) == 0); + } + + void p_TokenByScan_2() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - - argsOffset = sizeof(TokenByScanRequestHeader); - args = reinterpret_cast(&input[argsOffset]); - - cmd->NVALS = 6; - cmd->COP1 = COMPARE_NE; - cmd->BOP = BOP_AND; - cmd->OutputType = OT_DATAVALUE; - args->len = 5; - strncpy(args->data, "KENYA", 5); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 6; - strncpy(args->data, "BRAZIL", 6); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 8; - strncpy(args->data, "ETHIOPIA", 8); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 6; - strncpy(args->data, "CANADA", 6); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 13; - strncpy(args->data, "UNITED_STATES", 13); - - argsOffset += args->len + sizeof(DataValue); - args = reinterpret_cast(&input[argsOffset]); - args->len = 13; - strncpy(args->data, "UNITED STATES", 13); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 45); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + perror("open"); + return; } - void p_TokenByScan_like_1() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - int fd, argsOffset, i; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - - argsOffset = sizeof(TokenByScanRequestHeader); - args = reinterpret_cast(&input[argsOffset]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_LIKE; - cmd->BOP = BOP_AND; - cmd->OutputType = OT_DATAVALUE; - args->len = 4; - strncpy(args->data, "%NYA", 4); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 2); - - argsOffset = sizeof(TokenByScanResultHeader); - - for (i = 0; i < results->NVALS; i++) - { - args = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, args->data, args->len); - tmp[args->len] = '\0'; -// cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + args->len; - } - + perror("read"); + return; } - void p_TokenByScan_token_eq_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - TokenByScanRequestHeader* cmd; - TokenByScanResultHeader* results; - DataValue* args; - PrimToken* result; - int fd; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NVALS = 1; - cmd->COP1 = COMPARE_EQ; - cmd->OutputType = OT_TOKEN; - cmd->LBID = 13; // arbitrary # - args->len = 5; - strncpy(args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_TokenByScan(cmd, results, BLOCK_SIZE); - - result = reinterpret_cast(&results[1]); - CPPUNIT_ASSERT(results->NVALS == 1); - CPPUNIT_ASSERT(result->len == 5); - CPPUNIT_ASSERT(result->offset == 19); - CPPUNIT_ASSERT(result->LBID == 13); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_IdxList_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 2; + cmd->BOP = BOP_OR; + cmd->COP1 = COMPARE_EQ; + cmd->COP2 = COMPARE_EQ; + cmd->OutputType = OT_DATAVALUE; + args->len = 13; + strncpy(args->data, "UNITED STATES", 13); + + args = reinterpret_cast( + &input[sizeof(TokenByScanRequestHeader) + args->len + sizeof(DataValue)]); + args->len = 5; + strncpy(args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + args = reinterpret_cast(&results[1]); + memcpy(tmp, args->data, args->len); + tmp[args->len] = '\0'; + CPPUNIT_ASSERT(results->NVALS == 2); + // cout << "len is " << args->len << endl; + // cout << "data is " << tmp << endl; + + args = reinterpret_cast( + &output[sizeof(TokenByScanResultHeader) + args->len + sizeof(DataValue)]); + memcpy(tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << "len is " << args->len << endl; + // cout << "data is " << tmp << endl; + } + + void p_TokenByScan_3() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - - // there should be a list header (corresponds to the search value we used in - // p_IdxWalk) at FBO 10251 subblock 22 entry 12. The first LBID of the - // index list file is 10000 - - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], input[BLOCK_SIZE], output[BLOCK_SIZE]; - string filename("FILE_991.dat"); - IndexListHeader* hdr, *resultHdr; - IndexListParam* params; - IndexListEntry* results; - vector bigResults; - vector::iterator it; - int fd; - uint32_t err, i; - bool continuationPtr; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxList: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - hdr = reinterpret_cast(input); - params = reinterpret_cast(&input[sizeof(IndexListHeader)]); - resultHdr = reinterpret_cast(output); - results = reinterpret_cast(&output[sizeof(IndexListHeader)]); - - memset(hdr, 0, sizeof(IndexListHeader)); - hdr->NVALS = 1; - - params->fbo = 11; - params->sbid = 22; - params->entry = 12; - params->type = LIST_SIZE; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->fbo * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxList(hdr, resultHdr, 0); - - // scan for a continuation pointer - continuationPtr = false; - - for (i = 0; i < resultHdr->NVALS; i++) - { - if (results[i].type == LLP_SUBBLK || results[i].type == LLP_BLK) - { - // the header for the next iteration can stay the same, the - // argument has to be the LLP entry. - memcpy(params, &results[i], sizeof(IndexListEntry)); - CPPUNIT_ASSERT(params->fbo >= 20480); - CPPUNIT_ASSERT(params->fbo < 30720); - params->fbo -= 20480; // this happens to be the base LBID for the test data - continuationPtr = true; - } - else - { - CPPUNIT_ASSERT(results[i].type == RID); - bigResults.push_back(results[i]); - } - } - - /* - // this clause doesn't get used in this test b/c there is only one RID - if (results->type == LLP_SUBBLK || results->type == LLP_BLK) { - CPPUNIT_ASSERT(resultHdr->NVALS > 1); - - // the header for the next iteration can stay the same, the - // argument has to be the LLP entry. - memcpy(params, results, sizeof(IndexListEntry)); - // store the results returned so far - for (i = 1; i < resultHdr->NVALS; i++) - bigResults.push_back(results[i]); - } - else - for (i = 0; i < resultHdr->NVALS; i++) - bigResults.push_back(results[i]); - */ - } - while (continuationPtr); - - CPPUNIT_ASSERT(bigResults.size() == 1); - -// cout << endl << "RID count: " << bigResults.size() << endl; - for (i = 1, it = bigResults.begin(); it != bigResults.end(); it++, i++) - { -// cout << " " << i << ": type=" << (*it).type << " rid=" << (*it).value << endl; - CPPUNIT_ASSERT((*it).type == RID); - CPPUNIT_ASSERT((*it).value == 100); - } + perror("open"); + return; } - void p_IdxList_2() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - - // there should be a list header (corresponds to the search value we used in - // p_IdxWalk) at FBO 10251 subblock 22 entry 12. The first LBID of the - // index list file is 10000 - - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], input[BLOCK_SIZE], output[BLOCK_SIZE]; - string filename("FILE233.cdf"); - IndexListHeader* hdr, *resultHdr; - IndexListParam* params; - IndexListEntry* results; - vector bigResults; - vector::iterator it; - int fd; - uint32_t err, i; - bool continuationPtr; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_IdxList: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_IdxWalk: file not found..."); - return; - } - - hdr = reinterpret_cast(input); - params = reinterpret_cast(&input[sizeof(IndexListHeader)]); - resultHdr = reinterpret_cast(output); - results = reinterpret_cast(&output[sizeof(IndexListHeader)]); - - memset(hdr, 0, sizeof(IndexListHeader)); - hdr->NVALS = 1; - - params->fbo = 11; - params->sbid = 10; - params->entry = 28; - params->type = LIST_SIZE; - - pp.setBlockPtr((int*) block); - - do - { - lseek(fd, params->fbo * BLOCK_SIZE, SEEK_SET); - err = read(fd, block, BLOCK_SIZE); - - if (err <= 0) - { - cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; - throw runtime_error("p_IdxWalk: Couldn't read the file"); - } - - if (err != BLOCK_SIZE) - { - cerr << "p_IdxWalk: could not read a whole block" << endl; - throw runtime_error("p_IdxWalk: could not read a whole block"); - } - - pp.p_IdxList(hdr, resultHdr, 0); - - // scan for a continuation pointer - continuationPtr = false; - - for (i = 0; i < resultHdr->NVALS; i++) - { - if (results[i].type == LLP_SUBBLK || results[i].type == LLP_BLK) - { - // the header for the next iteration can stay the same, the - // argument has to be the LLP entry. - memcpy(params, &results[i], sizeof(IndexListEntry)); - CPPUNIT_ASSERT(params->fbo >= 20480); - CPPUNIT_ASSERT(params->fbo < 30720); - params->fbo -= 20480; // this happens to be the base LBID for the test data - continuationPtr = true; - } - else - { - CPPUNIT_ASSERT(results[i].type == RID); - bigResults.push_back(results[i]); - } - } - - /* - if (results->type == LLP_SUBBLK || results->type == LLP_BLK) { - CPPUNIT_ASSERT(resultHdr->NVALS > 1); - - // the header for the next iteration can stay the same, the - // argument has to be the LLP entry. - memcpy(params, results, sizeof(IndexListEntry)); - CPPUNIT_ASSERT(params->fbo >= 20480); - CPPUNIT_ASSERT(params->fbo < 30720); - params->fbo -= 20480; // this happens to be the base LBID for the test data - - // store the results returned so far - for (i = 1; i < resultHdr->NVALS; i++) - bigResults.push_back(results[i]); - } - else - for (i = 0; i < resultHdr->NVALS; i++) - bigResults.push_back(results[i]); - */ - } - while (continuationPtr); - - CPPUNIT_ASSERT(bigResults.size() == 8901); - -// cout << endl << "RID count: " << bigResults.size() << endl; - for (i = 1, it = bigResults.begin(); it != bigResults.end(); it++, i++) - { -// cout << " " << i << ": type=" << (*it).type << " rid=" << (*it).value << endl; - CPPUNIT_ASSERT((*it).type == RID); - CPPUNIT_ASSERT((*it).value == (uint32_t) i - 1); - } + perror("read"); + return; } - void p_Col_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - uint8_t* results; - int fd; - uint32_t i, written; - string filename("col1block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_1: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_1: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_1: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_1: could not read a whole block" << endl; - throw runtime_error("p_Col_1: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 1; - in->DataType = CalpontSystemCatalog::CHAR; - in->OutputType = OT_DATAVALUE; - in->NOPS = 0; - in->NVALS = 0; - in->InputFlags = 0; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = &output[sizeof(ColResultHeader)]; -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 8160); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == i % 255); - } - - close(fd); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Col_2() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 0; + cmd->COP1 = COMPARE_EQ; + cmd->OutputType = OT_DATAVALUE; + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + argsOffset = sizeof(TokenByScanResultHeader); + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 50); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - uint16_t* results; - uint32_t written, i; - int fd; - string filename("col2block.cdf"); + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_gt_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - cerr << "p_Col_2: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } + fd = open(filename.c_str(), O_RDONLY); - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_2: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_2: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_2: could not read a whole block" << endl; - throw runtime_error("p_Col_2: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 2; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 0; - in->NVALS = 0; - in->InputFlags = 0; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 4096); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == i); - } - - close(fd); + if (fd < 0) + { + perror("open"); + return; } - void p_Col_3() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - uint32_t* results; - uint32_t written, i; - int fd; - string filename("col4block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_3: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_3: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_2: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_3: could not read a whole block" << endl; - throw runtime_error("p_Col_3: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 4; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 0; - in->NVALS = 0; - in->InputFlags = 0; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 2048); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == (uint32_t) i); - } - - close(fd); + perror("read"); + return; } - void p_Col_4() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - u_int64_t* results; - uint32_t written, i; - int fd; - string filename("col8block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_4: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_4: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_4: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_4: could not read a whole block" << endl; - throw runtime_error("p_Col_4: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 0; - in->NVALS = 0; - in->InputFlags = 0; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 1024); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == (uint32_t) i); - } - - close(fd); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Col_5() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_GT; + cmd->OutputType = OT_DATAVALUE; + args->len = 7; + strncpy(args->data, "GERMANY", 7); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 41); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - uint8_t* results; - uint16_t* rids; - uint32_t written, i; - int fd; - string filename("col1block.cdf"); + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_gte_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - cerr << "p_Col_5: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } + fd = open(filename.c_str(), O_RDONLY); - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_5: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_5: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_5: could not read a whole block" << endl; - throw runtime_error("p_Col_5: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - rids = reinterpret_cast(&in[1]); - - in->DataSize = 1; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 0; - in->NVALS = 2; - in->InputFlags = 0; - rids[0] = 20; - rids[1] = 17; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 2); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == rids[i]); - } - - close(fd); + if (fd < 0) + { + perror("open"); + return; } - void p_Col_6() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - uint32_t* results; - uint32_t written, i; - int fd, tmp; - string filename("col4block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_6: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_6: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_6: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_6: could not read a whole block" << endl; - throw runtime_error("p_Col_6: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&in[1]); - - in->DataSize = 4; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 2; - in->BOP = BOP_AND; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 20; - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->DataSize]); - args->COP = COMPARE_GT; - tmp = 10; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 9); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == 11 + (uint32_t)i); - } - - close(fd); + perror("read"); + return; } - void p_Col_7() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - u_int64_t* results; - uint32_t written, i; - int fd; - int64_t tmp; - string filename("col8block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_7: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_7: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_7: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_7: could not read a whole block" << endl; - throw runtime_error("p_Col_7: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 2; - in->BOP = BOP_OR; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 10; - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->DataSize]); - args->COP = COMPARE_GT; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 33); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); - } - - close(fd); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Col_8() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_GE; + cmd->OutputType = OT_DATAVALUE; + args->len = 7; + strncpy(args->data, "GERMANY", 7); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 42); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - u_int64_t* results; - uint32_t written, i; - int fd; - int64_t tmp; - string filename("col8block.cdf"); + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_lt_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - cerr << "p_Col_7: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } + fd = open(filename.c_str(), O_RDONLY); - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_7: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_7: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_7: could not read a whole block" << endl; - throw runtime_error("p_Col_7: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 2; - in->BOP = BOP_OR; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 10; - args->COP = COMPARE_EQ; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->DataSize]); - args->COP = COMPARE_EQ; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 2); - CPPUNIT_ASSERT(results[0] == 10); - CPPUNIT_ASSERT(results[1] == 1000); -// for (i = 0; i < out->NVALS; i++) { -// cout << i << ": " << hex << (int)results[i] << dec << endl; -// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); -// } - - close(fd); + if (fd < 0) + { + perror("open"); + return; } - void p_Col_9() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - uint16_t* rids; - u_int64_t* results; - uint32_t written, i; - int fd; - int64_t tmp; - string filename("col8block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_9: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_9: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_9: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_9: could not read a whole block" << endl; - throw runtime_error("p_Col_9: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_DATAVALUE; - in->NOPS = 2; - in->BOP = BOP_OR; - in->NVALS = 2; - in->InputFlags = 0; - - tmp = 10; - args->COP = COMPARE_EQ; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->DataSize]); - args->COP = COMPARE_EQ; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - rids = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - 2 * (sizeof(ColArgs) + in->DataSize)]); - - rids[0] = 10; - rids[1] = 100; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 1); - CPPUNIT_ASSERT(results[0] == 10); -// for (i = 0; i < out->NVALS; i++) { -// cout << i << ": " << hex << (int)results[i] << dec << endl; -// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); -// } - - close(fd); + perror("read"); + return; } - void p_Col_10() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - int16_t* results; - uint32_t written, i; - int fd; - int64_t tmp; - string filename("col8block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_10: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_10: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_10: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_10: could not read a whole block" << endl; - throw runtime_error("p_Col_10: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_RID; - in->NOPS = 2; - in->BOP = BOP_OR; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 10; - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->DataSize]); - args->COP = COMPARE_GT; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 33); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == (i < 10 ? i : i - 10 + 1001)); - } - - close(fd); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Col_11() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_LT; + cmd->OutputType = OT_DATAVALUE; + args->len = 7; + strncpy(args->data, "GERMANY", 7); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 8); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - int16_t* resultRid; - int64_t* resultVal; - uint32_t written, i; - int fd; - int64_t tmp; - string filename("col8block.cdf"); + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_lte_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - cerr << "p_Col_11: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } + fd = open(filename.c_str(), O_RDONLY); - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_11: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_11: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_11: could not read a whole block" << endl; - throw runtime_error("p_Col_11: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_BOTH; - in->NOPS = 2; - in->BOP = BOP_OR; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 10; - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->DataSize]); - args->COP = COMPARE_GT; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 33); - - for (i = 0; i < out->NVALS; i++) - { - resultRid = reinterpret_cast(&output[ - sizeof(ColResultHeader) + i * (sizeof(Int16) + in->DataSize)]); - resultVal = reinterpret_cast(&resultRid[1]); - -// cout << i << ": rid:" << (int) *resultRid << " val:" << *resultVal << endl; - CPPUNIT_ASSERT(*resultRid == (i < 10 ? i : i - 10 + 1001)); - CPPUNIT_ASSERT(*resultVal == (i < 10 ? i : i - 10 + 1001)); - } - - close(fd); + if (fd < 0) + { + perror("open"); + return; } - void p_Col_12() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - uint8_t* results; - uint32_t written, i; - int fd; - string filename("col1block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_12: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_12: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_12: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_12: could not read a whole block" << endl; - throw runtime_error("p_Col_12: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 1; - in->DataType = CalpontSystemCatalog::CHAR; - in->OutputType = OT_DATAVALUE; - in->BOP = BOP_AND; - in->NOPS = 2; - in->NVALS = 0; - in->InputFlags = 0; - - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - args->COP = COMPARE_GT; - /* args->val[0] = 0xFC;*/ -// We need to test char values if the data type is char - args->val[0] = '2'; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 2]); - args->COP = COMPARE_LT; -// args->val[0] = 0; - args->val[0] = '4'; - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = &output[sizeof(ColResultHeader)]; -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 32); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << hex << (int)results[i] << dec << endl; - /* CPPUNIT_ASSERT(results[i] == 0xFD); - cout << i << ": " << (int)results[i] << endl; */ - CPPUNIT_ASSERT( (int)'3' == results[i] ); - } - - close(fd); + perror("read"); + return; } - void p_Col_13() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - int16_t* results; - uint32_t written, i; - int fd; - int32_t tmp; - int16_t ridTmp; - string filename("col4block.cdf"); - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_13: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_13: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_13: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_13: could not read a whole block" << endl; - throw runtime_error("p_Col_13: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = 4; - in->DataType = CalpontSystemCatalog::INT; - in->OutputType = OT_RID; - in->NOPS = 3; - in->BOP = BOP_OR; - in->NVALS = 3; - in->InputFlags = 1; - - // first argument "is RID 8 < 10?" Answer is yes - tmp = 10; // value to check - ridTmp = 8; - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, in->DataSize); - memcpy(&args->val[in->DataSize], &ridTmp, 2); - - // second argument "is RID 5 > 10?" Answer is no - args = reinterpret_cast(&args->val[in->DataSize + 2]); - args->COP = COMPARE_GT; - tmp = 10; - ridTmp = 5; - memcpy(args->val, &tmp, in->DataSize); - memcpy(&args->val[in->DataSize], &ridTmp, 2); - - // third argument "is RID 11 < 1000?" Answer is yes - args = reinterpret_cast(&args->val[in->DataSize + 2]); - args->COP = COMPARE_LT; - tmp = 1000; - ridTmp = 11; - memcpy(args->val, &tmp, in->DataSize); - memcpy(&args->val[in->DataSize], &ridTmp, 2); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 2); - CPPUNIT_ASSERT(results[0] == 8); - CPPUNIT_ASSERT(results[1] == 11); -// for (i = 0; i < out->NVALS; i++) { -// cout << i << ": " << (int)results[i] << endl; -// CPPUNIT_ASSERT(results[i] == (i < 10 ? i : i - 10 + 1001)); -// } - - close(fd); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Col_double_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_LE; + cmd->OutputType = OT_DATAVALUE; + args->len = 7; + strncpy(args->data, "GERMANY", 7); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 9); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - double* results; - uint32_t written, i; - int fd; - string filename("col_double_block.cdf"); - double tmp; + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_neq_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - cerr << "p_Col_double_1: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } + fd = open(filename.c_str(), O_RDONLY); - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_double_1: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_12: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_double_1: could not read a whole block" << endl; - throw runtime_error("p_Col_double_1: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::DOUBLE; - in->OutputType = OT_DATAVALUE; - in->BOP = BOP_AND; - in->NOPS = 2; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 10.5; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - args->COP = COMPARE_GT; - memcpy(args->val, &tmp, sizeof(tmp)); - tmp = 15; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 9]); - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, sizeof(tmp)); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 8); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == 11 + (i * 0.5)); - } - - close(fd); + if (fd < 0) + { + perror("open"); + return; } - void p_Col_float_1() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - float* resultVal; - uint32_t written, i; - int fd; - string filename("col_float_block.cdf"); - float tmp; - int16_t* resultRid; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_float_1: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_float_1: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_float_1: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_float_1: could not read a whole block" << endl; - throw runtime_error("p_Col_float_1: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 4; - in->DataType = CalpontSystemCatalog::FLOAT; - in->OutputType = OT_BOTH; - in->BOP = BOP_AND; - in->NOPS = 2; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = 10.5; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - args->COP = COMPARE_GT; - memcpy(args->val, &tmp, sizeof(tmp)); - tmp = 15; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 5]); - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, sizeof(tmp)); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 8); - - for (i = 0; i < out->NVALS; i++) - { - resultRid = reinterpret_cast(&output[ - sizeof(ColResultHeader) + i * (sizeof(Int16) + in->DataSize)]); - resultVal = reinterpret_cast(&resultRid[1]); - -// cout << i << ": rid:" << (int) *resultRid << " val:" << *resultVal << endl; - - CPPUNIT_ASSERT(*resultVal == 11 + (i * 0.5)); - } - - close(fd); + perror("read"); + return; } - void p_Col_neg_float_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - float* resultVal; - int16_t* resultRid; - uint32_t written, i; - int fd; - string filename("col_neg_float.cdf"); - float tmp; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_neg_float_1: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_neg_float_1: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_float_1: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_neg_float_1: could not read a whole block" << endl; - throw runtime_error("p_Col_neg_float_1: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 4; - in->DataType = CalpontSystemCatalog::FLOAT; - in->OutputType = OT_BOTH; - in->BOP = BOP_AND; - in->NOPS = 2; -// in->NOPS = 0; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = -5.0; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - args->COP = COMPARE_GT; - memcpy(args->val, &tmp, sizeof(tmp)); - tmp = 5.0; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 5]); - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, sizeof(tmp)); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 19); - - for (i = 0; i < out->NVALS; i++) - { - resultRid = reinterpret_cast(&output[ - sizeof(ColResultHeader) + i * (sizeof(Int16) + in->DataSize)]); - resultVal = reinterpret_cast(&resultRid[1]); - -// cout << i << ": rid:" << (int) *resultRid << " val:" << *resultVal << endl; - - CPPUNIT_ASSERT(*resultVal == -4.5 + (i * 0.5)); - } - - close(fd); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Col_neg_double_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_NE; + cmd->OutputType = OT_DATAVALUE; + args->len = 7; + strncpy(args->data, "GERMANY", 7); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 49); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - double* results; - uint32_t written, i; - int fd; - string filename("col_neg_double.cdf"); - double tmp; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - cerr << "p_Col_neg_double_1: skipping this test; needs the index list file " - << filename << endl; -// throw runtime_error("p_Col_1: file not found..."); - return; - } - - i = read(fd, block, BLOCK_SIZE); - - if (i <= 0) - { - cerr << "p_Col_neg_double_1: Couldn't read the file " << filename << endl; - throw runtime_error("p_Col_neg_double_1: Couldn't read the file"); - } - - if (i != BLOCK_SIZE) - { - cerr << "p_Col_neg_double_1: could not read a whole block" << endl; - throw runtime_error("p_Col_neg_double_1: could not read a whole block"); - } - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - - in->DataSize = 8; - in->DataType = CalpontSystemCatalog::DOUBLE; - in->OutputType = OT_DATAVALUE; - in->BOP = BOP_AND; - in->NOPS = 2; - in->NVALS = 0; - in->InputFlags = 0; - - tmp = -5.0; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - args->COP = COMPARE_GT; - memcpy(args->val, &tmp, sizeof(tmp)); - tmp = 5.0; - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 9]); - args->COP = COMPARE_LT; - memcpy(args->val, &tmp, sizeof(tmp)); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT(out->NVALS == 19); - - for (i = 0; i < out->NVALS; i++) - { -// cout << i << ": " << results[i] << dec << endl; - CPPUNIT_ASSERT(results[i] == -4.5 + (i * 0.5)); - } - - close(fd); + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; } - - template struct binary; - typedef binary<16> binary16; - typedef binary<32> binary32; - template - struct binary { - unsigned char data[W]; // May be ok for empty value ? - void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} - inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} - inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} - }; + } - void p_Col_bin_16() + void p_TokenByScan_range_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - binary16* results; - uint32_t written, i; - int fd; - binary16 tmp; - binary16* bin16 = (binary16*) block; - - for(int i = 0; i < BLOCK_SIZE/16; i++) - { - bin16[i] = 0; - } - - bin16[0].uint64(0) = 10UL; - - bin16[1].uint64(0) = 1000UL; - - bin16[3].uint64(0) = 1000UL; - bin16[3].uint64(1) = 1; - - bin16[4].uint64(0) = 256; - bin16[4].uint64(1) = 1; - - typedef char bin16_t[16]; - - *(uint64_t*)(((bin16_t*)block) + 5) = 500; - - *(uint64_t*)&((bin16_t*)block)[6] = 501; - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = sizeof(binary16); - in->DataType = execplan::CalpontSystemCatalog::DECIMAL; - in->OutputType = OT_DATAVALUE; - in->NOPS = 3; - in->BOP = BOP_OR; - in->NVALS = 0; - - tmp = 10; - args->COP = COMPARE_EQ; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast (args->val + in->DataSize); - - args->COP = COMPARE_EQ; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - args = reinterpret_cast (args->val + in->DataSize); - tmp.uint64(0) = 256; - tmp.uint64(1) = 1; - args->COP = COMPARE_EQ; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); - CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); - CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); - for (i = 0; i < out->NVALS; i++) { - printf("Result %d Value %016X%016X\n",i ,results[i].uint64(1),results[i].uint64(0) ); -// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); - } + perror("open"); + return; } - void p_Col_bin_32() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t input[2 * BLOCK_SIZE], output[8 * BLOCK_SIZE], block[BLOCK_SIZE]; - NewColRequestHeader* in; - ColResultHeader* out; - ColArgs* args; - binary32* results; - uint32_t written, i; - int fd; - binary32 tmp; - binary32* bin32 = (binary32*) block; - - for(int i = 0; i < BLOCK_SIZE/32; i++) - { - bin32[i].uint64(0) = 0; - } - - bin32[0].uint64(0) = 10UL; - - bin32[1].uint64(0) = 1000UL; - - bin32[3].uint64(0) = 1000UL; - bin32[3].uint64(1) = 1; - - bin32[4].uint64(0) = 256; - bin32[4].uint64(1) = 254; - bin32[4].uint64(2) = 253; - bin32[4].uint64(3) = 252; - - typedef char bin32_t[32]; - - *(uint64_t*)(((bin32_t*)block) + 5) = 500; - - *(uint64_t*)&((bin32_t*)block)[6] = 501; - - memset(input, 0, BLOCK_SIZE); - memset(output, 0, 4 * BLOCK_SIZE); - - in = reinterpret_cast(input); - out = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); - - in->DataSize = sizeof(binary32); - in->DataType = execplan::CalpontSystemCatalog::BINARY; - in->OutputType = OT_DATAVALUE; - in->NOPS = 3; - in->BOP = BOP_OR; - in->NVALS = 0; - - tmp = 10; - args->COP = COMPARE_EQ; - memcpy(args->val, &tmp, in->DataSize); - args = reinterpret_cast (args->val + in->DataSize); - - args->COP = COMPARE_EQ; - tmp = 1000; - memcpy(args->val, &tmp, in->DataSize); - - args = reinterpret_cast (args->val + in->DataSize); - tmp.uint64(0) = 256; - tmp.uint64(1) = 254; - tmp.uint64(2) = 253; - tmp.uint64(3) = 252; - - args->COP = COMPARE_EQ; - memcpy(args->val, &tmp, in->DataSize); - - pp.setBlockPtr((int*) block); - pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); - - results = reinterpret_cast(&output[sizeof(ColResultHeader)]); -// cout << "NVALS = " << out->NVALS << endl; - CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); -// CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); -// CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); - for (i = 0; i < out->NVALS; i++) { - printf("Result %d Value %016X%016X%016X%016X\n",i ,results[i].uint64(3),results[i].uint64(2),results[i].uint64(1),results[i].uint64(0) ); -// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); - } + perror("read"); + return; } - - - void p_Dictionary_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_EQ; - args->len = 5; - strncpy((char*)args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - - outValue = reinterpret_cast(&results[1]); - outValue->data[args->len] = '\0'; // not reusable in tests with multiple matches. - CPPUNIT_ASSERT(results->NVALS == 1); - CPPUNIT_ASSERT(outValue->len == 5); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Dictionary_2() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + + argsOffset = sizeof(TokenByScanRequestHeader); + args = reinterpret_cast(&input[argsOffset]); + + cmd->NVALS = 2; + cmd->COP1 = COMPARE_LT; + cmd->COP2 = COMPARE_GT; + cmd->BOP = BOP_AND; + cmd->OutputType = OT_DATAVALUE; + args->len = 5; + strncpy(args->data, "KENYA", 5); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 6; + strncpy(args->data, "BRAZIL", 6); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 12); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd; - uint32_t err; + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_eq_6() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - perror("open"); - return; - } + fd = open(filename.c_str(), O_RDONLY); - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 2; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_OR; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_EQ; - args->len = 13; - strncpy((char*)args->data, "UNITED STATES", 13); - - args = reinterpret_cast(&input[sizeof(DictInput) + args->len + - sizeof(DictFilterElement)]); - args->COP = COMPARE_EQ; - args->len = 5; - strncpy((char*)args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - - outValue = reinterpret_cast(&results[1]); - CPPUNIT_ASSERT(results->NVALS == 2); - CPPUNIT_ASSERT(outValue->len == 5); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); - - outValue = reinterpret_cast(&output[sizeof(DictOutput) + sizeof(DataValue) + outValue->len]); - CPPUNIT_ASSERT(outValue->len == 13); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "UNITED STATES", 13) == 0); + if (fd < 0) + { + perror("open"); + return; } - void p_Dictionary_3() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 0; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 50); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; -// cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + outValue->len; - } + perror("read"); + return; } - void p_Dictionary_gt_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_GT; - args->len = 7; - strncpy((char*)args->data, "GERMANY", 7); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 41); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; -// cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; - argsOffset += sizeof(DataValue) + outValue->len; - } + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Dictionary_token_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + + argsOffset = sizeof(TokenByScanRequestHeader); + args = reinterpret_cast(&input[argsOffset]); + + cmd->NVALS = 6; + cmd->COP1 = COMPARE_EQ; + cmd->BOP = BOP_OR; + cmd->OutputType = OT_DATAVALUE; + args->len = 5; + strncpy(args->data, "KENYA", 5); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 6; + strncpy(args->data, "BRAZIL", 6); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 8; + strncpy(args->data, "ETHIOPIA", 8); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 6; + strncpy(args->data, "CANADA", 6); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 13; + strncpy(args->data, "UNITED_STATES", 13); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 13; + strncpy(args->data, "UNITED STATES", 13); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 5); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - PrimToken* outToken; - int fd; - uint32_t err; + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_neq_6() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - perror("open"); - return; - } + fd = open(filename.c_str(), O_RDONLY); - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 2; - cmd->OutputType = OT_TOKEN; - cmd->BOP = BOP_OR; - cmd->LBID = 56878; //random magic - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_EQ; - args->len = 13; - strncpy((char*)args->data, "UNITED STATES", 13); - - args = reinterpret_cast(&input[sizeof(DictInput) + args->len + - sizeof(DictFilterElement)]); - args->COP = COMPARE_EQ; - args->len = 5; - strncpy((char*)args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - - CPPUNIT_ASSERT(results->NVALS == 2); - - outToken = reinterpret_cast(&results[1]); - CPPUNIT_ASSERT(outToken->len == 5); - CPPUNIT_ASSERT(outToken->LBID == cmd->LBID); - CPPUNIT_ASSERT(outToken->offset == 19); - CPPUNIT_ASSERT(outToken[1].len == 13); - CPPUNIT_ASSERT(outToken[1].LBID == cmd->LBID); - CPPUNIT_ASSERT(outToken[1].offset == 50); + if (fd < 0) + { + perror("open"); + return; } - void p_Dictionary_inputArg_1() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 2; - cmd->OutputType = OT_INPUTARG; - cmd->BOP = BOP_OR; - cmd->LBID = 56878; //random magic - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_EQ; - args->len = 13; - strncpy((char*)args->data, "UNITED STATES", 13); - - args = reinterpret_cast(&input[sizeof(DictInput) + args->len + - sizeof(DictFilterElement)]); - args->COP = COMPARE_EQ; - args->len = 5; - strncpy((char*)args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - - CPPUNIT_ASSERT(results->NVALS == 2); - - outValue = reinterpret_cast(&results[1]); - CPPUNIT_ASSERT(results->NVALS == 2); - CPPUNIT_ASSERT(outValue->len == 5); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); - - outValue = reinterpret_cast(&output[sizeof(DictOutput) + sizeof(DataValue) + outValue->len]); - CPPUNIT_ASSERT(outValue->len == 13); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "UNITED STATES", 13) == 0); + perror("read"); + return; } - void p_Dictionary_token_agg_1() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - PrimToken* outToken; - DictAggregate* agg; - DataValue* outValue; - int fd; - uint32_t err; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 2; - cmd->OutputType = OT_TOKEN | OT_AGGREGATE; - cmd->BOP = BOP_OR; - cmd->LBID = 56878; //random magic - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_EQ; - args->len = 13; - strncpy((char*)args->data, "UNITED STATES", 13); - - args = reinterpret_cast(&input[sizeof(DictInput) + args->len + - sizeof(DictFilterElement)]); - args->COP = COMPARE_EQ; - args->len = 5; - strncpy((char*)args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - - CPPUNIT_ASSERT(results->NVALS == 2); - - outToken = reinterpret_cast(&results[1]); - CPPUNIT_ASSERT(outToken->len == 5); - CPPUNIT_ASSERT(outToken->LBID == cmd->LBID); - CPPUNIT_ASSERT(outToken->offset == 19); - CPPUNIT_ASSERT(outToken[1].len == 13); - CPPUNIT_ASSERT(outToken[1].LBID == cmd->LBID); - CPPUNIT_ASSERT(outToken[1].offset == 50); - - agg = reinterpret_cast(&outToken[2]); - CPPUNIT_ASSERT(agg->Count == 50); - - // min follows - outValue = reinterpret_cast(&agg[1]); - CPPUNIT_ASSERT(outValue->len == 7); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "ALGERIA", 7) == 0); - // max follows - outValue = reinterpret_cast(((int8_t*)outValue) + - sizeof(DataValue) + outValue->len); - CPPUNIT_ASSERT(outValue->len == 6); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XUSSIA", 6) == 0); + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Dictionary_inToken_1() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + + argsOffset = sizeof(TokenByScanRequestHeader); + args = reinterpret_cast(&input[argsOffset]); + + cmd->NVALS = 6; + cmd->COP1 = COMPARE_NE; + cmd->BOP = BOP_AND; + cmd->OutputType = OT_DATAVALUE; + args->len = 5; + strncpy(args->data, "KENYA", 5); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 6; + strncpy(args->data, "BRAZIL", 6); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 8; + strncpy(args->data, "ETHIOPIA", 8); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 6; + strncpy(args->data, "CANADA", 6); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 13; + strncpy(args->data, "UNITED_STATES", 13); + + argsOffset += args->len + sizeof(DataValue); + args = reinterpret_cast(&input[argsOffset]); + args->len = 13; + strncpy(args->data, "UNITED STATES", 13); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 45); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DictAggregate* agg; - PrimToken* outToken; - DataValue* outValue; - int fd; - uint32_t err; + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } - fd = open(filename.c_str(), O_RDONLY); + void p_TokenByScan_like_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE], tmp[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + int fd, argsOffset, i; + uint32_t err; - if (fd < 0) - { - perror("open"); - return; - } + fd = open(filename.c_str(), O_RDONLY); - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&input[sizeof(DictInput) + 3 * sizeof(PrimToken)]); - - cmd->NOPS = 2; - cmd->OutputType = OT_TOKEN | OT_AGGREGATE; - cmd->BOP = BOP_OR; - cmd->LBID = 56878; //random magic - cmd->NVALS = 3; - cmd->InputFlags = 0; - cmd->tokens[0].LBID = 56878; - cmd->tokens[0].offset = 7838; // UNITED STATES - cmd->tokens[0].len = 13; - cmd->tokens[1].LBID = 56878; - cmd->tokens[1].offset = 7890; // ROMANIA - cmd->tokens[1].len = 7; - cmd->tokens[2].LBID = 56878; - cmd->tokens[2].offset = 8074; // XHINA - cmd->tokens[2].len = 5; - - args->COP = COMPARE_EQ; - args->len = 13; - strncpy((char*)args->data, "UNITED STATES", 13); - - args = reinterpret_cast(&input[sizeof(DictInput) + 3 * sizeof(PrimToken) + - args->len + sizeof(DictFilterElement)]); - args->COP = COMPARE_EQ; - args->len = 5; - strncpy((char*)args->data, "XHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - - CPPUNIT_ASSERT(results->NVALS == 2); - - outToken = reinterpret_cast(&results[1]); - CPPUNIT_ASSERT(outToken[0].len == 13); - CPPUNIT_ASSERT(outToken[0].LBID == cmd->LBID); - CPPUNIT_ASSERT(outToken[0].offset == 50); - CPPUNIT_ASSERT(outToken[1].len == 5); - CPPUNIT_ASSERT(outToken[1].LBID == cmd->LBID); - CPPUNIT_ASSERT(outToken[1].offset == 19); - - agg = reinterpret_cast(&outToken[2]); - CPPUNIT_ASSERT(agg->Count == 3); - - // min follows - outValue = reinterpret_cast(&agg[1]); - CPPUNIT_ASSERT(outValue->len == 7); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "ROMANIA", 7) == 0); - // max follows - outValue = reinterpret_cast(((int8_t*)outValue) + - sizeof(DataValue) + outValue->len); - CPPUNIT_ASSERT(outValue->len == 5); - CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); + if (fd < 0) + { + perror("open"); + return; } - void p_Dictionary_oldgetsig_1() - { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; + err = read(fd, block, BLOCK_SIZE); -#pragma pack(push,1) - struct OldSigArgs + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + + argsOffset = sizeof(TokenByScanRequestHeader); + args = reinterpret_cast(&input[argsOffset]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_LIKE; + cmd->BOP = BOP_AND; + cmd->OutputType = OT_DATAVALUE; + args->len = 4; + strncpy(args->data, "%NYA", 4); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 2); + + argsOffset = sizeof(TokenByScanResultHeader); + + for (i = 0; i < results->NVALS; i++) + { + args = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, args->data, args->len); + tmp[args->len] = '\0'; + // cout << " " << i << ": len=" << args->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + args->len; + } + } + + void p_TokenByScan_token_eq_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + TokenByScanRequestHeader* cmd; + TokenByScanResultHeader* results; + DataValue* args; + PrimToken* result; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NVALS = 1; + cmd->COP1 = COMPARE_EQ; + cmd->OutputType = OT_TOKEN; + cmd->LBID = 13; // arbitrary # + args->len = 5; + strncpy(args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_TokenByScan(cmd, results, BLOCK_SIZE); + + result = reinterpret_cast(&results[1]); + CPPUNIT_ASSERT(results->NVALS == 1); + CPPUNIT_ASSERT(result->len == 5); + CPPUNIT_ASSERT(result->offset == 19); + CPPUNIT_ASSERT(result->LBID == 13); + } + + void p_IdxList_1() + { + // there should be a list header (corresponds to the search value we used in + // p_IdxWalk) at FBO 10251 subblock 22 entry 12. The first LBID of the + // index list file is 10000 + + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], input[BLOCK_SIZE], output[BLOCK_SIZE]; + string filename("FILE_991.dat"); + IndexListHeader *hdr, *resultHdr; + IndexListParam* params; + IndexListEntry* results; + vector bigResults; + vector::iterator it; + int fd; + uint32_t err, i; + bool continuationPtr; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_IdxList: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; + } + + hdr = reinterpret_cast(input); + params = reinterpret_cast(&input[sizeof(IndexListHeader)]); + resultHdr = reinterpret_cast(output); + results = reinterpret_cast(&output[sizeof(IndexListHeader)]); + + memset(hdr, 0, sizeof(IndexListHeader)); + hdr->NVALS = 1; + + params->fbo = 11; + params->sbid = 22; + params->entry = 12; + params->type = LIST_SIZE; + + pp.setBlockPtr((int*)block); + + do + { + lseek(fd, params->fbo * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); + + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } + + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxList(hdr, resultHdr, 0); + + // scan for a continuation pointer + continuationPtr = false; + + for (i = 0; i < resultHdr->NVALS; i++) + { + if (results[i].type == LLP_SUBBLK || results[i].type == LLP_BLK) { - uint64_t rid; - uint16_t offsetIndex; - } *args; + // the header for the next iteration can stay the same, the + // argument has to be the LLP entry. + memcpy(params, &results[i], sizeof(IndexListEntry)); + CPPUNIT_ASSERT(params->fbo >= 20480); + CPPUNIT_ASSERT(params->fbo < 30720); + params->fbo -= 20480; // this happens to be the base LBID for the test data + continuationPtr = true; + } + else + { + CPPUNIT_ASSERT(results[i].type == RID); + bigResults.push_back(results[i]); + } + } + + /* + // this clause doesn't get used in this test b/c there is only one RID + if (results->type == LLP_SUBBLK || results->type == LLP_BLK) { + CPPUNIT_ASSERT(resultHdr->NVALS > 1); + + // the header for the next iteration can stay the same, the + // argument has to be the LLP entry. + memcpy(params, results, sizeof(IndexListEntry)); + // store the results returned so far + for (i = 1; i < resultHdr->NVALS; i++) + bigResults.push_back(results[i]); + } + else + for (i = 0; i < resultHdr->NVALS; i++) + bigResults.push_back(results[i]); + */ + } while (continuationPtr); + + CPPUNIT_ASSERT(bigResults.size() == 1); + + // cout << endl << "RID count: " << bigResults.size() << endl; + for (i = 1, it = bigResults.begin(); it != bigResults.end(); it++, i++) + { + // cout << " " << i << ": type=" << (*it).type << " rid=" << (*it).value << endl; + CPPUNIT_ASSERT((*it).type == RID); + CPPUNIT_ASSERT((*it).value == 100); + } + } + + void p_IdxList_2() + { + // there should be a list header (corresponds to the search value we used in + // p_IdxWalk) at FBO 10251 subblock 22 entry 12. The first LBID of the + // index list file is 10000 + + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], input[BLOCK_SIZE], output[BLOCK_SIZE]; + string filename("FILE233.cdf"); + IndexListHeader *hdr, *resultHdr; + IndexListParam* params; + IndexListEntry* results; + vector bigResults; + vector::iterator it; + int fd; + uint32_t err, i; + bool continuationPtr; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_IdxList: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_IdxWalk: file not found..."); + return; + } + + hdr = reinterpret_cast(input); + params = reinterpret_cast(&input[sizeof(IndexListHeader)]); + resultHdr = reinterpret_cast(output); + results = reinterpret_cast(&output[sizeof(IndexListHeader)]); + + memset(hdr, 0, sizeof(IndexListHeader)); + hdr->NVALS = 1; + + params->fbo = 11; + params->sbid = 10; + params->entry = 28; + params->type = LIST_SIZE; + + pp.setBlockPtr((int*)block); + + do + { + lseek(fd, params->fbo * BLOCK_SIZE, SEEK_SET); + err = read(fd, block, BLOCK_SIZE); + + if (err <= 0) + { + cerr << "p_IdxWalk: Couldn't read the file " << filename << endl; + throw runtime_error("p_IdxWalk: Couldn't read the file"); + } + + if (err != BLOCK_SIZE) + { + cerr << "p_IdxWalk: could not read a whole block" << endl; + throw runtime_error("p_IdxWalk: could not read a whole block"); + } + + pp.p_IdxList(hdr, resultHdr, 0); + + // scan for a continuation pointer + continuationPtr = false; + + for (i = 0; i < resultHdr->NVALS; i++) + { + if (results[i].type == LLP_SUBBLK || results[i].type == LLP_BLK) + { + // the header for the next iteration can stay the same, the + // argument has to be the LLP entry. + memcpy(params, &results[i], sizeof(IndexListEntry)); + CPPUNIT_ASSERT(params->fbo >= 20480); + CPPUNIT_ASSERT(params->fbo < 30720); + params->fbo -= 20480; // this happens to be the base LBID for the test data + continuationPtr = true; + } + else + { + CPPUNIT_ASSERT(results[i].type == RID); + bigResults.push_back(results[i]); + } + } + + /* + if (results->type == LLP_SUBBLK || results->type == LLP_BLK) { + CPPUNIT_ASSERT(resultHdr->NVALS > 1); + + // the header for the next iteration can stay the same, the + // argument has to be the LLP entry. + memcpy(params, results, sizeof(IndexListEntry)); + CPPUNIT_ASSERT(params->fbo >= 20480); + CPPUNIT_ASSERT(params->fbo < 30720); + params->fbo -= 20480; // this happens to be the base LBID for the test data + + // store the results returned so far + for (i = 1; i < resultHdr->NVALS; i++) + bigResults.push_back(results[i]); + } + else + for (i = 0; i < resultHdr->NVALS; i++) + bigResults.push_back(results[i]); + */ + } while (continuationPtr); + + CPPUNIT_ASSERT(bigResults.size() == 8901); + + // cout << endl << "RID count: " << bigResults.size() << endl; + for (i = 1, it = bigResults.begin(); it != bigResults.end(); it++, i++) + { + // cout << " " << i << ": type=" << (*it).type << " rid=" << (*it).value << endl; + CPPUNIT_ASSERT((*it).type == RID); + CPPUNIT_ASSERT((*it).value == (uint32_t)i - 1); + } + } + + void p_Col_1() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + uint8_t* results; + int fd; + uint32_t i, written; + string filename("col1block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_1: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_1: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_1: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_1: could not read a whole block" << endl; + throw runtime_error("p_Col_1: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 1; + in->DataType = CalpontSystemCatalog::CHAR; + in->OutputType = OT_DATAVALUE; + in->NOPS = 0; + in->NVALS = 0; + in->InputFlags = 0; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = &output[sizeof(ColResultHeader)]; + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 8160); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == i % 255); + } + + close(fd); + } + + void p_Col_2() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + uint16_t* results; + uint32_t written, i; + int fd; + string filename("col2block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_2: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_2: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_2: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_2: could not read a whole block" << endl; + throw runtime_error("p_Col_2: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 2; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 0; + in->NVALS = 0; + in->InputFlags = 0; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 4096); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == i); + } + + close(fd); + } + + void p_Col_3() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + uint32_t* results; + uint32_t written, i; + int fd; + string filename("col4block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_3: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_3: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_2: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_3: could not read a whole block" << endl; + throw runtime_error("p_Col_3: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 4; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 0; + in->NVALS = 0; + in->InputFlags = 0; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 2048); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == (uint32_t)i); + } + + close(fd); + } + + void p_Col_4() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + u_int64_t* results; + uint32_t written, i; + int fd; + string filename("col8block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_4: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_4: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_4: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_4: could not read a whole block" << endl; + throw runtime_error("p_Col_4: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 0; + in->NVALS = 0; + in->InputFlags = 0; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 1024); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == (uint32_t)i); + } + + close(fd); + } + + void p_Col_5() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + uint8_t* results; + uint16_t* rids; + uint32_t written, i; + int fd; + string filename("col1block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_5: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_5: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_5: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_5: could not read a whole block" << endl; + throw runtime_error("p_Col_5: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + rids = reinterpret_cast(&in[1]); + + in->DataSize = 1; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 0; + in->NVALS = 2; + in->InputFlags = 0; + rids[0] = 20; + rids[1] = 17; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 2); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == rids[i]); + } + + close(fd); + } + + void p_Col_6() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + uint32_t* results; + uint32_t written, i; + int fd, tmp; + string filename("col4block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_6: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_6: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_6: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_6: could not read a whole block" << endl; + throw runtime_error("p_Col_6: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&in[1]); + + in->DataSize = 4; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 2; + in->BOP = BOP_AND; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 20; + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->DataSize]); + args->COP = COMPARE_GT; + tmp = 10; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 9); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == 11 + (uint32_t)i); + } + + close(fd); + } + + void p_Col_7() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + u_int64_t* results; + uint32_t written, i; + int fd; + int64_t tmp; + string filename("col8block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_7: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_7: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_7: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_7: could not read a whole block" << endl; + throw runtime_error("p_Col_7: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 2; + in->BOP = BOP_OR; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 10; + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->DataSize]); + args->COP = COMPARE_GT; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 33); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == (uint32_t)(i < 10 ? i : i - 10 + 1001)); + } + + close(fd); + } + + void p_Col_8() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + u_int64_t* results; + uint32_t written, i; + int fd; + int64_t tmp; + string filename("col8block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_7: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_7: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_7: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_7: could not read a whole block" << endl; + throw runtime_error("p_Col_7: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 2; + in->BOP = BOP_OR; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->DataSize]); + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 2); + CPPUNIT_ASSERT(results[0] == 10); + CPPUNIT_ASSERT(results[1] == 1000); + // for (i = 0; i < out->NVALS; i++) { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + // CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + // } + + close(fd); + } + + void p_Col_9() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + uint16_t* rids; + u_int64_t* results; + uint32_t written, i; + int fd; + int64_t tmp; + string filename("col8block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_9: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_9: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_9: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_9: could not read a whole block" << endl; + throw runtime_error("p_Col_9: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_DATAVALUE; + in->NOPS = 2; + in->BOP = BOP_OR; + in->NVALS = 2; + in->InputFlags = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->DataSize]); + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + rids = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + 2 * (sizeof(ColArgs) + in->DataSize)]); + + rids[0] = 10; + rids[1] = 100; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 1); + CPPUNIT_ASSERT(results[0] == 10); + // for (i = 0; i < out->NVALS; i++) { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + // CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + // } + + close(fd); + } + + void p_Col_10() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + int16_t* results; + uint32_t written, i; + int fd; + int64_t tmp; + string filename("col8block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_10: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_10: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_10: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_10: could not read a whole block" << endl; + throw runtime_error("p_Col_10: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_RID; + in->NOPS = 2; + in->BOP = BOP_OR; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 10; + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->DataSize]); + args->COP = COMPARE_GT; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 33); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == (i < 10 ? i : i - 10 + 1001)); + } + + close(fd); + } + + void p_Col_11() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + int16_t* resultRid; + int64_t* resultVal; + uint32_t written, i; + int fd; + int64_t tmp; + string filename("col8block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_11: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_11: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_11: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_11: could not read a whole block" << endl; + throw runtime_error("p_Col_11: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_BOTH; + in->NOPS = 2; + in->BOP = BOP_OR; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 10; + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->DataSize]); + args->COP = COMPARE_GT; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 33); + + for (i = 0; i < out->NVALS; i++) + { + resultRid = + reinterpret_cast(&output[sizeof(ColResultHeader) + i * (sizeof(Int16) + in->DataSize)]); + resultVal = reinterpret_cast(&resultRid[1]); + + // cout << i << ": rid:" << (int) *resultRid << " val:" << *resultVal << endl; + CPPUNIT_ASSERT(*resultRid == (i < 10 ? i : i - 10 + 1001)); + CPPUNIT_ASSERT(*resultVal == (i < 10 ? i : i - 10 + 1001)); + } + + close(fd); + } + + void p_Col_12() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + uint8_t* results; + uint32_t written, i; + int fd; + string filename("col1block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_12: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_12: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_12: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_12: could not read a whole block" << endl; + throw runtime_error("p_Col_12: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 1; + in->DataType = CalpontSystemCatalog::CHAR; + in->OutputType = OT_DATAVALUE; + in->BOP = BOP_AND; + in->NOPS = 2; + in->NVALS = 0; + in->InputFlags = 0; + + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + args->COP = COMPARE_GT; + /* args->val[0] = 0xFC;*/ + // We need to test char values if the data type is char + args->val[0] = '2'; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 2]); + args->COP = COMPARE_LT; + // args->val[0] = 0; + args->val[0] = '4'; + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = &output[sizeof(ColResultHeader)]; + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 32); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << hex << (int)results[i] << dec << endl; + /* CPPUNIT_ASSERT(results[i] == 0xFD); + cout << i << ": " << (int)results[i] << endl; */ + CPPUNIT_ASSERT((int)'3' == results[i]); + } + + close(fd); + } + + void p_Col_13() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + int16_t* results; + uint32_t written, i; + int fd; + int32_t tmp; + int16_t ridTmp; + string filename("col4block.cdf"); + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_13: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_13: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_13: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_13: could not read a whole block" << endl; + throw runtime_error("p_Col_13: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = 4; + in->DataType = CalpontSystemCatalog::INT; + in->OutputType = OT_RID; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 3; + in->InputFlags = 1; + + // first argument "is RID 8 < 10?" Answer is yes + tmp = 10; // value to check + ridTmp = 8; + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, in->DataSize); + memcpy(&args->val[in->DataSize], &ridTmp, 2); + + // second argument "is RID 5 > 10?" Answer is no + args = reinterpret_cast(&args->val[in->DataSize + 2]); + args->COP = COMPARE_GT; + tmp = 10; + ridTmp = 5; + memcpy(args->val, &tmp, in->DataSize); + memcpy(&args->val[in->DataSize], &ridTmp, 2); + + // third argument "is RID 11 < 1000?" Answer is yes + args = reinterpret_cast(&args->val[in->DataSize + 2]); + args->COP = COMPARE_LT; + tmp = 1000; + ridTmp = 11; + memcpy(args->val, &tmp, in->DataSize); + memcpy(&args->val[in->DataSize], &ridTmp, 2); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 2); + CPPUNIT_ASSERT(results[0] == 8); + CPPUNIT_ASSERT(results[1] == 11); + // for (i = 0; i < out->NVALS; i++) { + // cout << i << ": " << (int)results[i] << endl; + // CPPUNIT_ASSERT(results[i] == (i < 10 ? i : i - 10 + 1001)); + // } + + close(fd); + } + + void p_Col_double_1() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + double* results; + uint32_t written, i; + int fd; + string filename("col_double_block.cdf"); + double tmp; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_double_1: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_double_1: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_12: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_double_1: could not read a whole block" << endl; + throw runtime_error("p_Col_double_1: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::DOUBLE; + in->OutputType = OT_DATAVALUE; + in->BOP = BOP_AND; + in->NOPS = 2; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 10.5; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + args->COP = COMPARE_GT; + memcpy(args->val, &tmp, sizeof(tmp)); + tmp = 15; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 9]); + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, sizeof(tmp)); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 8); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == 11 + (i * 0.5)); + } + + close(fd); + } + + void p_Col_float_1() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + float* resultVal; + uint32_t written, i; + int fd; + string filename("col_float_block.cdf"); + float tmp; + int16_t* resultRid; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_float_1: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_float_1: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_float_1: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_float_1: could not read a whole block" << endl; + throw runtime_error("p_Col_float_1: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 4; + in->DataType = CalpontSystemCatalog::FLOAT; + in->OutputType = OT_BOTH; + in->BOP = BOP_AND; + in->NOPS = 2; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = 10.5; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + args->COP = COMPARE_GT; + memcpy(args->val, &tmp, sizeof(tmp)); + tmp = 15; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 5]); + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, sizeof(tmp)); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 8); + + for (i = 0; i < out->NVALS; i++) + { + resultRid = + reinterpret_cast(&output[sizeof(ColResultHeader) + i * (sizeof(Int16) + in->DataSize)]); + resultVal = reinterpret_cast(&resultRid[1]); + + // cout << i << ": rid:" << (int) *resultRid << " val:" << *resultVal << endl; + + CPPUNIT_ASSERT(*resultVal == 11 + (i * 0.5)); + } + + close(fd); + } + + void p_Col_neg_float_1() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + float* resultVal; + int16_t* resultRid; + uint32_t written, i; + int fd; + string filename("col_neg_float.cdf"); + float tmp; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_neg_float_1: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_neg_float_1: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_float_1: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_neg_float_1: could not read a whole block" << endl; + throw runtime_error("p_Col_neg_float_1: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 4; + in->DataType = CalpontSystemCatalog::FLOAT; + in->OutputType = OT_BOTH; + in->BOP = BOP_AND; + in->NOPS = 2; + // in->NOPS = 0; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = -5.0; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + args->COP = COMPARE_GT; + memcpy(args->val, &tmp, sizeof(tmp)); + tmp = 5.0; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 5]); + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, sizeof(tmp)); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 19); + + for (i = 0; i < out->NVALS; i++) + { + resultRid = + reinterpret_cast(&output[sizeof(ColResultHeader) + i * (sizeof(Int16) + in->DataSize)]); + resultVal = reinterpret_cast(&resultRid[1]); + + // cout << i << ": rid:" << (int) *resultRid << " val:" << *resultVal << endl; + + CPPUNIT_ASSERT(*resultVal == -4.5 + (i * 0.5)); + } + + close(fd); + } + + void p_Col_neg_double_1() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + double* results; + uint32_t written, i; + int fd; + string filename("col_neg_double.cdf"); + double tmp; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + cerr << "p_Col_neg_double_1: skipping this test; needs the index list file " << filename << endl; + // throw runtime_error("p_Col_1: file not found..."); + return; + } + + i = read(fd, block, BLOCK_SIZE); + + if (i <= 0) + { + cerr << "p_Col_neg_double_1: Couldn't read the file " << filename << endl; + throw runtime_error("p_Col_neg_double_1: Couldn't read the file"); + } + + if (i != BLOCK_SIZE) + { + cerr << "p_Col_neg_double_1: could not read a whole block" << endl; + throw runtime_error("p_Col_neg_double_1: could not read a whole block"); + } + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + + in->DataSize = 8; + in->DataType = CalpontSystemCatalog::DOUBLE; + in->OutputType = OT_DATAVALUE; + in->BOP = BOP_AND; + in->NOPS = 2; + in->NVALS = 0; + in->InputFlags = 0; + + tmp = -5.0; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + args->COP = COMPARE_GT; + memcpy(args->val, &tmp, sizeof(tmp)); + tmp = 5.0; + args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + 9]); + args->COP = COMPARE_LT; + memcpy(args->val, &tmp, sizeof(tmp)); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT(out->NVALS == 19); + + for (i = 0; i < out->NVALS; i++) + { + // cout << i << ": " << results[i] << dec << endl; + CPPUNIT_ASSERT(results[i] == -4.5 + (i * 0.5)); + } + + close(fd); + } + + template + struct binary; + typedef binary<16> binary16; + typedef binary<32> binary32; + template + struct binary + { + unsigned char data[W]; // May be ok for empty value ? + void operator=(uint64_t v) + { + *((uint64_t*)data) = v; + memset(data + 8, 0, W - 8); + } + inline uint8_t& operator[](const int index) + { + return *((uint8_t*)(data + index)); + } + inline uint64_t& uint64(const int index) + { + return *((uint64_t*)(data + (index << 3))); + } + }; + + void p_Col_bin_16() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + binary16* results; + uint32_t written, i; + int fd; + binary16 tmp; + binary16* bin16 = (binary16*)block; + + for (int i = 0; i < BLOCK_SIZE / 16; i++) + { + bin16[i] = 0; + } + + bin16[0].uint64(0) = 10UL; + + bin16[1].uint64(0) = 1000UL; + + bin16[3].uint64(0) = 1000UL; + bin16[3].uint64(1) = 1; + + bin16[4].uint64(0) = 256; + bin16[4].uint64(1) = 1; + + typedef char bin16_t[16]; + + *(uint64_t*)(((bin16_t*)block) + 5) = 500; + + *(uint64_t*)&((bin16_t*)block)[6] = 501; + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = sizeof(binary16); + in->DataType = execplan::CalpontSystemCatalog::DECIMAL; + in->OutputType = OT_DATAVALUE; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(args->val + in->DataSize); + + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + args = reinterpret_cast(args->val + in->DataSize); + tmp.uint64(0) = 256; + tmp.uint64(1) = 1; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); + CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); + CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); + for (i = 0; i < out->NVALS; i++) + { + printf("Result %d Value %016X%016X\n", i, results[i].uint64(1), results[i].uint64(0)); + // CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + } + } + + void p_Col_bin_32() + { + PrimitiveProcessor pp; + uint8_t input[2 * BLOCK_SIZE], output[8 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + ColResultHeader* out; + ColArgs* args; + binary32* results; + uint32_t written, i; + int fd; + binary32 tmp; + binary32* bin32 = (binary32*)block; + + for (int i = 0; i < BLOCK_SIZE / 32; i++) + { + bin32[i].uint64(0) = 0; + } + + bin32[0].uint64(0) = 10UL; + + bin32[1].uint64(0) = 1000UL; + + bin32[3].uint64(0) = 1000UL; + bin32[3].uint64(1) = 1; + + bin32[4].uint64(0) = 256; + bin32[4].uint64(1) = 254; + bin32[4].uint64(2) = 253; + bin32[4].uint64(3) = 252; + + typedef char bin32_t[32]; + + *(uint64_t*)(((bin32_t*)block) + 5) = 500; + + *(uint64_t*)&((bin32_t*)block)[6] = 501; + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = sizeof(binary32); + in->DataType = execplan::CalpontSystemCatalog::BINARY; + in->OutputType = OT_DATAVALUE; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast(args->val + in->DataSize); + + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + args = reinterpret_cast(args->val + in->DataSize); + tmp.uint64(0) = 256; + tmp.uint64(1) = 254; + tmp.uint64(2) = 253; + tmp.uint64(3) = 252; + + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*)block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(ColResultHeader)]); + // cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); + // CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); + // CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); + for (i = 0; i < out->NVALS; i++) + { + printf("Result %d Value %016X%016X%016X%016X\n", i, results[i].uint64(3), results[i].uint64(2), + results[i].uint64(1), results[i].uint64(0)); + // CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + } + } + + void p_Dictionary_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_EQ; + args->len = 5; + strncpy((char*)args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + outValue = reinterpret_cast(&results[1]); + outValue->data[args->len] = '\0'; // not reusable in tests with multiple matches. + CPPUNIT_ASSERT(results->NVALS == 1); + CPPUNIT_ASSERT(outValue->len == 5); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); + } + + void p_Dictionary_2() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 2; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_OR; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_EQ; + args->len = 13; + strncpy((char*)args->data, "UNITED STATES", 13); + + args = reinterpret_cast( + &input[sizeof(DictInput) + args->len + sizeof(DictFilterElement)]); + args->COP = COMPARE_EQ; + args->len = 5; + strncpy((char*)args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + outValue = reinterpret_cast(&results[1]); + CPPUNIT_ASSERT(results->NVALS == 2); + CPPUNIT_ASSERT(outValue->len == 5); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); + + outValue = reinterpret_cast(&output[sizeof(DictOutput) + sizeof(DataValue) + outValue->len]); + CPPUNIT_ASSERT(outValue->len == 13); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "UNITED STATES", 13) == 0); + } + + void p_Dictionary_3() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 0; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 50); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + // cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_gt_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_GT; + args->len = 7; + strncpy((char*)args->data, "GERMANY", 7); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 41); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + // cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_token_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + PrimToken* outToken; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 2; + cmd->OutputType = OT_TOKEN; + cmd->BOP = BOP_OR; + cmd->LBID = 56878; // random magic + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_EQ; + args->len = 13; + strncpy((char*)args->data, "UNITED STATES", 13); + + args = reinterpret_cast( + &input[sizeof(DictInput) + args->len + sizeof(DictFilterElement)]); + args->COP = COMPARE_EQ; + args->len = 5; + strncpy((char*)args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + CPPUNIT_ASSERT(results->NVALS == 2); + + outToken = reinterpret_cast(&results[1]); + CPPUNIT_ASSERT(outToken->len == 5); + CPPUNIT_ASSERT(outToken->LBID == cmd->LBID); + CPPUNIT_ASSERT(outToken->offset == 19); + CPPUNIT_ASSERT(outToken[1].len == 13); + CPPUNIT_ASSERT(outToken[1].LBID == cmd->LBID); + CPPUNIT_ASSERT(outToken[1].offset == 50); + } + + void p_Dictionary_inputArg_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 2; + cmd->OutputType = OT_INPUTARG; + cmd->BOP = BOP_OR; + cmd->LBID = 56878; // random magic + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_EQ; + args->len = 13; + strncpy((char*)args->data, "UNITED STATES", 13); + + args = reinterpret_cast( + &input[sizeof(DictInput) + args->len + sizeof(DictFilterElement)]); + args->COP = COMPARE_EQ; + args->len = 5; + strncpy((char*)args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + CPPUNIT_ASSERT(results->NVALS == 2); + + outValue = reinterpret_cast(&results[1]); + CPPUNIT_ASSERT(results->NVALS == 2); + CPPUNIT_ASSERT(outValue->len == 5); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); + + outValue = reinterpret_cast(&output[sizeof(DictOutput) + sizeof(DataValue) + outValue->len]); + CPPUNIT_ASSERT(outValue->len == 13); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "UNITED STATES", 13) == 0); + } + + void p_Dictionary_token_agg_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + PrimToken* outToken; + DictAggregate* agg; + DataValue* outValue; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 2; + cmd->OutputType = OT_TOKEN | OT_AGGREGATE; + cmd->BOP = BOP_OR; + cmd->LBID = 56878; // random magic + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_EQ; + args->len = 13; + strncpy((char*)args->data, "UNITED STATES", 13); + + args = reinterpret_cast( + &input[sizeof(DictInput) + args->len + sizeof(DictFilterElement)]); + args->COP = COMPARE_EQ; + args->len = 5; + strncpy((char*)args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + CPPUNIT_ASSERT(results->NVALS == 2); + + outToken = reinterpret_cast(&results[1]); + CPPUNIT_ASSERT(outToken->len == 5); + CPPUNIT_ASSERT(outToken->LBID == cmd->LBID); + CPPUNIT_ASSERT(outToken->offset == 19); + CPPUNIT_ASSERT(outToken[1].len == 13); + CPPUNIT_ASSERT(outToken[1].LBID == cmd->LBID); + CPPUNIT_ASSERT(outToken[1].offset == 50); + + agg = reinterpret_cast(&outToken[2]); + CPPUNIT_ASSERT(agg->Count == 50); + + // min follows + outValue = reinterpret_cast(&agg[1]); + CPPUNIT_ASSERT(outValue->len == 7); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "ALGERIA", 7) == 0); + // max follows + outValue = reinterpret_cast(((int8_t*)outValue) + sizeof(DataValue) + outValue->len); + CPPUNIT_ASSERT(outValue->len == 6); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XUSSIA", 6) == 0); + } + + void p_Dictionary_inToken_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DictAggregate* agg; + PrimToken* outToken; + DataValue* outValue; + int fd; + uint32_t err; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(DictInput) + 3 * sizeof(PrimToken)]); + + cmd->NOPS = 2; + cmd->OutputType = OT_TOKEN | OT_AGGREGATE; + cmd->BOP = BOP_OR; + cmd->LBID = 56878; // random magic + cmd->NVALS = 3; + cmd->InputFlags = 0; + cmd->tokens[0].LBID = 56878; + cmd->tokens[0].offset = 7838; // UNITED STATES + cmd->tokens[0].len = 13; + cmd->tokens[1].LBID = 56878; + cmd->tokens[1].offset = 7890; // ROMANIA + cmd->tokens[1].len = 7; + cmd->tokens[2].LBID = 56878; + cmd->tokens[2].offset = 8074; // XHINA + cmd->tokens[2].len = 5; + + args->COP = COMPARE_EQ; + args->len = 13; + strncpy((char*)args->data, "UNITED STATES", 13); + + args = reinterpret_cast( + &input[sizeof(DictInput) + 3 * sizeof(PrimToken) + args->len + sizeof(DictFilterElement)]); + args->COP = COMPARE_EQ; + args->len = 5; + strncpy((char*)args->data, "XHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + CPPUNIT_ASSERT(results->NVALS == 2); + + outToken = reinterpret_cast(&results[1]); + CPPUNIT_ASSERT(outToken[0].len == 13); + CPPUNIT_ASSERT(outToken[0].LBID == cmd->LBID); + CPPUNIT_ASSERT(outToken[0].offset == 50); + CPPUNIT_ASSERT(outToken[1].len == 5); + CPPUNIT_ASSERT(outToken[1].LBID == cmd->LBID); + CPPUNIT_ASSERT(outToken[1].offset == 19); + + agg = reinterpret_cast(&outToken[2]); + CPPUNIT_ASSERT(agg->Count == 3); + + // min follows + outValue = reinterpret_cast(&agg[1]); + CPPUNIT_ASSERT(outValue->len == 7); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "ROMANIA", 7) == 0); + // max follows + outValue = reinterpret_cast(((int8_t*)outValue) + sizeof(DataValue) + outValue->len); + CPPUNIT_ASSERT(outValue->len == 5); + CPPUNIT_ASSERT(strncmp((char*)outValue->data, "XHINA", 5) == 0); + } + + void p_Dictionary_oldgetsig_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + +#pragma pack(push, 1) + struct OldSigArgs + { + uint64_t rid; + uint16_t offsetIndex; + } * args; #pragma pack(pop) - struct OldResults - { - uint64_t rid; - DataValue value; - } *oldResults; - - int fd, argsOffset; - uint32_t err; -// char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 0; - cmd->OutputType = OT_RID | OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 2; - cmd->InputFlags = 1; - args[0].rid = 0x1234567891011121LL; - args[0].offsetIndex = 48; // RUSSIA - args[1].rid = 0x1121110987654321LL; - args[1].offsetIndex = 38; // JAPAN - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << " nvals = " << results->NVALS << endl; - - CPPUNIT_ASSERT(results->NVALS == 2); - argsOffset = sizeof(DictOutput); - - oldResults = reinterpret_cast(&output[argsOffset]); -// strncpy(tmp, oldResults->value.data, oldResults->value.len); -// tmp[oldResults->value.len] = '\0'; -// cout << " oldResults.rid = " << oldResults->rid << endl; -// cout << " oldResults.value = " << tmp << endl; - CPPUNIT_ASSERT(oldResults->value.len == 6); - CPPUNIT_ASSERT(strncmp(oldResults->value.data, "RUSSIA", 6) == 0); - CPPUNIT_ASSERT(oldResults->rid == 0x1234567891011121LL); - argsOffset += 8 + 2 + oldResults->value.len; - - - oldResults = reinterpret_cast(&output[argsOffset]); -// strncpy(tmp, oldResults->value.data, oldResults->value.len); -// tmp[oldResults->value.len] = '\0'; -// cout << " oldResults.rid = " << oldResults->rid << endl; -// cout << " oldResults.value = " << tmp << endl; - CPPUNIT_ASSERT(oldResults->value.len == 5); - CPPUNIT_ASSERT(strncmp(oldResults->value.data, "JAPAN", 5) == 0); - CPPUNIT_ASSERT(oldResults->rid == 0x1121110987654321LL); - } - - void p_Dictionary_like_1() + struct OldResults { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; + uint64_t rid; + DataValue value; + } * oldResults; - fd = open(filename.c_str(), O_RDONLY); + int fd, argsOffset; + uint32_t err; + // char tmp[BLOCK_SIZE]; - if (fd < 0) - { - perror("open"); - return; - } + fd = open(filename.c_str(), O_RDONLY); - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 5; - strncpy((char*)args->data, "%HINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 2); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - -// cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; - if (i == 0) - CPPUNIT_ASSERT(strncmp(tmp, "XHINA", 5) == 0); - else if (i == 1) - CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); - - argsOffset += sizeof(DataValue) + outValue->len; - } - } - - void p_Dictionary_like_2() + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 5; - strncpy((char*)args->data, "%HIN%", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 2); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - -// cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; - if (i == 0) - CPPUNIT_ASSERT(strncmp(tmp, "XHINA", 5) == 0); - else if (i == 1) - CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); - - argsOffset += sizeof(DataValue) + outValue->len; - } + perror("open"); + return; } - void p_Dictionary_like_3() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 4; - strncpy((char*)args->data, "CHI%", 4); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 1); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); - argsOffset += sizeof(DataValue) + outValue->len; - } + perror("read"); + return; } - void p_Dictionary_like_4() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 5; - strncpy((char*)args->data, "CHINA", 5); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 1); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); - argsOffset += sizeof(DataValue) + outValue->len; - } + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Dictionary_like_5() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 0; + cmd->OutputType = OT_RID | OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 2; + cmd->InputFlags = 1; + args[0].rid = 0x1234567891011121LL; + args[0].offsetIndex = 48; // RUSSIA + args[1].rid = 0x1121110987654321LL; + args[1].offsetIndex = 38; // JAPAN + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << " nvals = " << results->NVALS << endl; + + CPPUNIT_ASSERT(results->NVALS == 2); + argsOffset = sizeof(DictOutput); + + oldResults = reinterpret_cast(&output[argsOffset]); + // strncpy(tmp, oldResults->value.data, oldResults->value.len); + // tmp[oldResults->value.len] = '\0'; + // cout << " oldResults.rid = " << oldResults->rid << endl; + // cout << " oldResults.value = " << tmp << endl; + CPPUNIT_ASSERT(oldResults->value.len == 6); + CPPUNIT_ASSERT(strncmp(oldResults->value.data, "RUSSIA", 6) == 0); + CPPUNIT_ASSERT(oldResults->rid == 0x1234567891011121LL); + argsOffset += 8 + 2 + oldResults->value.len; + + oldResults = reinterpret_cast(&output[argsOffset]); + // strncpy(tmp, oldResults->value.data, oldResults->value.len); + // tmp[oldResults->value.len] = '\0'; + // cout << " oldResults.rid = " << oldResults->rid << endl; + // cout << " oldResults.value = " << tmp << endl; + CPPUNIT_ASSERT(oldResults->value.len == 5); + CPPUNIT_ASSERT(strncmp(oldResults->value.data, "JAPAN", 5) == 0); + CPPUNIT_ASSERT(oldResults->rid == 0x1121110987654321LL); + } + + void p_Dictionary_like_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 7; - strncpy((char*)args->data, "_NDO%A%", 7); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 2); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - -// cout << " " << tmp << endl; - if (i == 0) - CPPUNIT_ASSERT(strncmp(tmp, "XNDONESIA", 9) == 0); - else - CPPUNIT_ASSERT(strncmp(tmp, "INDONESIA", 9) == 0); - - argsOffset += sizeof(DataValue) + outValue->len; - } + perror("open"); + return; } - void p_Dictionary_like_6() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 11; - strncpy((char*)args->data, "%NIT%ING%D%", 11); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 2); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - -// cout << " " << tmp << endl; - if (i == 0) - CPPUNIT_ASSERT(strncmp(tmp, "XNITED KINGDOM", 14) == 0); - else - CPPUNIT_ASSERT(strncmp(tmp, "UNITED KINGDOM", 14) == 0); - - argsOffset += sizeof(DataValue) + outValue->len; - } + perror("read"); + return; } - void p_Dictionary_like_7() + if (err != BLOCK_SIZE) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 7; - strncpy((char*)args->data, "UNI%TES", 7); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 1); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; -// cout << " " << tmp << endl; - CPPUNIT_ASSERT(strncmp(tmp, "UNITED STATES", 13) == 0); - argsOffset += sizeof(DataValue) + outValue->len; - } + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; } - void p_Dictionary_like_8() + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 5; + strncpy((char*)args->data, "%HINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 2); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - DataValue* outValue; - int fd, argsOffset; - uint32_t err, i; - char tmp[BLOCK_SIZE]; + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; - fd = open(filename.c_str(), O_RDONLY); + // cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; + if (i == 0) + CPPUNIT_ASSERT(strncmp(tmp, "XHINA", 5) == 0); + else if (i == 1) + CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 7; - strncpy((char*)args->data, "%TH_OP%", 7); - - pp.setBlockPtr((int*) block); - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - -// cout << "NVALS = " << results->NVALS << endl; - CPPUNIT_ASSERT(results->NVALS == 2); - - argsOffset = sizeof(DictOutput); - - for (i = 0; i < results->NVALS; i++) - { - outValue = reinterpret_cast(&output[argsOffset]); - strncpy((char*) tmp, outValue->data, outValue->len); - tmp[outValue->len] = '\0'; - -// cout << " " << tmp << endl; - if (i == 0) - CPPUNIT_ASSERT(strncmp(tmp, "XTHIOPIA", 8) == 0); - else - CPPUNIT_ASSERT(strncmp(tmp, "ETHIOPIA", 8) == 0); - - argsOffset += sizeof(DataValue) + outValue->len; - } + argsOffset += sizeof(DataValue) + outValue->len; } + } - void p_Dictionary_like_prefixbench_1() + void p_Dictionary_like_2() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - int fd; - uint32_t err; - uint64_t count; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 4; - strncpy((char*)args->data, "CHI%", 4); - - pp.setBlockPtr((int*) block); - - done = 0; - count = 0; - signal(SIGALRM, alarm_handler); - alarm(30); - - while (!done) - { - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - count++; - } - - cout << endl << " - did a LIKE comparison looking for CHI%, " << count << " times in 30 seconds" << endl; + perror("open"); + return; } - void p_Dictionary_like_substrbench_1() + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) { - PrimitiveProcessor pp; - uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; - string filename("dictblock.cdf"); - DictInput* cmd; - DictOutput* results; - DictFilterElement* args; - int fd; - uint32_t err; - uint64_t count; - - fd = open(filename.c_str(), O_RDONLY); - - if (fd < 0) - { - perror("open"); - return; - } - - err = read(fd, block, BLOCK_SIZE); - - if (err < 0) - { - perror("read"); - return; - } - - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - return; - } - - close(fd); - - cmd = reinterpret_cast(input); - results = reinterpret_cast(output); - args = reinterpret_cast(&cmd[1]); - - cmd->NOPS = 1; - cmd->OutputType = OT_DATAVALUE; - cmd->BOP = BOP_NONE; - cmd->NVALS = 0; - cmd->InputFlags = 0; - args->COP = COMPARE_LIKE; - args->len = 4; - strncpy((char*)args->data, "%HI%", 4); - - pp.setBlockPtr((int*) block); - - done = 0; - count = 0; - signal(SIGALRM, alarm_handler); - alarm(30); - - while (!done) - { - pp.p_Dictionary(cmd, results, BLOCK_SIZE); - count++; - } - - cout << endl << " - did a LIKE comparison looking for %HI%, " << count << " times in 30 seconds" << endl; + perror("read"); + return; } + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 5; + strncpy((char*)args->data, "%HIN%", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 2); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + + // cout << " " << i << ": len=" << outValue->len << " data=" << tmp << endl; + if (i == 0) + CPPUNIT_ASSERT(strncmp(tmp, "XHINA", 5) == 0); + else if (i == 1) + CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); + + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_3() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 4; + strncpy((char*)args->data, "CHI%", 4); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 1); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_4() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 5; + strncpy((char*)args->data, "CHINA", 5); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 1); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + CPPUNIT_ASSERT(strncmp(tmp, "CHINA", 5) == 0); + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_5() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 7; + strncpy((char*)args->data, "_NDO%A%", 7); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 2); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + + // cout << " " << tmp << endl; + if (i == 0) + CPPUNIT_ASSERT(strncmp(tmp, "XNDONESIA", 9) == 0); + else + CPPUNIT_ASSERT(strncmp(tmp, "INDONESIA", 9) == 0); + + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_6() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 11; + strncpy((char*)args->data, "%NIT%ING%D%", 11); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 2); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + + // cout << " " << tmp << endl; + if (i == 0) + CPPUNIT_ASSERT(strncmp(tmp, "XNITED KINGDOM", 14) == 0); + else + CPPUNIT_ASSERT(strncmp(tmp, "UNITED KINGDOM", 14) == 0); + + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_7() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 7; + strncpy((char*)args->data, "UNI%TES", 7); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 1); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + // cout << " " << tmp << endl; + CPPUNIT_ASSERT(strncmp(tmp, "UNITED STATES", 13) == 0); + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_8() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + DataValue* outValue; + int fd, argsOffset; + uint32_t err, i; + char tmp[BLOCK_SIZE]; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 7; + strncpy((char*)args->data, "%TH_OP%", 7); + + pp.setBlockPtr((int*)block); + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + + // cout << "NVALS = " << results->NVALS << endl; + CPPUNIT_ASSERT(results->NVALS == 2); + + argsOffset = sizeof(DictOutput); + + for (i = 0; i < results->NVALS; i++) + { + outValue = reinterpret_cast(&output[argsOffset]); + strncpy((char*)tmp, outValue->data, outValue->len); + tmp[outValue->len] = '\0'; + + // cout << " " << tmp << endl; + if (i == 0) + CPPUNIT_ASSERT(strncmp(tmp, "XTHIOPIA", 8) == 0); + else + CPPUNIT_ASSERT(strncmp(tmp, "ETHIOPIA", 8) == 0); + + argsOffset += sizeof(DataValue) + outValue->len; + } + } + + void p_Dictionary_like_prefixbench_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + int fd; + uint32_t err; + uint64_t count; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 4; + strncpy((char*)args->data, "CHI%", 4); + + pp.setBlockPtr((int*)block); + + done = 0; + count = 0; + signal(SIGALRM, alarm_handler); + alarm(30); + + while (!done) + { + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + count++; + } + + cout << endl << " - did a LIKE comparison looking for CHI%, " << count << " times in 30 seconds" << endl; + } + + void p_Dictionary_like_substrbench_1() + { + PrimitiveProcessor pp; + uint8_t block[BLOCK_SIZE], output[BLOCK_SIZE], input[BLOCK_SIZE]; + string filename("dictblock.cdf"); + DictInput* cmd; + DictOutput* results; + DictFilterElement* args; + int fd; + uint32_t err; + uint64_t count; + + fd = open(filename.c_str(), O_RDONLY); + + if (fd < 0) + { + perror("open"); + return; + } + + err = read(fd, block, BLOCK_SIZE); + + if (err < 0) + { + perror("read"); + return; + } + + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + return; + } + + close(fd); + + cmd = reinterpret_cast(input); + results = reinterpret_cast(output); + args = reinterpret_cast(&cmd[1]); + + cmd->NOPS = 1; + cmd->OutputType = OT_DATAVALUE; + cmd->BOP = BOP_NONE; + cmd->NVALS = 0; + cmd->InputFlags = 0; + args->COP = COMPARE_LIKE; + args->len = 4; + strncpy((char*)args->data, "%HI%", 4); + + pp.setBlockPtr((int*)block); + + done = 0; + count = 0; + signal(SIGALRM, alarm_handler); + alarm(30); + + while (!done) + { + pp.p_Dictionary(cmd, results, BLOCK_SIZE); + count++; + } + + cout << endl << " - did a LIKE comparison looking for %HI%, " << count << " times in 30 seconds" << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( PrimTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(PrimTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/primitives/linux-port/primitiveprocessor.cpp b/primitives/linux-port/primitiveprocessor.cpp index 09fb04029..fe746ccae 100644 --- a/primitives/linux-port/primitiveprocessor.cpp +++ b/primitives/linux-port/primitiveprocessor.cpp @@ -31,47 +31,42 @@ using namespace std; namespace primitives { - -PrimitiveProcessor::PrimitiveProcessor(int debugLevel) : - fDebugLevel(debugLevel), fStatsPtr(NULL), logicalBlockMode(false) +PrimitiveProcessor::PrimitiveProcessor(int debugLevel) + : fDebugLevel(debugLevel), fStatsPtr(NULL), logicalBlockMode(false) { + // This does + // masks[11] = { 0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023 }; + int acc, i; -// This does -// masks[11] = { 0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023 }; - int acc, i; - - for (acc = 0, i = 0; i < 11; i++) - { - masks[i] = acc; - acc = acc << 1 | 1; - } - + for (acc = 0, i = 0; i < 11; i++) + { + masks[i] = acc; + acc = acc << 1 | 1; + } } PrimitiveProcessor::~PrimitiveProcessor() { } - void PrimitiveProcessor::setParsedColumnFilter(boost::shared_ptr pcf) { - parsedColumnFilter = pcf; + parsedColumnFilter = pcf; } -ParsedColumnFilter::ParsedColumnFilter() : columnFilterMode(ALWAYS_TRUE), mFilterCount(0) +ParsedColumnFilter::ParsedColumnFilter() : columnFilterMode(ALWAYS_TRUE), mFilterCount(0) { } ParsedColumnFilter::ParsedColumnFilter(const uint32_t aFilterCount, const int BOP) - : columnFilterMode(ALWAYS_TRUE), mFilterCount(aFilterCount), mBOP(BOP) + : columnFilterMode(ALWAYS_TRUE), mFilterCount(aFilterCount), mBOP(BOP) { - prestored_rfs.reset(new uint8_t[mFilterCount]); - prestored_cops.reset(new uint8_t[mFilterCount]); + prestored_rfs.reset(new uint8_t[mFilterCount]); + prestored_cops.reset(new uint8_t[mFilterCount]); } ParsedColumnFilter::~ParsedColumnFilter() { } -} // namespace primitives - +} // namespace primitives diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index aaa4f9d00..baaf054f4 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -57,80 +57,79 @@ class PrimTest; namespace primitives { - enum ColumnFilterMode { - ALWAYS_TRUE, // empty filter is always true - SINGLE_COMPARISON, // exactly one comparison operation - ANY_COMPARISON_TRUE, // ANY comparison is true (BOP_OR) - ALL_COMPARISONS_TRUE, // ALL comparisons are true (BOP_AND) - XOR_COMPARISONS, // XORing results of comparisons (BOP_XOR) - ONE_OF_VALUES_IN_SET, // ONE of the values in the set is equal to the value checked (BOP_OR + all COMPARE_EQ) - NONE_OF_VALUES_IN_SET, // NONE of the values in the set is equal to the value checked (BOP_AND + all COMPARE_NE) - ONE_OF_VALUES_IN_ARRAY, // ONE of the values in the small set represented by an array (BOP_OR + all COMPARE_EQ) - NONE_OF_VALUES_IN_ARRAY,// NONE of the values in the small set represented by an array (BOP_AND + all COMPARE_NE) + ALWAYS_TRUE, // empty filter is always true + SINGLE_COMPARISON, // exactly one comparison operation + ANY_COMPARISON_TRUE, // ANY comparison is true (BOP_OR) + ALL_COMPARISONS_TRUE, // ALL comparisons are true (BOP_AND) + XOR_COMPARISONS, // XORing results of comparisons (BOP_XOR) + ONE_OF_VALUES_IN_SET, // ONE of the values in the set is equal to the value checked (BOP_OR + all + // COMPARE_EQ) + NONE_OF_VALUES_IN_SET, // NONE of the values in the set is equal to the value checked (BOP_AND + all + // COMPARE_NE) + ONE_OF_VALUES_IN_ARRAY, // ONE of the values in the small set represented by an array (BOP_OR + all + // COMPARE_EQ) + NONE_OF_VALUES_IN_ARRAY, // NONE of the values in the small set represented by an array (BOP_AND + all + // COMPARE_NE) }; // TBD Test if avalance makes lookup in the hash maps based on this hashers faster. class pcfHasher { -public: - inline size_t operator()(const int64_t i) const - { - return i; - } + public: + inline size_t operator()(const int64_t i) const + { + return i; + } }; class pcfEqual { -public: - inline size_t operator()(const int64_t f1, const int64_t f2) const - { - return f1 == f2; - } + public: + inline size_t operator()(const int64_t f1, const int64_t f2) const + { + return f1 == f2; + } }; class pcfHasher128 { -public: - inline size_t operator()(const int128_t i) const - { - return *reinterpret_cast(&i); - } + public: + inline size_t operator()(const int128_t i) const + { + return *reinterpret_cast(&i); + } }; class pcfEqual128 { -public: - inline bool operator()(const int128_t f1, const int128_t f2) const - { - return f1 == f2; - } + public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } }; // TBD Test robinhood as tr1 set replacement. typedef std::tr1::unordered_set prestored_set_t; typedef std::tr1::unordered_set prestored_set_t_128; - -class DictEqualityFilter: public std::tr1::unordered_set +class DictEqualityFilter : public std::tr1::unordered_set { -public: - DictEqualityFilter(const datatypes::Charset &cs) - :std::tr1::unordered_set - (10, - datatypes::CollationAwareHasher(cs), - datatypes::CollationAwareComparator(cs)) - { } - CHARSET_INFO & getCharset() const - { - idbassert(& _M_h1.getCharset() == & _M_eq.getCharset()); - return _M_h1.getCharset(); - } + public: + DictEqualityFilter(const datatypes::Charset& cs) + : std::tr1::unordered_set(10, datatypes::CollationAwareHasher(cs), + datatypes::CollationAwareComparator(cs)) + { + } + CHARSET_INFO& getCharset() const + { + idbassert(&_M_h1.getCharset() == &_M_eq.getCharset()); + return _M_h1.getCharset(); + } }; // Not the safest way b/c it doesn't cover uint128_t but the type @@ -138,25 +137,25 @@ public: template struct IntegralTypeToFilterType { - using type = int64_t; + using type = int64_t; }; -template<> +template <> struct IntegralTypeToFilterType { - using type = int128_t; + using type = int128_t; }; template struct IntegralTypeToFilterSetType { - using type = prestored_set_t; + using type = prestored_set_t; }; -template<> +template <> struct IntegralTypeToFilterSetType { - using type = prestored_set_t_128; + using type = prestored_set_t_128; }; // DRRTuy shared_arrays and shared_ptr looks redundant here @@ -164,124 +163,114 @@ struct IntegralTypeToFilterSetType // thus runtime calls ParsedColumnFilter dtor in the end. class ParsedColumnFilter { - public: - using CopsType = uint8_t; - using RFsType = uint8_t; - static constexpr uint32_t noSetFilterThreshold = 8; - ColumnFilterMode columnFilterMode; - // Very unfortunately prestored_argVals can also be used to store double/float values. - boost::shared_array prestored_argVals; - boost::shared_array prestored_argVals128; - boost::shared_array prestored_cops; - boost::shared_array prestored_rfs; - boost::shared_ptr prestored_set; - boost::shared_ptr prestored_set_128; + public: + using CopsType = uint8_t; + using RFsType = uint8_t; + static constexpr uint32_t noSetFilterThreshold = 8; + ColumnFilterMode columnFilterMode; + // Very unfortunately prestored_argVals can also be used to store double/float values. + boost::shared_array prestored_argVals; + boost::shared_array prestored_argVals128; + boost::shared_array prestored_cops; + boost::shared_array prestored_rfs; + boost::shared_ptr prestored_set; + boost::shared_ptr prestored_set_128; - ParsedColumnFilter(); - ParsedColumnFilter(const uint32_t aFilterCount, const int BOP); - ~ParsedColumnFilter(); + ParsedColumnFilter(); + ParsedColumnFilter(const uint32_t aFilterCount, const int BOP); + ~ParsedColumnFilter(); - template::value, T>::type* = nullptr> - T* getFilterVals() - { - return reinterpret_cast(prestored_argVals.get()); - } + template ::value, T>::type* = nullptr> + T* getFilterVals() + { + return reinterpret_cast(prestored_argVals.get()); + } - template::value, T>::type* = nullptr> - T* getFilterVals() - { - return prestored_argVals128.get(); - } + template ::value, T>::type* = nullptr> + T* getFilterVals() + { + return prestored_argVals128.get(); + } - template::value, T>::type* = nullptr> - T* getFilterSet() - { - return prestored_set.get(); - } + template ::value, T>::type* = nullptr> + T* getFilterSet() + { + return prestored_set.get(); + } - template::value, T>::type* = nullptr> - T* getFilterSet() - { - return prestored_set_128.get(); - } + template ::value, T>::type* = nullptr> + T* getFilterSet() + { + return prestored_set_128.get(); + } - template::type* = nullptr> - void storeFilterArg(const uint32_t argIndex, const T* argValPtr) - { - prestored_argVals[argIndex] = *argValPtr; - } + template ::type* = nullptr> + void storeFilterArg(const uint32_t argIndex, const T* argValPtr) + { + prestored_argVals[argIndex] = *argValPtr; + } - template::type* = nullptr> - void storeFilterArg(const uint32_t argIndex, const WT* argValPtr) - { - datatypes::TSInt128::assignPtrPtr(&(prestored_argVals128[argIndex]), - argValPtr); - } + template ::type* = nullptr> + void storeFilterArg(const uint32_t argIndex, const WT* argValPtr) + { + datatypes::TSInt128::assignPtrPtr(&(prestored_argVals128[argIndex]), argValPtr); + } - template::type* = nullptr> - void allocateSpaceForFilterArgs() - { - prestored_argVals.reset(new int64_t[mFilterCount]); - } + template ::type* = nullptr> + void allocateSpaceForFilterArgs() + { + prestored_argVals.reset(new int64_t[mFilterCount]); + } - template::type* = nullptr> - void allocateSpaceForFilterArgs() - { - prestored_argVals128.reset(new int128_t[mFilterCount]); - } + template ::type* = nullptr> + void allocateSpaceForFilterArgs() + { + prestored_argVals128.reset(new int128_t[mFilterCount]); + } - template::type* = nullptr> - void populatePrestoredSet() - { - prestored_set_128.reset(new prestored_set_t_128()); + template ::type* = nullptr> + void populatePrestoredSet() + { + prestored_set_128.reset(new prestored_set_t_128()); - // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression - for (uint32_t argIndex = 0; argIndex < mFilterCount; ++argIndex) - if (prestored_rfs[argIndex] == 0) - prestored_set_128->insert(prestored_argVals128[argIndex]); - } + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (uint32_t argIndex = 0; argIndex < mFilterCount; ++argIndex) + if (prestored_rfs[argIndex] == 0) + prestored_set_128->insert(prestored_argVals128[argIndex]); + } - template::type* = nullptr> - void populatePrestoredSet() - { - prestored_set.reset(new prestored_set_t()); + template ::type* = nullptr> + void populatePrestoredSet() + { + prestored_set.reset(new prestored_set_t()); - // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression - for (uint32_t argIndex = 0; argIndex < mFilterCount; argIndex++) - if (prestored_rfs[argIndex] == 0) - prestored_set->insert(prestored_argVals[argIndex]); - } + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (uint32_t argIndex = 0; argIndex < mFilterCount; argIndex++) + if (prestored_rfs[argIndex] == 0) + prestored_set->insert(prestored_argVals[argIndex]); + } - inline int getBOP() const - { - return mBOP; - } + inline int getBOP() const + { + return mBOP; + } - inline int getFilterCount() const - { - return mFilterCount; - } + inline int getFilterCount() const + { + return mFilterCount; + } - private: - uint32_t mFilterCount; - int mBOP; + private: + uint32_t mFilterCount; + int mBOP; }; //@bug 1828 These need to be public so that column operations can use it for 'like' struct p_DataValue { - int len; - const uint8_t* data; + int len; + const uint8_t* data; }; /** @brief This class encapsulates the primitive processing functionality of the system. @@ -290,304 +279,289 @@ struct p_DataValue */ class PrimitiveProcessor { -public: - PrimitiveProcessor(int debugLevel = 0); - virtual ~PrimitiveProcessor(); + public: + PrimitiveProcessor(int debugLevel = 0); + virtual ~PrimitiveProcessor(); - /** @brief Sets the block to operate on - * - * The primitive processing functions operate on one block at a time. The caller - * sets which block to operate on next with this function. - */ - void setBlockPtr(int* data) - { - block = data; - } - void setPMStatsPtr(dbbc::Stats* p) - { - fStatsPtr = p; - } + /** @brief Sets the block to operate on + * + * The primitive processing functions operate on one block at a time. The caller + * sets which block to operate on next with this function. + */ + void setBlockPtr(int* data) + { + block = data; + } + void setPMStatsPtr(dbbc::Stats* p) + { + fStatsPtr = p; + } + /** @brief The interface to Mark's NIOS primitive processing code. + * + * The interface to Mark's NIOS primitive processing code. Instead of reading + * and writing to a bus, it will read/write to buffers specified by inBuf + * and outBuf. The primitives implemented this way are: + * - p_Col and p_ColAggregate + * - p_GetSignature + * + * @param inBuf (in) The buffer containing a command to execute + * @param inLength (in) The size of inBuf in 4-byte words + * @param outBuf (in) The buffer to store the output in + * @param outLength (in) The size of outBuf in 4-byte words + * @param written (out) The number of bytes written to outBuf. + * @note Throws logic_error if the output buffer is too small for the result. + */ + void processBuffer(int* inBuf, unsigned inLength, int* outBuf, unsigned outLength, unsigned* written); - /** @brief The interface to Mark's NIOS primitive processing code. - * - * The interface to Mark's NIOS primitive processing code. Instead of reading - * and writing to a bus, it will read/write to buffers specified by inBuf - * and outBuf. The primitives implemented this way are: - * - p_Col and p_ColAggregate - * - p_GetSignature - * - * @param inBuf (in) The buffer containing a command to execute - * @param inLength (in) The size of inBuf in 4-byte words - * @param outBuf (in) The buffer to store the output in - * @param outLength (in) The size of outBuf in 4-byte words - * @param written (out) The number of bytes written to outBuf. - * @note Throws logic_error if the output buffer is too small for the result. - */ - void processBuffer(int* inBuf, unsigned inLength, int* outBuf, unsigned outLength, - unsigned* written); + /* Patrick */ - /* Patrick */ + /** @brief The p_TokenByScan primitive processor + * + * The p_TokenByScan primitive processor. It relies on the caller setting + * the block to operate on with setBlockPtr(). It assumes the continuation + * pointer is not used. + * @param t (in) The arguments to the primitive + * @param out (out) This must point to memory of some currently unknown max size + * @param outSize (in) The size of the output buffer in bytes. + * @note Throws logic_error if the output buffer is too small for the result. + */ + void p_TokenByScan(const TokenByScanRequestHeader* t, TokenByScanResultHeader* out, unsigned outSize, + boost::shared_ptr eqFilter); - /** @brief The p_TokenByScan primitive processor - * - * The p_TokenByScan primitive processor. It relies on the caller setting - * the block to operate on with setBlockPtr(). It assumes the continuation - * pointer is not used. - * @param t (in) The arguments to the primitive - * @param out (out) This must point to memory of some currently unknown max size - * @param outSize (in) The size of the output buffer in bytes. - * @note Throws logic_error if the output buffer is too small for the result. - */ - void p_TokenByScan(const TokenByScanRequestHeader* t, - TokenByScanResultHeader* out, unsigned outSize, - boost::shared_ptr eqFilter); + /** @brief The p_IdxWalk primitive processor + * + * The p_IdxWalk primitive processor. The caller must set the block to operate + * on with setBlockPtr(). This primitive can return intermediate results. + * All results returned will have an different LBID than the input. They can + * also be in varying states of completion. A result is final when + * Shift >= SSlen, otherwise it is intermediate and needs to be reissued with + * the specified LBID loaded. + * @note If in->NVALS > 2, new vectors may be returned in the result set, which + * will have to be deleted by the caller. The test to use right now is + * ({element}->NVALS > 2 && {element}->State == 0). If that condition is true, + * delete the vector, otherwise don't. This kludginess is for efficiency's sake + * and may go away for the sake of sanity later. + * @note It is safe to delete any vector passed in after the call. + * @param out The caller should pass in an empty vector. The results + * will be returned as elements of this vector. + */ + void p_IdxWalk(const IndexWalkHeader* in, std::vector* out) throw(); - /** @brief The p_IdxWalk primitive processor - * - * The p_IdxWalk primitive processor. The caller must set the block to operate - * on with setBlockPtr(). This primitive can return intermediate results. - * All results returned will have an different LBID than the input. They can - * also be in varying states of completion. A result is final when - * Shift >= SSlen, otherwise it is intermediate and needs to be reissued with - * the specified LBID loaded. - * @note If in->NVALS > 2, new vectors may be returned in the result set, which - * will have to be deleted by the caller. The test to use right now is - * ({element}->NVALS > 2 && {element}->State == 0). If that condition is true, - * delete the vector, otherwise don't. This kludginess is for efficiency's sake - * and may go away for the sake of sanity later. - * @note It is safe to delete any vector passed in after the call. - * @param out The caller should pass in an empty vector. The results - * will be returned as elements of this vector. - */ - void p_IdxWalk(const IndexWalkHeader* in, std::vector* out) throw(); + /** @brief The p_IdxList primitive processor. + * + * The p_IdxList primitive processor. The caller must set the block to operate + * on with setBlockPtr(). This primitive can return one intermediate result + * for every call made. If there is an intermediate result returned, it will + * be the first element, distinguished by its type field. If the + * first element has a type == RID (3) , there is no intermediate result. If + * the first element had a type == LLP_SUBBLK (4) or type == LLP_BLK (5), + * that element is the intermediate result. Its value field will be a pointer + * to the next section of the list. + * + * @param rqst (in) The request header followed by NVALS IndexWalkParams + * @param rslt (out) The caller passes in a buffer which will be filled + * by the primitive on return. It will consist of an IndexListHeader, + * followed by NVALS IndexListEntrys. + * @param mode (optional, in) 0 specifies old behavior (the last entry of a block might + * be a pointer). 1 specifies new behavior (the last entry should be ignored). + */ + void p_IdxList(const IndexListHeader* rqst, IndexListHeader* rslt, int mode = 1); - /** @brief The p_IdxList primitive processor. - * - * The p_IdxList primitive processor. The caller must set the block to operate - * on with setBlockPtr(). This primitive can return one intermediate result - * for every call made. If there is an intermediate result returned, it will - * be the first element, distinguished by its type field. If the - * first element has a type == RID (3) , there is no intermediate result. If - * the first element had a type == LLP_SUBBLK (4) or type == LLP_BLK (5), - * that element is the intermediate result. Its value field will be a pointer - * to the next section of the list. - * - * @param rqst (in) The request header followed by NVALS IndexWalkParams - * @param rslt (out) The caller passes in a buffer which will be filled - * by the primitive on return. It will consist of an IndexListHeader, - * followed by NVALS IndexListEntrys. - * @param mode (optional, in) 0 specifies old behavior (the last entry of a block might - * be a pointer). 1 specifies new behavior (the last entry should be ignored). - */ - void p_IdxList(const IndexListHeader* rqst, IndexListHeader* rslt, int mode = 1); + /** @brief The p_Col primitive processor. + * + * The p_Col primitive processor. It operates on a column block specified using setBlockPtr(). + * @param in The buffer containing the command parameters. + * The buffer should begin with a NewColRequestHeader structure, followed by + * an array of 'NOPS' defining the filter to apply (optional), + * followed by an array of RIDs to apply the filter to (optional). + * @param out The buffer that will contain the results. On return, it will start with + * a ColResultHeader, followed by the output type specified by in->OutputType. + * \li If OT_RID, it will be an array of RIDs + * \li If OT_DATAVALUE, it will be an array of matching data values stored in the column + * \li If OT_BOTH, it will be an array of pairs + * @param outSize The size of the output buffer in bytes. + * @param written (out parameter) A pointer to 1 int, which will contain the + * number of bytes written to out. + * @note See PrimitiveMsg.h for the type definitions. + */ + void p_Col(NewColRequestHeader* in, ColResultHeader* out, unsigned outSize, unsigned* written); - /** @brief The p_Col primitive processor. - * - * The p_Col primitive processor. It operates on a column block specified using setBlockPtr(). - * @param in The buffer containing the command parameters. - * The buffer should begin with a NewColRequestHeader structure, followed by - * an array of 'NOPS' defining the filter to apply (optional), - * followed by an array of RIDs to apply the filter to (optional). - * @param out The buffer that will contain the results. On return, it will start with - * a ColResultHeader, followed by the output type specified by in->OutputType. - * \li If OT_RID, it will be an array of RIDs - * \li If OT_DATAVALUE, it will be an array of matching data values stored in the column - * \li If OT_BOTH, it will be an array of pairs - * @param outSize The size of the output buffer in bytes. - * @param written (out parameter) A pointer to 1 int, which will contain the - * number of bytes written to out. - * @note See PrimitiveMsg.h for the type definitions. - */ - void p_Col(NewColRequestHeader* in, ColResultHeader* out, unsigned outSize, - unsigned* written); + template ::type* = nullptr> + void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); - template::type* = nullptr> - void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); + template ::type* = nullptr> + void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); + template ::type* = nullptr> + void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); + template ::type* = nullptr> + void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); - template::type* = nullptr> - void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); - template::type* = nullptr> - void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); - template::type* = nullptr> - void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); + template ::type* = nullptr> + void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); - template::type* = nullptr> - void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out); - - template - void columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out); + template + void columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out); - boost::shared_ptr parseColumnFilter(const uint8_t* filterString, - uint32_t colWidth, - uint32_t colType, - uint32_t filterCount, - uint32_t BOP); - void setParsedColumnFilter(boost::shared_ptr); + boost::shared_ptr parseColumnFilter(const uint8_t* filterString, uint32_t colWidth, + uint32_t colType, uint32_t filterCount, + uint32_t BOP); + void setParsedColumnFilter(boost::shared_ptr); - /** @brief The p_ColAggregate primitive processor. - * - * The p_ColAggregate primitive processor. It operates on a column block - * specified using setBlockPtr(). - * @param in The buffer containing the command parameters. The buffer should begin - * with a NewColAggRequestHeader, followed by an array of RIDs to generate - * the data for (optional). - * @param out The buffer to put the result in. On return, it will contain a - * NewCollAggResultHeader. - * @note See PrimitiveMsg.h for the type definitions. - */ -// void p_ColAggregate(const NewColAggRequestHeader *in, NewColAggResultHeader *out); + /** @brief The p_ColAggregate primitive processor. + * + * The p_ColAggregate primitive processor. It operates on a column block + * specified using setBlockPtr(). + * @param in The buffer containing the command parameters. The buffer should begin + * with a NewColAggRequestHeader, followed by an array of RIDs to generate + * the data for (optional). + * @param out The buffer to put the result in. On return, it will contain a + * NewCollAggResultHeader. + * @note See PrimitiveMsg.h for the type definitions. + */ + // void p_ColAggregate(const NewColAggRequestHeader *in, NewColAggResultHeader *out); - void p_Dictionary(const DictInput* in, std::vector* out, - bool skipNulls, uint32_t charsetNumber, - boost::shared_ptr eqFilter, - uint8_t eqOp); + void p_Dictionary(const DictInput* in, std::vector* out, bool skipNulls, uint32_t charsetNumber, + boost::shared_ptr eqFilter, uint8_t eqOp); - inline void setLogicalBlockMode(bool b) - { - logicalBlockMode = b; - } + inline void setLogicalBlockMode(bool b) + { + logicalBlockMode = b; + } -private: - PrimitiveProcessor(const PrimitiveProcessor& rhs); - PrimitiveProcessor& operator=(const PrimitiveProcessor& rhs); + private: + PrimitiveProcessor(const PrimitiveProcessor& rhs); + PrimitiveProcessor& operator=(const PrimitiveProcessor& rhs); - int* block; + int* block; - bool compare(const datatypes::Charset &cs, uint8_t COP, - const char *str1, size_t length1, - const char *str2, size_t length2) throw(); - int compare(int val1, int val2, uint8_t COP, bool lastStage) throw(); - void indexWalk_1(const IndexWalkHeader* in, std::vector* out) throw(); - void indexWalk_2(const IndexWalkHeader* in, std::vector* out) throw(); - void indexWalk_many(const IndexWalkHeader* in, std::vector* out) throw(); - void grabSubTree(const IndexWalkHeader* in, std::vector* out) throw(); + bool compare(const datatypes::Charset& cs, uint8_t COP, const char* str1, size_t length1, const char* str2, + size_t length2) throw(); + int compare(int val1, int val2, uint8_t COP, bool lastStage) throw(); + void indexWalk_1(const IndexWalkHeader* in, std::vector* out) throw(); + void indexWalk_2(const IndexWalkHeader* in, std::vector* out) throw(); + void indexWalk_many(const IndexWalkHeader* in, std::vector* out) throw(); + void grabSubTree(const IndexWalkHeader* in, std::vector* out) throw(); - void nextSig(int NVALS, const PrimToken* tokens, p_DataValue* ret, - uint8_t outputFlags = 0, bool oldGetSigBehavior = false, bool skipNulls = false) throw(); + void nextSig(int NVALS, const PrimToken* tokens, p_DataValue* ret, uint8_t outputFlags = 0, + bool oldGetSigBehavior = false, bool skipNulls = false) throw(); - uint64_t masks[11]; - int dict_OffsetIndex, currentOffsetIndex; // used by p_dictionary - int fDebugLevel; - dbbc::Stats* fStatsPtr; // pointer for pmstats - bool logicalBlockMode; + uint64_t masks[11]; + int dict_OffsetIndex, currentOffsetIndex; // used by p_dictionary + int fDebugLevel; + dbbc::Stats* fStatsPtr; // pointer for pmstats + bool logicalBlockMode; - boost::shared_ptr parsedColumnFilter; + boost::shared_ptr parsedColumnFilter; - friend class ::PrimTest; + friend class ::PrimTest; }; // -//COMPILE A COLUMN FILTER +// COMPILE A COLUMN FILTER // // Compile column filter from BLOB into structure optimized for fast filtering. // Return a shared_ptr for the compiled filter. -template // C++ integer type providing storage for colType +template // C++ integer type providing storage for colType boost::shared_ptr _parseColumnFilter( - const uint8_t* filterString, // Filter represented as BLOB - uint32_t colType, // Column datatype as ColDataType - uint32_t filterCount, // Number of filter elements contained in filterString - uint32_t BOP) // Operation (and/or/xor/none) that combines all filter elements + const uint8_t* filterString, // Filter represented as BLOB + uint32_t colType, // Column datatype as ColDataType + uint32_t filterCount, // Number of filter elements contained in filterString + uint32_t BOP) // Operation (and/or/xor/none) that combines all filter elements { - using UT = typename std::conditional::value || datatypes::is_uint128_t::value, T, typename datatypes::make_unsigned::type>::type; - const uint32_t WIDTH = sizeof(T); // Sizeof of the column to be filtered + using UT = typename std::conditional::value || datatypes::is_uint128_t::value, T, + typename datatypes::make_unsigned::type>::type; + const uint32_t WIDTH = sizeof(T); // Sizeof of the column to be filtered - boost::shared_ptr ret; // Place for building the value to return - if (filterCount == 0) - return ret; + boost::shared_ptr ret; // Place for building the value to return + if (filterCount == 0) + return ret; - // Allocate the compiled filter structure with space for filterCount filters. - // No need to init arrays since they will be filled on the fly. - ret.reset(new ParsedColumnFilter(filterCount, BOP)); - ret->allocateSpaceForFilterArgs(); + // Allocate the compiled filter structure with space for filterCount filters. + // No need to init arrays since they will be filled on the fly. + ret.reset(new ParsedColumnFilter(filterCount, BOP)); + ret->allocateSpaceForFilterArgs(); - // Choose initial filter mode based on operation and number of filter elements - if (filterCount == 1) - ret->columnFilterMode = SINGLE_COMPARISON; - else if (BOP == BOP_OR) - ret->columnFilterMode = ANY_COMPARISON_TRUE; - else if (BOP == BOP_AND) - ret->columnFilterMode = ALL_COMPARISONS_TRUE; - else if (BOP == BOP_XOR) - ret->columnFilterMode = XOR_COMPARISONS; + // Choose initial filter mode based on operation and number of filter elements + if (filterCount == 1) + ret->columnFilterMode = SINGLE_COMPARISON; + else if (BOP == BOP_OR) + ret->columnFilterMode = ANY_COMPARISON_TRUE; + else if (BOP == BOP_AND) + ret->columnFilterMode = ALL_COMPARISONS_TRUE; + else if (BOP == BOP_XOR) + ret->columnFilterMode = XOR_COMPARISONS; + else + idbassert(0); // BOP_NONE is compatible only with filterCount <= 1 + + // Size of single filter element in filterString BLOB + const uint32_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + WIDTH; + + // Parse the filter predicates and insert them into argVals and cops + for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) + { + // Pointer to ColArgs structure representing argIndex'th element in the BLOB + auto args = reinterpret_cast(filterString + (argIndex * filterSize)); + + ret->prestored_cops[argIndex] = args->COP; + ret->prestored_rfs[argIndex] = args->rf; + + if (datatypes::isUnsigned((execplan::CalpontSystemCatalog::ColDataType)colType)) + ret->storeFilterArg(argIndex, reinterpret_cast(args->val)); else - idbassert(0); // BOP_NONE is compatible only with filterCount <= 1 + ret->storeFilterArg(argIndex, reinterpret_cast(args->val)); + } - // Size of single filter element in filterString BLOB - const uint32_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + WIDTH; + /* Decide which structure to use. I think the only cases where we can use the set + are when NOPS > 1, BOP is OR, and every COP is ==, + and when NOPS > 1, BOP is AND, and every COP is !=. - // Parse the filter predicates and insert them into argVals and cops + If there were no predicates that violate the condition for using a set, + insert argVals into a set. + */ + if (filterCount > 1) + { + // Check that all COPs are of right kind that depends on BOP for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) { - // Pointer to ColArgs structure representing argIndex'th element in the BLOB - auto args = reinterpret_cast(filterString + (argIndex * filterSize)); - - ret->prestored_cops[argIndex] = args->COP; - ret->prestored_rfs[argIndex] = args->rf; - - if (datatypes::isUnsigned((execplan::CalpontSystemCatalog::ColDataType)colType)) - ret->storeFilterArg(argIndex, reinterpret_cast(args->val)); - else - ret->storeFilterArg(argIndex, reinterpret_cast(args->val)); + auto cop = ret->prestored_cops[argIndex]; + if (!((BOP == BOP_OR && cop == COMPARE_EQ) || (BOP == BOP_AND && cop == COMPARE_NE))) + { + goto skipConversion; + } } - /* Decide which structure to use. I think the only cases where we can use the set - are when NOPS > 1, BOP is OR, and every COP is ==, - and when NOPS > 1, BOP is AND, and every COP is !=. - - If there were no predicates that violate the condition for using a set, - insert argVals into a set. - */ - if (filterCount > 1) + // Now we found that conversion is possible. Let's choose between array-based search + // and set-based search depending on the set size. + // TODO: Tailor the threshold based on the actual search algorithms used and WIDTH/SIMD_WIDTH + if (filterCount <= ParsedColumnFilter::noSetFilterThreshold) { - // Check that all COPs are of right kind that depends on BOP - for (uint32_t argIndex = 0; argIndex < filterCount; argIndex++) - { - auto cop = ret->prestored_cops[argIndex]; - if (! ((BOP == BOP_OR && cop == COMPARE_EQ) || - (BOP == BOP_AND && cop == COMPARE_NE))) - { - goto skipConversion; - } - } + // Assign filter mode of array-based filtering + if (BOP == BOP_OR) + ret->columnFilterMode = ONE_OF_VALUES_IN_ARRAY; + else + ret->columnFilterMode = NONE_OF_VALUES_IN_ARRAY; + } + else + { + // Assign filter mode of set-based filtering + if (BOP == BOP_OR) + ret->columnFilterMode = ONE_OF_VALUES_IN_SET; + else + ret->columnFilterMode = NONE_OF_VALUES_IN_SET; - // Now we found that conversion is possible. Let's choose between array-based search - // and set-based search depending on the set size. - //TODO: Tailor the threshold based on the actual search algorithms used and WIDTH/SIMD_WIDTH - if (filterCount <= ParsedColumnFilter::noSetFilterThreshold) - { - // Assign filter mode of array-based filtering - if (BOP == BOP_OR) - ret->columnFilterMode = ONE_OF_VALUES_IN_ARRAY; - else - ret->columnFilterMode = NONE_OF_VALUES_IN_ARRAY; - } - else - { - // Assign filter mode of set-based filtering - if (BOP == BOP_OR) - ret->columnFilterMode = ONE_OF_VALUES_IN_SET; - else - ret->columnFilterMode = NONE_OF_VALUES_IN_SET; - - ret->populatePrestoredSet(); - } - - skipConversion:; + ret->populatePrestoredSet(); } - return ret; + skipConversion:; + } + + return ret; } -} //namespace primitives +} // namespace primitives #endif // vim:ts=4 sw=4: - diff --git a/primitives/linux-port/print_dictblock.cpp b/primitives/linux-port/print_dictblock.cpp index b62e4d9d1..dffa058e6 100644 --- a/primitives/linux-port/print_dictblock.cpp +++ b/primitives/linux-port/print_dictblock.cpp @@ -38,75 +38,72 @@ using namespace std; void usage(char* name) { - cerr << "Usage: " << name << " dict_block_filename" << endl; - exit(0); + cerr << "Usage: " << name << " dict_block_filename" << endl; + exit(0); } void parseDictBlock(char* block) { + uint16_t* offsets; + uint16_t* freeBytes; + u_int64_t* contPtr; + int offsetIndex, size; + char sig[BLOCK_SIZE + 1]; - uint16_t* offsets; - uint16_t* freeBytes; - u_int64_t* contPtr; - int offsetIndex, size; - char sig[BLOCK_SIZE + 1]; + freeBytes = reinterpret_cast(&block[0]); + contPtr = reinterpret_cast(&block[2]); + offsets = reinterpret_cast(&block[10]); - freeBytes = reinterpret_cast(&block[0]); - contPtr = reinterpret_cast(&block[2]); - offsets = reinterpret_cast(&block[10]); + cout << "Free Bytes: " << *freeBytes << endl; + cout << "Continuation Pointer: 0x" << hex << *contPtr << dec << endl; - cout << "Free Bytes: " << *freeBytes << endl; - cout << "Continuation Pointer: 0x" << hex << *contPtr << dec << endl; + for (offsetIndex = 0; offsets[offsetIndex + 1] != 0xffff; offsetIndex++) + { + size = offsets[offsetIndex] - offsets[offsetIndex + 1]; + memcpy(sig, &block[offsets[offsetIndex + 1]], size); + sig[size] = '\0'; + cout << "Offset #" << offsetIndex + 1 << ": size=" << size << " offset=" << offsets[offsetIndex + 1] + << endl; - for (offsetIndex = 0; offsets[offsetIndex + 1] != 0xffff; offsetIndex++) - { - size = offsets[offsetIndex] - offsets[offsetIndex + 1]; - memcpy(sig, &block[offsets[offsetIndex + 1]], size); - sig[size] = '\0'; - cout << "Offset #" << offsetIndex + 1 << ": size=" << size << " offset=" - << offsets[offsetIndex + 1] << endl; - -// Use these lines instead for non-ascii data. -// cout << " Signature: 0x"; -// for (i = 0; i < size; i++) -// cout << hex << (((int) sig[i]) & 0xff); - cout << " Signature: " << sig; - cout << dec << endl; - } + // Use these lines instead for non-ascii data. + // cout << " Signature: 0x"; + // for (i = 0; i < size; i++) + // cout << hex << (((int) sig[i]) & 0xff); + cout << " Signature: " << sig; + cout << dec << endl; + } } int main(int argc, char** argv) { - int fd, err; - char buf[BLOCK_SIZE]; + int fd, err; + char buf[BLOCK_SIZE]; - if (argc != 2) - usage(argv[0]); + if (argc != 2) + usage(argv[0]); - fd = open(argv[1], O_RDONLY); + fd = open(argv[1], O_RDONLY); - if (fd < 0) - { - perror("open"); - exit(1); - } + if (fd < 0) + { + perror("open"); + exit(1); + } - err = read(fd, buf, BLOCK_SIZE); + err = read(fd, buf, BLOCK_SIZE); - if (err < 0) - { - perror("read"); - exit(1); - } + if (err < 0) + { + perror("read"); + exit(1); + } - if (err != BLOCK_SIZE) - { - cerr << "Failed to read the file in one op, check the filelength and try again." - << endl; - exit(1); - } + if (err != BLOCK_SIZE) + { + cerr << "Failed to read the file in one op, check the filelength and try again." << endl; + exit(1); + } - parseDictBlock(buf); - exit(0); + parseDictBlock(buf); + exit(0); } - diff --git a/primitives/linux-port/print_indexlist.cpp b/primitives/linux-port/print_indexlist.cpp index f3640ab25..1fd027956 100644 --- a/primitives/linux-port/print_indexlist.cpp +++ b/primitives/linux-port/print_indexlist.cpp @@ -30,7 +30,6 @@ #include "calpontsystemcatalog.h" #include "we_index.h" - /** @file * Brief description of the file contents * @@ -41,134 +40,121 @@ using namespace std; void usage() { - cout << "Usage: print_indexlist filename block_offset subblock sbentry" << endl; - cout << " Where S=0 means print the whole block, S=1 means print the subblock" << endl; - cout << " Where subblock_number indicates which subblock to print." << endl; - cout << " Where sbentry indicates the first entry to print" << endl; - exit(1); + cout << "Usage: print_indexlist filename block_offset subblock sbentry" << endl; + cout << " Where S=0 means print the whole block, S=1 means print the subblock" << endl; + cout << " Where subblock_number indicates which subblock to print." << endl; + cout << " Where sbentry indicates the first entry to print" << endl; + exit(1); } int main(int argc, char** argv) { - char buf[8192]; - int s, fd, subblock, byteoffset, i, entries, sbentry; - string filename; - off_t offset; - off_t err; - IndexListEntry* entry; - IndexListParam* ptr; - uint32_t fbo; + char buf[8192]; + int s, fd, subblock, byteoffset, i, entries, sbentry; + string filename; + off_t offset; + off_t err; + IndexListEntry* entry; + IndexListParam* ptr; + uint32_t fbo; - if (argc == 1) - usage(); + if (argc == 1) + usage(); - filename = argv[1]; - s = atoi(argv[2]); - fbo = strtoul(argv[3], 0, 0); - subblock = atoi(argv[4]); - sbentry = atoi(argv[5]); + filename = argv[1]; + s = atoi(argv[2]); + fbo = strtoul(argv[3], 0, 0); + subblock = atoi(argv[4]); + sbentry = atoi(argv[5]); - fd = open(filename.c_str(), O_RDONLY); + fd = open(filename.c_str(), O_RDONLY); - if (fd < 0) - { - perror("open"); - exit(1); - } + if (fd < 0) + { + perror("open"); + exit(1); + } - offset = ((off_t)fbo * BLOCK_SIZE); -// cout << "BLOCK_SIZE = " << BLOCK_SIZE << " fbo=" << fbo << ", seeking to offset " << offset << endl; - err = lseek(fd, offset, SEEK_SET); + offset = ((off_t)fbo * BLOCK_SIZE); + // cout << "BLOCK_SIZE = " << BLOCK_SIZE << " fbo=" << fbo << ", seeking to offset " << offset << endl; + err = lseek(fd, offset, SEEK_SET); + if (err < 0) + { + perror("lseek"); + exit(1); + } + + err = read(fd, buf, BLOCK_SIZE); + + if (err != BLOCK_SIZE) + { if (err < 0) + perror("read"); + + cerr << "read error." << endl; + exit(1); + } + + close(fd); + + byteoffset = subblock * 256; + // cout << "subblock=" << subblock << " byte offset=" << byteoffset << endl; + entry = reinterpret_cast(&buf[byteoffset]); + + if (s == 0 && subblock == 0) + entries = 1024; + else + entries = 32; + + for (i = sbentry; i < entries; i++) + { + cout << i << ": "; + + switch (entry[i].type) { - perror("lseek"); - exit(1); - } + case LLP_SUBBLK: + ptr = reinterpret_cast(&entry[i]); + cout << "Subblock pointer. Rid count=" << entry[i].ridCt << " LBID=" << ptr->fbo + << " subblock=" << ptr->sbid << " SBentry=" << ptr->entry << endl; + break; - err = read(fd, buf, BLOCK_SIZE); + case LLP_BLK: + ptr = reinterpret_cast(&entry[i]); + cout << "Block pointer. Rid count=" << entry[i].ridCt << " LBID=" << ptr->fbo + << " subblock=" << ptr->sbid << " SBentry=" << ptr->entry << endl; + break; - if (err != BLOCK_SIZE) - { - if (err < 0) - perror("read"); + case RID: cout << "RID: " << entry[i].value << endl; break; - cerr << "read error." << endl; - exit(1); - } - - close(fd); - - byteoffset = subblock * 256; -// cout << "subblock=" << subblock << " byte offset=" << byteoffset << endl; - entry = reinterpret_cast(&buf[byteoffset]); - - if (s == 0 && subblock == 0) - entries = 1024; - else - entries = 32; - - for (i = sbentry; i < entries; i++) - { - - cout << i << ": "; - - switch (entry[i].type) + case LIST_SIZE: + if (i == sbentry) { - case LLP_SUBBLK: - ptr = reinterpret_cast(&entry[i]); - cout << "Subblock pointer. Rid count=" << entry[i].ridCt << - " LBID=" << ptr->fbo << " subblock=" << ptr->sbid << - " SBentry=" << ptr->entry << endl; - break; - - case LLP_BLK: - ptr = reinterpret_cast(&entry[i]); - cout << "Block pointer. Rid count=" << entry[i].ridCt << - " LBID=" << ptr->fbo << " subblock=" << - ptr->sbid << " SBentry=" << ptr->entry << endl; - break; - - case RID: - cout << "RID: " << entry[i].value << endl; - break; - - case LIST_SIZE: - if (i == sbentry) - { - u_int64_t* val = reinterpret_cast(&entry[i + 1]); - cout << "List Header. Rid count=" << entry[i].value; - cout << " key value=0x" << hex << *val << dec << endl; - i++; - } - else if (i + 1 < entries) - { - u_int64_t* val = reinterpret_cast(&entry[i + 1]); - cout << "List Size entry. Rid count=" << entry[i].value - << " (if a header) value=0x" << hex << *val << dec << endl; - } - else - cout << "List Size entry. Rid count=" << entry[i].value << endl; - - break; - - case NOT_IN_USE: - cout << "Not in use (ignored by p_idxlist)" << endl; - break; - - case EMPTY_LIST_PTR: - cout << "Empty List Pointer (?) (ignored by p_idxlist)" << endl; - break; - - case EMPTY_PTR: - cout << "Empty Pointer entry (?) (ignored by p_idxlist)" << endl; - break; - - default: - cout << "Unknown entry type (" << entry[i].type << ")" << endl; - break; + u_int64_t* val = reinterpret_cast(&entry[i + 1]); + cout << "List Header. Rid count=" << entry[i].value; + cout << " key value=0x" << hex << *val << dec << endl; + i++; } - } + else if (i + 1 < entries) + { + u_int64_t* val = reinterpret_cast(&entry[i + 1]); + cout << "List Size entry. Rid count=" << entry[i].value << " (if a header) value=0x" << hex << *val + << dec << endl; + } + else + cout << "List Size entry. Rid count=" << entry[i].value << endl; - return 0; + break; + + case NOT_IN_USE: cout << "Not in use (ignored by p_idxlist)" << endl; break; + + case EMPTY_LIST_PTR: cout << "Empty List Pointer (?) (ignored by p_idxlist)" << endl; break; + + case EMPTY_PTR: cout << "Empty Pointer entry (?) (ignored by p_idxlist)" << endl; break; + + default: cout << "Unknown entry type (" << entry[i].type << ")" << endl; break; + } + } + + return 0; } diff --git a/primitives/linux-port/print_indextree_subblock.cpp b/primitives/linux-port/print_indextree_subblock.cpp index 35116e64c..7de85fff9 100644 --- a/primitives/linux-port/print_indextree_subblock.cpp +++ b/primitives/linux-port/print_indextree_subblock.cpp @@ -38,54 +38,52 @@ using namespace std; void usage() { - cout << "Usage: print_indextree_subblock filename block_offset subblock_number" << endl; - exit(1); + cout << "Usage: print_indextree_subblock filename block_offset subblock_number" << endl; + exit(1); } int main(int argc, char** argv) { - char buf[256]; - int fd, err, subblock, fbo, byteoffset, i; - string filename; - WriteEngine::IdxBitTestEntry* entry; + char buf[256]; + int fd, err, subblock, fbo, byteoffset, i; + string filename; + WriteEngine::IdxBitTestEntry* entry; - if (argc != 4) - usage(); + if (argc != 4) + usage(); - filename = argv[1]; - fbo = atoi(argv[2]); - subblock = atoi(argv[3]); + filename = argv[1]; + fbo = atoi(argv[2]); + subblock = atoi(argv[3]); - cout << "FBO: " << fbo << " Subblock: " << subblock << endl; - fd = open(filename.c_str(), O_RDONLY); + cout << "FBO: " << fbo << " Subblock: " << subblock << endl; + fd = open(filename.c_str(), O_RDONLY); - if (fd < 0) - { - perror("open"); - exit(1); - } + if (fd < 0) + { + perror("open"); + exit(1); + } - byteoffset = fbo * BLOCK_SIZE + subblock * 256; - lseek(fd, byteoffset, SEEK_SET); - err = read(fd, buf, 256); + byteoffset = fbo * BLOCK_SIZE + subblock * 256; + lseek(fd, byteoffset, SEEK_SET); + err = read(fd, buf, 256); - if (err != 256) - { - perror("read"); - exit(1); - } + if (err != 256) + { + perror("read"); + exit(1); + } - close(fd); + close(fd); - for (i = 0, byteoffset = 0; byteoffset < 256; - byteoffset += sizeof(WriteEngine::IdxBitTestEntry), i++) - { - entry = (WriteEngine::IdxBitTestEntry*) &buf[byteoffset]; - cout << "Entry " << i << ": fbo=" << (int)entry->fbo << - " sbid=" << entry->sbid << " sbentry=" << entry->entry << - " group=" << entry->group << " bittest=" << entry->bitTest << - " type=" << entry->type << endl; - } + for (i = 0, byteoffset = 0; byteoffset < 256; byteoffset += sizeof(WriteEngine::IdxBitTestEntry), i++) + { + entry = (WriteEngine::IdxBitTestEntry*)&buf[byteoffset]; + cout << "Entry " << i << ": fbo=" << (int)entry->fbo << " sbid=" << entry->sbid + << " sbentry=" << entry->entry << " group=" << entry->group << " bittest=" << entry->bitTest + << " type=" << entry->type << endl; + } - exit(0); + exit(0); } diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 2feb62787..e16b7ff1e 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -71,7 +71,6 @@ using namespace joblist; namespace primitiveprocessor { - #ifdef PRIMPROC_STOPWATCH #include "poormanprofiler.inc" #endif @@ -87,118 +86,120 @@ extern int noVB; // copied from https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 uint nextPowOf2(uint x) { - x--; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - x |= x >> 8; - x |= x >> 16; - x++; - return x; + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + x++; + return x; } -BatchPrimitiveProcessor::BatchPrimitiveProcessor() : - ot(BPS_ELEMENT_TYPE), - txnID(0), - sessionID(0), - stepID(0), - uniqueID(0), - count(1), - baseRid(0), - ridCount(0), - needStrValues(false), - wideColumnsWidths(0), - filterCount(0), - projectCount(0), - sendRidsAtDelivery(false), - ridMap(0), - gotAbsRids(false), - gotValues(false), - hasScan(false), - validCPData(false), - minVal(MAX64), - maxVal(MIN64), - lbidForCP(0), - hasWideColumnOut(false), - busyLoaderCount(0), - physIO(0), - cachedIO(0), - touchedBlocks(0), - LBIDTrace(false), - fBusy(false), - doJoin(false), - hasFilterStep(false), - filtOnString(false), - prefetchThreshold(0), - mJOINHasSkewedKeyColumn(false), - mSmallSideRGPtr(nullptr), - mSmallSideKeyColumnsPtr(nullptr), - hasDictStep(false), - sockIndex(0), - endOfJoinerRan(false), - processorThreads(0), - ptMask(0), - firstInstance(false) +BatchPrimitiveProcessor::BatchPrimitiveProcessor() + : ot(BPS_ELEMENT_TYPE) + , txnID(0) + , sessionID(0) + , stepID(0) + , uniqueID(0) + , count(1) + , baseRid(0) + , ridCount(0) + , needStrValues(false) + , wideColumnsWidths(0) + , filterCount(0) + , projectCount(0) + , sendRidsAtDelivery(false) + , ridMap(0) + , gotAbsRids(false) + , gotValues(false) + , hasScan(false) + , validCPData(false) + , minVal(MAX64) + , maxVal(MIN64) + , lbidForCP(0) + , hasWideColumnOut(false) + , busyLoaderCount(0) + , physIO(0) + , cachedIO(0) + , touchedBlocks(0) + , LBIDTrace(false) + , fBusy(false) + , doJoin(false) + , hasFilterStep(false) + , filtOnString(false) + , prefetchThreshold(0) + , mJOINHasSkewedKeyColumn(false) + , mSmallSideRGPtr(nullptr) + , mSmallSideKeyColumnsPtr(nullptr) + , hasDictStep(false) + , sockIndex(0) + , endOfJoinerRan(false) + , processorThreads(0) + , ptMask(0) + , firstInstance(false) { - pp.setLogicalBlockMode(true); - pp.setBlockPtr((int*) blockData); - pthread_mutex_init(&objLock, NULL); + pp.setLogicalBlockMode(true); + pp.setBlockPtr((int*)blockData); + pthread_mutex_init(&objLock, NULL); } BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, - boost::shared_ptr bppst, uint _processorThreads) : - ot(BPS_ELEMENT_TYPE), - txnID(0), - sessionID(0), - stepID(0), - uniqueID(0), - count(1), - baseRid(0), - ridCount(0), - needStrValues(false), - wideColumnsWidths(0), - filterCount(0), - projectCount(0), - sendRidsAtDelivery(false), - ridMap(0), - gotAbsRids(false), - gotValues(false), - hasScan(false), - validCPData(false), - minVal(MAX64), - maxVal(MIN64), - lbidForCP(0), - hasWideColumnOut(false), - busyLoaderCount(0), - physIO(0), - cachedIO(0), - touchedBlocks(0), - LBIDTrace(false), - fBusy(false), - doJoin(false), - hasFilterStep(false), - filtOnString(false), - prefetchThreshold(prefetch), - mJOINHasSkewedKeyColumn(false), - mSmallSideRGPtr(nullptr), - mSmallSideKeyColumnsPtr(nullptr), - hasDictStep(false), - sockIndex(0), - endOfJoinerRan(false), - processorThreads(_processorThreads), - //processorThreads(32), - //ptMask(processorThreads - 1), - firstInstance(true) + boost::shared_ptr bppst, + uint _processorThreads) + : ot(BPS_ELEMENT_TYPE) + , txnID(0) + , sessionID(0) + , stepID(0) + , uniqueID(0) + , count(1) + , baseRid(0) + , ridCount(0) + , needStrValues(false) + , wideColumnsWidths(0) + , filterCount(0) + , projectCount(0) + , sendRidsAtDelivery(false) + , ridMap(0) + , gotAbsRids(false) + , gotValues(false) + , hasScan(false) + , validCPData(false) + , minVal(MAX64) + , maxVal(MIN64) + , lbidForCP(0) + , hasWideColumnOut(false) + , busyLoaderCount(0) + , physIO(0) + , cachedIO(0) + , touchedBlocks(0) + , LBIDTrace(false) + , fBusy(false) + , doJoin(false) + , hasFilterStep(false) + , filtOnString(false) + , prefetchThreshold(prefetch) + , mJOINHasSkewedKeyColumn(false) + , mSmallSideRGPtr(nullptr) + , mSmallSideKeyColumnsPtr(nullptr) + , hasDictStep(false) + , sockIndex(0) + , endOfJoinerRan(false) + , processorThreads(_processorThreads) + , + // processorThreads(32), + // ptMask(processorThreads - 1), + firstInstance(true) { - // promote processorThreads to next power of 2. also need to change the name to bucketCount or similar - processorThreads = nextPowOf2(processorThreads); - ptMask = processorThreads - 1; + // promote processorThreads to next power of 2. also need to change the name to bucketCount or similar + processorThreads = nextPowOf2(processorThreads); + ptMask = processorThreads - 1; - pp.setLogicalBlockMode(true); - pp.setBlockPtr((int*) blockData); - sendThread = bppst; - pthread_mutex_init(&objLock, NULL); - initBPP(b); + pp.setLogicalBlockMode(true); + pp.setBlockPtr((int*)blockData); + sendThread = bppst; + pthread_mutex_init(&objLock, NULL); + initBPP(b); } #if 0 @@ -210,18 +211,18 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(const BatchPrimitiveProcessor& BatchPrimitiveProcessor::~BatchPrimitiveProcessor() { - //FIXME: just do a sync fetch - counterLock.lock(); // need to make sure the loader has exited - - while (busyLoaderCount > 0) - { - counterLock.unlock(); - usleep(100000); - counterLock.lock(); - } + // FIXME: just do a sync fetch + counterLock.lock(); // need to make sure the loader has exited + while (busyLoaderCount > 0) + { counterLock.unlock(); - pthread_mutex_destroy(&objLock); + usleep(100000); + counterLock.lock(); + } + + counterLock.unlock(); + pthread_mutex_destroy(&objLock); } /** @@ -231,368 +232,366 @@ BatchPrimitiveProcessor::~BatchPrimitiveProcessor() void BatchPrimitiveProcessor::initBPP(ByteStream& bs) { - uint32_t i; - uint8_t tmp8; - uint16_t tmp16; - Command::CommandType type; + uint32_t i; + uint8_t tmp8; + uint16_t tmp16; + Command::CommandType type; - bs.advance(sizeof(ISMPacketHeader)); // skip the header + bs.advance(sizeof(ISMPacketHeader)); // skip the header + bs >> tmp8; + ot = static_cast(tmp8); + bs >> txnID; + bs >> sessionID; + bs >> stepID; + bs >> uniqueID; + bs >> versionInfo; + + bs >> tmp16; + needStrValues = tmp16 & NEED_STR_VALUES; + gotAbsRids = tmp16 & GOT_ABS_RIDS; + gotValues = tmp16 & GOT_VALUES; + LBIDTrace = tmp16 & LBID_TRACE; + sendRidsAtDelivery = tmp16 & SEND_RIDS_AT_DELIVERY; + doJoin = tmp16 & HAS_JOINER; + hasRowGroup = tmp16 & HAS_ROWGROUP; + getTupleJoinRowGroupData = tmp16 & JOIN_ROWGROUP_DATA; + bool hasWideColumnsIn = tmp16 & HAS_WIDE_COLUMNS; + + // This used to signify that there was input row data from previous jobsteps, and + // it never quite worked right. No need to fix it or update it; all BPP's have started + // with a scan for years. Took it out. + assert(!hasRowGroup); + + if (hasWideColumnsIn) + bs >> wideColumnsWidths; + + bs >> bop; + bs >> forHJ; + + if (ot == ROW_GROUP) + { + bs >> outputRG; + // outputRG.setUseStringTable(true); bs >> tmp8; - ot = static_cast(tmp8); - bs >> txnID; - bs >> sessionID; - bs >> stepID; - bs >> uniqueID; - bs >> versionInfo; - bs >> tmp16; - needStrValues = tmp16 & NEED_STR_VALUES; - gotAbsRids = tmp16 & GOT_ABS_RIDS; - gotValues = tmp16 & GOT_VALUES; - LBIDTrace = tmp16 & LBID_TRACE; - sendRidsAtDelivery = tmp16 & SEND_RIDS_AT_DELIVERY; - doJoin = tmp16 & HAS_JOINER; - hasRowGroup = tmp16 & HAS_ROWGROUP; - getTupleJoinRowGroupData = tmp16 & JOIN_ROWGROUP_DATA; - bool hasWideColumnsIn = tmp16 & HAS_WIDE_COLUMNS; + if (tmp8) + { + fe1.reset(new FuncExpWrapper()); + bs >> *fe1; + bs >> fe1Input; + } - // This used to signify that there was input row data from previous jobsteps, and - // it never quite worked right. No need to fix it or update it; all BPP's have started - // with a scan for years. Took it out. - assert(!hasRowGroup); + bs >> tmp8; - if (hasWideColumnsIn) - bs >> wideColumnsWidths; + if (tmp8) + { + fe2.reset(new FuncExpWrapper()); + bs >> *fe2; + bs >> fe2Output; + } + } - bs >> bop; - bs >> forHJ; + if (doJoin) + { + pthread_mutex_lock(&objLock); if (ot == ROW_GROUP) { - bs >> outputRG; - //outputRG.setUseStringTable(true); - bs >> tmp8; + bs >> joinerCount; + // cout << "joinerCount = " << joinerCount << endl; + joinTypes.reset(new JoinType[joinerCount]); - if (tmp8) + tJoiners.reset(new boost::shared_array >[joinerCount]); + for (uint j = 0; j < joinerCount; ++j) + tJoiners[j].reset(new boost::shared_ptr[processorThreads]); + + tlJoiners.reset(new boost::shared_array >[joinerCount]); + for (uint j = 0; j < joinerCount; ++j) + tlJoiners[j].reset(new boost::shared_ptr[processorThreads]); + + addToJoinerLocks.reset(new boost::scoped_array[joinerCount]); + for (uint j = 0; j < joinerCount; ++j) + addToJoinerLocks[j].reset(new boost::mutex[processorThreads]); + + smallSideDataLocks.reset(new boost::mutex[joinerCount]); + tJoinerSizes.reset(new std::atomic[joinerCount]); + largeSideKeyColumns.reset(new uint32_t[joinerCount]); + tlLargeSideKeyColumns.reset(new vector[joinerCount]); + tlSmallSideKeyColumns.reset(new std::vector); + typelessJoin.reset(new bool[joinerCount]); + tlSmallSideKeyLengths.reset(new uint32_t[joinerCount]); + + storedKeyAllocators.reset(new PoolAllocator[joinerCount]); + for (uint j = 0; j < joinerCount; ++j) + storedKeyAllocators[j].setUseLock(true); + + joinNullValues.reset(new uint64_t[joinerCount]); + doMatchNulls.reset(new bool[joinerCount]); + joinFEFilters.reset(new scoped_ptr[joinerCount]); + hasJoinFEFilters = false; + hasSmallOuterJoin = false; + bool smallSideRGRecvd = false; + + for (i = 0; i < joinerCount; i++) + { + doMatchNulls[i] = false; + uint32_t tmp32; + bs >> tmp32; + tJoinerSizes[i] = tmp32; + // bs >> tJoinerSizes[i]; + // cout << "joiner size = " << tJoinerSizes[i] << endl; + bs >> joinTypes[i]; + bs >> tmp8; + typelessJoin[i] = (bool)tmp8; + + if (joinTypes[i] & WITHFCNEXP) { - fe1.reset(new FuncExpWrapper()); - bs >> *fe1; - bs >> fe1Input; + hasJoinFEFilters = true; + joinFEFilters[i].reset(new FuncExpWrapper()); + bs >> *joinFEFilters[i]; } - bs >> tmp8; + if (joinTypes[i] & SMALLOUTER) + hasSmallOuterJoin = true; - if (tmp8) + if (!typelessJoin[i]) { - fe2.reset(new FuncExpWrapper()); - bs >> *fe2; - bs >> fe2Output; + bs >> joinNullValues[i]; + bs >> largeSideKeyColumns[i]; + // cout << "large side key is " << largeSideKeyColumns[i] << endl; + for (uint j = 0; j < processorThreads; ++j) + tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher())); } + else + { + deserializeVector(bs, tlLargeSideKeyColumns[i]); + bs >> tlSmallSideKeyLengths[i]; + bs >> tmp8; + mJOINHasSkewedKeyColumn = (bool)tmp8; + // Deser smallSideRG if key data types are different, e.g. INT vs wide-DECIMAL. + if (mJOINHasSkewedKeyColumn && !smallSideRGRecvd) + { + smallSideRGs.emplace_back(rowgroup::RowGroup(bs)); + // LargeSide key columns number equals to SmallSide key columns number. + deserializeVector(bs, *tlSmallSideKeyColumns); + mSmallSideRGPtr = &smallSideRGs[0]; + mSmallSideKeyColumnsPtr = &(*tlSmallSideKeyColumns); + smallSideRGRecvd = true; + } + + for (uint j = 0; j < processorThreads; ++j) + { + auto tlHasher = TupleJoiner::TypelessDataHasher(&outputRG, &tlLargeSideKeyColumns[i], + mSmallSideKeyColumnsPtr, mSmallSideRGPtr); + auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, &tlLargeSideKeyColumns[i], + mSmallSideKeyColumnsPtr, mSmallSideRGPtr); + tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator)); + } + } + } + + if (hasJoinFEFilters) + { + joinFERG.reset(new RowGroup()); + bs >> *joinFERG; + } + + if (getTupleJoinRowGroupData) + { + deserializeVector(bs, smallSideRGs); + // cout << "deserialized " << smallSideRGs.size() << " small-side + // rowgroups\n"; + idbassert(smallSideRGs.size() == joinerCount); + smallSideRowLengths.reset(new uint32_t[joinerCount]); + smallSideRowData.reset(new RGData[joinerCount]); + smallNullRowData.reset(new RGData[joinerCount]); + smallNullPointers.reset(new Row::Pointer[joinerCount]); + ssrdPos.reset(new uint64_t[joinerCount]); + + for (i = 0; i < joinerCount; i++) + { + smallSideRowLengths[i] = smallSideRGs[i].getRowSize(); + ; + smallSideRowData[i] = RGData(smallSideRGs[i], tJoinerSizes[i]); + // smallSideRowData[i].reset(new uint8_t[ + // smallSideRGs[i].getEmptySize() + + // (uint64_t) smallSideRowLengths[i] * tJoinerSizes[i]]); + smallSideRGs[i].setData(&smallSideRowData[i]); + smallSideRGs[i].resetRowGroup(0); + ssrdPos[i] = smallSideRGs[i].getEmptySize(); + + if (joinTypes[i] & (LARGEOUTER | SEMI | ANTI)) + { + Row smallRow; + smallSideRGs[i].initRow(&smallRow); + smallNullRowData[i] = RGData(smallSideRGs[i], 1); + smallSideRGs[i].setData(&smallNullRowData[i]); + smallSideRGs[i].getRow(0, &smallRow); + smallRow.initToNull(); + smallNullPointers[i] = smallRow.getPointer(); + smallSideRGs[i].setData(&smallSideRowData[i]); + } + } + + bs >> largeSideRG; + bs >> joinedRG; + // cout << "got the joined Rowgroup: " << joinedRG.toString() << "\n"; + } } - if (doJoin) - { - pthread_mutex_lock(&objLock); - - if (ot == ROW_GROUP) - { - bs >> joinerCount; -// cout << "joinerCount = " << joinerCount << endl; - joinTypes.reset(new JoinType[joinerCount]); - - tJoiners.reset(new boost::shared_array >[joinerCount]); - for (uint j = 0; j < joinerCount; ++j) - tJoiners[j].reset(new boost::shared_ptr[processorThreads]); - - tlJoiners.reset(new boost::shared_array >[joinerCount]); - for (uint j = 0; j < joinerCount; ++j) - tlJoiners[j].reset(new boost::shared_ptr[processorThreads]); - - addToJoinerLocks.reset(new boost::scoped_array[joinerCount]); - for (uint j = 0; j < joinerCount; ++j) - addToJoinerLocks[j].reset(new boost::mutex[processorThreads]); - - smallSideDataLocks.reset(new boost::mutex[joinerCount]); - tJoinerSizes.reset(new std::atomic[joinerCount]); - largeSideKeyColumns.reset(new uint32_t[joinerCount]); - tlLargeSideKeyColumns.reset(new vector[joinerCount]); - tlSmallSideKeyColumns.reset(new std::vector); - typelessJoin.reset(new bool[joinerCount]); - tlSmallSideKeyLengths.reset(new uint32_t[joinerCount]); - - storedKeyAllocators.reset(new PoolAllocator[joinerCount]); - for (uint j = 0; j < joinerCount; ++j) - storedKeyAllocators[j].setUseLock(true); - - joinNullValues.reset(new uint64_t[joinerCount]); - doMatchNulls.reset(new bool[joinerCount]); - joinFEFilters.reset(new scoped_ptr[joinerCount]); - hasJoinFEFilters = false; - hasSmallOuterJoin = false; - bool smallSideRGRecvd = false; - - for (i = 0; i < joinerCount; i++) - { - doMatchNulls[i] = false; - uint32_t tmp32; - bs >> tmp32; - tJoinerSizes[i] = tmp32; - //bs >> tJoinerSizes[i]; - //cout << "joiner size = " << tJoinerSizes[i] << endl; - bs >> joinTypes[i]; - bs >> tmp8; - typelessJoin[i] = (bool) tmp8; - - if (joinTypes[i] & WITHFCNEXP) - { - hasJoinFEFilters = true; - joinFEFilters[i].reset(new FuncExpWrapper()); - bs >> *joinFEFilters[i]; - } - - if (joinTypes[i] & SMALLOUTER) - hasSmallOuterJoin = true; - - if (!typelessJoin[i]) - { - bs >> joinNullValues[i]; - bs >> largeSideKeyColumns[i]; - //cout << "large side key is " << largeSideKeyColumns[i] << endl; - for (uint j = 0; j < processorThreads; ++j) - tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher())); - } - else - { - deserializeVector(bs, tlLargeSideKeyColumns[i]); - bs >> tlSmallSideKeyLengths[i]; - bs >> tmp8; - mJOINHasSkewedKeyColumn = (bool) tmp8; - // Deser smallSideRG if key data types are different, e.g. INT vs wide-DECIMAL. - if (mJOINHasSkewedKeyColumn && !smallSideRGRecvd) - { - smallSideRGs.emplace_back(rowgroup::RowGroup(bs)); - // LargeSide key columns number equals to SmallSide key columns number. - deserializeVector(bs, *tlSmallSideKeyColumns); - mSmallSideRGPtr = &smallSideRGs[0]; - mSmallSideKeyColumnsPtr = &(*tlSmallSideKeyColumns); - smallSideRGRecvd = true; - } - - for (uint j = 0; j < processorThreads; ++j) - { - auto tlHasher = TupleJoiner::TypelessDataHasher(&outputRG, - &tlLargeSideKeyColumns[i], - mSmallSideKeyColumnsPtr, - mSmallSideRGPtr); - auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, - &tlLargeSideKeyColumns[i], - mSmallSideKeyColumnsPtr, - mSmallSideRGPtr); - tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator)); - } - } - } - - if (hasJoinFEFilters) - { - joinFERG.reset(new RowGroup()); - bs >> *joinFERG; - } - - if (getTupleJoinRowGroupData) - { - deserializeVector(bs, smallSideRGs); -// cout << "deserialized " << smallSideRGs.size() << " small-side rowgroups\n"; - idbassert(smallSideRGs.size() == joinerCount); - smallSideRowLengths.reset(new uint32_t[joinerCount]); - smallSideRowData.reset(new RGData[joinerCount]); - smallNullRowData.reset(new RGData[joinerCount]); - smallNullPointers.reset(new Row::Pointer[joinerCount]); - ssrdPos.reset(new uint64_t[joinerCount]); - - for (i = 0; i < joinerCount; i++) - { - smallSideRowLengths[i] = smallSideRGs[i].getRowSize();; - smallSideRowData[i] = RGData(smallSideRGs[i], tJoinerSizes[i]); -// smallSideRowData[i].reset(new uint8_t[ -// smallSideRGs[i].getEmptySize() + -// (uint64_t) smallSideRowLengths[i] * tJoinerSizes[i]]); - smallSideRGs[i].setData(&smallSideRowData[i]); - smallSideRGs[i].resetRowGroup(0); - ssrdPos[i] = smallSideRGs[i].getEmptySize(); - - if (joinTypes[i] & (LARGEOUTER | SEMI | ANTI)) - { - Row smallRow; - smallSideRGs[i].initRow(&smallRow); - smallNullRowData[i] = RGData(smallSideRGs[i], 1); - smallSideRGs[i].setData(&smallNullRowData[i]); - smallSideRGs[i].getRow(0, &smallRow); - smallRow.initToNull(); - smallNullPointers[i] = smallRow.getPointer(); - smallSideRGs[i].setData(&smallSideRowData[i]); - } - } - - bs >> largeSideRG; - bs >> joinedRG; -// cout << "got the joined Rowgroup: " << joinedRG.toString() << "\n"; - } - } - #ifdef __FreeBSD__ - pthread_mutex_unlock(&objLock); + pthread_mutex_unlock(&objLock); #endif - } + } - bs >> filterCount; - filterSteps.resize(filterCount); - //cout << "deserializing " << filterCount << " filters\n"; - hasScan = false; - hasPassThru = false; + bs >> filterCount; + filterSteps.resize(filterCount); + // cout << "deserializing " << filterCount << " filters\n"; + hasScan = false; + hasPassThru = false; - for (i = 0; i < filterCount; ++i) + for (i = 0; i < filterCount; ++i) + { + // cout << "deserializing step " << i << endl; + filterSteps[i] = SCommand(Command::makeCommand(bs, &type, filterSteps)); + + if (type == Command::COLUMN_COMMAND) { - //cout << "deserializing step " << i << endl; - filterSteps[i] = SCommand(Command::makeCommand(bs, &type, filterSteps)); + ColumnCommand* col = (ColumnCommand*)filterSteps[i].get(); - if (type == Command::COLUMN_COMMAND) - { - ColumnCommand* col = (ColumnCommand*) filterSteps[i].get(); + if (col->isScan()) + hasScan = true; - if (col->isScan()) - hasScan = true; - - if (bop == BOP_OR) - col->setScan(true); - } - else if (type == Command::FILTER_COMMAND) - { - hasFilterStep = true; - - if (dynamic_cast(filterSteps[i].get()) != NULL) - filtOnString = true; - } - else if (type == Command::DICT_STEP || type == Command::RID_TO_STRING) - hasDictStep = true; + if (bop == BOP_OR) + col->setScan(true); } - - bs >> projectCount; - //cout << "deserializing " << projectCount << " projected columns\n\n"; - projectSteps.resize(projectCount); - - for (i = 0; i < projectCount; ++i) + else if (type == Command::FILTER_COMMAND) { - //cout << "deserializing step " << i << endl; - projectSteps[i] = SCommand(Command::makeCommand(bs, &type, projectSteps)); + hasFilterStep = true; - if (type == Command::PASS_THRU) - hasPassThru = true; - else if (type == Command::DICT_STEP || type == Command::RID_TO_STRING) - hasDictStep = true; + if (dynamic_cast(filterSteps[i].get()) != NULL) + filtOnString = true; } + else if (type == Command::DICT_STEP || type == Command::RID_TO_STRING) + hasDictStep = true; + } - if (ot == ROW_GROUP) + bs >> projectCount; + // cout << "deserializing " << projectCount << " projected columns\n\n"; + projectSteps.resize(projectCount); + + for (i = 0; i < projectCount; ++i) + { + // cout << "deserializing step " << i << endl; + projectSteps[i] = SCommand(Command::makeCommand(bs, &type, projectSteps)); + + if (type == Command::PASS_THRU) + hasPassThru = true; + else if (type == Command::DICT_STEP || type == Command::RID_TO_STRING) + hasDictStep = true; + } + + if (ot == ROW_GROUP) + { + bs >> tmp8; + + if (tmp8 > 0) { - bs >> tmp8; + bs >> fAggregateRG; + fAggregator.reset(new RowAggregation); + bs >> *(fAggregator.get()); - if (tmp8 > 0) + // If there's UDAF involved, set up for PM processing + for (uint64_t i = 0; i < fAggregator->getAggFunctions().size(); i++) + { + RowUDAFFunctionCol* rowUDAF = + dynamic_cast(fAggregator->getAggFunctions()[i].get()); + + if (rowUDAF) { - bs >> fAggregateRG; - fAggregator.reset(new RowAggregation); - bs >> *(fAggregator.get()); - - // If there's UDAF involved, set up for PM processing - for (uint64_t i = 0; i < fAggregator->getAggFunctions().size(); i++) - { - RowUDAFFunctionCol* rowUDAF = dynamic_cast(fAggregator->getAggFunctions()[i].get()); - - if (rowUDAF) - { - // On the PM, the aux column is not sent, but rather is output col + 1. - rowUDAF->fAuxColumnIndex = rowUDAF->fOutputColumnIndex + 1; - // Set the PM flag in case the UDAF cares. - rowUDAF->fUDAFContext.setContextFlags(rowUDAF->fUDAFContext.getContextFlags() - | mcsv1sdk::CONTEXT_IS_PM); - } - } + // On the PM, the aux column is not sent, but rather is output col + 1. + rowUDAF->fAuxColumnIndex = rowUDAF->fOutputColumnIndex + 1; + // Set the PM flag in case the UDAF cares. + rowUDAF->fUDAFContext.setContextFlags(rowUDAF->fUDAFContext.getContextFlags() | + mcsv1sdk::CONTEXT_IS_PM); } + } } + } - initProcessor(); + initProcessor(); } /** * resetBPP Parses the run messages from BatchPrimitiveProcessor-JL::runBPP() * Refer to that fcn for message format info. */ -void BatchPrimitiveProcessor::resetBPP(ByteStream& bs, const SP_UM_MUTEX& w, - const SP_UM_IOSOCK& s) +void BatchPrimitiveProcessor::resetBPP(ByteStream& bs, const SP_UM_MUTEX& w, const SP_UM_IOSOCK& s) { - uint32_t i; - vector preloads; + uint32_t i; + vector preloads; - pthread_mutex_lock(&objLock); + pthread_mutex_lock(&objLock); - writelock = w; - sock = s; - newConnection = true; + writelock = w; + sock = s; + newConnection = true; - // skip the header, sessionID, stepID, uniqueID, and priority - bs.advance(sizeof(ISMPacketHeader) + 16); - bs >> dbRoot; - bs >> count; - bs >> ridCount; + // skip the header, sessionID, stepID, uniqueID, and priority + bs.advance(sizeof(ISMPacketHeader) + 16); + bs >> dbRoot; + bs >> count; + bs >> ridCount; - if (gotAbsRids) + if (gotAbsRids) + { + assert(0); + memcpy(absRids.get(), bs.buf(), ridCount << 3); + bs.advance(ridCount << 3); + /* TODO: this loop isn't always necessary or sensible */ + ridMap = 0; + baseRid = absRids[0] & 0xffffffffffffe000ULL; + + for (uint32_t i = 0; i < ridCount; i++) { - assert(0); - memcpy(absRids.get(), bs.buf(), ridCount << 3); - bs.advance(ridCount << 3); - /* TODO: this loop isn't always necessary or sensible */ - ridMap = 0; - baseRid = absRids[0] & 0xffffffffffffe000ULL; - - for (uint32_t i = 0; i < ridCount; i++) - { - relRids[i] = absRids[i] - baseRid; - ridMap |= 1 << (relRids[i] >> 9); - } - } - else - { - bs >> ridMap; - bs >> baseRid; - memcpy(relRids, bs.buf(), ridCount << 1); - bs.advance(ridCount << 1); + relRids[i] = absRids[i] - baseRid; + ridMap |= 1 << (relRids[i] >> 9); } + } + else + { + bs >> ridMap; + bs >> baseRid; + memcpy(relRids, bs.buf(), ridCount << 1); + bs.advance(ridCount << 1); + } - if (gotValues) - { - memcpy(values, bs.buf(), ridCount << 3); - bs.advance(ridCount << 3); - } + if (gotValues) + { + memcpy(values, bs.buf(), ridCount << 3); + bs.advance(ridCount << 3); + } - for (i = 0; i < filterCount; ++i) - { - filterSteps[i]->resetCommand(bs); - } + for (i = 0; i < filterCount; ++i) + { + filterSteps[i]->resetCommand(bs); + } - for (i = 0; i < projectCount; ++i) - { - projectSteps[i]->resetCommand(bs); - } + for (i = 0; i < projectCount; ++i) + { + projectSteps[i]->resetCommand(bs); + } - idbassert(bs.length() == 0); + idbassert(bs.length() == 0); - /* init vars not part of the BS */ - currentBlockOffset = 0; - memset(relLBID.get(), 0, sizeof(uint64_t) * (projectCount + 1)); - memset(asyncLoaded.get(), 0, sizeof(bool) * (projectCount + 1)); + /* init vars not part of the BS */ + currentBlockOffset = 0; + memset(relLBID.get(), 0, sizeof(uint64_t) * (projectCount + 1)); + memset(asyncLoaded.get(), 0, sizeof(bool) * (projectCount + 1)); - buildVSSCache(count); + buildVSSCache(count); #ifdef __FreeBSD__ - pthread_mutex_unlock(&objLock); + pthread_mutex_unlock(&objLock); #endif } @@ -602,827 +601,821 @@ void BatchPrimitiveProcessor::resetBPP(ByteStream& bs, const SP_UM_MUTEX& w, // the lock for each shared table and inserts them there. void BatchPrimitiveProcessor::addToJoiner(ByteStream& bs) { -/* to get wall-time of hash table construction - idbassert(processorThreads != 0); - if (firstCallTime.is_not_a_date_time()) - firstCallTime = boost::posix_time::microsec_clock::universal_time(); -*/ + /* to get wall-time of hash table construction + idbassert(processorThreads != 0); + if (firstCallTime.is_not_a_date_time()) + firstCallTime = boost::posix_time::microsec_clock::universal_time(); + */ - uint32_t count, i, joinerNum, tlIndex, startPos, bucket; -#pragma pack(push,1) - struct JoinerElements - { - uint64_t key; - uint32_t value; - } *arr; + uint32_t count, i, joinerNum, tlIndex, startPos, bucket; +#pragma pack(push, 1) + struct JoinerElements + { + uint64_t key; + uint32_t value; + } * arr; #pragma pack(pop) - /* skip the header */ - bs.advance(sizeof(ISMPacketHeader) + 3 * sizeof(uint32_t)); + /* skip the header */ + bs.advance(sizeof(ISMPacketHeader) + 3 * sizeof(uint32_t)); - bs >> count; - bs >> startPos; + bs >> count; + bs >> startPos; - if (ot == ROW_GROUP) + if (ot == ROW_GROUP) + { + bs >> joinerNum; + idbassert(joinerNum < joinerCount); + arr = (JoinerElements*)bs.buf(); + + std::atomic& tJoinerSize = tJoinerSizes[joinerNum]; + + // XXXPAT: enormous if stmts are evil. TODO: move each block into + // properly-named functions for clarity. + if (typelessJoin[joinerNum]) { - bs >> joinerNum; - idbassert(joinerNum < joinerCount); - arr = (JoinerElements*) bs.buf(); - - std::atomic &tJoinerSize = tJoinerSizes[joinerNum]; - - // XXXPAT: enormous if stmts are evil. TODO: move each block into - // properly-named functions for clarity. - if (typelessJoin[joinerNum]) + utils::VLArray > > tmpBuckets(processorThreads); + uint8_t nullFlag; + PoolAllocator& storedKeyAllocator = storedKeyAllocators[joinerNum]; + // this first loop hashes incoming values into vectors that parallel the hash tables. + uint nullCount = 0; + for (i = 0; i < count; ++i) + { + bs >> nullFlag; + if (nullFlag == 0) { - utils::VLArray > > tmpBuckets(processorThreads); - uint8_t nullFlag; - PoolAllocator &storedKeyAllocator = storedKeyAllocators[joinerNum]; - // this first loop hashes incoming values into vectors that parallel the hash tables. - uint nullCount = 0; - for (i = 0; i < count; ++i) - { - bs >> nullFlag; - if (nullFlag == 0) - { - TypelessData tlSmallSideKey(bs, storedKeyAllocator); - if (mJOINHasSkewedKeyColumn) - tlSmallSideKey.setSmallSideWithSkewedData(); - else - tlSmallSideKey.setSmallSide(); - bs >> tlIndex; - // The bucket number corresponds with the index used later inserting TL keys into permanent JOIN hash map. - auto ha = tlSmallSideKey.hash(outputRG, - tlLargeSideKeyColumns[joinerNum], - mSmallSideKeyColumnsPtr, - mSmallSideRGPtr); - - bucket = ha & ptMask; - tmpBuckets[bucket].push_back(make_pair(tlSmallSideKey, tlIndex)); - } - else - ++nullCount; - } - tJoinerSize -= nullCount; - - bool done = false, didSomeWork; - //uint loopCounter = 0, noWorkCounter = 0; - // this loop moves the elements from each vector into its corresponding hash table. - while (!done) - { - //++loopCounter; - done = true; - didSomeWork = false; - for (i = 0; i < processorThreads; ++i) - { - if (!tmpBuckets[i].empty()) - { - bool gotIt = addToJoinerLocks[joinerNum][i].try_lock(); - if (!gotIt) - { - done = false; // didn't get it, don't block, try the next bucket - continue; - } - for (auto &element : tmpBuckets[i]) - tlJoiners[joinerNum][i]->insert(element); - addToJoinerLocks[joinerNum][i].unlock(); - tmpBuckets[i].clear(); - didSomeWork = true; - } - } - // if this iteration did no useful work, everything we need is locked; wait briefly - // and try again. - if (!done && !didSomeWork) - { - ::usleep(500 * processorThreads); - //++noWorkCounter; - } - } - //cout << "TL join insert. Took " << loopCounter << " loops" << endl; + TypelessData tlSmallSideKey(bs, storedKeyAllocator); + if (mJOINHasSkewedKeyColumn) + tlSmallSideKey.setSmallSideWithSkewedData(); + else + tlSmallSideKey.setSmallSide(); + bs >> tlIndex; + // The bucket number corresponds with the index used later inserting TL keys into permanent JOIN + // hash map. + auto ha = tlSmallSideKey.hash(outputRG, tlLargeSideKeyColumns[joinerNum], mSmallSideKeyColumnsPtr, + mSmallSideRGPtr); + bucket = ha & ptMask; + tmpBuckets[bucket].push_back(make_pair(tlSmallSideKey, tlIndex)); } else + ++nullCount; + } + tJoinerSize -= nullCount; + + bool done = false, didSomeWork; + // uint loopCounter = 0, noWorkCounter = 0; + // this loop moves the elements from each vector into its corresponding hash table. + while (!done) + { + //++loopCounter; + done = true; + didSomeWork = false; + for (i = 0; i < processorThreads; ++i) { - boost::shared_array > tJoiner = tJoiners[joinerNum]; - uint64_t nullValue = joinNullValues[joinerNum]; - bool &l_doMatchNulls = doMatchNulls[joinerNum]; - joblist::JoinType joinType = joinTypes[joinerNum]; - utils::VLArray > > tmpBuckets(processorThreads); - - if (joinType & MATCHNULLS) + if (!tmpBuckets[i].empty()) + { + bool gotIt = addToJoinerLocks[joinerNum][i].try_lock(); + if (!gotIt) { - // this first loop hashes incoming values into vectors that parallel the hash tables. - for (i = 0; i < count; ++i) - { - /* A minor optimization: the matchnull logic should only be used with - * the jointype specifies it and there's a null value in the small side */ - if (!l_doMatchNulls && arr[i].key == nullValue) - l_doMatchNulls = true; - bucket = bucketPicker((char *) &arr[i].key, 8, bpSeed) & ptMask; - tmpBuckets[bucket].push_back(make_pair(arr[i].key, arr[i].value)); - } - - - bool done = false, didSomeWork; - //uint loopCounter = 0, noWorkCounter = 0; - // this loop moves the elements from each vector into its corresponding hash table. - while (!done) - { - //++loopCounter; - done = true; - didSomeWork = false; - for (i = 0; i < processorThreads; ++i) - { - if (!tmpBuckets[i].empty()) - { - bool gotIt = addToJoinerLocks[joinerNum][i].try_lock(); - if (!gotIt) - { - done = false; // didn't get it, don't block, try the next bucket - continue; - } - for (auto &element : tmpBuckets[i]) - tJoiners[joinerNum][i]->insert(element); - addToJoinerLocks[joinerNum][i].unlock(); - tmpBuckets[i].clear(); - didSomeWork = true; - } - } - // if this iteration did no useful work, everything we need is locked; wait briefly - // and try again. - if (!done && !didSomeWork) - { - ::usleep(500 * processorThreads); - //++noWorkCounter; - } - } - - //cout << "T numeric join insert. Took " << loopCounter << " loops" << endl; + done = false; // didn't get it, don't block, try the next bucket + continue; } - else - { - // this first loop hashes incoming values into vectors that parallel the hash tables. - for (i = 0; i < count; ++i) - { - bucket = bucketPicker((char *) &arr[i].key, 8, bpSeed) & ptMask; - tmpBuckets[bucket].push_back(make_pair(arr[i].key, arr[i].value)); - } + for (auto& element : tmpBuckets[i]) + tlJoiners[joinerNum][i]->insert(element); + addToJoinerLocks[joinerNum][i].unlock(); + tmpBuckets[i].clear(); + didSomeWork = true; + } + } + // if this iteration did no useful work, everything we need is locked; wait briefly + // and try again. + if (!done && !didSomeWork) + { + ::usleep(500 * processorThreads); + //++noWorkCounter; + } + } + // cout << "TL join insert. Took " << loopCounter << " loops" << endl; + } + else + { + boost::shared_array > tJoiner = tJoiners[joinerNum]; + uint64_t nullValue = joinNullValues[joinerNum]; + bool& l_doMatchNulls = doMatchNulls[joinerNum]; + joblist::JoinType joinType = joinTypes[joinerNum]; + utils::VLArray > > tmpBuckets(processorThreads); - bool done = false; - bool didSomeWork; - //uint loopCounter = 0, noWorkCounter = 0; - // this loop moves the elements from each vector into its corresponding hash table. - while (!done) - { - //++loopCounter; - done = true; - didSomeWork = false; - for (i = 0; i < processorThreads; ++i) - { - if (!tmpBuckets[i].empty()) - { - bool gotIt = addToJoinerLocks[joinerNum][i].try_lock(); - if (!gotIt) - { - done = false; // didn't get it, don't block, try the next bucket - continue; - } - for (auto &element : tmpBuckets[i]) - tJoiners[joinerNum][i]->insert(element); - addToJoinerLocks[joinerNum][i].unlock(); - tmpBuckets[i].clear(); - didSomeWork = true; - } - } - // if this iteration did no useful work, everything we need is locked; wait briefly - // and try again. - if (!done && !didSomeWork) - { - ::usleep(500 * processorThreads); - //++noWorkCounter; - } - - } - //cout << "T numeric join insert 2. Took " << loopCounter << " loops," << - // " unproductive iterations = " << noWorkCounter << endl; - } + if (joinType & MATCHNULLS) + { + // this first loop hashes incoming values into vectors that parallel the hash tables. + for (i = 0; i < count; ++i) + { + /* A minor optimization: the matchnull logic should only be used with + * the jointype specifies it and there's a null value in the small side */ + if (!l_doMatchNulls && arr[i].key == nullValue) + l_doMatchNulls = true; + bucket = bucketPicker((char*)&arr[i].key, 8, bpSeed) & ptMask; + tmpBuckets[bucket].push_back(make_pair(arr[i].key, arr[i].value)); } - if (!typelessJoin[joinerNum]) - bs.advance(count * sizeof(JoinerElements)); - - if (getTupleJoinRowGroupData) + bool done = false, didSomeWork; + // uint loopCounter = 0, noWorkCounter = 0; + // this loop moves the elements from each vector into its corresponding hash table. + while (!done) { - RowGroup& smallSide = smallSideRGs[joinerNum]; - RGData offTheWire; - - // TODO: write an RGData fcn to let it interpret data within a ByteStream to avoid - // the extra copying. - offTheWire.deserialize(bs); - boost::mutex::scoped_lock lk(smallSideDataLocks[joinerNum]); - smallSide.setData(&smallSideRowData[joinerNum]); - smallSide.append(offTheWire, startPos); - - /* This prints the row data - smallSideRGs[joinerNum].initRow(&r); - for (i = 0; i < (tJoinerSizes[joinerNum] * smallSideRowLengths[joinerNum]); i+=r.getSize()) { - r.setData(&smallSideRowData[joinerNum][i + smallSideRGs[joinerNum].getEmptySize()]); - cout << " got row: " << r.toString() << endl; - } - */ + //++loopCounter; + done = true; + didSomeWork = false; + for (i = 0; i < processorThreads; ++i) + { + if (!tmpBuckets[i].empty()) + { + bool gotIt = addToJoinerLocks[joinerNum][i].try_lock(); + if (!gotIt) + { + done = false; // didn't get it, don't block, try the next bucket + continue; + } + for (auto& element : tmpBuckets[i]) + tJoiners[joinerNum][i]->insert(element); + addToJoinerLocks[joinerNum][i].unlock(); + tmpBuckets[i].clear(); + didSomeWork = true; + } + } + // if this iteration did no useful work, everything we need is locked; wait briefly + // and try again. + if (!done && !didSomeWork) + { + ::usleep(500 * processorThreads); + //++noWorkCounter; + } } + + // cout << "T numeric join insert. Took " << loopCounter << " loops" << endl; + } + else + { + // this first loop hashes incoming values into vectors that parallel the hash tables. + for (i = 0; i < count; ++i) + { + bucket = bucketPicker((char*)&arr[i].key, 8, bpSeed) & ptMask; + tmpBuckets[bucket].push_back(make_pair(arr[i].key, arr[i].value)); + } + + bool done = false; + bool didSomeWork; + // uint loopCounter = 0, noWorkCounter = 0; + // this loop moves the elements from each vector into its corresponding hash table. + while (!done) + { + //++loopCounter; + done = true; + didSomeWork = false; + for (i = 0; i < processorThreads; ++i) + { + if (!tmpBuckets[i].empty()) + { + bool gotIt = addToJoinerLocks[joinerNum][i].try_lock(); + if (!gotIt) + { + done = false; // didn't get it, don't block, try the next bucket + continue; + } + for (auto& element : tmpBuckets[i]) + tJoiners[joinerNum][i]->insert(element); + addToJoinerLocks[joinerNum][i].unlock(); + tmpBuckets[i].clear(); + didSomeWork = true; + } + } + // if this iteration did no useful work, everything we need is locked; wait briefly + // and try again. + if (!done && !didSomeWork) + { + ::usleep(500 * processorThreads); + //++noWorkCounter; + } + } + // cout << "T numeric join insert 2. Took " << loopCounter << " loops," << + // " unproductive iterations = " << noWorkCounter << endl; + } } - idbassert(bs.length() == 0); + if (!typelessJoin[joinerNum]) + bs.advance(count * sizeof(JoinerElements)); + + if (getTupleJoinRowGroupData) + { + RowGroup& smallSide = smallSideRGs[joinerNum]; + RGData offTheWire; + + // TODO: write an RGData fcn to let it interpret data within a ByteStream to avoid + // the extra copying. + offTheWire.deserialize(bs); + boost::mutex::scoped_lock lk(smallSideDataLocks[joinerNum]); + smallSide.setData(&smallSideRowData[joinerNum]); + smallSide.append(offTheWire, startPos); + + /* This prints the row data + smallSideRGs[joinerNum].initRow(&r); + for (i = 0; i < (tJoinerSizes[joinerNum] * smallSideRowLengths[joinerNum]); + i+=r.getSize()) { r.setData(&smallSideRowData[joinerNum][i + + smallSideRGs[joinerNum].getEmptySize()]); cout << " got row: " << r.toString() << endl; + } + */ + } + } + + idbassert(bs.length() == 0); } void BatchPrimitiveProcessor::doneSendingJoinerData() { - /* to get wall-time of hash table construction - if (!firstCallTime.is_not_a_date_time() && !(sessionID & 0x80000000)) - { - boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); - Logger logger; - ostringstream os; - os << "id " << uniqueID << ": joiner construction time = " << now-firstCallTime; - logger.logMessage(os.str()); - cout << os.str() << endl; - } - */ + /* to get wall-time of hash table construction +if (!firstCallTime.is_not_a_date_time() && !(sessionID & 0x80000000)) +{ + boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + Logger logger; + ostringstream os; + os << "id " << uniqueID << ": joiner construction time = " << now-firstCallTime; + logger.logMessage(os.str()); + cout << os.str() << endl; +} + */ } int BatchPrimitiveProcessor::endOfJoiner() { - /* Wait for all joiner elements to be added */ - uint32_t i; - size_t currentSize; - // it should be safe to run this without grabbing this lock - //boost::mutex::scoped_lock scoped(addToJoinerLock); + /* Wait for all joiner elements to be added */ + uint32_t i; + size_t currentSize; + // it should be safe to run this without grabbing this lock + // boost::mutex::scoped_lock scoped(addToJoinerLock); - if (endOfJoinerRan) - return 0; - - // minor hack / optimization. The instances not inserting the table data don't - // need to check that the table is complete. - if (!firstInstance) - { - endOfJoinerRan = true; - pthread_mutex_unlock(&objLock); - return 0; - } - - for (i = 0; i < joinerCount; i++) - { - if (!typelessJoin[i]) - { - currentSize = 0; - for (uint j = 0; j < processorThreads; ++j) - if (!tJoiners[i] || !tJoiners[i][j]) - return -1; - else - currentSize += tJoiners[i][j]->size(); - if (currentSize != tJoinerSizes[i]) - return -1; - //if ((!tJoiners[i] || tJoiners[i]->size() != tJoinerSizes[i])) - // return -1; - } - else - { - currentSize = 0; - for (uint j = 0; j < processorThreads; ++j) - if (!tlJoiners[i] || !tlJoiners[i][j]) - return -1; - else - currentSize += tlJoiners[i][j]->size(); - if (currentSize != tJoinerSizes[i]) - return -1; - //if ((!tJoiners[i] || tlJoiners[i]->size() != tJoinerSizes[i])) - // return -1; - } - } + if (endOfJoinerRan) + return 0; + // minor hack / optimization. The instances not inserting the table data don't + // need to check that the table is complete. + if (!firstInstance) + { endOfJoinerRan = true; + pthread_mutex_unlock(&objLock); + return 0; + } + + for (i = 0; i < joinerCount; i++) + { + if (!typelessJoin[i]) + { + currentSize = 0; + for (uint j = 0; j < processorThreads; ++j) + if (!tJoiners[i] || !tJoiners[i][j]) + return -1; + else + currentSize += tJoiners[i][j]->size(); + if (currentSize != tJoinerSizes[i]) + return -1; + // if ((!tJoiners[i] || tJoiners[i]->size() != tJoinerSizes[i])) + // return -1; + } + else + { + currentSize = 0; + for (uint j = 0; j < processorThreads; ++j) + if (!tlJoiners[i] || !tlJoiners[i][j]) + return -1; + else + currentSize += tlJoiners[i][j]->size(); + if (currentSize != tJoinerSizes[i]) + return -1; + // if ((!tJoiners[i] || tlJoiners[i]->size() != tJoinerSizes[i])) + // return -1; + } + } + + endOfJoinerRan = true; #ifndef __FreeBSD__ - pthread_mutex_unlock(&objLock); + pthread_mutex_unlock(&objLock); #endif - return 0; + return 0; } void BatchPrimitiveProcessor::initProcessor() { - uint32_t i, j; + uint32_t i, j; - if (gotAbsRids || needStrValues || hasRowGroup) - absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + if (gotAbsRids || needStrValues || hasRowGroup) + absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); - if (needStrValues) - strValues.reset(new string[LOGICAL_BLOCK_RIDS]); + if (needStrValues) + strValues.reset(new string[LOGICAL_BLOCK_RIDS]); - outMsgSize = defaultBufferSize; - outputMsg.reset(reinterpret_cast(aligned_alloc(utils::MAXCOLUMNWIDTH, outMsgSize))); + outMsgSize = defaultBufferSize; + outputMsg.reset(reinterpret_cast(aligned_alloc(utils::MAXCOLUMNWIDTH, outMsgSize))); - if (ot == ROW_GROUP) + if (ot == ROW_GROUP) + { + // calculate the projection -> outputRG mapping + projectionMap.reset(new int[projectCount]); + bool* reserved = (bool*)alloca(outputRG.getColumnCount() * sizeof(bool)); + + for (i = 0; i < outputRG.getColumnCount(); i++) + reserved[i] = false; + + for (i = 0; i < projectCount; i++) { - // calculate the projection -> outputRG mapping - projectionMap.reset(new int[projectCount]); - bool* reserved = (bool*)alloca(outputRG.getColumnCount() * sizeof(bool)); - - for (i = 0; i < outputRG.getColumnCount(); i++) - reserved[i] = false; - - for (i = 0; i < projectCount; i++) + for (j = 0; j < outputRG.getColumnCount(); j++) + if (projectSteps[i]->getTupleKey() == outputRG.getKeys()[j] && !reserved[j]) { - for (j = 0; j < outputRG.getColumnCount(); j++) - if (projectSteps[i]->getTupleKey() == outputRG.getKeys()[j] && !reserved[j]) - { - projectionMap[i] = j; - reserved[j] = true; - break; - } - - if (j == outputRG.getColumnCount()) - projectionMap[i] = -1; + projectionMap[i] = j; + reserved[j] = true; + break; } - if (doJoin) - { - outputRG.initRow(&oldRow); - outputRG.initRow(&newRow); - - tSmallSideMatches.reset(new MatchedData[joinerCount]); - keyColumnProj.reset(new bool[projectCount]); - - for (i = 0; i < projectCount; i++) - { - keyColumnProj[i] = false; - - for (j = 0; j < joinerCount; j++) - { - if (!typelessJoin[j]) - { - if (projectionMap[i] == (int) largeSideKeyColumns[j]) - { - keyColumnProj[i] = true; - break; - } - } - else - { - for (uint32_t k = 0; k < tlLargeSideKeyColumns[j].size(); k++) - { - if (projectionMap[i] == (int) tlLargeSideKeyColumns[j][k]) - { - keyColumnProj[i] = true; - break; - } - } - } - } - } - - if (hasJoinFEFilters) - { - joinFERG->initRow(&joinFERow, true); - joinFERowData.reset(new uint8_t[joinFERow.getSize()]); - joinFERow.setData(joinFERowData.get()); - joinFEMappings.reset(new shared_array[joinerCount + 1]); - - for (i = 0; i < joinerCount; i++) - joinFEMappings[i] = makeMapping(smallSideRGs[i], *joinFERG); - - joinFEMappings[joinerCount] = makeMapping(largeSideRG, *joinFERG); - } - } - - /* - Calculate the FE1 -> projection mapping - Calculate the projection step -> FE1 input mapping - */ - if (fe1) - { - fe1ToProjection = makeMapping(fe1Input, outputRG); - projectForFE1.reset(new int[projectCount]); - bool* reserved = (bool*)alloca(fe1Input.getColumnCount() * sizeof(bool)); - - for (i = 0; i < fe1Input.getColumnCount(); i++) - reserved[i] = false; - - for (i = 0; i < projectCount; i++) - { - projectForFE1[i] = -1; - - for (j = 0; j < fe1Input.getColumnCount(); j++) - { - if (projectSteps[i]->getTupleKey() == fe1Input.getKeys()[j] && !reserved[j]) - { - reserved[j] = true; - projectForFE1[i] = j; - break; - } - } - } - - fe1Input.initRow(&fe1In); - outputRG.initRow(&fe1Out); - } - - if (fe2) - { - fe2Input = (doJoin ? &joinedRG : &outputRG); - fe2Mapping = makeMapping(*fe2Input, fe2Output); - fe2Input->initRow(&fe2In); - fe2Output.initRow(&fe2Out); - } - - if (getTupleJoinRowGroupData) - { - gjrgPlaceHolders.reset(new uint32_t[joinerCount]); - outputRG.initRow(&largeRow); - joinedRG.initRow(&joinedRow); - joinedRG.initRow(&baseJRow, true); - smallRows.reset(new Row[joinerCount]); - - for (i = 0; i < joinerCount; i++) - smallSideRGs[i].initRow(&smallRows[i], true); - - baseJRowMem.reset(new uint8_t[baseJRow.getSize()]); - baseJRow.setData(baseJRowMem.get()); - gjrgMappings.reset(new shared_array[joinerCount + 1]); - - for (i = 0; i < joinerCount; i++) - gjrgMappings[i] = makeMapping(smallSideRGs[i], joinedRG); - - gjrgMappings[joinerCount] = makeMapping(outputRG, joinedRG); - } + if (j == outputRG.getColumnCount()) + projectionMap[i] = -1; } - // @bug 1051 - if (hasFilterStep) + if (doJoin) { - for (uint64_t i = 0; i < 2; i++) - { - fFiltRidCount[i] = 0; - fFiltCmdRids[i].reset(new uint16_t[LOGICAL_BLOCK_RIDS]); - fFiltCmdValues[i].reset(new int64_t[LOGICAL_BLOCK_RIDS]); - if (wideColumnsWidths | datatypes::MAXDECIMALWIDTH) - fFiltCmdBinaryValues[i].reset(new int128_t[LOGICAL_BLOCK_RIDS]); + outputRG.initRow(&oldRow); + outputRG.initRow(&newRow); - if (filtOnString) fFiltStrValues[i].reset(new string[LOGICAL_BLOCK_RIDS]); + tSmallSideMatches.reset(new MatchedData[joinerCount]); + keyColumnProj.reset(new bool[projectCount]); + + for (i = 0; i < projectCount; i++) + { + keyColumnProj[i] = false; + + for (j = 0; j < joinerCount; j++) + { + if (!typelessJoin[j]) + { + if (projectionMap[i] == (int)largeSideKeyColumns[j]) + { + keyColumnProj[i] = true; + break; + } + } + else + { + for (uint32_t k = 0; k < tlLargeSideKeyColumns[j].size(); k++) + { + if (projectionMap[i] == (int)tlLargeSideKeyColumns[j][k]) + { + keyColumnProj[i] = true; + break; + } + } + } } + } + + if (hasJoinFEFilters) + { + joinFERG->initRow(&joinFERow, true); + joinFERowData.reset(new uint8_t[joinFERow.getSize()]); + joinFERow.setData(joinFERowData.get()); + joinFEMappings.reset(new shared_array[joinerCount + 1]); + + for (i = 0; i < joinerCount; i++) + joinFEMappings[i] = makeMapping(smallSideRGs[i], *joinFERG); + + joinFEMappings[joinerCount] = makeMapping(largeSideRG, *joinFERG); + } } - /* init the Commands */ - if (filterCount > 0) + /* + Calculate the FE1 -> projection mapping + Calculate the projection step -> FE1 input mapping + */ + if (fe1) { - for (i = 0; i < (uint32_t) filterCount - 1; ++i) + fe1ToProjection = makeMapping(fe1Input, outputRG); + projectForFE1.reset(new int[projectCount]); + bool* reserved = (bool*)alloca(fe1Input.getColumnCount() * sizeof(bool)); + + for (i = 0; i < fe1Input.getColumnCount(); i++) + reserved[i] = false; + + for (i = 0; i < projectCount; i++) + { + projectForFE1[i] = -1; + + for (j = 0; j < fe1Input.getColumnCount(); j++) { -// cout << "prepping filter " << i << endl; - filterSteps[i]->setBatchPrimitiveProcessor(this); - - if (filterSteps[i + 1]->getCommandType() == Command::DICT_STEP) - filterSteps[i]->prep(OT_BOTH, true); - else if (filterSteps[i]->filterFeeder() != Command::NOT_FEEDER) - filterSteps[i]->prep(OT_BOTH, false); - else - filterSteps[i]->prep(OT_RID, false); + if (projectSteps[i]->getTupleKey() == fe1Input.getKeys()[j] && !reserved[j]) + { + reserved[j] = true; + projectForFE1[i] = j; + break; + } } + } -// cout << "prepping filter " << i << endl; - filterSteps[i]->setBatchPrimitiveProcessor(this); + fe1Input.initRow(&fe1In); + outputRG.initRow(&fe1Out); + } + + if (fe2) + { + fe2Input = (doJoin ? &joinedRG : &outputRG); + fe2Mapping = makeMapping(*fe2Input, fe2Output); + fe2Input->initRow(&fe2In); + fe2Output.initRow(&fe2Out); + } + + if (getTupleJoinRowGroupData) + { + gjrgPlaceHolders.reset(new uint32_t[joinerCount]); + outputRG.initRow(&largeRow); + joinedRG.initRow(&joinedRow); + joinedRG.initRow(&baseJRow, true); + smallRows.reset(new Row[joinerCount]); + + for (i = 0; i < joinerCount; i++) + smallSideRGs[i].initRow(&smallRows[i], true); + + baseJRowMem.reset(new uint8_t[baseJRow.getSize()]); + baseJRow.setData(baseJRowMem.get()); + gjrgMappings.reset(new shared_array[joinerCount + 1]); + + for (i = 0; i < joinerCount; i++) + gjrgMappings[i] = makeMapping(smallSideRGs[i], joinedRG); + + gjrgMappings[joinerCount] = makeMapping(outputRG, joinedRG); + } + } + + // @bug 1051 + if (hasFilterStep) + { + for (uint64_t i = 0; i < 2; i++) + { + fFiltRidCount[i] = 0; + fFiltCmdRids[i].reset(new uint16_t[LOGICAL_BLOCK_RIDS]); + fFiltCmdValues[i].reset(new int64_t[LOGICAL_BLOCK_RIDS]); + if (wideColumnsWidths | datatypes::MAXDECIMALWIDTH) + fFiltCmdBinaryValues[i].reset(new int128_t[LOGICAL_BLOCK_RIDS]); + + if (filtOnString) + fFiltStrValues[i].reset(new string[LOGICAL_BLOCK_RIDS]); + } + } + + /* init the Commands */ + if (filterCount > 0) + { + for (i = 0; i < (uint32_t)filterCount - 1; ++i) + { + // cout << "prepping filter " << i << endl; + filterSteps[i]->setBatchPrimitiveProcessor(this); + + if (filterSteps[i + 1]->getCommandType() == Command::DICT_STEP) + filterSteps[i]->prep(OT_BOTH, true); + else if (filterSteps[i]->filterFeeder() != Command::NOT_FEEDER) filterSteps[i]->prep(OT_BOTH, false); + else + filterSteps[i]->prep(OT_RID, false); } - for (i = 0; i < projectCount; ++i) - { -// cout << "prepping projection " << i << endl; - projectSteps[i]->setBatchPrimitiveProcessor(this); + // cout << "prepping filter " << i << endl; + filterSteps[i]->setBatchPrimitiveProcessor(this); + filterSteps[i]->prep(OT_BOTH, false); + } - if (noVB) - projectSteps[i]->prep(OT_BOTH, false); + for (i = 0; i < projectCount; ++i) + { + // cout << "prepping projection " << i << endl; + projectSteps[i]->setBatchPrimitiveProcessor(this); + + if (noVB) + projectSteps[i]->prep(OT_BOTH, false); + else + projectSteps[i]->prep(OT_DATAVALUE, false); + + if (0 < filterCount) + { + // if there is an rtscommand with a passThru, the passThru must make its own absRids + // unless there is only one project step, then the last filter step can make absRids + RTSCommand* rts = dynamic_cast(projectSteps[i].get()); + + if (rts && rts->isPassThru()) + { + if (1 == projectCount) + filterSteps[filterCount - 1]->setMakeAbsRids(true); else - projectSteps[i]->prep(OT_DATAVALUE, false); - - if (0 < filterCount ) - { - //if there is an rtscommand with a passThru, the passThru must make its own absRids - //unless there is only one project step, then the last filter step can make absRids - RTSCommand* rts = dynamic_cast(projectSteps[i].get()); - - if (rts && rts->isPassThru()) - { - if (1 == projectCount) - filterSteps[filterCount - 1]->setMakeAbsRids(true); - else rts->setAbsNull(); - } - } + rts->setAbsNull(); + } } + } - if (fAggregator.get() != NULL) + if (fAggregator.get() != NULL) + { + // fAggRowGroupData.reset(new uint8_t[fAggregateRG.getMaxDataSize()]); + fAggRowGroupData.reinit(fAggregateRG); + fAggregateRG.setData(&fAggRowGroupData); + + if (doJoin) { - //fAggRowGroupData.reset(new uint8_t[fAggregateRG.getMaxDataSize()]); - fAggRowGroupData.reinit(fAggregateRG); - fAggregateRG.setData(&fAggRowGroupData); - - if (doJoin) - { - fAggregator->setInputOutput(fe2 ? fe2Output : joinedRG, &fAggregateRG); - fAggregator->setJoinRowGroups(&smallSideRGs, &largeSideRG); - } - else - fAggregator->setInputOutput(fe2 ? fe2Output : outputRG, &fAggregateRG); - } - - if (LIKELY(!hasWideColumnOut)) - { - minVal = MAX64; - maxVal = MIN64; + fAggregator->setInputOutput(fe2 ? fe2Output : joinedRG, &fAggregateRG); + fAggregator->setJoinRowGroups(&smallSideRGs, &largeSideRG); } else - { - max128Val = datatypes::Decimal::minInt128; - min128Val = datatypes::Decimal::maxInt128; - } + fAggregator->setInputOutput(fe2 ? fe2Output : outputRG, &fAggregateRG); + } - // @bug 1269, initialize data used by execute() for async loading blocks - // +1 for the scan filter step with no predicate, if any - relLBID.reset(new uint64_t[projectCount + 1]); - asyncLoaded.reset(new bool[projectCount + 1]); + if (LIKELY(!hasWideColumnOut)) + { + minVal = MAX64; + maxVal = MIN64; + } + else + { + max128Val = datatypes::Decimal::minInt128; + min128Val = datatypes::Decimal::maxInt128; + } + + // @bug 1269, initialize data used by execute() for async loading blocks + // +1 for the scan filter step with no predicate, if any + relLBID.reset(new uint64_t[projectCount + 1]); + asyncLoaded.reset(new bool[projectCount + 1]); } /* This version does a join on projected rows */ -// In order to prevent super size result sets in the case of near cartesian joins on three or more joins, -// the startRid start at 0) is used to begin the rid loop and if we cut off processing early because of +// In order to prevent super size result sets in the case of near cartesian joins on three or more joins, +// the startRid start at 0) is used to begin the rid loop and if we cut off processing early because of // the size of the result set, we return the next rid to start with. If we finish ridCount rids, return 0- uint32_t BatchPrimitiveProcessor::executeTupleJoin(uint32_t startRid) { - uint32_t newRowCount = 0, i, j; - vector matches; - uint64_t largeKey; - uint64_t resultCount = 0; - uint32_t newStartRid = startRid; - outputRG.getRow(0, &oldRow); - outputRG.getRow(0, &newRow); + uint32_t newRowCount = 0, i, j; + vector matches; + uint64_t largeKey; + uint64_t resultCount = 0; + uint32_t newStartRid = startRid; + outputRG.getRow(0, &oldRow); + outputRG.getRow(0, &newRow); - //cout << "before join, RG has " << outputRG.getRowCount() << " BPP ridcount= " << ridCount << endl; - // ridCount gets modified based on the number of Rids actually processed during this call. - // origRidCount is the number of rids for this thread after filter, which are the total - // number of rids to be processed from all calls to this function during this thread. - for (i = startRid; i < origRidCount && !sendThread->aborted(); i++, oldRow.nextRow()) + // cout << "before join, RG has " << outputRG.getRowCount() << " BPP ridcount= " << ridCount << endl; + // ridCount gets modified based on the number of Rids actually processed during this call. + // origRidCount is the number of rids for this thread after filter, which are the total + // number of rids to be processed from all calls to this function during this thread. + for (i = startRid; i < origRidCount && !sendThread->aborted(); i++, oldRow.nextRow()) + { + /* Decide whether this large-side row belongs in the output. The breaks + * in the loop mean that it doesn't. + * + * In English the logic is: + * Reject the row if there's no match and it's not an anti or an outer join + * or if there is a match and it's an anti join with no filter. + * If there's an antijoin with a filter nothing can be eliminated at this stage. + * If there's an antijoin where the join op should match NULL values, and there + * are NULL values to match against, but there is no filter, all rows can be eliminated. + */ + + // cout << "large side row: " << oldRow.toString() << endl; + for (j = 0; j < joinerCount; j++) { - /* Decide whether this large-side row belongs in the output. The breaks - * in the loop mean that it doesn't. - * - * In English the logic is: - * Reject the row if there's no match and it's not an anti or an outer join - * or if there is a match and it's an anti join with no filter. - * If there's an antijoin with a filter nothing can be eliminated at this stage. - * If there's an antijoin where the join op should match NULL values, and there - * are NULL values to match against, but there is no filter, all rows can be eliminated. + bool found; + + if (UNLIKELY(joinTypes[j] & ANTI)) + { + if (joinTypes[j] & WITHFCNEXP) + continue; + else if (doMatchNulls[j]) + break; + } + + if (LIKELY(!typelessJoin[j])) + { + // cout << "not typeless join\n"; + bool isNull; + uint32_t colIndex = largeSideKeyColumns[j]; + + if (oldRow.isUnsigned(colIndex)) + largeKey = oldRow.getUintField(colIndex); + else + largeKey = oldRow.getIntField(colIndex); + uint bucket = bucketPicker((char*)&largeKey, 8, bpSeed) & ptMask; + + bool joinerIsEmpty = tJoiners[j][bucket]->empty() ? true : false; + + found = (tJoiners[j][bucket]->find(largeKey) != tJoiners[j][bucket]->end()); + isNull = oldRow.isNullValue(colIndex); + /* These conditions define when the row is NOT in the result set: + * - if the key is not in the small side, and the join isn't a large-outer or anti join + * - if the key is NULL, and the join isn't anti- or large-outer + * - if it's an anti-join and the key is either in the small side or it's NULL */ - //cout << "large side row: " << oldRow.toString() << endl; - for (j = 0; j < joinerCount; j++) + if (((!found || isNull) && !(joinTypes[j] & (LARGEOUTER | ANTI))) || + ((joinTypes[j] & ANTI) && !joinerIsEmpty && + ((isNull && (joinTypes[j] & MATCHNULLS)) || (found && !isNull)))) { - bool found; - - if (UNLIKELY(joinTypes[j] & ANTI)) - { - if (joinTypes[j] & WITHFCNEXP) - continue; - else if (doMatchNulls[j]) - break; - } - - - if (LIKELY(!typelessJoin[j])) - { - //cout << "not typeless join\n"; - bool isNull; - uint32_t colIndex = largeSideKeyColumns[j]; - - if (oldRow.isUnsigned(colIndex)) - largeKey = oldRow.getUintField(colIndex); - else - largeKey = oldRow.getIntField(colIndex); - uint bucket = bucketPicker((char *) &largeKey, 8, bpSeed) & ptMask; - - bool joinerIsEmpty = tJoiners[j][bucket]->empty() ? true : false; - - found = (tJoiners[j][bucket]->find(largeKey) != tJoiners[j][bucket]->end()); - isNull = oldRow.isNullValue(colIndex); - /* These conditions define when the row is NOT in the result set: - * - if the key is not in the small side, and the join isn't a large-outer or anti join - * - if the key is NULL, and the join isn't anti- or large-outer - * - if it's an anti-join and the key is either in the small side or it's NULL - */ - - if (((!found || isNull) && !(joinTypes[j] & (LARGEOUTER | ANTI))) || - ((joinTypes[j] & ANTI) && !joinerIsEmpty && ((isNull && (joinTypes[j] & MATCHNULLS)) || (found && !isNull)))) - { - //cout << " - not in the result set\n"; - break; - } - - //else - // cout << " - in the result set\n"; - } - else - { - //cout << " typeless join\n"; - // the null values are not sent by UM in typeless case. null -> !found - TypelessData tlLargeKey(&oldRow); - uint bucket = oldRow.hashTypeless(tlLargeSideKeyColumns[j], - mSmallSideKeyColumnsPtr, - mSmallSideRGPtr ? &mSmallSideRGPtr->getColWidths() : nullptr) & ptMask; - found = tlJoiners[j][bucket]->find(tlLargeKey) != tlJoiners[j][bucket]->end(); - - if ((!found && !(joinTypes[j] & (LARGEOUTER | ANTI))) || - (joinTypes[j] & ANTI)) - { - - /* Separated the ANTI join logic for readability. - * - */ - if (joinTypes[j] & ANTI) - { - if (found) - break; - else if (joinTypes[j] & MATCHNULLS) - { - bool hasNull = false; - - for (uint32_t z = 0; z < tlLargeSideKeyColumns[j].size(); z++) - if (oldRow.isNullValue(tlLargeSideKeyColumns[j][z])) - { - hasNull = true; - break; - } - - if (hasNull) // keys with nulls match everything - break; - else - continue; // non-null keys not in the small side - - // are in the result - } - else // signifies a not-exists query - continue; - } - - break; - } - } + // cout << " - not in the result set\n"; + break; } - if (j == joinerCount) + // else + // cout << " - in the result set\n"; + } + else + { + // cout << " typeless join\n"; + // the null values are not sent by UM in typeless case. null -> !found + TypelessData tlLargeKey(&oldRow); + uint bucket = oldRow.hashTypeless(tlLargeSideKeyColumns[j], mSmallSideKeyColumnsPtr, + mSmallSideRGPtr ? &mSmallSideRGPtr->getColWidths() : nullptr) & + ptMask; + found = tlJoiners[j][bucket]->find(tlLargeKey) != tlJoiners[j][bucket]->end(); + + if ((!found && !(joinTypes[j] & (LARGEOUTER | ANTI))) || (joinTypes[j] & ANTI)) { - uint32_t matchCount; - for (j = 0; j < joinerCount; j++) + /* Separated the ANTI join logic for readability. + * + */ + if (joinTypes[j] & ANTI) + { + if (found) + break; + else if (joinTypes[j] & MATCHNULLS) { - /* The result is already known if... - * -- anti-join with no fcnexp - * -- semi-join with no fcnexp and not scalar - * - * The ANTI join case isn't just a shortcut. getJoinResults() will produce results - * for a different case and generate the wrong result. Need to fix that, later. - */ - if ((joinTypes[j] & (SEMI | ANTI)) && !(joinTypes[j] & WITHFCNEXP) && !(joinTypes[j] & SCALAR)) + bool hasNull = false; + + for (uint32_t z = 0; z < tlLargeSideKeyColumns[j].size(); z++) + if (oldRow.isNullValue(tlLargeSideKeyColumns[j][z])) { - tSmallSideMatches[j][newRowCount].push_back(-1); - continue; + hasNull = true; + break; } - getJoinResults(oldRow, j, tSmallSideMatches[j][newRowCount]); - matchCount = tSmallSideMatches[j][newRowCount].size(); + if (hasNull) // keys with nulls match everything + break; + else + continue; // non-null keys not in the small side - if (joinTypes[j] & WITHFCNEXP) - { - vector newMatches; - applyMapping(joinFEMappings[joinerCount], oldRow, &joinFERow); - - for (uint32_t k = 0; k < matchCount; k++) - { - if (tSmallSideMatches[j][newRowCount][k] == (uint32_t) - 1) - smallRows[j].setPointer(smallNullPointers[j]); - else - { - smallSideRGs[j].getRow(tSmallSideMatches[j][newRowCount][k], &smallRows[j]); - //uint64_t rowOffset = ((uint64_t) tSmallSideMatches[j][newRowCount][k]) * - // smallRows[j].getSize() + smallSideRGs[j].getEmptySize(); - //smallRows[j].setData(&smallSideRowData[j][rowOffset]); - } - - applyMapping(joinFEMappings[j], smallRows[j], &joinFERow); - - if (joinFEFilters[j]->evaluate(&joinFERow)) - { - /* The first match includes it in a SEMI join result and excludes it from an ANTI join - * result. If it's SEMI & SCALAR however, it needs to continue. - */ - newMatches.push_back(tSmallSideMatches[j][newRowCount][k]); - - if ((joinTypes[j] & ANTI) || ((joinTypes[j] & (SEMI | SCALAR)) == SEMI)) - break; - } - } - - tSmallSideMatches[j][newRowCount].swap(newMatches); - matchCount = tSmallSideMatches[j][newRowCount].size(); - } - - if (matchCount == 0 && (joinTypes[j] & LARGEOUTER)) - { - tSmallSideMatches[j][newRowCount].push_back(-1); - matchCount = 1; - } - - /* Scalar check */ - if ((joinTypes[j] & SCALAR) && matchCount > 1) - throw scalar_exception(); - - /* Reverse the result for anti-join */ - if (joinTypes[j] & ANTI) - { - if (matchCount == 0) - { - tSmallSideMatches[j][newRowCount].push_back(-1); - matchCount = 1; - } - else - { - tSmallSideMatches[j][newRowCount].clear(); - matchCount = 0; - } - } - - /* For all join types, no matches here means it's not in the result */ - if (matchCount == 0) - break; - - /* Pair non-scalar semi-joins with a NULL row */ - if ((joinTypes[j] & SEMI) && !(joinTypes[j] & SCALAR)) - { - tSmallSideMatches[j][newRowCount].clear(); - tSmallSideMatches[j][newRowCount].push_back(-1); - matchCount = 1; - } - - resultCount += matchCount; + // are in the result } + else // signifies a not-exists query + continue; + } - /* Finally, copy the row into the output */ - if (j == joinerCount) - { - // We need to update 8 and 16 bytes in values and wide128Values buffers - // otherwise unrelated values will be observed in the JOIN-ed output RGData. - if (i != newRowCount) - { - values[newRowCount] = values[i]; - if (mJOINHasSkewedKeyColumn) - wide128Values[newRowCount] = wide128Values[i]; - relRids[newRowCount] = relRids[i]; - copyRow(oldRow, &newRow); - //cout << "joined row: " << newRow.toString() << endl; - } - - newRowCount++; - newRow.nextRow(); - } - //else - // cout << "j != joinerCount\n"; - } - // If we've accumulated more than maxResultCount -- 1048576 (2^20)_ of resultCounts, cut off processing. The caller will - // restart to continue where we left off. - if (resultCount >= maxResultCount) - { - newStartRid += newRowCount; - break; + break; } + } } - if (resultCount < maxResultCount) - newStartRid = 0; - - ridCount = newRowCount; - outputRG.setRowCount(ridCount); + if (j == joinerCount) + { + uint32_t matchCount; + for (j = 0; j < joinerCount; j++) + { + /* The result is already known if... + * -- anti-join with no fcnexp + * -- semi-join with no fcnexp and not scalar + * + * The ANTI join case isn't just a shortcut. getJoinResults() will produce results + * for a different case and generate the wrong result. Need to fix that, later. + */ + if ((joinTypes[j] & (SEMI | ANTI)) && !(joinTypes[j] & WITHFCNEXP) && !(joinTypes[j] & SCALAR)) + { + tSmallSideMatches[j][newRowCount].push_back(-1); + continue; + } - /* prints out the whole result set. - if (ridCount != 0) { - cout << "RG rowcount=" << outputRG.getRowCount() << " BPP ridcount=" << ridCount << endl; - for (i = 0; i < joinerCount; i++) { - for (j = 0; j < ridCount; j++) { - cout << "joiner " << i << " has " << tSmallSideMatches[i][j].size() << " entries" << endl; - cout << "row " << j << ":"; - for (uint32_t k = 0; k < tSmallSideMatches[i][j].size(); k++) - cout << " " << tSmallSideMatches[i][j][k]; - cout << endl; - } - cout << endl; - } - } - */ - return newStartRid; + getJoinResults(oldRow, j, tSmallSideMatches[j][newRowCount]); + matchCount = tSmallSideMatches[j][newRowCount].size(); + + if (joinTypes[j] & WITHFCNEXP) + { + vector newMatches; + applyMapping(joinFEMappings[joinerCount], oldRow, &joinFERow); + + for (uint32_t k = 0; k < matchCount; k++) + { + if (tSmallSideMatches[j][newRowCount][k] == (uint32_t)-1) + smallRows[j].setPointer(smallNullPointers[j]); + else + { + smallSideRGs[j].getRow(tSmallSideMatches[j][newRowCount][k], &smallRows[j]); + // uint64_t rowOffset = ((uint64_t) tSmallSideMatches[j][newRowCount][k]) * + // smallRows[j].getSize() + smallSideRGs[j].getEmptySize(); + // smallRows[j].setData(&smallSideRowData[j][rowOffset]); + } + + applyMapping(joinFEMappings[j], smallRows[j], &joinFERow); + + if (joinFEFilters[j]->evaluate(&joinFERow)) + { + /* The first match includes it in a SEMI join result and excludes it from an ANTI join + * result. If it's SEMI & SCALAR however, it needs to continue. + */ + newMatches.push_back(tSmallSideMatches[j][newRowCount][k]); + + if ((joinTypes[j] & ANTI) || ((joinTypes[j] & (SEMI | SCALAR)) == SEMI)) + break; + } + } + + tSmallSideMatches[j][newRowCount].swap(newMatches); + matchCount = tSmallSideMatches[j][newRowCount].size(); + } + + if (matchCount == 0 && (joinTypes[j] & LARGEOUTER)) + { + tSmallSideMatches[j][newRowCount].push_back(-1); + matchCount = 1; + } + + /* Scalar check */ + if ((joinTypes[j] & SCALAR) && matchCount > 1) + throw scalar_exception(); + + /* Reverse the result for anti-join */ + if (joinTypes[j] & ANTI) + { + if (matchCount == 0) + { + tSmallSideMatches[j][newRowCount].push_back(-1); + matchCount = 1; + } + else + { + tSmallSideMatches[j][newRowCount].clear(); + matchCount = 0; + } + } + + /* For all join types, no matches here means it's not in the result */ + if (matchCount == 0) + break; + + /* Pair non-scalar semi-joins with a NULL row */ + if ((joinTypes[j] & SEMI) && !(joinTypes[j] & SCALAR)) + { + tSmallSideMatches[j][newRowCount].clear(); + tSmallSideMatches[j][newRowCount].push_back(-1); + matchCount = 1; + } + + resultCount += matchCount; + } + + /* Finally, copy the row into the output */ + if (j == joinerCount) + { + // We need to update 8 and 16 bytes in values and wide128Values buffers + // otherwise unrelated values will be observed in the JOIN-ed output RGData. + if (i != newRowCount) + { + values[newRowCount] = values[i]; + if (mJOINHasSkewedKeyColumn) + wide128Values[newRowCount] = wide128Values[i]; + relRids[newRowCount] = relRids[i]; + copyRow(oldRow, &newRow); + // cout << "joined row: " << newRow.toString() << endl; + } + + newRowCount++; + newRow.nextRow(); + } + // else + // cout << "j != joinerCount\n"; + } + // If we've accumulated more than maxResultCount -- 1048576 (2^20)_ of resultCounts, cut off processing. + // The caller will restart to continue where we left off. + if (resultCount >= maxResultCount) + { + newStartRid += newRowCount; + break; + } + } + + if (resultCount < maxResultCount) + newStartRid = 0; + + ridCount = newRowCount; + outputRG.setRowCount(ridCount); + + /* prints out the whole result set. + if (ridCount != 0) { + cout << "RG rowcount=" << outputRG.getRowCount() << " BPP ridcount=" << ridCount << endl; + for (i = 0; i < joinerCount; i++) { + for (j = 0; j < ridCount; j++) { + cout << "joiner " << i << " has " << tSmallSideMatches[i][j].size() << " + entries" << endl; cout << "row " << j << ":"; for (uint32_t k = 0; k < tSmallSideMatches[i][j].size(); + k++) cout << " " << tSmallSideMatches[i][j][k]; cout << endl; + } + cout << endl; + } + } + */ + return newStartRid; } #ifdef PRIMPROC_STOPWATCH @@ -1431,529 +1424,519 @@ void BatchPrimitiveProcessor::execute(StopWatch* stopwatch) void BatchPrimitiveProcessor::execute() #endif { - uint8_t sendCount = 0; -// bool smoreRGs = false; -// uint32_t sStartRid = 0; - uint32_t i, j; + uint8_t sendCount = 0; + // bool smoreRGs = false; + // uint32_t sStartRid = 0; + uint32_t i, j; - try + try + { +#ifdef PRIMPROC_STOPWATCH + stopwatch->start("BatchPrimitiveProcessor::execute first part"); +#endif + + // if only one scan step which has no predicate, async load all columns + if (filterCount == 1 && hasScan) + { + ColumnCommand* col = dynamic_cast(filterSteps[0].get()); + + if ((col != NULL) && (col->getFilterCount() == 0) && (col->getLBID() != 0)) + { + // stored in last pos in relLBID[] and asyncLoaded[] + uint64_t p = projectCount; + asyncLoaded[p] = asyncLoaded[p] && (relLBID[p] % blocksReadAhead != 0); + relLBID[p] += col->getWidth(); + + if (!asyncLoaded[p] && col->willPrefetch()) + { + loadBlockAsync(col->getLBID(), versionInfo, txnID, col->getCompType(), &cachedIO, &physIO, + LBIDTrace, sessionID, &counterLock, &busyLoaderCount, sendThread, &vssCache); + asyncLoaded[p] = true; + } + + asyncLoadProjectColumns(); + } + } + +#ifdef PRIMPROC_STOPWATCH + stopwatch->stop("BatchPrimitiveProcessor::execute first part"); + stopwatch->start("BatchPrimitiveProcessor::execute second part"); +#endif + + // filters use relrids and values for intermediate results. + if (bop == BOP_AND) + for (j = 0; j < filterCount; ++j) + { +#ifdef PRIMPROC_STOPWATCH + stopwatch->start("- filterSteps[j]->execute()"); + filterSteps[j]->execute(); + stopwatch->stop("- filterSteps[j]->execute()"); +#else + filterSteps[j]->execute(); +#endif + } + else // BOP_OR + { + /* XXXPAT: This is a hacky impl of OR logic. Each filter is configured to + be a scan operation on init. This code runs each independently and + unions their output ridlists using accumulator. At the end it turns + accumulator into a final ridlist for subsequent steps. + + If there's a join or a passthru command in the projection list, the + values array has to contain values from the last filter step. In that + case, the last filter step isn't part of the "OR" filter processing. + JLF has added it to prep those operations, not to be a filter. + + 7/7/09 update: the multiple-table join required relocating the join op. It's + no longer necessary to add the loader columncommand to the filter array. + */ + + bool accumulator[LOGICAL_BLOCK_RIDS]; + // uint32_t realFilterCount = ((forHJ || hasPassThru) ? filterCount - 1 : filterCount); + uint32_t realFilterCount = filterCount; + + for (i = 0; i < LOGICAL_BLOCK_RIDS; i++) + accumulator[i] = false; + + if (!hasScan) // there are input rids + for (i = 0; i < ridCount; i++) + accumulator[relRids[i]] = true; + + ridCount = 0; + + for (i = 0; i < realFilterCount; ++i) + { + filterSteps[i]->execute(); + + if (!filterSteps[i]->filterFeeder()) + { + for (j = 0; j < ridCount; j++) + accumulator[relRids[j]] = true; + + ridCount = 0; + } + } + + for (ridMap = 0, i = 0; i < LOGICAL_BLOCK_RIDS; ++i) + { + if (accumulator[i]) + { + relRids[ridCount] = i; + ridMap |= 1 << (relRids[ridCount] >> 9); + ++ridCount; + } + } + } + +#ifdef PRIMPROC_STOPWATCH + stopwatch->stop("BatchPrimitiveProcessor::execute second part"); + stopwatch->start("BatchPrimitiveProcessor::execute third part"); +#endif + + if (projectCount > 0 || ot == ROW_GROUP) { #ifdef PRIMPROC_STOPWATCH - stopwatch->start("BatchPrimitiveProcessor::execute first part"); + stopwatch->start("- writeProjectionPreamble"); + writeProjectionPreamble(); + stopwatch->stop("- writeProjectionPreamble"); +#else + writeProjectionPreamble(); +#endif + } + + // async load blocks for project phase, if not alread loaded + if (ridCount > 0) + { +#ifdef PRIMPROC_STOPWATCH + stopwatch->start("- asyncLoadProjectColumns"); + asyncLoadProjectColumns(); + stopwatch->stop("- asyncLoadProjectColumns"); +#else + asyncLoadProjectColumns(); +#endif + } + +#ifdef PRIMPROC_STOPWATCH + stopwatch->stop("BatchPrimitiveProcessor::execute third part"); + stopwatch->start("BatchPrimitiveProcessor::execute fourth part"); #endif - // if only one scan step which has no predicate, async load all columns - if (filterCount == 1 && hasScan) + // projection commands read relrids and write output directly to a rowgroup + // or the serialized bytestream + if (ot != ROW_GROUP) + for (j = 0; j < projectCount; ++j) + { + projectSteps[j]->project(); + } + else + { + /* Function & Expression group 1 processing + - project for FE1 + - execute FE1 row by row + - if return value = true, map input row into the projection RG, adjust ridlist + */ +#ifdef PRIMPROC_STOPWATCH + stopwatch->start("- if(ot != ROW_GROUP) else"); +#endif + outputRG.resetRowGroup(baseRid); + + if (fe1) + { + uint32_t newRidCount = 0; + fe1Input.resetRowGroup(baseRid); + fe1Input.setRowCount(ridCount); + fe1Input.getRow(0, &fe1In); + outputRG.getRow(0, &fe1Out); + + for (j = 0; j < projectCount; j++) + if (projectForFE1[j] != -1) + projectSteps[j]->projectIntoRowGroup(fe1Input, projectForFE1[j]); + + for (j = 0; j < ridCount; j++, fe1In.nextRow()) + if (fe1->evaluate(&fe1In)) + { + applyMapping(fe1ToProjection, fe1In, &fe1Out); + relRids[newRidCount] = relRids[j]; + values[newRidCount++] = values[j]; + fe1Out.nextRow(); + } + + ridCount = newRidCount; + } + + outputRG.setRowCount(ridCount); + + if (sendRidsAtDelivery) + { + Row r; + outputRG.initRow(&r); + outputRG.getRow(0, &r); + + for (j = 0; j < ridCount; ++j) { - ColumnCommand* col = dynamic_cast(filterSteps[0].get()); + r.setRid(relRids[j]); + r.nextRow(); + } + } - if ((col != NULL) && (col->getFilterCount() == 0) && (col->getLBID() != 0)) + /* 7/7/09 PL: I Changed the projection alg to reduce block touches when there's + a join. The key columns get projected first, the join is executed to further + reduce the ridlist, then the rest of the columns get projected */ + + if (!doJoin) + { + for (j = 0; j < projectCount; ++j) + { + // cout << "projectionMap[" << j << "] = " << projectionMap[j] << endl; + if (projectionMap[j] != -1) + { +#ifdef PRIMPROC_STOPWATCH + stopwatch->start("-- projectIntoRowGroup"); + projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); + stopwatch->stop("-- projectIntoRowGroup"); +#else + projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); +#endif + } + + // else + // cout << " no target found for OID " << projectSteps[j]->getOID() << + //endl; + } + if (fe2) + { + /* functionize this -> processFE2() */ + fe2Output.resetRowGroup(baseRid); + fe2Output.getRow(0, &fe2Out); + fe2Input->getRow(0, &fe2In); + + // cerr << "input row: " << fe2In.toString() << endl; + for (j = 0; j < outputRG.getRowCount(); j++, fe2In.nextRow()) + { + if (fe2->evaluate(&fe2In)) { - // stored in last pos in relLBID[] and asyncLoaded[] - uint64_t p = projectCount; - asyncLoaded[p] = asyncLoaded[p] && (relLBID[p] % blocksReadAhead != 0); - relLBID[p] += col->getWidth(); - - if (!asyncLoaded[p] && col->willPrefetch()) - { - loadBlockAsync(col->getLBID(), - versionInfo, - txnID, - col->getCompType(), - &cachedIO, - &physIO, - LBIDTrace, - sessionID, - &counterLock, - &busyLoaderCount, - sendThread, - &vssCache); - asyncLoaded[p] = true; - } - - asyncLoadProjectColumns(); + applyMapping(fe2Mapping, fe2In, &fe2Out); + // cerr << " passed. output row: " << fe2Out.toString() << endl; + fe2Out.setRid(fe2In.getRelRid()); + fe2Output.incRowCount(); + fe2Out.nextRow(); } + } + + if (!fAggregator) + { + *serialized << (uint8_t)1; // the "count this msg" var + fe2Output.setDBRoot(dbRoot); + fe2Output.serializeRGData(*serialized); + //*serialized << fe2Output.getDataSize(); + // serialized->append(fe2Output.getData(), fe2Output.getDataSize()); + } + } + + if (fAggregator) + { + *serialized << (uint8_t)1; // the "count this msg" var + + RowGroup& toAggregate = (fe2 ? fe2Output : outputRG); + // toAggregate.convertToInlineDataInPlace(); + + if (fe2) + fe2Output.setDBRoot(dbRoot); + else + outputRG.setDBRoot(dbRoot); + + fAggregator->addRowGroup(&toAggregate); + + if ((currentBlockOffset + 1) == count) // @bug4507, 8k + { + fAggregator->loadResult(*serialized); // @bug4507, 8k + } // @bug4507, 8k + else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k + { + fAggregator->loadEmptySet(*serialized); // @bug4507, 8k + } // @bug4507, 8k + else // @bug4507, 8k + { + fAggregator->loadResult(*serialized); // @bug4507, 8k + fAggregator->aggReset(); // @bug4507, 8k + } // @bug4507, 8k + } + + if (!fAggregator && !fe2) + { + *serialized << (uint8_t)1; // the "count this msg" var + outputRG.setDBRoot(dbRoot); + // cerr << "serializing " << outputRG.toString() << endl; + outputRG.serializeRGData(*serialized); + + //*serialized << outputRG.getDataSize(); + // serialized->append(outputRG.getData(), outputRG.getDataSize()); } #ifdef PRIMPROC_STOPWATCH - stopwatch->stop("BatchPrimitiveProcessor::execute first part"); - stopwatch->start("BatchPrimitiveProcessor::execute second part"); + stopwatch->stop("- if(ot != ROW_GROUP) else"); #endif - - // filters use relrids and values for intermediate results. - if (bop == BOP_AND) - for (j = 0; j < filterCount; ++j) - { -#ifdef PRIMPROC_STOPWATCH - stopwatch->start("- filterSteps[j]->execute()"); - filterSteps[j]->execute(); - stopwatch->stop("- filterSteps[j]->execute()"); -#else - filterSteps[j]->execute(); -#endif - } - else // BOP_OR + } + else // Is doJoin + { + uint32_t startRid = 0; + ByteStream preamble = *serialized; + origRidCount = ridCount; // ridCount can get modified by executeTupleJoin(). We need to keep track of + // the original val. + /* project the key columns. If there's the filter IN the join, project everything. + Also need to project 'long' strings b/c executeTupleJoin may copy entire rows + using copyRow(), which will try to interpret the uninit'd string ptr. + Valgrind will legitimately complain about copying uninit'd values for the + other types but that is technically safe. */ + for (j = 0; j < projectCount; j++) { - - /* XXXPAT: This is a hacky impl of OR logic. Each filter is configured to - be a scan operation on init. This code runs each independently and - unions their output ridlists using accumulator. At the end it turns - accumulator into a final ridlist for subsequent steps. - - If there's a join or a passthru command in the projection list, the - values array has to contain values from the last filter step. In that - case, the last filter step isn't part of the "OR" filter processing. - JLF has added it to prep those operations, not to be a filter. - - 7/7/09 update: the multiple-table join required relocating the join op. It's - no longer necessary to add the loader columncommand to the filter array. - */ - - bool accumulator[LOGICAL_BLOCK_RIDS]; -// uint32_t realFilterCount = ((forHJ || hasPassThru) ? filterCount - 1 : filterCount); - uint32_t realFilterCount = filterCount; - - for (i = 0; i < LOGICAL_BLOCK_RIDS; i++) - accumulator[i] = false; - - if (!hasScan) // there are input rids - for (i = 0; i < ridCount; i++) - accumulator[relRids[i]] = true; - - ridCount = 0; - - for (i = 0; i < realFilterCount; ++i) - { - filterSteps[i]->execute(); - - if (! filterSteps[i]->filterFeeder()) - { - for (j = 0; j < ridCount; j++) - accumulator[relRids[j]] = true; - - ridCount = 0; - } - } - - for (ridMap = 0, i = 0; i < LOGICAL_BLOCK_RIDS; ++i) - { - if (accumulator[i]) - { - relRids[ridCount] = i; - ridMap |= 1 << (relRids[ridCount] >> 9); - ++ridCount; - } - } + if (keyColumnProj[j] || + (projectionMap[j] != -1 && (hasJoinFEFilters || oldRow.isLongString(projectionMap[j])))) + { +#ifdef PRIMPROC_STOPWATCH + stopwatch->start("-- projectIntoRowGroup"); + projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); + stopwatch->stop("-- projectIntoRowGroup"); +#else + projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); +#endif + } } -#ifdef PRIMPROC_STOPWATCH - stopwatch->stop("BatchPrimitiveProcessor::execute second part"); - stopwatch->start("BatchPrimitiveProcessor::execute third part"); -#endif - - if (projectCount > 0 || ot == ROW_GROUP) + do // while (startRid > 0) { #ifdef PRIMPROC_STOPWATCH - stopwatch->start("- writeProjectionPreamble"); - writeProjectionPreamble(); - stopwatch->stop("- writeProjectionPreamble"); + stopwatch->start("-- executeTupleJoin()"); + startRid = executeTupleJoin(startRid); + stopwatch->stop("-- executeTupleJoin()"); #else - writeProjectionPreamble(); -#endif - } - - // async load blocks for project phase, if not alread loaded - if (ridCount > 0) - { -#ifdef PRIMPROC_STOPWATCH - stopwatch->start("- asyncLoadProjectColumns"); - asyncLoadProjectColumns(); - stopwatch->stop("- asyncLoadProjectColumns"); -#else - asyncLoadProjectColumns(); -#endif - } - -#ifdef PRIMPROC_STOPWATCH - stopwatch->stop("BatchPrimitiveProcessor::execute third part"); - stopwatch->start("BatchPrimitiveProcessor::execute fourth part"); -#endif - - // projection commands read relrids and write output directly to a rowgroup - // or the serialized bytestream - if (ot != ROW_GROUP) - for (j = 0; j < projectCount; ++j) - { - projectSteps[j]->project(); - } - else - { - /* Function & Expression group 1 processing - - project for FE1 - - execute FE1 row by row - - if return value = true, map input row into the projection RG, adjust ridlist - */ -#ifdef PRIMPROC_STOPWATCH - stopwatch->start("- if(ot != ROW_GROUP) else"); -#endif - outputRG.resetRowGroup(baseRid); - - if (fe1) - { - uint32_t newRidCount = 0; - fe1Input.resetRowGroup(baseRid); - fe1Input.setRowCount(ridCount); - fe1Input.getRow(0, &fe1In); - outputRG.getRow(0, &fe1Out); - - for (j = 0; j < projectCount; j++) - if (projectForFE1[j] != -1) - projectSteps[j]->projectIntoRowGroup(fe1Input, projectForFE1[j]); - - for (j = 0; j < ridCount; j++, fe1In.nextRow()) - if (fe1->evaluate(&fe1In)) - { - applyMapping(fe1ToProjection, fe1In, &fe1Out); - relRids[newRidCount] = relRids[j]; - values[newRidCount++] = values[j]; - fe1Out.nextRow(); - } - - ridCount = newRidCount; - } - - outputRG.setRowCount(ridCount); - - if (sendRidsAtDelivery) - { - Row r; - outputRG.initRow(&r); - outputRG.getRow(0, &r); - - for (j = 0; j < ridCount; ++j) - { - r.setRid(relRids[j]); - r.nextRow(); - } - } - - /* 7/7/09 PL: I Changed the projection alg to reduce block touches when there's - a join. The key columns get projected first, the join is executed to further - reduce the ridlist, then the rest of the columns get projected */ - - if (!doJoin) - { - for (j = 0; j < projectCount; ++j) - { -// cout << "projectionMap[" << j << "] = " << projectionMap[j] << endl; - if (projectionMap[j] != -1) - { -#ifdef PRIMPROC_STOPWATCH - stopwatch->start("-- projectIntoRowGroup"); - projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); - stopwatch->stop("-- projectIntoRowGroup"); -#else - projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); -#endif - } - -// else -// cout << " no target found for OID " << projectSteps[j]->getOID() << endl; - } - if (fe2) - { - /* functionize this -> processFE2() */ - fe2Output.resetRowGroup ( baseRid ); - fe2Output.getRow ( 0, &fe2Out ); - fe2Input->getRow ( 0, &fe2In ); - - //cerr << "input row: " << fe2In.toString() << endl; - for ( j = 0; j < outputRG.getRowCount(); j++, fe2In.nextRow() ) - { - if ( fe2->evaluate ( &fe2In ) ) - { - applyMapping ( fe2Mapping, fe2In, &fe2Out ); - //cerr << " passed. output row: " << fe2Out.toString() << endl; - fe2Out.setRid ( fe2In.getRelRid() ); - fe2Output.incRowCount(); - fe2Out.nextRow(); - } - } - - if ( !fAggregator ) - { - *serialized << ( uint8_t ) 1; // the "count this msg" var - fe2Output.setDBRoot ( dbRoot ); - fe2Output.serializeRGData ( *serialized ); - //*serialized << fe2Output.getDataSize(); - //serialized->append(fe2Output.getData(), fe2Output.getDataSize()); - } - } - - if (fAggregator) - { - *serialized << ( uint8_t ) 1; // the "count this msg" var - - RowGroup& toAggregate = ( fe2 ? fe2Output : outputRG ); - //toAggregate.convertToInlineDataInPlace(); - - if ( fe2 ) - fe2Output.setDBRoot ( dbRoot ); - else - outputRG.setDBRoot ( dbRoot ); - - fAggregator->addRowGroup ( &toAggregate ); - - if ( ( currentBlockOffset + 1 ) == count ) // @bug4507, 8k - { - fAggregator->loadResult ( *serialized ); // @bug4507, 8k - } // @bug4507, 8k - else if ( utils::MonitorProcMem::isMemAvailable() ) // @bug4507, 8k - { - fAggregator->loadEmptySet ( *serialized ); // @bug4507, 8k - } // @bug4507, 8k - else // @bug4507, 8k - { - fAggregator->loadResult ( *serialized ); // @bug4507, 8k - fAggregator->aggReset(); // @bug4507, 8k - } // @bug4507, 8k - } - - if (!fAggregator && !fe2) - { - *serialized << ( uint8_t ) 1; // the "count this msg" var - outputRG.setDBRoot ( dbRoot ); - //cerr << "serializing " << outputRG.toString() << endl; - outputRG.serializeRGData ( *serialized ); - - //*serialized << outputRG.getDataSize(); - //serialized->append(outputRG.getData(), outputRG.getDataSize()); - } - -#ifdef PRIMPROC_STOPWATCH - stopwatch->stop ( "- if(ot != ROW_GROUP) else" ); -#endif - } - else // Is doJoin - { - uint32_t startRid = 0; - ByteStream preamble = *serialized; - origRidCount = ridCount; // ridCount can get modified by executeTupleJoin(). We need to keep track of the original val. - /* project the key columns. If there's the filter IN the join, project everything. - Also need to project 'long' strings b/c executeTupleJoin may copy entire rows - using copyRow(), which will try to interpret the uninit'd string ptr. - Valgrind will legitimately complain about copying uninit'd values for the - other types but that is technically safe. */ - for (j = 0; j < projectCount; j++) - { - if (keyColumnProj[j] || (projectionMap[j] != -1 && (hasJoinFEFilters || - oldRow.isLongString(projectionMap[j])))) - { -#ifdef PRIMPROC_STOPWATCH - stopwatch->start("-- projectIntoRowGroup"); - projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); - stopwatch->stop("-- projectIntoRowGroup"); -#else - projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); -#endif - } - } - - do //while (startRid > 0) - { -#ifdef PRIMPROC_STOPWATCH - stopwatch->start("-- executeTupleJoin()"); - startRid = executeTupleJoin(startRid); - stopwatch->stop("-- executeTupleJoin()"); -#else - startRid = executeTupleJoin(startRid); + startRid = executeTupleJoin(startRid); // sStartRid = startRid; #endif - /* project the non-key columns */ - for (j = 0; j < projectCount; ++j) - { - if (projectionMap[j] != -1 && !keyColumnProj[j] && !hasJoinFEFilters && - !oldRow.isLongString(projectionMap[j])) - { + /* project the non-key columns */ + for (j = 0; j < projectCount; ++j) + { + if (projectionMap[j] != -1 && !keyColumnProj[j] && !hasJoinFEFilters && + !oldRow.isLongString(projectionMap[j])) + { #ifdef PRIMPROC_STOPWATCH - stopwatch->start("-- projectIntoRowGroup"); - projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); - stopwatch->stop("-- projectIntoRowGroup"); + stopwatch->start("-- projectIntoRowGroup"); + projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); + stopwatch->stop("-- projectIntoRowGroup"); #else - projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); + projectSteps[j]->projectIntoRowGroup(outputRG, projectionMap[j]); #endif - } - } - /* The RowGroup is fully joined at this point. - * Add additional RowGroup processing here. - * TODO: Try to clean up all of the switching */ - - if (fe2 || fAggregator) - { - bool moreRGs = true; - initGJRG(); - - while (moreRGs && !sendThread->aborted()) - { - /* - * generate 1 rowgroup (8192 rows max) of joined rows - * if there's an FE2, run it - * -pack results into a new rowgroup - * -if there are < 8192 rows in the new RG, continue - * if there's an agg, run it - * send the result - */ - resetGJRG(); - moreRGs = generateJoinedRowGroup(baseJRow); -// smoreRGs = moreRGs; - sendCount = (uint8_t)(!moreRGs && !startRid); -// *serialized << (uint8_t)(!moreRGs && !startRid); // the "count this msg" var - *serialized << sendCount; - if (fe2) - { - /* functionize this -> processFE2()*/ - fe2Output.resetRowGroup(baseRid); - fe2Output.setDBRoot(dbRoot); - fe2Output.getRow(0, &fe2Out); - fe2Input->getRow(0, &fe2In); - - for (j = 0; j < joinedRG.getRowCount(); j++, fe2In.nextRow()) - if (fe2->evaluate(&fe2In)) - { - applyMapping(fe2Mapping, fe2In, &fe2Out); - fe2Out.setRid(fe2In.getRelRid()); - fe2Output.incRowCount(); - fe2Out.nextRow(); - } - } - - RowGroup& nextRG = (fe2 ? fe2Output : joinedRG); - nextRG.setDBRoot(dbRoot); - - if (fAggregator) - { - fAggregator->addRowGroup(&nextRG); - - if ((currentBlockOffset + 1) == count && moreRGs == false && startRid == 0) // @bug4507, 8k - { - fAggregator->loadResult(*serialized); // @bug4507, 8k - } // @bug4507, 8k - else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k - { - fAggregator->loadEmptySet(*serialized); // @bug4507, 8k - } // @bug4507, 8k - else // @bug4507, 8k - { - fAggregator->loadResult(*serialized); // @bug4507, 8k - fAggregator->aggReset(); // @bug4507, 8k - } // @bug4507, 8k - } - else - { - //cerr <<" * serialzing " << nextRG.toString() << endl; - nextRG.serializeRGData(*serialized); - } - - /* send the msg & reinit the BS */ - if (moreRGs) - { - sendResponse(); - serialized.reset(new ByteStream()); - *serialized = preamble; - } - } - - if ( hasSmallOuterJoin ) - { - // Should we happen to finish sending data rows right on the boundary of when moreRGs flips off, then - // we need to start a new buffer. I.e., it needs the count this message byte pushed. - if (serialized->length() == preamble.length()) - *serialized << ( uint8_t )(startRid > 0 ? 0 : 1); // the "count this msg" var - - *serialized << ridCount; - - for ( i = 0; i < joinerCount; i++ ) - { - for ( j = 0; j < ridCount; ++j ) - { - serializeInlineVector ( *serialized, - tSmallSideMatches[i][j] ); - tSmallSideMatches[i][j].clear(); - } - } - } - else - { - // We hae no more use for this allocation - for ( i = 0; i < joinerCount; i++ ) - for ( j = 0; j < ridCount; ++j ) - tSmallSideMatches[i][j].clear(); - } - } - else - { - *serialized << ( uint8_t )(startRid > 0 ? 0 : 1); // the "count this msg" var - outputRG.setDBRoot ( dbRoot ); - //cerr << "serializing " << outputRG.toString() << endl; - outputRG.serializeRGData ( *serialized ); - - //*serialized << outputRG.getDataSize(); - //serialized->append(outputRG.getData(), outputRG.getDataSize()); - for ( i = 0; i < joinerCount; i++ ) - { - for ( j = 0; j < ridCount; ++j ) - { - serializeInlineVector ( *serialized, - tSmallSideMatches[i][j] ); - tSmallSideMatches[i][j].clear(); - } - } - } - if (startRid > 0) - { - sendResponse(); - serialized.reset ( new ByteStream() ); - *serialized = preamble; - } - }while (startRid > 0); } + } + /* The RowGroup is fully joined at this point. + * Add additional RowGroup processing here. + * TODO: Try to clean up all of the switching */ + + if (fe2 || fAggregator) + { + bool moreRGs = true; + initGJRG(); + + while (moreRGs && !sendThread->aborted()) + { + /* + * generate 1 rowgroup (8192 rows max) of joined rows + * if there's an FE2, run it + * -pack results into a new rowgroup + * -if there are < 8192 rows in the new RG, continue + * if there's an agg, run it + * send the result + */ + resetGJRG(); + moreRGs = generateJoinedRowGroup(baseJRow); + // smoreRGs = moreRGs; + sendCount = (uint8_t)(!moreRGs && !startRid); + // *serialized << (uint8_t)(!moreRGs && !startRid); // the "count + // this msg" var + *serialized << sendCount; + if (fe2) + { + /* functionize this -> processFE2()*/ + fe2Output.resetRowGroup(baseRid); + fe2Output.setDBRoot(dbRoot); + fe2Output.getRow(0, &fe2Out); + fe2Input->getRow(0, &fe2In); + + for (j = 0; j < joinedRG.getRowCount(); j++, fe2In.nextRow()) + if (fe2->evaluate(&fe2In)) + { + applyMapping(fe2Mapping, fe2In, &fe2Out); + fe2Out.setRid(fe2In.getRelRid()); + fe2Output.incRowCount(); + fe2Out.nextRow(); + } + } + + RowGroup& nextRG = (fe2 ? fe2Output : joinedRG); + nextRG.setDBRoot(dbRoot); + + if (fAggregator) + { + fAggregator->addRowGroup(&nextRG); + + if ((currentBlockOffset + 1) == count && moreRGs == false && startRid == 0) // @bug4507, 8k + { + fAggregator->loadResult(*serialized); // @bug4507, 8k + } // @bug4507, 8k + else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k + { + fAggregator->loadEmptySet(*serialized); // @bug4507, 8k + } // @bug4507, 8k + else // @bug4507, 8k + { + fAggregator->loadResult(*serialized); // @bug4507, 8k + fAggregator->aggReset(); // @bug4507, 8k + } // @bug4507, 8k + } + else + { + // cerr <<" * serialzing " << nextRG.toString() << endl; + nextRG.serializeRGData(*serialized); + } + + /* send the msg & reinit the BS */ + if (moreRGs) + { + sendResponse(); + serialized.reset(new ByteStream()); + *serialized = preamble; + } + } + + if (hasSmallOuterJoin) + { + // Should we happen to finish sending data rows right on the boundary of when moreRGs flips off, + // then we need to start a new buffer. I.e., it needs the count this message byte pushed. + if (serialized->length() == preamble.length()) + *serialized << (uint8_t)(startRid > 0 ? 0 : 1); // the "count this msg" var + + *serialized << ridCount; + + for (i = 0; i < joinerCount; i++) + { + for (j = 0; j < ridCount; ++j) + { + serializeInlineVector(*serialized, tSmallSideMatches[i][j]); + tSmallSideMatches[i][j].clear(); + } + } + } + else + { + // We hae no more use for this allocation + for (i = 0; i < joinerCount; i++) + for (j = 0; j < ridCount; ++j) + tSmallSideMatches[i][j].clear(); + } + } + else + { + *serialized << (uint8_t)(startRid > 0 ? 0 : 1); // the "count this msg" var + outputRG.setDBRoot(dbRoot); + // cerr << "serializing " << outputRG.toString() << endl; + outputRG.serializeRGData(*serialized); + + //*serialized << outputRG.getDataSize(); + // serialized->append(outputRG.getData(), outputRG.getDataSize()); + for (i = 0; i < joinerCount; i++) + { + for (j = 0; j < ridCount; ++j) + { + serializeInlineVector(*serialized, tSmallSideMatches[i][j]); + tSmallSideMatches[i][j].clear(); + } + } + } + if (startRid > 0) + { + sendResponse(); + serialized.reset(new ByteStream()); + *serialized = preamble; + } + } while (startRid > 0); + } #ifdef PRIMPROC_STOPWATCH - stopwatch->stop("- if(ot != ROW_GROUP) else"); + stopwatch->stop("- if(ot != ROW_GROUP) else"); #endif - } - ridCount = origRidCount; // May not be needed, but just to be safe. -// std::cout << "end of send. startRid=" << sStartRid << " moreRG=" << smoreRGs << " sendCount=" << sendCount << std::endl; - if (projectCount > 0 || ot == ROW_GROUP) - { - *serialized << cachedIO; - cachedIO = 0; - *serialized << physIO; - physIO = 0; - *serialized << touchedBlocks; - touchedBlocks = 0; -// cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO << -// " touchedBlocks=" << touchedBlocks << endl; - } + } + ridCount = origRidCount; // May not be needed, but just to be safe. + // std::cout << "end of send. startRid=" << sStartRid << " moreRG=" << smoreRGs << " sendCount=" << + // sendCount << std::endl; + if (projectCount > 0 || ot == ROW_GROUP) + { + *serialized << cachedIO; + cachedIO = 0; + *serialized << physIO; + physIO = 0; + *serialized << touchedBlocks; + touchedBlocks = 0; + // cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO << + // " touchedBlocks=" << touchedBlocks << endl; + } #ifdef PRIMPROC_STOPWATCH - stopwatch->stop("BatchPrimitiveProcessor::execute fourth part"); + stopwatch->stop("BatchPrimitiveProcessor::execute fourth part"); #endif - - } - catch (logging::QueryDataExcept& qex) - { - writeErrorMsg(qex.what(), qex.errorCode()); - } - catch (logging::DictionaryBufferOverflow& db) - { - writeErrorMsg(db.what(), db.errorCode()); - } - catch (scalar_exception& se) - { - writeErrorMsg(IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW), ERR_MORE_THAN_1_ROW, false); - } - catch (NeedToRestartJob& n) - { + } + catch (logging::QueryDataExcept& qex) + { + writeErrorMsg(qex.what(), qex.errorCode()); + } + catch (logging::DictionaryBufferOverflow& db) + { + writeErrorMsg(db.what(), db.errorCode()); + } + catch (scalar_exception& se) + { + writeErrorMsg(IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW), ERR_MORE_THAN_1_ROW, false); + } + catch (NeedToRestartJob& n) + { #if 0 /* This block of code will flush the problematic OIDs from the @@ -2013,536 +1996,528 @@ void BatchPrimitiveProcessor::execute() #endif #ifndef __FreeBSD__ - pthread_mutex_unlock(&objLock); + pthread_mutex_unlock(&objLock); #endif - throw n; // need to pass this through to BPPSeeder - } - catch (IDBExcept& iex) - { - writeErrorMsg(iex.what(), iex.errorCode(), true, false); - } - catch (const std::exception& ex) - { - writeErrorMsg(ex.what(), logging::batchPrimitiveProcessorErr); - } - catch (...) - { - string msg("BatchPrimitiveProcessor caught an unknown exception"); - writeErrorMsg(msg, logging::batchPrimitiveProcessorErr); - } + throw n; // need to pass this through to BPPSeeder + } + catch (IDBExcept& iex) + { + writeErrorMsg(iex.what(), iex.errorCode(), true, false); + } + catch (const std::exception& ex) + { + writeErrorMsg(ex.what(), logging::batchPrimitiveProcessorErr); + } + catch (...) + { + string msg("BatchPrimitiveProcessor caught an unknown exception"); + writeErrorMsg(msg, logging::batchPrimitiveProcessorErr); + } } void BatchPrimitiveProcessor::writeErrorMsg(const string& error, uint16_t errCode, bool logIt, bool critical) { - ISMPacketHeader ism; - PrimitiveHeader ph; + ISMPacketHeader ism; + PrimitiveHeader ph; - // we don't need every field of these headers. Init'ing them anyway - // makes memory checkers happy. - void *ismp = static_cast(&ism); - void *php = static_cast(&ph); - memset(ismp, 0, sizeof(ISMPacketHeader)); - memset(php, 0, sizeof(PrimitiveHeader)); - ph.SessionID = sessionID; - ph.StepID = stepID; - ph.UniqueID = uniqueID; - ism.Status = errCode; + // we don't need every field of these headers. Init'ing them anyway + // makes memory checkers happy. + void* ismp = static_cast(&ism); + void* php = static_cast(&ph); + memset(ismp, 0, sizeof(ISMPacketHeader)); + memset(php, 0, sizeof(PrimitiveHeader)); + ph.SessionID = sessionID; + ph.StepID = stepID; + ph.UniqueID = uniqueID; + ism.Status = errCode; - serialized.reset(new ByteStream()); - serialized->append((uint8_t*) &ism, sizeof(ism)); - serialized->append((uint8_t*) &ph, sizeof(ph)); - *serialized << error; + serialized.reset(new ByteStream()); + serialized->append((uint8_t*)&ism, sizeof(ism)); + serialized->append((uint8_t*)&ph, sizeof(ph)); + *serialized << error; - if (logIt) - { - Logger log; - log.logMessage(error, critical); - } + if (logIt) + { + Logger log; + log.logMessage(error, critical); + } } void BatchPrimitiveProcessor::writeProjectionPreamble() { - ISMPacketHeader ism; - PrimitiveHeader ph; + ISMPacketHeader ism; + PrimitiveHeader ph; - // we don't need every field of these headers. Init'ing them anyway - // makes memory checkers happy. - void *ismp = static_cast(&ism); - void *php = static_cast(&ph); - memset(ismp, 0, sizeof(ISMPacketHeader)); - memset(php, 0, sizeof(PrimitiveHeader)); - ph.SessionID = sessionID; - ph.StepID = stepID; - ph.UniqueID = uniqueID; + // we don't need every field of these headers. Init'ing them anyway + // makes memory checkers happy. + void* ismp = static_cast(&ism); + void* php = static_cast(&ph); + memset(ismp, 0, sizeof(ISMPacketHeader)); + memset(php, 0, sizeof(PrimitiveHeader)); + ph.SessionID = sessionID; + ph.StepID = stepID; + ph.UniqueID = uniqueID; - serialized.reset(new ByteStream()); - serialized->append((uint8_t*) &ism, sizeof(ism)); - serialized->append((uint8_t*) &ph, sizeof(ph)); + serialized.reset(new ByteStream()); + serialized->append((uint8_t*)&ism, sizeof(ism)); + serialized->append((uint8_t*)&ph, sizeof(ph)); - /* add-ons */ - if (hasScan) + /* add-ons */ + if (hasScan) + { + if (validCPData) { - if (validCPData) - { - *serialized << (uint8_t) 1; - *serialized << lbidForCP; - if (UNLIKELY(hasWideColumnOut)) - { - // PSA width - *serialized << (uint8_t) wideColumnWidthOut; - *serialized << min128Val; - *serialized << max128Val; - } - else - { - *serialized << (uint8_t) utils::MAXLEGACYWIDTH; // width of min/max value - *serialized << (uint64_t) minVal; - *serialized << (uint64_t) maxVal; - } - } - else - { - *serialized << (uint8_t) 0; - *serialized << lbidForCP; - } + *serialized << (uint8_t)1; + *serialized << lbidForCP; + if (UNLIKELY(hasWideColumnOut)) + { + // PSA width + *serialized << (uint8_t)wideColumnWidthOut; + *serialized << min128Val; + *serialized << max128Val; + } + else + { + *serialized << (uint8_t)utils::MAXLEGACYWIDTH; // width of min/max value + *serialized << (uint64_t)minVal; + *serialized << (uint64_t)maxVal; + } } - -// ridsOut += ridCount; - /* results */ - - if (ot != ROW_GROUP) + else { - *serialized << ridCount; - - if (sendRidsAtDelivery) - { - *serialized << baseRid; - serialized->append((uint8_t*) relRids, ridCount << 1); - } + *serialized << (uint8_t)0; + *serialized << lbidForCP; } + } + + // ridsOut += ridCount; + /* results */ + + if (ot != ROW_GROUP) + { + *serialized << ridCount; + + if (sendRidsAtDelivery) + { + *serialized << baseRid; + serialized->append((uint8_t*)relRids, ridCount << 1); + } + } } void BatchPrimitiveProcessor::serializeElementTypes() { - - *serialized << baseRid; - *serialized << ridCount; - serialized->append((uint8_t*) relRids, ridCount << 1); - serialized->append((uint8_t*) values, ridCount << 3); + *serialized << baseRid; + *serialized << ridCount; + serialized->append((uint8_t*)relRids, ridCount << 1); + serialized->append((uint8_t*)values, ridCount << 3); } void BatchPrimitiveProcessor::serializeStrings() { + *serialized << ridCount; + serialized->append((uint8_t*)absRids.get(), ridCount << 3); - *serialized << ridCount; - serialized->append((uint8_t*) absRids.get(), ridCount << 3); - - for (uint32_t i = 0; i < ridCount; ++i) - *serialized << strValues[i]; + for (uint32_t i = 0; i < ridCount; ++i) + *serialized << strValues[i]; } void BatchPrimitiveProcessor::sendResponse() { + if (sendThread->flowControlEnabled()) + { + // newConnection should be set only for the first result of a batch job + // it tells sendthread it should consider it for the connection array + sendThread->sendResult(BPPSendThread::Msg_t(serialized, sock, writelock, sockIndex), newConnection); + newConnection = false; + } + else + { + boost::mutex::scoped_lock lk(*writelock); + sock->write(*serialized); + } - if (sendThread->flowControlEnabled()) - { - // newConnection should be set only for the first result of a batch job - // it tells sendthread it should consider it for the connection array - sendThread->sendResult(BPPSendThread::Msg_t(serialized, sock, writelock, sockIndex), newConnection); - newConnection = false; - } - else - { - boost::mutex::scoped_lock lk(*writelock); - sock->write(*serialized); - } - - serialized.reset(); + serialized.reset(); } /* The output of a filter chain is either ELEMENT_TYPE or STRING_ELEMENT_TYPE */ void BatchPrimitiveProcessor::makeResponse() { - ISMPacketHeader ism; - PrimitiveHeader ph; + ISMPacketHeader ism; + PrimitiveHeader ph; - // we don't need every field of these headers. Init'ing them anyway - // makes memory checkers happy. - void *ismp = static_cast(&ism); - void *php = static_cast(&ph); - memset(ismp, 0, sizeof(ISMPacketHeader)); - memset(php, 0, sizeof(PrimitiveHeader)); - ph.SessionID = sessionID; - ph.StepID = stepID; - ph.UniqueID = uniqueID; + // we don't need every field of these headers. Init'ing them anyway + // makes memory checkers happy. + void* ismp = static_cast(&ism); + void* php = static_cast(&ph); + memset(ismp, 0, sizeof(ISMPacketHeader)); + memset(php, 0, sizeof(PrimitiveHeader)); + ph.SessionID = sessionID; + ph.StepID = stepID; + ph.UniqueID = uniqueID; - serialized.reset(new ByteStream()); - serialized->append((uint8_t*) &ism, sizeof(ism)); - serialized->append((uint8_t*) &ph, sizeof(ph)); + serialized.reset(new ByteStream()); + serialized->append((uint8_t*)&ism, sizeof(ism)); + serialized->append((uint8_t*)&ph, sizeof(ph)); - /* add-ons */ - if (hasScan) + /* add-ons */ + if (hasScan) + { + if (validCPData) { - if (validCPData) - { - *serialized << (uint8_t) 1; - *serialized << lbidForCP; + *serialized << (uint8_t)1; + *serialized << lbidForCP; - if (UNLIKELY(hasWideColumnOut)) - { - // PSA width - // Remove the assert for >16 bytes DTs. - assert(wideColumnWidthOut == datatypes::MAXDECIMALWIDTH); - *serialized << (uint8_t) wideColumnWidthOut; - *serialized << min128Val; - *serialized << max128Val; - } - else - { - *serialized << (uint8_t) utils::MAXLEGACYWIDTH; // width of min/max value - *serialized << (uint64_t) minVal; - *serialized << (uint64_t) maxVal; - } - } - else - { - *serialized << (uint8_t) 0; - *serialized << lbidForCP; - } + if (UNLIKELY(hasWideColumnOut)) + { + // PSA width + // Remove the assert for >16 bytes DTs. + assert(wideColumnWidthOut == datatypes::MAXDECIMALWIDTH); + *serialized << (uint8_t)wideColumnWidthOut; + *serialized << min128Val; + *serialized << max128Val; + } + else + { + *serialized << (uint8_t)utils::MAXLEGACYWIDTH; // width of min/max value + *serialized << (uint64_t)minVal; + *serialized << (uint64_t)maxVal; + } + } + else + { + *serialized << (uint8_t)0; + *serialized << lbidForCP; + } + } + + /* results */ + /* Take the rid and value arrays, munge into OutputType ot */ + switch (ot) + { + case BPS_ELEMENT_TYPE: serializeElementTypes(); break; + + case STRING_ELEMENT_TYPE: serializeStrings(); break; + + default: + { + ostringstream oss; + oss << "BPP: makeResponse(): Bad output type: " << ot; + throw logic_error(oss.str()); } - /* results */ - /* Take the rid and value arrays, munge into OutputType ot */ - switch (ot) - { - case BPS_ELEMENT_TYPE: - serializeElementTypes(); - break; + // throw logic_error("BPP: makeResponse(): Bad output type"); + } - case STRING_ELEMENT_TYPE: - serializeStrings(); - break; + *serialized << cachedIO; + cachedIO = 0; + *serialized << physIO; + physIO = 0; + *serialized << touchedBlocks; + touchedBlocks = 0; - default: - { - ostringstream oss; - oss << "BPP: makeResponse(): Bad output type: " << ot; - throw logic_error(oss.str()); - } - - //throw logic_error("BPP: makeResponse(): Bad output type"); - } - - *serialized << cachedIO; - cachedIO = 0; - *serialized << physIO; - physIO = 0; - *serialized << touchedBlocks; - touchedBlocks = 0; - -// cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO << -// " touchedBlocks=" << touchedBlocks << endl; + // cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO << + // " touchedBlocks=" << touchedBlocks << endl; } int BatchPrimitiveProcessor::operator()() { - utils::setThreadName("PPBatchPrimProc"); + utils::setThreadName("PPBatchPrimProc"); #ifdef PRIMPROC_STOPWATCH - const static std::string msg{"BatchPrimitiveProcessor::operator()"}; - logging::StopWatch* stopwatch = profiler.getTimer(); + const static std::string msg{"BatchPrimitiveProcessor::operator()"}; + logging::StopWatch* stopwatch = profiler.getTimer(); #endif - if (currentBlockOffset == 0) - { + if (currentBlockOffset == 0) + { #ifdef PRIMPROC_STOPWATCH - stopwatch->start(msg); + stopwatch->start(msg); #endif - idbassert(count > 0); + idbassert(count > 0); + } + + if (fAggregator && currentBlockOffset == 0) // @bug4507, 8k + fAggregator->aggReset(); // @bug4507, 8k + + for (; currentBlockOffset < count; currentBlockOffset++) + { + if (!(sessionID & 0x80000000)) // can't do this with syscat queries + { + if (sendThread->aborted()) + break; + + if (sendThread->sizeTooBig()) + { + // The send buffer is full of messages yet to be sent, so this thread would block anyway. + freeLargeBuffers(); + return -1; // the reschedule error code + } } - if (fAggregator && currentBlockOffset == 0) // @bug4507, 8k - fAggregator->aggReset(); // @bug4507, 8k + allocLargeBuffers(); - for (; currentBlockOffset < count; currentBlockOffset++) + if (LIKELY(!hasWideColumnOut)) { - if (!(sessionID & 0x80000000)) // can't do this with syscat queries - { - if (sendThread->aborted()) - break; + minVal = MAX64; + maxVal = MIN64; + } + else + { + max128Val = datatypes::Decimal::minInt128; + min128Val = datatypes::Decimal::maxInt128; + } - if (sendThread->sizeTooBig()) - { - // The send buffer is full of messages yet to be sent, so this thread would block anyway. - freeLargeBuffers(); - return -1; // the reschedule error code - } - } - - allocLargeBuffers(); - - if (LIKELY(!hasWideColumnOut)) - { - minVal = MAX64; - maxVal = MIN64; - } - else - { - max128Val = datatypes::Decimal::minInt128; - min128Val = datatypes::Decimal::maxInt128; - } - - validCPData = false; + validCPData = false; #ifdef PRIMPROC_STOPWATCH - stopwatch->start("BPP() execute"); - execute(stopwatch); - stopwatch->stop("BPP() execute"); + stopwatch->start("BPP() execute"); + execute(stopwatch); + stopwatch->stop("BPP() execute"); #else - execute(); + execute(); #endif - if (projectCount == 0 && ot != ROW_GROUP) - makeResponse(); + if (projectCount == 0 && ot != ROW_GROUP) + makeResponse(); - try - { - sendResponse(); - } - catch (std::exception& e) - { - break; // If we make this throw, be sure to do the cleanup at the end - } - - // Bug 4475: Control outgoing socket so that all messages from a - // batch go out the same socket - sockIndex = (sockIndex + 1) % connectionsPerUM; - - nextLBID(); - - /* Base RIDs are now a combination of partition#, segment#, extent#, and block#. */ - uint32_t partNum; - uint16_t segNum; - uint8_t extentNum; - uint16_t blockNum; - rowgroup::getLocationFromRid(baseRid, &partNum, &segNum, &extentNum, &blockNum); - /* - cout << "baseRid=" << baseRid << " partNum=" << partNum << " segNum=" << segNum << - " extentNum=" << (int) extentNum - << " blockNum=" << blockNum << endl; - */ - blockNum++; - baseRid = rowgroup::convertToRid(partNum, segNum, extentNum, blockNum); - /* - cout << "-- baseRid=" << baseRid << " partNum=" << partNum << " extentNum=" << (int) extentNum - << " blockNum=" << blockNum << endl; - */ + try + { + sendResponse(); } - vssCache.clear(); + catch (std::exception& e) + { + break; // If we make this throw, be sure to do the cleanup at the end + } + + // Bug 4475: Control outgoing socket so that all messages from a + // batch go out the same socket + sockIndex = (sockIndex + 1) % connectionsPerUM; + + nextLBID(); + + /* Base RIDs are now a combination of partition#, segment#, extent#, and block#. */ + uint32_t partNum; + uint16_t segNum; + uint8_t extentNum; + uint16_t blockNum; + rowgroup::getLocationFromRid(baseRid, &partNum, &segNum, &extentNum, &blockNum); + /* + cout << "baseRid=" << baseRid << " partNum=" << partNum << " segNum=" << segNum << + " extentNum=" << (int) extentNum + << " blockNum=" << blockNum << endl; + */ + blockNum++; + baseRid = rowgroup::convertToRid(partNum, segNum, extentNum, blockNum); + /* + cout << "-- baseRid=" << baseRid << " partNum=" << partNum << " extentNum=" << (int) extentNum + << " blockNum=" << blockNum << endl; + */ + } + vssCache.clear(); #ifndef __FreeBSD__ - pthread_mutex_unlock(&objLock); + pthread_mutex_unlock(&objLock); #endif - freeLargeBuffers(); + freeLargeBuffers(); #ifdef PRIMPROC_STOPWATCH - stopwatch->stop(msg); + stopwatch->stop(msg); #endif -// cout << "sent " << count << " responses" << endl; - fBusy = false; - return 0; + // cout << "sent " << count << " responses" << endl; + fBusy = false; + return 0; } void BatchPrimitiveProcessor::allocLargeBuffers() { - if (ot == ROW_GROUP && !outRowGroupData) - { - //outputRG.setUseStringTable(true); - outRowGroupData.reset(new RGData(outputRG)); - outputRG.setData(outRowGroupData.get()); - } + if (ot == ROW_GROUP && !outRowGroupData) + { + // outputRG.setUseStringTable(true); + outRowGroupData.reset(new RGData(outputRG)); + outputRG.setData(outRowGroupData.get()); + } - if (fe1 && !fe1Data) - { - //fe1Input.setUseStringTable(true); - fe1Data.reset(new RGData(fe1Input)); - //fe1Data.reset(new uint8_t[fe1Input.getMaxDataSize()]); - fe1Input.setData(fe1Data.get()); - } + if (fe1 && !fe1Data) + { + // fe1Input.setUseStringTable(true); + fe1Data.reset(new RGData(fe1Input)); + // fe1Data.reset(new uint8_t[fe1Input.getMaxDataSize()]); + fe1Input.setData(fe1Data.get()); + } - if (fe2 && !fe2Data) - { - //fe2Output.setUseStringTable(true); - fe2Data.reset(new RGData(fe2Output)); - fe2Output.setData(fe2Data.get()); - } + if (fe2 && !fe2Data) + { + // fe2Output.setUseStringTable(true); + fe2Data.reset(new RGData(fe2Output)); + fe2Output.setData(fe2Data.get()); + } - if (getTupleJoinRowGroupData && !joinedRGMem) - { - //joinedRG.setUseStringTable(true); - joinedRGMem.reset(new RGData(joinedRG)); - joinedRG.setData(joinedRGMem.get()); - } + if (getTupleJoinRowGroupData && !joinedRGMem) + { + // joinedRG.setUseStringTable(true); + joinedRGMem.reset(new RGData(joinedRG)); + joinedRG.setData(joinedRGMem.get()); + } } void BatchPrimitiveProcessor::freeLargeBuffers() { - /* Get rid of large buffers */ - if (ot == ROW_GROUP && outputRG.getMaxDataSizeWithStrings() > maxIdleBufferSize) - outRowGroupData.reset(); + /* Get rid of large buffers */ + if (ot == ROW_GROUP && outputRG.getMaxDataSizeWithStrings() > maxIdleBufferSize) + outRowGroupData.reset(); - if (fe1 && fe1Input.getMaxDataSizeWithStrings() > maxIdleBufferSize) - fe1Data.reset(); + if (fe1 && fe1Input.getMaxDataSizeWithStrings() > maxIdleBufferSize) + fe1Data.reset(); - if (fe2 && fe2Output.getMaxDataSizeWithStrings() > maxIdleBufferSize) - fe2Data.reset(); + if (fe2 && fe2Output.getMaxDataSizeWithStrings() > maxIdleBufferSize) + fe2Data.reset(); - if (getTupleJoinRowGroupData && joinedRG.getMaxDataSizeWithStrings() > maxIdleBufferSize) - joinedRGMem.reset(); + if (getTupleJoinRowGroupData && joinedRG.getMaxDataSizeWithStrings() > maxIdleBufferSize) + joinedRGMem.reset(); } void BatchPrimitiveProcessor::nextLBID() { - uint32_t i; + uint32_t i; - for (i = 0; i < filterCount; i++) - filterSteps[i]->nextLBID(); + for (i = 0; i < filterCount; i++) + filterSteps[i]->nextLBID(); - for (i = 0; i < projectCount; i++) - projectSteps[i]->nextLBID(); + for (i = 0; i < projectCount; i++) + projectSteps[i]->nextLBID(); } SBPP BatchPrimitiveProcessor::duplicate() { - SBPP bpp; - uint32_t i; + SBPP bpp; + uint32_t i; -// cout << "duplicating a bpp\n"; + // cout << "duplicating a bpp\n"; - bpp.reset(new BatchPrimitiveProcessor()); - bpp->ot = ot; - bpp->versionInfo = versionInfo; - bpp->txnID = txnID; - bpp->sessionID = sessionID; - bpp->stepID = stepID; - bpp->uniqueID = uniqueID; - bpp->needStrValues = needStrValues; - bpp->wideColumnsWidths = wideColumnsWidths; - bpp->gotAbsRids = gotAbsRids; - bpp->gotValues = gotValues; - bpp->LBIDTrace = LBIDTrace; - bpp->hasScan = hasScan; - bpp->hasFilterStep = hasFilterStep; - bpp->filtOnString = filtOnString; - bpp->hasRowGroup = hasRowGroup; - bpp->getTupleJoinRowGroupData = getTupleJoinRowGroupData; - bpp->bop = bop; - bpp->hasPassThru = hasPassThru; - bpp->forHJ = forHJ; - bpp->processorThreads = processorThreads; // is a power-of-2 at this point - bpp->ptMask = processorThreads - 1; + bpp.reset(new BatchPrimitiveProcessor()); + bpp->ot = ot; + bpp->versionInfo = versionInfo; + bpp->txnID = txnID; + bpp->sessionID = sessionID; + bpp->stepID = stepID; + bpp->uniqueID = uniqueID; + bpp->needStrValues = needStrValues; + bpp->wideColumnsWidths = wideColumnsWidths; + bpp->gotAbsRids = gotAbsRids; + bpp->gotValues = gotValues; + bpp->LBIDTrace = LBIDTrace; + bpp->hasScan = hasScan; + bpp->hasFilterStep = hasFilterStep; + bpp->filtOnString = filtOnString; + bpp->hasRowGroup = hasRowGroup; + bpp->getTupleJoinRowGroupData = getTupleJoinRowGroupData; + bpp->bop = bop; + bpp->hasPassThru = hasPassThru; + bpp->forHJ = forHJ; + bpp->processorThreads = processorThreads; // is a power-of-2 at this point + bpp->ptMask = processorThreads - 1; - if (ot == ROW_GROUP) + if (ot == ROW_GROUP) + { + bpp->outputRG = outputRG; + + if (fe1) { - bpp->outputRG = outputRG; - - if (fe1) - { - bpp->fe1.reset(new FuncExpWrapper(*fe1)); - bpp->fe1Input = fe1Input; - } - - if (fe2) - { - bpp->fe2.reset(new FuncExpWrapper(*fe2)); - bpp->fe2Output = fe2Output; - } + bpp->fe1.reset(new FuncExpWrapper(*fe1)); + bpp->fe1Input = fe1Input; } - bpp->doJoin = doJoin; - - if (doJoin) + if (fe2) { - pthread_mutex_lock(&bpp->objLock); - /* There are add'l join vars, but only these are necessary for processing - a join */ - bpp->tJoinerSizes = tJoinerSizes; - bpp->joinerCount = joinerCount; - bpp->joinTypes = joinTypes; - bpp->largeSideKeyColumns = largeSideKeyColumns; - bpp->tJoiners = tJoiners; - //bpp->_pools = _pools; - bpp->typelessJoin = typelessJoin; - bpp->tlLargeSideKeyColumns = tlLargeSideKeyColumns; - bpp->tlSmallSideKeyColumns = tlSmallSideKeyColumns; - bpp->tlJoiners = tlJoiners; - bpp->tlSmallSideKeyLengths = tlSmallSideKeyLengths; - bpp->storedKeyAllocators = storedKeyAllocators; - bpp->joinNullValues = joinNullValues; - bpp->doMatchNulls = doMatchNulls; - bpp->hasJoinFEFilters = hasJoinFEFilters; - bpp->hasSmallOuterJoin = hasSmallOuterJoin; - bpp->mJOINHasSkewedKeyColumn = mJOINHasSkewedKeyColumn; - bpp->mSmallSideRGPtr = mSmallSideRGPtr; - bpp->mSmallSideKeyColumnsPtr = mSmallSideKeyColumnsPtr; - if (!getTupleJoinRowGroupData && mJOINHasSkewedKeyColumn) - { - idbassert(!smallSideRGs.empty()); - bpp->smallSideRGs.push_back(smallSideRGs[0]); - } + bpp->fe2.reset(new FuncExpWrapper(*fe2)); + bpp->fe2Output = fe2Output; + } + } - if (hasJoinFEFilters) - { - bpp->joinFERG = joinFERG; - bpp->joinFEFilters.reset(new scoped_ptr[joinerCount]); + bpp->doJoin = doJoin; - for (i = 0; i < joinerCount; i++) - if (joinFEFilters[i]) - bpp->joinFEFilters[i].reset(new FuncExpWrapper(*joinFEFilters[i])); - } + if (doJoin) + { + pthread_mutex_lock(&bpp->objLock); + /* There are add'l join vars, but only these are necessary for processing + a join */ + bpp->tJoinerSizes = tJoinerSizes; + bpp->joinerCount = joinerCount; + bpp->joinTypes = joinTypes; + bpp->largeSideKeyColumns = largeSideKeyColumns; + bpp->tJoiners = tJoiners; + // bpp->_pools = _pools; + bpp->typelessJoin = typelessJoin; + bpp->tlLargeSideKeyColumns = tlLargeSideKeyColumns; + bpp->tlSmallSideKeyColumns = tlSmallSideKeyColumns; + bpp->tlJoiners = tlJoiners; + bpp->tlSmallSideKeyLengths = tlSmallSideKeyLengths; + bpp->storedKeyAllocators = storedKeyAllocators; + bpp->joinNullValues = joinNullValues; + bpp->doMatchNulls = doMatchNulls; + bpp->hasJoinFEFilters = hasJoinFEFilters; + bpp->hasSmallOuterJoin = hasSmallOuterJoin; + bpp->mJOINHasSkewedKeyColumn = mJOINHasSkewedKeyColumn; + bpp->mSmallSideRGPtr = mSmallSideRGPtr; + bpp->mSmallSideKeyColumnsPtr = mSmallSideKeyColumnsPtr; + if (!getTupleJoinRowGroupData && mJOINHasSkewedKeyColumn) + { + idbassert(!smallSideRGs.empty()); + bpp->smallSideRGs.push_back(smallSideRGs[0]); + } - if (getTupleJoinRowGroupData) - { - bpp->smallSideRGs = smallSideRGs; - bpp->largeSideRG = largeSideRG; - bpp->smallSideRowLengths = smallSideRowLengths; - bpp->smallSideRowData = smallSideRowData; - bpp->smallNullRowData = smallNullRowData; - bpp->smallNullPointers = smallNullPointers; - bpp->joinedRG = joinedRG; - } + if (hasJoinFEFilters) + { + bpp->joinFERG = joinFERG; + bpp->joinFEFilters.reset(new scoped_ptr[joinerCount]); + + for (i = 0; i < joinerCount; i++) + if (joinFEFilters[i]) + bpp->joinFEFilters[i].reset(new FuncExpWrapper(*joinFEFilters[i])); + } + + if (getTupleJoinRowGroupData) + { + bpp->smallSideRGs = smallSideRGs; + bpp->largeSideRG = largeSideRG; + bpp->smallSideRowLengths = smallSideRowLengths; + bpp->smallSideRowData = smallSideRowData; + bpp->smallNullRowData = smallNullRowData; + bpp->smallNullPointers = smallNullPointers; + bpp->joinedRG = joinedRG; + } #ifdef __FreeBSD__ - pthread_mutex_unlock(&bpp->objLock); + pthread_mutex_unlock(&bpp->objLock); #endif - } + } - bpp->filterCount = filterCount; - bpp->filterSteps.resize(filterCount); + bpp->filterCount = filterCount; + bpp->filterSteps.resize(filterCount); - for (i = 0; i < filterCount; ++i) - bpp->filterSteps[i] = filterSteps[i]->duplicate(); + for (i = 0; i < filterCount; ++i) + bpp->filterSteps[i] = filterSteps[i]->duplicate(); - bpp->projectCount = projectCount; - bpp->projectSteps.resize(projectCount); + bpp->projectCount = projectCount; + bpp->projectSteps.resize(projectCount); - for (i = 0; i < projectCount; ++i) - bpp->projectSteps[i] = projectSteps[i]->duplicate(); + for (i = 0; i < projectCount; ++i) + bpp->projectSteps[i] = projectSteps[i]->duplicate(); - if (fAggregator.get() != NULL) - { - bpp->fAggregateRG = fAggregateRG; - bpp->fAggregator.reset(new RowAggregation( - fAggregator->getGroupByCols(), fAggregator->getAggFunctions())); - bpp->fAggregator->timeZone(fAggregator->timeZone()); - } + if (fAggregator.get() != NULL) + { + bpp->fAggregateRG = fAggregateRG; + bpp->fAggregator.reset(new RowAggregation(fAggregator->getGroupByCols(), fAggregator->getAggFunctions())); + bpp->fAggregator->timeZone(fAggregator->timeZone()); + } - bpp->sendRidsAtDelivery = sendRidsAtDelivery; - bpp->prefetchThreshold = prefetchThreshold; + bpp->sendRidsAtDelivery = sendRidsAtDelivery; + bpp->prefetchThreshold = prefetchThreshold; - bpp->sock = sock; - bpp->writelock = writelock; - bpp->hasDictStep = hasDictStep; - bpp->sendThread = sendThread; - bpp->newConnection = true; - bpp->initProcessor(); - return bpp; + bpp->sock = sock; + bpp->writelock = writelock; + bpp->hasDictStep = hasDictStep; + bpp->sendThread = sendThread; + bpp->newConnection = true; + bpp->initProcessor(); + return bpp; } #if 0 @@ -2612,236 +2587,223 @@ bool BatchPrimitiveProcessor::operator==(const BatchPrimitiveProcessor& bpp) con } #endif - void BatchPrimitiveProcessor::asyncLoadProjectColumns() { - // relLBID is the LBID related to the primMsg->LBID, - // it is used to keep the read ahead boundary for asyncLoads - // 1. scan driven case: load blocks in # to (# + blocksReadAhead - 1) range, - // where # is a multiple of ColScanReadAheadBlocks in Columnstore.xml - // 2. non-scan driven case: load blocks in the logical block. - // because 1 logical block per primMsg, asyncLoad only once per message. - for (uint64_t i = 0; i < projectCount; ++i) + // relLBID is the LBID related to the primMsg->LBID, + // it is used to keep the read ahead boundary for asyncLoads + // 1. scan driven case: load blocks in # to (# + blocksReadAhead - 1) range, + // where # is a multiple of ColScanReadAheadBlocks in Columnstore.xml + // 2. non-scan driven case: load blocks in the logical block. + // because 1 logical block per primMsg, asyncLoad only once per message. + for (uint64_t i = 0; i < projectCount; ++i) + { + // only care about column commands + ColumnCommand* col = dynamic_cast(projectSteps[i].get()); + + if (col != NULL) { - // only care about column commands - ColumnCommand* col = dynamic_cast(projectSteps[i].get()); + asyncLoaded[i] = asyncLoaded[i] && (relLBID[i] % blocksReadAhead != 0); + relLBID[i] += col->getWidth(); - if (col != NULL) - { - asyncLoaded[i] = asyncLoaded[i] && (relLBID[i] % blocksReadAhead != 0); - relLBID[i] += col->getWidth(); - - if (!asyncLoaded[i] && col->willPrefetch()) - { - loadBlockAsync(col->getLBID(), - versionInfo, - txnID, - col->getCompType(), - &cachedIO, - &physIO, - LBIDTrace, - sessionID, - &counterLock, - &busyLoaderCount, - sendThread, - &vssCache); - asyncLoaded[i] = true; - } - } + if (!asyncLoaded[i] && col->willPrefetch()) + { + loadBlockAsync(col->getLBID(), versionInfo, txnID, col->getCompType(), &cachedIO, &physIO, LBIDTrace, + sessionID, &counterLock, &busyLoaderCount, sendThread, &vssCache); + asyncLoaded[i] = true; + } } + } } bool BatchPrimitiveProcessor::generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth) { - Row& smallRow = smallRows[depth]; - const bool lowestLvl = (depth == joinerCount - 1); + Row& smallRow = smallRows[depth]; + const bool lowestLvl = (depth == joinerCount - 1); - while (gjrgRowNumber < ridCount && - gjrgPlaceHolders[depth] < tSmallSideMatches[depth][gjrgRowNumber].size() && - !gjrgFull) + while (gjrgRowNumber < ridCount && + gjrgPlaceHolders[depth] < tSmallSideMatches[depth][gjrgRowNumber].size() && !gjrgFull) + { + const vector& results = tSmallSideMatches[depth][gjrgRowNumber]; + const uint32_t size = results.size(); + + if (depth == 0) { - const vector& results = tSmallSideMatches[depth][gjrgRowNumber]; - const uint32_t size = results.size(); - - if (depth == 0) - { - outputRG.getRow(gjrgRowNumber, &largeRow); - applyMapping(gjrgMappings[joinerCount], largeRow, &baseRow); - baseRow.setRid(largeRow.getRelRid()); - } - - //cout << "rowNum = " << gjrgRowNumber << " at depth " << depth << " size is " << size << endl; - for (uint32_t& i = gjrgPlaceHolders[depth]; i < size && !gjrgFull; i++) - { - if (results[i] != (uint32_t) - 1) - { - smallSideRGs[depth].getRow(results[i], &smallRow); - //rowOffset = ((uint64_t) results[i]) * smallRowSize; - //smallRow.setData(&rowDataAtThisLvl.rowData[rowOffset] + emptySize); - } - else - smallRow.setPointer(smallNullPointers[depth]); - - //cout << "small row: " << smallRow.toString() << endl; - applyMapping(gjrgMappings[depth], smallRow, &baseRow); - - if (!lowestLvl) - generateJoinedRowGroup(baseRow, depth + 1); - else - { - copyRow(baseRow, &joinedRow); - //memcpy(joinedRow.getData(), baseRow.getData(), joinedRow.getSize()); - //cerr << "joined row " << joinedRG.getRowCount() << ": " << joinedRow.toString() << endl; - joinedRow.nextRow(); - joinedRG.incRowCount(); - - if (joinedRG.getRowCount() == 8192) - { - i++; - gjrgFull = true; - } - } - - if (gjrgFull) - break; - } - - if (depth == 0 && gjrgPlaceHolders[0] == tSmallSideMatches[0][gjrgRowNumber].size()) - { - gjrgPlaceHolders[0] = 0; - gjrgRowNumber++; - } + outputRG.getRow(gjrgRowNumber, &largeRow); + applyMapping(gjrgMappings[joinerCount], largeRow, &baseRow); + baseRow.setRid(largeRow.getRelRid()); } -// if (depth == 0) -// cout << "gjrg returning " << (uint32_t) gjrgFull << endl; - if (!gjrgFull) - gjrgPlaceHolders[depth] = 0; + // cout << "rowNum = " << gjrgRowNumber << " at depth " << depth << " size is " << size << endl; + for (uint32_t& i = gjrgPlaceHolders[depth]; i < size && !gjrgFull; i++) + { + if (results[i] != (uint32_t)-1) + { + smallSideRGs[depth].getRow(results[i], &smallRow); + // rowOffset = ((uint64_t) results[i]) * smallRowSize; + // smallRow.setData(&rowDataAtThisLvl.rowData[rowOffset] + emptySize); + } + else + smallRow.setPointer(smallNullPointers[depth]); - return gjrgFull; + // cout << "small row: " << smallRow.toString() << endl; + applyMapping(gjrgMappings[depth], smallRow, &baseRow); + + if (!lowestLvl) + generateJoinedRowGroup(baseRow, depth + 1); + else + { + copyRow(baseRow, &joinedRow); + // memcpy(joinedRow.getData(), baseRow.getData(), joinedRow.getSize()); + // cerr << "joined row " << joinedRG.getRowCount() << ": " << joinedRow.toString() << endl; + joinedRow.nextRow(); + joinedRG.incRowCount(); + + if (joinedRG.getRowCount() == 8192) + { + i++; + gjrgFull = true; + } + } + + if (gjrgFull) + break; + } + + if (depth == 0 && gjrgPlaceHolders[0] == tSmallSideMatches[0][gjrgRowNumber].size()) + { + gjrgPlaceHolders[0] = 0; + gjrgRowNumber++; + } + } + + // if (depth == 0) + // cout << "gjrg returning " << (uint32_t) gjrgFull << endl; + if (!gjrgFull) + gjrgPlaceHolders[depth] = 0; + + return gjrgFull; } void BatchPrimitiveProcessor::resetGJRG() { - gjrgFull = false; - joinedRG.resetRowGroup(baseRid); - joinedRG.getRow(0, &joinedRow); - joinedRG.setDBRoot(dbRoot); + gjrgFull = false; + joinedRG.resetRowGroup(baseRid); + joinedRG.getRow(0, &joinedRow); + joinedRG.setDBRoot(dbRoot); } void BatchPrimitiveProcessor::initGJRG() { - for (uint32_t z = 0; z < joinerCount; z++) - gjrgPlaceHolders[z] = 0; + for (uint32_t z = 0; z < joinerCount; z++) + gjrgPlaceHolders[z] = 0; - gjrgRowNumber = 0; + gjrgRowNumber = 0; } inline void BatchPrimitiveProcessor::getJoinResults(const Row& r, uint32_t jIndex, vector& v) { - uint bucket; + uint bucket; - if (!typelessJoin[jIndex]) + if (!typelessJoin[jIndex]) + { + if (r.isNullValue(largeSideKeyColumns[jIndex])) { - if (r.isNullValue(largeSideKeyColumns[jIndex])) - { - /* Bug 3524. This matches everything. */ - if (joinTypes[jIndex] & ANTI) - { - TJoiner::iterator it; + /* Bug 3524. This matches everything. */ + if (joinTypes[jIndex] & ANTI) + { + TJoiner::iterator it; - for (uint i = 0; i < processorThreads; ++i) - for (it = tJoiners[jIndex][i]->begin(); it != tJoiners[jIndex][i]->end(); ++it) - v.push_back(it->second); + for (uint i = 0; i < processorThreads; ++i) + for (it = tJoiners[jIndex][i]->begin(); it != tJoiners[jIndex][i]->end(); ++it) + v.push_back(it->second); - return; - } - else - return; - } + return; + } + else + return; + } - uint64_t largeKey; - uint32_t colIndex = largeSideKeyColumns[jIndex]; + uint64_t largeKey; + uint32_t colIndex = largeSideKeyColumns[jIndex]; - if (r.isUnsigned(colIndex)) - { - largeKey = r.getUintField(colIndex); - } - else - { - largeKey = r.getIntField(colIndex); - } - - bucket = bucketPicker((char *) &largeKey, 8, bpSeed) & ptMask; - pair range = tJoiners[jIndex][bucket]->equal_range(largeKey); - for (; range.first != range.second; ++range.first) - v.push_back(range.first->second); - - if (doMatchNulls[jIndex]) // add the nulls to the match list - { - bucket = bucketPicker((char *) &joinNullValues[jIndex], 8, bpSeed) & ptMask; - range = tJoiners[jIndex][bucket]->equal_range(joinNullValues[jIndex]); - for (; range.first != range.second; ++range.first) - v.push_back(range.first->second); - } + if (r.isUnsigned(colIndex)) + { + largeKey = r.getUintField(colIndex); } else { - /* Bug 3524. Large-side NULL + ANTI join matches everything. */ - if (joinTypes[jIndex] & ANTI) + largeKey = r.getIntField(colIndex); + } + + bucket = bucketPicker((char*)&largeKey, 8, bpSeed) & ptMask; + pair range = tJoiners[jIndex][bucket]->equal_range(largeKey); + for (; range.first != range.second; ++range.first) + v.push_back(range.first->second); + + if (doMatchNulls[jIndex]) // add the nulls to the match list + { + bucket = bucketPicker((char*)&joinNullValues[jIndex], 8, bpSeed) & ptMask; + range = tJoiners[jIndex][bucket]->equal_range(joinNullValues[jIndex]); + for (; range.first != range.second; ++range.first) + v.push_back(range.first->second); + } + } + else + { + /* Bug 3524. Large-side NULL + ANTI join matches everything. */ + if (joinTypes[jIndex] & ANTI) + { + bool hasNullValue = false; + + for (uint32_t i = 0; i < tlLargeSideKeyColumns[jIndex].size(); i++) + if (r.isNullValue(tlLargeSideKeyColumns[jIndex][i])) { - bool hasNullValue = false; - - for (uint32_t i = 0; i < tlLargeSideKeyColumns[jIndex].size(); i++) - if (r.isNullValue(tlLargeSideKeyColumns[jIndex][i])) - { - hasNullValue = true; - break; - } - - if (hasNullValue) - { - TLJoiner::iterator it; - for (uint i = 0; i < processorThreads; ++i) - for (it = tlJoiners[jIndex][i]->begin(); it != tlJoiners[jIndex][i]->end(); ++it) - v.push_back(it->second); - - return; - } + hasNullValue = true; + break; } - TypelessData largeKey(&r); - bucket = r.hashTypeless(tlLargeSideKeyColumns[jIndex], - mSmallSideKeyColumnsPtr, - mSmallSideRGPtr ? &mSmallSideRGPtr->getColWidths() : nullptr) & ptMask; - pair range = - tlJoiners[jIndex][bucket]->equal_range(largeKey); - for (; range.first != range.second; ++range.first) - v.push_back(range.first->second); + if (hasNullValue) + { + TLJoiner::iterator it; + for (uint i = 0; i < processorThreads; ++i) + for (it = tlJoiners[jIndex][i]->begin(); it != tlJoiners[jIndex][i]->end(); ++it) + v.push_back(it->second); + + return; + } } + + TypelessData largeKey(&r); + bucket = r.hashTypeless(tlLargeSideKeyColumns[jIndex], mSmallSideKeyColumnsPtr, + mSmallSideRGPtr ? &mSmallSideRGPtr->getColWidths() : nullptr) & + ptMask; + pair range = tlJoiners[jIndex][bucket]->equal_range(largeKey); + for (; range.first != range.second; ++range.first) + v.push_back(range.first->second); + } } void BatchPrimitiveProcessor::buildVSSCache(uint32_t loopCount) { - vector lbidList; - vector vssData; - uint32_t i; - int rc; + vector lbidList; + vector vssData; + uint32_t i; + int rc; - for (i = 0; i < filterCount; i++) - filterSteps[i]->getLBIDList(loopCount, &lbidList); + for (i = 0; i < filterCount; i++) + filterSteps[i]->getLBIDList(loopCount, &lbidList); - for (i = 0; i < projectCount; i++) - projectSteps[i]->getLBIDList(loopCount, &lbidList); + for (i = 0; i < projectCount; i++) + projectSteps[i]->getLBIDList(loopCount, &lbidList); - rc = brm->bulkVSSLookup(lbidList, versionInfo, (int) txnID, &vssData); + rc = brm->bulkVSSLookup(lbidList, versionInfo, (int)txnID, &vssData); - if (rc == 0) - for (i = 0; i < vssData.size(); i++) - vssCache.insert(make_pair(lbidList[i], vssData[i])); + if (rc == 0) + for (i = 0; i < vssData.size(); i++) + vssCache.insert(make_pair(lbidList[i], vssData[i])); -// cout << "buildVSSCache inserted " << vssCache.size() << " elements" << endl; + // cout << "buildVSSCache inserted " << vssCache.size() << " elements" << endl; } -} +} // namespace primitiveprocessor // vim:ts=4 sw=4: diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index 97673efd8..bcec5a3ce 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -68,378 +68,385 @@ typedef std::tr1::unordered_map VSSCache; namespace primitiveprocessor { - typedef boost::shared_ptr SBPP; class scalar_exception : public std::exception { - const char* what() const throw() - { - return "Not a scalar subquery."; - } + const char* what() const throw() + { + return "Not a scalar subquery."; + } }; class NeedToRestartJob : public std::runtime_error { -public: - NeedToRestartJob() : std::runtime_error("NeedToRestartJob") { } - NeedToRestartJob(const std::string& s) : - std::runtime_error(s) { } + public: + NeedToRestartJob() : std::runtime_error("NeedToRestartJob") + { + } + NeedToRestartJob(const std::string& s) : std::runtime_error(s) + { + } }; class BatchPrimitiveProcessor { -public: - BatchPrimitiveProcessor(messageqcpp::ByteStream&, double prefetchThresh, - boost::shared_ptr, uint processorThreads); + public: + BatchPrimitiveProcessor(messageqcpp::ByteStream&, double prefetchThresh, boost::shared_ptr, + uint processorThreads); - ~BatchPrimitiveProcessor(); + ~BatchPrimitiveProcessor(); - /* Interface used by primproc */ - void initBPP(messageqcpp::ByteStream&); - void resetBPP(messageqcpp::ByteStream&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& outputSock); - void addToJoiner(messageqcpp::ByteStream&); - int endOfJoiner(); - void doneSendingJoinerData(); - int operator()(); - void setLBIDForScan(uint64_t rid); + /* Interface used by primproc */ + void initBPP(messageqcpp::ByteStream&); + void resetBPP(messageqcpp::ByteStream&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& outputSock); + void addToJoiner(messageqcpp::ByteStream&); + int endOfJoiner(); + void doneSendingJoinerData(); + int operator()(); + void setLBIDForScan(uint64_t rid); - /* Duplicate() returns a deep copy of this object as it was init'd by initBPP. - It's thread-safe wrt resetBPP. */ - SBPP duplicate(); + /* Duplicate() returns a deep copy of this object as it was init'd by initBPP. + It's thread-safe wrt resetBPP. */ + SBPP duplicate(); - /* These need to be updated */ - //bool operator==(const BatchPrimitiveProcessor&) const; - //inline bool operator!=(const BatchPrimitiveProcessor& bpp) const - //{ - // return !(*this == bpp); - //} + /* These need to be updated */ + // bool operator==(const BatchPrimitiveProcessor&) const; + // inline bool operator!=(const BatchPrimitiveProcessor& bpp) const + //{ + // return !(*this == bpp); + //} - inline uint32_t getSessionID() - { - return sessionID; - } - inline uint32_t getStepID() - { - return stepID; - } - inline uint32_t getUniqueID() - { - return uniqueID; - } - inline bool busy() - { - return fBusy; - } - inline void busy(bool b) - { - fBusy = b; - } + inline uint32_t getSessionID() + { + return sessionID; + } + inline uint32_t getStepID() + { + return stepID; + } + inline uint32_t getUniqueID() + { + return uniqueID; + } + inline bool busy() + { + return fBusy; + } + inline void busy(bool b) + { + fBusy = b; + } - uint16_t FilterCount() const - { - return filterCount; - } - uint16_t ProjectCount() const - { - return projectCount; - } - uint32_t PhysIOCount() const - { - return physIO; - } - uint32_t CachedIOCount() const - { - return cachedIO; - } - uint32_t BlocksTouchedCount() const - { - return touchedBlocks; - } + uint16_t FilterCount() const + { + return filterCount; + } + uint16_t ProjectCount() const + { + return projectCount; + } + uint32_t PhysIOCount() const + { + return physIO; + } + uint32_t CachedIOCount() const + { + return cachedIO; + } + uint32_t BlocksTouchedCount() const + { + return touchedBlocks; + } - void setError(const std::string& error, logging::ErrorCodeValues errorCode) {} + void setError(const std::string& error, logging::ErrorCodeValues errorCode) + { + } - // these two functions are used by BPPV to create BPP instances - // on demand. TRY not to use unlock() for anything else. - void unlock() - { - pthread_mutex_unlock(&objLock); - } - bool hasJoin() - { - return doJoin; - } - primitives::PrimitiveProcessor& getPrimitiveProcessor() - { - return pp; - } - uint32_t getOutMsgSize() const - { - return outMsgSize; - } -private: - BatchPrimitiveProcessor(); - BatchPrimitiveProcessor(const BatchPrimitiveProcessor&); - BatchPrimitiveProcessor& operator=(const BatchPrimitiveProcessor&); + // these two functions are used by BPPV to create BPP instances + // on demand. TRY not to use unlock() for anything else. + void unlock() + { + pthread_mutex_unlock(&objLock); + } + bool hasJoin() + { + return doJoin; + } + primitives::PrimitiveProcessor& getPrimitiveProcessor() + { + return pp; + } + uint32_t getOutMsgSize() const + { + return outMsgSize; + } - void initProcessor(); + private: + BatchPrimitiveProcessor(); + BatchPrimitiveProcessor(const BatchPrimitiveProcessor&); + BatchPrimitiveProcessor& operator=(const BatchPrimitiveProcessor&); + + void initProcessor(); #ifdef PRIMPROC_STOPWATCH - void execute(logging::StopWatch* stopwatch); + void execute(logging::StopWatch* stopwatch); #else - void execute(); + void execute(); #endif - void writeProjectionPreamble(); - void makeResponse(); - void sendResponse(); - /* Used by scan operations to increment the LBIDs in successive steps */ - void nextLBID(); + void writeProjectionPreamble(); + void makeResponse(); + void sendResponse(); + /* Used by scan operations to increment the LBIDs in successive steps */ + void nextLBID(); - /* these send relative rids, should this be abs rids? */ - void serializeElementTypes(); - void serializeStrings(); + /* these send relative rids, should this be abs rids? */ + void serializeElementTypes(); + void serializeStrings(); - void asyncLoadProjectColumns(); - void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true); + void asyncLoadProjectColumns(); + void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true); - BPSOutputType ot; + BPSOutputType ot; - BRM::QueryContext versionInfo; - uint32_t txnID; - uint32_t sessionID; - uint32_t stepID; - uint32_t uniqueID; + BRM::QueryContext versionInfo; + uint32_t txnID; + uint32_t sessionID; + uint32_t stepID; + uint32_t uniqueID; - // # of times to loop over the command arrays - // ... This is 1, except when the first command is a scan, in which case - // this single BPP object produces count responses. - uint16_t count; - uint64_t baseRid; // first rid of the logical block + // # of times to loop over the command arrays + // ... This is 1, except when the first command is a scan, in which case + // this single BPP object produces count responses. + uint16_t count; + uint64_t baseRid; // first rid of the logical block - uint16_t relRids[LOGICAL_BLOCK_RIDS]; - int64_t values[LOGICAL_BLOCK_RIDS]; - int128_t wide128Values[LOGICAL_BLOCK_RIDS]; - boost::scoped_array absRids; - boost::scoped_array strValues; - uint16_t origRidCount; - uint16_t ridCount; - bool needStrValues; - uint16_t wideColumnsWidths; + uint16_t relRids[LOGICAL_BLOCK_RIDS]; + int64_t values[LOGICAL_BLOCK_RIDS]; + int128_t wide128Values[LOGICAL_BLOCK_RIDS]; + boost::scoped_array absRids; + boost::scoped_array strValues; + uint16_t origRidCount; + uint16_t ridCount; + bool needStrValues; + uint16_t wideColumnsWidths; - /* Common space for primitive data */ - alignas(utils::MAXCOLUMNWIDTH) uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; - boost::scoped_array outputMsg; - uint32_t outMsgSize; + /* Common space for primitive data */ + alignas(utils::MAXCOLUMNWIDTH) uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; + boost::scoped_array outputMsg; + uint32_t outMsgSize; - std::vector filterSteps; - std::vector projectSteps; - //@bug 1136 - uint16_t filterCount; - uint16_t projectCount; - bool sendRidsAtDelivery; - uint16_t ridMap; - bool gotAbsRids; - bool gotValues; + std::vector filterSteps; + std::vector projectSteps; + //@bug 1136 + uint16_t filterCount; + uint16_t projectCount; + bool sendRidsAtDelivery; + uint16_t ridMap; + bool gotAbsRids; + bool gotValues; - bool hasScan; - bool validCPData; - // CP data from a scanned column - union - { - int128_t min128Val; - int64_t minVal; - }; - union - { - int128_t max128Val; - int64_t maxVal; - }; + bool hasScan; + bool validCPData; + // CP data from a scanned column + union + { + int128_t min128Val; + int64_t minVal; + }; + union + { + int128_t max128Val; + int64_t maxVal; + }; - uint64_t lbidForCP; - bool hasWideColumnOut; - uint8_t wideColumnWidthOut; - // IO counters - boost::mutex counterLock; - uint32_t busyLoaderCount; + uint64_t lbidForCP; + bool hasWideColumnOut; + uint8_t wideColumnWidthOut; + // IO counters + boost::mutex counterLock; + uint32_t busyLoaderCount; - uint32_t physIO, cachedIO, touchedBlocks; + uint32_t physIO, cachedIO, touchedBlocks; - SP_UM_IOSOCK sock; - messageqcpp::SBS serialized; - SP_UM_MUTEX writelock; + SP_UM_IOSOCK sock; + messageqcpp::SBS serialized; + SP_UM_MUTEX writelock; - // MCOL-744 using pthread mutex instead of Boost mutex because - // in it is possible that this lock could be unlocked when it is - // already unlocked. In Ubuntu 16.04's Boost this triggers a - // crash. Whilst it is very hard to hit this it is still bad. - // Longer term TODO: fix/remove objLock and/or refactor BPP - pthread_mutex_t objLock; - bool LBIDTrace; - bool fBusy; + // MCOL-744 using pthread mutex instead of Boost mutex because + // in it is possible that this lock could be unlocked when it is + // already unlocked. In Ubuntu 16.04's Boost this triggers a + // crash. Whilst it is very hard to hit this it is still bad. + // Longer term TODO: fix/remove objLock and/or refactor BPP + pthread_mutex_t objLock; + bool LBIDTrace; + bool fBusy; - /* Join support TODO: Make join ops a seperate Command class. */ - bool doJoin; - boost::scoped_array > addToJoinerLocks; - boost::scoped_array smallSideDataLocks; + /* Join support TODO: Make join ops a seperate Command class. */ + bool doJoin; + boost::scoped_array> addToJoinerLocks; + boost::scoped_array smallSideDataLocks; -// uint32_t ridsIn, ridsOut; + // uint32_t ridsIn, ridsOut; - //@bug 1051 FilterStep on PM - bool hasFilterStep; - bool filtOnString; - boost::scoped_array fFiltCmdRids[2]; - boost::scoped_array fFiltCmdValues[2]; - boost::scoped_array fFiltCmdBinaryValues[2]; - boost::scoped_array fFiltStrValues[2]; - uint64_t fFiltRidCount[2]; + //@bug 1051 FilterStep on PM + bool hasFilterStep; + bool filtOnString; + boost::scoped_array fFiltCmdRids[2]; + boost::scoped_array fFiltCmdValues[2]; + boost::scoped_array fFiltCmdBinaryValues[2]; + boost::scoped_array fFiltStrValues[2]; + uint64_t fFiltRidCount[2]; - // query density threshold for prefetch & async loading - double prefetchThreshold; + // query density threshold for prefetch & async loading + double prefetchThreshold; - /* RowGroup support */ - rowgroup::RowGroup outputRG; - boost::scoped_ptr outRowGroupData; - boost::shared_array rgMap; // maps input cols to output cols - boost::shared_array projectionMap; // maps the projection steps to the output RG - bool hasRowGroup; + /* RowGroup support */ + rowgroup::RowGroup outputRG; + boost::scoped_ptr outRowGroupData; + boost::shared_array rgMap; // maps input cols to output cols + boost::shared_array projectionMap; // maps the projection steps to the output RG + bool hasRowGroup; - /* Rowgroups + join */ - typedef std::tr1::unordered_multimap, - utils::STLPoolAllocator > > TJoiner; + /* Rowgroups + join */ + typedef std::tr1::unordered_multimap, + utils::STLPoolAllocator>> + TJoiner; - typedef std::tr1::unordered_multimap > > TLJoiner; + typedef std::tr1::unordered_multimap< + joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher, + joiner::TupleJoiner::TypelessDataComparator, + utils::STLPoolAllocator>> + TLJoiner; - bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0); - /* generateJoinedRowGroup helper fcns & vars */ - void initGJRG(); // called once after joining - void resetGJRG(); // called after every rowgroup returned by generateJRG - boost::scoped_array gjrgPlaceHolders; - uint32_t gjrgRowNumber; - bool gjrgFull; - rowgroup::Row largeRow, joinedRow, baseJRow; - boost::scoped_array baseJRowMem; - boost::scoped_ptr joinedRGMem; - boost::scoped_array smallRows; - boost::shared_array > gjrgMappings; + bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0); + /* generateJoinedRowGroup helper fcns & vars */ + void initGJRG(); // called once after joining + void resetGJRG(); // called after every rowgroup returned by generateJRG + boost::scoped_array gjrgPlaceHolders; + uint32_t gjrgRowNumber; + bool gjrgFull; + rowgroup::Row largeRow, joinedRow, baseJRow; + boost::scoped_array baseJRowMem; + boost::scoped_ptr joinedRGMem; + boost::scoped_array smallRows; + boost::shared_array> gjrgMappings; - boost::shared_array > > tJoiners; - typedef std::vector MatchedData[LOGICAL_BLOCK_RIDS]; - boost::shared_array tSmallSideMatches; - uint32_t executeTupleJoin(uint32_t startRid); - bool getTupleJoinRowGroupData; - std::vector smallSideRGs; - rowgroup::RowGroup largeSideRG; - boost::shared_array smallSideRowData; - boost::shared_array smallNullRowData; - boost::shared_array smallNullPointers; - boost::shared_array ssrdPos; // this keeps track of position when building smallSideRowData - boost::shared_array smallSideRowLengths; - boost::shared_array joinTypes; - uint32_t joinerCount; - boost::shared_array > tJoinerSizes; - // LSKC[i] = the column in outputRG joiner i uses as its key column - boost::shared_array largeSideKeyColumns; - // KCPP[i] = true means a joiner uses projection step i as a key column - boost::shared_array keyColumnProj; - rowgroup::Row oldRow, newRow; // used by executeTupleJoin() - boost::shared_array joinNullValues; - boost::shared_array doMatchNulls; - boost::scoped_array > joinFEFilters; - bool hasJoinFEFilters; - bool hasSmallOuterJoin; + boost::shared_array>> tJoiners; + typedef std::vector MatchedData[LOGICAL_BLOCK_RIDS]; + boost::shared_array tSmallSideMatches; + uint32_t executeTupleJoin(uint32_t startRid); + bool getTupleJoinRowGroupData; + std::vector smallSideRGs; + rowgroup::RowGroup largeSideRG; + boost::shared_array smallSideRowData; + boost::shared_array smallNullRowData; + boost::shared_array smallNullPointers; + boost::shared_array ssrdPos; // this keeps track of position when building smallSideRowData + boost::shared_array smallSideRowLengths; + boost::shared_array joinTypes; + uint32_t joinerCount; + boost::shared_array> tJoinerSizes; + // LSKC[i] = the column in outputRG joiner i uses as its key column + boost::shared_array largeSideKeyColumns; + // KCPP[i] = true means a joiner uses projection step i as a key column + boost::shared_array keyColumnProj; + rowgroup::Row oldRow, newRow; // used by executeTupleJoin() + boost::shared_array joinNullValues; + boost::shared_array doMatchNulls; + boost::scoped_array> joinFEFilters; + bool hasJoinFEFilters; + bool hasSmallOuterJoin; - /* extra typeless join vars & fcns*/ - boost::shared_array typelessJoin; - boost::shared_array > tlLargeSideKeyColumns; - std::shared_ptr> tlSmallSideKeyColumns; - boost::shared_array > > tlJoiners; - boost::shared_array tlSmallSideKeyLengths; - // True if smallSide and largeSide TypelessData key column differs,e.g BIGINT vs DECIMAL(38). - bool mJOINHasSkewedKeyColumn; - const rowgroup::RowGroup* mSmallSideRGPtr; - const std::vector* mSmallSideKeyColumnsPtr; + /* extra typeless join vars & fcns*/ + boost::shared_array typelessJoin; + boost::shared_array> tlLargeSideKeyColumns; + std::shared_ptr> tlSmallSideKeyColumns; + boost::shared_array>> tlJoiners; + boost::shared_array tlSmallSideKeyLengths; + // True if smallSide and largeSide TypelessData key column differs,e.g BIGINT vs DECIMAL(38). + bool mJOINHasSkewedKeyColumn; + const rowgroup::RowGroup* mSmallSideRGPtr; + const std::vector* mSmallSideKeyColumnsPtr; - inline void getJoinResults(const rowgroup::Row& r, uint32_t jIndex, std::vector& v); - // these allocators hold the memory for the keys stored in tlJoiners - boost::shared_array storedKeyAllocators; + inline void getJoinResults(const rowgroup::Row& r, uint32_t jIndex, std::vector& v); + // these allocators hold the memory for the keys stored in tlJoiners + boost::shared_array storedKeyAllocators; - /* PM Aggregation */ - rowgroup::RowGroup joinedRG; // if there's a join, the rows are formatted with this - rowgroup::SP_ROWAGG_PM_t fAggregator; - rowgroup::RowGroup fAggregateRG; - rowgroup::RGData fAggRowGroupData; - //boost::scoped_array fAggRowGroupData; + /* PM Aggregation */ + rowgroup::RowGroup joinedRG; // if there's a join, the rows are formatted with this + rowgroup::SP_ROWAGG_PM_t fAggregator; + rowgroup::RowGroup fAggregateRG; + rowgroup::RGData fAggRowGroupData; + // boost::scoped_array fAggRowGroupData; - /* OR hacks */ - uint8_t bop; // BOP_AND or BOP_OR - bool hasPassThru; - uint8_t forHJ; + /* OR hacks */ + uint8_t bop; // BOP_AND or BOP_OR + bool hasPassThru; + uint8_t forHJ; - boost::scoped_ptr fe1, fe2; - rowgroup::RowGroup fe1Input, fe2Output, *fe2Input; - // note, joinFERG is only for metadata, and is shared between BPPs - boost::shared_ptr joinFERG; - boost::scoped_array joinFERowData; - boost::scoped_ptr fe1Data, fe2Data; // can probably make these RGDatas not pointers to RGDatas - boost::shared_array projectForFE1; - boost::shared_array fe1ToProjection, fe2Mapping; // RG mappings - boost::scoped_array > joinFEMappings; - rowgroup::Row fe1In, fe1Out, fe2In, fe2Out, joinFERow; + boost::scoped_ptr fe1, fe2; + rowgroup::RowGroup fe1Input, fe2Output, *fe2Input; + // note, joinFERG is only for metadata, and is shared between BPPs + boost::shared_ptr joinFERG; + boost::scoped_array joinFERowData; + boost::scoped_ptr fe1Data, + fe2Data; // can probably make these RGDatas not pointers to RGDatas + boost::shared_array projectForFE1; + boost::shared_array fe1ToProjection, fe2Mapping; // RG mappings + boost::scoped_array> joinFEMappings; + rowgroup::Row fe1In, fe1Out, fe2In, fe2Out, joinFERow; - bool hasDictStep; + bool hasDictStep; - primitives::PrimitiveProcessor pp; + primitives::PrimitiveProcessor pp; - /* VSS cache members */ - VSSCache vssCache; - void buildVSSCache(uint32_t loopCount); + /* VSS cache members */ + VSSCache vssCache; + void buildVSSCache(uint32_t loopCount); - /* To support limited DEC queues on the PM */ - boost::shared_ptr sendThread; - bool newConnection; // to support the load balancing code in sendThread + /* To support limited DEC queues on the PM */ + boost::shared_ptr sendThread; + bool newConnection; // to support the load balancing code in sendThread - /* To support reentrancy */ - uint32_t currentBlockOffset; - boost::scoped_array relLBID; - boost::scoped_array asyncLoaded; + /* To support reentrancy */ + uint32_t currentBlockOffset; + boost::scoped_array relLBID; + boost::scoped_array asyncLoaded; - /* To support a smaller memory footprint when idle */ - static const uint64_t maxIdleBufferSize = 16 * 1024 * 1024; // arbitrary - void allocLargeBuffers(); - void freeLargeBuffers(); + /* To support a smaller memory footprint when idle */ + static const uint64_t maxIdleBufferSize = 16 * 1024 * 1024; // arbitrary + void allocLargeBuffers(); + void freeLargeBuffers(); - /* To ensure all packets of an LBID go out the same socket */ - int sockIndex; + /* To ensure all packets of an LBID go out the same socket */ + int sockIndex; - /* Shared nothing vars */ - uint32_t dbRoot; + /* Shared nothing vars */ + uint32_t dbRoot; - bool endOfJoinerRan; - /* Some addJoiner() profiling stuff */ - boost::posix_time::ptime firstCallTime; - utils::Hasher_r bucketPicker; - const uint32_t bpSeed = 0xf22df448; // an arbitrary random # - uint processorThreads; - uint ptMask; - bool firstInstance; - - static const uint64_t maxResultCount = 1048576; // 2^20 + bool endOfJoinerRan; + /* Some addJoiner() profiling stuff */ + boost::posix_time::ptime firstCallTime; + utils::Hasher_r bucketPicker; + const uint32_t bpSeed = 0xf22df448; // an arbitrary random # + uint processorThreads; + uint ptMask; + bool firstInstance; - friend class Command; - friend class ColumnCommand; - friend class DictStep; - friend class PassThruCommand; - friend class RTSCommand; - friend class FilterCommand; - friend class ScaledFilterCmd; - friend class StrFilterCmd; - friend class PseudoCC; + static const uint64_t maxResultCount = 1048576; // 2^20 + + friend class Command; + friend class ColumnCommand; + friend class DictStep; + friend class PassThruCommand; + friend class RTSCommand; + friend class FilterCommand; + friend class ScaledFilterCmd; + friend class StrFilterCmd; + friend class PseudoCC; }; -} +} // namespace primitiveprocessor #endif diff --git a/primitives/primproc/bpp.h b/primitives/primproc/bpp.h index a3a1c19b1..2190129bb 100644 --- a/primitives/primproc/bpp.h +++ b/primitives/primproc/bpp.h @@ -42,4 +42,3 @@ #include "passthrucommand.h" #include "rtscommand.h" #include "pseudocc.h" - diff --git a/primitives/primproc/bppseeder.cpp b/primitives/primproc/bppseeder.cpp index 41744dbba..b81628820 100644 --- a/primitives/primproc/bppseeder.cpp +++ b/primitives/primproc/bppseeder.cpp @@ -50,25 +50,22 @@ typedef int pthread_t; using namespace messageqcpp; using namespace std; - - namespace primitiveprocessor { - struct PTLogs { - PTLogs() {}; - PTLogs(const int t, const char* fname): thdId(t) - { - logFD.open(fname, ios_base::app | ios_base::ate); - } - ~PTLogs() - { - logFD.close(); - } + PTLogs(){}; + PTLogs(const int t, const char* fname) : thdId(t) + { + logFD.open(fname, ios_base::app | ios_base::ate); + } + ~PTLogs() + { + logFD.close(); + } - int thdId; - ofstream logFD; + int thdId; + ofstream logFD; }; typedef PTLogs PTLogs_t; @@ -77,49 +74,48 @@ typedef std::tr1::unordered_map PTLogsMap_t; PTLogsMap_t gFDList; SPPTLogs_t gLogFD; -boost::mutex gFDMutex; //pthread_mutex_t gFDMutex=PTHREAD_MUTEX_INITIALIZER; +boost::mutex gFDMutex; // pthread_mutex_t gFDMutex=PTHREAD_MUTEX_INITIALIZER; int gThdCnt = 0; extern dbbc::BlockRequestProcessor** BRPp; extern int fCacheCount; - -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } -BPPSeeder::BPPSeeder(const SBS& b, - const SP_UM_MUTEX& w, - const SP_UM_IOSOCK& s, - const int pmThreads, - const bool trace) : - bs(b), writelock(w), sock(s), fPMThreads(pmThreads), fTrace(trace), - failCount(0), - firstRun(true) +BPPSeeder::BPPSeeder(const SBS& b, const SP_UM_MUTEX& w, const SP_UM_IOSOCK& s, const int pmThreads, + const bool trace) + : bs(b), writelock(w), sock(s), fPMThreads(pmThreads), fTrace(trace), failCount(0), firstRun(true) { - uint8_t* buf = b->buf(); - uint32_t pos = sizeof(ISMPacketHeader); + uint8_t* buf = b->buf(); + uint32_t pos = sizeof(ISMPacketHeader); - sessionID = *((uint32_t*) &buf[pos]); - pos += 4; - stepID = *((uint32_t*) &buf[pos]); - pos += 4; - uniqueID = *((uint32_t*) &buf[pos]); - pos += 4; - _priority = *((uint32_t*) &buf[pos]); + sessionID = *((uint32_t*)&buf[pos]); + pos += 4; + stepID = *((uint32_t*)&buf[pos]); + pos += 4; + uniqueID = *((uint32_t*)&buf[pos]); + pos += 4; + _priority = *((uint32_t*)&buf[pos]); - dieTime = boost::posix_time::second_clock::universal_time() + - boost::posix_time::seconds(100); + dieTime = boost::posix_time::second_clock::universal_time() + boost::posix_time::seconds(100); } BPPSeeder::BPPSeeder(const BPPSeeder& b) - : bs(b.bs), writelock(b.writelock), sock(b.sock), - fPMThreads(b.fPMThreads), fTrace(b.fTrace), uniqueID(b.uniqueID), - sessionID(b.sessionID), stepID(b.stepID), failCount(b.failCount), bpp(b.bpp), - firstRun(b.firstRun), _priority(b._priority) + : bs(b.bs) + , writelock(b.writelock) + , sock(b.sock) + , fPMThreads(b.fPMThreads) + , fTrace(b.fTrace) + , uniqueID(b.uniqueID) + , sessionID(b.sessionID) + , stepID(b.stepID) + , failCount(b.failCount) + , bpp(b.bpp) + , firstRun(b.firstRun) + , _priority(b._priority) { } @@ -129,59 +125,58 @@ BPPSeeder::~BPPSeeder() int BPPSeeder::operator()() { - uint32_t pos; - const uint8_t* buf = bs->buf(); - BPPMap::iterator it; - ostringstream logData; - struct timespec tm; - struct timespec tm2; - double tm3 = 0; - bool ptLock = false; - bool gotBPP = false; - PTLogs_t* logFD = NULL; - int ret = 0; - pthread_t tid = 0; - boost::mutex::scoped_lock scoped(bppLock, boost::defer_lock_t()); + uint32_t pos; + const uint8_t* buf = bs->buf(); + BPPMap::iterator it; + ostringstream logData; + struct timespec tm; + struct timespec tm2; + double tm3 = 0; + bool ptLock = false; + bool gotBPP = false; + PTLogs_t* logFD = NULL; + int ret = 0; + pthread_t tid = 0; + boost::mutex::scoped_lock scoped(bppLock, boost::defer_lock_t()); - - try + try + { + if (firstRun) { - if (firstRun) + pos = sizeof(ISMPacketHeader) - 2; + uint16_t status = *((uint16_t*)&buf[pos]); + pos += 2; + + sessionID = *((uint32_t*)&buf[pos]); + pos += 4; + stepID = *((uint32_t*)&buf[pos]); + pos += 4; + uniqueID = *((uint32_t*)&buf[pos]); + pos += 4; + _priority = *((uint32_t*)&buf[pos]); + + if (0 < status) + { + sendErrorMsg(uniqueID, status, stepID); + return ret; + } + + // if (!(sessionID & 0x80000000)) + // cout << "got request for <" << sessionID <<", " << stepID << ">\n"; + scoped.lock(); + + if (!bppv) + { + it = bppMap.find(uniqueID); + + if (it == bppMap.end()) { - pos = sizeof(ISMPacketHeader) - 2; - uint16_t status = *((uint16_t*) &buf[pos]); - pos += 2; + /* mitigate a small race between creation and use */ + scoped.unlock(); - sessionID = *((uint32_t*) &buf[pos]); - pos += 4; - stepID = *((uint32_t*) &buf[pos]); - pos += 4; - uniqueID = *((uint32_t*) &buf[pos]); - pos += 4; - _priority = *((uint32_t*) &buf[pos]); - - if (0 < status) - { - sendErrorMsg(uniqueID, status, stepID); - return ret; - } - - //if (!(sessionID & 0x80000000)) - //cout << "got request for <" << sessionID <<", " << stepID << ">\n"; - scoped.lock(); - - if (!bppv) - { - it = bppMap.find(uniqueID); - - if (it == bppMap.end()) - { - /* mitigate a small race between creation and use */ - scoped.unlock(); - - if (boost::posix_time::second_clock::universal_time() > dieTime) - { -#if 0 // for debugging + if (boost::posix_time::second_clock::universal_time() > dieTime) + { +#if 0 // for debugging #ifndef _MSC_VER boost::posix_time::ptime pt = boost::posix_time::microsec_clock::local_time(); @@ -195,228 +190,218 @@ int BPPSeeder::operator()() #endif throw logic_error("BPPSeeder couldn't find the sessionID/stepID pair"); #endif - return 0; - } + return 0; + } -// if (!isSysCat()) - return -1; -// else { // syscat queries aren't run by a threadpool, can't reschedule those jobs -// usleep(1000); -// goto retry; -// } - } + // if (!isSysCat()) + return -1; + // else { // syscat queries aren't run by a threadpool, can't reschedule those + //jobs usleep(1000); goto retry; + // } + } - bppv = it->second; - } + bppv = it->second; + } - if (bppv->aborted()) - return 0; + if (bppv->aborted()) + return 0; - bpp = bppv->next(); - scoped.unlock(); + bpp = bppv->next(); + scoped.unlock(); - if (!bpp) - { -// if (isSysCat()) { -// usleep(1000); -// goto retry; -// } - return -1; // all BPP instances are busy, make threadpool reschedule - } + if (!bpp) + { + // if (isSysCat()) { + // usleep(1000); + // goto retry; + // } + return -1; // all BPP instances are busy, make threadpool reschedule + } - gotBPP = true; - bpp->resetBPP(*bs, writelock, sock); - firstRun = false; - } // firstRun + gotBPP = true; + bpp->resetBPP(*bs, writelock, sock); + firstRun = false; + } // firstRun - - if (fTrace) - { - PTLogsMap_t::iterator it; + if (fTrace) + { + PTLogsMap_t::iterator it; #ifdef _MSC_VER - tid = GetCurrentThreadId(); + tid = GetCurrentThreadId(); #else - tid = pthread_self(); + tid = pthread_self(); #endif - // only lock map while inserted objects - // once there is an object for each thread - // there is not need to lock - if (gFDList.size() < (uint32_t)fPMThreads) - { - gFDMutex.lock(); - ptLock = true; - } + // only lock map while inserted objects + // once there is an object for each thread + // there is not need to lock + if (gFDList.size() < (uint32_t)fPMThreads) + { + gFDMutex.lock(); + ptLock = true; + } - it = gFDList.find(tid); + it = gFDList.find(tid); - if (it == gFDList.end()) - { - ostringstream LogFileName; - SPPTLogs_t spof; + if (it == gFDList.end()) + { + ostringstream LogFileName; + SPPTLogs_t spof; #ifdef _MSC_VER - LogFileName << "C:/Calpont/log/trace/pt." << tid; + LogFileName << "C:/Calpont/log/trace/pt." << tid; #else - LogFileName << MCSLOGDIR << "/trace/pt." << tid; + LogFileName << MCSLOGDIR << "/trace/pt." << tid; #endif - spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str())); - gThdCnt++; + spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str())); + gThdCnt++; - // TODO: add error checking - if (spof->logFD.is_open()) - { - gFDList[tid] = spof; - logFD = spof.get(); - } - } - else - logFD = (*it).second.get(); - - if (ptLock) - { - gFDMutex.unlock(); - ptLock = false; - } - - clock_gettime(CLOCK_MONOTONIC, &tm); - } // if (fTrace) - - uint32_t retries = 0; -restart: - - try + // TODO: add error checking + if (spof->logFD.is_open()) { - ret = (*bpp)(); + gFDList[tid] = spof; + logFD = spof.get(); } - catch (NeedToRestartJob& e) - { - ostringstream os; - // experimentally the race can exist longer than 10s. "No way" should - // it take 10 minutes. If it does, the user will have to resubmit their - // query. + } + else + logFD = (*it).second.get(); - // 9/27/12 - changed the timeout to 2 mins b/c people report the system - // is hung if it does nothing for 10 mins. 2 mins should still be more - // than enough - if (++retries == 120) - { - os << e.what() << ": Restarted a syscat job " << retries << " times, bailing\n"; - throw NeedToRestartJob(os.str()); - } + if (ptLock) + { + gFDMutex.unlock(); + ptLock = false; + } - flushSyscatOIDs(); - bs->rewind(); - bpp->resetBPP(*bs, writelock, sock); - sleep(1); - goto restart; - } + clock_gettime(CLOCK_MONOTONIC, &tm); + } // if (fTrace) - if (ret) - return ret; + uint32_t retries = 0; + restart: - if (fTrace) - if (logFD && logFD->logFD.is_open()) - { - clock_gettime(CLOCK_MONOTONIC, &tm2); - timespec_sub(tm, tm2, tm3); - logFD->logFD - << left << setw(3) << logFD->thdId - << right << fixed << ((double)(tm.tv_sec + (1.e-9 * tm.tv_nsec))) << " " - << right << fixed << tm3 << " " - << right << setw(6) << bpp->getSessionID() << " " - << right << setw(4) << bpp->getStepID() << " " - << right << setw(2) << bpp->FilterCount() << " " - << right << setw(2) << bpp->ProjectCount() << " " - << right << setw(9) << bpp->PhysIOCount() << " " - << right << setw(9) << bpp->CachedIOCount() << " " - << right << setw(9) << bpp->BlocksTouchedCount() - << endl; - } // if (logFD... - - } - catch (scalar_exception& se) + try { - if (gotBPP) - bpp->busy(false); - - if (ptLock) - { - gFDMutex.unlock(); - ptLock = false; - } + ret = (*bpp)(); } - catch (exception& ex) + catch (NeedToRestartJob& e) { - if (gotBPP) - bpp->busy(false); + ostringstream os; + // experimentally the race can exist longer than 10s. "No way" should + // it take 10 minutes. If it does, the user will have to resubmit their + // query. - if (ptLock) - { - gFDMutex.unlock(); - ptLock = false; - } + // 9/27/12 - changed the timeout to 2 mins b/c people report the system + // is hung if it does nothing for 10 mins. 2 mins should still be more + // than enough + if (++retries == 120) + { + os << e.what() << ": Restarted a syscat job " << retries << " times, bailing\n"; + throw NeedToRestartJob(os.str()); + } - catchHandler(ex.what(), uniqueID, stepID); - cout << "BPPSeeder step " << stepID << " caught an exception: " << ex.what() << endl; + flushSyscatOIDs(); + bs->rewind(); + bpp->resetBPP(*bs, writelock, sock); + sleep(1); + goto restart; } - catch (...) + + if (ret) + return ret; + + if (fTrace) + if (logFD && logFD->logFD.is_open()) + { + clock_gettime(CLOCK_MONOTONIC, &tm2); + timespec_sub(tm, tm2, tm3); + logFD->logFD << left << setw(3) << logFD->thdId << right << fixed + << ((double)(tm.tv_sec + (1.e-9 * tm.tv_nsec))) << " " << right << fixed << tm3 << " " + << right << setw(6) << bpp->getSessionID() << " " << right << setw(4) << bpp->getStepID() + << " " << right << setw(2) << bpp->FilterCount() << " " << right << setw(2) + << bpp->ProjectCount() << " " << right << setw(9) << bpp->PhysIOCount() << " " << right + << setw(9) << bpp->CachedIOCount() << " " << right << setw(9) + << bpp->BlocksTouchedCount() << endl; + } // if (logFD... + } + catch (scalar_exception& se) + { + if (gotBPP) + bpp->busy(false); + + if (ptLock) { - if (gotBPP) - bpp->busy(false); + gFDMutex.unlock(); + ptLock = false; + } + } + catch (exception& ex) + { + if (gotBPP) + bpp->busy(false); - if (ptLock) - { - gFDMutex.unlock(); - ptLock = false; - } - - string msg("BPPSeeder caught an unknown exception"); - catchHandler(msg, uniqueID, stepID); - cout << msg << endl; + if (ptLock) + { + gFDMutex.unlock(); + ptLock = false; } - return ret; + catchHandler(ex.what(), uniqueID, stepID); + cout << "BPPSeeder step " << stepID << " caught an exception: " << ex.what() << endl; + } + catch (...) + { + if (gotBPP) + bpp->busy(false); + + if (ptLock) + { + gFDMutex.unlock(); + ptLock = false; + } + + string msg("BPPSeeder caught an unknown exception"); + catchHandler(msg, uniqueID, stepID); + cout << msg << endl; + } + + return ret; } void BPPSeeder::catchHandler(const string& ex, uint32_t id, uint32_t step) { - Logger log; - log.logMessage(ex); - sendErrorMsg(id, logging::bppSeederErr, step); + Logger log; + log.logMessage(ex); + sendErrorMsg(id, logging::bppSeederErr, step); } - void BPPSeeder::sendErrorMsg(uint32_t id, uint16_t status, uint32_t step) { + ISMPacketHeader ism; + PrimitiveHeader ph = {0, 0, 0, 0, 0, 0}; - ISMPacketHeader ism; - PrimitiveHeader ph = {0,0,0,0,0,0}; + ism.Status = status; + ph.UniqueID = id; + ph.StepID = step; + ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + msg.append((uint8_t*)&ism, sizeof(ism)); + msg.append((uint8_t*)&ph, sizeof(ph)); - ism.Status = status; - ph.UniqueID = id; - ph.StepID = step; - ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - msg.append((uint8_t*) &ism, sizeof(ism)); - msg.append((uint8_t*) &ph, sizeof(ph)); - - boost::mutex::scoped_lock lk(*writelock); - sock->write(msg); + boost::mutex::scoped_lock lk(*writelock); + sock->write(msg); } bool BPPSeeder::isSysCat() { - const uint8_t* buf; - uint32_t sessionIDOffset = sizeof(ISMPacketHeader); - uint32_t sessionID; + const uint8_t* buf; + uint32_t sessionIDOffset = sizeof(ISMPacketHeader); + uint32_t sessionID; - buf = bs->buf(); - sessionID = *((uint32_t*) &buf[sessionIDOffset]); - return (sessionID & 0x80000000); + buf = bs->buf(); + sessionID = *((uint32_t*)&buf[sessionIDOffset]); + return (sessionID & 0x80000000); } uint32_t BPPSeeder::getID() { - return uniqueID; + return uniqueID; } /* This is part of the syscat-retry hack. We should get rid of it once we @@ -424,16 +409,15 @@ uint32_t BPPSeeder::getID() */ void BPPSeeder::flushSyscatOIDs() { - vector syscatOIDs; + vector syscatOIDs; - syscatOIDs = execplan::getAllSysCatOIDs(); + syscatOIDs = execplan::getAllSysCatOIDs(); - for (int i = 0; i < fCacheCount; i++) - { - dbbc::blockCacheClient bc(*BRPp[i]); - bc.flushOIDs((const uint32_t*) &syscatOIDs[0], syscatOIDs.size()); - } + for (int i = 0; i < fCacheCount; i++) + { + dbbc::blockCacheClient bc(*BRPp[i]); + bc.flushOIDs((const uint32_t*)&syscatOIDs[0], syscatOIDs.size()); + } } -}; - +}; // namespace primitiveprocessor diff --git a/primitives/primproc/bppseeder.h b/primitives/primproc/bppseeder.h index ec5c51dcb..c4e021fbe 100644 --- a/primitives/primproc/bppseeder.h +++ b/primitives/primproc/bppseeder.h @@ -50,54 +50,51 @@ namespace primitiveprocessor { class BPPSeeder : public threadpool::PriorityThreadPool::Functor { -public: - BPPSeeder(const messageqcpp::SBS&, - const SP_UM_MUTEX& wLock, - const SP_UM_IOSOCK& ios, - const int pmThreads, - const bool trace = false); - BPPSeeder(const BPPSeeder& b); + public: + BPPSeeder(const messageqcpp::SBS&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& ios, const int pmThreads, + const bool trace = false); + BPPSeeder(const BPPSeeder& b); - virtual ~BPPSeeder(); + virtual ~BPPSeeder(); - int operator()(); + int operator()(); - bool isSysCat(); - boost::shared_ptr spof; + bool isSysCat(); + boost::shared_ptr spof; - uint32_t getID(); + uint32_t getID(); - void priority(uint32_t p) - { - _priority = p; - } - uint32_t priority() - { - return _priority; - } + void priority(uint32_t p) + { + _priority = p; + } + uint32_t priority() + { + return _priority; + } -private: - BPPSeeder(); - void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step); - void sendErrorMsg(uint32_t id, uint16_t status, uint32_t step); - void flushSyscatOIDs(); + private: + BPPSeeder(); + void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step); + void sendErrorMsg(uint32_t id, uint16_t status, uint32_t step); + void flushSyscatOIDs(); - messageqcpp::SBS bs; - SP_UM_MUTEX writelock; - SP_UM_IOSOCK sock; - int fPMThreads; - bool fTrace; + messageqcpp::SBS bs; + SP_UM_MUTEX writelock; + SP_UM_IOSOCK sock; + int fPMThreads; + bool fTrace; - /* To support reentrancy */ - uint32_t uniqueID, sessionID, stepID, failCount; - boost::shared_ptr bpp; - SBPPV bppv; - bool firstRun; - boost::posix_time::ptime dieTime; + /* To support reentrancy */ + uint32_t uniqueID, sessionID, stepID, failCount; + boost::shared_ptr bpp; + SBPPV bppv; + bool firstRun; + boost::posix_time::ptime dieTime; - uint32_t _priority; + uint32_t _priority; }; -}; +}; // namespace primitiveprocessor #endif diff --git a/primitives/primproc/bppsendthread.cpp b/primitives/primproc/bppsendthread.cpp index 2ebccae23..580598dfc 100644 --- a/primitives/primproc/bppsendthread.cpp +++ b/primitives/primproc/bppsendthread.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* -***********************************************************************/ + * $Id$ + * + * + ***********************************************************************/ #include #include @@ -31,260 +31,272 @@ namespace primitiveprocessor extern uint32_t connectionsPerUM; extern uint32_t BPPCount; -BPPSendThread::BPPSendThread() : die(false), gotException(false), mainThreadWaiting(false), - sizeThreshold(100), msgsLeft(-1), waiting(false), sawAllConnections(false), - fcEnabled(false), currentByteSize(0) +BPPSendThread::BPPSendThread() + : die(false) + , gotException(false) + , mainThreadWaiting(false) + , sizeThreshold(100) + , msgsLeft(-1) + , waiting(false) + , sawAllConnections(false) + , fcEnabled(false) + , currentByteSize(0) { - maxByteSize = joblist::ResourceManager::instance()->getMaxBPPSendQueue(); - runner = boost::thread(Runner_t(this)); + maxByteSize = joblist::ResourceManager::instance()->getMaxBPPSendQueue(); + runner = boost::thread(Runner_t(this)); } -BPPSendThread::BPPSendThread(uint32_t initMsgsLeft) : die(false), gotException(false), - mainThreadWaiting(false), sizeThreshold(100), msgsLeft(initMsgsLeft), waiting(false), - sawAllConnections(false), fcEnabled(false), currentByteSize(0) +BPPSendThread::BPPSendThread(uint32_t initMsgsLeft) + : die(false) + , gotException(false) + , mainThreadWaiting(false) + , sizeThreshold(100) + , msgsLeft(initMsgsLeft) + , waiting(false) + , sawAllConnections(false) + , fcEnabled(false) + , currentByteSize(0) { - maxByteSize = joblist::ResourceManager::instance()->getMaxBPPSendQueue(); - runner = boost::thread(Runner_t(this)); + maxByteSize = joblist::ResourceManager::instance()->getMaxBPPSendQueue(); + runner = boost::thread(Runner_t(this)); } BPPSendThread::~BPPSendThread() { - abort(); - runner.join(); + abort(); + runner.join(); } void BPPSendThread::sendResult(const Msg_t& msg, bool newConnection) { - // Wait for the queue to empty out a bit if it's stuffed full - if (sizeTooBig()) + // Wait for the queue to empty out a bit if it's stuffed full + if (sizeTooBig()) + { + std::unique_lock sl1(respondLock); + while (currentByteSize >= maxByteSize && msgQueue.size() > 3 && !die) { - std::unique_lock sl1(respondLock); - while (currentByteSize >= maxByteSize && msgQueue.size() > 3 && !die) - { - respondWait = true; - fProcessorPool->incBlockedThreads(); - okToRespond.wait(sl1); - fProcessorPool->decBlockedThreads(); - respondWait = false; - } + respondWait = true; + fProcessorPool->incBlockedThreads(); + okToRespond.wait(sl1); + fProcessorPool->decBlockedThreads(); + respondWait = false; } - if (die) - return; - - std::unique_lock sl(msgQueueLock); - - if (gotException) - throw runtime_error(exceptionString); + } + if (die) + return; - (void)atomicops::atomicAdd(¤tByteSize, msg.msg->lengthWithHdrOverhead()); - msgQueue.push(msg); + std::unique_lock sl(msgQueueLock); - if (!sawAllConnections && newConnection) + if (gotException) + throw runtime_error(exceptionString); + + (void)atomicops::atomicAdd(¤tByteSize, msg.msg->lengthWithHdrOverhead()); + msgQueue.push(msg); + + if (!sawAllConnections && newConnection) + { + Connection_t ins(msg.sockLock, msg.sock); + bool inserted = connections_s.insert(ins).second; + + if (inserted) { - Connection_t ins(msg.sockLock, msg.sock); - bool inserted = connections_s.insert(ins).second; + connections_v.push_back(ins); - if (inserted) - { - connections_v.push_back(ins); - - if (connections_v.size() == connectionsPerUM) - { - connections_s.clear(); - sawAllConnections = true; - } - } + if (connections_v.size() == connectionsPerUM) + { + connections_s.clear(); + sawAllConnections = true; + } } + } - if (mainThreadWaiting) - queueNotEmpty.notify_one(); + if (mainThreadWaiting) + queueNotEmpty.notify_one(); } void BPPSendThread::sendResults(const vector& msgs, bool newConnection) { - // Wait for the queue to empty out a bit if it's stuffed full - if (sizeTooBig()) + // Wait for the queue to empty out a bit if it's stuffed full + if (sizeTooBig()) + { + std::unique_lock sl1(respondLock); + while (currentByteSize >= maxByteSize && msgQueue.size() > 3 && !die) { - std::unique_lock sl1(respondLock); - while (currentByteSize >= maxByteSize && msgQueue.size() > 3 && !die) - { - respondWait = true; - fProcessorPool->incBlockedThreads(); - okToRespond.wait(sl1); - fProcessorPool->decBlockedThreads(); - respondWait = false; - } + respondWait = true; + fProcessorPool->incBlockedThreads(); + okToRespond.wait(sl1); + fProcessorPool->decBlockedThreads(); + respondWait = false; } - if (die) - return; + } + if (die) + return; - std::unique_lock sl(msgQueueLock); - - if (gotException) - throw runtime_error(exceptionString); + std::unique_lock sl(msgQueueLock); - if (!sawAllConnections && newConnection) + if (gotException) + throw runtime_error(exceptionString); + + if (!sawAllConnections && newConnection) + { + idbassert(msgs.size() > 0); + Connection_t ins(msgs[0].sockLock, msgs[0].sock); + bool inserted = connections_s.insert(ins).second; + + if (inserted) { - idbassert(msgs.size() > 0); - Connection_t ins(msgs[0].sockLock, msgs[0].sock); - bool inserted = connections_s.insert(ins).second; + connections_v.push_back(ins); - if (inserted) - { - connections_v.push_back(ins); - - if (connections_v.size() == connectionsPerUM) - { - connections_s.clear(); - sawAllConnections = true; - } - } + if (connections_v.size() == connectionsPerUM) + { + connections_s.clear(); + sawAllConnections = true; + } } + } - for (uint32_t i = 0; i < msgs.size(); i++) - { - (void)atomicops::atomicAdd(¤tByteSize, msgs[i].msg->lengthWithHdrOverhead()); - msgQueue.push(msgs[i]); - } + for (uint32_t i = 0; i < msgs.size(); i++) + { + (void)atomicops::atomicAdd(¤tByteSize, msgs[i].msg->lengthWithHdrOverhead()); + msgQueue.push(msgs[i]); + } - if (mainThreadWaiting) - queueNotEmpty.notify_one(); + if (mainThreadWaiting) + queueNotEmpty.notify_one(); } void BPPSendThread::sendMore(int num) { - std::unique_lock sl(ackLock); + std::unique_lock sl(ackLock); -// cout << "got an ACK for " << num << " msgsLeft=" << msgsLeft << endl; - if (num == -1) - fcEnabled = false; - else if (num == 0) - { - fcEnabled = true; - msgsLeft = 0; - } - else - (void)atomicops::atomicAdd(&msgsLeft, num); + // cout << "got an ACK for " << num << " msgsLeft=" << msgsLeft << endl; + if (num == -1) + fcEnabled = false; + else if (num == 0) + { + fcEnabled = true; + msgsLeft = 0; + } + else + (void)atomicops::atomicAdd(&msgsLeft, num); - sl.unlock(); - if (waiting) - okToSend.notify_one(); + sl.unlock(); + if (waiting) + okToSend.notify_one(); } bool BPPSendThread::flowControlEnabled() { - return fcEnabled; + return fcEnabled; } void BPPSendThread::mainLoop() { - const uint32_t msgCap = 20; - boost::scoped_array msg; - uint32_t msgCount = 0, i, msgsSent; - SP_UM_MUTEX lock; - SP_UM_IOSOCK sock; - bool doLoadBalancing = false; + const uint32_t msgCap = 20; + boost::scoped_array msg; + uint32_t msgCount = 0, i, msgsSent; + SP_UM_MUTEX lock; + SP_UM_IOSOCK sock; + bool doLoadBalancing = false; - msg.reset(new Msg_t[msgCap]); + msg.reset(new Msg_t[msgCap]); - while (!die) + while (!die) + { + std::unique_lock sl(msgQueueLock); + + if (msgQueue.empty() && !die) { - std::unique_lock sl(msgQueueLock); - - if (msgQueue.empty() && !die) - { - mainThreadWaiting = true; - queueNotEmpty.wait(sl); - mainThreadWaiting = false; - continue; - } - - msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size()); - - for (i = 0; i < msgCount; i++) - { - msg[i] = msgQueue.front(); - msgQueue.pop(); - } - - doLoadBalancing = sawAllConnections; - sl.unlock(); - - /* In the send loop below, msgsSent tracks progress on sending the msg array, - * i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */ - msgsSent = 0; - - while (msgsSent < msgCount && !die) - { - uint64_t bsSize; - - if (msgsLeft <= 0 && fcEnabled && !die) - { - std::unique_lock sl2(ackLock); - while (msgsLeft <= 0 && fcEnabled && !die) - { - waiting = true; - okToSend.wait(sl2); - waiting = false; - } - } - - for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die; - msgsSent++, i++) - { - if (doLoadBalancing) - { - // Bug 4475 move control of sockIndex to batchPrimitiveProcessor - lock = connections_v[msg[msgsSent].sockIndex].sockLock; - sock = connections_v[msg[msgsSent].sockIndex].sock; - } - else - { - lock = msg[msgsSent].sockLock; - sock = msg[msgsSent].sock; - } - - bsSize = msg[msgsSent].msg->lengthWithHdrOverhead(); - - try - { - boost::mutex::scoped_lock sl2(*lock); - sock->write(*msg[msgsSent].msg); - //cout << "sent 1 msg\n"; - } - catch (std::exception& e) - { - sl.lock(); - exceptionString = e.what(); - gotException = true; - return; - } - - (void)atomicops::atomicDec(&msgsLeft); - (void)atomicops::atomicSub(¤tByteSize, bsSize); - msg[msgsSent].msg.reset(); - } - - if (respondWait && currentByteSize < maxByteSize) - { - okToRespond.notify_one(); - } - } + mainThreadWaiting = true; + queueNotEmpty.wait(sl); + mainThreadWaiting = false; + continue; } + + msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size()); + + for (i = 0; i < msgCount; i++) + { + msg[i] = msgQueue.front(); + msgQueue.pop(); + } + + doLoadBalancing = sawAllConnections; + sl.unlock(); + + /* In the send loop below, msgsSent tracks progress on sending the msg array, + * i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */ + msgsSent = 0; + + while (msgsSent < msgCount && !die) + { + uint64_t bsSize; + + if (msgsLeft <= 0 && fcEnabled && !die) + { + std::unique_lock sl2(ackLock); + while (msgsLeft <= 0 && fcEnabled && !die) + { + waiting = true; + okToSend.wait(sl2); + waiting = false; + } + } + + for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die; msgsSent++, i++) + { + if (doLoadBalancing) + { + // Bug 4475 move control of sockIndex to batchPrimitiveProcessor + lock = connections_v[msg[msgsSent].sockIndex].sockLock; + sock = connections_v[msg[msgsSent].sockIndex].sock; + } + else + { + lock = msg[msgsSent].sockLock; + sock = msg[msgsSent].sock; + } + + bsSize = msg[msgsSent].msg->lengthWithHdrOverhead(); + + try + { + boost::mutex::scoped_lock sl2(*lock); + sock->write(*msg[msgsSent].msg); + // cout << "sent 1 msg\n"; + } + catch (std::exception& e) + { + sl.lock(); + exceptionString = e.what(); + gotException = true; + return; + } + + (void)atomicops::atomicDec(&msgsLeft); + (void)atomicops::atomicSub(¤tByteSize, bsSize); + msg[msgsSent].msg.reset(); + } + + if (respondWait && currentByteSize < maxByteSize) + { + okToRespond.notify_one(); + } + } + } } void BPPSendThread::abort() { - std::lock_guard sl(msgQueueLock); - std::lock_guard sl2(ackLock); - std::lock_guard sl3(respondLock); - { - die = true; - } - queueNotEmpty.notify_all(); - okToSend.notify_all(); - okToRespond.notify_all(); -} - + std::lock_guard sl(msgQueueLock); + std::lock_guard sl2(ackLock); + std::lock_guard sl3(respondLock); + { + die = true; + } + queueNotEmpty.notify_all(); + okToSend.notify_all(); + okToRespond.notify_all(); } +} // namespace primitiveprocessor diff --git a/primitives/primproc/bppsendthread.h b/primitives/primproc/bppsendthread.h index 54dd62b76..deb6ff3d6 100644 --- a/primitives/primproc/bppsendthread.h +++ b/primitives/primproc/bppsendthread.h @@ -35,120 +35,125 @@ namespace primitiveprocessor { - class BPPSendThread { + public: + BPPSendThread(); // starts unthrottled + BPPSendThread(uint32_t initMsgsLeft); // starts throttled + virtual ~BPPSendThread(); -public: - BPPSendThread(); // starts unthrottled - BPPSendThread(uint32_t initMsgsLeft); // starts throttled - virtual ~BPPSendThread(); - - struct Msg_t + struct Msg_t + { + messageqcpp::SBS msg; + SP_UM_IOSOCK sock; + SP_UM_MUTEX sockLock; + int sockIndex; // Socket index for round robin control. Bug 4475 + Msg_t() : sockIndex(0) { - messageqcpp::SBS msg; - SP_UM_IOSOCK sock; - SP_UM_MUTEX sockLock; - int sockIndex; // Socket index for round robin control. Bug 4475 - Msg_t() : sockIndex(0) { } - Msg_t(const Msg_t& m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) { } - Msg_t& operator=(const Msg_t& m) - { - msg = m.msg; - sock = m.sock; - sockLock = m.sockLock; - sockIndex = m.sockIndex; - return *this; - } - Msg_t(const messageqcpp::SBS& m, const SP_UM_IOSOCK& so, const SP_UM_MUTEX& sl, int si) : - msg(m), sock(so), sockLock(sl), sockIndex(si) { } - }; - - bool sizeTooBig() - { - // keep the queue size below the 100 msg threshold & below the 250MB mark, - // but at least 3 msgs so there is always 1 ready to be sent. - return ((msgQueue.size() > sizeThreshold) || - (currentByteSize >= maxByteSize && msgQueue.size() > 3)) && !die; } - - - void sendMore(int num); - void sendResults(const std::vector& msgs, bool newConnection); - void sendResult(const Msg_t& msg, bool newConnection); - void mainLoop(); - bool flowControlEnabled(); - void abort(); - inline bool aborted() const + Msg_t(const Msg_t& m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) { - return die; } - void setProcessorPool(threadpool::PriorityThreadPool* processorPool) + Msg_t& operator=(const Msg_t& m) { - fProcessorPool = processorPool; + msg = m.msg; + sock = m.sock; + sockLock = m.sockLock; + sockIndex = m.sockIndex; + return *this; } - -private: - BPPSendThread(const BPPSendThread&); - BPPSendThread& operator=(const BPPSendThread&); - - struct Runner_t + Msg_t(const messageqcpp::SBS& m, const SP_UM_IOSOCK& so, const SP_UM_MUTEX& sl, int si) + : msg(m), sock(so), sockLock(sl), sockIndex(si) { - BPPSendThread* bppst; - Runner_t(BPPSendThread* b) : bppst(b) { } - void operator()() - { - utils::setThreadName("BPPSendThread"); - bppst->mainLoop(); - } - }; + } + }; - boost::thread runner; - std::queue msgQueue; - std::mutex msgQueueLock; - std::condition_variable queueNotEmpty; - volatile bool die, gotException, mainThreadWaiting; - std::string exceptionString; - uint32_t sizeThreshold; - volatile int32_t msgsLeft; - bool waiting; - std::mutex ackLock; - std::condition_variable okToSend; - // Condition to prevent run away queue - bool respondWait; - std::mutex respondLock; - std::condition_variable okToRespond; - - /* Load balancing structures */ - struct Connection_t + bool sizeTooBig() + { + // keep the queue size below the 100 msg threshold & below the 250MB mark, + // but at least 3 msgs so there is always 1 ready to be sent. + return ((msgQueue.size() > sizeThreshold) || (currentByteSize >= maxByteSize && msgQueue.size() > 3)) && + !die; + } + + void sendMore(int num); + void sendResults(const std::vector& msgs, bool newConnection); + void sendResult(const Msg_t& msg, bool newConnection); + void mainLoop(); + bool flowControlEnabled(); + void abort(); + inline bool aborted() const + { + return die; + } + void setProcessorPool(threadpool::PriorityThreadPool* processorPool) + { + fProcessorPool = processorPool; + } + + private: + BPPSendThread(const BPPSendThread&); + BPPSendThread& operator=(const BPPSendThread&); + + struct Runner_t + { + BPPSendThread* bppst; + Runner_t(BPPSendThread* b) : bppst(b) { - Connection_t(const SP_UM_MUTEX& lock, const SP_UM_IOSOCK& so) : - sockLock(lock), sock(so) { } - SP_UM_MUTEX sockLock; - SP_UM_IOSOCK sock; - bool operator<(const Connection_t& t) const - { - return sockLock.get() < t.sockLock.get(); - } - bool operator==(const Connection_t& t) const - { - return sockLock.get() == t.sockLock.get(); - } - }; - std::set connections_s; - std::vector connections_v; - bool sawAllConnections; - volatile bool fcEnabled; + } + void operator()() + { + utils::setThreadName("BPPSendThread"); + bppst->mainLoop(); + } + }; - /* secondary queue size restriction based on byte size */ - volatile uint64_t currentByteSize; - uint64_t maxByteSize; - // Used to tell the PriorityThreadPool It should consider additional threads because a - // queue full event has happened and a thread has been blocked. - threadpool::PriorityThreadPool* fProcessorPool; + boost::thread runner; + std::queue msgQueue; + std::mutex msgQueueLock; + std::condition_variable queueNotEmpty; + volatile bool die, gotException, mainThreadWaiting; + std::string exceptionString; + uint32_t sizeThreshold; + volatile int32_t msgsLeft; + bool waiting; + std::mutex ackLock; + std::condition_variable okToSend; + // Condition to prevent run away queue + bool respondWait; + std::mutex respondLock; + std::condition_variable okToRespond; + + /* Load balancing structures */ + struct Connection_t + { + Connection_t(const SP_UM_MUTEX& lock, const SP_UM_IOSOCK& so) : sockLock(lock), sock(so) + { + } + SP_UM_MUTEX sockLock; + SP_UM_IOSOCK sock; + bool operator<(const Connection_t& t) const + { + return sockLock.get() < t.sockLock.get(); + } + bool operator==(const Connection_t& t) const + { + return sockLock.get() == t.sockLock.get(); + } + }; + std::set connections_s; + std::vector connections_v; + bool sawAllConnections; + volatile bool fcEnabled; + + /* secondary queue size restriction based on byte size */ + volatile uint64_t currentByteSize; + uint64_t maxByteSize; + // Used to tell the PriorityThreadPool It should consider additional threads because a + // queue full event has happened and a thread has been blocked. + threadpool::PriorityThreadPool* fProcessorPool; }; -} +} // namespace primitiveprocessor #endif - diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 94e6ace75..16426f49d 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -55,1289 +55,1175 @@ using namespace logging; namespace primitiveprocessor { - extern int noVB; -ColumnCommand::ColumnCommand() : - Command(COLUMN_COMMAND), - blockCount(0), - loadCount(0), - suppressFilter(false) +ColumnCommand::ColumnCommand() : Command(COLUMN_COMMAND), blockCount(0), loadCount(0), suppressFilter(false) { } -ColumnCommand::ColumnCommand(execplan::CalpontSystemCatalog::ColType& aColType) -: ColumnCommand() +ColumnCommand::ColumnCommand(execplan::CalpontSystemCatalog::ColType& aColType) : ColumnCommand() { - colType = aColType; + colType = aColType; } -ColumnCommand::~ColumnCommand() { } +ColumnCommand::~ColumnCommand() +{ +} void ColumnCommand::_execute() { - if (_isScan) - makeScanMsg(); - else if (bpp->ridCount == 0) // this would cause a scan - { - blockCount += colType.colWidth; - return; // a step with no input rids does nothing - } - else - makeStepMsg(); + if (_isScan) + makeScanMsg(); + else if (bpp->ridCount == 0) // this would cause a scan + { + blockCount += colType.colWidth; + return; // a step with no input rids does nothing + } + else + makeStepMsg(); - issuePrimitive(); - processResult(); + issuePrimitive(); + processResult(); - // check if feeding a filtercommand - if (fFilterFeeder != NOT_FEEDER) - copyRidsForFilterCmd(); + // check if feeding a filtercommand + if (fFilterFeeder != NOT_FEEDER) + copyRidsForFilterCmd(); } void ColumnCommand::execute() { - if (fFilterFeeder == LEFT_FEEDER) - { - values = bpp->fFiltCmdValues[0].get(); - wide128Values = bpp->fFiltCmdBinaryValues[0].get(); - } - else if (fFilterFeeder == RIGHT_FEEDER) - { - values = bpp->fFiltCmdValues[1].get(); - wide128Values = bpp->fFiltCmdBinaryValues[1].get(); - } - else - { - values = bpp->values; - wide128Values = bpp->wide128Values; - } + if (fFilterFeeder == LEFT_FEEDER) + { + values = bpp->fFiltCmdValues[0].get(); + wide128Values = bpp->fFiltCmdBinaryValues[0].get(); + } + else if (fFilterFeeder == RIGHT_FEEDER) + { + values = bpp->fFiltCmdValues[1].get(); + wide128Values = bpp->fFiltCmdBinaryValues[1].get(); + } + else + { + values = bpp->values; + wide128Values = bpp->wide128Values; + } - _execute(); + _execute(); } void ColumnCommand::execute(int64_t* vals) { - values = vals; - _execute(); + values = vals; + _execute(); } void ColumnCommand::makeScanMsg() { - /* Finish the NewColRequestHeader. */ + /* Finish the NewColRequestHeader. */ - /* XXXPAT: if there is a Command preceeding this one, it's a DictScan feeding tokens - which need to become filter elements. Can we handle that with this design? - Implement that later. */ + /* XXXPAT: if there is a Command preceeding this one, it's a DictScan feeding tokens + which need to become filter elements. Can we handle that with this design? + Implement that later. */ - primMsg->ism.Size = baseMsgLength; - primMsg->NVALS = 0; - primMsg->LBID = lbid; - primMsg->RidFlags = 0xFFFF; + primMsg->ism.Size = baseMsgLength; + primMsg->NVALS = 0; + primMsg->LBID = lbid; + primMsg->RidFlags = 0xFFFF; -// cout << "scanning lbid " << lbid << " colwidth = " << primMsg->DataSize << -// " filterCount = " << filterCount << " outputType = " << -// (int) primMsg->OutputType << endl; + // cout << "scanning lbid " << lbid << " colwidth = " << primMsg->DataSize << + // " filterCount = " << filterCount << " outputType = " << + // (int) primMsg->OutputType << endl; } void ColumnCommand::makeStepMsg() { - memcpy(&inputMsg[baseMsgLength], bpp->relRids, bpp->ridCount << 1); - primMsg->RidFlags = bpp->ridMap; - primMsg->ism.Size = baseMsgLength + (bpp->ridCount << 1); - primMsg->NVALS = bpp->ridCount; - primMsg->LBID = lbid; -// cout << "lbid is " << lbid << endl; + memcpy(&inputMsg[baseMsgLength], bpp->relRids, bpp->ridCount << 1); + primMsg->RidFlags = bpp->ridMap; + primMsg->ism.Size = baseMsgLength + (bpp->ridCount << 1); + primMsg->NVALS = bpp->ridCount; + primMsg->LBID = lbid; + // cout << "lbid is " << lbid << endl; } template void ColumnCommand::_loadData() { - using ByteStreamType = typename messageqcpp::ByteStreamType::type; - uint32_t wasCached; - uint32_t blocksRead; - uint16_t _mask; - uint64_t oidLastLbid = 0; - bool lastBlockReached = false; - oidLastLbid = getLastLbid(); - uint32_t blocksToLoad = 0; - // The number of elements allocated equals to the number of - // iteratations of the first loop here. - BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(W * sizeof(BRM::LBID_t)); - uint8_t** blockPtrs = (uint8_t**) alloca(W * sizeof(uint8_t*)); - int i; + using ByteStreamType = typename messageqcpp::ByteStreamType::type; + uint32_t wasCached; + uint32_t blocksRead; + uint16_t _mask; + uint64_t oidLastLbid = 0; + bool lastBlockReached = false; + oidLastLbid = getLastLbid(); + uint32_t blocksToLoad = 0; + // The number of elements allocated equals to the number of + // iteratations of the first loop here. + BRM::LBID_t* lbids = (BRM::LBID_t*)alloca(W * sizeof(BRM::LBID_t)); + uint8_t** blockPtrs = (uint8_t**)alloca(W * sizeof(uint8_t*)); + int i; + _mask = mask; + // primMsg->RidFlags = 0xffff; // disables selective block loading + // cerr << "::ColumnCommand::_loadData OID " << getOID() << " l:" << primMsg->LBID << " ll: " << oidLastLbid + // << " primMsg->RidFlags " << primMsg->RidFlags << endl; - _mask = mask; - //primMsg->RidFlags = 0xffff; // disables selective block loading - //cerr << "::ColumnCommand::_loadData OID " << getOID() << " l:" << primMsg->LBID << " ll: " << oidLastLbid << " primMsg->RidFlags " << primMsg->RidFlags << endl; - - for (i = 0; i < W; ++i, _mask <<= shift) + for (i = 0; i < W; ++i, _mask <<= shift) + { + if ((!lastBlockReached && _isScan) || (!_isScan && primMsg->RidFlags & _mask)) { - if ((!lastBlockReached && _isScan) || (!_isScan && primMsg->RidFlags & _mask)) - { - lbids[blocksToLoad] = primMsg->LBID + i; - blockPtrs[blocksToLoad] = &bpp->blockData[i * BLOCK_SIZE]; - blocksToLoad++; - loadCount++; - } - else if (lastBlockReached && _isScan) - { - // fill remaining blocks with empty values when col scan - uint32_t blockLen = BLOCK_SIZE / W; - auto attrs = datatypes::SystemCatalog::TypeAttributesStd(W, - 0, - -1); - const auto* typeHandler = datatypes::TypeHandler::find(colType.colDataType, - attrs); - const uint8_t* emptyValue = typeHandler->getEmptyValueForType(attrs); - uint8_t* blockDataPtr = &bpp->blockData[i * BLOCK_SIZE]; + lbids[blocksToLoad] = primMsg->LBID + i; + blockPtrs[blocksToLoad] = &bpp->blockData[i * BLOCK_SIZE]; + blocksToLoad++; + loadCount++; + } + else if (lastBlockReached && _isScan) + { + // fill remaining blocks with empty values when col scan + uint32_t blockLen = BLOCK_SIZE / W; + auto attrs = datatypes::SystemCatalog::TypeAttributesStd(W, 0, -1); + const auto* typeHandler = datatypes::TypeHandler::find(colType.colDataType, attrs); + const uint8_t* emptyValue = typeHandler->getEmptyValueForType(attrs); + uint8_t* blockDataPtr = &bpp->blockData[i * BLOCK_SIZE]; - idbassert(blockDataPtr); - fillEmptyBlock(blockDataPtr, emptyValue, blockLen); - }// else + idbassert(blockDataPtr); + fillEmptyBlock(blockDataPtr, emptyValue, blockLen); + } // else - if ( (primMsg->LBID + i) == oidLastLbid) - lastBlockReached = true; + if ((primMsg->LBID + i) == oidLastLbid) + lastBlockReached = true; - blockCount++; - } // for + blockCount++; + } // for - /* Do the load */ - wasCached = primitiveprocessor::loadBlocks(lbids, - bpp->versionInfo, - bpp->txnID, - colType.compressionType, - blockPtrs, - &blocksRead, - bpp->LBIDTrace, - bpp->sessionID, - blocksToLoad, - &wasVersioned, - willPrefetch(), - &bpp->vssCache); - bpp->cachedIO += wasCached; - bpp->physIO += blocksRead; - bpp->touchedBlocks += blocksToLoad; + /* Do the load */ + wasCached = primitiveprocessor::loadBlocks(lbids, bpp->versionInfo, bpp->txnID, colType.compressionType, + blockPtrs, &blocksRead, bpp->LBIDTrace, bpp->sessionID, + blocksToLoad, &wasVersioned, willPrefetch(), &bpp->vssCache); + bpp->cachedIO += wasCached; + bpp->physIO += blocksRead; + bpp->touchedBlocks += blocksToLoad; } void ColumnCommand::loadData() { - switch(colType.colWidth) - { - case 1: - _loadData<1>(); - break; - case 2: - _loadData<2>(); - break; - case 4: - _loadData<4>(); - break; - case 8: - _loadData<8>(); - break; - case 16: - _loadData<16>(); - break; + switch (colType.colWidth) + { + case 1: _loadData<1>(); break; + case 2: _loadData<2>(); break; + case 4: _loadData<4>(); break; + case 8: _loadData<8>(); break; + case 16: _loadData<16>(); break; - default: - throw NotImplementedExcept(std::string("ColumnCommand::loadData does not support ") - + std::to_string(colType.colWidth) - + std::string(" byte width.")); - } + default: + throw NotImplementedExcept(std::string("ColumnCommand::loadData does not support ") + + std::to_string(colType.colWidth) + std::string(" byte width.")); + } } void ColumnCommand::issuePrimitive() { - loadData(); + loadData(); - if (!suppressFilter) - bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); + if (!suppressFilter) + bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); + else + bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); + + switch (colType.colWidth) + { + case 1: _issuePrimitive<1>(); break; + case 2: _issuePrimitive<2>(); break; + case 4: _issuePrimitive<4>(); break; + case 8: _issuePrimitive<8>(); break; + case 16: _issuePrimitive<16>(); break; + default: + throw NotImplementedExcept(std::string("ColumnCommand::_issuePrimitive does not support ") + + std::to_string(colType.colWidth) + std::string(" byte width.")); + } + + if (_isScan) + { + if (LIKELY(colType.isNarrow())) + updateCPDataNarrow(); else - bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); - - switch(colType.colWidth) - { - case 1: - _issuePrimitive<1>(); - break; - case 2: - _issuePrimitive<2>(); - break; - case 4: - _issuePrimitive<4>(); - break; - case 8: - _issuePrimitive<8>(); - break; - case 16: - _issuePrimitive<16>(); - break; - default: - throw NotImplementedExcept(std::string("ColumnCommand::_issuePrimitive does not support ") - + std::to_string(colType.colWidth) - + std::string(" byte width.")); - } - - if (_isScan) - { - if (LIKELY(colType.isNarrow())) - updateCPDataNarrow(); - else - updateCPDataWide(); - } + updateCPDataWide(); + } } -template +template void ColumnCommand::_issuePrimitiveNarrow() { - _loadData(); + _loadData(); - if (!suppressFilter) - bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); - else - bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); + if (!suppressFilter) + bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); + else + bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); - _issuePrimitive(); + _issuePrimitive(); - if (_isScan) - updateCPDataNarrow(); + if (_isScan) + updateCPDataNarrow(); } - -template +template void ColumnCommand::_issuePrimitive() { - using IntegralType = typename datatypes::WidthToSIntegralType::type; - // Down the call stack the code presumes outMsg buffer has enough space to store - // ColRequestHeader + uint16_t Rids[8192] + IntegralType[8192]. - bpp->getPrimitiveProcessor().columnScanAndFilter(primMsg, outMsg); -} // _issuePrimitive() + using IntegralType = typename datatypes::WidthToSIntegralType::type; + // Down the call stack the code presumes outMsg buffer has enough space to store + // ColRequestHeader + uint16_t Rids[8192] + IntegralType[8192]. + bpp->getPrimitiveProcessor().columnScanAndFilter(primMsg, outMsg); +} // _issuePrimitive() void ColumnCommand::updateCPDataNarrow() { - /* Update CP data, the PseudoColumn code should always be !_isScan. Should be safe - to leave this here for now. */ - if (_isScan) - { - bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); - bpp->lbidForCP = lbid; - bpp->maxVal = static_cast(outMsg->Max); - bpp->minVal = static_cast(outMsg->Min); - } + /* Update CP data, the PseudoColumn code should always be !_isScan. Should be safe + to leave this here for now. */ + if (_isScan) + { + bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); + bpp->lbidForCP = lbid; + bpp->maxVal = static_cast(outMsg->Max); + bpp->minVal = static_cast(outMsg->Min); + } } void ColumnCommand::updateCPDataWide() { - /* Update CP data, the PseudoColumn code should always be !_isScan. Should be safe - to leave this here for now. */ - if (_isScan) + /* Update CP data, the PseudoColumn code should always be !_isScan. Should be safe + to leave this here for now. */ + if (_isScan) + { + bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); + bpp->lbidForCP = lbid; + if (colType.isWideDecimalType()) { - bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); - bpp->lbidForCP = lbid; - if (colType.isWideDecimalType()) - { - bpp->hasWideColumnOut = true; - // colWidth is int32 and wideColumnWidthOut's - // value is expected to be at most uint8. - bpp->wideColumnWidthOut = colType.colWidth; - bpp->max128Val = outMsg->Max; - bpp->min128Val = outMsg->Min; - } - else - { - throw NotImplementedExcept("ColumnCommand::updateCPDataWide(): unsupported dataType " - + std::to_string(primMsg->colType.DataType) - + " with width " - + std::to_string(colType.colWidth)); - } + bpp->hasWideColumnOut = true; + // colWidth is int32 and wideColumnWidthOut's + // value is expected to be at most uint8. + bpp->wideColumnWidthOut = colType.colWidth; + bpp->max128Val = outMsg->Max; + bpp->min128Val = outMsg->Min; } + else + { + throw NotImplementedExcept("ColumnCommand::updateCPDataWide(): unsupported dataType " + + std::to_string(primMsg->colType.DataType) + " with width " + + std::to_string(colType.colWidth)); + } + } } -template +template void ColumnCommand::_process_OT_BOTH_wAbsRids() { - using T = typename datatypes::WidthToSIntegralType::type; - bpp->ridCount = outMsg->NVALS; - bpp->ridMap = outMsg->RidFlags; - uint8_t* outPtr = reinterpret_cast(&outMsg[1]); - auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); - T* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); + using T = typename datatypes::WidthToSIntegralType::type; + bpp->ridCount = outMsg->NVALS; + bpp->ridMap = outMsg->RidFlags; + uint8_t* outPtr = reinterpret_cast(&outMsg[1]); + auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); + T* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); - for (size_t i = 0; i < outMsg->NVALS; ++i) - { - bpp->relRids[i] = ridPos[i]; - bpp->absRids[i] = ridPos[i] + bpp->baseRid; - values[i] = valuesPos[i]; - } + for (size_t i = 0; i < outMsg->NVALS; ++i) + { + bpp->relRids[i] = ridPos[i]; + bpp->absRids[i] = ridPos[i] + bpp->baseRid; + values[i] = valuesPos[i]; + } } -template<> +template <> void ColumnCommand::_process_OT_BOTH_wAbsRids<16>() { - using T = typename datatypes::WidthToSIntegralType<16>::type; - bpp->ridCount = outMsg->NVALS; - bpp->ridMap = outMsg->RidFlags; - uint8_t* outPtr = reinterpret_cast(&outMsg[1]); - auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); - int128_t* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); + using T = typename datatypes::WidthToSIntegralType<16>::type; + bpp->ridCount = outMsg->NVALS; + bpp->ridMap = outMsg->RidFlags; + uint8_t* outPtr = reinterpret_cast(&outMsg[1]); + auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); + int128_t* valuesPos = + primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); - for (size_t i = 0; i < outMsg->NVALS; ++i) - { - bpp->relRids[i] = ridPos[i]; - bpp->absRids[i] = ridPos[i] + bpp->baseRid; - datatypes::TSInt128::assignPtrPtr(&wide128Values[i], &valuesPos[i]); - } + for (size_t i = 0; i < outMsg->NVALS; ++i) + { + bpp->relRids[i] = ridPos[i]; + bpp->absRids[i] = ridPos[i] + bpp->baseRid; + datatypes::TSInt128::assignPtrPtr(&wide128Values[i], &valuesPos[i]); + } } - -template +template void ColumnCommand::_process_OT_BOTH() { - using T = typename datatypes::WidthToSIntegralType::type; - bpp->ridCount = outMsg->NVALS; - bpp->ridMap = outMsg->RidFlags; - uint8_t* outPtr = reinterpret_cast(&outMsg[1]); - auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); - T* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); + using T = typename datatypes::WidthToSIntegralType::type; + bpp->ridCount = outMsg->NVALS; + bpp->ridMap = outMsg->RidFlags; + uint8_t* outPtr = reinterpret_cast(&outMsg[1]); + auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); + T* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); - for (size_t i = 0; i < outMsg->NVALS; ++i) - { - bpp->relRids[i] = ridPos[i]; - values[i] = valuesPos[i]; - } + for (size_t i = 0; i < outMsg->NVALS; ++i) + { + bpp->relRids[i] = ridPos[i]; + values[i] = valuesPos[i]; + } } -template<> +template <> void ColumnCommand::_process_OT_BOTH<16>() { - using T = typename datatypes::WidthToSIntegralType<16>::type; - bpp->ridCount = outMsg->NVALS; - bpp->ridMap = outMsg->RidFlags; - uint8_t* outPtr = reinterpret_cast(&outMsg[1]); - auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); - T* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); + using T = typename datatypes::WidthToSIntegralType<16>::type; + bpp->ridCount = outMsg->NVALS; + bpp->ridMap = outMsg->RidFlags; + uint8_t* outPtr = reinterpret_cast(&outMsg[1]); + auto* ridPos = primitives::getRIDArrayPosition(outPtr, 0); + T* valuesPos = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); - for (size_t i = 0; i < outMsg->NVALS; ++i) - { - bpp->relRids[i] = ridPos[i]; - datatypes::TSInt128::assignPtrPtr(&wide128Values[i], &valuesPos[i]); - } + for (size_t i = 0; i < outMsg->NVALS; ++i) + { + bpp->relRids[i] = ridPos[i]; + datatypes::TSInt128::assignPtrPtr(&wide128Values[i], &valuesPos[i]); + } } void ColumnCommand::process_OT_BOTH() { - if (makeAbsRids) + if (makeAbsRids) + { + switch (colType.colWidth) { - switch (colType.colWidth) - { - case 16: - _process_OT_BOTH_wAbsRids<16>(); - break; - case 8: - _process_OT_BOTH_wAbsRids<8>(); - break; - case 4: - _process_OT_BOTH_wAbsRids<4>(); - break; - case 2: - _process_OT_BOTH_wAbsRids<2>(); - break; - case 1: - _process_OT_BOTH_wAbsRids<1>(); - break; - default: - throw NotImplementedExcept(std::string("ColumnCommand::process_OT_BOTH with ABS RIDs does not support ") - + std::to_string(colType.colWidth) - + std::string(" byte width.")); - } + case 16: _process_OT_BOTH_wAbsRids<16>(); break; + case 8: _process_OT_BOTH_wAbsRids<8>(); break; + case 4: _process_OT_BOTH_wAbsRids<4>(); break; + case 2: _process_OT_BOTH_wAbsRids<2>(); break; + case 1: _process_OT_BOTH_wAbsRids<1>(); break; + default: + throw NotImplementedExcept( + std::string("ColumnCommand::process_OT_BOTH with ABS RIDs does not support ") + + std::to_string(colType.colWidth) + std::string(" byte width.")); } - else + } + else + { + switch (colType.colWidth) { - switch (colType.colWidth) - { - case 16: - _process_OT_BOTH<16>(); - break; - case 8: - _process_OT_BOTH<8>(); - break; - case 4: - _process_OT_BOTH<4>(); - break; - case 2: - _process_OT_BOTH<2>(); - break; - case 1: - _process_OT_BOTH<1>(); - break; - default: - throw NotImplementedExcept(std::string("ColumnCommand::process_OT_BOTH does not support ") - + std::to_string(colType.colWidth) - + std::string(" byte width.")); - } + case 16: _process_OT_BOTH<16>(); break; + case 8: _process_OT_BOTH<8>(); break; + case 4: _process_OT_BOTH<4>(); break; + case 2: _process_OT_BOTH<2>(); break; + case 1: _process_OT_BOTH<1>(); break; + default: + throw NotImplementedExcept(std::string("ColumnCommand::process_OT_BOTH does not support ") + + std::to_string(colType.colWidth) + std::string(" byte width.")); } + } } void ColumnCommand::process_OT_RID() { - memcpy(bpp->relRids, outMsg + 1, outMsg->NVALS << 1); - bpp->ridCount = outMsg->NVALS; - bpp->ridMap = outMsg->RidFlags; + memcpy(bpp->relRids, outMsg + 1, outMsg->NVALS << 1); + bpp->ridCount = outMsg->NVALS; + bpp->ridMap = outMsg->RidFlags; } // TODO This spec makes an impicit type conversion to fit types with sizeof(type) <= 4 // into 8 bytes. Treat values as a buffer and use memcpy to store the values in one go. -template +template void ColumnCommand::_process_OT_DATAVALUE() { - using T = typename datatypes::WidthToSIntegralType::type; - bpp->ridCount = outMsg->NVALS; - T* arr = (T*) (outMsg + 1); - for (size_t i = 0; i < outMsg->NVALS; ++i) - values[i] = arr[i]; + using T = typename datatypes::WidthToSIntegralType::type; + bpp->ridCount = outMsg->NVALS; + T* arr = (T*)(outMsg + 1); + for (size_t i = 0; i < outMsg->NVALS; ++i) + values[i] = arr[i]; } -template<> +template <> void ColumnCommand::_process_OT_DATAVALUE<8>() { - bpp->ridCount = outMsg->NVALS; - memcpy(values, outMsg + 1, outMsg->NVALS << 3); + bpp->ridCount = outMsg->NVALS; + memcpy(values, outMsg + 1, outMsg->NVALS << 3); } -template<> +template <> void ColumnCommand::_process_OT_DATAVALUE<16>() { - bpp->ridCount = outMsg->NVALS; - memcpy(wide128Values, outMsg + 1, outMsg->NVALS << 4); + bpp->ridCount = outMsg->NVALS; + memcpy(wide128Values, outMsg + 1, outMsg->NVALS << 4); } void ColumnCommand::process_OT_DATAVALUE() { - bpp->ridCount = outMsg->NVALS; + bpp->ridCount = outMsg->NVALS; - switch (colType.colWidth) - { - case 16: - _process_OT_DATAVALUE<16>(); - break; + switch (colType.colWidth) + { + case 16: _process_OT_DATAVALUE<16>(); break; - case 8: - _process_OT_DATAVALUE<8>(); - break; + case 8: _process_OT_DATAVALUE<8>(); break; - case 4: - _process_OT_DATAVALUE<4>(); - break; + case 4: _process_OT_DATAVALUE<4>(); break; - case 2: - _process_OT_DATAVALUE<2>(); - break; + case 2: _process_OT_DATAVALUE<2>(); break; - case 1: - _process_OT_DATAVALUE<2>(); - break; + case 1: _process_OT_DATAVALUE<2>(); break; - default: - throw NotImplementedExcept(std::string("ColumnCommand::process_OT_DATAVALUE does not support ") - + std::to_string(colType.colWidth) - + std::string(" byte width.")); - } + default: + throw NotImplementedExcept(std::string("ColumnCommand::process_OT_DATAVALUE does not support ") + + std::to_string(colType.colWidth) + std::string(" byte width.")); + } } void ColumnCommand::processResult() { - /* Switch on output type, turn pCol output into something useful, store it in - the containing BPP */ + /* Switch on output type, turn pCol output into something useful, store it in + the containing BPP */ - switch (outMsg->OutputType) - { - case OT_BOTH: - process_OT_BOTH(); - break; + switch (outMsg->OutputType) + { + case OT_BOTH: process_OT_BOTH(); break; - case OT_RID: - process_OT_RID(); - break; + case OT_RID: process_OT_RID(); break; - case OT_DATAVALUE: - process_OT_DATAVALUE(); - break; + case OT_DATAVALUE: process_OT_DATAVALUE(); break; - default: - cout << "outputType = " << outMsg->OutputType << endl; - throw logic_error("ColumnCommand got a bad OutputType"); - } + default: + cout << "outputType = " << outMsg->OutputType << endl; + throw logic_error("ColumnCommand got a bad OutputType"); + } - // check if feeding a filtercommand - if (fFilterFeeder == LEFT_FEEDER) - { - bpp->fFiltRidCount[0] = bpp->ridCount; + // check if feeding a filtercommand + if (fFilterFeeder == LEFT_FEEDER) + { + bpp->fFiltRidCount[0] = bpp->ridCount; - for (uint64_t i = 0; i < bpp->ridCount; i++) - bpp->fFiltCmdRids[0][i] = bpp->relRids[i]; - } - else if (fFilterFeeder == RIGHT_FEEDER) - { - bpp->fFiltRidCount[1] = bpp->ridCount; + for (uint64_t i = 0; i < bpp->ridCount; i++) + bpp->fFiltCmdRids[0][i] = bpp->relRids[i]; + } + else if (fFilterFeeder == RIGHT_FEEDER) + { + bpp->fFiltRidCount[1] = bpp->ridCount; - for (uint64_t i = 0; i < bpp->ridCount; i++) - bpp->fFiltCmdRids[1][i] = bpp->relRids[i]; - } + for (uint64_t i = 0; i < bpp->ridCount; i++) + bpp->fFiltCmdRids[1][i] = bpp->relRids[i]; + } } // Used by PseudoCC at least void ColumnCommand::createCommand(ByteStream& bs) { - uint8_t tmp8; - bs.advance(1); - colType.unserialize(bs); - bs >> tmp8; - _isScan = tmp8; - bs >> traceFlags; - bs >> filterString; - bs >> BOP; - bs >> filterCount; - deserializeInlineVector(bs, lastLbid); + uint8_t tmp8; + bs.advance(1); + colType.unserialize(bs); + bs >> tmp8; + _isScan = tmp8; + bs >> traceFlags; + bs >> filterString; + bs >> BOP; + bs >> filterCount; + deserializeInlineVector(bs, lastLbid); - Command::createCommand(bs); - switch (colType.colWidth) - { - case 1: - createColumnFilter::type>(); - break; + Command::createCommand(bs); + switch (colType.colWidth) + { + case 1: createColumnFilter::type>(); break; - case 2: - createColumnFilter::type>(); - break; + case 2: createColumnFilter::type>(); break; - case 4: - createColumnFilter::type>(); - break; + case 4: createColumnFilter::type>(); break; - case 8: - createColumnFilter::type>(); - break; + case 8: createColumnFilter::type>(); break; - case 16: - createColumnFilter::type>(); - break; + case 16: createColumnFilter::type>(); break; - default: - throw NotImplementedExcept(std::string("ColumnCommand::createCommand does not support ") - + std::to_string(colType.colWidth) - + std::string(" byte width.")); - } + default: + throw NotImplementedExcept(std::string("ColumnCommand::createCommand does not support ") + + std::to_string(colType.colWidth) + std::string(" byte width.")); + } } void ColumnCommand::createCommand(execplan::CalpontSystemCatalog::ColType& aColType, ByteStream& bs) { - colType = aColType; - uint8_t tmp8; + colType = aColType; + uint8_t tmp8; - bs >> tmp8; - _isScan = tmp8; - bs >> traceFlags; - bs >> filterString; - bs >> BOP; - bs >> filterCount; - deserializeInlineVector(bs, lastLbid); + bs >> tmp8; + _isScan = tmp8; + bs >> traceFlags; + bs >> filterString; + bs >> BOP; + bs >> filterCount; + deserializeInlineVector(bs, lastLbid); - Command::createCommand(bs); + Command::createCommand(bs); } void ColumnCommand::resetCommand(ByteStream& bs) { - bs >> lbid; + bs >> lbid; } void ColumnCommand::prep(int8_t outputType, bool absRids) { - fillInPrimitiveMessageHeader(outputType, absRids); + fillInPrimitiveMessageHeader(outputType, absRids); - switch (colType.colWidth) - { - case 1: - shift = 16; - mask = 0xFFFF; - break; + switch (colType.colWidth) + { + case 1: + shift = 16; + mask = 0xFFFF; + break; - case 2: - shift = 8; - mask = 0xFF; - break; + case 2: + shift = 8; + mask = 0xFF; + break; - case 4: - shift = 4; - mask = 0x0F; - break; + case 4: + shift = 4; + mask = 0x0F; + break; - case 8: - shift = 2; - mask = 0x03; - break; + case 8: + shift = 2; + mask = 0x03; + break; - case 16: - shift = 1; - mask = 0x01; - break; + case 16: + shift = 1; + mask = 0x01; + break; - default: - cout << "CC: colWidth is " << colType.colWidth << endl; - throw logic_error("ColumnCommand: bad column width?"); - } + default: + cout << "CC: colWidth is " << colType.colWidth << endl; + throw logic_error("ColumnCommand: bad column width?"); + } } void ColumnCommand::fillInPrimitiveMessageHeader(const int8_t outputType, const bool absRids) { - // WIP Align this structure or move input RIDs away. - baseMsgLength = sizeof(NewColRequestHeader) + - (suppressFilter ? 0 : filterString.length()); - size_t inputMsgBufSize = baseMsgLength + (LOGICAL_BLOCK_RIDS * sizeof(primitives::RIDType)); + // WIP Align this structure or move input RIDs away. + baseMsgLength = sizeof(NewColRequestHeader) + (suppressFilter ? 0 : filterString.length()); + size_t inputMsgBufSize = baseMsgLength + (LOGICAL_BLOCK_RIDS * sizeof(primitives::RIDType)); - if (!inputMsg) - inputMsg.reset(reinterpret_cast(aligned_alloc(utils::MAXCOLUMNWIDTH, - inputMsgBufSize))); + if (!inputMsg) + inputMsg.reset(reinterpret_cast(aligned_alloc(utils::MAXCOLUMNWIDTH, inputMsgBufSize))); - primMsg = (NewColRequestHeader*) inputMsg.get(); - outMsg = (ColResultHeader*) bpp->outputMsg.get(); - makeAbsRids = absRids; - primMsg->ism.Interleave = 0; - primMsg->ism.Flags = 0; - primMsg->ism.Command = COL_BY_SCAN; - primMsg->ism.Size = sizeof(NewColRequestHeader) + (suppressFilter ? 0 : filterString.length()); - primMsg->ism.Type = 2; - primMsg->hdr.SessionID = bpp->sessionID; - primMsg->hdr.TransactionID = bpp->txnID; - primMsg->hdr.VerID = bpp->versionInfo.currentScn; - primMsg->hdr.StepID = bpp->stepID; - primMsg->colType = ColRequestHeaderDataType(colType); - primMsg->OutputType = outputType; - primMsg->BOP = BOP; - primMsg->NOPS = (suppressFilter ? 0 : filterCount); - primMsg->sort = 0; + primMsg = (NewColRequestHeader*)inputMsg.get(); + outMsg = (ColResultHeader*)bpp->outputMsg.get(); + makeAbsRids = absRids; + primMsg->ism.Interleave = 0; + primMsg->ism.Flags = 0; + primMsg->ism.Command = COL_BY_SCAN; + primMsg->ism.Size = sizeof(NewColRequestHeader) + (suppressFilter ? 0 : filterString.length()); + primMsg->ism.Type = 2; + primMsg->hdr.SessionID = bpp->sessionID; + primMsg->hdr.TransactionID = bpp->txnID; + primMsg->hdr.VerID = bpp->versionInfo.currentScn; + primMsg->hdr.StepID = bpp->stepID; + primMsg->colType = ColRequestHeaderDataType(colType); + primMsg->OutputType = outputType; + primMsg->BOP = BOP; + primMsg->NOPS = (suppressFilter ? 0 : filterCount); + primMsg->sort = 0; } /* Assumes OT_DATAVALUE */ void ColumnCommand::projectResult() { - auto nvals = outMsg->NVALS; - if (primMsg->NVALS != nvals || nvals != bpp->ridCount ) - { - ostringstream os; - BRM::DBRM brm; - BRM::OID_t oid; - uint16_t l_dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t fbo; - brm.lookupLocal(lbid, 0, false, oid, l_dbroot, partNum, segNum, fbo); + auto nvals = outMsg->NVALS; + if (primMsg->NVALS != nvals || nvals != bpp->ridCount) + { + ostringstream os; + BRM::DBRM brm; + BRM::OID_t oid; + uint16_t l_dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t fbo; + brm.lookupLocal(lbid, 0, false, oid, l_dbroot, partNum, segNum, fbo); - os << __FILE__ << " error on projection for oid " << oid << " lbid " << lbid; + os << __FILE__ << " error on projection for oid " << oid << " lbid " << lbid; - if (primMsg->NVALS != outMsg->NVALS ) - os << ": input rids " << primMsg->NVALS; - else - os << ": ridcount " << bpp->ridCount; + if (primMsg->NVALS != outMsg->NVALS) + os << ": input rids " << primMsg->NVALS; + else + os << ": ridcount " << bpp->ridCount; - os << ", output rids " << nvals << endl; + os << ", output rids " << nvals << endl; - //cout << os.str(); - if (bpp->sessionID & 0x80000000) - throw NeedToRestartJob(os.str()); - else - throw PrimitiveColumnProjectResultExcept(os.str()); - } + // cout << os.str(); + if (bpp->sessionID & 0x80000000) + throw NeedToRestartJob(os.str()); + else + throw PrimitiveColumnProjectResultExcept(os.str()); + } - idbassert(primMsg->NVALS == nvals); - idbassert(bpp->ridCount == nvals); - uint32_t valuesByteSize = nvals * colType.colWidth; - *bpp->serialized << valuesByteSize; - bpp->serialized->append(primitives::getFirstValueArrayPosition(outMsg), valuesByteSize); + idbassert(primMsg->NVALS == nvals); + idbassert(bpp->ridCount == nvals); + uint32_t valuesByteSize = nvals * colType.colWidth; + *bpp->serialized << valuesByteSize; + bpp->serialized->append(primitives::getFirstValueArrayPosition(outMsg), valuesByteSize); } void ColumnCommand::removeRowsFromRowGroup(RowGroup& rg) { - uint32_t gapSize = colType.colWidth + 2; - uint8_t* msg8; - uint16_t rid; - Row oldRow, newRow; - uint32_t oldIdx, newIdx; + uint32_t gapSize = colType.colWidth + 2; + uint8_t* msg8; + uint16_t rid; + Row oldRow, newRow; + uint32_t oldIdx, newIdx; - rg.initRow(&oldRow); - rg.initRow(&newRow); - rg.getRow(0, &oldRow); - rg.getRow(0, &newRow); - msg8 = (uint8_t*) (outMsg + 1); + rg.initRow(&oldRow); + rg.initRow(&newRow); + rg.getRow(0, &oldRow); + rg.getRow(0, &newRow); + msg8 = (uint8_t*)(outMsg + 1); - for (oldIdx = newIdx = 0; newIdx < outMsg->NVALS; newIdx++, msg8 += gapSize) + for (oldIdx = newIdx = 0; newIdx < outMsg->NVALS; newIdx++, msg8 += gapSize) + { + rid = *((uint16_t*)msg8); + + while (rid != bpp->relRids[oldIdx]) { - rid = *((uint16_t*) msg8); - - while (rid != bpp->relRids[oldIdx]) - { - // need to find rid in relrids, and it is in there - oldIdx++; - oldRow.nextRow(); - } - - if (oldIdx != newIdx) - { - bpp->relRids[newIdx] = rid; - // we use a memcpy here instead of copyRow() to avoid expanding the string table; - memcpy(newRow.getData(), oldRow.getData(), newRow.getSize()); - } - - oldIdx++; - oldRow.nextRow(); - newRow.nextRow(); + // need to find rid in relrids, and it is in there + oldIdx++; + oldRow.nextRow(); } - rg.setRowCount(outMsg->NVALS); // this gets rid of trailing rows, no need to set them to NULL - bpp->ridCount = outMsg->NVALS; - primMsg->NVALS = outMsg->NVALS; + if (oldIdx != newIdx) + { + bpp->relRids[newIdx] = rid; + // we use a memcpy here instead of copyRow() to avoid expanding the string table; + memcpy(newRow.getData(), oldRow.getData(), newRow.getSize()); + } + + oldIdx++; + oldRow.nextRow(); + newRow.nextRow(); + } + + rg.setRowCount(outMsg->NVALS); // this gets rid of trailing rows, no need to set them to NULL + bpp->ridCount = outMsg->NVALS; + primMsg->NVALS = outMsg->NVALS; } -template -void ColumnCommand::_projectResultRGLoop(rowgroup::Row& r, - const T* valuesArray, - const uint32_t offset) +template +void ColumnCommand::_projectResultRGLoop(rowgroup::Row& r, const T* valuesArray, const uint32_t offset) { - for (size_t i = 0; i < outMsg->NVALS; ++i) - { - r.setIntField_offset(valuesArray[i], offset); - r.nextRow(rowSize); - } + for (size_t i = 0; i < outMsg->NVALS; ++i) + { + r.setIntField_offset(valuesArray[i], offset); + r.nextRow(rowSize); + } } -template<> -void ColumnCommand::_projectResultRGLoop(rowgroup::Row& r, - const int128_t* valuesArray, +template <> +void ColumnCommand::_projectResultRGLoop(rowgroup::Row& r, const int128_t* valuesArray, const uint32_t offset) { - for (size_t i = 0; i < outMsg->NVALS; ++i) - { - r.setBinaryField_offset(&valuesArray[i], colType.colWidth, offset); - r.nextRow(rowSize); - } + for (size_t i = 0; i < outMsg->NVALS; ++i) + { + r.setBinaryField_offset(&valuesArray[i], colType.colWidth, offset); + r.nextRow(rowSize); + } } -template +template void ColumnCommand::_projectResultRG(RowGroup& rg, uint32_t pos) { - using T = typename datatypes::WidthToSIntegralType::type; - T* valuesArray = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); - uint32_t offset; - auto nvals = outMsg->NVALS; + using T = typename datatypes::WidthToSIntegralType::type; + T* valuesArray = primitives::getValuesArrayPosition(primitives::getFirstValueArrayPosition(outMsg), 0); + uint32_t offset; + auto nvals = outMsg->NVALS; - rg.initRow(&r); - offset = r.getOffset(pos); - rowSize = r.getSize(); + rg.initRow(&r); + offset = r.getOffset(pos); + rowSize = r.getSize(); - if ((primMsg->NVALS != nvals || nvals != bpp->ridCount) && (!noVB || bpp->sessionID & 0x80000000)) - { - ostringstream os; - BRM::DBRM brm; - BRM::OID_t oid; - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t fbo; - brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo); + if ((primMsg->NVALS != nvals || nvals != bpp->ridCount) && (!noVB || bpp->sessionID & 0x80000000)) + { + ostringstream os; + BRM::DBRM brm; + BRM::OID_t oid; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t fbo; + brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo); - os << __FILE__ << " error on projectResultRG for oid " << oid << " lbid " << lbid; + os << __FILE__ << " error on projectResultRG for oid " << oid << " lbid " << lbid; - if (primMsg->NVALS != nvals ) - os << ": input rids " << primMsg->NVALS; - else - os << ": ridcount " << bpp->ridCount; + if (primMsg->NVALS != nvals) + os << ": input rids " << primMsg->NVALS; + else + os << ": ridcount " << bpp->ridCount; - os << ", output rids " << nvals; + os << ", output rids " << nvals; - os << endl; + os << endl; - if (bpp->sessionID & 0x80000000) - throw NeedToRestartJob(os.str()); - else - throw PrimitiveColumnProjectResultExcept(os.str()); - } - else if (primMsg->NVALS != nvals || nvals != bpp->ridCount) - removeRowsFromRowGroup(rg); + if (bpp->sessionID & 0x80000000) + throw NeedToRestartJob(os.str()); + else + throw PrimitiveColumnProjectResultExcept(os.str()); + } + else if (primMsg->NVALS != nvals || nvals != bpp->ridCount) + removeRowsFromRowGroup(rg); - idbassert(primMsg->NVALS == nvals); - idbassert(bpp->ridCount == nvals); - rg.getRow(0, &r); + idbassert(primMsg->NVALS == nvals); + idbassert(bpp->ridCount == nvals); + rg.getRow(0, &r); - _projectResultRGLoop(r, valuesArray, offset); + _projectResultRGLoop(r, valuesArray, offset); } void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) { - switch (colType.colWidth) - { - case 1: - _projectResultRG<1>(rg, pos); - break; - case 2: - _projectResultRG<2>(rg, pos); - break; - case 4: - _projectResultRG<4>(rg, pos); - break; - case 8: - _projectResultRG<8>(rg, pos); - break; - case 16: - _projectResultRG<16>(rg, pos); - break; - } + switch (colType.colWidth) + { + case 1: _projectResultRG<1>(rg, pos); break; + case 2: _projectResultRG<2>(rg, pos); break; + case 4: _projectResultRG<4>(rg, pos); break; + case 8: _projectResultRG<8>(rg, pos); break; + case 16: _projectResultRG<16>(rg, pos); break; + } } void ColumnCommand::project() { - /* bpp->ridCount == 0 would signify a scan operation */ - if (bpp->ridCount == 0) - { - *bpp->serialized << (uint32_t) 0; - blockCount += colType.colWidth; - return; - } + /* bpp->ridCount == 0 would signify a scan operation */ + if (bpp->ridCount == 0) + { + *bpp->serialized << (uint32_t)0; + blockCount += colType.colWidth; + return; + } - makeStepMsg(); - issuePrimitive(); - projectResult(); + makeStepMsg(); + issuePrimitive(); + projectResult(); } void ColumnCommand::projectIntoRowGroup(RowGroup& rg, uint32_t pos) { - if (bpp->ridCount == 0) - { - blockCount += colType.colWidth; - return; - } + if (bpp->ridCount == 0) + { + blockCount += colType.colWidth; + return; + } - makeStepMsg(); - issuePrimitive(); - projectResultRG(rg, pos); + makeStepMsg(); + issuePrimitive(); + projectResultRG(rg, pos); } void ColumnCommand::nextLBID() { - lbid += colType.colWidth; + lbid += colType.colWidth; } void ColumnCommand::duplicate(ColumnCommand* cc) { - cc->_isScan = _isScan; - cc->traceFlags = traceFlags; - cc->filterString = filterString; - cc->colType.colDataType = colType.colDataType; - cc->colType.compressionType = colType.compressionType; - cc->colType.colWidth = colType.colWidth; - cc->colType.charsetNumber = colType.charsetNumber; - cc->BOP = BOP; - cc->filterCount = filterCount; - cc->fFilterFeeder = fFilterFeeder; - cc->parsedColumnFilter = parsedColumnFilter; - cc->suppressFilter = suppressFilter; - cc->lastLbid = lastLbid; - cc->r = r; - cc->rowSize = rowSize; - cc->Command::operator=(*this); + cc->_isScan = _isScan; + cc->traceFlags = traceFlags; + cc->filterString = filterString; + cc->colType.colDataType = colType.colDataType; + cc->colType.compressionType = colType.compressionType; + cc->colType.colWidth = colType.colWidth; + cc->colType.charsetNumber = colType.charsetNumber; + cc->BOP = BOP; + cc->filterCount = filterCount; + cc->fFilterFeeder = fFilterFeeder; + cc->parsedColumnFilter = parsedColumnFilter; + cc->suppressFilter = suppressFilter; + cc->lastLbid = lastLbid; + cc->r = r; + cc->rowSize = rowSize; + cc->Command::operator=(*this); } SCommand ColumnCommand::duplicate() { - SCommand ret; + SCommand ret; - ret.reset(new ColumnCommand()); - duplicate((ColumnCommand*) ret.get()); - return ret; + ret.reset(new ColumnCommand()); + duplicate((ColumnCommand*)ret.get()); + return ret; } bool ColumnCommand::operator==(const ColumnCommand& cc) const { - if (_isScan != cc._isScan) - return false; + if (_isScan != cc._isScan) + return false; - if (BOP != cc.BOP) - return false; + if (BOP != cc.BOP) + return false; - if (filterString != cc.filterString) - return false; + if (filterString != cc.filterString) + return false; - if (filterCount != cc.filterCount) - return false; + if (filterCount != cc.filterCount) + return false; - if (makeAbsRids != cc.makeAbsRids) - return false; + if (makeAbsRids != cc.makeAbsRids) + return false; - if (colType.colWidth != cc.colType.colWidth) - return false; + if (colType.colWidth != cc.colType.colWidth) + return false; - if (colType.colDataType != cc.colType.colDataType) - return false; + if (colType.colDataType != cc.colType.colDataType) + return false; - return true; + return true; } bool ColumnCommand::operator!=(const ColumnCommand& cc) const { - return !(*this == cc); + return !(*this == cc); } - ColumnCommand& ColumnCommand::operator=(const ColumnCommand& c) { - _isScan = c._isScan; - traceFlags = c.traceFlags; - filterString = c.filterString; - colType.colDataType = c.colType.colDataType; - colType.compressionType = c.colType.compressionType; - colType.colWidth = c.colType.colWidth; - BOP = c.BOP; - filterCount = c.filterCount; - fFilterFeeder = c.fFilterFeeder; - parsedColumnFilter = c.parsedColumnFilter; - suppressFilter = c.suppressFilter; - lastLbid = c.lastLbid; - return *this; + _isScan = c._isScan; + traceFlags = c.traceFlags; + filterString = c.filterString; + colType.colDataType = c.colType.colDataType; + colType.compressionType = c.colType.compressionType; + colType.colWidth = c.colType.colWidth; + BOP = c.BOP; + filterCount = c.filterCount; + fFilterFeeder = c.fFilterFeeder; + parsedColumnFilter = c.parsedColumnFilter; + suppressFilter = c.suppressFilter; + lastLbid = c.lastLbid; + return *this; } bool ColumnCommand::willPrefetch() { - return (blockCount == 0 || ((double)loadCount) / ((double)blockCount) > - bpp->prefetchThreshold); + return (blockCount == 0 || ((double)loadCount) / ((double)blockCount) > bpp->prefetchThreshold); } void ColumnCommand::disableFilters() { - suppressFilter = true; - prep(primMsg->OutputType, makeAbsRids); + suppressFilter = true; + prep(primMsg->OutputType, makeAbsRids); } void ColumnCommand::enableFilters() { - suppressFilter = false; - prep(primMsg->OutputType, makeAbsRids); + suppressFilter = false; + prep(primMsg->OutputType, makeAbsRids); } void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) { - int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; + int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; - for (i = firstLBID; i <= lastLBID; i++) - lbids->push_back(i); + for (i = firstLBID; i <= lastLBID; i++) + lbids->push_back(i); } int64_t ColumnCommand::getLastLbid() { - if (!_isScan) - return 0; + if (!_isScan) + return 0; - return lastLbid[bpp->dbRoot - 1]; + return lastLbid[bpp->dbRoot - 1]; } ColumnCommand* ColumnCommandFabric::createCommand(messageqcpp::ByteStream& bs) { - bs.advance(1); // The higher dispatcher Command::makeCommand calls BS::peek so this increments BS ptr - execplan::CalpontSystemCatalog::ColType colType; - colType.unserialize(bs); - switch (colType.colWidth) - { - case 1: - return new ColumnCommandInt8(colType, bs); - break; + bs.advance(1); // The higher dispatcher Command::makeCommand calls BS::peek so this increments BS ptr + execplan::CalpontSystemCatalog::ColType colType; + colType.unserialize(bs); + switch (colType.colWidth) + { + case 1: return new ColumnCommandInt8(colType, bs); break; - case 2: - return new ColumnCommandInt16(colType, bs); - break; + case 2: return new ColumnCommandInt16(colType, bs); break; - case 4: - return new ColumnCommandInt32(colType, bs); - break; + case 4: return new ColumnCommandInt32(colType, bs); break; - case 8: - return new ColumnCommandInt64(colType, bs); - break; + case 8: return new ColumnCommandInt64(colType, bs); break; - case 16: - return new ColumnCommandInt128(colType, bs); - break; + case 16: return new ColumnCommandInt128(colType, bs); break; - default: - throw NotImplementedExcept(std::string("ColumnCommandFabric::createCommand: unsupported width ") - + std::to_string(colType.colWidth)); - } + default: + throw NotImplementedExcept(std::string("ColumnCommandFabric::createCommand: unsupported width ") + + std::to_string(colType.colWidth)); + } - return nullptr; + return nullptr; } -template +template void ColumnCommand::createColumnFilter() { - parsedColumnFilter = primitives::_parseColumnFilter(filterString.buf(), - colType.colDataType, - filterCount, BOP); - /* OR hack */ - emptyFilter = primitives::_parseColumnFilter(filterString.buf(), - colType.colDataType, - 0, BOP); + parsedColumnFilter = + primitives::_parseColumnFilter(filterString.buf(), colType.colDataType, filterCount, BOP); + /* OR hack */ + emptyFilter = primitives::_parseColumnFilter(filterString.buf(), colType.colDataType, 0, BOP); } ColumnCommand* ColumnCommandFabric::duplicate(const ColumnCommandUniquePtr& rhs) { - auto & command = *rhs; - if (LIKELY(typeid(command) == typeid(ColumnCommandInt64))) - { - ColumnCommandInt64* ret = new ColumnCommandInt64(); - *ret = *dynamic_cast(rhs.get()); - return ret; - } - else if (typeid(command) == typeid(ColumnCommandInt128)) - { - ColumnCommandInt128* ret = new ColumnCommandInt128(); - *ret = *dynamic_cast(rhs.get()); - return ret; - } - else if (typeid(command) == typeid(ColumnCommandInt8)) - { - ColumnCommandInt8* ret = new ColumnCommandInt8(); - *ret = *dynamic_cast(rhs.get()); - return ret; - } - else if (typeid(command) == typeid(ColumnCommandInt16)) - { - ColumnCommandInt16* ret = new ColumnCommandInt16(); - *ret = *dynamic_cast(rhs.get()); - return ret; - } - else if (typeid(command) == typeid(ColumnCommandInt32)) - { - ColumnCommandInt32* ret = new ColumnCommandInt32(); - *ret = *dynamic_cast(rhs.get()); - return ret; - } - else - throw NotImplementedExcept("ColumnCommandFabric::duplicate: Can not detect ColumnCommand child class"); + auto& command = *rhs; + if (LIKELY(typeid(command) == typeid(ColumnCommandInt64))) + { + ColumnCommandInt64* ret = new ColumnCommandInt64(); + *ret = *dynamic_cast(rhs.get()); + return ret; + } + else if (typeid(command) == typeid(ColumnCommandInt128)) + { + ColumnCommandInt128* ret = new ColumnCommandInt128(); + *ret = *dynamic_cast(rhs.get()); + return ret; + } + else if (typeid(command) == typeid(ColumnCommandInt8)) + { + ColumnCommandInt8* ret = new ColumnCommandInt8(); + *ret = *dynamic_cast(rhs.get()); + return ret; + } + else if (typeid(command) == typeid(ColumnCommandInt16)) + { + ColumnCommandInt16* ret = new ColumnCommandInt16(); + *ret = *dynamic_cast(rhs.get()); + return ret; + } + else if (typeid(command) == typeid(ColumnCommandInt32)) + { + ColumnCommandInt32* ret = new ColumnCommandInt32(); + *ret = *dynamic_cast(rhs.get()); + return ret; + } + else + throw NotImplementedExcept("ColumnCommandFabric::duplicate: Can not detect ColumnCommand child class"); - return nullptr; + return nullptr; } // Code duplication here for the future patch. -ColumnCommandInt8::ColumnCommandInt8(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream& bs) +ColumnCommandInt8::ColumnCommandInt8(execplan::CalpontSystemCatalog::ColType& aColType, + messageqcpp::ByteStream& bs) { - ColumnCommand::createCommand(aColType, bs); - createColumnFilter(); + ColumnCommand::createCommand(aColType, bs); + createColumnFilter(); } void ColumnCommandInt8::prep(int8_t outputType, bool absRids) { - shift = 16; - mask = 0xFFFF; - ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); + shift = 16; + mask = 0xFFFF; + ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); } void ColumnCommandInt8::loadData() { - _loadData(); + _loadData(); } void ColumnCommandInt8::process_OT_BOTH() { - if (makeAbsRids) - _process_OT_BOTH_wAbsRids(); - else - _process_OT_BOTH(); + if (makeAbsRids) + _process_OT_BOTH_wAbsRids(); + else + _process_OT_BOTH(); } void ColumnCommandInt8::process_OT_DATAVALUE() { - _process_OT_DATAVALUE(); + _process_OT_DATAVALUE(); } void ColumnCommandInt8::projectResultRG(RowGroup& rg, uint32_t pos) { - _projectResultRG(rg, pos); + _projectResultRG(rg, pos); } void ColumnCommandInt8::issuePrimitive() { - _issuePrimitiveNarrow(); + _issuePrimitiveNarrow(); } -ColumnCommandInt16::ColumnCommandInt16(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream& bs) +ColumnCommandInt16::ColumnCommandInt16(execplan::CalpontSystemCatalog::ColType& aColType, + messageqcpp::ByteStream& bs) { - ColumnCommand::createCommand(aColType, bs); - createColumnFilter(); + ColumnCommand::createCommand(aColType, bs); + createColumnFilter(); } void ColumnCommandInt16::prep(int8_t outputType, bool absRids) { - shift = 8; - mask = 0xFF; - ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); + shift = 8; + mask = 0xFF; + ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); } void ColumnCommandInt16::loadData() { - _loadData(); + _loadData(); } void ColumnCommandInt16::process_OT_BOTH() { - if (makeAbsRids) - _process_OT_BOTH_wAbsRids(); - else - _process_OT_BOTH(); + if (makeAbsRids) + _process_OT_BOTH_wAbsRids(); + else + _process_OT_BOTH(); } void ColumnCommandInt16::process_OT_DATAVALUE() { - _process_OT_DATAVALUE(); + _process_OT_DATAVALUE(); } void ColumnCommandInt16::projectResultRG(RowGroup& rg, uint32_t pos) { - _projectResultRG(rg, pos); + _projectResultRG(rg, pos); } void ColumnCommandInt16::issuePrimitive() { - _issuePrimitiveNarrow(); + _issuePrimitiveNarrow(); } -ColumnCommandInt32::ColumnCommandInt32(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream& bs) +ColumnCommandInt32::ColumnCommandInt32(execplan::CalpontSystemCatalog::ColType& aColType, + messageqcpp::ByteStream& bs) { - ColumnCommand::createCommand(aColType, bs); - createColumnFilter(); + ColumnCommand::createCommand(aColType, bs); + createColumnFilter(); } void ColumnCommandInt32::prep(int8_t outputType, bool absRids) { - shift = 4; - mask = 0x0F; - ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); + shift = 4; + mask = 0x0F; + ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); } void ColumnCommandInt32::loadData() { - _loadData(); + _loadData(); } void ColumnCommandInt32::process_OT_BOTH() { - if (makeAbsRids) - _process_OT_BOTH_wAbsRids(); - else - _process_OT_BOTH(); + if (makeAbsRids) + _process_OT_BOTH_wAbsRids(); + else + _process_OT_BOTH(); } void ColumnCommandInt32::process_OT_DATAVALUE() { - _process_OT_DATAVALUE(); + _process_OT_DATAVALUE(); } void ColumnCommandInt32::projectResultRG(RowGroup& rg, uint32_t pos) { - _projectResultRG(rg, pos); + _projectResultRG(rg, pos); } void ColumnCommandInt32::issuePrimitive() { - _issuePrimitiveNarrow(); + _issuePrimitiveNarrow(); } -ColumnCommandInt64::ColumnCommandInt64(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream& bs) +ColumnCommandInt64::ColumnCommandInt64(execplan::CalpontSystemCatalog::ColType& aColType, + messageqcpp::ByteStream& bs) { - ColumnCommand::createCommand(aColType, bs); - createColumnFilter(); + ColumnCommand::createCommand(aColType, bs); + createColumnFilter(); } void ColumnCommandInt64::prep(int8_t outputType, bool absRids) { - shift = 2; - mask = 0x03; - ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); + shift = 2; + mask = 0x03; + ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); } void ColumnCommandInt64::loadData() { - _loadData(); + _loadData(); } void ColumnCommandInt64::process_OT_BOTH() { - if (makeAbsRids) - _process_OT_BOTH_wAbsRids(); - else - _process_OT_BOTH(); + if (makeAbsRids) + _process_OT_BOTH_wAbsRids(); + else + _process_OT_BOTH(); } void ColumnCommandInt64::process_OT_DATAVALUE() { - _process_OT_DATAVALUE(); + _process_OT_DATAVALUE(); } void ColumnCommandInt64::projectResultRG(RowGroup& rg, uint32_t pos) { - _projectResultRG(rg, pos); + _projectResultRG(rg, pos); } void ColumnCommandInt64::issuePrimitive() { - _issuePrimitiveNarrow(); + _issuePrimitiveNarrow(); } -ColumnCommandInt128::ColumnCommandInt128(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream& bs) +ColumnCommandInt128::ColumnCommandInt128(execplan::CalpontSystemCatalog::ColType& aColType, + messageqcpp::ByteStream& bs) { - ColumnCommand::createCommand(aColType, bs); - createColumnFilter(); + ColumnCommand::createCommand(aColType, bs); + createColumnFilter(); } void ColumnCommandInt128::prep(int8_t outputType, bool absRids) { - shift = 1; - mask = 0x01; - ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); + shift = 1; + mask = 0x01; + ColumnCommand::fillInPrimitiveMessageHeader(outputType, absRids); } void ColumnCommandInt128::loadData() { - _loadData(); + _loadData(); } void ColumnCommandInt128::process_OT_BOTH() { - if (makeAbsRids) - _process_OT_BOTH_wAbsRids(); - else - _process_OT_BOTH(); + if (makeAbsRids) + _process_OT_BOTH_wAbsRids(); + else + _process_OT_BOTH(); } void ColumnCommandInt128::process_OT_DATAVALUE() { - _process_OT_DATAVALUE(); + _process_OT_DATAVALUE(); } void ColumnCommandInt128::projectResultRG(RowGroup& rg, uint32_t pos) { - _projectResultRG(rg, pos); + _projectResultRG(rg, pos); } void ColumnCommandInt128::issuePrimitive() { - loadData(); + loadData(); - if (!suppressFilter) - bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); - else - bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); + if (!suppressFilter) + bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); + else + bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); - _issuePrimitive(); - if (_isScan) - updateCPDataWide(); + _issuePrimitive(); + if (_isScan) + updateCPDataWide(); } -} +} // namespace primitiveprocessor // vim:ts=4 sw=4: diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index 91e4ee1da..b3ec49d71 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -34,288 +34,278 @@ #include "command.h" #include "calpontsystemcatalog.h" - namespace primitiveprocessor { - // Warning. As of 6.1.1 ColumnCommand has some code duplication. // There are number of derived classes specialized by column width. // There are also legacy generic CC methods used by PseudoCC. class ColumnCommand : public Command { -public: - ColumnCommand(); - ColumnCommand(execplan::CalpontSystemCatalog::ColType& aColType); - virtual ~ColumnCommand(); + public: + ColumnCommand(); + ColumnCommand(execplan::CalpontSystemCatalog::ColType& aColType); + virtual ~ColumnCommand(); - inline uint64_t getLBID() - { - return lbid; - } - inline uint8_t getWidth() - { - return colType.colWidth; - } - inline uint8_t getScale() - { - return colType.scale; - } - uint16_t getFilterCount() - { - return filterCount; - } - const execplan::CalpontSystemCatalog::ColType& getColType() - { - return colType; - } + inline uint64_t getLBID() + { + return lbid; + } + inline uint8_t getWidth() + { + return colType.colWidth; + } + inline uint8_t getScale() + { + return colType.scale; + } + uint16_t getFilterCount() + { + return filterCount; + } + const execplan::CalpontSystemCatalog::ColType& getColType() + { + return colType; + } - void execute(); - void execute(int64_t* vals); //used by RTSCommand to redirect values - virtual void prep(int8_t outputType, bool absRids); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos); - void nextLBID(); - bool isScan() - { - return _isScan; - } - void createCommand(messageqcpp::ByteStream&); - void createCommand(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - void setMakeAbsRids(bool m) - { - makeAbsRids = m; - } - bool willPrefetch(); - int64_t getLastLbid(); - void getLBIDList(uint32_t loopCount, std::vector* lbids); + void execute(); + void execute(int64_t* vals); // used by RTSCommand to redirect values + virtual void prep(int8_t outputType, bool absRids); + void project(); + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos); + void nextLBID(); + bool isScan() + { + return _isScan; + } + void createCommand(messageqcpp::ByteStream&); + void createCommand(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream&); + void resetCommand(messageqcpp::ByteStream&); + void setMakeAbsRids(bool m) + { + makeAbsRids = m; + } + bool willPrefetch(); + int64_t getLastLbid(); + void getLBIDList(uint32_t loopCount, std::vector* lbids); - virtual SCommand duplicate(); - bool operator==(const ColumnCommand&) const; - bool operator!=(const ColumnCommand&) const; + virtual SCommand duplicate(); + bool operator==(const ColumnCommand&) const; + bool operator!=(const ColumnCommand&) const; - /* OR hacks */ - void setScan(bool b) - { - _isScan = b; - } - void disableFilters(); - void enableFilters(); + /* OR hacks */ + void setScan(bool b) + { + _isScan = b; + } + void disableFilters(); + void enableFilters(); - int getCompType() const - { - return colType.compressionType; - } + int getCompType() const + { + return colType.compressionType; + } -protected: - virtual void loadData(); - template - void _loadData(); - void updateCPDataNarrow(); - void updateCPDataWide(); - template - void _issuePrimitiveNarrow(); - template - void _issuePrimitive(); - virtual void issuePrimitive(); - void duplicate(ColumnCommand*); - void fillInPrimitiveMessageHeader(const int8_t outputType, const bool absRids); - template - void createColumnFilter(); + protected: + virtual void loadData(); + template + void _loadData(); + void updateCPDataNarrow(); + void updateCPDataWide(); + template + void _issuePrimitiveNarrow(); + template + void _issuePrimitive(); + virtual void issuePrimitive(); + void duplicate(ColumnCommand*); + void fillInPrimitiveMessageHeader(const int8_t outputType, const bool absRids); + template + void createColumnFilter(); - // we only care about the width and type fields. - //On the PM the rest is uninitialized - execplan::CalpontSystemCatalog::ColType colType; + // we only care about the width and type fields. + // On the PM the rest is uninitialized + execplan::CalpontSystemCatalog::ColType colType; - ColumnCommand(const ColumnCommand&); - ColumnCommand& operator=(const ColumnCommand&); + ColumnCommand(const ColumnCommand&); + ColumnCommand& operator=(const ColumnCommand&); - void _execute(); - void processResult(); - template - void _process_OT_BOTH(); - template - void _process_OT_BOTH_wAbsRids(); - virtual void process_OT_BOTH(); - void process_OT_RID(); - virtual void process_OT_DATAVALUE(); - template - void _process_OT_DATAVALUE(); - void process_OT_ROWGROUP(); - void projectResult(); - template - void _projectResultRGLoop(rowgroup::Row& r, - const T* valuesArray, - const uint32_t offset); - template - void _projectResultRG(rowgroup::RowGroup& rg, uint32_t pos); - virtual void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos); - void removeRowsFromRowGroup(rowgroup::RowGroup&); - void makeScanMsg(); - void makeStepMsg(); - void setLBID(uint64_t rid); - template - inline void fillEmptyBlock(uint8_t* dst, - const uint8_t*emptyValue, - const uint32_t number) const; + void _execute(); + void processResult(); + template + void _process_OT_BOTH(); + template + void _process_OT_BOTH_wAbsRids(); + virtual void process_OT_BOTH(); + void process_OT_RID(); + virtual void process_OT_DATAVALUE(); + template + void _process_OT_DATAVALUE(); + void process_OT_ROWGROUP(); + void projectResult(); + template + void _projectResultRGLoop(rowgroup::Row& r, const T* valuesArray, const uint32_t offset); + template + void _projectResultRG(rowgroup::RowGroup& rg, uint32_t pos); + virtual void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos); + void removeRowsFromRowGroup(rowgroup::RowGroup&); + void makeScanMsg(); + void makeStepMsg(); + void setLBID(uint64_t rid); + template + inline void fillEmptyBlock(uint8_t* dst, const uint8_t* emptyValue, const uint32_t number) const; - bool _isScan; + bool _isScan; - boost::scoped_array inputMsg; - NewColRequestHeader* primMsg; - ColResultHeader* outMsg; + boost::scoped_array inputMsg; + NewColRequestHeader* primMsg; + ColResultHeader* outMsg; - // the length of base prim msg, which is everything up to the - // rid array for the pCol message - uint32_t baseMsgLength; + // the length of base prim msg, which is everything up to the + // rid array for the pCol message + uint32_t baseMsgLength; - uint64_t lbid; - uint32_t traceFlags; // probably move this to Command - uint8_t BOP; - messageqcpp::ByteStream filterString; - uint16_t filterCount; - bool makeAbsRids; - int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container - int128_t* wide128Values; + uint64_t lbid; + uint32_t traceFlags; // probably move this to Command + uint8_t BOP; + messageqcpp::ByteStream filterString; + uint16_t filterCount; + bool makeAbsRids; + int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container + int128_t* wide128Values; - uint16_t mask, shift; // vars for the selective block loader + uint16_t mask, shift; // vars for the selective block loader - // counters to decide whether to prefetch or not - uint32_t blockCount, loadCount; + // counters to decide whether to prefetch or not + uint32_t blockCount, loadCount; - boost::shared_ptr parsedColumnFilter; + boost::shared_ptr parsedColumnFilter; - /* OR hacks */ - boost::shared_ptr emptyFilter; - bool suppressFilter; + /* OR hacks */ + boost::shared_ptr emptyFilter; + bool suppressFilter; - std::vector lastLbid; + std::vector lastLbid; - /* speculative optimizations for projectintorowgroup() */ - rowgroup::Row r; - uint32_t rowSize; + /* speculative optimizations for projectintorowgroup() */ + rowgroup::Row r; + uint32_t rowSize; - bool wasVersioned; + bool wasVersioned; - friend class RTSCommand; + friend class RTSCommand; }; using ColumnCommandUniquePtr = std::unique_ptr; class ColumnCommandInt8 : public ColumnCommand { - public: - static constexpr uint8_t size = 1; - using IntegralType = datatypes::WidthToSIntegralType::type; - ColumnCommandInt8() : ColumnCommand() { }; - ColumnCommandInt8(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); - void prep(int8_t outputType, bool absRids) override; - void loadData() override; - void process_OT_BOTH() override; - void process_OT_DATAVALUE() override; - void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; - void issuePrimitive() override; + public: + static constexpr uint8_t size = 1; + using IntegralType = datatypes::WidthToSIntegralType::type; + ColumnCommandInt8() : ColumnCommand(){}; + ColumnCommandInt8(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); + void prep(int8_t outputType, bool absRids) override; + void loadData() override; + void process_OT_BOTH() override; + void process_OT_DATAVALUE() override; + void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; + void issuePrimitive() override; }; class ColumnCommandInt16 : public ColumnCommand { - public: - static constexpr uint8_t size = 2; - using IntegralType = datatypes::WidthToSIntegralType::type; - ColumnCommandInt16() : ColumnCommand() { }; - ColumnCommandInt16(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); - void prep(int8_t outputType, bool absRids) override; - void loadData() override; - void process_OT_BOTH() override; - void process_OT_DATAVALUE() override; - void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; - void issuePrimitive() override; + public: + static constexpr uint8_t size = 2; + using IntegralType = datatypes::WidthToSIntegralType::type; + ColumnCommandInt16() : ColumnCommand(){}; + ColumnCommandInt16(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); + void prep(int8_t outputType, bool absRids) override; + void loadData() override; + void process_OT_BOTH() override; + void process_OT_DATAVALUE() override; + void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; + void issuePrimitive() override; }; class ColumnCommandInt32 : public ColumnCommand { - public: - static constexpr uint8_t size = 4; - using IntegralType = datatypes::WidthToSIntegralType::type; - ColumnCommandInt32() : ColumnCommand() { }; - ColumnCommandInt32(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); - void prep(int8_t outputType, bool absRids) override; - void loadData() override; - void process_OT_BOTH() override; - void process_OT_DATAVALUE() override; - void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; - void issuePrimitive() override; + public: + static constexpr uint8_t size = 4; + using IntegralType = datatypes::WidthToSIntegralType::type; + ColumnCommandInt32() : ColumnCommand(){}; + ColumnCommandInt32(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); + void prep(int8_t outputType, bool absRids) override; + void loadData() override; + void process_OT_BOTH() override; + void process_OT_DATAVALUE() override; + void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; + void issuePrimitive() override; }; class ColumnCommandInt64 : public ColumnCommand { - public: - static constexpr uint8_t size = 8; - using IntegralType = datatypes::WidthToSIntegralType::type; - ColumnCommandInt64() : ColumnCommand() { }; - ColumnCommandInt64(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); - void prep(int8_t outputType, bool absRids) override; - void loadData() override; - void process_OT_BOTH() override; - void process_OT_DATAVALUE() override; - void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; - void issuePrimitive() override; + public: + static constexpr uint8_t size = 8; + using IntegralType = datatypes::WidthToSIntegralType::type; + ColumnCommandInt64() : ColumnCommand(){}; + ColumnCommandInt64(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); + void prep(int8_t outputType, bool absRids) override; + void loadData() override; + void process_OT_BOTH() override; + void process_OT_DATAVALUE() override; + void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; + void issuePrimitive() override; }; class ColumnCommandInt128 : public ColumnCommand { - public: - static constexpr uint8_t size = 16; - using IntegralType = datatypes::WidthToSIntegralType::type; - ColumnCommandInt128() : ColumnCommand() { }; - ColumnCommandInt128(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); - void prep(int8_t outputType, bool absRids) override; - void loadData() override; - void process_OT_BOTH() override; - void process_OT_DATAVALUE() override; - void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; - void issuePrimitive() override; + public: + static constexpr uint8_t size = 16; + using IntegralType = datatypes::WidthToSIntegralType::type; + ColumnCommandInt128() : ColumnCommand(){}; + ColumnCommandInt128(execplan::CalpontSystemCatalog::ColType& colType, messageqcpp::ByteStream& bs); + void prep(int8_t outputType, bool absRids) override; + void loadData() override; + void process_OT_BOTH() override; + void process_OT_DATAVALUE() override; + void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos) override; + void issuePrimitive() override; }; - class ColumnCommandFabric { - public: - ColumnCommandFabric() = default; - static ColumnCommand* createCommand(messageqcpp::ByteStream& bs); - static ColumnCommand* duplicate(const ColumnCommandUniquePtr& rhs); + public: + ColumnCommandFabric() = default; + static ColumnCommand* createCommand(messageqcpp::ByteStream& bs); + static ColumnCommand* duplicate(const ColumnCommandUniquePtr& rhs); }; -template -inline void ColumnCommand::fillEmptyBlock(uint8_t* dst, - const uint8_t*emptyValue, - const uint32_t number) const +template +inline void ColumnCommand::fillEmptyBlock(uint8_t* dst, const uint8_t* emptyValue, + const uint32_t number) const { - T* typedDst = reinterpret_cast(dst); - const T* typedEmptyValue = reinterpret_cast(emptyValue); - for (uint32_t idx = 0; idx < number; idx = idx + 4) - { - typedDst[idx] = *typedEmptyValue; - typedDst[idx+1] = *typedEmptyValue; - typedDst[idx+2] = *typedEmptyValue; - typedDst[idx+3] = *typedEmptyValue; - } + T* typedDst = reinterpret_cast(dst); + const T* typedEmptyValue = reinterpret_cast(emptyValue); + for (uint32_t idx = 0; idx < number; idx = idx + 4) + { + typedDst[idx] = *typedEmptyValue; + typedDst[idx + 1] = *typedEmptyValue; + typedDst[idx + 2] = *typedEmptyValue; + typedDst[idx + 3] = *typedEmptyValue; + } } -template<> +template <> inline void ColumnCommand::fillEmptyBlock(uint8_t* dst, - const uint8_t*emptyValue, - const uint32_t number) const + const uint8_t* emptyValue, + const uint32_t number) const { - for (uint32_t idx = 0; idx < number; idx++) - { - datatypes::TSInt128::assignPtrPtr(dst + idx * sizeof(messageqcpp::ByteStream::hexbyte), - emptyValue); - } + for (uint32_t idx = 0; idx < number; idx++) + { + datatypes::TSInt128::assignPtrPtr(dst + idx * sizeof(messageqcpp::ByteStream::hexbyte), emptyValue); + } } -} // namespace +} // namespace primitiveprocessor #endif // vim:ts=4 sw=4: - diff --git a/primitives/primproc/command.cpp b/primitives/primproc/command.cpp index 54432d277..ab1bc18b8 100644 --- a/primitives/primproc/command.cpp +++ b/primitives/primproc/command.cpp @@ -25,140 +25,130 @@ using namespace messageqcpp; namespace primitiveprocessor { +Command::Command(CommandType c) : cmdType(c), fFilterFeeder(NOT_FEEDER) +{ +} -Command::Command(CommandType c) : cmdType(c), fFilterFeeder(NOT_FEEDER) { } - -Command::~Command() { }; +Command::~Command(){}; void Command::createCommand(ByteStream& bs) { - bs >> OID; - bs >> tupleKey; - bs >> queryUuid; - bs >> stepUuid; + bs >> OID; + bs >> tupleKey; + bs >> queryUuid; + bs >> stepUuid; } -void Command::resetCommand(ByteStream& bs) { }; +void Command::resetCommand(ByteStream& bs){}; -void Command::setMakeAbsRids(bool) { } +void Command::setMakeAbsRids(bool) +{ +} Command* Command::makeCommand(ByteStream& bs, CommandType* type, vector& cmds) { - Command* ret; - uint8_t tmp8; + Command* ret; + uint8_t tmp8; - bs.peek(tmp8); - *type = (CommandType) tmp8; + bs.peek(tmp8); + *type = (CommandType)tmp8; - switch (*type) - { - case COLUMN_COMMAND: - return ColumnCommandFabric::createCommand(bs); - break; + switch (*type) + { + case COLUMN_COMMAND: return ColumnCommandFabric::createCommand(bs); break; - case DICT_STEP: - ret = new DictStep(); - break; + case DICT_STEP: ret = new DictStep(); break; - case PASS_THRU: - ret = new PassThruCommand(); - break; + case PASS_THRU: ret = new PassThruCommand(); break; - case RID_TO_STRING: - ret = new RTSCommand(); - break; + case RID_TO_STRING: ret = new RTSCommand(); break; - case FILTER_COMMAND: - ret = FilterCommand::makeFilterCommand(bs, cmds); - break; + case FILTER_COMMAND: ret = FilterCommand::makeFilterCommand(bs, cmds); break; - case PSEUDOCOLUMN: - ret = new PseudoCC(); - break; + case PSEUDOCOLUMN: ret = new PseudoCC(); break; - default: - throw logic_error("Command::makeCommand(): can't deserialize this bytestream"); - }; + default: throw logic_error("Command::makeCommand(): can't deserialize this bytestream"); + }; - ret->createCommand(bs); + ret->createCommand(bs); - return ret; + return ret; } void Command::setBatchPrimitiveProcessor(BatchPrimitiveProcessor* b) { - bpp = b; + bpp = b; } void Command::copyRidsForFilterCmd() { - if (fFilterFeeder == LEFT_FEEDER) - { - bpp->fFiltRidCount[0] = bpp->ridCount; + if (fFilterFeeder == LEFT_FEEDER) + { + bpp->fFiltRidCount[0] = bpp->ridCount; - for (uint64_t i = 0; i < bpp->ridCount; i++) - bpp->fFiltCmdRids[0][i] = bpp->relRids[i]; - } - else // if (fFilterFeeder == RIGHT_FEEDER) - { - bpp->fFiltRidCount[1] = bpp->ridCount; + for (uint64_t i = 0; i < bpp->ridCount; i++) + bpp->fFiltCmdRids[0][i] = bpp->relRids[i]; + } + else // if (fFilterFeeder == RIGHT_FEEDER) + { + bpp->fFiltRidCount[1] = bpp->ridCount; - for (uint64_t i = 0; i < bpp->ridCount; i++) - bpp->fFiltCmdRids[1][i] = bpp->relRids[i]; - } + for (uint64_t i = 0; i < bpp->ridCount; i++) + bpp->fFiltCmdRids[1][i] = bpp->relRids[i]; + } } bool Command::operator==(const Command& c) const { - const type_info& cType = typeid(c); + const type_info& cType = typeid(c); - if (cType != typeid(*this)) - return false; + if (cType != typeid(*this)) + return false; - if (cType == typeid(ColumnCommand)) - { - const ColumnCommand* cc = dynamic_cast(&c); - const ColumnCommand* t = dynamic_cast(this); + if (cType == typeid(ColumnCommand)) + { + const ColumnCommand* cc = dynamic_cast(&c); + const ColumnCommand* t = dynamic_cast(this); - if (*cc != *t) - return false; - } - else if (cType == typeid(DictStep)) - { - const DictStep* ds = dynamic_cast(&c); - const DictStep* t = dynamic_cast(this); + if (*cc != *t) + return false; + } + else if (cType == typeid(DictStep)) + { + const DictStep* ds = dynamic_cast(&c); + const DictStep* t = dynamic_cast(this); - if (*ds != *t) - return false; - } - else if (cType == typeid(PassThruCommand)) - { - const PassThruCommand* pt = dynamic_cast(&c); - const PassThruCommand* t = dynamic_cast(this); + if (*ds != *t) + return false; + } + else if (cType == typeid(PassThruCommand)) + { + const PassThruCommand* pt = dynamic_cast(&c); + const PassThruCommand* t = dynamic_cast(this); - if (*pt != *t) - return false; - } - else if (cType == typeid(RTSCommand)) - { - const RTSCommand* rts = dynamic_cast(&c); - const RTSCommand* t = dynamic_cast(this); + if (*pt != *t) + return false; + } + else if (cType == typeid(RTSCommand)) + { + const RTSCommand* rts = dynamic_cast(&c); + const RTSCommand* t = dynamic_cast(this); - if (*rts != *t) - return false; - } - else if (cType == typeid(FilterCommand)) - { - const FilterCommand* fc = dynamic_cast(&c); - const FilterCommand* t = dynamic_cast(this); + if (*rts != *t) + return false; + } + else if (cType == typeid(FilterCommand)) + { + const FilterCommand* fc = dynamic_cast(&c); + const FilterCommand* t = dynamic_cast(this); - if (*fc != *t) - return false; - } - else - cerr << "unknown Command type\n"; + if (*fc != *t) + return false; + } + else + cerr << "unknown Command type\n"; - return true; + return true; } -} +} // namespace primitiveprocessor diff --git a/primitives/primproc/command.h b/primitives/primproc/command.h index 64a77e0c5..bf00da7ac 100644 --- a/primitives/primproc/command.h +++ b/primitives/primproc/command.h @@ -27,100 +27,101 @@ namespace primitiveprocessor { - class BatchPrimitiveProcessor; class Command; typedef boost::shared_ptr SCommand; class Command { -public: - enum CommandType - { - NONE, - COLUMN_COMMAND, - DICT_STEP, - DICT_SCAN, - PASS_THRU, - RID_TO_STRING, - FILTER_COMMAND, - PSEUDOCOLUMN - }; + public: + enum CommandType + { + NONE, + COLUMN_COMMAND, + DICT_STEP, + DICT_SCAN, + PASS_THRU, + RID_TO_STRING, + FILTER_COMMAND, + PSEUDOCOLUMN + }; - static const uint8_t NOT_FEEDER = 0; - static const uint8_t FILT_FEEDER = 1; - static const uint8_t LEFT_FEEDER = 3; - static const uint8_t RIGHT_FEEDER = 5; + static const uint8_t NOT_FEEDER = 0; + static const uint8_t FILT_FEEDER = 1; + static const uint8_t LEFT_FEEDER = 3; + static const uint8_t RIGHT_FEEDER = 5; - Command(CommandType c); - virtual ~Command(); + Command(CommandType c); + virtual ~Command(); - virtual void execute() = 0; - virtual void project() = 0; - virtual void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t columnPosition) = 0; - virtual uint64_t getLBID() = 0; - virtual void getLBIDList(uint32_t loopCount, std::vector* out) {} // the default fcn returns 0 lbids - virtual void nextLBID() = 0; - virtual void createCommand(messageqcpp::ByteStream&); - virtual void resetCommand(messageqcpp::ByteStream&); + virtual void execute() = 0; + virtual void project() = 0; + virtual void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t columnPosition) = 0; + virtual uint64_t getLBID() = 0; + virtual void getLBIDList(uint32_t loopCount, std::vector* out) + { + } // the default fcn returns 0 lbids + virtual void nextLBID() = 0; + virtual void createCommand(messageqcpp::ByteStream&); + virtual void resetCommand(messageqcpp::ByteStream&); - /* Duplicate() makes a copy of this object as constructed by createCommand(). - It's thread-safe */ - virtual SCommand duplicate() = 0; - bool operator==(const Command&) const; - bool operator!=(const Command& c) const - { - return !(*this == c); - } + /* Duplicate() makes a copy of this object as constructed by createCommand(). + It's thread-safe */ + virtual SCommand duplicate() = 0; + bool operator==(const Command&) const; + bool operator!=(const Command& c) const + { + return !(*this == c); + } - /* Put bootstrap code here (ie, build the template primitive msg) */ - virtual void prep(int8_t outputType, bool makeAbsRids) = 0; - virtual void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*); - virtual void setMakeAbsRids(bool); + /* Put bootstrap code here (ie, build the template primitive msg) */ + virtual void prep(int8_t outputType, bool makeAbsRids) = 0; + virtual void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*); + virtual void setMakeAbsRids(bool); - CommandType getCommandType() const - { - return cmdType; - } + CommandType getCommandType() const + { + return cmdType; + } - // if feeding a filtercommand - // note: a filtercommand itself can feed another filtercommand - uint8_t filterFeeder() - { - return fFilterFeeder; - } - void filterFeeder(uint8_t f) - { - fFilterFeeder = f; - } - virtual void copyRidsForFilterCmd(); + // if feeding a filtercommand + // note: a filtercommand itself can feed another filtercommand + uint8_t filterFeeder() + { + return fFilterFeeder; + } + void filterFeeder(uint8_t f) + { + fFilterFeeder = f; + } + virtual void copyRidsForFilterCmd(); - static Command* makeCommand(messageqcpp::ByteStream&, CommandType*, std::vector&); + static Command* makeCommand(messageqcpp::ByteStream&, CommandType*, std::vector&); - uint32_t getOID() const - { - return OID; - } - uint32_t getTupleKey() const - { - return tupleKey; - } + uint32_t getOID() const + { + return OID; + } + uint32_t getTupleKey() const + { + return tupleKey; + } - virtual int getCompType() const = 0; + virtual int getCompType() const = 0; -protected: - BatchPrimitiveProcessor* bpp; - CommandType cmdType; - uint8_t fFilterFeeder; - uint32_t OID; - uint32_t tupleKey; - boost::uuids::uuid queryUuid; - boost::uuids::uuid stepUuid; + protected: + BatchPrimitiveProcessor* bpp; + CommandType cmdType; + uint8_t fFilterFeeder; + uint32_t OID; + uint32_t tupleKey; + boost::uuids::uuid queryUuid; + boost::uuids::uuid stepUuid; - Command(const Command &rhs) = default; - Command & operator=(const Command & rhs) = default; + Command(const Command& rhs) = default; + Command& operator=(const Command& rhs) = default; }; -} +} // namespace primitiveprocessor #endif diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index 2aa8566b0..60adfa573 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -42,11 +42,9 @@ using namespace rowgroup; namespace primitiveprocessor { - extern uint32_t dictBufferSize; -DictStep::DictStep() : Command(DICT_STEP), strValues(NULL), filterCount(0), - bufferSize(0) +DictStep::DictStep() : Command(DICT_STEP), strValues(NULL), filterCount(0), bufferSize(0) { } @@ -56,50 +54,50 @@ DictStep::~DictStep() DictStep& DictStep::operator=(const DictStep& d) { - // Not all fields are copied for the sake of efficiency. - // Right now, these only need to copy fields deserialized from the create msg. - BOP = d.BOP; - fFilterFeeder = d.fFilterFeeder; - compressionType = d.compressionType; - filterString = d.filterString; - hasEqFilter = d.hasEqFilter; - eqFilter = d.eqFilter; - eqOp = d.eqOp; - filterCount = d.filterCount; - charsetNumber = d.charsetNumber; - return *this; + // Not all fields are copied for the sake of efficiency. + // Right now, these only need to copy fields deserialized from the create msg. + BOP = d.BOP; + fFilterFeeder = d.fFilterFeeder; + compressionType = d.compressionType; + filterString = d.filterString; + hasEqFilter = d.hasEqFilter; + eqFilter = d.eqFilter; + eqOp = d.eqOp; + filterCount = d.filterCount; + charsetNumber = d.charsetNumber; + return *this; } void DictStep::createCommand(ByteStream& bs) { - uint8_t tmp8; + uint8_t tmp8; - bs.advance(1); - bs >> BOP; - bs >> tmp8; - compressionType = tmp8; - bs >> charsetNumber; - bs >> filterCount; - bs >> tmp8; - hasEqFilter = tmp8; + bs.advance(1); + bs >> BOP; + bs >> tmp8; + compressionType = tmp8; + bs >> charsetNumber; + bs >> filterCount; + bs >> tmp8; + hasEqFilter = tmp8; - if (hasEqFilter) + if (hasEqFilter) + { + string strTmp; + datatypes::Charset cs(charsetNumber); + eqFilter.reset(new primitives::DictEqualityFilter(cs)); + bs >> eqOp; + + for (uint32_t i = 0; i < filterCount; i++) { - string strTmp; - datatypes::Charset cs(charsetNumber); - eqFilter.reset(new primitives::DictEqualityFilter(cs)); - bs >> eqOp; - - for (uint32_t i = 0; i < filterCount; i++) - { - bs >> strTmp; - eqFilter->insert(strTmp); - } + bs >> strTmp; + eqFilter->insert(strTmp); } - else - bs >> filterString; + } + else + bs >> filterString; - Command::createCommand(bs); + Command::createCommand(bs); } void DictStep::resetCommand(ByteStream& bs) @@ -108,177 +106,174 @@ void DictStep::resetCommand(ByteStream& bs) void DictStep::prep(int8_t outputType, bool makeAbsRids) { - // (at most there are 8192 tokens to fetch) - bufferSize = sizeof(DictInput) + filterString.length() + (8192 * sizeof(OldGetSigParams)); - inputMsg.reset(new uint8_t[bufferSize]); - primMsg = (DictInput*) inputMsg.get(); + // (at most there are 8192 tokens to fetch) + bufferSize = sizeof(DictInput) + filterString.length() + (8192 * sizeof(OldGetSigParams)); + inputMsg.reset(new uint8_t[bufferSize]); + primMsg = (DictInput*)inputMsg.get(); - primMsg->ism.Interleave = 0; - primMsg->ism.Flags = 0; -// primMsg->ism.Flags = PrimitiveMsg::planFlagsToPrimFlags(traceFlags); - primMsg->ism.Command = DICT_SIGNATURE; - primMsg->ism.Size = bufferSize; - primMsg->ism.Type = 2; - primMsg->hdr.SessionID = bpp->sessionID; - //primMsg->hdr.StatementID = 0; - primMsg->hdr.TransactionID = bpp->txnID; - primMsg->hdr.VerID = bpp->versionInfo.currentScn; - primMsg->hdr.StepID = bpp->stepID; - primMsg->BOP = BOP; - primMsg->InputFlags = 1; //TODO: Use the new p_Dict functionality - primMsg->OutputType = (eqFilter || filterCount || fFilterFeeder != NOT_FEEDER - ? OT_RID : OT_RID | OT_DATAVALUE); - primMsg->NOPS = (eqFilter ? 0 : filterCount); - primMsg->NVALS = 0; + primMsg->ism.Interleave = 0; + primMsg->ism.Flags = 0; + // primMsg->ism.Flags = PrimitiveMsg::planFlagsToPrimFlags(traceFlags); + primMsg->ism.Command = DICT_SIGNATURE; + primMsg->ism.Size = bufferSize; + primMsg->ism.Type = 2; + primMsg->hdr.SessionID = bpp->sessionID; + // primMsg->hdr.StatementID = 0; + primMsg->hdr.TransactionID = bpp->txnID; + primMsg->hdr.VerID = bpp->versionInfo.currentScn; + primMsg->hdr.StepID = bpp->stepID; + primMsg->BOP = BOP; + primMsg->InputFlags = 1; // TODO: Use the new p_Dict functionality + primMsg->OutputType = + (eqFilter || filterCount || fFilterFeeder != NOT_FEEDER ? OT_RID : OT_RID | OT_DATAVALUE); + primMsg->NOPS = (eqFilter ? 0 : filterCount); + primMsg->NVALS = 0; } void DictStep::issuePrimitive(bool isFilter) { - bool wasCached; - uint32_t blocksRead; + bool wasCached; + uint32_t blocksRead; - if (!(primMsg->LBID & 0x8000000000000000LL)) - { - //std::cerr << "DS issuePrimitive lbid: " << (uint64_t)primMsg->LBID << endl; - primitiveprocessor::loadBlock(primMsg->LBID, - bpp->versionInfo, - bpp->txnID, - compressionType, - bpp->blockData, - &wasCached, - &blocksRead, - bpp->LBIDTrace, - bpp->sessionID); + if (!(primMsg->LBID & 0x8000000000000000LL)) + { + // std::cerr << "DS issuePrimitive lbid: " << (uint64_t)primMsg->LBID << endl; + primitiveprocessor::loadBlock(primMsg->LBID, bpp->versionInfo, bpp->txnID, compressionType, + bpp->blockData, &wasCached, &blocksRead, bpp->LBIDTrace, bpp->sessionID); - if (wasCached) - bpp->cachedIO++; + if (wasCached) + bpp->cachedIO++; - bpp->physIO += blocksRead; - bpp->touchedBlocks++; - } + bpp->physIO += blocksRead; + bpp->touchedBlocks++; + } - bpp->pp.p_Dictionary(primMsg, &result, isFilter, charsetNumber, eqFilter, eqOp); + bpp->pp.p_Dictionary(primMsg, &result, isFilter, charsetNumber, eqFilter, eqOp); } void DictStep::copyResultToTmpSpace(OrderedToken* ot) { - uint32_t i; - uint8_t* pos; - uint16_t len; - uint64_t rid64; - uint16_t rid16; + uint32_t i; + uint8_t* pos; + uint16_t len; + uint64_t rid64; + uint16_t rid16; - idbassert(primMsg->OutputType & OT_RID); - DictOutput* header = (DictOutput*) &result[0]; + idbassert(primMsg->OutputType & OT_RID); + DictOutput* header = (DictOutput*)&result[0]; - if (header->NVALS == 0) return; + if (header->NVALS == 0) + return; - pos = &result[sizeof(DictOutput)]; + pos = &result[sizeof(DictOutput)]; - for (i = 0; i < header->NVALS; i++) + for (i = 0; i < header->NVALS; i++) + { + rid64 = *((uint64_t*)pos); + pos += 8; + rid16 = rid64 & 0x1fff; + ot[rid16].inResult = true; + tmpResultCounter++; + + if (primMsg->OutputType & OT_DATAVALUE) { - rid64 = *((uint64_t*) pos); - pos += 8; - rid16 = rid64 & 0x1fff; - ot[rid16].inResult = true; - tmpResultCounter++; + len = *((uint16_t*)pos); + pos += 2; + ot[rid16].str = string((char*)pos, len); + pos += len; - if (primMsg->OutputType & OT_DATAVALUE) - { - len = *((uint16_t*) pos); - pos += 2; - ot[rid16].str = string((char*) pos, len); - pos += len; - - if (rid64 & 0x8000000000000000LL) - ot[rid16].str = joblist::CPNULLSTRMARK; - } + if (rid64 & 0x8000000000000000LL) + ot[rid16].str = joblist::CPNULLSTRMARK; } + } } void DictStep::copyResultToFinalPosition(OrderedToken* ot) { - uint32_t i, resultPos = 0; + uint32_t i, resultPos = 0; - for (i = 0; i < inputRidCount; i++) + for (i = 0; i < inputRidCount; i++) + { + if (ot[i].inResult) { - if (ot[i].inResult) - { - bpp->absRids[resultPos] = ot[i].rid; - bpp->relRids[resultPos] = ot[i].rid - bpp->baseRid; + bpp->absRids[resultPos] = ot[i].rid; + bpp->relRids[resultPos] = ot[i].rid - bpp->baseRid; - if (primMsg->OutputType & OT_DATAVALUE) - (*strValues)[resultPos] = ot[i].str; + if (primMsg->OutputType & OT_DATAVALUE) + (*strValues)[resultPos] = ot[i].str; - resultPos++; - } + resultPos++; } + } } void DictStep::processResult() { - uint32_t i; - uint8_t* pos; - uint16_t len; - DictOutput* header = (DictOutput*) &result[0]; + uint32_t i; + uint8_t* pos; + uint16_t len; + DictOutput* header = (DictOutput*)&result[0]; - if (header->NVALS == 0) return; + if (header->NVALS == 0) + return; - pos = &result[sizeof(DictOutput)]; + pos = &result[sizeof(DictOutput)]; - for (i = 0; i < header->NVALS; i++, tmpResultCounter++) + for (i = 0; i < header->NVALS; i++, tmpResultCounter++) + { + if (primMsg->OutputType & OT_RID) { - if (primMsg->OutputType & OT_RID) - { - bpp->absRids[tmpResultCounter] = *((uint64_t*) pos); - pos += 8; - //bpp->relRids[tmpResultCounter] = bpp->absRids[tmpResultCounter] & 0x1fff; - bpp->relRids[tmpResultCounter] = bpp->absRids[tmpResultCounter] - bpp->baseRid; - } - - if (primMsg->OutputType & OT_DATAVALUE) - { - len = *((uint16_t*) pos); - pos += 2; - (*strValues)[tmpResultCounter] = string((char*) pos, len); - pos += len; - } - - //cout << " stored " << (*strValues)[tmpResultCounter] << endl; - /* XXXPAT: disclaimer: this is how we do it in DictionaryStep; don't know - if it's necessary or not yet */ - if ((bpp->absRids[tmpResultCounter] & 0x8000000000000000LL) != 0) - { - if (primMsg->OutputType & OT_DATAVALUE) - (*strValues)[tmpResultCounter] = joblist::CPNULLSTRMARK.c_str(); - - bpp->absRids[tmpResultCounter] &= 0x7FFFFFFFFFFFFFFFLL; - } + bpp->absRids[tmpResultCounter] = *((uint64_t*)pos); + pos += 8; + // bpp->relRids[tmpResultCounter] = bpp->absRids[tmpResultCounter] & 0x1fff; + bpp->relRids[tmpResultCounter] = bpp->absRids[tmpResultCounter] - bpp->baseRid; } + + if (primMsg->OutputType & OT_DATAVALUE) + { + len = *((uint16_t*)pos); + pos += 2; + (*strValues)[tmpResultCounter] = string((char*)pos, len); + pos += len; + } + + // cout << " stored " << (*strValues)[tmpResultCounter] << endl; + /* XXXPAT: disclaimer: this is how we do it in DictionaryStep; don't know + if it's necessary or not yet */ + if ((bpp->absRids[tmpResultCounter] & 0x8000000000000000LL) != 0) + { + if (primMsg->OutputType & OT_DATAVALUE) + (*strValues)[tmpResultCounter] = joblist::CPNULLSTRMARK.c_str(); + + bpp->absRids[tmpResultCounter] &= 0x7FFFFFFFFFFFFFFFLL; + } + } } void DictStep::projectResult(string* strings) { - uint32_t i; - uint8_t* pos; - uint16_t len; - DictOutput* header = (DictOutput*) &result[0]; + uint32_t i; + uint8_t* pos; + uint16_t len; + DictOutput* header = (DictOutput*)&result[0]; - if (header->NVALS == 0) return; + if (header->NVALS == 0) + return; - pos = &result[sizeof(DictOutput)]; + pos = &result[sizeof(DictOutput)]; - //cout << "projectResult() l: " << primMsg->LBID << " NVALS: " << header->NVALS << endl; - for (i = 0; i < header->NVALS; i++) - { - len = *((uint16_t*) pos); - pos += 2; - strings[tmpResultCounter++] = string((char*) pos, len); - //cout << "serialized length is " << len << " string is " << strings[tmpResultCounter-1] << " string length = " << - // strings[tmpResultCounter-1].length() << endl; - pos += len; - totalResultLength += len + 4; - } + // cout << "projectResult() l: " << primMsg->LBID << " NVALS: " << header->NVALS << endl; + for (i = 0; i < header->NVALS; i++) + { + len = *((uint16_t*)pos); + pos += 2; + strings[tmpResultCounter++] = string((char*)pos, len); + // cout << "serialized length is " << len << " string is " << strings[tmpResultCounter-1] << " string + // length = " << + // strings[tmpResultCounter-1].length() << endl; + pos += len; + totalResultLength += len + 4; + } } // bug4901 - @@ -292,374 +287,371 @@ void DictStep::projectResult(string* strings) // before starting to serialize strings. void DictStep::projectResult(StringPtr* strings) { - uint32_t i; - uint8_t* pos; - uint16_t len; - DictOutput* header = (DictOutput*) &result[0]; + uint32_t i; + uint8_t* pos; + uint16_t len; + DictOutput* header = (DictOutput*)&result[0]; - if (header->NVALS == 0) return; + if (header->NVALS == 0) + return; - pos = &result[sizeof(DictOutput)]; + pos = &result[sizeof(DictOutput)]; - //cout << "projectResult() l: " << primMsg->LBID << " NVALS: " << header->NVALS << endl; - for (i = 0; i < header->NVALS; i++) - { - len = *((uint16_t*) pos); - pos += 2; - strings[tmpResultCounter++] = StringPtr(pos, len); - //cout << "serialized length is " << len << " string is " << strings[tmpResultCounter-1] << " string length = " << - // strings[tmpResultCounter-1].length() << endl; - pos += len; - totalResultLength += len + 4; - } + // cout << "projectResult() l: " << primMsg->LBID << " NVALS: " << header->NVALS << endl; + for (i = 0; i < header->NVALS; i++) + { + len = *((uint16_t*)pos); + pos += 2; + strings[tmpResultCounter++] = StringPtr(pos, len); + // cout << "serialized length is " << len << " string is " << strings[tmpResultCounter-1] << " string + // length = " << strings[tmpResultCounter-1].length() << endl; + pos += len; + totalResultLength += len + 4; + } } void DictStep::execute() { - if (fFilterFeeder == LEFT_FEEDER) - strValues = &(bpp->fFiltStrValues[0]); - else if (fFilterFeeder == RIGHT_FEEDER) - strValues = &(bpp->fFiltStrValues[1]); - else - strValues = &(bpp->strValues); + if (fFilterFeeder == LEFT_FEEDER) + strValues = &(bpp->fFiltStrValues[0]); + else if (fFilterFeeder == RIGHT_FEEDER) + strValues = &(bpp->fFiltStrValues[1]); + else + strValues = &(bpp->strValues); - _execute(); + _execute(); } void DictStep::_execute() { - /* Need to loop over bpp->values, issuing a primitive for each LBID */ - uint64_t i; - int64_t l_lbid; - OldGetSigParams* pt; - boost::scoped_array newRidList; + /* Need to loop over bpp->values, issuing a primitive for each LBID */ + uint64_t i; + int64_t l_lbid; + OldGetSigParams* pt; + boost::scoped_array newRidList; - // make the OrderedToken list - newRidList.reset(new OrderedToken[bpp->ridCount]); + // make the OrderedToken list + newRidList.reset(new OrderedToken[bpp->ridCount]); - for (i = 0; i < bpp->ridCount; i++) + for (i = 0; i < bpp->ridCount; i++) + { + newRidList[i].rid = bpp->absRids[i]; + newRidList[i].token = bpp->values[i]; + newRidList[i].pos = i; + } + + sort(&newRidList[0], &newRidList[bpp->ridCount], TokenSorter()); + + tmpResultCounter = 0; + i = 0; + + while (i < bpp->ridCount) + { + l_lbid = ((int64_t)newRidList[i].token) >> 10; + primMsg->LBID = (l_lbid == -1) ? l_lbid : l_lbid & 0xFFFFFFFFFL; + primMsg->NVALS = 0; + + /* When this is used as a filter, the strings can be thrown out. JLF currently + * constructs joblists s.t. only a FilterCommand will use the strings. + */ + primMsg->OutputType = (fFilterFeeder == NOT_FEEDER ? OT_RID : OT_RID | OT_DATAVALUE); + + pt = (OldGetSigParams*)(primMsg->tokens); + + while (i < bpp->ridCount && ((((int64_t)newRidList[i].token) >> 10) == l_lbid)) { - newRidList[i].rid = bpp->absRids[i]; - newRidList[i].token = bpp->values[i]; - newRidList[i].pos = i; + if (UNLIKELY(l_lbid < 0)) + pt[primMsg->NVALS].rid = (fFilterFeeder == NOT_FEEDER ? newRidList[i].rid : i) | 0x8000000000000000LL; + else + pt[primMsg->NVALS].rid = (fFilterFeeder == NOT_FEEDER ? newRidList[i].rid : i); + + pt[primMsg->NVALS].offsetIndex = newRidList[i].token & 0x3ff; + idbassert(pt[primMsg->NVALS].offsetIndex != 0); + primMsg->NVALS++; + i++; } - sort(&newRidList[0], &newRidList[bpp->ridCount], TokenSorter()); + memcpy(&pt[primMsg->NVALS], filterString.buf(), filterString.length()); + issuePrimitive(true); - tmpResultCounter = 0; - i = 0; + if (fFilterFeeder == NOT_FEEDER) + processResult(); + else + copyResultToTmpSpace(newRidList.get()); + } - while (i < bpp->ridCount) - { - l_lbid = ((int64_t) newRidList[i].token) >> 10; - primMsg->LBID = (l_lbid == -1) ? l_lbid : l_lbid & 0xFFFFFFFFFL; - primMsg->NVALS = 0; + inputRidCount = bpp->ridCount; + bpp->ridCount = tmpResultCounter; - /* When this is used as a filter, the strings can be thrown out. JLF currently - * constructs joblists s.t. only a FilterCommand will use the strings. - */ - primMsg->OutputType = (fFilterFeeder == NOT_FEEDER ? OT_RID : OT_RID | OT_DATAVALUE); + // check if feeding a filtercommand + if (fFilterFeeder != NOT_FEEDER) + { + sort(&newRidList[0], &newRidList[inputRidCount], PosSorter()); + copyResultToFinalPosition(newRidList.get()); + copyRidsForFilterCmd(); + } - pt = (OldGetSigParams*) (primMsg->tokens); - - while (i < bpp->ridCount && ((((int64_t) newRidList[i].token) >> 10) == l_lbid )) - { - if (UNLIKELY(l_lbid < 0)) - pt[primMsg->NVALS].rid = - (fFilterFeeder == NOT_FEEDER ? newRidList[i].rid : i) - | 0x8000000000000000LL; - else - pt[primMsg->NVALS].rid = - (fFilterFeeder == NOT_FEEDER ? newRidList[i].rid : i); - - pt[primMsg->NVALS].offsetIndex = newRidList[i].token & 0x3ff; - idbassert(pt[primMsg->NVALS].offsetIndex != 0); - primMsg->NVALS++; - i++; - } - - memcpy(&pt[primMsg->NVALS], filterString.buf(), filterString.length()); - issuePrimitive(true); - - if (fFilterFeeder == NOT_FEEDER) - processResult(); - else - copyResultToTmpSpace(newRidList.get()); - } - - inputRidCount = bpp->ridCount; - bpp->ridCount = tmpResultCounter; - - // check if feeding a filtercommand - if (fFilterFeeder != NOT_FEEDER) - { - sort(&newRidList[0], &newRidList[inputRidCount], PosSorter()); - copyResultToFinalPosition(newRidList.get()); - copyRidsForFilterCmd(); - } - - //cout << "DS: /_execute()\n"; + // cout << "DS: /_execute()\n"; } /* This will do the same thing as execute() but put the result in bpp->serialized */ void DictStep::_project() { - /* Need to loop over bpp->values, issuing a primitive for each LBID */ - uint32_t i; - int64_t l_lbid = 0; - OldGetSigParams* pt; - string tmpStrings[LOGICAL_BLOCK_RIDS]; - boost::scoped_array newRidList; + /* Need to loop over bpp->values, issuing a primitive for each LBID */ + uint32_t i; + int64_t l_lbid = 0; + OldGetSigParams* pt; + string tmpStrings[LOGICAL_BLOCK_RIDS]; + boost::scoped_array newRidList; - // make the OrderedToken list - newRidList.reset(new OrderedToken[bpp->ridCount]); + // make the OrderedToken list + newRidList.reset(new OrderedToken[bpp->ridCount]); - for (i = 0; i < bpp->ridCount; i++) + for (i = 0; i < bpp->ridCount; i++) + { + newRidList[i].rid = bpp->absRids[i]; + newRidList[i].token = values[i]; + newRidList[i].pos = i; + } + + // cout << "DS: _project()\n"; + tmpResultCounter = 0; + totalResultLength = 0; + i = 0; + + while (i < bpp->ridCount) + { + l_lbid = ((int64_t)newRidList[i].token) >> 10; + primMsg->LBID = (l_lbid == -1) ? l_lbid : l_lbid & 0xFFFFFFFFFL; + primMsg->NVALS = 0; + primMsg->OutputType = OT_DATAVALUE; + pt = (OldGetSigParams*)(primMsg->tokens); + + //@bug 972 + while (i < bpp->ridCount && ((((int64_t)newRidList[i].token) >> 10) == l_lbid)) { - newRidList[i].rid = bpp->absRids[i]; - newRidList[i].token = values[i]; - newRidList[i].pos = i; + if (l_lbid < 0) + pt[primMsg->NVALS].rid = newRidList[i].rid | 0x8000000000000000LL; + else + pt[primMsg->NVALS].rid = newRidList[i].rid; + + pt[primMsg->NVALS].offsetIndex = newRidList[i].token & 0x3ff; + idbassert(pt[primMsg->NVALS].offsetIndex > 0); + primMsg->NVALS++; + i++; } - //cout << "DS: _project()\n"; - tmpResultCounter = 0; - totalResultLength = 0; - i = 0; + memcpy(&pt[primMsg->NVALS], filterString.buf(), filterString.length()); + issuePrimitive(false); + projectResult(tmpStrings); + } - while (i < bpp->ridCount) - { - l_lbid = ((int64_t) newRidList[i].token) >> 10; - primMsg->LBID = (l_lbid == -1) ? l_lbid : l_lbid & 0xFFFFFFFFFL; - primMsg->NVALS = 0; - primMsg->OutputType = OT_DATAVALUE; - pt = (OldGetSigParams*) (primMsg->tokens); + idbassert(tmpResultCounter == bpp->ridCount); + *bpp->serialized << totalResultLength; - //@bug 972 - while (i < bpp->ridCount && ((((int64_t) newRidList[i].token) >> 10) == l_lbid)) - { - if (l_lbid < 0) - pt[primMsg->NVALS].rid = newRidList[i].rid | 0x8000000000000000LL; - else - pt[primMsg->NVALS].rid = newRidList[i].rid; + // cout << "_project() total length = " << totalResultLength << endl; + for (i = 0; i < tmpResultCounter; i++) + { + // cout << "serializing " << tmpStrings[i] << endl; + *bpp->serialized << tmpStrings[i]; + } - pt[primMsg->NVALS].offsetIndex = newRidList[i].token & 0x3ff; - idbassert(pt[primMsg->NVALS].offsetIndex > 0); - primMsg->NVALS++; - i++; - } - - memcpy(&pt[primMsg->NVALS], filterString.buf(), filterString.length()); - issuePrimitive(false); - projectResult(tmpStrings); - } - - idbassert(tmpResultCounter == bpp->ridCount); - *bpp->serialized << totalResultLength; - - //cout << "_project() total length = " << totalResultLength << endl; - for (i = 0; i < tmpResultCounter; i++) - { - //cout << "serializing " << tmpStrings[i] << endl; - *bpp->serialized << tmpStrings[i]; - } - - //cout << "DS: /_project() l: " << l_lbid << endl; + // cout << "DS: /_project() l: " << l_lbid << endl; } void DictStep::_projectToRG(RowGroup& rg, uint32_t col) { - /* Need to loop over bpp->values, issuing a primitive for each LBID */ - uint32_t i; - int64_t l_lbid = 0; - int64_t o_lbid = 0; - OldGetSigParams* pt; - StringPtr* tmpStrings = new StringPtr[LOGICAL_BLOCK_RIDS]; - rowgroup::Row r; - boost::scoped_array newRidList; + /* Need to loop over bpp->values, issuing a primitive for each LBID */ + uint32_t i; + int64_t l_lbid = 0; + int64_t o_lbid = 0; + OldGetSigParams* pt; + StringPtr* tmpStrings = new StringPtr[LOGICAL_BLOCK_RIDS]; + rowgroup::Row r; + boost::scoped_array newRidList; - // make the OrderedToken list - newRidList.reset(new OrderedToken[bpp->ridCount]); + // make the OrderedToken list + newRidList.reset(new OrderedToken[bpp->ridCount]); - for (i = 0; i < bpp->ridCount; i++) + for (i = 0; i < bpp->ridCount; i++) + { + newRidList[i].rid = bpp->absRids[i]; + newRidList[i].token = values[i]; + newRidList[i].pos = i; + } + + sort(&newRidList[0], &newRidList[bpp->ridCount], TokenSorter()); + + rg.initRow(&r); + uint32_t curResultCounter = 0; + tmpResultCounter = 0; + totalResultLength = 0; + i = 0; + + // cout << "DS: projectingToRG rids: " << bpp->ridCount << endl; + while (i < bpp->ridCount) + { + l_lbid = ((int64_t)newRidList[i].token) >> 10; + primMsg->LBID = (l_lbid == -1) ? l_lbid : l_lbid & 0xFFFFFFFFFL; + primMsg->NVALS = 0; + primMsg->OutputType = OT_DATAVALUE; + pt = (OldGetSigParams*)(primMsg->tokens); + + //@bug 972 + //@bug 1821 + while (i < bpp->ridCount && ((((int64_t)newRidList[i].token) >> 10) == l_lbid || l_lbid == -1 || + ((((int64_t)newRidList[i].token) >> 10) & 0x8000000000000000LL))) { - newRidList[i].rid = bpp->absRids[i]; - newRidList[i].token = values[i]; - newRidList[i].pos = i; + //@bug 1821 + if (newRidList[i].token == 0) + { + ostringstream oss; + ostringstream oss2; + oss << l_lbid; + logging::Message::Args args; + args.add("0"); + args.add(oss.str()); + oss2 << newRidList[i].rid; + args.add(oss2.str()); + primitiveprocessor::mlp->logMessage(logging::M0068, args, true); + newRidList[i].token = 0xfffffffffffffffeLL; + values[newRidList[i].pos] = 0xfffffffffffffffeLL; + } + + if ((((int64_t)newRidList[i].token) >> 10) < 0) + { + pt[primMsg->NVALS].rid = newRidList[i].rid | 0x8000000000000000LL; + } + else + { + if ((((int64_t)newRidList[i].token) >> 10) > 0 && o_lbid == 0) + l_lbid = o_lbid = (((int64_t)newRidList[i].token) >> 10); + + pt[primMsg->NVALS].rid = newRidList[i].rid; + } + + pt[primMsg->NVALS].offsetIndex = newRidList[i].token & 0x3ff; + idbassert(pt[primMsg->NVALS].offsetIndex > 0); + primMsg->NVALS++; + // pt++; + i++; } - sort(&newRidList[0], &newRidList[bpp->ridCount], TokenSorter()); + if (((int64_t)primMsg->LBID) < 0 && o_lbid > 0) + primMsg->LBID = o_lbid & 0xFFFFFFFFFL; - rg.initRow(&r); - uint32_t curResultCounter = 0; - tmpResultCounter = 0; - totalResultLength = 0; - i = 0; + memcpy(&pt[primMsg->NVALS], filterString.buf(), filterString.length()); + issuePrimitive(false); + projectResult(tmpStrings); + o_lbid = 0; + // cout << "DS: project & issue l: " << (int64_t)primMsg->LBID << " NVALS: " << primMsg->NVALS << endl; - //cout << "DS: projectingToRG rids: " << bpp->ridCount << endl; - while (i < bpp->ridCount) + // bug 4901 - move this inside the loop and call incrementally + // to save the unnecessary string copy + if ((rg.getColTypes()[col] != execplan::CalpontSystemCatalog::VARBINARY) && + (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::BLOB) && + (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::TEXT)) { - l_lbid = ((int64_t) newRidList[i].token) >> 10; - primMsg->LBID = (l_lbid == -1) ? l_lbid : l_lbid & 0xFFFFFFFFFL; - primMsg->NVALS = 0; - primMsg->OutputType = OT_DATAVALUE; - pt = (OldGetSigParams*) (primMsg->tokens); + for (i = curResultCounter; i < tmpResultCounter; i++) + { + rg.getRow(newRidList[i].pos, &r); + // std::cerr << "serializing " << tmpStrings[i] << endl; + r.setStringField(tmpStrings[i].getConstString(), col); + } + } + else + { + uint32_t firstTmpResultCounter = tmpResultCounter; + std::map result; - //@bug 972 - //@bug 1821 - while (i < bpp->ridCount && ((((int64_t)newRidList[i].token) >> 10) == l_lbid || l_lbid == -1 - || ((((int64_t)newRidList[i].token) >> 10) & 0x8000000000000000LL)) ) + for (i = curResultCounter; i < firstTmpResultCounter; i++) + { + rg.getRow(newRidList[i].pos, &r); + + // If this is a multi-block blob, get all the blocks + // We do string copy here, should maybe have a RowGroup + // function to append strings or something? + if (((newRidList[i].token >> 46) < 0x3FFFF) && ((newRidList[i].token >> 46) > 0)) { - //@bug 1821 - if (newRidList[i].token == 0) + StringPtr multi_part[1]; + uint16_t old_offset = primMsg->tokens[0].offset; + + if (result.empty()) + { + // String copy here because tmpStrings pointers will be blown away below + for (uint32_t x = curResultCounter; x < firstTmpResultCounter; x++) { - ostringstream oss; - ostringstream oss2; - oss << l_lbid; - logging::Message::Args args; - args.add("0"); - args.add(oss.str()); - oss2 << newRidList[i].rid; - args.add(oss2.str()); - primitiveprocessor::mlp->logMessage(logging::M0068, args, true); - newRidList[i].token = 0xfffffffffffffffeLL; - values[newRidList[i].pos] = 0xfffffffffffffffeLL; + result[x] = new string((char*)tmpStrings[x].ptr, tmpStrings[x].len); } + } - if ((((int64_t)newRidList[i].token) >> 10) < 0) - { - pt[primMsg->NVALS].rid = newRidList[i].rid | 0x8000000000000000LL; - } - else - { - if ((((int64_t)newRidList[i].token) >> 10) > 0 && o_lbid == 0) - l_lbid = o_lbid = (((int64_t)newRidList[i].token) >> 10); + uint64_t origin_lbid = primMsg->LBID; + uint32_t lbid_count = newRidList[i].token >> 46; + primMsg->tokens[0].offset = 1; // first offset of a sig - pt[primMsg->NVALS].rid = newRidList[i].rid; - } + for (uint32_t j = 1; j <= lbid_count; j++) + { + tmpResultCounter = 0; + primMsg->LBID = origin_lbid + j; + primMsg->NVALS = 1; + primMsg->tokens[0].LBID = origin_lbid + j; + issuePrimitive(false); + projectResult(multi_part); + result[i]->append((char*)multi_part[0].ptr, multi_part[0].len); + } - pt[primMsg->NVALS].offsetIndex = newRidList[i].token & 0x3ff; - idbassert(pt[primMsg->NVALS].offsetIndex > 0); - primMsg->NVALS++; -// pt++; - i++; - } - - if (((int64_t)primMsg->LBID) < 0 && o_lbid > 0) - primMsg->LBID = o_lbid & 0xFFFFFFFFFL; - - memcpy(&pt[primMsg->NVALS], filterString.buf(), filterString.length()); - issuePrimitive(false); - projectResult(tmpStrings); - o_lbid = 0; - //cout << "DS: project & issue l: " << (int64_t)primMsg->LBID << " NVALS: " << primMsg->NVALS << endl; - - // bug 4901 - move this inside the loop and call incrementally - // to save the unnecessary string copy - if ((rg.getColTypes()[col] != execplan::CalpontSystemCatalog::VARBINARY) && - (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::BLOB) && - (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::TEXT)) - { - for (i = curResultCounter; i < tmpResultCounter; i++) - { - rg.getRow(newRidList[i].pos, &r); - //std::cerr << "serializing " << tmpStrings[i] << endl; - r.setStringField(tmpStrings[i].getConstString(), col); - } + primMsg->tokens[0].offset = old_offset; + primMsg->LBID = origin_lbid; + tmpResultCounter = firstTmpResultCounter; + r.setVarBinaryField((unsigned char*)result[i]->c_str(), result[i]->length(), col); + delete result[i]; } else { - uint32_t firstTmpResultCounter = tmpResultCounter; - std::map result; - - for (i = curResultCounter; i < firstTmpResultCounter; i++) - { - rg.getRow(newRidList[i].pos, &r); - - // If this is a multi-block blob, get all the blocks - // We do string copy here, should maybe have a RowGroup - // function to append strings or something? - if (((newRidList[i].token >> 46) < 0x3FFFF) && - ((newRidList[i].token >> 46) > 0)) - { - StringPtr multi_part[1]; - uint16_t old_offset = primMsg->tokens[0].offset; - - if (result.empty()) - { - // String copy here because tmpStrings pointers will be blown away below - for (uint32_t x = curResultCounter; x < firstTmpResultCounter; x++) - { - result[x] = new string((char*)tmpStrings[x].ptr, tmpStrings[x].len); - } - } - - uint64_t origin_lbid = primMsg->LBID; - uint32_t lbid_count = newRidList[i].token >> 46; - primMsg->tokens[0].offset = 1; // first offset of a sig - - for (uint32_t j = 1; j <= lbid_count; j++) - { - tmpResultCounter = 0; - primMsg->LBID = origin_lbid + j; - primMsg->NVALS = 1; - primMsg->tokens[0].LBID = origin_lbid + j; - issuePrimitive(false); - projectResult(multi_part); - result[i]->append((char*)multi_part[0].ptr, multi_part[0].len); - } - - primMsg->tokens[0].offset = old_offset; - primMsg->LBID = origin_lbid; - tmpResultCounter = firstTmpResultCounter; - r.setVarBinaryField((unsigned char*)result[i]->c_str(), result[i]->length(), col); - delete result[i]; - } - else - { - r.setVarBinaryField(tmpStrings[i].ptr, tmpStrings[i].len, col); - } - } + r.setVarBinaryField(tmpStrings[i].ptr, tmpStrings[i].len, col); } - - curResultCounter = tmpResultCounter; + } } - //cout << "_projectToRG() total length = " << totalResultLength << endl; - idbassert(tmpResultCounter == bpp->ridCount); + curResultCounter = tmpResultCounter; + } - delete [] tmpStrings; - //cout << "DS: /projectingToRG l: " << (int64_t)primMsg->LBID - // << " len: " << tmpResultCounter - // << endl; + // cout << "_projectToRG() total length = " << totalResultLength << endl; + idbassert(tmpResultCounter == bpp->ridCount); + + delete[] tmpStrings; + // cout << "DS: /projectingToRG l: " << (int64_t)primMsg->LBID + // << " len: " << tmpResultCounter + // << endl; } void DictStep::project() { - values = bpp->values; - _project(); + values = bpp->values; + _project(); } void DictStep::project(int64_t* vals) { - values = vals; - _project(); + values = vals; + _project(); } void DictStep::projectIntoRowGroup(RowGroup& rg, uint32_t col) { - values = bpp->values; - _projectToRG(rg, col); + values = bpp->values; + _projectToRG(rg, col); } void DictStep::projectIntoRowGroup(RowGroup& rg, int64_t* vals, uint32_t col) { - values = vals; - _projectToRG(rg, col); + values = vals; + _projectToRG(rg, col); } uint64_t DictStep::getLBID() { - return 0; + return 0; } void DictStep::nextLBID() @@ -668,37 +660,33 @@ void DictStep::nextLBID() SCommand DictStep::duplicate() { - SCommand ret; - DictStep* ds; + SCommand ret; + DictStep* ds; - ret.reset(new DictStep()); - ds = (DictStep*) ret.get(); - ds->BOP = BOP; - ds->fFilterFeeder = fFilterFeeder; - ds->compressionType = compressionType; - ds->hasEqFilter = hasEqFilter; - ds->eqFilter = eqFilter; - ds->eqOp = eqOp; - ds->filterString = filterString; - ds->filterCount = filterCount; - ds->charsetNumber = charsetNumber; - ds->Command::operator=(*this); - return ret; + ret.reset(new DictStep()); + ds = (DictStep*)ret.get(); + ds->BOP = BOP; + ds->fFilterFeeder = fFilterFeeder; + ds->compressionType = compressionType; + ds->hasEqFilter = hasEqFilter; + ds->eqFilter = eqFilter; + ds->eqOp = eqOp; + ds->filterString = filterString; + ds->filterCount = filterCount; + ds->charsetNumber = charsetNumber; + ds->Command::operator=(*this); + return ret; } bool DictStep::operator==(const DictStep& ds) const { - return ((BOP == ds.BOP) && - (fFilterFeeder == ds.fFilterFeeder) && - (compressionType == ds.compressionType) && - (filterString == ds.filterString) && - (filterCount == ds.filterCount)); + return ((BOP == ds.BOP) && (fFilterFeeder == ds.fFilterFeeder) && (compressionType == ds.compressionType) && + (filterString == ds.filterString) && (filterCount == ds.filterCount)); } bool DictStep::operator!=(const DictStep& ds) const { - return !(*this == ds); + return !(*this == ds); } - -}; // namespace +}; // namespace primitiveprocessor diff --git a/primitives/primproc/dictstep.h b/primitives/primproc/dictstep.h index 0eb37ca7c..bb30db472 100644 --- a/primitives/primproc/dictstep.h +++ b/primitives/primproc/dictstep.h @@ -36,125 +36,133 @@ namespace primitiveprocessor { - - class DictStep : public Command { -public: - DictStep(); - virtual ~DictStep(); + public: + DictStep(); + virtual ~DictStep(); - void execute(); - void project(); - void project(int64_t* vals); //used by RTSCommand to redirect input - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row); - void projectIntoRowGroup(rowgroup::RowGroup& rg, int64_t* vals, uint32_t col); - uint64_t getLBID(); + void execute(); + void project(); + void project(int64_t* vals); // used by RTSCommand to redirect input + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row); + void projectIntoRowGroup(rowgroup::RowGroup& rg, int64_t* vals, uint32_t col); + uint64_t getLBID(); - /* This doesn't do anything for this class... make it column-specific or not? */ - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); + /* This doesn't do anything for this class... make it column-specific or not? */ + void nextLBID(); + void createCommand(messageqcpp::ByteStream&); + void resetCommand(messageqcpp::ByteStream&); - /* Put bootstrap code here (ie, build the template primitive msg) */ - void prep(int8_t outputType, bool makeAbsRids); + /* Put bootstrap code here (ie, build the template primitive msg) */ + void prep(int8_t outputType, bool makeAbsRids); - SCommand duplicate(); - bool operator==(const DictStep&) const; - bool operator!=(const DictStep&) const; + SCommand duplicate(); + bool operator==(const DictStep&) const; + bool operator!=(const DictStep&) const; - int getCompType() const + int getCompType() const + { + return compressionType; + } + void setCompType(int ct) + { + compressionType = ct; + } + + private: + DictStep(const DictStep&); + DictStep& operator=(const DictStep&); + + struct StringPtr + { + const uint8_t* ptr; + unsigned len; + + StringPtr() : ptr(NULL), len(0) { - return compressionType; + ; } - void setCompType(int ct) + StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) { - compressionType = ct; + ; } - -private: - DictStep(const DictStep&); - DictStep& operator=(const DictStep&); - - struct StringPtr + utils::ConstString getConstString() const { - const uint8_t* ptr; - unsigned len; + return utils::ConstString((const char*)ptr, len); + } + }; - StringPtr() : ptr(NULL), len(0) {;} - StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) {;} - utils::ConstString getConstString() const - { - return utils::ConstString((const char *) ptr, len); - } - }; + void _execute(); + void issuePrimitive(bool isProjection); + void processResult(); + void projectResult(std::string* tmpStrings); + void projectResult(StringPtr* tmpStrings); + void _project(); + void _projectToRG(rowgroup::RowGroup& rg, uint32_t col); - void _execute(); - void issuePrimitive(bool isProjection); - void processResult(); - void projectResult(std::string* tmpStrings); - void projectResult(StringPtr* tmpStrings); - void _project(); - void _projectToRG(rowgroup::RowGroup& rg, uint32_t col); - - // struct used for scratch space - struct OrderedToken + // struct used for scratch space + struct OrderedToken + { + uint64_t rid; + uint64_t token; + uint16_t pos; + std::string str; + bool inResult; + OrderedToken() : inResult(false) { - uint64_t rid; - uint64_t token; - uint16_t pos; - std::string str; - bool inResult; - OrderedToken() : inResult(false) { } - ~OrderedToken() { } - }; - struct TokenSorter + } + ~OrderedToken() { - inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const - { - return (c1.token < c2.token); - } - }; - struct PosSorter + } + }; + struct TokenSorter + { + inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const { - inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const - { - return (c1.pos < c2.pos); - } - }; + return (c1.token < c2.token); + } + }; + struct PosSorter + { + inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const + { + return (c1.pos < c2.pos); + } + }; - //bug 3679. FilterCommand depends on the result being in the same relative - //order as the input. These fcns help restore the original order. - void copyResultToTmpSpace(OrderedToken* ot); - void copyResultToFinalPosition(OrderedToken* ot); + // bug 3679. FilterCommand depends on the result being in the same relative + // order as the input. These fcns help restore the original order. + void copyResultToTmpSpace(OrderedToken* ot); + void copyResultToFinalPosition(OrderedToken* ot); - // Worst case, 8192 tokens in the msg. Each is 10 bytes. */ - boost::scoped_array inputMsg; - uint32_t tmpResultCounter; - uint32_t totalResultLength; - DictInput* primMsg; - std::vector result; + // Worst case, 8192 tokens in the msg. Each is 10 bytes. */ + boost::scoped_array inputMsg; + uint32_t tmpResultCounter; + uint32_t totalResultLength; + DictInput* primMsg; + std::vector result; - uint64_t lbid; - uint32_t fbo; - uint32_t traceFlags; // probably move this to Command - uint8_t BOP; - int64_t* values; - boost::scoped_array* strValues; - int compressionType; - messageqcpp::ByteStream filterString; - uint32_t filterCount; - uint32_t bufferSize; - uint32_t charsetNumber; - uint16_t inputRidCount; - - bool hasEqFilter; - boost::shared_ptr eqFilter; - uint8_t eqOp; // COMPARE_EQ or COMPARE_NE + uint64_t lbid; + uint32_t fbo; + uint32_t traceFlags; // probably move this to Command + uint8_t BOP; + int64_t* values; + boost::scoped_array* strValues; + int compressionType; + messageqcpp::ByteStream filterString; + uint32_t filterCount; + uint32_t bufferSize; + uint32_t charsetNumber; + uint16_t inputRidCount; - friend class RTSCommand; + bool hasEqFilter; + boost::shared_ptr eqFilter; + uint8_t eqOp; // COMPARE_EQ or COMPARE_NE + + friend class RTSCommand; }; -} // namespace +} // namespace primitiveprocessor #endif diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index 88ecb03e7..ba19098ce 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: filtercommand.cpp 2035 2013-01-21 14:12:19Z rdempsey $ -* -* -***********************************************************************/ + * $Id: filtercommand.cpp 2035 2013-01-21 14:12:19Z rdempsey $ + * + * + ***********************************************************************/ #include #include "primitiveserver.h" @@ -37,180 +37,170 @@ namespace { uint32_t cc = primitiveprocessor::Command::COLUMN_COMMAND; uint32_t ds = primitiveprocessor::Command::DICT_STEP; -const uint32_t CC = (cc << 8) | cc; -const uint32_t DCC = (ds << 16) | (cc << 8) | cc; -const uint32_t CDC = (cc << 16) | (ds << 8) | cc; +const uint32_t CC = (cc << 8) | cc; +const uint32_t DCC = (ds << 16) | (cc << 8) | cc; +const uint32_t CDC = (cc << 16) | (ds << 8) | cc; const uint32_t DCDC = (ds << 24) | (cc << 16) | (ds << 8) | cc; -}; - +}; // namespace namespace primitiveprocessor { - Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector& cmds) { - bs.advance(1); - // find out the # of commands in the cmds vector, - // vector::size() will not work, because cmds is resize() to filterCount - uint64_t nc = 0; + bs.advance(1); + // find out the # of commands in the cmds vector, + // vector::size() will not work, because cmds is resize() to filterCount + uint64_t nc = 0; - while (cmds[nc].get() != NULL) nc++; + while (cmds[nc].get() != NULL) + nc++; - // figure out the feeding commands - // must have 2 columncommands, may have 1 or 2 dictsteps. - uint64_t cols = 0; // # of columncommands - uint32_t columns = 0; - uint64_t i = nc; + // figure out the feeding commands + // must have 2 columncommands, may have 1 or 2 dictsteps. + uint64_t cols = 0; // # of columncommands + uint32_t columns = 0; + uint64_t i = nc; - while (i > 0 && cols < 2) + while (i > 0 && cols < 2) + { + Command::CommandType cmdType = cmds[i - 1]->getCommandType(); + + if (cmdType != Command::COLUMN_COMMAND && cmdType != Command::DICT_STEP) { - Command::CommandType cmdType = cmds[i - 1]->getCommandType(); - - if (cmdType != Command::COLUMN_COMMAND && cmdType != Command::DICT_STEP) - { - stringstream msg; - msg << "FilterCommand: feeded by " << cmdType << " is not supported."; - throw logic_error(msg.str()); - } - - columns = (columns << 8) + cmdType; - - if (cmdType == Command::COLUMN_COMMAND) - cols++; - - i--; + stringstream msg; + msg << "FilterCommand: feeded by " << cmdType << " is not supported."; + throw logic_error(msg.str()); } - // should not happen - if (cols < 2) throw logic_error("FilterCommand: not enough feeding ColumnCommands."); + columns = (columns << 8) + cmdType; + if (cmdType == Command::COLUMN_COMMAND) + cols++; - FilterCommand* fc = NULL; + i--; + } - // the order setting left/right feeder is important, left is the smaller index. - // because the doFilter relies on the rids of right-feeder is a subset of left. - if (columns == CC) + // should not happen + if (cols < 2) + throw logic_error("FilterCommand: not enough feeding ColumnCommands."); + + FilterCommand* fc = NULL; + + // the order setting left/right feeder is important, left is the smaller index. + // because the doFilter relies on the rids of right-feeder is a subset of left. + if (columns == CC) + { + cmds[nc - 2]->filterFeeder(LEFT_FEEDER); + cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); + + ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 2].get()); + ColumnCommand* cmd1 = dynamic_cast(cmds[nc - 1].get()); + uint32_t scale0 = cmd0->getScale(); + uint32_t scale1 = cmd1->getScale(); + + // char[] is stored as int, but cannot directly compare if length is different + // due to endian issue + if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR || + cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR || + cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::TEXT) { - cmds[nc - 2]->filterFeeder(LEFT_FEEDER); - cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); - - ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 2].get()); - ColumnCommand* cmd1 = dynamic_cast(cmds[nc - 1].get()); - uint32_t scale0 = cmd0->getScale(); - uint32_t scale1 = cmd1->getScale(); - - // char[] is stored as int, but cannot directly compare if length is different - // due to endian issue - if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR || - cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR || - cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::TEXT) - { - StrFilterCmd* sc = new StrFilterCmd(); - sc->setCompareFunc(CC); - fc = sc; - } - else if (scale0 == scale1) - { - fc = new FilterCommand(); - } - else - { - ScaledFilterCmd* sc = new ScaledFilterCmd(); - sc->setFactor(datatypes::scaleDivisor(scale1) / - datatypes::scaleDivisor(scale0)); - fc = sc; - } - - fc->setColTypes(cmd0->getColType(), cmd1->getColType()); + StrFilterCmd* sc = new StrFilterCmd(); + sc->setCompareFunc(CC); + fc = sc; } - else if (columns == DCDC) // both string + else if (scale0 == scale1) { - StrFilterCmd* sc = new StrFilterCmd(); - cmds[nc - 4]->filterFeeder(FILT_FEEDER); - cmds[nc - 3]->filterFeeder(LEFT_FEEDER); - cmds[nc - 2]->filterFeeder(FILT_FEEDER); - cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); - sc->setCompareFunc(DCDC); - fc = sc; - - ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 4].get()); - ColumnCommand* cmd2 = dynamic_cast(cmds[nc - 2].get()); - fc->setColTypes(cmd0->getColType(), cmd2->getColType()); - } - else if (columns == DCC) // lhs: char[]; rhs: string - { - StrFilterCmd* sc = new StrFilterCmd(); - cmds[nc - 3]->filterFeeder(LEFT_FEEDER); - cmds[nc - 2]->filterFeeder(FILT_FEEDER); - cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); - ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 3].get()); - ColumnCommand* cmd1 = dynamic_cast(cmds[nc - 2].get()); - size_t cl = cmd0->getWidth(); // char[] column - sc->setCharLength(cl); - sc->setCompareFunc(DCC); - fc = sc; - fc->setColTypes(cmd0->getColType(), cmd1->getColType()); - } - else if (columns == CDC) // lhs: string; rhs: char[] - { - StrFilterCmd* sc = new StrFilterCmd(); - cmds[nc - 3]->filterFeeder(FILT_FEEDER); - cmds[nc - 2]->filterFeeder(LEFT_FEEDER); - cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); - ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 3].get()); - ColumnCommand* cmd1 = dynamic_cast(cmds[nc - 1].get()); - size_t cl = cmd1->getWidth(); // char[] column - sc->setCharLength(cl); - sc->setCompareFunc(CDC); - fc = sc; - fc->setColTypes(cmd0->getColType(), cmd1->getColType()); + fc = new FilterCommand(); } else { - stringstream msg; - msg << "FilterCommand does not handle this column code: " << hex << columns << dec; - throw logic_error(msg.str()); + ScaledFilterCmd* sc = new ScaledFilterCmd(); + sc->setFactor(datatypes::scaleDivisor(scale1) / datatypes::scaleDivisor(scale0)); + fc = sc; } - return fc; + fc->setColTypes(cmd0->getColType(), cmd1->getColType()); + } + else if (columns == DCDC) // both string + { + StrFilterCmd* sc = new StrFilterCmd(); + cmds[nc - 4]->filterFeeder(FILT_FEEDER); + cmds[nc - 3]->filterFeeder(LEFT_FEEDER); + cmds[nc - 2]->filterFeeder(FILT_FEEDER); + cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); + sc->setCompareFunc(DCDC); + fc = sc; + + ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 4].get()); + ColumnCommand* cmd2 = dynamic_cast(cmds[nc - 2].get()); + fc->setColTypes(cmd0->getColType(), cmd2->getColType()); + } + else if (columns == DCC) // lhs: char[]; rhs: string + { + StrFilterCmd* sc = new StrFilterCmd(); + cmds[nc - 3]->filterFeeder(LEFT_FEEDER); + cmds[nc - 2]->filterFeeder(FILT_FEEDER); + cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); + ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 3].get()); + ColumnCommand* cmd1 = dynamic_cast(cmds[nc - 2].get()); + size_t cl = cmd0->getWidth(); // char[] column + sc->setCharLength(cl); + sc->setCompareFunc(DCC); + fc = sc; + fc->setColTypes(cmd0->getColType(), cmd1->getColType()); + } + else if (columns == CDC) // lhs: string; rhs: char[] + { + StrFilterCmd* sc = new StrFilterCmd(); + cmds[nc - 3]->filterFeeder(FILT_FEEDER); + cmds[nc - 2]->filterFeeder(LEFT_FEEDER); + cmds[nc - 1]->filterFeeder(RIGHT_FEEDER); + ColumnCommand* cmd0 = dynamic_cast(cmds[nc - 3].get()); + ColumnCommand* cmd1 = dynamic_cast(cmds[nc - 1].get()); + size_t cl = cmd1->getWidth(); // char[] column + sc->setCharLength(cl); + sc->setCompareFunc(CDC); + fc = sc; + fc->setColTypes(cmd0->getColType(), cmd1->getColType()); + } + else + { + stringstream msg; + msg << "FilterCommand does not handle this column code: " << hex << columns << dec; + throw logic_error(msg.str()); + } + + return fc; } - -FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0), - hasWideColumns(false) +FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0), hasWideColumns(false) { } - FilterCommand::~FilterCommand() { } - void FilterCommand::execute() { - doFilter(); + doFilter(); } - void FilterCommand::createCommand(ByteStream& bs) { - bs >> fBOP; - Command::createCommand(bs); + bs >> fBOP; + Command::createCommand(bs); } - void FilterCommand::resetCommand(ByteStream& bs) { } - void FilterCommand::prep(int8_t outputType, bool absRids) { } - void FilterCommand::project() { } @@ -219,337 +209,272 @@ void FilterCommand::projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) { } - uint64_t FilterCommand::getLBID() { - return 0; + return 0; } - void FilterCommand::nextLBID() { } - SCommand FilterCommand::duplicate() { - return SCommand(new FilterCommand(*this)); + return SCommand(new FilterCommand(*this)); } - void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& left, const execplan::CalpontSystemCatalog::ColType& right) { - leftColType = left; - rightColType = right; + leftColType = left; + rightColType = right; - if (left.isWideDecimalType() || right.isWideDecimalType()) - hasWideColumns = true; + if (left.isWideDecimalType() || right.isWideDecimalType()) + hasWideColumns = true; } - void FilterCommand::doFilter() { - bpp->ridMap = 0; - bpp->ridCount = 0; + bpp->ridMap = 0; + bpp->ridCount = 0; - bool (FilterCommand::*compareFunc)(uint64_t, uint64_t); + bool (FilterCommand::*compareFunc)(uint64_t, uint64_t); - if (hasWideColumns) - compareFunc = &FilterCommand::binaryCompare; - else - compareFunc = &FilterCommand::compare; + if (hasWideColumns) + compareFunc = &FilterCommand::binaryCompare; + else + compareFunc = &FilterCommand::compare; - // rids in [0] is used for scan [1], so [1] is a subset of [0], and same order. - // -- see makeFilterCommand() above. - for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; ) + // rids in [0] is used for scan [1], so [1] is a subset of [0], and same order. + // -- see makeFilterCommand() above. + for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1];) + { + if (bpp->fFiltCmdRids[0][i] != bpp->fFiltCmdRids[1][j]) { - if (bpp->fFiltCmdRids[0][i] != bpp->fFiltCmdRids[1][j]) - { - i++; - } - else - { - if ((this->*compareFunc)(i, j) == true) - { - bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; - if (leftColType.isWideDecimalType()) - bpp->wide128Values[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; - else - bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; - bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 9); - bpp->ridCount++; - } - - i++; - j++; - } + i++; } + else + { + if ((this->*compareFunc)(i, j) == true) + { + bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; + if (leftColType.isWideDecimalType()) + bpp->wide128Values[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; + else + bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; + bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 9); + bpp->ridCount++; + } - // bug 1247 -- reset the rid count - bpp->fFiltRidCount[0] = bpp->fFiltRidCount[1] = 0; + i++; + j++; + } + } + + // bug 1247 -- reset the rid count + bpp->fFiltRidCount[0] = bpp->fFiltRidCount[1] = 0; } - bool FilterCommand::compare(uint64_t i, uint64_t j) { - if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || - execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) - return false; + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || + execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; - switch (fBOP) - { - case COMPARE_GT: - return bpp->fFiltCmdValues[0][i] > bpp->fFiltCmdValues[1][j]; - break; + switch (fBOP) + { + case COMPARE_GT: return bpp->fFiltCmdValues[0][i] > bpp->fFiltCmdValues[1][j]; break; - case COMPARE_LT: - return bpp->fFiltCmdValues[0][i] < bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_LT: return bpp->fFiltCmdValues[0][i] < bpp->fFiltCmdValues[1][j]; break; - case COMPARE_EQ: - return bpp->fFiltCmdValues[0][i] == bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_EQ: return bpp->fFiltCmdValues[0][i] == bpp->fFiltCmdValues[1][j]; break; - case COMPARE_GE: - return bpp->fFiltCmdValues[0][i] >= bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_GE: return bpp->fFiltCmdValues[0][i] >= bpp->fFiltCmdValues[1][j]; break; - case COMPARE_LE: - return bpp->fFiltCmdValues[0][i] <= bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_LE: return bpp->fFiltCmdValues[0][i] <= bpp->fFiltCmdValues[1][j]; break; - case COMPARE_NE: - return bpp->fFiltCmdValues[0][i] != bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_NE: return bpp->fFiltCmdValues[0][i] != bpp->fFiltCmdValues[1][j]; break; - default: - return false; - break; - } + default: return false; break; + } } bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) { - // We type-promote to int128_t if either of the columns are - // not int128_t - int128_t leftVal, rightVal; + // We type-promote to int128_t if either of the columns are + // not int128_t + int128_t leftVal, rightVal; - if (leftColType.isWideDecimalType()) - { - if (execplan::isNull(bpp->fFiltCmdBinaryValues[0][i], leftColType)) - return false; - leftVal = bpp->fFiltCmdBinaryValues[0][i]; - } - else - { - if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType)) - return false; - leftVal = bpp->fFiltCmdValues[0][i]; - } + if (leftColType.isWideDecimalType()) + { + if (execplan::isNull(bpp->fFiltCmdBinaryValues[0][i], leftColType)) + return false; + leftVal = bpp->fFiltCmdBinaryValues[0][i]; + } + else + { + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType)) + return false; + leftVal = bpp->fFiltCmdValues[0][i]; + } - if (rightColType.isWideDecimalType()) - { - if (execplan::isNull(bpp->fFiltCmdBinaryValues[1][j], rightColType)) - return false; - rightVal = bpp->fFiltCmdBinaryValues[1][j]; - } - else - { - if (execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) - return false; - rightVal = bpp->fFiltCmdValues[1][j]; - } + if (rightColType.isWideDecimalType()) + { + if (execplan::isNull(bpp->fFiltCmdBinaryValues[1][j], rightColType)) + return false; + rightVal = bpp->fFiltCmdBinaryValues[1][j]; + } + else + { + if (execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; + rightVal = bpp->fFiltCmdValues[1][j]; + } - switch (fBOP) - { - case COMPARE_GT: - return leftVal > rightVal; - break; + switch (fBOP) + { + case COMPARE_GT: return leftVal > rightVal; break; - case COMPARE_LT: - return leftVal < rightVal; - break; + case COMPARE_LT: return leftVal < rightVal; break; - case COMPARE_EQ: - return leftVal == rightVal; - break; + case COMPARE_EQ: return leftVal == rightVal; break; - case COMPARE_GE: - return leftVal >= rightVal; - break; + case COMPARE_GE: return leftVal >= rightVal; break; - case COMPARE_LE: - return leftVal <= rightVal; - break; + case COMPARE_LE: return leftVal <= rightVal; break; - case COMPARE_NE: - return leftVal != rightVal; - break; + case COMPARE_NE: return leftVal != rightVal; break; - default: - return false; - break; - } + default: return false; break; + } } - bool FilterCommand::operator==(const FilterCommand& c) const { - return (fBOP == c.fBOP); + return (fBOP == c.fBOP); } - bool FilterCommand::operator!=(const FilterCommand& c) const { - return !(*this == c); + return !(*this == c); } - // == ScaledFilterCmd == ScaledFilterCmd::ScaledFilterCmd() : fFactor(1) { } - ScaledFilterCmd::~ScaledFilterCmd() { } - SCommand ScaledFilterCmd::duplicate() { - SCommand ret; - ScaledFilterCmd* filterCmd; + SCommand ret; + ScaledFilterCmd* filterCmd; - ret.reset(new ScaledFilterCmd()); - filterCmd = (ScaledFilterCmd*) ret.get(); - filterCmd->fBOP = fBOP; - filterCmd->fFactor = fFactor; + ret.reset(new ScaledFilterCmd()); + filterCmd = (ScaledFilterCmd*)ret.get(); + filterCmd->fBOP = fBOP; + filterCmd->fFactor = fFactor; - return ret; + return ret; } - bool ScaledFilterCmd::compare(uint64_t i, uint64_t j) { - if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || - execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) - return false; + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || + execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; - switch (fBOP) - { - case COMPARE_GT: - return bpp->fFiltCmdValues[0][i] * fFactor > bpp->fFiltCmdValues[1][j]; - break; + switch (fBOP) + { + case COMPARE_GT: return bpp->fFiltCmdValues[0][i] * fFactor > bpp->fFiltCmdValues[1][j]; break; - case COMPARE_LT: - return bpp->fFiltCmdValues[0][i] * fFactor < bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_LT: return bpp->fFiltCmdValues[0][i] * fFactor < bpp->fFiltCmdValues[1][j]; break; - case COMPARE_EQ: - return bpp->fFiltCmdValues[0][i] * fFactor == bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_EQ: return bpp->fFiltCmdValues[0][i] * fFactor == bpp->fFiltCmdValues[1][j]; break; - case COMPARE_GE: - return bpp->fFiltCmdValues[0][i] * fFactor >= bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_GE: return bpp->fFiltCmdValues[0][i] * fFactor >= bpp->fFiltCmdValues[1][j]; break; - case COMPARE_LE: - return bpp->fFiltCmdValues[0][i] * fFactor <= bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_LE: return bpp->fFiltCmdValues[0][i] * fFactor <= bpp->fFiltCmdValues[1][j]; break; - case COMPARE_NE: - return bpp->fFiltCmdValues[0][i] * fFactor != bpp->fFiltCmdValues[1][j]; - break; + case COMPARE_NE: return bpp->fFiltCmdValues[0][i] * fFactor != bpp->fFiltCmdValues[1][j]; break; - default: - return false; - break; - } + default: return false; break; + } } - void ScaledFilterCmd::setFactor(double f) { - fFactor = f; + fFactor = f; } - double ScaledFilterCmd::factor() { - return fFactor; + return fFactor; } - bool ScaledFilterCmd::operator==(const ScaledFilterCmd& c) const { - return ((fBOP == c.fBOP) && (fFactor == c.fFactor)); + return ((fBOP == c.fBOP) && (fFactor == c.fFactor)); } - bool ScaledFilterCmd::operator!=(const ScaledFilterCmd& c) const { - return !(*this == c); + return !(*this == c); } - // == StrFilterCmd == StrFilterCmd::StrFilterCmd() : fCompare(NULL), fCharLength(8) { } - StrFilterCmd::~StrFilterCmd() { } - SCommand StrFilterCmd::duplicate() { - return SCommand(new StrFilterCmd(*this)); + return SCommand(new StrFilterCmd(*this)); } - void StrFilterCmd::execute() { - doFilter(); + doFilter(); } - bool StrFilterCmd::compare(uint64_t i, uint64_t j) { - return (this->*fCompare)(i, j); + return (this->*fCompare)(i, j); } - void StrFilterCmd::setCompareFunc(uint32_t columns) { - if (columns == CC) // char[] : char - { - fCompare = &StrFilterCmd::compare_cc; - } - else if (columns == DCDC) // string : string - { - fCompare = &StrFilterCmd::compare_ss; - } - else if (columns == DCC) // char[] : string - { - fCompare = &StrFilterCmd::compare_cs; - } - else if (columns == CDC) // string : char[] - { - fCompare = &StrFilterCmd::compare_sc; - } - else - { - stringstream msg; - msg << "StrFilterCmd: unhandled column combination " << hex << columns << dec; - throw logic_error(msg.str()); - } + if (columns == CC) // char[] : char + { + fCompare = &StrFilterCmd::compare_cc; + } + else if (columns == DCDC) // string : string + { + fCompare = &StrFilterCmd::compare_ss; + } + else if (columns == DCC) // char[] : string + { + fCompare = &StrFilterCmd::compare_cs; + } + else if (columns == CDC) // string : char[] + { + fCompare = &StrFilterCmd::compare_sc; + } + else + { + stringstream msg; + msg << "StrFilterCmd: unhandled column combination " << hex << columns << dec; + throw logic_error(msg.str()); + } } - // TODO: // Move this function as a method to class Charset // and reuse it in: @@ -560,114 +485,97 @@ void StrFilterCmd::setCompareFunc(uint32_t columns) // a globally visible enum first, e.g. utils/common/mcs_basic_types.h // and all "fBOP" members in all classes should be changed to this enum. -static inline bool compareString(const datatypes::Charset &cs, - const utils::ConstString &s0, - const utils::ConstString &s1, - uint8_t fBOP) +static inline bool compareString(const datatypes::Charset& cs, const utils::ConstString& s0, + const utils::ConstString& s1, uint8_t fBOP) { - int cmp = cs.strnncollsp(s0, s1); - switch (fBOP) - { - case COMPARE_GT: - return cmp > 0; + int cmp = cs.strnncollsp(s0, s1); + switch (fBOP) + { + case COMPARE_GT: return cmp > 0; - case COMPARE_LT: - return cmp < 0; + case COMPARE_LT: return cmp < 0; - case COMPARE_EQ: - return cmp == 0; + case COMPARE_EQ: return cmp == 0; - case COMPARE_GE: - return cmp >= 0; + case COMPARE_GE: return cmp >= 0; - case COMPARE_LE: - return cmp <= 0; + case COMPARE_LE: return cmp <= 0; - case COMPARE_NE: - return cmp != 0; + case COMPARE_NE: return cmp != 0; - default: - break; - } - return false; + default: break; + } + return false; } - bool StrFilterCmd::compare_cc(uint64_t i, uint64_t j) { - if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || - execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) - return false; + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || + execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; - datatypes::Charset cs(leftColType.getCharset()); - datatypes::TCharShort s0(bpp->fFiltCmdValues[0][i]); - datatypes::TCharShort s1(bpp->fFiltCmdValues[1][j]); - return compareString(cs, s0.toConstString(leftColType.colWidth), - s1.toConstString(rightColType.colWidth), fBOP); + datatypes::Charset cs(leftColType.getCharset()); + datatypes::TCharShort s0(bpp->fFiltCmdValues[0][i]); + datatypes::TCharShort s1(bpp->fFiltCmdValues[1][j]); + return compareString(cs, s0.toConstString(leftColType.colWidth), s1.toConstString(rightColType.colWidth), + fBOP); } - bool StrFilterCmd::compare_ss(uint64_t i, uint64_t j) { - if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[1][j] == "" || - bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK) - return false; + if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[1][j] == "" || + bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || + bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK) + return false; - datatypes::Charset cs(leftColType.getCharset()); - utils::ConstString s0(utils::ConstString(bpp->fFiltStrValues[0][i])); - utils::ConstString s1(utils::ConstString(bpp->fFiltStrValues[1][j])); - return compareString(cs, s0, s1, fBOP); + datatypes::Charset cs(leftColType.getCharset()); + utils::ConstString s0(utils::ConstString(bpp->fFiltStrValues[0][i])); + utils::ConstString s1(utils::ConstString(bpp->fFiltStrValues[1][j])); + return compareString(cs, s0, s1, fBOP); } - bool StrFilterCmd::compare_cs(uint64_t i, uint64_t j) { - if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || - bpp->fFiltStrValues[1][j] == "" || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK) - return false; + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) || bpp->fFiltStrValues[1][j] == "" || + bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK) + return false; - datatypes::Charset cs(leftColType.getCharset()); - datatypes::TCharShort s0(bpp->fFiltCmdValues[0][i]); - utils::ConstString s1(bpp->fFiltStrValues[1][j]); - return compareString(cs, s0.toConstString(leftColType.colWidth), s1, fBOP); + datatypes::Charset cs(leftColType.getCharset()); + datatypes::TCharShort s0(bpp->fFiltCmdValues[0][i]); + utils::ConstString s1(bpp->fFiltStrValues[1][j]); + return compareString(cs, s0.toConstString(leftColType.colWidth), s1, fBOP); } - bool StrFilterCmd::compare_sc(uint64_t i, uint64_t j) { - if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || - execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) - return false; + if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || + execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; - datatypes::Charset cs(leftColType.getCharset()); - utils::ConstString s0(bpp->fFiltStrValues[0][i]); - datatypes::TCharShort s1(bpp->fFiltCmdValues[1][j]); - return compareString(cs, s0, s1.toConstString(rightColType.colWidth), fBOP); + datatypes::Charset cs(leftColType.getCharset()); + utils::ConstString s0(bpp->fFiltStrValues[0][i]); + datatypes::TCharShort s1(bpp->fFiltCmdValues[1][j]); + return compareString(cs, s0, s1.toConstString(rightColType.colWidth), fBOP); } - void StrFilterCmd::setCharLength(size_t l) { - fCharLength = l; + fCharLength = l; } - size_t StrFilterCmd::charLength() { - return fCharLength; + return fCharLength; } - bool StrFilterCmd::operator==(const StrFilterCmd& c) const { - return ((fBOP == c.fBOP) && fCharLength == c.fCharLength); + return ((fBOP == c.fBOP) && fCharLength == c.fCharLength); } - bool StrFilterCmd::operator!=(const StrFilterCmd& c) const { - return !(*this == c); + return !(*this == c); } - -}; +}; // namespace primitiveprocessor diff --git a/primitives/primproc/filtercommand.h b/primitives/primproc/filtercommand.h index 791c35c3e..280ce1265 100644 --- a/primitives/primproc/filtercommand.h +++ b/primitives/primproc/filtercommand.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: filtercommand.h 2035 2013-01-21 14:12:19Z rdempsey $ -* -* -***********************************************************************/ + * $Id: filtercommand.h 2035 2013-01-21 14:12:19Z rdempsey $ + * + * + ***********************************************************************/ /** @file * class FilterCommand interface */ @@ -33,140 +33,133 @@ #include "blocksize.h" #include "calpontsystemcatalog.h" - namespace primitiveprocessor { - class FilterCommand : public Command { -public: - FilterCommand(); - virtual ~FilterCommand(); + public: + FilterCommand(); + virtual ~FilterCommand(); - // returns a FilterCommand based on column types - static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector& cmds); + // returns a FilterCommand based on column types + static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector& cmds); - // virtuals from base class -- Command - void execute(); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); - uint64_t getLBID(); - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - SCommand duplicate(); - void prep(int8_t outputType, bool makeAbsRids); + // virtuals from base class -- Command + void execute(); + void project(); + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); + uint64_t getLBID(); + void nextLBID(); + void createCommand(messageqcpp::ByteStream&); + void resetCommand(messageqcpp::ByteStream&); + SCommand duplicate(); + void prep(int8_t outputType, bool makeAbsRids); - void setColTypes(const execplan::CalpontSystemCatalog::ColType& left, - const execplan::CalpontSystemCatalog::ColType& right); + void setColTypes(const execplan::CalpontSystemCatalog::ColType& left, + const execplan::CalpontSystemCatalog::ColType& right); - // operator override - bool operator==(const FilterCommand&) const; - bool operator!=(const FilterCommand&) const; + // operator override + bool operator==(const FilterCommand&) const; + bool operator!=(const FilterCommand&) const; - int getCompType() const - { - return 0; - } + int getCompType() const + { + return 0; + } -protected: - // filter operation - virtual void doFilter(); + protected: + // filter operation + virtual void doFilter(); - // compare method, take the indices to the values array - virtual bool compare(uint64_t, uint64_t); + // compare method, take the indices to the values array + virtual bool compare(uint64_t, uint64_t); - // compare method, take the indices to the values array - virtual bool binaryCompare(uint64_t, uint64_t); + // compare method, take the indices to the values array + virtual bool binaryCompare(uint64_t, uint64_t); - // binary operator - uint8_t fBOP; + // binary operator + uint8_t fBOP; - bool hasWideColumns; + bool hasWideColumns; - // column type for null check - execplan::CalpontSystemCatalog::ColType leftColType; - execplan::CalpontSystemCatalog::ColType rightColType; + // column type for null check + execplan::CalpontSystemCatalog::ColType leftColType; + execplan::CalpontSystemCatalog::ColType rightColType; - FilterCommand(const FilterCommand&rhs) = default; + FilterCommand(const FilterCommand& rhs) = default; -private: - // disabled copy operator - FilterCommand& operator=(const FilterCommand&); + private: + // disabled copy operator + FilterCommand& operator=(const FilterCommand&); }; - class ScaledFilterCmd : public FilterCommand { -public: - ScaledFilterCmd(); - virtual ~ScaledFilterCmd(); - SCommand duplicate(); + public: + ScaledFilterCmd(); + virtual ~ScaledFilterCmd(); + SCommand duplicate(); - void setFactor(double); - double factor(); + void setFactor(double); + double factor(); - // operator override - bool operator==(const ScaledFilterCmd&) const; - bool operator!=(const ScaledFilterCmd&) const; + // operator override + bool operator==(const ScaledFilterCmd&) const; + bool operator!=(const ScaledFilterCmd&) const; -protected: - // compare method, take the indices to the values array - bool compare(uint64_t, uint64_t); + protected: + // compare method, take the indices to the values array + bool compare(uint64_t, uint64_t); - // value used in comparison; - double fFactor; + // value used in comparison; + double fFactor; -private: - // disabled copy constructor and operator - ScaledFilterCmd(const ScaledFilterCmd&); - ScaledFilterCmd& operator=(const ScaledFilterCmd&); + private: + // disabled copy constructor and operator + ScaledFilterCmd(const ScaledFilterCmd&); + ScaledFilterCmd& operator=(const ScaledFilterCmd&); }; - class StrFilterCmd : public FilterCommand { -public: - StrFilterCmd(); - virtual ~StrFilterCmd(); + public: + StrFilterCmd(); + virtual ~StrFilterCmd(); - // override FilterCommand methods - void execute(); - SCommand duplicate(); + // override FilterCommand methods + void execute(); + SCommand duplicate(); - void setCompareFunc(uint32_t); - void setCharLength(size_t); - size_t charLength(); + void setCompareFunc(uint32_t); + void setCharLength(size_t); + size_t charLength(); - // operator override - bool operator==(const StrFilterCmd&) const; - bool operator!=(const StrFilterCmd&) const; + // operator override + bool operator==(const StrFilterCmd&) const; + bool operator!=(const StrFilterCmd&) const; -protected: - // compare method, take the indices to the values array - bool compare(uint64_t, uint64_t); + protected: + // compare method, take the indices to the values array + bool compare(uint64_t, uint64_t); - // compare method for differernt column combination, c--char[], s--string - // compare char[]-char[] is not the same as int-int due to endian issue. - bool compare_cc(uint64_t, uint64_t); - bool compare_ss(uint64_t, uint64_t); - bool compare_cs(uint64_t, uint64_t); - bool compare_sc(uint64_t, uint64_t); - bool (StrFilterCmd::*fCompare)(uint64_t, uint64_t); + // compare method for differernt column combination, c--char[], s--string + // compare char[]-char[] is not the same as int-int due to endian issue. + bool compare_cc(uint64_t, uint64_t); + bool compare_ss(uint64_t, uint64_t); + bool compare_cs(uint64_t, uint64_t); + bool compare_sc(uint64_t, uint64_t); + bool (StrFilterCmd::*fCompare)(uint64_t, uint64_t); - // colWidth of columns the don't need a dictionary - size_t fCharLength; + // colWidth of columns the don't need a dictionary + size_t fCharLength; - StrFilterCmd(const StrFilterCmd &rhs) = default; + StrFilterCmd(const StrFilterCmd& rhs) = default; -private: - // disabled copy operator - StrFilterCmd& operator=(const StrFilterCmd&); + private: + // disabled copy operator + StrFilterCmd& operator=(const StrFilterCmd&); }; +}; // namespace primitiveprocessor -}; - - -#endif // PRIMITIVES_FILTERCOMMAND_H_ - +#endif // PRIMITIVES_FILTERCOMMAND_H_ diff --git a/primitives/primproc/logger.cpp b/primitives/primproc/logger.cpp index 6bbe3310a..eedebe2be 100644 --- a/primitives/primproc/logger.cpp +++ b/primitives/primproc/logger.cpp @@ -31,58 +31,52 @@ using namespace logging; namespace primitiveprocessor { - -Logger::Logger() : - fMl1(LoggingID(28)) +Logger::Logger() : fMl1(LoggingID(28)) { - fMsgMap[logging::M0000] = Message(logging::M0000); - fMsgMap[logging::M0016] = Message(logging::M0016); - fMsgMap[logging::M0045] = Message(logging::M0045); - fMsgMap[logging::M0053] = Message(logging::M0053); - fMsgMap[logging::M0058] = Message(logging::M0058); - fMsgMap[logging::M0061] = Message(logging::M0061); - fMsgMap[logging::M0069] = Message(logging::M0069); - fMsgMap[logging::M0070] = Message(logging::M0070); - fMsgMap[logging::M0077] = Message(logging::M0077); + fMsgMap[logging::M0000] = Message(logging::M0000); + fMsgMap[logging::M0016] = Message(logging::M0016); + fMsgMap[logging::M0045] = Message(logging::M0045); + fMsgMap[logging::M0053] = Message(logging::M0053); + fMsgMap[logging::M0058] = Message(logging::M0058); + fMsgMap[logging::M0061] = Message(logging::M0061); + fMsgMap[logging::M0069] = Message(logging::M0069); + fMsgMap[logging::M0070] = Message(logging::M0070); + fMsgMap[logging::M0077] = Message(logging::M0077); } -void Logger::logMessage(const Message::MessageID mid, - const Message::Args& args, - bool critical) +void Logger::logMessage(const Message::MessageID mid, const Message::Args& args, bool critical) { - mutex::scoped_lock lk(fLogLock); - MsgMap::iterator msgIter = fMsgMap.find(mid); + mutex::scoped_lock lk(fLogLock); + MsgMap::iterator msgIter = fMsgMap.find(mid); - if (msgIter == fMsgMap.end()) - msgIter = fMsgMap.find(logging::M0000); + if (msgIter == fMsgMap.end()) + msgIter = fMsgMap.find(logging::M0000); - msgIter->second.reset(); - msgIter->second.format(args); + msgIter->second.reset(); + msgIter->second.format(args); - if (critical) - { - fMl1.logCriticalMessage(msgIter->second); - } - else - { - fMl1.logWarningMessage(msgIter->second); - } + if (critical) + { + fMl1.logCriticalMessage(msgIter->second); + } + else + { + fMl1.logWarningMessage(msgIter->second); + } } -void Logger::logInfoMessage(const Message::MessageID mid, - const Message::Args& args) +void Logger::logInfoMessage(const Message::MessageID mid, const Message::Args& args) { - mutex::scoped_lock lk(fLogLock); - MsgMap::iterator msgIter = fMsgMap.find(mid); + mutex::scoped_lock lk(fLogLock); + MsgMap::iterator msgIter = fMsgMap.find(mid); - if (msgIter == fMsgMap.end()) - msgIter = fMsgMap.find(logging::M0000); + if (msgIter == fMsgMap.end()) + msgIter = fMsgMap.find(logging::M0000); - msgIter->second.reset(); - msgIter->second.format(args); - - fMl1.logInfoMessage(msgIter->second); -} + msgIter->second.reset(); + msgIter->second.format(args); + fMl1.logInfoMessage(msgIter->second); } +} // namespace primitiveprocessor diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index 5062ce761..936d3466a 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -38,7 +38,6 @@ using namespace rowgroup; namespace primitiveprocessor { - PassThruCommand::PassThruCommand() : Command(PASS_THRU) { } @@ -49,23 +48,23 @@ PassThruCommand::~PassThruCommand() void PassThruCommand::prep(int8_t outputType, bool makeAbsRids) { - if (bpp->ot == ROW_GROUP) - { - bpp->outputRG.initRow(&r); - rowSize = r.getSize(); - } + if (bpp->ot == ROW_GROUP) + { + bpp->outputRG.initRow(&r); + rowSize = r.getSize(); + } } void PassThruCommand::execute() { -// throw logic_error("PassThruCommand isn't a filter step"); + // throw logic_error("PassThruCommand isn't a filter step"); } void PassThruCommand::project() { - uint32_t i; + uint32_t i; - *bpp->serialized << (uint32_t) (bpp->ridCount * colWidth); + *bpp->serialized << (uint32_t)(bpp->ridCount * colWidth); #if 0 cout << "pass thru serializing " << (uint32_t) (bpp->ridCount * colWidth) << " bytes:\n"; cout << "at relative position " << bpp->serialized->length() - sizeof(ISMPacketHeader) - sizeof(PrimitiveHeader) - 4 << endl; @@ -75,97 +74,92 @@ void PassThruCommand::project() #endif - switch (colWidth) - { - case 16: - bpp->serialized->append((uint8_t*) bpp->wide128Values, bpp->ridCount << 4); - break; - - case 8: - bpp->serialized->append((uint8_t*) bpp->values, bpp->ridCount << 3); - break; + switch (colWidth) + { + case 16: bpp->serialized->append((uint8_t*)bpp->wide128Values, bpp->ridCount << 4); break; - case 4: - for (i = 0; i < bpp->ridCount; i++) - *bpp->serialized << (uint32_t) bpp->values[i]; + case 8: bpp->serialized->append((uint8_t*)bpp->values, bpp->ridCount << 3); break; - break; + case 4: + for (i = 0; i < bpp->ridCount; i++) + *bpp->serialized << (uint32_t)bpp->values[i]; - case 2: - for (i = 0; i < bpp->ridCount; i++) - *bpp->serialized << (uint16_t) bpp->values[i]; + break; - break; + case 2: + for (i = 0; i < bpp->ridCount; i++) + *bpp->serialized << (uint16_t)bpp->values[i]; - case 1: - for (i = 0; i < bpp->ridCount; i++) - *bpp->serialized << (uint8_t) bpp->values[i]; + break; - break; + case 1: + for (i = 0; i < bpp->ridCount; i++) + *bpp->serialized << (uint8_t)bpp->values[i]; - default: - throw logic_error("PassThruCommand has a bad column width"); - } + break; + + default: throw logic_error("PassThruCommand has a bad column width"); + } } void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) { - uint32_t i; + uint32_t i; - rg.initRow(&r); - rg.getRow(0, &r); - uint32_t offset = r.getOffset(col); - rowSize = r.getSize(); + rg.initRow(&r); + rg.getRow(0, &r); + uint32_t offset = r.getOffset(col); + rowSize = r.getSize(); - switch (colWidth) - { - case 1: - for (i = 0; i < bpp->ridCount; i++) - { - r.setUintField_offset<1>(bpp->values[i], offset); - r.nextRow(rowSize); - } + switch (colWidth) + { + case 1: + for (i = 0; i < bpp->ridCount; i++) + { + r.setUintField_offset<1>(bpp->values[i], offset); + r.nextRow(rowSize); + } - break; + break; - case 2: - for (i = 0; i < bpp->ridCount; i++) - { - r.setUintField_offset<2>(bpp->values[i], offset); - r.nextRow(rowSize); - } + case 2: + for (i = 0; i < bpp->ridCount; i++) + { + r.setUintField_offset<2>(bpp->values[i], offset); + r.nextRow(rowSize); + } - break; + break; - case 4: - for (i = 0; i < bpp->ridCount; i++) - { - r.setUintField_offset<4>(bpp->values[i], offset); - r.nextRow(rowSize); - } + case 4: + for (i = 0; i < bpp->ridCount; i++) + { + r.setUintField_offset<4>(bpp->values[i], offset); + r.nextRow(rowSize); + } - break; + break; - case 8: - for (i = 0; i < bpp->ridCount; i++) - { - r.setUintField_offset<8>(bpp->values[i], offset); - r.nextRow(rowSize); - } + case 8: + for (i = 0; i < bpp->ridCount; i++) + { + r.setUintField_offset<8>(bpp->values[i], offset); + r.nextRow(rowSize); + } - break; - case 16: - for (i = 0; i < bpp->ridCount; i++) - { - r.setBinaryField_offset(&bpp->wide128Values[i], 16, offset); - r.nextRow(rowSize); - } - } + break; + case 16: + for (i = 0; i < bpp->ridCount; i++) + { + r.setBinaryField_offset(&bpp->wide128Values[i], 16, offset); + r.nextRow(rowSize); + } + } } uint64_t PassThruCommand::getLBID() { - return 0; + return 0; } void PassThruCommand::nextLBID() @@ -174,9 +168,9 @@ void PassThruCommand::nextLBID() void PassThruCommand::createCommand(ByteStream& bs) { - bs.advance(1); - bs >> colWidth; - Command::createCommand(bs); + bs.advance(1); + bs >> colWidth; + Command::createCommand(bs); } void PassThruCommand::resetCommand(ByteStream& bs) @@ -185,27 +179,27 @@ void PassThruCommand::resetCommand(ByteStream& bs) SCommand PassThruCommand::duplicate() { - SCommand ret; - PassThruCommand* p; + SCommand ret; + PassThruCommand* p; - ret.reset(new PassThruCommand()); - p = (PassThruCommand*) ret.get(); - p->colWidth = colWidth; - p->Command::operator=(*this); - return ret; + ret.reset(new PassThruCommand()); + p = (PassThruCommand*)ret.get(); + p->colWidth = colWidth; + p->Command::operator=(*this); + return ret; } bool PassThruCommand::operator==(const PassThruCommand& p) const { - if (colWidth != p.colWidth) - return false; + if (colWidth != p.colWidth) + return false; - return true; + return true; } bool PassThruCommand::operator!=(const PassThruCommand& p) const { - return !(*this == p); + return !(*this == p); } -}; +}; // namespace primitiveprocessor diff --git a/primitives/primproc/passthrucommand.h b/primitives/primproc/passthrucommand.h index 9aae9e996..672fddb37 100644 --- a/primitives/primproc/passthrucommand.h +++ b/primitives/primproc/passthrucommand.h @@ -35,39 +35,39 @@ namespace primitiveprocessor { - class PassThruCommand : public Command { -public: - PassThruCommand(); - virtual ~PassThruCommand(); + public: + PassThruCommand(); + virtual ~PassThruCommand(); - void prep(int8_t outputType, bool makeAbsRids); - void execute(); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); - uint64_t getLBID(); - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - SCommand duplicate(); - bool operator==(const PassThruCommand&) const; - bool operator!=(const PassThruCommand&) const; + void prep(int8_t outputType, bool makeAbsRids); + void execute(); + void project(); + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); + uint64_t getLBID(); + void nextLBID(); + void createCommand(messageqcpp::ByteStream&); + void resetCommand(messageqcpp::ByteStream&); + SCommand duplicate(); + bool operator==(const PassThruCommand&) const; + bool operator!=(const PassThruCommand&) const; - int getCompType() const - { - return 0; - } -private: - PassThruCommand(const PassThruCommand&); + int getCompType() const + { + return 0; + } - uint8_t colWidth; + private: + PassThruCommand(const PassThruCommand&); - /* Minor optimization for projectIntoRowGroup() */ - rowgroup::Row r; - uint32_t rowSize; + uint8_t colWidth; + + /* Minor optimization for projectIntoRowGroup() */ + rowgroup::Row r; + uint32_t rowSize; }; -} +} // namespace primitiveprocessor #endif diff --git a/primitives/primproc/pp_logger.h b/primitives/primproc/pp_logger.h index fcfc18fcc..ac2181996 100644 --- a/primitives/primproc/pp_logger.h +++ b/primitives/primproc/pp_logger.h @@ -32,41 +32,36 @@ namespace primitiveprocessor { - /** @brief message log wrapper class */ class Logger { -public: - Logger(); + public: + Logger(); - // critical=true logs CRITICAL; critical=false logs WARNING - void logMessage(const logging::Message::MessageID mid, - const logging::Message::Args& args, - bool critical = false); - void logInfoMessage(const logging::Message::MessageID mid, - const logging::Message::Args& args); + // critical=true logs CRITICAL; critical=false logs WARNING + void logMessage(const logging::Message::MessageID mid, const logging::Message::Args& args, + bool critical = false); + void logInfoMessage(const logging::Message::MessageID mid, const logging::Message::Args& args); - void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 ) - { - logging::Message::Args args; - args.add(msg); - logMessage(mid, args, critical); - } + void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0) + { + logging::Message::Args args; + args.add(msg); + logMessage(mid, args, critical); + } -private: - // defaults okay - //Logger(const Logger& rhs); - //Logger& operator=(const Logger& rhs); + private: + // defaults okay + // Logger(const Logger& rhs); + // Logger& operator=(const Logger& rhs); - typedef std::map MsgMap; + typedef std::map MsgMap; - MsgMap fMsgMap; - boost::mutex fLogLock; - logging::MessageLog fMl1; + MsgMap fMsgMap; + boost::mutex fLogLock; + logging::MessageLog fMl1; }; - -} +} // namespace primitiveprocessor #endif - diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index ae5824243..99aa7a315 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -95,16 +95,16 @@ using namespace threadpool; #include "atomicops.h" #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_DIRECT -# define O_DIRECT 0 +#define O_DIRECT 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif typedef tr1::unordered_set USOID; @@ -121,7 +121,7 @@ dbbc::Stats pmstats(statsName); oam::OamCache* oamCache = oam::OamCache::makeOamCache(); -//FIXME: there is an anon ns burried later in between 2 named namespaces... +// FIXME: there is an anon ns burried later in between 2 named namespaces... namespace primitiveprocessor { boost::shared_ptr OOBPool; @@ -144,32 +144,34 @@ uint32_t connectionsPerUM; uint32_t highPriorityThreads; uint32_t medPriorityThreads; uint32_t lowPriorityThreads; -int directIOFlag = O_DIRECT; -int noVB = 0; +int directIOFlag = O_DIRECT; +int noVB = 0; BPPMap bppMap; boost::mutex bppLock; #define DJLOCK_READ 0 #define DJLOCK_WRITE 1 -boost::mutex djMutex; // lock for djLock, lol. -std::map djLock; // djLock synchronizes destroy and joiner msgs, see bug 2619 +boost::mutex djMutex; // lock for djLock, lol. +std::map djLock; // djLock synchronizes destroy and joiner msgs, see bug 2619 volatile int32_t asyncCounter; -const int asyncMax = 20; // current number of asynchronous loads +const int asyncMax = 20; // current number of asynchronous loads struct preFetchCond { - //uint64_t lbid; - boost::condition cond; - unsigned waiters; + // uint64_t lbid; + boost::condition cond; + unsigned waiters; - preFetchCond(const uint64_t l) - { - waiters = 0; - } + preFetchCond(const uint64_t l) + { + waiters = 0; + } - ~preFetchCond() { } + ~preFetchCond() + { + } }; typedef preFetchCond preFetchBlock_t; @@ -177,932 +179,868 @@ typedef std::tr1::unordered_map pfBlockMap_t; typedef std::tr1::unordered_map::iterator pfBlockMapIter_t; pfBlockMap_t pfBlockMap; -boost::mutex pfbMutex; // = PTHREAD_MUTEX_INITIALIZER; +boost::mutex pfbMutex; // = PTHREAD_MUTEX_INITIALIZER; pfBlockMap_t pfExtentMap; -boost::mutex pfMutex; // = PTHREAD_MUTEX_INITIALIZER; +boost::mutex pfMutex; // = PTHREAD_MUTEX_INITIALIZER; map > dictEqualityFilters; boost::mutex eqFilterMutex; uint32_t cacheNum(uint64_t lbid) { - return ( lbid / brm->getExtentSize() ) % fCacheCount; + return (lbid / brm->getExtentSize()) % fCacheCount; } -void buildOidFileName(const BRM::OID_t oid, - const uint16_t dbRoot, - const uint16_t partNum, - const uint32_t segNum, - char* file_name) +void buildOidFileName(const BRM::OID_t oid, const uint16_t dbRoot, const uint16_t partNum, + const uint32_t segNum, char* file_name) { - WriteEngine::FileOp fileOp(false); + WriteEngine::FileOp fileOp(false); - if (fileOp.getFileName(oid, file_name, dbRoot, partNum, segNum) != WriteEngine::NO_ERROR) - { - file_name[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } + if (fileOp.getFileName(oid, file_name, dbRoot, partNum, segNum) != WriteEngine::NO_ERROR) + { + file_name[0] = 0; + throw std::runtime_error("fileOp.getFileName failed"); + } - //cout << "Oid2Filename o: " << oid << " n: " << file_name << endl; + // cout << "Oid2Filename o: " << oid << " n: " << file_name << endl; } - void waitForRetry(long count) { - timespec ts; - ts.tv_sec = 5L * count / 10L; - ts.tv_nsec = (5L * count % 10L) * 100000000L; + timespec ts; + ts.tv_sec = 5L * count / 10L; + ts.tv_nsec = (5L * count % 10L) * 100000000L; #ifdef _MSC_VER - Sleep(ts.tv_sec * 1000 + ts.tv_nsec / 1000 / 1000); + Sleep(ts.tv_sec * 1000 + ts.tv_nsec / 1000 / 1000); #else - nanosleep(&ts, 0); + nanosleep(&ts, 0); #endif } - -void prefetchBlocks(const uint64_t lbid, - const int compType, - uint32_t* rCount) +void prefetchBlocks(const uint64_t lbid, const int compType, uint32_t* rCount) { - uint16_t dbRoot; - uint32_t partNum; - uint16_t segNum; - uint32_t hwm; - uint32_t fbo; - uint32_t lowfbo; - uint32_t highfbo; - BRM::OID_t oid; - pfBlockMap_t::const_iterator iter; - uint64_t lowlbid = (lbid / blocksReadAhead) * blocksReadAhead; - blockCacheClient bc(*BRPp[cacheNum(lbid)]); - BRM::InlineLBIDRange range; - int err; + uint16_t dbRoot; + uint32_t partNum; + uint16_t segNum; + uint32_t hwm; + uint32_t fbo; + uint32_t lowfbo; + uint32_t highfbo; + BRM::OID_t oid; + pfBlockMap_t::const_iterator iter; + uint64_t lowlbid = (lbid / blocksReadAhead) * blocksReadAhead; + blockCacheClient bc(*BRPp[cacheNum(lbid)]); + BRM::InlineLBIDRange range; + int err; - pfbMutex.lock(); + pfbMutex.lock(); - iter = pfBlockMap.find(lowlbid); + iter = pfBlockMap.find(lowlbid); - if (iter != pfBlockMap.end()) - { - iter->second->waiters++; - iter->second->cond.wait(pfbMutex); - iter->second->waiters--; - pfbMutex.unlock(); - return; - } - - preFetchBlock_t* pfb = 0; - pfb = new preFetchBlock_t(lowlbid); - - pfBlockMap[lowlbid] = pfb; + if (iter != pfBlockMap.end()) + { + iter->second->waiters++; + iter->second->cond.wait(pfbMutex); + iter->second->waiters--; pfbMutex.unlock(); + return; + } - // loadBlock will catch a versioned block so vbflag can be set to false here - err = brm->lookupLocal(lbid, 0, false, oid, dbRoot, partNum, segNum, fbo); // need the oid + preFetchBlock_t* pfb = 0; + pfb = new preFetchBlock_t(lowlbid); - if (err < 0) + pfBlockMap[lowlbid] = pfb; + pfbMutex.unlock(); + + // loadBlock will catch a versioned block so vbflag can be set to false here + err = brm->lookupLocal(lbid, 0, false, oid, dbRoot, partNum, segNum, fbo); // need the oid + + if (err < 0) + { + cerr << "prefetchBlocks(): BRM lookupLocal failed! Expect more errors.\n"; + goto cleanup; + } + + // We ignore extState that tells us whether the extent is + // an outOfService extent to be ignored. The filtering for + // outOfService extents is done elsewhere. + int extState; + err = brm->getLocalHWM(oid, partNum, segNum, hwm, extState); + + if (err < 0) + { + cerr << "prefetchBlock(): BRM getLocalHWM failed! Expect more errors.\n"; + goto cleanup; + } + + lowfbo = fbo - (lbid - lowlbid); + highfbo = lowfbo + blocksReadAhead - 1; + range.start = lowlbid; + + if (hwm < highfbo) + range.size = hwm - lowfbo + 1; + else + range.size = blocksReadAhead; + + try + { + if (range.size > blocksReadAhead) { - cerr << "prefetchBlocks(): BRM lookupLocal failed! Expect more errors.\n"; - goto cleanup; + ostringstream os; + os << "Invalid Range from HWM for lbid " << lbid << ", range size should be <= blocksReadAhead: HWM " + << hwm << ", dbroot " << dbRoot << ", highfbo " << highfbo << ", lowfbo " << lowfbo + << ", blocksReadAhead " << blocksReadAhead << ", range size " << (int)range.size << endl; + throw logging::InvalidRangeHWMExcept(os.str()); } - // We ignore extState that tells us whether the extent is - // an outOfService extent to be ignored. The filtering for - // outOfService extents is done elsewhere. - int extState; - err = brm->getLocalHWM(oid, partNum, segNum, hwm, extState); + idbassert(range.size <= blocksReadAhead); - if (err < 0) - { - cerr << "prefetchBlock(): BRM getLocalHWM failed! Expect more errors.\n"; - goto cleanup; - } - - lowfbo = fbo - (lbid - lowlbid); - highfbo = lowfbo + blocksReadAhead - 1; - range.start = lowlbid; - - if (hwm < highfbo) - range.size = hwm - lowfbo + 1; - else - range.size = blocksReadAhead; - - try - { - if (range.size > blocksReadAhead) - { - ostringstream os; - os << "Invalid Range from HWM for lbid " << lbid - << ", range size should be <= blocksReadAhead: HWM " << hwm - << ", dbroot " << dbRoot - << ", highfbo " << highfbo << ", lowfbo " << lowfbo - << ", blocksReadAhead " << blocksReadAhead - << ", range size " << (int) range.size << endl; - throw logging::InvalidRangeHWMExcept(os.str()); - } - - idbassert(range.size <= blocksReadAhead); - - bc.check(range, QueryContext(numeric_limits::max()), 0, compType, *rCount); - } - catch (...) - { - // Perform necessary cleanup before rethrowing the exception - pfb->cond.notify_all(); - - pfbMutex.lock(); - - while (pfb->waiters > 0) - { - pfbMutex.unlock(); - //handle race condition with other threads going into wait before the broadcast above - pfb->cond.notify_one(); - usleep(1); - pfbMutex.lock(); - } - - if (pfBlockMap.erase(lowlbid) > 0) - delete pfb; - - pfb = 0; - pfbMutex.unlock(); - throw; - } - -cleanup: + bc.check(range, QueryContext(numeric_limits::max()), 0, compType, *rCount); + } + catch (...) + { + // Perform necessary cleanup before rethrowing the exception pfb->cond.notify_all(); pfbMutex.lock(); while (pfb->waiters > 0) { - pfbMutex.unlock(); - //handle race condition with other threads going into wait before the broadcast above - pfb->cond.notify_one(); - usleep(1); - pfbMutex.lock(); + pfbMutex.unlock(); + // handle race condition with other threads going into wait before the broadcast above + pfb->cond.notify_one(); + usleep(1); + pfbMutex.lock(); } if (pfBlockMap.erase(lowlbid) > 0) - delete pfb; + delete pfb; pfb = 0; pfbMutex.unlock(); + throw; + } -} // prefetchBlocks() +cleanup: + pfb->cond.notify_all(); + + pfbMutex.lock(); + + while (pfb->waiters > 0) + { + pfbMutex.unlock(); + // handle race condition with other threads going into wait before the broadcast above + pfb->cond.notify_one(); + usleep(1); + pfbMutex.lock(); + } + + if (pfBlockMap.erase(lowlbid) > 0) + delete pfb; + + pfb = 0; + pfbMutex.unlock(); + +} // prefetchBlocks() // returns the # that were cached. -uint32_t loadBlocks ( - LBID_t* lbids, - QueryContext qc, - VER_t txn, - int compType, - uint8_t** bufferPtrs, - uint32_t* rCount, - bool LBIDTrace, - uint32_t sessionID, - uint32_t blockCount, - bool* blocksWereVersioned, - bool doPrefetch, - VSSCache* vssCache) +uint32_t loadBlocks(LBID_t* lbids, QueryContext qc, VER_t txn, int compType, uint8_t** bufferPtrs, + uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, uint32_t blockCount, + bool* blocksWereVersioned, bool doPrefetch, VSSCache* vssCache) { - blockCacheClient bc(*BRPp[cacheNum(lbids[0])]); - uint32_t blksRead = 0; - VSSCache::iterator it; - uint32_t i, ret; - bool* vbFlags; - int* vssRCs; - bool* cacheThisBlock; - bool* wasCached; + blockCacheClient bc(*BRPp[cacheNum(lbids[0])]); + uint32_t blksRead = 0; + VSSCache::iterator it; + uint32_t i, ret; + bool* vbFlags; + int* vssRCs; + bool* cacheThisBlock; + bool* wasCached; - *blocksWereVersioned = false; + *blocksWereVersioned = false; #ifndef _MSC_VER - if (LBIDTrace) - { - for (i = 0; i < blockCount; i++) - { - - stats.touchedLBID(lbids[i], pthread_self(), sessionID); - } - } - -#endif - VER_t* vers = (VER_t*) alloca(blockCount * sizeof(VER_t)); - vbFlags = (bool*) alloca(blockCount); - vssRCs = (int*) alloca(blockCount * sizeof(int)); - cacheThisBlock = (bool*) alloca(blockCount); - wasCached = (bool*) alloca(blockCount); - + if (LBIDTrace) + { for (i = 0; i < blockCount; i++) { - if (vssCache) - { - it = vssCache->find(lbids[i]); - - if (it != vssCache->end()) - { - VSSData& vd = it->second; - vers[i] = vd.verID; - vbFlags[i] = vd.vbFlag; - vssRCs[i] = vd.returnCode; - - if (vssRCs[i] == ERR_SNAPSHOT_TOO_OLD) - throw runtime_error("Snapshot too old"); - } - } - - if (!vssCache || it == vssCache->end()) - vssRCs[i] = brm->vssLookup(lbids[i], qc, txn, &vers[i], &vbFlags[i]); - - *blocksWereVersioned |= vbFlags[i]; - - // If the block is being modified by this txn, set the useCache flag to false - if (txn > 0 && vers[i] == txn && !vbFlags[i]) - cacheThisBlock[i] = false; - else - cacheThisBlock[i] = true; + stats.touchedLBID(lbids[i], pthread_self(), sessionID); } - - /* - cout << " resolved ver #s: "; - for (uint32_t i = 0; i < blockCount; i++) - cout << " <" << vers[i] << ", " << (int) vbFlags[i] << ", " << (int) - cacheThisBlock[i] << ">"; - cout << endl; - */ - - ret = bc.getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount); - - // Do we want to check any VB flags here? Initial thought: no, because we have - // no idea whether any other blocks in the prefetch range are versioned, - // what's the difference if one in the visible range is? - if (ret != blockCount && doPrefetch) - { - prefetchBlocks(lbids[0], compType, &blksRead); - -#ifndef _MSC_VER - - if (fPMProfOn) - pmstats.markEvent(lbids[0], (pthread_t) - 1, sessionID, 'M'); - -#endif - /* After the prefetch they're all cached if they are in the same range, so - * prune the block list and try getCachedBlocks again first, then fall back - * to single-block IO requests if for some reason they aren't. */ - uint32_t l_blockCount = 0; - - for (i = 0; i < blockCount; i++) - { - if (!wasCached[i]) - { - lbids[l_blockCount] = lbids[i]; - vers[l_blockCount] = vers[i]; - bufferPtrs[l_blockCount] = bufferPtrs[i]; - vbFlags[l_blockCount] = vbFlags[i]; - cacheThisBlock[l_blockCount] = cacheThisBlock[i]; - ++l_blockCount; - } - } - - ret += bc.getCachedBlocks(lbids, vers, bufferPtrs, wasCached, l_blockCount); - - if (ret != blockCount) - { - for (i = 0; i < l_blockCount; i++) - if (!wasCached[i]) - { - bool ver; - - qc.currentScn = vers[i]; - bc.getBlock(lbids[i], qc, txn, compType, (void*) bufferPtrs[i], - vbFlags[i], wasCached[i], &ver, cacheThisBlock[i], false); - *blocksWereVersioned |= ver; - blksRead++; - } - } - } - /* Some blocks weren't cached, prefetch is disabled -> issue single-block IO requests, - * skip checking the cache again. */ - else if (ret != blockCount) - { - for (i = 0; i < blockCount; i++) - { - if (!wasCached[i]) - { - bool ver; - - qc.currentScn = vers[i]; - bc.getBlock(lbids[i], qc, txn, compType, (void*) bufferPtrs[i], vbFlags[i], - wasCached[i], &ver, cacheThisBlock[i], false); - *blocksWereVersioned |= ver; - blksRead++; - } - } - } - - if (rCount) - *rCount = blksRead; - - //if (*verBlocks) - // cout << "loadBlock says there were versioned blocks\n"; - return ret; -} - -void loadBlock ( - uint64_t lbid, - QueryContext v, - uint32_t t, - int compType, - void* bufferPtr, - bool* pWasBlockInCache, - uint32_t* rCount, - bool LBIDTrace, - uint32_t sessionID, - bool doPrefetch, - VSSCache* vssCache) -{ - bool flg = false; - BRM::OID_t oid; - BRM::VER_t txn = (BRM::VER_t)t; - uint16_t dbRoot = 0; - uint32_t partitionNum = 0; - uint16_t segmentNum = 0; - int rc; - BRM::VER_t ver; - blockCacheClient bc(*BRPp[cacheNum(lbid)]); - char file_name[WriteEngine::FILE_NAME_SIZE] = {0}; - char* fileNamePtr = file_name; - uint32_t blksRead = 0; - VSSCache::iterator it; - -#ifndef _MSC_VER - - if (LBIDTrace) - stats.touchedLBID(lbid, pthread_self(), sessionID); + } #endif + VER_t* vers = (VER_t*)alloca(blockCount * sizeof(VER_t)); + vbFlags = (bool*)alloca(blockCount); + vssRCs = (int*)alloca(blockCount * sizeof(int)); + cacheThisBlock = (bool*)alloca(blockCount); + wasCached = (bool*)alloca(blockCount); + for (i = 0; i < blockCount; i++) + { if (vssCache) { - it = vssCache->find(lbid); + it = vssCache->find(lbids[i]); - if (it != vssCache->end()) - { - VSSData& vd = it->second; - ver = vd.verID; - flg = vd.vbFlag; - rc = vd.returnCode; - } + if (it != vssCache->end()) + { + VSSData& vd = it->second; + vers[i] = vd.verID; + vbFlags[i] = vd.vbFlag; + vssRCs[i] = vd.returnCode; + + if (vssRCs[i] == ERR_SNAPSHOT_TOO_OLD) + throw runtime_error("Snapshot too old"); + } } if (!vssCache || it == vssCache->end()) - rc = brm->vssLookup((BRM::LBID_t)lbid, v, txn, &ver, &flg); + vssRCs[i] = brm->vssLookup(lbids[i], qc, txn, &vers[i], &vbFlags[i]); - v.currentScn = ver; - //cout << "VSS l/u: l=" << lbid << " v=" << ver << " t=" << txn << " flg=" << flg << " rc: " << rc << endl; + *blocksWereVersioned |= vbFlags[i]; - // if this block is locked by this session, don't cache it, just read it directly from disk - if (txn > 0 && ver == txn && !flg && !noVB) + // If the block is being modified by this txn, set the useCache flag to false + if (txn > 0 && vers[i] == txn && !vbFlags[i]) + cacheThisBlock[i] = false; + else + cacheThisBlock[i] = true; + } + + /* + cout << " resolved ver #s: "; + for (uint32_t i = 0; i < blockCount; i++) + cout << " <" << vers[i] << ", " << (int) vbFlags[i] << ", " << (int) + cacheThisBlock[i] << ">"; + cout << endl; + */ + + ret = bc.getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount); + + // Do we want to check any VB flags here? Initial thought: no, because we have + // no idea whether any other blocks in the prefetch range are versioned, + // what's the difference if one in the visible range is? + if (ret != blockCount && doPrefetch) + { + prefetchBlocks(lbids[0], compType, &blksRead); + +#ifndef _MSC_VER + + if (fPMProfOn) + pmstats.markEvent(lbids[0], (pthread_t)-1, sessionID, 'M'); + +#endif + /* After the prefetch they're all cached if they are in the same range, so + * prune the block list and try getCachedBlocks again first, then fall back + * to single-block IO requests if for some reason they aren't. */ + uint32_t l_blockCount = 0; + + for (i = 0; i < blockCount; i++) { - uint64_t offset; - uint32_t fbo; - boost::scoped_array newBufferSa; - boost::scoped_array cmpHdrBufSa; - boost::scoped_array cmpBufSa; - boost::scoped_array uCmpBufSa; + if (!wasCached[i]) + { + lbids[l_blockCount] = lbids[i]; + vers[l_blockCount] = vers[i]; + bufferPtrs[l_blockCount] = bufferPtrs[i]; + vbFlags[l_blockCount] = vbFlags[i]; + cacheThisBlock[l_blockCount] = cacheThisBlock[i]; + ++l_blockCount; + } + } + ret += bc.getCachedBlocks(lbids, vers, bufferPtrs, wasCached, l_blockCount); - - ptrdiff_t alignedBuffer = 0; - void* readBufferPtr = NULL; - char* cmpHdrBuf = NULL; - char* cmpBuf = NULL; - unsigned char* uCmpBuf = NULL; - uint64_t cmpBufLen = 0; - int blockReadRetryCount = 0; - unsigned idx = 0; - int pageSize = getpagesize(); - IDBDataFile* fp = 0; - - try + if (ret != blockCount) + { + for (i = 0; i < l_blockCount; i++) + if (!wasCached[i]) { - rc = brm->lookupLocal((BRM::LBID_t)lbid, - ver, - flg, - oid, - dbRoot, - partitionNum, - segmentNum, - fbo); + bool ver; + qc.currentScn = vers[i]; + bc.getBlock(lbids[i], qc, txn, compType, (void*)bufferPtrs[i], vbFlags[i], wasCached[i], &ver, + cacheThisBlock[i], false); + *blocksWereVersioned |= ver; + blksRead++; + } + } + } + /* Some blocks weren't cached, prefetch is disabled -> issue single-block IO requests, + * skip checking the cache again. */ + else if (ret != blockCount) + { + for (i = 0; i < blockCount; i++) + { + if (!wasCached[i]) + { + bool ver; - // load the block - buildOidFileName(oid, dbRoot, partitionNum, segmentNum, fileNamePtr); - int opts = directIOFlag ? IDBDataFile::USE_ODIRECT : 0; - fp = IDBDataFile::open( - IDBPolicy::getType( fileNamePtr, IDBPolicy::PRIMPROC ), - fileNamePtr, - "r", - opts); + qc.currentScn = vers[i]; + bc.getBlock(lbids[i], qc, txn, compType, (void*)bufferPtrs[i], vbFlags[i], wasCached[i], &ver, + cacheThisBlock[i], false); + *blocksWereVersioned |= ver; + blksRead++; + } + } + } - if ( fp == NULL ) - { - int errCode = errno; - SUMMARY_INFO2("open failed: ", fileNamePtr); - char errbuf[80]; - string errMsg; + if (rCount) + *rCount = blksRead; + + // if (*verBlocks) + // cout << "loadBlock says there were versioned blocks\n"; + return ret; +} + +void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bufferPtr, + bool* pWasBlockInCache, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, bool doPrefetch, + VSSCache* vssCache) +{ + bool flg = false; + BRM::OID_t oid; + BRM::VER_t txn = (BRM::VER_t)t; + uint16_t dbRoot = 0; + uint32_t partitionNum = 0; + uint16_t segmentNum = 0; + int rc; + BRM::VER_t ver; + blockCacheClient bc(*BRPp[cacheNum(lbid)]); + char file_name[WriteEngine::FILE_NAME_SIZE] = {0}; + char* fileNamePtr = file_name; + uint32_t blksRead = 0; + VSSCache::iterator it; + +#ifndef _MSC_VER + + if (LBIDTrace) + stats.touchedLBID(lbid, pthread_self(), sessionID); + +#endif + + if (vssCache) + { + it = vssCache->find(lbid); + + if (it != vssCache->end()) + { + VSSData& vd = it->second; + ver = vd.verID; + flg = vd.vbFlag; + rc = vd.returnCode; + } + } + + if (!vssCache || it == vssCache->end()) + rc = brm->vssLookup((BRM::LBID_t)lbid, v, txn, &ver, &flg); + + v.currentScn = ver; + // cout << "VSS l/u: l=" << lbid << " v=" << ver << " t=" << txn << " flg=" << flg << " rc: " << rc << endl; + + // if this block is locked by this session, don't cache it, just read it directly from disk + if (txn > 0 && ver == txn && !flg && !noVB) + { + uint64_t offset; + uint32_t fbo; + boost::scoped_array newBufferSa; + boost::scoped_array cmpHdrBufSa; + boost::scoped_array cmpBufSa; + boost::scoped_array uCmpBufSa; + + ptrdiff_t alignedBuffer = 0; + void* readBufferPtr = NULL; + char* cmpHdrBuf = NULL; + char* cmpBuf = NULL; + unsigned char* uCmpBuf = NULL; + uint64_t cmpBufLen = 0; + int blockReadRetryCount = 0; + unsigned idx = 0; + int pageSize = getpagesize(); + IDBDataFile* fp = 0; + + try + { + rc = brm->lookupLocal((BRM::LBID_t)lbid, ver, flg, oid, dbRoot, partitionNum, segmentNum, fbo); + + // load the block + buildOidFileName(oid, dbRoot, partitionNum, segmentNum, fileNamePtr); + int opts = directIOFlag ? IDBDataFile::USE_ODIRECT : 0; + fp = IDBDataFile::open(IDBPolicy::getType(fileNamePtr, IDBPolicy::PRIMPROC), fileNamePtr, "r", opts); + + if (fp == NULL) + { + int errCode = errno; + SUMMARY_INFO2("open failed: ", fileNamePtr); + char errbuf[80]; + string errMsg; #ifdef __linux__ -//#if STRERROR_R_CHAR_P - const char* p; + //#if STRERROR_R_CHAR_P + const char* p; - if ((p = strerror_r(errCode, errbuf, 80)) != 0) - errMsg = p; + if ((p = strerror_r(errCode, errbuf, 80)) != 0) + errMsg = p; #else - int p; + int p; - if ((p = strerror_r(errCode, errbuf, 80)) == 0) - errMsg = errbuf; + if ((p = strerror_r(errCode, errbuf, 80)) == 0) + errMsg = errbuf; #endif #ifndef _MSC_VER - if (errCode == EINVAL) - { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()-> - errorMsg(logging::ERR_O_DIRECT), logging::ERR_O_DIRECT); - } + if (errCode == EINVAL) + { + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT), + logging::ERR_O_DIRECT); + } #endif - string errStr(fileNamePtr); - errStr += ": open: "; - errStr += errMsg; - throw std::runtime_error(errStr); - } + string errStr(fileNamePtr); + errStr += ": open: "; + errStr += errMsg; + throw std::runtime_error(errStr); + } - // fd >= 0 must be true, otherwise above exception thrown. - offset = (uint64_t)fbo * (uint64_t)DATA_BLOCK_SIZE; - idx = offset / (4 * 1024 * 1024); + // fd >= 0 must be true, otherwise above exception thrown. + offset = (uint64_t)fbo * (uint64_t)DATA_BLOCK_SIZE; + idx = offset / (4 * 1024 * 1024); - errno = 0; - rc = 0; - int i = -1; + errno = 0; + rc = 0; + int i = -1; + if (compType == 0) + { + newBufferSa.reset(new uint8_t[DATA_BLOCK_SIZE + pageSize]); + alignedBuffer = (ptrdiff_t)newBufferSa.get(); - if (compType == 0) - { - newBufferSa.reset(new uint8_t[DATA_BLOCK_SIZE + pageSize]); - alignedBuffer = (ptrdiff_t) newBufferSa.get(); + if ((alignedBuffer % pageSize) != 0) + { + alignedBuffer &= ~((ptrdiff_t)pageSize - 1); + alignedBuffer += pageSize; + } - if ((alignedBuffer % pageSize) != 0) - { - alignedBuffer &= ~((ptrdiff_t)pageSize - 1); - alignedBuffer += pageSize; - } - - readBufferPtr = (void*) alignedBuffer; - i = fp->pread( readBufferPtr, offset, DATA_BLOCK_SIZE ); - memcpy(bufferPtr, readBufferPtr, i); + readBufferPtr = (void*)alignedBuffer; + i = fp->pread(readBufferPtr, offset, DATA_BLOCK_SIZE); + memcpy(bufferPtr, readBufferPtr, i); #ifdef IDB_COMP_POC_DEBUG - { - boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); - cout << "pread2(" << fd << ", 0x" << hex << (ptrdiff_t)readBufferPtr << dec << ", " << DATA_BLOCK_SIZE << ", " << offset << ") = " << i << endl; - } -#endif // IDB_COMP_POC_DEBUG - } // if (compType == 0) - else // if (compType != 0) - { -// retry if file is out of sync -- compressed column file only. -blockReadRetry: - - uCmpBufSa.reset(new unsigned char[4 * 1024 * 1024 + 4]); - uCmpBuf = uCmpBufSa.get(); - cmpHdrBufSa.reset(new char[4096 * 3 + pageSize]); - alignedBuffer = (ptrdiff_t) cmpHdrBufSa.get(); - - if ((alignedBuffer % pageSize) != 0) - { - alignedBuffer &= ~((ptrdiff_t)pageSize - 1); - alignedBuffer += pageSize; - } - - cmpHdrBuf = (char*) alignedBuffer; - - i = fp->pread( &cmpHdrBuf[0], 0, 4096 * 3); - - CompChunkPtrList ptrList; - std::unique_ptr decompressor( - compress::getCompressInterfaceByType( - compress::CompressInterface::getCompressionType( - &cmpHdrBuf[0]))); - - if (!decompressor) - { - // Use default? - decompressor.reset( - new compress::CompressInterfaceSnappy()); - } - - int dcrc = 0; - - if (i == 4096 * 3) - { - uint64_t numHdrs = 0; // extra headers - dcrc = compress::CompressInterface::getPtrList( - &cmpHdrBuf[4096], 4096, ptrList); - - if (dcrc == 0 && ptrList.size() > 0) - numHdrs = ptrList[0].first / 4096ULL - 2ULL; - - if (numHdrs > 0) - { - boost::scoped_array nextHdrBufsa(new char[numHdrs * 4096 + pageSize]); - alignedBuffer = (ptrdiff_t) nextHdrBufsa.get(); - - if ((alignedBuffer % pageSize) != 0) - { - alignedBuffer &= ~((ptrdiff_t)pageSize - 1); - alignedBuffer += pageSize; - } - - char* nextHdrBufPtr = (char*) alignedBuffer; - - i = fp->pread( &nextHdrBufPtr[0], 4096 * 2, numHdrs * 4096 ); - - CompChunkPtrList nextPtrList; - dcrc = compress::CompressInterface::getPtrList( - &nextHdrBufPtr[0], numHdrs * 4096, nextPtrList); - - if (dcrc == 0) - ptrList.insert(ptrList.end(), nextPtrList.begin(), nextPtrList.end()); - } - } - - if (dcrc != 0 || idx >= ptrList.size()) - { - // Due to race condition, the header on disk may not upated yet. - // Log an info message and retry. - if (blockReadRetryCount == 0) - { - logging::Message::Args args; - args.add(oid); - ostringstream infoMsg; - infoMsg << "retry read from " << fileNamePtr << ". dcrc=" << dcrc - << ", idx=" << idx << ", ptr.size=" << ptrList.size(); - args.add(infoMsg.str()); - mlp->logInfoMessage(logging::M0061, args); - } - - if (++blockReadRetryCount < 30) - { - waitForRetry(blockReadRetryCount); - goto blockReadRetry; - } - else - { - rc = -1004; - } - } - - if (rc == 0) - { - unsigned cmpBlkOff = offset % (4 * 1024 * 1024); - uint64_t cmpBufOff = ptrList[idx].first; - uint64_t cmpBufSz = ptrList[idx].second; - - if (cmpBufSa.get() == NULL || cmpBufLen < cmpBufSz) - { - cmpBufSa.reset(new char[cmpBufSz + pageSize]); - cmpBufLen = cmpBufSz; - alignedBuffer = (ptrdiff_t) cmpBufSa.get(); - - if ((alignedBuffer % pageSize) != 0) - { - alignedBuffer &= ~((ptrdiff_t)pageSize - 1); - alignedBuffer += pageSize; - } - - cmpBuf = (char*) alignedBuffer; - } - - size_t blen = 4 * 1024 * 1024; - - i = fp->pread( cmpBuf, cmpBufOff, cmpBufSz ); - - dcrc = decompressor->uncompressBlock(cmpBuf, cmpBufSz, uCmpBuf, blen); - - if (dcrc == 0) - { - memcpy(bufferPtr, &uCmpBuf[cmpBlkOff], DATA_BLOCK_SIZE); - } - else - { - // Due to race condition, the header on disk may not upated yet. - // Log an info message and retry. - if (blockReadRetryCount == 0) - { - logging::Message::Args args; - args.add(oid); - ostringstream infoMsg; - infoMsg << "retry read from " << fileNamePtr << ". dcrc=" << dcrc - << ", idx=" << idx << ", ptr.size=" << ptrList.size(); - args.add(infoMsg.str()); - mlp->logInfoMessage(logging::M0061, args); - } - - if (++blockReadRetryCount < 30) - { - waitForRetry(blockReadRetryCount); - goto blockReadRetry; - } - else - { - rc = -1006; - } - } - } - } - - if ( rc < 0 ) - { - string msg("pread failed"); - ostringstream infoMsg; - infoMsg << " rc:" << rc << ")"; - msg = msg + ", error:" + strerror(errno) + infoMsg.str(); - SUMMARY_INFO(msg); - //FIXME: free-up allocated memory! - throw std::runtime_error(msg); - } - } - catch (...) { - delete fp; - fp = 0; - throw; + boost::mutex::scoped_lock lk(primitiveprocessor::compDebugMutex); + cout << "pread2(" << fd << ", 0x" << hex << (ptrdiff_t)readBufferPtr << dec << ", " + << DATA_BLOCK_SIZE << ", " << offset << ") = " << i << endl; + } +#endif // IDB_COMP_POC_DEBUG + } // if (compType == 0) + else // if (compType != 0) + { + // retry if file is out of sync -- compressed column file only. + blockReadRetry: + + uCmpBufSa.reset(new unsigned char[4 * 1024 * 1024 + 4]); + uCmpBuf = uCmpBufSa.get(); + cmpHdrBufSa.reset(new char[4096 * 3 + pageSize]); + alignedBuffer = (ptrdiff_t)cmpHdrBufSa.get(); + + if ((alignedBuffer % pageSize) != 0) + { + alignedBuffer &= ~((ptrdiff_t)pageSize - 1); + alignedBuffer += pageSize; } - delete fp; - fp = 0; + cmpHdrBuf = (char*)alignedBuffer; - // log the retries - if (blockReadRetryCount > 0) + i = fp->pread(&cmpHdrBuf[0], 0, 4096 * 3); + + CompChunkPtrList ptrList; + std::unique_ptr decompressor(compress::getCompressInterfaceByType( + compress::CompressInterface::getCompressionType(&cmpHdrBuf[0]))); + + if (!decompressor) { + // Use default? + decompressor.reset(new compress::CompressInterfaceSnappy()); + } + + int dcrc = 0; + + if (i == 4096 * 3) + { + uint64_t numHdrs = 0; // extra headers + dcrc = compress::CompressInterface::getPtrList(&cmpHdrBuf[4096], 4096, ptrList); + + if (dcrc == 0 && ptrList.size() > 0) + numHdrs = ptrList[0].first / 4096ULL - 2ULL; + + if (numHdrs > 0) + { + boost::scoped_array nextHdrBufsa(new char[numHdrs * 4096 + pageSize]); + alignedBuffer = (ptrdiff_t)nextHdrBufsa.get(); + + if ((alignedBuffer % pageSize) != 0) + { + alignedBuffer &= ~((ptrdiff_t)pageSize - 1); + alignedBuffer += pageSize; + } + + char* nextHdrBufPtr = (char*)alignedBuffer; + + i = fp->pread(&nextHdrBufPtr[0], 4096 * 2, numHdrs * 4096); + + CompChunkPtrList nextPtrList; + dcrc = compress::CompressInterface::getPtrList(&nextHdrBufPtr[0], numHdrs * 4096, nextPtrList); + + if (dcrc == 0) + ptrList.insert(ptrList.end(), nextPtrList.begin(), nextPtrList.end()); + } + } + + if (dcrc != 0 || idx >= ptrList.size()) + { + // Due to race condition, the header on disk may not upated yet. + // Log an info message and retry. + if (blockReadRetryCount == 0) + { logging::Message::Args args; args.add(oid); ostringstream infoMsg; - infoMsg << "Successfully uncompress " << fileNamePtr << " chunk " - << idx << " @" << " blockReadRetry:" << blockReadRetryCount; + infoMsg << "retry read from " << fileNamePtr << ". dcrc=" << dcrc << ", idx=" << idx + << ", ptr.size=" << ptrList.size(); args.add(infoMsg.str()); - mlp->logInfoMessage(logging::M0006, args); + mlp->logInfoMessage(logging::M0061, args); + } + + if (++blockReadRetryCount < 30) + { + waitForRetry(blockReadRetryCount); + goto blockReadRetry; + } + else + { + rc = -1004; + } } - if (pWasBlockInCache) - *pWasBlockInCache = false; + if (rc == 0) + { + unsigned cmpBlkOff = offset % (4 * 1024 * 1024); + uint64_t cmpBufOff = ptrList[idx].first; + uint64_t cmpBufSz = ptrList[idx].second; - if (rCount) - *rCount = 1; + if (cmpBufSa.get() == NULL || cmpBufLen < cmpBufSz) + { + cmpBufSa.reset(new char[cmpBufSz + pageSize]); + cmpBufLen = cmpBufSz; + alignedBuffer = (ptrdiff_t)cmpBufSa.get(); - return; + if ((alignedBuffer % pageSize) != 0) + { + alignedBuffer &= ~((ptrdiff_t)pageSize - 1); + alignedBuffer += pageSize; + } + + cmpBuf = (char*)alignedBuffer; + } + + size_t blen = 4 * 1024 * 1024; + + i = fp->pread(cmpBuf, cmpBufOff, cmpBufSz); + + dcrc = decompressor->uncompressBlock(cmpBuf, cmpBufSz, uCmpBuf, blen); + + if (dcrc == 0) + { + memcpy(bufferPtr, &uCmpBuf[cmpBlkOff], DATA_BLOCK_SIZE); + } + else + { + // Due to race condition, the header on disk may not upated yet. + // Log an info message and retry. + if (blockReadRetryCount == 0) + { + logging::Message::Args args; + args.add(oid); + ostringstream infoMsg; + infoMsg << "retry read from " << fileNamePtr << ". dcrc=" << dcrc << ", idx=" << idx + << ", ptr.size=" << ptrList.size(); + args.add(infoMsg.str()); + mlp->logInfoMessage(logging::M0061, args); + } + + if (++blockReadRetryCount < 30) + { + waitForRetry(blockReadRetryCount); + goto blockReadRetry; + } + else + { + rc = -1006; + } + } + } + } + + if (rc < 0) + { + string msg("pread failed"); + ostringstream infoMsg; + infoMsg << " rc:" << rc << ")"; + msg = msg + ", error:" + strerror(errno) + infoMsg.str(); + SUMMARY_INFO(msg); + // FIXME: free-up allocated memory! + throw std::runtime_error(msg); + } + } + catch (...) + { + delete fp; + fp = 0; + throw; } - FileBuffer* fbPtr = 0; - bool wasBlockInCache = false; + delete fp; + fp = 0; - fbPtr = bc.getBlockPtr(lbid, ver, flg); - - if (fbPtr) + // log the retries + if (blockReadRetryCount > 0) { - memcpy(bufferPtr, fbPtr->getData(), BLOCK_SIZE); - wasBlockInCache = true; - } - - if (doPrefetch && !wasBlockInCache && !flg) - { - prefetchBlocks(lbid, compType, &blksRead); - -#ifndef _MSC_VER - - if (fPMProfOn) - pmstats.markEvent(lbid, (pthread_t) - 1, sessionID, 'M'); - -#endif - bc.getBlock(lbid, v, txn, compType, (uint8_t*) bufferPtr, flg, wasBlockInCache); - - if (!wasBlockInCache) - blksRead++; - } - else if (!wasBlockInCache) - { - bc.getBlock(lbid, v, txn, compType, (uint8_t*) bufferPtr, flg, wasBlockInCache); - - if (!wasBlockInCache) - blksRead++; + logging::Message::Args args; + args.add(oid); + ostringstream infoMsg; + infoMsg << "Successfully uncompress " << fileNamePtr << " chunk " << idx << " @" + << " blockReadRetry:" << blockReadRetryCount; + args.add(infoMsg.str()); + mlp->logInfoMessage(logging::M0006, args); } if (pWasBlockInCache) - *pWasBlockInCache = wasBlockInCache; + *pWasBlockInCache = false; if (rCount) - *rCount = blksRead; + *rCount = 1; + return; + } + + FileBuffer* fbPtr = 0; + bool wasBlockInCache = false; + + fbPtr = bc.getBlockPtr(lbid, ver, flg); + + if (fbPtr) + { + memcpy(bufferPtr, fbPtr->getData(), BLOCK_SIZE); + wasBlockInCache = true; + } + + if (doPrefetch && !wasBlockInCache && !flg) + { + prefetchBlocks(lbid, compType, &blksRead); + +#ifndef _MSC_VER + + if (fPMProfOn) + pmstats.markEvent(lbid, (pthread_t)-1, sessionID, 'M'); + +#endif + bc.getBlock(lbid, v, txn, compType, (uint8_t*)bufferPtr, flg, wasBlockInCache); + + if (!wasBlockInCache) + blksRead++; + } + else if (!wasBlockInCache) + { + bc.getBlock(lbid, v, txn, compType, (uint8_t*)bufferPtr, flg, wasBlockInCache); + + if (!wasBlockInCache) + blksRead++; + } + + if (pWasBlockInCache) + *pWasBlockInCache = wasBlockInCache; + + if (rCount) + *rCount = blksRead; } struct AsynchLoader { - AsynchLoader(uint64_t l, - const QueryContext& v, - uint32_t t, - int ct, - uint32_t* cCount, - uint32_t* rCount, - bool trace, - uint32_t sesID, - boost::mutex* m, - uint32_t* loaderCount, - boost::shared_ptr st, // sendThread for abort upon exception. - VSSCache* vCache) : - lbid(l), - ver(v), - txn(t), - compType(ct), - LBIDTrace(trace), - sessionID(sesID), - cacheCount(cCount), - readCount(rCount), - busyLoaders(loaderCount), - mutex(m), - sendThread(st), - vssCache(vCache) - { } + AsynchLoader(uint64_t l, const QueryContext& v, uint32_t t, int ct, uint32_t* cCount, uint32_t* rCount, + bool trace, uint32_t sesID, boost::mutex* m, uint32_t* loaderCount, + boost::shared_ptr st, // sendThread for abort upon exception. + VSSCache* vCache) + : lbid(l) + , ver(v) + , txn(t) + , compType(ct) + , LBIDTrace(trace) + , sessionID(sesID) + , cacheCount(cCount) + , readCount(rCount) + , busyLoaders(loaderCount) + , mutex(m) + , sendThread(st) + , vssCache(vCache) + { + } - void operator()() - { - utils::setThreadName("PPAsyncLoader"); - bool cached = false; - uint32_t rCount = 0; - char buf[BLOCK_SIZE]; - - //cout << "asynch started " << pthread_self() << " l: " << lbid << endl; - try - { - loadBlock(lbid, ver, txn, compType, buf, &cached, &rCount, LBIDTrace, true, vssCache); - } - catch (std::exception& ex) - { - sendThread->abort(); - cerr << "AsynchLoader caught loadBlock exception: " << ex.what() << endl; - idbassert(asyncCounter > 0); - (void)atomicops::atomicDec(&asyncCounter); - mutex->lock(); - --(*busyLoaders); - mutex->unlock(); - logging::Message::Args args; - args.add(string("PrimProc AsyncLoader caught error: ")); - args.add(ex.what()); - primitiveprocessor::mlp->logMessage(logging::M0000, args, false); - return; - } - catch (...) - { - sendThread->abort(); - cerr << "AsynchLoader caught unknown exception: " << endl; - //FIXME Use a locked processor primitive? - idbassert(asyncCounter > 0); - (void)atomicops::atomicDec(&asyncCounter); - mutex->lock(); - --(*busyLoaders); - mutex->unlock(); - logging::Message::Args args; - args.add(string("PrimProc AsyncLoader caught unknown error")); - primitiveprocessor::mlp->logMessage(logging::M0000, args, false); - return; - } - - idbassert(asyncCounter > 0); - (void)atomicops::atomicDec(&asyncCounter); - mutex->lock(); - - if (cached) - (*cacheCount)++; - - *readCount += rCount; - --(*busyLoaders); - mutex->unlock(); -// cerr << "done\n"; - } - -private: - uint64_t lbid; - QueryContext ver; - uint32_t txn; - int compType; - uint8_t dataWidth; - bool LBIDTrace; - uint32_t sessionID; - uint32_t* cacheCount; - uint32_t* readCount; - uint32_t* busyLoaders; - boost::mutex* mutex; - boost::shared_ptr sendThread; - VSSCache* vssCache; -}; - -void loadBlockAsync(uint64_t lbid, - const QueryContext& c, - uint32_t txn, - int compType, - uint32_t* cCount, - uint32_t* rCount, - bool LBIDTrace, - uint32_t sessionID, - boost::mutex* m, - uint32_t* busyLoaders, - boost::shared_ptr sendThread, // sendThread for abort upon exception. - VSSCache* vssCache) -{ - blockCacheClient bc(*BRPp[cacheNum(lbid)]); - bool vbFlag; - BRM::VER_t ver; - VSSCache::iterator it; - - if (vssCache) - { - it = vssCache->find(lbid); - - if (it != vssCache->end()) - { - //cout << "async: vss cache hit on " << lbid << endl; - VSSData& vd = it->second; - ver = vd.verID; - vbFlag = vd.vbFlag; - } - } - - if (!vssCache || it == vssCache->end()) - brm->vssLookup((BRM::LBID_t) lbid, c, txn, &ver, &vbFlag); - - if (bc.exists(lbid, ver)) - return; - - /* a quick and easy stand-in for a threadpool for loaders */ - atomicops::atomicMb(); - - if (asyncCounter >= asyncMax) - return; - - (void)atomicops::atomicInc(&asyncCounter); - - boost::mutex::scoped_lock sl(*m); + void operator()() + { + utils::setThreadName("PPAsyncLoader"); + bool cached = false; + uint32_t rCount = 0; + char buf[BLOCK_SIZE]; + // cout << "asynch started " << pthread_self() << " l: " << lbid << endl; try { - boost::thread thd(AsynchLoader(lbid, c, txn, compType, cCount, rCount, - LBIDTrace, sessionID, m, busyLoaders, sendThread, vssCache)); - (*busyLoaders)++; + loadBlock(lbid, ver, txn, compType, buf, &cached, &rCount, LBIDTrace, true, vssCache); } - catch (boost::thread_resource_error& e) + catch (std::exception& ex) { - cerr << "AsynchLoader: caught a thread resource error, need to lower asyncMax\n"; - idbassert(asyncCounter > 0); - (void)atomicops::atomicDec(&asyncCounter); + sendThread->abort(); + cerr << "AsynchLoader caught loadBlock exception: " << ex.what() << endl; + idbassert(asyncCounter > 0); + (void)atomicops::atomicDec(&asyncCounter); + mutex->lock(); + --(*busyLoaders); + mutex->unlock(); + logging::Message::Args args; + args.add(string("PrimProc AsyncLoader caught error: ")); + args.add(ex.what()); + primitiveprocessor::mlp->logMessage(logging::M0000, args, false); + return; } + catch (...) + { + sendThread->abort(); + cerr << "AsynchLoader caught unknown exception: " << endl; + // FIXME Use a locked processor primitive? + idbassert(asyncCounter > 0); + (void)atomicops::atomicDec(&asyncCounter); + mutex->lock(); + --(*busyLoaders); + mutex->unlock(); + logging::Message::Args args; + args.add(string("PrimProc AsyncLoader caught unknown error")); + primitiveprocessor::mlp->logMessage(logging::M0000, args, false); + return; + } + + idbassert(asyncCounter > 0); + (void)atomicops::atomicDec(&asyncCounter); + mutex->lock(); + + if (cached) + (*cacheCount)++; + + *readCount += rCount; + --(*busyLoaders); + mutex->unlock(); + // cerr << "done\n"; + } + + private: + uint64_t lbid; + QueryContext ver; + uint32_t txn; + int compType; + uint8_t dataWidth; + bool LBIDTrace; + uint32_t sessionID; + uint32_t* cacheCount; + uint32_t* readCount; + uint32_t* busyLoaders; + boost::mutex* mutex; + boost::shared_ptr sendThread; + VSSCache* vssCache; +}; + +void loadBlockAsync(uint64_t lbid, const QueryContext& c, uint32_t txn, int compType, uint32_t* cCount, + uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, boost::mutex* m, + uint32_t* busyLoaders, + boost::shared_ptr sendThread, // sendThread for abort upon exception. + VSSCache* vssCache) +{ + blockCacheClient bc(*BRPp[cacheNum(lbid)]); + bool vbFlag; + BRM::VER_t ver; + VSSCache::iterator it; + + if (vssCache) + { + it = vssCache->find(lbid); + + if (it != vssCache->end()) + { + // cout << "async: vss cache hit on " << lbid << endl; + VSSData& vd = it->second; + ver = vd.verID; + vbFlag = vd.vbFlag; + } + } + + if (!vssCache || it == vssCache->end()) + brm->vssLookup((BRM::LBID_t)lbid, c, txn, &ver, &vbFlag); + + if (bc.exists(lbid, ver)) + return; + + /* a quick and easy stand-in for a threadpool for loaders */ + atomicops::atomicMb(); + + if (asyncCounter >= asyncMax) + return; + + (void)atomicops::atomicInc(&asyncCounter); + + boost::mutex::scoped_lock sl(*m); + + try + { + boost::thread thd(AsynchLoader(lbid, c, txn, compType, cCount, rCount, LBIDTrace, sessionID, m, + busyLoaders, sendThread, vssCache)); + (*busyLoaders)++; + } + catch (boost::thread_resource_error& e) + { + cerr << "AsynchLoader: caught a thread resource error, need to lower asyncMax\n"; + idbassert(asyncCounter > 0); + (void)atomicops::atomicDec(&asyncCounter); + } } -} //namespace primitiveprocessor +} // namespace primitiveprocessor //#define DCT_DEBUG 1 -#define SETUP_GUARD \ -{ \ - unsigned char* o = outputp.get(); \ - memset(o, 0xa5, ouput_buf_size*3); \ -} +#define SETUP_GUARD \ + { \ + unsigned char* o = outputp.get(); \ + memset(o, 0xa5, ouput_buf_size * 3); \ + } #undef SETUP_GUARD #define SETUP_GUARD -#define CHECK_GUARD(lbid) \ -{ \ - unsigned char* o = outputp.get(); \ - for (int i = 0; i < ouput_buf_size; i++) \ - { \ - if (*o++ != 0xa5) \ - { \ - cerr << "Buffer underrun on LBID " << lbid << endl; \ - idbassert(0); \ - } \ - } \ - o += ouput_buf_size; \ - for (int i = 0; i < ouput_buf_size; i++) \ - { \ - if (*o++ != 0xa5) \ - { \ - cerr << "Buffer overrun on LBID " << lbid << endl; \ - idbassert(0); \ - } \ - } \ -} +#define CHECK_GUARD(lbid) \ + { \ + unsigned char* o = outputp.get(); \ + for (int i = 0; i < ouput_buf_size; i++) \ + { \ + if (*o++ != 0xa5) \ + { \ + cerr << "Buffer underrun on LBID " << lbid << endl; \ + idbassert(0); \ + } \ + } \ + o += ouput_buf_size; \ + for (int i = 0; i < ouput_buf_size; i++) \ + { \ + if (*o++ != 0xa5) \ + { \ + cerr << "Buffer overrun on LBID " << lbid << endl; \ + idbassert(0); \ + } \ + } \ + } #undef CHECK_GUARD #define CHECK_GUARD(x) @@ -1115,26 +1053,26 @@ using namespace primitiveprocessor; */ class DictScanJob : public threadpool::PriorityThreadPool::Functor { -public: - DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock); - virtual ~DictScanJob(); + public: + DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock); + virtual ~DictScanJob(); - void write(const ByteStream&); - int operator()(); - void catchHandler(const std::string& ex, uint32_t id, uint16_t code = logging::primitiveServerErr); - void sendErrorMsg(uint32_t id, uint16_t code); + void write(const ByteStream&); + int operator()(); + void catchHandler(const std::string& ex, uint32_t id, uint16_t code = logging::primitiveServerErr); + void sendErrorMsg(uint32_t id, uint16_t code); -private: - SP_UM_IOSOCK fIos; - SBS fByteStream; - SP_UM_MUTEX fWriteLock; - posix_time::ptime dieTime; + private: + SP_UM_IOSOCK fIos; + SBS fByteStream; + SP_UM_MUTEX fWriteLock; + posix_time::ptime dieTime; }; -DictScanJob::DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock) : - fIos(ios), fByteStream(bs), fWriteLock(writeLock) +DictScanJob::DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock) + : fIos(ios), fByteStream(bs), fWriteLock(writeLock) { - dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); + dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); } DictScanJob::~DictScanJob() @@ -1143,1395 +1081,1391 @@ DictScanJob::~DictScanJob() void DictScanJob::write(const ByteStream& bs) { - boost::mutex::scoped_lock lk(*fWriteLock); - fIos->write(bs); + boost::mutex::scoped_lock lk(*fWriteLock); + fIos->write(bs); } int DictScanJob::operator()() { - utils::setThreadName("PPDictScanJob"); - uint8_t data[DATA_BLOCK_SIZE]; - uint32_t output_buf_size = MAX_BUFFER_SIZE; - uint32_t session; - uint32_t uniqueId = 0; - bool wasBlockInCache = false; - uint32_t blocksRead = 0; - uint16_t runCount; + utils::setThreadName("PPDictScanJob"); + uint8_t data[DATA_BLOCK_SIZE]; + uint32_t output_buf_size = MAX_BUFFER_SIZE; + uint32_t session; + uint32_t uniqueId = 0; + bool wasBlockInCache = false; + uint32_t blocksRead = 0; + uint16_t runCount; + boost::shared_ptr eqFilter; + ByteStream results(output_buf_size); + TokenByScanRequestHeader* cmd; + PrimitiveProcessor pproc(gDebugLevel); + TokenByScanResultHeader* output; + QueryContext verInfo; - boost::shared_ptr eqFilter; - ByteStream results(output_buf_size); - TokenByScanRequestHeader* cmd; - PrimitiveProcessor pproc(gDebugLevel); - TokenByScanResultHeader* output; - QueryContext verInfo; - - try - { + try + { #ifdef DCT_DEBUG - DebugLevel oldDebugLevel = gDebugLevel; - gDebugLevel = VERBOSE; + DebugLevel oldDebugLevel = gDebugLevel; + gDebugLevel = VERBOSE; #endif - fByteStream->advance(sizeof(TokenByScanRequestHeader)); - *fByteStream >> verInfo; - cmd = (TokenByScanRequestHeader*) fByteStream->buf(); + fByteStream->advance(sizeof(TokenByScanRequestHeader)); + *fByteStream >> verInfo; + cmd = (TokenByScanRequestHeader*)fByteStream->buf(); - session = cmd->Hdr.SessionID; - uniqueId = cmd->Hdr.UniqueID; - runCount = cmd->Count; - output = (TokenByScanResultHeader*) results.getInputPtr(); + session = cmd->Hdr.SessionID; + uniqueId = cmd->Hdr.UniqueID; + runCount = cmd->Count; + output = (TokenByScanResultHeader*)results.getInputPtr(); #ifdef VALGRIND - memset(output, 0, sizeof(TokenByScanResultHeader)); + memset(output, 0, sizeof(TokenByScanResultHeader)); #endif - /* Grab the equality filter if one is specified */ - if (cmd->flags & HAS_EQ_FILTER) + /* Grab the equality filter if one is specified */ + if (cmd->flags & HAS_EQ_FILTER) + { + boost::mutex::scoped_lock sl(eqFilterMutex); + map >::iterator it; + it = dictEqualityFilters.find(uniqueId); + + if (it != dictEqualityFilters.end()) + eqFilter = it->second; + + sl.unlock(); + + if (!eqFilter) + { + if (posix_time::second_clock::universal_time() < dieTime) { - boost::mutex::scoped_lock sl(eqFilterMutex); - map >::iterator it; - it = dictEqualityFilters.find(uniqueId); - - if (it != dictEqualityFilters.end()) - eqFilter = it->second; - - sl.unlock(); - - if (!eqFilter) - { - if (posix_time::second_clock::universal_time() < dieTime) - { - fByteStream->rewind(); - return -1; // it's still being built, wait for it - } - else - return 0; // was probably aborted, go away... - } + fByteStream->rewind(); + return -1; // it's still being built, wait for it } + else + return 0; // was probably aborted, go away... + } + } - for (uint16_t i = 0; i < runCount; ++i) - { - loadBlock(cmd->LBID, - verInfo, - cmd->Hdr.TransactionID, - cmd->CompType, - data, - &wasBlockInCache, - &blocksRead, - fLBIDTraceOn, - session); - pproc.setBlockPtr((int*) data); - pproc.p_TokenByScan(cmd, output, output_buf_size, eqFilter); + for (uint16_t i = 0; i < runCount; ++i) + { + loadBlock(cmd->LBID, verInfo, cmd->Hdr.TransactionID, cmd->CompType, data, &wasBlockInCache, + &blocksRead, fLBIDTraceOn, session); + pproc.setBlockPtr((int*)data); + pproc.p_TokenByScan(cmd, output, output_buf_size, eqFilter); - if (wasBlockInCache) - output->CacheIO++; - else - output->PhysicalIO += blocksRead; + if (wasBlockInCache) + output->CacheIO++; + else + output->PhysicalIO += blocksRead; - results.advanceInputPtr(output->NBYTES); - write(results); - results.restart(); - cmd->LBID++; - } + results.advanceInputPtr(output->NBYTES); + write(results); + results.restart(); + cmd->LBID++; + } #ifdef DCT_DEBUG - gDebugLevel = oldDebugLevel; + gDebugLevel = oldDebugLevel; #endif - } - catch (logging::IDBExcept& iex) - { - cerr << "DictScanJob caught an IDBException: " << iex.what() << endl; - catchHandler(iex.what(), uniqueId, iex.errorCode()); - } - catch (std::exception& re) - { - cerr << "DictScanJob caught an exception: " << re.what() << endl; - catchHandler(re.what(), uniqueId); - } - catch (...) - { - string msg("Unknown exception caught in DictScanJob."); - cerr << msg << endl; - catchHandler(msg, uniqueId); - } + } + catch (logging::IDBExcept& iex) + { + cerr << "DictScanJob caught an IDBException: " << iex.what() << endl; + catchHandler(iex.what(), uniqueId, iex.errorCode()); + } + catch (std::exception& re) + { + cerr << "DictScanJob caught an exception: " << re.what() << endl; + catchHandler(re.what(), uniqueId); + } + catch (...) + { + string msg("Unknown exception caught in DictScanJob."); + cerr << msg << endl; + catchHandler(msg, uniqueId); + } - return 0; + return 0; } void DictScanJob::catchHandler(const string& ex, uint32_t id, uint16_t code) { - Logger log; - log.logMessage(ex); - sendErrorMsg(id, code); + Logger log; + log.logMessage(ex); + sendErrorMsg(id, code); } void DictScanJob::sendErrorMsg(uint32_t id, uint16_t code) { - ISMPacketHeader ism; - PrimitiveHeader ph; - ism.Status = code; - ph.UniqueID = id; + ISMPacketHeader ism; + PrimitiveHeader ph; + ism.Status = code; + ph.UniqueID = id; - ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - msg.append((uint8_t*) &ism, sizeof(ism)); - msg.append((uint8_t*) &ph, sizeof(ph)); + ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + msg.append((uint8_t*)&ism, sizeof(ism)); + msg.append((uint8_t*)&ph, sizeof(ph)); - write(msg); + write(msg); } struct BPPHandler { - BPPHandler(PrimitiveServer* ps) : fPrimitiveServerPtr(ps) { } + BPPHandler(PrimitiveServer* ps) : fPrimitiveServerPtr(ps) + { + } - // Keep a list of keys so that if connection fails we don't leave BPP - // threads lying around - std::vector bppKeys; - std::vector::iterator bppKeysIt; + // Keep a list of keys so that if connection fails we don't leave BPP + // threads lying around + std::vector bppKeys; + std::vector::iterator bppKeysIt; - ~BPPHandler() + ~BPPHandler() + { + boost::mutex::scoped_lock scoped(bppLock); + + for (bppKeysIt = bppKeys.begin(); bppKeysIt != bppKeys.end(); ++bppKeysIt) { - boost::mutex::scoped_lock scoped(bppLock); + uint32_t key = *bppKeysIt; + BPPMap::iterator it; - for (bppKeysIt = bppKeys.begin() ; bppKeysIt != bppKeys.end(); ++bppKeysIt) - { - uint32_t key = *bppKeysIt; - BPPMap::iterator it; + it = bppMap.find(key); - it = bppMap.find(key); + if (it != bppMap.end()) + { + it->second->abort(); + bppMap.erase(it); + } - if (it != bppMap.end()) - { - it->second->abort(); - bppMap.erase(it); - } - - fPrimitiveServerPtr->getProcessorThreadPool()->removeJobs(key); - OOBPool->removeJobs(key); - } - - scoped.unlock(); + fPrimitiveServerPtr->getProcessorThreadPool()->removeJobs(key); + OOBPool->removeJobs(key); } - struct BPPHandlerFunctor : public PriorityThreadPool::Functor + scoped.unlock(); + } + + struct BPPHandlerFunctor : public PriorityThreadPool::Functor + { + BPPHandlerFunctor(boost::shared_ptr r, SBS b) : bs(b) { - BPPHandlerFunctor(boost::shared_ptr r, SBS b) : bs(b) - { - rt = r; - dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); - } + rt = r; + dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); + } - boost::shared_ptr rt; - SBS bs; - posix_time::ptime dieTime; - }; + boost::shared_ptr rt; + SBS bs; + posix_time::ptime dieTime; + }; - struct LastJoiner : public BPPHandlerFunctor + struct LastJoiner : public BPPHandlerFunctor + { + LastJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { - LastJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { } - int operator()() - { - utils::setThreadName("PPHandLastJoiner"); - return rt->lastJoinerMsg(*bs, dieTime); - } - }; - - struct Create : public BPPHandlerFunctor + } + int operator()() { - Create(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { } - int operator()() - { - utils::setThreadName("PPHandCreate"); - rt->createBPP(*bs); - return 0; - } - }; + utils::setThreadName("PPHandLastJoiner"); + return rt->lastJoinerMsg(*bs, dieTime); + } + }; - struct Destroy : public BPPHandlerFunctor + struct Create : public BPPHandlerFunctor + { + Create(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { - Destroy(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { } - int operator()() - { - utils::setThreadName("PPHandDestroy"); - return rt->destroyBPP(*bs, dieTime); - } - }; - - struct AddJoiner : public BPPHandlerFunctor + } + int operator()() { - AddJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { } - int operator()() - { - utils::setThreadName("PPHandAddJoiner"); - return rt->addJoinerToBPP(*bs, dieTime); - } - }; + utils::setThreadName("PPHandCreate"); + rt->createBPP(*bs); + return 0; + } + }; - struct Abort : public BPPHandlerFunctor + struct Destroy : public BPPHandlerFunctor + { + Destroy(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { - Abort(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { } - int operator()() - { - utils::setThreadName("PPHandAbort"); - return rt->doAbort(*bs, dieTime); - } - }; - - int doAbort(ByteStream& bs, const posix_time::ptime& dieTime) + } + int operator()() { - uint32_t key; - BPPMap::iterator it; + utils::setThreadName("PPHandDestroy"); + return rt->destroyBPP(*bs, dieTime); + } + }; - try - { - bs.advance(sizeof(ISMPacketHeader)); - bs >> key; - } - catch (...) - { - // MCOL-857 We don't have the full packet yet - bs.rewind(); - return -1; - } + struct AddJoiner : public BPPHandlerFunctor + { + AddJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) + { + } + int operator()() + { + utils::setThreadName("PPHandAddJoiner"); + return rt->addJoinerToBPP(*bs, dieTime); + } + }; - boost::mutex::scoped_lock scoped(bppLock); - bppKeysIt = std::find(bppKeys.begin(), bppKeys.end(), key); + struct Abort : public BPPHandlerFunctor + { + Abort(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) + { + } + int operator()() + { + utils::setThreadName("PPHandAbort"); + return rt->doAbort(*bs, dieTime); + } + }; - if (bppKeysIt != bppKeys.end()) - { - bppKeys.erase(bppKeysIt); - } + int doAbort(ByteStream& bs, const posix_time::ptime& dieTime) + { + uint32_t key; + BPPMap::iterator it; - it = bppMap.find(key); + try + { + bs.advance(sizeof(ISMPacketHeader)); + bs >> key; + } + catch (...) + { + // MCOL-857 We don't have the full packet yet + bs.rewind(); + return -1; + } - if (it != bppMap.end()) - { - it->second->abort(); - bppMap.erase(it); - } - else - { - bs.rewind(); + boost::mutex::scoped_lock scoped(bppLock); + bppKeysIt = std::find(bppKeys.begin(), bppKeys.end(), key); - if (posix_time::second_clock::universal_time() > dieTime) - return 0; - else - return -1; - } + if (bppKeysIt != bppKeys.end()) + { + bppKeys.erase(bppKeysIt); + } - scoped.unlock(); - fPrimitiveServerPtr->getProcessorThreadPool()->removeJobs(key); - OOBPool->removeJobs(key); + it = bppMap.find(key); + + if (it != bppMap.end()) + { + it->second->abort(); + bppMap.erase(it); + } + else + { + bs.rewind(); + + if (posix_time::second_clock::universal_time() > dieTime) return 0; + else + return -1; } - int doAck(ByteStream& bs) + scoped.unlock(); + fPrimitiveServerPtr->getProcessorThreadPool()->removeJobs(key); + OOBPool->removeJobs(key); + return 0; + } + + int doAck(ByteStream& bs) + { + uint32_t key; + int16_t msgCount; + SBPPV bpps; + const ISMPacketHeader* ism = (const ISMPacketHeader*)bs.buf(); + + key = ism->Interleave; + msgCount = (int16_t)ism->Size; + bs.advance(sizeof(ISMPacketHeader)); + + bpps = grabBPPs(key); + + if (bpps) { - uint32_t key; - int16_t msgCount; - SBPPV bpps; - const ISMPacketHeader* ism = (const ISMPacketHeader*) bs.buf(); + bpps->getSendThread()->sendMore(msgCount); + return 0; + } + else + return -1; + } - key = ism->Interleave; - msgCount = (int16_t) ism->Size; - bs.advance(sizeof(ISMPacketHeader)); + void createBPP(ByteStream& bs) + { + uint32_t i; + uint32_t key, initMsgsLeft; + SBPP bpp; + SBPPV bppv; - bpps = grabBPPs(key); + // make the new BPP object + bppv.reset(new BPPV(fPrimitiveServerPtr)); + bpp.reset(new BatchPrimitiveProcessor(bs, fPrimitiveServerPtr->prefetchThreshold(), bppv->getSendThread(), + fPrimitiveServerPtr->ProcessorThreads())); - if (bpps) - { - bpps->getSendThread()->sendMore(msgCount); - return 0; - } - else - return -1; + if (bs.length() > 0) + bs >> initMsgsLeft; + else + { + initMsgsLeft = -1; } - void createBPP(ByteStream& bs) + idbassert(bs.length() == 0); + bppv->getSendThread()->sendMore(initMsgsLeft); + bppv->add(bpp); + + // this block of code creates some BPP instances up front for user queries, + // seems to get us slightly better query times + if ((bpp->getSessionID() & 0x80000000) == 0) { - uint32_t i; - uint32_t key, initMsgsLeft; - SBPP bpp; - SBPPV bppv; + for (i = 1; i < BPPCount / 8; i++) + { + SBPP dup = bpp->duplicate(); - // make the new BPP object - bppv.reset(new BPPV(fPrimitiveServerPtr)); - bpp.reset(new BatchPrimitiveProcessor(bs, fPrimitiveServerPtr->prefetchThreshold(), - bppv->getSendThread(), fPrimitiveServerPtr->ProcessorThreads())); - - if (bs.length() > 0) - bs >> initMsgsLeft; - else - { - initMsgsLeft = -1; - } - - idbassert(bs.length() == 0); - bppv->getSendThread()->sendMore(initMsgsLeft); - bppv->add(bpp); - - // this block of code creates some BPP instances up front for user queries, - // seems to get us slightly better query times - if ((bpp->getSessionID() & 0x80000000) == 0) - { - for (i = 1; i < BPPCount / 8; i++) - { - SBPP dup = bpp->duplicate(); - - /* Uncomment these lines to verify duplicate(). == op might need updating */ -// if (*bpp != *dup) -// cerr << "createBPP: duplicate mismatch at index " << i << endl; -// idbassert(*bpp == *dup); - bppv->add(dup); - } - } - - boost::mutex::scoped_lock scoped(bppLock); - key = bpp->getUniqueID(); - bppKeys.push_back(key); - bool newInsert; - newInsert = bppMap.insert(pair(key, bppv)).second; - //cout << "creating BPP # " << key << endl; - scoped.unlock(); - - if (!newInsert) - { - if (bpp->getSessionID() & 0x80000000) - cerr << "warning: createBPP() tried to clobber a BPP with duplicate sessionID & stepID. sessionID=" << - (int) (bpp->getSessionID() ^ 0x80000000) << " stepID=" << - bpp->getStepID() << " (syscat)" << endl; - else - cerr << "warning: createBPP() tried to clobber a BPP with duplicate sessionID & stepID. sessionID=" << - bpp->getSessionID() << " stepID=" << bpp->getStepID() << endl; - } + /* Uncomment these lines to verify duplicate(). == op might need updating */ + // if (*bpp != *dup) + // cerr << "createBPP: duplicate mismatch at index " << i << + // endl; + // idbassert(*bpp == *dup); + bppv->add(dup); + } } - inline SBPPV grabBPPs(uint32_t uniqueID) + boost::mutex::scoped_lock scoped(bppLock); + key = bpp->getUniqueID(); + bppKeys.push_back(key); + bool newInsert; + newInsert = bppMap.insert(pair(key, bppv)).second; + // cout << "creating BPP # " << key << endl; + scoped.unlock(); + + if (!newInsert) { - BPPMap::iterator it; - /* - uint32_t failCount = 0; - uint32_t maxFailCount = (fatal ? 500 : 5000); - */ - SBPPV ret; - - boost::mutex::scoped_lock scoped(bppLock); - it = bppMap.find(uniqueID); - - if (it != bppMap.end()) - return it->second; - else - return SBPPV(); - - /* - do - { - if (++failCount == maxFailCount) { - //cout << "grabBPPs couldn't find the BPPs for " << uniqueID << endl; - return ret; - //throw logic_error("grabBPPs couldn't find the unique ID"); - } - scoped.unlock(); - usleep(5000); - scoped.lock(); - it = bppMap.find(uniqueID); - } while (it == bppMap.end()); - - ret = it->second; - return ret; - */ + if (bpp->getSessionID() & 0x80000000) + cerr << "warning: createBPP() tried to clobber a BPP with duplicate sessionID & stepID. sessionID=" + << (int)(bpp->getSessionID() ^ 0x80000000) << " stepID=" << bpp->getStepID() << " (syscat)" + << endl; + else + cerr << "warning: createBPP() tried to clobber a BPP with duplicate sessionID & stepID. sessionID=" + << bpp->getSessionID() << " stepID=" << bpp->getStepID() << endl; } + } - inline shared_mutex & getDJLock(uint32_t uniqueID) + inline SBPPV grabBPPs(uint32_t uniqueID) + { + BPPMap::iterator it; + /* + uint32_t failCount = 0; + uint32_t maxFailCount = (fatal ? 500 : 5000); + */ + SBPPV ret; + + boost::mutex::scoped_lock scoped(bppLock); + it = bppMap.find(uniqueID); + + if (it != bppMap.end()) + return it->second; + else + return SBPPV(); + + /* + do + { + if (++failCount == maxFailCount) { + //cout << "grabBPPs couldn't find the BPPs for " << uniqueID << endl; + return ret; + //throw logic_error("grabBPPs couldn't find the unique ID"); + } + scoped.unlock(); + usleep(5000); + scoped.lock(); + it = bppMap.find(uniqueID); + } while (it == bppMap.end()); + + ret = it->second; + return ret; + */ + } + + inline shared_mutex& getDJLock(uint32_t uniqueID) + { + boost::mutex::scoped_lock lk(djMutex); + auto it = djLock.find(uniqueID); + if (it != djLock.end()) + return *it->second; + else { - boost::mutex::scoped_lock lk(djMutex); - auto it = djLock.find(uniqueID); - if (it != djLock.end()) - return *it->second; - else - { - auto ret = djLock.insert(make_pair(uniqueID, new shared_mutex())).first; - return *ret->second; - } + auto ret = djLock.insert(make_pair(uniqueID, new shared_mutex())).first; + return *ret->second; } + } - inline void deleteDJLock(uint32_t uniqueID) + inline void deleteDJLock(uint32_t uniqueID) + { + boost::mutex::scoped_lock lk(djMutex); + auto it = djLock.find(uniqueID); + if (it != djLock.end()) { - boost::mutex::scoped_lock lk(djMutex); - auto it = djLock.find(uniqueID); - if (it != djLock.end()) - { - delete it->second; - djLock.erase(it); - } + delete it->second; + djLock.erase(it); } + } - int addJoinerToBPP(ByteStream& bs, const posix_time::ptime& dieTime) + int addJoinerToBPP(ByteStream& bs, const posix_time::ptime& dieTime) + { + SBPPV bppv; + uint32_t uniqueID; + const uint8_t* buf; + + /* call addToJoiner() on the first BPP */ + + buf = bs.buf(); + /* the uniqueID is after the ISMPacketHeader, sessionID, and stepID */ + uniqueID = *((const uint32_t*)&buf[sizeof(ISMPacketHeader) + 2 * sizeof(uint32_t)]); + bppv = grabBPPs(uniqueID); + + if (bppv) { - SBPPV bppv; - uint32_t uniqueID; - const uint8_t* buf; - - /* call addToJoiner() on the first BPP */ - - buf = bs.buf(); - /* the uniqueID is after the ISMPacketHeader, sessionID, and stepID */ - uniqueID = *((const uint32_t*) &buf[sizeof(ISMPacketHeader) + 2 * sizeof(uint32_t)]); - bppv = grabBPPs(uniqueID); - - if (bppv) - { - shared_lock lk(getDJLock(uniqueID)); - bppv->get()[0]->addToJoiner(bs); - return 0; - } - else - { - if (posix_time::second_clock::universal_time() > dieTime) - return 0; - else - return -1; - } + shared_lock lk(getDJLock(uniqueID)); + bppv->get()[0]->addToJoiner(bs); + return 0; } - - int lastJoinerMsg(ByteStream& bs, const posix_time::ptime& dieTime) + else { - SBPPV bppv; - uint32_t uniqueID, i; - const uint8_t* buf; - int err; - - /* call endOfJoiner() on the every BPP */ - - buf = bs.buf(); - /* the uniqueID is after the ISMPacketHeader, sessionID, and stepID */ - uniqueID = *((const uint32_t*) &buf[sizeof(ISMPacketHeader) + 2 * sizeof(uint32_t)]); - - bppv = grabBPPs(uniqueID); - - if (!bppv) - { - //cout << "got a lastJoiner msg for an unknown obj " << uniqueID << endl; - if (posix_time::second_clock::universal_time() > dieTime) - return 0; - else - return -1; - } - - boost::unique_lock lk(getDJLock(uniqueID)); - - - for (i = 0; i < bppv->get().size(); i++) - { - err = bppv->get()[i]->endOfJoiner(); - - if (err == -1) - { - if (posix_time::second_clock::universal_time() > dieTime) - return 0; - else - return -1; - } - } - bppv->get()[0]->doneSendingJoinerData(); - - /* Note: some of the duplicate/run/join sync was moved to the BPPV class to do - more intelligent scheduling. Once the join data is received, BPPV will - start letting jobs run and create more BPP instances on demand. */ - - atomicops::atomicMb(); // make sure the joinDataReceived assignment doesn't migrate upward... - bppv->joinDataReceived = true; + if (posix_time::second_clock::universal_time() > dieTime) return 0; + else + return -1; } + } - int destroyBPP(ByteStream& bs, const posix_time::ptime& dieTime) + int lastJoinerMsg(ByteStream& bs, const posix_time::ptime& dieTime) + { + SBPPV bppv; + uint32_t uniqueID, i; + const uint8_t* buf; + int err; + + /* call endOfJoiner() on the every BPP */ + + buf = bs.buf(); + /* the uniqueID is after the ISMPacketHeader, sessionID, and stepID */ + uniqueID = *((const uint32_t*)&buf[sizeof(ISMPacketHeader) + 2 * sizeof(uint32_t)]); + + bppv = grabBPPs(uniqueID); + + if (!bppv) { - uint32_t uniqueID, sessionID, stepID; - BPPMap::iterator it; - - try - { - - bs.advance(sizeof(ISMPacketHeader)); - bs >> sessionID; - bs >> stepID; - bs >> uniqueID; - } - catch (...) - { - // MCOL-857 We don't appear to have the full packet yet! - bs.rewind(); - return -1; - } - - boost::unique_lock lk(getDJLock(uniqueID)); - boost::mutex::scoped_lock scoped(bppLock); - - bppKeysIt = std::find(bppKeys.begin(), bppKeys.end(), uniqueID); - - if (bppKeysIt != bppKeys.end()) - { - bppKeys.erase(bppKeysIt); - } - - it = bppMap.find(uniqueID); - - if (it != bppMap.end()) - { - boost::shared_ptr bppv = it->second; - - if (bppv->joinDataReceived) - { - bppv->abort(); - bppMap.erase(it); - } - else - { - // MCOL-5. On ubuntu, a crash was happening. Checking - // joinDataReceived here fixes it. - // We're not ready for a destroy. Reschedule. - return -1; - } - } - else - { - //cout << "got a destroy for an unknown obj " << uniqueID << endl; - bs.rewind(); - - if (posix_time::second_clock::universal_time() > dieTime) - { - // XXXPAT: going to let this fall through and delete jobs for - // uniqueID if there are any. Not clear what the downside is. - /* - lk.unlock(); - deleteDJLock(uniqueID); - return 0; - */ - } - else - return -1; - } - -// cout << " destroy: new size is " << bppMap.size() << endl; - /* - if (sessionID & 0x80000000) - cerr << "destroyed BPP instances for sessionID " << (int) - (sessionID ^ 0x80000000) << " stepID "<< stepID << " (syscat)\n"; - else - cerr << "destroyed BPP instances for sessionID " << sessionID << - " stepID "<< stepID << endl; - */ - fPrimitiveServerPtr->getProcessorThreadPool()->removeJobs(uniqueID); - OOBPool->removeJobs(uniqueID); - lk.unlock(); - deleteDJLock(uniqueID); + // cout << "got a lastJoiner msg for an unknown obj " << uniqueID << endl; + if (posix_time::second_clock::universal_time() > dieTime) return 0; + else + return -1; } - void setBPPToError(uint32_t uniqueID, const string& error, logging::ErrorCodeValues errorCode) + boost::unique_lock lk(getDJLock(uniqueID)); + + for (i = 0; i < bppv->get().size(); i++) { - SBPPV bppv; + err = bppv->get()[i]->endOfJoiner(); - bppv = grabBPPs(uniqueID); - - if (!bppv) - return; - - for (uint32_t i = 0; i < bppv->get().size(); i++) - bppv->get()[i]->setError(error, errorCode); - - if (bppv->get().empty() && !bppMap.empty() ) - bppMap.begin()->second.get()->get()[0]->setError(error, errorCode); + if (err == -1) + { + if (posix_time::second_clock::universal_time() > dieTime) + return 0; + else + return -1; + } } + bppv->get()[0]->doneSendingJoinerData(); - // Would be good to define the structure of these msgs somewhere... - inline uint32_t getUniqueID(SBS bs, uint8_t command) + /* Note: some of the duplicate/run/join sync was moved to the BPPV class to do + more intelligent scheduling. Once the join data is received, BPPV will + start letting jobs run and create more BPP instances on demand. */ + + atomicops::atomicMb(); // make sure the joinDataReceived assignment doesn't migrate upward... + bppv->joinDataReceived = true; + return 0; + } + + int destroyBPP(ByteStream& bs, const posix_time::ptime& dieTime) + { + uint32_t uniqueID, sessionID, stepID; + BPPMap::iterator it; + + try { - uint8_t* buf; - - buf = bs->buf(); - - switch (command) - { - case BATCH_PRIMITIVE_ABORT: - return *((uint32_t*) &buf[sizeof(ISMPacketHeader)]); - - case BATCH_PRIMITIVE_ACK: - { - ISMPacketHeader* ism = (ISMPacketHeader*) buf; - return ism->Interleave; - } - - case BATCH_PRIMITIVE_ADD_JOINER: - case BATCH_PRIMITIVE_END_JOINER: - case BATCH_PRIMITIVE_DESTROY: - return *((uint32_t*) &buf[sizeof(ISMPacketHeader) + 2 * sizeof(uint32_t)]); - - default: - return 0; - } + bs.advance(sizeof(ISMPacketHeader)); + bs >> sessionID; + bs >> stepID; + bs >> uniqueID; + } + catch (...) + { + // MCOL-857 We don't appear to have the full packet yet! + bs.rewind(); + return -1; } - PrimitiveServer* fPrimitiveServerPtr; + boost::unique_lock lk(getDJLock(uniqueID)); + boost::mutex::scoped_lock scoped(bppLock); + + bppKeysIt = std::find(bppKeys.begin(), bppKeys.end(), uniqueID); + + if (bppKeysIt != bppKeys.end()) + { + bppKeys.erase(bppKeysIt); + } + + it = bppMap.find(uniqueID); + + if (it != bppMap.end()) + { + boost::shared_ptr bppv = it->second; + + if (bppv->joinDataReceived) + { + bppv->abort(); + bppMap.erase(it); + } + else + { + // MCOL-5. On ubuntu, a crash was happening. Checking + // joinDataReceived here fixes it. + // We're not ready for a destroy. Reschedule. + return -1; + } + } + else + { + // cout << "got a destroy for an unknown obj " << uniqueID << endl; + bs.rewind(); + + if (posix_time::second_clock::universal_time() > dieTime) + { + // XXXPAT: going to let this fall through and delete jobs for + // uniqueID if there are any. Not clear what the downside is. + /* +lk.unlock(); +deleteDJLock(uniqueID); +return 0; + */ + } + else + return -1; + } + + // cout << " destroy: new size is " << bppMap.size() << endl; + /* + if (sessionID & 0x80000000) + cerr << "destroyed BPP instances for sessionID " << (int) + (sessionID ^ 0x80000000) << " stepID "<< stepID << " (syscat)\n"; + else + cerr << "destroyed BPP instances for sessionID " << sessionID << + " stepID "<< stepID << endl; + */ + fPrimitiveServerPtr->getProcessorThreadPool()->removeJobs(uniqueID); + OOBPool->removeJobs(uniqueID); + lk.unlock(); + deleteDJLock(uniqueID); + return 0; + } + + void setBPPToError(uint32_t uniqueID, const string& error, logging::ErrorCodeValues errorCode) + { + SBPPV bppv; + + bppv = grabBPPs(uniqueID); + + if (!bppv) + return; + + for (uint32_t i = 0; i < bppv->get().size(); i++) + bppv->get()[i]->setError(error, errorCode); + + if (bppv->get().empty() && !bppMap.empty()) + bppMap.begin()->second.get()->get()[0]->setError(error, errorCode); + } + + // Would be good to define the structure of these msgs somewhere... + inline uint32_t getUniqueID(SBS bs, uint8_t command) + { + uint8_t* buf; + + buf = bs->buf(); + + switch (command) + { + case BATCH_PRIMITIVE_ABORT: return *((uint32_t*)&buf[sizeof(ISMPacketHeader)]); + + case BATCH_PRIMITIVE_ACK: + { + ISMPacketHeader* ism = (ISMPacketHeader*)buf; + return ism->Interleave; + } + + case BATCH_PRIMITIVE_ADD_JOINER: + case BATCH_PRIMITIVE_END_JOINER: + case BATCH_PRIMITIVE_DESTROY: return *((uint32_t*)&buf[sizeof(ISMPacketHeader) + 2 * sizeof(uint32_t)]); + + default: return 0; + } + } + + PrimitiveServer* fPrimitiveServerPtr; }; class DictionaryOp : public PriorityThreadPool::Functor { -public: - DictionaryOp(SBS cmd) : bs(cmd) + public: + DictionaryOp(SBS cmd) : bs(cmd) + { + dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); + } + virtual int execute() = 0; + int operator()() + { + utils::setThreadName("PPDictOp"); + int ret; + ret = execute(); + + if (ret != 0) { - dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); + bs->rewind(); + + if (posix_time::second_clock::universal_time() > dieTime) + return 0; } - virtual int execute() = 0; - int operator()() - { - utils::setThreadName("PPDictOp"); - int ret; - ret = execute(); - if (ret != 0) - { - bs->rewind(); + return ret; + } - if (posix_time::second_clock::universal_time() > dieTime) - return 0; - } + protected: + SBS bs; - return ret; - } -protected: - SBS bs; -private: - posix_time::ptime dieTime; + private: + posix_time::ptime dieTime; }; class CreateEqualityFilter : public DictionaryOp { -public: - CreateEqualityFilter(SBS cmd) : DictionaryOp(cmd) { } - int execute() + public: + CreateEqualityFilter(SBS cmd) : DictionaryOp(cmd) + { + } + int execute() + { + createEqualityFilter(); + return 0; + } + + private: + void createEqualityFilter() + { + uint32_t uniqueID, count, i, charsetNumber; + string str; + bs->advance(sizeof(ISMPacketHeader)); + *bs >> uniqueID; + *bs >> charsetNumber; + + datatypes::Charset cs(charsetNumber); + boost::shared_ptr filter(new DictEqualityFilter(cs)); + + *bs >> count; + + for (i = 0; i < count; i++) { - createEqualityFilter(); - return 0; + *bs >> str; + filter->insert(str); } -private: - void createEqualityFilter() - { - uint32_t uniqueID, count, i, charsetNumber; - string str; - bs->advance(sizeof(ISMPacketHeader)); - *bs >> uniqueID; - *bs >> charsetNumber; - - datatypes::Charset cs(charsetNumber); - boost::shared_ptr filter(new DictEqualityFilter(cs)); - - *bs >> count; - - for (i = 0; i < count; i++) - { - *bs >> str; - filter->insert(str); - } - - boost::mutex::scoped_lock sl(eqFilterMutex); - dictEqualityFilters[uniqueID] = filter; - } + boost::mutex::scoped_lock sl(eqFilterMutex); + dictEqualityFilters[uniqueID] = filter; + } }; class DestroyEqualityFilter : public DictionaryOp { -public: - DestroyEqualityFilter(SBS cmd) : DictionaryOp(cmd) { } - int execute() + public: + DestroyEqualityFilter(SBS cmd) : DictionaryOp(cmd) + { + } + int execute() + { + return destroyEqualityFilter(); + } + + int destroyEqualityFilter() + { + uint32_t uniqueID; + map >::iterator it; + + bs->advance(sizeof(ISMPacketHeader)); + *bs >> uniqueID; + + boost::mutex::scoped_lock sl(eqFilterMutex); + it = dictEqualityFilters.find(uniqueID); + + if (it != dictEqualityFilters.end()) { - return destroyEqualityFilter(); - } - - int destroyEqualityFilter() - { - uint32_t uniqueID; - map >::iterator it; - - bs->advance(sizeof(ISMPacketHeader)); - *bs >> uniqueID; - - boost::mutex::scoped_lock sl(eqFilterMutex); - it = dictEqualityFilters.find(uniqueID); - - if (it != dictEqualityFilters.end()) - { - dictEqualityFilters.erase(it); - return 0; - } - else - return -1; + dictEqualityFilters.erase(it); + return 0; } + else + return -1; + } }; struct ReadThread { - ReadThread(const string& serverName, IOSocket& ios, PrimitiveServer* ps) : - fServerName(serverName), fIos(ios), fPrimitiveServerPtr(ps) + ReadThread(const string& serverName, IOSocket& ios, PrimitiveServer* ps) + : fServerName(serverName), fIos(ios), fPrimitiveServerPtr(ps) + { + fBPPHandler.reset(new BPPHandler(ps)); + } + + const ByteStream buildCacheOpResp(int32_t result) + { + const int msgsize = sizeof(ISMPacketHeader) + sizeof(int32_t); + ByteStream::byte msgbuf[msgsize]; + memset(msgbuf, 0, sizeof(ISMPacketHeader)); + ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); + hdrp->Command = CACHE_OP_RESULTS; + int32_t* resp = reinterpret_cast(&msgbuf[sizeof(ISMPacketHeader)]); + *resp = result; + return ByteStream(msgbuf, msgsize); + } + + /* Message format: + * ISMPacketHeader + * OID count - 32 bits + * OID array - 32 bits * count + */ + void doCacheFlushByOID(SP_UM_IOSOCK ios, ByteStream& bs) + { + uint8_t* buf = bs.buf(); + buf += sizeof(ISMPacketHeader); + uint32_t count = *((uint32_t*)buf); + buf += 4; + uint32_t* oids = (uint32_t*)buf; + + for (int i = 0; i < fCacheCount; i++) { - fBPPHandler.reset(new BPPHandler(ps)); + blockCacheClient bc(*BRPp[i]); + bc.flushOIDs(oids, count); } - const ByteStream buildCacheOpResp(int32_t result) + ios->write(buildCacheOpResp(0)); + } + + /* Message format: + * ISMPacketHeader + * Partition count - 32 bits + * Partition set - sizeof(LogicalPartition) boost::shared_ptr* count + * OID count - 32 bits + * OID array - 32 bits * count + */ + void doCacheFlushByPartition(SP_UM_IOSOCK ios, ByteStream& bs) + { + set partitions; + vector oids; + + bs.advance(sizeof(ISMPacketHeader)); + deserializeSet(bs, partitions); + deserializeInlineVector(bs, oids); + + idbassert(bs.length() == 0); + + for (int i = 0; i < fCacheCount; i++) { - const int msgsize = sizeof(ISMPacketHeader) + sizeof(int32_t); - ByteStream::byte msgbuf[msgsize]; - memset(msgbuf, 0, sizeof(ISMPacketHeader)); - ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); - hdrp->Command = CACHE_OP_RESULTS; - int32_t* resp = reinterpret_cast(&msgbuf[sizeof(ISMPacketHeader)]); - *resp = result; - return ByteStream(msgbuf, msgsize); + blockCacheClient bc(*BRPp[i]); + bc.flushPartition(oids, partitions); } - /* Message format: - * ISMPacketHeader - * OID count - 32 bits - * OID array - 32 bits * count - */ - void doCacheFlushByOID(SP_UM_IOSOCK ios, ByteStream& bs) - { - uint8_t* buf = bs.buf(); - buf += sizeof(ISMPacketHeader); - uint32_t count = *((uint32_t*) buf); - buf += 4; - uint32_t* oids = (uint32_t*) buf; + ios->write(buildCacheOpResp(0)); + } - for (int i = 0; i < fCacheCount; i++) + void doCacheFlushCmd(SP_UM_IOSOCK ios, const ByteStream& bs) + { + for (int i = 0; i < fCacheCount; i++) + { + blockCacheClient bc(*BRPp[i]); + bc.flushCache(); + } + + ios->write(buildCacheOpResp(0)); + } + + void doCacheDropFDs(SP_UM_IOSOCK ios, ByteStream& bs) + { + std::vector files; + + bs.advance(sizeof(ISMPacketHeader)); + dropFDCache(); + ios->write(buildCacheOpResp(0)); + } + + void doCachePurgeFDs(SP_UM_IOSOCK ios, ByteStream& bs) + { + std::vector files; + + bs.advance(sizeof(ISMPacketHeader)); + deserializeInlineVector(bs, files); + purgeFDCache(files); + ios->write(buildCacheOpResp(0)); + } + + // N.B. this fcn doesn't actually clean the VSS, but rather instructs PP to flush its + // cache of specific LBID's + void doCacheCleanVSSCmd(SP_UM_IOSOCK ios, const ByteStream& bs) + { + const ByteStream::byte* bytePtr = bs.buf(); + const uint32_t* cntp = reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader)]); + const LbidAtVer* itemp = + reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader) + sizeof(uint32_t)]); + + for (int i = 0; i < fCacheCount; i++) + { + blockCacheClient bc(*BRPp[i]); + bc.flushMany(itemp, *cntp); + } + + ios->write(buildCacheOpResp(0)); + } + + void doCacheFlushAllversion(SP_UM_IOSOCK ios, const ByteStream& bs) + { + const ByteStream::byte* bytePtr = bs.buf(); + const uint32_t* cntp = reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader)]); + const LBID_t* itemp = + reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader) + sizeof(uint32_t)]); + + for (int i = 0; i < fCacheCount; i++) + { + blockCacheClient bc(*BRPp[i]); + bc.flushManyAllversion(itemp, *cntp); + } + + ios->write(buildCacheOpResp(0)); + } + + void operator()() + { + utils::setThreadName("PPReadThread"); + threadpool::PriorityThreadPool* procPoolPtr = fPrimitiveServerPtr->getProcessorThreadPool(); + SBS bs; + UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance(); + + // Establish default output IOSocket (and mutex) based on the input + // IOSocket. If we end up rotating through multiple output sockets + // for the same UM, we will use UmSocketSelector to select output. + SP_UM_IOSOCK outIosDefault(new IOSocket(fIos)); + SP_UM_MUTEX writeLockDefault(new boost::mutex()); + + bool bRotateDest = fPrimitiveServerPtr->rotatingDestination(); + + if (bRotateDest) + { + // If we tried adding an IP address not listed as UM in config + // file; probably a DMLProc connection. We allow the connection + // but disable destination rotation since not in Columnstore.xml. + if (!pUmSocketSelector->addConnection(outIosDefault, writeLockDefault)) + { + bRotateDest = false; + } + } + + SP_UM_IOSOCK outIos(outIosDefault); + SP_UM_MUTEX writeLock(writeLockDefault); + + //..Loop to process incoming messages on IOSocket fIos + for (;;) + { + try + { + bs = fIos.read(); + } + catch (...) + { + // This connection is dead, nothing useful will come from it ever again + // We can't rely on the state of bs at this point... + if (bRotateDest && pUmSocketSelector) + pUmSocketSelector->delConnection(fIos); + + fIos.close(); + break; + } + + try + { + if (bs->length() != 0) { - blockCacheClient bc(*BRPp[i]); - bc.flushOIDs(oids, count); - } + idbassert(bs->length() >= sizeof(ISMPacketHeader)); - ios->write(buildCacheOpResp(0)); - } + const ISMPacketHeader* ismHdr = reinterpret_cast(bs->buf()); - /* Message format: - * ISMPacketHeader - * Partition count - 32 bits - * Partition set - sizeof(LogicalPartition) boost::shared_ptr* count - * OID count - 32 bits - * OID array - 32 bits * count - */ - void doCacheFlushByPartition(SP_UM_IOSOCK ios, ByteStream& bs) - { - set partitions; - vector oids; + /* This switch is for the OOB commands */ + switch (ismHdr->Command) + { + case CACHE_FLUSH_PARTITION: + doCacheFlushByPartition(outIos, *bs); + fIos.close(); + return; - bs.advance(sizeof(ISMPacketHeader)); - deserializeSet(bs, partitions); - deserializeInlineVector(bs, oids); + case CACHE_FLUSH_BY_OID: + doCacheFlushByOID(outIos, *bs); + fIos.close(); + return; - idbassert(bs.length() == 0); + case CACHE_FLUSH: + doCacheFlushCmd(outIos, *bs); + fIos.close(); + return; - for (int i = 0; i < fCacheCount; i++) - { - blockCacheClient bc(*BRPp[i]); - bc.flushPartition(oids, partitions); - } + case CACHE_CLEAN_VSS: + doCacheCleanVSSCmd(outIos, *bs); + fIos.close(); + return; - ios->write(buildCacheOpResp(0)); - } + case FLUSH_ALL_VERSION: + doCacheFlushAllversion(outIos, *bs); + fIos.close(); + return; - void doCacheFlushCmd(SP_UM_IOSOCK ios, const ByteStream& bs) - { - for (int i = 0; i < fCacheCount; i++) - { - blockCacheClient bc(*BRPp[i]); - bc.flushCache(); - } + case CACHE_DROP_FDS: + doCacheDropFDs(outIos, *bs); + fIos.close(); + return; - ios->write(buildCacheOpResp(0)); - } + case CACHE_PURGE_FDS: + doCachePurgeFDs(outIos, *bs); + fIos.close(); + return; - void doCacheDropFDs(SP_UM_IOSOCK ios, ByteStream& bs) - { - std::vector files; + default: break; + } - bs.advance(sizeof(ISMPacketHeader)); - dropFDCache(); - ios->write(buildCacheOpResp(0)); - } - - void doCachePurgeFDs(SP_UM_IOSOCK ios, ByteStream& bs) - { - std::vector files; - - bs.advance(sizeof(ISMPacketHeader)); - deserializeInlineVector(bs, files); - purgeFDCache(files); - ios->write(buildCacheOpResp(0)); - } - - //N.B. this fcn doesn't actually clean the VSS, but rather instructs PP to flush its - // cache of specific LBID's - void doCacheCleanVSSCmd(SP_UM_IOSOCK ios, const ByteStream& bs) - { - const ByteStream::byte* bytePtr = bs.buf(); - const uint32_t* cntp = reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader)]); - const LbidAtVer* itemp = - reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader) + sizeof(uint32_t)]); - - for (int i = 0; i < fCacheCount; i++) - { - blockCacheClient bc(*BRPp[i]); - bc.flushMany(itemp, *cntp); - } - - ios->write(buildCacheOpResp(0)); - } - - void doCacheFlushAllversion(SP_UM_IOSOCK ios, const ByteStream& bs) - { - const ByteStream::byte* bytePtr = bs.buf(); - const uint32_t* cntp = reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader)]); - const LBID_t* itemp = - reinterpret_cast(&bytePtr[sizeof(ISMPacketHeader) + sizeof(uint32_t)]); - - for (int i = 0; i < fCacheCount; i++) - { - blockCacheClient bc(*BRPp[i]); - bc.flushManyAllversion(itemp, *cntp); - } - - ios->write(buildCacheOpResp(0)); - } - - void operator()() - { - utils::setThreadName("PPReadThread"); - threadpool::PriorityThreadPool* procPoolPtr = fPrimitiveServerPtr->getProcessorThreadPool(); - SBS bs; - UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance(); - - // Establish default output IOSocket (and mutex) based on the input - // IOSocket. If we end up rotating through multiple output sockets - // for the same UM, we will use UmSocketSelector to select output. - SP_UM_IOSOCK outIosDefault(new IOSocket(fIos)); - SP_UM_MUTEX writeLockDefault(new boost::mutex()); - - bool bRotateDest = fPrimitiveServerPtr->rotatingDestination(); - - if (bRotateDest) - { - // If we tried adding an IP address not listed as UM in config - // file; probably a DMLProc connection. We allow the connection - // but disable destination rotation since not in Columnstore.xml. - if (!pUmSocketSelector->addConnection(outIosDefault, writeLockDefault)) + switch (ismHdr->Command) + { + case DICT_CREATE_EQUALITY_FILTER: { - bRotateDest = false; - } - } - - SP_UM_IOSOCK outIos(outIosDefault); - SP_UM_MUTEX writeLock(writeLockDefault); - - //..Loop to process incoming messages on IOSocket fIos - for (;;) - { - try - { - bs = fIos.read(); - } - catch (...) - { - //This connection is dead, nothing useful will come from it ever again - //We can't rely on the state of bs at this point... - if (bRotateDest && pUmSocketSelector) - pUmSocketSelector->delConnection(fIos); - - fIos.close(); - break; + PriorityThreadPool::Job job; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + job.functor = boost::shared_ptr(new CreateEqualityFilter(bs)); + OOBPool->addJob(job); + break; } - try + case DICT_DESTROY_EQUALITY_FILTER: { - if (bs->length() != 0) + PriorityThreadPool::Job job; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + job.functor = boost::shared_ptr(new DestroyEqualityFilter(bs)); + OOBPool->addJob(job); + break; + } + + case DICT_TOKEN_BY_SCAN_COMPARE: + { + idbassert(bs->length() >= sizeof(TokenByScanRequestHeader)); + TokenByScanRequestHeader* hdr = (TokenByScanRequestHeader*)ismHdr; + + if (bRotateDest) + { + if (!pUmSocketSelector->nextIOSocket(fIos, outIos, writeLock)) { - idbassert(bs->length() >= sizeof(ISMPacketHeader)); + // If we ever fall into this part of the + // code we have a "bug" of some sort. + // See handleUmSockSelErr() for more info. + // We reset ios and mutex to defaults. + handleUmSockSelErr(string("default cmd")); + outIos = outIosDefault; + writeLock = writeLockDefault; + pUmSocketSelector->delConnection(fIos); + bRotateDest = false; + } + } - const ISMPacketHeader* ismHdr = reinterpret_cast(bs->buf()); + PriorityThreadPool::Job job; + job.functor = boost::shared_ptr(new DictScanJob(outIos, bs, writeLock)); + job.id = hdr->Hdr.UniqueID; + job.weight = LOGICAL_BLOCK_RIDS; + job.priority = hdr->Hdr.Priority; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; - /* This switch is for the OOB commands */ - switch (ismHdr->Command) - { - case CACHE_FLUSH_PARTITION: - doCacheFlushByPartition(outIos, *bs); - fIos.close(); - return; + if (hdr->flags & IS_SYSCAT) + { + // boost::thread t(DictScanJob(outIos, bs, writeLock)); + // using already-existing threads may cut latency + // if it's changed back to running in an independent thread + // change the issyscat() checks in BPPSeeder as well + OOBPool->addJob(job); + } + else + { + procPoolPtr->addJob(job); + } - case CACHE_FLUSH_BY_OID: - doCacheFlushByOID(outIos, *bs); - fIos.close(); - return; + break; + } - case CACHE_FLUSH: - doCacheFlushCmd(outIos, *bs); - fIos.close(); - return; + case BATCH_PRIMITIVE_RUN: + { + if (bRotateDest) + { + if (!pUmSocketSelector->nextIOSocket(fIos, outIos, writeLock)) + { + // If we ever fall into this part of the + // code we have a "bug" of some sort. + // See handleUmSockSelErr() for more info. + // We reset ios and mutex to defaults. + handleUmSockSelErr(string("BPR cmd")); + outIos = outIosDefault; + writeLock = writeLockDefault; + pUmSocketSelector->delConnection(fIos); + bRotateDest = false; + } + } - case CACHE_CLEAN_VSS: - doCacheCleanVSSCmd(outIos, *bs); - fIos.close(); - return; - - case FLUSH_ALL_VERSION: - doCacheFlushAllversion(outIos, *bs); - fIos.close(); - return; - - case CACHE_DROP_FDS: - doCacheDropFDs(outIos, *bs); - fIos.close(); - return; - - case CACHE_PURGE_FDS: - doCachePurgeFDs(outIos, *bs); - fIos.close(); - return; - - default: - break; - } - - switch (ismHdr->Command) - { - case DICT_CREATE_EQUALITY_FILTER: - { - PriorityThreadPool::Job job; - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - job.functor = boost::shared_ptr(new CreateEqualityFilter(bs)); - OOBPool->addJob(job); - break; - } - - case DICT_DESTROY_EQUALITY_FILTER: - { - PriorityThreadPool::Job job; - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - job.functor = boost::shared_ptr(new DestroyEqualityFilter(bs)); - OOBPool->addJob(job); - break; - } - - case DICT_TOKEN_BY_SCAN_COMPARE: - { - idbassert(bs->length() >= sizeof(TokenByScanRequestHeader)); - TokenByScanRequestHeader* hdr = (TokenByScanRequestHeader*) ismHdr; - - if (bRotateDest) - { - if (!pUmSocketSelector->nextIOSocket( - fIos, outIos, writeLock)) - { - // If we ever fall into this part of the - // code we have a "bug" of some sort. - // See handleUmSockSelErr() for more info. - // We reset ios and mutex to defaults. - handleUmSockSelErr(string("default cmd")); - outIos = outIosDefault; - writeLock = writeLockDefault; - pUmSocketSelector->delConnection(fIos); - bRotateDest = false; - } - } - - PriorityThreadPool::Job job; - job.functor = boost::shared_ptr(new DictScanJob(outIos, - bs, writeLock)); - job.id = hdr->Hdr.UniqueID; - job.weight = LOGICAL_BLOCK_RIDS; - job.priority = hdr->Hdr.Priority; - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - - if (hdr->flags & IS_SYSCAT) - { - //boost::thread t(DictScanJob(outIos, bs, writeLock)); - // using already-existing threads may cut latency - // if it's changed back to running in an independent thread - // change the issyscat() checks in BPPSeeder as well - OOBPool->addJob(job); - } - else - { - procPoolPtr->addJob(job); - } - - break; - } - - case BATCH_PRIMITIVE_RUN: - { - if (bRotateDest) - { - if (!pUmSocketSelector->nextIOSocket( - fIos, outIos, writeLock)) - { - - // If we ever fall into this part of the - // code we have a "bug" of some sort. - // See handleUmSockSelErr() for more info. - // We reset ios and mutex to defaults. - handleUmSockSelErr(string("BPR cmd")); - outIos = outIosDefault; - writeLock = writeLockDefault; - pUmSocketSelector->delConnection(fIos); - bRotateDest = false; - } - } - - /* Decide whether this is a syscat call and run - right away instead of queueing */ - boost::shared_ptr bpps(new BPPSeeder(bs, writeLock, outIos, + /* Decide whether this is a syscat call and run + right away instead of queueing */ + boost::shared_ptr bpps(new BPPSeeder(bs, writeLock, outIos, fPrimitiveServerPtr->ProcessorThreads(), fPrimitiveServerPtr->PTTrace())); - PriorityThreadPool::Job job; - job.functor = bpps; - job.id = bpps->getID(); - job.weight = ismHdr->Size; - job.priority = bpps->priority(); - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; + PriorityThreadPool::Job job; + job.functor = bpps; + job.id = bpps->getID(); + job.weight = ismHdr->Size; + job.priority = bpps->priority(); + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; - if (bpps->isSysCat()) - { + if (bpps->isSysCat()) + { + // boost::thread t(*bpps); + // using already-existing threads may cut latency + // if it's changed back to running in an independent thread + // change the issyscat() checks in BPPSeeder as well + OOBPool->addJob(job); + } + else + { + procPoolPtr->addJob(job); + } - //boost::thread t(*bpps); - // using already-existing threads may cut latency - // if it's changed back to running in an independent thread - // change the issyscat() checks in BPPSeeder as well - OOBPool->addJob(job); - } - else - { - procPoolPtr->addJob(job); - } - - break; - } - - case BATCH_PRIMITIVE_CREATE: - { - PriorityThreadPool::Job job; - job.functor = boost::shared_ptr(new BPPHandler::Create(fBPPHandler, bs)); - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - OOBPool->addJob(job); - //fBPPHandler->createBPP(*bs); - break; - } - - case BATCH_PRIMITIVE_ADD_JOINER: - { - PriorityThreadPool::Job job; - job.functor = boost::shared_ptr(new BPPHandler::AddJoiner(fBPPHandler, bs)); - job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - OOBPool->addJob(job); - //fBPPHandler->addJoinerToBPP(*bs); - break; - } - - case BATCH_PRIMITIVE_END_JOINER: - { - // lastJoinerMsg can block; must do this in a different thread - //OOBPool->invoke(BPPHandler::LastJoiner(fBPPHandler, bs)); // needs a threadpool that can resched - //boost::thread tmp(BPPHandler::LastJoiner(fBPPHandler, bs)); - PriorityThreadPool::Job job; - job.functor = boost::shared_ptr(new BPPHandler::LastJoiner(fBPPHandler, bs)); - job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - OOBPool->addJob(job); - break; - } - - case BATCH_PRIMITIVE_DESTROY: - { - //OOBPool->invoke(BPPHandler::Destroy(fBPPHandler, bs)); // needs a threadpool that can resched - //boost::thread tmp(BPPHandler::Destroy(fBPPHandler, bs)); - PriorityThreadPool::Job job; - job.functor = boost::shared_ptr(new BPPHandler::Destroy(fBPPHandler, bs)); - job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - OOBPool->addJob(job); - //fBPPHandler->destroyBPP(*bs); - break; - } - - case BATCH_PRIMITIVE_ACK: - { - fBPPHandler->doAck(*bs); - break; - } - - case BATCH_PRIMITIVE_ABORT: - { - //OBPool->invoke(BPPHandler::Abort(fBPPHandler, bs)); - //fBPPHandler->doAbort(*bs); - PriorityThreadPool::Job job; - job.functor = boost::shared_ptr(new BPPHandler::Abort(fBPPHandler, bs)); - job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t* buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t*) &buf[pos + 6]); - job.uniqueID = *((uint32_t*) &buf[pos + 10]); - job.sock = outIos; - OOBPool->addJob(job); - break; - } - - default: - { - std::ostringstream os; - Logger log; - os << "unknown primitive cmd: " << ismHdr->Command; - log.logMessage(os.str()); - break; - } - } // the switch stmt - } - else // bs.length() == 0 - { - if (bRotateDest) - pUmSocketSelector->delConnection(fIos); - - fIos.close(); - break; - } - } // the try- surrounding the if stmt - catch (std::exception& e) - { - Logger logger; - logger.logMessage(e.what()); + break; } + + case BATCH_PRIMITIVE_CREATE: + { + PriorityThreadPool::Job job; + job.functor = + boost::shared_ptr(new BPPHandler::Create(fBPPHandler, bs)); + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + OOBPool->addJob(job); + // fBPPHandler->createBPP(*bs); + break; + } + + case BATCH_PRIMITIVE_ADD_JOINER: + { + PriorityThreadPool::Job job; + job.functor = + boost::shared_ptr(new BPPHandler::AddJoiner(fBPPHandler, bs)); + job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + OOBPool->addJob(job); + // fBPPHandler->addJoinerToBPP(*bs); + break; + } + + case BATCH_PRIMITIVE_END_JOINER: + { + // lastJoinerMsg can block; must do this in a different thread + // OOBPool->invoke(BPPHandler::LastJoiner(fBPPHandler, bs)); // needs a threadpool that can + // resched boost::thread tmp(BPPHandler::LastJoiner(fBPPHandler, bs)); + PriorityThreadPool::Job job; + job.functor = + boost::shared_ptr(new BPPHandler::LastJoiner(fBPPHandler, bs)); + job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + OOBPool->addJob(job); + break; + } + + case BATCH_PRIMITIVE_DESTROY: + { + // OOBPool->invoke(BPPHandler::Destroy(fBPPHandler, bs)); // needs a threadpool that can + // resched boost::thread tmp(BPPHandler::Destroy(fBPPHandler, bs)); + PriorityThreadPool::Job job; + job.functor = + boost::shared_ptr(new BPPHandler::Destroy(fBPPHandler, bs)); + job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + OOBPool->addJob(job); + // fBPPHandler->destroyBPP(*bs); + break; + } + + case BATCH_PRIMITIVE_ACK: + { + fBPPHandler->doAck(*bs); + break; + } + + case BATCH_PRIMITIVE_ABORT: + { + // OBPool->invoke(BPPHandler::Abort(fBPPHandler, bs)); + // fBPPHandler->doAbort(*bs); + PriorityThreadPool::Job job; + job.functor = + boost::shared_ptr(new BPPHandler::Abort(fBPPHandler, bs)); + job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*)&buf[pos + 6]); + job.uniqueID = *((uint32_t*)&buf[pos + 10]); + job.sock = outIos; + OOBPool->addJob(job); + break; + } + + default: + { + std::ostringstream os; + Logger log; + os << "unknown primitive cmd: " << ismHdr->Command; + log.logMessage(os.str()); + break; + } + } // the switch stmt } - } + else // bs.length() == 0 + { + if (bRotateDest) + pUmSocketSelector->delConnection(fIos); -// If this function is called, we have a "bug" of some sort. We added -// the "fIos" connection to UmSocketSelector earlier, so at the very -// least, UmSocketSelector should have been able to return that con- -// nection/port. We will try to recover by using the original fIos to -// send the response msg; but as stated, if this ever happens we have -// a bug we need to resolve. - void handleUmSockSelErr(const string& cmd) - { - ostringstream oss; - oss << "Unable to rotate through socket destinations (" << - cmd << ") for connection: " << fIos.toString(); - cerr << oss.str() << endl; - logging::Message::Args args; - args.add(oss.str()); - mlp->logMessage(logging::M0058, args, false); + fIos.close(); + break; + } + } // the try- surrounding the if stmt + catch (std::exception& e) + { + Logger logger; + logger.logMessage(e.what()); + } } + } - ~ReadThread() {} - string fServerName; - IOSocket fIos; - PrimitiveServer* fPrimitiveServerPtr; - boost::shared_ptr fBPPHandler; + // If this function is called, we have a "bug" of some sort. We added + // the "fIos" connection to UmSocketSelector earlier, so at the very + // least, UmSocketSelector should have been able to return that con- + // nection/port. We will try to recover by using the original fIos to + // send the response msg; but as stated, if this ever happens we have + // a bug we need to resolve. + void handleUmSockSelErr(const string& cmd) + { + ostringstream oss; + oss << "Unable to rotate through socket destinations (" << cmd << ") for connection: " << fIos.toString(); + cerr << oss.str() << endl; + logging::Message::Args args; + args.add(oss.str()); + mlp->logMessage(logging::M0058, args, false); + } + + ~ReadThread() + { + } + string fServerName; + IOSocket fIos; + PrimitiveServer* fPrimitiveServerPtr; + boost::shared_ptr fBPPHandler; }; /** @brief accept a primitive command from the user module */ struct ServerThread { - ServerThread(string serverName, PrimitiveServer* ps) : - fServerName(serverName), fPrimitiveServerPtr(ps) + ServerThread(string serverName, PrimitiveServer* ps) : fServerName(serverName), fPrimitiveServerPtr(ps) + { + SUMMARY_INFO2("starting server ", fServerName); + + bool tellUser = true; + bool toldUser = false; + + for (;;) { - SUMMARY_INFO2("starting server ", fServerName); + try + { + mqServerPtr = new MessageQueueServer(fServerName); + break; + } + catch (runtime_error& re) + { + string what = re.what(); - bool tellUser = true; - bool toldUser = false; - - for (;;) + if (what.find("Address already in use") != string::npos) { - try - { - mqServerPtr = new MessageQueueServer(fServerName); - break; - } - catch (runtime_error& re) - { - string what = re.what(); + if (tellUser) + { + cerr << "Address already in use, retrying..." << endl; + tellUser = false; + toldUser = true; + } - if (what.find("Address already in use") != string::npos) - { - if (tellUser) - { - cerr << "Address already in use, retrying..." << endl; - tellUser = false; - toldUser = true; - } - - sleep(5); - } - else - { - throw; - } - } + sleep(5); } - - if (toldUser) - cerr << "Ready." << endl; + else + { + throw; + } + } } - void operator()() - { - utils::setThreadName("PPServerThr"); - IOSocket ios; + if (toldUser) + cerr << "Ready." << endl; + } - try - { - for (;;) - { - ios = mqServerPtr->accept(); - //startup a detached thread to handle this socket's I/O - boost::thread rt(ReadThread(fServerName, ios, fPrimitiveServerPtr)); - } - } - catch (std::exception& ex) - { - SUMMARY_INFO2("exception caught in ServerThread: ", ex.what()); - } - catch (...) - { - SUMMARY_INFO("exception caught in ServerThread."); - } - } - - string fServerName; - PrimitiveServer* fPrimitiveServerPtr; - MessageQueueServer* mqServerPtr; -}; - -} // namespace anon - -namespace primitiveprocessor -{ -PrimitiveServer::PrimitiveServer(int serverThreads, - int serverQueueSize, - int processorWeight, - int processorQueueSize, - bool rotatingDestination, - uint32_t BRPBlocks, - int BRPThreads, - int cacheCount, - int maxBlocksPerRead, - int readAheadBlocks, - uint32_t deleteBlocks, - bool ptTrace, - double prefetch, - uint64_t smallSide - ): - fServerThreads(serverThreads), - fServerQueueSize(serverQueueSize), - fProcessorWeight(processorWeight), - fProcessorQueueSize(processorQueueSize), - fMaxBlocksPerRead(maxBlocksPerRead), - fReadAheadBlocks(readAheadBlocks), - fRotatingDestination(rotatingDestination), - fPTTrace(ptTrace), - fPrefetchThreshold(prefetch), - fPMSmallSide(smallSide) -{ - fCacheCount = cacheCount; - fServerpool.setMaxThreads(fServerThreads); - fServerpool.setQueueSize(fServerQueueSize); - fServerpool.setName("PrimitiveServer"); - - fProcessorPool = new threadpool::PriorityThreadPool(fProcessorWeight, highPriorityThreads, - medPriorityThreads, lowPriorityThreads, 0); - - // We're not using either the priority or the job-clustering features, just need a threadpool - // that can reschedule jobs, and an unlimited non-blocking queue - OOBPool.reset(new threadpool::PriorityThreadPool(1, 5, 0, 0, 1)); - - asyncCounter = 0; - - brm = new DBRM(); - - BRPp = new BlockRequestProcessor*[fCacheCount]; + void operator()() + { + utils::setThreadName("PPServerThr"); + IOSocket ios; try { - for (int i = 0; i < fCacheCount; i++) - BRPp[i] = new BlockRequestProcessor(BRPBlocks / fCacheCount, BRPThreads / fCacheCount, - fMaxBlocksPerRead, deleteBlocks / fCacheCount); + for (;;) + { + ios = mqServerPtr->accept(); + // startup a detached thread to handle this socket's I/O + boost::thread rt(ReadThread(fServerName, ios, fPrimitiveServerPtr)); + } + } + catch (std::exception& ex) + { + SUMMARY_INFO2("exception caught in ServerThread: ", ex.what()); } catch (...) { - cerr << "Unable to allocate " << BRPBlocks << " cache blocks. Adjust the DBBC config parameter " - "downward." << endl; - mlp->logMessage(logging::M0045, logging::Message::Args(), true); - exit(1); + SUMMARY_INFO("exception caught in ServerThread."); } + } + + string fServerName; + PrimitiveServer* fPrimitiveServerPtr; + MessageQueueServer* mqServerPtr; +}; + +} // namespace + +namespace primitiveprocessor +{ +PrimitiveServer::PrimitiveServer(int serverThreads, int serverQueueSize, int processorWeight, + int processorQueueSize, bool rotatingDestination, uint32_t BRPBlocks, + int BRPThreads, int cacheCount, int maxBlocksPerRead, int readAheadBlocks, + uint32_t deleteBlocks, bool ptTrace, double prefetch, uint64_t smallSide) + : fServerThreads(serverThreads) + , fServerQueueSize(serverQueueSize) + , fProcessorWeight(processorWeight) + , fProcessorQueueSize(processorQueueSize) + , fMaxBlocksPerRead(maxBlocksPerRead) + , fReadAheadBlocks(readAheadBlocks) + , fRotatingDestination(rotatingDestination) + , fPTTrace(ptTrace) + , fPrefetchThreshold(prefetch) + , fPMSmallSide(smallSide) +{ + fCacheCount = cacheCount; + fServerpool.setMaxThreads(fServerThreads); + fServerpool.setQueueSize(fServerQueueSize); + fServerpool.setName("PrimitiveServer"); + + fProcessorPool = new threadpool::PriorityThreadPool(fProcessorWeight, highPriorityThreads, + medPriorityThreads, lowPriorityThreads, 0); + + // We're not using either the priority or the job-clustering features, just need a threadpool + // that can reschedule jobs, and an unlimited non-blocking queue + OOBPool.reset(new threadpool::PriorityThreadPool(1, 5, 0, 0, 1)); + + asyncCounter = 0; + + brm = new DBRM(); + + BRPp = new BlockRequestProcessor*[fCacheCount]; + + try + { + for (int i = 0; i < fCacheCount; i++) + BRPp[i] = new BlockRequestProcessor(BRPBlocks / fCacheCount, BRPThreads / fCacheCount, + fMaxBlocksPerRead, deleteBlocks / fCacheCount); + } + catch (...) + { + cerr << "Unable to allocate " << BRPBlocks + << " cache blocks. Adjust the DBBC config parameter " + "downward." + << endl; + mlp->logMessage(logging::M0045, logging::Message::Args(), true); + exit(1); + } } PrimitiveServer::~PrimitiveServer() { } -void PrimitiveServer::start(Service *service) +void PrimitiveServer::start(Service* service) { - // start all the server threads - for ( int i = 1; i <= fServerThreads; i++) - { - string s("PMS"); - stringstream oss; - oss << s << i; + // start all the server threads + for (int i = 1; i <= fServerThreads; i++) + { + string s("PMS"); + stringstream oss; + oss << s << i; - fServerpool.invoke(ServerThread(oss.str(), this)); - } + fServerpool.invoke(ServerThread(oss.str(), this)); + } - service->NotifyServiceStarted(); + service->NotifyServiceStarted(); - fServerpool.wait(); + fServerpool.wait(); - cerr << "PrimitiveServer::start() exiting!" << endl; + cerr << "PrimitiveServer::start() exiting!" << endl; } BPPV::BPPV(PrimitiveServer* ps) { - sendThread.reset(new BPPSendThread()); - sendThread->setProcessorPool(ps->getProcessorThreadPool()); - v.reserve(BPPCount); - pos = 0; - joinDataReceived = false; + sendThread.reset(new BPPSendThread()); + sendThread->setProcessorPool(ps->getProcessorThreadPool()); + v.reserve(BPPCount); + pos = 0; + joinDataReceived = false; } BPPV::~BPPV() @@ -2540,37 +2474,36 @@ BPPV::~BPPV() void BPPV::add(boost::shared_ptr a) { - /* Right now BPP initialization locks the object if there is a join to - prevent the instance from being used before the join data is received. - The init/join/run sync code is quite old and confusing at this point, - and this makes it a little worse by circumventing the original design. - Due for a rewrite. */ + /* Right now BPP initialization locks the object if there is a join to + prevent the instance from being used before the join data is received. + The init/join/run sync code is quite old and confusing at this point, + and this makes it a little worse by circumventing the original design. + Due for a rewrite. */ - if (!unusedInstance) + if (!unusedInstance) + { + unusedInstance = a->duplicate(); + + if (a->hasJoin()) { - unusedInstance = a->duplicate(); - - if (a->hasJoin()) - { - joinDataReceived = false; - unusedInstance->unlock(); - } - else - joinDataReceived = true; + joinDataReceived = false; + unusedInstance->unlock(); } + else + joinDataReceived = true; + } - v.push_back(a); + v.push_back(a); } const vector >& BPPV::get() { - return v; + return v; } - boost::shared_ptr BPPV::next() { - uint32_t size = v.size(); - uint32_t i = 0; + uint32_t size = v.size(); + uint32_t i = 0; #if 0 @@ -2593,54 +2526,53 @@ boost::shared_ptr BPPV::next() return boost::shared_ptr(); #endif - //This block of code creates BPP instances if/when they are needed + // This block of code creates BPP instances if/when they are needed - // don't use a processor thread when it will just block, reschedule it instead - if (!joinDataReceived) - return boost::shared_ptr(); + // don't use a processor thread when it will just block, reschedule it instead + if (!joinDataReceived) + return boost::shared_ptr(); - for (i = 0; i < size; i++) + for (i = 0; i < size; i++) + { + uint32_t index = (i + pos) % size; + + if (!(v[index]->busy())) { - uint32_t index = (i + pos) % size; - - if (!(v[index]->busy())) - { - pos = (index + 1) % size; - v[index]->busy(true); - return v[index]; - } + pos = (index + 1) % size; + v[index]->busy(true); + return v[index]; } + } - // honor the BPPCount limit, mostly for debugging purposes. - if (size >= BPPCount) - return boost::shared_ptr(); + // honor the BPPCount limit, mostly for debugging purposes. + if (size >= BPPCount) + return boost::shared_ptr(); - SBPP newone = unusedInstance->duplicate(); + SBPP newone = unusedInstance->duplicate(); - if (newone->hasJoin()) - newone->unlock(); + if (newone->hasJoin()) + newone->unlock(); - newone->busy(true); - v.push_back(newone); - return newone; + newone->busy(true); + v.push_back(newone); + return newone; } void BPPV::abort() { - sendThread->abort(); - BOOST_FOREACH( boost::shared_ptr bpp, v ) - { - bpp->unlock(); - } + sendThread->abort(); + BOOST_FOREACH (boost::shared_ptr bpp, v) + { + bpp->unlock(); + } } bool BPPV::aborted() { - return sendThread->aborted(); + return sendThread->aborted(); } // end workaround -} // namespace primitiveprocessor +} // namespace primitiveprocessor // vim:ts=4 sw=4: - diff --git a/primitives/primproc/primitiveserver.h b/primitives/primproc/primitiveserver.h index af86204f7..543faade3 100644 --- a/primitives/primproc/primitiveserver.h +++ b/primitives/primproc/primitiveserver.h @@ -59,29 +59,30 @@ class PrimitiveServer; class BPPV { -public: - BPPV(PrimitiveServer* ps); - ~BPPV(); - boost::shared_ptr next(); - void add(boost::shared_ptr a); - const std::vector >& get(); - inline boost::shared_ptr getSendThread() - { - return sendThread; - } - void abort(); - bool aborted(); - volatile bool joinDataReceived; -private: - std::vector > v; - boost::shared_ptr sendThread; + public: + BPPV(PrimitiveServer* ps); + ~BPPV(); + boost::shared_ptr next(); + void add(boost::shared_ptr a); + const std::vector >& get(); + inline boost::shared_ptr getSendThread() + { + return sendThread; + } + void abort(); + bool aborted(); + volatile bool joinDataReceived; - // the instance other instances are created from - boost::shared_ptr unusedInstance; + private: + std::vector > v; + boost::shared_ptr sendThread; - // pos keeps the position of the last BPP returned by next(), - // next() will start searching at this pos on the next call. - uint32_t pos; + // the instance other instances are created from + boost::shared_ptr unusedInstance; + + // pos keeps the position of the last BPP returned by next(), + // next() will start searching at this pos on the next call. + uint32_t pos; }; typedef boost::shared_ptr SBPPV; @@ -93,99 +94,91 @@ void prefetchExtent(uint64_t lbid, uint32_t ver, uint32_t txn, uint32_t* rCount) void loadBlock(uint64_t lbid, BRM::QueryContext q, uint32_t txn, int compType, void* bufferPtr, bool* pWasBlockInCache, uint32_t* rCount = NULL, bool LBIDTrace = false, uint32_t sessionID = 0, bool doPrefetch = true, VSSCache* vssCache = NULL); -void loadBlockAsync(uint64_t lbid, const BRM::QueryContext& q, uint32_t txn, int CompType, - uint32_t* cCount, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, - boost::mutex* m, uint32_t* busyLoaders, boost::shared_ptr sendThread, VSSCache* vssCache = 0); +void loadBlockAsync(uint64_t lbid, const BRM::QueryContext& q, uint32_t txn, int CompType, uint32_t* cCount, + uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, boost::mutex* m, + uint32_t* busyLoaders, boost::shared_ptr sendThread, + VSSCache* vssCache = 0); uint32_t loadBlocks(BRM::LBID_t* lbids, BRM::QueryContext q, BRM::VER_t txn, int compType, uint8_t** bufferPtrs, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, - uint32_t blockCount, bool* wasVersioned, bool doPrefetch = true, VSSCache* vssCache = NULL); + uint32_t blockCount, bool* wasVersioned, bool doPrefetch = true, + VSSCache* vssCache = NULL); uint32_t cacheNum(uint64_t lbid); void buildFileName(BRM::OID_t oid, char* fileName); /** @brief process primitives as they arrive -*/ + */ class PrimitiveServer { -public: - /** @brief ctor - */ - PrimitiveServer(int serverThreads, - int serverQueueSize, - int processorWeight, - int processorQueueSize, - bool rotatingDestination, - uint32_t BRPBlocks = (1024 * 1024 * 2), - int BRPThreads = 64, - int cacheCount = 8, - int maxBlocksPerRead = 128, - int readAheadBlocks = 256, - uint32_t deleteBlocks = 0, - bool ptTrace = false, - double prefetchThreshold = 0, - uint64_t pmSmallSide = 0); + public: + /** @brief ctor + */ + PrimitiveServer(int serverThreads, int serverQueueSize, int processorWeight, int processorQueueSize, + bool rotatingDestination, uint32_t BRPBlocks = (1024 * 1024 * 2), int BRPThreads = 64, + int cacheCount = 8, int maxBlocksPerRead = 128, int readAheadBlocks = 256, + uint32_t deleteBlocks = 0, bool ptTrace = false, double prefetchThreshold = 0, + uint64_t pmSmallSide = 0); - /** @brief dtor - */ - ~PrimitiveServer(); + /** @brief dtor + */ + ~PrimitiveServer(); - /** @brief start the primitive server - * - */ - void start(Service *p); + /** @brief start the primitive server + * + */ + void start(Service* p); - /** @brief get a pointer the shared processor thread pool - */ - inline threadpool::PriorityThreadPool* getProcessorThreadPool() const - { - return fProcessorPool; - } + /** @brief get a pointer the shared processor thread pool + */ + inline threadpool::PriorityThreadPool* getProcessorThreadPool() const + { + return fProcessorPool; + } -// int fCacheCount; - int ReadAheadBlocks() const - { - return fReadAheadBlocks; - } - bool rotatingDestination() const - { - return fRotatingDestination; - } - bool PTTrace() const - { - return fPTTrace; - } - double prefetchThreshold() const - { - return fPrefetchThreshold; - } - uint32_t ProcessorThreads() const - { - return highPriorityThreads + medPriorityThreads + lowPriorityThreads; - } -protected: + // int fCacheCount; + int ReadAheadBlocks() const + { + return fReadAheadBlocks; + } + bool rotatingDestination() const + { + return fRotatingDestination; + } + bool PTTrace() const + { + return fPTTrace; + } + double prefetchThreshold() const + { + return fPrefetchThreshold; + } + uint32_t ProcessorThreads() const + { + return highPriorityThreads + medPriorityThreads + lowPriorityThreads; + } -private: - /** @brief the thread pool used to listen for - * incoming primitive commands - */ - threadpool::ThreadPool fServerpool; + protected: + private: + /** @brief the thread pool used to listen for + * incoming primitive commands + */ + threadpool::ThreadPool fServerpool; - /** @brief the thread pool used to process - * primitive commands - */ - threadpool::PriorityThreadPool* fProcessorPool; + /** @brief the thread pool used to process + * primitive commands + */ + threadpool::PriorityThreadPool* fProcessorPool; - int fServerThreads; - int fServerQueueSize; - int fProcessorWeight; - int fProcessorQueueSize; - int fMaxBlocksPerRead; - int fReadAheadBlocks; - bool fRotatingDestination; - bool fPTTrace; - double fPrefetchThreshold; - uint64_t fPMSmallSide; + int fServerThreads; + int fServerQueueSize; + int fProcessorWeight; + int fProcessorQueueSize; + int fMaxBlocksPerRead; + int fReadAheadBlocks; + bool fRotatingDestination; + bool fPTTrace; + double fPrefetchThreshold; + uint64_t fPMSmallSide; }; - -} // namespace primitiveprocessor -#endif //PRIMITIVESERVER_H +} // namespace primitiveprocessor +#endif // PRIMITIVESERVER_H diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 0464c6f46..bc901803d 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -64,7 +64,6 @@ using namespace primitiveprocessor; #include "liboamcpp.h" using namespace oam; - #include "IDBPolicy.h" using namespace idbdatafile; @@ -80,59 +79,49 @@ using namespace idbdatafile; class Opt { -public: - int m_debug; - bool m_fg; - Opt(int argc, char *argv[]) - :m_debug(0), - m_fg(false) - { - int c; + public: + int m_debug; + bool m_fg; + Opt(int argc, char* argv[]) : m_debug(0), m_fg(false) + { + int c; - while ((c = getopt(argc, argv, "df")) != EOF) - { - switch(c) - { - case 'd': - m_debug++; - break; - case 'f': - m_fg= true; - break; - case '?': - default: - break; - } - } + while ((c = getopt(argc, argv, "df")) != EOF) + { + switch (c) + { + case 'd': m_debug++; break; + case 'f': m_fg = true; break; + case '?': + default: break; + } + } } }; - -class ServicePrimProc: public Service, public Opt +class ServicePrimProc : public Service, public Opt { -public: - ServicePrimProc(const Opt &opt) - :Service("PrimProc"), Opt(opt) - { } - void LogErrno() override - { - cerr << strerror(errno) << endl; - } - void ParentLogChildMessage(const std::string &str) override - { - cout << str << endl; - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServicePrimProc(const Opt& opt) : Service("PrimProc"), Opt(opt) + { + } + void LogErrno() override + { + cerr << strerror(errno) << endl; + } + void ParentLogChildMessage(const std::string& str) override + { + cout << str << endl; + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - namespace primitiveprocessor { - extern uint32_t BPPCount; extern uint32_t blocksReadAhead; extern uint32_t defaultBufferSize; @@ -140,162 +129,157 @@ extern uint32_t connectionsPerUM; extern uint32_t highPriorityThreads; extern uint32_t medPriorityThreads; extern uint32_t lowPriorityThreads; -extern int directIOFlag; -extern int noVB; +extern int directIOFlag; +extern int noVB; DebugLevel gDebugLevel; Logger* mlp; -bool isDebug( const DebugLevel level ) +bool isDebug(const DebugLevel level) { - return level <= gDebugLevel; + return level <= gDebugLevel; } -} //namespace primitiveprocessor +} // namespace primitiveprocessor namespace { - int toInt(const string& val) { - if (val.length() == 0) - return -1; + if (val.length() == 0) + return -1; - return static_cast(Config::fromText(val)); + return static_cast(Config::fromText(val)); } void setupSignalHandlers() { #ifndef _MSC_VER - signal(SIGHUP, SIG_IGN); + signal(SIGHUP, SIG_IGN); - struct sigaction ign; + struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGUSR1, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGUSR2, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + sigaction(SIGUSR2, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); - sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGPIPE); - sigaddset(&sigset, SIGUSR1); - sigaddset(&sigset, SIGUSR2); - sigprocmask(SIG_BLOCK, &sigset, 0); + sigset_t sigset; + sigemptyset(&sigset); + sigaddset(&sigset, SIGPIPE); + sigaddset(&sigset, SIGUSR1); + sigaddset(&sigset, SIGUSR2); + sigprocmask(SIG_BLOCK, &sigset, 0); #endif } int8_t setupCwd(Config* cf) { - string workdir = startup::StartUp::tmpDir(); + string workdir = startup::StartUp::tmpDir(); - if (workdir.length() == 0) - workdir = "."; + if (workdir.length() == 0) + workdir = "."; - int8_t rc = chdir(workdir.c_str()); + int8_t rc = chdir(workdir.c_str()); - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); - return rc; + return rc; } int setupResources() { #ifndef _MSC_VER - struct rlimit rlim; + struct rlimit rlim; - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } - rlim.rlim_cur = rlim.rlim_max = 65536; + rlim.rlim_cur = rlim.rlim_max = 65536; - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } - if (rlim.rlim_cur != 65536) - { - return -4; - } + if (rlim.rlim_cur != 65536) + { + return -4; + } #endif - return 0; + return 0; } #ifdef QSIZE_DEBUG class QszMonThd { -public: - QszMonThd(PrimitiveServer* psp, ofstream* qszlog) : fPsp(psp), fQszLog(qszlog) + public: + QszMonThd(PrimitiveServer* psp, ofstream* qszlog) : fPsp(psp), fQszLog(qszlog) + { + } + + ~QszMonThd() + { + } + + void operator()() + { + utils::setThreadName("QszMonThd"); + for (;;) { + uint32_t qd = fPsp->getProcessorThreadPool()->getWaiting(); + + if (fQszLog) + { + // Get a timestamp for output. + struct tm tm; + struct timeval tv; + + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); + + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) + << tm.tm_sec << '.' << setw(4) << tv.tv_usec / 100; + + *fQszLog << oss.str() << ' ' << qd << endl; + } + + struct timespec req = {0, 1000 * 100}; // 100 usec + + nanosleep(&req, 0); } + } - ~QszMonThd() - { - } - - void operator()() - { - utils::setThreadName("QszMonThd"); - for (;;) - { - uint32_t qd = fPsp->getProcessorThreadPool()->getWaiting(); - - if (fQszLog) - { - // Get a timestamp for output. - struct tm tm; - struct timeval tv; - - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); - - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec - << '.' - << setw(4) << tv.tv_usec / 100 - ; - - *fQszLog << oss.str() << ' ' << qd << endl; - } - - struct timespec req = { 0, 1000 * 100 }; //100 usec - - nanosleep(&req, 0); - } - } -private: - //defaults okay - //QszMonThd(const QszMonThd& rhs); - //QszMonThd& operator=(const QszMonThd& rhs); - const PrimitiveServer* fPsp; - ofstream* fQszLog; + private: + // defaults okay + // QszMonThd(const QszMonThd& rhs); + // QszMonThd& operator=(const QszMonThd& rhs); + const PrimitiveServer* fPsp; + ofstream* fQszLog; }; #endif @@ -305,268 +289,260 @@ private: #ifdef DUMP_CACHE_CONTENTS void* waitForSIGUSR1(void* p) { - utils::setThreadName("waitForSIGUSR1"); + utils::setThreadName("waitForSIGUSR1"); #if defined(__LP64__) || defined(_MSC_VER) - ptrdiff_t tmp = reinterpret_cast(p); - int cacheCount = static_cast(tmp); + ptrdiff_t tmp = reinterpret_cast(p); + int cacheCount = static_cast(tmp); #else - int cacheCount = reinterpret_cast(p); + int cacheCount = reinterpret_cast(p); #endif #ifndef _MSC_VER - sigset_t oset; - int rec_sig; - int32_t rpt_state = 0; - sigfillset(&oset); - sigdelset(&oset, SIGUSR1); - sigdelset(&oset, SIGUSR2); - pthread_sigmask(SIG_SETMASK, &oset, 0); + sigset_t oset; + int rec_sig; + int32_t rpt_state = 0; + sigfillset(&oset); + sigdelset(&oset, SIGUSR1); + sigdelset(&oset, SIGUSR2); + pthread_sigmask(SIG_SETMASK, &oset, 0); - sigemptyset(&oset); - sigaddset(&oset, SIGUSR1); - sigaddset(&oset, SIGUSR2); + sigemptyset(&oset); + sigaddset(&oset, SIGUSR1); + sigaddset(&oset, SIGUSR2); #endif - for (;;) + for (;;) + { + sigwait(&oset, &rec_sig); + + if (rec_sig == SIGUSR1) { - sigwait(&oset, &rec_sig); + ostringstream out; - if (rec_sig == SIGUSR1) - { - ostringstream out; - - for (int i = 0; i < cacheCount; i++) - { - BRPp[i]->formatLRUList(out); - cout << out.str() << "###" << endl; - } - } - else if (rec_sig == SIGUSR2) - { - // is reporting currently on? - rpt_state = BRPp[0]->ReportingFrequency(); - - if (rpt_state > 0) - rpt_state = 0; // turn reporting off - else - rpt_state = 1; // fbm will set to the value from config file - - for (int i = 0; i < cacheCount; i++) - BRPp[i]->setReportingFrequency(rpt_state); - - cout << "@@@" << endl; - } + for (int i = 0; i < cacheCount; i++) + { + BRPp[i]->formatLRUList(out); + cout << out.str() << "###" << endl; + } } + else if (rec_sig == SIGUSR2) + { + // is reporting currently on? + rpt_state = BRPp[0]->ReportingFrequency(); - return 0; + if (rpt_state > 0) + rpt_state = 0; // turn reporting off + else + rpt_state = 1; // fbm will set to the value from config file + + for (int i = 0; i < cacheCount; i++) + BRPp[i]->setReportingFrequency(rpt_state); + + cout << "@@@" << endl; + } + } + + return 0; } #endif -} - +} // namespace int ServicePrimProc::Child() { - Config* cf = Config::makeConfig(); + Config* cf = Config::makeConfig(); - setupSignalHandlers(); + setupSignalHandlers(); - int err = 0; - err = setupCwd(cf); + int err = 0; + err = setupCwd(cf); - mlp = new primitiveprocessor::Logger(); + mlp = new primitiveprocessor::Logger(); - if (!m_debug) - err = setupResources(); - string errMsg; + if (!m_debug) + err = setupResources(); + string errMsg; - switch (err) - { - case -1: - case -3: - errMsg = "Error getting file limits, please see non-root install documentation"; - break; + switch (err) + { + case -1: + case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; - case -2: - errMsg = "Error setting file limits, please see non-root install documentation"; - break; + case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; - case -4: - errMsg = "Could not install file limits to required value, please see non-root install documentation"; - break; + case -4: + errMsg = "Could not install file limits to required value, please see non-root install documentation"; + break; - case -5: - errMsg = "Could not change into working directory"; - break; + case -5: errMsg = "Could not change into working directory"; break; - default: - break; - } + default: break; + } - if (err < 0) - { - Oam oam; - mlp->logMessage(errMsg); - cerr << errMsg << endl; + if (err < 0) + { + Oam oam; + mlp->logMessage(errMsg); + cerr << errMsg << endl; - NotifyServiceInitializationFailed(); - return 2; - } + NotifyServiceInitializationFailed(); + return 2; + } - int serverThreads = 1; - int serverQueueSize = 10; - int processorWeight = 8 * 1024; - int processorQueueSize = 10 * 1024; - int64_t BRPBlocksPct = 70; - uint32_t BRPBlocks = 1887437; - int BRPThreads = 16; - int cacheCount = 1; - int maxBlocksPerRead = 256; // 1MB - bool rotatingDestination = false; - uint32_t deleteBlocks = 128; - bool PTTrace = false; - int temp; - string strTemp; - int priority = -1; - const string primitiveServers("PrimitiveServers"); - const string jobListStr("JobList"); - const string dbbc("DBBC"); - const string ExtentMapStr("ExtentMap"); - uint64_t extentRows = 8 * 1024 * 1024; - uint64_t MaxExtentSize = 0; - double prefetchThreshold; - uint64_t PMSmallSide = 67108864; - BPPCount = 16; - int numCores = -1; - int configNumCores = -1; - uint32_t highPriorityPercentage, medPriorityPercentage, lowPriorityPercentage; - utils::CGroupConfigurator cg; + int serverThreads = 1; + int serverQueueSize = 10; + int processorWeight = 8 * 1024; + int processorQueueSize = 10 * 1024; + int64_t BRPBlocksPct = 70; + uint32_t BRPBlocks = 1887437; + int BRPThreads = 16; + int cacheCount = 1; + int maxBlocksPerRead = 256; // 1MB + bool rotatingDestination = false; + uint32_t deleteBlocks = 128; + bool PTTrace = false; + int temp; + string strTemp; + int priority = -1; + const string primitiveServers("PrimitiveServers"); + const string jobListStr("JobList"); + const string dbbc("DBBC"); + const string ExtentMapStr("ExtentMap"); + uint64_t extentRows = 8 * 1024 * 1024; + uint64_t MaxExtentSize = 0; + double prefetchThreshold; + uint64_t PMSmallSide = 67108864; + BPPCount = 16; + int numCores = -1; + int configNumCores = -1; + uint32_t highPriorityPercentage, medPriorityPercentage, lowPriorityPercentage; + utils::CGroupConfigurator cg; - gDebugLevel = primitiveprocessor::NONE; + gDebugLevel = primitiveprocessor::NONE; - temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); + temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); - if (temp > 0) - serverThreads = temp; + if (temp > 0) + serverThreads = temp; - temp = toInt(cf->getConfig(primitiveServers, "ServerQueueSize")); + temp = toInt(cf->getConfig(primitiveServers, "ServerQueueSize")); - if (temp > 0) - serverQueueSize = temp; + if (temp > 0) + serverQueueSize = temp; - temp = toInt(cf->getConfig(primitiveServers, "ProcessorThreshold")); + temp = toInt(cf->getConfig(primitiveServers, "ProcessorThreshold")); - if (temp > 0) - processorWeight = temp; + if (temp > 0) + processorWeight = temp; - temp = toInt(cf->getConfig(primitiveServers, "ProcessorQueueSize")); + temp = toInt(cf->getConfig(primitiveServers, "ProcessorQueueSize")); - if (temp > 0) - processorQueueSize = temp; + if (temp > 0) + processorQueueSize = temp; - temp = toInt(cf->getConfig(primitiveServers, "DebugLevel")); + temp = toInt(cf->getConfig(primitiveServers, "DebugLevel")); - if (temp > 0) - gDebugLevel = (DebugLevel)temp; + if (temp > 0) + gDebugLevel = (DebugLevel)temp; - highPriorityPercentage = 0; - temp = toInt(cf->getConfig(primitiveServers, "HighPriorityPercentage")); + highPriorityPercentage = 0; + temp = toInt(cf->getConfig(primitiveServers, "HighPriorityPercentage")); - if (temp >= 0) - highPriorityPercentage = temp; + if (temp >= 0) + highPriorityPercentage = temp; - medPriorityPercentage = 0; - temp = toInt(cf->getConfig(primitiveServers, "MediumPriorityPercentage")); + medPriorityPercentage = 0; + temp = toInt(cf->getConfig(primitiveServers, "MediumPriorityPercentage")); - if (temp >= 0) - medPriorityPercentage = temp; + if (temp >= 0) + medPriorityPercentage = temp; - lowPriorityPercentage = 0; - temp = toInt(cf->getConfig(primitiveServers, "LowPriorityPercentage")); + lowPriorityPercentage = 0; + temp = toInt(cf->getConfig(primitiveServers, "LowPriorityPercentage")); - if (temp >= 0) - lowPriorityPercentage = temp; + if (temp >= 0) + lowPriorityPercentage = temp; - temp = toInt(cf->getConfig(ExtentMapStr, "ExtentRows")); + temp = toInt(cf->getConfig(ExtentMapStr, "ExtentRows")); - if (temp > 0) - extentRows = temp; + if (temp > 0) + extentRows = temp; - temp = toInt(cf->getConfig(primitiveServers, "ConnectionsPerPrimProc")); + temp = toInt(cf->getConfig(primitiveServers, "ConnectionsPerPrimProc")); - if (temp > 0) - connectionsPerUM = temp; - else - connectionsPerUM = 1; + if (temp > 0) + connectionsPerUM = temp; + else + connectionsPerUM = 1; - // set to smallest extent size - // do not allow to read beyond the end of an extent - const int MaxReadAheadSz = (extentRows) / BLOCK_SIZE; - // Max size for a primitive message column buffer. Must be big enough - // to accomodate 8192 values of the widest non-dict column + 8192 RIDs + ohead. - defaultBufferSize = 150 * 1024; + // set to smallest extent size + // do not allow to read beyond the end of an extent + const int MaxReadAheadSz = (extentRows) / BLOCK_SIZE; + // Max size for a primitive message column buffer. Must be big enough + // to accomodate 8192 values of the widest non-dict column + 8192 RIDs + ohead. + defaultBufferSize = 150 * 1024; + // This parm controls whether we rotate through the output sockets + // when deciding where to send response messages, or whether to simply + // send the response to the socket of origin. Should normally be set + // to 'y', for install types 1 and 3. + string strVal = cf->getConfig(primitiveServers, "RotatingDestination"); + // XXX: Permanently disable for now... + strVal = "N"; - // This parm controls whether we rotate through the output sockets - // when deciding where to send response messages, or whether to simply - // send the response to the socket of origin. Should normally be set - // to 'y', for install types 1 and 3. - string strVal = cf->getConfig(primitiveServers, "RotatingDestination"); - //XXX: Permanently disable for now... - strVal = "N"; + if ((strVal == "y") || (strVal == "Y")) + { + rotatingDestination = true; - if ((strVal == "y") || (strVal == "Y")) - { - rotatingDestination = true; + // Disable destination rotation if UM and PM are running on same + // server, because we could accidentally end up sending DMLProc + // responses to ExeMgr and vice versa, if we rotated socket dest. + temp = toInt(cf->getConfig("Installation", "ServerTypeInstall")); - // Disable destination rotation if UM and PM are running on same - // server, because we could accidentally end up sending DMLProc - // responses to ExeMgr and vice versa, if we rotated socket dest. - temp = toInt(cf->getConfig("Installation", "ServerTypeInstall")); + if ((temp == oam::INSTALL_COMBINE_DM_UM_PM) || (temp == oam::INSTALL_COMBINE_PM_UM)) + rotatingDestination = false; + } - if ((temp == oam::INSTALL_COMBINE_DM_UM_PM) || - (temp == oam::INSTALL_COMBINE_PM_UM)) - rotatingDestination = false; - } - - string strBlockPct = cf->getConfig(dbbc, "NumBlocksPct"); - temp = atoi(strBlockPct.c_str()); + string strBlockPct = cf->getConfig(dbbc, "NumBlocksPct"); + temp = atoi(strBlockPct.c_str()); #ifdef _MSC_VER - /* TODO: implement handling for the 'm' or 'g' chars in NumBlocksPct */ - if (temp > 0) - BRPBlocksPct = temp; + /* TODO: implement handling for the 'm' or 'g' chars in NumBlocksPct */ + if (temp > 0) + BRPBlocksPct = temp; - MEMORYSTATUSEX memStat; - memStat.dwLength = sizeof(memStat); + MEMORYSTATUSEX memStat; + memStat.dwLength = sizeof(memStat); - if (GlobalMemoryStatusEx(&memStat) == 0) - //FIXME: Assume 2GB? - BRPBlocks = 2621 * BRPBlocksPct; - else - { + if (GlobalMemoryStatusEx(&memStat) == 0) + // FIXME: Assume 2GB? + BRPBlocks = 2621 * BRPBlocksPct; + else + { #ifndef _WIN64 - memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); + memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); #endif - //We now have the total phys mem in bytes - BRPBlocks = memStat.ullTotalPhys / (8 * 1024) / 100 * BRPBlocksPct; - } + // We now have the total phys mem in bytes + BRPBlocks = memStat.ullTotalPhys / (8 * 1024) / 100 * BRPBlocksPct; + } #else - bool absCache = false; - if (temp > 0) { - BRPBlocksPct = temp; - /* MCOL-1847. Did the user specify this as an absolute? */ - int len = strBlockPct.length(); - if ((strBlockPct[len-1] >= 'a' && strBlockPct[len-1] <= 'z') || - (strBlockPct[len-1] >= 'A' && strBlockPct[len-1] <= 'Z')) { - absCache = true; - BRPBlocksPct = Config::fromText(strBlockPct); - } + bool absCache = false; + if (temp > 0) + { + BRPBlocksPct = temp; + /* MCOL-1847. Did the user specify this as an absolute? */ + int len = strBlockPct.length(); + if ((strBlockPct[len - 1] >= 'a' && strBlockPct[len - 1] <= 'z') || + (strBlockPct[len - 1] >= 'A' && strBlockPct[len - 1] <= 'Z')) + { + absCache = true; + BRPBlocksPct = Config::fromText(strBlockPct); } - if (absCache) - BRPBlocks = BRPBlocksPct / 8192; - else - BRPBlocks = ((BRPBlocksPct / 100.0) * (double) cg.getTotalMemory()) / 8192; + } + if (absCache) + BRPBlocks = BRPBlocksPct / 8192; + else + BRPBlocks = ((BRPBlocksPct / 100.0) * (double)cg.getTotalMemory()) / 8192; #endif #if 0 temp = toInt(cf->getConfig(dbbc, "NumThreads")); @@ -575,240 +551,240 @@ int ServicePrimProc::Child() BRPThreads = temp; #endif - temp = toInt(cf->getConfig(dbbc, "NumCaches")); + temp = toInt(cf->getConfig(dbbc, "NumCaches")); - if (temp > 0) - cacheCount = temp; + if (temp > 0) + cacheCount = temp; - temp = toInt(cf->getConfig(dbbc, "NumDeleteBlocks")); + temp = toInt(cf->getConfig(dbbc, "NumDeleteBlocks")); - if (temp > 0) - deleteBlocks = temp; + if (temp > 0) + deleteBlocks = temp; - if ((uint32_t)(.01 * BRPBlocks) < deleteBlocks) - deleteBlocks = (uint32_t)(.01 * BRPBlocks); + if ((uint32_t)(.01 * BRPBlocks) < deleteBlocks) + deleteBlocks = (uint32_t)(.01 * BRPBlocks); - temp = toInt(cf->getConfig(primitiveServers, "ColScanBufferSizeBlocks")); + temp = toInt(cf->getConfig(primitiveServers, "ColScanBufferSizeBlocks")); - if (temp > (int) MaxReadAheadSz || temp < 1) - maxBlocksPerRead = MaxReadAheadSz; - else if (temp > 0) - maxBlocksPerRead = temp; + if (temp > (int)MaxReadAheadSz || temp < 1) + maxBlocksPerRead = MaxReadAheadSz; + else if (temp > 0) + maxBlocksPerRead = temp; - temp = toInt(cf->getConfig(primitiveServers, "ColScanReadAheadBlocks")); + temp = toInt(cf->getConfig(primitiveServers, "ColScanReadAheadBlocks")); - if (temp > (int) MaxReadAheadSz || temp < 0) - blocksReadAhead = MaxReadAheadSz; - else if (temp > 0) - { - //make sure we've got an integral factor of extent size - for (; (MaxExtentSize % temp) != 0; ++temp); + if (temp > (int)MaxReadAheadSz || temp < 0) + blocksReadAhead = MaxReadAheadSz; + else if (temp > 0) + { + // make sure we've got an integral factor of extent size + for (; (MaxExtentSize % temp) != 0; ++temp) + ; - blocksReadAhead = temp; - } + blocksReadAhead = temp; + } - temp = toInt(cf->getConfig(primitiveServers, "PTTrace")); + temp = toInt(cf->getConfig(primitiveServers, "PTTrace")); - if (temp > 0) - PTTrace = true; + if (temp > 0) + PTTrace = true; - temp = toInt(cf->getConfig(primitiveServers, "PrefetchThreshold")); + temp = toInt(cf->getConfig(primitiveServers, "PrefetchThreshold")); - if (temp < 0 || temp > 100) - prefetchThreshold = 0; - else - prefetchThreshold = temp / 100.0; + if (temp < 0 || temp > 100) + prefetchThreshold = 0; + else + prefetchThreshold = temp / 100.0; - int maxPct = 0; //disable by default - temp = toInt(cf->getConfig(primitiveServers, "MaxPct")); + int maxPct = 0; // disable by default + temp = toInt(cf->getConfig(primitiveServers, "MaxPct")); - if (temp >= 0) - maxPct = temp; + if (temp >= 0) + maxPct = temp; - // @bug4507, configurable pm aggregation AggregationMemoryCheck - // We could use this same mechanism for other growing buffers. - int aggPct = 95; - temp = toInt(cf->getConfig("SystemConfig", "MemoryCheckPercent")); + // @bug4507, configurable pm aggregation AggregationMemoryCheck + // We could use this same mechanism for other growing buffers. + int aggPct = 95; + temp = toInt(cf->getConfig("SystemConfig", "MemoryCheckPercent")); - if (temp >= 0) - aggPct = temp; + if (temp >= 0) + aggPct = temp; - //...Start the thread to monitor our memory usage - new boost::thread(utils::MonitorProcMem(maxPct, aggPct, 28)); + //...Start the thread to monitor our memory usage + new boost::thread(utils::MonitorProcMem(maxPct, aggPct, 28)); - // config file priority is 40..1 (highest..lowest) - string sPriority = cf->getConfig(primitiveServers, "Priority"); + // config file priority is 40..1 (highest..lowest) + string sPriority = cf->getConfig(primitiveServers, "Priority"); - if (sPriority.length() > 0) - temp = toInt(sPriority); - else - temp = 21; + if (sPriority.length() > 0) + temp = toInt(sPriority); + else + temp = 21; - // convert config file value to setpriority(2) value (-20..19, -1 is the default) - if (temp > 0) - priority = 20 - temp; - else if (temp < 0) - priority = 19; + // convert config file value to setpriority(2) value (-20..19, -1 is the default) + if (temp > 0) + priority = 20 - temp; + else if (temp < 0) + priority = 19; - if (priority < -20) priority = -20; + if (priority < -20) + priority = -20; #ifdef _MSC_VER - //FIXME: + // FIXME: #else - setpriority(PRIO_PROCESS, 0, priority); + setpriority(PRIO_PROCESS, 0, priority); #endif - //..Instantiate UmSocketSelector singleton. Disable rotating destination - //..selection if no UM IP addresses are in the Calpo67108864LLnt.xml file. - UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance(); + //..Instantiate UmSocketSelector singleton. Disable rotating destination + //..selection if no UM IP addresses are in the Calpo67108864LLnt.xml file. + UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance(); - if (rotatingDestination) - { - if (pUmSocketSelector->ipAddressCount() < 1) - rotatingDestination = false; - } + if (rotatingDestination) + { + if (pUmSocketSelector->ipAddressCount() < 1) + rotatingDestination = false; + } - //See if we want to override the calculated #cores - temp = toInt(cf->getConfig(primitiveServers, "NumCores")); + // See if we want to override the calculated #cores + temp = toInt(cf->getConfig(primitiveServers, "NumCores")); - if (temp > 0) - configNumCores = temp; + if (temp > 0) + configNumCores = temp; - if (configNumCores <= 0) - { - //count the actual #cores + if (configNumCores <= 0) + { + // count the actual #cores - numCores = cg.getNumCores(); + numCores = cg.getNumCores(); - if (numCores == 0) - numCores = 8; - } - else - numCores = configNumCores; + if (numCores == 0) + numCores = 8; + } + else + numCores = configNumCores; - //based on the #cores, calculate some thread parms - if (numCores > 0) - { - BRPThreads = 2 * numCores; - //there doesn't seem much benefit to having more than this, and sometimes it causes problems. - //DBBC.NumThreads can override this cap - BRPThreads = std::min(BRPThreads, 32); - } + // based on the #cores, calculate some thread parms + if (numCores > 0) + { + BRPThreads = 2 * numCores; + // there doesn't seem much benefit to having more than this, and sometimes it causes problems. + // DBBC.NumThreads can override this cap + BRPThreads = std::min(BRPThreads, 32); + } - // the default is ~10% low, 30% medium, 60% high, (where 2*cores = 100%) - if (highPriorityPercentage == 0 && medPriorityPercentage == 0 && - lowPriorityPercentage == 0) - { - lowPriorityThreads = max(1, (2 * numCores) / 10); - medPriorityThreads = max(1, (2 * numCores) / 3); - highPriorityThreads = (2 * numCores) - lowPriorityThreads - medPriorityThreads; - } - else - { - uint32_t totalThreads = (uint32_t) ((lowPriorityPercentage + medPriorityPercentage + - highPriorityPercentage) / 100.0 * (2 * numCores)); + // the default is ~10% low, 30% medium, 60% high, (where 2*cores = 100%) + if (highPriorityPercentage == 0 && medPriorityPercentage == 0 && lowPriorityPercentage == 0) + { + lowPriorityThreads = max(1, (2 * numCores) / 10); + medPriorityThreads = max(1, (2 * numCores) / 3); + highPriorityThreads = (2 * numCores) - lowPriorityThreads - medPriorityThreads; + } + else + { + uint32_t totalThreads = + (uint32_t)((lowPriorityPercentage + medPriorityPercentage + highPriorityPercentage) / 100.0 * + (2 * numCores)); - if (totalThreads == 0) - totalThreads = 1; + if (totalThreads == 0) + totalThreads = 1; - lowPriorityThreads = (uint32_t) (lowPriorityPercentage / 100.0 * (2 * numCores)); - medPriorityThreads = (uint32_t) (medPriorityPercentage / 100.0 * (2 * numCores)); - highPriorityThreads = totalThreads - lowPriorityThreads - medPriorityThreads; - } + lowPriorityThreads = (uint32_t)(lowPriorityPercentage / 100.0 * (2 * numCores)); + medPriorityThreads = (uint32_t)(medPriorityPercentage / 100.0 * (2 * numCores)); + highPriorityThreads = totalThreads - lowPriorityThreads - medPriorityThreads; + } - BPPCount = highPriorityThreads + medPriorityThreads + lowPriorityThreads; - - // let the user override if they want - temp = toInt(cf->getConfig(primitiveServers, "BPPCount")); + BPPCount = highPriorityThreads + medPriorityThreads + lowPriorityThreads; - if (temp > 0 && temp < (int) BPPCount) - BPPCount = temp; + // let the user override if they want + temp = toInt(cf->getConfig(primitiveServers, "BPPCount")); - temp = toInt(cf->getConfig(dbbc, "NumThreads")); + if (temp > 0 && temp < (int)BPPCount) + BPPCount = temp; - if (temp > 0) - BRPThreads = temp; + temp = toInt(cf->getConfig(dbbc, "NumThreads")); + + if (temp > 0) + BRPThreads = temp; #ifndef _MSC_VER - // @bug4598, switch for O_DIRECT to support gluster fs. - // directIOFlag == O_DIRECT, by default - strVal = cf->getConfig(primitiveServers, "DirectIO"); + // @bug4598, switch for O_DIRECT to support gluster fs. + // directIOFlag == O_DIRECT, by default + strVal = cf->getConfig(primitiveServers, "DirectIO"); - if ((strVal == "n") || (strVal == "N")) - directIOFlag = 0; + if ((strVal == "n") || (strVal == "N")) + directIOFlag = 0; #endif - IDBPolicy::configIDBPolicy(); + IDBPolicy::configIDBPolicy(); - // no versionbuffer if using HDFS for performance reason - if (IDBPolicy::useHdfs()) - noVB = 1; + // no versionbuffer if using HDFS for performance reason + if (IDBPolicy::useHdfs()) + noVB = 1; - cout << "Starting PrimitiveServer: st = " << serverThreads << ", sq = " << serverQueueSize << - ", pw = " << processorWeight << ", pq = " << processorQueueSize << - ", nb = " << BRPBlocks << ", nt = " << BRPThreads << ", nc = " << cacheCount << - ", ra = " << blocksReadAhead << ", db = " << deleteBlocks << ", mb = " << maxBlocksPerRead << - ", rd = " << rotatingDestination << ", tr = " << PTTrace << - ", ss = " << PMSmallSide << ", bp = " << BPPCount << endl; + cout << "Starting PrimitiveServer: st = " << serverThreads << ", sq = " << serverQueueSize + << ", pw = " << processorWeight << ", pq = " << processorQueueSize << ", nb = " << BRPBlocks + << ", nt = " << BRPThreads << ", nc = " << cacheCount << ", ra = " << blocksReadAhead + << ", db = " << deleteBlocks << ", mb = " << maxBlocksPerRead << ", rd = " << rotatingDestination + << ", tr = " << PTTrace << ", ss = " << PMSmallSide << ", bp = " << BPPCount << endl; - PrimitiveServer server(serverThreads, serverQueueSize, processorWeight, processorQueueSize, - rotatingDestination, BRPBlocks, BRPThreads, cacheCount, maxBlocksPerRead, blocksReadAhead, - deleteBlocks, PTTrace, prefetchThreshold, PMSmallSide); + PrimitiveServer server(serverThreads, serverQueueSize, processorWeight, processorQueueSize, + rotatingDestination, BRPBlocks, BRPThreads, cacheCount, maxBlocksPerRead, + blocksReadAhead, deleteBlocks, PTTrace, prefetchThreshold, PMSmallSide); #ifdef QSIZE_DEBUG - thread* qszMonThd; + thread* qszMonThd; - if (gDebugLevel >= STATS) - { + if (gDebugLevel >= STATS) + { #ifdef _MSC_VER - ofstream* qszLog = new ofstream("C:/Calpont/log/trace/ppqsz.dat"); + ofstream* qszLog = new ofstream("C:/Calpont/log/trace/ppqsz.dat"); #else - ofstream* qszLog = new ofstream("/var/log/mariadb/columnstore/trace/ppqsz.dat"); + ofstream* qszLog = new ofstream("/var/log/mariadb/columnstore/trace/ppqsz.dat"); #endif - if (!qszLog->good()) - { - qszLog->close(); - delete qszLog; - qszLog = 0; - } - - qszMonThd = new thread(QszMonThd(&server, qszLog)); + if (!qszLog->good()) + { + qszLog->close(); + delete qszLog; + qszLog = 0; } + qszMonThd = new thread(QszMonThd(&server, qszLog)); + } + #endif #ifdef DUMP_CACHE_CONTENTS - { - //Need to use pthreads API here... - pthread_t thd1; - pthread_attr_t attr1; - pthread_attr_init(&attr1); - pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); - pthread_create(&thd1, &attr1, waitForSIGUSR1, reinterpret_cast(cacheCount)); - } + { + // Need to use pthreads API here... + pthread_t thd1; + pthread_attr_t attr1; + pthread_attr_init(&attr1); + pthread_attr_setdetachstate(&attr1, PTHREAD_CREATE_DETACHED); + pthread_create(&thd1, &attr1, waitForSIGUSR1, reinterpret_cast(cacheCount)); + } #endif - server.start(this); + server.start(this); - cerr << "server.start() exited!" << endl; + cerr << "server.start() exited!" << endl; - return 1; + return 1; } - int main(int argc, char** argv) { - Opt opt(argc, argv); + Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - // This is unset due to the way we start it - program_invocation_short_name = const_cast("PrimProc"); - // Initialize the charset library - MY_INIT(argv[0]); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("PrimProc"); + // Initialize the charset library + MY_INIT(argv[0]); - return ServicePrimProc(opt).Run(); + return ServicePrimProc(opt).Run(); } // vim:ts=4 sw=4: diff --git a/primitives/primproc/primproc.h b/primitives/primproc/primproc.h index db003cd73..05a94e9bb 100644 --- a/primitives/primproc/primproc.h +++ b/primitives/primproc/primproc.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* -* $Id: primproc.h 2035 2013-01-21 14:12:19Z rdempsey $ -* -******************************************************************************************/ + * + * $Id: primproc.h 2035 2013-01-21 14:12:19Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -39,80 +39,79 @@ namespace primitiveprocessor { +#define SUMMARY_INFO(message) \ + if (isDebug(SUMMARY)) \ + { \ + std::cout << message << std::endl; \ + } -#define SUMMARY_INFO( message ) \ - if ( isDebug(SUMMARY) ) \ - { \ - std::cout << message << std::endl; \ - } +#define SUMMARY_INFO2(message1, message2) \ + if (isDebug(SUMMARY)) \ + { \ + std::cout << message1 << message2 << std::endl; \ + } -#define SUMMARY_INFO2( message1, message2 ) \ - if ( isDebug(SUMMARY) ) \ - { \ - std::cout << message1 << message2 << std::endl; \ - } +#define SUMMARY_INFO3(message1, message2, message3) \ + if (isDebug(SUMMARY)) \ + { \ + std::cout << message1 << message2 << message3 << std::endl; \ + } -#define SUMMARY_INFO3( message1, message2, message3 ) \ - if ( isDebug(SUMMARY) ) \ - { \ - std::cout << message1 << message2 << message3 << std::endl; \ - } +#define DETAIL_INFO(message) \ + if (isDebug(DETAIL)) \ + { \ + std::cout << message << std::endl; \ + } -#define DETAIL_INFO( message ) \ - if ( isDebug(DETAIL) ) \ - { \ - std::cout << message << std::endl; \ - } +#define DETAIL_INFO2(message1, message2) \ + if (isDebug(DETAIL)) \ + { \ + std::cout << message1 << message2 << std::endl; \ + } -#define DETAIL_INFO2( message1, message2 ) \ - if ( isDebug(DETAIL) ) \ - { \ - std::cout << message1 << message2 << std::endl; \ - } +#define DETAIL_INFO3(message1, message2, message3) \ + if (isDebug(DETAIL)) \ + { \ + std::cout << message1 << message2 << message3 << std::endl; \ + } -#define DETAIL_INFO3( message1, message2, message3 ) \ - if ( isDebug(DETAIL) ) \ - { \ - std::cout << message1 << message2 << message3 << std::endl; \ - } +#define VERBOSE_INFO(message) \ + if (isDebug(VERBOSE)) \ + { \ + std::cout << message << std::endl; \ + } -#define VERBOSE_INFO( message ) \ - if ( isDebug(VERBOSE) ) \ - { \ - std::cout << message << std::endl; \ - } +#define VERBOSE_INFO2(message1, message2) \ + if (isDebug(VERBOSE)) \ + { \ + std::cout << message1 << message2 << std::endl; \ + } -#define VERBOSE_INFO2( message1, message2 ) \ - if ( isDebug(VERBOSE) ) \ - { \ - std::cout << message1 << message2 << std::endl; \ - } +#define VERBOSE_INFO3(message1, message2, message3) \ + if (isDebug(VERBOSE)) \ + { \ + std::cout << message1 << message2 << message3 << std::endl; \ + } -#define VERBOSE_INFO3( message1, message2, message3 ) \ - if ( isDebug(VERBOSE) ) \ - { \ - std::cout << message1 << message2 << message3 << std::endl; \ - } - -enum DebugLevel /** @brief Debug level type enumeration */ +enum DebugLevel /** @brief Debug level type enumeration */ { - NONE = 0, /** @brief No debug info */ - STATS = 1, /** @brief stats info */ - SUMMARY = 2, /** @brief Summary level debug info */ - DETAIL = 3, /** @brief A little detail debug info */ - VERBOSE = 4, /** @brief Detailed debug info */ + NONE = 0, /** @brief No debug info */ + STATS = 1, /** @brief stats info */ + SUMMARY = 2, /** @brief Summary level debug info */ + DETAIL = 3, /** @brief A little detail debug info */ + VERBOSE = 4, /** @brief Detailed debug info */ }; -bool isDebug( const DebugLevel level ); +bool isDebug(const DebugLevel level); const int MAX_BUFFER_SIZE = 32768 * 2; // message log globals -//const logging::LoggingID lid1(28); -//extern logging::Message msg16; -//extern logging::MessageLog ml1; -//extern boost::mutex logLock; +// const logging::LoggingID lid1(28); +// extern logging::Message msg16; +// extern logging::MessageLog ml1; +// extern boost::mutex logLock; extern Logger* mlp; -} +} // namespace primitiveprocessor -#endif //PRIMPROC_H +#endif // PRIMPROC_H diff --git a/primitives/primproc/pseudocc.cpp b/primitives/primproc/pseudocc.cpp index 588723452..91751c169 100644 --- a/primitives/primproc/pseudocc.cpp +++ b/primitives/primproc/pseudocc.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "bpp.h" #include "pseudocolumn.h" @@ -24,7 +23,6 @@ using namespace execplan; namespace primitiveprocessor { - PseudoCC::PseudoCC() : ColumnCommand() { } @@ -35,43 +33,43 @@ PseudoCC::~PseudoCC() SCommand PseudoCC::duplicate() { - SCommand ret; - PseudoCC* pseudo; + SCommand ret; + PseudoCC* pseudo; - pseudo = new PseudoCC(); - ret.reset(pseudo); - pseudo->function = function; - pseudo->valueFromUM = valueFromUM; - pseudo->bigValueFromUM = bigValueFromUM; - ColumnCommand::duplicate(pseudo); - return ret; + pseudo = new PseudoCC(); + ret.reset(pseudo); + pseudo->function = function; + pseudo->valueFromUM = valueFromUM; + pseudo->bigValueFromUM = bigValueFromUM; + ColumnCommand::duplicate(pseudo); + return ret; } void PseudoCC::createCommand(messageqcpp::ByteStream& bs) { - bs.advance(1); - bs >> function; - ColumnCommand::createCommand(bs); + bs.advance(1); + bs >> function; + ColumnCommand::createCommand(bs); } void PseudoCC::resetCommand(messageqcpp::ByteStream& bs) { - if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN) - { - if (!colType.isWideDecimalType()) - bs >> valueFromUM; - else - bs >> bigValueFromUM; - } - else if (function == PSEUDO_EXTENTID) - { - bs >> valueFromUM; + if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN) + { + if (!colType.isWideDecimalType()) + bs >> valueFromUM; + else + bs >> bigValueFromUM; + } + else if (function == PSEUDO_EXTENTID) + { + bs >> valueFromUM; - if (colType.isWideDecimalType()) - bigValueFromUM = valueFromUM; - } + if (colType.isWideDecimalType()) + bigValueFromUM = valueFromUM; + } - ColumnCommand::resetCommand(bs); + ColumnCommand::resetCommand(bs); } #ifdef __GNUC__ @@ -84,225 +82,139 @@ __attribute__((optimize("no-tree-vectorize"))) void PseudoCC::loadData() { - switch (function) - { - case PSEUDO_PM: - switch (colType.colWidth) - { - case 1: - loadPMNumber(); - break; + switch (function) + { + case PSEUDO_PM: + switch (colType.colWidth) + { + case 1: loadPMNumber(); break; - case 2: - loadPMNumber(); - break; + case 2: loadPMNumber(); break; - case 4: - loadPMNumber(); - break; + case 4: loadPMNumber(); break; - case 8: - loadPMNumber(); - break; + case 8: loadPMNumber(); break; - case 16: - loadPMNumber(); - break; + case 16: loadPMNumber(); break; - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + default: cout << "PC::loadData(): bad column width" << endl; break; + } - break; + break; - case PSEUDO_EXTENTRELATIVERID: - switch (colType.colWidth) - { - case 1: - loadRIDs(); - break; + case PSEUDO_EXTENTRELATIVERID: + switch (colType.colWidth) + { + case 1: loadRIDs(); break; - case 2: - loadRIDs(); - break; + case 2: loadRIDs(); break; - case 4: - loadRIDs(); - break; + case 4: loadRIDs(); break; - case 8: - loadRIDs(); - break; + case 8: loadRIDs(); break; - case 16: - loadRIDs(); - break; + case 16: loadRIDs(); break; - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + default: cout << "PC::loadData(): bad column width" << endl; break; + } - break; + break; - case PSEUDO_SEGMENT: - switch (colType.colWidth) - { - case 1: - loadSegmentNum(); - break; + case PSEUDO_SEGMENT: + switch (colType.colWidth) + { + case 1: loadSegmentNum(); break; - case 2: - loadSegmentNum(); - break; + case 2: loadSegmentNum(); break; - case 4: - loadSegmentNum(); - break; + case 4: loadSegmentNum(); break; - case 8: - loadSegmentNum(); - break; + case 8: loadSegmentNum(); break; - case 16: - loadSegmentNum(); - break; + case 16: loadSegmentNum(); break; - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + default: cout << "PC::loadData(): bad column width" << endl; break; + } - break; + break; - case PSEUDO_SEGMENTDIR: - switch (colType.colWidth) - { - case 1: - loadPartitionNum(); - break; + case PSEUDO_SEGMENTDIR: + switch (colType.colWidth) + { + case 1: loadPartitionNum(); break; - case 2: - loadPartitionNum(); - break; + case 2: loadPartitionNum(); break; - case 4: - loadPartitionNum(); - break; + case 4: loadPartitionNum(); break; - case 8: - loadPartitionNum(); - break; + case 8: loadPartitionNum(); break; - case 16: - loadPartitionNum(); - break; + case 16: loadPartitionNum(); break; - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + default: cout << "PC::loadData(): bad column width" << endl; break; + } - break; + break; - case PSEUDO_BLOCKID: - switch (colType.colWidth) - { - case 1: - loadLBID(); - break; + case PSEUDO_BLOCKID: + switch (colType.colWidth) + { + case 1: loadLBID(); break; - case 2: - loadLBID(); - break; + case 2: loadLBID(); break; - case 4: - loadLBID(); - break; + case 4: loadLBID(); break; - case 8: - loadLBID(); - break; + case 8: loadLBID(); break; - case 16: - loadLBID(); - break; + case 16: loadLBID(); break; - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + default: cout << "PC::loadData(): bad column width" << endl; break; + } - break; + break; - case PSEUDO_DBROOT: - switch (colType.colWidth) - { - case 1: - loadDBRootNum(); - break; + case PSEUDO_DBROOT: + switch (colType.colWidth) + { + case 1: loadDBRootNum(); break; - case 2: - loadDBRootNum(); - break; + case 2: loadDBRootNum(); break; - case 4: - loadDBRootNum(); - break; + case 4: loadDBRootNum(); break; - case 8: - loadDBRootNum(); - break; + case 8: loadDBRootNum(); break; - case 16: - loadDBRootNum(); - break; - - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + case 16: loadDBRootNum(); break; - break; + default: cout << "PC::loadData(): bad column width" << endl; break; + } - /* cases where the value to use is sent from the UM */ - case PSEUDO_EXTENTMAX: - case PSEUDO_EXTENTMIN: - case PSEUDO_EXTENTID: - switch (colType.colWidth) - { - case 1: - loadSingleValue(valueFromUM); - break; + break; - case 2: - loadSingleValue(valueFromUM); - break; + /* cases where the value to use is sent from the UM */ + case PSEUDO_EXTENTMAX: + case PSEUDO_EXTENTMIN: + case PSEUDO_EXTENTID: + switch (colType.colWidth) + { + case 1: loadSingleValue(valueFromUM); break; - case 4: - loadSingleValue(valueFromUM); - break; + case 2: loadSingleValue(valueFromUM); break; - case 8: - loadSingleValue(valueFromUM); - break; + case 4: loadSingleValue(valueFromUM); break; - case 16: - loadSingleValue(bigValueFromUM); - break; - - default: - cout << "PC::loadData(): bad column width" << endl; - break; - } + case 8: loadSingleValue(valueFromUM); break; - break; + case 16: loadSingleValue(bigValueFromUM); break; - default: - cout << "PC::loadData(): bad function" << endl; - break; - } + default: cout << "PC::loadData(): bad column width" << endl; break; + } + + break; + + default: cout << "PC::loadData(): bad function" << endl; break; + } } -} +} // namespace primitiveprocessor diff --git a/primitives/primproc/pseudocc.h b/primitives/primproc/pseudocc.h index d59cc3360..6f6a9f8e9 100644 --- a/primitives/primproc/pseudocc.h +++ b/primitives/primproc/pseudocc.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef PSEUDOCC_H #define PSEUDOCC_H @@ -24,35 +23,40 @@ namespace primitiveprocessor { - class PseudoCC : public ColumnCommand { -public: - PseudoCC(); - virtual ~PseudoCC(); + public: + PseudoCC(); + virtual ~PseudoCC(); - virtual SCommand duplicate(); - virtual void createCommand(messageqcpp::ByteStream&); - virtual void resetCommand(messageqcpp::ByteStream&); -protected: - virtual void loadData(); + virtual SCommand duplicate(); + virtual void createCommand(messageqcpp::ByteStream&); + virtual void resetCommand(messageqcpp::ByteStream&); -private: + protected: + virtual void loadData(); - template void loadSingleValue(W val); - template void loadPMNumber(); - template void loadRIDs(); - template void loadSegmentNum(); - template void loadDBRootNum(); - template void loadPartitionNum(); - template void loadLBID(); + private: + template + void loadSingleValue(W val); + template + void loadPMNumber(); + template + void loadRIDs(); + template + void loadSegmentNum(); + template + void loadDBRootNum(); + template + void loadPartitionNum(); + template + void loadLBID(); - uint32_t function; - uint64_t valueFromUM; - uint128_t bigValueFromUM; + uint32_t function; + uint64_t valueFromUM; + uint128_t bigValueFromUM; }; - template #ifdef __GNUC__ // The "official" GCC version for InfiniDB is 4.5.3. This flag breaks at least 4.4.3 & 4.4.5, @@ -63,73 +67,73 @@ __attribute__((optimize("no-tree-vectorize"))) #endif void PseudoCC::loadSingleValue(W val) { - W* bData = (W*) bpp->blockData; + W* bData = (W*)bpp->blockData; - for (uint32_t i = 0; i < BLOCK_SIZE; i++) - bData[i] = val; // this breaks when using tree-vectorization + for (uint32_t i = 0; i < BLOCK_SIZE; i++) + bData[i] = val; // this breaks when using tree-vectorization - //memcpy(&bData[i], &val, sizeof(W)); // this always works + // memcpy(&bData[i], &val, sizeof(W)); // this always works } -template +template void PseudoCC::loadPMNumber() { - uint32_t pmNum = oamCache->getLocalPMId(); - loadSingleValue(pmNum); + uint32_t pmNum = oamCache->getLocalPMId(); + loadSingleValue(pmNum); } /* This returns extent-relative rids */ -template +template void PseudoCC::loadRIDs() { - uint32_t i; - uint64_t baseRid = rowgroup::getExtentRelativeRid(bpp->baseRid); - W* bData = (W*) bpp->blockData; + uint32_t i; + uint64_t baseRid = rowgroup::getExtentRelativeRid(bpp->baseRid); + W* bData = (W*)bpp->blockData; - for (i = 0; i < BLOCK_SIZE; i++) - { - //W val = baseRid + i; - //memcpy(&bData[i], &val, sizeof(W)); - bData[i] = baseRid + i; // breaks with tree-vectorization - } + for (i = 0; i < BLOCK_SIZE; i++) + { + // W val = baseRid + i; + // memcpy(&bData[i], &val, sizeof(W)); + bData[i] = baseRid + i; // breaks with tree-vectorization + } } -template +template void PseudoCC::loadSegmentNum() { - uint16_t segNum; - rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL); - loadSingleValue(segNum); + uint16_t segNum; + rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL); + loadSingleValue(segNum); } -template +template void PseudoCC::loadPartitionNum() { - uint32_t partNum; - rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL); - loadSingleValue(partNum); + uint32_t partNum; + rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL); + loadSingleValue(partNum); } -template +template void PseudoCC::loadDBRootNum() { - loadSingleValue(bpp->dbRoot); + loadSingleValue(bpp->dbRoot); } -template +template void PseudoCC::loadLBID() { - uint32_t i; - W* bData = (W*) bpp->blockData; + uint32_t i; + W* bData = (W*)bpp->blockData; - for (i = 0; i < BLOCK_SIZE; i++) - { - //W val = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; - //memcpy(&bData[i], &val, sizeof(W)); - bData[i] = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; // breaks with tree-vectorization - } + for (i = 0; i < BLOCK_SIZE; i++) + { + // W val = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; + // memcpy(&bData[i], &val, sizeof(W)); + bData[i] = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; // breaks with tree-vectorization + } } -} +} // namespace primitiveprocessor -#endif // PseudoCC_H +#endif // PseudoCC_H diff --git a/primitives/primproc/resource.h b/primitives/primproc/resource.h index 38e2b96fd..0bbe73c5c 100644 --- a/primitives/primproc/resource.h +++ b/primitives/primproc/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/primitives/primproc/rtscommand.cpp b/primitives/primproc/rtscommand.cpp index c4d310539..2cbdb8b7b 100644 --- a/primitives/primproc/rtscommand.cpp +++ b/primitives/primproc/rtscommand.cpp @@ -40,7 +40,6 @@ using namespace logging; namespace primitiveprocessor { - RTSCommand::RTSCommand() : Command(RID_TO_STRING), absNull(false) { } @@ -51,202 +50,203 @@ RTSCommand::~RTSCommand() void RTSCommand::execute() { - throw logic_error("RTSCommand shouldn't be used for filter steps"); + throw logic_error("RTSCommand shouldn't be used for filter steps"); } void RTSCommand::project() { - uint32_t i; + uint32_t i; - if (passThru) + if (passThru) + { + // if (bpp->absRids.get() == NULL) + if (absNull) //@bug 1003. Always need to remake absRids { -// if (bpp->absRids.get() == NULL) - if (absNull) //@bug 1003. Always need to remake absRids - { - bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); - for (i = 0; i < bpp->ridCount; i++) - bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid; - } - - // need something in values - - dict.project(); + for (i = 0; i < bpp->ridCount; i++) + bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid; } - else + + // need something in values + + dict.project(); + } + else + { + int64_t tmpValues[LOGICAL_BLOCK_RIDS]; + uint32_t old_rc = bpp->ridCount; + + if (bpp->absRids.get() == NULL) + bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + + col->execute(tmpValues); + + if (old_rc != bpp->ridCount) { - int64_t tmpValues[LOGICAL_BLOCK_RIDS]; - uint32_t old_rc = bpp->ridCount; + ostringstream os; - if (bpp->absRids.get() == NULL) - bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + os << __FILE__ << " (token column) error on projection for oid " << col->getOID() << " lbid " + << col->getLBID(); + os << ": input rids " << old_rc; + os << ", output rids " << bpp->ridCount << endl; - col->execute(tmpValues); - - if (old_rc != bpp->ridCount) - { - ostringstream os; - - os << __FILE__ << " (token column) error on projection for oid " << col->getOID() << " lbid " << col->getLBID(); - os << ": input rids " << old_rc; - os << ", output rids " << bpp->ridCount << endl; - - if (bpp->sessionID & 0x80000000) - throw NeedToRestartJob(os.str()); - else - { - throw PrimitiveColumnProjectResultExcept(os.str()); - } - } - - dict.project(tmpValues); + if (bpp->sessionID & 0x80000000) + throw NeedToRestartJob(os.str()); + else + { + throw PrimitiveColumnProjectResultExcept(os.str()); + } } + + dict.project(tmpValues); + } } void RTSCommand::projectIntoRowGroup(RowGroup& rg, uint32_t colNum) { - uint32_t i; + uint32_t i; - if (passThru) + if (passThru) + { + if (absNull) //@bug 1003. Always need to remake absRids { - if (absNull) //@bug 1003. Always need to remake absRids - { - bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); - for (i = 0; i < bpp->ridCount; i++) - bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid; - } - - dict.projectIntoRowGroup(rg, colNum); + for (i = 0; i < bpp->ridCount; i++) + bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid; } - else + + dict.projectIntoRowGroup(rg, colNum); + } + else + { + int64_t tmpValues[LOGICAL_BLOCK_RIDS]; + uint32_t old_rc = bpp->ridCount; + + if (bpp->absRids.get() == NULL) + bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + + col->execute(tmpValues); + + if (old_rc != bpp->ridCount) { - int64_t tmpValues[LOGICAL_BLOCK_RIDS]; - uint32_t old_rc = bpp->ridCount; + ostringstream os; - if (bpp->absRids.get() == NULL) - bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]); + os << __FILE__ << " (token column) error on projection for oid " << col->getOID() << " lbid " + << col->getLBID(); + os << ": input rids " << old_rc; + os << ", output rids " << bpp->ridCount << endl; - col->execute(tmpValues); - - if (old_rc != bpp->ridCount) - { - - ostringstream os; - - os << __FILE__ << " (token column) error on projection for oid " << col->getOID() << " lbid " << col->getLBID(); - os << ": input rids " << old_rc; - os << ", output rids " << bpp->ridCount << endl; - - if (bpp->sessionID & 0x80000000) - throw NeedToRestartJob(os.str()); - else - throw PrimitiveColumnProjectResultExcept(os.str()); - } - - dict.projectIntoRowGroup(rg, tmpValues, colNum); + if (bpp->sessionID & 0x80000000) + throw NeedToRestartJob(os.str()); + else + throw PrimitiveColumnProjectResultExcept(os.str()); } + + dict.projectIntoRowGroup(rg, tmpValues, colNum); + } } uint64_t RTSCommand::getLBID() { - if (!passThru) - return col->getLBID(); - else - return 0; + if (!passThru) + return col->getLBID(); + else + return 0; } void RTSCommand::nextLBID() { - if (!passThru) - col->nextLBID(); + if (!passThru) + col->nextLBID(); } void RTSCommand::prep(int8_t outputType, bool makeAbsRids) { - if (!passThru) - col->prep(OT_BOTH, true); + if (!passThru) + col->prep(OT_BOTH, true); - dict.prep(OT_DATAVALUE, true); + dict.prep(OT_DATAVALUE, true); } void RTSCommand::createCommand(ByteStream& bs) { - bs.advance(1); - bs >> passThru; + bs.advance(1); + bs >> passThru; - if (!passThru) - { - col.reset(ColumnCommandFabric::createCommand(bs)); - } + if (!passThru) + { + col.reset(ColumnCommandFabric::createCommand(bs)); + } - dict.createCommand(bs); - Command::createCommand(bs); + dict.createCommand(bs); + Command::createCommand(bs); } void RTSCommand::resetCommand(ByteStream& bs) { - if (!passThru) - col->resetCommand(bs); + if (!passThru) + col->resetCommand(bs); - dict.resetCommand(bs); + dict.resetCommand(bs); } SCommand RTSCommand::duplicate() { - SCommand ret; - RTSCommand* rts; + SCommand ret; + RTSCommand* rts; - ret.reset(new RTSCommand()); - rts = (RTSCommand*) ret.get(); - rts->passThru = passThru; + ret.reset(new RTSCommand()); + rts = (RTSCommand*)ret.get(); + rts->passThru = passThru; - if (!passThru) - rts->col.reset(ColumnCommandFabric::duplicate(col)); + if (!passThru) + rts->col.reset(ColumnCommandFabric::duplicate(col)); - rts->dict = dict; - rts->Command::operator=(*this); - return ret; + rts->dict = dict; + rts->Command::operator=(*this); + return ret; } bool RTSCommand::operator==(const RTSCommand& r) const { - if (passThru != r.passThru) - return false; + if (passThru != r.passThru) + return false; - if (!passThru) - if (col != r.col) - return false; + if (!passThru) + if (col != r.col) + return false; - if (absNull != r.absNull) - return false; + if (absNull != r.absNull) + return false; - if (dict != dict) - return false; + if (dict != dict) + return false; - return true; + return true; } bool RTSCommand::operator!=(const RTSCommand& r) const { - return !(*this == r); + return !(*this == r); } void RTSCommand::getLBIDList(uint32_t loopCount, vector* lbids) { - dict.getLBIDList(loopCount, lbids); + dict.getLBIDList(loopCount, lbids); - if (!passThru) - col->getLBIDList(loopCount, lbids); + if (!passThru) + col->getLBIDList(loopCount, lbids); } void RTSCommand::setBatchPrimitiveProcessor(BatchPrimitiveProcessor* b) { - Command::setBatchPrimitiveProcessor(b); - dict.setBatchPrimitiveProcessor(b); + Command::setBatchPrimitiveProcessor(b); + dict.setBatchPrimitiveProcessor(b); - if (!passThru) - col->setBatchPrimitiveProcessor(b); + if (!passThru) + col->setBatchPrimitiveProcessor(b); } -}; +}; // namespace primitiveprocessor diff --git a/primitives/primproc/rtscommand.h b/primitives/primproc/rtscommand.h index 42de09b79..0a2611da3 100644 --- a/primitives/primproc/rtscommand.h +++ b/primitives/primproc/rtscommand.h @@ -37,53 +37,52 @@ namespace primitiveprocessor { - - class RTSCommand : public Command { -public: - RTSCommand(); - virtual ~RTSCommand(); + public: + RTSCommand(); + virtual ~RTSCommand(); - void execute(); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); - uint64_t getLBID(); - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - SCommand duplicate(); - bool operator==(const RTSCommand&) const; - bool operator!=(const RTSCommand&) const; + void execute(); + void project(); + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); + uint64_t getLBID(); + void nextLBID(); + void createCommand(messageqcpp::ByteStream&); + void resetCommand(messageqcpp::ByteStream&); + SCommand duplicate(); + bool operator==(const RTSCommand&) const; + bool operator!=(const RTSCommand&) const; - void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*); + void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*); - /* Put bootstrap code here (ie, build the template primitive msg) */ - void prep(int8_t outputType, bool makeAbsRids); - uint8_t isPassThru() - { - return passThru; - } - void setAbsNull(bool a = true) - { - absNull = a; - } - void getLBIDList(uint32_t loopCount, std::vector* lbids); + /* Put bootstrap code here (ie, build the template primitive msg) */ + void prep(int8_t outputType, bool makeAbsRids); + uint8_t isPassThru() + { + return passThru; + } + void setAbsNull(bool a = true) + { + absNull = a; + } + void getLBIDList(uint32_t loopCount, std::vector* lbids); - //TODO: do we need to reference either col or dict? - int getCompType() const - { - return dict.getCompType(); - } -private: - RTSCommand(const RTSCommand&); + // TODO: do we need to reference either col or dict? + int getCompType() const + { + return dict.getCompType(); + } - ColumnCommandUniquePtr col; - DictStep dict; - uint8_t passThru; - bool absNull; + private: + RTSCommand(const RTSCommand&); + + ColumnCommandUniquePtr col; + DictStep dict; + uint8_t passThru; + bool absNull; }; -} +} // namespace primitiveprocessor #endif diff --git a/primitives/primproc/tdriver-umsocksel.cpp b/primitives/primproc/tdriver-umsocksel.cpp index c6bc5169f..96dd73f10 100644 --- a/primitives/primproc/tdriver-umsocksel.cpp +++ b/primitives/primproc/tdriver-umsocksel.cpp @@ -41,171 +41,161 @@ using namespace primitiveprocessor; int main() { - // Columnstore.xml file should be configured as follows: - // um1: 10.100.4.85 and 10.100.5.85 - // um2: 10.101.4.85 and 10.101.5.85 - sockaddr_in sa = { 1, 0, {0}, {' '} }; - char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"}; + // Columnstore.xml file should be configured as follows: + // um1: 10.100.4.85 and 10.100.5.85 + // um2: 10.101.4.85 and 10.101.5.85 + sockaddr_in sa = {1, 0, {0}, {' '}}; + char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"}; - // These are the IP addresses we use to test runtime connections - // "not" in the Columnstore.xml file. - sockaddr_in saUnknown = { 1, 0, {0}, {' '} }; - char* ipsUnknown[] = {"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"}; + // These are the IP addresses we use to test runtime connections + // "not" in the Columnstore.xml file. + sockaddr_in saUnknown = {1, 0, {0}, {' '}}; + char* ipsUnknown[] = {"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"}; - //-------------------------------------------------------------------------- - // Test initialization - //-------------------------------------------------------------------------- - UmSocketSelector* sockSel = UmSocketSelector::instance(); + //-------------------------------------------------------------------------- + // Test initialization + //-------------------------------------------------------------------------- + UmSocketSelector* sockSel = UmSocketSelector::instance(); - std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl; - std::cout << std::endl << "----Dump1 after initialization..." << std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump1 End...................." << std::endl << std::endl; + std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl; + std::cout << std::endl << "----Dump1 after initialization..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump1 End...................." << std::endl << std::endl; - IOSocket sock[4][4]; + IOSocket sock[4][4]; - for (int i = 0; i < 4; i++) + for (int i = 0; i < 4; i++) + { + inet_aton(ips[i], &sa.sin_addr); + + for (int j = 0; j < 4; j++) { - inet_aton(ips[i], &sa.sin_addr); - - for (int j = 0; j < 4; j++) - { - sock[i][j].setSocketImpl(new InetStreamSocket()); - sa.sin_port = htons((i * 4) + j); - sock[i][j].sa( sa ); - sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])), - SP_UM_MUTEX( new boost::mutex()) ); - } + sock[i][j].setSocketImpl(new InetStreamSocket()); + sa.sin_port = htons((i * 4) + j); + sock[i][j].sa(sa); + sockSel->addConnection(SP_UM_IOSOCK(new IOSocket(sock[i][j])), SP_UM_MUTEX(new boost::mutex())); } + } - std::cout << std::endl << "----Dump2 after adding 16 connections..." << - std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump2 End...................." << std::endl << std::endl; + std::cout << std::endl << "----Dump2 after adding 16 connections..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump2 End...................." << std::endl << std::endl; - //-------------------------------------------------------------------------- - // Test socket/port selection - //-------------------------------------------------------------------------- - std::cout << "Test socket/port selection logic..." << std::endl; + //-------------------------------------------------------------------------- + // Test socket/port selection + //-------------------------------------------------------------------------- + std::cout << "Test socket/port selection logic..." << std::endl; - for (unsigned k = 0; k < 17; k++) - { - SP_UM_IOSOCK outIos; - SP_UM_MUTEX writeLock; + for (unsigned k = 0; k < 17; k++) + { + SP_UM_IOSOCK outIos; + SP_UM_MUTEX writeLock; #if 1 - if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock )) - std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) << - "; port: " << ntohs(outIos->sa().sin_port) << std::endl; - else + if (sockSel->nextIOSocket(sock[0][0], outIos, writeLock)) + std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) + << "; port: " << ntohs(outIos->sa().sin_port) << std::endl; + else #else - if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock )) + if (!sockSel->nextIOSocket(sock[0][0], outIos, writeLock)) #endif - std::cout << "no nextIP found for " << sock[0][0] << std::endl; - } + std::cout << "no nextIP found for " << sock[0][0] << std::endl; + } - for (unsigned k = 0; k < 7; k++) + for (unsigned k = 0; k < 7; k++) + { + SP_UM_IOSOCK outIos; + SP_UM_MUTEX writeLock; + + if (sockSel->nextIOSocket(sock[2][0], outIos, writeLock)) + std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) + << "; port: " << ntohs(outIos->sa().sin_port) << std::endl; + else + std::cout << "no nextIP found for " << sock[2][0] << std::endl; + } + + std::cout << std::endl; + std::cout << "----Dump3 after selecting 17 connections from IP " << ips[0] << "; and 7 connections from IP " + << ips[2] << " ..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump3 End...................." << std::endl << std::endl; + + //-------------------------------------------------------------------------- + // Test connection deletions + //-------------------------------------------------------------------------- + for (unsigned k = 0; k < 4; k++) + { + sockSel->delConnection(sock[k][0]); + } + + std::cout << "----Dump4 after deleting first connection for each IP..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump4 End...................." << std::endl << std::endl; + + //-------------------------------------------------------------------------- + // Test addition of unknown connections + //-------------------------------------------------------------------------- + IOSocket sockUnknown[4][4]; + + for (int i = 0; i < 4; i++) + { + inet_aton(ipsUnknown[i], &saUnknown.sin_addr); + + for (int j = 0; j < 4; j++) { - SP_UM_IOSOCK outIos; - SP_UM_MUTEX writeLock; - - if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock )) - std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) << - "; port: " << ntohs(outIos->sa().sin_port) << std::endl; - else - std::cout << "no nextIP found for " << sock[2][0] << std::endl; + sockUnknown[i][j].setSocketImpl(new InetStreamSocket()); + saUnknown.sin_port = htons((i * 4) + j); + sockUnknown[i][j].sa(saUnknown); + sockSel->addConnection(SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])), SP_UM_MUTEX(new boost::mutex())); } + } - std::cout << std::endl; - std::cout << "----Dump3 after selecting 17 connections from IP " << - ips[0] << "; and 7 connections from IP " << ips[2] << " ..." << - std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump3 End...................." << std::endl << std::endl; + std::cout << "----Dump5 after adding connections for unknown IP's..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump5 End...................." << std::endl << std::endl; - //-------------------------------------------------------------------------- - // Test connection deletions - //-------------------------------------------------------------------------- - for (unsigned k = 0; k < 4; k++) - { - sockSel->delConnection( sock[k][0] ); - } + //-------------------------------------------------------------------------- + // Test resetting of "next" indexes after deleting all sockets from a + // specific IP address for which the "next" index is pointing. + //-------------------------------------------------------------------------- + sockSel->delConnection(sock[1][1]); + sockSel->delConnection(sock[1][2]); + sockSel->delConnection(sock[1][3]); - std::cout << "----Dump4 after deleting first connection for each IP..." << - std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump4 End...................." << std::endl << std::endl; + std::cout << "----Dump6 after deleting all connections for IP " << ips[1] << " ..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump6 End...................." << std::endl << std::endl; - //-------------------------------------------------------------------------- - // Test addition of unknown connections - //-------------------------------------------------------------------------- - IOSocket sockUnknown[4][4]; + //-------------------------------------------------------------------------- + // Test socket/port selection for an unknown module + //-------------------------------------------------------------------------- + std::cout << "Test socket/port selection logic..." << std::endl; - for (int i = 0; i < 4; i++) - { - inet_aton(ipsUnknown[i], &saUnknown.sin_addr); + for (unsigned k = 0; k < 11; k++) + { + SP_UM_IOSOCK outIos; + SP_UM_MUTEX writeLock; - for (int j = 0; j < 4; j++) - { - sockUnknown[i][j].setSocketImpl(new InetStreamSocket()); - saUnknown.sin_port = htons((i * 4) + j); - sockUnknown[i][j].sa( saUnknown ); - sockSel->addConnection( - SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])), - SP_UM_MUTEX( new boost::mutex()) ); - } - } + if (sockSel->nextIOSocket(sockUnknown[2][0], outIos, writeLock)) + std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) + << "; port: " << ntohs(outIos->sa().sin_port) << std::endl; + else + std::cout << "no nextIP found for " << sockUnknown[2][0] << std::endl; + } - std::cout << "----Dump5 after adding connections for unknown IP's..." << - std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump5 End...................." << std::endl << std::endl; + std::cout << std::endl; + std::cout << "----Dump7 after selecting 11 connections for IP " << ipsUnknown[2] << " ..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump7 End...................." << std::endl << std::endl; - //-------------------------------------------------------------------------- - // Test resetting of "next" indexes after deleting all sockets from a - // specific IP address for which the "next" index is pointing. - //-------------------------------------------------------------------------- - sockSel->delConnection( sock[1][1] ); - sockSel->delConnection( sock[1][2] ); - sockSel->delConnection( sock[1][3] ); + //-------------------------------------------------------------------------- + // Test deletion of last socket/port connection and resetting if the + // "next" index that is pointing to it. + //-------------------------------------------------------------------------- + sockSel->delConnection(sock[3][3]); + std::cout << "----Dump8 after deleting last connection for IP " << ips[3] << " ..." << std::endl; + std::cout << sockSel->toString(); + std::cout << "----Dump8 End...................." << std::endl << std::endl; - std::cout << "----Dump6 after deleting all connections for IP " << ips[1] << - " ..." << std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump6 End...................." << std::endl << std::endl; - - //-------------------------------------------------------------------------- - // Test socket/port selection for an unknown module - //-------------------------------------------------------------------------- - std::cout << "Test socket/port selection logic..." << std::endl; - - for (unsigned k = 0; k < 11; k++) - { - SP_UM_IOSOCK outIos; - SP_UM_MUTEX writeLock; - - if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock )) - std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) << - "; port: " << ntohs(outIos->sa().sin_port) << std::endl; - else - std::cout << "no nextIP found for " << sockUnknown[2][0] << std::endl; - } - - std::cout << std::endl; - std::cout << "----Dump7 after selecting 11 connections for IP " << - ipsUnknown[2] << " ..." << std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump7 End...................." << std::endl << std::endl; - - //-------------------------------------------------------------------------- - // Test deletion of last socket/port connection and resetting if the - // "next" index that is pointing to it. - //-------------------------------------------------------------------------- - sockSel->delConnection( sock[3][3] ); - std::cout << "----Dump8 after deleting last connection for IP " << - ips[3] << " ..." << std::endl; - std::cout << sockSel->toString(); - std::cout << "----Dump8 End...................." << std::endl << std::endl; - - return 0; + return 0; } diff --git a/primitives/primproc/tdriver.cpp b/primitives/primproc/tdriver.cpp index e9c533e1a..b22fb2bde 100644 --- a/primitives/primproc/tdriver.cpp +++ b/primitives/primproc/tdriver.cpp @@ -44,918 +44,896 @@ using namespace messageqcpp; void testColByScan() { - cout << "Sending COL_BY_SCAN primitive" << endl; + cout << "Sending COL_BY_SCAN primitive" << endl; - MessageQueueClient proc("PMS1"); - ByteStream obs, ibs; + MessageQueueClient proc("PMS1"); + ByteStream obs, ibs; - ISMPacketHeader packetHeader; - packetHeader.Reserve = 0; - packetHeader.Flow = 0; - packetHeader.Command = COL_BY_SCAN; - packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColByScanRequestHeader)); - packetHeader.Type = 2; - packetHeader.Source = 0; - packetHeader.Dest = 0; - packetHeader.FinalDest = 0; + ISMPacketHeader packetHeader; + packetHeader.Reserve = 0; + packetHeader.Flow = 0; + packetHeader.Command = COL_BY_SCAN; + packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColByScanRequestHeader)); + packetHeader.Type = 2; + packetHeader.Source = 0; + packetHeader.Dest = 0; + packetHeader.FinalDest = 0; - ColByScanRequestHeader colScan; - colScan.Hdr.SessionID = 1; - colScan.Hdr.StatementID = 1; - colScan.Hdr.TransactionID = 0; - colScan.Hdr.VerID = 0; - colScan.LBID = 0; - colScan.PBID = 0; - colScan.DataSize = 8; - colScan.OutputType = 1; - colScan.BOP = 0; - colScan.NOPS = 0; - colScan.NVALS = 1; + ColByScanRequestHeader colScan; + colScan.Hdr.SessionID = 1; + colScan.Hdr.StatementID = 1; + colScan.Hdr.TransactionID = 0; + colScan.Hdr.VerID = 0; + colScan.LBID = 0; + colScan.PBID = 0; + colScan.DataSize = 8; + colScan.OutputType = 1; + colScan.BOP = 0; + colScan.NOPS = 0; + colScan.NVALS = 1; - Int64 TempData; - TempData = 1; + Int64 TempData; + TempData = 1; - unsigned char message[10000]; + unsigned char message[10000]; - memmove(message, &packetHeader, sizeof(ISMPacketHeader) ); - memmove(message + sizeof(ISMPacketHeader), &colScan, sizeof(ColByScanRequestHeader) ); - memmove(message + sizeof(ISMPacketHeader) + sizeof(ColByScanRequestHeader), &TempData, sizeof(Int64)); + memmove(message, &packetHeader, sizeof(ISMPacketHeader)); + memmove(message + sizeof(ISMPacketHeader), &colScan, sizeof(ColByScanRequestHeader)); + memmove(message + sizeof(ISMPacketHeader) + sizeof(ColByScanRequestHeader), &TempData, sizeof(Int64)); - obs.append((messageqcpp::ByteStream::byte*)message, sizeof(ISMPacketHeader) + sizeof(ColByScanRequestHeader) + sizeof(Int64)); + obs.append((messageqcpp::ByteStream::byte*)message, + sizeof(ISMPacketHeader) + sizeof(ColByScanRequestHeader) + sizeof(Int64)); - cout << "Sending to Primitive Server" << endl; - proc.write( obs ); + cout << "Sending to Primitive Server" << endl; + proc.write(obs); - // TODO process results - cout << "Sent... awaiting results" << endl; + // TODO process results + cout << "Sent... awaiting results" << endl; - ibs = proc.read(); - int messageLen = ibs.length(); + ibs = proc.read(); + int messageLen = ibs.length(); - if (messageLen) + if (messageLen) + { + cout << "Received results" << endl << endl; + + ISMPacketHeader pktHeader; + ColResultHeader colResult; + + ByteStream::byte* bytePtr = new messageqcpp::ByteStream::byte[messageLen]; + ibs >> bytePtr; + memmove(((char*)&pktHeader), bytePtr, sizeof(ISMPacketHeader)); + memmove(((char*)&colResult), bytePtr + sizeof(ColResultHeader), sizeof(ColResultHeader)); + + int remaining = messageLen - sizeof(ISMPacketHeader) - sizeof(ColResultHeader); + + cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; + cout << "pktHeader.Flow: " << pktHeader.Flow << endl; + cout << "pktHeader.Command: " << pktHeader.Command << endl; + cout << "pktHeader.Size: " << pktHeader.Size << endl; + cout << "pktHeader.Type: " << pktHeader.Type << endl; + cout << "pktHeader.Source: " << pktHeader.Source << endl; + cout << "pktHeader.Dest: " << pktHeader.Dest << endl; + cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; + cout << "colResult.Hdr.SessionID: " << colResult.Hdr.SessionID << endl; + cout << "colResult.Hdr.StatementID: " << colResult.Hdr.StatementID << endl; + cout << "colResult.Hdr.TransactionID: " << colResult.Hdr.TransactionID << endl; + cout << "colResult.Hdr.VerID: " << colResult.Hdr.VerID << endl; + cout << "colResult.LBID: " << colResult.LBID << endl; + cout << "colResult.NVALS: " << colResult.NVALS << endl; + cout << "colResult.Pad1: " << colResult.Pad1 << endl; + cout << "colResult.Pad2: " << colResult.Pad2 << endl; + + cout << "Data" << endl; + cout << "----" << endl << endl; + cout << "Data Size: " << remaining << endl; + + if (remaining) { - cout << "Received results" << endl << endl; + char* data = new char[remaining]; + memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining); - ISMPacketHeader pktHeader; - ColResultHeader colResult; + char* ptr = data; - ByteStream::byte* bytePtr = new messageqcpp::ByteStream::byte[messageLen]; - ibs >> bytePtr; - memmove(((char*)&pktHeader), bytePtr, sizeof(ISMPacketHeader)); - memmove(((char*)&colResult), bytePtr + sizeof(ColResultHeader), sizeof(ColResultHeader) ); + for (int i = 0; i < remaining; i++) + { + for (int j = 0; j < 10 && j < remaining; j++) + printf("%02x ", *ptr++); - int remaining = messageLen - sizeof(ISMPacketHeader) - sizeof(ColResultHeader); + printf("\n"); + } - cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; - cout << "pktHeader.Flow: " << pktHeader.Flow << endl; - cout << "pktHeader.Command: " << pktHeader.Command << endl; - cout << "pktHeader.Size: " << pktHeader.Size << endl; - cout << "pktHeader.Type: " << pktHeader.Type << endl; - cout << "pktHeader.Source: " << pktHeader.Source << endl; - cout << "pktHeader.Dest: " << pktHeader.Dest << endl; - cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; - cout << "colResult.Hdr.SessionID: " << colResult.Hdr.SessionID << endl; - cout << "colResult.Hdr.StatementID: " << colResult.Hdr.StatementID << endl; - cout << "colResult.Hdr.TransactionID: " << colResult.Hdr.TransactionID << endl; - cout << "colResult.Hdr.VerID: " << colResult.Hdr.VerID << endl; - cout << "colResult.LBID: " << colResult.LBID << endl; - cout << "colResult.NVALS: " << colResult.NVALS << endl; - cout << "colResult.Pad1: " << colResult.Pad1 << endl; - cout << "colResult.Pad2: " << colResult.Pad2 << endl; - - cout << "Data" << endl; - cout << "----" << endl << endl; - cout << "Data Size: " << remaining << endl; - - if (remaining) - { - char* data = new char[remaining]; - memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining); - - char* ptr = data; - - for (int i = 0; i < remaining; i++) - { - for (int j = 0; j < 10 && j < remaining; j++) - printf("%02x ", *ptr++); - - printf("\n"); - } - - delete []data; - } - - delete []bytePtr; + delete[] data; } + delete[] bytePtr; + } } - void testColByRid() { - cout << "Sending COL_BY_RID primitive" << endl; + cout << "Sending COL_BY_RID primitive" << endl; - MessageQueueClient proc("PMS1"); - ByteStream obs, ibs; + MessageQueueClient proc("PMS1"); + ByteStream obs, ibs; - ISMPacketHeader packetHeader; - packetHeader.Reserve = 0; - packetHeader.Flow = 0; - packetHeader.Command = COL_BY_RID; - packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColByRIDRequestHeader)); - packetHeader.Type = 2; - packetHeader.Source = 0; - packetHeader.Dest = 0; - packetHeader.FinalDest = 0; + ISMPacketHeader packetHeader; + packetHeader.Reserve = 0; + packetHeader.Flow = 0; + packetHeader.Command = COL_BY_RID; + packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColByRIDRequestHeader)); + packetHeader.Type = 2; + packetHeader.Source = 0; + packetHeader.Dest = 0; + packetHeader.FinalDest = 0; - ColByRIDRequestHeader colByRid; - colByRid.Hdr.SessionID = 1; - colByRid.Hdr.StatementID = 1; - colByRid.Hdr.TransactionID = 0; - colByRid.Hdr.VerID = 0; - colByRid.LBID = 0; - colByRid.PBID = 0; - colByRid.DataSize = 8; - colByRid.OutputType = 3; - colByRid.BOP = 0; - colByRid.NOPS = 0; - colByRid.NVALS = 1; + ColByRIDRequestHeader colByRid; + colByRid.Hdr.SessionID = 1; + colByRid.Hdr.StatementID = 1; + colByRid.Hdr.TransactionID = 0; + colByRid.Hdr.VerID = 0; + colByRid.LBID = 0; + colByRid.PBID = 0; + colByRid.DataSize = 8; + colByRid.OutputType = 3; + colByRid.BOP = 0; + colByRid.NOPS = 0; + colByRid.NVALS = 1; - Int64 TempData; - TempData = 1; + Int64 TempData; + TempData = 1; - unsigned char message[10000]; + unsigned char message[10000]; - memmove(message, &packetHeader, sizeof(ISMPacketHeader) ); - memmove(message + sizeof(ISMPacketHeader), &colByRid, sizeof(ColByRIDRequestHeader) ); - memmove(message + sizeof(ISMPacketHeader) + sizeof(ColByRIDRequestHeader), &TempData, sizeof(Int64)); + memmove(message, &packetHeader, sizeof(ISMPacketHeader)); + memmove(message + sizeof(ISMPacketHeader), &colByRid, sizeof(ColByRIDRequestHeader)); + memmove(message + sizeof(ISMPacketHeader) + sizeof(ColByRIDRequestHeader), &TempData, sizeof(Int64)); - obs.append((messageqcpp::ByteStream::byte*)message, sizeof(ISMPacketHeader) + sizeof(ColByRIDRequestHeader) + sizeof(Int64)); + obs.append((messageqcpp::ByteStream::byte*)message, + sizeof(ISMPacketHeader) + sizeof(ColByRIDRequestHeader) + sizeof(Int64)); - cout << "Sending to Primitive Server" << endl; + cout << "Sending to Primitive Server" << endl; - proc.write(obs); + proc.write(obs); - cout << "Sent... awaiting results" << endl; + cout << "Sent... awaiting results" << endl; - ibs = proc.read(); - int messageLen = ibs.length(); + ibs = proc.read(); + int messageLen = ibs.length(); - if (messageLen) + if (messageLen) + { + cout << "Received results" << endl << endl; + + ISMPacketHeader pktHeader; + ColResultHeader colResult; + + ByteStream::byte* bytePtr = new messageqcpp::ByteStream::byte[messageLen]; + ibs >> bytePtr; + memmove(((char*)&pktHeader), bytePtr, sizeof(ISMPacketHeader)); + memmove(((char*)&colResult), bytePtr + sizeof(ColResultHeader), sizeof(ColResultHeader)); + + int remaining = messageLen - sizeof(ISMPacketHeader) - sizeof(ColResultHeader); + + cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; + cout << "pktHeader.Flow: " << pktHeader.Flow << endl; + cout << "pktHeader.Command: " << pktHeader.Command << endl; + cout << "pktHeader.Size: " << pktHeader.Size << endl; + cout << "pktHeader.Type: " << pktHeader.Type << endl; + cout << "pktHeader.Source: " << pktHeader.Source << endl; + cout << "pktHeader.Dest: " << pktHeader.Dest << endl; + cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; + cout << "colResult.Hdr.SessionID: " << colResult.Hdr.SessionID << endl; + cout << "colResult.Hdr.StatementID: " << colResult.Hdr.StatementID << endl; + cout << "colResult.Hdr.TransactionID: " << colResult.Hdr.TransactionID << endl; + cout << "colResult.Hdr.VerID: " << colResult.Hdr.VerID << endl; + cout << "colResult.LBID: " << colResult.LBID << endl; + cout << "colResult.NVALS: " << colResult.NVALS << endl; + cout << "colResult.Pad1: " << colResult.Pad1 << endl; + cout << "colResult.Pad2: " << colResult.Pad2 << endl; + + cout << "Data" << endl; + cout << "----" << endl << endl; + cout << "Data Size: " << remaining << endl; + + if (remaining) { + char* data = new char[remaining]; + memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining); - cout << "Received results" << endl << endl; + char* ptr = data; - ISMPacketHeader pktHeader; - ColResultHeader colResult; + for (int i = 0; i < remaining; i++) + { + for (int j = 0; j < 10 && j < remaining; j++) + printf("%02x ", *ptr++); - ByteStream::byte* bytePtr = new messageqcpp::ByteStream::byte[messageLen]; - ibs >> bytePtr; - memmove(((char*)&pktHeader), bytePtr, sizeof(ISMPacketHeader)); - memmove(((char*)&colResult), bytePtr + sizeof(ColResultHeader), sizeof(ColResultHeader) ); - - int remaining = messageLen - sizeof(ISMPacketHeader) - sizeof(ColResultHeader); - - cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; - cout << "pktHeader.Flow: " << pktHeader.Flow << endl; - cout << "pktHeader.Command: " << pktHeader.Command << endl; - cout << "pktHeader.Size: " << pktHeader.Size << endl; - cout << "pktHeader.Type: " << pktHeader.Type << endl; - cout << "pktHeader.Source: " << pktHeader.Source << endl; - cout << "pktHeader.Dest: " << pktHeader.Dest << endl; - cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; - cout << "colResult.Hdr.SessionID: " << colResult.Hdr.SessionID << endl; - cout << "colResult.Hdr.StatementID: " << colResult.Hdr.StatementID << endl; - cout << "colResult.Hdr.TransactionID: " << colResult.Hdr.TransactionID << endl; - cout << "colResult.Hdr.VerID: " << colResult.Hdr.VerID << endl; - cout << "colResult.LBID: " << colResult.LBID << endl; - cout << "colResult.NVALS: " << colResult.NVALS << endl; - cout << "colResult.Pad1: " << colResult.Pad1 << endl; - cout << "colResult.Pad2: " << colResult.Pad2 << endl; - - cout << "Data" << endl; - cout << "----" << endl << endl; - cout << "Data Size: " << remaining << endl; - - if (remaining) - { - char* data = new char[remaining]; - memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining); - - char* ptr = data; - - for (int i = 0; i < remaining; i++) - { - for (int j = 0; j < 10 && j < remaining; j++) - printf("%02x ", *ptr++); - - printf("\n"); - } - - delete []data; - } - - delete []bytePtr; + printf("\n"); + } + delete[] data; } + delete[] bytePtr; + } } - void testColAggByScan() { - cout << "Sending COL_AGG_BY_SCAN primitive" << endl; + cout << "Sending COL_AGG_BY_SCAN primitive" << endl; - MessageQueueClient proc("PMS1"); - ByteStream obs, ibs; + MessageQueueClient proc("PMS1"); + ByteStream obs, ibs; - ISMPacketHeader packetHeader; - packetHeader.Reserve = 0; - packetHeader.Flow = 0; - packetHeader.Command = COL_AGG_BY_SCAN; - packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader)); - packetHeader.Type = 2; - packetHeader.Source = 0; - packetHeader.Dest = 0; - packetHeader.FinalDest = 0; + ISMPacketHeader packetHeader; + packetHeader.Reserve = 0; + packetHeader.Flow = 0; + packetHeader.Command = COL_AGG_BY_SCAN; + packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader)); + packetHeader.Type = 2; + packetHeader.Source = 0; + packetHeader.Dest = 0; + packetHeader.FinalDest = 0; - ColAggByScanRequestHeader colAggByScan; - colAggByScan.Hdr.SessionID = 1; - colAggByScan.Hdr.StatementID = 1; - colAggByScan.Hdr.TransactionID = 0; - colAggByScan.Hdr.VerID = 0; - colAggByScan.LBID = 0; - colAggByScan.PBID = 0; - colAggByScan.DataSize = 8; - colAggByScan.OutputType = 1; - colAggByScan.BOP = 0; - colAggByScan.NOPS = 0; - colAggByScan.NVALS = 1; + ColAggByScanRequestHeader colAggByScan; + colAggByScan.Hdr.SessionID = 1; + colAggByScan.Hdr.StatementID = 1; + colAggByScan.Hdr.TransactionID = 0; + colAggByScan.Hdr.VerID = 0; + colAggByScan.LBID = 0; + colAggByScan.PBID = 0; + colAggByScan.DataSize = 8; + colAggByScan.OutputType = 1; + colAggByScan.BOP = 0; + colAggByScan.NOPS = 0; + colAggByScan.NVALS = 1; - Int64 TempData; - TempData = 1; + Int64 TempData; + TempData = 1; - unsigned char message[10000]; + unsigned char message[10000]; - memmove(message, &packetHeader, sizeof(ISMPacketHeader) ); - memmove(message + sizeof(ISMPacketHeader), &colAggByScan, sizeof(ColAggByScanRequestHeader) ); - memmove(message + sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader), &TempData, sizeof(Int64)); + memmove(message, &packetHeader, sizeof(ISMPacketHeader)); + memmove(message + sizeof(ISMPacketHeader), &colAggByScan, sizeof(ColAggByScanRequestHeader)); + memmove(message + sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader), &TempData, sizeof(Int64)); - obs.append((messageqcpp::ByteStream::byte*)message, sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader) - + sizeof(Int64)); + obs.append((messageqcpp::ByteStream::byte*)message, + sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader) + sizeof(Int64)); - cout << "Sending to Primitive Server" << endl; + cout << "Sending to Primitive Server" << endl; - proc.write(obs); + proc.write(obs); - cout << "Sent... awaiting results" << endl; + cout << "Sent... awaiting results" << endl; - ibs = proc.read(); - int messageLen = ibs.length(); + ibs = proc.read(); + int messageLen = ibs.length(); - if (messageLen) + if (messageLen) + { + ISMPacketHeader pktHeader; + ColAggResultHeader colAggResult; + + cout << "Received results" << endl << endl; + + ByteStream::byte* bytePtr = new messageqcpp::ByteStream::byte[messageLen]; + ibs >> bytePtr; + memmove(((char*)&pktHeader), bytePtr, sizeof(ISMPacketHeader)); + memmove(((char*)&colAggResult), bytePtr + sizeof(ColAggResultHeader), sizeof(ColAggResultHeader)); + + int remaining = messageLen - sizeof(ISMPacketHeader) - sizeof(ColAggResultHeader); + + cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; + cout << "pktHeader.Flow: " << pktHeader.Flow << endl; + cout << "pktHeader.Command: " << pktHeader.Command << endl; + cout << "pktHeader.Size: " << pktHeader.Size << endl; + cout << "pktHeader.Type: " << pktHeader.Type << endl; + cout << "pktHeader.Source: " << pktHeader.Source << endl; + cout << "pktHeader.Dest: " << pktHeader.Dest << endl; + cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; + cout << "colAggResult.Hdr.SessionID: " << colAggResult.Hdr.SessionID << endl; + cout << "colAggResult.Hdr.StatementID: " << colAggResult.Hdr.StatementID << endl; + cout << "colAggResult.Hdr.TransactionID: " << colAggResult.Hdr.TransactionID << endl; + cout << "colAggResult.Hdr.VerID: " << colAggResult.Hdr.VerID << endl; + cout << "colAggResult.LBID: " << colAggResult.LBID << endl; + cout << "colAggResult.MIN: " << colAggResult.MIN << endl; + cout << "colAggResult.MAX: " << colAggResult.MAX << endl; + cout << "colAggResult.SUM: " << colAggResult.SUM << endl; + cout << "colAggResult.SUMOverflow: " << colAggResult.SUMOverflow << endl; + cout << "colAggResult.NVALS: " << colAggResult.NVALS << endl; + cout << "colAggResult.Pad1: " << colAggResult.Pad1 << endl; + + cout << "Data" << endl; + cout << "----" << endl << endl; + cout << "Data Size: " << remaining << endl; + + if (remaining) { - ISMPacketHeader pktHeader; - ColAggResultHeader colAggResult; + char* data = new char[remaining]; + memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining); - cout << "Received results" << endl << endl; + char* ptr = data; - ByteStream::byte* bytePtr = new messageqcpp::ByteStream::byte[messageLen]; - ibs >> bytePtr; - memmove(((char*)&pktHeader), bytePtr, sizeof(ISMPacketHeader)); - memmove(((char*)&colAggResult), bytePtr + sizeof(ColAggResultHeader), sizeof(ColAggResultHeader) ); + for (int i = 0; i < remaining; i++) + { + for (int j = 0; j < 10 && j < remaining; j++) + printf("%02x ", *ptr++); - int remaining = messageLen - sizeof(ISMPacketHeader) - sizeof(ColAggResultHeader); - - cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; - cout << "pktHeader.Flow: " << pktHeader.Flow << endl; - cout << "pktHeader.Command: " << pktHeader.Command << endl; - cout << "pktHeader.Size: " << pktHeader.Size << endl; - cout << "pktHeader.Type: " << pktHeader.Type << endl; - cout << "pktHeader.Source: " << pktHeader.Source << endl; - cout << "pktHeader.Dest: " << pktHeader.Dest << endl; - cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; - cout << "colAggResult.Hdr.SessionID: " << colAggResult.Hdr.SessionID << endl; - cout << "colAggResult.Hdr.StatementID: " << colAggResult.Hdr.StatementID << endl; - cout << "colAggResult.Hdr.TransactionID: " << colAggResult.Hdr.TransactionID << endl; - cout << "colAggResult.Hdr.VerID: " << colAggResult.Hdr.VerID << endl; - cout << "colAggResult.LBID: " << colAggResult.LBID << endl; - cout << "colAggResult.MIN: " << colAggResult.MIN << endl; - cout << "colAggResult.MAX: " << colAggResult.MAX << endl; - cout << "colAggResult.SUM: " << colAggResult.SUM << endl; - cout << "colAggResult.SUMOverflow: " << colAggResult.SUMOverflow << endl; - cout << "colAggResult.NVALS: " << colAggResult.NVALS << endl; - cout << "colAggResult.Pad1: " << colAggResult.Pad1 << endl; - - cout << "Data" << endl; - cout << "----" << endl << endl; - cout << "Data Size: " << remaining << endl; - - if (remaining) - { - char* data = new char[remaining]; - memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining); - - char* ptr = data; - - for (int i = 0; i < remaining; i++) - { - for (int j = 0; j < 10 && j < remaining; j++) - printf("%02x ", *ptr++); - - printf("\n"); - } - - delete []data; - } - - delete []bytePtr; + printf("\n"); + } + delete[] data; } -} + delete[] bytePtr; + } +} void testColAggByRid() { - cout << "Sending COL_AGG_BY_RID primitive" << endl; + cout << "Sending COL_AGG_BY_RID primitive" << endl; - MessageQueueClient proc("PMS1"); - ByteStream obs, ibs; + MessageQueueClient proc("PMS1"); + ByteStream obs, ibs; - ByteStream::octbyte value64; - ByteStream::quadbyte value32; - ByteStream::doublebyte value16; - ByteStream::byte value8; + ByteStream::octbyte value64; + ByteStream::quadbyte value32; + ByteStream::doublebyte value16; + ByteStream::byte value8; - ISMPacketHeader packetHeader; - packetHeader.Reserve = 0; - packetHeader.Flow = 0; - packetHeader.Command = COL_AGG_BY_RID; - packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColAggByRIDRequestHeader)); - packetHeader.Type = 2; - packetHeader.Source = 0; - packetHeader.Dest = 0; - packetHeader.FinalDest = 0; + ISMPacketHeader packetHeader; + packetHeader.Reserve = 0; + packetHeader.Flow = 0; + packetHeader.Command = COL_AGG_BY_RID; + packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(ColAggByRIDRequestHeader)); + packetHeader.Type = 2; + packetHeader.Source = 0; + packetHeader.Dest = 0; + packetHeader.FinalDest = 0; - ColAggByRIDRequestHeader colAggByRID; - colAggByRID.Hdr.SessionID = 1; - colAggByRID.Hdr.StatementID = 1; - colAggByRID.Hdr.TransactionID = 0; - colAggByRID.Hdr.VerID = 0; - colAggByRID.LBID = 0; - colAggByRID.PBID = 0; - colAggByRID.DataSize = 8; - colAggByRID.OutputType = 1; - colAggByRID.BOP = 0; - colAggByRID.NOPS = 0; - colAggByRID.NVALS = 0; + ColAggByRIDRequestHeader colAggByRID; + colAggByRID.Hdr.SessionID = 1; + colAggByRID.Hdr.StatementID = 1; + colAggByRID.Hdr.TransactionID = 0; + colAggByRID.Hdr.VerID = 0; + colAggByRID.LBID = 0; + colAggByRID.PBID = 0; + colAggByRID.DataSize = 8; + colAggByRID.OutputType = 1; + colAggByRID.BOP = 0; + colAggByRID.NOPS = 0; + colAggByRID.NVALS = 0; - cout << "ISMPacketHeader" << endl; - cout << "---------------" << endl << endl; - cout << "Reserve: " << packetHeader.Reserve << endl; - value32 = packetHeader.Reserve; - obs << value32; + cout << "ISMPacketHeader" << endl; + cout << "---------------" << endl << endl; + cout << "Reserve: " << packetHeader.Reserve << endl; + value32 = packetHeader.Reserve; + obs << value32; - cout << "Flow: " << packetHeader.Flow << endl; - value16 = packetHeader.Flow; - obs << value16; + cout << "Flow: " << packetHeader.Flow << endl; + value16 = packetHeader.Flow; + obs << value16; - cout << "Command: " << packetHeader.Command << endl; - value8 = packetHeader.Command; - obs << value8; + cout << "Command: " << packetHeader.Command << endl; + value8 = packetHeader.Command; + obs << value8; - cout << "Size: " << packetHeader.Size << endl; - value16 = packetHeader.Size; - obs << value16; + cout << "Size: " << packetHeader.Size << endl; + value16 = packetHeader.Size; + obs << value16; - cout << "Type: " << packetHeader.Type << endl; - value8 = packetHeader.Type; - obs << value8; + cout << "Type: " << packetHeader.Type << endl; + value8 = packetHeader.Type; + obs << value8; - cout << "Source: " << packetHeader.Source << endl; - value64 = packetHeader.Source; - obs << value64; + cout << "Source: " << packetHeader.Source << endl; + value64 = packetHeader.Source; + obs << value64; - cout << "Dest: " << packetHeader.Dest << endl; - value64 = packetHeader.Dest; - obs << value64; + cout << "Dest: " << packetHeader.Dest << endl; + value64 = packetHeader.Dest; + obs << value64; - cout << "FinalDest: " << packetHeader.FinalDest << endl << endl; - value64 = packetHeader.FinalDest; - obs << value64; + cout << "FinalDest: " << packetHeader.FinalDest << endl << endl; + value64 = packetHeader.FinalDest; + obs << value64; - cout << "ColAggByRIDRequestHeader.PrimitiveHeader" << endl; - cout << "--------------------------------------" << endl << endl; - cout << "SessionID: " << colAggByRID.Hdr.SessionID << endl; - value16 = colAggByRID.Hdr.SessionID; - obs << value16; + cout << "ColAggByRIDRequestHeader.PrimitiveHeader" << endl; + cout << "--------------------------------------" << endl << endl; + cout << "SessionID: " << colAggByRID.Hdr.SessionID << endl; + value16 = colAggByRID.Hdr.SessionID; + obs << value16; - cout << "StatementID: " << colAggByRID.Hdr.StatementID << endl; - value16 = colAggByRID.Hdr.StatementID; - obs << value16; + cout << "StatementID: " << colAggByRID.Hdr.StatementID << endl; + value16 = colAggByRID.Hdr.StatementID; + obs << value16; - cout << "TransactionID: " << colAggByRID.Hdr.TransactionID << endl; - value16 = colAggByRID.Hdr.TransactionID; - obs << value16; + cout << "TransactionID: " << colAggByRID.Hdr.TransactionID << endl; + value16 = colAggByRID.Hdr.TransactionID; + obs << value16; - cout << "VerID: " << colAggByRID.Hdr.VerID << endl; - value16 = colAggByRID.Hdr.VerID; - obs << value16; + cout << "VerID: " << colAggByRID.Hdr.VerID << endl; + value16 = colAggByRID.Hdr.VerID; + obs << value16; - cout << "ColAggByRIDRequestHeader" << endl; - cout << "-------------------------" << endl << endl; + cout << "ColAggByRIDRequestHeader" << endl; + cout << "-------------------------" << endl << endl; - cout << "LBID: " << colAggByRID.LBID << endl; - value64 = colAggByRID.LBID; - obs << value64; + cout << "LBID: " << colAggByRID.LBID << endl; + value64 = colAggByRID.LBID; + obs << value64; - cout << "PBID: " << colAggByRID.PBID << endl; - value64 = colAggByRID.PBID; - obs << value64; + cout << "PBID: " << colAggByRID.PBID << endl; + value64 = colAggByRID.PBID; + obs << value64; - cout << "DataSize: " << colAggByRID.DataSize << endl; - value16 = colAggByRID.DataSize; - obs << value16; + cout << "DataSize: " << colAggByRID.DataSize << endl; + value16 = colAggByRID.DataSize; + obs << value16; - cout << "OutputType: " << colAggByRID.OutputType << endl; - value8 = colAggByRID.OutputType; - obs << value8; + cout << "OutputType: " << colAggByRID.OutputType << endl; + value8 = colAggByRID.OutputType; + obs << value8; - cout << "BOP: " << colAggByRID.BOP << endl; - value8 = colAggByRID.BOP; - obs << value8; + cout << "BOP: " << colAggByRID.BOP << endl; + value8 = colAggByRID.BOP; + obs << value8; - cout << "NOPS: " << colAggByRID.NOPS << endl; - value16 = colAggByRID.NOPS; - obs << value16; + cout << "NOPS: " << colAggByRID.NOPS << endl; + value16 = colAggByRID.NOPS; + obs << value16; - cout << "NVALS: " << colAggByRID.NVALS << endl; - value16 = colAggByRID.NVALS; - obs << value16; + cout << "NVALS: " << colAggByRID.NVALS << endl; + value16 = colAggByRID.NVALS; + obs << value16; - //obs << TempData; + // obs << TempData; - cout << "Sending to Primitive Server" << endl; + cout << "Sending to Primitive Server" << endl; - proc.write(obs); + proc.write(obs); - cout << "Sent... awaiting results" << endl; + cout << "Sent... awaiting results" << endl; - ibs = proc.read(); + ibs = proc.read(); - if (ibs.length() > 0) + if (ibs.length() > 0) + { + ISMPacketHeader pktHeader; + ColAggResultHeader colAggResult; + + cout << "Received results" << endl << endl; + + ibs >> value32; + pktHeader.Reserve = value32; + + ibs >> value16; + pktHeader.Flow = value16; + + ibs >> value8; + pktHeader.Command = value8; + + ibs >> value16; + pktHeader.Size = value16; + + ibs >> value8; + pktHeader.Type = value8; + + ibs >> value64; + pktHeader.Source = value64; + + ibs >> value64; + pktHeader.Dest = value64; + + ibs >> value64; + pktHeader.FinalDest = value64; + + ibs >> value16; + colAggResult.Hdr.SessionID = value16; + + ibs >> value16; + colAggResult.Hdr.StatementID = value16; + + ibs >> value16; + colAggResult.Hdr.TransactionID = value16; + + ibs >> value16; + colAggResult.Hdr.VerID = value16; + + ibs >> value64; + colAggResult.LBID = value64; + + ibs >> value64; + colAggResult.MIN = value64; + + ibs >> value64; + colAggResult.MAX = value64; + + ibs >> value64; + colAggResult.SUM = value64; + + ibs >> value32; + colAggResult.SUMOverflow = value32; + + ibs >> value16; + colAggResult.NVALS = value16; + + ibs >> value16; + colAggResult.Pad1 = value16; + + cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; + cout << "pktHeader.Flow: " << pktHeader.Flow << endl; + cout << "pktHeader.Command: " << pktHeader.Command << endl; + cout << "pktHeader.Size: " << pktHeader.Size << endl; + cout << "pktHeader.Type: " << pktHeader.Type << endl; + cout << "pktHeader.Source: " << pktHeader.Source << endl; + cout << "pktHeader.Dest: " << pktHeader.Dest << endl; + cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; + cout << "colAggResult.Hdr.SessionID: " << colAggResult.Hdr.SessionID << endl; + cout << "colAggResult.Hdr.StatementID: " << colAggResult.Hdr.StatementID << endl; + cout << "colAggResult.Hdr.TransactionID: " << colAggResult.Hdr.TransactionID << endl; + cout << "colAggResult.Hdr.VerID: " << colAggResult.Hdr.VerID << endl; + cout << "colAggResult.LBID: " << colAggResult.LBID << endl; + cout << "colAggResult.MIN: " << colAggResult.MIN << endl; + cout << "colAggResult.MAX: " << colAggResult.MAX << endl; + cout << "colAggResult.SUM: " << colAggResult.SUM << endl; + cout << "colAggResult.SUMOverflow: " << colAggResult.SUMOverflow << endl; + cout << "colAggResult.NVALS: " << colAggResult.NVALS << endl; + cout << "colAggResult.Pad1: " << colAggResult.Pad1 << endl; + + cout << "Data" << endl; + cout << "----" << endl << endl; + cout << "Data Size: " << ibs.length() << endl; + + for (int i = 0; i < colAggResult.NVALS && ibs.length(); i++) { - ISMPacketHeader pktHeader; - ColAggResultHeader colAggResult; - - cout << "Received results" << endl << endl; - - ibs >> value32; - pktHeader.Reserve = value32; - + if (colAggByRID.OutputType == 1 || colAggByRID.OutputType == 3) + { ibs >> value16; - pktHeader.Flow = value16; + cout << "RID: " << value16 << endl; + } - ibs >> value8; - pktHeader.Command = value8; + if (colAggByRID.OutputType == 2 || colAggByRID.OutputType == 3) + { + cout << "Token: "; - ibs >> value16; - pktHeader.Size = value16; - - ibs >> value8; - pktHeader.Type = value8; - - ibs >> value64; - pktHeader.Source = value64; - - ibs >> value64; - pktHeader.Dest = value64; - - ibs >> value64; - pktHeader.FinalDest = value64; - - ibs >> value16; - colAggResult.Hdr.SessionID = value16; - - ibs >> value16; - colAggResult.Hdr.StatementID = value16; - - ibs >> value16; - colAggResult.Hdr.TransactionID = value16; - - ibs >> value16; - colAggResult.Hdr.VerID = value16; - - ibs >> value64; - colAggResult.LBID = value64; - - ibs >> value64; - colAggResult.MIN = value64; - - ibs >> value64; - colAggResult.MAX = value64; - - ibs >> value64; - colAggResult.SUM = value64; - - ibs >> value32; - colAggResult.SUMOverflow = value32; - - ibs >> value16; - colAggResult.NVALS = value16; - - ibs >> value16; - colAggResult.Pad1 = value16; - - cout << "pktHeader.Reserve: " << pktHeader.Reserve << endl; - cout << "pktHeader.Flow: " << pktHeader.Flow << endl; - cout << "pktHeader.Command: " << pktHeader.Command << endl; - cout << "pktHeader.Size: " << pktHeader.Size << endl; - cout << "pktHeader.Type: " << pktHeader.Type << endl; - cout << "pktHeader.Source: " << pktHeader.Source << endl; - cout << "pktHeader.Dest: " << pktHeader.Dest << endl; - cout << "pktHeader.FinalDest: " << pktHeader.FinalDest << endl; - cout << "colAggResult.Hdr.SessionID: " << colAggResult.Hdr.SessionID << endl; - cout << "colAggResult.Hdr.StatementID: " << colAggResult.Hdr.StatementID << endl; - cout << "colAggResult.Hdr.TransactionID: " << colAggResult.Hdr.TransactionID << endl; - cout << "colAggResult.Hdr.VerID: " << colAggResult.Hdr.VerID << endl; - cout << "colAggResult.LBID: " << colAggResult.LBID << endl; - cout << "colAggResult.MIN: " << colAggResult.MIN << endl; - cout << "colAggResult.MAX: " << colAggResult.MAX << endl; - cout << "colAggResult.SUM: " << colAggResult.SUM << endl; - cout << "colAggResult.SUMOverflow: " << colAggResult.SUMOverflow << endl; - cout << "colAggResult.NVALS: " << colAggResult.NVALS << endl; - cout << "colAggResult.Pad1: " << colAggResult.Pad1 << endl; - - cout << "Data" << endl; - cout << "----" << endl << endl; - cout << "Data Size: " << ibs.length() << endl; - - for (int i = 0; i < colAggResult.NVALS && ibs.length(); i++) + switch (colAggByRID.DataSize) { - if (colAggByRID.OutputType == 1 || colAggByRID.OutputType == 3 ) - { - ibs >> value16; - cout << "RID: " << value16 << endl; - } + case 1: + ibs >> value8; + cout << value8 << endl; + break; - if (colAggByRID.OutputType == 2 || colAggByRID.OutputType == 3) - { - cout << "Token: "; + case 2: + ibs >> value16; + cout << value16 << endl; + break; - switch (colAggByRID.DataSize) - { - case 1: - ibs >> value8; - cout << value8 << endl; - break; + case 4: + ibs >> value32; + cout << value32 << endl; + break; - case 2: - ibs >> value16; - cout << value16 << endl; - break; - - case 4: - ibs >> value32; - cout << value32 << endl; - break; - - case 8: - default: - ibs >> value64; - cout << value64 << endl; - break; - } - } + case 8: + default: + ibs >> value64; + cout << value64 << endl; + break; } - + } } - + } } - void testDictTokenByIndexCompare() { - cout << "Sending DICT_TOKEN_BY_INDEX_COMPARE primitive" << endl; + cout << "Sending DICT_TOKEN_BY_INDEX_COMPARE primitive" << endl; - MessageQueueClient proc("PMS1"); - ByteStream obs, ibs; + MessageQueueClient proc("PMS1"); + ByteStream obs, ibs; - ByteStream::octbyte value64; - ByteStream::quadbyte value32; - ByteStream::doublebyte value16; - ByteStream::byte value8; + ByteStream::octbyte value64; + ByteStream::quadbyte value32; + ByteStream::doublebyte value16; + ByteStream::byte value8; - ISMPacketHeader packetHeader; - packetHeader.Reserve = 0; - packetHeader.Flow = 0; - packetHeader.Command = DICT_TOKEN_BY_INDEX_COMPARE; - packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(DictTokenByIndexRequestHeader)); - packetHeader.Type = 2; - packetHeader.Source = 0; - packetHeader.Dest = 0; - packetHeader.FinalDest = 0; + ISMPacketHeader packetHeader; + packetHeader.Reserve = 0; + packetHeader.Flow = 0; + packetHeader.Command = DICT_TOKEN_BY_INDEX_COMPARE; + packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(DictTokenByIndexRequestHeader)); + packetHeader.Type = 2; + packetHeader.Source = 0; + packetHeader.Dest = 0; + packetHeader.FinalDest = 0; - DictTokenByIndexRequestHeader dictTokenByIndex; - dictTokenByIndex.Hdr.SessionID = 1; - dictTokenByIndex.Hdr.StatementID = 1; - dictTokenByIndex.Hdr.TransactionID = 0; - dictTokenByIndex.Hdr.VerID = 0; - dictTokenByIndex.LBID = 0; - dictTokenByIndex.PBID = 0; - dictTokenByIndex.NVALS = 0; - dictTokenByIndex.Pad1 = 0; - dictTokenByIndex.Pad2 = 0; + DictTokenByIndexRequestHeader dictTokenByIndex; + dictTokenByIndex.Hdr.SessionID = 1; + dictTokenByIndex.Hdr.StatementID = 1; + dictTokenByIndex.Hdr.TransactionID = 0; + dictTokenByIndex.Hdr.VerID = 0; + dictTokenByIndex.LBID = 0; + dictTokenByIndex.PBID = 0; + dictTokenByIndex.NVALS = 0; + dictTokenByIndex.Pad1 = 0; + dictTokenByIndex.Pad2 = 0; - cout << "ISMPacketHeader" << endl; - cout << "---------------" << endl << endl; - cout << "Reserve: " << packetHeader.Reserve << endl; - value32 = packetHeader.Reserve; - obs << value32; + cout << "ISMPacketHeader" << endl; + cout << "---------------" << endl << endl; + cout << "Reserve: " << packetHeader.Reserve << endl; + value32 = packetHeader.Reserve; + obs << value32; - cout << "Flow: " << packetHeader.Flow << endl; - value16 = packetHeader.Flow; - obs << value16; + cout << "Flow: " << packetHeader.Flow << endl; + value16 = packetHeader.Flow; + obs << value16; - cout << "Command: " << packetHeader.Command << endl; - value8 = packetHeader.Command; - obs << value8; + cout << "Command: " << packetHeader.Command << endl; + value8 = packetHeader.Command; + obs << value8; - cout << "Size: " << packetHeader.Size << endl; - value16 = packetHeader.Size; - obs << value16; + cout << "Size: " << packetHeader.Size << endl; + value16 = packetHeader.Size; + obs << value16; - cout << "Type: " << packetHeader.Type << endl; - value8 = packetHeader.Type; - obs << value8; + cout << "Type: " << packetHeader.Type << endl; + value8 = packetHeader.Type; + obs << value8; - cout << "Source: " << packetHeader.Source << endl; - value64 = packetHeader.Source; - obs << value64; + cout << "Source: " << packetHeader.Source << endl; + value64 = packetHeader.Source; + obs << value64; - cout << "Dest: " << packetHeader.Dest << endl; - value64 = packetHeader.Dest; - obs << value64; + cout << "Dest: " << packetHeader.Dest << endl; + value64 = packetHeader.Dest; + obs << value64; - cout << "FinalDest: " << packetHeader.FinalDest << endl << endl; - value64 = packetHeader.FinalDest; - obs << value64; + cout << "FinalDest: " << packetHeader.FinalDest << endl << endl; + value64 = packetHeader.FinalDest; + obs << value64; - cout << "DictTokenByIndexRequestHeader.PrimitiveHeader" << endl; - cout << "--------------------------------------" << endl << endl; - cout << "SessionID: " << dictTokenByIndex.Hdr.SessionID << endl; - value16 = dictTokenByIndex.Hdr.SessionID; - obs << value16; + cout << "DictTokenByIndexRequestHeader.PrimitiveHeader" << endl; + cout << "--------------------------------------" << endl << endl; + cout << "SessionID: " << dictTokenByIndex.Hdr.SessionID << endl; + value16 = dictTokenByIndex.Hdr.SessionID; + obs << value16; - cout << "StatementID: " << dictTokenByIndex.Hdr.StatementID << endl; - value16 = dictTokenByIndex.Hdr.StatementID; - obs << value16; + cout << "StatementID: " << dictTokenByIndex.Hdr.StatementID << endl; + value16 = dictTokenByIndex.Hdr.StatementID; + obs << value16; - cout << "TransactionID: " << dictTokenByIndex.Hdr.TransactionID << endl; - value16 = dictTokenByIndex.Hdr.TransactionID; - obs << value16; + cout << "TransactionID: " << dictTokenByIndex.Hdr.TransactionID << endl; + value16 = dictTokenByIndex.Hdr.TransactionID; + obs << value16; - cout << "VerID: " << dictTokenByIndex.Hdr.VerID << endl; - value16 = dictTokenByIndex.Hdr.VerID; - obs << value16; + cout << "VerID: " << dictTokenByIndex.Hdr.VerID << endl; + value16 = dictTokenByIndex.Hdr.VerID; + obs << value16; - cout << "DictTokenByIndexRequestHeader" << endl; - cout << "-------------------------" << endl << endl; + cout << "DictTokenByIndexRequestHeader" << endl; + cout << "-------------------------" << endl << endl; - cout << "LBID: " << dictTokenByIndex.LBID << endl; - value64 = dictTokenByIndex.LBID; - obs << value64; + cout << "LBID: " << dictTokenByIndex.LBID << endl; + value64 = dictTokenByIndex.LBID; + obs << value64; - cout << "PBID: " << dictTokenByIndex.PBID << endl; - value64 = dictTokenByIndex.PBID; - obs << value64; + cout << "PBID: " << dictTokenByIndex.PBID << endl; + value64 = dictTokenByIndex.PBID; + obs << value64; - cout << "NVALS: " << dictTokenByIndex.NVALS << endl; - value16 = dictTokenByIndex.NVALS; - obs << value16; + cout << "NVALS: " << dictTokenByIndex.NVALS << endl; + value16 = dictTokenByIndex.NVALS; + obs << value16; - cout << "Pad1: " << dictTokenByIndex.Pad1 << endl; - value16 = dictTokenByIndex.Pad1; - obs << value16; + cout << "Pad1: " << dictTokenByIndex.Pad1 << endl; + value16 = dictTokenByIndex.Pad1; + obs << value16; - cout << "Pad2: " << dictTokenByIndex.Pad2 << endl; - value32 = dictTokenByIndex.Pad2; - obs << value32; + cout << "Pad2: " << dictTokenByIndex.Pad2 << endl; + value32 = dictTokenByIndex.Pad2; + obs << value32; - cout << "Sending to Primitive Server" << endl; + cout << "Sending to Primitive Server" << endl; - proc.write(obs); + proc.write(obs); - cout << "Sent... awaiting results" << endl; + cout << "Sent... awaiting results" << endl; - ibs = proc.read(); - cout << "Data" << endl; - cout << "----" << endl << endl; + ibs = proc.read(); + cout << "Data" << endl; + cout << "----" << endl << endl; - if (ibs.length() > 0) + if (ibs.length() > 0) + { + cout << "Data Size: " << ibs.length() << endl; + + while (ibs.length()) { - cout << "Data Size: " << ibs.length() << endl; - - while (ibs.length()) - { - ibs >> value16; - cout << value16 << endl; - } + ibs >> value16; + cout << value16 << endl; } + } } - void testDictTokenByScanCompare() { - cout << "Sending DICT_TOKEN_BY_SCAN_COMPARE primitive" << endl; + cout << "Sending DICT_TOKEN_BY_SCAN_COMPARE primitive" << endl; } - void testDictSignature() { - cout << "DICT_SIGNATURE primitive" << endl; + cout << "DICT_SIGNATURE primitive" << endl; - MessageQueueClient proc("PMS1"); - ByteStream obs, ibs; + MessageQueueClient proc("PMS1"); + ByteStream obs, ibs; - ByteStream::octbyte value64; - ByteStream::quadbyte value32; - ByteStream::doublebyte value16; - ByteStream::byte value8; + ByteStream::octbyte value64; + ByteStream::quadbyte value32; + ByteStream::doublebyte value16; + ByteStream::byte value8; - ISMPacketHeader packetHeader; - packetHeader.Reserve = 0; - packetHeader.Flow = 0; - packetHeader.Command = DICT_SIGNATURE; - packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(DictSignatureRequestHeader)); - packetHeader.Type = 2; - packetHeader.Source = 0; - packetHeader.Dest = 0; - packetHeader.FinalDest = 0; + ISMPacketHeader packetHeader; + packetHeader.Reserve = 0; + packetHeader.Flow = 0; + packetHeader.Command = DICT_SIGNATURE; + packetHeader.Size = (sizeof(ISMPacketHeader) + sizeof(DictSignatureRequestHeader)); + packetHeader.Type = 2; + packetHeader.Source = 0; + packetHeader.Dest = 0; + packetHeader.FinalDest = 0; - DictSignatureRequestHeader dictSignature; - dictSignature.Hdr.SessionID = 1; - dictSignature.Hdr.StatementID = 1; - dictSignature.Hdr.TransactionID = 0; - dictSignature.Hdr.VerID = 0; - dictSignature.LBID = 0; - dictSignature.PBID = 0; - dictSignature.NVALS = 0; - dictSignature.Pad1 = 0; - dictSignature.Pad2 = 0; + DictSignatureRequestHeader dictSignature; + dictSignature.Hdr.SessionID = 1; + dictSignature.Hdr.StatementID = 1; + dictSignature.Hdr.TransactionID = 0; + dictSignature.Hdr.VerID = 0; + dictSignature.LBID = 0; + dictSignature.PBID = 0; + dictSignature.NVALS = 0; + dictSignature.Pad1 = 0; + dictSignature.Pad2 = 0; - cout << "ISMPacketHeader" << endl; - cout << "---------------" << endl << endl; - cout << "Reserve: " << packetHeader.Reserve << endl; - value32 = packetHeader.Reserve; - obs << value32; + cout << "ISMPacketHeader" << endl; + cout << "---------------" << endl << endl; + cout << "Reserve: " << packetHeader.Reserve << endl; + value32 = packetHeader.Reserve; + obs << value32; - cout << "Flow: " << packetHeader.Flow << endl; - value16 = packetHeader.Flow; - obs << value16; + cout << "Flow: " << packetHeader.Flow << endl; + value16 = packetHeader.Flow; + obs << value16; - cout << "Command: " << packetHeader.Command << endl; - value8 = packetHeader.Command; - obs << value8; + cout << "Command: " << packetHeader.Command << endl; + value8 = packetHeader.Command; + obs << value8; - cout << "Size: " << packetHeader.Size << endl; - value16 = packetHeader.Size; - obs << value16; + cout << "Size: " << packetHeader.Size << endl; + value16 = packetHeader.Size; + obs << value16; - cout << "Type: " << packetHeader.Type << endl; - value8 = packetHeader.Type; - obs << value8; + cout << "Type: " << packetHeader.Type << endl; + value8 = packetHeader.Type; + obs << value8; - cout << "Source: " << packetHeader.Source << endl; - value64 = packetHeader.Source; - obs << value64; + cout << "Source: " << packetHeader.Source << endl; + value64 = packetHeader.Source; + obs << value64; - cout << "Dest: " << packetHeader.Dest << endl; - value64 = packetHeader.Dest; - obs << value64; + cout << "Dest: " << packetHeader.Dest << endl; + value64 = packetHeader.Dest; + obs << value64; - cout << "FinalDest: " << packetHeader.FinalDest << endl << endl; - value64 = packetHeader.FinalDest; - obs << value64; + cout << "FinalDest: " << packetHeader.FinalDest << endl << endl; + value64 = packetHeader.FinalDest; + obs << value64; - cout << "DictSignatureRequestHeader.PrimitiveHeader" << endl; - cout << "--------------------------------------" << endl << endl; - cout << "SessionID: " << dictSignature.Hdr.SessionID << endl; - value16 = dictSignature.Hdr.SessionID; - obs << value16; + cout << "DictSignatureRequestHeader.PrimitiveHeader" << endl; + cout << "--------------------------------------" << endl << endl; + cout << "SessionID: " << dictSignature.Hdr.SessionID << endl; + value16 = dictSignature.Hdr.SessionID; + obs << value16; - cout << "StatementID: " << dictSignature.Hdr.StatementID << endl; - value16 = dictSignature.Hdr.StatementID; - obs << value16; + cout << "StatementID: " << dictSignature.Hdr.StatementID << endl; + value16 = dictSignature.Hdr.StatementID; + obs << value16; - cout << "TransactionID: " << dictSignature.Hdr.TransactionID << endl; - value16 = dictSignature.Hdr.TransactionID; - obs << value16; + cout << "TransactionID: " << dictSignature.Hdr.TransactionID << endl; + value16 = dictSignature.Hdr.TransactionID; + obs << value16; - cout << "VerID: " << dictSignature.Hdr.VerID << endl; - value16 = dictSignature.Hdr.VerID; - obs << value16; + cout << "VerID: " << dictSignature.Hdr.VerID << endl; + value16 = dictSignature.Hdr.VerID; + obs << value16; - cout << "DictSignatureRequestHeader" << endl; - cout << "-------------------------" << endl << endl; + cout << "DictSignatureRequestHeader" << endl; + cout << "-------------------------" << endl << endl; - cout << "LBID: " << dictSignature.LBID << endl; - value64 = dictSignature.LBID; - obs << value64; + cout << "LBID: " << dictSignature.LBID << endl; + value64 = dictSignature.LBID; + obs << value64; - cout << "PBID: " << dictSignature.PBID << endl; - value64 = dictSignature.LBID; - obs << value64; + cout << "PBID: " << dictSignature.PBID << endl; + value64 = dictSignature.LBID; + obs << value64; - cout << "NVALS: " << dictSignature.NVALS << endl; - value16 = dictSignature.NVALS; - obs << value16; + cout << "NVALS: " << dictSignature.NVALS << endl; + value16 = dictSignature.NVALS; + obs << value16; - cout << "Pad1: " << dictSignature.Pad1 << endl; - value16 = dictSignature.Pad1; - obs << value16; + cout << "Pad1: " << dictSignature.Pad1 << endl; + value16 = dictSignature.Pad1; + obs << value16; - cout << "Pad2: " << dictSignature.Pad2 << endl; - value32 = dictSignature.Pad2; - obs << value32; + cout << "Pad2: " << dictSignature.Pad2 << endl; + value32 = dictSignature.Pad2; + obs << value32; - cout << "Sending to Primitive Server" << endl; + cout << "Sending to Primitive Server" << endl; - proc.write(obs); + proc.write(obs); - cout << "Sent... awaiting results" << endl; + cout << "Sent... awaiting results" << endl; - ibs = proc.read(); + ibs = proc.read(); - cout << "Data" << endl; - cout << "----" << endl << endl; + cout << "Data" << endl; + cout << "----" << endl << endl; - if (ibs.length() > 0) + if (ibs.length() > 0) + { + cout << "Data Size: " << ibs.length() << endl; + + while (ibs.length()) { - cout << "Data Size: " << ibs.length() << endl; - - while (ibs.length()) - { - ibs >> value16; - cout << value16 << endl; - } + ibs >> value16; + cout << value16 << endl; } - + } } - void testDictAggregate() { - cout << "DICT_AGGREGATE primitive" << endl; - + cout << "DICT_AGGREGATE primitive" << endl; } - void testIndexByCompare() { - cout << "INDEX_BY_COMPARE primitive" << endl; + cout << "INDEX_BY_COMPARE primitive" << endl; } - void testIndexByScan() { - cout << "INDEX_BY_SCAN primitive" << endl; + cout << "INDEX_BY_SCAN primitive" << endl; } - int main(int argc, char* argv[]) { + po::options_description desc("Allowed options"); + desc.add_options()("help", "produce help message")("all", "process all tests")("loop", + "loop processing all tests"); + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); - po::options_description desc ("Allowed options"); - desc.add_options () - ("help", "produce help message") - ("all", "process all tests" ) - ("loop", "loop processing all tests"); - po::variables_map vm; - po::store (po::parse_command_line (argc, argv, desc), vm); - po::notify (vm); - - if (vm.count ("all")) + if (vm.count("all")) + { + testColByScan(); + // testColByRid(); + // testColAggByScan(); + // testColAggByRid(); + // testDictSignature(); + // testDictTokenByIndexCompare(); + } + else if (vm.count("loop")) + { + while (1) { - testColByScan(); - //testColByRid(); - //testColAggByScan(); - //testColAggByRid(); - //testDictSignature(); - //testDictTokenByIndexCompare(); + testColByScan(); + testColByRid(); + testColAggByScan(); + testColAggByRid(); + testDictSignature(); + testDictTokenByIndexCompare(); } - else if (vm.count ("loop")) - { - while (1) - { - testColByScan(); - testColByRid(); - testColAggByScan(); - testColAggByRid(); - testDictSignature(); - testDictTokenByIndexCompare(); - } - - } - + } } diff --git a/primitives/primproc/udf.cpp b/primitives/primproc/udf.cpp index 833a63755..7a836590d 100644 --- a/primitives/primproc/udf.cpp +++ b/primitives/primproc/udf.cpp @@ -35,40 +35,38 @@ const string UdfLibName("libcaludf.so"); namespace primitiveprocessor { - void loadUDFs() { #ifndef _MSC_VER - int flags = RTLD_NOW; + int flags = RTLD_NOW; - void* libPtr = 0; + void* libPtr = 0; - UDFFcnPtr_t fcnPtr; + UDFFcnPtr_t fcnPtr; - libPtr = dlopen(UdfLibName.c_str(), flags); + libPtr = dlopen(UdfLibName.c_str(), flags); - if (libPtr == 0) - return; + if (libPtr == 0) + return; - unsigned i = 1; + unsigned i = 1; - for (;;) - { - ostringstream oss; - oss << "cpfunc" << i; - fcnPtr = (UDFFcnPtr_t)dlsym(libPtr, oss.str().c_str()); + for (;;) + { + ostringstream oss; + oss << "cpfunc" << i; + fcnPtr = (UDFFcnPtr_t)dlsym(libPtr, oss.str().c_str()); - if (fcnPtr == 0) - break; + if (fcnPtr == 0) + break; - UDFFcnMap[i] = fcnPtr; - i++; - } + UDFFcnMap[i] = fcnPtr; + i++; + } - cout << "loaded " << UDFFcnMap.size() << " UDF's" << endl; + cout << "loaded " << UDFFcnMap.size() << " UDF's" << endl; #endif } -} +} // namespace primitiveprocessor // vim:ts=4 sw=4: - diff --git a/primitives/primproc/umsocketselector.cpp b/primitives/primproc/umsocketselector.cpp index 5fa76f3bc..7b58fa73a 100644 --- a/primitives/primproc/umsocketselector.cpp +++ b/primitives/primproc/umsocketselector.cpp @@ -56,23 +56,22 @@ using namespace oam; namespace primitiveprocessor { /*static*/ const int32_t UmIPSocketConns::NEXT_IOSOCKET_UNASSIGNED = -1; -/*static*/ const int32_t UmModuleIPs::NEXT_IP_SOCKET_UNASSIGNED = -1; -/*static*/ UmSocketSelector* UmSocketSelector::fpUmSocketSelector = 0; +/*static*/ const int32_t UmModuleIPs::NEXT_IP_SOCKET_UNASSIGNED = -1; +/*static*/ UmSocketSelector* UmSocketSelector::fpUmSocketSelector = 0; //------------------------------------------------------------------------------ // UmSocketSelector methods //------------------------------------------------------------------------------ // UmSocketSelector Singleton accessor //------------------------------------------------------------------------------ -/* static */ UmSocketSelector* -UmSocketSelector::instance() +/* static */ UmSocketSelector* UmSocketSelector::instance() { - if (fpUmSocketSelector == 0) - { - fpUmSocketSelector = new UmSocketSelector(); - } + if (fpUmSocketSelector == 0) + { + fpUmSocketSelector = new UmSocketSelector(); + } - return fpUmSocketSelector; + return fpUmSocketSelector; } //------------------------------------------------------------------------------ @@ -80,24 +79,23 @@ UmSocketSelector::instance() //------------------------------------------------------------------------------ UmSocketSelector::UmSocketSelector() { - loadUMModuleInfo(); + loadUMModuleInfo(); } //------------------------------------------------------------------------------ // Returns the number of IP addresses defined in the Columnstore.xml file // return - uint32_t; the total number of IP addresses in the Columnstore.xml file //------------------------------------------------------------------------------ -uint32_t -UmSocketSelector::ipAddressCount() const +uint32_t UmSocketSelector::ipAddressCount() const { - uint32_t ipCount = 0; + uint32_t ipCount = 0; - for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i) - { - ipCount += fUmModuleIPs[i]->ipAddressCount(); - } + for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i) + { + ipCount += fUmModuleIPs[i]->ipAddressCount(); + } - return ipCount; + return ipCount; } //------------------------------------------------------------------------------ @@ -106,71 +104,67 @@ UmSocketSelector::ipAddressCount() const // does not insure thread safeness, but that's okay because it is assumed that // it will only be called once from the first call to instance(). //------------------------------------------------------------------------------ -void -UmSocketSelector::loadUMModuleInfo() +void UmSocketSelector::loadUMModuleInfo() { - Oam oam; - ModuleTypeConfig moduleTypeConfig; - const std::string UM_MODTYPE("um"); + Oam oam; + ModuleTypeConfig moduleTypeConfig; + const std::string UM_MODTYPE("um"); - oam.getSystemConfig(UM_MODTYPE, moduleTypeConfig); + oam.getSystemConfig(UM_MODTYPE, moduleTypeConfig); - int moduleCount = moduleTypeConfig.ModuleCount; - std::string moduleType = moduleTypeConfig.ModuleType; + int moduleCount = moduleTypeConfig.ModuleCount; + std::string moduleType = moduleTypeConfig.ModuleType; #ifdef LOAD_MODULE_DEBUG - std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl; - std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl; - std::cout << "ModuleCount = " << moduleCount << std::endl; - std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl; + std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl; + std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl; + std::cout << "ModuleCount = " << moduleCount << std::endl; + std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl; #endif - if ( moduleCount > 0 ) + if (moduleCount > 0) + { + //..Loop through the list of UM modules + for (DeviceNetworkList::iterator iter1 = moduleTypeConfig.ModuleNetworkList.begin(); + (iter1 != moduleTypeConfig.ModuleNetworkList.end()); ++iter1) { - //..Loop through the list of UM modules - for (DeviceNetworkList::iterator iter1 = - moduleTypeConfig.ModuleNetworkList.begin(); - (iter1 != moduleTypeConfig.ModuleNetworkList.end()); - ++iter1) + std::string moduleName = iter1->DeviceName; + +#ifdef LOAD_MODULE_DEBUG + std::cout << "ModuleName-" << moduleName << std::endl; +#endif + + //..Assign the UM index based on whether it is a new UM or one + // we have seen before + unsigned int umIdx = findOrAddUm(moduleName); + + //..Get the list of IP addresses (NIC's) for this UM module + for (HostConfigList::iterator iter2 = iter1->hostConfigList.begin(); + (iter2 != iter1->hostConfigList.end()); ++iter2) + { + std::string ipAddr = iter2->IPAddr; + +#ifdef LOAD_MODULE_DEBUG + std::cout << " NIC-" << iter2->NicID << "; host-" << iter2->HostName << "; IP-" << ipAddr + << std::endl; +#endif + + struct in_addr ip; + + if (inet_aton(ipAddr.c_str(), &ip)) { - std::string moduleName = iter1->DeviceName; - -#ifdef LOAD_MODULE_DEBUG - std::cout << "ModuleName-" << moduleName << std::endl; -#endif - - //..Assign the UM index based on whether it is a new UM or one - // we have seen before - unsigned int umIdx = findOrAddUm( moduleName ); - - //..Get the list of IP addresses (NIC's) for this UM module - for (HostConfigList::iterator iter2 = iter1->hostConfigList.begin(); - (iter2 != iter1->hostConfigList.end()); - ++iter2) - { - std::string ipAddr = iter2->IPAddr; - -#ifdef LOAD_MODULE_DEBUG - std::cout << " NIC-" << iter2->NicID << - "; host-" << iter2->HostName << - "; IP-" << ipAddr << std::endl; -#endif - - struct in_addr ip; - - if ( inet_aton(ipAddr.c_str(), &ip ) ) - { - fIpAddressUmMap[ ip.s_addr ] = umIdx; - fUmModuleIPs[umIdx]->addIP( ip.s_addr ); - } - else - { - std::cerr << "Invalid IP address in SystemModuleConfig " - "section: " << ipAddr << std::endl; - } - } // loop through the IP addresses for a UM module - } // loop through the list of UM modules - } // moduleCount > 0 + fIpAddressUmMap[ip.s_addr] = umIdx; + fUmModuleIPs[umIdx]->addIP(ip.s_addr); + } + else + { + std::cerr << "Invalid IP address in SystemModuleConfig " + "section: " + << ipAddr << std::endl; + } + } // loop through the IP addresses for a UM module + } // loop through the list of UM modules + } // moduleCount > 0 } //------------------------------------------------------------------------------ @@ -178,25 +172,24 @@ UmSocketSelector::loadUMModuleInfo() // fUmModuleIPs, if it is found. Else add the new moduleName. // return - unsigned int for the index into fUmModuleIPs for moduleName. //------------------------------------------------------------------------------ -unsigned int -UmSocketSelector::findOrAddUm( const std::string& moduleName ) +unsigned int UmSocketSelector::findOrAddUm(const std::string& moduleName) { - unsigned int umIdx = std::numeric_limits::max(); + unsigned int umIdx = std::numeric_limits::max(); - for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i) + for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i) + { + if (fUmModuleIPs[i]->moduleName() == moduleName) { - if (fUmModuleIPs[i]->moduleName() == moduleName) - { - umIdx = i; - return umIdx; - } + umIdx = i; + return umIdx; } + } - //..We have encountered a new UM module we should add to the list - fUmModuleIPs.push_back( SP_UM_MODIPS(new UmModuleIPs(moduleName)) ); - umIdx = fUmModuleIPs.size() - 1; + //..We have encountered a new UM module we should add to the list + fUmModuleIPs.push_back(SP_UM_MODIPS(new UmModuleIPs(moduleName))); + umIdx = fUmModuleIPs.size() - 1; - return umIdx; + return umIdx; } //------------------------------------------------------------------------------ @@ -206,55 +199,48 @@ UmSocketSelector::findOrAddUm( const std::string& moduleName ) // writeLock (in) - mutex to use when writing to ios. // return - boolean indicating whether socket/port connection was added. //------------------------------------------------------------------------------ -bool -UmSocketSelector::addConnection( - const SP_UM_IOSOCK& ios, - const SP_UM_MUTEX& writeLock ) +bool UmSocketSelector::addConnection(const SP_UM_IOSOCK& ios, const SP_UM_MUTEX& writeLock) { - bool bConnAdded = false; + bool bConnAdded = false; - sockaddr sa = ios->sa(); - const sockaddr_in* sinp = reinterpret_cast(&sa); - IpAddressUmMap_t::iterator mapIter = - fIpAddressUmMap.find ( sinp->sin_addr.s_addr ); + sockaddr sa = ios->sa(); + const sockaddr_in* sinp = reinterpret_cast(&sa); + IpAddressUmMap_t::iterator mapIter = fIpAddressUmMap.find(sinp->sin_addr.s_addr); - // Add this socket/port connection to the UM connection list it belongs to. - if ( mapIter != fIpAddressUmMap.end() ) - { - unsigned int umIdx = mapIter->second; - bConnAdded = fUmModuleIPs[umIdx]->addSocketConn( ios, writeLock ); - } + // Add this socket/port connection to the UM connection list it belongs to. + if (mapIter != fIpAddressUmMap.end()) + { + unsigned int umIdx = mapIter->second; + bConnAdded = fUmModuleIPs[umIdx]->addSocketConn(ios, writeLock); + } - if (!bConnAdded) - { + if (!bConnAdded) + { #ifdef SEL_CONN_DEBUG - std::ostringstream oss; - oss << "No UM/IP match found to add connection " << ios->toString() << - std::endl; - std::cout << oss.str(); + std::ostringstream oss; + oss << "No UM/IP match found to add connection " << ios->toString() << std::endl; + std::cout << oss.str(); #endif - } + } - return bConnAdded; + return bConnAdded; } //------------------------------------------------------------------------------ // Delete a socket/port connection from the UM for which it belongs. // ioSock (in) - socket/port connection to be deleted //------------------------------------------------------------------------------ -void -UmSocketSelector::delConnection( const messageqcpp::IOSocket& ios ) +void UmSocketSelector::delConnection(const messageqcpp::IOSocket& ios) { - sockaddr sa = ios.sa(); - const sockaddr_in* sinp = reinterpret_cast(&sa); - IpAddressUmMap_t::iterator mapIter = - fIpAddressUmMap.find ( sinp->sin_addr.s_addr ); + sockaddr sa = ios.sa(); + const sockaddr_in* sinp = reinterpret_cast(&sa); + IpAddressUmMap_t::iterator mapIter = fIpAddressUmMap.find(sinp->sin_addr.s_addr); - if ( mapIter != fIpAddressUmMap.end() ) - { - unsigned int umIdx = mapIter->second; - fUmModuleIPs[umIdx]->delSocketConn( ios ); - } + if (mapIter != fIpAddressUmMap.end()) + { + unsigned int umIdx = mapIter->second; + fUmModuleIPs[umIdx]->delSocketConn(ios); + } } //------------------------------------------------------------------------------ @@ -269,67 +255,58 @@ UmSocketSelector::delConnection( const messageqcpp::IOSocket& ios ) // return - bool indicating if socket/port connection was assigned to // outIos //------------------------------------------------------------------------------ -bool -UmSocketSelector::nextIOSocket( - const messageqcpp::IOSocket& ios, - SP_UM_IOSOCK& outIos, - SP_UM_MUTEX& writeLock ) +bool UmSocketSelector::nextIOSocket(const messageqcpp::IOSocket& ios, SP_UM_IOSOCK& outIos, + SP_UM_MUTEX& writeLock) { - sockaddr sa = ios.sa(); - const sockaddr_in* sinp = reinterpret_cast(&sa); - IpAddressUmMap_t::iterator mapIter = - fIpAddressUmMap.find ( sinp->sin_addr.s_addr ); + sockaddr sa = ios.sa(); + const sockaddr_in* sinp = reinterpret_cast(&sa); + IpAddressUmMap_t::iterator mapIter = fIpAddressUmMap.find(sinp->sin_addr.s_addr); - if ( mapIter != fIpAddressUmMap.end() ) + if (mapIter != fIpAddressUmMap.end()) + { + unsigned int umIdx = mapIter->second; + + if (fUmModuleIPs[umIdx]->nextIOSocket(outIos, writeLock)) { - unsigned int umIdx = mapIter->second; - - if (fUmModuleIPs[umIdx]->nextIOSocket( outIos, writeLock )) - { - #ifdef SEL_CONN_DEBUG - std::ostringstream oss; - oss << "UM " << fUmModuleIPs[umIdx]->moduleName() << - "; in: " << ios.toString() << - "; selected out: " << outIos->toString() << std::endl; - std::cout << oss.str(); + std::ostringstream oss; + oss << "UM " << fUmModuleIPs[umIdx]->moduleName() << "; in: " << ios.toString() + << "; selected out: " << outIos->toString() << std::endl; + std::cout << oss.str(); #endif - return true; - } + return true; } + } - //..This should not happen. Application is asking for next socket/port for - // a connection not in our UM module list. - return false; + //..This should not happen. Application is asking for next socket/port for + // a connection not in our UM module list. + return false; } //------------------------------------------------------------------------------ // Convert contents to string for logging, debugging, etc. //------------------------------------------------------------------------------ -const std::string -UmSocketSelector::toString() const +const std::string UmSocketSelector::toString() const { - std::ostringstream oss; + std::ostringstream oss; - oss << "IP Address to UM index map:" << std::endl; + oss << "IP Address to UM index map:" << std::endl; - for (IpAddressUmMap_t::const_iterator mapIter = fIpAddressUmMap.begin(); - (mapIter != fIpAddressUmMap.end()); - ++mapIter) - { - char ipString[INET_ADDRSTRLEN]; - oss << " IPAddress: " << - UmIPSocketConns::nwToString(mapIter->first, ipString) << - " maps to UM: " << mapIter->second << std::endl; - } + for (IpAddressUmMap_t::const_iterator mapIter = fIpAddressUmMap.begin(); (mapIter != fIpAddressUmMap.end()); + ++mapIter) + { + char ipString[INET_ADDRSTRLEN]; + oss << " IPAddress: " << UmIPSocketConns::nwToString(mapIter->first, ipString) + << " maps to UM: " << mapIter->second << std::endl; + } - for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i) - { - oss << std::endl << fUmModuleIPs[i]->toString(); - } + for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i) + { + oss << std::endl << fUmModuleIPs[i]->toString(); + } - return oss.str(); + return oss.str(); } //------------------------------------------------------------------------------ @@ -338,20 +315,19 @@ UmSocketSelector::toString() const // Add an IP address to be associated with this UM module. // ip (in) - IP address to associate with this UM module (in network byte order) //------------------------------------------------------------------------------ -void -UmModuleIPs::addIP( in_addr_t ip ) +void UmModuleIPs::addIP(in_addr_t ip) { - boost::mutex::scoped_lock lock( fUmModuleMutex ); + boost::mutex::scoped_lock lock(fUmModuleMutex); #ifdef MOD_CONN_DEBUG - std::ostringstream oss; - char ipString[INET_ADDRSTRLEN]; - oss << " UM " << fUmModuleName << "; adding IP: " << - UmIPSocketConns::nwToString(ip, ipString) << std::endl; - std::cout << oss.str(); + std::ostringstream oss; + char ipString[INET_ADDRSTRLEN]; + oss << " UM " << fUmModuleName << "; adding IP: " << UmIPSocketConns::nwToString(ip, ipString) + << std::endl; + std::cout << oss.str(); #endif - fUmIPSocketConns.push_back( SP_UM_IPCONNS(new UmIPSocketConns(ip)) ); + fUmIPSocketConns.push_back(SP_UM_IPCONNS(new UmIPSocketConns(ip))); } //------------------------------------------------------------------------------ @@ -361,81 +337,73 @@ UmModuleIPs::addIP( in_addr_t ip ) // writeLock (in) - mutex to use when writing to ioSock. // return - boolean indicating whether socket/port connection was added. //------------------------------------------------------------------------------ -bool -UmModuleIPs::addSocketConn( - const SP_UM_IOSOCK& ioSock, - const SP_UM_MUTEX& writeLock ) +bool UmModuleIPs::addSocketConn(const SP_UM_IOSOCK& ioSock, const SP_UM_MUTEX& writeLock) { - bool bConnAdded = false; + bool bConnAdded = false; - boost::mutex::scoped_lock lock( fUmModuleMutex ); + boost::mutex::scoped_lock lock(fUmModuleMutex); - for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i) + for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i) + { + sockaddr sa = ioSock->sa(); + const sockaddr_in* sinp = reinterpret_cast(&sa); + + if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr) { - sockaddr sa = ioSock->sa(); - const sockaddr_in* sinp = reinterpret_cast(&sa); - - if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr) - { - #ifdef MOD_CONN_DEBUG - std::ostringstream oss; - oss << "UM " << fUmModuleName << "; adding connection " << - ioSock->toString() << std::endl; - std::cout << oss.str(); + std::ostringstream oss; + oss << "UM " << fUmModuleName << "; adding connection " << ioSock->toString() << std::endl; + std::cout << oss.str(); #endif - fUmIPSocketConns[i]->addSocketConn ( ioSock, writeLock ); - bConnAdded = true; + fUmIPSocketConns[i]->addSocketConn(ioSock, writeLock); + bConnAdded = true; - //..Initialize fNextUmIPSocketIdx if this is the first socket/port - // connection for this UM. - if ( fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED) - fNextUmIPSocketIdx = i; + //..Initialize fNextUmIPSocketIdx if this is the first socket/port + // connection for this UM. + if (fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED) + fNextUmIPSocketIdx = i; - break; - } + break; } + } - return bConnAdded; + return bConnAdded; } //------------------------------------------------------------------------------ // Delete a socket/port connection from this UM. // ioSock (in) - socket/port connection to be deleted //------------------------------------------------------------------------------ -void -UmModuleIPs::delSocketConn( const messageqcpp::IOSocket& ioSock ) +void UmModuleIPs::delSocketConn(const messageqcpp::IOSocket& ioSock) { - boost::mutex::scoped_lock lock( fUmModuleMutex ); + boost::mutex::scoped_lock lock(fUmModuleMutex); - for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i) + for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i) + { + sockaddr sa = ioSock.sa(); + const sockaddr_in* sinp = reinterpret_cast(&sa); + + if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr) { - sockaddr sa = ioSock.sa(); - const sockaddr_in* sinp = reinterpret_cast(&sa); - - if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr) - { - #ifdef MOD_CONN_DEBUG - std::ostringstream oss; - oss << "UM " << fUmModuleName << "; deleting connection " << - ioSock.toString() << std::endl; - std::cout << oss.str(); + std::ostringstream oss; + oss << "UM " << fUmModuleName << "; deleting connection " << ioSock.toString() << std::endl; + std::cout << oss.str(); #endif - fUmIPSocketConns[i]->delSocketConn ( ioSock ); + fUmIPSocketConns[i]->delSocketConn(ioSock); - //..If we just deleted the last connection for this IP, then ad- - // vance fNextUmIPSocketIdx to an IP address having connections. - if (fUmIPSocketConns[i]->count() == 0) - { - advanceToNextIP(); - } + //..If we just deleted the last connection for this IP, then ad- + // vance fNextUmIPSocketIdx to an IP address having connections. + if (fUmIPSocketConns[i]->count() == 0) + { + advanceToNextIP(); + } - break; - } + break; } + } } //------------------------------------------------------------------------------ @@ -446,23 +414,21 @@ UmModuleIPs::delSocketConn( const messageqcpp::IOSocket& ioSock ) // return - bool indicating if socket/port connection was assigned to // outIos //------------------------------------------------------------------------------ -bool -UmModuleIPs::nextIOSocket( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock ) +bool UmModuleIPs::nextIOSocket(SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock) { - bool found = false; + bool found = false; - boost::mutex::scoped_lock lock( fUmModuleMutex ); + boost::mutex::scoped_lock lock(fUmModuleMutex); - if ((fUmIPSocketConns.size() > 0) && - (fNextUmIPSocketIdx != NEXT_IP_SOCKET_UNASSIGNED)) - { - assert (fNextUmIPSocketIdx < static_cast(fUmIPSocketConns.size())); - fUmIPSocketConns[fNextUmIPSocketIdx]->nextIOSocket( outIos, writeLock ); - advanceToNextIP(); - found = true; - } + if ((fUmIPSocketConns.size() > 0) && (fNextUmIPSocketIdx != NEXT_IP_SOCKET_UNASSIGNED)) + { + assert(fNextUmIPSocketIdx < static_cast(fUmIPSocketConns.size())); + fUmIPSocketConns[fNextUmIPSocketIdx]->nextIOSocket(outIos, writeLock); + advanceToNextIP(); + found = true; + } - return found; + return found; } //------------------------------------------------------------------------------ @@ -470,63 +436,57 @@ UmModuleIPs::nextIOSocket( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock ) // any IP's that have 0 socket/port connections. No mutex locking done, as // the calling methods will have already locked a mutex. //------------------------------------------------------------------------------ -void -UmModuleIPs::advanceToNextIP() +void UmModuleIPs::advanceToNextIP() { - if ( fUmIPSocketConns.size() > 1 ) + if (fUmIPSocketConns.size() > 1) + { + //..Search to end of IP list for an IP having 1 or more connections + for (int i = fNextUmIPSocketIdx + 1; i < static_cast(fUmIPSocketConns.size()); ++i) { - //..Search to end of IP list for an IP having 1 or more connections - for (int i = fNextUmIPSocketIdx + 1; - i < static_cast(fUmIPSocketConns.size()); - ++i) - { - if (fUmIPSocketConns[i]->count() > 0) - { - fNextUmIPSocketIdx = i; - return; - } - } - - //..Wrap back around to the start of the list, to continue the search - for (int i = 0; i < fNextUmIPSocketIdx; ++i) - { - if (fUmIPSocketConns[i]->count() > 0) - { - fNextUmIPSocketIdx = i; - return; - } - } - - fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED; + if (fUmIPSocketConns[i]->count() > 0) + { + fNextUmIPSocketIdx = i; + return; + } } - else // special logic to handle 0 fUmIPSocketConns, or a single empty one + + //..Wrap back around to the start of the list, to continue the search + for (int i = 0; i < fNextUmIPSocketIdx; ++i) { - if ((fUmIPSocketConns.size() == 0) || - (fUmIPSocketConns[0]->count() == 0)) - { - fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED; - } + if (fUmIPSocketConns[i]->count() > 0) + { + fNextUmIPSocketIdx = i; + return; + } } + + fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED; + } + else // special logic to handle 0 fUmIPSocketConns, or a single empty one + { + if ((fUmIPSocketConns.size() == 0) || (fUmIPSocketConns[0]->count() == 0)) + { + fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED; + } + } } //------------------------------------------------------------------------------ // Convert contents to string for logging, debugging, etc. //------------------------------------------------------------------------------ -const std::string -UmModuleIPs::toString() +const std::string UmModuleIPs::toString() { - std::ostringstream oss; + std::ostringstream oss; - boost::mutex::scoped_lock lock( fUmModuleMutex ); - oss << "UM module name: " << fUmModuleName << - "; nextUmIPIdx: " << fNextUmIPSocketIdx << std::endl; + boost::mutex::scoped_lock lock(fUmModuleMutex); + oss << "UM module name: " << fUmModuleName << "; nextUmIPIdx: " << fNextUmIPSocketIdx << std::endl; - for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i) - { - oss << fUmIPSocketConns[i]->toString(); - } + for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i) + { + oss << fUmIPSocketConns[i]->toString(); + } - return oss.str(); + return oss.str(); } //------------------------------------------------------------------------------ @@ -537,20 +497,14 @@ UmModuleIPs::toString() // ioSock (in) - socket/port connection to be added // writeLock (in) - mutex to use when writing to ioSock //------------------------------------------------------------------------------ -void -UmIPSocketConns::addSocketConn( - const SP_UM_IOSOCK& ioSock, - const SP_UM_MUTEX& writeLock ) +void UmIPSocketConns::addSocketConn(const SP_UM_IOSOCK& ioSock, const SP_UM_MUTEX& writeLock) { - UmIOSocketData sockData = - { - SP_UM_IOSOCK(ioSock), SP_UM_MUTEX(writeLock) - }; - fIOSockets.push_back( sockData ); + UmIOSocketData sockData = {SP_UM_IOSOCK(ioSock), SP_UM_MUTEX(writeLock)}; + fIOSockets.push_back(sockData); - //..Initialize fNextIOSocketIdx when we add first connection - if (fIOSockets.size() == 1) - fNextIOSocketIdx = 0; + //..Initialize fNextIOSocketIdx when we add first connection + if (fIOSockets.size() == 1) + fNextIOSocketIdx = 0; } //------------------------------------------------------------------------------ @@ -564,40 +518,39 @@ UmIPSocketConns::addSocketConn( // collection. Plus we want to use a vector over a list, so that nextIOSocket() // can benefit from quick random access. //------------------------------------------------------------------------------ -void -UmIPSocketConns::delSocketConn( const messageqcpp::IOSocket& ioSock ) +void UmIPSocketConns::delSocketConn(const messageqcpp::IOSocket& ioSock) { - for (unsigned int i = 0; i < fIOSockets.size(); ++i) + for (unsigned int i = 0; i < fIOSockets.size(); ++i) + { + sockaddr sa1 = fIOSockets[i].fSock->sa(); + const sockaddr_in* sinp1 = reinterpret_cast(&sa1); + sockaddr sa2 = ioSock.sa(); + const sockaddr_in* sinp2 = reinterpret_cast(&sa2); + + if (sinp1->sin_port == sinp2->sin_port) { - sockaddr sa1 = fIOSockets[i].fSock->sa(); - const sockaddr_in* sinp1 = reinterpret_cast(&sa1); - sockaddr sa2 = ioSock.sa(); - const sockaddr_in* sinp2 = reinterpret_cast(&sa2); + fIOSockets.erase(fIOSockets.begin() + i); - if (sinp1->sin_port == sinp2->sin_port) - { - fIOSockets.erase ( fIOSockets.begin() + i ); + //..Adjust fNextIOSocketIdx + // 1a. decrement if fNextIOSocketIdx is after deleted connection + // 1b. reset to start of vector if we deleted the last connection + // 2. reset fNextIOSocketIdx to -1 if we have no more connections + if (fIOSockets.size() > 0) + { + if (fNextIOSocketIdx > static_cast(i)) + fNextIOSocketIdx--; - //..Adjust fNextIOSocketIdx - // 1a. decrement if fNextIOSocketIdx is after deleted connection - // 1b. reset to start of vector if we deleted the last connection - // 2. reset fNextIOSocketIdx to -1 if we have no more connections - if (fIOSockets.size() > 0) - { - if (fNextIOSocketIdx > static_cast(i)) - fNextIOSocketIdx--; + if (fNextIOSocketIdx >= static_cast(fIOSockets.size())) + fNextIOSocketIdx = 0; + } + else + { + fNextIOSocketIdx = NEXT_IOSOCKET_UNASSIGNED; + } - if ( fNextIOSocketIdx >= static_cast(fIOSockets.size()) ) - fNextIOSocketIdx = 0; - } - else - { - fNextIOSocketIdx = NEXT_IOSOCKET_UNASSIGNED; - } - - break; - } + break; } + } } //------------------------------------------------------------------------------ @@ -606,62 +559,58 @@ UmIPSocketConns::delSocketConn( const messageqcpp::IOSocket& ioSock ) // outIos (out) - socket/port connection to use in sending next msg // writelock (out) - mutex lock to be used when writing to outIos //------------------------------------------------------------------------------ -void -UmIPSocketConns::nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock ) +void UmIPSocketConns::nextIOSocket(SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock) { - assert (fIOSockets.size() > 0); - assert (fNextIOSocketIdx != NEXT_IOSOCKET_UNASSIGNED); - assert (fNextIOSocketIdx < static_cast(fIOSockets.size())); + assert(fIOSockets.size() > 0); + assert(fNextIOSocketIdx != NEXT_IOSOCKET_UNASSIGNED); + assert(fNextIOSocketIdx < static_cast(fIOSockets.size())); - outIos = fIOSockets[fNextIOSocketIdx].fSock; - writeLock = fIOSockets[fNextIOSocketIdx].fMutex; + outIos = fIOSockets[fNextIOSocketIdx].fSock; + writeLock = fIOSockets[fNextIOSocketIdx].fMutex; - //..Update "next" index, being sure to wrap around to the start - // whenever we reach the end of the vector. - fNextIOSocketIdx++; + //..Update "next" index, being sure to wrap around to the start + // whenever we reach the end of the vector. + fNextIOSocketIdx++; - if (fNextIOSocketIdx >= static_cast(fIOSockets.size())) - fNextIOSocketIdx = 0; + if (fNextIOSocketIdx >= static_cast(fIOSockets.size())) + fNextIOSocketIdx = 0; } //------------------------------------------------------------------------------ // Convert network byte ordered IP address to a string. // return - char* is returned with the IP address string. //------------------------------------------------------------------------------ -/* static */ char* -UmIPSocketConns::nwToString( in_addr_t addr, char* ipString ) +/* static */ char* UmIPSocketConns::nwToString(in_addr_t addr, char* ipString) { - in_addr addrStruct = { addr }; + in_addr addrStruct = {addr}; #ifndef _MSC_VER - if (!inet_ntop(AF_INET, &addrStruct, ipString, INET_ADDRSTRLEN)) + if (!inet_ntop(AF_INET, &addrStruct, ipString, INET_ADDRSTRLEN)) #endif - strcpy(ipString, "unknown"); + strcpy(ipString, "unknown"); - return ipString; + return ipString; } //------------------------------------------------------------------------------ // Convert contents to string for logging, debugging, etc. //------------------------------------------------------------------------------ -const std::string -UmIPSocketConns::toString() const +const std::string UmIPSocketConns::toString() const { - std::ostringstream oss; + std::ostringstream oss; - char ipString[INET_ADDRSTRLEN]; - oss << " IPAddress: " << UmIPSocketConns::nwToString(fIpAddress, ipString) << - "; nextIOSocketIdx: " << fNextIOSocketIdx << std::endl; + char ipString[INET_ADDRSTRLEN]; + oss << " IPAddress: " << UmIPSocketConns::nwToString(fIpAddress, ipString) + << "; nextIOSocketIdx: " << fNextIOSocketIdx << std::endl; - for (unsigned int i = 0; i < fIOSockets.size(); ++i) - { - sockaddr sa = fIOSockets[i].fSock->sa(); - const sockaddr_in* sinp = reinterpret_cast(&sa); - oss << " port: " << ntohs(sinp->sin_port) << - std::endl; - } + for (unsigned int i = 0; i < fIOSockets.size(); ++i) + { + sockaddr sa = fIOSockets[i].fSock->sa(); + const sockaddr_in* sinp = reinterpret_cast(&sa); + oss << " port: " << ntohs(sinp->sin_port) << std::endl; + } - return oss.str(); + return oss.str(); } -} // end of primitiveprocessor namespace +} // namespace primitiveprocessor diff --git a/primitives/primproc/umsocketselector.h b/primitives/primproc/umsocketselector.h index c5be55be8..d1af33aff 100644 --- a/primitives/primproc/umsocketselector.h +++ b/primitives/primproc/umsocketselector.h @@ -57,10 +57,10 @@ namespace primitiveprocessor class UmModuleIPs; class UmIPSocketConns; -typedef boost::shared_ptr SP_UM_MODIPS; +typedef boost::shared_ptr SP_UM_MODIPS; typedef boost::shared_ptr SP_UM_IPCONNS; -typedef boost::shared_ptr SP_UM_IOSOCK; -typedef boost::shared_ptr SP_UM_MUTEX; +typedef boost::shared_ptr SP_UM_IOSOCK; +typedef boost::shared_ptr SP_UM_MUTEX; //------------------------------------------------------------------------------ /** @brief Public API class used to track and select socket for outgoing msgs. @@ -79,73 +79,70 @@ typedef boost::shared_ptr SP_UM_MUTEX; //------------------------------------------------------------------------------ class UmSocketSelector { -public: - typedef std::map IpAddressUmMap_t; + public: + typedef std::map IpAddressUmMap_t; - /** @brief Singleton accessor to UmSocketSelector instance. - * - * This method should be called once from the main thread to perform - * initialization from a single threaded environment. - */ - static UmSocketSelector* instance(); + /** @brief Singleton accessor to UmSocketSelector instance. + * + * This method should be called once from the main thread to perform + * initialization from a single threaded environment. + */ + static UmSocketSelector* instance(); - /** @brief UmSocketSelector destructor - * - */ - ~UmSocketSelector() - { }; + /** @brief UmSocketSelector destructor + * + */ + ~UmSocketSelector(){}; - /** @brief Accessor to total number of UM IP's in Columnstore.xml. - * - * @return Number of UM IP addresses read from Columnstore.xml. - */ - uint32_t ipAddressCount() const; + /** @brief Accessor to total number of UM IP's in Columnstore.xml. + * + * @return Number of UM IP addresses read from Columnstore.xml. + */ + uint32_t ipAddressCount() const; - /** @brief Add a socket/port connection to the connection list. - * - * @param ios (in) socket/port connection to be added. - * @param writeLock (in) mutex to use when writing to ios. - * @return boolean indicating if socket/port connection was added. - */ - bool addConnection( const SP_UM_IOSOCK& ios, - const SP_UM_MUTEX& writeLock ); + /** @brief Add a socket/port connection to the connection list. + * + * @param ios (in) socket/port connection to be added. + * @param writeLock (in) mutex to use when writing to ios. + * @return boolean indicating if socket/port connection was added. + */ + bool addConnection(const SP_UM_IOSOCK& ios, const SP_UM_MUTEX& writeLock); - /** @brief Delete a socket/port connection from the connection list. - * - * @param ios (in) socket/port connection to be removed. - */ - void delConnection( const messageqcpp::IOSocket& ios ); + /** @brief Delete a socket/port connection from the connection list. + * + * @param ios (in) socket/port connection to be removed. + */ + void delConnection(const messageqcpp::IOSocket& ios); - /** @brief Get the next output IOSocket to use for the specified ios. - * - * @param ios (in) socket/port connection for the incoming message. - * @param outIos (out) socket/port connection to use for the response. - * @param writeLock (out) mutex to use when writing to outIos. - * @return boolean indicating if operation was successful. - */ - bool nextIOSocket( const messageqcpp::IOSocket& ios, SP_UM_IOSOCK& outIos, - SP_UM_MUTEX& writeLock ); + /** @brief Get the next output IOSocket to use for the specified ios. + * + * @param ios (in) socket/port connection for the incoming message. + * @param outIos (out) socket/port connection to use for the response. + * @param writeLock (out) mutex to use when writing to outIos. + * @return boolean indicating if operation was successful. + */ + bool nextIOSocket(const messageqcpp::IOSocket& ios, SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock); - /** @brief toString method used in logging, debugging, etc. - * - */ - const std::string toString() const; + /** @brief toString method used in logging, debugging, etc. + * + */ + const std::string toString() const; -private: - //...Disable default copy constructor and assignment operator by - // declaring but not defining - UmSocketSelector (const UmSocketSelector& rhs); - UmSocketSelector& operator=(const UmSocketSelector& rhs); + private: + //...Disable default copy constructor and assignment operator by + // declaring but not defining + UmSocketSelector(const UmSocketSelector& rhs); + UmSocketSelector& operator=(const UmSocketSelector& rhs); - UmSocketSelector(); - void loadUMModuleInfo(); - unsigned int findOrAddUm( const std::string& moduleName ); + UmSocketSelector(); + void loadUMModuleInfo(); + unsigned int findOrAddUm(const std::string& moduleName); - static UmSocketSelector* fpUmSocketSelector; - std::vector fUmModuleIPs; // UM's and their sockets + static UmSocketSelector* fpUmSocketSelector; + std::vector fUmModuleIPs; // UM's and their sockets - // std::map that maps an IP address to an index into fUmModuleIPs - IpAddressUmMap_t fIpAddressUmMap; + // std::map that maps an IP address to an index into fUmModuleIPs + IpAddressUmMap_t fIpAddressUmMap; }; //------------------------------------------------------------------------------ @@ -162,84 +159,82 @@ private: //------------------------------------------------------------------------------ class UmModuleIPs { -public: - /** @brief UmModuleIPs constructor. - * - * @param moduleName (in) UM module name for this UmModuleIPs object. - */ - explicit UmModuleIPs ( const std::string& moduleName ) : - fUmModuleName(moduleName), - fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED) - { } + public: + /** @brief UmModuleIPs constructor. + * + * @param moduleName (in) UM module name for this UmModuleIPs object. + */ + explicit UmModuleIPs(const std::string& moduleName) + : fUmModuleName(moduleName), fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED) + { + } - /** @brief UmModuleIPs destructor. - * - */ - ~UmModuleIPs ( ) - { }; + /** @brief UmModuleIPs destructor. + * + */ + ~UmModuleIPs(){}; - /** @brief Accessor to number of IP's from Columnstore.xml for this UM. - * - * @return Number of IP addresses read from Columnstore.xml for this UM. - */ - uint32_t ipAddressCount() const - { - return fUmIPSocketConns.size(); - } + /** @brief Accessor to number of IP's from Columnstore.xml for this UM. + * + * @return Number of IP addresses read from Columnstore.xml for this UM. + */ + uint32_t ipAddressCount() const + { + return fUmIPSocketConns.size(); + } - /** @brief Accessor to the module name for this UmModuleIPs object. - * - * @return UM module name. - */ - const std::string& moduleName() const - { - return fUmModuleName; - } + /** @brief Accessor to the module name for this UmModuleIPs object. + * + * @return UM module name. + */ + const std::string& moduleName() const + { + return fUmModuleName; + } - /** @brief Add an IP address to this UM module. - * - * @param ip (in) IP address to be added (in network byte order) - */ - void addIP ( in_addr_t ip ); + /** @brief Add an IP address to this UM module. + * + * @param ip (in) IP address to be added (in network byte order) + */ + void addIP(in_addr_t ip); - /** @brief Add specified socket/port to the connection list for this UM. - * - * @param ioSock (in) socket/port to add to the connection list. - * @param writeLock (in) mutex to use when writing to ioSock. - * @return boolean indicating if socket/port connection was added. - */ - bool addSocketConn ( const SP_UM_IOSOCK& ioSock, - const SP_UM_MUTEX& writeLock ); + /** @brief Add specified socket/port to the connection list for this UM. + * + * @param ioSock (in) socket/port to add to the connection list. + * @param writeLock (in) mutex to use when writing to ioSock. + * @return boolean indicating if socket/port connection was added. + */ + bool addSocketConn(const SP_UM_IOSOCK& ioSock, const SP_UM_MUTEX& writeLock); - /** @brief Delete specified socket/port from the connection list. - * - * @param ioSock (in) socket/port to delete from the connection list. - */ - void delSocketConn ( const messageqcpp::IOSocket& ioSock ); + /** @brief Delete specified socket/port from the connection list. + * + * @param ioSock (in) socket/port to delete from the connection list. + */ + void delSocketConn(const messageqcpp::IOSocket& ioSock); - /** @brief Get the "next" available socket/port for this UM module. - * - * @param outIos (out) socket/port connection to use for the response. - * @param writeLock (out) mutex to use when writing to outIos. - * @return bool flag indicating whether a socket/port was available. - */ - bool nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock ); + /** @brief Get the "next" available socket/port for this UM module. + * + * @param outIos (out) socket/port connection to use for the response. + * @param writeLock (out) mutex to use when writing to outIos. + * @return bool flag indicating whether a socket/port was available. + */ + bool nextIOSocket(SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock); - /** @brief toString method used in logging, debugging, etc. - * - */ - const std::string toString(); + /** @brief toString method used in logging, debugging, etc. + * + */ + const std::string toString(); -private: - void advanceToNextIP(); + private: + void advanceToNextIP(); - static const int32_t NEXT_IP_SOCKET_UNASSIGNED; - std::string fUmModuleName; // UM module name - int32_t fNextUmIPSocketIdx; //index to "next" IP address - boost::mutex fUmModuleMutex; + static const int32_t NEXT_IP_SOCKET_UNASSIGNED; + std::string fUmModuleName; // UM module name + int32_t fNextUmIPSocketIdx; // index to "next" IP address + boost::mutex fUmModuleMutex; - // collection of IP addresses and their corresponding socket/port conns - std::vector fUmIPSocketConns; + // collection of IP addresses and their corresponding socket/port conns + std::vector fUmIPSocketConns; }; //------------------------------------------------------------------------------ @@ -257,87 +252,86 @@ private: //------------------------------------------------------------------------------ class UmIPSocketConns { -public: - struct UmIOSocketData - { - SP_UM_IOSOCK fSock; // an output IOSocket - SP_UM_MUTEX fMutex; // mutex to be use when writing to fSock - }; + public: + struct UmIOSocketData + { + SP_UM_IOSOCK fSock; // an output IOSocket + SP_UM_MUTEX fMutex; // mutex to be use when writing to fSock + }; - /** @brief UmIPSocketConns constructor. - * - * @param ip (in) IP address for this UmIPSocketConns object. - */ - explicit UmIPSocketConns( in_addr_t ip ) : - fIpAddress(ip), - fNextIOSocketIdx(NEXT_IOSOCKET_UNASSIGNED) - { } + /** @brief UmIPSocketConns constructor. + * + * @param ip (in) IP address for this UmIPSocketConns object. + */ + explicit UmIPSocketConns(in_addr_t ip) : fIpAddress(ip), fNextIOSocketIdx(NEXT_IOSOCKET_UNASSIGNED) + { + } - /** @brief UmIPSocketConns destructor. - * - */ - ~UmIPSocketConns( ) - { } + /** @brief UmIPSocketConns destructor. + * + */ + ~UmIPSocketConns() + { + } - /** @brief Accessor to the IP address for this UmIPSocketConns object. - * - * @return IP address (in network byte order). - */ - in_addr_t ipAddress ( ) - { - return fIpAddress; - } + /** @brief Accessor to the IP address for this UmIPSocketConns object. + * + * @return IP address (in network byte order). + */ + in_addr_t ipAddress() + { + return fIpAddress; + } - /** @brief Accessor to socket/port connection count for this IP address. - * - * @return socket/port connection count. - */ - uint32_t count( ) - { - return fIOSockets.size(); - } + /** @brief Accessor to socket/port connection count for this IP address. + * + * @return socket/port connection count. + */ + uint32_t count() + { + return fIOSockets.size(); + } - /** @brief Add specified socket/port to the connection list. - * - * @param ioSock (in) socket/port to add to the connection list. - * @param writeLock (in) mutex to use when writing to ioSock. - */ - void addSocketConn ( const SP_UM_IOSOCK& ioSock, - const SP_UM_MUTEX& writeLock ); + /** @brief Add specified socket/port to the connection list. + * + * @param ioSock (in) socket/port to add to the connection list. + * @param writeLock (in) mutex to use when writing to ioSock. + */ + void addSocketConn(const SP_UM_IOSOCK& ioSock, const SP_UM_MUTEX& writeLock); - /** @brief Delete specified socket/port from the connection list. - * - * @param ioSock (in) socket/port to delete from the connection list. - */ - void delSocketConn ( const messageqcpp::IOSocket& ioSock ); + /** @brief Delete specified socket/port from the connection list. + * + * @param ioSock (in) socket/port to delete from the connection list. + */ + void delSocketConn(const messageqcpp::IOSocket& ioSock); - /** @brief Get the "next" available socket/port for this IP address. - * - * @param outIos (out) socket/port connection to use for the response. - * @param writeLock (out) mutex to use when writing to outIos. - */ - void nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock ); + /** @brief Get the "next" available socket/port for this IP address. + * + * @param outIos (out) socket/port connection to use for the response. + * @param writeLock (out) mutex to use when writing to outIos. + */ + void nextIOSocket(SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock); - /** @brief Convert network byte ordered IP address to string - * - * @param ipString (out) IP address string; - * ipString must be an array of size INET_ADDRSTRLEN. - * @return IP address string (same as ipString) - */ - static char* nwToString( in_addr_t addr, char* ipString ); + /** @brief Convert network byte ordered IP address to string + * + * @param ipString (out) IP address string; + * ipString must be an array of size INET_ADDRSTRLEN. + * @return IP address string (same as ipString) + */ + static char* nwToString(in_addr_t addr, char* ipString); - /** @brief toString method used in logging, debugging, etc. - * - */ - const std::string toString() const; + /** @brief toString method used in logging, debugging, etc. + * + */ + const std::string toString() const; -private: - static const int32_t NEXT_IOSOCKET_UNASSIGNED; - in_addr_t fIpAddress; // IP address in network byte order - int32_t fNextIOSocketIdx;//index to "next" socket/port - std::vector fIOSockets;//socket/port list for fIpAddress + private: + static const int32_t NEXT_IOSOCKET_UNASSIGNED; + in_addr_t fIpAddress; // IP address in network byte order + int32_t fNextIOSocketIdx; // index to "next" socket/port + std::vector fIOSockets; // socket/port list for fIpAddress }; -} +} // namespace primitiveprocessor #endif diff --git a/storage-manager/include/messageFormat.h b/storage-manager/include/messageFormat.h old mode 100755 new mode 100644 index aefcea4da..23d397e93 --- a/storage-manager/include/messageFormat.h +++ b/storage-manager/include/messageFormat.h @@ -24,79 +24,81 @@ #include #include - namespace storagemanager { - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #pragma pack(push, 1) -struct sm_msg_header { - uint32_t type; // SM_MSG_{START,CONT,END} - uint32_t payloadLen; // refers to the length of what follows the header - uint8_t flags; // see below for valid values +struct sm_msg_header +{ + uint32_t type; // SM_MSG_{START,CONT,END} + uint32_t payloadLen; // refers to the length of what follows the header + uint8_t flags; // see below for valid values }; // current values for the flags field in sm_msg_header -static const uint8_t CONT = 0x1; // this means another message will follow as part of this request or response +static const uint8_t CONT = + 0x1; // this means another message will follow as part of this request or response -struct sm_request { - sm_msg_header header; - uint8_t payload[]; +struct sm_request +{ + sm_msg_header header; + uint8_t payload[]; }; -struct sm_response { - sm_msg_header header; - ssize_t returnCode; // if < 0 it indicates an error, and payload contains a 4-byte errno value - uint8_t payload[]; +struct sm_response +{ + sm_msg_header header; + ssize_t returnCode; // if < 0 it indicates an error, and payload contains a 4-byte errno value + uint8_t payload[]; }; // all msgs to and from StorageManager begin with this magic and a payload length -static const uint32_t SM_MSG_START=0xbf65a7e1; +static const uint32_t SM_MSG_START = 0xbf65a7e1; -// for read/write/append, which may break a message into chunks, messages not the -// beginning or end of the larger message will preface a chunk with this magic -static const uint32_t SM_MSG_CONT=0xfa371bd2; +// for read/write/append, which may break a message into chunks, messages not the +// beginning or end of the larger message will preface a chunk with this magic +static const uint32_t SM_MSG_CONT = 0xfa371bd2; // for read/write/append, the last chunk of a message should begin with this magic -static const uint32_t SM_MSG_END=0x9d5bc31b; +static const uint32_t SM_MSG_END = 0x9d5bc31b; static const uint32_t SM_HEADER_LEN = sizeof(sm_msg_header); // the unix socket StorageManager is listening on -__attribute__ ((unused)) -static const char *socket_name = "\0storagemanager"; +__attribute__((unused)) static const char* socket_name = "\0storagemanager"; #pragma GCC diagnostic pop // opcodes understood by StorageManager. Cast these to // a uint8_t to use them. -enum Opcodes { - OPEN, - READ, - WRITE, - STAT, - UNLINK, - APPEND, - TRUNCATE, - LIST_DIRECTORY, - PING, - COPY, - SYNC +enum Opcodes +{ + OPEN, + READ, + WRITE, + STAT, + UNLINK, + APPEND, + TRUNCATE, + LIST_DIRECTORY, + PING, + COPY, + SYNC }; /* All commands sent to and from StorageManager begin with SM_MSG_START, and a uint32_t for the length of the payload. - + In the payload, all responses from StorageManager begin with a return code, which is the same as the corresponding syscall. If the return code is an error (usually < 0), then a 4-byte value for errno follows. On success, no errno is sent. On success, what follows is any output parameters from the call. - + Note: filenames and pathnames in the following parameters should be absolute rather than relative. */ @@ -110,15 +112,17 @@ enum Opcodes { response format: struct stat */ -struct open_cmd { - uint8_t opcode; // == OPEN - int32_t openmode; - uint32_t flen; - char filename[]; +struct open_cmd +{ + uint8_t opcode; // == OPEN + int32_t openmode; + uint32_t flen; + char filename[]; }; -struct open_resp { - struct stat statbuf; +struct open_resp +{ + struct stat statbuf; }; /* @@ -130,16 +134,17 @@ struct open_resp { response format: data (size is stored in the return code) */ -struct read_cmd { - uint8_t opcode; // == READ - size_t count; - off_t offset; - uint32_t flen; - char filename[]; +struct read_cmd +{ + uint8_t opcode; // == READ + size_t count; + off_t offset; + uint32_t flen; + char filename[]; }; typedef uint8_t read_resp; - + /* WRITE ----- @@ -148,14 +153,15 @@ typedef uint8_t read_resp; response format: */ -struct write_cmd { - uint8_t opcode; // == WRITE - ssize_t count; - off_t offset; - uint32_t flen; - char filename[]; - // after this is the data to be written, ie at &filename[flen] -}; +struct write_cmd +{ + uint8_t opcode; // == WRITE + ssize_t count; + off_t offset; + uint32_t flen; + char filename[]; + // after this is the data to be written, ie at &filename[flen] +}; /* APPEND @@ -165,12 +171,13 @@ struct write_cmd { response format: */ -struct append_cmd { - uint8_t opcode; // == APPEND - ssize_t count; - uint32_t flen; - char filename[]; - // after this is the data to be written, ie at &filename[flen] +struct append_cmd +{ + uint8_t opcode; // == APPEND + ssize_t count; + uint32_t flen; + char filename[]; + // after this is the data to be written, ie at &filename[flen] }; /* @@ -181,10 +188,11 @@ struct append_cmd { response format: */ -struct unlink_cmd { - uint8_t opcode; // == UNLINK - uint32_t flen; - char filename[]; +struct unlink_cmd +{ + uint8_t opcode; // == UNLINK + uint32_t flen; + char filename[]; }; /* @@ -197,30 +205,33 @@ struct unlink_cmd { response format: struct stat */ -struct stat_cmd { - uint8_t opcode; // == STAT - uint32_t flen; - char filename[]; +struct stat_cmd +{ + uint8_t opcode; // == STAT + uint32_t flen; + char filename[]; }; -struct stat_resp { - struct stat statbuf; +struct stat_resp +{ + struct stat statbuf; }; /* - + TRUNCATE -------- command format: 1-byte opcode|off_t length|4-byte filename length|filename - + response format: */ -struct truncate_cmd { - uint8_t opcode; // == TRUNCATE - off_t length; - uint32_t flen; - char filename[]; +struct truncate_cmd +{ + uint8_t opcode; // == TRUNCATE + off_t length; + uint32_t flen; + char filename[]; }; /* @@ -234,21 +245,24 @@ struct truncate_cmd { (4-byte filename length|filename) * num elements */ -struct listdir_cmd { - uint8_t opcode; // == LIST_DIRECTORY - uint32_t plen; - char path[]; +struct listdir_cmd +{ + uint8_t opcode; // == LIST_DIRECTORY + uint32_t plen; + char path[]; }; -struct listdir_resp_entry { - uint32_t flen; - char filename[]; +struct listdir_resp_entry +{ + uint32_t flen; + char filename[]; }; -struct listdir_resp { - uint32_t elements; - listdir_resp_entry entries[]; - // followed by (elements * listdir_resp_entry) +struct listdir_resp +{ + uint32_t elements; + listdir_resp_entry entries[]; + // followed by (elements * listdir_resp_entry) }; /* @@ -260,11 +274,12 @@ struct listdir_resp { response format: nothing yet */ -struct ping_cmd { - uint8_t opcode; +struct ping_cmd +{ + uint8_t opcode; }; -/* +/* COPY ---- command format: @@ -273,20 +288,21 @@ struct ping_cmd { response format: */ -struct f_name { - uint32_t flen; - char filename[]; +struct f_name +{ + uint32_t flen; + char filename[]; }; -struct copy_cmd { - uint8_t opcode; - f_name file1; - // use f_name as an overlay at the end of file1 to get file2. +struct copy_cmd +{ + uint8_t opcode; + f_name file1; + // use f_name as an overlay at the end of file1 to get file2. }; - + #pragma pack(pop) -} +} // namespace storagemanager #endif - diff --git a/storage-manager/src/AppendTask.cpp b/storage-manager/src/AppendTask.cpp index 25a2da94d..d30211524 100644 --- a/storage-manager/src/AppendTask.cpp +++ b/storage-manager/src/AppendTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "AppendTask.h" #include "messageFormat.h" #include "SMLogging.h" @@ -25,7 +24,6 @@ using namespace std; namespace storagemanager { - AppendTask::AppendTask(int sock, uint len) : PosixTask(sock, len) { } @@ -35,86 +33,86 @@ AppendTask::~AppendTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } - + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } + #define min(x, y) (x < y ? x : y) bool AppendTask::run() { - SMLogging* logger = SMLogging::get(); - int success; - uint8_t cmdbuf[1024] = {0}; - ssize_t err; - - success = read(cmdbuf, sizeof(append_cmd)); - check_error("AppendTask read", false); - append_cmd *cmd = (append_cmd *) cmdbuf; - - if (cmd->flen > 1023 - sizeof(*cmd)) - { - handleError("AppendTask", ENAMETOOLONG); - return true; - } - success = read(&cmdbuf[sizeof(*cmd)], cmd->flen); - check_error("AppendTask read", false); - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"append %d bytes to %s.",cmd->count,cmd->filename); - #endif - - ssize_t readCount = 0, writeCount = 0; - vector databuf; - uint bufsize = min(100 << 20, cmd->count); // 100 MB - //uint bufsize = cmd->count; - databuf.resize(bufsize); - - while (readCount < cmd->count) - { - uint toRead = min(static_cast(cmd->count - readCount), bufsize); - success = read(&databuf[0], toRead); - check_error("AppendTask read data", false); - if (success==0) - break; - readCount += success; - uint writePos = 0; + SMLogging* logger = SMLogging::get(); + int success; + uint8_t cmdbuf[1024] = {0}; + ssize_t err; - while (writeCount < readCount) - { - try - { - err = ioc->append(cmd->filename, &databuf[writePos], success - writePos); - } - catch (exception &e) - { - logger->log(LOG_ERR, "AppendTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - if (err <= 0) - break; - writeCount += err; - writePos += err; - } - if (readCount != writeCount) - break; - } - - uint8_t respbuf[sizeof(sm_response) + 4]; - sm_response *resp = (sm_response *) respbuf; - uint payloadLen = 0; - if (cmd->count != 0 && writeCount == 0) + success = read(cmdbuf, sizeof(append_cmd)); + check_error("AppendTask read", false); + append_cmd* cmd = (append_cmd*)cmdbuf; + + if (cmd->flen > 1023 - sizeof(*cmd)) + { + handleError("AppendTask", ENAMETOOLONG); + return true; + } + success = read(&cmdbuf[sizeof(*cmd)], cmd->flen); + check_error("AppendTask read", false); + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "append %d bytes to %s.", cmd->count, cmd->filename); +#endif + + ssize_t readCount = 0, writeCount = 0; + vector databuf; + uint bufsize = min(100 << 20, cmd->count); // 100 MB + // uint bufsize = cmd->count; + databuf.resize(bufsize); + + while (readCount < cmd->count) + { + uint toRead = min(static_cast(cmd->count - readCount), bufsize); + success = read(&databuf[0], toRead); + check_error("AppendTask read data", false); + if (success == 0) + break; + readCount += success; + uint writePos = 0; + + while (writeCount < readCount) { - payloadLen = 4; - resp->returnCode = -1; - *((int *) &resp[1]) = errno; + try + { + err = ioc->append(cmd->filename, &databuf[writePos], success - writePos); + } + catch (exception& e) + { + logger->log(LOG_ERR, "AppendTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + if (err <= 0) + break; + writeCount += err; + writePos += err; } - else - resp->returnCode = writeCount; - return write(*resp, payloadLen); + if (readCount != writeCount) + break; + } + + uint8_t respbuf[sizeof(sm_response) + 4]; + sm_response* resp = (sm_response*)respbuf; + uint payloadLen = 0; + if (cmd->count != 0 && writeCount == 0) + { + payloadLen = 4; + resp->returnCode = -1; + *((int*)&resp[1]) = errno; + } + else + resp->returnCode = writeCount; + return write(*resp, payloadLen); } -} +} // namespace storagemanager diff --git a/storage-manager/src/AppendTask.h b/storage-manager/src/AppendTask.h index 4ab9d3c51..50c083cd4 100644 --- a/storage-manager/src/AppendTask.h +++ b/storage-manager/src/AppendTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef APPENDTASK_H_ #define APPENDTASK_H_ @@ -24,18 +22,17 @@ namespace storagemanager { - class AppendTask : public PosixTask { - public: - AppendTask(int sock, uint length); - virtual ~AppendTask(); - - bool run(); - - private: - AppendTask(); + public: + AppendTask(int sock, uint length); + virtual ~AppendTask(); + + bool run(); + + private: + AppendTask(); }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Cache.cpp b/storage-manager/src/Cache.cpp index 05a70adf3..296723e9c 100644 --- a/storage-manager/src/Cache.cpp +++ b/storage-manager/src/Cache.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "Cache.h" #include "Config.h" #include "Downloader.h" @@ -33,330 +32,328 @@ namespace bf = boost::filesystem; namespace { - boost::mutex m; - storagemanager::Cache *inst = NULL; -} +boost::mutex m; +storagemanager::Cache* inst = NULL; +} // namespace namespace storagemanager { - -Cache * Cache::get() +Cache* Cache::get() { - if (inst) - return inst; - boost::unique_lock s(m); - if (inst) - return inst; - inst = new Cache(); + if (inst) return inst; + boost::unique_lock s(m); + if (inst) + return inst; + inst = new Cache(); + return inst; } Cache::Cache() { - Config *conf = Config::get(); - logger = SMLogging::get(); - - configListener(); - conf->addConfigListener(this); - //cout << "Cache got cache size " << maxCacheSize << endl; - - string stmp = conf->getValue("ObjectStorage", "object_size"); - if (stmp.empty()) - { - logger->log(LOG_CRIT, "ObjectStorage/object_size is not set"); - throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file"); - } - try - { - objectSize = stoul(stmp); - } - catch (invalid_argument &) - { - logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number"); - throw runtime_error("Please set ObjectStorage/object_size to a number"); - } - - cachePrefix = conf->getValue("Cache", "path"); - if (cachePrefix.empty()) - { - logger->log(LOG_CRIT, "Cache/path is not set"); - throw runtime_error("Please set Cache/path in the storagemanager.cnf file"); - } - - try - { - bf::create_directories(cachePrefix); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what()); - throw e; - } - //cout << "Cache got cachePrefix " << cachePrefix << endl; - downloader.reset(new Downloader()); - - stmp = conf->getValue("ObjectStorage", "journal_path"); - if (stmp.empty()) - { - logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); - throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); - } - journalPrefix = stmp; + Config* conf = Config::get(); + logger = SMLogging::get(); + + configListener(); + conf->addConfigListener(this); + // cout << "Cache got cache size " << maxCacheSize << endl; + + string stmp = conf->getValue("ObjectStorage", "object_size"); + if (stmp.empty()) + { + logger->log(LOG_CRIT, "ObjectStorage/object_size is not set"); + throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file"); + } + try + { + objectSize = stoul(stmp); + } + catch (invalid_argument&) + { + logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number"); + throw runtime_error("Please set ObjectStorage/object_size to a number"); + } + + cachePrefix = conf->getValue("Cache", "path"); + if (cachePrefix.empty()) + { + logger->log(LOG_CRIT, "Cache/path is not set"); + throw runtime_error("Please set Cache/path in the storagemanager.cnf file"); + } + + try + { + bf::create_directories(cachePrefix); + } + catch (exception& e) + { + logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what()); + throw e; + } + // cout << "Cache got cachePrefix " << cachePrefix << endl; + downloader.reset(new Downloader()); + + stmp = conf->getValue("ObjectStorage", "journal_path"); + if (stmp.empty()) + { + logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); + throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); + } + journalPrefix = stmp; + bf::create_directories(journalPrefix); + try + { bf::create_directories(journalPrefix); - try - { - bf::create_directories(journalPrefix); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what()); - throw e; - } + } + catch (exception& e) + { + logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what()); + throw e; + } } Cache::~Cache() { - Config::get()->removeConfigListener(this); - for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) - delete it->second; + Config::get()->removeConfigListener(this); + for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) + delete it->second; } // be careful using this! SM should be idle. No ongoing reads or writes. void Cache::validateCacheSize() { - boost::unique_lock s(lru_mutex); - - for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) - it->second->validateCacheSize(); + boost::unique_lock s(lru_mutex); + + for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) + it->second->validateCacheSize(); } // new simplified version -void Cache::read(const bf::path &prefix, const vector &keys) +void Cache::read(const bf::path& prefix, const vector& keys) { - getPCache(prefix).read(keys); + getPCache(prefix).read(keys); } -void Cache::doneReading(const bf::path &prefix, const vector &keys) +void Cache::doneReading(const bf::path& prefix, const vector& keys) { - getPCache(prefix).doneReading(keys); + getPCache(prefix).doneReading(keys); } -void Cache::doneWriting(const bf::path &prefix) +void Cache::doneWriting(const bf::path& prefix) { - getPCache(prefix).makeSpace(0); + getPCache(prefix).makeSpace(0); } -const bf::path & Cache::getCachePath() const +const bf::path& Cache::getCachePath() const { - return cachePrefix; + return cachePrefix; } -const bf::path & Cache::getJournalPath() const +const bf::path& Cache::getJournalPath() const { - return journalPrefix; + return journalPrefix; } -const bf::path Cache::getCachePath(const bf::path &prefix) const +const bf::path Cache::getCachePath(const bf::path& prefix) const { - return cachePrefix/prefix; + return cachePrefix / prefix; } -const bf::path Cache::getJournalPath(const bf::path &prefix) const +const bf::path Cache::getJournalPath(const bf::path& prefix) const { - return journalPrefix/prefix; -} - -void Cache::exists(const bf::path &prefix, const vector &keys, vector *out) -{ - getPCache(prefix).exists(keys, out); + return journalPrefix / prefix; } -bool Cache::exists(const bf::path &prefix, const string &key) +void Cache::exists(const bf::path& prefix, const vector& keys, vector* out) { - return getPCache(prefix).exists(key); + getPCache(prefix).exists(keys, out); } -void Cache::newObject(const bf::path &prefix, const string &key, size_t size) +bool Cache::exists(const bf::path& prefix, const string& key) { - getPCache(prefix).newObject(key, size); + return getPCache(prefix).exists(key); } -void Cache::newJournalEntry(const bf::path &prefix, size_t size) +void Cache::newObject(const bf::path& prefix, const string& key, size_t size) { - getPCache(prefix).newJournalEntry(size); + getPCache(prefix).newObject(key, size); } -void Cache::deletedJournal(const bf::path &prefix, size_t size) +void Cache::newJournalEntry(const bf::path& prefix, size_t size) { - getPCache(prefix).deletedJournal(size); + getPCache(prefix).newJournalEntry(size); } -void Cache::deletedObject(const bf::path &prefix, const string &key, size_t size) +void Cache::deletedJournal(const bf::path& prefix, size_t size) { - getPCache(prefix).deletedObject(key, size); + getPCache(prefix).deletedJournal(size); +} + +void Cache::deletedObject(const bf::path& prefix, const string& key, size_t size) +{ + getPCache(prefix).deletedObject(key, size); } void Cache::setMaxCacheSize(size_t size) { - boost::unique_lock s(lru_mutex); - - maxCacheSize = size; - for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) - it->second->setMaxCacheSize(size); + boost::unique_lock s(lru_mutex); + + maxCacheSize = size; + for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) + it->second->setMaxCacheSize(size); } -void Cache::makeSpace(const bf::path &prefix, size_t size) +void Cache::makeSpace(const bf::path& prefix, size_t size) { - getPCache(prefix).makeSpace(size); + getPCache(prefix).makeSpace(size); } size_t Cache::getMaxCacheSize() const { - return maxCacheSize; + return maxCacheSize; } -void Cache::rename(const bf::path &prefix, const string &oldKey, const string &newKey, ssize_t sizediff) +void Cache::rename(const bf::path& prefix, const string& oldKey, const string& newKey, ssize_t sizediff) { - getPCache(prefix).rename(oldKey, newKey, sizediff); + getPCache(prefix).rename(oldKey, newKey, sizediff); } -int Cache::ifExistsThenDelete(const bf::path &prefix, const string &key) +int Cache::ifExistsThenDelete(const bf::path& prefix, const string& key) { - return getPCache(prefix).ifExistsThenDelete(key); + return getPCache(prefix).ifExistsThenDelete(key); } void Cache::printKPIs() const { - downloader->printKPIs(); + downloader->printKPIs(); } size_t Cache::getCurrentCacheSize() { - size_t totalSize = 0; + size_t totalSize = 0; - boost::unique_lock s(lru_mutex); - - for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) - totalSize += it->second->getCurrentCacheSize(); - return totalSize; + boost::unique_lock s(lru_mutex); + + for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) + totalSize += it->second->getCurrentCacheSize(); + return totalSize; } size_t Cache::getCurrentCacheElementCount() { - size_t totalCount = 0; + size_t totalCount = 0; - for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) - totalCount += it->second->getCurrentCacheElementCount(); - return totalCount; + for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) + totalCount += it->second->getCurrentCacheElementCount(); + return totalCount; } -size_t Cache::getCurrentCacheSize(const bf::path &prefix) +size_t Cache::getCurrentCacheSize(const bf::path& prefix) { - return getPCache(prefix).getCurrentCacheSize(); + return getPCache(prefix).getCurrentCacheSize(); } -size_t Cache::getCurrentCacheElementCount(const bf::path &prefix) +size_t Cache::getCurrentCacheElementCount(const bf::path& prefix) { - return getPCache(prefix).getCurrentCacheElementCount(); + return getPCache(prefix).getCurrentCacheElementCount(); } void Cache::reset() { - boost::unique_lock s(lru_mutex); - - for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) - it->second->reset(); + boost::unique_lock s(lru_mutex); + + for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it) + it->second->reset(); } -void Cache::newPrefix(const bf::path &prefix) +void Cache::newPrefix(const bf::path& prefix) { - boost::unique_lock s(lru_mutex); - - //cerr << "Cache: making new prefix " << prefix.string() << endl; - assert(prefixCaches.find(prefix) == prefixCaches.end()); - prefixCaches[prefix] = NULL; + boost::unique_lock s(lru_mutex); + + // cerr << "Cache: making new prefix " << prefix.string() << endl; + assert(prefixCaches.find(prefix) == prefixCaches.end()); + prefixCaches[prefix] = NULL; + s.unlock(); + PrefixCache* pCache = new PrefixCache(prefix); + s.lock(); + prefixCaches[prefix] = pCache; +} + +void Cache::dropPrefix(const bf::path& prefix) +{ + boost::unique_lock s(lru_mutex); + + auto* pCache = prefixCaches[prefix]; + prefixCaches.erase(prefix); + s.unlock(); + delete pCache; +} + +inline PrefixCache& Cache::getPCache(const bf::path& prefix) +{ + boost::unique_lock s(lru_mutex); + + // cerr << "Getting pcache for " << prefix.string() << endl; + PrefixCache* ret; + auto it = prefixCaches.find(prefix); + assert(it != prefixCaches.end()); + + ret = it->second; + while (ret == NULL) // wait for the new PC to init + { s.unlock(); - PrefixCache *pCache = new PrefixCache(prefix); + sleep(1); s.lock(); - prefixCaches[prefix] = pCache; + ret = prefixCaches[prefix]; + } + + return *ret; } -void Cache::dropPrefix(const bf::path &prefix) +Downloader* Cache::getDownloader() const { - boost::unique_lock s(lru_mutex); - - auto *pCache = prefixCaches[prefix]; - prefixCaches.erase(prefix); - s.unlock(); - delete pCache; -} - -inline PrefixCache & Cache::getPCache(const bf::path &prefix) -{ - boost::unique_lock s(lru_mutex); - - //cerr << "Getting pcache for " << prefix.string() << endl; - PrefixCache *ret; - auto it = prefixCaches.find(prefix); - assert(it != prefixCaches.end()); - - ret = it->second; - while (ret == NULL) // wait for the new PC to init - { - s.unlock(); - sleep(1); - s.lock(); - ret = prefixCaches[prefix]; - } - - return *ret; -} - -Downloader * Cache::getDownloader() const -{ - return downloader.get(); + return downloader.get(); } void Cache::shutdown() { - boost::unique_lock s(lru_mutex); - downloader.reset(); + boost::unique_lock s(lru_mutex); + downloader.reset(); } void Cache::configListener() { - Config *conf = Config::get(); - SMLogging* logger = SMLogging::get(); + Config* conf = Config::get(); + SMLogging* logger = SMLogging::get(); - if (maxCacheSize == 0) - maxCacheSize = 2147483648; - string stmp = conf->getValue("Cache", "cache_size"); - if (stmp.empty()) + if (maxCacheSize == 0) + maxCacheSize = 2147483648; + string stmp = conf->getValue("Cache", "cache_size"); + if (stmp.empty()) + { + logger->log(LOG_CRIT, "Cache/cache_size is not set. Using current value = %zi", maxCacheSize); + } + try + { + size_t newMaxCacheSize = stoull(stmp); + if (newMaxCacheSize != maxCacheSize) { - logger->log(LOG_CRIT, "Cache/cache_size is not set. Using current value = %zi",maxCacheSize); - } - try - { - size_t newMaxCacheSize = stoull(stmp); - if (newMaxCacheSize != maxCacheSize) - { - if (newMaxCacheSize >= MIN_CACHE_SIZE) - { - setMaxCacheSize(newMaxCacheSize); - logger->log(LOG_INFO, "Cache/cache_size = %zi",maxCacheSize); - } - else - { - logger->log(LOG_CRIT, "Cache/cache_size is below %u. Check value and suffix are correct in configuration file. Using current value = %zi",MIN_CACHE_SIZE,maxCacheSize); - } - } - } - catch (invalid_argument &) - { - logger->log(LOG_CRIT, "Cache/cache_size is not a number. Using current value = %zi",maxCacheSize); + if (newMaxCacheSize >= MIN_CACHE_SIZE) + { + setMaxCacheSize(newMaxCacheSize); + logger->log(LOG_INFO, "Cache/cache_size = %zi", maxCacheSize); + } + else + { + logger->log(LOG_CRIT, + "Cache/cache_size is below %u. Check value and suffix are correct in configuration file. " + "Using current value = %zi", + MIN_CACHE_SIZE, maxCacheSize); + } } + } + catch (invalid_argument&) + { + logger->log(LOG_CRIT, "Cache/cache_size is not a number. Using current value = %zi", maxCacheSize); + } } -} - - - - +} // namespace storagemanager diff --git a/storage-manager/src/Cache.h b/storage-manager/src/Cache.h index 12396550a..982e58945 100644 --- a/storage-manager/src/Cache.h +++ b/storage-manager/src/Cache.h @@ -18,12 +18,11 @@ #ifndef CACHE_H_ #define CACHE_H_ -/* PrefixCache manages the cache for one prefix managed by SM. +/* PrefixCache manages the cache for one prefix managed by SM. Cache is a map of prefix -> Cache, and holds the items that should be centralized like the Downloader */ - #include "Downloader.h" #include "SMLogging.h" #include "PrefixCache.h" @@ -44,82 +43,80 @@ namespace storagemanager { - -class Cache : public boost::noncopyable , public ConfigListener +class Cache : public boost::noncopyable, public ConfigListener { - public: - static Cache *get(); - virtual ~Cache(); - - //reading fcns - // read() marks objects to be read s.t. they do not get flushed. - // after reading them, unlock the 'logical file', and call doneReading(). - void read(const boost::filesystem::path &prefix, const std::vector &keys); - void doneReading(const boost::filesystem::path &prefix, const std::vector &keys); - bool exists(const boost::filesystem::path &prefix, const std::string &key); - void exists(const boost::filesystem::path &prefix, const std::vector &keys, - std::vector *out); - - // writing fcns - // new*() fcns tell the Cache data was added. After writing a set of objects, - // unlock the 'logical file', and call doneWriting(). - void newObject(const boost::filesystem::path &prefix, const std::string &key, size_t size); - void newJournalEntry(const boost::filesystem::path &prefix, size_t size); - void doneWriting(const boost::filesystem::path &prefix); - void deletedObject(const boost::filesystem::path &prefix, const std::string &key, size_t size); - void deletedJournal(const boost::filesystem::path &prefix, size_t size); - - // an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files. - // returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both exist - // This should be called while holding the file lock for key because it touches the journal file. - int ifExistsThenDelete(const boost::filesystem::path &prefix, const std::string &key); - - // rename is used when an old obj gets merged with its journal file - // the size will change in that process; sizediff is by how much - void rename(const boost::filesystem::path &prefix, const std::string &oldKey, - const std::string &newKey, ssize_t sizediff); - void setMaxCacheSize(size_t size); - void makeSpace(const boost::filesystem::path &prefix, size_t size); - size_t getCurrentCacheSize(); - size_t getCurrentCacheElementCount(); - size_t getCurrentCacheSize(const boost::filesystem::path &prefix); - size_t getCurrentCacheElementCount(const boost::filesystem::path &prefix); - size_t getMaxCacheSize() const; - - Downloader * getDownloader() const; - void newPrefix(const boost::filesystem::path &prefix); - void dropPrefix(const boost::filesystem::path &prefix); - void shutdown(); - void printKPIs() const; - - // test helpers - const boost::filesystem::path &getCachePath() const; - const boost::filesystem::path &getJournalPath() const; - const boost::filesystem::path getCachePath(const boost::filesystem::path &prefix) const; - const boost::filesystem::path getJournalPath(const boost::filesystem::path &prefix) const; - // this will delete everything in the Cache and journal paths, and empty all Cache structures. - void reset(); - void validateCacheSize(); - - virtual void configListener() override; - private: - Cache(); - - SMLogging *logger; - boost::filesystem::path cachePrefix; - boost::filesystem::path journalPrefix; - size_t maxCacheSize; - size_t objectSize; - boost::scoped_ptr downloader; - - PrefixCache & getPCache(const boost::filesystem::path &prefix); + public: + static Cache* get(); + virtual ~Cache(); - std::map prefixCaches; - mutable boost::mutex lru_mutex; // protects the prefixCaches + // reading fcns + // read() marks objects to be read s.t. they do not get flushed. + // after reading them, unlock the 'logical file', and call doneReading(). + void read(const boost::filesystem::path& prefix, const std::vector& keys); + void doneReading(const boost::filesystem::path& prefix, const std::vector& keys); + bool exists(const boost::filesystem::path& prefix, const std::string& key); + void exists(const boost::filesystem::path& prefix, const std::vector& keys, + std::vector* out); + + // writing fcns + // new*() fcns tell the Cache data was added. After writing a set of objects, + // unlock the 'logical file', and call doneWriting(). + void newObject(const boost::filesystem::path& prefix, const std::string& key, size_t size); + void newJournalEntry(const boost::filesystem::path& prefix, size_t size); + void doneWriting(const boost::filesystem::path& prefix); + void deletedObject(const boost::filesystem::path& prefix, const std::string& key, size_t size); + void deletedJournal(const boost::filesystem::path& prefix, size_t size); + + // an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files. + // returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both + // exist This should be called while holding the file lock for key because it touches the journal file. + int ifExistsThenDelete(const boost::filesystem::path& prefix, const std::string& key); + + // rename is used when an old obj gets merged with its journal file + // the size will change in that process; sizediff is by how much + void rename(const boost::filesystem::path& prefix, const std::string& oldKey, const std::string& newKey, + ssize_t sizediff); + void setMaxCacheSize(size_t size); + void makeSpace(const boost::filesystem::path& prefix, size_t size); + size_t getCurrentCacheSize(); + size_t getCurrentCacheElementCount(); + size_t getCurrentCacheSize(const boost::filesystem::path& prefix); + size_t getCurrentCacheElementCount(const boost::filesystem::path& prefix); + size_t getMaxCacheSize() const; + + Downloader* getDownloader() const; + void newPrefix(const boost::filesystem::path& prefix); + void dropPrefix(const boost::filesystem::path& prefix); + void shutdown(); + void printKPIs() const; + + // test helpers + const boost::filesystem::path& getCachePath() const; + const boost::filesystem::path& getJournalPath() const; + const boost::filesystem::path getCachePath(const boost::filesystem::path& prefix) const; + const boost::filesystem::path getJournalPath(const boost::filesystem::path& prefix) const; + // this will delete everything in the Cache and journal paths, and empty all Cache structures. + void reset(); + void validateCacheSize(); + + virtual void configListener() override; + + private: + Cache(); + + SMLogging* logger; + boost::filesystem::path cachePrefix; + boost::filesystem::path journalPrefix; + size_t maxCacheSize; + size_t objectSize; + boost::scoped_ptr downloader; + + PrefixCache& getPCache(const boost::filesystem::path& prefix); + + std::map prefixCaches; + mutable boost::mutex lru_mutex; // protects the prefixCaches }; - - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/ClientRequestProcessor.cpp b/storage-manager/src/ClientRequestProcessor.cpp index 83e3918e4..3f7edcd24 100644 --- a/storage-manager/src/ClientRequestProcessor.cpp +++ b/storage-manager/src/ClientRequestProcessor.cpp @@ -15,20 +15,18 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "ClientRequestProcessor.h" #include "ProcessTask.h" #include namespace { -storagemanager::ClientRequestProcessor *crp = NULL; +storagemanager::ClientRequestProcessor* crp = NULL; boost::mutex m; -}; +}; // namespace namespace storagemanager { - ClientRequestProcessor::ClientRequestProcessor() { } @@ -37,28 +35,26 @@ ClientRequestProcessor::~ClientRequestProcessor() { } -ClientRequestProcessor * ClientRequestProcessor::get() +ClientRequestProcessor* ClientRequestProcessor::get() { - if (crp) - return crp; - boost::mutex::scoped_lock s(m); - if (crp) - return crp; - crp = new ClientRequestProcessor(); + if (crp) return crp; + boost::mutex::scoped_lock s(m); + if (crp) + return crp; + crp = new ClientRequestProcessor(); + return crp; } void ClientRequestProcessor::processRequest(int sock, uint len) { - boost::shared_ptr t(new ProcessTask(sock, len)); - threadPool.addJob(t); + boost::shared_ptr t(new ProcessTask(sock, len)); + threadPool.addJob(t); } void ClientRequestProcessor::shutdown() { - delete crp; + delete crp; } -} - - +} // namespace storagemanager diff --git a/storage-manager/src/ClientRequestProcessor.h b/storage-manager/src/ClientRequestProcessor.h index c4566d36b..c91d72e38 100644 --- a/storage-manager/src/ClientRequestProcessor.h +++ b/storage-manager/src/ClientRequestProcessor.h @@ -15,31 +15,28 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef CLIENTREQUESTPROCESSOR_H_ #define CLIENTREQUESTPROCESSOR_H_ #include "ThreadPool.h" #include - namespace storagemanager { - class ClientRequestProcessor : public boost::noncopyable { - public: - static ClientRequestProcessor *get(); - virtual ~ClientRequestProcessor(); + public: + static ClientRequestProcessor* get(); + virtual ~ClientRequestProcessor(); - void processRequest(int sock, uint len); - void shutdown(); + void processRequest(int sock, uint len); + void shutdown(); - private: - ClientRequestProcessor(); - ThreadPool threadPool; + private: + ClientRequestProcessor(); + ThreadPool threadPool; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/CloudStorage.cpp b/storage-manager/src/CloudStorage.cpp index 327b47520..8e2465583 100644 --- a/storage-manager/src/CloudStorage.cpp +++ b/storage-manager/src/CloudStorage.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "CloudStorage.h" #include "Config.h" #include "S3Storage.h" @@ -30,66 +29,63 @@ using namespace std; namespace { - boost::mutex m; - storagemanager::CloudStorage *inst = NULL; - -string tolower(const string &s) +boost::mutex m; +storagemanager::CloudStorage* inst = NULL; + +string tolower(const string& s) { - string ret(s); - for (uint i = 0; i < ret.length(); i++) - ret[i] = ::tolower(ret[i]); - return ret; + string ret(s); + for (uint i = 0; i < ret.length(); i++) + ret[i] = ::tolower(ret[i]); + return ret; } -} +} // namespace namespace storagemanager { - -CloudStorage * CloudStorage::get() +CloudStorage* CloudStorage::get() { - if (inst) - return inst; - - SMLogging* logger = SMLogging::get(); - Config *conf = Config::get(); - string type = tolower(conf->getValue("ObjectStorage", "service")); - boost::mutex::scoped_lock s(m); - if (inst) - return inst; - if (type == "s3") - inst = new S3Storage(); - else if (type == "local" || type == "localstorage") - inst = new LocalStorage(); - else { - logger->log(LOG_CRIT,"CloudStorage: got unknown service provider: %s", type.c_str()); - throw runtime_error("CloudStorage: got unknown service provider"); - } - + if (inst) return inst; + + SMLogging* logger = SMLogging::get(); + Config* conf = Config::get(); + string type = tolower(conf->getValue("ObjectStorage", "service")); + boost::mutex::scoped_lock s(m); + if (inst) + return inst; + if (type == "s3") + inst = new S3Storage(); + else if (type == "local" || type == "localstorage") + inst = new LocalStorage(); + else + { + logger->log(LOG_CRIT, "CloudStorage: got unknown service provider: %s", type.c_str()); + throw runtime_error("CloudStorage: got unknown service provider"); + } + + return inst; } CloudStorage::CloudStorage() { - logger = SMLogging::get(); - - bytesUploaded = bytesDownloaded = objectsDeleted = objectsCopied = objectsGotten = objectsPut = - existenceChecks = 0; + logger = SMLogging::get(); + + bytesUploaded = bytesDownloaded = objectsDeleted = objectsCopied = objectsGotten = objectsPut = + existenceChecks = 0; } void CloudStorage::printKPIs() const { - cout << "CloudStorage" << endl; - cout << "\tbytesUplaoded = " << bytesUploaded << endl; - cout << "\tbytesDownloaded = " << bytesDownloaded << endl; - cout << "\tobjectsDeleted = " << objectsDeleted << endl; - cout << "\tobjectsCopied = " << objectsCopied << endl; - cout << "\tobjectsGotten = " << objectsGotten << endl; - cout << "\tobjectsPut = " << objectsPut << endl; - cout << "\texistenceChecks = " << existenceChecks << endl; + cout << "CloudStorage" << endl; + cout << "\tbytesUplaoded = " << bytesUploaded << endl; + cout << "\tbytesDownloaded = " << bytesDownloaded << endl; + cout << "\tobjectsDeleted = " << objectsDeleted << endl; + cout << "\tobjectsCopied = " << objectsCopied << endl; + cout << "\tobjectsGotten = " << objectsGotten << endl; + cout << "\tobjectsPut = " << objectsPut << endl; + cout << "\texistenceChecks = " << existenceChecks << endl; } - - - -} +} // namespace storagemanager diff --git a/storage-manager/src/CloudStorage.h b/storage-manager/src/CloudStorage.h index a844fd1e1..423eb9fde 100644 --- a/storage-manager/src/CloudStorage.h +++ b/storage-manager/src/CloudStorage.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef CLOUDSTORAGE_H_ #define CLOUDSTORAGE_H_ @@ -25,37 +24,36 @@ namespace storagemanager { - class CloudStorage { - public: - virtual ~CloudStorage(){}; - /* These behave like syscalls. return code -1 means an error, and errno is set */ - virtual int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL) = 0; - virtual int getObject(const std::string &sourceKey, boost::shared_array *data, size_t *size = NULL) = 0; - virtual int putObject(const std::string &sourceFile, const std::string &destKey) = 0; - virtual int putObject(const boost::shared_array data, size_t len, const std::string &destKey) = 0; - virtual int deleteObject(const std::string &key) = 0; - virtual int copyObject(const std::string &sourceKey, const std::string &destKey) = 0; - virtual int exists(const std::string &key, bool *out) = 0; - - virtual void printKPIs() const; - - // this will return a CloudStorage instance of the type specified in StorageManager.cnf - static CloudStorage *get(); - - protected: - SMLogging *logger; - CloudStorage(); - - // some KPIs - size_t bytesUploaded, bytesDownloaded, objectsDeleted, objectsCopied, objectsGotten, - objectsPut, existenceChecks; - - private: - + public: + virtual ~CloudStorage(){}; + /* These behave like syscalls. return code -1 means an error, and errno is set */ + virtual int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL) = 0; + virtual int getObject(const std::string& sourceKey, boost::shared_array* data, + size_t* size = NULL) = 0; + virtual int putObject(const std::string& sourceFile, const std::string& destKey) = 0; + virtual int putObject(const boost::shared_array data, size_t len, const std::string& destKey) = 0; + virtual int deleteObject(const std::string& key) = 0; + virtual int copyObject(const std::string& sourceKey, const std::string& destKey) = 0; + virtual int exists(const std::string& key, bool* out) = 0; + + virtual void printKPIs() const; + + // this will return a CloudStorage instance of the type specified in StorageManager.cnf + static CloudStorage* get(); + + protected: + SMLogging* logger; + CloudStorage(); + + // some KPIs + size_t bytesUploaded, bytesDownloaded, objectsDeleted, objectsCopied, objectsGotten, objectsPut, + existenceChecks; + + private: }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Config.cpp b/storage-manager/src/Config.cpp index 2238e0a2d..f634ececb 100644 --- a/storage-manager/src/Config.cpp +++ b/storage-manager/src/Config.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "Config.h" // This one is the build system config @@ -40,191 +39,193 @@ using namespace std; namespace { - boost::mutex m; - storagemanager::Config *inst = NULL; -} - +boost::mutex m; +storagemanager::Config* inst = NULL; +} // namespace + namespace storagemanager { - -Config * Config::get() +Config* Config::get() { - if (inst) - return inst; - boost::mutex::scoped_lock s(m); - if (inst) - return inst; - inst = new Config(); + if (inst) return inst; + boost::mutex::scoped_lock s(m); + if (inst) + return inst; + inst = new Config(); + return inst; } -Config * Config::get(const string &configFile) +Config* Config::get(const string& configFile) { - if (inst) - return inst; - boost::mutex::scoped_lock s(m); - if (inst) - return inst; - inst = new Config(configFile); + if (inst) return inst; + boost::mutex::scoped_lock s(m); + if (inst) + return inst; + inst = new Config(configFile); + return inst; } Config::Config() : die(false) { - /* This will search the current directory, - then $COLUMNSTORE_INSTALL_DIR/etc, - then /etc - looking for storagemanager.cnf - - We can change this however we need later. - */ - //char *cs_install_dir = getenv("COLUMNSTORE_INSTALL_DIR"); - - vector paths; - - // the paths to search in order - paths.push_back("."); - paths.push_back(string(MCSSYSCONFDIR) + "/columnstore"); - paths.push_back("/etc"); - - for (uint i = 0; i < paths.size(); i++) + /* This will search the current directory, + then $COLUMNSTORE_INSTALL_DIR/etc, + then /etc + looking for storagemanager.cnf + + We can change this however we need later. + */ + // char *cs_install_dir = getenv("COLUMNSTORE_INSTALL_DIR"); + + vector paths; + + // the paths to search in order + paths.push_back("."); + paths.push_back(string(MCSSYSCONFDIR) + "/columnstore"); + paths.push_back("/etc"); + + for (uint i = 0; i < paths.size(); i++) + { + // cout << "Config: searching '" << paths[i] << "'" << endl; + if (boost::filesystem::exists(paths[i] + "/storagemanager.cnf")) { - //cout << "Config: searching '" << paths[i] << "'" << endl; - if (boost::filesystem::exists(paths[i] + "/storagemanager.cnf")) - { - filename = paths[i] + "/storagemanager.cnf"; - SMLogging::get()->log(LOG_DEBUG, "Using the config file found at %s", filename.c_str()); - break; - } + filename = paths[i] + "/storagemanager.cnf"; + SMLogging::get()->log(LOG_DEBUG, "Using the config file found at %s", filename.c_str()); + break; } - if (filename.empty()) - throw runtime_error("Config: Could not find the config file for StorageManager"); - - reloadInterval = boost::posix_time::seconds(60); - last_mtime = {0, 0}; - reload(); - reloader = boost::thread([this] { this->reloadThreadFcn(); }); + } + if (filename.empty()) + throw runtime_error("Config: Could not find the config file for StorageManager"); + + reloadInterval = boost::posix_time::seconds(60); + last_mtime = {0, 0}; + reload(); + reloader = boost::thread([this] { this->reloadThreadFcn(); }); } -Config::Config(const string &configFile) : filename(configFile), die(false) +Config::Config(const string& configFile) : filename(configFile), die(false) { - if (!bf::is_regular_file(configFile)) - throw runtime_error("Config: Could not find the config file for StorageManager"); - - reloadInterval = boost::posix_time::seconds(60); - last_mtime = {0, 0}; - reload(); - reloader = boost::thread([this] { this->reloadThreadFcn(); }); + if (!bf::is_regular_file(configFile)) + throw runtime_error("Config: Could not find the config file for StorageManager"); + + reloadInterval = boost::posix_time::seconds(60); + last_mtime = {0, 0}; + reload(); + reloader = boost::thread([this] { this->reloadThreadFcn(); }); } Config::~Config() { - die = true; - reloader.interrupt(); - reloader.join(); + die = true; + reloader.interrupt(); + reloader.join(); } void Config::reloadThreadFcn() { - while (!die) + while (!die) + { + try { - try - { - if (reload()) - { - for (auto& listener : configListeners) - listener->configListener(); - } - boost::this_thread::sleep(reloadInterval); - } - catch (boost::property_tree::ini_parser_error &e) - { - // log the error. how to log things now? - } - catch (boost::thread_interrupted) - {} + if (reload()) + { + for (auto& listener : configListeners) + listener->configListener(); + } + boost::this_thread::sleep(reloadInterval); } + catch (boost::property_tree::ini_parser_error& e) + { + // log the error. how to log things now? + } + catch (boost::thread_interrupted) + { + } + } } bool Config::reload() { - bool rtn = false; - struct stat statbuf; - int err = stat(filename.c_str(), &statbuf); - if (err) - { - SMLogging::get()->log(LOG_ERR, "Config::reload error %s", filename.c_str()); - return rtn; - } - if ((statbuf.st_mtim.tv_sec == last_mtime.tv_sec) && (statbuf.st_mtim.tv_nsec == last_mtime.tv_nsec)) - return rtn; - last_mtime = statbuf.st_mtim; - rtn = true; - boost::unique_lock s(mutex); - contents.clear(); - boost::property_tree::ini_parser::read_ini(filename, contents); + bool rtn = false; + struct stat statbuf; + int err = stat(filename.c_str(), &statbuf); + if (err) + { + SMLogging::get()->log(LOG_ERR, "Config::reload error %s", filename.c_str()); return rtn; + } + if ((statbuf.st_mtim.tv_sec == last_mtime.tv_sec) && (statbuf.st_mtim.tv_nsec == last_mtime.tv_nsec)) + return rtn; + last_mtime = statbuf.st_mtim; + rtn = true; + boost::unique_lock s(mutex); + contents.clear(); + boost::property_tree::ini_parser::read_ini(filename, contents); + return rtn; } -string use_envvar(const boost::smatch &envvar) +string use_envvar(const boost::smatch& envvar) { - char *env = getenv(envvar[1].str().c_str()); - return (env ? env : ""); + char* env = getenv(envvar[1].str().c_str()); + return (env ? env : ""); } -string expand_numbers(const boost::smatch &match) +string expand_numbers(const boost::smatch& match) { - long long num = stol(match[1].str()); - char suffix = (char) ::tolower(match[2].str()[0]); + long long num = stol(match[1].str()); + char suffix = (char)::tolower(match[2].str()[0]); - if (suffix == 't') - num <<= 40; - else if (suffix == 'g') - num <<= 30; - else if (suffix == 'm') - num <<= 20; - else if (suffix == 'k') - num <<= 10; - return ::to_string(num); + if (suffix == 't') + num <<= 40; + else if (suffix == 'g') + num <<= 30; + else if (suffix == 'm') + num <<= 20; + else if (suffix == 'k') + num <<= 10; + return ::to_string(num); } - -string Config::getValue(const string §ion, const string &key) const + +string Config::getValue(const string& section, const string& key) const { - // if we care, move this envvar substition stuff to where the file is loaded - string ret; - boost::unique_lock s(mutex); - try { - ret = contents.get(section + "." + key); - } - catch (...) { - return ""; // debating whether it's necessary to tell the caller there was no entry. - } - s.unlock(); - - boost::regex re("\\$\\{(.+)\\}"); - - ret = boost::regex_replace(ret, re, use_envvar); - - // do the numeric substitutions. ex, the suffixes m, k, g - // ehhhhh. going to end up turning a string to a number, to a string, and then to a number again - // don't like that. OTOH who cares. - boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended); - ret = boost::regex_replace(ret, num_re, expand_numbers); - - return ret; + // if we care, move this envvar substition stuff to where the file is loaded + string ret; + boost::unique_lock s(mutex); + try + { + ret = contents.get(section + "." + key); + } + catch (...) + { + return ""; // debating whether it's necessary to tell the caller there was no entry. + } + s.unlock(); + + boost::regex re("\\$\\{(.+)\\}"); + + ret = boost::regex_replace(ret, re, use_envvar); + + // do the numeric substitutions. ex, the suffixes m, k, g + // ehhhhh. going to end up turning a string to a number, to a string, and then to a number again + // don't like that. OTOH who cares. + boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended); + ret = boost::regex_replace(ret, num_re, expand_numbers); + + return ret; } -void Config::addConfigListener(ConfigListener *listener) +void Config::addConfigListener(ConfigListener* listener) { - configListeners.push_back(listener); + configListeners.push_back(listener); } -void Config::removeConfigListener(ConfigListener *listener) +void Config::removeConfigListener(ConfigListener* listener) { - auto iterator = std::find(configListeners.begin(), configListeners.end(), listener); + auto iterator = std::find(configListeners.begin(), configListeners.end(), listener); - if (iterator != configListeners.end()) - configListeners.erase(iterator); + if (iterator != configListeners.end()) + configListeners.erase(iterator); } -} +} // namespace storagemanager diff --git a/storage-manager/src/Config.h b/storage-manager/src/Config.h index 9818ffc90..c70684283 100644 --- a/storage-manager/src/Config.h +++ b/storage-manager/src/Config.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef CONFIG_H_ #define CONFIG_H_ @@ -32,45 +31,42 @@ namespace storagemanager { class ConfigListener { - public: - virtual ~ConfigListener(){}; - virtual void configListener() = 0; + public: + virtual ~ConfigListener(){}; + virtual void configListener() = 0; }; class Config : public boost::noncopyable { - public: - static Config *get(); - virtual ~Config(); - - std::string getValue(const std::string §ion, const std::string &key) const; - - // for testing, lets caller specify a config file to use - static Config *get(const std::string &); + public: + static Config* get(); + virtual ~Config(); - void addConfigListener(ConfigListener *listener); - void removeConfigListener(ConfigListener *listener); + std::string getValue(const std::string& section, const std::string& key) const; - private: - Config(); - Config(const std::string &); - - bool reload(); - void reloadThreadFcn(); - std::vector configListeners; - struct ::timespec last_mtime; - mutable boost::mutex mutex; - boost::thread reloader; - boost::posix_time::time_duration reloadInterval; - - std::string filename; - boost::property_tree::ptree contents; - bool die; + // for testing, lets caller specify a config file to use + static Config* get(const std::string&); + + void addConfigListener(ConfigListener* listener); + void removeConfigListener(ConfigListener* listener); + + private: + Config(); + Config(const std::string&); + + bool reload(); + void reloadThreadFcn(); + std::vector configListeners; + struct ::timespec last_mtime; + mutable boost::mutex mutex; + boost::thread reloader; + boost::posix_time::time_duration reloadInterval; + + std::string filename; + boost::property_tree::ptree contents; + bool die; }; - - - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/CopyTask.cpp b/storage-manager/src/CopyTask.cpp index e18fabe44..7d6fe980e 100644 --- a/storage-manager/src/CopyTask.cpp +++ b/storage-manager/src/CopyTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "CopyTask.h" #include #include "SMLogging.h" @@ -25,7 +24,6 @@ using namespace std; namespace storagemanager { - CopyTask::CopyTask(int sock, uint len) : PosixTask(sock, len) { } @@ -35,55 +33,56 @@ CopyTask::~CopyTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } bool CopyTask::run() { - int success; - SMLogging* logger = SMLogging::get(); + int success; + SMLogging* logger = SMLogging::get(); - uint8_t buf[2048] = {0}; - - if (getLength() > 2047) - { - handleError("CopyTask read", ENAMETOOLONG); - return true; - } - - success = read(buf, getLength()); - check_error("CopyTask read", false); - copy_cmd *cmd = (copy_cmd *) buf; - string filename1(cmd->file1.filename, cmd->file1.flen); // need to copy this in case it's not null terminated - f_name *filename2 = (f_name *) &buf[sizeof(copy_cmd) + cmd->file1.flen]; - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"copy %s to %s.",filename1.c_str(),filename2->filename); - #endif - int err; - try - { - err = ioc->copyFile(filename1.c_str(), filename2->filename); - } - catch (exception &e) - { - logger->log(LOG_ERR, "CopyTask: caught %s", e.what()); - err = -1; - errno = EIO; - } - - if (err) - { - handleError("CopyTask copy", errno); - return true; - } - - sm_response *resp = (sm_response *) buf; - resp->returnCode = 0; - return write(*resp, 0); + uint8_t buf[2048] = {0}; + + if (getLength() > 2047) + { + handleError("CopyTask read", ENAMETOOLONG); + return true; + } + + success = read(buf, getLength()); + check_error("CopyTask read", false); + copy_cmd* cmd = (copy_cmd*)buf; + string filename1(cmd->file1.filename, + cmd->file1.flen); // need to copy this in case it's not null terminated + f_name* filename2 = (f_name*)&buf[sizeof(copy_cmd) + cmd->file1.flen]; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "copy %s to %s.", filename1.c_str(), filename2->filename); +#endif + int err; + try + { + err = ioc->copyFile(filename1.c_str(), filename2->filename); + } + catch (exception& e) + { + logger->log(LOG_ERR, "CopyTask: caught %s", e.what()); + err = -1; + errno = EIO; + } + + if (err) + { + handleError("CopyTask copy", errno); + return true; + } + + sm_response* resp = (sm_response*)buf; + resp->returnCode = 0; + return write(*resp, 0); } -} +} // namespace storagemanager diff --git a/storage-manager/src/CopyTask.h b/storage-manager/src/CopyTask.h index f33e9bf23..9f14f7280 100644 --- a/storage-manager/src/CopyTask.h +++ b/storage-manager/src/CopyTask.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef COPYTASK_H_ #define COPYTASK_H_ @@ -23,19 +22,17 @@ namespace storagemanager { - class CopyTask : public PosixTask { - public: - CopyTask(int sock, uint length); - virtual ~CopyTask(); - - bool run(); - - private: - CopyTask(); + public: + CopyTask(int sock, uint length); + virtual ~CopyTask(); + + bool run(); + + private: + CopyTask(); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Downloader.cpp b/storage-manager/src/Downloader.cpp index 0c230f566..7d21b7b4f 100644 --- a/storage-manager/src/Downloader.cpp +++ b/storage-manager/src/Downloader.cpp @@ -27,209 +27,211 @@ using namespace std; namespace bf = boost::filesystem; namespace storagemanager { - Downloader::Downloader() : maxDownloads(0) { - storage = CloudStorage::get(); - configListener(); - Config::get()->addConfigListener(this); - workers.setName("Downloader"); - logger = SMLogging::get(); - tmpPath = "downloading"; - bytesDownloaded = 0; + storage = CloudStorage::get(); + configListener(); + Config::get()->addConfigListener(this); + workers.setName("Downloader"); + logger = SMLogging::get(); + tmpPath = "downloading"; + bytesDownloaded = 0; } Downloader::~Downloader() { - Config::get()->removeConfigListener(this); + Config::get()->removeConfigListener(this); } -void Downloader::download(const vector &keys, vector *errnos, vector *sizes, - const bf::path &prefix, boost::mutex *cache_lock) +void Downloader::download(const vector& keys, vector* errnos, vector* sizes, + const bf::path& prefix, boost::mutex* cache_lock) { - uint counter = keys.size(); - boost::condition condvar; - DownloadListener listener(&counter, &condvar); - vector > ownedDownloads(keys.size()); - - // the caller is holding cache_lock - /* if a key is already being downloaded, attach listener to that Download instance. - if it is not already being downloaded, make a new Download instance. - wait for the listener to tell us that it's done. - */ - boost::unique_lock s(lock); - for (uint i = 0; i < keys.size(); i++) + uint counter = keys.size(); + boost::condition condvar; + DownloadListener listener(&counter, &condvar); + vector > ownedDownloads(keys.size()); + + // the caller is holding cache_lock + /* if a key is already being downloaded, attach listener to that Download instance. + if it is not already being downloaded, make a new Download instance. + wait for the listener to tell us that it's done. + */ + boost::unique_lock s(lock); + for (uint i = 0; i < keys.size(); i++) + { + boost::shared_ptr newDL(new Download(*keys[i], prefix, cache_lock, this)); + + auto it = downloads.find(newDL); // kinda sucks to have to search this way. + if (it == downloads.end()) { - boost::shared_ptr newDL(new Download(*keys[i], prefix, cache_lock, this)); - - auto it = downloads.find(newDL); // kinda sucks to have to search this way. - if (it == downloads.end()) - { - newDL->listeners.push_back(&listener); - ownedDownloads[i] = newDL; - downloads.insert(newDL); - workers.addJob(newDL); - } - else - { - //assert((*it)->key == *keys[i]); - //cout << "Waiting for the existing download of " << *keys[i] << endl; - - // a Download is technically in the map until all of the Downloads issued by its owner - // have finished. So, we have to test whether this existing download has already finished - // or not before waiting for it. We could have Downloads remove themselves on completion. - // TBD. Need to just get this working first. - if ((*it)->finished) - --counter; - else - (*it)->listeners.push_back(&listener); - } + newDL->listeners.push_back(&listener); + ownedDownloads[i] = newDL; + downloads.insert(newDL); + workers.addJob(newDL); } - s.unlock(); - // wait for the downloads to finish - while (counter > 0) - condvar.wait(*cache_lock); - - // check success, gather sizes from downloads started by this thread - sizes->resize(keys.size()); - errnos->resize(keys.size()); - char buf[80]; - s.lock(); - for (uint i = 0; i < keys.size(); i++) + else { - if (ownedDownloads[i]) - { - assert(ownedDownloads[i]->finished); - (*sizes)[i] = ownedDownloads[i]->size; - (*errnos)[i] = ownedDownloads[i]->dl_errno; - if ((*errnos)[i]) - logger->log(LOG_ERR, "Downloader: failed to download %s, got '%s'", keys[i]->c_str(), + // assert((*it)->key == *keys[i]); + // cout << "Waiting for the existing download of " << *keys[i] << endl; + + // a Download is technically in the map until all of the Downloads issued by its owner + // have finished. So, we have to test whether this existing download has already finished + // or not before waiting for it. We could have Downloads remove themselves on completion. + // TBD. Need to just get this working first. + if ((*it)->finished) + --counter; + else + (*it)->listeners.push_back(&listener); + } + } + s.unlock(); + // wait for the downloads to finish + while (counter > 0) + condvar.wait(*cache_lock); + + // check success, gather sizes from downloads started by this thread + sizes->resize(keys.size()); + errnos->resize(keys.size()); + char buf[80]; + s.lock(); + for (uint i = 0; i < keys.size(); i++) + { + if (ownedDownloads[i]) + { + assert(ownedDownloads[i]->finished); + (*sizes)[i] = ownedDownloads[i]->size; + (*errnos)[i] = ownedDownloads[i]->dl_errno; + if ((*errnos)[i]) + logger->log(LOG_ERR, "Downloader: failed to download %s, got '%s'", keys[i]->c_str(), strerror_r((*errnos)[i], buf, 80)); - downloads.erase(ownedDownloads[i]); - bytesDownloaded += (*sizes)[i]; - } - else - { - (*sizes)[i] = 0; - (*errnos)[i] = 0; - } + downloads.erase(ownedDownloads[i]); + bytesDownloaded += (*sizes)[i]; } + else + { + (*sizes)[i] = 0; + (*errnos)[i] = 0; + } + } } void Downloader::printKPIs() const { - cout << "Downloader: bytesDownloaded = " << bytesDownloaded << endl; + cout << "Downloader: bytesDownloaded = " << bytesDownloaded << endl; } -bool Downloader::inProgress(const string &key) +bool Downloader::inProgress(const string& key) { - boost::shared_ptr tmp(new Download(key)); - boost::unique_lock s(lock); - - auto it = downloads.find(tmp); - if (it != downloads.end()) - return !(*it)->finished; - return false; + boost::shared_ptr tmp(new Download(key)); + boost::unique_lock s(lock); + + auto it = downloads.find(tmp); + if (it != downloads.end()) + return !(*it)->finished; + return false; } -const bf::path & Downloader::getTmpPath() const +const bf::path& Downloader::getTmpPath() const { - return tmpPath; + return tmpPath; } /* The helper fcns */ -Downloader::Download::Download(const string &source, const bf::path &_dlPath, boost::mutex *_lock, Downloader *_dl) : - dlPath(_dlPath), key(source), dl_errno(0), size(0), lock(_lock), finished(false), itRan(false), dl(_dl) +Downloader::Download::Download(const string& source, const bf::path& _dlPath, boost::mutex* _lock, + Downloader* _dl) + : dlPath(_dlPath), key(source), dl_errno(0), size(0), lock(_lock), finished(false), itRan(false), dl(_dl) { } -Downloader::Download::Download(const string &source) : - key(source), dl_errno(0), size(0), lock(NULL), finished(false), itRan(false), dl(NULL) +Downloader::Download::Download(const string& source) + : key(source), dl_errno(0), size(0), lock(NULL), finished(false), itRan(false), dl(NULL) { } Downloader::Download::~Download() { - assert(!itRan || finished); + assert(!itRan || finished); } void Downloader::Download::operator()() { - itRan = true; - CloudStorage *storage = CloudStorage::get(); - // download to a tmp path - if (!bf::exists(dlPath / dl->getTmpPath())) - bf::create_directories(dlPath / dl->getTmpPath()); - bf::path tmpFile = dlPath / dl->getTmpPath() / key; - int err = storage->getObject(key, tmpFile.string(), &size); - if (err != 0) - { - dl_errno = errno; - bf::remove(tmpFile); - size = 0; - } - - // move it to its proper place - boost::system::error_code berr; - bf::rename(tmpFile, dlPath / key, berr); - if (berr) - { - dl_errno = berr.value(); - bf::remove(tmpFile); - size = 0; - } - - lock->lock(); - finished = true; - for (uint i = 0; i < listeners.size(); i++) - listeners[i]->downloadFinished(); - lock->unlock(); + itRan = true; + CloudStorage* storage = CloudStorage::get(); + // download to a tmp path + if (!bf::exists(dlPath / dl->getTmpPath())) + bf::create_directories(dlPath / dl->getTmpPath()); + bf::path tmpFile = dlPath / dl->getTmpPath() / key; + int err = storage->getObject(key, tmpFile.string(), &size); + if (err != 0) + { + dl_errno = errno; + bf::remove(tmpFile); + size = 0; + } + + // move it to its proper place + boost::system::error_code berr; + bf::rename(tmpFile, dlPath / key, berr); + if (berr) + { + dl_errno = berr.value(); + bf::remove(tmpFile); + size = 0; + } + + lock->lock(); + finished = true; + for (uint i = 0; i < listeners.size(); i++) + listeners[i]->downloadFinished(); + lock->unlock(); } -Downloader::DownloadListener::DownloadListener(uint *_counter, boost::condition *condvar) : counter(_counter), cond(condvar) +Downloader::DownloadListener::DownloadListener(uint* _counter, boost::condition* condvar) + : counter(_counter), cond(condvar) { } void Downloader::DownloadListener::downloadFinished() { - (*counter)--; - if ((*counter) == 0) - cond->notify_all(); + (*counter)--; + if ((*counter) == 0) + cond->notify_all(); } -inline size_t Downloader::DLHasher::operator()(const boost::shared_ptr &d) const +inline size_t Downloader::DLHasher::operator()(const boost::shared_ptr& d) const { - // since the keys start with a uuid, we can probably get away with just returning the first 8 chars - // or as a compromise, hashing only the first X chars. For later. - return hash()(d->key); + // since the keys start with a uuid, we can probably get away with just returning the first 8 chars + // or as a compromise, hashing only the first X chars. For later. + return hash()(d->key); } -inline bool Downloader::DLEquals::operator()(const boost::shared_ptr &d1, const boost::shared_ptr &d2) const +inline bool Downloader::DLEquals::operator()(const boost::shared_ptr& d1, + const boost::shared_ptr& d2) const { - return (d1->key == d2->key); + return (d1->key == d2->key); } void Downloader::configListener() { - // Downloader threads - string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_downloads"); - if (maxDownloads == 0) - maxDownloads = 20; - if (stmp.empty()) + // Downloader threads + string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_downloads"); + if (maxDownloads == 0) + maxDownloads = 20; + if (stmp.empty()) + { + logger->log(LOG_CRIT, "max_concurrent_downloads is not set. Using current value = %u", maxDownloads); + } + try + { + uint newValue = stoul(stmp); + if (newValue != maxDownloads) { - logger->log(LOG_CRIT, "max_concurrent_downloads is not set. Using current value = %u",maxDownloads); - } - try - { - uint newValue = stoul(stmp); - if (newValue != maxDownloads) - { - maxDownloads = newValue; - workers.setMaxThreads(maxDownloads); - logger->log(LOG_INFO, "max_concurrent_downloads = %u",maxDownloads); - } - } - catch (invalid_argument &) - { - logger->log(LOG_CRIT, "max_concurrent_downloads is not a number. Using current value = %u",maxDownloads); + maxDownloads = newValue; + workers.setMaxThreads(maxDownloads); + logger->log(LOG_INFO, "max_concurrent_downloads = %u", maxDownloads); } + } + catch (invalid_argument&) + { + logger->log(LOG_CRIT, "max_concurrent_downloads is not a number. Using current value = %u", maxDownloads); + } } -} +} // namespace storagemanager diff --git a/storage-manager/src/Downloader.h b/storage-manager/src/Downloader.h index 0c2e73410..341a8677f 100644 --- a/storage-manager/src/Downloader.h +++ b/storage-manager/src/Downloader.h @@ -35,82 +35,81 @@ namespace storagemanager { - class Downloader : public ConfigListener { - public: - Downloader(); - virtual ~Downloader(); - - // caller owns the memory for the strings. - // errors are reported through errnos - void download(const std::vector &keys, - std::vector *errnos, std::vector *sizes, const boost::filesystem::path &prefix, - boost::mutex *lock); - bool inProgress(const std::string &); // call this holding the cache's lock - const boost::filesystem::path & getTmpPath() const; - - void printKPIs() const; + public: + Downloader(); + virtual ~Downloader(); - virtual void configListener() override; - - private: - uint maxDownloads; - //boost::filesystem::path downloadPath; - boost::mutex lock; - - class DownloadListener - { - public: - DownloadListener(uint *counter, boost::condition *condvar); - void downloadFinished(); - private: - uint *counter; - boost::condition *cond; - }; - - /* Possible optimization. Downloads used to use pointers to strings to avoid an extra copy. - Out of paranoid during debugging, I made it copy the strings instead for a clearer lifecycle. - However, it _should_ be safe to do. - */ - struct Download : public ThreadPool::Job - { - Download(const std::string &source, const boost::filesystem::path &_dlPath, boost::mutex *, Downloader *); - Download(const std::string &source); - virtual ~Download(); - void operator()(); - boost::filesystem::path dlPath; - const std::string key; - int dl_errno; // to propagate errors from the download job to the caller - size_t size; - boost::mutex *lock; - bool finished, itRan; - Downloader *dl; - std::vector listeners; - }; - - struct DLHasher - { - size_t operator()(const boost::shared_ptr &d) const; - }; - - struct DLEquals - { - bool operator()(const boost::shared_ptr &d1, const boost::shared_ptr &d2) const; - }; - - typedef std::unordered_set, DLHasher, DLEquals> Downloads_t; - Downloads_t downloads; - boost::filesystem::path tmpPath; - - ThreadPool workers; - CloudStorage *storage; - SMLogging *logger; + // caller owns the memory for the strings. + // errors are reported through errnos + void download(const std::vector& keys, std::vector* errnos, + std::vector* sizes, const boost::filesystem::path& prefix, boost::mutex* lock); + bool inProgress(const std::string&); // call this holding the cache's lock + const boost::filesystem::path& getTmpPath() const; - // KPIs - size_t bytesDownloaded; + void printKPIs() const; + + virtual void configListener() override; + + private: + uint maxDownloads; + // boost::filesystem::path downloadPath; + boost::mutex lock; + + class DownloadListener + { + public: + DownloadListener(uint* counter, boost::condition* condvar); + void downloadFinished(); + + private: + uint* counter; + boost::condition* cond; + }; + + /* Possible optimization. Downloads used to use pointers to strings to avoid an extra copy. + Out of paranoid during debugging, I made it copy the strings instead for a clearer lifecycle. + However, it _should_ be safe to do. + */ + struct Download : public ThreadPool::Job + { + Download(const std::string& source, const boost::filesystem::path& _dlPath, boost::mutex*, Downloader*); + Download(const std::string& source); + virtual ~Download(); + void operator()(); + boost::filesystem::path dlPath; + const std::string key; + int dl_errno; // to propagate errors from the download job to the caller + size_t size; + boost::mutex* lock; + bool finished, itRan; + Downloader* dl; + std::vector listeners; + }; + + struct DLHasher + { + size_t operator()(const boost::shared_ptr& d) const; + }; + + struct DLEquals + { + bool operator()(const boost::shared_ptr& d1, const boost::shared_ptr& d2) const; + }; + + typedef std::unordered_set, DLHasher, DLEquals> Downloads_t; + Downloads_t downloads; + boost::filesystem::path tmpPath; + + ThreadPool workers; + CloudStorage* storage; + SMLogging* logger; + + // KPIs + size_t bytesDownloaded; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index 535a166dc..b17d502b2 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "IOCoordinator.h" #include "MetadataFile.h" #include "Synchronizer.h" @@ -39,1487 +38,1509 @@ using namespace std; namespace { - storagemanager::IOCoordinator *ioc = NULL; - boost::mutex m; -} +storagemanager::IOCoordinator* ioc = NULL; +boost::mutex m; +} // namespace namespace bf = boost::filesystem; namespace storagemanager { - IOCoordinator::IOCoordinator() { - config = Config::get(); - cache = Cache::get(); - logger = SMLogging::get(); - replicator = Replicator::get(); - - try - { - objectSize = stoul(config->getValue("ObjectStorage", "object_size")); - } - catch (...) - { - logger->log(LOG_ERR, "ObjectStorage/object_size must be set to a numeric value"); - throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file"); - } + config = Config::get(); + cache = Cache::get(); + logger = SMLogging::get(); + replicator = Replicator::get(); - try + try + { + objectSize = stoul(config->getValue("ObjectStorage", "object_size")); + } + catch (...) + { + logger->log(LOG_ERR, "ObjectStorage/object_size must be set to a numeric value"); + throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file"); + } + + try + { + metaPath = config->getValue("ObjectStorage", "metadata_path"); + if (metaPath.empty()) { - metaPath = config->getValue("ObjectStorage", "metadata_path"); - if (metaPath.empty()) - { - logger->log(LOG_ERR, "ObjectStorage/journal_path is not set"); - throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); - } + logger->log(LOG_ERR, "ObjectStorage/journal_path is not set"); + throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); } - catch (...) - { - logger->log(LOG_ERR, "ObjectStorage/metadata_path is not set"); - throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); - } - - cachePath = cache->getCachePath(); - journalPath = cache->getJournalPath(); - - bytesRead = bytesWritten = filesOpened = filesCreated = filesCopied = filesDeleted = - bytesCopied = filesTruncated = listingCount = callsToWrite = 0; - iocFilesOpened = iocObjectsCreated = iocJournalsCreated = iocBytesWritten = iocFilesDeleted = iocBytesRead = 0; + } + catch (...) + { + logger->log(LOG_ERR, "ObjectStorage/metadata_path is not set"); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + + cachePath = cache->getCachePath(); + journalPath = cache->getJournalPath(); + + bytesRead = bytesWritten = filesOpened = filesCreated = filesCopied = filesDeleted = bytesCopied = + filesTruncated = listingCount = callsToWrite = 0; + iocFilesOpened = iocObjectsCreated = iocJournalsCreated = iocBytesWritten = iocFilesDeleted = iocBytesRead = + 0; } IOCoordinator::~IOCoordinator() { } -IOCoordinator * IOCoordinator::get() +IOCoordinator* IOCoordinator::get() { - if (ioc) - return ioc; - boost::mutex::scoped_lock s(m); - if (ioc) - return ioc; - ioc = new IOCoordinator(); + if (ioc) return ioc; + boost::mutex::scoped_lock s(m); + if (ioc) + return ioc; + ioc = new IOCoordinator(); + return ioc; } void IOCoordinator::printKPIs() const { - cout << "IOCoordinator" << endl; - cout << "\tUser's POV" << endl; - cout << "\t\tbytesRead = " << bytesRead << endl; - cout << "\t\tbytesWritten = " << bytesWritten << endl; - cout << "\t\tbytesCopied = " << bytesCopied << endl; - cout << "\t\tfilesOpened = " << filesOpened << endl; - cout << "\t\tfilesCreated = " << filesCreated << endl; - cout << "\t\tfilesCopied = " << filesCopied << endl; - cout << "\t\tfilesDeleted = " << filesDeleted << endl; - cout << "\t\tfilesTruncated = " << filesTruncated << endl; - cout << "\t\tcallsToWrite = " << callsToWrite << endl; - cout << "\tIOC's POV" << endl; - cout << "\t\tiocFilesOpened = " << iocFilesOpened << endl; - cout << "\t\tiocObjectsCreated = " << iocObjectsCreated << endl; - cout << "\t\tiocJournalsCreated = " << iocJournalsCreated << endl; - cout << "\t\tiocBytesRead = " << iocBytesRead << endl; - cout << "\t\tiocBytesWritten = " << iocBytesWritten << endl; + cout << "IOCoordinator" << endl; + cout << "\tUser's POV" << endl; + cout << "\t\tbytesRead = " << bytesRead << endl; + cout << "\t\tbytesWritten = " << bytesWritten << endl; + cout << "\t\tbytesCopied = " << bytesCopied << endl; + cout << "\t\tfilesOpened = " << filesOpened << endl; + cout << "\t\tfilesCreated = " << filesCreated << endl; + cout << "\t\tfilesCopied = " << filesCopied << endl; + cout << "\t\tfilesDeleted = " << filesDeleted << endl; + cout << "\t\tfilesTruncated = " << filesTruncated << endl; + cout << "\t\tcallsToWrite = " << callsToWrite << endl; + cout << "\tIOC's POV" << endl; + cout << "\t\tiocFilesOpened = " << iocFilesOpened << endl; + cout << "\t\tiocObjectsCreated = " << iocObjectsCreated << endl; + cout << "\t\tiocJournalsCreated = " << iocJournalsCreated << endl; + cout << "\t\tiocBytesRead = " << iocBytesRead << endl; + cout << "\t\tiocBytesWritten = " << iocBytesWritten << endl; } - -int IOCoordinator::loadObject(int fd, uint8_t *data, off_t offset, size_t length) +int IOCoordinator::loadObject(int fd, uint8_t* data, off_t offset, size_t length) { - size_t count = 0; - int err; - - ::lseek(fd, offset, SEEK_SET); - while (count < length) + size_t count = 0; + int err; + + ::lseek(fd, offset, SEEK_SET); + while (count < length) + { + err = ::read(fd, &data[count], length - count); + if (err < 0) + return err; + else if (err == 0) { - err = ::read(fd, &data[count], length - count); - if (err < 0) - return err; - else if (err == 0) - { - errno = ENODATA; // better errno for early EOF? - return -1; - } - count += err; + errno = ENODATA; // better errno for early EOF? + return -1; } - iocBytesRead += count; - return 0; + count += err; + } + iocBytesRead += count; + return 0; } -int IOCoordinator::loadObjectAndJournal(const char *objFilename, const char *journalFilename, - uint8_t *data, off_t offset, size_t length) +int IOCoordinator::loadObjectAndJournal(const char* objFilename, const char* journalFilename, uint8_t* data, + off_t offset, size_t length) { - boost::shared_array argh; - - size_t tmp = 0; - argh = mergeJournal(objFilename, journalFilename, offset, length, &tmp); - if (!argh) - return -1; + boost::shared_array argh; + + size_t tmp = 0; + argh = mergeJournal(objFilename, journalFilename, offset, length, &tmp); + if (!argh) + return -1; + else + memcpy(data, argh.get(), length); + iocBytesRead += tmp; + return 0; +} + +ssize_t IOCoordinator::read(const char* _filename, uint8_t* data, off_t offset, size_t length) +{ + /* + This is a bit complex and verbose, so for the first cut, it will only return a partial + result where that is easy to do. Otherwise it will return an error. + */ + + /* + Get the read lock on filename + Figure out which objects are relevant to the request + call Cache::read(objects) + Open any journal files that exist to prevent deletion + open all object files to prevent deletion + release read lock + put together the response in data + */ + bf::path filename = ownership.get(_filename); + const bf::path firstDir = *(filename.begin()); + + ScopedReadLock fileLock(this, filename.string()); + MetadataFile meta(filename, MetadataFile::no_create_t(), true); + + if (!meta.exists()) + { + errno = ENOENT; + return -1; + } + + vector relevants = meta.metadataRead(offset, length); + map journalFDs, objectFDs; + map keyToJournalName, keyToObjectName; + utils::VLArray fdMinders(relevants.size() * 2); + int mindersIndex = 0; + char buf[80]; + + // load them into the cache + vector keys; + keys.reserve(relevants.size()); + for (const auto& object : relevants) + keys.push_back(object.key); + cache->read(firstDir, keys); + + // open the journal files and objects that exist to prevent them from being + // deleted mid-operation + for (const auto& key : keys) + { + // trying not to think about how expensive all these type conversions are. + // need to standardize on type. Or be smart and build filenames in a char []. + // later. not thinking about it for now. + + // open all of the journal files that exist + string jFilename = (journalPath / firstDir / (key + ".journal")).string(); + int fd = ::open(jFilename.c_str(), O_RDONLY); + if (fd >= 0) + { + keyToJournalName[key] = jFilename; + journalFDs[key] = fd; + fdMinders[mindersIndex++].fd = fd; + // fdMinders.push_back(SharedCloser(fd)); + } + else if (errno != ENOENT) + { + int l_errno = errno; + fileLock.unlock(); + cache->doneReading(firstDir, keys); + logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'", + jFilename.c_str(), strerror_r(l_errno, buf, 80)); + errno = l_errno; + return -1; + } + + // open all of the objects + string oFilename = (cachePath / firstDir / key).string(); + fd = ::open(oFilename.c_str(), O_RDONLY); + if (fd < 0) + { + int l_errno = errno; + fileLock.unlock(); + cache->doneReading(firstDir, keys); + logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'", + oFilename.c_str(), strerror_r(l_errno, buf, 80)); + errno = l_errno; + return -1; + } + keyToObjectName[key] = oFilename; + objectFDs[key] = fd; + fdMinders[mindersIndex++].fd = fd; + // fdMinders.push_back(SharedCloser(fd)); + } + // fileLock.unlock(); + + // ^^^ TODO: The original version unlocked the file at the above position. On second glance, + // I'm not sure how we can guarantee the journal files won't be modified during the loads below. + // If we do have a guarantee, and I'm just not seeing it now, add the explanation here and uncomment + // the unlock() call above. For now I will let this hold the read lock for the duration. + + // copy data from each object + journal into the returned data + size_t count = 0; + int err; + boost::shared_array mergedData; + for (auto& object : relevants) + { + const auto& jit = journalFDs.find(object.key); + + // if this is the first object, the offset to start reading at is offset - object->offset + off_t thisOffset = (count == 0 ? offset - object.offset : 0); + // This checks and returns if the read is starting past EOF + if (thisOffset >= (off_t)object.length) + goto out; + // if this is the last object, the length of the read is length - count, + // otherwise it is the length of the object - starting offset + + size_t thisLength = min(object.length - thisOffset, length - count); + if (jit == journalFDs.end()) + err = loadObject(objectFDs[object.key], &data[count], thisOffset, thisLength); else - memcpy(data, argh.get(), length); - iocBytesRead += tmp; - return 0; -} - -ssize_t IOCoordinator::read(const char *_filename, uint8_t *data, off_t offset, size_t length) -{ - /* - This is a bit complex and verbose, so for the first cut, it will only return a partial - result where that is easy to do. Otherwise it will return an error. - */ - - /* - Get the read lock on filename - Figure out which objects are relevant to the request - call Cache::read(objects) - Open any journal files that exist to prevent deletion - open all object files to prevent deletion - release read lock - put together the response in data - */ - bf::path filename = ownership.get(_filename); - const bf::path firstDir = *(filename.begin()); - - ScopedReadLock fileLock(this, filename.string()); - MetadataFile meta(filename, MetadataFile::no_create_t(),true); - - if (!meta.exists()) + err = loadObjectAndJournal(keyToObjectName[object.key].c_str(), keyToJournalName[object.key].c_str(), + &data[count], thisOffset, thisLength); + if (err) { - errno = ENOENT; + fileLock.unlock(); + cache->doneReading(firstDir, keys); + if (count == 0) return -1; + else + return count; } - - vector relevants = meta.metadataRead(offset, length); - map journalFDs, objectFDs; - map keyToJournalName, keyToObjectName; - utils::VLArray fdMinders(relevants.size() * 2); - int mindersIndex = 0; - char buf[80]; - - // load them into the cache - vector keys; - keys.reserve(relevants.size()); - for (const auto &object : relevants) - keys.push_back(object.key); - cache->read(firstDir, keys); - - // open the journal files and objects that exist to prevent them from being - // deleted mid-operation - for (const auto &key : keys) - { - // trying not to think about how expensive all these type conversions are. - // need to standardize on type. Or be smart and build filenames in a char []. - // later. not thinking about it for now. - - // open all of the journal files that exist - string jFilename = (journalPath/firstDir/(key + ".journal")).string(); - int fd = ::open(jFilename.c_str(), O_RDONLY); - if (fd >= 0) - { - keyToJournalName[key] = jFilename; - journalFDs[key] = fd; - fdMinders[mindersIndex++].fd = fd; - //fdMinders.push_back(SharedCloser(fd)); - } - else if (errno != ENOENT) - { - int l_errno = errno; - fileLock.unlock(); - cache->doneReading(firstDir, keys); - logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'", - jFilename.c_str(), strerror_r(l_errno, buf, 80)); - errno = l_errno; - return -1; - } - - // open all of the objects - string oFilename = (cachePath/firstDir/key).string(); - fd = ::open(oFilename.c_str(), O_RDONLY); - if (fd < 0) - { - int l_errno = errno; - fileLock.unlock(); - cache->doneReading(firstDir, keys); - logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'", - oFilename.c_str(), strerror_r(l_errno, buf, 80)); - errno = l_errno; - return -1; - } - keyToObjectName[key] = oFilename; - objectFDs[key] = fd; - fdMinders[mindersIndex++].fd = fd; - //fdMinders.push_back(SharedCloser(fd)); - } - //fileLock.unlock(); - - // ^^^ TODO: The original version unlocked the file at the above position. On second glance, - // I'm not sure how we can guarantee the journal files won't be modified during the loads below. - // If we do have a guarantee, and I'm just not seeing it now, add the explanation here and uncomment - // the unlock() call above. For now I will let this hold the read lock for the duration. - - // copy data from each object + journal into the returned data - size_t count = 0; - int err; - boost::shared_array mergedData; - for (auto &object : relevants) - { - const auto &jit = journalFDs.find(object.key); - - // if this is the first object, the offset to start reading at is offset - object->offset - off_t thisOffset = (count == 0 ? offset - object.offset : 0); - // This checks and returns if the read is starting past EOF - if (thisOffset >= (off_t) object.length) - goto out; - // if this is the last object, the length of the read is length - count, - // otherwise it is the length of the object - starting offset - - size_t thisLength = min(object.length - thisOffset, length - count); - if (jit == journalFDs.end()) - err = loadObject(objectFDs[object.key], &data[count], thisOffset, thisLength); - else - err = loadObjectAndJournal(keyToObjectName[object.key].c_str(), keyToJournalName[object.key].c_str(), - &data[count], thisOffset, thisLength); - if (err) - { - fileLock.unlock(); - cache->doneReading(firstDir, keys); - if (count == 0) - return -1; - else - return count; - } - - count += thisLength; - } - + + count += thisLength; + } + out: - fileLock.unlock(); - cache->doneReading(firstDir, keys); - // all done - bytesRead += length; - return count; + fileLock.unlock(); + cache->doneReading(firstDir, keys); + // all done + bytesRead += length; + return count; } -ssize_t IOCoordinator::write(const char *_filename, const uint8_t *data, off_t offset, size_t length) +ssize_t IOCoordinator::write(const char* _filename, const uint8_t* data, off_t offset, size_t length) { - ++callsToWrite; - bf::path filename = ownership.get(_filename); - const bf::path firstDir = *(filename.begin()); - ScopedWriteLock lock(this, filename.string()); - int ret = _write(filename, data, offset, length, firstDir); - lock.unlock(); - if (ret > 0) - bytesWritten += ret; - cache->doneWriting(firstDir); - return ret; + ++callsToWrite; + bf::path filename = ownership.get(_filename); + const bf::path firstDir = *(filename.begin()); + ScopedWriteLock lock(this, filename.string()); + int ret = _write(filename, data, offset, length, firstDir); + lock.unlock(); + if (ret > 0) + bytesWritten += ret; + cache->doneWriting(firstDir); + return ret; } -ssize_t IOCoordinator::_write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length, - const bf::path &firstDir) +ssize_t IOCoordinator::_write(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, + size_t length, const bf::path& firstDir) { - int err = 0, l_errno; - ssize_t count = 0; - uint64_t writeLength = 0; - uint64_t dataRemaining = length; - uint64_t objectOffset = 0; - vector objects; - vector newObjectKeys; - Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... + int err = 0, l_errno; + ssize_t count = 0; + uint64_t writeLength = 0; + uint64_t dataRemaining = length; + uint64_t objectOffset = 0; + vector objects; + vector newObjectKeys; + Synchronizer* synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... - MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(),true); - if (!metadata.exists()) + MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(), true); + if (!metadata.exists()) + { + errno = ENOENT; + return -1; + } + + // read metadata determine how many objects overlap + objects = metadata.metadataRead(offset, length); + + // if there are objects append the journalfile in replicator + if (!objects.empty()) + { + for (std::vector::const_iterator i = objects.begin(); i != objects.end(); ++i) { - errno = ENOENT; + // figure out how much data to write to this object + if (count == 0 && (uint64_t)offset > i->offset) + { + // first object in the list so start at offset and + // write to end of oject or all the data + // XXXBEN: FIX code here so that writes will use greater of + // objectSize or i-> offset in case objectSize was decreased. + objectOffset = offset - i->offset; + writeLength = min((objectSize - objectOffset), dataRemaining); + } + else + { + // starting at beginning of next object write the rest of data + // or until object length is reached + writeLength = min(objectSize, dataRemaining); + objectOffset = 0; + } + // cache->makeSpace(writeLength+JOURNAL_ENTRY_HEADER_SIZE); + + err = replicator->addJournalEntry((firstDir / i->key), &data[count], objectOffset, writeLength); + // assert((uint) err == writeLength); + + if (err < 0) + { + l_errno = errno; + replicator->updateMetadata(metadata); + // log error and abort + logger->log(LOG_ERR, + "IOCoordinator::write(): Failed addJournalEntry. Journal file likely has partially " + "written data and incorrect metadata."); + errno = l_errno; return -1; - } - - //read metadata determine how many objects overlap - objects = metadata.metadataRead(offset,length); - - // if there are objects append the journalfile in replicator - if(!objects.empty()) - { - for (std::vector::const_iterator i = objects.begin(); i != objects.end(); ++i) - { - // figure out how much data to write to this object - if (count == 0 && (uint64_t) offset > i->offset) - { - // first object in the list so start at offset and - // write to end of oject or all the data - // XXXBEN: FIX code here so that writes will use greater of - // objectSize or i-> offset in case objectSize was decreased. - objectOffset = offset - i->offset; - writeLength = min((objectSize - objectOffset),dataRemaining); - } - else - { - // starting at beginning of next object write the rest of data - // or until object length is reached - writeLength = min(objectSize,dataRemaining); - objectOffset = 0; - } - //cache->makeSpace(writeLength+JOURNAL_ENTRY_HEADER_SIZE); - - err = replicator->addJournalEntry((firstDir/i->key),&data[count],objectOffset,writeLength); - //assert((uint) err == writeLength); - - if (err < 0) - { - l_errno = errno; - replicator->updateMetadata(metadata); - //log error and abort - logger->log(LOG_ERR,"IOCoordinator::write(): Failed addJournalEntry. Journal file likely has partially written data and incorrect metadata."); - errno = l_errno; - return -1; - } - else if ((uint)err < writeLength) - { - dataRemaining -= err; - count += err; - iocBytesWritten += err; - if ((err + objectOffset) > i->length) - metadata.updateEntryLength(i->offset, (err + objectOffset)); - cache->newJournalEntry(firstDir, err+JOURNAL_ENTRY_HEADER_SIZE); - synchronizer->newJournalEntry(firstDir, i->key, err+JOURNAL_ENTRY_HEADER_SIZE); - replicator->updateMetadata(metadata); - //logger->log(LOG_ERR,"IOCoordinator::write(): addJournalEntry incomplete write, %u of %u bytes written.",count,length); - return count; - } - - if ((writeLength + objectOffset) > i->length) - metadata.updateEntryLength(i->offset, (writeLength + objectOffset)); - - cache->newJournalEntry(firstDir, writeLength+JOURNAL_ENTRY_HEADER_SIZE); - - synchronizer->newJournalEntry(firstDir, i->key, writeLength+JOURNAL_ENTRY_HEADER_SIZE); - count += writeLength; - dataRemaining -= writeLength; - iocBytesWritten += writeLength + JOURNAL_ENTRY_HEADER_SIZE; - } - } - - // TODO: Need to potentially create new all-0 objects here if offset is more the objectSize bytes - // beyond the current end of the file. - - // there is no overlapping data, or data goes beyond end of last object - while (dataRemaining > 0) - { - off_t currentEndofData = metadata.getMetadataNewObjectOffset(); - - // count is 0 so first write is beyond current end of file. - // offset is > than newObject.offset so we need to adjust offset for object - // unless offset is beyond newObject.offset + objectSize then we need to write null data to this object - if (count == 0 && offset > currentEndofData) - { - // First lets fill the last object with null data - objects = metadata.metadataRead(currentEndofData,1); - if (objects.size() == 1) - { - // last objeect needs data - metadataObject lastObject = objects[0]; - uint64_t nullJournalSize = (objectSize - lastObject.length); - utils::VLArray nullData(nullJournalSize); - memset(nullData, 0, nullJournalSize); - err = replicator->addJournalEntry((firstDir/lastObject.key),nullData.data(),lastObject.length,nullJournalSize); - if (err < 0) - { - l_errno = errno; - //log error and abort - logger->log(LOG_ERR,"IOCoordinator::write(): Failed addJournalEntry -- NullData. Journal file likely has partially written data and incorrect metadata."); - errno = l_errno; - return -1; - } - else if ((uint)err < nullJournalSize) - { - if ((err + lastObject.length) > lastObject.length) - metadata.updateEntryLength(lastObject.offset, (err + lastObject.length)); - cache->newJournalEntry(firstDir, err+JOURNAL_ENTRY_HEADER_SIZE); - synchronizer->newJournalEntry(firstDir, lastObject.key, err+JOURNAL_ENTRY_HEADER_SIZE); - replicator->updateMetadata(metadata); - //logger->log(LOG_ERR,"IOCoordinator::write(): addJournalEntry incomplete write, %u of %u bytes written.",count,length); - return count; - } - metadata.updateEntryLength(lastObject.offset, (nullJournalSize + lastObject.length)); - cache->newJournalEntry(firstDir, nullJournalSize+JOURNAL_ENTRY_HEADER_SIZE); - synchronizer->newJournalEntry(firstDir, lastObject.key, nullJournalSize+JOURNAL_ENTRY_HEADER_SIZE); - currentEndofData += nullJournalSize; - iocBytesWritten += nullJournalSize + JOURNAL_ENTRY_HEADER_SIZE; - } - // dd we need to do some full null data objects - while ((currentEndofData + (off_t)objectSize) <= offset) - { - metadataObject nullObject = metadata.addMetadataObject(filename,objectSize); - err = replicator->newNullObject((firstDir/nullObject.key),objectSize); - if (err < 0) - { - //log error and abort - l_errno = errno; - logger->log(LOG_ERR,"IOCoordinator::write(): Failed newNullObject."); - errno = l_errno; - if (count == 0) // if no data has been written yet, it's safe to return -1 here. - return -1; - goto out; - } - cache->newObject(firstDir, nullObject.key,objectSize); - newObjectKeys.push_back(nullObject.key); - iocBytesWritten += objectSize; - currentEndofData += objectSize; - } - - //this is starting beyond last object in metadata - //figure out if the offset is in this object - objectOffset = offset - currentEndofData; - writeLength = min((objectSize - objectOffset),dataRemaining); - } - else - { - // count != 0 we've already started writing and are going to new object - // start at beginning of the new object - writeLength = min(objectSize,dataRemaining); - objectOffset = 0; - } - // objectOffset is 0 unless the write starts beyond the end of data - // in that case need to add the null data to cachespace - //cache->makeSpace(writeLength + objectOffset); - metadataObject newObject = metadata.addMetadataObject(filename,(writeLength + objectOffset)); - // send to replicator - err = replicator->newObject((firstDir/newObject.key),&data[count],objectOffset,writeLength); - //assert((uint) err == writeLength); - if (err < 0) - { - //log error and abort - l_errno = errno; - logger->log(LOG_ERR,"IOCoordinator::write(): Failed newObject."); - metadata.removeEntry(newObject.offset); - replicator->remove(cachePath/firstDir/newObject.key); - errno = l_errno; - if (count == 0) // if no data has been written yet, it's safe to return -1 here. - return -1; - goto out; - } - else if (err == 0) - { - // remove the object created above; can't have 0-length objects - metadata.removeEntry(newObject.offset); - replicator->remove(cachePath/firstDir/newObject.key); - goto out; - } - else if ((uint)err < writeLength) - { - dataRemaining -= err; - count += err; - iocBytesWritten += err; - - // get a new name for the object - string oldKey = newObject.key; - newObject.key = metadata.getNewKeyFromOldKey(newObject.key, err + objectOffset); - ostringstream os; - os << "IOCoordinator::write(): renaming " << oldKey << " to " << newObject.key; - logger->log(LOG_DEBUG, os.str().c_str()); - int renameErr = ::rename((cachePath/firstDir/oldKey).string().c_str(), (cachePath/firstDir/newObject.key).string().c_str()); - int renameErrno = errno; - if (renameErr < 0) - { - ostringstream oss; - char buf[80]; - oss << "IOCoordinator::write(): Failed to rename " << (cachePath/firstDir/oldKey).string() << " to " << - (cachePath/firstDir/newObject.key).string() << "! Got " << strerror_r(renameErrno, buf, 80); - logger->log(LOG_ERR, oss.str().c_str()); - newObject.key = oldKey; - } - - // rename and resize the object in metadata - metadata.updateEntry(newObject.offset, newObject.key, (err + objectOffset)); - cache->newObject(firstDir, newObject.key,err + objectOffset); - newObjectKeys.push_back(newObject.key); - goto out; - } - - cache->newObject(firstDir, newObject.key,writeLength + objectOffset); - newObjectKeys.push_back(newObject.key); - - count += writeLength; - dataRemaining -= writeLength; - iocBytesWritten += writeLength; - } - -out: - l_errno = errno; - synchronizer->newObjects(firstDir, newObjectKeys); - replicator->updateMetadata(metadata); - errno = l_errno; - return count; -} - -ssize_t IOCoordinator::append(const char *_filename, const uint8_t *data, size_t length) -{ - bytesWritten += length; - bf::path filename = ownership.get(_filename); - const bf::path firstDir = *(filename.begin()); - int err, l_errno; - ssize_t count = 0; - uint64_t writeLength = 0; - uint64_t dataRemaining = length; - vector objects; - vector newObjectKeys; - Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... - - ScopedWriteLock lock(this, filename.string()); - - MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(),true); - if (!metadata.exists()) - { - errno = ENOENT; - return -1; - } - - uint64_t offset = metadata.getLength(); - - //read metadata determine if this fits in the last object - objects = metadata.metadataRead(offset,length); - - if(objects.size() == 1) - { - std::vector::const_iterator i = objects.begin(); - - // XXXPAT: Need to handle the case where objectSize has been reduced since i was created - // ie, i->length may be > objectSize here, so objectSize - i->length may be a huge positive #. - // Need to disable changing object size for now. - if ((objectSize - i->length) > 0) // if this is zero then we can't put anything else in this object - { - // figure out how much data to write to this object - writeLength = min((objectSize - i->length),dataRemaining); - - //cache->makeSpace(writeLength+JOURNAL_ENTRY_HEADER_SIZE); - - err = replicator->addJournalEntry((firstDir/i->key),&data[count],i->length,writeLength); - //assert((uint) err == writeLength); - if (err < 0) - { - l_errno = errno; - //log error and abort - logger->log(LOG_ERR,"IOCoordinator::append(): Failed addJournalEntry."); - errno = l_errno; - return -1; - } - - count += err; - dataRemaining -= err; - iocBytesWritten += err + JOURNAL_ENTRY_HEADER_SIZE; - metadata.updateEntryLength(i->offset, (err + i->length)); - cache->newJournalEntry(firstDir, err+JOURNAL_ENTRY_HEADER_SIZE); - synchronizer->newJournalEntry(firstDir, i->key, err+JOURNAL_ENTRY_HEADER_SIZE); - if (err < (int64_t) writeLength) - { - //logger->log(LOG_ERR,"IOCoordinator::append(): journal failed to complete write, %u of %u bytes written.",count,length); - goto out; - } - } - } - else if (objects.size() > 1) - { - //Something went wrong this shouldn't overlap multiple objects - logger->log(LOG_ERR,"IOCoordinator::append(): multiple overlapping objects found on append.",count,length); - assert(0); - errno = EIO; // a better option for 'programmer error'? - return -1; - } - // append is starting or adding to a new object - while (dataRemaining > 0) - { - //add a new metaDataObject - writeLength = min(objectSize,dataRemaining); - - //cache->makeSpace(writeLength); - // add a new metadata object, this will get a new objectKey NOTE: probably needs offset too - metadataObject newObject = metadata.addMetadataObject(filename,writeLength); - - // write the new object - err = replicator->newObject((firstDir/newObject.key),&data[count],0,writeLength); - //assert((uint) err == writeLength); - if (err < 0) - { - l_errno = errno; - //log error and abort - logger->log(LOG_ERR,"IOCoordinator::append(): Failed newObject."); - metadata.removeEntry(newObject.offset); - replicator->remove(cachePath/firstDir/newObject.key); - errno = l_errno; - // if no data was written successfully yet, it's safe to return -1 here. - if (count == 0) - return -1; - goto out; - } - else if (err == 0) - { - metadata.removeEntry(newObject.offset); - replicator->remove(cachePath/firstDir/newObject.key); - goto out; - } - - count += err; + } + else if ((uint)err < writeLength) + { dataRemaining -= err; + count += err; iocBytesWritten += err; - if (err < (int64_t) writeLength) - { - string oldKey = newObject.key; - newObject.key = metadata.getNewKeyFromOldKey(newObject.key, err + newObject.offset); - ostringstream os; - os << "IOCoordinator::append(): renaming " << oldKey << " to " << newObject.key; - logger->log(LOG_DEBUG, os.str().c_str()); - int renameErr = ::rename((cachePath/firstDir/oldKey).string().c_str(), (cachePath/firstDir/newObject.key).string().c_str()); - int renameErrno = errno; - if (renameErr < 0) - { - ostringstream oss; - char buf[80]; - oss << "IOCoordinator::append(): Failed to rename " << (cachePath/firstDir/oldKey).string() << " to " << - (cachePath/firstDir/newObject.key).string() << "! Got " << strerror_r(renameErrno, buf, 80); - logger->log(LOG_ERR, oss.str().c_str()); - newObject.key = oldKey; - } - - metadata.updateEntry(newObject.offset, newObject.key, err); - } + if ((err + objectOffset) > i->length) + metadata.updateEntryLength(i->offset, (err + objectOffset)); + cache->newJournalEntry(firstDir, err + JOURNAL_ENTRY_HEADER_SIZE); + synchronizer->newJournalEntry(firstDir, i->key, err + JOURNAL_ENTRY_HEADER_SIZE); + replicator->updateMetadata(metadata); + // logger->log(LOG_ERR,"IOCoordinator::write(): addJournalEntry incomplete write, %u of %u bytes + // written.",count,length); + return count; + } - cache->newObject(firstDir, newObject.key,err); - newObjectKeys.push_back(newObject.key); + if ((writeLength + objectOffset) > i->length) + metadata.updateEntryLength(i->offset, (writeLength + objectOffset)); - if (err < (int64_t) writeLength) + cache->newJournalEntry(firstDir, writeLength + JOURNAL_ENTRY_HEADER_SIZE); + + synchronizer->newJournalEntry(firstDir, i->key, writeLength + JOURNAL_ENTRY_HEADER_SIZE); + count += writeLength; + dataRemaining -= writeLength; + iocBytesWritten += writeLength + JOURNAL_ENTRY_HEADER_SIZE; + } + } + + // TODO: Need to potentially create new all-0 objects here if offset is more the objectSize bytes + // beyond the current end of the file. + + // there is no overlapping data, or data goes beyond end of last object + while (dataRemaining > 0) + { + off_t currentEndofData = metadata.getMetadataNewObjectOffset(); + + // count is 0 so first write is beyond current end of file. + // offset is > than newObject.offset so we need to adjust offset for object + // unless offset is beyond newObject.offset + objectSize then we need to write null data to this object + if (count == 0 && offset > currentEndofData) + { + // First lets fill the last object with null data + objects = metadata.metadataRead(currentEndofData, 1); + if (objects.size() == 1) + { + // last objeect needs data + metadataObject lastObject = objects[0]; + uint64_t nullJournalSize = (objectSize - lastObject.length); + utils::VLArray nullData(nullJournalSize); + memset(nullData, 0, nullJournalSize); + err = replicator->addJournalEntry((firstDir / lastObject.key), nullData.data(), lastObject.length, + nullJournalSize); + if (err < 0) { - //logger->log(LOG_ERR,"IOCoordinator::append(): newObject failed to complete write, %u of %u bytes written.",count,length); - goto out; + l_errno = errno; + // log error and abort + logger->log(LOG_ERR, + "IOCoordinator::write(): Failed addJournalEntry -- NullData. Journal file likely has " + "partially written data and incorrect metadata."); + errno = l_errno; + return -1; } + else if ((uint)err < nullJournalSize) + { + if ((err + lastObject.length) > lastObject.length) + metadata.updateEntryLength(lastObject.offset, (err + lastObject.length)); + cache->newJournalEntry(firstDir, err + JOURNAL_ENTRY_HEADER_SIZE); + synchronizer->newJournalEntry(firstDir, lastObject.key, err + JOURNAL_ENTRY_HEADER_SIZE); + replicator->updateMetadata(metadata); + // logger->log(LOG_ERR,"IOCoordinator::write(): addJournalEntry incomplete write, %u of %u bytes + // written.",count,length); + return count; + } + metadata.updateEntryLength(lastObject.offset, (nullJournalSize + lastObject.length)); + cache->newJournalEntry(firstDir, nullJournalSize + JOURNAL_ENTRY_HEADER_SIZE); + synchronizer->newJournalEntry(firstDir, lastObject.key, nullJournalSize + JOURNAL_ENTRY_HEADER_SIZE); + currentEndofData += nullJournalSize; + iocBytesWritten += nullJournalSize + JOURNAL_ENTRY_HEADER_SIZE; + } + // dd we need to do some full null data objects + while ((currentEndofData + (off_t)objectSize) <= offset) + { + metadataObject nullObject = metadata.addMetadataObject(filename, objectSize); + err = replicator->newNullObject((firstDir / nullObject.key), objectSize); + if (err < 0) + { + // log error and abort + l_errno = errno; + logger->log(LOG_ERR, "IOCoordinator::write(): Failed newNullObject."); + errno = l_errno; + if (count == 0) // if no data has been written yet, it's safe to return -1 here. + return -1; + goto out; + } + cache->newObject(firstDir, nullObject.key, objectSize); + newObjectKeys.push_back(nullObject.key); + iocBytesWritten += objectSize; + currentEndofData += objectSize; + } + + // this is starting beyond last object in metadata + // figure out if the offset is in this object + objectOffset = offset - currentEndofData; + writeLength = min((objectSize - objectOffset), dataRemaining); + } + else + { + // count != 0 we've already started writing and are going to new object + // start at beginning of the new object + writeLength = min(objectSize, dataRemaining); + objectOffset = 0; + } + // objectOffset is 0 unless the write starts beyond the end of data + // in that case need to add the null data to cachespace + // cache->makeSpace(writeLength + objectOffset); + metadataObject newObject = metadata.addMetadataObject(filename, (writeLength + objectOffset)); + // send to replicator + err = replicator->newObject((firstDir / newObject.key), &data[count], objectOffset, writeLength); + // assert((uint) err == writeLength); + if (err < 0) + { + // log error and abort + l_errno = errno; + logger->log(LOG_ERR, "IOCoordinator::write(): Failed newObject."); + metadata.removeEntry(newObject.offset); + replicator->remove(cachePath / firstDir / newObject.key); + errno = l_errno; + if (count == 0) // if no data has been written yet, it's safe to return -1 here. + return -1; + goto out; + } + else if (err == 0) + { + // remove the object created above; can't have 0-length objects + metadata.removeEntry(newObject.offset); + replicator->remove(cachePath / firstDir / newObject.key); + goto out; + } + else if ((uint)err < writeLength) + { + dataRemaining -= err; + count += err; + iocBytesWritten += err; + + // get a new name for the object + string oldKey = newObject.key; + newObject.key = metadata.getNewKeyFromOldKey(newObject.key, err + objectOffset); + ostringstream os; + os << "IOCoordinator::write(): renaming " << oldKey << " to " << newObject.key; + logger->log(LOG_DEBUG, os.str().c_str()); + int renameErr = ::rename((cachePath / firstDir / oldKey).string().c_str(), + (cachePath / firstDir / newObject.key).string().c_str()); + int renameErrno = errno; + if (renameErr < 0) + { + ostringstream oss; + char buf[80]; + oss << "IOCoordinator::write(): Failed to rename " << (cachePath / firstDir / oldKey).string() + << " to " << (cachePath / firstDir / newObject.key).string() << "! Got " + << strerror_r(renameErrno, buf, 80); + logger->log(LOG_ERR, oss.str().c_str()); + newObject.key = oldKey; + } + + // rename and resize the object in metadata + metadata.updateEntry(newObject.offset, newObject.key, (err + objectOffset)); + cache->newObject(firstDir, newObject.key, err + objectOffset); + newObjectKeys.push_back(newObject.key); + goto out; } + cache->newObject(firstDir, newObject.key, writeLength + objectOffset); + newObjectKeys.push_back(newObject.key); + + count += writeLength; + dataRemaining -= writeLength; + iocBytesWritten += writeLength; + } + out: - synchronizer->newObjects(firstDir, newObjectKeys); - replicator->updateMetadata(metadata); - // need to release the file lock before telling Cache that we're done writing. - lock.unlock(); - cache->doneWriting(firstDir); - - return count; + l_errno = errno; + synchronizer->newObjects(firstDir, newObjectKeys); + replicator->updateMetadata(metadata); + errno = l_errno; + return count; +} + +ssize_t IOCoordinator::append(const char* _filename, const uint8_t* data, size_t length) +{ + bytesWritten += length; + bf::path filename = ownership.get(_filename); + const bf::path firstDir = *(filename.begin()); + int err, l_errno; + ssize_t count = 0; + uint64_t writeLength = 0; + uint64_t dataRemaining = length; + vector objects; + vector newObjectKeys; + Synchronizer* synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... + + ScopedWriteLock lock(this, filename.string()); + + MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(), true); + if (!metadata.exists()) + { + errno = ENOENT; + return -1; + } + + uint64_t offset = metadata.getLength(); + + // read metadata determine if this fits in the last object + objects = metadata.metadataRead(offset, length); + + if (objects.size() == 1) + { + std::vector::const_iterator i = objects.begin(); + + // XXXPAT: Need to handle the case where objectSize has been reduced since i was created + // ie, i->length may be > objectSize here, so objectSize - i->length may be a huge positive #. + // Need to disable changing object size for now. + if ((objectSize - i->length) > 0) // if this is zero then we can't put anything else in this object + { + // figure out how much data to write to this object + writeLength = min((objectSize - i->length), dataRemaining); + + // cache->makeSpace(writeLength+JOURNAL_ENTRY_HEADER_SIZE); + + err = replicator->addJournalEntry((firstDir / i->key), &data[count], i->length, writeLength); + // assert((uint) err == writeLength); + if (err < 0) + { + l_errno = errno; + // log error and abort + logger->log(LOG_ERR, "IOCoordinator::append(): Failed addJournalEntry."); + errno = l_errno; + return -1; + } + + count += err; + dataRemaining -= err; + iocBytesWritten += err + JOURNAL_ENTRY_HEADER_SIZE; + metadata.updateEntryLength(i->offset, (err + i->length)); + cache->newJournalEntry(firstDir, err + JOURNAL_ENTRY_HEADER_SIZE); + synchronizer->newJournalEntry(firstDir, i->key, err + JOURNAL_ENTRY_HEADER_SIZE); + if (err < (int64_t)writeLength) + { + // logger->log(LOG_ERR,"IOCoordinator::append(): journal failed to complete write, %u of %u bytes + // written.",count,length); + goto out; + } + } + } + else if (objects.size() > 1) + { + // Something went wrong this shouldn't overlap multiple objects + logger->log(LOG_ERR, "IOCoordinator::append(): multiple overlapping objects found on append.", count, + length); + assert(0); + errno = EIO; // a better option for 'programmer error'? + return -1; + } + // append is starting or adding to a new object + while (dataRemaining > 0) + { + // add a new metaDataObject + writeLength = min(objectSize, dataRemaining); + + // cache->makeSpace(writeLength); + // add a new metadata object, this will get a new objectKey NOTE: probably needs offset too + metadataObject newObject = metadata.addMetadataObject(filename, writeLength); + + // write the new object + err = replicator->newObject((firstDir / newObject.key), &data[count], 0, writeLength); + // assert((uint) err == writeLength); + if (err < 0) + { + l_errno = errno; + // log error and abort + logger->log(LOG_ERR, "IOCoordinator::append(): Failed newObject."); + metadata.removeEntry(newObject.offset); + replicator->remove(cachePath / firstDir / newObject.key); + errno = l_errno; + // if no data was written successfully yet, it's safe to return -1 here. + if (count == 0) + return -1; + goto out; + } + else if (err == 0) + { + metadata.removeEntry(newObject.offset); + replicator->remove(cachePath / firstDir / newObject.key); + goto out; + } + + count += err; + dataRemaining -= err; + iocBytesWritten += err; + if (err < (int64_t)writeLength) + { + string oldKey = newObject.key; + newObject.key = metadata.getNewKeyFromOldKey(newObject.key, err + newObject.offset); + ostringstream os; + os << "IOCoordinator::append(): renaming " << oldKey << " to " << newObject.key; + logger->log(LOG_DEBUG, os.str().c_str()); + int renameErr = ::rename((cachePath / firstDir / oldKey).string().c_str(), + (cachePath / firstDir / newObject.key).string().c_str()); + int renameErrno = errno; + if (renameErr < 0) + { + ostringstream oss; + char buf[80]; + oss << "IOCoordinator::append(): Failed to rename " << (cachePath / firstDir / oldKey).string() + << " to " << (cachePath / firstDir / newObject.key).string() << "! Got " + << strerror_r(renameErrno, buf, 80); + logger->log(LOG_ERR, oss.str().c_str()); + newObject.key = oldKey; + } + + metadata.updateEntry(newObject.offset, newObject.key, err); + } + + cache->newObject(firstDir, newObject.key, err); + newObjectKeys.push_back(newObject.key); + + if (err < (int64_t)writeLength) + { + // logger->log(LOG_ERR,"IOCoordinator::append(): newObject failed to complete write, %u of %u bytes + // written.",count,length); + goto out; + } + } + +out: + synchronizer->newObjects(firstDir, newObjectKeys); + replicator->updateMetadata(metadata); + // need to release the file lock before telling Cache that we're done writing. + lock.unlock(); + cache->doneWriting(firstDir); + + return count; } // TODO: might need to support more open flags, ex: O_EXCL -int IOCoordinator::open(const char *_filename, int openmode, struct stat *out) +int IOCoordinator::open(const char* _filename, int openmode, struct stat* out) { - bf::path filename = ownership.get(_filename); - boost::scoped_ptr s; - - if (openmode & O_CREAT || openmode & O_TRUNC) - s.reset(new ScopedWriteLock(this, filename.string())); - else - s.reset(new ScopedReadLock(this, filename.string())); + bf::path filename = ownership.get(_filename); + boost::scoped_ptr s; - MetadataFile meta(filename, MetadataFile::no_create_t(),true); - - if ((openmode & O_CREAT) && !meta.exists()) { - ++filesCreated; - replicator->updateMetadata(meta); // this will end up creating filename - } - if ((openmode & O_TRUNC) && meta.exists()) - _truncate(filename, 0, s.get()); - - ++filesOpened; - return meta.stat(out); + if (openmode & O_CREAT || openmode & O_TRUNC) + s.reset(new ScopedWriteLock(this, filename.string())); + else + s.reset(new ScopedReadLock(this, filename.string())); + + MetadataFile meta(filename, MetadataFile::no_create_t(), true); + + if ((openmode & O_CREAT) && !meta.exists()) + { + ++filesCreated; + replicator->updateMetadata(meta); // this will end up creating filename + } + if ((openmode & O_TRUNC) && meta.exists()) + _truncate(filename, 0, s.get()); + + ++filesOpened; + return meta.stat(out); } -int IOCoordinator::listDirectory(const char *dirname, vector *listing) +int IOCoordinator::listDirectory(const char* dirname, vector* listing) { - bf::path p(metaPath / ownership.get(dirname, false)); - ++listingCount; - - listing->clear(); - if (!bf::exists(p)) - { - errno = ENOENT; - return -1; - } - if (!bf::is_directory(p)) - { - errno = ENOTDIR; - return -1; - } - - bf::directory_iterator end; - for (bf::directory_iterator it(p); it != end; it++) - { - if (bf::is_directory(it->path())) - listing->push_back(it->path().filename().string()); - else if (it->path().extension() == ".meta") - listing->push_back(it->path().stem().string()); - } + bf::path p(metaPath / ownership.get(dirname, false)); + ++listingCount; + + listing->clear(); + if (!bf::exists(p)) + { + errno = ENOENT; + return -1; + } + if (!bf::is_directory(p)) + { + errno = ENOTDIR; + return -1; + } + + bf::directory_iterator end; + for (bf::directory_iterator it(p); it != end; it++) + { + if (bf::is_directory(it->path())) + listing->push_back(it->path().filename().string()); + else if (it->path().extension() == ".meta") + listing->push_back(it->path().stem().string()); + } + return 0; +} + +int IOCoordinator::stat(const char* _path, struct stat* out) +{ + bf::path filename = ownership.get(_path, false); + + if (bf::is_directory(metaPath / filename)) + return ::stat((metaPath / filename).string().c_str(), out); + + ScopedReadLock s(this, filename.string()); + MetadataFile meta(filename, MetadataFile::no_create_t(), true); + return meta.stat(out); +} + +int IOCoordinator::truncate(const char* _path, size_t newSize) +{ + bf::path p = ownership.get(_path); + const char* path = p.string().c_str(); + + ScopedWriteLock lock(this, path); + return _truncate(p, newSize, &lock); +} + +int IOCoordinator::_truncate(const bf::path& bfpath, size_t newSize, ScopedFileLock* lock) +{ + /* + grab the write lock. + get the relevant metadata. + truncate the metadata. + tell replicator to write the new metadata + release the lock + + tell replicator to delete all of the objects that no longer exist & their journal files + tell cache they were deleted + tell synchronizer they were deleted + */ + const bf::path firstDir = *(bfpath.begin()); + + Synchronizer* synchronizer = + Synchronizer::get(); // needs to init sync here to break circular dependency... + + int err; + MetadataFile meta(bfpath, MetadataFile::no_create_t(), true); + if (!meta.exists()) + { + errno = ENOENT; + return -1; + } + + size_t filesize = meta.getLength(); + if (filesize == newSize) return 0; -} -int IOCoordinator::stat(const char *_path, struct stat *out) -{ - bf::path filename = ownership.get(_path, false); - - if (bf::is_directory(metaPath/filename)) - return ::stat((metaPath/filename).string().c_str(), out); - - ScopedReadLock s(this, filename.string()); - MetadataFile meta(filename, MetadataFile::no_create_t(),true); - return meta.stat(out); -} - -int IOCoordinator::truncate(const char *_path, size_t newSize) -{ - bf::path p = ownership.get(_path); - const char *path = p.string().c_str(); - - ScopedWriteLock lock(this, path); - return _truncate(p, newSize, &lock); -} - - -int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileLock *lock) -{ - /* - grab the write lock. - get the relevant metadata. - truncate the metadata. - tell replicator to write the new metadata - release the lock - - tell replicator to delete all of the objects that no longer exist & their journal files - tell cache they were deleted - tell synchronizer they were deleted - */ - const bf::path firstDir = *(bfpath.begin()); - - Synchronizer *synchronizer = Synchronizer::get(); // needs to init sync here to break circular dependency... - - int err; - MetadataFile meta(bfpath, MetadataFile::no_create_t(),true); - if (!meta.exists()) - { - errno = ENOENT; - return -1; - } - - size_t filesize = meta.getLength(); - if (filesize == newSize) - return 0; - - // extend the file, going to make IOC::write() do it - if (filesize < newSize) - { - uint8_t zero = 0; - err = _write(bfpath, &zero, newSize - 1, 1, firstDir); - lock->unlock(); - cache->doneWriting(firstDir); - if (err < 0) - return -1; - return 0; - } - - vector objects = meta.metadataRead(newSize, filesize - newSize); - - // truncate the file - if (newSize == objects[0].offset) - meta.removeEntry(objects[0].offset); - else - { - meta.updateEntryLength(objects[0].offset, newSize - objects[0].offset); - assert(utils::is_nonnegative(objects[0].offset) && objectSize > (newSize - objects[0].offset)); - } - for (uint i = 1; i < objects.size(); i++) - meta.removeEntry(objects[i].offset); - - err = replicator->updateMetadata(meta); - if (err) - return err; - //lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call - - uint i = (newSize == objects[0].offset ? 0 : 1); - vector deletedObjects; - for (; i < objects.size(); ++i) - { - int result = cache->ifExistsThenDelete(firstDir, objects[i].key); - if (result & 0x1) - replicator->remove(cachePath/firstDir/objects[i].key); - if (result & 0x2) - replicator->remove(journalPath/firstDir/(objects[i].key + ".journal")); - deletedObjects.push_back(objects[i].key); - } - if (!deletedObjects.empty()) - synchronizer->deletedObjects(firstDir, deletedObjects); - ++filesTruncated; + // extend the file, going to make IOC::write() do it + if (filesize < newSize) + { + uint8_t zero = 0; + err = _write(bfpath, &zero, newSize - 1, 1, firstDir); + lock->unlock(); + cache->doneWriting(firstDir); + if (err < 0) + return -1; return 0; -} + } + vector objects = meta.metadataRead(newSize, filesize - newSize); + // truncate the file + if (newSize == objects[0].offset) + meta.removeEntry(objects[0].offset); + else + { + meta.updateEntryLength(objects[0].offset, newSize - objects[0].offset); + assert(utils::is_nonnegative(objects[0].offset) && objectSize > (newSize - objects[0].offset)); + } + for (uint i = 1; i < objects.size(); i++) + meta.removeEntry(objects[i].offset); -void IOCoordinator::deleteMetaFile(const bf::path &file) -{ - /* - lock for writing - tell replicator to delete every object - tell cache they were deleted - tell synchronizer to delete them in cloud storage - */ - //cout << "deleteMetaFile called on " << file << endl; - - Synchronizer *synchronizer = Synchronizer::get(); + err = replicator->updateMetadata(meta); + if (err) + return err; + // lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call - ++filesDeleted; - - // this is kind of ugly. We need to lock on 'file' relative to metaPath, and without the .meta extension - string pita = file.string().substr(metaPath.string().length() + 1); // get rid of metapath - pita = pita.substr(0, pita.length() - 5); // get rid of the extension - const bf::path firstDir = *(bf::path(pita).begin()); - ScopedWriteLock lock(this, pita); - //cout << "file is " << file.string() << " locked on " << pita << endl; - - MetadataFile meta(file,MetadataFile::no_create_t(),false); - replicator->remove(file); - //lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call - - vector objects = meta.metadataRead(0, meta.getLength()); - vector deletedObjects; - for (auto &object : objects) - { - //cout << "deleting " << object.key << endl; - int result = cache->ifExistsThenDelete(firstDir, object.key); - if (result & 0x1) { - ++iocFilesDeleted; - replicator->remove(cachePath/firstDir/object.key); - } - if (result & 0x2) { - ++iocFilesDeleted; - replicator->remove(journalPath/firstDir/(object.key + ".journal")); - } - deletedObjects.push_back(object.key); - } + uint i = (newSize == objects[0].offset ? 0 : 1); + vector deletedObjects; + for (; i < objects.size(); ++i) + { + int result = cache->ifExistsThenDelete(firstDir, objects[i].key); + if (result & 0x1) + replicator->remove(cachePath / firstDir / objects[i].key); + if (result & 0x2) + replicator->remove(journalPath / firstDir / (objects[i].key + ".journal")); + deletedObjects.push_back(objects[i].key); + } + if (!deletedObjects.empty()) synchronizer->deletedObjects(firstDir, deletedObjects); - MetadataFile::deletedMeta(file); + ++filesTruncated; + return 0; } -void IOCoordinator::remove(const bf::path &p) +void IOCoordinator::deleteMetaFile(const bf::path& file) { - // recurse on dirs - if (bf::is_directory(p)) + /* + lock for writing + tell replicator to delete every object + tell cache they were deleted + tell synchronizer to delete them in cloud storage + */ + // cout << "deleteMetaFile called on " << file << endl; + + Synchronizer* synchronizer = Synchronizer::get(); + + ++filesDeleted; + + // this is kind of ugly. We need to lock on 'file' relative to metaPath, and without the .meta extension + string pita = file.string().substr(metaPath.string().length() + 1); // get rid of metapath + pita = pita.substr(0, pita.length() - 5); // get rid of the extension + const bf::path firstDir = *(bf::path(pita).begin()); + ScopedWriteLock lock(this, pita); + // cout << "file is " << file.string() << " locked on " << pita << endl; + + MetadataFile meta(file, MetadataFile::no_create_t(), false); + replicator->remove(file); + // lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call + + vector objects = meta.metadataRead(0, meta.getLength()); + vector deletedObjects; + for (auto& object : objects) + { + // cout << "deleting " << object.key << endl; + int result = cache->ifExistsThenDelete(firstDir, object.key); + if (result & 0x1) { - bf::directory_iterator dend; - bf::directory_iterator entry(p); - while (entry != dend) - { - remove(*entry); - ++entry; - } - //cout << "removing dir " << p << endl; - replicator->remove(p); - return; + ++iocFilesDeleted; + replicator->remove(cachePath / firstDir / object.key); } - - // if p is a metadata file call deleteMetaFile - if (p.extension() == ".meta" && bf::is_regular_file(p)) - deleteMetaFile(p); - else + if (result & 0x2) { - // if we were passed a 'logical' file, it needs to have the meta extension added - bf::path possibleMetaFile = p.string() + ".meta"; - if (bf::is_regular_file(possibleMetaFile)) - deleteMetaFile(possibleMetaFile); - else if (bf::exists(p)) - replicator->remove(p); // if p.meta doesn't exist, and it's not a dir, then just throw it out + ++iocFilesDeleted; + replicator->remove(journalPath / firstDir / (object.key + ".journal")); } + deletedObjects.push_back(object.key); + } + synchronizer->deletedObjects(firstDir, deletedObjects); + MetadataFile::deletedMeta(file); +} + +void IOCoordinator::remove(const bf::path& p) +{ + // recurse on dirs + if (bf::is_directory(p)) + { + bf::directory_iterator dend; + bf::directory_iterator entry(p); + while (entry != dend) + { + remove(*entry); + ++entry; + } + // cout << "removing dir " << p << endl; + replicator->remove(p); + return; + } + + // if p is a metadata file call deleteMetaFile + if (p.extension() == ".meta" && bf::is_regular_file(p)) + deleteMetaFile(p); + else + { + // if we were passed a 'logical' file, it needs to have the meta extension added + bf::path possibleMetaFile = p.string() + ".meta"; + if (bf::is_regular_file(possibleMetaFile)) + deleteMetaFile(possibleMetaFile); + else if (bf::exists(p)) + replicator->remove(p); // if p.meta doesn't exist, and it's not a dir, then just throw it out + } } /* Need to rename this one. The corresponding fcn in IDBFileSystem specifies that it deletes files, but also entire directories, like an 'rm -rf' command. Implementing that here will no longer make it like the 'unlink' syscall. */ -int IOCoordinator::unlink(const char *path) +int IOCoordinator::unlink(const char* path) { - /* recursively iterate over path - open every metadata file - lock for writing - tell replicator to delete every object - tell cache they were deleted - tell synchronizer to delete them in cloud storage - */ - - /* TODO! We need to make sure the input params to IOC fcns don't go up to parent dirs, - ex, if path = '../../../blahblah'. */ - bf::path p(metaPath/ownership.get(path)); - - try - { - remove(p); - } - catch (bf::filesystem_error &e) - { - cout << "IOC::unlink caught an error: " << e.what() << endl; - errno = e.code().value(); - return -1; - } - return 0; + /* recursively iterate over path + open every metadata file + lock for writing + tell replicator to delete every object + tell cache they were deleted + tell synchronizer to delete them in cloud storage + */ + + /* TODO! We need to make sure the input params to IOC fcns don't go up to parent dirs, + ex, if path = '../../../blahblah'. */ + bf::path p(metaPath / ownership.get(path)); + + try + { + remove(p); + } + catch (bf::filesystem_error& e) + { + cout << "IOC::unlink caught an error: " << e.what() << endl; + errno = e.code().value(); + return -1; + } + return 0; } // a helper to de-uglify error handling in copyFile struct CFException { - CFException(int err, const string &logEntry) : l_errno(err), entry(logEntry) { } - int l_errno; - string entry; + CFException(int err, const string& logEntry) : l_errno(err), entry(logEntry) + { + } + int l_errno; + string entry; }; - -int IOCoordinator::copyFile(const char *_filename1, const char *_filename2) -{ - /* - if filename2 exists, delete it - if filename1 does not exist return ENOENT - if filename1 is not a meta file return EINVAL - - get a new metadata object - get a read lock on filename1 - - for every object in filename1 - get a new key for the object - tell cloudstorage to copy obj1 to obj2 - if it errors out b/c obj1 doesn't exist - upload it with the new name from the cache - copy the journal file if any - add the new key to the new metadata - - on error, delete all of the newly created objects - - write the new metadata object - */ - - const bf::path p1 = ownership.get(_filename1); - const bf::path p2 = ownership.get(_filename2); - const bf::path firstDir1 = *(p1.begin()); - const bf::path firstDir2 = *(p2.begin()); - const char *filename1 = p1.string().c_str(); - const char *filename2 = p2.string().c_str(); - - CloudStorage *cs = CloudStorage::get(); - Synchronizer *sync = Synchronizer::get(); - bf::path metaFile1 = metaPath/(p1.string() + ".meta"); - bf::path metaFile2 = metaPath/(p2.string() + ".meta"); - int err; - char errbuf[80]; - - if (!bf::exists(metaFile1)) - { - errno = ENOENT; - return -1; - } - if (bf::exists(metaFile2)) - { - deleteMetaFile(metaFile2); - ++filesDeleted; - } - - // since we don't implement mkdir(), assume the caller did that and - // create any necessary parent dirs for filename2 - try - { - bf::create_directories(metaFile2.parent_path()); - } - catch(bf::filesystem_error &e) - { - logger->log(LOG_CRIT, "IOCoordinator::copyFile(): failed to create directory %s. Got %s", - metaFile2.parent_path().string().c_str(), strerror_r(e.code().value(), errbuf, 80)); - errno = e.code().value(); - return -1; - } - vector > newJournalEntries; - ScopedReadLock lock(this, filename1); - ScopedWriteLock lock2(this, filename2); - MetadataFile meta1(metaFile1, MetadataFile::no_create_t(),false); - MetadataFile meta2(metaFile2, MetadataFile::no_create_t(),false); - vector objects = meta1.metadataRead(0, meta1.getLength()); - bytesCopied += meta1.getLength(); - - if (meta2.exists()) +int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) +{ + /* + if filename2 exists, delete it + if filename1 does not exist return ENOENT + if filename1 is not a meta file return EINVAL + + get a new metadata object + get a read lock on filename1 + + for every object in filename1 + get a new key for the object + tell cloudstorage to copy obj1 to obj2 + if it errors out b/c obj1 doesn't exist + upload it with the new name from the cache + copy the journal file if any + add the new key to the new metadata + + on error, delete all of the newly created objects + + write the new metadata object + */ + + const bf::path p1 = ownership.get(_filename1); + const bf::path p2 = ownership.get(_filename2); + const bf::path firstDir1 = *(p1.begin()); + const bf::path firstDir2 = *(p2.begin()); + const char* filename1 = p1.string().c_str(); + const char* filename2 = p2.string().c_str(); + + CloudStorage* cs = CloudStorage::get(); + Synchronizer* sync = Synchronizer::get(); + bf::path metaFile1 = metaPath / (p1.string() + ".meta"); + bf::path metaFile2 = metaPath / (p2.string() + ".meta"); + int err; + char errbuf[80]; + + if (!bf::exists(metaFile1)) + { + errno = ENOENT; + return -1; + } + if (bf::exists(metaFile2)) + { + deleteMetaFile(metaFile2); + ++filesDeleted; + } + + // since we don't implement mkdir(), assume the caller did that and + // create any necessary parent dirs for filename2 + try + { + bf::create_directories(metaFile2.parent_path()); + } + catch (bf::filesystem_error& e) + { + logger->log(LOG_CRIT, "IOCoordinator::copyFile(): failed to create directory %s. Got %s", + metaFile2.parent_path().string().c_str(), strerror_r(e.code().value(), errbuf, 80)); + errno = e.code().value(); + return -1; + } + + vector > newJournalEntries; + ScopedReadLock lock(this, filename1); + ScopedWriteLock lock2(this, filename2); + MetadataFile meta1(metaFile1, MetadataFile::no_create_t(), false); + MetadataFile meta2(metaFile2, MetadataFile::no_create_t(), false); + vector objects = meta1.metadataRead(0, meta1.getLength()); + bytesCopied += meta1.getLength(); + + if (meta2.exists()) + { + meta2.removeAllEntries(); + ++filesDeleted; + } + + // TODO. I dislike large try-catch blocks, and large loops. Maybe a little refactoring is in order. + try + { + for (const auto& object : objects) { - meta2.removeAllEntries(); - ++filesDeleted; - } - - // TODO. I dislike large try-catch blocks, and large loops. Maybe a little refactoring is in order. - try - { - for (const auto &object : objects) + bf::path journalFile = journalPath / firstDir1 / (object.key + ".journal"); + + // originalLength = the length of the object before journal entries. + // the length in the metadata is the length after journal entries + size_t originalLength = MetadataFile::getLengthFromKey(object.key); + metadataObject newObj = meta2.addMetadataObject(filename2, originalLength); + if (originalLength != object.length) + meta2.updateEntryLength(newObj.offset, object.length); + err = cs->copyObject(object.key, newObj.key); + if (err) + { + if (errno == ENOENT) { - bf::path journalFile = journalPath/firstDir1/(object.key + ".journal"); - - // originalLength = the length of the object before journal entries. - // the length in the metadata is the length after journal entries - size_t originalLength = MetadataFile::getLengthFromKey(object.key); - metadataObject newObj = meta2.addMetadataObject(filename2, originalLength); - if (originalLength != object.length) - meta2.updateEntryLength(newObj.offset, object.length); - err = cs->copyObject(object.key, newObj.key); - if (err) - { - if (errno == ENOENT) - { - // it's not in cloudstorage, see if it's in the cache - bf::path cachedObjPath = cachePath/firstDir1/object.key; - bool objExists = bf::exists(cachedObjPath); - if (!objExists) - throw CFException(ENOENT, string("IOCoordinator::copyFile(): source = ") + filename1 + - ", dest = " + filename2 + ". Object " + object.key + " does not exist in either " - "cloud storage or the cache!"); + // it's not in cloudstorage, see if it's in the cache + bf::path cachedObjPath = cachePath / firstDir1 / object.key; + bool objExists = bf::exists(cachedObjPath); + if (!objExists) + throw CFException(ENOENT, string("IOCoordinator::copyFile(): source = ") + filename1 + + ", dest = " + filename2 + ". Object " + object.key + + " does not exist in either " + "cloud storage or the cache!"); - // put the copy in cloudstorage - err = cs->putObject(cachedObjPath.string(), newObj.key); - if (err) - throw CFException(errno, string("IOCoordinator::copyFile(): source = ") + filename1 + - ", dest = " + filename2 + ". Got an error uploading object " + object.key + ": " + - strerror_r(errno, errbuf, 80)); - } - else // the problem was something other than it not existing in cloud storage - throw CFException(errno, string("IOCoordinator::copyFile(): source = ") + filename1 + - ", dest = " + filename2 + ". Got an error copying object " + object.key + ": " + - strerror_r(errno, errbuf, 80)); - } - - - // if there's a journal file for this object, make a copy - if (bf::exists(journalFile)) - { - bf::path newJournalFile = journalPath/firstDir2/(newObj.key + ".journal"); - try - { - bf::copy_file(journalFile, newJournalFile); - size_t tmp = bf::file_size(newJournalFile); - ++iocJournalsCreated; - iocBytesRead += tmp; - iocBytesWritten += tmp; - cache->newJournalEntry(firstDir2, tmp); - newJournalEntries.push_back(pair(newObj.key, tmp)); - } - catch (bf::filesystem_error &e) - { - throw CFException(e.code().value(), string("IOCoordinator::copyFile(): source = ") + filename1 + - ", dest = " + filename2 + ". Got an error copying " + journalFile.string() + ": " + - strerror_r(e.code().value(), errbuf, 80)); - } - } + // put the copy in cloudstorage + err = cs->putObject(cachedObjPath.string(), newObj.key); + if (err) + throw CFException(errno, string("IOCoordinator::copyFile(): source = ") + filename1 + + ", dest = " + filename2 + ". Got an error uploading object " + + object.key + ": " + strerror_r(errno, errbuf, 80)); } - } - catch (CFException &e) - { - logger->log(LOG_CRIT, e.entry.c_str()); - for (const auto &newObject : meta2.metadataRead(0, meta2.getLength())) - cs->deleteObject(newObject.key); - for (auto &jEntry : newJournalEntries) + else // the problem was something other than it not existing in cloud storage + throw CFException(errno, string("IOCoordinator::copyFile(): source = ") + filename1 + + ", dest = " + filename2 + ". Got an error copying object " + + object.key + ": " + strerror_r(errno, errbuf, 80)); + } + + // if there's a journal file for this object, make a copy + if (bf::exists(journalFile)) + { + bf::path newJournalFile = journalPath / firstDir2 / (newObj.key + ".journal"); + try { - bf::path fullJournalPath = journalPath/firstDir2/(jEntry.first + ".journal"); - cache->deletedJournal(firstDir2, bf::file_size(fullJournalPath)); - bf::remove(fullJournalPath); + bf::copy_file(journalFile, newJournalFile); + size_t tmp = bf::file_size(newJournalFile); + ++iocJournalsCreated; + iocBytesRead += tmp; + iocBytesWritten += tmp; + cache->newJournalEntry(firstDir2, tmp); + newJournalEntries.push_back(pair(newObj.key, tmp)); } - errno = e.l_errno; - return -1; + catch (bf::filesystem_error& e) + { + throw CFException(e.code().value(), string("IOCoordinator::copyFile(): source = ") + filename1 + + ", dest = " + filename2 + ". Got an error copying " + + journalFile.string() + ": " + + strerror_r(e.code().value(), errbuf, 80)); + } + } } - lock.unlock(); - replicator->updateMetadata(meta2); - lock2.unlock(); - - for (auto &jEntry : newJournalEntries) - sync->newJournalEntry(firstDir2, jEntry.first, jEntry.second); - ++filesCopied; - return 0; + } + catch (CFException& e) + { + logger->log(LOG_CRIT, e.entry.c_str()); + for (const auto& newObject : meta2.metadataRead(0, meta2.getLength())) + cs->deleteObject(newObject.key); + for (auto& jEntry : newJournalEntries) + { + bf::path fullJournalPath = journalPath / firstDir2 / (jEntry.first + ".journal"); + cache->deletedJournal(firstDir2, bf::file_size(fullJournalPath)); + bf::remove(fullJournalPath); + } + errno = e.l_errno; + return -1; + } + lock.unlock(); + replicator->updateMetadata(meta2); + lock2.unlock(); + + for (auto& jEntry : newJournalEntries) + sync->newJournalEntry(firstDir2, jEntry.first, jEntry.second); + ++filesCopied; + return 0; } -const bf::path &IOCoordinator::getCachePath() const +const bf::path& IOCoordinator::getCachePath() const { - return cachePath; + return cachePath; } -const bf::path &IOCoordinator::getJournalPath() const +const bf::path& IOCoordinator::getJournalPath() const { - return journalPath; + return journalPath; } -const bf::path &IOCoordinator::getMetadataPath() const +const bf::path& IOCoordinator::getMetadataPath() const { - return metaPath; + return metaPath; } // this is not generic by any means. This is assuming a version 1 journal header, and is looking -// for the end of it, which is the first \0 char. It returns with fd pointing at the +// for the end of it, which is the first \0 char. It returns with fd pointing at the // first byte after the header. // update: had to make it also return the header; the boost json parser does not stop at either // a null char or the end of an object. -boost::shared_array seekToEndOfHeader1(int fd, size_t *_bytesRead) +boost::shared_array seekToEndOfHeader1(int fd, size_t* _bytesRead) { - //::lseek(fd, 0, SEEK_SET); - boost::shared_array ret(new char[100]); - int err; - - err = ::read(fd, ret.get(), 100); + //::lseek(fd, 0, SEEK_SET); + boost::shared_array ret(new char[100]); + int err; + + err = ::read(fd, ret.get(), 100); + if (err < 0) + { + char buf[80]; + throw runtime_error("seekToEndOfHeader1 got: " + string(strerror_r(errno, buf, 80))); + } + for (int i = 0; i < err; i++) + { + if (ret[i] == 0) + { + ::lseek(fd, i + 1, SEEK_SET); + *_bytesRead = i + 1; + return ret; + } + } + throw runtime_error("seekToEndOfHeader1: did not find the end of the header"); +} + +int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t offset, size_t* len) const +{ + throw runtime_error("IOCoordinator::mergeJournal(int, int, etc) is not implemented yet."); +} + +boost::shared_array IOCoordinator::mergeJournal(const char* object, const char* journal, + off_t offset, size_t len, + size_t* _bytesReadOut) const +{ + int objFD, journalFD; + boost::shared_array ret; + size_t l_bytesRead = 0; + + objFD = ::open(object, O_RDONLY); + if (objFD < 0) + { + *_bytesReadOut = 0; + return ret; + } + ScopedCloser s1(objFD); + + ret.reset(new uint8_t[len]); + + // read the object into memory + size_t count = 0; + if (offset != 0) + ::lseek(objFD, offset, SEEK_SET); + while (count < len) + { + int err = ::read(objFD, &ret[count], len - count); if (err < 0) { - char buf[80]; - throw runtime_error("seekToEndOfHeader1 got: " + string(strerror_r(errno, buf, 80))); + int l_errno = errno; + char buf[80]; + logger->log(LOG_CRIT, "IOC::mergeJournal(): failed to read %s, got '%s'", object, + strerror_r(l_errno, buf, 80)); + ret.reset(); + errno = l_errno; + *_bytesReadOut = count; + return ret; } - for (int i = 0; i < err; i++) + else if (err == 0) { - if (ret[i] == 0) - { - ::lseek(fd, i+1, SEEK_SET); - *_bytesRead = i + 1; - return ret; - } + // at the EOF of the object. The journal may contain entries that append to the data, + break; } - throw runtime_error("seekToEndOfHeader1: did not find the end of the header"); -} + count += err; + } + l_bytesRead += count; -int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t *buf, off_t offset, size_t *len) const -{ - throw runtime_error("IOCoordinator::mergeJournal(int, int, etc) is not implemented yet."); -} - -boost::shared_array IOCoordinator::mergeJournal(const char *object, const char *journal, off_t offset, - size_t len, size_t *_bytesReadOut) const -{ - int objFD, journalFD; - boost::shared_array ret; - size_t l_bytesRead = 0; - - objFD = ::open(object, O_RDONLY); - if (objFD < 0) - { - *_bytesReadOut = 0; - return ret; - } - ScopedCloser s1(objFD); - - ret.reset(new uint8_t[len]); - - // read the object into memory - size_t count = 0; - if (offset != 0) - ::lseek(objFD, offset, SEEK_SET); - while (count < len) { - int err = ::read(objFD, &ret[count], len - count); - if (err < 0) - { - int l_errno = errno; - char buf[80]; - logger->log(LOG_CRIT, "IOC::mergeJournal(): failed to read %s, got '%s'", object, strerror_r(l_errno, buf, 80)); - ret.reset(); - errno = l_errno; - *_bytesReadOut = count; - return ret; - } - else if (err == 0) - { - // at the EOF of the object. The journal may contain entries that append to the data, - break; - } - count += err; - } - l_bytesRead += count; - - // mergeJournalInMem has lower a IOPS requirement than the fully general code in this fcn. Use - // that if the caller requested the whole object to be merged - if (offset == 0 && (ssize_t) len >= ::lseek(objFD, 0, SEEK_END)) - { - size_t mjimBytesRead = 0; - int mjimerr = mergeJournalInMem(ret, len, journal, &mjimBytesRead); - if (mjimerr) - ret.reset(); - l_bytesRead += mjimBytesRead; - *_bytesReadOut = l_bytesRead; - return ret; - } - - journalFD = ::open(journal, O_RDONLY); - if (journalFD < 0) - { - *_bytesReadOut = l_bytesRead; - return ret; - } - ScopedCloser s2(journalFD); - - boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); - stringstream ss; - ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); - - // start processing the entries - while (1) - { - uint64_t offlen[2]; - int err = ::read(journalFD, &offlen, 16); - if (err == 0) // got EOF - break; - assert(err == 16); - l_bytesRead += 16; - - // if this entry overlaps, read the overlapping section - uint64_t lastJournalOffset = offlen[0] + offlen[1]; - uint64_t lastBufOffset = offset + len; - if (offlen[0] <= lastBufOffset && lastJournalOffset >= (uint64_t) offset) - { - uint64_t startReadingAt = max(offlen[0], (uint64_t) offset); - uint64_t lengthOfRead = min(lastBufOffset, lastJournalOffset) - startReadingAt; - - // seek to the portion of the entry to start reading at - if (startReadingAt != offlen[0]) - ::lseek(journalFD, startReadingAt - offlen[0], SEEK_CUR); - - uint count = 0; - while (count < lengthOfRead) - { - err = ::read(journalFD, &ret[startReadingAt - offset + count], lengthOfRead - count); - if (err < 0) - { - int l_errno = errno; - char buf[80]; - logger->log(LOG_ERR, "mergeJournal: got %s", strerror_r(l_errno, buf, 80)); - ret.reset(); - errno = l_errno; - l_bytesRead += count; - goto out; - } - else if (err == 0) - { - logger->log(LOG_ERR, "mergeJournal: got early EOF. offset=%ld, len=%ld, jOffset=%ld, jLen=%ld," - " startReadingAt=%ld, lengthOfRead=%ld", offset, len, offlen[0], offlen[1], startReadingAt, lengthOfRead); - ret.reset(); - l_bytesRead += count; - goto out; - } - count += err; - } - l_bytesRead += lengthOfRead; - - // advance the file pos if we didn't read to the end of the entry - if (startReadingAt - offlen[0] + lengthOfRead != offlen[1]) - ::lseek(journalFD, offlen[1] - (lengthOfRead + startReadingAt - offlen[0]), SEEK_CUR); - } - else - // skip over this journal entry - ::lseek(journalFD, offlen[1], SEEK_CUR); - } -out: + // mergeJournalInMem has lower a IOPS requirement than the fully general code in this fcn. Use + // that if the caller requested the whole object to be merged + if (offset == 0 && (ssize_t)len >= ::lseek(objFD, 0, SEEK_END)) + { + size_t mjimBytesRead = 0; + int mjimerr = mergeJournalInMem(ret, len, journal, &mjimBytesRead); + if (mjimerr) + ret.reset(); + l_bytesRead += mjimBytesRead; *_bytesReadOut = l_bytesRead; return ret; + } + + journalFD = ::open(journal, O_RDONLY); + if (journalFD < 0) + { + *_bytesReadOut = l_bytesRead; + return ret; + } + ScopedCloser s2(journalFD); + + boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); + stringstream ss; + ss << headertxt.get(); + boost::property_tree::ptree header; + boost::property_tree::json_parser::read_json(ss, header); + assert(header.get("version") == 1); + + // start processing the entries + while (1) + { + uint64_t offlen[2]; + int err = ::read(journalFD, &offlen, 16); + if (err == 0) // got EOF + break; + assert(err == 16); + l_bytesRead += 16; + + // if this entry overlaps, read the overlapping section + uint64_t lastJournalOffset = offlen[0] + offlen[1]; + uint64_t lastBufOffset = offset + len; + if (offlen[0] <= lastBufOffset && lastJournalOffset >= (uint64_t)offset) + { + uint64_t startReadingAt = max(offlen[0], (uint64_t)offset); + uint64_t lengthOfRead = min(lastBufOffset, lastJournalOffset) - startReadingAt; + + // seek to the portion of the entry to start reading at + if (startReadingAt != offlen[0]) + ::lseek(journalFD, startReadingAt - offlen[0], SEEK_CUR); + + uint count = 0; + while (count < lengthOfRead) + { + err = ::read(journalFD, &ret[startReadingAt - offset + count], lengthOfRead - count); + if (err < 0) + { + int l_errno = errno; + char buf[80]; + logger->log(LOG_ERR, "mergeJournal: got %s", strerror_r(l_errno, buf, 80)); + ret.reset(); + errno = l_errno; + l_bytesRead += count; + goto out; + } + else if (err == 0) + { + logger->log(LOG_ERR, + "mergeJournal: got early EOF. offset=%ld, len=%ld, jOffset=%ld, jLen=%ld," + " startReadingAt=%ld, lengthOfRead=%ld", + offset, len, offlen[0], offlen[1], startReadingAt, lengthOfRead); + ret.reset(); + l_bytesRead += count; + goto out; + } + count += err; + } + l_bytesRead += lengthOfRead; + + // advance the file pos if we didn't read to the end of the entry + if (startReadingAt - offlen[0] + lengthOfRead != offlen[1]) + ::lseek(journalFD, offlen[1] - (lengthOfRead + startReadingAt - offlen[0]), SEEK_CUR); + } + else + // skip over this journal entry + ::lseek(journalFD, offlen[1], SEEK_CUR); + } +out: + *_bytesReadOut = l_bytesRead; + return ret; } // MergeJournalInMem is a specialized version of mergeJournal(). This is currently only used by Synchronizer // and mergeJournal(), and only for merging the whole object with the whole journal. -int IOCoordinator::mergeJournalInMem(boost::shared_array &objData, size_t len, const char *journalPath, - size_t *_bytesReadOut) const +int IOCoordinator::mergeJournalInMem(boost::shared_array& objData, size_t len, + const char* journalPath, size_t* _bytesReadOut) const { - // if the journal is over some size threshold (100MB for now why not), - // use the original low-mem-usage version - if (len > (100 << 20)) - return mergeJournalInMem_bigJ(objData, len, journalPath, _bytesReadOut); + // if the journal is over some size threshold (100MB for now why not), + // use the original low-mem-usage version + if (len > (100 << 20)) + return mergeJournalInMem_bigJ(objData, len, journalPath, _bytesReadOut); - size_t l_bytesRead = 0; - int journalFD = ::open(journalPath, O_RDONLY); - if (journalFD < 0) + size_t l_bytesRead = 0; + int journalFD = ::open(journalPath, O_RDONLY); + if (journalFD < 0) + return -1; + ScopedCloser s(journalFD); + + // grab the journal header and make sure the version is 1 + boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); + stringstream ss; + ss << headertxt.get(); + boost::property_tree::ptree header; + boost::property_tree::json_parser::read_json(ss, header); + assert(header.get("version") == 1); + + // read the journal file into memory + size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; + ::lseek(journalFD, l_bytesRead, SEEK_SET); + boost::scoped_array journalData(new uint8_t[journalBytes]); + size_t readCount = 0; + while (readCount < journalBytes) + { + ssize_t err = ::read(journalFD, &journalData[readCount], journalBytes - readCount); + if (err < 0) + { + char buf[80]; + int l_errno = errno; + logger->log(LOG_ERR, "mergeJournalInMem: got %s", strerror_r(errno, buf, 80)); + errno = l_errno; + return -1; + } + else if (err == 0) + { + logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); + errno = ENODATA; // is there a better errno for early EOF? + return -1; + } + readCount += err; + l_bytesRead += err; + } + + // start processing the entries + size_t offset = 0; + while (offset < journalBytes) + { + if (offset + 16 >= journalBytes) + { + logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); + errno = ENODATA; // is there a better errno for early EOF? + return -1; + } + uint64_t* offlen = (uint64_t*)&journalData[offset]; + offset += 16; + + uint64_t startReadingAt = offlen[0]; + uint64_t lengthOfRead = offlen[1]; + + if (startReadingAt > len) + { + offset += offlen[1]; + continue; + } + + if (startReadingAt + lengthOfRead > len) + lengthOfRead = len - startReadingAt; + if (offset + lengthOfRead > journalBytes) + { + logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); + errno = ENODATA; // is there a better errno for early EOF? + return -1; + } + memcpy(&objData[startReadingAt], &journalData[offset], lengthOfRead); + offset += offlen[1]; + } + *_bytesReadOut = l_bytesRead; + return 0; +} + +int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array& objData, size_t len, + const char* journalPath, size_t* _bytesReadOut) const +{ + size_t l_bytesRead = 0; + int journalFD = ::open(journalPath, O_RDONLY); + if (journalFD < 0) + return -1; + ScopedCloser s(journalFD); + + // grab the journal header and make sure the version is 1 + boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); + stringstream ss; + ss << headertxt.get(); + boost::property_tree::ptree header; + boost::property_tree::json_parser::read_json(ss, header); + assert(header.get("version") == 1); + + // start processing the entries + while (1) + { + uint64_t offlen[2]; + int err = ::read(journalFD, &offlen, 16); + if (err == 0) // got EOF + break; + else if (err < 16) + { + // punting on this + cout << "mergeJournalInMem: failed to read a journal entry header in one attempt. fixme..." << endl; + errno = ENODATA; + return -1; + } + l_bytesRead += 16; + + uint64_t startReadingAt = offlen[0]; + uint64_t lengthOfRead = offlen[1]; + + if (startReadingAt > len) + { + ::lseek(journalFD, offlen[1], SEEK_CUR); + continue; + } + + if (startReadingAt + lengthOfRead > len) + lengthOfRead = len - startReadingAt; + + uint count = 0; + while (count < lengthOfRead) + { + err = ::read(journalFD, &objData[startReadingAt + count], lengthOfRead - count); + if (err < 0) + { + char buf[80]; + int l_errno = errno; + logger->log(LOG_ERR, "mergeJournalInMem: got %s", strerror_r(errno, buf, 80)); + errno = l_errno; return -1; - ScopedCloser s(journalFD); - - // grab the journal header and make sure the version is 1 - boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); - stringstream ss; - ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); - - // read the journal file into memory - size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; - ::lseek(journalFD, l_bytesRead, SEEK_SET); - boost::scoped_array journalData(new uint8_t[journalBytes]); - size_t readCount = 0; - while (readCount < journalBytes) - { - ssize_t err = ::read(journalFD, &journalData[readCount], journalBytes - readCount); - if (err < 0) - { - char buf[80]; - int l_errno = errno; - logger->log(LOG_ERR, "mergeJournalInMem: got %s", strerror_r(errno, buf, 80)); - errno = l_errno; - return -1; - } - else if (err == 0) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - readCount += err; - l_bytesRead += err; - } - - // start processing the entries - size_t offset = 0; - while (offset < journalBytes) - { - if (offset + 16 >= journalBytes) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - uint64_t *offlen = (uint64_t *) &journalData[offset]; - offset += 16; - - uint64_t startReadingAt = offlen[0]; - uint64_t lengthOfRead = offlen[1]; - - if (startReadingAt > len) - { - offset += offlen[1]; - continue; - } - - if (startReadingAt + lengthOfRead > len) - lengthOfRead = len - startReadingAt; - if (offset + lengthOfRead > journalBytes) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - memcpy(&objData[startReadingAt], &journalData[offset], lengthOfRead); - offset += offlen[1]; - } - *_bytesReadOut = l_bytesRead; - return 0; -} - -int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array &objData, size_t len, const char *journalPath, - size_t *_bytesReadOut) const -{ - size_t l_bytesRead = 0; - int journalFD = ::open(journalPath, O_RDONLY); - if (journalFD < 0) + } + else if (err == 0) + { + logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); + errno = ENODATA; // is there a better errno for early EOF? return -1; - ScopedCloser s(journalFD); - - // grab the journal header and make sure the version is 1 - boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); - stringstream ss; - ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); - - // start processing the entries - while (1) - { - uint64_t offlen[2]; - int err = ::read(journalFD, &offlen, 16); - if (err == 0) // got EOF - break; - else if (err < 16) - { - // punting on this - cout << "mergeJournalInMem: failed to read a journal entry header in one attempt. fixme..." << endl; - errno = ENODATA; - return -1; - } - l_bytesRead += 16; - - uint64_t startReadingAt = offlen[0]; - uint64_t lengthOfRead = offlen[1]; - - if (startReadingAt > len) - { - ::lseek(journalFD, offlen[1], SEEK_CUR); - continue; - } - - if (startReadingAt + lengthOfRead > len) - lengthOfRead = len - startReadingAt; - - uint count = 0; - while (count < lengthOfRead) - { - err = ::read(journalFD, &objData[startReadingAt + count], lengthOfRead - count); - if (err < 0) - { - char buf[80]; - int l_errno = errno; - logger->log(LOG_ERR, "mergeJournalInMem: got %s", strerror_r(errno, buf, 80)); - errno = l_errno; - return -1; - } - else if (err == 0) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - count += err; - } - l_bytesRead += lengthOfRead; - if (lengthOfRead < offlen[1]) - ::lseek(journalFD, offlen[1] - lengthOfRead, SEEK_CUR); + } + count += err; } - *_bytesReadOut = l_bytesRead; - return 0; + l_bytesRead += lengthOfRead; + if (lengthOfRead < offlen[1]) + ::lseek(journalFD, offlen[1] - lengthOfRead, SEEK_CUR); + } + *_bytesReadOut = l_bytesRead; + return 0; } -void IOCoordinator::readLock(const string &filename) +void IOCoordinator::readLock(const string& filename) { - boost::unique_lock s(lockMutex); + boost::unique_lock s(lockMutex); - //cout << "read-locking " << filename << endl; - assert(filename[0] != '/'); - auto ins = locks.insert(pair(filename, NULL)); - if (ins.second) - ins.first->second = new RWLock(); - ins.first->second->readLock(s); + // cout << "read-locking " << filename << endl; + assert(filename[0] != '/'); + auto ins = locks.insert(pair(filename, NULL)); + if (ins.second) + ins.first->second = new RWLock(); + ins.first->second->readLock(s); } -void IOCoordinator::readUnlock(const string &filename) +void IOCoordinator::readUnlock(const string& filename) { - boost::unique_lock s(lockMutex); - - auto it = locks.find(filename); - it->second->readUnlock(); - if (!it->second->inUse()) - { - delete it->second; - locks.erase(it); - } + boost::unique_lock s(lockMutex); + + auto it = locks.find(filename); + it->second->readUnlock(); + if (!it->second->inUse()) + { + delete it->second; + locks.erase(it); + } } -void IOCoordinator::writeLock(const string &filename) +void IOCoordinator::writeLock(const string& filename) { - boost::unique_lock s(lockMutex); - - //cout << "write-locking " << filename << endl; - assert(filename[0] != '/'); - auto ins = locks.insert(pair(filename, NULL)); - if (ins.second) - ins.first->second = new RWLock(); - ins.first->second->writeLock(s); + boost::unique_lock s(lockMutex); + + // cout << "write-locking " << filename << endl; + assert(filename[0] != '/'); + auto ins = locks.insert(pair(filename, NULL)); + if (ins.second) + ins.first->second = new RWLock(); + ins.first->second->writeLock(s); } -void IOCoordinator::writeUnlock(const string &filename) +void IOCoordinator::writeUnlock(const string& filename) { - boost::unique_lock s(lockMutex); - - auto it = locks.find(filename); - it->second->writeUnlock(); - if (!it->second->inUse()) - { - delete it->second; - locks.erase(it); - } + boost::unique_lock s(lockMutex); + + auto it = locks.find(filename); + it->second->writeUnlock(); + if (!it->second->inUse()) + { + delete it->second; + locks.erase(it); + } } -} +} // namespace storagemanager diff --git a/storage-manager/src/IOCoordinator.h b/storage-manager/src/IOCoordinator.h index 1783f6a76..8d2588fc7 100644 --- a/storage-manager/src/IOCoordinator.h +++ b/storage-manager/src/IOCoordinator.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef IOCOORDINATOR_H_ #define IOCOORDINATOR_H_ @@ -39,98 +38,96 @@ namespace storagemanager { - -boost::shared_array seekToEndOfHeader1(int fd, size_t *bytesRead); +boost::shared_array seekToEndOfHeader1(int fd, size_t* bytesRead); class IOCoordinator : public boost::noncopyable { - public: - static IOCoordinator *get(); - virtual ~IOCoordinator(); + public: + static IOCoordinator* get(); + virtual ~IOCoordinator(); - ssize_t read(const char *filename, uint8_t *data, off_t offset, size_t length); - ssize_t write(const char *filename, const uint8_t *data, off_t offset, size_t length); - ssize_t append(const char *filename, const uint8_t *data, size_t length); - int open(const char *filename, int openmode, struct stat *out); - int listDirectory(const char *filename, std::vector *listing); - int stat(const char *path, struct stat *out); - int truncate(const char *path, size_t newsize); - int unlink(const char *path); - int copyFile(const char *filename1, const char *filename2); - - // The shared logic for merging a journal file with its base file. - // len should be set to the length of the data requested - boost::shared_array mergeJournal(const char *objectPath, const char *journalPath, off_t offset, - size_t len, size_t *sizeRead) const; - - // this version modifies object data in memory, given the journal filename. Processes the whole object - // and whole journal file. - int mergeJournalInMem(boost::shared_array &objData, size_t len, const char *journalPath, - size_t *sizeRead) const; - - // this version of MJIM has a higher IOPS requirement and lower mem usage. - int mergeJournalInMem_bigJ(boost::shared_array &objData, size_t len, const char *journalPath, - size_t *sizeRead) const; - - // this version takes already-open file descriptors, and an already-allocated buffer as input. - // file descriptor are positioned, eh, best not to assume anything about their positions - // on return. - // Not implemented yet. At the point of this writing, we will use the existing versions, even though - // it's wasteful, and will leave this as a likely future optimization. - int mergeJournal(int objFD, int journalFD, uint8_t *buf, off_t offset, size_t *len) const; - - /* Lock manipulation fcns. They can lock on any param given to them. For convention's sake, - the parameter should mostly be the abs filename being accessed. */ - void readLock(const std::string &filename); - void writeLock(const std::string &filename); - void readUnlock(const std::string &filename); - void writeUnlock(const std::string &filename); - - // some accessors... - const boost::filesystem::path &getCachePath() const; - const boost::filesystem::path &getJournalPath() const; - const boost::filesystem::path &getMetadataPath() const; - - void printKPIs() const; - - private: - IOCoordinator(); - Config *config; - Cache *cache; - SMLogging *logger; - Replicator *replicator; - Ownership ownership; // ACK! Need a new name for this! + ssize_t read(const char* filename, uint8_t* data, off_t offset, size_t length); + ssize_t write(const char* filename, const uint8_t* data, off_t offset, size_t length); + ssize_t append(const char* filename, const uint8_t* data, size_t length); + int open(const char* filename, int openmode, struct stat* out); + int listDirectory(const char* filename, std::vector* listing); + int stat(const char* path, struct stat* out); + int truncate(const char* path, size_t newsize); + int unlink(const char* path); + int copyFile(const char* filename1, const char* filename2); - size_t objectSize; - boost::filesystem::path journalPath; - boost::filesystem::path cachePath; - boost::filesystem::path metaPath; - - std::map locks; - boost::mutex lockMutex; // lol - - void remove(const boost::filesystem::path &path); - void deleteMetaFile(const boost::filesystem::path &file); + // The shared logic for merging a journal file with its base file. + // len should be set to the length of the data requested + boost::shared_array mergeJournal(const char* objectPath, const char* journalPath, off_t offset, + size_t len, size_t* sizeRead) const; - int _truncate(const boost::filesystem::path &path, size_t newsize, ScopedFileLock *lock); - ssize_t _write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length, - const boost::filesystem::path &firstDir); - - int loadObjectAndJournal(const char *objFilename, const char *journalFilename, - uint8_t *data, off_t offset, size_t length); - int loadObject(int fd, uint8_t *data, off_t offset, size_t length); - - // some KPIs - // from the user's POV... - size_t bytesRead, bytesWritten, filesOpened, filesCreated, filesCopied; - size_t filesDeleted, bytesCopied, filesTruncated, listingCount, callsToWrite; - - // from IOC's pov... - size_t iocFilesOpened, iocObjectsCreated, iocJournalsCreated, iocFilesDeleted; - size_t iocBytesRead, iocBytesWritten; + // this version modifies object data in memory, given the journal filename. Processes the whole object + // and whole journal file. + int mergeJournalInMem(boost::shared_array& objData, size_t len, const char* journalPath, + size_t* sizeRead) const; + + // this version of MJIM has a higher IOPS requirement and lower mem usage. + int mergeJournalInMem_bigJ(boost::shared_array& objData, size_t len, const char* journalPath, + size_t* sizeRead) const; + + // this version takes already-open file descriptors, and an already-allocated buffer as input. + // file descriptor are positioned, eh, best not to assume anything about their positions + // on return. + // Not implemented yet. At the point of this writing, we will use the existing versions, even though + // it's wasteful, and will leave this as a likely future optimization. + int mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t offset, size_t* len) const; + + /* Lock manipulation fcns. They can lock on any param given to them. For convention's sake, + the parameter should mostly be the abs filename being accessed. */ + void readLock(const std::string& filename); + void writeLock(const std::string& filename); + void readUnlock(const std::string& filename); + void writeUnlock(const std::string& filename); + + // some accessors... + const boost::filesystem::path& getCachePath() const; + const boost::filesystem::path& getJournalPath() const; + const boost::filesystem::path& getMetadataPath() const; + + void printKPIs() const; + + private: + IOCoordinator(); + Config* config; + Cache* cache; + SMLogging* logger; + Replicator* replicator; + Ownership ownership; // ACK! Need a new name for this! + + size_t objectSize; + boost::filesystem::path journalPath; + boost::filesystem::path cachePath; + boost::filesystem::path metaPath; + + std::map locks; + boost::mutex lockMutex; // lol + + void remove(const boost::filesystem::path& path); + void deleteMetaFile(const boost::filesystem::path& file); + + int _truncate(const boost::filesystem::path& path, size_t newsize, ScopedFileLock* lock); + ssize_t _write(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length, + const boost::filesystem::path& firstDir); + + int loadObjectAndJournal(const char* objFilename, const char* journalFilename, uint8_t* data, off_t offset, + size_t length); + int loadObject(int fd, uint8_t* data, off_t offset, size_t length); + + // some KPIs + // from the user's POV... + size_t bytesRead, bytesWritten, filesOpened, filesCreated, filesCopied; + size_t filesDeleted, bytesCopied, filesTruncated, listingCount, callsToWrite; + + // from IOC's pov... + size_t iocFilesOpened, iocObjectsCreated, iocJournalsCreated, iocFilesDeleted; + size_t iocBytesRead, iocBytesWritten; }; -} - +} // namespace storagemanager #endif diff --git a/storage-manager/src/ListDirectoryTask.cpp b/storage-manager/src/ListDirectoryTask.cpp index 3cd9efb43..3c327a229 100644 --- a/storage-manager/src/ListDirectoryTask.cpp +++ b/storage-manager/src/ListDirectoryTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "ListDirectoryTask.h" #include "SMLogging.h" #include "messageFormat.h" @@ -27,7 +26,6 @@ using namespace std; namespace storagemanager { - ListDirectoryTask::ListDirectoryTask(int sock, uint len) : PosixTask(sock, len) { } @@ -37,108 +35,110 @@ ListDirectoryTask::~ListDirectoryTask() } #define check_error(msg, ret) \ - if (!success) \ - { \ - handleError(msg, errno); \ - return ret; \ - } - + if (!success) \ + { \ + handleError(msg, errno); \ + return ret; \ + } + #define min(x, y) (x < y ? x : y) -bool ListDirectoryTask::writeString(uint8_t *buf, int *offset, int size, const string &str) +bool ListDirectoryTask::writeString(uint8_t* buf, int* offset, int size, const string& str) { - bool success; - if (size - *offset < 4) // eh, let's not frag 4 bytes. + bool success; + if (size - *offset < 4) // eh, let's not frag 4 bytes. + { + success = write(buf, *offset); + check_error("ListDirectoryTask::writeString()", false); + *offset = 0; + } + int count = 0, len = str.length(); + *((uint32_t*)&buf[*offset]) = len; + *offset += 4; + while (count < len) + { + int toWrite = min(size - *offset, len); + memcpy(&buf[*offset], &str.data()[count], toWrite); + count += toWrite; + *offset += toWrite; + if (*offset == size) { - success = write(buf, *offset); - check_error("ListDirectoryTask::writeString()", false); - *offset = 0; + success = write(buf, *offset); + check_error("ListDirectoryTask::writeString()", false); + *offset = 0; } - int count = 0, len = str.length(); - *((uint32_t *) &buf[*offset]) = len; - *offset += 4; - while (count < len) - { - int toWrite = min(size - *offset, len); - memcpy(&buf[*offset], &str.data()[count], toWrite); - count += toWrite; - *offset += toWrite; - if (*offset == size) - { - success = write(buf, *offset); - check_error("ListDirectoryTask::writeString()", false); - *offset = 0; - } - } - return true; + } + return true; } - + bool ListDirectoryTask::run() { - SMLogging* logger = SMLogging::get(); - bool success; - uint8_t buf[1024] = {0}; - int err; - - if (getLength() > 1023) { - handleError("ListDirectoryTask read", ENAMETOOLONG); - return true; - } - - err = read(buf, getLength()); - if (err<0) - { - handleError("ListDirectoryTask read", errno); - return false; - } - listdir_cmd *cmd = (listdir_cmd *) buf; - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"list_directory %s.",cmd->path); - #endif - - vector listing; - try { - err = ioc->listDirectory(cmd->path, &listing); - } - catch (exception &e) - { - logger->log(LOG_ERR, "ListDirectoryTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - if (err) - { - handleError("ListDirectory", errno); - return true; - } - - // be careful modifying the listdir return types... - uint payloadLen = sizeof(listdir_resp) + (sizeof(listdir_resp_entry) * listing.size()); - for (uint i = 0; i < listing.size(); i++) - payloadLen += listing[i].size(); - - sm_response *resp = (sm_response *) buf; - resp->header.type = SM_MSG_START; - resp->header.payloadLen = payloadLen + sizeof(sm_response) - sizeof(sm_msg_header); - resp->header.flags = 0; - resp->returnCode = 0; - listdir_resp *r = (listdir_resp *) resp->payload; - r->elements = listing.size(); - - int offset = (uint64_t) r->entries - (uint64_t) buf; - for (uint i = 0; i < listing.size(); i++) - { - success = writeString(buf, &offset, 1024, listing[i]); - check_error("ListDirectoryTask write", false); - } - - if (offset != 0) - { - success = write(buf, offset); - check_error("ListDirectoryTask write", false); - } + SMLogging* logger = SMLogging::get(); + bool success; + uint8_t buf[1024] = {0}; + int err; + + if (getLength() > 1023) + { + handleError("ListDirectoryTask read", ENAMETOOLONG); return true; + } + + err = read(buf, getLength()); + if (err < 0) + { + handleError("ListDirectoryTask read", errno); + return false; + } + listdir_cmd* cmd = (listdir_cmd*)buf; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "list_directory %s.", cmd->path); +#endif + + vector listing; + try + { + err = ioc->listDirectory(cmd->path, &listing); + } + catch (exception& e) + { + logger->log(LOG_ERR, "ListDirectoryTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + if (err) + { + handleError("ListDirectory", errno); + return true; + } + + // be careful modifying the listdir return types... + uint payloadLen = sizeof(listdir_resp) + (sizeof(listdir_resp_entry) * listing.size()); + for (uint i = 0; i < listing.size(); i++) + payloadLen += listing[i].size(); + + sm_response* resp = (sm_response*)buf; + resp->header.type = SM_MSG_START; + resp->header.payloadLen = payloadLen + sizeof(sm_response) - sizeof(sm_msg_header); + resp->header.flags = 0; + resp->returnCode = 0; + listdir_resp* r = (listdir_resp*)resp->payload; + r->elements = listing.size(); + + int offset = (uint64_t)r->entries - (uint64_t)buf; + for (uint i = 0; i < listing.size(); i++) + { + success = writeString(buf, &offset, 1024, listing[i]); + check_error("ListDirectoryTask write", false); + } + + if (offset != 0) + { + success = write(buf, offset); + check_error("ListDirectoryTask write", false); + } + return true; } - -} + +} // namespace storagemanager diff --git a/storage-manager/src/ListDirectoryTask.h b/storage-manager/src/ListDirectoryTask.h index e2180a452..d5cb7156e 100644 --- a/storage-manager/src/ListDirectoryTask.h +++ b/storage-manager/src/ListDirectoryTask.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef LIST_DIRECTORYTASK_H_ #define LIST_DIRECTORYTASK_H_ @@ -24,21 +23,19 @@ namespace storagemanager { - class ListDirectoryTask : public PosixTask { - public: - ListDirectoryTask(int sock, uint length); - virtual ~ListDirectoryTask(); - - bool run(); - - private: - ListDirectoryTask(); - - bool writeString(uint8_t *buf, int *offset, int size, const std::string &str); + public: + ListDirectoryTask(int sock, uint length); + virtual ~ListDirectoryTask(); + + bool run(); + + private: + ListDirectoryTask(); + + bool writeString(uint8_t* buf, int* offset, int size, const std::string& str); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/LocalStorage.cpp b/storage-manager/src/LocalStorage.cpp index 0c2cd9d18..005e43838 100644 --- a/storage-manager/src/LocalStorage.cpp +++ b/storage-manager/src/LocalStorage.cpp @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #include #include #include @@ -31,41 +29,40 @@ namespace bf = boost::filesystem; namespace storagemanager { - LocalStorage::LocalStorage() { - prefix = Config::get()->getValue("LocalStorage", "path"); - //cout << "LS: got prefix " << prefix << endl; - if (!bf::is_directory(prefix)) + prefix = Config::get()->getValue("LocalStorage", "path"); + // cout << "LS: got prefix " << prefix << endl; + if (!bf::is_directory(prefix)) + { + try { - try - { - bf::create_directories(prefix); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what()); - throw e; - } + bf::create_directories(prefix); } - string stmp = Config::get()->getValue("LocalStorage", "fake_latency"); - if (!stmp.empty() && (stmp[0] == 'Y' || stmp[0] == 'y')) + catch (exception& e) { - fakeLatency = true; - stmp = Config::get()->getValue("LocalStorage", "max_latency"); - usecLatencyCap = strtoull(stmp.c_str(), NULL, 10); - if (usecLatencyCap == 0) - { - logger->log(LOG_CRIT, "LocalStorage: bad value for max_latency"); - throw runtime_error("LocalStorage: bad value for max_latency"); - } - r_seed = (uint) ::time(NULL); - logger->log(LOG_DEBUG, "LocalStorage: Will simulate cloud latency of max %llu us", usecLatencyCap); + logger->log(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what()); + throw e; } - else - fakeLatency = false; - - bytesRead = bytesWritten = 0; + } + string stmp = Config::get()->getValue("LocalStorage", "fake_latency"); + if (!stmp.empty() && (stmp[0] == 'Y' || stmp[0] == 'y')) + { + fakeLatency = true; + stmp = Config::get()->getValue("LocalStorage", "max_latency"); + usecLatencyCap = strtoull(stmp.c_str(), NULL, 10); + if (usecLatencyCap == 0) + { + logger->log(LOG_CRIT, "LocalStorage: bad value for max_latency"); + throw runtime_error("LocalStorage: bad value for max_latency"); + } + r_seed = (uint)::time(NULL); + logger->log(LOG_DEBUG, "LocalStorage: Will simulate cloud latency of max %llu us", usecLatencyCap); + } + else + fakeLatency = false; + + bytesRead = bytesWritten = 0; } LocalStorage::~LocalStorage() @@ -74,195 +71,199 @@ LocalStorage::~LocalStorage() void LocalStorage::printKPIs() const { - cout << "LocalStorage" << endl; - cout << "\tbytesRead = " << bytesRead << endl; - cout << "\tbytesWritten = " << bytesWritten << endl; - CloudStorage::printKPIs(); + cout << "LocalStorage" << endl; + cout << "\tbytesRead = " << bytesRead << endl; + cout << "\tbytesWritten = " << bytesWritten << endl; + CloudStorage::printKPIs(); } -const bf::path & LocalStorage::getPrefix() const +const bf::path& LocalStorage::getPrefix() const { - return prefix; + return prefix; } inline void LocalStorage::addLatency() { - if (fakeLatency) - { - uint64_t usec_delay = ((double) rand_r(&r_seed) / (double) RAND_MAX) * usecLatencyCap; - ::usleep(usec_delay); - } + if (fakeLatency) + { + uint64_t usec_delay = ((double)rand_r(&r_seed) / (double)RAND_MAX) * usecLatencyCap; + ::usleep(usec_delay); + } } -int LocalStorage::copy(const bf::path &source, const bf::path &dest) +int LocalStorage::copy(const bf::path& source, const bf::path& dest) { - boost::system::error_code err; - bf::copy_file(source, dest, bf::copy_option::fail_if_exists, err); - if (err) - { - errno = err.value(); - ::unlink(dest.string().c_str()); - return -1; - } - return 0; + boost::system::error_code err; + bf::copy_file(source, dest, bf::copy_option::fail_if_exists, err); + if (err) + { + errno = err.value(); + ::unlink(dest.string().c_str()); + return -1; + } + return 0; } -bf::path operator+(const bf::path &p1, const bf::path &p2) +bf::path operator+(const bf::path& p1, const bf::path& p2) { - bf::path ret(p1); - ret /= p2; + bf::path ret(p1); + ret /= p2; + return ret; +} + +int LocalStorage::getObject(const string& source, const string& dest, size_t* size) +{ + addLatency(); + + int ret = copy(prefix / source, dest); + if (ret) return ret; + size_t _size = bf::file_size(dest); + if (size) + *size = _size; + bytesRead += _size; + bytesWritten += _size; + ++objectsGotten; + return ret; } -int LocalStorage::getObject(const string &source, const string &dest, size_t *size) +int LocalStorage::getObject(const std::string& sourceKey, boost::shared_array* data, size_t* size) { - addLatency(); + addLatency(); - int ret = copy(prefix / source, dest); - if (ret) - return ret; - size_t _size = bf::file_size(dest); - if (size) - *size = _size; + bf::path source = prefix / sourceKey; + const char* c_source = source.string().c_str(); + // char buf[80]; + int l_errno; + + int fd = ::open(c_source, O_RDONLY); + if (fd < 0) + { + l_errno = errno; + // logger->log(LOG_WARNING, "LocalStorage::getObject() failed to open %s, got '%s'", c_source, + // strerror_r(errno, buf, 80)); + errno = l_errno; + return fd; + } + + size_t l_size = bf::file_size(source); + data->reset(new uint8_t[l_size]); + size_t count = 0; + while (count < l_size) + { + int err = ::read(fd, &(*data)[count], l_size - count); + if (err < 0) + { + l_errno = errno; + // logger->log(LOG_WARNING, "LocalStorage::getObject() failed to read %s, got '%s'", c_source, + // strerror_r(errno, buf, 80)); + close(fd); + bytesRead += count; + errno = l_errno; + return err; + } + count += err; + } + if (size) + *size = l_size; + close(fd); + bytesRead += l_size; + ++objectsGotten; + return 0; +} + +int LocalStorage::putObject(const string& source, const string& dest) +{ + addLatency(); + + int ret = copy(source, prefix / dest); + + if (ret == 0) + { + size_t _size = bf::file_size(source); bytesRead += _size; bytesWritten += _size; - ++objectsGotten; - return ret; -} - -int LocalStorage::getObject(const std::string &sourceKey, boost::shared_array *data, size_t *size) -{ - addLatency(); - - bf::path source = prefix / sourceKey; - const char *c_source = source.string().c_str(); - //char buf[80]; - int l_errno; - - int fd = ::open(c_source, O_RDONLY); - if (fd < 0) - { - l_errno = errno; - //logger->log(LOG_WARNING, "LocalStorage::getObject() failed to open %s, got '%s'", c_source, strerror_r(errno, buf, 80)); - errno = l_errno; - return fd; - } - - size_t l_size = bf::file_size(source); - data->reset(new uint8_t[l_size]); - size_t count = 0; - while (count < l_size) - { - int err = ::read(fd, &(*data)[count], l_size - count); - if (err < 0) - { - l_errno = errno; - //logger->log(LOG_WARNING, "LocalStorage::getObject() failed to read %s, got '%s'", c_source, strerror_r(errno, buf, 80)); - close(fd); - bytesRead += count; - errno = l_errno; - return err; - } - count += err; - } - if (size) - *size = l_size; - close(fd); - bytesRead += l_size; - ++objectsGotten; - return 0; -} - -int LocalStorage::putObject(const string &source, const string &dest) -{ - addLatency(); - - int ret = copy(source, prefix / dest); - - if (ret == 0) - { - size_t _size = bf::file_size(source); - bytesRead += _size; - bytesWritten += _size; - ++objectsPut; - } - return ret; -} - -int LocalStorage::putObject(boost::shared_array data, size_t len, const string &dest) -{ - addLatency(); - - bf::path destPath = prefix / dest; - const char *c_dest = destPath.string().c_str(); - //char buf[80]; - int l_errno; - - int fd = ::open(c_dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (fd < 0) - { - l_errno = errno; - //logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to open %s, got '%s'", c_dest, strerror_r(errno, buf, 80)); - errno = l_errno; - return fd; - } - - size_t count = 0; - int err; - while (count < len) - { - err = ::write(fd, &data[count], len - count); - if (err < 0) - { - l_errno = errno; - //logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to write to %s, got '%s'", c_dest, strerror_r(errno, buf, 80)); - close(fd); - ::unlink(c_dest); - errno = l_errno; - bytesWritten += count; - return err; - } - count += err; - } - close(fd); - bytesWritten += count; ++objectsPut; - return 0; + } + return ret; } -int LocalStorage::copyObject(const string &source, const string &dest) +int LocalStorage::putObject(boost::shared_array data, size_t len, const string& dest) { - addLatency(); - - int ret = copy(prefix / source, prefix / dest); - - if (ret == 0) + addLatency(); + + bf::path destPath = prefix / dest; + const char* c_dest = destPath.string().c_str(); + // char buf[80]; + int l_errno; + + int fd = ::open(c_dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd < 0) + { + l_errno = errno; + // logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to open %s, got '%s'", c_dest, + // strerror_r(errno, buf, 80)); + errno = l_errno; + return fd; + } + + size_t count = 0; + int err; + while (count < len) + { + err = ::write(fd, &data[count], len - count); + if (err < 0) { - ++objectsCopied; - size_t _size = bf::file_size(prefix/source); - bytesRead += _size; - bytesWritten += _size; + l_errno = errno; + // logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to write to %s, got '%s'", c_dest, + // strerror_r(errno, buf, 80)); + close(fd); + ::unlink(c_dest); + errno = l_errno; + bytesWritten += count; + return err; } - return ret; + count += err; + } + close(fd); + bytesWritten += count; + ++objectsPut; + return 0; } -int LocalStorage::deleteObject(const string &key) +int LocalStorage::copyObject(const string& source, const string& dest) { - addLatency(); - - ++objectsDeleted; - boost::system::error_code err; - bf::remove(prefix / key, err); - return 0; + addLatency(); + + int ret = copy(prefix / source, prefix / dest); + + if (ret == 0) + { + ++objectsCopied; + size_t _size = bf::file_size(prefix / source); + bytesRead += _size; + bytesWritten += _size; + } + return ret; } -int LocalStorage::exists(const std::string &key, bool *out) +int LocalStorage::deleteObject(const string& key) { - addLatency(); - - ++existenceChecks; - *out = bf::exists(prefix / key); - return 0; + addLatency(); + + ++objectsDeleted; + boost::system::error_code err; + bf::remove(prefix / key, err); + return 0; } +int LocalStorage::exists(const std::string& key, bool* out) +{ + addLatency(); + + ++existenceChecks; + *out = bf::exists(prefix / key); + return 0; } + +} // namespace storagemanager diff --git a/storage-manager/src/LocalStorage.h b/storage-manager/src/LocalStorage.h index 5185241e8..7dd6c5333 100644 --- a/storage-manager/src/LocalStorage.h +++ b/storage-manager/src/LocalStorage.h @@ -25,38 +25,37 @@ namespace storagemanager { - class LocalStorage : public CloudStorage { - public: - LocalStorage(); - virtual ~LocalStorage(); + public: + LocalStorage(); + virtual ~LocalStorage(); - int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL); - int getObject(const std::string &sourceKey, boost::shared_array *data, size_t *size = NULL); - int putObject(const std::string &sourceFile, const std::string &destKey); - int putObject(const boost::shared_array data, size_t len, const std::string &destKey); - int deleteObject(const std::string &key); - int copyObject(const std::string &sourceKey, const std::string &destKey); - int exists(const std::string &key, bool *out); - - const boost::filesystem::path & getPrefix() const; - void printKPIs() const; - - protected: - size_t bytesRead, bytesWritten; - - private: - boost::filesystem::path prefix; - int copy(const boost::filesystem::path &sourceKey, const boost::filesystem::path &destKey); - - // stuff for faking the latency on cloud ops - bool fakeLatency; - uint64_t usecLatencyCap; - uint r_seed; - void addLatency(); + int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL); + int getObject(const std::string& sourceKey, boost::shared_array* data, size_t* size = NULL); + int putObject(const std::string& sourceFile, const std::string& destKey); + int putObject(const boost::shared_array data, size_t len, const std::string& destKey); + int deleteObject(const std::string& key); + int copyObject(const std::string& sourceKey, const std::string& destKey); + int exists(const std::string& key, bool* out); + + const boost::filesystem::path& getPrefix() const; + void printKPIs() const; + + protected: + size_t bytesRead, bytesWritten; + + private: + boost::filesystem::path prefix; + int copy(const boost::filesystem::path& sourceKey, const boost::filesystem::path& destKey); + + // stuff for faking the latency on cloud ops + bool fakeLatency; + uint64_t usecLatencyCap; + uint r_seed; + void addLatency(); }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index cde04925d..e45c95173 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -37,157 +37,154 @@ namespace bpt = boost::property_tree; namespace bf = boost::filesystem; namespace -{ - boost::mutex mdfLock; - storagemanager::MetadataFile::MetadataConfig *inst = NULL; - uint64_t metadataFilesAccessed = 0; -} +{ +boost::mutex mdfLock; +storagemanager::MetadataFile::MetadataConfig* inst = NULL; +uint64_t metadataFilesAccessed = 0; +} // namespace namespace storagemanager { - -MetadataFile::MetadataConfig * MetadataFile::MetadataConfig::get() +MetadataFile::MetadataConfig* MetadataFile::MetadataConfig::get() { - if (inst) - return inst; - boost::unique_lock s(mdfLock); - if (inst) - return inst; - inst = new MetadataConfig(); + if (inst) return inst; + boost::unique_lock s(mdfLock); + if (inst) + return inst; + inst = new MetadataConfig(); + return inst; } MetadataFile::MetadataConfig::MetadataConfig() { - Config *config = Config::get(); - SMLogging *logger = SMLogging::get(); - - try - { - mObjectSize = stoul(config->getValue("ObjectStorage", "object_size")); - } - catch (...) - { - logger->log(LOG_CRIT, "ObjectStorage/object_size must be set to a numeric value"); - throw runtime_error("Please set ObjectStorage/object)size in the storagemanager.cnf file"); - } - - try - { - msMetadataPath = config->getValue("ObjectStorage", "metadata_path"); - if (msMetadataPath.empty()) - { - logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set"); - throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); - } - } - catch (...) - { - logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set"); - throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); - } + Config* config = Config::get(); + SMLogging* logger = SMLogging::get(); - try - { - boost::filesystem::create_directories(msMetadataPath); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); - throw e; - } + try + { + mObjectSize = stoul(config->getValue("ObjectStorage", "object_size")); + } + catch (...) + { + logger->log(LOG_CRIT, "ObjectStorage/object_size must be set to a numeric value"); + throw runtime_error("Please set ObjectStorage/object)size in the storagemanager.cnf file"); + } + try + { + msMetadataPath = config->getValue("ObjectStorage", "metadata_path"); + if (msMetadataPath.empty()) + { + logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set"); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + } + catch (...) + { + logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set"); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + + try + { + boost::filesystem::create_directories(msMetadataPath); + } + catch (exception& e) + { + logger->log(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); + throw e; + } } MetadataFile::MetadataFile() { - mpConfig = MetadataConfig::get(); - mpLogger = SMLogging::get(); - mVersion=1; - mRevision=1; - _exists = false; + mpConfig = MetadataConfig::get(); + mpLogger = SMLogging::get(); + mVersion = 1; + mRevision = 1; + _exists = false; } - -MetadataFile::MetadataFile(const boost::filesystem::path &filename) +MetadataFile::MetadataFile(const boost::filesystem::path& filename) { - mpConfig = MetadataConfig::get(); - mpLogger = SMLogging::get(); + mpConfig = MetadataConfig::get(); + mpLogger = SMLogging::get(); + _exists = true; + + mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta"); + + boost::unique_lock s(jsonCache.getMutex()); + jsontree = jsonCache.get(mFilename); + if (!jsontree) + { + if (boost::filesystem::exists(mFilename)) + { + jsontree.reset(new bpt::ptree()); + boost::property_tree::read_json(mFilename.string(), *jsontree); + jsonCache.put(mFilename, jsontree); + s.unlock(); + mVersion = 1; + mRevision = jsontree->get("revision"); + } + else + { + mVersion = 1; + mRevision = 1; + makeEmptyJsonTree(); + s.unlock(); + writeMetadata(); + } + } + else + { + s.unlock(); + mVersion = 1; + mRevision = jsontree->get("revision"); + } + ++metadataFilesAccessed; +} + +MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, bool appendExt) +{ + mpConfig = MetadataConfig::get(); + mpLogger = SMLogging::get(); + + mFilename = filename; + + if (appendExt) + mFilename = mpConfig->msMetadataPath / (mFilename.string() + ".meta"); + + boost::unique_lock s(jsonCache.getMutex()); + jsontree = jsonCache.get(mFilename); + if (!jsontree) + { + if (boost::filesystem::exists(mFilename)) + { + _exists = true; + jsontree.reset(new bpt::ptree()); + boost::property_tree::read_json(mFilename.string(), *jsontree); + jsonCache.put(mFilename, jsontree); + s.unlock(); + mVersion = 1; + mRevision = jsontree->get("revision"); + } + else + { + mVersion = 1; + mRevision = 1; + _exists = false; + makeEmptyJsonTree(); + } + } + else + { + s.unlock(); _exists = true; - - mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta"); - - boost::unique_lock s(jsonCache.getMutex()); - jsontree = jsonCache.get(mFilename); - if (!jsontree) - { - if (boost::filesystem::exists(mFilename)) - { - jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); - jsonCache.put(mFilename, jsontree); - s.unlock(); - mVersion = 1; - mRevision = jsontree->get("revision"); - } - else - { - mVersion = 1; - mRevision = 1; - makeEmptyJsonTree(); - s.unlock(); - writeMetadata(); - } - } - else - { - s.unlock(); - mVersion = 1; - mRevision = jsontree->get("revision"); - } - ++metadataFilesAccessed; -} - -MetadataFile::MetadataFile(const boost::filesystem::path &filename, no_create_t,bool appendExt) -{ - mpConfig = MetadataConfig::get(); - mpLogger = SMLogging::get(); - - mFilename = filename; - - if(appendExt) - mFilename = mpConfig->msMetadataPath /(mFilename.string() + ".meta"); - - boost::unique_lock s(jsonCache.getMutex()); - jsontree = jsonCache.get(mFilename); - if (!jsontree) - { - if (boost::filesystem::exists(mFilename)) - { - _exists = true; - jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); - jsonCache.put(mFilename, jsontree); - s.unlock(); - mVersion = 1; - mRevision = jsontree->get("revision"); - } - else - { - mVersion = 1; - mRevision = 1; - _exists = false; - makeEmptyJsonTree(); - } - } - else - { - s.unlock(); - _exists = true; - mVersion = 1; - mRevision = jsontree->get("revision"); - } - ++metadataFilesAccessed; + mVersion = 1; + mRevision = jsontree->get("revision"); + } + ++metadataFilesAccessed; } MetadataFile::~MetadataFile() @@ -196,381 +193,381 @@ MetadataFile::~MetadataFile() void MetadataFile::makeEmptyJsonTree() { - jsontree.reset(new bpt::ptree()); - boost::property_tree::ptree objs; - jsontree->put("version",mVersion); - jsontree->put("revision",mRevision); - jsontree->add_child("objects", objs); + jsontree.reset(new bpt::ptree()); + boost::property_tree::ptree objs; + jsontree->put("version", mVersion); + jsontree->put("revision", mRevision); + jsontree->add_child("objects", objs); } void MetadataFile::printKPIs() { - cout << "Metadata files accessed = " << metadataFilesAccessed << endl; + cout << "Metadata files accessed = " << metadataFilesAccessed << endl; } -int MetadataFile::stat(struct stat *out) const +int MetadataFile::stat(struct stat* out) const { - int err = ::stat(mFilename.c_str(), out); - if (err) - return err; - - out->st_size = getLength(); - return 0; + int err = ::stat(mFilename.c_str(), out); + if (err) + return err; + + out->st_size = getLength(); + return 0; } size_t MetadataFile::getLength() const { - size_t totalSize = 0; - - auto &objects = jsontree->get_child("objects"); - if (!objects.empty()) - { - auto &lastObject = objects.back().second; - totalSize = lastObject.get("offset") + lastObject.get("length"); - } - return totalSize; + size_t totalSize = 0; + + auto& objects = jsontree->get_child("objects"); + if (!objects.empty()) + { + auto& lastObject = objects.back().second; + totalSize = lastObject.get("offset") + lastObject.get("length"); + } + return totalSize; } bool MetadataFile::exists() const { - return _exists; + return _exists; } vector MetadataFile::metadataRead(off_t offset, size_t length) const { - // this version assumes mObjects is sorted by offset, and there are no gaps between objects - vector ret; - size_t foundLen = 0; - - /* For first cut of optimizations, going to generate mObjects as it was to use the existing alg - rather than write a new alg. - */ - set mObjects; - BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree->get_child("objects")) - { - mObjects.insert(metadataObject(v.second.get("offset"), v.second.get("length"), - v.second.get("key"))); - } - - if (mObjects.size() == 0) - return ret; - - uint64_t lastOffset = mObjects.rbegin()->offset; - auto i = mObjects.begin(); - // find the first object in range - // Note, the last object in mObjects may not be full, compare the last one against its maximum - // size rather than its current size. - while (i != mObjects.end()) - { - if ((uint64_t) offset <= (i->offset + i->length - 1) || - (i->offset == lastOffset && ((uint64_t) offset <= i->offset + mpConfig->mObjectSize - 1))) - { - foundLen = (i->offset == lastOffset ? mpConfig->mObjectSize : i->length) - (offset - i->offset); - ret.push_back(*i); - ++i; - break; - } - ++i; - } - - while (i != mObjects.end() && foundLen < length) - { - ret.push_back(*i); - foundLen += i->length; - ++i; - } + // this version assumes mObjects is sorted by offset, and there are no gaps between objects + vector ret; + size_t foundLen = 0; - assert(!(offset == 0 && length == getLength()) || (ret.size() == mObjects.size())); + /* For first cut of optimizations, going to generate mObjects as it was to use the existing alg + rather than write a new alg. + */ + set mObjects; + BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + { + mObjects.insert(metadataObject(v.second.get("offset"), v.second.get("length"), + v.second.get("key"))); + } + + if (mObjects.size() == 0) return ret; + + uint64_t lastOffset = mObjects.rbegin()->offset; + auto i = mObjects.begin(); + // find the first object in range + // Note, the last object in mObjects may not be full, compare the last one against its maximum + // size rather than its current size. + while (i != mObjects.end()) + { + if ((uint64_t)offset <= (i->offset + i->length - 1) || + (i->offset == lastOffset && ((uint64_t)offset <= i->offset + mpConfig->mObjectSize - 1))) + { + foundLen = (i->offset == lastOffset ? mpConfig->mObjectSize : i->length) - (offset - i->offset); + ret.push_back(*i); + ++i; + break; + } + ++i; + } + + while (i != mObjects.end() && foundLen < length) + { + ret.push_back(*i); + foundLen += i->length; + ++i; + } + + assert(!(offset == 0 && length == getLength()) || (ret.size() == mObjects.size())); + return ret; } -metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path &filename, size_t length) +metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& filename, size_t length) { - // this needs to handle if data write is beyond the end of the last object - // but not at start of new object - // - - metadataObject addObject; - auto &objects = jsontree->get_child("objects"); - if (!objects.empty()) - { - auto &lastObject = objects.back().second; - addObject.offset = lastObject.get("offset") + mpConfig->mObjectSize; - } - - addObject.length = length; - addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length); - boost::property_tree::ptree object; - object.put("offset", addObject.offset); - object.put("length", addObject.length); - object.put("key", addObject.key); - objects.push_back(make_pair("", object)); + // this needs to handle if data write is beyond the end of the last object + // but not at start of new object + // - return addObject; + metadataObject addObject; + auto& objects = jsontree->get_child("objects"); + if (!objects.empty()) + { + auto& lastObject = objects.back().second; + addObject.offset = lastObject.get("offset") + mpConfig->mObjectSize; + } + + addObject.length = length; + addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length); + boost::property_tree::ptree object; + object.put("offset", addObject.offset); + object.put("length", addObject.length); + object.put("key", addObject.key); + objects.push_back(make_pair("", object)); + + return addObject; } // TODO: Error handling...s int MetadataFile::writeMetadata() { - if (!boost::filesystem::exists(mFilename.parent_path())) - boost::filesystem::create_directories(mFilename.parent_path()); - - write_json(mFilename.string(), *jsontree); - _exists = true; - - boost::unique_lock s(jsonCache.getMutex()); - jsonCache.put(mFilename, jsontree); + if (!boost::filesystem::exists(mFilename.parent_path())) + boost::filesystem::create_directories(mFilename.parent_path()); - return 0; + write_json(mFilename.string(), *jsontree); + _exists = true; + + boost::unique_lock s(jsonCache.getMutex()); + jsonCache.put(mFilename, jsontree); + + return 0; } -bool MetadataFile::getEntry(off_t offset, metadataObject *out) const +bool MetadataFile::getEntry(off_t offset, metadataObject* out) const { - metadataObject addObject; - BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree->get_child("objects")) + metadataObject addObject; + BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + { + if (v.second.get("offset") == offset) { - if (v.second.get("offset") == offset) - { - out->offset = offset; - out->length = v.second.get("length"); - out->key = v.second.get("key"); - return true; - } + out->offset = offset; + out->length = v.second.get("length"); + out->key = v.second.get("key"); + return true; } - return false; + } + return false; } void MetadataFile::removeEntry(off_t offset) { - bpt::ptree &objects = jsontree->get_child("objects"); - for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it) + bpt::ptree& objects = jsontree->get_child("objects"); + for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it) + { + if (it->second.get("offset") == offset) { - if (it->second.get("offset") == offset) - { - objects.erase(it); - break; - } + objects.erase(it); + break; } + } } void MetadataFile::removeAllEntries() { - jsontree->get_child("objects").clear(); + jsontree->get_child("objects").clear(); } -void MetadataFile::deletedMeta(const bf::path &p) +void MetadataFile::deletedMeta(const bf::path& p) { - boost::unique_lock s(jsonCache.getMutex()); - jsonCache.erase(p); + boost::unique_lock s(jsonCache.getMutex()); + jsonCache.erase(p); } // There are more efficient ways to do it. Optimize if necessary. -void MetadataFile::breakout(const string &key, vector &ret) +void MetadataFile::breakout(const string& key, vector& ret) { - int indexes[3]; // positions of each '_' delimiter - ret.clear(); - indexes[0] = key.find_first_of('_'); - indexes[1] = key.find_first_of('_', indexes[0] + 1); - indexes[2] = key.find_first_of('_', indexes[1] + 1); - ret.push_back(key.substr(0, indexes[0])); - ret.push_back(key.substr(indexes[0] + 1, indexes[1] - indexes[0] - 1)); - ret.push_back(key.substr(indexes[1] + 1, indexes[2] - indexes[1] - 1)); - ret.push_back(key.substr(indexes[2] + 1)); + int indexes[3]; // positions of each '_' delimiter + ret.clear(); + indexes[0] = key.find_first_of('_'); + indexes[1] = key.find_first_of('_', indexes[0] + 1); + indexes[2] = key.find_first_of('_', indexes[1] + 1); + ret.push_back(key.substr(0, indexes[0])); + ret.push_back(key.substr(indexes[0] + 1, indexes[1] - indexes[0] - 1)); + ret.push_back(key.substr(indexes[1] + 1, indexes[2] - indexes[1] - 1)); + ret.push_back(key.substr(indexes[2] + 1)); } - -string MetadataFile::getNewKeyFromOldKey(const string &key, size_t length) + +string MetadataFile::getNewKeyFromOldKey(const string& key, size_t length) { - mdfLock.lock(); - boost::uuids::uuid u = boost::uuids::random_generator()(); - mdfLock.unlock(); - - vector split; - breakout(key, split); - ostringstream oss; - oss << u << "_" << split[1] << "_" << length << "_" << split[3]; - return oss.str(); + mdfLock.lock(); + boost::uuids::uuid u = boost::uuids::random_generator()(); + mdfLock.unlock(); + + vector split; + breakout(key, split); + ostringstream oss; + oss << u << "_" << split[1] << "_" << length << "_" << split[3]; + return oss.str(); } string MetadataFile::getNewKey(string sourceName, size_t offset, size_t length) { - mdfLock.lock(); - boost::uuids::uuid u = boost::uuids::random_generator()(); - mdfLock.unlock(); - stringstream ss; + mdfLock.lock(); + boost::uuids::uuid u = boost::uuids::random_generator()(); + mdfLock.unlock(); + stringstream ss; - for (uint i = 0; i < sourceName.length(); i++) + for (uint i = 0; i < sourceName.length(); i++) + { + if (sourceName[i] == '/') { - if (sourceName[i] == '/') - { - sourceName[i] = '~'; - } + sourceName[i] = '~'; } + } - ss << u << "_" << offset << "_" << length << "_" << sourceName; - return ss.str(); + ss << u << "_" << offset << "_" << length << "_" << sourceName; + return ss.str(); } -off_t MetadataFile::getOffsetFromKey(const string &key) +off_t MetadataFile::getOffsetFromKey(const string& key) { - vector split; - breakout(key, split); - return stoll(split[1]); + vector split; + breakout(key, split); + return stoll(split[1]); } -string MetadataFile::getSourceFromKey(const string &key) +string MetadataFile::getSourceFromKey(const string& key) { - vector split; - breakout(key, split); - - // this is to convert the munged filenames back to regular filenames - // for consistent use in IOC locks - for (uint i = 0; i < split[3].length(); i++) - if (split[3][i] == '~') - split[3][i] = '/'; - - return split[3]; + vector split; + breakout(key, split); + + // this is to convert the munged filenames back to regular filenames + // for consistent use in IOC locks + for (uint i = 0; i < split[3].length(); i++) + if (split[3][i] == '~') + split[3][i] = '/'; + + return split[3]; } -size_t MetadataFile::getLengthFromKey(const string &key) +size_t MetadataFile::getLengthFromKey(const string& key) { - vector split; - breakout(key, split); - return stoull(split[2]); + vector split; + breakout(key, split); + return stoull(split[2]); } // more efficient way to do these? -void MetadataFile::setOffsetInKey(string &key, off_t newOffset) +void MetadataFile::setOffsetInKey(string& key, off_t newOffset) { - vector split; - breakout(key, split); - ostringstream oss; - oss << split[0] << "_" << newOffset << "_" << split[2] << "_" << split[3]; - key = oss.str(); + vector split; + breakout(key, split); + ostringstream oss; + oss << split[0] << "_" << newOffset << "_" << split[2] << "_" << split[3]; + key = oss.str(); } -void MetadataFile::setLengthInKey(string &key, size_t newLength) +void MetadataFile::setLengthInKey(string& key, size_t newLength) { - vector split; - breakout(key, split); - ostringstream oss; - oss << split[0] << "_" << split[1] << "_" << newLength << "_" << split[3]; - key = oss.str(); + vector split; + breakout(key, split); + ostringstream oss; + oss << split[0] << "_" << split[1] << "_" << newLength << "_" << split[3]; + key = oss.str(); } void MetadataFile::printObjects() const { - BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree->get_child("objects")) - { - printf("Name: %s Length: %zu Offset: %lld\n", v.second.get("key").c_str(), - v.second.get("length"), (long long)v.second.get("offset")); - } + BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + { + printf("Name: %s Length: %zu Offset: %lld\n", v.second.get("key").c_str(), + v.second.get("length"), (long long)v.second.get("offset")); + } } -void MetadataFile::updateEntry(off_t offset, const string &newName, size_t newLength) +void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength) { - for (auto &v : jsontree->get_child("objects")) + for (auto& v : jsontree->get_child("objects")) + { + if (v.second.get("offset") == offset) { - if (v.second.get("offset") == offset) - { - v.second.put("key", newName); - v.second.put("length", newLength); - return; - } + v.second.put("key", newName); + v.second.put("length", newLength); + return; } - stringstream ss; - ss << "MetadataFile::updateEntry(): failed to find object at offset " << offset; - mpLogger->log(LOG_ERR, ss.str().c_str()); - throw logic_error(ss.str()); + } + stringstream ss; + ss << "MetadataFile::updateEntry(): failed to find object at offset " << offset; + mpLogger->log(LOG_ERR, ss.str().c_str()); + throw logic_error(ss.str()); } void MetadataFile::updateEntryLength(off_t offset, size_t newLength) { - for (auto &v : jsontree->get_child("objects")) + for (auto& v : jsontree->get_child("objects")) + { + if (v.second.get("offset") == offset) { - if (v.second.get("offset") == offset) - { - v.second.put("length", newLength); - return; - } + v.second.put("length", newLength); + return; } - stringstream ss; - ss << "MetadataFile::updateEntryLength(): failed to find object at offset " << offset; - mpLogger->log(LOG_ERR, ss.str().c_str()); - throw logic_error(ss.str()); + } + stringstream ss; + ss << "MetadataFile::updateEntryLength(): failed to find object at offset " << offset; + mpLogger->log(LOG_ERR, ss.str().c_str()); + throw logic_error(ss.str()); } off_t MetadataFile::getMetadataNewObjectOffset() { - auto &objects = jsontree->get_child("objects"); - if (objects.empty()) - return 0; - auto &lastObject = jsontree->get_child("objects").back().second; - return lastObject.get("offset") + lastObject.get("length"); + auto& objects = jsontree->get_child("objects"); + if (objects.empty()) + return 0; + auto& lastObject = jsontree->get_child("objects").back().second; + return lastObject.get("offset") + lastObject.get("length"); } metadataObject::metadataObject() : offset(0), length(0) -{} - -metadataObject::metadataObject(uint64_t _offset) : offset(_offset), length(0) -{} - -metadataObject::metadataObject(uint64_t _offset, uint64_t _length, const std::string &_key) : - offset(_offset), length(_length), key(_key) -{} - - - -MetadataFile::MetadataCache::MetadataCache() : max_lru_size(2000) // 2000 is an arbitrary #. Big enough for a large working set. -{} - -inline boost::mutex & MetadataFile::MetadataCache::getMutex() { - return mutex; } -MetadataFile::Jsontree_t MetadataFile::MetadataCache::get(const bf::path &p) +metadataObject::metadataObject(uint64_t _offset) : offset(_offset), length(0) { - auto it = lookup.find(p.string()); - if (it != lookup.end()) - { - lru.splice(lru.end(), lru, it->second.second); - return it->second.first; - } - - return storagemanager::MetadataFile::Jsontree_t(); +} + +metadataObject::metadataObject(uint64_t _offset, uint64_t _length, const std::string& _key) + : offset(_offset), length(_length), key(_key) +{ +} + +MetadataFile::MetadataCache::MetadataCache() + : max_lru_size(2000) // 2000 is an arbitrary #. Big enough for a large working set. +{ +} + +inline boost::mutex& MetadataFile::MetadataCache::getMutex() +{ + return mutex; +} + +MetadataFile::Jsontree_t MetadataFile::MetadataCache::get(const bf::path& p) +{ + auto it = lookup.find(p.string()); + if (it != lookup.end()) + { + lru.splice(lru.end(), lru, it->second.second); + return it->second.first; + } + + return storagemanager::MetadataFile::Jsontree_t(); } // note, does not change an existing jsontree. This should be OK. -void MetadataFile::MetadataCache::put(const bf::path &p, const Jsontree_t &j) +void MetadataFile::MetadataCache::put(const bf::path& p, const Jsontree_t& j) { - string sp = p.string(); - auto it = lookup.find(sp); - if (it == lookup.end()) + string sp = p.string(); + auto it = lookup.find(sp); + if (it == lookup.end()) + { + while (lru.size() >= max_lru_size) { - while (lru.size() >= max_lru_size) - { - lookup.erase(lru.front()); - lru.pop_front(); - } - lru.push_back(sp); - Lru_t::iterator last = lru.end(); - lookup.emplace(sp, make_pair(j, --last)); + lookup.erase(lru.front()); + lru.pop_front(); } + lru.push_back(sp); + Lru_t::iterator last = lru.end(); + lookup.emplace(sp, make_pair(j, --last)); + } } -void MetadataFile::MetadataCache::erase(const bf::path &p) +void MetadataFile::MetadataCache::erase(const bf::path& p) { - auto it = lookup.find(p.string()); - if (it != lookup.end()) - { - lru.erase(it->second.second); - lookup.erase(it); - } + auto it = lookup.find(p.string()); + if (it != lookup.end()) + { + lru.erase(it->second.second); + lookup.erase(it); + } } MetadataFile::MetadataCache MetadataFile::jsonCache; -} - - - +} // namespace storagemanager diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index 38a36c006..f08053844 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -31,115 +31,120 @@ namespace storagemanager { - -struct metadataObject { - metadataObject(); - metadataObject(uint64_t offset); // so we can search mObjects by integer - metadataObject(uint64_t offset, uint64_t length, const std::string &key); - uint64_t offset; - mutable uint64_t length; - mutable std::string key; - bool operator < (const metadataObject &b) const { return offset < b.offset; } +struct metadataObject +{ + metadataObject(); + metadataObject(uint64_t offset); // so we can search mObjects by integer + metadataObject(uint64_t offset, uint64_t length, const std::string& key); + uint64_t offset; + mutable uint64_t length; + mutable std::string key; + bool operator<(const metadataObject& b) const + { + return offset < b.offset; + } }; class MetadataFile { - public: - struct no_create_t {}; - MetadataFile(); - MetadataFile(const boost::filesystem::path &filename); - MetadataFile(const boost::filesystem::path &path, no_create_t,bool appendExt); // this one won't create it if it doesn't exist - - // this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile - // meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended. - // aside from that, it will behave like the no_create ctor variant above - //MetadataFile(const boost::filesystem::path &path); - ~MetadataFile(); + public: + struct no_create_t + { + }; + MetadataFile(); + MetadataFile(const boost::filesystem::path& filename); + MetadataFile(const boost::filesystem::path& path, no_create_t, + bool appendExt); // this one won't create it if it doesn't exist - bool exists() const; - void printObjects() const; - int stat(struct stat *) const; - size_t getLength() const; - // returns the objects needed to update - std::vector metadataRead(off_t offset, size_t length) const; - // updates the metadatafile with new object - //int writeMetadata(const boost::filesystem::path &filename); - int writeMetadata(); - - // updates the name and length fields of an entry, given the offset - void updateEntry(off_t offset, const std::string &newName, size_t newLength); - void updateEntryLength(off_t offset, size_t newLength); - metadataObject addMetadataObject(const boost::filesystem::path &filename, size_t length); - bool getEntry(off_t offset, metadataObject *out) const; - void removeEntry(off_t offset); - void removeAllEntries(); - - // removes p from the json cache. p should be a fully qualified metadata file - static void deletedMeta(const boost::filesystem::path &p); - - static std::string getNewKeyFromOldKey(const std::string &oldKey, size_t length=0); - static std::string getNewKey(std::string sourceName, size_t offset, size_t length); - static off_t getOffsetFromKey(const std::string &key); - static std::string getSourceFromKey(const std::string &key); - static size_t getLengthFromKey(const std::string &key); - static void setOffsetInKey(std::string &key, off_t newOffset); - static void setLengthInKey(std::string &key, size_t newLength); - - // breaks a key into its consitituent fields - static void breakout(const std::string &key, std::vector &out); - - off_t getMetadataNewObjectOffset(); - // this will be a singleton, which stores the config used - // by all MetadataFile instances so we don't have to keep bothering Config. - // members are public b/c i don't want to write accessors right now. Also who cares. - class MetadataConfig - { - public: - static MetadataConfig *get(); - size_t mObjectSize; - boost::filesystem::path msMetadataPath; - - private: - MetadataConfig(); - }; - - static void printKPIs(); - - typedef boost::shared_ptr Jsontree_t; - private: - MetadataConfig *mpConfig; - SMLogging *mpLogger; - int mVersion; - int mRevision; - boost::filesystem::path mFilename; - Jsontree_t jsontree; - //std::set mObjects; - bool _exists; - void makeEmptyJsonTree(); - - class MetadataCache - { - public: - MetadataCache(); - Jsontree_t get(const boost::filesystem::path &); - void put(const boost::filesystem::path &, const Jsontree_t &); - void erase(const boost::filesystem::path &); - boost::mutex &getMutex(); - private: - // there's a more efficient way to do this, KISS for now. - typedef std::list Lru_t; - typedef std::unordered_map > Lookup_t; - Lookup_t lookup; - Lru_t lru; - uint max_lru_size; - boost::mutex mutex; - }; - static MetadataCache jsonCache; - + // this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile + // meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended. + // aside from that, it will behave like the no_create ctor variant above + // MetadataFile(const boost::filesystem::path &path); + ~MetadataFile(); + + bool exists() const; + void printObjects() const; + int stat(struct stat*) const; + size_t getLength() const; + // returns the objects needed to update + std::vector metadataRead(off_t offset, size_t length) const; + // updates the metadatafile with new object + // int writeMetadata(const boost::filesystem::path &filename); + int writeMetadata(); + + // updates the name and length fields of an entry, given the offset + void updateEntry(off_t offset, const std::string& newName, size_t newLength); + void updateEntryLength(off_t offset, size_t newLength); + metadataObject addMetadataObject(const boost::filesystem::path& filename, size_t length); + bool getEntry(off_t offset, metadataObject* out) const; + void removeEntry(off_t offset); + void removeAllEntries(); + + // removes p from the json cache. p should be a fully qualified metadata file + static void deletedMeta(const boost::filesystem::path& p); + + static std::string getNewKeyFromOldKey(const std::string& oldKey, size_t length = 0); + static std::string getNewKey(std::string sourceName, size_t offset, size_t length); + static off_t getOffsetFromKey(const std::string& key); + static std::string getSourceFromKey(const std::string& key); + static size_t getLengthFromKey(const std::string& key); + static void setOffsetInKey(std::string& key, off_t newOffset); + static void setLengthInKey(std::string& key, size_t newLength); + + // breaks a key into its consitituent fields + static void breakout(const std::string& key, std::vector& out); + + off_t getMetadataNewObjectOffset(); + // this will be a singleton, which stores the config used + // by all MetadataFile instances so we don't have to keep bothering Config. + // members are public b/c i don't want to write accessors right now. Also who cares. + class MetadataConfig + { + public: + static MetadataConfig* get(); + size_t mObjectSize; + boost::filesystem::path msMetadataPath; + + private: + MetadataConfig(); + }; + + static void printKPIs(); + + typedef boost::shared_ptr Jsontree_t; + + private: + MetadataConfig* mpConfig; + SMLogging* mpLogger; + int mVersion; + int mRevision; + boost::filesystem::path mFilename; + Jsontree_t jsontree; + // std::set mObjects; + bool _exists; + void makeEmptyJsonTree(); + + class MetadataCache + { + public: + MetadataCache(); + Jsontree_t get(const boost::filesystem::path&); + void put(const boost::filesystem::path&, const Jsontree_t&); + void erase(const boost::filesystem::path&); + boost::mutex& getMutex(); + + private: + // there's a more efficient way to do this, KISS for now. + typedef std::list Lru_t; + typedef std::unordered_map > Lookup_t; + Lookup_t lookup; + Lru_t lru; + uint max_lru_size; + boost::mutex mutex; + }; + static MetadataCache jsonCache; }; - - -} +} // namespace storagemanager #endif /* METADATAFILE_H_ */ diff --git a/storage-manager/src/OpenTask.cpp b/storage-manager/src/OpenTask.cpp index a5b8e7265..267d253ee 100644 --- a/storage-manager/src/OpenTask.cpp +++ b/storage-manager/src/OpenTask.cpp @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #include "OpenTask.h" #include "messageFormat.h" #include "SMLogging.h" @@ -28,7 +26,6 @@ using namespace std; namespace storagemanager { - OpenTask::OpenTask(int sock, uint len) : PosixTask(sock, len) { } @@ -39,57 +36,57 @@ OpenTask::~OpenTask() bool OpenTask::run() { - /* - get the parameters - call IOManager to do the work - return the result - */ - SMLogging* logger = SMLogging::get(); - int success; - uint8_t buf[1024] = {0}; - - if (getLength() > 1023) - { - handleError("OpenTask read1", ENAMETOOLONG); - return true; - } - - success = read(buf, getLength()); - if (success<0) - { - handleError("OpenTask read2", errno); - return false; - } - - open_cmd *cmd = (open_cmd *) buf; + /* + get the parameters + call IOManager to do the work + return the result + */ + SMLogging* logger = SMLogging::get(); + int success; + uint8_t buf[1024] = {0}; - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"open filename %s mode %o.",cmd->filename,cmd->openmode); - #endif - sm_response *resp = (sm_response *) buf; - int err; - - try - { - err = ioc->open(cmd->filename, cmd->openmode, (struct stat *) &resp->payload); - } - catch (exception &e) - { - logger->log(LOG_ERR, "OpenTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - if (err) - { - handleError("OpenTask open", errno); - return true; - } - - resp->returnCode = 0; - success = write(*resp, sizeof(struct stat)); - if (!success) - handleError("OpenTask write", errno); - return success; + if (getLength() > 1023) + { + handleError("OpenTask read1", ENAMETOOLONG); + return true; + } + + success = read(buf, getLength()); + if (success < 0) + { + handleError("OpenTask read2", errno); + return false; + } + + open_cmd* cmd = (open_cmd*)buf; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "open filename %s mode %o.", cmd->filename, cmd->openmode); +#endif + sm_response* resp = (sm_response*)buf; + int err; + + try + { + err = ioc->open(cmd->filename, cmd->openmode, (struct stat*)&resp->payload); + } + catch (exception& e) + { + logger->log(LOG_ERR, "OpenTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + if (err) + { + handleError("OpenTask open", errno); + return true; + } + + resp->returnCode = 0; + success = write(*resp, sizeof(struct stat)); + if (!success) + handleError("OpenTask write", errno); + return success; } -} +} // namespace storagemanager diff --git a/storage-manager/src/OpenTask.h b/storage-manager/src/OpenTask.h index 4c2524c55..2581b0e6c 100644 --- a/storage-manager/src/OpenTask.h +++ b/storage-manager/src/OpenTask.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef OPENTASK_H_ #define OPENTASK_H_ @@ -23,19 +22,17 @@ namespace storagemanager { - class OpenTask : public PosixTask { - public: - OpenTask(int sock, uint length); - virtual ~OpenTask(); - - bool run(); - - private: - OpenTask(); + public: + OpenTask(int sock, uint length); + virtual ~OpenTask(); + + bool run(); + + private: + OpenTask(); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index 569a3c86f..cadcebc22 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -26,303 +26,308 @@ #include using namespace std; -namespace bf=boost::filesystem; +namespace bf = boost::filesystem; namespace storagemanager { - Ownership::Ownership() { - Config *config = Config::get(); - logger = SMLogging::get(); - - string sPrefixDepth = config->getValue("ObjectStorage", "common_prefix_depth"); - if (sPrefixDepth.empty()) - { - const char *msg = "Ownership: Need to specify ObjectStorage/common_prefix_depth in the storagemanager.cnf file"; - logger->log(LOG_CRIT, msg); - throw runtime_error(msg); - } - try - { - prefixDepth = stoul(sPrefixDepth, NULL, 0); - } - catch (invalid_argument &e) - { - const char *msg = "Ownership: Invalid value in ObjectStorage/common_prefix_depth"; - logger->log(LOG_CRIT, msg); - throw runtime_error(msg); - } - - metadataPrefix = config->getValue("ObjectStorage", "metadata_path"); - if (metadataPrefix.empty()) - { - const char *msg = "Ownership: Need to specify ObjectStorage/metadata_path in the storagemanager.cnf file"; - logger->log(LOG_CRIT, msg); - throw runtime_error(msg); - } - monitor = new Monitor(this); + Config* config = Config::get(); + logger = SMLogging::get(); + + string sPrefixDepth = config->getValue("ObjectStorage", "common_prefix_depth"); + if (sPrefixDepth.empty()) + { + const char* msg = + "Ownership: Need to specify ObjectStorage/common_prefix_depth in the storagemanager.cnf file"; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + try + { + prefixDepth = stoul(sPrefixDepth, NULL, 0); + } + catch (invalid_argument& e) + { + const char* msg = "Ownership: Invalid value in ObjectStorage/common_prefix_depth"; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + + metadataPrefix = config->getValue("ObjectStorage", "metadata_path"); + if (metadataPrefix.empty()) + { + const char* msg = "Ownership: Need to specify ObjectStorage/metadata_path in the storagemanager.cnf file"; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + monitor = new Monitor(this); } Ownership::~Ownership() { - delete monitor; - for (auto &it : ownedPrefixes) - releaseOwnership(it.first, true); + delete monitor; + for (auto& it : ownedPrefixes) + releaseOwnership(it.first, true); } -bf::path Ownership::get(const bf::path &p, bool getOwnership) +bf::path Ownership::get(const bf::path& p, bool getOwnership) { - bf::path ret, prefix, normalizedPath(p); - bf::path::const_iterator pit; - int i, levels; + bf::path ret, prefix, normalizedPath(p); + bf::path::const_iterator pit; + int i, levels; - normalizedPath.normalize(); - //cerr << "Ownership::get() param = " << normalizedPath.string() << endl; - if (prefixDepth > 0) - { - for (i = 0, pit = normalizedPath.begin(); i <= prefixDepth && pit != normalizedPath.end(); ++i, ++pit) - ; - if (pit != normalizedPath.end()) - prefix = *pit; - //cerr << "prefix is " << prefix.string() << endl; - for (levels = 0; pit != normalizedPath.end(); ++levels, ++pit) - ret /= *pit; - if (!getOwnership) - return ret; - if (levels <= 1) - throw runtime_error("Ownership: given path " + normalizedPath.string() + - " does not have minimum number of directories"); - } - else - { - ret = normalizedPath; - prefix = *(normalizedPath.begin()); - } - + normalizedPath.normalize(); + // cerr << "Ownership::get() param = " << normalizedPath.string() << endl; + if (prefixDepth > 0) + { + for (i = 0, pit = normalizedPath.begin(); i <= prefixDepth && pit != normalizedPath.end(); ++i, ++pit) + ; + if (pit != normalizedPath.end()) + prefix = *pit; + // cerr << "prefix is " << prefix.string() << endl; + for (levels = 0; pit != normalizedPath.end(); ++levels, ++pit) + ret /= *pit; if (!getOwnership) - return ret; - - mutex.lock(); - if (ownedPrefixes.find(prefix) == ownedPrefixes.end()) - { - mutex.unlock(); - takeOwnership(prefix); - } - else - { - // todo... replace this polling, and the similar polling in Cache, with proper condition vars. - while (ownedPrefixes[prefix] == false) - { - mutex.unlock(); - sleep(1); - mutex.lock(); - } - mutex.unlock(); - } - //cerr << "returning " << ret.string() << endl; + return ret; + if (levels <= 1) + throw runtime_error("Ownership: given path " + normalizedPath.string() + + " does not have minimum number of directories"); + } + else + { + ret = normalizedPath; + prefix = *(normalizedPath.begin()); + } + + if (!getOwnership) return ret; + + mutex.lock(); + if (ownedPrefixes.find(prefix) == ownedPrefixes.end()) + { + mutex.unlock(); + takeOwnership(prefix); + } + else + { + // todo... replace this polling, and the similar polling in Cache, with proper condition vars. + while (ownedPrefixes[prefix] == false) + { + mutex.unlock(); + sleep(1); + mutex.lock(); + } + mutex.unlock(); + } + // cerr << "returning " << ret.string() << endl; + return ret; } // minor timesaver -#define TOUCH(p, f) { \ - int fd = ::open((metadataPrefix/p/f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \ - if (fd >= 0) \ - ::close(fd); \ - else \ - { \ - char buf[80]; int saved_errno = errno; \ - cerr << "failed to touch " << metadataPrefix/p/f << " got " << strerror_r(saved_errno, buf, 80) << endl; \ - } \ +#define TOUCH(p, f) \ + { \ + int fd = ::open((metadataPrefix / p / f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \ + if (fd >= 0) \ + ::close(fd); \ + else \ + { \ + char buf[80]; \ + int saved_errno = errno; \ + cerr << "failed to touch " << metadataPrefix / p / f << " got " << strerror_r(saved_errno, buf, 80) \ + << endl; \ + } \ + } + +#define DELETE(p, f) ::unlink((metadataPrefix / p / f).string().c_str()); + +void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const +{ + while (!*doneFlushing) + { + TOUCH(prefix, "FLUSHING"); + try + { + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + } + catch (boost::thread_interrupted&) + { + } + } } -#define DELETE(p, f) ::unlink((metadataPrefix/p/f).string().c_str()); - -void Ownership::touchFlushing(const bf::path &prefix, volatile bool *doneFlushing) const +void Ownership::releaseOwnership(const bf::path& p, bool isDtor) { - while (!*doneFlushing) - { - TOUCH(prefix, "FLUSHING"); - try - { - boost::this_thread::sleep_for(boost::chrono::seconds(1)); - } - catch (boost::thread_interrupted &) - { } - } -} + logger->log(LOG_DEBUG, "Ownership: releasing ownership of %s", p.string().c_str()); + boost::unique_lock s(mutex); -void Ownership::releaseOwnership(const bf::path &p, bool isDtor) -{ - logger->log(LOG_DEBUG, "Ownership: releasing ownership of %s", p.string().c_str()); - boost::unique_lock s(mutex); - - auto it = ownedPrefixes.find(p); - if (it == ownedPrefixes.end()) - { - logger->log(LOG_DEBUG, "Ownership::releaseOwnership(): told to disown %s, but do not own it", p.string().c_str()); - return; - } - - if (isDtor) - { - // This is a quick release. If this is being destroyed, then it is through the graceful - // shutdown mechanism, which will flush data separately. - DELETE(p, "OWNED"); - DELETE(p, "FLUSHING"); - return; - } - else - ownedPrefixes.erase(it); - - s.unlock(); - - volatile bool done = false; - - // start flushing - boost::thread xfer([this, &p, &done] { this->touchFlushing(p, &done); }); - Synchronizer::get()->dropPrefix(p); - Cache::get()->dropPrefix(p); - done = true; - xfer.interrupt(); - xfer.join(); - - // update state + auto it = ownedPrefixes.find(p); + if (it == ownedPrefixes.end()) + { + logger->log(LOG_DEBUG, "Ownership::releaseOwnership(): told to disown %s, but do not own it", + p.string().c_str()); + return; + } + + if (isDtor) + { + // This is a quick release. If this is being destroyed, then it is through the graceful + // shutdown mechanism, which will flush data separately. DELETE(p, "OWNED"); DELETE(p, "FLUSHING"); + return; + } + else + ownedPrefixes.erase(it); + + s.unlock(); + + volatile bool done = false; + + // start flushing + boost::thread xfer([this, &p, &done] { this->touchFlushing(p, &done); }); + Synchronizer::get()->dropPrefix(p); + Cache::get()->dropPrefix(p); + done = true; + xfer.interrupt(); + xfer.join(); + + // update state + DELETE(p, "OWNED"); + DELETE(p, "FLUSHING"); } -void Ownership::_takeOwnership(const bf::path &p) +void Ownership::_takeOwnership(const bf::path& p) { - logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str()); - DELETE(p, "FLUSHING"); - DELETE(p, "REQUEST_TRANSFER"); - // TODO: need to consider errors taking ownership - TOUCH(p, "OWNED"); - mutex.lock(); - ownedPrefixes[p] = true; - mutex.unlock(); - Synchronizer::get()->newPrefix(p); - Cache::get()->newPrefix(p); + logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str()); + DELETE(p, "FLUSHING"); + DELETE(p, "REQUEST_TRANSFER"); + // TODO: need to consider errors taking ownership + TOUCH(p, "OWNED"); + mutex.lock(); + ownedPrefixes[p] = true; + mutex.unlock(); + Synchronizer::get()->newPrefix(p); + Cache::get()->newPrefix(p); } -void Ownership::takeOwnership(const bf::path &p) +void Ownership::takeOwnership(const bf::path& p) { - // If the prefix doesn't exist, ownership isn't possible yet. - if (!bf::is_directory(metadataPrefix/p)) - return; + // If the prefix doesn't exist, ownership isn't possible yet. + if (!bf::is_directory(metadataPrefix / p)) + return; - boost::unique_lock s(mutex); - - auto it = ownedPrefixes.find(p); - if (it != ownedPrefixes.end()) - return; - ownedPrefixes[p] = NULL; - s.unlock(); - - bool okToTransfer = false; - struct stat statbuf; - int err; - char buf[80]; - bf::path ownedPath = metadataPrefix/p/"OWNED"; - bf::path flushingPath = metadataPrefix/p/"FLUSHING"; - - // if it's not already owned, then we can take possession - err = ::stat(ownedPath.string().c_str(), &statbuf); - if (err && errno == ENOENT) - { - _takeOwnership(p); - return; - } - - TOUCH(p, "REQUEST_TRANSFER"); - time_t lastFlushTime = time(NULL); - while (!okToTransfer && time(NULL) < lastFlushTime + 10) - { - // if the OWNED file is deleted or if the flushing file isn't touched after 10 secs - // it is ok to take possession. - err = ::stat(ownedPath.string().c_str(), &statbuf); - if (err) - { - if (errno == ENOENT) - okToTransfer = true; - else - logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", strerror_r(errno, buf, 80), - ownedPath.string().c_str()); - } - err = ::stat(flushingPath.string().c_str(), &statbuf); - if (err && errno != ENOENT) - logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", strerror_r(errno, buf, 80), - flushingPath.string().c_str()); - else - { - logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str()); - if (!err) - lastFlushTime = statbuf.st_mtime; - } - if (!okToTransfer) - sleep(1); - } + boost::unique_lock s(mutex); + + auto it = ownedPrefixes.find(p); + if (it != ownedPrefixes.end()) + return; + ownedPrefixes[p] = NULL; + s.unlock(); + + bool okToTransfer = false; + struct stat statbuf; + int err; + char buf[80]; + bf::path ownedPath = metadataPrefix / p / "OWNED"; + bf::path flushingPath = metadataPrefix / p / "FLUSHING"; + + // if it's not already owned, then we can take possession + err = ::stat(ownedPath.string().c_str(), &statbuf); + if (err && errno == ENOENT) + { _takeOwnership(p); + return; + } + + TOUCH(p, "REQUEST_TRANSFER"); + time_t lastFlushTime = time(NULL); + while (!okToTransfer && time(NULL) < lastFlushTime + 10) + { + // if the OWNED file is deleted or if the flushing file isn't touched after 10 secs + // it is ok to take possession. + err = ::stat(ownedPath.string().c_str(), &statbuf); + if (err) + { + if (errno == ENOENT) + okToTransfer = true; + else + logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", + strerror_r(errno, buf, 80), ownedPath.string().c_str()); + } + err = ::stat(flushingPath.string().c_str(), &statbuf); + if (err && errno != ENOENT) + logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", + strerror_r(errno, buf, 80), flushingPath.string().c_str()); + else + { + logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str()); + if (!err) + lastFlushTime = statbuf.st_mtime; + } + if (!okToTransfer) + sleep(1); + } + _takeOwnership(p); } -Ownership::Monitor::Monitor(Ownership *_owner) : owner(_owner), stop(false) +Ownership::Monitor::Monitor(Ownership* _owner) : owner(_owner), stop(false) { - thread = boost::thread([this] { this->watchForInterlopers(); }); + thread = boost::thread([this] { this->watchForInterlopers(); }); } Ownership::Monitor::~Monitor() { - stop = true; - thread.interrupt(); - thread.join(); + stop = true; + thread.interrupt(); + thread.join(); } void Ownership::Monitor::watchForInterlopers() { - // look for requests to transfer ownership - struct stat statbuf; - int err; - char buf[80]; - vector releaseList; - - while (!stop) + // look for requests to transfer ownership + struct stat statbuf; + int err; + char buf[80]; + vector releaseList; + + while (!stop) + { + releaseList.clear(); + boost::unique_lock s(owner->mutex); + + for (auto& prefix : owner->ownedPrefixes) { - releaseList.clear(); - boost::unique_lock s(owner->mutex); - - for (auto &prefix : owner->ownedPrefixes) - { - if (stop) - break; - if (prefix.second == false) - continue; - bf::path p(owner->metadataPrefix/(prefix.first)/"REQUEST_TRANSFER"); - const char *cp = p.string().c_str(); + if (stop) + break; + if (prefix.second == false) + continue; + bf::path p(owner->metadataPrefix / (prefix.first) / "REQUEST_TRANSFER"); + const char* cp = p.string().c_str(); - err = ::stat(cp, &statbuf); - // release it if there's a release request only. Log it if there's an error other than - // that the file isn't there. - if (err == 0) - releaseList.push_back(prefix.first); - if (err < 0 && errno != ENOENT) - owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp, - strerror_r(errno, buf, 80)); - } - s.unlock(); - - for (auto &prefix : releaseList) - owner->releaseOwnership(prefix); - if (stop) - break; - try - { - boost::this_thread::sleep_for(boost::chrono::seconds(1)); - } - catch (boost::thread_interrupted &) - { } + err = ::stat(cp, &statbuf); + // release it if there's a release request only. Log it if there's an error other than + // that the file isn't there. + if (err == 0) + releaseList.push_back(prefix.first); + if (err < 0 && errno != ENOENT) + owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp, + strerror_r(errno, buf, 80)); } + s.unlock(); + + for (auto& prefix : releaseList) + owner->releaseOwnership(prefix); + if (stop) + break; + try + { + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + } + catch (boost::thread_interrupted&) + { + } + } } -} - +} // namespace storagemanager diff --git a/storage-manager/src/Ownership.h b/storage-manager/src/Ownership.h index 5327c940e..cae131b53 100644 --- a/storage-manager/src/Ownership.h +++ b/storage-manager/src/Ownership.h @@ -24,57 +24,56 @@ #include #include "SMLogging.h" -/* This class tracks the ownership of each prefix and manages ownership transfer. +/* This class tracks the ownership of each prefix and manages ownership transfer. Could we come up with a better name btw? */ namespace storagemanager { - class Ownership : public boost::noncopyable { - public: - Ownership(); - ~Ownership(); + public: + Ownership(); + ~Ownership(); - bool sharedFS(); - // returns the path "right shifted" by prefixDepth, and with ownership of that path. - // on error it throws a runtime exception - // setting getOwnership to false will return the modified path but not also take ownership - // of the returned prefix. - boost::filesystem::path get(const boost::filesystem::path &, bool getOwnership=true); - - - private: - int prefixDepth; - boost::filesystem::path metadataPrefix; - SMLogging *logger; - - void touchFlushing(const boost::filesystem::path &, volatile bool *) const; - void takeOwnership(const boost::filesystem::path &); - void releaseOwnership(const boost::filesystem::path &, bool isDtor = false); - void _takeOwnership(const boost::filesystem::path &); - - struct Monitor - { - Monitor(Ownership *); - ~Monitor(); - boost::thread thread; - Ownership *owner; - volatile bool stop; - void watchForInterlopers(); - }; - - // maps a prefix to a state. ownedPrefixes[p] == false means it's being init'd, == true means it's ready for use. - std::map ownedPrefixes; - Monitor *monitor; - boost::mutex mutex; + bool sharedFS(); + // returns the path "right shifted" by prefixDepth, and with ownership of that path. + // on error it throws a runtime exception + // setting getOwnership to false will return the modified path but not also take ownership + // of the returned prefix. + boost::filesystem::path get(const boost::filesystem::path&, bool getOwnership = true); + + private: + int prefixDepth; + boost::filesystem::path metadataPrefix; + SMLogging* logger; + + void touchFlushing(const boost::filesystem::path&, volatile bool*) const; + void takeOwnership(const boost::filesystem::path&); + void releaseOwnership(const boost::filesystem::path&, bool isDtor = false); + void _takeOwnership(const boost::filesystem::path&); + + struct Monitor + { + Monitor(Ownership*); + ~Monitor(); + boost::thread thread; + Ownership* owner; + volatile bool stop; + void watchForInterlopers(); + }; + + // maps a prefix to a state. ownedPrefixes[p] == false means it's being init'd, == true means it's ready + // for use. + std::map ownedPrefixes; + Monitor* monitor; + boost::mutex mutex; }; inline bool Ownership::sharedFS() { - return prefixDepth >= 0; + return prefixDepth >= 0; } -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/PingTask.cpp b/storage-manager/src/PingTask.cpp index 73e460cea..8451b7757 100644 --- a/storage-manager/src/PingTask.cpp +++ b/storage-manager/src/PingTask.cpp @@ -15,14 +15,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "PingTask.h" #include "messageFormat.h" #include namespace storagemanager { - PingTask::PingTask(int sock, uint len) : PosixTask(sock, len) { } @@ -33,28 +31,28 @@ PingTask::~PingTask() bool PingTask::run() { - // not much to check on for Milestone 1 - - uint8_t buf; - - if (getLength() > 1) - { - handleError("PingTask", E2BIG); - return true; - } - // consume the msg - int success = read(&buf, getLength()); - if (success<0) - { - handleError("PingTask", errno); - return false; - } - - // send generic success response - sm_response ret; - ret.returnCode = 0; - success = write(ret, 0); - return success; + // not much to check on for Milestone 1 + + uint8_t buf; + + if (getLength() > 1) + { + handleError("PingTask", E2BIG); + return true; + } + // consume the msg + int success = read(&buf, getLength()); + if (success < 0) + { + handleError("PingTask", errno); + return false; + } + + // send generic success response + sm_response ret; + ret.returnCode = 0; + success = write(ret, 0); + return success; } -} +} // namespace storagemanager diff --git a/storage-manager/src/PingTask.h b/storage-manager/src/PingTask.h index 735ad8828..225810754 100644 --- a/storage-manager/src/PingTask.h +++ b/storage-manager/src/PingTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef PINGTASK_H_ #define PINGTASK_H_ @@ -24,18 +22,17 @@ namespace storagemanager { - class PingTask : public PosixTask { - public: - PingTask(int sock, uint length); - virtual ~PingTask(); - - bool run(); - - private: - PingTask(); + public: + PingTask(int sock, uint length); + virtual ~PingTask(); + + bool run(); + + private: + PingTask(); }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/PosixTask.cpp b/storage-manager/src/PosixTask.cpp index a7096c136..2bd4dc790 100644 --- a/storage-manager/src/PosixTask.cpp +++ b/storage-manager/src/PosixTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "PosixTask.h" #include "messageFormat.h" #include "SMLogging.h" @@ -30,193 +29,192 @@ using namespace std; namespace storagemanager { - -PosixTask::PosixTask(int _sock, uint _length) : - sock(_sock), - totalLength(_length), - remainingLengthInStream(_length), - remainingLengthForCaller(_length), - bufferPos(0), - bufferLen(0) +PosixTask::PosixTask(int _sock, uint _length) + : sock(_sock) + , totalLength(_length) + , remainingLengthInStream(_length) + , remainingLengthForCaller(_length) + , bufferPos(0) + , bufferLen(0) { - ioc = IOCoordinator::get(); + ioc = IOCoordinator::get(); } PosixTask::~PosixTask() { - assert(remainingLengthForCaller == 0); - assert(remainingLengthInStream == 0); - consumeMsg(); + assert(remainingLengthForCaller == 0); + assert(remainingLengthInStream == 0); + consumeMsg(); } -void PosixTask::handleError(const char *name, int errCode) +void PosixTask::handleError(const char* name, int errCode) { - SMLogging* logger = SMLogging::get(); - char buf[80]; - - // send an error response if possible - sm_response *resp = (sm_response *) buf; - resp->returnCode = -1; - *((int *) resp->payload) = errCode; - bool success = write(*resp, 4); - - if (!success) - logger->log(LOG_ERR, "%s caught an error: %s.", name, strerror_r(errCode, buf, 80)); + SMLogging* logger = SMLogging::get(); + char buf[80]; + + // send an error response if possible + sm_response* resp = (sm_response*)buf; + resp->returnCode = -1; + *((int*)resp->payload) = errCode; + bool success = write(*resp, 4); + + if (!success) + logger->log(LOG_ERR, "%s caught an error: %s.", name, strerror_r(errCode, buf, 80)); } uint PosixTask::getRemainingLength() { - return remainingLengthForCaller; + return remainingLengthForCaller; } uint PosixTask::getLength() { - return totalLength; + return totalLength; } // todo, need this to return an int instead of a bool b/c it modifies the length of the read -int PosixTask::read(uint8_t *buf, uint length) +int PosixTask::read(uint8_t* buf, uint length) { - if (length > remainingLengthForCaller) - length = remainingLengthForCaller; - if (length == 0) - return 0; - - uint count = 0; - int err; - + if (length > remainingLengthForCaller) + length = remainingLengthForCaller; + if (length == 0) + return 0; - // copy data from the local buffer first. - uint dataInBuffer = bufferLen - bufferPos; + uint count = 0; + int err; - if (length <= dataInBuffer) - { - memcpy(buf, &localBuffer[bufferPos], length); - count = length; - bufferPos += length; - remainingLengthForCaller -= length; - } - else if (dataInBuffer > 0) - { - memcpy(buf, &localBuffer[bufferPos], dataInBuffer); - count = dataInBuffer; - bufferPos += dataInBuffer; - remainingLengthForCaller -= dataInBuffer; - } - - // read the remaining requested amount from the stream. - // ideally, combine the recv here with the recv below that refills the local - // buffer. - while (count < length) - { - err = ::recv(sock, &buf[count], length - count, 0); - if (err < 0) - return err; + // copy data from the local buffer first. + uint dataInBuffer = bufferLen - bufferPos; - count += err; - remainingLengthInStream -= err; - remainingLengthForCaller -= err; - } - - /* The caller's request has been satisfied here. If there is remaining data in the stream - get what's available. */ - primeBuffer(); - return count; + if (length <= dataInBuffer) + { + memcpy(buf, &localBuffer[bufferPos], length); + count = length; + bufferPos += length; + remainingLengthForCaller -= length; + } + else if (dataInBuffer > 0) + { + memcpy(buf, &localBuffer[bufferPos], dataInBuffer); + count = dataInBuffer; + bufferPos += dataInBuffer; + remainingLengthForCaller -= dataInBuffer; + } + + // read the remaining requested amount from the stream. + // ideally, combine the recv here with the recv below that refills the local + // buffer. + while (count < length) + { + err = ::recv(sock, &buf[count], length - count, 0); + if (err < 0) + return err; + + count += err; + remainingLengthInStream -= err; + remainingLengthForCaller -= err; + } + + /* The caller's request has been satisfied here. If there is remaining data in the stream + get what's available. */ + primeBuffer(); + return count; } void PosixTask::primeBuffer() { - if (remainingLengthInStream > 0) + if (remainingLengthInStream > 0) + { + // Reset the buffer to allow a larger read. + if (bufferLen == bufferPos) { - // Reset the buffer to allow a larger read. - if (bufferLen == bufferPos) - { - bufferLen = 0; - bufferPos = 0; - } - else if (bufferLen - bufferPos < 1024) // if < 1024 in the buffer, move data to the front - { - // debating whether it is more efficient to use a circular buffer + more - // recv's, or to move data to reduce the # of recv's. WAG: moving data. - memmove(localBuffer, &localBuffer[bufferPos], bufferLen - bufferPos); - bufferLen -= bufferPos; - bufferPos = 0; - } - - uint toRead = min(remainingLengthInStream, bufferSize - bufferLen); - int err = ::recv(sock, &localBuffer[bufferLen], toRead, MSG_DONTWAIT); - // ignoring errors here since this is supposed to be silent. - // errors will be caught by the next read - if (err > 0) - { - bufferLen += err; - remainingLengthInStream -= err; - } + bufferLen = 0; + bufferPos = 0; } + else if (bufferLen - bufferPos < 1024) // if < 1024 in the buffer, move data to the front + { + // debating whether it is more efficient to use a circular buffer + more + // recv's, or to move data to reduce the # of recv's. WAG: moving data. + memmove(localBuffer, &localBuffer[bufferPos], bufferLen - bufferPos); + bufferLen -= bufferPos; + bufferPos = 0; + } + + uint toRead = min(remainingLengthInStream, bufferSize - bufferLen); + int err = ::recv(sock, &localBuffer[bufferLen], toRead, MSG_DONTWAIT); + // ignoring errors here since this is supposed to be silent. + // errors will be caught by the next read + if (err > 0) + { + bufferLen += err; + remainingLengthInStream -= err; + } + } } -bool PosixTask::write(const uint8_t *buf, uint len) +bool PosixTask::write(const uint8_t* buf, uint len) { - int err; - uint count = 0; - - while (count < len) - { - err = ::send(sock, &buf[count], len - count, 0); - if (err < 0) - return false; - count += err; - } - return true; + int err; + uint count = 0; + + while (count < len) + { + err = ::send(sock, &buf[count], len - count, 0); + if (err < 0) + return false; + count += err; + } + return true; } -bool PosixTask::write(sm_response &resp, uint payloadLength) +bool PosixTask::write(sm_response& resp, uint payloadLength) { - int err; - uint count = 0; - uint8_t *buf = (uint8_t *) &resp; - - resp.header.type = SM_MSG_START; - resp.header.flags = 0; - resp.header.payloadLen = payloadLength + sizeof(sm_response) - sizeof(sm_msg_header); - uint toSend = payloadLength + sizeof(sm_response); - - while (count < toSend) - { - err = ::send(sock, &buf[count], toSend - count, 0); - if (err < 0) - return false; - count += err; - } - return true; + int err; + uint count = 0; + uint8_t* buf = (uint8_t*)&resp; + + resp.header.type = SM_MSG_START; + resp.header.flags = 0; + resp.header.payloadLen = payloadLength + sizeof(sm_response) - sizeof(sm_msg_header); + uint toSend = payloadLength + sizeof(sm_response); + + while (count < toSend) + { + err = ::send(sock, &buf[count], toSend - count, 0); + if (err < 0) + return false; + count += err; + } + return true; } -bool PosixTask::write(const vector &buf) +bool PosixTask::write(const vector& buf) { - return write(&buf[0], buf.size()); + return write(&buf[0], buf.size()); } void PosixTask::consumeMsg() { - SMLogging* logger = SMLogging::get(); + SMLogging* logger = SMLogging::get(); - uint8_t buf[1024]; - int err; - - bufferLen = 0; - bufferPos = 0; - remainingLengthForCaller = 0; - - while (remainingLengthInStream > 0) + uint8_t buf[1024]; + int err; + + bufferLen = 0; + bufferPos = 0; + remainingLengthForCaller = 0; + + while (remainingLengthInStream > 0) + { + logger->log(LOG_WARNING, "PosixTask::consumeMsg(): Discarding the tail end of a partial msg."); + err = ::recv(sock, buf, min(remainingLengthInStream, 1024), 0); + if (err <= 0) { - logger->log(LOG_WARNING, "PosixTask::consumeMsg(): Discarding the tail end of a partial msg."); - err = ::recv(sock, buf, min(remainingLengthInStream, 1024), 0); - if (err <= 0) { - remainingLengthInStream = 0; - break; - } - remainingLengthInStream -= err; + remainingLengthInStream = 0; + break; } + remainingLengthInStream -= err; + } } -} +} // namespace storagemanager diff --git a/storage-manager/src/PosixTask.h b/storage-manager/src/PosixTask.h index c8025eb88..427b0f8fd 100644 --- a/storage-manager/src/PosixTask.h +++ b/storage-manager/src/PosixTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef POSIX_TASK_H_ #define POSIX_TASK_H_ @@ -29,41 +27,40 @@ namespace storagemanager { - class PosixTask { - public: - PosixTask(int sock, uint length); - virtual ~PosixTask(); - - // this should return false if there was a network error, true otherwise including for other errors - virtual bool run() = 0; - void primeBuffer(); - - protected: - int read(uint8_t *buf, uint length); - bool write(const std::vector &buf); - bool write(sm_response &resp, uint payloadLength); - bool write(const uint8_t *buf, uint length); - void consumeMsg(); // drains the remaining portion of the message - uint getLength(); // returns the total length of the msg - uint getRemainingLength(); // returns the remaining length from the caller's perspective - void handleError(const char *name, int errCode); - - IOCoordinator *ioc; - - private: - PosixTask(); - - int sock; - int totalLength; - uint remainingLengthInStream; - uint remainingLengthForCaller; - static const uint bufferSize = 4096; - uint8_t localBuffer[bufferSize]; - uint bufferPos; - uint bufferLen; + public: + PosixTask(int sock, uint length); + virtual ~PosixTask(); + + // this should return false if there was a network error, true otherwise including for other errors + virtual bool run() = 0; + void primeBuffer(); + + protected: + int read(uint8_t* buf, uint length); + bool write(const std::vector& buf); + bool write(sm_response& resp, uint payloadLength); + bool write(const uint8_t* buf, uint length); + void consumeMsg(); // drains the remaining portion of the message + uint getLength(); // returns the total length of the msg + uint getRemainingLength(); // returns the remaining length from the caller's perspective + void handleError(const char* name, int errCode); + + IOCoordinator* ioc; + + private: + PosixTask(); + + int sock; + int totalLength; + uint remainingLengthInStream; + uint remainingLengthForCaller; + static const uint bufferSize = 4096; + uint8_t localBuffer[bufferSize]; + uint bufferPos; + uint bufferLen; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/PrefixCache.cpp b/storage-manager/src/PrefixCache.cpp index 86192faae..cb1faaee8 100644 --- a/storage-manager/src/PrefixCache.cpp +++ b/storage-manager/src/PrefixCache.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "PrefixCache.h" #include "Cache.h" #include "Config.h" @@ -34,642 +33,647 @@ namespace bf = boost::filesystem; namespace storagemanager { - -PrefixCache::PrefixCache(const bf::path &prefix) : firstDir(prefix), currentCacheSize(0) +PrefixCache::PrefixCache(const bf::path& prefix) : firstDir(prefix), currentCacheSize(0) { - Config *conf = Config::get(); - logger = SMLogging::get(); - replicator = Replicator::get(); - downloader = Cache::get()->getDownloader(); - - string stmp = conf->getValue("Cache", "cache_size"); - if (stmp.empty()) - { - logger->log(LOG_CRIT, "Cache/cache_size is not set"); - throw runtime_error("Please set Cache/cache_size in the storagemanager.cnf file"); - } - try - { - maxCacheSize = stoul(stmp); - } - catch (invalid_argument &) - { - logger->log(LOG_CRIT, "Cache/cache_size is not a number"); - throw runtime_error("Please set Cache/cache_size to a number"); - } - //cout << "Cache got cache size " << maxCacheSize << endl; - - stmp = conf->getValue("ObjectStorage", "object_size"); - if (stmp.empty()) - { - logger->log(LOG_CRIT, "ObjectStorage/object_size is not set"); - throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file"); - } - try - { - objectSize = stoul(stmp); - } - catch (invalid_argument &) - { - logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number"); - throw runtime_error("Please set ObjectStorage/object_size to a number"); - } - - cachePrefix = conf->getValue("Cache", "path"); - if (cachePrefix.empty()) - { - logger->log(LOG_CRIT, "Cache/path is not set"); - throw runtime_error("Please set Cache/path in the storagemanager.cnf file"); - } - cachePrefix /= firstDir; - - try - { - bf::create_directories(cachePrefix); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what()); - throw e; - } - - stmp = conf->getValue("ObjectStorage", "journal_path"); - if (stmp.empty()) - { - logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); - throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); - } - journalPrefix = stmp; - journalPrefix /= firstDir; - + Config* conf = Config::get(); + logger = SMLogging::get(); + replicator = Replicator::get(); + downloader = Cache::get()->getDownloader(); + + string stmp = conf->getValue("Cache", "cache_size"); + if (stmp.empty()) + { + logger->log(LOG_CRIT, "Cache/cache_size is not set"); + throw runtime_error("Please set Cache/cache_size in the storagemanager.cnf file"); + } + try + { + maxCacheSize = stoul(stmp); + } + catch (invalid_argument&) + { + logger->log(LOG_CRIT, "Cache/cache_size is not a number"); + throw runtime_error("Please set Cache/cache_size to a number"); + } + // cout << "Cache got cache size " << maxCacheSize << endl; + + stmp = conf->getValue("ObjectStorage", "object_size"); + if (stmp.empty()) + { + logger->log(LOG_CRIT, "ObjectStorage/object_size is not set"); + throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file"); + } + try + { + objectSize = stoul(stmp); + } + catch (invalid_argument&) + { + logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number"); + throw runtime_error("Please set ObjectStorage/object_size to a number"); + } + + cachePrefix = conf->getValue("Cache", "path"); + if (cachePrefix.empty()) + { + logger->log(LOG_CRIT, "Cache/path is not set"); + throw runtime_error("Please set Cache/path in the storagemanager.cnf file"); + } + cachePrefix /= firstDir; + + try + { + bf::create_directories(cachePrefix); + } + catch (exception& e) + { + logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what()); + throw e; + } + + stmp = conf->getValue("ObjectStorage", "journal_path"); + if (stmp.empty()) + { + logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); + throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); + } + journalPrefix = stmp; + journalPrefix /= firstDir; + + bf::create_directories(journalPrefix); + try + { bf::create_directories(journalPrefix); - try - { - bf::create_directories(journalPrefix); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what()); - throw e; - } - - lru_mutex.lock(); // unlocked by populate() when it's done - // Ideally put this in background but has to be synchronous with write calls - populate(); - //boost::thread t([this] { this->populate(); }); - //t.detach(); + } + catch (exception& e) + { + logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what()); + throw e; + } + + lru_mutex.lock(); // unlocked by populate() when it's done + // Ideally put this in background but has to be synchronous with write calls + populate(); + // boost::thread t([this] { this->populate(); }); + // t.detach(); } PrefixCache::~PrefixCache() { - /* This and shutdown() need to do whatever is necessary to leave cache contents in a safe - state on disk. Does anything need to be done toward that? - */ + /* This and shutdown() need to do whatever is necessary to leave cache contents in a safe + state on disk. Does anything need to be done toward that? + */ } void PrefixCache::populate() { - Synchronizer *sync = Synchronizer::get(); - bf::directory_iterator dir(cachePrefix); - bf::directory_iterator dend; - vector newObjects; - while (dir != dend) + Synchronizer* sync = Synchronizer::get(); + bf::directory_iterator dir(cachePrefix); + bf::directory_iterator dend; + vector newObjects; + while (dir != dend) + { + // put everything in lru & m_lru + const bf::path& p = dir->path(); + if (bf::is_regular_file(p)) { - // put everything in lru & m_lru - const bf::path &p = dir->path(); - if (bf::is_regular_file(p)) - { - lru.push_back(p.filename().string()); - auto last = lru.end(); - m_lru.insert(--last); - currentCacheSize += bf::file_size(*dir); - newObjects.push_back(p.filename().string()); - } - else if (p != cachePrefix/downloader->getTmpPath()) - logger->log(LOG_WARNING, "Cache: found something in the cache that does not belong '%s'", p.string().c_str()); - ++dir; + lru.push_back(p.filename().string()); + auto last = lru.end(); + m_lru.insert(--last); + currentCacheSize += bf::file_size(*dir); + newObjects.push_back(p.filename().string()); } - sync->newObjects(firstDir, newObjects); - newObjects.clear(); - - // account for what's in the journal dir - vector > newJournals; - dir = bf::directory_iterator(journalPrefix); - while (dir != dend) + else if (p != cachePrefix / downloader->getTmpPath()) + logger->log(LOG_WARNING, "Cache: found something in the cache that does not belong '%s'", + p.string().c_str()); + ++dir; + } + sync->newObjects(firstDir, newObjects); + newObjects.clear(); + + // account for what's in the journal dir + vector > newJournals; + dir = bf::directory_iterator(journalPrefix); + while (dir != dend) + { + const bf::path& p = dir->path(); + if (bf::is_regular_file(p)) { - const bf::path &p = dir->path(); - if (bf::is_regular_file(p)) - { - if (p.extension() == ".journal") - { - size_t s = bf::file_size(*dir); - currentCacheSize += s; - newJournals.push_back(pair(p.stem().string(), s)); - } - else - logger->log(LOG_WARNING, "Cache: found a file in the journal dir that does not belong '%s'", p.string().c_str()); - } - else - logger->log(LOG_WARNING, "Cache: found something in the journal dir that does not belong '%s'", p.string().c_str()); - ++dir; + if (p.extension() == ".journal") + { + size_t s = bf::file_size(*dir); + currentCacheSize += s; + newJournals.push_back(pair(p.stem().string(), s)); + } + else + logger->log(LOG_WARNING, "Cache: found a file in the journal dir that does not belong '%s'", + p.string().c_str()); } - lru_mutex.unlock(); - sync->newJournalEntries(firstDir, newJournals); + else + logger->log(LOG_WARNING, "Cache: found something in the journal dir that does not belong '%s'", + p.string().c_str()); + ++dir; + } + lru_mutex.unlock(); + sync->newJournalEntries(firstDir, newJournals); } // be careful using this! SM should be idle. No ongoing reads or writes. void PrefixCache::validateCacheSize() { - boost::unique_lock s(lru_mutex); - - if (!doNotEvict.empty() || !toBeDeleted.empty()) - { - cout << "Not safe to use validateCacheSize() at the moment." << endl; - return; - } - - size_t oldSize = currentCacheSize; - currentCacheSize = 0; - m_lru.clear(); - lru.clear(); - populate(); - - if (oldSize != currentCacheSize) - logger->log(LOG_DEBUG, "PrefixCache::validateCacheSize(): found a discrepancy. Actual size is %lld, had %lld.", - currentCacheSize, oldSize); - else - logger->log(LOG_DEBUG, "PrefixCache::validateCacheSize(): Cache size accounting agrees with reality for now."); -} - -void PrefixCache::read(const vector &keys) -{ - /* Move existing keys to the back of the LRU, start downloading nonexistant keys. - */ - vector keysToFetch; - vector dlErrnos; - vector dlSizes; - - boost::unique_lock s(lru_mutex); - - M_LRU_t::iterator mit; - for (const string &key : keys) - { - mit = m_lru.find(key); - if (mit != m_lru.end()) - { - addToDNE(mit->lit); - lru.splice(lru.end(), lru, mit->lit); // move them to the back so they are last to pick for eviction - } - else - { - // There's window where the file has been downloaded but is not yet - // added to the lru structs. However it is in the DNE. If it is in the DNE, then it is also - // in Downloader's map. So, this thread needs to start the download if it's not in the - // DNE or if there's an existing download that hasn't finished yet. Starting the download - // includes waiting for an existing download to finish, which from this class's pov is the - // same thing. - if (doNotEvict.find(key) == doNotEvict.end() || downloader->inProgress(key)) - keysToFetch.push_back(&key); - else - cout << "Cache: detected and stopped a racey download" << endl; - addToDNE(key); - } - } - if (keysToFetch.empty()) - return; - - downloader->download(keysToFetch, &dlErrnos, &dlSizes, cachePrefix, &lru_mutex); - - size_t sum_sizes = 0; - for (uint i = 0; i < keysToFetch.size(); ++i) - { - // downloads with size 0 didn't actually happen, either because it - // was a preexisting download (another read() call owns it), or because - // there was an error downloading it. Use size == 0 as an indication of - // what to add to the cache. Also needs to verify that the file was not deleted, - // indicated by existence in doNotEvict. - if (dlSizes[i] != 0) - { - if (doNotEvict.find(*keysToFetch[i]) != doNotEvict.end()) - { - sum_sizes += dlSizes[i]; - lru.push_back(*keysToFetch[i]); - LRU_t::iterator lit = lru.end(); - m_lru.insert(--lit); // I dislike this way of grabbing the last iterator in a list. - } - else // it was downloaded, but a deletion happened so we have to toss it - { - cout << "removing a file that was deleted by another thread during download" << endl; - bf::remove(cachePrefix / (*keysToFetch[i])); - } - } - } - - // move everything in keys to the back of the lru (yes, again) - for (const string &key : keys) - { - mit = m_lru.find(key); - if (mit != m_lru.end()) // all of the files exist, just not all of them are 'owned by' this thread. - lru.splice(lru.end(), lru, mit->lit); - } - - // fix cache size - //_makeSpace(sum_sizes); - currentCacheSize += sum_sizes; + boost::unique_lock s(lru_mutex); + + if (!doNotEvict.empty() || !toBeDeleted.empty()) + { + cout << "Not safe to use validateCacheSize() at the moment." << endl; + return; + } + + size_t oldSize = currentCacheSize; + currentCacheSize = 0; + m_lru.clear(); + lru.clear(); + populate(); + + if (oldSize != currentCacheSize) + logger->log(LOG_DEBUG, + "PrefixCache::validateCacheSize(): found a discrepancy. Actual size is %lld, had %lld.", + currentCacheSize, oldSize); + else + logger->log(LOG_DEBUG, + "PrefixCache::validateCacheSize(): Cache size accounting agrees with reality for now."); } -void PrefixCache::doneReading(const vector &keys) +void PrefixCache::read(const vector& keys) { - boost::unique_lock s(lru_mutex); - for (const string &key : keys) + /* Move existing keys to the back of the LRU, start downloading nonexistant keys. + */ + vector keysToFetch; + vector dlErrnos; + vector dlSizes; + + boost::unique_lock s(lru_mutex); + + M_LRU_t::iterator mit; + for (const string& key : keys) + { + mit = m_lru.find(key); + if (mit != m_lru.end()) { - removeFromDNE(key); - // most should be in the map. - // debateable whether it's faster to look up the list iterator and use it - // or whether it's faster to bypass that and use strings only. - - //const auto &it = m_lru.find(key); - //if (it != m_lru.end()) - // removeFromDNE(it->lit); + addToDNE(mit->lit); + lru.splice(lru.end(), lru, mit->lit); // move them to the back so they are last to pick for eviction } - _makeSpace(0); + else + { + // There's window where the file has been downloaded but is not yet + // added to the lru structs. However it is in the DNE. If it is in the DNE, then it is also + // in Downloader's map. So, this thread needs to start the download if it's not in the + // DNE or if there's an existing download that hasn't finished yet. Starting the download + // includes waiting for an existing download to finish, which from this class's pov is the + // same thing. + if (doNotEvict.find(key) == doNotEvict.end() || downloader->inProgress(key)) + keysToFetch.push_back(&key); + else + cout << "Cache: detected and stopped a racey download" << endl; + addToDNE(key); + } + } + if (keysToFetch.empty()) + return; + + downloader->download(keysToFetch, &dlErrnos, &dlSizes, cachePrefix, &lru_mutex); + + size_t sum_sizes = 0; + for (uint i = 0; i < keysToFetch.size(); ++i) + { + // downloads with size 0 didn't actually happen, either because it + // was a preexisting download (another read() call owns it), or because + // there was an error downloading it. Use size == 0 as an indication of + // what to add to the cache. Also needs to verify that the file was not deleted, + // indicated by existence in doNotEvict. + if (dlSizes[i] != 0) + { + if (doNotEvict.find(*keysToFetch[i]) != doNotEvict.end()) + { + sum_sizes += dlSizes[i]; + lru.push_back(*keysToFetch[i]); + LRU_t::iterator lit = lru.end(); + m_lru.insert(--lit); // I dislike this way of grabbing the last iterator in a list. + } + else // it was downloaded, but a deletion happened so we have to toss it + { + cout << "removing a file that was deleted by another thread during download" << endl; + bf::remove(cachePrefix / (*keysToFetch[i])); + } + } + } + + // move everything in keys to the back of the lru (yes, again) + for (const string& key : keys) + { + mit = m_lru.find(key); + if (mit != m_lru.end()) // all of the files exist, just not all of them are 'owned by' this thread. + lru.splice(lru.end(), lru, mit->lit); + } + + // fix cache size + //_makeSpace(sum_sizes); + currentCacheSize += sum_sizes; +} + +void PrefixCache::doneReading(const vector& keys) +{ + boost::unique_lock s(lru_mutex); + for (const string& key : keys) + { + removeFromDNE(key); + // most should be in the map. + // debateable whether it's faster to look up the list iterator and use it + // or whether it's faster to bypass that and use strings only. + + // const auto &it = m_lru.find(key); + // if (it != m_lru.end()) + // removeFromDNE(it->lit); + } + _makeSpace(0); } void PrefixCache::doneWriting() { - makeSpace(0); + makeSpace(0); } -PrefixCache::DNEElement::DNEElement(const LRU_t::iterator &k) : key(k), refCount(1) +PrefixCache::DNEElement::DNEElement(const LRU_t::iterator& k) : key(k), refCount(1) { } -PrefixCache::DNEElement::DNEElement(const string &k) : sKey(k), refCount(1) +PrefixCache::DNEElement::DNEElement(const string& k) : sKey(k), refCount(1) { } -void PrefixCache::addToDNE(const DNEElement &key) +void PrefixCache::addToDNE(const DNEElement& key) { - DNE_t::iterator it = doNotEvict.find(key); - if (it != doNotEvict.end()) - { - DNEElement &dnee = const_cast(*it); - ++(dnee.refCount); - } - else - doNotEvict.insert(key); -} - -void PrefixCache::removeFromDNE(const DNEElement &key) -{ - DNE_t::iterator it = doNotEvict.find(key); - if (it == doNotEvict.end()) - return; - DNEElement &dnee = const_cast(*it); - if (--(dnee.refCount) == 0) - doNotEvict.erase(it); -} - -const bf::path & PrefixCache::getCachePath() -{ - return cachePrefix; + DNE_t::iterator it = doNotEvict.find(key); + if (it != doNotEvict.end()) + { + DNEElement& dnee = const_cast(*it); + ++(dnee.refCount); + } + else + doNotEvict.insert(key); } -const bf::path & PrefixCache::getJournalPath() +void PrefixCache::removeFromDNE(const DNEElement& key) { - return journalPrefix; -} - -void PrefixCache::exists(const vector &keys, vector *out) const -{ - out->resize(keys.size()); - boost::unique_lock s(lru_mutex); - for (uint i = 0; i < keys.size(); i++) - (*out)[i] = (m_lru.find(keys[i]) != m_lru.end()); + DNE_t::iterator it = doNotEvict.find(key); + if (it == doNotEvict.end()) + return; + DNEElement& dnee = const_cast(*it); + if (--(dnee.refCount) == 0) + doNotEvict.erase(it); } -bool PrefixCache::exists(const string &key) const +const bf::path& PrefixCache::getCachePath() { - boost::unique_lock s(lru_mutex); - return m_lru.find(key) != m_lru.end(); + return cachePrefix; } -void PrefixCache::newObject(const string &key, size_t size) +const bf::path& PrefixCache::getJournalPath() { - boost::unique_lock s(lru_mutex); - assert(m_lru.find(key) == m_lru.end()); - if (m_lru.find(key) != m_lru.end()) - { - //This should never happen but was in MCOL-3499 - //Remove this when PrefixCache ctor can call populate() synchronous with write calls - logger->log(LOG_ERR, "PrefixCache::newObject(): key exists in m_lru already %s",key.c_str()); - } - //_makeSpace(size); - lru.push_back(key); - LRU_t::iterator back = lru.end(); - m_lru.insert(--back); - currentCacheSize += size; + return journalPrefix; +} + +void PrefixCache::exists(const vector& keys, vector* out) const +{ + out->resize(keys.size()); + boost::unique_lock s(lru_mutex); + for (uint i = 0; i < keys.size(); i++) + (*out)[i] = (m_lru.find(keys[i]) != m_lru.end()); +} + +bool PrefixCache::exists(const string& key) const +{ + boost::unique_lock s(lru_mutex); + return m_lru.find(key) != m_lru.end(); +} + +void PrefixCache::newObject(const string& key, size_t size) +{ + boost::unique_lock s(lru_mutex); + assert(m_lru.find(key) == m_lru.end()); + if (m_lru.find(key) != m_lru.end()) + { + // This should never happen but was in MCOL-3499 + // Remove this when PrefixCache ctor can call populate() synchronous with write calls + logger->log(LOG_ERR, "PrefixCache::newObject(): key exists in m_lru already %s", key.c_str()); + } + //_makeSpace(size); + lru.push_back(key); + LRU_t::iterator back = lru.end(); + m_lru.insert(--back); + currentCacheSize += size; } void PrefixCache::newJournalEntry(size_t size) { - boost::unique_lock s(lru_mutex); - //_makeSpace(size); - currentCacheSize += size; + boost::unique_lock s(lru_mutex); + //_makeSpace(size); + currentCacheSize += size; } void PrefixCache::deletedJournal(size_t size) { - boost::unique_lock s(lru_mutex); + boost::unique_lock s(lru_mutex); - if (currentCacheSize >= size) - currentCacheSize -= size; - else - { - ostringstream oss; - oss << "PrefixCache::deletedJournal(): Detected an accounting error."; - logger->log(LOG_WARNING, oss.str().c_str()); - currentCacheSize = 0; - } + if (currentCacheSize >= size) + currentCacheSize -= size; + else + { + ostringstream oss; + oss << "PrefixCache::deletedJournal(): Detected an accounting error."; + logger->log(LOG_WARNING, oss.str().c_str()); + currentCacheSize = 0; + } } -void PrefixCache::deletedObject(const string &key, size_t size) +void PrefixCache::deletedObject(const string& key, size_t size) { - boost::unique_lock s(lru_mutex); + boost::unique_lock s(lru_mutex); - M_LRU_t::iterator mit = m_lru.find(key); - assert(mit != m_lru.end()); - - // if it's being flushed, let makeSpace() do the deleting - if (toBeDeleted.find(mit->lit) == toBeDeleted.end()) + M_LRU_t::iterator mit = m_lru.find(key); + assert(mit != m_lru.end()); + + // if it's being flushed, let makeSpace() do the deleting + if (toBeDeleted.find(mit->lit) == toBeDeleted.end()) + { + doNotEvict.erase(mit->lit); + lru.erase(mit->lit); + m_lru.erase(mit); + if (currentCacheSize >= size) + currentCacheSize -= size; + else { - doNotEvict.erase(mit->lit); - lru.erase(mit->lit); - m_lru.erase(mit); - if (currentCacheSize >= size) - currentCacheSize -= size; - else - { - ostringstream oss; - oss << "PrefixCache::deletedObject(): Detected an accounting error."; - logger->log(LOG_WARNING, oss.str().c_str()); - currentCacheSize = 0; - } + ostringstream oss; + oss << "PrefixCache::deletedObject(): Detected an accounting error."; + logger->log(LOG_WARNING, oss.str().c_str()); + currentCacheSize = 0; } + } } void PrefixCache::setMaxCacheSize(size_t size) { - boost::unique_lock s(lru_mutex); - if (size < maxCacheSize) - _makeSpace(maxCacheSize - size); - maxCacheSize = size; + boost::unique_lock s(lru_mutex); + if (size < maxCacheSize) + _makeSpace(maxCacheSize - size); + maxCacheSize = size; } void PrefixCache::makeSpace(size_t size) { - boost::unique_lock s(lru_mutex); - _makeSpace(size); + boost::unique_lock s(lru_mutex); + _makeSpace(size); } size_t PrefixCache::getMaxCacheSize() const { - return maxCacheSize; + return maxCacheSize; } // call this holding lru_mutex void PrefixCache::_makeSpace(size_t size) { - ssize_t thisMuch = currentCacheSize + size - maxCacheSize; - if (thisMuch <= 0) - return; - - LRU_t::iterator it; - while (thisMuch > 0 && !lru.empty()) + ssize_t thisMuch = currentCacheSize + size - maxCacheSize; + if (thisMuch <= 0) + return; + + LRU_t::iterator it; + while (thisMuch > 0 && !lru.empty()) + { + it = lru.begin(); + // find the first element not being either read() right now or being processed by another + // makeSpace() call. + while (it != lru.end()) { - it = lru.begin(); - // find the first element not being either read() right now or being processed by another - // makeSpace() call. - while (it != lru.end()) - { - // make sure it's not currently being read or being flushed by another _makeSpace() call - if ((doNotEvict.find(it) == doNotEvict.end()) && (toBeDeleted.find(it) == toBeDeleted.end())) - break; - ++it; - } - if (it == lru.end()) - { - // nothing can be deleted right now - return; - } - - // ran into this a couple times, still happens as of commit 948ee1aa5 - // BT: made this more visable in logging. - // likely related to MCOL-3499 and lru containing double entries. - if (!bf::exists(cachePrefix / *it)) - logger->log(LOG_WARNING, "PrefixCache::makeSpace(): doesn't exist, %s/%s",cachePrefix.string().c_str(),((string)(*it)).c_str()); - assert(bf::exists(cachePrefix / *it)); - /* - tell Synchronizer that this key will be evicted - delete the file - remove it from our structs - update current size - */ - - //logger->log(LOG_WARNING, "Cache: flushing!"); - toBeDeleted.insert(it); - - string key = *it; // need to make a copy; it could get changed after unlocking. - - lru_mutex.unlock(); - try - { - Synchronizer::get()->flushObject(firstDir, key); - } - catch (...) - { - // it gets logged by Sync - lru_mutex.lock(); - toBeDeleted.erase(it); - continue; - } - lru_mutex.lock(); - - // check doNotEvict again in case this object is now being read - if (doNotEvict.find(it) == doNotEvict.end()) - { - bf::path cachedFile = cachePrefix / *it; - m_lru.erase(*it); - toBeDeleted.erase(it); - lru.erase(it); - size_t newSize = bf::file_size(cachedFile); - replicator->remove(cachedFile, Replicator::LOCAL_ONLY); - if (newSize < currentCacheSize) - { - currentCacheSize -= newSize; - thisMuch -= newSize; - } - else - { - logger->log(LOG_WARNING, "PrefixCache::makeSpace(): accounting error. Almost wrapped currentCacheSize on flush."); - currentCacheSize = 0; - thisMuch = 0; - } - } - else - toBeDeleted.erase(it); + // make sure it's not currently being read or being flushed by another _makeSpace() call + if ((doNotEvict.find(it) == doNotEvict.end()) && (toBeDeleted.find(it) == toBeDeleted.end())) + break; + ++it; } + if (it == lru.end()) + { + // nothing can be deleted right now + return; + } + + // ran into this a couple times, still happens as of commit 948ee1aa5 + // BT: made this more visable in logging. + // likely related to MCOL-3499 and lru containing double entries. + if (!bf::exists(cachePrefix / *it)) + logger->log(LOG_WARNING, "PrefixCache::makeSpace(): doesn't exist, %s/%s", cachePrefix.string().c_str(), + ((string)(*it)).c_str()); + assert(bf::exists(cachePrefix / *it)); + /* + tell Synchronizer that this key will be evicted + delete the file + remove it from our structs + update current size + */ + + // logger->log(LOG_WARNING, "Cache: flushing!"); + toBeDeleted.insert(it); + + string key = *it; // need to make a copy; it could get changed after unlocking. + + lru_mutex.unlock(); + try + { + Synchronizer::get()->flushObject(firstDir, key); + } + catch (...) + { + // it gets logged by Sync + lru_mutex.lock(); + toBeDeleted.erase(it); + continue; + } + lru_mutex.lock(); + + // check doNotEvict again in case this object is now being read + if (doNotEvict.find(it) == doNotEvict.end()) + { + bf::path cachedFile = cachePrefix / *it; + m_lru.erase(*it); + toBeDeleted.erase(it); + lru.erase(it); + size_t newSize = bf::file_size(cachedFile); + replicator->remove(cachedFile, Replicator::LOCAL_ONLY); + if (newSize < currentCacheSize) + { + currentCacheSize -= newSize; + thisMuch -= newSize; + } + else + { + logger->log(LOG_WARNING, + "PrefixCache::makeSpace(): accounting error. Almost wrapped currentCacheSize on flush."); + currentCacheSize = 0; + thisMuch = 0; + } + } + else + toBeDeleted.erase(it); + } } -void PrefixCache::rename(const string &oldKey, const string &newKey, ssize_t sizediff) +void PrefixCache::rename(const string& oldKey, const string& newKey, ssize_t sizediff) { - // rename it in the LRU - // erase/insert to rehash it everywhere else + // rename it in the LRU + // erase/insert to rehash it everywhere else - boost::unique_lock s(lru_mutex); - auto it = m_lru.find(oldKey); - if (it == m_lru.end()) - return; - - auto lit = it->lit; - m_lru.erase(it); - int refCount = 0; - auto dne_it = doNotEvict.find(lit); - if (dne_it != doNotEvict.end()) - { - refCount = dne_it->refCount; - doNotEvict.erase(dne_it); - } - - auto tbd_it = toBeDeleted.find(lit); - bool hasTBDEntry = (tbd_it != toBeDeleted.end()); - if (hasTBDEntry) - toBeDeleted.erase(tbd_it); - - *lit = newKey; - - if (hasTBDEntry) - toBeDeleted.insert(lit); - if (refCount != 0) - { - pair dne_tmp = doNotEvict.insert(lit); - const_cast(*(dne_tmp.first)).refCount = refCount; - } - - m_lru.insert(lit); - currentCacheSize += sizediff; + boost::unique_lock s(lru_mutex); + auto it = m_lru.find(oldKey); + if (it == m_lru.end()) + return; + + auto lit = it->lit; + m_lru.erase(it); + int refCount = 0; + auto dne_it = doNotEvict.find(lit); + if (dne_it != doNotEvict.end()) + { + refCount = dne_it->refCount; + doNotEvict.erase(dne_it); + } + + auto tbd_it = toBeDeleted.find(lit); + bool hasTBDEntry = (tbd_it != toBeDeleted.end()); + if (hasTBDEntry) + toBeDeleted.erase(tbd_it); + + *lit = newKey; + + if (hasTBDEntry) + toBeDeleted.insert(lit); + if (refCount != 0) + { + pair dne_tmp = doNotEvict.insert(lit); + const_cast(*(dne_tmp.first)).refCount = refCount; + } + + m_lru.insert(lit); + currentCacheSize += sizediff; } -int PrefixCache::ifExistsThenDelete(const string &key) +int PrefixCache::ifExistsThenDelete(const string& key) { - bf::path cachedPath = cachePrefix / key; - bf::path journalPath = journalPrefix / (key + ".journal"); + bf::path cachedPath = cachePrefix / key; + bf::path journalPath = journalPrefix / (key + ".journal"); - boost::unique_lock s(lru_mutex); - bool objectExists = false; - - auto it = m_lru.find(key); - if (it != m_lru.end()) + boost::unique_lock s(lru_mutex); + bool objectExists = false; + + auto it = m_lru.find(key); + if (it != m_lru.end()) + { + if (toBeDeleted.find(it->lit) == toBeDeleted.end()) { - if (toBeDeleted.find(it->lit) == toBeDeleted.end()) - { - doNotEvict.erase(it->lit); - lru.erase(it->lit); - m_lru.erase(it); - objectExists = true; - } - else // let makeSpace() delete it if it's already in progress - return 0; + doNotEvict.erase(it->lit); + lru.erase(it->lit); + m_lru.erase(it); + objectExists = true; } - bool journalExists = bf::exists(journalPath); - //assert(objectExists == bf::exists(cachedPath)); - - size_t objectSize = (objectExists ? bf::file_size(cachedPath) : 0); - //size_t objectSize = (objectExists ? MetadataFile::getLengthFromKey(key) : 0); - size_t journalSize = (journalExists ? bf::file_size(journalPath) : 0); - currentCacheSize -= (objectSize + journalSize); - - //assert(!objectExists || objectSize == bf::file_size(cachedPath)); - - return (objectExists ? 1 : 0) | (journalExists ? 2 : 0); + else // let makeSpace() delete it if it's already in progress + return 0; + } + bool journalExists = bf::exists(journalPath); + // assert(objectExists == bf::exists(cachedPath)); + + size_t objectSize = (objectExists ? bf::file_size(cachedPath) : 0); + // size_t objectSize = (objectExists ? MetadataFile::getLengthFromKey(key) : 0); + size_t journalSize = (journalExists ? bf::file_size(journalPath) : 0); + currentCacheSize -= (objectSize + journalSize); + + // assert(!objectExists || objectSize == bf::file_size(cachedPath)); + + return (objectExists ? 1 : 0) | (journalExists ? 2 : 0); } size_t PrefixCache::getCurrentCacheSize() const { - return currentCacheSize; + return currentCacheSize; } size_t PrefixCache::getCurrentCacheElementCount() const { - boost::unique_lock s(lru_mutex); - assert(m_lru.size() == lru.size()); - return m_lru.size(); + boost::unique_lock s(lru_mutex); + assert(m_lru.size() == lru.size()); + return m_lru.size(); } void PrefixCache::reset() { - boost::unique_lock s(lru_mutex); - m_lru.clear(); - lru.clear(); - toBeDeleted.clear(); - doNotEvict.clear(); - - bf::directory_iterator dir; - bf::directory_iterator dend; - for (dir = bf::directory_iterator(cachePrefix); dir != dend; ++dir) - bf::remove_all(dir->path()); - - for (dir = bf::directory_iterator(journalPrefix); dir != dend; ++dir) - bf::remove_all(dir->path()); - currentCacheSize = 0; + boost::unique_lock s(lru_mutex); + m_lru.clear(); + lru.clear(); + toBeDeleted.clear(); + doNotEvict.clear(); + + bf::directory_iterator dir; + bf::directory_iterator dend; + for (dir = bf::directory_iterator(cachePrefix); dir != dend; ++dir) + bf::remove_all(dir->path()); + + for (dir = bf::directory_iterator(journalPrefix); dir != dend; ++dir) + bf::remove_all(dir->path()); + currentCacheSize = 0; } void PrefixCache::shutdown() { - /* Does this need to do something anymore? */ + /* Does this need to do something anymore? */ } /* The helper classes */ -PrefixCache::M_LRU_element_t::M_LRU_element_t(const string *k) : key(k) -{} - -PrefixCache::M_LRU_element_t::M_LRU_element_t(const string &k) : key(&k) -{} - -PrefixCache::M_LRU_element_t::M_LRU_element_t(const LRU_t::iterator &i) : key(&(*i)), lit(i) -{} - -inline size_t PrefixCache::KeyHasher::operator()(const M_LRU_element_t &l) const +PrefixCache::M_LRU_element_t::M_LRU_element_t(const string* k) : key(k) { - return hash()(*(l.key)); } -inline bool PrefixCache::KeyEquals::operator()(const M_LRU_element_t &l1, const M_LRU_element_t &l2) const +PrefixCache::M_LRU_element_t::M_LRU_element_t(const string& k) : key(&k) { - return (*(l1.key) == *(l2.key)); } -inline size_t PrefixCache::DNEHasher::operator()(const DNEElement &l) const +PrefixCache::M_LRU_element_t::M_LRU_element_t(const LRU_t::iterator& i) : key(&(*i)), lit(i) { - return (l.sKey.empty() ? hash()(*(l.key)) : hash()(l.sKey)); } -inline bool PrefixCache::DNEEquals::operator()(const DNEElement &l1, const DNEElement &l2) const +inline size_t PrefixCache::KeyHasher::operator()(const M_LRU_element_t& l) const { - const string *s1, *s2; - s1 = l1.sKey.empty() ? &(*(l1.key)) : &(l1.sKey); - s2 = l2.sKey.empty() ? &(*(l2.key)) : &(l2.sKey); - - return (*s1 == *s2); + return hash()(*(l.key)); } -inline bool PrefixCache::TBDLess::operator()(const LRU_t::iterator &i1, const LRU_t::iterator &i2) const +inline bool PrefixCache::KeyEquals::operator()(const M_LRU_element_t& l1, const M_LRU_element_t& l2) const { - return *i1 < *i2; + return (*(l1.key) == *(l2.key)); } +inline size_t PrefixCache::DNEHasher::operator()(const DNEElement& l) const +{ + return (l.sKey.empty() ? hash()(*(l.key)) : hash()(l.sKey)); } +inline bool PrefixCache::DNEEquals::operator()(const DNEElement& l1, const DNEElement& l2) const +{ + const string *s1, *s2; + s1 = l1.sKey.empty() ? &(*(l1.key)) : &(l1.sKey); + s2 = l2.sKey.empty() ? &(*(l2.key)) : &(l2.sKey); + return (*s1 == *s2); +} +inline bool PrefixCache::TBDLess::operator()(const LRU_t::iterator& i1, const LRU_t::iterator& i2) const +{ + return *i1 < *i2; +} +} // namespace storagemanager diff --git a/storage-manager/src/PrefixCache.h b/storage-manager/src/PrefixCache.h index 1121275c7..108fee8ff 100644 --- a/storage-manager/src/PrefixCache.h +++ b/storage-manager/src/PrefixCache.h @@ -18,12 +18,11 @@ #ifndef PREFIXCACHE_H_ #define PREFIXCACHE_H_ -/* PrefixCache manages the cache for one prefix managed by SM. +/* PrefixCache manages the cache for one prefix managed by SM. Cache is a map of prefix -> PrefixCache, and holds the items that should be centralized like the Downloader */ - #include "Downloader.h" #include "SMLogging.h" #include "Replicator.h" @@ -38,133 +37,132 @@ namespace storagemanager { - class PrefixCache : public boost::noncopyable { - public: - PrefixCache(const boost::filesystem::path &prefix); - virtual ~PrefixCache(); - - //reading fcns - // read() marks objects to be read s.t. they do not get flushed. - // after reading them, unlock the 'logical file', and call doneReading(). - void read(const std::vector &keys); - void doneReading(const std::vector &keys); - bool exists(const std::string &key) const; - void exists(const std::vector &keys, std::vector *out) const; - - // writing fcns - // new*() fcns tell the PrefixCache data was added. After writing a set of objects, - // unlock the 'logical file', and call doneWriting(). - void newObject(const std::string &key, size_t size); - void newJournalEntry(size_t size); - void doneWriting(); - void deletedObject(const std::string &key, size_t size); - void deletedJournal(size_t size); - - // an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files. - // returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both exist - // This should be called while holding the file lock for key because it touches the journal file. - int ifExistsThenDelete(const std::string &key); - - // rename is used when an old obj gets merged with its journal file - // the size will change in that process; sizediff is by how much - void rename(const std::string &oldKey, const std::string &newKey, ssize_t sizediff); - void setMaxCacheSize(size_t size); - void makeSpace(size_t size); - size_t getCurrentCacheSize() const; - size_t getCurrentCacheElementCount() const; - size_t getMaxCacheSize() const; - void shutdown(); + public: + PrefixCache(const boost::filesystem::path& prefix); + virtual ~PrefixCache(); - // test helpers - const boost::filesystem::path &getCachePath(); - const boost::filesystem::path &getJournalPath(); - // this will delete everything in the PrefixCache and journal paths, and empty all PrefixCache structures. - void reset(); - void validateCacheSize(); - - private: - PrefixCache(); - - boost::filesystem::path cachePrefix; - boost::filesystem::path journalPrefix; - boost::filesystem::path firstDir; - size_t maxCacheSize; - size_t objectSize; - size_t currentCacheSize; - Replicator *replicator; - SMLogging *logger; - Downloader *downloader; - - void populate(); - void _makeSpace(size_t size); + // reading fcns + // read() marks objects to be read s.t. they do not get flushed. + // after reading them, unlock the 'logical file', and call doneReading(). + void read(const std::vector& keys); + void doneReading(const std::vector& keys); + bool exists(const std::string& key) const; + void exists(const std::vector& keys, std::vector* out) const; - /* The main PrefixCache structures */ - // lru owns the string memory for the filenames it manages. m_lru and DNE point to those strings. - typedef std::list LRU_t; - LRU_t lru; - - struct M_LRU_element_t - { - M_LRU_element_t(const std::string &); - M_LRU_element_t(const std::string *); - M_LRU_element_t(const LRU_t::iterator &); - const std::string *key; - LRU_t::iterator lit; - }; - struct KeyHasher - { - size_t operator()(const M_LRU_element_t &l) const; - }; + // writing fcns + // new*() fcns tell the PrefixCache data was added. After writing a set of objects, + // unlock the 'logical file', and call doneWriting(). + void newObject(const std::string& key, size_t size); + void newJournalEntry(size_t size); + void doneWriting(); + void deletedObject(const std::string& key, size_t size); + void deletedJournal(size_t size); - struct KeyEquals - { - bool operator()(const M_LRU_element_t &l1, const M_LRU_element_t &l2) const; - }; - - typedef std::unordered_set M_LRU_t; - M_LRU_t m_lru; // the LRU entries as a hash table - - /* The do-not-evict list stuff. */ - struct DNEElement - { - DNEElement(const LRU_t::iterator &); - DNEElement(const std::string &); - LRU_t::iterator key; - std::string sKey; - uint refCount; - }; - - struct DNEHasher - { - size_t operator()(const DNEElement &d) const; - }; - - struct DNEEquals - { - bool operator()(const DNEElement &d1, const DNEElement &d2) const; - }; - - typedef std::unordered_set DNE_t; - DNE_t doNotEvict; - void addToDNE(const DNEElement &); - void removeFromDNE(const DNEElement &); - - // the to-be-deleted set. Elements removed from the LRU but not yet deleted will be here. - // Elements are inserted and removed by makeSpace(). If read() references a file that is in this, - // it will remove it, signalling to makeSpace that it should not be deleted - struct TBDLess - { - bool operator()(const LRU_t::iterator &, const LRU_t::iterator &) const; - }; + // an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files. + // returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both + // exist This should be called while holding the file lock for key because it touches the journal file. + int ifExistsThenDelete(const std::string& key); - typedef std::set TBD_t; - TBD_t toBeDeleted; - - mutable boost::mutex lru_mutex; // protects the main PrefixCache structures & the do-not-evict set + // rename is used when an old obj gets merged with its journal file + // the size will change in that process; sizediff is by how much + void rename(const std::string& oldKey, const std::string& newKey, ssize_t sizediff); + void setMaxCacheSize(size_t size); + void makeSpace(size_t size); + size_t getCurrentCacheSize() const; + size_t getCurrentCacheElementCount() const; + size_t getMaxCacheSize() const; + void shutdown(); + + // test helpers + const boost::filesystem::path& getCachePath(); + const boost::filesystem::path& getJournalPath(); + // this will delete everything in the PrefixCache and journal paths, and empty all PrefixCache structures. + void reset(); + void validateCacheSize(); + + private: + PrefixCache(); + + boost::filesystem::path cachePrefix; + boost::filesystem::path journalPrefix; + boost::filesystem::path firstDir; + size_t maxCacheSize; + size_t objectSize; + size_t currentCacheSize; + Replicator* replicator; + SMLogging* logger; + Downloader* downloader; + + void populate(); + void _makeSpace(size_t size); + + /* The main PrefixCache structures */ + // lru owns the string memory for the filenames it manages. m_lru and DNE point to those strings. + typedef std::list LRU_t; + LRU_t lru; + + struct M_LRU_element_t + { + M_LRU_element_t(const std::string&); + M_LRU_element_t(const std::string*); + M_LRU_element_t(const LRU_t::iterator&); + const std::string* key; + LRU_t::iterator lit; + }; + struct KeyHasher + { + size_t operator()(const M_LRU_element_t& l) const; + }; + + struct KeyEquals + { + bool operator()(const M_LRU_element_t& l1, const M_LRU_element_t& l2) const; + }; + + typedef std::unordered_set M_LRU_t; + M_LRU_t m_lru; // the LRU entries as a hash table + + /* The do-not-evict list stuff. */ + struct DNEElement + { + DNEElement(const LRU_t::iterator&); + DNEElement(const std::string&); + LRU_t::iterator key; + std::string sKey; + uint refCount; + }; + + struct DNEHasher + { + size_t operator()(const DNEElement& d) const; + }; + + struct DNEEquals + { + bool operator()(const DNEElement& d1, const DNEElement& d2) const; + }; + + typedef std::unordered_set DNE_t; + DNE_t doNotEvict; + void addToDNE(const DNEElement&); + void removeFromDNE(const DNEElement&); + + // the to-be-deleted set. Elements removed from the LRU but not yet deleted will be here. + // Elements are inserted and removed by makeSpace(). If read() references a file that is in this, + // it will remove it, signalling to makeSpace that it should not be deleted + struct TBDLess + { + bool operator()(const LRU_t::iterator&, const LRU_t::iterator&) const; + }; + + typedef std::set TBD_t; + TBD_t toBeDeleted; + + mutable boost::mutex lru_mutex; // protects the main PrefixCache structures & the do-not-evict set }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/ProcessTask.cpp b/storage-manager/src/ProcessTask.cpp index c6cdc337f..a849fd303 100644 --- a/storage-manager/src/ProcessTask.cpp +++ b/storage-manager/src/ProcessTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "ProcessTask.h" #include #include @@ -41,97 +40,73 @@ using namespace std; namespace storagemanager { - ProcessTask::ProcessTask(int _sock, uint _length) : sock(_sock), length(_length), returnedSock(false) { - assert(length > 0); + assert(length > 0); } ProcessTask::~ProcessTask() { - if (!returnedSock) - (SessionManager::get())->returnSocket(sock); + if (!returnedSock) + (SessionManager::get())->returnSocket(sock); } void ProcessTask::handleError(int saved_errno) { - SMLogging* logger = SMLogging::get(); - SessionManager::get()->socketError(sock); - returnedSock = true; - char buf[80]; - logger->log(LOG_ERR,"ProcessTask: got an error during a socket read: %s.",strerror_r(saved_errno, buf, 80)); + SMLogging* logger = SMLogging::get(); + SessionManager::get()->socketError(sock); + returnedSock = true; + char buf[80]; + logger->log(LOG_ERR, "ProcessTask: got an error during a socket read: %s.", + strerror_r(saved_errno, buf, 80)); } void ProcessTask::operator()() { - /* - Read the command from the socket - Create the appropriate PosixTask - Run it - */ - vector msg; - int err; - uint8_t opcode; - - err = ::recv(sock, &opcode, 1, MSG_PEEK); - if (err <= 0) - { - handleError(errno); - return; - } - - boost::scoped_ptr task; - switch(opcode) - { - case OPEN: - task.reset(new OpenTask(sock, length)); - break; - case READ: - task.reset(new ReadTask(sock, length)); - break; - case WRITE: - task.reset(new WriteTask(sock, length)); - break; - case STAT: - task.reset(new StatTask(sock, length)); - break; - case UNLINK: - task.reset(new UnlinkTask(sock, length)); - break; - case APPEND: - task.reset(new AppendTask(sock, length)); - break; - case TRUNCATE: - task.reset(new TruncateTask(sock, length)); - break; - case LIST_DIRECTORY: - task.reset(new ListDirectoryTask(sock, length)); - break; - case PING: - task.reset(new PingTask(sock, length)); - break; - case SYNC: - task.reset(new SyncTask(sock, length)); - break; - case COPY: - task.reset(new CopyTask(sock, length)); - break; - default: - throw runtime_error("ProcessTask: got an unknown opcode"); - } - task->primeBuffer(); - bool success = task->run(); - if (!success) - { - SessionManager::get()->socketError(sock); - returnedSock = true; - } - else - { - SessionManager::get()->returnSocket(sock); - returnedSock = true; - } + /* + Read the command from the socket + Create the appropriate PosixTask + Run it + */ + vector msg; + int err; + uint8_t opcode; + + err = ::recv(sock, &opcode, 1, MSG_PEEK); + if (err <= 0) + { + handleError(errno); + return; + } + + boost::scoped_ptr task; + switch (opcode) + { + case OPEN: task.reset(new OpenTask(sock, length)); break; + case READ: task.reset(new ReadTask(sock, length)); break; + case WRITE: task.reset(new WriteTask(sock, length)); break; + case STAT: task.reset(new StatTask(sock, length)); break; + case UNLINK: task.reset(new UnlinkTask(sock, length)); break; + case APPEND: task.reset(new AppendTask(sock, length)); break; + case TRUNCATE: task.reset(new TruncateTask(sock, length)); break; + case LIST_DIRECTORY: task.reset(new ListDirectoryTask(sock, length)); break; + case PING: task.reset(new PingTask(sock, length)); break; + case SYNC: task.reset(new SyncTask(sock, length)); break; + case COPY: task.reset(new CopyTask(sock, length)); break; + default: throw runtime_error("ProcessTask: got an unknown opcode"); + } + task->primeBuffer(); + bool success = task->run(); + if (!success) + { + SessionManager::get()->socketError(sock); + returnedSock = true; + } + else + { + SessionManager::get()->returnSocket(sock); + returnedSock = true; + } } - -} +} // namespace storagemanager diff --git a/storage-manager/src/ProcessTask.h b/storage-manager/src/ProcessTask.h index a9b1008aa..390e88058 100644 --- a/storage-manager/src/ProcessTask.h +++ b/storage-manager/src/ProcessTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef PROCESS_TASK_H_ #define PROCESS_TASK_H_ @@ -24,29 +22,23 @@ namespace storagemanager { - class ProcessTask : public ThreadPool::Job { - public: - ProcessTask(int sock, uint length); // _sock is the socket to read from - virtual ~ProcessTask(); - - void operator()(); - - private: - ProcessTask(); - - void handleError(int errCode); - int sock; - uint length; - bool returnedSock; + public: + ProcessTask(int sock, uint length); // _sock is the socket to read from + virtual ~ProcessTask(); + + void operator()(); + + private: + ProcessTask(); + + void handleError(int errCode); + int sock; + uint length; + bool returnedSock; }; - - -} - - - +} // namespace storagemanager #endif diff --git a/storage-manager/src/RWLock.cpp b/storage-manager/src/RWLock.cpp index 886b40e9d..c9cda4af1 100644 --- a/storage-manager/src/RWLock.cpp +++ b/storage-manager/src/RWLock.cpp @@ -19,95 +19,94 @@ namespace storagemanager { - RWLock::RWLock() : readersWaiting(0), readersRunning(0), writersWaiting(0), writersRunning(0) { } RWLock::~RWLock() { - assert(!readersWaiting); - assert(!readersRunning); - assert(!writersWaiting); - assert(!writersRunning); + assert(!readersWaiting); + assert(!readersRunning); + assert(!writersWaiting); + assert(!writersRunning); } bool RWLock::inUse() { - boost::unique_lock s(m); - return readersWaiting || readersRunning || writersWaiting || writersRunning; + boost::unique_lock s(m); + return readersWaiting || readersRunning || writersWaiting || writersRunning; } void RWLock::readLock() { - boost::unique_lock s(m); - - ++readersWaiting; - while (writersWaiting != 0 || writersRunning != 0) - okToRead.wait(s); - - ++readersRunning; - --readersWaiting; + boost::unique_lock s(m); + + ++readersWaiting; + while (writersWaiting != 0 || writersRunning != 0) + okToRead.wait(s); + + ++readersRunning; + --readersWaiting; } -void RWLock::readLock(boost::unique_lock &l) +void RWLock::readLock(boost::unique_lock& l) { - boost::unique_lock s(m); - l.unlock(); - - ++readersWaiting; - while (writersWaiting != 0 || writersRunning != 0) - okToRead.wait(s); - - ++readersRunning; - --readersWaiting; + boost::unique_lock s(m); + l.unlock(); + + ++readersWaiting; + while (writersWaiting != 0 || writersRunning != 0) + okToRead.wait(s); + + ++readersRunning; + --readersWaiting; } void RWLock::readUnlock() { - boost::unique_lock s(m); - - assert(readersRunning > 0); - --readersRunning; - if (readersRunning == 0 && writersWaiting != 0) - okToWrite.notify_one(); + boost::unique_lock s(m); + + assert(readersRunning > 0); + --readersRunning; + if (readersRunning == 0 && writersWaiting != 0) + okToWrite.notify_one(); } void RWLock::writeLock() { - boost::unique_lock s(m); - - ++writersWaiting; - while (readersRunning != 0 || writersRunning != 0) - okToWrite.wait(s); - - ++writersRunning; - --writersWaiting; + boost::unique_lock s(m); + + ++writersWaiting; + while (readersRunning != 0 || writersRunning != 0) + okToWrite.wait(s); + + ++writersRunning; + --writersWaiting; } -void RWLock::writeLock(boost::unique_lock &l) +void RWLock::writeLock(boost::unique_lock& l) { - boost::unique_lock s(m); - l.unlock(); - - ++writersWaiting; - while (readersRunning != 0 || writersRunning != 0) - okToWrite.wait(s); - - ++writersRunning; - --writersWaiting; + boost::unique_lock s(m); + l.unlock(); + + ++writersWaiting; + while (readersRunning != 0 || writersRunning != 0) + okToWrite.wait(s); + + ++writersRunning; + --writersWaiting; } void RWLock::writeUnlock() { - boost::unique_lock s(m); - - assert(writersRunning > 0); - --writersRunning; - if (writersWaiting != 0) - okToWrite.notify_one(); - else if (readersWaiting != 0) - okToRead.notify_all(); + boost::unique_lock s(m); + + assert(writersRunning > 0); + --writersRunning; + if (writersWaiting != 0) + okToWrite.notify_one(); + else if (readersWaiting != 0) + okToRead.notify_all(); } -} +} // namespace storagemanager diff --git a/storage-manager/src/RWLock.h b/storage-manager/src/RWLock.h index 169cb3965..38ec3a8df 100644 --- a/storage-manager/src/RWLock.h +++ b/storage-manager/src/RWLock.h @@ -21,32 +21,31 @@ /* Quicky impl of a read-write lock that prioritizes writers. */ namespace storagemanager { - class RWLock - { - public: - RWLock(); - ~RWLock(); - - void readLock(); - // this version will release the lock in the parameter after locking this instance - void readLock(boost::unique_lock &); - void readUnlock(); - void writeLock(); - // this version will release the lock in the parameter after locking this instance - void writeLock(boost::unique_lock &); - void writeUnlock(); - - // returns true if anything is blocked on or owns this lock instance. - bool inUse(); - - private: - uint readersWaiting; - uint readersRunning; - uint writersWaiting; - uint writersRunning; - boost::mutex m; - boost::condition okToWrite; - boost::condition okToRead; - }; -} +class RWLock +{ + public: + RWLock(); + ~RWLock(); + void readLock(); + // this version will release the lock in the parameter after locking this instance + void readLock(boost::unique_lock&); + void readUnlock(); + void writeLock(); + // this version will release the lock in the parameter after locking this instance + void writeLock(boost::unique_lock&); + void writeUnlock(); + + // returns true if anything is blocked on or owns this lock instance. + bool inUse(); + + private: + uint readersWaiting; + uint readersRunning; + uint writersWaiting; + uint writersRunning; + boost::mutex m; + boost::condition okToWrite; + boost::condition okToRead; +}; +} // namespace storagemanager diff --git a/storage-manager/src/ReadTask.cpp b/storage-manager/src/ReadTask.cpp index 42d30404c..cfed24665 100644 --- a/storage-manager/src/ReadTask.cpp +++ b/storage-manager/src/ReadTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "ReadTask.h" #include "messageFormat.h" #include "SMLogging.h" @@ -25,7 +24,6 @@ using namespace std; namespace storagemanager { - ReadTask::ReadTask(int sock, uint len) : PosixTask(sock, len) { } @@ -35,77 +33,78 @@ ReadTask::~ReadTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } #define max(x, y) (x > y ? x : y) - + bool ReadTask::run() { - SMLogging* logger = SMLogging::get(); - uint8_t buf[1024] = {0}; + SMLogging* logger = SMLogging::get(); + uint8_t buf[1024] = {0}; - // get the parameters - if (getLength() > 1023) { - handleError("ReadTask read", EFAULT); - return true; - } - - int success; - success = read(buf, getLength()); - check_error("ReadTask read cmd", false); - read_cmd *cmd = (read_cmd *) buf; - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"read %s count %i offset %i.",cmd->filename,cmd->count,cmd->offset); - #endif - - // read from IOC, write to the socket - vector outbuf; - if (cmd->count > (100 << 20)) - cmd->count = (100 << 20); // cap a read request at 100MB - outbuf.resize(max(cmd->count, 4) + sizeof(sm_response)); - sm_response *resp = (sm_response *) &outbuf[0]; - - resp->returnCode = 0; - uint payloadLen = 0; - - // todo: do the reading and writing in chunks - // todo: need to make this use O_DIRECT on the IOC side - ssize_t err; - while ((uint) resp->returnCode < cmd->count) + // get the parameters + if (getLength() > 1023) + { + handleError("ReadTask read", EFAULT); + return true; + } + + int success; + success = read(buf, getLength()); + check_error("ReadTask read cmd", false); + read_cmd* cmd = (read_cmd*)buf; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "read %s count %i offset %i.", cmd->filename, cmd->count, cmd->offset); +#endif + + // read from IOC, write to the socket + vector outbuf; + if (cmd->count > (100 << 20)) + cmd->count = (100 << 20); // cap a read request at 100MB + outbuf.resize(max(cmd->count, 4) + sizeof(sm_response)); + sm_response* resp = (sm_response*)&outbuf[0]; + + resp->returnCode = 0; + uint payloadLen = 0; + + // todo: do the reading and writing in chunks + // todo: need to make this use O_DIRECT on the IOC side + ssize_t err; + while ((uint)resp->returnCode < cmd->count) + { + try { - try - { - err = ioc->read(cmd->filename, &resp->payload[resp->returnCode], cmd->offset + resp->returnCode, - cmd->count - resp->returnCode); - } - catch (exception &e) - { - logger->log(LOG_ERR, "ReadTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - if (err < 0) { - if (resp->returnCode == 0) { - resp->returnCode = err; - payloadLen = 4; - *((int32_t *) resp->payload) = errno; - } - break; - } - if (err == 0) - break; - resp->returnCode += err; + err = ioc->read(cmd->filename, &resp->payload[resp->returnCode], cmd->offset + resp->returnCode, + cmd->count - resp->returnCode); } - if (resp->returnCode >= 0) - payloadLen = resp->returnCode; - return write(*resp, payloadLen); + catch (exception& e) + { + logger->log(LOG_ERR, "ReadTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + if (err < 0) + { + if (resp->returnCode == 0) + { + resp->returnCode = err; + payloadLen = 4; + *((int32_t*)resp->payload) = errno; + } + break; + } + if (err == 0) + break; + resp->returnCode += err; + } + if (resp->returnCode >= 0) + payloadLen = resp->returnCode; + return write(*resp, payloadLen); } - - -} +} // namespace storagemanager diff --git a/storage-manager/src/ReadTask.h b/storage-manager/src/ReadTask.h index db0eeb35a..392d7eafa 100644 --- a/storage-manager/src/ReadTask.h +++ b/storage-manager/src/ReadTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef READTASK_H_ #define READTASK_H_ @@ -24,18 +22,17 @@ namespace storagemanager { - class ReadTask : public PosixTask { - public: - ReadTask(int sock, uint length); - virtual ~ReadTask(); + public: + ReadTask(int sock, uint length); + virtual ~ReadTask(); - bool run(); + bool run(); - private: - ReadTask(); + private: + ReadTask(); }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Replicator.cpp b/storage-manager/src/Replicator.cpp index 542f00060..10fc07002 100644 --- a/storage-manager/src/Replicator.cpp +++ b/storage-manager/src/Replicator.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "Replicator.h" #include "IOCoordinator.h" #include "SMLogging.h" @@ -38,439 +37,456 @@ using namespace std; namespace { - storagemanager::Replicator *rep = NULL; - boost::mutex m; -} +storagemanager::Replicator* rep = NULL; +boost::mutex m; +} // namespace namespace storagemanager { - Replicator::Replicator() { - mpConfig = Config::get(); - mpLogger = SMLogging::get(); - try + mpConfig = Config::get(); + mpLogger = SMLogging::get(); + try + { + msJournalPath = mpConfig->getValue("ObjectStorage", "journal_path"); + if (msJournalPath.empty()) { - msJournalPath = mpConfig->getValue("ObjectStorage", "journal_path"); - if (msJournalPath.empty()) - { - mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); - throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); - } + mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); + throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); } - catch (...) - { - mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file."); - throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); - } - try - { - boost::filesystem::create_directories(msJournalPath); - } - catch (exception &e) - { - syslog(LOG_CRIT, "Failed to create %s, got: %s", msJournalPath.c_str(), e.what()); - throw e; - } - msCachePath = mpConfig->getValue("Cache", "path"); - if (msCachePath.empty()) - { - mpLogger->log(LOG_CRIT, "Cache/path is not set"); - throw runtime_error("Please set Cache/path in the storagemanager.cnf file"); - } - try - { - boost::filesystem::create_directories(msCachePath); - } - catch (exception &e) - { - mpLogger->log(LOG_CRIT, "Failed to create %s, got: %s", msCachePath.c_str(), e.what()); - throw e; - } - repUserDataWritten = repHeaderDataWritten = replicatorObjectsCreated = replicatorJournalsCreated = 0; + } + catch (...) + { + mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file."); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + try + { + boost::filesystem::create_directories(msJournalPath); + } + catch (exception& e) + { + syslog(LOG_CRIT, "Failed to create %s, got: %s", msJournalPath.c_str(), e.what()); + throw e; + } + msCachePath = mpConfig->getValue("Cache", "path"); + if (msCachePath.empty()) + { + mpLogger->log(LOG_CRIT, "Cache/path is not set"); + throw runtime_error("Please set Cache/path in the storagemanager.cnf file"); + } + try + { + boost::filesystem::create_directories(msCachePath); + } + catch (exception& e) + { + mpLogger->log(LOG_CRIT, "Failed to create %s, got: %s", msCachePath.c_str(), e.what()); + throw e; + } + repUserDataWritten = repHeaderDataWritten = replicatorObjectsCreated = replicatorJournalsCreated = 0; } Replicator::~Replicator() { } -Replicator * Replicator::get() +Replicator* Replicator::get() { - if (rep) - return rep; - boost::mutex::scoped_lock s(m); - if (rep) - return rep; - rep = new Replicator(); + if (rep) return rep; + boost::mutex::scoped_lock s(m); + if (rep) + return rep; + rep = new Replicator(); + return rep; } void Replicator::printKPIs() const { - cout << "Replicator" << endl; - cout << "\treplicatorUserDataWritten = " << repUserDataWritten << endl; - cout << "\treplicatorHeaderDataWritten = " << repHeaderDataWritten << endl; + cout << "Replicator" << endl; + cout << "\treplicatorUserDataWritten = " << repUserDataWritten << endl; + cout << "\treplicatorHeaderDataWritten = " << repHeaderDataWritten << endl; - cout << "\treplicatorObjectsCreated = " << replicatorObjectsCreated << endl; - cout << "\treplicatorJournalsCreated = " << replicatorJournalsCreated << endl; + cout << "\treplicatorObjectsCreated = " << replicatorObjectsCreated << endl; + cout << "\treplicatorJournalsCreated = " << replicatorJournalsCreated << endl; } -#define OPEN(name, mode) \ - fd = ::open(name, mode, 0600); \ - if (fd < 0) \ - return fd; \ - ScopedCloser sc(fd); +#define OPEN(name, mode) \ + fd = ::open(name, mode, 0600); \ + if (fd < 0) \ + return fd; \ + ScopedCloser sc(fd); -int Replicator::newObject(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length ) +int Replicator::newObject(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, + size_t length) { - int fd, err; - string objectFilename = msCachePath + "/" + filename.string(); + int fd, err; + string objectFilename = msCachePath + "/" + filename.string(); - OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT); - size_t count = 0; - while (count < length) { - err = ::pwrite(fd, &data[count], length - count, offset + count); - if (err <= 0) - { - if (count > 0) // return what was successfully written - return count; - else - return err; - } - count += err; - } - repUserDataWritten += count; - ++replicatorObjectsCreated; - return count; -} - -int Replicator::newNullObject(const boost::filesystem::path &filename,size_t length ) -{ - int fd, err; - string objectFilename = msCachePath + "/" + filename.string(); - - OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT); - err = ftruncate(fd,length); - - return err; -} - -ssize_t Replicator::_pwrite(int fd, const void *data, size_t length, off_t offset) -{ - ssize_t err; - size_t count = 0; - uint8_t *bData = (uint8_t *) data; - - do + OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT); + size_t count = 0; + while (count < length) + { + err = ::pwrite(fd, &data[count], length - count, offset + count); + if (err <= 0) { - err = ::pwrite(fd, &bData[count], length - count, offset + count); - if (err < 0 || (err == 0 && errno != EINTR)) - { - if (count > 0) - return count; - else - return err; - } - count += err; - } while (count < length); - - return count; -} - -ssize_t Replicator::_write(int fd, const void *data, size_t length) -{ - ssize_t err; - size_t count = 0; - uint8_t *bData = (uint8_t *) data; - - do - { - err = ::write(fd, &bData[count], length - count); - if (err < 0 || (err == 0 && errno != EINTR)) - { - if (count > 0) - return count; - else - return err; - } - count += err; - } while (count < length); - - return count; -} - -/* XXXPAT: I think we'll have to rewrite this function some; we'll have to at least clearly define - what happens in the various error scenarios. - - To be more resilent in the face of hard errors, we may also want to redefine what a journal file is. - If/when we cannot fix the journal file in the face of an error, there are scenarios that the read code - will not be able to cope with. Ex, a journal entry that says it's 200 bytes long, but there are only - really 100 bytes. The read code has no way to tell the difference if there is an entry that follows - the bad entry, and that will cause an unrecoverable error. - - Initial thought on a sol'n. Make each journal entry its own file in a tmp dir, ordered by a sequence - number in the filename. Then, one entry cannot affect the others, and the end of the file is unambiguously - the end of the data. On successful write, move the file to where it should be. This would also prevent - the readers from ever seeing bad data, and possibly reduce the size of some critical sections. - - Benefits would be data integrity, and possibly add'l parallelism. The downside is of course, a higher - number of IO ops for the same operation. -*/ -int Replicator::addJournalEntry(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length) -{ - int fd, err; - uint64_t offlen[] = {(uint64_t) offset,length}; - size_t count = 0; - int version = 1; - int l_errno; - char errbuf[80]; - bool bHeaderChanged = false; - string headerRollback = ""; - string journalFilename = msJournalPath + "/" + filename.string() + ".journal"; - boost::filesystem::path firstDir = *((filename).begin()); - uint64_t thisEntryMaxOffset = (offset + length - 1); - - uint64_t currentMaxOffset = 0; - bool exists = boost::filesystem::exists(journalFilename); - OPEN(journalFilename.c_str(), (exists ? O_RDWR : O_WRONLY | O_CREAT)) - - if (!exists) - { - bHeaderChanged = true; - // create new journal file with header - string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset).str(); - err = _write(fd, header.c_str(), header.length() + 1); - l_errno = errno; - repHeaderDataWritten += (header.length() + 1); - if ((uint)err != (header.length() + 1)) - { - // return the error because the header for this entry on a new journal file failed - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Writing journal header failed (%s).", - strerror_r(l_errno, errbuf, 80)); - errno = l_errno; - return err; - } - Cache::get()->newJournalEntry(firstDir, header.length() + 1); - ++replicatorJournalsCreated; - } - else - { - // read the existing header and check if max_offset needs to be updated - size_t tmp; - boost::shared_array headertxt; - try - { - headertxt = seekToEndOfHeader1(fd, &tmp); - } - catch (std::runtime_error& e) - { - mpLogger->log(LOG_CRIT,"%s",e.what()); - errno = EIO; - return -1; - } - catch (...) - { - mpLogger->log(LOG_CRIT,"Unknown exception caught during seekToEndOfHeader1."); - errno = EIO; - return -1; - } - stringstream ss; - ss << headertxt.get(); - headerRollback = headertxt.get(); - boost::property_tree::ptree header; - try - { - boost::property_tree::json_parser::read_json(ss, header); - } - catch (const boost::property_tree::json_parser::json_parser_error& e) - { - mpLogger->log(LOG_CRIT,"%s",e.what()); - errno = EIO; - return -1; - } - catch (...) - { - mpLogger->log(LOG_CRIT,"Unknown exception caught during read_json."); - errno = EIO; - return -1; - } - assert(header.get("version") == 1); - uint64_t currentMaxOffset = header.get("max_offset"); - if (thisEntryMaxOffset > currentMaxOffset) - { - bHeaderChanged = true; - string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset).str(); - err = _pwrite(fd, header.c_str(), header.length() + 1,0); - l_errno = errno; - repHeaderDataWritten += (header.length() + 1); - if ((uint)err != (header.length() + 1)) - { - // only the header was possibly changed rollback attempt - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Updating journal header failed. " - "Attempting to rollback and continue."); - int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1,0); - if ((uint)rollbackErr == (headerRollback.length() + 1)) - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header success."); - else - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed!"); - errno = l_errno; - if (err < 0) - return err; - else - return 0; - } - } - } - - off_t entryHeaderOffset = ::lseek(fd, 0, SEEK_END); - - err = _write(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE); - l_errno = errno; - repHeaderDataWritten += JOURNAL_ENTRY_HEADER_SIZE; - if (err != JOURNAL_ENTRY_HEADER_SIZE) - { - // this entry failed so if the header was updated roll it back - if (bHeaderChanged) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: write journal entry header failed. Attempting to rollback and continue."); - //attempt to rollback top level header - int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1,0); - if ((uint)rollbackErr != (headerRollback.length() + 1)) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - if (err < 0) - return err; - else - return 0; - } - } - int rollbackErr = ::ftruncate(fd,entryHeaderOffset); - if (rollbackErr != 0) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Truncate to previous EOF failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - if (err < 0) - return err; - else - return 0; - } - l_errno = errno; + if (count > 0) // return what was successfully written + return count; + else return err; } - while (count < length) { - err = _write(fd, &data[count], length - count); - if (err < 0 ) - { - l_errno = errno; - /* XXXBEN: Attempt to update entry header with the partial write and write it. - IF the write fails to update entry header report an error to IOC and logging. - */ - if (count > 0) // return what was successfully written - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Got '%s' writing a journal entry. " - "Attempting to update and continue.", strerror_r(l_errno, errbuf, 80)); - // Update the file header max_offset if necessary and possible - thisEntryMaxOffset = (offset + count - 1); - if (thisEntryMaxOffset > currentMaxOffset) - { - string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset).str(); - int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1,0); - if ((uint)rollbackErr != (header.length() + 1)) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal header failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - } - // Update the journal entry header - offlen[1] = count; - int rollbackErr = _pwrite(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE,entryHeaderOffset); - if ((uint)rollbackErr != JOURNAL_ENTRY_HEADER_SIZE) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal entry header failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - // return back what we did write - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Partial write success."); - repUserDataWritten += count; - return count; - } - else - { - //If the header was changed rollback and reset EOF - //Like this never happened - //Currently since this returns the err from the first write. IOC returns -1 and writeTask returns an error - //So system is likely broken in some way - if (bHeaderChanged) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: write journal entry failed (%s). " - "Attempting to rollback and continue.", strerror_r(l_errno, errbuf, 80)); - //attempt to rollback top level header - string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % 0).str(); - int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1,0); - if ((uint)rollbackErr != (header.length() + 1)) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed (%s)!", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - } - int rollbackErr = ::ftruncate(fd,entryHeaderOffset); - if (rollbackErr != 0) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Remove entry header failed (%s)!", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Write failed. Journal file restored."); - errno = l_errno; - return err; - } - } - count += err; - } - - repUserDataWritten += count; - return count; + count += err; + } + repUserDataWritten += count; + ++replicatorObjectsCreated; + return count; } -int Replicator::remove(const boost::filesystem::path &filename, Flags flags) +int Replicator::newNullObject(const boost::filesystem::path& filename, size_t length) { - int ret = 0; - - if (flags & NO_LOCAL) - return 0; // not implemented yet - + int fd, err; + string objectFilename = msCachePath + "/" + filename.string(); + + OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT); + err = ftruncate(fd, length); + + return err; +} + +ssize_t Replicator::_pwrite(int fd, const void* data, size_t length, off_t offset) +{ + ssize_t err; + size_t count = 0; + uint8_t* bData = (uint8_t*)data; + + do + { + err = ::pwrite(fd, &bData[count], length - count, offset + count); + if (err < 0 || (err == 0 && errno != EINTR)) + { + if (count > 0) + return count; + else + return err; + } + count += err; + } while (count < length); + + return count; +} + +ssize_t Replicator::_write(int fd, const void* data, size_t length) +{ + ssize_t err; + size_t count = 0; + uint8_t* bData = (uint8_t*)data; + + do + { + err = ::write(fd, &bData[count], length - count); + if (err < 0 || (err == 0 && errno != EINTR)) + { + if (count > 0) + return count; + else + return err; + } + count += err; + } while (count < length); + + return count; +} + +/* XXXPAT: I think we'll have to rewrite this function some; we'll have to at least clearly define + what happens in the various error scenarios. + + To be more resilent in the face of hard errors, we may also want to redefine what a journal file is. + If/when we cannot fix the journal file in the face of an error, there are scenarios that the read code + will not be able to cope with. Ex, a journal entry that says it's 200 bytes long, but there are only + really 100 bytes. The read code has no way to tell the difference if there is an entry that follows + the bad entry, and that will cause an unrecoverable error. + + Initial thought on a sol'n. Make each journal entry its own file in a tmp dir, ordered by a sequence + number in the filename. Then, one entry cannot affect the others, and the end of the file is unambiguously + the end of the data. On successful write, move the file to where it should be. This would also prevent + the readers from ever seeing bad data, and possibly reduce the size of some critical sections. + + Benefits would be data integrity, and possibly add'l parallelism. The downside is of course, a higher + number of IO ops for the same operation. +*/ +int Replicator::addJournalEntry(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, + size_t length) +{ + int fd, err; + uint64_t offlen[] = {(uint64_t)offset, length}; + size_t count = 0; + int version = 1; + int l_errno; + char errbuf[80]; + bool bHeaderChanged = false; + string headerRollback = ""; + string journalFilename = msJournalPath + "/" + filename.string() + ".journal"; + boost::filesystem::path firstDir = *((filename).begin()); + uint64_t thisEntryMaxOffset = (offset + length - 1); + + uint64_t currentMaxOffset = 0; + bool exists = boost::filesystem::exists(journalFilename); + OPEN(journalFilename.c_str(), (exists ? O_RDWR : O_WRONLY | O_CREAT)) + + if (!exists) + { + bHeaderChanged = true; + // create new journal file with header + string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % + thisEntryMaxOffset) + .str(); + err = _write(fd, header.c_str(), header.length() + 1); + l_errno = errno; + repHeaderDataWritten += (header.length() + 1); + if ((uint)err != (header.length() + 1)) + { + // return the error because the header for this entry on a new journal file failed + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Writing journal header failed (%s).", + strerror_r(l_errno, errbuf, 80)); + errno = l_errno; + return err; + } + Cache::get()->newJournalEntry(firstDir, header.length() + 1); + ++replicatorJournalsCreated; + } + else + { + // read the existing header and check if max_offset needs to be updated + size_t tmp; + boost::shared_array headertxt; try { - //#ifndef NDEBUG - // assert(boost::filesystem::remove_all(filename) > 0); - //#else - boost::filesystem::remove_all(filename); - //#endif + headertxt = seekToEndOfHeader1(fd, &tmp); } - catch(boost::filesystem::filesystem_error &e) + catch (std::runtime_error& e) { - #ifndef NDEBUG - cout << "Replicator::remove(): caught an execption: " << e.what() << endl; - assert(0); - #endif - errno = e.code().value(); - ret = -1; + mpLogger->log(LOG_CRIT, "%s", e.what()); + errno = EIO; + return -1; } - return ret; + catch (...) + { + mpLogger->log(LOG_CRIT, "Unknown exception caught during seekToEndOfHeader1."); + errno = EIO; + return -1; + } + stringstream ss; + ss << headertxt.get(); + headerRollback = headertxt.get(); + boost::property_tree::ptree header; + try + { + boost::property_tree::json_parser::read_json(ss, header); + } + catch (const boost::property_tree::json_parser::json_parser_error& e) + { + mpLogger->log(LOG_CRIT, "%s", e.what()); + errno = EIO; + return -1; + } + catch (...) + { + mpLogger->log(LOG_CRIT, "Unknown exception caught during read_json."); + errno = EIO; + return -1; + } + assert(header.get("version") == 1); + uint64_t currentMaxOffset = header.get("max_offset"); + if (thisEntryMaxOffset > currentMaxOffset) + { + bHeaderChanged = true; + string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % + thisEntryMaxOffset) + .str(); + err = _pwrite(fd, header.c_str(), header.length() + 1, 0); + l_errno = errno; + repHeaderDataWritten += (header.length() + 1); + if ((uint)err != (header.length() + 1)) + { + // only the header was possibly changed rollback attempt + mpLogger->log(LOG_CRIT, + "Replicator::addJournalEntry: Updating journal header failed. " + "Attempting to rollback and continue."); + int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1, 0); + if ((uint)rollbackErr == (headerRollback.length() + 1)) + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header success."); + else + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed!"); + errno = l_errno; + if (err < 0) + return err; + else + return 0; + } + } + } + + off_t entryHeaderOffset = ::lseek(fd, 0, SEEK_END); + + err = _write(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE); + l_errno = errno; + repHeaderDataWritten += JOURNAL_ENTRY_HEADER_SIZE; + if (err != JOURNAL_ENTRY_HEADER_SIZE) + { + // this entry failed so if the header was updated roll it back + if (bHeaderChanged) + { + mpLogger->log(LOG_CRIT, + "Replicator::addJournalEntry: write journal entry header failed. Attempting to rollback " + "and continue."); + // attempt to rollback top level header + int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1, 0); + if ((uint)rollbackErr != (headerRollback.length() + 1)) + { + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed! (%s)", + strerror_r(errno, errbuf, 80)); + errno = l_errno; + if (err < 0) + return err; + else + return 0; + } + } + int rollbackErr = ::ftruncate(fd, entryHeaderOffset); + if (rollbackErr != 0) + { + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Truncate to previous EOF failed! (%s)", + strerror_r(errno, errbuf, 80)); + errno = l_errno; + if (err < 0) + return err; + else + return 0; + } + l_errno = errno; + return err; + } + while (count < length) + { + err = _write(fd, &data[count], length - count); + if (err < 0) + { + l_errno = errno; + /* XXXBEN: Attempt to update entry header with the partial write and write it. + IF the write fails to update entry header report an error to IOC and logging. + */ + if (count > 0) // return what was successfully written + { + mpLogger->log(LOG_CRIT, + "Replicator::addJournalEntry: Got '%s' writing a journal entry. " + "Attempting to update and continue.", + strerror_r(l_errno, errbuf, 80)); + // Update the file header max_offset if necessary and possible + thisEntryMaxOffset = (offset + count - 1); + if (thisEntryMaxOffset > currentMaxOffset) + { + string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % + thisEntryMaxOffset) + .str(); + int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1, 0); + if ((uint)rollbackErr != (header.length() + 1)) + { + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal header failed! (%s)", + strerror_r(errno, errbuf, 80)); + errno = l_errno; + return err; + } + } + // Update the journal entry header + offlen[1] = count; + int rollbackErr = _pwrite(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE, entryHeaderOffset); + if ((uint)rollbackErr != JOURNAL_ENTRY_HEADER_SIZE) + { + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal entry header failed! (%s)", + strerror_r(errno, errbuf, 80)); + errno = l_errno; + return err; + } + // return back what we did write + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Partial write success."); + repUserDataWritten += count; + return count; + } + else + { + // If the header was changed rollback and reset EOF + // Like this never happened + // Currently since this returns the err from the first write. IOC returns -1 and writeTask returns an + // error So system is likely broken in some way + if (bHeaderChanged) + { + mpLogger->log(LOG_CRIT, + "Replicator::addJournalEntry: write journal entry failed (%s). " + "Attempting to rollback and continue.", + strerror_r(l_errno, errbuf, 80)); + // attempt to rollback top level header + string header = + (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % 0).str(); + int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1, 0); + if ((uint)rollbackErr != (header.length() + 1)) + { + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed (%s)!", + strerror_r(errno, errbuf, 80)); + errno = l_errno; + return err; + } + } + int rollbackErr = ::ftruncate(fd, entryHeaderOffset); + if (rollbackErr != 0) + { + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Remove entry header failed (%s)!", + strerror_r(errno, errbuf, 80)); + errno = l_errno; + return err; + } + mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Write failed. Journal file restored."); + errno = l_errno; + return err; + } + } + count += err; + } + + repUserDataWritten += count; + return count; } -int Replicator::updateMetadata(MetadataFile &meta) +int Replicator::remove(const boost::filesystem::path& filename, Flags flags) { - return meta.writeMetadata(); + int ret = 0; + + if (flags & NO_LOCAL) + return 0; // not implemented yet + + try + { + //#ifndef NDEBUG + // assert(boost::filesystem::remove_all(filename) > 0); + //#else + boost::filesystem::remove_all(filename); + //#endif + } + catch (boost::filesystem::filesystem_error& e) + { +#ifndef NDEBUG + cout << "Replicator::remove(): caught an execption: " << e.what() << endl; + assert(0); +#endif + errno = e.code().value(); + ret = -1; + } + return ret; } +int Replicator::updateMetadata(MetadataFile& meta) +{ + return meta.writeMetadata(); } + +} // namespace storagemanager diff --git a/storage-manager/src/Replicator.h b/storage-manager/src/Replicator.h index 4cf96ac92..ab0142d90 100644 --- a/storage-manager/src/Replicator.h +++ b/storage-manager/src/Replicator.h @@ -28,52 +28,51 @@ namespace storagemanager { - // 64-bit offset // 64-bit length // class Replicator { - public: - static Replicator *get(); - virtual ~Replicator(); + public: + static Replicator* get(); + virtual ~Replicator(); - enum Flags - { - NONE = 0, - LOCAL_ONLY = 0x1, - NO_LOCAL = 0x2 - }; - - int addJournalEntry(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length); - int newObject(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length); - int newNullObject(const boost::filesystem::path &filename,size_t length ); + enum Flags + { + NONE = 0, + LOCAL_ONLY = 0x1, + NO_LOCAL = 0x2 + }; - int remove(const boost::filesystem::path &file, Flags flags = NONE); - - int updateMetadata(MetadataFile &meta); + int addJournalEntry(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, + size_t length); + int newObject(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length); + int newNullObject(const boost::filesystem::path& filename, size_t length); - void printKPIs() const; + int remove(const boost::filesystem::path& file, Flags flags = NONE); - private: - Replicator(); - - // a couple helpers - ssize_t _write(int fd, const void *data, size_t len); - ssize_t _pwrite(int fd, const void *data, size_t len, off_t offset); - - Config *mpConfig; - SMLogging *mpLogger; - std::string msJournalPath; - std::string msCachePath; - //ThreadPool threadPool; + int updateMetadata(MetadataFile& meta); - size_t repUserDataWritten, repHeaderDataWritten; - size_t replicatorObjectsCreated, replicatorJournalsCreated; + void printKPIs() const; + private: + Replicator(); + + // a couple helpers + ssize_t _write(int fd, const void* data, size_t len); + ssize_t _pwrite(int fd, const void* data, size_t len, off_t offset); + + Config* mpConfig; + SMLogging* mpLogger; + std::string msJournalPath; + std::string msCachePath; + // ThreadPool threadPool; + + size_t repUserDataWritten, repHeaderDataWritten; + size_t replicatorObjectsCreated, replicatorJournalsCreated; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index 8a19840a4..758d9b2e6 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "S3Storage.h" #include "Config.h" #include @@ -36,31 +35,25 @@ using namespace std; namespace storagemanager { -string tolower(const string &s) +string tolower(const string& s) { - string ret(s); - for (uint i = 0; i < ret.length(); i++) - ret[i] = ::tolower(ret[i]); - return ret; + string ret(s); + for (uint i = 0; i < ret.length(); i++) + ret[i] = ::tolower(ret[i]); + return ret; } -static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) +static size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* userp) { - ((std::string*)userp)->append((char*)contents, size * nmemb); - return size * nmemb; + ((std::string*)userp)->append((char*)contents, size * nmemb); + return size * nmemb; } inline bool retryable_error(uint8_t s3err) { - return ( - s3err == MS3_ERR_RESPONSE_PARSE || - s3err == MS3_ERR_REQUEST_ERROR || - s3err == MS3_ERR_OOM || - s3err == MS3_ERR_IMPOSSIBLE || - s3err == MS3_ERR_AUTH || - s3err == MS3_ERR_SERVER || - s3err == MS3_ERR_AUTH_ROLE - ); + return (s3err == MS3_ERR_RESPONSE_PARSE || s3err == MS3_ERR_REQUEST_ERROR || s3err == MS3_ERR_OOM || + s3err == MS3_ERR_IMPOSSIBLE || s3err == MS3_ERR_AUTH || s3err == MS3_ERR_SERVER || + s3err == MS3_ERR_AUTH_ROLE); } // Best effort to map the errors returned by the ms3 API to linux errnos @@ -68,543 +61,569 @@ inline bool retryable_error(uint8_t s3err) // nonsensical or misleading, so we should allow non-errno values to be propagated upward. const int s3err_to_errno[] = { 0, - EINVAL, // 1 MS3_ERR_PARAMETER. Best effort. D'oh. - ENODATA, // 2 MS3_ERR_NO_DATA + EINVAL, // 1 MS3_ERR_PARAMETER. Best effort. D'oh. + ENODATA, // 2 MS3_ERR_NO_DATA ENAMETOOLONG, // 3 MS3_ERR_URI_TOO_LONG - EBADMSG, // 4 MS3_ERR_RESPONSE_PARSE - ECOMM, // 5 MS3_ERR_REQUEST_ERROR - ENOMEM, // 6 MS3_ERR_OOM - EINVAL, // 7 MS3_ERR_IMPOSSIBLE. Will have to look through the code to find out what this is exactly. + EBADMSG, // 4 MS3_ERR_RESPONSE_PARSE + ECOMM, // 5 MS3_ERR_REQUEST_ERROR + ENOMEM, // 6 MS3_ERR_OOM + EINVAL, // 7 MS3_ERR_IMPOSSIBLE. Will have to look through the code to find out what this is exactly. EKEYREJECTED, // 8 MS3_ERR_AUTH - ENOENT, // 9 MS3_ERR_NOT_FOUND - EPROTO, // 10 MS3_ERR_SERVER - EMSGSIZE, // 11 MS3_ERR_TOO_BIG + ENOENT, // 9 MS3_ERR_NOT_FOUND + EPROTO, // 10 MS3_ERR_SERVER + EMSGSIZE, // 11 MS3_ERR_TOO_BIG EKEYREJECTED // 12 MS3_ERR_AUTH_ROLE }; -const char *s3err_msgs[] = { - "All is well", - "A required function parameter is missing", - "No data is supplied to a function that requires data", - "The generated URI for the request is too long", - "The API could not parse the response", - "The API could not send the request", - "Could not allocate required memory", - "An impossible condition occurred, how unlucky are you?", - "Authentication failed", - "Object not found", - "Unknown error code in response", - "Data to PUT is too large; 4GB maximum length", - "Authentication failed, token has expired" -}; +const char* s3err_msgs[] = {"All is well", + "A required function parameter is missing", + "No data is supplied to a function that requires data", + "The generated URI for the request is too long", + "The API could not parse the response", + "The API could not send the request", + "Could not allocate required memory", + "An impossible condition occurred, how unlucky are you?", + "Authentication failed", + "Object not found", + "Unknown error code in response", + "Data to PUT is too large; 4GB maximum length", + "Authentication failed, token has expired"}; - -S3Storage::ScopedConnection::ScopedConnection(S3Storage *s, ms3_st *m) : s3(s), conn(m) +S3Storage::ScopedConnection::ScopedConnection(S3Storage* s, ms3_st* m) : s3(s), conn(m) { - assert(conn); + assert(conn); } S3Storage::ScopedConnection::~ScopedConnection() { - s3->returnConnection(conn); + s3->returnConnection(conn); } S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry) { - /* Check creds from envvars - Get necessary vars from config - Init an ms3_st object - */ - Config *config = Config::get(); - const char *keyerr = "S3 access requires setting AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars, " - " or setting aws_access_key_id and aws_secret_access_key, or configure an IAM role with ec2_iam_mode=enabled." - " Check storagemanager.cnf file for more information."; - key = config->getValue("S3", "aws_access_key_id"); - secret = config->getValue("S3", "aws_secret_access_key"); - IAMrole = config->getValue("S3", "iam_role_name"); - STSendpoint = config->getValue("S3", "sts_endpoint"); - STSregion = config->getValue("S3", "sts_region"); - string ec2_mode = tolower(config->getValue("S3", "ec2_iam_mode")); - string use_http = tolower(config->getValue("S3", "use_http")); - string ssl_verify = tolower(config->getValue("S3", "ssl_verify")); - string port_number = config->getValue("S3", "port_number"); + /* Check creds from envvars + Get necessary vars from config + Init an ms3_st object + */ + Config* config = Config::get(); + const char* keyerr = + "S3 access requires setting AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars, " + " or setting aws_access_key_id and aws_secret_access_key, or configure an IAM role with " + "ec2_iam_mode=enabled." + " Check storagemanager.cnf file for more information."; + key = config->getValue("S3", "aws_access_key_id"); + secret = config->getValue("S3", "aws_secret_access_key"); + IAMrole = config->getValue("S3", "iam_role_name"); + STSendpoint = config->getValue("S3", "sts_endpoint"); + STSregion = config->getValue("S3", "sts_region"); + string ec2_mode = tolower(config->getValue("S3", "ec2_iam_mode")); + string use_http = tolower(config->getValue("S3", "use_http")); + string ssl_verify = tolower(config->getValue("S3", "ssl_verify")); + string port_number = config->getValue("S3", "port_number"); + bool keyMissing = false; + isEC2Instance = false; + ec2iamEnabled = false; + useHTTP = false; + sslVerify = true; + portNumber = 0; - bool keyMissing = false; - isEC2Instance = false; - ec2iamEnabled = false; - useHTTP = false; - sslVerify = true; - portNumber = 0; + if (!port_number.empty()) + { + portNumber = stoi(port_number); + } - if (!port_number.empty()) + if (ec2_mode == "enabled") + { + ec2iamEnabled = true; + } + + if (use_http == "enabled") + { + useHTTP = true; + } + + if (ssl_verify == "disabled") + { + sslVerify = false; + } + + if (key.empty()) + { + char* _key_id = getenv("AWS_ACCESS_KEY_ID"); + if (!_key_id) { - portNumber = stoi(port_number); + keyMissing = true; } - - if (ec2_mode == "enabled") + else + key = _key_id; + } + if (secret.empty()) + { + char* _secret_id = getenv("AWS_SECRET_ACCESS_KEY"); + if (!_secret_id) { - ec2iamEnabled = true; + keyMissing = true; } + else + secret = _secret_id; + } - if (use_http == "enabled") + // Valid to not have keys configured if running on ec2 instance. + // Attempt to get those credentials from instance. + if (keyMissing) + { + if (ec2iamEnabled) { - useHTTP = true; + getIAMRoleFromMetadataEC2(); } + if (!IAMrole.empty() && getCredentialsFromMetadataEC2()) + { + isEC2Instance = true; + } + else + { + logger->log(LOG_ERR, keyerr); + throw runtime_error(keyerr); + } + } - if (ssl_verify == "disabled") - { - sslVerify = false; - } + region = config->getValue("S3", "region"); + bucket = config->getValue("S3", "bucket"); + prefix = config->getValue("S3", "prefix"); + if (bucket.empty()) + { + const char* msg = "S3 access requires setting S3/bucket in storagemanager.cnf"; + logger->log(LOG_ERR, msg); + throw runtime_error(msg); + } - if (key.empty()) - { - char *_key_id = getenv("AWS_ACCESS_KEY_ID"); - if (!_key_id) - { - keyMissing = true; - } - else - key = _key_id; - } - if (secret.empty()) - { - char *_secret_id = getenv("AWS_SECRET_ACCESS_KEY"); - if (!_secret_id) - { - keyMissing = true; - } - else - secret = _secret_id; - } + endpoint = config->getValue("S3", "endpoint"); - // Valid to not have keys configured if running on ec2 instance. - // Attempt to get those credentials from instance. - if (keyMissing) - { - if (ec2iamEnabled) - { - getIAMRoleFromMetadataEC2(); - } - if (!IAMrole.empty() && getCredentialsFromMetadataEC2()) - { - isEC2Instance = true; - } - else - { - logger->log(LOG_ERR, keyerr); - throw runtime_error(keyerr); - } - } - - region = config->getValue("S3", "region"); - bucket = config->getValue("S3", "bucket"); - prefix = config->getValue("S3", "prefix"); - if (bucket.empty()) - { - const char *msg = "S3 access requires setting S3/bucket in storagemanager.cnf"; - logger->log(LOG_ERR, msg); - throw runtime_error(msg); - } - - endpoint = config->getValue("S3", "endpoint"); - - ms3_library_init(); - //ms3_debug(); - testConnectivityAndPerms(); + ms3_library_init(); + // ms3_debug(); + testConnectivityAndPerms(); } S3Storage::~S3Storage() { - for (auto &conn : freeConns) - ms3_deinit(conn.conn); - ms3_library_deinit(); + for (auto& conn : freeConns) + ms3_deinit(conn.conn); + ms3_library_deinit(); } // convenience macro for testConnectivityAndPerms() -#define FAIL(OP) { \ - const char *msg = "S3Storage: failed to " #OP ", check log files for specific error"; \ - logger->log(LOG_ERR, msg); \ - throw runtime_error(msg); \ -} - +#define FAIL(OP) \ + { \ + const char* msg = "S3Storage: failed to " #OP ", check log files for specific error"; \ + logger->log(LOG_ERR, msg); \ + throw runtime_error(msg); \ + } bool S3Storage::getIAMRoleFromMetadataEC2() { - CURL *curl = NULL; - CURLcode curl_res; - string readBuffer; - string instanceMetadata = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"; - curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, instanceMetadata.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); - curl_res = curl_easy_perform(curl); - if (curl_res != CURLE_OK) - { - logger->log(LOG_ERR, "CURL fail %u",curl_res); - return false; - } - IAMrole = readBuffer; - //logger->log(LOG_INFO, "S3Storage: IAM Role Name = %s",IAMrole.c_str()); + CURL* curl = NULL; + CURLcode curl_res; + string readBuffer; + string instanceMetadata = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"; + curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_URL, instanceMetadata.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + curl_res = curl_easy_perform(curl); + if (curl_res != CURLE_OK) + { + logger->log(LOG_ERR, "CURL fail %u", curl_res); + return false; + } + IAMrole = readBuffer; + // logger->log(LOG_INFO, "S3Storage: IAM Role Name = %s",IAMrole.c_str()); - return true; + return true; } bool S3Storage::getCredentialsFromMetadataEC2() { - CURL *curl = NULL; - CURLcode curl_res; - std::string readBuffer; - string instanceMetadata = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + IAMrole; - curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, instanceMetadata.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); - curl_res = curl_easy_perform(curl); - if (curl_res != CURLE_OK) - { - logger->log(LOG_ERR, "CURL fail %u",curl_res); - return false; - } - stringstream credentials(readBuffer); - boost::property_tree::ptree pt; - boost::property_tree::read_json(credentials, pt); - key = pt.get("AccessKeyId"); - secret = pt.get("SecretAccessKey"); - token = pt.get("Token"); - //logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token = %s",key.c_str(),secret.c_str(),token.c_str()); + CURL* curl = NULL; + CURLcode curl_res; + std::string readBuffer; + string instanceMetadata = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + IAMrole; + curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_URL, instanceMetadata.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + curl_res = curl_easy_perform(curl); + if (curl_res != CURLE_OK) + { + logger->log(LOG_ERR, "CURL fail %u", curl_res); + return false; + } + stringstream credentials(readBuffer); + boost::property_tree::ptree pt; + boost::property_tree::read_json(credentials, pt); + key = pt.get("AccessKeyId"); + secret = pt.get("SecretAccessKey"); + token = pt.get("Token"); + // logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token = + // %s",key.c_str(),secret.c_str(),token.c_str()); - return true; + return true; } void S3Storage::testConnectivityAndPerms() { - boost::shared_array testObj(new uint8_t[1]); - testObj[0] = 0; - boost::uuids::uuid u = boost::uuids::random_generator()(); - ostringstream oss; - oss << u << "connectivity_test"; - - string testObjKey = oss.str(); - - int err = putObject(testObj, 1, testObjKey); - if (err) - FAIL(PUT) - bool _exists; - err = exists(testObjKey, &_exists); - if (err) - FAIL(HEAD) - size_t len; - err = getObject(testObjKey, &testObj, &len); - if (err) - FAIL(GET) - err = deleteObject(testObjKey); - if (err) - FAIL(DELETE) - logger->log(LOG_INFO, "S3Storage: S3 connectivity & permissions are OK"); + boost::shared_array testObj(new uint8_t[1]); + testObj[0] = 0; + boost::uuids::uuid u = boost::uuids::random_generator()(); + ostringstream oss; + oss << u << "connectivity_test"; + + string testObjKey = oss.str(); + + int err = putObject(testObj, 1, testObjKey); + if (err) + FAIL(PUT) + bool _exists; + err = exists(testObjKey, &_exists); + if (err) + FAIL(HEAD) + size_t len; + err = getObject(testObjKey, &testObj, &len); + if (err) + FAIL(GET) + err = deleteObject(testObjKey); + if (err) + FAIL(DELETE) + logger->log(LOG_INFO, "S3Storage: S3 connectivity & permissions are OK"); } -int S3Storage::getObject(const string &sourceKey, const string &destFile, size_t *size) +int S3Storage::getObject(const string& sourceKey, const string& destFile, size_t* size) { - int fd, err; - boost::shared_array data; - size_t len, count = 0; - char buf[80]; - - err = getObject(sourceKey, &data, &len); - if (err) - return err; - - fd = ::open(destFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (fd < 0) + int fd, err; + boost::shared_array data; + size_t len, count = 0; + char buf[80]; + + err = getObject(sourceKey, &data, &len); + if (err) + return err; + + fd = ::open(destFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd < 0) + { + int l_errno = errno; + logger->log(LOG_ERR, "S3Storage::getObject(): Failed to open %s, got %s", destFile.c_str(), + strerror_r(l_errno, buf, 80)); + errno = l_errno; + return err; + } + ScopedCloser s(fd); + while (count < len) + { + err = ::write(fd, &data[count], len - count); + if (err < 0) { - int l_errno = errno; - logger->log(LOG_ERR, "S3Storage::getObject(): Failed to open %s, got %s", destFile.c_str(), - strerror_r(l_errno, buf, 80)); - errno = l_errno; - return err; + int l_errno = errno; + logger->log(LOG_ERR, "S3Storage::getObject(): Failed to write to %s, got %s", destFile.c_str(), + strerror_r(errno, buf, 80)); + errno = l_errno; + return -1; } - ScopedCloser s(fd); - while (count < len) - { - err = ::write(fd, &data[count], len - count); - if (err < 0) - { - int l_errno = errno; - logger->log(LOG_ERR, "S3Storage::getObject(): Failed to write to %s, got %s", destFile.c_str(), - strerror_r(errno, buf, 80)); - errno = l_errno; - return -1; - } - count += err; - } - if (size) - *size = len; - return 0; + count += err; + } + if (size) + *size = len; + return 0; } -int S3Storage::getObject(const string &_sourceKey, boost::shared_array *data, size_t *size) +int S3Storage::getObject(const string& _sourceKey, boost::shared_array* data, size_t* size) { - uint8_t err; - size_t len = 0; - uint8_t *_data = NULL; - string sourceKey = prefix + _sourceKey; - - ms3_st *creds = getConnection(); - if (!creds) + uint8_t err; + size_t len = 0; + uint8_t* _data = NULL; + string sourceKey = prefix + _sourceKey; + + ms3_st* creds = getConnection(); + if (!creds) + { + logger->log(LOG_ERR, + "S3Storage::getObject(): failed to GET, S3Storage::getConnection() returned NULL on init"); + errno = EINVAL; + return -1; + } + ScopedConnection sc(this, creds); + + do + { + err = ms3_get(creds, bucket.c_str(), sourceKey.c_str(), &_data, &len); + if (err && (!skipRetryableErrors && retryable_error(err))) { - logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, S3Storage::getConnection() returned NULL on init"); - errno = EINVAL; - return -1; - } - ScopedConnection sc(this, creds); - - do { - err = ms3_get(creds, bucket.c_str(), sourceKey.c_str(), &_data, &len); - if (err && (!skipRetryableErrors && retryable_error(err))) - { - if (ms3_server_error(creds)) - logger->log(LOG_WARNING, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s." - " Retrying...", ms3_server_error(creds), bucket.c_str(), sourceKey.c_str()); - else - logger->log(LOG_WARNING, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s. Retrying...", + if (ms3_server_error(creds)) + logger->log(LOG_WARNING, + "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s." + " Retrying...", + ms3_server_error(creds), bucket.c_str(), sourceKey.c_str()); + else + logger->log(LOG_WARNING, + "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[err], bucket.c_str(), sourceKey.c_str()); - if (ec2iamEnabled) - { - getIAMRoleFromMetadataEC2(); - getCredentialsFromMetadataEC2(); - ms3_ec2_set_cred(creds,IAMrole.c_str(),key.c_str(),secret.c_str(),token.c_str()); - } - else if(!IAMrole.empty()) - { - ms3_assume_role(creds); - } - sleep(5); - } - } while (err && (!skipRetryableErrors && retryable_error(err))); - if (err) - { - if (ms3_server_error(creds)) - logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s.", - ms3_server_error(creds), bucket.c_str(), sourceKey.c_str()); - else - logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s.", - s3err_msgs[err], bucket.c_str(), sourceKey.c_str()); - data->reset(); - errno = s3err_to_errno[err]; - return -1; + if (ec2iamEnabled) + { + getIAMRoleFromMetadataEC2(); + getCredentialsFromMetadataEC2(); + ms3_ec2_set_cred(creds, IAMrole.c_str(), key.c_str(), secret.c_str(), token.c_str()); + } + else if (!IAMrole.empty()) + { + ms3_assume_role(creds); + } + sleep(5); } + } while (err && (!skipRetryableErrors && retryable_error(err))); + if (err) + { + if (ms3_server_error(creds)) + logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s.", + ms3_server_error(creds), bucket.c_str(), sourceKey.c_str()); + else + logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s.", + s3err_msgs[err], bucket.c_str(), sourceKey.c_str()); + data->reset(); + errno = s3err_to_errno[err]; + return -1; + } - data->reset(_data, free); - if (size) - *size = len; - return 0; + data->reset(_data, free); + if (size) + *size = len; + return 0; } - -int S3Storage::putObject(const string &sourceFile, const string &destKey) +int S3Storage::putObject(const string& sourceFile, const string& destKey) { - boost::shared_array data; - int err, fd; - size_t len, count = 0; - char buf[80]; - boost::system::error_code boost_err; - - len = boost::filesystem::file_size(sourceFile, boost_err); - if (boost_err) + boost::shared_array data; + int err, fd; + size_t len, count = 0; + char buf[80]; + boost::system::error_code boost_err; + + len = boost::filesystem::file_size(sourceFile, boost_err); + if (boost_err) + { + errno = boost_err.value(); + return -1; + } + data.reset(new uint8_t[len]); + fd = ::open(sourceFile.c_str(), O_RDONLY); + if (fd < 0) + { + int l_errno = errno; + logger->log(LOG_ERR, "S3Storage::putObject(): Failed to open %s, got %s", sourceFile.c_str(), + strerror_r(l_errno, buf, 80)); + errno = l_errno; + return -1; + } + ScopedCloser s(fd); + while (count < len) + { + err = ::read(fd, &data[count], len - count); + if (err < 0) { - errno = boost_err.value(); - return -1; + int l_errno = errno; + logger->log(LOG_ERR, "S3Storage::putObject(): Failed to read %s @ position %ld, got %s", + sourceFile.c_str(), count, strerror_r(l_errno, buf, 80)); + errno = l_errno; + return -1; } - data.reset(new uint8_t[len]); - fd = ::open(sourceFile.c_str(), O_RDONLY); - if (fd < 0) + else if (err == 0) { - int l_errno = errno; - logger->log(LOG_ERR, "S3Storage::putObject(): Failed to open %s, got %s", sourceFile.c_str(), - strerror_r(l_errno, buf, 80)); - errno = l_errno; - return -1; + // this shouldn't happen, we just checked the size + logger->log(LOG_ERR, "S3Storage::putObject(): Got early EOF reading %s @ position %ld", + sourceFile.c_str(), count); + errno = ENODATA; // is there a better errno for early eof? + return -1; } - ScopedCloser s(fd); - while (count < len) - { - err = ::read(fd, &data[count], len - count); - if (err < 0) - { - int l_errno = errno; - logger->log(LOG_ERR, "S3Storage::putObject(): Failed to read %s @ position %ld, got %s", sourceFile.c_str(), - count, strerror_r(l_errno, buf, 80)); - errno = l_errno; - return -1; - } - else if (err == 0) - { - // this shouldn't happen, we just checked the size - logger->log(LOG_ERR, "S3Storage::putObject(): Got early EOF reading %s @ position %ld", sourceFile.c_str(), - count); - errno = ENODATA; // is there a better errno for early eof? - return -1; - } - count += err; - } - - return putObject(data, len, destKey); + count += err; + } + + return putObject(data, len, destKey); } -int S3Storage::putObject(const boost::shared_array data, size_t len, const string &_destKey) +int S3Storage::putObject(const boost::shared_array data, size_t len, const string& _destKey) { - string destKey = prefix + _destKey; - uint8_t s3err; - ms3_st *creds = getConnection(); - if (!creds) + string destKey = prefix + _destKey; + uint8_t s3err; + ms3_st* creds = getConnection(); + if (!creds) + { + logger->log(LOG_ERR, + "S3Storage::putObject(): failed to PUT, S3Storage::getConnection() returned NULL on init"); + errno = EINVAL; + return -1; + } + ScopedConnection sc(this, creds); + + do + { + s3err = ms3_put(creds, bucket.c_str(), destKey.c_str(), data.get(), len); + if (s3err && (!skipRetryableErrors && retryable_error(s3err))) { - logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, S3Storage::getConnection() returned NULL on init"); - errno = EINVAL; - return -1; + if (ms3_server_error(creds)) + logger->log(LOG_WARNING, + "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s." + " Retrying...", + ms3_server_error(creds), bucket.c_str(), destKey.c_str()); + else + logger->log(LOG_WARNING, + "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s." + " Retrying...", + s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); + if (ec2iamEnabled) + { + getIAMRoleFromMetadataEC2(); + getCredentialsFromMetadataEC2(); + ms3_ec2_set_cred(creds, IAMrole.c_str(), key.c_str(), secret.c_str(), token.c_str()); + } + else if (!IAMrole.empty()) + { + ms3_assume_role(creds); + } + sleep(5); } - ScopedConnection sc(this, creds); - - do { - s3err = ms3_put(creds, bucket.c_str(), destKey.c_str(), data.get(), len); - if (s3err && (!skipRetryableErrors && retryable_error(s3err))) - { - if (ms3_server_error(creds)) - logger->log(LOG_WARNING, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s." - " Retrying...", ms3_server_error(creds), bucket.c_str(), destKey.c_str()); - else - logger->log(LOG_WARNING, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s." - " Retrying...", s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); - if (ec2iamEnabled) - { - getIAMRoleFromMetadataEC2(); - getCredentialsFromMetadataEC2(); - ms3_ec2_set_cred(creds,IAMrole.c_str(),key.c_str(),secret.c_str(),token.c_str()); - } - else if(!IAMrole.empty()) - { - ms3_assume_role(creds); - } - sleep(5); - } - } while (s3err && (!skipRetryableErrors && retryable_error(s3err))); - if (s3err) - { - if (ms3_server_error(creds)) - logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s.", - ms3_server_error(creds), bucket.c_str(), destKey.c_str()); - else - logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s.", - s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); - errno = s3err_to_errno[s3err]; - return -1; - } - return 0; + } while (s3err && (!skipRetryableErrors && retryable_error(s3err))); + if (s3err) + { + if (ms3_server_error(creds)) + logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s.", + ms3_server_error(creds), bucket.c_str(), destKey.c_str()); + else + logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s.", + s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); + errno = s3err_to_errno[s3err]; + return -1; + } + return 0; } -int S3Storage::deleteObject(const string &_key) +int S3Storage::deleteObject(const string& _key) { - uint8_t s3err; - string deleteKey = prefix + _key; - ms3_st *creds = getConnection(); - if (!creds) - { - logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, S3Storage::getConnection() returned NULL on init"); - errno = EINVAL; - return -1; - } - ScopedConnection sc(this, creds); + uint8_t s3err; + string deleteKey = prefix + _key; + ms3_st* creds = getConnection(); + if (!creds) + { + logger->log( + LOG_ERR, + "S3Storage::deleteObject(): failed to DELETE, S3Storage::getConnection() returned NULL on init"); + errno = EINVAL; + return -1; + } + ScopedConnection sc(this, creds); - do { - s3err = ms3_delete(creds, bucket.c_str(), deleteKey.c_str()); - if (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))) - { - if (ms3_server_error(creds)) - logger->log(LOG_WARNING, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s." - " Retrying...", ms3_server_error(creds), bucket.c_str(), deleteKey.c_str()); - else - logger->log(LOG_WARNING, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s. Retrying...", - s3err_msgs[s3err], bucket.c_str(), deleteKey.c_str()); - if (ec2iamEnabled) - { - getIAMRoleFromMetadataEC2(); - getCredentialsFromMetadataEC2(); - ms3_ec2_set_cred(creds,IAMrole.c_str(),key.c_str(),secret.c_str(),token.c_str()); - } - else if(!IAMrole.empty()) - { - ms3_assume_role(creds); - } - sleep(5); - } - } while (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))); - - if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND) + do + { + s3err = ms3_delete(creds, bucket.c_str(), deleteKey.c_str()); + if (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))) { - if (ms3_server_error(creds)) - logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s.", - ms3_server_error(creds), bucket.c_str(), deleteKey.c_str()); - else - logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s.", - s3err_msgs[s3err], bucket.c_str(), deleteKey.c_str()); - return -1; + if (ms3_server_error(creds)) + logger->log(LOG_WARNING, + "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s." + " Retrying...", + ms3_server_error(creds), bucket.c_str(), deleteKey.c_str()); + else + logger->log( + LOG_WARNING, + "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s. Retrying...", + s3err_msgs[s3err], bucket.c_str(), deleteKey.c_str()); + if (ec2iamEnabled) + { + getIAMRoleFromMetadataEC2(); + getCredentialsFromMetadataEC2(); + ms3_ec2_set_cred(creds, IAMrole.c_str(), key.c_str(), secret.c_str(), token.c_str()); + } + else if (!IAMrole.empty()) + { + ms3_assume_role(creds); + } + sleep(5); } - return 0; + } while (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))); + + if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND) + { + if (ms3_server_error(creds)) + logger->log(LOG_ERR, + "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s.", + ms3_server_error(creds), bucket.c_str(), deleteKey.c_str()); + else + logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s.", + s3err_msgs[s3err], bucket.c_str(), deleteKey.c_str()); + return -1; + } + return 0; } -int S3Storage::copyObject(const string &_sourceKey, const string &_destKey) +int S3Storage::copyObject(const string& _sourceKey, const string& _destKey) { - string sourceKey = prefix + _sourceKey, destKey = prefix + _destKey; - uint8_t s3err; - ms3_st *creds = getConnection(); - if (!creds) + string sourceKey = prefix + _sourceKey, destKey = prefix + _destKey; + uint8_t s3err; + ms3_st* creds = getConnection(); + if (!creds) + { + logger->log(LOG_ERR, + "S3Storage::copyObject(): failed to copy, S3Storage::getConnection() returned NULL on init"); + errno = EINVAL; + return -1; + } + ScopedConnection sc(this, creds); + + do + { + s3err = ms3_copy(creds, bucket.c_str(), sourceKey.c_str(), bucket.c_str(), destKey.c_str()); + if (s3err && (!skipRetryableErrors && retryable_error(s3err))) { - logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, S3Storage::getConnection() returned NULL on init"); - errno = EINVAL; - return -1; + if (ms3_server_error(creds)) + logger->log(LOG_WARNING, + "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, " + "destkey = %s. Retrying...", + ms3_server_error(creds), bucket.c_str(), sourceKey.c_str(), destKey.c_str()); + else + logger->log(LOG_WARNING, + "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, " + " destkey = %s. Retrying...", + s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); + if (ec2iamEnabled) + { + getIAMRoleFromMetadataEC2(); + getCredentialsFromMetadataEC2(); + ms3_ec2_set_cred(creds, IAMrole.c_str(), key.c_str(), secret.c_str(), token.c_str()); + } + else if (!IAMrole.empty()) + { + ms3_assume_role(creds); + } + sleep(5); } - ScopedConnection sc(this, creds); - - do - { - s3err = ms3_copy(creds, bucket.c_str(), sourceKey.c_str(), bucket.c_str(), destKey.c_str()); - if (s3err && (!skipRetryableErrors && retryable_error(s3err))) - { - if (ms3_server_error(creds)) - logger->log(LOG_WARNING, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, " - "destkey = %s. Retrying...", ms3_server_error(creds), bucket.c_str(), sourceKey.c_str(), destKey.c_str()); - else - logger->log(LOG_WARNING, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, " - " destkey = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); - if (ec2iamEnabled) - { - getIAMRoleFromMetadataEC2(); - getCredentialsFromMetadataEC2(); - ms3_ec2_set_cred(creds,IAMrole.c_str(),key.c_str(),secret.c_str(),token.c_str()); - } - else if(!IAMrole.empty()) - { - ms3_assume_role(creds); - } - sleep(5); - } - } while (s3err && (!skipRetryableErrors && retryable_error(s3err))); - - if (s3err) - { - // added the add'l check MS3_ERR_NOT_FOUND to suppress error msgs for a legitimate case in IOC::copyFile() - if (ms3_server_error(creds) && s3err != MS3_ERR_NOT_FOUND) - logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, " - "destkey = %s.", ms3_server_error(creds), bucket.c_str(), sourceKey.c_str(), destKey.c_str()); - else if (s3err != MS3_ERR_NOT_FOUND) - logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, " - "destkey = %s.", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); - errno = s3err_to_errno[s3err]; - return -1; - } - return 0; - + } while (s3err && (!skipRetryableErrors && retryable_error(s3err))); + + if (s3err) + { + // added the add'l check MS3_ERR_NOT_FOUND to suppress error msgs for a legitimate case in IOC::copyFile() + if (ms3_server_error(creds) && s3err != MS3_ERR_NOT_FOUND) + logger->log(LOG_ERR, + "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, " + "destkey = %s.", + ms3_server_error(creds), bucket.c_str(), sourceKey.c_str(), destKey.c_str()); + else if (s3err != MS3_ERR_NOT_FOUND) + logger->log(LOG_ERR, + "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, " + "destkey = %s.", + s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); + errno = s3err_to_errno[s3err]; + return -1; + } + return 0; + #if 0 // no s3-s3 copy yet. get & put for now. @@ -618,152 +637,162 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey) #endif } -int S3Storage::exists(const string &_key, bool *out) +int S3Storage::exists(const string& _key, bool* out) { - string existsKey = prefix + _key; - uint8_t s3err; - ms3_status_st status; - ms3_st *creds = getConnection(); - if (!creds) + string existsKey = prefix + _key; + uint8_t s3err; + ms3_status_st status; + ms3_st* creds = getConnection(); + if (!creds) + { + logger->log(LOG_ERR, + "S3Storage::exists(): failed to HEAD, S3Storage::getConnection() returned NULL on init"); + errno = EINVAL; + return -1; + } + ScopedConnection sc(this, creds); + + do + { + s3err = ms3_status(creds, bucket.c_str(), existsKey.c_str(), &status); + if (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))) { - logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, S3Storage::getConnection() returned NULL on init"); - errno = EINVAL; - return -1; - } - ScopedConnection sc(this, creds); - - do { - s3err = ms3_status(creds, bucket.c_str(), existsKey.c_str(), &status); - if (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))) - { - if (ms3_server_error(creds)) - logger->log(LOG_WARNING, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s." - " Retrying...", ms3_server_error(creds), bucket.c_str(), existsKey.c_str()); - else - logger->log(LOG_WARNING, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s. Retrying...", + if (ms3_server_error(creds)) + logger->log(LOG_WARNING, + "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s." + " Retrying...", + ms3_server_error(creds), bucket.c_str(), existsKey.c_str()); + else + logger->log(LOG_WARNING, + "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), existsKey.c_str()); - if (ec2iamEnabled) - { - getIAMRoleFromMetadataEC2(); - getCredentialsFromMetadataEC2(); - ms3_ec2_set_cred(creds,IAMrole.c_str(),key.c_str(),secret.c_str(),token.c_str()); - } - else if(!IAMrole.empty()) - { - ms3_assume_role(creds); - } - sleep(5); - } - } while (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))); - - if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND) - { - if (ms3_server_error(creds)) - logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s.", - ms3_server_error(creds), bucket.c_str(), existsKey.c_str()); - else - logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s.", - s3err_msgs[s3err], bucket.c_str(), existsKey.c_str()); - errno = s3err_to_errno[s3err]; - return -1; + if (ec2iamEnabled) + { + getIAMRoleFromMetadataEC2(); + getCredentialsFromMetadataEC2(); + ms3_ec2_set_cred(creds, IAMrole.c_str(), key.c_str(), secret.c_str(), token.c_str()); + } + else if (!IAMrole.empty()) + { + ms3_assume_role(creds); + } + sleep(5); } - - *out = (s3err == 0); - return 0; + } while (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))); + + if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND) + { + if (ms3_server_error(creds)) + logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s.", + ms3_server_error(creds), bucket.c_str(), existsKey.c_str()); + else + logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s.", + s3err_msgs[s3err], bucket.c_str(), existsKey.c_str()); + errno = s3err_to_errno[s3err]; + return -1; + } + + *out = (s3err == 0); + return 0; } -ms3_st * S3Storage::getConnection() +ms3_st* S3Storage::getConnection() { - boost::unique_lock s(connMutex); - - // prune the list. Most-idle connections are at the back. - timespec now; - clock_gettime(CLOCK_MONOTONIC_COARSE, &now); - while (!freeConns.empty()) - { - Connection &back = freeConns.back(); - if (back.idleSince.tv_sec + maxIdleSecs <= now.tv_sec) - { - ms3_deinit(back.conn); - //connMutexes.erase(back.conn); - back.conn = NULL; - freeConns.pop_back(); - } - else - break; // everything in front of this entry will not have been idle long enough - } - - // get a connection - ms3_st *ret = NULL; - uint8_t res = 0; - if (freeConns.empty()) - { - ret = ms3_init(key.c_str(), secret.c_str(), region.c_str(), (endpoint.empty() ? NULL : endpoint.c_str())); - // Something went wrong with libmarias3 init - if (ret == NULL) - { - logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_init returned NULL, no specific info to report"); - } - // Set option for use http instead of https - if (useHTTP) - { - ms3_set_option(ret, MS3_OPT_USE_HTTP, NULL); - } - // Set option to disable SSL Verification - if (!sslVerify) - { - ms3_set_option(ret, MS3_OPT_DISABLE_SSL_VERIFY, NULL); - } - // Port number is not 0 so it was set by cnf file - if (portNumber != 0) - { - ms3_set_option(ret, MS3_OPT_PORT_NUMBER, &portNumber); - } - // IAM role setup for keys - if (!IAMrole.empty()) - { - if (isEC2Instance) - { - res = ms3_ec2_set_cred(ret,IAMrole.c_str(),key.c_str(),secret.c_str(),token.c_str()); - } - else - { - res = ms3_init_assume_role(ret, (IAMrole.empty() ? NULL : IAMrole.c_str()), - (STSendpoint.empty() ? NULL : STSendpoint.c_str()), - (STSregion.empty() ? NULL : STSregion.c_str())); - } + boost::unique_lock s(connMutex); - if (res) - { - // Something is wrong with the assume role so abort as if the ms3_init failed - logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, aws_secret_access_key values. Also check sts_region and sts_endpoint if configured.",IAMrole.c_str()); - if (ms3_server_error(ret)) - logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s' role name = %s", ms3_server_error(ret), IAMrole.c_str()); - ms3_deinit(ret); - ret = NULL; - } - } - //assert(connMutexes[ret].try_lock()); - s.unlock(); - return ret; + // prune the list. Most-idle connections are at the back. + timespec now; + clock_gettime(CLOCK_MONOTONIC_COARSE, &now); + while (!freeConns.empty()) + { + Connection& back = freeConns.back(); + if (back.idleSince.tv_sec + maxIdleSecs <= now.tv_sec) + { + ms3_deinit(back.conn); + // connMutexes.erase(back.conn); + back.conn = NULL; + freeConns.pop_back(); } - assert(freeConns.front().idleSince.tv_sec + maxIdleSecs > now.tv_sec); - ret = freeConns.front().conn; - freeConns.pop_front(); - //assert(connMutexes[ret].try_lock()); + else + break; // everything in front of this entry will not have been idle long enough + } + + // get a connection + ms3_st* ret = NULL; + uint8_t res = 0; + if (freeConns.empty()) + { + ret = ms3_init(key.c_str(), secret.c_str(), region.c_str(), (endpoint.empty() ? NULL : endpoint.c_str())); + // Something went wrong with libmarias3 init + if (ret == NULL) + { + logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_init returned NULL, no specific info to report"); + } + // Set option for use http instead of https + if (useHTTP) + { + ms3_set_option(ret, MS3_OPT_USE_HTTP, NULL); + } + // Set option to disable SSL Verification + if (!sslVerify) + { + ms3_set_option(ret, MS3_OPT_DISABLE_SSL_VERIFY, NULL); + } + // Port number is not 0 so it was set by cnf file + if (portNumber != 0) + { + ms3_set_option(ret, MS3_OPT_PORT_NUMBER, &portNumber); + } + // IAM role setup for keys + if (!IAMrole.empty()) + { + if (isEC2Instance) + { + res = ms3_ec2_set_cred(ret, IAMrole.c_str(), key.c_str(), secret.c_str(), token.c_str()); + } + else + { + res = ms3_init_assume_role(ret, (IAMrole.empty() ? NULL : IAMrole.c_str()), + (STSendpoint.empty() ? NULL : STSendpoint.c_str()), + (STSregion.empty() ? NULL : STSregion.c_str())); + } + + if (res) + { + // Something is wrong with the assume role so abort as if the ms3_init failed + logger->log(LOG_ERR, + "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, " + "aws_access_key_id, aws_secret_access_key values. Also check sts_region and sts_endpoint " + "if configured.", + IAMrole.c_str()); + if (ms3_server_error(ret)) + logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s' role name = %s", + ms3_server_error(ret), IAMrole.c_str()); + ms3_deinit(ret); + ret = NULL; + } + } + // assert(connMutexes[ret].try_lock()); + s.unlock(); return ret; + } + assert(freeConns.front().idleSince.tv_sec + maxIdleSecs > now.tv_sec); + ret = freeConns.front().conn; + freeConns.pop_front(); + // assert(connMutexes[ret].try_lock()); + return ret; } -void S3Storage::returnConnection(ms3_st *ms3) +void S3Storage::returnConnection(ms3_st* ms3) { - assert(ms3); - Connection conn; - conn.conn = ms3; - clock_gettime(CLOCK_MONOTONIC_COARSE, &conn.idleSince); - - boost::unique_lock s(connMutex); - freeConns.push_front(conn); - //connMutexes[ms3].unlock(); + assert(ms3); + Connection conn; + conn.conn = ms3; + clock_gettime(CLOCK_MONOTONIC_COARSE, &conn.idleSince); + + boost::unique_lock s(connMutex); + freeConns.push_front(conn); + // connMutexes[ms3].unlock(); } -} +} // namespace storagemanager diff --git a/storage-manager/src/S3Storage.h b/storage-manager/src/S3Storage.h index 8c79d141b..338de2f6b 100644 --- a/storage-manager/src/S3Storage.h +++ b/storage-manager/src/S3Storage.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef S3STORAGE_H_ #define S3STORAGE_H_ @@ -28,71 +27,67 @@ namespace storagemanager { - class S3Storage : public CloudStorage { - public: - S3Storage(bool skipRetry = false); + public: + S3Storage(bool skipRetry = false); - virtual ~S3Storage(); + virtual ~S3Storage(); - int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL); - int getObject(const std::string &sourceKey, boost::shared_array *data, size_t *size = NULL); - int putObject(const std::string &sourceFile, const std::string &destKey); - int putObject(const boost::shared_array data, size_t len, const std::string &destKey); - int deleteObject(const std::string &key); - int copyObject(const std::string &sourceKey, const std::string &destKey); - int exists(const std::string &key, bool *out); + int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL); + int getObject(const std::string& sourceKey, boost::shared_array* data, size_t* size = NULL); + int putObject(const std::string& sourceFile, const std::string& destKey); + int putObject(const boost::shared_array data, size_t len, const std::string& destKey); + int deleteObject(const std::string& key); + int copyObject(const std::string& sourceKey, const std::string& destKey); + int exists(const std::string& key, bool* out); - private: - bool getIAMRoleFromMetadataEC2(); - bool getCredentialsFromMetadataEC2(); - void testConnectivityAndPerms(); - ms3_st *getConnection(); - void returnConnection(ms3_st *); - - bool skipRetryableErrors; + private: + bool getIAMRoleFromMetadataEC2(); + bool getCredentialsFromMetadataEC2(); + void testConnectivityAndPerms(); + ms3_st* getConnection(); + void returnConnection(ms3_st*); - std::string bucket; // might store this as a char *, since it's only used that way - std::string prefix; - std::string region; - std::string key; - std::string secret; - std::string token; - std::string endpoint; - std::string IAMrole; - std::string STSendpoint; - std::string STSregion; - bool isEC2Instance; - bool ec2iamEnabled; - bool useHTTP; - bool sslVerify; - int portNumber; - - struct Connection - { - ms3_st *conn; - timespec idleSince; - }; - struct ScopedConnection - { - ScopedConnection(S3Storage *, ms3_st *); - ~ScopedConnection(); - S3Storage *s3; - ms3_st *conn; - }; - - // for sanity checking - //std::map connMutexes; - - boost::mutex connMutex; - std::deque freeConns; // using this as a stack to keep lru objects together - const time_t maxIdleSecs = 30; + bool skipRetryableErrors; + + std::string bucket; // might store this as a char *, since it's only used that way + std::string prefix; + std::string region; + std::string key; + std::string secret; + std::string token; + std::string endpoint; + std::string IAMrole; + std::string STSendpoint; + std::string STSregion; + bool isEC2Instance; + bool ec2iamEnabled; + bool useHTTP; + bool sslVerify; + int portNumber; + + struct Connection + { + ms3_st* conn; + timespec idleSince; + }; + struct ScopedConnection + { + ScopedConnection(S3Storage*, ms3_st*); + ~ScopedConnection(); + S3Storage* s3; + ms3_st* conn; + }; + + // for sanity checking + // std::map connMutexes; + + boost::mutex connMutex; + std::deque freeConns; // using this as a stack to keep lru objects together + const time_t maxIdleSecs = 30; }; - - - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/SMLogging.cpp b/storage-manager/src/SMLogging.cpp index 21a4e4a96..d69811821 100644 --- a/storage-manager/src/SMLogging.cpp +++ b/storage-manager/src/SMLogging.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include #include #include "SMLogging.h" @@ -24,52 +23,51 @@ using namespace std; namespace { -storagemanager::SMLogging *smLog = NULL; +storagemanager::SMLogging* smLog = NULL; boost::mutex m; -}; +}; // namespace namespace storagemanager { - SMLogging::SMLogging() { - //TODO: make this configurable - setlogmask (LOG_UPTO (LOG_DEBUG)); - openlog ("StorageManager", LOG_PID | LOG_NDELAY | LOG_PERROR | LOG_CONS, LOG_LOCAL1); + // TODO: make this configurable + setlogmask(LOG_UPTO(LOG_DEBUG)); + openlog("StorageManager", LOG_PID | LOG_NDELAY | LOG_PERROR | LOG_CONS, LOG_LOCAL1); } SMLogging::~SMLogging() { - syslog(LOG_INFO, "CloseLog"); - closelog(); + syslog(LOG_INFO, "CloseLog"); + closelog(); } -SMLogging * SMLogging::get() +SMLogging* SMLogging::get() { - if (smLog) - return smLog; - boost::mutex::scoped_lock s(m); - if (smLog) - return smLog; - smLog = new SMLogging(); + if (smLog) return smLog; + boost::mutex::scoped_lock s(m); + if (smLog) + return smLog; + smLog = new SMLogging(); + return smLog; } -void SMLogging::log(int priority,const char *format, ...) +void SMLogging::log(int priority, const char* format, ...) { - va_list args; - va_start(args, format); - - #ifdef DEBUG - va_list args2; - va_copy(args2, args); - vfprintf(stderr, format, args2); - fprintf(stderr, "\n"); - va_end(args2); - #endif - vsyslog(priority, format, args); + va_list args; + va_start(args, format); - va_end(args); +#ifdef DEBUG + va_list args2; + va_copy(args2, args); + vfprintf(stderr, format, args2); + fprintf(stderr, "\n"); + va_end(args2); +#endif + vsyslog(priority, format, args); + + va_end(args); } -} +} // namespace storagemanager diff --git a/storage-manager/src/SMLogging.h b/storage-manager/src/SMLogging.h index 769fe29b0..d284898bd 100644 --- a/storage-manager/src/SMLogging.h +++ b/storage-manager/src/SMLogging.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef SM_LOGGING_H_ #define SM_LOGGING_H_ @@ -24,23 +23,19 @@ namespace storagemanager { - class SMLogging { - public: + public: + static SMLogging* get(); + ~SMLogging(); - static SMLogging *get(); - ~SMLogging(); - - void log(int priority, const char *format, ...); - - private: - SMLogging(); - //SMConfig& config; + void log(int priority, const char* format, ...); + private: + SMLogging(); + // SMConfig& config; }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/SessionManager.cpp b/storage-manager/src/SessionManager.cpp index f1b7b7c11..593e4255e 100644 --- a/storage-manager/src/SessionManager.cpp +++ b/storage-manager/src/SessionManager.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include #include #include @@ -41,449 +40,450 @@ using namespace std; namespace { - storagemanager::SessionManager *sm = NULL; - boost::mutex m; -} +storagemanager::SessionManager* sm = NULL; +boost::mutex m; +} // namespace namespace storagemanager { - SessionManager::SessionManager() { - crp = ClientRequestProcessor::get(); - socketCtrl[0] = -1; - socketCtrl[1] = -1; + crp = ClientRequestProcessor::get(); + socketCtrl[0] = -1; + socketCtrl[1] = -1; } SessionManager::~SessionManager() { } -SessionManager * SessionManager::get() +SessionManager* SessionManager::get() { - if (sm) - return sm; - boost::mutex::scoped_lock s(m); - if (sm) - return sm; - sm = new SessionManager(); + if (sm) return sm; + boost::mutex::scoped_lock s(m); + if (sm) + return sm; + sm = new SessionManager(); + return sm; } int SessionManager::start() { - SMLogging* logger = SMLogging::get(); - int rc,listenSockfd,incomingSockfd,on = 1; - struct sockaddr_un addr; - int nfds; - int pollTimeout = -1; - int socketIncr; - int current_size = 0; + SMLogging* logger = SMLogging::get(); + int rc, listenSockfd, incomingSockfd, on = 1; + struct sockaddr_un addr; + int nfds; + int pollTimeout = -1; + int socketIncr; + int current_size = 0; - bool shutdown = false; - bool running = true; + bool shutdown = false; + bool running = true; - if (pipe(socketCtrl)==-1) - { - logger->log(LOG_CRIT,"Pipe Failed: %s", strerror(errno)); - return 1; - } - - listenSockfd = ::socket(AF_UNIX, SOCK_STREAM, 0); - if (listenSockfd < 0) - { - logger->log(LOG_CRIT,"socket() failed: %s", strerror(errno)); - return -1; - } - - rc = ::setsockopt(listenSockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)); - if (rc < 0) - { - logger->log(LOG_CRIT,"setsockopt() failed: %s", strerror(errno)); - close(listenSockfd); - return -1; - } - - rc = ::ioctl(listenSockfd, FIONBIO, (char *)&on); - if (rc < 0) - { - logger->log(LOG_CRIT,"ioctl() failed: %s", strerror(errno)); - close(listenSockfd); - return -1; - } - - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null... - rc = ::bind(listenSockfd,(struct sockaddr *)&addr, sizeof(addr)); - if (rc < 0) - { - logger->log(LOG_CRIT,"bind() failed: %s", strerror(errno)); - close(listenSockfd); - return -1; - } - - rc = ::listen(listenSockfd, 32); - if (rc < 0) - { - logger->log(LOG_CRIT,"listen() failed: %s", strerror(errno)); - close(listenSockfd); - return -1; - } - - memset(fds, 0 , sizeof(fds)); - fds[0].fd = listenSockfd; - fds[0].events = POLLIN; - fds[1].fd = socketCtrl[0]; - fds[1].events = POLLIN; - nfds = 2; - - logger->log(LOG_INFO,"SessionManager waiting for sockets."); - while (running) - { - try - { - //if (current_size != nfds) - //logger->log(LOG_DEBUG,"polling %i fds %i", nfds,fds); - //cout << "polling " << nfds << " fds" << endl; - rc = ::poll(fds, nfds, pollTimeout); - if (rc < 0 && errno != EINTR) - { - logger->log(LOG_CRIT,"poll() failed: %s", strerror(errno)); - break; - } - else if (rc < 0) - continue; - current_size = nfds; - for (socketIncr = 0; socketIncr < current_size; socketIncr++) - { - - if(fds[socketIncr].revents == 0) - continue; - //if (socketIncr >= 2) - //logger->log(LOG_DEBUG,"got event on fd %i index is %i", fds[socketIncr].fd,socketIncr); - if(fds[socketIncr].revents != POLLIN) - { - //logger->log(LOG_DEBUG,"Error! revents = %d", fds[socketIncr].revents,); - if (fds[socketIncr].fd == -1) - logger->log(LOG_DEBUG,"!= POLLIN, closing fd -1"); - close(fds[socketIncr].fd); - fds[socketIncr].fd = -1; - continue; - } - if (fds[socketIncr].fd == listenSockfd) - { - //logger->log(LOG_DEBUG,"Listening socket is readable"); - incomingSockfd = 0; - while (incomingSockfd != -1 && !shutdown) - { - if (nfds >= MAX_SM_SOCKETS) - break; // try to free up some room - - incomingSockfd = ::accept(listenSockfd, NULL, NULL); - if (incomingSockfd < 0) - { - if (errno != EWOULDBLOCK) - { - logger->log(LOG_CRIT,"accept() failed: %s", strerror(errno)); - shutdown = true; - } - break; - } - //logger->log(LOG_DEBUG,"New incoming connection - %d",incomingSockfd); - fds[nfds].fd = incomingSockfd; - fds[nfds].events = POLLIN; - nfds++; - } - } - else if (fds[socketIncr].fd == socketCtrl[0]) - { - //logger->log(LOG_DEBUG,"SocketControl is readable"); - uint8_t ctrlCode; - int len,socket; - - len = ::read(socketCtrl[0], &ctrlCode, 1); - if (len <= 0) - { - continue; - } - switch(ctrlCode) - { - case ADDFD: - len = ::read(socketCtrl[0], &socket, sizeof(socket)); - if (len <= 0) - { - continue; - } - for (int i = 0; i < nfds; i++) - { - if(fds[i].fd == socket) - { - if (shutdown) - { - logger->log(LOG_DEBUG,"Shutdown in progress, closed socket %i at index %i", fds[i].fd,i); - close(socket); - break; - } - fds[i].events = (POLLIN | POLLPRI); - break; - } - } - break; - case REMOVEFD: - len = ::read(socketCtrl[0], &socket, sizeof(socket)); - if (len <= 0) - { - continue; - } - for (int i = 0; i < nfds; i++) - { - if (fds[i].fd == socket) - { - if (socket == -1) - logger->log(LOG_DEBUG,"REMOVEFD told to remove fd -1"); - close(socket); - fds[i].fd = -1; - break; - } - } - break; - case SHUTDOWN: - logger->log(LOG_DEBUG,"Shutdown StorageManager..."); - close(fds[0].fd); - shutdown = true; - break; - default: - break; - } - } - else - { - //logger->log(LOG_DEBUG,"socketIncr %d -- Descriptor %d is readable",socketIncr,fds[socketIncr].fd); - bool closeConn = false; - char recv_buffer[8192]; - uint recvMsgLength = 0; - uint recvMsgStart = 0; - uint remainingBytes = 0; - uint endOfData, i; - int peakLength,len; - - //logger->log(LOG_DEBUG,"reading from fd %i index is %i", fds[socketIncr].fd,socketIncr); - if (sockState.find(fds[socketIncr].fd) != sockState.end()) - { - SockState &state = sockState[fds[socketIncr].fd]; - remainingBytes = state.remainingBytes; - memcpy(recv_buffer, state.remainingData, remainingBytes); - sockState.erase(fds[socketIncr].fd); - } - while(true) - { - peakLength = ::recv(fds[socketIncr].fd, &recv_buffer[remainingBytes], sizeof(recv_buffer)-remainingBytes, MSG_PEEK | MSG_DONTWAIT); - if (peakLength < 0) - { - if (errno != EWOULDBLOCK) - { - closeConn = true; - break; - } - - // there's no data available at this point; go back to the poll loop. - assert(remainingBytes < SM_HEADER_LEN); - SockState state; - state.remainingBytes = remainingBytes; - memcpy(state.remainingData, recv_buffer, remainingBytes); - sockState[fds[socketIncr].fd] = state; - break; - } - //logger->log(LOG_DEBUG,"recv got %i bytes", peakLength); - endOfData = remainingBytes + peakLength; - if (endOfData < SM_HEADER_LEN) - { - //read this snippet and keep going - len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength); - remainingBytes = endOfData; - if (len != peakLength) - logger->log(LOG_ERR,"Read returned length != peakLength. ( %i != %i )", len, peakLength); - continue; - } - - //Look for SM_MSG_START - for (i = 0; i <= endOfData - SM_HEADER_LEN; i++) - { - if (*((uint *) &recv_buffer[i]) == SM_MSG_START) - { - //logger->log(LOG_DEBUG,"Received SM_MSG_START"); - //found it set msgLength and recvMsgStart offset of SM_MSG_START - recvMsgLength = *((uint *) &recv_buffer[i+4]); - //logger->log(LOG_DEBUG,"got length = %i", recvMsgLength); - recvMsgStart = i + SM_HEADER_LEN; - //logger->log(LOG_DEBUG,"recvMsgLength %d recvMsgStart %d endofData %d", recvMsgLength,recvMsgStart,endOfData); - // if >= endOfData then the start of the message data is the beginning of next message - if (recvMsgStart >= endOfData) - recvMsgStart = 0; - break; - } - } - - // didn't find SM_MSG_START in this message consume the data and loop back through on next message - if (recvMsgLength == 0) - { - //logger->log(LOG_DEBUG,"No SM_MSG_START"); - len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength); - if (len != peakLength) - logger->log(LOG_ERR,"Read returned length != peakLength. ( %i != %i )", len, peakLength); - // we know the msg header isn't in position [0, endOfData - i), so throw that out - // and copy [i, endOfData) to the front of the buffer to be - // checked by the next iteration. - memcpy(recv_buffer, &recv_buffer[i], endOfData - i); - remainingBytes = endOfData - i; - } - else - { - //found msg start - //remove the junk in front of the message - if (recvMsgStart > 0) - { - //logger->log(LOG_DEBUG,"SM_MSG_START data is here"); - // how many to consume here... - // recvMsgStart is the position in the buffer - // peakLength is the amount peeked this time - // remainingBytes is the amount carried over from previous reads - len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], recvMsgStart - remainingBytes); - } - else - { - //logger->log(LOG_DEBUG,"SM_MSG_START data is next message"); - len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength); - if (len != peakLength) - logger->log(LOG_ERR,"Read returned length != peakLength. ( %i != %i )", len, peakLength); - } - //Disable polling on this socket - fds[socketIncr].events = 0; - //Pass the socket to CRP and the message length. next read will be start of message - crp->processRequest(fds[socketIncr].fd,recvMsgLength); - - /* - //Doing this to work with cloudio_component_test - len = ::read(fds[socketIncr].fd, out, recvMsgLength); - logger->log(LOG_DEBUG,"Read %d bytes.",len); - //Debug test lets send a reponse back - uint32_t response[4] = { storagemanager::SM_MSG_START, 8, (uint32_t ) -1, EINVAL }; - len = ::send(fds[socketIncr].fd, response, 16, 0); - */ - break; - } - } - - if (closeConn) - { - if (fds[socketIncr].fd == -1) - logger->log(LOG_DEBUG,"closeConn closing fd -1"); - close(fds[socketIncr].fd); - fds[socketIncr].fd = -1; - } - } - } - } - catch (std::exception& ex) - { - throw std::exception(); - } - catch (...) - { - throw std::exception(); - } - - int i, j; - if (shutdown) - { - for (i = 2; i < nfds; ++i) - { - if (fds[i].events == (POLLIN | POLLPRI)) - { - close(fds[i].fd); - fds[i].fd = -1; - } - } - } - /* get rid of fds == -1 */ - for (i = 2; i < nfds; ++i) - { - if (fds[i].fd == -1) - break; - } - for (j = i + 1; j < nfds; ++j) - { - if (fds[j].fd != -1) - { - fds[i].fd = fds[j].fd; - fds[i].events = fds[j].events; - ++i; - } - } - nfds = i; - - if(shutdown) - { - if (nfds <= 2) - running = false; - } - } - - //Shutdown Done - crp->shutdown(); + if (pipe(socketCtrl) == -1) + { + logger->log(LOG_CRIT, "Pipe Failed: %s", strerror(errno)); + return 1; + } + listenSockfd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (listenSockfd < 0) + { + logger->log(LOG_CRIT, "socket() failed: %s", strerror(errno)); return -1; + } + + rc = ::setsockopt(listenSockfd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)); + if (rc < 0) + { + logger->log(LOG_CRIT, "setsockopt() failed: %s", strerror(errno)); + close(listenSockfd); + return -1; + } + + rc = ::ioctl(listenSockfd, FIONBIO, (char*)&on); + if (rc < 0) + { + logger->log(LOG_CRIT, "ioctl() failed: %s", strerror(errno)); + close(listenSockfd); + return -1; + } + + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null... + rc = ::bind(listenSockfd, (struct sockaddr*)&addr, sizeof(addr)); + if (rc < 0) + { + logger->log(LOG_CRIT, "bind() failed: %s", strerror(errno)); + close(listenSockfd); + return -1; + } + + rc = ::listen(listenSockfd, 32); + if (rc < 0) + { + logger->log(LOG_CRIT, "listen() failed: %s", strerror(errno)); + close(listenSockfd); + return -1; + } + + memset(fds, 0, sizeof(fds)); + fds[0].fd = listenSockfd; + fds[0].events = POLLIN; + fds[1].fd = socketCtrl[0]; + fds[1].events = POLLIN; + nfds = 2; + + logger->log(LOG_INFO, "SessionManager waiting for sockets."); + while (running) + { + try + { + // if (current_size != nfds) + // logger->log(LOG_DEBUG,"polling %i fds %i", nfds,fds); + // cout << "polling " << nfds << " fds" << endl; + rc = ::poll(fds, nfds, pollTimeout); + if (rc < 0 && errno != EINTR) + { + logger->log(LOG_CRIT, "poll() failed: %s", strerror(errno)); + break; + } + else if (rc < 0) + continue; + current_size = nfds; + for (socketIncr = 0; socketIncr < current_size; socketIncr++) + { + if (fds[socketIncr].revents == 0) + continue; + // if (socketIncr >= 2) + // logger->log(LOG_DEBUG,"got event on fd %i index is %i", fds[socketIncr].fd,socketIncr); + if (fds[socketIncr].revents != POLLIN) + { + // logger->log(LOG_DEBUG,"Error! revents = %d", fds[socketIncr].revents,); + if (fds[socketIncr].fd == -1) + logger->log(LOG_DEBUG, "!= POLLIN, closing fd -1"); + close(fds[socketIncr].fd); + fds[socketIncr].fd = -1; + continue; + } + if (fds[socketIncr].fd == listenSockfd) + { + // logger->log(LOG_DEBUG,"Listening socket is readable"); + incomingSockfd = 0; + while (incomingSockfd != -1 && !shutdown) + { + if (nfds >= MAX_SM_SOCKETS) + break; // try to free up some room + + incomingSockfd = ::accept(listenSockfd, NULL, NULL); + if (incomingSockfd < 0) + { + if (errno != EWOULDBLOCK) + { + logger->log(LOG_CRIT, "accept() failed: %s", strerror(errno)); + shutdown = true; + } + break; + } + // logger->log(LOG_DEBUG,"New incoming connection - %d",incomingSockfd); + fds[nfds].fd = incomingSockfd; + fds[nfds].events = POLLIN; + nfds++; + } + } + else if (fds[socketIncr].fd == socketCtrl[0]) + { + // logger->log(LOG_DEBUG,"SocketControl is readable"); + uint8_t ctrlCode; + int len, socket; + + len = ::read(socketCtrl[0], &ctrlCode, 1); + if (len <= 0) + { + continue; + } + switch (ctrlCode) + { + case ADDFD: + len = ::read(socketCtrl[0], &socket, sizeof(socket)); + if (len <= 0) + { + continue; + } + for (int i = 0; i < nfds; i++) + { + if (fds[i].fd == socket) + { + if (shutdown) + { + logger->log(LOG_DEBUG, "Shutdown in progress, closed socket %i at index %i", fds[i].fd, + i); + close(socket); + break; + } + fds[i].events = (POLLIN | POLLPRI); + break; + } + } + break; + case REMOVEFD: + len = ::read(socketCtrl[0], &socket, sizeof(socket)); + if (len <= 0) + { + continue; + } + for (int i = 0; i < nfds; i++) + { + if (fds[i].fd == socket) + { + if (socket == -1) + logger->log(LOG_DEBUG, "REMOVEFD told to remove fd -1"); + close(socket); + fds[i].fd = -1; + break; + } + } + break; + case SHUTDOWN: + logger->log(LOG_DEBUG, "Shutdown StorageManager..."); + close(fds[0].fd); + shutdown = true; + break; + default: break; + } + } + else + { + // logger->log(LOG_DEBUG,"socketIncr %d -- Descriptor %d is + // readable",socketIncr,fds[socketIncr].fd); + bool closeConn = false; + char recv_buffer[8192]; + uint recvMsgLength = 0; + uint recvMsgStart = 0; + uint remainingBytes = 0; + uint endOfData, i; + int peakLength, len; + + // logger->log(LOG_DEBUG,"reading from fd %i index is %i", fds[socketIncr].fd,socketIncr); + if (sockState.find(fds[socketIncr].fd) != sockState.end()) + { + SockState& state = sockState[fds[socketIncr].fd]; + remainingBytes = state.remainingBytes; + memcpy(recv_buffer, state.remainingData, remainingBytes); + sockState.erase(fds[socketIncr].fd); + } + while (true) + { + peakLength = ::recv(fds[socketIncr].fd, &recv_buffer[remainingBytes], + sizeof(recv_buffer) - remainingBytes, MSG_PEEK | MSG_DONTWAIT); + if (peakLength < 0) + { + if (errno != EWOULDBLOCK) + { + closeConn = true; + break; + } + + // there's no data available at this point; go back to the poll loop. + assert(remainingBytes < SM_HEADER_LEN); + SockState state; + state.remainingBytes = remainingBytes; + memcpy(state.remainingData, recv_buffer, remainingBytes); + sockState[fds[socketIncr].fd] = state; + break; + } + // logger->log(LOG_DEBUG,"recv got %i bytes", peakLength); + endOfData = remainingBytes + peakLength; + if (endOfData < SM_HEADER_LEN) + { + // read this snippet and keep going + len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength); + remainingBytes = endOfData; + if (len != peakLength) + logger->log(LOG_ERR, "Read returned length != peakLength. ( %i != %i )", len, peakLength); + continue; + } + + // Look for SM_MSG_START + for (i = 0; i <= endOfData - SM_HEADER_LEN; i++) + { + if (*((uint*)&recv_buffer[i]) == SM_MSG_START) + { + // logger->log(LOG_DEBUG,"Received SM_MSG_START"); + // found it set msgLength and recvMsgStart offset of SM_MSG_START + recvMsgLength = *((uint*)&recv_buffer[i + 4]); + // logger->log(LOG_DEBUG,"got length = %i", recvMsgLength); + recvMsgStart = i + SM_HEADER_LEN; + // logger->log(LOG_DEBUG,"recvMsgLength %d recvMsgStart %d endofData %d", + // recvMsgLength,recvMsgStart,endOfData); + // if >= endOfData then the start of the message data is the beginning of next message + if (recvMsgStart >= endOfData) + recvMsgStart = 0; + break; + } + } + + // didn't find SM_MSG_START in this message consume the data and loop back through on next message + if (recvMsgLength == 0) + { + // logger->log(LOG_DEBUG,"No SM_MSG_START"); + len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength); + if (len != peakLength) + logger->log(LOG_ERR, "Read returned length != peakLength. ( %i != %i )", len, peakLength); + // we know the msg header isn't in position [0, endOfData - i), so throw that out + // and copy [i, endOfData) to the front of the buffer to be + // checked by the next iteration. + memcpy(recv_buffer, &recv_buffer[i], endOfData - i); + remainingBytes = endOfData - i; + } + else + { + // found msg start + // remove the junk in front of the message + if (recvMsgStart > 0) + { + // logger->log(LOG_DEBUG,"SM_MSG_START data is here"); + // how many to consume here... + // recvMsgStart is the position in the buffer + // peakLength is the amount peeked this time + // remainingBytes is the amount carried over from previous reads + len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], recvMsgStart - remainingBytes); + } + else + { + // logger->log(LOG_DEBUG,"SM_MSG_START data is next message"); + len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength); + if (len != peakLength) + logger->log(LOG_ERR, "Read returned length != peakLength. ( %i != %i )", len, peakLength); + } + // Disable polling on this socket + fds[socketIncr].events = 0; + // Pass the socket to CRP and the message length. next read will be start of message + crp->processRequest(fds[socketIncr].fd, recvMsgLength); + + /* + //Doing this to work with cloudio_component_test + len = ::read(fds[socketIncr].fd, out, recvMsgLength); + logger->log(LOG_DEBUG,"Read %d bytes.",len); + //Debug test lets send a reponse back + uint32_t response[4] = { storagemanager::SM_MSG_START, 8, (uint32_t ) -1, EINVAL }; + len = ::send(fds[socketIncr].fd, response, 16, 0); + */ + break; + } + } + + if (closeConn) + { + if (fds[socketIncr].fd == -1) + logger->log(LOG_DEBUG, "closeConn closing fd -1"); + close(fds[socketIncr].fd); + fds[socketIncr].fd = -1; + } + } + } + } + catch (std::exception& ex) + { + throw std::exception(); + } + catch (...) + { + throw std::exception(); + } + + int i, j; + if (shutdown) + { + for (i = 2; i < nfds; ++i) + { + if (fds[i].events == (POLLIN | POLLPRI)) + { + close(fds[i].fd); + fds[i].fd = -1; + } + } + } + /* get rid of fds == -1 */ + for (i = 2; i < nfds; ++i) + { + if (fds[i].fd == -1) + break; + } + for (j = i + 1; j < nfds; ++j) + { + if (fds[j].fd != -1) + { + fds[i].fd = fds[j].fd; + fds[i].events = fds[j].events; + ++i; + } + } + nfds = i; + + if (shutdown) + { + if (nfds <= 2) + running = false; + } + } + + // Shutdown Done + crp->shutdown(); + + return -1; } void SessionManager::returnSocket(int socket) { - boost::mutex::scoped_lock s(ctrlMutex); - int err; - uint8_t ctrlCode = ADDFD; - err = ::write(socketCtrl[1], &ctrlCode, 1); - if (err <= 0) - { - return; - } - err = ::write(socketCtrl[1], &socket, sizeof(socket)); - if (err <= 0) - { - return; - } + boost::mutex::scoped_lock s(ctrlMutex); + int err; + uint8_t ctrlCode = ADDFD; + err = ::write(socketCtrl[1], &ctrlCode, 1); + if (err <= 0) + { + return; + } + err = ::write(socketCtrl[1], &socket, sizeof(socket)); + if (err <= 0) + { + return; + } } void SessionManager::socketError(int socket) { - boost::mutex::scoped_lock s(ctrlMutex); - SMLogging* logger = SMLogging::get(); - logger->log(LOG_CRIT," ****** socket error!"); - int err; - uint8_t ctrlCode = REMOVEFD; - err = ::write(socketCtrl[1], &ctrlCode, 1); - if (err <= 0) - { - return; - } - err = ::write(socketCtrl[1], &socket, sizeof(socket)); - if (err <= 0) - { - return; - } -} - -void SessionManager::shutdownSM(int sig){ - boost::mutex::scoped_lock s(ctrlMutex); - SMLogging* logger = SMLogging::get(); - logger->log(LOG_DEBUG,"SessionManager Caught Signal %i",sig); - int err; - uint8_t ctrlCode = SHUTDOWN; - err = ::write(socketCtrl[1], &ctrlCode, 1); - if (err <= 0) - { - return; - } + boost::mutex::scoped_lock s(ctrlMutex); + SMLogging* logger = SMLogging::get(); + logger->log(LOG_CRIT, " ****** socket error!"); + int err; + uint8_t ctrlCode = REMOVEFD; + err = ::write(socketCtrl[1], &ctrlCode, 1); + if (err <= 0) + { + return; + } + err = ::write(socketCtrl[1], &socket, sizeof(socket)); + if (err <= 0) + { + return; + } } +void SessionManager::shutdownSM(int sig) +{ + boost::mutex::scoped_lock s(ctrlMutex); + SMLogging* logger = SMLogging::get(); + logger->log(LOG_DEBUG, "SessionManager Caught Signal %i", sig); + int err; + uint8_t ctrlCode = SHUTDOWN; + err = ::write(socketCtrl[1], &ctrlCode, 1); + if (err <= 0) + { + return; + } } +} // namespace storagemanager diff --git a/storage-manager/src/SessionManager.h b/storage-manager/src/SessionManager.h index 5ba06666a..dc8de64ad 100644 --- a/storage-manager/src/SessionManager.h +++ b/storage-manager/src/SessionManager.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef STORAGEMANGER_H_ #define STORAGEMANGER_H_ @@ -28,14 +26,13 @@ #include #include - namespace storagemanager { - -enum sessionCtrl { - ADDFD, - REMOVEFD, - SHUTDOWN +enum sessionCtrl +{ + ADDFD, + REMOVEFD, + SHUTDOWN }; #define MAX_SM_SOCKETS 200 @@ -45,38 +42,38 @@ enum sessionCtrl { */ class SessionManager { -public: - static SessionManager *get(); - ~SessionManager(); + public: + static SessionManager* get(); + ~SessionManager(); - /** - * start and manage socket connections - */ - int start(); + /** + * start and manage socket connections + */ + int start(); - void returnSocket(int socket); - void socketError(int socket); - void CRPTest(int socket,uint length); - void shutdownSM(int sig); + void returnSocket(int socket); + void socketError(int socket); + void CRPTest(int socket, uint length); + void shutdownSM(int sig); -private: - SessionManager(); - //SMConfig& config; - ClientRequestProcessor *crp; - struct pollfd fds[MAX_SM_SOCKETS]; - int socketCtrl[2]; - boost::mutex ctrlMutex; - - // These map a socket fd to its state between read iterations if a message header could not be found in the data - // available at the time. - struct SockState { - char remainingData[SM_HEADER_LEN]; - uint remainingBytes; - }; - std::tr1::unordered_map sockState; - + private: + SessionManager(); + // SMConfig& config; + ClientRequestProcessor* crp; + struct pollfd fds[MAX_SM_SOCKETS]; + int socketCtrl[2]; + boost::mutex ctrlMutex; + + // These map a socket fd to its state between read iterations if a message header could not be found in the + // data available at the time. + struct SockState + { + char remainingData[SM_HEADER_LEN]; + uint remainingBytes; + }; + std::tr1::unordered_map sockState; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/StatTask.cpp b/storage-manager/src/StatTask.cpp index c61bb4fc7..b34e12fe2 100644 --- a/storage-manager/src/StatTask.cpp +++ b/storage-manager/src/StatTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "StatTask.h" #include "messageFormat.h" #include "SMLogging.h" @@ -29,7 +28,6 @@ using namespace std; namespace storagemanager { - StatTask::StatTask(int sock, uint len) : PosixTask(sock, len) { } @@ -39,53 +37,54 @@ StatTask::~StatTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } bool StatTask::run() { - SMLogging* logger = SMLogging::get(); - int success; - uint8_t buf[1024] = {0}; - - if (getLength() > 1023) { - handleError("StatTask read", ENAMETOOLONG); - return true; - } - - success = read(buf, getLength()); - check_error("StatTask read", false); - stat_cmd *cmd = (stat_cmd *) buf; - sm_response *resp = (sm_response *) buf; - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"stat %s.",cmd->filename); - #endif - int err; - try - { - err = ioc->stat(cmd->filename, (struct stat *) resp->payload); - } - catch (exception &e) - { - logger->log(LOG_ERR, "StatTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - - resp->returnCode = err; - uint payloadLen; - if (!err) - payloadLen = sizeof(struct stat); - else { - payloadLen = 4; - *((int32_t *) resp->payload) = errno; - } - return write(*resp, payloadLen); -} + SMLogging* logger = SMLogging::get(); + int success; + uint8_t buf[1024] = {0}; + if (getLength() > 1023) + { + handleError("StatTask read", ENAMETOOLONG); + return true; + } + + success = read(buf, getLength()); + check_error("StatTask read", false); + stat_cmd* cmd = (stat_cmd*)buf; + sm_response* resp = (sm_response*)buf; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "stat %s.", cmd->filename); +#endif + int err; + try + { + err = ioc->stat(cmd->filename, (struct stat*)resp->payload); + } + catch (exception& e) + { + logger->log(LOG_ERR, "StatTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + + resp->returnCode = err; + uint payloadLen; + if (!err) + payloadLen = sizeof(struct stat); + else + { + payloadLen = 4; + *((int32_t*)resp->payload) = errno; + } + return write(*resp, payloadLen); } +} // namespace storagemanager diff --git a/storage-manager/src/StatTask.h b/storage-manager/src/StatTask.h index 1920d07dd..679dfbe93 100644 --- a/storage-manager/src/StatTask.h +++ b/storage-manager/src/StatTask.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef STATTASK_H_ #define STATTASK_H_ @@ -23,19 +22,17 @@ namespace storagemanager { - class StatTask : public PosixTask { - public: - StatTask(int sock, uint length); - virtual ~StatTask(); - - bool run(); - - private: - StatTask(); + public: + StatTask(int sock, uint length); + virtual ~StatTask(); + + bool run(); + + private: + StatTask(); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/SyncTask.cpp b/storage-manager/src/SyncTask.cpp index 4415822e1..b669d12d8 100644 --- a/storage-manager/src/SyncTask.cpp +++ b/storage-manager/src/SyncTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "SyncTask.h" #include "Synchronizer.h" #include "messageFormat.h" @@ -23,7 +22,6 @@ namespace storagemanager { - SyncTask::SyncTask(int sock, uint len) : PosixTask(sock, len) { } @@ -34,29 +32,29 @@ SyncTask::~SyncTask() bool SyncTask::run() { - // force flush synchronizer - uint8_t buf; + // force flush synchronizer + uint8_t buf; - if (getLength() > 1) - { - handleError("SyncTask", E2BIG); - return true; - } - // consume the msg - int success = read(&buf, getLength()); - if (success<0) - { - handleError("SyncTask", errno); - return false; - } + if (getLength() > 1) + { + handleError("SyncTask", E2BIG); + return true; + } + // consume the msg + int success = read(&buf, getLength()); + if (success < 0) + { + handleError("SyncTask", errno); + return false; + } - Synchronizer::get()->syncNow(); + Synchronizer::get()->syncNow(); - // send generic success response - sm_response ret; - ret.returnCode = 0; - success = write(ret, 0); - return success; + // send generic success response + sm_response ret; + ret.returnCode = 0; + success = write(ret, 0); + return success; } -} +} // namespace storagemanager diff --git a/storage-manager/src/SyncTask.h b/storage-manager/src/SyncTask.h index c6b3a4abe..184c98ff9 100644 --- a/storage-manager/src/SyncTask.h +++ b/storage-manager/src/SyncTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef SYNCTASK_H_ #define SYNCTASK_H_ @@ -24,18 +22,17 @@ namespace storagemanager { - class SyncTask : public PosixTask { - public: - SyncTask(int sock, uint length); - virtual ~SyncTask(); + public: + SyncTask(int sock, uint length); + virtual ~SyncTask(); - bool run(); + bool run(); - private: - SyncTask(); + private: + SyncTask(); }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 5a510f7cc..ed02f5eb1 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "Synchronizer.h" #include "Cache.h" #include "IOCoordinator.h" @@ -31,791 +30,819 @@ using namespace std; namespace { - storagemanager::Synchronizer *instance = NULL; - boost::mutex inst_mutex; -} +storagemanager::Synchronizer* instance = NULL; +boost::mutex inst_mutex; +} // namespace namespace bf = boost::filesystem; namespace storagemanager { - -Synchronizer * Synchronizer::get() +Synchronizer* Synchronizer::get() { - if (instance) - return instance; - boost::unique_lock lock(inst_mutex); - if (instance) - return instance; - instance = new Synchronizer(); + if (instance) return instance; + boost::unique_lock lock(inst_mutex); + if (instance) + return instance; + instance = new Synchronizer(); + return instance; } Synchronizer::Synchronizer() : maxUploads(0) { - Config *config = Config::get(); - logger = SMLogging::get(); - cache = Cache::get(); - replicator = Replicator::get(); - ioc = IOCoordinator::get(); - cs = CloudStorage::get(); - - numBytesRead = numBytesWritten = numBytesUploaded = numBytesDownloaded = mergeDiff = - flushesTriggeredBySize = flushesTriggeredByTimer = journalsMerged = - objectsSyncedWithNoJournal = bytesReadBySync = bytesReadBySyncWithJournal = 0; + Config* config = Config::get(); + logger = SMLogging::get(); + cache = Cache::get(); + replicator = Replicator::get(); + ioc = IOCoordinator::get(); + cs = CloudStorage::get(); - journalPath = cache->getJournalPath(); - cachePath = cache->getCachePath(); - threadPool.reset(new ThreadPool()); - configListener(); - config->addConfigListener(this); - die = false; - journalSizeThreshold = cache->getMaxCacheSize() / 2; - blockNewJobs = false; - syncThread = boost::thread([this] () { this->periodicSync(); }); + numBytesRead = numBytesWritten = numBytesUploaded = numBytesDownloaded = mergeDiff = + flushesTriggeredBySize = flushesTriggeredByTimer = journalsMerged = objectsSyncedWithNoJournal = + bytesReadBySync = bytesReadBySyncWithJournal = 0; + + journalPath = cache->getJournalPath(); + cachePath = cache->getCachePath(); + threadPool.reset(new ThreadPool()); + configListener(); + config->addConfigListener(this); + die = false; + journalSizeThreshold = cache->getMaxCacheSize() / 2; + blockNewJobs = false; + syncThread = boost::thread([this]() { this->periodicSync(); }); } Synchronizer::~Synchronizer() { - /* should this wait until all pending work is done, - or save the list it's working on..... - For milestone 2, this will do the safe thing and finish working first. - Later we can get fancy. */ - Config::get()->removeConfigListener(this); - forceFlush(); - die = true; - syncThread.join(); - threadPool.reset(); + /* should this wait until all pending work is done, + or save the list it's working on..... + For milestone 2, this will do the safe thing and finish working first. + Later we can get fancy. */ + Config::get()->removeConfigListener(this); + forceFlush(); + die = true; + syncThread.join(); + threadPool.reset(); } enum OpFlags { - NOOP = 0, - JOURNAL = 0x1, - DELETE = 0x2, - NEW_OBJECT = 0x4, + NOOP = 0, + JOURNAL = 0x1, + DELETE = 0x2, + NEW_OBJECT = 0x4, }; /* XXXPAT. Need to revisit this later. To make multiple prefix functionality as minimal as possible, -I limited the changes to key string manipulation where possible. The keys it manages have the prefix they +I limited the changes to key string manipulation where possible. The keys it manages have the prefix they belong to prepended. So key 12345 in prefix p1 becomes p1/12345. This is not the most elegant or performant option, just the least invasive. */ -void Synchronizer::newPrefix(const bf::path &p) +void Synchronizer::newPrefix(const bf::path& p) { - uncommittedJournalSize[p] = 0; + uncommittedJournalSize[p] = 0; } -void Synchronizer::dropPrefix(const bf::path &p) +void Synchronizer::dropPrefix(const bf::path& p) { - syncNow(p); - boost::unique_lock s(mutex); - uncommittedJournalSize.erase(p); + syncNow(p); + boost::unique_lock s(mutex); + uncommittedJournalSize.erase(p); } -void Synchronizer::_newJournalEntry(const bf::path &prefix, const string &_key, size_t size) +void Synchronizer::_newJournalEntry(const bf::path& prefix, const string& _key, size_t size) { - string key = (prefix/_key).string(); - uncommittedJournalSize[prefix] += size; - auto it = pendingOps.find(key); + string key = (prefix / _key).string(); + uncommittedJournalSize[prefix] += size; + auto it = pendingOps.find(key); + if (it != pendingOps.end()) + { + it->second->opFlags |= JOURNAL; + return; + } + // makeJob(key); + pendingOps[key] = boost::shared_ptr(new PendingOps(JOURNAL)); +} + +void Synchronizer::newJournalEntry(const bf::path& prefix, const string& _key, size_t size) +{ + boost::unique_lock s(mutex); + _newJournalEntry(prefix, _key, size); + if (uncommittedJournalSize[prefix] > journalSizeThreshold) + { + uncommittedJournalSize[prefix] = 0; + s.unlock(); + forceFlush(); + } +} + +void Synchronizer::newJournalEntries(const bf::path& prefix, const vector >& keys) +{ + boost::unique_lock s(mutex); + for (auto& keysize : keys) + _newJournalEntry(prefix, keysize.first, keysize.second); + if (uncommittedJournalSize[prefix] > journalSizeThreshold) + { + uncommittedJournalSize[prefix] = 0; + s.unlock(); + forceFlush(); + } +} + +void Synchronizer::newObjects(const bf::path& prefix, const vector& keys) +{ + boost::unique_lock s(mutex); + + for (const string& _key : keys) + { + bf::path key(prefix / _key); + assert(pendingOps.find(key.string()) == pendingOps.end()); + // makeJob(key); + pendingOps[key.string()] = boost::shared_ptr(new PendingOps(NEW_OBJECT)); + } +} + +void Synchronizer::deletedObjects(const bf::path& prefix, const vector& keys) +{ + boost::unique_lock s(mutex); + + for (const string& _key : keys) + { + bf::path key(prefix / _key); + auto it = pendingOps.find(key.string()); if (it != pendingOps.end()) - { - it->second->opFlags |= JOURNAL; - return; - } - //makeJob(key); - pendingOps[key] = boost::shared_ptr(new PendingOps(JOURNAL)); -} - -void Synchronizer::newJournalEntry(const bf::path &prefix, const string &_key, size_t size) -{ - boost::unique_lock s(mutex); - _newJournalEntry(prefix, _key, size); - if (uncommittedJournalSize[prefix] > journalSizeThreshold) - { - uncommittedJournalSize[prefix] = 0; - s.unlock(); - forceFlush(); - } + it->second->opFlags |= DELETE; + else + pendingOps[key.string()] = boost::shared_ptr(new PendingOps(DELETE)); + } + // would be good to signal to the things in opsInProgress that these were deleted. That would + // quiet down the logging somewhat. How to do that efficiently, and w/o gaps or deadlock... } -void Synchronizer::newJournalEntries(const bf::path &prefix, const vector > &keys) +void Synchronizer::flushObject(const bf::path& prefix, const string& _key) { - boost::unique_lock s(mutex); - for (auto &keysize : keys) - _newJournalEntry(prefix, keysize.first, keysize.second); - if (uncommittedJournalSize[prefix] > journalSizeThreshold) + string key = (prefix / _key).string(); + + while (blockNewJobs) + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + + boost::unique_lock s(mutex); + + // if there is something to do on key, it should be either in pendingOps or opsInProgress + // if it is is pending ops, start the job now. If it is in progress, wait for it to finish. + // If there are no jobs to do on key, it will verify it exists in cloud storage. + // The existence check is currently necessary on a startup where the cache is populated but + // synchronizer isn't. + + bool noExistingJob = false; + auto it = pendingOps.find(key); + if (it != pendingOps.end()) + { + objNames.push_front(key); + auto nameIt = objNames.begin(); + s.unlock(); + process(nameIt); + s.lock(); + } + else + { + auto op = opsInProgress.find(key); + // it's already in progress + if (op != opsInProgress.end()) { - uncommittedJournalSize[prefix] = 0; - s.unlock(); - forceFlush(); - } -} - -void Synchronizer::newObjects(const bf::path &prefix, const vector &keys) -{ - boost::unique_lock s(mutex); - - for (const string &_key : keys) - { - bf::path key(prefix/_key); - assert(pendingOps.find(key.string()) == pendingOps.end()); - //makeJob(key); - pendingOps[key.string()] = boost::shared_ptr(new PendingOps(NEW_OBJECT)); - } -} - -void Synchronizer::deletedObjects(const bf::path &prefix, const vector &keys) -{ - boost::unique_lock s(mutex); - - for (const string &_key : keys) - { - bf::path key(prefix/_key); - auto it = pendingOps.find(key.string()); - if (it != pendingOps.end()) - it->second->opFlags |= DELETE; - else - pendingOps[key.string()] = boost::shared_ptr(new PendingOps(DELETE)); - } - // would be good to signal to the things in opsInProgress that these were deleted. That would - // quiet down the logging somewhat. How to do that efficiently, and w/o gaps or deadlock... -} - -void Synchronizer::flushObject(const bf::path &prefix, const string &_key) -{ - string key = (prefix/_key).string(); - - while (blockNewJobs) - boost::this_thread::sleep_for(boost::chrono::seconds(1)); - - boost::unique_lock s(mutex); - - // if there is something to do on key, it should be either in pendingOps or opsInProgress - // if it is is pending ops, start the job now. If it is in progress, wait for it to finish. - // If there are no jobs to do on key, it will verify it exists in cloud storage. - // The existence check is currently necessary on a startup where the cache is populated but - // synchronizer isn't. - - bool noExistingJob = false; - auto it = pendingOps.find(key); - if (it != pendingOps.end()) - { - objNames.push_front(key); - auto nameIt = objNames.begin(); - s.unlock(); - process(nameIt); - s.lock(); + boost::shared_ptr tmp = op->second; + tmp->wait(&mutex); } else { - auto op = opsInProgress.find(key); - // it's already in progress - if (op != opsInProgress.end()) { - boost::shared_ptr tmp = op->second; - tmp->wait(&mutex); - } - else - { - // it's not in either one, trigger existence check - noExistingJob = true; - } + // it's not in either one, trigger existence check + noExistingJob = true; } - - if (!noExistingJob) - return; - - // check whether this key is in cloud storage - bool keyExists, journalExists; - int err; - do { - err = cs->exists(_key.c_str(), &keyExists); - if (err) - { - char buf[80]; - logger->log(LOG_CRIT, "Sync::flushObject(): cloud existence check failed, got '%s'", strerror_r(errno, buf, 80)); - sleep(5); - } - } while (err); - journalExists = bf::exists(journalPath/(key + ".journal")); - - if (journalExists) + } + + if (!noExistingJob) + return; + + // check whether this key is in cloud storage + bool keyExists, journalExists; + int err; + do + { + err = cs->exists(_key.c_str(), &keyExists); + if (err) { - logger->log(LOG_DEBUG, "Sync::flushObject(): %s has a journal, " - "and there is no job for it. Merging & uploading now.", key.c_str()); - pendingOps[key] = boost::shared_ptr(new PendingOps(JOURNAL)); - objNames.push_front(key); - auto nameIt = objNames.begin(); - s.unlock(); - process(nameIt); - } - else if (!keyExists) - { - logger->log(LOG_DEBUG, "Sync::flushObject(): %s does not exist in cloud storage, " - "and there is no job for it. Uploading it now.", key.c_str()); - pendingOps[key] = boost::shared_ptr(new PendingOps(NEW_OBJECT)); - objNames.push_front(key); - auto nameIt = objNames.begin(); - s.unlock(); - process(nameIt); + char buf[80]; + logger->log(LOG_CRIT, "Sync::flushObject(): cloud existence check failed, got '%s'", + strerror_r(errno, buf, 80)); + sleep(5); } + } while (err); + journalExists = bf::exists(journalPath / (key + ".journal")); + + if (journalExists) + { + logger->log(LOG_DEBUG, + "Sync::flushObject(): %s has a journal, " + "and there is no job for it. Merging & uploading now.", + key.c_str()); + pendingOps[key] = boost::shared_ptr(new PendingOps(JOURNAL)); + objNames.push_front(key); + auto nameIt = objNames.begin(); + s.unlock(); + process(nameIt); + } + else if (!keyExists) + { + logger->log(LOG_DEBUG, + "Sync::flushObject(): %s does not exist in cloud storage, " + "and there is no job for it. Uploading it now.", + key.c_str()); + pendingOps[key] = boost::shared_ptr(new PendingOps(NEW_OBJECT)); + objNames.push_front(key); + auto nameIt = objNames.begin(); + s.unlock(); + process(nameIt); + } } void Synchronizer::periodicSync() { - boost::unique_lock lock(mutex); - while (!die) + boost::unique_lock lock(mutex); + while (!die) + { + lock.unlock(); + bool wasTriggeredBySize = false; + try { - lock.unlock(); - bool wasTriggeredBySize = false; - try - { - boost::this_thread::sleep_for(syncInterval); - } - catch (const boost::thread_interrupted) - { - //logger->log(LOG_DEBUG,"Synchronizer Force Flush."); - wasTriggeredBySize = true; - } - lock.lock(); - if (blockNewJobs) - continue; - if (!pendingOps.empty()) - { - if (wasTriggeredBySize) - ++flushesTriggeredBySize; - else - ++flushesTriggeredByTimer; - } - //cout << "Sync'ing " << pendingOps.size() << " objects" << " queue size is " << - // threadPool.currentQueueSize() << endl; - for (auto &job : pendingOps) - makeJob(job.first); - for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) - it->second = 0; + boost::this_thread::sleep_for(syncInterval); } + catch (const boost::thread_interrupted) + { + // logger->log(LOG_DEBUG,"Synchronizer Force Flush."); + wasTriggeredBySize = true; + } + lock.lock(); + if (blockNewJobs) + continue; + if (!pendingOps.empty()) + { + if (wasTriggeredBySize) + ++flushesTriggeredBySize; + else + ++flushesTriggeredByTimer; + } + // cout << "Sync'ing " << pendingOps.size() << " objects" << " queue size is " << + // threadPool.currentQueueSize() << endl; + for (auto& job : pendingOps) + makeJob(job.first); + for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) + it->second = 0; + } } -void Synchronizer::syncNow(const bf::path &prefix) +void Synchronizer::syncNow(const bf::path& prefix) { - boost::unique_lock lock(mutex); + boost::unique_lock lock(mutex); - // This should ensure that all pendingOps have been added as jobs - // and waits for them to complete. until pendingOps is empty. - // this should be redone to only remove items of given prefix eventually + // This should ensure that all pendingOps have been added as jobs + // and waits for them to complete. until pendingOps is empty. + // this should be redone to only remove items of given prefix eventually - blockNewJobs = true; - while (pendingOps.size() != 0 || opsInProgress.size() != 0) - { - for (auto &job : pendingOps) - makeJob(job.first); - for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) - it->second = 0; - lock.unlock(); - while (opsInProgress.size() > 0) - boost::this_thread::sleep_for(boost::chrono::seconds(1)); - lock.lock(); - } - blockNewJobs = false; + blockNewJobs = true; + while (pendingOps.size() != 0 || opsInProgress.size() != 0) + { + for (auto& job : pendingOps) + makeJob(job.first); + for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) + it->second = 0; + lock.unlock(); + while (opsInProgress.size() > 0) + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + lock.lock(); + } + blockNewJobs = false; } void Synchronizer::syncNow() { - boost::unique_lock lock(mutex); + boost::unique_lock lock(mutex); - // This should ensure that all pendingOps have been added as jobs - // and waits for them to complete. until pendingOps is empty. - // Used by the mcsadmin command suspendDatabaseWrites. - // Leaving S3 storage and local metadata directories sync'd for snapshot backups. + // This should ensure that all pendingOps have been added as jobs + // and waits for them to complete. until pendingOps is empty. + // Used by the mcsadmin command suspendDatabaseWrites. + // Leaving S3 storage and local metadata directories sync'd for snapshot backups. - blockNewJobs = true; - while (pendingOps.size() != 0 || opsInProgress.size() != 0) - { - for (auto &job : pendingOps) - makeJob(job.first); - for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) - it->second = 0; - lock.unlock(); - while (opsInProgress.size() > 0) - boost::this_thread::sleep_for(boost::chrono::seconds(1)); - lock.lock(); - } - blockNewJobs = false; + blockNewJobs = true; + while (pendingOps.size() != 0 || opsInProgress.size() != 0) + { + for (auto& job : pendingOps) + makeJob(job.first); + for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) + it->second = 0; + lock.unlock(); + while (opsInProgress.size() > 0) + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + lock.lock(); + } + blockNewJobs = false; } - void Synchronizer::forceFlush() { - boost::unique_lock lock(mutex); + boost::unique_lock lock(mutex); - syncThread.interrupt(); + syncThread.interrupt(); } -void Synchronizer::makeJob(const string &key) +void Synchronizer::makeJob(const string& key) { - objNames.push_front(key); - - boost::shared_ptr j(new Job(this, objNames.begin())); - threadPool->addJob(j); + objNames.push_front(key); + + boost::shared_ptr j(new Job(this, objNames.begin())); + threadPool->addJob(j); } void Synchronizer::process(list::iterator name) { - /* - check if there is a pendingOp for name - if yes, start processing it - if no, - check if there is an ongoing op and block on it - if not, return - */ + /* + check if there is a pendingOp for name + if yes, start processing it + if no, + check if there is an ongoing op and block on it + if not, return + */ - boost::unique_lock s(mutex); - - string &key = *name; - auto it = pendingOps.find(key); - if (it == pendingOps.end()) - { - auto op = opsInProgress.find(key); - // it's already in progress - if (op != opsInProgress.end()) - { - boost::shared_ptr tmp = op->second; - tmp->wait(&mutex); - objNames.erase(name); - return; - } - else - { - // it's not in pending or opsinprogress, nothing to do - objNames.erase(name); - return; - } - } - - boost::shared_ptr pending = it->second; - bool inserted = opsInProgress.insert(*it).second; - if (!inserted) - { - objNames.erase(name); - return; // the one in pending will have to wait until the next time to avoid clobbering waiting threads - } - - // Because of the ownership thing and the odd set of changes that it required, - // we need to strip the prefix from key. - size_t first_slash_pos = key.find_first_of('/'); - string realKey = key.substr(first_slash_pos + 1); - string sourceFile = MetadataFile::getSourceFromKey(realKey); - pendingOps.erase(it); - s.unlock(); - - bool success = false; - int retryCount = 0; - while (!success) - { - assert(!s.owns_lock()); - try { - // Exceptions should only happen b/c of cloud service errors that can't be retried. - // This code is intentionally racy to avoid having to grab big locks. - // In particular, it's possible that by the time synchronize() runs, - // the file to sync has already been deleted. When one of these functions - // encounters a state that doesn't make sense, such as being told to upload a file - // that doesn't exist, it will return successfully under the assumption that - // things are working as they should upstream, and a syncDelete() call will be coming - // shortly. - if (pending->opFlags & DELETE) - synchronizeDelete(sourceFile, name); - else if (pending->opFlags & JOURNAL) - synchronizeWithJournal(sourceFile, name); - else if (pending->opFlags & NEW_OBJECT) - synchronize(sourceFile, name); - else - throw logic_error("Synchronizer::process(): got an unknown op flag"); - s.lock(); - pending->notify(); - success = true; - } - catch(exception &e) { - // these are often self-resolving, so we will suppress logging it for 10 iterations, then escalate - // to error, then to crit - //if (++retryCount >= 10) - logger->log((retryCount < 20 ? LOG_ERR : LOG_CRIT), "Synchronizer::process(): error sync'ing %s opFlags=%d, got '%s'. Retrying...", key.c_str(), - pending->opFlags, e.what()); - success = false; - sleep(1); - continue; - /* TODO: Need to think this about this requeue logic again. The potential problem is that - there may be threads waiting for this job to finish. If the insert doesn't happen because - there is already a job in pendingOps for the same file, then the threads waiting on this - job never get woken, right?? Or, can that never happen for some reason? - */ - s.lock(); - auto inserted = pendingOps.insert(pair >(key, pending)); - if (!inserted.second) - inserted.first->second->opFlags |= pending->opFlags; - opsInProgress.erase(key); - makeJob(key); - objNames.erase(name); - return; - } - } - - opsInProgress.erase(*name); - objNames.erase(name); -} + boost::unique_lock s(mutex); -void Synchronizer::synchronize(const string &sourceFile, list::iterator &it) -{ - ScopedReadLock s(ioc, sourceFile); - - string key = *it; - size_t pos = key.find_first_of('/'); - bf::path prefix = key.substr(0, pos); - string cloudKey = key.substr(pos + 1); - char buf[80]; - bool exists = false; - int err; - bf::path objectPath = cachePath/key; - MetadataFile md(sourceFile, MetadataFile::no_create_t(),true); - - if (!md.exists()) + string& key = *name; + auto it = pendingOps.find(key); + if (it == pendingOps.end()) + { + auto op = opsInProgress.find(key); + // it's already in progress + if (op != opsInProgress.end()) { - logger->log(LOG_DEBUG, "synchronize(): no metadata found for %s. It must have been deleted.", sourceFile.c_str()); - try - { - if (!bf::exists(objectPath)) - return; - size_t size = bf::file_size(objectPath); - replicator->remove(objectPath); - cache->deletedObject(prefix, cloudKey, size); - cs->deleteObject(cloudKey); - } - catch (exception &e) - { - logger->log(LOG_DEBUG, "synchronize(): failed to remove orphaned object '%s' from the cache, got %s", - objectPath.string().c_str(), e.what()); - } - return; - } - - metadataObject mdEntry; - bool entryExists = md.getEntry(MetadataFile::getOffsetFromKey(cloudKey), &mdEntry); - if (!entryExists || cloudKey != mdEntry.key) - { - logger->log(LOG_DEBUG, "synchronize(): %s does not exist in metadata for %s. This suggests truncation.", key.c_str(), sourceFile.c_str()); - return; - } - - //assert(key == mdEntry->key); <-- This could fail b/c of truncation + a write/append before this job runs. - - err = cs->exists(cloudKey, &exists); - if (err) - throw runtime_error(string("synchronize(): checking existence of ") + key + ", got " + - strerror_r(errno, buf, 80)); - if (exists) - return; - - exists = cache->exists(prefix, cloudKey); - if (!exists) - { - logger->log(LOG_DEBUG, "synchronize(): was told to upload %s but it does not exist locally", key.c_str()); - return; - } - - err = cs->putObject(objectPath.string(), cloudKey); - if (err) - throw runtime_error(string("synchronize(): uploading ") + key + ", got " + strerror_r(errno, buf, 80)); - - numBytesRead += mdEntry.length; - bytesReadBySync += mdEntry.length; - numBytesUploaded += mdEntry.length; - ++objectsSyncedWithNoJournal; - replicator->remove(objectPath, Replicator::NO_LOCAL); -} - -void Synchronizer::synchronizeDelete(const string &sourceFile, list::iterator &it) -{ - ScopedWriteLock s(ioc, sourceFile); - string cloudKey = it->substr(it->find('/') + 1); - cs->deleteObject(cloudKey); -} - -void Synchronizer::synchronizeWithJournal(const string &sourceFile, list::iterator &lit) -{ - ScopedWriteLock s(ioc, sourceFile); - - char buf[80]; - string key = *lit; - size_t pos = key.find_first_of('/'); - bf::path prefix = key.substr(0, pos); - string cloudKey = key.substr(pos + 1); - - MetadataFile md(sourceFile, MetadataFile::no_create_t(),true); - - if (!md.exists()) - { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): no metadata found for %s. It must have been deleted.", sourceFile.c_str()); - try - { - bf::path objectPath = cachePath/key; - if (bf::exists(objectPath)) - { - size_t objSize = bf::file_size(objectPath); - replicator->remove(objectPath); - cache->deletedObject(prefix, cloudKey, objSize); - cs->deleteObject(cloudKey); - } - bf::path jPath = journalPath/(key + ".journal"); - if (bf::exists(jPath)) - { - size_t jSize = bf::file_size(jPath); - replicator->remove(jPath); - cache->deletedJournal(prefix, jSize); - } - } - catch(exception &e) - { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): failed to remove orphaned object '%s' from the cache, got %s", - (cachePath/key).string().c_str(), e.what()); - } - return; - } - - metadataObject mdEntry; - bool metaExists = md.getEntry(MetadataFile::getOffsetFromKey(cloudKey), &mdEntry); - if (!metaExists || cloudKey != mdEntry.key) - { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): %s does not exist in metadata for %s. This suggests truncation.", key.c_str(), sourceFile.c_str()); - return; - } - //assert(key == mdEntry->key); <--- I suspect this can happen in a truncate + write situation + a deep sync queue - - bf::path oldCachePath = cachePath / key; - string journalName = (journalPath/(key + ".journal")).string(); - - if (!bf::exists(journalName)) - { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): no journal file found for %s", key.c_str()); - - // sanity check + add'l info. Test whether the object exists in cloud storage. If so, complain, - // and run synchronize() instead. - bool existsOnCloud; - int err = cs->exists(cloudKey, &existsOnCloud); - if (err) - throw runtime_error(string("Synchronizer: cs->exists() failed: ") + strerror_r(errno, buf, 80)); - if (!existsOnCloud) - { - if (cache->exists(prefix, cloudKey)) - { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): %s has no journal and does not exist in the cloud, calling " - "synchronize() instead. Need to explain how this happens.", key.c_str()); - s.unlock(); - synchronize(sourceFile, lit); - } - else - logger->log(LOG_DEBUG, "synchronizeWithJournal(): %s has no journal, and does not exist in the cloud or in " - " the local cache. Need to explain how this happens.", key.c_str()); - return; - } - else - logger->log(LOG_DEBUG, "synchronizeWithJournal(): %s has no journal, but it does exist in the cloud. " - " This indicates that an overlapping syncWithJournal() call handled it properly.", key.c_str()); - - return; - } - - int err; - boost::shared_array data; - size_t count = 0, size = mdEntry.length, originalSize = 0; - - bool oldObjIsCached = cache->exists(prefix, cloudKey); - - // get the base object if it is not already cached - // merge it with its journal file - if (!oldObjIsCached) - { - err = cs->getObject(cloudKey, &data, &size); - if (err) - { - if (errno == ENOENT) - { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): %s does not exist in cache nor in cloud storage", key.c_str()); - return; - } - throw runtime_error(string("Synchronizer: getObject() failed: ") + strerror_r(errno, buf, 80)); - } - - numBytesDownloaded += size; - originalSize += size; - - //TODO!! This sucks. Need a way to pass in a larger array to cloud storage, and also have it not - // do any add'l alloc'ing or copying - if (size < mdEntry.length) - { - boost::shared_array tmp(new uint8_t[mdEntry.length]()); - memcpy(tmp.get(), data.get(), size); - memset(&tmp[size], 0, mdEntry.length - size); - data.swap(tmp); - } - size = mdEntry.length; // reset length. Using the metadata as a source of truth (truncation), not actual file length. - - size_t _bytesRead = 0; - err = ioc->mergeJournalInMem(data, size, journalName.c_str(), &_bytesRead); - if (err) - { - if (!bf::exists(journalName)) - logger->log(LOG_DEBUG, "synchronizeWithJournal(): journal %s was deleted mid-operation, check locking", - journalName.c_str()); - else - logger->log(LOG_ERR, "synchronizeWithJournal(): unexpected error merging journal for %s", key.c_str()); - return; - } - numBytesRead += _bytesRead; - bytesReadBySyncWithJournal += _bytesRead; - originalSize += _bytesRead; + boost::shared_ptr tmp = op->second; + tmp->wait(&mutex); + objNames.erase(name); + return; } else { - size_t _bytesRead = 0; - data = ioc->mergeJournal(oldCachePath.string().c_str(), journalName.c_str(), 0, size, &_bytesRead); - if (!data) - { - if (!bf::exists(journalName)) - logger->log(LOG_DEBUG, "synchronizeWithJournal(): journal %s was deleted mid-operation, check locking", - journalName.c_str()); - else - logger->log(LOG_ERR, "synchronizeWithJournal(): unexpected error merging journal for %s", key.c_str()); - return; - } - numBytesRead += _bytesRead; - bytesReadBySyncWithJournal += _bytesRead; - originalSize = _bytesRead; + // it's not in pending or opsinprogress, nothing to do + objNames.erase(name); + return; } - - // original size here should be == objectsize + journalsize - - // get a new key for the resolved version & upload it - string newCloudKey = MetadataFile::getNewKeyFromOldKey(cloudKey, size); - string newKey = (prefix/newCloudKey).string(); - err = cs->putObject(data, size, newCloudKey); - if (err) - { - // try to delete it in cloud storage... unlikely it is there in the first place, and if it is - // this probably won't work - int l_errno = errno; - cs->deleteObject(newCloudKey); - throw runtime_error(string("Synchronizer: putObject() failed: ") + strerror_r(l_errno, buf, 80)); - } - numBytesUploaded += size; - - // if the object was cached... - // write the new data to disk, - // tell the cache about the rename - // rename the file in any pending ops in Synchronizer - - if (oldObjIsCached) - { - // Is this the only thing outside of Replicator that writes files? - // If so move this write loop to Replicator. - bf::path newCachePath = cachePath / newKey; - int newFD = ::open(newCachePath.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (newFD < 0) - throw runtime_error(string("Synchronizer: Failed to open a new object in local storage! Got ") - + strerror_r(errno, buf, 80)); - ScopedCloser scloser(newFD); - - while (count < size) - { - err = ::write(newFD, &data[count], size - count); - if (err < 0) - { - ::unlink(newCachePath.string().c_str()); - throw runtime_error(string("Synchronizer: Failed to write to a new object in local storage! Got ") - + strerror_r(errno, buf, 80)); - } - count += err; - } - numBytesWritten += size; - - size_t oldSize = bf::file_size(oldCachePath); - - cache->rename(prefix, cloudKey, newCloudKey, size - oldSize); - replicator->remove(oldCachePath); - - // This condition is probably irrelevant for correct functioning now, - // but it should be very rare so what the hell. - if (oldSize != MetadataFile::getLengthFromKey(cloudKey)) - { - ostringstream oss; - oss << "Synchronizer::synchronizeWithJournal(): detected a mismatch between file size and " << - "length stored in the object name. object name = " << cloudKey << " length-in-name = " << - MetadataFile::getLengthFromKey(cloudKey) << " real-length = " << oldSize; - logger->log(LOG_WARNING, oss.str().c_str()); - } - } - - mergeDiff += size - originalSize; - ++journalsMerged; - // update the metadata for the source file - - md.updateEntry(MetadataFile::getOffsetFromKey(cloudKey), newCloudKey, size); - replicator->updateMetadata(md); + } - rename(key, newKey); - - // delete the old object & journal file - cache->deletedJournal(prefix, bf::file_size(journalName)); - replicator->remove(journalName); - cs->deleteObject(cloudKey); + boost::shared_ptr pending = it->second; + bool inserted = opsInProgress.insert(*it).second; + if (!inserted) + { + objNames.erase(name); + return; // the one in pending will have to wait until the next time to avoid clobbering waiting threads + } + + // Because of the ownership thing and the odd set of changes that it required, + // we need to strip the prefix from key. + size_t first_slash_pos = key.find_first_of('/'); + string realKey = key.substr(first_slash_pos + 1); + string sourceFile = MetadataFile::getSourceFromKey(realKey); + pendingOps.erase(it); + s.unlock(); + + bool success = false; + int retryCount = 0; + while (!success) + { + assert(!s.owns_lock()); + try + { + // Exceptions should only happen b/c of cloud service errors that can't be retried. + // This code is intentionally racy to avoid having to grab big locks. + // In particular, it's possible that by the time synchronize() runs, + // the file to sync has already been deleted. When one of these functions + // encounters a state that doesn't make sense, such as being told to upload a file + // that doesn't exist, it will return successfully under the assumption that + // things are working as they should upstream, and a syncDelete() call will be coming + // shortly. + if (pending->opFlags & DELETE) + synchronizeDelete(sourceFile, name); + else if (pending->opFlags & JOURNAL) + synchronizeWithJournal(sourceFile, name); + else if (pending->opFlags & NEW_OBJECT) + synchronize(sourceFile, name); + else + throw logic_error("Synchronizer::process(): got an unknown op flag"); + s.lock(); + pending->notify(); + success = true; + } + catch (exception& e) + { + // these are often self-resolving, so we will suppress logging it for 10 iterations, then escalate + // to error, then to crit + // if (++retryCount >= 10) + logger->log((retryCount < 20 ? LOG_ERR : LOG_CRIT), + "Synchronizer::process(): error sync'ing %s opFlags=%d, got '%s'. Retrying...", + key.c_str(), pending->opFlags, e.what()); + success = false; + sleep(1); + continue; + /* TODO: Need to think this about this requeue logic again. The potential problem is that + there may be threads waiting for this job to finish. If the insert doesn't happen because + there is already a job in pendingOps for the same file, then the threads waiting on this + job never get woken, right?? Or, can that never happen for some reason? + */ + s.lock(); + auto inserted = pendingOps.insert(pair >(key, pending)); + if (!inserted.second) + inserted.first->second->opFlags |= pending->opFlags; + opsInProgress.erase(key); + makeJob(key); + objNames.erase(name); + return; + } + } + + opsInProgress.erase(*name); + objNames.erase(name); } -void Synchronizer::rename(const string &oldKey, const string &newKey) +void Synchronizer::synchronize(const string& sourceFile, list::iterator& it) { - boost::unique_lock s(mutex); + ScopedReadLock s(ioc, sourceFile); - auto it = pendingOps.find(oldKey); - if (it != pendingOps.end()) + string key = *it; + size_t pos = key.find_first_of('/'); + bf::path prefix = key.substr(0, pos); + string cloudKey = key.substr(pos + 1); + char buf[80]; + bool exists = false; + int err; + bf::path objectPath = cachePath / key; + MetadataFile md(sourceFile, MetadataFile::no_create_t(), true); + + if (!md.exists()) + { + logger->log(LOG_DEBUG, "synchronize(): no metadata found for %s. It must have been deleted.", + sourceFile.c_str()); + try { - pendingOps[newKey] = it->second; - pendingOps.erase(it); + if (!bf::exists(objectPath)) + return; + size_t size = bf::file_size(objectPath); + replicator->remove(objectPath); + cache->deletedObject(prefix, cloudKey, size); + cs->deleteObject(cloudKey); } - it = opsInProgress.find(oldKey); - if (it != opsInProgress.end()) + catch (exception& e) { - opsInProgress[newKey] = it->second; - opsInProgress.erase(it); + logger->log(LOG_DEBUG, "synchronize(): failed to remove orphaned object '%s' from the cache, got %s", + objectPath.string().c_str(), e.what()); } - - for (auto &name: objNames) - if (name == oldKey) - name = newKey; + return; + } + + metadataObject mdEntry; + bool entryExists = md.getEntry(MetadataFile::getOffsetFromKey(cloudKey), &mdEntry); + if (!entryExists || cloudKey != mdEntry.key) + { + logger->log(LOG_DEBUG, "synchronize(): %s does not exist in metadata for %s. This suggests truncation.", + key.c_str(), sourceFile.c_str()); + return; + } + + // assert(key == mdEntry->key); <-- This could fail b/c of truncation + a write/append before this job + // runs. + + err = cs->exists(cloudKey, &exists); + if (err) + throw runtime_error(string("synchronize(): checking existence of ") + key + ", got " + + strerror_r(errno, buf, 80)); + if (exists) + return; + + exists = cache->exists(prefix, cloudKey); + if (!exists) + { + logger->log(LOG_DEBUG, "synchronize(): was told to upload %s but it does not exist locally", key.c_str()); + return; + } + + err = cs->putObject(objectPath.string(), cloudKey); + if (err) + throw runtime_error(string("synchronize(): uploading ") + key + ", got " + strerror_r(errno, buf, 80)); + + numBytesRead += mdEntry.length; + bytesReadBySync += mdEntry.length; + numBytesUploaded += mdEntry.length; + ++objectsSyncedWithNoJournal; + replicator->remove(objectPath, Replicator::NO_LOCAL); +} + +void Synchronizer::synchronizeDelete(const string& sourceFile, list::iterator& it) +{ + ScopedWriteLock s(ioc, sourceFile); + string cloudKey = it->substr(it->find('/') + 1); + cs->deleteObject(cloudKey); +} + +void Synchronizer::synchronizeWithJournal(const string& sourceFile, list::iterator& lit) +{ + ScopedWriteLock s(ioc, sourceFile); + + char buf[80]; + string key = *lit; + size_t pos = key.find_first_of('/'); + bf::path prefix = key.substr(0, pos); + string cloudKey = key.substr(pos + 1); + + MetadataFile md(sourceFile, MetadataFile::no_create_t(), true); + + if (!md.exists()) + { + logger->log(LOG_DEBUG, "synchronizeWithJournal(): no metadata found for %s. It must have been deleted.", + sourceFile.c_str()); + try + { + bf::path objectPath = cachePath / key; + if (bf::exists(objectPath)) + { + size_t objSize = bf::file_size(objectPath); + replicator->remove(objectPath); + cache->deletedObject(prefix, cloudKey, objSize); + cs->deleteObject(cloudKey); + } + bf::path jPath = journalPath / (key + ".journal"); + if (bf::exists(jPath)) + { + size_t jSize = bf::file_size(jPath); + replicator->remove(jPath); + cache->deletedJournal(prefix, jSize); + } + } + catch (exception& e) + { + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): failed to remove orphaned object '%s' from the cache, got %s", + (cachePath / key).string().c_str(), e.what()); + } + return; + } + + metadataObject mdEntry; + bool metaExists = md.getEntry(MetadataFile::getOffsetFromKey(cloudKey), &mdEntry); + if (!metaExists || cloudKey != mdEntry.key) + { + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): %s does not exist in metadata for %s. This suggests truncation.", + key.c_str(), sourceFile.c_str()); + return; + } + // assert(key == mdEntry->key); <--- I suspect this can happen in a truncate + write situation + a deep + // sync queue + + bf::path oldCachePath = cachePath / key; + string journalName = (journalPath / (key + ".journal")).string(); + + if (!bf::exists(journalName)) + { + logger->log(LOG_DEBUG, "synchronizeWithJournal(): no journal file found for %s", key.c_str()); + + // sanity check + add'l info. Test whether the object exists in cloud storage. If so, complain, + // and run synchronize() instead. + bool existsOnCloud; + int err = cs->exists(cloudKey, &existsOnCloud); + if (err) + throw runtime_error(string("Synchronizer: cs->exists() failed: ") + strerror_r(errno, buf, 80)); + if (!existsOnCloud) + { + if (cache->exists(prefix, cloudKey)) + { + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): %s has no journal and does not exist in the cloud, calling " + "synchronize() instead. Need to explain how this happens.", + key.c_str()); + s.unlock(); + synchronize(sourceFile, lit); + } + else + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): %s has no journal, and does not exist in the cloud or in " + " the local cache. Need to explain how this happens.", + key.c_str()); + return; + } + else + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): %s has no journal, but it does exist in the cloud. " + " This indicates that an overlapping syncWithJournal() call handled it properly.", + key.c_str()); + + return; + } + + int err; + boost::shared_array data; + size_t count = 0, size = mdEntry.length, originalSize = 0; + + bool oldObjIsCached = cache->exists(prefix, cloudKey); + + // get the base object if it is not already cached + // merge it with its journal file + if (!oldObjIsCached) + { + err = cs->getObject(cloudKey, &data, &size); + if (err) + { + if (errno == ENOENT) + { + logger->log(LOG_DEBUG, "synchronizeWithJournal(): %s does not exist in cache nor in cloud storage", + key.c_str()); + return; + } + throw runtime_error(string("Synchronizer: getObject() failed: ") + strerror_r(errno, buf, 80)); + } + + numBytesDownloaded += size; + originalSize += size; + + // TODO!! This sucks. Need a way to pass in a larger array to cloud storage, and also have it not + // do any add'l alloc'ing or copying + if (size < mdEntry.length) + { + boost::shared_array tmp(new uint8_t[mdEntry.length]()); + memcpy(tmp.get(), data.get(), size); + memset(&tmp[size], 0, mdEntry.length - size); + data.swap(tmp); + } + size = mdEntry.length; // reset length. Using the metadata as a source of truth (truncation), not actual + // file length. + + size_t _bytesRead = 0; + err = ioc->mergeJournalInMem(data, size, journalName.c_str(), &_bytesRead); + if (err) + { + if (!bf::exists(journalName)) + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): journal %s was deleted mid-operation, check locking", + journalName.c_str()); + else + logger->log(LOG_ERR, "synchronizeWithJournal(): unexpected error merging journal for %s", + key.c_str()); + return; + } + numBytesRead += _bytesRead; + bytesReadBySyncWithJournal += _bytesRead; + originalSize += _bytesRead; + } + else + { + size_t _bytesRead = 0; + data = ioc->mergeJournal(oldCachePath.string().c_str(), journalName.c_str(), 0, size, &_bytesRead); + if (!data) + { + if (!bf::exists(journalName)) + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): journal %s was deleted mid-operation, check locking", + journalName.c_str()); + else + logger->log(LOG_ERR, "synchronizeWithJournal(): unexpected error merging journal for %s", + key.c_str()); + return; + } + numBytesRead += _bytesRead; + bytesReadBySyncWithJournal += _bytesRead; + originalSize = _bytesRead; + } + + // original size here should be == objectsize + journalsize + + // get a new key for the resolved version & upload it + string newCloudKey = MetadataFile::getNewKeyFromOldKey(cloudKey, size); + string newKey = (prefix / newCloudKey).string(); + err = cs->putObject(data, size, newCloudKey); + if (err) + { + // try to delete it in cloud storage... unlikely it is there in the first place, and if it is + // this probably won't work + int l_errno = errno; + cs->deleteObject(newCloudKey); + throw runtime_error(string("Synchronizer: putObject() failed: ") + strerror_r(l_errno, buf, 80)); + } + numBytesUploaded += size; + + // if the object was cached... + // write the new data to disk, + // tell the cache about the rename + // rename the file in any pending ops in Synchronizer + + if (oldObjIsCached) + { + // Is this the only thing outside of Replicator that writes files? + // If so move this write loop to Replicator. + bf::path newCachePath = cachePath / newKey; + int newFD = ::open(newCachePath.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (newFD < 0) + throw runtime_error(string("Synchronizer: Failed to open a new object in local storage! Got ") + + strerror_r(errno, buf, 80)); + ScopedCloser scloser(newFD); + + while (count < size) + { + err = ::write(newFD, &data[count], size - count); + if (err < 0) + { + ::unlink(newCachePath.string().c_str()); + throw runtime_error(string("Synchronizer: Failed to write to a new object in local storage! Got ") + + strerror_r(errno, buf, 80)); + } + count += err; + } + numBytesWritten += size; + + size_t oldSize = bf::file_size(oldCachePath); + + cache->rename(prefix, cloudKey, newCloudKey, size - oldSize); + replicator->remove(oldCachePath); + + // This condition is probably irrelevant for correct functioning now, + // but it should be very rare so what the hell. + if (oldSize != MetadataFile::getLengthFromKey(cloudKey)) + { + ostringstream oss; + oss << "Synchronizer::synchronizeWithJournal(): detected a mismatch between file size and " + << "length stored in the object name. object name = " << cloudKey + << " length-in-name = " << MetadataFile::getLengthFromKey(cloudKey) << " real-length = " << oldSize; + logger->log(LOG_WARNING, oss.str().c_str()); + } + } + + mergeDiff += size - originalSize; + ++journalsMerged; + // update the metadata for the source file + + md.updateEntry(MetadataFile::getOffsetFromKey(cloudKey), newCloudKey, size); + replicator->updateMetadata(md); + + rename(key, newKey); + + // delete the old object & journal file + cache->deletedJournal(prefix, bf::file_size(journalName)); + replicator->remove(journalName); + cs->deleteObject(cloudKey); +} + +void Synchronizer::rename(const string& oldKey, const string& newKey) +{ + boost::unique_lock s(mutex); + + auto it = pendingOps.find(oldKey); + if (it != pendingOps.end()) + { + pendingOps[newKey] = it->second; + pendingOps.erase(it); + } + it = opsInProgress.find(oldKey); + if (it != opsInProgress.end()) + { + opsInProgress[newKey] = it->second; + opsInProgress.erase(it); + } + + for (auto& name : objNames) + if (name == oldKey) + name = newKey; } bf::path Synchronizer::getJournalPath() { - return journalPath; + return journalPath; } bf::path Synchronizer::getCachePath() { - return cachePath; + return cachePath; } void Synchronizer::printKPIs() const { - cout << "Synchronizer" << endl; - cout << "\tnumBytesRead: " << numBytesRead << endl; - cout << "\tbytesReadBySync: " << bytesReadBySync << endl; - cout << "\tbytesReadBySyncWithJournal: " << bytesReadBySyncWithJournal << endl; - cout << "\tnumBytesWritten: " << numBytesWritten << endl; - cout << "\tnumBytesUploaded: " << numBytesUploaded << endl; - cout << "\tnumBytesDownloaded: " << numBytesDownloaded << endl; - cout << "\tmergeDiff: " << mergeDiff << endl; - cout << "\tflushesTriggeredBySize: " << flushesTriggeredBySize << endl; - cout << "\tflushesTriggeredByTimer: " << flushesTriggeredByTimer << endl; - cout << "\tjournalsMerged: " << journalsMerged << endl; - cout << "\tobjectsSyncedWithNoJournal: " << objectsSyncedWithNoJournal << endl; + cout << "Synchronizer" << endl; + cout << "\tnumBytesRead: " << numBytesRead << endl; + cout << "\tbytesReadBySync: " << bytesReadBySync << endl; + cout << "\tbytesReadBySyncWithJournal: " << bytesReadBySyncWithJournal << endl; + cout << "\tnumBytesWritten: " << numBytesWritten << endl; + cout << "\tnumBytesUploaded: " << numBytesUploaded << endl; + cout << "\tnumBytesDownloaded: " << numBytesDownloaded << endl; + cout << "\tmergeDiff: " << mergeDiff << endl; + cout << "\tflushesTriggeredBySize: " << flushesTriggeredBySize << endl; + cout << "\tflushesTriggeredByTimer: " << flushesTriggeredByTimer << endl; + cout << "\tjournalsMerged: " << journalsMerged << endl; + cout << "\tobjectsSyncedWithNoJournal: " << objectsSyncedWithNoJournal << endl; } - + /* The helper objects & fcns */ Synchronizer::PendingOps::PendingOps(int flags) : opFlags(flags), waiters(0), finished(false) @@ -824,56 +851,57 @@ Synchronizer::PendingOps::PendingOps(int flags) : opFlags(flags), waiters(0), fi Synchronizer::PendingOps::~PendingOps() { - assert(waiters == 0); + assert(waiters == 0); } void Synchronizer::PendingOps::notify() { - finished = true; - condvar.notify_all(); + finished = true; + condvar.notify_all(); } -void Synchronizer::PendingOps::wait(boost::mutex *m) +void Synchronizer::PendingOps::wait(boost::mutex* m) { - while (!finished) - { - waiters++; - condvar.wait(*m); - waiters--; - } + while (!finished) + { + waiters++; + condvar.wait(*m); + waiters--; + } } -Synchronizer::Job::Job(Synchronizer *s, std::list::iterator i) : sync(s), it(i) -{ } +Synchronizer::Job::Job(Synchronizer* s, std::list::iterator i) : sync(s), it(i) +{ +} void Synchronizer::Job::operator()() -{ - sync->process(it); +{ + sync->process(it); } void Synchronizer::configListener() { - // Uploader threads - string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_uploads"); - if (maxUploads == 0) - maxUploads = 20; - if (stmp.empty()) + // Uploader threads + string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_uploads"); + if (maxUploads == 0) + maxUploads = 20; + if (stmp.empty()) + { + logger->log(LOG_CRIT, "max_concurrent_uploads is not set. Using current value = %u", maxUploads); + } + try + { + uint newValue = stoul(stmp); + if (newValue != maxUploads) { - logger->log(LOG_CRIT, "max_concurrent_uploads is not set. Using current value = %u",maxUploads); - } - try - { - uint newValue = stoul(stmp); - if (newValue != maxUploads) - { - maxUploads = newValue; - threadPool->setMaxThreads(maxUploads); - logger->log(LOG_INFO, "max_concurrent_uploads = %u",maxUploads); - } - } - catch (invalid_argument &) - { - logger->log(LOG_CRIT, "max_concurrent_uploads is not a number. Using current value = %u",maxUploads); + maxUploads = newValue; + threadPool->setMaxThreads(maxUploads); + logger->log(LOG_INFO, "max_concurrent_uploads = %u", maxUploads); } + } + catch (invalid_argument&) + { + logger->log(LOG_CRIT, "max_concurrent_uploads is not a number. Using current value = %u", maxUploads); + } } -} +} // namespace storagemanager diff --git a/storage-manager/src/Synchronizer.h b/storage-manager/src/Synchronizer.h index a11dc4131..59680bfe1 100644 --- a/storage-manager/src/Synchronizer.h +++ b/storage-manager/src/Synchronizer.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef SYNCHRONIZER_H_ #define SYNCHRONIZER_H_ @@ -35,107 +34,107 @@ namespace storagemanager { - class Cache; // break circular dependency in header files class IOCoordinator; -class Synchronizer : public boost::noncopyable , public ConfigListener +class Synchronizer : public boost::noncopyable, public ConfigListener { - public: - static Synchronizer *get(); - virtual ~Synchronizer(); - - // these take keys as parameters, not full path names, ex, pass in '12345' not - // 'cache/12345'. - void newJournalEntry(const boost::filesystem::path &firstDir, const std::string &key, size_t len); - void newJournalEntries(const boost::filesystem::path &firstDir, const std::vector > &keys); - void newObjects(const boost::filesystem::path &firstDir, const std::vector &keys); - void deletedObjects(const boost::filesystem::path &firstDir, const std::vector &keys); - void flushObject(const boost::filesystem::path &firstDir, const std::string &key); - void forceFlush(); // ideally, make a version of this that takes a firstDir parameter - void syncNow(); // synchronous version of force for SyncTask + public: + static Synchronizer* get(); + virtual ~Synchronizer(); - void newPrefix(const boost::filesystem::path &p); - void dropPrefix(const boost::filesystem::path &p); - - // for testing primarily - boost::filesystem::path getJournalPath(); - boost::filesystem::path getCachePath(); - void printKPIs() const; + // these take keys as parameters, not full path names, ex, pass in '12345' not + // 'cache/12345'. + void newJournalEntry(const boost::filesystem::path& firstDir, const std::string& key, size_t len); + void newJournalEntries(const boost::filesystem::path& firstDir, + const std::vector >& keys); + void newObjects(const boost::filesystem::path& firstDir, const std::vector& keys); + void deletedObjects(const boost::filesystem::path& firstDir, const std::vector& keys); + void flushObject(const boost::filesystem::path& firstDir, const std::string& key); + void forceFlush(); // ideally, make a version of this that takes a firstDir parameter + void syncNow(); // synchronous version of force for SyncTask - virtual void configListener() override; + void newPrefix(const boost::filesystem::path& p); + void dropPrefix(const boost::filesystem::path& p); - private: - Synchronizer(); - - void _newJournalEntry(const boost::filesystem::path &firstDir, const std::string &key, size_t len); - void process(std::list::iterator key); - void synchronize(const std::string &sourceFile, std::list::iterator &it); - void synchronizeDelete(const std::string &sourceFile, std::list::iterator &it); - void synchronizeWithJournal(const std::string &sourceFile, std::list::iterator &it); - void rename(const std::string &oldkey, const std::string &newkey); - void makeJob(const std::string &key); - - // this struct kind of got sloppy. Need to clean it at some point. - struct PendingOps - { - PendingOps(int flags); - ~PendingOps(); - int opFlags; - int waiters; - bool finished; - boost::condition condvar; - void wait(boost::mutex *); - void notify(); - }; - - struct Job : public ThreadPool::Job - { - virtual ~Job(){}; - Job(Synchronizer *s, std::list::iterator i); - void operator()(); - Synchronizer *sync; - std::list::iterator it; - }; - - uint maxUploads; - boost::scoped_ptr threadPool; - std::map > pendingOps; - std::map > opsInProgress; - - // this is a bit of a kludge to handle objects being renamed. Jobs can be issued - // against name1, but when the job starts running, the target may be name2. - // some consolidation should be possible between this and the two maps above, tbd. - // in general the code got kludgier b/c of renaming, needs a cleanup pass. - std::list objNames; - - // this thread will start jobs for entries in pendingOps every 10 seconds - bool die; - boost::thread syncThread; - const boost::chrono::seconds syncInterval = boost::chrono::seconds(10); - void periodicSync(); - std::map uncommittedJournalSize; - size_t journalSizeThreshold; - bool blockNewJobs; - - void syncNow(const boost::filesystem::path &prefix); // a synchronous version of forceFlush() + // for testing primarily + boost::filesystem::path getJournalPath(); + boost::filesystem::path getCachePath(); + void printKPIs() const; - // some KPIs - size_t numBytesRead, numBytesWritten, numBytesUploaded, numBytesDownloaded, - flushesTriggeredBySize, flushesTriggeredByTimer, journalsMerged, objectsSyncedWithNoJournal, - bytesReadBySync, bytesReadBySyncWithJournal; - ssize_t mergeDiff; - - SMLogging *logger; - Cache *cache; - Replicator *replicator; - IOCoordinator *ioc; - CloudStorage *cs; - - boost::filesystem::path cachePath; - boost::filesystem::path journalPath; - boost::mutex mutex; + virtual void configListener() override; + + private: + Synchronizer(); + + void _newJournalEntry(const boost::filesystem::path& firstDir, const std::string& key, size_t len); + void process(std::list::iterator key); + void synchronize(const std::string& sourceFile, std::list::iterator& it); + void synchronizeDelete(const std::string& sourceFile, std::list::iterator& it); + void synchronizeWithJournal(const std::string& sourceFile, std::list::iterator& it); + void rename(const std::string& oldkey, const std::string& newkey); + void makeJob(const std::string& key); + + // this struct kind of got sloppy. Need to clean it at some point. + struct PendingOps + { + PendingOps(int flags); + ~PendingOps(); + int opFlags; + int waiters; + bool finished; + boost::condition condvar; + void wait(boost::mutex*); + void notify(); + }; + + struct Job : public ThreadPool::Job + { + virtual ~Job(){}; + Job(Synchronizer* s, std::list::iterator i); + void operator()(); + Synchronizer* sync; + std::list::iterator it; + }; + + uint maxUploads; + boost::scoped_ptr threadPool; + std::map > pendingOps; + std::map > opsInProgress; + + // this is a bit of a kludge to handle objects being renamed. Jobs can be issued + // against name1, but when the job starts running, the target may be name2. + // some consolidation should be possible between this and the two maps above, tbd. + // in general the code got kludgier b/c of renaming, needs a cleanup pass. + std::list objNames; + + // this thread will start jobs for entries in pendingOps every 10 seconds + bool die; + boost::thread syncThread; + const boost::chrono::seconds syncInterval = boost::chrono::seconds(10); + void periodicSync(); + std::map uncommittedJournalSize; + size_t journalSizeThreshold; + bool blockNewJobs; + + void syncNow(const boost::filesystem::path& prefix); // a synchronous version of forceFlush() + + // some KPIs + size_t numBytesRead, numBytesWritten, numBytesUploaded, numBytesDownloaded, flushesTriggeredBySize, + flushesTriggeredByTimer, journalsMerged, objectsSyncedWithNoJournal, bytesReadBySync, + bytesReadBySyncWithJournal; + ssize_t mergeDiff; + + SMLogging* logger; + Cache* cache; + Replicator* replicator; + IOCoordinator* ioc; + CloudStorage* cs; + + boost::filesystem::path cachePath; + boost::filesystem::path journalPath; + boost::mutex mutex; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/ThreadPool.cpp b/storage-manager/src/ThreadPool.cpp index 1a12579d3..01c97376e 100644 --- a/storage-manager/src/ThreadPool.cpp +++ b/storage-manager/src/ThreadPool.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "ThreadPool.h" #include @@ -23,165 +22,168 @@ using namespace std; namespace storagemanager { - ThreadPool::ThreadPool() : maxThreads(1000), die(false), processQueueOnExit(true), threadsWaiting(0) { - // Using this ctor effectively limits the # of threads here to the natural limit of the - // context it's used in. In the CRP class for example, the # of active threads would be - // naturally limited by the # of concurrent operations. - logger = SMLogging::get(); - pruner = boost::thread([this] { this->prune(); } ); + // Using this ctor effectively limits the # of threads here to the natural limit of the + // context it's used in. In the CRP class for example, the # of active threads would be + // naturally limited by the # of concurrent operations. + logger = SMLogging::get(); + pruner = boost::thread([this] { this->prune(); }); } -ThreadPool::ThreadPool(uint num_threads, bool _processQueueOnExit) : maxThreads(num_threads), die(false), - processQueueOnExit(_processQueueOnExit), threadsWaiting(0) +ThreadPool::ThreadPool(uint num_threads, bool _processQueueOnExit) + : maxThreads(num_threads), die(false), processQueueOnExit(_processQueueOnExit), threadsWaiting(0) { - logger = SMLogging::get(); - pruner = boost::thread([this] { this->prune(); } ); + logger = SMLogging::get(); + pruner = boost::thread([this] { this->prune(); }); } ThreadPool::~ThreadPool() { - boost::unique_lock s(mutex); - die = true; - if (!processQueueOnExit) - jobs.clear(); - - jobAvailable.notify_all(); - s.unlock(); - - threads.join_all(); - pruner.interrupt(); - pruner.join(); + boost::unique_lock s(mutex); + die = true; + if (!processQueueOnExit) + jobs.clear(); + + jobAvailable.notify_all(); + s.unlock(); + + threads.join_all(); + pruner.interrupt(); + pruner.join(); } -void ThreadPool::setName(const string &_name) +void ThreadPool::setName(const string& _name) { - name = _name; + name = _name; } -void ThreadPool::addJob(const boost::shared_ptr &j) +void ThreadPool::addJob(const boost::shared_ptr& j) { - boost::unique_lock s(mutex); - if (die) - return; - - jobs.push_back(j); - // Start another thread if necessary - if (threadsWaiting == 0 && (threads.size() - pruneable.size()) < maxThreads) { - boost::thread *thread = threads.create_thread([this] { this->processingLoop(); }); - s_threads.insert(thread); - //if (name == "Downloader") - // cout << "Threadpool (" << name << "): created a new thread. live threads = " << - // threads.size() - pruneable.size() << " job count = " << jobs.size() << endl; - } - else - jobAvailable.notify_one(); + boost::unique_lock s(mutex); + if (die) + return; + + jobs.push_back(j); + // Start another thread if necessary + if (threadsWaiting == 0 && (threads.size() - pruneable.size()) < maxThreads) + { + boost::thread* thread = threads.create_thread([this] { this->processingLoop(); }); + s_threads.insert(thread); + // if (name == "Downloader") + // cout << "Threadpool (" << name << "): created a new thread. live threads = " << + // threads.size() - pruneable.size() << " job count = " << jobs.size() << endl; + } + else + jobAvailable.notify_one(); } void ThreadPool::prune() { - set::iterator it; - boost::unique_lock s(mutex); - - while (1) + set::iterator it; + boost::unique_lock s(mutex); + + while (1) + { + while (pruneable.empty() && !die) + somethingToPrune.wait(s); + + if (die) + break; + + for (auto& id : pruneable) { - while (pruneable.empty() && !die) - somethingToPrune.wait(s); - - if (die) - break; - - for (auto &id : pruneable) - { - it = s_threads.find(id); - assert(it != s_threads.end()); - it->thrd->join(); - threads.remove_thread(it->thrd); - s_threads.erase(it); - } - pruneable.clear(); + it = s_threads.find(id); + assert(it != s_threads.end()); + it->thrd->join(); + threads.remove_thread(it->thrd); + s_threads.erase(it); } + pruneable.clear(); + } } void ThreadPool::setMaxThreads(uint newMax) { - boost::unique_lock s(mutex); - maxThreads = newMax; + boost::unique_lock s(mutex); + maxThreads = newMax; } int ThreadPool::currentQueueSize() const { - boost::unique_lock s(mutex); - return jobs.size(); + boost::unique_lock s(mutex); + return jobs.size(); } void ThreadPool::processingLoop() { - boost::unique_lock s(mutex); + boost::unique_lock s(mutex); + try + { + _processingLoop(s); + } + catch (...) + { + } + // _processingLoop returns with the lock held + + pruneable.push_back(boost::this_thread::get_id()); + somethingToPrune.notify_one(); + // if (name == "Downloader") + // cout << "threadpool " << name << " thread exiting. jobs = " << jobs.size() << + // " live thread count = " << threads.size() - pruneable.size() << endl; +} + +void ThreadPool::_processingLoop(boost::unique_lock& s) +{ + // cout << "Thread started" << endl; + while (threads.size() - pruneable.size() <= + maxThreads) // this is the scale-down mechanism when maxThreads is changed + { + while (jobs.empty() && !die) + { + threadsWaiting++; + bool timedout = !jobAvailable.timed_wait<>(s, idleThreadTimeout); + threadsWaiting--; + if (timedout && jobs.empty()) + { + // cout << "Thread exiting b/c of timeout" << endl; + return; + } + } + if (jobs.empty()) + { // die was set, and there are no jobs left to process + // cout << "Thread exiting b/c of die (?)" << endl; + return; + } + boost::shared_ptr job = jobs.front(); + jobs.pop_front(); + + s.unlock(); try { - _processingLoop(s); + (*job)(); } - catch (...) - {} - // _processingLoop returns with the lock held - - pruneable.push_back(boost::this_thread::get_id()); - somethingToPrune.notify_one(); - //if (name == "Downloader") - // cout << "threadpool " << name << " thread exiting. jobs = " << jobs.size() << - // " live thread count = " << threads.size() - pruneable.size() << endl; -} - -void ThreadPool::_processingLoop(boost::unique_lock &s) -{ - //cout << "Thread started" << endl; - while (threads.size() - pruneable.size() <= maxThreads) // this is the scale-down mechanism when maxThreads is changed + catch (exception& e) { - while (jobs.empty() && !die) - { - threadsWaiting++; - bool timedout = !jobAvailable.timed_wait<>(s, idleThreadTimeout); - threadsWaiting--; - if (timedout && jobs.empty()) - { - //cout << "Thread exiting b/c of timeout" << endl; - return; - } - } - if (jobs.empty()) { // die was set, and there are no jobs left to process - //cout << "Thread exiting b/c of die (?)" << endl; - return; - } - boost::shared_ptr job = jobs.front(); - jobs.pop_front(); - - s.unlock(); - try - { - (*job)(); - } - catch (exception &e) - { - logger->log(LOG_CRIT, "ThreadPool (%s): caught '%s' from a job", name.c_str(), e.what()); - } - s.lock(); + logger->log(LOG_CRIT, "ThreadPool (%s): caught '%s' from a job", name.c_str(), e.what()); } - //cout << "Thread exiting b/c of pruning logic" << endl; + s.lock(); + } + // cout << "Thread exiting b/c of pruning logic" << endl; } -inline bool ThreadPool::id_compare::operator()(const ID_Thread &t1, const ID_Thread &t2) const +inline bool ThreadPool::id_compare::operator()(const ID_Thread& t1, const ID_Thread& t2) const { - return t1.id < t2.id; + return t1.id < t2.id; } -ThreadPool::ID_Thread::ID_Thread(boost::thread::id &i): id(i) +ThreadPool::ID_Thread::ID_Thread(boost::thread::id& i) : id(i) { } -ThreadPool::ID_Thread::ID_Thread(boost::thread *t): id(t->get_id()), thrd(t) +ThreadPool::ID_Thread::ID_Thread(boost::thread* t) : id(t->get_id()), thrd(t) { } -} +} // namespace storagemanager diff --git a/storage-manager/src/ThreadPool.h b/storage-manager/src/ThreadPool.h index 651ca8110..09b1bf853 100644 --- a/storage-manager/src/ThreadPool.h +++ b/storage-manager/src/ThreadPool.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef SM_THREADPOOL_H_ #define SM_THREADPOOL_H_ @@ -28,67 +27,66 @@ namespace storagemanager { - class ThreadPool : public boost::noncopyable { - public: - ThreadPool(); // this ctor uses an 'unbounded' # of threads; relies on the context for a natural max, - ThreadPool(uint num_threads, bool processQueueOnExit = false); // this ctor lets caller specify a limit. - virtual ~ThreadPool(); - - // addJob doesn't block - class Job - { - public: - virtual ~Job(){}; - virtual void operator()() = 0; - }; - - void addJob(const boost::shared_ptr &j); - void setMaxThreads(uint newMax); - int currentQueueSize() const; - void setName(const std::string &); // set the name of this threadpool (for debugging) + public: + ThreadPool(); // this ctor uses an 'unbounded' # of threads; relies on the context for a natural max, + ThreadPool(uint num_threads, bool processQueueOnExit = false); // this ctor lets caller specify a limit. + virtual ~ThreadPool(); - private: - void processingLoop(); // the fcn run by each thread - void _processingLoop(boost::unique_lock &); // processingLoop() wraps _processingLoop() with thread management stuff. - - SMLogging *logger; - uint maxThreads; - bool die; - bool processQueueOnExit; - int threadsWaiting; - std::string name; - boost::thread_group threads; - - // the set s_threads below is intended to make pruning idle threads efficient. - // there should be a cleaner way to do it. - struct ID_Thread - { - ID_Thread(boost::thread::id &); - ID_Thread(boost::thread *); - boost::thread::id id; - boost::thread *thrd; - }; - - struct id_compare - { - bool operator()(const ID_Thread &, const ID_Thread &) const; - }; - std::set s_threads; - - boost::condition jobAvailable; - std::deque > jobs; - mutable boost::mutex mutex; - - const boost::posix_time::time_duration idleThreadTimeout = boost::posix_time::seconds(60); - boost::thread pruner; - boost::condition somethingToPrune; - std::vector pruneable; // when a thread is about to return it puts its id here - void prune(); + // addJob doesn't block + class Job + { + public: + virtual ~Job(){}; + virtual void operator()() = 0; + }; + + void addJob(const boost::shared_ptr& j); + void setMaxThreads(uint newMax); + int currentQueueSize() const; + void setName(const std::string&); // set the name of this threadpool (for debugging) + + private: + void processingLoop(); // the fcn run by each thread + void _processingLoop(boost::unique_lock&); // processingLoop() wraps _processingLoop() with + // thread management stuff. + + SMLogging* logger; + uint maxThreads; + bool die; + bool processQueueOnExit; + int threadsWaiting; + std::string name; + boost::thread_group threads; + + // the set s_threads below is intended to make pruning idle threads efficient. + // there should be a cleaner way to do it. + struct ID_Thread + { + ID_Thread(boost::thread::id&); + ID_Thread(boost::thread*); + boost::thread::id id; + boost::thread* thrd; + }; + + struct id_compare + { + bool operator()(const ID_Thread&, const ID_Thread&) const; + }; + std::set s_threads; + + boost::condition jobAvailable; + std::deque > jobs; + mutable boost::mutex mutex; + + const boost::posix_time::time_duration idleThreadTimeout = boost::posix_time::seconds(60); + boost::thread pruner; + boost::condition somethingToPrune; + std::vector pruneable; // when a thread is about to return it puts its id here + void prune(); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/TruncateTask.cpp b/storage-manager/src/TruncateTask.cpp index acd77823b..019fef01e 100644 --- a/storage-manager/src/TruncateTask.cpp +++ b/storage-manager/src/TruncateTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "TruncateTask.h" #include #include "messageFormat.h" @@ -25,7 +24,6 @@ using namespace std; namespace storagemanager { - TruncateTask::TruncateTask(int sock, uint len) : PosixTask(sock, len) { } @@ -35,51 +33,52 @@ TruncateTask::~TruncateTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } bool TruncateTask::run() { - SMLogging* logger = SMLogging::get(); - int success; - uint8_t buf[1024] = {0}; - - if (getLength() > 1023) { - handleError("TruncateTask read", ENAMETOOLONG); - return false; - } - - success = read(buf, getLength()); - check_error("TruncateTask read", false); - truncate_cmd *cmd = (truncate_cmd *) buf; - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"truncate %s newlength %i.",cmd->filename,cmd->length); - #endif - int err; - - try - { - err = ioc->truncate(cmd->filename, cmd->length); - } - catch (exception &e) - { - logger->log(LOG_ERR, "TruncateTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - if (err) - { - handleError("TruncateTask truncate", errno); - return true; - } - - sm_response *resp = (sm_response *) buf; - resp->returnCode = 0; - return write(*resp, 0); + SMLogging* logger = SMLogging::get(); + int success; + uint8_t buf[1024] = {0}; + + if (getLength() > 1023) + { + handleError("TruncateTask read", ENAMETOOLONG); + return false; + } + + success = read(buf, getLength()); + check_error("TruncateTask read", false); + truncate_cmd* cmd = (truncate_cmd*)buf; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "truncate %s newlength %i.", cmd->filename, cmd->length); +#endif + int err; + + try + { + err = ioc->truncate(cmd->filename, cmd->length); + } + catch (exception& e) + { + logger->log(LOG_ERR, "TruncateTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + if (err) + { + handleError("TruncateTask truncate", errno); + return true; + } + + sm_response* resp = (sm_response*)buf; + resp->returnCode = 0; + return write(*resp, 0); } -} +} // namespace storagemanager diff --git a/storage-manager/src/TruncateTask.h b/storage-manager/src/TruncateTask.h index 664f674c5..91ac6dd85 100644 --- a/storage-manager/src/TruncateTask.h +++ b/storage-manager/src/TruncateTask.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef TRUNCATETASK_H_ #define TRUNCATETASK_H_ @@ -23,19 +22,17 @@ namespace storagemanager { - class TruncateTask : public PosixTask { - public: - TruncateTask(int sock, uint length); - virtual ~TruncateTask(); - - bool run(); - - private: - TruncateTask(); + public: + TruncateTask(int sock, uint length); + virtual ~TruncateTask(); + + bool run(); + + private: + TruncateTask(); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/UnlinkTask.cpp b/storage-manager/src/UnlinkTask.cpp index ad913acdb..4e84995fb 100644 --- a/storage-manager/src/UnlinkTask.cpp +++ b/storage-manager/src/UnlinkTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "UnlinkTask.h" #include #include "messageFormat.h" @@ -25,7 +24,6 @@ using namespace std; namespace storagemanager { - UnlinkTask::UnlinkTask(int sock, uint len) : PosixTask(sock, len) { } @@ -35,53 +33,53 @@ UnlinkTask::~UnlinkTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } - + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } bool UnlinkTask::run() { - SMLogging* logger = SMLogging::get(); - int success; - uint8_t buf[1024] = {0}; - - if (getLength() > 1023) { - handleError("UnlinkTask read", ENAMETOOLONG); - return true; - } - - success = read(buf, getLength()); - check_error("UnlinkTask read", false); - unlink_cmd *cmd = (unlink_cmd *) buf; - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"unlink %s.",cmd->filename); - #endif - int err; - - try - { - err = ioc->unlink(cmd->filename); - } - catch (exception &e) - { - logger->log(LOG_ERR, "UnlinkTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - - if (err) - { - handleError("UnlinkTask unlink", errno); - return true; - } - - sm_response *resp = (sm_response *) buf; - resp->returnCode = 0; - return write(*resp, 0); + SMLogging* logger = SMLogging::get(); + int success; + uint8_t buf[1024] = {0}; + + if (getLength() > 1023) + { + handleError("UnlinkTask read", ENAMETOOLONG); + return true; + } + + success = read(buf, getLength()); + check_error("UnlinkTask read", false); + unlink_cmd* cmd = (unlink_cmd*)buf; + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "unlink %s.", cmd->filename); +#endif + int err; + + try + { + err = ioc->unlink(cmd->filename); + } + catch (exception& e) + { + logger->log(LOG_ERR, "UnlinkTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + + if (err) + { + handleError("UnlinkTask unlink", errno); + return true; + } + + sm_response* resp = (sm_response*)buf; + resp->returnCode = 0; + return write(*resp, 0); } -} +} // namespace storagemanager diff --git a/storage-manager/src/UnlinkTask.h b/storage-manager/src/UnlinkTask.h index 856b6da8a..e132f6a22 100644 --- a/storage-manager/src/UnlinkTask.h +++ b/storage-manager/src/UnlinkTask.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef UNLINKTASK_H_ #define UNLINKTASK_H_ @@ -23,19 +22,17 @@ namespace storagemanager { - class UnlinkTask : public PosixTask { - public: - UnlinkTask(int sock, uint length); - virtual ~UnlinkTask(); - - bool run(); - - private: - UnlinkTask(); + public: + UnlinkTask(int sock, uint length); + virtual ~UnlinkTask(); + + bool run(); + + private: + UnlinkTask(); }; - -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/Utilities.cpp b/storage-manager/src/Utilities.cpp index 9a904e485..351eb0f8a 100644 --- a/storage-manager/src/Utilities.cpp +++ b/storage-manager/src/Utilities.cpp @@ -20,8 +20,7 @@ namespace storagemanager { - -ScopedFileLock::ScopedFileLock(IOCoordinator *i, const std::string &k) : ioc(i), locked(false), key(k) +ScopedFileLock::ScopedFileLock(IOCoordinator* i, const std::string& k) : ioc(i), locked(false), key(k) { } @@ -29,91 +28,97 @@ ScopedFileLock::~ScopedFileLock() { } -ScopedReadLock::ScopedReadLock(IOCoordinator *i, const std::string &k) : ScopedFileLock(i, k) +ScopedReadLock::ScopedReadLock(IOCoordinator* i, const std::string& k) : ScopedFileLock(i, k) { - lock(); + lock(); } ScopedReadLock::~ScopedReadLock() { - unlock(); + unlock(); } void ScopedReadLock::lock() { - assert(!locked); - ioc->readLock(key); - locked = true; + assert(!locked); + ioc->readLock(key); + locked = true; } void ScopedReadLock::unlock() { - if (locked) - { - ioc->readUnlock(key); - locked = false; - } + if (locked) + { + ioc->readUnlock(key); + locked = false; + } } -ScopedWriteLock::ScopedWriteLock(IOCoordinator *i, const std::string &k) : ScopedFileLock(i, k) +ScopedWriteLock::ScopedWriteLock(IOCoordinator* i, const std::string& k) : ScopedFileLock(i, k) { - lock(); + lock(); } ScopedWriteLock::~ScopedWriteLock() { - unlock(); + unlock(); } void ScopedWriteLock::lock() { - assert(!locked); - ioc->writeLock(key); - locked = true; + assert(!locked); + ioc->writeLock(key); + locked = true; } void ScopedWriteLock::unlock() { - if (locked) - { - ioc->writeUnlock(key); - locked = false; - } + if (locked) + { + ioc->writeUnlock(key); + locked = false; + } } -ScopedCloser::ScopedCloser() : fd(-1) { } -ScopedCloser::ScopedCloser(int f) : fd(f) { assert(f != -1); } -ScopedCloser::~ScopedCloser() { - if (fd < 0) - return; - int s_errno = errno; - ::close(fd); - errno = s_errno; -} - -SharedCloser::SharedCloser(int f) -{ - block = new CtrlBlock(); - assert(f >= 0); - block->fd = f; - block->refCount = 1; -} - -SharedCloser::SharedCloser(const SharedCloser &s) : block(s.block) +ScopedCloser::ScopedCloser() : fd(-1) { - block->refCount++; +} +ScopedCloser::ScopedCloser(int f) : fd(f) +{ + assert(f != -1); +} +ScopedCloser::~ScopedCloser() +{ + if (fd < 0) + return; + int s_errno = errno; + ::close(fd); + errno = s_errno; +} + +SharedCloser::SharedCloser(int f) +{ + block = new CtrlBlock(); + assert(f >= 0); + block->fd = f; + block->refCount = 1; +} + +SharedCloser::SharedCloser(const SharedCloser& s) : block(s.block) +{ + block->refCount++; } SharedCloser::~SharedCloser() { - block->refCount--; - if (block->refCount == 0) - { - int s_errno = errno; - ::close(block->fd); - delete block; - errno = s_errno; - } -} - + block->refCount--; + if (block->refCount == 0) + { + int s_errno = errno; + ::close(block->fd); + delete block; + errno = s_errno; + } } + +} // namespace storagemanager diff --git a/storage-manager/src/Utilities.h b/storage-manager/src/Utilities.h index fe7a6c93c..2aab933c7 100644 --- a/storage-manager/src/Utilities.h +++ b/storage-manager/src/Utilities.h @@ -20,10 +20,8 @@ #include - namespace storagemanager { - class IOCoordinator; // a few utility classes we've coded here and there, now de-duped and centralized. @@ -31,58 +29,58 @@ class IOCoordinator; struct ScopedFileLock { - ScopedFileLock(IOCoordinator *i, const std::string &k); - virtual ~ScopedFileLock(); - - virtual void lock() = 0; - virtual void unlock() = 0; - - IOCoordinator *ioc; - bool locked; - const std::string key; + ScopedFileLock(IOCoordinator* i, const std::string& k); + virtual ~ScopedFileLock(); + + virtual void lock() = 0; + virtual void unlock() = 0; + + IOCoordinator* ioc; + bool locked; + const std::string key; }; struct ScopedReadLock : public ScopedFileLock { - ScopedReadLock(IOCoordinator *i, const std::string &k); - virtual ~ScopedReadLock(); - void lock(); - void unlock(); + ScopedReadLock(IOCoordinator* i, const std::string& k); + virtual ~ScopedReadLock(); + void lock(); + void unlock(); }; struct ScopedWriteLock : public ScopedFileLock { - ScopedWriteLock(IOCoordinator *i, const std::string &k); - virtual ~ScopedWriteLock(); - void lock(); - void unlock(); + ScopedWriteLock(IOCoordinator* i, const std::string& k); + virtual ~ScopedWriteLock(); + void lock(); + void unlock(); }; struct ScopedCloser { - ScopedCloser(); - ScopedCloser(int f); - ~ScopedCloser(); - int fd; + ScopedCloser(); + ScopedCloser(int f); + ~ScopedCloser(); + int fd; }; class SharedCloser { - public: - SharedCloser(int f); - SharedCloser(const SharedCloser &); - ~SharedCloser(); - - private: - struct CtrlBlock - { - int fd; - uint refCount; - }; - - CtrlBlock *block; + public: + SharedCloser(int f); + SharedCloser(const SharedCloser&); + ~SharedCloser(); + + private: + struct CtrlBlock + { + int fd; + uint refCount; + }; + + CtrlBlock* block; }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/WriteTask.cpp b/storage-manager/src/WriteTask.cpp index 70cc7d7b0..3e9511613 100644 --- a/storage-manager/src/WriteTask.cpp +++ b/storage-manager/src/WriteTask.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "WriteTask.h" #include "messageFormat.h" #include "IOCoordinator.h" @@ -26,7 +25,6 @@ using namespace std; namespace storagemanager { - WriteTask::WriteTask(int sock, uint len) : PosixTask(sock, len) { } @@ -36,85 +34,85 @@ WriteTask::~WriteTask() } #define check_error(msg, ret) \ - if (success<0) \ - { \ - handleError(msg, errno); \ - return ret; \ - } + if (success < 0) \ + { \ + handleError(msg, errno); \ + return ret; \ + } #define min(x, y) (x < y ? x : y) bool WriteTask::run() { - SMLogging* logger = SMLogging::get(); - int success; - uint8_t cmdbuf[1024] = {0}; - - success = read(cmdbuf, sizeof(write_cmd)); - check_error("WriteTask read", false); - write_cmd *cmd = (write_cmd *) cmdbuf; - - if (cmd->flen > 1023 - sizeof(*cmd)) - { - handleError("WriteTask", ENAMETOOLONG); - return true; - } - success = read(&cmdbuf[sizeof(*cmd)], cmd->flen); - check_error("WriteTask read", false); - - #ifdef SM_TRACE - logger->log(LOG_DEBUG,"write filename %s offset %i count %i.",cmd->filename,cmd->offset,cmd->count); - #endif - - ssize_t readCount = 0, writeCount = 0; - vector databuf; - uint bufsize = min(100 << 20, cmd->count); // 100 MB - //uint bufsize = cmd->count; - databuf.resize(bufsize); + SMLogging* logger = SMLogging::get(); + int success; + uint8_t cmdbuf[1024] = {0}; - while (readCount < cmd->count) + success = read(cmdbuf, sizeof(write_cmd)); + check_error("WriteTask read", false); + write_cmd* cmd = (write_cmd*)cmdbuf; + + if (cmd->flen > 1023 - sizeof(*cmd)) + { + handleError("WriteTask", ENAMETOOLONG); + return true; + } + success = read(&cmdbuf[sizeof(*cmd)], cmd->flen); + check_error("WriteTask read", false); + +#ifdef SM_TRACE + logger->log(LOG_DEBUG, "write filename %s offset %i count %i.", cmd->filename, cmd->offset, cmd->count); +#endif + + ssize_t readCount = 0, writeCount = 0; + vector databuf; + uint bufsize = min(100 << 20, cmd->count); // 100 MB + // uint bufsize = cmd->count; + databuf.resize(bufsize); + + while (readCount < cmd->count) + { + uint toRead = min(static_cast(cmd->count - readCount), bufsize); + success = read(&databuf[0], toRead); + check_error("WriteTask read data", false); + if (success == 0) + break; + readCount += success; + uint writePos = 0; + ssize_t err; + while (writeCount < readCount) { - uint toRead = min(static_cast(cmd->count - readCount), bufsize); - success = read(&databuf[0], toRead); - check_error("WriteTask read data", false); - if (success==0) - break; - readCount += success; - uint writePos = 0; - ssize_t err; - while (writeCount < readCount) - { - try - { - err = ioc->write(cmd->filename, &databuf[writePos], cmd->offset + writeCount, success - writePos); - } - catch (exception &e) - { - logger->log(LOG_ERR, "WriteTask: caught '%s'", e.what()); - errno = EIO; - err = -1; - } - if (err <= 0) - break; - writeCount += err; - writePos += err; - } - if (writeCount != readCount) - break; + try + { + err = ioc->write(cmd->filename, &databuf[writePos], cmd->offset + writeCount, success - writePos); + } + catch (exception& e) + { + logger->log(LOG_ERR, "WriteTask: caught '%s'", e.what()); + errno = EIO; + err = -1; + } + if (err <= 0) + break; + writeCount += err; + writePos += err; } - - uint8_t respbuf[sizeof(sm_response) + 4]; - sm_response *resp = (sm_response *) respbuf; - uint payloadLen = 0; - if (cmd->count != 0 && writeCount == 0) - { - payloadLen = 4; - resp->returnCode = -1; - *((int *) &resp[1]) = errno; - } - else - resp->returnCode = writeCount; - return write(*resp, payloadLen); + if (writeCount != readCount) + break; + } + + uint8_t respbuf[sizeof(sm_response) + 4]; + sm_response* resp = (sm_response*)respbuf; + uint payloadLen = 0; + if (cmd->count != 0 && writeCount == 0) + { + payloadLen = 4; + resp->returnCode = -1; + *((int*)&resp[1]) = errno; + } + else + resp->returnCode = writeCount; + return write(*resp, payloadLen); } -} +} // namespace storagemanager diff --git a/storage-manager/src/WriteTask.h b/storage-manager/src/WriteTask.h index 399368328..c1c636123 100644 --- a/storage-manager/src/WriteTask.h +++ b/storage-manager/src/WriteTask.h @@ -15,8 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - #ifndef WRITETASK_H_ #define WRITETASK_H_ @@ -24,18 +22,17 @@ namespace storagemanager { - class WriteTask : public PosixTask { - public: - WriteTask(int sock, uint length); - virtual ~WriteTask(); - - bool run(); - - private: - WriteTask(); + public: + WriteTask(int sock, uint length); + virtual ~WriteTask(); + + bool run(); + + private: + WriteTask(); }; -} +} // namespace storagemanager #endif diff --git a/storage-manager/src/main.cpp b/storage-manager/src/main.cpp index f6a85dc1a..6ca9be9df 100644 --- a/storage-manager/src/main.cpp +++ b/storage-manager/src/main.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include #include #include @@ -39,160 +38,150 @@ using namespace std; using namespace storagemanager; - class Opt { -public: - const char *m_progname; + public: + const char* m_progname; bool m_fg; - Opt(int argc, char **argv) - :m_progname(argv[0]), - m_fg(argc >= 2 && string(argv[1]) == "fg") - { } + Opt(int argc, char** argv) : m_progname(argv[0]), m_fg(argc >= 2 && string(argv[1]) == "fg") + { + } }; - -class ServiceStorageManager: public Service, public Opt +class ServiceStorageManager : public Service, public Opt { -protected: - void setupChildSignalHandlers(); + protected: + void setupChildSignalHandlers(); -public: - ServiceStorageManager(const Opt &opt) - :Service("StorageManager"), Opt(opt) - { } - void LogErrno() override - { - SMLogging::get()->log(LOG_ERR, "%s", strerror(errno)); - } - void ParentLogChildMessage(const std::string &str) override - { - SMLogging::get()->log(LOG_INFO, "%.*s", (int) str.length(), str.data()); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServiceStorageManager(const Opt& opt) : Service("StorageManager"), Opt(opt) + { + } + void LogErrno() override + { + SMLogging::get()->log(LOG_ERR, "%s", strerror(errno)); + } + void ParentLogChildMessage(const std::string& str) override + { + SMLogging::get()->log(LOG_INFO, "%.*s", (int)str.length(), str.data()); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - bool signalCaught = false; void printCacheUsage(int sig) { - Cache::get()->validateCacheSize(); - cout << "Current cache size = " << Cache::get()->getCurrentCacheSize() << endl; - cout << "Cache element count = " << Cache::get()->getCurrentCacheElementCount() << endl; + Cache::get()->validateCacheSize(); + cout << "Current cache size = " << Cache::get()->getCurrentCacheSize() << endl; + cout << "Cache element count = " << Cache::get()->getCurrentCacheElementCount() << endl; } void printKPIs(int sig) { - IOCoordinator::get()->printKPIs(); - Cache::get()->printKPIs(); - Synchronizer::get()->printKPIs(); - CloudStorage::get()->printKPIs(); - Replicator::get()->printKPIs(); + IOCoordinator::get()->printKPIs(); + Cache::get()->printKPIs(); + Synchronizer::get()->printKPIs(); + CloudStorage::get()->printKPIs(); + Replicator::get()->printKPIs(); } void shutdownSM(int sig) { - if (!signalCaught) - { - (SessionManager::get())->shutdownSM(sig); - } - signalCaught = true; + if (!signalCaught) + { + (SessionManager::get())->shutdownSM(sig); + } + signalCaught = true; } void coreSM(int sig) { - if (!signalCaught) - { - fatalHandler(sig); - } - signalCaught = true; + if (!signalCaught) + { + fatalHandler(sig); + } + signalCaught = true; } - void ServiceStorageManager::setupChildSignalHandlers() { - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); - std::vector shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGPOLL, - SIGPROF, SIGPWR, SIGTERM, SIGVTALRM}; + std::vector shutdownSignals{SIGALRM, SIGHUP, SIGINT, SIGPOLL, SIGPROF, SIGPWR, SIGTERM, SIGVTALRM}; - std::vector coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, - SIGQUIT, SIGSEGV, SIGSYS, SIGTRAP, - SIGXCPU, SIGXFSZ}; + std::vector coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGQUIT, + SIGSEGV, SIGSYS, SIGTRAP, SIGXCPU, SIGXFSZ}; - sa.sa_handler = shutdownSM; - for (int sig : shutdownSignals) - sigaction(sig, &sa, NULL); - - sa.sa_handler = coreSM; - for (int sig : coreSignals) - sigaction(sig, &sa, NULL); + sa.sa_handler = shutdownSM; + for (int sig : shutdownSignals) + sigaction(sig, &sa, NULL); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, NULL); - - sa.sa_handler = printCacheUsage; - sigaction(SIGUSR1, &sa, NULL); - - sa.sa_handler = printKPIs; - sigaction(SIGUSR2, &sa, NULL); + sa.sa_handler = coreSM; + for (int sig : coreSignals) + sigaction(sig, &sa, NULL); + + sa.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sa, NULL); + + sa.sa_handler = printCacheUsage; + sigaction(SIGUSR1, &sa, NULL); + + sa.sa_handler = printKPIs; + sigaction(SIGUSR2, &sa, NULL); } - int ServiceStorageManager::Child() { + SMLogging* logger = SMLogging::get(); + IOCoordinator* ioc = NULL; + Cache* cache = NULL; + Synchronizer* sync = NULL; + Replicator* rep = NULL; - SMLogging* logger = SMLogging::get(); - IOCoordinator* ioc = NULL; - Cache* cache = NULL; - Synchronizer* sync = NULL; - Replicator* rep = NULL; + /* Instantiate objects to have them verify config settings before continuing */ + try + { + ioc = IOCoordinator::get(); + cache = Cache::get(); + sync = Synchronizer::get(); + rep = Replicator::get(); + } + catch (exception& e) + { + logger->log(LOG_INFO, "StorageManager init FAIL: %s", e.what()); + return -1; + } - /* Instantiate objects to have them verify config settings before continuing */ - try - { - ioc = IOCoordinator::get(); - cache = Cache::get(); - sync = Synchronizer::get(); - rep = Replicator::get(); - } - catch (exception &e) - { - logger->log(LOG_INFO, "StorageManager init FAIL: %s", e.what()); - return -1; - } + setupChildSignalHandlers(); - setupChildSignalHandlers(); + int ret = 0; - int ret = 0; + logger->log(LOG_NOTICE, "StorageManager started."); - logger->log(LOG_NOTICE,"StorageManager started."); + SessionManager* sm = SessionManager::get(); - SessionManager* sm = SessionManager::get(); + NotifyServiceStarted(); - NotifyServiceStarted(); + ret = sm->start(); - ret = sm->start(); + cache->shutdown(); - cache->shutdown(); - - delete sync; - delete cache; - delete ioc; - delete rep; - logger->log(LOG_INFO,"StorageManager Shutdown Complete."); + delete sync; + delete cache; + delete ioc; + delete rep; + logger->log(LOG_INFO, "StorageManager Shutdown Complete."); - return ret; + return ret; } - int main(int argc, char** argv) { - return ServiceStorageManager(Opt(argc, argv)).Run(); + return ServiceStorageManager(Opt(argc, argv)).Run(); } diff --git a/storage-manager/src/smcat.cpp b/storage-manager/src/smcat.cpp index de7a2674e..bc1b1e1a9 100644 --- a/storage-manager/src/smcat.cpp +++ b/storage-manager/src/smcat.cpp @@ -28,139 +28,143 @@ using namespace std; using namespace storagemanager; -void usage(const char *progname) +void usage(const char* progname) { - cerr << progname << " is 'cat' for files managed by StorageManager." << endl; - cerr << "Usage: " << progname << " file1 file2 ... fileN" << endl; + cerr << progname << " is 'cat' for files managed by StorageManager." << endl; + cerr << "Usage: " << progname << " file1 file2 ... fileN" << endl; } bool SMOnline() { - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... - int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); - int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr)); - if (err >= 0) - { - ::close(clientSocket); - return true; - } - return false; + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... + int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); + int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr)); + if (err >= 0) + { + ::close(clientSocket); + return true; + } + return false; } -void catFileOffline(const char *filename, int prefixlen) +void catFileOffline(const char* filename, int prefixlen) { - uint8_t data[8192]; - off_t offset = 0; - int read_err, write_err, count; - try + uint8_t data[8192]; + off_t offset = 0; + int read_err, write_err, count; + try + { + boost::scoped_ptr ioc(IOCoordinator::get()); + + do { - boost::scoped_ptr ioc(IOCoordinator::get()); + count = 0; + read_err = ioc->read(filename, data, offset, 8192); + if (read_err < 0) + { + int l_errno = errno; + cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192) + << endl; + } - do { - count = 0; - read_err = ioc->read(filename, data, offset, 8192); - if (read_err < 0) - { - int l_errno = errno; - cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char *) data, 8192) << endl; - } - - while (count < read_err) - { - write_err = write(STDOUT_FILENO, &data[count], read_err - count); - if (write_err < 0) - { - int l_errno = errno; - cerr << "Error writing to stdout: " << strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - count += write_err; - } - offset += read_err; - } while (read_err > 0); - } - catch (exception &e) - { - cerr << "smcat catFileOffline FAIL: " << e.what() << endl; - } -} - -void catFileOnline(const char *filename, int prefixlen) -{ - uint8_t data[8192]; - off_t offset = 0; - int read_err, write_err, count; - idbdatafile::SMDataFile df(filename, O_RDONLY, 0); - - do { - count = 0; - read_err = df.read(data, 8192); - if (read_err < 0) + while (count < read_err) + { + write_err = write(STDOUT_FILENO, &data[count], read_err - count); + if (write_err < 0) { - int l_errno = errno; - cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char *) data, 8192) << endl; + int l_errno = errno; + cerr << "Error writing to stdout: " << strerror_r(l_errno, (char*)data, 8192) << endl; + exit(1); } - - while (count < read_err) - { - write_err = write(STDOUT_FILENO, &data[count], read_err - count); - if (write_err < 0) - { - int l_errno = errno; - cerr << "Error writing to stdout: " << strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - count += write_err; - } - offset += read_err; + count += write_err; + } + offset += read_err; } while (read_err > 0); + } + catch (exception& e) + { + cerr << "smcat catFileOffline FAIL: " << e.what() << endl; + } } -int makePathPrefix(char *target, int targetlen) +void catFileOnline(const char* filename, int prefixlen) { - // MCOL-3438 -> add bogus directories to the front of each param - Config *config = Config::get(); - int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); - target[0] = '/'; - target[1] = 0; - int bufpos = 1; - - for (int i = 0; i < prefixDepth; i++) + uint8_t data[8192]; + off_t offset = 0; + int read_err, write_err, count; + idbdatafile::SMDataFile df(filename, O_RDONLY, 0); + + do + { + count = 0; + read_err = df.read(data, 8192); + if (read_err < 0) { - if (bufpos + 3 >= targetlen) - { - cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; - exit(1); - } - memcpy(&target[bufpos], "x/\0", 3); - bufpos += 2; + int l_errno = errno; + cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192) + << endl; } - return bufpos; + + while (count < read_err) + { + write_err = write(STDOUT_FILENO, &data[count], read_err - count); + if (write_err < 0) + { + int l_errno = errno; + cerr << "Error writing to stdout: " << strerror_r(l_errno, (char*)data, 8192) << endl; + exit(1); + } + count += write_err; + } + offset += read_err; + } while (read_err > 0); } -int main(int argc, char **argv) +int makePathPrefix(char* target, int targetlen) { - if (argc < 2) - { - usage(argv[0]); - return 1; - } - - char prefix[8192]; - int prefixlen = makePathPrefix(prefix, 8192); + // MCOL-3438 -> add bogus directories to the front of each param + Config* config = Config::get(); + int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); + target[0] = '/'; + target[1] = 0; + int bufpos = 1; - for (int i = 1; i < argc; i++) + for (int i = 0; i < prefixDepth; i++) + { + if (bufpos + 3 >= targetlen) { - strncat(&prefix[prefixlen], argv[i], 8192 - prefixlen); - - if (SMOnline()) - catFileOnline(prefix, prefixlen); - else - catFileOffline(prefix, prefixlen); + cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; + exit(1); } - - return 0; + memcpy(&target[bufpos], "x/\0", 3); + bufpos += 2; + } + return bufpos; +} + +int main(int argc, char** argv) +{ + if (argc < 2) + { + usage(argv[0]); + return 1; + } + + char prefix[8192]; + int prefixlen = makePathPrefix(prefix, 8192); + + for (int i = 1; i < argc; i++) + { + strncat(&prefix[prefixlen], argv[i], 8192 - prefixlen); + + if (SMOnline()) + catFileOnline(prefix, prefixlen); + else + catFileOffline(prefix, prefixlen); + } + + return 0; } diff --git a/storage-manager/src/smls.cpp b/storage-manager/src/smls.cpp index e9487c069..f41b7f5f6 100644 --- a/storage-manager/src/smls.cpp +++ b/storage-manager/src/smls.cpp @@ -34,155 +34,154 @@ using namespace std; using namespace storagemanager; -void usage(const char *progname) +void usage(const char* progname) { - cerr << progname << " is like 'ls -l' for files managed by StorageManager" << endl; - cerr << "Usage: " << progname << " directory" << endl; + cerr << progname << " is like 'ls -l' for files managed by StorageManager" << endl; + cerr << "Usage: " << progname << " directory" << endl; } bool SMOnline() { - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... - int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); - int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr)); - if (err >= 0) - { - ::close(clientSocket); - return true; - } - return false; + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... + int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); + int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr)); + if (err >= 0) + { + ::close(clientSocket); + return true; + } + return false; } -void lsOffline(const char *path) +void lsOffline(const char* path) { - try - { - boost::scoped_ptr ioc(IOCoordinator::get()); - vector listing; + try + { + boost::scoped_ptr ioc(IOCoordinator::get()); + vector listing; - int err = ioc->listDirectory(path, &listing); - if (err) - exit(1); - - struct stat _stat; - boost::filesystem::path base(path); - boost::filesystem::path p; - cout.fill(' '); - for (auto &entry : listing) - { - p = base / entry; - err = ioc->stat(p.string().c_str(), &_stat); - if (!err) - { - if (_stat.st_mode & S_IFDIR) - { - cout << "d"; - cout.width(14); - } - else - cout.width(15); - - struct tm *my_tm = localtime(&_stat.st_mtim.tv_sec); - char date[100]; - strftime(date, 100, "%b %e %H:%M", my_tm); - cout << right << _stat.st_size << left << " " << date << left << " " << entry << endl; - } - else - { - cout.width(15); - cout << right << "error" << left << " " << entry << endl; - } - } - } - catch (exception &e) - { - cerr << "smls lsOffline FAIL: " << e.what() << endl; - } -} - -void lsOnline(const char *path) -{ - idbdatafile::SMFileSystem fs; - list listing; - - int err = fs.listDirectory(path, listing); + int err = ioc->listDirectory(path, &listing); if (err) - exit(1); - + exit(1); + + struct stat _stat; boost::filesystem::path base(path); boost::filesystem::path p; cout.fill(' '); - for (auto &entry : listing) + for (auto& entry : listing) { - p = base / entry; - bool isDir = fs.isDir(p.string().c_str()); - ssize_t size = fs.size(p.string().c_str()); - idbdatafile::SMDataFile df(p.string().c_str(),O_RDONLY,1); - time_t mtime = df.mtime(); - if (size >= 0) + p = base / entry; + err = ioc->stat(p.string().c_str(), &_stat); + if (!err) + { + if (_stat.st_mode & S_IFDIR) { - if (isDir) - { - cout << "d"; - cout.width(14); - } - else - cout.width(15); - - struct tm *my_tm = localtime(&mtime); - char date[100]; - strftime(date, 100, "%b %e %H:%M", my_tm); - cout << right << size << left << " " << date << left << " " << entry << endl; + cout << "d"; + cout.width(14); } else - { - cout.width(15); - cout << right << "error" << left << " " << entry << endl; - } + cout.width(15); + + struct tm* my_tm = localtime(&_stat.st_mtim.tv_sec); + char date[100]; + strftime(date, 100, "%b %e %H:%M", my_tm); + cout << right << _stat.st_size << left << " " << date << left << " " << entry << endl; + } + else + { + cout.width(15); + cout << right << "error" << left << " " << entry << endl; + } } + } + catch (exception& e) + { + cerr << "smls lsOffline FAIL: " << e.what() << endl; + } } -int makePathPrefix(char *target, int targetlen) +void lsOnline(const char* path) { - // MCOL-3438 -> add bogus directories to the front of each param - Config *config = Config::get(); - int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); - target[0] = '/'; - target[1] = 0; - int bufpos = 1; - - for (int i = 0; i < prefixDepth; i++) - { - if (bufpos + 3 >= targetlen) - { - cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; - exit(1); - } - memcpy(&target[bufpos], "x/\0", 3); - bufpos += 2; - } - return bufpos; -} + idbdatafile::SMFileSystem fs; + list listing; -int main(int argc, char **argv) -{ - if (argc != 2) + int err = fs.listDirectory(path, listing); + if (err) + exit(1); + + boost::filesystem::path base(path); + boost::filesystem::path p; + cout.fill(' '); + for (auto& entry : listing) + { + p = base / entry; + bool isDir = fs.isDir(p.string().c_str()); + ssize_t size = fs.size(p.string().c_str()); + idbdatafile::SMDataFile df(p.string().c_str(), O_RDONLY, 1); + time_t mtime = df.mtime(); + if (size >= 0) { - usage(argv[0]); - return 1; + if (isDir) + { + cout << "d"; + cout.width(14); + } + else + cout.width(15); + + struct tm* my_tm = localtime(&mtime); + char date[100]; + strftime(date, 100, "%b %e %H:%M", my_tm); + cout << right << size << left << " " << date << left << " " << entry << endl; } - - char prefix[8192]; - int prefixlen = makePathPrefix(prefix, 8192); - - if (SMOnline()) - lsOnline(strncat(prefix, argv[1], 8192 - prefixlen)); else - lsOffline(strncat(prefix, argv[1], 8192 - prefixlen)); - - return 0; + { + cout.width(15); + cout << right << "error" << left << " " << entry << endl; + } + } } +int makePathPrefix(char* target, int targetlen) +{ + // MCOL-3438 -> add bogus directories to the front of each param + Config* config = Config::get(); + int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); + target[0] = '/'; + target[1] = 0; + int bufpos = 1; + + for (int i = 0; i < prefixDepth; i++) + { + if (bufpos + 3 >= targetlen) + { + cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; + exit(1); + } + memcpy(&target[bufpos], "x/\0", 3); + bufpos += 2; + } + return bufpos; +} + +int main(int argc, char** argv) +{ + if (argc != 2) + { + usage(argv[0]); + return 1; + } + + char prefix[8192]; + int prefixlen = makePathPrefix(prefix, 8192); + + if (SMOnline()) + lsOnline(strncat(prefix, argv[1], 8192 - prefixlen)); + else + lsOffline(strncat(prefix, argv[1], 8192 - prefixlen)); + + return 0; +} diff --git a/storage-manager/src/smput.cpp b/storage-manager/src/smput.cpp index b81dbe9b4..35f723804 100644 --- a/storage-manager/src/smput.cpp +++ b/storage-manager/src/smput.cpp @@ -30,158 +30,156 @@ using namespace std; using namespace storagemanager; -void usage(const char *progname) +void usage(const char* progname) { - cerr << progname << " reads from stdin and puts it in a file managed by StorageManager" << endl; - cerr << "Usage: " << progname << " output_file" << endl; + cerr << progname << " reads from stdin and puts it in a file managed by StorageManager" << endl; + cerr << "Usage: " << progname << " output_file" << endl; } bool SMOnline() { - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... - int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); - int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr)); - if (err >= 0) - { - ::close(clientSocket); - return true; - } - return false; + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... + int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); + int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr)); + if (err >= 0) + { + ::close(clientSocket); + return true; + } + return false; } -void putOffline(const char *fname, int prefixlen) +void putOffline(const char* fname, int prefixlen) { - uint8_t data[8192]; - int read_err, write_err; - ssize_t count, offset = 0; - try + uint8_t data[8192]; + int read_err, write_err; + ssize_t count, offset = 0; + try + { + boost::scoped_ptr ioc(IOCoordinator::get()); + struct stat _stat; + read_err = ioc->open(fname, O_CREAT | O_TRUNC | O_WRONLY, &_stat); + if (read_err < 0) { - boost::scoped_ptr ioc(IOCoordinator::get()); - struct stat _stat; - read_err = ioc->open(fname, O_CREAT | O_TRUNC | O_WRONLY, &_stat); - if (read_err < 0) - { - int l_errno = errno; - cerr << "Failed to open/create " << &fname[prefixlen] << ": " << - strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - - do - { - read_err = ::read(STDIN_FILENO, data, 8192); - if (read_err < 0) - { - int l_errno = errno; - cerr << "Error reading stdin: " << strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - count = 0; - while (count < read_err) - { - write_err = ioc->write(fname, &data[count], offset + count, read_err - count); - if (write_err < 0) - { - int l_errno = errno; - cerr << "Error writing to " << &fname[prefixlen] << ": " << - strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - count += write_err; - } - offset += read_err; - } while (read_err > 0); + int l_errno = errno; + cerr << "Failed to open/create " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192) + << endl; + exit(1); } - catch (exception &e) - { - cerr << "smput putOffline FAIL: " << e.what() << endl; - } -} -void putOnline(const char *fname, int prefixlen) -{ - uint8_t data[8192]; - int read_err, write_err; - ssize_t count; - - idbdatafile::SMFileFactory ffactory; - boost::scoped_ptr df( - dynamic_cast(ffactory.open(fname, "w", 0, 0))); - - if (!df) - { - int l_errno = errno; - cerr << "Failed to open/create " << &fname[prefixlen] << ": " << - strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - do { - read_err = ::read(STDIN_FILENO, data, 8192); - if (read_err < 0) + read_err = ::read(STDIN_FILENO, data, 8192); + if (read_err < 0) + { + int l_errno = errno; + cerr << "Error reading stdin: " << strerror_r(l_errno, (char*)data, 8192) << endl; + exit(1); + } + count = 0; + while (count < read_err) + { + write_err = ioc->write(fname, &data[count], offset + count, read_err - count); + if (write_err < 0) { - int l_errno = errno; - cerr << "Error reading stdin: " << strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - count = 0; - while (count < read_err) - { - write_err = df->write(&data[count], read_err - count); - if (write_err < 0) - { - int l_errno = errno; - cerr << "Error writing to " << &fname[prefixlen] << ": " << - strerror_r(l_errno, (char *) data, 8192) << endl; - exit(1); - } - count += write_err; + int l_errno = errno; + cerr << "Error writing to " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192) + << endl; + exit(1); } + count += write_err; + } + offset += read_err; } while (read_err > 0); + } + catch (exception& e) + { + cerr << "smput putOffline FAIL: " << e.what() << endl; + } } -int makePathPrefix(char *target, int targetlen) +void putOnline(const char* fname, int prefixlen) { - // MCOL-3438 -> add bogus directories to the front of each param - Config *config = Config::get(); - int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); - target[0] = '/'; - target[1] = 0; - int bufpos = 1; - - for (int i = 0; i < prefixDepth; i++) + uint8_t data[8192]; + int read_err, write_err; + ssize_t count; + + idbdatafile::SMFileFactory ffactory; + boost::scoped_ptr df( + dynamic_cast(ffactory.open(fname, "w", 0, 0))); + + if (!df) + { + int l_errno = errno; + cerr << "Failed to open/create " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192) + << endl; + exit(1); + } + + do + { + read_err = ::read(STDIN_FILENO, data, 8192); + if (read_err < 0) { - if (bufpos + 3 >= targetlen) - { - cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; - exit(1); - } - memcpy(&target[bufpos], "x/\0", 3); - bufpos += 2; + int l_errno = errno; + cerr << "Error reading stdin: " << strerror_r(l_errno, (char*)data, 8192) << endl; + exit(1); } - return bufpos; + count = 0; + while (count < read_err) + { + write_err = df->write(&data[count], read_err - count); + if (write_err < 0) + { + int l_errno = errno; + cerr << "Error writing to " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192) + << endl; + exit(1); + } + count += write_err; + } + } while (read_err > 0); } -int main(int argc, char **argv) +int makePathPrefix(char* target, int targetlen) { - if (argc != 2) + // MCOL-3438 -> add bogus directories to the front of each param + Config* config = Config::get(); + int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); + target[0] = '/'; + target[1] = 0; + int bufpos = 1; + + for (int i = 0; i < prefixDepth; i++) + { + if (bufpos + 3 >= targetlen) { - usage(argv[0]); - return 1; + cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; + exit(1); } - - char prefix[8192]; - int prefixlen = makePathPrefix(prefix, 8192); - - if (SMOnline()) - putOnline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen); - else - putOffline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen); - return 0; + memcpy(&target[bufpos], "x/\0", 3); + bufpos += 2; + } + return bufpos; +} + +int main(int argc, char** argv) +{ + if (argc != 2) + { + usage(argv[0]); + return 1; + } + + char prefix[8192]; + int prefixlen = makePathPrefix(prefix, 8192); + + if (SMOnline()) + putOnline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen); + else + putOffline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen); + return 0; } - - diff --git a/storage-manager/src/smrm.cpp b/storage-manager/src/smrm.cpp index 6100116d5..176e1c84f 100644 --- a/storage-manager/src/smrm.cpp +++ b/storage-manager/src/smrm.cpp @@ -24,103 +24,103 @@ #include #include - using namespace std; using namespace storagemanager; -void usage(const char *progname) +void usage(const char* progname) { - cerr << progname << " is like 'rm -rf' for files managed by StorageManager, but with no options or globbing" << endl; - cerr << "Usage: " << progname << " file-or-dir1 file-or-dir2 .. file-or-dirN" << endl; + cerr << progname << " is like 'rm -rf' for files managed by StorageManager, but with no options or globbing" + << endl; + cerr << "Usage: " << progname << " file-or-dir1 file-or-dir2 .. file-or-dirN" << endl; } bool SMOnline() { - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... - int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); - int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr)); - if (err >= 0) - { - ::close(clientSocket); - return true; - } - return false; + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null... + int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); + int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr)); + if (err >= 0) + { + ::close(clientSocket); + return true; + } + return false; } #define min(x, y) (x < y ? x : y) -void rmOffline(int argCount, const char **args, const char *prefix, uint prefixlen) +void rmOffline(int argCount, const char** args, const char* prefix, uint prefixlen) { - try - { - boost::scoped_ptr ioc(IOCoordinator::get()); - char buf[16384]; - strncpy(buf, prefix, prefixlen); - - for (int i = 1; i < argCount; i++) - { - memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1); - ioc->unlink(buf); - } - } - catch (exception &e) - { - cerr << "smrm rmOffline FAIL: " << e.what() << endl; - } -} - -void rmOnline(int argCount, const char **args, const char *prefix, uint prefixlen) -{ - idbdatafile::SMFileSystem fs; + try + { + boost::scoped_ptr ioc(IOCoordinator::get()); char buf[16384]; strncpy(buf, prefix, prefixlen); - + for (int i = 1; i < argCount; i++) { - memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1); - fs.remove((char *) memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1)); + memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1); + ioc->unlink(buf); } + } + catch (exception& e) + { + cerr << "smrm rmOffline FAIL: " << e.what() << endl; + } } -int makePathPrefix(char *target, int targetlen) +void rmOnline(int argCount, const char** args, const char* prefix, uint prefixlen) { - // MCOL-3438 -> add bogus directories to the front of each param - Config *config = Config::get(); - int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); - target[0] = '/'; - target[1] = 0; - int bufpos = 1; - - for (int i = 0; i < prefixDepth; i++) - { - if (bufpos + 3 >= targetlen) - { - cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; - exit(1); - } - memcpy(&target[bufpos], "x/\0", 3); - bufpos += 2; - } - return bufpos; + idbdatafile::SMFileSystem fs; + char buf[16384]; + strncpy(buf, prefix, prefixlen); + + for (int i = 1; i < argCount; i++) + { + memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1); + fs.remove((char*)memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1)); + } } -int main(int argc, const char **argv) +int makePathPrefix(char* target, int targetlen) { - if (argc < 2) + // MCOL-3438 -> add bogus directories to the front of each param + Config* config = Config::get(); + int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth")); + target[0] = '/'; + target[1] = 0; + int bufpos = 1; + + for (int i = 0; i < prefixDepth; i++) + { + if (bufpos + 3 >= targetlen) { - usage(argv[0]); - return 1; + cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth"; + exit(1); } - - char prefix[8192]; - uint prefixlen = makePathPrefix(prefix, 8192); - - if (SMOnline()) - rmOnline(argc, argv, prefix, prefixlen); - else - rmOffline(argc, argv, prefix, prefixlen); - return 0; + memcpy(&target[bufpos], "x/\0", 3); + bufpos += 2; + } + return bufpos; +} + +int main(int argc, const char** argv) +{ + if (argc < 2) + { + usage(argv[0]); + return 1; + } + + char prefix[8192]; + uint prefixlen = makePathPrefix(prefix, 8192); + + if (SMOnline()) + rmOnline(argc, argv, prefix, prefixlen); + else + rmOffline(argc, argv, prefix, prefixlen); + return 0; } diff --git a/storage-manager/src/testS3Connection.cpp b/storage-manager/src/testS3Connection.cpp index a20d8d828..99e344cd6 100644 --- a/storage-manager/src/testS3Connection.cpp +++ b/storage-manager/src/testS3Connection.cpp @@ -24,45 +24,46 @@ using namespace storagemanager; using namespace std; -void printUsage() { - cout << "MariaDB Columnstore Storage Manager Test Configuration Connectivity.\n" << endl; - cout << "Usage: testS3Connection \n" << endl; - cout << "Returns Success=0 Failure=1\n" << endl; +void printUsage() +{ + cout << "MariaDB Columnstore Storage Manager Test Configuration Connectivity.\n" << endl; + cout << "Usage: testS3Connection \n" << endl; + cout << "Returns Success=0 Failure=1\n" << endl; } int s3TestConnection() { - S3Storage* s3 = NULL; - int ret = 0; - try - { - S3Storage* s3 = new S3Storage(true); - cout << "S3 Storage Manager Configuration OK" << endl; - delete s3; - } - catch (exception &e) - { - cout << "S3 Storage Manager Configuration Error:" << endl; - cout << e.what() << endl; - if (s3) - delete s3; - ret = 1; - } - return ret; + S3Storage* s3 = NULL; + int ret = 0; + try + { + S3Storage* s3 = new S3Storage(true); + cout << "S3 Storage Manager Configuration OK" << endl; + delete s3; + } + catch (exception& e) + { + cout << "S3 Storage Manager Configuration Error:" << endl; + cout << e.what() << endl; + if (s3) + delete s3; + ret = 1; + } + return ret; } int main(int argc, char* argv[]) { - int option; - while ((option = getopt(argc, argv, "h")) != EOF ) + int option; + while ((option = getopt(argc, argv, "h")) != EOF) + { + switch (option) { - switch (option) - { - case 'h': - default: - printUsage(); - return 0; - break; - } + case 'h': + default: + printUsage(); + return 0; + break; } - return s3TestConnection(); + } + return s3TestConnection(); } diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index bd3037a72..1864939f4 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -50,7 +50,6 @@ #include #include - #undef NDEBUG #include @@ -58,65 +57,71 @@ using namespace storagemanager; using namespace std; namespace bf = boost::filesystem; -void printUsage() { - cout << "MariaDB Columnstore Storage Manager Unit Test\n" << endl << - "Usage unit_test [OPTION] " << endl << - "-d [test_data] Location of test_data included with source code" << endl << - " Default = ./" << endl << - "-p [prefix] This directory will be used as scratch space for tests run" << endl << - " Default = unittest" << endl; +void printUsage() +{ + cout << "MariaDB Columnstore Storage Manager Unit Test\n" + << endl + << "Usage unit_test [OPTION] " << endl + << "-d [test_data] Location of test_data included with source code" << endl + << " Default = ./" << endl + << "-p [prefix] This directory will be used as scratch space for tests run" << endl + << " Default = unittest" << endl; } -struct scoped_closer { - scoped_closer(int f) : fd(f) { } - ~scoped_closer() { - int s_errno = errno; - ::close(fd); - errno = s_errno; - } - int fd; +struct scoped_closer +{ + scoped_closer(int f) : fd(f) + { + } + ~scoped_closer() + { + int s_errno = errno; + ::close(fd); + errno = s_errno; + } + int fd; }; // (ints) 0 1 2 3 ... 2048 -void makeTestObject(const char *dest) +void makeTestObject(const char* dest) { - int objFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); - assert(objFD >= 0); - scoped_closer s1(objFD); + int objFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); + assert(objFD >= 0); + scoped_closer s1(objFD); - for (int i = 0; i < 2048; i++) - assert(write(objFD, &i, 4) == 4); + for (int i = 0; i < 2048; i++) + assert(write(objFD, &i, 4) == 4); } // the merged version should look like // (ints) 0 1 2 3 4 0 1 2 3 4 10 11 12 13... -void makeTestJournal(const char *dest) +void makeTestJournal(const char* dest) { - int journalFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); - assert(journalFD >= 0); - scoped_closer s2(journalFD); + int journalFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); + assert(journalFD >= 0); + scoped_closer s2(journalFD); - char header[] = "{ \"version\" : 1, \"max_offset\": 39 }"; - size_t result = write(journalFD, header, strlen(header) + 1); - assert(result==(strlen(header) + 1)); - uint64_t offlen[2] = { 20, 20 }; - result = write(journalFD, offlen, 16); - for (int i = 0; i < 5; i++) - assert(write(journalFD, &i, 4) == 4); + char header[] = "{ \"version\" : 1, \"max_offset\": 39 }"; + size_t result = write(journalFD, header, strlen(header) + 1); + assert(result == (strlen(header) + 1)); + uint64_t offlen[2] = {20, 20}; + result = write(journalFD, offlen, 16); + for (int i = 0; i < 5; i++) + assert(write(journalFD, &i, 4) == 4); } bf::path testDirPath = "./"; bf::path homepath = getenv("HOME"); string prefix = "unittest"; -string testObjKey = "12345_0_8192_" + prefix +"~test-file"; -string copyfileObjKey = "12345_0_8192_" + prefix +"~source"; +string testObjKey = "12345_0_8192_" + prefix + "~test-file"; +string copyfileObjKey = "12345_0_8192_" + prefix + "~source"; string metaTestFile = prefix + "/test-file"; bf::path testFilePath = homepath / metaTestFile; const char* testFile = testFilePath.string().c_str(); string _metadata = -"{ \n\ + "{ \n\ \"version\" : 1, \n\ \"revision\" : 1, \n\ \"objects\" : \n\ @@ -129,459 +134,461 @@ string _metadata = ] \n\ }\n"; -void makeTestMetadata(const char *dest, string &key) +void makeTestMetadata(const char* dest, string& key) { - int metaFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); - assert(metaFD >= 0); - scoped_closer sc(metaFD); - boost::algorithm::replace_all(_metadata,"xxx",key); - // need to parameterize the object name in the objects list - size_t result = write(metaFD, _metadata.c_str(), _metadata.length()); - assert(result==_metadata.length()); - boost::algorithm::replace_all(_metadata,key,"xxx"); + int metaFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); + assert(metaFD >= 0); + scoped_closer sc(metaFD); + boost::algorithm::replace_all(_metadata, "xxx", key); + // need to parameterize the object name in the objects list + size_t result = write(metaFD, _metadata.c_str(), _metadata.length()); + assert(result == _metadata.length()); + boost::algorithm::replace_all(_metadata, key, "xxx"); } int getSocket() { - int sock = ::socket(AF_UNIX, SOCK_STREAM, 0); - assert(sock >= 0); - return sock; + int sock = ::socket(AF_UNIX, SOCK_STREAM, 0); + assert(sock >= 0); + return sock; } int sessionSock = -1; // tester uses this end of the connection int serverSock = -1; -int clientSock = -1; // have the Tasks use this end of the connection +int clientSock = -1; // have the Tasks use this end of the connection void acceptConnection() { - int err; - if (serverSock == -1) { - serverSock = getSocket(); + int err; + if (serverSock == -1) + { + serverSock = getSocket(); - struct sockaddr_un sa; - memset(&sa, 0, sizeof(sa)); - sa.sun_family = AF_UNIX; - memcpy(&sa.sun_path[1], "testing", 7); - - err = ::bind(serverSock, (struct sockaddr *) &sa, sizeof(sa)); - assert(err == 0); - err = ::listen(serverSock, 2); - assert(err == 0); - } - - sessionSock = ::accept(serverSock, NULL, NULL); - assert(sessionSock > 0); -} - -// connects sessionSock to clientSock -void makeConnection() -{ - boost::thread t(acceptConnection); struct sockaddr_un sa; memset(&sa, 0, sizeof(sa)); sa.sun_family = AF_UNIX; memcpy(&sa.sun_path[1], "testing", 7); - clientSock = ::socket(AF_UNIX, SOCK_STREAM, 0); - assert(clientSock > 0); - sleep(1); // let server thread get to accept() - int err = ::connect(clientSock, (struct sockaddr *) &sa, sizeof(sa)); + err = ::bind(serverSock, (struct sockaddr*)&sa, sizeof(sa)); assert(err == 0); - t.join(); + err = ::listen(serverSock, 2); + assert(err == 0); + } + + sessionSock = ::accept(serverSock, NULL, NULL); + assert(sessionSock > 0); } -bool opentask(bool connectionTest=false) +// connects sessionSock to clientSock +void makeConnection() { - // going to rely on msgs being smaller than the buffer here - int err=0; - uint8_t buf[1024]; - sm_msg_header *hdr = (sm_msg_header *) buf; - open_cmd *cmd = (open_cmd *) &hdr[1]; - string testFile = "metadataJournalTest"; - // open/create a file named 'opentest1' - std::string filename = homepath.string()+"/"+prefix+"/"+testFile; - hdr->type = SM_MSG_START; - hdr->flags = 0; - hdr->payloadLen = sizeof(*cmd) + filename.size(); - cmd->opcode = OPEN; - cmd->openmode = O_WRONLY | O_CREAT; - cmd->flen = 19; - strcpy((char *) cmd->filename, filename.c_str()); + boost::thread t(acceptConnection); + struct sockaddr_un sa; + memset(&sa, 0, sizeof(sa)); + sa.sun_family = AF_UNIX; + memcpy(&sa.sun_path[1], "testing", 7); - cout << "open file " << filename << endl; - ::unlink(filename.c_str()); + clientSock = ::socket(AF_UNIX, SOCK_STREAM, 0); + assert(clientSock > 0); + sleep(1); // let server thread get to accept() + int err = ::connect(clientSock, (struct sockaddr*)&sa, sizeof(sa)); + assert(err == 0); + t.join(); +} - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - hdr->payloadLen -= 2; +bool opentask(bool connectionTest = false) +{ + // going to rely on msgs being smaller than the buffer here + int err = 0; + uint8_t buf[1024]; + sm_msg_header* hdr = (sm_msg_header*)buf; + open_cmd* cmd = (open_cmd*)&hdr[1]; + string testFile = "metadataJournalTest"; + // open/create a file named 'opentest1' + std::string filename = homepath.string() + "/" + prefix + "/" + testFile; + hdr->type = SM_MSG_START; + hdr->flags = 0; + hdr->payloadLen = sizeof(*cmd) + filename.size(); + cmd->opcode = OPEN; + cmd->openmode = O_WRONLY | O_CREAT; + cmd->flen = 19; + strcpy((char*)cmd->filename, filename.c_str()); - size_t result = ::write(sessionSock, cmd, hdr->payloadLen); - assert(result==(hdr->payloadLen)); + cout << "open file " << filename << endl; + ::unlink(filename.c_str()); - // set payload to be correct length again - if (connectionTest) - hdr->payloadLen += 2; + // set payload to be shorter than actual message lengh + // and send a shortened message. + if (connectionTest) + hdr->payloadLen -= 2; - // process task will look for the full length and - // will wait on the rest of the message. - ProcessTask pt(clientSock, hdr->payloadLen); - boost::thread t(pt); + size_t result = ::write(sessionSock, cmd, hdr->payloadLen); + assert(result == (hdr->payloadLen)); - if (connectionTest) - { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - // read the response - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(struct stat) + sizeof(sm_response)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - struct stat *_stat = (struct stat *) resp->payload; + // set payload to be correct length again + if (connectionTest) + hdr->payloadLen += 2; - // what can we verify about the stat... - assert(_stat->st_size == 0); - /* verify the file is there */ - string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); - assert(!metaPath.empty()); - metaPath += string("/" + prefix + "/" + testFile + ".meta"); + // process task will look for the full length and + // will wait on the rest of the message. + ProcessTask pt(clientSock, hdr->payloadLen); + boost::thread t(pt); - assert(boost::filesystem::exists(metaPath)); - } + if (connectionTest) + { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else + { + t.join(); + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err == sizeof(struct stat) + sizeof(sm_response)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 0); + struct stat* _stat = (struct stat*)resp->payload; - cout << "opentask OK" << endl; - return true; + // what can we verify about the stat... + assert(_stat->st_size == 0); + /* verify the file is there */ + string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); + assert(!metaPath.empty()); + metaPath += string("/" + prefix + "/" + testFile + ".meta"); + + assert(boost::filesystem::exists(metaPath)); + } + + cout << "opentask OK" << endl; + return true; } bool replicatorTest() { - Config* config = Config::get(); - string metaPath = config->getValue("ObjectStorage", "metadata_path"); - string journalPath = config->getValue("ObjectStorage", "journal_path"); - string cacehPath = config->getValue("Cache", "path"); + Config* config = Config::get(); + string metaPath = config->getValue("ObjectStorage", "metadata_path"); + string journalPath = config->getValue("ObjectStorage", "journal_path"); + string cacehPath = config->getValue("Cache", "path"); - Replicator *repli = Replicator::get(); - int err,fd; - string newobject = prefix + "/newobjectTest"; - string newObjectJournalFullPath = journalPath + "/" + prefix + "/newobjectTest.journal"; - string newObjectCacheFullPath = cacehPath + "/" + prefix + "/newobjectTest"; - uint8_t buf[1024]; - uint8_t data[1024]; - int version = 1; - uint64_t max_offset = 0; - memcpy(data,"1234567890",10); - string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % max_offset).str(); + Replicator* repli = Replicator::get(); + int err, fd; + string newobject = prefix + "/newobjectTest"; + string newObjectJournalFullPath = journalPath + "/" + prefix + "/newobjectTest.journal"; + string newObjectCacheFullPath = cacehPath + "/" + prefix + "/newobjectTest"; + uint8_t buf[1024]; + uint8_t data[1024]; + int version = 1; + uint64_t max_offset = 0; + memcpy(data, "1234567890", 10); + string header = + (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % max_offset).str(); - // test newObject - repli->newObject(newobject,data,0,10); + // test newObject + repli->newObject(newobject, data, 0, 10); - //check file contents - fd = ::open(newObjectCacheFullPath.c_str(), O_RDONLY); - err = ::read(fd, buf, sizeof(buf)); - assert(err == 10); - buf[10] = 0; - assert(!strcmp("1234567890", (const char *) buf)); - cout << "replicator newObject OK" << endl; - ::close(fd); + // check file contents + fd = ::open(newObjectCacheFullPath.c_str(), O_RDONLY); + err = ::read(fd, buf, sizeof(buf)); + assert(err == 10); + buf[10] = 0; + assert(!strcmp("1234567890", (const char*)buf)); + cout << "replicator newObject OK" << endl; + ::close(fd); - // test addJournalEntry - repli->addJournalEntry(newobject,data,0,10); + // test addJournalEntry + repli->addJournalEntry(newobject, data, 0, 10); - fd = ::open(newObjectJournalFullPath.c_str(), O_RDONLY); - err = ::read(fd, buf, sizeof(buf)); - assert((uint) err == (header.length() + 1 + 16 + 10)); - buf[err] = 0; - assert(!strcmp("1234567890", (const char *) buf + header.length() + 1 + 16)); - cout << "replicator addJournalEntry OK" << endl; - ::close(fd); + fd = ::open(newObjectJournalFullPath.c_str(), O_RDONLY); + err = ::read(fd, buf, sizeof(buf)); + assert((uint)err == (header.length() + 1 + 16 + 10)); + buf[err] = 0; + assert(!strcmp("1234567890", (const char*)buf + header.length() + 1 + 16)); + cout << "replicator addJournalEntry OK" << endl; + ::close(fd); - repli->remove(newObjectCacheFullPath.c_str()); - repli->remove(newObjectJournalFullPath.c_str()); - assert(!boost::filesystem::exists(newObjectCacheFullPath.c_str())); - cout << "replicator remove OK" << endl; - return true; + repli->remove(newObjectCacheFullPath.c_str()); + repli->remove(newObjectJournalFullPath.c_str()); + assert(!boost::filesystem::exists(newObjectCacheFullPath.c_str())); + cout << "replicator remove OK" << endl; + return true; } void metadataJournalTest(std::size_t size, off_t offset) { - // make an empty file to write to - bf::path fullPath = homepath / prefix / "metadataJournalTest"; - const char *filename = fullPath.string().c_str(); - std::vector buf(sizeof(write_cmd)+std::strlen(filename)+size); - uint64_t *data; + // make an empty file to write to + bf::path fullPath = homepath / prefix / "metadataJournalTest"; + const char* filename = fullPath.string().c_str(); + std::vector buf(sizeof(write_cmd) + std::strlen(filename) + size); + uint64_t* data; - sm_msg_header *hdr = (sm_msg_header *) buf.data(); - write_cmd *cmd = (write_cmd *) &hdr[1]; - cmd->opcode = WRITE; - cmd->offset = offset; - cmd->count = size; - cmd->flen = std::strlen(filename); - memcpy(&cmd->filename, filename, cmd->flen); - data = (uint64_t *) &cmd->filename[cmd->flen]; - int count = 0; - for (uint64_t i = 0; i < (size/sizeof(uint64_t)); i++) - { - data[i] = i; - count++; - } - hdr->type = SM_MSG_START; - hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; - WriteTask w(clientSock, hdr->payloadLen); - int err = ::write(sessionSock, cmd, hdr->payloadLen); + sm_msg_header* hdr = (sm_msg_header*)buf.data(); + write_cmd* cmd = (write_cmd*)&hdr[1]; + cmd->opcode = WRITE; + cmd->offset = offset; + cmd->count = size; + cmd->flen = std::strlen(filename); + memcpy(&cmd->filename, filename, cmd->flen); + data = (uint64_t*)&cmd->filename[cmd->flen]; + int count = 0; + for (uint64_t i = 0; i < (size / sizeof(uint64_t)); i++) + { + data[i] = i; + count++; + } + hdr->type = SM_MSG_START; + hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; + WriteTask w(clientSock, hdr->payloadLen); + int err = ::write(sessionSock, cmd, hdr->payloadLen); - w.run(); + w.run(); - // verify response - uint8_t bufRead[1024]; - err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); - sm_response *resp = (sm_response *) bufRead; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == (int) size); + // verify response + uint8_t bufRead[1024]; + err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + sm_response* resp = (sm_response*)bufRead; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == (int)size); } void metadataJournalTest_append(std::size_t size) { - // make an empty file to write to - bf::path fullPath = homepath / prefix / "metadataJournalTest"; - const char *filename = fullPath.string().c_str(); + // make an empty file to write to + bf::path fullPath = homepath / prefix / "metadataJournalTest"; + const char* filename = fullPath.string().c_str(); - std::vector buf(sizeof(write_cmd)+std::strlen(filename)+size); - uint64_t *data; + std::vector buf(sizeof(write_cmd) + std::strlen(filename) + size); + uint64_t* data; - sm_msg_header *hdr = (sm_msg_header *) buf.data(); - append_cmd *cmd = (append_cmd *) &hdr[1]; - cmd->opcode = APPEND; - cmd->count = size; - cmd->flen = std::strlen(filename); - memcpy(&cmd->filename, filename, cmd->flen); - data = (uint64_t *) &cmd->filename[cmd->flen]; - int count = 0; - for (uint64_t i = 0; i < (size/sizeof(uint64_t)); i++) - { - data[i] = i; - count++; - } - hdr->type = SM_MSG_START; - hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; - AppendTask a(clientSock, hdr->payloadLen); - int err = ::write(sessionSock, cmd, hdr->payloadLen); + sm_msg_header* hdr = (sm_msg_header*)buf.data(); + append_cmd* cmd = (append_cmd*)&hdr[1]; + cmd->opcode = APPEND; + cmd->count = size; + cmd->flen = std::strlen(filename); + memcpy(&cmd->filename, filename, cmd->flen); + data = (uint64_t*)&cmd->filename[cmd->flen]; + int count = 0; + for (uint64_t i = 0; i < (size / sizeof(uint64_t)); i++) + { + data[i] = i; + count++; + } + hdr->type = SM_MSG_START; + hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; + AppendTask a(clientSock, hdr->payloadLen); + int err = ::write(sessionSock, cmd, hdr->payloadLen); - a.run(); + a.run(); - // verify response - uint8_t bufRead[1024]; - err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); - sm_response *resp = (sm_response *) bufRead; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == (int) size); + // verify response + uint8_t bufRead[1024]; + err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + sm_response* resp = (sm_response*)bufRead; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == (int)size); } void metadataJournalTestCleanup() { - IOCoordinator *ioc = IOCoordinator::get(); - Synchronizer *sync = Synchronizer::get(); - bf::path fullPath = homepath / prefix / "metadataJournalTest"; - ioc->unlink(fullPath.string().c_str()); - sync->forceFlush(); + IOCoordinator* ioc = IOCoordinator::get(); + Synchronizer* sync = Synchronizer::get(); + bf::path fullPath = homepath / prefix / "metadataJournalTest"; + ioc->unlink(fullPath.string().c_str()); + sync->forceFlush(); } bool writetask() { - // make an empty file to write to - bf::path fullPath = homepath / prefix / "writetest1"; - const char *filename = fullPath.string().c_str(); - ::unlink(filename); - int fd = ::open(filename, O_CREAT | O_RDWR, 0666); - assert(fd > 0); - scoped_closer f(fd); + // make an empty file to write to + bf::path fullPath = homepath / prefix / "writetest1"; + const char* filename = fullPath.string().c_str(); + ::unlink(filename); + int fd = ::open(filename, O_CREAT | O_RDWR, 0666); + assert(fd > 0); + scoped_closer f(fd); - uint8_t buf[1024]; - sm_msg_header *hdr = (sm_msg_header *) buf; - write_cmd *cmd = (write_cmd *) &hdr[1]; - uint8_t *data; + uint8_t buf[1024]; + sm_msg_header* hdr = (sm_msg_header*)buf; + write_cmd* cmd = (write_cmd*)&hdr[1]; + uint8_t* data; - cmd->opcode = WRITE; - cmd->offset = 0; - cmd->count = 9; - cmd->flen = 10; - memcpy(&cmd->filename, filename, cmd->flen); - data = (uint8_t *) &cmd->filename[cmd->flen]; - memcpy(data, "123456789", cmd->count); + cmd->opcode = WRITE; + cmd->offset = 0; + cmd->count = 9; + cmd->flen = 10; + memcpy(&cmd->filename, filename, cmd->flen); + data = (uint8_t*)&cmd->filename[cmd->flen]; + memcpy(data, "123456789", cmd->count); - hdr->type = SM_MSG_START; - hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; + hdr->type = SM_MSG_START; + hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; - WriteTask w(clientSock, hdr->payloadLen); - ssize_t result = ::write(sessionSock, cmd, hdr->payloadLen); - assert(result == static_cast(hdr->payloadLen)); + WriteTask w(clientSock, hdr->payloadLen); + ssize_t result = ::write(sessionSock, cmd, hdr->payloadLen); + assert(result == static_cast(hdr->payloadLen)); - w.run(); + w.run(); - // verify response - int err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 9); + // verify response + int err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 9); - //check file contents - err = ::read(fd, buf, sizeof(buf)); - assert(err == 9); - buf[9] = 0; - assert(!strcmp("123456789", (const char *) buf)); - ::unlink(filename); - cout << "write task OK" << endl; - return true; + // check file contents + err = ::read(fd, buf, sizeof(buf)); + assert(err == 9); + buf[9] = 0; + assert(!strcmp("123456789", (const char*)buf)); + ::unlink(filename); + cout << "write task OK" << endl; + return true; } bool appendtask() { - // make a file and put some stuff in it - bf::path fullPath = homepath / prefix / "appendtest1"; - const char *filename = fullPath.string().c_str(); - ::unlink(filename); - int fd = ::open(filename, O_CREAT | O_RDWR, 0666); - assert(fd > 0); - scoped_closer f(fd); - int err = ::write(fd, "testjunk", 8); - assert(err == 8); + // make a file and put some stuff in it + bf::path fullPath = homepath / prefix / "appendtest1"; + const char* filename = fullPath.string().c_str(); + ::unlink(filename); + int fd = ::open(filename, O_CREAT | O_RDWR, 0666); + assert(fd > 0); + scoped_closer f(fd); + int err = ::write(fd, "testjunk", 8); + assert(err == 8); - uint8_t buf[1024]; - append_cmd *cmd = (append_cmd *) buf; - uint8_t *data; + uint8_t buf[1024]; + append_cmd* cmd = (append_cmd*)buf; + uint8_t* data; - cmd->opcode = APPEND; - cmd->count = 9; - cmd->flen = 11; - memcpy(&cmd->filename, filename, cmd->flen); - data = (uint8_t *) &cmd->filename[cmd->flen]; - memcpy(data, "123456789", cmd->count); + cmd->opcode = APPEND; + cmd->count = 9; + cmd->flen = 11; + memcpy(&cmd->filename, filename, cmd->flen); + data = (uint8_t*)&cmd->filename[cmd->flen]; + memcpy(data, "123456789", cmd->count); - int payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; + int payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; - AppendTask a(clientSock, payloadLen); - ssize_t result = ::write(sessionSock, cmd, payloadLen); - assert(result==(payloadLen)); + AppendTask a(clientSock, payloadLen); + ssize_t result = ::write(sessionSock, cmd, payloadLen); + assert(result == (payloadLen)); - a.run(); + a.run(); - // verify response + // verify response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 9); + + // check file contents + ::lseek(fd, 0, SEEK_SET); + err = ::read(fd, buf, sizeof(buf)); + assert(err == 17); + buf[17] = 0; + assert(!strcmp("testjunk123456789", (const char*)buf)); + ::unlink(filename); + cout << "append task OK" << endl; + return true; +} + +void unlinktask(bool connectionTest = false) +{ + int err = 0; + // make a meta file and delete it + bf::path fullPath = homepath / prefix / "unlinktest1"; + string pathMeta = prefix + "/unlinktest1"; + const char* Metafilename = pathMeta.c_str(); + const char* filename = fullPath.string().c_str(); + + IOCoordinator* ioc = IOCoordinator::get(); + bf::path fullPathMeta = ioc->getMetadataPath() / (string(Metafilename) + ".meta"); + bf::remove(fullPathMeta); + + MetadataFile meta(Metafilename); + meta.writeMetadata(); + + assert(bf::exists(fullPathMeta)); + + uint8_t buf[1024]; + unlink_cmd* cmd = (unlink_cmd*)buf; + + cmd->opcode = UNLINK; + cmd->flen = strlen(filename); + memcpy(&cmd->filename, filename, cmd->flen); + + // set payload to be shorter than actual message lengh + // and send a shortened message. + if (connectionTest) + cmd->flen -= 2; + + size_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); + assert(result == (sizeof(unlink_cmd) + cmd->flen)); + + // set payload to be correct length again + if (connectionTest) + cmd->flen += 2; + + // process task will look for the full length and + // will wait on the rest of the message. + ProcessTask pt(clientSock, sizeof(unlink_cmd) + cmd->flen); + boost::thread t(pt); + + if (connectionTest) + { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path + sleep(1); + close(sessionSock); + close(clientSock); err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; + assert(err == -1); + t.join(); + } + else + { + t.join(); + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; assert(err == sizeof(*resp)); assert(resp->header.type == SM_MSG_START); assert(resp->header.payloadLen == sizeof(ssize_t)); assert(resp->header.flags == 0); - assert(resp->returnCode == 9); + assert(resp->returnCode == 0); + // confirm it no longer exists + assert(!bf::exists(fullPathMeta)); + } - //check file contents - ::lseek(fd, 0, SEEK_SET); - err = ::read(fd, buf, sizeof(buf)); - assert(err == 17); - buf[17] = 0; - assert(!strcmp("testjunk123456789", (const char *) buf)); - ::unlink(filename); - cout << "append task OK" << endl; - return true; -} - -void unlinktask(bool connectionTest=false) -{ - int err=0; - // make a meta file and delete it - bf::path fullPath = homepath / prefix / "unlinktest1"; - string pathMeta = prefix + "/unlinktest1"; - const char *Metafilename = pathMeta.c_str(); - const char *filename = fullPath.string().c_str(); - - IOCoordinator *ioc = IOCoordinator::get(); - bf::path fullPathMeta = ioc->getMetadataPath()/(string(Metafilename) + ".meta"); - bf::remove(fullPathMeta); - - MetadataFile meta(Metafilename); - meta.writeMetadata(); - - assert(bf::exists(fullPathMeta)); - - uint8_t buf[1024]; - unlink_cmd *cmd = (unlink_cmd *) buf; - - cmd->opcode = UNLINK; - cmd->flen = strlen(filename); - memcpy(&cmd->filename, filename, cmd->flen); - - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - cmd->flen -= 2; - - size_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); - assert(result==(sizeof(unlink_cmd) + cmd->flen)); - - // set payload to be correct length again - if (connectionTest) - cmd->flen += 2; - - // process task will look for the full length and - // will wait on the rest of the message. - ProcessTask pt(clientSock, sizeof(unlink_cmd) + cmd->flen); - boost::thread t(pt); - - if (connectionTest) - { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - // read the response - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - // confirm it no longer exists - assert(!bf::exists(fullPathMeta)); - } - - // delete it again, make sure we get an error message & reasonable error code - // Interesting. boost::filesystem::remove() doesn't consider it an error if the file doesn't - // exist. Need to look into the reasoning for that, and decide whether IOC - // should return an error anyway. For now, this test below doesn't get - // an error msg. - #if 0 +// delete it again, make sure we get an error message & reasonable error code +// Interesting. boost::filesystem::remove() doesn't consider it an error if the file doesn't +// exist. Need to look into the reasoning for that, and decide whether IOC +// should return an error anyway. For now, this test below doesn't get +// an error msg. +#if 0 memset(buf, 0, 1024); cmd->opcode = UNLINK; cmd->flen = strlen(filename); @@ -603,1139 +610,1137 @@ void unlinktask(bool connectionTest=false) assert(resp->returnCode == -1); err = (*(int *) resp->payload); assert(err == ENOENT); - #endif +#endif - cout << "unlink task OK" << endl; + cout << "unlink task OK" << endl; } -bool stattask(bool connectionTest=false) +bool stattask(bool connectionTest = false) { - int err=0; - bf::path fullPath = homepath / prefix / "stattest1"; - string filename = fullPath.string(); - string Metafilename = prefix + "/stattest1"; + int err = 0; + bf::path fullPath = homepath / prefix / "stattest1"; + string filename = fullPath.string(); + string Metafilename = prefix + "/stattest1"; - string fullFilename = Config::get()->getValue("ObjectStorage", "metadata_path") + "/" + Metafilename + ".meta"; + string fullFilename = + Config::get()->getValue("ObjectStorage", "metadata_path") + "/" + Metafilename + ".meta"; - ::unlink(fullFilename.c_str()); - makeTestMetadata(fullFilename.c_str(),testObjKey); + ::unlink(fullFilename.c_str()); + makeTestMetadata(fullFilename.c_str(), testObjKey); - uint8_t buf[1024]; - stat_cmd *cmd = (stat_cmd *) buf; + uint8_t buf[1024]; + stat_cmd* cmd = (stat_cmd*)buf; - cmd->opcode = STAT; - cmd->flen = filename.length(); - strcpy((char *) cmd->filename, filename.c_str()); + cmd->opcode = STAT; + cmd->flen = filename.length(); + strcpy((char*)cmd->filename, filename.c_str()); - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - cmd->flen -= 2; + // set payload to be shorter than actual message lengh + // and send a shortened message. + if (connectionTest) + cmd->flen -= 2; - size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); - assert(result==(sizeof(*cmd) + cmd->flen)); + size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); + assert(result == (sizeof(*cmd) + cmd->flen)); - // set payload to be correct length again - if (connectionTest) - cmd->flen += 2; + // set payload to be correct length again + if (connectionTest) + cmd->flen += 2; - // process task will look for the full length and - // will wait on the rest of the message. - ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); - boost::thread t(pt); + // process task will look for the full length and + // will wait on the rest of the message. + ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); + boost::thread t(pt); - if (connectionTest) - { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - // read the response - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(struct stat) + sizeof(sm_response)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.flags == 0); - assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t)); - assert(resp->returnCode == 0); - struct stat *_stat = (struct stat *) resp->payload; + if (connectionTest) + { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else + { + t.join(); + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err == sizeof(struct stat) + sizeof(sm_response)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.flags == 0); + assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t)); + assert(resp->returnCode == 0); + struct stat* _stat = (struct stat*)resp->payload; - // what can we verify about the stat... - assert(_stat->st_uid == getuid()); - assert(_stat->st_gid == getgid()); - assert(_stat->st_size == 8192); - } + // what can we verify about the stat... + assert(_stat->st_uid == getuid()); + assert(_stat->st_gid == getgid()); + assert(_stat->st_size == 8192); + } - ::unlink(fullFilename.c_str()); - cout << "stattask OK" << endl; - return true; + ::unlink(fullFilename.c_str()); + cout << "stattask OK" << endl; + return true; } bool IOCTruncate() { - - IOCoordinator *ioc = IOCoordinator::get(); - CloudStorage *cs = CloudStorage::get(); - Synchronizer *sync = Synchronizer::get(); - LocalStorage *ls = dynamic_cast(cs); - if (!ls) - { - cout << "IOCTruncate() currently requires using Local storage" << endl; - return true; - } - Cache *cache = Cache::get(); - cache->reset(); - - bf::path cachePath = ioc->getCachePath(); - bf::path journalPath = ioc->getJournalPath(); - bf::path metaPath = ioc->getMetadataPath(); - bf::path cloudPath = ls->getPrefix(); - - // metaPath doesn't necessarily exist until a MetadataFile instance is created - bf::create_directories(metaPath); - - /* start with one object in cloud storage - truncate past the end of the object - verify nothing changed & got success - truncate at 4000 bytes - verify everything sees the 'file' as 4000 bytes - - IOC + meta - truncate at 0 bytes - verify file now looks empty - verify the object was deleted - - add 2 8k test objects and a journal against the second one - truncate @ 10000 bytes - verify all files still exist - truncate @ 6000 bytes, 2nd object & journal were deleted - truncate @ 0 bytes, verify no files are left - */ - - bf::path metadataFile = metaPath/prefix/"test-file.meta"; - bf::path objectPath = cloudPath/testObjKey; - bf::path cachedObjectPath = cachePath/prefix/testObjKey; - makeTestMetadata(metadataFile.string().c_str(),testObjKey); - makeTestObject(objectPath.string().c_str()); - - int err; - uint8_t buf[1<<14]; - int *buf32 = (int *) buf; - - /* Need to enable this later. - // Extend the test file to 10000 bytes - err = ioc->truncate(testFile, 10000); - assert(!err); - err = ioc->read(testFile, buf, 0, 10000); - assert(err == 10000); - // verity the data is what it should be - for (int i = 0; i < 2048; i++) - assert(buf32[i] == i); - for (int i = 2048; i < 2500; i++) - assert(buf32[i] == 0); - */ - - err = ioc->truncate(testFile, 4000); - assert(!err); - MetadataFile meta(metaTestFile); - assert(meta.getLength() == 4000); - - // read the data, make sure there are only 4000 bytes & the object still exists - err = ioc->read(testFile, buf, 0, 8192); - assert(err == 4000); - err = ioc->read(testFile, buf, 4000, 1); - assert(err == 0); - err = ioc->read(testFile, buf, 4005, 1); - assert(err == 0); - assert(bf::exists(objectPath)); - - // truncate to 0 bytes, make sure everything is consistent with that, and the object no longer exists - err = ioc->truncate(testFile, 0); - assert(!err); - meta = MetadataFile(metaTestFile); - assert(meta.getLength() == 0); - err = ioc->read(testFile, buf, 0, 1); - assert(err == 0); - err = ioc->read(testFile, buf, 4000, 1); - assert(err == 0); - sync->forceFlush(); - sleep(1); // give Sync a chance to delete the object from the cloud - assert(!bf::exists(objectPath)); - - // recreate the meta file, make a 2-object version - ioc->unlink(testFile); - makeTestMetadata(metadataFile.string().c_str(),testObjKey); - makeTestObject(objectPath.string().c_str()); - - meta = MetadataFile(metaTestFile); - bf::path secondObjectPath = cloudPath / meta.addMetadataObject(testFile, 8192).key; - bf::path cachedSecondObject = cachePath/prefix/secondObjectPath.filename(); - makeTestObject(secondObjectPath.string().c_str()); - meta.writeMetadata(); - - // make sure there are 16k bytes, and the data is valid before going forward - memset(buf, 0, sizeof(buf)); - err = ioc->read(testFile, buf, 0, sizeof(buf)); - assert(err == sizeof(buf)); - for (int i = 0; i < (int)sizeof(buf)/4; i++) - assert(buf32[i] == (i % 2048)); - assert(bf::exists(cachedSecondObject)); - assert(bf::exists(cachedObjectPath)); - - // truncate to 10k, make sure everything looks right - err = ioc->truncate(testFile, 10240); - assert(!err); - meta = MetadataFile(metaTestFile); - assert(meta.getLength() == 10240); - memset(buf, 0, sizeof(buf)); - err = ioc->read(testFile, buf, 0, 10240); - for (int i = 0; i < 10240/4; i++) - assert(buf32[i] == (i % 2048)); - err = ioc->read(testFile, buf, 10239, 10); - assert(err == 1); - - // truncate to 6000 bytes, make sure second object got deleted - err = ioc->truncate(testFile, 6000); - meta = MetadataFile(metaTestFile); - assert(meta.getLength() == 6000); - err = ioc->read(testFile, buf, 0, 8192); - assert(err == 6000); - sync->forceFlush(); - sleep(1); // give Synchronizer a chance to delete the file from the 'cloud' - assert(!bf::exists(secondObjectPath)); - assert(!bf::exists(cachedSecondObject)); - - cache->reset(); - ioc->unlink(testFile); - - cout << "IOCTruncate OK" << endl; + IOCoordinator* ioc = IOCoordinator::get(); + CloudStorage* cs = CloudStorage::get(); + Synchronizer* sync = Synchronizer::get(); + LocalStorage* ls = dynamic_cast(cs); + if (!ls) + { + cout << "IOCTruncate() currently requires using Local storage" << endl; return true; + } + Cache* cache = Cache::get(); + cache->reset(); + + bf::path cachePath = ioc->getCachePath(); + bf::path journalPath = ioc->getJournalPath(); + bf::path metaPath = ioc->getMetadataPath(); + bf::path cloudPath = ls->getPrefix(); + + // metaPath doesn't necessarily exist until a MetadataFile instance is created + bf::create_directories(metaPath); + + /* start with one object in cloud storage + truncate past the end of the object + verify nothing changed & got success + truncate at 4000 bytes + verify everything sees the 'file' as 4000 bytes + - IOC + meta + truncate at 0 bytes + verify file now looks empty + verify the object was deleted + + add 2 8k test objects and a journal against the second one + truncate @ 10000 bytes + verify all files still exist + truncate @ 6000 bytes, 2nd object & journal were deleted + truncate @ 0 bytes, verify no files are left + */ + + bf::path metadataFile = metaPath / prefix / "test-file.meta"; + bf::path objectPath = cloudPath / testObjKey; + bf::path cachedObjectPath = cachePath / prefix / testObjKey; + makeTestMetadata(metadataFile.string().c_str(), testObjKey); + makeTestObject(objectPath.string().c_str()); + + int err; + uint8_t buf[1 << 14]; + int* buf32 = (int*)buf; + + /* Need to enable this later. + // Extend the test file to 10000 bytes + err = ioc->truncate(testFile, 10000); + assert(!err); + err = ioc->read(testFile, buf, 0, 10000); + assert(err == 10000); + // verity the data is what it should be + for (int i = 0; i < 2048; i++) + assert(buf32[i] == i); + for (int i = 2048; i < 2500; i++) + assert(buf32[i] == 0); + */ + + err = ioc->truncate(testFile, 4000); + assert(!err); + MetadataFile meta(metaTestFile); + assert(meta.getLength() == 4000); + + // read the data, make sure there are only 4000 bytes & the object still exists + err = ioc->read(testFile, buf, 0, 8192); + assert(err == 4000); + err = ioc->read(testFile, buf, 4000, 1); + assert(err == 0); + err = ioc->read(testFile, buf, 4005, 1); + assert(err == 0); + assert(bf::exists(objectPath)); + + // truncate to 0 bytes, make sure everything is consistent with that, and the object no longer exists + err = ioc->truncate(testFile, 0); + assert(!err); + meta = MetadataFile(metaTestFile); + assert(meta.getLength() == 0); + err = ioc->read(testFile, buf, 0, 1); + assert(err == 0); + err = ioc->read(testFile, buf, 4000, 1); + assert(err == 0); + sync->forceFlush(); + sleep(1); // give Sync a chance to delete the object from the cloud + assert(!bf::exists(objectPath)); + + // recreate the meta file, make a 2-object version + ioc->unlink(testFile); + makeTestMetadata(metadataFile.string().c_str(), testObjKey); + makeTestObject(objectPath.string().c_str()); + + meta = MetadataFile(metaTestFile); + bf::path secondObjectPath = cloudPath / meta.addMetadataObject(testFile, 8192).key; + bf::path cachedSecondObject = cachePath / prefix / secondObjectPath.filename(); + makeTestObject(secondObjectPath.string().c_str()); + meta.writeMetadata(); + + // make sure there are 16k bytes, and the data is valid before going forward + memset(buf, 0, sizeof(buf)); + err = ioc->read(testFile, buf, 0, sizeof(buf)); + assert(err == sizeof(buf)); + for (int i = 0; i < (int)sizeof(buf) / 4; i++) + assert(buf32[i] == (i % 2048)); + assert(bf::exists(cachedSecondObject)); + assert(bf::exists(cachedObjectPath)); + + // truncate to 10k, make sure everything looks right + err = ioc->truncate(testFile, 10240); + assert(!err); + meta = MetadataFile(metaTestFile); + assert(meta.getLength() == 10240); + memset(buf, 0, sizeof(buf)); + err = ioc->read(testFile, buf, 0, 10240); + for (int i = 0; i < 10240 / 4; i++) + assert(buf32[i] == (i % 2048)); + err = ioc->read(testFile, buf, 10239, 10); + assert(err == 1); + + // truncate to 6000 bytes, make sure second object got deleted + err = ioc->truncate(testFile, 6000); + meta = MetadataFile(metaTestFile); + assert(meta.getLength() == 6000); + err = ioc->read(testFile, buf, 0, 8192); + assert(err == 6000); + sync->forceFlush(); + sleep(1); // give Synchronizer a chance to delete the file from the 'cloud' + assert(!bf::exists(secondObjectPath)); + assert(!bf::exists(cachedSecondObject)); + + cache->reset(); + ioc->unlink(testFile); + + cout << "IOCTruncate OK" << endl; + return true; } - -bool truncatetask(bool connectionTest=false) +bool truncatetask(bool connectionTest = false) { - IOCoordinator *ioc = IOCoordinator::get(); - Cache *cache = Cache::get(); + IOCoordinator* ioc = IOCoordinator::get(); + Cache* cache = Cache::get(); - bf::path metaPath = ioc->getMetadataPath(); - bf::path fullPath = homepath / prefix / "trunctest1"; - string metaStr = prefix + "/trunctest1"; - const char *filename = fullPath.string().c_str(); - const char *Metafilename = metaStr.c_str(); - int err=0; - // get the metafile created - string metaFullName = (metaPath/Metafilename).string() + ".meta"; - ::unlink(metaFullName.c_str()); - MetadataFile meta(Metafilename); + bf::path metaPath = ioc->getMetadataPath(); + bf::path fullPath = homepath / prefix / "trunctest1"; + string metaStr = prefix + "/trunctest1"; + const char* filename = fullPath.string().c_str(); + const char* Metafilename = metaStr.c_str(); + int err = 0; + // get the metafile created + string metaFullName = (metaPath / Metafilename).string() + ".meta"; + ::unlink(metaFullName.c_str()); + MetadataFile meta(Metafilename); - uint8_t buf[1024]; - truncate_cmd *cmd = (truncate_cmd *) buf; + uint8_t buf[1024]; + truncate_cmd* cmd = (truncate_cmd*)buf; - cmd->opcode = TRUNCATE; - cmd->length = 1000; - cmd->flen = strlen(filename); - strcpy((char *) cmd->filename, filename); + cmd->opcode = TRUNCATE; + cmd->length = 1000; + cmd->flen = strlen(filename); + strcpy((char*)cmd->filename, filename); - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - cmd->flen -= 2; + // set payload to be shorter than actual message lengh + // and send a shortened message. + if (connectionTest) + cmd->flen -= 2; - size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); - assert(result==(sizeof(*cmd) + cmd->flen)); + size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); + assert(result == (sizeof(*cmd) + cmd->flen)); - // set payload to be correct length again - if (connectionTest) - cmd->flen += 2; + // set payload to be correct length again + if (connectionTest) + cmd->flen += 2; - // process task will look for the full length and - // will wait on the rest of the message. - ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); - boost::thread t(pt); + // process task will look for the full length and + // will wait on the rest of the message. + ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); + boost::thread t(pt); - if (connectionTest) - { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - // read the response - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(sm_response)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.flags == 0); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->returnCode == 0); - // reload the metadata, check that it is 1000 bytes - meta = MetadataFile(Metafilename); - assert(meta.getLength() == 1000); - } + if (connectionTest) + { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else + { + t.join(); + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err == sizeof(sm_response)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.flags == 0); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->returnCode == 0); + // reload the metadata, check that it is 1000 bytes + meta = MetadataFile(Metafilename); + assert(meta.getLength() == 1000); + } - cache->reset(); - ::unlink(metaFullName.c_str()); - cout << "truncate task OK" << endl; - return true; + cache->reset(); + ::unlink(metaFullName.c_str()); + cout << "truncate task OK" << endl; + return true; } -bool listdirtask(bool connectionTest=false) +bool listdirtask(bool connectionTest = false) { - IOCoordinator *ioc = IOCoordinator::get(); - const bf::path metaPath = ioc->getMetadataPath(); - bf::path fullPath = homepath / prefix / "listdirtask"; - string metaStr = prefix + "/listdirtask"; - const char *relPath = fullPath.string().c_str(); - const char *MetarelPath = metaStr.c_str(); + IOCoordinator* ioc = IOCoordinator::get(); + const bf::path metaPath = ioc->getMetadataPath(); + bf::path fullPath = homepath / prefix / "listdirtask"; + string metaStr = prefix + "/listdirtask"; + const char* relPath = fullPath.string().c_str(); + const char* MetarelPath = metaStr.c_str(); - bf::path tmpPath = metaPath/MetarelPath; + bf::path tmpPath = metaPath / MetarelPath; - // make some dummy files, make sure they are in the list returned. - set files; - int err; - vector fdMinders; + // make some dummy files, make sure they are in the list returned. + set files; + int err; + vector fdMinders; - bf::create_directories(tmpPath); - for (int i = 0; i < 10; i++) { - string file(tmpPath.string() + "/dummy" + to_string(i)); - files.insert(file); - file += ".meta"; - err = ::open(file.c_str(), O_CREAT | O_WRONLY, 0600); - assert(err >= 0); - fdMinders.push_back(err); - } + bf::create_directories(tmpPath); + for (int i = 0; i < 10; i++) + { + string file(tmpPath.string() + "/dummy" + to_string(i)); + files.insert(file); + file += ".meta"; + err = ::open(file.c_str(), O_CREAT | O_WRONLY, 0600); + assert(err >= 0); + fdMinders.push_back(err); + } - uint8_t buf[8192]; - memset(buf,0,sizeof(buf)); - listdir_cmd *cmd = (listdir_cmd *) buf; + uint8_t buf[8192]; + memset(buf, 0, sizeof(buf)); + listdir_cmd* cmd = (listdir_cmd*)buf; - cmd->opcode = LIST_DIRECTORY; - cmd->plen = strlen(relPath); - memcpy(cmd->path, relPath, cmd->plen); + cmd->opcode = LIST_DIRECTORY; + cmd->plen = strlen(relPath); + memcpy(cmd->path, relPath, cmd->plen); - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - cmd->plen -= 2; + // set payload to be shorter than actual message lengh + // and send a shortened message. + if (connectionTest) + cmd->plen -= 2; - size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->plen); - assert(result==(sizeof(*cmd) + cmd->plen)); + size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->plen); + assert(result == (sizeof(*cmd) + cmd->plen)); - // set payload to be correct length again - if (connectionTest) - cmd->plen += 2; + // set payload to be correct length again + if (connectionTest) + cmd->plen += 2; - // process task will look for the full length and - // will wait on the rest of the message. - ProcessTask pt(clientSock, sizeof(*cmd) + cmd->plen); - boost::thread t(pt); + // process task will look for the full length and + // will wait on the rest of the message. + ProcessTask pt(clientSock, sizeof(*cmd) + cmd->plen); + boost::thread t(pt); - if (connectionTest) + if (connectionTest) + { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else + { + t.join(); + /* going to keep this simple. Don't run this in a big dir. */ + /* maybe later I'll make a dir, put a file in it, and etc. For now run it in a small dir. */ + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err > 0); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.flags == 0); + assert(resp->returnCode == 0); + listdir_resp* r = (listdir_resp*)resp->payload; + assert(r->elements == 10); + int off = sizeof(sm_response) + sizeof(listdir_resp); + uint fileCounter = 0; + while (off < err) { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - /* going to keep this simple. Don't run this in a big dir. */ - /* maybe later I'll make a dir, put a file in it, and etc. For now run it in a small dir. */ - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err > 0); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - listdir_resp *r = (listdir_resp *) resp->payload; - assert(r->elements == 10); - int off = sizeof(sm_response) + sizeof(listdir_resp); - uint fileCounter = 0; - while (off < err) - { - listdir_resp_entry *e = (listdir_resp_entry *) &buf[off]; - //cout << "len = " << e->flen << endl; - assert(off + e->flen + sizeof(listdir_resp_entry) < 8192); - string file(e->filename, e->flen); - assert(files.find((tmpPath/file).string()) != files.end()); - fileCounter++; - //cout << "name = " << file << endl; - off += e->flen + sizeof(listdir_resp_entry); - } - assert(fileCounter == r->elements); + listdir_resp_entry* e = (listdir_resp_entry*)&buf[off]; + // cout << "len = " << e->flen << endl; + assert(off + e->flen + sizeof(listdir_resp_entry) < 8192); + string file(e->filename, e->flen); + assert(files.find((tmpPath / file).string()) != files.end()); + fileCounter++; + // cout << "name = " << file << endl; + off += e->flen + sizeof(listdir_resp_entry); } + assert(fileCounter == r->elements); + } - bf::remove_all(tmpPath); + bf::remove_all(tmpPath); - cout << "listdir task OK" << endl; + cout << "listdir task OK" << endl; - return true; + return true; } void pingtask() { - int err=0; - uint8_t buf[1024]; - ping_cmd *cmd = (ping_cmd *) buf; - cmd->opcode = PING; + int err = 0; + uint8_t buf[1024]; + ping_cmd* cmd = (ping_cmd*)buf; + cmd->opcode = PING; - size_t len = sizeof(*cmd); + size_t len = sizeof(*cmd); - ssize_t result = ::write(sessionSock, cmd, sizeof(*cmd)); - assert(result==(sizeof(*cmd))); + ssize_t result = ::write(sessionSock, cmd, sizeof(*cmd)); + assert(result == (sizeof(*cmd))); + // process task will look for the full length and + // will wait on the rest of the message. + // don't test connection loss here since this message is only 1 byte + ProcessTask pt(clientSock, len); + boost::thread t(pt); - // process task will look for the full length and - // will wait on the rest of the message. - // don't test connection loss here since this message is only 1 byte - ProcessTask pt(clientSock, len); - boost::thread t(pt); + t.join(); + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response* resp = (sm_response*)buf; + assert(err == sizeof(sm_response)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 0); + cout << "pingtask OK" << endl; +} + +bool copytask(bool connectionTest = false) +{ + /* + make a file + copy it + verify it exists + */ + bf::path fullPathSrc = homepath / prefix / "dummy1"; + string metaStrSrc = prefix + "/dummy1"; + const char* source = fullPathSrc.string().c_str(); + const char* Metasource = metaStrSrc.c_str(); + + bf::path fullPathDest = homepath / prefix / "dummy2"; + string metaStrDest = prefix + "/dummy2"; + const char* dest = fullPathDest.string().c_str(); + const char* Metadest = metaStrDest.c_str(); + + MetadataFile meta1(Metasource); + + uint8_t buf[1024]; + copy_cmd* cmd = (copy_cmd*)buf; + cmd->opcode = COPY; + cmd->file1.flen = strlen(source); + strncpy(cmd->file1.filename, source, cmd->file1.flen); + f_name* file2 = (f_name*)&cmd->file1.filename[cmd->file1.flen]; + file2->flen = strlen(dest); + strncpy(file2->filename, dest, file2->flen); + + uint len = (uint64_t)&file2->filename[file2->flen] - (uint64_t)buf; + + // set payload to be shorter than actual message lengh + // and send a shortened message. + if (connectionTest) + len -= 2; + + ssize_t result = ::write(sessionSock, buf, len); + assert(result == static_cast(len)); + + int err = 0; + + // set payload to be correct length again + if (connectionTest) + len += 2; + + // process task will look for the full length and + // will wait on the rest of the message. + ProcessTask pt(clientSock, len); + boost::thread t(pt); + + if (connectionTest) + { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else + { t.join(); // read the response err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; + sm_response* resp = (sm_response*)buf; assert(err == sizeof(sm_response)); assert(resp->header.type == SM_MSG_START); assert(resp->header.payloadLen == sizeof(ssize_t)); assert(resp->header.flags == 0); assert(resp->returnCode == 0); + // verify copytest2 is there + MetadataFile meta2(Metadest, MetadataFile::no_create_t(), true); + assert(meta2.exists()); + } - cout << "pingtask OK" << endl; -} - -bool copytask(bool connectionTest=false) -{ - /* - make a file - copy it - verify it exists - */ - bf::path fullPathSrc = homepath / prefix / "dummy1"; - string metaStrSrc = prefix + "/dummy1"; - const char *source = fullPathSrc.string().c_str(); - const char *Metasource = metaStrSrc.c_str(); - - bf::path fullPathDest = homepath / prefix / "dummy2"; - string metaStrDest = prefix + "/dummy2"; - const char *dest = fullPathDest.string().c_str(); - const char *Metadest = metaStrDest.c_str(); - - - MetadataFile meta1(Metasource); - - uint8_t buf[1024]; - copy_cmd *cmd = (copy_cmd *) buf; - cmd->opcode = COPY; - cmd->file1.flen = strlen(source); - strncpy(cmd->file1.filename, source, cmd->file1.flen); - f_name *file2 = (f_name *) &cmd->file1.filename[cmd->file1.flen]; - file2->flen = strlen(dest); - strncpy(file2->filename, dest, file2->flen); - - uint len = (uint64_t) &file2->filename[file2->flen] - (uint64_t) buf; - - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - len -= 2; - - ssize_t result = ::write(sessionSock, buf, len); - assert(result==static_cast(len)); - - int err=0; - - // set payload to be correct length again - if (connectionTest) - len += 2; - - // process task will look for the full length and - // will wait on the rest of the message. - ProcessTask pt(clientSock, len); - boost::thread t(pt); - - if (connectionTest) - { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - // read the response - err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(sm_response)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - // verify copytest2 is there - MetadataFile meta2(Metadest, MetadataFile::no_create_t(),true); - assert(meta2.exists()); - } - - bf::path metaPath = IOCoordinator::get()->getMetadataPath(); - bf::remove(metaPath/(metaStrSrc + ".meta")); - bf::remove(metaPath/(metaStrDest + ".meta")); - cout << "copytask OK " << endl; - return true; + bf::path metaPath = IOCoordinator::get()->getMetadataPath(); + bf::remove(metaPath / (metaStrSrc + ".meta")); + bf::remove(metaPath / (metaStrDest + ".meta")); + cout << "copytask OK " << endl; + return true; } bool localstorageTest1() { - LocalStorage ls; + LocalStorage ls; - /* TODO: Some stuff */ - cout << "local storage test 1 OK" << endl; - return true; + /* TODO: Some stuff */ + cout << "local storage test 1 OK" << endl; + return true; } bool cacheTest1() { + Cache* cache = Cache::get(); + CloudStorage* cs = CloudStorage::get(); + LocalStorage* ls = dynamic_cast(cs); + if (ls == NULL) + { + cout << "Cache test 1 requires using local storage" << endl; + return false; + } - Cache *cache = Cache::get(); - CloudStorage *cs = CloudStorage::get(); - LocalStorage *ls = dynamic_cast(cs); - if (ls == NULL) { - cout << "Cache test 1 requires using local storage" << endl; - return false; - } + cache->reset(); + assert(cache->getCurrentCacheSize() == 0); - cache->reset(); - assert(cache->getCurrentCacheSize() == 0); + bf::path storagePath = ls->getPrefix(); + bf::path cachePath = cache->getCachePath() / prefix; + vector v_bogus; + vector exists; - bf::path storagePath = ls->getPrefix(); - bf::path cachePath = cache->getCachePath() / prefix; - vector v_bogus; - vector exists; + // make sure nothing shows up in the cache path for files that don't exist + v_bogus.push_back("does-not-exist"); + cache->read(prefix, v_bogus); + assert(!bf::exists(cachePath / "does-not-exist")); + cache->exists(prefix, v_bogus, &exists); + assert(exists.size() == 1); + assert(!exists[0]); - // make sure nothing shows up in the cache path for files that don't exist - v_bogus.push_back("does-not-exist"); - cache->read(prefix, v_bogus); - assert(!bf::exists(cachePath / "does-not-exist")); - cache->exists(prefix, v_bogus, &exists); - assert(exists.size() == 1); - assert(!exists[0]); + // make sure a file that does exist does show up in the cache path - // make sure a file that does exist does show up in the cache path + makeTestObject((storagePath / testObjKey).string().c_str()); - makeTestObject((storagePath/testObjKey).string().c_str()); + v_bogus[0] = testObjKey.c_str(); + cache->read(prefix, v_bogus); + assert(bf::exists(cachePath / testObjKey)); + exists.clear(); + cache->exists(prefix, v_bogus, &exists); + assert(exists.size() == 1); + assert(exists[0]); + size_t currentSize = cache->getCurrentCacheSize(); + assert(currentSize == bf::file_size(cachePath / testObjKey)); - v_bogus[0] = testObjKey.c_str(); - cache->read(prefix, v_bogus); - assert(bf::exists(cachePath / testObjKey)); - exists.clear(); - cache->exists(prefix, v_bogus, &exists); - assert(exists.size() == 1); - assert(exists[0]); - size_t currentSize = cache->getCurrentCacheSize(); - assert(currentSize == bf::file_size(cachePath / testObjKey)); + // lie about the file being deleted and then replaced + cache->deletedObject(prefix, testObjKey, currentSize); + assert(cache->getCurrentCacheSize() == 0); + cache->newObject(prefix, testObjKey, currentSize); + assert(cache->getCurrentCacheSize() == currentSize); + cache->exists(prefix, v_bogus, &exists); + assert(exists.size() == 1); + assert(exists[0]); - // lie about the file being deleted and then replaced - cache->deletedObject(prefix, testObjKey, currentSize); - assert(cache->getCurrentCacheSize() == 0); - cache->newObject(prefix, testObjKey, currentSize); - assert(cache->getCurrentCacheSize() == currentSize); - cache->exists(prefix, v_bogus, &exists); - assert(exists.size() == 1); - assert(exists[0]); - - // cleanup - bf::remove(cachePath / testObjKey); - bf::remove(storagePath / testObjKey); - cout << "cache test 1 OK" << endl; - return true; + // cleanup + bf::remove(cachePath / testObjKey); + bf::remove(storagePath / testObjKey); + cout << "cache test 1 OK" << endl; + return true; } bool mergeJournalTest() { - /* - create a dummy object and a dummy journal - call mergeJournal to process it with various params - verify the expected values - */ + /* + create a dummy object and a dummy journal + call mergeJournal to process it with various params + verify the expected values + */ - makeTestObject("test-object"); - makeTestJournal("test-journal"); + makeTestObject("test-object"); + makeTestJournal("test-journal"); - int i; - IOCoordinator *ioc = IOCoordinator::get(); - size_t len = 8192, tmp; - boost::shared_array data = ioc->mergeJournal("test-object", "test-journal", 0, len, &tmp); - assert(data); - int *idata = (int *) data.get(); - for (i = 0; i < 5; i++) - assert(idata[i] == i); - for (; i < 10; i++) - assert(idata[i] == i-5); - for (; i < 2048; i++) - assert(idata[i] == i); + int i; + IOCoordinator* ioc = IOCoordinator::get(); + size_t len = 8192, tmp; + boost::shared_array data = ioc->mergeJournal("test-object", "test-journal", 0, len, &tmp); + assert(data); + int* idata = (int*)data.get(); + for (i = 0; i < 5; i++) + assert(idata[i] == i); + for (; i < 10; i++) + assert(idata[i] == i - 5); + for (; i < 2048; i++) + assert(idata[i] == i); - // try different range parameters - // read at the beginning of the change - len = 40; - data = ioc->mergeJournal("test-object", "test-journal", 20, len, &tmp); - assert(data); - idata = (int *) data.get(); - for (i = 0; i < 5; i++) - assert(idata[i] == i); - for (; i < 10; i++) - assert(idata[i] == i+5); + // try different range parameters + // read at the beginning of the change + len = 40; + data = ioc->mergeJournal("test-object", "test-journal", 20, len, &tmp); + assert(data); + idata = (int*)data.get(); + for (i = 0; i < 5; i++) + assert(idata[i] == i); + for (; i < 10; i++) + assert(idata[i] == i + 5); - // read s.t. beginning of the change is in the middle of the range - len = 24; - data = ioc->mergeJournal("test-object", "test-journal", 8, len, &tmp); - assert(data); - idata = (int *) data.get(); - for (i = 0; i < 3; i++) - assert(idata[i] == i + 2); - for (; i < 6; i++) - assert(idata[i] == i - 3); + // read s.t. beginning of the change is in the middle of the range + len = 24; + data = ioc->mergeJournal("test-object", "test-journal", 8, len, &tmp); + assert(data); + idata = (int*)data.get(); + for (i = 0; i < 3; i++) + assert(idata[i] == i + 2); + for (; i < 6; i++) + assert(idata[i] == i - 3); - // read s.t. end of the change is in the middle of the range - len = 20; - data = ioc->mergeJournal("test-object", "test-journal", 28, len, &tmp); - assert(data); - idata = (int *) data.get(); - for (i = 0; i < 3; i++) - assert(idata[i] == i + 2); - for (; i < 3; i++) - assert(idata[i] == i + 7); + // read s.t. end of the change is in the middle of the range + len = 20; + data = ioc->mergeJournal("test-object", "test-journal", 28, len, &tmp); + assert(data); + idata = (int*)data.get(); + for (i = 0; i < 3; i++) + assert(idata[i] == i + 2); + for (; i < 3; i++) + assert(idata[i] == i + 7); - // cleanup - bf::remove("test-object"); - bf::remove("test-journal"); - cout << "mergeJournalTest OK" << endl; - return true; + // cleanup + bf::remove("test-object"); + bf::remove("test-journal"); + cout << "mergeJournalTest OK" << endl; + return true; } bool syncTest1() { - IOCoordinator *ioc = IOCoordinator::get(); - Config *config = Config::get(); - Synchronizer *sync = Synchronizer::get(); - Cache *cache = Cache::get(); - CloudStorage *cs = CloudStorage::get(); - LocalStorage *ls = dynamic_cast(cs); - if (!ls) - { - cout << "syncTest1() requires using local storage at the moment." << endl; - return true; - } - - cache->reset(); - - // delete everything in the fake cloud to make it easier to list later - bf::path fakeCloudPath = ls->getPrefix(); - cout << "fakeCLoudPath = " << fakeCloudPath << endl; - for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) - bf::remove(dir->path()); - - bf::path cachePath = sync->getCachePath(); - bf::path journalPath = sync->getJournalPath(); - - string stmp = config->getValue("ObjectStorage", "metadata_path"); - assert(!stmp.empty()); - bf::path metaPath = stmp; - // nothing creates the dir yet - bf::create_directories(metaPath); - - // make the test obj, journal, and metadata - string journalName = prefix + "/" + testObjKey + ".journal"; - - makeTestObject((cachePath/prefix/testObjKey).string().c_str()); - makeTestJournal((journalPath/journalName).string().c_str()); - makeTestMetadata((metaPath/string(metaTestFile + ".meta")).string().c_str(),testObjKey); - - cache->newObject(prefix, testObjKey, bf::file_size(cachePath/prefix/testObjKey)); - cache->newJournalEntry(prefix, bf::file_size(journalPath/journalName)); - - vector vObj; - vObj.push_back(testObjKey); - - sync->newObjects(prefix, vObj); - sync->forceFlush(); - sleep(2); // wait for the job to run - - // make sure that it made it to the cloud - bool exists = false; - int err = cs->exists(testObjKey, &exists); - assert(!err); - assert(exists); - - sync->newJournalEntry(prefix, testObjKey, 0); - sync->forceFlush(); - sleep(1); // let it do what it does - - // check that the original objects no longer exist - assert(!cache->exists(prefix, testObjKey)); - assert(!bf::exists(journalPath/journalName)); - - // Replicator doesn't implement all of its functionality yet, need to delete key from the cache manually for now - bf::remove(cachePath/testObjKey); - - // check that a new version of object exists in cloud storage - // D'oh, this would have to list the objects to find it, not going to implement - // that everywhere just now. For now, making this test require LocalStorage. - bool foundIt = false; - string newKey; - for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir) - { - newKey = dir->path().filename().string(); - foundIt = (MetadataFile::getSourceFromKey(newKey) == metaTestFile); - if (foundIt) - { - assert(cache->exists(prefix, newKey)); - cs->deleteObject(newKey); - break; - } - } - - assert(foundIt); - cache->makeSpace(prefix, cache->getMaxCacheSize()); // clear the cache & make it call sync->flushObject() - - // the key should now be back in cloud storage and deleted from the cache - assert(!cache->exists(prefix, newKey)); - err = cs->exists(newKey, &exists); - assert(!err && exists); - - // make the journal again, call sync->newJournalObject() - makeTestJournal((journalPath / prefix / (newKey + ".journal")).string().c_str()); - cache->newJournalEntry(prefix, bf::file_size(journalPath / prefix / (newKey + ".journal"))); - sync->newJournalEntry(prefix, newKey, 0); - sync->forceFlush(); - sleep(1); - - // verify that newkey is no longer in cloud storage, and that another permutation is - err = cs->exists(newKey, &exists); - assert(!err && !exists); - foundIt = false; - for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir) - { - testObjKey = dir->path().filename().string(); - foundIt = (MetadataFile::getSourceFromKey(testObjKey) == metaTestFile); - } - assert(foundIt); - - // TODO test error paths, pass in some junk - - // cleanup, just blow away everything for now - cache->reset(); - vector keys; - for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) - keys.push_back(dir->path().filename().string()); - sync->deletedObjects(prefix, keys); - sync->forceFlush(); - sleep(1); - ioc->unlink(testFile); - - cout << "Sync test 1 OK" << endl; + IOCoordinator* ioc = IOCoordinator::get(); + Config* config = Config::get(); + Synchronizer* sync = Synchronizer::get(); + Cache* cache = Cache::get(); + CloudStorage* cs = CloudStorage::get(); + LocalStorage* ls = dynamic_cast(cs); + if (!ls) + { + cout << "syncTest1() requires using local storage at the moment." << endl; return true; + } + + cache->reset(); + + // delete everything in the fake cloud to make it easier to list later + bf::path fakeCloudPath = ls->getPrefix(); + cout << "fakeCLoudPath = " << fakeCloudPath << endl; + for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) + bf::remove(dir->path()); + + bf::path cachePath = sync->getCachePath(); + bf::path journalPath = sync->getJournalPath(); + + string stmp = config->getValue("ObjectStorage", "metadata_path"); + assert(!stmp.empty()); + bf::path metaPath = stmp; + // nothing creates the dir yet + bf::create_directories(metaPath); + + // make the test obj, journal, and metadata + string journalName = prefix + "/" + testObjKey + ".journal"; + + makeTestObject((cachePath / prefix / testObjKey).string().c_str()); + makeTestJournal((journalPath / journalName).string().c_str()); + makeTestMetadata((metaPath / string(metaTestFile + ".meta")).string().c_str(), testObjKey); + + cache->newObject(prefix, testObjKey, bf::file_size(cachePath / prefix / testObjKey)); + cache->newJournalEntry(prefix, bf::file_size(journalPath / journalName)); + + vector vObj; + vObj.push_back(testObjKey); + + sync->newObjects(prefix, vObj); + sync->forceFlush(); + sleep(2); // wait for the job to run + + // make sure that it made it to the cloud + bool exists = false; + int err = cs->exists(testObjKey, &exists); + assert(!err); + assert(exists); + + sync->newJournalEntry(prefix, testObjKey, 0); + sync->forceFlush(); + sleep(1); // let it do what it does + + // check that the original objects no longer exist + assert(!cache->exists(prefix, testObjKey)); + assert(!bf::exists(journalPath / journalName)); + + // Replicator doesn't implement all of its functionality yet, need to delete key from the cache manually for + // now + bf::remove(cachePath / testObjKey); + + // check that a new version of object exists in cloud storage + // D'oh, this would have to list the objects to find it, not going to implement + // that everywhere just now. For now, making this test require LocalStorage. + bool foundIt = false; + string newKey; + for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir) + { + newKey = dir->path().filename().string(); + foundIt = (MetadataFile::getSourceFromKey(newKey) == metaTestFile); + if (foundIt) + { + assert(cache->exists(prefix, newKey)); + cs->deleteObject(newKey); + break; + } + } + + assert(foundIt); + cache->makeSpace(prefix, cache->getMaxCacheSize()); // clear the cache & make it call sync->flushObject() + + // the key should now be back in cloud storage and deleted from the cache + assert(!cache->exists(prefix, newKey)); + err = cs->exists(newKey, &exists); + assert(!err && exists); + + // make the journal again, call sync->newJournalObject() + makeTestJournal((journalPath / prefix / (newKey + ".journal")).string().c_str()); + cache->newJournalEntry(prefix, bf::file_size(journalPath / prefix / (newKey + ".journal"))); + sync->newJournalEntry(prefix, newKey, 0); + sync->forceFlush(); + sleep(1); + + // verify that newkey is no longer in cloud storage, and that another permutation is + err = cs->exists(newKey, &exists); + assert(!err && !exists); + foundIt = false; + for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir) + { + testObjKey = dir->path().filename().string(); + foundIt = (MetadataFile::getSourceFromKey(testObjKey) == metaTestFile); + } + assert(foundIt); + + // TODO test error paths, pass in some junk + + // cleanup, just blow away everything for now + cache->reset(); + vector keys; + for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) + keys.push_back(dir->path().filename().string()); + sync->deletedObjects(prefix, keys); + sync->forceFlush(); + sleep(1); + ioc->unlink(testFile); + + cout << "Sync test 1 OK" << endl; + return true; } void metadataUpdateTest() { - Config* config = Config::get(); - string metaPath = config->getValue("ObjectStorage", "metadata_path"); - MetadataFile mdfTest("metadataUpdateTest"); - mdfTest.addMetadataObject("metadataUpdateTest",100); - mdfTest.printObjects(); - mdfTest.updateEntryLength(0,200); - mdfTest.printObjects(); - //mdfTest.updateEntryLength(0,100); - //mdfTest.printObjects(); - string metaFilePath = metaPath + "/" + "metadataUpdateTest.meta"; - ::unlink(metaFilePath.c_str()); - + Config* config = Config::get(); + string metaPath = config->getValue("ObjectStorage", "metadata_path"); + MetadataFile mdfTest("metadataUpdateTest"); + mdfTest.addMetadataObject("metadataUpdateTest", 100); + mdfTest.printObjects(); + mdfTest.updateEntryLength(0, 200); + mdfTest.printObjects(); + // mdfTest.updateEntryLength(0,100); + // mdfTest.printObjects(); + string metaFilePath = metaPath + "/" + "metadataUpdateTest.meta"; + ::unlink(metaFilePath.c_str()); } void s3storageTest1() { + try + { + S3Storage s3; + bool exists; + int err; + string testFile = "storagemanager.cnf"; + string testFile2 = testFile + "2"; + + exists = bf::exists(testFile); + if (!exists) + { + cout << "s3storageTest1() requires having " << testFile << " in the current directory."; + return; + } + try { - S3Storage s3; - bool exists; - int err; - string testFile = "storagemanager.cnf"; - string testFile2 = testFile + "2"; + err = s3.exists(testFile, &exists); + assert(!err); + if (exists) + s3.deleteObject(testFile); - exists = bf::exists(testFile); - if (!exists) - { - cout << "s3storageTest1() requires having " << testFile << " in the current directory."; - return; - } + err = s3.exists(testFile2, &exists); + assert(!err); + if (exists) + s3.deleteObject(testFile2); - try { - err = s3.exists(testFile, &exists); - assert(!err); - if (exists) - s3.deleteObject(testFile); + // put it & get it + err = s3.putObject(testFile, testFile); + assert(!err); + err = s3.exists(testFile, &exists); + assert(!err); + assert(exists); + err = s3.getObject(testFile, testFile2); + assert(!err); + exists = bf::exists(testFile2); + assert(bf::file_size(testFile) == bf::file_size(testFile2)); - err = s3.exists(testFile2, &exists); - assert(!err); - if (exists) - s3.deleteObject(testFile2); + // do a deep compare testFile vs testFile2 + size_t len = bf::file_size(testFile); + int fd1 = open(testFile.c_str(), O_RDONLY); + assert(fd1 >= 0); + int fd2 = open(testFile2.c_str(), O_RDONLY); + assert(fd2 >= 0); - // put it & get it - err = s3.putObject(testFile, testFile); - assert(!err); - err = s3.exists(testFile, &exists); - assert(!err); - assert(exists); - err = s3.getObject(testFile, testFile2); - assert(!err); - exists = bf::exists(testFile2); - assert(bf::file_size(testFile) == bf::file_size(testFile2)); + uint8_t* data1 = new uint8_t[len]; + uint8_t* data2 = new uint8_t[len]; + err = read(fd1, data1, len); + assert(err == (int)len); + err = read(fd2, data2, len); + assert(err == (int)len); + assert(!memcmp(data1, data2, len)); + close(fd1); + close(fd2); + delete[] data1; + delete[] data2; - // do a deep compare testFile vs testFile2 - size_t len = bf::file_size(testFile); - int fd1 = open(testFile.c_str(), O_RDONLY); - assert(fd1 >= 0); - int fd2 = open(testFile2.c_str(), O_RDONLY); - assert(fd2 >= 0); + err = s3.copyObject(testFile, testFile2); + assert(!err); + err = s3.exists(testFile2, &exists); + assert(!err); + assert(exists); + s3.deleteObject(testFile); + s3.deleteObject(testFile2); - uint8_t *data1 = new uint8_t[len]; - uint8_t *data2 = new uint8_t[len]; - err = read(fd1, data1, len); - assert(err == (int) len); - err = read(fd2, data2, len); - assert(err == (int) len); - assert(!memcmp(data1, data2, len)); - close(fd1); - close(fd2); - delete [] data1; - delete [] data2; - - err = s3.copyObject(testFile, testFile2); - assert(!err); - err = s3.exists(testFile2, &exists); - assert(!err); - assert(exists); - s3.deleteObject(testFile); - s3.deleteObject(testFile2); - - err = s3.copyObject("this-does-not-exist", testFile2); - assert(err < 0); - assert(errno == ENOENT); - } - catch(exception &e) - { - cout << __FUNCTION__ << " caught " << e.what() << endl; - assert(0); - } - cout << "S3Storage Test 1 OK" << endl; + err = s3.copyObject("this-does-not-exist", testFile2); + assert(err < 0); + assert(errno == ENOENT); } - catch (exception &e) + catch (exception& e) { - cout << e.what() << endl; + cout << __FUNCTION__ << " caught " << e.what() << endl; + assert(0); } + cout << "S3Storage Test 1 OK" << endl; + } + catch (exception& e) + { + cout << e.what() << endl; + } } void IOCReadTest1() { - /* Generate the test object & metadata - read it, verify result + /* Generate the test object & metadata + read it, verify result - Generate the journal object - read it, verify the merged result + Generate the journal object + read it, verify the merged result - TODO: do partial reads with an offset similar to what the mergeJournal tests do - TODO: some error path testing - */ - Cache *cache = Cache::get(); - CloudStorage *cs = CloudStorage::get(); - IOCoordinator *ioc = IOCoordinator::get(); - Config *config = Config::get(); - LocalStorage *ls = dynamic_cast(cs); - if (!ls) - { - cout << "IOC read test 1 requires LocalStorage for now." << endl; - return; - } - testObjKey = "12345_0_8192_" + prefix +"~test-file"; + TODO: do partial reads with an offset similar to what the mergeJournal tests do + TODO: some error path testing + */ + Cache* cache = Cache::get(); + CloudStorage* cs = CloudStorage::get(); + IOCoordinator* ioc = IOCoordinator::get(); + Config* config = Config::get(); + LocalStorage* ls = dynamic_cast(cs); + if (!ls) + { + cout << "IOC read test 1 requires LocalStorage for now." << endl; + return; + } + testObjKey = "12345_0_8192_" + prefix + "~test-file"; - cache->reset(); + cache->reset(); - bf::path storagePath = ls->getPrefix(); - bf::path cachePath = cache->getCachePath(); - bf::path journalPath = cache->getJournalPath(); - bf::path metaPath = config->getValue("ObjectStorage", "metadata_path"); - assert(!metaPath.empty()); - bf::create_directories(metaPath/prefix); + bf::path storagePath = ls->getPrefix(); + bf::path cachePath = cache->getCachePath(); + bf::path journalPath = cache->getJournalPath(); + bf::path metaPath = config->getValue("ObjectStorage", "metadata_path"); + assert(!metaPath.empty()); + bf::create_directories(metaPath / prefix); - string objFilename = (storagePath/testObjKey).string(); - string journalFilename = (journalPath/prefix/testObjKey).string() + ".journal"; - string metaFilename = (metaPath/metaTestFile).string() + ".meta"; + string objFilename = (storagePath / testObjKey).string(); + string journalFilename = (journalPath / prefix / testObjKey).string() + ".journal"; + string metaFilename = (metaPath / metaTestFile).string() + ".meta"; - boost::scoped_array data(new uint8_t[1<<20]); + boost::scoped_array data(new uint8_t[1 << 20]); - memset(data.get(), 0, 1<<20); + memset(data.get(), 0, 1 << 20); - int err; - err = ioc->read(testFile, data.get(), 0, 1<<20); - assert(err < 0); - assert(errno == ENOENT); - makeTestObject(objFilename.c_str()); - makeTestMetadata(metaFilename.c_str(),testObjKey); - size_t objSize = bf::file_size(objFilename); - err = ioc->read(testFile, data.get(), 0, 1<<20); - assert(err == (int) objSize); + int err; + err = ioc->read(testFile, data.get(), 0, 1 << 20); + assert(err < 0); + assert(errno == ENOENT); + makeTestObject(objFilename.c_str()); + makeTestMetadata(metaFilename.c_str(), testObjKey); + size_t objSize = bf::file_size(objFilename); + err = ioc->read(testFile, data.get(), 0, 1 << 20); + assert(err == (int)objSize); - // verify the data - int *data32 = (int *) data.get(); - int i; - for (i = 0; i < 2048; i++) - assert(data32[i] == i); - for (; i < (1<<20)/4; i++) - assert(data32[i] == 0); + // verify the data + int* data32 = (int*)data.get(); + int i; + for (i = 0; i < 2048; i++) + assert(data32[i] == i); + for (; i < (1 << 20) / 4; i++) + assert(data32[i] == 0); - makeTestJournal(journalFilename.c_str()); + makeTestJournal(journalFilename.c_str()); - err = ioc->read(testFile, data.get(), 0, 1<<20); - assert(err == (int) objSize); - for (i = 0; i < 5; i++) - assert(data32[i] == i); - for (; i < 10; i++) - assert(data32[i] == i-5); - for (; i < 2048; i++) - assert(data32[i] == i); - for (; i < (1<<20)/4; i++) - assert(data32[i] == 0); + err = ioc->read(testFile, data.get(), 0, 1 << 20); + assert(err == (int)objSize); + for (i = 0; i < 5; i++) + assert(data32[i] == i); + for (; i < 10; i++) + assert(data32[i] == i - 5); + for (; i < 2048; i++) + assert(data32[i] == i); + for (; i < (1 << 20) / 4; i++) + assert(data32[i] == 0); - err = ioc->read(testFile, data.get(), 9000, 4000); - assert(err==0); + err = ioc->read(testFile, data.get(), 9000, 4000); + assert(err == 0); - cache->reset(); - err = ioc->unlink(testFile); - assert(err>= 0); + cache->reset(); + err = ioc->unlink(testFile); + assert(err >= 0); - cout << "IOC read test 1 OK" << endl; + cout << "IOC read test 1 OK" << endl; } void IOCUnlink() { - IOCoordinator *ioc = IOCoordinator::get(); - CloudStorage *cs = CloudStorage::get(); - Cache *cache = Cache::get(); - Synchronizer *sync = Synchronizer::get(); + IOCoordinator* ioc = IOCoordinator::get(); + CloudStorage* cs = CloudStorage::get(); + Cache* cache = Cache::get(); + Synchronizer* sync = Synchronizer::get(); - cache->reset(); + cache->reset(); - /* - Make a metadata file with a complex path - make the test object and test journal - delete it at the parent dir level - make sure the parent dir was deleted - make sure the object and journal were deleted - */ + /* + Make a metadata file with a complex path + make the test object and test journal + delete it at the parent dir level + make sure the parent dir was deleted + make sure the object and journal were deleted + */ - bf::path metaPath = ioc->getMetadataPath(); - bf::path cachePath = ioc->getCachePath(); - bf::path journalPath = ioc->getJournalPath(); - bf::path cachedObjPath = cachePath/prefix/testObjKey; - bf::path cachedJournalPath = journalPath/prefix/(string(testObjKey) + ".journal"); - bf::path metadataFile = metaPath/(string(metaTestFile) + ".meta"); - makeTestMetadata(metadataFile.string().c_str(),testObjKey); - makeTestObject(cachedObjPath.string().c_str()); - makeTestJournal(cachedJournalPath.string().c_str()); + bf::path metaPath = ioc->getMetadataPath(); + bf::path cachePath = ioc->getCachePath(); + bf::path journalPath = ioc->getJournalPath(); + bf::path cachedObjPath = cachePath / prefix / testObjKey; + bf::path cachedJournalPath = journalPath / prefix / (string(testObjKey) + ".journal"); + bf::path metadataFile = metaPath / (string(metaTestFile) + ".meta"); + makeTestMetadata(metadataFile.string().c_str(), testObjKey); + makeTestObject(cachedObjPath.string().c_str()); + makeTestJournal(cachedJournalPath.string().c_str()); - cache->newObject(prefix, cachedObjPath.filename().string(), bf::file_size(cachedObjPath)); - cache->newJournalEntry(prefix, bf::file_size(cachedJournalPath)); - vector keys; - keys.push_back(cachedObjPath.filename().string()); - sync->newObjects(prefix, keys); - //sync->newJournalEntry(keys[0]); don't want to end up renaming it - sync->forceFlush(); - sleep(1); + cache->newObject(prefix, cachedObjPath.filename().string(), bf::file_size(cachedObjPath)); + cache->newJournalEntry(prefix, bf::file_size(cachedJournalPath)); + vector keys; + keys.push_back(cachedObjPath.filename().string()); + sync->newObjects(prefix, keys); + // sync->newJournalEntry(keys[0]); don't want to end up renaming it + sync->forceFlush(); + sleep(1); - // ok, they should be fully 'in the system' now. - // verify that they are - assert(bf::exists(metadataFile)); - assert(bf::exists(cachedObjPath)); - assert(bf::exists(cachedJournalPath)); - bool exists; - cs->exists(cachedObjPath.filename().string(), &exists); - assert(exists); + // ok, they should be fully 'in the system' now. + // verify that they are + assert(bf::exists(metadataFile)); + assert(bf::exists(cachedObjPath)); + assert(bf::exists(cachedJournalPath)); + bool exists; + cs->exists(cachedObjPath.filename().string(), &exists); + assert(exists); - int err = ioc->unlink(testFile); - assert(err == 0); + int err = ioc->unlink(testFile); + assert(err == 0); - assert(!bf::exists(metadataFile)); - assert(!bf::exists(cachedObjPath)); - assert(!bf::exists(cachedJournalPath)); - sync->forceFlush(); - sleep(1); // stall for sync - cs->exists(cachedObjPath.filename().string(), &exists); - assert(!exists); - assert(cache->getCurrentCacheSize() == 0); + assert(!bf::exists(metadataFile)); + assert(!bf::exists(cachedObjPath)); + assert(!bf::exists(cachedJournalPath)); + sync->forceFlush(); + sleep(1); // stall for sync + cs->exists(cachedObjPath.filename().string(), &exists); + assert(!exists); + assert(cache->getCurrentCacheSize() == 0); - cout << "IOC unlink test OK" << endl; + cout << "IOC unlink test OK" << endl; } - void IOCCopyFile1() { - /* - Make our usual test files - with metadata in a subdir - with object in cloud storage - call ioc::copyFile() - with dest in a different subdir - verify the contents - */ - IOCoordinator *ioc = IOCoordinator::get(); - Cache *cache = Cache::get(); - CloudStorage *cs = CloudStorage::get(); - LocalStorage *ls = dynamic_cast(cs); - Synchronizer *sync = Synchronizer::get(); + /* + Make our usual test files + with metadata in a subdir + with object in cloud storage + call ioc::copyFile() + with dest in a different subdir + verify the contents + */ + IOCoordinator* ioc = IOCoordinator::get(); + Cache* cache = Cache::get(); + CloudStorage* cs = CloudStorage::get(); + LocalStorage* ls = dynamic_cast(cs); + Synchronizer* sync = Synchronizer::get(); - if (!ls) - { - cout << "IOCCopyFile1 requires local storage at the moment" << endl; - return; - } + if (!ls) + { + cout << "IOCCopyFile1 requires local storage at the moment" << endl; + return; + } - bf::path metaPath = ioc->getMetadataPath(); - bf::path cachePath = ioc->getCachePath(); - bf::path csPath = ls->getPrefix(); - bf::path journalPath = ioc->getJournalPath(); - bf::path cachedObjPath = cachePath/prefix/copyfileObjKey; - bf::path sourcePath = metaPath/prefix/"source.meta"; - bf::path destPath = metaPath/prefix/"dest.meta"; - bf::path l_sourceFile = homepath / prefix / string("source"); - bf::path l_destFile = homepath / prefix / string("dest"); + bf::path metaPath = ioc->getMetadataPath(); + bf::path cachePath = ioc->getCachePath(); + bf::path csPath = ls->getPrefix(); + bf::path journalPath = ioc->getJournalPath(); + bf::path cachedObjPath = cachePath / prefix / copyfileObjKey; + bf::path sourcePath = metaPath / prefix / "source.meta"; + bf::path destPath = metaPath / prefix / "dest.meta"; + bf::path l_sourceFile = homepath / prefix / string("source"); + bf::path l_destFile = homepath / prefix / string("dest"); - cache->reset(); + cache->reset(); - bf::create_directories(sourcePath.parent_path()); - makeTestMetadata(sourcePath.string().c_str(),copyfileObjKey); - makeTestObject((csPath/copyfileObjKey).string().c_str()); - makeTestJournal((journalPath/prefix/(string(copyfileObjKey) + ".journal")).string().c_str()); - cache->newJournalEntry(prefix, bf::file_size(journalPath/prefix/(string(copyfileObjKey) + ".journal"))); + bf::create_directories(sourcePath.parent_path()); + makeTestMetadata(sourcePath.string().c_str(), copyfileObjKey); + makeTestObject((csPath / copyfileObjKey).string().c_str()); + makeTestJournal((journalPath / prefix / (string(copyfileObjKey) + ".journal")).string().c_str()); + cache->newJournalEntry(prefix, bf::file_size(journalPath / prefix / (string(copyfileObjKey) + ".journal"))); - int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str()); - assert(!err); - uint8_t buf1[8192], buf2[8192]; - err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, sizeof(buf1)); - assert(err == sizeof(buf1)); - err = ioc->read(l_destFile.string().c_str(), buf2, 0, sizeof(buf2)); - assert(err == sizeof(buf2)); - assert(memcmp(buf1, buf2, 8192) == 0); + int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str()); + assert(!err); + uint8_t buf1[8192], buf2[8192]; + err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, sizeof(buf1)); + assert(err == sizeof(buf1)); + err = ioc->read(l_destFile.string().c_str(), buf2, 0, sizeof(buf2)); + assert(err == sizeof(buf2)); + assert(memcmp(buf1, buf2, 8192) == 0); - assert(ioc->unlink(l_sourceFile.string().c_str()) == 0); - assert(ioc->unlink(l_destFile.string().c_str()) == 0); - sync->forceFlush(); - assert(cache->getCurrentCacheSize() == 0); + assert(ioc->unlink(l_sourceFile.string().c_str()) == 0); + assert(ioc->unlink(l_destFile.string().c_str()) == 0); + sync->forceFlush(); + assert(cache->getCurrentCacheSize() == 0); - cout << "IOC copy file 1 OK" << endl; + cout << "IOC copy file 1 OK" << endl; } void IOCCopyFile2() { - // call IOC::copyFile() with non-existant file - IOCoordinator *ioc = IOCoordinator::get(); + // call IOC::copyFile() with non-existant file + IOCoordinator* ioc = IOCoordinator::get(); - bf::path fullPath = homepath / prefix / "not-there"; - const char *source = fullPath.string().c_str(); - bf::path fullPath2 = homepath / prefix / "not-there2"; - const char *dest = fullPath2.string().c_str(); + bf::path fullPath = homepath / prefix / "not-there"; + const char* source = fullPath.string().c_str(); + bf::path fullPath2 = homepath / prefix / "not-there2"; + const char* dest = fullPath2.string().c_str(); - bf::path metaPath = ioc->getMetadataPath(); - bf::remove(metaPath/prefix/"not-there.meta"); - bf::remove(metaPath/prefix/"not-there2.meta"); + bf::path metaPath = ioc->getMetadataPath(); + bf::remove(metaPath / prefix / "not-there.meta"); + bf::remove(metaPath / prefix / "not-there2.meta"); - int err = ioc->copyFile(source, dest); - assert(err); - assert(errno == ENOENT); - assert(!bf::exists(metaPath/"not-there.meta")); - assert(!bf::exists(metaPath/"not-there2.meta")); + int err = ioc->copyFile(source, dest); + assert(err); + assert(errno == ENOENT); + assert(!bf::exists(metaPath / "not-there.meta")); + assert(!bf::exists(metaPath / "not-there2.meta")); - cout << "IOC copy file 2 OK" << endl; + cout << "IOC copy file 2 OK" << endl; } void IOCCopyFile3() { - /* - Make our usual test files - with object in the cache not in CS - call ioc::copyFile() - verify dest file exists - */ - IOCoordinator *ioc = IOCoordinator::get(); - Cache *cache = Cache::get(); - Synchronizer *sync = Synchronizer::get(); + /* + Make our usual test files + with object in the cache not in CS + call ioc::copyFile() + verify dest file exists + */ + IOCoordinator* ioc = IOCoordinator::get(); + Cache* cache = Cache::get(); + Synchronizer* sync = Synchronizer::get(); - bf::path metaPath = ioc->getMetadataPath(); - bf::path journalPath = ioc->getJournalPath(); - bf::path cachePath = ioc->getCachePath(); - bf::path sourcePath = metaPath/prefix/"source.meta"; - bf::path destPath = metaPath/prefix/"dest.meta"; - bf::path l_sourceFile = homepath / prefix / string("source"); - bf::path l_destFile = homepath / prefix / string("dest"); + bf::path metaPath = ioc->getMetadataPath(); + bf::path journalPath = ioc->getJournalPath(); + bf::path cachePath = ioc->getCachePath(); + bf::path sourcePath = metaPath / prefix / "source.meta"; + bf::path destPath = metaPath / prefix / "dest.meta"; + bf::path l_sourceFile = homepath / prefix / string("source"); + bf::path l_destFile = homepath / prefix / string("dest"); - cache->reset(); + cache->reset(); - makeTestObject((cachePath/prefix/copyfileObjKey).string().c_str()); - makeTestJournal((journalPath/prefix/(string(copyfileObjKey) + ".journal")).string().c_str()); - makeTestMetadata(sourcePath.string().c_str(),copyfileObjKey); + makeTestObject((cachePath / prefix / copyfileObjKey).string().c_str()); + makeTestJournal((journalPath / prefix / (string(copyfileObjKey) + ".journal")).string().c_str()); + makeTestMetadata(sourcePath.string().c_str(), copyfileObjKey); - cache->newObject(prefix, copyfileObjKey, bf::file_size(cachePath/prefix/copyfileObjKey)); - cache->newJournalEntry(prefix, bf::file_size(journalPath/prefix/(string(copyfileObjKey) + ".journal"))); + cache->newObject(prefix, copyfileObjKey, bf::file_size(cachePath / prefix / copyfileObjKey)); + cache->newJournalEntry(prefix, bf::file_size(journalPath / prefix / (string(copyfileObjKey) + ".journal"))); - int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str()); - assert(!err); - uint8_t buf1[8192], buf2[8192]; - err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, sizeof(buf1)); - assert(err == sizeof(buf1)); - err = ioc->read(l_destFile.string().c_str(), buf2, 0, sizeof(buf2)); - assert(err == sizeof(buf2)); - assert(memcmp(buf1, buf2, 8192) == 0); + int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str()); + assert(!err); + uint8_t buf1[8192], buf2[8192]; + err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, sizeof(buf1)); + assert(err == sizeof(buf1)); + err = ioc->read(l_destFile.string().c_str(), buf2, 0, sizeof(buf2)); + assert(err == sizeof(buf2)); + assert(memcmp(buf1, buf2, 8192) == 0); - assert(ioc->unlink(l_sourceFile.string().c_str()) == 0); - assert(ioc->unlink(l_destFile.string().c_str()) == 0); - sync->forceFlush(); - assert(cache->getCurrentCacheSize() == 0); + assert(ioc->unlink(l_sourceFile.string().c_str()) == 0); + assert(ioc->unlink(l_destFile.string().c_str()) == 0); + sync->forceFlush(); + assert(cache->getCurrentCacheSize() == 0); - cout << "IOC copy file 3 OK" << endl; + cout << "IOC copy file 3 OK" << endl; } void IOCCopyFile() { - IOCCopyFile1(); - IOCCopyFile2(); - IOCCopyFile3(); + IOCCopyFile1(); + IOCCopyFile2(); + IOCCopyFile3(); } /* Correctness was tested by inspecting debugging outputs in mergeJournal(). With a little more work @@ -1743,111 +1748,111 @@ void IOCCopyFile() */ void bigMergeJournal1() { - const char *jName = "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~" - "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal"; - const char *fName = "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~" - "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em"; - bf::path jNamePath = testDirPath / jName; - bf::path fNamePath = testDirPath / fName; + const char* jName = + "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~" + "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal"; + const char* fName = + "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~" + "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em"; + bf::path jNamePath = testDirPath / jName; + bf::path fNamePath = testDirPath / fName; - if (!bf::is_directory(testDirPath/"test_data")) - { - cout << "bigMergeJournal1 test_data directory not found at " << (testDirPath/"test_data") << endl << - " Check if -d option needs to be provided or changed." << endl; - return; - } - IOCoordinator *ioc = IOCoordinator::get(); - boost::shared_array buf; - size_t tmp; - buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 0, 68332, &tmp); - assert(buf); - buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 100, 68232, &tmp); - assert(buf); - buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 0, 68232, &tmp); - assert(buf); - buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 100, 68132, &tmp); - assert(buf); - buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 100, 10, &tmp); - assert(buf); + if (!bf::is_directory(testDirPath / "test_data")) + { + cout << "bigMergeJournal1 test_data directory not found at " << (testDirPath / "test_data") << endl + << " Check if -d option needs to be provided or changed." << endl; + return; + } + IOCoordinator* ioc = IOCoordinator::get(); + boost::shared_array buf; + size_t tmp; + buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 0, 68332, &tmp); + assert(buf); + buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 100, 68232, &tmp); + assert(buf); + buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 0, 68232, &tmp); + assert(buf); + buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 100, 68132, &tmp); + assert(buf); + buf = ioc->mergeJournal(fNamePath.string().c_str(), jNamePath.string().c_str(), 100, 10, &tmp); + assert(buf); } - // This should write an incomplete msg(s) to make sure SM does the right thing. Not // done yet, handing this off to Ben. void shortMsg() { - IOCoordinator *ioc = IOCoordinator::get(); + IOCoordinator* ioc = IOCoordinator::get(); - struct stat _stat; - bf::path fullPath = homepath / prefix / "writetest1"; - const char *filename = fullPath.string().c_str(); - ::unlink(filename); - ioc->open(filename,O_WRONLY | O_CREAT,&_stat); + struct stat _stat; + bf::path fullPath = homepath / prefix / "writetest1"; + const char* filename = fullPath.string().c_str(); + ::unlink(filename); + ioc->open(filename, O_WRONLY | O_CREAT, &_stat); - size_t size = 27; - std::vector bufWrite(sizeof(write_cmd)+std::strlen(filename)+size); + size_t size = 27; + std::vector bufWrite(sizeof(write_cmd) + std::strlen(filename) + size); - sm_msg_header *hdrWrite = (sm_msg_header *) bufWrite.data(); - write_cmd *cmdWrite = (write_cmd *) &hdrWrite[1]; - uint8_t *dataWrite; + sm_msg_header* hdrWrite = (sm_msg_header*)bufWrite.data(); + write_cmd* cmdWrite = (write_cmd*)&hdrWrite[1]; + uint8_t* dataWrite; - cmdWrite->opcode = WRITE; - cmdWrite->offset = 0; - cmdWrite->count = size; - cmdWrite->flen = std::strlen(filename); - memcpy(&cmdWrite->filename, filename, cmdWrite->flen); - dataWrite = (uint8_t *) &cmdWrite->filename[cmdWrite->flen]; - memcpy(dataWrite, "123456789123456789123456789", cmdWrite->count); + cmdWrite->opcode = WRITE; + cmdWrite->offset = 0; + cmdWrite->count = size; + cmdWrite->flen = std::strlen(filename); + memcpy(&cmdWrite->filename, filename, cmdWrite->flen); + dataWrite = (uint8_t*)&cmdWrite->filename[cmdWrite->flen]; + memcpy(dataWrite, "123456789123456789123456789", cmdWrite->count); - hdrWrite->type = SM_MSG_START; - hdrWrite->payloadLen = sizeof(*cmdWrite) + cmdWrite->flen + 9; + hdrWrite->type = SM_MSG_START; + hdrWrite->payloadLen = sizeof(*cmdWrite) + cmdWrite->flen + 9; - WriteTask w(clientSock, hdrWrite->payloadLen); - ssize_t result = ::write(sessionSock, cmdWrite, hdrWrite->payloadLen); - assert(result==static_cast(hdrWrite->payloadLen)); + WriteTask w(clientSock, hdrWrite->payloadLen); + ssize_t result = ::write(sessionSock, cmdWrite, hdrWrite->payloadLen); + assert(result == static_cast(hdrWrite->payloadLen)); - w.run(); + w.run(); - // verify response - uint8_t bufRead[1024]; - int err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); - sm_response *resp = (sm_response *) bufRead; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 9); + // verify response + uint8_t bufRead[1024]; + int err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + sm_response* resp = (sm_response*)bufRead; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 9); + std::vector bufAppend(sizeof(append_cmd) + std::strlen(filename) + size); + uint8_t* dataAppend; - std::vector bufAppend(sizeof(append_cmd)+std::strlen(filename)+size); - uint8_t *dataAppend; + sm_msg_header* hdrAppend = (sm_msg_header*)bufAppend.data(); + append_cmd* cmdAppend = (append_cmd*)&hdrAppend[1]; + cmdAppend->opcode = APPEND; + cmdAppend->count = size; + cmdAppend->flen = std::strlen(filename); + memcpy(&cmdAppend->filename, filename, cmdAppend->flen); + dataAppend = (uint8_t*)&cmdAppend->filename[cmdAppend->flen]; + memcpy(dataAppend, "123456789123456789123456789", cmdAppend->count); + hdrAppend->type = SM_MSG_START; + hdrAppend->payloadLen = sizeof(*cmdAppend) + cmdAppend->flen + 9; - sm_msg_header *hdrAppend = (sm_msg_header *) bufAppend.data(); - append_cmd *cmdAppend = (append_cmd *) &hdrAppend[1]; - cmdAppend->opcode = APPEND; - cmdAppend->count = size; - cmdAppend->flen = std::strlen(filename); - memcpy(&cmdAppend->filename, filename, cmdAppend->flen); - dataAppend = (uint8_t *) &cmdAppend->filename[cmdAppend->flen]; - memcpy(dataAppend, "123456789123456789123456789", cmdAppend->count); - hdrAppend->type = SM_MSG_START; - hdrAppend->payloadLen = sizeof(*cmdAppend) + cmdAppend->flen + 9; + AppendTask a(clientSock, hdrAppend->payloadLen); + err = ::write(sessionSock, cmdAppend, hdrAppend->payloadLen); - AppendTask a(clientSock, hdrAppend->payloadLen); - err = ::write(sessionSock, cmdAppend, hdrAppend->payloadLen); + a.run(); - a.run(); - - // verify response - err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); - resp = (sm_response *) bufRead; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 9); - ioc->unlink(fullPath.string().c_str()); - cout << "shortWriteMsg Test OK" << endl; + // verify response + err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + resp = (sm_response*)bufRead; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 9); + ioc->unlink(fullPath.string().c_str()); + cout << "shortWriteMsg Test OK" << endl; } // write and append are the biggest vulnerabilities here b/c those msgs could be sent in multiple @@ -1857,150 +1862,150 @@ void shortMsg() // tested as well. void shortMsgTests() { - shortMsg(); + shortMsg(); } int main(int argc, char* argv[]) { - std::size_t sizeKB = 1024; - int option; - while ((option = getopt(argc, argv, "d:p:h:")) != EOF ) + std::size_t sizeKB = 1024; + int option; + while ((option = getopt(argc, argv, "d:p:h:")) != EOF) + { + switch (option) { - switch (option) - { - case 'd': - testDirPath = optarg; - cout << "test_dir path is " << testDirPath << endl; - break; - case 'p': - prefix = optarg; - testObjKey = "12345_0_8192_" + prefix +"~test-file"; - copyfileObjKey = "12345_0_8192_" + prefix +"~source"; - metaTestFile = prefix + "/test-file"; - testFilePath = homepath / metaTestFile; - testFile = testFilePath.string().c_str(); - cout << "TestFile is " << testFile << endl; - break; - case 'h': - default: - printUsage(); - return 0; - break; - } + case 'd': + testDirPath = optarg; + cout << "test_dir path is " << testDirPath << endl; + break; + case 'p': + prefix = optarg; + testObjKey = "12345_0_8192_" + prefix + "~test-file"; + copyfileObjKey = "12345_0_8192_" + prefix + "~source"; + metaTestFile = prefix + "/test-file"; + testFilePath = homepath / metaTestFile; + testFile = testFilePath.string().c_str(); + cout << "TestFile is " << testFile << endl; + break; + case 'h': + default: + printUsage(); + return 0; + break; } + } - if (!bf::is_regular_file("test_data/storagemanager.cnf")) - { - cerr << "This should be run in a dir where ./test_data/storagemanager.cnf exists" << endl; - exit(1); - } - Config *config = Config::get("test_data/storagemanager.cnf"); - cout << "Cleaning out debris from previous runs" << endl; - bf::remove_all(config->getValue("ObjectStorage", "metadata_path")); - bf::remove_all(config->getValue("ObjectStorage", "journal_path")); - bf::remove_all(config->getValue("LocalStorage", "path")); - bf::remove_all(config->getValue("Cache", "path")); + if (!bf::is_regular_file("test_data/storagemanager.cnf")) + { + cerr << "This should be run in a dir where ./test_data/storagemanager.cnf exists" << endl; + exit(1); + } + Config* config = Config::get("test_data/storagemanager.cnf"); + cout << "Cleaning out debris from previous runs" << endl; + bf::remove_all(config->getValue("ObjectStorage", "metadata_path")); + bf::remove_all(config->getValue("ObjectStorage", "journal_path")); + bf::remove_all(config->getValue("LocalStorage", "path")); + bf::remove_all(config->getValue("Cache", "path")); - cout << "connecting" << endl; - makeConnection(); - cout << "connected" << endl; - scoped_closer sc1(serverSock), sc2(sessionSock), sc3(clientSock); + cout << "connecting" << endl; + makeConnection(); + cout << "connected" << endl; + scoped_closer sc1(serverSock), sc2(sessionSock), sc3(clientSock); - opentask(); - //metadataUpdateTest(); - // create the metadatafile to use - // requires 8K object size to test boundries - //Case 1 new write that spans full object - metadataJournalTest((10*sizeKB),0); - //Case 2 write data beyond end of data in object 2 that still ends in object 2 - metadataJournalTest((4*sizeKB),(8*sizeKB)); - //Case 3 write spans 2 journal objects - metadataJournalTest((8*sizeKB),(4*sizeKB)); - //Case 4 write starts object1 ends object3 - metadataJournalTest((16*sizeKB),(7*sizeKB)); - //Case 5 write starts in new object at 0 offset after null objects - metadataJournalTest((8*sizeKB),(32*sizeKB)); - // overwrite null objects - metadataJournalTest((10*sizeKB),(40*sizeKB)); - metadataJournalTest((8*sizeKB),(24*sizeKB)); - // overwrite whole file and create new objects - metadataJournalTest((96*sizeKB),(0)); + opentask(); + // metadataUpdateTest(); + // create the metadatafile to use + // requires 8K object size to test boundries + // Case 1 new write that spans full object + metadataJournalTest((10 * sizeKB), 0); + // Case 2 write data beyond end of data in object 2 that still ends in object 2 + metadataJournalTest((4 * sizeKB), (8 * sizeKB)); + // Case 3 write spans 2 journal objects + metadataJournalTest((8 * sizeKB), (4 * sizeKB)); + // Case 4 write starts object1 ends object3 + metadataJournalTest((16 * sizeKB), (7 * sizeKB)); + // Case 5 write starts in new object at 0 offset after null objects + metadataJournalTest((8 * sizeKB), (32 * sizeKB)); + // overwrite null objects + metadataJournalTest((10 * sizeKB), (40 * sizeKB)); + metadataJournalTest((8 * sizeKB), (24 * sizeKB)); + // overwrite whole file and create new objects + metadataJournalTest((96 * sizeKB), (0)); - //append test - // first one appends file to end of 8K object - metadataJournalTest_append((7*sizeKB)); - // this apppends that starts on new object - metadataJournalTest_append((7*sizeKB)); - // this starts in one object and crosses into new object - metadataJournalTest_append((7*sizeKB)); + // append test + // first one appends file to end of 8K object + metadataJournalTest_append((7 * sizeKB)); + // this apppends that starts on new object + metadataJournalTest_append((7 * sizeKB)); + // this starts in one object and crosses into new object + metadataJournalTest_append((7 * sizeKB)); - //writetask(); - //appendtask(); - unlinktask(); - stattask(); - truncatetask(); - listdirtask(); - pingtask(); - copytask(); + // writetask(); + // appendtask(); + unlinktask(); + stattask(); + truncatetask(); + listdirtask(); + pingtask(); + copytask(); - localstorageTest1(); - cacheTest1(); - mergeJournalTest(); - replicatorTest(); - syncTest1(); + localstorageTest1(); + cacheTest1(); + mergeJournalTest(); + replicatorTest(); + syncTest1(); - IOCReadTest1(); - IOCTruncate(); - IOCUnlink(); - IOCCopyFile(); - shortMsgTests(); + IOCReadTest1(); + IOCTruncate(); + IOCUnlink(); + IOCCopyFile(); + shortMsgTests(); - // For the moment, this next one just verifies no error happens as reported by the fcns called. - // It doesn't verify the result yet. - bigMergeJournal1(); + // For the moment, this next one just verifies no error happens as reported by the fcns called. + // It doesn't verify the result yet. + bigMergeJournal1(); - // skip the s3 test if s3 is not configured - if (config->getValue("S3", "region") != "") - { - s3storageTest1(); - } - else - cout << "To run the S3Storage unit tests, configure the S3 section of test-data/storagemanager.cnf" - << endl; + // skip the s3 test if s3 is not configured + if (config->getValue("S3", "region") != "") + { + s3storageTest1(); + } + else + cout << "To run the S3Storage unit tests, configure the S3 section of test-data/storagemanager.cnf" + << endl; - cout << "Cleanup"; - metadataJournalTestCleanup(); - cout << " DONE" << endl; + cout << "Cleanup"; + metadataJournalTestCleanup(); + cout << " DONE" << endl; - cout << "Testing connection loss..." << endl << endl; + cout << "Testing connection loss..." << endl << endl; - cout << "Check log files for lines:" << endl; - cout << "[NameTask read] caught an error: Bad file descriptor." << endl; - cout << "****** socket error!" << endl; - cout << "PosixTask::consumeMsg(): Discarding the tail end of a partial msg." << endl << endl; - //opentask(); - cout << "OpenTask read2 connection test " << endl; - opentask(true); - makeConnection(); - cout << "UnlinkTask read connection test " << endl; - unlinktask(true); - makeConnection(); - cout << "StatTask read connection test " << endl; - stattask(true); - makeConnection(); - cout << "TruncateTask read connection test " << endl; - truncatetask(true); - makeConnection(); - cout << "ListDirextoryTask read connection test " << endl; - listdirtask(true); - makeConnection(); - cout << "CopyTask read connection test " << endl; - copytask(true); + cout << "Check log files for lines:" << endl; + cout << "[NameTask read] caught an error: Bad file descriptor." << endl; + cout << "****** socket error!" << endl; + cout << "PosixTask::consumeMsg(): Discarding the tail end of a partial msg." << endl << endl; + // opentask(); + cout << "OpenTask read2 connection test " << endl; + opentask(true); + makeConnection(); + cout << "UnlinkTask read connection test " << endl; + unlinktask(true); + makeConnection(); + cout << "StatTask read connection test " << endl; + stattask(true); + makeConnection(); + cout << "TruncateTask read connection test " << endl; + truncatetask(true); + makeConnection(); + cout << "ListDirextoryTask read connection test " << endl; + listdirtask(true); + makeConnection(); + cout << "CopyTask read connection test " << endl; + copytask(true); - (Cache::get())->shutdown(); - delete (Synchronizer::get()); - delete (Cache::get()); - delete (IOCoordinator::get()); + (Cache::get())->shutdown(); + delete (Synchronizer::get()); + delete (Cache::get()); + delete (IOCoordinator::get()); - return 0; + return 0; } diff --git a/tests/arithmeticoperator-tests.cpp b/tests/arithmeticoperator-tests.cpp index 5f883eb9d..ddd1a1540 100644 --- a/tests/arithmeticoperator-tests.cpp +++ b/tests/arithmeticoperator-tests.cpp @@ -15,12 +15,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include // googletest header file +#include // googletest header file #include "mcs_basic_types.h" -//using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; +// using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; TEST(SimpleCheck, check1) { - EXPECT_EQ(true, true); + EXPECT_EQ(true, true); } diff --git a/tests/col16block.h b/tests/col16block.h index 67f68b496..68bc15e4a 100644 --- a/tests/col16block.h +++ b/tests/col16block.h @@ -18,690 +18,488 @@ #ifndef HAVE_COL16BLOCK #define HAVE_COL16BLOCK unsigned char ___bin_col16block_cdf[] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xab, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xba, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xde, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x14, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x23, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x24, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x26, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x27, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x29, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x32, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x33, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x35, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x36, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x38, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x42, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x44, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x47, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x51, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x53, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x54, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x56, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x57, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x59, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x60, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x62, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x63, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x65, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x66, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x68, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x69, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x71, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x72, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x77, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x78, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x81, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x83, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x84, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x86, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x87, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x89, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x92, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x93, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x95, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x96, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x98, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa9, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xaa, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xab, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xad, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xba, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc7, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xca, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xce, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xda, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xde, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xea, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xec, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xed, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xef, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfe, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xec, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x21, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x31, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x32, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x42, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x51, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x52, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x53, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x54, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x62, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x63, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x65, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x72, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x73, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x76, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x83, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x84, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x85, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x86, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x87, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x94, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x95, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x96, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x97, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x98, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xba, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xca, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xda, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xdb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xea, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xeb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xec, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xed, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xfa, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xfb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xfc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; unsigned int ___bin_col16block_cdf_len = 8192; constexpr int __col16block_cdf_umin = 2; constexpr int __col16block_cdf_umax = 511; diff --git a/tests/col1block.h b/tests/col1block.h index ab836f243..a9c70f00a 100644 --- a/tests/col1block.h +++ b/tests/col1block.h @@ -18,690 +18,488 @@ #ifndef HAVE_COL1BLOCK #define HAVE_COL1BLOCK unsigned char __col1block_cdf[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, - 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, - 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, - 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, - 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, - 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, - 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, - 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, - 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, - 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, - 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, - 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, - 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, - 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, - 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, - 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, - 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, - 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, - 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, - 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, - 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff -}; + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, + 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, + 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, + 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, + 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, + 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, + 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, + 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, + 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, + 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, + 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, + 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, + 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, + 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, + 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, + 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, + 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, + 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, + 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, + 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, + 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, + 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, + 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, + 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, + 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, + 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, + 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, + 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, + 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, + 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, + 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, + 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, + 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, + 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, + 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, + 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, + 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, + 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, + 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, + 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, + 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, + 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, + 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, + 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, + 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, + 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, + 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, + 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, + 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, + 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, + 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, + 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, + 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, + 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, + 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, + 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, + 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, + 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, + 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, + 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, + 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, + 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, + 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, + 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, + 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, + 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, + 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, + 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, + 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, + 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, + 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, + 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, + 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, + 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, + 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, + 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, + 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, + 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, + 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, + 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, + 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, + 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, + 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, + 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, + 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, + 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, + 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, + 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, + 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, + 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, + 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, + 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, + 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, + 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, + 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, + 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, + 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, + 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, + 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, + 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, + 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, + 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, + 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, + 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, + 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, + 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, + 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, + 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, + 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, + 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, + 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, + 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, + 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, + 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, + 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, + 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, + 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, + 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, + 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, + 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, + 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, + 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, + 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, + 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, + 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, + 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, + 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, + 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, + 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, + 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, + 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, + 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, + 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, + 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, + 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, + 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, + 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, + 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, + 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, + 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, + 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, + 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, + 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, + 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, + 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, + 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, + 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, + 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, + 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, + 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, + 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, + 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, + 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, + 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, + 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, + 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, + 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, + 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, + 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, + 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, + 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, + 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, + 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, + 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, + 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, + 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, + 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, + 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, + 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, + 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, + 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, + 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, + 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, + 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, + 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, + 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, + 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, + 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, + 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, + 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, + 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, + 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, + 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, + 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, + 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, + 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, + 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, + 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, + 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, + 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, + 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, + 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, + 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, + 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, + 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, + 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, + 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, + 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, + 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, + 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, + 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, + 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, + 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, + 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, + 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, + 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, + 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, + 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, + 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, + 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, + 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, + 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, + 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, + 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, + 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, + 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, + 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, + 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, + 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, + 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, + 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, + 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, + 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, + 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, + 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, + 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, + 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, + 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, + 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, + 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, + 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, + 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, + 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, + 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, + 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, + 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, + 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, + 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, + 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, + 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, + 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, + 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, + 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, + 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, + 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, + 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, + 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, + 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, + 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, + 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, + 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, + 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, + 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, + 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, + 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, + 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, + 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, + 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, + 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, + 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, + 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, + 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, + 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, + 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, + 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, + 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, + 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, + 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, + 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, + 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, + 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, + 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, + 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, + 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, + 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, + 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, + 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, + 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}; unsigned int __col1block_cdf_len = 8192; constexpr int __col1block_cdf_umin = -126; constexpr int __col1block_cdf_umax = 127; diff --git a/tests/col2block.h b/tests/col2block.h index 3637654a0..a47d0cfa5 100644 --- a/tests/col2block.h +++ b/tests/col2block.h @@ -18,690 +18,488 @@ #ifndef HAVE_COL2BLOCK #define HAVE_COL2BLOCK unsigned char __col2block_cdf[] = { - 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09, 0x00, 0x0a, 0x00, 0x0b, 0x00, - 0x0c, 0x00, 0x0d, 0x00, 0x0e, 0x00, 0x0f, 0x00, 0x10, 0x00, 0x11, 0x00, - 0x12, 0x00, 0x13, 0x00, 0x14, 0x00, 0x15, 0x00, 0x16, 0x00, 0x17, 0x00, - 0x18, 0x00, 0x19, 0x00, 0x1a, 0x00, 0x1b, 0x00, 0x1c, 0x00, 0x1d, 0x00, - 0x1e, 0x00, 0x1f, 0x00, 0x20, 0x00, 0x21, 0x00, 0x22, 0x00, 0x23, 0x00, - 0x24, 0x00, 0x25, 0x00, 0x26, 0x00, 0x27, 0x00, 0x28, 0x00, 0x29, 0x00, - 0x2a, 0x00, 0x2b, 0x00, 0x2c, 0x00, 0x2d, 0x00, 0x2e, 0x00, 0x2f, 0x00, - 0x30, 0x00, 0x31, 0x00, 0x32, 0x00, 0x33, 0x00, 0x34, 0x00, 0x35, 0x00, - 0x36, 0x00, 0x37, 0x00, 0x38, 0x00, 0x39, 0x00, 0x3a, 0x00, 0x3b, 0x00, - 0x3c, 0x00, 0x3d, 0x00, 0x3e, 0x00, 0x3f, 0x00, 0x40, 0x00, 0x41, 0x00, - 0x42, 0x00, 0x43, 0x00, 0x44, 0x00, 0x45, 0x00, 0x46, 0x00, 0x47, 0x00, - 0x48, 0x00, 0x49, 0x00, 0x4a, 0x00, 0x4b, 0x00, 0x4c, 0x00, 0x4d, 0x00, - 0x4e, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x51, 0x00, 0x52, 0x00, 0x53, 0x00, - 0x54, 0x00, 0x55, 0x00, 0x56, 0x00, 0x57, 0x00, 0x58, 0x00, 0x59, 0x00, - 0x5a, 0x00, 0x5b, 0x00, 0x5c, 0x00, 0x5d, 0x00, 0x5e, 0x00, 0x5f, 0x00, - 0x60, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00, 0x64, 0x00, 0x65, 0x00, - 0x66, 0x00, 0x67, 0x00, 0x68, 0x00, 0x69, 0x00, 0x6a, 0x00, 0x6b, 0x00, - 0x6c, 0x00, 0x6d, 0x00, 0x6e, 0x00, 0x6f, 0x00, 0x70, 0x00, 0x71, 0x00, - 0x72, 0x00, 0x73, 0x00, 0x74, 0x00, 0x75, 0x00, 0x76, 0x00, 0x77, 0x00, - 0x78, 0x00, 0x79, 0x00, 0x7a, 0x00, 0x7b, 0x00, 0x7c, 0x00, 0x7d, 0x00, - 0x7e, 0x00, 0x7f, 0x00, 0x80, 0x00, 0x81, 0x00, 0x82, 0x00, 0x83, 0x00, - 0x84, 0x00, 0x85, 0x00, 0x86, 0x00, 0x87, 0x00, 0x88, 0x00, 0x89, 0x00, - 0x8a, 0x00, 0x8b, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8e, 0x00, 0x8f, 0x00, - 0x90, 0x00, 0x91, 0x00, 0x92, 0x00, 0x93, 0x00, 0x94, 0x00, 0x95, 0x00, - 0x96, 0x00, 0x97, 0x00, 0x98, 0x00, 0x99, 0x00, 0x9a, 0x00, 0x9b, 0x00, - 0x9c, 0x00, 0x9d, 0x00, 0x9e, 0x00, 0x9f, 0x00, 0xa0, 0x00, 0xa1, 0x00, - 0xa2, 0x00, 0xa3, 0x00, 0xa4, 0x00, 0xa5, 0x00, 0xa6, 0x00, 0xa7, 0x00, - 0xa8, 0x00, 0xa9, 0x00, 0xaa, 0x00, 0xab, 0x00, 0xac, 0x00, 0xad, 0x00, - 0xae, 0x00, 0xaf, 0x00, 0xb0, 0x00, 0xb1, 0x00, 0xb2, 0x00, 0xb3, 0x00, - 0xb4, 0x00, 0xb5, 0x00, 0xb6, 0x00, 0xb7, 0x00, 0xb8, 0x00, 0xb9, 0x00, - 0xba, 0x00, 0xbb, 0x00, 0xbc, 0x00, 0xbd, 0x00, 0xbe, 0x00, 0xbf, 0x00, - 0xc0, 0x00, 0xc1, 0x00, 0xc2, 0x00, 0xc3, 0x00, 0xc4, 0x00, 0xc5, 0x00, - 0xc6, 0x00, 0xc7, 0x00, 0xc8, 0x00, 0xc9, 0x00, 0xca, 0x00, 0xcb, 0x00, - 0xcc, 0x00, 0xcd, 0x00, 0xce, 0x00, 0xcf, 0x00, 0xd0, 0x00, 0xd1, 0x00, - 0xd2, 0x00, 0xd3, 0x00, 0xd4, 0x00, 0xd5, 0x00, 0xd6, 0x00, 0xd7, 0x00, - 0xd8, 0x00, 0xd9, 0x00, 0xda, 0x00, 0xdb, 0x00, 0xdc, 0x00, 0xdd, 0x00, - 0xde, 0x00, 0xdf, 0x00, 0xe0, 0x00, 0xe1, 0x00, 0xe2, 0x00, 0xe3, 0x00, - 0xe4, 0x00, 0xe5, 0x00, 0xe6, 0x00, 0xe7, 0x00, 0xe8, 0x00, 0xe9, 0x00, - 0xea, 0x00, 0xeb, 0x00, 0xec, 0x00, 0xed, 0x00, 0xee, 0x00, 0xef, 0x00, - 0xf0, 0x00, 0xf1, 0x00, 0xf2, 0x00, 0xf3, 0x00, 0xf4, 0x00, 0xf5, 0x00, - 0xf6, 0x00, 0xf7, 0x00, 0xf8, 0x00, 0xf9, 0x00, 0xfa, 0x00, 0xfb, 0x00, - 0xfc, 0x00, 0xfd, 0x00, 0xfe, 0x00, 0xff, 0x00, 0x00, 0x01, 0x01, 0x01, - 0x02, 0x01, 0x03, 0x01, 0x04, 0x01, 0x05, 0x01, 0x06, 0x01, 0x07, 0x01, - 0x08, 0x01, 0x09, 0x01, 0x0a, 0x01, 0x0b, 0x01, 0x0c, 0x01, 0x0d, 0x01, - 0x0e, 0x01, 0x0f, 0x01, 0x10, 0x01, 0x11, 0x01, 0x12, 0x01, 0x13, 0x01, - 0x14, 0x01, 0x15, 0x01, 0x16, 0x01, 0x17, 0x01, 0x18, 0x01, 0x19, 0x01, - 0x1a, 0x01, 0x1b, 0x01, 0x1c, 0x01, 0x1d, 0x01, 0x1e, 0x01, 0x1f, 0x01, - 0x20, 0x01, 0x21, 0x01, 0x22, 0x01, 0x23, 0x01, 0x24, 0x01, 0x25, 0x01, - 0x26, 0x01, 0x27, 0x01, 0x28, 0x01, 0x29, 0x01, 0x2a, 0x01, 0x2b, 0x01, - 0x2c, 0x01, 0x2d, 0x01, 0x2e, 0x01, 0x2f, 0x01, 0x30, 0x01, 0x31, 0x01, - 0x32, 0x01, 0x33, 0x01, 0x34, 0x01, 0x35, 0x01, 0x36, 0x01, 0x37, 0x01, - 0x38, 0x01, 0x39, 0x01, 0x3a, 0x01, 0x3b, 0x01, 0x3c, 0x01, 0x3d, 0x01, - 0x3e, 0x01, 0x3f, 0x01, 0x40, 0x01, 0x41, 0x01, 0x42, 0x01, 0x43, 0x01, - 0x44, 0x01, 0x45, 0x01, 0x46, 0x01, 0x47, 0x01, 0x48, 0x01, 0x49, 0x01, - 0x4a, 0x01, 0x4b, 0x01, 0x4c, 0x01, 0x4d, 0x01, 0x4e, 0x01, 0x4f, 0x01, - 0x50, 0x01, 0x51, 0x01, 0x52, 0x01, 0x53, 0x01, 0x54, 0x01, 0x55, 0x01, - 0x56, 0x01, 0x57, 0x01, 0x58, 0x01, 0x59, 0x01, 0x5a, 0x01, 0x5b, 0x01, - 0x5c, 0x01, 0x5d, 0x01, 0x5e, 0x01, 0x5f, 0x01, 0x60, 0x01, 0x61, 0x01, - 0x62, 0x01, 0x63, 0x01, 0x64, 0x01, 0x65, 0x01, 0x66, 0x01, 0x67, 0x01, - 0x68, 0x01, 0x69, 0x01, 0x6a, 0x01, 0x6b, 0x01, 0x6c, 0x01, 0x6d, 0x01, - 0x6e, 0x01, 0x6f, 0x01, 0x70, 0x01, 0x71, 0x01, 0x72, 0x01, 0x73, 0x01, - 0x74, 0x01, 0x75, 0x01, 0x76, 0x01, 0x77, 0x01, 0x78, 0x01, 0x79, 0x01, - 0x7a, 0x01, 0x7b, 0x01, 0x7c, 0x01, 0x7d, 0x01, 0x7e, 0x01, 0x7f, 0x01, - 0x80, 0x01, 0x81, 0x01, 0x82, 0x01, 0x83, 0x01, 0x84, 0x01, 0x85, 0x01, - 0x86, 0x01, 0x87, 0x01, 0x88, 0x01, 0x89, 0x01, 0x8a, 0x01, 0x8b, 0x01, - 0x8c, 0x01, 0x8d, 0x01, 0x8e, 0x01, 0x8f, 0x01, 0x90, 0x01, 0x91, 0x01, - 0x92, 0x01, 0x93, 0x01, 0x94, 0x01, 0x95, 0x01, 0x96, 0x01, 0x97, 0x01, - 0x98, 0x01, 0x99, 0x01, 0x9a, 0x01, 0x9b, 0x01, 0x9c, 0x01, 0x9d, 0x01, - 0x9e, 0x01, 0x9f, 0x01, 0xa0, 0x01, 0xa1, 0x01, 0xa2, 0x01, 0xa3, 0x01, - 0xa4, 0x01, 0xa5, 0x01, 0xa6, 0x01, 0xa7, 0x01, 0xa8, 0x01, 0xa9, 0x01, - 0xaa, 0x01, 0xab, 0x01, 0xac, 0x01, 0xad, 0x01, 0xae, 0x01, 0xaf, 0x01, - 0xb0, 0x01, 0xb1, 0x01, 0xb2, 0x01, 0xb3, 0x01, 0xb4, 0x01, 0xb5, 0x01, - 0xb6, 0x01, 0xb7, 0x01, 0xb8, 0x01, 0xb9, 0x01, 0xba, 0x01, 0xbb, 0x01, - 0xbc, 0x01, 0xbd, 0x01, 0xbe, 0x01, 0xbf, 0x01, 0xc0, 0x01, 0xc1, 0x01, - 0xc2, 0x01, 0xc3, 0x01, 0xc4, 0x01, 0xc5, 0x01, 0xc6, 0x01, 0xc7, 0x01, - 0xc8, 0x01, 0xc9, 0x01, 0xca, 0x01, 0xcb, 0x01, 0xcc, 0x01, 0xcd, 0x01, - 0xce, 0x01, 0xcf, 0x01, 0xd0, 0x01, 0xd1, 0x01, 0xd2, 0x01, 0xd3, 0x01, - 0xd4, 0x01, 0xd5, 0x01, 0xd6, 0x01, 0xd7, 0x01, 0xd8, 0x01, 0xd9, 0x01, - 0xda, 0x01, 0xdb, 0x01, 0xdc, 0x01, 0xdd, 0x01, 0xde, 0x01, 0xdf, 0x01, - 0xe0, 0x01, 0xe1, 0x01, 0xe2, 0x01, 0xe3, 0x01, 0xe4, 0x01, 0xe5, 0x01, - 0xe6, 0x01, 0xe7, 0x01, 0xe8, 0x01, 0xe9, 0x01, 0xea, 0x01, 0xeb, 0x01, - 0xec, 0x01, 0xed, 0x01, 0xee, 0x01, 0xef, 0x01, 0xf0, 0x01, 0xf1, 0x01, - 0xf2, 0x01, 0xf3, 0x01, 0xf4, 0x01, 0xf5, 0x01, 0xf6, 0x01, 0xf7, 0x01, - 0xf8, 0x01, 0xf9, 0x01, 0xfa, 0x01, 0xfb, 0x01, 0xfc, 0x01, 0xfd, 0x01, - 0xfe, 0x01, 0xff, 0x01, 0x00, 0x02, 0x01, 0x02, 0x02, 0x02, 0x03, 0x02, - 0x04, 0x02, 0x05, 0x02, 0x06, 0x02, 0x07, 0x02, 0x08, 0x02, 0x09, 0x02, - 0x0a, 0x02, 0x0b, 0x02, 0x0c, 0x02, 0x0d, 0x02, 0x0e, 0x02, 0x0f, 0x02, - 0x10, 0x02, 0x11, 0x02, 0x12, 0x02, 0x13, 0x02, 0x14, 0x02, 0x15, 0x02, - 0x16, 0x02, 0x17, 0x02, 0x18, 0x02, 0x19, 0x02, 0x1a, 0x02, 0x1b, 0x02, - 0x1c, 0x02, 0x1d, 0x02, 0x1e, 0x02, 0x1f, 0x02, 0x20, 0x02, 0x21, 0x02, - 0x22, 0x02, 0x23, 0x02, 0x24, 0x02, 0x25, 0x02, 0x26, 0x02, 0x27, 0x02, - 0x28, 0x02, 0x29, 0x02, 0x2a, 0x02, 0x2b, 0x02, 0x2c, 0x02, 0x2d, 0x02, - 0x2e, 0x02, 0x2f, 0x02, 0x30, 0x02, 0x31, 0x02, 0x32, 0x02, 0x33, 0x02, - 0x34, 0x02, 0x35, 0x02, 0x36, 0x02, 0x37, 0x02, 0x38, 0x02, 0x39, 0x02, - 0x3a, 0x02, 0x3b, 0x02, 0x3c, 0x02, 0x3d, 0x02, 0x3e, 0x02, 0x3f, 0x02, - 0x40, 0x02, 0x41, 0x02, 0x42, 0x02, 0x43, 0x02, 0x44, 0x02, 0x45, 0x02, - 0x46, 0x02, 0x47, 0x02, 0x48, 0x02, 0x49, 0x02, 0x4a, 0x02, 0x4b, 0x02, - 0x4c, 0x02, 0x4d, 0x02, 0x4e, 0x02, 0x4f, 0x02, 0x50, 0x02, 0x51, 0x02, - 0x52, 0x02, 0x53, 0x02, 0x54, 0x02, 0x55, 0x02, 0x56, 0x02, 0x57, 0x02, - 0x58, 0x02, 0x59, 0x02, 0x5a, 0x02, 0x5b, 0x02, 0x5c, 0x02, 0x5d, 0x02, - 0x5e, 0x02, 0x5f, 0x02, 0x60, 0x02, 0x61, 0x02, 0x62, 0x02, 0x63, 0x02, - 0x64, 0x02, 0x65, 0x02, 0x66, 0x02, 0x67, 0x02, 0x68, 0x02, 0x69, 0x02, - 0x6a, 0x02, 0x6b, 0x02, 0x6c, 0x02, 0x6d, 0x02, 0x6e, 0x02, 0x6f, 0x02, - 0x70, 0x02, 0x71, 0x02, 0x72, 0x02, 0x73, 0x02, 0x74, 0x02, 0x75, 0x02, - 0x76, 0x02, 0x77, 0x02, 0x78, 0x02, 0x79, 0x02, 0x7a, 0x02, 0x7b, 0x02, - 0x7c, 0x02, 0x7d, 0x02, 0x7e, 0x02, 0x7f, 0x02, 0x80, 0x02, 0x81, 0x02, - 0x82, 0x02, 0x83, 0x02, 0x84, 0x02, 0x85, 0x02, 0x86, 0x02, 0x87, 0x02, - 0x88, 0x02, 0x89, 0x02, 0x8a, 0x02, 0x8b, 0x02, 0x8c, 0x02, 0x8d, 0x02, - 0x8e, 0x02, 0x8f, 0x02, 0x90, 0x02, 0x91, 0x02, 0x92, 0x02, 0x93, 0x02, - 0x94, 0x02, 0x95, 0x02, 0x96, 0x02, 0x97, 0x02, 0x98, 0x02, 0x99, 0x02, - 0x9a, 0x02, 0x9b, 0x02, 0x9c, 0x02, 0x9d, 0x02, 0x9e, 0x02, 0x9f, 0x02, - 0xa0, 0x02, 0xa1, 0x02, 0xa2, 0x02, 0xa3, 0x02, 0xa4, 0x02, 0xa5, 0x02, - 0xa6, 0x02, 0xa7, 0x02, 0xa8, 0x02, 0xa9, 0x02, 0xaa, 0x02, 0xab, 0x02, - 0xac, 0x02, 0xad, 0x02, 0xae, 0x02, 0xaf, 0x02, 0xb0, 0x02, 0xb1, 0x02, - 0xb2, 0x02, 0xb3, 0x02, 0xb4, 0x02, 0xb5, 0x02, 0xb6, 0x02, 0xb7, 0x02, - 0xb8, 0x02, 0xb9, 0x02, 0xba, 0x02, 0xbb, 0x02, 0xbc, 0x02, 0xbd, 0x02, - 0xbe, 0x02, 0xbf, 0x02, 0xc0, 0x02, 0xc1, 0x02, 0xc2, 0x02, 0xc3, 0x02, - 0xc4, 0x02, 0xc5, 0x02, 0xc6, 0x02, 0xc7, 0x02, 0xc8, 0x02, 0xc9, 0x02, - 0xca, 0x02, 0xcb, 0x02, 0xcc, 0x02, 0xcd, 0x02, 0xce, 0x02, 0xcf, 0x02, - 0xd0, 0x02, 0xd1, 0x02, 0xd2, 0x02, 0xd3, 0x02, 0xd4, 0x02, 0xd5, 0x02, - 0xd6, 0x02, 0xd7, 0x02, 0xd8, 0x02, 0xd9, 0x02, 0xda, 0x02, 0xdb, 0x02, - 0xdc, 0x02, 0xdd, 0x02, 0xde, 0x02, 0xdf, 0x02, 0xe0, 0x02, 0xe1, 0x02, - 0xe2, 0x02, 0xe3, 0x02, 0xe4, 0x02, 0xe5, 0x02, 0xe6, 0x02, 0xe7, 0x02, - 0xe8, 0x02, 0xe9, 0x02, 0xea, 0x02, 0xeb, 0x02, 0xec, 0x02, 0xed, 0x02, - 0xee, 0x02, 0xef, 0x02, 0xf0, 0x02, 0xf1, 0x02, 0xf2, 0x02, 0xf3, 0x02, - 0xf4, 0x02, 0xf5, 0x02, 0xf6, 0x02, 0xf7, 0x02, 0xf8, 0x02, 0xf9, 0x02, - 0xfa, 0x02, 0xfb, 0x02, 0xfc, 0x02, 0xfd, 0x02, 0xfe, 0x02, 0xff, 0x02, - 0x00, 0x03, 0x01, 0x03, 0x02, 0x03, 0x03, 0x03, 0x04, 0x03, 0x05, 0x03, - 0x06, 0x03, 0x07, 0x03, 0x08, 0x03, 0x09, 0x03, 0x0a, 0x03, 0x0b, 0x03, - 0x0c, 0x03, 0x0d, 0x03, 0x0e, 0x03, 0x0f, 0x03, 0x10, 0x03, 0x11, 0x03, - 0x12, 0x03, 0x13, 0x03, 0x14, 0x03, 0x15, 0x03, 0x16, 0x03, 0x17, 0x03, - 0x18, 0x03, 0x19, 0x03, 0x1a, 0x03, 0x1b, 0x03, 0x1c, 0x03, 0x1d, 0x03, - 0x1e, 0x03, 0x1f, 0x03, 0x20, 0x03, 0x21, 0x03, 0x22, 0x03, 0x23, 0x03, - 0x24, 0x03, 0x25, 0x03, 0x26, 0x03, 0x27, 0x03, 0x28, 0x03, 0x29, 0x03, - 0x2a, 0x03, 0x2b, 0x03, 0x2c, 0x03, 0x2d, 0x03, 0x2e, 0x03, 0x2f, 0x03, - 0x30, 0x03, 0x31, 0x03, 0x32, 0x03, 0x33, 0x03, 0x34, 0x03, 0x35, 0x03, - 0x36, 0x03, 0x37, 0x03, 0x38, 0x03, 0x39, 0x03, 0x3a, 0x03, 0x3b, 0x03, - 0x3c, 0x03, 0x3d, 0x03, 0x3e, 0x03, 0x3f, 0x03, 0x40, 0x03, 0x41, 0x03, - 0x42, 0x03, 0x43, 0x03, 0x44, 0x03, 0x45, 0x03, 0x46, 0x03, 0x47, 0x03, - 0x48, 0x03, 0x49, 0x03, 0x4a, 0x03, 0x4b, 0x03, 0x4c, 0x03, 0x4d, 0x03, - 0x4e, 0x03, 0x4f, 0x03, 0x50, 0x03, 0x51, 0x03, 0x52, 0x03, 0x53, 0x03, - 0x54, 0x03, 0x55, 0x03, 0x56, 0x03, 0x57, 0x03, 0x58, 0x03, 0x59, 0x03, - 0x5a, 0x03, 0x5b, 0x03, 0x5c, 0x03, 0x5d, 0x03, 0x5e, 0x03, 0x5f, 0x03, - 0x60, 0x03, 0x61, 0x03, 0x62, 0x03, 0x63, 0x03, 0x64, 0x03, 0x65, 0x03, - 0x66, 0x03, 0x67, 0x03, 0x68, 0x03, 0x69, 0x03, 0x6a, 0x03, 0x6b, 0x03, - 0x6c, 0x03, 0x6d, 0x03, 0x6e, 0x03, 0x6f, 0x03, 0x70, 0x03, 0x71, 0x03, - 0x72, 0x03, 0x73, 0x03, 0x74, 0x03, 0x75, 0x03, 0x76, 0x03, 0x77, 0x03, - 0x78, 0x03, 0x79, 0x03, 0x7a, 0x03, 0x7b, 0x03, 0x7c, 0x03, 0x7d, 0x03, - 0x7e, 0x03, 0x7f, 0x03, 0x80, 0x03, 0x81, 0x03, 0x82, 0x03, 0x83, 0x03, - 0x84, 0x03, 0x85, 0x03, 0x86, 0x03, 0x87, 0x03, 0x88, 0x03, 0x89, 0x03, - 0x8a, 0x03, 0x8b, 0x03, 0x8c, 0x03, 0x8d, 0x03, 0x8e, 0x03, 0x8f, 0x03, - 0x90, 0x03, 0x91, 0x03, 0x92, 0x03, 0x93, 0x03, 0x94, 0x03, 0x95, 0x03, - 0x96, 0x03, 0x97, 0x03, 0x98, 0x03, 0x99, 0x03, 0x9a, 0x03, 0x9b, 0x03, - 0x9c, 0x03, 0x9d, 0x03, 0x9e, 0x03, 0x9f, 0x03, 0xa0, 0x03, 0xa1, 0x03, - 0xa2, 0x03, 0xa3, 0x03, 0xa4, 0x03, 0xa5, 0x03, 0xa6, 0x03, 0xa7, 0x03, - 0xa8, 0x03, 0xa9, 0x03, 0xaa, 0x03, 0xab, 0x03, 0xac, 0x03, 0xad, 0x03, - 0xae, 0x03, 0xaf, 0x03, 0xb0, 0x03, 0xb1, 0x03, 0xb2, 0x03, 0xb3, 0x03, - 0xb4, 0x03, 0xb5, 0x03, 0xb6, 0x03, 0xb7, 0x03, 0xb8, 0x03, 0xb9, 0x03, - 0xba, 0x03, 0xbb, 0x03, 0xbc, 0x03, 0xbd, 0x03, 0xbe, 0x03, 0xbf, 0x03, - 0xc0, 0x03, 0xc1, 0x03, 0xc2, 0x03, 0xc3, 0x03, 0xc4, 0x03, 0xc5, 0x03, - 0xc6, 0x03, 0xc7, 0x03, 0xc8, 0x03, 0xc9, 0x03, 0xca, 0x03, 0xcb, 0x03, - 0xcc, 0x03, 0xcd, 0x03, 0xce, 0x03, 0xcf, 0x03, 0xd0, 0x03, 0xd1, 0x03, - 0xd2, 0x03, 0xd3, 0x03, 0xd4, 0x03, 0xd5, 0x03, 0xd6, 0x03, 0xd7, 0x03, - 0xd8, 0x03, 0xd9, 0x03, 0xda, 0x03, 0xdb, 0x03, 0xdc, 0x03, 0xdd, 0x03, - 0xde, 0x03, 0xdf, 0x03, 0xe0, 0x03, 0xe1, 0x03, 0xe2, 0x03, 0xe3, 0x03, - 0xe4, 0x03, 0xe5, 0x03, 0xe6, 0x03, 0xe7, 0x03, 0xe8, 0x03, 0xe9, 0x03, - 0xea, 0x03, 0xeb, 0x03, 0xec, 0x03, 0xed, 0x03, 0xee, 0x03, 0xef, 0x03, - 0xf0, 0x03, 0xf1, 0x03, 0xf2, 0x03, 0xf3, 0x03, 0xf4, 0x03, 0xf5, 0x03, - 0xf6, 0x03, 0xf7, 0x03, 0xf8, 0x03, 0xf9, 0x03, 0xfa, 0x03, 0xfb, 0x03, - 0xfc, 0x03, 0xfd, 0x03, 0xfe, 0x03, 0xff, 0x03, 0x00, 0x04, 0x01, 0x04, - 0x02, 0x04, 0x03, 0x04, 0x04, 0x04, 0x05, 0x04, 0x06, 0x04, 0x07, 0x04, - 0x08, 0x04, 0x09, 0x04, 0x0a, 0x04, 0x0b, 0x04, 0x0c, 0x04, 0x0d, 0x04, - 0x0e, 0x04, 0x0f, 0x04, 0x10, 0x04, 0x11, 0x04, 0x12, 0x04, 0x13, 0x04, - 0x14, 0x04, 0x15, 0x04, 0x16, 0x04, 0x17, 0x04, 0x18, 0x04, 0x19, 0x04, - 0x1a, 0x04, 0x1b, 0x04, 0x1c, 0x04, 0x1d, 0x04, 0x1e, 0x04, 0x1f, 0x04, - 0x20, 0x04, 0x21, 0x04, 0x22, 0x04, 0x23, 0x04, 0x24, 0x04, 0x25, 0x04, - 0x26, 0x04, 0x27, 0x04, 0x28, 0x04, 0x29, 0x04, 0x2a, 0x04, 0x2b, 0x04, - 0x2c, 0x04, 0x2d, 0x04, 0x2e, 0x04, 0x2f, 0x04, 0x30, 0x04, 0x31, 0x04, - 0x32, 0x04, 0x33, 0x04, 0x34, 0x04, 0x35, 0x04, 0x36, 0x04, 0x37, 0x04, - 0x38, 0x04, 0x39, 0x04, 0x3a, 0x04, 0x3b, 0x04, 0x3c, 0x04, 0x3d, 0x04, - 0x3e, 0x04, 0x3f, 0x04, 0x40, 0x04, 0x41, 0x04, 0x42, 0x04, 0x43, 0x04, - 0x44, 0x04, 0x45, 0x04, 0x46, 0x04, 0x47, 0x04, 0x48, 0x04, 0x49, 0x04, - 0x4a, 0x04, 0x4b, 0x04, 0x4c, 0x04, 0x4d, 0x04, 0x4e, 0x04, 0x4f, 0x04, - 0x50, 0x04, 0x51, 0x04, 0x52, 0x04, 0x53, 0x04, 0x54, 0x04, 0x55, 0x04, - 0x56, 0x04, 0x57, 0x04, 0x58, 0x04, 0x59, 0x04, 0x5a, 0x04, 0x5b, 0x04, - 0x5c, 0x04, 0x5d, 0x04, 0x5e, 0x04, 0x5f, 0x04, 0x60, 0x04, 0x61, 0x04, - 0x62, 0x04, 0x63, 0x04, 0x64, 0x04, 0x65, 0x04, 0x66, 0x04, 0x67, 0x04, - 0x68, 0x04, 0x69, 0x04, 0x6a, 0x04, 0x6b, 0x04, 0x6c, 0x04, 0x6d, 0x04, - 0x6e, 0x04, 0x6f, 0x04, 0x70, 0x04, 0x71, 0x04, 0x72, 0x04, 0x73, 0x04, - 0x74, 0x04, 0x75, 0x04, 0x76, 0x04, 0x77, 0x04, 0x78, 0x04, 0x79, 0x04, - 0x7a, 0x04, 0x7b, 0x04, 0x7c, 0x04, 0x7d, 0x04, 0x7e, 0x04, 0x7f, 0x04, - 0x80, 0x04, 0x81, 0x04, 0x82, 0x04, 0x83, 0x04, 0x84, 0x04, 0x85, 0x04, - 0x86, 0x04, 0x87, 0x04, 0x88, 0x04, 0x89, 0x04, 0x8a, 0x04, 0x8b, 0x04, - 0x8c, 0x04, 0x8d, 0x04, 0x8e, 0x04, 0x8f, 0x04, 0x90, 0x04, 0x91, 0x04, - 0x92, 0x04, 0x93, 0x04, 0x94, 0x04, 0x95, 0x04, 0x96, 0x04, 0x97, 0x04, - 0x98, 0x04, 0x99, 0x04, 0x9a, 0x04, 0x9b, 0x04, 0x9c, 0x04, 0x9d, 0x04, - 0x9e, 0x04, 0x9f, 0x04, 0xa0, 0x04, 0xa1, 0x04, 0xa2, 0x04, 0xa3, 0x04, - 0xa4, 0x04, 0xa5, 0x04, 0xa6, 0x04, 0xa7, 0x04, 0xa8, 0x04, 0xa9, 0x04, - 0xaa, 0x04, 0xab, 0x04, 0xac, 0x04, 0xad, 0x04, 0xae, 0x04, 0xaf, 0x04, - 0xb0, 0x04, 0xb1, 0x04, 0xb2, 0x04, 0xb3, 0x04, 0xb4, 0x04, 0xb5, 0x04, - 0xb6, 0x04, 0xb7, 0x04, 0xb8, 0x04, 0xb9, 0x04, 0xba, 0x04, 0xbb, 0x04, - 0xbc, 0x04, 0xbd, 0x04, 0xbe, 0x04, 0xbf, 0x04, 0xc0, 0x04, 0xc1, 0x04, - 0xc2, 0x04, 0xc3, 0x04, 0xc4, 0x04, 0xc5, 0x04, 0xc6, 0x04, 0xc7, 0x04, - 0xc8, 0x04, 0xc9, 0x04, 0xca, 0x04, 0xcb, 0x04, 0xcc, 0x04, 0xcd, 0x04, - 0xce, 0x04, 0xcf, 0x04, 0xd0, 0x04, 0xd1, 0x04, 0xd2, 0x04, 0xd3, 0x04, - 0xd4, 0x04, 0xd5, 0x04, 0xd6, 0x04, 0xd7, 0x04, 0xd8, 0x04, 0xd9, 0x04, - 0xda, 0x04, 0xdb, 0x04, 0xdc, 0x04, 0xdd, 0x04, 0xde, 0x04, 0xdf, 0x04, - 0xe0, 0x04, 0xe1, 0x04, 0xe2, 0x04, 0xe3, 0x04, 0xe4, 0x04, 0xe5, 0x04, - 0xe6, 0x04, 0xe7, 0x04, 0xe8, 0x04, 0xe9, 0x04, 0xea, 0x04, 0xeb, 0x04, - 0xec, 0x04, 0xed, 0x04, 0xee, 0x04, 0xef, 0x04, 0xf0, 0x04, 0xf1, 0x04, - 0xf2, 0x04, 0xf3, 0x04, 0xf4, 0x04, 0xf5, 0x04, 0xf6, 0x04, 0xf7, 0x04, - 0xf8, 0x04, 0xf9, 0x04, 0xfa, 0x04, 0xfb, 0x04, 0xfc, 0x04, 0xfd, 0x04, - 0xfe, 0x04, 0xff, 0x04, 0x00, 0x05, 0x01, 0x05, 0x02, 0x05, 0x03, 0x05, - 0x04, 0x05, 0x05, 0x05, 0x06, 0x05, 0x07, 0x05, 0x08, 0x05, 0x09, 0x05, - 0x0a, 0x05, 0x0b, 0x05, 0x0c, 0x05, 0x0d, 0x05, 0x0e, 0x05, 0x0f, 0x05, - 0x10, 0x05, 0x11, 0x05, 0x12, 0x05, 0x13, 0x05, 0x14, 0x05, 0x15, 0x05, - 0x16, 0x05, 0x17, 0x05, 0x18, 0x05, 0x19, 0x05, 0x1a, 0x05, 0x1b, 0x05, - 0x1c, 0x05, 0x1d, 0x05, 0x1e, 0x05, 0x1f, 0x05, 0x20, 0x05, 0x21, 0x05, - 0x22, 0x05, 0x23, 0x05, 0x24, 0x05, 0x25, 0x05, 0x26, 0x05, 0x27, 0x05, - 0x28, 0x05, 0x29, 0x05, 0x2a, 0x05, 0x2b, 0x05, 0x2c, 0x05, 0x2d, 0x05, - 0x2e, 0x05, 0x2f, 0x05, 0x30, 0x05, 0x31, 0x05, 0x32, 0x05, 0x33, 0x05, - 0x34, 0x05, 0x35, 0x05, 0x36, 0x05, 0x37, 0x05, 0x38, 0x05, 0x39, 0x05, - 0x3a, 0x05, 0x3b, 0x05, 0x3c, 0x05, 0x3d, 0x05, 0x3e, 0x05, 0x3f, 0x05, - 0x40, 0x05, 0x41, 0x05, 0x42, 0x05, 0x43, 0x05, 0x44, 0x05, 0x45, 0x05, - 0x46, 0x05, 0x47, 0x05, 0x48, 0x05, 0x49, 0x05, 0x4a, 0x05, 0x4b, 0x05, - 0x4c, 0x05, 0x4d, 0x05, 0x4e, 0x05, 0x4f, 0x05, 0x50, 0x05, 0x51, 0x05, - 0x52, 0x05, 0x53, 0x05, 0x54, 0x05, 0x55, 0x05, 0x56, 0x05, 0x57, 0x05, - 0x58, 0x05, 0x59, 0x05, 0x5a, 0x05, 0x5b, 0x05, 0x5c, 0x05, 0x5d, 0x05, - 0x5e, 0x05, 0x5f, 0x05, 0x60, 0x05, 0x61, 0x05, 0x62, 0x05, 0x63, 0x05, - 0x64, 0x05, 0x65, 0x05, 0x66, 0x05, 0x67, 0x05, 0x68, 0x05, 0x69, 0x05, - 0x6a, 0x05, 0x6b, 0x05, 0x6c, 0x05, 0x6d, 0x05, 0x6e, 0x05, 0x6f, 0x05, - 0x70, 0x05, 0x71, 0x05, 0x72, 0x05, 0x73, 0x05, 0x74, 0x05, 0x75, 0x05, - 0x76, 0x05, 0x77, 0x05, 0x78, 0x05, 0x79, 0x05, 0x7a, 0x05, 0x7b, 0x05, - 0x7c, 0x05, 0x7d, 0x05, 0x7e, 0x05, 0x7f, 0x05, 0x80, 0x05, 0x81, 0x05, - 0x82, 0x05, 0x83, 0x05, 0x84, 0x05, 0x85, 0x05, 0x86, 0x05, 0x87, 0x05, - 0x88, 0x05, 0x89, 0x05, 0x8a, 0x05, 0x8b, 0x05, 0x8c, 0x05, 0x8d, 0x05, - 0x8e, 0x05, 0x8f, 0x05, 0x90, 0x05, 0x91, 0x05, 0x92, 0x05, 0x93, 0x05, - 0x94, 0x05, 0x95, 0x05, 0x96, 0x05, 0x97, 0x05, 0x98, 0x05, 0x99, 0x05, - 0x9a, 0x05, 0x9b, 0x05, 0x9c, 0x05, 0x9d, 0x05, 0x9e, 0x05, 0x9f, 0x05, - 0xa0, 0x05, 0xa1, 0x05, 0xa2, 0x05, 0xa3, 0x05, 0xa4, 0x05, 0xa5, 0x05, - 0xa6, 0x05, 0xa7, 0x05, 0xa8, 0x05, 0xa9, 0x05, 0xaa, 0x05, 0xab, 0x05, - 0xac, 0x05, 0xad, 0x05, 0xae, 0x05, 0xaf, 0x05, 0xb0, 0x05, 0xb1, 0x05, - 0xb2, 0x05, 0xb3, 0x05, 0xb4, 0x05, 0xb5, 0x05, 0xb6, 0x05, 0xb7, 0x05, - 0xb8, 0x05, 0xb9, 0x05, 0xba, 0x05, 0xbb, 0x05, 0xbc, 0x05, 0xbd, 0x05, - 0xbe, 0x05, 0xbf, 0x05, 0xc0, 0x05, 0xc1, 0x05, 0xc2, 0x05, 0xc3, 0x05, - 0xc4, 0x05, 0xc5, 0x05, 0xc6, 0x05, 0xc7, 0x05, 0xc8, 0x05, 0xc9, 0x05, - 0xca, 0x05, 0xcb, 0x05, 0xcc, 0x05, 0xcd, 0x05, 0xce, 0x05, 0xcf, 0x05, - 0xd0, 0x05, 0xd1, 0x05, 0xd2, 0x05, 0xd3, 0x05, 0xd4, 0x05, 0xd5, 0x05, - 0xd6, 0x05, 0xd7, 0x05, 0xd8, 0x05, 0xd9, 0x05, 0xda, 0x05, 0xdb, 0x05, - 0xdc, 0x05, 0xdd, 0x05, 0xde, 0x05, 0xdf, 0x05, 0xe0, 0x05, 0xe1, 0x05, - 0xe2, 0x05, 0xe3, 0x05, 0xe4, 0x05, 0xe5, 0x05, 0xe6, 0x05, 0xe7, 0x05, - 0xe8, 0x05, 0xe9, 0x05, 0xea, 0x05, 0xeb, 0x05, 0xec, 0x05, 0xed, 0x05, - 0xee, 0x05, 0xef, 0x05, 0xf0, 0x05, 0xf1, 0x05, 0xf2, 0x05, 0xf3, 0x05, - 0xf4, 0x05, 0xf5, 0x05, 0xf6, 0x05, 0xf7, 0x05, 0xf8, 0x05, 0xf9, 0x05, - 0xfa, 0x05, 0xfb, 0x05, 0xfc, 0x05, 0xfd, 0x05, 0xfe, 0x05, 0xff, 0x05, - 0x00, 0x06, 0x01, 0x06, 0x02, 0x06, 0x03, 0x06, 0x04, 0x06, 0x05, 0x06, - 0x06, 0x06, 0x07, 0x06, 0x08, 0x06, 0x09, 0x06, 0x0a, 0x06, 0x0b, 0x06, - 0x0c, 0x06, 0x0d, 0x06, 0x0e, 0x06, 0x0f, 0x06, 0x10, 0x06, 0x11, 0x06, - 0x12, 0x06, 0x13, 0x06, 0x14, 0x06, 0x15, 0x06, 0x16, 0x06, 0x17, 0x06, - 0x18, 0x06, 0x19, 0x06, 0x1a, 0x06, 0x1b, 0x06, 0x1c, 0x06, 0x1d, 0x06, - 0x1e, 0x06, 0x1f, 0x06, 0x20, 0x06, 0x21, 0x06, 0x22, 0x06, 0x23, 0x06, - 0x24, 0x06, 0x25, 0x06, 0x26, 0x06, 0x27, 0x06, 0x28, 0x06, 0x29, 0x06, - 0x2a, 0x06, 0x2b, 0x06, 0x2c, 0x06, 0x2d, 0x06, 0x2e, 0x06, 0x2f, 0x06, - 0x30, 0x06, 0x31, 0x06, 0x32, 0x06, 0x33, 0x06, 0x34, 0x06, 0x35, 0x06, - 0x36, 0x06, 0x37, 0x06, 0x38, 0x06, 0x39, 0x06, 0x3a, 0x06, 0x3b, 0x06, - 0x3c, 0x06, 0x3d, 0x06, 0x3e, 0x06, 0x3f, 0x06, 0x40, 0x06, 0x41, 0x06, - 0x42, 0x06, 0x43, 0x06, 0x44, 0x06, 0x45, 0x06, 0x46, 0x06, 0x47, 0x06, - 0x48, 0x06, 0x49, 0x06, 0x4a, 0x06, 0x4b, 0x06, 0x4c, 0x06, 0x4d, 0x06, - 0x4e, 0x06, 0x4f, 0x06, 0x50, 0x06, 0x51, 0x06, 0x52, 0x06, 0x53, 0x06, - 0x54, 0x06, 0x55, 0x06, 0x56, 0x06, 0x57, 0x06, 0x58, 0x06, 0x59, 0x06, - 0x5a, 0x06, 0x5b, 0x06, 0x5c, 0x06, 0x5d, 0x06, 0x5e, 0x06, 0x5f, 0x06, - 0x60, 0x06, 0x61, 0x06, 0x62, 0x06, 0x63, 0x06, 0x64, 0x06, 0x65, 0x06, - 0x66, 0x06, 0x67, 0x06, 0x68, 0x06, 0x69, 0x06, 0x6a, 0x06, 0x6b, 0x06, - 0x6c, 0x06, 0x6d, 0x06, 0x6e, 0x06, 0x6f, 0x06, 0x70, 0x06, 0x71, 0x06, - 0x72, 0x06, 0x73, 0x06, 0x74, 0x06, 0x75, 0x06, 0x76, 0x06, 0x77, 0x06, - 0x78, 0x06, 0x79, 0x06, 0x7a, 0x06, 0x7b, 0x06, 0x7c, 0x06, 0x7d, 0x06, - 0x7e, 0x06, 0x7f, 0x06, 0x80, 0x06, 0x81, 0x06, 0x82, 0x06, 0x83, 0x06, - 0x84, 0x06, 0x85, 0x06, 0x86, 0x06, 0x87, 0x06, 0x88, 0x06, 0x89, 0x06, - 0x8a, 0x06, 0x8b, 0x06, 0x8c, 0x06, 0x8d, 0x06, 0x8e, 0x06, 0x8f, 0x06, - 0x90, 0x06, 0x91, 0x06, 0x92, 0x06, 0x93, 0x06, 0x94, 0x06, 0x95, 0x06, - 0x96, 0x06, 0x97, 0x06, 0x98, 0x06, 0x99, 0x06, 0x9a, 0x06, 0x9b, 0x06, - 0x9c, 0x06, 0x9d, 0x06, 0x9e, 0x06, 0x9f, 0x06, 0xa0, 0x06, 0xa1, 0x06, - 0xa2, 0x06, 0xa3, 0x06, 0xa4, 0x06, 0xa5, 0x06, 0xa6, 0x06, 0xa7, 0x06, - 0xa8, 0x06, 0xa9, 0x06, 0xaa, 0x06, 0xab, 0x06, 0xac, 0x06, 0xad, 0x06, - 0xae, 0x06, 0xaf, 0x06, 0xb0, 0x06, 0xb1, 0x06, 0xb2, 0x06, 0xb3, 0x06, - 0xb4, 0x06, 0xb5, 0x06, 0xb6, 0x06, 0xb7, 0x06, 0xb8, 0x06, 0xb9, 0x06, - 0xba, 0x06, 0xbb, 0x06, 0xbc, 0x06, 0xbd, 0x06, 0xbe, 0x06, 0xbf, 0x06, - 0xc0, 0x06, 0xc1, 0x06, 0xc2, 0x06, 0xc3, 0x06, 0xc4, 0x06, 0xc5, 0x06, - 0xc6, 0x06, 0xc7, 0x06, 0xc8, 0x06, 0xc9, 0x06, 0xca, 0x06, 0xcb, 0x06, - 0xcc, 0x06, 0xcd, 0x06, 0xce, 0x06, 0xcf, 0x06, 0xd0, 0x06, 0xd1, 0x06, - 0xd2, 0x06, 0xd3, 0x06, 0xd4, 0x06, 0xd5, 0x06, 0xd6, 0x06, 0xd7, 0x06, - 0xd8, 0x06, 0xd9, 0x06, 0xda, 0x06, 0xdb, 0x06, 0xdc, 0x06, 0xdd, 0x06, - 0xde, 0x06, 0xdf, 0x06, 0xe0, 0x06, 0xe1, 0x06, 0xe2, 0x06, 0xe3, 0x06, - 0xe4, 0x06, 0xe5, 0x06, 0xe6, 0x06, 0xe7, 0x06, 0xe8, 0x06, 0xe9, 0x06, - 0xea, 0x06, 0xeb, 0x06, 0xec, 0x06, 0xed, 0x06, 0xee, 0x06, 0xef, 0x06, - 0xf0, 0x06, 0xf1, 0x06, 0xf2, 0x06, 0xf3, 0x06, 0xf4, 0x06, 0xf5, 0x06, - 0xf6, 0x06, 0xf7, 0x06, 0xf8, 0x06, 0xf9, 0x06, 0xfa, 0x06, 0xfb, 0x06, - 0xfc, 0x06, 0xfd, 0x06, 0xfe, 0x06, 0xff, 0x06, 0x00, 0x07, 0x01, 0x07, - 0x02, 0x07, 0x03, 0x07, 0x04, 0x07, 0x05, 0x07, 0x06, 0x07, 0x07, 0x07, - 0x08, 0x07, 0x09, 0x07, 0x0a, 0x07, 0x0b, 0x07, 0x0c, 0x07, 0x0d, 0x07, - 0x0e, 0x07, 0x0f, 0x07, 0x10, 0x07, 0x11, 0x07, 0x12, 0x07, 0x13, 0x07, - 0x14, 0x07, 0x15, 0x07, 0x16, 0x07, 0x17, 0x07, 0x18, 0x07, 0x19, 0x07, - 0x1a, 0x07, 0x1b, 0x07, 0x1c, 0x07, 0x1d, 0x07, 0x1e, 0x07, 0x1f, 0x07, - 0x20, 0x07, 0x21, 0x07, 0x22, 0x07, 0x23, 0x07, 0x24, 0x07, 0x25, 0x07, - 0x26, 0x07, 0x27, 0x07, 0x28, 0x07, 0x29, 0x07, 0x2a, 0x07, 0x2b, 0x07, - 0x2c, 0x07, 0x2d, 0x07, 0x2e, 0x07, 0x2f, 0x07, 0x30, 0x07, 0x31, 0x07, - 0x32, 0x07, 0x33, 0x07, 0x34, 0x07, 0x35, 0x07, 0x36, 0x07, 0x37, 0x07, - 0x38, 0x07, 0x39, 0x07, 0x3a, 0x07, 0x3b, 0x07, 0x3c, 0x07, 0x3d, 0x07, - 0x3e, 0x07, 0x3f, 0x07, 0x40, 0x07, 0x41, 0x07, 0x42, 0x07, 0x43, 0x07, - 0x44, 0x07, 0x45, 0x07, 0x46, 0x07, 0x47, 0x07, 0x48, 0x07, 0x49, 0x07, - 0x4a, 0x07, 0x4b, 0x07, 0x4c, 0x07, 0x4d, 0x07, 0x4e, 0x07, 0x4f, 0x07, - 0x50, 0x07, 0x51, 0x07, 0x52, 0x07, 0x53, 0x07, 0x54, 0x07, 0x55, 0x07, - 0x56, 0x07, 0x57, 0x07, 0x58, 0x07, 0x59, 0x07, 0x5a, 0x07, 0x5b, 0x07, - 0x5c, 0x07, 0x5d, 0x07, 0x5e, 0x07, 0x5f, 0x07, 0x60, 0x07, 0x61, 0x07, - 0x62, 0x07, 0x63, 0x07, 0x64, 0x07, 0x65, 0x07, 0x66, 0x07, 0x67, 0x07, - 0x68, 0x07, 0x69, 0x07, 0x6a, 0x07, 0x6b, 0x07, 0x6c, 0x07, 0x6d, 0x07, - 0x6e, 0x07, 0x6f, 0x07, 0x70, 0x07, 0x71, 0x07, 0x72, 0x07, 0x73, 0x07, - 0x74, 0x07, 0x75, 0x07, 0x76, 0x07, 0x77, 0x07, 0x78, 0x07, 0x79, 0x07, - 0x7a, 0x07, 0x7b, 0x07, 0x7c, 0x07, 0x7d, 0x07, 0x7e, 0x07, 0x7f, 0x07, - 0x80, 0x07, 0x81, 0x07, 0x82, 0x07, 0x83, 0x07, 0x84, 0x07, 0x85, 0x07, - 0x86, 0x07, 0x87, 0x07, 0x88, 0x07, 0x89, 0x07, 0x8a, 0x07, 0x8b, 0x07, - 0x8c, 0x07, 0x8d, 0x07, 0x8e, 0x07, 0x8f, 0x07, 0x90, 0x07, 0x91, 0x07, - 0x92, 0x07, 0x93, 0x07, 0x94, 0x07, 0x95, 0x07, 0x96, 0x07, 0x97, 0x07, - 0x98, 0x07, 0x99, 0x07, 0x9a, 0x07, 0x9b, 0x07, 0x9c, 0x07, 0x9d, 0x07, - 0x9e, 0x07, 0x9f, 0x07, 0xa0, 0x07, 0xa1, 0x07, 0xa2, 0x07, 0xa3, 0x07, - 0xa4, 0x07, 0xa5, 0x07, 0xa6, 0x07, 0xa7, 0x07, 0xa8, 0x07, 0xa9, 0x07, - 0xaa, 0x07, 0xab, 0x07, 0xac, 0x07, 0xad, 0x07, 0xae, 0x07, 0xaf, 0x07, - 0xb0, 0x07, 0xb1, 0x07, 0xb2, 0x07, 0xb3, 0x07, 0xb4, 0x07, 0xb5, 0x07, - 0xb6, 0x07, 0xb7, 0x07, 0xb8, 0x07, 0xb9, 0x07, 0xba, 0x07, 0xbb, 0x07, - 0xbc, 0x07, 0xbd, 0x07, 0xbe, 0x07, 0xbf, 0x07, 0xc0, 0x07, 0xc1, 0x07, - 0xc2, 0x07, 0xc3, 0x07, 0xc4, 0x07, 0xc5, 0x07, 0xc6, 0x07, 0xc7, 0x07, - 0xc8, 0x07, 0xc9, 0x07, 0xca, 0x07, 0xcb, 0x07, 0xcc, 0x07, 0xcd, 0x07, - 0xce, 0x07, 0xcf, 0x07, 0xd0, 0x07, 0xd1, 0x07, 0xd2, 0x07, 0xd3, 0x07, - 0xd4, 0x07, 0xd5, 0x07, 0xd6, 0x07, 0xd7, 0x07, 0xd8, 0x07, 0xd9, 0x07, - 0xda, 0x07, 0xdb, 0x07, 0xdc, 0x07, 0xdd, 0x07, 0xde, 0x07, 0xdf, 0x07, - 0xe0, 0x07, 0xe1, 0x07, 0xe2, 0x07, 0xe3, 0x07, 0xe4, 0x07, 0xe5, 0x07, - 0xe6, 0x07, 0xe7, 0x07, 0xe8, 0x07, 0xe9, 0x07, 0xea, 0x07, 0xeb, 0x07, - 0xec, 0x07, 0xed, 0x07, 0xee, 0x07, 0xef, 0x07, 0xf0, 0x07, 0xf1, 0x07, - 0xf2, 0x07, 0xf3, 0x07, 0xf4, 0x07, 0xf5, 0x07, 0xf6, 0x07, 0xf7, 0x07, - 0xf8, 0x07, 0xf9, 0x07, 0xfa, 0x07, 0xfb, 0x07, 0xfc, 0x07, 0xfd, 0x07, - 0xfe, 0x07, 0xff, 0x07, 0x00, 0x08, 0x01, 0x08, 0x02, 0x08, 0x03, 0x08, - 0x04, 0x08, 0x05, 0x08, 0x06, 0x08, 0x07, 0x08, 0x08, 0x08, 0x09, 0x08, - 0x0a, 0x08, 0x0b, 0x08, 0x0c, 0x08, 0x0d, 0x08, 0x0e, 0x08, 0x0f, 0x08, - 0x10, 0x08, 0x11, 0x08, 0x12, 0x08, 0x13, 0x08, 0x14, 0x08, 0x15, 0x08, - 0x16, 0x08, 0x17, 0x08, 0x18, 0x08, 0x19, 0x08, 0x1a, 0x08, 0x1b, 0x08, - 0x1c, 0x08, 0x1d, 0x08, 0x1e, 0x08, 0x1f, 0x08, 0x20, 0x08, 0x21, 0x08, - 0x22, 0x08, 0x23, 0x08, 0x24, 0x08, 0x25, 0x08, 0x26, 0x08, 0x27, 0x08, - 0x28, 0x08, 0x29, 0x08, 0x2a, 0x08, 0x2b, 0x08, 0x2c, 0x08, 0x2d, 0x08, - 0x2e, 0x08, 0x2f, 0x08, 0x30, 0x08, 0x31, 0x08, 0x32, 0x08, 0x33, 0x08, - 0x34, 0x08, 0x35, 0x08, 0x36, 0x08, 0x37, 0x08, 0x38, 0x08, 0x39, 0x08, - 0x3a, 0x08, 0x3b, 0x08, 0x3c, 0x08, 0x3d, 0x08, 0x3e, 0x08, 0x3f, 0x08, - 0x40, 0x08, 0x41, 0x08, 0x42, 0x08, 0x43, 0x08, 0x44, 0x08, 0x45, 0x08, - 0x46, 0x08, 0x47, 0x08, 0x48, 0x08, 0x49, 0x08, 0x4a, 0x08, 0x4b, 0x08, - 0x4c, 0x08, 0x4d, 0x08, 0x4e, 0x08, 0x4f, 0x08, 0x50, 0x08, 0x51, 0x08, - 0x52, 0x08, 0x53, 0x08, 0x54, 0x08, 0x55, 0x08, 0x56, 0x08, 0x57, 0x08, - 0x58, 0x08, 0x59, 0x08, 0x5a, 0x08, 0x5b, 0x08, 0x5c, 0x08, 0x5d, 0x08, - 0x5e, 0x08, 0x5f, 0x08, 0x60, 0x08, 0x61, 0x08, 0x62, 0x08, 0x63, 0x08, - 0x64, 0x08, 0x65, 0x08, 0x66, 0x08, 0x67, 0x08, 0x68, 0x08, 0x69, 0x08, - 0x6a, 0x08, 0x6b, 0x08, 0x6c, 0x08, 0x6d, 0x08, 0x6e, 0x08, 0x6f, 0x08, - 0x70, 0x08, 0x71, 0x08, 0x72, 0x08, 0x73, 0x08, 0x74, 0x08, 0x75, 0x08, - 0x76, 0x08, 0x77, 0x08, 0x78, 0x08, 0x79, 0x08, 0x7a, 0x08, 0x7b, 0x08, - 0x7c, 0x08, 0x7d, 0x08, 0x7e, 0x08, 0x7f, 0x08, 0x80, 0x08, 0x81, 0x08, - 0x82, 0x08, 0x83, 0x08, 0x84, 0x08, 0x85, 0x08, 0x86, 0x08, 0x87, 0x08, - 0x88, 0x08, 0x89, 0x08, 0x8a, 0x08, 0x8b, 0x08, 0x8c, 0x08, 0x8d, 0x08, - 0x8e, 0x08, 0x8f, 0x08, 0x90, 0x08, 0x91, 0x08, 0x92, 0x08, 0x93, 0x08, - 0x94, 0x08, 0x95, 0x08, 0x96, 0x08, 0x97, 0x08, 0x98, 0x08, 0x99, 0x08, - 0x9a, 0x08, 0x9b, 0x08, 0x9c, 0x08, 0x9d, 0x08, 0x9e, 0x08, 0x9f, 0x08, - 0xa0, 0x08, 0xa1, 0x08, 0xa2, 0x08, 0xa3, 0x08, 0xa4, 0x08, 0xa5, 0x08, - 0xa6, 0x08, 0xa7, 0x08, 0xa8, 0x08, 0xa9, 0x08, 0xaa, 0x08, 0xab, 0x08, - 0xac, 0x08, 0xad, 0x08, 0xae, 0x08, 0xaf, 0x08, 0xb0, 0x08, 0xb1, 0x08, - 0xb2, 0x08, 0xb3, 0x08, 0xb4, 0x08, 0xb5, 0x08, 0xb6, 0x08, 0xb7, 0x08, - 0xb8, 0x08, 0xb9, 0x08, 0xba, 0x08, 0xbb, 0x08, 0xbc, 0x08, 0xbd, 0x08, - 0xbe, 0x08, 0xbf, 0x08, 0xc0, 0x08, 0xc1, 0x08, 0xc2, 0x08, 0xc3, 0x08, - 0xc4, 0x08, 0xc5, 0x08, 0xc6, 0x08, 0xc7, 0x08, 0xc8, 0x08, 0xc9, 0x08, - 0xca, 0x08, 0xcb, 0x08, 0xcc, 0x08, 0xcd, 0x08, 0xce, 0x08, 0xcf, 0x08, - 0xd0, 0x08, 0xd1, 0x08, 0xd2, 0x08, 0xd3, 0x08, 0xd4, 0x08, 0xd5, 0x08, - 0xd6, 0x08, 0xd7, 0x08, 0xd8, 0x08, 0xd9, 0x08, 0xda, 0x08, 0xdb, 0x08, - 0xdc, 0x08, 0xdd, 0x08, 0xde, 0x08, 0xdf, 0x08, 0xe0, 0x08, 0xe1, 0x08, - 0xe2, 0x08, 0xe3, 0x08, 0xe4, 0x08, 0xe5, 0x08, 0xe6, 0x08, 0xe7, 0x08, - 0xe8, 0x08, 0xe9, 0x08, 0xea, 0x08, 0xeb, 0x08, 0xec, 0x08, 0xed, 0x08, - 0xee, 0x08, 0xef, 0x08, 0xf0, 0x08, 0xf1, 0x08, 0xf2, 0x08, 0xf3, 0x08, - 0xf4, 0x08, 0xf5, 0x08, 0xf6, 0x08, 0xf7, 0x08, 0xf8, 0x08, 0xf9, 0x08, - 0xfa, 0x08, 0xfb, 0x08, 0xfc, 0x08, 0xfd, 0x08, 0xfe, 0x08, 0xff, 0x08, - 0x00, 0x09, 0x01, 0x09, 0x02, 0x09, 0x03, 0x09, 0x04, 0x09, 0x05, 0x09, - 0x06, 0x09, 0x07, 0x09, 0x08, 0x09, 0x09, 0x09, 0x0a, 0x09, 0x0b, 0x09, - 0x0c, 0x09, 0x0d, 0x09, 0x0e, 0x09, 0x0f, 0x09, 0x10, 0x09, 0x11, 0x09, - 0x12, 0x09, 0x13, 0x09, 0x14, 0x09, 0x15, 0x09, 0x16, 0x09, 0x17, 0x09, - 0x18, 0x09, 0x19, 0x09, 0x1a, 0x09, 0x1b, 0x09, 0x1c, 0x09, 0x1d, 0x09, - 0x1e, 0x09, 0x1f, 0x09, 0x20, 0x09, 0x21, 0x09, 0x22, 0x09, 0x23, 0x09, - 0x24, 0x09, 0x25, 0x09, 0x26, 0x09, 0x27, 0x09, 0x28, 0x09, 0x29, 0x09, - 0x2a, 0x09, 0x2b, 0x09, 0x2c, 0x09, 0x2d, 0x09, 0x2e, 0x09, 0x2f, 0x09, - 0x30, 0x09, 0x31, 0x09, 0x32, 0x09, 0x33, 0x09, 0x34, 0x09, 0x35, 0x09, - 0x36, 0x09, 0x37, 0x09, 0x38, 0x09, 0x39, 0x09, 0x3a, 0x09, 0x3b, 0x09, - 0x3c, 0x09, 0x3d, 0x09, 0x3e, 0x09, 0x3f, 0x09, 0x40, 0x09, 0x41, 0x09, - 0x42, 0x09, 0x43, 0x09, 0x44, 0x09, 0x45, 0x09, 0x46, 0x09, 0x47, 0x09, - 0x48, 0x09, 0x49, 0x09, 0x4a, 0x09, 0x4b, 0x09, 0x4c, 0x09, 0x4d, 0x09, - 0x4e, 0x09, 0x4f, 0x09, 0x50, 0x09, 0x51, 0x09, 0x52, 0x09, 0x53, 0x09, - 0x54, 0x09, 0x55, 0x09, 0x56, 0x09, 0x57, 0x09, 0x58, 0x09, 0x59, 0x09, - 0x5a, 0x09, 0x5b, 0x09, 0x5c, 0x09, 0x5d, 0x09, 0x5e, 0x09, 0x5f, 0x09, - 0x60, 0x09, 0x61, 0x09, 0x62, 0x09, 0x63, 0x09, 0x64, 0x09, 0x65, 0x09, - 0x66, 0x09, 0x67, 0x09, 0x68, 0x09, 0x69, 0x09, 0x6a, 0x09, 0x6b, 0x09, - 0x6c, 0x09, 0x6d, 0x09, 0x6e, 0x09, 0x6f, 0x09, 0x70, 0x09, 0x71, 0x09, - 0x72, 0x09, 0x73, 0x09, 0x74, 0x09, 0x75, 0x09, 0x76, 0x09, 0x77, 0x09, - 0x78, 0x09, 0x79, 0x09, 0x7a, 0x09, 0x7b, 0x09, 0x7c, 0x09, 0x7d, 0x09, - 0x7e, 0x09, 0x7f, 0x09, 0x80, 0x09, 0x81, 0x09, 0x82, 0x09, 0x83, 0x09, - 0x84, 0x09, 0x85, 0x09, 0x86, 0x09, 0x87, 0x09, 0x88, 0x09, 0x89, 0x09, - 0x8a, 0x09, 0x8b, 0x09, 0x8c, 0x09, 0x8d, 0x09, 0x8e, 0x09, 0x8f, 0x09, - 0x90, 0x09, 0x91, 0x09, 0x92, 0x09, 0x93, 0x09, 0x94, 0x09, 0x95, 0x09, - 0x96, 0x09, 0x97, 0x09, 0x98, 0x09, 0x99, 0x09, 0x9a, 0x09, 0x9b, 0x09, - 0x9c, 0x09, 0x9d, 0x09, 0x9e, 0x09, 0x9f, 0x09, 0xa0, 0x09, 0xa1, 0x09, - 0xa2, 0x09, 0xa3, 0x09, 0xa4, 0x09, 0xa5, 0x09, 0xa6, 0x09, 0xa7, 0x09, - 0xa8, 0x09, 0xa9, 0x09, 0xaa, 0x09, 0xab, 0x09, 0xac, 0x09, 0xad, 0x09, - 0xae, 0x09, 0xaf, 0x09, 0xb0, 0x09, 0xb1, 0x09, 0xb2, 0x09, 0xb3, 0x09, - 0xb4, 0x09, 0xb5, 0x09, 0xb6, 0x09, 0xb7, 0x09, 0xb8, 0x09, 0xb9, 0x09, - 0xba, 0x09, 0xbb, 0x09, 0xbc, 0x09, 0xbd, 0x09, 0xbe, 0x09, 0xbf, 0x09, - 0xc0, 0x09, 0xc1, 0x09, 0xc2, 0x09, 0xc3, 0x09, 0xc4, 0x09, 0xc5, 0x09, - 0xc6, 0x09, 0xc7, 0x09, 0xc8, 0x09, 0xc9, 0x09, 0xca, 0x09, 0xcb, 0x09, - 0xcc, 0x09, 0xcd, 0x09, 0xce, 0x09, 0xcf, 0x09, 0xd0, 0x09, 0xd1, 0x09, - 0xd2, 0x09, 0xd3, 0x09, 0xd4, 0x09, 0xd5, 0x09, 0xd6, 0x09, 0xd7, 0x09, - 0xd8, 0x09, 0xd9, 0x09, 0xda, 0x09, 0xdb, 0x09, 0xdc, 0x09, 0xdd, 0x09, - 0xde, 0x09, 0xdf, 0x09, 0xe0, 0x09, 0xe1, 0x09, 0xe2, 0x09, 0xe3, 0x09, - 0xe4, 0x09, 0xe5, 0x09, 0xe6, 0x09, 0xe7, 0x09, 0xe8, 0x09, 0xe9, 0x09, - 0xea, 0x09, 0xeb, 0x09, 0xec, 0x09, 0xed, 0x09, 0xee, 0x09, 0xef, 0x09, - 0xf0, 0x09, 0xf1, 0x09, 0xf2, 0x09, 0xf3, 0x09, 0xf4, 0x09, 0xf5, 0x09, - 0xf6, 0x09, 0xf7, 0x09, 0xf8, 0x09, 0xf9, 0x09, 0xfa, 0x09, 0xfb, 0x09, - 0xfc, 0x09, 0xfd, 0x09, 0xfe, 0x09, 0xff, 0x09, 0x00, 0x0a, 0x01, 0x0a, - 0x02, 0x0a, 0x03, 0x0a, 0x04, 0x0a, 0x05, 0x0a, 0x06, 0x0a, 0x07, 0x0a, - 0x08, 0x0a, 0x09, 0x0a, 0x0a, 0x0a, 0x0b, 0x0a, 0x0c, 0x0a, 0x0d, 0x0a, - 0x0e, 0x0a, 0x0f, 0x0a, 0x10, 0x0a, 0x11, 0x0a, 0x12, 0x0a, 0x13, 0x0a, - 0x14, 0x0a, 0x15, 0x0a, 0x16, 0x0a, 0x17, 0x0a, 0x18, 0x0a, 0x19, 0x0a, - 0x1a, 0x0a, 0x1b, 0x0a, 0x1c, 0x0a, 0x1d, 0x0a, 0x1e, 0x0a, 0x1f, 0x0a, - 0x20, 0x0a, 0x21, 0x0a, 0x22, 0x0a, 0x23, 0x0a, 0x24, 0x0a, 0x25, 0x0a, - 0x26, 0x0a, 0x27, 0x0a, 0x28, 0x0a, 0x29, 0x0a, 0x2a, 0x0a, 0x2b, 0x0a, - 0x2c, 0x0a, 0x2d, 0x0a, 0x2e, 0x0a, 0x2f, 0x0a, 0x30, 0x0a, 0x31, 0x0a, - 0x32, 0x0a, 0x33, 0x0a, 0x34, 0x0a, 0x35, 0x0a, 0x36, 0x0a, 0x37, 0x0a, - 0x38, 0x0a, 0x39, 0x0a, 0x3a, 0x0a, 0x3b, 0x0a, 0x3c, 0x0a, 0x3d, 0x0a, - 0x3e, 0x0a, 0x3f, 0x0a, 0x40, 0x0a, 0x41, 0x0a, 0x42, 0x0a, 0x43, 0x0a, - 0x44, 0x0a, 0x45, 0x0a, 0x46, 0x0a, 0x47, 0x0a, 0x48, 0x0a, 0x49, 0x0a, - 0x4a, 0x0a, 0x4b, 0x0a, 0x4c, 0x0a, 0x4d, 0x0a, 0x4e, 0x0a, 0x4f, 0x0a, - 0x50, 0x0a, 0x51, 0x0a, 0x52, 0x0a, 0x53, 0x0a, 0x54, 0x0a, 0x55, 0x0a, - 0x56, 0x0a, 0x57, 0x0a, 0x58, 0x0a, 0x59, 0x0a, 0x5a, 0x0a, 0x5b, 0x0a, - 0x5c, 0x0a, 0x5d, 0x0a, 0x5e, 0x0a, 0x5f, 0x0a, 0x60, 0x0a, 0x61, 0x0a, - 0x62, 0x0a, 0x63, 0x0a, 0x64, 0x0a, 0x65, 0x0a, 0x66, 0x0a, 0x67, 0x0a, - 0x68, 0x0a, 0x69, 0x0a, 0x6a, 0x0a, 0x6b, 0x0a, 0x6c, 0x0a, 0x6d, 0x0a, - 0x6e, 0x0a, 0x6f, 0x0a, 0x70, 0x0a, 0x71, 0x0a, 0x72, 0x0a, 0x73, 0x0a, - 0x74, 0x0a, 0x75, 0x0a, 0x76, 0x0a, 0x77, 0x0a, 0x78, 0x0a, 0x79, 0x0a, - 0x7a, 0x0a, 0x7b, 0x0a, 0x7c, 0x0a, 0x7d, 0x0a, 0x7e, 0x0a, 0x7f, 0x0a, - 0x80, 0x0a, 0x81, 0x0a, 0x82, 0x0a, 0x83, 0x0a, 0x84, 0x0a, 0x85, 0x0a, - 0x86, 0x0a, 0x87, 0x0a, 0x88, 0x0a, 0x89, 0x0a, 0x8a, 0x0a, 0x8b, 0x0a, - 0x8c, 0x0a, 0x8d, 0x0a, 0x8e, 0x0a, 0x8f, 0x0a, 0x90, 0x0a, 0x91, 0x0a, - 0x92, 0x0a, 0x93, 0x0a, 0x94, 0x0a, 0x95, 0x0a, 0x96, 0x0a, 0x97, 0x0a, - 0x98, 0x0a, 0x99, 0x0a, 0x9a, 0x0a, 0x9b, 0x0a, 0x9c, 0x0a, 0x9d, 0x0a, - 0x9e, 0x0a, 0x9f, 0x0a, 0xa0, 0x0a, 0xa1, 0x0a, 0xa2, 0x0a, 0xa3, 0x0a, - 0xa4, 0x0a, 0xa5, 0x0a, 0xa6, 0x0a, 0xa7, 0x0a, 0xa8, 0x0a, 0xa9, 0x0a, - 0xaa, 0x0a, 0xab, 0x0a, 0xac, 0x0a, 0xad, 0x0a, 0xae, 0x0a, 0xaf, 0x0a, - 0xb0, 0x0a, 0xb1, 0x0a, 0xb2, 0x0a, 0xb3, 0x0a, 0xb4, 0x0a, 0xb5, 0x0a, - 0xb6, 0x0a, 0xb7, 0x0a, 0xb8, 0x0a, 0xb9, 0x0a, 0xba, 0x0a, 0xbb, 0x0a, - 0xbc, 0x0a, 0xbd, 0x0a, 0xbe, 0x0a, 0xbf, 0x0a, 0xc0, 0x0a, 0xc1, 0x0a, - 0xc2, 0x0a, 0xc3, 0x0a, 0xc4, 0x0a, 0xc5, 0x0a, 0xc6, 0x0a, 0xc7, 0x0a, - 0xc8, 0x0a, 0xc9, 0x0a, 0xca, 0x0a, 0xcb, 0x0a, 0xcc, 0x0a, 0xcd, 0x0a, - 0xce, 0x0a, 0xcf, 0x0a, 0xd0, 0x0a, 0xd1, 0x0a, 0xd2, 0x0a, 0xd3, 0x0a, - 0xd4, 0x0a, 0xd5, 0x0a, 0xd6, 0x0a, 0xd7, 0x0a, 0xd8, 0x0a, 0xd9, 0x0a, - 0xda, 0x0a, 0xdb, 0x0a, 0xdc, 0x0a, 0xdd, 0x0a, 0xde, 0x0a, 0xdf, 0x0a, - 0xe0, 0x0a, 0xe1, 0x0a, 0xe2, 0x0a, 0xe3, 0x0a, 0xe4, 0x0a, 0xe5, 0x0a, - 0xe6, 0x0a, 0xe7, 0x0a, 0xe8, 0x0a, 0xe9, 0x0a, 0xea, 0x0a, 0xeb, 0x0a, - 0xec, 0x0a, 0xed, 0x0a, 0xee, 0x0a, 0xef, 0x0a, 0xf0, 0x0a, 0xf1, 0x0a, - 0xf2, 0x0a, 0xf3, 0x0a, 0xf4, 0x0a, 0xf5, 0x0a, 0xf6, 0x0a, 0xf7, 0x0a, - 0xf8, 0x0a, 0xf9, 0x0a, 0xfa, 0x0a, 0xfb, 0x0a, 0xfc, 0x0a, 0xfd, 0x0a, - 0xfe, 0x0a, 0xff, 0x0a, 0x00, 0x0b, 0x01, 0x0b, 0x02, 0x0b, 0x03, 0x0b, - 0x04, 0x0b, 0x05, 0x0b, 0x06, 0x0b, 0x07, 0x0b, 0x08, 0x0b, 0x09, 0x0b, - 0x0a, 0x0b, 0x0b, 0x0b, 0x0c, 0x0b, 0x0d, 0x0b, 0x0e, 0x0b, 0x0f, 0x0b, - 0x10, 0x0b, 0x11, 0x0b, 0x12, 0x0b, 0x13, 0x0b, 0x14, 0x0b, 0x15, 0x0b, - 0x16, 0x0b, 0x17, 0x0b, 0x18, 0x0b, 0x19, 0x0b, 0x1a, 0x0b, 0x1b, 0x0b, - 0x1c, 0x0b, 0x1d, 0x0b, 0x1e, 0x0b, 0x1f, 0x0b, 0x20, 0x0b, 0x21, 0x0b, - 0x22, 0x0b, 0x23, 0x0b, 0x24, 0x0b, 0x25, 0x0b, 0x26, 0x0b, 0x27, 0x0b, - 0x28, 0x0b, 0x29, 0x0b, 0x2a, 0x0b, 0x2b, 0x0b, 0x2c, 0x0b, 0x2d, 0x0b, - 0x2e, 0x0b, 0x2f, 0x0b, 0x30, 0x0b, 0x31, 0x0b, 0x32, 0x0b, 0x33, 0x0b, - 0x34, 0x0b, 0x35, 0x0b, 0x36, 0x0b, 0x37, 0x0b, 0x38, 0x0b, 0x39, 0x0b, - 0x3a, 0x0b, 0x3b, 0x0b, 0x3c, 0x0b, 0x3d, 0x0b, 0x3e, 0x0b, 0x3f, 0x0b, - 0x40, 0x0b, 0x41, 0x0b, 0x42, 0x0b, 0x43, 0x0b, 0x44, 0x0b, 0x45, 0x0b, - 0x46, 0x0b, 0x47, 0x0b, 0x48, 0x0b, 0x49, 0x0b, 0x4a, 0x0b, 0x4b, 0x0b, - 0x4c, 0x0b, 0x4d, 0x0b, 0x4e, 0x0b, 0x4f, 0x0b, 0x50, 0x0b, 0x51, 0x0b, - 0x52, 0x0b, 0x53, 0x0b, 0x54, 0x0b, 0x55, 0x0b, 0x56, 0x0b, 0x57, 0x0b, - 0x58, 0x0b, 0x59, 0x0b, 0x5a, 0x0b, 0x5b, 0x0b, 0x5c, 0x0b, 0x5d, 0x0b, - 0x5e, 0x0b, 0x5f, 0x0b, 0x60, 0x0b, 0x61, 0x0b, 0x62, 0x0b, 0x63, 0x0b, - 0x64, 0x0b, 0x65, 0x0b, 0x66, 0x0b, 0x67, 0x0b, 0x68, 0x0b, 0x69, 0x0b, - 0x6a, 0x0b, 0x6b, 0x0b, 0x6c, 0x0b, 0x6d, 0x0b, 0x6e, 0x0b, 0x6f, 0x0b, - 0x70, 0x0b, 0x71, 0x0b, 0x72, 0x0b, 0x73, 0x0b, 0x74, 0x0b, 0x75, 0x0b, - 0x76, 0x0b, 0x77, 0x0b, 0x78, 0x0b, 0x79, 0x0b, 0x7a, 0x0b, 0x7b, 0x0b, - 0x7c, 0x0b, 0x7d, 0x0b, 0x7e, 0x0b, 0x7f, 0x0b, 0x80, 0x0b, 0x81, 0x0b, - 0x82, 0x0b, 0x83, 0x0b, 0x84, 0x0b, 0x85, 0x0b, 0x86, 0x0b, 0x87, 0x0b, - 0x88, 0x0b, 0x89, 0x0b, 0x8a, 0x0b, 0x8b, 0x0b, 0x8c, 0x0b, 0x8d, 0x0b, - 0x8e, 0x0b, 0x8f, 0x0b, 0x90, 0x0b, 0x91, 0x0b, 0x92, 0x0b, 0x93, 0x0b, - 0x94, 0x0b, 0x95, 0x0b, 0x96, 0x0b, 0x97, 0x0b, 0x98, 0x0b, 0x99, 0x0b, - 0x9a, 0x0b, 0x9b, 0x0b, 0x9c, 0x0b, 0x9d, 0x0b, 0x9e, 0x0b, 0x9f, 0x0b, - 0xa0, 0x0b, 0xa1, 0x0b, 0xa2, 0x0b, 0xa3, 0x0b, 0xa4, 0x0b, 0xa5, 0x0b, - 0xa6, 0x0b, 0xa7, 0x0b, 0xa8, 0x0b, 0xa9, 0x0b, 0xaa, 0x0b, 0xab, 0x0b, - 0xac, 0x0b, 0xad, 0x0b, 0xae, 0x0b, 0xaf, 0x0b, 0xb0, 0x0b, 0xb1, 0x0b, - 0xb2, 0x0b, 0xb3, 0x0b, 0xb4, 0x0b, 0xb5, 0x0b, 0xb6, 0x0b, 0xb7, 0x0b, - 0xb8, 0x0b, 0xb9, 0x0b, 0xba, 0x0b, 0xbb, 0x0b, 0xbc, 0x0b, 0xbd, 0x0b, - 0xbe, 0x0b, 0xbf, 0x0b, 0xc0, 0x0b, 0xc1, 0x0b, 0xc2, 0x0b, 0xc3, 0x0b, - 0xc4, 0x0b, 0xc5, 0x0b, 0xc6, 0x0b, 0xc7, 0x0b, 0xc8, 0x0b, 0xc9, 0x0b, - 0xca, 0x0b, 0xcb, 0x0b, 0xcc, 0x0b, 0xcd, 0x0b, 0xce, 0x0b, 0xcf, 0x0b, - 0xd0, 0x0b, 0xd1, 0x0b, 0xd2, 0x0b, 0xd3, 0x0b, 0xd4, 0x0b, 0xd5, 0x0b, - 0xd6, 0x0b, 0xd7, 0x0b, 0xd8, 0x0b, 0xd9, 0x0b, 0xda, 0x0b, 0xdb, 0x0b, - 0xdc, 0x0b, 0xdd, 0x0b, 0xde, 0x0b, 0xdf, 0x0b, 0xe0, 0x0b, 0xe1, 0x0b, - 0xe2, 0x0b, 0xe3, 0x0b, 0xe4, 0x0b, 0xe5, 0x0b, 0xe6, 0x0b, 0xe7, 0x0b, - 0xe8, 0x0b, 0xe9, 0x0b, 0xea, 0x0b, 0xeb, 0x0b, 0xec, 0x0b, 0xed, 0x0b, - 0xee, 0x0b, 0xef, 0x0b, 0xf0, 0x0b, 0xf1, 0x0b, 0xf2, 0x0b, 0xf3, 0x0b, - 0xf4, 0x0b, 0xf5, 0x0b, 0xf6, 0x0b, 0xf7, 0x0b, 0xf8, 0x0b, 0xf9, 0x0b, - 0xfa, 0x0b, 0xfb, 0x0b, 0xfc, 0x0b, 0xfd, 0x0b, 0xfe, 0x0b, 0xff, 0x0b, - 0x00, 0x0c, 0x01, 0x0c, 0x02, 0x0c, 0x03, 0x0c, 0x04, 0x0c, 0x05, 0x0c, - 0x06, 0x0c, 0x07, 0x0c, 0x08, 0x0c, 0x09, 0x0c, 0x0a, 0x0c, 0x0b, 0x0c, - 0x0c, 0x0c, 0x0d, 0x0c, 0x0e, 0x0c, 0x0f, 0x0c, 0x10, 0x0c, 0x11, 0x0c, - 0x12, 0x0c, 0x13, 0x0c, 0x14, 0x0c, 0x15, 0x0c, 0x16, 0x0c, 0x17, 0x0c, - 0x18, 0x0c, 0x19, 0x0c, 0x1a, 0x0c, 0x1b, 0x0c, 0x1c, 0x0c, 0x1d, 0x0c, - 0x1e, 0x0c, 0x1f, 0x0c, 0x20, 0x0c, 0x21, 0x0c, 0x22, 0x0c, 0x23, 0x0c, - 0x24, 0x0c, 0x25, 0x0c, 0x26, 0x0c, 0x27, 0x0c, 0x28, 0x0c, 0x29, 0x0c, - 0x2a, 0x0c, 0x2b, 0x0c, 0x2c, 0x0c, 0x2d, 0x0c, 0x2e, 0x0c, 0x2f, 0x0c, - 0x30, 0x0c, 0x31, 0x0c, 0x32, 0x0c, 0x33, 0x0c, 0x34, 0x0c, 0x35, 0x0c, - 0x36, 0x0c, 0x37, 0x0c, 0x38, 0x0c, 0x39, 0x0c, 0x3a, 0x0c, 0x3b, 0x0c, - 0x3c, 0x0c, 0x3d, 0x0c, 0x3e, 0x0c, 0x3f, 0x0c, 0x40, 0x0c, 0x41, 0x0c, - 0x42, 0x0c, 0x43, 0x0c, 0x44, 0x0c, 0x45, 0x0c, 0x46, 0x0c, 0x47, 0x0c, - 0x48, 0x0c, 0x49, 0x0c, 0x4a, 0x0c, 0x4b, 0x0c, 0x4c, 0x0c, 0x4d, 0x0c, - 0x4e, 0x0c, 0x4f, 0x0c, 0x50, 0x0c, 0x51, 0x0c, 0x52, 0x0c, 0x53, 0x0c, - 0x54, 0x0c, 0x55, 0x0c, 0x56, 0x0c, 0x57, 0x0c, 0x58, 0x0c, 0x59, 0x0c, - 0x5a, 0x0c, 0x5b, 0x0c, 0x5c, 0x0c, 0x5d, 0x0c, 0x5e, 0x0c, 0x5f, 0x0c, - 0x60, 0x0c, 0x61, 0x0c, 0x62, 0x0c, 0x63, 0x0c, 0x64, 0x0c, 0x65, 0x0c, - 0x66, 0x0c, 0x67, 0x0c, 0x68, 0x0c, 0x69, 0x0c, 0x6a, 0x0c, 0x6b, 0x0c, - 0x6c, 0x0c, 0x6d, 0x0c, 0x6e, 0x0c, 0x6f, 0x0c, 0x70, 0x0c, 0x71, 0x0c, - 0x72, 0x0c, 0x73, 0x0c, 0x74, 0x0c, 0x75, 0x0c, 0x76, 0x0c, 0x77, 0x0c, - 0x78, 0x0c, 0x79, 0x0c, 0x7a, 0x0c, 0x7b, 0x0c, 0x7c, 0x0c, 0x7d, 0x0c, - 0x7e, 0x0c, 0x7f, 0x0c, 0x80, 0x0c, 0x81, 0x0c, 0x82, 0x0c, 0x83, 0x0c, - 0x84, 0x0c, 0x85, 0x0c, 0x86, 0x0c, 0x87, 0x0c, 0x88, 0x0c, 0x89, 0x0c, - 0x8a, 0x0c, 0x8b, 0x0c, 0x8c, 0x0c, 0x8d, 0x0c, 0x8e, 0x0c, 0x8f, 0x0c, - 0x90, 0x0c, 0x91, 0x0c, 0x92, 0x0c, 0x93, 0x0c, 0x94, 0x0c, 0x95, 0x0c, - 0x96, 0x0c, 0x97, 0x0c, 0x98, 0x0c, 0x99, 0x0c, 0x9a, 0x0c, 0x9b, 0x0c, - 0x9c, 0x0c, 0x9d, 0x0c, 0x9e, 0x0c, 0x9f, 0x0c, 0xa0, 0x0c, 0xa1, 0x0c, - 0xa2, 0x0c, 0xa3, 0x0c, 0xa4, 0x0c, 0xa5, 0x0c, 0xa6, 0x0c, 0xa7, 0x0c, - 0xa8, 0x0c, 0xa9, 0x0c, 0xaa, 0x0c, 0xab, 0x0c, 0xac, 0x0c, 0xad, 0x0c, - 0xae, 0x0c, 0xaf, 0x0c, 0xb0, 0x0c, 0xb1, 0x0c, 0xb2, 0x0c, 0xb3, 0x0c, - 0xb4, 0x0c, 0xb5, 0x0c, 0xb6, 0x0c, 0xb7, 0x0c, 0xb8, 0x0c, 0xb9, 0x0c, - 0xba, 0x0c, 0xbb, 0x0c, 0xbc, 0x0c, 0xbd, 0x0c, 0xbe, 0x0c, 0xbf, 0x0c, - 0xc0, 0x0c, 0xc1, 0x0c, 0xc2, 0x0c, 0xc3, 0x0c, 0xc4, 0x0c, 0xc5, 0x0c, - 0xc6, 0x0c, 0xc7, 0x0c, 0xc8, 0x0c, 0xc9, 0x0c, 0xca, 0x0c, 0xcb, 0x0c, - 0xcc, 0x0c, 0xcd, 0x0c, 0xce, 0x0c, 0xcf, 0x0c, 0xd0, 0x0c, 0xd1, 0x0c, - 0xd2, 0x0c, 0xd3, 0x0c, 0xd4, 0x0c, 0xd5, 0x0c, 0xd6, 0x0c, 0xd7, 0x0c, - 0xd8, 0x0c, 0xd9, 0x0c, 0xda, 0x0c, 0xdb, 0x0c, 0xdc, 0x0c, 0xdd, 0x0c, - 0xde, 0x0c, 0xdf, 0x0c, 0xe0, 0x0c, 0xe1, 0x0c, 0xe2, 0x0c, 0xe3, 0x0c, - 0xe4, 0x0c, 0xe5, 0x0c, 0xe6, 0x0c, 0xe7, 0x0c, 0xe8, 0x0c, 0xe9, 0x0c, - 0xea, 0x0c, 0xeb, 0x0c, 0xec, 0x0c, 0xed, 0x0c, 0xee, 0x0c, 0xef, 0x0c, - 0xf0, 0x0c, 0xf1, 0x0c, 0xf2, 0x0c, 0xf3, 0x0c, 0xf4, 0x0c, 0xf5, 0x0c, - 0xf6, 0x0c, 0xf7, 0x0c, 0xf8, 0x0c, 0xf9, 0x0c, 0xfa, 0x0c, 0xfb, 0x0c, - 0xfc, 0x0c, 0xfd, 0x0c, 0xfe, 0x0c, 0xff, 0x0c, 0x00, 0x0d, 0x01, 0x0d, - 0x02, 0x0d, 0x03, 0x0d, 0x04, 0x0d, 0x05, 0x0d, 0x06, 0x0d, 0x07, 0x0d, - 0x08, 0x0d, 0x09, 0x0d, 0x0a, 0x0d, 0x0b, 0x0d, 0x0c, 0x0d, 0x0d, 0x0d, - 0x0e, 0x0d, 0x0f, 0x0d, 0x10, 0x0d, 0x11, 0x0d, 0x12, 0x0d, 0x13, 0x0d, - 0x14, 0x0d, 0x15, 0x0d, 0x16, 0x0d, 0x17, 0x0d, 0x18, 0x0d, 0x19, 0x0d, - 0x1a, 0x0d, 0x1b, 0x0d, 0x1c, 0x0d, 0x1d, 0x0d, 0x1e, 0x0d, 0x1f, 0x0d, - 0x20, 0x0d, 0x21, 0x0d, 0x22, 0x0d, 0x23, 0x0d, 0x24, 0x0d, 0x25, 0x0d, - 0x26, 0x0d, 0x27, 0x0d, 0x28, 0x0d, 0x29, 0x0d, 0x2a, 0x0d, 0x2b, 0x0d, - 0x2c, 0x0d, 0x2d, 0x0d, 0x2e, 0x0d, 0x2f, 0x0d, 0x30, 0x0d, 0x31, 0x0d, - 0x32, 0x0d, 0x33, 0x0d, 0x34, 0x0d, 0x35, 0x0d, 0x36, 0x0d, 0x37, 0x0d, - 0x38, 0x0d, 0x39, 0x0d, 0x3a, 0x0d, 0x3b, 0x0d, 0x3c, 0x0d, 0x3d, 0x0d, - 0x3e, 0x0d, 0x3f, 0x0d, 0x40, 0x0d, 0x41, 0x0d, 0x42, 0x0d, 0x43, 0x0d, - 0x44, 0x0d, 0x45, 0x0d, 0x46, 0x0d, 0x47, 0x0d, 0x48, 0x0d, 0x49, 0x0d, - 0x4a, 0x0d, 0x4b, 0x0d, 0x4c, 0x0d, 0x4d, 0x0d, 0x4e, 0x0d, 0x4f, 0x0d, - 0x50, 0x0d, 0x51, 0x0d, 0x52, 0x0d, 0x53, 0x0d, 0x54, 0x0d, 0x55, 0x0d, - 0x56, 0x0d, 0x57, 0x0d, 0x58, 0x0d, 0x59, 0x0d, 0x5a, 0x0d, 0x5b, 0x0d, - 0x5c, 0x0d, 0x5d, 0x0d, 0x5e, 0x0d, 0x5f, 0x0d, 0x60, 0x0d, 0x61, 0x0d, - 0x62, 0x0d, 0x63, 0x0d, 0x64, 0x0d, 0x65, 0x0d, 0x66, 0x0d, 0x67, 0x0d, - 0x68, 0x0d, 0x69, 0x0d, 0x6a, 0x0d, 0x6b, 0x0d, 0x6c, 0x0d, 0x6d, 0x0d, - 0x6e, 0x0d, 0x6f, 0x0d, 0x70, 0x0d, 0x71, 0x0d, 0x72, 0x0d, 0x73, 0x0d, - 0x74, 0x0d, 0x75, 0x0d, 0x76, 0x0d, 0x77, 0x0d, 0x78, 0x0d, 0x79, 0x0d, - 0x7a, 0x0d, 0x7b, 0x0d, 0x7c, 0x0d, 0x7d, 0x0d, 0x7e, 0x0d, 0x7f, 0x0d, - 0x80, 0x0d, 0x81, 0x0d, 0x82, 0x0d, 0x83, 0x0d, 0x84, 0x0d, 0x85, 0x0d, - 0x86, 0x0d, 0x87, 0x0d, 0x88, 0x0d, 0x89, 0x0d, 0x8a, 0x0d, 0x8b, 0x0d, - 0x8c, 0x0d, 0x8d, 0x0d, 0x8e, 0x0d, 0x8f, 0x0d, 0x90, 0x0d, 0x91, 0x0d, - 0x92, 0x0d, 0x93, 0x0d, 0x94, 0x0d, 0x95, 0x0d, 0x96, 0x0d, 0x97, 0x0d, - 0x98, 0x0d, 0x99, 0x0d, 0x9a, 0x0d, 0x9b, 0x0d, 0x9c, 0x0d, 0x9d, 0x0d, - 0x9e, 0x0d, 0x9f, 0x0d, 0xa0, 0x0d, 0xa1, 0x0d, 0xa2, 0x0d, 0xa3, 0x0d, - 0xa4, 0x0d, 0xa5, 0x0d, 0xa6, 0x0d, 0xa7, 0x0d, 0xa8, 0x0d, 0xa9, 0x0d, - 0xaa, 0x0d, 0xab, 0x0d, 0xac, 0x0d, 0xad, 0x0d, 0xae, 0x0d, 0xaf, 0x0d, - 0xb0, 0x0d, 0xb1, 0x0d, 0xb2, 0x0d, 0xb3, 0x0d, 0xb4, 0x0d, 0xb5, 0x0d, - 0xb6, 0x0d, 0xb7, 0x0d, 0xb8, 0x0d, 0xb9, 0x0d, 0xba, 0x0d, 0xbb, 0x0d, - 0xbc, 0x0d, 0xbd, 0x0d, 0xbe, 0x0d, 0xbf, 0x0d, 0xc0, 0x0d, 0xc1, 0x0d, - 0xc2, 0x0d, 0xc3, 0x0d, 0xc4, 0x0d, 0xc5, 0x0d, 0xc6, 0x0d, 0xc7, 0x0d, - 0xc8, 0x0d, 0xc9, 0x0d, 0xca, 0x0d, 0xcb, 0x0d, 0xcc, 0x0d, 0xcd, 0x0d, - 0xce, 0x0d, 0xcf, 0x0d, 0xd0, 0x0d, 0xd1, 0x0d, 0xd2, 0x0d, 0xd3, 0x0d, - 0xd4, 0x0d, 0xd5, 0x0d, 0xd6, 0x0d, 0xd7, 0x0d, 0xd8, 0x0d, 0xd9, 0x0d, - 0xda, 0x0d, 0xdb, 0x0d, 0xdc, 0x0d, 0xdd, 0x0d, 0xde, 0x0d, 0xdf, 0x0d, - 0xe0, 0x0d, 0xe1, 0x0d, 0xe2, 0x0d, 0xe3, 0x0d, 0xe4, 0x0d, 0xe5, 0x0d, - 0xe6, 0x0d, 0xe7, 0x0d, 0xe8, 0x0d, 0xe9, 0x0d, 0xea, 0x0d, 0xeb, 0x0d, - 0xec, 0x0d, 0xed, 0x0d, 0xee, 0x0d, 0xef, 0x0d, 0xf0, 0x0d, 0xf1, 0x0d, - 0xf2, 0x0d, 0xf3, 0x0d, 0xf4, 0x0d, 0xf5, 0x0d, 0xf6, 0x0d, 0xf7, 0x0d, - 0xf8, 0x0d, 0xf9, 0x0d, 0xfa, 0x0d, 0xfb, 0x0d, 0xfc, 0x0d, 0xfd, 0x0d, - 0xfe, 0x0d, 0xff, 0x0d, 0x00, 0x0e, 0x01, 0x0e, 0x02, 0x0e, 0x03, 0x0e, - 0x04, 0x0e, 0x05, 0x0e, 0x06, 0x0e, 0x07, 0x0e, 0x08, 0x0e, 0x09, 0x0e, - 0x0a, 0x0e, 0x0b, 0x0e, 0x0c, 0x0e, 0x0d, 0x0e, 0x0e, 0x0e, 0x0f, 0x0e, - 0x10, 0x0e, 0x11, 0x0e, 0x12, 0x0e, 0x13, 0x0e, 0x14, 0x0e, 0x15, 0x0e, - 0x16, 0x0e, 0x17, 0x0e, 0x18, 0x0e, 0x19, 0x0e, 0x1a, 0x0e, 0x1b, 0x0e, - 0x1c, 0x0e, 0x1d, 0x0e, 0x1e, 0x0e, 0x1f, 0x0e, 0x20, 0x0e, 0x21, 0x0e, - 0x22, 0x0e, 0x23, 0x0e, 0x24, 0x0e, 0x25, 0x0e, 0x26, 0x0e, 0x27, 0x0e, - 0x28, 0x0e, 0x29, 0x0e, 0x2a, 0x0e, 0x2b, 0x0e, 0x2c, 0x0e, 0x2d, 0x0e, - 0x2e, 0x0e, 0x2f, 0x0e, 0x30, 0x0e, 0x31, 0x0e, 0x32, 0x0e, 0x33, 0x0e, - 0x34, 0x0e, 0x35, 0x0e, 0x36, 0x0e, 0x37, 0x0e, 0x38, 0x0e, 0x39, 0x0e, - 0x3a, 0x0e, 0x3b, 0x0e, 0x3c, 0x0e, 0x3d, 0x0e, 0x3e, 0x0e, 0x3f, 0x0e, - 0x40, 0x0e, 0x41, 0x0e, 0x42, 0x0e, 0x43, 0x0e, 0x44, 0x0e, 0x45, 0x0e, - 0x46, 0x0e, 0x47, 0x0e, 0x48, 0x0e, 0x49, 0x0e, 0x4a, 0x0e, 0x4b, 0x0e, - 0x4c, 0x0e, 0x4d, 0x0e, 0x4e, 0x0e, 0x4f, 0x0e, 0x50, 0x0e, 0x51, 0x0e, - 0x52, 0x0e, 0x53, 0x0e, 0x54, 0x0e, 0x55, 0x0e, 0x56, 0x0e, 0x57, 0x0e, - 0x58, 0x0e, 0x59, 0x0e, 0x5a, 0x0e, 0x5b, 0x0e, 0x5c, 0x0e, 0x5d, 0x0e, - 0x5e, 0x0e, 0x5f, 0x0e, 0x60, 0x0e, 0x61, 0x0e, 0x62, 0x0e, 0x63, 0x0e, - 0x64, 0x0e, 0x65, 0x0e, 0x66, 0x0e, 0x67, 0x0e, 0x68, 0x0e, 0x69, 0x0e, - 0x6a, 0x0e, 0x6b, 0x0e, 0x6c, 0x0e, 0x6d, 0x0e, 0x6e, 0x0e, 0x6f, 0x0e, - 0x70, 0x0e, 0x71, 0x0e, 0x72, 0x0e, 0x73, 0x0e, 0x74, 0x0e, 0x75, 0x0e, - 0x76, 0x0e, 0x77, 0x0e, 0x78, 0x0e, 0x79, 0x0e, 0x7a, 0x0e, 0x7b, 0x0e, - 0x7c, 0x0e, 0x7d, 0x0e, 0x7e, 0x0e, 0x7f, 0x0e, 0x80, 0x0e, 0x81, 0x0e, - 0x82, 0x0e, 0x83, 0x0e, 0x84, 0x0e, 0x85, 0x0e, 0x86, 0x0e, 0x87, 0x0e, - 0x88, 0x0e, 0x89, 0x0e, 0x8a, 0x0e, 0x8b, 0x0e, 0x8c, 0x0e, 0x8d, 0x0e, - 0x8e, 0x0e, 0x8f, 0x0e, 0x90, 0x0e, 0x91, 0x0e, 0x92, 0x0e, 0x93, 0x0e, - 0x94, 0x0e, 0x95, 0x0e, 0x96, 0x0e, 0x97, 0x0e, 0x98, 0x0e, 0x99, 0x0e, - 0x9a, 0x0e, 0x9b, 0x0e, 0x9c, 0x0e, 0x9d, 0x0e, 0x9e, 0x0e, 0x9f, 0x0e, - 0xa0, 0x0e, 0xa1, 0x0e, 0xa2, 0x0e, 0xa3, 0x0e, 0xa4, 0x0e, 0xa5, 0x0e, - 0xa6, 0x0e, 0xa7, 0x0e, 0xa8, 0x0e, 0xa9, 0x0e, 0xaa, 0x0e, 0xab, 0x0e, - 0xac, 0x0e, 0xad, 0x0e, 0xae, 0x0e, 0xaf, 0x0e, 0xb0, 0x0e, 0xb1, 0x0e, - 0xb2, 0x0e, 0xb3, 0x0e, 0xb4, 0x0e, 0xb5, 0x0e, 0xb6, 0x0e, 0xb7, 0x0e, - 0xb8, 0x0e, 0xb9, 0x0e, 0xba, 0x0e, 0xbb, 0x0e, 0xbc, 0x0e, 0xbd, 0x0e, - 0xbe, 0x0e, 0xbf, 0x0e, 0xc0, 0x0e, 0xc1, 0x0e, 0xc2, 0x0e, 0xc3, 0x0e, - 0xc4, 0x0e, 0xc5, 0x0e, 0xc6, 0x0e, 0xc7, 0x0e, 0xc8, 0x0e, 0xc9, 0x0e, - 0xca, 0x0e, 0xcb, 0x0e, 0xcc, 0x0e, 0xcd, 0x0e, 0xce, 0x0e, 0xcf, 0x0e, - 0xd0, 0x0e, 0xd1, 0x0e, 0xd2, 0x0e, 0xd3, 0x0e, 0xd4, 0x0e, 0xd5, 0x0e, - 0xd6, 0x0e, 0xd7, 0x0e, 0xd8, 0x0e, 0xd9, 0x0e, 0xda, 0x0e, 0xdb, 0x0e, - 0xdc, 0x0e, 0xdd, 0x0e, 0xde, 0x0e, 0xdf, 0x0e, 0xe0, 0x0e, 0xe1, 0x0e, - 0xe2, 0x0e, 0xe3, 0x0e, 0xe4, 0x0e, 0xe5, 0x0e, 0xe6, 0x0e, 0xe7, 0x0e, - 0xe8, 0x0e, 0xe9, 0x0e, 0xea, 0x0e, 0xeb, 0x0e, 0xec, 0x0e, 0xed, 0x0e, - 0xee, 0x0e, 0xef, 0x0e, 0xf0, 0x0e, 0xf1, 0x0e, 0xf2, 0x0e, 0xf3, 0x0e, - 0xf4, 0x0e, 0xf5, 0x0e, 0xf6, 0x0e, 0xf7, 0x0e, 0xf8, 0x0e, 0xf9, 0x0e, - 0xfa, 0x0e, 0xfb, 0x0e, 0xfc, 0x0e, 0xfd, 0x0e, 0xfe, 0x0e, 0xff, 0x0e, - 0x00, 0x0f, 0x01, 0x0f, 0x02, 0x0f, 0x03, 0x0f, 0x04, 0x0f, 0x05, 0x0f, - 0x06, 0x0f, 0x07, 0x0f, 0x08, 0x0f, 0x09, 0x0f, 0x0a, 0x0f, 0x0b, 0x0f, - 0x0c, 0x0f, 0x0d, 0x0f, 0x0e, 0x0f, 0x0f, 0x0f, 0x10, 0x0f, 0x11, 0x0f, - 0x12, 0x0f, 0x13, 0x0f, 0x14, 0x0f, 0x15, 0x0f, 0x16, 0x0f, 0x17, 0x0f, - 0x18, 0x0f, 0x19, 0x0f, 0x1a, 0x0f, 0x1b, 0x0f, 0x1c, 0x0f, 0x1d, 0x0f, - 0x1e, 0x0f, 0x1f, 0x0f, 0x20, 0x0f, 0x21, 0x0f, 0x22, 0x0f, 0x23, 0x0f, - 0x24, 0x0f, 0x25, 0x0f, 0x26, 0x0f, 0x27, 0x0f, 0x28, 0x0f, 0x29, 0x0f, - 0x2a, 0x0f, 0x2b, 0x0f, 0x2c, 0x0f, 0x2d, 0x0f, 0x2e, 0x0f, 0x2f, 0x0f, - 0x30, 0x0f, 0x31, 0x0f, 0x32, 0x0f, 0x33, 0x0f, 0x34, 0x0f, 0x35, 0x0f, - 0x36, 0x0f, 0x37, 0x0f, 0x38, 0x0f, 0x39, 0x0f, 0x3a, 0x0f, 0x3b, 0x0f, - 0x3c, 0x0f, 0x3d, 0x0f, 0x3e, 0x0f, 0x3f, 0x0f, 0x40, 0x0f, 0x41, 0x0f, - 0x42, 0x0f, 0x43, 0x0f, 0x44, 0x0f, 0x45, 0x0f, 0x46, 0x0f, 0x47, 0x0f, - 0x48, 0x0f, 0x49, 0x0f, 0x4a, 0x0f, 0x4b, 0x0f, 0x4c, 0x0f, 0x4d, 0x0f, - 0x4e, 0x0f, 0x4f, 0x0f, 0x50, 0x0f, 0x51, 0x0f, 0x52, 0x0f, 0x53, 0x0f, - 0x54, 0x0f, 0x55, 0x0f, 0x56, 0x0f, 0x57, 0x0f, 0x58, 0x0f, 0x59, 0x0f, - 0x5a, 0x0f, 0x5b, 0x0f, 0x5c, 0x0f, 0x5d, 0x0f, 0x5e, 0x0f, 0x5f, 0x0f, - 0x60, 0x0f, 0x61, 0x0f, 0x62, 0x0f, 0x63, 0x0f, 0x64, 0x0f, 0x65, 0x0f, - 0x66, 0x0f, 0x67, 0x0f, 0x68, 0x0f, 0x69, 0x0f, 0x6a, 0x0f, 0x6b, 0x0f, - 0x6c, 0x0f, 0x6d, 0x0f, 0x6e, 0x0f, 0x6f, 0x0f, 0x70, 0x0f, 0x71, 0x0f, - 0x72, 0x0f, 0x73, 0x0f, 0x74, 0x0f, 0x75, 0x0f, 0x76, 0x0f, 0x77, 0x0f, - 0x78, 0x0f, 0x79, 0x0f, 0x7a, 0x0f, 0x7b, 0x0f, 0x7c, 0x0f, 0x7d, 0x0f, - 0x7e, 0x0f, 0x7f, 0x0f, 0x80, 0x0f, 0x81, 0x0f, 0x82, 0x0f, 0x83, 0x0f, - 0x84, 0x0f, 0x85, 0x0f, 0x86, 0x0f, 0x87, 0x0f, 0x88, 0x0f, 0x89, 0x0f, - 0x8a, 0x0f, 0x8b, 0x0f, 0x8c, 0x0f, 0x8d, 0x0f, 0x8e, 0x0f, 0x8f, 0x0f, - 0x90, 0x0f, 0x91, 0x0f, 0x92, 0x0f, 0x93, 0x0f, 0x94, 0x0f, 0x95, 0x0f, - 0x96, 0x0f, 0x97, 0x0f, 0x98, 0x0f, 0x99, 0x0f, 0x9a, 0x0f, 0x9b, 0x0f, - 0x9c, 0x0f, 0x9d, 0x0f, 0x9e, 0x0f, 0x9f, 0x0f, 0xa0, 0x0f, 0xa1, 0x0f, - 0xa2, 0x0f, 0xa3, 0x0f, 0xa4, 0x0f, 0xa5, 0x0f, 0xa6, 0x0f, 0xa7, 0x0f, - 0xa8, 0x0f, 0xa9, 0x0f, 0xaa, 0x0f, 0xab, 0x0f, 0xac, 0x0f, 0xad, 0x0f, - 0xae, 0x0f, 0xaf, 0x0f, 0xb0, 0x0f, 0xb1, 0x0f, 0xb2, 0x0f, 0xb3, 0x0f, - 0xb4, 0x0f, 0xb5, 0x0f, 0xb6, 0x0f, 0xb7, 0x0f, 0xb8, 0x0f, 0xb9, 0x0f, - 0xba, 0x0f, 0xbb, 0x0f, 0xbc, 0x0f, 0xbd, 0x0f, 0xbe, 0x0f, 0xbf, 0x0f, - 0xc0, 0x0f, 0xc1, 0x0f, 0xc2, 0x0f, 0xc3, 0x0f, 0xc4, 0x0f, 0xc5, 0x0f, - 0xc6, 0x0f, 0xc7, 0x0f, 0xc8, 0x0f, 0xc9, 0x0f, 0xca, 0x0f, 0xcb, 0x0f, - 0xcc, 0x0f, 0xcd, 0x0f, 0xce, 0x0f, 0xcf, 0x0f, 0xd0, 0x0f, 0xd1, 0x0f, - 0xd2, 0x0f, 0xd3, 0x0f, 0xd4, 0x0f, 0xd5, 0x0f, 0xd6, 0x0f, 0xd7, 0x0f, - 0xd8, 0x0f, 0xd9, 0x0f, 0xda, 0x0f, 0xdb, 0x0f, 0xdc, 0x0f, 0xdd, 0x0f, - 0xde, 0x0f, 0xdf, 0x0f, 0xe0, 0x0f, 0xe1, 0x0f, 0xe2, 0x0f, 0xe3, 0x0f, - 0xe4, 0x0f, 0xe5, 0x0f, 0xe6, 0x0f, 0xe7, 0x0f, 0xe8, 0x0f, 0xe9, 0x0f, - 0xea, 0x0f, 0xeb, 0x0f, 0xec, 0x0f, 0xed, 0x0f, 0xee, 0x0f, 0xef, 0x0f, - 0xf0, 0x0f, 0xf1, 0x0f, 0xf2, 0x0f, 0xf3, 0x0f, 0xf4, 0x0f, 0xf5, 0x0f, - 0xf6, 0x0f, 0xf7, 0x0f, 0xf8, 0x0f, 0xf9, 0x0f, 0xfa, 0x0f, 0xfb, 0x0f, - 0xfc, 0x0f, 0xfd, 0x0f, 0xfe, 0x0f, 0xff, 0x0f -}; + 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x08, + 0x00, 0x09, 0x00, 0x0a, 0x00, 0x0b, 0x00, 0x0c, 0x00, 0x0d, 0x00, 0x0e, 0x00, 0x0f, 0x00, 0x10, 0x00, + 0x11, 0x00, 0x12, 0x00, 0x13, 0x00, 0x14, 0x00, 0x15, 0x00, 0x16, 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, + 0x00, 0x1a, 0x00, 0x1b, 0x00, 0x1c, 0x00, 0x1d, 0x00, 0x1e, 0x00, 0x1f, 0x00, 0x20, 0x00, 0x21, 0x00, + 0x22, 0x00, 0x23, 0x00, 0x24, 0x00, 0x25, 0x00, 0x26, 0x00, 0x27, 0x00, 0x28, 0x00, 0x29, 0x00, 0x2a, + 0x00, 0x2b, 0x00, 0x2c, 0x00, 0x2d, 0x00, 0x2e, 0x00, 0x2f, 0x00, 0x30, 0x00, 0x31, 0x00, 0x32, 0x00, + 0x33, 0x00, 0x34, 0x00, 0x35, 0x00, 0x36, 0x00, 0x37, 0x00, 0x38, 0x00, 0x39, 0x00, 0x3a, 0x00, 0x3b, + 0x00, 0x3c, 0x00, 0x3d, 0x00, 0x3e, 0x00, 0x3f, 0x00, 0x40, 0x00, 0x41, 0x00, 0x42, 0x00, 0x43, 0x00, + 0x44, 0x00, 0x45, 0x00, 0x46, 0x00, 0x47, 0x00, 0x48, 0x00, 0x49, 0x00, 0x4a, 0x00, 0x4b, 0x00, 0x4c, + 0x00, 0x4d, 0x00, 0x4e, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x51, 0x00, 0x52, 0x00, 0x53, 0x00, 0x54, 0x00, + 0x55, 0x00, 0x56, 0x00, 0x57, 0x00, 0x58, 0x00, 0x59, 0x00, 0x5a, 0x00, 0x5b, 0x00, 0x5c, 0x00, 0x5d, + 0x00, 0x5e, 0x00, 0x5f, 0x00, 0x60, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00, 0x64, 0x00, 0x65, 0x00, + 0x66, 0x00, 0x67, 0x00, 0x68, 0x00, 0x69, 0x00, 0x6a, 0x00, 0x6b, 0x00, 0x6c, 0x00, 0x6d, 0x00, 0x6e, + 0x00, 0x6f, 0x00, 0x70, 0x00, 0x71, 0x00, 0x72, 0x00, 0x73, 0x00, 0x74, 0x00, 0x75, 0x00, 0x76, 0x00, + 0x77, 0x00, 0x78, 0x00, 0x79, 0x00, 0x7a, 0x00, 0x7b, 0x00, 0x7c, 0x00, 0x7d, 0x00, 0x7e, 0x00, 0x7f, + 0x00, 0x80, 0x00, 0x81, 0x00, 0x82, 0x00, 0x83, 0x00, 0x84, 0x00, 0x85, 0x00, 0x86, 0x00, 0x87, 0x00, + 0x88, 0x00, 0x89, 0x00, 0x8a, 0x00, 0x8b, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8e, 0x00, 0x8f, 0x00, 0x90, + 0x00, 0x91, 0x00, 0x92, 0x00, 0x93, 0x00, 0x94, 0x00, 0x95, 0x00, 0x96, 0x00, 0x97, 0x00, 0x98, 0x00, + 0x99, 0x00, 0x9a, 0x00, 0x9b, 0x00, 0x9c, 0x00, 0x9d, 0x00, 0x9e, 0x00, 0x9f, 0x00, 0xa0, 0x00, 0xa1, + 0x00, 0xa2, 0x00, 0xa3, 0x00, 0xa4, 0x00, 0xa5, 0x00, 0xa6, 0x00, 0xa7, 0x00, 0xa8, 0x00, 0xa9, 0x00, + 0xaa, 0x00, 0xab, 0x00, 0xac, 0x00, 0xad, 0x00, 0xae, 0x00, 0xaf, 0x00, 0xb0, 0x00, 0xb1, 0x00, 0xb2, + 0x00, 0xb3, 0x00, 0xb4, 0x00, 0xb5, 0x00, 0xb6, 0x00, 0xb7, 0x00, 0xb8, 0x00, 0xb9, 0x00, 0xba, 0x00, + 0xbb, 0x00, 0xbc, 0x00, 0xbd, 0x00, 0xbe, 0x00, 0xbf, 0x00, 0xc0, 0x00, 0xc1, 0x00, 0xc2, 0x00, 0xc3, + 0x00, 0xc4, 0x00, 0xc5, 0x00, 0xc6, 0x00, 0xc7, 0x00, 0xc8, 0x00, 0xc9, 0x00, 0xca, 0x00, 0xcb, 0x00, + 0xcc, 0x00, 0xcd, 0x00, 0xce, 0x00, 0xcf, 0x00, 0xd0, 0x00, 0xd1, 0x00, 0xd2, 0x00, 0xd3, 0x00, 0xd4, + 0x00, 0xd5, 0x00, 0xd6, 0x00, 0xd7, 0x00, 0xd8, 0x00, 0xd9, 0x00, 0xda, 0x00, 0xdb, 0x00, 0xdc, 0x00, + 0xdd, 0x00, 0xde, 0x00, 0xdf, 0x00, 0xe0, 0x00, 0xe1, 0x00, 0xe2, 0x00, 0xe3, 0x00, 0xe4, 0x00, 0xe5, + 0x00, 0xe6, 0x00, 0xe7, 0x00, 0xe8, 0x00, 0xe9, 0x00, 0xea, 0x00, 0xeb, 0x00, 0xec, 0x00, 0xed, 0x00, + 0xee, 0x00, 0xef, 0x00, 0xf0, 0x00, 0xf1, 0x00, 0xf2, 0x00, 0xf3, 0x00, 0xf4, 0x00, 0xf5, 0x00, 0xf6, + 0x00, 0xf7, 0x00, 0xf8, 0x00, 0xf9, 0x00, 0xfa, 0x00, 0xfb, 0x00, 0xfc, 0x00, 0xfd, 0x00, 0xfe, 0x00, + 0xff, 0x00, 0x00, 0x01, 0x01, 0x01, 0x02, 0x01, 0x03, 0x01, 0x04, 0x01, 0x05, 0x01, 0x06, 0x01, 0x07, + 0x01, 0x08, 0x01, 0x09, 0x01, 0x0a, 0x01, 0x0b, 0x01, 0x0c, 0x01, 0x0d, 0x01, 0x0e, 0x01, 0x0f, 0x01, + 0x10, 0x01, 0x11, 0x01, 0x12, 0x01, 0x13, 0x01, 0x14, 0x01, 0x15, 0x01, 0x16, 0x01, 0x17, 0x01, 0x18, + 0x01, 0x19, 0x01, 0x1a, 0x01, 0x1b, 0x01, 0x1c, 0x01, 0x1d, 0x01, 0x1e, 0x01, 0x1f, 0x01, 0x20, 0x01, + 0x21, 0x01, 0x22, 0x01, 0x23, 0x01, 0x24, 0x01, 0x25, 0x01, 0x26, 0x01, 0x27, 0x01, 0x28, 0x01, 0x29, + 0x01, 0x2a, 0x01, 0x2b, 0x01, 0x2c, 0x01, 0x2d, 0x01, 0x2e, 0x01, 0x2f, 0x01, 0x30, 0x01, 0x31, 0x01, + 0x32, 0x01, 0x33, 0x01, 0x34, 0x01, 0x35, 0x01, 0x36, 0x01, 0x37, 0x01, 0x38, 0x01, 0x39, 0x01, 0x3a, + 0x01, 0x3b, 0x01, 0x3c, 0x01, 0x3d, 0x01, 0x3e, 0x01, 0x3f, 0x01, 0x40, 0x01, 0x41, 0x01, 0x42, 0x01, + 0x43, 0x01, 0x44, 0x01, 0x45, 0x01, 0x46, 0x01, 0x47, 0x01, 0x48, 0x01, 0x49, 0x01, 0x4a, 0x01, 0x4b, + 0x01, 0x4c, 0x01, 0x4d, 0x01, 0x4e, 0x01, 0x4f, 0x01, 0x50, 0x01, 0x51, 0x01, 0x52, 0x01, 0x53, 0x01, + 0x54, 0x01, 0x55, 0x01, 0x56, 0x01, 0x57, 0x01, 0x58, 0x01, 0x59, 0x01, 0x5a, 0x01, 0x5b, 0x01, 0x5c, + 0x01, 0x5d, 0x01, 0x5e, 0x01, 0x5f, 0x01, 0x60, 0x01, 0x61, 0x01, 0x62, 0x01, 0x63, 0x01, 0x64, 0x01, + 0x65, 0x01, 0x66, 0x01, 0x67, 0x01, 0x68, 0x01, 0x69, 0x01, 0x6a, 0x01, 0x6b, 0x01, 0x6c, 0x01, 0x6d, + 0x01, 0x6e, 0x01, 0x6f, 0x01, 0x70, 0x01, 0x71, 0x01, 0x72, 0x01, 0x73, 0x01, 0x74, 0x01, 0x75, 0x01, + 0x76, 0x01, 0x77, 0x01, 0x78, 0x01, 0x79, 0x01, 0x7a, 0x01, 0x7b, 0x01, 0x7c, 0x01, 0x7d, 0x01, 0x7e, + 0x01, 0x7f, 0x01, 0x80, 0x01, 0x81, 0x01, 0x82, 0x01, 0x83, 0x01, 0x84, 0x01, 0x85, 0x01, 0x86, 0x01, + 0x87, 0x01, 0x88, 0x01, 0x89, 0x01, 0x8a, 0x01, 0x8b, 0x01, 0x8c, 0x01, 0x8d, 0x01, 0x8e, 0x01, 0x8f, + 0x01, 0x90, 0x01, 0x91, 0x01, 0x92, 0x01, 0x93, 0x01, 0x94, 0x01, 0x95, 0x01, 0x96, 0x01, 0x97, 0x01, + 0x98, 0x01, 0x99, 0x01, 0x9a, 0x01, 0x9b, 0x01, 0x9c, 0x01, 0x9d, 0x01, 0x9e, 0x01, 0x9f, 0x01, 0xa0, + 0x01, 0xa1, 0x01, 0xa2, 0x01, 0xa3, 0x01, 0xa4, 0x01, 0xa5, 0x01, 0xa6, 0x01, 0xa7, 0x01, 0xa8, 0x01, + 0xa9, 0x01, 0xaa, 0x01, 0xab, 0x01, 0xac, 0x01, 0xad, 0x01, 0xae, 0x01, 0xaf, 0x01, 0xb0, 0x01, 0xb1, + 0x01, 0xb2, 0x01, 0xb3, 0x01, 0xb4, 0x01, 0xb5, 0x01, 0xb6, 0x01, 0xb7, 0x01, 0xb8, 0x01, 0xb9, 0x01, + 0xba, 0x01, 0xbb, 0x01, 0xbc, 0x01, 0xbd, 0x01, 0xbe, 0x01, 0xbf, 0x01, 0xc0, 0x01, 0xc1, 0x01, 0xc2, + 0x01, 0xc3, 0x01, 0xc4, 0x01, 0xc5, 0x01, 0xc6, 0x01, 0xc7, 0x01, 0xc8, 0x01, 0xc9, 0x01, 0xca, 0x01, + 0xcb, 0x01, 0xcc, 0x01, 0xcd, 0x01, 0xce, 0x01, 0xcf, 0x01, 0xd0, 0x01, 0xd1, 0x01, 0xd2, 0x01, 0xd3, + 0x01, 0xd4, 0x01, 0xd5, 0x01, 0xd6, 0x01, 0xd7, 0x01, 0xd8, 0x01, 0xd9, 0x01, 0xda, 0x01, 0xdb, 0x01, + 0xdc, 0x01, 0xdd, 0x01, 0xde, 0x01, 0xdf, 0x01, 0xe0, 0x01, 0xe1, 0x01, 0xe2, 0x01, 0xe3, 0x01, 0xe4, + 0x01, 0xe5, 0x01, 0xe6, 0x01, 0xe7, 0x01, 0xe8, 0x01, 0xe9, 0x01, 0xea, 0x01, 0xeb, 0x01, 0xec, 0x01, + 0xed, 0x01, 0xee, 0x01, 0xef, 0x01, 0xf0, 0x01, 0xf1, 0x01, 0xf2, 0x01, 0xf3, 0x01, 0xf4, 0x01, 0xf5, + 0x01, 0xf6, 0x01, 0xf7, 0x01, 0xf8, 0x01, 0xf9, 0x01, 0xfa, 0x01, 0xfb, 0x01, 0xfc, 0x01, 0xfd, 0x01, + 0xfe, 0x01, 0xff, 0x01, 0x00, 0x02, 0x01, 0x02, 0x02, 0x02, 0x03, 0x02, 0x04, 0x02, 0x05, 0x02, 0x06, + 0x02, 0x07, 0x02, 0x08, 0x02, 0x09, 0x02, 0x0a, 0x02, 0x0b, 0x02, 0x0c, 0x02, 0x0d, 0x02, 0x0e, 0x02, + 0x0f, 0x02, 0x10, 0x02, 0x11, 0x02, 0x12, 0x02, 0x13, 0x02, 0x14, 0x02, 0x15, 0x02, 0x16, 0x02, 0x17, + 0x02, 0x18, 0x02, 0x19, 0x02, 0x1a, 0x02, 0x1b, 0x02, 0x1c, 0x02, 0x1d, 0x02, 0x1e, 0x02, 0x1f, 0x02, + 0x20, 0x02, 0x21, 0x02, 0x22, 0x02, 0x23, 0x02, 0x24, 0x02, 0x25, 0x02, 0x26, 0x02, 0x27, 0x02, 0x28, + 0x02, 0x29, 0x02, 0x2a, 0x02, 0x2b, 0x02, 0x2c, 0x02, 0x2d, 0x02, 0x2e, 0x02, 0x2f, 0x02, 0x30, 0x02, + 0x31, 0x02, 0x32, 0x02, 0x33, 0x02, 0x34, 0x02, 0x35, 0x02, 0x36, 0x02, 0x37, 0x02, 0x38, 0x02, 0x39, + 0x02, 0x3a, 0x02, 0x3b, 0x02, 0x3c, 0x02, 0x3d, 0x02, 0x3e, 0x02, 0x3f, 0x02, 0x40, 0x02, 0x41, 0x02, + 0x42, 0x02, 0x43, 0x02, 0x44, 0x02, 0x45, 0x02, 0x46, 0x02, 0x47, 0x02, 0x48, 0x02, 0x49, 0x02, 0x4a, + 0x02, 0x4b, 0x02, 0x4c, 0x02, 0x4d, 0x02, 0x4e, 0x02, 0x4f, 0x02, 0x50, 0x02, 0x51, 0x02, 0x52, 0x02, + 0x53, 0x02, 0x54, 0x02, 0x55, 0x02, 0x56, 0x02, 0x57, 0x02, 0x58, 0x02, 0x59, 0x02, 0x5a, 0x02, 0x5b, + 0x02, 0x5c, 0x02, 0x5d, 0x02, 0x5e, 0x02, 0x5f, 0x02, 0x60, 0x02, 0x61, 0x02, 0x62, 0x02, 0x63, 0x02, + 0x64, 0x02, 0x65, 0x02, 0x66, 0x02, 0x67, 0x02, 0x68, 0x02, 0x69, 0x02, 0x6a, 0x02, 0x6b, 0x02, 0x6c, + 0x02, 0x6d, 0x02, 0x6e, 0x02, 0x6f, 0x02, 0x70, 0x02, 0x71, 0x02, 0x72, 0x02, 0x73, 0x02, 0x74, 0x02, + 0x75, 0x02, 0x76, 0x02, 0x77, 0x02, 0x78, 0x02, 0x79, 0x02, 0x7a, 0x02, 0x7b, 0x02, 0x7c, 0x02, 0x7d, + 0x02, 0x7e, 0x02, 0x7f, 0x02, 0x80, 0x02, 0x81, 0x02, 0x82, 0x02, 0x83, 0x02, 0x84, 0x02, 0x85, 0x02, + 0x86, 0x02, 0x87, 0x02, 0x88, 0x02, 0x89, 0x02, 0x8a, 0x02, 0x8b, 0x02, 0x8c, 0x02, 0x8d, 0x02, 0x8e, + 0x02, 0x8f, 0x02, 0x90, 0x02, 0x91, 0x02, 0x92, 0x02, 0x93, 0x02, 0x94, 0x02, 0x95, 0x02, 0x96, 0x02, + 0x97, 0x02, 0x98, 0x02, 0x99, 0x02, 0x9a, 0x02, 0x9b, 0x02, 0x9c, 0x02, 0x9d, 0x02, 0x9e, 0x02, 0x9f, + 0x02, 0xa0, 0x02, 0xa1, 0x02, 0xa2, 0x02, 0xa3, 0x02, 0xa4, 0x02, 0xa5, 0x02, 0xa6, 0x02, 0xa7, 0x02, + 0xa8, 0x02, 0xa9, 0x02, 0xaa, 0x02, 0xab, 0x02, 0xac, 0x02, 0xad, 0x02, 0xae, 0x02, 0xaf, 0x02, 0xb0, + 0x02, 0xb1, 0x02, 0xb2, 0x02, 0xb3, 0x02, 0xb4, 0x02, 0xb5, 0x02, 0xb6, 0x02, 0xb7, 0x02, 0xb8, 0x02, + 0xb9, 0x02, 0xba, 0x02, 0xbb, 0x02, 0xbc, 0x02, 0xbd, 0x02, 0xbe, 0x02, 0xbf, 0x02, 0xc0, 0x02, 0xc1, + 0x02, 0xc2, 0x02, 0xc3, 0x02, 0xc4, 0x02, 0xc5, 0x02, 0xc6, 0x02, 0xc7, 0x02, 0xc8, 0x02, 0xc9, 0x02, + 0xca, 0x02, 0xcb, 0x02, 0xcc, 0x02, 0xcd, 0x02, 0xce, 0x02, 0xcf, 0x02, 0xd0, 0x02, 0xd1, 0x02, 0xd2, + 0x02, 0xd3, 0x02, 0xd4, 0x02, 0xd5, 0x02, 0xd6, 0x02, 0xd7, 0x02, 0xd8, 0x02, 0xd9, 0x02, 0xda, 0x02, + 0xdb, 0x02, 0xdc, 0x02, 0xdd, 0x02, 0xde, 0x02, 0xdf, 0x02, 0xe0, 0x02, 0xe1, 0x02, 0xe2, 0x02, 0xe3, + 0x02, 0xe4, 0x02, 0xe5, 0x02, 0xe6, 0x02, 0xe7, 0x02, 0xe8, 0x02, 0xe9, 0x02, 0xea, 0x02, 0xeb, 0x02, + 0xec, 0x02, 0xed, 0x02, 0xee, 0x02, 0xef, 0x02, 0xf0, 0x02, 0xf1, 0x02, 0xf2, 0x02, 0xf3, 0x02, 0xf4, + 0x02, 0xf5, 0x02, 0xf6, 0x02, 0xf7, 0x02, 0xf8, 0x02, 0xf9, 0x02, 0xfa, 0x02, 0xfb, 0x02, 0xfc, 0x02, + 0xfd, 0x02, 0xfe, 0x02, 0xff, 0x02, 0x00, 0x03, 0x01, 0x03, 0x02, 0x03, 0x03, 0x03, 0x04, 0x03, 0x05, + 0x03, 0x06, 0x03, 0x07, 0x03, 0x08, 0x03, 0x09, 0x03, 0x0a, 0x03, 0x0b, 0x03, 0x0c, 0x03, 0x0d, 0x03, + 0x0e, 0x03, 0x0f, 0x03, 0x10, 0x03, 0x11, 0x03, 0x12, 0x03, 0x13, 0x03, 0x14, 0x03, 0x15, 0x03, 0x16, + 0x03, 0x17, 0x03, 0x18, 0x03, 0x19, 0x03, 0x1a, 0x03, 0x1b, 0x03, 0x1c, 0x03, 0x1d, 0x03, 0x1e, 0x03, + 0x1f, 0x03, 0x20, 0x03, 0x21, 0x03, 0x22, 0x03, 0x23, 0x03, 0x24, 0x03, 0x25, 0x03, 0x26, 0x03, 0x27, + 0x03, 0x28, 0x03, 0x29, 0x03, 0x2a, 0x03, 0x2b, 0x03, 0x2c, 0x03, 0x2d, 0x03, 0x2e, 0x03, 0x2f, 0x03, + 0x30, 0x03, 0x31, 0x03, 0x32, 0x03, 0x33, 0x03, 0x34, 0x03, 0x35, 0x03, 0x36, 0x03, 0x37, 0x03, 0x38, + 0x03, 0x39, 0x03, 0x3a, 0x03, 0x3b, 0x03, 0x3c, 0x03, 0x3d, 0x03, 0x3e, 0x03, 0x3f, 0x03, 0x40, 0x03, + 0x41, 0x03, 0x42, 0x03, 0x43, 0x03, 0x44, 0x03, 0x45, 0x03, 0x46, 0x03, 0x47, 0x03, 0x48, 0x03, 0x49, + 0x03, 0x4a, 0x03, 0x4b, 0x03, 0x4c, 0x03, 0x4d, 0x03, 0x4e, 0x03, 0x4f, 0x03, 0x50, 0x03, 0x51, 0x03, + 0x52, 0x03, 0x53, 0x03, 0x54, 0x03, 0x55, 0x03, 0x56, 0x03, 0x57, 0x03, 0x58, 0x03, 0x59, 0x03, 0x5a, + 0x03, 0x5b, 0x03, 0x5c, 0x03, 0x5d, 0x03, 0x5e, 0x03, 0x5f, 0x03, 0x60, 0x03, 0x61, 0x03, 0x62, 0x03, + 0x63, 0x03, 0x64, 0x03, 0x65, 0x03, 0x66, 0x03, 0x67, 0x03, 0x68, 0x03, 0x69, 0x03, 0x6a, 0x03, 0x6b, + 0x03, 0x6c, 0x03, 0x6d, 0x03, 0x6e, 0x03, 0x6f, 0x03, 0x70, 0x03, 0x71, 0x03, 0x72, 0x03, 0x73, 0x03, + 0x74, 0x03, 0x75, 0x03, 0x76, 0x03, 0x77, 0x03, 0x78, 0x03, 0x79, 0x03, 0x7a, 0x03, 0x7b, 0x03, 0x7c, + 0x03, 0x7d, 0x03, 0x7e, 0x03, 0x7f, 0x03, 0x80, 0x03, 0x81, 0x03, 0x82, 0x03, 0x83, 0x03, 0x84, 0x03, + 0x85, 0x03, 0x86, 0x03, 0x87, 0x03, 0x88, 0x03, 0x89, 0x03, 0x8a, 0x03, 0x8b, 0x03, 0x8c, 0x03, 0x8d, + 0x03, 0x8e, 0x03, 0x8f, 0x03, 0x90, 0x03, 0x91, 0x03, 0x92, 0x03, 0x93, 0x03, 0x94, 0x03, 0x95, 0x03, + 0x96, 0x03, 0x97, 0x03, 0x98, 0x03, 0x99, 0x03, 0x9a, 0x03, 0x9b, 0x03, 0x9c, 0x03, 0x9d, 0x03, 0x9e, + 0x03, 0x9f, 0x03, 0xa0, 0x03, 0xa1, 0x03, 0xa2, 0x03, 0xa3, 0x03, 0xa4, 0x03, 0xa5, 0x03, 0xa6, 0x03, + 0xa7, 0x03, 0xa8, 0x03, 0xa9, 0x03, 0xaa, 0x03, 0xab, 0x03, 0xac, 0x03, 0xad, 0x03, 0xae, 0x03, 0xaf, + 0x03, 0xb0, 0x03, 0xb1, 0x03, 0xb2, 0x03, 0xb3, 0x03, 0xb4, 0x03, 0xb5, 0x03, 0xb6, 0x03, 0xb7, 0x03, + 0xb8, 0x03, 0xb9, 0x03, 0xba, 0x03, 0xbb, 0x03, 0xbc, 0x03, 0xbd, 0x03, 0xbe, 0x03, 0xbf, 0x03, 0xc0, + 0x03, 0xc1, 0x03, 0xc2, 0x03, 0xc3, 0x03, 0xc4, 0x03, 0xc5, 0x03, 0xc6, 0x03, 0xc7, 0x03, 0xc8, 0x03, + 0xc9, 0x03, 0xca, 0x03, 0xcb, 0x03, 0xcc, 0x03, 0xcd, 0x03, 0xce, 0x03, 0xcf, 0x03, 0xd0, 0x03, 0xd1, + 0x03, 0xd2, 0x03, 0xd3, 0x03, 0xd4, 0x03, 0xd5, 0x03, 0xd6, 0x03, 0xd7, 0x03, 0xd8, 0x03, 0xd9, 0x03, + 0xda, 0x03, 0xdb, 0x03, 0xdc, 0x03, 0xdd, 0x03, 0xde, 0x03, 0xdf, 0x03, 0xe0, 0x03, 0xe1, 0x03, 0xe2, + 0x03, 0xe3, 0x03, 0xe4, 0x03, 0xe5, 0x03, 0xe6, 0x03, 0xe7, 0x03, 0xe8, 0x03, 0xe9, 0x03, 0xea, 0x03, + 0xeb, 0x03, 0xec, 0x03, 0xed, 0x03, 0xee, 0x03, 0xef, 0x03, 0xf0, 0x03, 0xf1, 0x03, 0xf2, 0x03, 0xf3, + 0x03, 0xf4, 0x03, 0xf5, 0x03, 0xf6, 0x03, 0xf7, 0x03, 0xf8, 0x03, 0xf9, 0x03, 0xfa, 0x03, 0xfb, 0x03, + 0xfc, 0x03, 0xfd, 0x03, 0xfe, 0x03, 0xff, 0x03, 0x00, 0x04, 0x01, 0x04, 0x02, 0x04, 0x03, 0x04, 0x04, + 0x04, 0x05, 0x04, 0x06, 0x04, 0x07, 0x04, 0x08, 0x04, 0x09, 0x04, 0x0a, 0x04, 0x0b, 0x04, 0x0c, 0x04, + 0x0d, 0x04, 0x0e, 0x04, 0x0f, 0x04, 0x10, 0x04, 0x11, 0x04, 0x12, 0x04, 0x13, 0x04, 0x14, 0x04, 0x15, + 0x04, 0x16, 0x04, 0x17, 0x04, 0x18, 0x04, 0x19, 0x04, 0x1a, 0x04, 0x1b, 0x04, 0x1c, 0x04, 0x1d, 0x04, + 0x1e, 0x04, 0x1f, 0x04, 0x20, 0x04, 0x21, 0x04, 0x22, 0x04, 0x23, 0x04, 0x24, 0x04, 0x25, 0x04, 0x26, + 0x04, 0x27, 0x04, 0x28, 0x04, 0x29, 0x04, 0x2a, 0x04, 0x2b, 0x04, 0x2c, 0x04, 0x2d, 0x04, 0x2e, 0x04, + 0x2f, 0x04, 0x30, 0x04, 0x31, 0x04, 0x32, 0x04, 0x33, 0x04, 0x34, 0x04, 0x35, 0x04, 0x36, 0x04, 0x37, + 0x04, 0x38, 0x04, 0x39, 0x04, 0x3a, 0x04, 0x3b, 0x04, 0x3c, 0x04, 0x3d, 0x04, 0x3e, 0x04, 0x3f, 0x04, + 0x40, 0x04, 0x41, 0x04, 0x42, 0x04, 0x43, 0x04, 0x44, 0x04, 0x45, 0x04, 0x46, 0x04, 0x47, 0x04, 0x48, + 0x04, 0x49, 0x04, 0x4a, 0x04, 0x4b, 0x04, 0x4c, 0x04, 0x4d, 0x04, 0x4e, 0x04, 0x4f, 0x04, 0x50, 0x04, + 0x51, 0x04, 0x52, 0x04, 0x53, 0x04, 0x54, 0x04, 0x55, 0x04, 0x56, 0x04, 0x57, 0x04, 0x58, 0x04, 0x59, + 0x04, 0x5a, 0x04, 0x5b, 0x04, 0x5c, 0x04, 0x5d, 0x04, 0x5e, 0x04, 0x5f, 0x04, 0x60, 0x04, 0x61, 0x04, + 0x62, 0x04, 0x63, 0x04, 0x64, 0x04, 0x65, 0x04, 0x66, 0x04, 0x67, 0x04, 0x68, 0x04, 0x69, 0x04, 0x6a, + 0x04, 0x6b, 0x04, 0x6c, 0x04, 0x6d, 0x04, 0x6e, 0x04, 0x6f, 0x04, 0x70, 0x04, 0x71, 0x04, 0x72, 0x04, + 0x73, 0x04, 0x74, 0x04, 0x75, 0x04, 0x76, 0x04, 0x77, 0x04, 0x78, 0x04, 0x79, 0x04, 0x7a, 0x04, 0x7b, + 0x04, 0x7c, 0x04, 0x7d, 0x04, 0x7e, 0x04, 0x7f, 0x04, 0x80, 0x04, 0x81, 0x04, 0x82, 0x04, 0x83, 0x04, + 0x84, 0x04, 0x85, 0x04, 0x86, 0x04, 0x87, 0x04, 0x88, 0x04, 0x89, 0x04, 0x8a, 0x04, 0x8b, 0x04, 0x8c, + 0x04, 0x8d, 0x04, 0x8e, 0x04, 0x8f, 0x04, 0x90, 0x04, 0x91, 0x04, 0x92, 0x04, 0x93, 0x04, 0x94, 0x04, + 0x95, 0x04, 0x96, 0x04, 0x97, 0x04, 0x98, 0x04, 0x99, 0x04, 0x9a, 0x04, 0x9b, 0x04, 0x9c, 0x04, 0x9d, + 0x04, 0x9e, 0x04, 0x9f, 0x04, 0xa0, 0x04, 0xa1, 0x04, 0xa2, 0x04, 0xa3, 0x04, 0xa4, 0x04, 0xa5, 0x04, + 0xa6, 0x04, 0xa7, 0x04, 0xa8, 0x04, 0xa9, 0x04, 0xaa, 0x04, 0xab, 0x04, 0xac, 0x04, 0xad, 0x04, 0xae, + 0x04, 0xaf, 0x04, 0xb0, 0x04, 0xb1, 0x04, 0xb2, 0x04, 0xb3, 0x04, 0xb4, 0x04, 0xb5, 0x04, 0xb6, 0x04, + 0xb7, 0x04, 0xb8, 0x04, 0xb9, 0x04, 0xba, 0x04, 0xbb, 0x04, 0xbc, 0x04, 0xbd, 0x04, 0xbe, 0x04, 0xbf, + 0x04, 0xc0, 0x04, 0xc1, 0x04, 0xc2, 0x04, 0xc3, 0x04, 0xc4, 0x04, 0xc5, 0x04, 0xc6, 0x04, 0xc7, 0x04, + 0xc8, 0x04, 0xc9, 0x04, 0xca, 0x04, 0xcb, 0x04, 0xcc, 0x04, 0xcd, 0x04, 0xce, 0x04, 0xcf, 0x04, 0xd0, + 0x04, 0xd1, 0x04, 0xd2, 0x04, 0xd3, 0x04, 0xd4, 0x04, 0xd5, 0x04, 0xd6, 0x04, 0xd7, 0x04, 0xd8, 0x04, + 0xd9, 0x04, 0xda, 0x04, 0xdb, 0x04, 0xdc, 0x04, 0xdd, 0x04, 0xde, 0x04, 0xdf, 0x04, 0xe0, 0x04, 0xe1, + 0x04, 0xe2, 0x04, 0xe3, 0x04, 0xe4, 0x04, 0xe5, 0x04, 0xe6, 0x04, 0xe7, 0x04, 0xe8, 0x04, 0xe9, 0x04, + 0xea, 0x04, 0xeb, 0x04, 0xec, 0x04, 0xed, 0x04, 0xee, 0x04, 0xef, 0x04, 0xf0, 0x04, 0xf1, 0x04, 0xf2, + 0x04, 0xf3, 0x04, 0xf4, 0x04, 0xf5, 0x04, 0xf6, 0x04, 0xf7, 0x04, 0xf8, 0x04, 0xf9, 0x04, 0xfa, 0x04, + 0xfb, 0x04, 0xfc, 0x04, 0xfd, 0x04, 0xfe, 0x04, 0xff, 0x04, 0x00, 0x05, 0x01, 0x05, 0x02, 0x05, 0x03, + 0x05, 0x04, 0x05, 0x05, 0x05, 0x06, 0x05, 0x07, 0x05, 0x08, 0x05, 0x09, 0x05, 0x0a, 0x05, 0x0b, 0x05, + 0x0c, 0x05, 0x0d, 0x05, 0x0e, 0x05, 0x0f, 0x05, 0x10, 0x05, 0x11, 0x05, 0x12, 0x05, 0x13, 0x05, 0x14, + 0x05, 0x15, 0x05, 0x16, 0x05, 0x17, 0x05, 0x18, 0x05, 0x19, 0x05, 0x1a, 0x05, 0x1b, 0x05, 0x1c, 0x05, + 0x1d, 0x05, 0x1e, 0x05, 0x1f, 0x05, 0x20, 0x05, 0x21, 0x05, 0x22, 0x05, 0x23, 0x05, 0x24, 0x05, 0x25, + 0x05, 0x26, 0x05, 0x27, 0x05, 0x28, 0x05, 0x29, 0x05, 0x2a, 0x05, 0x2b, 0x05, 0x2c, 0x05, 0x2d, 0x05, + 0x2e, 0x05, 0x2f, 0x05, 0x30, 0x05, 0x31, 0x05, 0x32, 0x05, 0x33, 0x05, 0x34, 0x05, 0x35, 0x05, 0x36, + 0x05, 0x37, 0x05, 0x38, 0x05, 0x39, 0x05, 0x3a, 0x05, 0x3b, 0x05, 0x3c, 0x05, 0x3d, 0x05, 0x3e, 0x05, + 0x3f, 0x05, 0x40, 0x05, 0x41, 0x05, 0x42, 0x05, 0x43, 0x05, 0x44, 0x05, 0x45, 0x05, 0x46, 0x05, 0x47, + 0x05, 0x48, 0x05, 0x49, 0x05, 0x4a, 0x05, 0x4b, 0x05, 0x4c, 0x05, 0x4d, 0x05, 0x4e, 0x05, 0x4f, 0x05, + 0x50, 0x05, 0x51, 0x05, 0x52, 0x05, 0x53, 0x05, 0x54, 0x05, 0x55, 0x05, 0x56, 0x05, 0x57, 0x05, 0x58, + 0x05, 0x59, 0x05, 0x5a, 0x05, 0x5b, 0x05, 0x5c, 0x05, 0x5d, 0x05, 0x5e, 0x05, 0x5f, 0x05, 0x60, 0x05, + 0x61, 0x05, 0x62, 0x05, 0x63, 0x05, 0x64, 0x05, 0x65, 0x05, 0x66, 0x05, 0x67, 0x05, 0x68, 0x05, 0x69, + 0x05, 0x6a, 0x05, 0x6b, 0x05, 0x6c, 0x05, 0x6d, 0x05, 0x6e, 0x05, 0x6f, 0x05, 0x70, 0x05, 0x71, 0x05, + 0x72, 0x05, 0x73, 0x05, 0x74, 0x05, 0x75, 0x05, 0x76, 0x05, 0x77, 0x05, 0x78, 0x05, 0x79, 0x05, 0x7a, + 0x05, 0x7b, 0x05, 0x7c, 0x05, 0x7d, 0x05, 0x7e, 0x05, 0x7f, 0x05, 0x80, 0x05, 0x81, 0x05, 0x82, 0x05, + 0x83, 0x05, 0x84, 0x05, 0x85, 0x05, 0x86, 0x05, 0x87, 0x05, 0x88, 0x05, 0x89, 0x05, 0x8a, 0x05, 0x8b, + 0x05, 0x8c, 0x05, 0x8d, 0x05, 0x8e, 0x05, 0x8f, 0x05, 0x90, 0x05, 0x91, 0x05, 0x92, 0x05, 0x93, 0x05, + 0x94, 0x05, 0x95, 0x05, 0x96, 0x05, 0x97, 0x05, 0x98, 0x05, 0x99, 0x05, 0x9a, 0x05, 0x9b, 0x05, 0x9c, + 0x05, 0x9d, 0x05, 0x9e, 0x05, 0x9f, 0x05, 0xa0, 0x05, 0xa1, 0x05, 0xa2, 0x05, 0xa3, 0x05, 0xa4, 0x05, + 0xa5, 0x05, 0xa6, 0x05, 0xa7, 0x05, 0xa8, 0x05, 0xa9, 0x05, 0xaa, 0x05, 0xab, 0x05, 0xac, 0x05, 0xad, + 0x05, 0xae, 0x05, 0xaf, 0x05, 0xb0, 0x05, 0xb1, 0x05, 0xb2, 0x05, 0xb3, 0x05, 0xb4, 0x05, 0xb5, 0x05, + 0xb6, 0x05, 0xb7, 0x05, 0xb8, 0x05, 0xb9, 0x05, 0xba, 0x05, 0xbb, 0x05, 0xbc, 0x05, 0xbd, 0x05, 0xbe, + 0x05, 0xbf, 0x05, 0xc0, 0x05, 0xc1, 0x05, 0xc2, 0x05, 0xc3, 0x05, 0xc4, 0x05, 0xc5, 0x05, 0xc6, 0x05, + 0xc7, 0x05, 0xc8, 0x05, 0xc9, 0x05, 0xca, 0x05, 0xcb, 0x05, 0xcc, 0x05, 0xcd, 0x05, 0xce, 0x05, 0xcf, + 0x05, 0xd0, 0x05, 0xd1, 0x05, 0xd2, 0x05, 0xd3, 0x05, 0xd4, 0x05, 0xd5, 0x05, 0xd6, 0x05, 0xd7, 0x05, + 0xd8, 0x05, 0xd9, 0x05, 0xda, 0x05, 0xdb, 0x05, 0xdc, 0x05, 0xdd, 0x05, 0xde, 0x05, 0xdf, 0x05, 0xe0, + 0x05, 0xe1, 0x05, 0xe2, 0x05, 0xe3, 0x05, 0xe4, 0x05, 0xe5, 0x05, 0xe6, 0x05, 0xe7, 0x05, 0xe8, 0x05, + 0xe9, 0x05, 0xea, 0x05, 0xeb, 0x05, 0xec, 0x05, 0xed, 0x05, 0xee, 0x05, 0xef, 0x05, 0xf0, 0x05, 0xf1, + 0x05, 0xf2, 0x05, 0xf3, 0x05, 0xf4, 0x05, 0xf5, 0x05, 0xf6, 0x05, 0xf7, 0x05, 0xf8, 0x05, 0xf9, 0x05, + 0xfa, 0x05, 0xfb, 0x05, 0xfc, 0x05, 0xfd, 0x05, 0xfe, 0x05, 0xff, 0x05, 0x00, 0x06, 0x01, 0x06, 0x02, + 0x06, 0x03, 0x06, 0x04, 0x06, 0x05, 0x06, 0x06, 0x06, 0x07, 0x06, 0x08, 0x06, 0x09, 0x06, 0x0a, 0x06, + 0x0b, 0x06, 0x0c, 0x06, 0x0d, 0x06, 0x0e, 0x06, 0x0f, 0x06, 0x10, 0x06, 0x11, 0x06, 0x12, 0x06, 0x13, + 0x06, 0x14, 0x06, 0x15, 0x06, 0x16, 0x06, 0x17, 0x06, 0x18, 0x06, 0x19, 0x06, 0x1a, 0x06, 0x1b, 0x06, + 0x1c, 0x06, 0x1d, 0x06, 0x1e, 0x06, 0x1f, 0x06, 0x20, 0x06, 0x21, 0x06, 0x22, 0x06, 0x23, 0x06, 0x24, + 0x06, 0x25, 0x06, 0x26, 0x06, 0x27, 0x06, 0x28, 0x06, 0x29, 0x06, 0x2a, 0x06, 0x2b, 0x06, 0x2c, 0x06, + 0x2d, 0x06, 0x2e, 0x06, 0x2f, 0x06, 0x30, 0x06, 0x31, 0x06, 0x32, 0x06, 0x33, 0x06, 0x34, 0x06, 0x35, + 0x06, 0x36, 0x06, 0x37, 0x06, 0x38, 0x06, 0x39, 0x06, 0x3a, 0x06, 0x3b, 0x06, 0x3c, 0x06, 0x3d, 0x06, + 0x3e, 0x06, 0x3f, 0x06, 0x40, 0x06, 0x41, 0x06, 0x42, 0x06, 0x43, 0x06, 0x44, 0x06, 0x45, 0x06, 0x46, + 0x06, 0x47, 0x06, 0x48, 0x06, 0x49, 0x06, 0x4a, 0x06, 0x4b, 0x06, 0x4c, 0x06, 0x4d, 0x06, 0x4e, 0x06, + 0x4f, 0x06, 0x50, 0x06, 0x51, 0x06, 0x52, 0x06, 0x53, 0x06, 0x54, 0x06, 0x55, 0x06, 0x56, 0x06, 0x57, + 0x06, 0x58, 0x06, 0x59, 0x06, 0x5a, 0x06, 0x5b, 0x06, 0x5c, 0x06, 0x5d, 0x06, 0x5e, 0x06, 0x5f, 0x06, + 0x60, 0x06, 0x61, 0x06, 0x62, 0x06, 0x63, 0x06, 0x64, 0x06, 0x65, 0x06, 0x66, 0x06, 0x67, 0x06, 0x68, + 0x06, 0x69, 0x06, 0x6a, 0x06, 0x6b, 0x06, 0x6c, 0x06, 0x6d, 0x06, 0x6e, 0x06, 0x6f, 0x06, 0x70, 0x06, + 0x71, 0x06, 0x72, 0x06, 0x73, 0x06, 0x74, 0x06, 0x75, 0x06, 0x76, 0x06, 0x77, 0x06, 0x78, 0x06, 0x79, + 0x06, 0x7a, 0x06, 0x7b, 0x06, 0x7c, 0x06, 0x7d, 0x06, 0x7e, 0x06, 0x7f, 0x06, 0x80, 0x06, 0x81, 0x06, + 0x82, 0x06, 0x83, 0x06, 0x84, 0x06, 0x85, 0x06, 0x86, 0x06, 0x87, 0x06, 0x88, 0x06, 0x89, 0x06, 0x8a, + 0x06, 0x8b, 0x06, 0x8c, 0x06, 0x8d, 0x06, 0x8e, 0x06, 0x8f, 0x06, 0x90, 0x06, 0x91, 0x06, 0x92, 0x06, + 0x93, 0x06, 0x94, 0x06, 0x95, 0x06, 0x96, 0x06, 0x97, 0x06, 0x98, 0x06, 0x99, 0x06, 0x9a, 0x06, 0x9b, + 0x06, 0x9c, 0x06, 0x9d, 0x06, 0x9e, 0x06, 0x9f, 0x06, 0xa0, 0x06, 0xa1, 0x06, 0xa2, 0x06, 0xa3, 0x06, + 0xa4, 0x06, 0xa5, 0x06, 0xa6, 0x06, 0xa7, 0x06, 0xa8, 0x06, 0xa9, 0x06, 0xaa, 0x06, 0xab, 0x06, 0xac, + 0x06, 0xad, 0x06, 0xae, 0x06, 0xaf, 0x06, 0xb0, 0x06, 0xb1, 0x06, 0xb2, 0x06, 0xb3, 0x06, 0xb4, 0x06, + 0xb5, 0x06, 0xb6, 0x06, 0xb7, 0x06, 0xb8, 0x06, 0xb9, 0x06, 0xba, 0x06, 0xbb, 0x06, 0xbc, 0x06, 0xbd, + 0x06, 0xbe, 0x06, 0xbf, 0x06, 0xc0, 0x06, 0xc1, 0x06, 0xc2, 0x06, 0xc3, 0x06, 0xc4, 0x06, 0xc5, 0x06, + 0xc6, 0x06, 0xc7, 0x06, 0xc8, 0x06, 0xc9, 0x06, 0xca, 0x06, 0xcb, 0x06, 0xcc, 0x06, 0xcd, 0x06, 0xce, + 0x06, 0xcf, 0x06, 0xd0, 0x06, 0xd1, 0x06, 0xd2, 0x06, 0xd3, 0x06, 0xd4, 0x06, 0xd5, 0x06, 0xd6, 0x06, + 0xd7, 0x06, 0xd8, 0x06, 0xd9, 0x06, 0xda, 0x06, 0xdb, 0x06, 0xdc, 0x06, 0xdd, 0x06, 0xde, 0x06, 0xdf, + 0x06, 0xe0, 0x06, 0xe1, 0x06, 0xe2, 0x06, 0xe3, 0x06, 0xe4, 0x06, 0xe5, 0x06, 0xe6, 0x06, 0xe7, 0x06, + 0xe8, 0x06, 0xe9, 0x06, 0xea, 0x06, 0xeb, 0x06, 0xec, 0x06, 0xed, 0x06, 0xee, 0x06, 0xef, 0x06, 0xf0, + 0x06, 0xf1, 0x06, 0xf2, 0x06, 0xf3, 0x06, 0xf4, 0x06, 0xf5, 0x06, 0xf6, 0x06, 0xf7, 0x06, 0xf8, 0x06, + 0xf9, 0x06, 0xfa, 0x06, 0xfb, 0x06, 0xfc, 0x06, 0xfd, 0x06, 0xfe, 0x06, 0xff, 0x06, 0x00, 0x07, 0x01, + 0x07, 0x02, 0x07, 0x03, 0x07, 0x04, 0x07, 0x05, 0x07, 0x06, 0x07, 0x07, 0x07, 0x08, 0x07, 0x09, 0x07, + 0x0a, 0x07, 0x0b, 0x07, 0x0c, 0x07, 0x0d, 0x07, 0x0e, 0x07, 0x0f, 0x07, 0x10, 0x07, 0x11, 0x07, 0x12, + 0x07, 0x13, 0x07, 0x14, 0x07, 0x15, 0x07, 0x16, 0x07, 0x17, 0x07, 0x18, 0x07, 0x19, 0x07, 0x1a, 0x07, + 0x1b, 0x07, 0x1c, 0x07, 0x1d, 0x07, 0x1e, 0x07, 0x1f, 0x07, 0x20, 0x07, 0x21, 0x07, 0x22, 0x07, 0x23, + 0x07, 0x24, 0x07, 0x25, 0x07, 0x26, 0x07, 0x27, 0x07, 0x28, 0x07, 0x29, 0x07, 0x2a, 0x07, 0x2b, 0x07, + 0x2c, 0x07, 0x2d, 0x07, 0x2e, 0x07, 0x2f, 0x07, 0x30, 0x07, 0x31, 0x07, 0x32, 0x07, 0x33, 0x07, 0x34, + 0x07, 0x35, 0x07, 0x36, 0x07, 0x37, 0x07, 0x38, 0x07, 0x39, 0x07, 0x3a, 0x07, 0x3b, 0x07, 0x3c, 0x07, + 0x3d, 0x07, 0x3e, 0x07, 0x3f, 0x07, 0x40, 0x07, 0x41, 0x07, 0x42, 0x07, 0x43, 0x07, 0x44, 0x07, 0x45, + 0x07, 0x46, 0x07, 0x47, 0x07, 0x48, 0x07, 0x49, 0x07, 0x4a, 0x07, 0x4b, 0x07, 0x4c, 0x07, 0x4d, 0x07, + 0x4e, 0x07, 0x4f, 0x07, 0x50, 0x07, 0x51, 0x07, 0x52, 0x07, 0x53, 0x07, 0x54, 0x07, 0x55, 0x07, 0x56, + 0x07, 0x57, 0x07, 0x58, 0x07, 0x59, 0x07, 0x5a, 0x07, 0x5b, 0x07, 0x5c, 0x07, 0x5d, 0x07, 0x5e, 0x07, + 0x5f, 0x07, 0x60, 0x07, 0x61, 0x07, 0x62, 0x07, 0x63, 0x07, 0x64, 0x07, 0x65, 0x07, 0x66, 0x07, 0x67, + 0x07, 0x68, 0x07, 0x69, 0x07, 0x6a, 0x07, 0x6b, 0x07, 0x6c, 0x07, 0x6d, 0x07, 0x6e, 0x07, 0x6f, 0x07, + 0x70, 0x07, 0x71, 0x07, 0x72, 0x07, 0x73, 0x07, 0x74, 0x07, 0x75, 0x07, 0x76, 0x07, 0x77, 0x07, 0x78, + 0x07, 0x79, 0x07, 0x7a, 0x07, 0x7b, 0x07, 0x7c, 0x07, 0x7d, 0x07, 0x7e, 0x07, 0x7f, 0x07, 0x80, 0x07, + 0x81, 0x07, 0x82, 0x07, 0x83, 0x07, 0x84, 0x07, 0x85, 0x07, 0x86, 0x07, 0x87, 0x07, 0x88, 0x07, 0x89, + 0x07, 0x8a, 0x07, 0x8b, 0x07, 0x8c, 0x07, 0x8d, 0x07, 0x8e, 0x07, 0x8f, 0x07, 0x90, 0x07, 0x91, 0x07, + 0x92, 0x07, 0x93, 0x07, 0x94, 0x07, 0x95, 0x07, 0x96, 0x07, 0x97, 0x07, 0x98, 0x07, 0x99, 0x07, 0x9a, + 0x07, 0x9b, 0x07, 0x9c, 0x07, 0x9d, 0x07, 0x9e, 0x07, 0x9f, 0x07, 0xa0, 0x07, 0xa1, 0x07, 0xa2, 0x07, + 0xa3, 0x07, 0xa4, 0x07, 0xa5, 0x07, 0xa6, 0x07, 0xa7, 0x07, 0xa8, 0x07, 0xa9, 0x07, 0xaa, 0x07, 0xab, + 0x07, 0xac, 0x07, 0xad, 0x07, 0xae, 0x07, 0xaf, 0x07, 0xb0, 0x07, 0xb1, 0x07, 0xb2, 0x07, 0xb3, 0x07, + 0xb4, 0x07, 0xb5, 0x07, 0xb6, 0x07, 0xb7, 0x07, 0xb8, 0x07, 0xb9, 0x07, 0xba, 0x07, 0xbb, 0x07, 0xbc, + 0x07, 0xbd, 0x07, 0xbe, 0x07, 0xbf, 0x07, 0xc0, 0x07, 0xc1, 0x07, 0xc2, 0x07, 0xc3, 0x07, 0xc4, 0x07, + 0xc5, 0x07, 0xc6, 0x07, 0xc7, 0x07, 0xc8, 0x07, 0xc9, 0x07, 0xca, 0x07, 0xcb, 0x07, 0xcc, 0x07, 0xcd, + 0x07, 0xce, 0x07, 0xcf, 0x07, 0xd0, 0x07, 0xd1, 0x07, 0xd2, 0x07, 0xd3, 0x07, 0xd4, 0x07, 0xd5, 0x07, + 0xd6, 0x07, 0xd7, 0x07, 0xd8, 0x07, 0xd9, 0x07, 0xda, 0x07, 0xdb, 0x07, 0xdc, 0x07, 0xdd, 0x07, 0xde, + 0x07, 0xdf, 0x07, 0xe0, 0x07, 0xe1, 0x07, 0xe2, 0x07, 0xe3, 0x07, 0xe4, 0x07, 0xe5, 0x07, 0xe6, 0x07, + 0xe7, 0x07, 0xe8, 0x07, 0xe9, 0x07, 0xea, 0x07, 0xeb, 0x07, 0xec, 0x07, 0xed, 0x07, 0xee, 0x07, 0xef, + 0x07, 0xf0, 0x07, 0xf1, 0x07, 0xf2, 0x07, 0xf3, 0x07, 0xf4, 0x07, 0xf5, 0x07, 0xf6, 0x07, 0xf7, 0x07, + 0xf8, 0x07, 0xf9, 0x07, 0xfa, 0x07, 0xfb, 0x07, 0xfc, 0x07, 0xfd, 0x07, 0xfe, 0x07, 0xff, 0x07, 0x00, + 0x08, 0x01, 0x08, 0x02, 0x08, 0x03, 0x08, 0x04, 0x08, 0x05, 0x08, 0x06, 0x08, 0x07, 0x08, 0x08, 0x08, + 0x09, 0x08, 0x0a, 0x08, 0x0b, 0x08, 0x0c, 0x08, 0x0d, 0x08, 0x0e, 0x08, 0x0f, 0x08, 0x10, 0x08, 0x11, + 0x08, 0x12, 0x08, 0x13, 0x08, 0x14, 0x08, 0x15, 0x08, 0x16, 0x08, 0x17, 0x08, 0x18, 0x08, 0x19, 0x08, + 0x1a, 0x08, 0x1b, 0x08, 0x1c, 0x08, 0x1d, 0x08, 0x1e, 0x08, 0x1f, 0x08, 0x20, 0x08, 0x21, 0x08, 0x22, + 0x08, 0x23, 0x08, 0x24, 0x08, 0x25, 0x08, 0x26, 0x08, 0x27, 0x08, 0x28, 0x08, 0x29, 0x08, 0x2a, 0x08, + 0x2b, 0x08, 0x2c, 0x08, 0x2d, 0x08, 0x2e, 0x08, 0x2f, 0x08, 0x30, 0x08, 0x31, 0x08, 0x32, 0x08, 0x33, + 0x08, 0x34, 0x08, 0x35, 0x08, 0x36, 0x08, 0x37, 0x08, 0x38, 0x08, 0x39, 0x08, 0x3a, 0x08, 0x3b, 0x08, + 0x3c, 0x08, 0x3d, 0x08, 0x3e, 0x08, 0x3f, 0x08, 0x40, 0x08, 0x41, 0x08, 0x42, 0x08, 0x43, 0x08, 0x44, + 0x08, 0x45, 0x08, 0x46, 0x08, 0x47, 0x08, 0x48, 0x08, 0x49, 0x08, 0x4a, 0x08, 0x4b, 0x08, 0x4c, 0x08, + 0x4d, 0x08, 0x4e, 0x08, 0x4f, 0x08, 0x50, 0x08, 0x51, 0x08, 0x52, 0x08, 0x53, 0x08, 0x54, 0x08, 0x55, + 0x08, 0x56, 0x08, 0x57, 0x08, 0x58, 0x08, 0x59, 0x08, 0x5a, 0x08, 0x5b, 0x08, 0x5c, 0x08, 0x5d, 0x08, + 0x5e, 0x08, 0x5f, 0x08, 0x60, 0x08, 0x61, 0x08, 0x62, 0x08, 0x63, 0x08, 0x64, 0x08, 0x65, 0x08, 0x66, + 0x08, 0x67, 0x08, 0x68, 0x08, 0x69, 0x08, 0x6a, 0x08, 0x6b, 0x08, 0x6c, 0x08, 0x6d, 0x08, 0x6e, 0x08, + 0x6f, 0x08, 0x70, 0x08, 0x71, 0x08, 0x72, 0x08, 0x73, 0x08, 0x74, 0x08, 0x75, 0x08, 0x76, 0x08, 0x77, + 0x08, 0x78, 0x08, 0x79, 0x08, 0x7a, 0x08, 0x7b, 0x08, 0x7c, 0x08, 0x7d, 0x08, 0x7e, 0x08, 0x7f, 0x08, + 0x80, 0x08, 0x81, 0x08, 0x82, 0x08, 0x83, 0x08, 0x84, 0x08, 0x85, 0x08, 0x86, 0x08, 0x87, 0x08, 0x88, + 0x08, 0x89, 0x08, 0x8a, 0x08, 0x8b, 0x08, 0x8c, 0x08, 0x8d, 0x08, 0x8e, 0x08, 0x8f, 0x08, 0x90, 0x08, + 0x91, 0x08, 0x92, 0x08, 0x93, 0x08, 0x94, 0x08, 0x95, 0x08, 0x96, 0x08, 0x97, 0x08, 0x98, 0x08, 0x99, + 0x08, 0x9a, 0x08, 0x9b, 0x08, 0x9c, 0x08, 0x9d, 0x08, 0x9e, 0x08, 0x9f, 0x08, 0xa0, 0x08, 0xa1, 0x08, + 0xa2, 0x08, 0xa3, 0x08, 0xa4, 0x08, 0xa5, 0x08, 0xa6, 0x08, 0xa7, 0x08, 0xa8, 0x08, 0xa9, 0x08, 0xaa, + 0x08, 0xab, 0x08, 0xac, 0x08, 0xad, 0x08, 0xae, 0x08, 0xaf, 0x08, 0xb0, 0x08, 0xb1, 0x08, 0xb2, 0x08, + 0xb3, 0x08, 0xb4, 0x08, 0xb5, 0x08, 0xb6, 0x08, 0xb7, 0x08, 0xb8, 0x08, 0xb9, 0x08, 0xba, 0x08, 0xbb, + 0x08, 0xbc, 0x08, 0xbd, 0x08, 0xbe, 0x08, 0xbf, 0x08, 0xc0, 0x08, 0xc1, 0x08, 0xc2, 0x08, 0xc3, 0x08, + 0xc4, 0x08, 0xc5, 0x08, 0xc6, 0x08, 0xc7, 0x08, 0xc8, 0x08, 0xc9, 0x08, 0xca, 0x08, 0xcb, 0x08, 0xcc, + 0x08, 0xcd, 0x08, 0xce, 0x08, 0xcf, 0x08, 0xd0, 0x08, 0xd1, 0x08, 0xd2, 0x08, 0xd3, 0x08, 0xd4, 0x08, + 0xd5, 0x08, 0xd6, 0x08, 0xd7, 0x08, 0xd8, 0x08, 0xd9, 0x08, 0xda, 0x08, 0xdb, 0x08, 0xdc, 0x08, 0xdd, + 0x08, 0xde, 0x08, 0xdf, 0x08, 0xe0, 0x08, 0xe1, 0x08, 0xe2, 0x08, 0xe3, 0x08, 0xe4, 0x08, 0xe5, 0x08, + 0xe6, 0x08, 0xe7, 0x08, 0xe8, 0x08, 0xe9, 0x08, 0xea, 0x08, 0xeb, 0x08, 0xec, 0x08, 0xed, 0x08, 0xee, + 0x08, 0xef, 0x08, 0xf0, 0x08, 0xf1, 0x08, 0xf2, 0x08, 0xf3, 0x08, 0xf4, 0x08, 0xf5, 0x08, 0xf6, 0x08, + 0xf7, 0x08, 0xf8, 0x08, 0xf9, 0x08, 0xfa, 0x08, 0xfb, 0x08, 0xfc, 0x08, 0xfd, 0x08, 0xfe, 0x08, 0xff, + 0x08, 0x00, 0x09, 0x01, 0x09, 0x02, 0x09, 0x03, 0x09, 0x04, 0x09, 0x05, 0x09, 0x06, 0x09, 0x07, 0x09, + 0x08, 0x09, 0x09, 0x09, 0x0a, 0x09, 0x0b, 0x09, 0x0c, 0x09, 0x0d, 0x09, 0x0e, 0x09, 0x0f, 0x09, 0x10, + 0x09, 0x11, 0x09, 0x12, 0x09, 0x13, 0x09, 0x14, 0x09, 0x15, 0x09, 0x16, 0x09, 0x17, 0x09, 0x18, 0x09, + 0x19, 0x09, 0x1a, 0x09, 0x1b, 0x09, 0x1c, 0x09, 0x1d, 0x09, 0x1e, 0x09, 0x1f, 0x09, 0x20, 0x09, 0x21, + 0x09, 0x22, 0x09, 0x23, 0x09, 0x24, 0x09, 0x25, 0x09, 0x26, 0x09, 0x27, 0x09, 0x28, 0x09, 0x29, 0x09, + 0x2a, 0x09, 0x2b, 0x09, 0x2c, 0x09, 0x2d, 0x09, 0x2e, 0x09, 0x2f, 0x09, 0x30, 0x09, 0x31, 0x09, 0x32, + 0x09, 0x33, 0x09, 0x34, 0x09, 0x35, 0x09, 0x36, 0x09, 0x37, 0x09, 0x38, 0x09, 0x39, 0x09, 0x3a, 0x09, + 0x3b, 0x09, 0x3c, 0x09, 0x3d, 0x09, 0x3e, 0x09, 0x3f, 0x09, 0x40, 0x09, 0x41, 0x09, 0x42, 0x09, 0x43, + 0x09, 0x44, 0x09, 0x45, 0x09, 0x46, 0x09, 0x47, 0x09, 0x48, 0x09, 0x49, 0x09, 0x4a, 0x09, 0x4b, 0x09, + 0x4c, 0x09, 0x4d, 0x09, 0x4e, 0x09, 0x4f, 0x09, 0x50, 0x09, 0x51, 0x09, 0x52, 0x09, 0x53, 0x09, 0x54, + 0x09, 0x55, 0x09, 0x56, 0x09, 0x57, 0x09, 0x58, 0x09, 0x59, 0x09, 0x5a, 0x09, 0x5b, 0x09, 0x5c, 0x09, + 0x5d, 0x09, 0x5e, 0x09, 0x5f, 0x09, 0x60, 0x09, 0x61, 0x09, 0x62, 0x09, 0x63, 0x09, 0x64, 0x09, 0x65, + 0x09, 0x66, 0x09, 0x67, 0x09, 0x68, 0x09, 0x69, 0x09, 0x6a, 0x09, 0x6b, 0x09, 0x6c, 0x09, 0x6d, 0x09, + 0x6e, 0x09, 0x6f, 0x09, 0x70, 0x09, 0x71, 0x09, 0x72, 0x09, 0x73, 0x09, 0x74, 0x09, 0x75, 0x09, 0x76, + 0x09, 0x77, 0x09, 0x78, 0x09, 0x79, 0x09, 0x7a, 0x09, 0x7b, 0x09, 0x7c, 0x09, 0x7d, 0x09, 0x7e, 0x09, + 0x7f, 0x09, 0x80, 0x09, 0x81, 0x09, 0x82, 0x09, 0x83, 0x09, 0x84, 0x09, 0x85, 0x09, 0x86, 0x09, 0x87, + 0x09, 0x88, 0x09, 0x89, 0x09, 0x8a, 0x09, 0x8b, 0x09, 0x8c, 0x09, 0x8d, 0x09, 0x8e, 0x09, 0x8f, 0x09, + 0x90, 0x09, 0x91, 0x09, 0x92, 0x09, 0x93, 0x09, 0x94, 0x09, 0x95, 0x09, 0x96, 0x09, 0x97, 0x09, 0x98, + 0x09, 0x99, 0x09, 0x9a, 0x09, 0x9b, 0x09, 0x9c, 0x09, 0x9d, 0x09, 0x9e, 0x09, 0x9f, 0x09, 0xa0, 0x09, + 0xa1, 0x09, 0xa2, 0x09, 0xa3, 0x09, 0xa4, 0x09, 0xa5, 0x09, 0xa6, 0x09, 0xa7, 0x09, 0xa8, 0x09, 0xa9, + 0x09, 0xaa, 0x09, 0xab, 0x09, 0xac, 0x09, 0xad, 0x09, 0xae, 0x09, 0xaf, 0x09, 0xb0, 0x09, 0xb1, 0x09, + 0xb2, 0x09, 0xb3, 0x09, 0xb4, 0x09, 0xb5, 0x09, 0xb6, 0x09, 0xb7, 0x09, 0xb8, 0x09, 0xb9, 0x09, 0xba, + 0x09, 0xbb, 0x09, 0xbc, 0x09, 0xbd, 0x09, 0xbe, 0x09, 0xbf, 0x09, 0xc0, 0x09, 0xc1, 0x09, 0xc2, 0x09, + 0xc3, 0x09, 0xc4, 0x09, 0xc5, 0x09, 0xc6, 0x09, 0xc7, 0x09, 0xc8, 0x09, 0xc9, 0x09, 0xca, 0x09, 0xcb, + 0x09, 0xcc, 0x09, 0xcd, 0x09, 0xce, 0x09, 0xcf, 0x09, 0xd0, 0x09, 0xd1, 0x09, 0xd2, 0x09, 0xd3, 0x09, + 0xd4, 0x09, 0xd5, 0x09, 0xd6, 0x09, 0xd7, 0x09, 0xd8, 0x09, 0xd9, 0x09, 0xda, 0x09, 0xdb, 0x09, 0xdc, + 0x09, 0xdd, 0x09, 0xde, 0x09, 0xdf, 0x09, 0xe0, 0x09, 0xe1, 0x09, 0xe2, 0x09, 0xe3, 0x09, 0xe4, 0x09, + 0xe5, 0x09, 0xe6, 0x09, 0xe7, 0x09, 0xe8, 0x09, 0xe9, 0x09, 0xea, 0x09, 0xeb, 0x09, 0xec, 0x09, 0xed, + 0x09, 0xee, 0x09, 0xef, 0x09, 0xf0, 0x09, 0xf1, 0x09, 0xf2, 0x09, 0xf3, 0x09, 0xf4, 0x09, 0xf5, 0x09, + 0xf6, 0x09, 0xf7, 0x09, 0xf8, 0x09, 0xf9, 0x09, 0xfa, 0x09, 0xfb, 0x09, 0xfc, 0x09, 0xfd, 0x09, 0xfe, + 0x09, 0xff, 0x09, 0x00, 0x0a, 0x01, 0x0a, 0x02, 0x0a, 0x03, 0x0a, 0x04, 0x0a, 0x05, 0x0a, 0x06, 0x0a, + 0x07, 0x0a, 0x08, 0x0a, 0x09, 0x0a, 0x0a, 0x0a, 0x0b, 0x0a, 0x0c, 0x0a, 0x0d, 0x0a, 0x0e, 0x0a, 0x0f, + 0x0a, 0x10, 0x0a, 0x11, 0x0a, 0x12, 0x0a, 0x13, 0x0a, 0x14, 0x0a, 0x15, 0x0a, 0x16, 0x0a, 0x17, 0x0a, + 0x18, 0x0a, 0x19, 0x0a, 0x1a, 0x0a, 0x1b, 0x0a, 0x1c, 0x0a, 0x1d, 0x0a, 0x1e, 0x0a, 0x1f, 0x0a, 0x20, + 0x0a, 0x21, 0x0a, 0x22, 0x0a, 0x23, 0x0a, 0x24, 0x0a, 0x25, 0x0a, 0x26, 0x0a, 0x27, 0x0a, 0x28, 0x0a, + 0x29, 0x0a, 0x2a, 0x0a, 0x2b, 0x0a, 0x2c, 0x0a, 0x2d, 0x0a, 0x2e, 0x0a, 0x2f, 0x0a, 0x30, 0x0a, 0x31, + 0x0a, 0x32, 0x0a, 0x33, 0x0a, 0x34, 0x0a, 0x35, 0x0a, 0x36, 0x0a, 0x37, 0x0a, 0x38, 0x0a, 0x39, 0x0a, + 0x3a, 0x0a, 0x3b, 0x0a, 0x3c, 0x0a, 0x3d, 0x0a, 0x3e, 0x0a, 0x3f, 0x0a, 0x40, 0x0a, 0x41, 0x0a, 0x42, + 0x0a, 0x43, 0x0a, 0x44, 0x0a, 0x45, 0x0a, 0x46, 0x0a, 0x47, 0x0a, 0x48, 0x0a, 0x49, 0x0a, 0x4a, 0x0a, + 0x4b, 0x0a, 0x4c, 0x0a, 0x4d, 0x0a, 0x4e, 0x0a, 0x4f, 0x0a, 0x50, 0x0a, 0x51, 0x0a, 0x52, 0x0a, 0x53, + 0x0a, 0x54, 0x0a, 0x55, 0x0a, 0x56, 0x0a, 0x57, 0x0a, 0x58, 0x0a, 0x59, 0x0a, 0x5a, 0x0a, 0x5b, 0x0a, + 0x5c, 0x0a, 0x5d, 0x0a, 0x5e, 0x0a, 0x5f, 0x0a, 0x60, 0x0a, 0x61, 0x0a, 0x62, 0x0a, 0x63, 0x0a, 0x64, + 0x0a, 0x65, 0x0a, 0x66, 0x0a, 0x67, 0x0a, 0x68, 0x0a, 0x69, 0x0a, 0x6a, 0x0a, 0x6b, 0x0a, 0x6c, 0x0a, + 0x6d, 0x0a, 0x6e, 0x0a, 0x6f, 0x0a, 0x70, 0x0a, 0x71, 0x0a, 0x72, 0x0a, 0x73, 0x0a, 0x74, 0x0a, 0x75, + 0x0a, 0x76, 0x0a, 0x77, 0x0a, 0x78, 0x0a, 0x79, 0x0a, 0x7a, 0x0a, 0x7b, 0x0a, 0x7c, 0x0a, 0x7d, 0x0a, + 0x7e, 0x0a, 0x7f, 0x0a, 0x80, 0x0a, 0x81, 0x0a, 0x82, 0x0a, 0x83, 0x0a, 0x84, 0x0a, 0x85, 0x0a, 0x86, + 0x0a, 0x87, 0x0a, 0x88, 0x0a, 0x89, 0x0a, 0x8a, 0x0a, 0x8b, 0x0a, 0x8c, 0x0a, 0x8d, 0x0a, 0x8e, 0x0a, + 0x8f, 0x0a, 0x90, 0x0a, 0x91, 0x0a, 0x92, 0x0a, 0x93, 0x0a, 0x94, 0x0a, 0x95, 0x0a, 0x96, 0x0a, 0x97, + 0x0a, 0x98, 0x0a, 0x99, 0x0a, 0x9a, 0x0a, 0x9b, 0x0a, 0x9c, 0x0a, 0x9d, 0x0a, 0x9e, 0x0a, 0x9f, 0x0a, + 0xa0, 0x0a, 0xa1, 0x0a, 0xa2, 0x0a, 0xa3, 0x0a, 0xa4, 0x0a, 0xa5, 0x0a, 0xa6, 0x0a, 0xa7, 0x0a, 0xa8, + 0x0a, 0xa9, 0x0a, 0xaa, 0x0a, 0xab, 0x0a, 0xac, 0x0a, 0xad, 0x0a, 0xae, 0x0a, 0xaf, 0x0a, 0xb0, 0x0a, + 0xb1, 0x0a, 0xb2, 0x0a, 0xb3, 0x0a, 0xb4, 0x0a, 0xb5, 0x0a, 0xb6, 0x0a, 0xb7, 0x0a, 0xb8, 0x0a, 0xb9, + 0x0a, 0xba, 0x0a, 0xbb, 0x0a, 0xbc, 0x0a, 0xbd, 0x0a, 0xbe, 0x0a, 0xbf, 0x0a, 0xc0, 0x0a, 0xc1, 0x0a, + 0xc2, 0x0a, 0xc3, 0x0a, 0xc4, 0x0a, 0xc5, 0x0a, 0xc6, 0x0a, 0xc7, 0x0a, 0xc8, 0x0a, 0xc9, 0x0a, 0xca, + 0x0a, 0xcb, 0x0a, 0xcc, 0x0a, 0xcd, 0x0a, 0xce, 0x0a, 0xcf, 0x0a, 0xd0, 0x0a, 0xd1, 0x0a, 0xd2, 0x0a, + 0xd3, 0x0a, 0xd4, 0x0a, 0xd5, 0x0a, 0xd6, 0x0a, 0xd7, 0x0a, 0xd8, 0x0a, 0xd9, 0x0a, 0xda, 0x0a, 0xdb, + 0x0a, 0xdc, 0x0a, 0xdd, 0x0a, 0xde, 0x0a, 0xdf, 0x0a, 0xe0, 0x0a, 0xe1, 0x0a, 0xe2, 0x0a, 0xe3, 0x0a, + 0xe4, 0x0a, 0xe5, 0x0a, 0xe6, 0x0a, 0xe7, 0x0a, 0xe8, 0x0a, 0xe9, 0x0a, 0xea, 0x0a, 0xeb, 0x0a, 0xec, + 0x0a, 0xed, 0x0a, 0xee, 0x0a, 0xef, 0x0a, 0xf0, 0x0a, 0xf1, 0x0a, 0xf2, 0x0a, 0xf3, 0x0a, 0xf4, 0x0a, + 0xf5, 0x0a, 0xf6, 0x0a, 0xf7, 0x0a, 0xf8, 0x0a, 0xf9, 0x0a, 0xfa, 0x0a, 0xfb, 0x0a, 0xfc, 0x0a, 0xfd, + 0x0a, 0xfe, 0x0a, 0xff, 0x0a, 0x00, 0x0b, 0x01, 0x0b, 0x02, 0x0b, 0x03, 0x0b, 0x04, 0x0b, 0x05, 0x0b, + 0x06, 0x0b, 0x07, 0x0b, 0x08, 0x0b, 0x09, 0x0b, 0x0a, 0x0b, 0x0b, 0x0b, 0x0c, 0x0b, 0x0d, 0x0b, 0x0e, + 0x0b, 0x0f, 0x0b, 0x10, 0x0b, 0x11, 0x0b, 0x12, 0x0b, 0x13, 0x0b, 0x14, 0x0b, 0x15, 0x0b, 0x16, 0x0b, + 0x17, 0x0b, 0x18, 0x0b, 0x19, 0x0b, 0x1a, 0x0b, 0x1b, 0x0b, 0x1c, 0x0b, 0x1d, 0x0b, 0x1e, 0x0b, 0x1f, + 0x0b, 0x20, 0x0b, 0x21, 0x0b, 0x22, 0x0b, 0x23, 0x0b, 0x24, 0x0b, 0x25, 0x0b, 0x26, 0x0b, 0x27, 0x0b, + 0x28, 0x0b, 0x29, 0x0b, 0x2a, 0x0b, 0x2b, 0x0b, 0x2c, 0x0b, 0x2d, 0x0b, 0x2e, 0x0b, 0x2f, 0x0b, 0x30, + 0x0b, 0x31, 0x0b, 0x32, 0x0b, 0x33, 0x0b, 0x34, 0x0b, 0x35, 0x0b, 0x36, 0x0b, 0x37, 0x0b, 0x38, 0x0b, + 0x39, 0x0b, 0x3a, 0x0b, 0x3b, 0x0b, 0x3c, 0x0b, 0x3d, 0x0b, 0x3e, 0x0b, 0x3f, 0x0b, 0x40, 0x0b, 0x41, + 0x0b, 0x42, 0x0b, 0x43, 0x0b, 0x44, 0x0b, 0x45, 0x0b, 0x46, 0x0b, 0x47, 0x0b, 0x48, 0x0b, 0x49, 0x0b, + 0x4a, 0x0b, 0x4b, 0x0b, 0x4c, 0x0b, 0x4d, 0x0b, 0x4e, 0x0b, 0x4f, 0x0b, 0x50, 0x0b, 0x51, 0x0b, 0x52, + 0x0b, 0x53, 0x0b, 0x54, 0x0b, 0x55, 0x0b, 0x56, 0x0b, 0x57, 0x0b, 0x58, 0x0b, 0x59, 0x0b, 0x5a, 0x0b, + 0x5b, 0x0b, 0x5c, 0x0b, 0x5d, 0x0b, 0x5e, 0x0b, 0x5f, 0x0b, 0x60, 0x0b, 0x61, 0x0b, 0x62, 0x0b, 0x63, + 0x0b, 0x64, 0x0b, 0x65, 0x0b, 0x66, 0x0b, 0x67, 0x0b, 0x68, 0x0b, 0x69, 0x0b, 0x6a, 0x0b, 0x6b, 0x0b, + 0x6c, 0x0b, 0x6d, 0x0b, 0x6e, 0x0b, 0x6f, 0x0b, 0x70, 0x0b, 0x71, 0x0b, 0x72, 0x0b, 0x73, 0x0b, 0x74, + 0x0b, 0x75, 0x0b, 0x76, 0x0b, 0x77, 0x0b, 0x78, 0x0b, 0x79, 0x0b, 0x7a, 0x0b, 0x7b, 0x0b, 0x7c, 0x0b, + 0x7d, 0x0b, 0x7e, 0x0b, 0x7f, 0x0b, 0x80, 0x0b, 0x81, 0x0b, 0x82, 0x0b, 0x83, 0x0b, 0x84, 0x0b, 0x85, + 0x0b, 0x86, 0x0b, 0x87, 0x0b, 0x88, 0x0b, 0x89, 0x0b, 0x8a, 0x0b, 0x8b, 0x0b, 0x8c, 0x0b, 0x8d, 0x0b, + 0x8e, 0x0b, 0x8f, 0x0b, 0x90, 0x0b, 0x91, 0x0b, 0x92, 0x0b, 0x93, 0x0b, 0x94, 0x0b, 0x95, 0x0b, 0x96, + 0x0b, 0x97, 0x0b, 0x98, 0x0b, 0x99, 0x0b, 0x9a, 0x0b, 0x9b, 0x0b, 0x9c, 0x0b, 0x9d, 0x0b, 0x9e, 0x0b, + 0x9f, 0x0b, 0xa0, 0x0b, 0xa1, 0x0b, 0xa2, 0x0b, 0xa3, 0x0b, 0xa4, 0x0b, 0xa5, 0x0b, 0xa6, 0x0b, 0xa7, + 0x0b, 0xa8, 0x0b, 0xa9, 0x0b, 0xaa, 0x0b, 0xab, 0x0b, 0xac, 0x0b, 0xad, 0x0b, 0xae, 0x0b, 0xaf, 0x0b, + 0xb0, 0x0b, 0xb1, 0x0b, 0xb2, 0x0b, 0xb3, 0x0b, 0xb4, 0x0b, 0xb5, 0x0b, 0xb6, 0x0b, 0xb7, 0x0b, 0xb8, + 0x0b, 0xb9, 0x0b, 0xba, 0x0b, 0xbb, 0x0b, 0xbc, 0x0b, 0xbd, 0x0b, 0xbe, 0x0b, 0xbf, 0x0b, 0xc0, 0x0b, + 0xc1, 0x0b, 0xc2, 0x0b, 0xc3, 0x0b, 0xc4, 0x0b, 0xc5, 0x0b, 0xc6, 0x0b, 0xc7, 0x0b, 0xc8, 0x0b, 0xc9, + 0x0b, 0xca, 0x0b, 0xcb, 0x0b, 0xcc, 0x0b, 0xcd, 0x0b, 0xce, 0x0b, 0xcf, 0x0b, 0xd0, 0x0b, 0xd1, 0x0b, + 0xd2, 0x0b, 0xd3, 0x0b, 0xd4, 0x0b, 0xd5, 0x0b, 0xd6, 0x0b, 0xd7, 0x0b, 0xd8, 0x0b, 0xd9, 0x0b, 0xda, + 0x0b, 0xdb, 0x0b, 0xdc, 0x0b, 0xdd, 0x0b, 0xde, 0x0b, 0xdf, 0x0b, 0xe0, 0x0b, 0xe1, 0x0b, 0xe2, 0x0b, + 0xe3, 0x0b, 0xe4, 0x0b, 0xe5, 0x0b, 0xe6, 0x0b, 0xe7, 0x0b, 0xe8, 0x0b, 0xe9, 0x0b, 0xea, 0x0b, 0xeb, + 0x0b, 0xec, 0x0b, 0xed, 0x0b, 0xee, 0x0b, 0xef, 0x0b, 0xf0, 0x0b, 0xf1, 0x0b, 0xf2, 0x0b, 0xf3, 0x0b, + 0xf4, 0x0b, 0xf5, 0x0b, 0xf6, 0x0b, 0xf7, 0x0b, 0xf8, 0x0b, 0xf9, 0x0b, 0xfa, 0x0b, 0xfb, 0x0b, 0xfc, + 0x0b, 0xfd, 0x0b, 0xfe, 0x0b, 0xff, 0x0b, 0x00, 0x0c, 0x01, 0x0c, 0x02, 0x0c, 0x03, 0x0c, 0x04, 0x0c, + 0x05, 0x0c, 0x06, 0x0c, 0x07, 0x0c, 0x08, 0x0c, 0x09, 0x0c, 0x0a, 0x0c, 0x0b, 0x0c, 0x0c, 0x0c, 0x0d, + 0x0c, 0x0e, 0x0c, 0x0f, 0x0c, 0x10, 0x0c, 0x11, 0x0c, 0x12, 0x0c, 0x13, 0x0c, 0x14, 0x0c, 0x15, 0x0c, + 0x16, 0x0c, 0x17, 0x0c, 0x18, 0x0c, 0x19, 0x0c, 0x1a, 0x0c, 0x1b, 0x0c, 0x1c, 0x0c, 0x1d, 0x0c, 0x1e, + 0x0c, 0x1f, 0x0c, 0x20, 0x0c, 0x21, 0x0c, 0x22, 0x0c, 0x23, 0x0c, 0x24, 0x0c, 0x25, 0x0c, 0x26, 0x0c, + 0x27, 0x0c, 0x28, 0x0c, 0x29, 0x0c, 0x2a, 0x0c, 0x2b, 0x0c, 0x2c, 0x0c, 0x2d, 0x0c, 0x2e, 0x0c, 0x2f, + 0x0c, 0x30, 0x0c, 0x31, 0x0c, 0x32, 0x0c, 0x33, 0x0c, 0x34, 0x0c, 0x35, 0x0c, 0x36, 0x0c, 0x37, 0x0c, + 0x38, 0x0c, 0x39, 0x0c, 0x3a, 0x0c, 0x3b, 0x0c, 0x3c, 0x0c, 0x3d, 0x0c, 0x3e, 0x0c, 0x3f, 0x0c, 0x40, + 0x0c, 0x41, 0x0c, 0x42, 0x0c, 0x43, 0x0c, 0x44, 0x0c, 0x45, 0x0c, 0x46, 0x0c, 0x47, 0x0c, 0x48, 0x0c, + 0x49, 0x0c, 0x4a, 0x0c, 0x4b, 0x0c, 0x4c, 0x0c, 0x4d, 0x0c, 0x4e, 0x0c, 0x4f, 0x0c, 0x50, 0x0c, 0x51, + 0x0c, 0x52, 0x0c, 0x53, 0x0c, 0x54, 0x0c, 0x55, 0x0c, 0x56, 0x0c, 0x57, 0x0c, 0x58, 0x0c, 0x59, 0x0c, + 0x5a, 0x0c, 0x5b, 0x0c, 0x5c, 0x0c, 0x5d, 0x0c, 0x5e, 0x0c, 0x5f, 0x0c, 0x60, 0x0c, 0x61, 0x0c, 0x62, + 0x0c, 0x63, 0x0c, 0x64, 0x0c, 0x65, 0x0c, 0x66, 0x0c, 0x67, 0x0c, 0x68, 0x0c, 0x69, 0x0c, 0x6a, 0x0c, + 0x6b, 0x0c, 0x6c, 0x0c, 0x6d, 0x0c, 0x6e, 0x0c, 0x6f, 0x0c, 0x70, 0x0c, 0x71, 0x0c, 0x72, 0x0c, 0x73, + 0x0c, 0x74, 0x0c, 0x75, 0x0c, 0x76, 0x0c, 0x77, 0x0c, 0x78, 0x0c, 0x79, 0x0c, 0x7a, 0x0c, 0x7b, 0x0c, + 0x7c, 0x0c, 0x7d, 0x0c, 0x7e, 0x0c, 0x7f, 0x0c, 0x80, 0x0c, 0x81, 0x0c, 0x82, 0x0c, 0x83, 0x0c, 0x84, + 0x0c, 0x85, 0x0c, 0x86, 0x0c, 0x87, 0x0c, 0x88, 0x0c, 0x89, 0x0c, 0x8a, 0x0c, 0x8b, 0x0c, 0x8c, 0x0c, + 0x8d, 0x0c, 0x8e, 0x0c, 0x8f, 0x0c, 0x90, 0x0c, 0x91, 0x0c, 0x92, 0x0c, 0x93, 0x0c, 0x94, 0x0c, 0x95, + 0x0c, 0x96, 0x0c, 0x97, 0x0c, 0x98, 0x0c, 0x99, 0x0c, 0x9a, 0x0c, 0x9b, 0x0c, 0x9c, 0x0c, 0x9d, 0x0c, + 0x9e, 0x0c, 0x9f, 0x0c, 0xa0, 0x0c, 0xa1, 0x0c, 0xa2, 0x0c, 0xa3, 0x0c, 0xa4, 0x0c, 0xa5, 0x0c, 0xa6, + 0x0c, 0xa7, 0x0c, 0xa8, 0x0c, 0xa9, 0x0c, 0xaa, 0x0c, 0xab, 0x0c, 0xac, 0x0c, 0xad, 0x0c, 0xae, 0x0c, + 0xaf, 0x0c, 0xb0, 0x0c, 0xb1, 0x0c, 0xb2, 0x0c, 0xb3, 0x0c, 0xb4, 0x0c, 0xb5, 0x0c, 0xb6, 0x0c, 0xb7, + 0x0c, 0xb8, 0x0c, 0xb9, 0x0c, 0xba, 0x0c, 0xbb, 0x0c, 0xbc, 0x0c, 0xbd, 0x0c, 0xbe, 0x0c, 0xbf, 0x0c, + 0xc0, 0x0c, 0xc1, 0x0c, 0xc2, 0x0c, 0xc3, 0x0c, 0xc4, 0x0c, 0xc5, 0x0c, 0xc6, 0x0c, 0xc7, 0x0c, 0xc8, + 0x0c, 0xc9, 0x0c, 0xca, 0x0c, 0xcb, 0x0c, 0xcc, 0x0c, 0xcd, 0x0c, 0xce, 0x0c, 0xcf, 0x0c, 0xd0, 0x0c, + 0xd1, 0x0c, 0xd2, 0x0c, 0xd3, 0x0c, 0xd4, 0x0c, 0xd5, 0x0c, 0xd6, 0x0c, 0xd7, 0x0c, 0xd8, 0x0c, 0xd9, + 0x0c, 0xda, 0x0c, 0xdb, 0x0c, 0xdc, 0x0c, 0xdd, 0x0c, 0xde, 0x0c, 0xdf, 0x0c, 0xe0, 0x0c, 0xe1, 0x0c, + 0xe2, 0x0c, 0xe3, 0x0c, 0xe4, 0x0c, 0xe5, 0x0c, 0xe6, 0x0c, 0xe7, 0x0c, 0xe8, 0x0c, 0xe9, 0x0c, 0xea, + 0x0c, 0xeb, 0x0c, 0xec, 0x0c, 0xed, 0x0c, 0xee, 0x0c, 0xef, 0x0c, 0xf0, 0x0c, 0xf1, 0x0c, 0xf2, 0x0c, + 0xf3, 0x0c, 0xf4, 0x0c, 0xf5, 0x0c, 0xf6, 0x0c, 0xf7, 0x0c, 0xf8, 0x0c, 0xf9, 0x0c, 0xfa, 0x0c, 0xfb, + 0x0c, 0xfc, 0x0c, 0xfd, 0x0c, 0xfe, 0x0c, 0xff, 0x0c, 0x00, 0x0d, 0x01, 0x0d, 0x02, 0x0d, 0x03, 0x0d, + 0x04, 0x0d, 0x05, 0x0d, 0x06, 0x0d, 0x07, 0x0d, 0x08, 0x0d, 0x09, 0x0d, 0x0a, 0x0d, 0x0b, 0x0d, 0x0c, + 0x0d, 0x0d, 0x0d, 0x0e, 0x0d, 0x0f, 0x0d, 0x10, 0x0d, 0x11, 0x0d, 0x12, 0x0d, 0x13, 0x0d, 0x14, 0x0d, + 0x15, 0x0d, 0x16, 0x0d, 0x17, 0x0d, 0x18, 0x0d, 0x19, 0x0d, 0x1a, 0x0d, 0x1b, 0x0d, 0x1c, 0x0d, 0x1d, + 0x0d, 0x1e, 0x0d, 0x1f, 0x0d, 0x20, 0x0d, 0x21, 0x0d, 0x22, 0x0d, 0x23, 0x0d, 0x24, 0x0d, 0x25, 0x0d, + 0x26, 0x0d, 0x27, 0x0d, 0x28, 0x0d, 0x29, 0x0d, 0x2a, 0x0d, 0x2b, 0x0d, 0x2c, 0x0d, 0x2d, 0x0d, 0x2e, + 0x0d, 0x2f, 0x0d, 0x30, 0x0d, 0x31, 0x0d, 0x32, 0x0d, 0x33, 0x0d, 0x34, 0x0d, 0x35, 0x0d, 0x36, 0x0d, + 0x37, 0x0d, 0x38, 0x0d, 0x39, 0x0d, 0x3a, 0x0d, 0x3b, 0x0d, 0x3c, 0x0d, 0x3d, 0x0d, 0x3e, 0x0d, 0x3f, + 0x0d, 0x40, 0x0d, 0x41, 0x0d, 0x42, 0x0d, 0x43, 0x0d, 0x44, 0x0d, 0x45, 0x0d, 0x46, 0x0d, 0x47, 0x0d, + 0x48, 0x0d, 0x49, 0x0d, 0x4a, 0x0d, 0x4b, 0x0d, 0x4c, 0x0d, 0x4d, 0x0d, 0x4e, 0x0d, 0x4f, 0x0d, 0x50, + 0x0d, 0x51, 0x0d, 0x52, 0x0d, 0x53, 0x0d, 0x54, 0x0d, 0x55, 0x0d, 0x56, 0x0d, 0x57, 0x0d, 0x58, 0x0d, + 0x59, 0x0d, 0x5a, 0x0d, 0x5b, 0x0d, 0x5c, 0x0d, 0x5d, 0x0d, 0x5e, 0x0d, 0x5f, 0x0d, 0x60, 0x0d, 0x61, + 0x0d, 0x62, 0x0d, 0x63, 0x0d, 0x64, 0x0d, 0x65, 0x0d, 0x66, 0x0d, 0x67, 0x0d, 0x68, 0x0d, 0x69, 0x0d, + 0x6a, 0x0d, 0x6b, 0x0d, 0x6c, 0x0d, 0x6d, 0x0d, 0x6e, 0x0d, 0x6f, 0x0d, 0x70, 0x0d, 0x71, 0x0d, 0x72, + 0x0d, 0x73, 0x0d, 0x74, 0x0d, 0x75, 0x0d, 0x76, 0x0d, 0x77, 0x0d, 0x78, 0x0d, 0x79, 0x0d, 0x7a, 0x0d, + 0x7b, 0x0d, 0x7c, 0x0d, 0x7d, 0x0d, 0x7e, 0x0d, 0x7f, 0x0d, 0x80, 0x0d, 0x81, 0x0d, 0x82, 0x0d, 0x83, + 0x0d, 0x84, 0x0d, 0x85, 0x0d, 0x86, 0x0d, 0x87, 0x0d, 0x88, 0x0d, 0x89, 0x0d, 0x8a, 0x0d, 0x8b, 0x0d, + 0x8c, 0x0d, 0x8d, 0x0d, 0x8e, 0x0d, 0x8f, 0x0d, 0x90, 0x0d, 0x91, 0x0d, 0x92, 0x0d, 0x93, 0x0d, 0x94, + 0x0d, 0x95, 0x0d, 0x96, 0x0d, 0x97, 0x0d, 0x98, 0x0d, 0x99, 0x0d, 0x9a, 0x0d, 0x9b, 0x0d, 0x9c, 0x0d, + 0x9d, 0x0d, 0x9e, 0x0d, 0x9f, 0x0d, 0xa0, 0x0d, 0xa1, 0x0d, 0xa2, 0x0d, 0xa3, 0x0d, 0xa4, 0x0d, 0xa5, + 0x0d, 0xa6, 0x0d, 0xa7, 0x0d, 0xa8, 0x0d, 0xa9, 0x0d, 0xaa, 0x0d, 0xab, 0x0d, 0xac, 0x0d, 0xad, 0x0d, + 0xae, 0x0d, 0xaf, 0x0d, 0xb0, 0x0d, 0xb1, 0x0d, 0xb2, 0x0d, 0xb3, 0x0d, 0xb4, 0x0d, 0xb5, 0x0d, 0xb6, + 0x0d, 0xb7, 0x0d, 0xb8, 0x0d, 0xb9, 0x0d, 0xba, 0x0d, 0xbb, 0x0d, 0xbc, 0x0d, 0xbd, 0x0d, 0xbe, 0x0d, + 0xbf, 0x0d, 0xc0, 0x0d, 0xc1, 0x0d, 0xc2, 0x0d, 0xc3, 0x0d, 0xc4, 0x0d, 0xc5, 0x0d, 0xc6, 0x0d, 0xc7, + 0x0d, 0xc8, 0x0d, 0xc9, 0x0d, 0xca, 0x0d, 0xcb, 0x0d, 0xcc, 0x0d, 0xcd, 0x0d, 0xce, 0x0d, 0xcf, 0x0d, + 0xd0, 0x0d, 0xd1, 0x0d, 0xd2, 0x0d, 0xd3, 0x0d, 0xd4, 0x0d, 0xd5, 0x0d, 0xd6, 0x0d, 0xd7, 0x0d, 0xd8, + 0x0d, 0xd9, 0x0d, 0xda, 0x0d, 0xdb, 0x0d, 0xdc, 0x0d, 0xdd, 0x0d, 0xde, 0x0d, 0xdf, 0x0d, 0xe0, 0x0d, + 0xe1, 0x0d, 0xe2, 0x0d, 0xe3, 0x0d, 0xe4, 0x0d, 0xe5, 0x0d, 0xe6, 0x0d, 0xe7, 0x0d, 0xe8, 0x0d, 0xe9, + 0x0d, 0xea, 0x0d, 0xeb, 0x0d, 0xec, 0x0d, 0xed, 0x0d, 0xee, 0x0d, 0xef, 0x0d, 0xf0, 0x0d, 0xf1, 0x0d, + 0xf2, 0x0d, 0xf3, 0x0d, 0xf4, 0x0d, 0xf5, 0x0d, 0xf6, 0x0d, 0xf7, 0x0d, 0xf8, 0x0d, 0xf9, 0x0d, 0xfa, + 0x0d, 0xfb, 0x0d, 0xfc, 0x0d, 0xfd, 0x0d, 0xfe, 0x0d, 0xff, 0x0d, 0x00, 0x0e, 0x01, 0x0e, 0x02, 0x0e, + 0x03, 0x0e, 0x04, 0x0e, 0x05, 0x0e, 0x06, 0x0e, 0x07, 0x0e, 0x08, 0x0e, 0x09, 0x0e, 0x0a, 0x0e, 0x0b, + 0x0e, 0x0c, 0x0e, 0x0d, 0x0e, 0x0e, 0x0e, 0x0f, 0x0e, 0x10, 0x0e, 0x11, 0x0e, 0x12, 0x0e, 0x13, 0x0e, + 0x14, 0x0e, 0x15, 0x0e, 0x16, 0x0e, 0x17, 0x0e, 0x18, 0x0e, 0x19, 0x0e, 0x1a, 0x0e, 0x1b, 0x0e, 0x1c, + 0x0e, 0x1d, 0x0e, 0x1e, 0x0e, 0x1f, 0x0e, 0x20, 0x0e, 0x21, 0x0e, 0x22, 0x0e, 0x23, 0x0e, 0x24, 0x0e, + 0x25, 0x0e, 0x26, 0x0e, 0x27, 0x0e, 0x28, 0x0e, 0x29, 0x0e, 0x2a, 0x0e, 0x2b, 0x0e, 0x2c, 0x0e, 0x2d, + 0x0e, 0x2e, 0x0e, 0x2f, 0x0e, 0x30, 0x0e, 0x31, 0x0e, 0x32, 0x0e, 0x33, 0x0e, 0x34, 0x0e, 0x35, 0x0e, + 0x36, 0x0e, 0x37, 0x0e, 0x38, 0x0e, 0x39, 0x0e, 0x3a, 0x0e, 0x3b, 0x0e, 0x3c, 0x0e, 0x3d, 0x0e, 0x3e, + 0x0e, 0x3f, 0x0e, 0x40, 0x0e, 0x41, 0x0e, 0x42, 0x0e, 0x43, 0x0e, 0x44, 0x0e, 0x45, 0x0e, 0x46, 0x0e, + 0x47, 0x0e, 0x48, 0x0e, 0x49, 0x0e, 0x4a, 0x0e, 0x4b, 0x0e, 0x4c, 0x0e, 0x4d, 0x0e, 0x4e, 0x0e, 0x4f, + 0x0e, 0x50, 0x0e, 0x51, 0x0e, 0x52, 0x0e, 0x53, 0x0e, 0x54, 0x0e, 0x55, 0x0e, 0x56, 0x0e, 0x57, 0x0e, + 0x58, 0x0e, 0x59, 0x0e, 0x5a, 0x0e, 0x5b, 0x0e, 0x5c, 0x0e, 0x5d, 0x0e, 0x5e, 0x0e, 0x5f, 0x0e, 0x60, + 0x0e, 0x61, 0x0e, 0x62, 0x0e, 0x63, 0x0e, 0x64, 0x0e, 0x65, 0x0e, 0x66, 0x0e, 0x67, 0x0e, 0x68, 0x0e, + 0x69, 0x0e, 0x6a, 0x0e, 0x6b, 0x0e, 0x6c, 0x0e, 0x6d, 0x0e, 0x6e, 0x0e, 0x6f, 0x0e, 0x70, 0x0e, 0x71, + 0x0e, 0x72, 0x0e, 0x73, 0x0e, 0x74, 0x0e, 0x75, 0x0e, 0x76, 0x0e, 0x77, 0x0e, 0x78, 0x0e, 0x79, 0x0e, + 0x7a, 0x0e, 0x7b, 0x0e, 0x7c, 0x0e, 0x7d, 0x0e, 0x7e, 0x0e, 0x7f, 0x0e, 0x80, 0x0e, 0x81, 0x0e, 0x82, + 0x0e, 0x83, 0x0e, 0x84, 0x0e, 0x85, 0x0e, 0x86, 0x0e, 0x87, 0x0e, 0x88, 0x0e, 0x89, 0x0e, 0x8a, 0x0e, + 0x8b, 0x0e, 0x8c, 0x0e, 0x8d, 0x0e, 0x8e, 0x0e, 0x8f, 0x0e, 0x90, 0x0e, 0x91, 0x0e, 0x92, 0x0e, 0x93, + 0x0e, 0x94, 0x0e, 0x95, 0x0e, 0x96, 0x0e, 0x97, 0x0e, 0x98, 0x0e, 0x99, 0x0e, 0x9a, 0x0e, 0x9b, 0x0e, + 0x9c, 0x0e, 0x9d, 0x0e, 0x9e, 0x0e, 0x9f, 0x0e, 0xa0, 0x0e, 0xa1, 0x0e, 0xa2, 0x0e, 0xa3, 0x0e, 0xa4, + 0x0e, 0xa5, 0x0e, 0xa6, 0x0e, 0xa7, 0x0e, 0xa8, 0x0e, 0xa9, 0x0e, 0xaa, 0x0e, 0xab, 0x0e, 0xac, 0x0e, + 0xad, 0x0e, 0xae, 0x0e, 0xaf, 0x0e, 0xb0, 0x0e, 0xb1, 0x0e, 0xb2, 0x0e, 0xb3, 0x0e, 0xb4, 0x0e, 0xb5, + 0x0e, 0xb6, 0x0e, 0xb7, 0x0e, 0xb8, 0x0e, 0xb9, 0x0e, 0xba, 0x0e, 0xbb, 0x0e, 0xbc, 0x0e, 0xbd, 0x0e, + 0xbe, 0x0e, 0xbf, 0x0e, 0xc0, 0x0e, 0xc1, 0x0e, 0xc2, 0x0e, 0xc3, 0x0e, 0xc4, 0x0e, 0xc5, 0x0e, 0xc6, + 0x0e, 0xc7, 0x0e, 0xc8, 0x0e, 0xc9, 0x0e, 0xca, 0x0e, 0xcb, 0x0e, 0xcc, 0x0e, 0xcd, 0x0e, 0xce, 0x0e, + 0xcf, 0x0e, 0xd0, 0x0e, 0xd1, 0x0e, 0xd2, 0x0e, 0xd3, 0x0e, 0xd4, 0x0e, 0xd5, 0x0e, 0xd6, 0x0e, 0xd7, + 0x0e, 0xd8, 0x0e, 0xd9, 0x0e, 0xda, 0x0e, 0xdb, 0x0e, 0xdc, 0x0e, 0xdd, 0x0e, 0xde, 0x0e, 0xdf, 0x0e, + 0xe0, 0x0e, 0xe1, 0x0e, 0xe2, 0x0e, 0xe3, 0x0e, 0xe4, 0x0e, 0xe5, 0x0e, 0xe6, 0x0e, 0xe7, 0x0e, 0xe8, + 0x0e, 0xe9, 0x0e, 0xea, 0x0e, 0xeb, 0x0e, 0xec, 0x0e, 0xed, 0x0e, 0xee, 0x0e, 0xef, 0x0e, 0xf0, 0x0e, + 0xf1, 0x0e, 0xf2, 0x0e, 0xf3, 0x0e, 0xf4, 0x0e, 0xf5, 0x0e, 0xf6, 0x0e, 0xf7, 0x0e, 0xf8, 0x0e, 0xf9, + 0x0e, 0xfa, 0x0e, 0xfb, 0x0e, 0xfc, 0x0e, 0xfd, 0x0e, 0xfe, 0x0e, 0xff, 0x0e, 0x00, 0x0f, 0x01, 0x0f, + 0x02, 0x0f, 0x03, 0x0f, 0x04, 0x0f, 0x05, 0x0f, 0x06, 0x0f, 0x07, 0x0f, 0x08, 0x0f, 0x09, 0x0f, 0x0a, + 0x0f, 0x0b, 0x0f, 0x0c, 0x0f, 0x0d, 0x0f, 0x0e, 0x0f, 0x0f, 0x0f, 0x10, 0x0f, 0x11, 0x0f, 0x12, 0x0f, + 0x13, 0x0f, 0x14, 0x0f, 0x15, 0x0f, 0x16, 0x0f, 0x17, 0x0f, 0x18, 0x0f, 0x19, 0x0f, 0x1a, 0x0f, 0x1b, + 0x0f, 0x1c, 0x0f, 0x1d, 0x0f, 0x1e, 0x0f, 0x1f, 0x0f, 0x20, 0x0f, 0x21, 0x0f, 0x22, 0x0f, 0x23, 0x0f, + 0x24, 0x0f, 0x25, 0x0f, 0x26, 0x0f, 0x27, 0x0f, 0x28, 0x0f, 0x29, 0x0f, 0x2a, 0x0f, 0x2b, 0x0f, 0x2c, + 0x0f, 0x2d, 0x0f, 0x2e, 0x0f, 0x2f, 0x0f, 0x30, 0x0f, 0x31, 0x0f, 0x32, 0x0f, 0x33, 0x0f, 0x34, 0x0f, + 0x35, 0x0f, 0x36, 0x0f, 0x37, 0x0f, 0x38, 0x0f, 0x39, 0x0f, 0x3a, 0x0f, 0x3b, 0x0f, 0x3c, 0x0f, 0x3d, + 0x0f, 0x3e, 0x0f, 0x3f, 0x0f, 0x40, 0x0f, 0x41, 0x0f, 0x42, 0x0f, 0x43, 0x0f, 0x44, 0x0f, 0x45, 0x0f, + 0x46, 0x0f, 0x47, 0x0f, 0x48, 0x0f, 0x49, 0x0f, 0x4a, 0x0f, 0x4b, 0x0f, 0x4c, 0x0f, 0x4d, 0x0f, 0x4e, + 0x0f, 0x4f, 0x0f, 0x50, 0x0f, 0x51, 0x0f, 0x52, 0x0f, 0x53, 0x0f, 0x54, 0x0f, 0x55, 0x0f, 0x56, 0x0f, + 0x57, 0x0f, 0x58, 0x0f, 0x59, 0x0f, 0x5a, 0x0f, 0x5b, 0x0f, 0x5c, 0x0f, 0x5d, 0x0f, 0x5e, 0x0f, 0x5f, + 0x0f, 0x60, 0x0f, 0x61, 0x0f, 0x62, 0x0f, 0x63, 0x0f, 0x64, 0x0f, 0x65, 0x0f, 0x66, 0x0f, 0x67, 0x0f, + 0x68, 0x0f, 0x69, 0x0f, 0x6a, 0x0f, 0x6b, 0x0f, 0x6c, 0x0f, 0x6d, 0x0f, 0x6e, 0x0f, 0x6f, 0x0f, 0x70, + 0x0f, 0x71, 0x0f, 0x72, 0x0f, 0x73, 0x0f, 0x74, 0x0f, 0x75, 0x0f, 0x76, 0x0f, 0x77, 0x0f, 0x78, 0x0f, + 0x79, 0x0f, 0x7a, 0x0f, 0x7b, 0x0f, 0x7c, 0x0f, 0x7d, 0x0f, 0x7e, 0x0f, 0x7f, 0x0f, 0x80, 0x0f, 0x81, + 0x0f, 0x82, 0x0f, 0x83, 0x0f, 0x84, 0x0f, 0x85, 0x0f, 0x86, 0x0f, 0x87, 0x0f, 0x88, 0x0f, 0x89, 0x0f, + 0x8a, 0x0f, 0x8b, 0x0f, 0x8c, 0x0f, 0x8d, 0x0f, 0x8e, 0x0f, 0x8f, 0x0f, 0x90, 0x0f, 0x91, 0x0f, 0x92, + 0x0f, 0x93, 0x0f, 0x94, 0x0f, 0x95, 0x0f, 0x96, 0x0f, 0x97, 0x0f, 0x98, 0x0f, 0x99, 0x0f, 0x9a, 0x0f, + 0x9b, 0x0f, 0x9c, 0x0f, 0x9d, 0x0f, 0x9e, 0x0f, 0x9f, 0x0f, 0xa0, 0x0f, 0xa1, 0x0f, 0xa2, 0x0f, 0xa3, + 0x0f, 0xa4, 0x0f, 0xa5, 0x0f, 0xa6, 0x0f, 0xa7, 0x0f, 0xa8, 0x0f, 0xa9, 0x0f, 0xaa, 0x0f, 0xab, 0x0f, + 0xac, 0x0f, 0xad, 0x0f, 0xae, 0x0f, 0xaf, 0x0f, 0xb0, 0x0f, 0xb1, 0x0f, 0xb2, 0x0f, 0xb3, 0x0f, 0xb4, + 0x0f, 0xb5, 0x0f, 0xb6, 0x0f, 0xb7, 0x0f, 0xb8, 0x0f, 0xb9, 0x0f, 0xba, 0x0f, 0xbb, 0x0f, 0xbc, 0x0f, + 0xbd, 0x0f, 0xbe, 0x0f, 0xbf, 0x0f, 0xc0, 0x0f, 0xc1, 0x0f, 0xc2, 0x0f, 0xc3, 0x0f, 0xc4, 0x0f, 0xc5, + 0x0f, 0xc6, 0x0f, 0xc7, 0x0f, 0xc8, 0x0f, 0xc9, 0x0f, 0xca, 0x0f, 0xcb, 0x0f, 0xcc, 0x0f, 0xcd, 0x0f, + 0xce, 0x0f, 0xcf, 0x0f, 0xd0, 0x0f, 0xd1, 0x0f, 0xd2, 0x0f, 0xd3, 0x0f, 0xd4, 0x0f, 0xd5, 0x0f, 0xd6, + 0x0f, 0xd7, 0x0f, 0xd8, 0x0f, 0xd9, 0x0f, 0xda, 0x0f, 0xdb, 0x0f, 0xdc, 0x0f, 0xdd, 0x0f, 0xde, 0x0f, + 0xdf, 0x0f, 0xe0, 0x0f, 0xe1, 0x0f, 0xe2, 0x0f, 0xe3, 0x0f, 0xe4, 0x0f, 0xe5, 0x0f, 0xe6, 0x0f, 0xe7, + 0x0f, 0xe8, 0x0f, 0xe9, 0x0f, 0xea, 0x0f, 0xeb, 0x0f, 0xec, 0x0f, 0xed, 0x0f, 0xee, 0x0f, 0xef, 0x0f, + 0xf0, 0x0f, 0xf1, 0x0f, 0xf2, 0x0f, 0xf3, 0x0f, 0xf4, 0x0f, 0xf5, 0x0f, 0xf6, 0x0f, 0xf7, 0x0f, 0xf8, + 0x0f, 0xf9, 0x0f, 0xfa, 0x0f, 0xfb, 0x0f, 0xfc, 0x0f, 0xfd, 0x0f, 0xfe, 0x0f, 0xff, 0x0f}; unsigned int __col2block_cdf_len = 8192; constexpr int __col2block_cdf_umin = 0; constexpr int __col2block_cdf_umax = 4095; diff --git a/tests/col4block.h b/tests/col4block.h index b9dde31a6..d594e4973 100644 --- a/tests/col4block.h +++ b/tests/col4block.h @@ -18,690 +18,488 @@ #ifndef HAVE_COL4BLOCK #define HAVE_COL4BLOCK unsigned char __col4block_cdf[] = { - 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x09, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, - 0x0f, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, - 0x12, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x15, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, - 0x1b, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, - 0x1e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, - 0x21, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, - 0x24, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, - 0x27, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, - 0x2a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, - 0x2d, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, - 0x30, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, - 0x33, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, - 0x36, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, - 0x39, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, - 0x3c, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, - 0x3f, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, - 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, - 0x45, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, - 0x48, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, - 0x4b, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, - 0x4e, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, - 0x51, 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, - 0x54, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, - 0x57, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, - 0x5a, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, - 0x5d, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, - 0x60, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, - 0x63, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, - 0x66, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, - 0x69, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, - 0x6c, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, - 0x6f, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, - 0x75, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, - 0x78, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x7a, 0x00, 0x00, 0x00, - 0x7b, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, - 0x7e, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, - 0x81, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, - 0x84, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, - 0x87, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, - 0x8a, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, - 0x8d, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, - 0x90, 0x00, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, - 0x93, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, - 0x96, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, - 0x99, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x9b, 0x00, 0x00, 0x00, - 0x9c, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, - 0x9f, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, - 0xa2, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, - 0xa5, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, - 0xa8, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, - 0xab, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, - 0xae, 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, - 0xb1, 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, - 0xb4, 0x00, 0x00, 0x00, 0xb5, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, - 0xb7, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, - 0xba, 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, - 0xbd, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00, - 0xc0, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, - 0xc3, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, - 0xc6, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, - 0xc9, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0xcb, 0x00, 0x00, 0x00, - 0xcc, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, - 0xcf, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, 0x00, - 0xd2, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, - 0xd5, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, - 0xd8, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, - 0xdb, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, - 0xde, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, - 0xe1, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, - 0xe4, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, - 0xe7, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, - 0xea, 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0xec, 0x00, 0x00, 0x00, - 0xed, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, 0xef, 0x00, 0x00, 0x00, - 0xf0, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0xf2, 0x00, 0x00, 0x00, - 0xf3, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x00, 0x00, - 0xf6, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, - 0xf9, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, - 0xfc, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0x00, - 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, - 0x02, 0x01, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, - 0x05, 0x01, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, - 0x08, 0x01, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, - 0x0b, 0x01, 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, - 0x0e, 0x01, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, - 0x11, 0x01, 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, - 0x14, 0x01, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, - 0x17, 0x01, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, - 0x1a, 0x01, 0x00, 0x00, 0x1b, 0x01, 0x00, 0x00, 0x1c, 0x01, 0x00, 0x00, - 0x1d, 0x01, 0x00, 0x00, 0x1e, 0x01, 0x00, 0x00, 0x1f, 0x01, 0x00, 0x00, - 0x20, 0x01, 0x00, 0x00, 0x21, 0x01, 0x00, 0x00, 0x22, 0x01, 0x00, 0x00, - 0x23, 0x01, 0x00, 0x00, 0x24, 0x01, 0x00, 0x00, 0x25, 0x01, 0x00, 0x00, - 0x26, 0x01, 0x00, 0x00, 0x27, 0x01, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, - 0x29, 0x01, 0x00, 0x00, 0x2a, 0x01, 0x00, 0x00, 0x2b, 0x01, 0x00, 0x00, - 0x2c, 0x01, 0x00, 0x00, 0x2d, 0x01, 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, - 0x2f, 0x01, 0x00, 0x00, 0x30, 0x01, 0x00, 0x00, 0x31, 0x01, 0x00, 0x00, - 0x32, 0x01, 0x00, 0x00, 0x33, 0x01, 0x00, 0x00, 0x34, 0x01, 0x00, 0x00, - 0x35, 0x01, 0x00, 0x00, 0x36, 0x01, 0x00, 0x00, 0x37, 0x01, 0x00, 0x00, - 0x38, 0x01, 0x00, 0x00, 0x39, 0x01, 0x00, 0x00, 0x3a, 0x01, 0x00, 0x00, - 0x3b, 0x01, 0x00, 0x00, 0x3c, 0x01, 0x00, 0x00, 0x3d, 0x01, 0x00, 0x00, - 0x3e, 0x01, 0x00, 0x00, 0x3f, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, - 0x41, 0x01, 0x00, 0x00, 0x42, 0x01, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, - 0x44, 0x01, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x46, 0x01, 0x00, 0x00, - 0x47, 0x01, 0x00, 0x00, 0x48, 0x01, 0x00, 0x00, 0x49, 0x01, 0x00, 0x00, - 0x4a, 0x01, 0x00, 0x00, 0x4b, 0x01, 0x00, 0x00, 0x4c, 0x01, 0x00, 0x00, - 0x4d, 0x01, 0x00, 0x00, 0x4e, 0x01, 0x00, 0x00, 0x4f, 0x01, 0x00, 0x00, - 0x50, 0x01, 0x00, 0x00, 0x51, 0x01, 0x00, 0x00, 0x52, 0x01, 0x00, 0x00, - 0x53, 0x01, 0x00, 0x00, 0x54, 0x01, 0x00, 0x00, 0x55, 0x01, 0x00, 0x00, - 0x56, 0x01, 0x00, 0x00, 0x57, 0x01, 0x00, 0x00, 0x58, 0x01, 0x00, 0x00, - 0x59, 0x01, 0x00, 0x00, 0x5a, 0x01, 0x00, 0x00, 0x5b, 0x01, 0x00, 0x00, - 0x5c, 0x01, 0x00, 0x00, 0x5d, 0x01, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, - 0x5f, 0x01, 0x00, 0x00, 0x60, 0x01, 0x00, 0x00, 0x61, 0x01, 0x00, 0x00, - 0x62, 0x01, 0x00, 0x00, 0x63, 0x01, 0x00, 0x00, 0x64, 0x01, 0x00, 0x00, - 0x65, 0x01, 0x00, 0x00, 0x66, 0x01, 0x00, 0x00, 0x67, 0x01, 0x00, 0x00, - 0x68, 0x01, 0x00, 0x00, 0x69, 0x01, 0x00, 0x00, 0x6a, 0x01, 0x00, 0x00, - 0x6b, 0x01, 0x00, 0x00, 0x6c, 0x01, 0x00, 0x00, 0x6d, 0x01, 0x00, 0x00, - 0x6e, 0x01, 0x00, 0x00, 0x6f, 0x01, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, - 0x71, 0x01, 0x00, 0x00, 0x72, 0x01, 0x00, 0x00, 0x73, 0x01, 0x00, 0x00, - 0x74, 0x01, 0x00, 0x00, 0x75, 0x01, 0x00, 0x00, 0x76, 0x01, 0x00, 0x00, - 0x77, 0x01, 0x00, 0x00, 0x78, 0x01, 0x00, 0x00, 0x79, 0x01, 0x00, 0x00, - 0x7a, 0x01, 0x00, 0x00, 0x7b, 0x01, 0x00, 0x00, 0x7c, 0x01, 0x00, 0x00, - 0x7d, 0x01, 0x00, 0x00, 0x7e, 0x01, 0x00, 0x00, 0x7f, 0x01, 0x00, 0x00, - 0x80, 0x01, 0x00, 0x00, 0x81, 0x01, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, - 0x83, 0x01, 0x00, 0x00, 0x84, 0x01, 0x00, 0x00, 0x85, 0x01, 0x00, 0x00, - 0x86, 0x01, 0x00, 0x00, 0x87, 0x01, 0x00, 0x00, 0x88, 0x01, 0x00, 0x00, - 0x89, 0x01, 0x00, 0x00, 0x8a, 0x01, 0x00, 0x00, 0x8b, 0x01, 0x00, 0x00, - 0x8c, 0x01, 0x00, 0x00, 0x8d, 0x01, 0x00, 0x00, 0x8e, 0x01, 0x00, 0x00, - 0x8f, 0x01, 0x00, 0x00, 0x90, 0x01, 0x00, 0x00, 0x91, 0x01, 0x00, 0x00, - 0x92, 0x01, 0x00, 0x00, 0x93, 0x01, 0x00, 0x00, 0x94, 0x01, 0x00, 0x00, - 0x95, 0x01, 0x00, 0x00, 0x96, 0x01, 0x00, 0x00, 0x97, 0x01, 0x00, 0x00, - 0x98, 0x01, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, 0x9a, 0x01, 0x00, 0x00, - 0x9b, 0x01, 0x00, 0x00, 0x9c, 0x01, 0x00, 0x00, 0x9d, 0x01, 0x00, 0x00, - 0x9e, 0x01, 0x00, 0x00, 0x9f, 0x01, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, - 0xa1, 0x01, 0x00, 0x00, 0xa2, 0x01, 0x00, 0x00, 0xa3, 0x01, 0x00, 0x00, - 0xa4, 0x01, 0x00, 0x00, 0xa5, 0x01, 0x00, 0x00, 0xa6, 0x01, 0x00, 0x00, - 0xa7, 0x01, 0x00, 0x00, 0xa8, 0x01, 0x00, 0x00, 0xa9, 0x01, 0x00, 0x00, - 0xaa, 0x01, 0x00, 0x00, 0xab, 0x01, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, - 0xad, 0x01, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0xaf, 0x01, 0x00, 0x00, - 0xb0, 0x01, 0x00, 0x00, 0xb1, 0x01, 0x00, 0x00, 0xb2, 0x01, 0x00, 0x00, - 0xb3, 0x01, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0xb5, 0x01, 0x00, 0x00, - 0xb6, 0x01, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0xb8, 0x01, 0x00, 0x00, - 0xb9, 0x01, 0x00, 0x00, 0xba, 0x01, 0x00, 0x00, 0xbb, 0x01, 0x00, 0x00, - 0xbc, 0x01, 0x00, 0x00, 0xbd, 0x01, 0x00, 0x00, 0xbe, 0x01, 0x00, 0x00, - 0xbf, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc1, 0x01, 0x00, 0x00, - 0xc2, 0x01, 0x00, 0x00, 0xc3, 0x01, 0x00, 0x00, 0xc4, 0x01, 0x00, 0x00, - 0xc5, 0x01, 0x00, 0x00, 0xc6, 0x01, 0x00, 0x00, 0xc7, 0x01, 0x00, 0x00, - 0xc8, 0x01, 0x00, 0x00, 0xc9, 0x01, 0x00, 0x00, 0xca, 0x01, 0x00, 0x00, - 0xcb, 0x01, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, 0xcd, 0x01, 0x00, 0x00, - 0xce, 0x01, 0x00, 0x00, 0xcf, 0x01, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, - 0xd1, 0x01, 0x00, 0x00, 0xd2, 0x01, 0x00, 0x00, 0xd3, 0x01, 0x00, 0x00, - 0xd4, 0x01, 0x00, 0x00, 0xd5, 0x01, 0x00, 0x00, 0xd6, 0x01, 0x00, 0x00, - 0xd7, 0x01, 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, 0xd9, 0x01, 0x00, 0x00, - 0xda, 0x01, 0x00, 0x00, 0xdb, 0x01, 0x00, 0x00, 0xdc, 0x01, 0x00, 0x00, - 0xdd, 0x01, 0x00, 0x00, 0xde, 0x01, 0x00, 0x00, 0xdf, 0x01, 0x00, 0x00, - 0xe0, 0x01, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, - 0xe3, 0x01, 0x00, 0x00, 0xe4, 0x01, 0x00, 0x00, 0xe5, 0x01, 0x00, 0x00, - 0xe6, 0x01, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, 0xe8, 0x01, 0x00, 0x00, - 0xe9, 0x01, 0x00, 0x00, 0xea, 0x01, 0x00, 0x00, 0xeb, 0x01, 0x00, 0x00, - 0xec, 0x01, 0x00, 0x00, 0xed, 0x01, 0x00, 0x00, 0xee, 0x01, 0x00, 0x00, - 0xef, 0x01, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, 0xf1, 0x01, 0x00, 0x00, - 0xf2, 0x01, 0x00, 0x00, 0xf3, 0x01, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, - 0xf5, 0x01, 0x00, 0x00, 0xf6, 0x01, 0x00, 0x00, 0xf7, 0x01, 0x00, 0x00, - 0xf8, 0x01, 0x00, 0x00, 0xf9, 0x01, 0x00, 0x00, 0xfa, 0x01, 0x00, 0x00, - 0xfb, 0x01, 0x00, 0x00, 0xfc, 0x01, 0x00, 0x00, 0xfd, 0x01, 0x00, 0x00, - 0xfe, 0x01, 0x00, 0x00, 0xff, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, - 0x01, 0x02, 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, - 0x04, 0x02, 0x00, 0x00, 0x05, 0x02, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, - 0x07, 0x02, 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x09, 0x02, 0x00, 0x00, - 0x0a, 0x02, 0x00, 0x00, 0x0b, 0x02, 0x00, 0x00, 0x0c, 0x02, 0x00, 0x00, - 0x0d, 0x02, 0x00, 0x00, 0x0e, 0x02, 0x00, 0x00, 0x0f, 0x02, 0x00, 0x00, - 0x10, 0x02, 0x00, 0x00, 0x11, 0x02, 0x00, 0x00, 0x12, 0x02, 0x00, 0x00, - 0x13, 0x02, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x15, 0x02, 0x00, 0x00, - 0x16, 0x02, 0x00, 0x00, 0x17, 0x02, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, - 0x19, 0x02, 0x00, 0x00, 0x1a, 0x02, 0x00, 0x00, 0x1b, 0x02, 0x00, 0x00, - 0x1c, 0x02, 0x00, 0x00, 0x1d, 0x02, 0x00, 0x00, 0x1e, 0x02, 0x00, 0x00, - 0x1f, 0x02, 0x00, 0x00, 0x20, 0x02, 0x00, 0x00, 0x21, 0x02, 0x00, 0x00, - 0x22, 0x02, 0x00, 0x00, 0x23, 0x02, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, - 0x25, 0x02, 0x00, 0x00, 0x26, 0x02, 0x00, 0x00, 0x27, 0x02, 0x00, 0x00, - 0x28, 0x02, 0x00, 0x00, 0x29, 0x02, 0x00, 0x00, 0x2a, 0x02, 0x00, 0x00, - 0x2b, 0x02, 0x00, 0x00, 0x2c, 0x02, 0x00, 0x00, 0x2d, 0x02, 0x00, 0x00, - 0x2e, 0x02, 0x00, 0x00, 0x2f, 0x02, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, - 0x31, 0x02, 0x00, 0x00, 0x32, 0x02, 0x00, 0x00, 0x33, 0x02, 0x00, 0x00, - 0x34, 0x02, 0x00, 0x00, 0x35, 0x02, 0x00, 0x00, 0x36, 0x02, 0x00, 0x00, - 0x37, 0x02, 0x00, 0x00, 0x38, 0x02, 0x00, 0x00, 0x39, 0x02, 0x00, 0x00, - 0x3a, 0x02, 0x00, 0x00, 0x3b, 0x02, 0x00, 0x00, 0x3c, 0x02, 0x00, 0x00, - 0x3d, 0x02, 0x00, 0x00, 0x3e, 0x02, 0x00, 0x00, 0x3f, 0x02, 0x00, 0x00, - 0x40, 0x02, 0x00, 0x00, 0x41, 0x02, 0x00, 0x00, 0x42, 0x02, 0x00, 0x00, - 0x43, 0x02, 0x00, 0x00, 0x44, 0x02, 0x00, 0x00, 0x45, 0x02, 0x00, 0x00, - 0x46, 0x02, 0x00, 0x00, 0x47, 0x02, 0x00, 0x00, 0x48, 0x02, 0x00, 0x00, - 0x49, 0x02, 0x00, 0x00, 0x4a, 0x02, 0x00, 0x00, 0x4b, 0x02, 0x00, 0x00, - 0x4c, 0x02, 0x00, 0x00, 0x4d, 0x02, 0x00, 0x00, 0x4e, 0x02, 0x00, 0x00, - 0x4f, 0x02, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, 0x51, 0x02, 0x00, 0x00, - 0x52, 0x02, 0x00, 0x00, 0x53, 0x02, 0x00, 0x00, 0x54, 0x02, 0x00, 0x00, - 0x55, 0x02, 0x00, 0x00, 0x56, 0x02, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, - 0x58, 0x02, 0x00, 0x00, 0x59, 0x02, 0x00, 0x00, 0x5a, 0x02, 0x00, 0x00, - 0x5b, 0x02, 0x00, 0x00, 0x5c, 0x02, 0x00, 0x00, 0x5d, 0x02, 0x00, 0x00, - 0x5e, 0x02, 0x00, 0x00, 0x5f, 0x02, 0x00, 0x00, 0x60, 0x02, 0x00, 0x00, - 0x61, 0x02, 0x00, 0x00, 0x62, 0x02, 0x00, 0x00, 0x63, 0x02, 0x00, 0x00, - 0x64, 0x02, 0x00, 0x00, 0x65, 0x02, 0x00, 0x00, 0x66, 0x02, 0x00, 0x00, - 0x67, 0x02, 0x00, 0x00, 0x68, 0x02, 0x00, 0x00, 0x69, 0x02, 0x00, 0x00, - 0x6a, 0x02, 0x00, 0x00, 0x6b, 0x02, 0x00, 0x00, 0x6c, 0x02, 0x00, 0x00, - 0x6d, 0x02, 0x00, 0x00, 0x6e, 0x02, 0x00, 0x00, 0x6f, 0x02, 0x00, 0x00, - 0x70, 0x02, 0x00, 0x00, 0x71, 0x02, 0x00, 0x00, 0x72, 0x02, 0x00, 0x00, - 0x73, 0x02, 0x00, 0x00, 0x74, 0x02, 0x00, 0x00, 0x75, 0x02, 0x00, 0x00, - 0x76, 0x02, 0x00, 0x00, 0x77, 0x02, 0x00, 0x00, 0x78, 0x02, 0x00, 0x00, - 0x79, 0x02, 0x00, 0x00, 0x7a, 0x02, 0x00, 0x00, 0x7b, 0x02, 0x00, 0x00, - 0x7c, 0x02, 0x00, 0x00, 0x7d, 0x02, 0x00, 0x00, 0x7e, 0x02, 0x00, 0x00, - 0x7f, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x81, 0x02, 0x00, 0x00, - 0x82, 0x02, 0x00, 0x00, 0x83, 0x02, 0x00, 0x00, 0x84, 0x02, 0x00, 0x00, - 0x85, 0x02, 0x00, 0x00, 0x86, 0x02, 0x00, 0x00, 0x87, 0x02, 0x00, 0x00, - 0x88, 0x02, 0x00, 0x00, 0x89, 0x02, 0x00, 0x00, 0x8a, 0x02, 0x00, 0x00, - 0x8b, 0x02, 0x00, 0x00, 0x8c, 0x02, 0x00, 0x00, 0x8d, 0x02, 0x00, 0x00, - 0x8e, 0x02, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, - 0x91, 0x02, 0x00, 0x00, 0x92, 0x02, 0x00, 0x00, 0x93, 0x02, 0x00, 0x00, - 0x94, 0x02, 0x00, 0x00, 0x95, 0x02, 0x00, 0x00, 0x96, 0x02, 0x00, 0x00, - 0x97, 0x02, 0x00, 0x00, 0x98, 0x02, 0x00, 0x00, 0x99, 0x02, 0x00, 0x00, - 0x9a, 0x02, 0x00, 0x00, 0x9b, 0x02, 0x00, 0x00, 0x9c, 0x02, 0x00, 0x00, - 0x9d, 0x02, 0x00, 0x00, 0x9e, 0x02, 0x00, 0x00, 0x9f, 0x02, 0x00, 0x00, - 0xa0, 0x02, 0x00, 0x00, 0xa1, 0x02, 0x00, 0x00, 0xa2, 0x02, 0x00, 0x00, - 0xa3, 0x02, 0x00, 0x00, 0xa4, 0x02, 0x00, 0x00, 0xa5, 0x02, 0x00, 0x00, - 0xa6, 0x02, 0x00, 0x00, 0xa7, 0x02, 0x00, 0x00, 0xa8, 0x02, 0x00, 0x00, - 0xa9, 0x02, 0x00, 0x00, 0xaa, 0x02, 0x00, 0x00, 0xab, 0x02, 0x00, 0x00, - 0xac, 0x02, 0x00, 0x00, 0xad, 0x02, 0x00, 0x00, 0xae, 0x02, 0x00, 0x00, - 0xaf, 0x02, 0x00, 0x00, 0xb0, 0x02, 0x00, 0x00, 0xb1, 0x02, 0x00, 0x00, - 0xb2, 0x02, 0x00, 0x00, 0xb3, 0x02, 0x00, 0x00, 0xb4, 0x02, 0x00, 0x00, - 0xb5, 0x02, 0x00, 0x00, 0xb6, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, - 0xb8, 0x02, 0x00, 0x00, 0xb9, 0x02, 0x00, 0x00, 0xba, 0x02, 0x00, 0x00, - 0xbb, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0xbd, 0x02, 0x00, 0x00, - 0xbe, 0x02, 0x00, 0x00, 0xbf, 0x02, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, - 0xc1, 0x02, 0x00, 0x00, 0xc2, 0x02, 0x00, 0x00, 0xc3, 0x02, 0x00, 0x00, - 0xc4, 0x02, 0x00, 0x00, 0xc5, 0x02, 0x00, 0x00, 0xc6, 0x02, 0x00, 0x00, - 0xc7, 0x02, 0x00, 0x00, 0xc8, 0x02, 0x00, 0x00, 0xc9, 0x02, 0x00, 0x00, - 0xca, 0x02, 0x00, 0x00, 0xcb, 0x02, 0x00, 0x00, 0xcc, 0x02, 0x00, 0x00, - 0xcd, 0x02, 0x00, 0x00, 0xce, 0x02, 0x00, 0x00, 0xcf, 0x02, 0x00, 0x00, - 0xd0, 0x02, 0x00, 0x00, 0xd1, 0x02, 0x00, 0x00, 0xd2, 0x02, 0x00, 0x00, - 0xd3, 0x02, 0x00, 0x00, 0xd4, 0x02, 0x00, 0x00, 0xd5, 0x02, 0x00, 0x00, - 0xd6, 0x02, 0x00, 0x00, 0xd7, 0x02, 0x00, 0x00, 0xd8, 0x02, 0x00, 0x00, - 0xd9, 0x02, 0x00, 0x00, 0xda, 0x02, 0x00, 0x00, 0xdb, 0x02, 0x00, 0x00, - 0xdc, 0x02, 0x00, 0x00, 0xdd, 0x02, 0x00, 0x00, 0xde, 0x02, 0x00, 0x00, - 0xdf, 0x02, 0x00, 0x00, 0xe0, 0x02, 0x00, 0x00, 0xe1, 0x02, 0x00, 0x00, - 0xe2, 0x02, 0x00, 0x00, 0xe3, 0x02, 0x00, 0x00, 0xe4, 0x02, 0x00, 0x00, - 0xe5, 0x02, 0x00, 0x00, 0xe6, 0x02, 0x00, 0x00, 0xe7, 0x02, 0x00, 0x00, - 0xe8, 0x02, 0x00, 0x00, 0xe9, 0x02, 0x00, 0x00, 0xea, 0x02, 0x00, 0x00, - 0xeb, 0x02, 0x00, 0x00, 0xec, 0x02, 0x00, 0x00, 0xed, 0x02, 0x00, 0x00, - 0xee, 0x02, 0x00, 0x00, 0xef, 0x02, 0x00, 0x00, 0xf0, 0x02, 0x00, 0x00, - 0xf1, 0x02, 0x00, 0x00, 0xf2, 0x02, 0x00, 0x00, 0xf3, 0x02, 0x00, 0x00, - 0xf4, 0x02, 0x00, 0x00, 0xf5, 0x02, 0x00, 0x00, 0xf6, 0x02, 0x00, 0x00, - 0xf7, 0x02, 0x00, 0x00, 0xf8, 0x02, 0x00, 0x00, 0xf9, 0x02, 0x00, 0x00, - 0xfa, 0x02, 0x00, 0x00, 0xfb, 0x02, 0x00, 0x00, 0xfc, 0x02, 0x00, 0x00, - 0xfd, 0x02, 0x00, 0x00, 0xfe, 0x02, 0x00, 0x00, 0xff, 0x02, 0x00, 0x00, - 0x00, 0x03, 0x00, 0x00, 0x01, 0x03, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, - 0x03, 0x03, 0x00, 0x00, 0x04, 0x03, 0x00, 0x00, 0x05, 0x03, 0x00, 0x00, - 0x06, 0x03, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x08, 0x03, 0x00, 0x00, - 0x09, 0x03, 0x00, 0x00, 0x0a, 0x03, 0x00, 0x00, 0x0b, 0x03, 0x00, 0x00, - 0x0c, 0x03, 0x00, 0x00, 0x0d, 0x03, 0x00, 0x00, 0x0e, 0x03, 0x00, 0x00, - 0x0f, 0x03, 0x00, 0x00, 0x10, 0x03, 0x00, 0x00, 0x11, 0x03, 0x00, 0x00, - 0x12, 0x03, 0x00, 0x00, 0x13, 0x03, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, - 0x15, 0x03, 0x00, 0x00, 0x16, 0x03, 0x00, 0x00, 0x17, 0x03, 0x00, 0x00, - 0x18, 0x03, 0x00, 0x00, 0x19, 0x03, 0x00, 0x00, 0x1a, 0x03, 0x00, 0x00, - 0x1b, 0x03, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, 0x1d, 0x03, 0x00, 0x00, - 0x1e, 0x03, 0x00, 0x00, 0x1f, 0x03, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, - 0x21, 0x03, 0x00, 0x00, 0x22, 0x03, 0x00, 0x00, 0x23, 0x03, 0x00, 0x00, - 0x24, 0x03, 0x00, 0x00, 0x25, 0x03, 0x00, 0x00, 0x26, 0x03, 0x00, 0x00, - 0x27, 0x03, 0x00, 0x00, 0x28, 0x03, 0x00, 0x00, 0x29, 0x03, 0x00, 0x00, - 0x2a, 0x03, 0x00, 0x00, 0x2b, 0x03, 0x00, 0x00, 0x2c, 0x03, 0x00, 0x00, - 0x2d, 0x03, 0x00, 0x00, 0x2e, 0x03, 0x00, 0x00, 0x2f, 0x03, 0x00, 0x00, - 0x30, 0x03, 0x00, 0x00, 0x31, 0x03, 0x00, 0x00, 0x32, 0x03, 0x00, 0x00, - 0x33, 0x03, 0x00, 0x00, 0x34, 0x03, 0x00, 0x00, 0x35, 0x03, 0x00, 0x00, - 0x36, 0x03, 0x00, 0x00, 0x37, 0x03, 0x00, 0x00, 0x38, 0x03, 0x00, 0x00, - 0x39, 0x03, 0x00, 0x00, 0x3a, 0x03, 0x00, 0x00, 0x3b, 0x03, 0x00, 0x00, - 0x3c, 0x03, 0x00, 0x00, 0x3d, 0x03, 0x00, 0x00, 0x3e, 0x03, 0x00, 0x00, - 0x3f, 0x03, 0x00, 0x00, 0x40, 0x03, 0x00, 0x00, 0x41, 0x03, 0x00, 0x00, - 0x42, 0x03, 0x00, 0x00, 0x43, 0x03, 0x00, 0x00, 0x44, 0x03, 0x00, 0x00, - 0x45, 0x03, 0x00, 0x00, 0x46, 0x03, 0x00, 0x00, 0x47, 0x03, 0x00, 0x00, - 0x48, 0x03, 0x00, 0x00, 0x49, 0x03, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, - 0x4b, 0x03, 0x00, 0x00, 0x4c, 0x03, 0x00, 0x00, 0x4d, 0x03, 0x00, 0x00, - 0x4e, 0x03, 0x00, 0x00, 0x4f, 0x03, 0x00, 0x00, 0x50, 0x03, 0x00, 0x00, - 0x51, 0x03, 0x00, 0x00, 0x52, 0x03, 0x00, 0x00, 0x53, 0x03, 0x00, 0x00, - 0x54, 0x03, 0x00, 0x00, 0x55, 0x03, 0x00, 0x00, 0x56, 0x03, 0x00, 0x00, - 0x57, 0x03, 0x00, 0x00, 0x58, 0x03, 0x00, 0x00, 0x59, 0x03, 0x00, 0x00, - 0x5a, 0x03, 0x00, 0x00, 0x5b, 0x03, 0x00, 0x00, 0x5c, 0x03, 0x00, 0x00, - 0x5d, 0x03, 0x00, 0x00, 0x5e, 0x03, 0x00, 0x00, 0x5f, 0x03, 0x00, 0x00, - 0x60, 0x03, 0x00, 0x00, 0x61, 0x03, 0x00, 0x00, 0x62, 0x03, 0x00, 0x00, - 0x63, 0x03, 0x00, 0x00, 0x64, 0x03, 0x00, 0x00, 0x65, 0x03, 0x00, 0x00, - 0x66, 0x03, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, 0x68, 0x03, 0x00, 0x00, - 0x69, 0x03, 0x00, 0x00, 0x6a, 0x03, 0x00, 0x00, 0x6b, 0x03, 0x00, 0x00, - 0x6c, 0x03, 0x00, 0x00, 0x6d, 0x03, 0x00, 0x00, 0x6e, 0x03, 0x00, 0x00, - 0x6f, 0x03, 0x00, 0x00, 0x70, 0x03, 0x00, 0x00, 0x71, 0x03, 0x00, 0x00, - 0x72, 0x03, 0x00, 0x00, 0x73, 0x03, 0x00, 0x00, 0x74, 0x03, 0x00, 0x00, - 0x75, 0x03, 0x00, 0x00, 0x76, 0x03, 0x00, 0x00, 0x77, 0x03, 0x00, 0x00, - 0x78, 0x03, 0x00, 0x00, 0x79, 0x03, 0x00, 0x00, 0x7a, 0x03, 0x00, 0x00, - 0x7b, 0x03, 0x00, 0x00, 0x7c, 0x03, 0x00, 0x00, 0x7d, 0x03, 0x00, 0x00, - 0x7e, 0x03, 0x00, 0x00, 0x7f, 0x03, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, - 0x81, 0x03, 0x00, 0x00, 0x82, 0x03, 0x00, 0x00, 0x83, 0x03, 0x00, 0x00, - 0x84, 0x03, 0x00, 0x00, 0x85, 0x03, 0x00, 0x00, 0x86, 0x03, 0x00, 0x00, - 0x87, 0x03, 0x00, 0x00, 0x88, 0x03, 0x00, 0x00, 0x89, 0x03, 0x00, 0x00, - 0x8a, 0x03, 0x00, 0x00, 0x8b, 0x03, 0x00, 0x00, 0x8c, 0x03, 0x00, 0x00, - 0x8d, 0x03, 0x00, 0x00, 0x8e, 0x03, 0x00, 0x00, 0x8f, 0x03, 0x00, 0x00, - 0x90, 0x03, 0x00, 0x00, 0x91, 0x03, 0x00, 0x00, 0x92, 0x03, 0x00, 0x00, - 0x93, 0x03, 0x00, 0x00, 0x94, 0x03, 0x00, 0x00, 0x95, 0x03, 0x00, 0x00, - 0x96, 0x03, 0x00, 0x00, 0x97, 0x03, 0x00, 0x00, 0x98, 0x03, 0x00, 0x00, - 0x99, 0x03, 0x00, 0x00, 0x9a, 0x03, 0x00, 0x00, 0x9b, 0x03, 0x00, 0x00, - 0x9c, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9e, 0x03, 0x00, 0x00, - 0x9f, 0x03, 0x00, 0x00, 0xa0, 0x03, 0x00, 0x00, 0xa1, 0x03, 0x00, 0x00, - 0xa2, 0x03, 0x00, 0x00, 0xa3, 0x03, 0x00, 0x00, 0xa4, 0x03, 0x00, 0x00, - 0xa5, 0x03, 0x00, 0x00, 0xa6, 0x03, 0x00, 0x00, 0xa7, 0x03, 0x00, 0x00, - 0xa8, 0x03, 0x00, 0x00, 0xa9, 0x03, 0x00, 0x00, 0xaa, 0x03, 0x00, 0x00, - 0xab, 0x03, 0x00, 0x00, 0xac, 0x03, 0x00, 0x00, 0xad, 0x03, 0x00, 0x00, - 0xae, 0x03, 0x00, 0x00, 0xaf, 0x03, 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, - 0xb1, 0x03, 0x00, 0x00, 0xb2, 0x03, 0x00, 0x00, 0xb3, 0x03, 0x00, 0x00, - 0xb4, 0x03, 0x00, 0x00, 0xb5, 0x03, 0x00, 0x00, 0xb6, 0x03, 0x00, 0x00, - 0xb7, 0x03, 0x00, 0x00, 0xb8, 0x03, 0x00, 0x00, 0xb9, 0x03, 0x00, 0x00, - 0xba, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbc, 0x03, 0x00, 0x00, - 0xbd, 0x03, 0x00, 0x00, 0xbe, 0x03, 0x00, 0x00, 0xbf, 0x03, 0x00, 0x00, - 0xc0, 0x03, 0x00, 0x00, 0xc1, 0x03, 0x00, 0x00, 0xc2, 0x03, 0x00, 0x00, - 0xc3, 0x03, 0x00, 0x00, 0xc4, 0x03, 0x00, 0x00, 0xc5, 0x03, 0x00, 0x00, - 0xc6, 0x03, 0x00, 0x00, 0xc7, 0x03, 0x00, 0x00, 0xc8, 0x03, 0x00, 0x00, - 0xc9, 0x03, 0x00, 0x00, 0xca, 0x03, 0x00, 0x00, 0xcb, 0x03, 0x00, 0x00, - 0xcc, 0x03, 0x00, 0x00, 0xcd, 0x03, 0x00, 0x00, 0xce, 0x03, 0x00, 0x00, - 0xcf, 0x03, 0x00, 0x00, 0xd0, 0x03, 0x00, 0x00, 0xd1, 0x03, 0x00, 0x00, - 0xd2, 0x03, 0x00, 0x00, 0xd3, 0x03, 0x00, 0x00, 0xd4, 0x03, 0x00, 0x00, - 0xd5, 0x03, 0x00, 0x00, 0xd6, 0x03, 0x00, 0x00, 0xd7, 0x03, 0x00, 0x00, - 0xd8, 0x03, 0x00, 0x00, 0xd9, 0x03, 0x00, 0x00, 0xda, 0x03, 0x00, 0x00, - 0xdb, 0x03, 0x00, 0x00, 0xdc, 0x03, 0x00, 0x00, 0xdd, 0x03, 0x00, 0x00, - 0xde, 0x03, 0x00, 0x00, 0xdf, 0x03, 0x00, 0x00, 0xe0, 0x03, 0x00, 0x00, - 0xe1, 0x03, 0x00, 0x00, 0xe2, 0x03, 0x00, 0x00, 0xe3, 0x03, 0x00, 0x00, - 0xe4, 0x03, 0x00, 0x00, 0xe5, 0x03, 0x00, 0x00, 0xe6, 0x03, 0x00, 0x00, - 0xe7, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, - 0xea, 0x03, 0x00, 0x00, 0xeb, 0x03, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, - 0xed, 0x03, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0xef, 0x03, 0x00, 0x00, - 0xf0, 0x03, 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, - 0xf3, 0x03, 0x00, 0x00, 0xf4, 0x03, 0x00, 0x00, 0xf5, 0x03, 0x00, 0x00, - 0xf6, 0x03, 0x00, 0x00, 0xf7, 0x03, 0x00, 0x00, 0xf8, 0x03, 0x00, 0x00, - 0xf9, 0x03, 0x00, 0x00, 0xfa, 0x03, 0x00, 0x00, 0xfb, 0x03, 0x00, 0x00, - 0xfc, 0x03, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, - 0xff, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, - 0x02, 0x04, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, - 0x05, 0x04, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x07, 0x04, 0x00, 0x00, - 0x08, 0x04, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x0a, 0x04, 0x00, 0x00, - 0x0b, 0x04, 0x00, 0x00, 0x0c, 0x04, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, - 0x0e, 0x04, 0x00, 0x00, 0x0f, 0x04, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, - 0x11, 0x04, 0x00, 0x00, 0x12, 0x04, 0x00, 0x00, 0x13, 0x04, 0x00, 0x00, - 0x14, 0x04, 0x00, 0x00, 0x15, 0x04, 0x00, 0x00, 0x16, 0x04, 0x00, 0x00, - 0x17, 0x04, 0x00, 0x00, 0x18, 0x04, 0x00, 0x00, 0x19, 0x04, 0x00, 0x00, - 0x1a, 0x04, 0x00, 0x00, 0x1b, 0x04, 0x00, 0x00, 0x1c, 0x04, 0x00, 0x00, - 0x1d, 0x04, 0x00, 0x00, 0x1e, 0x04, 0x00, 0x00, 0x1f, 0x04, 0x00, 0x00, - 0x20, 0x04, 0x00, 0x00, 0x21, 0x04, 0x00, 0x00, 0x22, 0x04, 0x00, 0x00, - 0x23, 0x04, 0x00, 0x00, 0x24, 0x04, 0x00, 0x00, 0x25, 0x04, 0x00, 0x00, - 0x26, 0x04, 0x00, 0x00, 0x27, 0x04, 0x00, 0x00, 0x28, 0x04, 0x00, 0x00, - 0x29, 0x04, 0x00, 0x00, 0x2a, 0x04, 0x00, 0x00, 0x2b, 0x04, 0x00, 0x00, - 0x2c, 0x04, 0x00, 0x00, 0x2d, 0x04, 0x00, 0x00, 0x2e, 0x04, 0x00, 0x00, - 0x2f, 0x04, 0x00, 0x00, 0x30, 0x04, 0x00, 0x00, 0x31, 0x04, 0x00, 0x00, - 0x32, 0x04, 0x00, 0x00, 0x33, 0x04, 0x00, 0x00, 0x34, 0x04, 0x00, 0x00, - 0x35, 0x04, 0x00, 0x00, 0x36, 0x04, 0x00, 0x00, 0x37, 0x04, 0x00, 0x00, - 0x38, 0x04, 0x00, 0x00, 0x39, 0x04, 0x00, 0x00, 0x3a, 0x04, 0x00, 0x00, - 0x3b, 0x04, 0x00, 0x00, 0x3c, 0x04, 0x00, 0x00, 0x3d, 0x04, 0x00, 0x00, - 0x3e, 0x04, 0x00, 0x00, 0x3f, 0x04, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, - 0x41, 0x04, 0x00, 0x00, 0x42, 0x04, 0x00, 0x00, 0x43, 0x04, 0x00, 0x00, - 0x44, 0x04, 0x00, 0x00, 0x45, 0x04, 0x00, 0x00, 0x46, 0x04, 0x00, 0x00, - 0x47, 0x04, 0x00, 0x00, 0x48, 0x04, 0x00, 0x00, 0x49, 0x04, 0x00, 0x00, - 0x4a, 0x04, 0x00, 0x00, 0x4b, 0x04, 0x00, 0x00, 0x4c, 0x04, 0x00, 0x00, - 0x4d, 0x04, 0x00, 0x00, 0x4e, 0x04, 0x00, 0x00, 0x4f, 0x04, 0x00, 0x00, - 0x50, 0x04, 0x00, 0x00, 0x51, 0x04, 0x00, 0x00, 0x52, 0x04, 0x00, 0x00, - 0x53, 0x04, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x55, 0x04, 0x00, 0x00, - 0x56, 0x04, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, 0x58, 0x04, 0x00, 0x00, - 0x59, 0x04, 0x00, 0x00, 0x5a, 0x04, 0x00, 0x00, 0x5b, 0x04, 0x00, 0x00, - 0x5c, 0x04, 0x00, 0x00, 0x5d, 0x04, 0x00, 0x00, 0x5e, 0x04, 0x00, 0x00, - 0x5f, 0x04, 0x00, 0x00, 0x60, 0x04, 0x00, 0x00, 0x61, 0x04, 0x00, 0x00, - 0x62, 0x04, 0x00, 0x00, 0x63, 0x04, 0x00, 0x00, 0x64, 0x04, 0x00, 0x00, - 0x65, 0x04, 0x00, 0x00, 0x66, 0x04, 0x00, 0x00, 0x67, 0x04, 0x00, 0x00, - 0x68, 0x04, 0x00, 0x00, 0x69, 0x04, 0x00, 0x00, 0x6a, 0x04, 0x00, 0x00, - 0x6b, 0x04, 0x00, 0x00, 0x6c, 0x04, 0x00, 0x00, 0x6d, 0x04, 0x00, 0x00, - 0x6e, 0x04, 0x00, 0x00, 0x6f, 0x04, 0x00, 0x00, 0x70, 0x04, 0x00, 0x00, - 0x71, 0x04, 0x00, 0x00, 0x72, 0x04, 0x00, 0x00, 0x73, 0x04, 0x00, 0x00, - 0x74, 0x04, 0x00, 0x00, 0x75, 0x04, 0x00, 0x00, 0x76, 0x04, 0x00, 0x00, - 0x77, 0x04, 0x00, 0x00, 0x78, 0x04, 0x00, 0x00, 0x79, 0x04, 0x00, 0x00, - 0x7a, 0x04, 0x00, 0x00, 0x7b, 0x04, 0x00, 0x00, 0x7c, 0x04, 0x00, 0x00, - 0x7d, 0x04, 0x00, 0x00, 0x7e, 0x04, 0x00, 0x00, 0x7f, 0x04, 0x00, 0x00, - 0x80, 0x04, 0x00, 0x00, 0x81, 0x04, 0x00, 0x00, 0x82, 0x04, 0x00, 0x00, - 0x83, 0x04, 0x00, 0x00, 0x84, 0x04, 0x00, 0x00, 0x85, 0x04, 0x00, 0x00, - 0x86, 0x04, 0x00, 0x00, 0x87, 0x04, 0x00, 0x00, 0x88, 0x04, 0x00, 0x00, - 0x89, 0x04, 0x00, 0x00, 0x8a, 0x04, 0x00, 0x00, 0x8b, 0x04, 0x00, 0x00, - 0x8c, 0x04, 0x00, 0x00, 0x8d, 0x04, 0x00, 0x00, 0x8e, 0x04, 0x00, 0x00, - 0x8f, 0x04, 0x00, 0x00, 0x90, 0x04, 0x00, 0x00, 0x91, 0x04, 0x00, 0x00, - 0x92, 0x04, 0x00, 0x00, 0x93, 0x04, 0x00, 0x00, 0x94, 0x04, 0x00, 0x00, - 0x95, 0x04, 0x00, 0x00, 0x96, 0x04, 0x00, 0x00, 0x97, 0x04, 0x00, 0x00, - 0x98, 0x04, 0x00, 0x00, 0x99, 0x04, 0x00, 0x00, 0x9a, 0x04, 0x00, 0x00, - 0x9b, 0x04, 0x00, 0x00, 0x9c, 0x04, 0x00, 0x00, 0x9d, 0x04, 0x00, 0x00, - 0x9e, 0x04, 0x00, 0x00, 0x9f, 0x04, 0x00, 0x00, 0xa0, 0x04, 0x00, 0x00, - 0xa1, 0x04, 0x00, 0x00, 0xa2, 0x04, 0x00, 0x00, 0xa3, 0x04, 0x00, 0x00, - 0xa4, 0x04, 0x00, 0x00, 0xa5, 0x04, 0x00, 0x00, 0xa6, 0x04, 0x00, 0x00, - 0xa7, 0x04, 0x00, 0x00, 0xa8, 0x04, 0x00, 0x00, 0xa9, 0x04, 0x00, 0x00, - 0xaa, 0x04, 0x00, 0x00, 0xab, 0x04, 0x00, 0x00, 0xac, 0x04, 0x00, 0x00, - 0xad, 0x04, 0x00, 0x00, 0xae, 0x04, 0x00, 0x00, 0xaf, 0x04, 0x00, 0x00, - 0xb0, 0x04, 0x00, 0x00, 0xb1, 0x04, 0x00, 0x00, 0xb2, 0x04, 0x00, 0x00, - 0xb3, 0x04, 0x00, 0x00, 0xb4, 0x04, 0x00, 0x00, 0xb5, 0x04, 0x00, 0x00, - 0xb6, 0x04, 0x00, 0x00, 0xb7, 0x04, 0x00, 0x00, 0xb8, 0x04, 0x00, 0x00, - 0xb9, 0x04, 0x00, 0x00, 0xba, 0x04, 0x00, 0x00, 0xbb, 0x04, 0x00, 0x00, - 0xbc, 0x04, 0x00, 0x00, 0xbd, 0x04, 0x00, 0x00, 0xbe, 0x04, 0x00, 0x00, - 0xbf, 0x04, 0x00, 0x00, 0xc0, 0x04, 0x00, 0x00, 0xc1, 0x04, 0x00, 0x00, - 0xc2, 0x04, 0x00, 0x00, 0xc3, 0x04, 0x00, 0x00, 0xc4, 0x04, 0x00, 0x00, - 0xc5, 0x04, 0x00, 0x00, 0xc6, 0x04, 0x00, 0x00, 0xc7, 0x04, 0x00, 0x00, - 0xc8, 0x04, 0x00, 0x00, 0xc9, 0x04, 0x00, 0x00, 0xca, 0x04, 0x00, 0x00, - 0xcb, 0x04, 0x00, 0x00, 0xcc, 0x04, 0x00, 0x00, 0xcd, 0x04, 0x00, 0x00, - 0xce, 0x04, 0x00, 0x00, 0xcf, 0x04, 0x00, 0x00, 0xd0, 0x04, 0x00, 0x00, - 0xd1, 0x04, 0x00, 0x00, 0xd2, 0x04, 0x00, 0x00, 0xd3, 0x04, 0x00, 0x00, - 0xd4, 0x04, 0x00, 0x00, 0xd5, 0x04, 0x00, 0x00, 0xd6, 0x04, 0x00, 0x00, - 0xd7, 0x04, 0x00, 0x00, 0xd8, 0x04, 0x00, 0x00, 0xd9, 0x04, 0x00, 0x00, - 0xda, 0x04, 0x00, 0x00, 0xdb, 0x04, 0x00, 0x00, 0xdc, 0x04, 0x00, 0x00, - 0xdd, 0x04, 0x00, 0x00, 0xde, 0x04, 0x00, 0x00, 0xdf, 0x04, 0x00, 0x00, - 0xe0, 0x04, 0x00, 0x00, 0xe1, 0x04, 0x00, 0x00, 0xe2, 0x04, 0x00, 0x00, - 0xe3, 0x04, 0x00, 0x00, 0xe4, 0x04, 0x00, 0x00, 0xe5, 0x04, 0x00, 0x00, - 0xe6, 0x04, 0x00, 0x00, 0xe7, 0x04, 0x00, 0x00, 0xe8, 0x04, 0x00, 0x00, - 0xe9, 0x04, 0x00, 0x00, 0xea, 0x04, 0x00, 0x00, 0xeb, 0x04, 0x00, 0x00, - 0xec, 0x04, 0x00, 0x00, 0xed, 0x04, 0x00, 0x00, 0xee, 0x04, 0x00, 0x00, - 0xef, 0x04, 0x00, 0x00, 0xf0, 0x04, 0x00, 0x00, 0xf1, 0x04, 0x00, 0x00, - 0xf2, 0x04, 0x00, 0x00, 0xf3, 0x04, 0x00, 0x00, 0xf4, 0x04, 0x00, 0x00, - 0xf5, 0x04, 0x00, 0x00, 0xf6, 0x04, 0x00, 0x00, 0xf7, 0x04, 0x00, 0x00, - 0xf8, 0x04, 0x00, 0x00, 0xf9, 0x04, 0x00, 0x00, 0xfa, 0x04, 0x00, 0x00, - 0xfb, 0x04, 0x00, 0x00, 0xfc, 0x04, 0x00, 0x00, 0xfd, 0x04, 0x00, 0x00, - 0xfe, 0x04, 0x00, 0x00, 0xff, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, - 0x01, 0x05, 0x00, 0x00, 0x02, 0x05, 0x00, 0x00, 0x03, 0x05, 0x00, 0x00, - 0x04, 0x05, 0x00, 0x00, 0x05, 0x05, 0x00, 0x00, 0x06, 0x05, 0x00, 0x00, - 0x07, 0x05, 0x00, 0x00, 0x08, 0x05, 0x00, 0x00, 0x09, 0x05, 0x00, 0x00, - 0x0a, 0x05, 0x00, 0x00, 0x0b, 0x05, 0x00, 0x00, 0x0c, 0x05, 0x00, 0x00, - 0x0d, 0x05, 0x00, 0x00, 0x0e, 0x05, 0x00, 0x00, 0x0f, 0x05, 0x00, 0x00, - 0x10, 0x05, 0x00, 0x00, 0x11, 0x05, 0x00, 0x00, 0x12, 0x05, 0x00, 0x00, - 0x13, 0x05, 0x00, 0x00, 0x14, 0x05, 0x00, 0x00, 0x15, 0x05, 0x00, 0x00, - 0x16, 0x05, 0x00, 0x00, 0x17, 0x05, 0x00, 0x00, 0x18, 0x05, 0x00, 0x00, - 0x19, 0x05, 0x00, 0x00, 0x1a, 0x05, 0x00, 0x00, 0x1b, 0x05, 0x00, 0x00, - 0x1c, 0x05, 0x00, 0x00, 0x1d, 0x05, 0x00, 0x00, 0x1e, 0x05, 0x00, 0x00, - 0x1f, 0x05, 0x00, 0x00, 0x20, 0x05, 0x00, 0x00, 0x21, 0x05, 0x00, 0x00, - 0x22, 0x05, 0x00, 0x00, 0x23, 0x05, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, - 0x25, 0x05, 0x00, 0x00, 0x26, 0x05, 0x00, 0x00, 0x27, 0x05, 0x00, 0x00, - 0x28, 0x05, 0x00, 0x00, 0x29, 0x05, 0x00, 0x00, 0x2a, 0x05, 0x00, 0x00, - 0x2b, 0x05, 0x00, 0x00, 0x2c, 0x05, 0x00, 0x00, 0x2d, 0x05, 0x00, 0x00, - 0x2e, 0x05, 0x00, 0x00, 0x2f, 0x05, 0x00, 0x00, 0x30, 0x05, 0x00, 0x00, - 0x31, 0x05, 0x00, 0x00, 0x32, 0x05, 0x00, 0x00, 0x33, 0x05, 0x00, 0x00, - 0x34, 0x05, 0x00, 0x00, 0x35, 0x05, 0x00, 0x00, 0x36, 0x05, 0x00, 0x00, - 0x37, 0x05, 0x00, 0x00, 0x38, 0x05, 0x00, 0x00, 0x39, 0x05, 0x00, 0x00, - 0x3a, 0x05, 0x00, 0x00, 0x3b, 0x05, 0x00, 0x00, 0x3c, 0x05, 0x00, 0x00, - 0x3d, 0x05, 0x00, 0x00, 0x3e, 0x05, 0x00, 0x00, 0x3f, 0x05, 0x00, 0x00, - 0x40, 0x05, 0x00, 0x00, 0x41, 0x05, 0x00, 0x00, 0x42, 0x05, 0x00, 0x00, - 0x43, 0x05, 0x00, 0x00, 0x44, 0x05, 0x00, 0x00, 0x45, 0x05, 0x00, 0x00, - 0x46, 0x05, 0x00, 0x00, 0x47, 0x05, 0x00, 0x00, 0x48, 0x05, 0x00, 0x00, - 0x49, 0x05, 0x00, 0x00, 0x4a, 0x05, 0x00, 0x00, 0x4b, 0x05, 0x00, 0x00, - 0x4c, 0x05, 0x00, 0x00, 0x4d, 0x05, 0x00, 0x00, 0x4e, 0x05, 0x00, 0x00, - 0x4f, 0x05, 0x00, 0x00, 0x50, 0x05, 0x00, 0x00, 0x51, 0x05, 0x00, 0x00, - 0x52, 0x05, 0x00, 0x00, 0x53, 0x05, 0x00, 0x00, 0x54, 0x05, 0x00, 0x00, - 0x55, 0x05, 0x00, 0x00, 0x56, 0x05, 0x00, 0x00, 0x57, 0x05, 0x00, 0x00, - 0x58, 0x05, 0x00, 0x00, 0x59, 0x05, 0x00, 0x00, 0x5a, 0x05, 0x00, 0x00, - 0x5b, 0x05, 0x00, 0x00, 0x5c, 0x05, 0x00, 0x00, 0x5d, 0x05, 0x00, 0x00, - 0x5e, 0x05, 0x00, 0x00, 0x5f, 0x05, 0x00, 0x00, 0x60, 0x05, 0x00, 0x00, - 0x61, 0x05, 0x00, 0x00, 0x62, 0x05, 0x00, 0x00, 0x63, 0x05, 0x00, 0x00, - 0x64, 0x05, 0x00, 0x00, 0x65, 0x05, 0x00, 0x00, 0x66, 0x05, 0x00, 0x00, - 0x67, 0x05, 0x00, 0x00, 0x68, 0x05, 0x00, 0x00, 0x69, 0x05, 0x00, 0x00, - 0x6a, 0x05, 0x00, 0x00, 0x6b, 0x05, 0x00, 0x00, 0x6c, 0x05, 0x00, 0x00, - 0x6d, 0x05, 0x00, 0x00, 0x6e, 0x05, 0x00, 0x00, 0x6f, 0x05, 0x00, 0x00, - 0x70, 0x05, 0x00, 0x00, 0x71, 0x05, 0x00, 0x00, 0x72, 0x05, 0x00, 0x00, - 0x73, 0x05, 0x00, 0x00, 0x74, 0x05, 0x00, 0x00, 0x75, 0x05, 0x00, 0x00, - 0x76, 0x05, 0x00, 0x00, 0x77, 0x05, 0x00, 0x00, 0x78, 0x05, 0x00, 0x00, - 0x79, 0x05, 0x00, 0x00, 0x7a, 0x05, 0x00, 0x00, 0x7b, 0x05, 0x00, 0x00, - 0x7c, 0x05, 0x00, 0x00, 0x7d, 0x05, 0x00, 0x00, 0x7e, 0x05, 0x00, 0x00, - 0x7f, 0x05, 0x00, 0x00, 0x80, 0x05, 0x00, 0x00, 0x81, 0x05, 0x00, 0x00, - 0x82, 0x05, 0x00, 0x00, 0x83, 0x05, 0x00, 0x00, 0x84, 0x05, 0x00, 0x00, - 0x85, 0x05, 0x00, 0x00, 0x86, 0x05, 0x00, 0x00, 0x87, 0x05, 0x00, 0x00, - 0x88, 0x05, 0x00, 0x00, 0x89, 0x05, 0x00, 0x00, 0x8a, 0x05, 0x00, 0x00, - 0x8b, 0x05, 0x00, 0x00, 0x8c, 0x05, 0x00, 0x00, 0x8d, 0x05, 0x00, 0x00, - 0x8e, 0x05, 0x00, 0x00, 0x8f, 0x05, 0x00, 0x00, 0x90, 0x05, 0x00, 0x00, - 0x91, 0x05, 0x00, 0x00, 0x92, 0x05, 0x00, 0x00, 0x93, 0x05, 0x00, 0x00, - 0x94, 0x05, 0x00, 0x00, 0x95, 0x05, 0x00, 0x00, 0x96, 0x05, 0x00, 0x00, - 0x97, 0x05, 0x00, 0x00, 0x98, 0x05, 0x00, 0x00, 0x99, 0x05, 0x00, 0x00, - 0x9a, 0x05, 0x00, 0x00, 0x9b, 0x05, 0x00, 0x00, 0x9c, 0x05, 0x00, 0x00, - 0x9d, 0x05, 0x00, 0x00, 0x9e, 0x05, 0x00, 0x00, 0x9f, 0x05, 0x00, 0x00, - 0xa0, 0x05, 0x00, 0x00, 0xa1, 0x05, 0x00, 0x00, 0xa2, 0x05, 0x00, 0x00, - 0xa3, 0x05, 0x00, 0x00, 0xa4, 0x05, 0x00, 0x00, 0xa5, 0x05, 0x00, 0x00, - 0xa6, 0x05, 0x00, 0x00, 0xa7, 0x05, 0x00, 0x00, 0xa8, 0x05, 0x00, 0x00, - 0xa9, 0x05, 0x00, 0x00, 0xaa, 0x05, 0x00, 0x00, 0xab, 0x05, 0x00, 0x00, - 0xac, 0x05, 0x00, 0x00, 0xad, 0x05, 0x00, 0x00, 0xae, 0x05, 0x00, 0x00, - 0xaf, 0x05, 0x00, 0x00, 0xb0, 0x05, 0x00, 0x00, 0xb1, 0x05, 0x00, 0x00, - 0xb2, 0x05, 0x00, 0x00, 0xb3, 0x05, 0x00, 0x00, 0xb4, 0x05, 0x00, 0x00, - 0xb5, 0x05, 0x00, 0x00, 0xb6, 0x05, 0x00, 0x00, 0xb7, 0x05, 0x00, 0x00, - 0xb8, 0x05, 0x00, 0x00, 0xb9, 0x05, 0x00, 0x00, 0xba, 0x05, 0x00, 0x00, - 0xbb, 0x05, 0x00, 0x00, 0xbc, 0x05, 0x00, 0x00, 0xbd, 0x05, 0x00, 0x00, - 0xbe, 0x05, 0x00, 0x00, 0xbf, 0x05, 0x00, 0x00, 0xc0, 0x05, 0x00, 0x00, - 0xc1, 0x05, 0x00, 0x00, 0xc2, 0x05, 0x00, 0x00, 0xc3, 0x05, 0x00, 0x00, - 0xc4, 0x05, 0x00, 0x00, 0xc5, 0x05, 0x00, 0x00, 0xc6, 0x05, 0x00, 0x00, - 0xc7, 0x05, 0x00, 0x00, 0xc8, 0x05, 0x00, 0x00, 0xc9, 0x05, 0x00, 0x00, - 0xca, 0x05, 0x00, 0x00, 0xcb, 0x05, 0x00, 0x00, 0xcc, 0x05, 0x00, 0x00, - 0xcd, 0x05, 0x00, 0x00, 0xce, 0x05, 0x00, 0x00, 0xcf, 0x05, 0x00, 0x00, - 0xd0, 0x05, 0x00, 0x00, 0xd1, 0x05, 0x00, 0x00, 0xd2, 0x05, 0x00, 0x00, - 0xd3, 0x05, 0x00, 0x00, 0xd4, 0x05, 0x00, 0x00, 0xd5, 0x05, 0x00, 0x00, - 0xd6, 0x05, 0x00, 0x00, 0xd7, 0x05, 0x00, 0x00, 0xd8, 0x05, 0x00, 0x00, - 0xd9, 0x05, 0x00, 0x00, 0xda, 0x05, 0x00, 0x00, 0xdb, 0x05, 0x00, 0x00, - 0xdc, 0x05, 0x00, 0x00, 0xdd, 0x05, 0x00, 0x00, 0xde, 0x05, 0x00, 0x00, - 0xdf, 0x05, 0x00, 0x00, 0xe0, 0x05, 0x00, 0x00, 0xe1, 0x05, 0x00, 0x00, - 0xe2, 0x05, 0x00, 0x00, 0xe3, 0x05, 0x00, 0x00, 0xe4, 0x05, 0x00, 0x00, - 0xe5, 0x05, 0x00, 0x00, 0xe6, 0x05, 0x00, 0x00, 0xe7, 0x05, 0x00, 0x00, - 0xe8, 0x05, 0x00, 0x00, 0xe9, 0x05, 0x00, 0x00, 0xea, 0x05, 0x00, 0x00, - 0xeb, 0x05, 0x00, 0x00, 0xec, 0x05, 0x00, 0x00, 0xed, 0x05, 0x00, 0x00, - 0xee, 0x05, 0x00, 0x00, 0xef, 0x05, 0x00, 0x00, 0xf0, 0x05, 0x00, 0x00, - 0xf1, 0x05, 0x00, 0x00, 0xf2, 0x05, 0x00, 0x00, 0xf3, 0x05, 0x00, 0x00, - 0xf4, 0x05, 0x00, 0x00, 0xf5, 0x05, 0x00, 0x00, 0xf6, 0x05, 0x00, 0x00, - 0xf7, 0x05, 0x00, 0x00, 0xf8, 0x05, 0x00, 0x00, 0xf9, 0x05, 0x00, 0x00, - 0xfa, 0x05, 0x00, 0x00, 0xfb, 0x05, 0x00, 0x00, 0xfc, 0x05, 0x00, 0x00, - 0xfd, 0x05, 0x00, 0x00, 0xfe, 0x05, 0x00, 0x00, 0xff, 0x05, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x00, 0x01, 0x06, 0x00, 0x00, 0x02, 0x06, 0x00, 0x00, - 0x03, 0x06, 0x00, 0x00, 0x04, 0x06, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, - 0x06, 0x06, 0x00, 0x00, 0x07, 0x06, 0x00, 0x00, 0x08, 0x06, 0x00, 0x00, - 0x09, 0x06, 0x00, 0x00, 0x0a, 0x06, 0x00, 0x00, 0x0b, 0x06, 0x00, 0x00, - 0x0c, 0x06, 0x00, 0x00, 0x0d, 0x06, 0x00, 0x00, 0x0e, 0x06, 0x00, 0x00, - 0x0f, 0x06, 0x00, 0x00, 0x10, 0x06, 0x00, 0x00, 0x11, 0x06, 0x00, 0x00, - 0x12, 0x06, 0x00, 0x00, 0x13, 0x06, 0x00, 0x00, 0x14, 0x06, 0x00, 0x00, - 0x15, 0x06, 0x00, 0x00, 0x16, 0x06, 0x00, 0x00, 0x17, 0x06, 0x00, 0x00, - 0x18, 0x06, 0x00, 0x00, 0x19, 0x06, 0x00, 0x00, 0x1a, 0x06, 0x00, 0x00, - 0x1b, 0x06, 0x00, 0x00, 0x1c, 0x06, 0x00, 0x00, 0x1d, 0x06, 0x00, 0x00, - 0x1e, 0x06, 0x00, 0x00, 0x1f, 0x06, 0x00, 0x00, 0x20, 0x06, 0x00, 0x00, - 0x21, 0x06, 0x00, 0x00, 0x22, 0x06, 0x00, 0x00, 0x23, 0x06, 0x00, 0x00, - 0x24, 0x06, 0x00, 0x00, 0x25, 0x06, 0x00, 0x00, 0x26, 0x06, 0x00, 0x00, - 0x27, 0x06, 0x00, 0x00, 0x28, 0x06, 0x00, 0x00, 0x29, 0x06, 0x00, 0x00, - 0x2a, 0x06, 0x00, 0x00, 0x2b, 0x06, 0x00, 0x00, 0x2c, 0x06, 0x00, 0x00, - 0x2d, 0x06, 0x00, 0x00, 0x2e, 0x06, 0x00, 0x00, 0x2f, 0x06, 0x00, 0x00, - 0x30, 0x06, 0x00, 0x00, 0x31, 0x06, 0x00, 0x00, 0x32, 0x06, 0x00, 0x00, - 0x33, 0x06, 0x00, 0x00, 0x34, 0x06, 0x00, 0x00, 0x35, 0x06, 0x00, 0x00, - 0x36, 0x06, 0x00, 0x00, 0x37, 0x06, 0x00, 0x00, 0x38, 0x06, 0x00, 0x00, - 0x39, 0x06, 0x00, 0x00, 0x3a, 0x06, 0x00, 0x00, 0x3b, 0x06, 0x00, 0x00, - 0x3c, 0x06, 0x00, 0x00, 0x3d, 0x06, 0x00, 0x00, 0x3e, 0x06, 0x00, 0x00, - 0x3f, 0x06, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, 0x41, 0x06, 0x00, 0x00, - 0x42, 0x06, 0x00, 0x00, 0x43, 0x06, 0x00, 0x00, 0x44, 0x06, 0x00, 0x00, - 0x45, 0x06, 0x00, 0x00, 0x46, 0x06, 0x00, 0x00, 0x47, 0x06, 0x00, 0x00, - 0x48, 0x06, 0x00, 0x00, 0x49, 0x06, 0x00, 0x00, 0x4a, 0x06, 0x00, 0x00, - 0x4b, 0x06, 0x00, 0x00, 0x4c, 0x06, 0x00, 0x00, 0x4d, 0x06, 0x00, 0x00, - 0x4e, 0x06, 0x00, 0x00, 0x4f, 0x06, 0x00, 0x00, 0x50, 0x06, 0x00, 0x00, - 0x51, 0x06, 0x00, 0x00, 0x52, 0x06, 0x00, 0x00, 0x53, 0x06, 0x00, 0x00, - 0x54, 0x06, 0x00, 0x00, 0x55, 0x06, 0x00, 0x00, 0x56, 0x06, 0x00, 0x00, - 0x57, 0x06, 0x00, 0x00, 0x58, 0x06, 0x00, 0x00, 0x59, 0x06, 0x00, 0x00, - 0x5a, 0x06, 0x00, 0x00, 0x5b, 0x06, 0x00, 0x00, 0x5c, 0x06, 0x00, 0x00, - 0x5d, 0x06, 0x00, 0x00, 0x5e, 0x06, 0x00, 0x00, 0x5f, 0x06, 0x00, 0x00, - 0x60, 0x06, 0x00, 0x00, 0x61, 0x06, 0x00, 0x00, 0x62, 0x06, 0x00, 0x00, - 0x63, 0x06, 0x00, 0x00, 0x64, 0x06, 0x00, 0x00, 0x65, 0x06, 0x00, 0x00, - 0x66, 0x06, 0x00, 0x00, 0x67, 0x06, 0x00, 0x00, 0x68, 0x06, 0x00, 0x00, - 0x69, 0x06, 0x00, 0x00, 0x6a, 0x06, 0x00, 0x00, 0x6b, 0x06, 0x00, 0x00, - 0x6c, 0x06, 0x00, 0x00, 0x6d, 0x06, 0x00, 0x00, 0x6e, 0x06, 0x00, 0x00, - 0x6f, 0x06, 0x00, 0x00, 0x70, 0x06, 0x00, 0x00, 0x71, 0x06, 0x00, 0x00, - 0x72, 0x06, 0x00, 0x00, 0x73, 0x06, 0x00, 0x00, 0x74, 0x06, 0x00, 0x00, - 0x75, 0x06, 0x00, 0x00, 0x76, 0x06, 0x00, 0x00, 0x77, 0x06, 0x00, 0x00, - 0x78, 0x06, 0x00, 0x00, 0x79, 0x06, 0x00, 0x00, 0x7a, 0x06, 0x00, 0x00, - 0x7b, 0x06, 0x00, 0x00, 0x7c, 0x06, 0x00, 0x00, 0x7d, 0x06, 0x00, 0x00, - 0x7e, 0x06, 0x00, 0x00, 0x7f, 0x06, 0x00, 0x00, 0x80, 0x06, 0x00, 0x00, - 0x81, 0x06, 0x00, 0x00, 0x82, 0x06, 0x00, 0x00, 0x83, 0x06, 0x00, 0x00, - 0x84, 0x06, 0x00, 0x00, 0x85, 0x06, 0x00, 0x00, 0x86, 0x06, 0x00, 0x00, - 0x87, 0x06, 0x00, 0x00, 0x88, 0x06, 0x00, 0x00, 0x89, 0x06, 0x00, 0x00, - 0x8a, 0x06, 0x00, 0x00, 0x8b, 0x06, 0x00, 0x00, 0x8c, 0x06, 0x00, 0x00, - 0x8d, 0x06, 0x00, 0x00, 0x8e, 0x06, 0x00, 0x00, 0x8f, 0x06, 0x00, 0x00, - 0x90, 0x06, 0x00, 0x00, 0x91, 0x06, 0x00, 0x00, 0x92, 0x06, 0x00, 0x00, - 0x93, 0x06, 0x00, 0x00, 0x94, 0x06, 0x00, 0x00, 0x95, 0x06, 0x00, 0x00, - 0x96, 0x06, 0x00, 0x00, 0x97, 0x06, 0x00, 0x00, 0x98, 0x06, 0x00, 0x00, - 0x99, 0x06, 0x00, 0x00, 0x9a, 0x06, 0x00, 0x00, 0x9b, 0x06, 0x00, 0x00, - 0x9c, 0x06, 0x00, 0x00, 0x9d, 0x06, 0x00, 0x00, 0x9e, 0x06, 0x00, 0x00, - 0x9f, 0x06, 0x00, 0x00, 0xa0, 0x06, 0x00, 0x00, 0xa1, 0x06, 0x00, 0x00, - 0xa2, 0x06, 0x00, 0x00, 0xa3, 0x06, 0x00, 0x00, 0xa4, 0x06, 0x00, 0x00, - 0xa5, 0x06, 0x00, 0x00, 0xa6, 0x06, 0x00, 0x00, 0xa7, 0x06, 0x00, 0x00, - 0xa8, 0x06, 0x00, 0x00, 0xa9, 0x06, 0x00, 0x00, 0xaa, 0x06, 0x00, 0x00, - 0xab, 0x06, 0x00, 0x00, 0xac, 0x06, 0x00, 0x00, 0xad, 0x06, 0x00, 0x00, - 0xae, 0x06, 0x00, 0x00, 0xaf, 0x06, 0x00, 0x00, 0xb0, 0x06, 0x00, 0x00, - 0xb1, 0x06, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0xb3, 0x06, 0x00, 0x00, - 0xb4, 0x06, 0x00, 0x00, 0xb5, 0x06, 0x00, 0x00, 0xb6, 0x06, 0x00, 0x00, - 0xb7, 0x06, 0x00, 0x00, 0xb8, 0x06, 0x00, 0x00, 0xb9, 0x06, 0x00, 0x00, - 0xba, 0x06, 0x00, 0x00, 0xbb, 0x06, 0x00, 0x00, 0xbc, 0x06, 0x00, 0x00, - 0xbd, 0x06, 0x00, 0x00, 0xbe, 0x06, 0x00, 0x00, 0xbf, 0x06, 0x00, 0x00, - 0xc0, 0x06, 0x00, 0x00, 0xc1, 0x06, 0x00, 0x00, 0xc2, 0x06, 0x00, 0x00, - 0xc3, 0x06, 0x00, 0x00, 0xc4, 0x06, 0x00, 0x00, 0xc5, 0x06, 0x00, 0x00, - 0xc6, 0x06, 0x00, 0x00, 0xc7, 0x06, 0x00, 0x00, 0xc8, 0x06, 0x00, 0x00, - 0xc9, 0x06, 0x00, 0x00, 0xca, 0x06, 0x00, 0x00, 0xcb, 0x06, 0x00, 0x00, - 0xcc, 0x06, 0x00, 0x00, 0xcd, 0x06, 0x00, 0x00, 0xce, 0x06, 0x00, 0x00, - 0xcf, 0x06, 0x00, 0x00, 0xd0, 0x06, 0x00, 0x00, 0xd1, 0x06, 0x00, 0x00, - 0xd2, 0x06, 0x00, 0x00, 0xd3, 0x06, 0x00, 0x00, 0xd4, 0x06, 0x00, 0x00, - 0xd5, 0x06, 0x00, 0x00, 0xd6, 0x06, 0x00, 0x00, 0xd7, 0x06, 0x00, 0x00, - 0xd8, 0x06, 0x00, 0x00, 0xd9, 0x06, 0x00, 0x00, 0xda, 0x06, 0x00, 0x00, - 0xdb, 0x06, 0x00, 0x00, 0xdc, 0x06, 0x00, 0x00, 0xdd, 0x06, 0x00, 0x00, - 0xde, 0x06, 0x00, 0x00, 0xdf, 0x06, 0x00, 0x00, 0xe0, 0x06, 0x00, 0x00, - 0xe1, 0x06, 0x00, 0x00, 0xe2, 0x06, 0x00, 0x00, 0xe3, 0x06, 0x00, 0x00, - 0xe4, 0x06, 0x00, 0x00, 0xe5, 0x06, 0x00, 0x00, 0xe6, 0x06, 0x00, 0x00, - 0xe7, 0x06, 0x00, 0x00, 0xe8, 0x06, 0x00, 0x00, 0xe9, 0x06, 0x00, 0x00, - 0xea, 0x06, 0x00, 0x00, 0xeb, 0x06, 0x00, 0x00, 0xec, 0x06, 0x00, 0x00, - 0xed, 0x06, 0x00, 0x00, 0xee, 0x06, 0x00, 0x00, 0xef, 0x06, 0x00, 0x00, - 0xf0, 0x06, 0x00, 0x00, 0xf1, 0x06, 0x00, 0x00, 0xf2, 0x06, 0x00, 0x00, - 0xf3, 0x06, 0x00, 0x00, 0xf4, 0x06, 0x00, 0x00, 0xf5, 0x06, 0x00, 0x00, - 0xf6, 0x06, 0x00, 0x00, 0xf7, 0x06, 0x00, 0x00, 0xf8, 0x06, 0x00, 0x00, - 0xf9, 0x06, 0x00, 0x00, 0xfa, 0x06, 0x00, 0x00, 0xfb, 0x06, 0x00, 0x00, - 0xfc, 0x06, 0x00, 0x00, 0xfd, 0x06, 0x00, 0x00, 0xfe, 0x06, 0x00, 0x00, - 0xff, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x01, 0x07, 0x00, 0x00, - 0x02, 0x07, 0x00, 0x00, 0x03, 0x07, 0x00, 0x00, 0x04, 0x07, 0x00, 0x00, - 0x05, 0x07, 0x00, 0x00, 0x06, 0x07, 0x00, 0x00, 0x07, 0x07, 0x00, 0x00, - 0x08, 0x07, 0x00, 0x00, 0x09, 0x07, 0x00, 0x00, 0x0a, 0x07, 0x00, 0x00, - 0x0b, 0x07, 0x00, 0x00, 0x0c, 0x07, 0x00, 0x00, 0x0d, 0x07, 0x00, 0x00, - 0x0e, 0x07, 0x00, 0x00, 0x0f, 0x07, 0x00, 0x00, 0x10, 0x07, 0x00, 0x00, - 0x11, 0x07, 0x00, 0x00, 0x12, 0x07, 0x00, 0x00, 0x13, 0x07, 0x00, 0x00, - 0x14, 0x07, 0x00, 0x00, 0x15, 0x07, 0x00, 0x00, 0x16, 0x07, 0x00, 0x00, - 0x17, 0x07, 0x00, 0x00, 0x18, 0x07, 0x00, 0x00, 0x19, 0x07, 0x00, 0x00, - 0x1a, 0x07, 0x00, 0x00, 0x1b, 0x07, 0x00, 0x00, 0x1c, 0x07, 0x00, 0x00, - 0x1d, 0x07, 0x00, 0x00, 0x1e, 0x07, 0x00, 0x00, 0x1f, 0x07, 0x00, 0x00, - 0x20, 0x07, 0x00, 0x00, 0x21, 0x07, 0x00, 0x00, 0x22, 0x07, 0x00, 0x00, - 0x23, 0x07, 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, 0x25, 0x07, 0x00, 0x00, - 0x26, 0x07, 0x00, 0x00, 0x27, 0x07, 0x00, 0x00, 0x28, 0x07, 0x00, 0x00, - 0x29, 0x07, 0x00, 0x00, 0x2a, 0x07, 0x00, 0x00, 0x2b, 0x07, 0x00, 0x00, - 0x2c, 0x07, 0x00, 0x00, 0x2d, 0x07, 0x00, 0x00, 0x2e, 0x07, 0x00, 0x00, - 0x2f, 0x07, 0x00, 0x00, 0x30, 0x07, 0x00, 0x00, 0x31, 0x07, 0x00, 0x00, - 0x32, 0x07, 0x00, 0x00, 0x33, 0x07, 0x00, 0x00, 0x34, 0x07, 0x00, 0x00, - 0x35, 0x07, 0x00, 0x00, 0x36, 0x07, 0x00, 0x00, 0x37, 0x07, 0x00, 0x00, - 0x38, 0x07, 0x00, 0x00, 0x39, 0x07, 0x00, 0x00, 0x3a, 0x07, 0x00, 0x00, - 0x3b, 0x07, 0x00, 0x00, 0x3c, 0x07, 0x00, 0x00, 0x3d, 0x07, 0x00, 0x00, - 0x3e, 0x07, 0x00, 0x00, 0x3f, 0x07, 0x00, 0x00, 0x40, 0x07, 0x00, 0x00, - 0x41, 0x07, 0x00, 0x00, 0x42, 0x07, 0x00, 0x00, 0x43, 0x07, 0x00, 0x00, - 0x44, 0x07, 0x00, 0x00, 0x45, 0x07, 0x00, 0x00, 0x46, 0x07, 0x00, 0x00, - 0x47, 0x07, 0x00, 0x00, 0x48, 0x07, 0x00, 0x00, 0x49, 0x07, 0x00, 0x00, - 0x4a, 0x07, 0x00, 0x00, 0x4b, 0x07, 0x00, 0x00, 0x4c, 0x07, 0x00, 0x00, - 0x4d, 0x07, 0x00, 0x00, 0x4e, 0x07, 0x00, 0x00, 0x4f, 0x07, 0x00, 0x00, - 0x50, 0x07, 0x00, 0x00, 0x51, 0x07, 0x00, 0x00, 0x52, 0x07, 0x00, 0x00, - 0x53, 0x07, 0x00, 0x00, 0x54, 0x07, 0x00, 0x00, 0x55, 0x07, 0x00, 0x00, - 0x56, 0x07, 0x00, 0x00, 0x57, 0x07, 0x00, 0x00, 0x58, 0x07, 0x00, 0x00, - 0x59, 0x07, 0x00, 0x00, 0x5a, 0x07, 0x00, 0x00, 0x5b, 0x07, 0x00, 0x00, - 0x5c, 0x07, 0x00, 0x00, 0x5d, 0x07, 0x00, 0x00, 0x5e, 0x07, 0x00, 0x00, - 0x5f, 0x07, 0x00, 0x00, 0x60, 0x07, 0x00, 0x00, 0x61, 0x07, 0x00, 0x00, - 0x62, 0x07, 0x00, 0x00, 0x63, 0x07, 0x00, 0x00, 0x64, 0x07, 0x00, 0x00, - 0x65, 0x07, 0x00, 0x00, 0x66, 0x07, 0x00, 0x00, 0x67, 0x07, 0x00, 0x00, - 0x68, 0x07, 0x00, 0x00, 0x69, 0x07, 0x00, 0x00, 0x6a, 0x07, 0x00, 0x00, - 0x6b, 0x07, 0x00, 0x00, 0x6c, 0x07, 0x00, 0x00, 0x6d, 0x07, 0x00, 0x00, - 0x6e, 0x07, 0x00, 0x00, 0x6f, 0x07, 0x00, 0x00, 0x70, 0x07, 0x00, 0x00, - 0x71, 0x07, 0x00, 0x00, 0x72, 0x07, 0x00, 0x00, 0x73, 0x07, 0x00, 0x00, - 0x74, 0x07, 0x00, 0x00, 0x75, 0x07, 0x00, 0x00, 0x76, 0x07, 0x00, 0x00, - 0x77, 0x07, 0x00, 0x00, 0x78, 0x07, 0x00, 0x00, 0x79, 0x07, 0x00, 0x00, - 0x7a, 0x07, 0x00, 0x00, 0x7b, 0x07, 0x00, 0x00, 0x7c, 0x07, 0x00, 0x00, - 0x7d, 0x07, 0x00, 0x00, 0x7e, 0x07, 0x00, 0x00, 0x7f, 0x07, 0x00, 0x00, - 0x80, 0x07, 0x00, 0x00, 0x81, 0x07, 0x00, 0x00, 0x82, 0x07, 0x00, 0x00, - 0x83, 0x07, 0x00, 0x00, 0x84, 0x07, 0x00, 0x00, 0x85, 0x07, 0x00, 0x00, - 0x86, 0x07, 0x00, 0x00, 0x87, 0x07, 0x00, 0x00, 0x88, 0x07, 0x00, 0x00, - 0x89, 0x07, 0x00, 0x00, 0x8a, 0x07, 0x00, 0x00, 0x8b, 0x07, 0x00, 0x00, - 0x8c, 0x07, 0x00, 0x00, 0x8d, 0x07, 0x00, 0x00, 0x8e, 0x07, 0x00, 0x00, - 0x8f, 0x07, 0x00, 0x00, 0x90, 0x07, 0x00, 0x00, 0x91, 0x07, 0x00, 0x00, - 0x92, 0x07, 0x00, 0x00, 0x93, 0x07, 0x00, 0x00, 0x94, 0x07, 0x00, 0x00, - 0x95, 0x07, 0x00, 0x00, 0x96, 0x07, 0x00, 0x00, 0x97, 0x07, 0x00, 0x00, - 0x98, 0x07, 0x00, 0x00, 0x99, 0x07, 0x00, 0x00, 0x9a, 0x07, 0x00, 0x00, - 0x9b, 0x07, 0x00, 0x00, 0x9c, 0x07, 0x00, 0x00, 0x9d, 0x07, 0x00, 0x00, - 0x9e, 0x07, 0x00, 0x00, 0x9f, 0x07, 0x00, 0x00, 0xa0, 0x07, 0x00, 0x00, - 0xa1, 0x07, 0x00, 0x00, 0xa2, 0x07, 0x00, 0x00, 0xa3, 0x07, 0x00, 0x00, - 0xa4, 0x07, 0x00, 0x00, 0xa5, 0x07, 0x00, 0x00, 0xa6, 0x07, 0x00, 0x00, - 0xa7, 0x07, 0x00, 0x00, 0xa8, 0x07, 0x00, 0x00, 0xa9, 0x07, 0x00, 0x00, - 0xaa, 0x07, 0x00, 0x00, 0xab, 0x07, 0x00, 0x00, 0xac, 0x07, 0x00, 0x00, - 0xad, 0x07, 0x00, 0x00, 0xae, 0x07, 0x00, 0x00, 0xaf, 0x07, 0x00, 0x00, - 0xb0, 0x07, 0x00, 0x00, 0xb1, 0x07, 0x00, 0x00, 0xb2, 0x07, 0x00, 0x00, - 0xb3, 0x07, 0x00, 0x00, 0xb4, 0x07, 0x00, 0x00, 0xb5, 0x07, 0x00, 0x00, - 0xb6, 0x07, 0x00, 0x00, 0xb7, 0x07, 0x00, 0x00, 0xb8, 0x07, 0x00, 0x00, - 0xb9, 0x07, 0x00, 0x00, 0xba, 0x07, 0x00, 0x00, 0xbb, 0x07, 0x00, 0x00, - 0xbc, 0x07, 0x00, 0x00, 0xbd, 0x07, 0x00, 0x00, 0xbe, 0x07, 0x00, 0x00, - 0xbf, 0x07, 0x00, 0x00, 0xc0, 0x07, 0x00, 0x00, 0xc1, 0x07, 0x00, 0x00, - 0xc2, 0x07, 0x00, 0x00, 0xc3, 0x07, 0x00, 0x00, 0xc4, 0x07, 0x00, 0x00, - 0xc5, 0x07, 0x00, 0x00, 0xc6, 0x07, 0x00, 0x00, 0xc7, 0x07, 0x00, 0x00, - 0xc8, 0x07, 0x00, 0x00, 0xc9, 0x07, 0x00, 0x00, 0xca, 0x07, 0x00, 0x00, - 0xcb, 0x07, 0x00, 0x00, 0xcc, 0x07, 0x00, 0x00, 0xcd, 0x07, 0x00, 0x00, - 0xce, 0x07, 0x00, 0x00, 0xcf, 0x07, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, - 0xd1, 0x07, 0x00, 0x00, 0xd2, 0x07, 0x00, 0x00, 0xd3, 0x07, 0x00, 0x00, - 0xd4, 0x07, 0x00, 0x00, 0xd5, 0x07, 0x00, 0x00, 0xd6, 0x07, 0x00, 0x00, - 0xd7, 0x07, 0x00, 0x00, 0xd8, 0x07, 0x00, 0x00, 0xd9, 0x07, 0x00, 0x00, - 0xda, 0x07, 0x00, 0x00, 0xdb, 0x07, 0x00, 0x00, 0xdc, 0x07, 0x00, 0x00, - 0xdd, 0x07, 0x00, 0x00, 0xde, 0x07, 0x00, 0x00, 0xdf, 0x07, 0x00, 0x00, - 0xe0, 0x07, 0x00, 0x00, 0xe1, 0x07, 0x00, 0x00, 0xe2, 0x07, 0x00, 0x00, - 0xe3, 0x07, 0x00, 0x00, 0xe4, 0x07, 0x00, 0x00, 0xe5, 0x07, 0x00, 0x00, - 0xe6, 0x07, 0x00, 0x00, 0xe7, 0x07, 0x00, 0x00, 0xe8, 0x07, 0x00, 0x00, - 0xe9, 0x07, 0x00, 0x00, 0xea, 0x07, 0x00, 0x00, 0xeb, 0x07, 0x00, 0x00, - 0xec, 0x07, 0x00, 0x00, 0xed, 0x07, 0x00, 0x00, 0xee, 0x07, 0x00, 0x00, - 0xef, 0x07, 0x00, 0x00, 0xf0, 0x07, 0x00, 0x00, 0xf1, 0x07, 0x00, 0x00, - 0xf2, 0x07, 0x00, 0x00, 0xf3, 0x07, 0x00, 0x00, 0xf4, 0x07, 0x00, 0x00, - 0xf5, 0x07, 0x00, 0x00, 0xf6, 0x07, 0x00, 0x00, 0xf7, 0x07, 0x00, 0x00, - 0xf8, 0x07, 0x00, 0x00, 0xf9, 0x07, 0x00, 0x00, 0xfa, 0x07, 0x00, 0x00, - 0xfb, 0x07, 0x00, 0x00, 0xfc, 0x07, 0x00, 0x00, 0xfd, 0x07, 0x00, 0x00, - 0xfe, 0x07, 0x00, 0x00, 0xff, 0x07, 0x00, 0x00 -}; + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, + 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x15, + 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x19, 0x00, + 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, + 0x00, 0x1e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x26, + 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x2a, 0x00, + 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, + 0x00, 0x2f, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x37, + 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x3b, 0x00, + 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, + 0x00, 0x40, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, + 0x44, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x48, + 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x4c, 0x00, + 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x4e, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, + 0x00, 0x51, 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, + 0x55, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x59, + 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x5d, 0x00, + 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, + 0x00, 0x62, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, + 0x66, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x6a, + 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x6e, 0x00, + 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, + 0x00, 0x73, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, + 0x77, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x7a, 0x00, 0x00, 0x00, 0x7b, + 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x7f, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, + 0x00, 0x84, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, + 0x88, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x8c, + 0x00, 0x00, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, 0x90, 0x00, + 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, + 0x99, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x9b, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x9d, + 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, 0xa1, 0x00, + 0x00, 0x00, 0xa2, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, + 0x00, 0xa6, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, + 0xaa, 0x00, 0x00, 0x00, 0xab, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0xae, + 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0xb1, 0x00, 0x00, 0x00, 0xb2, 0x00, + 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0xb5, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, + 0x00, 0xb7, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, + 0xbb, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, 0xbd, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0xbf, + 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, 0xc3, 0x00, + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, + 0x00, 0xc8, 0x00, 0x00, 0x00, 0xc9, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0xcb, 0x00, 0x00, 0x00, + 0xcc, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0x00, 0xd0, + 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, 0x00, 0xd2, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0xd4, 0x00, + 0x00, 0x00, 0xd5, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, + 0x00, 0xd9, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, + 0xdd, 0x00, 0x00, 0x00, 0xde, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xe1, + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0xe5, 0x00, + 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, 0xe7, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0xe9, 0x00, 0x00, + 0x00, 0xea, 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0xec, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, + 0xee, 0x00, 0x00, 0x00, 0xef, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0xf2, + 0x00, 0x00, 0x00, 0xf3, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x00, 0x00, 0xf6, 0x00, + 0x00, 0x00, 0xf7, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, + 0x00, 0xfb, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0x00, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x03, + 0x01, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x07, 0x01, + 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x0b, 0x01, 0x00, + 0x00, 0x0c, 0x01, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x0e, 0x01, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, + 0x10, 0x01, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x14, + 0x01, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0x17, 0x01, 0x00, 0x00, 0x18, 0x01, + 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0x1a, 0x01, 0x00, 0x00, 0x1b, 0x01, 0x00, 0x00, 0x1c, 0x01, 0x00, + 0x00, 0x1d, 0x01, 0x00, 0x00, 0x1e, 0x01, 0x00, 0x00, 0x1f, 0x01, 0x00, 0x00, 0x20, 0x01, 0x00, 0x00, + 0x21, 0x01, 0x00, 0x00, 0x22, 0x01, 0x00, 0x00, 0x23, 0x01, 0x00, 0x00, 0x24, 0x01, 0x00, 0x00, 0x25, + 0x01, 0x00, 0x00, 0x26, 0x01, 0x00, 0x00, 0x27, 0x01, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, 0x29, 0x01, + 0x00, 0x00, 0x2a, 0x01, 0x00, 0x00, 0x2b, 0x01, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x2d, 0x01, 0x00, + 0x00, 0x2e, 0x01, 0x00, 0x00, 0x2f, 0x01, 0x00, 0x00, 0x30, 0x01, 0x00, 0x00, 0x31, 0x01, 0x00, 0x00, + 0x32, 0x01, 0x00, 0x00, 0x33, 0x01, 0x00, 0x00, 0x34, 0x01, 0x00, 0x00, 0x35, 0x01, 0x00, 0x00, 0x36, + 0x01, 0x00, 0x00, 0x37, 0x01, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, 0x39, 0x01, 0x00, 0x00, 0x3a, 0x01, + 0x00, 0x00, 0x3b, 0x01, 0x00, 0x00, 0x3c, 0x01, 0x00, 0x00, 0x3d, 0x01, 0x00, 0x00, 0x3e, 0x01, 0x00, + 0x00, 0x3f, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x41, 0x01, 0x00, 0x00, 0x42, 0x01, 0x00, 0x00, + 0x43, 0x01, 0x00, 0x00, 0x44, 0x01, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x46, 0x01, 0x00, 0x00, 0x47, + 0x01, 0x00, 0x00, 0x48, 0x01, 0x00, 0x00, 0x49, 0x01, 0x00, 0x00, 0x4a, 0x01, 0x00, 0x00, 0x4b, 0x01, + 0x00, 0x00, 0x4c, 0x01, 0x00, 0x00, 0x4d, 0x01, 0x00, 0x00, 0x4e, 0x01, 0x00, 0x00, 0x4f, 0x01, 0x00, + 0x00, 0x50, 0x01, 0x00, 0x00, 0x51, 0x01, 0x00, 0x00, 0x52, 0x01, 0x00, 0x00, 0x53, 0x01, 0x00, 0x00, + 0x54, 0x01, 0x00, 0x00, 0x55, 0x01, 0x00, 0x00, 0x56, 0x01, 0x00, 0x00, 0x57, 0x01, 0x00, 0x00, 0x58, + 0x01, 0x00, 0x00, 0x59, 0x01, 0x00, 0x00, 0x5a, 0x01, 0x00, 0x00, 0x5b, 0x01, 0x00, 0x00, 0x5c, 0x01, + 0x00, 0x00, 0x5d, 0x01, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, 0x5f, 0x01, 0x00, 0x00, 0x60, 0x01, 0x00, + 0x00, 0x61, 0x01, 0x00, 0x00, 0x62, 0x01, 0x00, 0x00, 0x63, 0x01, 0x00, 0x00, 0x64, 0x01, 0x00, 0x00, + 0x65, 0x01, 0x00, 0x00, 0x66, 0x01, 0x00, 0x00, 0x67, 0x01, 0x00, 0x00, 0x68, 0x01, 0x00, 0x00, 0x69, + 0x01, 0x00, 0x00, 0x6a, 0x01, 0x00, 0x00, 0x6b, 0x01, 0x00, 0x00, 0x6c, 0x01, 0x00, 0x00, 0x6d, 0x01, + 0x00, 0x00, 0x6e, 0x01, 0x00, 0x00, 0x6f, 0x01, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x71, 0x01, 0x00, + 0x00, 0x72, 0x01, 0x00, 0x00, 0x73, 0x01, 0x00, 0x00, 0x74, 0x01, 0x00, 0x00, 0x75, 0x01, 0x00, 0x00, + 0x76, 0x01, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x78, 0x01, 0x00, 0x00, 0x79, 0x01, 0x00, 0x00, 0x7a, + 0x01, 0x00, 0x00, 0x7b, 0x01, 0x00, 0x00, 0x7c, 0x01, 0x00, 0x00, 0x7d, 0x01, 0x00, 0x00, 0x7e, 0x01, + 0x00, 0x00, 0x7f, 0x01, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x81, 0x01, 0x00, 0x00, 0x82, 0x01, 0x00, + 0x00, 0x83, 0x01, 0x00, 0x00, 0x84, 0x01, 0x00, 0x00, 0x85, 0x01, 0x00, 0x00, 0x86, 0x01, 0x00, 0x00, + 0x87, 0x01, 0x00, 0x00, 0x88, 0x01, 0x00, 0x00, 0x89, 0x01, 0x00, 0x00, 0x8a, 0x01, 0x00, 0x00, 0x8b, + 0x01, 0x00, 0x00, 0x8c, 0x01, 0x00, 0x00, 0x8d, 0x01, 0x00, 0x00, 0x8e, 0x01, 0x00, 0x00, 0x8f, 0x01, + 0x00, 0x00, 0x90, 0x01, 0x00, 0x00, 0x91, 0x01, 0x00, 0x00, 0x92, 0x01, 0x00, 0x00, 0x93, 0x01, 0x00, + 0x00, 0x94, 0x01, 0x00, 0x00, 0x95, 0x01, 0x00, 0x00, 0x96, 0x01, 0x00, 0x00, 0x97, 0x01, 0x00, 0x00, + 0x98, 0x01, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, 0x9a, 0x01, 0x00, 0x00, 0x9b, 0x01, 0x00, 0x00, 0x9c, + 0x01, 0x00, 0x00, 0x9d, 0x01, 0x00, 0x00, 0x9e, 0x01, 0x00, 0x00, 0x9f, 0x01, 0x00, 0x00, 0xa0, 0x01, + 0x00, 0x00, 0xa1, 0x01, 0x00, 0x00, 0xa2, 0x01, 0x00, 0x00, 0xa3, 0x01, 0x00, 0x00, 0xa4, 0x01, 0x00, + 0x00, 0xa5, 0x01, 0x00, 0x00, 0xa6, 0x01, 0x00, 0x00, 0xa7, 0x01, 0x00, 0x00, 0xa8, 0x01, 0x00, 0x00, + 0xa9, 0x01, 0x00, 0x00, 0xaa, 0x01, 0x00, 0x00, 0xab, 0x01, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, 0xad, + 0x01, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0xaf, 0x01, 0x00, 0x00, 0xb0, 0x01, 0x00, 0x00, 0xb1, 0x01, + 0x00, 0x00, 0xb2, 0x01, 0x00, 0x00, 0xb3, 0x01, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0xb5, 0x01, 0x00, + 0x00, 0xb6, 0x01, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0xb8, 0x01, 0x00, 0x00, 0xb9, 0x01, 0x00, 0x00, + 0xba, 0x01, 0x00, 0x00, 0xbb, 0x01, 0x00, 0x00, 0xbc, 0x01, 0x00, 0x00, 0xbd, 0x01, 0x00, 0x00, 0xbe, + 0x01, 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc1, 0x01, 0x00, 0x00, 0xc2, 0x01, + 0x00, 0x00, 0xc3, 0x01, 0x00, 0x00, 0xc4, 0x01, 0x00, 0x00, 0xc5, 0x01, 0x00, 0x00, 0xc6, 0x01, 0x00, + 0x00, 0xc7, 0x01, 0x00, 0x00, 0xc8, 0x01, 0x00, 0x00, 0xc9, 0x01, 0x00, 0x00, 0xca, 0x01, 0x00, 0x00, + 0xcb, 0x01, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, 0xcd, 0x01, 0x00, 0x00, 0xce, 0x01, 0x00, 0x00, 0xcf, + 0x01, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, 0xd1, 0x01, 0x00, 0x00, 0xd2, 0x01, 0x00, 0x00, 0xd3, 0x01, + 0x00, 0x00, 0xd4, 0x01, 0x00, 0x00, 0xd5, 0x01, 0x00, 0x00, 0xd6, 0x01, 0x00, 0x00, 0xd7, 0x01, 0x00, + 0x00, 0xd8, 0x01, 0x00, 0x00, 0xd9, 0x01, 0x00, 0x00, 0xda, 0x01, 0x00, 0x00, 0xdb, 0x01, 0x00, 0x00, + 0xdc, 0x01, 0x00, 0x00, 0xdd, 0x01, 0x00, 0x00, 0xde, 0x01, 0x00, 0x00, 0xdf, 0x01, 0x00, 0x00, 0xe0, + 0x01, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, 0xe3, 0x01, 0x00, 0x00, 0xe4, 0x01, + 0x00, 0x00, 0xe5, 0x01, 0x00, 0x00, 0xe6, 0x01, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, 0xe8, 0x01, 0x00, + 0x00, 0xe9, 0x01, 0x00, 0x00, 0xea, 0x01, 0x00, 0x00, 0xeb, 0x01, 0x00, 0x00, 0xec, 0x01, 0x00, 0x00, + 0xed, 0x01, 0x00, 0x00, 0xee, 0x01, 0x00, 0x00, 0xef, 0x01, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, 0xf1, + 0x01, 0x00, 0x00, 0xf2, 0x01, 0x00, 0x00, 0xf3, 0x01, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0xf5, 0x01, + 0x00, 0x00, 0xf6, 0x01, 0x00, 0x00, 0xf7, 0x01, 0x00, 0x00, 0xf8, 0x01, 0x00, 0x00, 0xf9, 0x01, 0x00, + 0x00, 0xfa, 0x01, 0x00, 0x00, 0xfb, 0x01, 0x00, 0x00, 0xfc, 0x01, 0x00, 0x00, 0xfd, 0x01, 0x00, 0x00, + 0xfe, 0x01, 0x00, 0x00, 0xff, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x02, + 0x02, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00, 0x05, 0x02, 0x00, 0x00, 0x06, 0x02, + 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x09, 0x02, 0x00, 0x00, 0x0a, 0x02, 0x00, + 0x00, 0x0b, 0x02, 0x00, 0x00, 0x0c, 0x02, 0x00, 0x00, 0x0d, 0x02, 0x00, 0x00, 0x0e, 0x02, 0x00, 0x00, + 0x0f, 0x02, 0x00, 0x00, 0x10, 0x02, 0x00, 0x00, 0x11, 0x02, 0x00, 0x00, 0x12, 0x02, 0x00, 0x00, 0x13, + 0x02, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x15, 0x02, 0x00, 0x00, 0x16, 0x02, 0x00, 0x00, 0x17, 0x02, + 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x19, 0x02, 0x00, 0x00, 0x1a, 0x02, 0x00, 0x00, 0x1b, 0x02, 0x00, + 0x00, 0x1c, 0x02, 0x00, 0x00, 0x1d, 0x02, 0x00, 0x00, 0x1e, 0x02, 0x00, 0x00, 0x1f, 0x02, 0x00, 0x00, + 0x20, 0x02, 0x00, 0x00, 0x21, 0x02, 0x00, 0x00, 0x22, 0x02, 0x00, 0x00, 0x23, 0x02, 0x00, 0x00, 0x24, + 0x02, 0x00, 0x00, 0x25, 0x02, 0x00, 0x00, 0x26, 0x02, 0x00, 0x00, 0x27, 0x02, 0x00, 0x00, 0x28, 0x02, + 0x00, 0x00, 0x29, 0x02, 0x00, 0x00, 0x2a, 0x02, 0x00, 0x00, 0x2b, 0x02, 0x00, 0x00, 0x2c, 0x02, 0x00, + 0x00, 0x2d, 0x02, 0x00, 0x00, 0x2e, 0x02, 0x00, 0x00, 0x2f, 0x02, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, + 0x31, 0x02, 0x00, 0x00, 0x32, 0x02, 0x00, 0x00, 0x33, 0x02, 0x00, 0x00, 0x34, 0x02, 0x00, 0x00, 0x35, + 0x02, 0x00, 0x00, 0x36, 0x02, 0x00, 0x00, 0x37, 0x02, 0x00, 0x00, 0x38, 0x02, 0x00, 0x00, 0x39, 0x02, + 0x00, 0x00, 0x3a, 0x02, 0x00, 0x00, 0x3b, 0x02, 0x00, 0x00, 0x3c, 0x02, 0x00, 0x00, 0x3d, 0x02, 0x00, + 0x00, 0x3e, 0x02, 0x00, 0x00, 0x3f, 0x02, 0x00, 0x00, 0x40, 0x02, 0x00, 0x00, 0x41, 0x02, 0x00, 0x00, + 0x42, 0x02, 0x00, 0x00, 0x43, 0x02, 0x00, 0x00, 0x44, 0x02, 0x00, 0x00, 0x45, 0x02, 0x00, 0x00, 0x46, + 0x02, 0x00, 0x00, 0x47, 0x02, 0x00, 0x00, 0x48, 0x02, 0x00, 0x00, 0x49, 0x02, 0x00, 0x00, 0x4a, 0x02, + 0x00, 0x00, 0x4b, 0x02, 0x00, 0x00, 0x4c, 0x02, 0x00, 0x00, 0x4d, 0x02, 0x00, 0x00, 0x4e, 0x02, 0x00, + 0x00, 0x4f, 0x02, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, 0x51, 0x02, 0x00, 0x00, 0x52, 0x02, 0x00, 0x00, + 0x53, 0x02, 0x00, 0x00, 0x54, 0x02, 0x00, 0x00, 0x55, 0x02, 0x00, 0x00, 0x56, 0x02, 0x00, 0x00, 0x57, + 0x02, 0x00, 0x00, 0x58, 0x02, 0x00, 0x00, 0x59, 0x02, 0x00, 0x00, 0x5a, 0x02, 0x00, 0x00, 0x5b, 0x02, + 0x00, 0x00, 0x5c, 0x02, 0x00, 0x00, 0x5d, 0x02, 0x00, 0x00, 0x5e, 0x02, 0x00, 0x00, 0x5f, 0x02, 0x00, + 0x00, 0x60, 0x02, 0x00, 0x00, 0x61, 0x02, 0x00, 0x00, 0x62, 0x02, 0x00, 0x00, 0x63, 0x02, 0x00, 0x00, + 0x64, 0x02, 0x00, 0x00, 0x65, 0x02, 0x00, 0x00, 0x66, 0x02, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x68, + 0x02, 0x00, 0x00, 0x69, 0x02, 0x00, 0x00, 0x6a, 0x02, 0x00, 0x00, 0x6b, 0x02, 0x00, 0x00, 0x6c, 0x02, + 0x00, 0x00, 0x6d, 0x02, 0x00, 0x00, 0x6e, 0x02, 0x00, 0x00, 0x6f, 0x02, 0x00, 0x00, 0x70, 0x02, 0x00, + 0x00, 0x71, 0x02, 0x00, 0x00, 0x72, 0x02, 0x00, 0x00, 0x73, 0x02, 0x00, 0x00, 0x74, 0x02, 0x00, 0x00, + 0x75, 0x02, 0x00, 0x00, 0x76, 0x02, 0x00, 0x00, 0x77, 0x02, 0x00, 0x00, 0x78, 0x02, 0x00, 0x00, 0x79, + 0x02, 0x00, 0x00, 0x7a, 0x02, 0x00, 0x00, 0x7b, 0x02, 0x00, 0x00, 0x7c, 0x02, 0x00, 0x00, 0x7d, 0x02, + 0x00, 0x00, 0x7e, 0x02, 0x00, 0x00, 0x7f, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x81, 0x02, 0x00, + 0x00, 0x82, 0x02, 0x00, 0x00, 0x83, 0x02, 0x00, 0x00, 0x84, 0x02, 0x00, 0x00, 0x85, 0x02, 0x00, 0x00, + 0x86, 0x02, 0x00, 0x00, 0x87, 0x02, 0x00, 0x00, 0x88, 0x02, 0x00, 0x00, 0x89, 0x02, 0x00, 0x00, 0x8a, + 0x02, 0x00, 0x00, 0x8b, 0x02, 0x00, 0x00, 0x8c, 0x02, 0x00, 0x00, 0x8d, 0x02, 0x00, 0x00, 0x8e, 0x02, + 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, 0x91, 0x02, 0x00, 0x00, 0x92, 0x02, 0x00, + 0x00, 0x93, 0x02, 0x00, 0x00, 0x94, 0x02, 0x00, 0x00, 0x95, 0x02, 0x00, 0x00, 0x96, 0x02, 0x00, 0x00, + 0x97, 0x02, 0x00, 0x00, 0x98, 0x02, 0x00, 0x00, 0x99, 0x02, 0x00, 0x00, 0x9a, 0x02, 0x00, 0x00, 0x9b, + 0x02, 0x00, 0x00, 0x9c, 0x02, 0x00, 0x00, 0x9d, 0x02, 0x00, 0x00, 0x9e, 0x02, 0x00, 0x00, 0x9f, 0x02, + 0x00, 0x00, 0xa0, 0x02, 0x00, 0x00, 0xa1, 0x02, 0x00, 0x00, 0xa2, 0x02, 0x00, 0x00, 0xa3, 0x02, 0x00, + 0x00, 0xa4, 0x02, 0x00, 0x00, 0xa5, 0x02, 0x00, 0x00, 0xa6, 0x02, 0x00, 0x00, 0xa7, 0x02, 0x00, 0x00, + 0xa8, 0x02, 0x00, 0x00, 0xa9, 0x02, 0x00, 0x00, 0xaa, 0x02, 0x00, 0x00, 0xab, 0x02, 0x00, 0x00, 0xac, + 0x02, 0x00, 0x00, 0xad, 0x02, 0x00, 0x00, 0xae, 0x02, 0x00, 0x00, 0xaf, 0x02, 0x00, 0x00, 0xb0, 0x02, + 0x00, 0x00, 0xb1, 0x02, 0x00, 0x00, 0xb2, 0x02, 0x00, 0x00, 0xb3, 0x02, 0x00, 0x00, 0xb4, 0x02, 0x00, + 0x00, 0xb5, 0x02, 0x00, 0x00, 0xb6, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0xb8, 0x02, 0x00, 0x00, + 0xb9, 0x02, 0x00, 0x00, 0xba, 0x02, 0x00, 0x00, 0xbb, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0xbd, + 0x02, 0x00, 0x00, 0xbe, 0x02, 0x00, 0x00, 0xbf, 0x02, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0xc1, 0x02, + 0x00, 0x00, 0xc2, 0x02, 0x00, 0x00, 0xc3, 0x02, 0x00, 0x00, 0xc4, 0x02, 0x00, 0x00, 0xc5, 0x02, 0x00, + 0x00, 0xc6, 0x02, 0x00, 0x00, 0xc7, 0x02, 0x00, 0x00, 0xc8, 0x02, 0x00, 0x00, 0xc9, 0x02, 0x00, 0x00, + 0xca, 0x02, 0x00, 0x00, 0xcb, 0x02, 0x00, 0x00, 0xcc, 0x02, 0x00, 0x00, 0xcd, 0x02, 0x00, 0x00, 0xce, + 0x02, 0x00, 0x00, 0xcf, 0x02, 0x00, 0x00, 0xd0, 0x02, 0x00, 0x00, 0xd1, 0x02, 0x00, 0x00, 0xd2, 0x02, + 0x00, 0x00, 0xd3, 0x02, 0x00, 0x00, 0xd4, 0x02, 0x00, 0x00, 0xd5, 0x02, 0x00, 0x00, 0xd6, 0x02, 0x00, + 0x00, 0xd7, 0x02, 0x00, 0x00, 0xd8, 0x02, 0x00, 0x00, 0xd9, 0x02, 0x00, 0x00, 0xda, 0x02, 0x00, 0x00, + 0xdb, 0x02, 0x00, 0x00, 0xdc, 0x02, 0x00, 0x00, 0xdd, 0x02, 0x00, 0x00, 0xde, 0x02, 0x00, 0x00, 0xdf, + 0x02, 0x00, 0x00, 0xe0, 0x02, 0x00, 0x00, 0xe1, 0x02, 0x00, 0x00, 0xe2, 0x02, 0x00, 0x00, 0xe3, 0x02, + 0x00, 0x00, 0xe4, 0x02, 0x00, 0x00, 0xe5, 0x02, 0x00, 0x00, 0xe6, 0x02, 0x00, 0x00, 0xe7, 0x02, 0x00, + 0x00, 0xe8, 0x02, 0x00, 0x00, 0xe9, 0x02, 0x00, 0x00, 0xea, 0x02, 0x00, 0x00, 0xeb, 0x02, 0x00, 0x00, + 0xec, 0x02, 0x00, 0x00, 0xed, 0x02, 0x00, 0x00, 0xee, 0x02, 0x00, 0x00, 0xef, 0x02, 0x00, 0x00, 0xf0, + 0x02, 0x00, 0x00, 0xf1, 0x02, 0x00, 0x00, 0xf2, 0x02, 0x00, 0x00, 0xf3, 0x02, 0x00, 0x00, 0xf4, 0x02, + 0x00, 0x00, 0xf5, 0x02, 0x00, 0x00, 0xf6, 0x02, 0x00, 0x00, 0xf7, 0x02, 0x00, 0x00, 0xf8, 0x02, 0x00, + 0x00, 0xf9, 0x02, 0x00, 0x00, 0xfa, 0x02, 0x00, 0x00, 0xfb, 0x02, 0x00, 0x00, 0xfc, 0x02, 0x00, 0x00, + 0xfd, 0x02, 0x00, 0x00, 0xfe, 0x02, 0x00, 0x00, 0xff, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x01, + 0x03, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x04, 0x03, 0x00, 0x00, 0x05, 0x03, + 0x00, 0x00, 0x06, 0x03, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x08, 0x03, 0x00, 0x00, 0x09, 0x03, 0x00, + 0x00, 0x0a, 0x03, 0x00, 0x00, 0x0b, 0x03, 0x00, 0x00, 0x0c, 0x03, 0x00, 0x00, 0x0d, 0x03, 0x00, 0x00, + 0x0e, 0x03, 0x00, 0x00, 0x0f, 0x03, 0x00, 0x00, 0x10, 0x03, 0x00, 0x00, 0x11, 0x03, 0x00, 0x00, 0x12, + 0x03, 0x00, 0x00, 0x13, 0x03, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x16, 0x03, + 0x00, 0x00, 0x17, 0x03, 0x00, 0x00, 0x18, 0x03, 0x00, 0x00, 0x19, 0x03, 0x00, 0x00, 0x1a, 0x03, 0x00, + 0x00, 0x1b, 0x03, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, 0x1d, 0x03, 0x00, 0x00, 0x1e, 0x03, 0x00, 0x00, + 0x1f, 0x03, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x21, 0x03, 0x00, 0x00, 0x22, 0x03, 0x00, 0x00, 0x23, + 0x03, 0x00, 0x00, 0x24, 0x03, 0x00, 0x00, 0x25, 0x03, 0x00, 0x00, 0x26, 0x03, 0x00, 0x00, 0x27, 0x03, + 0x00, 0x00, 0x28, 0x03, 0x00, 0x00, 0x29, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2b, 0x03, 0x00, + 0x00, 0x2c, 0x03, 0x00, 0x00, 0x2d, 0x03, 0x00, 0x00, 0x2e, 0x03, 0x00, 0x00, 0x2f, 0x03, 0x00, 0x00, + 0x30, 0x03, 0x00, 0x00, 0x31, 0x03, 0x00, 0x00, 0x32, 0x03, 0x00, 0x00, 0x33, 0x03, 0x00, 0x00, 0x34, + 0x03, 0x00, 0x00, 0x35, 0x03, 0x00, 0x00, 0x36, 0x03, 0x00, 0x00, 0x37, 0x03, 0x00, 0x00, 0x38, 0x03, + 0x00, 0x00, 0x39, 0x03, 0x00, 0x00, 0x3a, 0x03, 0x00, 0x00, 0x3b, 0x03, 0x00, 0x00, 0x3c, 0x03, 0x00, + 0x00, 0x3d, 0x03, 0x00, 0x00, 0x3e, 0x03, 0x00, 0x00, 0x3f, 0x03, 0x00, 0x00, 0x40, 0x03, 0x00, 0x00, + 0x41, 0x03, 0x00, 0x00, 0x42, 0x03, 0x00, 0x00, 0x43, 0x03, 0x00, 0x00, 0x44, 0x03, 0x00, 0x00, 0x45, + 0x03, 0x00, 0x00, 0x46, 0x03, 0x00, 0x00, 0x47, 0x03, 0x00, 0x00, 0x48, 0x03, 0x00, 0x00, 0x49, 0x03, + 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x4b, 0x03, 0x00, 0x00, 0x4c, 0x03, 0x00, 0x00, 0x4d, 0x03, 0x00, + 0x00, 0x4e, 0x03, 0x00, 0x00, 0x4f, 0x03, 0x00, 0x00, 0x50, 0x03, 0x00, 0x00, 0x51, 0x03, 0x00, 0x00, + 0x52, 0x03, 0x00, 0x00, 0x53, 0x03, 0x00, 0x00, 0x54, 0x03, 0x00, 0x00, 0x55, 0x03, 0x00, 0x00, 0x56, + 0x03, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0x58, 0x03, 0x00, 0x00, 0x59, 0x03, 0x00, 0x00, 0x5a, 0x03, + 0x00, 0x00, 0x5b, 0x03, 0x00, 0x00, 0x5c, 0x03, 0x00, 0x00, 0x5d, 0x03, 0x00, 0x00, 0x5e, 0x03, 0x00, + 0x00, 0x5f, 0x03, 0x00, 0x00, 0x60, 0x03, 0x00, 0x00, 0x61, 0x03, 0x00, 0x00, 0x62, 0x03, 0x00, 0x00, + 0x63, 0x03, 0x00, 0x00, 0x64, 0x03, 0x00, 0x00, 0x65, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x67, + 0x03, 0x00, 0x00, 0x68, 0x03, 0x00, 0x00, 0x69, 0x03, 0x00, 0x00, 0x6a, 0x03, 0x00, 0x00, 0x6b, 0x03, + 0x00, 0x00, 0x6c, 0x03, 0x00, 0x00, 0x6d, 0x03, 0x00, 0x00, 0x6e, 0x03, 0x00, 0x00, 0x6f, 0x03, 0x00, + 0x00, 0x70, 0x03, 0x00, 0x00, 0x71, 0x03, 0x00, 0x00, 0x72, 0x03, 0x00, 0x00, 0x73, 0x03, 0x00, 0x00, + 0x74, 0x03, 0x00, 0x00, 0x75, 0x03, 0x00, 0x00, 0x76, 0x03, 0x00, 0x00, 0x77, 0x03, 0x00, 0x00, 0x78, + 0x03, 0x00, 0x00, 0x79, 0x03, 0x00, 0x00, 0x7a, 0x03, 0x00, 0x00, 0x7b, 0x03, 0x00, 0x00, 0x7c, 0x03, + 0x00, 0x00, 0x7d, 0x03, 0x00, 0x00, 0x7e, 0x03, 0x00, 0x00, 0x7f, 0x03, 0x00, 0x00, 0x80, 0x03, 0x00, + 0x00, 0x81, 0x03, 0x00, 0x00, 0x82, 0x03, 0x00, 0x00, 0x83, 0x03, 0x00, 0x00, 0x84, 0x03, 0x00, 0x00, + 0x85, 0x03, 0x00, 0x00, 0x86, 0x03, 0x00, 0x00, 0x87, 0x03, 0x00, 0x00, 0x88, 0x03, 0x00, 0x00, 0x89, + 0x03, 0x00, 0x00, 0x8a, 0x03, 0x00, 0x00, 0x8b, 0x03, 0x00, 0x00, 0x8c, 0x03, 0x00, 0x00, 0x8d, 0x03, + 0x00, 0x00, 0x8e, 0x03, 0x00, 0x00, 0x8f, 0x03, 0x00, 0x00, 0x90, 0x03, 0x00, 0x00, 0x91, 0x03, 0x00, + 0x00, 0x92, 0x03, 0x00, 0x00, 0x93, 0x03, 0x00, 0x00, 0x94, 0x03, 0x00, 0x00, 0x95, 0x03, 0x00, 0x00, + 0x96, 0x03, 0x00, 0x00, 0x97, 0x03, 0x00, 0x00, 0x98, 0x03, 0x00, 0x00, 0x99, 0x03, 0x00, 0x00, 0x9a, + 0x03, 0x00, 0x00, 0x9b, 0x03, 0x00, 0x00, 0x9c, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9e, 0x03, + 0x00, 0x00, 0x9f, 0x03, 0x00, 0x00, 0xa0, 0x03, 0x00, 0x00, 0xa1, 0x03, 0x00, 0x00, 0xa2, 0x03, 0x00, + 0x00, 0xa3, 0x03, 0x00, 0x00, 0xa4, 0x03, 0x00, 0x00, 0xa5, 0x03, 0x00, 0x00, 0xa6, 0x03, 0x00, 0x00, + 0xa7, 0x03, 0x00, 0x00, 0xa8, 0x03, 0x00, 0x00, 0xa9, 0x03, 0x00, 0x00, 0xaa, 0x03, 0x00, 0x00, 0xab, + 0x03, 0x00, 0x00, 0xac, 0x03, 0x00, 0x00, 0xad, 0x03, 0x00, 0x00, 0xae, 0x03, 0x00, 0x00, 0xaf, 0x03, + 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, 0xb1, 0x03, 0x00, 0x00, 0xb2, 0x03, 0x00, 0x00, 0xb3, 0x03, 0x00, + 0x00, 0xb4, 0x03, 0x00, 0x00, 0xb5, 0x03, 0x00, 0x00, 0xb6, 0x03, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, + 0xb8, 0x03, 0x00, 0x00, 0xb9, 0x03, 0x00, 0x00, 0xba, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbc, + 0x03, 0x00, 0x00, 0xbd, 0x03, 0x00, 0x00, 0xbe, 0x03, 0x00, 0x00, 0xbf, 0x03, 0x00, 0x00, 0xc0, 0x03, + 0x00, 0x00, 0xc1, 0x03, 0x00, 0x00, 0xc2, 0x03, 0x00, 0x00, 0xc3, 0x03, 0x00, 0x00, 0xc4, 0x03, 0x00, + 0x00, 0xc5, 0x03, 0x00, 0x00, 0xc6, 0x03, 0x00, 0x00, 0xc7, 0x03, 0x00, 0x00, 0xc8, 0x03, 0x00, 0x00, + 0xc9, 0x03, 0x00, 0x00, 0xca, 0x03, 0x00, 0x00, 0xcb, 0x03, 0x00, 0x00, 0xcc, 0x03, 0x00, 0x00, 0xcd, + 0x03, 0x00, 0x00, 0xce, 0x03, 0x00, 0x00, 0xcf, 0x03, 0x00, 0x00, 0xd0, 0x03, 0x00, 0x00, 0xd1, 0x03, + 0x00, 0x00, 0xd2, 0x03, 0x00, 0x00, 0xd3, 0x03, 0x00, 0x00, 0xd4, 0x03, 0x00, 0x00, 0xd5, 0x03, 0x00, + 0x00, 0xd6, 0x03, 0x00, 0x00, 0xd7, 0x03, 0x00, 0x00, 0xd8, 0x03, 0x00, 0x00, 0xd9, 0x03, 0x00, 0x00, + 0xda, 0x03, 0x00, 0x00, 0xdb, 0x03, 0x00, 0x00, 0xdc, 0x03, 0x00, 0x00, 0xdd, 0x03, 0x00, 0x00, 0xde, + 0x03, 0x00, 0x00, 0xdf, 0x03, 0x00, 0x00, 0xe0, 0x03, 0x00, 0x00, 0xe1, 0x03, 0x00, 0x00, 0xe2, 0x03, + 0x00, 0x00, 0xe3, 0x03, 0x00, 0x00, 0xe4, 0x03, 0x00, 0x00, 0xe5, 0x03, 0x00, 0x00, 0xe6, 0x03, 0x00, + 0x00, 0xe7, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, 0xea, 0x03, 0x00, 0x00, + 0xeb, 0x03, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, 0xed, 0x03, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0xef, + 0x03, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, 0xf3, 0x03, + 0x00, 0x00, 0xf4, 0x03, 0x00, 0x00, 0xf5, 0x03, 0x00, 0x00, 0xf6, 0x03, 0x00, 0x00, 0xf7, 0x03, 0x00, + 0x00, 0xf8, 0x03, 0x00, 0x00, 0xf9, 0x03, 0x00, 0x00, 0xfa, 0x03, 0x00, 0x00, 0xfb, 0x03, 0x00, 0x00, + 0xfc, 0x03, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x04, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x04, 0x04, + 0x00, 0x00, 0x05, 0x04, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x07, 0x04, 0x00, 0x00, 0x08, 0x04, 0x00, + 0x00, 0x09, 0x04, 0x00, 0x00, 0x0a, 0x04, 0x00, 0x00, 0x0b, 0x04, 0x00, 0x00, 0x0c, 0x04, 0x00, 0x00, + 0x0d, 0x04, 0x00, 0x00, 0x0e, 0x04, 0x00, 0x00, 0x0f, 0x04, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x11, + 0x04, 0x00, 0x00, 0x12, 0x04, 0x00, 0x00, 0x13, 0x04, 0x00, 0x00, 0x14, 0x04, 0x00, 0x00, 0x15, 0x04, + 0x00, 0x00, 0x16, 0x04, 0x00, 0x00, 0x17, 0x04, 0x00, 0x00, 0x18, 0x04, 0x00, 0x00, 0x19, 0x04, 0x00, + 0x00, 0x1a, 0x04, 0x00, 0x00, 0x1b, 0x04, 0x00, 0x00, 0x1c, 0x04, 0x00, 0x00, 0x1d, 0x04, 0x00, 0x00, + 0x1e, 0x04, 0x00, 0x00, 0x1f, 0x04, 0x00, 0x00, 0x20, 0x04, 0x00, 0x00, 0x21, 0x04, 0x00, 0x00, 0x22, + 0x04, 0x00, 0x00, 0x23, 0x04, 0x00, 0x00, 0x24, 0x04, 0x00, 0x00, 0x25, 0x04, 0x00, 0x00, 0x26, 0x04, + 0x00, 0x00, 0x27, 0x04, 0x00, 0x00, 0x28, 0x04, 0x00, 0x00, 0x29, 0x04, 0x00, 0x00, 0x2a, 0x04, 0x00, + 0x00, 0x2b, 0x04, 0x00, 0x00, 0x2c, 0x04, 0x00, 0x00, 0x2d, 0x04, 0x00, 0x00, 0x2e, 0x04, 0x00, 0x00, + 0x2f, 0x04, 0x00, 0x00, 0x30, 0x04, 0x00, 0x00, 0x31, 0x04, 0x00, 0x00, 0x32, 0x04, 0x00, 0x00, 0x33, + 0x04, 0x00, 0x00, 0x34, 0x04, 0x00, 0x00, 0x35, 0x04, 0x00, 0x00, 0x36, 0x04, 0x00, 0x00, 0x37, 0x04, + 0x00, 0x00, 0x38, 0x04, 0x00, 0x00, 0x39, 0x04, 0x00, 0x00, 0x3a, 0x04, 0x00, 0x00, 0x3b, 0x04, 0x00, + 0x00, 0x3c, 0x04, 0x00, 0x00, 0x3d, 0x04, 0x00, 0x00, 0x3e, 0x04, 0x00, 0x00, 0x3f, 0x04, 0x00, 0x00, + 0x40, 0x04, 0x00, 0x00, 0x41, 0x04, 0x00, 0x00, 0x42, 0x04, 0x00, 0x00, 0x43, 0x04, 0x00, 0x00, 0x44, + 0x04, 0x00, 0x00, 0x45, 0x04, 0x00, 0x00, 0x46, 0x04, 0x00, 0x00, 0x47, 0x04, 0x00, 0x00, 0x48, 0x04, + 0x00, 0x00, 0x49, 0x04, 0x00, 0x00, 0x4a, 0x04, 0x00, 0x00, 0x4b, 0x04, 0x00, 0x00, 0x4c, 0x04, 0x00, + 0x00, 0x4d, 0x04, 0x00, 0x00, 0x4e, 0x04, 0x00, 0x00, 0x4f, 0x04, 0x00, 0x00, 0x50, 0x04, 0x00, 0x00, + 0x51, 0x04, 0x00, 0x00, 0x52, 0x04, 0x00, 0x00, 0x53, 0x04, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x55, + 0x04, 0x00, 0x00, 0x56, 0x04, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, 0x58, 0x04, 0x00, 0x00, 0x59, 0x04, + 0x00, 0x00, 0x5a, 0x04, 0x00, 0x00, 0x5b, 0x04, 0x00, 0x00, 0x5c, 0x04, 0x00, 0x00, 0x5d, 0x04, 0x00, + 0x00, 0x5e, 0x04, 0x00, 0x00, 0x5f, 0x04, 0x00, 0x00, 0x60, 0x04, 0x00, 0x00, 0x61, 0x04, 0x00, 0x00, + 0x62, 0x04, 0x00, 0x00, 0x63, 0x04, 0x00, 0x00, 0x64, 0x04, 0x00, 0x00, 0x65, 0x04, 0x00, 0x00, 0x66, + 0x04, 0x00, 0x00, 0x67, 0x04, 0x00, 0x00, 0x68, 0x04, 0x00, 0x00, 0x69, 0x04, 0x00, 0x00, 0x6a, 0x04, + 0x00, 0x00, 0x6b, 0x04, 0x00, 0x00, 0x6c, 0x04, 0x00, 0x00, 0x6d, 0x04, 0x00, 0x00, 0x6e, 0x04, 0x00, + 0x00, 0x6f, 0x04, 0x00, 0x00, 0x70, 0x04, 0x00, 0x00, 0x71, 0x04, 0x00, 0x00, 0x72, 0x04, 0x00, 0x00, + 0x73, 0x04, 0x00, 0x00, 0x74, 0x04, 0x00, 0x00, 0x75, 0x04, 0x00, 0x00, 0x76, 0x04, 0x00, 0x00, 0x77, + 0x04, 0x00, 0x00, 0x78, 0x04, 0x00, 0x00, 0x79, 0x04, 0x00, 0x00, 0x7a, 0x04, 0x00, 0x00, 0x7b, 0x04, + 0x00, 0x00, 0x7c, 0x04, 0x00, 0x00, 0x7d, 0x04, 0x00, 0x00, 0x7e, 0x04, 0x00, 0x00, 0x7f, 0x04, 0x00, + 0x00, 0x80, 0x04, 0x00, 0x00, 0x81, 0x04, 0x00, 0x00, 0x82, 0x04, 0x00, 0x00, 0x83, 0x04, 0x00, 0x00, + 0x84, 0x04, 0x00, 0x00, 0x85, 0x04, 0x00, 0x00, 0x86, 0x04, 0x00, 0x00, 0x87, 0x04, 0x00, 0x00, 0x88, + 0x04, 0x00, 0x00, 0x89, 0x04, 0x00, 0x00, 0x8a, 0x04, 0x00, 0x00, 0x8b, 0x04, 0x00, 0x00, 0x8c, 0x04, + 0x00, 0x00, 0x8d, 0x04, 0x00, 0x00, 0x8e, 0x04, 0x00, 0x00, 0x8f, 0x04, 0x00, 0x00, 0x90, 0x04, 0x00, + 0x00, 0x91, 0x04, 0x00, 0x00, 0x92, 0x04, 0x00, 0x00, 0x93, 0x04, 0x00, 0x00, 0x94, 0x04, 0x00, 0x00, + 0x95, 0x04, 0x00, 0x00, 0x96, 0x04, 0x00, 0x00, 0x97, 0x04, 0x00, 0x00, 0x98, 0x04, 0x00, 0x00, 0x99, + 0x04, 0x00, 0x00, 0x9a, 0x04, 0x00, 0x00, 0x9b, 0x04, 0x00, 0x00, 0x9c, 0x04, 0x00, 0x00, 0x9d, 0x04, + 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, 0x9f, 0x04, 0x00, 0x00, 0xa0, 0x04, 0x00, 0x00, 0xa1, 0x04, 0x00, + 0x00, 0xa2, 0x04, 0x00, 0x00, 0xa3, 0x04, 0x00, 0x00, 0xa4, 0x04, 0x00, 0x00, 0xa5, 0x04, 0x00, 0x00, + 0xa6, 0x04, 0x00, 0x00, 0xa7, 0x04, 0x00, 0x00, 0xa8, 0x04, 0x00, 0x00, 0xa9, 0x04, 0x00, 0x00, 0xaa, + 0x04, 0x00, 0x00, 0xab, 0x04, 0x00, 0x00, 0xac, 0x04, 0x00, 0x00, 0xad, 0x04, 0x00, 0x00, 0xae, 0x04, + 0x00, 0x00, 0xaf, 0x04, 0x00, 0x00, 0xb0, 0x04, 0x00, 0x00, 0xb1, 0x04, 0x00, 0x00, 0xb2, 0x04, 0x00, + 0x00, 0xb3, 0x04, 0x00, 0x00, 0xb4, 0x04, 0x00, 0x00, 0xb5, 0x04, 0x00, 0x00, 0xb6, 0x04, 0x00, 0x00, + 0xb7, 0x04, 0x00, 0x00, 0xb8, 0x04, 0x00, 0x00, 0xb9, 0x04, 0x00, 0x00, 0xba, 0x04, 0x00, 0x00, 0xbb, + 0x04, 0x00, 0x00, 0xbc, 0x04, 0x00, 0x00, 0xbd, 0x04, 0x00, 0x00, 0xbe, 0x04, 0x00, 0x00, 0xbf, 0x04, + 0x00, 0x00, 0xc0, 0x04, 0x00, 0x00, 0xc1, 0x04, 0x00, 0x00, 0xc2, 0x04, 0x00, 0x00, 0xc3, 0x04, 0x00, + 0x00, 0xc4, 0x04, 0x00, 0x00, 0xc5, 0x04, 0x00, 0x00, 0xc6, 0x04, 0x00, 0x00, 0xc7, 0x04, 0x00, 0x00, + 0xc8, 0x04, 0x00, 0x00, 0xc9, 0x04, 0x00, 0x00, 0xca, 0x04, 0x00, 0x00, 0xcb, 0x04, 0x00, 0x00, 0xcc, + 0x04, 0x00, 0x00, 0xcd, 0x04, 0x00, 0x00, 0xce, 0x04, 0x00, 0x00, 0xcf, 0x04, 0x00, 0x00, 0xd0, 0x04, + 0x00, 0x00, 0xd1, 0x04, 0x00, 0x00, 0xd2, 0x04, 0x00, 0x00, 0xd3, 0x04, 0x00, 0x00, 0xd4, 0x04, 0x00, + 0x00, 0xd5, 0x04, 0x00, 0x00, 0xd6, 0x04, 0x00, 0x00, 0xd7, 0x04, 0x00, 0x00, 0xd8, 0x04, 0x00, 0x00, + 0xd9, 0x04, 0x00, 0x00, 0xda, 0x04, 0x00, 0x00, 0xdb, 0x04, 0x00, 0x00, 0xdc, 0x04, 0x00, 0x00, 0xdd, + 0x04, 0x00, 0x00, 0xde, 0x04, 0x00, 0x00, 0xdf, 0x04, 0x00, 0x00, 0xe0, 0x04, 0x00, 0x00, 0xe1, 0x04, + 0x00, 0x00, 0xe2, 0x04, 0x00, 0x00, 0xe3, 0x04, 0x00, 0x00, 0xe4, 0x04, 0x00, 0x00, 0xe5, 0x04, 0x00, + 0x00, 0xe6, 0x04, 0x00, 0x00, 0xe7, 0x04, 0x00, 0x00, 0xe8, 0x04, 0x00, 0x00, 0xe9, 0x04, 0x00, 0x00, + 0xea, 0x04, 0x00, 0x00, 0xeb, 0x04, 0x00, 0x00, 0xec, 0x04, 0x00, 0x00, 0xed, 0x04, 0x00, 0x00, 0xee, + 0x04, 0x00, 0x00, 0xef, 0x04, 0x00, 0x00, 0xf0, 0x04, 0x00, 0x00, 0xf1, 0x04, 0x00, 0x00, 0xf2, 0x04, + 0x00, 0x00, 0xf3, 0x04, 0x00, 0x00, 0xf4, 0x04, 0x00, 0x00, 0xf5, 0x04, 0x00, 0x00, 0xf6, 0x04, 0x00, + 0x00, 0xf7, 0x04, 0x00, 0x00, 0xf8, 0x04, 0x00, 0x00, 0xf9, 0x04, 0x00, 0x00, 0xfa, 0x04, 0x00, 0x00, + 0xfb, 0x04, 0x00, 0x00, 0xfc, 0x04, 0x00, 0x00, 0xfd, 0x04, 0x00, 0x00, 0xfe, 0x04, 0x00, 0x00, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x01, 0x05, 0x00, 0x00, 0x02, 0x05, 0x00, 0x00, 0x03, 0x05, + 0x00, 0x00, 0x04, 0x05, 0x00, 0x00, 0x05, 0x05, 0x00, 0x00, 0x06, 0x05, 0x00, 0x00, 0x07, 0x05, 0x00, + 0x00, 0x08, 0x05, 0x00, 0x00, 0x09, 0x05, 0x00, 0x00, 0x0a, 0x05, 0x00, 0x00, 0x0b, 0x05, 0x00, 0x00, + 0x0c, 0x05, 0x00, 0x00, 0x0d, 0x05, 0x00, 0x00, 0x0e, 0x05, 0x00, 0x00, 0x0f, 0x05, 0x00, 0x00, 0x10, + 0x05, 0x00, 0x00, 0x11, 0x05, 0x00, 0x00, 0x12, 0x05, 0x00, 0x00, 0x13, 0x05, 0x00, 0x00, 0x14, 0x05, + 0x00, 0x00, 0x15, 0x05, 0x00, 0x00, 0x16, 0x05, 0x00, 0x00, 0x17, 0x05, 0x00, 0x00, 0x18, 0x05, 0x00, + 0x00, 0x19, 0x05, 0x00, 0x00, 0x1a, 0x05, 0x00, 0x00, 0x1b, 0x05, 0x00, 0x00, 0x1c, 0x05, 0x00, 0x00, + 0x1d, 0x05, 0x00, 0x00, 0x1e, 0x05, 0x00, 0x00, 0x1f, 0x05, 0x00, 0x00, 0x20, 0x05, 0x00, 0x00, 0x21, + 0x05, 0x00, 0x00, 0x22, 0x05, 0x00, 0x00, 0x23, 0x05, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, 0x25, 0x05, + 0x00, 0x00, 0x26, 0x05, 0x00, 0x00, 0x27, 0x05, 0x00, 0x00, 0x28, 0x05, 0x00, 0x00, 0x29, 0x05, 0x00, + 0x00, 0x2a, 0x05, 0x00, 0x00, 0x2b, 0x05, 0x00, 0x00, 0x2c, 0x05, 0x00, 0x00, 0x2d, 0x05, 0x00, 0x00, + 0x2e, 0x05, 0x00, 0x00, 0x2f, 0x05, 0x00, 0x00, 0x30, 0x05, 0x00, 0x00, 0x31, 0x05, 0x00, 0x00, 0x32, + 0x05, 0x00, 0x00, 0x33, 0x05, 0x00, 0x00, 0x34, 0x05, 0x00, 0x00, 0x35, 0x05, 0x00, 0x00, 0x36, 0x05, + 0x00, 0x00, 0x37, 0x05, 0x00, 0x00, 0x38, 0x05, 0x00, 0x00, 0x39, 0x05, 0x00, 0x00, 0x3a, 0x05, 0x00, + 0x00, 0x3b, 0x05, 0x00, 0x00, 0x3c, 0x05, 0x00, 0x00, 0x3d, 0x05, 0x00, 0x00, 0x3e, 0x05, 0x00, 0x00, + 0x3f, 0x05, 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, 0x41, 0x05, 0x00, 0x00, 0x42, 0x05, 0x00, 0x00, 0x43, + 0x05, 0x00, 0x00, 0x44, 0x05, 0x00, 0x00, 0x45, 0x05, 0x00, 0x00, 0x46, 0x05, 0x00, 0x00, 0x47, 0x05, + 0x00, 0x00, 0x48, 0x05, 0x00, 0x00, 0x49, 0x05, 0x00, 0x00, 0x4a, 0x05, 0x00, 0x00, 0x4b, 0x05, 0x00, + 0x00, 0x4c, 0x05, 0x00, 0x00, 0x4d, 0x05, 0x00, 0x00, 0x4e, 0x05, 0x00, 0x00, 0x4f, 0x05, 0x00, 0x00, + 0x50, 0x05, 0x00, 0x00, 0x51, 0x05, 0x00, 0x00, 0x52, 0x05, 0x00, 0x00, 0x53, 0x05, 0x00, 0x00, 0x54, + 0x05, 0x00, 0x00, 0x55, 0x05, 0x00, 0x00, 0x56, 0x05, 0x00, 0x00, 0x57, 0x05, 0x00, 0x00, 0x58, 0x05, + 0x00, 0x00, 0x59, 0x05, 0x00, 0x00, 0x5a, 0x05, 0x00, 0x00, 0x5b, 0x05, 0x00, 0x00, 0x5c, 0x05, 0x00, + 0x00, 0x5d, 0x05, 0x00, 0x00, 0x5e, 0x05, 0x00, 0x00, 0x5f, 0x05, 0x00, 0x00, 0x60, 0x05, 0x00, 0x00, + 0x61, 0x05, 0x00, 0x00, 0x62, 0x05, 0x00, 0x00, 0x63, 0x05, 0x00, 0x00, 0x64, 0x05, 0x00, 0x00, 0x65, + 0x05, 0x00, 0x00, 0x66, 0x05, 0x00, 0x00, 0x67, 0x05, 0x00, 0x00, 0x68, 0x05, 0x00, 0x00, 0x69, 0x05, + 0x00, 0x00, 0x6a, 0x05, 0x00, 0x00, 0x6b, 0x05, 0x00, 0x00, 0x6c, 0x05, 0x00, 0x00, 0x6d, 0x05, 0x00, + 0x00, 0x6e, 0x05, 0x00, 0x00, 0x6f, 0x05, 0x00, 0x00, 0x70, 0x05, 0x00, 0x00, 0x71, 0x05, 0x00, 0x00, + 0x72, 0x05, 0x00, 0x00, 0x73, 0x05, 0x00, 0x00, 0x74, 0x05, 0x00, 0x00, 0x75, 0x05, 0x00, 0x00, 0x76, + 0x05, 0x00, 0x00, 0x77, 0x05, 0x00, 0x00, 0x78, 0x05, 0x00, 0x00, 0x79, 0x05, 0x00, 0x00, 0x7a, 0x05, + 0x00, 0x00, 0x7b, 0x05, 0x00, 0x00, 0x7c, 0x05, 0x00, 0x00, 0x7d, 0x05, 0x00, 0x00, 0x7e, 0x05, 0x00, + 0x00, 0x7f, 0x05, 0x00, 0x00, 0x80, 0x05, 0x00, 0x00, 0x81, 0x05, 0x00, 0x00, 0x82, 0x05, 0x00, 0x00, + 0x83, 0x05, 0x00, 0x00, 0x84, 0x05, 0x00, 0x00, 0x85, 0x05, 0x00, 0x00, 0x86, 0x05, 0x00, 0x00, 0x87, + 0x05, 0x00, 0x00, 0x88, 0x05, 0x00, 0x00, 0x89, 0x05, 0x00, 0x00, 0x8a, 0x05, 0x00, 0x00, 0x8b, 0x05, + 0x00, 0x00, 0x8c, 0x05, 0x00, 0x00, 0x8d, 0x05, 0x00, 0x00, 0x8e, 0x05, 0x00, 0x00, 0x8f, 0x05, 0x00, + 0x00, 0x90, 0x05, 0x00, 0x00, 0x91, 0x05, 0x00, 0x00, 0x92, 0x05, 0x00, 0x00, 0x93, 0x05, 0x00, 0x00, + 0x94, 0x05, 0x00, 0x00, 0x95, 0x05, 0x00, 0x00, 0x96, 0x05, 0x00, 0x00, 0x97, 0x05, 0x00, 0x00, 0x98, + 0x05, 0x00, 0x00, 0x99, 0x05, 0x00, 0x00, 0x9a, 0x05, 0x00, 0x00, 0x9b, 0x05, 0x00, 0x00, 0x9c, 0x05, + 0x00, 0x00, 0x9d, 0x05, 0x00, 0x00, 0x9e, 0x05, 0x00, 0x00, 0x9f, 0x05, 0x00, 0x00, 0xa0, 0x05, 0x00, + 0x00, 0xa1, 0x05, 0x00, 0x00, 0xa2, 0x05, 0x00, 0x00, 0xa3, 0x05, 0x00, 0x00, 0xa4, 0x05, 0x00, 0x00, + 0xa5, 0x05, 0x00, 0x00, 0xa6, 0x05, 0x00, 0x00, 0xa7, 0x05, 0x00, 0x00, 0xa8, 0x05, 0x00, 0x00, 0xa9, + 0x05, 0x00, 0x00, 0xaa, 0x05, 0x00, 0x00, 0xab, 0x05, 0x00, 0x00, 0xac, 0x05, 0x00, 0x00, 0xad, 0x05, + 0x00, 0x00, 0xae, 0x05, 0x00, 0x00, 0xaf, 0x05, 0x00, 0x00, 0xb0, 0x05, 0x00, 0x00, 0xb1, 0x05, 0x00, + 0x00, 0xb2, 0x05, 0x00, 0x00, 0xb3, 0x05, 0x00, 0x00, 0xb4, 0x05, 0x00, 0x00, 0xb5, 0x05, 0x00, 0x00, + 0xb6, 0x05, 0x00, 0x00, 0xb7, 0x05, 0x00, 0x00, 0xb8, 0x05, 0x00, 0x00, 0xb9, 0x05, 0x00, 0x00, 0xba, + 0x05, 0x00, 0x00, 0xbb, 0x05, 0x00, 0x00, 0xbc, 0x05, 0x00, 0x00, 0xbd, 0x05, 0x00, 0x00, 0xbe, 0x05, + 0x00, 0x00, 0xbf, 0x05, 0x00, 0x00, 0xc0, 0x05, 0x00, 0x00, 0xc1, 0x05, 0x00, 0x00, 0xc2, 0x05, 0x00, + 0x00, 0xc3, 0x05, 0x00, 0x00, 0xc4, 0x05, 0x00, 0x00, 0xc5, 0x05, 0x00, 0x00, 0xc6, 0x05, 0x00, 0x00, + 0xc7, 0x05, 0x00, 0x00, 0xc8, 0x05, 0x00, 0x00, 0xc9, 0x05, 0x00, 0x00, 0xca, 0x05, 0x00, 0x00, 0xcb, + 0x05, 0x00, 0x00, 0xcc, 0x05, 0x00, 0x00, 0xcd, 0x05, 0x00, 0x00, 0xce, 0x05, 0x00, 0x00, 0xcf, 0x05, + 0x00, 0x00, 0xd0, 0x05, 0x00, 0x00, 0xd1, 0x05, 0x00, 0x00, 0xd2, 0x05, 0x00, 0x00, 0xd3, 0x05, 0x00, + 0x00, 0xd4, 0x05, 0x00, 0x00, 0xd5, 0x05, 0x00, 0x00, 0xd6, 0x05, 0x00, 0x00, 0xd7, 0x05, 0x00, 0x00, + 0xd8, 0x05, 0x00, 0x00, 0xd9, 0x05, 0x00, 0x00, 0xda, 0x05, 0x00, 0x00, 0xdb, 0x05, 0x00, 0x00, 0xdc, + 0x05, 0x00, 0x00, 0xdd, 0x05, 0x00, 0x00, 0xde, 0x05, 0x00, 0x00, 0xdf, 0x05, 0x00, 0x00, 0xe0, 0x05, + 0x00, 0x00, 0xe1, 0x05, 0x00, 0x00, 0xe2, 0x05, 0x00, 0x00, 0xe3, 0x05, 0x00, 0x00, 0xe4, 0x05, 0x00, + 0x00, 0xe5, 0x05, 0x00, 0x00, 0xe6, 0x05, 0x00, 0x00, 0xe7, 0x05, 0x00, 0x00, 0xe8, 0x05, 0x00, 0x00, + 0xe9, 0x05, 0x00, 0x00, 0xea, 0x05, 0x00, 0x00, 0xeb, 0x05, 0x00, 0x00, 0xec, 0x05, 0x00, 0x00, 0xed, + 0x05, 0x00, 0x00, 0xee, 0x05, 0x00, 0x00, 0xef, 0x05, 0x00, 0x00, 0xf0, 0x05, 0x00, 0x00, 0xf1, 0x05, + 0x00, 0x00, 0xf2, 0x05, 0x00, 0x00, 0xf3, 0x05, 0x00, 0x00, 0xf4, 0x05, 0x00, 0x00, 0xf5, 0x05, 0x00, + 0x00, 0xf6, 0x05, 0x00, 0x00, 0xf7, 0x05, 0x00, 0x00, 0xf8, 0x05, 0x00, 0x00, 0xf9, 0x05, 0x00, 0x00, + 0xfa, 0x05, 0x00, 0x00, 0xfb, 0x05, 0x00, 0x00, 0xfc, 0x05, 0x00, 0x00, 0xfd, 0x05, 0x00, 0x00, 0xfe, + 0x05, 0x00, 0x00, 0xff, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x01, 0x06, 0x00, 0x00, 0x02, 0x06, + 0x00, 0x00, 0x03, 0x06, 0x00, 0x00, 0x04, 0x06, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, 0x06, 0x06, 0x00, + 0x00, 0x07, 0x06, 0x00, 0x00, 0x08, 0x06, 0x00, 0x00, 0x09, 0x06, 0x00, 0x00, 0x0a, 0x06, 0x00, 0x00, + 0x0b, 0x06, 0x00, 0x00, 0x0c, 0x06, 0x00, 0x00, 0x0d, 0x06, 0x00, 0x00, 0x0e, 0x06, 0x00, 0x00, 0x0f, + 0x06, 0x00, 0x00, 0x10, 0x06, 0x00, 0x00, 0x11, 0x06, 0x00, 0x00, 0x12, 0x06, 0x00, 0x00, 0x13, 0x06, + 0x00, 0x00, 0x14, 0x06, 0x00, 0x00, 0x15, 0x06, 0x00, 0x00, 0x16, 0x06, 0x00, 0x00, 0x17, 0x06, 0x00, + 0x00, 0x18, 0x06, 0x00, 0x00, 0x19, 0x06, 0x00, 0x00, 0x1a, 0x06, 0x00, 0x00, 0x1b, 0x06, 0x00, 0x00, + 0x1c, 0x06, 0x00, 0x00, 0x1d, 0x06, 0x00, 0x00, 0x1e, 0x06, 0x00, 0x00, 0x1f, 0x06, 0x00, 0x00, 0x20, + 0x06, 0x00, 0x00, 0x21, 0x06, 0x00, 0x00, 0x22, 0x06, 0x00, 0x00, 0x23, 0x06, 0x00, 0x00, 0x24, 0x06, + 0x00, 0x00, 0x25, 0x06, 0x00, 0x00, 0x26, 0x06, 0x00, 0x00, 0x27, 0x06, 0x00, 0x00, 0x28, 0x06, 0x00, + 0x00, 0x29, 0x06, 0x00, 0x00, 0x2a, 0x06, 0x00, 0x00, 0x2b, 0x06, 0x00, 0x00, 0x2c, 0x06, 0x00, 0x00, + 0x2d, 0x06, 0x00, 0x00, 0x2e, 0x06, 0x00, 0x00, 0x2f, 0x06, 0x00, 0x00, 0x30, 0x06, 0x00, 0x00, 0x31, + 0x06, 0x00, 0x00, 0x32, 0x06, 0x00, 0x00, 0x33, 0x06, 0x00, 0x00, 0x34, 0x06, 0x00, 0x00, 0x35, 0x06, + 0x00, 0x00, 0x36, 0x06, 0x00, 0x00, 0x37, 0x06, 0x00, 0x00, 0x38, 0x06, 0x00, 0x00, 0x39, 0x06, 0x00, + 0x00, 0x3a, 0x06, 0x00, 0x00, 0x3b, 0x06, 0x00, 0x00, 0x3c, 0x06, 0x00, 0x00, 0x3d, 0x06, 0x00, 0x00, + 0x3e, 0x06, 0x00, 0x00, 0x3f, 0x06, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, 0x41, 0x06, 0x00, 0x00, 0x42, + 0x06, 0x00, 0x00, 0x43, 0x06, 0x00, 0x00, 0x44, 0x06, 0x00, 0x00, 0x45, 0x06, 0x00, 0x00, 0x46, 0x06, + 0x00, 0x00, 0x47, 0x06, 0x00, 0x00, 0x48, 0x06, 0x00, 0x00, 0x49, 0x06, 0x00, 0x00, 0x4a, 0x06, 0x00, + 0x00, 0x4b, 0x06, 0x00, 0x00, 0x4c, 0x06, 0x00, 0x00, 0x4d, 0x06, 0x00, 0x00, 0x4e, 0x06, 0x00, 0x00, + 0x4f, 0x06, 0x00, 0x00, 0x50, 0x06, 0x00, 0x00, 0x51, 0x06, 0x00, 0x00, 0x52, 0x06, 0x00, 0x00, 0x53, + 0x06, 0x00, 0x00, 0x54, 0x06, 0x00, 0x00, 0x55, 0x06, 0x00, 0x00, 0x56, 0x06, 0x00, 0x00, 0x57, 0x06, + 0x00, 0x00, 0x58, 0x06, 0x00, 0x00, 0x59, 0x06, 0x00, 0x00, 0x5a, 0x06, 0x00, 0x00, 0x5b, 0x06, 0x00, + 0x00, 0x5c, 0x06, 0x00, 0x00, 0x5d, 0x06, 0x00, 0x00, 0x5e, 0x06, 0x00, 0x00, 0x5f, 0x06, 0x00, 0x00, + 0x60, 0x06, 0x00, 0x00, 0x61, 0x06, 0x00, 0x00, 0x62, 0x06, 0x00, 0x00, 0x63, 0x06, 0x00, 0x00, 0x64, + 0x06, 0x00, 0x00, 0x65, 0x06, 0x00, 0x00, 0x66, 0x06, 0x00, 0x00, 0x67, 0x06, 0x00, 0x00, 0x68, 0x06, + 0x00, 0x00, 0x69, 0x06, 0x00, 0x00, 0x6a, 0x06, 0x00, 0x00, 0x6b, 0x06, 0x00, 0x00, 0x6c, 0x06, 0x00, + 0x00, 0x6d, 0x06, 0x00, 0x00, 0x6e, 0x06, 0x00, 0x00, 0x6f, 0x06, 0x00, 0x00, 0x70, 0x06, 0x00, 0x00, + 0x71, 0x06, 0x00, 0x00, 0x72, 0x06, 0x00, 0x00, 0x73, 0x06, 0x00, 0x00, 0x74, 0x06, 0x00, 0x00, 0x75, + 0x06, 0x00, 0x00, 0x76, 0x06, 0x00, 0x00, 0x77, 0x06, 0x00, 0x00, 0x78, 0x06, 0x00, 0x00, 0x79, 0x06, + 0x00, 0x00, 0x7a, 0x06, 0x00, 0x00, 0x7b, 0x06, 0x00, 0x00, 0x7c, 0x06, 0x00, 0x00, 0x7d, 0x06, 0x00, + 0x00, 0x7e, 0x06, 0x00, 0x00, 0x7f, 0x06, 0x00, 0x00, 0x80, 0x06, 0x00, 0x00, 0x81, 0x06, 0x00, 0x00, + 0x82, 0x06, 0x00, 0x00, 0x83, 0x06, 0x00, 0x00, 0x84, 0x06, 0x00, 0x00, 0x85, 0x06, 0x00, 0x00, 0x86, + 0x06, 0x00, 0x00, 0x87, 0x06, 0x00, 0x00, 0x88, 0x06, 0x00, 0x00, 0x89, 0x06, 0x00, 0x00, 0x8a, 0x06, + 0x00, 0x00, 0x8b, 0x06, 0x00, 0x00, 0x8c, 0x06, 0x00, 0x00, 0x8d, 0x06, 0x00, 0x00, 0x8e, 0x06, 0x00, + 0x00, 0x8f, 0x06, 0x00, 0x00, 0x90, 0x06, 0x00, 0x00, 0x91, 0x06, 0x00, 0x00, 0x92, 0x06, 0x00, 0x00, + 0x93, 0x06, 0x00, 0x00, 0x94, 0x06, 0x00, 0x00, 0x95, 0x06, 0x00, 0x00, 0x96, 0x06, 0x00, 0x00, 0x97, + 0x06, 0x00, 0x00, 0x98, 0x06, 0x00, 0x00, 0x99, 0x06, 0x00, 0x00, 0x9a, 0x06, 0x00, 0x00, 0x9b, 0x06, + 0x00, 0x00, 0x9c, 0x06, 0x00, 0x00, 0x9d, 0x06, 0x00, 0x00, 0x9e, 0x06, 0x00, 0x00, 0x9f, 0x06, 0x00, + 0x00, 0xa0, 0x06, 0x00, 0x00, 0xa1, 0x06, 0x00, 0x00, 0xa2, 0x06, 0x00, 0x00, 0xa3, 0x06, 0x00, 0x00, + 0xa4, 0x06, 0x00, 0x00, 0xa5, 0x06, 0x00, 0x00, 0xa6, 0x06, 0x00, 0x00, 0xa7, 0x06, 0x00, 0x00, 0xa8, + 0x06, 0x00, 0x00, 0xa9, 0x06, 0x00, 0x00, 0xaa, 0x06, 0x00, 0x00, 0xab, 0x06, 0x00, 0x00, 0xac, 0x06, + 0x00, 0x00, 0xad, 0x06, 0x00, 0x00, 0xae, 0x06, 0x00, 0x00, 0xaf, 0x06, 0x00, 0x00, 0xb0, 0x06, 0x00, + 0x00, 0xb1, 0x06, 0x00, 0x00, 0xb2, 0x06, 0x00, 0x00, 0xb3, 0x06, 0x00, 0x00, 0xb4, 0x06, 0x00, 0x00, + 0xb5, 0x06, 0x00, 0x00, 0xb6, 0x06, 0x00, 0x00, 0xb7, 0x06, 0x00, 0x00, 0xb8, 0x06, 0x00, 0x00, 0xb9, + 0x06, 0x00, 0x00, 0xba, 0x06, 0x00, 0x00, 0xbb, 0x06, 0x00, 0x00, 0xbc, 0x06, 0x00, 0x00, 0xbd, 0x06, + 0x00, 0x00, 0xbe, 0x06, 0x00, 0x00, 0xbf, 0x06, 0x00, 0x00, 0xc0, 0x06, 0x00, 0x00, 0xc1, 0x06, 0x00, + 0x00, 0xc2, 0x06, 0x00, 0x00, 0xc3, 0x06, 0x00, 0x00, 0xc4, 0x06, 0x00, 0x00, 0xc5, 0x06, 0x00, 0x00, + 0xc6, 0x06, 0x00, 0x00, 0xc7, 0x06, 0x00, 0x00, 0xc8, 0x06, 0x00, 0x00, 0xc9, 0x06, 0x00, 0x00, 0xca, + 0x06, 0x00, 0x00, 0xcb, 0x06, 0x00, 0x00, 0xcc, 0x06, 0x00, 0x00, 0xcd, 0x06, 0x00, 0x00, 0xce, 0x06, + 0x00, 0x00, 0xcf, 0x06, 0x00, 0x00, 0xd0, 0x06, 0x00, 0x00, 0xd1, 0x06, 0x00, 0x00, 0xd2, 0x06, 0x00, + 0x00, 0xd3, 0x06, 0x00, 0x00, 0xd4, 0x06, 0x00, 0x00, 0xd5, 0x06, 0x00, 0x00, 0xd6, 0x06, 0x00, 0x00, + 0xd7, 0x06, 0x00, 0x00, 0xd8, 0x06, 0x00, 0x00, 0xd9, 0x06, 0x00, 0x00, 0xda, 0x06, 0x00, 0x00, 0xdb, + 0x06, 0x00, 0x00, 0xdc, 0x06, 0x00, 0x00, 0xdd, 0x06, 0x00, 0x00, 0xde, 0x06, 0x00, 0x00, 0xdf, 0x06, + 0x00, 0x00, 0xe0, 0x06, 0x00, 0x00, 0xe1, 0x06, 0x00, 0x00, 0xe2, 0x06, 0x00, 0x00, 0xe3, 0x06, 0x00, + 0x00, 0xe4, 0x06, 0x00, 0x00, 0xe5, 0x06, 0x00, 0x00, 0xe6, 0x06, 0x00, 0x00, 0xe7, 0x06, 0x00, 0x00, + 0xe8, 0x06, 0x00, 0x00, 0xe9, 0x06, 0x00, 0x00, 0xea, 0x06, 0x00, 0x00, 0xeb, 0x06, 0x00, 0x00, 0xec, + 0x06, 0x00, 0x00, 0xed, 0x06, 0x00, 0x00, 0xee, 0x06, 0x00, 0x00, 0xef, 0x06, 0x00, 0x00, 0xf0, 0x06, + 0x00, 0x00, 0xf1, 0x06, 0x00, 0x00, 0xf2, 0x06, 0x00, 0x00, 0xf3, 0x06, 0x00, 0x00, 0xf4, 0x06, 0x00, + 0x00, 0xf5, 0x06, 0x00, 0x00, 0xf6, 0x06, 0x00, 0x00, 0xf7, 0x06, 0x00, 0x00, 0xf8, 0x06, 0x00, 0x00, + 0xf9, 0x06, 0x00, 0x00, 0xfa, 0x06, 0x00, 0x00, 0xfb, 0x06, 0x00, 0x00, 0xfc, 0x06, 0x00, 0x00, 0xfd, + 0x06, 0x00, 0x00, 0xfe, 0x06, 0x00, 0x00, 0xff, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x01, 0x07, + 0x00, 0x00, 0x02, 0x07, 0x00, 0x00, 0x03, 0x07, 0x00, 0x00, 0x04, 0x07, 0x00, 0x00, 0x05, 0x07, 0x00, + 0x00, 0x06, 0x07, 0x00, 0x00, 0x07, 0x07, 0x00, 0x00, 0x08, 0x07, 0x00, 0x00, 0x09, 0x07, 0x00, 0x00, + 0x0a, 0x07, 0x00, 0x00, 0x0b, 0x07, 0x00, 0x00, 0x0c, 0x07, 0x00, 0x00, 0x0d, 0x07, 0x00, 0x00, 0x0e, + 0x07, 0x00, 0x00, 0x0f, 0x07, 0x00, 0x00, 0x10, 0x07, 0x00, 0x00, 0x11, 0x07, 0x00, 0x00, 0x12, 0x07, + 0x00, 0x00, 0x13, 0x07, 0x00, 0x00, 0x14, 0x07, 0x00, 0x00, 0x15, 0x07, 0x00, 0x00, 0x16, 0x07, 0x00, + 0x00, 0x17, 0x07, 0x00, 0x00, 0x18, 0x07, 0x00, 0x00, 0x19, 0x07, 0x00, 0x00, 0x1a, 0x07, 0x00, 0x00, + 0x1b, 0x07, 0x00, 0x00, 0x1c, 0x07, 0x00, 0x00, 0x1d, 0x07, 0x00, 0x00, 0x1e, 0x07, 0x00, 0x00, 0x1f, + 0x07, 0x00, 0x00, 0x20, 0x07, 0x00, 0x00, 0x21, 0x07, 0x00, 0x00, 0x22, 0x07, 0x00, 0x00, 0x23, 0x07, + 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, 0x25, 0x07, 0x00, 0x00, 0x26, 0x07, 0x00, 0x00, 0x27, 0x07, 0x00, + 0x00, 0x28, 0x07, 0x00, 0x00, 0x29, 0x07, 0x00, 0x00, 0x2a, 0x07, 0x00, 0x00, 0x2b, 0x07, 0x00, 0x00, + 0x2c, 0x07, 0x00, 0x00, 0x2d, 0x07, 0x00, 0x00, 0x2e, 0x07, 0x00, 0x00, 0x2f, 0x07, 0x00, 0x00, 0x30, + 0x07, 0x00, 0x00, 0x31, 0x07, 0x00, 0x00, 0x32, 0x07, 0x00, 0x00, 0x33, 0x07, 0x00, 0x00, 0x34, 0x07, + 0x00, 0x00, 0x35, 0x07, 0x00, 0x00, 0x36, 0x07, 0x00, 0x00, 0x37, 0x07, 0x00, 0x00, 0x38, 0x07, 0x00, + 0x00, 0x39, 0x07, 0x00, 0x00, 0x3a, 0x07, 0x00, 0x00, 0x3b, 0x07, 0x00, 0x00, 0x3c, 0x07, 0x00, 0x00, + 0x3d, 0x07, 0x00, 0x00, 0x3e, 0x07, 0x00, 0x00, 0x3f, 0x07, 0x00, 0x00, 0x40, 0x07, 0x00, 0x00, 0x41, + 0x07, 0x00, 0x00, 0x42, 0x07, 0x00, 0x00, 0x43, 0x07, 0x00, 0x00, 0x44, 0x07, 0x00, 0x00, 0x45, 0x07, + 0x00, 0x00, 0x46, 0x07, 0x00, 0x00, 0x47, 0x07, 0x00, 0x00, 0x48, 0x07, 0x00, 0x00, 0x49, 0x07, 0x00, + 0x00, 0x4a, 0x07, 0x00, 0x00, 0x4b, 0x07, 0x00, 0x00, 0x4c, 0x07, 0x00, 0x00, 0x4d, 0x07, 0x00, 0x00, + 0x4e, 0x07, 0x00, 0x00, 0x4f, 0x07, 0x00, 0x00, 0x50, 0x07, 0x00, 0x00, 0x51, 0x07, 0x00, 0x00, 0x52, + 0x07, 0x00, 0x00, 0x53, 0x07, 0x00, 0x00, 0x54, 0x07, 0x00, 0x00, 0x55, 0x07, 0x00, 0x00, 0x56, 0x07, + 0x00, 0x00, 0x57, 0x07, 0x00, 0x00, 0x58, 0x07, 0x00, 0x00, 0x59, 0x07, 0x00, 0x00, 0x5a, 0x07, 0x00, + 0x00, 0x5b, 0x07, 0x00, 0x00, 0x5c, 0x07, 0x00, 0x00, 0x5d, 0x07, 0x00, 0x00, 0x5e, 0x07, 0x00, 0x00, + 0x5f, 0x07, 0x00, 0x00, 0x60, 0x07, 0x00, 0x00, 0x61, 0x07, 0x00, 0x00, 0x62, 0x07, 0x00, 0x00, 0x63, + 0x07, 0x00, 0x00, 0x64, 0x07, 0x00, 0x00, 0x65, 0x07, 0x00, 0x00, 0x66, 0x07, 0x00, 0x00, 0x67, 0x07, + 0x00, 0x00, 0x68, 0x07, 0x00, 0x00, 0x69, 0x07, 0x00, 0x00, 0x6a, 0x07, 0x00, 0x00, 0x6b, 0x07, 0x00, + 0x00, 0x6c, 0x07, 0x00, 0x00, 0x6d, 0x07, 0x00, 0x00, 0x6e, 0x07, 0x00, 0x00, 0x6f, 0x07, 0x00, 0x00, + 0x70, 0x07, 0x00, 0x00, 0x71, 0x07, 0x00, 0x00, 0x72, 0x07, 0x00, 0x00, 0x73, 0x07, 0x00, 0x00, 0x74, + 0x07, 0x00, 0x00, 0x75, 0x07, 0x00, 0x00, 0x76, 0x07, 0x00, 0x00, 0x77, 0x07, 0x00, 0x00, 0x78, 0x07, + 0x00, 0x00, 0x79, 0x07, 0x00, 0x00, 0x7a, 0x07, 0x00, 0x00, 0x7b, 0x07, 0x00, 0x00, 0x7c, 0x07, 0x00, + 0x00, 0x7d, 0x07, 0x00, 0x00, 0x7e, 0x07, 0x00, 0x00, 0x7f, 0x07, 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, + 0x81, 0x07, 0x00, 0x00, 0x82, 0x07, 0x00, 0x00, 0x83, 0x07, 0x00, 0x00, 0x84, 0x07, 0x00, 0x00, 0x85, + 0x07, 0x00, 0x00, 0x86, 0x07, 0x00, 0x00, 0x87, 0x07, 0x00, 0x00, 0x88, 0x07, 0x00, 0x00, 0x89, 0x07, + 0x00, 0x00, 0x8a, 0x07, 0x00, 0x00, 0x8b, 0x07, 0x00, 0x00, 0x8c, 0x07, 0x00, 0x00, 0x8d, 0x07, 0x00, + 0x00, 0x8e, 0x07, 0x00, 0x00, 0x8f, 0x07, 0x00, 0x00, 0x90, 0x07, 0x00, 0x00, 0x91, 0x07, 0x00, 0x00, + 0x92, 0x07, 0x00, 0x00, 0x93, 0x07, 0x00, 0x00, 0x94, 0x07, 0x00, 0x00, 0x95, 0x07, 0x00, 0x00, 0x96, + 0x07, 0x00, 0x00, 0x97, 0x07, 0x00, 0x00, 0x98, 0x07, 0x00, 0x00, 0x99, 0x07, 0x00, 0x00, 0x9a, 0x07, + 0x00, 0x00, 0x9b, 0x07, 0x00, 0x00, 0x9c, 0x07, 0x00, 0x00, 0x9d, 0x07, 0x00, 0x00, 0x9e, 0x07, 0x00, + 0x00, 0x9f, 0x07, 0x00, 0x00, 0xa0, 0x07, 0x00, 0x00, 0xa1, 0x07, 0x00, 0x00, 0xa2, 0x07, 0x00, 0x00, + 0xa3, 0x07, 0x00, 0x00, 0xa4, 0x07, 0x00, 0x00, 0xa5, 0x07, 0x00, 0x00, 0xa6, 0x07, 0x00, 0x00, 0xa7, + 0x07, 0x00, 0x00, 0xa8, 0x07, 0x00, 0x00, 0xa9, 0x07, 0x00, 0x00, 0xaa, 0x07, 0x00, 0x00, 0xab, 0x07, + 0x00, 0x00, 0xac, 0x07, 0x00, 0x00, 0xad, 0x07, 0x00, 0x00, 0xae, 0x07, 0x00, 0x00, 0xaf, 0x07, 0x00, + 0x00, 0xb0, 0x07, 0x00, 0x00, 0xb1, 0x07, 0x00, 0x00, 0xb2, 0x07, 0x00, 0x00, 0xb3, 0x07, 0x00, 0x00, + 0xb4, 0x07, 0x00, 0x00, 0xb5, 0x07, 0x00, 0x00, 0xb6, 0x07, 0x00, 0x00, 0xb7, 0x07, 0x00, 0x00, 0xb8, + 0x07, 0x00, 0x00, 0xb9, 0x07, 0x00, 0x00, 0xba, 0x07, 0x00, 0x00, 0xbb, 0x07, 0x00, 0x00, 0xbc, 0x07, + 0x00, 0x00, 0xbd, 0x07, 0x00, 0x00, 0xbe, 0x07, 0x00, 0x00, 0xbf, 0x07, 0x00, 0x00, 0xc0, 0x07, 0x00, + 0x00, 0xc1, 0x07, 0x00, 0x00, 0xc2, 0x07, 0x00, 0x00, 0xc3, 0x07, 0x00, 0x00, 0xc4, 0x07, 0x00, 0x00, + 0xc5, 0x07, 0x00, 0x00, 0xc6, 0x07, 0x00, 0x00, 0xc7, 0x07, 0x00, 0x00, 0xc8, 0x07, 0x00, 0x00, 0xc9, + 0x07, 0x00, 0x00, 0xca, 0x07, 0x00, 0x00, 0xcb, 0x07, 0x00, 0x00, 0xcc, 0x07, 0x00, 0x00, 0xcd, 0x07, + 0x00, 0x00, 0xce, 0x07, 0x00, 0x00, 0xcf, 0x07, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xd1, 0x07, 0x00, + 0x00, 0xd2, 0x07, 0x00, 0x00, 0xd3, 0x07, 0x00, 0x00, 0xd4, 0x07, 0x00, 0x00, 0xd5, 0x07, 0x00, 0x00, + 0xd6, 0x07, 0x00, 0x00, 0xd7, 0x07, 0x00, 0x00, 0xd8, 0x07, 0x00, 0x00, 0xd9, 0x07, 0x00, 0x00, 0xda, + 0x07, 0x00, 0x00, 0xdb, 0x07, 0x00, 0x00, 0xdc, 0x07, 0x00, 0x00, 0xdd, 0x07, 0x00, 0x00, 0xde, 0x07, + 0x00, 0x00, 0xdf, 0x07, 0x00, 0x00, 0xe0, 0x07, 0x00, 0x00, 0xe1, 0x07, 0x00, 0x00, 0xe2, 0x07, 0x00, + 0x00, 0xe3, 0x07, 0x00, 0x00, 0xe4, 0x07, 0x00, 0x00, 0xe5, 0x07, 0x00, 0x00, 0xe6, 0x07, 0x00, 0x00, + 0xe7, 0x07, 0x00, 0x00, 0xe8, 0x07, 0x00, 0x00, 0xe9, 0x07, 0x00, 0x00, 0xea, 0x07, 0x00, 0x00, 0xeb, + 0x07, 0x00, 0x00, 0xec, 0x07, 0x00, 0x00, 0xed, 0x07, 0x00, 0x00, 0xee, 0x07, 0x00, 0x00, 0xef, 0x07, + 0x00, 0x00, 0xf0, 0x07, 0x00, 0x00, 0xf1, 0x07, 0x00, 0x00, 0xf2, 0x07, 0x00, 0x00, 0xf3, 0x07, 0x00, + 0x00, 0xf4, 0x07, 0x00, 0x00, 0xf5, 0x07, 0x00, 0x00, 0xf6, 0x07, 0x00, 0x00, 0xf7, 0x07, 0x00, 0x00, + 0xf8, 0x07, 0x00, 0x00, 0xf9, 0x07, 0x00, 0x00, 0xfa, 0x07, 0x00, 0x00, 0xfb, 0x07, 0x00, 0x00, 0xfc, + 0x07, 0x00, 0x00, 0xfd, 0x07, 0x00, 0x00, 0xfe, 0x07, 0x00, 0x00, 0xff, 0x07, 0x00, 0x00}; unsigned int __col4block_cdf_len = 8192; constexpr int __col4block_cdf_umin = 0; constexpr int __col4block_cdf_umax = 2047; diff --git a/tests/col8block.h b/tests/col8block.h index 7cf0de38a..c03b9106e 100644 --- a/tests/col8block.h +++ b/tests/col8block.h @@ -18,690 +18,488 @@ #ifndef HAVE_COL8BLOCK #define HAVE_COL8BLOCK unsigned char ___bin_col8block_cdf[] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xab, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xba, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xde, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xec, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x14, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x22, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x23, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x25, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x26, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x29, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x31, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x32, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x34, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x35, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x37, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x38, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x44, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x46, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x47, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x52, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x53, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x55, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x56, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x58, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x59, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x61, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x62, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x65, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x68, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x71, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x73, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x76, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x77, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x83, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x84, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x85, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x86, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x88, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x89, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x91, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x92, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x94, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x95, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x97, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x98, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa2, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa5, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xaa, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xad, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xaf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbe, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc9, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xca, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xce, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd5, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xda, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdb, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xea, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xeb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xec, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xee, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xef, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfa, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfe, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x07, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x13, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x16, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x19, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x21, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x22, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x25, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x27, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x28, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x31, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x33, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x34, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x36, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x37, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x39, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x42, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x43, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x45, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x46, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x49, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x51, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x52, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x54, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x55, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x58, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x60, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x61, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x63, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x64, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x66, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x67, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x69, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x70, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x73, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x75, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x76, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x78, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x79, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x81, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x82, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x84, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x85, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x88, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x91, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x93, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x94, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x96, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x97, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x99, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa7, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xab, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xac, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xae, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xaf, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb3, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb6, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xba, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbd, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbe, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xca, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcd, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcf, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xda, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xde, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdf, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe9, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xea, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xeb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xed, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xee, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfa, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfd, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x05, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x06, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x08, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x09, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x11, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x12, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x17, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x18, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x21, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x23, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x24, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x26, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x27, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x29, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x2f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x30, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x32, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x35, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x36, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x38, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x39, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x41, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x42, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x45, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x47, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x48, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x51, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x53, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x54, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x56, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x57, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x59, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x60, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x62, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x63, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x65, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x66, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x68, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x69, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x71, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x72, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x75, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x77, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x78, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x7e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x81, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x83, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x84, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x86, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x87, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x89, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x8d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x8f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x90, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x92, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x93, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x95, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x96, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x98, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x99, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x9e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xa7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xa8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa9, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xaa, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xab, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xad, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xae, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xb9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xba, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbc, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xbd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xbf, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc7, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xca, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xcb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcc, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xce, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xcf, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xd8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xdb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xde, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xea, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xed, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xef, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xf9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfc, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xbb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xca, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe7, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xec, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x1f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x21, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x2e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x32, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x3f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x41, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x4e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x52, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x54, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x5f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x61, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x63, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x65, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x72, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x76, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x81, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x83, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x84, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x85, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x87, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x92, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x94, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x96, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x98, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa2, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb3, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xba, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xca, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd5, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xea, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xeb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xed, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf6, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xfa, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfc, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x1e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x29, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x2d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x31, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x3e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x42, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x4d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x5e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x62, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x64, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x6f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x71, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x73, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x75, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x82, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x84, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x86, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x91, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x93, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x95, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x97, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb4, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xca, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xda, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe6, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe7, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xea, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xec, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf5, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf9, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfb, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfd, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x17, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x19, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x1d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x2c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x39, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x3d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x3f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x41, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x4c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x52, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x5d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x5f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x61, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x6e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x72, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x7f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x81, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x83, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x84, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x85, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x8e, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x90, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x92, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x94, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x96, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9c, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x9f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa2, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa9, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb3, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc4, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xca, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd5, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe5, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xea, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xeb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf1, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf5, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf6, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfa, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfc, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; unsigned int ___bin_col8block_cdf_len = 8192; constexpr int __col8block_cdf_umin = 0; constexpr int __col8block_cdf_umax = 1023; diff --git a/tests/col_double_block.h b/tests/col_double_block.h index 90012e2b7..78c43eb6e 100644 --- a/tests/col_double_block.h +++ b/tests/col_double_block.h @@ -18,690 +18,487 @@ #ifndef HAVE_COL_DOUBLE_BLOCK #define HAVE_COL_DOUBLE_BLOCK unsigned char ___bin_col_double_block_cdf[] = { - 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xfa, 0xff, 0xaa, 0xaa, 0xaa, 0xaa, - 0xaa, 0xaa, 0xfa, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x0c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x14, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x1a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x23, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x26, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x29, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x2c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x2f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x30, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x31, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x31, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x33, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x34, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x34, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x36, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x37, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x37, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x39, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x3a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x3d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x42, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x43, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x44, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x45, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x46, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x47, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x48, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x49, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x50, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x51, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x51, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x51, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x52, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x52, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x53, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x53, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x54, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x54, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x54, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x55, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x55, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x56, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x56, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x57, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x57, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x57, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x58, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x58, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x59, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x59, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x80, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x80, 0x40 -}; + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xfa, 0xff, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xfa, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x16, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x1a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x2c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x30, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x31, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x33, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x34, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x36, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x37, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x39, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x3f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x41, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x42, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x45, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x45, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x46, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x46, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x47, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x49, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4f, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4f, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x51, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x51, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x51, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x60, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, + 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x53, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x53, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x53, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x53, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x54, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x55, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x55, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x55, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x56, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x57, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x57, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x57, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x58, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x58, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x59, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x59, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, + 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5e, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, + 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x60, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x60, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x60, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x60, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, + 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x61, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x61, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x61, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x61, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x90, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, + 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x62, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x62, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x62, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x62, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x62, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x63, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x63, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x63, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x63, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x90, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x63, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x64, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x64, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x64, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x64, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x64, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x65, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x65, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x65, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x90, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x65, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x65, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x66, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x66, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x66, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x66, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x66, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x67, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x67, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x67, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x67, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x67, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, + 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x68, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x68, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x68, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x68, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x68, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, + 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x69, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x69, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x69, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x69, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, + 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x30, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, + 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x60, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, + 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6e, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, + 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6f, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6f, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6f, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x70, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x70, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x28, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x70, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x38, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x48, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x58, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, + 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x70, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x70, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x70, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x70, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x70, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, + 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x71, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x71, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x71, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x38, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x71, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x48, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x58, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x68, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, + 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x71, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x71, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x71, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x71, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x71, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x72, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x72, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x72, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x72, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x48, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x72, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x58, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x68, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, + 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x72, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x72, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x72, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x72, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x72, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x73, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x73, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x73, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x73, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x58, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x73, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x68, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x78, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, + 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x73, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x73, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x73, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x73, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x73, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x74, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x74, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x74, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x58, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x74, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x68, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x74, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x78, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x88, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x98, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, + 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x74, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x74, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x74, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x74, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x74, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x75, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x75, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x75, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x68, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x75, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x78, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x75, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x88, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x98, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, + 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x75, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x75, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x75, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x75, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x76, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x76, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x76, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x76, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x78, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x76, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x88, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x76, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x98, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, + 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x76, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x76, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x76, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x77, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x77, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, + 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x77, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x77, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x77, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x88, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x77, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x98, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x77, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, + 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x77, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x77, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x78, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x78, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x78, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x30, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, + 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x78, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x78, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x78, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x98, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x78, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x78, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, + 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x78, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x79, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x79, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x79, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x79, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, + 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x79, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x79, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x79, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x79, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x79, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x60, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, + 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, + 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, + 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x08, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x18, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, + 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x90, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, + 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x18, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, + 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7e, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x90, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7f, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x18, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x28, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x38, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, + 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7f, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7f, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x90, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7f, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, + 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x80, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x80, 0x40}; unsigned int ___bin_col_double_block_cdf_len = 8192; #endif - diff --git a/tests/col_float_block.h b/tests/col_float_block.h index 8f6b577b6..fe418486e 100644 --- a/tests/col_float_block.h +++ b/tests/col_float_block.h @@ -18,689 +18,487 @@ #ifndef HAVE_COL_FLOAT_BLOCK #define HAVE_COL_FLOAT_BLOCK unsigned char ___bin_col_float_block_cdf[] = { - 0x00, 0x00, 0xc0, 0x3f, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x20, 0x40, - 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x80, 0x40, - 0x00, 0x00, 0x90, 0x40, 0x00, 0x00, 0xa0, 0x40, 0x00, 0x00, 0xb0, 0x40, - 0x00, 0x00, 0xc0, 0x40, 0x00, 0x00, 0xd0, 0x40, 0x00, 0x00, 0xe0, 0x40, - 0x00, 0x00, 0xf0, 0x40, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x08, 0x41, - 0x00, 0x00, 0x10, 0x41, 0x00, 0x00, 0x18, 0x41, 0x00, 0x00, 0x20, 0x41, - 0x00, 0x00, 0x28, 0x41, 0x00, 0x00, 0x30, 0x41, 0x00, 0x00, 0x38, 0x41, - 0x00, 0x00, 0x40, 0x41, 0x00, 0x00, 0x48, 0x41, 0x00, 0x00, 0x50, 0x41, - 0x00, 0x00, 0x58, 0x41, 0x00, 0x00, 0x60, 0x41, 0x00, 0x00, 0x68, 0x41, - 0x00, 0x00, 0x70, 0x41, 0x00, 0x00, 0x78, 0x41, 0x00, 0x00, 0x80, 0x41, - 0x00, 0x00, 0x84, 0x41, 0x00, 0x00, 0x88, 0x41, 0x00, 0x00, 0x8c, 0x41, - 0x00, 0x00, 0x90, 0x41, 0x00, 0x00, 0x94, 0x41, 0x00, 0x00, 0x98, 0x41, - 0x00, 0x00, 0x9c, 0x41, 0x00, 0x00, 0xa0, 0x41, 0x00, 0x00, 0xa4, 0x41, - 0x00, 0x00, 0xa8, 0x41, 0x00, 0x00, 0xac, 0x41, 0x00, 0x00, 0xb0, 0x41, - 0x00, 0x00, 0xb4, 0x41, 0x00, 0x00, 0xb8, 0x41, 0x00, 0x00, 0xbc, 0x41, - 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xc4, 0x41, 0x00, 0x00, 0xc8, 0x41, - 0x00, 0x00, 0xcc, 0x41, 0x00, 0x00, 0xd0, 0x41, 0x00, 0x00, 0xd4, 0x41, - 0x00, 0x00, 0xd8, 0x41, 0x00, 0x00, 0xdc, 0x41, 0x00, 0x00, 0xe0, 0x41, - 0x00, 0x00, 0xe4, 0x41, 0x00, 0x00, 0xe8, 0x41, 0x00, 0x00, 0xec, 0x41, - 0x00, 0x00, 0xf0, 0x41, 0x00, 0x00, 0xf4, 0x41, 0x00, 0x00, 0xf8, 0x41, - 0x00, 0x00, 0xfc, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x02, 0x42, - 0x00, 0x00, 0x04, 0x42, 0x00, 0x00, 0x06, 0x42, 0x00, 0x00, 0x08, 0x42, - 0x00, 0x00, 0x0a, 0x42, 0x00, 0x00, 0x0c, 0x42, 0x00, 0x00, 0x0e, 0x42, - 0x00, 0x00, 0x10, 0x42, 0x00, 0x00, 0x12, 0x42, 0x00, 0x00, 0x14, 0x42, - 0x00, 0x00, 0x16, 0x42, 0x00, 0x00, 0x18, 0x42, 0x00, 0x00, 0x1a, 0x42, - 0x00, 0x00, 0x1c, 0x42, 0x00, 0x00, 0x1e, 0x42, 0x00, 0x00, 0x20, 0x42, - 0x00, 0x00, 0x22, 0x42, 0x00, 0x00, 0x24, 0x42, 0x00, 0x00, 0x26, 0x42, - 0x00, 0x00, 0x28, 0x42, 0x00, 0x00, 0x2a, 0x42, 0x00, 0x00, 0x2c, 0x42, - 0x00, 0x00, 0x2e, 0x42, 0x00, 0x00, 0x30, 0x42, 0x00, 0x00, 0x32, 0x42, - 0x00, 0x00, 0x34, 0x42, 0x00, 0x00, 0x36, 0x42, 0x00, 0x00, 0x38, 0x42, - 0x00, 0x00, 0x3a, 0x42, 0x00, 0x00, 0x3c, 0x42, 0x00, 0x00, 0x3e, 0x42, - 0x00, 0x00, 0x40, 0x42, 0x00, 0x00, 0x42, 0x42, 0x00, 0x00, 0x44, 0x42, - 0x00, 0x00, 0x46, 0x42, 0x00, 0x00, 0x48, 0x42, 0x00, 0x00, 0x4a, 0x42, - 0x00, 0x00, 0x4c, 0x42, 0x00, 0x00, 0x4e, 0x42, 0x00, 0x00, 0x50, 0x42, - 0x00, 0x00, 0x52, 0x42, 0x00, 0x00, 0x54, 0x42, 0x00, 0x00, 0x56, 0x42, - 0x00, 0x00, 0x58, 0x42, 0x00, 0x00, 0x5a, 0x42, 0x00, 0x00, 0x5c, 0x42, - 0x00, 0x00, 0x5e, 0x42, 0x00, 0x00, 0x60, 0x42, 0x00, 0x00, 0x62, 0x42, - 0x00, 0x00, 0x64, 0x42, 0x00, 0x00, 0x66, 0x42, 0x00, 0x00, 0x68, 0x42, - 0x00, 0x00, 0x6a, 0x42, 0x00, 0x00, 0x6c, 0x42, 0x00, 0x00, 0x6e, 0x42, - 0x00, 0x00, 0x70, 0x42, 0x00, 0x00, 0x72, 0x42, 0x00, 0x00, 0x74, 0x42, - 0x00, 0x00, 0x76, 0x42, 0x00, 0x00, 0x78, 0x42, 0x00, 0x00, 0x7a, 0x42, - 0x00, 0x00, 0x7c, 0x42, 0x00, 0x00, 0x7e, 0x42, 0x00, 0x00, 0x80, 0x42, - 0x00, 0x00, 0x81, 0x42, 0x00, 0x00, 0x82, 0x42, 0x00, 0x00, 0x83, 0x42, - 0x00, 0x00, 0x84, 0x42, 0x00, 0x00, 0x85, 0x42, 0x00, 0x00, 0x86, 0x42, - 0x00, 0x00, 0x87, 0x42, 0x00, 0x00, 0x88, 0x42, 0x00, 0x00, 0x89, 0x42, - 0x00, 0x00, 0x8a, 0x42, 0x00, 0x00, 0x8b, 0x42, 0x00, 0x00, 0x8c, 0x42, - 0x00, 0x00, 0x8d, 0x42, 0x00, 0x00, 0x8e, 0x42, 0x00, 0x00, 0x8f, 0x42, - 0x00, 0x00, 0x90, 0x42, 0x00, 0x00, 0x91, 0x42, 0x00, 0x00, 0x92, 0x42, - 0x00, 0x00, 0x93, 0x42, 0x00, 0x00, 0x94, 0x42, 0x00, 0x00, 0x95, 0x42, - 0x00, 0x00, 0x96, 0x42, 0x00, 0x00, 0x97, 0x42, 0x00, 0x00, 0x98, 0x42, - 0x00, 0x00, 0x99, 0x42, 0x00, 0x00, 0x9a, 0x42, 0x00, 0x00, 0x9b, 0x42, - 0x00, 0x00, 0x9c, 0x42, 0x00, 0x00, 0x9d, 0x42, 0x00, 0x00, 0x9e, 0x42, - 0x00, 0x00, 0x9f, 0x42, 0x00, 0x00, 0xa0, 0x42, 0x00, 0x00, 0xa1, 0x42, - 0x00, 0x00, 0xa2, 0x42, 0x00, 0x00, 0xa3, 0x42, 0x00, 0x00, 0xa4, 0x42, - 0x00, 0x00, 0xa5, 0x42, 0x00, 0x00, 0xa6, 0x42, 0x00, 0x00, 0xa7, 0x42, - 0x00, 0x00, 0xa8, 0x42, 0x00, 0x00, 0xa9, 0x42, 0x00, 0x00, 0xaa, 0x42, - 0x00, 0x00, 0xab, 0x42, 0x00, 0x00, 0xac, 0x42, 0x00, 0x00, 0xad, 0x42, - 0x00, 0x00, 0xae, 0x42, 0x00, 0x00, 0xaf, 0x42, 0x00, 0x00, 0xb0, 0x42, - 0x00, 0x00, 0xb1, 0x42, 0x00, 0x00, 0xb2, 0x42, 0x00, 0x00, 0xb3, 0x42, - 0x00, 0x00, 0xb4, 0x42, 0x00, 0x00, 0xb5, 0x42, 0x00, 0x00, 0xb6, 0x42, - 0x00, 0x00, 0xb7, 0x42, 0x00, 0x00, 0xb8, 0x42, 0x00, 0x00, 0xb9, 0x42, - 0x00, 0x00, 0xba, 0x42, 0x00, 0x00, 0xbb, 0x42, 0x00, 0x00, 0xbc, 0x42, - 0x00, 0x00, 0xbd, 0x42, 0x00, 0x00, 0xbe, 0x42, 0x00, 0x00, 0xbf, 0x42, - 0x00, 0x00, 0xc0, 0x42, 0x00, 0x00, 0xc1, 0x42, 0x00, 0x00, 0xc2, 0x42, - 0x00, 0x00, 0xc3, 0x42, 0x00, 0x00, 0xc4, 0x42, 0x00, 0x00, 0xc5, 0x42, - 0x00, 0x00, 0xc6, 0x42, 0x00, 0x00, 0xc7, 0x42, 0x00, 0x00, 0xc8, 0x42, - 0x00, 0x00, 0xc9, 0x42, 0x00, 0x00, 0xca, 0x42, 0x00, 0x00, 0xcb, 0x42, - 0x00, 0x00, 0xcc, 0x42, 0x00, 0x00, 0xcd, 0x42, 0x00, 0x00, 0xce, 0x42, - 0x00, 0x00, 0xcf, 0x42, 0x00, 0x00, 0xd0, 0x42, 0x00, 0x00, 0xd1, 0x42, - 0x00, 0x00, 0xd2, 0x42, 0x00, 0x00, 0xd3, 0x42, 0x00, 0x00, 0xd4, 0x42, - 0x00, 0x00, 0xd5, 0x42, 0x00, 0x00, 0xd6, 0x42, 0x00, 0x00, 0xd7, 0x42, - 0x00, 0x00, 0xd8, 0x42, 0x00, 0x00, 0xd9, 0x42, 0x00, 0x00, 0xda, 0x42, - 0x00, 0x00, 0xdb, 0x42, 0x00, 0x00, 0xdc, 0x42, 0x00, 0x00, 0xdd, 0x42, - 0x00, 0x00, 0xde, 0x42, 0x00, 0x00, 0xdf, 0x42, 0x00, 0x00, 0xe0, 0x42, - 0x00, 0x00, 0xe1, 0x42, 0x00, 0x00, 0xe2, 0x42, 0x00, 0x00, 0xe3, 0x42, - 0x00, 0x00, 0xe4, 0x42, 0x00, 0x00, 0xe5, 0x42, 0x00, 0x00, 0xe6, 0x42, - 0x00, 0x00, 0xe7, 0x42, 0x00, 0x00, 0xe8, 0x42, 0x00, 0x00, 0xe9, 0x42, - 0x00, 0x00, 0xea, 0x42, 0x00, 0x00, 0xeb, 0x42, 0x00, 0x00, 0xec, 0x42, - 0x00, 0x00, 0xed, 0x42, 0x00, 0x00, 0xee, 0x42, 0x00, 0x00, 0xef, 0x42, - 0x00, 0x00, 0xf0, 0x42, 0x00, 0x00, 0xf1, 0x42, 0x00, 0x00, 0xf2, 0x42, - 0x00, 0x00, 0xf3, 0x42, 0x00, 0x00, 0xf4, 0x42, 0x00, 0x00, 0xf5, 0x42, - 0x00, 0x00, 0xf6, 0x42, 0x00, 0x00, 0xf7, 0x42, 0x00, 0x00, 0xf8, 0x42, - 0x00, 0x00, 0xf9, 0x42, 0x00, 0x00, 0xfa, 0x42, 0x00, 0x00, 0xfb, 0x42, - 0x00, 0x00, 0xfc, 0x42, 0x00, 0x00, 0xfd, 0x42, 0x00, 0x00, 0xfe, 0x42, - 0x00, 0x00, 0xff, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x80, 0x00, 0x43, - 0x00, 0x00, 0x01, 0x43, 0x00, 0x80, 0x01, 0x43, 0x00, 0x00, 0x02, 0x43, - 0x00, 0x80, 0x02, 0x43, 0x00, 0x00, 0x03, 0x43, 0x00, 0x80, 0x03, 0x43, - 0x00, 0x00, 0x04, 0x43, 0x00, 0x80, 0x04, 0x43, 0x00, 0x00, 0x05, 0x43, - 0x00, 0x80, 0x05, 0x43, 0x00, 0x00, 0x06, 0x43, 0x00, 0x80, 0x06, 0x43, - 0x00, 0x00, 0x07, 0x43, 0x00, 0x80, 0x07, 0x43, 0x00, 0x00, 0x08, 0x43, - 0x00, 0x80, 0x08, 0x43, 0x00, 0x00, 0x09, 0x43, 0x00, 0x80, 0x09, 0x43, - 0x00, 0x00, 0x0a, 0x43, 0x00, 0x80, 0x0a, 0x43, 0x00, 0x00, 0x0b, 0x43, - 0x00, 0x80, 0x0b, 0x43, 0x00, 0x00, 0x0c, 0x43, 0x00, 0x80, 0x0c, 0x43, - 0x00, 0x00, 0x0d, 0x43, 0x00, 0x80, 0x0d, 0x43, 0x00, 0x00, 0x0e, 0x43, - 0x00, 0x80, 0x0e, 0x43, 0x00, 0x00, 0x0f, 0x43, 0x00, 0x80, 0x0f, 0x43, - 0x00, 0x00, 0x10, 0x43, 0x00, 0x80, 0x10, 0x43, 0x00, 0x00, 0x11, 0x43, - 0x00, 0x80, 0x11, 0x43, 0x00, 0x00, 0x12, 0x43, 0x00, 0x80, 0x12, 0x43, - 0x00, 0x00, 0x13, 0x43, 0x00, 0x80, 0x13, 0x43, 0x00, 0x00, 0x14, 0x43, - 0x00, 0x80, 0x14, 0x43, 0x00, 0x00, 0x15, 0x43, 0x00, 0x80, 0x15, 0x43, - 0x00, 0x00, 0x16, 0x43, 0x00, 0x80, 0x16, 0x43, 0x00, 0x00, 0x17, 0x43, - 0x00, 0x80, 0x17, 0x43, 0x00, 0x00, 0x18, 0x43, 0x00, 0x80, 0x18, 0x43, - 0x00, 0x00, 0x19, 0x43, 0x00, 0x80, 0x19, 0x43, 0x00, 0x00, 0x1a, 0x43, - 0x00, 0x80, 0x1a, 0x43, 0x00, 0x00, 0x1b, 0x43, 0x00, 0x80, 0x1b, 0x43, - 0x00, 0x00, 0x1c, 0x43, 0x00, 0x80, 0x1c, 0x43, 0x00, 0x00, 0x1d, 0x43, - 0x00, 0x80, 0x1d, 0x43, 0x00, 0x00, 0x1e, 0x43, 0x00, 0x80, 0x1e, 0x43, - 0x00, 0x00, 0x1f, 0x43, 0x00, 0x80, 0x1f, 0x43, 0x00, 0x00, 0x20, 0x43, - 0x00, 0x80, 0x20, 0x43, 0x00, 0x00, 0x21, 0x43, 0x00, 0x80, 0x21, 0x43, - 0x00, 0x00, 0x22, 0x43, 0x00, 0x80, 0x22, 0x43, 0x00, 0x00, 0x23, 0x43, - 0x00, 0x80, 0x23, 0x43, 0x00, 0x00, 0x24, 0x43, 0x00, 0x80, 0x24, 0x43, - 0x00, 0x00, 0x25, 0x43, 0x00, 0x80, 0x25, 0x43, 0x00, 0x00, 0x26, 0x43, - 0x00, 0x80, 0x26, 0x43, 0x00, 0x00, 0x27, 0x43, 0x00, 0x80, 0x27, 0x43, - 0x00, 0x00, 0x28, 0x43, 0x00, 0x80, 0x28, 0x43, 0x00, 0x00, 0x29, 0x43, - 0x00, 0x80, 0x29, 0x43, 0x00, 0x00, 0x2a, 0x43, 0x00, 0x80, 0x2a, 0x43, - 0x00, 0x00, 0x2b, 0x43, 0x00, 0x80, 0x2b, 0x43, 0x00, 0x00, 0x2c, 0x43, - 0x00, 0x80, 0x2c, 0x43, 0x00, 0x00, 0x2d, 0x43, 0x00, 0x80, 0x2d, 0x43, - 0x00, 0x00, 0x2e, 0x43, 0x00, 0x80, 0x2e, 0x43, 0x00, 0x00, 0x2f, 0x43, - 0x00, 0x80, 0x2f, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x80, 0x30, 0x43, - 0x00, 0x00, 0x31, 0x43, 0x00, 0x80, 0x31, 0x43, 0x00, 0x00, 0x32, 0x43, - 0x00, 0x80, 0x32, 0x43, 0x00, 0x00, 0x33, 0x43, 0x00, 0x80, 0x33, 0x43, - 0x00, 0x00, 0x34, 0x43, 0x00, 0x80, 0x34, 0x43, 0x00, 0x00, 0x35, 0x43, - 0x00, 0x80, 0x35, 0x43, 0x00, 0x00, 0x36, 0x43, 0x00, 0x80, 0x36, 0x43, - 0x00, 0x00, 0x37, 0x43, 0x00, 0x80, 0x37, 0x43, 0x00, 0x00, 0x38, 0x43, - 0x00, 0x80, 0x38, 0x43, 0x00, 0x00, 0x39, 0x43, 0x00, 0x80, 0x39, 0x43, - 0x00, 0x00, 0x3a, 0x43, 0x00, 0x80, 0x3a, 0x43, 0x00, 0x00, 0x3b, 0x43, - 0x00, 0x80, 0x3b, 0x43, 0x00, 0x00, 0x3c, 0x43, 0x00, 0x80, 0x3c, 0x43, - 0x00, 0x00, 0x3d, 0x43, 0x00, 0x80, 0x3d, 0x43, 0x00, 0x00, 0x3e, 0x43, - 0x00, 0x80, 0x3e, 0x43, 0x00, 0x00, 0x3f, 0x43, 0x00, 0x80, 0x3f, 0x43, - 0x00, 0x00, 0x40, 0x43, 0x00, 0x80, 0x40, 0x43, 0x00, 0x00, 0x41, 0x43, - 0x00, 0x80, 0x41, 0x43, 0x00, 0x00, 0x42, 0x43, 0x00, 0x80, 0x42, 0x43, - 0x00, 0x00, 0x43, 0x43, 0x00, 0x80, 0x43, 0x43, 0x00, 0x00, 0x44, 0x43, - 0x00, 0x80, 0x44, 0x43, 0x00, 0x00, 0x45, 0x43, 0x00, 0x80, 0x45, 0x43, - 0x00, 0x00, 0x46, 0x43, 0x00, 0x80, 0x46, 0x43, 0x00, 0x00, 0x47, 0x43, - 0x00, 0x80, 0x47, 0x43, 0x00, 0x00, 0x48, 0x43, 0x00, 0x80, 0x48, 0x43, - 0x00, 0x00, 0x49, 0x43, 0x00, 0x80, 0x49, 0x43, 0x00, 0x00, 0x4a, 0x43, - 0x00, 0x80, 0x4a, 0x43, 0x00, 0x00, 0x4b, 0x43, 0x00, 0x80, 0x4b, 0x43, - 0x00, 0x00, 0x4c, 0x43, 0x00, 0x80, 0x4c, 0x43, 0x00, 0x00, 0x4d, 0x43, - 0x00, 0x80, 0x4d, 0x43, 0x00, 0x00, 0x4e, 0x43, 0x00, 0x80, 0x4e, 0x43, - 0x00, 0x00, 0x4f, 0x43, 0x00, 0x80, 0x4f, 0x43, 0x00, 0x00, 0x50, 0x43, - 0x00, 0x80, 0x50, 0x43, 0x00, 0x00, 0x51, 0x43, 0x00, 0x80, 0x51, 0x43, - 0x00, 0x00, 0x52, 0x43, 0x00, 0x80, 0x52, 0x43, 0x00, 0x00, 0x53, 0x43, - 0x00, 0x80, 0x53, 0x43, 0x00, 0x00, 0x54, 0x43, 0x00, 0x80, 0x54, 0x43, - 0x00, 0x00, 0x55, 0x43, 0x00, 0x80, 0x55, 0x43, 0x00, 0x00, 0x56, 0x43, - 0x00, 0x80, 0x56, 0x43, 0x00, 0x00, 0x57, 0x43, 0x00, 0x80, 0x57, 0x43, - 0x00, 0x00, 0x58, 0x43, 0x00, 0x80, 0x58, 0x43, 0x00, 0x00, 0x59, 0x43, - 0x00, 0x80, 0x59, 0x43, 0x00, 0x00, 0x5a, 0x43, 0x00, 0x80, 0x5a, 0x43, - 0x00, 0x00, 0x5b, 0x43, 0x00, 0x80, 0x5b, 0x43, 0x00, 0x00, 0x5c, 0x43, - 0x00, 0x80, 0x5c, 0x43, 0x00, 0x00, 0x5d, 0x43, 0x00, 0x80, 0x5d, 0x43, - 0x00, 0x00, 0x5e, 0x43, 0x00, 0x80, 0x5e, 0x43, 0x00, 0x00, 0x5f, 0x43, - 0x00, 0x80, 0x5f, 0x43, 0x00, 0x00, 0x60, 0x43, 0x00, 0x80, 0x60, 0x43, - 0x00, 0x00, 0x61, 0x43, 0x00, 0x80, 0x61, 0x43, 0x00, 0x00, 0x62, 0x43, - 0x00, 0x80, 0x62, 0x43, 0x00, 0x00, 0x63, 0x43, 0x00, 0x80, 0x63, 0x43, - 0x00, 0x00, 0x64, 0x43, 0x00, 0x80, 0x64, 0x43, 0x00, 0x00, 0x65, 0x43, - 0x00, 0x80, 0x65, 0x43, 0x00, 0x00, 0x66, 0x43, 0x00, 0x80, 0x66, 0x43, - 0x00, 0x00, 0x67, 0x43, 0x00, 0x80, 0x67, 0x43, 0x00, 0x00, 0x68, 0x43, - 0x00, 0x80, 0x68, 0x43, 0x00, 0x00, 0x69, 0x43, 0x00, 0x80, 0x69, 0x43, - 0x00, 0x00, 0x6a, 0x43, 0x00, 0x80, 0x6a, 0x43, 0x00, 0x00, 0x6b, 0x43, - 0x00, 0x80, 0x6b, 0x43, 0x00, 0x00, 0x6c, 0x43, 0x00, 0x80, 0x6c, 0x43, - 0x00, 0x00, 0x6d, 0x43, 0x00, 0x80, 0x6d, 0x43, 0x00, 0x00, 0x6e, 0x43, - 0x00, 0x80, 0x6e, 0x43, 0x00, 0x00, 0x6f, 0x43, 0x00, 0x80, 0x6f, 0x43, - 0x00, 0x00, 0x70, 0x43, 0x00, 0x80, 0x70, 0x43, 0x00, 0x00, 0x71, 0x43, - 0x00, 0x80, 0x71, 0x43, 0x00, 0x00, 0x72, 0x43, 0x00, 0x80, 0x72, 0x43, - 0x00, 0x00, 0x73, 0x43, 0x00, 0x80, 0x73, 0x43, 0x00, 0x00, 0x74, 0x43, - 0x00, 0x80, 0x74, 0x43, 0x00, 0x00, 0x75, 0x43, 0x00, 0x80, 0x75, 0x43, - 0x00, 0x00, 0x76, 0x43, 0x00, 0x80, 0x76, 0x43, 0x00, 0x00, 0x77, 0x43, - 0x00, 0x80, 0x77, 0x43, 0x00, 0x00, 0x78, 0x43, 0x00, 0x80, 0x78, 0x43, - 0x00, 0x00, 0x79, 0x43, 0x00, 0x80, 0x79, 0x43, 0x00, 0x00, 0x7a, 0x43, - 0x00, 0x80, 0x7a, 0x43, 0x00, 0x00, 0x7b, 0x43, 0x00, 0x80, 0x7b, 0x43, - 0x00, 0x00, 0x7c, 0x43, 0x00, 0x80, 0x7c, 0x43, 0x00, 0x00, 0x7d, 0x43, - 0x00, 0x80, 0x7d, 0x43, 0x00, 0x00, 0x7e, 0x43, 0x00, 0x80, 0x7e, 0x43, - 0x00, 0x00, 0x7f, 0x43, 0x00, 0x80, 0x7f, 0x43, 0x00, 0x00, 0x80, 0x43, - 0x00, 0x40, 0x80, 0x43, 0x00, 0x80, 0x80, 0x43, 0x00, 0xc0, 0x80, 0x43, - 0x00, 0x00, 0x81, 0x43, 0x00, 0x40, 0x81, 0x43, 0x00, 0x80, 0x81, 0x43, - 0x00, 0xc0, 0x81, 0x43, 0x00, 0x00, 0x82, 0x43, 0x00, 0x40, 0x82, 0x43, - 0x00, 0x80, 0x82, 0x43, 0x00, 0xc0, 0x82, 0x43, 0x00, 0x00, 0x83, 0x43, - 0x00, 0x40, 0x83, 0x43, 0x00, 0x80, 0x83, 0x43, 0x00, 0xc0, 0x83, 0x43, - 0x00, 0x00, 0x84, 0x43, 0x00, 0x40, 0x84, 0x43, 0x00, 0x80, 0x84, 0x43, - 0x00, 0xc0, 0x84, 0x43, 0x00, 0x00, 0x85, 0x43, 0x00, 0x40, 0x85, 0x43, - 0x00, 0x80, 0x85, 0x43, 0x00, 0xc0, 0x85, 0x43, 0x00, 0x00, 0x86, 0x43, - 0x00, 0x40, 0x86, 0x43, 0x00, 0x80, 0x86, 0x43, 0x00, 0xc0, 0x86, 0x43, - 0x00, 0x00, 0x87, 0x43, 0x00, 0x40, 0x87, 0x43, 0x00, 0x80, 0x87, 0x43, - 0x00, 0xc0, 0x87, 0x43, 0x00, 0x00, 0x88, 0x43, 0x00, 0x40, 0x88, 0x43, - 0x00, 0x80, 0x88, 0x43, 0x00, 0xc0, 0x88, 0x43, 0x00, 0x00, 0x89, 0x43, - 0x00, 0x40, 0x89, 0x43, 0x00, 0x80, 0x89, 0x43, 0x00, 0xc0, 0x89, 0x43, - 0x00, 0x00, 0x8a, 0x43, 0x00, 0x40, 0x8a, 0x43, 0x00, 0x80, 0x8a, 0x43, - 0x00, 0xc0, 0x8a, 0x43, 0x00, 0x00, 0x8b, 0x43, 0x00, 0x40, 0x8b, 0x43, - 0x00, 0x80, 0x8b, 0x43, 0x00, 0xc0, 0x8b, 0x43, 0x00, 0x00, 0x8c, 0x43, - 0x00, 0x40, 0x8c, 0x43, 0x00, 0x80, 0x8c, 0x43, 0x00, 0xc0, 0x8c, 0x43, - 0x00, 0x00, 0x8d, 0x43, 0x00, 0x40, 0x8d, 0x43, 0x00, 0x80, 0x8d, 0x43, - 0x00, 0xc0, 0x8d, 0x43, 0x00, 0x00, 0x8e, 0x43, 0x00, 0x40, 0x8e, 0x43, - 0x00, 0x80, 0x8e, 0x43, 0x00, 0xc0, 0x8e, 0x43, 0x00, 0x00, 0x8f, 0x43, - 0x00, 0x40, 0x8f, 0x43, 0x00, 0x80, 0x8f, 0x43, 0x00, 0xc0, 0x8f, 0x43, - 0x00, 0x00, 0x90, 0x43, 0x00, 0x40, 0x90, 0x43, 0x00, 0x80, 0x90, 0x43, - 0x00, 0xc0, 0x90, 0x43, 0x00, 0x00, 0x91, 0x43, 0x00, 0x40, 0x91, 0x43, - 0x00, 0x80, 0x91, 0x43, 0x00, 0xc0, 0x91, 0x43, 0x00, 0x00, 0x92, 0x43, - 0x00, 0x40, 0x92, 0x43, 0x00, 0x80, 0x92, 0x43, 0x00, 0xc0, 0x92, 0x43, - 0x00, 0x00, 0x93, 0x43, 0x00, 0x40, 0x93, 0x43, 0x00, 0x80, 0x93, 0x43, - 0x00, 0xc0, 0x93, 0x43, 0x00, 0x00, 0x94, 0x43, 0x00, 0x40, 0x94, 0x43, - 0x00, 0x80, 0x94, 0x43, 0x00, 0xc0, 0x94, 0x43, 0x00, 0x00, 0x95, 0x43, - 0x00, 0x40, 0x95, 0x43, 0x00, 0x80, 0x95, 0x43, 0x00, 0xc0, 0x95, 0x43, - 0x00, 0x00, 0x96, 0x43, 0x00, 0x40, 0x96, 0x43, 0x00, 0x80, 0x96, 0x43, - 0x00, 0xc0, 0x96, 0x43, 0x00, 0x00, 0x97, 0x43, 0x00, 0x40, 0x97, 0x43, - 0x00, 0x80, 0x97, 0x43, 0x00, 0xc0, 0x97, 0x43, 0x00, 0x00, 0x98, 0x43, - 0x00, 0x40, 0x98, 0x43, 0x00, 0x80, 0x98, 0x43, 0x00, 0xc0, 0x98, 0x43, - 0x00, 0x00, 0x99, 0x43, 0x00, 0x40, 0x99, 0x43, 0x00, 0x80, 0x99, 0x43, - 0x00, 0xc0, 0x99, 0x43, 0x00, 0x00, 0x9a, 0x43, 0x00, 0x40, 0x9a, 0x43, - 0x00, 0x80, 0x9a, 0x43, 0x00, 0xc0, 0x9a, 0x43, 0x00, 0x00, 0x9b, 0x43, - 0x00, 0x40, 0x9b, 0x43, 0x00, 0x80, 0x9b, 0x43, 0x00, 0xc0, 0x9b, 0x43, - 0x00, 0x00, 0x9c, 0x43, 0x00, 0x40, 0x9c, 0x43, 0x00, 0x80, 0x9c, 0x43, - 0x00, 0xc0, 0x9c, 0x43, 0x00, 0x00, 0x9d, 0x43, 0x00, 0x40, 0x9d, 0x43, - 0x00, 0x80, 0x9d, 0x43, 0x00, 0xc0, 0x9d, 0x43, 0x00, 0x00, 0x9e, 0x43, - 0x00, 0x40, 0x9e, 0x43, 0x00, 0x80, 0x9e, 0x43, 0x00, 0xc0, 0x9e, 0x43, - 0x00, 0x00, 0x9f, 0x43, 0x00, 0x40, 0x9f, 0x43, 0x00, 0x80, 0x9f, 0x43, - 0x00, 0xc0, 0x9f, 0x43, 0x00, 0x00, 0xa0, 0x43, 0x00, 0x40, 0xa0, 0x43, - 0x00, 0x80, 0xa0, 0x43, 0x00, 0xc0, 0xa0, 0x43, 0x00, 0x00, 0xa1, 0x43, - 0x00, 0x40, 0xa1, 0x43, 0x00, 0x80, 0xa1, 0x43, 0x00, 0xc0, 0xa1, 0x43, - 0x00, 0x00, 0xa2, 0x43, 0x00, 0x40, 0xa2, 0x43, 0x00, 0x80, 0xa2, 0x43, - 0x00, 0xc0, 0xa2, 0x43, 0x00, 0x00, 0xa3, 0x43, 0x00, 0x40, 0xa3, 0x43, - 0x00, 0x80, 0xa3, 0x43, 0x00, 0xc0, 0xa3, 0x43, 0x00, 0x00, 0xa4, 0x43, - 0x00, 0x40, 0xa4, 0x43, 0x00, 0x80, 0xa4, 0x43, 0x00, 0xc0, 0xa4, 0x43, - 0x00, 0x00, 0xa5, 0x43, 0x00, 0x40, 0xa5, 0x43, 0x00, 0x80, 0xa5, 0x43, - 0x00, 0xc0, 0xa5, 0x43, 0x00, 0x00, 0xa6, 0x43, 0x00, 0x40, 0xa6, 0x43, - 0x00, 0x80, 0xa6, 0x43, 0x00, 0xc0, 0xa6, 0x43, 0x00, 0x00, 0xa7, 0x43, - 0x00, 0x40, 0xa7, 0x43, 0x00, 0x80, 0xa7, 0x43, 0x00, 0xc0, 0xa7, 0x43, - 0x00, 0x00, 0xa8, 0x43, 0x00, 0x40, 0xa8, 0x43, 0x00, 0x80, 0xa8, 0x43, - 0x00, 0xc0, 0xa8, 0x43, 0x00, 0x00, 0xa9, 0x43, 0x00, 0x40, 0xa9, 0x43, - 0x00, 0x80, 0xa9, 0x43, 0x00, 0xc0, 0xa9, 0x43, 0x00, 0x00, 0xaa, 0x43, - 0x00, 0x40, 0xaa, 0x43, 0x00, 0x80, 0xaa, 0x43, 0x00, 0xc0, 0xaa, 0x43, - 0x00, 0x00, 0xab, 0x43, 0x00, 0x40, 0xab, 0x43, 0x00, 0x80, 0xab, 0x43, - 0x00, 0xc0, 0xab, 0x43, 0x00, 0x00, 0xac, 0x43, 0x00, 0x40, 0xac, 0x43, - 0x00, 0x80, 0xac, 0x43, 0x00, 0xc0, 0xac, 0x43, 0x00, 0x00, 0xad, 0x43, - 0x00, 0x40, 0xad, 0x43, 0x00, 0x80, 0xad, 0x43, 0x00, 0xc0, 0xad, 0x43, - 0x00, 0x00, 0xae, 0x43, 0x00, 0x40, 0xae, 0x43, 0x00, 0x80, 0xae, 0x43, - 0x00, 0xc0, 0xae, 0x43, 0x00, 0x00, 0xaf, 0x43, 0x00, 0x40, 0xaf, 0x43, - 0x00, 0x80, 0xaf, 0x43, 0x00, 0xc0, 0xaf, 0x43, 0x00, 0x00, 0xb0, 0x43, - 0x00, 0x40, 0xb0, 0x43, 0x00, 0x80, 0xb0, 0x43, 0x00, 0xc0, 0xb0, 0x43, - 0x00, 0x00, 0xb1, 0x43, 0x00, 0x40, 0xb1, 0x43, 0x00, 0x80, 0xb1, 0x43, - 0x00, 0xc0, 0xb1, 0x43, 0x00, 0x00, 0xb2, 0x43, 0x00, 0x40, 0xb2, 0x43, - 0x00, 0x80, 0xb2, 0x43, 0x00, 0xc0, 0xb2, 0x43, 0x00, 0x00, 0xb3, 0x43, - 0x00, 0x40, 0xb3, 0x43, 0x00, 0x80, 0xb3, 0x43, 0x00, 0xc0, 0xb3, 0x43, - 0x00, 0x00, 0xb4, 0x43, 0x00, 0x40, 0xb4, 0x43, 0x00, 0x80, 0xb4, 0x43, - 0x00, 0xc0, 0xb4, 0x43, 0x00, 0x00, 0xb5, 0x43, 0x00, 0x40, 0xb5, 0x43, - 0x00, 0x80, 0xb5, 0x43, 0x00, 0xc0, 0xb5, 0x43, 0x00, 0x00, 0xb6, 0x43, - 0x00, 0x40, 0xb6, 0x43, 0x00, 0x80, 0xb6, 0x43, 0x00, 0xc0, 0xb6, 0x43, - 0x00, 0x00, 0xb7, 0x43, 0x00, 0x40, 0xb7, 0x43, 0x00, 0x80, 0xb7, 0x43, - 0x00, 0xc0, 0xb7, 0x43, 0x00, 0x00, 0xb8, 0x43, 0x00, 0x40, 0xb8, 0x43, - 0x00, 0x80, 0xb8, 0x43, 0x00, 0xc0, 0xb8, 0x43, 0x00, 0x00, 0xb9, 0x43, - 0x00, 0x40, 0xb9, 0x43, 0x00, 0x80, 0xb9, 0x43, 0x00, 0xc0, 0xb9, 0x43, - 0x00, 0x00, 0xba, 0x43, 0x00, 0x40, 0xba, 0x43, 0x00, 0x80, 0xba, 0x43, - 0x00, 0xc0, 0xba, 0x43, 0x00, 0x00, 0xbb, 0x43, 0x00, 0x40, 0xbb, 0x43, - 0x00, 0x80, 0xbb, 0x43, 0x00, 0xc0, 0xbb, 0x43, 0x00, 0x00, 0xbc, 0x43, - 0x00, 0x40, 0xbc, 0x43, 0x00, 0x80, 0xbc, 0x43, 0x00, 0xc0, 0xbc, 0x43, - 0x00, 0x00, 0xbd, 0x43, 0x00, 0x40, 0xbd, 0x43, 0x00, 0x80, 0xbd, 0x43, - 0x00, 0xc0, 0xbd, 0x43, 0x00, 0x00, 0xbe, 0x43, 0x00, 0x40, 0xbe, 0x43, - 0x00, 0x80, 0xbe, 0x43, 0x00, 0xc0, 0xbe, 0x43, 0x00, 0x00, 0xbf, 0x43, - 0x00, 0x40, 0xbf, 0x43, 0x00, 0x80, 0xbf, 0x43, 0x00, 0xc0, 0xbf, 0x43, - 0x00, 0x00, 0xc0, 0x43, 0x00, 0x40, 0xc0, 0x43, 0x00, 0x80, 0xc0, 0x43, - 0x00, 0xc0, 0xc0, 0x43, 0x00, 0x00, 0xc1, 0x43, 0x00, 0x40, 0xc1, 0x43, - 0x00, 0x80, 0xc1, 0x43, 0x00, 0xc0, 0xc1, 0x43, 0x00, 0x00, 0xc2, 0x43, - 0x00, 0x40, 0xc2, 0x43, 0x00, 0x80, 0xc2, 0x43, 0x00, 0xc0, 0xc2, 0x43, - 0x00, 0x00, 0xc3, 0x43, 0x00, 0x40, 0xc3, 0x43, 0x00, 0x80, 0xc3, 0x43, - 0x00, 0xc0, 0xc3, 0x43, 0x00, 0x00, 0xc4, 0x43, 0x00, 0x40, 0xc4, 0x43, - 0x00, 0x80, 0xc4, 0x43, 0x00, 0xc0, 0xc4, 0x43, 0x00, 0x00, 0xc5, 0x43, - 0x00, 0x40, 0xc5, 0x43, 0x00, 0x80, 0xc5, 0x43, 0x00, 0xc0, 0xc5, 0x43, - 0x00, 0x00, 0xc6, 0x43, 0x00, 0x40, 0xc6, 0x43, 0x00, 0x80, 0xc6, 0x43, - 0x00, 0xc0, 0xc6, 0x43, 0x00, 0x00, 0xc7, 0x43, 0x00, 0x40, 0xc7, 0x43, - 0x00, 0x80, 0xc7, 0x43, 0x00, 0xc0, 0xc7, 0x43, 0x00, 0x00, 0xc8, 0x43, - 0x00, 0x40, 0xc8, 0x43, 0x00, 0x80, 0xc8, 0x43, 0x00, 0xc0, 0xc8, 0x43, - 0x00, 0x00, 0xc9, 0x43, 0x00, 0x40, 0xc9, 0x43, 0x00, 0x80, 0xc9, 0x43, - 0x00, 0xc0, 0xc9, 0x43, 0x00, 0x00, 0xca, 0x43, 0x00, 0x40, 0xca, 0x43, - 0x00, 0x80, 0xca, 0x43, 0x00, 0xc0, 0xca, 0x43, 0x00, 0x00, 0xcb, 0x43, - 0x00, 0x40, 0xcb, 0x43, 0x00, 0x80, 0xcb, 0x43, 0x00, 0xc0, 0xcb, 0x43, - 0x00, 0x00, 0xcc, 0x43, 0x00, 0x40, 0xcc, 0x43, 0x00, 0x80, 0xcc, 0x43, - 0x00, 0xc0, 0xcc, 0x43, 0x00, 0x00, 0xcd, 0x43, 0x00, 0x40, 0xcd, 0x43, - 0x00, 0x80, 0xcd, 0x43, 0x00, 0xc0, 0xcd, 0x43, 0x00, 0x00, 0xce, 0x43, - 0x00, 0x40, 0xce, 0x43, 0x00, 0x80, 0xce, 0x43, 0x00, 0xc0, 0xce, 0x43, - 0x00, 0x00, 0xcf, 0x43, 0x00, 0x40, 0xcf, 0x43, 0x00, 0x80, 0xcf, 0x43, - 0x00, 0xc0, 0xcf, 0x43, 0x00, 0x00, 0xd0, 0x43, 0x00, 0x40, 0xd0, 0x43, - 0x00, 0x80, 0xd0, 0x43, 0x00, 0xc0, 0xd0, 0x43, 0x00, 0x00, 0xd1, 0x43, - 0x00, 0x40, 0xd1, 0x43, 0x00, 0x80, 0xd1, 0x43, 0x00, 0xc0, 0xd1, 0x43, - 0x00, 0x00, 0xd2, 0x43, 0x00, 0x40, 0xd2, 0x43, 0x00, 0x80, 0xd2, 0x43, - 0x00, 0xc0, 0xd2, 0x43, 0x00, 0x00, 0xd3, 0x43, 0x00, 0x40, 0xd3, 0x43, - 0x00, 0x80, 0xd3, 0x43, 0x00, 0xc0, 0xd3, 0x43, 0x00, 0x00, 0xd4, 0x43, - 0x00, 0x40, 0xd4, 0x43, 0x00, 0x80, 0xd4, 0x43, 0x00, 0xc0, 0xd4, 0x43, - 0x00, 0x00, 0xd5, 0x43, 0x00, 0x40, 0xd5, 0x43, 0x00, 0x80, 0xd5, 0x43, - 0x00, 0xc0, 0xd5, 0x43, 0x00, 0x00, 0xd6, 0x43, 0x00, 0x40, 0xd6, 0x43, - 0x00, 0x80, 0xd6, 0x43, 0x00, 0xc0, 0xd6, 0x43, 0x00, 0x00, 0xd7, 0x43, - 0x00, 0x40, 0xd7, 0x43, 0x00, 0x80, 0xd7, 0x43, 0x00, 0xc0, 0xd7, 0x43, - 0x00, 0x00, 0xd8, 0x43, 0x00, 0x40, 0xd8, 0x43, 0x00, 0x80, 0xd8, 0x43, - 0x00, 0xc0, 0xd8, 0x43, 0x00, 0x00, 0xd9, 0x43, 0x00, 0x40, 0xd9, 0x43, - 0x00, 0x80, 0xd9, 0x43, 0x00, 0xc0, 0xd9, 0x43, 0x00, 0x00, 0xda, 0x43, - 0x00, 0x40, 0xda, 0x43, 0x00, 0x80, 0xda, 0x43, 0x00, 0xc0, 0xda, 0x43, - 0x00, 0x00, 0xdb, 0x43, 0x00, 0x40, 0xdb, 0x43, 0x00, 0x80, 0xdb, 0x43, - 0x00, 0xc0, 0xdb, 0x43, 0x00, 0x00, 0xdc, 0x43, 0x00, 0x40, 0xdc, 0x43, - 0x00, 0x80, 0xdc, 0x43, 0x00, 0xc0, 0xdc, 0x43, 0x00, 0x00, 0xdd, 0x43, - 0x00, 0x40, 0xdd, 0x43, 0x00, 0x80, 0xdd, 0x43, 0x00, 0xc0, 0xdd, 0x43, - 0x00, 0x00, 0xde, 0x43, 0x00, 0x40, 0xde, 0x43, 0x00, 0x80, 0xde, 0x43, - 0x00, 0xc0, 0xde, 0x43, 0x00, 0x00, 0xdf, 0x43, 0x00, 0x40, 0xdf, 0x43, - 0x00, 0x80, 0xdf, 0x43, 0x00, 0xc0, 0xdf, 0x43, 0x00, 0x00, 0xe0, 0x43, - 0x00, 0x40, 0xe0, 0x43, 0x00, 0x80, 0xe0, 0x43, 0x00, 0xc0, 0xe0, 0x43, - 0x00, 0x00, 0xe1, 0x43, 0x00, 0x40, 0xe1, 0x43, 0x00, 0x80, 0xe1, 0x43, - 0x00, 0xc0, 0xe1, 0x43, 0x00, 0x00, 0xe2, 0x43, 0x00, 0x40, 0xe2, 0x43, - 0x00, 0x80, 0xe2, 0x43, 0x00, 0xc0, 0xe2, 0x43, 0x00, 0x00, 0xe3, 0x43, - 0x00, 0x40, 0xe3, 0x43, 0x00, 0x80, 0xe3, 0x43, 0x00, 0xc0, 0xe3, 0x43, - 0x00, 0x00, 0xe4, 0x43, 0x00, 0x40, 0xe4, 0x43, 0x00, 0x80, 0xe4, 0x43, - 0x00, 0xc0, 0xe4, 0x43, 0x00, 0x00, 0xe5, 0x43, 0x00, 0x40, 0xe5, 0x43, - 0x00, 0x80, 0xe5, 0x43, 0x00, 0xc0, 0xe5, 0x43, 0x00, 0x00, 0xe6, 0x43, - 0x00, 0x40, 0xe6, 0x43, 0x00, 0x80, 0xe6, 0x43, 0x00, 0xc0, 0xe6, 0x43, - 0x00, 0x00, 0xe7, 0x43, 0x00, 0x40, 0xe7, 0x43, 0x00, 0x80, 0xe7, 0x43, - 0x00, 0xc0, 0xe7, 0x43, 0x00, 0x00, 0xe8, 0x43, 0x00, 0x40, 0xe8, 0x43, - 0x00, 0x80, 0xe8, 0x43, 0x00, 0xc0, 0xe8, 0x43, 0x00, 0x00, 0xe9, 0x43, - 0x00, 0x40, 0xe9, 0x43, 0x00, 0x80, 0xe9, 0x43, 0x00, 0xc0, 0xe9, 0x43, - 0x00, 0x00, 0xea, 0x43, 0x00, 0x40, 0xea, 0x43, 0x00, 0x80, 0xea, 0x43, - 0x00, 0xc0, 0xea, 0x43, 0x00, 0x00, 0xeb, 0x43, 0x00, 0x40, 0xeb, 0x43, - 0x00, 0x80, 0xeb, 0x43, 0x00, 0xc0, 0xeb, 0x43, 0x00, 0x00, 0xec, 0x43, - 0x00, 0x40, 0xec, 0x43, 0x00, 0x80, 0xec, 0x43, 0x00, 0xc0, 0xec, 0x43, - 0x00, 0x00, 0xed, 0x43, 0x00, 0x40, 0xed, 0x43, 0x00, 0x80, 0xed, 0x43, - 0x00, 0xc0, 0xed, 0x43, 0x00, 0x00, 0xee, 0x43, 0x00, 0x40, 0xee, 0x43, - 0x00, 0x80, 0xee, 0x43, 0x00, 0xc0, 0xee, 0x43, 0x00, 0x00, 0xef, 0x43, - 0x00, 0x40, 0xef, 0x43, 0x00, 0x80, 0xef, 0x43, 0x00, 0xc0, 0xef, 0x43, - 0x00, 0x00, 0xf0, 0x43, 0x00, 0x40, 0xf0, 0x43, 0x00, 0x80, 0xf0, 0x43, - 0x00, 0xc0, 0xf0, 0x43, 0x00, 0x00, 0xf1, 0x43, 0x00, 0x40, 0xf1, 0x43, - 0x00, 0x80, 0xf1, 0x43, 0x00, 0xc0, 0xf1, 0x43, 0x00, 0x00, 0xf2, 0x43, - 0x00, 0x40, 0xf2, 0x43, 0x00, 0x80, 0xf2, 0x43, 0x00, 0xc0, 0xf2, 0x43, - 0x00, 0x00, 0xf3, 0x43, 0x00, 0x40, 0xf3, 0x43, 0x00, 0x80, 0xf3, 0x43, - 0x00, 0xc0, 0xf3, 0x43, 0x00, 0x00, 0xf4, 0x43, 0x00, 0x40, 0xf4, 0x43, - 0x00, 0x80, 0xf4, 0x43, 0x00, 0xc0, 0xf4, 0x43, 0x00, 0x00, 0xf5, 0x43, - 0x00, 0x40, 0xf5, 0x43, 0x00, 0x80, 0xf5, 0x43, 0x00, 0xc0, 0xf5, 0x43, - 0x00, 0x00, 0xf6, 0x43, 0x00, 0x40, 0xf6, 0x43, 0x00, 0x80, 0xf6, 0x43, - 0x00, 0xc0, 0xf6, 0x43, 0x00, 0x00, 0xf7, 0x43, 0x00, 0x40, 0xf7, 0x43, - 0x00, 0x80, 0xf7, 0x43, 0x00, 0xc0, 0xf7, 0x43, 0x00, 0x00, 0xf8, 0x43, - 0x00, 0x40, 0xf8, 0x43, 0x00, 0x80, 0xf8, 0x43, 0x00, 0xc0, 0xf8, 0x43, - 0x00, 0x00, 0xf9, 0x43, 0x00, 0x40, 0xf9, 0x43, 0x00, 0x80, 0xf9, 0x43, - 0x00, 0xc0, 0xf9, 0x43, 0x00, 0x00, 0xfa, 0x43, 0x00, 0x40, 0xfa, 0x43, - 0x00, 0x80, 0xfa, 0x43, 0x00, 0xc0, 0xfa, 0x43, 0x00, 0x00, 0xfb, 0x43, - 0x00, 0x40, 0xfb, 0x43, 0x00, 0x80, 0xfb, 0x43, 0x00, 0xc0, 0xfb, 0x43, - 0x00, 0x00, 0xfc, 0x43, 0x00, 0x40, 0xfc, 0x43, 0x00, 0x80, 0xfc, 0x43, - 0x00, 0xc0, 0xfc, 0x43, 0x00, 0x00, 0xfd, 0x43, 0x00, 0x40, 0xfd, 0x43, - 0x00, 0x80, 0xfd, 0x43, 0x00, 0xc0, 0xfd, 0x43, 0x00, 0x00, 0xfe, 0x43, - 0x00, 0x40, 0xfe, 0x43, 0x00, 0x80, 0xfe, 0x43, 0x00, 0xc0, 0xfe, 0x43, - 0x00, 0x00, 0xff, 0x43, 0x00, 0x40, 0xff, 0x43, 0x00, 0x80, 0xff, 0x43, - 0x00, 0xc0, 0xff, 0x43, 0x00, 0x00, 0x00, 0x44, 0x00, 0x20, 0x00, 0x44, - 0x00, 0x40, 0x00, 0x44, 0x00, 0x60, 0x00, 0x44, 0x00, 0x80, 0x00, 0x44, - 0x00, 0xa0, 0x00, 0x44, 0x00, 0xc0, 0x00, 0x44, 0x00, 0xe0, 0x00, 0x44, - 0x00, 0x00, 0x01, 0x44, 0x00, 0x20, 0x01, 0x44, 0x00, 0x40, 0x01, 0x44, - 0x00, 0x60, 0x01, 0x44, 0x00, 0x80, 0x01, 0x44, 0x00, 0xa0, 0x01, 0x44, - 0x00, 0xc0, 0x01, 0x44, 0x00, 0xe0, 0x01, 0x44, 0x00, 0x00, 0x02, 0x44, - 0x00, 0x20, 0x02, 0x44, 0x00, 0x40, 0x02, 0x44, 0x00, 0x60, 0x02, 0x44, - 0x00, 0x80, 0x02, 0x44, 0x00, 0xa0, 0x02, 0x44, 0x00, 0xc0, 0x02, 0x44, - 0x00, 0xe0, 0x02, 0x44, 0x00, 0x00, 0x03, 0x44, 0x00, 0x20, 0x03, 0x44, - 0x00, 0x40, 0x03, 0x44, 0x00, 0x60, 0x03, 0x44, 0x00, 0x80, 0x03, 0x44, - 0x00, 0xa0, 0x03, 0x44, 0x00, 0xc0, 0x03, 0x44, 0x00, 0xe0, 0x03, 0x44, - 0x00, 0x00, 0x04, 0x44, 0x00, 0x20, 0x04, 0x44, 0x00, 0x40, 0x04, 0x44, - 0x00, 0x60, 0x04, 0x44, 0x00, 0x80, 0x04, 0x44, 0x00, 0xa0, 0x04, 0x44, - 0x00, 0xc0, 0x04, 0x44, 0x00, 0xe0, 0x04, 0x44, 0x00, 0x00, 0x05, 0x44, - 0x00, 0x20, 0x05, 0x44, 0x00, 0x40, 0x05, 0x44, 0x00, 0x60, 0x05, 0x44, - 0x00, 0x80, 0x05, 0x44, 0x00, 0xa0, 0x05, 0x44, 0x00, 0xc0, 0x05, 0x44, - 0x00, 0xe0, 0x05, 0x44, 0x00, 0x00, 0x06, 0x44, 0x00, 0x20, 0x06, 0x44, - 0x00, 0x40, 0x06, 0x44, 0x00, 0x60, 0x06, 0x44, 0x00, 0x80, 0x06, 0x44, - 0x00, 0xa0, 0x06, 0x44, 0x00, 0xc0, 0x06, 0x44, 0x00, 0xe0, 0x06, 0x44, - 0x00, 0x00, 0x07, 0x44, 0x00, 0x20, 0x07, 0x44, 0x00, 0x40, 0x07, 0x44, - 0x00, 0x60, 0x07, 0x44, 0x00, 0x80, 0x07, 0x44, 0x00, 0xa0, 0x07, 0x44, - 0x00, 0xc0, 0x07, 0x44, 0x00, 0xe0, 0x07, 0x44, 0x00, 0x00, 0x08, 0x44, - 0x00, 0x20, 0x08, 0x44, 0x00, 0x40, 0x08, 0x44, 0x00, 0x60, 0x08, 0x44, - 0x00, 0x80, 0x08, 0x44, 0x00, 0xa0, 0x08, 0x44, 0x00, 0xc0, 0x08, 0x44, - 0x00, 0xe0, 0x08, 0x44, 0x00, 0x00, 0x09, 0x44, 0x00, 0x20, 0x09, 0x44, - 0x00, 0x40, 0x09, 0x44, 0x00, 0x60, 0x09, 0x44, 0x00, 0x80, 0x09, 0x44, - 0x00, 0xa0, 0x09, 0x44, 0x00, 0xc0, 0x09, 0x44, 0x00, 0xe0, 0x09, 0x44, - 0x00, 0x00, 0x0a, 0x44, 0x00, 0x20, 0x0a, 0x44, 0x00, 0x40, 0x0a, 0x44, - 0x00, 0x60, 0x0a, 0x44, 0x00, 0x80, 0x0a, 0x44, 0x00, 0xa0, 0x0a, 0x44, - 0x00, 0xc0, 0x0a, 0x44, 0x00, 0xe0, 0x0a, 0x44, 0x00, 0x00, 0x0b, 0x44, - 0x00, 0x20, 0x0b, 0x44, 0x00, 0x40, 0x0b, 0x44, 0x00, 0x60, 0x0b, 0x44, - 0x00, 0x80, 0x0b, 0x44, 0x00, 0xa0, 0x0b, 0x44, 0x00, 0xc0, 0x0b, 0x44, - 0x00, 0xe0, 0x0b, 0x44, 0x00, 0x00, 0x0c, 0x44, 0x00, 0x20, 0x0c, 0x44, - 0x00, 0x40, 0x0c, 0x44, 0x00, 0x60, 0x0c, 0x44, 0x00, 0x80, 0x0c, 0x44, - 0x00, 0xa0, 0x0c, 0x44, 0x00, 0xc0, 0x0c, 0x44, 0x00, 0xe0, 0x0c, 0x44, - 0x00, 0x00, 0x0d, 0x44, 0x00, 0x20, 0x0d, 0x44, 0x00, 0x40, 0x0d, 0x44, - 0x00, 0x60, 0x0d, 0x44, 0x00, 0x80, 0x0d, 0x44, 0x00, 0xa0, 0x0d, 0x44, - 0x00, 0xc0, 0x0d, 0x44, 0x00, 0xe0, 0x0d, 0x44, 0x00, 0x00, 0x0e, 0x44, - 0x00, 0x20, 0x0e, 0x44, 0x00, 0x40, 0x0e, 0x44, 0x00, 0x60, 0x0e, 0x44, - 0x00, 0x80, 0x0e, 0x44, 0x00, 0xa0, 0x0e, 0x44, 0x00, 0xc0, 0x0e, 0x44, - 0x00, 0xe0, 0x0e, 0x44, 0x00, 0x00, 0x0f, 0x44, 0x00, 0x20, 0x0f, 0x44, - 0x00, 0x40, 0x0f, 0x44, 0x00, 0x60, 0x0f, 0x44, 0x00, 0x80, 0x0f, 0x44, - 0x00, 0xa0, 0x0f, 0x44, 0x00, 0xc0, 0x0f, 0x44, 0x00, 0xe0, 0x0f, 0x44, - 0x00, 0x00, 0x10, 0x44, 0x00, 0x20, 0x10, 0x44, 0x00, 0x40, 0x10, 0x44, - 0x00, 0x60, 0x10, 0x44, 0x00, 0x80, 0x10, 0x44, 0x00, 0xa0, 0x10, 0x44, - 0x00, 0xc0, 0x10, 0x44, 0x00, 0xe0, 0x10, 0x44, 0x00, 0x00, 0x11, 0x44, - 0x00, 0x20, 0x11, 0x44, 0x00, 0x40, 0x11, 0x44, 0x00, 0x60, 0x11, 0x44, - 0x00, 0x80, 0x11, 0x44, 0x00, 0xa0, 0x11, 0x44, 0x00, 0xc0, 0x11, 0x44, - 0x00, 0xe0, 0x11, 0x44, 0x00, 0x00, 0x12, 0x44, 0x00, 0x20, 0x12, 0x44, - 0x00, 0x40, 0x12, 0x44, 0x00, 0x60, 0x12, 0x44, 0x00, 0x80, 0x12, 0x44, - 0x00, 0xa0, 0x12, 0x44, 0x00, 0xc0, 0x12, 0x44, 0x00, 0xe0, 0x12, 0x44, - 0x00, 0x00, 0x13, 0x44, 0x00, 0x20, 0x13, 0x44, 0x00, 0x40, 0x13, 0x44, - 0x00, 0x60, 0x13, 0x44, 0x00, 0x80, 0x13, 0x44, 0x00, 0xa0, 0x13, 0x44, - 0x00, 0xc0, 0x13, 0x44, 0x00, 0xe0, 0x13, 0x44, 0x00, 0x00, 0x14, 0x44, - 0x00, 0x20, 0x14, 0x44, 0x00, 0x40, 0x14, 0x44, 0x00, 0x60, 0x14, 0x44, - 0x00, 0x80, 0x14, 0x44, 0x00, 0xa0, 0x14, 0x44, 0x00, 0xc0, 0x14, 0x44, - 0x00, 0xe0, 0x14, 0x44, 0x00, 0x00, 0x15, 0x44, 0x00, 0x20, 0x15, 0x44, - 0x00, 0x40, 0x15, 0x44, 0x00, 0x60, 0x15, 0x44, 0x00, 0x80, 0x15, 0x44, - 0x00, 0xa0, 0x15, 0x44, 0x00, 0xc0, 0x15, 0x44, 0x00, 0xe0, 0x15, 0x44, - 0x00, 0x00, 0x16, 0x44, 0x00, 0x20, 0x16, 0x44, 0x00, 0x40, 0x16, 0x44, - 0x00, 0x60, 0x16, 0x44, 0x00, 0x80, 0x16, 0x44, 0x00, 0xa0, 0x16, 0x44, - 0x00, 0xc0, 0x16, 0x44, 0x00, 0xe0, 0x16, 0x44, 0x00, 0x00, 0x17, 0x44, - 0x00, 0x20, 0x17, 0x44, 0x00, 0x40, 0x17, 0x44, 0x00, 0x60, 0x17, 0x44, - 0x00, 0x80, 0x17, 0x44, 0x00, 0xa0, 0x17, 0x44, 0x00, 0xc0, 0x17, 0x44, - 0x00, 0xe0, 0x17, 0x44, 0x00, 0x00, 0x18, 0x44, 0x00, 0x20, 0x18, 0x44, - 0x00, 0x40, 0x18, 0x44, 0x00, 0x60, 0x18, 0x44, 0x00, 0x80, 0x18, 0x44, - 0x00, 0xa0, 0x18, 0x44, 0x00, 0xc0, 0x18, 0x44, 0x00, 0xe0, 0x18, 0x44, - 0x00, 0x00, 0x19, 0x44, 0x00, 0x20, 0x19, 0x44, 0x00, 0x40, 0x19, 0x44, - 0x00, 0x60, 0x19, 0x44, 0x00, 0x80, 0x19, 0x44, 0x00, 0xa0, 0x19, 0x44, - 0x00, 0xc0, 0x19, 0x44, 0x00, 0xe0, 0x19, 0x44, 0x00, 0x00, 0x1a, 0x44, - 0x00, 0x20, 0x1a, 0x44, 0x00, 0x40, 0x1a, 0x44, 0x00, 0x60, 0x1a, 0x44, - 0x00, 0x80, 0x1a, 0x44, 0x00, 0xa0, 0x1a, 0x44, 0x00, 0xc0, 0x1a, 0x44, - 0x00, 0xe0, 0x1a, 0x44, 0x00, 0x00, 0x1b, 0x44, 0x00, 0x20, 0x1b, 0x44, - 0x00, 0x40, 0x1b, 0x44, 0x00, 0x60, 0x1b, 0x44, 0x00, 0x80, 0x1b, 0x44, - 0x00, 0xa0, 0x1b, 0x44, 0x00, 0xc0, 0x1b, 0x44, 0x00, 0xe0, 0x1b, 0x44, - 0x00, 0x00, 0x1c, 0x44, 0x00, 0x20, 0x1c, 0x44, 0x00, 0x40, 0x1c, 0x44, - 0x00, 0x60, 0x1c, 0x44, 0x00, 0x80, 0x1c, 0x44, 0x00, 0xa0, 0x1c, 0x44, - 0x00, 0xc0, 0x1c, 0x44, 0x00, 0xe0, 0x1c, 0x44, 0x00, 0x00, 0x1d, 0x44, - 0x00, 0x20, 0x1d, 0x44, 0x00, 0x40, 0x1d, 0x44, 0x00, 0x60, 0x1d, 0x44, - 0x00, 0x80, 0x1d, 0x44, 0x00, 0xa0, 0x1d, 0x44, 0x00, 0xc0, 0x1d, 0x44, - 0x00, 0xe0, 0x1d, 0x44, 0x00, 0x00, 0x1e, 0x44, 0x00, 0x20, 0x1e, 0x44, - 0x00, 0x40, 0x1e, 0x44, 0x00, 0x60, 0x1e, 0x44, 0x00, 0x80, 0x1e, 0x44, - 0x00, 0xa0, 0x1e, 0x44, 0x00, 0xc0, 0x1e, 0x44, 0x00, 0xe0, 0x1e, 0x44, - 0x00, 0x00, 0x1f, 0x44, 0x00, 0x20, 0x1f, 0x44, 0x00, 0x40, 0x1f, 0x44, - 0x00, 0x60, 0x1f, 0x44, 0x00, 0x80, 0x1f, 0x44, 0x00, 0xa0, 0x1f, 0x44, - 0x00, 0xc0, 0x1f, 0x44, 0x00, 0xe0, 0x1f, 0x44, 0x00, 0x00, 0x20, 0x44, - 0x00, 0x20, 0x20, 0x44, 0x00, 0x40, 0x20, 0x44, 0x00, 0x60, 0x20, 0x44, - 0x00, 0x80, 0x20, 0x44, 0x00, 0xa0, 0x20, 0x44, 0x00, 0xc0, 0x20, 0x44, - 0x00, 0xe0, 0x20, 0x44, 0x00, 0x00, 0x21, 0x44, 0x00, 0x20, 0x21, 0x44, - 0x00, 0x40, 0x21, 0x44, 0x00, 0x60, 0x21, 0x44, 0x00, 0x80, 0x21, 0x44, - 0x00, 0xa0, 0x21, 0x44, 0x00, 0xc0, 0x21, 0x44, 0x00, 0xe0, 0x21, 0x44, - 0x00, 0x00, 0x22, 0x44, 0x00, 0x20, 0x22, 0x44, 0x00, 0x40, 0x22, 0x44, - 0x00, 0x60, 0x22, 0x44, 0x00, 0x80, 0x22, 0x44, 0x00, 0xa0, 0x22, 0x44, - 0x00, 0xc0, 0x22, 0x44, 0x00, 0xe0, 0x22, 0x44, 0x00, 0x00, 0x23, 0x44, - 0x00, 0x20, 0x23, 0x44, 0x00, 0x40, 0x23, 0x44, 0x00, 0x60, 0x23, 0x44, - 0x00, 0x80, 0x23, 0x44, 0x00, 0xa0, 0x23, 0x44, 0x00, 0xc0, 0x23, 0x44, - 0x00, 0xe0, 0x23, 0x44, 0x00, 0x00, 0x24, 0x44, 0x00, 0x20, 0x24, 0x44, - 0x00, 0x40, 0x24, 0x44, 0x00, 0x60, 0x24, 0x44, 0x00, 0x80, 0x24, 0x44, - 0x00, 0xa0, 0x24, 0x44, 0x00, 0xc0, 0x24, 0x44, 0x00, 0xe0, 0x24, 0x44, - 0x00, 0x00, 0x25, 0x44, 0x00, 0x20, 0x25, 0x44, 0x00, 0x40, 0x25, 0x44, - 0x00, 0x60, 0x25, 0x44, 0x00, 0x80, 0x25, 0x44, 0x00, 0xa0, 0x25, 0x44, - 0x00, 0xc0, 0x25, 0x44, 0x00, 0xe0, 0x25, 0x44, 0x00, 0x00, 0x26, 0x44, - 0x00, 0x20, 0x26, 0x44, 0x00, 0x40, 0x26, 0x44, 0x00, 0x60, 0x26, 0x44, - 0x00, 0x80, 0x26, 0x44, 0x00, 0xa0, 0x26, 0x44, 0x00, 0xc0, 0x26, 0x44, - 0x00, 0xe0, 0x26, 0x44, 0x00, 0x00, 0x27, 0x44, 0x00, 0x20, 0x27, 0x44, - 0x00, 0x40, 0x27, 0x44, 0x00, 0x60, 0x27, 0x44, 0x00, 0x80, 0x27, 0x44, - 0x00, 0xa0, 0x27, 0x44, 0x00, 0xc0, 0x27, 0x44, 0x00, 0xe0, 0x27, 0x44, - 0x00, 0x00, 0x28, 0x44, 0x00, 0x20, 0x28, 0x44, 0x00, 0x40, 0x28, 0x44, - 0x00, 0x60, 0x28, 0x44, 0x00, 0x80, 0x28, 0x44, 0x00, 0xa0, 0x28, 0x44, - 0x00, 0xc0, 0x28, 0x44, 0x00, 0xe0, 0x28, 0x44, 0x00, 0x00, 0x29, 0x44, - 0x00, 0x20, 0x29, 0x44, 0x00, 0x40, 0x29, 0x44, 0x00, 0x60, 0x29, 0x44, - 0x00, 0x80, 0x29, 0x44, 0x00, 0xa0, 0x29, 0x44, 0x00, 0xc0, 0x29, 0x44, - 0x00, 0xe0, 0x29, 0x44, 0x00, 0x00, 0x2a, 0x44, 0x00, 0x20, 0x2a, 0x44, - 0x00, 0x40, 0x2a, 0x44, 0x00, 0x60, 0x2a, 0x44, 0x00, 0x80, 0x2a, 0x44, - 0x00, 0xa0, 0x2a, 0x44, 0x00, 0xc0, 0x2a, 0x44, 0x00, 0xe0, 0x2a, 0x44, - 0x00, 0x00, 0x2b, 0x44, 0x00, 0x20, 0x2b, 0x44, 0x00, 0x40, 0x2b, 0x44, - 0x00, 0x60, 0x2b, 0x44, 0x00, 0x80, 0x2b, 0x44, 0x00, 0xa0, 0x2b, 0x44, - 0x00, 0xc0, 0x2b, 0x44, 0x00, 0xe0, 0x2b, 0x44, 0x00, 0x00, 0x2c, 0x44, - 0x00, 0x20, 0x2c, 0x44, 0x00, 0x40, 0x2c, 0x44, 0x00, 0x60, 0x2c, 0x44, - 0x00, 0x80, 0x2c, 0x44, 0x00, 0xa0, 0x2c, 0x44, 0x00, 0xc0, 0x2c, 0x44, - 0x00, 0xe0, 0x2c, 0x44, 0x00, 0x00, 0x2d, 0x44, 0x00, 0x20, 0x2d, 0x44, - 0x00, 0x40, 0x2d, 0x44, 0x00, 0x60, 0x2d, 0x44, 0x00, 0x80, 0x2d, 0x44, - 0x00, 0xa0, 0x2d, 0x44, 0x00, 0xc0, 0x2d, 0x44, 0x00, 0xe0, 0x2d, 0x44, - 0x00, 0x00, 0x2e, 0x44, 0x00, 0x20, 0x2e, 0x44, 0x00, 0x40, 0x2e, 0x44, - 0x00, 0x60, 0x2e, 0x44, 0x00, 0x80, 0x2e, 0x44, 0x00, 0xa0, 0x2e, 0x44, - 0x00, 0xc0, 0x2e, 0x44, 0x00, 0xe0, 0x2e, 0x44, 0x00, 0x00, 0x2f, 0x44, - 0x00, 0x20, 0x2f, 0x44, 0x00, 0x40, 0x2f, 0x44, 0x00, 0x60, 0x2f, 0x44, - 0x00, 0x80, 0x2f, 0x44, 0x00, 0xa0, 0x2f, 0x44, 0x00, 0xc0, 0x2f, 0x44, - 0x00, 0xe0, 0x2f, 0x44, 0x00, 0x00, 0x30, 0x44, 0x00, 0x20, 0x30, 0x44, - 0x00, 0x40, 0x30, 0x44, 0x00, 0x60, 0x30, 0x44, 0x00, 0x80, 0x30, 0x44, - 0x00, 0xa0, 0x30, 0x44, 0x00, 0xc0, 0x30, 0x44, 0x00, 0xe0, 0x30, 0x44, - 0x00, 0x00, 0x31, 0x44, 0x00, 0x20, 0x31, 0x44, 0x00, 0x40, 0x31, 0x44, - 0x00, 0x60, 0x31, 0x44, 0x00, 0x80, 0x31, 0x44, 0x00, 0xa0, 0x31, 0x44, - 0x00, 0xc0, 0x31, 0x44, 0x00, 0xe0, 0x31, 0x44, 0x00, 0x00, 0x32, 0x44, - 0x00, 0x20, 0x32, 0x44, 0x00, 0x40, 0x32, 0x44, 0x00, 0x60, 0x32, 0x44, - 0x00, 0x80, 0x32, 0x44, 0x00, 0xa0, 0x32, 0x44, 0x00, 0xc0, 0x32, 0x44, - 0x00, 0xe0, 0x32, 0x44, 0x00, 0x00, 0x33, 0x44, 0x00, 0x20, 0x33, 0x44, - 0x00, 0x40, 0x33, 0x44, 0x00, 0x60, 0x33, 0x44, 0x00, 0x80, 0x33, 0x44, - 0x00, 0xa0, 0x33, 0x44, 0x00, 0xc0, 0x33, 0x44, 0x00, 0xe0, 0x33, 0x44, - 0x00, 0x00, 0x34, 0x44, 0x00, 0x20, 0x34, 0x44, 0x00, 0x40, 0x34, 0x44, - 0x00, 0x60, 0x34, 0x44, 0x00, 0x80, 0x34, 0x44, 0x00, 0xa0, 0x34, 0x44, - 0x00, 0xc0, 0x34, 0x44, 0x00, 0xe0, 0x34, 0x44, 0x00, 0x00, 0x35, 0x44, - 0x00, 0x20, 0x35, 0x44, 0x00, 0x40, 0x35, 0x44, 0x00, 0x60, 0x35, 0x44, - 0x00, 0x80, 0x35, 0x44, 0x00, 0xa0, 0x35, 0x44, 0x00, 0xc0, 0x35, 0x44, - 0x00, 0xe0, 0x35, 0x44, 0x00, 0x00, 0x36, 0x44, 0x00, 0x20, 0x36, 0x44, - 0x00, 0x40, 0x36, 0x44, 0x00, 0x60, 0x36, 0x44, 0x00, 0x80, 0x36, 0x44, - 0x00, 0xa0, 0x36, 0x44, 0x00, 0xc0, 0x36, 0x44, 0x00, 0xe0, 0x36, 0x44, - 0x00, 0x00, 0x37, 0x44, 0x00, 0x20, 0x37, 0x44, 0x00, 0x40, 0x37, 0x44, - 0x00, 0x60, 0x37, 0x44, 0x00, 0x80, 0x37, 0x44, 0x00, 0xa0, 0x37, 0x44, - 0x00, 0xc0, 0x37, 0x44, 0x00, 0xe0, 0x37, 0x44, 0x00, 0x00, 0x38, 0x44, - 0x00, 0x20, 0x38, 0x44, 0x00, 0x40, 0x38, 0x44, 0x00, 0x60, 0x38, 0x44, - 0x00, 0x80, 0x38, 0x44, 0x00, 0xa0, 0x38, 0x44, 0x00, 0xc0, 0x38, 0x44, - 0x00, 0xe0, 0x38, 0x44, 0x00, 0x00, 0x39, 0x44, 0x00, 0x20, 0x39, 0x44, - 0x00, 0x40, 0x39, 0x44, 0x00, 0x60, 0x39, 0x44, 0x00, 0x80, 0x39, 0x44, - 0x00, 0xa0, 0x39, 0x44, 0x00, 0xc0, 0x39, 0x44, 0x00, 0xe0, 0x39, 0x44, - 0x00, 0x00, 0x3a, 0x44, 0x00, 0x20, 0x3a, 0x44, 0x00, 0x40, 0x3a, 0x44, - 0x00, 0x60, 0x3a, 0x44, 0x00, 0x80, 0x3a, 0x44, 0x00, 0xa0, 0x3a, 0x44, - 0x00, 0xc0, 0x3a, 0x44, 0x00, 0xe0, 0x3a, 0x44, 0x00, 0x00, 0x3b, 0x44, - 0x00, 0x20, 0x3b, 0x44, 0x00, 0x40, 0x3b, 0x44, 0x00, 0x60, 0x3b, 0x44, - 0x00, 0x80, 0x3b, 0x44, 0x00, 0xa0, 0x3b, 0x44, 0x00, 0xc0, 0x3b, 0x44, - 0x00, 0xe0, 0x3b, 0x44, 0x00, 0x00, 0x3c, 0x44, 0x00, 0x20, 0x3c, 0x44, - 0x00, 0x40, 0x3c, 0x44, 0x00, 0x60, 0x3c, 0x44, 0x00, 0x80, 0x3c, 0x44, - 0x00, 0xa0, 0x3c, 0x44, 0x00, 0xc0, 0x3c, 0x44, 0x00, 0xe0, 0x3c, 0x44, - 0x00, 0x00, 0x3d, 0x44, 0x00, 0x20, 0x3d, 0x44, 0x00, 0x40, 0x3d, 0x44, - 0x00, 0x60, 0x3d, 0x44, 0x00, 0x80, 0x3d, 0x44, 0x00, 0xa0, 0x3d, 0x44, - 0x00, 0xc0, 0x3d, 0x44, 0x00, 0xe0, 0x3d, 0x44, 0x00, 0x00, 0x3e, 0x44, - 0x00, 0x20, 0x3e, 0x44, 0x00, 0x40, 0x3e, 0x44, 0x00, 0x60, 0x3e, 0x44, - 0x00, 0x80, 0x3e, 0x44, 0x00, 0xa0, 0x3e, 0x44, 0x00, 0xc0, 0x3e, 0x44, - 0x00, 0xe0, 0x3e, 0x44, 0x00, 0x00, 0x3f, 0x44, 0x00, 0x20, 0x3f, 0x44, - 0x00, 0x40, 0x3f, 0x44, 0x00, 0x60, 0x3f, 0x44, 0x00, 0x80, 0x3f, 0x44, - 0x00, 0xa0, 0x3f, 0x44, 0x00, 0xc0, 0x3f, 0x44, 0x00, 0xe0, 0x3f, 0x44, - 0x00, 0x00, 0x40, 0x44, 0x00, 0x20, 0x40, 0x44, 0x00, 0x40, 0x40, 0x44, - 0x00, 0x60, 0x40, 0x44, 0x00, 0x80, 0x40, 0x44, 0x00, 0xa0, 0x40, 0x44, - 0x00, 0xc0, 0x40, 0x44, 0x00, 0xe0, 0x40, 0x44, 0x00, 0x00, 0x41, 0x44, - 0x00, 0x20, 0x41, 0x44, 0x00, 0x40, 0x41, 0x44, 0x00, 0x60, 0x41, 0x44, - 0x00, 0x80, 0x41, 0x44, 0x00, 0xa0, 0x41, 0x44, 0x00, 0xc0, 0x41, 0x44, - 0x00, 0xe0, 0x41, 0x44, 0x00, 0x00, 0x42, 0x44, 0x00, 0x20, 0x42, 0x44, - 0x00, 0x40, 0x42, 0x44, 0x00, 0x60, 0x42, 0x44, 0x00, 0x80, 0x42, 0x44, - 0x00, 0xa0, 0x42, 0x44, 0x00, 0xc0, 0x42, 0x44, 0x00, 0xe0, 0x42, 0x44, - 0x00, 0x00, 0x43, 0x44, 0x00, 0x20, 0x43, 0x44, 0x00, 0x40, 0x43, 0x44, - 0x00, 0x60, 0x43, 0x44, 0x00, 0x80, 0x43, 0x44, 0x00, 0xa0, 0x43, 0x44, - 0x00, 0xc0, 0x43, 0x44, 0x00, 0xe0, 0x43, 0x44, 0x00, 0x00, 0x44, 0x44, - 0x00, 0x20, 0x44, 0x44, 0x00, 0x40, 0x44, 0x44, 0x00, 0x60, 0x44, 0x44, - 0x00, 0x80, 0x44, 0x44, 0x00, 0xa0, 0x44, 0x44, 0x00, 0xc0, 0x44, 0x44, - 0x00, 0xe0, 0x44, 0x44, 0x00, 0x00, 0x45, 0x44, 0x00, 0x20, 0x45, 0x44, - 0x00, 0x40, 0x45, 0x44, 0x00, 0x60, 0x45, 0x44, 0x00, 0x80, 0x45, 0x44, - 0x00, 0xa0, 0x45, 0x44, 0x00, 0xc0, 0x45, 0x44, 0x00, 0xe0, 0x45, 0x44, - 0x00, 0x00, 0x46, 0x44, 0x00, 0x20, 0x46, 0x44, 0x00, 0x40, 0x46, 0x44, - 0x00, 0x60, 0x46, 0x44, 0x00, 0x80, 0x46, 0x44, 0x00, 0xa0, 0x46, 0x44, - 0x00, 0xc0, 0x46, 0x44, 0x00, 0xe0, 0x46, 0x44, 0x00, 0x00, 0x47, 0x44, - 0x00, 0x20, 0x47, 0x44, 0x00, 0x40, 0x47, 0x44, 0x00, 0x60, 0x47, 0x44, - 0x00, 0x80, 0x47, 0x44, 0x00, 0xa0, 0x47, 0x44, 0x00, 0xc0, 0x47, 0x44, - 0x00, 0xe0, 0x47, 0x44, 0x00, 0x00, 0x48, 0x44, 0x00, 0x20, 0x48, 0x44, - 0x00, 0x40, 0x48, 0x44, 0x00, 0x60, 0x48, 0x44, 0x00, 0x80, 0x48, 0x44, - 0x00, 0xa0, 0x48, 0x44, 0x00, 0xc0, 0x48, 0x44, 0x00, 0xe0, 0x48, 0x44, - 0x00, 0x00, 0x49, 0x44, 0x00, 0x20, 0x49, 0x44, 0x00, 0x40, 0x49, 0x44, - 0x00, 0x60, 0x49, 0x44, 0x00, 0x80, 0x49, 0x44, 0x00, 0xa0, 0x49, 0x44, - 0x00, 0xc0, 0x49, 0x44, 0x00, 0xe0, 0x49, 0x44, 0x00, 0x00, 0x4a, 0x44, - 0x00, 0x20, 0x4a, 0x44, 0x00, 0x40, 0x4a, 0x44, 0x00, 0x60, 0x4a, 0x44, - 0x00, 0x80, 0x4a, 0x44, 0x00, 0xa0, 0x4a, 0x44, 0x00, 0xc0, 0x4a, 0x44, - 0x00, 0xe0, 0x4a, 0x44, 0x00, 0x00, 0x4b, 0x44, 0x00, 0x20, 0x4b, 0x44, - 0x00, 0x40, 0x4b, 0x44, 0x00, 0x60, 0x4b, 0x44, 0x00, 0x80, 0x4b, 0x44, - 0x00, 0xa0, 0x4b, 0x44, 0x00, 0xc0, 0x4b, 0x44, 0x00, 0xe0, 0x4b, 0x44, - 0x00, 0x00, 0x4c, 0x44, 0x00, 0x20, 0x4c, 0x44, 0x00, 0x40, 0x4c, 0x44, - 0x00, 0x60, 0x4c, 0x44, 0x00, 0x80, 0x4c, 0x44, 0x00, 0xa0, 0x4c, 0x44, - 0x00, 0xc0, 0x4c, 0x44, 0x00, 0xe0, 0x4c, 0x44, 0x00, 0x00, 0x4d, 0x44, - 0x00, 0x20, 0x4d, 0x44, 0x00, 0x40, 0x4d, 0x44, 0x00, 0x60, 0x4d, 0x44, - 0x00, 0x80, 0x4d, 0x44, 0x00, 0xa0, 0x4d, 0x44, 0x00, 0xc0, 0x4d, 0x44, - 0x00, 0xe0, 0x4d, 0x44, 0x00, 0x00, 0x4e, 0x44, 0x00, 0x20, 0x4e, 0x44, - 0x00, 0x40, 0x4e, 0x44, 0x00, 0x60, 0x4e, 0x44, 0x00, 0x80, 0x4e, 0x44, - 0x00, 0xa0, 0x4e, 0x44, 0x00, 0xc0, 0x4e, 0x44, 0x00, 0xe0, 0x4e, 0x44, - 0x00, 0x00, 0x4f, 0x44, 0x00, 0x20, 0x4f, 0x44, 0x00, 0x40, 0x4f, 0x44, - 0x00, 0x60, 0x4f, 0x44, 0x00, 0x80, 0x4f, 0x44, 0x00, 0xa0, 0x4f, 0x44, - 0x00, 0xc0, 0x4f, 0x44, 0x00, 0xe0, 0x4f, 0x44, 0x00, 0x00, 0x50, 0x44, - 0x00, 0x20, 0x50, 0x44, 0x00, 0x40, 0x50, 0x44, 0x00, 0x60, 0x50, 0x44, - 0x00, 0x80, 0x50, 0x44, 0x00, 0xa0, 0x50, 0x44, 0x00, 0xc0, 0x50, 0x44, - 0x00, 0xe0, 0x50, 0x44, 0x00, 0x00, 0x51, 0x44, 0x00, 0x20, 0x51, 0x44, - 0x00, 0x40, 0x51, 0x44, 0x00, 0x60, 0x51, 0x44, 0x00, 0x80, 0x51, 0x44, - 0x00, 0xa0, 0x51, 0x44, 0x00, 0xc0, 0x51, 0x44, 0x00, 0xe0, 0x51, 0x44, - 0x00, 0x00, 0x52, 0x44, 0x00, 0x20, 0x52, 0x44, 0x00, 0x40, 0x52, 0x44, - 0x00, 0x60, 0x52, 0x44, 0x00, 0x80, 0x52, 0x44, 0x00, 0xa0, 0x52, 0x44, - 0x00, 0xc0, 0x52, 0x44, 0x00, 0xe0, 0x52, 0x44, 0x00, 0x00, 0x53, 0x44, - 0x00, 0x20, 0x53, 0x44, 0x00, 0x40, 0x53, 0x44, 0x00, 0x60, 0x53, 0x44, - 0x00, 0x80, 0x53, 0x44, 0x00, 0xa0, 0x53, 0x44, 0x00, 0xc0, 0x53, 0x44, - 0x00, 0xe0, 0x53, 0x44, 0x00, 0x00, 0x54, 0x44, 0x00, 0x20, 0x54, 0x44, - 0x00, 0x40, 0x54, 0x44, 0x00, 0x60, 0x54, 0x44, 0x00, 0x80, 0x54, 0x44, - 0x00, 0xa0, 0x54, 0x44, 0x00, 0xc0, 0x54, 0x44, 0x00, 0xe0, 0x54, 0x44, - 0x00, 0x00, 0x55, 0x44, 0x00, 0x20, 0x55, 0x44, 0x00, 0x40, 0x55, 0x44, - 0x00, 0x60, 0x55, 0x44, 0x00, 0x80, 0x55, 0x44, 0x00, 0xa0, 0x55, 0x44, - 0x00, 0xc0, 0x55, 0x44, 0x00, 0xe0, 0x55, 0x44, 0x00, 0x00, 0x56, 0x44, - 0x00, 0x20, 0x56, 0x44, 0x00, 0x40, 0x56, 0x44, 0x00, 0x60, 0x56, 0x44, - 0x00, 0x80, 0x56, 0x44, 0x00, 0xa0, 0x56, 0x44, 0x00, 0xc0, 0x56, 0x44, - 0x00, 0xe0, 0x56, 0x44, 0x00, 0x00, 0x57, 0x44, 0x00, 0x20, 0x57, 0x44, - 0x00, 0x40, 0x57, 0x44, 0x00, 0x60, 0x57, 0x44, 0x00, 0x80, 0x57, 0x44, - 0x00, 0xa0, 0x57, 0x44, 0x00, 0xc0, 0x57, 0x44, 0x00, 0xe0, 0x57, 0x44, - 0x00, 0x00, 0x58, 0x44, 0x00, 0x20, 0x58, 0x44, 0x00, 0x40, 0x58, 0x44, - 0x00, 0x60, 0x58, 0x44, 0x00, 0x80, 0x58, 0x44, 0x00, 0xa0, 0x58, 0x44, - 0x00, 0xc0, 0x58, 0x44, 0x00, 0xe0, 0x58, 0x44, 0x00, 0x00, 0x59, 0x44, - 0x00, 0x20, 0x59, 0x44, 0x00, 0x40, 0x59, 0x44, 0x00, 0x60, 0x59, 0x44, - 0x00, 0x80, 0x59, 0x44, 0x00, 0xa0, 0x59, 0x44, 0x00, 0xc0, 0x59, 0x44, - 0x00, 0xe0, 0x59, 0x44, 0x00, 0x00, 0x5a, 0x44, 0x00, 0x20, 0x5a, 0x44, - 0x00, 0x40, 0x5a, 0x44, 0x00, 0x60, 0x5a, 0x44, 0x00, 0x80, 0x5a, 0x44, - 0x00, 0xa0, 0x5a, 0x44, 0x00, 0xc0, 0x5a, 0x44, 0x00, 0xe0, 0x5a, 0x44, - 0x00, 0x00, 0x5b, 0x44, 0x00, 0x20, 0x5b, 0x44, 0x00, 0x40, 0x5b, 0x44, - 0x00, 0x60, 0x5b, 0x44, 0x00, 0x80, 0x5b, 0x44, 0x00, 0xa0, 0x5b, 0x44, - 0x00, 0xc0, 0x5b, 0x44, 0x00, 0xe0, 0x5b, 0x44, 0x00, 0x00, 0x5c, 0x44, - 0x00, 0x20, 0x5c, 0x44, 0x00, 0x40, 0x5c, 0x44, 0x00, 0x60, 0x5c, 0x44, - 0x00, 0x80, 0x5c, 0x44, 0x00, 0xa0, 0x5c, 0x44, 0x00, 0xc0, 0x5c, 0x44, - 0x00, 0xe0, 0x5c, 0x44, 0x00, 0x00, 0x5d, 0x44, 0x00, 0x20, 0x5d, 0x44, - 0x00, 0x40, 0x5d, 0x44, 0x00, 0x60, 0x5d, 0x44, 0x00, 0x80, 0x5d, 0x44, - 0x00, 0xa0, 0x5d, 0x44, 0x00, 0xc0, 0x5d, 0x44, 0x00, 0xe0, 0x5d, 0x44, - 0x00, 0x00, 0x5e, 0x44, 0x00, 0x20, 0x5e, 0x44, 0x00, 0x40, 0x5e, 0x44, - 0x00, 0x60, 0x5e, 0x44, 0x00, 0x80, 0x5e, 0x44, 0x00, 0xa0, 0x5e, 0x44, - 0x00, 0xc0, 0x5e, 0x44, 0x00, 0xe0, 0x5e, 0x44, 0x00, 0x00, 0x5f, 0x44, - 0x00, 0x20, 0x5f, 0x44, 0x00, 0x40, 0x5f, 0x44, 0x00, 0x60, 0x5f, 0x44, - 0x00, 0x80, 0x5f, 0x44, 0x00, 0xa0, 0x5f, 0x44, 0x00, 0xc0, 0x5f, 0x44, - 0x00, 0xe0, 0x5f, 0x44, 0x00, 0x00, 0x60, 0x44, 0x00, 0x20, 0x60, 0x44, - 0x00, 0x40, 0x60, 0x44, 0x00, 0x60, 0x60, 0x44, 0x00, 0x80, 0x60, 0x44, - 0x00, 0xa0, 0x60, 0x44, 0x00, 0xc0, 0x60, 0x44, 0x00, 0xe0, 0x60, 0x44, - 0x00, 0x00, 0x61, 0x44, 0x00, 0x20, 0x61, 0x44, 0x00, 0x40, 0x61, 0x44, - 0x00, 0x60, 0x61, 0x44, 0x00, 0x80, 0x61, 0x44, 0x00, 0xa0, 0x61, 0x44, - 0x00, 0xc0, 0x61, 0x44, 0x00, 0xe0, 0x61, 0x44, 0x00, 0x00, 0x62, 0x44, - 0x00, 0x20, 0x62, 0x44, 0x00, 0x40, 0x62, 0x44, 0x00, 0x60, 0x62, 0x44, - 0x00, 0x80, 0x62, 0x44, 0x00, 0xa0, 0x62, 0x44, 0x00, 0xc0, 0x62, 0x44, - 0x00, 0xe0, 0x62, 0x44, 0x00, 0x00, 0x63, 0x44, 0x00, 0x20, 0x63, 0x44, - 0x00, 0x40, 0x63, 0x44, 0x00, 0x60, 0x63, 0x44, 0x00, 0x80, 0x63, 0x44, - 0x00, 0xa0, 0x63, 0x44, 0x00, 0xc0, 0x63, 0x44, 0x00, 0xe0, 0x63, 0x44, - 0x00, 0x00, 0x64, 0x44, 0x00, 0x20, 0x64, 0x44, 0x00, 0x40, 0x64, 0x44, - 0x00, 0x60, 0x64, 0x44, 0x00, 0x80, 0x64, 0x44, 0x00, 0xa0, 0x64, 0x44, - 0x00, 0xc0, 0x64, 0x44, 0x00, 0xe0, 0x64, 0x44, 0x00, 0x00, 0x65, 0x44, - 0x00, 0x20, 0x65, 0x44, 0x00, 0x40, 0x65, 0x44, 0x00, 0x60, 0x65, 0x44, - 0x00, 0x80, 0x65, 0x44, 0x00, 0xa0, 0x65, 0x44, 0x00, 0xc0, 0x65, 0x44, - 0x00, 0xe0, 0x65, 0x44, 0x00, 0x00, 0x66, 0x44, 0x00, 0x20, 0x66, 0x44, - 0x00, 0x40, 0x66, 0x44, 0x00, 0x60, 0x66, 0x44, 0x00, 0x80, 0x66, 0x44, - 0x00, 0xa0, 0x66, 0x44, 0x00, 0xc0, 0x66, 0x44, 0x00, 0xe0, 0x66, 0x44, - 0x00, 0x00, 0x67, 0x44, 0x00, 0x20, 0x67, 0x44, 0x00, 0x40, 0x67, 0x44, - 0x00, 0x60, 0x67, 0x44, 0x00, 0x80, 0x67, 0x44, 0x00, 0xa0, 0x67, 0x44, - 0x00, 0xc0, 0x67, 0x44, 0x00, 0xe0, 0x67, 0x44, 0x00, 0x00, 0x68, 0x44, - 0x00, 0x20, 0x68, 0x44, 0x00, 0x40, 0x68, 0x44, 0x00, 0x60, 0x68, 0x44, - 0x00, 0x80, 0x68, 0x44, 0x00, 0xa0, 0x68, 0x44, 0x00, 0xc0, 0x68, 0x44, - 0x00, 0xe0, 0x68, 0x44, 0x00, 0x00, 0x69, 0x44, 0x00, 0x20, 0x69, 0x44, - 0x00, 0x40, 0x69, 0x44, 0x00, 0x60, 0x69, 0x44, 0x00, 0x80, 0x69, 0x44, - 0x00, 0xa0, 0x69, 0x44, 0x00, 0xc0, 0x69, 0x44, 0x00, 0xe0, 0x69, 0x44, - 0x00, 0x00, 0x6a, 0x44, 0x00, 0x20, 0x6a, 0x44, 0x00, 0x40, 0x6a, 0x44, - 0x00, 0x60, 0x6a, 0x44, 0x00, 0x80, 0x6a, 0x44, 0x00, 0xa0, 0x6a, 0x44, - 0x00, 0xc0, 0x6a, 0x44, 0x00, 0xe0, 0x6a, 0x44, 0x00, 0x00, 0x6b, 0x44, - 0x00, 0x20, 0x6b, 0x44, 0x00, 0x40, 0x6b, 0x44, 0x00, 0x60, 0x6b, 0x44, - 0x00, 0x80, 0x6b, 0x44, 0x00, 0xa0, 0x6b, 0x44, 0x00, 0xc0, 0x6b, 0x44, - 0x00, 0xe0, 0x6b, 0x44, 0x00, 0x00, 0x6c, 0x44, 0x00, 0x20, 0x6c, 0x44, - 0x00, 0x40, 0x6c, 0x44, 0x00, 0x60, 0x6c, 0x44, 0x00, 0x80, 0x6c, 0x44, - 0x00, 0xa0, 0x6c, 0x44, 0x00, 0xc0, 0x6c, 0x44, 0x00, 0xe0, 0x6c, 0x44, - 0x00, 0x00, 0x6d, 0x44, 0x00, 0x20, 0x6d, 0x44, 0x00, 0x40, 0x6d, 0x44, - 0x00, 0x60, 0x6d, 0x44, 0x00, 0x80, 0x6d, 0x44, 0x00, 0xa0, 0x6d, 0x44, - 0x00, 0xc0, 0x6d, 0x44, 0x00, 0xe0, 0x6d, 0x44, 0x00, 0x00, 0x6e, 0x44, - 0x00, 0x20, 0x6e, 0x44, 0x00, 0x40, 0x6e, 0x44, 0x00, 0x60, 0x6e, 0x44, - 0x00, 0x80, 0x6e, 0x44, 0x00, 0xa0, 0x6e, 0x44, 0x00, 0xc0, 0x6e, 0x44, - 0x00, 0xe0, 0x6e, 0x44, 0x00, 0x00, 0x6f, 0x44, 0x00, 0x20, 0x6f, 0x44, - 0x00, 0x40, 0x6f, 0x44, 0x00, 0x60, 0x6f, 0x44, 0x00, 0x80, 0x6f, 0x44, - 0x00, 0xa0, 0x6f, 0x44, 0x00, 0xc0, 0x6f, 0x44, 0x00, 0xe0, 0x6f, 0x44, - 0x00, 0x00, 0x70, 0x44, 0x00, 0x20, 0x70, 0x44, 0x00, 0x40, 0x70, 0x44, - 0x00, 0x60, 0x70, 0x44, 0x00, 0x80, 0x70, 0x44, 0x00, 0xa0, 0x70, 0x44, - 0x00, 0xc0, 0x70, 0x44, 0x00, 0xe0, 0x70, 0x44, 0x00, 0x00, 0x71, 0x44, - 0x00, 0x20, 0x71, 0x44, 0x00, 0x40, 0x71, 0x44, 0x00, 0x60, 0x71, 0x44, - 0x00, 0x80, 0x71, 0x44, 0x00, 0xa0, 0x71, 0x44, 0x00, 0xc0, 0x71, 0x44, - 0x00, 0xe0, 0x71, 0x44, 0x00, 0x00, 0x72, 0x44, 0x00, 0x20, 0x72, 0x44, - 0x00, 0x40, 0x72, 0x44, 0x00, 0x60, 0x72, 0x44, 0x00, 0x80, 0x72, 0x44, - 0x00, 0xa0, 0x72, 0x44, 0x00, 0xc0, 0x72, 0x44, 0x00, 0xe0, 0x72, 0x44, - 0x00, 0x00, 0x73, 0x44, 0x00, 0x20, 0x73, 0x44, 0x00, 0x40, 0x73, 0x44, - 0x00, 0x60, 0x73, 0x44, 0x00, 0x80, 0x73, 0x44, 0x00, 0xa0, 0x73, 0x44, - 0x00, 0xc0, 0x73, 0x44, 0x00, 0xe0, 0x73, 0x44, 0x00, 0x00, 0x74, 0x44, - 0x00, 0x20, 0x74, 0x44, 0x00, 0x40, 0x74, 0x44, 0x00, 0x60, 0x74, 0x44, - 0x00, 0x80, 0x74, 0x44, 0x00, 0xa0, 0x74, 0x44, 0x00, 0xc0, 0x74, 0x44, - 0x00, 0xe0, 0x74, 0x44, 0x00, 0x00, 0x75, 0x44, 0x00, 0x20, 0x75, 0x44, - 0x00, 0x40, 0x75, 0x44, 0x00, 0x60, 0x75, 0x44, 0x00, 0x80, 0x75, 0x44, - 0x00, 0xa0, 0x75, 0x44, 0x00, 0xc0, 0x75, 0x44, 0x00, 0xe0, 0x75, 0x44, - 0x00, 0x00, 0x76, 0x44, 0x00, 0x20, 0x76, 0x44, 0x00, 0x40, 0x76, 0x44, - 0x00, 0x60, 0x76, 0x44, 0x00, 0x80, 0x76, 0x44, 0x00, 0xa0, 0x76, 0x44, - 0x00, 0xc0, 0x76, 0x44, 0x00, 0xe0, 0x76, 0x44, 0x00, 0x00, 0x77, 0x44, - 0x00, 0x20, 0x77, 0x44, 0x00, 0x40, 0x77, 0x44, 0x00, 0x60, 0x77, 0x44, - 0x00, 0x80, 0x77, 0x44, 0x00, 0xa0, 0x77, 0x44, 0x00, 0xc0, 0x77, 0x44, - 0x00, 0xe0, 0x77, 0x44, 0x00, 0x00, 0x78, 0x44, 0x00, 0x20, 0x78, 0x44, - 0x00, 0x40, 0x78, 0x44, 0x00, 0x60, 0x78, 0x44, 0x00, 0x80, 0x78, 0x44, - 0x00, 0xa0, 0x78, 0x44, 0x00, 0xc0, 0x78, 0x44, 0x00, 0xe0, 0x78, 0x44, - 0x00, 0x00, 0x79, 0x44, 0x00, 0x20, 0x79, 0x44, 0x00, 0x40, 0x79, 0x44, - 0x00, 0x60, 0x79, 0x44, 0x00, 0x80, 0x79, 0x44, 0x00, 0xa0, 0x79, 0x44, - 0x00, 0xc0, 0x79, 0x44, 0x00, 0xe0, 0x79, 0x44, 0x00, 0x00, 0x7a, 0x44, - 0x00, 0x20, 0x7a, 0x44, 0x00, 0x40, 0x7a, 0x44, 0x00, 0x60, 0x7a, 0x44, - 0x00, 0x80, 0x7a, 0x44, 0x00, 0xa0, 0x7a, 0x44, 0x00, 0xc0, 0x7a, 0x44, - 0x00, 0xe0, 0x7a, 0x44, 0x00, 0x00, 0x7b, 0x44, 0x00, 0x20, 0x7b, 0x44, - 0x00, 0x40, 0x7b, 0x44, 0x00, 0x60, 0x7b, 0x44, 0x00, 0x80, 0x7b, 0x44, - 0x00, 0xa0, 0x7b, 0x44, 0x00, 0xc0, 0x7b, 0x44, 0x00, 0xe0, 0x7b, 0x44, - 0x00, 0x00, 0x7c, 0x44, 0x00, 0x20, 0x7c, 0x44, 0x00, 0x40, 0x7c, 0x44, - 0x00, 0x60, 0x7c, 0x44, 0x00, 0x80, 0x7c, 0x44, 0x00, 0xa0, 0x7c, 0x44, - 0x00, 0xc0, 0x7c, 0x44, 0x00, 0xe0, 0x7c, 0x44, 0x00, 0x00, 0x7d, 0x44, - 0x00, 0x20, 0x7d, 0x44, 0x00, 0x40, 0x7d, 0x44, 0x00, 0x60, 0x7d, 0x44, - 0x00, 0x80, 0x7d, 0x44, 0x00, 0xa0, 0x7d, 0x44, 0x00, 0xc0, 0x7d, 0x44, - 0x00, 0xe0, 0x7d, 0x44, 0x00, 0x00, 0x7e, 0x44, 0x00, 0x20, 0x7e, 0x44, - 0x00, 0x40, 0x7e, 0x44, 0x00, 0x60, 0x7e, 0x44, 0x00, 0x80, 0x7e, 0x44, - 0x00, 0xa0, 0x7e, 0x44, 0x00, 0xc0, 0x7e, 0x44, 0x00, 0xe0, 0x7e, 0x44, - 0x00, 0x00, 0x7f, 0x44, 0x00, 0x20, 0x7f, 0x44, 0x00, 0x40, 0x7f, 0x44, - 0x00, 0x60, 0x7f, 0x44, 0x00, 0x80, 0x7f, 0x44, 0x00, 0xa0, 0x7f, 0x44, - 0x00, 0xc0, 0x7f, 0x44, 0x00, 0xe0, 0x7f, 0x44, 0x00, 0x00, 0x80, 0x44, - 0x00, 0x10, 0x80, 0x44, 0x00, 0x20, 0x80, 0x44 -}; + 0x00, 0x00, 0xc0, 0x3f, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, + 0x00, 0x60, 0x40, 0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0x90, 0x40, 0x00, 0x00, 0xa0, 0x40, 0x00, 0x00, + 0xb0, 0x40, 0x00, 0x00, 0xc0, 0x40, 0x00, 0x00, 0xd0, 0x40, 0x00, 0x00, 0xe0, 0x40, 0x00, 0x00, 0xf0, + 0x40, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x08, 0x41, 0x00, 0x00, 0x10, 0x41, 0x00, 0x00, 0x18, 0x41, + 0x00, 0x00, 0x20, 0x41, 0x00, 0x00, 0x28, 0x41, 0x00, 0x00, 0x30, 0x41, 0x00, 0x00, 0x38, 0x41, 0x00, + 0x00, 0x40, 0x41, 0x00, 0x00, 0x48, 0x41, 0x00, 0x00, 0x50, 0x41, 0x00, 0x00, 0x58, 0x41, 0x00, 0x00, + 0x60, 0x41, 0x00, 0x00, 0x68, 0x41, 0x00, 0x00, 0x70, 0x41, 0x00, 0x00, 0x78, 0x41, 0x00, 0x00, 0x80, + 0x41, 0x00, 0x00, 0x84, 0x41, 0x00, 0x00, 0x88, 0x41, 0x00, 0x00, 0x8c, 0x41, 0x00, 0x00, 0x90, 0x41, + 0x00, 0x00, 0x94, 0x41, 0x00, 0x00, 0x98, 0x41, 0x00, 0x00, 0x9c, 0x41, 0x00, 0x00, 0xa0, 0x41, 0x00, + 0x00, 0xa4, 0x41, 0x00, 0x00, 0xa8, 0x41, 0x00, 0x00, 0xac, 0x41, 0x00, 0x00, 0xb0, 0x41, 0x00, 0x00, + 0xb4, 0x41, 0x00, 0x00, 0xb8, 0x41, 0x00, 0x00, 0xbc, 0x41, 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xc4, + 0x41, 0x00, 0x00, 0xc8, 0x41, 0x00, 0x00, 0xcc, 0x41, 0x00, 0x00, 0xd0, 0x41, 0x00, 0x00, 0xd4, 0x41, + 0x00, 0x00, 0xd8, 0x41, 0x00, 0x00, 0xdc, 0x41, 0x00, 0x00, 0xe0, 0x41, 0x00, 0x00, 0xe4, 0x41, 0x00, + 0x00, 0xe8, 0x41, 0x00, 0x00, 0xec, 0x41, 0x00, 0x00, 0xf0, 0x41, 0x00, 0x00, 0xf4, 0x41, 0x00, 0x00, + 0xf8, 0x41, 0x00, 0x00, 0xfc, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x02, 0x42, 0x00, 0x00, 0x04, + 0x42, 0x00, 0x00, 0x06, 0x42, 0x00, 0x00, 0x08, 0x42, 0x00, 0x00, 0x0a, 0x42, 0x00, 0x00, 0x0c, 0x42, + 0x00, 0x00, 0x0e, 0x42, 0x00, 0x00, 0x10, 0x42, 0x00, 0x00, 0x12, 0x42, 0x00, 0x00, 0x14, 0x42, 0x00, + 0x00, 0x16, 0x42, 0x00, 0x00, 0x18, 0x42, 0x00, 0x00, 0x1a, 0x42, 0x00, 0x00, 0x1c, 0x42, 0x00, 0x00, + 0x1e, 0x42, 0x00, 0x00, 0x20, 0x42, 0x00, 0x00, 0x22, 0x42, 0x00, 0x00, 0x24, 0x42, 0x00, 0x00, 0x26, + 0x42, 0x00, 0x00, 0x28, 0x42, 0x00, 0x00, 0x2a, 0x42, 0x00, 0x00, 0x2c, 0x42, 0x00, 0x00, 0x2e, 0x42, + 0x00, 0x00, 0x30, 0x42, 0x00, 0x00, 0x32, 0x42, 0x00, 0x00, 0x34, 0x42, 0x00, 0x00, 0x36, 0x42, 0x00, + 0x00, 0x38, 0x42, 0x00, 0x00, 0x3a, 0x42, 0x00, 0x00, 0x3c, 0x42, 0x00, 0x00, 0x3e, 0x42, 0x00, 0x00, + 0x40, 0x42, 0x00, 0x00, 0x42, 0x42, 0x00, 0x00, 0x44, 0x42, 0x00, 0x00, 0x46, 0x42, 0x00, 0x00, 0x48, + 0x42, 0x00, 0x00, 0x4a, 0x42, 0x00, 0x00, 0x4c, 0x42, 0x00, 0x00, 0x4e, 0x42, 0x00, 0x00, 0x50, 0x42, + 0x00, 0x00, 0x52, 0x42, 0x00, 0x00, 0x54, 0x42, 0x00, 0x00, 0x56, 0x42, 0x00, 0x00, 0x58, 0x42, 0x00, + 0x00, 0x5a, 0x42, 0x00, 0x00, 0x5c, 0x42, 0x00, 0x00, 0x5e, 0x42, 0x00, 0x00, 0x60, 0x42, 0x00, 0x00, + 0x62, 0x42, 0x00, 0x00, 0x64, 0x42, 0x00, 0x00, 0x66, 0x42, 0x00, 0x00, 0x68, 0x42, 0x00, 0x00, 0x6a, + 0x42, 0x00, 0x00, 0x6c, 0x42, 0x00, 0x00, 0x6e, 0x42, 0x00, 0x00, 0x70, 0x42, 0x00, 0x00, 0x72, 0x42, + 0x00, 0x00, 0x74, 0x42, 0x00, 0x00, 0x76, 0x42, 0x00, 0x00, 0x78, 0x42, 0x00, 0x00, 0x7a, 0x42, 0x00, + 0x00, 0x7c, 0x42, 0x00, 0x00, 0x7e, 0x42, 0x00, 0x00, 0x80, 0x42, 0x00, 0x00, 0x81, 0x42, 0x00, 0x00, + 0x82, 0x42, 0x00, 0x00, 0x83, 0x42, 0x00, 0x00, 0x84, 0x42, 0x00, 0x00, 0x85, 0x42, 0x00, 0x00, 0x86, + 0x42, 0x00, 0x00, 0x87, 0x42, 0x00, 0x00, 0x88, 0x42, 0x00, 0x00, 0x89, 0x42, 0x00, 0x00, 0x8a, 0x42, + 0x00, 0x00, 0x8b, 0x42, 0x00, 0x00, 0x8c, 0x42, 0x00, 0x00, 0x8d, 0x42, 0x00, 0x00, 0x8e, 0x42, 0x00, + 0x00, 0x8f, 0x42, 0x00, 0x00, 0x90, 0x42, 0x00, 0x00, 0x91, 0x42, 0x00, 0x00, 0x92, 0x42, 0x00, 0x00, + 0x93, 0x42, 0x00, 0x00, 0x94, 0x42, 0x00, 0x00, 0x95, 0x42, 0x00, 0x00, 0x96, 0x42, 0x00, 0x00, 0x97, + 0x42, 0x00, 0x00, 0x98, 0x42, 0x00, 0x00, 0x99, 0x42, 0x00, 0x00, 0x9a, 0x42, 0x00, 0x00, 0x9b, 0x42, + 0x00, 0x00, 0x9c, 0x42, 0x00, 0x00, 0x9d, 0x42, 0x00, 0x00, 0x9e, 0x42, 0x00, 0x00, 0x9f, 0x42, 0x00, + 0x00, 0xa0, 0x42, 0x00, 0x00, 0xa1, 0x42, 0x00, 0x00, 0xa2, 0x42, 0x00, 0x00, 0xa3, 0x42, 0x00, 0x00, + 0xa4, 0x42, 0x00, 0x00, 0xa5, 0x42, 0x00, 0x00, 0xa6, 0x42, 0x00, 0x00, 0xa7, 0x42, 0x00, 0x00, 0xa8, + 0x42, 0x00, 0x00, 0xa9, 0x42, 0x00, 0x00, 0xaa, 0x42, 0x00, 0x00, 0xab, 0x42, 0x00, 0x00, 0xac, 0x42, + 0x00, 0x00, 0xad, 0x42, 0x00, 0x00, 0xae, 0x42, 0x00, 0x00, 0xaf, 0x42, 0x00, 0x00, 0xb0, 0x42, 0x00, + 0x00, 0xb1, 0x42, 0x00, 0x00, 0xb2, 0x42, 0x00, 0x00, 0xb3, 0x42, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x00, + 0xb5, 0x42, 0x00, 0x00, 0xb6, 0x42, 0x00, 0x00, 0xb7, 0x42, 0x00, 0x00, 0xb8, 0x42, 0x00, 0x00, 0xb9, + 0x42, 0x00, 0x00, 0xba, 0x42, 0x00, 0x00, 0xbb, 0x42, 0x00, 0x00, 0xbc, 0x42, 0x00, 0x00, 0xbd, 0x42, + 0x00, 0x00, 0xbe, 0x42, 0x00, 0x00, 0xbf, 0x42, 0x00, 0x00, 0xc0, 0x42, 0x00, 0x00, 0xc1, 0x42, 0x00, + 0x00, 0xc2, 0x42, 0x00, 0x00, 0xc3, 0x42, 0x00, 0x00, 0xc4, 0x42, 0x00, 0x00, 0xc5, 0x42, 0x00, 0x00, + 0xc6, 0x42, 0x00, 0x00, 0xc7, 0x42, 0x00, 0x00, 0xc8, 0x42, 0x00, 0x00, 0xc9, 0x42, 0x00, 0x00, 0xca, + 0x42, 0x00, 0x00, 0xcb, 0x42, 0x00, 0x00, 0xcc, 0x42, 0x00, 0x00, 0xcd, 0x42, 0x00, 0x00, 0xce, 0x42, + 0x00, 0x00, 0xcf, 0x42, 0x00, 0x00, 0xd0, 0x42, 0x00, 0x00, 0xd1, 0x42, 0x00, 0x00, 0xd2, 0x42, 0x00, + 0x00, 0xd3, 0x42, 0x00, 0x00, 0xd4, 0x42, 0x00, 0x00, 0xd5, 0x42, 0x00, 0x00, 0xd6, 0x42, 0x00, 0x00, + 0xd7, 0x42, 0x00, 0x00, 0xd8, 0x42, 0x00, 0x00, 0xd9, 0x42, 0x00, 0x00, 0xda, 0x42, 0x00, 0x00, 0xdb, + 0x42, 0x00, 0x00, 0xdc, 0x42, 0x00, 0x00, 0xdd, 0x42, 0x00, 0x00, 0xde, 0x42, 0x00, 0x00, 0xdf, 0x42, + 0x00, 0x00, 0xe0, 0x42, 0x00, 0x00, 0xe1, 0x42, 0x00, 0x00, 0xe2, 0x42, 0x00, 0x00, 0xe3, 0x42, 0x00, + 0x00, 0xe4, 0x42, 0x00, 0x00, 0xe5, 0x42, 0x00, 0x00, 0xe6, 0x42, 0x00, 0x00, 0xe7, 0x42, 0x00, 0x00, + 0xe8, 0x42, 0x00, 0x00, 0xe9, 0x42, 0x00, 0x00, 0xea, 0x42, 0x00, 0x00, 0xeb, 0x42, 0x00, 0x00, 0xec, + 0x42, 0x00, 0x00, 0xed, 0x42, 0x00, 0x00, 0xee, 0x42, 0x00, 0x00, 0xef, 0x42, 0x00, 0x00, 0xf0, 0x42, + 0x00, 0x00, 0xf1, 0x42, 0x00, 0x00, 0xf2, 0x42, 0x00, 0x00, 0xf3, 0x42, 0x00, 0x00, 0xf4, 0x42, 0x00, + 0x00, 0xf5, 0x42, 0x00, 0x00, 0xf6, 0x42, 0x00, 0x00, 0xf7, 0x42, 0x00, 0x00, 0xf8, 0x42, 0x00, 0x00, + 0xf9, 0x42, 0x00, 0x00, 0xfa, 0x42, 0x00, 0x00, 0xfb, 0x42, 0x00, 0x00, 0xfc, 0x42, 0x00, 0x00, 0xfd, + 0x42, 0x00, 0x00, 0xfe, 0x42, 0x00, 0x00, 0xff, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x80, 0x00, 0x43, + 0x00, 0x00, 0x01, 0x43, 0x00, 0x80, 0x01, 0x43, 0x00, 0x00, 0x02, 0x43, 0x00, 0x80, 0x02, 0x43, 0x00, + 0x00, 0x03, 0x43, 0x00, 0x80, 0x03, 0x43, 0x00, 0x00, 0x04, 0x43, 0x00, 0x80, 0x04, 0x43, 0x00, 0x00, + 0x05, 0x43, 0x00, 0x80, 0x05, 0x43, 0x00, 0x00, 0x06, 0x43, 0x00, 0x80, 0x06, 0x43, 0x00, 0x00, 0x07, + 0x43, 0x00, 0x80, 0x07, 0x43, 0x00, 0x00, 0x08, 0x43, 0x00, 0x80, 0x08, 0x43, 0x00, 0x00, 0x09, 0x43, + 0x00, 0x80, 0x09, 0x43, 0x00, 0x00, 0x0a, 0x43, 0x00, 0x80, 0x0a, 0x43, 0x00, 0x00, 0x0b, 0x43, 0x00, + 0x80, 0x0b, 0x43, 0x00, 0x00, 0x0c, 0x43, 0x00, 0x80, 0x0c, 0x43, 0x00, 0x00, 0x0d, 0x43, 0x00, 0x80, + 0x0d, 0x43, 0x00, 0x00, 0x0e, 0x43, 0x00, 0x80, 0x0e, 0x43, 0x00, 0x00, 0x0f, 0x43, 0x00, 0x80, 0x0f, + 0x43, 0x00, 0x00, 0x10, 0x43, 0x00, 0x80, 0x10, 0x43, 0x00, 0x00, 0x11, 0x43, 0x00, 0x80, 0x11, 0x43, + 0x00, 0x00, 0x12, 0x43, 0x00, 0x80, 0x12, 0x43, 0x00, 0x00, 0x13, 0x43, 0x00, 0x80, 0x13, 0x43, 0x00, + 0x00, 0x14, 0x43, 0x00, 0x80, 0x14, 0x43, 0x00, 0x00, 0x15, 0x43, 0x00, 0x80, 0x15, 0x43, 0x00, 0x00, + 0x16, 0x43, 0x00, 0x80, 0x16, 0x43, 0x00, 0x00, 0x17, 0x43, 0x00, 0x80, 0x17, 0x43, 0x00, 0x00, 0x18, + 0x43, 0x00, 0x80, 0x18, 0x43, 0x00, 0x00, 0x19, 0x43, 0x00, 0x80, 0x19, 0x43, 0x00, 0x00, 0x1a, 0x43, + 0x00, 0x80, 0x1a, 0x43, 0x00, 0x00, 0x1b, 0x43, 0x00, 0x80, 0x1b, 0x43, 0x00, 0x00, 0x1c, 0x43, 0x00, + 0x80, 0x1c, 0x43, 0x00, 0x00, 0x1d, 0x43, 0x00, 0x80, 0x1d, 0x43, 0x00, 0x00, 0x1e, 0x43, 0x00, 0x80, + 0x1e, 0x43, 0x00, 0x00, 0x1f, 0x43, 0x00, 0x80, 0x1f, 0x43, 0x00, 0x00, 0x20, 0x43, 0x00, 0x80, 0x20, + 0x43, 0x00, 0x00, 0x21, 0x43, 0x00, 0x80, 0x21, 0x43, 0x00, 0x00, 0x22, 0x43, 0x00, 0x80, 0x22, 0x43, + 0x00, 0x00, 0x23, 0x43, 0x00, 0x80, 0x23, 0x43, 0x00, 0x00, 0x24, 0x43, 0x00, 0x80, 0x24, 0x43, 0x00, + 0x00, 0x25, 0x43, 0x00, 0x80, 0x25, 0x43, 0x00, 0x00, 0x26, 0x43, 0x00, 0x80, 0x26, 0x43, 0x00, 0x00, + 0x27, 0x43, 0x00, 0x80, 0x27, 0x43, 0x00, 0x00, 0x28, 0x43, 0x00, 0x80, 0x28, 0x43, 0x00, 0x00, 0x29, + 0x43, 0x00, 0x80, 0x29, 0x43, 0x00, 0x00, 0x2a, 0x43, 0x00, 0x80, 0x2a, 0x43, 0x00, 0x00, 0x2b, 0x43, + 0x00, 0x80, 0x2b, 0x43, 0x00, 0x00, 0x2c, 0x43, 0x00, 0x80, 0x2c, 0x43, 0x00, 0x00, 0x2d, 0x43, 0x00, + 0x80, 0x2d, 0x43, 0x00, 0x00, 0x2e, 0x43, 0x00, 0x80, 0x2e, 0x43, 0x00, 0x00, 0x2f, 0x43, 0x00, 0x80, + 0x2f, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x80, 0x30, 0x43, 0x00, 0x00, 0x31, 0x43, 0x00, 0x80, 0x31, + 0x43, 0x00, 0x00, 0x32, 0x43, 0x00, 0x80, 0x32, 0x43, 0x00, 0x00, 0x33, 0x43, 0x00, 0x80, 0x33, 0x43, + 0x00, 0x00, 0x34, 0x43, 0x00, 0x80, 0x34, 0x43, 0x00, 0x00, 0x35, 0x43, 0x00, 0x80, 0x35, 0x43, 0x00, + 0x00, 0x36, 0x43, 0x00, 0x80, 0x36, 0x43, 0x00, 0x00, 0x37, 0x43, 0x00, 0x80, 0x37, 0x43, 0x00, 0x00, + 0x38, 0x43, 0x00, 0x80, 0x38, 0x43, 0x00, 0x00, 0x39, 0x43, 0x00, 0x80, 0x39, 0x43, 0x00, 0x00, 0x3a, + 0x43, 0x00, 0x80, 0x3a, 0x43, 0x00, 0x00, 0x3b, 0x43, 0x00, 0x80, 0x3b, 0x43, 0x00, 0x00, 0x3c, 0x43, + 0x00, 0x80, 0x3c, 0x43, 0x00, 0x00, 0x3d, 0x43, 0x00, 0x80, 0x3d, 0x43, 0x00, 0x00, 0x3e, 0x43, 0x00, + 0x80, 0x3e, 0x43, 0x00, 0x00, 0x3f, 0x43, 0x00, 0x80, 0x3f, 0x43, 0x00, 0x00, 0x40, 0x43, 0x00, 0x80, + 0x40, 0x43, 0x00, 0x00, 0x41, 0x43, 0x00, 0x80, 0x41, 0x43, 0x00, 0x00, 0x42, 0x43, 0x00, 0x80, 0x42, + 0x43, 0x00, 0x00, 0x43, 0x43, 0x00, 0x80, 0x43, 0x43, 0x00, 0x00, 0x44, 0x43, 0x00, 0x80, 0x44, 0x43, + 0x00, 0x00, 0x45, 0x43, 0x00, 0x80, 0x45, 0x43, 0x00, 0x00, 0x46, 0x43, 0x00, 0x80, 0x46, 0x43, 0x00, + 0x00, 0x47, 0x43, 0x00, 0x80, 0x47, 0x43, 0x00, 0x00, 0x48, 0x43, 0x00, 0x80, 0x48, 0x43, 0x00, 0x00, + 0x49, 0x43, 0x00, 0x80, 0x49, 0x43, 0x00, 0x00, 0x4a, 0x43, 0x00, 0x80, 0x4a, 0x43, 0x00, 0x00, 0x4b, + 0x43, 0x00, 0x80, 0x4b, 0x43, 0x00, 0x00, 0x4c, 0x43, 0x00, 0x80, 0x4c, 0x43, 0x00, 0x00, 0x4d, 0x43, + 0x00, 0x80, 0x4d, 0x43, 0x00, 0x00, 0x4e, 0x43, 0x00, 0x80, 0x4e, 0x43, 0x00, 0x00, 0x4f, 0x43, 0x00, + 0x80, 0x4f, 0x43, 0x00, 0x00, 0x50, 0x43, 0x00, 0x80, 0x50, 0x43, 0x00, 0x00, 0x51, 0x43, 0x00, 0x80, + 0x51, 0x43, 0x00, 0x00, 0x52, 0x43, 0x00, 0x80, 0x52, 0x43, 0x00, 0x00, 0x53, 0x43, 0x00, 0x80, 0x53, + 0x43, 0x00, 0x00, 0x54, 0x43, 0x00, 0x80, 0x54, 0x43, 0x00, 0x00, 0x55, 0x43, 0x00, 0x80, 0x55, 0x43, + 0x00, 0x00, 0x56, 0x43, 0x00, 0x80, 0x56, 0x43, 0x00, 0x00, 0x57, 0x43, 0x00, 0x80, 0x57, 0x43, 0x00, + 0x00, 0x58, 0x43, 0x00, 0x80, 0x58, 0x43, 0x00, 0x00, 0x59, 0x43, 0x00, 0x80, 0x59, 0x43, 0x00, 0x00, + 0x5a, 0x43, 0x00, 0x80, 0x5a, 0x43, 0x00, 0x00, 0x5b, 0x43, 0x00, 0x80, 0x5b, 0x43, 0x00, 0x00, 0x5c, + 0x43, 0x00, 0x80, 0x5c, 0x43, 0x00, 0x00, 0x5d, 0x43, 0x00, 0x80, 0x5d, 0x43, 0x00, 0x00, 0x5e, 0x43, + 0x00, 0x80, 0x5e, 0x43, 0x00, 0x00, 0x5f, 0x43, 0x00, 0x80, 0x5f, 0x43, 0x00, 0x00, 0x60, 0x43, 0x00, + 0x80, 0x60, 0x43, 0x00, 0x00, 0x61, 0x43, 0x00, 0x80, 0x61, 0x43, 0x00, 0x00, 0x62, 0x43, 0x00, 0x80, + 0x62, 0x43, 0x00, 0x00, 0x63, 0x43, 0x00, 0x80, 0x63, 0x43, 0x00, 0x00, 0x64, 0x43, 0x00, 0x80, 0x64, + 0x43, 0x00, 0x00, 0x65, 0x43, 0x00, 0x80, 0x65, 0x43, 0x00, 0x00, 0x66, 0x43, 0x00, 0x80, 0x66, 0x43, + 0x00, 0x00, 0x67, 0x43, 0x00, 0x80, 0x67, 0x43, 0x00, 0x00, 0x68, 0x43, 0x00, 0x80, 0x68, 0x43, 0x00, + 0x00, 0x69, 0x43, 0x00, 0x80, 0x69, 0x43, 0x00, 0x00, 0x6a, 0x43, 0x00, 0x80, 0x6a, 0x43, 0x00, 0x00, + 0x6b, 0x43, 0x00, 0x80, 0x6b, 0x43, 0x00, 0x00, 0x6c, 0x43, 0x00, 0x80, 0x6c, 0x43, 0x00, 0x00, 0x6d, + 0x43, 0x00, 0x80, 0x6d, 0x43, 0x00, 0x00, 0x6e, 0x43, 0x00, 0x80, 0x6e, 0x43, 0x00, 0x00, 0x6f, 0x43, + 0x00, 0x80, 0x6f, 0x43, 0x00, 0x00, 0x70, 0x43, 0x00, 0x80, 0x70, 0x43, 0x00, 0x00, 0x71, 0x43, 0x00, + 0x80, 0x71, 0x43, 0x00, 0x00, 0x72, 0x43, 0x00, 0x80, 0x72, 0x43, 0x00, 0x00, 0x73, 0x43, 0x00, 0x80, + 0x73, 0x43, 0x00, 0x00, 0x74, 0x43, 0x00, 0x80, 0x74, 0x43, 0x00, 0x00, 0x75, 0x43, 0x00, 0x80, 0x75, + 0x43, 0x00, 0x00, 0x76, 0x43, 0x00, 0x80, 0x76, 0x43, 0x00, 0x00, 0x77, 0x43, 0x00, 0x80, 0x77, 0x43, + 0x00, 0x00, 0x78, 0x43, 0x00, 0x80, 0x78, 0x43, 0x00, 0x00, 0x79, 0x43, 0x00, 0x80, 0x79, 0x43, 0x00, + 0x00, 0x7a, 0x43, 0x00, 0x80, 0x7a, 0x43, 0x00, 0x00, 0x7b, 0x43, 0x00, 0x80, 0x7b, 0x43, 0x00, 0x00, + 0x7c, 0x43, 0x00, 0x80, 0x7c, 0x43, 0x00, 0x00, 0x7d, 0x43, 0x00, 0x80, 0x7d, 0x43, 0x00, 0x00, 0x7e, + 0x43, 0x00, 0x80, 0x7e, 0x43, 0x00, 0x00, 0x7f, 0x43, 0x00, 0x80, 0x7f, 0x43, 0x00, 0x00, 0x80, 0x43, + 0x00, 0x40, 0x80, 0x43, 0x00, 0x80, 0x80, 0x43, 0x00, 0xc0, 0x80, 0x43, 0x00, 0x00, 0x81, 0x43, 0x00, + 0x40, 0x81, 0x43, 0x00, 0x80, 0x81, 0x43, 0x00, 0xc0, 0x81, 0x43, 0x00, 0x00, 0x82, 0x43, 0x00, 0x40, + 0x82, 0x43, 0x00, 0x80, 0x82, 0x43, 0x00, 0xc0, 0x82, 0x43, 0x00, 0x00, 0x83, 0x43, 0x00, 0x40, 0x83, + 0x43, 0x00, 0x80, 0x83, 0x43, 0x00, 0xc0, 0x83, 0x43, 0x00, 0x00, 0x84, 0x43, 0x00, 0x40, 0x84, 0x43, + 0x00, 0x80, 0x84, 0x43, 0x00, 0xc0, 0x84, 0x43, 0x00, 0x00, 0x85, 0x43, 0x00, 0x40, 0x85, 0x43, 0x00, + 0x80, 0x85, 0x43, 0x00, 0xc0, 0x85, 0x43, 0x00, 0x00, 0x86, 0x43, 0x00, 0x40, 0x86, 0x43, 0x00, 0x80, + 0x86, 0x43, 0x00, 0xc0, 0x86, 0x43, 0x00, 0x00, 0x87, 0x43, 0x00, 0x40, 0x87, 0x43, 0x00, 0x80, 0x87, + 0x43, 0x00, 0xc0, 0x87, 0x43, 0x00, 0x00, 0x88, 0x43, 0x00, 0x40, 0x88, 0x43, 0x00, 0x80, 0x88, 0x43, + 0x00, 0xc0, 0x88, 0x43, 0x00, 0x00, 0x89, 0x43, 0x00, 0x40, 0x89, 0x43, 0x00, 0x80, 0x89, 0x43, 0x00, + 0xc0, 0x89, 0x43, 0x00, 0x00, 0x8a, 0x43, 0x00, 0x40, 0x8a, 0x43, 0x00, 0x80, 0x8a, 0x43, 0x00, 0xc0, + 0x8a, 0x43, 0x00, 0x00, 0x8b, 0x43, 0x00, 0x40, 0x8b, 0x43, 0x00, 0x80, 0x8b, 0x43, 0x00, 0xc0, 0x8b, + 0x43, 0x00, 0x00, 0x8c, 0x43, 0x00, 0x40, 0x8c, 0x43, 0x00, 0x80, 0x8c, 0x43, 0x00, 0xc0, 0x8c, 0x43, + 0x00, 0x00, 0x8d, 0x43, 0x00, 0x40, 0x8d, 0x43, 0x00, 0x80, 0x8d, 0x43, 0x00, 0xc0, 0x8d, 0x43, 0x00, + 0x00, 0x8e, 0x43, 0x00, 0x40, 0x8e, 0x43, 0x00, 0x80, 0x8e, 0x43, 0x00, 0xc0, 0x8e, 0x43, 0x00, 0x00, + 0x8f, 0x43, 0x00, 0x40, 0x8f, 0x43, 0x00, 0x80, 0x8f, 0x43, 0x00, 0xc0, 0x8f, 0x43, 0x00, 0x00, 0x90, + 0x43, 0x00, 0x40, 0x90, 0x43, 0x00, 0x80, 0x90, 0x43, 0x00, 0xc0, 0x90, 0x43, 0x00, 0x00, 0x91, 0x43, + 0x00, 0x40, 0x91, 0x43, 0x00, 0x80, 0x91, 0x43, 0x00, 0xc0, 0x91, 0x43, 0x00, 0x00, 0x92, 0x43, 0x00, + 0x40, 0x92, 0x43, 0x00, 0x80, 0x92, 0x43, 0x00, 0xc0, 0x92, 0x43, 0x00, 0x00, 0x93, 0x43, 0x00, 0x40, + 0x93, 0x43, 0x00, 0x80, 0x93, 0x43, 0x00, 0xc0, 0x93, 0x43, 0x00, 0x00, 0x94, 0x43, 0x00, 0x40, 0x94, + 0x43, 0x00, 0x80, 0x94, 0x43, 0x00, 0xc0, 0x94, 0x43, 0x00, 0x00, 0x95, 0x43, 0x00, 0x40, 0x95, 0x43, + 0x00, 0x80, 0x95, 0x43, 0x00, 0xc0, 0x95, 0x43, 0x00, 0x00, 0x96, 0x43, 0x00, 0x40, 0x96, 0x43, 0x00, + 0x80, 0x96, 0x43, 0x00, 0xc0, 0x96, 0x43, 0x00, 0x00, 0x97, 0x43, 0x00, 0x40, 0x97, 0x43, 0x00, 0x80, + 0x97, 0x43, 0x00, 0xc0, 0x97, 0x43, 0x00, 0x00, 0x98, 0x43, 0x00, 0x40, 0x98, 0x43, 0x00, 0x80, 0x98, + 0x43, 0x00, 0xc0, 0x98, 0x43, 0x00, 0x00, 0x99, 0x43, 0x00, 0x40, 0x99, 0x43, 0x00, 0x80, 0x99, 0x43, + 0x00, 0xc0, 0x99, 0x43, 0x00, 0x00, 0x9a, 0x43, 0x00, 0x40, 0x9a, 0x43, 0x00, 0x80, 0x9a, 0x43, 0x00, + 0xc0, 0x9a, 0x43, 0x00, 0x00, 0x9b, 0x43, 0x00, 0x40, 0x9b, 0x43, 0x00, 0x80, 0x9b, 0x43, 0x00, 0xc0, + 0x9b, 0x43, 0x00, 0x00, 0x9c, 0x43, 0x00, 0x40, 0x9c, 0x43, 0x00, 0x80, 0x9c, 0x43, 0x00, 0xc0, 0x9c, + 0x43, 0x00, 0x00, 0x9d, 0x43, 0x00, 0x40, 0x9d, 0x43, 0x00, 0x80, 0x9d, 0x43, 0x00, 0xc0, 0x9d, 0x43, + 0x00, 0x00, 0x9e, 0x43, 0x00, 0x40, 0x9e, 0x43, 0x00, 0x80, 0x9e, 0x43, 0x00, 0xc0, 0x9e, 0x43, 0x00, + 0x00, 0x9f, 0x43, 0x00, 0x40, 0x9f, 0x43, 0x00, 0x80, 0x9f, 0x43, 0x00, 0xc0, 0x9f, 0x43, 0x00, 0x00, + 0xa0, 0x43, 0x00, 0x40, 0xa0, 0x43, 0x00, 0x80, 0xa0, 0x43, 0x00, 0xc0, 0xa0, 0x43, 0x00, 0x00, 0xa1, + 0x43, 0x00, 0x40, 0xa1, 0x43, 0x00, 0x80, 0xa1, 0x43, 0x00, 0xc0, 0xa1, 0x43, 0x00, 0x00, 0xa2, 0x43, + 0x00, 0x40, 0xa2, 0x43, 0x00, 0x80, 0xa2, 0x43, 0x00, 0xc0, 0xa2, 0x43, 0x00, 0x00, 0xa3, 0x43, 0x00, + 0x40, 0xa3, 0x43, 0x00, 0x80, 0xa3, 0x43, 0x00, 0xc0, 0xa3, 0x43, 0x00, 0x00, 0xa4, 0x43, 0x00, 0x40, + 0xa4, 0x43, 0x00, 0x80, 0xa4, 0x43, 0x00, 0xc0, 0xa4, 0x43, 0x00, 0x00, 0xa5, 0x43, 0x00, 0x40, 0xa5, + 0x43, 0x00, 0x80, 0xa5, 0x43, 0x00, 0xc0, 0xa5, 0x43, 0x00, 0x00, 0xa6, 0x43, 0x00, 0x40, 0xa6, 0x43, + 0x00, 0x80, 0xa6, 0x43, 0x00, 0xc0, 0xa6, 0x43, 0x00, 0x00, 0xa7, 0x43, 0x00, 0x40, 0xa7, 0x43, 0x00, + 0x80, 0xa7, 0x43, 0x00, 0xc0, 0xa7, 0x43, 0x00, 0x00, 0xa8, 0x43, 0x00, 0x40, 0xa8, 0x43, 0x00, 0x80, + 0xa8, 0x43, 0x00, 0xc0, 0xa8, 0x43, 0x00, 0x00, 0xa9, 0x43, 0x00, 0x40, 0xa9, 0x43, 0x00, 0x80, 0xa9, + 0x43, 0x00, 0xc0, 0xa9, 0x43, 0x00, 0x00, 0xaa, 0x43, 0x00, 0x40, 0xaa, 0x43, 0x00, 0x80, 0xaa, 0x43, + 0x00, 0xc0, 0xaa, 0x43, 0x00, 0x00, 0xab, 0x43, 0x00, 0x40, 0xab, 0x43, 0x00, 0x80, 0xab, 0x43, 0x00, + 0xc0, 0xab, 0x43, 0x00, 0x00, 0xac, 0x43, 0x00, 0x40, 0xac, 0x43, 0x00, 0x80, 0xac, 0x43, 0x00, 0xc0, + 0xac, 0x43, 0x00, 0x00, 0xad, 0x43, 0x00, 0x40, 0xad, 0x43, 0x00, 0x80, 0xad, 0x43, 0x00, 0xc0, 0xad, + 0x43, 0x00, 0x00, 0xae, 0x43, 0x00, 0x40, 0xae, 0x43, 0x00, 0x80, 0xae, 0x43, 0x00, 0xc0, 0xae, 0x43, + 0x00, 0x00, 0xaf, 0x43, 0x00, 0x40, 0xaf, 0x43, 0x00, 0x80, 0xaf, 0x43, 0x00, 0xc0, 0xaf, 0x43, 0x00, + 0x00, 0xb0, 0x43, 0x00, 0x40, 0xb0, 0x43, 0x00, 0x80, 0xb0, 0x43, 0x00, 0xc0, 0xb0, 0x43, 0x00, 0x00, + 0xb1, 0x43, 0x00, 0x40, 0xb1, 0x43, 0x00, 0x80, 0xb1, 0x43, 0x00, 0xc0, 0xb1, 0x43, 0x00, 0x00, 0xb2, + 0x43, 0x00, 0x40, 0xb2, 0x43, 0x00, 0x80, 0xb2, 0x43, 0x00, 0xc0, 0xb2, 0x43, 0x00, 0x00, 0xb3, 0x43, + 0x00, 0x40, 0xb3, 0x43, 0x00, 0x80, 0xb3, 0x43, 0x00, 0xc0, 0xb3, 0x43, 0x00, 0x00, 0xb4, 0x43, 0x00, + 0x40, 0xb4, 0x43, 0x00, 0x80, 0xb4, 0x43, 0x00, 0xc0, 0xb4, 0x43, 0x00, 0x00, 0xb5, 0x43, 0x00, 0x40, + 0xb5, 0x43, 0x00, 0x80, 0xb5, 0x43, 0x00, 0xc0, 0xb5, 0x43, 0x00, 0x00, 0xb6, 0x43, 0x00, 0x40, 0xb6, + 0x43, 0x00, 0x80, 0xb6, 0x43, 0x00, 0xc0, 0xb6, 0x43, 0x00, 0x00, 0xb7, 0x43, 0x00, 0x40, 0xb7, 0x43, + 0x00, 0x80, 0xb7, 0x43, 0x00, 0xc0, 0xb7, 0x43, 0x00, 0x00, 0xb8, 0x43, 0x00, 0x40, 0xb8, 0x43, 0x00, + 0x80, 0xb8, 0x43, 0x00, 0xc0, 0xb8, 0x43, 0x00, 0x00, 0xb9, 0x43, 0x00, 0x40, 0xb9, 0x43, 0x00, 0x80, + 0xb9, 0x43, 0x00, 0xc0, 0xb9, 0x43, 0x00, 0x00, 0xba, 0x43, 0x00, 0x40, 0xba, 0x43, 0x00, 0x80, 0xba, + 0x43, 0x00, 0xc0, 0xba, 0x43, 0x00, 0x00, 0xbb, 0x43, 0x00, 0x40, 0xbb, 0x43, 0x00, 0x80, 0xbb, 0x43, + 0x00, 0xc0, 0xbb, 0x43, 0x00, 0x00, 0xbc, 0x43, 0x00, 0x40, 0xbc, 0x43, 0x00, 0x80, 0xbc, 0x43, 0x00, + 0xc0, 0xbc, 0x43, 0x00, 0x00, 0xbd, 0x43, 0x00, 0x40, 0xbd, 0x43, 0x00, 0x80, 0xbd, 0x43, 0x00, 0xc0, + 0xbd, 0x43, 0x00, 0x00, 0xbe, 0x43, 0x00, 0x40, 0xbe, 0x43, 0x00, 0x80, 0xbe, 0x43, 0x00, 0xc0, 0xbe, + 0x43, 0x00, 0x00, 0xbf, 0x43, 0x00, 0x40, 0xbf, 0x43, 0x00, 0x80, 0xbf, 0x43, 0x00, 0xc0, 0xbf, 0x43, + 0x00, 0x00, 0xc0, 0x43, 0x00, 0x40, 0xc0, 0x43, 0x00, 0x80, 0xc0, 0x43, 0x00, 0xc0, 0xc0, 0x43, 0x00, + 0x00, 0xc1, 0x43, 0x00, 0x40, 0xc1, 0x43, 0x00, 0x80, 0xc1, 0x43, 0x00, 0xc0, 0xc1, 0x43, 0x00, 0x00, + 0xc2, 0x43, 0x00, 0x40, 0xc2, 0x43, 0x00, 0x80, 0xc2, 0x43, 0x00, 0xc0, 0xc2, 0x43, 0x00, 0x00, 0xc3, + 0x43, 0x00, 0x40, 0xc3, 0x43, 0x00, 0x80, 0xc3, 0x43, 0x00, 0xc0, 0xc3, 0x43, 0x00, 0x00, 0xc4, 0x43, + 0x00, 0x40, 0xc4, 0x43, 0x00, 0x80, 0xc4, 0x43, 0x00, 0xc0, 0xc4, 0x43, 0x00, 0x00, 0xc5, 0x43, 0x00, + 0x40, 0xc5, 0x43, 0x00, 0x80, 0xc5, 0x43, 0x00, 0xc0, 0xc5, 0x43, 0x00, 0x00, 0xc6, 0x43, 0x00, 0x40, + 0xc6, 0x43, 0x00, 0x80, 0xc6, 0x43, 0x00, 0xc0, 0xc6, 0x43, 0x00, 0x00, 0xc7, 0x43, 0x00, 0x40, 0xc7, + 0x43, 0x00, 0x80, 0xc7, 0x43, 0x00, 0xc0, 0xc7, 0x43, 0x00, 0x00, 0xc8, 0x43, 0x00, 0x40, 0xc8, 0x43, + 0x00, 0x80, 0xc8, 0x43, 0x00, 0xc0, 0xc8, 0x43, 0x00, 0x00, 0xc9, 0x43, 0x00, 0x40, 0xc9, 0x43, 0x00, + 0x80, 0xc9, 0x43, 0x00, 0xc0, 0xc9, 0x43, 0x00, 0x00, 0xca, 0x43, 0x00, 0x40, 0xca, 0x43, 0x00, 0x80, + 0xca, 0x43, 0x00, 0xc0, 0xca, 0x43, 0x00, 0x00, 0xcb, 0x43, 0x00, 0x40, 0xcb, 0x43, 0x00, 0x80, 0xcb, + 0x43, 0x00, 0xc0, 0xcb, 0x43, 0x00, 0x00, 0xcc, 0x43, 0x00, 0x40, 0xcc, 0x43, 0x00, 0x80, 0xcc, 0x43, + 0x00, 0xc0, 0xcc, 0x43, 0x00, 0x00, 0xcd, 0x43, 0x00, 0x40, 0xcd, 0x43, 0x00, 0x80, 0xcd, 0x43, 0x00, + 0xc0, 0xcd, 0x43, 0x00, 0x00, 0xce, 0x43, 0x00, 0x40, 0xce, 0x43, 0x00, 0x80, 0xce, 0x43, 0x00, 0xc0, + 0xce, 0x43, 0x00, 0x00, 0xcf, 0x43, 0x00, 0x40, 0xcf, 0x43, 0x00, 0x80, 0xcf, 0x43, 0x00, 0xc0, 0xcf, + 0x43, 0x00, 0x00, 0xd0, 0x43, 0x00, 0x40, 0xd0, 0x43, 0x00, 0x80, 0xd0, 0x43, 0x00, 0xc0, 0xd0, 0x43, + 0x00, 0x00, 0xd1, 0x43, 0x00, 0x40, 0xd1, 0x43, 0x00, 0x80, 0xd1, 0x43, 0x00, 0xc0, 0xd1, 0x43, 0x00, + 0x00, 0xd2, 0x43, 0x00, 0x40, 0xd2, 0x43, 0x00, 0x80, 0xd2, 0x43, 0x00, 0xc0, 0xd2, 0x43, 0x00, 0x00, + 0xd3, 0x43, 0x00, 0x40, 0xd3, 0x43, 0x00, 0x80, 0xd3, 0x43, 0x00, 0xc0, 0xd3, 0x43, 0x00, 0x00, 0xd4, + 0x43, 0x00, 0x40, 0xd4, 0x43, 0x00, 0x80, 0xd4, 0x43, 0x00, 0xc0, 0xd4, 0x43, 0x00, 0x00, 0xd5, 0x43, + 0x00, 0x40, 0xd5, 0x43, 0x00, 0x80, 0xd5, 0x43, 0x00, 0xc0, 0xd5, 0x43, 0x00, 0x00, 0xd6, 0x43, 0x00, + 0x40, 0xd6, 0x43, 0x00, 0x80, 0xd6, 0x43, 0x00, 0xc0, 0xd6, 0x43, 0x00, 0x00, 0xd7, 0x43, 0x00, 0x40, + 0xd7, 0x43, 0x00, 0x80, 0xd7, 0x43, 0x00, 0xc0, 0xd7, 0x43, 0x00, 0x00, 0xd8, 0x43, 0x00, 0x40, 0xd8, + 0x43, 0x00, 0x80, 0xd8, 0x43, 0x00, 0xc0, 0xd8, 0x43, 0x00, 0x00, 0xd9, 0x43, 0x00, 0x40, 0xd9, 0x43, + 0x00, 0x80, 0xd9, 0x43, 0x00, 0xc0, 0xd9, 0x43, 0x00, 0x00, 0xda, 0x43, 0x00, 0x40, 0xda, 0x43, 0x00, + 0x80, 0xda, 0x43, 0x00, 0xc0, 0xda, 0x43, 0x00, 0x00, 0xdb, 0x43, 0x00, 0x40, 0xdb, 0x43, 0x00, 0x80, + 0xdb, 0x43, 0x00, 0xc0, 0xdb, 0x43, 0x00, 0x00, 0xdc, 0x43, 0x00, 0x40, 0xdc, 0x43, 0x00, 0x80, 0xdc, + 0x43, 0x00, 0xc0, 0xdc, 0x43, 0x00, 0x00, 0xdd, 0x43, 0x00, 0x40, 0xdd, 0x43, 0x00, 0x80, 0xdd, 0x43, + 0x00, 0xc0, 0xdd, 0x43, 0x00, 0x00, 0xde, 0x43, 0x00, 0x40, 0xde, 0x43, 0x00, 0x80, 0xde, 0x43, 0x00, + 0xc0, 0xde, 0x43, 0x00, 0x00, 0xdf, 0x43, 0x00, 0x40, 0xdf, 0x43, 0x00, 0x80, 0xdf, 0x43, 0x00, 0xc0, + 0xdf, 0x43, 0x00, 0x00, 0xe0, 0x43, 0x00, 0x40, 0xe0, 0x43, 0x00, 0x80, 0xe0, 0x43, 0x00, 0xc0, 0xe0, + 0x43, 0x00, 0x00, 0xe1, 0x43, 0x00, 0x40, 0xe1, 0x43, 0x00, 0x80, 0xe1, 0x43, 0x00, 0xc0, 0xe1, 0x43, + 0x00, 0x00, 0xe2, 0x43, 0x00, 0x40, 0xe2, 0x43, 0x00, 0x80, 0xe2, 0x43, 0x00, 0xc0, 0xe2, 0x43, 0x00, + 0x00, 0xe3, 0x43, 0x00, 0x40, 0xe3, 0x43, 0x00, 0x80, 0xe3, 0x43, 0x00, 0xc0, 0xe3, 0x43, 0x00, 0x00, + 0xe4, 0x43, 0x00, 0x40, 0xe4, 0x43, 0x00, 0x80, 0xe4, 0x43, 0x00, 0xc0, 0xe4, 0x43, 0x00, 0x00, 0xe5, + 0x43, 0x00, 0x40, 0xe5, 0x43, 0x00, 0x80, 0xe5, 0x43, 0x00, 0xc0, 0xe5, 0x43, 0x00, 0x00, 0xe6, 0x43, + 0x00, 0x40, 0xe6, 0x43, 0x00, 0x80, 0xe6, 0x43, 0x00, 0xc0, 0xe6, 0x43, 0x00, 0x00, 0xe7, 0x43, 0x00, + 0x40, 0xe7, 0x43, 0x00, 0x80, 0xe7, 0x43, 0x00, 0xc0, 0xe7, 0x43, 0x00, 0x00, 0xe8, 0x43, 0x00, 0x40, + 0xe8, 0x43, 0x00, 0x80, 0xe8, 0x43, 0x00, 0xc0, 0xe8, 0x43, 0x00, 0x00, 0xe9, 0x43, 0x00, 0x40, 0xe9, + 0x43, 0x00, 0x80, 0xe9, 0x43, 0x00, 0xc0, 0xe9, 0x43, 0x00, 0x00, 0xea, 0x43, 0x00, 0x40, 0xea, 0x43, + 0x00, 0x80, 0xea, 0x43, 0x00, 0xc0, 0xea, 0x43, 0x00, 0x00, 0xeb, 0x43, 0x00, 0x40, 0xeb, 0x43, 0x00, + 0x80, 0xeb, 0x43, 0x00, 0xc0, 0xeb, 0x43, 0x00, 0x00, 0xec, 0x43, 0x00, 0x40, 0xec, 0x43, 0x00, 0x80, + 0xec, 0x43, 0x00, 0xc0, 0xec, 0x43, 0x00, 0x00, 0xed, 0x43, 0x00, 0x40, 0xed, 0x43, 0x00, 0x80, 0xed, + 0x43, 0x00, 0xc0, 0xed, 0x43, 0x00, 0x00, 0xee, 0x43, 0x00, 0x40, 0xee, 0x43, 0x00, 0x80, 0xee, 0x43, + 0x00, 0xc0, 0xee, 0x43, 0x00, 0x00, 0xef, 0x43, 0x00, 0x40, 0xef, 0x43, 0x00, 0x80, 0xef, 0x43, 0x00, + 0xc0, 0xef, 0x43, 0x00, 0x00, 0xf0, 0x43, 0x00, 0x40, 0xf0, 0x43, 0x00, 0x80, 0xf0, 0x43, 0x00, 0xc0, + 0xf0, 0x43, 0x00, 0x00, 0xf1, 0x43, 0x00, 0x40, 0xf1, 0x43, 0x00, 0x80, 0xf1, 0x43, 0x00, 0xc0, 0xf1, + 0x43, 0x00, 0x00, 0xf2, 0x43, 0x00, 0x40, 0xf2, 0x43, 0x00, 0x80, 0xf2, 0x43, 0x00, 0xc0, 0xf2, 0x43, + 0x00, 0x00, 0xf3, 0x43, 0x00, 0x40, 0xf3, 0x43, 0x00, 0x80, 0xf3, 0x43, 0x00, 0xc0, 0xf3, 0x43, 0x00, + 0x00, 0xf4, 0x43, 0x00, 0x40, 0xf4, 0x43, 0x00, 0x80, 0xf4, 0x43, 0x00, 0xc0, 0xf4, 0x43, 0x00, 0x00, + 0xf5, 0x43, 0x00, 0x40, 0xf5, 0x43, 0x00, 0x80, 0xf5, 0x43, 0x00, 0xc0, 0xf5, 0x43, 0x00, 0x00, 0xf6, + 0x43, 0x00, 0x40, 0xf6, 0x43, 0x00, 0x80, 0xf6, 0x43, 0x00, 0xc0, 0xf6, 0x43, 0x00, 0x00, 0xf7, 0x43, + 0x00, 0x40, 0xf7, 0x43, 0x00, 0x80, 0xf7, 0x43, 0x00, 0xc0, 0xf7, 0x43, 0x00, 0x00, 0xf8, 0x43, 0x00, + 0x40, 0xf8, 0x43, 0x00, 0x80, 0xf8, 0x43, 0x00, 0xc0, 0xf8, 0x43, 0x00, 0x00, 0xf9, 0x43, 0x00, 0x40, + 0xf9, 0x43, 0x00, 0x80, 0xf9, 0x43, 0x00, 0xc0, 0xf9, 0x43, 0x00, 0x00, 0xfa, 0x43, 0x00, 0x40, 0xfa, + 0x43, 0x00, 0x80, 0xfa, 0x43, 0x00, 0xc0, 0xfa, 0x43, 0x00, 0x00, 0xfb, 0x43, 0x00, 0x40, 0xfb, 0x43, + 0x00, 0x80, 0xfb, 0x43, 0x00, 0xc0, 0xfb, 0x43, 0x00, 0x00, 0xfc, 0x43, 0x00, 0x40, 0xfc, 0x43, 0x00, + 0x80, 0xfc, 0x43, 0x00, 0xc0, 0xfc, 0x43, 0x00, 0x00, 0xfd, 0x43, 0x00, 0x40, 0xfd, 0x43, 0x00, 0x80, + 0xfd, 0x43, 0x00, 0xc0, 0xfd, 0x43, 0x00, 0x00, 0xfe, 0x43, 0x00, 0x40, 0xfe, 0x43, 0x00, 0x80, 0xfe, + 0x43, 0x00, 0xc0, 0xfe, 0x43, 0x00, 0x00, 0xff, 0x43, 0x00, 0x40, 0xff, 0x43, 0x00, 0x80, 0xff, 0x43, + 0x00, 0xc0, 0xff, 0x43, 0x00, 0x00, 0x00, 0x44, 0x00, 0x20, 0x00, 0x44, 0x00, 0x40, 0x00, 0x44, 0x00, + 0x60, 0x00, 0x44, 0x00, 0x80, 0x00, 0x44, 0x00, 0xa0, 0x00, 0x44, 0x00, 0xc0, 0x00, 0x44, 0x00, 0xe0, + 0x00, 0x44, 0x00, 0x00, 0x01, 0x44, 0x00, 0x20, 0x01, 0x44, 0x00, 0x40, 0x01, 0x44, 0x00, 0x60, 0x01, + 0x44, 0x00, 0x80, 0x01, 0x44, 0x00, 0xa0, 0x01, 0x44, 0x00, 0xc0, 0x01, 0x44, 0x00, 0xe0, 0x01, 0x44, + 0x00, 0x00, 0x02, 0x44, 0x00, 0x20, 0x02, 0x44, 0x00, 0x40, 0x02, 0x44, 0x00, 0x60, 0x02, 0x44, 0x00, + 0x80, 0x02, 0x44, 0x00, 0xa0, 0x02, 0x44, 0x00, 0xc0, 0x02, 0x44, 0x00, 0xe0, 0x02, 0x44, 0x00, 0x00, + 0x03, 0x44, 0x00, 0x20, 0x03, 0x44, 0x00, 0x40, 0x03, 0x44, 0x00, 0x60, 0x03, 0x44, 0x00, 0x80, 0x03, + 0x44, 0x00, 0xa0, 0x03, 0x44, 0x00, 0xc0, 0x03, 0x44, 0x00, 0xe0, 0x03, 0x44, 0x00, 0x00, 0x04, 0x44, + 0x00, 0x20, 0x04, 0x44, 0x00, 0x40, 0x04, 0x44, 0x00, 0x60, 0x04, 0x44, 0x00, 0x80, 0x04, 0x44, 0x00, + 0xa0, 0x04, 0x44, 0x00, 0xc0, 0x04, 0x44, 0x00, 0xe0, 0x04, 0x44, 0x00, 0x00, 0x05, 0x44, 0x00, 0x20, + 0x05, 0x44, 0x00, 0x40, 0x05, 0x44, 0x00, 0x60, 0x05, 0x44, 0x00, 0x80, 0x05, 0x44, 0x00, 0xa0, 0x05, + 0x44, 0x00, 0xc0, 0x05, 0x44, 0x00, 0xe0, 0x05, 0x44, 0x00, 0x00, 0x06, 0x44, 0x00, 0x20, 0x06, 0x44, + 0x00, 0x40, 0x06, 0x44, 0x00, 0x60, 0x06, 0x44, 0x00, 0x80, 0x06, 0x44, 0x00, 0xa0, 0x06, 0x44, 0x00, + 0xc0, 0x06, 0x44, 0x00, 0xe0, 0x06, 0x44, 0x00, 0x00, 0x07, 0x44, 0x00, 0x20, 0x07, 0x44, 0x00, 0x40, + 0x07, 0x44, 0x00, 0x60, 0x07, 0x44, 0x00, 0x80, 0x07, 0x44, 0x00, 0xa0, 0x07, 0x44, 0x00, 0xc0, 0x07, + 0x44, 0x00, 0xe0, 0x07, 0x44, 0x00, 0x00, 0x08, 0x44, 0x00, 0x20, 0x08, 0x44, 0x00, 0x40, 0x08, 0x44, + 0x00, 0x60, 0x08, 0x44, 0x00, 0x80, 0x08, 0x44, 0x00, 0xa0, 0x08, 0x44, 0x00, 0xc0, 0x08, 0x44, 0x00, + 0xe0, 0x08, 0x44, 0x00, 0x00, 0x09, 0x44, 0x00, 0x20, 0x09, 0x44, 0x00, 0x40, 0x09, 0x44, 0x00, 0x60, + 0x09, 0x44, 0x00, 0x80, 0x09, 0x44, 0x00, 0xa0, 0x09, 0x44, 0x00, 0xc0, 0x09, 0x44, 0x00, 0xe0, 0x09, + 0x44, 0x00, 0x00, 0x0a, 0x44, 0x00, 0x20, 0x0a, 0x44, 0x00, 0x40, 0x0a, 0x44, 0x00, 0x60, 0x0a, 0x44, + 0x00, 0x80, 0x0a, 0x44, 0x00, 0xa0, 0x0a, 0x44, 0x00, 0xc0, 0x0a, 0x44, 0x00, 0xe0, 0x0a, 0x44, 0x00, + 0x00, 0x0b, 0x44, 0x00, 0x20, 0x0b, 0x44, 0x00, 0x40, 0x0b, 0x44, 0x00, 0x60, 0x0b, 0x44, 0x00, 0x80, + 0x0b, 0x44, 0x00, 0xa0, 0x0b, 0x44, 0x00, 0xc0, 0x0b, 0x44, 0x00, 0xe0, 0x0b, 0x44, 0x00, 0x00, 0x0c, + 0x44, 0x00, 0x20, 0x0c, 0x44, 0x00, 0x40, 0x0c, 0x44, 0x00, 0x60, 0x0c, 0x44, 0x00, 0x80, 0x0c, 0x44, + 0x00, 0xa0, 0x0c, 0x44, 0x00, 0xc0, 0x0c, 0x44, 0x00, 0xe0, 0x0c, 0x44, 0x00, 0x00, 0x0d, 0x44, 0x00, + 0x20, 0x0d, 0x44, 0x00, 0x40, 0x0d, 0x44, 0x00, 0x60, 0x0d, 0x44, 0x00, 0x80, 0x0d, 0x44, 0x00, 0xa0, + 0x0d, 0x44, 0x00, 0xc0, 0x0d, 0x44, 0x00, 0xe0, 0x0d, 0x44, 0x00, 0x00, 0x0e, 0x44, 0x00, 0x20, 0x0e, + 0x44, 0x00, 0x40, 0x0e, 0x44, 0x00, 0x60, 0x0e, 0x44, 0x00, 0x80, 0x0e, 0x44, 0x00, 0xa0, 0x0e, 0x44, + 0x00, 0xc0, 0x0e, 0x44, 0x00, 0xe0, 0x0e, 0x44, 0x00, 0x00, 0x0f, 0x44, 0x00, 0x20, 0x0f, 0x44, 0x00, + 0x40, 0x0f, 0x44, 0x00, 0x60, 0x0f, 0x44, 0x00, 0x80, 0x0f, 0x44, 0x00, 0xa0, 0x0f, 0x44, 0x00, 0xc0, + 0x0f, 0x44, 0x00, 0xe0, 0x0f, 0x44, 0x00, 0x00, 0x10, 0x44, 0x00, 0x20, 0x10, 0x44, 0x00, 0x40, 0x10, + 0x44, 0x00, 0x60, 0x10, 0x44, 0x00, 0x80, 0x10, 0x44, 0x00, 0xa0, 0x10, 0x44, 0x00, 0xc0, 0x10, 0x44, + 0x00, 0xe0, 0x10, 0x44, 0x00, 0x00, 0x11, 0x44, 0x00, 0x20, 0x11, 0x44, 0x00, 0x40, 0x11, 0x44, 0x00, + 0x60, 0x11, 0x44, 0x00, 0x80, 0x11, 0x44, 0x00, 0xa0, 0x11, 0x44, 0x00, 0xc0, 0x11, 0x44, 0x00, 0xe0, + 0x11, 0x44, 0x00, 0x00, 0x12, 0x44, 0x00, 0x20, 0x12, 0x44, 0x00, 0x40, 0x12, 0x44, 0x00, 0x60, 0x12, + 0x44, 0x00, 0x80, 0x12, 0x44, 0x00, 0xa0, 0x12, 0x44, 0x00, 0xc0, 0x12, 0x44, 0x00, 0xe0, 0x12, 0x44, + 0x00, 0x00, 0x13, 0x44, 0x00, 0x20, 0x13, 0x44, 0x00, 0x40, 0x13, 0x44, 0x00, 0x60, 0x13, 0x44, 0x00, + 0x80, 0x13, 0x44, 0x00, 0xa0, 0x13, 0x44, 0x00, 0xc0, 0x13, 0x44, 0x00, 0xe0, 0x13, 0x44, 0x00, 0x00, + 0x14, 0x44, 0x00, 0x20, 0x14, 0x44, 0x00, 0x40, 0x14, 0x44, 0x00, 0x60, 0x14, 0x44, 0x00, 0x80, 0x14, + 0x44, 0x00, 0xa0, 0x14, 0x44, 0x00, 0xc0, 0x14, 0x44, 0x00, 0xe0, 0x14, 0x44, 0x00, 0x00, 0x15, 0x44, + 0x00, 0x20, 0x15, 0x44, 0x00, 0x40, 0x15, 0x44, 0x00, 0x60, 0x15, 0x44, 0x00, 0x80, 0x15, 0x44, 0x00, + 0xa0, 0x15, 0x44, 0x00, 0xc0, 0x15, 0x44, 0x00, 0xe0, 0x15, 0x44, 0x00, 0x00, 0x16, 0x44, 0x00, 0x20, + 0x16, 0x44, 0x00, 0x40, 0x16, 0x44, 0x00, 0x60, 0x16, 0x44, 0x00, 0x80, 0x16, 0x44, 0x00, 0xa0, 0x16, + 0x44, 0x00, 0xc0, 0x16, 0x44, 0x00, 0xe0, 0x16, 0x44, 0x00, 0x00, 0x17, 0x44, 0x00, 0x20, 0x17, 0x44, + 0x00, 0x40, 0x17, 0x44, 0x00, 0x60, 0x17, 0x44, 0x00, 0x80, 0x17, 0x44, 0x00, 0xa0, 0x17, 0x44, 0x00, + 0xc0, 0x17, 0x44, 0x00, 0xe0, 0x17, 0x44, 0x00, 0x00, 0x18, 0x44, 0x00, 0x20, 0x18, 0x44, 0x00, 0x40, + 0x18, 0x44, 0x00, 0x60, 0x18, 0x44, 0x00, 0x80, 0x18, 0x44, 0x00, 0xa0, 0x18, 0x44, 0x00, 0xc0, 0x18, + 0x44, 0x00, 0xe0, 0x18, 0x44, 0x00, 0x00, 0x19, 0x44, 0x00, 0x20, 0x19, 0x44, 0x00, 0x40, 0x19, 0x44, + 0x00, 0x60, 0x19, 0x44, 0x00, 0x80, 0x19, 0x44, 0x00, 0xa0, 0x19, 0x44, 0x00, 0xc0, 0x19, 0x44, 0x00, + 0xe0, 0x19, 0x44, 0x00, 0x00, 0x1a, 0x44, 0x00, 0x20, 0x1a, 0x44, 0x00, 0x40, 0x1a, 0x44, 0x00, 0x60, + 0x1a, 0x44, 0x00, 0x80, 0x1a, 0x44, 0x00, 0xa0, 0x1a, 0x44, 0x00, 0xc0, 0x1a, 0x44, 0x00, 0xe0, 0x1a, + 0x44, 0x00, 0x00, 0x1b, 0x44, 0x00, 0x20, 0x1b, 0x44, 0x00, 0x40, 0x1b, 0x44, 0x00, 0x60, 0x1b, 0x44, + 0x00, 0x80, 0x1b, 0x44, 0x00, 0xa0, 0x1b, 0x44, 0x00, 0xc0, 0x1b, 0x44, 0x00, 0xe0, 0x1b, 0x44, 0x00, + 0x00, 0x1c, 0x44, 0x00, 0x20, 0x1c, 0x44, 0x00, 0x40, 0x1c, 0x44, 0x00, 0x60, 0x1c, 0x44, 0x00, 0x80, + 0x1c, 0x44, 0x00, 0xa0, 0x1c, 0x44, 0x00, 0xc0, 0x1c, 0x44, 0x00, 0xe0, 0x1c, 0x44, 0x00, 0x00, 0x1d, + 0x44, 0x00, 0x20, 0x1d, 0x44, 0x00, 0x40, 0x1d, 0x44, 0x00, 0x60, 0x1d, 0x44, 0x00, 0x80, 0x1d, 0x44, + 0x00, 0xa0, 0x1d, 0x44, 0x00, 0xc0, 0x1d, 0x44, 0x00, 0xe0, 0x1d, 0x44, 0x00, 0x00, 0x1e, 0x44, 0x00, + 0x20, 0x1e, 0x44, 0x00, 0x40, 0x1e, 0x44, 0x00, 0x60, 0x1e, 0x44, 0x00, 0x80, 0x1e, 0x44, 0x00, 0xa0, + 0x1e, 0x44, 0x00, 0xc0, 0x1e, 0x44, 0x00, 0xe0, 0x1e, 0x44, 0x00, 0x00, 0x1f, 0x44, 0x00, 0x20, 0x1f, + 0x44, 0x00, 0x40, 0x1f, 0x44, 0x00, 0x60, 0x1f, 0x44, 0x00, 0x80, 0x1f, 0x44, 0x00, 0xa0, 0x1f, 0x44, + 0x00, 0xc0, 0x1f, 0x44, 0x00, 0xe0, 0x1f, 0x44, 0x00, 0x00, 0x20, 0x44, 0x00, 0x20, 0x20, 0x44, 0x00, + 0x40, 0x20, 0x44, 0x00, 0x60, 0x20, 0x44, 0x00, 0x80, 0x20, 0x44, 0x00, 0xa0, 0x20, 0x44, 0x00, 0xc0, + 0x20, 0x44, 0x00, 0xe0, 0x20, 0x44, 0x00, 0x00, 0x21, 0x44, 0x00, 0x20, 0x21, 0x44, 0x00, 0x40, 0x21, + 0x44, 0x00, 0x60, 0x21, 0x44, 0x00, 0x80, 0x21, 0x44, 0x00, 0xa0, 0x21, 0x44, 0x00, 0xc0, 0x21, 0x44, + 0x00, 0xe0, 0x21, 0x44, 0x00, 0x00, 0x22, 0x44, 0x00, 0x20, 0x22, 0x44, 0x00, 0x40, 0x22, 0x44, 0x00, + 0x60, 0x22, 0x44, 0x00, 0x80, 0x22, 0x44, 0x00, 0xa0, 0x22, 0x44, 0x00, 0xc0, 0x22, 0x44, 0x00, 0xe0, + 0x22, 0x44, 0x00, 0x00, 0x23, 0x44, 0x00, 0x20, 0x23, 0x44, 0x00, 0x40, 0x23, 0x44, 0x00, 0x60, 0x23, + 0x44, 0x00, 0x80, 0x23, 0x44, 0x00, 0xa0, 0x23, 0x44, 0x00, 0xc0, 0x23, 0x44, 0x00, 0xe0, 0x23, 0x44, + 0x00, 0x00, 0x24, 0x44, 0x00, 0x20, 0x24, 0x44, 0x00, 0x40, 0x24, 0x44, 0x00, 0x60, 0x24, 0x44, 0x00, + 0x80, 0x24, 0x44, 0x00, 0xa0, 0x24, 0x44, 0x00, 0xc0, 0x24, 0x44, 0x00, 0xe0, 0x24, 0x44, 0x00, 0x00, + 0x25, 0x44, 0x00, 0x20, 0x25, 0x44, 0x00, 0x40, 0x25, 0x44, 0x00, 0x60, 0x25, 0x44, 0x00, 0x80, 0x25, + 0x44, 0x00, 0xa0, 0x25, 0x44, 0x00, 0xc0, 0x25, 0x44, 0x00, 0xe0, 0x25, 0x44, 0x00, 0x00, 0x26, 0x44, + 0x00, 0x20, 0x26, 0x44, 0x00, 0x40, 0x26, 0x44, 0x00, 0x60, 0x26, 0x44, 0x00, 0x80, 0x26, 0x44, 0x00, + 0xa0, 0x26, 0x44, 0x00, 0xc0, 0x26, 0x44, 0x00, 0xe0, 0x26, 0x44, 0x00, 0x00, 0x27, 0x44, 0x00, 0x20, + 0x27, 0x44, 0x00, 0x40, 0x27, 0x44, 0x00, 0x60, 0x27, 0x44, 0x00, 0x80, 0x27, 0x44, 0x00, 0xa0, 0x27, + 0x44, 0x00, 0xc0, 0x27, 0x44, 0x00, 0xe0, 0x27, 0x44, 0x00, 0x00, 0x28, 0x44, 0x00, 0x20, 0x28, 0x44, + 0x00, 0x40, 0x28, 0x44, 0x00, 0x60, 0x28, 0x44, 0x00, 0x80, 0x28, 0x44, 0x00, 0xa0, 0x28, 0x44, 0x00, + 0xc0, 0x28, 0x44, 0x00, 0xe0, 0x28, 0x44, 0x00, 0x00, 0x29, 0x44, 0x00, 0x20, 0x29, 0x44, 0x00, 0x40, + 0x29, 0x44, 0x00, 0x60, 0x29, 0x44, 0x00, 0x80, 0x29, 0x44, 0x00, 0xa0, 0x29, 0x44, 0x00, 0xc0, 0x29, + 0x44, 0x00, 0xe0, 0x29, 0x44, 0x00, 0x00, 0x2a, 0x44, 0x00, 0x20, 0x2a, 0x44, 0x00, 0x40, 0x2a, 0x44, + 0x00, 0x60, 0x2a, 0x44, 0x00, 0x80, 0x2a, 0x44, 0x00, 0xa0, 0x2a, 0x44, 0x00, 0xc0, 0x2a, 0x44, 0x00, + 0xe0, 0x2a, 0x44, 0x00, 0x00, 0x2b, 0x44, 0x00, 0x20, 0x2b, 0x44, 0x00, 0x40, 0x2b, 0x44, 0x00, 0x60, + 0x2b, 0x44, 0x00, 0x80, 0x2b, 0x44, 0x00, 0xa0, 0x2b, 0x44, 0x00, 0xc0, 0x2b, 0x44, 0x00, 0xe0, 0x2b, + 0x44, 0x00, 0x00, 0x2c, 0x44, 0x00, 0x20, 0x2c, 0x44, 0x00, 0x40, 0x2c, 0x44, 0x00, 0x60, 0x2c, 0x44, + 0x00, 0x80, 0x2c, 0x44, 0x00, 0xa0, 0x2c, 0x44, 0x00, 0xc0, 0x2c, 0x44, 0x00, 0xe0, 0x2c, 0x44, 0x00, + 0x00, 0x2d, 0x44, 0x00, 0x20, 0x2d, 0x44, 0x00, 0x40, 0x2d, 0x44, 0x00, 0x60, 0x2d, 0x44, 0x00, 0x80, + 0x2d, 0x44, 0x00, 0xa0, 0x2d, 0x44, 0x00, 0xc0, 0x2d, 0x44, 0x00, 0xe0, 0x2d, 0x44, 0x00, 0x00, 0x2e, + 0x44, 0x00, 0x20, 0x2e, 0x44, 0x00, 0x40, 0x2e, 0x44, 0x00, 0x60, 0x2e, 0x44, 0x00, 0x80, 0x2e, 0x44, + 0x00, 0xa0, 0x2e, 0x44, 0x00, 0xc0, 0x2e, 0x44, 0x00, 0xe0, 0x2e, 0x44, 0x00, 0x00, 0x2f, 0x44, 0x00, + 0x20, 0x2f, 0x44, 0x00, 0x40, 0x2f, 0x44, 0x00, 0x60, 0x2f, 0x44, 0x00, 0x80, 0x2f, 0x44, 0x00, 0xa0, + 0x2f, 0x44, 0x00, 0xc0, 0x2f, 0x44, 0x00, 0xe0, 0x2f, 0x44, 0x00, 0x00, 0x30, 0x44, 0x00, 0x20, 0x30, + 0x44, 0x00, 0x40, 0x30, 0x44, 0x00, 0x60, 0x30, 0x44, 0x00, 0x80, 0x30, 0x44, 0x00, 0xa0, 0x30, 0x44, + 0x00, 0xc0, 0x30, 0x44, 0x00, 0xe0, 0x30, 0x44, 0x00, 0x00, 0x31, 0x44, 0x00, 0x20, 0x31, 0x44, 0x00, + 0x40, 0x31, 0x44, 0x00, 0x60, 0x31, 0x44, 0x00, 0x80, 0x31, 0x44, 0x00, 0xa0, 0x31, 0x44, 0x00, 0xc0, + 0x31, 0x44, 0x00, 0xe0, 0x31, 0x44, 0x00, 0x00, 0x32, 0x44, 0x00, 0x20, 0x32, 0x44, 0x00, 0x40, 0x32, + 0x44, 0x00, 0x60, 0x32, 0x44, 0x00, 0x80, 0x32, 0x44, 0x00, 0xa0, 0x32, 0x44, 0x00, 0xc0, 0x32, 0x44, + 0x00, 0xe0, 0x32, 0x44, 0x00, 0x00, 0x33, 0x44, 0x00, 0x20, 0x33, 0x44, 0x00, 0x40, 0x33, 0x44, 0x00, + 0x60, 0x33, 0x44, 0x00, 0x80, 0x33, 0x44, 0x00, 0xa0, 0x33, 0x44, 0x00, 0xc0, 0x33, 0x44, 0x00, 0xe0, + 0x33, 0x44, 0x00, 0x00, 0x34, 0x44, 0x00, 0x20, 0x34, 0x44, 0x00, 0x40, 0x34, 0x44, 0x00, 0x60, 0x34, + 0x44, 0x00, 0x80, 0x34, 0x44, 0x00, 0xa0, 0x34, 0x44, 0x00, 0xc0, 0x34, 0x44, 0x00, 0xe0, 0x34, 0x44, + 0x00, 0x00, 0x35, 0x44, 0x00, 0x20, 0x35, 0x44, 0x00, 0x40, 0x35, 0x44, 0x00, 0x60, 0x35, 0x44, 0x00, + 0x80, 0x35, 0x44, 0x00, 0xa0, 0x35, 0x44, 0x00, 0xc0, 0x35, 0x44, 0x00, 0xe0, 0x35, 0x44, 0x00, 0x00, + 0x36, 0x44, 0x00, 0x20, 0x36, 0x44, 0x00, 0x40, 0x36, 0x44, 0x00, 0x60, 0x36, 0x44, 0x00, 0x80, 0x36, + 0x44, 0x00, 0xa0, 0x36, 0x44, 0x00, 0xc0, 0x36, 0x44, 0x00, 0xe0, 0x36, 0x44, 0x00, 0x00, 0x37, 0x44, + 0x00, 0x20, 0x37, 0x44, 0x00, 0x40, 0x37, 0x44, 0x00, 0x60, 0x37, 0x44, 0x00, 0x80, 0x37, 0x44, 0x00, + 0xa0, 0x37, 0x44, 0x00, 0xc0, 0x37, 0x44, 0x00, 0xe0, 0x37, 0x44, 0x00, 0x00, 0x38, 0x44, 0x00, 0x20, + 0x38, 0x44, 0x00, 0x40, 0x38, 0x44, 0x00, 0x60, 0x38, 0x44, 0x00, 0x80, 0x38, 0x44, 0x00, 0xa0, 0x38, + 0x44, 0x00, 0xc0, 0x38, 0x44, 0x00, 0xe0, 0x38, 0x44, 0x00, 0x00, 0x39, 0x44, 0x00, 0x20, 0x39, 0x44, + 0x00, 0x40, 0x39, 0x44, 0x00, 0x60, 0x39, 0x44, 0x00, 0x80, 0x39, 0x44, 0x00, 0xa0, 0x39, 0x44, 0x00, + 0xc0, 0x39, 0x44, 0x00, 0xe0, 0x39, 0x44, 0x00, 0x00, 0x3a, 0x44, 0x00, 0x20, 0x3a, 0x44, 0x00, 0x40, + 0x3a, 0x44, 0x00, 0x60, 0x3a, 0x44, 0x00, 0x80, 0x3a, 0x44, 0x00, 0xa0, 0x3a, 0x44, 0x00, 0xc0, 0x3a, + 0x44, 0x00, 0xe0, 0x3a, 0x44, 0x00, 0x00, 0x3b, 0x44, 0x00, 0x20, 0x3b, 0x44, 0x00, 0x40, 0x3b, 0x44, + 0x00, 0x60, 0x3b, 0x44, 0x00, 0x80, 0x3b, 0x44, 0x00, 0xa0, 0x3b, 0x44, 0x00, 0xc0, 0x3b, 0x44, 0x00, + 0xe0, 0x3b, 0x44, 0x00, 0x00, 0x3c, 0x44, 0x00, 0x20, 0x3c, 0x44, 0x00, 0x40, 0x3c, 0x44, 0x00, 0x60, + 0x3c, 0x44, 0x00, 0x80, 0x3c, 0x44, 0x00, 0xa0, 0x3c, 0x44, 0x00, 0xc0, 0x3c, 0x44, 0x00, 0xe0, 0x3c, + 0x44, 0x00, 0x00, 0x3d, 0x44, 0x00, 0x20, 0x3d, 0x44, 0x00, 0x40, 0x3d, 0x44, 0x00, 0x60, 0x3d, 0x44, + 0x00, 0x80, 0x3d, 0x44, 0x00, 0xa0, 0x3d, 0x44, 0x00, 0xc0, 0x3d, 0x44, 0x00, 0xe0, 0x3d, 0x44, 0x00, + 0x00, 0x3e, 0x44, 0x00, 0x20, 0x3e, 0x44, 0x00, 0x40, 0x3e, 0x44, 0x00, 0x60, 0x3e, 0x44, 0x00, 0x80, + 0x3e, 0x44, 0x00, 0xa0, 0x3e, 0x44, 0x00, 0xc0, 0x3e, 0x44, 0x00, 0xe0, 0x3e, 0x44, 0x00, 0x00, 0x3f, + 0x44, 0x00, 0x20, 0x3f, 0x44, 0x00, 0x40, 0x3f, 0x44, 0x00, 0x60, 0x3f, 0x44, 0x00, 0x80, 0x3f, 0x44, + 0x00, 0xa0, 0x3f, 0x44, 0x00, 0xc0, 0x3f, 0x44, 0x00, 0xe0, 0x3f, 0x44, 0x00, 0x00, 0x40, 0x44, 0x00, + 0x20, 0x40, 0x44, 0x00, 0x40, 0x40, 0x44, 0x00, 0x60, 0x40, 0x44, 0x00, 0x80, 0x40, 0x44, 0x00, 0xa0, + 0x40, 0x44, 0x00, 0xc0, 0x40, 0x44, 0x00, 0xe0, 0x40, 0x44, 0x00, 0x00, 0x41, 0x44, 0x00, 0x20, 0x41, + 0x44, 0x00, 0x40, 0x41, 0x44, 0x00, 0x60, 0x41, 0x44, 0x00, 0x80, 0x41, 0x44, 0x00, 0xa0, 0x41, 0x44, + 0x00, 0xc0, 0x41, 0x44, 0x00, 0xe0, 0x41, 0x44, 0x00, 0x00, 0x42, 0x44, 0x00, 0x20, 0x42, 0x44, 0x00, + 0x40, 0x42, 0x44, 0x00, 0x60, 0x42, 0x44, 0x00, 0x80, 0x42, 0x44, 0x00, 0xa0, 0x42, 0x44, 0x00, 0xc0, + 0x42, 0x44, 0x00, 0xe0, 0x42, 0x44, 0x00, 0x00, 0x43, 0x44, 0x00, 0x20, 0x43, 0x44, 0x00, 0x40, 0x43, + 0x44, 0x00, 0x60, 0x43, 0x44, 0x00, 0x80, 0x43, 0x44, 0x00, 0xa0, 0x43, 0x44, 0x00, 0xc0, 0x43, 0x44, + 0x00, 0xe0, 0x43, 0x44, 0x00, 0x00, 0x44, 0x44, 0x00, 0x20, 0x44, 0x44, 0x00, 0x40, 0x44, 0x44, 0x00, + 0x60, 0x44, 0x44, 0x00, 0x80, 0x44, 0x44, 0x00, 0xa0, 0x44, 0x44, 0x00, 0xc0, 0x44, 0x44, 0x00, 0xe0, + 0x44, 0x44, 0x00, 0x00, 0x45, 0x44, 0x00, 0x20, 0x45, 0x44, 0x00, 0x40, 0x45, 0x44, 0x00, 0x60, 0x45, + 0x44, 0x00, 0x80, 0x45, 0x44, 0x00, 0xa0, 0x45, 0x44, 0x00, 0xc0, 0x45, 0x44, 0x00, 0xe0, 0x45, 0x44, + 0x00, 0x00, 0x46, 0x44, 0x00, 0x20, 0x46, 0x44, 0x00, 0x40, 0x46, 0x44, 0x00, 0x60, 0x46, 0x44, 0x00, + 0x80, 0x46, 0x44, 0x00, 0xa0, 0x46, 0x44, 0x00, 0xc0, 0x46, 0x44, 0x00, 0xe0, 0x46, 0x44, 0x00, 0x00, + 0x47, 0x44, 0x00, 0x20, 0x47, 0x44, 0x00, 0x40, 0x47, 0x44, 0x00, 0x60, 0x47, 0x44, 0x00, 0x80, 0x47, + 0x44, 0x00, 0xa0, 0x47, 0x44, 0x00, 0xc0, 0x47, 0x44, 0x00, 0xe0, 0x47, 0x44, 0x00, 0x00, 0x48, 0x44, + 0x00, 0x20, 0x48, 0x44, 0x00, 0x40, 0x48, 0x44, 0x00, 0x60, 0x48, 0x44, 0x00, 0x80, 0x48, 0x44, 0x00, + 0xa0, 0x48, 0x44, 0x00, 0xc0, 0x48, 0x44, 0x00, 0xe0, 0x48, 0x44, 0x00, 0x00, 0x49, 0x44, 0x00, 0x20, + 0x49, 0x44, 0x00, 0x40, 0x49, 0x44, 0x00, 0x60, 0x49, 0x44, 0x00, 0x80, 0x49, 0x44, 0x00, 0xa0, 0x49, + 0x44, 0x00, 0xc0, 0x49, 0x44, 0x00, 0xe0, 0x49, 0x44, 0x00, 0x00, 0x4a, 0x44, 0x00, 0x20, 0x4a, 0x44, + 0x00, 0x40, 0x4a, 0x44, 0x00, 0x60, 0x4a, 0x44, 0x00, 0x80, 0x4a, 0x44, 0x00, 0xa0, 0x4a, 0x44, 0x00, + 0xc0, 0x4a, 0x44, 0x00, 0xe0, 0x4a, 0x44, 0x00, 0x00, 0x4b, 0x44, 0x00, 0x20, 0x4b, 0x44, 0x00, 0x40, + 0x4b, 0x44, 0x00, 0x60, 0x4b, 0x44, 0x00, 0x80, 0x4b, 0x44, 0x00, 0xa0, 0x4b, 0x44, 0x00, 0xc0, 0x4b, + 0x44, 0x00, 0xe0, 0x4b, 0x44, 0x00, 0x00, 0x4c, 0x44, 0x00, 0x20, 0x4c, 0x44, 0x00, 0x40, 0x4c, 0x44, + 0x00, 0x60, 0x4c, 0x44, 0x00, 0x80, 0x4c, 0x44, 0x00, 0xa0, 0x4c, 0x44, 0x00, 0xc0, 0x4c, 0x44, 0x00, + 0xe0, 0x4c, 0x44, 0x00, 0x00, 0x4d, 0x44, 0x00, 0x20, 0x4d, 0x44, 0x00, 0x40, 0x4d, 0x44, 0x00, 0x60, + 0x4d, 0x44, 0x00, 0x80, 0x4d, 0x44, 0x00, 0xa0, 0x4d, 0x44, 0x00, 0xc0, 0x4d, 0x44, 0x00, 0xe0, 0x4d, + 0x44, 0x00, 0x00, 0x4e, 0x44, 0x00, 0x20, 0x4e, 0x44, 0x00, 0x40, 0x4e, 0x44, 0x00, 0x60, 0x4e, 0x44, + 0x00, 0x80, 0x4e, 0x44, 0x00, 0xa0, 0x4e, 0x44, 0x00, 0xc0, 0x4e, 0x44, 0x00, 0xe0, 0x4e, 0x44, 0x00, + 0x00, 0x4f, 0x44, 0x00, 0x20, 0x4f, 0x44, 0x00, 0x40, 0x4f, 0x44, 0x00, 0x60, 0x4f, 0x44, 0x00, 0x80, + 0x4f, 0x44, 0x00, 0xa0, 0x4f, 0x44, 0x00, 0xc0, 0x4f, 0x44, 0x00, 0xe0, 0x4f, 0x44, 0x00, 0x00, 0x50, + 0x44, 0x00, 0x20, 0x50, 0x44, 0x00, 0x40, 0x50, 0x44, 0x00, 0x60, 0x50, 0x44, 0x00, 0x80, 0x50, 0x44, + 0x00, 0xa0, 0x50, 0x44, 0x00, 0xc0, 0x50, 0x44, 0x00, 0xe0, 0x50, 0x44, 0x00, 0x00, 0x51, 0x44, 0x00, + 0x20, 0x51, 0x44, 0x00, 0x40, 0x51, 0x44, 0x00, 0x60, 0x51, 0x44, 0x00, 0x80, 0x51, 0x44, 0x00, 0xa0, + 0x51, 0x44, 0x00, 0xc0, 0x51, 0x44, 0x00, 0xe0, 0x51, 0x44, 0x00, 0x00, 0x52, 0x44, 0x00, 0x20, 0x52, + 0x44, 0x00, 0x40, 0x52, 0x44, 0x00, 0x60, 0x52, 0x44, 0x00, 0x80, 0x52, 0x44, 0x00, 0xa0, 0x52, 0x44, + 0x00, 0xc0, 0x52, 0x44, 0x00, 0xe0, 0x52, 0x44, 0x00, 0x00, 0x53, 0x44, 0x00, 0x20, 0x53, 0x44, 0x00, + 0x40, 0x53, 0x44, 0x00, 0x60, 0x53, 0x44, 0x00, 0x80, 0x53, 0x44, 0x00, 0xa0, 0x53, 0x44, 0x00, 0xc0, + 0x53, 0x44, 0x00, 0xe0, 0x53, 0x44, 0x00, 0x00, 0x54, 0x44, 0x00, 0x20, 0x54, 0x44, 0x00, 0x40, 0x54, + 0x44, 0x00, 0x60, 0x54, 0x44, 0x00, 0x80, 0x54, 0x44, 0x00, 0xa0, 0x54, 0x44, 0x00, 0xc0, 0x54, 0x44, + 0x00, 0xe0, 0x54, 0x44, 0x00, 0x00, 0x55, 0x44, 0x00, 0x20, 0x55, 0x44, 0x00, 0x40, 0x55, 0x44, 0x00, + 0x60, 0x55, 0x44, 0x00, 0x80, 0x55, 0x44, 0x00, 0xa0, 0x55, 0x44, 0x00, 0xc0, 0x55, 0x44, 0x00, 0xe0, + 0x55, 0x44, 0x00, 0x00, 0x56, 0x44, 0x00, 0x20, 0x56, 0x44, 0x00, 0x40, 0x56, 0x44, 0x00, 0x60, 0x56, + 0x44, 0x00, 0x80, 0x56, 0x44, 0x00, 0xa0, 0x56, 0x44, 0x00, 0xc0, 0x56, 0x44, 0x00, 0xe0, 0x56, 0x44, + 0x00, 0x00, 0x57, 0x44, 0x00, 0x20, 0x57, 0x44, 0x00, 0x40, 0x57, 0x44, 0x00, 0x60, 0x57, 0x44, 0x00, + 0x80, 0x57, 0x44, 0x00, 0xa0, 0x57, 0x44, 0x00, 0xc0, 0x57, 0x44, 0x00, 0xe0, 0x57, 0x44, 0x00, 0x00, + 0x58, 0x44, 0x00, 0x20, 0x58, 0x44, 0x00, 0x40, 0x58, 0x44, 0x00, 0x60, 0x58, 0x44, 0x00, 0x80, 0x58, + 0x44, 0x00, 0xa0, 0x58, 0x44, 0x00, 0xc0, 0x58, 0x44, 0x00, 0xe0, 0x58, 0x44, 0x00, 0x00, 0x59, 0x44, + 0x00, 0x20, 0x59, 0x44, 0x00, 0x40, 0x59, 0x44, 0x00, 0x60, 0x59, 0x44, 0x00, 0x80, 0x59, 0x44, 0x00, + 0xa0, 0x59, 0x44, 0x00, 0xc0, 0x59, 0x44, 0x00, 0xe0, 0x59, 0x44, 0x00, 0x00, 0x5a, 0x44, 0x00, 0x20, + 0x5a, 0x44, 0x00, 0x40, 0x5a, 0x44, 0x00, 0x60, 0x5a, 0x44, 0x00, 0x80, 0x5a, 0x44, 0x00, 0xa0, 0x5a, + 0x44, 0x00, 0xc0, 0x5a, 0x44, 0x00, 0xe0, 0x5a, 0x44, 0x00, 0x00, 0x5b, 0x44, 0x00, 0x20, 0x5b, 0x44, + 0x00, 0x40, 0x5b, 0x44, 0x00, 0x60, 0x5b, 0x44, 0x00, 0x80, 0x5b, 0x44, 0x00, 0xa0, 0x5b, 0x44, 0x00, + 0xc0, 0x5b, 0x44, 0x00, 0xe0, 0x5b, 0x44, 0x00, 0x00, 0x5c, 0x44, 0x00, 0x20, 0x5c, 0x44, 0x00, 0x40, + 0x5c, 0x44, 0x00, 0x60, 0x5c, 0x44, 0x00, 0x80, 0x5c, 0x44, 0x00, 0xa0, 0x5c, 0x44, 0x00, 0xc0, 0x5c, + 0x44, 0x00, 0xe0, 0x5c, 0x44, 0x00, 0x00, 0x5d, 0x44, 0x00, 0x20, 0x5d, 0x44, 0x00, 0x40, 0x5d, 0x44, + 0x00, 0x60, 0x5d, 0x44, 0x00, 0x80, 0x5d, 0x44, 0x00, 0xa0, 0x5d, 0x44, 0x00, 0xc0, 0x5d, 0x44, 0x00, + 0xe0, 0x5d, 0x44, 0x00, 0x00, 0x5e, 0x44, 0x00, 0x20, 0x5e, 0x44, 0x00, 0x40, 0x5e, 0x44, 0x00, 0x60, + 0x5e, 0x44, 0x00, 0x80, 0x5e, 0x44, 0x00, 0xa0, 0x5e, 0x44, 0x00, 0xc0, 0x5e, 0x44, 0x00, 0xe0, 0x5e, + 0x44, 0x00, 0x00, 0x5f, 0x44, 0x00, 0x20, 0x5f, 0x44, 0x00, 0x40, 0x5f, 0x44, 0x00, 0x60, 0x5f, 0x44, + 0x00, 0x80, 0x5f, 0x44, 0x00, 0xa0, 0x5f, 0x44, 0x00, 0xc0, 0x5f, 0x44, 0x00, 0xe0, 0x5f, 0x44, 0x00, + 0x00, 0x60, 0x44, 0x00, 0x20, 0x60, 0x44, 0x00, 0x40, 0x60, 0x44, 0x00, 0x60, 0x60, 0x44, 0x00, 0x80, + 0x60, 0x44, 0x00, 0xa0, 0x60, 0x44, 0x00, 0xc0, 0x60, 0x44, 0x00, 0xe0, 0x60, 0x44, 0x00, 0x00, 0x61, + 0x44, 0x00, 0x20, 0x61, 0x44, 0x00, 0x40, 0x61, 0x44, 0x00, 0x60, 0x61, 0x44, 0x00, 0x80, 0x61, 0x44, + 0x00, 0xa0, 0x61, 0x44, 0x00, 0xc0, 0x61, 0x44, 0x00, 0xe0, 0x61, 0x44, 0x00, 0x00, 0x62, 0x44, 0x00, + 0x20, 0x62, 0x44, 0x00, 0x40, 0x62, 0x44, 0x00, 0x60, 0x62, 0x44, 0x00, 0x80, 0x62, 0x44, 0x00, 0xa0, + 0x62, 0x44, 0x00, 0xc0, 0x62, 0x44, 0x00, 0xe0, 0x62, 0x44, 0x00, 0x00, 0x63, 0x44, 0x00, 0x20, 0x63, + 0x44, 0x00, 0x40, 0x63, 0x44, 0x00, 0x60, 0x63, 0x44, 0x00, 0x80, 0x63, 0x44, 0x00, 0xa0, 0x63, 0x44, + 0x00, 0xc0, 0x63, 0x44, 0x00, 0xe0, 0x63, 0x44, 0x00, 0x00, 0x64, 0x44, 0x00, 0x20, 0x64, 0x44, 0x00, + 0x40, 0x64, 0x44, 0x00, 0x60, 0x64, 0x44, 0x00, 0x80, 0x64, 0x44, 0x00, 0xa0, 0x64, 0x44, 0x00, 0xc0, + 0x64, 0x44, 0x00, 0xe0, 0x64, 0x44, 0x00, 0x00, 0x65, 0x44, 0x00, 0x20, 0x65, 0x44, 0x00, 0x40, 0x65, + 0x44, 0x00, 0x60, 0x65, 0x44, 0x00, 0x80, 0x65, 0x44, 0x00, 0xa0, 0x65, 0x44, 0x00, 0xc0, 0x65, 0x44, + 0x00, 0xe0, 0x65, 0x44, 0x00, 0x00, 0x66, 0x44, 0x00, 0x20, 0x66, 0x44, 0x00, 0x40, 0x66, 0x44, 0x00, + 0x60, 0x66, 0x44, 0x00, 0x80, 0x66, 0x44, 0x00, 0xa0, 0x66, 0x44, 0x00, 0xc0, 0x66, 0x44, 0x00, 0xe0, + 0x66, 0x44, 0x00, 0x00, 0x67, 0x44, 0x00, 0x20, 0x67, 0x44, 0x00, 0x40, 0x67, 0x44, 0x00, 0x60, 0x67, + 0x44, 0x00, 0x80, 0x67, 0x44, 0x00, 0xa0, 0x67, 0x44, 0x00, 0xc0, 0x67, 0x44, 0x00, 0xe0, 0x67, 0x44, + 0x00, 0x00, 0x68, 0x44, 0x00, 0x20, 0x68, 0x44, 0x00, 0x40, 0x68, 0x44, 0x00, 0x60, 0x68, 0x44, 0x00, + 0x80, 0x68, 0x44, 0x00, 0xa0, 0x68, 0x44, 0x00, 0xc0, 0x68, 0x44, 0x00, 0xe0, 0x68, 0x44, 0x00, 0x00, + 0x69, 0x44, 0x00, 0x20, 0x69, 0x44, 0x00, 0x40, 0x69, 0x44, 0x00, 0x60, 0x69, 0x44, 0x00, 0x80, 0x69, + 0x44, 0x00, 0xa0, 0x69, 0x44, 0x00, 0xc0, 0x69, 0x44, 0x00, 0xe0, 0x69, 0x44, 0x00, 0x00, 0x6a, 0x44, + 0x00, 0x20, 0x6a, 0x44, 0x00, 0x40, 0x6a, 0x44, 0x00, 0x60, 0x6a, 0x44, 0x00, 0x80, 0x6a, 0x44, 0x00, + 0xa0, 0x6a, 0x44, 0x00, 0xc0, 0x6a, 0x44, 0x00, 0xe0, 0x6a, 0x44, 0x00, 0x00, 0x6b, 0x44, 0x00, 0x20, + 0x6b, 0x44, 0x00, 0x40, 0x6b, 0x44, 0x00, 0x60, 0x6b, 0x44, 0x00, 0x80, 0x6b, 0x44, 0x00, 0xa0, 0x6b, + 0x44, 0x00, 0xc0, 0x6b, 0x44, 0x00, 0xe0, 0x6b, 0x44, 0x00, 0x00, 0x6c, 0x44, 0x00, 0x20, 0x6c, 0x44, + 0x00, 0x40, 0x6c, 0x44, 0x00, 0x60, 0x6c, 0x44, 0x00, 0x80, 0x6c, 0x44, 0x00, 0xa0, 0x6c, 0x44, 0x00, + 0xc0, 0x6c, 0x44, 0x00, 0xe0, 0x6c, 0x44, 0x00, 0x00, 0x6d, 0x44, 0x00, 0x20, 0x6d, 0x44, 0x00, 0x40, + 0x6d, 0x44, 0x00, 0x60, 0x6d, 0x44, 0x00, 0x80, 0x6d, 0x44, 0x00, 0xa0, 0x6d, 0x44, 0x00, 0xc0, 0x6d, + 0x44, 0x00, 0xe0, 0x6d, 0x44, 0x00, 0x00, 0x6e, 0x44, 0x00, 0x20, 0x6e, 0x44, 0x00, 0x40, 0x6e, 0x44, + 0x00, 0x60, 0x6e, 0x44, 0x00, 0x80, 0x6e, 0x44, 0x00, 0xa0, 0x6e, 0x44, 0x00, 0xc0, 0x6e, 0x44, 0x00, + 0xe0, 0x6e, 0x44, 0x00, 0x00, 0x6f, 0x44, 0x00, 0x20, 0x6f, 0x44, 0x00, 0x40, 0x6f, 0x44, 0x00, 0x60, + 0x6f, 0x44, 0x00, 0x80, 0x6f, 0x44, 0x00, 0xa0, 0x6f, 0x44, 0x00, 0xc0, 0x6f, 0x44, 0x00, 0xe0, 0x6f, + 0x44, 0x00, 0x00, 0x70, 0x44, 0x00, 0x20, 0x70, 0x44, 0x00, 0x40, 0x70, 0x44, 0x00, 0x60, 0x70, 0x44, + 0x00, 0x80, 0x70, 0x44, 0x00, 0xa0, 0x70, 0x44, 0x00, 0xc0, 0x70, 0x44, 0x00, 0xe0, 0x70, 0x44, 0x00, + 0x00, 0x71, 0x44, 0x00, 0x20, 0x71, 0x44, 0x00, 0x40, 0x71, 0x44, 0x00, 0x60, 0x71, 0x44, 0x00, 0x80, + 0x71, 0x44, 0x00, 0xa0, 0x71, 0x44, 0x00, 0xc0, 0x71, 0x44, 0x00, 0xe0, 0x71, 0x44, 0x00, 0x00, 0x72, + 0x44, 0x00, 0x20, 0x72, 0x44, 0x00, 0x40, 0x72, 0x44, 0x00, 0x60, 0x72, 0x44, 0x00, 0x80, 0x72, 0x44, + 0x00, 0xa0, 0x72, 0x44, 0x00, 0xc0, 0x72, 0x44, 0x00, 0xe0, 0x72, 0x44, 0x00, 0x00, 0x73, 0x44, 0x00, + 0x20, 0x73, 0x44, 0x00, 0x40, 0x73, 0x44, 0x00, 0x60, 0x73, 0x44, 0x00, 0x80, 0x73, 0x44, 0x00, 0xa0, + 0x73, 0x44, 0x00, 0xc0, 0x73, 0x44, 0x00, 0xe0, 0x73, 0x44, 0x00, 0x00, 0x74, 0x44, 0x00, 0x20, 0x74, + 0x44, 0x00, 0x40, 0x74, 0x44, 0x00, 0x60, 0x74, 0x44, 0x00, 0x80, 0x74, 0x44, 0x00, 0xa0, 0x74, 0x44, + 0x00, 0xc0, 0x74, 0x44, 0x00, 0xe0, 0x74, 0x44, 0x00, 0x00, 0x75, 0x44, 0x00, 0x20, 0x75, 0x44, 0x00, + 0x40, 0x75, 0x44, 0x00, 0x60, 0x75, 0x44, 0x00, 0x80, 0x75, 0x44, 0x00, 0xa0, 0x75, 0x44, 0x00, 0xc0, + 0x75, 0x44, 0x00, 0xe0, 0x75, 0x44, 0x00, 0x00, 0x76, 0x44, 0x00, 0x20, 0x76, 0x44, 0x00, 0x40, 0x76, + 0x44, 0x00, 0x60, 0x76, 0x44, 0x00, 0x80, 0x76, 0x44, 0x00, 0xa0, 0x76, 0x44, 0x00, 0xc0, 0x76, 0x44, + 0x00, 0xe0, 0x76, 0x44, 0x00, 0x00, 0x77, 0x44, 0x00, 0x20, 0x77, 0x44, 0x00, 0x40, 0x77, 0x44, 0x00, + 0x60, 0x77, 0x44, 0x00, 0x80, 0x77, 0x44, 0x00, 0xa0, 0x77, 0x44, 0x00, 0xc0, 0x77, 0x44, 0x00, 0xe0, + 0x77, 0x44, 0x00, 0x00, 0x78, 0x44, 0x00, 0x20, 0x78, 0x44, 0x00, 0x40, 0x78, 0x44, 0x00, 0x60, 0x78, + 0x44, 0x00, 0x80, 0x78, 0x44, 0x00, 0xa0, 0x78, 0x44, 0x00, 0xc0, 0x78, 0x44, 0x00, 0xe0, 0x78, 0x44, + 0x00, 0x00, 0x79, 0x44, 0x00, 0x20, 0x79, 0x44, 0x00, 0x40, 0x79, 0x44, 0x00, 0x60, 0x79, 0x44, 0x00, + 0x80, 0x79, 0x44, 0x00, 0xa0, 0x79, 0x44, 0x00, 0xc0, 0x79, 0x44, 0x00, 0xe0, 0x79, 0x44, 0x00, 0x00, + 0x7a, 0x44, 0x00, 0x20, 0x7a, 0x44, 0x00, 0x40, 0x7a, 0x44, 0x00, 0x60, 0x7a, 0x44, 0x00, 0x80, 0x7a, + 0x44, 0x00, 0xa0, 0x7a, 0x44, 0x00, 0xc0, 0x7a, 0x44, 0x00, 0xe0, 0x7a, 0x44, 0x00, 0x00, 0x7b, 0x44, + 0x00, 0x20, 0x7b, 0x44, 0x00, 0x40, 0x7b, 0x44, 0x00, 0x60, 0x7b, 0x44, 0x00, 0x80, 0x7b, 0x44, 0x00, + 0xa0, 0x7b, 0x44, 0x00, 0xc0, 0x7b, 0x44, 0x00, 0xe0, 0x7b, 0x44, 0x00, 0x00, 0x7c, 0x44, 0x00, 0x20, + 0x7c, 0x44, 0x00, 0x40, 0x7c, 0x44, 0x00, 0x60, 0x7c, 0x44, 0x00, 0x80, 0x7c, 0x44, 0x00, 0xa0, 0x7c, + 0x44, 0x00, 0xc0, 0x7c, 0x44, 0x00, 0xe0, 0x7c, 0x44, 0x00, 0x00, 0x7d, 0x44, 0x00, 0x20, 0x7d, 0x44, + 0x00, 0x40, 0x7d, 0x44, 0x00, 0x60, 0x7d, 0x44, 0x00, 0x80, 0x7d, 0x44, 0x00, 0xa0, 0x7d, 0x44, 0x00, + 0xc0, 0x7d, 0x44, 0x00, 0xe0, 0x7d, 0x44, 0x00, 0x00, 0x7e, 0x44, 0x00, 0x20, 0x7e, 0x44, 0x00, 0x40, + 0x7e, 0x44, 0x00, 0x60, 0x7e, 0x44, 0x00, 0x80, 0x7e, 0x44, 0x00, 0xa0, 0x7e, 0x44, 0x00, 0xc0, 0x7e, + 0x44, 0x00, 0xe0, 0x7e, 0x44, 0x00, 0x00, 0x7f, 0x44, 0x00, 0x20, 0x7f, 0x44, 0x00, 0x40, 0x7f, 0x44, + 0x00, 0x60, 0x7f, 0x44, 0x00, 0x80, 0x7f, 0x44, 0x00, 0xa0, 0x7f, 0x44, 0x00, 0xc0, 0x7f, 0x44, 0x00, + 0xe0, 0x7f, 0x44, 0x00, 0x00, 0x80, 0x44, 0x00, 0x10, 0x80, 0x44, 0x00, 0x20, 0x80, 0x44}; unsigned int ___bin_col_float_block_cdf_len = 8192; #endif diff --git a/tests/col_neg_double.h b/tests/col_neg_double.h index 7f0607e4b..c877636b6 100644 --- a/tests/col_neg_double.h +++ b/tests/col_neg_double.h @@ -18,690 +18,487 @@ #ifndef HAVE_COL_NEG_DOUBLE #define HAVE_COL_NEG_DOUBLE unsigned char ___bin_col_neg_double_cdf[] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0xc0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x24, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0xc0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0xc0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x21, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xc0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xc0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x1c, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0xc0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xc0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x16, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0xc0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xc0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x10, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xc0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0xc0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x04, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xbf, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xf0, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0xbf, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xe0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x3f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x0c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x14, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x1a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x23, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x26, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x29, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x2c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x2f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x30, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x31, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x31, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x33, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x34, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x34, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x36, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x37, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x37, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x39, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x3a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x3d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x42, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x43, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x44, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x45, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x46, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x47, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x48, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x49, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x50, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x51, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x51, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x51, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x52, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x52, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x53, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x53, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x54, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x54, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x54, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x55, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x55, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x56, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x56, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x57, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x57, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x57, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x58, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x58, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x59, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x59, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x60, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x61, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x62, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x63, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x64, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x65, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x66, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x67, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x68, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x69, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x70, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x71, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x72, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x73, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x74, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x75, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x76, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x77, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x78, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x79, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7a, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7b, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x58, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x88, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7c, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x78, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7d, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x08, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x38, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x68, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x80, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x98, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7e, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x28, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x7f, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7f, 0x40 -}; + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0xc0, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0xc0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x21, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xc0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1e, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xc0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1a, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x16, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x12, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, + 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xbf, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0xbf, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x3f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x3f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x08, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x21, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x23, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x25, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x27, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x29, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x30, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x31, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x31, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x32, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x33, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x33, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x34, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x34, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x35, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x36, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x37, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x38, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x39, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x3a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x3b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x3c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x3d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x40, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x40, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x41, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x42, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x43, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x43, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x44, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x44, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x45, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x45, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x46, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x47, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x48, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x48, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x48, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x49, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x49, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x49, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x4c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x4d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x4f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x50, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x50, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x50, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x51, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x51, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x52, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x52, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x52, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe0, 0x52, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x53, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x53, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x54, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x54, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x54, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x54, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x55, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x55, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x55, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x56, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x56, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x56, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x57, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x57, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x57, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, + 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x58, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x58, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x58, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x59, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x59, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x59, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x59, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe0, 0x5b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, + 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x5d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x5e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x5f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x5f, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5f, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe0, 0x5f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, + 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x60, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x60, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x60, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe0, 0x60, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x60, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x61, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x61, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x61, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x61, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x61, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x62, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x30, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, + 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x62, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x62, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x62, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x62, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x63, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x63, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x60, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, + 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x63, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x63, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x63, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x64, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x64, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, + 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x64, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x64, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x64, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x65, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x65, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x65, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x65, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x65, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x66, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x66, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x66, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x90, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, + 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x66, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x67, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x67, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x67, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x67, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, + 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x67, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x68, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x68, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x68, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x68, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x90, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, + 0x68, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x69, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x69, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x69, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x69, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x69, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x69, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x90, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf0, 0x6a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x90, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x6c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6d, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6e, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x6e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf0, 0x6e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, + 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x6f, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x6f, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe0, 0x6f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x6f, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x70, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x70, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x70, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x68, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x70, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x78, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x70, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x88, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x98, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa8, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, + 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x70, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x70, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x70, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x70, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x70, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x71, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x71, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x71, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x71, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x78, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x71, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x88, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x71, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x98, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb8, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, + 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x71, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x71, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x71, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x71, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x72, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x72, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, + 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x72, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x72, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x72, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x88, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x72, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x98, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x72, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc8, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, + 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x72, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x72, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x72, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x73, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x73, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x73, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x30, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, + 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x73, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x73, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x73, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x98, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x73, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x73, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, + 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x73, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x73, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x74, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x74, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x74, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x74, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, + 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x74, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x74, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x74, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x74, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x74, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe8, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0x74, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x75, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x75, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x75, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x75, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x75, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x60, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x75, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x75, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x75, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x75, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x75, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf8, 0x75, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x76, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x76, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x76, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x76, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x76, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, + 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x76, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x76, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x76, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x76, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x76, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf8, 0x76, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, + 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x77, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x77, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x77, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x77, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x60, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x77, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, + 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x77, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x77, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x77, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x77, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x77, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf8, 0x77, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x08, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x18, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, + 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x78, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x78, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x78, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x78, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x78, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x90, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa0, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, + 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x78, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x78, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x78, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x78, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0x78, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x18, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, + 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x79, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x79, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x79, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x79, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x79, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x90, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x79, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x79, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x79, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf8, 0x79, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x18, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x28, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x38, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, + 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x7a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7a, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7a, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x90, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7a, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xa0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, + 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x7a, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7a, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7a, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x38, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x48, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, + 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x7b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7b, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7b, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x90, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7b, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7b, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd0, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, + 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7b, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7b, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x28, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x38, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x48, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x58, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, + 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x7c, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7c, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7c, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7c, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7c, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, + 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7c, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x38, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x48, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x58, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x68, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, + 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x7d, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7d, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7d, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7d, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7d, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xd0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf0, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7d, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x7e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7e, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x48, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x58, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x68, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, + 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x7e, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x7e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x7e, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x7e, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x7e, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xe0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x7e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x7f, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x7f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x7f, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x7f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x7f, 0x40}; unsigned int ___bin_col_neg_double_cdf_len = 8192; #endif - diff --git a/tests/col_neg_float.h b/tests/col_neg_float.h index 203893379..ad7b4d7c8 100644 --- a/tests/col_neg_float.h +++ b/tests/col_neg_float.h @@ -18,689 +18,487 @@ #ifndef HAVE_COL_NEG_FLOAT #define HAVE_COL_NEG_FLOAT unsigned char ___bin_col_neg_float_cdf[] = { - 0x00, 0x00, 0x28, 0xc1, 0x00, 0x00, 0x20, 0xc1, 0x00, 0x00, 0x18, 0xc1, - 0x00, 0x00, 0x10, 0xc1, 0x00, 0x00, 0x08, 0xc1, 0x00, 0x00, 0x00, 0xc1, - 0x00, 0x00, 0xf0, 0xc0, 0x00, 0x00, 0xe0, 0xc0, 0x00, 0x00, 0xd0, 0xc0, - 0x00, 0x00, 0xc0, 0xc0, 0x00, 0x00, 0xb0, 0xc0, 0x00, 0x00, 0xa0, 0xc0, - 0x00, 0x00, 0x90, 0xc0, 0x00, 0x00, 0x80, 0xc0, 0x00, 0x00, 0x60, 0xc0, - 0x00, 0x00, 0x40, 0xc0, 0x00, 0x00, 0x20, 0xc0, 0x00, 0x00, 0x00, 0xc0, - 0x00, 0x00, 0xc0, 0xbf, 0x00, 0x00, 0x80, 0xbf, 0x00, 0x00, 0x00, 0xbf, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x80, 0x3f, - 0x00, 0x00, 0xc0, 0x3f, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x20, 0x40, - 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x80, 0x40, - 0x00, 0x00, 0x90, 0x40, 0x00, 0x00, 0xa0, 0x40, 0x00, 0x00, 0xb0, 0x40, - 0x00, 0x00, 0xc0, 0x40, 0x00, 0x00, 0xd0, 0x40, 0x00, 0x00, 0xe0, 0x40, - 0x00, 0x00, 0xf0, 0x40, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x08, 0x41, - 0x00, 0x00, 0x10, 0x41, 0x00, 0x00, 0x18, 0x41, 0x00, 0x00, 0x20, 0x41, - 0x00, 0x00, 0x28, 0x41, 0x00, 0x00, 0x30, 0x41, 0x00, 0x00, 0x38, 0x41, - 0x00, 0x00, 0x40, 0x41, 0x00, 0x00, 0x48, 0x41, 0x00, 0x00, 0x50, 0x41, - 0x00, 0x00, 0x58, 0x41, 0x00, 0x00, 0x60, 0x41, 0x00, 0x00, 0x68, 0x41, - 0x00, 0x00, 0x70, 0x41, 0x00, 0x00, 0x78, 0x41, 0x00, 0x00, 0x80, 0x41, - 0x00, 0x00, 0x84, 0x41, 0x00, 0x00, 0x88, 0x41, 0x00, 0x00, 0x8c, 0x41, - 0x00, 0x00, 0x90, 0x41, 0x00, 0x00, 0x94, 0x41, 0x00, 0x00, 0x98, 0x41, - 0x00, 0x00, 0x9c, 0x41, 0x00, 0x00, 0xa0, 0x41, 0x00, 0x00, 0xa4, 0x41, - 0x00, 0x00, 0xa8, 0x41, 0x00, 0x00, 0xac, 0x41, 0x00, 0x00, 0xb0, 0x41, - 0x00, 0x00, 0xb4, 0x41, 0x00, 0x00, 0xb8, 0x41, 0x00, 0x00, 0xbc, 0x41, - 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xc4, 0x41, 0x00, 0x00, 0xc8, 0x41, - 0x00, 0x00, 0xcc, 0x41, 0x00, 0x00, 0xd0, 0x41, 0x00, 0x00, 0xd4, 0x41, - 0x00, 0x00, 0xd8, 0x41, 0x00, 0x00, 0xdc, 0x41, 0x00, 0x00, 0xe0, 0x41, - 0x00, 0x00, 0xe4, 0x41, 0x00, 0x00, 0xe8, 0x41, 0x00, 0x00, 0xec, 0x41, - 0x00, 0x00, 0xf0, 0x41, 0x00, 0x00, 0xf4, 0x41, 0x00, 0x00, 0xf8, 0x41, - 0x00, 0x00, 0xfc, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x02, 0x42, - 0x00, 0x00, 0x04, 0x42, 0x00, 0x00, 0x06, 0x42, 0x00, 0x00, 0x08, 0x42, - 0x00, 0x00, 0x0a, 0x42, 0x00, 0x00, 0x0c, 0x42, 0x00, 0x00, 0x0e, 0x42, - 0x00, 0x00, 0x10, 0x42, 0x00, 0x00, 0x12, 0x42, 0x00, 0x00, 0x14, 0x42, - 0x00, 0x00, 0x16, 0x42, 0x00, 0x00, 0x18, 0x42, 0x00, 0x00, 0x1a, 0x42, - 0x00, 0x00, 0x1c, 0x42, 0x00, 0x00, 0x1e, 0x42, 0x00, 0x00, 0x20, 0x42, - 0x00, 0x00, 0x22, 0x42, 0x00, 0x00, 0x24, 0x42, 0x00, 0x00, 0x26, 0x42, - 0x00, 0x00, 0x28, 0x42, 0x00, 0x00, 0x2a, 0x42, 0x00, 0x00, 0x2c, 0x42, - 0x00, 0x00, 0x2e, 0x42, 0x00, 0x00, 0x30, 0x42, 0x00, 0x00, 0x32, 0x42, - 0x00, 0x00, 0x34, 0x42, 0x00, 0x00, 0x36, 0x42, 0x00, 0x00, 0x38, 0x42, - 0x00, 0x00, 0x3a, 0x42, 0x00, 0x00, 0x3c, 0x42, 0x00, 0x00, 0x3e, 0x42, - 0x00, 0x00, 0x40, 0x42, 0x00, 0x00, 0x42, 0x42, 0x00, 0x00, 0x44, 0x42, - 0x00, 0x00, 0x46, 0x42, 0x00, 0x00, 0x48, 0x42, 0x00, 0x00, 0x4a, 0x42, - 0x00, 0x00, 0x4c, 0x42, 0x00, 0x00, 0x4e, 0x42, 0x00, 0x00, 0x50, 0x42, - 0x00, 0x00, 0x52, 0x42, 0x00, 0x00, 0x54, 0x42, 0x00, 0x00, 0x56, 0x42, - 0x00, 0x00, 0x58, 0x42, 0x00, 0x00, 0x5a, 0x42, 0x00, 0x00, 0x5c, 0x42, - 0x00, 0x00, 0x5e, 0x42, 0x00, 0x00, 0x60, 0x42, 0x00, 0x00, 0x62, 0x42, - 0x00, 0x00, 0x64, 0x42, 0x00, 0x00, 0x66, 0x42, 0x00, 0x00, 0x68, 0x42, - 0x00, 0x00, 0x6a, 0x42, 0x00, 0x00, 0x6c, 0x42, 0x00, 0x00, 0x6e, 0x42, - 0x00, 0x00, 0x70, 0x42, 0x00, 0x00, 0x72, 0x42, 0x00, 0x00, 0x74, 0x42, - 0x00, 0x00, 0x76, 0x42, 0x00, 0x00, 0x78, 0x42, 0x00, 0x00, 0x7a, 0x42, - 0x00, 0x00, 0x7c, 0x42, 0x00, 0x00, 0x7e, 0x42, 0x00, 0x00, 0x80, 0x42, - 0x00, 0x00, 0x81, 0x42, 0x00, 0x00, 0x82, 0x42, 0x00, 0x00, 0x83, 0x42, - 0x00, 0x00, 0x84, 0x42, 0x00, 0x00, 0x85, 0x42, 0x00, 0x00, 0x86, 0x42, - 0x00, 0x00, 0x87, 0x42, 0x00, 0x00, 0x88, 0x42, 0x00, 0x00, 0x89, 0x42, - 0x00, 0x00, 0x8a, 0x42, 0x00, 0x00, 0x8b, 0x42, 0x00, 0x00, 0x8c, 0x42, - 0x00, 0x00, 0x8d, 0x42, 0x00, 0x00, 0x8e, 0x42, 0x00, 0x00, 0x8f, 0x42, - 0x00, 0x00, 0x90, 0x42, 0x00, 0x00, 0x91, 0x42, 0x00, 0x00, 0x92, 0x42, - 0x00, 0x00, 0x93, 0x42, 0x00, 0x00, 0x94, 0x42, 0x00, 0x00, 0x95, 0x42, - 0x00, 0x00, 0x96, 0x42, 0x00, 0x00, 0x97, 0x42, 0x00, 0x00, 0x98, 0x42, - 0x00, 0x00, 0x99, 0x42, 0x00, 0x00, 0x9a, 0x42, 0x00, 0x00, 0x9b, 0x42, - 0x00, 0x00, 0x9c, 0x42, 0x00, 0x00, 0x9d, 0x42, 0x00, 0x00, 0x9e, 0x42, - 0x00, 0x00, 0x9f, 0x42, 0x00, 0x00, 0xa0, 0x42, 0x00, 0x00, 0xa1, 0x42, - 0x00, 0x00, 0xa2, 0x42, 0x00, 0x00, 0xa3, 0x42, 0x00, 0x00, 0xa4, 0x42, - 0x00, 0x00, 0xa5, 0x42, 0x00, 0x00, 0xa6, 0x42, 0x00, 0x00, 0xa7, 0x42, - 0x00, 0x00, 0xa8, 0x42, 0x00, 0x00, 0xa9, 0x42, 0x00, 0x00, 0xaa, 0x42, - 0x00, 0x00, 0xab, 0x42, 0x00, 0x00, 0xac, 0x42, 0x00, 0x00, 0xad, 0x42, - 0x00, 0x00, 0xae, 0x42, 0x00, 0x00, 0xaf, 0x42, 0x00, 0x00, 0xb0, 0x42, - 0x00, 0x00, 0xb1, 0x42, 0x00, 0x00, 0xb2, 0x42, 0x00, 0x00, 0xb3, 0x42, - 0x00, 0x00, 0xb4, 0x42, 0x00, 0x00, 0xb5, 0x42, 0x00, 0x00, 0xb6, 0x42, - 0x00, 0x00, 0xb7, 0x42, 0x00, 0x00, 0xb8, 0x42, 0x00, 0x00, 0xb9, 0x42, - 0x00, 0x00, 0xba, 0x42, 0x00, 0x00, 0xbb, 0x42, 0x00, 0x00, 0xbc, 0x42, - 0x00, 0x00, 0xbd, 0x42, 0x00, 0x00, 0xbe, 0x42, 0x00, 0x00, 0xbf, 0x42, - 0x00, 0x00, 0xc0, 0x42, 0x00, 0x00, 0xc1, 0x42, 0x00, 0x00, 0xc2, 0x42, - 0x00, 0x00, 0xc3, 0x42, 0x00, 0x00, 0xc4, 0x42, 0x00, 0x00, 0xc5, 0x42, - 0x00, 0x00, 0xc6, 0x42, 0x00, 0x00, 0xc7, 0x42, 0x00, 0x00, 0xc8, 0x42, - 0x00, 0x00, 0xc9, 0x42, 0x00, 0x00, 0xca, 0x42, 0x00, 0x00, 0xcb, 0x42, - 0x00, 0x00, 0xcc, 0x42, 0x00, 0x00, 0xcd, 0x42, 0x00, 0x00, 0xce, 0x42, - 0x00, 0x00, 0xcf, 0x42, 0x00, 0x00, 0xd0, 0x42, 0x00, 0x00, 0xd1, 0x42, - 0x00, 0x00, 0xd2, 0x42, 0x00, 0x00, 0xd3, 0x42, 0x00, 0x00, 0xd4, 0x42, - 0x00, 0x00, 0xd5, 0x42, 0x00, 0x00, 0xd6, 0x42, 0x00, 0x00, 0xd7, 0x42, - 0x00, 0x00, 0xd8, 0x42, 0x00, 0x00, 0xd9, 0x42, 0x00, 0x00, 0xda, 0x42, - 0x00, 0x00, 0xdb, 0x42, 0x00, 0x00, 0xdc, 0x42, 0x00, 0x00, 0xdd, 0x42, - 0x00, 0x00, 0xde, 0x42, 0x00, 0x00, 0xdf, 0x42, 0x00, 0x00, 0xe0, 0x42, - 0x00, 0x00, 0xe1, 0x42, 0x00, 0x00, 0xe2, 0x42, 0x00, 0x00, 0xe3, 0x42, - 0x00, 0x00, 0xe4, 0x42, 0x00, 0x00, 0xe5, 0x42, 0x00, 0x00, 0xe6, 0x42, - 0x00, 0x00, 0xe7, 0x42, 0x00, 0x00, 0xe8, 0x42, 0x00, 0x00, 0xe9, 0x42, - 0x00, 0x00, 0xea, 0x42, 0x00, 0x00, 0xeb, 0x42, 0x00, 0x00, 0xec, 0x42, - 0x00, 0x00, 0xed, 0x42, 0x00, 0x00, 0xee, 0x42, 0x00, 0x00, 0xef, 0x42, - 0x00, 0x00, 0xf0, 0x42, 0x00, 0x00, 0xf1, 0x42, 0x00, 0x00, 0xf2, 0x42, - 0x00, 0x00, 0xf3, 0x42, 0x00, 0x00, 0xf4, 0x42, 0x00, 0x00, 0xf5, 0x42, - 0x00, 0x00, 0xf6, 0x42, 0x00, 0x00, 0xf7, 0x42, 0x00, 0x00, 0xf8, 0x42, - 0x00, 0x00, 0xf9, 0x42, 0x00, 0x00, 0xfa, 0x42, 0x00, 0x00, 0xfb, 0x42, - 0x00, 0x00, 0xfc, 0x42, 0x00, 0x00, 0xfd, 0x42, 0x00, 0x00, 0xfe, 0x42, - 0x00, 0x00, 0xff, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x80, 0x00, 0x43, - 0x00, 0x00, 0x01, 0x43, 0x00, 0x80, 0x01, 0x43, 0x00, 0x00, 0x02, 0x43, - 0x00, 0x80, 0x02, 0x43, 0x00, 0x00, 0x03, 0x43, 0x00, 0x80, 0x03, 0x43, - 0x00, 0x00, 0x04, 0x43, 0x00, 0x80, 0x04, 0x43, 0x00, 0x00, 0x05, 0x43, - 0x00, 0x80, 0x05, 0x43, 0x00, 0x00, 0x06, 0x43, 0x00, 0x80, 0x06, 0x43, - 0x00, 0x00, 0x07, 0x43, 0x00, 0x80, 0x07, 0x43, 0x00, 0x00, 0x08, 0x43, - 0x00, 0x80, 0x08, 0x43, 0x00, 0x00, 0x09, 0x43, 0x00, 0x80, 0x09, 0x43, - 0x00, 0x00, 0x0a, 0x43, 0x00, 0x80, 0x0a, 0x43, 0x00, 0x00, 0x0b, 0x43, - 0x00, 0x80, 0x0b, 0x43, 0x00, 0x00, 0x0c, 0x43, 0x00, 0x80, 0x0c, 0x43, - 0x00, 0x00, 0x0d, 0x43, 0x00, 0x80, 0x0d, 0x43, 0x00, 0x00, 0x0e, 0x43, - 0x00, 0x80, 0x0e, 0x43, 0x00, 0x00, 0x0f, 0x43, 0x00, 0x80, 0x0f, 0x43, - 0x00, 0x00, 0x10, 0x43, 0x00, 0x80, 0x10, 0x43, 0x00, 0x00, 0x11, 0x43, - 0x00, 0x80, 0x11, 0x43, 0x00, 0x00, 0x12, 0x43, 0x00, 0x80, 0x12, 0x43, - 0x00, 0x00, 0x13, 0x43, 0x00, 0x80, 0x13, 0x43, 0x00, 0x00, 0x14, 0x43, - 0x00, 0x80, 0x14, 0x43, 0x00, 0x00, 0x15, 0x43, 0x00, 0x80, 0x15, 0x43, - 0x00, 0x00, 0x16, 0x43, 0x00, 0x80, 0x16, 0x43, 0x00, 0x00, 0x17, 0x43, - 0x00, 0x80, 0x17, 0x43, 0x00, 0x00, 0x18, 0x43, 0x00, 0x80, 0x18, 0x43, - 0x00, 0x00, 0x19, 0x43, 0x00, 0x80, 0x19, 0x43, 0x00, 0x00, 0x1a, 0x43, - 0x00, 0x80, 0x1a, 0x43, 0x00, 0x00, 0x1b, 0x43, 0x00, 0x80, 0x1b, 0x43, - 0x00, 0x00, 0x1c, 0x43, 0x00, 0x80, 0x1c, 0x43, 0x00, 0x00, 0x1d, 0x43, - 0x00, 0x80, 0x1d, 0x43, 0x00, 0x00, 0x1e, 0x43, 0x00, 0x80, 0x1e, 0x43, - 0x00, 0x00, 0x1f, 0x43, 0x00, 0x80, 0x1f, 0x43, 0x00, 0x00, 0x20, 0x43, - 0x00, 0x80, 0x20, 0x43, 0x00, 0x00, 0x21, 0x43, 0x00, 0x80, 0x21, 0x43, - 0x00, 0x00, 0x22, 0x43, 0x00, 0x80, 0x22, 0x43, 0x00, 0x00, 0x23, 0x43, - 0x00, 0x80, 0x23, 0x43, 0x00, 0x00, 0x24, 0x43, 0x00, 0x80, 0x24, 0x43, - 0x00, 0x00, 0x25, 0x43, 0x00, 0x80, 0x25, 0x43, 0x00, 0x00, 0x26, 0x43, - 0x00, 0x80, 0x26, 0x43, 0x00, 0x00, 0x27, 0x43, 0x00, 0x80, 0x27, 0x43, - 0x00, 0x00, 0x28, 0x43, 0x00, 0x80, 0x28, 0x43, 0x00, 0x00, 0x29, 0x43, - 0x00, 0x80, 0x29, 0x43, 0x00, 0x00, 0x2a, 0x43, 0x00, 0x80, 0x2a, 0x43, - 0x00, 0x00, 0x2b, 0x43, 0x00, 0x80, 0x2b, 0x43, 0x00, 0x00, 0x2c, 0x43, - 0x00, 0x80, 0x2c, 0x43, 0x00, 0x00, 0x2d, 0x43, 0x00, 0x80, 0x2d, 0x43, - 0x00, 0x00, 0x2e, 0x43, 0x00, 0x80, 0x2e, 0x43, 0x00, 0x00, 0x2f, 0x43, - 0x00, 0x80, 0x2f, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x80, 0x30, 0x43, - 0x00, 0x00, 0x31, 0x43, 0x00, 0x80, 0x31, 0x43, 0x00, 0x00, 0x32, 0x43, - 0x00, 0x80, 0x32, 0x43, 0x00, 0x00, 0x33, 0x43, 0x00, 0x80, 0x33, 0x43, - 0x00, 0x00, 0x34, 0x43, 0x00, 0x80, 0x34, 0x43, 0x00, 0x00, 0x35, 0x43, - 0x00, 0x80, 0x35, 0x43, 0x00, 0x00, 0x36, 0x43, 0x00, 0x80, 0x36, 0x43, - 0x00, 0x00, 0x37, 0x43, 0x00, 0x80, 0x37, 0x43, 0x00, 0x00, 0x38, 0x43, - 0x00, 0x80, 0x38, 0x43, 0x00, 0x00, 0x39, 0x43, 0x00, 0x80, 0x39, 0x43, - 0x00, 0x00, 0x3a, 0x43, 0x00, 0x80, 0x3a, 0x43, 0x00, 0x00, 0x3b, 0x43, - 0x00, 0x80, 0x3b, 0x43, 0x00, 0x00, 0x3c, 0x43, 0x00, 0x80, 0x3c, 0x43, - 0x00, 0x00, 0x3d, 0x43, 0x00, 0x80, 0x3d, 0x43, 0x00, 0x00, 0x3e, 0x43, - 0x00, 0x80, 0x3e, 0x43, 0x00, 0x00, 0x3f, 0x43, 0x00, 0x80, 0x3f, 0x43, - 0x00, 0x00, 0x40, 0x43, 0x00, 0x80, 0x40, 0x43, 0x00, 0x00, 0x41, 0x43, - 0x00, 0x80, 0x41, 0x43, 0x00, 0x00, 0x42, 0x43, 0x00, 0x80, 0x42, 0x43, - 0x00, 0x00, 0x43, 0x43, 0x00, 0x80, 0x43, 0x43, 0x00, 0x00, 0x44, 0x43, - 0x00, 0x80, 0x44, 0x43, 0x00, 0x00, 0x45, 0x43, 0x00, 0x80, 0x45, 0x43, - 0x00, 0x00, 0x46, 0x43, 0x00, 0x80, 0x46, 0x43, 0x00, 0x00, 0x47, 0x43, - 0x00, 0x80, 0x47, 0x43, 0x00, 0x00, 0x48, 0x43, 0x00, 0x80, 0x48, 0x43, - 0x00, 0x00, 0x49, 0x43, 0x00, 0x80, 0x49, 0x43, 0x00, 0x00, 0x4a, 0x43, - 0x00, 0x80, 0x4a, 0x43, 0x00, 0x00, 0x4b, 0x43, 0x00, 0x80, 0x4b, 0x43, - 0x00, 0x00, 0x4c, 0x43, 0x00, 0x80, 0x4c, 0x43, 0x00, 0x00, 0x4d, 0x43, - 0x00, 0x80, 0x4d, 0x43, 0x00, 0x00, 0x4e, 0x43, 0x00, 0x80, 0x4e, 0x43, - 0x00, 0x00, 0x4f, 0x43, 0x00, 0x80, 0x4f, 0x43, 0x00, 0x00, 0x50, 0x43, - 0x00, 0x80, 0x50, 0x43, 0x00, 0x00, 0x51, 0x43, 0x00, 0x80, 0x51, 0x43, - 0x00, 0x00, 0x52, 0x43, 0x00, 0x80, 0x52, 0x43, 0x00, 0x00, 0x53, 0x43, - 0x00, 0x80, 0x53, 0x43, 0x00, 0x00, 0x54, 0x43, 0x00, 0x80, 0x54, 0x43, - 0x00, 0x00, 0x55, 0x43, 0x00, 0x80, 0x55, 0x43, 0x00, 0x00, 0x56, 0x43, - 0x00, 0x80, 0x56, 0x43, 0x00, 0x00, 0x57, 0x43, 0x00, 0x80, 0x57, 0x43, - 0x00, 0x00, 0x58, 0x43, 0x00, 0x80, 0x58, 0x43, 0x00, 0x00, 0x59, 0x43, - 0x00, 0x80, 0x59, 0x43, 0x00, 0x00, 0x5a, 0x43, 0x00, 0x80, 0x5a, 0x43, - 0x00, 0x00, 0x5b, 0x43, 0x00, 0x80, 0x5b, 0x43, 0x00, 0x00, 0x5c, 0x43, - 0x00, 0x80, 0x5c, 0x43, 0x00, 0x00, 0x5d, 0x43, 0x00, 0x80, 0x5d, 0x43, - 0x00, 0x00, 0x5e, 0x43, 0x00, 0x80, 0x5e, 0x43, 0x00, 0x00, 0x5f, 0x43, - 0x00, 0x80, 0x5f, 0x43, 0x00, 0x00, 0x60, 0x43, 0x00, 0x80, 0x60, 0x43, - 0x00, 0x00, 0x61, 0x43, 0x00, 0x80, 0x61, 0x43, 0x00, 0x00, 0x62, 0x43, - 0x00, 0x80, 0x62, 0x43, 0x00, 0x00, 0x63, 0x43, 0x00, 0x80, 0x63, 0x43, - 0x00, 0x00, 0x64, 0x43, 0x00, 0x80, 0x64, 0x43, 0x00, 0x00, 0x65, 0x43, - 0x00, 0x80, 0x65, 0x43, 0x00, 0x00, 0x66, 0x43, 0x00, 0x80, 0x66, 0x43, - 0x00, 0x00, 0x67, 0x43, 0x00, 0x80, 0x67, 0x43, 0x00, 0x00, 0x68, 0x43, - 0x00, 0x80, 0x68, 0x43, 0x00, 0x00, 0x69, 0x43, 0x00, 0x80, 0x69, 0x43, - 0x00, 0x00, 0x6a, 0x43, 0x00, 0x80, 0x6a, 0x43, 0x00, 0x00, 0x6b, 0x43, - 0x00, 0x80, 0x6b, 0x43, 0x00, 0x00, 0x6c, 0x43, 0x00, 0x80, 0x6c, 0x43, - 0x00, 0x00, 0x6d, 0x43, 0x00, 0x80, 0x6d, 0x43, 0x00, 0x00, 0x6e, 0x43, - 0x00, 0x80, 0x6e, 0x43, 0x00, 0x00, 0x6f, 0x43, 0x00, 0x80, 0x6f, 0x43, - 0x00, 0x00, 0x70, 0x43, 0x00, 0x80, 0x70, 0x43, 0x00, 0x00, 0x71, 0x43, - 0x00, 0x80, 0x71, 0x43, 0x00, 0x00, 0x72, 0x43, 0x00, 0x80, 0x72, 0x43, - 0x00, 0x00, 0x73, 0x43, 0x00, 0x80, 0x73, 0x43, 0x00, 0x00, 0x74, 0x43, - 0x00, 0x80, 0x74, 0x43, 0x00, 0x00, 0x75, 0x43, 0x00, 0x80, 0x75, 0x43, - 0x00, 0x00, 0x76, 0x43, 0x00, 0x80, 0x76, 0x43, 0x00, 0x00, 0x77, 0x43, - 0x00, 0x80, 0x77, 0x43, 0x00, 0x00, 0x78, 0x43, 0x00, 0x80, 0x78, 0x43, - 0x00, 0x00, 0x79, 0x43, 0x00, 0x80, 0x79, 0x43, 0x00, 0x00, 0x7a, 0x43, - 0x00, 0x80, 0x7a, 0x43, 0x00, 0x00, 0x7b, 0x43, 0x00, 0x80, 0x7b, 0x43, - 0x00, 0x00, 0x7c, 0x43, 0x00, 0x80, 0x7c, 0x43, 0x00, 0x00, 0x7d, 0x43, - 0x00, 0x80, 0x7d, 0x43, 0x00, 0x00, 0x7e, 0x43, 0x00, 0x80, 0x7e, 0x43, - 0x00, 0x00, 0x7f, 0x43, 0x00, 0x80, 0x7f, 0x43, 0x00, 0x00, 0x80, 0x43, - 0x00, 0x40, 0x80, 0x43, 0x00, 0x80, 0x80, 0x43, 0x00, 0xc0, 0x80, 0x43, - 0x00, 0x00, 0x81, 0x43, 0x00, 0x40, 0x81, 0x43, 0x00, 0x80, 0x81, 0x43, - 0x00, 0xc0, 0x81, 0x43, 0x00, 0x00, 0x82, 0x43, 0x00, 0x40, 0x82, 0x43, - 0x00, 0x80, 0x82, 0x43, 0x00, 0xc0, 0x82, 0x43, 0x00, 0x00, 0x83, 0x43, - 0x00, 0x40, 0x83, 0x43, 0x00, 0x80, 0x83, 0x43, 0x00, 0xc0, 0x83, 0x43, - 0x00, 0x00, 0x84, 0x43, 0x00, 0x40, 0x84, 0x43, 0x00, 0x80, 0x84, 0x43, - 0x00, 0xc0, 0x84, 0x43, 0x00, 0x00, 0x85, 0x43, 0x00, 0x40, 0x85, 0x43, - 0x00, 0x80, 0x85, 0x43, 0x00, 0xc0, 0x85, 0x43, 0x00, 0x00, 0x86, 0x43, - 0x00, 0x40, 0x86, 0x43, 0x00, 0x80, 0x86, 0x43, 0x00, 0xc0, 0x86, 0x43, - 0x00, 0x00, 0x87, 0x43, 0x00, 0x40, 0x87, 0x43, 0x00, 0x80, 0x87, 0x43, - 0x00, 0xc0, 0x87, 0x43, 0x00, 0x00, 0x88, 0x43, 0x00, 0x40, 0x88, 0x43, - 0x00, 0x80, 0x88, 0x43, 0x00, 0xc0, 0x88, 0x43, 0x00, 0x00, 0x89, 0x43, - 0x00, 0x40, 0x89, 0x43, 0x00, 0x80, 0x89, 0x43, 0x00, 0xc0, 0x89, 0x43, - 0x00, 0x00, 0x8a, 0x43, 0x00, 0x40, 0x8a, 0x43, 0x00, 0x80, 0x8a, 0x43, - 0x00, 0xc0, 0x8a, 0x43, 0x00, 0x00, 0x8b, 0x43, 0x00, 0x40, 0x8b, 0x43, - 0x00, 0x80, 0x8b, 0x43, 0x00, 0xc0, 0x8b, 0x43, 0x00, 0x00, 0x8c, 0x43, - 0x00, 0x40, 0x8c, 0x43, 0x00, 0x80, 0x8c, 0x43, 0x00, 0xc0, 0x8c, 0x43, - 0x00, 0x00, 0x8d, 0x43, 0x00, 0x40, 0x8d, 0x43, 0x00, 0x80, 0x8d, 0x43, - 0x00, 0xc0, 0x8d, 0x43, 0x00, 0x00, 0x8e, 0x43, 0x00, 0x40, 0x8e, 0x43, - 0x00, 0x80, 0x8e, 0x43, 0x00, 0xc0, 0x8e, 0x43, 0x00, 0x00, 0x8f, 0x43, - 0x00, 0x40, 0x8f, 0x43, 0x00, 0x80, 0x8f, 0x43, 0x00, 0xc0, 0x8f, 0x43, - 0x00, 0x00, 0x90, 0x43, 0x00, 0x40, 0x90, 0x43, 0x00, 0x80, 0x90, 0x43, - 0x00, 0xc0, 0x90, 0x43, 0x00, 0x00, 0x91, 0x43, 0x00, 0x40, 0x91, 0x43, - 0x00, 0x80, 0x91, 0x43, 0x00, 0xc0, 0x91, 0x43, 0x00, 0x00, 0x92, 0x43, - 0x00, 0x40, 0x92, 0x43, 0x00, 0x80, 0x92, 0x43, 0x00, 0xc0, 0x92, 0x43, - 0x00, 0x00, 0x93, 0x43, 0x00, 0x40, 0x93, 0x43, 0x00, 0x80, 0x93, 0x43, - 0x00, 0xc0, 0x93, 0x43, 0x00, 0x00, 0x94, 0x43, 0x00, 0x40, 0x94, 0x43, - 0x00, 0x80, 0x94, 0x43, 0x00, 0xc0, 0x94, 0x43, 0x00, 0x00, 0x95, 0x43, - 0x00, 0x40, 0x95, 0x43, 0x00, 0x80, 0x95, 0x43, 0x00, 0xc0, 0x95, 0x43, - 0x00, 0x00, 0x96, 0x43, 0x00, 0x40, 0x96, 0x43, 0x00, 0x80, 0x96, 0x43, - 0x00, 0xc0, 0x96, 0x43, 0x00, 0x00, 0x97, 0x43, 0x00, 0x40, 0x97, 0x43, - 0x00, 0x80, 0x97, 0x43, 0x00, 0xc0, 0x97, 0x43, 0x00, 0x00, 0x98, 0x43, - 0x00, 0x40, 0x98, 0x43, 0x00, 0x80, 0x98, 0x43, 0x00, 0xc0, 0x98, 0x43, - 0x00, 0x00, 0x99, 0x43, 0x00, 0x40, 0x99, 0x43, 0x00, 0x80, 0x99, 0x43, - 0x00, 0xc0, 0x99, 0x43, 0x00, 0x00, 0x9a, 0x43, 0x00, 0x40, 0x9a, 0x43, - 0x00, 0x80, 0x9a, 0x43, 0x00, 0xc0, 0x9a, 0x43, 0x00, 0x00, 0x9b, 0x43, - 0x00, 0x40, 0x9b, 0x43, 0x00, 0x80, 0x9b, 0x43, 0x00, 0xc0, 0x9b, 0x43, - 0x00, 0x00, 0x9c, 0x43, 0x00, 0x40, 0x9c, 0x43, 0x00, 0x80, 0x9c, 0x43, - 0x00, 0xc0, 0x9c, 0x43, 0x00, 0x00, 0x9d, 0x43, 0x00, 0x40, 0x9d, 0x43, - 0x00, 0x80, 0x9d, 0x43, 0x00, 0xc0, 0x9d, 0x43, 0x00, 0x00, 0x9e, 0x43, - 0x00, 0x40, 0x9e, 0x43, 0x00, 0x80, 0x9e, 0x43, 0x00, 0xc0, 0x9e, 0x43, - 0x00, 0x00, 0x9f, 0x43, 0x00, 0x40, 0x9f, 0x43, 0x00, 0x80, 0x9f, 0x43, - 0x00, 0xc0, 0x9f, 0x43, 0x00, 0x00, 0xa0, 0x43, 0x00, 0x40, 0xa0, 0x43, - 0x00, 0x80, 0xa0, 0x43, 0x00, 0xc0, 0xa0, 0x43, 0x00, 0x00, 0xa1, 0x43, - 0x00, 0x40, 0xa1, 0x43, 0x00, 0x80, 0xa1, 0x43, 0x00, 0xc0, 0xa1, 0x43, - 0x00, 0x00, 0xa2, 0x43, 0x00, 0x40, 0xa2, 0x43, 0x00, 0x80, 0xa2, 0x43, - 0x00, 0xc0, 0xa2, 0x43, 0x00, 0x00, 0xa3, 0x43, 0x00, 0x40, 0xa3, 0x43, - 0x00, 0x80, 0xa3, 0x43, 0x00, 0xc0, 0xa3, 0x43, 0x00, 0x00, 0xa4, 0x43, - 0x00, 0x40, 0xa4, 0x43, 0x00, 0x80, 0xa4, 0x43, 0x00, 0xc0, 0xa4, 0x43, - 0x00, 0x00, 0xa5, 0x43, 0x00, 0x40, 0xa5, 0x43, 0x00, 0x80, 0xa5, 0x43, - 0x00, 0xc0, 0xa5, 0x43, 0x00, 0x00, 0xa6, 0x43, 0x00, 0x40, 0xa6, 0x43, - 0x00, 0x80, 0xa6, 0x43, 0x00, 0xc0, 0xa6, 0x43, 0x00, 0x00, 0xa7, 0x43, - 0x00, 0x40, 0xa7, 0x43, 0x00, 0x80, 0xa7, 0x43, 0x00, 0xc0, 0xa7, 0x43, - 0x00, 0x00, 0xa8, 0x43, 0x00, 0x40, 0xa8, 0x43, 0x00, 0x80, 0xa8, 0x43, - 0x00, 0xc0, 0xa8, 0x43, 0x00, 0x00, 0xa9, 0x43, 0x00, 0x40, 0xa9, 0x43, - 0x00, 0x80, 0xa9, 0x43, 0x00, 0xc0, 0xa9, 0x43, 0x00, 0x00, 0xaa, 0x43, - 0x00, 0x40, 0xaa, 0x43, 0x00, 0x80, 0xaa, 0x43, 0x00, 0xc0, 0xaa, 0x43, - 0x00, 0x00, 0xab, 0x43, 0x00, 0x40, 0xab, 0x43, 0x00, 0x80, 0xab, 0x43, - 0x00, 0xc0, 0xab, 0x43, 0x00, 0x00, 0xac, 0x43, 0x00, 0x40, 0xac, 0x43, - 0x00, 0x80, 0xac, 0x43, 0x00, 0xc0, 0xac, 0x43, 0x00, 0x00, 0xad, 0x43, - 0x00, 0x40, 0xad, 0x43, 0x00, 0x80, 0xad, 0x43, 0x00, 0xc0, 0xad, 0x43, - 0x00, 0x00, 0xae, 0x43, 0x00, 0x40, 0xae, 0x43, 0x00, 0x80, 0xae, 0x43, - 0x00, 0xc0, 0xae, 0x43, 0x00, 0x00, 0xaf, 0x43, 0x00, 0x40, 0xaf, 0x43, - 0x00, 0x80, 0xaf, 0x43, 0x00, 0xc0, 0xaf, 0x43, 0x00, 0x00, 0xb0, 0x43, - 0x00, 0x40, 0xb0, 0x43, 0x00, 0x80, 0xb0, 0x43, 0x00, 0xc0, 0xb0, 0x43, - 0x00, 0x00, 0xb1, 0x43, 0x00, 0x40, 0xb1, 0x43, 0x00, 0x80, 0xb1, 0x43, - 0x00, 0xc0, 0xb1, 0x43, 0x00, 0x00, 0xb2, 0x43, 0x00, 0x40, 0xb2, 0x43, - 0x00, 0x80, 0xb2, 0x43, 0x00, 0xc0, 0xb2, 0x43, 0x00, 0x00, 0xb3, 0x43, - 0x00, 0x40, 0xb3, 0x43, 0x00, 0x80, 0xb3, 0x43, 0x00, 0xc0, 0xb3, 0x43, - 0x00, 0x00, 0xb4, 0x43, 0x00, 0x40, 0xb4, 0x43, 0x00, 0x80, 0xb4, 0x43, - 0x00, 0xc0, 0xb4, 0x43, 0x00, 0x00, 0xb5, 0x43, 0x00, 0x40, 0xb5, 0x43, - 0x00, 0x80, 0xb5, 0x43, 0x00, 0xc0, 0xb5, 0x43, 0x00, 0x00, 0xb6, 0x43, - 0x00, 0x40, 0xb6, 0x43, 0x00, 0x80, 0xb6, 0x43, 0x00, 0xc0, 0xb6, 0x43, - 0x00, 0x00, 0xb7, 0x43, 0x00, 0x40, 0xb7, 0x43, 0x00, 0x80, 0xb7, 0x43, - 0x00, 0xc0, 0xb7, 0x43, 0x00, 0x00, 0xb8, 0x43, 0x00, 0x40, 0xb8, 0x43, - 0x00, 0x80, 0xb8, 0x43, 0x00, 0xc0, 0xb8, 0x43, 0x00, 0x00, 0xb9, 0x43, - 0x00, 0x40, 0xb9, 0x43, 0x00, 0x80, 0xb9, 0x43, 0x00, 0xc0, 0xb9, 0x43, - 0x00, 0x00, 0xba, 0x43, 0x00, 0x40, 0xba, 0x43, 0x00, 0x80, 0xba, 0x43, - 0x00, 0xc0, 0xba, 0x43, 0x00, 0x00, 0xbb, 0x43, 0x00, 0x40, 0xbb, 0x43, - 0x00, 0x80, 0xbb, 0x43, 0x00, 0xc0, 0xbb, 0x43, 0x00, 0x00, 0xbc, 0x43, - 0x00, 0x40, 0xbc, 0x43, 0x00, 0x80, 0xbc, 0x43, 0x00, 0xc0, 0xbc, 0x43, - 0x00, 0x00, 0xbd, 0x43, 0x00, 0x40, 0xbd, 0x43, 0x00, 0x80, 0xbd, 0x43, - 0x00, 0xc0, 0xbd, 0x43, 0x00, 0x00, 0xbe, 0x43, 0x00, 0x40, 0xbe, 0x43, - 0x00, 0x80, 0xbe, 0x43, 0x00, 0xc0, 0xbe, 0x43, 0x00, 0x00, 0xbf, 0x43, - 0x00, 0x40, 0xbf, 0x43, 0x00, 0x80, 0xbf, 0x43, 0x00, 0xc0, 0xbf, 0x43, - 0x00, 0x00, 0xc0, 0x43, 0x00, 0x40, 0xc0, 0x43, 0x00, 0x80, 0xc0, 0x43, - 0x00, 0xc0, 0xc0, 0x43, 0x00, 0x00, 0xc1, 0x43, 0x00, 0x40, 0xc1, 0x43, - 0x00, 0x80, 0xc1, 0x43, 0x00, 0xc0, 0xc1, 0x43, 0x00, 0x00, 0xc2, 0x43, - 0x00, 0x40, 0xc2, 0x43, 0x00, 0x80, 0xc2, 0x43, 0x00, 0xc0, 0xc2, 0x43, - 0x00, 0x00, 0xc3, 0x43, 0x00, 0x40, 0xc3, 0x43, 0x00, 0x80, 0xc3, 0x43, - 0x00, 0xc0, 0xc3, 0x43, 0x00, 0x00, 0xc4, 0x43, 0x00, 0x40, 0xc4, 0x43, - 0x00, 0x80, 0xc4, 0x43, 0x00, 0xc0, 0xc4, 0x43, 0x00, 0x00, 0xc5, 0x43, - 0x00, 0x40, 0xc5, 0x43, 0x00, 0x80, 0xc5, 0x43, 0x00, 0xc0, 0xc5, 0x43, - 0x00, 0x00, 0xc6, 0x43, 0x00, 0x40, 0xc6, 0x43, 0x00, 0x80, 0xc6, 0x43, - 0x00, 0xc0, 0xc6, 0x43, 0x00, 0x00, 0xc7, 0x43, 0x00, 0x40, 0xc7, 0x43, - 0x00, 0x80, 0xc7, 0x43, 0x00, 0xc0, 0xc7, 0x43, 0x00, 0x00, 0xc8, 0x43, - 0x00, 0x40, 0xc8, 0x43, 0x00, 0x80, 0xc8, 0x43, 0x00, 0xc0, 0xc8, 0x43, - 0x00, 0x00, 0xc9, 0x43, 0x00, 0x40, 0xc9, 0x43, 0x00, 0x80, 0xc9, 0x43, - 0x00, 0xc0, 0xc9, 0x43, 0x00, 0x00, 0xca, 0x43, 0x00, 0x40, 0xca, 0x43, - 0x00, 0x80, 0xca, 0x43, 0x00, 0xc0, 0xca, 0x43, 0x00, 0x00, 0xcb, 0x43, - 0x00, 0x40, 0xcb, 0x43, 0x00, 0x80, 0xcb, 0x43, 0x00, 0xc0, 0xcb, 0x43, - 0x00, 0x00, 0xcc, 0x43, 0x00, 0x40, 0xcc, 0x43, 0x00, 0x80, 0xcc, 0x43, - 0x00, 0xc0, 0xcc, 0x43, 0x00, 0x00, 0xcd, 0x43, 0x00, 0x40, 0xcd, 0x43, - 0x00, 0x80, 0xcd, 0x43, 0x00, 0xc0, 0xcd, 0x43, 0x00, 0x00, 0xce, 0x43, - 0x00, 0x40, 0xce, 0x43, 0x00, 0x80, 0xce, 0x43, 0x00, 0xc0, 0xce, 0x43, - 0x00, 0x00, 0xcf, 0x43, 0x00, 0x40, 0xcf, 0x43, 0x00, 0x80, 0xcf, 0x43, - 0x00, 0xc0, 0xcf, 0x43, 0x00, 0x00, 0xd0, 0x43, 0x00, 0x40, 0xd0, 0x43, - 0x00, 0x80, 0xd0, 0x43, 0x00, 0xc0, 0xd0, 0x43, 0x00, 0x00, 0xd1, 0x43, - 0x00, 0x40, 0xd1, 0x43, 0x00, 0x80, 0xd1, 0x43, 0x00, 0xc0, 0xd1, 0x43, - 0x00, 0x00, 0xd2, 0x43, 0x00, 0x40, 0xd2, 0x43, 0x00, 0x80, 0xd2, 0x43, - 0x00, 0xc0, 0xd2, 0x43, 0x00, 0x00, 0xd3, 0x43, 0x00, 0x40, 0xd3, 0x43, - 0x00, 0x80, 0xd3, 0x43, 0x00, 0xc0, 0xd3, 0x43, 0x00, 0x00, 0xd4, 0x43, - 0x00, 0x40, 0xd4, 0x43, 0x00, 0x80, 0xd4, 0x43, 0x00, 0xc0, 0xd4, 0x43, - 0x00, 0x00, 0xd5, 0x43, 0x00, 0x40, 0xd5, 0x43, 0x00, 0x80, 0xd5, 0x43, - 0x00, 0xc0, 0xd5, 0x43, 0x00, 0x00, 0xd6, 0x43, 0x00, 0x40, 0xd6, 0x43, - 0x00, 0x80, 0xd6, 0x43, 0x00, 0xc0, 0xd6, 0x43, 0x00, 0x00, 0xd7, 0x43, - 0x00, 0x40, 0xd7, 0x43, 0x00, 0x80, 0xd7, 0x43, 0x00, 0xc0, 0xd7, 0x43, - 0x00, 0x00, 0xd8, 0x43, 0x00, 0x40, 0xd8, 0x43, 0x00, 0x80, 0xd8, 0x43, - 0x00, 0xc0, 0xd8, 0x43, 0x00, 0x00, 0xd9, 0x43, 0x00, 0x40, 0xd9, 0x43, - 0x00, 0x80, 0xd9, 0x43, 0x00, 0xc0, 0xd9, 0x43, 0x00, 0x00, 0xda, 0x43, - 0x00, 0x40, 0xda, 0x43, 0x00, 0x80, 0xda, 0x43, 0x00, 0xc0, 0xda, 0x43, - 0x00, 0x00, 0xdb, 0x43, 0x00, 0x40, 0xdb, 0x43, 0x00, 0x80, 0xdb, 0x43, - 0x00, 0xc0, 0xdb, 0x43, 0x00, 0x00, 0xdc, 0x43, 0x00, 0x40, 0xdc, 0x43, - 0x00, 0x80, 0xdc, 0x43, 0x00, 0xc0, 0xdc, 0x43, 0x00, 0x00, 0xdd, 0x43, - 0x00, 0x40, 0xdd, 0x43, 0x00, 0x80, 0xdd, 0x43, 0x00, 0xc0, 0xdd, 0x43, - 0x00, 0x00, 0xde, 0x43, 0x00, 0x40, 0xde, 0x43, 0x00, 0x80, 0xde, 0x43, - 0x00, 0xc0, 0xde, 0x43, 0x00, 0x00, 0xdf, 0x43, 0x00, 0x40, 0xdf, 0x43, - 0x00, 0x80, 0xdf, 0x43, 0x00, 0xc0, 0xdf, 0x43, 0x00, 0x00, 0xe0, 0x43, - 0x00, 0x40, 0xe0, 0x43, 0x00, 0x80, 0xe0, 0x43, 0x00, 0xc0, 0xe0, 0x43, - 0x00, 0x00, 0xe1, 0x43, 0x00, 0x40, 0xe1, 0x43, 0x00, 0x80, 0xe1, 0x43, - 0x00, 0xc0, 0xe1, 0x43, 0x00, 0x00, 0xe2, 0x43, 0x00, 0x40, 0xe2, 0x43, - 0x00, 0x80, 0xe2, 0x43, 0x00, 0xc0, 0xe2, 0x43, 0x00, 0x00, 0xe3, 0x43, - 0x00, 0x40, 0xe3, 0x43, 0x00, 0x80, 0xe3, 0x43, 0x00, 0xc0, 0xe3, 0x43, - 0x00, 0x00, 0xe4, 0x43, 0x00, 0x40, 0xe4, 0x43, 0x00, 0x80, 0xe4, 0x43, - 0x00, 0xc0, 0xe4, 0x43, 0x00, 0x00, 0xe5, 0x43, 0x00, 0x40, 0xe5, 0x43, - 0x00, 0x80, 0xe5, 0x43, 0x00, 0xc0, 0xe5, 0x43, 0x00, 0x00, 0xe6, 0x43, - 0x00, 0x40, 0xe6, 0x43, 0x00, 0x80, 0xe6, 0x43, 0x00, 0xc0, 0xe6, 0x43, - 0x00, 0x00, 0xe7, 0x43, 0x00, 0x40, 0xe7, 0x43, 0x00, 0x80, 0xe7, 0x43, - 0x00, 0xc0, 0xe7, 0x43, 0x00, 0x00, 0xe8, 0x43, 0x00, 0x40, 0xe8, 0x43, - 0x00, 0x80, 0xe8, 0x43, 0x00, 0xc0, 0xe8, 0x43, 0x00, 0x00, 0xe9, 0x43, - 0x00, 0x40, 0xe9, 0x43, 0x00, 0x80, 0xe9, 0x43, 0x00, 0xc0, 0xe9, 0x43, - 0x00, 0x00, 0xea, 0x43, 0x00, 0x40, 0xea, 0x43, 0x00, 0x80, 0xea, 0x43, - 0x00, 0xc0, 0xea, 0x43, 0x00, 0x00, 0xeb, 0x43, 0x00, 0x40, 0xeb, 0x43, - 0x00, 0x80, 0xeb, 0x43, 0x00, 0xc0, 0xeb, 0x43, 0x00, 0x00, 0xec, 0x43, - 0x00, 0x40, 0xec, 0x43, 0x00, 0x80, 0xec, 0x43, 0x00, 0xc0, 0xec, 0x43, - 0x00, 0x00, 0xed, 0x43, 0x00, 0x40, 0xed, 0x43, 0x00, 0x80, 0xed, 0x43, - 0x00, 0xc0, 0xed, 0x43, 0x00, 0x00, 0xee, 0x43, 0x00, 0x40, 0xee, 0x43, - 0x00, 0x80, 0xee, 0x43, 0x00, 0xc0, 0xee, 0x43, 0x00, 0x00, 0xef, 0x43, - 0x00, 0x40, 0xef, 0x43, 0x00, 0x80, 0xef, 0x43, 0x00, 0xc0, 0xef, 0x43, - 0x00, 0x00, 0xf0, 0x43, 0x00, 0x40, 0xf0, 0x43, 0x00, 0x80, 0xf0, 0x43, - 0x00, 0xc0, 0xf0, 0x43, 0x00, 0x00, 0xf1, 0x43, 0x00, 0x40, 0xf1, 0x43, - 0x00, 0x80, 0xf1, 0x43, 0x00, 0xc0, 0xf1, 0x43, 0x00, 0x00, 0xf2, 0x43, - 0x00, 0x40, 0xf2, 0x43, 0x00, 0x80, 0xf2, 0x43, 0x00, 0xc0, 0xf2, 0x43, - 0x00, 0x00, 0xf3, 0x43, 0x00, 0x40, 0xf3, 0x43, 0x00, 0x80, 0xf3, 0x43, - 0x00, 0xc0, 0xf3, 0x43, 0x00, 0x00, 0xf4, 0x43, 0x00, 0x40, 0xf4, 0x43, - 0x00, 0x80, 0xf4, 0x43, 0x00, 0xc0, 0xf4, 0x43, 0x00, 0x00, 0xf5, 0x43, - 0x00, 0x40, 0xf5, 0x43, 0x00, 0x80, 0xf5, 0x43, 0x00, 0xc0, 0xf5, 0x43, - 0x00, 0x00, 0xf6, 0x43, 0x00, 0x40, 0xf6, 0x43, 0x00, 0x80, 0xf6, 0x43, - 0x00, 0xc0, 0xf6, 0x43, 0x00, 0x00, 0xf7, 0x43, 0x00, 0x40, 0xf7, 0x43, - 0x00, 0x80, 0xf7, 0x43, 0x00, 0xc0, 0xf7, 0x43, 0x00, 0x00, 0xf8, 0x43, - 0x00, 0x40, 0xf8, 0x43, 0x00, 0x80, 0xf8, 0x43, 0x00, 0xc0, 0xf8, 0x43, - 0x00, 0x00, 0xf9, 0x43, 0x00, 0x40, 0xf9, 0x43, 0x00, 0x80, 0xf9, 0x43, - 0x00, 0xc0, 0xf9, 0x43, 0x00, 0x00, 0xfa, 0x43, 0x00, 0x40, 0xfa, 0x43, - 0x00, 0x80, 0xfa, 0x43, 0x00, 0xc0, 0xfa, 0x43, 0x00, 0x00, 0xfb, 0x43, - 0x00, 0x40, 0xfb, 0x43, 0x00, 0x80, 0xfb, 0x43, 0x00, 0xc0, 0xfb, 0x43, - 0x00, 0x00, 0xfc, 0x43, 0x00, 0x40, 0xfc, 0x43, 0x00, 0x80, 0xfc, 0x43, - 0x00, 0xc0, 0xfc, 0x43, 0x00, 0x00, 0xfd, 0x43, 0x00, 0x40, 0xfd, 0x43, - 0x00, 0x80, 0xfd, 0x43, 0x00, 0xc0, 0xfd, 0x43, 0x00, 0x00, 0xfe, 0x43, - 0x00, 0x40, 0xfe, 0x43, 0x00, 0x80, 0xfe, 0x43, 0x00, 0xc0, 0xfe, 0x43, - 0x00, 0x00, 0xff, 0x43, 0x00, 0x40, 0xff, 0x43, 0x00, 0x80, 0xff, 0x43, - 0x00, 0xc0, 0xff, 0x43, 0x00, 0x00, 0x00, 0x44, 0x00, 0x20, 0x00, 0x44, - 0x00, 0x40, 0x00, 0x44, 0x00, 0x60, 0x00, 0x44, 0x00, 0x80, 0x00, 0x44, - 0x00, 0xa0, 0x00, 0x44, 0x00, 0xc0, 0x00, 0x44, 0x00, 0xe0, 0x00, 0x44, - 0x00, 0x00, 0x01, 0x44, 0x00, 0x20, 0x01, 0x44, 0x00, 0x40, 0x01, 0x44, - 0x00, 0x60, 0x01, 0x44, 0x00, 0x80, 0x01, 0x44, 0x00, 0xa0, 0x01, 0x44, - 0x00, 0xc0, 0x01, 0x44, 0x00, 0xe0, 0x01, 0x44, 0x00, 0x00, 0x02, 0x44, - 0x00, 0x20, 0x02, 0x44, 0x00, 0x40, 0x02, 0x44, 0x00, 0x60, 0x02, 0x44, - 0x00, 0x80, 0x02, 0x44, 0x00, 0xa0, 0x02, 0x44, 0x00, 0xc0, 0x02, 0x44, - 0x00, 0xe0, 0x02, 0x44, 0x00, 0x00, 0x03, 0x44, 0x00, 0x20, 0x03, 0x44, - 0x00, 0x40, 0x03, 0x44, 0x00, 0x60, 0x03, 0x44, 0x00, 0x80, 0x03, 0x44, - 0x00, 0xa0, 0x03, 0x44, 0x00, 0xc0, 0x03, 0x44, 0x00, 0xe0, 0x03, 0x44, - 0x00, 0x00, 0x04, 0x44, 0x00, 0x20, 0x04, 0x44, 0x00, 0x40, 0x04, 0x44, - 0x00, 0x60, 0x04, 0x44, 0x00, 0x80, 0x04, 0x44, 0x00, 0xa0, 0x04, 0x44, - 0x00, 0xc0, 0x04, 0x44, 0x00, 0xe0, 0x04, 0x44, 0x00, 0x00, 0x05, 0x44, - 0x00, 0x20, 0x05, 0x44, 0x00, 0x40, 0x05, 0x44, 0x00, 0x60, 0x05, 0x44, - 0x00, 0x80, 0x05, 0x44, 0x00, 0xa0, 0x05, 0x44, 0x00, 0xc0, 0x05, 0x44, - 0x00, 0xe0, 0x05, 0x44, 0x00, 0x00, 0x06, 0x44, 0x00, 0x20, 0x06, 0x44, - 0x00, 0x40, 0x06, 0x44, 0x00, 0x60, 0x06, 0x44, 0x00, 0x80, 0x06, 0x44, - 0x00, 0xa0, 0x06, 0x44, 0x00, 0xc0, 0x06, 0x44, 0x00, 0xe0, 0x06, 0x44, - 0x00, 0x00, 0x07, 0x44, 0x00, 0x20, 0x07, 0x44, 0x00, 0x40, 0x07, 0x44, - 0x00, 0x60, 0x07, 0x44, 0x00, 0x80, 0x07, 0x44, 0x00, 0xa0, 0x07, 0x44, - 0x00, 0xc0, 0x07, 0x44, 0x00, 0xe0, 0x07, 0x44, 0x00, 0x00, 0x08, 0x44, - 0x00, 0x20, 0x08, 0x44, 0x00, 0x40, 0x08, 0x44, 0x00, 0x60, 0x08, 0x44, - 0x00, 0x80, 0x08, 0x44, 0x00, 0xa0, 0x08, 0x44, 0x00, 0xc0, 0x08, 0x44, - 0x00, 0xe0, 0x08, 0x44, 0x00, 0x00, 0x09, 0x44, 0x00, 0x20, 0x09, 0x44, - 0x00, 0x40, 0x09, 0x44, 0x00, 0x60, 0x09, 0x44, 0x00, 0x80, 0x09, 0x44, - 0x00, 0xa0, 0x09, 0x44, 0x00, 0xc0, 0x09, 0x44, 0x00, 0xe0, 0x09, 0x44, - 0x00, 0x00, 0x0a, 0x44, 0x00, 0x20, 0x0a, 0x44, 0x00, 0x40, 0x0a, 0x44, - 0x00, 0x60, 0x0a, 0x44, 0x00, 0x80, 0x0a, 0x44, 0x00, 0xa0, 0x0a, 0x44, - 0x00, 0xc0, 0x0a, 0x44, 0x00, 0xe0, 0x0a, 0x44, 0x00, 0x00, 0x0b, 0x44, - 0x00, 0x20, 0x0b, 0x44, 0x00, 0x40, 0x0b, 0x44, 0x00, 0x60, 0x0b, 0x44, - 0x00, 0x80, 0x0b, 0x44, 0x00, 0xa0, 0x0b, 0x44, 0x00, 0xc0, 0x0b, 0x44, - 0x00, 0xe0, 0x0b, 0x44, 0x00, 0x00, 0x0c, 0x44, 0x00, 0x20, 0x0c, 0x44, - 0x00, 0x40, 0x0c, 0x44, 0x00, 0x60, 0x0c, 0x44, 0x00, 0x80, 0x0c, 0x44, - 0x00, 0xa0, 0x0c, 0x44, 0x00, 0xc0, 0x0c, 0x44, 0x00, 0xe0, 0x0c, 0x44, - 0x00, 0x00, 0x0d, 0x44, 0x00, 0x20, 0x0d, 0x44, 0x00, 0x40, 0x0d, 0x44, - 0x00, 0x60, 0x0d, 0x44, 0x00, 0x80, 0x0d, 0x44, 0x00, 0xa0, 0x0d, 0x44, - 0x00, 0xc0, 0x0d, 0x44, 0x00, 0xe0, 0x0d, 0x44, 0x00, 0x00, 0x0e, 0x44, - 0x00, 0x20, 0x0e, 0x44, 0x00, 0x40, 0x0e, 0x44, 0x00, 0x60, 0x0e, 0x44, - 0x00, 0x80, 0x0e, 0x44, 0x00, 0xa0, 0x0e, 0x44, 0x00, 0xc0, 0x0e, 0x44, - 0x00, 0xe0, 0x0e, 0x44, 0x00, 0x00, 0x0f, 0x44, 0x00, 0x20, 0x0f, 0x44, - 0x00, 0x40, 0x0f, 0x44, 0x00, 0x60, 0x0f, 0x44, 0x00, 0x80, 0x0f, 0x44, - 0x00, 0xa0, 0x0f, 0x44, 0x00, 0xc0, 0x0f, 0x44, 0x00, 0xe0, 0x0f, 0x44, - 0x00, 0x00, 0x10, 0x44, 0x00, 0x20, 0x10, 0x44, 0x00, 0x40, 0x10, 0x44, - 0x00, 0x60, 0x10, 0x44, 0x00, 0x80, 0x10, 0x44, 0x00, 0xa0, 0x10, 0x44, - 0x00, 0xc0, 0x10, 0x44, 0x00, 0xe0, 0x10, 0x44, 0x00, 0x00, 0x11, 0x44, - 0x00, 0x20, 0x11, 0x44, 0x00, 0x40, 0x11, 0x44, 0x00, 0x60, 0x11, 0x44, - 0x00, 0x80, 0x11, 0x44, 0x00, 0xa0, 0x11, 0x44, 0x00, 0xc0, 0x11, 0x44, - 0x00, 0xe0, 0x11, 0x44, 0x00, 0x00, 0x12, 0x44, 0x00, 0x20, 0x12, 0x44, - 0x00, 0x40, 0x12, 0x44, 0x00, 0x60, 0x12, 0x44, 0x00, 0x80, 0x12, 0x44, - 0x00, 0xa0, 0x12, 0x44, 0x00, 0xc0, 0x12, 0x44, 0x00, 0xe0, 0x12, 0x44, - 0x00, 0x00, 0x13, 0x44, 0x00, 0x20, 0x13, 0x44, 0x00, 0x40, 0x13, 0x44, - 0x00, 0x60, 0x13, 0x44, 0x00, 0x80, 0x13, 0x44, 0x00, 0xa0, 0x13, 0x44, - 0x00, 0xc0, 0x13, 0x44, 0x00, 0xe0, 0x13, 0x44, 0x00, 0x00, 0x14, 0x44, - 0x00, 0x20, 0x14, 0x44, 0x00, 0x40, 0x14, 0x44, 0x00, 0x60, 0x14, 0x44, - 0x00, 0x80, 0x14, 0x44, 0x00, 0xa0, 0x14, 0x44, 0x00, 0xc0, 0x14, 0x44, - 0x00, 0xe0, 0x14, 0x44, 0x00, 0x00, 0x15, 0x44, 0x00, 0x20, 0x15, 0x44, - 0x00, 0x40, 0x15, 0x44, 0x00, 0x60, 0x15, 0x44, 0x00, 0x80, 0x15, 0x44, - 0x00, 0xa0, 0x15, 0x44, 0x00, 0xc0, 0x15, 0x44, 0x00, 0xe0, 0x15, 0x44, - 0x00, 0x00, 0x16, 0x44, 0x00, 0x20, 0x16, 0x44, 0x00, 0x40, 0x16, 0x44, - 0x00, 0x60, 0x16, 0x44, 0x00, 0x80, 0x16, 0x44, 0x00, 0xa0, 0x16, 0x44, - 0x00, 0xc0, 0x16, 0x44, 0x00, 0xe0, 0x16, 0x44, 0x00, 0x00, 0x17, 0x44, - 0x00, 0x20, 0x17, 0x44, 0x00, 0x40, 0x17, 0x44, 0x00, 0x60, 0x17, 0x44, - 0x00, 0x80, 0x17, 0x44, 0x00, 0xa0, 0x17, 0x44, 0x00, 0xc0, 0x17, 0x44, - 0x00, 0xe0, 0x17, 0x44, 0x00, 0x00, 0x18, 0x44, 0x00, 0x20, 0x18, 0x44, - 0x00, 0x40, 0x18, 0x44, 0x00, 0x60, 0x18, 0x44, 0x00, 0x80, 0x18, 0x44, - 0x00, 0xa0, 0x18, 0x44, 0x00, 0xc0, 0x18, 0x44, 0x00, 0xe0, 0x18, 0x44, - 0x00, 0x00, 0x19, 0x44, 0x00, 0x20, 0x19, 0x44, 0x00, 0x40, 0x19, 0x44, - 0x00, 0x60, 0x19, 0x44, 0x00, 0x80, 0x19, 0x44, 0x00, 0xa0, 0x19, 0x44, - 0x00, 0xc0, 0x19, 0x44, 0x00, 0xe0, 0x19, 0x44, 0x00, 0x00, 0x1a, 0x44, - 0x00, 0x20, 0x1a, 0x44, 0x00, 0x40, 0x1a, 0x44, 0x00, 0x60, 0x1a, 0x44, - 0x00, 0x80, 0x1a, 0x44, 0x00, 0xa0, 0x1a, 0x44, 0x00, 0xc0, 0x1a, 0x44, - 0x00, 0xe0, 0x1a, 0x44, 0x00, 0x00, 0x1b, 0x44, 0x00, 0x20, 0x1b, 0x44, - 0x00, 0x40, 0x1b, 0x44, 0x00, 0x60, 0x1b, 0x44, 0x00, 0x80, 0x1b, 0x44, - 0x00, 0xa0, 0x1b, 0x44, 0x00, 0xc0, 0x1b, 0x44, 0x00, 0xe0, 0x1b, 0x44, - 0x00, 0x00, 0x1c, 0x44, 0x00, 0x20, 0x1c, 0x44, 0x00, 0x40, 0x1c, 0x44, - 0x00, 0x60, 0x1c, 0x44, 0x00, 0x80, 0x1c, 0x44, 0x00, 0xa0, 0x1c, 0x44, - 0x00, 0xc0, 0x1c, 0x44, 0x00, 0xe0, 0x1c, 0x44, 0x00, 0x00, 0x1d, 0x44, - 0x00, 0x20, 0x1d, 0x44, 0x00, 0x40, 0x1d, 0x44, 0x00, 0x60, 0x1d, 0x44, - 0x00, 0x80, 0x1d, 0x44, 0x00, 0xa0, 0x1d, 0x44, 0x00, 0xc0, 0x1d, 0x44, - 0x00, 0xe0, 0x1d, 0x44, 0x00, 0x00, 0x1e, 0x44, 0x00, 0x20, 0x1e, 0x44, - 0x00, 0x40, 0x1e, 0x44, 0x00, 0x60, 0x1e, 0x44, 0x00, 0x80, 0x1e, 0x44, - 0x00, 0xa0, 0x1e, 0x44, 0x00, 0xc0, 0x1e, 0x44, 0x00, 0xe0, 0x1e, 0x44, - 0x00, 0x00, 0x1f, 0x44, 0x00, 0x20, 0x1f, 0x44, 0x00, 0x40, 0x1f, 0x44, - 0x00, 0x60, 0x1f, 0x44, 0x00, 0x80, 0x1f, 0x44, 0x00, 0xa0, 0x1f, 0x44, - 0x00, 0xc0, 0x1f, 0x44, 0x00, 0xe0, 0x1f, 0x44, 0x00, 0x00, 0x20, 0x44, - 0x00, 0x20, 0x20, 0x44, 0x00, 0x40, 0x20, 0x44, 0x00, 0x60, 0x20, 0x44, - 0x00, 0x80, 0x20, 0x44, 0x00, 0xa0, 0x20, 0x44, 0x00, 0xc0, 0x20, 0x44, - 0x00, 0xe0, 0x20, 0x44, 0x00, 0x00, 0x21, 0x44, 0x00, 0x20, 0x21, 0x44, - 0x00, 0x40, 0x21, 0x44, 0x00, 0x60, 0x21, 0x44, 0x00, 0x80, 0x21, 0x44, - 0x00, 0xa0, 0x21, 0x44, 0x00, 0xc0, 0x21, 0x44, 0x00, 0xe0, 0x21, 0x44, - 0x00, 0x00, 0x22, 0x44, 0x00, 0x20, 0x22, 0x44, 0x00, 0x40, 0x22, 0x44, - 0x00, 0x60, 0x22, 0x44, 0x00, 0x80, 0x22, 0x44, 0x00, 0xa0, 0x22, 0x44, - 0x00, 0xc0, 0x22, 0x44, 0x00, 0xe0, 0x22, 0x44, 0x00, 0x00, 0x23, 0x44, - 0x00, 0x20, 0x23, 0x44, 0x00, 0x40, 0x23, 0x44, 0x00, 0x60, 0x23, 0x44, - 0x00, 0x80, 0x23, 0x44, 0x00, 0xa0, 0x23, 0x44, 0x00, 0xc0, 0x23, 0x44, - 0x00, 0xe0, 0x23, 0x44, 0x00, 0x00, 0x24, 0x44, 0x00, 0x20, 0x24, 0x44, - 0x00, 0x40, 0x24, 0x44, 0x00, 0x60, 0x24, 0x44, 0x00, 0x80, 0x24, 0x44, - 0x00, 0xa0, 0x24, 0x44, 0x00, 0xc0, 0x24, 0x44, 0x00, 0xe0, 0x24, 0x44, - 0x00, 0x00, 0x25, 0x44, 0x00, 0x20, 0x25, 0x44, 0x00, 0x40, 0x25, 0x44, - 0x00, 0x60, 0x25, 0x44, 0x00, 0x80, 0x25, 0x44, 0x00, 0xa0, 0x25, 0x44, - 0x00, 0xc0, 0x25, 0x44, 0x00, 0xe0, 0x25, 0x44, 0x00, 0x00, 0x26, 0x44, - 0x00, 0x20, 0x26, 0x44, 0x00, 0x40, 0x26, 0x44, 0x00, 0x60, 0x26, 0x44, - 0x00, 0x80, 0x26, 0x44, 0x00, 0xa0, 0x26, 0x44, 0x00, 0xc0, 0x26, 0x44, - 0x00, 0xe0, 0x26, 0x44, 0x00, 0x00, 0x27, 0x44, 0x00, 0x20, 0x27, 0x44, - 0x00, 0x40, 0x27, 0x44, 0x00, 0x60, 0x27, 0x44, 0x00, 0x80, 0x27, 0x44, - 0x00, 0xa0, 0x27, 0x44, 0x00, 0xc0, 0x27, 0x44, 0x00, 0xe0, 0x27, 0x44, - 0x00, 0x00, 0x28, 0x44, 0x00, 0x20, 0x28, 0x44, 0x00, 0x40, 0x28, 0x44, - 0x00, 0x60, 0x28, 0x44, 0x00, 0x80, 0x28, 0x44, 0x00, 0xa0, 0x28, 0x44, - 0x00, 0xc0, 0x28, 0x44, 0x00, 0xe0, 0x28, 0x44, 0x00, 0x00, 0x29, 0x44, - 0x00, 0x20, 0x29, 0x44, 0x00, 0x40, 0x29, 0x44, 0x00, 0x60, 0x29, 0x44, - 0x00, 0x80, 0x29, 0x44, 0x00, 0xa0, 0x29, 0x44, 0x00, 0xc0, 0x29, 0x44, - 0x00, 0xe0, 0x29, 0x44, 0x00, 0x00, 0x2a, 0x44, 0x00, 0x20, 0x2a, 0x44, - 0x00, 0x40, 0x2a, 0x44, 0x00, 0x60, 0x2a, 0x44, 0x00, 0x80, 0x2a, 0x44, - 0x00, 0xa0, 0x2a, 0x44, 0x00, 0xc0, 0x2a, 0x44, 0x00, 0xe0, 0x2a, 0x44, - 0x00, 0x00, 0x2b, 0x44, 0x00, 0x20, 0x2b, 0x44, 0x00, 0x40, 0x2b, 0x44, - 0x00, 0x60, 0x2b, 0x44, 0x00, 0x80, 0x2b, 0x44, 0x00, 0xa0, 0x2b, 0x44, - 0x00, 0xc0, 0x2b, 0x44, 0x00, 0xe0, 0x2b, 0x44, 0x00, 0x00, 0x2c, 0x44, - 0x00, 0x20, 0x2c, 0x44, 0x00, 0x40, 0x2c, 0x44, 0x00, 0x60, 0x2c, 0x44, - 0x00, 0x80, 0x2c, 0x44, 0x00, 0xa0, 0x2c, 0x44, 0x00, 0xc0, 0x2c, 0x44, - 0x00, 0xe0, 0x2c, 0x44, 0x00, 0x00, 0x2d, 0x44, 0x00, 0x20, 0x2d, 0x44, - 0x00, 0x40, 0x2d, 0x44, 0x00, 0x60, 0x2d, 0x44, 0x00, 0x80, 0x2d, 0x44, - 0x00, 0xa0, 0x2d, 0x44, 0x00, 0xc0, 0x2d, 0x44, 0x00, 0xe0, 0x2d, 0x44, - 0x00, 0x00, 0x2e, 0x44, 0x00, 0x20, 0x2e, 0x44, 0x00, 0x40, 0x2e, 0x44, - 0x00, 0x60, 0x2e, 0x44, 0x00, 0x80, 0x2e, 0x44, 0x00, 0xa0, 0x2e, 0x44, - 0x00, 0xc0, 0x2e, 0x44, 0x00, 0xe0, 0x2e, 0x44, 0x00, 0x00, 0x2f, 0x44, - 0x00, 0x20, 0x2f, 0x44, 0x00, 0x40, 0x2f, 0x44, 0x00, 0x60, 0x2f, 0x44, - 0x00, 0x80, 0x2f, 0x44, 0x00, 0xa0, 0x2f, 0x44, 0x00, 0xc0, 0x2f, 0x44, - 0x00, 0xe0, 0x2f, 0x44, 0x00, 0x00, 0x30, 0x44, 0x00, 0x20, 0x30, 0x44, - 0x00, 0x40, 0x30, 0x44, 0x00, 0x60, 0x30, 0x44, 0x00, 0x80, 0x30, 0x44, - 0x00, 0xa0, 0x30, 0x44, 0x00, 0xc0, 0x30, 0x44, 0x00, 0xe0, 0x30, 0x44, - 0x00, 0x00, 0x31, 0x44, 0x00, 0x20, 0x31, 0x44, 0x00, 0x40, 0x31, 0x44, - 0x00, 0x60, 0x31, 0x44, 0x00, 0x80, 0x31, 0x44, 0x00, 0xa0, 0x31, 0x44, - 0x00, 0xc0, 0x31, 0x44, 0x00, 0xe0, 0x31, 0x44, 0x00, 0x00, 0x32, 0x44, - 0x00, 0x20, 0x32, 0x44, 0x00, 0x40, 0x32, 0x44, 0x00, 0x60, 0x32, 0x44, - 0x00, 0x80, 0x32, 0x44, 0x00, 0xa0, 0x32, 0x44, 0x00, 0xc0, 0x32, 0x44, - 0x00, 0xe0, 0x32, 0x44, 0x00, 0x00, 0x33, 0x44, 0x00, 0x20, 0x33, 0x44, - 0x00, 0x40, 0x33, 0x44, 0x00, 0x60, 0x33, 0x44, 0x00, 0x80, 0x33, 0x44, - 0x00, 0xa0, 0x33, 0x44, 0x00, 0xc0, 0x33, 0x44, 0x00, 0xe0, 0x33, 0x44, - 0x00, 0x00, 0x34, 0x44, 0x00, 0x20, 0x34, 0x44, 0x00, 0x40, 0x34, 0x44, - 0x00, 0x60, 0x34, 0x44, 0x00, 0x80, 0x34, 0x44, 0x00, 0xa0, 0x34, 0x44, - 0x00, 0xc0, 0x34, 0x44, 0x00, 0xe0, 0x34, 0x44, 0x00, 0x00, 0x35, 0x44, - 0x00, 0x20, 0x35, 0x44, 0x00, 0x40, 0x35, 0x44, 0x00, 0x60, 0x35, 0x44, - 0x00, 0x80, 0x35, 0x44, 0x00, 0xa0, 0x35, 0x44, 0x00, 0xc0, 0x35, 0x44, - 0x00, 0xe0, 0x35, 0x44, 0x00, 0x00, 0x36, 0x44, 0x00, 0x20, 0x36, 0x44, - 0x00, 0x40, 0x36, 0x44, 0x00, 0x60, 0x36, 0x44, 0x00, 0x80, 0x36, 0x44, - 0x00, 0xa0, 0x36, 0x44, 0x00, 0xc0, 0x36, 0x44, 0x00, 0xe0, 0x36, 0x44, - 0x00, 0x00, 0x37, 0x44, 0x00, 0x20, 0x37, 0x44, 0x00, 0x40, 0x37, 0x44, - 0x00, 0x60, 0x37, 0x44, 0x00, 0x80, 0x37, 0x44, 0x00, 0xa0, 0x37, 0x44, - 0x00, 0xc0, 0x37, 0x44, 0x00, 0xe0, 0x37, 0x44, 0x00, 0x00, 0x38, 0x44, - 0x00, 0x20, 0x38, 0x44, 0x00, 0x40, 0x38, 0x44, 0x00, 0x60, 0x38, 0x44, - 0x00, 0x80, 0x38, 0x44, 0x00, 0xa0, 0x38, 0x44, 0x00, 0xc0, 0x38, 0x44, - 0x00, 0xe0, 0x38, 0x44, 0x00, 0x00, 0x39, 0x44, 0x00, 0x20, 0x39, 0x44, - 0x00, 0x40, 0x39, 0x44, 0x00, 0x60, 0x39, 0x44, 0x00, 0x80, 0x39, 0x44, - 0x00, 0xa0, 0x39, 0x44, 0x00, 0xc0, 0x39, 0x44, 0x00, 0xe0, 0x39, 0x44, - 0x00, 0x00, 0x3a, 0x44, 0x00, 0x20, 0x3a, 0x44, 0x00, 0x40, 0x3a, 0x44, - 0x00, 0x60, 0x3a, 0x44, 0x00, 0x80, 0x3a, 0x44, 0x00, 0xa0, 0x3a, 0x44, - 0x00, 0xc0, 0x3a, 0x44, 0x00, 0xe0, 0x3a, 0x44, 0x00, 0x00, 0x3b, 0x44, - 0x00, 0x20, 0x3b, 0x44, 0x00, 0x40, 0x3b, 0x44, 0x00, 0x60, 0x3b, 0x44, - 0x00, 0x80, 0x3b, 0x44, 0x00, 0xa0, 0x3b, 0x44, 0x00, 0xc0, 0x3b, 0x44, - 0x00, 0xe0, 0x3b, 0x44, 0x00, 0x00, 0x3c, 0x44, 0x00, 0x20, 0x3c, 0x44, - 0x00, 0x40, 0x3c, 0x44, 0x00, 0x60, 0x3c, 0x44, 0x00, 0x80, 0x3c, 0x44, - 0x00, 0xa0, 0x3c, 0x44, 0x00, 0xc0, 0x3c, 0x44, 0x00, 0xe0, 0x3c, 0x44, - 0x00, 0x00, 0x3d, 0x44, 0x00, 0x20, 0x3d, 0x44, 0x00, 0x40, 0x3d, 0x44, - 0x00, 0x60, 0x3d, 0x44, 0x00, 0x80, 0x3d, 0x44, 0x00, 0xa0, 0x3d, 0x44, - 0x00, 0xc0, 0x3d, 0x44, 0x00, 0xe0, 0x3d, 0x44, 0x00, 0x00, 0x3e, 0x44, - 0x00, 0x20, 0x3e, 0x44, 0x00, 0x40, 0x3e, 0x44, 0x00, 0x60, 0x3e, 0x44, - 0x00, 0x80, 0x3e, 0x44, 0x00, 0xa0, 0x3e, 0x44, 0x00, 0xc0, 0x3e, 0x44, - 0x00, 0xe0, 0x3e, 0x44, 0x00, 0x00, 0x3f, 0x44, 0x00, 0x20, 0x3f, 0x44, - 0x00, 0x40, 0x3f, 0x44, 0x00, 0x60, 0x3f, 0x44, 0x00, 0x80, 0x3f, 0x44, - 0x00, 0xa0, 0x3f, 0x44, 0x00, 0xc0, 0x3f, 0x44, 0x00, 0xe0, 0x3f, 0x44, - 0x00, 0x00, 0x40, 0x44, 0x00, 0x20, 0x40, 0x44, 0x00, 0x40, 0x40, 0x44, - 0x00, 0x60, 0x40, 0x44, 0x00, 0x80, 0x40, 0x44, 0x00, 0xa0, 0x40, 0x44, - 0x00, 0xc0, 0x40, 0x44, 0x00, 0xe0, 0x40, 0x44, 0x00, 0x00, 0x41, 0x44, - 0x00, 0x20, 0x41, 0x44, 0x00, 0x40, 0x41, 0x44, 0x00, 0x60, 0x41, 0x44, - 0x00, 0x80, 0x41, 0x44, 0x00, 0xa0, 0x41, 0x44, 0x00, 0xc0, 0x41, 0x44, - 0x00, 0xe0, 0x41, 0x44, 0x00, 0x00, 0x42, 0x44, 0x00, 0x20, 0x42, 0x44, - 0x00, 0x40, 0x42, 0x44, 0x00, 0x60, 0x42, 0x44, 0x00, 0x80, 0x42, 0x44, - 0x00, 0xa0, 0x42, 0x44, 0x00, 0xc0, 0x42, 0x44, 0x00, 0xe0, 0x42, 0x44, - 0x00, 0x00, 0x43, 0x44, 0x00, 0x20, 0x43, 0x44, 0x00, 0x40, 0x43, 0x44, - 0x00, 0x60, 0x43, 0x44, 0x00, 0x80, 0x43, 0x44, 0x00, 0xa0, 0x43, 0x44, - 0x00, 0xc0, 0x43, 0x44, 0x00, 0xe0, 0x43, 0x44, 0x00, 0x00, 0x44, 0x44, - 0x00, 0x20, 0x44, 0x44, 0x00, 0x40, 0x44, 0x44, 0x00, 0x60, 0x44, 0x44, - 0x00, 0x80, 0x44, 0x44, 0x00, 0xa0, 0x44, 0x44, 0x00, 0xc0, 0x44, 0x44, - 0x00, 0xe0, 0x44, 0x44, 0x00, 0x00, 0x45, 0x44, 0x00, 0x20, 0x45, 0x44, - 0x00, 0x40, 0x45, 0x44, 0x00, 0x60, 0x45, 0x44, 0x00, 0x80, 0x45, 0x44, - 0x00, 0xa0, 0x45, 0x44, 0x00, 0xc0, 0x45, 0x44, 0x00, 0xe0, 0x45, 0x44, - 0x00, 0x00, 0x46, 0x44, 0x00, 0x20, 0x46, 0x44, 0x00, 0x40, 0x46, 0x44, - 0x00, 0x60, 0x46, 0x44, 0x00, 0x80, 0x46, 0x44, 0x00, 0xa0, 0x46, 0x44, - 0x00, 0xc0, 0x46, 0x44, 0x00, 0xe0, 0x46, 0x44, 0x00, 0x00, 0x47, 0x44, - 0x00, 0x20, 0x47, 0x44, 0x00, 0x40, 0x47, 0x44, 0x00, 0x60, 0x47, 0x44, - 0x00, 0x80, 0x47, 0x44, 0x00, 0xa0, 0x47, 0x44, 0x00, 0xc0, 0x47, 0x44, - 0x00, 0xe0, 0x47, 0x44, 0x00, 0x00, 0x48, 0x44, 0x00, 0x20, 0x48, 0x44, - 0x00, 0x40, 0x48, 0x44, 0x00, 0x60, 0x48, 0x44, 0x00, 0x80, 0x48, 0x44, - 0x00, 0xa0, 0x48, 0x44, 0x00, 0xc0, 0x48, 0x44, 0x00, 0xe0, 0x48, 0x44, - 0x00, 0x00, 0x49, 0x44, 0x00, 0x20, 0x49, 0x44, 0x00, 0x40, 0x49, 0x44, - 0x00, 0x60, 0x49, 0x44, 0x00, 0x80, 0x49, 0x44, 0x00, 0xa0, 0x49, 0x44, - 0x00, 0xc0, 0x49, 0x44, 0x00, 0xe0, 0x49, 0x44, 0x00, 0x00, 0x4a, 0x44, - 0x00, 0x20, 0x4a, 0x44, 0x00, 0x40, 0x4a, 0x44, 0x00, 0x60, 0x4a, 0x44, - 0x00, 0x80, 0x4a, 0x44, 0x00, 0xa0, 0x4a, 0x44, 0x00, 0xc0, 0x4a, 0x44, - 0x00, 0xe0, 0x4a, 0x44, 0x00, 0x00, 0x4b, 0x44, 0x00, 0x20, 0x4b, 0x44, - 0x00, 0x40, 0x4b, 0x44, 0x00, 0x60, 0x4b, 0x44, 0x00, 0x80, 0x4b, 0x44, - 0x00, 0xa0, 0x4b, 0x44, 0x00, 0xc0, 0x4b, 0x44, 0x00, 0xe0, 0x4b, 0x44, - 0x00, 0x00, 0x4c, 0x44, 0x00, 0x20, 0x4c, 0x44, 0x00, 0x40, 0x4c, 0x44, - 0x00, 0x60, 0x4c, 0x44, 0x00, 0x80, 0x4c, 0x44, 0x00, 0xa0, 0x4c, 0x44, - 0x00, 0xc0, 0x4c, 0x44, 0x00, 0xe0, 0x4c, 0x44, 0x00, 0x00, 0x4d, 0x44, - 0x00, 0x20, 0x4d, 0x44, 0x00, 0x40, 0x4d, 0x44, 0x00, 0x60, 0x4d, 0x44, - 0x00, 0x80, 0x4d, 0x44, 0x00, 0xa0, 0x4d, 0x44, 0x00, 0xc0, 0x4d, 0x44, - 0x00, 0xe0, 0x4d, 0x44, 0x00, 0x00, 0x4e, 0x44, 0x00, 0x20, 0x4e, 0x44, - 0x00, 0x40, 0x4e, 0x44, 0x00, 0x60, 0x4e, 0x44, 0x00, 0x80, 0x4e, 0x44, - 0x00, 0xa0, 0x4e, 0x44, 0x00, 0xc0, 0x4e, 0x44, 0x00, 0xe0, 0x4e, 0x44, - 0x00, 0x00, 0x4f, 0x44, 0x00, 0x20, 0x4f, 0x44, 0x00, 0x40, 0x4f, 0x44, - 0x00, 0x60, 0x4f, 0x44, 0x00, 0x80, 0x4f, 0x44, 0x00, 0xa0, 0x4f, 0x44, - 0x00, 0xc0, 0x4f, 0x44, 0x00, 0xe0, 0x4f, 0x44, 0x00, 0x00, 0x50, 0x44, - 0x00, 0x20, 0x50, 0x44, 0x00, 0x40, 0x50, 0x44, 0x00, 0x60, 0x50, 0x44, - 0x00, 0x80, 0x50, 0x44, 0x00, 0xa0, 0x50, 0x44, 0x00, 0xc0, 0x50, 0x44, - 0x00, 0xe0, 0x50, 0x44, 0x00, 0x00, 0x51, 0x44, 0x00, 0x20, 0x51, 0x44, - 0x00, 0x40, 0x51, 0x44, 0x00, 0x60, 0x51, 0x44, 0x00, 0x80, 0x51, 0x44, - 0x00, 0xa0, 0x51, 0x44, 0x00, 0xc0, 0x51, 0x44, 0x00, 0xe0, 0x51, 0x44, - 0x00, 0x00, 0x52, 0x44, 0x00, 0x20, 0x52, 0x44, 0x00, 0x40, 0x52, 0x44, - 0x00, 0x60, 0x52, 0x44, 0x00, 0x80, 0x52, 0x44, 0x00, 0xa0, 0x52, 0x44, - 0x00, 0xc0, 0x52, 0x44, 0x00, 0xe0, 0x52, 0x44, 0x00, 0x00, 0x53, 0x44, - 0x00, 0x20, 0x53, 0x44, 0x00, 0x40, 0x53, 0x44, 0x00, 0x60, 0x53, 0x44, - 0x00, 0x80, 0x53, 0x44, 0x00, 0xa0, 0x53, 0x44, 0x00, 0xc0, 0x53, 0x44, - 0x00, 0xe0, 0x53, 0x44, 0x00, 0x00, 0x54, 0x44, 0x00, 0x20, 0x54, 0x44, - 0x00, 0x40, 0x54, 0x44, 0x00, 0x60, 0x54, 0x44, 0x00, 0x80, 0x54, 0x44, - 0x00, 0xa0, 0x54, 0x44, 0x00, 0xc0, 0x54, 0x44, 0x00, 0xe0, 0x54, 0x44, - 0x00, 0x00, 0x55, 0x44, 0x00, 0x20, 0x55, 0x44, 0x00, 0x40, 0x55, 0x44, - 0x00, 0x60, 0x55, 0x44, 0x00, 0x80, 0x55, 0x44, 0x00, 0xa0, 0x55, 0x44, - 0x00, 0xc0, 0x55, 0x44, 0x00, 0xe0, 0x55, 0x44, 0x00, 0x00, 0x56, 0x44, - 0x00, 0x20, 0x56, 0x44, 0x00, 0x40, 0x56, 0x44, 0x00, 0x60, 0x56, 0x44, - 0x00, 0x80, 0x56, 0x44, 0x00, 0xa0, 0x56, 0x44, 0x00, 0xc0, 0x56, 0x44, - 0x00, 0xe0, 0x56, 0x44, 0x00, 0x00, 0x57, 0x44, 0x00, 0x20, 0x57, 0x44, - 0x00, 0x40, 0x57, 0x44, 0x00, 0x60, 0x57, 0x44, 0x00, 0x80, 0x57, 0x44, - 0x00, 0xa0, 0x57, 0x44, 0x00, 0xc0, 0x57, 0x44, 0x00, 0xe0, 0x57, 0x44, - 0x00, 0x00, 0x58, 0x44, 0x00, 0x20, 0x58, 0x44, 0x00, 0x40, 0x58, 0x44, - 0x00, 0x60, 0x58, 0x44, 0x00, 0x80, 0x58, 0x44, 0x00, 0xa0, 0x58, 0x44, - 0x00, 0xc0, 0x58, 0x44, 0x00, 0xe0, 0x58, 0x44, 0x00, 0x00, 0x59, 0x44, - 0x00, 0x20, 0x59, 0x44, 0x00, 0x40, 0x59, 0x44, 0x00, 0x60, 0x59, 0x44, - 0x00, 0x80, 0x59, 0x44, 0x00, 0xa0, 0x59, 0x44, 0x00, 0xc0, 0x59, 0x44, - 0x00, 0xe0, 0x59, 0x44, 0x00, 0x00, 0x5a, 0x44, 0x00, 0x20, 0x5a, 0x44, - 0x00, 0x40, 0x5a, 0x44, 0x00, 0x60, 0x5a, 0x44, 0x00, 0x80, 0x5a, 0x44, - 0x00, 0xa0, 0x5a, 0x44, 0x00, 0xc0, 0x5a, 0x44, 0x00, 0xe0, 0x5a, 0x44, - 0x00, 0x00, 0x5b, 0x44, 0x00, 0x20, 0x5b, 0x44, 0x00, 0x40, 0x5b, 0x44, - 0x00, 0x60, 0x5b, 0x44, 0x00, 0x80, 0x5b, 0x44, 0x00, 0xa0, 0x5b, 0x44, - 0x00, 0xc0, 0x5b, 0x44, 0x00, 0xe0, 0x5b, 0x44, 0x00, 0x00, 0x5c, 0x44, - 0x00, 0x20, 0x5c, 0x44, 0x00, 0x40, 0x5c, 0x44, 0x00, 0x60, 0x5c, 0x44, - 0x00, 0x80, 0x5c, 0x44, 0x00, 0xa0, 0x5c, 0x44, 0x00, 0xc0, 0x5c, 0x44, - 0x00, 0xe0, 0x5c, 0x44, 0x00, 0x00, 0x5d, 0x44, 0x00, 0x20, 0x5d, 0x44, - 0x00, 0x40, 0x5d, 0x44, 0x00, 0x60, 0x5d, 0x44, 0x00, 0x80, 0x5d, 0x44, - 0x00, 0xa0, 0x5d, 0x44, 0x00, 0xc0, 0x5d, 0x44, 0x00, 0xe0, 0x5d, 0x44, - 0x00, 0x00, 0x5e, 0x44, 0x00, 0x20, 0x5e, 0x44, 0x00, 0x40, 0x5e, 0x44, - 0x00, 0x60, 0x5e, 0x44, 0x00, 0x80, 0x5e, 0x44, 0x00, 0xa0, 0x5e, 0x44, - 0x00, 0xc0, 0x5e, 0x44, 0x00, 0xe0, 0x5e, 0x44, 0x00, 0x00, 0x5f, 0x44, - 0x00, 0x20, 0x5f, 0x44, 0x00, 0x40, 0x5f, 0x44, 0x00, 0x60, 0x5f, 0x44, - 0x00, 0x80, 0x5f, 0x44, 0x00, 0xa0, 0x5f, 0x44, 0x00, 0xc0, 0x5f, 0x44, - 0x00, 0xe0, 0x5f, 0x44, 0x00, 0x00, 0x60, 0x44, 0x00, 0x20, 0x60, 0x44, - 0x00, 0x40, 0x60, 0x44, 0x00, 0x60, 0x60, 0x44, 0x00, 0x80, 0x60, 0x44, - 0x00, 0xa0, 0x60, 0x44, 0x00, 0xc0, 0x60, 0x44, 0x00, 0xe0, 0x60, 0x44, - 0x00, 0x00, 0x61, 0x44, 0x00, 0x20, 0x61, 0x44, 0x00, 0x40, 0x61, 0x44, - 0x00, 0x60, 0x61, 0x44, 0x00, 0x80, 0x61, 0x44, 0x00, 0xa0, 0x61, 0x44, - 0x00, 0xc0, 0x61, 0x44, 0x00, 0xe0, 0x61, 0x44, 0x00, 0x00, 0x62, 0x44, - 0x00, 0x20, 0x62, 0x44, 0x00, 0x40, 0x62, 0x44, 0x00, 0x60, 0x62, 0x44, - 0x00, 0x80, 0x62, 0x44, 0x00, 0xa0, 0x62, 0x44, 0x00, 0xc0, 0x62, 0x44, - 0x00, 0xe0, 0x62, 0x44, 0x00, 0x00, 0x63, 0x44, 0x00, 0x20, 0x63, 0x44, - 0x00, 0x40, 0x63, 0x44, 0x00, 0x60, 0x63, 0x44, 0x00, 0x80, 0x63, 0x44, - 0x00, 0xa0, 0x63, 0x44, 0x00, 0xc0, 0x63, 0x44, 0x00, 0xe0, 0x63, 0x44, - 0x00, 0x00, 0x64, 0x44, 0x00, 0x20, 0x64, 0x44, 0x00, 0x40, 0x64, 0x44, - 0x00, 0x60, 0x64, 0x44, 0x00, 0x80, 0x64, 0x44, 0x00, 0xa0, 0x64, 0x44, - 0x00, 0xc0, 0x64, 0x44, 0x00, 0xe0, 0x64, 0x44, 0x00, 0x00, 0x65, 0x44, - 0x00, 0x20, 0x65, 0x44, 0x00, 0x40, 0x65, 0x44, 0x00, 0x60, 0x65, 0x44, - 0x00, 0x80, 0x65, 0x44, 0x00, 0xa0, 0x65, 0x44, 0x00, 0xc0, 0x65, 0x44, - 0x00, 0xe0, 0x65, 0x44, 0x00, 0x00, 0x66, 0x44, 0x00, 0x20, 0x66, 0x44, - 0x00, 0x40, 0x66, 0x44, 0x00, 0x60, 0x66, 0x44, 0x00, 0x80, 0x66, 0x44, - 0x00, 0xa0, 0x66, 0x44, 0x00, 0xc0, 0x66, 0x44, 0x00, 0xe0, 0x66, 0x44, - 0x00, 0x00, 0x67, 0x44, 0x00, 0x20, 0x67, 0x44, 0x00, 0x40, 0x67, 0x44, - 0x00, 0x60, 0x67, 0x44, 0x00, 0x80, 0x67, 0x44, 0x00, 0xa0, 0x67, 0x44, - 0x00, 0xc0, 0x67, 0x44, 0x00, 0xe0, 0x67, 0x44, 0x00, 0x00, 0x68, 0x44, - 0x00, 0x20, 0x68, 0x44, 0x00, 0x40, 0x68, 0x44, 0x00, 0x60, 0x68, 0x44, - 0x00, 0x80, 0x68, 0x44, 0x00, 0xa0, 0x68, 0x44, 0x00, 0xc0, 0x68, 0x44, - 0x00, 0xe0, 0x68, 0x44, 0x00, 0x00, 0x69, 0x44, 0x00, 0x20, 0x69, 0x44, - 0x00, 0x40, 0x69, 0x44, 0x00, 0x60, 0x69, 0x44, 0x00, 0x80, 0x69, 0x44, - 0x00, 0xa0, 0x69, 0x44, 0x00, 0xc0, 0x69, 0x44, 0x00, 0xe0, 0x69, 0x44, - 0x00, 0x00, 0x6a, 0x44, 0x00, 0x20, 0x6a, 0x44, 0x00, 0x40, 0x6a, 0x44, - 0x00, 0x60, 0x6a, 0x44, 0x00, 0x80, 0x6a, 0x44, 0x00, 0xa0, 0x6a, 0x44, - 0x00, 0xc0, 0x6a, 0x44, 0x00, 0xe0, 0x6a, 0x44, 0x00, 0x00, 0x6b, 0x44, - 0x00, 0x20, 0x6b, 0x44, 0x00, 0x40, 0x6b, 0x44, 0x00, 0x60, 0x6b, 0x44, - 0x00, 0x80, 0x6b, 0x44, 0x00, 0xa0, 0x6b, 0x44, 0x00, 0xc0, 0x6b, 0x44, - 0x00, 0xe0, 0x6b, 0x44, 0x00, 0x00, 0x6c, 0x44, 0x00, 0x20, 0x6c, 0x44, - 0x00, 0x40, 0x6c, 0x44, 0x00, 0x60, 0x6c, 0x44, 0x00, 0x80, 0x6c, 0x44, - 0x00, 0xa0, 0x6c, 0x44, 0x00, 0xc0, 0x6c, 0x44, 0x00, 0xe0, 0x6c, 0x44, - 0x00, 0x00, 0x6d, 0x44, 0x00, 0x20, 0x6d, 0x44, 0x00, 0x40, 0x6d, 0x44, - 0x00, 0x60, 0x6d, 0x44, 0x00, 0x80, 0x6d, 0x44, 0x00, 0xa0, 0x6d, 0x44, - 0x00, 0xc0, 0x6d, 0x44, 0x00, 0xe0, 0x6d, 0x44, 0x00, 0x00, 0x6e, 0x44, - 0x00, 0x20, 0x6e, 0x44, 0x00, 0x40, 0x6e, 0x44, 0x00, 0x60, 0x6e, 0x44, - 0x00, 0x80, 0x6e, 0x44, 0x00, 0xa0, 0x6e, 0x44, 0x00, 0xc0, 0x6e, 0x44, - 0x00, 0xe0, 0x6e, 0x44, 0x00, 0x00, 0x6f, 0x44, 0x00, 0x20, 0x6f, 0x44, - 0x00, 0x40, 0x6f, 0x44, 0x00, 0x60, 0x6f, 0x44, 0x00, 0x80, 0x6f, 0x44, - 0x00, 0xa0, 0x6f, 0x44, 0x00, 0xc0, 0x6f, 0x44, 0x00, 0xe0, 0x6f, 0x44, - 0x00, 0x00, 0x70, 0x44, 0x00, 0x20, 0x70, 0x44, 0x00, 0x40, 0x70, 0x44, - 0x00, 0x60, 0x70, 0x44, 0x00, 0x80, 0x70, 0x44, 0x00, 0xa0, 0x70, 0x44, - 0x00, 0xc0, 0x70, 0x44, 0x00, 0xe0, 0x70, 0x44, 0x00, 0x00, 0x71, 0x44, - 0x00, 0x20, 0x71, 0x44, 0x00, 0x40, 0x71, 0x44, 0x00, 0x60, 0x71, 0x44, - 0x00, 0x80, 0x71, 0x44, 0x00, 0xa0, 0x71, 0x44, 0x00, 0xc0, 0x71, 0x44, - 0x00, 0xe0, 0x71, 0x44, 0x00, 0x00, 0x72, 0x44, 0x00, 0x20, 0x72, 0x44, - 0x00, 0x40, 0x72, 0x44, 0x00, 0x60, 0x72, 0x44, 0x00, 0x80, 0x72, 0x44, - 0x00, 0xa0, 0x72, 0x44, 0x00, 0xc0, 0x72, 0x44, 0x00, 0xe0, 0x72, 0x44, - 0x00, 0x00, 0x73, 0x44, 0x00, 0x20, 0x73, 0x44, 0x00, 0x40, 0x73, 0x44, - 0x00, 0x60, 0x73, 0x44, 0x00, 0x80, 0x73, 0x44, 0x00, 0xa0, 0x73, 0x44, - 0x00, 0xc0, 0x73, 0x44, 0x00, 0xe0, 0x73, 0x44, 0x00, 0x00, 0x74, 0x44, - 0x00, 0x20, 0x74, 0x44, 0x00, 0x40, 0x74, 0x44, 0x00, 0x60, 0x74, 0x44, - 0x00, 0x80, 0x74, 0x44, 0x00, 0xa0, 0x74, 0x44, 0x00, 0xc0, 0x74, 0x44, - 0x00, 0xe0, 0x74, 0x44, 0x00, 0x00, 0x75, 0x44, 0x00, 0x20, 0x75, 0x44, - 0x00, 0x40, 0x75, 0x44, 0x00, 0x60, 0x75, 0x44, 0x00, 0x80, 0x75, 0x44, - 0x00, 0xa0, 0x75, 0x44, 0x00, 0xc0, 0x75, 0x44, 0x00, 0xe0, 0x75, 0x44, - 0x00, 0x00, 0x76, 0x44, 0x00, 0x20, 0x76, 0x44, 0x00, 0x40, 0x76, 0x44, - 0x00, 0x60, 0x76, 0x44, 0x00, 0x80, 0x76, 0x44, 0x00, 0xa0, 0x76, 0x44, - 0x00, 0xc0, 0x76, 0x44, 0x00, 0xe0, 0x76, 0x44, 0x00, 0x00, 0x77, 0x44, - 0x00, 0x20, 0x77, 0x44, 0x00, 0x40, 0x77, 0x44, 0x00, 0x60, 0x77, 0x44, - 0x00, 0x80, 0x77, 0x44, 0x00, 0xa0, 0x77, 0x44, 0x00, 0xc0, 0x77, 0x44, - 0x00, 0xe0, 0x77, 0x44, 0x00, 0x00, 0x78, 0x44, 0x00, 0x20, 0x78, 0x44, - 0x00, 0x40, 0x78, 0x44, 0x00, 0x60, 0x78, 0x44, 0x00, 0x80, 0x78, 0x44, - 0x00, 0xa0, 0x78, 0x44, 0x00, 0xc0, 0x78, 0x44, 0x00, 0xe0, 0x78, 0x44, - 0x00, 0x00, 0x79, 0x44, 0x00, 0x20, 0x79, 0x44, 0x00, 0x40, 0x79, 0x44, - 0x00, 0x60, 0x79, 0x44, 0x00, 0x80, 0x79, 0x44, 0x00, 0xa0, 0x79, 0x44, - 0x00, 0xc0, 0x79, 0x44, 0x00, 0xe0, 0x79, 0x44, 0x00, 0x00, 0x7a, 0x44, - 0x00, 0x20, 0x7a, 0x44, 0x00, 0x40, 0x7a, 0x44, 0x00, 0x60, 0x7a, 0x44, - 0x00, 0x80, 0x7a, 0x44, 0x00, 0xa0, 0x7a, 0x44, 0x00, 0xc0, 0x7a, 0x44, - 0x00, 0xe0, 0x7a, 0x44, 0x00, 0x00, 0x7b, 0x44, 0x00, 0x20, 0x7b, 0x44, - 0x00, 0x40, 0x7b, 0x44, 0x00, 0x60, 0x7b, 0x44, 0x00, 0x80, 0x7b, 0x44, - 0x00, 0xa0, 0x7b, 0x44, 0x00, 0xc0, 0x7b, 0x44, 0x00, 0xe0, 0x7b, 0x44, - 0x00, 0x00, 0x7c, 0x44, 0x00, 0x20, 0x7c, 0x44, 0x00, 0x40, 0x7c, 0x44, - 0x00, 0x60, 0x7c, 0x44, 0x00, 0x80, 0x7c, 0x44, 0x00, 0xa0, 0x7c, 0x44, - 0x00, 0xc0, 0x7c, 0x44, 0x00, 0xe0, 0x7c, 0x44, 0x00, 0x00, 0x7d, 0x44, - 0x00, 0x20, 0x7d, 0x44, 0x00, 0x40, 0x7d, 0x44 -}; + 0x00, 0x00, 0x28, 0xc1, 0x00, 0x00, 0x20, 0xc1, 0x00, 0x00, 0x18, 0xc1, 0x00, 0x00, 0x10, 0xc1, 0x00, + 0x00, 0x08, 0xc1, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0xf0, 0xc0, 0x00, 0x00, 0xe0, 0xc0, 0x00, 0x00, + 0xd0, 0xc0, 0x00, 0x00, 0xc0, 0xc0, 0x00, 0x00, 0xb0, 0xc0, 0x00, 0x00, 0xa0, 0xc0, 0x00, 0x00, 0x90, + 0xc0, 0x00, 0x00, 0x80, 0xc0, 0x00, 0x00, 0x60, 0xc0, 0x00, 0x00, 0x40, 0xc0, 0x00, 0x00, 0x20, 0xc0, + 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0xc0, 0xbf, 0x00, 0x00, 0x80, 0xbf, 0x00, 0x00, 0x00, 0xbf, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0xc0, 0x3f, 0x00, 0x00, + 0x00, 0x40, 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x60, 0x40, 0x00, 0x00, 0x80, + 0x40, 0x00, 0x00, 0x90, 0x40, 0x00, 0x00, 0xa0, 0x40, 0x00, 0x00, 0xb0, 0x40, 0x00, 0x00, 0xc0, 0x40, + 0x00, 0x00, 0xd0, 0x40, 0x00, 0x00, 0xe0, 0x40, 0x00, 0x00, 0xf0, 0x40, 0x00, 0x00, 0x00, 0x41, 0x00, + 0x00, 0x08, 0x41, 0x00, 0x00, 0x10, 0x41, 0x00, 0x00, 0x18, 0x41, 0x00, 0x00, 0x20, 0x41, 0x00, 0x00, + 0x28, 0x41, 0x00, 0x00, 0x30, 0x41, 0x00, 0x00, 0x38, 0x41, 0x00, 0x00, 0x40, 0x41, 0x00, 0x00, 0x48, + 0x41, 0x00, 0x00, 0x50, 0x41, 0x00, 0x00, 0x58, 0x41, 0x00, 0x00, 0x60, 0x41, 0x00, 0x00, 0x68, 0x41, + 0x00, 0x00, 0x70, 0x41, 0x00, 0x00, 0x78, 0x41, 0x00, 0x00, 0x80, 0x41, 0x00, 0x00, 0x84, 0x41, 0x00, + 0x00, 0x88, 0x41, 0x00, 0x00, 0x8c, 0x41, 0x00, 0x00, 0x90, 0x41, 0x00, 0x00, 0x94, 0x41, 0x00, 0x00, + 0x98, 0x41, 0x00, 0x00, 0x9c, 0x41, 0x00, 0x00, 0xa0, 0x41, 0x00, 0x00, 0xa4, 0x41, 0x00, 0x00, 0xa8, + 0x41, 0x00, 0x00, 0xac, 0x41, 0x00, 0x00, 0xb0, 0x41, 0x00, 0x00, 0xb4, 0x41, 0x00, 0x00, 0xb8, 0x41, + 0x00, 0x00, 0xbc, 0x41, 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xc4, 0x41, 0x00, 0x00, 0xc8, 0x41, 0x00, + 0x00, 0xcc, 0x41, 0x00, 0x00, 0xd0, 0x41, 0x00, 0x00, 0xd4, 0x41, 0x00, 0x00, 0xd8, 0x41, 0x00, 0x00, + 0xdc, 0x41, 0x00, 0x00, 0xe0, 0x41, 0x00, 0x00, 0xe4, 0x41, 0x00, 0x00, 0xe8, 0x41, 0x00, 0x00, 0xec, + 0x41, 0x00, 0x00, 0xf0, 0x41, 0x00, 0x00, 0xf4, 0x41, 0x00, 0x00, 0xf8, 0x41, 0x00, 0x00, 0xfc, 0x41, + 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x02, 0x42, 0x00, 0x00, 0x04, 0x42, 0x00, 0x00, 0x06, 0x42, 0x00, + 0x00, 0x08, 0x42, 0x00, 0x00, 0x0a, 0x42, 0x00, 0x00, 0x0c, 0x42, 0x00, 0x00, 0x0e, 0x42, 0x00, 0x00, + 0x10, 0x42, 0x00, 0x00, 0x12, 0x42, 0x00, 0x00, 0x14, 0x42, 0x00, 0x00, 0x16, 0x42, 0x00, 0x00, 0x18, + 0x42, 0x00, 0x00, 0x1a, 0x42, 0x00, 0x00, 0x1c, 0x42, 0x00, 0x00, 0x1e, 0x42, 0x00, 0x00, 0x20, 0x42, + 0x00, 0x00, 0x22, 0x42, 0x00, 0x00, 0x24, 0x42, 0x00, 0x00, 0x26, 0x42, 0x00, 0x00, 0x28, 0x42, 0x00, + 0x00, 0x2a, 0x42, 0x00, 0x00, 0x2c, 0x42, 0x00, 0x00, 0x2e, 0x42, 0x00, 0x00, 0x30, 0x42, 0x00, 0x00, + 0x32, 0x42, 0x00, 0x00, 0x34, 0x42, 0x00, 0x00, 0x36, 0x42, 0x00, 0x00, 0x38, 0x42, 0x00, 0x00, 0x3a, + 0x42, 0x00, 0x00, 0x3c, 0x42, 0x00, 0x00, 0x3e, 0x42, 0x00, 0x00, 0x40, 0x42, 0x00, 0x00, 0x42, 0x42, + 0x00, 0x00, 0x44, 0x42, 0x00, 0x00, 0x46, 0x42, 0x00, 0x00, 0x48, 0x42, 0x00, 0x00, 0x4a, 0x42, 0x00, + 0x00, 0x4c, 0x42, 0x00, 0x00, 0x4e, 0x42, 0x00, 0x00, 0x50, 0x42, 0x00, 0x00, 0x52, 0x42, 0x00, 0x00, + 0x54, 0x42, 0x00, 0x00, 0x56, 0x42, 0x00, 0x00, 0x58, 0x42, 0x00, 0x00, 0x5a, 0x42, 0x00, 0x00, 0x5c, + 0x42, 0x00, 0x00, 0x5e, 0x42, 0x00, 0x00, 0x60, 0x42, 0x00, 0x00, 0x62, 0x42, 0x00, 0x00, 0x64, 0x42, + 0x00, 0x00, 0x66, 0x42, 0x00, 0x00, 0x68, 0x42, 0x00, 0x00, 0x6a, 0x42, 0x00, 0x00, 0x6c, 0x42, 0x00, + 0x00, 0x6e, 0x42, 0x00, 0x00, 0x70, 0x42, 0x00, 0x00, 0x72, 0x42, 0x00, 0x00, 0x74, 0x42, 0x00, 0x00, + 0x76, 0x42, 0x00, 0x00, 0x78, 0x42, 0x00, 0x00, 0x7a, 0x42, 0x00, 0x00, 0x7c, 0x42, 0x00, 0x00, 0x7e, + 0x42, 0x00, 0x00, 0x80, 0x42, 0x00, 0x00, 0x81, 0x42, 0x00, 0x00, 0x82, 0x42, 0x00, 0x00, 0x83, 0x42, + 0x00, 0x00, 0x84, 0x42, 0x00, 0x00, 0x85, 0x42, 0x00, 0x00, 0x86, 0x42, 0x00, 0x00, 0x87, 0x42, 0x00, + 0x00, 0x88, 0x42, 0x00, 0x00, 0x89, 0x42, 0x00, 0x00, 0x8a, 0x42, 0x00, 0x00, 0x8b, 0x42, 0x00, 0x00, + 0x8c, 0x42, 0x00, 0x00, 0x8d, 0x42, 0x00, 0x00, 0x8e, 0x42, 0x00, 0x00, 0x8f, 0x42, 0x00, 0x00, 0x90, + 0x42, 0x00, 0x00, 0x91, 0x42, 0x00, 0x00, 0x92, 0x42, 0x00, 0x00, 0x93, 0x42, 0x00, 0x00, 0x94, 0x42, + 0x00, 0x00, 0x95, 0x42, 0x00, 0x00, 0x96, 0x42, 0x00, 0x00, 0x97, 0x42, 0x00, 0x00, 0x98, 0x42, 0x00, + 0x00, 0x99, 0x42, 0x00, 0x00, 0x9a, 0x42, 0x00, 0x00, 0x9b, 0x42, 0x00, 0x00, 0x9c, 0x42, 0x00, 0x00, + 0x9d, 0x42, 0x00, 0x00, 0x9e, 0x42, 0x00, 0x00, 0x9f, 0x42, 0x00, 0x00, 0xa0, 0x42, 0x00, 0x00, 0xa1, + 0x42, 0x00, 0x00, 0xa2, 0x42, 0x00, 0x00, 0xa3, 0x42, 0x00, 0x00, 0xa4, 0x42, 0x00, 0x00, 0xa5, 0x42, + 0x00, 0x00, 0xa6, 0x42, 0x00, 0x00, 0xa7, 0x42, 0x00, 0x00, 0xa8, 0x42, 0x00, 0x00, 0xa9, 0x42, 0x00, + 0x00, 0xaa, 0x42, 0x00, 0x00, 0xab, 0x42, 0x00, 0x00, 0xac, 0x42, 0x00, 0x00, 0xad, 0x42, 0x00, 0x00, + 0xae, 0x42, 0x00, 0x00, 0xaf, 0x42, 0x00, 0x00, 0xb0, 0x42, 0x00, 0x00, 0xb1, 0x42, 0x00, 0x00, 0xb2, + 0x42, 0x00, 0x00, 0xb3, 0x42, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x00, 0xb5, 0x42, 0x00, 0x00, 0xb6, 0x42, + 0x00, 0x00, 0xb7, 0x42, 0x00, 0x00, 0xb8, 0x42, 0x00, 0x00, 0xb9, 0x42, 0x00, 0x00, 0xba, 0x42, 0x00, + 0x00, 0xbb, 0x42, 0x00, 0x00, 0xbc, 0x42, 0x00, 0x00, 0xbd, 0x42, 0x00, 0x00, 0xbe, 0x42, 0x00, 0x00, + 0xbf, 0x42, 0x00, 0x00, 0xc0, 0x42, 0x00, 0x00, 0xc1, 0x42, 0x00, 0x00, 0xc2, 0x42, 0x00, 0x00, 0xc3, + 0x42, 0x00, 0x00, 0xc4, 0x42, 0x00, 0x00, 0xc5, 0x42, 0x00, 0x00, 0xc6, 0x42, 0x00, 0x00, 0xc7, 0x42, + 0x00, 0x00, 0xc8, 0x42, 0x00, 0x00, 0xc9, 0x42, 0x00, 0x00, 0xca, 0x42, 0x00, 0x00, 0xcb, 0x42, 0x00, + 0x00, 0xcc, 0x42, 0x00, 0x00, 0xcd, 0x42, 0x00, 0x00, 0xce, 0x42, 0x00, 0x00, 0xcf, 0x42, 0x00, 0x00, + 0xd0, 0x42, 0x00, 0x00, 0xd1, 0x42, 0x00, 0x00, 0xd2, 0x42, 0x00, 0x00, 0xd3, 0x42, 0x00, 0x00, 0xd4, + 0x42, 0x00, 0x00, 0xd5, 0x42, 0x00, 0x00, 0xd6, 0x42, 0x00, 0x00, 0xd7, 0x42, 0x00, 0x00, 0xd8, 0x42, + 0x00, 0x00, 0xd9, 0x42, 0x00, 0x00, 0xda, 0x42, 0x00, 0x00, 0xdb, 0x42, 0x00, 0x00, 0xdc, 0x42, 0x00, + 0x00, 0xdd, 0x42, 0x00, 0x00, 0xde, 0x42, 0x00, 0x00, 0xdf, 0x42, 0x00, 0x00, 0xe0, 0x42, 0x00, 0x00, + 0xe1, 0x42, 0x00, 0x00, 0xe2, 0x42, 0x00, 0x00, 0xe3, 0x42, 0x00, 0x00, 0xe4, 0x42, 0x00, 0x00, 0xe5, + 0x42, 0x00, 0x00, 0xe6, 0x42, 0x00, 0x00, 0xe7, 0x42, 0x00, 0x00, 0xe8, 0x42, 0x00, 0x00, 0xe9, 0x42, + 0x00, 0x00, 0xea, 0x42, 0x00, 0x00, 0xeb, 0x42, 0x00, 0x00, 0xec, 0x42, 0x00, 0x00, 0xed, 0x42, 0x00, + 0x00, 0xee, 0x42, 0x00, 0x00, 0xef, 0x42, 0x00, 0x00, 0xf0, 0x42, 0x00, 0x00, 0xf1, 0x42, 0x00, 0x00, + 0xf2, 0x42, 0x00, 0x00, 0xf3, 0x42, 0x00, 0x00, 0xf4, 0x42, 0x00, 0x00, 0xf5, 0x42, 0x00, 0x00, 0xf6, + 0x42, 0x00, 0x00, 0xf7, 0x42, 0x00, 0x00, 0xf8, 0x42, 0x00, 0x00, 0xf9, 0x42, 0x00, 0x00, 0xfa, 0x42, + 0x00, 0x00, 0xfb, 0x42, 0x00, 0x00, 0xfc, 0x42, 0x00, 0x00, 0xfd, 0x42, 0x00, 0x00, 0xfe, 0x42, 0x00, + 0x00, 0xff, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x80, 0x00, 0x43, 0x00, 0x00, 0x01, 0x43, 0x00, 0x80, + 0x01, 0x43, 0x00, 0x00, 0x02, 0x43, 0x00, 0x80, 0x02, 0x43, 0x00, 0x00, 0x03, 0x43, 0x00, 0x80, 0x03, + 0x43, 0x00, 0x00, 0x04, 0x43, 0x00, 0x80, 0x04, 0x43, 0x00, 0x00, 0x05, 0x43, 0x00, 0x80, 0x05, 0x43, + 0x00, 0x00, 0x06, 0x43, 0x00, 0x80, 0x06, 0x43, 0x00, 0x00, 0x07, 0x43, 0x00, 0x80, 0x07, 0x43, 0x00, + 0x00, 0x08, 0x43, 0x00, 0x80, 0x08, 0x43, 0x00, 0x00, 0x09, 0x43, 0x00, 0x80, 0x09, 0x43, 0x00, 0x00, + 0x0a, 0x43, 0x00, 0x80, 0x0a, 0x43, 0x00, 0x00, 0x0b, 0x43, 0x00, 0x80, 0x0b, 0x43, 0x00, 0x00, 0x0c, + 0x43, 0x00, 0x80, 0x0c, 0x43, 0x00, 0x00, 0x0d, 0x43, 0x00, 0x80, 0x0d, 0x43, 0x00, 0x00, 0x0e, 0x43, + 0x00, 0x80, 0x0e, 0x43, 0x00, 0x00, 0x0f, 0x43, 0x00, 0x80, 0x0f, 0x43, 0x00, 0x00, 0x10, 0x43, 0x00, + 0x80, 0x10, 0x43, 0x00, 0x00, 0x11, 0x43, 0x00, 0x80, 0x11, 0x43, 0x00, 0x00, 0x12, 0x43, 0x00, 0x80, + 0x12, 0x43, 0x00, 0x00, 0x13, 0x43, 0x00, 0x80, 0x13, 0x43, 0x00, 0x00, 0x14, 0x43, 0x00, 0x80, 0x14, + 0x43, 0x00, 0x00, 0x15, 0x43, 0x00, 0x80, 0x15, 0x43, 0x00, 0x00, 0x16, 0x43, 0x00, 0x80, 0x16, 0x43, + 0x00, 0x00, 0x17, 0x43, 0x00, 0x80, 0x17, 0x43, 0x00, 0x00, 0x18, 0x43, 0x00, 0x80, 0x18, 0x43, 0x00, + 0x00, 0x19, 0x43, 0x00, 0x80, 0x19, 0x43, 0x00, 0x00, 0x1a, 0x43, 0x00, 0x80, 0x1a, 0x43, 0x00, 0x00, + 0x1b, 0x43, 0x00, 0x80, 0x1b, 0x43, 0x00, 0x00, 0x1c, 0x43, 0x00, 0x80, 0x1c, 0x43, 0x00, 0x00, 0x1d, + 0x43, 0x00, 0x80, 0x1d, 0x43, 0x00, 0x00, 0x1e, 0x43, 0x00, 0x80, 0x1e, 0x43, 0x00, 0x00, 0x1f, 0x43, + 0x00, 0x80, 0x1f, 0x43, 0x00, 0x00, 0x20, 0x43, 0x00, 0x80, 0x20, 0x43, 0x00, 0x00, 0x21, 0x43, 0x00, + 0x80, 0x21, 0x43, 0x00, 0x00, 0x22, 0x43, 0x00, 0x80, 0x22, 0x43, 0x00, 0x00, 0x23, 0x43, 0x00, 0x80, + 0x23, 0x43, 0x00, 0x00, 0x24, 0x43, 0x00, 0x80, 0x24, 0x43, 0x00, 0x00, 0x25, 0x43, 0x00, 0x80, 0x25, + 0x43, 0x00, 0x00, 0x26, 0x43, 0x00, 0x80, 0x26, 0x43, 0x00, 0x00, 0x27, 0x43, 0x00, 0x80, 0x27, 0x43, + 0x00, 0x00, 0x28, 0x43, 0x00, 0x80, 0x28, 0x43, 0x00, 0x00, 0x29, 0x43, 0x00, 0x80, 0x29, 0x43, 0x00, + 0x00, 0x2a, 0x43, 0x00, 0x80, 0x2a, 0x43, 0x00, 0x00, 0x2b, 0x43, 0x00, 0x80, 0x2b, 0x43, 0x00, 0x00, + 0x2c, 0x43, 0x00, 0x80, 0x2c, 0x43, 0x00, 0x00, 0x2d, 0x43, 0x00, 0x80, 0x2d, 0x43, 0x00, 0x00, 0x2e, + 0x43, 0x00, 0x80, 0x2e, 0x43, 0x00, 0x00, 0x2f, 0x43, 0x00, 0x80, 0x2f, 0x43, 0x00, 0x00, 0x30, 0x43, + 0x00, 0x80, 0x30, 0x43, 0x00, 0x00, 0x31, 0x43, 0x00, 0x80, 0x31, 0x43, 0x00, 0x00, 0x32, 0x43, 0x00, + 0x80, 0x32, 0x43, 0x00, 0x00, 0x33, 0x43, 0x00, 0x80, 0x33, 0x43, 0x00, 0x00, 0x34, 0x43, 0x00, 0x80, + 0x34, 0x43, 0x00, 0x00, 0x35, 0x43, 0x00, 0x80, 0x35, 0x43, 0x00, 0x00, 0x36, 0x43, 0x00, 0x80, 0x36, + 0x43, 0x00, 0x00, 0x37, 0x43, 0x00, 0x80, 0x37, 0x43, 0x00, 0x00, 0x38, 0x43, 0x00, 0x80, 0x38, 0x43, + 0x00, 0x00, 0x39, 0x43, 0x00, 0x80, 0x39, 0x43, 0x00, 0x00, 0x3a, 0x43, 0x00, 0x80, 0x3a, 0x43, 0x00, + 0x00, 0x3b, 0x43, 0x00, 0x80, 0x3b, 0x43, 0x00, 0x00, 0x3c, 0x43, 0x00, 0x80, 0x3c, 0x43, 0x00, 0x00, + 0x3d, 0x43, 0x00, 0x80, 0x3d, 0x43, 0x00, 0x00, 0x3e, 0x43, 0x00, 0x80, 0x3e, 0x43, 0x00, 0x00, 0x3f, + 0x43, 0x00, 0x80, 0x3f, 0x43, 0x00, 0x00, 0x40, 0x43, 0x00, 0x80, 0x40, 0x43, 0x00, 0x00, 0x41, 0x43, + 0x00, 0x80, 0x41, 0x43, 0x00, 0x00, 0x42, 0x43, 0x00, 0x80, 0x42, 0x43, 0x00, 0x00, 0x43, 0x43, 0x00, + 0x80, 0x43, 0x43, 0x00, 0x00, 0x44, 0x43, 0x00, 0x80, 0x44, 0x43, 0x00, 0x00, 0x45, 0x43, 0x00, 0x80, + 0x45, 0x43, 0x00, 0x00, 0x46, 0x43, 0x00, 0x80, 0x46, 0x43, 0x00, 0x00, 0x47, 0x43, 0x00, 0x80, 0x47, + 0x43, 0x00, 0x00, 0x48, 0x43, 0x00, 0x80, 0x48, 0x43, 0x00, 0x00, 0x49, 0x43, 0x00, 0x80, 0x49, 0x43, + 0x00, 0x00, 0x4a, 0x43, 0x00, 0x80, 0x4a, 0x43, 0x00, 0x00, 0x4b, 0x43, 0x00, 0x80, 0x4b, 0x43, 0x00, + 0x00, 0x4c, 0x43, 0x00, 0x80, 0x4c, 0x43, 0x00, 0x00, 0x4d, 0x43, 0x00, 0x80, 0x4d, 0x43, 0x00, 0x00, + 0x4e, 0x43, 0x00, 0x80, 0x4e, 0x43, 0x00, 0x00, 0x4f, 0x43, 0x00, 0x80, 0x4f, 0x43, 0x00, 0x00, 0x50, + 0x43, 0x00, 0x80, 0x50, 0x43, 0x00, 0x00, 0x51, 0x43, 0x00, 0x80, 0x51, 0x43, 0x00, 0x00, 0x52, 0x43, + 0x00, 0x80, 0x52, 0x43, 0x00, 0x00, 0x53, 0x43, 0x00, 0x80, 0x53, 0x43, 0x00, 0x00, 0x54, 0x43, 0x00, + 0x80, 0x54, 0x43, 0x00, 0x00, 0x55, 0x43, 0x00, 0x80, 0x55, 0x43, 0x00, 0x00, 0x56, 0x43, 0x00, 0x80, + 0x56, 0x43, 0x00, 0x00, 0x57, 0x43, 0x00, 0x80, 0x57, 0x43, 0x00, 0x00, 0x58, 0x43, 0x00, 0x80, 0x58, + 0x43, 0x00, 0x00, 0x59, 0x43, 0x00, 0x80, 0x59, 0x43, 0x00, 0x00, 0x5a, 0x43, 0x00, 0x80, 0x5a, 0x43, + 0x00, 0x00, 0x5b, 0x43, 0x00, 0x80, 0x5b, 0x43, 0x00, 0x00, 0x5c, 0x43, 0x00, 0x80, 0x5c, 0x43, 0x00, + 0x00, 0x5d, 0x43, 0x00, 0x80, 0x5d, 0x43, 0x00, 0x00, 0x5e, 0x43, 0x00, 0x80, 0x5e, 0x43, 0x00, 0x00, + 0x5f, 0x43, 0x00, 0x80, 0x5f, 0x43, 0x00, 0x00, 0x60, 0x43, 0x00, 0x80, 0x60, 0x43, 0x00, 0x00, 0x61, + 0x43, 0x00, 0x80, 0x61, 0x43, 0x00, 0x00, 0x62, 0x43, 0x00, 0x80, 0x62, 0x43, 0x00, 0x00, 0x63, 0x43, + 0x00, 0x80, 0x63, 0x43, 0x00, 0x00, 0x64, 0x43, 0x00, 0x80, 0x64, 0x43, 0x00, 0x00, 0x65, 0x43, 0x00, + 0x80, 0x65, 0x43, 0x00, 0x00, 0x66, 0x43, 0x00, 0x80, 0x66, 0x43, 0x00, 0x00, 0x67, 0x43, 0x00, 0x80, + 0x67, 0x43, 0x00, 0x00, 0x68, 0x43, 0x00, 0x80, 0x68, 0x43, 0x00, 0x00, 0x69, 0x43, 0x00, 0x80, 0x69, + 0x43, 0x00, 0x00, 0x6a, 0x43, 0x00, 0x80, 0x6a, 0x43, 0x00, 0x00, 0x6b, 0x43, 0x00, 0x80, 0x6b, 0x43, + 0x00, 0x00, 0x6c, 0x43, 0x00, 0x80, 0x6c, 0x43, 0x00, 0x00, 0x6d, 0x43, 0x00, 0x80, 0x6d, 0x43, 0x00, + 0x00, 0x6e, 0x43, 0x00, 0x80, 0x6e, 0x43, 0x00, 0x00, 0x6f, 0x43, 0x00, 0x80, 0x6f, 0x43, 0x00, 0x00, + 0x70, 0x43, 0x00, 0x80, 0x70, 0x43, 0x00, 0x00, 0x71, 0x43, 0x00, 0x80, 0x71, 0x43, 0x00, 0x00, 0x72, + 0x43, 0x00, 0x80, 0x72, 0x43, 0x00, 0x00, 0x73, 0x43, 0x00, 0x80, 0x73, 0x43, 0x00, 0x00, 0x74, 0x43, + 0x00, 0x80, 0x74, 0x43, 0x00, 0x00, 0x75, 0x43, 0x00, 0x80, 0x75, 0x43, 0x00, 0x00, 0x76, 0x43, 0x00, + 0x80, 0x76, 0x43, 0x00, 0x00, 0x77, 0x43, 0x00, 0x80, 0x77, 0x43, 0x00, 0x00, 0x78, 0x43, 0x00, 0x80, + 0x78, 0x43, 0x00, 0x00, 0x79, 0x43, 0x00, 0x80, 0x79, 0x43, 0x00, 0x00, 0x7a, 0x43, 0x00, 0x80, 0x7a, + 0x43, 0x00, 0x00, 0x7b, 0x43, 0x00, 0x80, 0x7b, 0x43, 0x00, 0x00, 0x7c, 0x43, 0x00, 0x80, 0x7c, 0x43, + 0x00, 0x00, 0x7d, 0x43, 0x00, 0x80, 0x7d, 0x43, 0x00, 0x00, 0x7e, 0x43, 0x00, 0x80, 0x7e, 0x43, 0x00, + 0x00, 0x7f, 0x43, 0x00, 0x80, 0x7f, 0x43, 0x00, 0x00, 0x80, 0x43, 0x00, 0x40, 0x80, 0x43, 0x00, 0x80, + 0x80, 0x43, 0x00, 0xc0, 0x80, 0x43, 0x00, 0x00, 0x81, 0x43, 0x00, 0x40, 0x81, 0x43, 0x00, 0x80, 0x81, + 0x43, 0x00, 0xc0, 0x81, 0x43, 0x00, 0x00, 0x82, 0x43, 0x00, 0x40, 0x82, 0x43, 0x00, 0x80, 0x82, 0x43, + 0x00, 0xc0, 0x82, 0x43, 0x00, 0x00, 0x83, 0x43, 0x00, 0x40, 0x83, 0x43, 0x00, 0x80, 0x83, 0x43, 0x00, + 0xc0, 0x83, 0x43, 0x00, 0x00, 0x84, 0x43, 0x00, 0x40, 0x84, 0x43, 0x00, 0x80, 0x84, 0x43, 0x00, 0xc0, + 0x84, 0x43, 0x00, 0x00, 0x85, 0x43, 0x00, 0x40, 0x85, 0x43, 0x00, 0x80, 0x85, 0x43, 0x00, 0xc0, 0x85, + 0x43, 0x00, 0x00, 0x86, 0x43, 0x00, 0x40, 0x86, 0x43, 0x00, 0x80, 0x86, 0x43, 0x00, 0xc0, 0x86, 0x43, + 0x00, 0x00, 0x87, 0x43, 0x00, 0x40, 0x87, 0x43, 0x00, 0x80, 0x87, 0x43, 0x00, 0xc0, 0x87, 0x43, 0x00, + 0x00, 0x88, 0x43, 0x00, 0x40, 0x88, 0x43, 0x00, 0x80, 0x88, 0x43, 0x00, 0xc0, 0x88, 0x43, 0x00, 0x00, + 0x89, 0x43, 0x00, 0x40, 0x89, 0x43, 0x00, 0x80, 0x89, 0x43, 0x00, 0xc0, 0x89, 0x43, 0x00, 0x00, 0x8a, + 0x43, 0x00, 0x40, 0x8a, 0x43, 0x00, 0x80, 0x8a, 0x43, 0x00, 0xc0, 0x8a, 0x43, 0x00, 0x00, 0x8b, 0x43, + 0x00, 0x40, 0x8b, 0x43, 0x00, 0x80, 0x8b, 0x43, 0x00, 0xc0, 0x8b, 0x43, 0x00, 0x00, 0x8c, 0x43, 0x00, + 0x40, 0x8c, 0x43, 0x00, 0x80, 0x8c, 0x43, 0x00, 0xc0, 0x8c, 0x43, 0x00, 0x00, 0x8d, 0x43, 0x00, 0x40, + 0x8d, 0x43, 0x00, 0x80, 0x8d, 0x43, 0x00, 0xc0, 0x8d, 0x43, 0x00, 0x00, 0x8e, 0x43, 0x00, 0x40, 0x8e, + 0x43, 0x00, 0x80, 0x8e, 0x43, 0x00, 0xc0, 0x8e, 0x43, 0x00, 0x00, 0x8f, 0x43, 0x00, 0x40, 0x8f, 0x43, + 0x00, 0x80, 0x8f, 0x43, 0x00, 0xc0, 0x8f, 0x43, 0x00, 0x00, 0x90, 0x43, 0x00, 0x40, 0x90, 0x43, 0x00, + 0x80, 0x90, 0x43, 0x00, 0xc0, 0x90, 0x43, 0x00, 0x00, 0x91, 0x43, 0x00, 0x40, 0x91, 0x43, 0x00, 0x80, + 0x91, 0x43, 0x00, 0xc0, 0x91, 0x43, 0x00, 0x00, 0x92, 0x43, 0x00, 0x40, 0x92, 0x43, 0x00, 0x80, 0x92, + 0x43, 0x00, 0xc0, 0x92, 0x43, 0x00, 0x00, 0x93, 0x43, 0x00, 0x40, 0x93, 0x43, 0x00, 0x80, 0x93, 0x43, + 0x00, 0xc0, 0x93, 0x43, 0x00, 0x00, 0x94, 0x43, 0x00, 0x40, 0x94, 0x43, 0x00, 0x80, 0x94, 0x43, 0x00, + 0xc0, 0x94, 0x43, 0x00, 0x00, 0x95, 0x43, 0x00, 0x40, 0x95, 0x43, 0x00, 0x80, 0x95, 0x43, 0x00, 0xc0, + 0x95, 0x43, 0x00, 0x00, 0x96, 0x43, 0x00, 0x40, 0x96, 0x43, 0x00, 0x80, 0x96, 0x43, 0x00, 0xc0, 0x96, + 0x43, 0x00, 0x00, 0x97, 0x43, 0x00, 0x40, 0x97, 0x43, 0x00, 0x80, 0x97, 0x43, 0x00, 0xc0, 0x97, 0x43, + 0x00, 0x00, 0x98, 0x43, 0x00, 0x40, 0x98, 0x43, 0x00, 0x80, 0x98, 0x43, 0x00, 0xc0, 0x98, 0x43, 0x00, + 0x00, 0x99, 0x43, 0x00, 0x40, 0x99, 0x43, 0x00, 0x80, 0x99, 0x43, 0x00, 0xc0, 0x99, 0x43, 0x00, 0x00, + 0x9a, 0x43, 0x00, 0x40, 0x9a, 0x43, 0x00, 0x80, 0x9a, 0x43, 0x00, 0xc0, 0x9a, 0x43, 0x00, 0x00, 0x9b, + 0x43, 0x00, 0x40, 0x9b, 0x43, 0x00, 0x80, 0x9b, 0x43, 0x00, 0xc0, 0x9b, 0x43, 0x00, 0x00, 0x9c, 0x43, + 0x00, 0x40, 0x9c, 0x43, 0x00, 0x80, 0x9c, 0x43, 0x00, 0xc0, 0x9c, 0x43, 0x00, 0x00, 0x9d, 0x43, 0x00, + 0x40, 0x9d, 0x43, 0x00, 0x80, 0x9d, 0x43, 0x00, 0xc0, 0x9d, 0x43, 0x00, 0x00, 0x9e, 0x43, 0x00, 0x40, + 0x9e, 0x43, 0x00, 0x80, 0x9e, 0x43, 0x00, 0xc0, 0x9e, 0x43, 0x00, 0x00, 0x9f, 0x43, 0x00, 0x40, 0x9f, + 0x43, 0x00, 0x80, 0x9f, 0x43, 0x00, 0xc0, 0x9f, 0x43, 0x00, 0x00, 0xa0, 0x43, 0x00, 0x40, 0xa0, 0x43, + 0x00, 0x80, 0xa0, 0x43, 0x00, 0xc0, 0xa0, 0x43, 0x00, 0x00, 0xa1, 0x43, 0x00, 0x40, 0xa1, 0x43, 0x00, + 0x80, 0xa1, 0x43, 0x00, 0xc0, 0xa1, 0x43, 0x00, 0x00, 0xa2, 0x43, 0x00, 0x40, 0xa2, 0x43, 0x00, 0x80, + 0xa2, 0x43, 0x00, 0xc0, 0xa2, 0x43, 0x00, 0x00, 0xa3, 0x43, 0x00, 0x40, 0xa3, 0x43, 0x00, 0x80, 0xa3, + 0x43, 0x00, 0xc0, 0xa3, 0x43, 0x00, 0x00, 0xa4, 0x43, 0x00, 0x40, 0xa4, 0x43, 0x00, 0x80, 0xa4, 0x43, + 0x00, 0xc0, 0xa4, 0x43, 0x00, 0x00, 0xa5, 0x43, 0x00, 0x40, 0xa5, 0x43, 0x00, 0x80, 0xa5, 0x43, 0x00, + 0xc0, 0xa5, 0x43, 0x00, 0x00, 0xa6, 0x43, 0x00, 0x40, 0xa6, 0x43, 0x00, 0x80, 0xa6, 0x43, 0x00, 0xc0, + 0xa6, 0x43, 0x00, 0x00, 0xa7, 0x43, 0x00, 0x40, 0xa7, 0x43, 0x00, 0x80, 0xa7, 0x43, 0x00, 0xc0, 0xa7, + 0x43, 0x00, 0x00, 0xa8, 0x43, 0x00, 0x40, 0xa8, 0x43, 0x00, 0x80, 0xa8, 0x43, 0x00, 0xc0, 0xa8, 0x43, + 0x00, 0x00, 0xa9, 0x43, 0x00, 0x40, 0xa9, 0x43, 0x00, 0x80, 0xa9, 0x43, 0x00, 0xc0, 0xa9, 0x43, 0x00, + 0x00, 0xaa, 0x43, 0x00, 0x40, 0xaa, 0x43, 0x00, 0x80, 0xaa, 0x43, 0x00, 0xc0, 0xaa, 0x43, 0x00, 0x00, + 0xab, 0x43, 0x00, 0x40, 0xab, 0x43, 0x00, 0x80, 0xab, 0x43, 0x00, 0xc0, 0xab, 0x43, 0x00, 0x00, 0xac, + 0x43, 0x00, 0x40, 0xac, 0x43, 0x00, 0x80, 0xac, 0x43, 0x00, 0xc0, 0xac, 0x43, 0x00, 0x00, 0xad, 0x43, + 0x00, 0x40, 0xad, 0x43, 0x00, 0x80, 0xad, 0x43, 0x00, 0xc0, 0xad, 0x43, 0x00, 0x00, 0xae, 0x43, 0x00, + 0x40, 0xae, 0x43, 0x00, 0x80, 0xae, 0x43, 0x00, 0xc0, 0xae, 0x43, 0x00, 0x00, 0xaf, 0x43, 0x00, 0x40, + 0xaf, 0x43, 0x00, 0x80, 0xaf, 0x43, 0x00, 0xc0, 0xaf, 0x43, 0x00, 0x00, 0xb0, 0x43, 0x00, 0x40, 0xb0, + 0x43, 0x00, 0x80, 0xb0, 0x43, 0x00, 0xc0, 0xb0, 0x43, 0x00, 0x00, 0xb1, 0x43, 0x00, 0x40, 0xb1, 0x43, + 0x00, 0x80, 0xb1, 0x43, 0x00, 0xc0, 0xb1, 0x43, 0x00, 0x00, 0xb2, 0x43, 0x00, 0x40, 0xb2, 0x43, 0x00, + 0x80, 0xb2, 0x43, 0x00, 0xc0, 0xb2, 0x43, 0x00, 0x00, 0xb3, 0x43, 0x00, 0x40, 0xb3, 0x43, 0x00, 0x80, + 0xb3, 0x43, 0x00, 0xc0, 0xb3, 0x43, 0x00, 0x00, 0xb4, 0x43, 0x00, 0x40, 0xb4, 0x43, 0x00, 0x80, 0xb4, + 0x43, 0x00, 0xc0, 0xb4, 0x43, 0x00, 0x00, 0xb5, 0x43, 0x00, 0x40, 0xb5, 0x43, 0x00, 0x80, 0xb5, 0x43, + 0x00, 0xc0, 0xb5, 0x43, 0x00, 0x00, 0xb6, 0x43, 0x00, 0x40, 0xb6, 0x43, 0x00, 0x80, 0xb6, 0x43, 0x00, + 0xc0, 0xb6, 0x43, 0x00, 0x00, 0xb7, 0x43, 0x00, 0x40, 0xb7, 0x43, 0x00, 0x80, 0xb7, 0x43, 0x00, 0xc0, + 0xb7, 0x43, 0x00, 0x00, 0xb8, 0x43, 0x00, 0x40, 0xb8, 0x43, 0x00, 0x80, 0xb8, 0x43, 0x00, 0xc0, 0xb8, + 0x43, 0x00, 0x00, 0xb9, 0x43, 0x00, 0x40, 0xb9, 0x43, 0x00, 0x80, 0xb9, 0x43, 0x00, 0xc0, 0xb9, 0x43, + 0x00, 0x00, 0xba, 0x43, 0x00, 0x40, 0xba, 0x43, 0x00, 0x80, 0xba, 0x43, 0x00, 0xc0, 0xba, 0x43, 0x00, + 0x00, 0xbb, 0x43, 0x00, 0x40, 0xbb, 0x43, 0x00, 0x80, 0xbb, 0x43, 0x00, 0xc0, 0xbb, 0x43, 0x00, 0x00, + 0xbc, 0x43, 0x00, 0x40, 0xbc, 0x43, 0x00, 0x80, 0xbc, 0x43, 0x00, 0xc0, 0xbc, 0x43, 0x00, 0x00, 0xbd, + 0x43, 0x00, 0x40, 0xbd, 0x43, 0x00, 0x80, 0xbd, 0x43, 0x00, 0xc0, 0xbd, 0x43, 0x00, 0x00, 0xbe, 0x43, + 0x00, 0x40, 0xbe, 0x43, 0x00, 0x80, 0xbe, 0x43, 0x00, 0xc0, 0xbe, 0x43, 0x00, 0x00, 0xbf, 0x43, 0x00, + 0x40, 0xbf, 0x43, 0x00, 0x80, 0xbf, 0x43, 0x00, 0xc0, 0xbf, 0x43, 0x00, 0x00, 0xc0, 0x43, 0x00, 0x40, + 0xc0, 0x43, 0x00, 0x80, 0xc0, 0x43, 0x00, 0xc0, 0xc0, 0x43, 0x00, 0x00, 0xc1, 0x43, 0x00, 0x40, 0xc1, + 0x43, 0x00, 0x80, 0xc1, 0x43, 0x00, 0xc0, 0xc1, 0x43, 0x00, 0x00, 0xc2, 0x43, 0x00, 0x40, 0xc2, 0x43, + 0x00, 0x80, 0xc2, 0x43, 0x00, 0xc0, 0xc2, 0x43, 0x00, 0x00, 0xc3, 0x43, 0x00, 0x40, 0xc3, 0x43, 0x00, + 0x80, 0xc3, 0x43, 0x00, 0xc0, 0xc3, 0x43, 0x00, 0x00, 0xc4, 0x43, 0x00, 0x40, 0xc4, 0x43, 0x00, 0x80, + 0xc4, 0x43, 0x00, 0xc0, 0xc4, 0x43, 0x00, 0x00, 0xc5, 0x43, 0x00, 0x40, 0xc5, 0x43, 0x00, 0x80, 0xc5, + 0x43, 0x00, 0xc0, 0xc5, 0x43, 0x00, 0x00, 0xc6, 0x43, 0x00, 0x40, 0xc6, 0x43, 0x00, 0x80, 0xc6, 0x43, + 0x00, 0xc0, 0xc6, 0x43, 0x00, 0x00, 0xc7, 0x43, 0x00, 0x40, 0xc7, 0x43, 0x00, 0x80, 0xc7, 0x43, 0x00, + 0xc0, 0xc7, 0x43, 0x00, 0x00, 0xc8, 0x43, 0x00, 0x40, 0xc8, 0x43, 0x00, 0x80, 0xc8, 0x43, 0x00, 0xc0, + 0xc8, 0x43, 0x00, 0x00, 0xc9, 0x43, 0x00, 0x40, 0xc9, 0x43, 0x00, 0x80, 0xc9, 0x43, 0x00, 0xc0, 0xc9, + 0x43, 0x00, 0x00, 0xca, 0x43, 0x00, 0x40, 0xca, 0x43, 0x00, 0x80, 0xca, 0x43, 0x00, 0xc0, 0xca, 0x43, + 0x00, 0x00, 0xcb, 0x43, 0x00, 0x40, 0xcb, 0x43, 0x00, 0x80, 0xcb, 0x43, 0x00, 0xc0, 0xcb, 0x43, 0x00, + 0x00, 0xcc, 0x43, 0x00, 0x40, 0xcc, 0x43, 0x00, 0x80, 0xcc, 0x43, 0x00, 0xc0, 0xcc, 0x43, 0x00, 0x00, + 0xcd, 0x43, 0x00, 0x40, 0xcd, 0x43, 0x00, 0x80, 0xcd, 0x43, 0x00, 0xc0, 0xcd, 0x43, 0x00, 0x00, 0xce, + 0x43, 0x00, 0x40, 0xce, 0x43, 0x00, 0x80, 0xce, 0x43, 0x00, 0xc0, 0xce, 0x43, 0x00, 0x00, 0xcf, 0x43, + 0x00, 0x40, 0xcf, 0x43, 0x00, 0x80, 0xcf, 0x43, 0x00, 0xc0, 0xcf, 0x43, 0x00, 0x00, 0xd0, 0x43, 0x00, + 0x40, 0xd0, 0x43, 0x00, 0x80, 0xd0, 0x43, 0x00, 0xc0, 0xd0, 0x43, 0x00, 0x00, 0xd1, 0x43, 0x00, 0x40, + 0xd1, 0x43, 0x00, 0x80, 0xd1, 0x43, 0x00, 0xc0, 0xd1, 0x43, 0x00, 0x00, 0xd2, 0x43, 0x00, 0x40, 0xd2, + 0x43, 0x00, 0x80, 0xd2, 0x43, 0x00, 0xc0, 0xd2, 0x43, 0x00, 0x00, 0xd3, 0x43, 0x00, 0x40, 0xd3, 0x43, + 0x00, 0x80, 0xd3, 0x43, 0x00, 0xc0, 0xd3, 0x43, 0x00, 0x00, 0xd4, 0x43, 0x00, 0x40, 0xd4, 0x43, 0x00, + 0x80, 0xd4, 0x43, 0x00, 0xc0, 0xd4, 0x43, 0x00, 0x00, 0xd5, 0x43, 0x00, 0x40, 0xd5, 0x43, 0x00, 0x80, + 0xd5, 0x43, 0x00, 0xc0, 0xd5, 0x43, 0x00, 0x00, 0xd6, 0x43, 0x00, 0x40, 0xd6, 0x43, 0x00, 0x80, 0xd6, + 0x43, 0x00, 0xc0, 0xd6, 0x43, 0x00, 0x00, 0xd7, 0x43, 0x00, 0x40, 0xd7, 0x43, 0x00, 0x80, 0xd7, 0x43, + 0x00, 0xc0, 0xd7, 0x43, 0x00, 0x00, 0xd8, 0x43, 0x00, 0x40, 0xd8, 0x43, 0x00, 0x80, 0xd8, 0x43, 0x00, + 0xc0, 0xd8, 0x43, 0x00, 0x00, 0xd9, 0x43, 0x00, 0x40, 0xd9, 0x43, 0x00, 0x80, 0xd9, 0x43, 0x00, 0xc0, + 0xd9, 0x43, 0x00, 0x00, 0xda, 0x43, 0x00, 0x40, 0xda, 0x43, 0x00, 0x80, 0xda, 0x43, 0x00, 0xc0, 0xda, + 0x43, 0x00, 0x00, 0xdb, 0x43, 0x00, 0x40, 0xdb, 0x43, 0x00, 0x80, 0xdb, 0x43, 0x00, 0xc0, 0xdb, 0x43, + 0x00, 0x00, 0xdc, 0x43, 0x00, 0x40, 0xdc, 0x43, 0x00, 0x80, 0xdc, 0x43, 0x00, 0xc0, 0xdc, 0x43, 0x00, + 0x00, 0xdd, 0x43, 0x00, 0x40, 0xdd, 0x43, 0x00, 0x80, 0xdd, 0x43, 0x00, 0xc0, 0xdd, 0x43, 0x00, 0x00, + 0xde, 0x43, 0x00, 0x40, 0xde, 0x43, 0x00, 0x80, 0xde, 0x43, 0x00, 0xc0, 0xde, 0x43, 0x00, 0x00, 0xdf, + 0x43, 0x00, 0x40, 0xdf, 0x43, 0x00, 0x80, 0xdf, 0x43, 0x00, 0xc0, 0xdf, 0x43, 0x00, 0x00, 0xe0, 0x43, + 0x00, 0x40, 0xe0, 0x43, 0x00, 0x80, 0xe0, 0x43, 0x00, 0xc0, 0xe0, 0x43, 0x00, 0x00, 0xe1, 0x43, 0x00, + 0x40, 0xe1, 0x43, 0x00, 0x80, 0xe1, 0x43, 0x00, 0xc0, 0xe1, 0x43, 0x00, 0x00, 0xe2, 0x43, 0x00, 0x40, + 0xe2, 0x43, 0x00, 0x80, 0xe2, 0x43, 0x00, 0xc0, 0xe2, 0x43, 0x00, 0x00, 0xe3, 0x43, 0x00, 0x40, 0xe3, + 0x43, 0x00, 0x80, 0xe3, 0x43, 0x00, 0xc0, 0xe3, 0x43, 0x00, 0x00, 0xe4, 0x43, 0x00, 0x40, 0xe4, 0x43, + 0x00, 0x80, 0xe4, 0x43, 0x00, 0xc0, 0xe4, 0x43, 0x00, 0x00, 0xe5, 0x43, 0x00, 0x40, 0xe5, 0x43, 0x00, + 0x80, 0xe5, 0x43, 0x00, 0xc0, 0xe5, 0x43, 0x00, 0x00, 0xe6, 0x43, 0x00, 0x40, 0xe6, 0x43, 0x00, 0x80, + 0xe6, 0x43, 0x00, 0xc0, 0xe6, 0x43, 0x00, 0x00, 0xe7, 0x43, 0x00, 0x40, 0xe7, 0x43, 0x00, 0x80, 0xe7, + 0x43, 0x00, 0xc0, 0xe7, 0x43, 0x00, 0x00, 0xe8, 0x43, 0x00, 0x40, 0xe8, 0x43, 0x00, 0x80, 0xe8, 0x43, + 0x00, 0xc0, 0xe8, 0x43, 0x00, 0x00, 0xe9, 0x43, 0x00, 0x40, 0xe9, 0x43, 0x00, 0x80, 0xe9, 0x43, 0x00, + 0xc0, 0xe9, 0x43, 0x00, 0x00, 0xea, 0x43, 0x00, 0x40, 0xea, 0x43, 0x00, 0x80, 0xea, 0x43, 0x00, 0xc0, + 0xea, 0x43, 0x00, 0x00, 0xeb, 0x43, 0x00, 0x40, 0xeb, 0x43, 0x00, 0x80, 0xeb, 0x43, 0x00, 0xc0, 0xeb, + 0x43, 0x00, 0x00, 0xec, 0x43, 0x00, 0x40, 0xec, 0x43, 0x00, 0x80, 0xec, 0x43, 0x00, 0xc0, 0xec, 0x43, + 0x00, 0x00, 0xed, 0x43, 0x00, 0x40, 0xed, 0x43, 0x00, 0x80, 0xed, 0x43, 0x00, 0xc0, 0xed, 0x43, 0x00, + 0x00, 0xee, 0x43, 0x00, 0x40, 0xee, 0x43, 0x00, 0x80, 0xee, 0x43, 0x00, 0xc0, 0xee, 0x43, 0x00, 0x00, + 0xef, 0x43, 0x00, 0x40, 0xef, 0x43, 0x00, 0x80, 0xef, 0x43, 0x00, 0xc0, 0xef, 0x43, 0x00, 0x00, 0xf0, + 0x43, 0x00, 0x40, 0xf0, 0x43, 0x00, 0x80, 0xf0, 0x43, 0x00, 0xc0, 0xf0, 0x43, 0x00, 0x00, 0xf1, 0x43, + 0x00, 0x40, 0xf1, 0x43, 0x00, 0x80, 0xf1, 0x43, 0x00, 0xc0, 0xf1, 0x43, 0x00, 0x00, 0xf2, 0x43, 0x00, + 0x40, 0xf2, 0x43, 0x00, 0x80, 0xf2, 0x43, 0x00, 0xc0, 0xf2, 0x43, 0x00, 0x00, 0xf3, 0x43, 0x00, 0x40, + 0xf3, 0x43, 0x00, 0x80, 0xf3, 0x43, 0x00, 0xc0, 0xf3, 0x43, 0x00, 0x00, 0xf4, 0x43, 0x00, 0x40, 0xf4, + 0x43, 0x00, 0x80, 0xf4, 0x43, 0x00, 0xc0, 0xf4, 0x43, 0x00, 0x00, 0xf5, 0x43, 0x00, 0x40, 0xf5, 0x43, + 0x00, 0x80, 0xf5, 0x43, 0x00, 0xc0, 0xf5, 0x43, 0x00, 0x00, 0xf6, 0x43, 0x00, 0x40, 0xf6, 0x43, 0x00, + 0x80, 0xf6, 0x43, 0x00, 0xc0, 0xf6, 0x43, 0x00, 0x00, 0xf7, 0x43, 0x00, 0x40, 0xf7, 0x43, 0x00, 0x80, + 0xf7, 0x43, 0x00, 0xc0, 0xf7, 0x43, 0x00, 0x00, 0xf8, 0x43, 0x00, 0x40, 0xf8, 0x43, 0x00, 0x80, 0xf8, + 0x43, 0x00, 0xc0, 0xf8, 0x43, 0x00, 0x00, 0xf9, 0x43, 0x00, 0x40, 0xf9, 0x43, 0x00, 0x80, 0xf9, 0x43, + 0x00, 0xc0, 0xf9, 0x43, 0x00, 0x00, 0xfa, 0x43, 0x00, 0x40, 0xfa, 0x43, 0x00, 0x80, 0xfa, 0x43, 0x00, + 0xc0, 0xfa, 0x43, 0x00, 0x00, 0xfb, 0x43, 0x00, 0x40, 0xfb, 0x43, 0x00, 0x80, 0xfb, 0x43, 0x00, 0xc0, + 0xfb, 0x43, 0x00, 0x00, 0xfc, 0x43, 0x00, 0x40, 0xfc, 0x43, 0x00, 0x80, 0xfc, 0x43, 0x00, 0xc0, 0xfc, + 0x43, 0x00, 0x00, 0xfd, 0x43, 0x00, 0x40, 0xfd, 0x43, 0x00, 0x80, 0xfd, 0x43, 0x00, 0xc0, 0xfd, 0x43, + 0x00, 0x00, 0xfe, 0x43, 0x00, 0x40, 0xfe, 0x43, 0x00, 0x80, 0xfe, 0x43, 0x00, 0xc0, 0xfe, 0x43, 0x00, + 0x00, 0xff, 0x43, 0x00, 0x40, 0xff, 0x43, 0x00, 0x80, 0xff, 0x43, 0x00, 0xc0, 0xff, 0x43, 0x00, 0x00, + 0x00, 0x44, 0x00, 0x20, 0x00, 0x44, 0x00, 0x40, 0x00, 0x44, 0x00, 0x60, 0x00, 0x44, 0x00, 0x80, 0x00, + 0x44, 0x00, 0xa0, 0x00, 0x44, 0x00, 0xc0, 0x00, 0x44, 0x00, 0xe0, 0x00, 0x44, 0x00, 0x00, 0x01, 0x44, + 0x00, 0x20, 0x01, 0x44, 0x00, 0x40, 0x01, 0x44, 0x00, 0x60, 0x01, 0x44, 0x00, 0x80, 0x01, 0x44, 0x00, + 0xa0, 0x01, 0x44, 0x00, 0xc0, 0x01, 0x44, 0x00, 0xe0, 0x01, 0x44, 0x00, 0x00, 0x02, 0x44, 0x00, 0x20, + 0x02, 0x44, 0x00, 0x40, 0x02, 0x44, 0x00, 0x60, 0x02, 0x44, 0x00, 0x80, 0x02, 0x44, 0x00, 0xa0, 0x02, + 0x44, 0x00, 0xc0, 0x02, 0x44, 0x00, 0xe0, 0x02, 0x44, 0x00, 0x00, 0x03, 0x44, 0x00, 0x20, 0x03, 0x44, + 0x00, 0x40, 0x03, 0x44, 0x00, 0x60, 0x03, 0x44, 0x00, 0x80, 0x03, 0x44, 0x00, 0xa0, 0x03, 0x44, 0x00, + 0xc0, 0x03, 0x44, 0x00, 0xe0, 0x03, 0x44, 0x00, 0x00, 0x04, 0x44, 0x00, 0x20, 0x04, 0x44, 0x00, 0x40, + 0x04, 0x44, 0x00, 0x60, 0x04, 0x44, 0x00, 0x80, 0x04, 0x44, 0x00, 0xa0, 0x04, 0x44, 0x00, 0xc0, 0x04, + 0x44, 0x00, 0xe0, 0x04, 0x44, 0x00, 0x00, 0x05, 0x44, 0x00, 0x20, 0x05, 0x44, 0x00, 0x40, 0x05, 0x44, + 0x00, 0x60, 0x05, 0x44, 0x00, 0x80, 0x05, 0x44, 0x00, 0xa0, 0x05, 0x44, 0x00, 0xc0, 0x05, 0x44, 0x00, + 0xe0, 0x05, 0x44, 0x00, 0x00, 0x06, 0x44, 0x00, 0x20, 0x06, 0x44, 0x00, 0x40, 0x06, 0x44, 0x00, 0x60, + 0x06, 0x44, 0x00, 0x80, 0x06, 0x44, 0x00, 0xa0, 0x06, 0x44, 0x00, 0xc0, 0x06, 0x44, 0x00, 0xe0, 0x06, + 0x44, 0x00, 0x00, 0x07, 0x44, 0x00, 0x20, 0x07, 0x44, 0x00, 0x40, 0x07, 0x44, 0x00, 0x60, 0x07, 0x44, + 0x00, 0x80, 0x07, 0x44, 0x00, 0xa0, 0x07, 0x44, 0x00, 0xc0, 0x07, 0x44, 0x00, 0xe0, 0x07, 0x44, 0x00, + 0x00, 0x08, 0x44, 0x00, 0x20, 0x08, 0x44, 0x00, 0x40, 0x08, 0x44, 0x00, 0x60, 0x08, 0x44, 0x00, 0x80, + 0x08, 0x44, 0x00, 0xa0, 0x08, 0x44, 0x00, 0xc0, 0x08, 0x44, 0x00, 0xe0, 0x08, 0x44, 0x00, 0x00, 0x09, + 0x44, 0x00, 0x20, 0x09, 0x44, 0x00, 0x40, 0x09, 0x44, 0x00, 0x60, 0x09, 0x44, 0x00, 0x80, 0x09, 0x44, + 0x00, 0xa0, 0x09, 0x44, 0x00, 0xc0, 0x09, 0x44, 0x00, 0xe0, 0x09, 0x44, 0x00, 0x00, 0x0a, 0x44, 0x00, + 0x20, 0x0a, 0x44, 0x00, 0x40, 0x0a, 0x44, 0x00, 0x60, 0x0a, 0x44, 0x00, 0x80, 0x0a, 0x44, 0x00, 0xa0, + 0x0a, 0x44, 0x00, 0xc0, 0x0a, 0x44, 0x00, 0xe0, 0x0a, 0x44, 0x00, 0x00, 0x0b, 0x44, 0x00, 0x20, 0x0b, + 0x44, 0x00, 0x40, 0x0b, 0x44, 0x00, 0x60, 0x0b, 0x44, 0x00, 0x80, 0x0b, 0x44, 0x00, 0xa0, 0x0b, 0x44, + 0x00, 0xc0, 0x0b, 0x44, 0x00, 0xe0, 0x0b, 0x44, 0x00, 0x00, 0x0c, 0x44, 0x00, 0x20, 0x0c, 0x44, 0x00, + 0x40, 0x0c, 0x44, 0x00, 0x60, 0x0c, 0x44, 0x00, 0x80, 0x0c, 0x44, 0x00, 0xa0, 0x0c, 0x44, 0x00, 0xc0, + 0x0c, 0x44, 0x00, 0xe0, 0x0c, 0x44, 0x00, 0x00, 0x0d, 0x44, 0x00, 0x20, 0x0d, 0x44, 0x00, 0x40, 0x0d, + 0x44, 0x00, 0x60, 0x0d, 0x44, 0x00, 0x80, 0x0d, 0x44, 0x00, 0xa0, 0x0d, 0x44, 0x00, 0xc0, 0x0d, 0x44, + 0x00, 0xe0, 0x0d, 0x44, 0x00, 0x00, 0x0e, 0x44, 0x00, 0x20, 0x0e, 0x44, 0x00, 0x40, 0x0e, 0x44, 0x00, + 0x60, 0x0e, 0x44, 0x00, 0x80, 0x0e, 0x44, 0x00, 0xa0, 0x0e, 0x44, 0x00, 0xc0, 0x0e, 0x44, 0x00, 0xe0, + 0x0e, 0x44, 0x00, 0x00, 0x0f, 0x44, 0x00, 0x20, 0x0f, 0x44, 0x00, 0x40, 0x0f, 0x44, 0x00, 0x60, 0x0f, + 0x44, 0x00, 0x80, 0x0f, 0x44, 0x00, 0xa0, 0x0f, 0x44, 0x00, 0xc0, 0x0f, 0x44, 0x00, 0xe0, 0x0f, 0x44, + 0x00, 0x00, 0x10, 0x44, 0x00, 0x20, 0x10, 0x44, 0x00, 0x40, 0x10, 0x44, 0x00, 0x60, 0x10, 0x44, 0x00, + 0x80, 0x10, 0x44, 0x00, 0xa0, 0x10, 0x44, 0x00, 0xc0, 0x10, 0x44, 0x00, 0xe0, 0x10, 0x44, 0x00, 0x00, + 0x11, 0x44, 0x00, 0x20, 0x11, 0x44, 0x00, 0x40, 0x11, 0x44, 0x00, 0x60, 0x11, 0x44, 0x00, 0x80, 0x11, + 0x44, 0x00, 0xa0, 0x11, 0x44, 0x00, 0xc0, 0x11, 0x44, 0x00, 0xe0, 0x11, 0x44, 0x00, 0x00, 0x12, 0x44, + 0x00, 0x20, 0x12, 0x44, 0x00, 0x40, 0x12, 0x44, 0x00, 0x60, 0x12, 0x44, 0x00, 0x80, 0x12, 0x44, 0x00, + 0xa0, 0x12, 0x44, 0x00, 0xc0, 0x12, 0x44, 0x00, 0xe0, 0x12, 0x44, 0x00, 0x00, 0x13, 0x44, 0x00, 0x20, + 0x13, 0x44, 0x00, 0x40, 0x13, 0x44, 0x00, 0x60, 0x13, 0x44, 0x00, 0x80, 0x13, 0x44, 0x00, 0xa0, 0x13, + 0x44, 0x00, 0xc0, 0x13, 0x44, 0x00, 0xe0, 0x13, 0x44, 0x00, 0x00, 0x14, 0x44, 0x00, 0x20, 0x14, 0x44, + 0x00, 0x40, 0x14, 0x44, 0x00, 0x60, 0x14, 0x44, 0x00, 0x80, 0x14, 0x44, 0x00, 0xa0, 0x14, 0x44, 0x00, + 0xc0, 0x14, 0x44, 0x00, 0xe0, 0x14, 0x44, 0x00, 0x00, 0x15, 0x44, 0x00, 0x20, 0x15, 0x44, 0x00, 0x40, + 0x15, 0x44, 0x00, 0x60, 0x15, 0x44, 0x00, 0x80, 0x15, 0x44, 0x00, 0xa0, 0x15, 0x44, 0x00, 0xc0, 0x15, + 0x44, 0x00, 0xe0, 0x15, 0x44, 0x00, 0x00, 0x16, 0x44, 0x00, 0x20, 0x16, 0x44, 0x00, 0x40, 0x16, 0x44, + 0x00, 0x60, 0x16, 0x44, 0x00, 0x80, 0x16, 0x44, 0x00, 0xa0, 0x16, 0x44, 0x00, 0xc0, 0x16, 0x44, 0x00, + 0xe0, 0x16, 0x44, 0x00, 0x00, 0x17, 0x44, 0x00, 0x20, 0x17, 0x44, 0x00, 0x40, 0x17, 0x44, 0x00, 0x60, + 0x17, 0x44, 0x00, 0x80, 0x17, 0x44, 0x00, 0xa0, 0x17, 0x44, 0x00, 0xc0, 0x17, 0x44, 0x00, 0xe0, 0x17, + 0x44, 0x00, 0x00, 0x18, 0x44, 0x00, 0x20, 0x18, 0x44, 0x00, 0x40, 0x18, 0x44, 0x00, 0x60, 0x18, 0x44, + 0x00, 0x80, 0x18, 0x44, 0x00, 0xa0, 0x18, 0x44, 0x00, 0xc0, 0x18, 0x44, 0x00, 0xe0, 0x18, 0x44, 0x00, + 0x00, 0x19, 0x44, 0x00, 0x20, 0x19, 0x44, 0x00, 0x40, 0x19, 0x44, 0x00, 0x60, 0x19, 0x44, 0x00, 0x80, + 0x19, 0x44, 0x00, 0xa0, 0x19, 0x44, 0x00, 0xc0, 0x19, 0x44, 0x00, 0xe0, 0x19, 0x44, 0x00, 0x00, 0x1a, + 0x44, 0x00, 0x20, 0x1a, 0x44, 0x00, 0x40, 0x1a, 0x44, 0x00, 0x60, 0x1a, 0x44, 0x00, 0x80, 0x1a, 0x44, + 0x00, 0xa0, 0x1a, 0x44, 0x00, 0xc0, 0x1a, 0x44, 0x00, 0xe0, 0x1a, 0x44, 0x00, 0x00, 0x1b, 0x44, 0x00, + 0x20, 0x1b, 0x44, 0x00, 0x40, 0x1b, 0x44, 0x00, 0x60, 0x1b, 0x44, 0x00, 0x80, 0x1b, 0x44, 0x00, 0xa0, + 0x1b, 0x44, 0x00, 0xc0, 0x1b, 0x44, 0x00, 0xe0, 0x1b, 0x44, 0x00, 0x00, 0x1c, 0x44, 0x00, 0x20, 0x1c, + 0x44, 0x00, 0x40, 0x1c, 0x44, 0x00, 0x60, 0x1c, 0x44, 0x00, 0x80, 0x1c, 0x44, 0x00, 0xa0, 0x1c, 0x44, + 0x00, 0xc0, 0x1c, 0x44, 0x00, 0xe0, 0x1c, 0x44, 0x00, 0x00, 0x1d, 0x44, 0x00, 0x20, 0x1d, 0x44, 0x00, + 0x40, 0x1d, 0x44, 0x00, 0x60, 0x1d, 0x44, 0x00, 0x80, 0x1d, 0x44, 0x00, 0xa0, 0x1d, 0x44, 0x00, 0xc0, + 0x1d, 0x44, 0x00, 0xe0, 0x1d, 0x44, 0x00, 0x00, 0x1e, 0x44, 0x00, 0x20, 0x1e, 0x44, 0x00, 0x40, 0x1e, + 0x44, 0x00, 0x60, 0x1e, 0x44, 0x00, 0x80, 0x1e, 0x44, 0x00, 0xa0, 0x1e, 0x44, 0x00, 0xc0, 0x1e, 0x44, + 0x00, 0xe0, 0x1e, 0x44, 0x00, 0x00, 0x1f, 0x44, 0x00, 0x20, 0x1f, 0x44, 0x00, 0x40, 0x1f, 0x44, 0x00, + 0x60, 0x1f, 0x44, 0x00, 0x80, 0x1f, 0x44, 0x00, 0xa0, 0x1f, 0x44, 0x00, 0xc0, 0x1f, 0x44, 0x00, 0xe0, + 0x1f, 0x44, 0x00, 0x00, 0x20, 0x44, 0x00, 0x20, 0x20, 0x44, 0x00, 0x40, 0x20, 0x44, 0x00, 0x60, 0x20, + 0x44, 0x00, 0x80, 0x20, 0x44, 0x00, 0xa0, 0x20, 0x44, 0x00, 0xc0, 0x20, 0x44, 0x00, 0xe0, 0x20, 0x44, + 0x00, 0x00, 0x21, 0x44, 0x00, 0x20, 0x21, 0x44, 0x00, 0x40, 0x21, 0x44, 0x00, 0x60, 0x21, 0x44, 0x00, + 0x80, 0x21, 0x44, 0x00, 0xa0, 0x21, 0x44, 0x00, 0xc0, 0x21, 0x44, 0x00, 0xe0, 0x21, 0x44, 0x00, 0x00, + 0x22, 0x44, 0x00, 0x20, 0x22, 0x44, 0x00, 0x40, 0x22, 0x44, 0x00, 0x60, 0x22, 0x44, 0x00, 0x80, 0x22, + 0x44, 0x00, 0xa0, 0x22, 0x44, 0x00, 0xc0, 0x22, 0x44, 0x00, 0xe0, 0x22, 0x44, 0x00, 0x00, 0x23, 0x44, + 0x00, 0x20, 0x23, 0x44, 0x00, 0x40, 0x23, 0x44, 0x00, 0x60, 0x23, 0x44, 0x00, 0x80, 0x23, 0x44, 0x00, + 0xa0, 0x23, 0x44, 0x00, 0xc0, 0x23, 0x44, 0x00, 0xe0, 0x23, 0x44, 0x00, 0x00, 0x24, 0x44, 0x00, 0x20, + 0x24, 0x44, 0x00, 0x40, 0x24, 0x44, 0x00, 0x60, 0x24, 0x44, 0x00, 0x80, 0x24, 0x44, 0x00, 0xa0, 0x24, + 0x44, 0x00, 0xc0, 0x24, 0x44, 0x00, 0xe0, 0x24, 0x44, 0x00, 0x00, 0x25, 0x44, 0x00, 0x20, 0x25, 0x44, + 0x00, 0x40, 0x25, 0x44, 0x00, 0x60, 0x25, 0x44, 0x00, 0x80, 0x25, 0x44, 0x00, 0xa0, 0x25, 0x44, 0x00, + 0xc0, 0x25, 0x44, 0x00, 0xe0, 0x25, 0x44, 0x00, 0x00, 0x26, 0x44, 0x00, 0x20, 0x26, 0x44, 0x00, 0x40, + 0x26, 0x44, 0x00, 0x60, 0x26, 0x44, 0x00, 0x80, 0x26, 0x44, 0x00, 0xa0, 0x26, 0x44, 0x00, 0xc0, 0x26, + 0x44, 0x00, 0xe0, 0x26, 0x44, 0x00, 0x00, 0x27, 0x44, 0x00, 0x20, 0x27, 0x44, 0x00, 0x40, 0x27, 0x44, + 0x00, 0x60, 0x27, 0x44, 0x00, 0x80, 0x27, 0x44, 0x00, 0xa0, 0x27, 0x44, 0x00, 0xc0, 0x27, 0x44, 0x00, + 0xe0, 0x27, 0x44, 0x00, 0x00, 0x28, 0x44, 0x00, 0x20, 0x28, 0x44, 0x00, 0x40, 0x28, 0x44, 0x00, 0x60, + 0x28, 0x44, 0x00, 0x80, 0x28, 0x44, 0x00, 0xa0, 0x28, 0x44, 0x00, 0xc0, 0x28, 0x44, 0x00, 0xe0, 0x28, + 0x44, 0x00, 0x00, 0x29, 0x44, 0x00, 0x20, 0x29, 0x44, 0x00, 0x40, 0x29, 0x44, 0x00, 0x60, 0x29, 0x44, + 0x00, 0x80, 0x29, 0x44, 0x00, 0xa0, 0x29, 0x44, 0x00, 0xc0, 0x29, 0x44, 0x00, 0xe0, 0x29, 0x44, 0x00, + 0x00, 0x2a, 0x44, 0x00, 0x20, 0x2a, 0x44, 0x00, 0x40, 0x2a, 0x44, 0x00, 0x60, 0x2a, 0x44, 0x00, 0x80, + 0x2a, 0x44, 0x00, 0xa0, 0x2a, 0x44, 0x00, 0xc0, 0x2a, 0x44, 0x00, 0xe0, 0x2a, 0x44, 0x00, 0x00, 0x2b, + 0x44, 0x00, 0x20, 0x2b, 0x44, 0x00, 0x40, 0x2b, 0x44, 0x00, 0x60, 0x2b, 0x44, 0x00, 0x80, 0x2b, 0x44, + 0x00, 0xa0, 0x2b, 0x44, 0x00, 0xc0, 0x2b, 0x44, 0x00, 0xe0, 0x2b, 0x44, 0x00, 0x00, 0x2c, 0x44, 0x00, + 0x20, 0x2c, 0x44, 0x00, 0x40, 0x2c, 0x44, 0x00, 0x60, 0x2c, 0x44, 0x00, 0x80, 0x2c, 0x44, 0x00, 0xa0, + 0x2c, 0x44, 0x00, 0xc0, 0x2c, 0x44, 0x00, 0xe0, 0x2c, 0x44, 0x00, 0x00, 0x2d, 0x44, 0x00, 0x20, 0x2d, + 0x44, 0x00, 0x40, 0x2d, 0x44, 0x00, 0x60, 0x2d, 0x44, 0x00, 0x80, 0x2d, 0x44, 0x00, 0xa0, 0x2d, 0x44, + 0x00, 0xc0, 0x2d, 0x44, 0x00, 0xe0, 0x2d, 0x44, 0x00, 0x00, 0x2e, 0x44, 0x00, 0x20, 0x2e, 0x44, 0x00, + 0x40, 0x2e, 0x44, 0x00, 0x60, 0x2e, 0x44, 0x00, 0x80, 0x2e, 0x44, 0x00, 0xa0, 0x2e, 0x44, 0x00, 0xc0, + 0x2e, 0x44, 0x00, 0xe0, 0x2e, 0x44, 0x00, 0x00, 0x2f, 0x44, 0x00, 0x20, 0x2f, 0x44, 0x00, 0x40, 0x2f, + 0x44, 0x00, 0x60, 0x2f, 0x44, 0x00, 0x80, 0x2f, 0x44, 0x00, 0xa0, 0x2f, 0x44, 0x00, 0xc0, 0x2f, 0x44, + 0x00, 0xe0, 0x2f, 0x44, 0x00, 0x00, 0x30, 0x44, 0x00, 0x20, 0x30, 0x44, 0x00, 0x40, 0x30, 0x44, 0x00, + 0x60, 0x30, 0x44, 0x00, 0x80, 0x30, 0x44, 0x00, 0xa0, 0x30, 0x44, 0x00, 0xc0, 0x30, 0x44, 0x00, 0xe0, + 0x30, 0x44, 0x00, 0x00, 0x31, 0x44, 0x00, 0x20, 0x31, 0x44, 0x00, 0x40, 0x31, 0x44, 0x00, 0x60, 0x31, + 0x44, 0x00, 0x80, 0x31, 0x44, 0x00, 0xa0, 0x31, 0x44, 0x00, 0xc0, 0x31, 0x44, 0x00, 0xe0, 0x31, 0x44, + 0x00, 0x00, 0x32, 0x44, 0x00, 0x20, 0x32, 0x44, 0x00, 0x40, 0x32, 0x44, 0x00, 0x60, 0x32, 0x44, 0x00, + 0x80, 0x32, 0x44, 0x00, 0xa0, 0x32, 0x44, 0x00, 0xc0, 0x32, 0x44, 0x00, 0xe0, 0x32, 0x44, 0x00, 0x00, + 0x33, 0x44, 0x00, 0x20, 0x33, 0x44, 0x00, 0x40, 0x33, 0x44, 0x00, 0x60, 0x33, 0x44, 0x00, 0x80, 0x33, + 0x44, 0x00, 0xa0, 0x33, 0x44, 0x00, 0xc0, 0x33, 0x44, 0x00, 0xe0, 0x33, 0x44, 0x00, 0x00, 0x34, 0x44, + 0x00, 0x20, 0x34, 0x44, 0x00, 0x40, 0x34, 0x44, 0x00, 0x60, 0x34, 0x44, 0x00, 0x80, 0x34, 0x44, 0x00, + 0xa0, 0x34, 0x44, 0x00, 0xc0, 0x34, 0x44, 0x00, 0xe0, 0x34, 0x44, 0x00, 0x00, 0x35, 0x44, 0x00, 0x20, + 0x35, 0x44, 0x00, 0x40, 0x35, 0x44, 0x00, 0x60, 0x35, 0x44, 0x00, 0x80, 0x35, 0x44, 0x00, 0xa0, 0x35, + 0x44, 0x00, 0xc0, 0x35, 0x44, 0x00, 0xe0, 0x35, 0x44, 0x00, 0x00, 0x36, 0x44, 0x00, 0x20, 0x36, 0x44, + 0x00, 0x40, 0x36, 0x44, 0x00, 0x60, 0x36, 0x44, 0x00, 0x80, 0x36, 0x44, 0x00, 0xa0, 0x36, 0x44, 0x00, + 0xc0, 0x36, 0x44, 0x00, 0xe0, 0x36, 0x44, 0x00, 0x00, 0x37, 0x44, 0x00, 0x20, 0x37, 0x44, 0x00, 0x40, + 0x37, 0x44, 0x00, 0x60, 0x37, 0x44, 0x00, 0x80, 0x37, 0x44, 0x00, 0xa0, 0x37, 0x44, 0x00, 0xc0, 0x37, + 0x44, 0x00, 0xe0, 0x37, 0x44, 0x00, 0x00, 0x38, 0x44, 0x00, 0x20, 0x38, 0x44, 0x00, 0x40, 0x38, 0x44, + 0x00, 0x60, 0x38, 0x44, 0x00, 0x80, 0x38, 0x44, 0x00, 0xa0, 0x38, 0x44, 0x00, 0xc0, 0x38, 0x44, 0x00, + 0xe0, 0x38, 0x44, 0x00, 0x00, 0x39, 0x44, 0x00, 0x20, 0x39, 0x44, 0x00, 0x40, 0x39, 0x44, 0x00, 0x60, + 0x39, 0x44, 0x00, 0x80, 0x39, 0x44, 0x00, 0xa0, 0x39, 0x44, 0x00, 0xc0, 0x39, 0x44, 0x00, 0xe0, 0x39, + 0x44, 0x00, 0x00, 0x3a, 0x44, 0x00, 0x20, 0x3a, 0x44, 0x00, 0x40, 0x3a, 0x44, 0x00, 0x60, 0x3a, 0x44, + 0x00, 0x80, 0x3a, 0x44, 0x00, 0xa0, 0x3a, 0x44, 0x00, 0xc0, 0x3a, 0x44, 0x00, 0xe0, 0x3a, 0x44, 0x00, + 0x00, 0x3b, 0x44, 0x00, 0x20, 0x3b, 0x44, 0x00, 0x40, 0x3b, 0x44, 0x00, 0x60, 0x3b, 0x44, 0x00, 0x80, + 0x3b, 0x44, 0x00, 0xa0, 0x3b, 0x44, 0x00, 0xc0, 0x3b, 0x44, 0x00, 0xe0, 0x3b, 0x44, 0x00, 0x00, 0x3c, + 0x44, 0x00, 0x20, 0x3c, 0x44, 0x00, 0x40, 0x3c, 0x44, 0x00, 0x60, 0x3c, 0x44, 0x00, 0x80, 0x3c, 0x44, + 0x00, 0xa0, 0x3c, 0x44, 0x00, 0xc0, 0x3c, 0x44, 0x00, 0xe0, 0x3c, 0x44, 0x00, 0x00, 0x3d, 0x44, 0x00, + 0x20, 0x3d, 0x44, 0x00, 0x40, 0x3d, 0x44, 0x00, 0x60, 0x3d, 0x44, 0x00, 0x80, 0x3d, 0x44, 0x00, 0xa0, + 0x3d, 0x44, 0x00, 0xc0, 0x3d, 0x44, 0x00, 0xe0, 0x3d, 0x44, 0x00, 0x00, 0x3e, 0x44, 0x00, 0x20, 0x3e, + 0x44, 0x00, 0x40, 0x3e, 0x44, 0x00, 0x60, 0x3e, 0x44, 0x00, 0x80, 0x3e, 0x44, 0x00, 0xa0, 0x3e, 0x44, + 0x00, 0xc0, 0x3e, 0x44, 0x00, 0xe0, 0x3e, 0x44, 0x00, 0x00, 0x3f, 0x44, 0x00, 0x20, 0x3f, 0x44, 0x00, + 0x40, 0x3f, 0x44, 0x00, 0x60, 0x3f, 0x44, 0x00, 0x80, 0x3f, 0x44, 0x00, 0xa0, 0x3f, 0x44, 0x00, 0xc0, + 0x3f, 0x44, 0x00, 0xe0, 0x3f, 0x44, 0x00, 0x00, 0x40, 0x44, 0x00, 0x20, 0x40, 0x44, 0x00, 0x40, 0x40, + 0x44, 0x00, 0x60, 0x40, 0x44, 0x00, 0x80, 0x40, 0x44, 0x00, 0xa0, 0x40, 0x44, 0x00, 0xc0, 0x40, 0x44, + 0x00, 0xe0, 0x40, 0x44, 0x00, 0x00, 0x41, 0x44, 0x00, 0x20, 0x41, 0x44, 0x00, 0x40, 0x41, 0x44, 0x00, + 0x60, 0x41, 0x44, 0x00, 0x80, 0x41, 0x44, 0x00, 0xa0, 0x41, 0x44, 0x00, 0xc0, 0x41, 0x44, 0x00, 0xe0, + 0x41, 0x44, 0x00, 0x00, 0x42, 0x44, 0x00, 0x20, 0x42, 0x44, 0x00, 0x40, 0x42, 0x44, 0x00, 0x60, 0x42, + 0x44, 0x00, 0x80, 0x42, 0x44, 0x00, 0xa0, 0x42, 0x44, 0x00, 0xc0, 0x42, 0x44, 0x00, 0xe0, 0x42, 0x44, + 0x00, 0x00, 0x43, 0x44, 0x00, 0x20, 0x43, 0x44, 0x00, 0x40, 0x43, 0x44, 0x00, 0x60, 0x43, 0x44, 0x00, + 0x80, 0x43, 0x44, 0x00, 0xa0, 0x43, 0x44, 0x00, 0xc0, 0x43, 0x44, 0x00, 0xe0, 0x43, 0x44, 0x00, 0x00, + 0x44, 0x44, 0x00, 0x20, 0x44, 0x44, 0x00, 0x40, 0x44, 0x44, 0x00, 0x60, 0x44, 0x44, 0x00, 0x80, 0x44, + 0x44, 0x00, 0xa0, 0x44, 0x44, 0x00, 0xc0, 0x44, 0x44, 0x00, 0xe0, 0x44, 0x44, 0x00, 0x00, 0x45, 0x44, + 0x00, 0x20, 0x45, 0x44, 0x00, 0x40, 0x45, 0x44, 0x00, 0x60, 0x45, 0x44, 0x00, 0x80, 0x45, 0x44, 0x00, + 0xa0, 0x45, 0x44, 0x00, 0xc0, 0x45, 0x44, 0x00, 0xe0, 0x45, 0x44, 0x00, 0x00, 0x46, 0x44, 0x00, 0x20, + 0x46, 0x44, 0x00, 0x40, 0x46, 0x44, 0x00, 0x60, 0x46, 0x44, 0x00, 0x80, 0x46, 0x44, 0x00, 0xa0, 0x46, + 0x44, 0x00, 0xc0, 0x46, 0x44, 0x00, 0xe0, 0x46, 0x44, 0x00, 0x00, 0x47, 0x44, 0x00, 0x20, 0x47, 0x44, + 0x00, 0x40, 0x47, 0x44, 0x00, 0x60, 0x47, 0x44, 0x00, 0x80, 0x47, 0x44, 0x00, 0xa0, 0x47, 0x44, 0x00, + 0xc0, 0x47, 0x44, 0x00, 0xe0, 0x47, 0x44, 0x00, 0x00, 0x48, 0x44, 0x00, 0x20, 0x48, 0x44, 0x00, 0x40, + 0x48, 0x44, 0x00, 0x60, 0x48, 0x44, 0x00, 0x80, 0x48, 0x44, 0x00, 0xa0, 0x48, 0x44, 0x00, 0xc0, 0x48, + 0x44, 0x00, 0xe0, 0x48, 0x44, 0x00, 0x00, 0x49, 0x44, 0x00, 0x20, 0x49, 0x44, 0x00, 0x40, 0x49, 0x44, + 0x00, 0x60, 0x49, 0x44, 0x00, 0x80, 0x49, 0x44, 0x00, 0xa0, 0x49, 0x44, 0x00, 0xc0, 0x49, 0x44, 0x00, + 0xe0, 0x49, 0x44, 0x00, 0x00, 0x4a, 0x44, 0x00, 0x20, 0x4a, 0x44, 0x00, 0x40, 0x4a, 0x44, 0x00, 0x60, + 0x4a, 0x44, 0x00, 0x80, 0x4a, 0x44, 0x00, 0xa0, 0x4a, 0x44, 0x00, 0xc0, 0x4a, 0x44, 0x00, 0xe0, 0x4a, + 0x44, 0x00, 0x00, 0x4b, 0x44, 0x00, 0x20, 0x4b, 0x44, 0x00, 0x40, 0x4b, 0x44, 0x00, 0x60, 0x4b, 0x44, + 0x00, 0x80, 0x4b, 0x44, 0x00, 0xa0, 0x4b, 0x44, 0x00, 0xc0, 0x4b, 0x44, 0x00, 0xe0, 0x4b, 0x44, 0x00, + 0x00, 0x4c, 0x44, 0x00, 0x20, 0x4c, 0x44, 0x00, 0x40, 0x4c, 0x44, 0x00, 0x60, 0x4c, 0x44, 0x00, 0x80, + 0x4c, 0x44, 0x00, 0xa0, 0x4c, 0x44, 0x00, 0xc0, 0x4c, 0x44, 0x00, 0xe0, 0x4c, 0x44, 0x00, 0x00, 0x4d, + 0x44, 0x00, 0x20, 0x4d, 0x44, 0x00, 0x40, 0x4d, 0x44, 0x00, 0x60, 0x4d, 0x44, 0x00, 0x80, 0x4d, 0x44, + 0x00, 0xa0, 0x4d, 0x44, 0x00, 0xc0, 0x4d, 0x44, 0x00, 0xe0, 0x4d, 0x44, 0x00, 0x00, 0x4e, 0x44, 0x00, + 0x20, 0x4e, 0x44, 0x00, 0x40, 0x4e, 0x44, 0x00, 0x60, 0x4e, 0x44, 0x00, 0x80, 0x4e, 0x44, 0x00, 0xa0, + 0x4e, 0x44, 0x00, 0xc0, 0x4e, 0x44, 0x00, 0xe0, 0x4e, 0x44, 0x00, 0x00, 0x4f, 0x44, 0x00, 0x20, 0x4f, + 0x44, 0x00, 0x40, 0x4f, 0x44, 0x00, 0x60, 0x4f, 0x44, 0x00, 0x80, 0x4f, 0x44, 0x00, 0xa0, 0x4f, 0x44, + 0x00, 0xc0, 0x4f, 0x44, 0x00, 0xe0, 0x4f, 0x44, 0x00, 0x00, 0x50, 0x44, 0x00, 0x20, 0x50, 0x44, 0x00, + 0x40, 0x50, 0x44, 0x00, 0x60, 0x50, 0x44, 0x00, 0x80, 0x50, 0x44, 0x00, 0xa0, 0x50, 0x44, 0x00, 0xc0, + 0x50, 0x44, 0x00, 0xe0, 0x50, 0x44, 0x00, 0x00, 0x51, 0x44, 0x00, 0x20, 0x51, 0x44, 0x00, 0x40, 0x51, + 0x44, 0x00, 0x60, 0x51, 0x44, 0x00, 0x80, 0x51, 0x44, 0x00, 0xa0, 0x51, 0x44, 0x00, 0xc0, 0x51, 0x44, + 0x00, 0xe0, 0x51, 0x44, 0x00, 0x00, 0x52, 0x44, 0x00, 0x20, 0x52, 0x44, 0x00, 0x40, 0x52, 0x44, 0x00, + 0x60, 0x52, 0x44, 0x00, 0x80, 0x52, 0x44, 0x00, 0xa0, 0x52, 0x44, 0x00, 0xc0, 0x52, 0x44, 0x00, 0xe0, + 0x52, 0x44, 0x00, 0x00, 0x53, 0x44, 0x00, 0x20, 0x53, 0x44, 0x00, 0x40, 0x53, 0x44, 0x00, 0x60, 0x53, + 0x44, 0x00, 0x80, 0x53, 0x44, 0x00, 0xa0, 0x53, 0x44, 0x00, 0xc0, 0x53, 0x44, 0x00, 0xe0, 0x53, 0x44, + 0x00, 0x00, 0x54, 0x44, 0x00, 0x20, 0x54, 0x44, 0x00, 0x40, 0x54, 0x44, 0x00, 0x60, 0x54, 0x44, 0x00, + 0x80, 0x54, 0x44, 0x00, 0xa0, 0x54, 0x44, 0x00, 0xc0, 0x54, 0x44, 0x00, 0xe0, 0x54, 0x44, 0x00, 0x00, + 0x55, 0x44, 0x00, 0x20, 0x55, 0x44, 0x00, 0x40, 0x55, 0x44, 0x00, 0x60, 0x55, 0x44, 0x00, 0x80, 0x55, + 0x44, 0x00, 0xa0, 0x55, 0x44, 0x00, 0xc0, 0x55, 0x44, 0x00, 0xe0, 0x55, 0x44, 0x00, 0x00, 0x56, 0x44, + 0x00, 0x20, 0x56, 0x44, 0x00, 0x40, 0x56, 0x44, 0x00, 0x60, 0x56, 0x44, 0x00, 0x80, 0x56, 0x44, 0x00, + 0xa0, 0x56, 0x44, 0x00, 0xc0, 0x56, 0x44, 0x00, 0xe0, 0x56, 0x44, 0x00, 0x00, 0x57, 0x44, 0x00, 0x20, + 0x57, 0x44, 0x00, 0x40, 0x57, 0x44, 0x00, 0x60, 0x57, 0x44, 0x00, 0x80, 0x57, 0x44, 0x00, 0xa0, 0x57, + 0x44, 0x00, 0xc0, 0x57, 0x44, 0x00, 0xe0, 0x57, 0x44, 0x00, 0x00, 0x58, 0x44, 0x00, 0x20, 0x58, 0x44, + 0x00, 0x40, 0x58, 0x44, 0x00, 0x60, 0x58, 0x44, 0x00, 0x80, 0x58, 0x44, 0x00, 0xa0, 0x58, 0x44, 0x00, + 0xc0, 0x58, 0x44, 0x00, 0xe0, 0x58, 0x44, 0x00, 0x00, 0x59, 0x44, 0x00, 0x20, 0x59, 0x44, 0x00, 0x40, + 0x59, 0x44, 0x00, 0x60, 0x59, 0x44, 0x00, 0x80, 0x59, 0x44, 0x00, 0xa0, 0x59, 0x44, 0x00, 0xc0, 0x59, + 0x44, 0x00, 0xe0, 0x59, 0x44, 0x00, 0x00, 0x5a, 0x44, 0x00, 0x20, 0x5a, 0x44, 0x00, 0x40, 0x5a, 0x44, + 0x00, 0x60, 0x5a, 0x44, 0x00, 0x80, 0x5a, 0x44, 0x00, 0xa0, 0x5a, 0x44, 0x00, 0xc0, 0x5a, 0x44, 0x00, + 0xe0, 0x5a, 0x44, 0x00, 0x00, 0x5b, 0x44, 0x00, 0x20, 0x5b, 0x44, 0x00, 0x40, 0x5b, 0x44, 0x00, 0x60, + 0x5b, 0x44, 0x00, 0x80, 0x5b, 0x44, 0x00, 0xa0, 0x5b, 0x44, 0x00, 0xc0, 0x5b, 0x44, 0x00, 0xe0, 0x5b, + 0x44, 0x00, 0x00, 0x5c, 0x44, 0x00, 0x20, 0x5c, 0x44, 0x00, 0x40, 0x5c, 0x44, 0x00, 0x60, 0x5c, 0x44, + 0x00, 0x80, 0x5c, 0x44, 0x00, 0xa0, 0x5c, 0x44, 0x00, 0xc0, 0x5c, 0x44, 0x00, 0xe0, 0x5c, 0x44, 0x00, + 0x00, 0x5d, 0x44, 0x00, 0x20, 0x5d, 0x44, 0x00, 0x40, 0x5d, 0x44, 0x00, 0x60, 0x5d, 0x44, 0x00, 0x80, + 0x5d, 0x44, 0x00, 0xa0, 0x5d, 0x44, 0x00, 0xc0, 0x5d, 0x44, 0x00, 0xe0, 0x5d, 0x44, 0x00, 0x00, 0x5e, + 0x44, 0x00, 0x20, 0x5e, 0x44, 0x00, 0x40, 0x5e, 0x44, 0x00, 0x60, 0x5e, 0x44, 0x00, 0x80, 0x5e, 0x44, + 0x00, 0xa0, 0x5e, 0x44, 0x00, 0xc0, 0x5e, 0x44, 0x00, 0xe0, 0x5e, 0x44, 0x00, 0x00, 0x5f, 0x44, 0x00, + 0x20, 0x5f, 0x44, 0x00, 0x40, 0x5f, 0x44, 0x00, 0x60, 0x5f, 0x44, 0x00, 0x80, 0x5f, 0x44, 0x00, 0xa0, + 0x5f, 0x44, 0x00, 0xc0, 0x5f, 0x44, 0x00, 0xe0, 0x5f, 0x44, 0x00, 0x00, 0x60, 0x44, 0x00, 0x20, 0x60, + 0x44, 0x00, 0x40, 0x60, 0x44, 0x00, 0x60, 0x60, 0x44, 0x00, 0x80, 0x60, 0x44, 0x00, 0xa0, 0x60, 0x44, + 0x00, 0xc0, 0x60, 0x44, 0x00, 0xe0, 0x60, 0x44, 0x00, 0x00, 0x61, 0x44, 0x00, 0x20, 0x61, 0x44, 0x00, + 0x40, 0x61, 0x44, 0x00, 0x60, 0x61, 0x44, 0x00, 0x80, 0x61, 0x44, 0x00, 0xa0, 0x61, 0x44, 0x00, 0xc0, + 0x61, 0x44, 0x00, 0xe0, 0x61, 0x44, 0x00, 0x00, 0x62, 0x44, 0x00, 0x20, 0x62, 0x44, 0x00, 0x40, 0x62, + 0x44, 0x00, 0x60, 0x62, 0x44, 0x00, 0x80, 0x62, 0x44, 0x00, 0xa0, 0x62, 0x44, 0x00, 0xc0, 0x62, 0x44, + 0x00, 0xe0, 0x62, 0x44, 0x00, 0x00, 0x63, 0x44, 0x00, 0x20, 0x63, 0x44, 0x00, 0x40, 0x63, 0x44, 0x00, + 0x60, 0x63, 0x44, 0x00, 0x80, 0x63, 0x44, 0x00, 0xa0, 0x63, 0x44, 0x00, 0xc0, 0x63, 0x44, 0x00, 0xe0, + 0x63, 0x44, 0x00, 0x00, 0x64, 0x44, 0x00, 0x20, 0x64, 0x44, 0x00, 0x40, 0x64, 0x44, 0x00, 0x60, 0x64, + 0x44, 0x00, 0x80, 0x64, 0x44, 0x00, 0xa0, 0x64, 0x44, 0x00, 0xc0, 0x64, 0x44, 0x00, 0xe0, 0x64, 0x44, + 0x00, 0x00, 0x65, 0x44, 0x00, 0x20, 0x65, 0x44, 0x00, 0x40, 0x65, 0x44, 0x00, 0x60, 0x65, 0x44, 0x00, + 0x80, 0x65, 0x44, 0x00, 0xa0, 0x65, 0x44, 0x00, 0xc0, 0x65, 0x44, 0x00, 0xe0, 0x65, 0x44, 0x00, 0x00, + 0x66, 0x44, 0x00, 0x20, 0x66, 0x44, 0x00, 0x40, 0x66, 0x44, 0x00, 0x60, 0x66, 0x44, 0x00, 0x80, 0x66, + 0x44, 0x00, 0xa0, 0x66, 0x44, 0x00, 0xc0, 0x66, 0x44, 0x00, 0xe0, 0x66, 0x44, 0x00, 0x00, 0x67, 0x44, + 0x00, 0x20, 0x67, 0x44, 0x00, 0x40, 0x67, 0x44, 0x00, 0x60, 0x67, 0x44, 0x00, 0x80, 0x67, 0x44, 0x00, + 0xa0, 0x67, 0x44, 0x00, 0xc0, 0x67, 0x44, 0x00, 0xe0, 0x67, 0x44, 0x00, 0x00, 0x68, 0x44, 0x00, 0x20, + 0x68, 0x44, 0x00, 0x40, 0x68, 0x44, 0x00, 0x60, 0x68, 0x44, 0x00, 0x80, 0x68, 0x44, 0x00, 0xa0, 0x68, + 0x44, 0x00, 0xc0, 0x68, 0x44, 0x00, 0xe0, 0x68, 0x44, 0x00, 0x00, 0x69, 0x44, 0x00, 0x20, 0x69, 0x44, + 0x00, 0x40, 0x69, 0x44, 0x00, 0x60, 0x69, 0x44, 0x00, 0x80, 0x69, 0x44, 0x00, 0xa0, 0x69, 0x44, 0x00, + 0xc0, 0x69, 0x44, 0x00, 0xe0, 0x69, 0x44, 0x00, 0x00, 0x6a, 0x44, 0x00, 0x20, 0x6a, 0x44, 0x00, 0x40, + 0x6a, 0x44, 0x00, 0x60, 0x6a, 0x44, 0x00, 0x80, 0x6a, 0x44, 0x00, 0xa0, 0x6a, 0x44, 0x00, 0xc0, 0x6a, + 0x44, 0x00, 0xe0, 0x6a, 0x44, 0x00, 0x00, 0x6b, 0x44, 0x00, 0x20, 0x6b, 0x44, 0x00, 0x40, 0x6b, 0x44, + 0x00, 0x60, 0x6b, 0x44, 0x00, 0x80, 0x6b, 0x44, 0x00, 0xa0, 0x6b, 0x44, 0x00, 0xc0, 0x6b, 0x44, 0x00, + 0xe0, 0x6b, 0x44, 0x00, 0x00, 0x6c, 0x44, 0x00, 0x20, 0x6c, 0x44, 0x00, 0x40, 0x6c, 0x44, 0x00, 0x60, + 0x6c, 0x44, 0x00, 0x80, 0x6c, 0x44, 0x00, 0xa0, 0x6c, 0x44, 0x00, 0xc0, 0x6c, 0x44, 0x00, 0xe0, 0x6c, + 0x44, 0x00, 0x00, 0x6d, 0x44, 0x00, 0x20, 0x6d, 0x44, 0x00, 0x40, 0x6d, 0x44, 0x00, 0x60, 0x6d, 0x44, + 0x00, 0x80, 0x6d, 0x44, 0x00, 0xa0, 0x6d, 0x44, 0x00, 0xc0, 0x6d, 0x44, 0x00, 0xe0, 0x6d, 0x44, 0x00, + 0x00, 0x6e, 0x44, 0x00, 0x20, 0x6e, 0x44, 0x00, 0x40, 0x6e, 0x44, 0x00, 0x60, 0x6e, 0x44, 0x00, 0x80, + 0x6e, 0x44, 0x00, 0xa0, 0x6e, 0x44, 0x00, 0xc0, 0x6e, 0x44, 0x00, 0xe0, 0x6e, 0x44, 0x00, 0x00, 0x6f, + 0x44, 0x00, 0x20, 0x6f, 0x44, 0x00, 0x40, 0x6f, 0x44, 0x00, 0x60, 0x6f, 0x44, 0x00, 0x80, 0x6f, 0x44, + 0x00, 0xa0, 0x6f, 0x44, 0x00, 0xc0, 0x6f, 0x44, 0x00, 0xe0, 0x6f, 0x44, 0x00, 0x00, 0x70, 0x44, 0x00, + 0x20, 0x70, 0x44, 0x00, 0x40, 0x70, 0x44, 0x00, 0x60, 0x70, 0x44, 0x00, 0x80, 0x70, 0x44, 0x00, 0xa0, + 0x70, 0x44, 0x00, 0xc0, 0x70, 0x44, 0x00, 0xe0, 0x70, 0x44, 0x00, 0x00, 0x71, 0x44, 0x00, 0x20, 0x71, + 0x44, 0x00, 0x40, 0x71, 0x44, 0x00, 0x60, 0x71, 0x44, 0x00, 0x80, 0x71, 0x44, 0x00, 0xa0, 0x71, 0x44, + 0x00, 0xc0, 0x71, 0x44, 0x00, 0xe0, 0x71, 0x44, 0x00, 0x00, 0x72, 0x44, 0x00, 0x20, 0x72, 0x44, 0x00, + 0x40, 0x72, 0x44, 0x00, 0x60, 0x72, 0x44, 0x00, 0x80, 0x72, 0x44, 0x00, 0xa0, 0x72, 0x44, 0x00, 0xc0, + 0x72, 0x44, 0x00, 0xe0, 0x72, 0x44, 0x00, 0x00, 0x73, 0x44, 0x00, 0x20, 0x73, 0x44, 0x00, 0x40, 0x73, + 0x44, 0x00, 0x60, 0x73, 0x44, 0x00, 0x80, 0x73, 0x44, 0x00, 0xa0, 0x73, 0x44, 0x00, 0xc0, 0x73, 0x44, + 0x00, 0xe0, 0x73, 0x44, 0x00, 0x00, 0x74, 0x44, 0x00, 0x20, 0x74, 0x44, 0x00, 0x40, 0x74, 0x44, 0x00, + 0x60, 0x74, 0x44, 0x00, 0x80, 0x74, 0x44, 0x00, 0xa0, 0x74, 0x44, 0x00, 0xc0, 0x74, 0x44, 0x00, 0xe0, + 0x74, 0x44, 0x00, 0x00, 0x75, 0x44, 0x00, 0x20, 0x75, 0x44, 0x00, 0x40, 0x75, 0x44, 0x00, 0x60, 0x75, + 0x44, 0x00, 0x80, 0x75, 0x44, 0x00, 0xa0, 0x75, 0x44, 0x00, 0xc0, 0x75, 0x44, 0x00, 0xe0, 0x75, 0x44, + 0x00, 0x00, 0x76, 0x44, 0x00, 0x20, 0x76, 0x44, 0x00, 0x40, 0x76, 0x44, 0x00, 0x60, 0x76, 0x44, 0x00, + 0x80, 0x76, 0x44, 0x00, 0xa0, 0x76, 0x44, 0x00, 0xc0, 0x76, 0x44, 0x00, 0xe0, 0x76, 0x44, 0x00, 0x00, + 0x77, 0x44, 0x00, 0x20, 0x77, 0x44, 0x00, 0x40, 0x77, 0x44, 0x00, 0x60, 0x77, 0x44, 0x00, 0x80, 0x77, + 0x44, 0x00, 0xa0, 0x77, 0x44, 0x00, 0xc0, 0x77, 0x44, 0x00, 0xe0, 0x77, 0x44, 0x00, 0x00, 0x78, 0x44, + 0x00, 0x20, 0x78, 0x44, 0x00, 0x40, 0x78, 0x44, 0x00, 0x60, 0x78, 0x44, 0x00, 0x80, 0x78, 0x44, 0x00, + 0xa0, 0x78, 0x44, 0x00, 0xc0, 0x78, 0x44, 0x00, 0xe0, 0x78, 0x44, 0x00, 0x00, 0x79, 0x44, 0x00, 0x20, + 0x79, 0x44, 0x00, 0x40, 0x79, 0x44, 0x00, 0x60, 0x79, 0x44, 0x00, 0x80, 0x79, 0x44, 0x00, 0xa0, 0x79, + 0x44, 0x00, 0xc0, 0x79, 0x44, 0x00, 0xe0, 0x79, 0x44, 0x00, 0x00, 0x7a, 0x44, 0x00, 0x20, 0x7a, 0x44, + 0x00, 0x40, 0x7a, 0x44, 0x00, 0x60, 0x7a, 0x44, 0x00, 0x80, 0x7a, 0x44, 0x00, 0xa0, 0x7a, 0x44, 0x00, + 0xc0, 0x7a, 0x44, 0x00, 0xe0, 0x7a, 0x44, 0x00, 0x00, 0x7b, 0x44, 0x00, 0x20, 0x7b, 0x44, 0x00, 0x40, + 0x7b, 0x44, 0x00, 0x60, 0x7b, 0x44, 0x00, 0x80, 0x7b, 0x44, 0x00, 0xa0, 0x7b, 0x44, 0x00, 0xc0, 0x7b, + 0x44, 0x00, 0xe0, 0x7b, 0x44, 0x00, 0x00, 0x7c, 0x44, 0x00, 0x20, 0x7c, 0x44, 0x00, 0x40, 0x7c, 0x44, + 0x00, 0x60, 0x7c, 0x44, 0x00, 0x80, 0x7c, 0x44, 0x00, 0xa0, 0x7c, 0x44, 0x00, 0xc0, 0x7c, 0x44, 0x00, + 0xe0, 0x7c, 0x44, 0x00, 0x00, 0x7d, 0x44, 0x00, 0x20, 0x7d, 0x44, 0x00, 0x40, 0x7d, 0x44}; unsigned int ___bin_col_neg_float_cdf_len = 8192; #endif diff --git a/tests/comparators-tests.cpp b/tests/comparators-tests.cpp index f31a2526e..9d237b477 100644 --- a/tests/comparators-tests.cpp +++ b/tests/comparators-tests.cpp @@ -49,409 +49,394 @@ using namespace messageqcpp; // Timer class used by this tdriver to output elapsed times, etc. class Timer { -public: - void start(const string& message) + public: + void start(const string& message) + { + if (!fHeaderDisplayed) { - if (!fHeaderDisplayed) - { - header(); - fHeaderDisplayed = true; - } - - gettimeofday(&fTvStart, 0); - cout << timestr() << " Start " << message << endl; + header(); + fHeaderDisplayed = true; } - void stop(const string& message) - { - time_t now; - time(&now); - string secondsElapsed; - getTimeElapsed(secondsElapsed); - cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; - } + gettimeofday(&fTvStart, 0); + cout << timestr() << " Start " << message << endl; + } - Timer() : fHeaderDisplayed(false) {} + void stop(const string& message) + { + time_t now; + time(&now); + string secondsElapsed; + getTimeElapsed(secondsElapsed); + cout << timestr() << " " << secondsElapsed << " Stop " << message << endl; + } -private: + Timer() : fHeaderDisplayed(false) + { + } - struct timeval fTvStart; - bool fHeaderDisplayed; + private: + struct timeval fTvStart; + bool fHeaderDisplayed; - double getTimeElapsed(string& seconds) - { - struct timeval tvStop; - gettimeofday(&tvStop, 0); - double secondsElapsed = - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); - ostringstream oss; - oss << secondsElapsed; - seconds = oss.str(); - seconds.resize(8, '0'); - return secondsElapsed; - } + double getTimeElapsed(string& seconds) + { + struct timeval tvStop; + gettimeofday(&tvStop, 0); + double secondsElapsed = + (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); + ostringstream oss; + oss << secondsElapsed; + seconds = oss.str(); + seconds.resize(8, '0'); + return secondsElapsed; + } - string timestr() - { - struct tm tm; - struct timeval tv; + string timestr() + { + struct tm tm; + struct timeval tv; - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec << '.' - << setw(6) << tv.tv_usec - ; - return oss.str(); - } + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec + << '.' << setw(6) << tv.tv_usec; + return oss.str(); + } - void header() - { - cout << endl; - cout << "Time Seconds Activity" << endl; - } + void header() + { + cout << endl; + cout << "Time Seconds Activity" << endl; + } }; class FilterDriver : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(FilterDriver); - CPPUNIT_TEST_SUITE(FilterDriver); + CPPUNIT_TEST(INT_TEST); + CPPUNIT_TEST(FLOAT_TEST); + CPPUNIT_TEST(WIDEDT_TEST); - CPPUNIT_TEST(INT_TEST); - CPPUNIT_TEST(FLOAT_TEST); - CPPUNIT_TEST(WIDEDT_TEST); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + // The tests creates an RG with 1 column of the cscDt type + // then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL. + // Then creates comparator structures and run a number of tests. v1 < v2 + void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, uint32_t width, + bool generateRandValues, uint8_t precision) + { + std::cout << std::endl << "------------------------------------------------------------" << std::endl; + uint32_t oid = 3001; + std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector charSetNumVec; + std::vector types; + offsets.push_back(2); + offsets.push_back(2 + width); + roids.push_back(oid); + tkeys.push_back(1); + types.push_back(cscDt); + cscale.push_back(0); + cprecision.push_back(precision); + charSetNumVec.push_back(8); + rowgroup::RowGroup inRG(1, // column count + offsets, // oldOffset + roids, // column oids + tkeys, // keys + types, // types + charSetNumVec, // charset numbers + cscale, // scale + cprecision, // precision + 20, // sTableThreshold + false // useStringTable + ); -private: - // The tests creates an RG with 1 column of the cscDt type - // then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL. - // Then creates comparator structures and run a number of tests. v1 < v2 - void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, - uint32_t width, - bool generateRandValues, - uint8_t precision) + rowgroup::RGData rgD = rowgroup::RGData(inRG); + inRG.setData(&rgD); + rowgroup::Row r, r1, r2, r3; + inRG.initRow(&r); + uint32_t rowSize = r.getSize(); + inRG.getRow(0, &r); + + // Sorting spec describes sorting direction and NULL comparision + // preferences + ordering::IdbSortSpec spec = ordering::IdbSortSpec(0, // column index + true, // ascending + true); // NULLs first + std::vector specVect; + specVect.push_back(spec); + + switch (cscDt) { - std::cout << std::endl << "------------------------------------------------------------" << std::endl; - uint32_t oid =3001; - std::vector offsets, roids, tkeys, cscale, cprecision; - std::vector charSetNumVec; - std::vector types; - offsets.push_back(2); offsets.push_back(2+width); - roids.push_back(oid); - tkeys.push_back(1); - types.push_back(cscDt); - cscale.push_back(0); - cprecision.push_back(precision); - charSetNumVec.push_back(8); - rowgroup::RowGroup inRG(1, //column count - offsets, //oldOffset - roids, // column oids - tkeys, //keys - types, // types - charSetNumVec, // charset numbers - cscale, //scale - cprecision, // precision - 20, // sTableThreshold - false //useStringTable - ); - - rowgroup::RGData rgD = rowgroup::RGData(inRG); - inRG.setData(&rgD); - rowgroup::Row r, r1, r2, r3; - inRG.initRow(&r); - uint32_t rowSize = r.getSize(); - inRG.getRow(0, &r); - - // Sorting spec describes sorting direction and NULL comparision - // preferences - ordering::IdbSortSpec spec = ordering::IdbSortSpec(0, // column index - true, // ascending - true); // NULLs first - std::vector specVect; - specVect.push_back(spec); - - switch(cscDt) + case execplan::CalpontSystemCatalog::UTINYINT: + { + std::cout << "UTINYINT " << std::endl; + r.setUintField<1>(42, 0); + r.nextRow(rowSize); + r.setUintField<1>(43, 0); + r.nextRow(rowSize); + r.setUintField<1>(joblist::UTINYINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::USMALLINT: + { + std::cout << "USMALLINT " << std::endl; + r.setUintField<2>(42, 0); + r.nextRow(rowSize); + r.setUintField<2>(43, 0); + r.nextRow(rowSize); + r.setUintField<2>(joblist::USMALLINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + { + std::cout << "UINT " << std::endl; + r.setUintField<4>(42, 0); + r.nextRow(rowSize); + r.setUintField<4>(43, 0); + r.nextRow(rowSize); + r.setUintField<4>(joblist::UINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::UBIGINT: + { + std::cout << "UBIGINT " << std::endl; + r.setUintField<8>(42, 0); + r.nextRow(rowSize); + r.setUintField<8>(43, 0); + r.nextRow(rowSize); + r.setUintField<8>(joblist::UBIGINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::TINYINT: + { + std::cout << "TINYINT " << std::endl; + r.setIntField<1>(42, 0); + r.nextRow(rowSize); + r.setIntField<1>(43, 0); + r.nextRow(rowSize); + r.setIntField<1>(joblist::TINYINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::SMALLINT: + { + std::cout << "SMALLINT " << std::endl; + r.setIntField<2>(42, 0); + r.nextRow(rowSize); + r.setIntField<2>(43, 0); + r.nextRow(rowSize); + r.setIntField<2>(joblist::SMALLINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::DATE: + { + if (cscDt == execplan::CalpontSystemCatalog::DATE) + std::cout << "DATE" << std::endl; + else + std::cout << "INT " << std::endl; + r.setIntField<4>(42, 0); + r.nextRow(rowSize); + r.setIntField<4>(43, 0); + r.nextRow(rowSize); + if (cscDt == execplan::CalpontSystemCatalog::DATE) + r.setIntField<4>(joblist::DATENULL, 0); + else + r.setIntField<4>(joblist::INTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::BIGINT: + { + std::cout << "BIGINT " << std::endl; + r.setIntField<8>(42, 0); + r.nextRow(rowSize); + r.setIntField<8>(43, 0); + r.nextRow(rowSize); + r.setIntField<8>(joblist::BIGINTNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + std::cout << "DECIMAL " << std::endl; + switch (width) { - case execplan::CalpontSystemCatalog::UTINYINT: - { - std::cout << "UTINYINT " << std::endl; - r.setUintField<1>(42, 0); - r.nextRow(rowSize); - r.setUintField<1>(43, 0); - r.nextRow(rowSize); - r.setUintField<1>(joblist::UTINYINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::USMALLINT: - { - std::cout << "USMALLINT " << std::endl; - r.setUintField<2>(42, 0); - r.nextRow(rowSize); - r.setUintField<2>(43, 0); - r.nextRow(rowSize); - r.setUintField<2>(joblist::USMALLINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - std::cout << "UINT " << std::endl; - r.setUintField<4>(42, 0); - r.nextRow(rowSize); - r.setUintField<4>(43, 0); - r.nextRow(rowSize); - r.setUintField<4>(joblist::UINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::UBIGINT: - { - std::cout << "UBIGINT " << std::endl; - r.setUintField<8>(42, 0); - r.nextRow(rowSize); - r.setUintField<8>(43, 0); - r.nextRow(rowSize); - r.setUintField<8>(joblist::UBIGINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::TINYINT: - { - std::cout << "TINYINT " << std::endl; - r.setIntField<1>(42, 0); - r.nextRow(rowSize); - r.setIntField<1>(43, 0); - r.nextRow(rowSize); - r.setIntField<1>(joblist::TINYINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::SMALLINT: - { - std::cout << "SMALLINT " << std::endl; - r.setIntField<2>(42, 0); - r.nextRow(rowSize); - r.setIntField<2>(43, 0); - r.nextRow(rowSize); - r.setIntField<2>(joblist::SMALLINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::DATE: - { - if (cscDt == execplan::CalpontSystemCatalog::DATE) - std::cout << "DATE" << std::endl; - else - std::cout << "INT " << std::endl; - r.setIntField<4>(42, 0); - r.nextRow(rowSize); - r.setIntField<4>(43, 0); - r.nextRow(rowSize); - if (cscDt == execplan::CalpontSystemCatalog::DATE) - r.setIntField<4>(joblist::DATENULL, 0); - else - r.setIntField<4>(joblist::INTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::BIGINT: - { - std::cout << "BIGINT " << std::endl; - r.setIntField<8>(42, 0); - r.nextRow(rowSize); - r.setIntField<8>(43, 0); - r.nextRow(rowSize); - r.setIntField<8>(joblist::BIGINTNULL, 0); - break; - } - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - std::cout << "DECIMAL " << std::endl; - switch (width) - { - case 1 : - { - r.setUintField<1>(42, 0); - r.nextRow(rowSize); - r.setUintField<1>(43, 0); - r.nextRow(rowSize); - r.setUintField<1>(joblist::TINYINTNULL, 0); - break; - } - case 2 : - { - r.setUintField<2>(42, 0); - r.nextRow(rowSize); - r.setUintField<2>(43, 0); - r.nextRow(rowSize); - r.setUintField<2>(joblist::SMALLINTNULL, 0); - break; - } - case 4 : - { - r.setUintField<4>(42, 0); - r.nextRow(rowSize); - r.setUintField<4>(43, 0); - r.nextRow(rowSize); - r.setUintField<4>(joblist::INTNULL, 0); - break; - } - case 8 : - { - r.setUintField<8>(42, 0); - r.nextRow(rowSize); - r.setUintField<8>(43, 0); - r.nextRow(rowSize); - r.setUintField<8>(joblist::BIGINTNULL, 0); - break; - } - case 16 : - { - uint128_t dec = 42; - r.setBinaryField(&dec, 0); - r.nextRow(rowSize); - dec++; - r.setBinaryField(&dec, 0); - r.nextRow(rowSize); - r.setBinaryField( - const_cast(&datatypes::Decimal128Null), - 0); - break; - } - default : - std::cout << " This is the end. My only friend the end... " << std::endl; - } - break; - } - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - std::cout << "FLOAT " << std::endl; - r.setFloatField(42.1, 0); - r.nextRow(rowSize); - r.setFloatField(43.1, 0); - r.nextRow(rowSize); - r.setUintField(joblist::FLOATNULL, 0); - break; - - } - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - std::cout << "DOUBLE " << std::endl; - r.setDoubleField(42.1, 0); - r.nextRow(rowSize); - r.setDoubleField(43.1, 0); - r.nextRow(rowSize); - r.setUintField(joblist::DOUBLENULL, 0); - break; - } - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - r.setLongDoubleField(42.1, 0); - r.nextRow(rowSize); - r.setLongDoubleField(43.1, 0); - r.nextRow(rowSize); - r.setLongDoubleField(joblist::LONGDOUBLENULL, 0); - break; - } - default: - { - break; - } + case 1: + { + r.setUintField<1>(42, 0); + r.nextRow(rowSize); + r.setUintField<1>(43, 0); + r.nextRow(rowSize); + r.setUintField<1>(joblist::TINYINTNULL, 0); + break; + } + case 2: + { + r.setUintField<2>(42, 0); + r.nextRow(rowSize); + r.setUintField<2>(43, 0); + r.nextRow(rowSize); + r.setUintField<2>(joblist::SMALLINTNULL, 0); + break; + } + case 4: + { + r.setUintField<4>(42, 0); + r.nextRow(rowSize); + r.setUintField<4>(43, 0); + r.nextRow(rowSize); + r.setUintField<4>(joblist::INTNULL, 0); + break; + } + case 8: + { + r.setUintField<8>(42, 0); + r.nextRow(rowSize); + r.setUintField<8>(43, 0); + r.nextRow(rowSize); + r.setUintField<8>(joblist::BIGINTNULL, 0); + break; + } + case 16: + { + uint128_t dec = 42; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + dec++; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + r.setBinaryField(const_cast(&datatypes::Decimal128Null), 0); + break; + } + default: std::cout << " This is the end. My only friend the end... " << std::endl; } - - inRG.setRowCount(3); - inRG.initRow(&r1); - inRG.initRow(&r2); - inRG.initRow(&r3); - inRG.getRow(0, &r1); - inRG.getRow(1, &r2); - inRG.getRow(2, &r3); - - std::cout<< "r1 " << r1.toString() << " r2 " << r2.toString() - << " r3 " << r3.toString() << std::endl; - - ordering::IdbCompare idbCompare; - idbCompare.initialize(inRG); - ordering::OrderByData odbData = ordering::OrderByData(specVect, inRG); - bool result = odbData(r1.getPointer(), r2.getPointer()); - std::cout << r1.toString() << " < " << r2.toString() << " is " - << ((result) ? "true" : "false") << std::endl; - CPPUNIT_ASSERT(result == true); - result = odbData(r2.getPointer(), r1.getPointer()); - std::cout << r2.toString() << " < " << r1.toString() << " is " - << ((result) ? "true" : "false") << std::endl; - CPPUNIT_ASSERT(result == false); - result = odbData(r2.getPointer(), r2.getPointer()); - std::cout << r2.toString() << " < " << r2.toString() << " is " - << ((result) ? "true" : "false") << std::endl; - CPPUNIT_ASSERT(result == false); - // Compare value with NULL. if spec.fNf then NULLs goes first - result = odbData(r3.getPointer(), r1.getPointer()); - std::cout << r3.toString() << " < " << r1.toString() << " is " - << ((result) ? "true" : "false") << std::endl; - CPPUNIT_ASSERT(result == true); - // Compare NULL with NULL - result = odbData(r3.getPointer(), r1.getPointer()); - std::cout << r3.toString() << " < " << r3.toString() << " is " - << ((result) ? "true" : "false") << std::endl; - CPPUNIT_ASSERT(result == true); + break; + } + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + std::cout << "FLOAT " << std::endl; + r.setFloatField(42.1, 0); + r.nextRow(rowSize); + r.setFloatField(43.1, 0); + r.nextRow(rowSize); + r.setUintField(joblist::FLOATNULL, 0); + break; + } + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + std::cout << "DOUBLE " << std::endl; + r.setDoubleField(42.1, 0); + r.nextRow(rowSize); + r.setDoubleField(43.1, 0); + r.nextRow(rowSize); + r.setUintField(joblist::DOUBLENULL, 0); + break; + } + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + r.setLongDoubleField(42.1, 0); + r.nextRow(rowSize); + r.setLongDoubleField(43.1, 0); + r.nextRow(rowSize); + r.setLongDoubleField(joblist::LONGDOUBLENULL, 0); + break; + } + default: + { + break; + } } - void INT_TEST() - { + inRG.setRowCount(3); + inRG.initRow(&r1); + inRG.initRow(&r2); + inRG.initRow(&r3); + inRG.getRow(0, &r1); + inRG.getRow(1, &r2); + inRG.getRow(2, &r3); + + std::cout << "r1 " << r1.toString() << " r2 " << r2.toString() << " r3 " << r3.toString() << std::endl; + + ordering::IdbCompare idbCompare; + idbCompare.initialize(inRG); + ordering::OrderByData odbData = ordering::OrderByData(specVect, inRG); + bool result = odbData(r1.getPointer(), r2.getPointer()); + std::cout << r1.toString() << " < " << r2.toString() << " is " << ((result) ? "true" : "false") + << std::endl; + CPPUNIT_ASSERT(result == true); + result = odbData(r2.getPointer(), r1.getPointer()); + std::cout << r2.toString() << " < " << r1.toString() << " is " << ((result) ? "true" : "false") + << std::endl; + CPPUNIT_ASSERT(result == false); + result = odbData(r2.getPointer(), r2.getPointer()); + std::cout << r2.toString() << " < " << r2.toString() << " is " << ((result) ? "true" : "false") + << std::endl; + CPPUNIT_ASSERT(result == false); + // Compare value with NULL. if spec.fNf then NULLs goes first + result = odbData(r3.getPointer(), r1.getPointer()); + std::cout << r3.toString() << " < " << r1.toString() << " is " << ((result) ? "true" : "false") + << std::endl; + CPPUNIT_ASSERT(result == true); + // Compare NULL with NULL + result = odbData(r3.getPointer(), r1.getPointer()); + std::cout << r3.toString() << " < " << r3.toString() << " is " << ((result) ? "true" : "false") + << std::endl; + CPPUNIT_ASSERT(result == true); + } + + void INT_TEST() + { #ifdef __x86_64__ - //bool generateValues = true; - bool fixedValues = false; - testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues, 20); + // bool generateValues = true; + bool fixedValues = false; + testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues, 20); #elif __arm__ - // TODO: add arm tests + // TODO: add arm tests #endif + } - } + void FLOAT_TEST() + { + } - void FLOAT_TEST() - { - } - - void WIDEDT_TEST() - { - bool fixedValues = false; - testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 16, fixedValues, 38); - } + void WIDEDT_TEST() + { + bool fixedValues = false; + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 16, fixedValues, 38); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver); - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/tests/compression-tests.cpp b/tests/compression-tests.cpp index b0d0868b4..256b81f79 100644 --- a/tests/compression-tests.cpp +++ b/tests/compression-tests.cpp @@ -23,104 +23,86 @@ class CompressionTest : public ::testing::Test { + protected: + std::string genPermutations(string& data) + { + std::string generated; + generate(data, 0, generated); + return generated; + } - protected: - std::string genPermutations(string& data) + private: + void generate(string& data, uint32_t i, std::string& generated) + { + if (i == data.size()) { - std::string generated; - generate(data, 0, generated); - return generated; + generated.append(data); + return; } - private: - void generate(string& data, uint32_t i, std::string& generated) + for (uint32_t k = i, e = data.size(); k < e; ++k) { - if (i == data.size()) - { - generated.append(data); - return; - } - - for (uint32_t k = i, e = data.size(); k < e; ++k) - { - std::swap(data[i], data[k]); - generate(data, i + 1, generated); - std::swap(data[i], data[k]); - } + std::swap(data[i], data[k]); + generate(data, i + 1, generated); + std::swap(data[i], data[k]); } + } }; TEST_F(CompressionTest, LZ4CanCompress) { - std::string originalData = - "This program is free software; you can redistribute it and/or" - "modify it under the terms of the GNU General Public License" - "as published by the Free Software Foundation; version 2 of" - "the License."; + std::string originalData = + "This program is free software; you can redistribute it and/or" + "modify it under the terms of the GNU General Public License" + "as published by the Free Software Foundation; version 2 of" + "the License."; - std::unique_ptr compressor( - new compress::CompressInterfaceLZ4()); + std::unique_ptr compressor(new compress::CompressInterfaceLZ4()); - size_t originalSize = originalData.size(); - size_t compressedSize = compressor->maxCompressedSize(originalSize); - std::unique_ptr compressedData(new char[compressedSize]); - std::memset(compressedData.get(), 0, compressedSize); + size_t originalSize = originalData.size(); + size_t compressedSize = compressor->maxCompressedSize(originalSize); + std::unique_ptr compressedData(new char[compressedSize]); + std::memset(compressedData.get(), 0, compressedSize); - auto rc = compressor->compress(originalData.data(), originalSize, - compressedData.get(), &compressedSize); - ASSERT_EQ(rc, 0); + auto rc = compressor->compress(originalData.data(), originalSize, compressedData.get(), &compressedSize); + ASSERT_EQ(rc, 0); - std::unique_ptr uncompressedData(new char[originalSize]); - rc = compressor->uncompress(compressedData.get(), compressedSize, - uncompressedData.get(), &originalSize); - ASSERT_EQ(rc, 0); - std::string result(uncompressedData.get()); - EXPECT_EQ(originalData, result); + std::unique_ptr uncompressedData(new char[originalSize]); + rc = compressor->uncompress(compressedData.get(), compressedSize, uncompressedData.get(), &originalSize); + ASSERT_EQ(rc, 0); + std::string result(uncompressedData.get()); + EXPECT_EQ(originalData, result); } TEST_F(CompressionTest, LZvsSnappyUnique) { - std::unique_ptr lz4Compressor( - new compress::CompressInterfaceLZ4()); - std::unique_ptr snappyCompressor( - new compress::CompressInterfaceSnappy()); - // Generate permutations. - // 9! * 9 == 3265920 (closer to current chunk size) - std::vector dataPool{"abcdefghi", "aaadefghi", "aaaaafghi", - "aaaaaaahi", "aaaaaaaaj"}; + std::unique_ptr lz4Compressor(new compress::CompressInterfaceLZ4()); + std::unique_ptr snappyCompressor(new compress::CompressInterfaceSnappy()); + // Generate permutations. + // 9! * 9 == 3265920 (closer to current chunk size) + std::vector dataPool{"abcdefghi", "aaadefghi", "aaaaafghi", "aaaaaaahi", "aaaaaaaaj"}; - for (auto& data : dataPool) - { - std::cout << "Permutations generated for: " << data << std::endl; - auto generated = genPermutations(data); - auto generatedSize = generated.size(); + for (auto& data : dataPool) + { + std::cout << "Permutations generated for: " << data << std::endl; + auto generated = genPermutations(data); + auto generatedSize = generated.size(); - auto compressedSizeLZ4 = - lz4Compressor->maxCompressedSize(generatedSize); - auto compressedSizeSnappy = - snappyCompressor->maxCompressedSize(generatedSize); + auto compressedSizeLZ4 = lz4Compressor->maxCompressedSize(generatedSize); + auto compressedSizeSnappy = snappyCompressor->maxCompressedSize(generatedSize); - std::unique_ptr lz4CompressedData(new char[compressedSizeLZ4]); - auto rc = lz4Compressor->compress(generated.data(), generatedSize, - lz4CompressedData.get(), - &compressedSizeLZ4); - ASSERT_EQ(rc, 0); + std::unique_ptr lz4CompressedData(new char[compressedSizeLZ4]); + auto rc = + lz4Compressor->compress(generated.data(), generatedSize, lz4CompressedData.get(), &compressedSizeLZ4); + ASSERT_EQ(rc, 0); - std::unique_ptr snappyCompressedData( - new char[compressedSizeSnappy]); - rc = snappyCompressor->compress(generated.data(), generatedSize, - snappyCompressedData.get(), - &compressedSizeSnappy); - ASSERT_EQ(rc, 0); + std::unique_ptr snappyCompressedData(new char[compressedSizeSnappy]); + rc = snappyCompressor->compress(generated.data(), generatedSize, snappyCompressedData.get(), + &compressedSizeSnappy); + ASSERT_EQ(rc, 0); - std::cout << "LZ ratio: " - << (float) ((float) generatedSize / - (float) compressedSizeLZ4) - << std::endl; + std::cout << "LZ ratio: " << (float)((float)generatedSize / (float)compressedSizeLZ4) << std::endl; - std::cout << "Snappy ratio: " - << (float) ((float) generatedSize / - (float) compressedSizeSnappy) - << std::endl; - } + std::cout << "Snappy ratio: " << (float)((float)generatedSize / (float)compressedSizeSnappy) << std::endl; + } } diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp index a6c7179b5..627f3eed2 100644 --- a/tests/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -27,519 +27,582 @@ using namespace dataconvert; TEST(DataConvertTest, Strtoll128) { - char *ep = NULL; - bool saturate = false; - string str; - int128_t val, valMax; - bitset<64> b1, b2, b3, b4; + char* ep = NULL; + bool saturate = false; + string str; + int128_t val, valMax; + bitset<64> b1, b2, b3, b4; - // test empty - str = ""; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, 0); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); + // test empty + str = ""; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 0); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); - // test simple values - str = "123"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, 123); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); - str = " 123"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, 123); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); - str = " 123.45"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, 123); - EXPECT_NE(*ep, '\0'); - EXPECT_FALSE(saturate); - str = "-123"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, -123); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); - str = " -123"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, -123); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); - str = " -123.45"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - EXPECT_EQ(val, -123); - EXPECT_NE(*ep, '\0'); - EXPECT_FALSE(saturate); + // test simple values + str = "123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " 123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " 123.45"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_NE(*ep, '\0'); + EXPECT_FALSE(saturate); + str = "-123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " -123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " -123.45"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_NE(*ep, '\0'); + EXPECT_FALSE(saturate); - // test max/min values - // test max - str = "170141183460469231731687303715884105727"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - b1 = *(reinterpret_cast(&val)); - b2 = *(reinterpret_cast(&val) + 1); - valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); - // test min - str = "-170141183460469231731687303715884105728"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - b1 = *(reinterpret_cast(&val)); - b2 = *(reinterpret_cast(&val) + 1); - valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; - valMax = -valMax - 1; - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_EQ(*ep, '\0'); - EXPECT_FALSE(saturate); + // test max/min values + // test max + str = "170141183460469231731687303715884105727"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000) + + 715884105) * + 1000) + + 727; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + // test min + str = "-170141183460469231731687303715884105728"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000) + + 715884105) * + 1000) + + 727; + valMax = -valMax - 1; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); - // test saturation - // test saturation to max - str = "170141183460469231731687303715884105728"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - b1 = *(reinterpret_cast(&val)); - b2 = *(reinterpret_cast(&val) + 1); - valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_EQ(*ep, '\0'); - EXPECT_TRUE(saturate); - // test saturation to min - str = "-170141183460469231731687303715884105729"; - saturate = false; - val = strtoll128(str.c_str(), saturate, &ep); - b1 = *(reinterpret_cast(&val)); - b2 = *(reinterpret_cast(&val) + 1); - valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; - valMax = -valMax - 1; - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_EQ(*ep, '\0'); - EXPECT_TRUE(saturate); + // test saturation + // test saturation to max + str = "170141183460469231731687303715884105728"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000) + + 715884105) * + 1000) + + 727; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_TRUE(saturate); + // test saturation to min + str = "-170141183460469231731687303715884105729"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000) + + 715884105) * + 1000) + + 727; + valMax = -valMax - 1; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_TRUE(saturate); } TEST(DataConvertTest, NumberIntValue) { - datatypes::SystemCatalog::TypeAttributesStd ct; + datatypes::SystemCatalog::TypeAttributesStd ct; - int128_t res, valMax; - string data; - bool noRoundup = false; - bool pushWarning; - bitset<64> b1, b2, b3, b4; + int128_t res, valMax; + string data; + bool noRoundup = false; + bool pushWarning; + bitset<64> b1, b2, b3, b4; - // tests for signed decimal - // behaviour of number_int_value for unsigned decimal - // is similar to the signed case. - datatypes::SystemCatalog::ColDataType typecode = datatypes::SystemCatalog::DECIMAL; - // test with decimal(38,0) - ct.precision = 38; - ct.scale = 0; - // test simple values - //data = ""; - data = "0"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 0); - EXPECT_FALSE(pushWarning); - data = "1234"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 1234); - EXPECT_FALSE(pushWarning); - data = "12.0"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 12); - EXPECT_FALSE(pushWarning); - data = "12.34"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 12); - EXPECT_TRUE(pushWarning); - data = "-1234"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, -1234); - EXPECT_FALSE(pushWarning); - data = "-12.34"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, -12); - EXPECT_TRUE(pushWarning); - // test max - data = "99999999999999999999999999999999999999"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test min - data = "-99999999999999999999999999999999999999"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test rounding - data = "12.56"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 13); - EXPECT_TRUE(pushWarning); - data = "-12.56"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, -13); - EXPECT_TRUE(pushWarning); - // test saturation - data = "999999999999999999999999999999999999999"; // data has 39 9's - // valMax has 38 9's - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - data = "-999999999999999999999999999999999999999"; // data has 39 9's - // valMax has 38 9's - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - // test scientific notation - data = "1.23e37"; - valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - data = "1.23e38"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); + // tests for signed decimal + // behaviour of number_int_value for unsigned decimal + // is similar to the signed case. + datatypes::SystemCatalog::ColDataType typecode = datatypes::SystemCatalog::DECIMAL; + // test with decimal(38,0) + ct.precision = 38; + ct.scale = 0; + // test simple values + // data = ""; + data = "0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 1234); + EXPECT_FALSE(pushWarning); + data = "12.0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12); + EXPECT_FALSE(pushWarning); + data = "12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12); + EXPECT_TRUE(pushWarning); + data = "-1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -1234); + EXPECT_FALSE(pushWarning); + data = "-12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -12); + EXPECT_TRUE(pushWarning); + // test max + data = "99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "12.56"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 13); + EXPECT_TRUE(pushWarning); + data = "-12.56"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -13); + EXPECT_TRUE(pushWarning); + // test saturation + data = "999999999999999999999999999999999999999"; // data has 39 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-999999999999999999999999999999999999999"; // data has 39 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "1.23e37"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "1.23e38"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); - // test with decimal(38,10) - ct.scale = 10; - data = "0"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 0); - EXPECT_FALSE(pushWarning); - data = "1234"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 12340000000000); - EXPECT_FALSE(pushWarning); - data = "12.0"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 120000000000); - EXPECT_FALSE(pushWarning); - data = "12.34"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 123400000000); - EXPECT_FALSE(pushWarning); - data = "-1234"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, -12340000000000); - EXPECT_FALSE(pushWarning); - data = "-12.34"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, -123400000000); - EXPECT_FALSE(pushWarning); - // test max - data = "9999999999999999999999999999.9999999999"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test min - data = "-9999999999999999999999999999.9999999999"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test rounding - data = "12.11111111119"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 121111111112); - EXPECT_TRUE(pushWarning); - data = "-12.11111111119"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, -121111111112); - EXPECT_TRUE(pushWarning); - // test saturation - data = "99999999999999999999999999999"; // data has 29 9's - // valMax has 38 9's - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - data = "-99999999999999999999999999999"; // data has 29 9's - // valMax has 38 9's - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - // test scientific notation - data = "1.23e9"; - valMax = ((((int128_t)123000000 * 1000000000) + 0) * 100); - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - data = "1.23e28"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - // test with decimal(38,38) - ct.scale = 38; - data = "0"; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - EXPECT_EQ(res, 0); - EXPECT_FALSE(pushWarning); - data = "1.234"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - data = "0.123"; - valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - data = "-1.234"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - data = "-0.123"; - valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test max - data = "0.99999999999999999999999999999999999999"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test min - data = "-0.99999999999999999999999999999999999999"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - // test rounding - data = "0.199999999999999999999999999999999999999"; - valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - data = "-0.199999999999999999999999999999999999999"; - valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - // test saturation - data = "99999999999999999999999999999"; // data has 29 9's - // valMax has 38 9's - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - data = "-99999999999999999999999999999"; // data has 29 9's - // valMax has 38 9's - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - valMax = -valMax; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); - // test scientific notation - data = "123e-4"; - valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 10) + 0; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_FALSE(pushWarning); - data = "123e-2"; - valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - pushWarning = false; - number_int_value(data, typecode, ct, pushWarning, noRoundup, res); - b1 = *(reinterpret_cast(&res)); - b2 = *(reinterpret_cast(&res) + 1); - b3 = *(reinterpret_cast(&valMax)); - b4 = *(reinterpret_cast(&valMax) + 1); - EXPECT_EQ(b1, b3); - EXPECT_EQ(b2, b4); - EXPECT_TRUE(pushWarning); + // test with decimal(38,10) + ct.scale = 10; + data = "0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12340000000000); + EXPECT_FALSE(pushWarning); + data = "12.0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 120000000000); + EXPECT_FALSE(pushWarning); + data = "12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 123400000000); + EXPECT_FALSE(pushWarning); + data = "-1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -12340000000000); + EXPECT_FALSE(pushWarning); + data = "-12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -123400000000); + EXPECT_FALSE(pushWarning); + // test max + data = "9999999999999999999999999999.9999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-9999999999999999999999999999.9999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "12.11111111119"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 121111111112); + EXPECT_TRUE(pushWarning); + data = "-12.11111111119"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -121111111112); + EXPECT_TRUE(pushWarning); + // test saturation + data = "99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "1.23e9"; + valMax = ((((int128_t)123000000 * 1000000000) + 0) * 100); + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "1.23e28"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test with decimal(38,38) + ct.scale = 38; + data = "0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1.234"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "0.123"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "-1.234"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-0.123"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000) + 0) * 100) + 0; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test max + data = "0.99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-0.99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "0.199999999999999999999999999999999999999"; + valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-0.199999999999999999999999999999999999999"; + valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000) + 0) * 100) + 0; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test saturation + data = "99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "123e-4"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000) + 0) * 10) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "123e-2"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); } TEST(DataConvertTest, ConvertColumnData) { diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp index 62598d7af..354ab9b45 100644 --- a/tests/mcs_decimal-tests.cpp +++ b/tests/mcs_decimal-tests.cpp @@ -25,1572 +25,1571 @@ TEST(Decimal, compareCheckInt64) { - // remainder-based checks - // Equality checks - // l value = r value, L scale = R scale - { - datatypes::Decimal l(420, 10, 18); - datatypes::Decimal r(420, 10, 18); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // remainder-based checks + // Equality checks + // l value = r value, L scale = R scale + { + datatypes::Decimal l(420, 10, 18); + datatypes::Decimal r(420, 10, 18); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(420, 11, 18); - datatypes::Decimal r(42, 10, 18); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(420, 11, 18); + datatypes::Decimal r(42, 10, 18); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value < r value, L scale > R scale - { - datatypes::Decimal l(42, 10, 18); - datatypes::Decimal r(420, 11, 18); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value < r value, L scale > R scale + { + datatypes::Decimal l(42, 10, 18); + datatypes::Decimal r(420, 11, 18); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // Inequality checks - // l value = r value, L scale < R scale - { - datatypes::Decimal l(42, 10, 18); - datatypes::Decimal r(42, 13, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // Inequality checks + // l value = r value, L scale < R scale + { + datatypes::Decimal l(42, 10, 18); + datatypes::Decimal r(42, 13, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value = r value, L scale < R scale - { - datatypes::Decimal l(42, 13, 18); - datatypes::Decimal r(42, 10, 18); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value = r value, L scale < R scale + { + datatypes::Decimal l(42, 13, 18); + datatypes::Decimal r(42, 10, 18); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale > R scale - { - datatypes::Decimal l(420, 13, 18); - datatypes::Decimal r(42, 10, 18); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale > R scale + { + datatypes::Decimal l(420, 13, 18); + datatypes::Decimal r(42, 10, 18); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(4200, 11, 18); - datatypes::Decimal r(42, 12, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(4200, 11, 18); + datatypes::Decimal r(42, 12, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(99, 10, 18); - datatypes::Decimal r(420, 11, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(99, 10, 18); + datatypes::Decimal r(420, 11, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(42, 10, 18); - datatypes::Decimal r(420, 13, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(42, 10, 18); + datatypes::Decimal r(420, 13, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // quotinent-based checks - // l value = r value, L scale = R scale - { - datatypes::Decimal l(420, 1, 18); - datatypes::Decimal r(420, 1, 18); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // quotinent-based checks + // l value = r value, L scale = R scale + { + datatypes::Decimal l(420, 1, 18); + datatypes::Decimal r(420, 1, 18); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(4200, 2, 18); - datatypes::Decimal r(420, 1, 18); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(4200, 2, 18); + datatypes::Decimal r(420, 1, 18); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value < r value, L scale > R scale - { - datatypes::Decimal l(42, 0, 18); - datatypes::Decimal r(420, 1, 18); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value < r value, L scale > R scale + { + datatypes::Decimal l(42, 0, 18); + datatypes::Decimal r(420, 1, 18); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // Inequality checks - // l value = r value, L scale < R scale - { - datatypes::Decimal l(42000, 0, 18); - datatypes::Decimal r(42000, 3, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // Inequality checks + // l value = r value, L scale < R scale + { + datatypes::Decimal l(42000, 0, 18); + datatypes::Decimal r(42000, 3, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value = r value, L scale < R scale - { - datatypes::Decimal l(420000, 3, 18); - datatypes::Decimal r(420000, 0, 18); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value = r value, L scale < R scale + { + datatypes::Decimal l(420000, 3, 18); + datatypes::Decimal r(420000, 0, 18); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale > R scale - { - datatypes::Decimal l(420000, 3, 18); - datatypes::Decimal r(42000, 0, 18); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale > R scale + { + datatypes::Decimal l(420000, 3, 18); + datatypes::Decimal r(42000, 0, 18); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(420000, 1, 18); - datatypes::Decimal r(4200, 2, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(420000, 1, 18); + datatypes::Decimal r(4200, 2, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(990, 0, 18); - datatypes::Decimal r(4200, 1, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(990, 0, 18); + datatypes::Decimal r(4200, 1, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(420, 0, 18); - datatypes::Decimal r(4200, 3, 18); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(420, 0, 18); + datatypes::Decimal r(4200, 3, 18); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } } TEST(Decimal, compareCheckInt128) { - // remainer-based checks - // Equality checks - // l value = r value, L scale = R scale - { - datatypes::Decimal l(datatypes::TSInt128(420), 20, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 20, 38); - EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // remainer-based checks + // Equality checks + // l value = r value, L scale = R scale + { + datatypes::Decimal l(datatypes::TSInt128(420), 20, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 20, 38); + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(420), 21, 38); - datatypes::Decimal r(datatypes::TSInt128(42), 20, 38); - EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(420), 21, 38); + datatypes::Decimal r(datatypes::TSInt128(42), 20, 38); + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value < r value, L scale > R scale - { - datatypes::Decimal l(datatypes::TSInt128(42), 20, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 21, 38); - EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value < r value, L scale > R scale + { + datatypes::Decimal l(datatypes::TSInt128(42), 20, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 21, 38); + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // Inequality checks - // l value = r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(42), 20, 38); - datatypes::Decimal r(datatypes::TSInt128(42), 23, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // Inequality checks + // l value = r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(42), 20, 38); + datatypes::Decimal r(datatypes::TSInt128(42), 23, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value = r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(42), 23, 38); - datatypes::Decimal r(datatypes::TSInt128(42), 20, 38); - EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value = r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(42), 23, 38); + datatypes::Decimal r(datatypes::TSInt128(42), 20, 38); + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale > R scale - { - datatypes::Decimal l(datatypes::TSInt128(420), 23, 38); - datatypes::Decimal r(datatypes::TSInt128(42), 20, 38); - EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale > R scale + { + datatypes::Decimal l(datatypes::TSInt128(420), 23, 38); + datatypes::Decimal r(datatypes::TSInt128(42), 20, 38); + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(4200), 21, 38); - datatypes::Decimal r(datatypes::TSInt128(42), 22, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(4200), 21, 38); + datatypes::Decimal r(datatypes::TSInt128(42), 22, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(99), 20, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 21, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(99), 20, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 21, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(42), 20, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 23, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(42), 20, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 23, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // quotinent-based checks - // l value = r value, L scale = R scale - { - datatypes::Decimal l(datatypes::TSInt128(420), 1, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); - EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // quotinent-based checks + // l value = r value, L scale = R scale + { + datatypes::Decimal l(datatypes::TSInt128(420), 1, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(4200), 2, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); - EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(4200), 2, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value < r value, L scale > R scale - { - datatypes::Decimal l(datatypes::TSInt128(42), 0, 38); - datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); - EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value < r value, L scale > R scale + { + datatypes::Decimal l(datatypes::TSInt128(42), 0, 38); + datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // Inequality checks - // l value = r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(42000), 0, 38); - datatypes::Decimal r(datatypes::TSInt128(42000), 3, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // Inequality checks + // l value = r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(42000), 0, 38); + datatypes::Decimal r(datatypes::TSInt128(42000), 3, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value = r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(420000), 3, 38); - datatypes::Decimal r(datatypes::TSInt128(420000), 0, 38); - EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value = r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(420000), 3, 38); + datatypes::Decimal r(datatypes::TSInt128(420000), 0, 38); + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale > R scale - { - datatypes::Decimal l(datatypes::TSInt128(420000), 3, 38); - datatypes::Decimal r(datatypes::TSInt128(42000), 0, 38); - EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale > R scale + { + datatypes::Decimal l(datatypes::TSInt128(420000), 3, 38); + datatypes::Decimal r(datatypes::TSInt128(42000), 0, 38); + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(420000), 1, 38); - datatypes::Decimal r(datatypes::TSInt128(4200), 2, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(420000), 1, 38); + datatypes::Decimal r(datatypes::TSInt128(4200), 2, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(990), 0, 38); - datatypes::Decimal r(datatypes::TSInt128(4200), 1, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(990), 0, 38); + datatypes::Decimal r(datatypes::TSInt128(4200), 1, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(datatypes::TSInt128(420), 0, 38); - datatypes::Decimal r(datatypes::TSInt128(4200), 3, 38); - EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(datatypes::TSInt128(420), 0, 38); + datatypes::Decimal r(datatypes::TSInt128(4200), 3, 38); + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } } TEST(Decimal, compareCheckInt64_Int128) { - // remainer-based checks - // Equality checks - // l value = r value, L scale = R scale - { - datatypes::Decimal l(420, 10, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 10, 38); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // remainer-based checks + // Equality checks + // l value = r value, L scale = R scale + { + datatypes::Decimal l(420, 10, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 10, 38); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(420, 11, 18); - datatypes::Decimal r(datatypes::TSInt128(42), 10, 38); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(420, 11, 18); + datatypes::Decimal r(datatypes::TSInt128(42), 10, 38); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value < r value, L scale > R scale - { - datatypes::Decimal l(42, 10, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 11, 38); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value < r value, L scale > R scale + { + datatypes::Decimal l(42, 10, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 11, 38); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // Inequality checks - // l value = r value, L scale < R scale - { - datatypes::Decimal l(42, 10, 18); - datatypes::Decimal r(datatypes::TSInt128(42), 13, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // Inequality checks + // l value = r value, L scale < R scale + { + datatypes::Decimal l(42, 10, 18); + datatypes::Decimal r(datatypes::TSInt128(42), 13, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value = r value, L scale < R scale - { - datatypes::Decimal l(42, 13, 18); - datatypes::Decimal r(datatypes::TSInt128(42), 10, 38); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value = r value, L scale < R scale + { + datatypes::Decimal l(42, 13, 18); + datatypes::Decimal r(datatypes::TSInt128(42), 10, 38); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale > R scale - { - datatypes::Decimal l(420, 13, 18); - datatypes::Decimal r(datatypes::TSInt128(42), 10, 38); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale > R scale + { + datatypes::Decimal l(420, 13, 18); + datatypes::Decimal r(datatypes::TSInt128(42), 10, 38); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(4200, 11, 18); - datatypes::Decimal r(datatypes::TSInt128(42), 12, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(4200, 11, 18); + datatypes::Decimal r(datatypes::TSInt128(42), 12, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(99, 10, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 11, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(99, 10, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 11, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(42, 10, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 13, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(42, 10, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 13, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // quotinent-based checks - // l value = r value, L scale = R scale - { - datatypes::Decimal l(420, 1, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // quotinent-based checks + // l value = r value, L scale = R scale + { + datatypes::Decimal l(420, 1, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(4200, 2, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(4200, 2, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // l value < r value, L scale > R scale - { - datatypes::Decimal l(42, 0, 18); - datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); - EXPECT_TRUE(l == r); - EXPECT_TRUE(l >= r); - } + // l value < r value, L scale > R scale + { + datatypes::Decimal l(42, 0, 18); + datatypes::Decimal r(datatypes::TSInt128(420), 1, 38); + EXPECT_TRUE(l == r); + EXPECT_TRUE(l >= r); + } - // Inequality checks - // l value = r value, L scale < R scale - { - datatypes::Decimal l(42000, 0, 18); - datatypes::Decimal r(datatypes::TSInt128(42000), 3, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // Inequality checks + // l value = r value, L scale < R scale + { + datatypes::Decimal l(42000, 0, 18); + datatypes::Decimal r(datatypes::TSInt128(42000), 3, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value = r value, L scale < R scale - { - datatypes::Decimal l(420000, 3, 18); - datatypes::Decimal r(datatypes::TSInt128(420000), 0, 38); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value = r value, L scale < R scale + { + datatypes::Decimal l(420000, 3, 18); + datatypes::Decimal r(datatypes::TSInt128(420000), 0, 38); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale > R scale - { - datatypes::Decimal l(420000, 3, 18); - datatypes::Decimal r(datatypes::TSInt128(42000), 0, 38); - EXPECT_FALSE(l > r); - EXPECT_FALSE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale > R scale + { + datatypes::Decimal l(420000, 3, 18); + datatypes::Decimal r(datatypes::TSInt128(42000), 0, 38); + EXPECT_FALSE(l > r); + EXPECT_FALSE(l >= r); + EXPECT_FALSE(l == r); + } - // l value > r value, L scale < R scale - { - datatypes::Decimal l(420000, 1, 18); - datatypes::Decimal r(datatypes::TSInt128(4200), 2, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value > r value, L scale < R scale + { + datatypes::Decimal l(420000, 1, 18); + datatypes::Decimal r(datatypes::TSInt128(4200), 2, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(990, 0, 18); - datatypes::Decimal r(datatypes::TSInt128(4200), 1, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(990, 0, 18); + datatypes::Decimal r(datatypes::TSInt128(4200), 1, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } - // l value < r value, L scale < R scale - { - datatypes::Decimal l(420, 0, 18); - datatypes::Decimal r(datatypes::TSInt128(4200), 3, 38); - EXPECT_TRUE(l > r); - EXPECT_TRUE(l >= r); - EXPECT_FALSE(l == r); - } + // l value < r value, L scale < R scale + { + datatypes::Decimal l(420, 0, 18); + datatypes::Decimal r(datatypes::TSInt128(4200), 3, 38); + EXPECT_TRUE(l > r); + EXPECT_TRUE(l >= r); + EXPECT_FALSE(l == r); + } } TEST(Decimal, additionNoOverflowCheck) { - // Addition w/o overflow check - execplan::IDB_Decimal l, r, result; - // same precision, same scale, both positive values - l.scale = 38; - l.precision = 38; - l.s128Value = 42; + // Addition w/o overflow check + execplan::IDB_Decimal l, r, result; + // same precision, same scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; - r.scale = 38; - r.precision = 38; - r.s128Value = 420; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + r.scale = 38; + r.precision = 38; + r.s128Value = 420; - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(462, result.s128Value); + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - // same precision, same scale, both negative values - l.s128Value = -42; - r.s128Value = -420; - result.s128Value = 0; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(462, result.s128Value); - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(-462, result.s128Value); + // same precision, same scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - // same precision, same scale, +- values - l.s128Value = 42; - r.s128Value = -420; - result.s128Value = 0; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(-462, result.s128Value); - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(-378, result.s128Value); + // same precision, same scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; - // same precision, same scale, both 0 - l.s128Value = 0; - r.s128Value = 0; - result.s128Value = 0; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(-378, result.s128Value); - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(0, result.s128Value); + // same precision, same scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; - // different scale - // same precision, L scale > R scale, both positive values - l.scale = 38; - l.precision = 38; - l.s128Value = 42; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(0, result.s128Value); - r.scale = 15; - r.precision = 38; - r.s128Value = 420; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + // different scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); + r.scale = 15; + r.precision = 38; + r.s128Value = 420; - // same precision, L scale > R scale, both negative values - l.s128Value = -42; - r.s128Value = -420; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ("-0.00000000000042000000000000000000000042", result.toString()); + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); - // same precision, L scale > R scale, +- values - l.s128Value = 42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, L scale > R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("-0.00000000000042000000000000000000000042", result.toString()); - // same precision, L scale > R scale, both 0 - l.s128Value = 0; - r.s128Value = 0; - result.s128Value = 0; + // same precision, L scale > R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); - // same precision, L scale < R scale, both positive values - l.scale = 15; - l.precision = 38; - l.s128Value = 42; + // same precision, L scale > R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; - r.scale = 38; - r.precision = 38; - r.s128Value = 420; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(0, result.s128Value); - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; - // same precision, L scale < R scale, both negative values - l.s128Value = -42; - r.s128Value = -420; - result.s128Value = 0; + r.scale = 38; + r.precision = 38; + r.s128Value = 420; - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ("-0.00000000000004200000000000000000000420", result.toString()); + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - // same precision, L scale < R scale, +- values - l.s128Value = 42; - r.s128Value = -420; - result.s128Value = 0; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); + // same precision, L scale < R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - // same precision, L scale < R scale, both 0 - l.s128Value = 0; - r.s128Value = 0; - result.s128Value = 0; + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("-0.00000000000004200000000000000000000420", result.toString()); - datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(0, result.s128Value); + // same precision, L scale < R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); + + // same precision, L scale < R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(0, result.s128Value); } TEST(Decimal, divisionNoOverflowCheck) { - // DIVISION - // same precision, same scale, both positive values - execplan::IDB_Decimal l, r, result; + // DIVISION + // same precision, same scale, both positive values + execplan::IDB_Decimal l, r, result; - l.scale = 38; - l.precision = 38; - l.s128Value = 43; + l.scale = 38; + l.precision = 38; + l.s128Value = 43; - r.scale = 38; - r.precision = 38; - r.s128Value = 420; - - result.scale = 10; - result.precision = 38; - result.s128Value = 0; + r.scale = 38; + r.precision = 38; + r.s128Value = 420; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("9.7674418605", result.toString()); + result.scale = 10; + result.precision = 38; + result.s128Value = 0; - // same precision, same scale, both negative values - l.s128Value = -43; - r.s128Value = -420; - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("9.7674418605", result.toString()); - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("9.7674418605", result.toString()); + // same precision, same scale, both negative values + l.s128Value = -43; + r.s128Value = -420; + result.s128Value = 0; - // same precision, same scale, +- values - l.s128Value = 2200000; - r.s128Value = -1900; - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("9.7674418605", result.toString()); - datatypes::Decimal::division(l, r, result); - EXPECT_EQ("-1157.8947368421", result.toString()); + // same precision, same scale, +- values + l.s128Value = 2200000; + r.s128Value = -1900; + result.s128Value = 0; - // same precision, same scale, l = 0 - l.s128Value = 0; - r.s128Value = 42424242; - result.s128Value = 0; + datatypes::Decimal::division(l, r, result); + EXPECT_EQ("-1157.8947368421", result.toString()); - datatypes::Decimal::division(l, r, result); - EXPECT_EQ(0, result.s128Value); + // same precision, same scale, l = 0 + l.s128Value = 0; + r.s128Value = 42424242; + result.s128Value = 0; - // diff scale - // same precision, L scale > R scale, both positive values - l.scale = 38; - l.s128Value = 19; + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(0, result.s128Value); - r.scale = 15; - r.s128Value = 22; - - result.scale = 10; - result.s128Value = 0; + // diff scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.s128Value = 19; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("115789473684210526315789.4736842105", result.toString()); + r.scale = 15; + r.s128Value = 22; - // same precision, L scale > R scale, both negative values - l.s128Value = -22; - r.s128Value = -19; - result.s128Value = 0; + result.scale = 10; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("86363636363636363636363.6363636364", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("115789473684210526315789.4736842105", result.toString()); - // same precision, L scale > R scale, +- values - l.s128Value = 19; - r.s128Value = -22; - result.s128Value = 0; + // same precision, L scale > R scale, both negative values + l.s128Value = -22; + r.s128Value = -19; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("-115789473684210526315789.4736842105", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("86363636363636363636363.6363636364", result.toString()); - // same precision, L scale > R scale, R = 0 - l.s128Value = 424242; - r.s128Value = 0; - result.s128Value = 0; + // same precision, L scale > R scale, +- values + l.s128Value = 19; + r.s128Value = -22; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("-115789473684210526315789.4736842105", result.toString()); - // same precision, L scale > R scale, both MAX positive values - utils::int128Max(l.s128Value); - utils::int128Max(r.s128Value); - result.s128Value = 0; + // same precision, L scale > R scale, R = 0 + l.s128Value = 424242; + r.s128Value = 0; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); - // same precision, L scale > R scale, both MIN negative values - utils::int128Min(l.s128Value); - utils::int128Min(r.s128Value); - result.s128Value = 0; + // same precision, L scale > R scale, both MAX positive values + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); - // same precision, L scale < R scale, both positive values - l.scale = 15; - l.precision = 38; - l.s128Value = 42; + // same precision, L scale > R scale, both MIN negative values + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; - r.scale = 38; - r.precision = 38; - r.s128Value = 42; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; - // same precision, L scale < R scale, both negative values - l.s128Value = -22; - r.s128Value = -19; - result.s128Value = 0; + r.scale = 38; + r.precision = 38; + r.s128Value = 42; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("0.00000000000000000000000863636363636364", result.toString()); + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - // same precision, L scale < R scale, +- values - l.s128Value = 22; - r.s128Value = -19; - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("-0.00000000000000000000000863636363636364", result.toString()); + // same precision, L scale < R scale, both negative values + l.s128Value = -22; + r.s128Value = -19; + result.s128Value = 0; - // same precision, L scale < R scale, R = 0 - l.s128Value = 42; - r.s128Value = 0; - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000000863636363636364", result.toString()); - datatypes::Decimal::division(r, l, result); - EXPECT_EQ(0, result.s128Value); + // same precision, L scale < R scale, +- values + l.s128Value = 22; + r.s128Value = -19; + result.s128Value = 0; - // same precision, L scale < R scale, both MAX positive values - // WIP Investigate the next two - utils::int128Max(l.s128Value); - utils::int128Max(r.s128Value); - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("-0.00000000000000000000000863636363636364", result.toString()); - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); + // same precision, L scale < R scale, R = 0 + l.s128Value = 42; + r.s128Value = 0; + result.s128Value = 0; - // same precision, L scale < R scale, both MIN negative values - utils::int128Min(l.s128Value); - utils::int128Min(r.s128Value); - result.s128Value = 0; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); + // same precision, L scale < R scale, both MAX positive values + // WIP Investigate the next two + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; - // same precision, L scale < R scale, result.scale < (r.scale-l.scale) - // both positive values - l.scale = 37; - l.precision = 38; - l.s128Value = 43; + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); - r.scale = 38; - r.precision = 38; - r.s128Value = 420; - - result.scale = 0; - result.precision = 38; - result.s128Value = 0; + // same precision, L scale < R scale, both MIN negative values + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("1", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); - // same precision, L scale < R scale, result.scale < (r.scale-l.scale) - // both negative values - l.s128Value = -22; - r.s128Value = -1900; - result.s128Value = 0; + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both positive values + l.scale = 37; + l.precision = 38; + l.s128Value = 43; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("9", result.toString()); + r.scale = 38; + r.precision = 38; + r.s128Value = 420; - // same precision, L scale < R scale, result.scale < (r.scale-l.scale) - // +- values - l.s128Value = 22; - r.s128Value = -1900; - result.s128Value = 0; + result.scale = 0; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("-9", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("1", result.toString()); - // same precision, L scale < R scale, result.scale < (r.scale-l.scale) - // R = 0 - l.s128Value = 42; - r.s128Value = 0; - result.s128Value = 0; + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both negative values + l.s128Value = -22; + r.s128Value = -1900; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("9", result.toString()); - // same precision, L scale < R scale, result.scale < (r.scale-l.scale) - // both MAX positive values - utils::int128Max(l.s128Value); - utils::int128Max(r.s128Value); - result.s128Value = 0; + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // +- values + l.s128Value = 22; + r.s128Value = -1900; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("0", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("-9", result.toString()); - // same precision, L scale < R scale, result.scale < (r.scale-l.scale) - // both MIN negative values - utils::int128Min(l.s128Value); - utils::int128Min(r.s128Value); - result.s128Value = 0; + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // R = 0 + l.s128Value = 42; + r.s128Value = 0; + result.s128Value = 0; - datatypes::Decimal::division(r, l, result); - EXPECT_EQ("0", result.toString()); + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both MAX positive values + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0", result.toString()); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both MIN negative values + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0", result.toString()); } -void doDiv(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result) +void doDiv(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - datatypes::Decimal::division(l, r, result); + datatypes::Decimal::division(l, r, result); } TEST(Decimal, divisionWithOverflowCheck) { - // Divide min int128 by -1 - execplan::IDB_Decimal l, r, result; - l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::minInt128; + // Divide min int128 by -1 + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::minInt128; - r.scale = 0; - r.precision = 38; - r.s128Value = -1; - - result.scale = 0; - result.precision = 38; - result.s128Value = 42; - EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); + r.scale = 0; + r.precision = 38; + r.s128Value = -1; - // Divide two ints one of which overflows after the scaling. - // TODO We currently do not test overflow due to scaling in - // case of division. Re-enable this test when we check for overflow - /*l.s128Value = datatypes::Decimal::maxInt128; - r.scale = 1; - r.s128Value = 42; - - result.scale = 1; - result.s128Value = 42; - EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept);*/ + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); - // Normal execution w/o overflow - l.scale = 0; - l.s128Value = datatypes::Decimal::maxInt128 - 1; + // Divide two ints one of which overflows after the scaling. + // TODO We currently do not test overflow due to scaling in + // case of division. Re-enable this test when we check for overflow + /*l.s128Value = datatypes::Decimal::maxInt128; + r.scale = 1; + r.s128Value = 42; - r.scale = 0; - r.s128Value = 0xFFFFFFFFFFFFFFFF; - - result.scale = 0; - result.s128Value = 0; + result.scale = 1; + result.s128Value = 42; + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept);*/ - EXPECT_NO_THROW(doDiv(l, r, result)); - EXPECT_EQ("9223372036854775809", result.toString()); + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::maxInt128 - 1; + + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doDiv(l, r, result)); + EXPECT_EQ("9223372036854775809", result.toString()); } -void doAdd(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result) +void doAdd(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - datatypes::Decimal::addition(l, r, result); + datatypes::Decimal::addition(l, r, result); } TEST(Decimal, additionWithOverflowCheck) { - // Add two max ints - execplan::IDB_Decimal l, r, result; - l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128 - 1; + // Add two max ints + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128 - 1; - r.scale = 0; - r.precision = 38; - r.s128Value = datatypes::Decimal::maxInt128 - 1; - - result.scale = 0; - result.precision = 38; - result.s128Value = 42; - EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + r.scale = 0; + r.precision = 38; + r.s128Value = datatypes::Decimal::maxInt128 - 1; - // Add two ints one of which overflows after the scaling. - l.s128Value = datatypes::Decimal::maxInt128 - 1; - r.scale = 1; - r.s128Value = 0xFFFFFFFFFFFFFFFF; - - result.scale = 1; - result.precision = 38; - result.s128Value = 0; - EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); - // Normal execution w/o overflow - l.scale = 0; - l.s128Value = datatypes::Decimal::minInt128; + // Add two ints one of which overflows after the scaling. + l.s128Value = datatypes::Decimal::maxInt128 - 1; + r.scale = 1; + r.s128Value = 0xFFFFFFFFFFFFFFFF; - r.scale = 0; - r.s128Value = 0xFFFFFFFFFFFFFFFF; + result.scale = 1; + result.precision = 38; + result.s128Value = 0; + EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); - result.scale = 0; - result.s128Value = 0; + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::minInt128; - EXPECT_NO_THROW(doAdd(l, r, result)); - EXPECT_EQ("-170141183460469231713240559642174554113", result.toString()); + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doAdd(l, r, result)); + EXPECT_EQ("-170141183460469231713240559642174554113", result.toString()); } TEST(Decimal, subtractionNoOverflowCheck) { - // Subtractio w/o overflow check - execplan::IDB_Decimal l, r, result; - // same precision, same scale, both positive values - l.scale = 38; - l.precision = 38; - l.s128Value = 42; + // Subtractio w/o overflow check + execplan::IDB_Decimal l, r, result; + // same precision, same scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; - r.scale = 38; - r.precision = 38; - r.s128Value = 420; + r.scale = 38; + r.precision = 38; + r.s128Value = 420; - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("-0.00000000000000000000000000000000000378", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("-0.00000000000000000000000000000000000378", result.toString()); - // same precision, same scale, both negative values - l.s128Value = -42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, same scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("0.00000000000000000000000000000000000378", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000000000000000000000000000378", result.toString()); - // same precision, same scale, +- values - l.s128Value = 42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, same scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("0.00000000000000000000000000000000000462", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000000000000000000000000000462", result.toString()); - // same precision, same scale, both 0 - l.s128Value = 0; - r.s128Value = 0; - result.s128Value = 0; + // same precision, same scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); - // different scale - // same precision, L scale > R scale, both positive values - l.scale = 38; - l.precision = 38; - l.s128Value = 42; + // different scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; - r.scale = 15; - r.precision = 38; - r.s128Value = 420; + r.scale = 15; + r.precision = 38; + r.s128Value = 420; - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); - // same precision, L scale > R scale, both negative values - l.s128Value = -42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, L scale > R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("0.00000000000041999999999999999999999958", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000041999999999999999999999958", result.toString()); - // same precision, L scale > R scale, +- values - l.s128Value = 42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, L scale > R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); - // same precision, L scale > R scale, both 0 - l.s128Value = 0; - r.s128Value = 0; - result.s128Value = 0; + // same precision, L scale > R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); - // same precision, L scale < R scale, both positive values - l.scale = 15; - l.precision = 38; - l.s128Value = 42; + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; - r.scale = 38; - r.precision = 38; - r.s128Value = 420; + r.scale = 38; + r.precision = 38; + r.s128Value = 420; - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); - // same precision, L scale < R scale, both negative values - l.s128Value = -42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, L scale < R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("-0.00000000000004199999999999999999999580", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("-0.00000000000004199999999999999999999580", result.toString()); - // same precision, L scale < R scale, +- values - l.s128Value = 42; - r.s128Value = -420; - result.s128Value = 0; + // same precision, L scale < R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); - // same precision, L scale < R scale, both 0 - l.s128Value = 0; - r.s128Value = 0; - result.s128Value = 0; + // same precision, L scale < R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; - datatypes::Decimal::subtraction(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); } -void doSubtract(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result) +void doSubtract(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - datatypes::Decimal::subtraction(l, r, result); + datatypes::Decimal::subtraction(l, r, result); } TEST(Decimal, subtractionWithOverflowCheck) { - // Subtract a max int from a min int - execplan::IDB_Decimal l, r, result; - l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::minInt128 + 1; + // Subtract a max int from a min int + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::minInt128 + 1; - r.scale = 0; - r.precision = 38; - r.s128Value = datatypes::Decimal::maxInt128 - 1; + r.scale = 0; + r.precision = 38; + r.s128Value = datatypes::Decimal::maxInt128 - 1; - result.scale = 0; - result.precision = 38; - result.s128Value = 42; - EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); - // Subtract two ints one of which overflows after the scaling. - l.s128Value = datatypes::Decimal::minInt128 + 1; - r.scale = 1; - r.s128Value = 0xFFFFFFFFFFFFFFFF; + // Subtract two ints one of which overflows after the scaling. + l.s128Value = datatypes::Decimal::minInt128 + 1; + r.scale = 1; + r.s128Value = 0xFFFFFFFFFFFFFFFF; - result.scale = 1; - result.precision = 38; - result.s128Value = 0; + result.scale = 1; + result.precision = 38; + result.s128Value = 0; - EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); + EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); - // Normal execution w/o overflow - l.scale = 0; - l.s128Value = datatypes::Decimal::maxInt128; + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::maxInt128; - r.scale = 0; - r.s128Value = 0xFFFFFFFFFFFFFFFF; + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; - result.scale = 0; - result.s128Value = 0; + result.scale = 0; + result.s128Value = 0; - EXPECT_NO_THROW(doSubtract(l, r, result)); - EXPECT_EQ("170141183460469231713240559642174554112", result.toString()); + EXPECT_NO_THROW(doSubtract(l, r, result)); + EXPECT_EQ("170141183460469231713240559642174554112", result.toString()); } TEST(Decimal, multiplicationNoOverflowCheck) { - // Multiplication - // same precision, l.scale + r.scale = result.scale, both positive values - execplan::IDB_Decimal l, r, result; + // Multiplication + // same precision, l.scale + r.scale = result.scale, both positive values + execplan::IDB_Decimal l, r, result; - l.scale = 19; - l.precision = 38; - l.s128Value = 4611686018427387904; + l.scale = 19; + l.precision = 38; + l.s128Value = 4611686018427387904; - r.scale = 19; - r.precision = 38; - r.s128Value = UINT64_MAX; + r.scale = 19; + r.precision = 38; + r.s128Value = UINT64_MAX; - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::multiplication(r, l, result); - EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); - // same precision, l.scale + r.scale = result.scale, both negative values - l.s128Value = -4611686018427387904; - r.s128Value = UINT64_MAX; - r.s128Value = -r.s128Value; - result.s128Value = 0; + // same precision, l.scale + r.scale = result.scale, both negative values + l.s128Value = -4611686018427387904; + r.s128Value = UINT64_MAX; + r.s128Value = -r.s128Value; + result.s128Value = 0; - datatypes::Decimal::multiplication(r, l, result); - EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); - // same precision, l.scale + r.scale = result.scale, +- values - l.s128Value = -4611686018427387904; - r.s128Value = UINT64_MAX; - result.s128Value = 0; + // same precision, l.scale + r.scale = result.scale, +- values + l.s128Value = -4611686018427387904; + r.s128Value = UINT64_MAX; + result.s128Value = 0; - datatypes::Decimal::multiplication(l, r, result); - EXPECT_EQ("-0.85070591730234615861231965839514664960", result.toString()); + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ("-0.85070591730234615861231965839514664960", result.toString()); - // same precision, l.scale + r.scale = result.scale, l = 0 - l.s128Value = 0; - r.s128Value = UINT64_MAX; - result.s128Value = 0; + // same precision, l.scale + r.scale = result.scale, l = 0 + l.s128Value = 0; + r.s128Value = UINT64_MAX; + result.s128Value = 0; - datatypes::Decimal::multiplication(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); - // same precision, l.scale + r.scale < result.scale, both positive values - l.scale = 18; - l.precision = 38; - l.s128Value = 72057594037927936; + // same precision, l.scale + r.scale < result.scale, both positive values + l.scale = 18; + l.precision = 38; + l.s128Value = 72057594037927936; - r.scale = 18; - r.precision = 38; - r.s128Value = 9223372036854775808ULL; + r.scale = 18; + r.precision = 38; + r.s128Value = 9223372036854775808ULL; - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::multiplication(r, l, result); - EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); - // same precision, l.scale + r.scale < result.scale, both negative values - l.s128Value = -72057594037927936; - r.s128Value = 9223372036854775808ULL; - r.s128Value = -r.s128Value; - result.s128Value = 0; + // same precision, l.scale + r.scale < result.scale, both negative values + l.s128Value = -72057594037927936; + r.s128Value = 9223372036854775808ULL; + r.s128Value = -r.s128Value; + result.s128Value = 0; - datatypes::Decimal::multiplication(r, l, result); - EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); - // same precision, l.scale + r.scale < result.scale, +- values - l.s128Value = -72057594037927936; - r.s128Value = 9223372036854775808ULL; - result.s128Value = 0; + // same precision, l.scale + r.scale < result.scale, +- values + l.s128Value = -72057594037927936; + r.s128Value = 9223372036854775808ULL; + result.s128Value = 0; - datatypes::Decimal::multiplication(l, r, result); - EXPECT_EQ("-0.66461399789245793645190353014017228800", result.toString()); + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ("-0.66461399789245793645190353014017228800", result.toString()); - // same precision, l.scale + r.scale < result.scale, l = 0 - l.s128Value = 0; - r.s128Value = 9223372036854775808ULL; - result.s128Value = 0; + // same precision, l.scale + r.scale < result.scale, l = 0 + l.s128Value = 0; + r.s128Value = 9223372036854775808ULL; + result.s128Value = 0; - datatypes::Decimal::multiplication(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); - // same precision, l.scale + r.scale > result.scale, both positive values - l.scale = 38; - l.precision = 38; - l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + - 1234567890123456789); + // same precision, l.scale + r.scale > result.scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + 1234567890123456789); - r.scale = 38; - r.precision = 38; - r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + - 1234567890123456789ULL); + r.scale = 38; + r.precision = 38; + r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + 1234567890123456789ULL); - result.scale = 38; - result.precision = 38; - result.s128Value = 0; + result.scale = 38; + result.precision = 38; + result.s128Value = 0; - datatypes::Decimal::multiplication(r, l, result); - EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); - // same precision, l.scale + r.scale > result.scale, both negative values - l.s128Value = -l.s128Value; - r.s128Value = -r.s128Value; - result.s128Value = 0; + // same precision, l.scale + r.scale > result.scale, both negative values + l.s128Value = -l.s128Value; + r.s128Value = -r.s128Value; + result.s128Value = 0; - datatypes::Decimal::multiplication(r, l, result); - EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); - // same precision, l.scale + r.scale > result.scale, +- values - r.s128Value = -r.s128Value; - result.s128Value = 0; + // same precision, l.scale + r.scale > result.scale, +- values + r.s128Value = -r.s128Value; + result.s128Value = 0; - datatypes::Decimal::multiplication(l, r, result); - EXPECT_EQ("-0.01524157875323883675019051998750190521", result.toString()); + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ("-0.01524157875323883675019051998750190521", result.toString()); - // same precision, l.scale + r.scale > result.scale, l = 0 - r.s128Value = 0; - result.s128Value = 0; + // same precision, l.scale + r.scale > result.scale, l = 0 + r.s128Value = 0; + result.s128Value = 0; - datatypes::Decimal::multiplication(l, r, result); - EXPECT_EQ(0, result.s128Value); + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); } -void doMultiply(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result) +void doMultiply(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - datatypes::Decimal::multiplication(l, r, result); + datatypes::Decimal::multiplication(l, r, result); } TEST(Decimal, multiplicationWithOverflowCheck) { - execplan::IDB_Decimal l, r, result; - // result.scale >= l.scale + r.scale - l.scale = 0; - l.precision = 38; - l.s128Value = UINT64_MAX; + execplan::IDB_Decimal l, r, result; + // result.scale >= l.scale + r.scale + l.scale = 0; + l.precision = 38; + l.s128Value = UINT64_MAX; - r.scale = 0; - r.precision = 38; - r.s128Value = UINT64_MAX; + r.scale = 0; + r.precision = 38; + r.s128Value = UINT64_MAX; - result.scale = 0; - result.precision = 38; - result.s128Value = 42; - EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); - // result.scale < l.scale + r.scale - l.scale = 36; - l.precision = 38; - l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + - 1234567890123456789); + // result.scale < l.scale + r.scale + l.scale = 36; + l.precision = 38; + l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + 1234567890123456789); - r.scale = 36; - r.precision = 38; - r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + - 1234567890123456789); + r.scale = 36; + r.precision = 38; + r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + 1234567890123456789); - result.scale = 38; - result.precision = 38; - result.s128Value = 42; - EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + result.scale = 38; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); - // Normal execution w/o overflow - l.scale = 0; - l.s128Value = 4611686018427387904; + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = 4611686018427387904; - r.scale = 0; - r.s128Value = 4611686018427387904; + r.scale = 0; + r.s128Value = 4611686018427387904; - result.scale = 0; - result.s128Value = 0; + result.scale = 0; + result.s128Value = 0; - EXPECT_NO_THROW(doMultiply(l, r, result)); - EXPECT_EQ("21267647932558653966460912964485513216", result.toString()); + EXPECT_NO_THROW(doMultiply(l, r, result)); + EXPECT_EQ("21267647932558653966460912964485513216", result.toString()); - l.setTSInt128Value((int128_t)1 << 122); - l.setScale(0); - r.setTSInt128Value(100); - r.setScale(0); - EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + l.setTSInt128Value((int128_t)1 << 122); + l.setScale(0); + r.setTSInt128Value(100); + r.setScale(0); + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); - l.setTSInt128Value((int128_t)1 << 65); - l.setScale(0); - r.setTSInt128Value((int128_t)1 << 64); - r.setScale(0); - EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); - - l.setTSInt128Value((int128_t)1 << 122); - l.setScale(0); - r.setTSInt128Value(2); - r.setScale(0); - EXPECT_NO_THROW(doMultiply(l, r, result)); - EXPECT_EQ("10633823966279326983230456482242756608", result.toString()); + l.setTSInt128Value((int128_t)1 << 65); + l.setScale(0); + r.setTSInt128Value((int128_t)1 << 64); + r.setScale(0); + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + l.setTSInt128Value((int128_t)1 << 122); + l.setScale(0); + r.setTSInt128Value(2); + r.setScale(0); + EXPECT_NO_THROW(doMultiply(l, r, result)); + EXPECT_EQ("10633823966279326983230456482242756608", result.toString()); } TEST(Decimal, DecimalToStringCheckScale0) { - string input, expected; - int128_t res; - int precision = 38; - int scale = 0; - res = 0; - datatypes::Decimal dec(0, scale, precision, res); + string input, expected; + int128_t res; + int precision = 38; + int scale = 0; + res = 0; + datatypes::Decimal dec(0, scale, precision, res); - // test simple values - expected = "0"; - EXPECT_EQ(dec.toString(), expected); - res = 2; - expected = "2"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -2; - expected = "-2"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = 123; - expected = "123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -123; - expected = "-123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test simple values + expected = "0"; + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "2"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-2"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "99999999999999999999999999999999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-99999999999999999999999999999999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + expected = "99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test trailing zeros - res = 123000; - expected = "123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test trailing zeros + res = 123000; + expected = "123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); } TEST(Decimal, DecimalToStringCheckScale10) { - string input, expected; - int128_t res; - int precision = 38; - int scale = 10; - res = 0; - datatypes::Decimal dec(0, scale, precision, res); + string input, expected; + int128_t res; + int precision = 38; + int scale = 10; + res = 0; + datatypes::Decimal dec(0, scale, precision, res); - // test simple values - expected = "0.0000000000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test simple values + expected = "0.0000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - res = 2; - expected = "0.0000000002"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.0000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - res = -2; - expected = "-0.0000000002"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.0000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - res = 123; - expected = "0.0000000123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -123; - expected = "-0.0000000123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = 12345678901; - expected = "1.2345678901"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -12345678901; - expected = "-1.2345678901"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.0000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.0000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 12345678901; + expected = "1.2345678901"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -12345678901; + expected = "-1.2345678901"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "9999999999999999999999999999.9999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-9999999999999999999999999999.9999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + expected = "9999999999999999999999999999.9999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-9999999999999999999999999999.9999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test trailing zeros - res = 123000; - expected = "0.0000123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-0.0000123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test trailing zeros + res = 123000; + expected = "0.0000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.0000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test leading zeros - res = 10000000009; - expected = "1.0000000009"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-1.0000000009"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test leading zeros + res = 10000000009; + expected = "1.0000000009"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-1.0000000009"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); } TEST(Decimal, DecimalToStringCheckScale38) { - string input, expected; - int128_t res; - int precision = 38; - int scale = 38; - res = 0; - datatypes::Decimal dec(0, scale, precision, res); + string input, expected; + int128_t res; + int precision = 38; + int scale = 38; + res = 0; + datatypes::Decimal dec(0, scale, precision, res); - // test simple values - res = 0; - expected = "0.00000000000000000000000000000000000000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = 2; - expected = "0.00000000000000000000000000000000000002"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -2; - expected = "-0.00000000000000000000000000000000000002"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = 123; - expected = "0.00000000000000000000000000000000000123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -123; - expected = "-0.00000000000000000000000000000000000123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; - expected = "0.12345678901234567890123456789012345678"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-0.12345678901234567890123456789012345678"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test simple values + res = 0; + expected = "0.00000000000000000000000000000000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.00000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.00000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.00000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.00000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000) + + 12345678; + expected = "0.12345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.12345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "0.99999999999999999999999999999999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-0.99999999999999999999999999999999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + expected = "0.99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test trailing zeros - res = 123000; - expected = "0.00000000000000000000000000000000123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-0.00000000000000000000000000000000123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test trailing zeros + res = 123000; + expected = "0.00000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.00000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); } TEST(Decimal, DecimalToStringCheckScale37) { - string input, expected; - int128_t res; - int precision = 38; - int scale = 37; - res = 0; - datatypes::Decimal dec(0, scale, precision, res); + string input, expected; + int128_t res; + int precision = 38; + int scale = 37; + res = 0; + datatypes::Decimal dec(0, scale, precision, res); - // test simple values - res = 0; - expected = "0.0000000000000000000000000000000000000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = 2; - expected = "0.0000000000000000000000000000000000002"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -2; - expected = "-0.0000000000000000000000000000000000002"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = 123; - expected = "0.0000000000000000000000000000000000123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -123; - expected = "-0.0000000000000000000000000000000000123"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; - expected = "1.2345678901234567890123456789012345678"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-1.2345678901234567890123456789012345678"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test simple values + res = 0; + expected = "0.0000000000000000000000000000000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.0000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.0000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.0000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.0000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000) + + 12345678; + expected = "1.2345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-1.2345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "9.9999999999999999999999999999999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-9.9999999999999999999999999999999999999"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000) + + 999999999) * + 100) + + 99; + expected = "9.9999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-9.9999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); - // test trailing zeros - res = 123000; - expected = "0.0000000000000000000000000000000123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); - res = -res; - expected = "-0.0000000000000000000000000000000123000"; - dec.setTSInt128Value(res); - EXPECT_EQ(dec.toString(), expected); + // test trailing zeros + res = 123000; + expected = "0.0000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.0000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); } - - diff --git a/tests/primitives_column_scan_and_filter.cpp b/tests/primitives_column_scan_and_filter.cpp index 12a1b7174..70f921b38 100644 --- a/tests/primitives_column_scan_and_filter.cpp +++ b/tests/primitives_column_scan_and_filter.cpp @@ -42,7 +42,7 @@ using namespace std; class ColumnScanFilterTest : public ::testing::Test { -protected: + protected: PrimitiveProcessor pp; uint8_t input[BLOCK_SIZE]; alignas(utils::MAXCOLUMNWIDTH) uint8_t output[4 * BLOCK_SIZE]; @@ -52,7 +52,7 @@ protected: NewColRequestHeader* in; ColResultHeader* out; ColArgs* args; - + void SetUp() override { memset(input, 0, BLOCK_SIZE); @@ -72,32 +72,31 @@ protected: if (fd < 0) { - cerr << "getBlock(): skipping this test; needs the index list file " - << fileName << endl; - return nullptr; + cerr << "getBlock(): skipping this test; needs the index list file " << fileName << endl; + return nullptr; } i = read(fd, block, BLOCK_SIZE); if (i <= 0) { - cerr << "getBlock(): Couldn't read the file " << fileName << endl; - close(fd); - return nullptr; + cerr << "getBlock(): Couldn't read the file " << fileName << endl; + close(fd); + return nullptr; } if (i != BLOCK_SIZE) { - cerr << "getBlock(): could not read a whole block" << endl; - close(fd); - return nullptr; + cerr << "getBlock(): could not read a whole block" << endl; + close(fd); + return nullptr; } close(fd); return block; - } - uint8_t* readBlockFromLiteralArray(const std::string& fileName, uint8_t* block) - { + } + uint8_t* readBlockFromLiteralArray(const std::string& fileName, uint8_t* block) + { if (fileName == std::string("col1block.cdf")) return &__col1block_cdf[0]; else if (fileName == std::string("col2block.cdf")) @@ -118,7 +117,7 @@ protected: return &___bin_col_neg_double_cdf[0]; return nullptr; - } + } }; TEST_F(ColumnScanFilterTest, ColumnScan1Byte) @@ -134,14 +133,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan1Byte) in->NOPS = 0; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col1block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); EXPECT_EQ(out->NVALS, 8160); for (i = 0; i < 300; i++) - EXPECT_EQ(results[i],i % 255); + EXPECT_EQ(results[i], i % 255); // Can't check Min/Max for char columns until MCOL-4871 } @@ -158,23 +157,23 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteVectorized) in->NOPS = 0; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col1block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); EXPECT_EQ(out->NVALS, 8160); for (i = 0; i < 128; ++i) - EXPECT_EQ(results[i],i); + EXPECT_EQ(results[i], i); for (i = 129; i < 255; ++i) - EXPECT_EQ(results[i],i + 1); + EXPECT_EQ(results[i], i + 1); EXPECT_EQ(results[8032], 0x7F); EXPECT_EQ(results[8033], 0x80); for (i = 8034; i < 8160; ++i) - EXPECT_EQ(results[i],i % 255 + 1); + EXPECT_EQ(results[i], i % 255 + 1); EXPECT_EQ(out->Max, __col1block_cdf_umax); EXPECT_EQ(out->Min, __col1block_cdf_umin); @@ -192,14 +191,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes) in->NOPS = 0; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col2block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col2block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); EXPECT_EQ(out->NVALS, 4096); for (i = 0; i < out->NVALS; i++) - EXPECT_EQ(results[i], i); + EXPECT_EQ(results[i], i); EXPECT_EQ(out->Max, __col2block_cdf_umax); EXPECT_EQ(out->Min, __col2block_cdf_umin); @@ -217,14 +216,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes) in->NOPS = 0; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col4block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); EXPECT_EQ(out->NVALS, 2048); for (i = 0; i < out->NVALS; i++) - EXPECT_EQ(results[i], (uint32_t) i); + EXPECT_EQ(results[i], (uint32_t)i); EXPECT_EQ(out->Max, __col4block_cdf_umax); EXPECT_EQ(out->Min, __col4block_cdf_umin); } @@ -241,14 +240,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes) in->NOPS = 0; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); ASSERT_EQ(out->NVALS, 1024); for (i = 0; i < out->NVALS; i++) - ASSERT_EQ(results[i], (uint32_t) i); + ASSERT_EQ(results[i], (uint32_t)i); EXPECT_EQ(out->Max, __col8block_cdf_umax); EXPECT_EQ(out->Min, __col8block_cdf_umin); } @@ -267,14 +266,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes1EqFilter) in->NOPS = 1; in->BOP = BOP_AND; in->NVALS = 0; - + tmp = 50; args->COP = COMPARE_LE; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col2block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col2block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -299,14 +298,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingRID) rids[0] = 20; rids[1] = 17; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col1block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); ASSERT_EQ(out->NVALS, 2); for (i = 0; i < out->NVALS; ++i) - ASSERT_EQ(results[i], rids[i]); + ASSERT_EQ(results[i], rids[i]); } TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingMultipleRIDs) @@ -328,7 +327,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingMultipleRIDs) rids[1] = 17; rids[126] = 8189; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col1block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -353,14 +352,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes1EqFilter) in->NOPS = 1; in->BOP = BOP_AND; in->NVALS = 0; - + tmp = 2040; args->COP = COMPARE_GE; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col4block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -387,8 +386,8 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesUsingMultipleRIDs) rids[0] = 20; rids[1] = 17; rids[126] = 1020; - - pp.setBlockPtr((int*) readBlockFromLiteralArray("col4block.cdf", block)); + + pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -416,13 +415,13 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes2Filters) tmp = 20; args->COP = COMPARE_LT; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_GT; tmp = 10; memcpy(args->val, &tmp, in->colType.DataSize); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col4block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -451,14 +450,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes1EqFilter) in->NOPS = 1; in->BOP = BOP_AND; in->NVALS = 0; - + tmp = 11; args->COP = COMPARE_LT; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -488,8 +487,8 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesUsingMultipleRIDs) rids[0] = 20; rids[1] = 17; rids[126] = 1020; - - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -500,7 +499,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesUsingMultipleRIDs) ASSERT_EQ(results[126], 1020); } -//void p_Col_7() +// void p_Col_7() TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2CompFilters) { constexpr const uint8_t W = 8; @@ -515,24 +514,24 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2CompFilters) in->NOPS = 2; in->BOP = BOP_OR; in->NVALS = 0; - + tmp = 10; args->COP = COMPARE_LT; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_GT; tmp = 1000; memcpy(args->val, &tmp, in->colType.DataSize); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); ASSERT_EQ(out->NVALS, 33); for (i = 0; i < out->NVALS; i++) - ASSERT_EQ(results[i], (uint32_t) (i < 10 ? i : i - 10 + 1001)); + ASSERT_EQ(results[i], (uint32_t)(i < 10 ? i : i - 10 + 1001)); EXPECT_EQ(out->Max, __col8block_cdf_umax); EXPECT_EQ(out->Min, __col8block_cdf_umin); @@ -556,13 +555,13 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFilters) tmp = 10; args->COP = COMPARE_EQ; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_EQ; tmp = 1000; memcpy(args->val, &tmp, in->colType.DataSize); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -592,19 +591,19 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRID) tmp = 10; args->COP = COMPARE_EQ; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_EQ; tmp = 1000; memcpy(args->val, &tmp, in->colType.DataSize); - rids = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - 2 * (sizeof(ColArgs) + in->colType.DataSize)]); + rids = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + 2 * (sizeof(ColArgs) + in->colType.DataSize)]); rids[0] = 10; rids[1] = 100; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -625,17 +624,17 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2FiltersRIDOutputRid) in->NOPS = 2; in->BOP = BOP_OR; in->NVALS = 0; - + tmp = 10; args->COP = COMPARE_LT; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_GT; tmp = 1000; memcpy(args->val, &tmp, in->colType.DataSize); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstRIDArrayPosition(out), 0); @@ -661,31 +660,31 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRIDOutputBoth) in->NOPS = 2; in->BOP = BOP_OR; in->NVALS = 0; - + tmp = 10; args->COP = COMPARE_LT; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_GT; tmp = 1000; memcpy(args->val, &tmp, in->colType.DataSize); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col8block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.columnScanAndFilter(in, out); ASSERT_EQ(out->NVALS, 33); for (i = 0; i < out->NVALS; i++) { - ASSERT_EQ(resultRid[i], (i < 10 ? i : i - 10 + 1001)); - ASSERT_EQ(resultVal[i], (i < 10 ? i : i - 10 + 1001)); + ASSERT_EQ(resultRid[i], (i < 10 ? i : i - 10 + 1001)); + ASSERT_EQ(resultVal[i], (i < 10 ? i : i - 10 + 1001)); } ASSERT_EQ(out->Max, __col8block_cdf_umax); ASSERT_EQ(out->Min, __col8block_cdf_umin); } -//void p_Col_12() +// void p_Col_12() TEST_F(ColumnScanFilterTest, ColumnScan1Byte2CompFilters) { constexpr const uint8_t W = 1; @@ -708,14 +707,14 @@ TEST_F(ColumnScanFilterTest, ColumnScan1Byte2CompFilters) args->COP = COMPARE_LT; args->val[0] = '4'; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col1block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); ASSERT_EQ(out->NVALS, 32); for (i = 0; i < out->NVALS; i++) - ASSERT_EQ( (int)'3', results[i]); + ASSERT_EQ((int)'3', results[i]); } /* The code doesn't support filters with RID and literal @@ -766,7 +765,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes2CompFiltersOutputRID) ASSERT_EQ(results[1], 11); } */ -//void p_Col_double_1() +// void p_Col_double_1() TEST_F(ColumnScanFilterTest, ColumnScan8BytesDouble2CompFilters) { constexpr const uint8_t W = 8; @@ -789,17 +788,17 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesDouble2CompFilters) args->COP = COMPARE_LT; memcpy(args->val, &tmp, sizeof(tmp)); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col_double_block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col_double_block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); ASSERT_EQ(out->NVALS, 8); for (i = 0; i < out->NVALS; i++) - ASSERT_EQ(results[i], 11 + (i * 0.5)); + ASSERT_EQ(results[i], 11 + (i * 0.5)); } -//void p_Col_float_1() +// void p_Col_float_1() TEST_F(ColumnScanFilterTest, ColumnScan4BytesFloat2CompFiltersOutputBoth) { constexpr const uint8_t W = 4; @@ -814,7 +813,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesFloat2CompFiltersOutputBoth) in->BOP = BOP_AND; in->NOPS = 2; in->NVALS = 0; - + tmp = 10.5; args->COP = COMPARE_GT; memcpy(args->val, &tmp, sizeof(tmp)); @@ -823,19 +822,19 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesFloat2CompFiltersOutputBoth) args->COP = COMPARE_LT; memcpy(args->val, &tmp, sizeof(tmp)); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col_float_block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col_float_block.cdf", block)); pp.columnScanAndFilter(in, out); ASSERT_EQ(out->NVALS, 8); for (i = 0; i < out->NVALS; i++) { - ASSERT_EQ(resultRid[i], 19 + i); - ASSERT_EQ(resultVal[i], 11 + (i * 0.5)); + ASSERT_EQ(resultRid[i], 19 + i); + ASSERT_EQ(resultVal[i], 11 + (i * 0.5)); } } -//void p_Col_neg_float_1() +// void p_Col_neg_float_1() TEST_F(ColumnScanFilterTest, ColumnScan4BytesNegFloat2CompFiltersOutputBoth) { using IntegralType = float; @@ -849,7 +848,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesNegFloat2CompFiltersOutputBoth) in->BOP = BOP_AND; in->NOPS = 2; in->NVALS = 0; - + tmp = -5.0; args->COP = COMPARE_GT; memcpy(args->val, &tmp, sizeof(tmp)); @@ -858,18 +857,18 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesNegFloat2CompFiltersOutputBoth) args->COP = COMPARE_LT; memcpy(args->val, &tmp, sizeof(tmp)); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col_neg_float.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col_neg_float.cdf", block)); pp.columnScanAndFilter(in, out); ASSERT_EQ(out->NVALS, 19); for (i = 0; i < out->NVALS; i++) { - ASSERT_EQ(resultRid[i], 12 + i); - ASSERT_EQ(resultVal[i], -4.5 + (i * 0.5)); + ASSERT_EQ(resultRid[i], 12 + i); + ASSERT_EQ(resultVal[i], -4.5 + (i * 0.5)); } } -//void p_Col_neg_double_1() +// void p_Col_neg_double_1() TEST_F(ColumnScanFilterTest, ColumnScan8BytesNegDouble2CompFilters) { constexpr const uint8_t W = 8; @@ -892,7 +891,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesNegDouble2CompFilters) args->COP = COMPARE_LT; memcpy(args->val, &tmp, sizeof(tmp)); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col_neg_double.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col_neg_double.cdf", block)); pp.columnScanAndFilter(in, out); ASSERT_EQ(out->NVALS, 19); @@ -916,7 +915,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes) in->BOP = BOP_OR; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray("col16block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col16block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); @@ -926,7 +925,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes) int128_t NullValue = int128_t(0x8000000000000000LL) << 64; ASSERT_EQ(results[0], NullValue); for (i = 1; i < out->NVALS; ++i) - ASSERT_EQ(results[i], i+1); + ASSERT_EQ(results[i], i + 1); EXPECT_EQ(out->Max, __col16block_cdf_umax); EXPECT_EQ(out->Min, __col16block_cdf_umin); } @@ -948,13 +947,13 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) tmp = 10; args->COP = COMPARE_EQ; memcpy(args->val, &tmp, in->colType.DataSize); - args = reinterpret_cast(&input[sizeof(NewColRequestHeader) + - sizeof(ColArgs) + in->colType.DataSize]); + args = reinterpret_cast( + &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); args->COP = COMPARE_EQ; tmp = 510; memcpy(args->val, &tmp, in->colType.DataSize); - pp.setBlockPtr((int*) readBlockFromLiteralArray("col16block.cdf", block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray("col16block.cdf", block)); pp.columnScanAndFilter(in, out); results = getValuesArrayPosition(getFirstValueArrayPosition(out), 0); diff --git a/tests/primitives_scan_bench.cpp b/tests/primitives_scan_bench.cpp index 84e6f4716..2d6b77c9a 100644 --- a/tests/primitives_scan_bench.cpp +++ b/tests/primitives_scan_bench.cpp @@ -39,24 +39,24 @@ using namespace std; uint8_t* readBlockFromLiteralArray(const std::string& fileName, uint8_t* block) { - if (fileName == std::string("col1block.cdf")) - return &__col1block_cdf[0]; - else if (fileName == std::string("col2block.cdf")) - return &__col2block_cdf[0]; - else if (fileName == std::string("col4block.cdf")) - return &__col4block_cdf[0]; - else if (fileName == std::string("col8block.cdf")) - return &___bin_col8block_cdf[0]; - else if (fileName == std::string("col_float_block.cdf")) - return &___bin_col_float_block_cdf[0]; - else if (fileName == std::string("col_double_block.cdf")) - return &___bin_col_double_block_cdf[0]; - else if (fileName == std::string("col_neg_float.cdf")) - return &___bin_col_neg_float_cdf[0]; - else if (fileName == std::string("col_neg_double.cdf")) - return &___bin_col_neg_double_cdf[0]; + if (fileName == std::string("col1block.cdf")) + return &__col1block_cdf[0]; + else if (fileName == std::string("col2block.cdf")) + return &__col2block_cdf[0]; + else if (fileName == std::string("col4block.cdf")) + return &__col4block_cdf[0]; + else if (fileName == std::string("col8block.cdf")) + return &___bin_col8block_cdf[0]; + else if (fileName == std::string("col_float_block.cdf")) + return &___bin_col_float_block_cdf[0]; + else if (fileName == std::string("col_double_block.cdf")) + return &___bin_col_double_block_cdf[0]; + else if (fileName == std::string("col_neg_float.cdf")) + return &___bin_col_neg_float_cdf[0]; + else if (fileName == std::string("col_neg_double.cdf")) + return &___bin_col_neg_double_cdf[0]; - return nullptr; + return nullptr; } class FilterBenchFixture : public benchmark::Fixture @@ -89,7 +89,8 @@ class FilterBenchFixture : public benchmark::Fixture SetUp(const_cast(state)); } - void inTestRunSetUp(const std::string& dataName, const size_t dataSize, const uint8_t dataType, const uint32_t outputType, ColArgs* args) + void inTestRunSetUp(const std::string& dataName, const size_t dataSize, const uint8_t dataType, + const uint32_t outputType, ColArgs* args) { in->colType = ColRequestHeaderDataType(); in->colType.DataSize = dataSize; @@ -97,21 +98,21 @@ class FilterBenchFixture : public benchmark::Fixture in->OutputType = outputType; in->NOPS = 0; in->NVALS = 0; - pp.setBlockPtr((int*) readBlockFromLiteralArray(dataName, block)); + pp.setBlockPtr((int*)readBlockFromLiteralArray(dataName, block)); } void runFilterBenchLegacy() { pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); } - template + template void runFilterBenchTemplated() { using IntegralType = typename datatypes::WidthToSIntegralType::type; pp.columnScanAndFilter(in, out); } - template + template void setUp1EqFilter() { using IntegralType = typename datatypes::WidthToSIntegralType::type; @@ -121,7 +122,6 @@ class FilterBenchFixture : public benchmark::Fixture args->COP = COMPARE_EQ; memcpy(args->val, &tmp, W); } - }; BENCHMARK_DEFINE_F(FilterBenchFixture, BM_ColumnScan1ByteLegacyCode)(benchmark::State& state) diff --git a/tests/rebuild-em-tests.cpp b/tests/rebuild-em-tests.cpp index d9c4b6a1e..464b314e2 100644 --- a/tests/rebuild-em-tests.cpp +++ b/tests/rebuild-em-tests.cpp @@ -27,20 +27,23 @@ using namespace WriteEngine; class RebuildEMTest : public ::testing::Test { -protected: + protected: struct FileId { - FileId() : oid(0), partition(0), segment(0) {} - FileId(uint32_t oid, uint32_t partition, uint32_t segment) - : oid(oid), partition(partition), segment(segment) - { - } - uint32_t oid; - uint32_t partition; - uint32_t segment; + FileId() : oid(0), partition(0), segment(0) + { + } + FileId(uint32_t oid, uint32_t partition, uint32_t segment) + : oid(oid), partition(partition), segment(segment) + { + } + uint32_t oid; + uint32_t partition; + uint32_t segment; }; - static uint32_t getOid(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + static uint32_t getOid(uint32_t a, uint32_t b, uint32_t c, uint32_t d) + { uint32_t oid = 0; oid |= a << 24; oid |= b << 16; @@ -52,111 +55,100 @@ protected: TEST_F(RebuildEMTest, File2OidCheckFileFormatTest) { - // Valid file names. - auto aFileName = "001.dir/002.dir/003.dir/004.dir/005.dir/FILE006.cdf"; - FileId aExpected(getOid(1, 2, 3, 4), 5, 6); + // Valid file names. + auto aFileName = "001.dir/002.dir/003.dir/004.dir/005.dir/FILE006.cdf"; + FileId aExpected(getOid(1, 2, 3, 4), 5, 6); - auto bFileName = "/011.dir/022.dir/033.dir/044.dir/055.dir/FILE066.cdf"; - FileId bExpected(getOid(11, 22, 33, 44), 55, 66); + auto bFileName = "/011.dir/022.dir/033.dir/044.dir/055.dir/FILE066.cdf"; + FileId bExpected(getOid(11, 22, 33, 44), 55, 66); - auto cFileName = - "data1/255.dir/255.dir/255.dir/255.dir/255.dir/FILE255.cdf"; - FileId cExpected(getOid(255, 255, 255, 255), 255, 255); + auto cFileName = "data1/255.dir/255.dir/255.dir/255.dir/255.dir/FILE255.cdf"; + FileId cExpected(getOid(255, 255, 255, 255), 255, 255); - auto dFileName = - "/data1/000.dir/000.dir/000.dir/001.dir/001.dir/FILE001.cdf"; - FileId dExpected(getOid(0, 0, 0, 1), 1, 1); + auto dFileName = "/data1/000.dir/000.dir/000.dir/001.dir/001.dir/FILE001.cdf"; + FileId dExpected(getOid(0, 0, 0, 1), 1, 1); - auto eFileName = "/data0/data1/data2/data3/data4/data5/data6/../../../../" - "../data1/000.dir/000.dir/000.dir/000.dir/" - "000.dir/FILE000.cdf"; - FileId eExpected(getOid(0, 0, 0, 0), 0, 0); + auto eFileName = + "/data0/data1/data2/data3/data4/data5/data6/../../../../" + "../data1/000.dir/000.dir/000.dir/000.dir/" + "000.dir/FILE000.cdf"; + FileId eExpected(getOid(0, 0, 0, 0), 0, 0); - auto fFileName = "data1/data2/data3/data4/data5/data6/data7/000.dir/" - "000.dir/008.dir/028.dir/000.dir/FILE079.cdf"; - FileId fExpected(getOid(0, 0, 8, 28), 00, 79); + auto fFileName = + "data1/data2/data3/data4/data5/data6/data7/000.dir/" + "000.dir/008.dir/028.dir/000.dir/FILE079.cdf"; + FileId fExpected(getOid(0, 0, 8, 28), 00, 79); - std::vector> expectedFileIds = { - make_pair(aFileName, aExpected), make_pair(bFileName, bExpected), - make_pair(cFileName, cExpected), make_pair(dFileName, dExpected), - make_pair(eFileName, eExpected), make_pair(fFileName, fExpected)}; + std::vector> expectedFileIds = { + make_pair(aFileName, aExpected), make_pair(bFileName, bExpected), make_pair(cFileName, cExpected), + make_pair(dFileName, dExpected), make_pair(eFileName, eExpected), make_pair(fFileName, fExpected)}; - for (const auto& expectedPair : expectedFileIds) - { - FileId calculated; - auto rc = WriteEngine::Convertor::fileName2Oid( - expectedPair.first, calculated.oid, calculated.partition, - calculated.segment); + for (const auto& expectedPair : expectedFileIds) + { + FileId calculated; + auto rc = WriteEngine::Convertor::fileName2Oid(expectedPair.first, calculated.oid, calculated.partition, + calculated.segment); - ASSERT_EQ(rc, 0); - EXPECT_EQ(expectedPair.second.oid, calculated.oid); - EXPECT_EQ(expectedPair.second.partition, calculated.partition); - EXPECT_EQ(expectedPair.second.segment, calculated.segment); - } + ASSERT_EQ(rc, 0); + EXPECT_EQ(expectedPair.second.oid, calculated.oid); + EXPECT_EQ(expectedPair.second.partition, calculated.partition); + EXPECT_EQ(expectedPair.second.segment, calculated.segment); + } - // Invalid file names. - // Dir exceed 255. - auto aInvalidFileName = - "256.dir/000.dir/001.dir/002.dir/003.dir/FILE004.cdf"; - // Segment exceed 255. - auto bInvalidFileName = - "000.dir/000.dir/001.dir/002.dir/003.dir/FILE256.cdf"; - // Just a random path. - auto cInvalidFileName = "/usr////bin//lib///"; - // Empty string. - auto dInvalidFileName = ""; - // Does not have A dir. - auto eInvalidFileName = - "/data1/000.dir/001.dir/002.dir/003.dir/FILE000.cdf"; - // Invalid partition name dir. - auto fInvalidFileName = - "/000.dir/000.dir/001.dir/002.dir/003.ir/FILE000.cdf"; - // Invalid segment name. - auto gInvalidFileName = - "/000.dir/000.dir/001.dir/002.dir/003.ir/FILE00.cdf"; - // Invalid dir name. - auto hInvalidFileName = "/00.dir/00.dir/001.dir/002.dir/003.ir/FIE000.cdf"; - // Invalid amount of dirs. - auto iInvalidFileName = "/002.dir/003.ir/FIE000.cdf"; + // Invalid file names. + // Dir exceed 255. + auto aInvalidFileName = "256.dir/000.dir/001.dir/002.dir/003.dir/FILE004.cdf"; + // Segment exceed 255. + auto bInvalidFileName = "000.dir/000.dir/001.dir/002.dir/003.dir/FILE256.cdf"; + // Just a random path. + auto cInvalidFileName = "/usr////bin//lib///"; + // Empty string. + auto dInvalidFileName = ""; + // Does not have A dir. + auto eInvalidFileName = "/data1/000.dir/001.dir/002.dir/003.dir/FILE000.cdf"; + // Invalid partition name dir. + auto fInvalidFileName = "/000.dir/000.dir/001.dir/002.dir/003.ir/FILE000.cdf"; + // Invalid segment name. + auto gInvalidFileName = "/000.dir/000.dir/001.dir/002.dir/003.ir/FILE00.cdf"; + // Invalid dir name. + auto hInvalidFileName = "/00.dir/00.dir/001.dir/002.dir/003.ir/FIE000.cdf"; + // Invalid amount of dirs. + auto iInvalidFileName = "/002.dir/003.ir/FIE000.cdf"; - std::vector invalidFileNames = { - aInvalidFileName, bInvalidFileName, cInvalidFileName, - dInvalidFileName, eInvalidFileName, fInvalidFileName, - gInvalidFileName, hInvalidFileName, iInvalidFileName}; + std::vector invalidFileNames = {aInvalidFileName, bInvalidFileName, cInvalidFileName, + dInvalidFileName, eInvalidFileName, fInvalidFileName, + gInvalidFileName, hInvalidFileName, iInvalidFileName}; - for (const auto& invalidFileName : invalidFileNames) - { - FileId calculated; - auto rc = WriteEngine::Convertor::fileName2Oid( - invalidFileName, calculated.oid, calculated.partition, - calculated.segment); - ASSERT_NE(rc, 0); - } + for (const auto& invalidFileName : invalidFileNames) + { + FileId calculated; + auto rc = WriteEngine::Convertor::fileName2Oid(invalidFileName, calculated.oid, calculated.partition, + calculated.segment); + ASSERT_NE(rc, 0); + } } TEST_F(RebuildEMTest, File2OidCalculationTest) { - char dbDirName[20][20]; - char fileName[64]; + char dbDirName[20][20]; + char fileName[64]; - for (uint32_t i = 3; i < 256; ++i) - { - FileId expectedFileId(getOid(i, i - 1, i - 2, i - 3), i, i); - memset(fileName, 0, 64); - // Generate the filename by the oid, partition and segment. - auto rc = WriteEngine::Convertor::oid2FileName( - expectedFileId.oid, fileName, dbDirName, expectedFileId.partition, - expectedFileId.segment); - ASSERT_EQ(rc, 0); + for (uint32_t i = 3; i < 256; ++i) + { + FileId expectedFileId(getOid(i, i - 1, i - 2, i - 3), i, i); + memset(fileName, 0, 64); + // Generate the filename by the oid, partition and segment. + auto rc = WriteEngine::Convertor::oid2FileName(expectedFileId.oid, fileName, dbDirName, + expectedFileId.partition, expectedFileId.segment); + ASSERT_EQ(rc, 0); - FileId calculatedFileId; - // Generate an oid, partition and segment from the given file name. - rc = WriteEngine::Convertor::fileName2Oid( - fileName, calculatedFileId.oid, calculatedFileId.partition, - calculatedFileId.segment); - ASSERT_EQ(rc, 0); - EXPECT_EQ(expectedFileId.oid, calculatedFileId.oid); - EXPECT_EQ(expectedFileId.partition, calculatedFileId.partition); - EXPECT_EQ(expectedFileId.segment, calculatedFileId.segment); - } + FileId calculatedFileId; + // Generate an oid, partition and segment from the given file name. + rc = WriteEngine::Convertor::fileName2Oid(fileName, calculatedFileId.oid, calculatedFileId.partition, + calculatedFileId.segment); + ASSERT_EQ(rc, 0); + EXPECT_EQ(expectedFileId.oid, calculatedFileId.oid); + EXPECT_EQ(expectedFileId.partition, calculatedFileId.partition); + EXPECT_EQ(expectedFileId.segment, calculatedFileId.segment); + } } diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index 38eebc1b7..b1c6d5fa5 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -15,7 +15,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include // googletest header file +#include // googletest header file #include #include "rowgroup.h" @@ -30,126 +30,124 @@ using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; class RowDecimalTest : public ::testing::Test { -protected: - void SetUp() override + protected: + void SetUp() override + { + uint32_t precision = WIDE_DEC_PRECISION; + uint32_t oid = 3001; + + std::vector types; + std::vector precisionVec; + std::vector roids, tkeys, cscale; + std::vector widthVec; + std::vector charSetNumVec; + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + + for (size_t i = 0; i <= 3; i++) + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + + for (size_t i = 0; i < types.size(); i++) { - uint32_t precision = WIDE_DEC_PRECISION; - uint32_t oid = 3001; - - std::vector types; - std::vector precisionVec; - std::vector roids, tkeys, cscale; - std::vector widthVec; - std::vector charSetNumVec; - types.push_back(execplan::CalpontSystemCatalog::DECIMAL); - types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); - - for (size_t i = 0; i <= 3; i++) - types.push_back(execplan::CalpontSystemCatalog::DECIMAL); - - precisionVec.push_back(precision); - precisionVec.push_back(precision); - precisionVec.push_back(18); - precisionVec.push_back(9); - precisionVec.push_back(4); - precisionVec.push_back(2); - - uint32_t offset = INITIAL_ROW_OFFSET; - offsets.push_back(offset); - - for (size_t i = 0; i < types.size(); i++) - { - uint8_t width = utils::widthByPrecision(precisionVec[i]); - widthVec.push_back(width); - offset += width; - offsets.push_back(offset); - roids.push_back(oid + i); - tkeys.push_back(i + 1); - cscale.push_back(0); - charSetNumVec.push_back(8); - } - - rowgroup::RowGroup inRG(roids.size(), // column count - offsets, // oldOffset - roids, // column oids - tkeys, // keys - types, // types - charSetNumVec, // charset numbers - cscale, // scale - precisionVec, // precision - 20, // sTableThreshold - false // useStringTable - ); - - rg = rgOut = inRG; - rgD.reinit(rg); - rgDOut.reinit(rgOut); - rg.setData(&rgD); - rgOut.setData(&rgDOut); - - rg.initRow(&r); - rg.initRow(&rOutMappingCheck); - rowSize = r.getSize(); - rg.getRow(0, &r); - - int128_t nullValue = 0; - int128_t bigValue = 0; - - datatypes::Decimal::setWideDecimalNullValue(nullValue); - bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; - - sValueVector.push_back(nullValue); - sValueVector.push_back(-42); - sValueVector.push_back(bigValue); - sValueVector.push_back(0); - sValueVector.push_back(nullValue-1); - - anotherValueVector.push_back(nullValue); - anotherValueVector.push_back(42); - anotherValueVector.push_back(bigValue); - anotherValueVector.push_back(0); - anotherValueVector.push_back(nullValue-1); - s8ValueVector.push_back(joblist::TINYINTNULL); - s8ValueVector.push_back(-0x79); - s8ValueVector.push_back(0); - s8ValueVector.push_back(0x81); - s8ValueVector.push_back(joblist::TINYINTNULL-1); - - s16ValueVector.push_back(joblist::SMALLINTNULL); - s16ValueVector.push_back(-0x79); - s16ValueVector.push_back(0); - s16ValueVector.push_back(0x81); - s16ValueVector.push_back(joblist::SMALLINTNULL-1); - - s32ValueVector.push_back(joblist::INTNULL); - s32ValueVector.push_back(-0x79); - s32ValueVector.push_back(0); - s32ValueVector.push_back(0x81); - s32ValueVector.push_back(joblist::INTNULL-1); - - s64ValueVector.push_back(joblist::BIGINTNULL); - s64ValueVector.push_back(-0x79); - s64ValueVector.push_back(0); - s64ValueVector.push_back(0x81); - s64ValueVector.push_back(joblist::BIGINTNULL-1); - - for (size_t i = 0; i < sValueVector.size(); i++) - { - r.setBinaryField_offset(&sValueVector[i], - sizeof(sValueVector[0]), offsets[0]); - r.setBinaryField_offset(&anotherValueVector[i], - sizeof(anotherValueVector[0]), offsets[1]); - r.setIntField(s64ValueVector[i], 2); - r.setIntField(s32ValueVector[i], 3); - r.setIntField(s16ValueVector[i], 4); - r.setIntField(s8ValueVector[i], 5); - r.nextRow(rowSize); - } - - rowCount = sValueVector.size(); + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid + i); + tkeys.push_back(i + 1); + cscale.push_back(0); + charSetNumVec.push_back(8); } -// void TearDown() override {} + rowgroup::RowGroup inRG(roids.size(), // column count + offsets, // oldOffset + roids, // column oids + tkeys, // keys + types, // types + charSetNumVec, // charset numbers + cscale, // scale + precisionVec, // precision + 20, // sTableThreshold + false // useStringTable + ); + + rg = rgOut = inRG; + rgD.reinit(rg); + rgDOut.reinit(rgOut); + rg.setData(&rgD); + rgOut.setData(&rgDOut); + + rg.initRow(&r); + rg.initRow(&rOutMappingCheck); + rowSize = r.getSize(); + rg.getRow(0, &r); + + int128_t nullValue = 0; + int128_t bigValue = 0; + + datatypes::Decimal::setWideDecimalNullValue(nullValue); + bigValue = -static_cast(0xFFFFFFFF) * 0xFFFFFFFFFFFFFFFF; + + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(bigValue); + sValueVector.push_back(0); + sValueVector.push_back(nullValue - 1); + + anotherValueVector.push_back(nullValue); + anotherValueVector.push_back(42); + anotherValueVector.push_back(bigValue); + anotherValueVector.push_back(0); + anotherValueVector.push_back(nullValue - 1); + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL - 1); + + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL - 1); + + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL - 1); + + s64ValueVector.push_back(joblist::BIGINTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::BIGINTNULL - 1); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + r.setBinaryField_offset(&sValueVector[i], sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&anotherValueVector[i], sizeof(anotherValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + r.nextRow(rowSize); + } + + rowCount = sValueVector.size(); + } + + // void TearDown() override {} rowgroup::Row r, rOut, sameRow, nonequiRow; rowgroup::Row rOutMappingCheck; @@ -168,71 +166,74 @@ protected: TEST_F(RowDecimalTest, NonNullValueCheck) { - rg.getRow(1, &r); + rg.getRow(1, &r); - for (size_t i = 1; i <= sValueVector.size(); i++) - { - EXPECT_FALSE(r.isNullValue(0)); - EXPECT_FALSE(r.isNullValue(1)); - EXPECT_FALSE(r.isNullValue(2)); - EXPECT_FALSE(r.isNullValue(3)); - EXPECT_FALSE(r.isNullValue(4)); - EXPECT_FALSE(r.isNullValue(5)); - r.nextRow(rowSize); - } + for (size_t i = 1; i <= sValueVector.size(); i++) + { + EXPECT_FALSE(r.isNullValue(0)); + EXPECT_FALSE(r.isNullValue(1)); + EXPECT_FALSE(r.isNullValue(2)); + EXPECT_FALSE(r.isNullValue(3)); + EXPECT_FALSE(r.isNullValue(4)); + EXPECT_FALSE(r.isNullValue(5)); + r.nextRow(rowSize); + } } TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck) { - rg.getRow(0, &r); - r.initToNull(); + rg.getRow(0, &r); + r.initToNull(); - EXPECT_TRUE(r.isNullValue(0)); - EXPECT_TRUE(r.isNullValue(1)); - EXPECT_TRUE(r.isNullValue(2)); - EXPECT_TRUE(r.isNullValue(3)); - EXPECT_TRUE(r.isNullValue(4)); - EXPECT_TRUE(r.isNullValue(5)); + EXPECT_TRUE(r.isNullValue(0)); + EXPECT_TRUE(r.isNullValue(1)); + EXPECT_TRUE(r.isNullValue(2)); + EXPECT_TRUE(r.isNullValue(3)); + EXPECT_TRUE(r.isNullValue(4)); + EXPECT_TRUE(r.isNullValue(5)); } TEST_F(RowDecimalTest, GetBinaryFieldCheck) { - rg.getRow(0, &r); - int128_t* a128Value; - int128_t* s128Value; + rg.getRow(0, &r); + int128_t* a128Value; + int128_t* s128Value; - for (size_t i = 0; i < sValueVector.size(); i++) - { - s128Value = r.getBinaryField(0); - EXPECT_EQ(sValueVector[i], *s128Value); - a128Value = r.getBinaryField(1); - EXPECT_EQ(anotherValueVector[i], *a128Value); - //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); - //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); - //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); - //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); - r.nextRow(rowSize); - } + for (size_t i = 0; i < sValueVector.size(); i++) + { + s128Value = r.getBinaryField(0); + EXPECT_EQ(sValueVector[i], *s128Value); + a128Value = r.getBinaryField(1); + EXPECT_EQ(anotherValueVector[i], *a128Value); + // EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + // EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); + // EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); + // EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); + r.nextRow(rowSize); + } } TEST_F(RowDecimalTest, ToStringCheck) { - std::vector exemplarVector; + std::vector exemplarVector; - exemplarVector.push_back(std::string("[ 0]: 0: NULL NULL NULL NULL NULL NULL ")); - exemplarVector.push_back(std::string("[ 0]: 0: -42 42 -121 -121 -121 -121 ")); - exemplarVector.push_back(std::string("[ 0]: 0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); - exemplarVector.push_back(std::string("[ 0]: 0: 0 0 129 129 129 -127 ")); - exemplarVector.push_back(std::string("[ 0]: 0: 170141183460469231731687303715884105727 170141183460469231731687303715884105727 9223372036854775807 2147483647 32767 127 ")); + exemplarVector.push_back(std::string("[ 0]: 0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("[ 0]: 0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back( + std::string("[ 0]: 0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("[ 0]: 0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back( + std::string("[ 0]: 0: 170141183460469231731687303715884105727 " + "170141183460469231731687303715884105727 9223372036854775807 2147483647 32767 127 ")); - rg.getRow(0, &r); - r.initToNull(); + rg.getRow(0, &r); + r.initToNull(); - for (auto& el: exemplarVector) - { - EXPECT_EQ(el, r.toString()); - r.nextRow(rowSize); - } + for (auto& el : exemplarVector) + { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } } TEST_F(RowDecimalTest, ToCSVCheck) @@ -241,75 +242,75 @@ TEST_F(RowDecimalTest, ToCSVCheck) TEST_F(RowDecimalTest, ApplyMappingCheck) { - int mapping[] = {0, 1, -1, -1, -1, -1}; - rg.getRow(1, &r); - rg.getRow(2, &rOutMappingCheck); - int128_t* s128Value = rOutMappingCheck.getBinaryField(0); - int128_t* a128Value = rOutMappingCheck.getBinaryField(1); - EXPECT_NE(sValueVector[1], *s128Value); - EXPECT_NE(anotherValueVector[1], *a128Value); - applyMapping(mapping, r, &rOutMappingCheck); - s128Value = rOutMappingCheck.getBinaryField(0); - a128Value = rOutMappingCheck.getBinaryField(1); - EXPECT_EQ(sValueVector[1], *s128Value); - EXPECT_EQ(anotherValueVector[1], *a128Value); + int mapping[] = {0, 1, -1, -1, -1, -1}; + rg.getRow(1, &r); + rg.getRow(2, &rOutMappingCheck); + int128_t* s128Value = rOutMappingCheck.getBinaryField(0); + int128_t* a128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_NE(sValueVector[1], *s128Value); + EXPECT_NE(anotherValueVector[1], *a128Value); + applyMapping(mapping, r, &rOutMappingCheck); + s128Value = rOutMappingCheck.getBinaryField(0); + a128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(sValueVector[1], *s128Value); + EXPECT_EQ(anotherValueVector[1], *a128Value); } TEST_F(RowDecimalTest, CopyBinaryFieldCheck) { - int128_t constVal = 1; - int128_t *col1Out, *col1In; - int128_t *col2Out, *col2In; - rgOut.getRow(0, &rOut); + int128_t constVal = 1; + int128_t *col1Out, *col1In; + int128_t *col2Out, *col2In; + rgOut.getRow(0, &rOut); - for (size_t i = 0; i < sValueVector.size(); i++) - { - rOut.setBinaryField_offset(&constVal, 16, offsets[0]); - rOut.setBinaryField_offset(&constVal, 16, offsets[1]); - rOut.nextRow(rowSize); - } + for (size_t i = 0; i < sValueVector.size(); i++) + { + rOut.setBinaryField_offset(&constVal, 16, offsets[0]); + rOut.setBinaryField_offset(&constVal, 16, offsets[1]); + rOut.nextRow(rowSize); + } - rgOut.initRow(&rOut); - rgOut.getRow(0, &rOut); - rg.getRow(0, &r); + rgOut.initRow(&rOut); + rgOut.getRow(0, &rOut); + rg.getRow(0, &r); - for (size_t i = 0; i < sValueVector.size(); i++) - { - col1In = r.getBinaryField(0); - col1Out = rOut.getBinaryField(0); - col2In = r.getBinaryField(1); - col2Out = rOut.getBinaryField(1); - EXPECT_NE(*col1In, *col1Out); - EXPECT_NE(*col2In, *col2Out); - r.copyBinaryField(rOut, 0, 0); - r.copyBinaryField(rOut, 1, 1); - col1Out = rOut.getBinaryField(0); - col2Out = rOut.getBinaryField(1); - EXPECT_EQ(*col1In, *col1Out); - EXPECT_EQ(*col2In, *col2Out); - r.nextRow(rowSize); - rOut.nextRow(rowSize); - } + for (size_t i = 0; i < sValueVector.size(); i++) + { + col1In = r.getBinaryField(0); + col1Out = rOut.getBinaryField(0); + col2In = r.getBinaryField(1); + col2Out = rOut.getBinaryField(1); + EXPECT_NE(*col1In, *col1Out); + EXPECT_NE(*col2In, *col2Out); + r.copyBinaryField(rOut, 0, 0); + r.copyBinaryField(rOut, 1, 1); + col1Out = rOut.getBinaryField(0); + col2Out = rOut.getBinaryField(1); + EXPECT_EQ(*col1In, *col1Out); + EXPECT_EQ(*col2In, *col2Out); + r.nextRow(rowSize); + rOut.nextRow(rowSize); + } } TEST_F(RowDecimalTest, RowEqualsCheck) { - rg.getRow(0, &r); - rg.getRow(0, &sameRow); - rg.getRow(1, &nonequiRow); + rg.getRow(0, &r); + rg.getRow(0, &sameRow); + rg.getRow(1, &nonequiRow); - for (size_t i = 0; i < sValueVector.size(); i++) + for (size_t i = 0; i < sValueVector.size(); i++) + { + EXPECT_TRUE(r.equals(sameRow)); + if (i < sValueVector.size() - 1) { - EXPECT_TRUE(r.equals(sameRow)); - if (i < sValueVector.size()-1) - { - EXPECT_FALSE(r.equals(nonequiRow)); - } - r.nextRow(rowSize); - sameRow.nextRow(rowSize); - if (i < sValueVector.size()-1) - { - nonequiRow.nextRow(rowSize); - } + EXPECT_FALSE(r.equals(nonequiRow)); } + r.nextRow(rowSize); + sameRow.nextRow(rowSize); + if (i < sValueVector.size() - 1) + { + nonequiRow.nextRow(rowSize); + } + } } diff --git a/tests/shared_components_tests.cpp b/tests/shared_components_tests.cpp index 7f302599b..c76ebe7ab 100644 --- a/tests/shared_components_tests.cpp +++ b/tests/shared_components_tests.cpp @@ -16,7 +16,7 @@ MA 02110-1301, USA. */ // Define __STDC_LIMIT_MACROS so that we can pick up UINT64_MAX from stdint.h -#define __STDC_LIMIT_MACROS 1 +#define __STDC_LIMIT_MACROS 1 #include #include #include @@ -63,1685 +63,1653 @@ int compare1(const void* a, const void* b) class SharedTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(SharedTest); - CPPUNIT_TEST_SUITE( SharedTest ); + CPPUNIT_TEST(setUp); + // CPPUNIT_TEST( test1 ); -CPPUNIT_TEST(setUp); -//CPPUNIT_TEST( test1 ); + // File operation testing + // CPPUNIT_TEST( testFileNameOp ); + // CPPUNIT_TEST( testFileHandleOp ); + CPPUNIT_TEST(testDirBasic); + CPPUNIT_TEST(testCreateDeleteFile); -// File operation testing -// CPPUNIT_TEST( testFileNameOp ); -// CPPUNIT_TEST( testFileHandleOp ); - CPPUNIT_TEST( testDirBasic ); - CPPUNIT_TEST( testCreateDeleteFile ); + // Data block related testing + // CPPUNIT_TEST( testCalculateRowIdBitmap ); + // CPPUNIT_TEST( testBlockBuffer ); + // CPPUNIT_TEST( testBitBasic ); + // CPPUNIT_TEST( testBufferBit ); + // CPPUNIT_TEST( testBitShift ); + // CPPUNIT_TEST( testEmptyRowValue ); + // CPPUNIT_TEST( testCorrectRowWidth ); + // DB File Block related testing + // CPPUNIT_TEST( testDbBlock ); -// Data block related testing -// CPPUNIT_TEST( testCalculateRowIdBitmap ); -// CPPUNIT_TEST( testBlockBuffer ); -// CPPUNIT_TEST( testBitBasic ); -// CPPUNIT_TEST( testBufferBit ); -// CPPUNIT_TEST( testBitShift ); -// CPPUNIT_TEST( testEmptyRowValue ); -// CPPUNIT_TEST( testCorrectRowWidth ); -// DB File Block related testing -// CPPUNIT_TEST( testDbBlock ); + // CPPUNIT_TEST( testCopyDbFile ); -// CPPUNIT_TEST( testCopyDbFile ); + // Extent & dict related testing + CPPUNIT_TEST(testExtensionWOPrealloc); + CPPUNIT_TEST(testDictExtensionWOPrealloc); + // CPPUNIT_TEST( testExtentCrWOPreallocBin ); + // Semaphore related testing + // CPPUNIT_TEST( testSem ); -// Extent & dict related testing - CPPUNIT_TEST( testExtensionWOPrealloc ); - CPPUNIT_TEST( testDictExtensionWOPrealloc ); -// CPPUNIT_TEST( testExtentCrWOPreallocBin ); - // Semaphore related testing -// CPPUNIT_TEST( testSem ); + // Log related testing + CPPUNIT_TEST(testLog); -// Log related testing - CPPUNIT_TEST( testLog ); + // Version Buffer related testing + // CPPUNIT_TEST( testHWM ); + // CPPUNIT_TEST( testVB ); -// Version Buffer related testing -// CPPUNIT_TEST( testHWM ); -// CPPUNIT_TEST( testVB ); + // Disk manager related testing + // CPPUNIT_TEST( testDM ); + // CPPUNIT_TEST( tearDown ); -// Disk manager related testing -// CPPUNIT_TEST( testDM ); -// CPPUNIT_TEST( tearDown ); + // Cache related testing + // CPPUNIT_TEST( testCacheBasic ); + // CPPUNIT_TEST( testCacheReadWrite ); -// Cache related testing -// CPPUNIT_TEST( testCacheBasic ); -// CPPUNIT_TEST( testCacheReadWrite ); + CPPUNIT_TEST(testCleanup); // NEVER COMMENT OUT THIS LINE + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST( testCleanup ); // NEVER COMMENT OUT THIS LINE - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: + void tearDown() + { + } -public: - void setUp() + void test1() + { + int numOfBlock = 1024; + FILE* pFile; + unsigned char writeBuf[BYTE_PER_BLOCK * 10]; + char diskBuf[BYTE_PER_BLOCK * 11]; + char fileName[40]; + + for (int k = 0; k < 273; k++) { + snprintf(fileName, sizeof(fileName), "testwr%d.tst", k); + pFile = fopen(fileName, "w+b"); + if (pFile != NULL) + { + setvbuf(pFile, diskBuf, _IOFBF, 81920); + memset(writeBuf, 0, BYTE_PER_BLOCK * sizeof(unsigned char)); + + for (int i = 0; i < numOfBlock; i++) + fwrite(writeBuf, sizeof(writeBuf), 1, pFile); + + // fwrite( writeBuf, sizeof( writeBuf ), numOfBlock, pFile ); + fclose(pFile); + } } + } + /* + void testFileNameOp() + { + FileOp fileOp; + bool bStatus; + char fileName[20]; + int rc; + + strcpy( fileName, "FILE_999.dat" ); + BRMWrapper::setUseBrm(true); + // file opertaions + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + rc = fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( rc == ERR_FILE_NOT_EXIST ); + + rc = fileOp.createFile( fileName, 10 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = fileOp.createFile( fileName, 10 ); + CPPUNIT_ASSERT( rc == ERR_FILE_EXIST ); + + bStatus = fileOp.exists( fileName ); + CPPUNIT_ASSERT( bStatus == true ); + + // init different value + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + rc = fileOp.createFile( fileName, 10, 0xFF, 1 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + rc = fileOp.createFile( fileName, 10, 0x8001, 2 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + rc = fileOp.createFile( fileName, 10, 0xFFAAAAAB, 4 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + //Not use block resolution manager + BRMWrapper::setUseBrm(false); + // file opertaions + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + rc = fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( rc == ERR_FILE_NOT_EXIST ); + + rc = fileOp.createFile( fileName, 10 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = fileOp.createFile( fileName, 10 ); + CPPUNIT_ASSERT( rc == ERR_FILE_EXIST ); + + bStatus = fileOp.exists( fileName ); + CPPUNIT_ASSERT( bStatus == true ); + + // init different value + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + rc = fileOp.createFile( fileName, 10, 0xFF, 1 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + rc = fileOp.createFile( fileName, 10, 0x8001, 2 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + rc = fileOp.createFile( fileName, 10, 0xFFAAAAAB, 4 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + } + + void testFileHandleOp() + { + FileOp fileOp; + char fileName[100], testStr[50]; + unsigned char dataBuf[BYTE_PER_BLOCK]; + long fileSize; + FILE* pFile; + int rc; + int allocSize = 0; + + BRMWrapper::setUseBrm(true); + // strcpy( fileName, "FILE_999.dat" ); + strcpy( testStr, "String write test data" ); + + fileOp.deleteFile( 999 ); + rc = fileOp.createFile( 999, 10, allocSize); + CPPUNIT_ASSERT( rc == NO_ERROR ); + pFile = fileOp.openFile( 999 ); + CPPUNIT_ASSERT( pFile != NULL ); + + fileSize = fileOp.getFileSize( 999 ); + CPPUNIT_ASSERT( fileSize == allocSize * BYTE_PER_BLOCK ); + + rc = fileOp.setFileOffset( pFile, -1, SEEK_SET ); + CPPUNIT_ASSERT( rc == ERR_FILE_SEEK ); + + uint64_t lbid1; + rc = BRMWrapper::getInstance()->getBrmInfo( 1999, 1, lbid1 ); + CPPUNIT_ASSERT( rc != NO_ERROR ); + rc = BRMWrapper::getInstance()->getBrmInfo( 999, 1, lbid1 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = fileOp.setFileOffset( pFile, lbid1 * BYTE_PER_BLOCK, SEEK_SET ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + uint64_t lbid9; + rc = BRMWrapper::getInstance()->getBrmInfo( 999, 9, lbid9 ); + + CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = fileOp.setFileOffset( pFile, BYTE_PER_BLOCK * lbid9, SEEK_SET ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // write something to block 1 + memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); + memcpy( dataBuf + 8, testStr, strlen( testStr ) ); + + CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, lbid1 * BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); + rc = fileOp.writeFile( pFile, dataBuf, BYTE_PER_BLOCK ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = fileOp.writeFile( NULL, dataBuf, BYTE_PER_BLOCK ); + CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); + + // read something from block 1 + memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); + + rc = BRMWrapper::getInstance()->getBrmInfo( 999, 1, lbid1 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, lbid1 * BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); + rc = fileOp.readFile( pFile, dataBuf, BYTE_PER_BLOCK ); + + printf( "\ndataBuf=%s\n", dataBuf + 8 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( memcmp( dataBuf + 8, testStr, strlen(testStr)) == 0 ); + + CPPUNIT_ASSERT( fileOp.getFileSize( 999 ) == allocSize * 8192 ); + fileOp.closeFile( pFile ); + fileOp.deleteFile(999); + + // No block resolution manager + BRMWrapper::setUseBrm(false); + // strcpy( fileName, "FILE_999.dat" ); + rc = fileOp.getFileName( 999, fileName ); + strcpy( testStr, "String write test data" ); + fileOp.deleteFile( 999 ); + CPPUNIT_ASSERT( fileOp.exists( 999 ) == false ); + + rc = fileOp.createFile( 999, 10, allocSize); + CPPUNIT_ASSERT( rc == NO_ERROR ); + pFile = fileOp.openFile( fileName ); + CPPUNIT_ASSERT( pFile != NULL ); + + fileSize = fileOp.getFileSize( 999 ); + CPPUNIT_ASSERT( fileSize == 81920 ); + + rc = fileOp.setFileOffset( pFile, -1, SEEK_SET ); + CPPUNIT_ASSERT( rc == ERR_FILE_SEEK ); + + rc = fileOp.setFileOffset( pFile, BYTE_PER_BLOCK, SEEK_SET ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = fileOp.setFileOffset( pFile, BYTE_PER_BLOCK * 9, SEEK_SET ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // write something to block 1 + memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); + memcpy( dataBuf + 8, testStr, strlen( testStr ) ); + + CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); + rc = fileOp.writeFile( pFile, dataBuf, BYTE_PER_BLOCK ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = fileOp.writeFile( NULL, dataBuf, BYTE_PER_BLOCK ); + CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); + + // read something from block 1 + memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); + + CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); + rc = fileOp.readFile( pFile, dataBuf, BYTE_PER_BLOCK ); + + printf( "\ndataBuf=%s\n", dataBuf + 8 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( memcmp( dataBuf + 8, testStr, strlen(testStr)) == 0 ); + + CPPUNIT_ASSERT( fileOp.getFileSize( 999 ) == 81920 ); + fileOp.closeFile( pFile ); + fileOp.deleteFile(999); + + } + + */ + void testDirBasic() + { + FileOp fileOp; + char dirName[30]; + int rc; + + printf("\nRunning testDirBasic \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + IDBFileSystem& fs = IDBPolicy::getFs("/tmp"); + strcpy(dirName, "/tmp/testdir42"); + fs.remove(dirName); + CPPUNIT_ASSERT(fileOp.isDir(dirName) == false); + + rc = fileOp.createDir(dirName); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(fileOp.isDir(dirName) == true); + + fs.remove(dirName); + } + + void testCreateDeleteFile() + { + IDBDataFile* pFile = NULL; + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + + printf("\nRunning testCreateDeleteFile \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy(fileName, "versionbuffer"); + fileOp.compressionType(1); + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + int width = blockOp.getCorrectRowWidth(execplan::CalpontSystemCatalog::BIGINT, 8); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + const uint8_t* emptyVal = blockOp.getEmptyRowValue(execplan::CalpontSystemCatalog::BIGINT, 8); + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile(fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + execplan::CalpontSystemCatalog::BIGINT, + 1); // dbroot + CPPUNIT_ASSERT(rc == NO_ERROR); + + fileOp.closeFile(pFile); + + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT(rc == NO_ERROR); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + } + /* + void testCalculateRowIdBitmap() + { + BlockOp blockOp; + RID rowId = numeric_limitsmax(); + int fbo = 0, bio = 0, bbo = 0; + + // Assuming 1024 per data block, 8 byte width + CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 1024, 8, fbo, bio ) == false ); + + rowId = 1026; + CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 1024, 8, fbo, bio ) == true ); + CPPUNIT_ASSERT( fbo == 1 ); + CPPUNIT_ASSERT( bio == 16 ); + + // Assuming 2048 per data block, 4 byte width + // rowId = 2049; + // CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); + // CPPUNIT_ASSERT( fbo == 1 ); + // CPPUNIT_ASSERT( bio == 16 ); + // + // // Assuming 4096 per data block, 2 byte width + // rowId = 2049; + // CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); + // CPPUNIT_ASSERT( fbo == 1 ); + // CPPUNIT_ASSERT( bio == 16 ); + // + // // Assuming 8192 per data block, 1 byte width + // rowId = 2049; + // CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); + // CPPUNIT_ASSERT( fbo == 1 ); + // CPPUNIT_ASSERT( bio == 16 ); + // + rowId = 65546; + CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == true ); + CPPUNIT_ASSERT( fbo == 1 ); + CPPUNIT_ASSERT( bio == 1 ); + CPPUNIT_ASSERT( bbo == 2 ); + + // CPPUNIT_ASSERT( blockOp.getRowId( fbo, bio, bbo ) == 65546 ); + + rowId = numeric_limitsmax(); + CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == false ); + } + + + void testBlockBuffer() + { + BlockOp blockOp; + unsigned char buf[BYTE_PER_BLOCK]; + int writeVal = 80, readVal; + + blockOp.resetBuf( buf, sizeof( buf ) ); + CPPUNIT_ASSERT( buf[1] == '\0' ); + + // copy to byte 8 area + blockOp.writeBufValue( buf + 8, &writeVal, 4, true ); + + // blockOp.writeBufValue( buf + 8, &writeVal, 4 ); + blockOp.readBufValue( buf + 8, &readVal, 4 ); + CPPUNIT_ASSERT( readVal == 80 ); + } + + void testBitBasic() + { + BlockOp blockOp; + unsigned int val; + + val = 7; + CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 3 ); + val = 33; + CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 2 ); + val = 255; + CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 8 ); + val = 0; + CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 0 ); + + val = 3; + CPPUNIT_ASSERT( blockOp.getBitFlag( val, 0 ) == 1 ); + CPPUNIT_ASSERT( blockOp.getBitFlag( val, 1 ) == 1 ); + CPPUNIT_ASSERT( blockOp.getBitFlag( val, 2 ) == 0 ); + CPPUNIT_ASSERT( blockOp.getBitFlag( val, 3 ) == 0 ); + + blockOp.setBitFlag( &val, 2, true ); + CPPUNIT_ASSERT( blockOp.getBitFlag( val, 2 ) == true ); + CPPUNIT_ASSERT( val == 7 ); + + blockOp.setBitFlag( &val, 1, false ); + CPPUNIT_ASSERT( blockOp.getBitFlag( val, 1 ) == false ); + CPPUNIT_ASSERT( val == 5 ); + } + + void testBufferBit() + { + BlockOp blockOp; + unsigned char buf[BYTE_PER_BLOCK]; + + blockOp.resetBuf( buf, sizeof( buf ) ); + + blockOp.setBitFlag( (unsigned int*)&buf[1], 0, true ); + blockOp.setBitFlag( (unsigned int*)&buf[1], 3, true ); + + blockOp.setBitFlag( (unsigned int*)&buf[6], 1, true ); + blockOp.setBitFlag( (unsigned int*)&buf[6], 2, true ); + blockOp.setBitFlag( (unsigned int*)&buf[6], 3, true ); + blockOp.setBitFlag( (unsigned int*)&buf[6], 5, true ); + + CPPUNIT_ASSERT( blockOp.getBufTotalOnBit( buf, sizeof( buf )) == 6 ); + + CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 6, 5 ) == true ); + CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 6, 3 ) == true ); + CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 1, 0 ) == true ); + CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 1, 3 ) == true ); + + blockOp.setBufByteBitFlag( buf, 6, 2, false ); + CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 6, 2 ) == false ); + CPPUNIT_ASSERT( blockOp.getBufTotalOnBit( buf, sizeof( buf )) == 5 ); + } + + void testBitShift() + { + BlockOp blockOp; + uint64_t curVal, result, mask; + + curVal = 0x6a; + mask = 0xe0; // first three bit + result = blockOp.getBitValue( curVal, 5, 0xF ); + CPPUNIT_ASSERT( result == 3 ); + + result = blockOp.setBitValue( curVal, 0x4, 5, mask ); + CPPUNIT_ASSERT( result == 138 ); + } + + void testEmptyRowValue( ) + { + BlockOp blockOp; + uint64_t curVal; + RID curRID; + + curRID = blockOp.getRowId( 0, 1, 3 ); + CPPUNIT_ASSERT( curRID == 3 ); + + curRID = blockOp.getRowId( 0, 2, 3 ); + CPPUNIT_ASSERT( curRID == 3 ); + + curRID = blockOp.getRowId( 2, 4, 3 ); + CPPUNIT_ASSERT( curRID == 4099 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::TINYINT, 1 ); + CPPUNIT_ASSERT( curVal == 0x81 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::SMALLINT, 2 ); + CPPUNIT_ASSERT( curVal == 0x8001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::MEDINT, 4 ); + CPPUNIT_ASSERT( curVal == 0x80000001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::INT, 4 ); + CPPUNIT_ASSERT( curVal == 0x80000001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::BIGINT, 8 ); + CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::FLOAT, 4 ); + CPPUNIT_ASSERT( curVal == 0xFFAAAAAB ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DOUBLE, 8 ); + CPPUNIT_ASSERT( curVal == 0xFFFAAAAAAAAAAAABLL ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 1 ); + CPPUNIT_ASSERT( curVal == 0x8001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 2 ); + CPPUNIT_ASSERT( curVal == 0x8001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 3 ); + CPPUNIT_ASSERT( curVal == 0x80000001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 4 ); + CPPUNIT_ASSERT( curVal == 0x80000001 ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 5 ); + CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 6 ); + CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - void tearDown() - { - - } - - void test1() - { - int numOfBlock = 1024; - FILE* pFile; - unsigned char writeBuf[BYTE_PER_BLOCK * 10]; - char diskBuf[BYTE_PER_BLOCK * 11]; - char fileName[40]; - - for ( int k = 0; k < 273; k++ ) - { - snprintf( fileName, sizeof(fileName), "testwr%d.tst", k ); - pFile = fopen( fileName, "w+b" ); - - if ( pFile != NULL ) - { - setvbuf ( pFile, diskBuf, _IOFBF, 81920 ); - memset( writeBuf, 0, BYTE_PER_BLOCK * sizeof( unsigned char )); - - for ( int i = 0; i < numOfBlock; i++ ) - fwrite( writeBuf, sizeof( writeBuf ), 1, pFile ); - - // fwrite( writeBuf, sizeof( writeBuf ), numOfBlock, pFile ); - fclose( pFile ); - } - } - } -/* - void testFileNameOp() - { - FileOp fileOp; - bool bStatus; - char fileName[20]; - int rc; - - strcpy( fileName, "FILE_999.dat" ); - BRMWrapper::setUseBrm(true); - // file opertaions - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - rc = fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( rc == ERR_FILE_NOT_EXIST ); - - rc = fileOp.createFile( fileName, 10 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = fileOp.createFile( fileName, 10 ); - CPPUNIT_ASSERT( rc == ERR_FILE_EXIST ); - - bStatus = fileOp.exists( fileName ); - CPPUNIT_ASSERT( bStatus == true ); - - // init different value - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - rc = fileOp.createFile( fileName, 10, 0xFF, 1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - rc = fileOp.createFile( fileName, 10, 0x8001, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - rc = fileOp.createFile( fileName, 10, 0xFFAAAAAB, 4 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - //Not use block resolution manager - BRMWrapper::setUseBrm(false); - // file opertaions - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - rc = fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( rc == ERR_FILE_NOT_EXIST ); - - rc = fileOp.createFile( fileName, 10 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = fileOp.createFile( fileName, 10 ); - CPPUNIT_ASSERT( rc == ERR_FILE_EXIST ); - - bStatus = fileOp.exists( fileName ); - CPPUNIT_ASSERT( bStatus == true ); - - // init different value - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - rc = fileOp.createFile( fileName, 10, 0xFF, 1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - rc = fileOp.createFile( fileName, 10, 0x8001, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - rc = fileOp.createFile( fileName, 10, 0xFFAAAAAB, 4 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - } - - void testFileHandleOp() - { - FileOp fileOp; - char fileName[100], testStr[50]; - unsigned char dataBuf[BYTE_PER_BLOCK]; - long fileSize; - FILE* pFile; - int rc; - int allocSize = 0; - - BRMWrapper::setUseBrm(true); -// strcpy( fileName, "FILE_999.dat" ); - strcpy( testStr, "String write test data" ); - - fileOp.deleteFile( 999 ); - rc = fileOp.createFile( 999, 10, allocSize); - CPPUNIT_ASSERT( rc == NO_ERROR ); - pFile = fileOp.openFile( 999 ); - CPPUNIT_ASSERT( pFile != NULL ); - - fileSize = fileOp.getFileSize( 999 ); - CPPUNIT_ASSERT( fileSize == allocSize * BYTE_PER_BLOCK ); - - rc = fileOp.setFileOffset( pFile, -1, SEEK_SET ); - CPPUNIT_ASSERT( rc == ERR_FILE_SEEK ); - - uint64_t lbid1; - rc = BRMWrapper::getInstance()->getBrmInfo( 1999, 1, lbid1 ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - rc = BRMWrapper::getInstance()->getBrmInfo( 999, 1, lbid1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = fileOp.setFileOffset( pFile, lbid1 * BYTE_PER_BLOCK, SEEK_SET ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - uint64_t lbid9; - rc = BRMWrapper::getInstance()->getBrmInfo( 999, 9, lbid9 ); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = fileOp.setFileOffset( pFile, BYTE_PER_BLOCK * lbid9, SEEK_SET ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // write something to block 1 - memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); - memcpy( dataBuf + 8, testStr, strlen( testStr ) ); - - CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, lbid1 * BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); - rc = fileOp.writeFile( pFile, dataBuf, BYTE_PER_BLOCK ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = fileOp.writeFile( NULL, dataBuf, BYTE_PER_BLOCK ); - CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); - - // read something from block 1 - memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); - - rc = BRMWrapper::getInstance()->getBrmInfo( 999, 1, lbid1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, lbid1 * BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); - rc = fileOp.readFile( pFile, dataBuf, BYTE_PER_BLOCK ); - - printf( "\ndataBuf=%s\n", dataBuf + 8 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( memcmp( dataBuf + 8, testStr, strlen(testStr)) == 0 ); - - CPPUNIT_ASSERT( fileOp.getFileSize( 999 ) == allocSize * 8192 ); - fileOp.closeFile( pFile ); - fileOp.deleteFile(999); - - // No block resolution manager - BRMWrapper::setUseBrm(false); -// strcpy( fileName, "FILE_999.dat" ); - rc = fileOp.getFileName( 999, fileName ); - strcpy( testStr, "String write test data" ); - fileOp.deleteFile( 999 ); - CPPUNIT_ASSERT( fileOp.exists( 999 ) == false ); - - rc = fileOp.createFile( 999, 10, allocSize); - CPPUNIT_ASSERT( rc == NO_ERROR ); - pFile = fileOp.openFile( fileName ); - CPPUNIT_ASSERT( pFile != NULL ); - - fileSize = fileOp.getFileSize( 999 ); - CPPUNIT_ASSERT( fileSize == 81920 ); - - rc = fileOp.setFileOffset( pFile, -1, SEEK_SET ); - CPPUNIT_ASSERT( rc == ERR_FILE_SEEK ); - - rc = fileOp.setFileOffset( pFile, BYTE_PER_BLOCK, SEEK_SET ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = fileOp.setFileOffset( pFile, BYTE_PER_BLOCK * 9, SEEK_SET ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // write something to block 1 - memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); - memcpy( dataBuf + 8, testStr, strlen( testStr ) ); - - CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); - rc = fileOp.writeFile( pFile, dataBuf, BYTE_PER_BLOCK ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = fileOp.writeFile( NULL, dataBuf, BYTE_PER_BLOCK ); - CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); - - // read something from block 1 - memset( dataBuf, 0, sizeof( unsigned char ) * BYTE_PER_BLOCK ); - - CPPUNIT_ASSERT( fileOp.setFileOffset( pFile, BYTE_PER_BLOCK, SEEK_SET ) == NO_ERROR ); - rc = fileOp.readFile( pFile, dataBuf, BYTE_PER_BLOCK ); - - printf( "\ndataBuf=%s\n", dataBuf + 8 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( memcmp( dataBuf + 8, testStr, strlen(testStr)) == 0 ); - - CPPUNIT_ASSERT( fileOp.getFileSize( 999 ) == 81920 ); - fileOp.closeFile( pFile ); - fileOp.deleteFile(999); - - } - -*/ - void testDirBasic() - { - FileOp fileOp; - char dirName[30]; - int rc; - - printf("\nRunning testDirBasic \n"); - idbdatafile::IDBPolicy::init(true, false, "", 0); - IDBFileSystem& fs = IDBPolicy::getFs( "/tmp" ); - strcpy( dirName, "/tmp/testdir42" ); - fs.remove( dirName ); - CPPUNIT_ASSERT( fileOp.isDir( dirName ) == false ); - - rc = fileOp.createDir( dirName ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( fileOp.isDir( dirName ) == true ); - - fs.remove( dirName ); - } - - void testCreateDeleteFile() - { - IDBDataFile* pFile = NULL; - FileOp fileOp; - BlockOp blockOp; - char fileName[20]; - int rc; - char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ]; - - printf("\nRunning testCreateDeleteFile \n"); - idbdatafile::IDBPolicy::init(true, false, "", 0); - // Set to versionbuffer to satisfy IDBPolicy::getType - strcpy( fileName, "versionbuffer" ); - fileOp.compressionType(1); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 ); - int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; - const uint8_t* emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 ); - // createFile runs IDBDataFile::open + initAbrevCompColumnExtent - // under the hood - // bigint column file - rc = fileOp.createFile( fileName, - nBlocks, // number of blocks - emptyVal, // NULL value - width, // width - execplan::CalpontSystemCatalog::BIGINT, - 1 ); // dbroot - CPPUNIT_ASSERT( rc == NO_ERROR ); - - fileOp.closeFile(pFile); - - pFile = IDBDataFile::open(IDBPolicy::getType(fileName, - IDBPolicy::WRITEENG), fileName, "rb", 1); - - rc = pFile->seek(0, 0); - CPPUNIT_ASSERT(rc == NO_ERROR); - rc = fileOp.readHeaders(pFile, hdrs); - CPPUNIT_ASSERT( rc == NO_ERROR ); - // Couldn't use IDBDataFile->close() here w/o excplicit cast - fileOp.closeFile(pFile); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - } -/* - void testCalculateRowIdBitmap() - { - BlockOp blockOp; - RID rowId = numeric_limitsmax(); - int fbo = 0, bio = 0, bbo = 0; - - // Assuming 1024 per data block, 8 byte width - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 1024, 8, fbo, bio ) == false ); - - rowId = 1026; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 1024, 8, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - - // Assuming 2048 per data block, 4 byte width -// rowId = 2049; -// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); -// CPPUNIT_ASSERT( fbo == 1 ); -// CPPUNIT_ASSERT( bio == 16 ); -// -// // Assuming 4096 per data block, 2 byte width -// rowId = 2049; -// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); -// CPPUNIT_ASSERT( fbo == 1 ); -// CPPUNIT_ASSERT( bio == 16 ); -// -// // Assuming 8192 per data block, 1 byte width -// rowId = 2049; -// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); -// CPPUNIT_ASSERT( fbo == 1 ); -// CPPUNIT_ASSERT( bio == 16 ); -// - rowId = 65546; - CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 1 ); - CPPUNIT_ASSERT( bbo == 2 ); - -// CPPUNIT_ASSERT( blockOp.getRowId( fbo, bio, bbo ) == 65546 ); - - rowId = numeric_limitsmax(); - CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == false ); - } - - - void testBlockBuffer() - { - BlockOp blockOp; - unsigned char buf[BYTE_PER_BLOCK]; - int writeVal = 80, readVal; - - blockOp.resetBuf( buf, sizeof( buf ) ); - CPPUNIT_ASSERT( buf[1] == '\0' ); - - // copy to byte 8 area - blockOp.writeBufValue( buf + 8, &writeVal, 4, true ); - -// blockOp.writeBufValue( buf + 8, &writeVal, 4 ); - blockOp.readBufValue( buf + 8, &readVal, 4 ); - CPPUNIT_ASSERT( readVal == 80 ); - } - - void testBitBasic() - { - BlockOp blockOp; - unsigned int val; - - val = 7; - CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 3 ); - val = 33; - CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 2 ); - val = 255; - CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 8 ); - val = 0; - CPPUNIT_ASSERT( blockOp.getByteTotalOnBit( val, 8 ) == 0 ); - - val = 3; - CPPUNIT_ASSERT( blockOp.getBitFlag( val, 0 ) == 1 ); - CPPUNIT_ASSERT( blockOp.getBitFlag( val, 1 ) == 1 ); - CPPUNIT_ASSERT( blockOp.getBitFlag( val, 2 ) == 0 ); - CPPUNIT_ASSERT( blockOp.getBitFlag( val, 3 ) == 0 ); - - blockOp.setBitFlag( &val, 2, true ); - CPPUNIT_ASSERT( blockOp.getBitFlag( val, 2 ) == true ); - CPPUNIT_ASSERT( val == 7 ); - - blockOp.setBitFlag( &val, 1, false ); - CPPUNIT_ASSERT( blockOp.getBitFlag( val, 1 ) == false ); - CPPUNIT_ASSERT( val == 5 ); - } - - void testBufferBit() - { - BlockOp blockOp; - unsigned char buf[BYTE_PER_BLOCK]; - - blockOp.resetBuf( buf, sizeof( buf ) ); - - blockOp.setBitFlag( (unsigned int*)&buf[1], 0, true ); - blockOp.setBitFlag( (unsigned int*)&buf[1], 3, true ); - - blockOp.setBitFlag( (unsigned int*)&buf[6], 1, true ); - blockOp.setBitFlag( (unsigned int*)&buf[6], 2, true ); - blockOp.setBitFlag( (unsigned int*)&buf[6], 3, true ); - blockOp.setBitFlag( (unsigned int*)&buf[6], 5, true ); - - CPPUNIT_ASSERT( blockOp.getBufTotalOnBit( buf, sizeof( buf )) == 6 ); - - CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 6, 5 ) == true ); - CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 6, 3 ) == true ); - CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 1, 0 ) == true ); - CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 1, 3 ) == true ); - - blockOp.setBufByteBitFlag( buf, 6, 2, false ); - CPPUNIT_ASSERT( blockOp.getBufByteBitFlag( buf, 6, 2 ) == false ); - CPPUNIT_ASSERT( blockOp.getBufTotalOnBit( buf, sizeof( buf )) == 5 ); - } - - void testBitShift() - { - BlockOp blockOp; - uint64_t curVal, result, mask; - - curVal = 0x6a; - mask = 0xe0; // first three bit - result = blockOp.getBitValue( curVal, 5, 0xF ); - CPPUNIT_ASSERT( result == 3 ); - - result = blockOp.setBitValue( curVal, 0x4, 5, mask ); - CPPUNIT_ASSERT( result == 138 ); - } - - void testEmptyRowValue( ) - { - BlockOp blockOp; - uint64_t curVal; - RID curRID; - - curRID = blockOp.getRowId( 0, 1, 3 ); - CPPUNIT_ASSERT( curRID == 3 ); - - curRID = blockOp.getRowId( 0, 2, 3 ); - CPPUNIT_ASSERT( curRID == 3 ); - - curRID = blockOp.getRowId( 2, 4, 3 ); - CPPUNIT_ASSERT( curRID == 4099 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::TINYINT, 1 ); - CPPUNIT_ASSERT( curVal == 0x81 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::SMALLINT, 2 ); - CPPUNIT_ASSERT( curVal == 0x8001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::MEDINT, 4 ); - CPPUNIT_ASSERT( curVal == 0x80000001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::INT, 4 ); - CPPUNIT_ASSERT( curVal == 0x80000001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::BIGINT, 8 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::FLOAT, 4 ); - CPPUNIT_ASSERT( curVal == 0xFFAAAAAB ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DOUBLE, 8 ); - CPPUNIT_ASSERT( curVal == 0xFFFAAAAAAAAAAAABLL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 1 ); - CPPUNIT_ASSERT( curVal == 0x8001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 2 ); - CPPUNIT_ASSERT( curVal == 0x8001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 3 ); - CPPUNIT_ASSERT( curVal == 0x80000001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 4 ); - CPPUNIT_ASSERT( curVal == 0x80000001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 5 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 6 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 7 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 8 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - -// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); -// CPPUNIT_ASSERT( curVal == 0x80000001 ); -// -// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); -// CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); -// -// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); -// CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); -// -// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); -// CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); -// - curVal = blockOp.getEmptyRowValue( WriteEngine::DATE, 4 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DATETIME, 8 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 1 ); - CPPUNIT_ASSERT( curVal == 0xFF ); - curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 2 ); - CPPUNIT_ASSERT( curVal == 0xFFFF ); - curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 4 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); - curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 5 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 8 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 10 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 1 ); - CPPUNIT_ASSERT( curVal == 0xFFFF ); - curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 2 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); - curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 3 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); - curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 4 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 5 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 10 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - } - void testCorrectRowWidth() - { - BlockOp blockOp; - int curVal; - - curVal = blockOp.getCorrectRowWidth( WriteEngine::TINYINT, 1 ); - CPPUNIT_ASSERT( curVal == 1 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::SMALLINT, 1 ); // intentionally put 1 rather than 2 - CPPUNIT_ASSERT( curVal == 2 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::MEDINT, 8 ); // intentionally put 8 rather than 4 - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::INT, 4 ); - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::BIGINT, 4 ); // should be 8 - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::FLOAT, 8 ); // should be 4 - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DOUBLE, 4 ); // should be 8 - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 1 ); - CPPUNIT_ASSERT( curVal == 2 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 2 ); - CPPUNIT_ASSERT( curVal == 2 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 3 ); - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 4 ); - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 5 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 6 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 7 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 8 ); - CPPUNIT_ASSERT( curVal == 8 ); - -// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); -// CPPUNIT_ASSERT( curVal == 4 ); -// -// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); -// CPPUNIT_ASSERT( curVal == 8 ); -// -// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); -// CPPUNIT_ASSERT( curVal == 8 ); -// -// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); -// CPPUNIT_ASSERT( curVal == 8 ); -// - curVal = blockOp.getCorrectRowWidth( WriteEngine::DATE, 8 ); - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DATETIME, 4 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 1 ); - CPPUNIT_ASSERT( curVal == 1 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 2 ); - CPPUNIT_ASSERT( curVal == 2 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 3 ); - CPPUNIT_ASSERT( curVal == 4 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 5 ); - CPPUNIT_ASSERT( curVal == 8 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 8 ); - CPPUNIT_ASSERT( curVal == 8 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 10 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 1 ); - CPPUNIT_ASSERT( curVal == 2 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 2 ); - CPPUNIT_ASSERT( curVal == 4 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 3 ); - CPPUNIT_ASSERT( curVal == 4 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 4 ); - CPPUNIT_ASSERT( curVal == 8 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 5 ); - CPPUNIT_ASSERT( curVal == 8 ); - curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 10 ); - CPPUNIT_ASSERT( curVal == 8 ); - } - - - void testDbBlock() - { - DbFileOp dbFileOp; - DataBlock block; - uint64_t outVal = 12345, inVal; - int rc; - FILE* pFile; - int allocSize = 0; + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 7 ); + CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 8 ); + CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); + + // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); + // CPPUNIT_ASSERT( curVal == 0x80000001 ); + // + // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); + // CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); + // + // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); + // CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); + // + // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); + // CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + // + curVal = blockOp.getEmptyRowValue( WriteEngine::DATE, 4 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::DATETIME, 8 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 1 ); + CPPUNIT_ASSERT( curVal == 0xFF ); + curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 2 ); + CPPUNIT_ASSERT( curVal == 0xFFFF ); + curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 4 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); + curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 5 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 8 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + curVal = blockOp.getEmptyRowValue( WriteEngine::CHAR, 10 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + + curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 1 ); + CPPUNIT_ASSERT( curVal == 0xFFFF ); + curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 2 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); + curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 3 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); + curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 4 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 5 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + curVal = blockOp.getEmptyRowValue( WriteEngine::VARCHAR, 10 ); + CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); + } + void testCorrectRowWidth() + { + BlockOp blockOp; + int curVal; + + curVal = blockOp.getCorrectRowWidth( WriteEngine::TINYINT, 1 ); + CPPUNIT_ASSERT( curVal == 1 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::SMALLINT, 1 ); // intentionally put 1 rather than + 2 CPPUNIT_ASSERT( curVal == 2 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::MEDINT, 8 ); // intentionally put 8 rather than + 4 CPPUNIT_ASSERT( curVal == 4 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::INT, 4 ); + CPPUNIT_ASSERT( curVal == 4 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::BIGINT, 4 ); // should be 8 + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::FLOAT, 8 ); // should be 4 + CPPUNIT_ASSERT( curVal == 4 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DOUBLE, 4 ); // should be 8 + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 1 ); + CPPUNIT_ASSERT( curVal == 2 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 2 ); + CPPUNIT_ASSERT( curVal == 2 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 3 ); + CPPUNIT_ASSERT( curVal == 4 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 4 ); + CPPUNIT_ASSERT( curVal == 4 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 5 ); + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 6 ); + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 7 ); + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 8 ); + CPPUNIT_ASSERT( curVal == 8 ); + + // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); + // CPPUNIT_ASSERT( curVal == 4 ); + // + // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); + // CPPUNIT_ASSERT( curVal == 8 ); + // + // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); + // CPPUNIT_ASSERT( curVal == 8 ); + // + // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); + // CPPUNIT_ASSERT( curVal == 8 ); + // + curVal = blockOp.getCorrectRowWidth( WriteEngine::DATE, 8 ); + CPPUNIT_ASSERT( curVal == 4 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::DATETIME, 4 ); + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 1 ); + CPPUNIT_ASSERT( curVal == 1 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 2 ); + CPPUNIT_ASSERT( curVal == 2 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 3 ); + CPPUNIT_ASSERT( curVal == 4 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 5 ); + CPPUNIT_ASSERT( curVal == 8 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 8 ); + CPPUNIT_ASSERT( curVal == 8 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::CHAR, 10 ); + CPPUNIT_ASSERT( curVal == 8 ); + + curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 1 ); + CPPUNIT_ASSERT( curVal == 2 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 2 ); + CPPUNIT_ASSERT( curVal == 4 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 3 ); + CPPUNIT_ASSERT( curVal == 4 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 4 ); + CPPUNIT_ASSERT( curVal == 8 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 5 ); + CPPUNIT_ASSERT( curVal == 8 ); + curVal = blockOp.getCorrectRowWidth( WriteEngine::VARCHAR, 10 ); + CPPUNIT_ASSERT( curVal == 8 ); + } + + + void testDbBlock() + { + DbFileOp dbFileOp; + DataBlock block; + uint64_t outVal = 12345, inVal; + int rc; + FILE* pFile; + int allocSize = 0; - Stats::setUseStats( true ); - BRMWrapper::setUseBrm(true); - dbFileOp.deleteFile( 999 ); - CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); + Stats::setUseStats( true ); + BRMWrapper::setUseBrm(true); + dbFileOp.deleteFile( 999 ); + CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); - CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize ) == NO_ERROR ); + CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize ) == NO_ERROR ); - pFile = dbFileOp.openFile( 999 ); - CPPUNIT_ASSERT( pFile != NULL ); + pFile = dbFileOp.openFile( 999 ); + CPPUNIT_ASSERT( pFile != NULL ); - uint64_t lbid0; - BRMWrapper::getInstance()->getBrmInfo( 999, 0, lbid0 ); - rc = dbFileOp.readDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + uint64_t lbid0; + BRMWrapper::getInstance()->getBrmInfo( 999, 0, lbid0 ); + rc = dbFileOp.readDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - dbFileOp.setSubBlockEntry( block.data, 1, 2, 8, &outVal ); - - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.readSubBlockEntry( pFile, &block, lbid0, 1, 2, 8, &inVal ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( inVal == outVal ); - - inVal = 0; - dbFileOp.getSubBlockEntry( block.data, 1, 2, 8, &inVal ); - CPPUNIT_ASSERT( inVal == outVal ); - - rc = dbFileOp.writeSubBlockEntry( pFile, &block, lbid0, 1, 2, 8, &inVal ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.readDBFile( pFile, &block, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.writeDBFile( pFile, &block, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // Not Use Block resolution manager - BRMWrapper::setUseBrm(false); - dbFileOp.deleteFile( 999 ); - CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); - - CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize ) == NO_ERROR ); - - pFile = dbFileOp.openFile( 999 ); - CPPUNIT_ASSERT( pFile != NULL ); - - rc = dbFileOp.readDBFile( pFile, block.data, 0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - dbFileOp.setSubBlockEntry( block.data, 1, 2, 8, &outVal ); - - rc = dbFileOp.writeDBFile( pFile, block.data, 0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.readSubBlockEntry( pFile, &block, 0, 1, 2, 8, &inVal ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( inVal == outVal ); - - inVal = 0; - dbFileOp.getSubBlockEntry( block.data, 1, 2, 8, &inVal ); - CPPUNIT_ASSERT( inVal == outVal ); - - rc = dbFileOp.writeSubBlockEntry( pFile, &block, (uint64_t)0, 1, 2, 8, &inVal ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.readDBFile( pFile, &block, 0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + dbFileOp.setSubBlockEntry( block.data, 1, 2, 8, &outVal ); + + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.readSubBlockEntry( pFile, &block, lbid0, 1, 2, 8, &inVal ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + CPPUNIT_ASSERT( inVal == outVal ); + + inVal = 0; + dbFileOp.getSubBlockEntry( block.data, 1, 2, 8, &inVal ); + CPPUNIT_ASSERT( inVal == outVal ); + + rc = dbFileOp.writeSubBlockEntry( pFile, &block, lbid0, 1, 2, 8, &inVal ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.readDBFile( pFile, &block, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.writeDBFile( pFile, &block, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // Not Use Block resolution manager + BRMWrapper::setUseBrm(false); + dbFileOp.deleteFile( 999 ); + CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); + + CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize ) == NO_ERROR ); + + pFile = dbFileOp.openFile( 999 ); + CPPUNIT_ASSERT( pFile != NULL ); + + rc = dbFileOp.readDBFile( pFile, block.data, 0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + dbFileOp.setSubBlockEntry( block.data, 1, 2, 8, &outVal ); + + rc = dbFileOp.writeDBFile( pFile, block.data, 0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.readSubBlockEntry( pFile, &block, 0, 1, 2, 8, &inVal ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + CPPUNIT_ASSERT( inVal == outVal ); + + inVal = 0; + dbFileOp.getSubBlockEntry( block.data, 1, 2, 8, &inVal ); + CPPUNIT_ASSERT( inVal == outVal ); + + rc = dbFileOp.writeSubBlockEntry( pFile, &block, (uint64_t)0, 1, 2, 8, &inVal ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.readDBFile( pFile, &block, 0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = dbFileOp.writeDBFile( pFile, &block, 0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = dbFileOp.writeDBFile( pFile, &block, 0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - long readCounter, writeCounter; - readCounter = Stats::getIoBlockRead(); - writeCounter = Stats::getIoBlockWrite(); - printf( "\nTotal number of blocks read: %ld", readCounter ); - printf( "\nTotal number of blocks write: %ld", writeCounter ); - Stats::setUseStats( false ); - } + long readCounter, writeCounter; + readCounter = Stats::getIoBlockRead(); + writeCounter = Stats::getIoBlockWrite(); + printf( "\nTotal number of blocks read: %ld", readCounter ); + printf( "\nTotal number of blocks write: %ld", writeCounter ); + Stats::setUseStats( false ); + } - void testCopyDbFile() - { - DbFileOp dbFileOp; - int rc, startFbo = 5, copyBlock = 15, bufBlock = 30, extendBlock; - FILE* pSourceFile = NULL; - FILE* pTargetFile = NULL; - int allocSize999 = 0, allocSize998 = 0; + void testCopyDbFile() + { + DbFileOp dbFileOp; + int rc, startFbo = 5, copyBlock = 15, bufBlock = 30, extendBlock; + FILE* pSourceFile = NULL; + FILE* pTargetFile = NULL; + int allocSize999 = 0, allocSize998 = 0; - BRMWrapper::setUseBrm(true); + BRMWrapper::setUseBrm(true); - rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, - copyBlock, bufBlock, extendBlock ); - CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); + rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + copyBlock, bufBlock, extendBlock ); + CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); - dbFileOp.deleteFile( 999 ); - dbFileOp.deleteFile( 998 ); - CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); - CPPUNIT_ASSERT( dbFileOp.exists( 998 ) == false ); + dbFileOp.deleteFile( 999 ); + dbFileOp.deleteFile( 998 ); + CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); + CPPUNIT_ASSERT( dbFileOp.exists( 998 ) == false ); - CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize999 ) == NO_ERROR ); - CPPUNIT_ASSERT( dbFileOp.createFile( 998, 20, allocSize998, 0xFFFF, 2 ) == NO_ERROR ); + CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize999 ) == NO_ERROR ); + CPPUNIT_ASSERT( dbFileOp.createFile( 998, 20, allocSize998, 0xFFFF, 2 ) == NO_ERROR ); - pTargetFile = dbFileOp.openFile( 999 ); - CPPUNIT_ASSERT( pTargetFile != NULL ); + pTargetFile = dbFileOp.openFile( 999 ); + CPPUNIT_ASSERT( pTargetFile != NULL ); - pSourceFile = dbFileOp.openFile( 998 ); - CPPUNIT_ASSERT( pSourceFile != NULL ); + pSourceFile = dbFileOp.openFile( 998 ); + CPPUNIT_ASSERT( pSourceFile != NULL ); -// rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, -// copyBlock, bufBlock, extendBlock ); -// CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); - //CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); + // rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + // copyBlock, bufBlock, extendBlock ); + // CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); + //CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); - //copyBlock = 20; - copyBlock = allocSize998; - bufBlock = allocSize999; + //copyBlock = 20; + copyBlock = allocSize998; + bufBlock = allocSize999; - rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, - copyBlock, bufBlock, extendBlock ); - CPPUNIT_ASSERT( rc == ERR_FILE_NEED_EXTEND ); - //CPPUNIT_ASSERT( extendBlock == 10245 ); - - int allocExtSize999 = 0; - rc = dbFileOp.extendFile( pTargetFile, 999, extendBlock, allocExtSize999, 0, 8 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, - copyBlock, bufBlock, extendBlock ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( extendBlock == 0 ); - - uint64_t lbidTarget; - BRMWrapper::getInstance()->getBrmInfo( 999, startFbo, lbidTarget ); - rc = dbFileOp.copyDBFile( pSourceFile, pTargetFile, lbidTarget ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - dbFileOp.closeFile( pSourceFile ); - dbFileOp.closeFile( pTargetFile ); - - //Not Use BRM - BRMWrapper::setUseBrm(false); - pSourceFile = NULL; - pTargetFile = NULL; - rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, - copyBlock, bufBlock, extendBlock ); - CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); - - dbFileOp.deleteFile( 999 ); - dbFileOp.deleteFile( 998 ); - CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); - CPPUNIT_ASSERT( dbFileOp.exists( 998 ) == false ); - - CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize999 ) == NO_ERROR ); - CPPUNIT_ASSERT( dbFileOp.createFile( 998, 20, allocSize998, 0xFFFF, 2 ) == NO_ERROR ); - - pTargetFile = dbFileOp.openFile( 999 ); - CPPUNIT_ASSERT( pTargetFile != NULL ); - - pSourceFile = dbFileOp.openFile( 998 ); - CPPUNIT_ASSERT( pSourceFile != NULL ); - -// rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, -// copyBlock, bufBlock, extendBlock ); -// CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); - - //CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); - //CPPUNIT_ASSERT( rc == NO_ERROR); - - copyBlock = 20; - bufBlock = 30; - rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, - copyBlock, bufBlock, extendBlock ); - CPPUNIT_ASSERT( rc == ERR_FILE_NEED_EXTEND ); - CPPUNIT_ASSERT( extendBlock == 45 ); - - rc = dbFileOp.extendFile( pTargetFile, 999, extendBlock, 0, 8 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, - copyBlock, bufBlock, extendBlock ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( extendBlock == 0 ); - - rc = dbFileOp.copyDBFile( pSourceFile, pTargetFile, startFbo ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - dbFileOp.closeFile( pSourceFile ); - dbFileOp.closeFile( pTargetFile ); - - } -*/ - - void testExtensionWOPrealloc() - { - IDBDataFile* pFile = NULL; - FileOp fileOp; - BlockOp blockOp; - char fileName[20]; - int rc; - char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ]; - int dbRoot = 1; - - printf("\nRunning testExtensionWOPrealloc \n"); - idbdatafile::IDBPolicy::init(true, false, "", 0); - // Set to versionbuffer to satisfy IDBPolicy::getType - strcpy( fileName, "versionbuffer" ); - fileOp.compressionType(1); - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - - int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 ); - int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; - const uint8_t* emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 ); - // createFile runs IDBDataFile::open + initAbrevCompColumnExtent - // under the hood - // bigint column file - rc = fileOp.createFile( fileName, - nBlocks, // number of blocks - emptyVal, // NULL value - width, // width - execplan::CalpontSystemCatalog::BIGINT, - dbRoot ); // dbroot - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // open created compressed file and check its header - pFile = IDBDataFile::open(IDBPolicy::getType(fileName, - IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - - rc = pFile->seek(0, 0); - CPPUNIT_ASSERT(rc == NO_ERROR); - rc = fileOp.readHeaders(pFile, hdrs); - CPPUNIT_ASSERT( rc == NO_ERROR ); - // Couldn't use IDBDataFile->close() here w/o excplicit cast - fileOp.closeFile(pFile); - - // Extend the extent up to 64MB - // first run w preallocation - idbdatafile::BufferedFile* bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); - pFile = dynamic_cast(bFile); - - rc = fileOp.initColumnExtent(pFile, - dbRoot, - BYTE_PER_BLOCK, // number of blocks - emptyVal, - width, - execplan::CalpontSystemCatalog::BIGINT, - false, // use existing file - true, // expand the extent - false, // add full (not abbreviated) extent - false); // don't optimize extention - - - CPPUNIT_ASSERT(rc == NO_ERROR); - CPPUNIT_ASSERT(bFile->size() == 2105344); - fileOp.closeFile(pFile); - // file has been extended delete the file before - // the second run - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); - - // second run with disabled preallocation - rc = fileOp.createFile( fileName, - nBlocks, // number of blocks - emptyVal, // NULL value - width, // width - execplan::CalpontSystemCatalog::BIGINT, - dbRoot ); // dbroot - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // open created compressed file and check its header - pFile = IDBDataFile::open(IDBPolicy::getType(fileName, - IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - - rc = pFile->seek(0, 0); - CPPUNIT_ASSERT(rc == NO_ERROR); - rc = fileOp.readHeaders(pFile, hdrs); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fileOp.closeFile(pFile); - - bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); - pFile = dynamic_cast(bFile); - - // enable disk space preallocation and extend - idbdatafile::IDBPolicy::enablePreallocSpace(dbRoot); - rc = fileOp.initColumnExtent(pFile, - dbRoot, - BYTE_PER_BLOCK, // number of blocks - emptyVal, - width, - execplan::CalpontSystemCatalog::BIGINT, - false, // use existing file - true, // expand the extent - false, // add full (not abbreviated) extent - true); // optimize extention - - - CPPUNIT_ASSERT(rc == NO_ERROR); - CPPUNIT_ASSERT(bFile->size() == 67108864); - fileOp.closeFile(pFile); - // file has been extended - - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); - } - - // Create a dict file. Extend it w and w/o preallocation. - // Check the file sizes. - void testDictExtensionWOPrealloc() - { - FileOp fileOp; - BlockOp blockOp; - char fileName[20]; - int rc; - int dbRoot = 1; - int colWidth = 65535; - - DctnryCompress1 m_Dctnry(/*compressionType=*/1); - // This is the magic for the stub in FileOp::oid2FileName - int oId = 42; - - printf("\nRunning testDictExtensionWOPrealloc "); - printf("There could be InetStreamSocket::connect errors \n"); - m_Dctnry.setDebugLevel( DEBUG_3 ); - - idbdatafile::IDBPolicy::init(true, false, "", 0); - // Set to versionbuffer to satisfy IDBPolicy::getType - strcpy( fileName, "versionbuffer" ); - - rc = m_Dctnry.dropDctnry(oId); - // FileOp::oid2FileName is called under the hood - // Dctnry::createDctnry could be used with running CS - // createDctnryFile also uses DBRM under the hood it works though. - IDBDataFile* m_dFile = m_Dctnry.createDctnryFileUnit(fileName, - colWidth, - "w+b", - DEFAULT_BUFSIZ); - - idbdatafile::BufferedFile* bFile = (idbdatafile::BufferedFile*)m_dFile; - CPPUNIT_ASSERT(m_dFile != NULL); - - const int m_totalHdrBytes = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE; - - m_Dctnry.compressionType(1); - rc = m_Dctnry.initDctnryExtent( m_dFile, - dbRoot, - BYTE_PER_BLOCK, // 8192 - const_cast(m_Dctnry.getDctnryHeader2()), - m_totalHdrBytes, - false, - false ); //enable preallocation - // Check the file size and remove the file - CPPUNIT_ASSERT(bFile->size() == 483328); - CPPUNIT_ASSERT(rc == NO_ERROR); - fileOp.deleteFile( fileName ); - CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); - - // Create a Dictionary for the second time - m_dFile = m_Dctnry.createDctnryFileUnit(fileName, - colWidth, - "w+b", - DEFAULT_BUFSIZ); - - // Get the file size later - bFile = (idbdatafile::BufferedFile*)m_dFile; - CPPUNIT_ASSERT(m_dFile != NULL); - - // enable preallocation and create a Dictionary - idbdatafile::IDBPolicy::enablePreallocSpace(dbRoot); - m_Dctnry.compressionType(1); - rc = m_Dctnry.initDctnryExtent( m_dFile, - dbRoot, - BYTE_PER_BLOCK, - const_cast(m_Dctnry.getDctnryHeader2()), - m_totalHdrBytes, - false, - true ); //skip preallocation - - // Check the size and remove the file. - CPPUNIT_ASSERT(bFile->size() == 67379200); - CPPUNIT_ASSERT(rc == NO_ERROR); - fileOp.deleteFile(fileName); - CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); - } - - - - - - void testLog() - { - Log log; - FileOp fileOp; - string msg; - int iVal = 3; - char logFile[] = "test1.log"; - char logErrFile[] = "test1err.log"; - - log.setLogFileName( logFile, logErrFile ); - - msg = Convertor::int2Str( iVal ); - log.logMsg( msg + " this is a info message", MSGLVL_INFO1 ); - msg = Convertor::getTimeStr(); - log.logMsg( " this is a warning message", MSGLVL_WARNING ); - log.logMsg( "this is an error message ", 1011, MSGLVL_ERROR ); - log.logMsg( "this is a critical message", 1211, MSGLVL_CRITICAL ); - CPPUNIT_ASSERT( fileOp.exists( logFile ) == true ); - CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == true ); - fileOp.deleteFile( logFile ); - fileOp.deleteFile( logErrFile ); - CPPUNIT_ASSERT( fileOp.exists( logFile ) == false ); - CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == false ); - } - -/* - - void testHWM() - { - int rc ; - int hwm = 0; - int allocSize = 0, allocSize1 = 0; - OID oid = 999; - DbFileOp dbFileOp; - VER_t transId = 11; - - BRMWrapper::setUseBrm(true); - BRMWrapper::getInstance()->rollBack( transId ); - - // create something to increase LBID number - dbFileOp.deleteFile( oid - 1 ); - rc = dbFileOp.createFile( oid - 1, 10, allocSize1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - dbFileOp.deleteFile( oid ); - CPPUNIT_ASSERT( dbFileOp.exists( oid ) == false ); - - CPPUNIT_ASSERT( dbFileOp.createFile( oid, 10, allocSize1 ) == NO_ERROR ); - - rc = BRMWrapper::getInstance()->getHWM( oid, hwm); - CPPUNIT_ASSERT( rc == NO_ERROR ); - hwm = 10; - rc = BRMWrapper::getInstance()->setHWM( oid, hwm ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = BRMWrapper::getInstance()->getHWM( oid, hwm); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( hwm == 10 ); - - rc = BRMWrapper::getInstance()->allocateExtent( oid, -1, allocSize); - CPPUNIT_ASSERT( rc != NO_ERROR ); - rc = BRMWrapper::getInstance()->allocateExtent( oid, 20, allocSize); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocSize > 0 ); - } - - void testVB() - { - FileOp fileOp; - FILE* pFile; - OID oid = 999; - LBIDRange range; - vector rangeList; - vector fboList; - uint64_t curLbid; - int rc; - VER_t transId = 11; - - BRMWrapper::getInstance()->setDebugLevel( DEBUG_3 ); - - // change fbo 3 - rc = BRMWrapper::getInstance()->getBrmInfo( oid, 3, curLbid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( curLbid >= 0 ); - - range.start = 0; - range.size = 1; - - rangeList.push_back( range ); - printf( "\ncur lbid=%ld", (long)curLbid ); - - pFile = fileOp.openFile( oid ); - CPPUNIT_ASSERT( pFile != NULL ); - - fboList.push_back( 3 ); - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // commit - rc = BRMWrapper::getInstance()->commit( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - - // write to the same block - transId++; - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // commit - rc = BRMWrapper::getInstance()->commit( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // the same block should be locked by BRM - transId++; - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - transId++; - // rollback - fboList[0] += 1; - rangeList[0].start += 1; - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = BRMWrapper::getInstance()->commit( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - transId++; - // write lbid - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, curLbid + 5); - CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + copyBlock, bufBlock, extendBlock ); + CPPUNIT_ASSERT( rc == ERR_FILE_NEED_EXTEND ); + //CPPUNIT_ASSERT( extendBlock == 10245 ); + + int allocExtSize999 = 0; + rc = dbFileOp.extendFile( pTargetFile, 999, extendBlock, allocExtSize999, 0, 8 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + copyBlock, bufBlock, extendBlock ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( extendBlock == 0 ); + + uint64_t lbidTarget; + BRMWrapper::getInstance()->getBrmInfo( 999, startFbo, lbidTarget ); + rc = dbFileOp.copyDBFile( pSourceFile, pTargetFile, lbidTarget ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + dbFileOp.closeFile( pSourceFile ); + dbFileOp.closeFile( pTargetFile ); + + //Not Use BRM + BRMWrapper::setUseBrm(false); + pSourceFile = NULL; + pTargetFile = NULL; + rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + copyBlock, bufBlock, extendBlock ); + CPPUNIT_ASSERT( rc == ERR_FILE_NULL ); + + dbFileOp.deleteFile( 999 ); + dbFileOp.deleteFile( 998 ); + CPPUNIT_ASSERT( dbFileOp.exists( 999 ) == false ); + CPPUNIT_ASSERT( dbFileOp.exists( 998 ) == false ); + + CPPUNIT_ASSERT( dbFileOp.createFile( 999, 10, allocSize999 ) == NO_ERROR ); + CPPUNIT_ASSERT( dbFileOp.createFile( 998, 20, allocSize998, 0xFFFF, 2 ) == NO_ERROR ); + + pTargetFile = dbFileOp.openFile( 999 ); + CPPUNIT_ASSERT( pTargetFile != NULL ); + + pSourceFile = dbFileOp.openFile( 998 ); + CPPUNIT_ASSERT( pSourceFile != NULL ); + + // rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + // copyBlock, bufBlock, extendBlock ); + // CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); + + //CPPUNIT_ASSERT( rc == ERR_SIZE_NOT_MATCH ); + //CPPUNIT_ASSERT( rc == NO_ERROR); + + copyBlock = 20; + bufBlock = 30; + rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + copyBlock, bufBlock, extendBlock ); + CPPUNIT_ASSERT( rc == ERR_FILE_NEED_EXTEND ); + CPPUNIT_ASSERT( extendBlock == 45 ); + + rc = dbFileOp.extendFile( pTargetFile, 999, extendBlock, 0, 8 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = dbFileOp.copyFileValidation( pSourceFile, pTargetFile, startFbo, + copyBlock, bufBlock, extendBlock ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( extendBlock == 0 ); + + rc = dbFileOp.copyDBFile( pSourceFile, pTargetFile, startFbo ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + dbFileOp.closeFile( pSourceFile ); + dbFileOp.closeFile( pTargetFile ); + + } + */ + + void testExtensionWOPrealloc() + { + IDBDataFile* pFile = NULL; + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + int dbRoot = 1; + + printf("\nRunning testExtensionWOPrealloc \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy(fileName, "versionbuffer"); + fileOp.compressionType(1); + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + int width = blockOp.getCorrectRowWidth(execplan::CalpontSystemCatalog::BIGINT, 8); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + const uint8_t* emptyVal = blockOp.getEmptyRowValue(execplan::CalpontSystemCatalog::BIGINT, 8); + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile(fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + execplan::CalpontSystemCatalog::BIGINT, + dbRoot); // dbroot + CPPUNIT_ASSERT(rc == NO_ERROR); + + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT(rc == NO_ERROR); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + // Extend the extent up to 64MB + // first run w preallocation + idbdatafile::BufferedFile* bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); + pFile = dynamic_cast(bFile); + + rc = fileOp.initColumnExtent(pFile, dbRoot, + BYTE_PER_BLOCK, // number of blocks + emptyVal, width, execplan::CalpontSystemCatalog::BIGINT, + false, // use existing file + true, // expand the extent + false, // add full (not abbreviated) extent + false); // don't optimize extention + + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(bFile->size() == 2105344); + fileOp.closeFile(pFile); + // file has been extended delete the file before + // the second run + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + // second run with disabled preallocation + rc = fileOp.createFile(fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + execplan::CalpontSystemCatalog::BIGINT, + dbRoot); // dbroot + CPPUNIT_ASSERT(rc == NO_ERROR); + + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.closeFile(pFile); + + bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); + pFile = dynamic_cast(bFile); + + // enable disk space preallocation and extend + idbdatafile::IDBPolicy::enablePreallocSpace(dbRoot); + rc = fileOp.initColumnExtent(pFile, dbRoot, + BYTE_PER_BLOCK, // number of blocks + emptyVal, width, execplan::CalpontSystemCatalog::BIGINT, + false, // use existing file + true, // expand the extent + false, // add full (not abbreviated) extent + true); // optimize extention + + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(bFile->size() == 67108864); + fileOp.closeFile(pFile); + // file has been extended + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + } + + // Create a dict file. Extend it w and w/o preallocation. + // Check the file sizes. + void testDictExtensionWOPrealloc() + { + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + int dbRoot = 1; + int colWidth = 65535; + + DctnryCompress1 m_Dctnry(/*compressionType=*/1); + // This is the magic for the stub in FileOp::oid2FileName + int oId = 42; + + printf("\nRunning testDictExtensionWOPrealloc "); + printf("There could be InetStreamSocket::connect errors \n"); + m_Dctnry.setDebugLevel(DEBUG_3); + + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy(fileName, "versionbuffer"); + + rc = m_Dctnry.dropDctnry(oId); + // FileOp::oid2FileName is called under the hood + // Dctnry::createDctnry could be used with running CS + // createDctnryFile also uses DBRM under the hood it works though. + IDBDataFile* m_dFile = m_Dctnry.createDctnryFileUnit(fileName, colWidth, "w+b", DEFAULT_BUFSIZ); + + idbdatafile::BufferedFile* bFile = (idbdatafile::BufferedFile*)m_dFile; + CPPUNIT_ASSERT(m_dFile != NULL); + + const int m_totalHdrBytes = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE; + + m_Dctnry.compressionType(1); + rc = m_Dctnry.initDctnryExtent(m_dFile, dbRoot, + BYTE_PER_BLOCK, // 8192 + const_cast(m_Dctnry.getDctnryHeader2()), m_totalHdrBytes, + false, + false); // enable preallocation + // Check the file size and remove the file + CPPUNIT_ASSERT(bFile->size() == 483328); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + // Create a Dictionary for the second time + m_dFile = m_Dctnry.createDctnryFileUnit(fileName, colWidth, "w+b", DEFAULT_BUFSIZ); + + // Get the file size later + bFile = (idbdatafile::BufferedFile*)m_dFile; + CPPUNIT_ASSERT(m_dFile != NULL); + + // enable preallocation and create a Dictionary + idbdatafile::IDBPolicy::enablePreallocSpace(dbRoot); + m_Dctnry.compressionType(1); + rc = m_Dctnry.initDctnryExtent(m_dFile, dbRoot, BYTE_PER_BLOCK, + const_cast(m_Dctnry.getDctnryHeader2()), m_totalHdrBytes, + false, + true); // skip preallocation + + // Check the size and remove the file. + CPPUNIT_ASSERT(bFile->size() == 67379200); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + } + + void testLog() + { + Log log; + FileOp fileOp; + string msg; + int iVal = 3; + char logFile[] = "test1.log"; + char logErrFile[] = "test1err.log"; + + log.setLogFileName(logFile, logErrFile); + + msg = Convertor::int2Str(iVal); + log.logMsg(msg + " this is a info message", MSGLVL_INFO1); + msg = Convertor::getTimeStr(); + log.logMsg(" this is a warning message", MSGLVL_WARNING); + log.logMsg("this is an error message ", 1011, MSGLVL_ERROR); + log.logMsg("this is a critical message", 1211, MSGLVL_CRITICAL); + CPPUNIT_ASSERT(fileOp.exists(logFile) == true); + CPPUNIT_ASSERT(fileOp.exists(logErrFile) == true); + fileOp.deleteFile(logFile); + fileOp.deleteFile(logErrFile); + CPPUNIT_ASSERT(fileOp.exists(logFile) == false); + CPPUNIT_ASSERT(fileOp.exists(logErrFile) == false); + } + + /* + + void testHWM() + { + int rc ; + int hwm = 0; + int allocSize = 0, allocSize1 = 0; + OID oid = 999; + DbFileOp dbFileOp; + VER_t transId = 11; + + BRMWrapper::setUseBrm(true); + BRMWrapper::getInstance()->rollBack( transId ); + + // create something to increase LBID number + dbFileOp.deleteFile( oid - 1 ); + rc = dbFileOp.createFile( oid - 1, 10, allocSize1 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + dbFileOp.deleteFile( oid ); + CPPUNIT_ASSERT( dbFileOp.exists( oid ) == false ); + + CPPUNIT_ASSERT( dbFileOp.createFile( oid, 10, allocSize1 ) == NO_ERROR ); + + rc = BRMWrapper::getInstance()->getHWM( oid, hwm); + CPPUNIT_ASSERT( rc == NO_ERROR ); + hwm = 10; + rc = BRMWrapper::getInstance()->setHWM( oid, hwm ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = BRMWrapper::getInstance()->getHWM( oid, hwm); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( hwm == 10 ); + + rc = BRMWrapper::getInstance()->allocateExtent( oid, -1, allocSize); + CPPUNIT_ASSERT( rc != NO_ERROR ); + rc = BRMWrapper::getInstance()->allocateExtent( oid, 20, allocSize); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( allocSize > 0 ); + } + + void testVB() + { + FileOp fileOp; + FILE* pFile; + OID oid = 999; + LBIDRange range; + vector rangeList; + vector fboList; + uint64_t curLbid; + int rc; + VER_t transId = 11; + + BRMWrapper::getInstance()->setDebugLevel( DEBUG_3 ); + + // change fbo 3 + rc = BRMWrapper::getInstance()->getBrmInfo( oid, 3, curLbid ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( curLbid >= 0 ); + + range.start = 0; + range.size = 1; + + rangeList.push_back( range ); + printf( "\ncur lbid=%ld", (long)curLbid ); + + pFile = fileOp.openFile( oid ); + CPPUNIT_ASSERT( pFile != NULL ); + + fboList.push_back( 3 ); + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // commit + rc = BRMWrapper::getInstance()->commit( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + + // write to the same block + transId++; + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // commit + rc = BRMWrapper::getInstance()->commit( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // the same block should be locked by BRM + transId++; + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + transId++; + // rollback + fboList[0] += 1; + rangeList[0].start += 1; + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList, rangeList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = BRMWrapper::getInstance()->commit( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + transId++; + // write lbid + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, curLbid + 5); + CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, curLbid + 6); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = BRMWrapper::getInstance()->commit( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - fileOp.closeFile( pFile ); + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, curLbid + 6); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = BRMWrapper::getInstance()->commit( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.closeFile( pFile ); - // test rollback whether can write back the original value after three insert - printf( "\n\nThis is the test for rollback after three insert for the same trans\n" ); - DbFileOp dbFileOp; - DataBlock block, originBlock; - uint64_t lbid0; - - pFile = dbFileOp.openFile( oid ); - CPPUNIT_ASSERT( pFile != NULL ); - - BRMWrapper::getInstance()->getBrmInfo( oid, 10, lbid0 ); - memset( block.data, 'a', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // test rollback whether can write back the original value after three insert + printf( "\n\nThis is the test for rollback after three insert for the same trans\n" ); + DbFileOp dbFileOp; + DataBlock block, originBlock; + uint64_t lbid0; + + pFile = dbFileOp.openFile( oid ); + CPPUNIT_ASSERT( pFile != NULL ); + + BRMWrapper::getInstance()->getBrmInfo( oid, 10, lbid0 ); + memset( block.data, 'a', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = dbFileOp.readDBFile( pFile, originBlock.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = dbFileOp.readDBFile( pFile, originBlock.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - transId++; - fboList.clear(); - fboList.push_back( 10 ); + transId++; + fboList.clear(); + fboList.push_back( 10 ); - // first time - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // first time + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - memset( block.data, 'b', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + memset( block.data, 'b', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - // second time - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // second time + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - memset( block.data, 'c', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + memset( block.data, 'c', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - // third time - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // third time + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - memset( block.data, 'd', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + memset( block.data, 'd', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - fflush(pFile); - rc = BRMWrapper::getInstance()->rollBack( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + fflush(pFile); + rc = BRMWrapper::getInstance()->rollBack( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); - fflush(pFile); - rc = dbFileOp.readDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( memcmp( block.data, originBlock.data, BYTE_PER_BLOCK ) == 0 ); - - // test commit twice and rollback to see whether rollback commit values - printf( "\n\nThis is the test for commit twice and then rollback to see whether the previous two committed one got rollback\n" ); - - // first commit - transId++; - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - memset( block.data, 'e', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = BRMWrapper::getInstance()->commit( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // second commit - transId++; - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - memset( block.data, 'f', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fflush(pFile); - - rc = BRMWrapper::getInstance()->commit( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - memcpy( originBlock.data, block.data, BYTE_PER_BLOCK ); - - // rollback - transId++; - rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - memset( block.data, 'g', BYTE_PER_BLOCK ); - rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fflush(pFile); - - rc = BRMWrapper::getInstance()->rollBack( transId ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // check whether they are the same one - rc = dbFileOp.readDBFile( pFile, block.data, lbid0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fflush(pFile); - - CPPUNIT_ASSERT( memcmp( block.data, originBlock.data, BYTE_PER_BLOCK ) == 0 ); - } - - void testDM() - { - FileOp fileOp; - char fileName[FILE_NAME_SIZE]; - int rc; - - rc = fileOp.oid2FileName( 100, fileName, true ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = fileOp.oid2FileName( 3071, fileName, true ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - - } - - void testCacheBasic() - { - DataBlock block; - OID oid = 15; - uint64_t lbid = 1234, fbo = 3; - int rc; - CacheKey key; - CommBlock cb; - - Cache::init(); - cb.file.oid = oid; - memset( block.data, 'a', BYTE_PER_BLOCK ); - rc = Cache::insertLRUList( cb, lbid, fbo, block ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( Cache::cacheKeyExist( oid, lbid ) == true ); - - // add a second block - block.data[1] = '1'; - lbid = 1235; - rc = Cache::insertLRUList( cb, lbid, fbo, block ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( Cache::cacheKeyExist( oid, lbid ) == true ); - - // add one more block - block.data[1] = '2'; - lbid = 1237; - rc = Cache::insertLRUList( cb, lbid, fbo, block ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( Cache::cacheKeyExist( oid, lbid ) == true ); - - // load one from lruList - memset( block.data, 0, BYTE_PER_BLOCK ); - lbid = 1235; - key = Cache::getCacheKey( oid, lbid ); - rc = Cache::loadCacheBlock( key, block ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( block.data[1] == '1' ); - CPPUNIT_ASSERT( block.data[2] == 'a' ); - - // move one block from lrulist to write list - block.data[2] = 'm'; - rc = Cache::modifyCacheBlock( key, block ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - memset( block.data, 0, BYTE_PER_BLOCK ); // clear - rc = Cache::loadCacheBlock( key, block ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( block.data[2] == 'm' ); - - Cache::printCacheList(); - - // flush cache - Cache::flushCache(); - Cache::printCacheList(); - - // shutdown -// Cache::freeMemory(); - } - - void testCacheReadWrite() - { - DbFileOp dbFileOp; - DataBlock block; - CommBlock cb; - OID oid = 16; - uint64_t lbid = 1234; - int rc, totalBlock; - int allocSize = 0; - - BRMWrapper::setUseBrm(true); - dbFileOp.setTransId( 3 ); - Cache::clear(); - - cb.session.txnid = 10; - cb.file.oid = oid; - - dbFileOp.deleteFile( oid ); - CPPUNIT_ASSERT( dbFileOp.exists( oid ) == false ); - - CPPUNIT_ASSERT( dbFileOp.createFile( oid, 10, allocSize ) == NO_ERROR ); - - cb.file.pFile = dbFileOp.openFile( oid ); - CPPUNIT_ASSERT( cb.file.pFile != NULL ); - - BRMWrapper::getInstance()->getBrmInfo( oid, 3, lbid ); - - Cache::setUseCache( true ); - Cache::clear(); - int size; - - size = Cache::getListSize( FREE_LIST ); - CPPUNIT_ASSERT( size == DEFAULT_CACHE_BLOCK ); - size = Cache::getListSize( WRITE_LIST ); - CPPUNIT_ASSERT( size == 0 ); - - - totalBlock = Cache::getTotalBlock(); - - for ( long i = 0; i < totalBlock; i++ ) - { - - rc = dbFileOp.readDBFile( cb, block.data, lbid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - block.data[1] = 'k'; - rc = dbFileOp.writeDBFile( cb, block.data, lbid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - lbid++; - - if ( i == DEFAULT_CACHE_BLOCK - 20 ) - BRMWrapper::getInstance()->getBrmInfo( oid, 3, lbid ); - - if ( i % 5000 == 0 && i != 0 ) - { - printf( "\ni=%ld", i ); - Cache::printCacheList(); - } - - if ( i == totalBlock - 2 ) - { - dbFileOp.flushCache(); - Cache::setUseCache( false ); - } - } - - } - - template struct binary; - typedef binary<16> binary16; - typedef binary<32> binary32; - template - struct binary { - unsigned char data[W]; // May be ok for empty value ? - void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} - inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} - inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} - }; - - void testExtentCrWOPreallocBin() { - IDBDataFile* pFile = NULL; - ColumnOpCompress1 fileOp; - BlockOp blockOp; - char fileName[20]; - int rc; - char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ]; - int dbRoot = 1; - - idbdatafile::IDBPolicy::init(true, false, "", 0); - // Set to versionbuffer to satisfy IDBPolicy::getType - strcpy(fileName, "versionbuffer"); - fileOp.compressionType(1); - - fileOp.deleteFile(fileName); - CPPUNIT_ASSERT(fileOp.exists(fileName) == false); - - uint64_t emptyVal = blockOp.getEmptyRowValue(execplan::CalpontSystemCatalog::BIGINT, 8); - int width = blockOp.getCorrectRowWidth(execplan::CalpontSystemCatalog::BINARY, sizeof (binary16)); - int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; - - // createFile runs IDBDataFile::open + initAbrevCompColumnExtent - // under the hood - // bigint column file - rc = fileOp.createFile(fileName, - nBlocks, // number of blocks - emptyVal, // NULL value - width, // width - dbRoot); // dbroot - CPPUNIT_ASSERT(rc == NO_ERROR); - - fileOp.closeFile(pFile); - - // open created compressed file and check its header - pFile = IDBDataFile::open(IDBPolicy::getType(fileName, - IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - - rc = pFile->seek(0, 0); - CPPUNIT_ASSERT(rc == NO_ERROR); - rc = fileOp.readHeaders(pFile, hdrs); - CPPUNIT_ASSERT(rc == NO_ERROR); - // Couldn't use IDBDataFile->close() here w/o excplicit cast - fileOp.closeFile(pFile); - - // Extend the extent up to 64MB - pFile = IDBDataFile::open(IDBPolicy::getType(fileName, - IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - - // disable disk space preallocation - idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); - rc = fileOp.initColumnExtent(pFile, - dbRoot, - BYTE_PER_BLOCK - nBlocks, // number of blocks - emptyVal, - width, - false, // use existing file - false, // don't expand; new extent - false, // add full (not abbreviated) extent - true); // optimize extention - - CPPUNIT_ASSERT(rc == NO_ERROR); - fileOp.closeFile(pFile); - // file has been extended - cout << endl << "file has been extended"; - - // write up to INITIAL_EXTENT_ROWS_TO_DISK + 1 rows into the file - - Column curCol; - binary16 valArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; - RID rowIdArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; - // This is the magic for the stub in FileOp::oid2FileName - int fid = 42; - - for (uint64_t it = 0; it <= INITIAL_EXTENT_ROWS_TO_DISK; it++) { - rowIdArray[it] = it; - valArray[it].uint64(0) = it + 3; - valArray[it].uint64(1) = it + 5; - } - - fileOp.initColumn(curCol); - fileOp.setColParam(curCol, - 1, // column number - width, - execplan::CalpontSystemCatalog::BINARY, - WriteEngine::WR_BINARY, - fid, - 1); //compression type - - string segFile; - // openColumnFile uses DBRM's oid server but we - // have to get the chunks' pointers from the header. - curCol.dataFile.pFile = fileOp.openFile( - curCol, - dbRoot, - 0, - 0, - segFile, - false, - "r+b", - BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB - - CPPUNIT_ASSERT(rc == NO_ERROR); - - rc = fileOp.writeRow(curCol, INITIAL_EXTENT_ROWS_TO_DISK + 1, - (RID*) rowIdArray, valArray); - CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); // I prefer this way as it prints values - - // flush and close the file used for reading - fileOp.clearColumn(curCol); - - std::map oids; - oids[fid] = fid; - - // flush changed chunks from the Manager - int rtn1 = fileOp.chunkManager()->flushChunks(rc, oids); - - // read back the file - cout << endl << "Read file "; - DataBlock block; - binary16* bin16 = (binary16*) block.data; - - fileOp.initColumn(curCol); - fileOp.setColParam(curCol, - 1, // column number - width, - execplan::CalpontSystemCatalog::BINARY, - WriteEngine::WR_BINARY, - fid, - 1); //compression type - - curCol.dataFile.pFile = fileOp.openFile( - curCol, - dbRoot, - 0, - 0, - segFile, - false, - "r+b", - BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB - - CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); - - int blocks = fileOp.blocksInFile(curCol.dataFile.pFile); - - for (int b = 0; b < blocks; b++) { - rc = fileOp.chunkManager()->readBlock(curCol.dataFile.pFile, block.data, b); // ColumnOpCompress1.readBlock() is protected so ... - CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); - //cout << endl << bin16[0].uint64(0); - CPPUNIT_ASSERT_EQUAL(b * 512UL + 3, bin16[0].uint64(0)); // Checking just first value of each block as it was written before - CPPUNIT_ASSERT_EQUAL(b * 512UL + 5, bin16[0].uint64(1)); - } - - fileOp.clearColumn(curCol); - fileOp.closeFile(curCol.dataFile.pFile); // Seems done by clearColumn, but anyways... - - cout << endl << "Delete file "; - - fileOp.deleteFile(fileName); - CPPUNIT_ASSERT(fileOp.exists(fileName) == false); - cout << endl << "End of test"; - } - */ - - void testCleanup() - { - // shutdown - Cache::freeMemory(); - } + fflush(pFile); + rc = dbFileOp.readDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( memcmp( block.data, originBlock.data, BYTE_PER_BLOCK ) == 0 ); + + // test commit twice and rollback to see whether rollback commit values + printf( "\n\nThis is the test for commit twice and then rollback to see whether the previous two + committed one got rollback\n" ); + + // first commit + transId++; + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + memset( block.data, 'e', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = BRMWrapper::getInstance()->commit( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // second commit + transId++; + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + memset( block.data, 'f', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + fflush(pFile); + + rc = BRMWrapper::getInstance()->commit( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + memcpy( originBlock.data, block.data, BYTE_PER_BLOCK ); + + // rollback + transId++; + rc = BRMWrapper::getInstance()->writeVB( pFile, transId, oid, fboList ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + memset( block.data, 'g', BYTE_PER_BLOCK ); + rc = dbFileOp.writeDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + fflush(pFile); + + rc = BRMWrapper::getInstance()->rollBack( transId ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + // check whether they are the same one + rc = dbFileOp.readDBFile( pFile, block.data, lbid0 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + fflush(pFile); + + CPPUNIT_ASSERT( memcmp( block.data, originBlock.data, BYTE_PER_BLOCK ) == 0 ); + } + + void testDM() + { + FileOp fileOp; + char fileName[FILE_NAME_SIZE]; + int rc; + + rc = fileOp.oid2FileName( 100, fileName, true ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = fileOp.oid2FileName( 3071, fileName, true ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + + } + + void testCacheBasic() + { + DataBlock block; + OID oid = 15; + uint64_t lbid = 1234, fbo = 3; + int rc; + CacheKey key; + CommBlock cb; + + Cache::init(); + cb.file.oid = oid; + memset( block.data, 'a', BYTE_PER_BLOCK ); + rc = Cache::insertLRUList( cb, lbid, fbo, block ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( Cache::cacheKeyExist( oid, lbid ) == true ); + + // add a second block + block.data[1] = '1'; + lbid = 1235; + rc = Cache::insertLRUList( cb, lbid, fbo, block ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( Cache::cacheKeyExist( oid, lbid ) == true ); + + // add one more block + block.data[1] = '2'; + lbid = 1237; + rc = Cache::insertLRUList( cb, lbid, fbo, block ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( Cache::cacheKeyExist( oid, lbid ) == true ); + + // load one from lruList + memset( block.data, 0, BYTE_PER_BLOCK ); + lbid = 1235; + key = Cache::getCacheKey( oid, lbid ); + rc = Cache::loadCacheBlock( key, block ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( block.data[1] == '1' ); + CPPUNIT_ASSERT( block.data[2] == 'a' ); + + // move one block from lrulist to write list + block.data[2] = 'm'; + rc = Cache::modifyCacheBlock( key, block ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + memset( block.data, 0, BYTE_PER_BLOCK ); // clear + rc = Cache::loadCacheBlock( key, block ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT( block.data[2] == 'm' ); + + Cache::printCacheList(); + + // flush cache + Cache::flushCache(); + Cache::printCacheList(); + + // shutdown + // Cache::freeMemory(); + } + + void testCacheReadWrite() + { + DbFileOp dbFileOp; + DataBlock block; + CommBlock cb; + OID oid = 16; + uint64_t lbid = 1234; + int rc, totalBlock; + int allocSize = 0; + + BRMWrapper::setUseBrm(true); + dbFileOp.setTransId( 3 ); + Cache::clear(); + + cb.session.txnid = 10; + cb.file.oid = oid; + + dbFileOp.deleteFile( oid ); + CPPUNIT_ASSERT( dbFileOp.exists( oid ) == false ); + + CPPUNIT_ASSERT( dbFileOp.createFile( oid, 10, allocSize ) == NO_ERROR ); + + cb.file.pFile = dbFileOp.openFile( oid ); + CPPUNIT_ASSERT( cb.file.pFile != NULL ); + + BRMWrapper::getInstance()->getBrmInfo( oid, 3, lbid ); + + Cache::setUseCache( true ); + Cache::clear(); + int size; + + size = Cache::getListSize( FREE_LIST ); + CPPUNIT_ASSERT( size == DEFAULT_CACHE_BLOCK ); + size = Cache::getListSize( WRITE_LIST ); + CPPUNIT_ASSERT( size == 0 ); + + + totalBlock = Cache::getTotalBlock(); + + for ( long i = 0; i < totalBlock; i++ ) + { + + rc = dbFileOp.readDBFile( cb, block.data, lbid ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + block.data[1] = 'k'; + rc = dbFileOp.writeDBFile( cb, block.data, lbid ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + lbid++; + + if ( i == DEFAULT_CACHE_BLOCK - 20 ) + BRMWrapper::getInstance()->getBrmInfo( oid, 3, lbid ); + + if ( i % 5000 == 0 && i != 0 ) + { + printf( "\ni=%ld", i ); + Cache::printCacheList(); + } + + if ( i == totalBlock - 2 ) + { + dbFileOp.flushCache(); + Cache::setUseCache( false ); + } + } + + } + + template struct binary; + typedef binary<16> binary16; + typedef binary<32> binary32; + template + struct binary { + unsigned char data[W]; // May be ok for empty value ? + void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} + inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} + inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} + }; + + void testExtentCrWOPreallocBin() { + IDBDataFile* pFile = NULL; + ColumnOpCompress1 fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ]; + int dbRoot = 1; + + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy(fileName, "versionbuffer"); + fileOp.compressionType(1); + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + uint64_t emptyVal = blockOp.getEmptyRowValue(execplan::CalpontSystemCatalog::BIGINT, 8); + int width = blockOp.getCorrectRowWidth(execplan::CalpontSystemCatalog::BINARY, sizeof (binary16)); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile(fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot); // dbroot + CPPUNIT_ASSERT(rc == NO_ERROR); + + fileOp.closeFile(pFile); + + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT(rc == NO_ERROR); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + // Extend the extent up to 64MB + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + // disable disk space preallocation + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK - nBlocks, // number of blocks + emptyVal, + width, + false, // use existing file + false, // don't expand; new extent + false, // add full (not abbreviated) extent + true); // optimize extention + + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.closeFile(pFile); + // file has been extended + cout << endl << "file has been extended"; + + // write up to INITIAL_EXTENT_ROWS_TO_DISK + 1 rows into the file + + Column curCol; + binary16 valArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; + RID rowIdArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; + // This is the magic for the stub in FileOp::oid2FileName + int fid = 42; + + for (uint64_t it = 0; it <= INITIAL_EXTENT_ROWS_TO_DISK; it++) { + rowIdArray[it] = it; + valArray[it].uint64(0) = it + 3; + valArray[it].uint64(1) = it + 5; + } + + fileOp.initColumn(curCol); + fileOp.setColParam(curCol, + 1, // column number + width, + execplan::CalpontSystemCatalog::BINARY, + WriteEngine::WR_BINARY, + fid, + 1); //compression type + + string segFile; + // openColumnFile uses DBRM's oid server but we + // have to get the chunks' pointers from the header. + curCol.dataFile.pFile = fileOp.openFile( + curCol, + dbRoot, + 0, + 0, + segFile, + false, + "r+b", + BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB + + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = fileOp.writeRow(curCol, INITIAL_EXTENT_ROWS_TO_DISK + 1, + (RID*) rowIdArray, valArray); + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); // I prefer this way as it prints values + + // flush and close the file used for reading + fileOp.clearColumn(curCol); + + std::map oids; + oids[fid] = fid; + + // flush changed chunks from the Manager + int rtn1 = fileOp.chunkManager()->flushChunks(rc, oids); + + // read back the file + cout << endl << "Read file "; + DataBlock block; + binary16* bin16 = (binary16*) block.data; + + fileOp.initColumn(curCol); + fileOp.setColParam(curCol, + 1, // column number + width, + execplan::CalpontSystemCatalog::BINARY, + WriteEngine::WR_BINARY, + fid, + 1); //compression type + + curCol.dataFile.pFile = fileOp.openFile( + curCol, + dbRoot, + 0, + 0, + segFile, + false, + "r+b", + BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB + + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); + + int blocks = fileOp.blocksInFile(curCol.dataFile.pFile); + + for (int b = 0; b < blocks; b++) { + rc = fileOp.chunkManager()->readBlock(curCol.dataFile.pFile, block.data, b); // + ColumnOpCompress1.readBlock() is protected so ... CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); + //cout << endl << bin16[0].uint64(0); + CPPUNIT_ASSERT_EQUAL(b * 512UL + 3, bin16[0].uint64(0)); // Checking just first value of each + block as it was written before CPPUNIT_ASSERT_EQUAL(b * 512UL + 5, bin16[0].uint64(1)); + } + + fileOp.clearColumn(curCol); + fileOp.closeFile(curCol.dataFile.pFile); // Seems done by clearColumn, but anyways... + + cout << endl << "Delete file "; + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + cout << endl << "End of test"; + } + */ + + void testCleanup() + { + // shutdown + Cache::freeMemory(); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( SharedTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(SharedTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/tools/bincvt/li2bin.cpp b/tools/bincvt/li2bin.cpp index 40ce02a8d..51b1d8b64 100644 --- a/tools/bincvt/li2bin.cpp +++ b/tools/bincvt/li2bin.cpp @@ -22,36 +22,37 @@ namespace { inline long long atoll(const char* s) { - return _strtoi64(s, 0, 10); + return _strtoi64(s, 0, 10); } inline uint64_t strtoull(const char* s, char** a, int b) { - return _strtoi64(s, a, b); -} + return _strtoi64(s, a, b); } +} // namespace #endif -//1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular courts above the| +// 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular +// courts above the| #pragma pack(1) struct lineitem_image { - int32_t l_orderkey; - int32_t l_partkey; - int32_t l_suppkey; //12 - int64_t l_linenumber; - int64_t l_quantity; - int64_t l_extendedprice; - int64_t l_discount; - int64_t l_tax; //40 - char l_returnflag; - char l_linestatus; //2 - int32_t l_shipdate; - int32_t l_commitdate; - int32_t l_receiptdate; //12 - char l_shipinstruct[25]; - char l_shipmode[10]; - char l_comment[44]; //79 + int32_t l_orderkey; + int32_t l_partkey; + int32_t l_suppkey; // 12 + int64_t l_linenumber; + int64_t l_quantity; + int64_t l_extendedprice; + int64_t l_discount; + int64_t l_tax; // 40 + char l_returnflag; + char l_linestatus; // 2 + int32_t l_shipdate; + int32_t l_commitdate; + int32_t l_receiptdate; // 12 + char l_shipinstruct[25]; + char l_shipmode[10]; + char l_comment[44]; // 79 }; BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145); @@ -59,38 +60,38 @@ BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145); // Decimal, date, and date/time columns are still treated as integer. struct unsigned_lineitem_image { - uint32_t l_orderkey; - uint32_t l_partkey; - uint32_t l_suppkey; //12 - uint64_t l_linenumber; - int64_t l_quantity; - int64_t l_extendedprice; - int64_t l_discount; - int64_t l_tax; //40 - char l_returnflag; - char l_linestatus; //2 - int32_t l_shipdate; - int32_t l_commitdate; - int32_t l_receiptdate; //12 - char l_shipinstruct[25]; - char l_shipmode[10]; - char l_comment[44]; //79 + uint32_t l_orderkey; + uint32_t l_partkey; + uint32_t l_suppkey; // 12 + uint64_t l_linenumber; + int64_t l_quantity; + int64_t l_extendedprice; + int64_t l_discount; + int64_t l_tax; // 40 + char l_returnflag; + char l_linestatus; // 2 + int32_t l_shipdate; + int32_t l_commitdate; + int32_t l_receiptdate; // 12 + char l_shipinstruct[25]; + char l_shipmode[10]; + char l_comment[44]; // 79 }; BOOST_STATIC_ASSERT(sizeof(struct unsigned_lineitem_image) == 145); struct Date { - unsigned spare : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; + unsigned spare : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; }; BOOST_STATIC_ASSERT(sizeof(struct Date) == 4); union date_image { - struct Date d; - int32_t i; + struct Date d; + int32_t i; }; namespace @@ -99,183 +100,182 @@ typedef boost::tokenizer > tokenizer; int64_t idbbigdec2(const string& str) { - int64_t val = 0; - string::size_type idx = string::npos; - string tmp(str); - idx = tmp.find('.'); + int64_t val = 0; + string::size_type idx = string::npos; + string tmp(str); + idx = tmp.find('.'); - if (idx != string::npos) - tmp.erase(idx, 1); - else - tmp.append("00"); + if (idx != string::npos) + tmp.erase(idx, 1); + else + tmp.append("00"); - val = atoll(tmp.c_str()); - return val; + val = atoll(tmp.c_str()); + return val; } int32_t idbdate(const string& str) { - date_image di; - di.i = 0; - boost::char_separator sep("-"); - tokenizer tokens(str, sep); - tokenizer::iterator tok_iter = tokens.begin(); + date_image di; + di.i = 0; + boost::char_separator sep("-"); + tokenizer tokens(str, sep); + tokenizer::iterator tok_iter = tokens.begin(); - // Note that the spare bits must be set, instead of leaving them - // initialized to 0; to be compatible with internal date format. - if (distance(tok_iter, tokens.end()) == 3) - { - di.d.spare = 0x3E; - di.d.year = atoi(tok_iter->c_str()); - ++tok_iter; - di.d.month = atoi(tok_iter->c_str()); - ++tok_iter; - di.d.day = atoi(tok_iter->c_str()); - ++tok_iter; - } + // Note that the spare bits must be set, instead of leaving them + // initialized to 0; to be compatible with internal date format. + if (distance(tok_iter, tokens.end()) == 3) + { + di.d.spare = 0x3E; + di.d.year = atoi(tok_iter->c_str()); + ++tok_iter; + di.d.month = atoi(tok_iter->c_str()); + ++tok_iter; + di.d.day = atoi(tok_iter->c_str()); + ++tok_iter; + } - return di.i; + return di.i; } int parseinto(lineitem_image& img, const string& buf) { - memset(&img, 0, sizeof(img)); + memset(&img, 0, sizeof(img)); - boost::char_separator sep("|"); - tokenizer tokens(buf, sep); - tokenizer::iterator tok_iter = tokens.begin(); + boost::char_separator sep("|"); + tokenizer tokens(buf, sep); + tokenizer::iterator tok_iter = tokens.begin(); - if (distance(tok_iter, tokens.end()) < 16) - return -1; + if (distance(tok_iter, tokens.end()) < 16) + return -1; - img.l_orderkey = atoi(tok_iter->c_str()); - ++tok_iter; - img.l_partkey = atoi(tok_iter->c_str()); - ++tok_iter; - img.l_suppkey = atoi(tok_iter->c_str()); - ++tok_iter; - img.l_linenumber = atoll(tok_iter->c_str()); - ++tok_iter; - img.l_quantity = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_extendedprice = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_discount = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_tax = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_returnflag = tok_iter->at(0); - ++tok_iter; - img.l_linestatus = tok_iter->at(0); - ++tok_iter; - img.l_shipdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_commitdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_receiptdate = idbdate(tok_iter->c_str()); - ++tok_iter; - memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; + img.l_orderkey = atoi(tok_iter->c_str()); + ++tok_iter; + img.l_partkey = atoi(tok_iter->c_str()); + ++tok_iter; + img.l_suppkey = atoi(tok_iter->c_str()); + ++tok_iter; + img.l_linenumber = atoll(tok_iter->c_str()); + ++tok_iter; + img.l_quantity = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_extendedprice = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_discount = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_tax = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_returnflag = tok_iter->at(0); + ++tok_iter; + img.l_linestatus = tok_iter->at(0); + ++tok_iter; + img.l_shipdate = idbdate(tok_iter->c_str()); + ++tok_iter; + img.l_commitdate = idbdate(tok_iter->c_str()); + ++tok_iter; + img.l_receiptdate = idbdate(tok_iter->c_str()); + ++tok_iter; + memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size()); + ++tok_iter; + memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size()); + ++tok_iter; + memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size()); + ++tok_iter; - return 0; + return 0; } int unsigned_parseinto(unsigned_lineitem_image& img, const string& buf) { - memset(&img, 0, sizeof(img)); + memset(&img, 0, sizeof(img)); - boost::char_separator sep("|"); - tokenizer tokens(buf, sep); - tokenizer::iterator tok_iter = tokens.begin(); + boost::char_separator sep("|"); + tokenizer tokens(buf, sep); + tokenizer::iterator tok_iter = tokens.begin(); - if (distance(tok_iter, tokens.end()) < 16) - return -1; + if (distance(tok_iter, tokens.end()) < 16) + return -1; - img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_partkey = strtoul(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_quantity = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_extendedprice = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_discount = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_tax = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_returnflag = tok_iter->at(0); - ++tok_iter; - img.l_linestatus = tok_iter->at(0); - ++tok_iter; - img.l_shipdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_commitdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_receiptdate = idbdate(tok_iter->c_str()); - ++tok_iter; - memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; + img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10); + ++tok_iter; + img.l_partkey = strtoul(tok_iter->c_str(), 0, 10); + ++tok_iter; + img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10); + ++tok_iter; + img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10); + ++tok_iter; + img.l_quantity = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_extendedprice = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_discount = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_tax = idbbigdec2(tok_iter->c_str()); + ++tok_iter; + img.l_returnflag = tok_iter->at(0); + ++tok_iter; + img.l_linestatus = tok_iter->at(0); + ++tok_iter; + img.l_shipdate = idbdate(tok_iter->c_str()); + ++tok_iter; + img.l_commitdate = idbdate(tok_iter->c_str()); + ++tok_iter; + img.l_receiptdate = idbdate(tok_iter->c_str()); + ++tok_iter; + memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size()); + ++tok_iter; + memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size()); + ++tok_iter; + memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size()); + ++tok_iter; - return 0; + return 0; } -} +} // namespace int main(int argc, char** argv) { - if ((argc > 1) && (strcmp(argv[1], "-h") == 0)) - { - std::cerr << "li2bin [-u]" << std::endl; - std::cerr << " -u Create first 4 fields as unsigned integers" << std::endl; - return 0; - } + if ((argc > 1) && (strcmp(argv[1], "-h") == 0)) + { + std::cerr << "li2bin [-u]" << std::endl; + std::cerr << " -u Create first 4 fields as unsigned integers" << std::endl; + return 0; + } - string input; + string input; #ifdef _MSC_VER - _setmode(1, _O_BINARY); + _setmode(1, _O_BINARY); #endif - getline(cin, input); + getline(cin, input); - if ((argc > 1) && (strcmp(argv[1], "-u") == 0)) + if ((argc > 1) && (strcmp(argv[1], "-u") == 0)) + { + unsigned_lineitem_image i; + + while (!cin.eof()) { - unsigned_lineitem_image i; + if (unsigned_parseinto(i, input) == 0) + cout.write(reinterpret_cast(&i), sizeof(i)); - while (!cin.eof()) - { - if (unsigned_parseinto(i, input) == 0) - cout.write(reinterpret_cast(&i), sizeof(i)); - - getline(cin, input); - } + getline(cin, input); } - else + } + else + { + lineitem_image i; + + while (!cin.eof()) { - lineitem_image i; + if (parseinto(i, input) == 0) + cout.write(reinterpret_cast(&i), sizeof(i)); - while (!cin.eof()) - { - if (parseinto(i, input) == 0) - cout.write(reinterpret_cast(&i), sizeof(i)); - - getline(cin, input); - } + getline(cin, input); } + } - return 0; + return 0; } // vim:ts=4 sw=4: - diff --git a/tools/cfread/cfread.cpp b/tools/cfread/cfread.cpp index c0390fc15..e6fe14c2a 100644 --- a/tools/cfread/cfread.cpp +++ b/tools/cfread/cfread.cpp @@ -10,20 +10,18 @@ using namespace std; -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } } // TODO: @@ -35,161 +33,151 @@ void timespec_sub(const struct timespec& tv1, // int main(int argc, char** argv) { - uint64_t acc = 0; - uint64_t readBlocks = 0; // read size ib blocks - uint64_t readSize = 0; // read size ib bytes - uint64_t readBufferSz = 0; - const uint64_t blockSize = 8192; - char* alignedbuff = 0; - boost::scoped_array realbuff; - const unsigned pageSize = 4096; //getpagesize(); - WriteEngine::FileOp fFileOp; - BRM::OID_t oid; - char fname[256]; - struct timespec tm; - struct timespec tm2; - struct timespec tm3; - struct timespec starttm; - struct timespec endtm; - struct timespec tottm; - bool odirect = true; - int fd = 0; - char response = 'Y'; + uint64_t acc = 0; + uint64_t readBlocks = 0; // read size ib blocks + uint64_t readSize = 0; // read size ib bytes + uint64_t readBufferSz = 0; + const uint64_t blockSize = 8192; + char* alignedbuff = 0; + boost::scoped_array realbuff; + const unsigned pageSize = 4096; // getpagesize(); + WriteEngine::FileOp fFileOp; + BRM::OID_t oid; + char fname[256]; + struct timespec tm; + struct timespec tm2; + struct timespec tm3; + struct timespec starttm; + struct timespec endtm; + struct timespec tottm; + bool odirect = true; + int fd = 0; + char response = 'Y'; - if (argc <= 1) - { - cerr << "usage: testread " << endl; - return -1; - } + if (argc <= 1) + { + cerr << "usage: testread " << endl; + return -1; + } - oid = atoi(argv[1]); + oid = atoi(argv[1]); - if (oid <= 0) - exit(-1); + if (oid <= 0) + exit(-1); - if (argc >= 2) - { - readBlocks = atoi(argv[2]); + if (argc >= 2) + { + readBlocks = atoi(argv[2]); - if (readBlocks <= 0) - readBlocks = 8; - } + if (readBlocks <= 0) + readBlocks = 8; + } - if (argc >= 4) - { - odirect = false; - } + if (argc >= 4) + { + odirect = false; + } - readSize = readBlocks * blockSize; - readBufferSz = readSize + pageSize; + readSize = readBlocks * blockSize; + readBufferSz = readSize + pageSize; - realbuff.reset(new char[readBufferSz]); + realbuff.reset(new char[readBufferSz]); - if (realbuff.get() == 0) - { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return 0; - } + if (realbuff.get() == 0) + { + cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; + return 0; + } - if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR) - { - fname[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } - else - { - cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl; - } + if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR) + { + fname[0] = 0; + throw std::runtime_error("fileOp.getFileName failed"); + } + else + { + cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl; + } #if __LP64__ - alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize); + alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize); #else - alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize); + alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize); #endif - idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize); - idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0); + idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize); + idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0); - if (odirect) - fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME); - else - fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME); + if (odirect) + fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME); + else + fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME); - if (fd < 0) + if (fd < 0) + { + cerr << "Open failed" << endl; + perror("open"); + throw runtime_error("Error opening file"); + } + + while (toupper(response) != 'N') + { + uint64_t i = 1; + uint64_t rCnt = 0; + clock_gettime(CLOCK_REALTIME, &starttm); + + while (i != 0) { - cerr << "Open failed" << endl; - perror("open"); - throw runtime_error("Error opening file"); - } + // clock_gettime(CLOCK_REALTIME, &tm); + i = pread(fd, alignedbuff, readSize, acc); + // clock_gettime(CLOCK_REALTIME, &tm2); + idbassert(i == 0 || i == readSize); + idbassert(i % pageSize == 0); + idbassert(acc % pageSize == 0); - while (toupper(response) != 'N') - { - uint64_t i = 1; - uint64_t rCnt = 0; - clock_gettime(CLOCK_REALTIME, &starttm); + if (i < 0 && errno == EINTR) + { + timespec_sub(tm, tm2, tm3); + cout << "* " << i << " " << right << setw(2) << setfill(' ') << tm3.tv_sec << "." << right << setw(9) + << setfill('0') << tm3.tv_nsec << endl; + continue; + } + else if (i < 0) + { + timespec_sub(tm, tm2, tm3); + cout << "* i: " << i << " sz: " << readSize << " acc: " << acc << right << setw(2) << setfill(' ') + << tm3.tv_sec << " " << right << tm3.tv_nsec << endl; + perror("pread"); + // make loop exit + i = 0; + } - while (i != 0) - { - //clock_gettime(CLOCK_REALTIME, &tm); - i = pread(fd, alignedbuff, readSize, acc); - //clock_gettime(CLOCK_REALTIME, &tm2); - idbassert(i == 0 || i == readSize); - idbassert(i % pageSize == 0); - idbassert(acc % pageSize == 0); + acc += i; - if (i < 0 && errno == EINTR) - { - timespec_sub(tm, tm2, tm3); - cout << "* " - << i << " " - << right << setw(2) << setfill(' ') << tm3.tv_sec << "." - << right << setw(9) << setfill('0') << tm3.tv_nsec - << endl; - continue; - } - else if (i < 0) - { - timespec_sub(tm, tm2, tm3); - cout << "* i: " - << i << " sz: " << readSize << " acc: " << acc - << right << setw(2) << setfill(' ') << tm3.tv_sec << " " - << right << tm3.tv_nsec - << endl; - perror("pread"); - //make loop exit - i = 0; - } + if (i > 0) + rCnt++; - acc += i; + // timespec_sub(tm, tm2, tm3); + // cout + // << i << " " + // << right << setw(2) << setfill(' ') << tm3.tv_sec << " " + // << right << tm3.tv_nsec + // << endl; - if (i > 0) - rCnt++; + } // while(acc... - //timespec_sub(tm, tm2, tm3); - //cout - // << i << " " - // << right << setw(2) << setfill(' ') << tm3.tv_sec << " " - // << right << tm3.tv_nsec - // << endl; + clock_gettime(CLOCK_REALTIME, &endtm); + timespec_sub(starttm, endtm, tottm); - } // while(acc... + cout << "Total reads: " << rCnt << " sz: " << acc / (1024 * 1024) << "MB" + << " tm: " << tottm.tv_sec << "secs " << tottm.tv_nsec << "ns" << endl; - clock_gettime(CLOCK_REALTIME, &endtm); - timespec_sub(starttm, endtm, tottm); + cout << "Repeat the last scan[Y,N]?" << endl; + cin >> response; + acc = 0; - cout << "Total reads: " << rCnt - << " sz: " << acc / (1024 * 1024) << "MB" - << " tm: " << tottm.tv_sec << "secs " - << tottm.tv_nsec << "ns" - << endl; + } // while response... - cout << "Repeat the last scan[Y,N]?" << endl; - cin >> response; - acc = 0; - - } // while response... - - close(fd); - return 0; - -} //main + close(fd); + return 0; +} // main diff --git a/tools/cfread/mtread.cpp b/tools/cfread/mtread.cpp index debc81ef1..d51c990cb 100644 --- a/tools/cfread/mtread.cpp +++ b/tools/cfread/mtread.cpp @@ -11,20 +11,18 @@ using namespace std; -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } } // TODO: @@ -33,256 +31,239 @@ void timespec_sub(const struct timespec& tv1, struct readThr { -public: - readThr(const int oid, const int rSize) + public: + readThr(const int oid, const int rSize) + { + fblockSize = 8192; + fpageSize = getpagesize(); + freadBlocks = rSize; + freadSize = rSize * fblockSize; // read size ib bytes + freadBufferSz = freadSize + fpageSize; + falignedbuff = 0; + foid = oid; + facc = 0; + memset(fname, 0, sizeof(fname)); + memset((char*)&ftm, 0, sizeof(ftm)); + memset((char*)&ftm2, 0, sizeof(ftm2)); + memset((char*)&ftm3, 0, sizeof(ftm3)); + memset((char*)&fstarttm, 0, sizeof(fstarttm)); + memset((char*)&fendtm, 0, sizeof(fendtm)); + memset((char*)&ftottm, 0, sizeof(ftottm)); + fodirect = true; + fd = 0; + cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl; + } + + void operator()() + { + WriteEngine::FileOp fFileOp; + char frealbuff[freadBufferSz]; + memset(frealbuff, 0, freadBufferSz); + + if (frealbuff == 0) { - fblockSize = 8192; - fpageSize = getpagesize(); - freadBlocks = rSize; - freadSize = rSize * fblockSize; // read size ib bytes - freadBufferSz = freadSize + fpageSize; - falignedbuff = 0; - foid = oid; - facc = 0; - memset(fname, 0, sizeof(fname)); - memset((char*)&ftm, 0, sizeof(ftm)); - memset((char*)&ftm2, 0, sizeof(ftm2)); - memset((char*)&ftm3, 0, sizeof(ftm3)); - memset((char*)&fstarttm, 0, sizeof(fstarttm)); - memset((char*)&fendtm, 0, sizeof(fendtm)); - memset((char*)&ftottm, 0, sizeof(ftottm)); - fodirect = true; - fd = 0; - cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl; + cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; + return; } - void operator() () + if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR) { - - WriteEngine::FileOp fFileOp; - char frealbuff[freadBufferSz]; - memset(frealbuff, 0, freadBufferSz); - - if (frealbuff == 0) - { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return; - } - - if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR) - { - fname[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } - else - { - cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl; - } + fname[0] = 0; + throw std::runtime_error("fileOp.getFileName failed"); + } + else + { + cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl; + } #if __LP64__ - falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize); + falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize); #else - falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize); + falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize); #endif - idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize); - idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0); + idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize); + idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0); - if (fodirect) - fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME); - else - fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME); + if (fodirect) + fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME); + else + fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME); - if (fd < 0) - { - cerr << "Open failed" << endl; - perror("open"); - throw runtime_error("Error opening file"); - } + if (fd < 0) + { + cerr << "Open failed" << endl; + perror("open"); + throw runtime_error("Error opening file"); + } - uint64_t i = 1; - uint64_t rCnt = 0; + uint64_t i = 1; + uint64_t rCnt = 0; - clock_gettime(CLOCK_REALTIME, &fstarttm); + clock_gettime(CLOCK_REALTIME, &fstarttm); - while (i > 0) - { - clock_gettime(CLOCK_REALTIME, &ftm); - i = pread(fd, falignedbuff, freadSize, facc); - clock_gettime(CLOCK_REALTIME, &ftm2); + while (i > 0) + { + clock_gettime(CLOCK_REALTIME, &ftm); + i = pread(fd, falignedbuff, freadSize, facc); + clock_gettime(CLOCK_REALTIME, &ftm2); - idbassert(i == 0 || i == freadSize); - idbassert(i % fpageSize == 0); - idbassert(facc % fpageSize == 0); + idbassert(i == 0 || i == freadSize); + idbassert(i % fpageSize == 0); + idbassert(facc % fpageSize == 0); - if (i < 0 && errno == EINTR) - { - timespec_sub(ftm, ftm2, ftm3); - cout << "* " - << i << " " - << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." - << right << setw(9) << setfill('0') << ftm3.tv_nsec - << endl; - continue; - } - else if (i < 0) - { - timespec_sub(ftm, ftm2, ftm3); - cout << "* i: " - << i << " sz: " << freadSize << " acc: " << facc - << right << setw(2) << setfill(' ') << ftm3.tv_sec << " " - << right << ftm3.tv_nsec - << endl; - perror("pread"); - } + if (i < 0 && errno == EINTR) + { + timespec_sub(ftm, ftm2, ftm3); + cout << "* " << i << " " << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." << right << setw(9) + << setfill('0') << ftm3.tv_nsec << endl; + continue; + } + else if (i < 0) + { + timespec_sub(ftm, ftm2, ftm3); + cout << "* i: " << i << " sz: " << freadSize << " acc: " << facc << right << setw(2) << setfill(' ') + << ftm3.tv_sec << " " << right << ftm3.tv_nsec << endl; + perror("pread"); + } - facc += i; + facc += i; - if (i > 0) - rCnt++; + if (i > 0) + rCnt++; - /** - timespec_sub(ftm, ftm2, ftm3); - cout - << rCnt << " " << facc/(1024*1024) - << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." - << right << ftm3.tv_nsec << " i: " << i/(1024*1024) - << endl; - **/ + /** + timespec_sub(ftm, ftm2, ftm3); + cout + << rCnt << " " << facc/(1024*1024) + << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." + << right << ftm3.tv_nsec << " i: " << i/(1024*1024) + << endl; + **/ - } // while(acc... + } // while(acc... - clock_gettime(CLOCK_REALTIME, &fendtm); - timespec_sub(fstarttm, fendtm, ftottm); + clock_gettime(CLOCK_REALTIME, &fendtm); + timespec_sub(fstarttm, fendtm, ftottm); - cout << "Total reads: " << rCnt - << " sz: " << facc / (1024 * 1024) << "MB" - << " tm: " << ftottm.tv_sec << "secs " - << ftottm.tv_nsec << "ns" - << endl; + cout << "Total reads: " << rCnt << " sz: " << facc / (1024 * 1024) << "MB" + << " tm: " << ftottm.tv_sec << "secs " << ftottm.tv_nsec << "ns" << endl; - facc = 0; - close(fd); - } // operator() + facc = 0; + close(fd); + } // operator() -public: - uint64_t facc; - uint64_t freadBlocks; // read size ib blocks - uint64_t freadSize; // read size ib bytes - uint64_t freadBufferSz; - uint64_t fblockSize; - char* falignedbuff; - unsigned fpageSize ; - BRM::OID_t foid; - char fname[256]; - struct timespec ftm; - struct timespec ftm2; - struct timespec ftm3; - struct timespec fstarttm; - struct timespec fendtm; - struct timespec ftottm; - bool fodirect; - int fd; + public: + uint64_t facc; + uint64_t freadBlocks; // read size ib blocks + uint64_t freadSize; // read size ib bytes + uint64_t freadBufferSz; + uint64_t fblockSize; + char* falignedbuff; + unsigned fpageSize; + BRM::OID_t foid; + char fname[256]; + struct timespec ftm; + struct timespec ftm2; + struct timespec ftm3; + struct timespec fstarttm; + struct timespec fendtm; + struct timespec ftottm; + bool fodirect; + int fd; -}; // struct readThr +}; // struct readThr // void usage() { - cerr << "usage: mtread -o -s " << endl; + cerr << "usage: mtread -o -s " << endl; } // main() // int main(int argc, char** argv) { - int ch = 0; - int readBlocks = 0; - BRM::OID_t oid = 0; - std::vector oidList; + int ch = 0; + int readBlocks = 0; + BRM::OID_t oid = 0; + std::vector oidList; - enum CLA_ENUM + enum CLA_ENUM + { + OID = (int)0, + READSIZE + }; + + // longopt struct + // struct option { + // const char *name; + // int has_arg; + // int *flag; + // int val; + //}; + + static struct option long_options[] = {//{const char *name, int has_arg, int *flag, int val}, + {"oid", required_argument, NULL, OID}, + {"rsize", required_argument, NULL, READSIZE}, + {0, 0, 0, 0}}; + + if (argc <= 1) + { + return -1; + } + + // process command line arguments + while ((ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1) + { + // pid_t pidId = getpid(); + switch (ch) { - OID = (int)0, - READSIZE - }; + case OID: + case 'o': + oid = atoi(optarg); + oidList.push_back(oid); + cout << "oid: " << optarg << endl; + break; - //longopt struct - //struct option { - //const char *name; - //int has_arg; - //int *flag; - //int val; - //}; + case READSIZE: + case 's': + readBlocks = atoi(optarg); + cout << "read size: " << optarg << endl; - static struct - option long_options[] = - { - //{const char *name, int has_arg, int *flag, int val}, - {"oid", required_argument, NULL, OID}, - {"rsize", required_argument, NULL, READSIZE}, - {0, 0, 0, 0} - }; + if (readBlocks <= 0) + readBlocks = 1; - if (argc <= 1) - { - return -1; - } + break; - // process command line arguments - while ( (ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1 ) - { - //pid_t pidId = getpid(); - switch (ch) - { + case '?': + default: + cout << "optarg " << optarg << endl; + usage(); + break; - case OID: - case 'o': - oid = atoi(optarg); - oidList.push_back(oid); - cout << "oid: " << optarg << endl; - break; + } // switch - case READSIZE: - case 's': - readBlocks = atoi(optarg); - cout << "read size: " << optarg << endl; + } // while... - if (readBlocks <= 0) - readBlocks = 1; + uint32_t idx = 0; + std::vector thrList; - break; + while (idx < oidList.size()) + { + struct readThr rdr(oidList[idx++], readBlocks); + boost::thread* thr = new boost::thread(rdr); + thrList.push_back(thr); + } - case '?': - default: - cout << "optarg " << optarg << endl; - usage(); - break; + idx = 0; - } // switch + while (idx < thrList.size()) + { + boost::thread* thr = thrList[idx++]; + thr->join(); + delete thr; + } - } // while... - - - uint32_t idx = 0; - std::vector thrList; - - while (idx < oidList.size()) - { - struct readThr rdr(oidList[idx++], readBlocks); - boost::thread* thr = new boost::thread(rdr); - thrList.push_back(thr); - } - - idx = 0; - - while (idx < thrList.size()) - { - boost::thread* thr = thrList[idx++]; - thr->join(); - delete thr; - } - - thrList.clear(); - -} //main + thrList.clear(); +} // main diff --git a/tools/clearShm/main.cpp b/tools/clearShm/main.cpp index f1a33ef78..31e07b9a8 100644 --- a/tools/clearShm/main.cpp +++ b/tools/clearShm/main.cpp @@ -17,7 +17,6 @@ // $Id: main.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - #include "mcsconfig.h" #include @@ -45,206 +44,199 @@ using namespace BRM; namespace { - bool vFlg; bool nFlg; std::mutex coutMutex; void shmDoit(key_t shm_key, const string& label) { - string key_name = ShmKeys::keyToName(shm_key); + string key_name = ShmKeys::keyToName(shm_key); - if (vFlg) + if (vFlg) + { + try { - try - { - bi::shared_memory_object memObj(bi::open_only, - key_name.c_str(), - bi::read_only); - bi::offset_t memSize = 0; - memObj.get_size(memSize); - std::lock_guard lk(coutMutex); - cout << label << ": shm_key: " << shm_key << - "; key_name: " << key_name << - "; size: " << memSize << endl; - } - catch (...) - { - } + bi::shared_memory_object memObj(bi::open_only, key_name.c_str(), bi::read_only); + bi::offset_t memSize = 0; + memObj.get_size(memSize); + std::lock_guard lk(coutMutex); + cout << label << ": shm_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize + << endl; } + catch (...) + { + } + } - if (!nFlg) - { - bi::shared_memory_object::remove(key_name.c_str()); - } + if (!nFlg) + { + bi::shared_memory_object::remove(key_name.c_str()); + } } void shmDoitRange(key_t shm_key, const string& label) { - if (shm_key == 0) return; + if (shm_key == 0) + return; - unsigned shm_key_cnt; + unsigned shm_key_cnt; - for (shm_key_cnt = 0; shm_key_cnt < ShmKeys::KEYRANGE_SIZE; shm_key_cnt++, shm_key++) - { - shmDoit(shm_key, label); - } + for (shm_key_cnt = 0; shm_key_cnt < ShmKeys::KEYRANGE_SIZE; shm_key_cnt++, shm_key++) + { + shmDoit(shm_key, label); + } } void semDoit(key_t sem_key, const string& label) { - string key_name = ShmKeys::keyToName(sem_key); + string key_name = ShmKeys::keyToName(sem_key); - if (vFlg) + if (vFlg) + { + try { - try - { - bi::shared_memory_object memObj(bi::open_only, - key_name.c_str(), - bi::read_only); - bi::offset_t memSize = 0; - memObj.get_size(memSize); - std::lock_guard lk(coutMutex); - cout << label << ": sem_key: " << sem_key << - "; key_name: " << key_name << - "; size: " << memSize << endl; - } - catch (...) - { - } + bi::shared_memory_object memObj(bi::open_only, key_name.c_str(), bi::read_only); + bi::offset_t memSize = 0; + memObj.get_size(memSize); + std::lock_guard lk(coutMutex); + cout << label << ": sem_key: " << sem_key << "; key_name: " << key_name << "; size: " << memSize + << endl; } + catch (...) + { + } + } - if (!nFlg) - { - bi::shared_memory_object::remove(key_name.c_str()); - } + if (!nFlg) + { + bi::shared_memory_object::remove(key_name.c_str()); + } } void usage() { - cout << "usage: clearShm [-cvnh]" << endl; - cout << " delete all ColumnStore shared memory data" << endl; - cout << " -h display this help" << endl; - cout << " -c only clear ColumnStore Engine data, leave OAM intact" << endl; - cout << " -v verbose output" << endl; - cout << " -n don't actually delete anything (implies -v)" << endl; + cout << "usage: clearShm [-cvnh]" << endl; + cout << " delete all ColumnStore shared memory data" << endl; + cout << " -h display this help" << endl; + cout << " -c only clear ColumnStore Engine data, leave OAM intact" << endl; + cout << " -v verbose output" << endl; + cout << " -n don't actually delete anything (implies -v)" << endl; } class ThdFunc { -public: - ThdFunc() : fShm_key(0) { } - ThdFunc(key_t shm_key, const string& label) : fShm_key(shm_key), fLabel(label) { } + public: + ThdFunc() : fShm_key(0) + { + } + ThdFunc(key_t shm_key, const string& label) : fShm_key(shm_key), fLabel(label) + { + } - ~ThdFunc() { } + ~ThdFunc() + { + } - void operator()() const - { - shmDoitRange(fShm_key, fLabel); - } + void operator()() const + { + shmDoitRange(fShm_key, fLabel); + } -private: - //ThdFunc(const ThdFunc& rhs); - //ThdFunc& operator=(const ThdFunc& rhs); + private: + // ThdFunc(const ThdFunc& rhs); + // ThdFunc& operator=(const ThdFunc& rhs); - key_t fShm_key; - string fLabel; + key_t fShm_key; + string fLabel; }; -} +} // namespace int main(int argc, char** argv) { #ifdef _MSC_VER - const char* envp = getenv("SystemRoot"); - string SystemRoot; + const char* envp = getenv("SystemRoot"); + string SystemRoot; - if (envp && *envp) - SystemRoot = envp; - else - SystemRoot = "C:\\WINDOWS"; + if (envp && *envp) + SystemRoot = envp; + else + SystemRoot = "C:\\WINDOWS"; - string tmpEnv = "TMP=" + SystemRoot + "\\Temp"; - _putenv(tmpEnv.c_str()); + string tmpEnv = "TMP=" + SystemRoot + "\\Temp"; + _putenv(tmpEnv.c_str()); #endif - int c; - opterr = 0; - bool cFlg = false; - vFlg = false; - nFlg = false; + int c; + opterr = 0; + bool cFlg = false; + vFlg = false; + nFlg = false; - while ((c = getopt(argc, argv, "cvnh")) != EOF) - switch (c) - { - case 'c': - cFlg = true; - break; - - case 'v': - vFlg = true; - break; - - case 'n': - nFlg = true; - break; - - case 'h': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if (nFlg) - vFlg = true; - - ShmKeys BrmKeys; - - boost::thread_group tg; - boost::thread* tp = 0; - tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_CL_BASE, "COPYLOCK ")); - tg.add_thread(tp); - tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EXTENTMAP_BASE, "EXTMAP ")); - tg.add_thread(tp); - tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE")); - tg.add_thread(tp); - tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM ")); - tg.add_thread(tp); - tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VSS_BASE, "VSS ")); - tg.add_thread(tp); - tg.join_all(); - - shmDoit(BrmKeys.MST_SYSVKEY, "MST "); - - if (!cFlg) + while ((c = getopt(argc, argv, "cvnh")) != EOF) + switch (c) { - shmDoit(BrmKeys.PROCESSSTATUS_SYSVKEY, "PROC_STAT "); - shmDoit(BrmKeys.SYSTEMSTATUS_SYSVKEY, "SYS_STAT "); - shmDoit(BrmKeys.SWITCHSTATUS_SYSVKEY, "SW_STAT "); - shmDoit(BrmKeys.NICSTATUS_SYSVKEY, "NIC_STAT "); - shmDoit(BrmKeys.DBROOTSTATUS_SYSVKEY, "DBROOT_STAT"); + case 'c': cFlg = true; break; + + case 'v': vFlg = true; break; + + case 'n': nFlg = true; break; + + case 'h': + default: + usage(); + return (c == 'h' ? 0 : 1); + break; } - shmDoit(BrmKeys.DECOMSVRMUTEX_SYSVKEY, "DCMSVRMUTEX"); + if (nFlg) + vFlg = true; - semDoit(BrmKeys.KEYRANGE_CL_BASE, "COPYLOCK "); - semDoit(BrmKeys.KEYRANGE_EXTENTMAP_BASE, "EXTMAP "); - semDoit(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE"); - semDoit(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM "); - semDoit(BrmKeys.KEYRANGE_VSS_BASE, "VSS "); - semDoit(BrmKeys.MST_SYSVKEY, "MST "); + ShmKeys BrmKeys; - if (!cFlg) - { - semDoit(BrmKeys.PROCESSSTATUS_SYSVKEY, "PROC_STAT "); - semDoit(BrmKeys.SYSTEMSTATUS_SYSVKEY, "SYS_STAT "); - semDoit(BrmKeys.SWITCHSTATUS_SYSVKEY, "SW_STAT "); - semDoit(BrmKeys.NICSTATUS_SYSVKEY, "NIC_STAT "); - shmDoit(BrmKeys.DBROOTSTATUS_SYSVKEY, "DBROOT_STAT"); - } + boost::thread_group tg; + boost::thread* tp = 0; + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_CL_BASE, "COPYLOCK ")); + tg.add_thread(tp); + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EXTENTMAP_BASE, "EXTMAP ")); + tg.add_thread(tp); + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE")); + tg.add_thread(tp); + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM ")); + tg.add_thread(tp); + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VSS_BASE, "VSS ")); + tg.add_thread(tp); + tg.join_all(); - return 0; + shmDoit(BrmKeys.MST_SYSVKEY, "MST "); + + if (!cFlg) + { + shmDoit(BrmKeys.PROCESSSTATUS_SYSVKEY, "PROC_STAT "); + shmDoit(BrmKeys.SYSTEMSTATUS_SYSVKEY, "SYS_STAT "); + shmDoit(BrmKeys.SWITCHSTATUS_SYSVKEY, "SW_STAT "); + shmDoit(BrmKeys.NICSTATUS_SYSVKEY, "NIC_STAT "); + shmDoit(BrmKeys.DBROOTSTATUS_SYSVKEY, "DBROOT_STAT"); + } + + shmDoit(BrmKeys.DECOMSVRMUTEX_SYSVKEY, "DCMSVRMUTEX"); + + semDoit(BrmKeys.KEYRANGE_CL_BASE, "COPYLOCK "); + semDoit(BrmKeys.KEYRANGE_EXTENTMAP_BASE, "EXTMAP "); + semDoit(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE"); + semDoit(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM "); + semDoit(BrmKeys.KEYRANGE_VSS_BASE, "VSS "); + semDoit(BrmKeys.MST_SYSVKEY, "MST "); + + if (!cFlg) + { + semDoit(BrmKeys.PROCESSSTATUS_SYSVKEY, "PROC_STAT "); + semDoit(BrmKeys.SYSTEMSTATUS_SYSVKEY, "SYS_STAT "); + semDoit(BrmKeys.SWITCHSTATUS_SYSVKEY, "SW_STAT "); + semDoit(BrmKeys.NICSTATUS_SYSVKEY, "NIC_STAT "); + shmDoit(BrmKeys.DBROOTSTATUS_SYSVKEY, "DBROOT_STAT"); + } + + return 0; } // vim:ts=4 sw=4: - diff --git a/tools/cleartablelock/cleartablelock.cpp b/tools/cleartablelock/cleartablelock.cpp index 3a332be22..4d3b2e0f1 100644 --- a/tools/cleartablelock/cleartablelock.cpp +++ b/tools/cleartablelock/cleartablelock.cpp @@ -16,9 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: cleartablelock.cpp 2336 2013-06-25 19:11:36Z rdempsey $ -*/ - + * $Id: cleartablelock.cpp 2336 2013-06-25 19:11:36Z rdempsey $ + */ #include #include @@ -46,167 +45,151 @@ namespace { - //------------------------------------------------------------------------------ // Print command line usage //------------------------------------------------------------------------------ void usage() { - std::cout << "Usage: cleartablelock lockID" << std::endl - << " -h to display this menu" << std::endl; + std::cout << "Usage: cleartablelock lockID" << std::endl << " -h to display this menu" << std::endl; } - + //------------------------------------------------------------------------------ // Log initial syslog message // tableName - // tableLockID - //------------------------------------------------------------------------------ -void logInitStatus( - const std::string& tableName, - uint64_t tableLockID) +void logInitStatus(const std::string& tableName, uint64_t tableLockID) { - const std::string APPLNAME("cleartablelock command"); - const int SUBSYSTEM_ID = 19; // writeengine? - const int INIT_MSG_NUM = logging::M0088; - logging::Message::Args msgArgs; - logging::Message logMsg( INIT_MSG_NUM ); - msgArgs.add( APPLNAME ); - msgArgs.add( tableName ); - msgArgs.add( tableLockID ); - logMsg.format( msgArgs ); - logging::LoggingID lid( SUBSYSTEM_ID ); - logging::MessageLog ml( lid ); - ml.logInfoMessage( logMsg ); + const std::string APPLNAME("cleartablelock command"); + const int SUBSYSTEM_ID = 19; // writeengine? + const int INIT_MSG_NUM = logging::M0088; + logging::Message::Args msgArgs; + logging::Message logMsg(INIT_MSG_NUM); + msgArgs.add(APPLNAME); + msgArgs.add(tableName); + msgArgs.add(tableLockID); + logMsg.format(msgArgs); + logging::LoggingID lid(SUBSYSTEM_ID); + logging::MessageLog ml(lid); + ml.logInfoMessage(logMsg); } - + //------------------------------------------------------------------------------ // Log final status syslog message // tableName - Name of table of interest // tableLockID - Table lock ID associated with tableName // statusMsg - Error message to be logged; if empty then success is assumed. //------------------------------------------------------------------------------ -void logFinalStatus( - const std::string& tableName, - uint64_t tableLockID, - const std::string& statusMsg ) +void logFinalStatus(const std::string& tableName, uint64_t tableLockID, const std::string& statusMsg) { - const std::string APPLNAME("cleartablelock command"); - const int SUBSYSTEM_ID = 19; // writeengine? - const int COMP_MSG_NUM = logging::M0089; - logging::Message::Args msgArgs; - logging::Message logMsg( COMP_MSG_NUM ); - msgArgs.add( APPLNAME ); - msgArgs.add( tableName ); - msgArgs.add( tableLockID ); - std::string finalStatus; + const std::string APPLNAME("cleartablelock command"); + const int SUBSYSTEM_ID = 19; // writeengine? + const int COMP_MSG_NUM = logging::M0089; + logging::Message::Args msgArgs; + logging::Message logMsg(COMP_MSG_NUM); + msgArgs.add(APPLNAME); + msgArgs.add(tableName); + msgArgs.add(tableLockID); + std::string finalStatus; - if ( statusMsg.empty() ) - { - finalStatus = "Completed successfully"; - } - else - { - finalStatus = "Encountered errors: "; - finalStatus += statusMsg; - } + if (statusMsg.empty()) + { + finalStatus = "Completed successfully"; + } + else + { + finalStatus = "Encountered errors: "; + finalStatus += statusMsg; + } - msgArgs.add( finalStatus ); - logMsg.format( msgArgs ); - logging::LoggingID lid( SUBSYSTEM_ID ); - logging::MessageLog ml( lid ); - ml.logInfoMessage( logMsg ); + msgArgs.add(finalStatus); + logMsg.format(msgArgs); + logging::LoggingID lid(SUBSYSTEM_ID); + logging::MessageLog ml(lid); + ml.logInfoMessage(logMsg); } - + //------------------------------------------------------------------------------ // Construct list of all PMs. // pmList - (output) List of "all" PMs where msgs are to be sent. //------------------------------------------------------------------------------ -int constructPMList( std::vector& pmList ) +int constructPMList(std::vector& pmList) { - try - { - // Get OAM information - oam::Oam oam; - oam::ModuleTypeConfig modTypeConfig; - oam.getSystemConfig("pm", modTypeConfig); + try + { + // Get OAM information + oam::Oam oam; + oam::ModuleTypeConfig modTypeConfig; + oam.getSystemConfig("pm", modTypeConfig); - unsigned int pmCount = modTypeConfig.ModuleCount; + unsigned int pmCount = modTypeConfig.ModuleCount; - for (unsigned i = 0; i < pmCount; i++) - { - std::string devName = modTypeConfig.ModuleNetworkList[i].DeviceName; - int moduleID = ::atoi( - devName.substr(oam::MAX_MODULE_TYPE_SIZE, - oam::MAX_MODULE_ID_SIZE).c_str() ); - pmList.push_back( moduleID ); - } - } - catch (std::exception& ex) + for (unsigned i = 0; i < pmCount; i++) { - std::cout << "Error getting PM systemConfig. " << ex.what() << - std::endl; - return 4; - } - catch (...) - { - std::cout << "Error getting PM systemConfig. Unknown exception." << - std::endl; - return 4; + std::string devName = modTypeConfig.ModuleNetworkList[i].DeviceName; + int moduleID = ::atoi(devName.substr(oam::MAX_MODULE_TYPE_SIZE, oam::MAX_MODULE_ID_SIZE).c_str()); + pmList.push_back(moduleID); } + } + catch (std::exception& ex) + { + std::cout << "Error getting PM systemConfig. " << ex.what() << std::endl; + return 4; + } + catch (...) + { + std::cout << "Error getting PM systemConfig. Unknown exception." << std::endl; + return 4; + } - return 0; + return 0; } - + //------------------------------------------------------------------------------ // Construct partial PM list based on specified list of DBRoots. // dbRootList - List of applicable DBRoots with PMs to be notified. // pmList - (output) List of PMs to receive msgs, based on dbRootList. //------------------------------------------------------------------------------ -int constructPMList( const std::vector& dbRootList, - std::vector& pmList ) +int constructPMList(const std::vector& dbRootList, std::vector& pmList) { - uint32_t dbRoot = 0; + uint32_t dbRoot = 0; - try - { - // Get OAM information - oam::Oam oam; - std::set pmSet; // used to filter out duplicates - int pm; + try + { + // Get OAM information + oam::Oam oam; + std::set pmSet; // used to filter out duplicates + int pm; - for (unsigned j = 0; j < dbRootList.size(); j++) - { - dbRoot = dbRootList[j]; - oam.getDbrootPmConfig( dbRootList[j], pm ); - pmSet.insert( pm ); - } - - // Store unique set of PM IDs into output vector - for (std::set::const_iterator iter = pmSet.begin(); - iter != pmSet.end(); - ++iter) - { - pmList.push_back( *iter ); - } - } - catch (std::exception& ex) + for (unsigned j = 0; j < dbRootList.size(); j++) { - std::cout << "Error mapping DBRoot " << dbRoot << " to a PM. " << - ex.what() << ". Verify that this DBRoot is in Columnstore.xml." << - std::endl; - return 5; - } - catch (...) - { - std::cout << "Error mapping DBRoot " << dbRoot << " to a PM. " << - "Unknown exception. Verify that this DBRoot is in Columnstore.xml" << - std::endl; - return 5; + dbRoot = dbRootList[j]; + oam.getDbrootPmConfig(dbRootList[j], pm); + pmSet.insert(pm); } - return 0; + // Store unique set of PM IDs into output vector + for (std::set::const_iterator iter = pmSet.begin(); iter != pmSet.end(); ++iter) + { + pmList.push_back(*iter); + } + } + catch (std::exception& ex) + { + std::cout << "Error mapping DBRoot " << dbRoot << " to a PM. " << ex.what() + << ". Verify that this DBRoot is in Columnstore.xml." << std::endl; + return 5; + } + catch (...) + { + std::cout << "Error mapping DBRoot " << dbRoot << " to a PM. " + << "Unknown exception. Verify that this DBRoot is in Columnstore.xml" << std::endl; + return 5; + } + + return 0; } - + //------------------------------------------------------------------------------ // Create MessageQueueClient objects // pmList - (input) List of PMs where msgs are to be sent. @@ -214,51 +197,46 @@ int constructPMList( const std::vector& dbRootList, // errMsg - (output) Any applicable error message if nonzero return // code is returned. //------------------------------------------------------------------------------ -int createMsgQueClts( - const std::vector& pmList, - std::vector >& - msgQueClts, - std::string& errMsg) +int createMsgQueClts(const std::vector& pmList, + std::vector >& msgQueClts, + std::string& errMsg) { - errMsg.clear(); + errMsg.clear(); - std::string srvName; + std::string srvName; - try + try + { + // Send bulk rollback request to each specified PM. + for (unsigned k = 0; k < pmList.size(); k++) { - // Send bulk rollback request to each specified PM. - for (unsigned k = 0; k < pmList.size(); k++) - { - std::ostringstream weServerName; - weServerName << "pm" << pmList[k] << "_WriteEngineServer"; - srvName = weServerName.str(); - //std::cout << "Connecting to " << srvName << std::endl; + std::ostringstream weServerName; + weServerName << "pm" << pmList[k] << "_WriteEngineServer"; + srvName = weServerName.str(); + // std::cout << "Connecting to " << srvName << std::endl; - messageqcpp::MessageQueueClient* cl = - new messageqcpp::MessageQueueClient( srvName ); - msgQueClts.push_back( - boost::shared_ptr(cl) ); - } - } - catch (std::exception& ex) - { - std::ostringstream ossStatus; - ossStatus << "Error creating connection to " << - srvName << ": " << ex.what(); - errMsg = ossStatus.str(); - return 3; - } - catch (...) - { - std::ostringstream ossStatus; - ossStatus << "Error creating connection to " << srvName; - errMsg = ossStatus.str(); - return 3; + messageqcpp::MessageQueueClient* cl = new messageqcpp::MessageQueueClient(srvName); + msgQueClts.push_back(boost::shared_ptr(cl)); } + } + catch (std::exception& ex) + { + std::ostringstream ossStatus; + ossStatus << "Error creating connection to " << srvName << ": " << ex.what(); + errMsg = ossStatus.str(); + return 3; + } + catch (...) + { + std::ostringstream ossStatus; + ossStatus << "Error creating connection to " << srvName; + errMsg = ossStatus.str(); + return 3; + } - return 0; + return 0; } - + //------------------------------------------------------------------------------ // Send Request to each PM and Wait for corresponding Responses // msgQueClts - MessageQueueClients with connections to be used in sending @@ -271,64 +249,57 @@ int createMsgQueClts( // errMsg - (output) Any applicable error message if nonzero return // code is returned. //------------------------------------------------------------------------------ -int sendRequest( - const std::vector >& - msgQueClts, - const std::vector& pmList, - BRM::DBRM* brm, - const BRM::TableLockInfo& tInfo, - const std::string& tblName, - ClearTableLockThread::CLRTBLLOCK_MSGTYPE msgType, - std::string& errMsg) +int sendRequest(const std::vector >& msgQueClts, + const std::vector& pmList, BRM::DBRM* brm, const BRM::TableLockInfo& tInfo, + const std::string& tblName, ClearTableLockThread::CLRTBLLOCK_MSGTYPE msgType, + std::string& errMsg) { - unsigned int pmCount = msgQueClts.size(); - boost::thread_group tg; - std::vector statusVec; + unsigned int pmCount = msgQueClts.size(); + boost::thread_group tg; + std::vector statusVec; - // Send request to each PM. - for (unsigned i = 0; i < pmCount; i++) + // Send request to each PM. + for (unsigned i = 0; i < pmCount; i++) + { + int moduleID = pmList[i]; + + ClearTableLockStatus* pStatus = new ClearTableLockStatus(moduleID); + statusVec.push_back(pStatus); + + tg.create_thread(ClearTableLockThread(brm, msgQueClts[i].get(), tInfo, tblName, msgType, pStatus)); + } + + // Wait for all the requests to complete + tg.join_all(); + + // Determine the completion status of each request + int rc = 0; + std::ostringstream ossStatus; + + for (unsigned i = 0; i < statusVec.size(); i++) + { + int retStatus = statusVec[i]->retStatus(); + + if (retStatus != 0) { - int moduleID = pmList[i]; + rc = retStatus; - ClearTableLockStatus* pStatus = new ClearTableLockStatus(moduleID); - statusVec.push_back( pStatus ); - - tg.create_thread( ClearTableLockThread( - brm, msgQueClts[i].get(), tInfo, tblName, msgType, - pStatus )); + if (!statusVec[i]->retMsg().empty()) + { + ossStatus << "From PM" << statusVec[i]->moduleID() << ": " << statusVec[i]->retMsg() << ". " + << std::endl; + } } - // Wait for all the requests to complete - tg.join_all(); + delete statusVec[i]; + } - // Determine the completion status of each request - int rc = 0; - std::ostringstream ossStatus; + if (!(ossStatus.str().empty())) + errMsg = ossStatus.str(); - for (unsigned i = 0; i < statusVec.size(); i++) - { - int retStatus = statusVec[i]->retStatus(); - - if (retStatus != 0) - { - rc = retStatus; - - if (!statusVec[i]->retMsg().empty()) - { - ossStatus << "From PM" << statusVec[i]->moduleID() << - ": " << statusVec[i]->retMsg() << ". " << std::endl; - } - } - - delete statusVec[i]; - } - - if (!(ossStatus.str().empty())) - errMsg = ossStatus.str(); - - return rc; + return rc; } - + //------------------------------------------------------------------------------ // We send the bulk rollback request to all the PMs and not just the ones // referenced in the table lock, because there is always the possiblity that @@ -345,66 +316,57 @@ int sendRequest( // errMsg - (output) Any applicable error message if nonzero return // code is returned. //------------------------------------------------------------------------------ -int execBulkRollbackReq( - const std::vector >& - msgQueClts, - const std::vector& pmList, - BRM::DBRM* brm, - const BRM::TableLockInfo& tInfo, - const std::string& tblName, - bool rollbackOnly, - std::string& errMsg) +int execBulkRollbackReq(const std::vector >& msgQueClts, + const std::vector& pmList, BRM::DBRM* brm, const BRM::TableLockInfo& tInfo, + const std::string& tblName, bool rollbackOnly, std::string& errMsg) { - errMsg.clear(); + errMsg.clear(); - try + try + { + if (!rollbackOnly) { - if (!rollbackOnly) - { - // Take over ownership of stale lock. - std::string processName("clearTableLock"); - uint32_t processID = ::getpid(); - int32_t sessionID = -1; - int32_t transID = -1; - bool ownerChanged = brm->changeOwner( - tInfo.id, processName, processID, sessionID, transID); + // Take over ownership of stale lock. + std::string processName("clearTableLock"); + uint32_t processID = ::getpid(); + int32_t sessionID = -1; + int32_t transID = -1; + bool ownerChanged = brm->changeOwner(tInfo.id, processName, processID, sessionID, transID); - if (!ownerChanged) - { - errMsg = "Unable to grab lock; Lock not found or still in use."; - return 2; - } - } - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Error changing table lock owner; " << ex.what(); - errMsg = oss.str(); - return 2; - } - catch (...) - { - std::ostringstream oss; - oss << "Unknown error changing table lock owner;"; - errMsg = oss.str(); + if (!ownerChanged) + { + errMsg = "Unable to grab lock; Lock not found or still in use."; return 2; + } } + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error changing table lock owner; " << ex.what(); + errMsg = oss.str(); + return 2; + } + catch (...) + { + std::ostringstream oss; + oss << "Unknown error changing table lock owner;"; + errMsg = oss.str(); + return 2; + } - // Perform bulk rollback if state is in LOADING state - int rc = 0; + // Perform bulk rollback if state is in LOADING state + int rc = 0; - if (tInfo.state == BRM::LOADING) - { - rc = sendRequest( msgQueClts, pmList, - brm, tInfo, tblName, - ClearTableLockThread::CLRTBLLOCK_MSGTYPE_ROLLBACK, - errMsg ); - } + if (tInfo.state == BRM::LOADING) + { + rc = sendRequest(msgQueClts, pmList, brm, tInfo, tblName, + ClearTableLockThread::CLRTBLLOCK_MSGTYPE_ROLLBACK, errMsg); + } - return rc; + return rc; } - + //------------------------------------------------------------------------------ // We send the file cleanup request to all the PMs and not just the ones // referenced in the table lock, because there is always the possiblity that @@ -420,349 +382,333 @@ int execBulkRollbackReq( // errMsg - (output) Any applicable error message if nonzero return // code is returned. //------------------------------------------------------------------------------ -int execFileCleanupReq( - const std::vector >& - msgQueClts, - const std::vector& pmList, - BRM::DBRM* brm, - const BRM::TableLockInfo& tInfo, - const std::string& tblName, - std::string& errMsg) +int execFileCleanupReq(const std::vector >& msgQueClts, + const std::vector& pmList, BRM::DBRM* brm, const BRM::TableLockInfo& tInfo, + const std::string& tblName, std::string& errMsg) { - errMsg.clear(); + errMsg.clear(); - int rc = sendRequest( msgQueClts, pmList, - brm, tInfo, tblName, - ClearTableLockThread::CLRTBLLOCK_MSGTYPE_CLEANUP, - errMsg ); + int rc = sendRequest(msgQueClts, pmList, brm, tInfo, tblName, + ClearTableLockThread::CLRTBLLOCK_MSGTYPE_CLEANUP, errMsg); - return rc; + return rc; } -} - +} // namespace + //------------------------------------------------------------------------------ // Main entry point to this program //------------------------------------------------------------------------------ int main(int argc, char** argv) { - int c; - bool clearLockOnly = false; - bool rollbackOnly = false; - uint32_t tableOID = 0; + int c; + bool clearLockOnly = false; + bool rollbackOnly = false; + uint32_t tableOID = 0; - while ((c = getopt(argc, argv, "hlr:")) != EOF) + while ((c = getopt(argc, argv, "hlr:")) != EOF) + { + switch (c) { - switch (c) - { - case 'l': - clearLockOnly = true; - break; -//Only allow '-r' option for development/debugging + case 'l': + clearLockOnly = true; + break; +// Only allow '-r' option for development/debugging #if 1 - case 'r': // hidden option to rollback specified table w/o a lock - rollbackOnly = true; - tableOID = ::strtoull(optarg, 0, 10); - break; + case 'r': // hidden option to rollback specified table w/o a lock + rollbackOnly = true; + tableOID = ::strtoull(optarg, 0, 10); + break; #endif - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - } - - if ((argc - optind) != 1 ) - { + case 'h': + case '?': + default: usage(); + return (c == 'h' ? 0 : 1); + break; + } + } + + if ((argc - optind) != 1) + { + usage(); + return 1; + } + + // Get the table lock ID specified by the user + uint64_t lockID = ::strtoull(argv[optind], 0, 10); + + // If user specified both clearlock and rollback then we need to do both + if (clearLockOnly && rollbackOnly) + { + clearLockOnly = false; + rollbackOnly = false; + } + + //-------------------------------------------------------------------------- + // Verify that BRM is up and in a read/write state + //-------------------------------------------------------------------------- + BRM::DBRM brm; + int brmRc = brm.isReadWrite(); + + if (brmRc != BRM::ERR_OK) + { + std::string brmErrMsg; + BRM::errString(brmRc, brmErrMsg); + std::cout << "BRM error: " << brmErrMsg << std::endl; + std::cout << "Table lock " << lockID << " is not cleared." << std::endl; + return 1; + } + + if (brm.getSystemReady() < 1) + { + std::cout << "System is not ready. Verify that ColumnStore " + "is up and ready before running this program" + << std::endl; + return 1; + } + + //-------------------------------------------------------------------------- + //@Bug 3711 Check whether the table is locked; does the table lock exist. + //-------------------------------------------------------------------------- + execplan::CalpontSystemCatalog::TableName tblName; + BRM::TableLockInfo tInfo; + std::string task; + std::vector pmList; + int rc; + const std::string taskSysCat("getting system catalog information"); + + try + { + if (rollbackOnly) + { + tInfo.id = lockID; + tInfo.tableOID = tableOID; + tInfo.ownerPID = 0; + tInfo.ownerSessionID = 1; + tInfo.state = BRM::LOADING; + + // Construct PM list using all PMs, since we don't have a table + // lock with the list of applicable DBRoots. + task = "constructing total PM list"; + rc = constructPMList(pmList); + + if (rc != 0) + { return 1; + } + } + else + { + task = "getting table lock"; + bool lockExists = brm.getTableLockInfo(lockID, &tInfo); + + if (!lockExists) + { + std::cout << "Table lock " << lockID << " does not exist." << std::endl; + return 1; + } + + // Construct PM list based on list of affected DBRoots + task = "mapping DBRoots to PMs"; + rc = constructPMList(tInfo.dbrootList, pmList); + + if (rc != 0) + { + return 1; + } } - // Get the table lock ID specified by the user - uint64_t lockID = ::strtoull(argv[optind], 0, 10); + uint32_t sessionID = 1; + task = taskSysCat; + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); + tblName = systemCatalogPtr->tableName(tInfo.tableOID); + } + catch (std::exception& ex) + { + std::cout << "Error " << task << ". " << ex.what() << std::endl; - // If user specified both clearlock and rollback then we need to do both - if (clearLockOnly && rollbackOnly) + if (clearLockOnly && (task == taskSysCat)) { - clearLockOnly = false; - rollbackOnly = false; + tblName.schema = "[unknown name]"; + tblName.table.clear(); + std::cout << "Will still try to clear table lock." << std::endl; + } + else + { + std::cout << "Table lock " << lockID << " is not cleared." << std::endl; + return 1; + } + } + catch (...) + { + std::cout << "Error " << task << ". " << std::endl; + + if (clearLockOnly && (task == taskSysCat)) + { + tblName.schema = "[unknown name]"; + tblName.table.clear(); + std::cout << "Will still try to clear table lock." << std::endl; + } + else + { + std::cout << "Table lock " << lockID << " is not cleared." << std::endl; + return 1; + } + } + + logInitStatus(tblName.toString(), lockID); + + if (rollbackOnly) + { + std::cout << "Rolling back table " << tblName.toString() << std::endl << std::endl; + } + else if (clearLockOnly) + { + std::cout << "Clearing table lock " << lockID << " for table " << tblName.toString() << std::endl + << std::endl; + } + else + { + std::cout << "Rolling back and clearing table lock for table " << tblName.toString() << "; table lock " + << lockID << std::endl + << std::endl; + } + + //-------------------------------------------------------------------------- + // Perform bulk rollback + //-------------------------------------------------------------------------- + std::string errMsg; + + if (!clearLockOnly) + { + std::vector > msgQueClts; + rc = createMsgQueClts(pmList, msgQueClts, errMsg); + + if (rc != 0) + { + logFinalStatus(tblName.toString(), lockID, errMsg); + std::cout << errMsg << std::endl; + std::cout << "Table lock " << lockID << " for table " << tblName.toString() << " is not cleared." + << std::endl; + return rc; } - //-------------------------------------------------------------------------- - // Verify that BRM is up and in a read/write state - //-------------------------------------------------------------------------- - BRM::DBRM brm; - int brmRc = brm.isReadWrite(); + rc = execBulkRollbackReq(msgQueClts, pmList, &brm, tInfo, tblName.toString(), rollbackOnly, errMsg); - if (brmRc != BRM::ERR_OK) + BRM::TxnID txn; + txn.id = tInfo.ownerTxnID; + txn.valid = true; + brm.rolledback(txn); + + if (rc != 0) { - std::string brmErrMsg; - BRM::errString(brmRc, brmErrMsg); - std::cout << "BRM error: " << brmErrMsg << std::endl; + logFinalStatus(tblName.toString(), lockID, errMsg); + std::cout << "Rollback error: " << errMsg << std::endl; + std::cout << "Table lock " << lockID << " for table " << tblName.toString() << " is not cleared." + << std::endl; + return rc; + } + + //---------------------------------------------------------------------- + // Change the state of the table lock to cleanup state. + // We ignore the return stateChange flag. + //---------------------------------------------------------------------- + if (!rollbackOnly) + { + try + { + brm.changeState(tInfo.id, BRM::CLEANUP); + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error changing state. " << ex.what(); + logFinalStatus(tblName.toString(), lockID, oss.str()); + std::cout << oss.str() << std::endl; std::cout << "Table lock " << lockID << " is not cleared." << std::endl; return 1; - } - - if (brm.getSystemReady() < 1) - { - std::cout << "System is not ready. Verify that ColumnStore " - "is up and ready before running this program" << std::endl; + } + catch (...) + { + std::ostringstream oss; + oss << "Error changing state. " << std::endl; + logFinalStatus(tblName.toString(), lockID, oss.str()); + std::cout << oss.str() << std::endl; + std::cout << "Table lock " << lockID << " is not cleared." << std::endl; return 1; + } } - //-------------------------------------------------------------------------- - //@Bug 3711 Check whether the table is locked; does the table lock exist. - //-------------------------------------------------------------------------- - execplan::CalpontSystemCatalog::TableName tblName; - BRM::TableLockInfo tInfo; - std::string task; - std::vector pmList; - int rc; - const std::string taskSysCat("getting system catalog information"); + //---------------------------------------------------------------------- + // Delete the meta data files + //---------------------------------------------------------------------- + rc = execFileCleanupReq(msgQueClts, pmList, &brm, tInfo, tblName.toString(), errMsg); + if (rc != 0) + { + //@Bug 4517. Release tablelock if remove meta files failed + const std::string APPLNAME("cleartablelock command"); + const int SUBSYSTEM_ID = 19; // writeengine? + const int COMP_MSG_NUM = logging::M0089; + logging::Message::Args msgArgs; + logging::Message logMsg(COMP_MSG_NUM); + msgArgs.add(APPLNAME); + msgArgs.add(tblName.toString()); + msgArgs.add(lockID); + msgArgs.add(errMsg); + logMsg.format(msgArgs); + logging::LoggingID lid(SUBSYSTEM_ID); + logging::MessageLog ml(lid); + ml.logWarningMessage(logMsg); + + std::cout << "File cleanup error: " << errMsg << std::endl; + } + } // end of: if (!clearLockOnly) + + //-------------------------------------------------------------------------- + // Finally, release the actual table lock + //-------------------------------------------------------------------------- + std::cout << std::endl; + + if (rollbackOnly) + { + logFinalStatus(tblName.toString(), lockID, std::string()); + std::cout << "Bulk Rollback Only for table " << tblName.toString() << " completed successfully." + << std::endl; + } + else + { try { - if (rollbackOnly) - { - tInfo.id = lockID; - tInfo.tableOID = tableOID; - tInfo.ownerPID = 0; - tInfo.ownerSessionID = 1; - tInfo.state = BRM::LOADING; - - // Construct PM list using all PMs, since we don't have a table - // lock with the list of applicable DBRoots. - task = "constructing total PM list"; - rc = constructPMList( pmList ); - - if (rc != 0) - { - return 1; - } - } - else - { - task = "getting table lock"; - bool lockExists = brm.getTableLockInfo( lockID, &tInfo ); - - if (!lockExists) - { - std::cout << "Table lock " << lockID << " does not exist." << - std::endl; - return 1; - } - - // Construct PM list based on list of affected DBRoots - task = "mapping DBRoots to PMs"; - rc = constructPMList( tInfo.dbrootList, pmList ); - - if (rc != 0) - { - return 1; - } - } - - uint32_t sessionID = 1; - task = taskSysCat; - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - sessionID ); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); - tblName = systemCatalogPtr->tableName( tInfo.tableOID ); + brm.releaseTableLock(lockID); // ignore boolean return value } catch (std::exception& ex) { - std::cout << "Error " << task << ". " << ex.what() << std::endl; - - if (clearLockOnly && (task == taskSysCat)) - { - tblName.schema = "[unknown name]"; - tblName.table.clear(); - std::cout << "Will still try to clear table lock." << std::endl; - } - else - { - std::cout << "Table lock " << lockID << " is not cleared." << - std::endl; - return 1; - } + logFinalStatus(tblName.toString(), lockID, ex.what()); + std::cout << "Error releasing table lock " << lockID << " for table " << tblName.toString() + << std::endl; + std::cout << ex.what() << std::endl; + return 1; } catch (...) { - std::cout << "Error " << task << ". " << std::endl; - - if (clearLockOnly && (task == taskSysCat)) - { - tblName.schema = "[unknown name]"; - tblName.table.clear(); - std::cout << "Will still try to clear table lock." << std::endl; - } - else - { - std::cout << "Table lock " << lockID << " is not cleared." << - std::endl; - return 1; - } + std::string unknownErr("Unknown exception"); + logFinalStatus(tblName.toString(), lockID, unknownErr); + std::cout << "Error releasing table lock " << lockID << " for table " << tblName.toString() + << std::endl; + std::cout << unknownErr << std::endl; + return 1; } - logInitStatus( tblName.toString(), lockID ); + logFinalStatus(tblName.toString(), lockID, std::string()); + std::cout << "Table lock " << lockID << " for table " << tblName.toString() << " is cleared." + << std::endl; + } - if (rollbackOnly) - { - std::cout << "Rolling back table " << tblName.toString() << - std::endl << std::endl; - } - else if (clearLockOnly) - { - std::cout << "Clearing table lock " << lockID << - " for table " << tblName.toString() << std::endl << std::endl; - } - else - { - std::cout << "Rolling back and clearing table lock for table " << - tblName.toString() << "; table lock " << lockID << - std::endl << std::endl; - } - - //-------------------------------------------------------------------------- - // Perform bulk rollback - //-------------------------------------------------------------------------- - std::string errMsg; - - if (!clearLockOnly) - { - std::vector > - msgQueClts; - rc = createMsgQueClts(pmList, msgQueClts, errMsg); - - if (rc != 0) - { - logFinalStatus( tblName.toString(), lockID, errMsg ); - std::cout << errMsg << std::endl; - std::cout << "Table lock " << lockID << " for table " << - tblName.toString() << " is not cleared." << std::endl; - return rc; - } - - rc = execBulkRollbackReq( msgQueClts, pmList, - &brm, tInfo, tblName.toString(), rollbackOnly, errMsg ); - - BRM::TxnID txn; - txn.id = tInfo.ownerTxnID; - txn.valid = true; - brm.rolledback(txn); - - if (rc != 0) - { - logFinalStatus( tblName.toString(), lockID, errMsg ); - std::cout << "Rollback error: " << errMsg << std::endl; - std::cout << "Table lock " << lockID << " for table " << - tblName.toString() << " is not cleared." << std::endl; - return rc; - } - - //---------------------------------------------------------------------- - // Change the state of the table lock to cleanup state. - // We ignore the return stateChange flag. - //---------------------------------------------------------------------- - if (!rollbackOnly) - { - try - { - brm.changeState( tInfo.id, BRM::CLEANUP ); - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Error changing state. " << ex.what(); - logFinalStatus( tblName.toString(), lockID, oss.str() ); - std::cout << oss.str() << std::endl; - std::cout << "Table lock " << lockID << " is not cleared." << - std::endl; - return 1; - } - catch (...) - { - std::ostringstream oss; - oss << "Error changing state. " << std::endl; - logFinalStatus( tblName.toString(), lockID, oss.str() ); - std::cout << oss.str() << std::endl; - std::cout << "Table lock " << lockID << " is not cleared." << - std::endl; - return 1; - } - } - - //---------------------------------------------------------------------- - // Delete the meta data files - //---------------------------------------------------------------------- - rc = execFileCleanupReq( msgQueClts, pmList, - &brm, tInfo, tblName.toString(), errMsg ); - - if (rc != 0) - { - //@Bug 4517. Release tablelock if remove meta files failed - const std::string APPLNAME("cleartablelock command"); - const int SUBSYSTEM_ID = 19; // writeengine? - const int COMP_MSG_NUM = logging::M0089; - logging::Message::Args msgArgs; - logging::Message logMsg( COMP_MSG_NUM ); - msgArgs.add( APPLNAME ); - msgArgs.add( tblName.toString() ); - msgArgs.add( lockID ); - msgArgs.add( errMsg ); - logMsg.format( msgArgs ); - logging::LoggingID lid( SUBSYSTEM_ID ); - logging::MessageLog ml( lid ); - ml.logWarningMessage( logMsg ); - - std::cout << "File cleanup error: " << errMsg << std::endl; - } - } // end of: if (!clearLockOnly) - - //-------------------------------------------------------------------------- - // Finally, release the actual table lock - //-------------------------------------------------------------------------- - std::cout << std::endl; - - if (rollbackOnly) - { - logFinalStatus( tblName.toString(), lockID, std::string() ); - std::cout << "Bulk Rollback Only for table " << tblName.toString() << - " completed successfully." << std::endl; - } - else - { - try - { - brm.releaseTableLock( lockID ); // ignore boolean return value - } - catch (std::exception& ex) - { - logFinalStatus( tblName.toString(), lockID, ex.what() ); - std::cout << "Error releasing table lock " << lockID << - " for table " << tblName.toString() << std::endl; - std::cout << ex.what() << std::endl; - return 1; - } - catch (...) - { - std::string unknownErr("Unknown exception"); - logFinalStatus( tblName.toString(), lockID, unknownErr ); - std::cout << "Error releasing table lock " << lockID << - " for table " << tblName.toString() << std::endl; - std::cout << unknownErr << std::endl; - return 1; - } - - logFinalStatus( tblName.toString(), lockID, std::string() ); - std::cout << "Table lock " << lockID << " for table " << - tblName.toString() << " is cleared." << std::endl; - } - - return 0; + return 0; } diff --git a/tools/cleartablelock/cleartablelockthread.cpp b/tools/cleartablelock/cleartablelockthread.cpp index cf58c11f5..fbafad1d6 100644 --- a/tools/cleartablelock/cleartablelockthread.cpp +++ b/tools/cleartablelock/cleartablelockthread.cpp @@ -28,190 +28,174 @@ #include /*static*/ boost::mutex ClearTableLockThread::fStdOutLock; - + //------------------------------------------------------------------------------ // ClearTableLockThread constructor //------------------------------------------------------------------------------ -ClearTableLockThread::ClearTableLockThread( - BRM::DBRM* brm, - messageqcpp::MessageQueueClient* clt, - const BRM::TableLockInfo& tInfo, - const std::string& tblName, - CLRTBLLOCK_MSGTYPE msgType, - ClearTableLockStatus* pStatus) : - fTableLockInfo(tInfo), - fBrm(brm), - fClt(clt), - fTblName(tblName), - fMsgType(msgType), - fStatus(pStatus) +ClearTableLockThread::ClearTableLockThread(BRM::DBRM* brm, messageqcpp::MessageQueueClient* clt, + const BRM::TableLockInfo& tInfo, const std::string& tblName, + CLRTBLLOCK_MSGTYPE msgType, ClearTableLockStatus* pStatus) + : fTableLockInfo(tInfo), fBrm(brm), fClt(clt), fTblName(tblName), fMsgType(msgType), fStatus(pStatus) { } - + //------------------------------------------------------------------------------ // Main entry point for a ClearTableLockThread object, used to forward a // cleartablelock tool command to the WriteEngineServer specified to the ctor. //------------------------------------------------------------------------------ -void ClearTableLockThread::operator() () +void ClearTableLockThread::operator()() { - try - { - if (fMsgType == CLRTBLLOCK_MSGTYPE_ROLLBACK) - executeRollback( ); - else if (fMsgType == CLRTBLLOCK_MSGTYPE_CLEANUP) - executeFileCleanup( ); - } - catch (std::exception& ex) - { - setStatus( 101, ex.what() ); - } - catch (...) - { - std::string errMsg("Unknown exception."); - setStatus( 102, errMsg ); - } + try + { + if (fMsgType == CLRTBLLOCK_MSGTYPE_ROLLBACK) + executeRollback(); + else if (fMsgType == CLRTBLLOCK_MSGTYPE_CLEANUP) + executeFileCleanup(); + } + catch (std::exception& ex) + { + setStatus(101, ex.what()); + } + catch (...) + { + std::string errMsg("Unknown exception."); + setStatus(102, errMsg); + } } - + //------------------------------------------------------------------------------ // Process bulk rollback portion of cleartablelock command. //------------------------------------------------------------------------------ -void ClearTableLockThread::executeRollback( ) +void ClearTableLockThread::executeRollback() { - boost::shared_ptr bsIn; - messageqcpp::ByteStream bsOut; + boost::shared_ptr bsIn; + messageqcpp::ByteStream bsOut; - const std::string APPLNAME("cleartablelock command"); + const std::string APPLNAME("cleartablelock command"); + + //-------------------------------------------------------------------------- + // Send rollback msg to the writeengine server connected to fClt. + //-------------------------------------------------------------------------- + { + boost::mutex::scoped_lock lk(fStdOutLock); + std::cout << "Sending rollback request to PM" << fStatus->moduleID() << "..." << std::endl; + // std::cout << "cleartablelock rollback: tableLock-" <write(bsOut); + + // Wait for the response, and check for any errors + std::string rollbackErrMsg; + bsIn.reset(new messageqcpp::ByteStream()); + bsIn = fClt->read(); + + if (bsIn->length() == 0) + { + std::string errMsg("Network error, PM rollback"); + setStatus(103, errMsg); + + boost::mutex::scoped_lock lk(fStdOutLock); + std::cout << "No response from PM" << fStatus->moduleID() << std::endl; + return; + } + else + { + messageqcpp::ByteStream::byte rc; + *bsIn >> rc; + *bsIn >> rollbackErrMsg; - //-------------------------------------------------------------------------- - // Send rollback msg to the writeengine server connected to fClt. - //-------------------------------------------------------------------------- { - boost::mutex::scoped_lock lk(fStdOutLock); - std::cout << "Sending rollback request to PM" << - fStatus->moduleID() << "..." << std::endl; -// std::cout << "cleartablelock rollback: tableLock-" <moduleID() << std::endl; + else + std::cout << "Unsuccessful rollback response from PM" << fStatus->moduleID() << "; " << rollbackErrMsg + << std::endl; + + // std::cout << "cleartablelock rollback response; rc-" << (int)rc << + // "; retMsg: <" << rollbackErrMsg << '>' << std::endl; } - bsOut << (messageqcpp::ByteStream::byte) - WriteEngine::WE_CLT_SRV_CLEAR_TABLE_LOCK; - bsOut << fTableLockInfo.id; - bsOut << fTableLockInfo.tableOID; - bsOut << fTblName; - bsOut << APPLNAME; - fClt->write(bsOut); - - // Wait for the response, and check for any errors - std::string rollbackErrMsg; - bsIn.reset(new messageqcpp::ByteStream()); - bsIn = fClt->read(); - - if (bsIn->length() == 0) + if (rc != 0) { - std::string errMsg("Network error, PM rollback"); - setStatus( 103, errMsg ); - - boost::mutex::scoped_lock lk(fStdOutLock); - std::cout << "No response from PM" << fStatus->moduleID() << std::endl; - return; - } - else - { - messageqcpp::ByteStream::byte rc; - *bsIn >> rc; - *bsIn >> rollbackErrMsg; - - { - boost::mutex::scoped_lock lk(fStdOutLock); - - if (rc == 0) - std::cout << "Successful rollback response from PM" << - fStatus->moduleID() << std::endl; - else - std::cout << "Unsuccessful rollback response from PM" << - fStatus->moduleID() << "; " << rollbackErrMsg << std::endl; - -// std::cout << "cleartablelock rollback response; rc-" << (int)rc << -// "; retMsg: <" << rollbackErrMsg << '>' << std::endl; - } - - if (rc != 0) - { - std::string errMsg("PM rollback error: "); - errMsg += rollbackErrMsg; - setStatus( 104, errMsg ); - return; - } - + std::string errMsg("PM rollback error: "); + errMsg += rollbackErrMsg; + setStatus(104, errMsg); + return; } + } } - + //------------------------------------------------------------------------------ // Process file cleanup portion of cleartablelock command. //------------------------------------------------------------------------------ -void ClearTableLockThread::executeFileCleanup( ) +void ClearTableLockThread::executeFileCleanup() { - boost::shared_ptr bsIn; - messageqcpp::ByteStream bsOut; + boost::shared_ptr bsIn; + messageqcpp::ByteStream bsOut; + + //-------------------------------------------------------------------------- + // Send cleanup msg (to delete rb files) to the we server connected to fClt. + //-------------------------------------------------------------------------- + { + boost::mutex::scoped_lock lk(fStdOutLock); + std::cout << "Sending cleanup request to PM" << fStatus->moduleID() << "..." << std::endl; + // std::cout << "cleartablelock cleanup: " << + // "oid-" << fTableLockInfo.tableOID << std::endl; + } + + bsOut << (messageqcpp::ByteStream::byte)WriteEngine::WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP; + bsOut << fTableLockInfo.tableOID; + fClt->write(bsOut); + + // Wait for the response, and check for any errors + std::string fileDeleteErrMsg; + bsIn.reset(new messageqcpp::ByteStream()); + bsIn = fClt->read(); + + if (bsIn->length() == 0) + { + std::string errMsg("Network error; PM rollback cleanup"); + setStatus(105, errMsg); + + boost::mutex::scoped_lock lk(fStdOutLock); + std::cout << "No response from PM" << fStatus->moduleID() << std::endl; + return; + } + else + { + messageqcpp::ByteStream::byte rc; + *bsIn >> rc; + *bsIn >> fileDeleteErrMsg; - //-------------------------------------------------------------------------- - // Send cleanup msg (to delete rb files) to the we server connected to fClt. - //-------------------------------------------------------------------------- { - boost::mutex::scoped_lock lk(fStdOutLock); - std::cout << "Sending cleanup request to PM" << - fStatus->moduleID() << "..." << std::endl; -// std::cout << "cleartablelock cleanup: " << -// "oid-" << fTableLockInfo.tableOID << std::endl; + boost::mutex::scoped_lock lk(fStdOutLock); + + if (rc == 0) + std::cout << "Successful cleanup response from PM" << fStatus->moduleID() << std::endl; + else + std::cout << "Unsuccessful cleanup response from PM" << fStatus->moduleID() << "; " + << fileDeleteErrMsg << std::endl; + + // std::cout << "cleartablelock cleanup response; rc-" << (int)rc << + // "; retMsg: <" << fileDeleteErrMsg << '>' << std::endl; } - bsOut << (messageqcpp::ByteStream::byte) - WriteEngine::WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP; - bsOut << fTableLockInfo.tableOID; - fClt->write(bsOut); - - // Wait for the response, and check for any errors - std::string fileDeleteErrMsg; - bsIn.reset(new messageqcpp::ByteStream()); - bsIn = fClt->read(); - - if (bsIn->length() == 0) + if (rc != 0) { - std::string errMsg("Network error; PM rollback cleanup"); - setStatus( 105, errMsg ); - - boost::mutex::scoped_lock lk(fStdOutLock); - std::cout << "No response from PM" << fStatus->moduleID() << std::endl; - return; - } - else - { - messageqcpp::ByteStream::byte rc; - *bsIn >> rc; - *bsIn >> fileDeleteErrMsg; - - { - boost::mutex::scoped_lock lk(fStdOutLock); - - if (rc == 0) - std::cout << "Successful cleanup response from PM" << - fStatus->moduleID() << std::endl; - else - std::cout << "Unsuccessful cleanup response from PM" << - fStatus->moduleID() << "; " << fileDeleteErrMsg << std::endl; - -// std::cout << "cleartablelock cleanup response; rc-" << (int)rc << -// "; retMsg: <" << fileDeleteErrMsg << '>' << std::endl; - } - - if (rc != 0) - { - std::string errMsg("PM rollback cleanup error: "); - errMsg += fileDeleteErrMsg; - setStatus( 106, errMsg ); - return; - } + std::string errMsg("PM rollback cleanup error: "); + errMsg += fileDeleteErrMsg; + setStatus(106, errMsg); + return; } + } } diff --git a/tools/cleartablelock/cleartablelockthread.h b/tools/cleartablelock/cleartablelockthread.h index 37c1b98fc..ed435f1f1 100644 --- a/tools/cleartablelock/cleartablelockthread.h +++ b/tools/cleartablelock/cleartablelockthread.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: cleartablelockthread.h 2101 2013-01-21 14:12:52Z rdempsey $ -* -*******************************************************************************/ + * $Id: cleartablelockthread.h 2101 2013-01-21 14:12:52Z rdempsey $ + * + *******************************************************************************/ #ifndef ClearTableLockThread_H__ #define ClearTableLockThread_H__ @@ -34,54 +34,55 @@ */ class ClearTableLockStatus { -public: - /** @brief ClearTableLockStatus constructor - * @param moduleID PM module ID relevant to this status object - */ - explicit ClearTableLockStatus(int moduleID) : - fModuleID(moduleID), fReturnStatus(0) { } + public: + /** @brief ClearTableLockStatus constructor + * @param moduleID PM module ID relevant to this status object + */ + explicit ClearTableLockStatus(int moduleID) : fModuleID(moduleID), fReturnStatus(0) + { + } - /** @brief Accessor to return status - */ - int retStatus() const - { - return fReturnStatus; - } + /** @brief Accessor to return status + */ + int retStatus() const + { + return fReturnStatus; + } - /** @brief Accessor to return message - */ - const std::string& retMsg() const - { - return fReturnMsg; - } + /** @brief Accessor to return message + */ + const std::string& retMsg() const + { + return fReturnMsg; + } - /** @brief Accessor to PM module ID - */ - int moduleID() const - { - return fModuleID; - } + /** @brief Accessor to PM module ID + */ + int moduleID() const + { + return fModuleID; + } - /** @brief Mutator used to set the return status - * @param stat Status to be saved - */ - void retStatus(int stat) - { - fReturnStatus = stat; - } + /** @brief Mutator used to set the return status + * @param stat Status to be saved + */ + void retStatus(int stat) + { + fReturnStatus = stat; + } - /** @brief Mutator used to set the return message - * @param msg Status message to be saved - */ - void retMsg (const std::string& msg) - { - fReturnMsg = msg; - } + /** @brief Mutator used to set the return message + * @param msg Status message to be saved + */ + void retMsg(const std::string& msg) + { + fReturnMsg = msg; + } -private: - int fModuleID; // PM module ID associated with this request - int fReturnStatus; // Return status from the PM - std::string fReturnMsg; // Return message from the PM + private: + int fModuleID; // PM module ID associated with this request + int fReturnStatus; // Return status from the PM + std::string fReturnMsg; // Return message from the PM }; //------------------------------------------------------------------------------ @@ -89,49 +90,44 @@ private: */ class ClearTableLockThread { -public: - enum CLRTBLLOCK_MSGTYPE - { - CLRTBLLOCK_MSGTYPE_ROLLBACK = 1, - CLRTBLLOCK_MSGTYPE_CLEANUP = 2 - }; + public: + enum CLRTBLLOCK_MSGTYPE + { + CLRTBLLOCK_MSGTYPE_ROLLBACK = 1, + CLRTBLLOCK_MSGTYPE_CLEANUP = 2 + }; - /** @brief ClearTableLockThread constructor - * @param brm Handle to DBRM - * @param clt MessageQueueClient used to communicate with PM - * @param tInfo Initial table lock information - * @param tblName Name of table referenced by tInfo - * @param msgType Message to process - * @param pStatus Status object used to track this bulkload rollback req - */ - ClearTableLockThread( - BRM::DBRM* brm, - messageqcpp::MessageQueueClient* clt, - const BRM::TableLockInfo& tInfo, - const std::string& tblName, - CLRTBLLOCK_MSGTYPE msgType, - ClearTableLockStatus* pStatus); + /** @brief ClearTableLockThread constructor + * @param brm Handle to DBRM + * @param clt MessageQueueClient used to communicate with PM + * @param tInfo Initial table lock information + * @param tblName Name of table referenced by tInfo + * @param msgType Message to process + * @param pStatus Status object used to track this bulkload rollback req + */ + ClearTableLockThread(BRM::DBRM* brm, messageqcpp::MessageQueueClient* clt, const BRM::TableLockInfo& tInfo, + const std::string& tblName, CLRTBLLOCK_MSGTYPE msgType, ClearTableLockStatus* pStatus); - /** @brief Entry point for thread execution - */ - void operator() (); + /** @brief Entry point for thread execution + */ + void operator()(); -private: - void executeRollback ( ); - void executeFileCleanup( ); - void setStatus(int status, const std::string& msg) - { - fStatus->retStatus( status ); - fStatus->retMsg ( msg ); - } + private: + void executeRollback(); + void executeFileCleanup(); + void setStatus(int status, const std::string& msg) + { + fStatus->retStatus(status); + fStatus->retMsg(msg); + } - BRM::TableLockInfo fTableLockInfo; // Initial table lock information - BRM::DBRM* fBrm; // Handle to DBRM - messageqcpp::MessageQueueClient* fClt;// Msg queue client to send/rcv msgs - std::string fTblName; // Name of relevant table - CLRTBLLOCK_MSGTYPE fMsgType; // Msg type to process - ClearTableLockStatus* fStatus; // Status object used to track request - static boost::mutex fStdOutLock; // Synchronize logging to stdout + BRM::TableLockInfo fTableLockInfo; // Initial table lock information + BRM::DBRM* fBrm; // Handle to DBRM + messageqcpp::MessageQueueClient* fClt; // Msg queue client to send/rcv msgs + std::string fTblName; // Name of relevant table + CLRTBLLOCK_MSGTYPE fMsgType; // Msg type to process + ClearTableLockStatus* fStatus; // Status object used to track request + static boost::mutex fStdOutLock; // Synchronize logging to stdout }; #endif diff --git a/tools/configMgt/autoConfigure.cpp b/tools/configMgt/autoConfigure.cpp index abf6c42e4..b5d7334c5 100644 --- a/tools/configMgt/autoConfigure.cpp +++ b/tools/configMgt/autoConfigure.cpp @@ -17,19 +17,18 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: configure.cpp 64 2006-10-12 22:21:51Z dhill $ -* -* -* List of files being updated by configure: -* Columnstore/etc/Columnstore.xml -* -* -******************************************************************************************/ + * $Id: configure.cpp 64 2006-10-12 22:21:51Z dhill $ + * + * + * List of files being updated by configure: + * Columnstore/etc/Columnstore.xml + * + * + ******************************************************************************************/ /** * @file */ - #include #include #include @@ -51,7 +50,6 @@ #include "configcpp.h" #include "installdir.h" - #include "mcsconfig.h" using namespace std; @@ -60,1926 +58,1979 @@ using namespace config; typedef struct Performance_Module_struct { - std::string moduleIP1; - std::string moduleIP2; - std::string moduleIP3; - std::string moduleIP4; + std::string moduleIP1; + std::string moduleIP2; + std::string moduleIP3; + std::string moduleIP4; } PerformanceModule; typedef std::vector PerformanceModuleList; int main(int argc, char* argv[]) { - setenv("CALPONT_HOME", "./", 1); + setenv("CALPONT_HOME", "./", 1); - Oam oam; - string systemParentOAMModuleName; - string parentOAMModuleIPAddr; - PerformanceModuleList performancemodulelist; - int pmNumber = 0; - string prompt; - int DBRMworkernodeID = 0; - string remote_installer_debug = "0"; - Config* sysConfigOld; - Config* sysConfigNew; - string systemName; - bool extextMapCheckOnly = false; + Oam oam; + string systemParentOAMModuleName; + string parentOAMModuleIPAddr; + PerformanceModuleList performancemodulelist; + int pmNumber = 0; + string prompt; + int DBRMworkernodeID = 0; + string remote_installer_debug = "0"; + Config* sysConfigOld; + Config* sysConfigNew; + string systemName; + bool extextMapCheckOnly = false; - if (argc > 1) + if (argc > 1) + { + string arg1 = argv[1]; + + if (arg1 == "-e") + extextMapCheckOnly = true; + else + systemName = arg1; + } + else + systemName = oam::UnassignedName; + + try + { + std::string configFilePathOld = std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml"); + std::string configFilePathNew = + std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml.new"); + sysConfigOld = Config::makeConfig(configFilePathOld); // system version + sysConfigNew = Config::makeConfig(configFilePathNew); // released version + } + catch (...) + { + cout << "ERROR: Problem reading Columnstore.xml files\n"; + exit(-1); + } + + string SystemSection = "SystemConfig"; + string InstallSection = "Installation"; + + // read cloud parameter to see if OLD is a 3.0+ or pre-3.0 build being installed + string cloud; + + try + { + cloud = sysConfigOld->getConfig(InstallSection, "Cloud"); + } + catch (...) + { + } + + // build3 is for 3 and above 4 + bool OLDbuild3 = false; + + if (!cloud.empty()) + OLDbuild3 = true; + + // read cloud parameter to see if NEW is a 3.0+ or pre-3.0 build being installed + try + { + cloud = sysConfigNew->getConfig(InstallSection, "Cloud"); + } + catch (...) + { + } + + // build3 is for 3 and above 4 + bool build3 = false; + + if (!cloud.empty()) + build3 = true; + + bool build40 = false; + bool build401 = true; + + // check and update PMwithUM + try + { + string PMwithUM = sysConfigOld->getConfig(InstallSection, "PMwithUM"); + + if (!PMwithUM.empty()) { - string arg1 = argv[1]; + try + { + sysConfigNew->setConfig(InstallSection, "PMwithUM", PMwithUM); + } + catch (...) + { + cout << "ERROR: Problem setting PMwithUM in the Columnstore System Configuration file" << endl; + exit(-1); + } + } + } + catch (...) + { + } - if ( arg1 == "-e" ) - extextMapCheckOnly = true; - else - systemName = arg1; + // check and update PMwithUM + try + { + string MySQLRep = sysConfigOld->getConfig(InstallSection, "MySQLRep"); + + if (!MySQLRep.empty()) + { + try + { + sysConfigNew->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch (...) + { + cout << "ERROR: Problem setting MySQLRep in the Columnstore System Configuration file" << endl; + exit(-1); + } + } + } + catch (...) + { + } + + // set gluster flag if it exists + string DataRedundancyConfig; + string DataRedundancyCopies; + string DataRedundancyStorageType; + string DataRedundancyNetworkType; + + try + { + DataRedundancyConfig = sysConfigOld->getConfig(InstallSection, "DataRedundancyConfig"); + DataRedundancyCopies = sysConfigOld->getConfig(InstallSection, "DataRedundancyCopies"); + DataRedundancyStorageType = sysConfigOld->getConfig(InstallSection, "DataRedundancyStorageType"); + DataRedundancyNetworkType = sysConfigOld->getConfig(InstallSection, "DataRedundancyNetworkType"); + } + catch (...) + { + } + + if (!DataRedundancyConfig.empty()) + { + try + { + sysConfigNew->setConfig(InstallSection, "DataRedundancyConfig", DataRedundancyConfig); + sysConfigNew->setConfig(InstallSection, "DataRedundancyCopies", DataRedundancyCopies); + sysConfigNew->setConfig(InstallSection, "DataRedundancyStorageType", DataRedundancyStorageType); + sysConfigNew->setConfig(InstallSection, "DataRedundancyNetworkType", DataRedundancyNetworkType); + } + catch (...) + { + } + } + + // check and make sure the ExtentMap variables don't get changed at install + string oldFilesPerColumnPartition; + string oldExtentsPerSegmentFile; + string newFilesPerColumnPartition; + string newExtentsPerSegmentFile; + + try + { + oldFilesPerColumnPartition = sysConfigOld->getConfig("ExtentMap", "FilesPerColumnPartition"); + oldExtentsPerSegmentFile = sysConfigOld->getConfig("ExtentMap", "ExtentsPerSegmentFile"); + + newFilesPerColumnPartition = sysConfigNew->getConfig("ExtentMap", "FilesPerColumnPartition"); + newExtentsPerSegmentFile = sysConfigNew->getConfig("ExtentMap", "ExtentsPerSegmentFile"); + + if (oldFilesPerColumnPartition != newFilesPerColumnPartition) + { + try + { + sysConfigNew->setConfig("ExtentMap", "FilesPerColumnPartition", oldFilesPerColumnPartition); + } + catch (...) + { + cout << "ERROR: Problem setting FilesPerColumnPartition in the Columnstore System Configuration file" + << endl; + exit(-1); + } + } + + if (oldExtentsPerSegmentFile != newExtentsPerSegmentFile) + { + try + { + sysConfigNew->setConfig("ExtentMap", "ExtentsPerSegmentFile", oldExtentsPerSegmentFile); + } + catch (...) + { + cout << "ERROR: Problem setting ExtentsPerSegmentFile in the Columnstore System Configuration file" + << endl; + exit(-1); + } + } + } + catch (...) + { + } + + sysConfigNew->write(); + + if (extextMapCheckOnly) + exit(0); + + systemParentOAMModuleName = "pm1"; + + // check if systemParentOAMModuleName (pm1) is configured, if not set to 'pm2' + // string IPaddr = sysConfigOld->getConfig("pm1_ProcessMonitor", "IPAddr"); + // if ( IPaddr == "0.0.0.0" ) + // systemParentOAMModuleName = "pm2"; + + // set Parent OAM Module Name + try + { + sysConfigNew->setConfig(SystemSection, "ParentOAMModuleName", systemParentOAMModuleName); + } + catch (...) + { + cout << "ERROR: Problem updating the Columnstore System Configuration file" << endl; + exit(-1); + } + + // setup System Name + string oldSystemName; + + try + { + oldSystemName = sysConfigOld->getConfig(SystemSection, "SystemName"); + } + catch (...) + { + } + + if (!oldSystemName.empty()) + systemName = oldSystemName; + + try + { + sysConfigNew->setConfig(SystemSection, "SystemName", systemName); + } + catch (...) + { + cout << "ERROR: Problem setting SystemName from the Columnstore System Configuration file" << endl; + exit(-1); + } + + // WaitPeriod + try + { + string waitPeriod = sysConfigOld->getConfig(SystemSection, "WaitPeriod"); + if (waitPeriod.length() > 0) + { + sysConfigNew->setConfig(SystemSection, "WaitPeriod", waitPeriod); + } + } + catch (...) + { + } + + // setup HA IP Address + string HA_IPadd; + + try + { + HA_IPadd = sysConfigOld->getConfig("ProcMgr_HA", "IPAddr"); + } + catch (...) + { + } + + if (!HA_IPadd.empty()) + { + try + { + sysConfigNew->setConfig("ProcMgr_HA", "IPAddr", HA_IPadd); + } + catch (...) + { + cout << "ERROR: Problem setting ProcMgr_HA from the Columnstore System Configuration file" << endl; + exit(-1); + } + } + + // setup CMP IP Addresses + string CMP_IPadd; + string CMP_port; + + for (int id = 1;; id++) + { + string cmpName = "CMP" + oam.itoa(id); + + try + { + CMP_IPadd = sysConfigOld->getConfig(cmpName, "IPAddr"); + } + catch (...) + { + } + + if (!CMP_IPadd.empty()) + { + try + { + CMP_port = sysConfigOld->getConfig(cmpName, "Port"); + } + catch (...) + { + } + + try + { + sysConfigNew->setConfig(cmpName, "IPAddr", CMP_IPadd); + sysConfigNew->setConfig(cmpName, "Port", CMP_port); + } + catch (...) + { + cout << "ERROR: Problem setting CMP from the Columnstore System Configuration file" << endl; + exit(-1); + } } else - systemName = oam::UnassignedName; + break; + } - try + // make DBRM backwards compatiable for pre 1.0.0.157 load + string dbrmMainProc = "DBRM_Controller"; + string dbrmSubProc = "DBRM_Worker"; + string numSubProc = "NumWorkers"; + + // CrossEngineSupport + string Host = ""; + string Port = "3306"; + string User = ""; + string Password = ""; + string TLSCA = ""; + string TLSClientCert = ""; + string TLSClientKey = ""; + + try + { + Host = sysConfigOld->getConfig("CrossEngineSupport", "Host"); + Port = sysConfigOld->getConfig("CrossEngineSupport", "Port"); + User = sysConfigOld->getConfig("CrossEngineSupport", "User"); + Password = sysConfigOld->getConfig("CrossEngineSupport", "Password"); + } + catch (...) + { + Host = ""; + Port = "3306"; + User = ""; + Password = ""; + } + + try + { + sysConfigNew->setConfig("CrossEngineSupport", "Host", Host); + sysConfigNew->setConfig("CrossEngineSupport", "Port", Port); + sysConfigNew->setConfig("CrossEngineSupport", "User", User); + sysConfigNew->setConfig("CrossEngineSupport", "Password", Password); + } + catch (...) + { + } + + try + { + TLSCA = sysConfigOld->getConfig("CrossEngineSupport", "TLSCA"); + TLSClientCert = sysConfigOld->getConfig("CrossEngineSupport", "TLSClientCert"); + TLSClientKey = sysConfigOld->getConfig("CrossEngineSupport", "TLSClientKey"); + } + catch (...) + { + TLSCA = ""; + TLSClientCert = ""; + TLSClientKey = ""; + } + + try + { + sysConfigNew->setConfig("CrossEngineSupport", "TLSCA", TLSCA); + sysConfigNew->setConfig("CrossEngineSupport", "TLSClientCert", TLSClientCert); + sysConfigNew->setConfig("CrossEngineSupport", "TLSClientKey", TLSClientKey); + } + catch (...) + { + } + + // QueryStats and UserPriority + string QueryStats = "N"; + string UserPriority = "N"; + + try + { + QueryStats = sysConfigOld->getConfig("QueryStats", "Enabled"); + UserPriority = sysConfigOld->getConfig("UserPriority", "Enabled"); + } + catch (...) + { + QueryStats = "N"; + UserPriority = "N"; + } + + try + { + sysConfigNew->setConfig("QueryStats", "Enabled", QueryStats); + sysConfigNew->setConfig("UserPriority", "Enabled", UserPriority); + } + catch (...) + { + } + + // @bug4598, DirectIO setting + string directIO = "y"; + + try + { + directIO = sysConfigOld->getConfig("PrimitiveServers", "DirectIO"); + } + catch (...) + { + directIO = "y"; + } + + try + { + sysConfigNew->setConfig("PrimitiveServers", "DirectIO", directIO); + } + catch (...) + { + } + + // @bug4507, configurable pm MemoryCheck + string memCheck; + + try + { + memCheck = sysConfigOld->getConfig("SystemConfig", "MemoryCheckPercent"); + + if (!(memCheck.empty() || memCheck == "")) { - std::string configFilePathOld = std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml"); - std::string configFilePathNew = std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml.new"); - sysConfigOld = Config::makeConfig(configFilePathOld); // system version - sysConfigNew = Config::makeConfig(configFilePathNew); // released version + sysConfigNew->setConfig("SystemConfig", "MemoryCheckPercent", memCheck); } - catch (...) + } + catch (...) + { + } + + // Priority Settings + string HighPriorityPercentage; + + try + { + HighPriorityPercentage = sysConfigOld->getConfig("PrimitiveServers", "HighPriorityPercentage"); + sysConfigNew->setConfig("PrimitiveServers", "HighPriorityPercentage", HighPriorityPercentage); + } + catch (...) + { + } + + string MediumPriorityPercentage; + + try + { + MediumPriorityPercentage = sysConfigOld->getConfig("PrimitiveServers", "MediumPriorityPercentage"); + sysConfigNew->setConfig("PrimitiveServers", "MediumPriorityPercentage", MediumPriorityPercentage); + } + catch (...) + { + } + + string LowPriorityPercentage; + + try + { + LowPriorityPercentage = sysConfigOld->getConfig("PrimitiveServers", "LowPriorityPercentage"); + sysConfigNew->setConfig("PrimitiveServers", "LowPriorityPercentage", LowPriorityPercentage); + } + catch (...) + { + } + + // default to single-server install type + string OserverTypeInstall = oam.itoa(oam::INSTALL_COMBINE_DM_UM_PM); + ; + string NserverTypeInstall; + string OSingleServerInstall = "y"; + int IserverTypeInstall; + + try + { + OserverTypeInstall = sysConfigOld->getConfig(InstallSection, "ServerTypeInstall"); + OSingleServerInstall = sysConfigOld->getConfig(InstallSection, "SingleServerInstall"); + } + catch (...) + { + // default to Normal mult-server install type + OserverTypeInstall = oam.itoa(oam::INSTALL_COMBINE_DM_UM_PM); + OSingleServerInstall = "y"; + } + + // set Server Installation Type + try + { + sysConfigNew->setConfig(InstallSection, "ServerTypeInstall", OserverTypeInstall); + sysConfigNew->setConfig(InstallSection, "SingleServerInstall", OSingleServerInstall); + } + catch (...) + { + } + + NserverTypeInstall = OserverTypeInstall; + + IserverTypeInstall = atoi(NserverTypeInstall.c_str()); + + // set RotatingDestination + switch (IserverTypeInstall) + { + case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { - cout << "ERROR: Problem reading Columnstore.xml files\n"; + try + { + sysConfigNew->setConfig("PrimitiveServers", "RotatingDestination", "n"); + } + catch (...) + { + cout << "ERROR: Problem setting RotatingDestination in the Columnstore System Configuration file" + << endl; exit(-1); + } + + break; } + } - string SystemSection = "SystemConfig"; - string InstallSection = "Installation"; + string parentOAMModuleType = systemParentOAMModuleName.substr(0, MAX_MODULE_TYPE_SIZE); - //read cloud parameter to see if OLD is a 3.0+ or pre-3.0 build being installed - string cloud; + // + // get Data storage Mount + // + string DBRootStorageType; + + int DBRootCount; + string deviceName; + + try + { + DBRootStorageType = sysConfigOld->getConfig(InstallSection, "DBRootStorageType"); + DBRootCount = strtol(sysConfigOld->getConfig(SystemSection, "DBRootCount").c_str(), 0, 0); + } + catch (...) + { + cout << "ERROR: Problem getting DB Storage Data from the Columnstore System Configuration file" << endl; + exit(-1); + } + + // 2.2 to 3.x+ DBRootStorageTypeconversion + if (DBRootStorageType == "local") + DBRootStorageType = "internal"; + + if (DBRootStorageType == "storage") + DBRootStorageType = "external"; + + try + { + sysConfigNew->setConfig(InstallSection, "DBRootStorageType", DBRootStorageType); + } + catch (...) + { + cout << "ERROR: Problem setting DBRootStorageType in the Columnstore System Configuration file" << endl; + exit(-1); + } + + try + { + sysConfigNew->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); + } + catch (...) + { + cout << "ERROR: Problem setting DBRoot Count in the Columnstore System Configuration file" << endl; + exit(-1); + } + + // + // Update memory and cache settings + // + + string NumBlocksPct; + + try + { + NumBlocksPct = sysConfigOld->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } + + if ((NumBlocksPct.empty() || NumBlocksPct == "") && IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM) + { + NumBlocksPct = "50"; + } + + if (!(NumBlocksPct.empty() || NumBlocksPct == "")) + { try { - cloud = sysConfigOld->getConfig(InstallSection, "Cloud"); + sysConfigNew->setConfig("DBBC", "NumBlocksPct", NumBlocksPct); } catch (...) - {} + { + } + } - // build3 is for 3 and above 4 - bool OLDbuild3 = false; + string TotalUmMemory; - if ( !cloud.empty() ) - OLDbuild3 = true; + try + { + TotalUmMemory = sysConfigOld->getConfig("HashJoin", "TotalUmMemory"); + } + catch (...) + { + } - //read cloud parameter to see if NEW is a 3.0+ or pre-3.0 build being installed + try + { + sysConfigNew->setConfig("HashJoin", "TotalUmMemory", TotalUmMemory); + } + catch (...) + { + } + + string TotalPmUmMemory; + + try + { + TotalPmUmMemory = sysConfigOld->getConfig("HashJoin", "TotalPmUmMemory"); + } + catch (...) + { + } + + try + { + sysConfigNew->setConfig("HashJoin", "TotalPmUmMemory", TotalPmUmMemory); + } + catch (...) + { + } + + string strNumThreads; + + try + { + strNumThreads = sysConfigOld->getConfig("DBBC", "NumThreads"); + } + catch (...) + { + } + + if (!(strNumThreads.empty() || strNumThreads == "")) + { try { - cloud = sysConfigNew->getConfig(InstallSection, "Cloud"); + sysConfigNew->setConfig("DBBC", "NumThreads", strNumThreads); } catch (...) - {} - - // build3 is for 3 and above 4 - bool build3 = false; - - if ( !cloud.empty() ) - build3 = true; - - - bool build40 = false; - bool build401 = true; - - - //check and update PMwithUM - try { - string PMwithUM = sysConfigOld->getConfig(InstallSection, "PMwithUM"); + } + } - if ( !PMwithUM.empty() ) + string MySQLPort = "3306"; + + try + { + MySQLPort = sysConfigOld->getConfig("Installation", "MySQLPort"); + } + catch (...) + { + MySQLPort = "3306"; + } + + try + { + sysConfigNew->setConfig("Installation", "MySQLPort", MySQLPort); + } + catch (...) + { + } + + sysConfigNew->write(); + + // Get list of configured system modules + SystemModuleTypeConfig sysModuleTypeConfig; + + try + { + oam.getSystemConfig(sysModuleTypeConfig); + } + catch (...) + { + cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; + exit(-1); + } + + // + // Module Configuration + // + string ModuleSection = "SystemModuleConfig"; + unsigned int maxPMNicCount = 1; + + for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) + { + string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType; + string moduleDesc = sysModuleTypeConfig.moduletypeconfig[i].ModuleDesc; + int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount; + + // verify and setup of modules count + switch (IserverTypeInstall) + { + case (oam::INSTALL_COMBINE_DM_UM_PM): + { + if (moduleType == "um") { - try - { - sysConfigNew->setConfig(InstallSection, "PMwithUM", PMwithUM); - } - catch (...) - { - cout << "ERROR: Problem setting PMwithUM in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - } - catch (...) - { - } + moduleCount = 0; - //check and update PMwithUM - try - { - string MySQLRep = sysConfigOld->getConfig(InstallSection, "MySQLRep"); - - if ( !MySQLRep.empty() ) - { - try - { - sysConfigNew->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch (...) - { - cout << "ERROR: Problem setting MySQLRep in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - } - catch (...) - { - } - - //set gluster flag if it exists - string DataRedundancyConfig; - string DataRedundancyCopies; - string DataRedundancyStorageType; - string DataRedundancyNetworkType; - - try - { - DataRedundancyConfig = sysConfigOld->getConfig(InstallSection, "DataRedundancyConfig"); - DataRedundancyCopies = sysConfigOld->getConfig(InstallSection, "DataRedundancyCopies"); - DataRedundancyStorageType = sysConfigOld->getConfig(InstallSection, "DataRedundancyStorageType"); - DataRedundancyNetworkType = sysConfigOld->getConfig(InstallSection, "DataRedundancyNetworkType"); - } - catch (...) - {} - - if ( !DataRedundancyConfig.empty() ) - { - try - { - sysConfigNew->setConfig(InstallSection, "DataRedundancyConfig", DataRedundancyConfig); - sysConfigNew->setConfig(InstallSection, "DataRedundancyCopies", DataRedundancyCopies); - sysConfigNew->setConfig(InstallSection, "DataRedundancyStorageType", DataRedundancyStorageType); - sysConfigNew->setConfig(InstallSection, "DataRedundancyNetworkType", DataRedundancyNetworkType); - } - catch (...) - {} - } - - //check and make sure the ExtentMap variables don't get changed at install - string oldFilesPerColumnPartition; - string oldExtentsPerSegmentFile; - string newFilesPerColumnPartition; - string newExtentsPerSegmentFile; - - try - { - oldFilesPerColumnPartition = sysConfigOld->getConfig("ExtentMap", "FilesPerColumnPartition"); - oldExtentsPerSegmentFile = sysConfigOld->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - newFilesPerColumnPartition = sysConfigNew->getConfig("ExtentMap", "FilesPerColumnPartition"); - newExtentsPerSegmentFile = sysConfigNew->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - if ( oldFilesPerColumnPartition != newFilesPerColumnPartition ) - { - try - { - sysConfigNew->setConfig("ExtentMap", "FilesPerColumnPartition", oldFilesPerColumnPartition); - } - catch (...) - { - cout << "ERROR: Problem setting FilesPerColumnPartition in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - - if ( oldExtentsPerSegmentFile != newExtentsPerSegmentFile ) - { - try - { - sysConfigNew->setConfig("ExtentMap", "ExtentsPerSegmentFile", oldExtentsPerSegmentFile); - } - catch (...) - { - cout << "ERROR: Problem setting ExtentsPerSegmentFile in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - } - catch (...) - { - } - - sysConfigNew->write(); - - if ( extextMapCheckOnly ) - exit(0); - - systemParentOAMModuleName = "pm1"; - - //check if systemParentOAMModuleName (pm1) is configured, if not set to 'pm2' -// string IPaddr = sysConfigOld->getConfig("pm1_ProcessMonitor", "IPAddr"); -// if ( IPaddr == "0.0.0.0" ) -// systemParentOAMModuleName = "pm2"; - - //set Parent OAM Module Name - try - { - sysConfigNew->setConfig(SystemSection, "ParentOAMModuleName", systemParentOAMModuleName); - } - catch (...) - { - cout << "ERROR: Problem updating the Columnstore System Configuration file" << endl; - exit(-1); - } - - //setup System Name - string oldSystemName; - - try - { - oldSystemName = sysConfigOld->getConfig(SystemSection, "SystemName"); - } - catch (...) - { } - - if ( !oldSystemName.empty() ) - systemName = oldSystemName; - - try - { - sysConfigNew->setConfig(SystemSection, "SystemName", systemName); - } - catch (...) - { - cout << "ERROR: Problem setting SystemName from the Columnstore System Configuration file" << endl; - exit(-1); - } - - // WaitPeriod - try - { - string waitPeriod = sysConfigOld->getConfig(SystemSection, "WaitPeriod"); - if (waitPeriod.length() > 0) - { - sysConfigNew->setConfig(SystemSection, "WaitPeriod", waitPeriod); - } - } - catch (...) - { } - - //setup HA IP Address - string HA_IPadd; - - try - { - HA_IPadd = sysConfigOld->getConfig("ProcMgr_HA", "IPAddr"); - } - catch (...) - { } - - if ( !HA_IPadd.empty() ) - { - try - { - sysConfigNew->setConfig("ProcMgr_HA", "IPAddr", HA_IPadd); - } - catch (...) - { - cout << "ERROR: Problem setting ProcMgr_HA from the Columnstore System Configuration file" << endl; + try + { + string ModuleCountParm = "ModuleCount" + oam.itoa(i + 1); + sysConfigNew->setConfig(ModuleSection, ModuleCountParm, oam.itoa(moduleCount)); + continue; + } + catch (...) + { + cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" + << endl; exit(-1); - } - } - - //setup CMP IP Addresses - string CMP_IPadd; - string CMP_port; - - for ( int id = 1 ;; id ++ ) - { - string cmpName = "CMP" + oam.itoa(id); - - try - { - CMP_IPadd = sysConfigOld->getConfig(cmpName, "IPAddr"); - } - catch (...) - { } - - if ( !CMP_IPadd.empty() ) - { - try - { - CMP_port = sysConfigOld->getConfig(cmpName, "Port"); - } - catch (...) - { } - - try - { - sysConfigNew->setConfig(cmpName, "IPAddr", CMP_IPadd); - sysConfigNew->setConfig(cmpName, "Port", CMP_port); - } - catch (...) - { - cout << "ERROR: Problem setting CMP from the Columnstore System Configuration file" << endl; - exit(-1); - } + } } else - break; - } - - // make DBRM backwards compatiable for pre 1.0.0.157 load - string dbrmMainProc = "DBRM_Controller"; - string dbrmSubProc = "DBRM_Worker"; - string numSubProc = "NumWorkers"; - - //CrossEngineSupport - string Host = ""; - string Port = "3306"; - string User = ""; - string Password = ""; - string TLSCA = ""; - string TLSClientCert = ""; - string TLSClientKey = ""; - - try - { - Host = sysConfigOld->getConfig("CrossEngineSupport", "Host"); - Port = sysConfigOld->getConfig("CrossEngineSupport", "Port"); - User = sysConfigOld->getConfig("CrossEngineSupport", "User"); - Password = sysConfigOld->getConfig("CrossEngineSupport", "Password"); - } - catch (...) - { - Host = ""; - Port = "3306"; - User = ""; - Password = ""; - } - - try - { - sysConfigNew->setConfig("CrossEngineSupport", "Host", Host); - sysConfigNew->setConfig("CrossEngineSupport", "Port", Port); - sysConfigNew->setConfig("CrossEngineSupport", "User", User); - sysConfigNew->setConfig("CrossEngineSupport", "Password", Password); - } - catch (...) - {} - - try - { - TLSCA = sysConfigOld->getConfig("CrossEngineSupport", "TLSCA"); - TLSClientCert = sysConfigOld->getConfig("CrossEngineSupport", "TLSClientCert"); - TLSClientKey = sysConfigOld->getConfig("CrossEngineSupport", "TLSClientKey"); - } - catch (...) - { - TLSCA = ""; - TLSClientCert = ""; - TLSClientKey = ""; - } - - try - { - sysConfigNew->setConfig("CrossEngineSupport", "TLSCA", TLSCA); - sysConfigNew->setConfig("CrossEngineSupport", "TLSClientCert", TLSClientCert); - sysConfigNew->setConfig("CrossEngineSupport", "TLSClientKey", TLSClientKey); - } - catch (...) - {} - - //QueryStats and UserPriority - string QueryStats = "N"; - string UserPriority = "N"; - - try - { - QueryStats = sysConfigOld->getConfig("QueryStats", "Enabled"); - UserPriority = sysConfigOld->getConfig("UserPriority", "Enabled"); - } - catch (...) - { - QueryStats = "N"; - UserPriority = "N"; - } - - try - { - sysConfigNew->setConfig("QueryStats", "Enabled", QueryStats); - sysConfigNew->setConfig("UserPriority", "Enabled", UserPriority); - } - catch (...) - {} - - // @bug4598, DirectIO setting - string directIO = "y"; - - try - { - directIO = sysConfigOld->getConfig("PrimitiveServers", "DirectIO"); - } - catch (...) - { - directIO = "y"; - } - - try - { - sysConfigNew->setConfig("PrimitiveServers", "DirectIO", directIO); - } - catch (...) - {} - - // @bug4507, configurable pm MemoryCheck - string memCheck; - - try - { - memCheck = sysConfigOld->getConfig("SystemConfig", "MemoryCheckPercent"); - - if ( !( memCheck.empty() || memCheck == "" ) ) { - sysConfigNew->setConfig("SystemConfig", "MemoryCheckPercent", memCheck); + try + { + string ModuleCountParm = "ModuleCount" + oam.itoa(i + 1); + sysConfigNew->setConfig(ModuleSection, ModuleCountParm, oam.itoa(moduleCount)); + } + catch (...) + { + cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" + << endl; + exit(-1); + } } - } - catch (...) - { - } - //Priority Settings - string HighPriorityPercentage; + break; + } - try - { - HighPriorityPercentage = sysConfigOld->getConfig("PrimitiveServers", "HighPriorityPercentage"); - sysConfigNew->setConfig("PrimitiveServers", "HighPriorityPercentage", HighPriorityPercentage); - } - catch (...) {} - - string MediumPriorityPercentage; - - try - { - MediumPriorityPercentage = sysConfigOld->getConfig("PrimitiveServers", "MediumPriorityPercentage"); - sysConfigNew->setConfig("PrimitiveServers", "MediumPriorityPercentage", MediumPriorityPercentage); - } - catch (...) {} - - string LowPriorityPercentage; - - try - { - LowPriorityPercentage = sysConfigOld->getConfig("PrimitiveServers", "LowPriorityPercentage"); - sysConfigNew->setConfig("PrimitiveServers", "LowPriorityPercentage", LowPriorityPercentage); - } - catch (...) {} - - // default to single-server install type - string OserverTypeInstall = oam.itoa(oam::INSTALL_COMBINE_DM_UM_PM);; - string NserverTypeInstall; - string OSingleServerInstall = "y"; - int IserverTypeInstall; - - try - { - OserverTypeInstall = sysConfigOld->getConfig(InstallSection, "ServerTypeInstall"); - OSingleServerInstall = sysConfigOld->getConfig(InstallSection, "SingleServerInstall"); - } - catch (...) - { - // default to Normal mult-server install type - OserverTypeInstall = oam.itoa(oam::INSTALL_COMBINE_DM_UM_PM); - OSingleServerInstall = "y"; - } - - // set Server Installation Type - try - { - sysConfigNew->setConfig(InstallSection, "ServerTypeInstall", OserverTypeInstall); - sysConfigNew->setConfig(InstallSection, "SingleServerInstall", OSingleServerInstall); - } - catch (...) - {} - - NserverTypeInstall = OserverTypeInstall; - - IserverTypeInstall = atoi(NserverTypeInstall.c_str()); - - //set RotatingDestination - switch ( IserverTypeInstall ) - { - case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server - { - try - { - sysConfigNew->setConfig("PrimitiveServers", "RotatingDestination", "n"); - } - catch (...) - { - cout << "ERROR: Problem setting RotatingDestination in the Columnstore System Configuration file" << endl; - exit(-1); - } - - break; - } - } - - string parentOAMModuleType = systemParentOAMModuleName.substr(0, MAX_MODULE_TYPE_SIZE); - - // - // get Data storage Mount - // - - string DBRootStorageType; - - int DBRootCount; - string deviceName; - - try - { - DBRootStorageType = sysConfigOld->getConfig(InstallSection, "DBRootStorageType"); - DBRootCount = strtol(sysConfigOld->getConfig(SystemSection, "DBRootCount").c_str(), 0, 0); - } - catch (...) - { - cout << "ERROR: Problem getting DB Storage Data from the Columnstore System Configuration file" << endl; - exit(-1); - } - - // 2.2 to 3.x+ DBRootStorageTypeconversion - if ( DBRootStorageType == "local" ) - DBRootStorageType = "internal"; - - if ( DBRootStorageType == "storage" ) - DBRootStorageType = "external"; - - try - { - sysConfigNew->setConfig(InstallSection, "DBRootStorageType", DBRootStorageType); - } - catch (...) - { - cout << "ERROR: Problem setting DBRootStorageType in the Columnstore System Configuration file" << endl; - exit(-1); - } - - try - { - sysConfigNew->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); - } - catch (...) - { - cout << "ERROR: Problem setting DBRoot Count in the Columnstore System Configuration file" << endl; - exit(-1); - } - - // - //Update memory and cache settings - // - - string NumBlocksPct; - - try - { - NumBlocksPct = sysConfigOld->getConfig("DBBC", "NumBlocksPct"); - } - catch (...) - { } - - if ( ( NumBlocksPct.empty() || NumBlocksPct == "" ) && - IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) - { - NumBlocksPct = "50"; - } - - if ( !( NumBlocksPct.empty() || NumBlocksPct == "" ) ) - { + default: + { try { - sysConfigNew->setConfig("DBBC", "NumBlocksPct", NumBlocksPct); + string ModuleCountParm = "ModuleCount" + oam.itoa(i + 1); + sysConfigNew->setConfig(ModuleSection, ModuleCountParm, oam.itoa(moduleCount)); } catch (...) - {} + { + cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" << endl; + exit(-1); + } + + break; + } } - string TotalUmMemory; + if (moduleCount == 0) + // no modules equipped for this Module Type, skip + continue; - try + if (moduleType == "pm") + pmNumber = moduleCount; + + // for 2.x to 3.x upgrade dbroot assignments + int dbrootNum = 0; + int systemDBRootCount = 0; + int dbrootCountPerModule = 0; + + if (moduleType == "pm" && !OLDbuild3) { - TotalUmMemory = sysConfigOld->getConfig("HashJoin", "TotalUmMemory"); + dbrootNum = 1; + systemDBRootCount = DBRootCount; + + if (pmNumber > 0) + dbrootCountPerModule = DBRootCount / pmNumber; + + if (dbrootCountPerModule == 0) + dbrootCountPerModule = 1; } - catch (...) - { } - try + // get Module Name IP addresses and Host Names + DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end(); listPT++) { - sysConfigNew->setConfig("HashJoin", "TotalUmMemory", TotalUmMemory); - } - catch (...) - {} + PerformanceModule performancemodule; + string moduleName = (*listPT).DeviceName; + int moduleID = atoi(moduleName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - string TotalPmUmMemory; + string moduleDisableState = (*listPT).DisableState; - try - { - TotalPmUmMemory = sysConfigOld->getConfig("HashJoin", "TotalPmUmMemory"); - } - catch (...) - { } + // set Module Disable State + string moduleDisableStateParm = "ModuleDisableState" + oam.itoa(moduleID) + "-" + oam.itoa(i + 1); - try - { - sysConfigNew->setConfig("HashJoin", "TotalPmUmMemory", TotalPmUmMemory); - } - catch (...) - {} + try + { + sysConfigNew->setConfig(ModuleSection, moduleDisableStateParm, moduleDisableState); + } + catch (...) + { + cout + << "ERROR: Problem setting ModuleDisableState in the Columnstore System Configuration file for " + + moduleName + << endl; + exit(-1); + } - string strNumThreads; + for (unsigned int nicID = 1; nicID < MAX_NIC + 1; nicID++) + { + string moduleIPAddr = oam::UnassignedIpAddr; + string moduleHostName = oam::UnassignedName; - try - { - strNumThreads = sysConfigOld->getConfig("DBBC", "NumThreads"); - } - catch (...) - { } + HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); + + for (; pt1 != (*listPT).hostConfigList.end(); pt1++) + { + if (moduleName == (*listPT).DeviceName && (*pt1).NicID == nicID) + { + moduleIPAddr = (*pt1).IPAddr; + moduleHostName = (*pt1).HostName; + break; + } + } + + if (moduleHostName.empty() || (moduleHostName == oam::UnassignedName)) + // exit out to next module ID + break; + + if (moduleIPAddr.empty()) + moduleIPAddr = oam::UnassignedIpAddr; + + string moduleNameDesc = moduleDesc + " #" + oam.itoa(moduleID); + + // set New Module Host Name + string moduleHostNameParm = + "ModuleHostName" + oam.itoa(moduleID) + "-" + oam.itoa(nicID) + "-" + oam.itoa(i + 1); - if ( !( strNumThreads.empty() || strNumThreads == "" ) ) - { try { - sysConfigNew->setConfig("DBBC", "NumThreads", strNumThreads); + sysConfigNew->setConfig(ModuleSection, moduleHostNameParm, moduleHostName); } catch (...) - {} - } - - string MySQLPort = "3306"; - - try - { - MySQLPort = sysConfigOld->getConfig("Installation", "MySQLPort"); - } - catch (...) - { - MySQLPort = "3306"; - } - - try - { - sysConfigNew->setConfig("Installation", "MySQLPort", MySQLPort); - } - catch (...) - {} - - sysConfigNew->write(); - - //Get list of configured system modules - SystemModuleTypeConfig sysModuleTypeConfig; - - try - { - oam.getSystemConfig(sysModuleTypeConfig); - } - catch (...) - { - cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; - exit(-1); - } - - // - // Module Configuration - // - string ModuleSection = "SystemModuleConfig"; - unsigned int maxPMNicCount = 1; - - for ( unsigned int i = 0 ; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) - { - string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType; - string moduleDesc = sysModuleTypeConfig.moduletypeconfig[i].ModuleDesc; - int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount; - - //verify and setup of modules count - switch ( IserverTypeInstall ) { - case (oam::INSTALL_COMBINE_DM_UM_PM): - { - if ( moduleType == "um" ) - { - moduleCount = 0; - - try - { - string ModuleCountParm = "ModuleCount" + oam.itoa(i + 1); - sysConfigNew->setConfig(ModuleSection, ModuleCountParm, oam.itoa(moduleCount)); - continue; - } - catch (...) - { - cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - else - { - try - { - string ModuleCountParm = "ModuleCount" + oam.itoa(i + 1); - sysConfigNew->setConfig(ModuleSection, ModuleCountParm, oam.itoa(moduleCount)); - } - catch (...) - { - cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - - break; - } - - default: - { - try - { - string ModuleCountParm = "ModuleCount" + oam.itoa(i + 1); - sysConfigNew->setConfig(ModuleSection, ModuleCountParm, oam.itoa(moduleCount)); - } - catch (...) - { - cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" << endl; - exit(-1); - } - - break; - } + cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; + exit(-1); } - if ( moduleCount == 0 ) - //no modules equipped for this Module Type, skip - continue; + // set Module IP address + string moduleIPAddrNameParm = + "ModuleIPAddr" + oam.itoa(moduleID) + "-" + oam.itoa(nicID) + "-" + oam.itoa(i + 1); - if ( moduleType == "pm" ) - pmNumber = moduleCount; - - //for 2.x to 3.x upgrade dbroot assignments - int dbrootNum = 0; - int systemDBRootCount = 0; - int dbrootCountPerModule = 0; - - if ( moduleType == "pm" && !OLDbuild3) + try { - dbrootNum = 1; - systemDBRootCount = DBRootCount; - - if ( pmNumber > 0 ) - dbrootCountPerModule = DBRootCount / pmNumber; - - if ( dbrootCountPerModule == 0 ) - dbrootCountPerModule = 1; + sysConfigNew->setConfig(ModuleSection, moduleIPAddrNameParm, moduleIPAddr); + } + catch (...) + { + cout << "ERROR: Problem setting IP address in the Columnstore System Configuration file" << endl; + exit(-1); } - //get Module Name IP addresses and Host Names - DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for ( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) + if (moduleType == "pm") { - PerformanceModule performancemodule; - string moduleName = (*listPT).DeviceName; - int moduleID = atoi(moduleName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); + switch (nicID) + { + case 1: performancemodule.moduleIP1 = moduleIPAddr; break; - string moduleDisableState = (*listPT).DisableState; + case 2: performancemodule.moduleIP2 = moduleIPAddr; break; + case 3: performancemodule.moduleIP3 = moduleIPAddr; break; - //set Module Disable State - string moduleDisableStateParm = "ModuleDisableState" + oam.itoa(moduleID) + "-" + oam.itoa(i + 1); + case 4: performancemodule.moduleIP4 = moduleIPAddr; break; + } - try - { - sysConfigNew->setConfig(ModuleSection, moduleDisableStateParm, moduleDisableState); - } - catch (...) - { - cout << "ERROR: Problem setting ModuleDisableState in the Columnstore System Configuration file for " + moduleName << endl; - exit(-1); - } + if (maxPMNicCount < nicID) + maxPMNicCount = nicID; + } - for ( unsigned int nicID = 1 ; nicID < MAX_NIC + 1 ; nicID++ ) - { - string moduleIPAddr = oam::UnassignedIpAddr; - string moduleHostName = oam::UnassignedName; + if (nicID > 1) + continue; - HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); + // set port addresses + if (moduleName == systemParentOAMModuleName) + { + parentOAMModuleIPAddr = moduleIPAddr; - for ( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) - { - if ( moduleName == (*listPT).DeviceName && (*pt1).NicID == nicID) - { - moduleIPAddr = (*pt1).IPAddr; - moduleHostName = (*pt1).HostName; - break; - } - } + // exit out if parentOAMModuleIPAddr is NOT set, this means the System Columnstore.xml isn't + // configured + if (parentOAMModuleIPAddr == "0.0.0.0") + { + cout << "ERROR: System Columnstore.xml not configured" << endl; + exit(-1); + } - if (moduleHostName.empty() || (moduleHostName == oam::UnassignedName) ) - // exit out to next module ID - break; + // set Parent Processes Port IP Address + string parentProcessMonitor = systemParentOAMModuleName + "_ProcessMonitor"; + sysConfigNew->setConfig(parentProcessMonitor, "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig(parentProcessMonitor, "Port", "8800"); + sysConfigNew->setConfig("ProcMgr", "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig("ProcMgr_Alarm", "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig("ProcStatusControl", "IPAddr", parentOAMModuleIPAddr); + string parentServerMonitor = systemParentOAMModuleName + "_ServerMonitor"; + sysConfigNew->setConfig(parentServerMonitor, "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig(parentServerMonitor, "Port", "8622"); - if (moduleIPAddr.empty()) - moduleIPAddr = oam::UnassignedIpAddr; + if (build3) + { + string portName = systemParentOAMModuleName + "_WriteEngineServer"; + sysConfigNew->setConfig(portName, "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig(portName, "Port", "8630"); + } + else + { + sysConfigNew->setConfig("DDLProc", "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig("DMLProc", "IPAddr", parentOAMModuleIPAddr); + } - string moduleNameDesc = moduleDesc + " #" + oam.itoa(moduleID); + if (IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM) + { + // set User Module's IP Addresses + string Section = "ExeMgr" + oam.itoa(moduleID); - //set New Module Host Name - string moduleHostNameParm = "ModuleHostName" + oam.itoa(moduleID) + "-" + oam.itoa(nicID) + "-" + oam.itoa(i + 1); + sysConfigNew->setConfig(Section, "IPAddr", moduleIPAddr); + sysConfigNew->setConfig(Section, "Port", "8601"); - try - { - sysConfigNew->setConfig(ModuleSection, moduleHostNameParm, moduleHostName); - } - catch (...) - { - cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; - exit(-1); - } + // set Performance Module's IP's to first NIC IP entered + sysConfigNew->setConfig("DDLProc", "IPAddr", moduleIPAddr); + sysConfigNew->setConfig("DMLProc", "IPAddr", moduleIPAddr); + } + } + else + { + // set child Process Monitor Port IP Address + string portName = moduleName + "_ProcessMonitor"; + sysConfigNew->setConfig(portName, "IPAddr", moduleIPAddr); + sysConfigNew->setConfig(portName, "Port", "8800"); - //set Module IP address - string moduleIPAddrNameParm = "ModuleIPAddr" + oam.itoa(moduleID) + "-" + oam.itoa(nicID) + "-" + oam.itoa(i + 1); + // set child Server Monitor Port IP Address + portName = moduleName + "_ServerMonitor"; + sysConfigNew->setConfig(portName, "IPAddr", moduleIPAddr); + sysConfigNew->setConfig(portName, "Port", "8622"); - try - { - sysConfigNew->setConfig(ModuleSection, moduleIPAddrNameParm, moduleIPAddr); - } - catch (...) - { - cout << "ERROR: Problem setting IP address in the Columnstore System Configuration file" << endl; - exit(-1); - } + // set Performance Module WriteEngineServer Port IP Address + if (moduleType == "pm" && build3) + { + portName = moduleName + "_WriteEngineServer"; + sysConfigNew->setConfig(portName, "IPAddr", moduleIPAddr); + sysConfigNew->setConfig(portName, "Port", "8630"); + } - if ( moduleType == "pm" ) - { + // set User Module's IP Addresses + if (moduleType == "um" || + (moduleType == "pm" && IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM)) + { + string Section = "ExeMgr" + oam.itoa(moduleID); - switch (nicID) - { - case 1: - performancemodule.moduleIP1 = moduleIPAddr; - break; + sysConfigNew->setConfig(Section, "IPAddr", moduleIPAddr); + sysConfigNew->setConfig(Section, "Port", "8601"); + } + } - case 2: - performancemodule.moduleIP2 = moduleIPAddr; - break; + // set Performance Module's IP's to first NIC IP entered + if (moduleName == "um1" && build3) + { + sysConfigNew->setConfig("DDLProc", "IPAddr", moduleIPAddr); + sysConfigNew->setConfig("DMLProc", "IPAddr", moduleIPAddr); + } - case 3: - performancemodule.moduleIP3 = moduleIPAddr; - break; + // setup DBRM processes + if (moduleName == systemParentOAMModuleName) + sysConfigNew->setConfig(dbrmMainProc, "IPAddr", moduleIPAddr); - case 4: - performancemodule.moduleIP4 = moduleIPAddr; - break; - } + // if ( moduleDisableState == oam::ENABLEDSTATE ) + //{ + DBRMworkernodeID++; + string DBRMSection = dbrmSubProc + oam.itoa(DBRMworkernodeID); + sysConfigNew->setConfig(DBRMSection, "IPAddr", moduleIPAddr); + sysConfigNew->setConfig(DBRMSection, "Module", moduleName); + //} + } // end of nicID loop - if ( maxPMNicCount < nicID ) - maxPMNicCount = nicID; - } + // set dbroot assigments + DeviceDBRootList::iterator pt3 = sysModuleTypeConfig.moduletypeconfig[i].ModuleDBRootList.begin(); - if ( nicID > 1 ) - continue; + // this will be empty if upgrading from 2.2 + if (sysModuleTypeConfig.moduletypeconfig[i].ModuleDBRootList.size() == 0) + { + if (!OLDbuild3 && moduleType == "pm") + { + int dbrootCount = dbrootCountPerModule; + string moduleCountParm = "ModuleDBRootCount" + oam.itoa(moduleID) + "-" + oam.itoa(i + 1); - //set port addresses - if ( moduleName == systemParentOAMModuleName ) - { - parentOAMModuleIPAddr = moduleIPAddr; + try + { + sysConfigNew->setConfig(ModuleSection, moduleCountParm, oam.itoa(dbrootCount)); + } + catch (...) + { + cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; + exit(-1); + } - //exit out if parentOAMModuleIPAddr is NOT set, this means the System Columnstore.xml isn't configured - if ( parentOAMModuleIPAddr == "0.0.0.0" ) - { - cout << "ERROR: System Columnstore.xml not configured" << endl; - exit(-1); - } + int entry = 1; - //set Parent Processes Port IP Address - string parentProcessMonitor = systemParentOAMModuleName + "_ProcessMonitor"; - sysConfigNew->setConfig(parentProcessMonitor, "IPAddr", parentOAMModuleIPAddr); - sysConfigNew->setConfig(parentProcessMonitor, "Port", "8800"); - sysConfigNew->setConfig("ProcMgr", "IPAddr", parentOAMModuleIPAddr); - sysConfigNew->setConfig("ProcMgr_Alarm", "IPAddr", parentOAMModuleIPAddr); - sysConfigNew->setConfig("ProcStatusControl", "IPAddr", parentOAMModuleIPAddr); - string parentServerMonitor = systemParentOAMModuleName + "_ServerMonitor"; - sysConfigNew->setConfig(parentServerMonitor, "IPAddr", parentOAMModuleIPAddr); - sysConfigNew->setConfig(parentServerMonitor, "Port", "8622"); + for (; entry < dbrootCountPerModule + 1; entry++) + { + int dbrootid = dbrootNum; - if (build3) - { - string portName = systemParentOAMModuleName + "_WriteEngineServer"; - sysConfigNew->setConfig(portName, "IPAddr", parentOAMModuleIPAddr); - sysConfigNew->setConfig(portName, "Port", "8630"); - } - else - { - sysConfigNew->setConfig("DDLProc", "IPAddr", parentOAMModuleIPAddr); - sysConfigNew->setConfig("DMLProc", "IPAddr", parentOAMModuleIPAddr); - } - - if ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) - { - - //set User Module's IP Addresses - string Section = "ExeMgr" + oam.itoa(moduleID); - - sysConfigNew->setConfig(Section, "IPAddr", moduleIPAddr); - sysConfigNew->setConfig(Section, "Port", "8601"); - - //set Performance Module's IP's to first NIC IP entered - sysConfigNew->setConfig("DDLProc", "IPAddr", moduleIPAddr); - sysConfigNew->setConfig("DMLProc", "IPAddr", moduleIPAddr); - } - } - else - { - //set child Process Monitor Port IP Address - string portName = moduleName + "_ProcessMonitor"; - sysConfigNew->setConfig(portName, "IPAddr", moduleIPAddr); - sysConfigNew->setConfig(portName, "Port", "8800"); - - //set child Server Monitor Port IP Address - portName = moduleName + "_ServerMonitor"; - sysConfigNew->setConfig(portName, "IPAddr", moduleIPAddr); - sysConfigNew->setConfig(portName, "Port", "8622"); - - //set Performance Module WriteEngineServer Port IP Address - if ( moduleType == "pm" && build3) - { - portName = moduleName + "_WriteEngineServer"; - sysConfigNew->setConfig(portName, "IPAddr", moduleIPAddr); - sysConfigNew->setConfig(portName, "Port", "8630"); - } - - //set User Module's IP Addresses - if ( moduleType == "um" || - ( moduleType == "pm" && IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) ) - { - - string Section = "ExeMgr" + oam.itoa(moduleID); - - sysConfigNew->setConfig(Section, "IPAddr", moduleIPAddr); - sysConfigNew->setConfig(Section, "Port", "8601"); - } - } - - //set Performance Module's IP's to first NIC IP entered - if ( moduleName == "um1" && build3) - { - sysConfigNew->setConfig("DDLProc", "IPAddr", moduleIPAddr); - sysConfigNew->setConfig("DMLProc", "IPAddr", moduleIPAddr); - } - - //setup DBRM processes - if ( moduleName == systemParentOAMModuleName ) - sysConfigNew->setConfig(dbrmMainProc, "IPAddr", moduleIPAddr); - - //if ( moduleDisableState == oam::ENABLEDSTATE ) - //{ - DBRMworkernodeID++; - string DBRMSection = dbrmSubProc + oam.itoa(DBRMworkernodeID); - sysConfigNew->setConfig(DBRMSection, "IPAddr", moduleIPAddr); - sysConfigNew->setConfig(DBRMSection, "Module", moduleName); - //} - } //end of nicID loop - - //set dbroot assigments - DeviceDBRootList::iterator pt3 = sysModuleTypeConfig.moduletypeconfig[i].ModuleDBRootList.begin(); - - //this will be empty if upgrading from 2.2 - if ( sysModuleTypeConfig.moduletypeconfig[i].ModuleDBRootList.size() == 0 ) - { - if ( !OLDbuild3 && moduleType == "pm") - { - int dbrootCount = dbrootCountPerModule; - string moduleCountParm = "ModuleDBRootCount" + oam.itoa(moduleID) + "-" + oam.itoa(i + 1); - - try - { - sysConfigNew->setConfig(ModuleSection, moduleCountParm, oam.itoa(dbrootCount)); - } - catch (...) - { - cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; - exit(-1); - } - - int entry = 1; - - for ( ; entry < dbrootCountPerModule + 1 ; entry++) - { - int dbrootid = dbrootNum; - - if ( dbrootNum > systemDBRootCount ) - dbrootid = 0; - else - dbrootNum++; - - string moduleDBRootIDParm = "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(entry) + "-" + oam.itoa(i + 1); - - try - { - sysConfigNew->setConfig(ModuleSection, moduleDBRootIDParm, oam.itoa(dbrootid)); - } - catch (...) - { - cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - } - } + if (dbrootNum > systemDBRootCount) + dbrootid = 0; else + dbrootNum++; + + string moduleDBRootIDParm = + "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(entry) + "-" + oam.itoa(i + 1); + + try { - for ( ; pt3 != sysModuleTypeConfig.moduletypeconfig[i].ModuleDBRootList.end() ; pt3++) - { - if ( (*pt3).dbrootConfigList.size() > 0 ) - { - int moduleID = (*pt3).DeviceID; - - DBRootConfigList::iterator pt4 = (*pt3).dbrootConfigList.begin(); - - int dbrootCount = (*pt3).dbrootConfigList.size(); - string moduleCountParm = "ModuleDBRootCount" + oam.itoa(moduleID) + "-" + oam.itoa(i + 1); - - try - { - sysConfigNew->setConfig(ModuleSection, moduleCountParm, oam.itoa(dbrootCount)); - } - catch (...) - { - cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; - exit(-1); - } - - int entry = 1; - - for ( ; pt4 != (*pt3).dbrootConfigList.end() ; pt4++, entry++) - { - int dbrootid = *pt4; - - string moduleDBRootIDParm = "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(entry) + "-" + oam.itoa(i + 1); - - try - { - sysConfigNew->setConfig(ModuleSection, moduleDBRootIDParm, oam.itoa(dbrootid)); - } - catch (...) - { - cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - } - } + sysConfigNew->setConfig(ModuleSection, moduleDBRootIDParm, oam.itoa(dbrootid)); } - - if ( ( moduleType == "pm" ) || - ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) ) - performancemodulelist.push_back(performancemodule); - - } //end of module loop - - sysConfigNew->write(); - - } //end of i for loop - - if ( performancemodulelist.size() == 0 ) - { - cout << "ERROR: performancemodulelist is empty, exiting..." << endl; - exit(-1); - } - - //set dm count to 0 always - try - { - sysConfigNew->setConfig(ModuleSection, "ModuleCount1", "0"); - } - catch (...) - { - cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" << endl; - exit(-1); - } - - //setup DBRM Controller - sysConfigNew->setConfig(dbrmMainProc, numSubProc, oam.itoa(DBRMworkernodeID)); - - //setup PrimitiveServers parameters - try - { - sysConfigNew->setConfig("PrimitiveServers", "ConnectionsPerPrimProc", oam.itoa(maxPMNicCount * 2)); - } - catch (...) - { - cout << "ERROR: Problem setting ConnectionsPerPrimProc in the Columnstore System Configuration file" << endl; - exit(-1); - } - - //set the PM Ports based on Number of PM modules equipped, if any equipped - int minPmPorts = 32; - sysConfigNew->setConfig("PrimitiveServers", "Count", oam.itoa(pmNumber)); - - int pmPorts = pmNumber * (maxPMNicCount * 2); - - if ( pmPorts < minPmPorts ) - pmPorts = minPmPorts; - - if ( pmNumber > 0 || - ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) ) - { - const string PM = "PMS"; - - for ( int pmsID = 1; pmsID < pmPorts + 1 ; ) - { - for (unsigned int j = 1 ; j < maxPMNicCount + 1 ; j++) + catch (...) { - PerformanceModuleList::iterator list1 = performancemodulelist.begin(); - - for (; list1 != performancemodulelist.end() ; list1++) - { - string pmName = PM + oam.itoa(pmsID); - string IpAddr; - - switch (j) - { - case 1: - IpAddr = (*list1).moduleIP1; - break; - - case 2: - IpAddr = (*list1).moduleIP2; - break; - - case 3: - IpAddr = (*list1).moduleIP3; - break; - - case 4: - IpAddr = (*list1).moduleIP4; - break; - } - - if ( !IpAddr.empty() && IpAddr != oam::UnassignedIpAddr ) - { - sysConfigNew->setConfig(pmName, "IPAddr", IpAddr); - pmsID++; - - if ( pmsID > pmPorts ) - break; - } - } - - if ( pmsID > pmPorts ) - break; + cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; + exit(-1); } + } } - } + } + else + { + for (; pt3 != sysModuleTypeConfig.moduletypeconfig[i].ModuleDBRootList.end(); pt3++) + { + if ((*pt3).dbrootConfigList.size() > 0) + { + int moduleID = (*pt3).DeviceID; + + DBRootConfigList::iterator pt4 = (*pt3).dbrootConfigList.begin(); + + int dbrootCount = (*pt3).dbrootConfigList.size(); + string moduleCountParm = "ModuleDBRootCount" + oam.itoa(moduleID) + "-" + oam.itoa(i + 1); + + try + { + sysConfigNew->setConfig(ModuleSection, moduleCountParm, oam.itoa(dbrootCount)); + } + catch (...) + { + cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" << endl; + exit(-1); + } + + int entry = 1; + + for (; pt4 != (*pt3).dbrootConfigList.end(); pt4++, entry++) + { + int dbrootid = *pt4; + + string moduleDBRootIDParm = + "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(entry) + "-" + oam.itoa(i + 1); + + try + { + sysConfigNew->setConfig(ModuleSection, moduleDBRootIDParm, oam.itoa(dbrootid)); + } + catch (...) + { + cout << "ERROR: Problem setting Host Name in the Columnstore System Configuration file" + << endl; + exit(-1); + } + } + } + } + } + + if ((moduleType == "pm") || (IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM)) + performancemodulelist.push_back(performancemodule); + + } // end of module loop sysConfigNew->write(); - // - // Configure NMS Addresses - // + } // end of i for loop - string NMSIPAddress; + if (performancemodulelist.size() == 0) + { + cout << "ERROR: performancemodulelist is empty, exiting..." << endl; + exit(-1); + } + + // set dm count to 0 always + try + { + sysConfigNew->setConfig(ModuleSection, "ModuleCount1", "0"); + } + catch (...) + { + cout << "ERROR: Problem setting Module Count in the Columnstore System Configuration file" << endl; + exit(-1); + } + + // setup DBRM Controller + sysConfigNew->setConfig(dbrmMainProc, numSubProc, oam.itoa(DBRMworkernodeID)); + + // setup PrimitiveServers parameters + try + { + sysConfigNew->setConfig("PrimitiveServers", "ConnectionsPerPrimProc", oam.itoa(maxPMNicCount * 2)); + } + catch (...) + { + cout << "ERROR: Problem setting ConnectionsPerPrimProc in the Columnstore System Configuration file" + << endl; + exit(-1); + } + + // set the PM Ports based on Number of PM modules equipped, if any equipped + int minPmPorts = 32; + sysConfigNew->setConfig("PrimitiveServers", "Count", oam.itoa(pmNumber)); + + int pmPorts = pmNumber * (maxPMNicCount * 2); + + if (pmPorts < minPmPorts) + pmPorts = minPmPorts; + + if (pmNumber > 0 || (IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM)) + { + const string PM = "PMS"; + + for (int pmsID = 1; pmsID < pmPorts + 1;) + { + for (unsigned int j = 1; j < maxPMNicCount + 1; j++) + { + PerformanceModuleList::iterator list1 = performancemodulelist.begin(); + + for (; list1 != performancemodulelist.end(); list1++) + { + string pmName = PM + oam.itoa(pmsID); + string IpAddr; + + switch (j) + { + case 1: IpAddr = (*list1).moduleIP1; break; + + case 2: IpAddr = (*list1).moduleIP2; break; + + case 3: IpAddr = (*list1).moduleIP3; break; + + case 4: IpAddr = (*list1).moduleIP4; break; + } + + if (!IpAddr.empty() && IpAddr != oam::UnassignedIpAddr) + { + sysConfigNew->setConfig(pmName, "IPAddr", IpAddr); + pmsID++; + + if (pmsID > pmPorts) + break; + } + } + + if (pmsID > pmPorts) + break; + } + } + } + + sysConfigNew->write(); + + // + // Configure NMS Addresses + // + + string NMSIPAddress; + + try + { + NMSIPAddress = sysConfigOld->getConfig(SystemSection, "NMSIPAddress"); + } + catch (...) + { + cout << "ERROR: Problem getting NMSIPAddress from Columnstore System Configuration file" << endl; + exit(-1); + } + + try + { + sysConfigNew->setConfig(SystemSection, "NMSIPAddress", NMSIPAddress); + } + catch (...) + { + cout << "ERROR: Problem setting NMSIPAddress in the Columnstore System Configuration file" << endl; + exit(-1); + } + + // + // setup TransactionArchivePeriod + // + + string transactionArchivePeriod; + + try + { + transactionArchivePeriod = sysConfigOld->getConfig(SystemSection, "TransactionArchivePeriod"); + } + catch (...) + { + cout << "ERROR: Problem getting transactionArchivePeriod from Columnstore System Configuration file" + << endl; + exit(-1); + } + + try + { + sysConfigNew->setConfig(SystemSection, "TransactionArchivePeriod", transactionArchivePeriod); + } + catch (...) + { + cout << "ERROR: Problem setting IP address in the Columnstore System Configuration file" << endl; + exit(-1); + } + + // + // 3 and above configuration items + // + if (build3) + { + // setup cloud parameters + string UMStorageType; + string PMInstanceType; + string UMInstanceType; + string UMSecurityGroup; + string UMVolumeSize; + string PMVolumeSize; + string AmazonAutoTagging; + string AmazonVPCNextPrivateIP; + string AmazonDeviceName; + string UMVolumeType; + string UMVolumeIOPS; + string PMVolumeType; + string PMVolumeIOPS; try { - NMSIPAddress = sysConfigOld->getConfig(SystemSection, "NMSIPAddress"); + cloud = sysConfigOld->getConfig(InstallSection, "Cloud"); + UMStorageType = sysConfigOld->getConfig(InstallSection, "UMStorageType"); + PMInstanceType = sysConfigOld->getConfig(InstallSection, "PMInstanceType"); + UMInstanceType = sysConfigOld->getConfig(InstallSection, "UMInstanceType"); + UMSecurityGroup = sysConfigOld->getConfig(InstallSection, "UMSecurityGroup"); + UMVolumeSize = sysConfigOld->getConfig(InstallSection, "UMVolumeSize"); + PMVolumeSize = sysConfigOld->getConfig(InstallSection, "PMVolumeSize"); + AmazonAutoTagging = sysConfigOld->getConfig(InstallSection, "AmazonAutoTagging"); + AmazonVPCNextPrivateIP = sysConfigOld->getConfig(InstallSection, "AmazonVPCNextPrivateIP"); + AmazonDeviceName = sysConfigOld->getConfig(InstallSection, "AmazonDeviceName"); + UMVolumeType = sysConfigOld->getConfig(InstallSection, "UMVolumeType"); + UMVolumeIOPS = sysConfigOld->getConfig(InstallSection, "UMVolumeIOPS"); + PMVolumeType = sysConfigOld->getConfig(InstallSection, "PMVolumeType"); + PMVolumeIOPS = sysConfigOld->getConfig(InstallSection, "PMVolumeIOPS"); } catch (...) { - cout << "ERROR: Problem getting NMSIPAddress from Columnstore System Configuration file" << endl; - exit(-1); + } + + // this is for 2.2 to 4.x builds + if (UMStorageType.empty() || UMStorageType == "") + UMStorageType = "internal"; + + // 3.x upgrade + if (build3 && !build40 && !build401) + { + if (cloud == "no" || cloud == oam::UnassignedName) + cloud = "n"; + + if (cloud == "amazon-ec2" || cloud == "amazon-vpc") + cloud = "amazon"; + } + + // 4.0 upgrade + if (build40 && !build401) + { + if (cloud == "no" || cloud == "n") + cloud = oam::UnassignedName; + } + + // 4.0.1+ upgrade + if (build401) + { + if (cloud == "no" || cloud == "n") + cloud = oam::UnassignedName; + + if (cloud == "amazon") + cloud = "amazon-ec2"; + + if (AmazonVPCNextPrivateIP.empty()) + AmazonVPCNextPrivateIP = oam::UnassignedName; + + try + { + sysConfigNew->setConfig(InstallSection, "AmazonVPCNextPrivateIP", AmazonVPCNextPrivateIP); + } + catch (...) + { + // cout << "ERROR: Problem setting Cloud Parameters from the Columnstore System + //Configuration file" << endl; exit(-1); + } } try { - sysConfigNew->setConfig(SystemSection, "NMSIPAddress", NMSIPAddress); + sysConfigNew->setConfig(InstallSection, "Cloud", cloud); + sysConfigNew->setConfig(InstallSection, "UMStorageType", UMStorageType); + sysConfigNew->setConfig(InstallSection, "PMInstanceType", PMInstanceType); + sysConfigNew->setConfig(InstallSection, "UMInstanceType", UMInstanceType); + sysConfigNew->setConfig(InstallSection, "UMSecurityGroup", UMSecurityGroup); + sysConfigNew->setConfig(InstallSection, "UMVolumeSize", UMVolumeSize); + sysConfigNew->setConfig(InstallSection, "PMVolumeSize", PMVolumeSize); + sysConfigNew->setConfig(InstallSection, "AmazonAutoTagging", AmazonAutoTagging); + sysConfigNew->setConfig(InstallSection, "AmazonDeviceName", AmazonDeviceName); + + sysConfigNew->setConfig(InstallSection, "UMVolumeType", UMVolumeType); + sysConfigNew->setConfig(InstallSection, "UMVolumeIOPS", UMVolumeIOPS); + sysConfigNew->setConfig(InstallSection, "PMVolumeType", PMVolumeType); + sysConfigNew->setConfig(InstallSection, "PMVolumeIOPS", PMVolumeIOPS); } catch (...) { - cout << "ERROR: Problem setting NMSIPAddress in the Columnstore System Configuration file" << endl; - exit(-1); + // cout << "ERROR: Problem setting Cloud Parameters from the Columnstore System + //Configuration file" << endl; exit(-1); } - // - // setup TransactionArchivePeriod - // + if (cloud == "amazon-ec2" || cloud == "amazon-vpc") + cloud = "amazon"; - string transactionArchivePeriod; + // setup um storage + if (cloud == "amazon" && UMStorageType == "external") + { + try + { + systemStorageInfo_t t; + t = oam.getStorageConfig(); + ModuleTypeConfig moduletypeconfig; + oam.getSystemConfig("um", moduletypeconfig); + + for (int id = 1; id < moduletypeconfig.ModuleCount + 1; id++) + { + string volumeNameID = "UMVolumeName" + oam.itoa(id); + string volumeName = oam::UnassignedName; + string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id); + string deviceName = oam::UnassignedName; + + try + { + volumeName = sysConfigOld->getConfig(InstallSection, volumeNameID); + deviceName = sysConfigOld->getConfig(InstallSection, deviceNameID); + } + catch (...) + { + } + + try + { + sysConfigNew->setConfig(InstallSection, volumeNameID, volumeName); + sysConfigNew->setConfig(InstallSection, deviceNameID, deviceName); + } + catch (...) + { + } + } + } + catch (exception& e) + { + cout << endl << "**** getStorageConfig Failed : " << e.what() << endl; + } + } + + // setup dbroot storage try { - transactionArchivePeriod = sysConfigOld->getConfig(SystemSection, "TransactionArchivePeriod"); - } - catch (...) - { - cout << "ERROR: Problem getting transactionArchivePeriod from Columnstore System Configuration file" << endl; - exit(-1); - } + DBRootConfigList dbrootConfigList; + oam.getSystemDbrootConfig(dbrootConfigList); - try - { - sysConfigNew->setConfig(SystemSection, "TransactionArchivePeriod", transactionArchivePeriod); - } - catch (...) - { - cout << "ERROR: Problem setting IP address in the Columnstore System Configuration file" << endl; - exit(-1); - } - - // - // 3 and above configuration items - // - if (build3) - { - //setup cloud parameters - string UMStorageType; - string PMInstanceType; - string UMInstanceType; - string UMSecurityGroup; - string UMVolumeSize; - string PMVolumeSize; - string AmazonAutoTagging; - string AmazonVPCNextPrivateIP; - string AmazonDeviceName; - string UMVolumeType; - string UMVolumeIOPS; - string PMVolumeType; - string PMVolumeIOPS; + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + for (; pt != dbrootConfigList.end(); pt++) + { + int id = *pt; + string DBrootID = "DBRoot" + oam.itoa(id); + ; + string pathID = "/var/lib/columnstore/data" + oam.itoa(id); try { - cloud = sysConfigOld->getConfig(InstallSection, "Cloud"); - UMStorageType = sysConfigOld->getConfig(InstallSection, "UMStorageType"); - PMInstanceType = sysConfigOld->getConfig(InstallSection, "PMInstanceType"); - UMInstanceType = sysConfigOld->getConfig(InstallSection, "UMInstanceType"); - UMSecurityGroup = sysConfigOld->getConfig(InstallSection, "UMSecurityGroup"); + sysConfigNew->setConfig(SystemSection, DBrootID, pathID); + } + catch (...) + { + cout << "ERROR: Problem setting DBRoot in the Columnstore System Configuration file" << endl; + exit(-1); + } + + if (cloud == "amazon" && DBRootStorageType == "external") + { + string volumeNameID = "PMVolumeName" + oam.itoa(id); + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + oam.itoa(id); + string deviceName = oam::UnassignedName; + string amazondeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(id); + string amazondeviceName = oam::UnassignedName; + + try + { + volumeName = sysConfigOld->getConfig(InstallSection, volumeNameID); + deviceName = sysConfigOld->getConfig(InstallSection, deviceNameID); + amazondeviceName = sysConfigOld->getConfig(InstallSection, amazondeviceNameID); + } + catch (...) + { + } + + try + { + sysConfigNew->setConfig(InstallSection, volumeNameID, volumeName); + sysConfigNew->setConfig(InstallSection, deviceNameID, deviceName); + sysConfigNew->setConfig(InstallSection, amazondeviceNameID, amazondeviceName); + } + catch (...) + { + } + + string UMVolumeSize = oam::UnassignedName; + string PMVolumeSize = oam::UnassignedName; + + try + { UMVolumeSize = sysConfigOld->getConfig(InstallSection, "UMVolumeSize"); PMVolumeSize = sysConfigOld->getConfig(InstallSection, "PMVolumeSize"); - AmazonAutoTagging = sysConfigOld->getConfig(InstallSection, "AmazonAutoTagging"); - AmazonVPCNextPrivateIP = sysConfigOld->getConfig(InstallSection, "AmazonVPCNextPrivateIP"); - AmazonDeviceName = sysConfigOld->getConfig(InstallSection, "AmazonDeviceName"); - UMVolumeType = sysConfigOld->getConfig(InstallSection, "UMVolumeType"); - UMVolumeIOPS = sysConfigOld->getConfig(InstallSection, "UMVolumeIOPS"); - PMVolumeType = sysConfigOld->getConfig(InstallSection, "PMVolumeType"); - PMVolumeIOPS = sysConfigOld->getConfig(InstallSection, "PMVolumeIOPS"); - } - catch (...) - { } + } + catch (...) + { + } - //this is for 2.2 to 4.x builds - if ( UMStorageType.empty() || UMStorageType == "" ) - UMStorageType = "internal"; - - // 3.x upgrade - if (build3 && !build40 && !build401) - { - if ( cloud == "no" || cloud == oam::UnassignedName) - cloud = "n"; - - if ( cloud == "amazon-ec2" || cloud == "amazon-vpc") - cloud = "amazon"; - } - - // 4.0 upgrade - if (build40 && !build401) - { - if ( cloud == "no" || cloud == "n" ) - cloud = oam::UnassignedName; - } - - // 4.0.1+ upgrade - if (build401) - { - if ( cloud == "no" || cloud == "n" ) - cloud = oam::UnassignedName; - - if ( cloud == "amazon") - cloud = "amazon-ec2"; - - if ( AmazonVPCNextPrivateIP.empty() ) - AmazonVPCNextPrivateIP = oam::UnassignedName; - - try - { - sysConfigNew->setConfig(InstallSection, "AmazonVPCNextPrivateIP", AmazonVPCNextPrivateIP); - } - catch (...) - { - // cout << "ERROR: Problem setting Cloud Parameters from the Columnstore System Configuration file" << endl; - // exit(-1); - } - } - - try - { - sysConfigNew->setConfig(InstallSection, "Cloud", cloud); - sysConfigNew->setConfig(InstallSection, "UMStorageType", UMStorageType); - sysConfigNew->setConfig(InstallSection, "PMInstanceType", PMInstanceType); - sysConfigNew->setConfig(InstallSection, "UMInstanceType", UMInstanceType); - sysConfigNew->setConfig(InstallSection, "UMSecurityGroup", UMSecurityGroup); + try + { sysConfigNew->setConfig(InstallSection, "UMVolumeSize", UMVolumeSize); sysConfigNew->setConfig(InstallSection, "PMVolumeSize", PMVolumeSize); - sysConfigNew->setConfig(InstallSection, "AmazonAutoTagging", AmazonAutoTagging); - sysConfigNew->setConfig(InstallSection, "AmazonDeviceName", AmazonDeviceName); + } + catch (...) + { + } + } - sysConfigNew->setConfig(InstallSection, "UMVolumeType", UMVolumeType); - sysConfigNew->setConfig(InstallSection, "UMVolumeIOPS", UMVolumeIOPS); - sysConfigNew->setConfig(InstallSection, "PMVolumeType", PMVolumeType); - sysConfigNew->setConfig(InstallSection, "PMVolumeIOPS", PMVolumeIOPS); + if (!DataRedundancyConfig.empty()) + { + try + { + string dbrootPMsID = "DBRoot" + oam.itoa(id) + "PMs"; + string dbrootPMs = sysConfigOld->getConfig("DataRedundancyConfig", dbrootPMsID); + + try + { + sysConfigNew->setConfig("DataRedundancyConfig", dbrootPMsID, dbrootPMs); + } + catch (...) + { + } + } + catch (...) + { + } + } + } + } + catch (exception& e) + { + cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; + } + } + else + { + // pre 3.0 only + + string DBRootStorageLoc; + + for (int i = 1; i < DBRootCount + 1; i++) + { + if (DBRootStorageType != "local") + { + string DBRootStorageLocID = "DBRootStorageLoc" + oam.itoa(i); + + try + { + DBRootStorageLoc = sysConfigOld->getConfig(InstallSection, DBRootStorageLocID); } catch (...) { - // cout << "ERROR: Problem setting Cloud Parameters from the Columnstore System Configuration file" << endl; - // exit(-1); + cout << "ERROR: Problem getting '" + DBRootStorageLocID + + "' from the Columnstore System Configuration file" + << endl; + exit(-1); } - - if ( cloud == "amazon-ec2" || cloud == "amazon-vpc") - cloud = "amazon"; - - //setup um storage - if ( cloud == "amazon" && UMStorageType == "external") - { - - try - { - systemStorageInfo_t t; - t = oam.getStorageConfig(); - - ModuleTypeConfig moduletypeconfig; - oam.getSystemConfig("um", moduletypeconfig); - - for ( int id = 1; id < moduletypeconfig.ModuleCount + 1 ; id++) - { - string volumeNameID = "UMVolumeName" + oam.itoa(id); - string volumeName = oam::UnassignedName; - string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id); - string deviceName = oam::UnassignedName; - - try - { - volumeName = sysConfigOld->getConfig(InstallSection, volumeNameID); - deviceName = sysConfigOld->getConfig(InstallSection, deviceNameID); - } - catch (...) - {} - - try - { - sysConfigNew->setConfig(InstallSection, volumeNameID, volumeName); - sysConfigNew->setConfig(InstallSection, deviceNameID, deviceName); - } - catch (...) - {} - } - } - catch (exception& e) - { - cout << endl << "**** getStorageConfig Failed : " << e.what() << endl; - } - } - - //setup dbroot storage try { - DBRootConfigList dbrootConfigList; - oam.getSystemDbrootConfig(dbrootConfigList); - - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - - for ( ; pt != dbrootConfigList.end() ; pt++) - { - int id = *pt; - string DBrootID = "DBRoot" + oam.itoa(id);; - string pathID = "/var/lib/columnstore/data" + oam.itoa(id); - - try - { - sysConfigNew->setConfig(SystemSection, DBrootID, pathID); - } - catch (...) - { - cout << "ERROR: Problem setting DBRoot in the Columnstore System Configuration file" << endl; - exit(-1); - } - - if ( cloud == "amazon" && DBRootStorageType == "external") - { - - string volumeNameID = "PMVolumeName" + oam.itoa(id); - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + oam.itoa(id); - string deviceName = oam::UnassignedName; - string amazondeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(id); - string amazondeviceName = oam::UnassignedName; - - try - { - volumeName = sysConfigOld->getConfig(InstallSection, volumeNameID); - deviceName = sysConfigOld->getConfig(InstallSection, deviceNameID); - amazondeviceName = sysConfigOld->getConfig(InstallSection, amazondeviceNameID); - } - catch (...) - {} - - try - { - sysConfigNew->setConfig(InstallSection, volumeNameID, volumeName); - sysConfigNew->setConfig(InstallSection, deviceNameID, deviceName); - sysConfigNew->setConfig(InstallSection, amazondeviceNameID, amazondeviceName); - } - catch (...) - {} - - - string UMVolumeSize = oam::UnassignedName; - string PMVolumeSize = oam::UnassignedName; - - try - { - UMVolumeSize = sysConfigOld->getConfig(InstallSection, "UMVolumeSize"); - PMVolumeSize = sysConfigOld->getConfig(InstallSection, "PMVolumeSize"); - } - catch (...) - {} - - try - { - sysConfigNew->setConfig(InstallSection, "UMVolumeSize", UMVolumeSize); - sysConfigNew->setConfig(InstallSection, "PMVolumeSize", PMVolumeSize); - } - catch (...) - {} - } - - if ( !DataRedundancyConfig.empty() ) - { - try - { - string dbrootPMsID = "DBRoot" + oam.itoa(id) + "PMs"; - string dbrootPMs = sysConfigOld->getConfig("DataRedundancyConfig", dbrootPMsID); - - try - { - sysConfigNew->setConfig("DataRedundancyConfig", dbrootPMsID, dbrootPMs); - } - catch (...) - {} - } - catch (...) - {} - } - } + sysConfigNew->setConfig(InstallSection, DBRootStorageLocID, DBRootStorageLoc); } - catch (exception& e) + catch (...) { - cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; + cout << "ERROR: Problem setting '" + DBRootStorageLocID + + "' in the Columnstore System Configuration file" + << endl; + exit(-1); } + } + + string DBrootID = "DBRoot" + oam.itoa(i); + string pathID = "/var/lib/columnstore/data" + oam.itoa(i); + + try + { + sysConfigNew->setConfig(SystemSection, DBrootID, pathID); + } + catch (...) + { + cout << "ERROR: Problem setting DBRoot in the Columnstore System Configuration file" << endl; + exit(-1); + } } - else + } + + // do elastic IP configuration + int AmazonElasticIPCount = 0; + + try + { + AmazonElasticIPCount = atoi(sysConfigOld->getConfig(InstallSection, "AmazonElasticIPCount").c_str()); + + if (AmazonElasticIPCount > 0) { - // pre 3.0 only + for (int id = 1; id < AmazonElasticIPCount + 1; id++) + { + string AmazonElasticModule = "AmazonElasticModule" + oam.itoa(id); + string ELmoduleName; + string AmazonElasticIPAddr = "AmazonElasticIPAddr" + oam.itoa(id); + string ELIPaddress; - string DBRootStorageLoc; + ELmoduleName = sysConfigOld->getConfig(InstallSection, AmazonElasticModule); + ELIPaddress = sysConfigOld->getConfig(InstallSection, AmazonElasticIPAddr); - for ( int i = 1 ; i < DBRootCount + 1 ; i++) + try { - if ( DBRootStorageType != "local") - { - - string DBRootStorageLocID = "DBRootStorageLoc" + oam.itoa(i); - - try - { - DBRootStorageLoc = sysConfigOld->getConfig(InstallSection, DBRootStorageLocID); - } - catch (...) - { - cout << "ERROR: Problem getting '" + DBRootStorageLocID + "' from the Columnstore System Configuration file" << endl; - exit(-1); - } - - try - { - sysConfigNew->setConfig(InstallSection, DBRootStorageLocID, DBRootStorageLoc); - } - catch (...) - { - cout << "ERROR: Problem setting '" + DBRootStorageLocID + "' in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - - string DBrootID = "DBRoot" + oam.itoa(i); - string pathID = "/var/lib/columnstore/data" + oam.itoa(i); - - try - { - sysConfigNew->setConfig(SystemSection, DBrootID, pathID); - } - catch (...) - { - cout << "ERROR: Problem setting DBRoot in the Columnstore System Configuration file" << endl; - exit(-1); - } + sysConfigNew->setConfig(InstallSection, "AmazonElasticIPCount", oam.itoa(AmazonElasticIPCount)); + sysConfigNew->setConfig(InstallSection, AmazonElasticModule, ELmoduleName); + sysConfigNew->setConfig(InstallSection, AmazonElasticIPAddr, ELIPaddress); } - } - - //do elastic IP configuration - int AmazonElasticIPCount = 0; - - try - { - AmazonElasticIPCount = atoi(sysConfigOld->getConfig(InstallSection, "AmazonElasticIPCount").c_str()); - - if (AmazonElasticIPCount > 0 ) + catch (...) { - for ( int id = 1 ; id < AmazonElasticIPCount + 1 ; id++ ) - { - string AmazonElasticModule = "AmazonElasticModule" + oam.itoa(id); - string ELmoduleName; - string AmazonElasticIPAddr = "AmazonElasticIPAddr" + oam.itoa(id); - string ELIPaddress; - - ELmoduleName = sysConfigOld->getConfig(InstallSection, AmazonElasticModule); - ELIPaddress = sysConfigOld->getConfig(InstallSection, AmazonElasticIPAddr); - - try - { - sysConfigNew->setConfig(InstallSection, "AmazonElasticIPCount", oam.itoa(AmazonElasticIPCount)); - sysConfigNew->setConfig(InstallSection, AmazonElasticModule, ELmoduleName); - sysConfigNew->setConfig(InstallSection, AmazonElasticIPAddr, ELIPaddress); - } - catch (...) - {} - } } + } } - catch (...) - {} + } + catch (...) + { + } - try + try + { + oam.getSystemConfig("AmazonElasticIPCount", AmazonElasticIPCount); + } + catch (...) + { + AmazonElasticIPCount = 0; + } + + // ConcurrentTransactions + string ConcurrentTransactions; + + try + { + ConcurrentTransactions = sysConfigOld->getConfig(SystemSection, "ConcurrentTransactions"); + + if (!ConcurrentTransactions.empty()) { - oam.getSystemConfig("AmazonElasticIPCount", AmazonElasticIPCount); + try + { + sysConfigNew->setConfig(SystemSection, "ConcurrentTransactions", ConcurrentTransactions); + } + catch (...) + { + cout << "ERROR: Problem setting ConcurrentTransactions in the Columnstore System Configuration file" + << endl; + exit(-1); + } } - catch (...) + } + catch (...) + { + } + + // NetworkCompression Enabled + string NetworkCompression; + + try + { + NetworkCompression = sysConfigOld->getConfig("NetworkCompression", "Enabled"); + + if (!NetworkCompression.empty()) { - AmazonElasticIPCount = 0; + try + { + sysConfigNew->setConfig("NetworkCompression", "Enabled", NetworkCompression); + } + catch (...) + { + cout << "ERROR: Problem setting NetworkCompression in the Columnstore System Configuration file" + << endl; + exit(-1); + } } + } + catch (...) + { + } - // ConcurrentTransactions - string ConcurrentTransactions; + // hadoop + string DataFilePlugin; - try + try + { + DataFilePlugin = sysConfigOld->getConfig(SystemSection, "DataFilePlugin"); + + if (!DataFilePlugin.empty()) { - ConcurrentTransactions = sysConfigOld->getConfig(SystemSection, "ConcurrentTransactions"); + try + { + sysConfigNew->setConfig(SystemSection, "DataFilePlugin", DataFilePlugin); + } + catch (...) + { + cout << "ERROR: Problem setting DataFilePlugin in the Columnstore System Configuration file" << endl; + exit(-1); + } - if ( !ConcurrentTransactions.empty() ) + string ExtentsPerSegmentFile; + + try + { + ExtentsPerSegmentFile = sysConfigOld->getConfig("ExtentMap", "ExtentsPerSegmentFile"); + + try { - try - { - sysConfigNew->setConfig(SystemSection, "ConcurrentTransactions", ConcurrentTransactions); - } - catch (...) - { - cout << "ERROR: Problem setting ConcurrentTransactions in the Columnstore System Configuration file" << endl; - exit(-1); - } + sysConfigNew->setConfig("ExtentMap", "ExtentsPerSegmentFile", ExtentsPerSegmentFile); } + catch (...) + { + cout << "ERROR: Problem setting ExtentsPerSegmentFile in the Columnstore System Configuration file" + << endl; + exit(-1); + } + } + catch (...) + { + } } - catch (...) - {} + } + catch (...) + { + } - // NetworkCompression Enabled - string NetworkCompression; + string DataFileLog; - try + try + { + DataFileLog = sysConfigOld->getConfig(SystemSection, "DataFileLog"); + + if (!DataFileLog.empty()) { - NetworkCompression = sysConfigOld->getConfig("NetworkCompression", "Enabled"); - - if ( !NetworkCompression.empty() ) - { - try - { - sysConfigNew->setConfig("NetworkCompression", "Enabled", NetworkCompression); - } - catch (...) - { - cout << "ERROR: Problem setting NetworkCompression in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig(SystemSection, "DataFileLog", DataFileLog); + } + catch (...) + { + cout << "ERROR: Problem setting DataFileLog in the Columnstore System Configuration file" << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } - //hadoop - string DataFilePlugin; + string AllowDiskBasedJoin; + string TempFileCompression; - try + try + { + AllowDiskBasedJoin = sysConfigOld->getConfig("HashJoin", "AllowDiskBasedJoin"); + + if (!AllowDiskBasedJoin.empty()) { - DataFilePlugin = sysConfigOld->getConfig(SystemSection, "DataFilePlugin"); + TempFileCompression = sysConfigOld->getConfig("HashJoin", "TempFileCompression"); - if ( !DataFilePlugin.empty() ) - { - try - { - sysConfigNew->setConfig(SystemSection, "DataFilePlugin", DataFilePlugin); - } - catch (...) - { - cout << "ERROR: Problem setting DataFilePlugin in the Columnstore System Configuration file" << endl; - exit(-1); - } - - string ExtentsPerSegmentFile; - - try - { - ExtentsPerSegmentFile = sysConfigOld->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - try - { - sysConfigNew->setConfig("ExtentMap", "ExtentsPerSegmentFile", ExtentsPerSegmentFile); - } - catch (...) - { - cout << "ERROR: Problem setting ExtentsPerSegmentFile in the Columnstore System Configuration file" << endl; - exit(-1); - } - } - catch (...) - {} - } + try + { + sysConfigNew->setConfig("HashJoin", "AllowDiskBasedJoin", AllowDiskBasedJoin); + sysConfigNew->setConfig("HashJoin", "TempFileCompression", TempFileCompression); + } + catch (...) + { + cout << "ERROR: Problem setting AllowDiskBasedJoin in the Columnstore System Configuration file" + << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } - string DataFileLog; + string AllowDiskBasedAggregation; + // string TempFileCompression; - try + try + { + AllowDiskBasedAggregation = sysConfigOld->getConfig("RowAggregation", "AllowDiskBasedAggregation"); + + if (!AllowDiskBasedAggregation.empty()) { - DataFileLog = sysConfigOld->getConfig(SystemSection, "DataFileLog"); - - if ( !DataFileLog.empty() ) - { - try - { - sysConfigNew->setConfig(SystemSection, "DataFileLog", DataFileLog); - } - catch (...) - { - cout << "ERROR: Problem setting DataFileLog in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig("RowAggregation", "AllowDiskBasedAggregation", AllowDiskBasedAggregation); + } + catch (...) + { + cout + << "ERROR: Problem setting AllowDiskBasedAggregation in the Columnstore System Configuration file" + << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } + string SystemTempFileDir; - string AllowDiskBasedJoin; - string TempFileCompression; + try + { + SystemTempFileDir = sysConfigOld->getConfig("SystemConfig", "SystemTempFileDir"); - try + if (!SystemTempFileDir.empty()) { - AllowDiskBasedJoin = sysConfigOld->getConfig("HashJoin", "AllowDiskBasedJoin"); + SystemTempFileDir = sysConfigOld->getConfig("SystemConfig", "SystemTempFileDir"); - if ( !AllowDiskBasedJoin.empty() ) - { - TempFileCompression = sysConfigOld->getConfig("HashJoin", "TempFileCompression"); - - try - { - sysConfigNew->setConfig("HashJoin", "AllowDiskBasedJoin", AllowDiskBasedJoin); - sysConfigNew->setConfig("HashJoin", "TempFileCompression", TempFileCompression); - } - catch (...) - { - cout << "ERROR: Problem setting AllowDiskBasedJoin in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig("SystemConfig", "SystemTempFileDir", SystemTempFileDir); + } + catch (...) + { + cout << "ERROR: Problem setting SystemTempFileDir in the Columnstore System Configuration file" + << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } - string AllowDiskBasedAggregation; -// string TempFileCompression; + try + { + Host = sysConfigOld->getConfig("QueryTele", "Host"); - try + if (!Host.empty()) { - AllowDiskBasedAggregation = sysConfigOld->getConfig("RowAggregation", "AllowDiskBasedAggregation"); + Port = sysConfigOld->getConfig("QueryTele", "Port"); - if ( !AllowDiskBasedAggregation.empty() ) - { - try - { - sysConfigNew->setConfig("RowAggregation", "AllowDiskBasedAggregation", AllowDiskBasedAggregation); - } - catch (...) - { - cout << "ERROR: Problem setting AllowDiskBasedAggregation in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig("QueryTele", "Host", Host); + sysConfigNew->setConfig("QueryTele", "Port", Port); + } + catch (...) + { + cout << "ERROR: Problem setting QueryTele in the Columnstore System Configuration file" << endl; + exit(-1); + } } - catch (...) - {} - - string SystemTempFileDir; + } + catch (...) + { + } - try + try + { + string AmazonAccessKey = sysConfigOld->getConfig("Installation", "AmazonAccessKey"); + + if (!AmazonAccessKey.empty()) { - SystemTempFileDir = sysConfigOld->getConfig("SystemConfig", "SystemTempFileDir"); - - if ( !SystemTempFileDir.empty() ) - { - SystemTempFileDir = sysConfigOld->getConfig("SystemConfig", "SystemTempFileDir"); - - try - { - sysConfigNew->setConfig("SystemConfig", "SystemTempFileDir", SystemTempFileDir); - } - catch (...) - { - cout << "ERROR: Problem setting SystemTempFileDir in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig("Installation", "AmazonAccessKey", AmazonAccessKey); + } + catch (...) + { + cout << "ERROR: Problem setting AmazonAccessKey in the Columnstore System Configuration file" << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } - try + try + { + string AmazonSecretKey = sysConfigOld->getConfig("Installation", "AmazonSecretKey"); + + if (!AmazonSecretKey.empty()) { - Host = sysConfigOld->getConfig("QueryTele", "Host"); - - if ( !Host.empty() ) - { - Port = sysConfigOld->getConfig("QueryTele", "Port"); - - try - { - sysConfigNew->setConfig("QueryTele", "Host", Host); - sysConfigNew->setConfig("QueryTele", "Port", Port); - } - catch (...) - { - cout << "ERROR: Problem setting QueryTele in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig("Installation", "AmazonSecretKey", AmazonSecretKey); + } + catch (...) + { + cout << "ERROR: Problem setting AmazonSecretKey in the Columnstore System Configuration file" << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } - try + try + { + string LockFileDirectory = sysConfigOld->getConfig("Installation", "LockFileDirectory"); + + if (!LockFileDirectory.empty()) { - string AmazonAccessKey = sysConfigOld->getConfig("Installation", "AmazonAccessKey"); - - if ( !AmazonAccessKey.empty() ) - { - try - { - sysConfigNew->setConfig("Installation", "AmazonAccessKey", AmazonAccessKey); - } - catch (...) - { - cout << "ERROR: Problem setting AmazonAccessKey in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + try + { + sysConfigNew->setConfig("Installation", "LockFileDirectory", LockFileDirectory); + } + catch (...) + { + cout << "ERROR: Problem setting LockFileDirectory in the Columnstore System Configuration file" + << endl; + exit(-1); + } } - catch (...) - {} + } + catch (...) + { + } - try + // add entries from tuning guide + + string ColScanReadAheadBlocks; + string PrefetchThreshold; + string MaxOutstandingRequests; + string PmMaxMemorySmallSide; + string ThreadPoolSize; + + try + { + ColScanReadAheadBlocks = sysConfigOld->getConfig("PrimitiveServers", "ColScanReadAheadBlocks"); + PrefetchThreshold = sysConfigOld->getConfig("PrimitiveServers", "PrefetchThreshold"); + PmMaxMemorySmallSide = sysConfigOld->getConfig("HashJoin", "PmMaxMemorySmallSide"); + ThreadPoolSize = sysConfigOld->getConfig("JobList", "ThreadPoolSize"); + } + catch (...) + { + } + + try + { + sysConfigNew->setConfig("PrimitiveServers", "ColScanReadAheadBlocks", ColScanReadAheadBlocks); + sysConfigNew->setConfig("PrimitiveServers", "PrefetchThreshold", PrefetchThreshold); + sysConfigNew->setConfig("HashJoin", "PmMaxMemorySmallSide", PmMaxMemorySmallSide); + sysConfigNew->setConfig("JobList", "ThreadPoolSize", ThreadPoolSize); + } + catch (...) + { + } + + // ExeMgr Optional settings + try + { + // threadpool size. This is the size the pool will idle down to if it grows bigger + string threadPoolSize; + // Time between checks to see if memory is exausted + string secondsBetweenMemChecks; + // Max percent of total memory used by everything before we kill ourself + string maxPct; + // Maximum number of concurrent queries. Any more will have to wait. + string execQueueSize; + threadPoolSize = sysConfigOld->getConfig("ExeMgr1", "ThreadPoolSize"); + if (!threadPoolSize.empty()) { - string AmazonSecretKey = sysConfigOld->getConfig("Installation", "AmazonSecretKey"); - - if ( !AmazonSecretKey.empty() ) - { - try - { - sysConfigNew->setConfig("Installation", "AmazonSecretKey", AmazonSecretKey); - } - catch (...) - { - cout << "ERROR: Problem setting AmazonSecretKey in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + sysConfigNew->setConfig("ExeMgr1", "ThreadPoolSize", threadPoolSize); } - catch (...) - {} - try + secondsBetweenMemChecks = sysConfigOld->getConfig("ExeMgr1", "SecondsBetweenMemChecks"); + if (!secondsBetweenMemChecks.empty()) { - string LockFileDirectory = sysConfigOld->getConfig("Installation", "LockFileDirectory"); - - if ( !LockFileDirectory.empty() ) - { - try - { - sysConfigNew->setConfig("Installation", "LockFileDirectory", LockFileDirectory); - } - catch (...) - { - cout << "ERROR: Problem setting LockFileDirectory in the Columnstore System Configuration file" << endl; - exit(-1); - } - } + sysConfigNew->setConfig("ExeMgr1", "SecondsBetweenMemChecks", secondsBetweenMemChecks); } - catch (...) - {} - - // add entries from tuning guide - - string ColScanReadAheadBlocks; - string PrefetchThreshold; - string MaxOutstandingRequests; - string PmMaxMemorySmallSide; - string ThreadPoolSize; - - - try + maxPct = sysConfigOld->getConfig("ExeMgr1", "MaxPct"); + if (!maxPct.empty()) { - ColScanReadAheadBlocks = sysConfigOld->getConfig("PrimitiveServers", "ColScanReadAheadBlocks"); - PrefetchThreshold = sysConfigOld->getConfig("PrimitiveServers", "PrefetchThreshold"); - PmMaxMemorySmallSide = sysConfigOld->getConfig("HashJoin", "PmMaxMemorySmallSide"); - ThreadPoolSize = sysConfigOld->getConfig("JobList", "ThreadPoolSize"); + sysConfigNew->setConfig("ExeMgr1", "MaxPct", maxPct); } - catch (...) - {} - try + execQueueSize = sysConfigOld->getConfig("ExeMgr1", "ExecQueueSize"); + if (!execQueueSize.empty()) { - sysConfigNew->setConfig("PrimitiveServers", "ColScanReadAheadBlocks", ColScanReadAheadBlocks); - sysConfigNew->setConfig("PrimitiveServers", "PrefetchThreshold", PrefetchThreshold); - sysConfigNew->setConfig("HashJoin", "PmMaxMemorySmallSide", PmMaxMemorySmallSide); - sysConfigNew->setConfig("JobList", "ThreadPoolSize", ThreadPoolSize); + sysConfigNew->setConfig("ExeMgr1", "ExecQueueSize", execQueueSize); } - catch (...) - {} - - // ExeMgr Optional settings - try - { - // threadpool size. This is the size the pool will idle down to if it grows bigger - string threadPoolSize; - // Time between checks to see if memory is exausted - string secondsBetweenMemChecks; - // Max percent of total memory used by everything before we kill ourself - string maxPct; - // Maximum number of concurrent queries. Any more will have to wait. - string execQueueSize; - threadPoolSize = sysConfigOld->getConfig("ExeMgr1", "ThreadPoolSize"); - if ( !threadPoolSize.empty() ) - { - sysConfigNew->setConfig("ExeMgr1", "ThreadPoolSize", threadPoolSize); - } + } + catch (...) + { + } - secondsBetweenMemChecks = sysConfigOld->getConfig("ExeMgr1", "SecondsBetweenMemChecks"); - if ( !secondsBetweenMemChecks.empty() ) - { - sysConfigNew->setConfig("ExeMgr1", "SecondsBetweenMemChecks", secondsBetweenMemChecks); - } - - maxPct = sysConfigOld->getConfig("ExeMgr1", "MaxPct"); - if ( !maxPct.empty() ) - { - sysConfigNew->setConfig("ExeMgr1", "MaxPct", maxPct); - } - - execQueueSize = sysConfigOld->getConfig("ExeMgr1", "ExecQueueSize"); - if ( !execQueueSize.empty() ) - { - sysConfigNew->setConfig("ExeMgr1", "ExecQueueSize", execQueueSize); - } - } - catch (...) - {} - - // PrimProc optional parameters - // Max percent of total memory used by everything before we kill the current query - // For 5.6.1, this setting uses the same mechanism that ExeMgr uses to kill itself - try + // PrimProc optional parameters + // Max percent of total memory used by everything before we kill the current query + // For 5.6.1, this setting uses the same mechanism that ExeMgr uses to kill itself + try + { + string maxPct; + maxPct = sysConfigOld->getConfig("PrimitiveServers", "MaxPct"); + if (!maxPct.empty()) { - string maxPct; - maxPct = sysConfigOld->getConfig("PrimitiveServers", "MaxPct"); - if ( !maxPct.empty() ) - { - sysConfigNew->setConfig("PrimitiveServers", "MaxPct", maxPct); - } + sysConfigNew->setConfig("PrimitiveServers", "MaxPct", maxPct); } - catch (...) - {} - - //Write out Updated System Configuration File - sysConfigNew->write(); + } + catch (...) + { + } + + // Write out Updated System Configuration File + sysConfigNew->write(); } diff --git a/tools/cplogger/main.cpp b/tools/cplogger/main.cpp index 4bf673c57..e416aa857 100644 --- a/tools/cplogger/main.cpp +++ b/tools/cplogger/main.cpp @@ -31,72 +31,70 @@ namespace { void usage() { - cout << "usage: cplogger [-s subsys] [-cwi] [-h] msg_id [args...]" << endl; -} + cout << "usage: cplogger [-s subsys] [-cwi] [-h] msg_id [args...]" << endl; } +} // namespace int main(int argc, char** argv) { - int c; - opterr = 0; - bool cflg = true; - bool wflg = false; - unsigned subsysID = 8; //oamcpp + int c; + opterr = 0; + bool cflg = true; + bool wflg = false; + unsigned subsysID = 8; // oamcpp - while ((c = getopt(argc, argv, "s:cwih")) != EOF) - switch (c) - { - case 'c': - cflg = true; - wflg = false; - break; - - case 'w': - cflg = false; - wflg = true; - break; - - case 'i': - cflg = false; - wflg = false; - break; - - case 's': - subsysID = strtoul(optarg, 0, 0); - break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if ((argc - optind) < 1) + while ((c = getopt(argc, argv, "s:cwih")) != EOF) + switch (c) { + case 'c': + cflg = true; + wflg = false; + break; + + case 'w': + cflg = false; + wflg = true; + break; + + case 'i': + cflg = false; + wflg = false; + break; + + case 's': subsysID = strtoul(optarg, 0, 0); break; + + case 'h': + case '?': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } - Message::MessageID mid = strtoul(argv[optind++], 0, 0);; - Message::Args args; + if ((argc - optind) < 1) + { + usage(); + return 1; + } - for (int i = optind; i < argc; i++) - args.add(argv[optind++]); + Message::MessageID mid = strtoul(argv[optind++], 0, 0); + ; + Message::Args args; - LoggingID logInfo(subsysID); - Message msg(mid); - msg.format(args); - MessageLog log(logInfo); + for (int i = optind; i < argc; i++) + args.add(argv[optind++]); - if (cflg) - log.logCriticalMessage(msg); - else if (wflg) - log.logWarningMessage(msg); - else - log.logInfoMessage(msg); + LoggingID logInfo(subsysID); + Message msg(mid); + msg.format(args); + MessageLog log(logInfo); - return 0; + if (cflg) + log.logCriticalMessage(msg); + else if (wflg) + log.logWarningMessage(msg); + else + log.logInfoMessage(msg); + + return 0; } - diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 5885a6f78..38b9ffd54 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: dbbuilder.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -* -*******************************************************************************/ + * $Id: dbbuilder.cpp 2101 2013-01-21 14:12:52Z rdempsey $ + * + *******************************************************************************/ #include #include @@ -50,291 +50,278 @@ string logFile; enum BUILD_OPTION { - SYSCATALOG_ONLY = 7, //Create systables only + SYSCATALOG_ONLY = 7, // Create systables only }; namespace { - int setUp() { #ifndef _MSC_VER - string cmd = "/bin/rm -f " + logFile + " >/dev/null 2>&1"; - int rc = system(cmd.c_str()); - cmd = "/bin/touch -f " + logFile + " >/dev/null 2>&1"; - rc = system(cmd.c_str()); + string cmd = "/bin/rm -f " + logFile + " >/dev/null 2>&1"; + int rc = system(cmd.c_str()); + cmd = "/bin/touch -f " + logFile + " >/dev/null 2>&1"; + rc = system(cmd.c_str()); #endif - return rc; + return rc; } int checkNotThere(WriteEngine::FID fid) { - WriteEngine::FileOp fileOp; + WriteEngine::FileOp fileOp; - return (fileOp.existsOIDDir(fid) ? -1 : 0); + return (fileOp.existsOIDDir(fid) ? -1 : 0); } void usage() { - cerr << "Usage: dbbuilder [-h|f] function" << endl - << " -h Display this help info" << endl - << " -f Necessary to use any fcn other than 7" << endl - << " fcn" << endl - << " 7 Build system tables only" << endl - << endl - << "WARNING! Using this tool improperly can render your database unusable!" << endl - ; + cerr << "Usage: dbbuilder [-h|f] function" << endl + << " -h Display this help info" << endl + << " -f Necessary to use any fcn other than 7" << endl + << " fcn" << endl + << " 7 Build system tables only" << endl + << endl + << "WARNING! Using this tool improperly can render your database unusable!" << endl; } const unsigned sysCatalogErr = logging::M0060; -void errorHandler(const unsigned mid, - const string& src, - const string& msg, - bool isCritErr = true) +void errorHandler(const unsigned mid, const string& src, const string& msg, bool isCritErr = true) { - logging::LoggingID lid(19); - logging::MessageLog ml(lid); - logging::Message::Args args; - logging::Message message(mid); + logging::LoggingID lid(19); + logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message message(mid); - if (isCritErr) - { - args.add(string("error")); - args.add(msg); - message.format( args ); - ml.logCriticalMessage(message ); - cout << src << " was not successful. " << msg << endl; - } - else - { - args.add(string("status")); - args.add(msg); - message.format( args ); - ml.logInfoMessage(message ); - cout << src << " was not completed. " << msg << endl; - } + if (isCritErr) + { + args.add(string("error")); + args.add(msg); + message.format(args); + ml.logCriticalMessage(message); + cout << src << " was not successful. " << msg << endl; + } + else + { + args.add(string("status")); + args.add(msg); + message.format(args); + ml.logInfoMessage(message); + cout << src << " was not completed. " << msg << endl; + } } - -} +} // namespace int main(int argc, char* argv[]) { - int buildOption; - int c; - std::string schema("tpch"); - Oam oam; - bool fFlg = false; - int rc = 0; + int buildOption; + int c; + std::string schema("tpch"); + Oam oam; + bool fFlg = false; + int rc = 0; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "u:fh")) != EOF) - switch (c) - { - case 'u': - schema = optarg; - break; - - case 'f': - fFlg = true; - break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if ((argc - optind) < 1) + while ((c = getopt(argc, argv, "u:fh")) != EOF) + switch (c) { + case 'u': schema = optarg; break; + + case 'f': fFlg = true; break; + + case 'h': + case '?': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } - oamModuleInfo_t t; - bool parentOAMModuleFlag = false; + if ((argc - optind) < 1) + { + usage(); + return 1; + } + + oamModuleInfo_t t; + bool parentOAMModuleFlag = false; + + // get local module info; validate running on Active Parent OAM Module + try + { + t = oam.getModuleInfo(); + parentOAMModuleFlag = boost::get<4>(t); + } + catch (exception&) + { + parentOAMModuleFlag = true; + } + + if (!parentOAMModuleFlag) + { + cerr << "Exiting, dbbuilder can only be run on the Active " + "Parent OAM Module" + << endl; + return 1; + } + + logFile = string(MCSLOGDIR) + "/install/dbbuilder.status"; + + buildOption = atoi(argv[optind++]); + + if (buildOption != 7 && !fFlg) + { + usage(); + return 1; + } + + if (buildOption == SYSCATALOG_ONLY) + { + if (setUp()) + { + cerr << "setUp() call error " << endl; + } + + bool canWrite = true; + + if (access(logFile.c_str(), W_OK) != 0) + canWrite = false; - //get local module info; validate running on Active Parent OAM Module try { - t = oam.getModuleInfo(); - parentOAMModuleFlag = boost::get<4>(t); - } - catch (exception&) - { - parentOAMModuleFlag = true; - } + if (checkNotThere(1001) != 0) + { + string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; - if ( !parentOAMModuleFlag ) - { - cerr << "Exiting, dbbuilder can only be run on the Active " - "Parent OAM Module" << endl; - return 1; - } - - logFile = string(MCSLOGDIR) + "/install/dbbuilder.status"; - - buildOption = atoi(argv[optind++]); - - if (buildOption != 7 && !fFlg) - { - usage(); - return 1; - } - - if ( buildOption == SYSCATALOG_ONLY ) - { - if ( setUp() ) + if (canWrite) { - cerr << "setUp() call error " << endl; + rc = system(cmd.c_str()); + } + else + { + cerr << cmd << endl; } - bool canWrite = true; + errorHandler(sysCatalogErr, "Build system catalog", + "System catalog appears to exist. It will remain intact " + "for reuse. The database is not recreated.", + false); + return 1; + } - if (access(logFile.c_str(), W_OK) != 0) - canWrite = false; + //@bug5554, make sure IDBPolicy matches the Columnstore.xml config + try + { + string calpontConfigFile(std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml"); + config::Config* sysConfig = config::Config::makeConfig(calpontConfigFile.c_str()); + string tmp = sysConfig->getConfig("Installation", "DBRootStorageType"); - try + if (boost::iequals(tmp, "hdfs")) { - if (checkNotThere(1001) != 0) - { - string cmd = "echo 'FAILED: buildOption=" + - oam.itoa(buildOption) + - "' > " + logFile; + // HDFS is configured + if (!IDBPolicy::useHdfs()) // error install plugin + throw runtime_error("HDFS is not enabled, installPlugin may have failed."); - if (canWrite) - { - rc = system(cmd.c_str()); - } - else - { - cerr << cmd << endl; - } + else if (!IDBFileSystem::getFs(IDBDataFile::HDFS).filesystemIsUp()) + throw runtime_error("HDFS FS is NULL, check env variables."); + } + } + catch (const exception& ex) + { + string cmd(string("echo 'FAILED: ") + ex.what() + "' > " + logFile); - errorHandler(sysCatalogErr, - "Build system catalog", - "System catalog appears to exist. It will remain intact " - "for reuse. The database is not recreated.", - false); - return 1; - } + if (canWrite) + rc = system(cmd.c_str()); + else + cerr << cmd << endl; - //@bug5554, make sure IDBPolicy matches the Columnstore.xml config - try - { - string calpontConfigFile(std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml"); - config::Config* sysConfig = config::Config::makeConfig(calpontConfigFile.c_str()); - string tmp = sysConfig->getConfig("Installation", "DBRootStorageType"); + errorHandler(sysCatalogErr, "Build system catalog", ex.what(), false); + return 1; + } + catch (...) + { + string cmd = "echo 'FAILED: HDFS checking.' > " + logFile; - if (boost::iequals(tmp, "hdfs")) - { - // HDFS is configured - if (!IDBPolicy::useHdfs()) // error install plugin - throw runtime_error("HDFS is not enabled, installPlugin may have failed."); + if (canWrite) + rc = system(cmd.c_str()); + else + cerr << cmd << endl; - else if (!IDBFileSystem::getFs(IDBDataFile::HDFS).filesystemIsUp()) - throw runtime_error("HDFS FS is NULL, check env variables."); - } - } - catch (const exception& ex) - { - string cmd(string("echo 'FAILED: ") + ex.what() + "' > " + logFile); + errorHandler(sysCatalogErr, "Build system catalog", "HDFS check failed.", false); + return 1; + } - if (canWrite) - rc = system(cmd.c_str()); - else - cerr << cmd << endl; + // create an initial oid bitmap file + { + ObjectIDManager oidm; + } - errorHandler(sysCatalogErr, "Build system catalog", ex.what(), false); - return 1; - } - catch (...) - { - string cmd = "echo 'FAILED: HDFS checking.' > " + logFile; + SystemCatalog sysCatalog; + sysCatalog.build(); - if (canWrite) - rc = system(cmd.c_str()); - else - cerr << cmd << endl; + std::string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; - errorHandler(sysCatalogErr, "Build system catalog", "HDFS check failed.", false); - return 1; - } - - //create an initial oid bitmap file - { - ObjectIDManager oidm; - } - - SystemCatalog sysCatalog; - sysCatalog.build(); - - std::string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; - - if (canWrite) - rc = system(cmd.c_str()); - else + if (canWrite) + rc = system(cmd.c_str()); + else #ifdef _MSC_VER - (void)0; + (void)0; #else - cerr << cmd << endl; + cerr << cmd << endl; #endif - cmd = "save_brm"; + cmd = "save_brm"; - if (canWrite) - { - rc = system(cmd.c_str()); + if (canWrite) + { + rc = system(cmd.c_str()); - if (rc != 0) - { - ostringstream os; - os << "Warning: running " << cmd << " failed. This is usually non-fatal."; - cerr << os.str() << endl; - errorHandler(sysCatalogErr, "Save BRM", os.str()); - } - } - else - cerr << cmd << endl; - - return 0; - } - catch (exception& ex) + if (rc != 0) { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; - - if (canWrite) - rc = system(cmd.c_str()); - else - cerr << cmd << endl; - - errorHandler(sysCatalogErr, "Build system catalog", ex.what()); + ostringstream os; + os << "Warning: running " << cmd << " failed. This is usually non-fatal."; + cerr << os.str() << endl; + errorHandler(sysCatalogErr, "Save BRM", os.str()); } - catch (...) - { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; + } + else + cerr << cmd << endl; - if (canWrite) - rc = system(cmd.c_str()); - else - cerr << cmd << endl; - - string err("Caught unknown exception!"); - errorHandler(sysCatalogErr, "Build system catalog", err); - } + return 0; } - else + catch (exception& ex) { - usage(); - return 1; - } + string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; + if (canWrite) + rc = system(cmd.c_str()); + else + cerr << cmd << endl; + + errorHandler(sysCatalogErr, "Build system catalog", ex.what()); + } + catch (...) + { + string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; + + if (canWrite) + rc = system(cmd.c_str()); + else + cerr << cmd << endl; + + string err("Caught unknown exception!"); + errorHandler(sysCatalogErr, "Build system catalog", err); + } + } + else + { + usage(); return 1; + } + + return 1; } // vim:ts=4 sw=4: - diff --git a/tools/dbbuilder/dbbuilder.h b/tools/dbbuilder/dbbuilder.h index b0578ebff..debee930c 100644 --- a/tools/dbbuilder/dbbuilder.h +++ b/tools/dbbuilder/dbbuilder.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: dbbuilder.h 2122 2013-02-14 14:58:28Z chao $ -* -******************************************************************************************/ + * $Id: dbbuilder.h 2122 2013-02-14 14:58:28Z chao $ + * + ******************************************************************************************/ /** * @file */ @@ -58,4 +58,4 @@ #include "messagelog.h" #include "liboamcpp.h" -#endif //DBBUILDER_H +#endif // DBBUILDER_H diff --git a/tools/dbbuilder/systemcatalog.cpp b/tools/dbbuilder/systemcatalog.cpp index 1bcafe2c4..0cf855a73 100644 --- a/tools/dbbuilder/systemcatalog.cpp +++ b/tools/dbbuilder/systemcatalog.cpp @@ -36,500 +36,579 @@ using namespace joblist; void SystemCatalog::build() { - TxnID txnID = 0; - int rc; - //int t= 1000; - - remove(); - - cout << "Creating System Catalog..." << endl; - cout << endl; - // SYSTABLE - - timeval startTime; - gettimeofday( &startTime, 0); - ostringstream msg; - WErrorCodes ec; - - //------------------------------------------------------------------------------ - // Get the DBRoot count, and rotate the tables through those DBRoots. - // All the columns in the first table (SYSTABLE) start on DBRoot1, all the - // columns in the second table (SYSCOLUMN) start on DBRoot2, etc. - //------------------------------------------------------------------------------ - config::Config* cf = config::Config::makeConfig(); - string root = cf->getConfig("SystemConfig", "DBRootCount"); - uint32_t dbRootCount = cf->uFromText(root); - - //------------------------------------------------------------------------------ - // Create SYSTABLE table - //------------------------------------------------------------------------------ - uint32_t dbRoot = 1; - int compressionType = 0; - uint32_t partition = 0; - uint16_t segment = 0; - - ResourceManager* rm = ResourceManager::instance(); - std::map oids; + TxnID txnID = 0; + int rc; + // int t= 1000; + + remove(); + + cout << "Creating System Catalog..." << endl; + cout << endl; + // SYSTABLE + + timeval startTime; + gettimeofday(&startTime, 0); + ostringstream msg; + WErrorCodes ec; + + //------------------------------------------------------------------------------ + // Get the DBRoot count, and rotate the tables through those DBRoots. + // All the columns in the first table (SYSTABLE) start on DBRoot1, all the + // columns in the second table (SYSCOLUMN) start on DBRoot2, etc. + //------------------------------------------------------------------------------ + config::Config* cf = config::Config::makeConfig(); + string root = cf->getConfig("SystemConfig", "DBRootCount"); + uint32_t dbRootCount = cf->uFromText(root); + + //------------------------------------------------------------------------------ + // Create SYSTABLE table + //------------------------------------------------------------------------------ + uint32_t dbRoot = 1; + int compressionType = 0; + uint32_t partition = 0; + uint16_t segment = 0; + + ResourceManager* rm = ResourceManager::instance(); + std::map oids; + + if (rm->useHdfs()) + { + compressionType = 2; + oids[OID_SYSTABLE_TABLENAME] = OID_SYSTABLE_TABLENAME; + oids[DICTOID_SYSTABLE_TABLENAME] = DICTOID_SYSTABLE_TABLENAME; + oids[OID_SYSTABLE_SCHEMA] = OID_SYSTABLE_SCHEMA; + oids[DICTOID_SYSTABLE_SCHEMA] = DICTOID_SYSTABLE_SCHEMA; + oids[OID_SYSTABLE_OBJECTID] = OID_SYSTABLE_OBJECTID; + oids[OID_SYSTABLE_CREATEDATE] = OID_SYSTABLE_CREATEDATE; + oids[OID_SYSTABLE_LASTUPDATE] = OID_SYSTABLE_LASTUPDATE; + oids[OID_SYSTABLE_INIT] = OID_SYSTABLE_INIT; + oids[OID_SYSTABLE_NEXT] = OID_SYSTABLE_NEXT; + oids[OID_SYSTABLE_NUMOFROWS] = OID_SYSTABLE_NUMOFROWS; + oids[OID_SYSTABLE_AVGROWLEN] = OID_SYSTABLE_AVGROWLEN; + oids[OID_SYSTABLE_NUMOFBLOCKS] = OID_SYSTABLE_NUMOFBLOCKS; + oids[OID_SYSTABLE_AUTOINCREMENT] = OID_SYSTABLE_AUTOINCREMENT; + } + + fWriteEngine.setTransId(1); + fWriteEngine.setBulkFlag(true); + cout << "Creating SYSTABLE" << endl; + cout << "---------------------------------------" << endl; + // TableName + msg << " Creating TableName column OID: " << OID_SYSTABLE_TABLENAME; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_TABLENAME, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + msg << " Creating TableName column dictionary"; + // Dictionary files + cout << msg.str() << endl; + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSTABLE_TABLENAME, 65, dbRoot, partition, segment, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // Schema + msg << " Creating Schema column OID: " << OID_SYSTABLE_SCHEMA; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_SCHEMA, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(" Creating Schema column dictionary"); + cout << msg.str() << endl; + // Dictionary files + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSTABLE_SCHEMA, 65, dbRoot, partition, segment, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // ObjectId + msg << " Creating ObjectId column OID: " << OID_SYSTABLE_OBJECTID; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_OBJECTID, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // CreateDate + msg << " Creating CreateDate column OID: " << OID_SYSTABLE_CREATEDATE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_CREATEDATE, CalpontSystemCatalog::DATE, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // LastUpdateDate + msg << " Creating LastUpdate column OID: " << OID_SYSTABLE_LASTUPDATE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_LASTUPDATE, CalpontSystemCatalog::DATE, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // INIT + msg << " Creating INIT column OID: " << OID_SYSTABLE_INIT; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_INIT, CalpontSystemCatalog::INT, 4, dbRoot, partition, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // NEXT + msg << " Creating NEXT column OID: " << OID_SYSTABLE_NEXT; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_NEXT, CalpontSystemCatalog::INT, 4, dbRoot, partition, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // NUMOFROWS + msg << " Creating NUMOFROWS column OID: " << OID_SYSTABLE_NUMOFROWS; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_NUMOFROWS, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // AVGROWLEN + msg << " Creating AVGROWLEN column OID: " << OID_SYSTABLE_AVGROWLEN; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_AVGROWLEN, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // NUMOFBLOCKS + msg << " Creating NUMOFBLOCKS column OID: " << OID_SYSTABLE_NUMOFBLOCKS; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_NUMOFBLOCKS, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // AUTOINCREMENT + msg << " Creating AUTOINCREMENT column OID: " << OID_SYSTABLE_AUTOINCREMENT; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSTABLE_AUTOINCREMENT, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + //------------------------------------------------------------------------------ + // Create SYSCOLUMN table + //------------------------------------------------------------------------------ + // dbRoot++; + // if (dbRoot > dbRootCount) + // dbRoot = 1; + + // SYSCOLUMN + if (rm->useHdfs()) + { + oids[OID_SYSCOLUMN_SCHEMA] = OID_SYSCOLUMN_SCHEMA; + oids[DICTOID_SYSCOLUMN_SCHEMA] = DICTOID_SYSCOLUMN_SCHEMA; + oids[OID_SYSCOLUMN_TABLENAME] = OID_SYSCOLUMN_TABLENAME; + oids[DICTOID_SYSCOLUMN_TABLENAME] = DICTOID_SYSCOLUMN_TABLENAME; + oids[OID_SYSCOLUMN_COLNAME] = OID_SYSCOLUMN_COLNAME; + oids[DICTOID_SYSCOLUMN_COLNAME] = DICTOID_SYSCOLUMN_COLNAME; + oids[OID_SYSCOLUMN_OBJECTID] = OID_SYSCOLUMN_OBJECTID; + oids[OID_SYSCOLUMN_DICTOID] = OID_SYSCOLUMN_DICTOID; + oids[OID_SYSCOLUMN_LISTOBJID] = OID_SYSCOLUMN_LISTOBJID; + oids[OID_SYSCOLUMN_TREEOBJID] = OID_SYSCOLUMN_TREEOBJID; + oids[OID_SYSCOLUMN_DATATYPE] = OID_SYSCOLUMN_DATATYPE; + oids[OID_SYSCOLUMN_COLUMNLEN] = OID_SYSCOLUMN_COLUMNLEN; + oids[OID_SYSCOLUMN_COLUMNPOS] = OID_SYSCOLUMN_COLUMNPOS; + oids[OID_SYSCOLUMN_LASTUPDATE] = OID_SYSCOLUMN_LASTUPDATE; + oids[OID_SYSCOLUMN_DEFAULTVAL] = OID_SYSCOLUMN_DEFAULTVAL; + oids[DICTOID_SYSCOLUMN_DEFAULTVAL] = DICTOID_SYSCOLUMN_DEFAULTVAL; + oids[OID_SYSCOLUMN_NULLABLE] = OID_SYSCOLUMN_NULLABLE; + oids[OID_SYSCOLUMN_SCALE] = OID_SYSCOLUMN_SCALE; + oids[OID_SYSCOLUMN_PRECISION] = OID_SYSCOLUMN_PRECISION; + oids[OID_SYSCOLUMN_AUTOINC] = OID_SYSCOLUMN_AUTOINC; + oids[OID_SYSCOLUMN_DISTCOUNT] = OID_SYSCOLUMN_DISTCOUNT; + oids[OID_SYSCOLUMN_NULLCOUNT] = OID_SYSCOLUMN_NULLCOUNT; + oids[OID_SYSCOLUMN_MINVALUE] = OID_SYSCOLUMN_MINVALUE; + oids[DICTOID_SYSCOLUMN_MINVALUE] = DICTOID_SYSCOLUMN_MINVALUE; + oids[OID_SYSCOLUMN_MAXVALUE] = OID_SYSCOLUMN_MAXVALUE; + oids[DICTOID_SYSCOLUMN_MAXVALUE] = DICTOID_SYSCOLUMN_MAXVALUE; + oids[OID_SYSCOLUMN_COMPRESSIONTYPE] = OID_SYSCOLUMN_COMPRESSIONTYPE; + oids[OID_SYSCOLUMN_NEXTVALUE] = OID_SYSCOLUMN_NEXTVALUE; + } + + cout << endl; + cout << "Creating SYSCOLUMN" << endl; + // Schema + cout << "---------------------------------------" << endl; + msg << " Creating Schema column OID: " << OID_SYSCOLUMN_SCHEMA; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_SCHEMA, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(" Creating Schema column dictionary..."); + // Dictionary files + cout << msg.str() << endl; + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_SCHEMA, 65, dbRoot, partition, segment, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // TableName + msg << " Creating TableName column OID: " << OID_SYSCOLUMN_TABLENAME; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_TABLENAME, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(" Creating TableName column dictionary..."); + // Dictionary files + cout << msg.str() << endl; + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_TABLENAME, 65, dbRoot, partition, segment, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // ColumnName + msg << " Creating ColumnName column OID: " << OID_SYSCOLUMN_COLNAME; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_COLNAME, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(" Creating ColumnName column dictionary..."); + // Dictionary files + cout << msg.str() << endl; + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_COLNAME, 65, dbRoot, partition, segment, + compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // ObjectID + msg << " Creating ObjectID column OID: " << OID_SYSCOLUMN_OBJECTID; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_OBJECTID, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // DictOID + msg << " Creating DictOID column OID: " << OID_SYSCOLUMN_DICTOID; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_DICTOID, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); + + msg.str(""); + + // ListOID + msg << " Creating ListOID column OID: " << OID_SYSCOLUMN_LISTOBJID; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_LISTOBJID, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - if ( rm->useHdfs() ) - { - compressionType = 2; - oids[OID_SYSTABLE_TABLENAME] = OID_SYSTABLE_TABLENAME; - oids[DICTOID_SYSTABLE_TABLENAME] = DICTOID_SYSTABLE_TABLENAME; - oids[OID_SYSTABLE_SCHEMA] = OID_SYSTABLE_SCHEMA; - oids[DICTOID_SYSTABLE_SCHEMA] = DICTOID_SYSTABLE_SCHEMA; - oids[OID_SYSTABLE_OBJECTID] = OID_SYSTABLE_OBJECTID; - oids[OID_SYSTABLE_CREATEDATE] = OID_SYSTABLE_CREATEDATE; - oids[OID_SYSTABLE_LASTUPDATE] = OID_SYSTABLE_LASTUPDATE; - oids[OID_SYSTABLE_INIT] = OID_SYSTABLE_INIT; - oids[OID_SYSTABLE_NEXT] = OID_SYSTABLE_NEXT; - oids[OID_SYSTABLE_NUMOFROWS] = OID_SYSTABLE_NUMOFROWS; - oids[OID_SYSTABLE_AVGROWLEN] = OID_SYSTABLE_AVGROWLEN; - oids[OID_SYSTABLE_NUMOFBLOCKS] = OID_SYSTABLE_NUMOFBLOCKS; - oids[OID_SYSTABLE_AUTOINCREMENT] = OID_SYSTABLE_AUTOINCREMENT; - } + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - fWriteEngine.setTransId(1); - fWriteEngine.setBulkFlag(true); - cout << "Creating SYSTABLE" << endl; - cout << "---------------------------------------" << endl; - // TableName - msg << " Creating TableName column OID: " << OID_SYSTABLE_TABLENAME; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_TABLENAME, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); + msg.str(""); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + // TreeOID + msg << " Creating TreeOID column OID: " << OID_SYSCOLUMN_TREEOBJID; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_TREEOBJID, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - msg.str(""); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg << " Creating TableName column dictionary"; - //Dictionary files - cout << msg.str() << endl; - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSTABLE_TABLENAME, 65, dbRoot, partition, segment, compressionType ); + msg.str(""); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + // DataType + msg << " Creating DataType column OID: " << OID_SYSCOLUMN_DATATYPE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_DATATYPE, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - msg.str(""); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - // Schema - msg << " Creating Schema column OID: " << OID_SYSTABLE_SCHEMA; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_SCHEMA, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); + msg.str(""); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + // ColumnLength + msg << " Creating ColumnLength column OID: " << OID_SYSCOLUMN_COLUMNLEN; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_COLUMNLEN, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(" Creating Schema column dictionary"); - cout << msg.str() << endl; - //Dictionary files - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSTABLE_SCHEMA, 65, dbRoot, partition, segment, compressionType); + msg.str(""); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + // ColumnPos + msg << " Creating ColumnPos column OID: " << OID_SYSCOLUMN_COLUMNPOS; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_COLUMNPOS, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - msg.str(""); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - // ObjectId - msg << " Creating ObjectId column OID: " << OID_SYSTABLE_OBJECTID; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_OBJECTID, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // CreateDate - msg << " Creating CreateDate column OID: " << OID_SYSTABLE_CREATEDATE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_CREATEDATE, CalpontSystemCatalog::DATE, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // LastUpdateDate - msg << " Creating LastUpdate column OID: " << OID_SYSTABLE_LASTUPDATE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_LASTUPDATE, CalpontSystemCatalog::DATE, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // INIT - msg << " Creating INIT column OID: " << OID_SYSTABLE_INIT; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_INIT, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // NEXT - msg << " Creating NEXT column OID: " << OID_SYSTABLE_NEXT; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_NEXT, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - //NUMOFROWS - msg << " Creating NUMOFROWS column OID: " << OID_SYSTABLE_NUMOFROWS; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_NUMOFROWS, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(""); + // LastUpdate + msg << " Creating LastUpdate column OID: " << OID_SYSCOLUMN_LASTUPDATE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_LASTUPDATE, CalpontSystemCatalog::DATE, 4, dbRoot, + partition, compressionType); - //AVGROWLEN - msg << " Creating AVGROWLEN column OID: " << OID_SYSTABLE_AVGROWLEN; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_AVGROWLEN, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(""); + // DefaultValue + msg << " Creating DefaultValue column OID: " << OID_SYSCOLUMN_DEFAULTVAL; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_DEFAULTVAL, CalpontSystemCatalog::VARCHAR, 8, dbRoot, + partition, compressionType); - //NUMOFBLOCKS - msg << " Creating NUMOFBLOCKS column OID: " << OID_SYSTABLE_NUMOFBLOCKS; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_NUMOFBLOCKS, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(""); + msg.str(" Creating DefaultValue column dictionary..."); + // Dictionary files + cout << msg.str() << endl; + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_DEFAULTVAL, 9, dbRoot, partition, segment, + compressionType); - //AUTOINCREMENT - msg << " Creating AUTOINCREMENT column OID: " << OID_SYSTABLE_AUTOINCREMENT; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSTABLE_AUTOINCREMENT, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(""); + // Nullable + msg << " Creating Nullable column OID: " << OID_SYSCOLUMN_NULLABLE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_NULLABLE, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - //------------------------------------------------------------------------------ - // Create SYSCOLUMN table - //------------------------------------------------------------------------------ - //dbRoot++; - //if (dbRoot > dbRootCount) - // dbRoot = 1; + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - //SYSCOLUMN - if ( rm->useHdfs() ) - { - oids[OID_SYSCOLUMN_SCHEMA] = OID_SYSCOLUMN_SCHEMA; - oids[DICTOID_SYSCOLUMN_SCHEMA] = DICTOID_SYSCOLUMN_SCHEMA; - oids[OID_SYSCOLUMN_TABLENAME] = OID_SYSCOLUMN_TABLENAME; - oids[DICTOID_SYSCOLUMN_TABLENAME] = DICTOID_SYSCOLUMN_TABLENAME; - oids[OID_SYSCOLUMN_COLNAME] = OID_SYSCOLUMN_COLNAME; - oids[DICTOID_SYSCOLUMN_COLNAME] = DICTOID_SYSCOLUMN_COLNAME; - oids[OID_SYSCOLUMN_OBJECTID] = OID_SYSCOLUMN_OBJECTID; - oids[OID_SYSCOLUMN_DICTOID] = OID_SYSCOLUMN_DICTOID; - oids[OID_SYSCOLUMN_LISTOBJID] = OID_SYSCOLUMN_LISTOBJID; - oids[OID_SYSCOLUMN_TREEOBJID] = OID_SYSCOLUMN_TREEOBJID; - oids[OID_SYSCOLUMN_DATATYPE] = OID_SYSCOLUMN_DATATYPE; - oids[OID_SYSCOLUMN_COLUMNLEN] = OID_SYSCOLUMN_COLUMNLEN; - oids[OID_SYSCOLUMN_COLUMNPOS] = OID_SYSCOLUMN_COLUMNPOS; - oids[OID_SYSCOLUMN_LASTUPDATE] = OID_SYSCOLUMN_LASTUPDATE; - oids[OID_SYSCOLUMN_DEFAULTVAL] = OID_SYSCOLUMN_DEFAULTVAL; - oids[DICTOID_SYSCOLUMN_DEFAULTVAL] = DICTOID_SYSCOLUMN_DEFAULTVAL; - oids[OID_SYSCOLUMN_NULLABLE] = OID_SYSCOLUMN_NULLABLE; - oids[OID_SYSCOLUMN_SCALE] = OID_SYSCOLUMN_SCALE; - oids[OID_SYSCOLUMN_PRECISION] = OID_SYSCOLUMN_PRECISION; - oids[OID_SYSCOLUMN_AUTOINC] = OID_SYSCOLUMN_AUTOINC; - oids[OID_SYSCOLUMN_DISTCOUNT] = OID_SYSCOLUMN_DISTCOUNT; - oids[OID_SYSCOLUMN_NULLCOUNT] = OID_SYSCOLUMN_NULLCOUNT; - oids[OID_SYSCOLUMN_MINVALUE] = OID_SYSCOLUMN_MINVALUE; - oids[DICTOID_SYSCOLUMN_MINVALUE] = DICTOID_SYSCOLUMN_MINVALUE; - oids[OID_SYSCOLUMN_MAXVALUE] = OID_SYSCOLUMN_MAXVALUE; - oids[DICTOID_SYSCOLUMN_MAXVALUE] = DICTOID_SYSCOLUMN_MAXVALUE; - oids[OID_SYSCOLUMN_COMPRESSIONTYPE] = OID_SYSCOLUMN_COMPRESSIONTYPE; - oids[OID_SYSCOLUMN_NEXTVALUE] = OID_SYSCOLUMN_NEXTVALUE; - } + msg.str(""); - cout << endl; - cout << "Creating SYSCOLUMN" << endl; - // Schema - cout << "---------------------------------------" << endl; - msg << " Creating Schema column OID: " << OID_SYSCOLUMN_SCHEMA; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_SCHEMA, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); + // Scale + msg << " Creating Scale column OID: " << OID_SYSCOLUMN_SCALE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_SCALE, CalpontSystemCatalog::INT, 4, dbRoot, partition, + compressionType); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(" Creating Schema column dictionary..."); - //Dictionary files - cout << msg.str() << endl; - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_SCHEMA, 65, dbRoot, partition, segment, compressionType); + msg.str(""); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + // Precision + msg << " Creating Precision column OID: " << OID_SYSCOLUMN_PRECISION; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_PRECISION, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - msg.str(""); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - // TableName - msg << " Creating TableName column OID: " << OID_SYSCOLUMN_TABLENAME; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_TABLENAME, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); + msg.str(""); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + // AutoInc + msg << " Creating AutoInc column OID: " << OID_SYSCOLUMN_AUTOINC; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_AUTOINC, CalpontSystemCatalog::CHAR, 1, dbRoot, + partition, compressionType); - msg.str(" Creating TableName column dictionary..."); - //Dictionary files - cout << msg.str() << endl; - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_TABLENAME, 65, dbRoot, partition, segment, compressionType); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(""); + // DISTCOUNT + msg << " Creating DISTCOUNT column OID: " << OID_SYSCOLUMN_DISTCOUNT; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_DISTCOUNT, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - // ColumnName - msg << " Creating ColumnName column OID: " << OID_SYSCOLUMN_COLNAME; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_COLNAME, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(" Creating ColumnName column dictionary..."); - //Dictionary files - cout << msg.str() << endl; - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_COLNAME, 65, dbRoot, partition, segment, compressionType); + // NULLCOUNT + msg << " Creating NULLCOUNT column OID: " << OID_SYSCOLUMN_NULLCOUNT; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_NULLCOUNT, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(""); + msg.str(""); - // ObjectID - msg << " Creating ObjectID column OID: " << OID_SYSCOLUMN_OBJECTID; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_OBJECTID, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + // MINVALUE + msg << " Creating MINVALUE column OID: " << OID_SYSCOLUMN_MINVALUE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_MINVALUE, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(""); + msg.str(" Creating MINVALUE column dictionary..."); + cout << msg.str() << endl; + // Dictionary files + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_MINVALUE, 65, dbRoot, partition, segment, + compressionType); - // DictOID - msg << " Creating DictOID column OID: " << OID_SYSCOLUMN_DICTOID; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_DICTOID, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(""); - msg.str(""); + // MAXVALUE + msg << " Creating MAXVALUE column OID: " << OID_SYSCOLUMN_MAXVALUE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_MAXVALUE, CalpontSystemCatalog::VARCHAR, 40, dbRoot, + partition, compressionType); - // ListOID - msg << " Creating ListOID column OID: " << OID_SYSCOLUMN_LISTOBJID; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_LISTOBJID, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + msg.str(" Creating MAXVALUE column dictionary..."); + // Dictionary files + cout << msg.str() << endl; + rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_MAXVALUE, 65, dbRoot, partition, segment, + compressionType); - msg.str(""); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - // TreeOID - msg << " Creating TreeOID column OID: " << OID_SYSCOLUMN_TREEOBJID; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_TREEOBJID, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + // CompressionType + msg << " Creating CompressionType column OID: " << OID_SYSCOLUMN_COMPRESSIONTYPE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_COMPRESSIONTYPE, CalpontSystemCatalog::INT, 4, dbRoot, + partition, compressionType); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(""); + msg.str(""); - // DataType - msg << " Creating DataType column OID: " << OID_SYSCOLUMN_DATATYPE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_DATATYPE, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + // nextvalue + msg << " Creating NEXTVALUE column OID: " << OID_SYSCOLUMN_NEXTVALUE; + cout << msg.str() << endl; + rc = fWriteEngine.createColumn(txnID, OID_SYSCOLUMN_NEXTVALUE, CalpontSystemCatalog::UBIGINT, 8, dbRoot, + partition, compressionType); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(""); + msg.str(""); - // ColumnLength - msg << " Creating ColumnLength column OID: " << OID_SYSCOLUMN_COLUMNLEN; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_COLUMNLEN, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + //------------------------------------------------------------------------------ + // Create SYSCONSTRAINT table + //------------------------------------------------------------------------------ + dbRoot++; - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (dbRoot > dbRootCount) + dbRoot = 1; - msg.str(""); + // flush data files + fWriteEngine.flushDataFiles(rc, 1, oids); + // save brm + msg.str(" BRMWrapper saving state "); - // ColumnPos - msg << " Creating ColumnPos column OID: " << OID_SYSCOLUMN_COLUMNPOS; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_COLUMNPOS, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); + rc = BRMWrapper::getInstance()->saveState(); - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); + if (rc) + throw runtime_error(msg.str() + ec.errorString(rc)); - msg.str(""); + timeval endTime; + gettimeofday(&endTime, 0); + double elapsedTime = + (endTime.tv_sec + (endTime.tv_usec / 1000000.0)) - (startTime.tv_sec + (startTime.tv_usec / 1000000.0)); + cout << "System Catalog creation took: " << elapsedTime << " seconds to complete." << endl; - // LastUpdate - msg << " Creating LastUpdate column OID: " << OID_SYSCOLUMN_LASTUPDATE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_LASTUPDATE, CalpontSystemCatalog::DATE, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // DefaultValue - msg << " Creating DefaultValue column OID: " << OID_SYSCOLUMN_DEFAULTVAL; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_DEFAULTVAL, CalpontSystemCatalog::VARCHAR, 8, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - msg.str(" Creating DefaultValue column dictionary..."); - //Dictionary files - cout << msg.str() << endl; - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_DEFAULTVAL, 9, dbRoot, partition, segment, compressionType); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // Nullable - msg << " Creating Nullable column OID: " << OID_SYSCOLUMN_NULLABLE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_NULLABLE, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // Scale - msg << " Creating Scale column OID: " << OID_SYSCOLUMN_SCALE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_SCALE, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // Precision - msg << " Creating Precision column OID: " << OID_SYSCOLUMN_PRECISION; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_PRECISION, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // AutoInc - msg << " Creating AutoInc column OID: " << OID_SYSCOLUMN_AUTOINC; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_AUTOINC, CalpontSystemCatalog::CHAR, 1, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // DISTCOUNT - msg << " Creating DISTCOUNT column OID: " << OID_SYSCOLUMN_DISTCOUNT; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_DISTCOUNT, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // NULLCOUNT - msg << " Creating NULLCOUNT column OID: " << OID_SYSCOLUMN_NULLCOUNT; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_NULLCOUNT, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // MINVALUE - msg << " Creating MINVALUE column OID: " << OID_SYSCOLUMN_MINVALUE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_MINVALUE, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(" Creating MINVALUE column dictionary..."); - cout << msg.str() << endl; - //Dictionary files - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_MINVALUE, 65, dbRoot, partition, segment, compressionType); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // MAXVALUE - msg << " Creating MAXVALUE column OID: " << OID_SYSCOLUMN_MAXVALUE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_MAXVALUE, CalpontSystemCatalog::VARCHAR, 40, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(" Creating MAXVALUE column dictionary..."); - //Dictionary files - cout << msg.str() << endl; - rc = fWriteEngine.createDctnry(txnID, DICTOID_SYSCOLUMN_MAXVALUE, 65, dbRoot, partition, segment, compressionType); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - // CompressionType - msg << " Creating CompressionType column OID: " << OID_SYSCOLUMN_COMPRESSIONTYPE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_COMPRESSIONTYPE, CalpontSystemCatalog::INT, 4, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - // nextvalue - msg << " Creating NEXTVALUE column OID: " << OID_SYSCOLUMN_NEXTVALUE; - cout << msg.str() << endl; - rc = fWriteEngine.createColumn( txnID, OID_SYSCOLUMN_NEXTVALUE, CalpontSystemCatalog::UBIGINT, 8, dbRoot, partition, compressionType ); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - msg.str(""); - - //------------------------------------------------------------------------------ - // Create SYSCONSTRAINT table - //------------------------------------------------------------------------------ - dbRoot++; - - if (dbRoot > dbRootCount) - dbRoot = 1; - -//flush data files - fWriteEngine.flushDataFiles(rc, 1, oids); - // save brm - msg.str(" BRMWrapper saving state "); - - rc = BRMWrapper::getInstance()->saveState(); - - if (rc) throw runtime_error(msg.str() + ec.errorString(rc)); - - timeval endTime; - gettimeofday( &endTime, 0); - double elapsedTime = - (endTime.tv_sec + (endTime.tv_usec / 1000000.0)) - - (startTime.tv_sec + (startTime.tv_usec / 1000000.0)); - cout << "System Catalog creation took: " << elapsedTime << - " seconds to complete." << endl; - - cout << endl; - cout << "System Catalog created" << endl; - cout << endl; + cout << endl; + cout << "System Catalog created" << endl; + cout << endl; } void SystemCatalog::remove() { - ColumnOpCompress0 colOp; + ColumnOpCompress0 colOp; - for ( int c = 1001; c <= 1074; c++ ) - colOp.deleteFile( c ); - - for ( int d = 2001; d <= 2312; d++ ) - colOp.deleteFile( d ); + for (int c = 1001; c <= 1074; c++) + colOp.deleteFile(c); + for (int d = 2001; d <= 2312; d++) + colOp.deleteFile(d); } diff --git a/tools/dbbuilder/systemcatalog.h b/tools/dbbuilder/systemcatalog.h index 23798df84..0375fe9f0 100644 --- a/tools/dbbuilder/systemcatalog.h +++ b/tools/dbbuilder/systemcatalog.h @@ -27,15 +27,12 @@ class SystemCatalog { -public: - void build(); - void remove(); - -private: - WriteEngine::WriteEngineWrapper fWriteEngine; + public: + void build(); + void remove(); + private: + WriteEngine::WriteEngineWrapper fWriteEngine; }; - - -#endif //SYSTEMCATLOG_H +#endif // SYSTEMCATLOG_H diff --git a/tools/dbbuilder/tpchpopulate.cpp b/tools/dbbuilder/tpchpopulate.cpp index 461a4ab35..301e1bb60 100644 --- a/tools/dbbuilder/tpchpopulate.cpp +++ b/tools/dbbuilder/tpchpopulate.cpp @@ -16,201 +16,208 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: tpchpopulate.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -* -******************************************************************************************/ + * $Id: tpchpopulate.cpp 2101 2013-01-21 14:12:52Z rdempsey $ + * + ******************************************************************************************/ #include #include "dbbuilder.h" #include "tpchpopulate.h" - using namespace dmlpackageprocessor; using namespace dmlpackage; using namespace std; void TpchPopulate::populateFromFile(std::string tableName, string& fileName) { + cout << endl; + cout << "Populating " << tableName << endl; + cout << "---------------------------------------" << endl; - cout << endl; - cout << "Populating " << tableName << endl; - cout << "---------------------------------------" << endl; + std::string dmlStatement; + ifstream ifdml(fileName.c_str(), ios::in); - std::string dmlStatement; - ifstream ifdml(fileName.c_str(), ios::in); - - if (ifdml) + if (ifdml) + { + while (ifdml.good()) { - while (ifdml.good()) - { - char line[256]; - line[0] = '\0';; - ifdml.getline(line, 256); + char line[256]; + line[0] = '\0'; + ; + ifdml.getline(line, 256); - //if ( line[0] == '\0' ) - if ( strlen(line) == 0 ) - continue; + // if ( line[0] == '\0' ) + if (strlen(line) == 0) + continue; - dmlStatement = "insert into "; - dmlStatement += tableName; - dmlStatement += " values("; - dmlStatement += line; - dmlStatement += ");"; - - insert(dmlStatement); - } + dmlStatement = "insert into "; + dmlStatement += tableName; + dmlStatement += " values("; + dmlStatement += line; + dmlStatement += ");"; + insert(dmlStatement); } - else - { - perror(fileName.c_str()); - } + } + else + { + perror(fileName.c_str()); + } - cout << endl; - cout << "Finished Populating " << tableName << endl; + cout << endl; + cout << "Finished Populating " << tableName << endl; } void TpchPopulate::populate_part() { + std::string dmlStatement = + "insert into tpch.part values('1', 'goldenrod lace spring peru powder', 'Manufac#1', 'Brand#13', " + "'PROMO BURNISHED COPPER', '7', 'JUMBO PKG', '901.00', 'final deposits s' );"; + insert(dmlStatement); - std::string dmlStatement = "insert into tpch.part values('1', 'goldenrod lace spring peru powder', 'Manufac#1', 'Brand#13', 'PROMO BURNISHED COPPER', '7', 'JUMBO PKG', '901.00', 'final deposits s' );"; - insert(dmlStatement); + dmlStatement = + "insert into tpch.part values(2, 'blush rosy metallic lemon navajo', 'Manufac#1', 'Brand#13','LARGE " + "BRUSHED BRASS', 1, 'LG CASE', 902.00, 'final platelets hang f');"; + insert(dmlStatement); - dmlStatement = "insert into tpch.part values(2, 'blush rosy metallic lemon navajo', 'Manufac#1', 'Brand#13','LARGE BRUSHED BRASS', 1, 'LG CASE', 902.00, 'final platelets hang f');"; - insert(dmlStatement); + dmlStatement = + "insert into tpch.part values(3, 'dark green antique puff wheat', 'Manufac#4', 'Brand#42', 'STANDARD " + "POLISHED BRASS', 21, 'WRAP CASE', 903.00, 'unusual excuses ac');"; + insert(dmlStatement); - dmlStatement = "insert into tpch.part values(3, 'dark green antique puff wheat', 'Manufac#4', 'Brand#42', 'STANDARD POLISHED BRASS', 21, 'WRAP CASE', 903.00, 'unusual excuses ac');"; - insert(dmlStatement); + dmlStatement = + "insert into tpch.part values(4, 'chocolate metallic smoke ghost drab', 'Manufac#3', 'Brand#34', " + "'SMALL PLATED BRASS', 14, 'MED DRUM', 904.00, 'ironi');"; + insert(dmlStatement); - dmlStatement = "insert into tpch.part values(4, 'chocolate metallic smoke ghost drab', 'Manufac#3', 'Brand#34', 'SMALL PLATED BRASS', 14, 'MED DRUM', 904.00, 'ironi');"; - insert(dmlStatement); + dmlStatement = + "insert into tpch.part values(5, 'forest blush chiffon thistle chocolate', 'Manufac#3', 'Brand#32', " + "'STANDARD POLISHED TIN', 15, 'SM PKG', 905.00, 'pending, spe');"; + insert(dmlStatement); - dmlStatement = "insert into tpch.part values(5, 'forest blush chiffon thistle chocolate', 'Manufac#3', 'Brand#32', 'STANDARD POLISHED TIN', 15, 'SM PKG', 905.00, 'pending, spe');"; - insert(dmlStatement); + cout << endl; + cout << "Commiting inserting to tpch.part table ..." << endl; + std::string command("COMMIT;"); + VendorDMLStatement dml_command(command, 1); + CalpontDMLPackage* dmlCommandPkgPtr = CalpontDMLFactory::makeCalpontDMLPackage(dml_command); - cout << endl; - cout << "Commiting inserting to tpch.part table ..." << endl; - std::string command("COMMIT;"); - VendorDMLStatement dml_command(command, 1); - CalpontDMLPackage* dmlCommandPkgPtr = CalpontDMLFactory::makeCalpontDMLPackage(dml_command); + DMLPackageProcessor* pkgProcPtr = + DMLPackageProcessorFactory::makePackageProcessor(DML_COMMAND, *dmlCommandPkgPtr); - DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor( DML_COMMAND, *dmlCommandPkgPtr ); + DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage(*dmlCommandPkgPtr); - DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage( *dmlCommandPkgPtr ); + if (DMLPackageProcessor::NO_ERROR != result.result) + { + cout << "Command process failed!" << endl; + } - if ( DMLPackageProcessor::NO_ERROR != result.result ) - { - cout << "Command process failed!" << endl; - } + delete pkgProcPtr; + delete dmlCommandPkgPtr; - delete pkgProcPtr; - delete dmlCommandPkgPtr; - - cout << endl; - cout << "Finished Populating tpch.part table" << endl; + cout << endl; + cout << "Finished Populating tpch.part table" << endl; } void TpchPopulate::populate_customer() { - cout << endl; - cout << "Populating tpch.customer table" << endl; - cout << "---------------------------------------" << endl; + cout << endl; + cout << "Populating tpch.customer table" << endl; + cout << "---------------------------------------" << endl; - std::string dmlStatement = "insert into tpch.customer values(1,'Customer#000000001', 'IVhzIApeRb ot,c,E',15, '25-989-741-2988',711.56, 'BUILDING', 'regular, regular platelets are fluffily according to the even attainments. blithely iron');"; + std::string dmlStatement = + "insert into tpch.customer values(1,'Customer#000000001', 'IVhzIApeRb ot,c,E',15, " + "'25-989-741-2988',711.56, 'BUILDING', 'regular, regular platelets are fluffily according to the even " + "attainments. blithely iron');"; - insert(dmlStatement); + insert(dmlStatement); - cout << endl; - cout << "Finished Populating tpch.customer table" << endl; + cout << endl; + cout << "Finished Populating tpch.customer table" << endl; } void TpchPopulate::populate_tpch() { - std::string tableName; - std::string filePath; - cout << endl; - cout << "Populating region table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.region"; - filePath = "region.tbl"; - populateFromFile(tableName, filePath ); + std::string tableName; + std::string filePath; + cout << endl; + cout << "Populating region table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.region"; + filePath = "region.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating nation table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.nation"; - filePath = "nation.tbl"; - populateFromFile(tableName, filePath ); + cout << "Populating nation table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.nation"; + filePath = "nation.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating customer table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.customer"; - filePath = "customer.tbl"; - populateFromFile(tableName, filePath ); + cout << "Populating customer table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.customer"; + filePath = "customer.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating orders table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.orders"; - filePath = "orders.tbl"; - populateFromFile(tableName, filePath ); + cout << "Populating orders table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.orders"; + filePath = "orders.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating part table" << endl; - cout << "---------------------------------------" << endl; - tableName = "part.nation"; - filePath = "part.tbl"; - populateFromFile(tableName, filePath ); + cout << "Populating part table" << endl; + cout << "---------------------------------------" << endl; + tableName = "part.nation"; + filePath = "part.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating supplier table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.supplier"; - filePath = "supplier.tbl"; - populateFromFile(tableName, filePath ); + cout << "Populating supplier table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.supplier"; + filePath = "supplier.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating partsupp table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.partsupp"; - filePath = "partsupp.tbl"; - populateFromFile(tableName, filePath ); + cout << "Populating partsupp table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.partsupp"; + filePath = "partsupp.tbl"; + populateFromFile(tableName, filePath); - cout << "Populating lineitem table" << endl; - cout << "---------------------------------------" << endl; - tableName = "tpch.lineitem"; - filePath = "lineitem.tbl"; - populateFromFile(tableName, filePath ); - - cout << endl; - cout << "Finished Populating TPCH tables." << endl; + cout << "Populating lineitem table" << endl; + cout << "---------------------------------------" << endl; + tableName = "tpch.lineitem"; + filePath = "lineitem.tbl"; + populateFromFile(tableName, filePath); + cout << endl; + cout << "Finished Populating TPCH tables." << endl; } void TpchPopulate::insert(string insertStmt) { + cout << endl; + cout << insertStmt << endl; + cout << "---------------------------------------" << endl; - cout << endl; - cout << insertStmt << endl; - cout << "---------------------------------------" << endl; + boost::timer theTimer; + VendorDMLStatement dmlStmt(insertStmt, 1); + CalpontDMLPackage* dmlPkgPtr = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - boost::timer theTimer; - VendorDMLStatement dmlStmt(insertStmt, 1); - CalpontDMLPackage* dmlPkgPtr = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor(DML_INSERT, *dmlPkgPtr); - DMLPackageProcessor* pkgProcPtr = DMLPackageProcessorFactory::makePackageProcessor( DML_INSERT, *dmlPkgPtr ); + // pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); + DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage(*dmlPkgPtr); - //pkgProcPtr->setDebugLevel(DMLPackageProcessor::VERBOSE); - DMLPackageProcessor::DMLResult result = pkgProcPtr->processPackage( *dmlPkgPtr ); + if (DMLPackageProcessor::NO_ERROR != result.result) + { + cout << "Insert failed!" << endl; + } + else + { + cout << "Insert successful" << endl; + } - if ( DMLPackageProcessor::NO_ERROR != result.result ) - { - cout << "Insert failed!" << endl; - } - else - { - cout << "Insert successful" << endl; - } - - cout << "Insert took :" << theTimer.elapsed() << " seconds to complete." << endl; - - delete pkgProcPtr; - delete dmlPkgPtr; + cout << "Insert took :" << theTimer.elapsed() << " seconds to complete." << endl; + delete pkgProcPtr; + delete dmlPkgPtr; } - diff --git a/tools/dbbuilder/tpchpopulate.h b/tools/dbbuilder/tpchpopulate.h index 1668d5f92..41a0b9b9a 100644 --- a/tools/dbbuilder/tpchpopulate.h +++ b/tools/dbbuilder/tpchpopulate.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: tpchpopulate.h 2101 2013-01-21 14:12:52Z rdempsey $ -* -******************************************************************************************/ + * $Id: tpchpopulate.h 2101 2013-01-21 14:12:52Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -27,18 +27,15 @@ class TpchPopulate { + public: + void populate_part(); + void populate_customer(); + void populate_tpch(); + void populateFromFile(std::string tableName, std::string& fileName); -public: - void populate_part(); - void populate_customer(); - void populate_tpch(); - void populateFromFile(std::string tableName, std::string& fileName); -protected: - - -private: - void insert(std::string insertStmt); - + protected: + private: + void insert(std::string insertStmt); }; -#endif //TPCHPOPULATE_H +#endif // TPCHPOPULATE_H diff --git a/tools/dbbuilder/tpchschema.cpp b/tools/dbbuilder/tpchschema.cpp index c11ebff49..d67b16830 100644 --- a/tools/dbbuilder/tpchschema.cpp +++ b/tools/dbbuilder/tpchschema.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: tpchschema.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -* -******************************************************************************************/ + * $Id: tpchschema.cpp 2101 2013-01-21 14:12:52Z rdempsey $ + * + ******************************************************************************************/ #include #include "dbbuilder.h" @@ -33,56 +33,77 @@ using namespace std; void TpchSchema::build() { - cout << "Creating tpch database..." << endl; - cout << endl; - cout << "create table tpch.region" << endl; - cout << "---------------------------------------" << endl; - std::string createStmt = "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; - buildTable(createStmt); + cout << "Creating tpch database..." << endl; + cout << endl; + cout << "create table tpch.region" << endl; + cout << "---------------------------------------" << endl; + std::string createStmt = + "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.nation" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT NULL,n_comment varchar(152))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.nation" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT " + "NULL,n_comment varchar(152))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.customer" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment varchar(117))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.customer" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), " + "c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment " + "varchar(117))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.orders" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), o_shippriority integer,o_comment varchar(79))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.orders" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), " + "o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), " + "o_shippriority integer,o_comment varchar(79))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.part" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , p_comment varchar(23))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.part" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand " + "char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , " + "p_comment varchar(23))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.supplier" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.supplier" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), " + "s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.partsupp" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , ps_supplycost number(9,2) , ps_comment varchar(199))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.partsupp" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , " + "ps_supplycost number(9,2) , ps_comment varchar(199))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.lineitem" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.lineitem" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey " + "integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice " + "number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag " + "char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , " + "l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; + buildTable(createStmt); - cout << endl; + cout << endl; #if 0 cout << "create index l_partkey_idx ON tpch.lineitem(l_partkey)" << endl; cout << "---------------------------------------" << endl; @@ -115,63 +136,82 @@ void TpchSchema::build() cout << endl; #endif - cout << "Finished creating tpch database" << endl; + cout << "Finished creating tpch database" << endl; } void TpchSchema::buildTpchTables(std::string schema) { - cout << "Creating tpch database..." << endl; - cout << endl; - cout << "create table " << schema << ".region" << endl; - cout << "---------------------------------------" << endl; - std::string createStart("create table "); - std::string createFirst = createStart + schema; - std::string createStmt = ".region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; - buildTable(createFirst + createStmt); + cout << "Creating tpch database..." << endl; + cout << endl; + cout << "create table " << schema << ".region" << endl; + cout << "---------------------------------------" << endl; + std::string createStart("create table "); + std::string createFirst = createStart + schema; + std::string createStmt = ".region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".nation" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT NULL,n_comment varchar(152))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".nation" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT NULL,n_comment " + "varchar(152))"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".customer" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment varchar(117))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".customer" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), c_nationkey " + "integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment varchar(117))"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".orders" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), o_shippriority integer,o_comment varchar(79))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".orders" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), o_totalprice " + "number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), o_shippriority " + "integer,o_comment varchar(79))"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".part" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , p_comment varchar(23))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".part" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand char(10) , p_type " + "varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , p_comment " + "varchar(23))"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".supplier" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".supplier" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), s_nationkey " + "integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".partsupp" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , ps_supplycost number(9,2) , ps_comment varchar(199))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".partsupp" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , ps_supplycost " + "number(9,2) , ps_comment varchar(199))"; + buildTable(createFirst + createStmt); - cout << endl; - cout << "create table " << schema << ".lineitem" << endl; - cout << "---------------------------------------" << endl; - createStmt = ".lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; - buildTable(createFirst + createStmt); + cout << endl; + cout << "create table " << schema << ".lineitem" << endl; + cout << "---------------------------------------" << endl; + createStmt = + ".lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey integer NOT NULL, " + "l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice number(9,2) NOT NULL, " + "l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag char(1) , l_linestatus " + "char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct char(25), " + "l_shipmode char(10),l_comment varchar(44))"; + buildTable(createFirst + createStmt); - cout << endl; + cout << endl; #if 0 createStart = "create index "; std::string createIndex("l_partkey_idx"); @@ -216,61 +256,82 @@ void TpchSchema::buildTpchTables(std::string schema) cout << endl; #endif - cout << "Finished creating tpch database" << endl; + cout << "Finished creating tpch database" << endl; } void TpchSchema::buildMultiColumnIndex() { - cout << "Creating tpch database..." << endl; - cout << endl; - cout << "create table tpch.region" << endl; - cout << "---------------------------------------" << endl; - std::string createStmt = "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; - buildTable(createStmt); + cout << "Creating tpch database..." << endl; + cout << endl; + cout << "create table tpch.region" << endl; + cout << "---------------------------------------" << endl; + std::string createStmt = + "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.nation" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT NULL,n_comment varchar(152))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.nation" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT " + "NULL,n_comment varchar(152))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.customer" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment varchar(117))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.customer" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), " + "c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment " + "varchar(117))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.orders" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), o_shippriority integer,o_comment varchar(79))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.orders" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), " + "o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), " + "o_shippriority integer,o_comment varchar(79))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.part" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , p_comment varchar(23), PRIMARY KEY(p_partkey))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.part" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand " + "char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , " + "p_comment varchar(23), PRIMARY KEY(p_partkey))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.supplier" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.supplier" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), " + "s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.partsupp" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , ps_supplycost number(9,2) , ps_comment varchar(199))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.partsupp" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , " + "ps_supplycost number(9,2) , ps_comment varchar(199))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.lineitem" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.lineitem" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey " + "integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice " + "number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag " + "char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , " + "l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; + buildTable(createStmt); - cout << endl; + cout << endl; #if 0 cout << "create index l_partkey_idx ON tpch.lineitem(l_partkey)" << endl; cout << "---------------------------------------" << endl; @@ -303,98 +364,118 @@ void TpchSchema::buildMultiColumnIndex() cout << endl; #endif - cout << "Finished creating tpch database" << endl; + cout << "Finished creating tpch database" << endl; } void TpchSchema::buildTable() { - cout << "Creating tpch database..." << endl; - cout << endl; - cout << "create table tpch.region" << endl; - cout << "---------------------------------------" << endl; - std::string createStmt = "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; - buildTable(createStmt); + cout << "Creating tpch database..." << endl; + cout << endl; + cout << "create table tpch.region" << endl; + cout << "---------------------------------------" << endl; + std::string createStmt = + "create table tpch.region( r_regionkey integer NOT NULL, r_name char(25) ,r_comment varchar(152));"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.nation" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT NULL,n_comment varchar(152))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.nation" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.nation( n_nationkey integer NOT NULL , n_name char(25) ,n_regionkey integer NOT " + "NULL,n_comment varchar(152))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.customer" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment varchar(117))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.customer" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.customer(c_custkey integer NOT NULL, c_name varchar(25) ,c_address varchar(40), " + "c_nationkey integer ,c_phone char(15) ,c_acctbal number(9,2) , c_mktsegment char(10) , c_comment " + "varchar(117))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.orders" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), o_shippriority integer,o_comment varchar(79))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.orders" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.orders(o_orderkey integer NOT NULL, o_custkey integer , o_orderstatus char(1), " + "o_totalprice number(9,2) , o_orderdate date , o_orderpriority char(15) , o_clerk char(15), " + "o_shippriority integer,o_comment varchar(79))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.part" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , p_comment varchar(23), PRIMARY KEY(p_partkey))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.part" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.part( p_partkey integer NOT NULL ,p_name varchar(55) , p_mfgr char(25), p_brand " + "char(10) , p_type varchar(25) , p_size integer , p_container char(10) ,p_retailprice number(9,2) , " + "p_comment varchar(23), PRIMARY KEY(p_partkey))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.supplier" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.supplier" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.supplier( s_suppkey integer NOT NULL, s_name char(25) , s_address varchar(40), " + "s_nationkey integer , s_phone char(15) , s_acctbal number(9,2), s_comment varchar(101))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.partsupp" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , ps_supplycost number(9,2) , ps_comment varchar(199))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.partsupp" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.partsupp( ps_partkey integer NOT NULL, ps_suppkey integer,ps_availqty integer , " + "ps_supplycost number(9,2) , ps_comment varchar(199))"; + buildTable(createStmt); - cout << endl; - cout << "create table tpch.lineitem" << endl; - cout << "---------------------------------------" << endl; - createStmt = "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; - buildTable(createStmt); + cout << endl; + cout << "create table tpch.lineitem" << endl; + cout << "---------------------------------------" << endl; + createStmt = + "create table tpch.lineitem( l_orderkey integer NOT NULL , l_partkey integer NOT NULL,l_suppkey " + "integer NOT NULL, l_linenumber integer NOT NULL, l_quantity number(9,2) NOT NULL,l_extendedprice " + "number(9,2) NOT NULL, l_discount number(9,2) NOT NULL, l_tax number(9,2) NOT NULL, l_returnflag " + "char(1) , l_linestatus char(1) , l_shipdate date , l_commitdate date,l_receiptdate date , " + "l_shipinstruct char(25), l_shipmode char(10),l_comment varchar(44))"; + buildTable(createStmt); - cout << endl; - cout << "Finished creating tpch database" << endl; + cout << endl; + cout << "Finished creating tpch database" << endl; } void TpchSchema::buildTable(string createText) { - cout << endl; - cout << createText << endl; + cout << endl; + cout << createText << endl; - boost::timer theTimer; - ddlpackage::SqlParser parser; - parser.Parse(createText.c_str()); + boost::timer theTimer; + ddlpackage::SqlParser parser; + parser.Parse(createText.c_str()); - if (parser.Good()) + if (parser.Good()) + { + const ddlpackage::ParseTree& ptree = parser.GetParseTree(); + + try { - const ddlpackage::ParseTree& ptree = parser.GetParseTree(); + cout << ptree << endl; + ddlpackageprocessor::CreateTableProcessor processor; + processor.setDebugLevel(ddlpackageprocessor::CreateTableProcessor::VERBOSE); - try - { - cout << ptree << endl; - ddlpackageprocessor::CreateTableProcessor processor; - processor.setDebugLevel(ddlpackageprocessor::CreateTableProcessor::VERBOSE); - - ddlpackage::SqlStatement& stmt = *ptree.fList[0]; - ddlpackageprocessor::CreateTableProcessor::DDLResult result; - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog( 1 ); - result = processor.processPackage(dynamic_cast(stmt)); - systemCatalogPtr->removeCalpontSystemCatalog(1); - std::cout << "return: " << result.result << std::endl; - } - catch (...) - { - - throw; - } + ddlpackage::SqlStatement& stmt = *ptree.fList[0]; + ddlpackageprocessor::CreateTableProcessor::DDLResult result; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(1); + result = processor.processPackage(dynamic_cast(stmt)); + systemCatalogPtr->removeCalpontSystemCatalog(1); + std::cout << "return: " << result.result << std::endl; } + catch (...) + { + throw; + } + } - cout << "Create table took :" << theTimer.elapsed() << " seconds to complete." << endl; + cout << "Create table took :" << theTimer.elapsed() << " seconds to complete." << endl; } #if 0 diff --git a/tools/dbbuilder/tpchschema.h b/tools/dbbuilder/tpchschema.h index 01408c731..31c34cef6 100644 --- a/tools/dbbuilder/tpchschema.h +++ b/tools/dbbuilder/tpchschema.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: tpchschema.h 2101 2013-01-21 14:12:52Z rdempsey $ -* -******************************************************************************************/ + * $Id: tpchschema.h 2101 2013-01-21 14:12:52Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -27,19 +27,17 @@ class TpchSchema { + public: + void build(); + void buildTpchTables(std::string schema); + void buildTable(); + void buildMultiColumnIndex(); + void buildIndex(); -public: - void build(); - void buildTpchTables(std::string schema); - void buildTable(); - void buildMultiColumnIndex(); - void buildIndex(); -protected: - - -private: - void buildTable(std::string createText); - void createindex(std::string createText); + protected: + private: + void buildTable(std::string createText); + void createindex(std::string createText); }; -#endif //TPCHSCHEMA_H +#endif // TPCHSCHEMA_H diff --git a/tools/dbloadxml/colxml.cpp b/tools/dbloadxml/colxml.cpp index 0042d21c1..80921f6f3 100644 --- a/tools/dbloadxml/colxml.cpp +++ b/tools/dbloadxml/colxml.cpp @@ -20,7 +20,6 @@ * ******************************************************************************/ - #include #include @@ -43,177 +42,174 @@ using namespace bulkloadxml; int main(int argc, char** argv) { - const int DEBUG_LVL_TO_DUMP_SYSCAT_RPT = 4; - // set effective ID to root - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - WriteEngine::Config::initConfigCache(); // load Columnstore.xml config settings + const int DEBUG_LVL_TO_DUMP_SYSCAT_RPT = 4; + // set effective ID to root + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); + WriteEngine::Config::initConfigCache(); // load Columnstore.xml config settings - //Bug 6137 - std::string aBulkRoot = WriteEngine::Config::getBulkRoot(); + // Bug 6137 + std::string aBulkRoot = WriteEngine::Config::getBulkRoot(); - if (!aBulkRoot.empty()) + if (!aBulkRoot.empty()) + { + if (!boost::filesystem::exists(aBulkRoot.c_str())) { - if (!boost::filesystem::exists(aBulkRoot.c_str())) - { - cout << "Creating directory : " << aBulkRoot << endl; - boost::filesystem::create_directories(aBulkRoot.c_str()); - } + cout << "Creating directory : " << aBulkRoot << endl; + boost::filesystem::create_directories(aBulkRoot.c_str()); + } - if (boost::filesystem::exists(aBulkRoot.c_str())) - { - std::ostringstream aSS; - aSS << aBulkRoot; - aSS << "/job"; - std::string jobDir = aSS.str(); + if (boost::filesystem::exists(aBulkRoot.c_str())) + { + std::ostringstream aSS; + aSS << aBulkRoot; + aSS << "/job"; + std::string jobDir = aSS.str(); - if (!boost::filesystem::exists(jobDir.c_str())) - { - cout << "Creating directory : " << jobDir << endl; - bool aSuccess = boost::filesystem::create_directories(jobDir.c_str()); + if (!boost::filesystem::exists(jobDir.c_str())) + { + cout << "Creating directory : " << jobDir << endl; + bool aSuccess = boost::filesystem::create_directories(jobDir.c_str()); - if (!aSuccess) - { - cout << "\nFailed to create job directory, please check permissions\n" << endl; - return -1; - } - } - } - else + if (!aSuccess) { - cout << "\nFailed to create bulk directory, check for permissions\n" << endl; - return -1; + cout << "\nFailed to create job directory, please check permissions\n" << endl; + return -1; } + } } else { - cout << "\nBulkRoot is empty in config file. Failed to create job file.\n\n"; - return -1; + cout << "\nFailed to create bulk directory, check for permissions\n" << endl; + return -1; } + } + else + { + cout << "\nBulkRoot is empty in config file. Failed to create job file.\n\n"; + return -1; + } - InputMgr mgr("299"); //@bug 391 + InputMgr mgr("299"); //@bug 391 - if (! mgr.input(argc, argv)) - return 1; + if (!mgr.input(argc, argv)) + return 1; - int debugLevel = atoi(mgr.getParm( - WriteEngine::XMLGenData::RPT_DEBUG).c_str()); + int debugLevel = atoi(mgr.getParm(WriteEngine::XMLGenData::RPT_DEBUG).c_str()); - bool bUseLogFile = true; - bool bSysCatRpt = false; + bool bUseLogFile = true; + bool bSysCatRpt = false; + if (debugLevel == DEBUG_LVL_TO_DUMP_SYSCAT_RPT) + { + cout << "\nRunning colxml to dump system catalog report:\n\n"; + bUseLogFile = false; + bSysCatRpt = true; + } + else + { + cout << "\nRunning colxml with the following parameters:\n"; + } + + WriteEngine::XMLGenProc curJob(&mgr, bUseLogFile, bSysCatRpt); + + if (debugLevel > 0 && debugLevel <= 3) + { + curJob.setDebugLevel(debugLevel); + cout << "\nDebug level is set to " << debugLevel << endl; + } + + BRM::DBRM dbrm; + + if (dbrm.getSystemReady() < 1) + { + std::string errMsg( + "System is not ready. Verify that ColumnStore is up and ready " + "before running colxml."); + + if (bUseLogFile) + curJob.logErrorMessage(errMsg); + else + cout << errMsg << endl; + + return 1; + } + + bool rc = false; + + const WriteEngine::XMLGenData::TableList& tables = mgr.getTables(); + + try + { + if (tables.empty()) + mgr.loadCatalogTables(); + + if (tables.empty()) + { + string msg = + "Either schema name is invalid or no table " + "is in the schema."; + curJob.logErrorMessage(msg); + } + else + { + curJob.startXMLFile(); + + for (InputMgr::TableList::const_iterator tbl = tables.begin(); tbl != tables.end(); ++tbl) + { + curJob.makeTableData(*tbl); + rc = curJob.makeColumnData(*tbl); + + if (!rc) + cout << "No columns for " << tbl->table << endl; + } + } + } + catch (runtime_error& ex) + { + curJob.logErrorMessage(string("colxml runtime exception: ") + ex.what()); + cout << curJob.errorString() << endl; + return 1; + } + catch (exception& ex) + { + curJob.logErrorMessage(string("colxml exception: ") + ex.what()); + cout << curJob.errorString() << endl; + return 1; + } + catch (...) + { + curJob.logErrorMessage(string("colxml unknown exception ")); + cout << curJob.errorString() << endl; + return 1; + } + + if (rc) + { if (debugLevel == DEBUG_LVL_TO_DUMP_SYSCAT_RPT) { - cout << "\nRunning colxml to dump system catalog report:\n\n"; - bUseLogFile = false; - bSysCatRpt = true; + std::string xmlFileName("-"); + curJob.writeXMLFile(xmlFileName); + cout << "\nDump completed for tables:\n\t"; } else { - cout << "\nRunning colxml with the following parameters:\n"; + std::string xmlFileName = curJob.genJobXMLFileName(); + cout << "Creating job description file: " << xmlFileName << endl; + curJob.writeXMLFile(xmlFileName); + cout << "File completed for tables:\n\t"; } - WriteEngine::XMLGenProc curJob(&mgr, bUseLogFile, bSysCatRpt); + copy(tables.begin(), tables.end(), + ostream_iterator(cout, "\n\t")); + cout << "\nNormal exit.\n"; + } + else + { + cout << "File not made.\n"; + cout << curJob.errorString(); + return 1; + } - if ( debugLevel > 0 && debugLevel <= 3 ) - { - curJob.setDebugLevel( debugLevel ); - cout << "\nDebug level is set to " << debugLevel << endl; - } - - BRM::DBRM dbrm; - - if (dbrm.getSystemReady() < 1) - { - std::string errMsg( - "System is not ready. Verify that ColumnStore is up and ready " - "before running colxml."); - - if (bUseLogFile) - curJob.logErrorMessage(errMsg); - else - cout << errMsg << endl; - - return 1; - } - - bool rc = false; - - const WriteEngine::XMLGenData::TableList& tables = mgr.getTables(); - - try - { - if (tables.empty()) - mgr.loadCatalogTables(); - - if (tables.empty()) - { - string msg = "Either schema name is invalid or no table " - "is in the schema."; - curJob.logErrorMessage(msg); - } - else - { - curJob.startXMLFile( ); - - for (InputMgr::TableList::const_iterator tbl = tables.begin(); - tbl != tables.end() ; ++tbl) - { - curJob.makeTableData( *tbl ); - rc = curJob.makeColumnData(*tbl); - - if (!rc) - cout << "No columns for " << tbl->table << endl; - } - } - } - catch (runtime_error& ex) - { - curJob.logErrorMessage(string( "colxml runtime exception: ") + - ex.what() ); - cout << curJob.errorString() << endl; - return 1; - } - catch (exception& ex) - { - curJob.logErrorMessage(string( "colxml exception: ") + ex.what() ); - cout << curJob.errorString() << endl; - return 1; - } - catch (...) - { - curJob.logErrorMessage(string("colxml unknown exception ")); - cout << curJob.errorString() << endl; - return 1; - } - - if (rc) - { - if (debugLevel == DEBUG_LVL_TO_DUMP_SYSCAT_RPT) - { - std::string xmlFileName("-"); - curJob.writeXMLFile( xmlFileName ); - cout << "\nDump completed for tables:\n\t"; - } - else - { - std::string xmlFileName = curJob.genJobXMLFileName( ); - cout << "Creating job description file: " << xmlFileName << endl; - curJob.writeXMLFile( xmlFileName ); - cout << "File completed for tables:\n\t"; - } - - copy(tables.begin(), tables.end(), - ostream_iterator - (cout, "\n\t")); - cout << "\nNormal exit.\n"; - } - else - { - cout << "File not made.\n"; - cout << curJob.errorString(); - return 1; - } - - return 0; + return 0; } diff --git a/tools/dbloadxml/inputmgr.cpp b/tools/dbloadxml/inputmgr.cpp index 24736fc82..fa0d9d5bc 100644 --- a/tools/dbloadxml/inputmgr.cpp +++ b/tools/dbloadxml/inputmgr.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: inputmgr.cpp 2258 2013-05-08 17:33:40Z dcathey $ -* -*******************************************************************************/ + * $Id: inputmgr.cpp 2258 2013-05-08 17:33:40Z dcathey $ + * + *******************************************************************************/ #include "inputmgr.h" @@ -36,20 +36,19 @@ using namespace execplan; namespace bulkloadxml { - //------------------------------------------------------------------------------ // InputMgr constructor //------------------------------------------------------------------------------ InputMgr::InputMgr(const string& job) { - fParms[ JOBID] = job; // add or override default value + fParms[JOBID] = job; // add or override default value } //------------------------------------------------------------------------------ // InputMgr destructor //------------------------------------------------------------------------------ /* virtual */ -InputMgr::~InputMgr( ) +InputMgr::~InputMgr() { } @@ -58,30 +57,33 @@ InputMgr::~InputMgr( ) //------------------------------------------------------------------------------ void InputMgr::printUsage() { - //@bug 391 - cerr << "Usage: " << "colxml [options] dbName" << endl << endl; - cerr << "Options: " << endl; - cerr << " -d Delimiter (default '|')\n"; - cerr << " -e Maximum allowable errors (per table)\n"; - cerr << " -h Print this message\n"; - cerr << " -j Job id (numeric)\n"; - cerr << " -l Load file name\n"; - cerr << " -n \"name in quotes\"\n"; - cerr << " -p Path for XML job description file that is generated\n"; - cerr << " -s \"Description in quotes\"\n"; - cerr << " -t Table name\n" ; - cerr << " -u User\n"; - cerr << " -r Number of read buffers\n"; - cerr << " -c Application read buffer size (in bytes)\n"; - cerr << " -w I/O library buffer size (in bytes), used to read files\n"; - cerr << " -x Extension of file name (default \".tbl\")\n"; - cerr << " -E EnclosedByChar (if data has enclosed values)\n"; - cerr << " -C EscapeChar\n"; - cerr << " -b Debug level (1-3)\n\n"; - cerr << " dbName - Required parm specifying the name of the database;" << - endl << " all others are optional\n\n" ; - cerr << "Example:\n\t" << "colxml -t lineitem -j 123 tpch\n"; - // exit(0); + //@bug 391 + cerr << "Usage: " + << "colxml [options] dbName" << endl + << endl; + cerr << "Options: " << endl; + cerr << " -d Delimiter (default '|')\n"; + cerr << " -e Maximum allowable errors (per table)\n"; + cerr << " -h Print this message\n"; + cerr << " -j Job id (numeric)\n"; + cerr << " -l Load file name\n"; + cerr << " -n \"name in quotes\"\n"; + cerr << " -p Path for XML job description file that is generated\n"; + cerr << " -s \"Description in quotes\"\n"; + cerr << " -t Table name\n"; + cerr << " -u User\n"; + cerr << " -r Number of read buffers\n"; + cerr << " -c Application read buffer size (in bytes)\n"; + cerr << " -w I/O library buffer size (in bytes), used to read files\n"; + cerr << " -x Extension of file name (default \".tbl\")\n"; + cerr << " -E EnclosedByChar (if data has enclosed values)\n"; + cerr << " -C EscapeChar\n"; + cerr << " -b Debug level (1-3)\n\n"; + cerr << " dbName - Required parm specifying the name of the database;" << endl + << " all others are optional\n\n"; + cerr << "Example:\n\t" + << "colxml -t lineitem -j 123 tpch\n"; + // exit(0); } //------------------------------------------------------------------------------ @@ -89,139 +91,132 @@ void InputMgr::printUsage() //------------------------------------------------------------------------------ bool InputMgr::input(int argc, char** argv) { - std::vector l_tableList; - int ch; + std::vector l_tableList; + int ch; - while ( (ch = getopt(argc, argv, "b:d:s:j:l:e:n:p:t:u:r:c:w:x:hE:C:")) != EOF ) + while ((ch = getopt(argc, argv, "b:d:s:j:l:e:n:p:t:u:r:c:w:x:hE:C:")) != EOF) + { + switch (ch) { - switch (ch) + case 't': + { + l_tableList.push_back(optarg); + break; + } + + case 'l': + { + fLoadNames.push_back(optarg); + break; + } + + case 'r': + case 'c': + case 'w': + case 'j': + case 'b': + case 'e': + { + if (verifyArgument(optarg) < 0) { - case 't': - { - l_tableList.push_back(optarg); - break; - } - - case 'l': - { - fLoadNames.push_back(optarg); - break; - } - - case 'r': - case 'c': - case 'w': - case 'j': - case 'b': - case 'e': - { - if (verifyArgument(optarg) < 0) - { - cout << "Argument associated with option -" << - (char)ch << " is not a number." << endl; - return false; - } - - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if (errno != 0) - { - cout << "Option -" << (char)ch << - " is invalid or out of range" << endl; - return false; - } - - // Limit to INT_MAX because we eventually store in an "int" - if ( ((ch == 'e') && (lValue < 0)) || - ((ch != 'e') && (lValue < 1)) || - (lValue > INT_MAX)) - { - cout << "Option -" << (char)ch << - " is invalid or out of range." << endl; - return false; - } - } - /* FALLTHRU */ - case 'd': - case 's': - case 'f': - case 'n': - case 'p': - case 'u': - case 'x': - case 'E': - case 'C': - { - char l_option[4]; - snprintf(l_option, sizeof(l_option), "-%c", ch); - ParmList::iterator p = fParms.find(l_option); - - if ( fParms.end() != p ) - p->second = optarg; - - break; - } - - case 'h': - { - printUsage(); - return false; - } - - default : - { - cout << "Try '" << argv[0] << - " -h' for more information." << endl; - return false; - } + cout << "Argument associated with option -" << (char)ch << " is not a number." << endl; + return false; } + + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if (errno != 0) + { + cout << "Option -" << (char)ch << " is invalid or out of range" << endl; + return false; + } + + // Limit to INT_MAX because we eventually store in an "int" + if (((ch == 'e') && (lValue < 0)) || ((ch != 'e') && (lValue < 1)) || (lValue > INT_MAX)) + { + cout << "Option -" << (char)ch << " is invalid or out of range." << endl; + return false; + } + } + /* FALLTHRU */ + case 'd': + case 's': + case 'f': + case 'n': + case 'p': + case 'u': + case 'x': + case 'E': + case 'C': + { + char l_option[4]; + snprintf(l_option, sizeof(l_option), "-%c", ch); + ParmList::iterator p = fParms.find(l_option); + + if (fParms.end() != p) + p->second = optarg; + + break; + } + + case 'h': + { + printUsage(); + return false; + } + + default: + { + cout << "Try '" << argv[0] << " -h' for more information." << endl; + return false; + } } + } + + if (optind < argc) + { + fSchema = argv[optind++]; if (optind < argc) { - fSchema = argv[optind++]; - - if ( optind < argc ) - { - cout << "Extraneous arguments are ignored." << endl; - } - } - else - { - cout << "Schema value is required." << endl; - cout << "Try '" << argv[0] << " -h' for more information." << endl; - return false; + cout << "Extraneous arguments are ignored." << endl; } + } + else + { + cout << "Schema value is required." << endl; + cout << "Try '" << argv[0] << " -h' for more information." << endl; + return false; + } - for (unsigned ndx = 0; ndx < l_tableList.size(); ndx++) - { - fTables.push_back(execplan::CalpontSystemCatalog::TableName( - fSchema, l_tableList[ndx])); - } + for (unsigned ndx = 0; ndx < l_tableList.size(); ndx++) + { + fTables.push_back(execplan::CalpontSystemCatalog::TableName(fSchema, l_tableList[ndx])); + } - return true; + return true; } //------------------------------------------------------------------------------ // Get list of tables for fSchema, and store the list in fTables. //------------------------------------------------------------------------------ -bool InputMgr::loadCatalogTables() +bool InputMgr::loadCatalogTables() { - boost::shared_ptr systemCatPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - systemCatPtr->identity(execplan::CalpontSystemCatalog::EC); - const vector< pair > tables - = systemCatPtr->getTables( fSchema ); + boost::shared_ptr systemCatPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + systemCatPtr->identity(execplan::CalpontSystemCatalog::EC); + const vector > tables = + systemCatPtr->getTables(fSchema); - for (vector >::const_iterator it = tables.begin(); - it != tables.end(); ++it) - { - fTables.push_back((*it).second); - } + for (vector >::const_iterator it = + tables.begin(); + it != tables.end(); ++it) + { + fTables.push_back((*it).second); + } - return (! fTables.empty()); + return (!fTables.empty()); } //------------------------------------------------------------------------------ @@ -229,17 +224,17 @@ bool InputMgr::loadCatalogTables() //------------------------------------------------------------------------------ int InputMgr::verifyArgument(char* arg) { - while (*arg != 0) + while (*arg != 0) + { + if (!isdigit(*arg)) { - if (!isdigit(*arg)) - { - return -1; - } - - arg++; + return -1; } - return 0; + arg++; + } + + return 0; } //------------------------------------------------------------------------------ @@ -248,7 +243,7 @@ int InputMgr::verifyArgument(char* arg) /* virtual */ void InputMgr::print(std::ostream& os) const { - os << *this; + os << *this; } //------------------------------------------------------------------------------ @@ -256,31 +251,29 @@ void InputMgr::print(std::ostream& os) const //------------------------------------------------------------------------------ std::ostream& operator<<(std::ostream& os, const InputMgr& m) { - os << "\n\tSchema:\t" << m.fSchema << "\n\t" << "Tables:\t"; + os << "\n\tSchema:\t" << m.fSchema << "\n\t" + << "Tables:\t"; - for ( InputMgr::TableList::const_iterator i = m.fTables.begin(); - i != m.fTables.end(); ++i) - os << i->table << " "; + for (InputMgr::TableList::const_iterator i = m.fTables.begin(); i != m.fTables.end(); ++i) + os << i->table << " "; - os << "\n\t" << "Load Files: "; + os << "\n\t" + << "Load Files: "; - for ( InputMgr::LoadNames::const_iterator i = m.fLoadNames.begin(); - i != m.fLoadNames.end(); ++i) - os << *i << " "; + for (InputMgr::LoadNames::const_iterator i = m.fLoadNames.begin(); i != m.fLoadNames.end(); ++i) + os << *i << " "; - for (InputMgr::ParmList::const_iterator i = m.fParms.begin(); - i != m.fParms.end(); ++i) - { - // Don't report the enclosedBy if not enabled by user - if ((i->first == XMLGenData::ENCLOSED_BY_CHAR) && - (i->second.length() < 1)) - continue; + for (InputMgr::ParmList::const_iterator i = m.fParms.begin(); i != m.fParms.end(); ++i) + { + // Don't report the enclosedBy if not enabled by user + if ((i->first == XMLGenData::ENCLOSED_BY_CHAR) && (i->second.length() < 1)) + continue; - os << "\n\t" << i->first << "\t" << i->second; - } + os << "\n\t" << i->first << "\t" << i->second; + } - os << std::endl; - return os; + os << std::endl; + return os; } -} //namespace bulkloadxml +} // namespace bulkloadxml diff --git a/tools/dbloadxml/inputmgr.h b/tools/dbloadxml/inputmgr.h index 815b8d7cd..d70ed80d8 100644 --- a/tools/dbloadxml/inputmgr.h +++ b/tools/dbloadxml/inputmgr.h @@ -28,43 +28,41 @@ namespace bulkloadxml { - /** @brief Stores Input to colxml; used to generate Job XML file for cpimport. */ class InputMgr : public WriteEngine::XMLGenData { -public: + public: + /** @brief Constructor to manage colxml input. + * + * @param job is the Job Number + */ + InputMgr(const std::string& job); + virtual ~InputMgr(); - /** @brief Constructor to manage colxml input. - * - * @param job is the Job Number - */ - InputMgr(const std::string& job); - virtual ~InputMgr( ); + /** @brief Specify parameter data to be written to Job XML file + * + * @param argc Number of arguments in argv + * @param argv Input arguments + * @return Return true if input accepted; false indicates an error + */ + bool input(int argc, char** argv); - /** @brief Specify parameter data to be written to Job XML file - * - * @param argc Number of arguments in argv - * @param argv Input arguments - * @return Return true if input accepted; false indicates an error - */ - bool input(int argc, char** argv); + /** @brief Load list of tables in system catalog for relevant schema. + */ + bool loadCatalogTables(); - /** @brief Load list of tables in system catalog for relevant schema. - */ - bool loadCatalogTables(); + /** @brief Print contents of this object + */ + virtual void print(std::ostream& os) const; - /** @brief Print contents of this object - */ - virtual void print(std::ostream& os) const; + friend std::ostream& operator<<(std::ostream& os, const InputMgr& m); - friend std::ostream& operator<<(std::ostream& os, const InputMgr& m); - -private: - void printUsage(); - int verifyArgument(char* arg); + private: + void printUsage(); + int verifyArgument(char* arg); }; -} +} // namespace bulkloadxml #endif diff --git a/tools/dbloadxml/resource.h b/tools/dbloadxml/resource.h index e50160fb2..2f3717887 100644 --- a/tools/dbloadxml/resource.h +++ b/tools/dbloadxml/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/tools/ddlcleanup/ddlcleanup.cpp b/tools/ddlcleanup/ddlcleanup.cpp index d557519e0..bbbbac437 100644 --- a/tools/ddlcleanup/ddlcleanup.cpp +++ b/tools/ddlcleanup/ddlcleanup.cpp @@ -16,9 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: ddlcleanup.cpp 967 2009-10-15 13:57:29Z rdempsey $ -*/ - + * $Id: ddlcleanup.cpp 967 2009-10-15 13:57:29Z rdempsey $ + */ #include #include @@ -57,31 +56,28 @@ namespace fs = boost::filesystem; namespace { - - void usage() { - cout << "Usage: ddlcleanup" << endl; -} + cout << "Usage: ddlcleanup" << endl; } +} // namespace int main(int argc, char** argv) { - int c; + int c; - while ((c = getopt(argc, argv, "h")) != EOF) - switch (c) - { - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } + while ((c = getopt(argc, argv, "h")) != EOF) + switch (c) + { + case 'h': + case '?': + default: + usage(); + return (c == 'h' ? 0 : 1); + break; + } - return ddlcleanuputil::ddl_cleanup(); + return ddlcleanuputil::ddl_cleanup(); } // vim:ts=4 sw=4: - diff --git a/tools/ddldriver/ddldriver.cpp b/tools/ddldriver/ddldriver.cpp index 05b05795c..0bf1fa727 100644 --- a/tools/ddldriver/ddldriver.cpp +++ b/tools/ddldriver/ddldriver.cpp @@ -36,99 +36,98 @@ namespace { void usage() { - cout << "usage: ddlriver [-h] schema sql_text" << endl; + cout << "usage: ddlriver [-h] schema sql_text" << endl; } const string toupper_(const string& in) { - string::const_iterator iter = in.begin(); - string::const_iterator end = in.end(); - ostringstream oss; + string::const_iterator iter = in.begin(); + string::const_iterator end = in.end(); + ostringstream oss; - while (iter != end) - { - oss << static_cast(toupper(*iter)); - ++iter; - } + while (iter != end) + { + oss << static_cast(toupper(*iter)); + ++iter; + } - return oss.str(); -} + return oss.str(); } +} // namespace int main(int argc, char** argv) { - int c; + int c; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "h")) != EOF) - switch (c) - { - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if (argc - optind < 2) + while ((c = getopt(argc, argv, "h")) != EOF) + switch (c) { + case 'h': + case '?': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } - string owner(toupper_(argv[optind++])); + if (argc - optind < 2) + { + usage(); + return 1; + } - SqlParser parser; - parser.setDefaultSchema(owner); + string owner(toupper_(argv[optind++])); - string stmtStr(toupper_(argv[optind++])); - parser.Parse(stmtStr.c_str()); + SqlParser parser; + parser.setDefaultSchema(owner); - if (!parser.Good()) - { - cerr << "Failed to parse statement: " << stmtStr << endl; - return 1; - } + string stmtStr(toupper_(argv[optind++])); + parser.Parse(stmtStr.c_str()); - const ParseTree& ptree = parser.GetParseTree(); - SqlStatement& stmt = *ptree.fList[0]; + if (!parser.Good()) + { + cerr << "Failed to parse statement: " << stmtStr << endl; + return 1; + } - stmt.fSessionID = 1; - stmt.fSql = stmtStr; - stmt.fOwner = owner; + const ParseTree& ptree = parser.GetParseTree(); + SqlStatement& stmt = *ptree.fList[0]; - ByteStream bytestream; - bytestream << stmt.fSessionID; - stmt.serialize(bytestream); - MessageQueueClient mq("DDLProc"); - ByteStream::byte b; - string errorMsg; + stmt.fSessionID = 1; + stmt.fSql = stmtStr; + stmt.fOwner = owner; - try - { - mq.write(bytestream); - bytestream = mq.read(); - bytestream >> b; - bytestream >> errorMsg; - } - catch (runtime_error& rex) - { - cerr << "runtime_error in engine: " << rex.what() << endl; - return 1; - } - catch (...) - { - cerr << "uknown error in engine" << endl; - return 1; - } + ByteStream bytestream; + bytestream << stmt.fSessionID; + stmt.serialize(bytestream); + MessageQueueClient mq("DDLProc"); + ByteStream::byte b; + string errorMsg; - if (b != 0) - { - cerr << "DDLProc error: " << errorMsg << endl; - return 1; - } + try + { + mq.write(bytestream); + bytestream = mq.read(); + bytestream >> b; + bytestream >> errorMsg; + } + catch (runtime_error& rex) + { + cerr << "runtime_error in engine: " << rex.what() << endl; + return 1; + } + catch (...) + { + cerr << "uknown error in engine" << endl; + return 1; + } - return 0; + if (b != 0) + { + cerr << "DDLProc error: " << errorMsg << endl; + return 1; + } + + return 0; } - diff --git a/tools/dmldriver/dmldriver.cpp b/tools/dmldriver/dmldriver.cpp index c492fae76..506a6c745 100644 --- a/tools/dmldriver/dmldriver.cpp +++ b/tools/dmldriver/dmldriver.cpp @@ -42,178 +42,164 @@ bool dflg; void usage() { - cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]" << endl; - cout << " -c intvl \tcommit every intvl statements" << endl; - cout << " -f file \tread statements from file (max 15KB/stmt)" << endl; - cout << " -s sid \tset sid as session id" << endl; - cout << " -t flgs \tset trace flags" << endl; - cout << " -v \tdisplay affected row count(s)" << endl; - cout << " -d \tdisplay debug info" << endl; - cout << " -r file \tread orderkeys from file for TPC-H RF2" << endl; - cout << " -p cnt \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl; - cout << " -e schema\tset the schema name (only w/ -r)" << endl; - cout << " -h \tdisplay this help text" << endl; -} + cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]" + << endl; + cout << " -c intvl \tcommit every intvl statements" << endl; + cout << " -f file \tread statements from file (max 15KB/stmt)" << endl; + cout << " -s sid \tset sid as session id" << endl; + cout << " -t flgs \tset trace flags" << endl; + cout << " -v \tdisplay affected row count(s)" << endl; + cout << " -d \tdisplay debug info" << endl; + cout << " -r file \tread orderkeys from file for TPC-H RF2" << endl; + cout << " -p cnt \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl; + cout << " -e schema\tset the schema name (only w/ -r)" << endl; + cout << " -h \tdisplay this help text" << endl; } +} // namespace int main(int argc, char** argv) { - int c; + int c; - opterr = 0; + opterr = 0; - vflg = false; - dflg = false; + vflg = false; + dflg = false; - bool fflg = false; - string infilename; + bool fflg = false; + string infilename; - int cIntvl = numeric_limits::max(); + int cIntvl = numeric_limits::max(); - uint32_t sessionID = time(0) & 0x7fffffff; + uint32_t sessionID = time(0) & 0x7fffffff; - bool rflg = false; + bool rflg = false; - int packCnt = 1; + int packCnt = 1; - uint32_t tflg = 0; + uint32_t tflg = 0; - string schema; + string schema; - while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF) - switch (c) - { - case 'v': - vflg = true; - break; - - case 'd': - dflg = true; - break; - - case 'f': - fflg = true; - infilename = optarg; - break; - - case 'c': - cIntvl = static_cast(strtol(optarg, 0, 0)); - break; - - case 's': - sessionID = static_cast(strtoul(optarg, 0, 0)); - break; - - case 't': - tflg = static_cast(strtoul(optarg, 0, 0)); - break; - - case 'r': - rflg = true; - infilename = optarg; - break; - - case 'p': - packCnt = static_cast(strtol(optarg, 0, 0)); - break; - - case 'e': - schema = optarg; - break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if (!fflg && !rflg && ((argc - optind) < 1)) + while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF) + switch (c) { + case 'v': vflg = true; break; + + case 'd': dflg = true; break; + + case 'f': + fflg = true; + infilename = optarg; + break; + + case 'c': cIntvl = static_cast(strtol(optarg, 0, 0)); break; + + case 's': sessionID = static_cast(strtoul(optarg, 0, 0)); break; + + case 't': tflg = static_cast(strtoul(optarg, 0, 0)); break; + + case 'r': + rflg = true; + infilename = optarg; + break; + + case 'p': packCnt = static_cast(strtol(optarg, 0, 0)); break; + + case 'e': schema = optarg; break; + + case 'h': + case '?': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } - if (fflg && rflg) + if (!fflg && !rflg && ((argc - optind) < 1)) + { + usage(); + return 1; + } + + if (fflg && rflg) + { + cout << "-f and -r are mutually exclusive!" << endl << endl; + usage(); + return 1; + } + + if (!schema.empty() && !rflg) + { + cout << "-e requires -r!" << endl << endl; + usage(); + return 1; + } + + string stmtStr; + + if (!fflg && !rflg) + stmtStr = argv[optind++]; + + int rc = 0; + + dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg); + + if (fflg) + { + ifstream ifs(infilename.c_str()); + + if (!ifs.good()) { - cout << "-f and -r are mutually exclusive!" << endl << endl; - usage(); - return 1; + cerr << "Error accessing file " << infilename << endl; + return 1; } - if (!schema.empty() && !rflg) + const streamsize ilinelen = 15 * 1024; + scoped_array iline(new char[ilinelen]); + int cnt = 0; + + for (;;) { - cout << "-e requires -r!" << endl << endl; - usage(); - return 1; - } + ifs.getline(iline.get(), ilinelen); - string stmtStr; + if (ifs.eof()) + break; - if (!fflg && !rflg) - stmtStr = argv[optind++]; + rc = dmlif.sendOne(iline.get()); - int rc = 0; + if (rc != 0) + break; - dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg); + cnt++; - if (fflg) - { - ifstream ifs(infilename.c_str()); - - if (!ifs.good()) - { - cerr << "Error accessing file " << infilename << endl; - return 1; - } - - const streamsize ilinelen = 15 * 1024; - scoped_array iline(new char[ilinelen]); - int cnt = 0; - - for (;;) - { - ifs.getline(iline.get(), ilinelen); - - if (ifs.eof()) - break; - - rc = dmlif.sendOne(iline.get()); - - if (rc != 0) - break; - - cnt++; - - if ((cnt % cIntvl) == 0) - dmlif.sendOne("COMMIT;"); - } - } - else if (rflg) - { - ifstream ifs(infilename.c_str()); - - if (!ifs.good()) - { - cerr << "Error accessing file " << infilename << endl; - return 1; - } - - if (schema.empty()) - schema = "tpch"; - - tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg); - rc = rf2.run(ifs); - } - else - { - rc = dmlif.sendOne(stmtStr); - } - - if (rc == 0) + if ((cnt % cIntvl) == 0) dmlif.sendOne("COMMIT;"); + } + } + else if (rflg) + { + ifstream ifs(infilename.c_str()); - return 0; + if (!ifs.good()) + { + cerr << "Error accessing file " << infilename << endl; + return 1; + } + + if (schema.empty()) + schema = "tpch"; + + tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg); + rc = rf2.run(ifs); + } + else + { + rc = dmlif.sendOne(stmtStr); + } + + if (rc == 0) + dmlif.sendOne("COMMIT;"); + + return 0; } - diff --git a/tools/dmldriver/dmlif.cpp b/tools/dmldriver/dmlif.cpp index d38000ea9..2887f17d8 100644 --- a/tools/dmldriver/dmlif.cpp +++ b/tools/dmldriver/dmlif.cpp @@ -50,11 +50,10 @@ using namespace dmlif; namespace dmlif { - -DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg) : - fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0) +DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg) + : fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0) { - fMqp.reset(new MessageQueueClient("DMLProc")); + fMqp.reset(new MessageQueueClient("DMLProc")); } DMLIF::~DMLIF() @@ -63,234 +62,75 @@ DMLIF::~DMLIF() int DMLIF::sendOne(const string& stmt) { - int rc; + int rc; - string tStmt(stmt); + string tStmt(stmt); - if (*tStmt.rbegin() != ';') - tStmt += ";"; + if (*tStmt.rbegin() != ';') + tStmt += ";"; - VendorDMLStatement dmlStmt(tStmt, fSessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + VendorDMLStatement dmlStmt(tStmt, fSessionID); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - if (pDMLPackage == 0) - { - cerr << "Failed to parse statement: " << tStmt << endl; - return -1; - } + if (pDMLPackage == 0) + { + cerr << "Failed to parse statement: " << tStmt << endl; + return -1; + } - string queryString = pDMLPackage->get_QueryString(); + string queryString = pDMLPackage->get_QueryString(); - if (fDflg) cout << "qs: >" << queryString << '<' << endl; + if (fDflg) + cout << "qs: >" << queryString << '<' << endl; - string dmlStatement = pDMLPackage->get_DMLStatement(); + string dmlStatement = pDMLPackage->get_DMLStatement(); - if (fDflg) cout << "DML: " << dmlStatement << endl; + if (fDflg) + cout << "DML: " << dmlStatement << endl; - bool isDML = true; + bool isDML = true; - if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK") - { - isDML = false; - } + if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK") + { + isDML = false; + } - if (isDML) - { - char_separator sep(" "); - tokenizer > tok(queryString, sep); - tokenizer >::iterator iter = tok.begin(); - idbassert(iter != tok.end()); - string where = *iter; - ++iter; - idbassert(iter != tok.end()); - string col1 = *iter; - ++iter; - idbassert(iter != tok.end()); - string op = *iter; - ++iter; - idbassert(iter != tok.end()); - string col2 = *iter; - ++iter; - idbassert(iter == tok.end()); + if (isDML) + { + char_separator sep(" "); + tokenizer > tok(queryString, sep); + tokenizer >::iterator iter = tok.begin(); + idbassert(iter != tok.end()); + string where = *iter; + ++iter; + idbassert(iter != tok.end()); + string col1 = *iter; + ++iter; + idbassert(iter != tok.end()); + string op = *iter; + ++iter; + idbassert(iter != tok.end()); + string col2 = *iter; + ++iter; + idbassert(iter == tok.end()); - if (fDflg) cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl; + if (fDflg) + cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl; - if (fDflg) cout << "hf: " << pDMLPackage->HasFilter() << endl; + if (fDflg) + cout << "hf: " << pDMLPackage->HasFilter() << endl; - DMLTable* tp = pDMLPackage->get_Table(); + DMLTable* tp = pDMLPackage->get_Table(); - if (fDflg) cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl; + if (fDflg) + cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl; - if (fDflg) cout << "row count: " << tp->get_RowList().size() << endl; + if (fDflg) + cout << "row count: " << tp->get_RowList().size() << endl; - SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID)); - CalpontSelectExecutionPlan::ColumnMap cm; - cm.insert(make_pair(col1, srcp)); - pDMLPackage->get_ExecutionPlan()->columnMap(cm); - CalpontSelectExecutionPlan::ReturnedColumnList rcl; - rcl.push_back(srcp); - pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); - pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); - pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); - SessionManager sm; - BRM::TxnID txnid = sm.getTxnID(fSessionID); - - if (!txnid.valid) - txnid = sm.newTxnID(fSessionID); - - pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); - pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); - ParseTree* pt = new ParseTree(); - ReturnedColumn* rc1 = srcp->clone(); - ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM); - SOP sop(new Operator(op)); - SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2); - pt->data(sf); - pDMLPackage->get_ExecutionPlan()->filters(pt); - - if (fDflg) cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; - } - - ByteStream bytestream; - pDMLPackage->write(bytestream); - delete pDMLPackage; - ByteStream::octbyte rows; - - rc = DMLSend(bytestream, rows); - - if (isDML && fVflg) - cout << rows << " rows affected" << endl; - - return rc; -} - -int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows) -{ - ByteStream::byte b; - string errorMsg; - - try - { - fMqp->connect(); - fMqp->write(bytestream); - bytestream = fMqp->read(); - fMqp->shutdown(); - - if (fDflg) cout << "read " << bytestream.length() << " bytes from DMLProc" << endl; - - bytestream >> b; - - if (fDflg) cout << "b = " << (int)b << endl; - - bytestream >> rows; - - if (fDflg) cout << "rows = " << rows << endl; - - bytestream >> errorMsg; - - if (fDflg) cout << "errorMsg = " << errorMsg << endl; - } - catch (runtime_error& rex) - { - cerr << "runtime_error in engine: " << rex.what() << endl; - return -1; - } - catch (...) - { - cerr << "uknown error in engine" << endl; - return -1; - } - - if (b != 0) - { - cerr << "DMLProc error: " << errorMsg << endl; - return -1; - } - - return 0; -} - -void DMLIF::rf2Start(const string& sn) -{ - fSchema = sn; - fOFilterStr = ""; - fLFilterStr = ""; - fOPt = 0; - fLPt = 0; -} - -void DMLIF::rf2Add(int64_t okey) -{ - ostringstream oss; - oss << okey; - string okeyStr(oss.str()); - - if (fOFilterStr.empty()) - { - fOFilterStr = "o_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2); - sop.reset(new Operator("or")); - cf->op(sop); - fOPt = new ParseTree(cf); - } - else - { - fOFilterStr += " or o_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = dynamic_cast(fOPt->data()); - cf->pushFilter(new SimpleFilter(sop, rc1, rc2)); - } - - if (fLFilterStr.empty()) - { - fLFilterStr = "l_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2); - sop.reset(new Operator("or")); - cf->op(sop); - fLPt = new ParseTree(cf); - } - else - { - fLFilterStr += " or l_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = dynamic_cast(fLPt->data()); - cf->pushFilter(new SimpleFilter(sop, rc1, rc2)); - } -} - -int DMLIF::rf2Send() -{ - if (fOFilterStr.empty()) - return -1; - - int rc = 0; - string dmlstr; - dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';'; - - if (fDflg) cout << dmlstr << endl; - - VendorDMLStatement dmlStmt(dmlstr, fSessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - - if (pDMLPackage == 0) - { - cerr << "Failed to parse statement: " << dmlstr << endl; - return -1; - } - - SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID)); + SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID)); CalpontSelectExecutionPlan::ColumnMap cm; - cm.insert(make_pair("o_orderkey", srcp)); + cm.insert(make_pair(col1, srcp)); pDMLPackage->get_ExecutionPlan()->columnMap(cm); CalpontSelectExecutionPlan::ReturnedColumnList rcl; rcl.push_back(srcp); @@ -301,71 +141,244 @@ int DMLIF::rf2Send() BRM::TxnID txnid = sm.getTxnID(fSessionID); if (!txnid.valid) - txnid = sm.newTxnID(fSessionID); + txnid = sm.newTxnID(fSessionID); pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); - pDMLPackage->get_ExecutionPlan()->filters(fOPt); + ParseTree* pt = new ParseTree(); + ReturnedColumn* rc1 = srcp->clone(); + ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM); + SOP sop(new Operator(op)); + SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2); + pt->data(sf); + pDMLPackage->get_ExecutionPlan()->filters(pt); - if (fDflg) cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; + if (fDflg) + cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; + } - ByteStream bytestream; - pDMLPackage->write(bytestream); - delete pDMLPackage; - pDMLPackage = 0; - ByteStream::octbyte rows = 0; + ByteStream bytestream; + pDMLPackage->write(bytestream); + delete pDMLPackage; + ByteStream::octbyte rows; - rc = DMLSend(bytestream, rows); + rc = DMLSend(bytestream, rows); - if (fVflg) - cout << rows << " rows affected" << endl; + if (isDML && fVflg) + cout << rows << " rows affected" << endl; - dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';'; - - if (fDflg) cout << dmlstr << endl; - - VendorDMLStatement dmlStmt1(dmlstr, fSessionID); - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1); - - if (pDMLPackage == 0) - { - cerr << "Failed to parse statement: " << dmlstr << endl; - return -1; - } - - srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID)); - cm.clear(); - cm.insert(make_pair("l_orderkey", srcp)); - pDMLPackage->get_ExecutionPlan()->columnMap(cm); - rcl.clear(); - rcl.push_back(srcp); - pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); - pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); - pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); - txnid = sm.getTxnID(fSessionID); - - if (!txnid.valid) - txnid = sm.newTxnID(fSessionID); - - pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); - pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); - pDMLPackage->get_ExecutionPlan()->filters(fLPt); - - if (fDflg) cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; - - bytestream.reset(); - pDMLPackage->write(bytestream); - delete pDMLPackage; - pDMLPackage = 0; - rows = 0; - - rc = DMLSend(bytestream, rows); - - if (fVflg) - cout << rows << " rows affected" << endl; - - return 0; + return rc; } +int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows) +{ + ByteStream::byte b; + string errorMsg; + + try + { + fMqp->connect(); + fMqp->write(bytestream); + bytestream = fMqp->read(); + fMqp->shutdown(); + + if (fDflg) + cout << "read " << bytestream.length() << " bytes from DMLProc" << endl; + + bytestream >> b; + + if (fDflg) + cout << "b = " << (int)b << endl; + + bytestream >> rows; + + if (fDflg) + cout << "rows = " << rows << endl; + + bytestream >> errorMsg; + + if (fDflg) + cout << "errorMsg = " << errorMsg << endl; + } + catch (runtime_error& rex) + { + cerr << "runtime_error in engine: " << rex.what() << endl; + return -1; + } + catch (...) + { + cerr << "uknown error in engine" << endl; + return -1; + } + + if (b != 0) + { + cerr << "DMLProc error: " << errorMsg << endl; + return -1; + } + + return 0; } +void DMLIF::rf2Start(const string& sn) +{ + fSchema = sn; + fOFilterStr = ""; + fLFilterStr = ""; + fOPt = 0; + fLPt = 0; +} + +void DMLIF::rf2Add(int64_t okey) +{ + ostringstream oss; + oss << okey; + string okeyStr(oss.str()); + + if (fOFilterStr.empty()) + { + fOFilterStr = "o_orderkey=" + okeyStr; + ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID); + ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); + SOP sop(new Operator("=")); + ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2); + sop.reset(new Operator("or")); + cf->op(sop); + fOPt = new ParseTree(cf); + } + else + { + fOFilterStr += " or o_orderkey=" + okeyStr; + ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID); + ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); + SOP sop(new Operator("=")); + ConstantFilter* cf = dynamic_cast(fOPt->data()); + cf->pushFilter(new SimpleFilter(sop, rc1, rc2)); + } + + if (fLFilterStr.empty()) + { + fLFilterStr = "l_orderkey=" + okeyStr; + ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID); + ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); + SOP sop(new Operator("=")); + ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2); + sop.reset(new Operator("or")); + cf->op(sop); + fLPt = new ParseTree(cf); + } + else + { + fLFilterStr += " or l_orderkey=" + okeyStr; + ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID); + ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); + SOP sop(new Operator("=")); + ConstantFilter* cf = dynamic_cast(fLPt->data()); + cf->pushFilter(new SimpleFilter(sop, rc1, rc2)); + } +} + +int DMLIF::rf2Send() +{ + if (fOFilterStr.empty()) + return -1; + + int rc = 0; + string dmlstr; + dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';'; + + if (fDflg) + cout << dmlstr << endl; + + VendorDMLStatement dmlStmt(dmlstr, fSessionID); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + + if (pDMLPackage == 0) + { + cerr << "Failed to parse statement: " << dmlstr << endl; + return -1; + } + + SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID)); + CalpontSelectExecutionPlan::ColumnMap cm; + cm.insert(make_pair("o_orderkey", srcp)); + pDMLPackage->get_ExecutionPlan()->columnMap(cm); + CalpontSelectExecutionPlan::ReturnedColumnList rcl; + rcl.push_back(srcp); + pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); + pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); + pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); + SessionManager sm; + BRM::TxnID txnid = sm.getTxnID(fSessionID); + + if (!txnid.valid) + txnid = sm.newTxnID(fSessionID); + + pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); + pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); + pDMLPackage->get_ExecutionPlan()->filters(fOPt); + + if (fDflg) + cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; + + ByteStream bytestream; + pDMLPackage->write(bytestream); + delete pDMLPackage; + pDMLPackage = 0; + ByteStream::octbyte rows = 0; + + rc = DMLSend(bytestream, rows); + + if (fVflg) + cout << rows << " rows affected" << endl; + + dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';'; + + if (fDflg) + cout << dmlstr << endl; + + VendorDMLStatement dmlStmt1(dmlstr, fSessionID); + pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1); + + if (pDMLPackage == 0) + { + cerr << "Failed to parse statement: " << dmlstr << endl; + return -1; + } + + srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID)); + cm.clear(); + cm.insert(make_pair("l_orderkey", srcp)); + pDMLPackage->get_ExecutionPlan()->columnMap(cm); + rcl.clear(); + rcl.push_back(srcp); + pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); + pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); + pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); + txnid = sm.getTxnID(fSessionID); + + if (!txnid.valid) + txnid = sm.newTxnID(fSessionID); + + pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); + pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); + pDMLPackage->get_ExecutionPlan()->filters(fLPt); + + if (fDflg) + cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; + + bytestream.reset(); + pDMLPackage->write(bytestream); + delete pDMLPackage; + pDMLPackage = 0; + rows = 0; + + rc = DMLSend(bytestream, rows); + + if (fVflg) + cout << rows << " rows affected" << endl; + + return 0; +} + +} // namespace dmlif diff --git a/tools/dmldriver/dmlif.h b/tools/dmldriver/dmlif.h index 2c935d69e..0fc648d74 100644 --- a/tools/dmldriver/dmlif.h +++ b/tools/dmldriver/dmlif.h @@ -29,42 +29,40 @@ namespace dmlif { - class DMLIF { -public: - DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false); - ~DMLIF(); + public: + DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false); + ~DMLIF(); - int sendOne(const std::string& stmt); + int sendOne(const std::string& stmt); - void rf2Start(const std::string& sn); - void rf2Add(int64_t okey); - int rf2Send(); + void rf2Start(const std::string& sn); + void rf2Add(int64_t okey); + int rf2Send(); -protected: - int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows); + protected: + int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows); -private: - //DMLIF(const DMLIF& rhs); - //DMLIF& operator=(const DMLIF& rhs); + private: + // DMLIF(const DMLIF& rhs); + // DMLIF& operator=(const DMLIF& rhs); - uint32_t fSessionID; - uint32_t fTflg; - bool fDflg; - bool fVflg; + uint32_t fSessionID; + uint32_t fTflg; + bool fDflg; + bool fVflg; - boost::scoped_ptr fMqp; + boost::scoped_ptr fMqp; - std::string fSchema; - std::string fOFilterStr; - std::string fLFilterStr; + std::string fSchema; + std::string fOFilterStr; + std::string fLFilterStr; - execplan::ParseTree* fOPt; - execplan::ParseTree* fLPt; + execplan::ParseTree* fOPt; + execplan::ParseTree* fLPt; }; -} +} // namespace dmlif #endif - diff --git a/tools/dmldriver/tpchrf2.cpp b/tools/dmldriver/tpchrf2.cpp index 9c9edb918..03ef3742d 100644 --- a/tools/dmldriver/tpchrf2.cpp +++ b/tools/dmldriver/tpchrf2.cpp @@ -29,9 +29,8 @@ using namespace boost; namespace tpch { - -RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v) : - fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v) +RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v) + : fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v) { } @@ -41,47 +40,47 @@ RF2::~RF2() int RF2::run(istream& in) { - const streamsize ilinelen = 1024; - scoped_array iline(new char[ilinelen]); - int cnt = 0; - dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg); + const streamsize ilinelen = 1024; + scoped_array iline(new char[ilinelen]); + int cnt = 0; + dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg); - for (;;) + for (;;) + { + dmlif.rf2Start(fSchema); + + for (int i = 0; i < fPack; i++) { - dmlif.rf2Start(fSchema); + in.getline(iline.get(), ilinelen); - for (int i = 0; i < fPack; i++) - { - in.getline(iline.get(), ilinelen); + if (in.eof()) + break; - if (in.eof()) - break; - - typedef char_separator cs; - typedef tokenizer tk; - cs sep("|"); - tk tok(string(iline.get()), sep); - tk::iterator iter = tok.begin(); - idbassert(iter != tok.end()); - string keystr = *iter; - ++iter; - //idbassert(iter == tok.end()); - int64_t okey = strtol(keystr.c_str(), 0, 0); - dmlif.rf2Add(okey); - } - - dmlif.rf2Send(); - cnt++; - - if ((cnt % fIntvl) == 0) - dmlif.sendOne("COMMIT;"); - - if (in.eof()) - break; + typedef char_separator cs; + typedef tokenizer tk; + cs sep("|"); + tk tok(string(iline.get()), sep); + tk::iterator iter = tok.begin(); + idbassert(iter != tok.end()); + string keystr = *iter; + ++iter; + // idbassert(iter == tok.end()); + int64_t okey = strtol(keystr.c_str(), 0, 0); + dmlif.rf2Add(okey); } - dmlif.sendOne("COMMIT;"); - return 0; + dmlif.rf2Send(); + cnt++; + + if ((cnt % fIntvl) == 0) + dmlif.sendOne("COMMIT;"); + + if (in.eof()) + break; + } + + dmlif.sendOne("COMMIT;"); + return 0; } -} +} // namespace tpch diff --git a/tools/dmldriver/tpchrf2.h b/tools/dmldriver/tpchrf2.h index 43bb79f0f..018676f24 100644 --- a/tools/dmldriver/tpchrf2.h +++ b/tools/dmldriver/tpchrf2.h @@ -28,29 +28,28 @@ namespace tpch { - class RF2 { -public: - RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits::max(), - int p = 1, bool d = false, bool v = false); - ~RF2(); + public: + RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits::max(), + int p = 1, bool d = false, bool v = false); + ~RF2(); - int run(std::istream& in); + int run(std::istream& in); -private: - //RF2(const RF2& rhs); - //RF2& operator=(const RF2& rhs); + private: + // RF2(const RF2& rhs); + // RF2& operator=(const RF2& rhs); - std::string fSchema; - uint32_t fSessionID; - uint32_t fTflg; - int fIntvl; - int fPack; - bool fDflg; - bool fVflg; + std::string fSchema; + uint32_t fSessionID; + uint32_t fTflg; + int fIntvl; + int fPack; + bool fDflg; + bool fVflg; }; -} +} // namespace tpch #endif diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index e106db90d..2dcb6ce81 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -16,9 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: editem.cpp 2336 2013-06-25 19:11:36Z rdempsey $ -*/ - + * $Id: editem.cpp 2336 2013-06-25 19:11:36Z rdempsey $ + */ #include #include @@ -50,14 +49,18 @@ using namespace dataconvert; #undef REALLY_DANGEROUS -#define CHECK(cmd) { int rc = (cmd);\ -if ((rc) != 0)\ -{ cerr << "Error in DBRM call " #cmd "; returnCode: " << rc << endl;\ -return 1; } } +#define CHECK(cmd) \ + { \ + int rc = (cmd); \ + if ((rc) != 0) \ + { \ + cerr << "Error in DBRM call " #cmd "; returnCode: " << rc << endl; \ + return 1; \ + } \ + } namespace { - OID_t MaxOID; DBRM* emp = 0; @@ -74,42 +77,34 @@ bool uflg = false; struct SortExtentsByPartitionFirst { - bool operator() (const EMEntry& entry1, const EMEntry& entry2) - { - if ( (entry1.partitionNum < entry2.partitionNum) || - ((entry1.partitionNum == entry2.partitionNum) && - (entry1.dbRoot < entry2.dbRoot)) || - ((entry1.partitionNum == entry2.partitionNum) && - (entry1.dbRoot == entry2.dbRoot) && - (entry1.segmentNum < entry2.segmentNum)) || - ((entry1.partitionNum == entry2.partitionNum) && - (entry1.dbRoot == entry2.dbRoot) && - (entry1.segmentNum == entry2.segmentNum) && - (entry1.blockOffset < entry2.blockOffset)) ) - return true; - else - return false; - } + bool operator()(const EMEntry& entry1, const EMEntry& entry2) + { + if ((entry1.partitionNum < entry2.partitionNum) || + ((entry1.partitionNum == entry2.partitionNum) && (entry1.dbRoot < entry2.dbRoot)) || + ((entry1.partitionNum == entry2.partitionNum) && (entry1.dbRoot == entry2.dbRoot) && + (entry1.segmentNum < entry2.segmentNum)) || + ((entry1.partitionNum == entry2.partitionNum) && (entry1.dbRoot == entry2.dbRoot) && + (entry1.segmentNum == entry2.segmentNum) && (entry1.blockOffset < entry2.blockOffset))) + return true; + else + return false; + } }; struct SortExtentsByDBRootFirst { - bool operator() (const EMEntry& entry1, const EMEntry& entry2) - { - if ( (entry1.dbRoot < entry2.dbRoot) || - ((entry1.dbRoot == entry2.dbRoot) && - (entry1.partitionNum < entry2.partitionNum)) || - ((entry1.dbRoot == entry2.dbRoot) && - (entry1.partitionNum == entry2.partitionNum) && - (entry1.segmentNum < entry2.segmentNum)) || - ((entry1.dbRoot == entry2.dbRoot) && - (entry1.partitionNum == entry2.partitionNum) && - (entry1.segmentNum == entry2.segmentNum) && - (entry1.blockOffset < entry2.blockOffset)) ) - return true; - else - return false; - } + bool operator()(const EMEntry& entry1, const EMEntry& entry2) + { + if ((entry1.dbRoot < entry2.dbRoot) || + ((entry1.dbRoot == entry2.dbRoot) && (entry1.partitionNum < entry2.partitionNum)) || + ((entry1.dbRoot == entry2.dbRoot) && (entry1.partitionNum == entry2.partitionNum) && + (entry1.segmentNum < entry2.segmentNum)) || + ((entry1.dbRoot == entry2.dbRoot) && (entry1.partitionNum == entry2.partitionNum) && + (entry1.segmentNum == entry2.segmentNum) && (entry1.blockOffset < entry2.blockOffset))) + return true; + else + return false; + } }; //------------------------------------------------------------------------------ @@ -117,32 +112,33 @@ struct SortExtentsByDBRootFirst //------------------------------------------------------------------------------ void usage(const string& pname) { - cout << "usage: " << pname << - " [-tsahvm] [-di]|[-o oid -S opt]|[-c oid]|[-x]|[-e oid]|[-r oid]|" - "[-w oid]|[-l]|[-b lbid][-C][-p dbr]" << endl << - " examins/modifies the extent map." << endl << - " -h \tdisplay this help text" << endl << - " -o oid \tdisplay extent map for oid" << endl << - " -S opt \tSort order for -o (1-partition, dbroot, seg#, fbo;" << endl << - " \t 2-dbroot, partition, seg#, fbo;" << endl << - " \t default is unsorted)" << endl << - " -d \tdump the entire extent map" << endl << - " -c oid \tclear the min/max vals for oid" << endl << - " -C \tclear all min/max vals" << endl << - " -t \tdisplay min/max values as dates" << endl << - " -s \tdisplay min/max values as timestamps" << endl << - " -a \tdisplay min/max values as char strings" << endl << - " -u \tdisplay min/max values as unsigned integers" << endl << - " -x \tcreate/extend one or more oids" << endl << - " -e oid \tdelete oid" << endl << - " -r oid \trollback or delete extents" << endl << - " -v \tdisplay verbose output" << endl << - " -w oid \tedit HWM for an oid" << endl << - " -l \tdump the free list" << endl << - " -b lbid\tdisplay info about lbid" << endl << - " -i \tformat the output for import (implies -dm)" << endl << - " -m \tdisplay actual min/max values" << endl << - " -p dbr \tdelete all extents on dbroot dbr" << endl; + cout << "usage: " << pname + << " [-tsahvm] [-di]|[-o oid -S opt]|[-c oid]|[-x]|[-e oid]|[-r oid]|" + "[-w oid]|[-l]|[-b lbid][-C][-p dbr]" + << endl + << " examins/modifies the extent map." << endl + << " -h \tdisplay this help text" << endl + << " -o oid \tdisplay extent map for oid" << endl + << " -S opt \tSort order for -o (1-partition, dbroot, seg#, fbo;" << endl + << " \t 2-dbroot, partition, seg#, fbo;" << endl + << " \t default is unsorted)" << endl + << " -d \tdump the entire extent map" << endl + << " -c oid \tclear the min/max vals for oid" << endl + << " -C \tclear all min/max vals" << endl + << " -t \tdisplay min/max values as dates" << endl + << " -s \tdisplay min/max values as timestamps" << endl + << " -a \tdisplay min/max values as char strings" << endl + << " -u \tdisplay min/max values as unsigned integers" << endl + << " -x \tcreate/extend one or more oids" << endl + << " -e oid \tdelete oid" << endl + << " -r oid \trollback or delete extents" << endl + << " -v \tdisplay verbose output" << endl + << " -w oid \tedit HWM for an oid" << endl + << " -l \tdump the free list" << endl + << " -b lbid\tdisplay info about lbid" << endl + << " -i \tformat the output for import (implies -dm)" << endl + << " -m \tdisplay actual min/max values" << endl + << " -p dbr \tdelete all extents on dbroot dbr" << endl; } //------------------------------------------------------------------------------ @@ -150,95 +146,94 @@ void usage(const string& pname) //------------------------------------------------------------------------------ const string charcolToString(int64_t v) { - ostringstream oss; - char c; + ostringstream oss; + char c; - for (int i = 0; i < 8; i++) - { - c = v & 0xff; - oss << c; - v >>= 8; - } + for (int i = 0; i < 8; i++) + { + c = v & 0xff; + oss << c; + v >>= 8; + } - return oss.str(); + return oss.str(); } //------------------------------------------------------------------------------ // Formats an integer to it's date, datetime, or char equivalent //------------------------------------------------------------------------------ -template +template const string fmt(T v) { - ostringstream oss; + ostringstream oss; - if (tflg) + if (tflg) + { + oss << DataConvert::dateToString((int64_t)v); + } + else if (sflg) + { + oss << DataConvert::datetimeToString((int64_t)v); + } + else if (aflg) + { + oss << charcolToString((int64_t)v); + } + else if (mflg) + { + if (typeid(T) != typeid(int128_t)) { - oss << DataConvert::dateToString((int64_t) v); - } - else if (sflg) - { - oss << DataConvert::datetimeToString((int64_t) v); - } - else if (aflg) - { - oss << charcolToString((int64_t) v); - } - else if (mflg) - { - if (typeid(T) != typeid(int128_t)) - { - oss << (int64_t) v; - } - else - { - oss << datatypes::TSInt128(v); - } - } - else if (uflg) - { - if (typeid(T) != typeid(int128_t)) - { - if (static_cast(v) > numeric_limits::max() - 2) - oss << "notset"; - else - oss << static_cast(v); - } - else - { - if (v <= utils::minInt128 + 1) - { - oss << "notset"; - } - else - { - oss << datatypes::TSInt128(v); - } - } + oss << (int64_t)v; } else { - if (typeid(T) != typeid(int128_t)) - { - if (v == numeric_limits::max() || - v <= (numeric_limits::min() + 1)) - oss << "notset"; - else - oss << (int64_t) v; - } - else - { - if (v == utils::maxInt128 || (v <= utils::minInt128 + 1)) - { - oss << "notset"; - } - else - { - oss << datatypes::TSInt128(v); - } - } + oss << datatypes::TSInt128(v); } + } + else if (uflg) + { + if (typeid(T) != typeid(int128_t)) + { + if (static_cast(v) > numeric_limits::max() - 2) + oss << "notset"; + else + oss << static_cast(v); + } + else + { + if (v <= utils::minInt128 + 1) + { + oss << "notset"; + } + else + { + oss << datatypes::TSInt128(v); + } + } + } + else + { + if (typeid(T) != typeid(int128_t)) + { + if (v == numeric_limits::max() || v <= (numeric_limits::min() + 1)) + oss << "notset"; + else + oss << (int64_t)v; + } + else + { + if (v == utils::maxInt128 || (v <= utils::minInt128 + 1)) + { + oss << "notset"; + } + else + { + oss << datatypes::TSInt128(v); + } + } + } - return oss.str(); + return oss.str(); } //------------------------------------------------------------------------------ @@ -251,15 +246,18 @@ const string fmt(T v) //@bug 4914: Validate that correct number of input parameters is given bool isInputValid() { - if (cin.good()) - return true; + if (cin.good()) + return true; - cin.clear(); - cin.ignore( numeric_limits::max(), '\n'); - cout << endl << "Invalid input; try again, be sure to enter spaces " - "between input parameters." << endl << endl; + cin.clear(); + cin.ignore(numeric_limits::max(), '\n'); + cout << endl + << "Invalid input; try again, be sure to enter spaces " + "between input parameters." + << endl + << endl; - return false; + return false; } //------------------------------------------------------------------------------ @@ -267,147 +265,126 @@ bool isInputValid() //------------------------------------------------------------------------------ int dumpone(OID_t oid, unsigned int sortOrder) { - std::vector entries; - std::vector::iterator iter; - std::vector::iterator end; - int64_t max; - int64_t min; - int128_t bigMax; - int128_t bigMin; - int32_t seqNum; - bool header; - bool needtrailer = false; - unsigned extentRows = emp->getExtentRows(); - unsigned colWidth = 0; + std::vector entries; + std::vector::iterator iter; + std::vector::iterator end; + int64_t max; + int64_t min; + int128_t bigMax; + int128_t bigMin; + int32_t seqNum; + bool header; + bool needtrailer = false; + unsigned extentRows = emp->getExtentRows(); + unsigned colWidth = 0; - CHECK(emp->getExtents(oid, entries, false, false, true)); + CHECK(emp->getExtents(oid, entries, false, false, true)); - if (entries.size() > 0) + if (entries.size() > 0) + { + if (sortOrder == 1) { - if (sortOrder == 1) - { - SortExtentsByPartitionFirst sorter; - std::sort( entries.begin(), entries.end(), sorter ); - } - else if (sortOrder == 2) - { - SortExtentsByDBRootFirst sorter; - std::sort( entries.begin(), entries.end(), sorter ); - } - - header = false; - iter = entries.begin(); - end = entries.end(); - - while (iter != end) - { - uint32_t lbidRangeSize = iter->range.size * 1024; - seqNum = iter->partition.cprange.sequenceNum; - int state = iter->partition.cprange.isValid; - - if (!header) - { - if ( iter->colWid > 0 ) - { - cout << "Col OID = " << oid << ", NumExtents = " << - entries.size() << ", width = " << iter->colWid << endl; - colWidth = iter->colWid; - } - else - { - cout << "Dct OID = " << oid << endl; - colWidth = DICT_COL_WIDTH; - } - - header = true; - } - - if (vflg) - cout << oid << ' '; - - if (iter->colWid != datatypes::MAXDECIMALWIDTH) - { - max = iter->partition.cprange.hiVal; - min = iter->partition.cprange.loVal; - - cout << iter->range.start << " - " << - (iter->range.start + lbidRangeSize - 1) << - " (" << lbidRangeSize << ") min: " << fmt(min) << - ", max: " << fmt(max) << ", seqNum: " << seqNum << ", state: "; - } - else - { - bigMax = iter->partition.cprange.bigHiVal; - bigMin = iter->partition.cprange.bigLoVal; - - cout << iter->range.start << " - " << - (iter->range.start + lbidRangeSize - 1) << - " (" << lbidRangeSize << ") min: " << fmt(bigMin) << - ", max: " << fmt(bigMax) << ", seqNum: " << seqNum << ", state: "; - } - - switch (state) - { - case 0: - cout << "invalid"; - break; - - case 1: - cout << "updating"; - break; - - case 2: - cout << "valid"; - break; - - default: - cout << "unknown"; - break; - } - - cout << ", fbo: " << iter->blockOffset; - cout << ", DBRoot: " << iter->dbRoot << - ", part#: " << iter->partitionNum << - ", seg#: " << iter->segmentNum; - cout << ", HWM: " << iter->HWM; - - switch (iter->status) - { - case EXTENTAVAILABLE: - cout << "; status: avail"; - break; - - case EXTENTUNAVAILABLE: - cout << "; status: unavail"; - break; - - case EXTENTOUTOFSERVICE: - cout << "; status: outOfSrv"; - break; - - default: - cout << "; status: unknown"; - break; - } - - cout << endl; - - //Complain loudly if there's a mis-match - if (lbidRangeSize != (extentRows * colWidth / BLOCK_SIZE)) - { - cout << endl; - throw logic_error( - "Extent Map entries do match config file setting!"); - } - - needtrailer = true; - ++iter; - } - - if (needtrailer) cout << endl; + SortExtentsByPartitionFirst sorter; + std::sort(entries.begin(), entries.end(), sorter); + } + else if (sortOrder == 2) + { + SortExtentsByDBRootFirst sorter; + std::sort(entries.begin(), entries.end(), sorter); } - return 0; + header = false; + iter = entries.begin(); + end = entries.end(); + + while (iter != end) + { + uint32_t lbidRangeSize = iter->range.size * 1024; + seqNum = iter->partition.cprange.sequenceNum; + int state = iter->partition.cprange.isValid; + + if (!header) + { + if (iter->colWid > 0) + { + cout << "Col OID = " << oid << ", NumExtents = " << entries.size() << ", width = " << iter->colWid + << endl; + colWidth = iter->colWid; + } + else + { + cout << "Dct OID = " << oid << endl; + colWidth = DICT_COL_WIDTH; + } + + header = true; + } + + if (vflg) + cout << oid << ' '; + + if (iter->colWid != datatypes::MAXDECIMALWIDTH) + { + max = iter->partition.cprange.hiVal; + min = iter->partition.cprange.loVal; + + cout << iter->range.start << " - " << (iter->range.start + lbidRangeSize - 1) << " (" << lbidRangeSize + << ") min: " << fmt(min) << ", max: " << fmt(max) << ", seqNum: " << seqNum << ", state: "; + } + else + { + bigMax = iter->partition.cprange.bigHiVal; + bigMin = iter->partition.cprange.bigLoVal; + + cout << iter->range.start << " - " << (iter->range.start + lbidRangeSize - 1) << " (" << lbidRangeSize + << ") min: " << fmt(bigMin) << ", max: " << fmt(bigMax) << ", seqNum: " << seqNum << ", state: "; + } + + switch (state) + { + case 0: cout << "invalid"; break; + + case 1: cout << "updating"; break; + + case 2: cout << "valid"; break; + + default: cout << "unknown"; break; + } + + cout << ", fbo: " << iter->blockOffset; + cout << ", DBRoot: " << iter->dbRoot << ", part#: " << iter->partitionNum + << ", seg#: " << iter->segmentNum; + cout << ", HWM: " << iter->HWM; + + switch (iter->status) + { + case EXTENTAVAILABLE: cout << "; status: avail"; break; + + case EXTENTUNAVAILABLE: cout << "; status: unavail"; break; + + case EXTENTOUTOFSERVICE: cout << "; status: outOfSrv"; break; + + default: cout << "; status: unknown"; break; + } + + cout << endl; + + // Complain loudly if there's a mis-match + if (lbidRangeSize != (extentRows * colWidth / BLOCK_SIZE)) + { + cout << endl; + throw logic_error("Extent Map entries do match config file setting!"); + } + + needtrailer = true; + ++iter; + } + + if (needtrailer) + cout << endl; + } + + return 0; } //------------------------------------------------------------------------------ @@ -415,12 +392,12 @@ int dumpone(OID_t oid, unsigned int sortOrder) //------------------------------------------------------------------------------ int dumpall() { - for (OID_t oid = 0; oid <= MaxOID; oid++) - { - dumpone(oid, 0 /* no sorting */); - } + for (OID_t oid = 0; oid <= MaxOID; oid++) + { + dumpone(oid, 0 /* no sorting */); + } - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -429,23 +406,23 @@ int dumpall() int zapit() { #ifdef REALLY_DANGEROUS - LBIDRange_v range; + LBIDRange_v range; - for (OID_t oid = 0; oid <= MaxOID; oid++) + for (OID_t oid = 0; oid <= MaxOID; oid++) + { + CHECK(emp->lookup(oid, range)); + + if (range.size() > 0) { - CHECK(emp->lookup(oid, range)); - - if (range.size() > 0) - { - CHECK(emp->deleteOID(oid)); - CHECK(emp->confirmChanges()); - } + CHECK(emp->deleteOID(oid)); + CHECK(emp->confirmChanges()); } + } #else - cerr << "Sorry, I'm not going to do that." << endl; + cerr << "Sorry, I'm not going to do that." << endl; #endif - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -453,53 +430,53 @@ int zapit() //------------------------------------------------------------------------------ int clearAllCPData() { - BRM::LBIDRange_v ranges; - int oid, err; + BRM::LBIDRange_v ranges; + int oid, err; - for (oid = 0; oid < MaxOID; oid++) + for (oid = 0; oid < MaxOID; oid++) + { + err = emp->lookup(oid, ranges); + + if (err == 0 && ranges.size() > 0) { - err = emp->lookup(oid, ranges); + // Get the extents for a given OID to determine it's column width + std::vector entries; + CHECK(emp->getExtents(oid, entries, false, false, true)); - if (err == 0 && ranges.size() > 0) - { - // Get the extents for a given OID to determine it's column width - std::vector entries; - CHECK(emp->getExtents(oid, entries, false, false, true)); + if (entries.empty()) + continue; - if (entries.empty()) - continue; + bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); - bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); + BRM::CPInfo cpInfo; + BRM::CPInfoList_t vCpInfo; - BRM::CPInfo cpInfo; - BRM::CPInfoList_t vCpInfo; + if (!isBinaryColumn) + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } - if (!isBinaryColumn) - { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); - } - else - { - utils::int128Min(cpInfo.bigMax); - utils::int128Max(cpInfo.bigMin); - } + cpInfo.seqNum = -1; + cpInfo.isBinaryColumn = isBinaryColumn; - cpInfo.seqNum = -1; - cpInfo.isBinaryColumn = isBinaryColumn; + for (uint32_t i = 0; i < ranges.size(); i++) + { + BRM::LBIDRange r = ranges.at(i); + cpInfo.firstLbid = r.start; + vCpInfo.push_back(cpInfo); + } - for (uint32_t i = 0; i < ranges.size(); i++) - { - BRM::LBIDRange r = ranges.at(i); - cpInfo.firstLbid = r.start; - vCpInfo.push_back(cpInfo); - } - - CHECK(emp->setExtentsMaxMin(vCpInfo)); - } + CHECK(emp->setExtentsMaxMin(vCpInfo)); } + } - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -507,168 +484,162 @@ int clearAllCPData() //------------------------------------------------------------------------------ int clearmm(OID_t oid) { - BRM::LBIDRange_v ranges; - CHECK(emp->lookup(oid, ranges)); - BRM::LBIDRange_v::size_type rcount = ranges.size(); + BRM::LBIDRange_v ranges; + CHECK(emp->lookup(oid, ranges)); + BRM::LBIDRange_v::size_type rcount = ranges.size(); - // Get the extents for a given OID to determine it's column width - std::vector entries; - CHECK(emp->getExtents(oid, entries, false, false, true)); - if (entries.empty()) - { - cerr << "There are no entries in the Extent Map for OID: " << oid << endl; - return 1; - } + // Get the extents for a given OID to determine it's column width + std::vector entries; + CHECK(emp->getExtents(oid, entries, false, false, true)); + if (entries.empty()) + { + cerr << "There are no entries in the Extent Map for OID: " << oid << endl; + return 1; + } - bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); + bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); - // @bug 2280. Changed to use the batch interface to clear the CP info to make the clear option faster. - BRM::CPInfo cpInfo; - BRM::CPInfoList_t vCpInfo; + // @bug 2280. Changed to use the batch interface to clear the CP info to make the clear option faster. + BRM::CPInfo cpInfo; + BRM::CPInfoList_t vCpInfo; - if (!isBinaryColumn) - { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); - } - else - { - utils::int128Min(cpInfo.bigMax); - utils::int128Max(cpInfo.bigMin); - } + if (!isBinaryColumn) + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } - cpInfo.seqNum = -1; - cpInfo.isBinaryColumn = isBinaryColumn; + cpInfo.seqNum = -1; + cpInfo.isBinaryColumn = isBinaryColumn; - for (unsigned i = 0; i < rcount; i++) - { - BRM::LBIDRange r = ranges.at(i); - cpInfo.firstLbid = r.start; - vCpInfo.push_back(cpInfo); - } + for (unsigned i = 0; i < rcount; i++) + { + BRM::LBIDRange r = ranges.at(i); + cpInfo.firstLbid = r.start; + vCpInfo.push_back(cpInfo); + } - CHECK(emp->setExtentsMaxMin(vCpInfo)); + CHECK(emp->setExtentsMaxMin(vCpInfo)); - return 0; + return 0; } //------------------------------------------------------------------------------ // Create/add extents to dictionary OID, or a list of column OIDs. //------------------------------------------------------------------------------ -int extendOids( ) +int extendOids() { - uint16_t dbRoot; - uint32_t partNum; - uint16_t segNum; - OID_t oid; - uint32_t colWidth; - char DictStoreOIDFlag; + uint16_t dbRoot; + uint32_t partNum; + uint16_t segNum; + OID_t oid; + uint32_t colWidth; + char DictStoreOIDFlag; - vector cols; + vector cols; - cout << "Are you extending a dictionary store oid (y/n)? "; - cin >> DictStoreOIDFlag; + cout << "Are you extending a dictionary store oid (y/n)? "; + cin >> DictStoreOIDFlag; - if ((DictStoreOIDFlag == 'y') || (DictStoreOIDFlag == 'Y')) + if ((DictStoreOIDFlag == 'y') || (DictStoreOIDFlag == 'Y')) + { + LBID_t lbid; + int allocd; + + cout << "Enter OID, DBRoot, and Partition#, and Segment# " + "(separated by spaces): "; + + while (1) { - LBID_t lbid; - int allocd; + cin >> oid >> dbRoot >> partNum >> segNum; - cout << "Enter OID, DBRoot, and Partition#, and Segment# " - "(separated by spaces): "; - - while (1) - { - cin >> oid >> dbRoot >> partNum >> segNum; - - if (isInputValid()) - break; - } - - CHECK(emp->createDictStoreExtent ( oid, dbRoot, partNum, segNum, - lbid, allocd)); - - if (vflg) - { - cout << oid << " created/extended w/ " << allocd << " blocks; " - "beginning LBID: " << lbid << - "; DBRoot: " << dbRoot << - "; Part#: " << partNum << - "; Seg#: " << segNum << endl; - } - } - else - { - while (1) - { - bool bFinished = false; - - while (1) - { - cout << "Enter OID and column width (separated by spaces); " - "0 OID represents end of list: "; - cin >> oid >> colWidth; - - if (oid == 0) - { - bFinished = true; - break; - } - - if (isInputValid()) - break; - } - - if (bFinished) - break; - - CreateStripeColumnExtentsArgIn colArg; - colArg.oid = oid; - colArg.width = colWidth; - - if (uflg) - { - colArg.colDataType = execplan::CalpontSystemCatalog::UBIGINT; - } - else - { - colArg.colDataType = execplan::CalpontSystemCatalog::BIGINT; - } - - cols.push_back( colArg ); - } - - vector newExtents; - - while (1) - { - cout << "Enter DBRoot and partition# (partition " - "only used for empty DBRoot): "; - cin >> dbRoot >> partNum; - - if (isInputValid()) - break; - } - - CHECK(emp->createStripeColumnExtents(cols, dbRoot, partNum, - segNum, newExtents)); - - cout << "Extents created in partition " << partNum << - ", segment " << segNum << endl; - - if (vflg) - { - for (unsigned k = 0; k < newExtents.size(); k++) - { - cout << "Column OID-" << cols[k].oid << - "; LBID-" << newExtents[k].startLbid << - "; nblks-" << newExtents[k].allocSize << - "; fbo-" << newExtents[k].startBlkOffset << endl; - } - } + if (isInputValid()) + break; } - return 0; + CHECK(emp->createDictStoreExtent(oid, dbRoot, partNum, segNum, lbid, allocd)); + + if (vflg) + { + cout << oid << " created/extended w/ " << allocd + << " blocks; " + "beginning LBID: " + << lbid << "; DBRoot: " << dbRoot << "; Part#: " << partNum << "; Seg#: " << segNum << endl; + } + } + else + { + while (1) + { + bool bFinished = false; + + while (1) + { + cout << "Enter OID and column width (separated by spaces); " + "0 OID represents end of list: "; + cin >> oid >> colWidth; + + if (oid == 0) + { + bFinished = true; + break; + } + + if (isInputValid()) + break; + } + + if (bFinished) + break; + + CreateStripeColumnExtentsArgIn colArg; + colArg.oid = oid; + colArg.width = colWidth; + + if (uflg) + { + colArg.colDataType = execplan::CalpontSystemCatalog::UBIGINT; + } + else + { + colArg.colDataType = execplan::CalpontSystemCatalog::BIGINT; + } + + cols.push_back(colArg); + } + + vector newExtents; + + while (1) + { + cout << "Enter DBRoot and partition# (partition " + "only used for empty DBRoot): "; + cin >> dbRoot >> partNum; + + if (isInputValid()) + break; + } + + CHECK(emp->createStripeColumnExtents(cols, dbRoot, partNum, segNum, newExtents)); + + cout << "Extents created in partition " << partNum << ", segment " << segNum << endl; + + if (vflg) + { + for (unsigned k = 0; k < newExtents.size(); k++) + { + cout << "Column OID-" << cols[k].oid << "; LBID-" << newExtents[k].startLbid << "; nblks-" + << newExtents[k].allocSize << "; fbo-" << newExtents[k].startBlkOffset << endl; + } + } + } + + return 0; } //------------------------------------------------------------------------------ @@ -677,67 +648,65 @@ int extendOids( ) //------------------------------------------------------------------------------ int rollbackExtents(OID_t oid) { - char DictStoreOIDFlag; - uint32_t partNum; - uint16_t dbRoot; - uint16_t segNum; - HWM_t hwm; + char DictStoreOIDFlag; + uint32_t partNum; + uint16_t dbRoot; + uint16_t segNum; + HWM_t hwm; - cout << "Are you rolling back extents for a dictionary store oid (y/n)? "; - cin >> DictStoreOIDFlag; + cout << "Are you rolling back extents for a dictionary store oid (y/n)? "; + cin >> DictStoreOIDFlag; - if ((DictStoreOIDFlag == 'y') || (DictStoreOIDFlag == 'Y')) + if ((DictStoreOIDFlag == 'y') || (DictStoreOIDFlag == 'Y')) + { + unsigned int hwmCount = 0; + vector segNums; + vector hwms; + + while (1) { - unsigned int hwmCount = 0; - vector segNums; - vector hwms; + cout << "Enter DBRoot#, part#, and the number of HWMs to be entered " + "(separated by spaces): "; + cin >> dbRoot >> partNum >> hwmCount; - while (1) - { - cout << "Enter DBRoot#, part#, and the number of HWMs to be entered " - "(separated by spaces): "; - cin >> dbRoot >> partNum >> hwmCount; - - if (isInputValid()) - break; - } - - for (unsigned int k = 0; k < hwmCount; k++) - { - while (1) - { - cout << "Enter seg# and HWM for that segment file " << - "(separated by spaces): "; - cin >> segNum >> hwm; - - if (isInputValid()) - break; - } - - hwms.push_back(hwm); - segNums.push_back(segNum); - } - - CHECK(emp->rollbackDictStoreExtents_DBroot( oid, dbRoot, partNum, - segNums, hwms )); - } - else - { - while (1) - { - cout << "Enter DBRoot#, part#, seg#, and HWM for the last extent " - "on that DBRoot (separated by spaces): "; - cin >> dbRoot >> partNum >> segNum >> hwm; - - if (isInputValid()) - break; - } - - CHECK(emp->rollbackColumnExtents_DBroot( oid, false, - dbRoot, partNum, segNum, hwm )); + if (isInputValid()) + break; } - return 0; + for (unsigned int k = 0; k < hwmCount; k++) + { + while (1) + { + cout << "Enter seg# and HWM for that segment file " + << "(separated by spaces): "; + cin >> segNum >> hwm; + + if (isInputValid()) + break; + } + + hwms.push_back(hwm); + segNums.push_back(segNum); + } + + CHECK(emp->rollbackDictStoreExtents_DBroot(oid, dbRoot, partNum, segNums, hwms)); + } + else + { + while (1) + { + cout << "Enter DBRoot#, part#, seg#, and HWM for the last extent " + "on that DBRoot (separated by spaces): "; + cin >> dbRoot >> partNum >> segNum >> hwm; + + if (isInputValid()) + break; + } + + CHECK(emp->rollbackColumnExtents_DBroot(oid, false, dbRoot, partNum, segNum, hwm)); + } + + return 0; } //------------------------------------------------------------------------------ @@ -745,26 +714,26 @@ int rollbackExtents(OID_t oid) //------------------------------------------------------------------------------ int deleteOid(OID_t oid) { - if (!fflg) - { - cout << "WARNING! This operation cannot be undone. Enter 'yes' to continue: "; - string resp; - cin >> resp; + if (!fflg) + { + cout << "WARNING! This operation cannot be undone. Enter 'yes' to continue: "; + string resp; + cin >> resp; - if (resp.empty()) - return 1; + if (resp.empty()) + return 1; - string::const_iterator p = resp.begin(); + string::const_iterator p = resp.begin(); - if (*p != 'y' && *p != 'Y') - return 1; - } + if (*p != 'y' && *p != 'Y') + return 1; + } - cout << "Deleting extent map info for " << oid << endl; + cout << "Deleting extent map info for " << oid << endl; - CHECK(emp->deleteOID(oid)); + CHECK(emp->deleteOID(oid)); - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -772,31 +741,30 @@ int deleteOid(OID_t oid) //------------------------------------------------------------------------------ int editHWM(OID_t oid) { - HWM_t oldHWM; - HWM_t newHWM; - uint32_t partNum = 0; - uint16_t segNum = 0; + HWM_t oldHWM; + HWM_t newHWM; + uint32_t partNum = 0; + uint16_t segNum = 0; - while (1) - { - cout << "Enter Partition#, and Segment# (separated by spaces): "; - cin >> partNum >> segNum; + while (1) + { + cout << "Enter Partition#, and Segment# (separated by spaces): "; + cin >> partNum >> segNum; - if (isInputValid()) - break; - } + if (isInputValid()) + break; + } - int extState; - CHECK(emp->getLocalHWM(oid, partNum, segNum, oldHWM, extState)); + int extState; + CHECK(emp->getLocalHWM(oid, partNum, segNum, oldHWM, extState)); - cout << "HWM for partition " << partNum << " and segment " << segNum << - " is currently " << oldHWM << - ". Enter new value: "; - cin >> newHWM; + cout << "HWM for partition " << partNum << " and segment " << segNum << " is currently " << oldHWM + << ". Enter new value: "; + cin >> newHWM; - CHECK(emp->setLocalHWM(oid, partNum, segNum, newHWM)); + CHECK(emp->setLocalHWM(oid, partNum, segNum, newHWM)); - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -804,20 +772,20 @@ int editHWM(OID_t oid) //------------------------------------------------------------------------------ int dumpFL() { - vector v = emp->getEMFreeListEntries(); + vector v = emp->getEMFreeListEntries(); - vector::iterator iter = v.begin(); - vector::iterator end = v.end(); + vector::iterator iter = v.begin(); + vector::iterator end = v.end(); - while (iter != end) - { - if (iter->size || vflg) - cout << iter->start << '\t' << iter->size << endl; + while (iter != end) + { + if (iter->size || vflg) + cout << iter->start << '\t' << iter->size << endl; - ++iter; - } + ++iter; + } - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -825,21 +793,18 @@ int dumpFL() //------------------------------------------------------------------------------ int dumpLBID(LBID_t lbid) { - uint16_t ver = 0; - BRM::OID_t oid; - uint16_t dbroot; - uint32_t partNum; - uint16_t segNum; - uint32_t fbo; - int rc; - rc = emp->lookupLocal(lbid, ver, false, oid, dbroot, partNum, segNum, fbo); - idbassert(rc == 0); - cout << "LBID " << lbid << " is part of OID " << oid << - "; DbRoot " << dbroot << - "; partition# " << partNum << - "; segment# " << segNum << - "; at FBO " << fbo << endl; - return 0; + uint16_t ver = 0; + BRM::OID_t oid; + uint16_t dbroot; + uint32_t partNum; + uint16_t segNum; + uint32_t fbo; + int rc; + rc = emp->lookupLocal(lbid, ver, false, oid, dbroot, partNum, segNum, fbo); + idbassert(rc == 0); + cout << "LBID " << lbid << " is part of OID " << oid << "; DbRoot " << dbroot << "; partition# " << partNum + << "; segment# " << segNum << "; at FBO " << fbo << endl; + return 0; } //------------------------------------------------------------------------------ @@ -847,256 +812,232 @@ int dumpLBID(LBID_t lbid) //------------------------------------------------------------------------------ int deleteAllOnDBRoot(uint16_t dbroot) { - int rc; - rc = emp->deleteDBRoot(dbroot); - idbassert(rc == 0); - return 0; + int rc; + rc = emp->deleteDBRoot(dbroot); + idbassert(rc == 0); + return 0; } -} +} // namespace //------------------------------------------------------------------------------ // main entry point into this program //------------------------------------------------------------------------------ int main(int argc, char** argv) { - int c; - pname = argv[0]; - bool dflg = false; - int zflg = 0; - bool cflg = false; - bool Cflg = false; - OID_t oid = 0; - bool oflg = false; - bool xflg = false; - bool eflg = false; - bool rflg = false; - bool wflg = false; - bool lflg = false; - bool bflg = false; - bool iflg = false; - LBID_t lbid = 0; - bool pflg = false; - uint16_t dbroot = 0; - unsigned int sortOrder = 0; // value of 0 means no sorting + int c; + pname = argv[0]; + bool dflg = false; + int zflg = 0; + bool cflg = false; + bool Cflg = false; + OID_t oid = 0; + bool oflg = false; + bool xflg = false; + bool eflg = false; + bool rflg = false; + bool wflg = false; + bool lflg = false; + bool bflg = false; + bool iflg = false; + LBID_t lbid = 0; + bool pflg = false; + uint16_t dbroot = 0; + unsigned int sortOrder = 0; // value of 0 means no sorting - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "dzCc:o:tsxue:r:fvhw:lb:aimp:S:")) != EOF) - switch (c) - { - case 'd': - dflg = true; - break; - - case 'z': - zflg++; - break; - - case 'C': - Cflg = true; - break; - - case 'c': - cflg = true; - oid = (OID_t)strtoul(optarg, 0, 0); - break; - - case 'o': - oflg = true; - oid = (OID_t)strtoul(optarg, 0, 0); - break; - - case 't': - tflg = true; - break; - - case 'u': - uflg = true; - break; - - case 's': - sflg = true; - break; - - case 'x': - xflg = true; - break; - - case 'e': - eflg = true; - oid = (OID_t)strtoul(optarg, 0, 0); - break; - - case 'r': - rflg = true; - oid = (OID_t)strtoul(optarg, 0, 0); - break; - - case 'f': - fflg = true; - break; - - case 'v': - vflg = true; - break; - - case 'w': - wflg = true; - oid = (OID_t)strtoul(optarg, 0, 0); - break; - - case 'l': - lflg = true; - break; - - case 'b': - bflg = true; - lbid = (LBID_t)strtoull(optarg, 0, 0); - break; - - case 'a': - aflg = true; - break; - - case 'i': - iflg = true; - break; - - case 'm': - mflg = true; - break; - - case 'p': - pflg = true; - dbroot = (uint16_t)strtoul(optarg, 0, 0); - break; - - case 'S': - sortOrder = strtoul(optarg, 0, 0); - break; - - case 'h': - case '?': - default: - usage(pname); - return (c == 'h' ? 0 : 1); - break; - } - - (void)Config::makeConfig(); - - //IF this is UM in a multi-node system, there may not (won't) be an OID bitmap file, so move on... - oam::oamModuleInfo_t modInfo; - oam::Oam oam; - string localModuleType("pm"); - - try + while ((c = getopt(argc, argv, "dzCc:o:tsxue:r:fvhw:lb:aimp:S:")) != EOF) + switch (c) { - modInfo = oam.getModuleInfo(); - localModuleType = modInfo.get<1>(); - } - catch (...) - { - } + case 'd': dflg = true; break; - emp = new DBRM(); + case 'z': zflg++; break; - if (!emp->isDBRMReady()) - { - cerr << endl << "Error! The DBRM is currently not responding!" << endl - << "editem can't continue" << endl << endl; - return 1; - } + case 'C': Cflg = true; break; - MaxOID = -1; + case 'c': + cflg = true; + oid = (OID_t)strtoul(optarg, 0, 0); + break; - if (localModuleType != "um") - { - ObjectIDManager oidm; - MaxOID = oidm.size(); - } + case 'o': + oflg = true; + oid = (OID_t)strtoul(optarg, 0, 0); + break; - if (emp->isReadWrite() != ERR_OK) - { - cerr << endl << "Warning! The DBRM is currently in Read-Only mode!" << endl - << "Updates will not propagate!" << endl << endl; - } + case 't': tflg = true; break; - if (iflg) - { - dflg = mflg = true; - ExtentMap em; - cout << em; - return 0; - } + case 'u': uflg = true; break; - if ((int)dflg + (int)cflg + (int)oflg + (int)xflg + (int)eflg + (int)wflg + - (int)lflg + (int)bflg + (int)Cflg + (int)rflg + (int)pflg > 1) - { - cerr << "Only one of d/c/o/x/e/w/l/b/r/C/p can be specified." << endl; + case 's': sflg = true; break; + + case 'x': xflg = true; break; + + case 'e': + eflg = true; + oid = (OID_t)strtoul(optarg, 0, 0); + break; + + case 'r': + rflg = true; + oid = (OID_t)strtoul(optarg, 0, 0); + break; + + case 'f': fflg = true; break; + + case 'v': vflg = true; break; + + case 'w': + wflg = true; + oid = (OID_t)strtoul(optarg, 0, 0); + break; + + case 'l': lflg = true; break; + + case 'b': + bflg = true; + lbid = (LBID_t)strtoull(optarg, 0, 0); + break; + + case 'a': aflg = true; break; + + case 'i': iflg = true; break; + + case 'm': mflg = true; break; + + case 'p': + pflg = true; + dbroot = (uint16_t)strtoul(optarg, 0, 0); + break; + + case 'S': sortOrder = strtoul(optarg, 0, 0); break; + + case 'h': + case '?': + default: usage(pname); - return 1; + return (c == 'h' ? 0 : 1); + break; } - if ((int)sflg + (int)tflg + (int)aflg + (int)uflg > 1) - { - cerr << "Only one of s/t/a/u can be specified." << endl; - usage(pname); - return 1; - } + (void)Config::makeConfig(); - if (MaxOID < 0 && ((int)dflg + (int)Cflg + zflg) > 0) - { - cerr << "Can't use d/C flag on module type " << localModuleType << endl; - usage(pname); - return 1; - } + // IF this is UM in a multi-node system, there may not (won't) be an OID bitmap file, so move on... + oam::oamModuleInfo_t modInfo; + oam::Oam oam; + string localModuleType("pm"); - if (pflg) - return deleteAllOnDBRoot(dbroot); + try + { + modInfo = oam.getModuleInfo(); + localModuleType = modInfo.get<1>(); + } + catch (...) + { + } - if (oflg) - return dumpone(oid, sortOrder); + emp = new DBRM(); - if (dflg) - return dumpall(); + if (!emp->isDBRMReady()) + { + cerr << endl + << "Error! The DBRM is currently not responding!" << endl + << "editem can't continue" << endl + << endl; + return 1; + } - if (zflg >= 2) - return zapit(); - else if (zflg) - { - cerr << "Not enough z's to zap extent map." << endl; - return 1; - } + MaxOID = -1; - if (Cflg) - return clearAllCPData(); + if (localModuleType != "um") + { + ObjectIDManager oidm; + MaxOID = oidm.size(); + } - if (cflg) - return clearmm(oid); - - if (xflg) - return extendOids(); - - if (eflg) - return deleteOid(oid); - - if (wflg) - return editHWM(oid); - - if (rflg) - return rollbackExtents(oid); - - if (lflg) - return dumpFL(); - - if (bflg) - return dumpLBID(lbid); - - usage(pname); + if (emp->isReadWrite() != ERR_OK) + { + cerr << endl + << "Warning! The DBRM is currently in Read-Only mode!" << endl + << "Updates will not propagate!" << endl + << endl; + } + if (iflg) + { + dflg = mflg = true; + ExtentMap em; + cout << em; return 0; + } + + if ((int)dflg + (int)cflg + (int)oflg + (int)xflg + (int)eflg + (int)wflg + (int)lflg + (int)bflg + + (int)Cflg + (int)rflg + (int)pflg > + 1) + { + cerr << "Only one of d/c/o/x/e/w/l/b/r/C/p can be specified." << endl; + usage(pname); + return 1; + } + + if ((int)sflg + (int)tflg + (int)aflg + (int)uflg > 1) + { + cerr << "Only one of s/t/a/u can be specified." << endl; + usage(pname); + return 1; + } + + if (MaxOID < 0 && ((int)dflg + (int)Cflg + zflg) > 0) + { + cerr << "Can't use d/C flag on module type " << localModuleType << endl; + usage(pname); + return 1; + } + + if (pflg) + return deleteAllOnDBRoot(dbroot); + + if (oflg) + return dumpone(oid, sortOrder); + + if (dflg) + return dumpall(); + + if (zflg >= 2) + return zapit(); + else if (zflg) + { + cerr << "Not enough z's to zap extent map." << endl; + return 1; + } + + if (Cflg) + return clearAllCPData(); + + if (cflg) + return clearmm(oid); + + if (xflg) + return extendOids(); + + if (eflg) + return deleteOid(oid); + + if (wflg) + return editHWM(oid); + + if (rflg) + return rollbackExtents(oid); + + if (lflg) + return dumpFL(); + + if (bflg) + return dumpLBID(lbid); + + usage(pname); + + return 0; } // vim:ts=4 sw=4: - diff --git a/tools/fixdate/fixdate.cpp b/tools/fixdate/fixdate.cpp index 6b48c6ae0..73b1fa643 100644 --- a/tools/fixdate/fixdate.cpp +++ b/tools/fixdate/fixdate.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /* -* $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -* -*/ + * $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $ + * + */ #include #include @@ -42,7 +42,6 @@ using namespace dmlpackageprocessor; namespace { - const streamsize blkSz = 8192; void usage() @@ -51,9 +50,9 @@ void usage() u_int64_t fixDate(ByteStream& bs, ostream& out) { - ByteStream fixed; - ByteStream::quadbyte o; - u_int64_t cnt = 0; + ByteStream fixed; + ByteStream::quadbyte o; + u_int64_t cnt = 0; #if 0 DMLPackageProcessor::Date minDate; DMLPackageProcessor::Date maxDate; @@ -68,108 +67,107 @@ u_int64_t fixDate(ByteStream& bs, ostream& out) mxd = *(reinterpret_cast(&maxDate)); mnd = *(reinterpret_cast(&minDate)); #endif - DMLPackageProcessor::Date fixDate; - fixDate.spare = 0; - ByteStream::quadbyte f; + DMLPackageProcessor::Date fixDate; + fixDate.spare = 0; + ByteStream::quadbyte f; - while (bs.length() > 0) + while (bs.length() > 0) + { + bs >> o; + + if (o >= 0xfffffffe) { - bs >> o; + fixed << o; + continue; + } - if (o >= 0xfffffffe) - { - fixed << o; - continue; - } - - f = o & 0xffff; - fixDate.year = f; - o >>= 16; - f = o & 0xf; - fixDate.month = f; - o >>= 4; - f = o & 0x3f; - fixDate.day = f; - //o >>= 6; - o = *(reinterpret_cast(&fixDate)); - fixed << o; + f = o & 0xffff; + fixDate.year = f; + o >>= 16; + f = o & 0xf; + fixDate.month = f; + o >>= 4; + f = o & 0x3f; + fixDate.day = f; + // o >>= 6; + o = *(reinterpret_cast(&fixDate)); + fixed << o; #if 0 cout << DMLPackageProcessor::dateToString(o) << endl; idbassert(o >= mnd && o <= mxd); cnt++; #endif - } + } - out << fixed; - return cnt; + out << fixed; + return cnt; } -} +} // namespace int main(int argc, char* argv[]) { - int c; + int c; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "h")) != EOF) - switch (c) - { - case 'h': - usage(); - return 0; - break; - - default: - usage(); - return 1; - break; - } - - if ((argc - optind) < 1) + while ((c = getopt(argc, argv, "h")) != EOF) + switch (c) { + case 'h': + usage(); + return 0; + break; + + default: usage(); return 1; + break; } - ByteStream bs; + if ((argc - optind) < 1) + { + usage(); + return 1; + } - ifstream ifs(argv[optind + 0]); - ByteStream::byte inbuf[blkSz]; - streampos fLen; - u_int64_t blkNo = 0; + ByteStream bs; - ifs.seekg(0, ios_base::end); - fLen = ifs.tellg(); - ifs.seekg(0, ios_base::beg); + ifstream ifs(argv[optind + 0]); + ByteStream::byte inbuf[blkSz]; + streampos fLen; + u_int64_t blkNo = 0; - idbassert((fLen % blkSz) == 0); - u_int64_t numBlks = fLen / blkSz; - cout << numBlks << " blocks to fix..." << endl; + ifs.seekg(0, ios_base::end); + fLen = ifs.tellg(); + ifs.seekg(0, ios_base::beg); - ofstream ofs("fixdate.cdf"); + idbassert((fLen % blkSz) == 0); + u_int64_t numBlks = fLen / blkSz; + cout << numBlks << " blocks to fix..." << endl; - cout << "pct done: " << setw(3); + ofstream ofs("fixdate.cdf"); - for (;;) - { + cout << "pct done: " << setw(3); - ifs.read(reinterpret_cast(inbuf), blkSz); + for (;;) + { + ifs.read(reinterpret_cast(inbuf), blkSz); - if (ifs.eof()) break; + if (ifs.eof()) + break; - bs.load(inbuf, blkSz); + bs.load(inbuf, blkSz); - fixDate(bs, ofs); - cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks); - //cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl; + fixDate(bs, ofs); + cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks); + // cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl; - blkNo++; - } + blkNo++; + } - cout << "\b\b\b" << setw(3) << 100 << endl; - //cout << setw(3) << 100 << endl; + cout << "\b\b\b" << setw(3) << 100 << endl; + // cout << setw(3) << 100 << endl; - return 0; + return 0; } - diff --git a/tools/getConfig/main.cpp b/tools/getConfig/main.cpp index 0456a8080..62c7721c2 100644 --- a/tools/getConfig/main.cpp +++ b/tools/getConfig/main.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /***************************************************************************** -* $Id: main.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -* -*****************************************************************************/ + * $Id: main.cpp 2101 2013-01-21 14:12:52Z rdempsey $ + * + *****************************************************************************/ #include #include @@ -35,134 +35,124 @@ using namespace config; namespace { - void usage(const string& pname) { - cout << "usage: " << pname << " [-vh] [-c config_file] section param" << endl << - " Displays configuration variable param from section section." << endl << - " -c config_file use config file config_file" << endl << - " -a display all configuration values" << endl << - " -i display all configuration values in .ini-file format (implies -a)" << endl << - " -v display verbose information" << endl << - " -h display this help text" << endl; + cout << "usage: " << pname << " [-vh] [-c config_file] section param" << endl + << " Displays configuration variable param from section section." << endl + << " -c config_file use config file config_file" << endl + << " -a display all configuration values" << endl + << " -i display all configuration values in .ini-file format (implies -a)" << endl + << " -v display verbose information" << endl + << " -h display this help text" << endl; } -} +} // namespace int main(int argc, char** argv) { - int c; - string pname(argv[0]); - bool vflg = false; - string configFile; - bool aflg = false; - bool iflg = false; + int c; + string pname(argv[0]); + bool vflg = false; + string configFile; + bool aflg = false; + bool iflg = false; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "c:vaih")) != EOF) - switch (c) - { - case 'v': - vflg = true; - break; - - case 'c': - configFile = optarg; - break; - - case 'a': - aflg = true; - break; - - case 'i': - iflg = aflg = true; - break; - - case 'h': - case '?': - default: - usage(pname); - return (c == 'h' ? 0 : 1); - break; - } - - if (!aflg && ( (argc - optind) < 2 )) + while ((c = getopt(argc, argv, "c:vaih")) != EOF) + switch (c) { + case 'v': vflg = true; break; + + case 'c': configFile = optarg; break; + + case 'a': aflg = true; break; + + case 'i': iflg = aflg = true; break; + + case 'h': + case '?': + default: usage(pname); - return 1; + return (c == 'h' ? 0 : 1); + break; } - try + if (!aflg && ((argc - optind) < 2)) + { + usage(pname); + return 1; + } + + try + { + openlog("mcsGetConfig", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_USER); + Config* cf; + + if (configFile.length() > 0) + cf = Config::makeConfig(configFile); + else + cf = Config::makeConfig(); + + if (vflg) { - openlog("mcsGetConfig", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_USER); - Config* cf; + cout << "Using config file: " << cf->configFile() << endl; + } - if (configFile.length() > 0) - cf = Config::makeConfig(configFile); - else - cf = Config::makeConfig(); + if (aflg) + { + vector secs; + vector parms; + secs = cf->enumConfig(); + vector::iterator siter; + vector::iterator send; + vector::iterator piter; + vector::iterator pend; + siter = secs.begin(); + send = secs.end(); - if (vflg) + while (siter != send) + { + if (iflg) + cout << '[' << *siter << ']' << endl; + + parms = cf->enumSection(*siter); + piter = parms.begin(); + pend = parms.end(); + + while (piter != pend) { - cout << "Using config file: " << cf->configFile() << endl; + if (iflg) + cout << *piter << " = " << cf->getConfig(*siter, *piter) << endl; + else + cout << *siter << '.' << *piter << " = " << cf->getConfig(*siter, *piter) << endl; + + ++piter; } - if (aflg) - { - vector secs; - vector parms; - secs = cf->enumConfig(); - vector::iterator siter; - vector::iterator send; - vector::iterator piter; - vector::iterator pend; - siter = secs.begin(); - send = secs.end(); + ++siter; - while (siter != send) - { - if (iflg) - cout << '[' << *siter << ']' << endl; + if (iflg) + cout << endl; + } - parms = cf->enumSection(*siter); - piter = parms.begin(); - pend = parms.end(); - - while (piter != pend) - { - if (iflg) - cout << *piter << " = " << cf->getConfig(*siter, *piter) << endl; - else - cout << *siter << '.' << *piter << " = " << - cf->getConfig(*siter, *piter) << endl; - - ++piter; - } - - ++siter; - - if (iflg) - cout << endl; - } - - return 0; - } - - cout << cf->getConfig(argv[optind + 0], argv[optind + 1]) << endl; - closelog(); - } - catch (exception& e) - { - syslog(LOG_ERR, "Exception in mcsGetConfig for %s %s : %s", argv[optind + 0], argv[optind + 1], e.what()); - closelog(); - } - catch (...) - { - syslog(LOG_ERR, "Exception in mcsGetConfig for %s %s : Unknown exception", argv[optind + 0], argv[optind + 1]); - closelog(); + return 0; } - return 0; + cout << cf->getConfig(argv[optind + 0], argv[optind + 1]) << endl; + closelog(); + } + catch (exception& e) + { + syslog(LOG_ERR, "Exception in mcsGetConfig for %s %s : %s", argv[optind + 0], argv[optind + 1], e.what()); + closelog(); + } + catch (...) + { + syslog(LOG_ERR, "Exception in mcsGetConfig for %s %s : Unknown exception", argv[optind + 0], + argv[optind + 1]); + closelog(); + } + + return 0; } - diff --git a/tools/idb_comp/main.cpp b/tools/idb_comp/main.cpp index edff43b77..474591f4e 100644 --- a/tools/idb_comp/main.cpp +++ b/tools/idb_comp/main.cpp @@ -26,248 +26,240 @@ unsigned vflg; const string lzo_rctos(int r) { - switch (r) - { - case LZO_E_INPUT_NOT_CONSUMED: - return "LZO_E_INPUT_NOT_CONSUMED"; + switch (r) + { + case LZO_E_INPUT_NOT_CONSUMED: return "LZO_E_INPUT_NOT_CONSUMED"; - default: - break; - } + default: break; + } - return "Unknown Error!"; + return "Unknown Error!"; } void usage() { - cout << "usage: idb_comp [-e size][-h] file(s)" << endl - << "\t-e size size (in KB) of expansion buffer" << endl - << "\t-h display this help" << endl; + cout << "usage: idb_comp [-e size][-h] file(s)" << endl + << "\t-e size size (in KB) of expansion buffer" << endl + << "\t-h display this help" << endl; } int doit(const string& infile) { - int rc = 0; - int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT); + int rc = 0; + int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT); - if (ifd < 0) - { - cerr << infile << ": open: " << strerror(errno) << endl; - return 1; - } + if (ifd < 0) + { + cerr << infile << ": open: " << strerror(errno) << endl; + return 1; + } - struct stat istatbuf; + struct stat istatbuf; - fstat(ifd, &istatbuf); + fstat(ifd, &istatbuf); - string outname(infile); + string outname(infile); - string::size_type ptr; + string::size_type ptr; - ptr = outname.find_last_of('.'); + ptr = outname.find_last_of('.'); - if (ptr != string::npos) - outname.erase(ptr); + if (ptr != string::npos) + outname.erase(ptr); - ptr = outname.find_last_of('/'); + ptr = outname.find_last_of('/'); - if (ptr != string::npos) - outname.erase(0, ptr + 1); + if (ptr != string::npos) + outname.erase(0, ptr + 1); - outname = "./" + outname + ".cmp"; + outname = "./" + outname + ".cmp"; - int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644); + int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644); - if (ofd < 0) - { - cerr << outname << ": open: " << strerror(errno) << endl; - close(ifd); - return 1; - } + if (ofd < 0) + { + cerr << outname << ": open: " << strerror(errno) << endl; + close(ifd); + return 1; + } - lzo_init(); + lzo_init(); - ssize_t nread = -1; - ssize_t nwritten = -1; - lzo_bytep ibuf; - lzo_bytep tibuf; - lzo_bytep cbuf; - lzo_bytep tcbuf; - lzo_bytep wkmem; - lzo_uint32 ibuf_len = 0; - lzo_uint cbuf_len = 0; + ssize_t nread = -1; + ssize_t nwritten = -1; + lzo_bytep ibuf; + lzo_bytep tibuf; + lzo_bytep cbuf; + lzo_bytep tcbuf; + lzo_bytep wkmem; + lzo_uint32 ibuf_len = 0; + lzo_uint cbuf_len = 0; - ibuf_len = 512 * 1024 * 8; - tibuf = new lzo_byte[ibuf_len + 4095]; + ibuf_len = 512 * 1024 * 8; + tibuf = new lzo_byte[ibuf_len + 4095]; - if ((ptrdiff_t)tibuf & 0xfffULL) - ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096; - else - ibuf = tibuf; + if ((ptrdiff_t)tibuf & 0xfffULL) + ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096; + else + ibuf = tibuf; + cbuf_len = 512 * 1024 * 8; + tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024]; + + if ((ptrdiff_t)tcbuf & 0xfff) + cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096; + else + cbuf = tcbuf; + + wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS]; + + int r = LZO_E_OK; + + const int TOTAL_HDR_LEN = 4096 * 2; + char* hdrbuf = new char[TOTAL_HDR_LEN + 4095]; + memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095); + char* hdrbufp = 0; + + if ((ptrdiff_t)hdrbuf & 0xfff) + hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096; + else + hdrbufp = hdrbuf; + + struct compHdr + { + uint64_t ptrs[512]; + }; + + idbassert(sizeof(compHdr) <= 4096); + + compHdr* hdrptr1 = (compHdr*)hdrbufp; + compHdr* hdrptr = hdrptr1 + 1; // advance to 2nd hdr to store compression ptrs + lseek(ofd, TOTAL_HDR_LEN, SEEK_SET); + + nread = read(ifd, ibuf, ibuf_len); + + int idx = 0; + off_t cmpoff = TOTAL_HDR_LEN; + + while (nread > 0) + { cbuf_len = 512 * 1024 * 8; - tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024]; + memset(cbuf, 0, cbuf_len); + r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem); - if ((ptrdiff_t)tcbuf & 0xfff) - cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096; - else - cbuf = tcbuf; - - wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS]; - - int r = LZO_E_OK; - - const int TOTAL_HDR_LEN = 4096 * 2; - char* hdrbuf = new char[TOTAL_HDR_LEN + 4095]; - memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095); - char* hdrbufp = 0; - - if ((ptrdiff_t)hdrbuf & 0xfff) - hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096; - else - hdrbufp = hdrbuf; - - struct compHdr + if (r != LZO_E_OK) { - uint64_t ptrs[512]; - }; - - idbassert(sizeof(compHdr) <= 4096); - - compHdr* hdrptr1 = (compHdr*)hdrbufp; - compHdr* hdrptr = hdrptr1 + 1; // advance to 2nd hdr to store compression ptrs - lseek(ofd, TOTAL_HDR_LEN, SEEK_SET); - - nread = read(ifd, ibuf, ibuf_len); - - int idx = 0; - off_t cmpoff = TOTAL_HDR_LEN; - - while (nread > 0) - { - cbuf_len = 512 * 1024 * 8; - memset(cbuf, 0, cbuf_len); - r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem); - - if (r != LZO_E_OK) - { - cerr << "compression failed!: " << r << endl; - rc = 1; - goto out; - } - - if (cbuf_len > (unsigned)nread) - { - cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl; - idbassert((cbuf_len - nread) <= exp_buf_size * 1024); - } - - if (cbuf_len & 0xfff) - cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096; - - //cbuf_len = 512 * 1024 * 8; - nwritten = write(ofd, cbuf, cbuf_len); - - if (nwritten < 0 || (unsigned)nwritten != cbuf_len) - { - cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl; - rc = 1; - goto out; - } - - if (vflg > 0) - { - lzo_bytep tbuf; - lzo_uint tbuflen = 4 * 1024 * 1024 + 4; - boost::scoped_array tbuf_sa(new lzo_byte[tbuflen]); - tbuf = tbuf_sa.get(); - cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len; - r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0); - cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl; - } - - hdrptr->ptrs[idx] = cmpoff; - idx++; - cmpoff += cbuf_len; - - nread = read(ifd, ibuf, ibuf_len); + cerr << "compression failed!: " << r << endl; + rc = 1; + goto out; } - if (nread < 0) + if (cbuf_len > (unsigned)nread) { - cerr << infile << ": read: " << strerror(errno) << endl; - rc = 1; - goto out; + cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl; + idbassert((cbuf_len - nread) <= exp_buf_size * 1024); + } + + if (cbuf_len & 0xfff) + cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096; + + // cbuf_len = 512 * 1024 * 8; + nwritten = write(ofd, cbuf, cbuf_len); + + if (nwritten < 0 || (unsigned)nwritten != cbuf_len) + { + cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl; + rc = 1; + goto out; + } + + if (vflg > 0) + { + lzo_bytep tbuf; + lzo_uint tbuflen = 4 * 1024 * 1024 + 4; + boost::scoped_array tbuf_sa(new lzo_byte[tbuflen]); + tbuf = tbuf_sa.get(); + cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len; + r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0); + cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl; } hdrptr->ptrs[idx] = cmpoff; - idbassert(idx <= 64); + idx++; + cmpoff += cbuf_len; - // Fill in meta-data information in first header - hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL; - hdrptr1->ptrs[1] = 1; - hdrptr1->ptrs[2] = 1; + nread = read(ifd, ibuf, ibuf_len); + } - nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0); - idbassert(nwritten == TOTAL_HDR_LEN); + if (nread < 0) + { + cerr << infile << ": read: " << strerror(errno) << endl; + rc = 1; + goto out; + } + + hdrptr->ptrs[idx] = cmpoff; + idbassert(idx <= 64); + + // Fill in meta-data information in first header + hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL; + hdrptr1->ptrs[1] = 1; + hdrptr1->ptrs[2] = 1; + + nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0); + idbassert(nwritten == TOTAL_HDR_LEN); out: - delete [] wkmem; - delete [] tcbuf; - delete [] tibuf; - fsync(ofd); - struct stat ostatbuf; - fstat(ofd, &ostatbuf); - idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]); - delete [] hdrbuf; - cout << infile << ": Input Size: " << istatbuf.st_size - << " Output size: " << ostatbuf.st_size - << " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl; - close(ofd); - close(ifd); + delete[] wkmem; + delete[] tcbuf; + delete[] tibuf; + fsync(ofd); + struct stat ostatbuf; + fstat(ofd, &ostatbuf); + idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]); + delete[] hdrbuf; + cout << infile << ": Input Size: " << istatbuf.st_size << " Output size: " << ostatbuf.st_size + << " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl; + close(ofd); + close(ifd); - return rc; + return rc; } -} +} // namespace int main(int argc, char** argv) { - opterr = 0; - int c; - exp_buf_size = 128; - vflg = 0; + opterr = 0; + int c; + exp_buf_size = 128; + vflg = 0; - while ((c = getopt(argc, argv, "e:vh")) != -1) - switch (c) - { - case 'e': - exp_buf_size = atoi(optarg); - break; - - case 'v': - vflg++; - break; - - case 'h': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if ((argc - optind) < 1) + while ((c = getopt(argc, argv, "e:vh")) != -1) + switch (c) { + case 'e': exp_buf_size = atoi(optarg); break; + + case 'v': vflg++; break; + + case 'h': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } - int rc = 0; + if ((argc - optind) < 1) + { + usage(); + return 1; + } - for (int i = optind; i < argc; i++) - if (doit(argv[i])) - rc = 1; + int rc = 0; - return rc; + for (int i = optind; i < argc; i++) + if (doit(argv[i])) + rc = 1; + + return rc; } - diff --git a/tools/idbmeminfo/idbmeminfo.cpp b/tools/idbmeminfo/idbmeminfo.cpp index 800385f61..03f6c031e 100644 --- a/tools/idbmeminfo/idbmeminfo.cpp +++ b/tools/idbmeminfo/idbmeminfo.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifdef _MSC_VER #define WIN32_LEAN_AND_MEAN #include @@ -33,155 +32,145 @@ using namespace std; namespace { - struct memInfo { - size_t mtotal; - size_t mfree; + size_t mtotal; + size_t mfree; }; memInfo getMemInfo() { - size_t memTot; - size_t memFree; + size_t memTot; + size_t memFree; #if defined(_MSC_VER) - MEMORYSTATUSEX memStat; - memStat.dwLength = sizeof(memStat); + MEMORYSTATUSEX memStat; + memStat.dwLength = sizeof(memStat); - if (GlobalMemoryStatusEx(&memStat) == 0) - { - //FIXME: Assume 2GB? - memTot = 2 * 1024 * 1024; - memFree = 0; - } - else - { - //We now have the total phys mem in bytes - memTot = memStat.ullTotalPhys / 1024; - memFree = memStat.ullAvailPhys / 1024; - } + if (GlobalMemoryStatusEx(&memStat) == 0) + { + // FIXME: Assume 2GB? + memTot = 2 * 1024 * 1024; + memFree = 0; + } + else + { + // We now have the total phys mem in bytes + memTot = memStat.ullTotalPhys / 1024; + memFree = memStat.ullAvailPhys / 1024; + } #elif defined(__FreeBSD__) - string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);} /Free Memory Pages/ {print $NF;}'"); - FILE* cmdPipe; - char input[80]; - cmdPipe = popen(cmd.c_str(), "r"); - input[0] = '\0'; - fgets(input, 80, cmdPipe); - input[79] = '\0'; - memFree = atoi(input); - input[0] = '\0'; - fgets(input, 80, cmdPipe); - input[79] = '\0'; - memTot = atoi(input); - pclose(cmdPipe); + string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);} /Free Memory Pages/ {print $NF;}'"); + FILE* cmdPipe; + char input[80]; + cmdPipe = popen(cmd.c_str(), "r"); + input[0] = '\0'; + fgets(input, 80, cmdPipe); + input[79] = '\0'; + memFree = atoi(input); + input[0] = '\0'; + fgets(input, 80, cmdPipe); + input[79] = '\0'; + memTot = atoi(input); + pclose(cmdPipe); #elif defined(__linux__) - ifstream in("/proc/meminfo"); - string input; - string x; - getline(in, input); - { - stringstream ss; - ss << input; - ss >> x; - ss >> memTot; - } - getline(in, input); - { - stringstream ss; - ss << input; - ss >> x; - ss >> memFree; - } + ifstream in("/proc/meminfo"); + string input; + string x; + getline(in, input); + { + stringstream ss; + ss << input; + ss >> x; + ss >> memTot; + } + getline(in, input); + { + stringstream ss; + ss << input; + ss >> x; + ss >> memFree; + } #else #error O/S not supported! #endif - //memTot is now in KB, convert to MB - memTot /= 1024; - memFree /= 1024; + // memTot is now in KB, convert to MB + memTot /= 1024; + memFree /= 1024; - memInfo mi; - mi.mtotal = memTot; - mi.mfree = memFree; + memInfo mi; + mi.mtotal = memTot; + mi.mfree = memFree; - return mi; + return mi; } size_t memTotal() { - return getMemInfo().mtotal; + return getMemInfo().mtotal; } size_t memFree() { - return getMemInfo().mfree; + return getMemInfo().mfree; } void usage() { - cout << "usage: idbmeminfo [-t|f] [-g|m|h]" << endl - << "\t-t display total system memory (default)" << endl - << "\t-f display available system memory" << endl - << "\t-m display memory in MB (default)" << endl - << "\t-g display memory in GB" << endl - << "\t-h display this help" << endl; + cout << "usage: idbmeminfo [-t|f] [-g|m|h]" << endl + << "\t-t display total system memory (default)" << endl + << "\t-f display available system memory" << endl + << "\t-m display memory in MB (default)" << endl + << "\t-g display memory in GB" << endl + << "\t-h display this help" << endl; } -} +} // namespace int main(int argc, char** argv) { - opterr = 0; - int c; - bool gFlg = false; - bool tFlg = true; + opterr = 0; + int c; + bool gFlg = false; + bool tFlg = true; - while ((c = getopt(argc, argv, "tfgmh")) != EOF) - switch (c) - { - case 'm': - gFlg = false; - break; - - case 'g': - gFlg = true; - break; - - case 't': - tFlg = true; - break; - - case 'f': - tFlg = false; - break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - size_t mi; - - if (tFlg) - mi = memTotal(); - else - mi = memFree(); - - if (gFlg) + while ((c = getopt(argc, argv, "tfgmh")) != EOF) + switch (c) { - if (mi <= 500) - mi = 0; - else - mi = (mi + 1023) / 1024; + case 'm': gFlg = false; break; + + case 'g': gFlg = true; break; + + case 't': tFlg = true; break; + + case 'f': tFlg = false; break; + + case 'h': + case '?': + default: + usage(); + return (c == 'h' ? 0 : 1); + break; } - cout << mi << endl; + size_t mi; - return 0; + if (tFlg) + mi = memTotal(); + else + mi = memFree(); + + if (gFlg) + { + if (mi <= 500) + mi = 0; + else + mi = (mi + 1023) / 1024; + } + + cout << mi << endl; + + return 0; } - diff --git a/tools/passwd/cskeys.cpp b/tools/passwd/cskeys.cpp index 4599e9261..181503605 100644 --- a/tools/passwd/cskeys.cpp +++ b/tools/passwd/cskeys.cpp @@ -25,12 +25,8 @@ using std::string; using ByteVec = std::vector; -struct option options[] = -{ - {"help", no_argument, nullptr, 'h'}, - {"user", required_argument, nullptr, 'u'}, - {nullptr, 0, nullptr, 0 } -}; +struct option options[] = { + {"help", no_argument, nullptr, 'h'}, {"user", required_argument, nullptr, 'u'}, {nullptr, 0, nullptr, 0}}; const string default_user = "mysql"; @@ -38,8 +34,8 @@ ByteVec generate_encryption_key(); void print_usage(const char* executable, const char* default_directory) { - const char msg[] = - R"(usage: %s [-h|--help] [directory] + const char msg[] = + R"(usage: %s [-h|--help] [directory] This utility generates a random AES encryption key and init vector and writes them to disk. The data is written to the file '%s', in the specified directory. The key and init vector are used by the utility 'cspasswd' to @@ -51,75 +47,69 @@ configuration files. -u, --user Designate the owner of the generated file (default: '%s') directory : The directory where to store the file in (default: '%s') )"; - printf(msg, executable, SECRETS_FILENAME, default_user.c_str(), default_directory); + printf(msg, executable, SECRETS_FILENAME, default_user.c_str(), default_directory); } int main(int argc, char** argv) { - const string default_directory = string(MCSDATADIR); - string username = default_user; + const string default_directory = string(MCSDATADIR); + string username = default_user; - int c; - while ((c = getopt_long(argc, argv, "hu:", options, nullptr)) != -1) + int c; + while ((c = getopt_long(argc, argv, "hu:", options, nullptr)) != -1) + { + switch (c) { - switch (c) - { - case 'h': - print_usage(argv[0], default_directory.c_str()); - return EXIT_SUCCESS; + case 'h': print_usage(argv[0], default_directory.c_str()); return EXIT_SUCCESS; - case 'u': - username = optarg; - break; + case 'u': username = optarg; break; - default: - print_usage(argv[0], default_directory.c_str()); - return EXIT_FAILURE; - } + default: print_usage(argv[0], default_directory.c_str()); return EXIT_FAILURE; } + } - string filepath = default_directory; - if (optind < argc) - { - filepath = argv[optind]; - } - filepath.append("/").append(SECRETS_FILENAME); + string filepath = default_directory; + if (optind < argc) + { + filepath = argv[optind]; + } + filepath.append("/").append(SECRETS_FILENAME); - // Check that the file doesn't exist. - errno = 0; - auto filepathc = filepath.c_str(); - if (access(filepathc, F_OK) == 0) - { - printf("Secrets file '%s' already exists. Delete it before generating a new encryption key.\n", - filepathc); - return EXIT_FAILURE; - } - else if (errno != ENOENT) - { - printf("stat() for secrets file '%s' failed unexpectedly. Error %i, %s.\n", - filepathc, errno, strerror(errno)); - return EXIT_FAILURE; - } + // Check that the file doesn't exist. + errno = 0; + auto filepathc = filepath.c_str(); + if (access(filepathc, F_OK) == 0) + { + printf("Secrets file '%s' already exists. Delete it before generating a new encryption key.\n", + filepathc); + return EXIT_FAILURE; + } + else if (errno != ENOENT) + { + printf("stat() for secrets file '%s' failed unexpectedly. Error %i, %s.\n", filepathc, errno, + strerror(errno)); + return EXIT_FAILURE; + } - int rval = EXIT_FAILURE; - auto new_key = generate_encryption_key(); - if (!new_key.empty() && secrets_write_keys(new_key, filepath, username)) - { - rval = EXIT_SUCCESS; - } - return rval; + int rval = EXIT_FAILURE; + auto new_key = generate_encryption_key(); + if (!new_key.empty() && secrets_write_keys(new_key, filepath, username)) + { + rval = EXIT_SUCCESS; + } + return rval; } ByteVec generate_encryption_key() { - int keylen = EVP_CIPHER_key_length(secrets_cipher()); - ByteVec key(keylen); - // Generate random bytes using OpenSSL. - if (RAND_bytes(key.data(), keylen) != 1) - { - auto errornum = ERR_get_error(); - printf("OpenSSL RAND_bytes() failed. %s.\n", ERR_error_string(errornum, nullptr)); - key.clear(); - } - return key; + int keylen = EVP_CIPHER_key_length(secrets_cipher()); + ByteVec key(keylen); + // Generate random bytes using OpenSSL. + if (RAND_bytes(key.data(), keylen) != 1) + { + auto errornum = ERR_get_error(); + printf("OpenSSL RAND_bytes() failed. %s.\n", ERR_error_string(errornum, nullptr)); + key.clear(); + } + return key; } diff --git a/tools/passwd/cspasswd.cpp b/tools/passwd/cspasswd.cpp index deadd5a4e..7621b079b 100644 --- a/tools/passwd/cspasswd.cpp +++ b/tools/passwd/cspasswd.cpp @@ -32,18 +32,15 @@ using std::endl; using std::flush; using std::string; -struct option options[] = -{ - {"help", no_argument, nullptr, 'h'}, - {"decrypt", no_argument, nullptr, 'd'}, - {"interactive", no_argument, nullptr, 'i'}, - {nullptr, 0, nullptr, 0 } -}; +struct option options[] = {{"help", no_argument, nullptr, 'h'}, + {"decrypt", no_argument, nullptr, 'd'}, + {"interactive", no_argument, nullptr, 'i'}, + {nullptr, 0, nullptr, 0}}; void print_usage(const char* executable, const char* directory) { - const char msg[] = - R"(Usage: %s [-h|--help] [-i|--interactive] [-d|--decrypt] [path] password + const char msg[] = + R"(Usage: %s [-h|--help] [-i|--interactive] [-d|--decrypt] [path] password Encrypt a Columnstore plaintext password using the encryption key in the key file '%s'. The key file may be generated using the 'cskeys'-utility. -h, --help Display this help. @@ -58,211 +55,200 @@ Encrypt a Columnstore plaintext password using the encryption key in the key fil password The password to encrypt or decrypt )"; - printf(msg, executable, SECRETS_FILENAME, directory); + printf(msg, executable, SECRETS_FILENAME, directory); } bool read_password(string* pPassword) { - bool rv = false; - string password; + bool rv = false; + string password; - if (isatty(STDIN_FILENO)) + if (isatty(STDIN_FILENO)) + { + struct termios tty; + tcgetattr(STDIN_FILENO, &tty); + + bool echo = (tty.c_lflag & ECHO); + if (echo) { - struct termios tty; - tcgetattr(STDIN_FILENO, &tty); + tty.c_lflag &= ~ECHO; + tcsetattr(STDIN_FILENO, TCSANOW, &tty); + } - bool echo = (tty.c_lflag & ECHO); - if (echo) - { - tty.c_lflag &= ~ECHO; - tcsetattr(STDIN_FILENO, TCSANOW, &tty); - } + cout << "Enter password : " << flush; + string s1; + std::getline(std::cin, s1); + cout << endl; - cout << "Enter password : " << flush; - string s1; - std::getline(std::cin, s1); - cout << endl; + cout << "Repeat password: " << flush; + string s2; + std::getline(std::cin, s2); + cout << endl; - cout << "Repeat password: " << flush; - string s2; - std::getline(std::cin, s2); - cout << endl; + if (echo) + { + tty.c_lflag |= ECHO; + tcsetattr(STDIN_FILENO, TCSANOW, &tty); + } - if (echo) - { - tty.c_lflag |= ECHO; - tcsetattr(STDIN_FILENO, TCSANOW, &tty); - } - - if (s1 == s2) - { - password = s1; - rv = true; - } - else - { - cout << "Passwords are not identical." << endl; - } + if (s1 == s2) + { + password = s1; + rv = true; } else { - std::getline(std::cin, password); - rv = true; + cout << "Passwords are not identical." << endl; } + } + else + { + std::getline(std::cin, password); + rv = true; + } - if (rv) - { - *pPassword = password; - } + if (rv) + { + *pPassword = password; + } - return rv; + return rv; } - int main(int argc, char** argv) { - std::ios::sync_with_stdio(); - const string default_directory = string(MCSDATADIR); + std::ios::sync_with_stdio(); + const string default_directory = string(MCSDATADIR); - enum class Mode + enum class Mode + { + ENCRYPT, + DECRYPT + }; + + auto mode = Mode::ENCRYPT; + bool interactive = false; + + int c; + while ((c = getopt_long(argc, argv, "hdi", options, NULL)) != -1) + { + switch (c) { - ENCRYPT, - DECRYPT - }; + case 'h': print_usage(argv[0], default_directory.c_str()); return EXIT_SUCCESS; - auto mode = Mode::ENCRYPT; - bool interactive = false; + case 'd': mode = Mode::DECRYPT; break; - int c; - while ((c = getopt_long(argc, argv, "hdi", options, NULL)) != -1) - { - switch (c) - { - case 'h': - print_usage(argv[0], default_directory.c_str()); - return EXIT_SUCCESS; + case 'i': interactive = true; break; - case 'd': - mode = Mode::DECRYPT; - break; - - case 'i': - interactive = true; - break; - - default: - print_usage(argv[0], default_directory.c_str()); - return EXIT_FAILURE; - } + default: print_usage(argv[0], default_directory.c_str()); return EXIT_FAILURE; } + } - string input; - string path = default_directory; + string input; + string path = default_directory; - switch (argc - optind) - { + switch (argc - optind) + { case 2: - // Two args provided. - path = argv[optind]; - if (!interactive) - { - input = argv[optind + 1]; - } - else - { - print_usage(argv[0], default_directory.c_str()); - return EXIT_FAILURE; - } - break; - - case 1: - // One arg provided. - if (!interactive) - { - input = argv[optind]; - } - else - { - path = argv[optind]; - } - break; - - case 0: - if (!interactive) - { - print_usage(argv[0], default_directory.c_str()); - return EXIT_FAILURE; - } - break; - - default: + // Two args provided. + path = argv[optind]; + if (!interactive) + { + input = argv[optind + 1]; + } + else + { print_usage(argv[0], default_directory.c_str()); return EXIT_FAILURE; - } + } + break; - if (interactive) + case 1: + // One arg provided. + if (!interactive) + { + input = argv[optind]; + } + else + { + path = argv[optind]; + } + break; + + case 0: + if (!interactive) + { + print_usage(argv[0], default_directory.c_str()); + return EXIT_FAILURE; + } + break; + + default: print_usage(argv[0], default_directory.c_str()); return EXIT_FAILURE; + } + + if (interactive) + { + if (!read_password(&input)) { - if (!read_password(&input)) - { - return EXIT_FAILURE; - } + return EXIT_FAILURE; } + } - int rval = EXIT_FAILURE; + int rval = EXIT_FAILURE; - string filepath = path; - filepath.append("/").append(SECRETS_FILENAME); + string filepath = path; + filepath.append("/").append(SECRETS_FILENAME); - auto keydata = secrets_readkeys(filepath); - if (keydata.ok) + auto keydata = secrets_readkeys(filepath); + if (keydata.ok) + { + bool encrypting = (mode == Mode::ENCRYPT); + bool new_mode = keydata.iv.empty(); // false -> constant IV from file + if (keydata.key.empty()) { - bool encrypting = (mode == Mode::ENCRYPT); - bool new_mode = keydata.iv.empty(); // false -> constant IV from file - if (keydata.key.empty()) - { - printf("Password encryption key file '%s' not found, cannot %s password.\n", - filepath.c_str(), encrypting ? "encrypt" : "decrypt"); - } - else if (encrypting) - { - string encrypted = new_mode ? encrypt_password(keydata.key, input) : - encrypt_password_old(keydata.key, keydata.iv, input); - if (!encrypted.empty()) - { - printf("%s\n", encrypted.c_str()); - rval = EXIT_SUCCESS; - } - else - { - printf("Password encryption failed.\n"); - } - } - else - { - auto is_hex = std::all_of(input.begin(), input.end(), isxdigit); - if (is_hex && input.length() % 2 == 0) - { - string decrypted = new_mode ? decrypt_password(keydata.key, input) : - decrypt_password_old(keydata.key, keydata.iv, input); - if (!decrypted.empty()) - { - printf("%s\n", decrypted.c_str()); - rval = EXIT_SUCCESS; - } - else - { - printf("Password decryption failed.\n"); - } - } - else - { - printf("Input is not a valid hex-encoded encrypted password.\n"); - } - } + printf("Password encryption key file '%s' not found, cannot %s password.\n", filepath.c_str(), + encrypting ? "encrypt" : "decrypt"); + } + else if (encrypting) + { + string encrypted = new_mode ? encrypt_password(keydata.key, input) + : encrypt_password_old(keydata.key, keydata.iv, input); + if (!encrypted.empty()) + { + printf("%s\n", encrypted.c_str()); + rval = EXIT_SUCCESS; + } + else + { + printf("Password encryption failed.\n"); + } } else { - printf("Could not read encryption key file '%s'.\n", filepath.c_str()); + auto is_hex = std::all_of(input.begin(), input.end(), isxdigit); + if (is_hex && input.length() % 2 == 0) + { + string decrypted = new_mode ? decrypt_password(keydata.key, input) + : decrypt_password_old(keydata.key, keydata.iv, input); + if (!decrypted.empty()) + { + printf("%s\n", decrypted.c_str()); + rval = EXIT_SUCCESS; + } + else + { + printf("Password decryption failed.\n"); + } + } + else + { + printf("Input is not a valid hex-encoded encrypted password.\n"); + } } - return rval; + } + else + { + printf("Could not read encryption key file '%s'.\n", filepath.c_str()); + } + return rval; } diff --git a/tools/passwd/secrets.cpp b/tools/passwd/secrets.cpp index a915d7825..a7635e880 100644 --- a/tools/passwd/secrets.cpp +++ b/tools/passwd/secrets.cpp @@ -36,57 +36,55 @@ using std::string; const char* const SECRETS_FILENAME = ".secrets"; - namespace { -CSPasswdLogging *passwdLog = NULL; +CSPasswdLogging* passwdLog = NULL; boost::mutex m; -} +} // namespace CSPasswdLogging::CSPasswdLogging() { - //TODO: make this configurable - setlogmask (LOG_UPTO (LOG_DEBUG)); - openlog ("CSPasswd", LOG_PID | LOG_NDELAY | LOG_PERROR | LOG_CONS, LOG_LOCAL1); + // TODO: make this configurable + setlogmask(LOG_UPTO(LOG_DEBUG)); + openlog("CSPasswd", LOG_PID | LOG_NDELAY | LOG_PERROR | LOG_CONS, LOG_LOCAL1); } CSPasswdLogging::~CSPasswdLogging() { - syslog(LOG_INFO, "CloseLog"); - closelog(); + syslog(LOG_INFO, "CloseLog"); + closelog(); } -CSPasswdLogging * CSPasswdLogging::get() +CSPasswdLogging* CSPasswdLogging::get() { - if (passwdLog) - return passwdLog; - boost::mutex::scoped_lock s(m); - if (passwdLog) - return passwdLog; - passwdLog = new CSPasswdLogging(); + if (passwdLog) return passwdLog; + boost::mutex::scoped_lock s(m); + if (passwdLog) + return passwdLog; + passwdLog = new CSPasswdLogging(); + return passwdLog; } -void CSPasswdLogging::log(int priority,const char *format, ...) +void CSPasswdLogging::log(int priority, const char* format, ...) { - va_list args; - va_start(args, format); + va_list args; + va_start(args, format); - #ifdef DEBUG - va_list args2; - va_copy(args2, args); - vfprintf(stderr, format, args2); - fprintf(stderr, "\n"); - va_end(args2); - #endif - vsyslog(priority, format, args); +#ifdef DEBUG + va_list args2; + va_copy(args2, args); + vfprintf(stderr, format, args2); + fprintf(stderr, "\n"); + va_end(args2); +#endif + vsyslog(priority, format, args); - va_end(args); + va_end(args); } namespace { - using HexLookupTable = std::array; HexLookupTable init_hex_lookup_table() noexcept; @@ -98,83 +96,84 @@ const char hex_upper[] = "0123456789ABCDEF"; HexLookupTable init_hex_lookup_table() noexcept { - auto char_val = [](char c) -> uint8_t { - if (c >= '0' && c <= '9') - { - return c - '0'; - } - else if (c >= 'A' && c <= 'F') - { - return c - 'A' + 10; - } - else if (c >= 'a' && c <= 'f') - { - return c - 'a' + 10; - } - else - { - return '\177'; - } - }; - - HexLookupTable rval; - for (size_t i = 0; i < rval.size(); i++) + auto char_val = [](char c) -> uint8_t + { + if (c >= '0' && c <= '9') { - rval[i] = char_val(i); + return c - '0'; } - return rval; + else if (c >= 'A' && c <= 'F') + { + return c - 'A' + 10; + } + else if (c >= 'a' && c <= 'f') + { + return c - 'a' + 10; + } + else + { + return '\177'; + } + }; + + HexLookupTable rval; + for (size_t i = 0; i < rval.size(); i++) + { + rval[i] = char_val(i); + } + return rval; } bool hex2bin(const char* in, unsigned int in_len, uint8_t* out) { - // Input length must be multiple of two. - if (!in || in_len == 0 || in_len % 2 != 0) - { - return false; - } + // Input length must be multiple of two. + if (!in || in_len == 0 || in_len % 2 != 0) + { + return false; + } - const char* in_end = in + in_len; - while (in < in_end) - { - // One byte is formed from two hex chars, with the first char forming the high bits. - uint8_t high_half = hex_lookup_table[*in++]; - uint8_t low_half = hex_lookup_table[*in++]; - uint8_t total = (high_half << 4) | low_half; - *out++ = total; - } - return true; + const char* in_end = in + in_len; + while (in < in_end) + { + // One byte is formed from two hex chars, with the first char forming the high bits. + uint8_t high_half = hex_lookup_table[*in++]; + uint8_t low_half = hex_lookup_table[*in++]; + uint8_t total = (high_half << 4) | low_half; + *out++ = total; + } + return true; } char* bin2hex(const uint8_t* in, unsigned int len, char* out) { - const uint8_t* in_end = in + len; - if (len == 0 || in == NULL) - { - return NULL; - } + const uint8_t* in_end = in + len; + if (len == 0 || in == NULL) + { + return NULL; + } - for (; in != in_end; ++in) - { - *out++ = hex_upper[((uint8_t) * in) >> 4]; - *out++ = hex_upper[((uint8_t) * in) & 0x0F]; - } - *out = '\0'; + for (; in != in_end; ++in) + { + *out++ = hex_upper[((uint8_t)*in) >> 4]; + *out++ = hex_upper[((uint8_t)*in) & 0x0F]; + } + *out = '\0'; - return out; + return out; } struct ThisUnit { - ByteVec key; /**< Password decryption key, assigned at startup */ - ByteVec iv; /**< Decryption init vector, assigned at startup. Only used with old-format keys */ + ByteVec key; /**< Password decryption key, assigned at startup */ + ByteVec iv; /**< Decryption init vector, assigned at startup. Only used with old-format keys */ }; ThisUnit this_unit; enum class ProcessingMode { - ENCRYPT, - DECRYPT, - DECRYPT_IGNORE_ERRORS + ENCRYPT, + DECRYPT, + DECRYPT_IGNORE_ERRORS }; const char field_desc[] = "description"; @@ -184,7 +183,7 @@ const char field_key[] = "encryption_key"; const char desc[] = "Columnstore encryption/decryption key"; #define SECRETS_CIPHER EVP_aes_256_cbc -#define STRINGIFY(X) #X +#define STRINGIFY(X) #X #define STRINGIFY2(X) STRINGIFY(X) const char CIPHER_NAME[] = STRINGIFY2(SECRETS_CIPHER); @@ -201,84 +200,84 @@ void print_openSSL_errors(const char* operation); * @param output_len Produced output length is written here * @return True on success */ -bool encrypt_or_decrypt(const uint8_t* key, const uint8_t* iv, ProcessingMode mode, - const uint8_t* input, int input_len, uint8_t* output, int* output_len) +bool encrypt_or_decrypt(const uint8_t* key, const uint8_t* iv, ProcessingMode mode, const uint8_t* input, + int input_len, uint8_t* output, int* output_len) { - auto ctx = EVP_CIPHER_CTX_new(); - int enc = (mode == ProcessingMode::ENCRYPT) ? AES_ENCRYPT : AES_DECRYPT; - bool ignore_errors = (mode == ProcessingMode::DECRYPT_IGNORE_ERRORS); - bool ok = false; + auto ctx = EVP_CIPHER_CTX_new(); + int enc = (mode == ProcessingMode::ENCRYPT) ? AES_ENCRYPT : AES_DECRYPT; + bool ignore_errors = (mode == ProcessingMode::DECRYPT_IGNORE_ERRORS); + bool ok = false; - if (EVP_CipherInit_ex(ctx, secrets_cipher(), nullptr, key, iv, enc) == 1 || ignore_errors) + if (EVP_CipherInit_ex(ctx, secrets_cipher(), nullptr, key, iv, enc) == 1 || ignore_errors) + { + int output_written = 0; + if (EVP_CipherUpdate(ctx, output, &output_written, input, input_len) == 1 || ignore_errors) { - int output_written = 0; - if (EVP_CipherUpdate(ctx, output, &output_written, input, input_len) == 1 || ignore_errors) - { - int total_output_len = output_written; - if (EVP_CipherFinal_ex(ctx, output + total_output_len, &output_written) == 1 || ignore_errors) - { - total_output_len += output_written; - *output_len = total_output_len; - ok = true; - } - } + int total_output_len = output_written; + if (EVP_CipherFinal_ex(ctx, output + total_output_len, &output_written) == 1 || ignore_errors) + { + total_output_len += output_written; + *output_len = total_output_len; + ok = true; + } } + } - EVP_CIPHER_CTX_free(ctx); - if (!ok) - { - const char* operation = (mode == ProcessingMode::ENCRYPT) ? "when encrypting password" : - "when decrypting password"; - print_openSSL_errors(operation); - } - return ok; + EVP_CIPHER_CTX_free(ctx); + if (!ok) + { + const char* operation = + (mode == ProcessingMode::ENCRYPT) ? "when encrypting password" : "when decrypting password"; + print_openSSL_errors(operation); + } + return ok; } void print_openSSL_errors(const char* operation) { - // It's unclear how thread(unsafe) OpenSSL error functions are. Minimize such possibilities by - // using a local buffer. - constexpr size_t bufsize = 256; // Should be enough according to some googling. - char buf[bufsize]; - buf[0] = '\0'; + // It's unclear how thread(unsafe) OpenSSL error functions are. Minimize such possibilities by + // using a local buffer. + constexpr size_t bufsize = 256; // Should be enough according to some googling. + char buf[bufsize]; + buf[0] = '\0'; - auto errornum = ERR_get_error(); - auto errornum2 = ERR_get_error(); - ERR_error_string_n(errornum, buf, bufsize); + auto errornum = ERR_get_error(); + auto errornum2 = ERR_get_error(); + ERR_error_string_n(errornum, buf, bufsize); - if (errornum2 == 0) + if (errornum2 == 0) + { + // One error. + CSPasswdLogging::get()->log(LOG_ERR, "OpenSSL error %s. %s", operation, buf); + } + else + { + // Multiple errors, print all as separate messages. + CSPasswdLogging::get()->log(LOG_ERR, "Multiple OpenSSL errors %s. Detailed messages below.", operation); + CSPasswdLogging::get()->log(LOG_ERR, "%s", buf); + while (errornum2 != 0) { - // One error. - CSPasswdLogging::get()->log(LOG_ERR,"OpenSSL error %s. %s", operation, buf); - } - else - { - // Multiple errors, print all as separate messages. - CSPasswdLogging::get()->log(LOG_ERR,"Multiple OpenSSL errors %s. Detailed messages below.", operation); - CSPasswdLogging::get()->log(LOG_ERR,"%s", buf); - while (errornum2 != 0) - { - ERR_error_string_n(errornum2, buf, bufsize); - CSPasswdLogging::get()->log(LOG_ERR,"%s", buf); - errornum2 = ERR_get_error(); - } + ERR_error_string_n(errornum2, buf, bufsize); + CSPasswdLogging::get()->log(LOG_ERR, "%s", buf); + errornum2 = ERR_get_error(); } + } } -} +} // namespace const EVP_CIPHER* secrets_cipher() { - return SECRETS_CIPHER(); + return SECRETS_CIPHER(); } int secrets_keylen() { - return EVP_CIPHER_key_length(secrets_cipher()); + return EVP_CIPHER_key_length(secrets_cipher()); } int secrets_ivlen() { - return EVP_CIPHER_iv_length(secrets_cipher()); + return EVP_CIPHER_iv_length(secrets_cipher()); } /** @@ -291,164 +290,174 @@ int secrets_ivlen() */ ReadKeyResult secrets_readkeys(const string& filepath) { - ReadKeyResult rval; - auto filepathc = filepath.c_str(); + ReadKeyResult rval; + auto filepathc = filepath.c_str(); - const int binary_key_len = secrets_keylen(); - const int binary_iv_len = secrets_ivlen(); - const int binary_total_len = binary_key_len + binary_iv_len; + const int binary_key_len = secrets_keylen(); + const int binary_iv_len = secrets_ivlen(); + const int binary_total_len = binary_key_len + binary_iv_len; - // Before opening the file, check its size and permissions. - struct stat filestats; - bool stat_error = false; - bool old_format = false; - errno = 0; - if (stat(filepathc, &filestats) == 0) + // Before opening the file, check its size and permissions. + struct stat filestats; + bool stat_error = false; + bool old_format = false; + errno = 0; + if (stat(filepathc, &filestats) == 0) + { + auto filesize = filestats.st_size; + if (filesize == binary_total_len) { - auto filesize = filestats.st_size; - if (filesize == binary_total_len) - { - old_format = true; - CSPasswdLogging::get()->log(LOG_WARNING,"File format of '%s' is deprecated. Please generate a new encryption key ('maxkeys') " - "and re-encrypt passwords ('maxpasswd').", filepathc); - } - - auto filemode = filestats.st_mode; - if (!S_ISREG(filemode)) - { - CSPasswdLogging::get()->log(LOG_ERR,"Secrets file '%s' is not a regular file.", filepathc); - stat_error = true; - } - else if ((filemode & (S_IRWXU | S_IRWXG | S_IRWXO)) != S_IRUSR) - { - CSPasswdLogging::get()->log(LOG_ERR,"Secrets file '%s' permissions are wrong. The only permission on the file should be " - "owner:read.", filepathc); - stat_error = true; - } - } - else if (errno == ENOENT) - { - // The file does not exist. This is ok. Return empty result. - rval.ok = true; - return rval; - } - else - { - CSPasswdLogging::get()->log(LOG_ERR,"stat() for secrets file '%s' failed. Error %d, %s.", - filepathc, errno, strerror(errno)); - stat_error = true; + old_format = true; + CSPasswdLogging::get()->log( + LOG_WARNING, + "File format of '%s' is deprecated. Please generate a new encryption key ('maxkeys') " + "and re-encrypt passwords ('maxpasswd').", + filepathc); } - if (stat_error) + auto filemode = filestats.st_mode; + if (!S_ISREG(filemode)) { - return rval; + CSPasswdLogging::get()->log(LOG_ERR, "Secrets file '%s' is not a regular file.", filepathc); + stat_error = true; } - - if (old_format) + else if ((filemode & (S_IRWXU | S_IRWXG | S_IRWXO)) != S_IRUSR) { - errno = 0; - std::ifstream file(filepath, std::ios_base::binary); - if (file.is_open()) - { - // Read all data from file. - utils::VLArray readbuf(binary_total_len); - //char readbuf[binary_total_len]; - file.read(readbuf.data(), binary_total_len); - if (file.good()) - { - // Success, copy contents to key structure. - rval.key.assign(readbuf.data(), readbuf.data() + binary_key_len); - rval.iv.assign(readbuf.data() + binary_key_len, readbuf.data() + binary_total_len); - rval.ok = true; - } - else - { - CSPasswdLogging::get()->log(LOG_ERR,"Read from secrets file %s failed. Read %li, expected %i bytes. Error %d, %s.", - filepathc, file.gcount(), binary_total_len, errno, strerror(errno)); - } - } - else - { - CSPasswdLogging::get()->log(LOG_ERR,"Could not open secrets file '%s'. Error %d, %s.", - filepathc, errno, strerror(errno)); - } - } - else - { - // File contents should be json. - //json_error_t err; - boost::property_tree::ptree jsontree; - try - { - boost::property_tree::read_json(filepath, jsontree); - } - catch(boost::property_tree::json_parser::json_parser_error &je) - { - std::cout << "Error reading JSON from secrets file: " << je.filename() << " on line: " << je.line() << std::endl; - std::cout << je.message() << std::endl; - } - catch(...) - { - printf("Error reading JSON from secrets file '%s' failed. Error %d, %s.\n", - filepathc, errno, strerror(errno)); - } - //json_t* obj = json_load_file(filepathc, 0, &err); - string enc_cipher = jsontree.get(field_cipher); - string enc_key = jsontree.get(field_key); - //const char* enc_cipher = json_string_value(json_object_get(obj, field_cipher)); - //const char* enc_key = json_string_value(json_object_get(obj, field_key)); - bool cipher_ok = !enc_cipher.empty() && (enc_cipher == CIPHER_NAME); - if (cipher_ok && !enc_key.empty()) - { - int read_hex_key_len = enc_key.length(); - int expected_hex_key_len = 2 * binary_key_len; - if (read_hex_key_len == expected_hex_key_len) - { - rval.key.resize(binary_key_len); - hex2bin(enc_key.c_str(), read_hex_key_len, rval.key.data()); - rval.ok = true; - } - else - { - CSPasswdLogging::get()->log(LOG_ERR,"Wrong encryption key length in secrets file '%s': found %i, expected %i.", - filepathc, read_hex_key_len, expected_hex_key_len); - } - } - else - { - CSPasswdLogging::get()->log(LOG_ERR,"Secrets file '%s' does not contain expected string fields '%s' and '%s', " - "or '%s' is not '%s'.", - filepathc, field_cipher, field_key, field_cipher, CIPHER_NAME); - } - jsontree.clear(); + CSPasswdLogging::get()->log( + LOG_ERR, + "Secrets file '%s' permissions are wrong. The only permission on the file should be " + "owner:read.", + filepathc); + stat_error = true; } + } + else if (errno == ENOENT) + { + // The file does not exist. This is ok. Return empty result. + rval.ok = true; return rval; + } + else + { + CSPasswdLogging::get()->log(LOG_ERR, "stat() for secrets file '%s' failed. Error %d, %s.", filepathc, + errno, strerror(errno)); + stat_error = true; + } + + if (stat_error) + { + return rval; + } + + if (old_format) + { + errno = 0; + std::ifstream file(filepath, std::ios_base::binary); + if (file.is_open()) + { + // Read all data from file. + utils::VLArray readbuf(binary_total_len); + // char readbuf[binary_total_len]; + file.read(readbuf.data(), binary_total_len); + if (file.good()) + { + // Success, copy contents to key structure. + rval.key.assign(readbuf.data(), readbuf.data() + binary_key_len); + rval.iv.assign(readbuf.data() + binary_key_len, readbuf.data() + binary_total_len); + rval.ok = true; + } + else + { + CSPasswdLogging::get()->log( + LOG_ERR, "Read from secrets file %s failed. Read %li, expected %i bytes. Error %d, %s.", + filepathc, file.gcount(), binary_total_len, errno, strerror(errno)); + } + } + else + { + CSPasswdLogging::get()->log(LOG_ERR, "Could not open secrets file '%s'. Error %d, %s.", filepathc, + errno, strerror(errno)); + } + } + else + { + // File contents should be json. + // json_error_t err; + boost::property_tree::ptree jsontree; + try + { + boost::property_tree::read_json(filepath, jsontree); + } + catch (boost::property_tree::json_parser::json_parser_error& je) + { + std::cout << "Error reading JSON from secrets file: " << je.filename() << " on line: " << je.line() + << std::endl; + std::cout << je.message() << std::endl; + } + catch (...) + { + printf("Error reading JSON from secrets file '%s' failed. Error %d, %s.\n", filepathc, errno, + strerror(errno)); + } + // json_t* obj = json_load_file(filepathc, 0, &err); + string enc_cipher = jsontree.get(field_cipher); + string enc_key = jsontree.get(field_key); + // const char* enc_cipher = json_string_value(json_object_get(obj, field_cipher)); + // const char* enc_key = json_string_value(json_object_get(obj, field_key)); + bool cipher_ok = !enc_cipher.empty() && (enc_cipher == CIPHER_NAME); + if (cipher_ok && !enc_key.empty()) + { + int read_hex_key_len = enc_key.length(); + int expected_hex_key_len = 2 * binary_key_len; + if (read_hex_key_len == expected_hex_key_len) + { + rval.key.resize(binary_key_len); + hex2bin(enc_key.c_str(), read_hex_key_len, rval.key.data()); + rval.ok = true; + } + else + { + CSPasswdLogging::get()->log( + LOG_ERR, "Wrong encryption key length in secrets file '%s': found %i, expected %i.", filepathc, + read_hex_key_len, expected_hex_key_len); + } + } + else + { + CSPasswdLogging::get()->log(LOG_ERR, + "Secrets file '%s' does not contain expected string fields '%s' and '%s', " + "or '%s' is not '%s'.", + filepathc, field_cipher, field_key, field_cipher, CIPHER_NAME); + } + jsontree.clear(); + } + return rval; } string decrypt_password(const string& input) { - const auto& key = this_unit.key; - string rval; - if (key.empty()) + const auto& key = this_unit.key; + string rval; + if (key.empty()) + { + // Password encryption is not used, return original. + rval = input; + } + else + { + // If the input is not a HEX string, return the input as is. + auto is_hex = std::all_of(input.begin(), input.end(), isxdigit); + if (is_hex) { - // Password encryption is not used, return original. - rval = input; + const auto& iv = this_unit.iv; + rval = iv.empty() ? ::decrypt_password(key, input) : decrypt_password_old(key, iv, input); } else { - // If the input is not a HEX string, return the input as is. - auto is_hex = std::all_of(input.begin(), input.end(), isxdigit); - if (is_hex) - { - const auto& iv = this_unit.iv; - rval = iv.empty() ? ::decrypt_password(key, input) : decrypt_password_old(key, iv, input); - } - else - { - rval = input; - } + rval = input; } - return rval; + } + return rval; } /** @@ -462,69 +471,70 @@ string decrypt_password(const string& input) */ string decrypt_password_old(const ByteVec& key, const ByteVec& iv, const std::string& input) { - string rval; - // Convert to binary. - size_t hex_len = input.length(); - auto bin_len = hex_len / 2; - utils::VLArray encrypted_bin(bin_len); - hex2bin(input.c_str(), hex_len, encrypted_bin.data()); - utils::VLArray plain(bin_len); - int decrypted_len = 0; - if (encrypt_or_decrypt(key.data(), iv.data(), ProcessingMode::DECRYPT_IGNORE_ERRORS, encrypted_bin.data(), - bin_len, plain.data(), &decrypted_len)) + string rval; + // Convert to binary. + size_t hex_len = input.length(); + auto bin_len = hex_len / 2; + utils::VLArray encrypted_bin(bin_len); + hex2bin(input.c_str(), hex_len, encrypted_bin.data()); + utils::VLArray plain(bin_len); + int decrypted_len = 0; + if (encrypt_or_decrypt(key.data(), iv.data(), ProcessingMode::DECRYPT_IGNORE_ERRORS, encrypted_bin.data(), + bin_len, plain.data(), &decrypted_len)) + { + if (decrypted_len > 0) { - if (decrypted_len > 0) - { - // Success, password was encrypted using 2.5. Decrypted data should be text. - auto output_data = reinterpret_cast(plain.data()); - rval.assign(output_data, decrypted_len); - } - else - { - // Failure, password was likely encrypted in 2.4. Try to decrypt using 2.4 code. - AES_KEY aeskey; - AES_set_decrypt_key(key.data(), 8 * key.size(), &aeskey); - auto iv_copy = iv; - memset(plain.data(), '\0', bin_len); - AES_cbc_encrypt(encrypted_bin.data(), plain.data(), bin_len, &aeskey, iv_copy.data(), AES_DECRYPT); - rval = reinterpret_cast(plain.data()); - } + // Success, password was encrypted using 2.5. Decrypted data should be text. + auto output_data = reinterpret_cast(plain.data()); + rval.assign(output_data, decrypted_len); } - return rval; + else + { + // Failure, password was likely encrypted in 2.4. Try to decrypt using 2.4 code. + AES_KEY aeskey; + AES_set_decrypt_key(key.data(), 8 * key.size(), &aeskey); + auto iv_copy = iv; + memset(plain.data(), '\0', bin_len); + AES_cbc_encrypt(encrypted_bin.data(), plain.data(), bin_len, &aeskey, iv_copy.data(), AES_DECRYPT); + rval = reinterpret_cast(plain.data()); + } + } + return rval; } string decrypt_password(const ByteVec& key, const std::string& input) { - int total_hex_len = input.length(); - string rval; + int total_hex_len = input.length(); + string rval; - // Extract IV. - auto ptr = input.data(); - int iv_bin_len = secrets_ivlen(); - int iv_hex_len = 2 * iv_bin_len; - utils::VLArray iv_bin(iv_bin_len); + // Extract IV. + auto ptr = input.data(); + int iv_bin_len = secrets_ivlen(); + int iv_hex_len = 2 * iv_bin_len; + utils::VLArray iv_bin(iv_bin_len); - if (total_hex_len >= iv_hex_len) + if (total_hex_len >= iv_hex_len) + { + hex2bin(ptr, iv_hex_len, iv_bin.data()); + + int encrypted_hex_len = total_hex_len - iv_hex_len; + int encrypted_bin_len = encrypted_hex_len / 2; + utils::VLArray encrypted_bin(encrypted_bin_len); + hex2bin(ptr + iv_hex_len, encrypted_hex_len, encrypted_bin.data()); + + utils::VLArray decrypted( + encrypted_bin_len); // Decryption output cannot be longer than input data. + int decrypted_len = 0; + if (encrypt_or_decrypt(key.data(), iv_bin.data(), ProcessingMode::DECRYPT, encrypted_bin.data(), + encrypted_bin_len, decrypted.data(), &decrypted_len)) { - hex2bin(ptr, iv_hex_len, iv_bin.data()); - - int encrypted_hex_len = total_hex_len - iv_hex_len; - int encrypted_bin_len = encrypted_hex_len / 2; - utils::VLArray encrypted_bin(encrypted_bin_len); - hex2bin(ptr + iv_hex_len, encrypted_hex_len, encrypted_bin.data()); - - utils::VLArray decrypted(encrypted_bin_len); // Decryption output cannot be longer than input data. - int decrypted_len = 0; - if (encrypt_or_decrypt(key.data(), iv_bin.data(), ProcessingMode::DECRYPT, encrypted_bin.data(), encrypted_bin_len, - decrypted.data(), &decrypted_len)) - { - // Decrypted data should be text. - auto output_data = reinterpret_cast(decrypted.data()); - rval.assign(output_data, decrypted_len); - } + // Decrypted data should be text. + auto output_data = reinterpret_cast(decrypted.data()); + rval.assign(output_data, decrypted_len); } + } - return rval; + return rval; } /** @@ -536,78 +546,79 @@ string decrypt_password(const ByteVec& key, const std::string& input) */ string encrypt_password_old(const ByteVec& key, const ByteVec& iv, const string& input) { - string rval; - // Output can be a block length longer than input. - auto input_len = input.length(); - utils::VLArray encrypted_bin(input_len + AES_BLOCK_SIZE); + string rval; + // Output can be a block length longer than input. + auto input_len = input.length(); + utils::VLArray encrypted_bin(input_len + AES_BLOCK_SIZE); - // Although input is text, interpret as binary. - auto input_data = reinterpret_cast(input.c_str()); - int encrypted_len = 0; - if (encrypt_or_decrypt(key.data(), iv.data(), ProcessingMode::ENCRYPT, - input_data, input_len, encrypted_bin.data(), &encrypted_len)) - { - int hex_len = 2 * encrypted_len; - utils::VLArray hex_output(hex_len + 1); + // Although input is text, interpret as binary. + auto input_data = reinterpret_cast(input.c_str()); + int encrypted_len = 0; + if (encrypt_or_decrypt(key.data(), iv.data(), ProcessingMode::ENCRYPT, input_data, input_len, + encrypted_bin.data(), &encrypted_len)) + { + int hex_len = 2 * encrypted_len; + utils::VLArray hex_output(hex_len + 1); - bin2hex(encrypted_bin.data(), encrypted_len, hex_output.data()); - rval.assign(hex_output.data(), hex_len); - } - return rval; + bin2hex(encrypted_bin.data(), encrypted_len, hex_output.data()); + rval.assign(hex_output.data(), hex_len); + } + return rval; } string encrypt_password(const ByteVec& key, const string& input) { - string rval; - // Generate random IV. - auto ivlen = secrets_ivlen(); - utils::VLArray iv_bin(ivlen); - if (RAND_bytes(iv_bin.data(), ivlen) != 1) - { - printf("OpenSSL RAND_bytes() failed. %s.\n", ERR_error_string(ERR_get_error(), nullptr)); - return rval; - } - - // Output can be a block length longer than input. - auto input_len = input.length(); - utils::VLArray encrypted_bin(input_len + EVP_CIPHER_block_size(secrets_cipher())); - // Although input is text, interpret as binary. - auto input_data = reinterpret_cast(input.c_str()); - int encrypted_len = 0; - if (encrypt_or_decrypt(key.data(), iv_bin.data(), ProcessingMode::ENCRYPT, - input_data, input_len, encrypted_bin.data(), &encrypted_len)) - { - // Form one string with IV in front. - int iv_hex_len = 2 * ivlen; - int encrypted_hex_len = 2 * encrypted_len; - int total_hex_len = iv_hex_len + encrypted_hex_len; - utils::VLArray hex_output(total_hex_len + 1); - bin2hex(iv_bin.data(), ivlen, hex_output.data()); - bin2hex(encrypted_bin.data(), encrypted_len, hex_output.data() + iv_hex_len); - rval.assign(hex_output.data(), total_hex_len); - } + string rval; + // Generate random IV. + auto ivlen = secrets_ivlen(); + utils::VLArray iv_bin(ivlen); + if (RAND_bytes(iv_bin.data(), ivlen) != 1) + { + printf("OpenSSL RAND_bytes() failed. %s.\n", ERR_error_string(ERR_get_error(), nullptr)); return rval; + } + + // Output can be a block length longer than input. + auto input_len = input.length(); + utils::VLArray encrypted_bin(input_len + EVP_CIPHER_block_size(secrets_cipher())); + // Although input is text, interpret as binary. + auto input_data = reinterpret_cast(input.c_str()); + int encrypted_len = 0; + if (encrypt_or_decrypt(key.data(), iv_bin.data(), ProcessingMode::ENCRYPT, input_data, input_len, + encrypted_bin.data(), &encrypted_len)) + { + // Form one string with IV in front. + int iv_hex_len = 2 * ivlen; + int encrypted_hex_len = 2 * encrypted_len; + int total_hex_len = iv_hex_len + encrypted_hex_len; + utils::VLArray hex_output(total_hex_len + 1); + bin2hex(iv_bin.data(), ivlen, hex_output.data()); + bin2hex(encrypted_bin.data(), encrypted_len, hex_output.data() + iv_hex_len); + rval.assign(hex_output.data(), total_hex_len); + } + return rval; } bool load_encryption_keys() { - if(this_unit.key.empty() || this_unit.iv.empty()) + if (this_unit.key.empty() || this_unit.iv.empty()) + { + string path(MCSDATADIR); + path.append("/").append(SECRETS_FILENAME); + auto ret = secrets_readkeys(path); + if (ret.ok) { - string path(MCSDATADIR); - path.append("/").append(SECRETS_FILENAME); - auto ret = secrets_readkeys(path); - if (ret.ok) - { - if (!ret.key.empty()) - { - //CSPasswdLogging::get()->log(LOG_INFO,"Using encrypted passwords. Encryption key read from '%s'.", path.c_str()); - this_unit.key = move(ret.key); - this_unit.iv = move(ret.iv); - } - return ret.ok; - } + if (!ret.key.empty()) + { + // CSPasswdLogging::get()->log(LOG_INFO,"Using encrypted passwords. Encryption key read from '%s'.", + // path.c_str()); + this_unit.key = move(ret.key); + this_unit.iv = move(ret.iv); + } + return ret.ok; } - return true; + } + return true; } /** @@ -620,70 +631,69 @@ bool load_encryption_keys() */ bool secrets_write_keys(const ByteVec& key, const string& filepath, const string& owner) { - auto keylen = key.size(); - utils::VLArray key_hex(2 * keylen + 1); - bin2hex(key.data(), keylen, key_hex.data()); + auto keylen = key.size(); + utils::VLArray key_hex(2 * keylen + 1); + bin2hex(key.data(), keylen, key_hex.data()); - boost::property_tree::ptree jsontree; - jsontree.put(field_desc,desc); - jsontree.put(field_version,columnstore_version.c_str()); - jsontree.put(field_cipher,CIPHER_NAME); - jsontree.put(field_key,(const char*)key_hex.data()); + boost::property_tree::ptree jsontree; + jsontree.put(field_desc, desc); + jsontree.put(field_version, columnstore_version.c_str()); + jsontree.put(field_cipher, CIPHER_NAME); + jsontree.put(field_key, (const char*)key_hex.data()); - auto filepathc = filepath.c_str(); - bool write_ok = false; + auto filepathc = filepath.c_str(); + bool write_ok = false; + errno = 0; + try + { + write_json(filepathc, jsontree); + } + catch (boost::property_tree::json_parser::json_parser_error& je) + { + std::cout << "Write to secrets file: " << je.filename() << " on line: " << je.line() << std::endl; + std::cout << je.message() << std::endl; + } + catch (...) + { + printf("Write to secrets file '%s' failed. Error %d, %s.\n", filepathc, errno, strerror(errno)); + } + write_ok = true; + + jsontree.clear(); + bool rval = false; + if (write_ok) + { + // Change file permissions to prevent modifications. errno = 0; - try + if (chmod(filepathc, S_IRUSR) == 0) { - write_json(filepathc, jsontree); - } - catch(boost::property_tree::json_parser::json_parser_error &je) - { - std::cout << "Write to secrets file: " << je.filename() << " on line: " << je.line() << std::endl; - std::cout << je.message() << std::endl; - } - catch(...) - { - printf("Write to secrets file '%s' failed. Error %d, %s.\n", - filepathc, errno, strerror(errno)); - } - write_ok = true; - - jsontree.clear(); - bool rval = false; - if (write_ok) - { - // Change file permissions to prevent modifications. - errno = 0; - if (chmod(filepathc, S_IRUSR) == 0) + printf("Permissions of '%s' set to owner:read.\n", filepathc); + auto ownerz = owner.c_str(); + auto userinfo = getpwnam(ownerz); + if (userinfo) + { + if (chown(filepathc, userinfo->pw_uid, userinfo->pw_gid) == 0) { - printf("Permissions of '%s' set to owner:read.\n", filepathc); - auto ownerz = owner.c_str(); - auto userinfo = getpwnam(ownerz); - if (userinfo) - { - if (chown(filepathc, userinfo->pw_uid, userinfo->pw_gid) == 0) - { - printf("Ownership of '%s' given to %s.\n", filepathc, ownerz); - rval = true; - } - else - { - printf("Failed to give '%s' ownership of '%s': %d, %s.\n", - ownerz, filepathc, errno, strerror(errno)); - } - } - else - { - printf("Could not find user '%s' when attempting to change ownership of '%s': %d, %s.\n", - ownerz, filepathc, errno, strerror(errno)); - } + printf("Ownership of '%s' given to %s.\n", filepathc, ownerz); + rval = true; } else { - printf("Failed to change the permissions of the secrets file '%s'. Error %d, %s.\n", - filepathc, errno, strerror(errno)); + printf("Failed to give '%s' ownership of '%s': %d, %s.\n", ownerz, filepathc, errno, + strerror(errno)); } + } + else + { + printf("Could not find user '%s' when attempting to change ownership of '%s': %d, %s.\n", ownerz, + filepathc, errno, strerror(errno)); + } } - return rval; + else + { + printf("Failed to change the permissions of the secrets file '%s'. Error %d, %s.\n", filepathc, errno, + strerror(errno)); + } + } + return rval; } diff --git a/tools/passwd/secrets.h b/tools/passwd/secrets.h index 781583ece..107d2e16b 100644 --- a/tools/passwd/secrets.h +++ b/tools/passwd/secrets.h @@ -17,13 +17,14 @@ class CSPasswdLogging { - public: - static CSPasswdLogging *get(); - ~CSPasswdLogging(); + public: + static CSPasswdLogging* get(); + ~CSPasswdLogging(); - void log(int priority, const char *format, ...); - private: - CSPasswdLogging(); + void log(int priority, const char* format, ...); + + private: + CSPasswdLogging(); }; /** @@ -63,7 +64,7 @@ int secrets_keylen(); */ int secrets_ivlen(); -bool load_encryption_keys(); +bool load_encryption_keys(); std::string encrypt_password_old(const ByteVec& key, const ByteVec& iv, const std::string& input); std::string encrypt_password(const ByteVec& key, const std::string& input); @@ -72,10 +73,10 @@ std::string decrypt_password(const ByteVec& key, const std::string& input); struct ReadKeyResult { - bool ok {false}; - ByteVec key; - ByteVec iv; + bool ok{false}; + ByteVec key; + ByteVec iv; }; ReadKeyResult secrets_readkeys(const std::string& filepath); -bool secrets_write_keys(const ByteVec& key, const std::string& filepath, const std::string& owner); +bool secrets_write_keys(const ByteVec& key, const std::string& filepath, const std::string& owner); diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp index e417a2baa..618420f1f 100644 --- a/tools/pingproc/pingproc.cpp +++ b/tools/pingproc/pingproc.cpp @@ -52,246 +52,244 @@ bool thdFcnFailure; namespace { - -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - struct timespec& diff) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) { - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec + 1000000000; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } -} // timespec_sub + if (tv2.tv_nsec < tv1.tv_nsec) + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec + 1000000000; + } + else + { + diff.tv_sec = tv2.tv_sec - tv1.tv_sec; + diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; + } +} // timespec_sub // // class OidOperation { + public: + enum OpType_t + { + SCAN = 0, + BLOCK = 1, + LOOPBACK = 2, + NONE = 3, + BATCHSCAN = 4, + BATCHSTEP = 5, + BATCHFILT = 6 + }; -public: + OidOperation(const OID_t oid, const OpType_t opType, const uint32_t sessionId = 0); + ~OidOperation(){}; - enum OpType_t - { - SCAN = 0, - BLOCK = 1, - LOOPBACK = 2, - NONE = 3, - BATCHSCAN = 4, - BATCHSTEP = 5, - BATCHFILT = 6 - }; + void addFilter(const int8_t COP, const int64_t value); - OidOperation(const OID_t oid, const OpType_t opType, const uint32_t sessionId = 0); - ~OidOperation() {}; + const OID_t OID() const + { + return fOid; + } + const OpType_t OpType() const + { + return fOpType; + } - void addFilter(const int8_t COP, const int64_t value); + const CalpontSystemCatalog::ColType& ColumnType() + { + return fColType; + } + const uint32_t ColumnWidth() const + { + return fColType.colWidth; + } - const OID_t OID() const - { - return fOid; - } - const OpType_t OpType() const - { - return fOpType; - } + const uint32_t FilterCount() const + { + return fFilterCount; + } + const ByteStream& FilterString() + { + return fFilterList; + } + const uint32_t SessionId() const + { + return fSessionId; + } + const uint32_t DataType() const + { + return fColType.colDataType; + } + const uint32_t BOP() const + { + return fBOP; + } + void BOP(const uint32_t bop) + { + if (FilterCount() >= 2) + fBOP = bop; + } + const uint32_t COP1() const + { + return fCOP1; + } + void COP1(const uint32_t cop) + { + fCOP1 = cop; + } + const uint32_t COP2() const + { + return fCOP2; + } + void COP2(const uint32_t cop) + { + fCOP2 = cop; + } + bool isIntegralDataType(); + void setLbidTraceOn(); + void setPMProfileOn(); + bool LbidTrace() + { + return fLbidTrace; + } + bool PMProfile() + { + return fPMProfile; + } - const CalpontSystemCatalog::ColType& ColumnType() - { - return fColType; - } - const uint32_t ColumnWidth() const - { - return fColType.colWidth; - } + void deSerializeFilter(int8_t& COP, int64_t& value); - const uint32_t FilterCount() const - { - return fFilterCount; - } - const ByteStream& FilterString() - { - return fFilterList; - } - const uint32_t SessionId() const - { - return fSessionId; - } - const uint32_t DataType() const - { - return fColType.colDataType; - } - const uint32_t BOP() const - { - return fBOP; - } - void BOP(const uint32_t bop) - { - if (FilterCount() >= 2) fBOP = bop; - } - const uint32_t COP1() const - { - return fCOP1; - } - void COP1(const uint32_t cop) - { - fCOP1 = cop; - } - const uint32_t COP2() const - { - return fCOP2; - } - void COP2(const uint32_t cop) - { - fCOP2 = cop; - } - bool isIntegralDataType(); - void setLbidTraceOn(); - void setPMProfileOn(); - bool LbidTrace() - { - return fLbidTrace; - } - bool PMProfile() - { - return fPMProfile; - } - - void deSerializeFilter(int8_t& COP, int64_t& value); - -// private: - OidOperation() {}; - OID_t fOid; - - ByteStream fFilterList; - uint32_t fFilterCount; - OpType_t fOpType; - CalpontSystemCatalog::ColType fColType; - uint32_t fSessionId; - uint32_t fBOP; - uint32_t fCOP1; - uint32_t fCOP2; - bool fLbidTrace; - bool fPMProfile; -}; // class OidOperation + // private: + OidOperation(){}; + OID_t fOid; + ByteStream fFilterList; + uint32_t fFilterCount; + OpType_t fOpType; + CalpontSystemCatalog::ColType fColType; + uint32_t fSessionId; + uint32_t fBOP; + uint32_t fCOP1; + uint32_t fCOP2; + bool fLbidTrace; + bool fPMProfile; +}; // class OidOperation // // OidOperation::OidOperation(const OID_t oid, const OpType_t opType, const uint32_t sessionId) - : fOid(oid), fFilterList(), fFilterCount(0), fOpType(opType), fSessionId(sessionId), fBOP(BOP_NONE), fCOP1(COMPARE_NIL), fCOP2(COMPARE_NIL), fLbidTrace(false), fPMProfile(false) + : fOid(oid) + , fFilterList() + , fFilterCount(0) + , fOpType(opType) + , fSessionId(sessionId) + , fBOP(BOP_NONE) + , fCOP1(COMPARE_NIL) + , fCOP2(COMPARE_NIL) + , fLbidTrace(false) + , fPMProfile(false) { - boost::shared_ptr cat = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - fColType = cat->colType(oid); - fFilterList.reset(); + boost::shared_ptr cat = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); + fColType = cat->colType(oid); + fFilterList.reset(); } void OidOperation::setLbidTraceOn() { - fLbidTrace = true; + fLbidTrace = true; } void OidOperation::setPMProfileOn() { - fPMProfile = true; + fPMProfile = true; } void OidOperation::addFilter(const int8_t COP, const int64_t value) { + if (fFilterCount == 2) + return; - if (fFilterCount == 2) - return; + fFilterList << (uint8_t)COP; - fFilterList << (uint8_t) COP; + // converts to a type of the appropriate width, then bitwise + // copies into the filter ByteStream + switch (ColumnWidth()) + { + case 1: + int8_t tmp8; + tmp8 = value; + fFilterList << *((uint8_t*)&tmp8); + break; - // converts to a type of the appropriate width, then bitwise - // copies into the filter ByteStream - switch (ColumnWidth()) - { - case 1: - int8_t tmp8; - tmp8 = value; - fFilterList << *((uint8_t*) &tmp8); - break; + case 2: + int16_t tmp16; + tmp16 = value; + fFilterList << *((uint16_t*)&tmp16); + break; - case 2: - int16_t tmp16; - tmp16 = value; - fFilterList << *((uint16_t*) &tmp16); - break; + case 4: + int32_t tmp32; + tmp32 = value; + fFilterList << *((uint32_t*)&tmp32); + break; - case 4: - int32_t tmp32; - tmp32 = value; - fFilterList << *((uint32_t*) &tmp32); - break; + case 8: fFilterList << *((uint64_t*)&value); break; - case 8: - fFilterList << *((uint64_t*) &value); - break; + default: + ostringstream o; - default: - ostringstream o; + o << "addFilter: colType says OID " + << " has a width of " << ColumnWidth(); + throw runtime_error(o.str()); + } - o << "addFilter: colType says OID " << - " has a width of " << ColumnWidth(); - throw runtime_error(o.str()); - } - - fFilterCount++; + fFilterCount++; } - void OidOperation::deSerializeFilter(int8_t& COP, int64_t& value) { + if (fFilterCount == 0) + { + COP = COMPARE_NIL; + return; + } - if (fFilterCount == 0) - { - COP = COMPARE_NIL; - return; - } + fFilterList >> *(uint8_t*)&COP; - fFilterList >> *(uint8_t*) &COP; + switch (ColumnWidth()) + { + case 1: + int8_t tmp8; + tmp8 = value; + fFilterList >> *((uint8_t*)&tmp8); + value = tmp8; + break; - switch (ColumnWidth()) - { - case 1: - int8_t tmp8; - tmp8 = value; - fFilterList >> *((uint8_t*) &tmp8); - value = tmp8; - break; + case 2: + int16_t tmp16; + fFilterList >> *((uint16_t*)&tmp16); + value = tmp16; + break; - case 2: - int16_t tmp16; - fFilterList >> *((uint16_t*) &tmp16); - value = tmp16; - break; + case 4: + int32_t tmp32; + fFilterList >> *((uint32_t*)&tmp32); + value = tmp32; + break; - case 4: - int32_t tmp32; - fFilterList >> *((uint32_t*) &tmp32); - value = tmp32; - break; + case 8: fFilterList >> *((uint64_t*)&value); break; - case 8: - fFilterList >> *((uint64_t*) &value); - break; + default: + ostringstream o; - default: - ostringstream o; + o << "deSerializeFilter: colType says OID " + << " has a width of " << ColumnWidth(); + throw runtime_error(o.str()); + } - o << "deSerializeFilter: colType says OID " << - " has a width of " << ColumnWidth(); - throw runtime_error(o.str()); - } - - fFilterCount--; + fFilterCount--; } typedef vector OperationList; @@ -300,236 +298,215 @@ typedef vector OperationList; // bool OidOperation::isIntegralDataType() { - if (DataType() == CalpontSystemCatalog::BIT || - DataType() == CalpontSystemCatalog::TINYINT || - DataType() == CalpontSystemCatalog::SMALLINT || - DataType() == CalpontSystemCatalog::MEDINT || - DataType() == CalpontSystemCatalog::INT || - DataType() == CalpontSystemCatalog::DATE || - DataType() == CalpontSystemCatalog::BIGINT || - DataType() == CalpontSystemCatalog::DATETIME || - DataType() == CalpontSystemCatalog::TIMESTAMP || - DataType() == CalpontSystemCatalog::TIME || - DataType() == CalpontSystemCatalog::UTINYINT || - DataType() == CalpontSystemCatalog::USMALLINT || - DataType() == CalpontSystemCatalog::UMEDINT || - DataType() == CalpontSystemCatalog::UINT || - DataType() == CalpontSystemCatalog::UBIGINT) - return true; + if (DataType() == CalpontSystemCatalog::BIT || DataType() == CalpontSystemCatalog::TINYINT || + DataType() == CalpontSystemCatalog::SMALLINT || DataType() == CalpontSystemCatalog::MEDINT || + DataType() == CalpontSystemCatalog::INT || DataType() == CalpontSystemCatalog::DATE || + DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || + DataType() == CalpontSystemCatalog::TIMESTAMP || DataType() == CalpontSystemCatalog::TIME || + DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || + DataType() == CalpontSystemCatalog::UMEDINT || DataType() == CalpontSystemCatalog::UINT || + DataType() == CalpontSystemCatalog::UBIGINT) + return true; - if (DataType() == CalpontSystemCatalog::CHAR && 1 == fColType.colWidth) - return true; + if (DataType() == CalpontSystemCatalog::CHAR && 1 == fColType.colWidth) + return true; - return false; + return false; } const ByteStream formatLoopBackMsg(const uint32_t sessionId, uint32_t uniqueId) { - ByteStream primMsg; - ISMPacketHeader ism; - memset(&ism, 0, sizeof(ism)); - ism.Command = COL_LOOPBACK; - ism.Size = sizeof(ism) + sizeof(ColLoopback); - ism.Type = 2; - primMsg.load((const uint8_t*) &ism, sizeof(ism)); - struct ColLoopback lb; + ByteStream primMsg; + ISMPacketHeader ism; + memset(&ism, 0, sizeof(ism)); + ism.Command = COL_LOOPBACK; + ism.Size = sizeof(ism) + sizeof(ColLoopback); + ism.Type = 2; + primMsg.load((const uint8_t*)&ism, sizeof(ism)); + struct ColLoopback lb; - memset(&lb, 0, sizeof(lb)); + memset(&lb, 0, sizeof(lb)); - lb.Hdr.SessionID = sessionId; - lb.Hdr.StatementID = 0; - lb.Hdr.TransactionID = sessionId; - lb.Hdr.VerID = 0; - lb.Hdr.StepID = sessionId; - lb.Hdr.UniqueID = uniqueId; - primMsg.append((const uint8_t*) &lb, sizeof(lb)); + lb.Hdr.SessionID = sessionId; + lb.Hdr.StatementID = 0; + lb.Hdr.TransactionID = sessionId; + lb.Hdr.VerID = 0; + lb.Hdr.StepID = sessionId; + lb.Hdr.UniqueID = uniqueId; + primMsg.append((const uint8_t*)&lb, sizeof(lb)); - return primMsg; -} // formatLoopBackMsg + return primMsg; +} // formatLoopBackMsg -const ByteStream formatDictionaryMsg( - const uint64_t lbid, - ByteStream& ridList, - const uint16_t ridCount, - OidOperation& oidOp) +const ByteStream formatDictionaryMsg(const uint64_t lbid, ByteStream& ridList, const uint16_t ridCount, + OidOperation& oidOp) { - ByteStream primMsg; - DictSignatureRequestHeader hdr; - ISMPacketHeader ism; + ByteStream primMsg; + DictSignatureRequestHeader hdr; + ISMPacketHeader ism; - ism.Flags = 0; //planFlagsToPrimFlags(fTraceFlags); - ism.Command = DICT_SIGNATURE; - ism.Size = sizeof(DictSignatureRequestHeader) + ridList.length(); - ism.Type = 2; + ism.Flags = 0; // planFlagsToPrimFlags(fTraceFlags); + ism.Command = DICT_SIGNATURE; + ism.Size = sizeof(DictSignatureRequestHeader) + ridList.length(); + ism.Type = 2; - hdr.Hdr.SessionID = oidOp.SessionId(); - hdr.Hdr.StatementID = 0; - hdr.Hdr.TransactionID = oidOp.SessionId(); - hdr.Hdr.VerID = 0; - hdr.Hdr.StepID = 0; + hdr.Hdr.SessionID = oidOp.SessionId(); + hdr.Hdr.StatementID = 0; + hdr.Hdr.TransactionID = oidOp.SessionId(); + hdr.Hdr.VerID = 0; + hdr.Hdr.StepID = 0; - hdr.LBID = lbid; - hdr.PBID = 0; - idbassert(ridCount <= 8000); - hdr.NVALS = ridCount; + hdr.LBID = lbid; + hdr.PBID = 0; + idbassert(ridCount <= 8000); + hdr.NVALS = ridCount; - primMsg.load((const uint8_t*) &ism, sizeof(ism)); - primMsg.append((const uint8_t*) &hdr, sizeof(DictSignatureRequestHeader)); - primMsg += ridList; + primMsg.load((const uint8_t*)&ism, sizeof(ism)); + primMsg.append((const uint8_t*)&hdr, sizeof(DictSignatureRequestHeader)); + primMsg += ridList; - return primMsg; + return primMsg; } - -const ByteStream formatColStepMsg( - const uint64_t lbid, - ByteStream& ridList, - const uint16_t ridCount, - OidOperation& oidOp, - uint32_t uniqueId) +const ByteStream formatColStepMsg(const uint64_t lbid, ByteStream& ridList, const uint16_t ridCount, + OidOperation& oidOp, uint32_t uniqueId) { - ByteStream primMsg; - NewColRequestHeader hdr; + ByteStream primMsg; + NewColRequestHeader hdr; - memset(&hdr, 0, sizeof(hdr)); + memset(&hdr, 0, sizeof(hdr)); - hdr.ism.Reserve = 0; - hdr.ism.Flags = 0; + hdr.ism.Reserve = 0; + hdr.ism.Flags = 0; - if (oidOp.LbidTrace() == true) - hdr.ism.Flags |= PF_LBID_TRACE; + if (oidOp.LbidTrace() == true) + hdr.ism.Flags |= PF_LBID_TRACE; - if (oidOp.PMProfile() == true) - hdr.ism.Flags |= PF_PM_PROF; + if (oidOp.PMProfile() == true) + hdr.ism.Flags |= PF_PM_PROF; - hdr.ism.Command = COL_BY_SCAN; - hdr.ism.Size = sizeof(NewColRequestHeader) + oidOp.FilterString().length() + ridList.length(); - hdr.ism.Type = 2; + hdr.ism.Command = COL_BY_SCAN; + hdr.ism.Size = sizeof(NewColRequestHeader) + oidOp.FilterString().length() + ridList.length(); + hdr.ism.Type = 2; - hdr.hdr.SessionID = oidOp.SessionId(); - hdr.hdr.StatementID = 0; - hdr.hdr.TransactionID = oidOp.SessionId(); - hdr.hdr.VerID = 0; - hdr.hdr.StepID = oidOp.SessionId(); - hdr.hdr.UniqueID = uniqueId; + hdr.hdr.SessionID = oidOp.SessionId(); + hdr.hdr.StatementID = 0; + hdr.hdr.TransactionID = oidOp.SessionId(); + hdr.hdr.VerID = 0; + hdr.hdr.StepID = oidOp.SessionId(); + hdr.hdr.UniqueID = uniqueId; - hdr.LBID = lbid; - idbassert(hdr.LBID > 0); - hdr.PBID = 0; - hdr.DataSize = oidOp.ColumnWidth(); - hdr.DataType = oidOp.DataType(); - hdr.OutputType = OT_BOTH; - hdr.BOP = BOP_NONE; -// hdr.InputFlags = 0; - hdr.NOPS = oidOp.FilterCount(); - hdr.NVALS = ridCount; - hdr.sort = 0; + hdr.LBID = lbid; + idbassert(hdr.LBID > 0); + hdr.PBID = 0; + hdr.DataSize = oidOp.ColumnWidth(); + hdr.DataType = oidOp.DataType(); + hdr.OutputType = OT_BOTH; + hdr.BOP = BOP_NONE; + // hdr.InputFlags = 0; + hdr.NOPS = oidOp.FilterCount(); + hdr.NVALS = ridCount; + hdr.sort = 0; - primMsg.load((const uint8_t*) &hdr, sizeof(NewColRequestHeader)); + primMsg.load((const uint8_t*)&hdr, sizeof(NewColRequestHeader)); - if (oidOp.FilterCount() > 0) - primMsg += oidOp.FilterString(); - - if (ridCount > 0) - primMsg += ridList; - - return primMsg; - -} //formatColStepMsg - -const ByteStream formatDictionaryScanMsg(const uint64_t lbid, - const uint16_t count, - OidOperation& oidOp) -{ - ByteStream primMsg; - DictTokenByScanRequestHeader hdr; - - hdr.ism.Reserve = 0; - hdr.ism.Flags = 0; - hdr.ism.Command = DICT_TOKEN_BY_SCAN_COMPARE; - hdr.ism.Size = sizeof(DictTokenByScanRequestHeader) + - oidOp.FilterString().length(); - hdr.ism.Type = 2; - - hdr.Hdr.SessionID = oidOp.SessionId(); - hdr.Hdr.StatementID = 0; - hdr.Hdr.TransactionID = 0; - hdr.Hdr.VerID = 0; - hdr.Hdr.StepID = oidOp.SessionId(); - - hdr.LBID = lbid; - idbassert(hdr.LBID >= 0); - hdr.PBID = 0; - hdr.OutputType = OT_TOKEN; - hdr.BOP = oidOp.BOP(); - hdr.COP1 = oidOp.COP1(); - hdr.COP2 = oidOp.COP2(); - hdr.NVALS = oidOp.FilterCount(); - hdr.Count = count; - hdr.charsetNumber = oidOp.ColumnType().charsetNumber; - idbassert(hdr.Count > 0); - - primMsg.load((const uint8_t*) &hdr.ism, sizeof(ISMPacketHeader)); - primMsg.append((const uint8_t*) &hdr, sizeof(DictTokenByScanRequestHeader)); + if (oidOp.FilterCount() > 0) primMsg += oidOp.FilterString(); - return primMsg; + if (ridCount > 0) + primMsg += ridList; + + return primMsg; + +} // formatColStepMsg + +const ByteStream formatDictionaryScanMsg(const uint64_t lbid, const uint16_t count, OidOperation& oidOp) +{ + ByteStream primMsg; + DictTokenByScanRequestHeader hdr; + + hdr.ism.Reserve = 0; + hdr.ism.Flags = 0; + hdr.ism.Command = DICT_TOKEN_BY_SCAN_COMPARE; + hdr.ism.Size = sizeof(DictTokenByScanRequestHeader) + oidOp.FilterString().length(); + hdr.ism.Type = 2; + + hdr.Hdr.SessionID = oidOp.SessionId(); + hdr.Hdr.StatementID = 0; + hdr.Hdr.TransactionID = 0; + hdr.Hdr.VerID = 0; + hdr.Hdr.StepID = oidOp.SessionId(); + + hdr.LBID = lbid; + idbassert(hdr.LBID >= 0); + hdr.PBID = 0; + hdr.OutputType = OT_TOKEN; + hdr.BOP = oidOp.BOP(); + hdr.COP1 = oidOp.COP1(); + hdr.COP2 = oidOp.COP2(); + hdr.NVALS = oidOp.FilterCount(); + hdr.Count = count; + hdr.charsetNumber = oidOp.ColumnType().charsetNumber; + idbassert(hdr.Count > 0); + + primMsg.load((const uint8_t*)&hdr.ism, sizeof(ISMPacketHeader)); + primMsg.append((const uint8_t*)&hdr, sizeof(DictTokenByScanRequestHeader)); + primMsg += oidOp.FilterString(); + + return primMsg; } -const ByteStream formatColScanMsg(const uint64_t lbid, - const uint16_t count, - OidOperation& oidOp, +const ByteStream formatColScanMsg(const uint64_t lbid, const uint16_t count, OidOperation& oidOp, uint32_t uniqueId) { - ByteStream primMsg; - ISMPacketHeader ism; - ColByScanRangeRequestHeader fMsgHeader; + ByteStream primMsg; + ISMPacketHeader ism; + ColByScanRangeRequestHeader fMsgHeader; - memset(&fMsgHeader, 0, sizeof(fMsgHeader)); - memset(&ism, 0, sizeof(ism)); + memset(&fMsgHeader, 0, sizeof(fMsgHeader)); + memset(&ism, 0, sizeof(ism)); - ism.Reserve = 0; - ism.Flags = 0; + ism.Reserve = 0; + ism.Flags = 0; - if (oidOp.LbidTrace() == true) - ism.Flags |= PF_LBID_TRACE; + if (oidOp.LbidTrace() == true) + ism.Flags |= PF_LBID_TRACE; - if (oidOp.PMProfile() == true) - ism.Flags |= PF_PM_PROF; + if (oidOp.PMProfile() == true) + ism.Flags |= PF_PM_PROF; - ism.Command = COL_BY_SCAN_RANGE; - ism.Size = sizeof(fMsgHeader) + sizeof(ism) + oidOp.FilterString().length(); - ism.Type = 2; + ism.Command = COL_BY_SCAN_RANGE; + ism.Size = sizeof(fMsgHeader) + sizeof(ism) + oidOp.FilterString().length(); + ism.Type = 2; - primMsg.load((const uint8_t*) &ism, sizeof(ism)); + primMsg.load((const uint8_t*)&ism, sizeof(ism)); - fMsgHeader.LBID = lbid; - idbassert(fMsgHeader.LBID >= 0); - fMsgHeader.PBID = 0; - fMsgHeader.DataSize = oidOp.ColumnWidth(); - fMsgHeader.DataType = oidOp.DataType(); - fMsgHeader.OutputType = OT_BOTH; - fMsgHeader.BOP = oidOp.BOP(); - fMsgHeader.NOPS = oidOp.FilterCount(); - fMsgHeader.NVALS = 0; - fMsgHeader.Count = count; //(hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - idbassert(fMsgHeader.Count > 0); - fMsgHeader.Hdr.SessionID = oidOp.SessionId(); - fMsgHeader.Hdr.StatementID = 0; - fMsgHeader.Hdr.TransactionID = oidOp.SessionId(); - fMsgHeader.Hdr.VerID = 0; - fMsgHeader.Hdr.StepID = oidOp.SessionId(); - fMsgHeader.Hdr.UniqueID = uniqueId; + fMsgHeader.LBID = lbid; + idbassert(fMsgHeader.LBID >= 0); + fMsgHeader.PBID = 0; + fMsgHeader.DataSize = oidOp.ColumnWidth(); + fMsgHeader.DataType = oidOp.DataType(); + fMsgHeader.OutputType = OT_BOTH; + fMsgHeader.BOP = oidOp.BOP(); + fMsgHeader.NOPS = oidOp.FilterCount(); + fMsgHeader.NVALS = 0; + fMsgHeader.Count = count; //(hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + idbassert(fMsgHeader.Count > 0); + fMsgHeader.Hdr.SessionID = oidOp.SessionId(); + fMsgHeader.Hdr.StatementID = 0; + fMsgHeader.Hdr.TransactionID = oidOp.SessionId(); + fMsgHeader.Hdr.VerID = 0; + fMsgHeader.Hdr.StepID = oidOp.SessionId(); + fMsgHeader.Hdr.UniqueID = uniqueId; - primMsg.append((const uint8_t*) &fMsgHeader, sizeof(fMsgHeader)); + primMsg.append((const uint8_t*)&fMsgHeader, sizeof(fMsgHeader)); - if (oidOp.FilterCount() > 0) - primMsg += oidOp.FilterString(); + if (oidOp.FilterCount() > 0) + primMsg += oidOp.FilterString(); - return primMsg; - -} //formatColScanMsg + return primMsg; +} // formatColScanMsg // void doBatchOp_scan(OidOperation& OidOp); @@ -545,1264 +522,1245 @@ void doDictStep(OidOperation& OidOp); // struct ThdFcn { - void operator()() + void operator()() + { + uint64_t totalBytes = 0; + + try { - uint64_t totalBytes = 0; + ByteStream ibs; - try - { - ByteStream ibs; + if (debug) + cout << "Waiting on " << fNumMsgs << " messages." << endl; - if (debug) - cout << "Waiting on " << fNumMsgs << " messages." << endl; - - for (uint32_t k = 0; k < fNumMsgs; k++) - { -// cout << "reading msg #" << k << "...\n"; - ibs = fDec->read(uniqueID); - -// cout << "got msg #" << k << endl; - if (debug) - if (k % 10240 == 0) - cout - << "ThdFcn: read " << fSessionid - << " " << k << "/" << fNumMsgs << " " - << ibs.length() << "/" << totalBytes - << endl; - - if (ibs.length() == 0) - break; - - totalBytes += ibs.length(); - } - } - catch (exception& e) - { - cerr << "read exception: " << e.what() << endl; - thdFcnFailure = true; - } + for (uint32_t k = 0; k < fNumMsgs; k++) + { + // cout << "reading msg #" << k << "...\n"; + ibs = fDec->read(uniqueID); + // cout << "got msg #" << k << endl; if (debug) - cout << totalBytes << " bytes read in " - << fNumMsgs << " messages" << endl; + if (k % 10240 == 0) + cout << "ThdFcn: read " << fSessionid << " " << k << "/" << fNumMsgs << " " << ibs.length() << "/" + << totalBytes << endl; - } // void operator() + if (ibs.length() == 0) + break; - uint32_t fSessionid; - uint32_t uniqueID; - DistributedEngineComm* fDec; - unsigned fNumMsgs; + totalBytes += ibs.length(); + } + } + catch (exception& e) + { + cerr << "read exception: " << e.what() << endl; + thdFcnFailure = true; + } -}; // struct ThdFcn + if (debug) + cout << totalBytes << " bytes read in " << fNumMsgs << " messages" << endl; + } // void operator() + + uint32_t fSessionid; + uint32_t uniqueID; + DistributedEngineComm* fDec; + unsigned fNumMsgs; + +}; // struct ThdFcn struct QryThdFcn { - void operator()() + void operator()() + { + uint64_t totalBytes = 0; + int64_t min; + int64_t max; + uint64_t lbid; + uint32_t cachedIO; + uint32_t physIO; + uint32_t touchedBlocks; + bool validCPData; + + try { - uint64_t totalBytes = 0; - int64_t min; - int64_t max; - uint64_t lbid; - uint32_t cachedIO; - uint32_t physIO; - uint32_t touchedBlocks; - bool validCPData; + ByteStream ibs; + ByteStream obs; - try - { - ByteStream ibs; - ByteStream obs; + if (debug) + cout << "Waiting on " << fNumMsgs << " messages." << endl; - if (debug) - cout << "Waiting on " << fNumMsgs << " messages." << endl; - - for (uint32_t k = 0; k < fNumMsgs; k++) - { -// cout << "reading msg #" << k << "...\n"; - ibs = fDec->read(uniqueID); - -// cout << "got msg #" << k << endl; - if (debug) - if (k % 10240 == 0) - cout - << "QryThdFcn: read " << fSessionid - << " " << k << "/" << fNumMsgs << " " - << ibs.length() << "/" << totalBytes - << " rows: " << fRows << endl; - - if (ibs.length() == 0) - break; - - totalBytes += ibs.length(); - fRows += fBpp.getTableBand (ibs, &obs, &validCPData, &lbid, &min, &max, &cachedIO, &physIO, &touchedBlocks ); - fBlockTouched += touchedBlocks; - } - } - catch (exception& e) - { - cerr << "read exception: " << e.what() << endl; - thdFcnFailure = true; - } + for (uint32_t k = 0; k < fNumMsgs; k++) + { + // cout << "reading msg #" << k << "...\n"; + ibs = fDec->read(uniqueID); + // cout << "got msg #" << k << endl; if (debug) - cout << totalBytes << " bytes read in " - << fNumMsgs << " messages for " - << fRows << " rows and " << fBlockTouched << " blocks\n"; + if (k % 10240 == 0) + cout << "QryThdFcn: read " << fSessionid << " " << k << "/" << fNumMsgs << " " << ibs.length() + << "/" << totalBytes << " rows: " << fRows << endl; - } // void operator() + if (ibs.length() == 0) + break; - QryThdFcn(BatchPrimitiveProcessorJL& bpp, uint64_t& rows, uint32_t& blk): fBpp(bpp), fNumMsgs(0), fRows(rows), fBlockTouched(blk) {} - BatchPrimitiveProcessorJL& fBpp; - uint32_t fSessionid; - DistributedEngineComm* fDec; - unsigned fNumMsgs; - uint64_t& fRows; - uint32_t& fBlockTouched; - uint32_t uniqueID; + totalBytes += ibs.length(); + fRows += + fBpp.getTableBand(ibs, &obs, &validCPData, &lbid, &min, &max, &cachedIO, &physIO, &touchedBlocks); + fBlockTouched += touchedBlocks; + } + } + catch (exception& e) + { + cerr << "read exception: " << e.what() << endl; + thdFcnFailure = true; + } -}; // struct ThdFcn + if (debug) + cout << totalBytes << " bytes read in " << fNumMsgs << " messages for " << fRows << " rows and " + << fBlockTouched << " blocks\n"; + } // void operator() + + QryThdFcn(BatchPrimitiveProcessorJL& bpp, uint64_t& rows, uint32_t& blk) + : fBpp(bpp), fNumMsgs(0), fRows(rows), fBlockTouched(blk) + { + } + BatchPrimitiveProcessorJL& fBpp; + uint32_t fSessionid; + DistributedEngineComm* fDec; + unsigned fNumMsgs; + uint64_t& fRows; + uint32_t& fBlockTouched; + uint32_t uniqueID; + +}; // struct ThdFcn struct BatchScanThr { - BatchScanThr(OidOperation& oidOp): fOidOp(oidOp) {} + BatchScanThr(OidOperation& oidOp) : fOidOp(oidOp) + { + } - void operator()() - { - doBatchOp_scan(fOidOp); - } // void operator() + void operator()() + { + doBatchOp_scan(fOidOp); + } // void operator() - OidOperation& fOidOp; + OidOperation& fOidOp; -}; // struct BatchScanThr +}; // struct BatchScanThr struct BatchStepThr { - BatchStepThr(OidOperation& oidOp): fOidOp(oidOp) {} + BatchStepThr(OidOperation& oidOp) : fOidOp(oidOp) + { + } - void operator()() - { - doBatchOp_step(fOidOp); - } // void operator() + void operator()() + { + doBatchOp_step(fOidOp); + } // void operator() - OidOperation& fOidOp; + OidOperation& fOidOp; -}; // struct BatchStepThr +}; // struct BatchStepThr struct BatchFiltThr { - BatchFiltThr(OidOperation& oidOp): fOidOp(oidOp) {} + BatchFiltThr(OidOperation& oidOp) : fOidOp(oidOp) + { + } - void operator()() - { - doBatchOp_filt(fOidOp); - } // void operator() + void operator()() + { + doBatchOp_filt(fOidOp); + } // void operator() - OidOperation& fOidOp; + OidOperation& fOidOp; -}; // struct BatchFiltThr +}; // struct BatchFiltThr struct BatchQueryThr { - BatchQueryThr(OperationList& oidOps): fOidOps(oidOps) {} + BatchQueryThr(OperationList& oidOps) : fOidOps(oidOps) + { + } - void operator()() - { - doBatchQueryOp(fOidOps); - } // void operator() + void operator()() + { + doBatchQueryOp(fOidOps); + } // void operator() - OperationList& fOidOps; - -}; // struct + OperationList& fOidOps; +}; // struct struct ColStepThr { - ColStepThr(OidOperation& oidOp): fOidOp(oidOp) {} + ColStepThr(OidOperation& oidOp) : fOidOp(oidOp) + { + } - void operator()() - { - doColStep(fOidOp); - } // void operator() + void operator()() + { + doColStep(fOidOp); + } // void operator() - OidOperation& fOidOp; + OidOperation& fOidOp; -}; // struct ColStepThr +}; // struct ColStepThr struct ColScanThr { + ColScanThr(OidOperation& oidOp) : fOidOp(oidOp) + { + } - ColScanThr(OidOperation& oidOp): fOidOp(oidOp) {} + void operator()() + { + doColScan(fOidOp); + } // void operator() - void operator()() - { - doColScan(fOidOp); - } // void operator() + OidOperation& fOidOp; - OidOperation& fOidOp; - -}; // struct ColStepThr +}; // struct ColStepThr struct DictSigThr { -}; // DictSigThr +}; // DictSigThr struct DictScanThr { -}; // DictScanThr +}; // DictScanThr // doColScan void doDictionaryScan(OidOperation& OidOp) { } - // doColScan void doColScan(OidOperation& OidOp) { + if (debug) + cout << "beginning doColScan\n"; - if (debug) cout << "beginning doColScan\n"; + BRM::LBIDRange_v lbidRanges; + HWM_t hwm = 0; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + struct timespec ts1; + struct timespec ts2; + struct timespec diff; + uint32_t uniqueID; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - struct timespec ts1; - struct timespec ts2; - struct timespec diff; - uint32_t uniqueID; + uint32_t sessionid = getpid(); + uint32_t totalBlks = 0; + // dec->addSession(sessionid); + // dec->addStep(sessionid, sessionid); + DBRM dbrm; - uint32_t sessionid = getpid(); - uint32_t totalBlks = 0; -// dec->addSession(sessionid); -// dec->addStep(sessionid, sessionid); - DBRM dbrm; + uniqueID = dbrm.getUnique32(); + dec->addQueue(uniqueID); - uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); + int err = dbrm.lookup(OidOp.OID(), lbidRanges); - int err = dbrm.lookup(OidOp.OID(), lbidRanges); + if (err) + throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); + err = dbrm.getHWM(OidOp.OID(), hwm); - err = dbrm.getHWM(OidOp.OID(), hwm); + if (err) + throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); + LBIDRange_v::iterator it; + OID_t tmp; + uint32_t fbo; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; + ThdFcn f1; + f1.fSessionid = sessionid; + f1.uniqueID = uniqueID; + f1.fDec = dec; + f1.fNumMsgs = 0; + thdFcnFailure = false; + uint32_t rangeSize = 0; + ByteStream obs; - ThdFcn f1; - f1.fSessionid = sessionid; - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = 0; - thdFcnFailure = false; - uint32_t rangeSize = 0; - ByteStream obs; + // calculate the expected number of messages + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; - // calculate the expected number of messages - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) { + cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; + abort(); + } - BRM::LBID_t lbid = (*it).start; + if (hwm < fbo) + continue; - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + totalBlks += rangeSize; + } // for - if (hwm < fbo) - continue; + f1.fNumMsgs = (totalBlks / OidOp.ColumnWidth()); - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - totalBlks += rangeSize; - } // for + if (0 < totalBlks % OidOp.ColumnWidth()) + ++f1.fNumMsgs; - f1.fNumMsgs = (totalBlks / OidOp.ColumnWidth()); + idbassert(f1.fNumMsgs); - if (0 < totalBlks % OidOp.ColumnWidth()) ++f1.fNumMsgs; + if (debug) + cout << "Scanning OID " << OidOp.OID() << " " << f1.fNumMsgs << " msgs" << endl; - idbassert(f1.fNumMsgs); + thread t1(f1); + clock_gettime(CLOCK_REALTIME, &ts1); + // send the primitive requests + int rCount = 0; + totalBlks = 0; - if (debug) - cout - << "Scanning OID " << OidOp.OID() - << " " << f1.fNumMsgs << " msgs" << endl; - - thread t1(f1); - clock_gettime(CLOCK_REALTIME, &ts1); - // send the primitive requests - int rCount = 0; - totalBlks = 0; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + try { + BRM::LBID_t lbid = (*it).start; - try - { + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) + { + cerr << "doAColScan dbrm.lookup failed for lbid " << lbid << endl; + abort(); + } - BRM::LBID_t lbid = (*it).start; + if (hwm < fbo) + continue; - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "doAColScan dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + obs = formatColScanMsg(lbid, rangeSize, OidOp, uniqueID); - if (hwm < fbo) - continue; + if (obs.length() > 0) + { + dec->write(obs); + rCount++; - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - obs = formatColScanMsg(lbid, rangeSize, OidOp, uniqueID); + if (debug) + cout << "colScan: " << rCount << "/" << lbidRanges.size() << " sending " << obs.length() + << " bytes " + << " lbid " << lbid << " sz " << rangeSize << endl; + } + } + catch (exception& e) + { + cerr << "catch " << e.what() << endl; + } - if (obs.length() > 0) - { - dec->write(obs); - rCount++; + totalBlks += rangeSize; - if (debug) - cout << "colScan: " << rCount << "/" - << lbidRanges.size() << " sending " << obs.length() << " bytes " - << " lbid " << lbid << " sz " << rangeSize << endl; - } - } - catch (exception& e) - { - cerr << "catch " << e.what() << endl; - } + } // for (lbidRanges ... - totalBlks += rangeSize; + t1.join(); + clock_gettime(CLOCK_REALTIME, &ts2); + // dec->removeSession(sessionid); + dec->removeQueue(uniqueID); - } // for (lbidRanges ... + timespec_sub(ts1, ts2, diff); - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); -// dec->removeSession(sessionid); - dec->removeQueue(uniqueID); + cout << "ColScan stats OID: " << OidOp.OID() << "\tFilter: " << (int)OidOp.FilterCount() + << "\tBlocks: " << (int)totalBlks << "\tElapse: " << diff.tv_sec + (diff.tv_nsec / 1000000000.0) + << "s"; - timespec_sub(ts1, ts2, diff); + float rate = 0; + rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << " Blks/sec : " << rate << endl; - cout << "ColScan stats OID: " << OidOp.OID() - << "\tFilter: " << (int) OidOp.FilterCount() - << "\tBlocks: " << (int) totalBlks - << "\tElapse: " << diff.tv_sec + (diff.tv_nsec / 1000000000.0) << "s"; + if (thdFcnFailure) + cout << "There was a failure in the read thread." << endl; - float rate = 0; - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << " Blks/sec : " << rate << endl; + cout << endl; - if (thdFcnFailure) - cout << "There was a failure in the read thread." << endl; - - cout << endl; - -} // doAScan +} // doAScan // void usage() { - cerr << - "PingProc operation [filter] [operation] [filter] [reporting]" << endl << - "PingProc -c -s -gt 0 -t -d " << endl << - "\t---- operation flags ----" << endl << - "\t--scan -s " << endl << - "\t--block -t " << endl << - "\t--BatchPrimitiveScan -B " << endl << - "\t--BatchPrimitiveStep -Z " << endl << - "\t--concurrent -c perform each operation in its own thread" << endl << - "\t--lbid-trace -lb set lbid trace flag in request" << endl << - "\t---- filter flags ----" << endl << - "\t--equal -eq equivalency test of values in a block>" << endl << - "\t--greater-than -gt - greater than (>) test of values in a block" << endl << - "\t--greater-than-equal -ge greater than or equal to (>=) test of values in a block" << endl << - "\t--less-than -lt less than (<) test of values in a block" << endl << - "\t--less-than-equal -le less than or equal to (<=) test of values in a block" << endl << - "\t--not-equal -ne not equal to (!=) test of values in a block" << endl << - "\t--bop <1 or 0> binary operator when 2 comparison filters are present" << endl << - "\t---- reporting flags ----" << endl << - "\t--debug -d turn debug output on>" << endl << - "\t--list -list -l print out all oids and their ranges" << endl << - "\t--loopback -p send count loopback requests" << endl << - "\t--query -q run batch query for all the oids (enter with -B or -Z)" << endl; + cerr << "PingProc operation [filter] [operation] [filter] [reporting]" << endl + << "PingProc -c -s -gt 0 -t -d " << endl + << "\t---- operation flags ----" << endl + << "\t--scan -s " << endl + << "\t--block -t " << endl + << "\t--BatchPrimitiveScan -B " << endl + << "\t--BatchPrimitiveStep -Z " << endl + << "\t--concurrent -c perform each operation in its own thread" << endl + << "\t--lbid-trace -lb set lbid trace flag in request" << endl + << "\t---- filter flags ----" << endl + << "\t--equal -eq equivalency test of values in a block>" << endl + << "\t--greater-than -gt - greater than (>) test of values in a block" << endl + << "\t--greater-than-equal -ge greater than or equal to (>=) test of values in a block" << endl + << "\t--less-than -lt less than (<) test of values in a block" << endl + << "\t--less-than-equal -le less than or equal to (<=) test of values in a block" << endl + << "\t--not-equal -ne not equal to (!=) test of values in a block" << endl + << "\t--bop <1 or 0> binary operator when 2 comparison filters are present" << endl + << "\t---- reporting flags ----" << endl + << "\t--debug -d turn debug output on>" << endl + << "\t--list -list -l print out all oids and their ranges" << endl + << "\t--loopback -p send count loopback requests" << endl + << "\t--query -q run batch query for all the oids (enter with -B or -Z)" << endl; -} // usage() +} // usage() const int64_t getInt(string s) { + if (s.length() <= 0) + return -1; - if (s.length() <= 0) - return -1; + // if (atoll(s.data()) < 0) + // return -1; - //if (atoll(s.data()) < 0) - // return -1; + return atoll(s.data()); - return atoll(s.data()); - -} //getInt +} // getInt // dictionary void doDictionarySig(OidOperation& OidOp) { -} // doDictionarySig +} // doDictionarySig // col step void doColStep(OidOperation& OidOp) { + struct timespec ts1; + struct timespec ts2; + struct timespec diff; + DBRM dbrm; + BRM::LBIDRange_v lbidRanges; + HWM_t hwm = 0; + LBIDRange_v::iterator it; + OID_t tmp; + uint32_t fbo; + uint32_t totalBlks = 0; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + ThdFcn f1; - struct timespec ts1; - struct timespec ts2; - struct timespec diff; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - uint32_t totalBlks = 0; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; + // dec->addSession(OidOp.SessionId()); + // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); + uint32_t uniqueID = dbrm.getUnique32(); + dec->addQueue(uniqueID); -// dec->addSession(OidOp.SessionId()); -// dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - uint32_t uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); + f1.fSessionid = OidOp.SessionId(); + f1.uniqueID = uniqueID; + f1.fDec = dec; + f1.fNumMsgs = 0; + thdFcnFailure = false; + ByteStream ridlist; + uint16_t ridCount = 0; // BLOCK_SIZE/OidOp.ColumnWidth(); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = 0; - thdFcnFailure = false; - ByteStream ridlist; - uint16_t ridCount = 0; // BLOCK_SIZE/OidOp.ColumnWidth(); + for (uint16_t i = 0; i < ridCount; i++) + ridlist << i; - for (uint16_t i = 0; i < ridCount; i++) - ridlist << i; + int err = dbrm.lookup(OidOp.OID(), lbidRanges); - int err = dbrm.lookup(OidOp.OID(), lbidRanges); + if (err) + throw runtime_error("doAColStep: BRM LBID range lookup failure (1)"); - if (err) - throw runtime_error("doAColStep: BRM LBID range lookup failure (1)"); + err = dbrm.getHWM(OidOp.OID(), hwm); - err = dbrm.getHWM(OidOp.OID(), hwm); + if (err) + throw runtime_error("doAColStep: BRM HWM lookup failure (3)"); - if (err) - throw runtime_error("doAColStep: BRM HWM lookup failure (3)"); + uint32_t rangeSize = 0; - uint32_t rangeSize = 0; + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "pColStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - break; //continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - totalBlks += rangeSize; - } // for - - uint32_t colwidth = OidOp.ColumnWidth(); - f1.fNumMsgs = totalBlks / (colwidth); - - if (0 < totalBlks % colwidth) ++f1.fNumMsgs; - - idbassert(f1.fNumMsgs); - thread t1(f1); - ByteStream obs; - totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - if (debug) - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - - abort(); - } - - if (hwm < fbo) - break; //continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - - for (unsigned i = 0; i < rangeSize; i++) - { - obs += formatColStepMsg(lbid + i, ridlist, ridCount, OidOp, uniqueID); - - if (0 == (i + 1) % colwidth) - { - dec->write(obs); - - if (debug && i + 1 == rangeSize) - cout << "colStep: " << i << "/" - << rangeSize << " " << obs.length() - << " lbid " << lbid + i << endl; - - obs.restart(); - } - } - - totalBlks += rangeSize; - } // for - - if (obs.length()) - { - dec->write(obs); - - if (debug) - cout << "colStep: last" << "/" - << rangeSize << " " << obs.length() << endl; + cerr << "pColStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; + abort(); } - obs.reset(); + if (hwm < fbo) + break; // continue; - t1.join(); //@bug 849 moved join here and changed output to be like pColScan. - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); -// t1.join(); - cout << "ColStep stats OID: " << OidOp.OID() - << "\tFilter: " << (int) OidOp.FilterCount() - << "\tBlocks: " << (int) totalBlks - << "\tElapse: " << diff.tv_sec + (diff.tv_nsec / 1000000000.0) << "s"; + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + totalBlks += rangeSize; + } // for - float rate = 0; - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; + uint32_t colwidth = OidOp.ColumnWidth(); + f1.fNumMsgs = totalBlks / (colwidth); - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; + if (0 < totalBlks % colwidth) + ++f1.fNumMsgs; - cout << endl; + idbassert(f1.fNumMsgs); + thread t1(f1); + ByteStream obs; + totalBlks = 0; + clock_gettime(CLOCK_REALTIME, &ts1); -} // doColStep + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; + + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) + { + if (debug) + cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; + + abort(); + } + + if (hwm < fbo) + break; // continue; + + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + + for (unsigned i = 0; i < rangeSize; i++) + { + obs += formatColStepMsg(lbid + i, ridlist, ridCount, OidOp, uniqueID); + + if (0 == (i + 1) % colwidth) + { + dec->write(obs); + + if (debug && i + 1 == rangeSize) + cout << "colStep: " << i << "/" << rangeSize << " " << obs.length() << " lbid " << lbid + i << endl; + + obs.restart(); + } + } + + totalBlks += rangeSize; + } // for + + if (obs.length()) + { + dec->write(obs); + + if (debug) + cout << "colStep: last" + << "/" << rangeSize << " " << obs.length() << endl; + } + + obs.reset(); + + t1.join(); //@bug 849 moved join here and changed output to be like pColScan. + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + // t1.join(); + cout << "ColStep stats OID: " << OidOp.OID() << "\tFilter: " << (int)OidOp.FilterCount() + << "\tBlocks: " << (int)totalBlks << "\tElapse: " << diff.tv_sec + (diff.tv_nsec / 1000000000.0) + << "s"; + + float rate = 0; + rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\tBlks/sec " << rate << endl; + + if (thdFcnFailure) + cerr << "There was a failure in the read thread." << endl; + + cout << endl; + +} // doColStep void doBatchOp_scan(OidOperation& OidOp) { - struct timespec ts1, ts2, diff; - JobStepAssociation injs, outjs; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - boost::shared_ptr sysCat = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); + struct timespec ts1, ts2, diff; + JobStepAssociation injs, outjs; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + ThdFcn f1; + boost::shared_ptr sysCat = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - pColScanStep scan(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, - OidOp.fSessionId, 0, OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, rm); + pColScanStep scan(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, OidOp.fSessionId, 0, OidOp.fSessionId, + OidOp.fSessionId, OidOp.fSessionId, rm); - int32_t filters = OidOp.FilterCount(); + int32_t filters = OidOp.FilterCount(); - while (OidOp.FilterCount() > 0) + while (OidOp.FilterCount() > 0) + { + int8_t cop; + int64_t value; + OidOp.deSerializeFilter(cop, value); + scan.addFilter(cop, value); + } + + BatchPrimitiveProcessorJL bpp; + ByteStream bs; + DBRM dbrm; + BRM::LBIDRange_v lbidRanges; + HWM_t hwm = 0; + LBIDRange_v::iterator it; + OID_t tmp; + uint32_t fbo; + + uint32_t uniqueID = dbrm.getUnique32(); + bpp.setUniqueID(uniqueID); + + bpp.setSessionID(OidOp.SessionId()); + bpp.setStepID(OidOp.SessionId()); + bpp.addFilterStep(scan); + + cout << "session number = " << OidOp.SessionId() << endl; + // dec->addSession(OidOp.SessionId()); + // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); + dec->addQueue(uniqueID); + f1.fSessionid = OidOp.SessionId(); + f1.uniqueID = uniqueID; + f1.fDec = dec; + thdFcnFailure = false; + int err = dbrm.lookup(OidOp.OID(), lbidRanges); + + if (err) + { + cerr << "doAColScan: BRM LBID range lookup failure (1)\n"; + throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); + } + + err = dbrm.getHWM(OidOp.OID(), hwm); + + if (err) + { + cerr << "doAColScan: BRM HWM lookup failure (3)" << endl; + throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); + } + + f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); + + thread t1(f1); + + bpp.createBPP(bs); + dec->write(bs); + bs.restart(); + + uint32_t rangeSize = 0, totalBlks = 0; + clock_gettime(CLOCK_REALTIME, &ts1); + + // cout << "BPP scaning\n"; + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; + + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - scan.addFilter(cop, value); + cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; + abort(); } - BatchPrimitiveProcessorJL bpp; - ByteStream bs; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; + if (hwm < fbo) + continue; - uint32_t uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - - bpp.setSessionID(OidOp.SessionId()); - bpp.setStepID(OidOp.SessionId()); - bpp.addFilterStep(scan); - - cout << "session number = " << OidOp.SessionId() << endl; -// dec->addSession(OidOp.SessionId()); -// dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - dec->addQueue(uniqueID); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - thdFcnFailure = false; - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - { - cerr << "doAColScan: BRM LBID range lookup failure (1)\n"; - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - } - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - { - cerr << "doAColScan: BRM HWM lookup failure (3)" << endl; - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - } - - f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); - - thread t1(f1); - - bpp.createBPP(bs); + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + bpp.setLBID(lbid); + bpp.setCount(rangeSize / OidOp.fColType.colWidth + (rangeSize % OidOp.fColType.colWidth ? 1 : 0)); + bpp.runBPP(bs); dec->write(bs); + // cout << "sending the BPP\n"; + bpp.reset(); bs.restart(); + totalBlks += rangeSize; + } - uint32_t rangeSize = 0, totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); + t1.join(); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + float rate = 0; + cout << "ColStep stats OID: " << OidOp.OID() << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" + << "\tFilters: " << filters << "\tBlocks : " << (int)totalBlks; -// cout << "BPP scaning\n"; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; + rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\tBlks/sec " << rate << endl; - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - bpp.setLBID(lbid); - bpp.setCount(rangeSize / OidOp.fColType.colWidth + (rangeSize % OidOp.fColType.colWidth ? 1 : 0)); - bpp.runBPP(bs); - dec->write(bs); -// cout << "sending the BPP\n"; - bpp.reset(); - bs.restart(); - totalBlks += rangeSize; - } - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "ColStep stats OID: " << OidOp.OID() - << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tFilters: " << filters - << "\tBlocks : " << (int) totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(bs); - dec->write(bs); - cout << endl; + if (thdFcnFailure) + cerr << "There was a failure in the read thread." << endl; + bpp.destroyBPP(bs); + dec->write(bs); + cout << endl; } void doBatchOp_filt(OidOperation& OidOp) { - struct timespec ts1, ts2, diff; - JobStepAssociation injs, outjs; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - boost::shared_ptr sysCat = CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); + struct timespec ts1, ts2, diff; + JobStepAssociation injs, outjs; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + ThdFcn f1; + boost::shared_ptr sysCat = CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - ByteStream bs; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - uint32_t uniqueID; + ByteStream bs; + DBRM dbrm; + BRM::LBIDRange_v lbidRanges; + HWM_t hwm = 0; + LBIDRange_v::iterator it; + OID_t tmp; + uint32_t fbo; + uint32_t uniqueID; - BatchPrimitiveProcessorJL bpp; - uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - bpp.setSessionID(OidOp.SessionId()); - bpp.setStepID(OidOp.SessionId()); + BatchPrimitiveProcessorJL bpp; + uniqueID = dbrm.getUnique32(); + bpp.setUniqueID(uniqueID); + bpp.setSessionID(OidOp.SessionId()); + bpp.setStepID(OidOp.SessionId()); - pColScanStep scan(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, - OidOp.fSessionId, 0, OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, rm); + pColScanStep scan(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, OidOp.fSessionId, 0, OidOp.fSessionId, + OidOp.fSessionId, OidOp.fSessionId, rm); - while (OidOp.FilterCount() > 0) + while (OidOp.FilterCount() > 0) + { + int8_t cop; + int64_t value; + OidOp.deSerializeFilter(cop, value); + scan.addFilter(cop, value); + } + + bpp.addFilterStep(scan); + + pColStep step(injs, outjs, dec, sysCat, OidOp.fOid + 1, OidOp.fOid + 1, OidOp.fSessionId, 0, + OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, rm); + + while (OidOp.FilterCount() > 0) + { + int8_t cop; + int64_t value; + OidOp.deSerializeFilter(cop, value); + step.addFilter(cop, value); + } + + bpp.addFilterStep(step); + + execplan::CalpontSystemCatalog::ColType colType; + FilterStep filt(OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, colType); + filt.setBOP(OidOp.BOP()); + bpp.addFilterStep(filt); + + cout << "session number = " << OidOp.SessionId() << endl; + // dec->addSession(OidOp.SessionId()); + // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); + dec->addQueue(uniqueID); + f1.fSessionid = OidOp.SessionId(); + f1.uniqueID = uniqueID; + f1.fDec = dec; + thdFcnFailure = false; + int err = dbrm.lookup(OidOp.OID(), lbidRanges); + + if (err) + { + cerr << "doBatchOp_filt: BRM LBID range lookup failure (1)\n"; + throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); + } + + err = dbrm.getHWM(OidOp.OID(), hwm); + + if (err) + { + cerr << "doBatchOp_filt: BRM HWM lookup failure (2)" << endl; + throw runtime_error("doBatchOp_filt: BRM HWM lookup failure (2)"); + } + + f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); + + thread t1(f1); + + bpp.createBPP(bs); + dec->write(bs); + bs.restart(); + + uint32_t rangeSize = 0, totalBlks = 0; + clock_gettime(CLOCK_REALTIME, &ts1); + + // cout << "BPP scaning\n"; + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; + + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - scan.addFilter(cop, value); + cerr << "doBatchOp_filt: dbrm.lookup failed for lbid (3)" << lbid << endl; + abort(); } - bpp.addFilterStep(scan); + if (hwm < fbo) + continue; - pColStep step(injs, outjs, dec, sysCat, OidOp.fOid + 1, OidOp.fOid + 1, - OidOp.fSessionId, 0, OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, rm); - - while (OidOp.FilterCount() > 0) - { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - step.addFilter(cop, value); - } - - bpp.addFilterStep(step); - - execplan::CalpontSystemCatalog::ColType colType; - FilterStep filt(OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, colType); - filt.setBOP(OidOp.BOP()); - bpp.addFilterStep(filt); - - cout << "session number = " << OidOp.SessionId() << endl; -// dec->addSession(OidOp.SessionId()); -// dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - dec->addQueue(uniqueID); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - thdFcnFailure = false; - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - { - cerr << "doBatchOp_filt: BRM LBID range lookup failure (1)\n"; - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - } - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - { - cerr << "doBatchOp_filt: BRM HWM lookup failure (2)" << endl; - throw runtime_error("doBatchOp_filt: BRM HWM lookup failure (2)"); - } - - f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); - - thread t1(f1); - - bpp.createBPP(bs); + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + bpp.setLBID(lbid); + bpp.setCount(rangeSize / OidOp.fColType.colWidth + (rangeSize % OidOp.fColType.colWidth ? 1 : 0)); + bpp.runBPP(bs); dec->write(bs); + // cout << "sending the BPP\n"; + bpp.reset(); bs.restart(); + totalBlks += rangeSize; + } - uint32_t rangeSize = 0, totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); + t1.join(); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + float rate = 0; + cout << "doBatchOp_filt stats OID: " << OidOp.OID() << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) + << "s" + << "\tBlocks : " << (int)totalBlks; -// cout << "BPP scaning\n"; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; + rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\tBlks/sec " << rate << endl; - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "doBatchOp_filt: dbrm.lookup failed for lbid (3)" << lbid << endl; - abort(); - } + if (thdFcnFailure) + cerr << "There was a failure in the read thread." << endl; - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - bpp.setLBID(lbid); - bpp.setCount(rangeSize / OidOp.fColType.colWidth + (rangeSize % OidOp.fColType.colWidth ? 1 : 0)); - bpp.runBPP(bs); - dec->write(bs); -// cout << "sending the BPP\n"; - bpp.reset(); - bs.restart(); - totalBlks += rangeSize; - } - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "doBatchOp_filt stats OID: " << OidOp.OID() - << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tBlocks : " << (int) totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(bs); - dec->write(bs); - cout << endl; + bpp.destroyBPP(bs); + dec->write(bs); + cout << endl; } - void doBatchQueryOp(OperationList& OidOps) { - struct timespec ts1, ts2, diff; + struct timespec ts1, ts2, diff; - JobStepAssociation injs, outjs; - BatchPrimitiveProcessorJL bpp; - uint64_t rows = 0; - uint32_t blockTouched = 0; - DBRM dbrm; + JobStepAssociation injs, outjs; + BatchPrimitiveProcessorJL bpp; + uint64_t rows = 0; + uint32_t blockTouched = 0; + DBRM dbrm; - QryThdFcn f1(bpp, rows, blockTouched); + QryThdFcn f1(bpp, rows, blockTouched); - OperationList::iterator filterOp = OidOps.begin(); - uint32_t sessionId = (*filterOp)->SessionId(); - uint32_t uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - bpp.setSessionID(sessionId); - bpp.setStepID(sessionId); - cout << "session number = " << sessionId << endl; + OperationList::iterator filterOp = OidOps.begin(); + uint32_t sessionId = (*filterOp)->SessionId(); + uint32_t uniqueID = dbrm.getUnique32(); + bpp.setUniqueID(uniqueID); + bpp.setSessionID(sessionId); + bpp.setStepID(sessionId); + cout << "session number = " << sessionId << endl; + f1.fSessionid = sessionId; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + // dec->addSession(sessionId); + // dec->addStep(sessionId, sessionId); + dec->addQueue(uniqueID); - f1.fSessionid = sessionId; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); -// dec->addSession(sessionId); -// dec->addStep(sessionId, sessionId); - dec->addQueue(uniqueID); + f1.fDec = dec; + f1.uniqueID = uniqueID; + // boost::shared_ptr sysCat = + //execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - f1.fDec = dec; - f1.uniqueID = uniqueID; -// boost::shared_ptr sysCat = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); + // first column is made into the first scan filter step including filters + OID_t scanOid = (*filterOp)->fOid; + uint32_t scanWidth = (*filterOp)->ColumnWidth(); + uint32_t maxWidth = scanWidth; -//first column is made into the first scan filter step including filters - OID_t scanOid = (*filterOp)->fOid; - uint32_t scanWidth = (*filterOp)->ColumnWidth(); - uint32_t maxWidth = scanWidth; + uint32_t pid = getpid(); + pColScanStep scan(injs, outjs, dec, execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(pid), scanOid, + scanOid, sessionId, 0, sessionId, sessionId, sessionId, rm); - uint32_t pid = getpid(); - pColScanStep scan(injs, outjs, dec, execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(pid), scanOid, scanOid, sessionId, 0, sessionId, sessionId, sessionId, rm); + uint32_t filterCount = (*filterOp)->FilterCount(); - uint32_t filterCount = (*filterOp)->FilterCount(); + while ((*filterOp)->FilterCount() > 0) + { + int8_t cop; + int64_t value; + (*filterOp)->deSerializeFilter(cop, value); + scan.addFilter(cop, value); + } - while ((*filterOp)->FilterCount() > 0) + bpp.addFilterStep(scan); + + // Any other columns that are batch scans are added as filter steps, the rest as project steps. + // The last filter step is added as a passthru step into the project list. + + OperationList::iterator listend = OidOps.end(); + + for (OperationList::iterator op = OidOps.begin() + 1; op != listend; ++op) + { + pColStep step(injs, outjs, dec, execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(pid), + (*op)->fOid, (*op)->fOid, sessionId, 0, sessionId, sessionId, sessionId, rm); + + if ((*op)->OpType() == OidOperation::BATCHSCAN) { + filterCount += (*op)->FilterCount(); + + while ((*op)->FilterCount() > 0) + { int8_t cop; - int64_t value; - (*filterOp)->deSerializeFilter(cop, value); - scan.addFilter(cop, value); + int64_t value; + (*op)->deSerializeFilter(cop, value); + step.addFilter(cop, value); + } + + filterOp = op; + bpp.addFilterStep(step); } - - bpp.addFilterStep(scan); - -//Any other columns that are batch scans are added as filter steps, the rest as project steps. -//The last filter step is added as a passthru step into the project list. - - OperationList::iterator listend = OidOps.end(); - - for (OperationList::iterator op = OidOps.begin() + 1; op != listend; ++op) + else { - pColStep step(injs, outjs, dec, execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(pid), (*op)->fOid, (*op)->fOid, sessionId, 0, sessionId, sessionId, sessionId, rm); - - if ((*op)->OpType() == OidOperation::BATCHSCAN) - { - filterCount += (*op)->FilterCount(); - - while ((*op)->FilterCount() > 0) - { - int8_t cop; - int64_t value; - (*op)->deSerializeFilter(cop, value); - step.addFilter(cop, value); - } - - filterOp = op; - bpp.addFilterStep(step); - } - else - { - bpp.addProjectStep(step); - } - - if ( (*op)->ColumnWidth() > maxWidth) - maxWidth = (*op)->ColumnWidth(); + bpp.addProjectStep(step); } - PassThruStep pass(injs, outjs, dec, (*filterOp)->ColumnType(), (*filterOp)->fOid, (*filterOp)->fOid, - sessionId, 0, sessionId, sessionId, sessionId, false, rm); - bpp.addProjectStep(pass); + if ((*op)->ColumnWidth() > maxWidth) + maxWidth = (*op)->ColumnWidth(); + } - ByteStream bs; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; + PassThruStep pass(injs, outjs, dec, (*filterOp)->ColumnType(), (*filterOp)->fOid, (*filterOp)->fOid, + sessionId, 0, sessionId, sessionId, sessionId, false, rm); + bpp.addProjectStep(pass); - thdFcnFailure = false; - int err = dbrm.lookup(scanOid, lbidRanges); + ByteStream bs; + BRM::LBIDRange_v lbidRanges; + HWM_t hwm = 0; + LBIDRange_v::iterator it; + OID_t tmp; + uint32_t fbo; - if (err) + thdFcnFailure = false; + int err = dbrm.lookup(scanOid, lbidRanges); + + if (err) + { + cerr << "doQueryScan: BRM LBID range lookup failure (1)\n"; + throw runtime_error("doQueryScan: BRM LBID range lookup failure (1)"); + } + + err = dbrm.getHWM(scanOid, hwm); + + if (err) + { + cerr << "doQueryScan: BRM HWM lookup failure (3)" << endl; + throw runtime_error("doQueryScan: BRM HWM lookup failure (3)"); + } + + f1.fNumMsgs = hwm / scanWidth + (hwm % scanWidth ? 1 : 0); + + thread t1(f1); + uint32_t cnt = dbrm.getExtentSize() / maxWidth; + + bpp.createBPP(bs); + dec->write(bs); + bs.restart(); + uint32_t rangeSize = 0, totalBlks = 0; + clock_gettime(CLOCK_REALTIME, &ts1); + + // cout << "BPP scaning\n"; + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; + + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) { - cerr << "doQueryScan: BRM LBID range lookup failure (1)\n"; - throw runtime_error("doQueryScan: BRM LBID range lookup failure (1)"); + cerr << "doBatchQuery dbrm.lookup failed for lbid " << lbid << endl; + abort(); } - err = dbrm.getHWM(scanOid, hwm); + if (hwm < fbo) + continue; - if (err) + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + uint32_t totallbid = rangeSize / scanWidth + (0 < rangeSize % scanWidth ? 1 : 0); + + while (0 < totallbid) { - cerr << "doQueryScan: BRM HWM lookup failure (3)" << endl; - throw runtime_error("doQueryScan: BRM HWM lookup failure (3)"); + if (totallbid < cnt) + cnt = totallbid; + + bpp.setLBID(lbid); + bpp.setCount(cnt); + bpp.runBPP(bs); + dec->write(bs); + // cout << "sending the BPP with range cnt " << cnt << " lbid " << lbid << "\n"; + bpp.reset(); + bs.restart(); + lbid += cnt * scanWidth; + totallbid -= cnt; } - f1.fNumMsgs = hwm / scanWidth + (hwm % scanWidth ? 1 : 0); - - thread t1(f1); - uint32_t cnt = dbrm.getExtentSize() / maxWidth; - - bpp.createBPP(bs); - dec->write(bs); - bs.restart(); - uint32_t rangeSize = 0, totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); - -// cout << "BPP scaning\n"; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + for (OperationList::iterator op = OidOps.begin(); op != OidOps.end(); ++op) { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "doBatchQuery dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - uint32_t totallbid = rangeSize / scanWidth + (0 < rangeSize % scanWidth ? 1 : 0) ; - - while ( 0 < totallbid) - { - if (totallbid < cnt) cnt = totallbid; - - bpp.setLBID(lbid); - bpp.setCount(cnt); - bpp.runBPP(bs); - dec->write(bs); -// cout << "sending the BPP with range cnt " << cnt << " lbid " << lbid << "\n"; - bpp.reset(); - bs.restart(); - lbid += cnt * scanWidth; - totallbid -= cnt; - } - - for (OperationList::iterator op = OidOps.begin(); op != OidOps.end(); ++op) - { - totalBlks += (uint32_t)(rangeSize * (double)( (double)(*op)->ColumnWidth() / scanWidth)); - } + totalBlks += (uint32_t)(rangeSize * (double)((double)(*op)->ColumnWidth() / scanWidth)); } + } - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); + t1.join(); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "QueryScan stats - " << bpp.toString() - << "\tElapsed: " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tFilters: " << filterCount - << "\tBlocks : " << (int) totalBlks; + float rate = 0; + cout << "QueryScan stats - " << bpp.toString() + << "\tElapsed: " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" + << "\tFilters: " << filterCount << "\tBlocks : " << (int)totalBlks; - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - cout << "\tTouched Blocks: " << blockTouched; - rate = blockTouched / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tTouched Blks/sec " << rate << endl; + rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\tBlks/sec " << rate << endl; + cout << "\tTouched Blocks: " << blockTouched; + rate = blockTouched / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\tTouched Blks/sec " << rate << endl; - cout << "\tRows: " << rows; - rate = rows / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\t\tRows/sec " << fixed << setprecision(2) << rate << endl; + cout << "\tRows: " << rows; + rate = rows / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\t\tRows/sec " << fixed << setprecision(2) << rate << endl; + if (thdFcnFailure) + cerr << "There was a failure in the read thread." << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(bs); - dec->write(bs); - cout << endl; + bpp.destroyBPP(bs); + dec->write(bs); + cout << endl; } void doBatchOp_step(OidOperation& OidOp) { - struct timespec ts1, ts2, diff; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - uint32_t totalBlks = 0; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - JobStepAssociation injs, outjs; + struct timespec ts1, ts2, diff; + DBRM dbrm; + BRM::LBIDRange_v lbidRanges; + HWM_t hwm = 0; + LBIDRange_v::iterator it; + OID_t tmp; + uint32_t fbo; + uint32_t totalBlks = 0; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + ThdFcn f1; + JobStepAssociation injs, outjs; - boost::shared_ptr sysCat = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); + boost::shared_ptr sysCat = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - pColStep step(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, - OidOp.fSessionId, 0, OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, rm); + pColStep step(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, OidOp.fSessionId, 0, OidOp.fSessionId, + OidOp.fSessionId, OidOp.fSessionId, rm); - int32_t filters = OidOp.FilterCount(); + int32_t filters = OidOp.FilterCount(); - while (OidOp.FilterCount() > 0) + while (OidOp.FilterCount() > 0) + { + int8_t cop; + int64_t value; + OidOp.deSerializeFilter(cop, value); + step.addFilter(cop, value); + } + + BatchPrimitiveProcessorJL bpp; + ElementType et; + ByteStream obs; + uint32_t uniqueID = dbrm.getUnique32(); + bpp.setUniqueID(uniqueID); + bpp.setSessionID(OidOp.SessionId()); + bpp.setStepID(OidOp.SessionId()); + bpp.addFilterStep(step); + + // dec->addSession(OidOp.SessionId()); + // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); + dec->addQueue(uniqueID); + f1.fSessionid = OidOp.SessionId(); + f1.uniqueID = uniqueID; + f1.fDec = dec; + f1.fNumMsgs = 0; + thdFcnFailure = false; + ByteStream ridlist; + uint16_t ridCount = 0; // BLOCK_SIZE/OidOp.ColumnWidth(); + + for (uint16_t i = 0; i < ridCount; i++) + ridlist << i; + + int err = dbrm.lookup(OidOp.OID(), lbidRanges); + + if (err) + throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); + + err = dbrm.getHWM(OidOp.OID(), hwm); + + if (err) + throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); + + uint32_t rangeSize = 0; + f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); + thread t1(f1); + + totalBlks = 0; + bpp.createBPP(obs); + dec->write(obs); + obs.restart(); + + clock_gettime(CLOCK_REALTIME, &ts1); + + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + BRM::LBID_t lbid = (*it).start; + + if (dbrm.lookup(lbid, 0, false, tmp, fbo)) { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - step.addFilter(cop, value); + if (debug) + cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; + + abort(); } - BatchPrimitiveProcessorJL bpp; - ElementType et; - ByteStream obs; - uint32_t uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - bpp.setSessionID(OidOp.SessionId()); - bpp.setStepID(OidOp.SessionId()); - bpp.addFilterStep(step); + if (hwm < fbo) + continue; -// dec->addSession(OidOp.SessionId()); -// dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - dec->addQueue(uniqueID); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = 0; - thdFcnFailure = false; - ByteStream ridlist; - uint16_t ridCount = 0; // BLOCK_SIZE/OidOp.ColumnWidth(); + rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - for (uint16_t i = 0; i < ridCount; i++) - ridlist << i; - - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - - uint32_t rangeSize = 0; - f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); - thread t1(f1); - - totalBlks = 0; - bpp.createBPP(obs); - dec->write(obs); - obs.restart(); - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + for (unsigned i = 0; i < rangeSize; i++) { - BRM::LBID_t lbid = (*it).start; + /* insert all rids for this LBID */ + for (uint32_t j = 0; j < BLOCK_SIZE / OidOp.fColType.colWidth; ++j) + { + et.first = ((fbo + i) * BLOCK_SIZE / OidOp.fColType.colWidth) + j; + et.second = j; + bpp.addElementType(et); + } - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - if (debug) - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; + /* If on a logical block boundary, send the primitive */ + if (i % OidOp.fColType.colWidth == (unsigned)OidOp.fColType.colWidth - 1) + { + // cout << "serializing at extent offset " << i << endl; + bpp.runBPP(obs); + dec->write(obs); + bpp.reset(); + obs.restart(); + } - abort(); - } + if (debug && i + 1 == rangeSize) + cout << "colStep: " << i + 1 << "/" << rangeSize << " " << obs.length() << " lbid " << lbid + i + << endl; + } - if (hwm < fbo) - continue; + if (rangeSize % OidOp.fColType.colWidth) + { + // cout << "serializing last msg\n"; + bpp.runBPP(obs); + dec->write(obs); + bpp.reset(); + obs.restart(); + } - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); + totalBlks += rangeSize; + } // for - for (unsigned i = 0; i < rangeSize; i++) - { - /* insert all rids for this LBID */ - for (uint32_t j = 0; j < BLOCK_SIZE / OidOp.fColType.colWidth; ++j) - { - et.first = ((fbo + i) * BLOCK_SIZE / OidOp.fColType.colWidth) + j; - et.second = j; - bpp.addElementType(et); - } + t1.join(); + clock_gettime(CLOCK_REALTIME, &ts2); + timespec_sub(ts1, ts2, diff); + float rate = 0; + cout << "ColStep stats OID: " << OidOp.OID() << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" + << "\tFilters: " << filters << "\tBlocks : " << (int)totalBlks; - /* If on a logical block boundary, send the primitive */ - if (i % OidOp.fColType.colWidth == (unsigned)OidOp.fColType.colWidth - 1) - { -// cout << "serializing at extent offset " << i << endl; - bpp.runBPP(obs); - dec->write(obs); - bpp.reset(); - obs.restart(); - } + rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\tBlks/sec " << rate << endl; - if (debug && i + 1 == rangeSize) - cout - << "colStep: " << i + 1 << "/" - << rangeSize << " " << obs.length() - << " lbid " << lbid + i << endl; - } + if (thdFcnFailure) + cerr << "There was a failure in the read thread." << endl; - if (rangeSize % OidOp.fColType.colWidth) - { -// cout << "serializing last msg\n"; - bpp.runBPP(obs); - dec->write(obs); - bpp.reset(); - obs.restart(); - } - - totalBlks += rangeSize; - } // for - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "ColStep stats OID: " << OidOp.OID() - << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tFilters: " << filters - << "\tBlocks : " << (int) totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(obs); - dec->write(obs); - - cout << endl; + bpp.destroyBPP(obs); + dec->write(obs); + cout << endl; } // // void doListOp(const OID_t o = 0) { + DBRM dbrm; + BRM::LBIDRange_v lbidRanges; + LBIDRange_v::iterator it; + OID_t oid = 3000; + HWM_t hwm = 0; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - LBIDRange_v::iterator it; - OID_t oid = 3000; - HWM_t hwm = 0; + if (o != 0) + { + int err = dbrm.lookup(o, lbidRanges); - if (o != 0) + if (err) + throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); + + err = dbrm.getHWM(o, hwm); + + if (err) + throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); + + cout << "Object ID: " << o << " HWM: " << hwm << endl; + + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + cout << "\tStart: " << (*it).start << " sz: " << (*it).size << endl; + } + else + { + for (; oid < 100000; oid++) { - int err = dbrm.lookup(o, lbidRanges); + int err = dbrm.lookup(oid, lbidRanges); - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); + if (lbidRanges.size() == 0) + continue; - err = dbrm.getHWM(o, hwm); + if (err) + throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); + err = dbrm.getHWM(oid, hwm); - cout << "Object ID: " << o << " HWM: " << hwm << endl; + if (err) + throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - cout << "\tStart: " << (*it).start << " sz: " << (*it).size << endl; - } - else - { - for (; oid < 100000; oid++) - { + cout << "Object ID: " << oid << " HWM: " << hwm << endl; - int err = dbrm.lookup(oid, lbidRanges); + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + cout << "\tStart: " << (*it).start << " sz: " << (*it).size << endl; - if (lbidRanges.size() == 0) - continue; + hwm = 0; + lbidRanges.clear(); + } // for (; oid... + } // else - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(oid, hwm); - - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - - cout << "Object ID: " << oid << " HWM: " << hwm << endl; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - cout << "\tStart: " << (*it).start << " sz: " << (*it).size << endl; - - hwm = 0; - lbidRanges.clear(); - } // for (; oid... - } // else - -} // doListOp +} // doListOp // // do LoopBackOp void doLoopBack(const uint64_t loopcount) { + ByteStream lbMsg; + struct timespec ts1; + struct timespec ts2; + struct timespec diff; + uint32_t sessionid = getpid(); + DBRM dbrm; + ResourceManager* rm = ResourceManager::instance(); + DistributedEngineComm* dec = DistributedEngineComm::instance(rm); + ThdFcn f1; - ByteStream lbMsg; - struct timespec ts1; - struct timespec ts2; - struct timespec diff; - uint32_t sessionid = getpid(); - DBRM dbrm; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; + // dec->addSession(sessionid); + // dec->addStep(sessionid, sessionid); + uint32_t uniqueID = dbrm.getUnique32(); + dec->addQueue(uniqueID); + f1.fSessionid = sessionid; + f1.uniqueID = uniqueID; + f1.fDec = dec; + f1.fNumMsgs = loopcount; + thdFcnFailure = false; + thread t1(f1); + lbMsg = formatLoopBackMsg(sessionid, uniqueID); -// dec->addSession(sessionid); -// dec->addStep(sessionid, sessionid); - uint32_t uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); - f1.fSessionid = sessionid; - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = loopcount; - thdFcnFailure = false; - thread t1(f1); + cout << "Sending " << loopcount << " LOOPBACK requests" << endl; + clock_gettime(CLOCK_REALTIME, &ts1); + + for (uint64_t i = 0; i < loopcount; i++) + { lbMsg = formatLoopBackMsg(sessionid, uniqueID); + dec->write(lbMsg); + } - cout << "Sending " << loopcount << " LOOPBACK requests" << endl; - clock_gettime(CLOCK_REALTIME, &ts1); + clock_gettime(CLOCK_REALTIME, &ts2); + cout << loopcount << " LOOPBACK msgs sent" << endl; - for (uint64_t i = 0; i < loopcount; i++) - { - lbMsg = formatLoopBackMsg(sessionid, uniqueID); - dec->write(lbMsg); - } + t1.join(); + timespec_sub(ts1, ts2, diff); + cout << "\ttotal runtime: " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" << endl; - clock_gettime(CLOCK_REALTIME, &ts2); - cout << loopcount << " LOOPBACK msgs sent" << endl; + float rate = 0; + rate = loopcount / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); + cout << "\t" << rate << " rqsts/s" << endl; + dec->removeQueue(uniqueID); - t1.join(); - timespec_sub(ts1, ts2, diff); - cout << "\ttotal runtime: " << (diff.tv_sec - + (diff.tv_nsec / 1000000000.0)) << "s" << endl; +} // doLoopBack - float rate = 0; - rate = loopcount / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\t" << rate << " rqsts/s" << endl; - dec->removeQueue(uniqueID); - -} //doLoopBack - -} //namespace +} // namespace /*--------------------------------------------------------------------------- //Command line parameter definition @@ -1825,592 +1783,575 @@ void doLoopBack(const uint64_t loopcount) // --pm-profile -pmp turn on pm profiling -----------------------------------------------------------------------------*/ - - int main(int argc, char** argv) { + int64_t eq_val = 0; + int64_t gt_val = 0; + int64_t ge_val = 0; + int64_t lt_val = 0; + int64_t le_val = 0; + int64_t ne_val = 0; + int64_t bop_val = 0; + int64_t loopback_count = 100000; + int64_t scanOid = 0; + int64_t stepOid = 0; + bool list = false; + // OID_t listOid=0; + string oidString; + vector oidv; + int ch = 0; + bool concurrent_flag = false; + bool lbidtrace_flag = false; + bool pmprofile_flag = false; + bool query_flag = false; - int64_t eq_val = 0; - int64_t gt_val = 0; - int64_t ge_val = 0; - int64_t lt_val = 0; - int64_t le_val = 0; - int64_t ne_val = 0; - int64_t bop_val = 0; - int64_t loopback_count = 100000; - int64_t scanOid = 0; - int64_t stepOid = 0; - bool list = false; -//OID_t listOid=0; - string oidString; - vector oidv; - int ch = 0; - bool concurrent_flag = false; - bool lbidtrace_flag = false; - bool pmprofile_flag = false; - bool query_flag = false; + enum CLA_ENUM + { + OID = (int)0, + SCANOP = (int)1, + BLOCKOP = (int)2, + DEBUG = (int)3, + EQFILTER = (int)4, + GTFILTER = (int)5, + GEFILTER = (int)6, + LTFILTER = (int)7, + LEFILTER = (int)8, + NEFILTER = (int)9, + BOP = (int)10, + LISTOP = (int)11, + LOOPBACKOP = (int)12, + CONCURRENT = (int)13, + LBIDTRACE = (int)14, + PMPROFILE = (int)15, + INVALIDOP = (int)16, + BATCHSCANOP = (int)17, + BATCHSTEPOP = (int)18, + BATCHFILTOP = (int)19, + QUERYOP = (int)20 + }; - enum CLA_ENUM + /** + // longopt struct + struct option { + const char *name; + int has_arg; + int *flag; + int val; + }; + **/ + + static struct option long_options[] = { + // {const char *name, int has_arg, int *flag, int val}, + {"scan", required_argument, NULL, SCANOP}, + {"block", required_argument, NULL, BLOCKOP}, + {"debug", no_argument, NULL, DEBUG}, + {"equal", required_argument, NULL, EQFILTER}, + {"eq", required_argument, NULL, EQFILTER}, + {"greater-than", required_argument, NULL, GTFILTER}, + {"gt", required_argument, NULL, GTFILTER}, + {"greater-than-equal", required_argument, NULL, GEFILTER}, + {"ge", required_argument, NULL, GEFILTER}, + {"less-than", required_argument, NULL, LTFILTER}, + {"lt", required_argument, NULL, LTFILTER}, + {"less-than-equal", required_argument, NULL, LEFILTER}, + {"le", required_argument, NULL, LEFILTER}, + {"not-equal", required_argument, NULL, NEFILTER}, + {"ne", required_argument, NULL, NEFILTER}, + {"bop", optional_argument, NULL, BOP}, + {"list", no_argument, NULL, LISTOP}, + {"loopback", optional_argument, NULL, LOOPBACKOP}, + {"concurrent", no_argument, NULL, CONCURRENT}, + {"lbid-trace", no_argument, NULL, LBIDTRACE}, + {"lb", no_argument, NULL, LBIDTRACE}, + {"pm-prof", no_argument, NULL, PMPROFILE}, + {"pm-profile", no_argument, NULL, PMPROFILE}, + {"pmp", no_argument, NULL, PMPROFILE}, + {"batchscan", required_argument, NULL, BATCHSCANOP}, + {"batchstep", required_argument, NULL, BATCHSTEPOP}, + {"batchfilt", required_argument, NULL, BATCHFILTOP}, + {"queryop", no_argument, NULL, QUERYOP}, + {0, 0, 0, 0}}; + + OidOperation* currOp = NULL; + OperationList OpList; + + if (argc <= 1) + { + usage(); + } + + // process command line arguments + while ((ch = getopt_long_only(argc, argv, "B:Z:F:ds:t:lcqp:", long_options, NULL)) != -1) + { + pid_t pidId = getpid(); + + switch (ch) { - OID = (int)0, - SCANOP = (int)1, - BLOCKOP = (int)2, - DEBUG = (int)3, - EQFILTER = (int)4, - GTFILTER = (int)5, - GEFILTER = (int)6, - LTFILTER = (int)7, - LEFILTER = (int)8, - NEFILTER = (int)9, - BOP = (int)10, - LISTOP = (int)11, - LOOPBACKOP = (int)12, - CONCURRENT = (int)13, - LBIDTRACE = (int)14, - PMPROFILE = (int)15, - INVALIDOP = (int)16, - BATCHSCANOP = (int)17, - BATCHSTEPOP = (int)18, - BATCHFILTOP = (int)19, - QUERYOP = (int)20 + case SCANOP: + case 's': + if (optarg) + scanOid = getInt(optarg); - }; + // cout << "OPT=" << ch << " ARG " << scanOid << endl; + currOp = NULL; - /** - // longopt struct - struct option { - const char *name; - int has_arg; - int *flag; - int val; - }; - **/ - - static struct - option long_options[] = - { -// {const char *name, int has_arg, int *flag, int val}, - {"scan", required_argument, NULL, SCANOP }, - {"block", required_argument, NULL, BLOCKOP }, - {"debug", no_argument, NULL, DEBUG }, - {"equal", required_argument, NULL, EQFILTER }, - {"eq", required_argument, NULL, EQFILTER }, - {"greater-than", required_argument, NULL, GTFILTER }, - {"gt", required_argument, NULL, GTFILTER }, - {"greater-than-equal", required_argument, NULL, GEFILTER }, - {"ge", required_argument, NULL, GEFILTER }, - {"less-than", required_argument, NULL, LTFILTER }, - {"lt", required_argument, NULL, LTFILTER }, - {"less-than-equal", required_argument, NULL, LEFILTER }, - {"le", required_argument, NULL, LEFILTER }, - {"not-equal", required_argument, NULL, NEFILTER }, - {"ne", required_argument, NULL, NEFILTER }, - {"bop", optional_argument, NULL, BOP }, - {"list", no_argument, NULL, LISTOP }, - {"loopback", optional_argument, NULL, LOOPBACKOP }, - {"concurrent", no_argument, NULL, CONCURRENT }, - {"lbid-trace", no_argument, NULL, LBIDTRACE }, - {"lb", no_argument, NULL, LBIDTRACE }, - {"pm-prof", no_argument, NULL, PMPROFILE }, - {"pm-profile", no_argument, NULL, PMPROFILE }, - {"pmp", no_argument, NULL, PMPROFILE }, - {"batchscan", required_argument, NULL, BATCHSCANOP }, - {"batchstep", required_argument, NULL, BATCHSTEPOP }, - {"batchfilt", required_argument, NULL, BATCHFILTOP }, - {"queryop", no_argument, NULL, QUERYOP }, - {0, 0, 0, 0} - }; - - OidOperation* currOp = NULL; - OperationList OpList; - - if (argc <= 1) - { - usage(); - } - -// process command line arguments - while ( (ch = getopt_long_only(argc, argv, "B:Z:F:ds:t:lcqp:", long_options, NULL)) != -1 ) - { - - pid_t pidId = getpid(); - - switch (ch) + if (scanOid > 0) + currOp = new OidOperation(scanOid, OidOperation::SCAN, pidId); + else { - - case SCANOP: - case 's': - if (optarg) - scanOid = getInt(optarg); - - //cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::SCAN, pidId ); - else - { - cout << "PingProc: scan missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BATCHSCANOP: - case 'B': - if (optarg) - scanOid = getInt(optarg); - else - cout << "no optarg\n"; - - cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::BATCHSCAN, pidId ); - else - { - cout << "PingProc: batch scan missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BATCHSTEPOP: - case 'Z': - if (optarg) - scanOid = getInt(optarg); - else - cout << "no optarg\n"; - - cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::BATCHSTEP, pidId ); - else - { - cout << "PingProc: batch step missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BATCHFILTOP: - case 'F': - if (optarg) - scanOid = getInt(optarg); - else - cout << "no optarg\n"; - - cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::BATCHFILT, pidId ); - else - { - cout << "PingProc: batch filter missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BLOCKOP: - case 't': - if (optarg) - stepOid = getInt(optarg); - - //cout << "OPT=" << ch << " ARG " << stepOid << endl; - currOp = NULL; - - if (stepOid > 0) - currOp = new OidOperation(stepOid, OidOperation::BLOCK, pidId ); - else - { - cout << "PingProc: step missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << stepOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case DEBUG: - case 'd': - //cout << "OPT=" << ch << endl; - debug = true; - break; - - case EQFILTER: - if (optarg) - eq_val = getInt(optarg); - else - eq_val = 0; - - cout << "OPT=" << ch << " ARG=" << eq_val << endl; - - if (currOp) - currOp->addFilter(COMPARE_EQ, eq_val); - else - ; // TODO: error Processing - - break; - - case GTFILTER: - if (optarg) - gt_val = getInt(optarg); - else - gt_val = 0; - - //cout << "OPT=" << ch << " ARG=" << gt_val << endl; - if (currOp) - currOp->addFilter(COMPARE_GT, gt_val); - - break; - - case GEFILTER: - if (optarg) - ge_val = getInt(optarg); - else - ge_val = 0; - - //cout << "OPT=" << ch << " ARG=" << ge_val << endl; - if (currOp) - currOp->addFilter(COMPARE_GE, ge_val); - - break; - - case LTFILTER: - if (optarg) - lt_val = getInt(optarg); - else - lt_val = 0; - - //cout << "OPT=" << ch << " ARG=" << lt_val << endl; - if (currOp) - currOp->addFilter(COMPARE_LT, lt_val); - - break; - - case LEFILTER: - if (optarg) - le_val = getInt(optarg); - else - le_val = 0; - - //cout << "OPT=" << ch << " ARG=" << le_val << endl; - if (currOp) - currOp->addFilter(COMPARE_LE, le_val); - - break; - - case NEFILTER: - case 'n': - if (optarg) - ne_val = getInt(optarg); - else - ne_val = 0; - - //cout << "OPT=" << ch << " ARG=" << ne_val << endl; - if (currOp) - currOp->addFilter(COMPARE_NE, ne_val); - - break; - - case BOP: - case 'b': - if (optarg) - bop_val = getInt(optarg); - else - bop_val = 1; // assume AND - - //cout << "OPT=" << ch << " ARG=" << bop_val << endl; - if (currOp) - currOp->BOP(bop_val ? BOP_AND : BOP_OR); - - break; - - case LISTOP: - case 'l': - /** - if (optarg) - listOid=getInt(optarg); - else - listOid=0; - cout << "OPT=" << ch << " LISTOP " << listOid << endl; - **/ - list = true; - break; - - case LOOPBACKOP: - case 'p': - if (optarg) - loopback_count = getInt(optarg); - - //cout << "OPT=" << ch << " LOOPBACKOP " << loopback_count << endl; - currOp = NULL; - currOp = new OidOperation(0, OidOperation::LOOPBACK, pidId ); - OpList.push_back(currOp); - break; - - case 'c': - case CONCURRENT: - concurrent_flag = true; - //cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; - break; - - case LBIDTRACE: - lbidtrace_flag = true; - - //cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; - if (currOp) - currOp->setLbidTraceOn(); - - break; - - case PMPROFILE: - pmprofile_flag = true; - - //cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; - if (currOp) - currOp->setPMProfileOn(); - - break; - - case 'q': - case QUERYOP: - query_flag = true; -// cout << "OPT=" << ch << " QUERY FLAG " << query_flag << endl; - break; - - - case '?': - default: - cout << "optarg " << optarg << endl; - usage(); - + cout << "PingProc: scan missing or invalid OID parameter value" << endl; + break; } - if (list == true) - break; + if (currOp && currOp->isIntegralDataType()) + OpList.push_back(currOp); + else + { + cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; + delete currOp; + currOp = NULL; + } - } // while + break; -// if list is requested, print the listing and exit -// - vector BatchScanThreads; - vector BatchStepThreads; - vector BatchFiltThreads; - vector ColScanThreads; - vector ColStepThreads; - vector DictScanThreads; - vector DictSigThreads; - vector thrArray; + case BATCHSCANOP: + case 'B': + if (optarg) + scanOid = getInt(optarg); + else + cout << "no optarg\n"; - if (query_flag) + cout << "OPT=" << ch << " ARG " << scanOid << endl; + currOp = NULL; + + if (scanOid > 0) + currOp = new OidOperation(scanOid, OidOperation::BATCHSCAN, pidId); + else + { + cout << "PingProc: batch scan missing or invalid OID parameter value" << endl; + break; + } + + if (currOp && currOp->isIntegralDataType()) + OpList.push_back(currOp); + else + { + cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; + delete currOp; + currOp = NULL; + } + + break; + + case BATCHSTEPOP: + case 'Z': + if (optarg) + scanOid = getInt(optarg); + else + cout << "no optarg\n"; + + cout << "OPT=" << ch << " ARG " << scanOid << endl; + currOp = NULL; + + if (scanOid > 0) + currOp = new OidOperation(scanOid, OidOperation::BATCHSTEP, pidId); + else + { + cout << "PingProc: batch step missing or invalid OID parameter value" << endl; + break; + } + + if (currOp && currOp->isIntegralDataType()) + OpList.push_back(currOp); + else + { + cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; + delete currOp; + currOp = NULL; + } + + break; + + case BATCHFILTOP: + case 'F': + if (optarg) + scanOid = getInt(optarg); + else + cout << "no optarg\n"; + + cout << "OPT=" << ch << " ARG " << scanOid << endl; + currOp = NULL; + + if (scanOid > 0) + currOp = new OidOperation(scanOid, OidOperation::BATCHFILT, pidId); + else + { + cout << "PingProc: batch filter missing or invalid OID parameter value" << endl; + break; + } + + if (currOp && currOp->isIntegralDataType()) + OpList.push_back(currOp); + else + { + cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; + delete currOp; + currOp = NULL; + } + + break; + + case BLOCKOP: + case 't': + if (optarg) + stepOid = getInt(optarg); + + // cout << "OPT=" << ch << " ARG " << stepOid << endl; + currOp = NULL; + + if (stepOid > 0) + currOp = new OidOperation(stepOid, OidOperation::BLOCK, pidId); + else + { + cout << "PingProc: step missing or invalid OID parameter value" << endl; + break; + } + + if (currOp && currOp->isIntegralDataType()) + OpList.push_back(currOp); + else + { + cout << "PingProc cannot process this ColumnType-oid: " << stepOid << endl; + delete currOp; + currOp = NULL; + } + + break; + + case DEBUG: + case 'd': + // cout << "OPT=" << ch << endl; + debug = true; + break; + + case EQFILTER: + if (optarg) + eq_val = getInt(optarg); + else + eq_val = 0; + + cout << "OPT=" << ch << " ARG=" << eq_val << endl; + + if (currOp) + currOp->addFilter(COMPARE_EQ, eq_val); + else + ; // TODO: error Processing + + break; + + case GTFILTER: + if (optarg) + gt_val = getInt(optarg); + else + gt_val = 0; + + // cout << "OPT=" << ch << " ARG=" << gt_val << endl; + if (currOp) + currOp->addFilter(COMPARE_GT, gt_val); + + break; + + case GEFILTER: + if (optarg) + ge_val = getInt(optarg); + else + ge_val = 0; + + // cout << "OPT=" << ch << " ARG=" << ge_val << endl; + if (currOp) + currOp->addFilter(COMPARE_GE, ge_val); + + break; + + case LTFILTER: + if (optarg) + lt_val = getInt(optarg); + else + lt_val = 0; + + // cout << "OPT=" << ch << " ARG=" << lt_val << endl; + if (currOp) + currOp->addFilter(COMPARE_LT, lt_val); + + break; + + case LEFILTER: + if (optarg) + le_val = getInt(optarg); + else + le_val = 0; + + // cout << "OPT=" << ch << " ARG=" << le_val << endl; + if (currOp) + currOp->addFilter(COMPARE_LE, le_val); + + break; + + case NEFILTER: + case 'n': + if (optarg) + ne_val = getInt(optarg); + else + ne_val = 0; + + // cout << "OPT=" << ch << " ARG=" << ne_val << endl; + if (currOp) + currOp->addFilter(COMPARE_NE, ne_val); + + break; + + case BOP: + case 'b': + if (optarg) + bop_val = getInt(optarg); + else + bop_val = 1; // assume AND + + // cout << "OPT=" << ch << " ARG=" << bop_val << endl; + if (currOp) + currOp->BOP(bop_val ? BOP_AND : BOP_OR); + + break; + + case LISTOP: + case 'l': + /** + if (optarg) + listOid=getInt(optarg); + else + listOid=0; + cout << "OPT=" << ch << " LISTOP " << listOid << endl; + **/ + list = true; + break; + + case LOOPBACKOP: + case 'p': + if (optarg) + loopback_count = getInt(optarg); + + // cout << "OPT=" << ch << " LOOPBACKOP " << loopback_count << endl; + currOp = NULL; + currOp = new OidOperation(0, OidOperation::LOOPBACK, pidId); + OpList.push_back(currOp); + break; + + case 'c': + case CONCURRENT: + concurrent_flag = true; + // cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; + break; + + case LBIDTRACE: + lbidtrace_flag = true; + + // cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; + if (currOp) + currOp->setLbidTraceOn(); + + break; + + case PMPROFILE: + pmprofile_flag = true; + + // cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; + if (currOp) + currOp->setPMProfileOn(); + + break; + + case 'q': + case QUERYOP: + query_flag = true; + // cout << "OPT=" << ch << " QUERY FLAG " << query_flag << endl; + break; + + case '?': + default: cout << "optarg " << optarg << endl; usage(); + } + + if (list == true) + break; + + } // while + + // if list is requested, print the listing and exit + // + vector BatchScanThreads; + vector BatchStepThreads; + vector BatchFiltThreads; + vector ColScanThreads; + vector ColStepThreads; + vector DictScanThreads; + vector DictSigThreads; + vector thrArray; + + if (query_flag) + { + cout << "starting batch query thread\n"; + struct BatchQueryThr* qt = new struct BatchQueryThr(OpList); + thread* t1 = new thread(*qt); + + if (concurrent_flag) + thrArray.push_back(t1); + else + t1->join(); + } + else if (list) + { + doListOp(); + } + else + { + for (uint32_t i = 0; i < OpList.size(); i++) { - cout << "starting batch query thread\n"; - struct BatchQueryThr* qt = new struct BatchQueryThr(OpList); - thread* t1 = new thread(*qt); + if (OpList[i]->OpType() == OidOperation::LOOPBACK) + { + doLoopBack(loopback_count); + } + + else if (OpList[i]->OpType() == OidOperation::SCAN) + { + struct ColScanThr* cst = new struct ColScanThr(*OpList[i]); + ColScanThreads.push_back(cst); + thread* t1 = new thread(*cst); if (concurrent_flag) - thrArray.push_back(t1); + thrArray.push_back(t1); else - t1->join(); - } - else if (list) - { - doListOp(); - } - else - { - - for (uint32_t i = 0; i < OpList.size(); i++) { + t1->join(); + delete t1; + } + } - if (OpList[i]->OpType() == OidOperation::LOOPBACK) - { - doLoopBack(loopback_count); - } + else if (OpList[i]->OpType() == OidOperation::BLOCK) + { + struct ColStepThr* cst = new struct ColStepThr(*OpList[i]); + ColStepThreads.push_back(cst); + thread* t1 = new thread(*cst); - else if (OpList[i]->OpType() == OidOperation::SCAN) - { - struct ColScanThr* cst = new struct ColScanThr(*OpList[i]); - ColScanThreads.push_back(cst); - thread* t1 = new thread(*cst); + if (concurrent_flag) + thrArray.push_back(t1); + else + { + t1->join(); + delete t1; + } + } - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } + else if (OpList[i]->OpType() == OidOperation::BATCHSCAN) + { + cout << "starting batch scan thread\n"; + struct BatchScanThr* cst = new struct BatchScanThr(*OpList[i]); + BatchScanThreads.push_back(cst); + thread* t1 = new thread(*cst); - else if (OpList[i]->OpType() == OidOperation::BLOCK) - { - struct ColStepThr* cst = new struct ColStepThr(*OpList[i]); - ColStepThreads.push_back(cst); - thread* t1 = new thread(*cst); + if (concurrent_flag) + thrArray.push_back(t1); + else + { + t1->join(); + delete t1; + } + } - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } + else if (OpList[i]->OpType() == OidOperation::BATCHSTEP) + { + cout << "starting batch step thread\n"; + struct BatchStepThr* cst = new struct BatchStepThr(*OpList[i]); + BatchStepThreads.push_back(cst); + thread* t1 = new thread(*cst); - else if (OpList[i]->OpType() == OidOperation::BATCHSCAN) - { - cout << "starting batch scan thread\n"; - struct BatchScanThr* cst = new struct BatchScanThr(*OpList[i]); - BatchScanThreads.push_back(cst); - thread* t1 = new thread(*cst); + if (concurrent_flag) + thrArray.push_back(t1); + else + { + t1->join(); + delete t1; + } + } - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } + else if (OpList[i]->OpType() == OidOperation::BATCHFILT) + { + cout << "starting batch filt thread\n"; + struct BatchFiltThr* cst = new struct BatchFiltThr(*OpList[i]); + BatchFiltThreads.push_back(cst); + thread* t1 = new thread(*cst); - else if (OpList[i]->OpType() == OidOperation::BATCHSTEP) - { - cout << "starting batch step thread\n"; - struct BatchStepThr* cst = new struct BatchStepThr(*OpList[i]); - BatchStepThreads.push_back(cst); - thread* t1 = new thread(*cst); + if (concurrent_flag) + thrArray.push_back(t1); + else + { + t1->join(); + delete t1; + } - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } + i += 2; + } + } // for - else if (OpList[i]->OpType() == OidOperation::BATCHFILT) - { - cout << "starting batch filt thread\n"; - struct BatchFiltThr* cst = new struct BatchFiltThr(*OpList[i]); - BatchFiltThreads.push_back(cst); - thread* t1 = new thread(*cst); + } // else - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } + // join threads to main + for (uint32_t i = 0; i < thrArray.size(); i++) + thrArray[i]->join(); - i += 2; - } - } // for + // clean up + for (uint32_t i = 0; i < OpList.size(); i++) + delete OpList[i]; - } // else + OpList.clear(); + for (uint32_t i = 0; i < BatchScanThreads.size(); i++) + delete BatchScanThreads[i]; -// join threads to main - for (uint32_t i = 0; i < thrArray.size(); i++) - thrArray[i]->join(); + BatchScanThreads.clear(); -// clean up - for (uint32_t i = 0; i < OpList.size(); i++) - delete OpList[i]; + for (uint32_t i = 0; i < BatchStepThreads.size(); i++) + delete BatchStepThreads[i]; - OpList.clear(); + BatchStepThreads.clear(); - for (uint32_t i = 0; i < BatchScanThreads.size(); i++) - delete BatchScanThreads[i]; + for (uint32_t i = 0; i < BatchFiltThreads.size(); i++) + delete BatchFiltThreads[i]; - BatchScanThreads.clear(); + BatchFiltThreads.clear(); - for (uint32_t i = 0; i < BatchStepThreads.size(); i++) - delete BatchStepThreads[i]; + for (uint32_t i = 0; i < ColScanThreads.size(); i++) + delete ColScanThreads[i]; - BatchStepThreads.clear(); + ColScanThreads.clear(); - for (uint32_t i = 0; i < BatchFiltThreads.size(); i++) - delete BatchFiltThreads[i]; + for (uint32_t i = 0; i < ColStepThreads.size(); i++) + delete ColStepThreads[i]; - BatchFiltThreads.clear(); + ColStepThreads.clear(); - for (uint32_t i = 0; i < ColScanThreads.size(); i++) - delete ColScanThreads[i]; + for (uint32_t i = 0; i < DictScanThreads.size(); i++) + delete DictScanThreads[i]; - ColScanThreads.clear(); + DictScanThreads.clear(); - for (uint32_t i = 0; i < ColStepThreads.size(); i++) - delete ColStepThreads[i]; + for (uint32_t i = 0; i < DictSigThreads.size(); i++) + delete DictSigThreads[i]; - ColStepThreads.clear(); + DictSigThreads.clear(); - for (uint32_t i = 0; i < DictScanThreads.size(); i++) - delete DictScanThreads[i]; + for (uint32_t i = 0; i < thrArray.size(); i++) + delete thrArray[i]; - DictScanThreads.clear(); - - for (uint32_t i = 0; i < DictSigThreads.size(); i++) - delete DictSigThreads[i]; - - DictSigThreads.clear(); - - for (uint32_t i = 0; i < thrArray.size(); i++) - delete thrArray[i]; - - thrArray.clear(); - -} //main() + thrArray.clear(); +} // main() diff --git a/tools/qfe/bison-win.cpp b/tools/qfe/bison-win.cpp index 2c1e94a73..6e33a8378 100644 --- a/tools/qfe/bison-win.cpp +++ b/tools/qfe/bison-win.cpp @@ -64,14 +64,13 @@ #define YYLSP_NEEDED 0 /* Substitute the variable and function names. */ -#define yyparse qfeparse -#define yylex qfelex -#define yyerror qfeerror -#define yylval qfelval -#define yychar qfechar -#define yydebug qfedebug -#define yynerrs qfenerrs - +#define yyparse qfeparse +#define yylex qfelex +#define yyerror qfeerror +#define yylval qfelval +#define yychar qfechar +#define yydebug qfedebug +#define yynerrs qfenerrs /* Copy the first part of user declarations. */ @@ -104,87 +103,77 @@ class ReturnedColumn; #include "cseputils.h" - - /* Line 189 of yacc.c */ #line 111 "bison-win.cpp" /* Enabling traces. */ #ifndef YYDEBUG -# define YYDEBUG 0 +#define YYDEBUG 0 #endif /* Enabling verbose error messages. */ #ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 +#undef YYERROR_VERBOSE +#define YYERROR_VERBOSE 1 #else -# define YYERROR_VERBOSE 0 +#define YYERROR_VERBOSE 0 #endif /* Enabling the token table. */ #ifndef YYTOKEN_TABLE -# define YYTOKEN_TABLE 0 +#define YYTOKEN_TABLE 0 #endif - /* Tokens. */ #ifndef YYTOKENTYPE -# define YYTOKENTYPE +#define YYTOKENTYPE /* Put the tokens into the symbol table, so that GDB and other debuggers know about them. */ enum yytokentype { - CHAR_CONST = 258, - RELOP = 259, - LOGICOP = 260, - QFEP_SELECT = 261, - QFEP_FROM = 262, - QFEP_WHERE = 263, - GROUPBY = 264, - OBJNAME = 265, - INT_CONST = 266, - LIMIT = 267, - ORDERBY = 268, - ASC = 269, - DESC = 270, - AS = 271, - FUNC = 272 + CHAR_CONST = 258, + RELOP = 259, + LOGICOP = 260, + QFEP_SELECT = 261, + QFEP_FROM = 262, + QFEP_WHERE = 263, + GROUPBY = 264, + OBJNAME = 265, + INT_CONST = 266, + LIMIT = 267, + ORDERBY = 268, + ASC = 269, + DESC = 270, + AS = 271, + FUNC = 272 }; #endif - - -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED typedef union YYSTYPE { - - /* Line 214 of yacc.c */ + /* Line 214 of yacc.c */ #line 31 "qfeparser.ypp" - execplan::ReturnedColumn* rcp; - std::string* cp; - std::pair* cvp; + execplan::ReturnedColumn* rcp; + std::string* cp; + std::pair* cvp; - - - /* Line 214 of yacc.c */ + /* Line 214 of yacc.c */ #line 172 "bison-win.cpp" } YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 +#define YYSTYPE_IS_TRIVIAL 1 +#define yystype YYSTYPE /* obsolescent; will be withdrawn */ +#define YYSTYPE_IS_DECLARED 1 #endif - /* Copy the second part of user declarations. */ - /* Line 264 of yacc.c */ #line 184 "bison-win.cpp" #ifdef short -# undef short +#undef short #endif #ifdef YYTYPE_UINT8 @@ -195,8 +184,7 @@ typedef unsigned char yytype_uint8; #ifdef YYTYPE_INT8 typedef YYTYPE_INT8 yytype_int8; -#elif (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) +#elif (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) typedef signed char yytype_int8; #else typedef short int yytype_int8; @@ -215,450 +203,379 @@ typedef short int yytype_int16; #endif #ifndef YYSIZE_T -# ifdef __SIZE_TYPE__ -# define YYSIZE_T __SIZE_TYPE__ -# elif defined size_t -# define YYSIZE_T size_t -# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# define YYSIZE_T size_t -# else -# define YYSIZE_T unsigned int -# endif +#ifdef __SIZE_TYPE__ +#define YYSIZE_T __SIZE_TYPE__ +#elif defined size_t +#define YYSIZE_T size_t +#elif !defined YYSIZE_T && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +#include /* INFRINGES ON USER NAME SPACE */ +#define YYSIZE_T size_t +#else +#define YYSIZE_T unsigned int +#endif #endif -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) +#define YYSIZE_MAXIMUM ((YYSIZE_T)-1) #ifndef YY_ -# if YYENABLE_NLS -# if ENABLE_NLS -# include /* INFRINGES ON USER NAME SPACE */ -# define YY_(msgid) dgettext ("bison-runtime", msgid) -# endif -# endif -# ifndef YY_ -# define YY_(msgid) msgid -# endif +#if YYENABLE_NLS +#if ENABLE_NLS +#include /* INFRINGES ON USER NAME SPACE */ +#define YY_(msgid) dgettext("bison-runtime", msgid) +#endif +#endif +#ifndef YY_ +#define YY_(msgid) msgid +#endif #endif /* Suppress unused-variable warnings by "using" E. */ -#if ! defined lint || defined __GNUC__ -# define YYUSE(e) ((void) (e)) +#if !defined lint || defined __GNUC__ +#define YYUSE(e) ((void)(e)) #else -# define YYUSE(e) /* empty */ +#define YYUSE(e) /* empty */ #endif /* Identity function, used to suppress warnings about constant conditions. */ #ifndef lint -# define YYID(n) (n) +#define YYID(n) (n) #else -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static int -YYID (int yyi) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static int YYID(int yyi) #else -static int -YYID (yyi) -int yyi; +static int YYID(yyi) int yyi; #endif { - return yyi; + return yyi; } #endif -#if ! defined yyoverflow || YYERROR_VERBOSE +#if !defined yyoverflow || YYERROR_VERBOSE /* The parser invokes alloca or malloc; define the necessary symbols. */ -# ifdef YYSTACK_USE_ALLOCA -# if YYSTACK_USE_ALLOCA -# ifdef __GNUC__ -# define YYSTACK_ALLOC __builtin_alloca -# elif defined __BUILTIN_VA_ARG_INCR -# include /* INFRINGES ON USER NAME SPACE */ -# elif defined _AIX -# define YYSTACK_ALLOC __alloca -# elif defined _MSC_VER -# include /* INFRINGES ON USER NAME SPACE */ -# define alloca _alloca -# else -# define YYSTACK_ALLOC alloca -# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# endif -# endif -# endif +#ifdef YYSTACK_USE_ALLOCA +#if YYSTACK_USE_ALLOCA +#ifdef __GNUC__ +#define YYSTACK_ALLOC __builtin_alloca +#elif defined __BUILTIN_VA_ARG_INCR +#include /* INFRINGES ON USER NAME SPACE */ +#elif defined _AIX +#define YYSTACK_ALLOC __alloca +#elif defined _MSC_VER +#include /* INFRINGES ON USER NAME SPACE */ +#define alloca _alloca +#else +#define YYSTACK_ALLOC alloca +#if !defined _ALLOCA_H && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +#include /* INFRINGES ON USER NAME SPACE */ +#ifndef _STDLIB_H +#define _STDLIB_H 1 +#endif +#endif +#endif +#endif +#endif -# ifdef YYSTACK_ALLOC +#ifdef YYSTACK_ALLOC /* Pacify GCC's `empty if-body' warning. */ -# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0)) -# ifndef YYSTACK_ALLOC_MAXIMUM +#define YYSTACK_FREE(Ptr) \ + do \ + { /* empty */ \ + ; \ + } while (YYID(0)) +#ifndef YYSTACK_ALLOC_MAXIMUM /* The OS might guarantee only one guard page at the bottom of the stack, and a page size can be as small as 4096 bytes. So we cannot safely invoke alloca (N) if N exceeds 4096. Use a slightly smaller number to allow for a few compiler-allocated temporary stack slots. */ -# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ -# endif -# else -# define YYSTACK_ALLOC YYMALLOC -# define YYSTACK_FREE YYFREE -# ifndef YYSTACK_ALLOC_MAXIMUM -# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM -# endif -# if (defined __cplusplus && ! defined _STDLIB_H \ - && ! ((defined YYMALLOC || defined malloc) \ - && (defined YYFREE || defined free))) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# ifndef YYMALLOC -# define YYMALLOC malloc -# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void* malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# ifndef YYFREE -# define YYFREE free -# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void free (void*); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# endif +#define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +#endif +#else +#define YYSTACK_ALLOC YYMALLOC +#define YYSTACK_FREE YYFREE +#ifndef YYSTACK_ALLOC_MAXIMUM +#define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +#endif +#if (defined __cplusplus && !defined _STDLIB_H && \ + !((defined YYMALLOC || defined malloc) && (defined YYFREE || defined free))) +#include /* INFRINGES ON USER NAME SPACE */ +#ifndef _STDLIB_H +#define _STDLIB_H 1 +#endif +#endif +#ifndef YYMALLOC +#define YYMALLOC malloc +#if !defined malloc && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +void* malloc(YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +#endif +#endif +#ifndef YYFREE +#define YYFREE free +#if !defined free && !defined _STDLIB_H && \ + (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +void free(void*); /* INFRINGES ON USER NAME SPACE */ +#endif +#endif +#endif #endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - -#if (! defined yyoverflow \ - && (! defined __cplusplus \ - || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) +#if (!defined yyoverflow && (!defined __cplusplus || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) /* A type that is properly aligned for any stack member. */ union yyalloc { - yytype_int16 yyss_alloc; - YYSTYPE yyvs_alloc; + yytype_int16 yyss_alloc; + YYSTYPE yyvs_alloc; }; /* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) +#define YYSTACK_GAP_MAXIMUM (sizeof(union yyalloc) - 1) /* The size of an array large to enough to hold all stacks, each with N elements. */ -# define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ - + YYSTACK_GAP_MAXIMUM) +#define YYSTACK_BYTES(N) ((N) * (sizeof(yytype_int16) + sizeof(YYSTYPE)) + YYSTACK_GAP_MAXIMUM) /* Copy COUNT objects from FROM to TO. The source and destination do not overlap. */ -# ifndef YYCOPY -# if defined __GNUC__ && 1 < __GNUC__ -# define YYCOPY(To, From, Count) \ - __builtin_memcpy (To, From, (Count) * sizeof (*(From))) -# else -# define YYCOPY(To, From, Count) \ - do \ - { \ - YYSIZE_T yyi; \ - for (yyi = 0; yyi < (Count); yyi++) \ - (To)[yyi] = (From)[yyi]; \ - } \ - while (YYID (0)) -# endif -# endif +#ifndef YYCOPY +#if defined __GNUC__ && 1 < __GNUC__ +#define YYCOPY(To, From, Count) __builtin_memcpy(To, From, (Count) * sizeof(*(From))) +#else +#define YYCOPY(To, From, Count) \ + do \ + { \ + YYSIZE_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (To)[yyi] = (From)[yyi]; \ + } while (YYID(0)) +#endif +#endif /* Relocate STACK from its old location to the new one. The local variables YYSIZE and YYSTACKSIZE give the old and new number of elements in the stack, and YYPTR gives the new location of the stack. Advance YYPTR to a properly aligned location for the next stack. */ -# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ - do \ - { \ - YYSIZE_T yynewbytes; \ - YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ - Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ - } \ - while (YYID (0)) +#define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYSIZE_T yynewbytes; \ + YYCOPY(&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * sizeof(*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / sizeof(*yyptr); \ + } while (YYID(0)) #endif /* YYFINAL -- State number of the termination state. */ -#define YYFINAL 12 +#define YYFINAL 12 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 60 +#define YYLAST 60 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 24 +#define YYNTOKENS 24 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 22 +#define YYNNTS 22 /* YYNRULES -- Number of rules. */ -#define YYNRULES 41 +#define YYNRULES 41 /* YYNRULES -- Number of states. */ -#define YYNSTATES 67 +#define YYNSTATES 67 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ -#define YYUNDEFTOK 2 -#define YYMAXUTOK 272 +#define YYUNDEFTOK 2 +#define YYMAXUTOK 272 -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) +#define YYTRANSLATE(YYX) ((unsigned int)(YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) /* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ -static const yytype_uint8 yytranslate[] = -{ - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 20, 21, 2, 2, 19, 2, 22, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 23, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17 -}; +static const yytype_uint8 yytranslate[] = { + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 20, 21, 2, 2, 19, 2, 22, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 23, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; #if YYDEBUG /* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in YYRHS. */ -static const yytype_uint8 yyprhs[] = -{ - 0, 0, 3, 13, 15, 19, 21, 23, 28, 31, - 32, 35, 38, 39, 41, 43, 45, 48, 49, 52, - 56, 62, 64, 68, 74, 76, 80, 82, 84, 85, - 88, 90, 94, 96, 98, 99, 103, 104, 106, 108, - 109, 112 -}; +static const yytype_uint8 yyprhs[] = {0, 0, 3, 13, 15, 19, 21, 23, 28, 31, 32, 35, 38, 39, + 41, 43, 45, 48, 49, 52, 56, 62, 64, 68, 74, 76, 80, 82, + 84, 85, 88, 90, 94, 96, 98, 99, 103, 104, 106, 108, 109, 112}; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ -static const yytype_int8 yyrhs[] = -{ - 25, 0, -1, 6, 26, 7, 32, 35, 40, 43, - 45, 18, -1, 29, -1, 26, 19, 29, -1, 37, - -1, 28, -1, 17, 20, 27, 21, -1, 27, 30, - -1, -1, 31, 10, -1, 31, 3, -1, -1, 16, - -1, 34, -1, 37, -1, 33, 30, -1, -1, 8, - 36, -1, 37, 4, 39, -1, 36, 5, 37, 4, - 39, -1, 38, -1, 38, 22, 38, -1, 38, 22, - 38, 22, 38, -1, 10, -1, 23, 10, 23, -1, - 3, -1, 11, -1, -1, 9, 41, -1, 42, -1, - 41, 19, 42, -1, 37, -1, 11, -1, -1, 13, - 41, 44, -1, -1, 14, -1, 15, -1, -1, 12, - 11, -1, 12, 11, 19, 11, -1 -}; +static const yytype_int8 yyrhs[] = { + 25, 0, -1, 6, 26, 7, 32, 35, 40, 43, 45, 18, -1, 29, -1, 26, 19, 29, -1, 37, -1, 28, -1, 17, + 20, 27, 21, -1, 27, 30, -1, -1, 31, 10, -1, 31, 3, -1, -1, 16, -1, 34, -1, 37, -1, 33, 30, -1, + -1, 8, 36, -1, 37, 4, 39, -1, 36, 5, 37, 4, 39, -1, 38, -1, 38, 22, 38, -1, 38, 22, 38, 22, + 38, -1, 10, -1, 23, 10, 23, -1, 3, -1, 11, -1, -1, 9, 41, -1, 42, -1, 41, 19, 42, -1, 37, -1, + 11, -1, -1, 13, 41, 44, -1, -1, 14, -1, 15, -1, -1, 12, 11, -1, 12, 11, 19, 11, -1}; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ -static const yytype_uint16 yyrline[] = -{ - 0, 55, 55, 63, 69, 77, 87, 90, 98, 102, - 103, 108, 116, 117, 120, 124, 144, 147, 148, 151, - 163, 179, 190, 202, 217, 218, 224, 230, 238, 239, - 243, 244, 248, 249, 256, 257, 261, 262, 263, 267, - 268, 274 -}; +static const yytype_uint16 yyrline[] = {0, 55, 55, 63, 69, 77, 87, 90, 98, 102, 103, 108, 116, 117, + 120, 124, 144, 147, 148, 151, 163, 179, 190, 202, 217, 218, 224, 230, + 238, 239, 243, 244, 248, 249, 256, 257, 261, 262, 263, 267, 268, 274}; #endif #if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. First, the terminals, then, starting at YYNTOKENS, nonterminals. */ -static const char* const yytname[] = -{ - "$end", "error", "$undefined", "CHAR_CONST", "RELOP", "LOGICOP", - "QFEP_SELECT", "QFEP_FROM", "QFEP_WHERE", "GROUPBY", "OBJNAME", - "INT_CONST", "LIMIT", "ORDERBY", "ASC", "DESC", "AS", "FUNC", "';'", - "','", "'('", "')'", "'.'", "'`'", "$accept", "statement", - "select_column_list", "select_column_name", "func_col", - "aliased_column_name", "opt_alias", "opt_as", "table_spec", "table_name", - "aliased_table_name", "opt_where_clause", "filter_predicate_list", - "obj_name", "obj_name_aux", "constant", "opt_groupby_clause", - "groupby_items_list", "groupby_item", "opt_orderby_clause", - "opt_direction", "opt_limit_clause", 0 -}; +static const char* const yytname[] = {"$end", + "error", + "$undefined", + "CHAR_CONST", + "RELOP", + "LOGICOP", + "QFEP_SELECT", + "QFEP_FROM", + "QFEP_WHERE", + "GROUPBY", + "OBJNAME", + "INT_CONST", + "LIMIT", + "ORDERBY", + "ASC", + "DESC", + "AS", + "FUNC", + "';'", + "','", + "'('", + "')'", + "'.'", + "'`'", + "$accept", + "statement", + "select_column_list", + "select_column_name", + "func_col", + "aliased_column_name", + "opt_alias", + "opt_as", + "table_spec", + "table_name", + "aliased_table_name", + "opt_where_clause", + "filter_predicate_list", + "obj_name", + "obj_name_aux", + "constant", + "opt_groupby_clause", + "groupby_items_list", + "groupby_item", + "opt_orderby_clause", + "opt_direction", + "opt_limit_clause", + 0}; #endif -# ifdef YYPRINT +#ifdef YYPRINT /* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to token YYLEX-NUM. */ -static const yytype_uint16 yytoknum[] = -{ - 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 59, 44, - 40, 41, 46, 96 -}; -# endif +static const yytype_uint16 yytoknum[] = {0, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 59, 44, 40, 41, 46, 96}; +#endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = -{ - 0, 24, 25, 26, 26, 27, 27, 28, 29, 30, - 30, 30, 31, 31, 32, 33, 34, 35, 35, 36, - 36, 37, 37, 37, 38, 38, 39, 39, 40, 40, - 41, 41, 42, 42, 43, 43, 44, 44, 44, 45, - 45, 45 -}; +static const yytype_uint8 yyr1[] = {0, 24, 25, 26, 26, 27, 27, 28, 29, 30, 30, 30, 31, 31, + 32, 33, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 39, 39, + 40, 40, 41, 41, 42, 42, 43, 43, 44, 44, 44, 45, 45, 45}; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = -{ - 0, 2, 9, 1, 3, 1, 1, 4, 2, 0, - 2, 2, 0, 1, 1, 1, 2, 0, 2, 3, - 5, 1, 3, 5, 1, 3, 1, 1, 0, 2, - 1, 3, 1, 1, 0, 3, 0, 1, 1, 0, - 2, 4 -}; +static const yytype_uint8 yyr2[] = {0, 2, 9, 1, 3, 1, 1, 4, 2, 0, 2, 2, 0, 1, 1, 1, 2, 0, 2, 3, 5, + 1, 3, 5, 1, 3, 1, 1, 0, 2, 1, 3, 1, 1, 0, 3, 0, 1, 1, 0, 2, 4}; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state STATE-NUM when YYTABLE doesn't specify something else to do. Zero means the default is an error. */ -static const yytype_uint8 yydefact[] = -{ - 0, 0, 0, 24, 0, 0, 0, 9, 6, 3, - 5, 21, 1, 0, 0, 0, 0, 13, 8, 0, - 0, 0, 25, 17, 9, 14, 15, 4, 11, 10, - 22, 7, 0, 28, 16, 0, 18, 0, 0, 34, - 23, 0, 0, 33, 32, 29, 30, 0, 39, 0, - 26, 27, 19, 0, 36, 0, 0, 0, 31, 37, - 38, 35, 40, 2, 20, 0, 41 -}; +static const yytype_uint8 yydefact[] = {0, 0, 0, 24, 0, 0, 0, 9, 6, 3, 5, 21, 1, 0, 0, 0, 0, + 13, 8, 0, 0, 0, 25, 17, 9, 14, 15, 4, 11, 10, 22, 7, 0, 28, + 16, 0, 18, 0, 0, 34, 23, 0, 0, 33, 32, 29, 30, 0, 39, 0, 26, + 27, 19, 0, 36, 0, 0, 0, 31, 37, 38, 35, 40, 2, 20, 0, 41}; /* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int8 yydefgoto[] = -{ - -1, 2, 6, 7, 8, 9, 18, 19, 23, 24, - 25, 33, 36, 10, 11, 52, 39, 45, 46, 48, - 61, 56 - }; +static const yytype_int8 yydefgoto[] = {-1, 2, 6, 7, 8, 9, 18, 19, 23, 24, 25, + 33, 36, 10, 11, 52, 39, 45, 46, 48, 61, 56}; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ #define YYPACT_NINF -20 -static const yytype_int8 yypact[] = -{ - 8, -8, 4, -20, 5, 17, 3, 2, -20, -20, - -20, 9, -20, -8, 1, -2, -8, -20, -20, 10, - -2, 13, -20, 22, 2, -20, -20, -20, -20, -20, - 15, -20, -2, 26, -20, -2, 31, 35, -4, 27, - -20, -2, 0, -20, -20, 23, -20, -4, 29, 39, - -20, -20, -20, -4, 14, 33, 28, 0, -20, -20, - -20, -20, 30, -20, -20, 34, -20 -}; +static const yytype_int8 yypact[] = {8, -8, 4, -20, 5, 17, 3, 2, -20, -20, -20, 9, -20, -8, + 1, -2, -8, -20, -20, 10, -2, 13, -20, 22, 2, -20, -20, -20, + -20, -20, 15, -20, -2, 26, -20, -2, 31, 35, -4, 27, -20, -2, + 0, -20, -20, 23, -20, -4, 29, 39, -20, -20, -20, -4, 14, 33, + 28, 0, -20, -20, -20, -20, 30, -20, -20, 34, -20}; /* YYPGOTO[NTERM-NUM]. */ -static const yytype_int8 yypgoto[] = -{ - -20, -20, -20, 37, -20, 32, 36, -20, -20, -20, - -20, -20, -20, -15, -19, -10, -20, 6, -1, -20, - -20, -20 - }; +static const yytype_int8 yypgoto[] = {-20, -20, -20, 37, -20, 32, 36, -20, -20, -20, -20, + -20, -20, -15, -19, -10, -20, 6, -1, -20, -20, -20}; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ #define YYTABLE_NINF -13 -static const yytype_int8 yytable[] = -{ - 26, 30, 3, 50, 12, -12, 3, 43, 3, 4, - 15, 51, -12, 28, 1, 5, 40, 37, 17, 5, - 29, 5, 16, 44, 22, 13, 49, 14, 59, 60, - 32, 20, 44, 53, 31, 38, 41, 35, 44, 42, - 47, 55, 53, 57, 62, 66, 63, 64, 27, 65, - 21, 0, 58, 54, 0, 0, 0, 0, 0, 0, - 34 -}; +static const yytype_int8 yytable[] = {26, 30, 3, 50, 12, -12, 3, 43, 3, 4, 15, 51, -12, 28, 1, 5, + 40, 37, 17, 5, 29, 5, 16, 44, 22, 13, 49, 14, 59, 60, 32, 20, + 44, 53, 31, 38, 41, 35, 44, 42, 47, 55, 53, 57, 62, 66, 63, 64, + 27, 65, 21, 0, 58, 54, 0, 0, 0, 0, 0, 0, 34}; -static const yytype_int8 yycheck[] = -{ - 15, 20, 10, 3, 0, 3, 10, 11, 10, 17, - 7, 11, 10, 3, 6, 23, 35, 32, 16, 23, - 10, 23, 19, 38, 23, 20, 41, 10, 14, 15, - 8, 22, 47, 19, 21, 9, 5, 22, 53, 4, - 13, 12, 19, 4, 11, 11, 18, 57, 16, 19, - 13, -1, 53, 47, -1, -1, -1, -1, -1, -1, - 24 -}; +static const yytype_int8 yycheck[] = {15, 20, 10, 3, 0, 3, 10, 11, 10, 17, 7, 11, 10, 3, 6, 23, + 35, 32, 16, 23, 10, 23, 19, 38, 23, 20, 41, 10, 14, 15, 8, 22, + 47, 19, 21, 9, 5, 22, 53, 4, 13, 12, 19, 4, 11, 11, 18, 57, + 16, 19, 13, -1, 53, 47, -1, -1, -1, -1, -1, -1, 24}; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = -{ - 0, 6, 25, 10, 17, 23, 26, 27, 28, 29, - 37, 38, 0, 20, 10, 7, 19, 16, 30, 31, - 22, 27, 23, 32, 33, 34, 37, 29, 3, 10, - 38, 21, 8, 35, 30, 22, 36, 37, 9, 40, - 38, 5, 4, 11, 37, 41, 42, 13, 43, 37, - 3, 11, 39, 19, 41, 12, 45, 4, 42, 14, - 15, 44, 11, 18, 39, 19, 11 -}; +static const yytype_uint8 yystos[] = {0, 6, 25, 10, 17, 23, 26, 27, 28, 29, 37, 38, 0, 20, 10, 7, 19, + 16, 30, 31, 22, 27, 23, 32, 33, 34, 37, 29, 3, 10, 38, 21, 8, 35, + 30, 22, 36, 37, 9, 40, 38, 5, 4, 11, 37, 41, 42, 13, 43, 37, 3, + 11, 39, 19, 41, 12, 45, 4, 42, 14, 15, 44, 11, 18, 39, 19, 11}; -#define yyerrok (yyerrstatus = 0) -#define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 - -#define YYACCEPT goto yyacceptlab -#define YYABORT goto yyabortlab -#define YYERROR goto yyerrorlab +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) +#define YYEMPTY (-2) +#define YYEOF 0 +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab /* Like YYERROR except do call yyerror. This remains here temporarily to ease the transition to the new meaning of YYERROR, for GCC. Once GCC version 2 has supplanted version 1, this can go. */ -#define YYFAIL goto yyerrlab +#define YYFAIL goto yyerrlab -#define YYRECOVERING() (!!yyerrstatus) +#define YYRECOVERING() (!!yyerrstatus) -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY && yylen == 1) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - yytoken = YYTRANSLATE (yychar); \ - YYPOPSTACK (1); \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (YYID (0)) - - -#define YYTERROR 1 -#define YYERRCODE 256 +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY && yylen == 1) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + yytoken = YYTRANSLATE(yychar); \ + YYPOPSTACK(1); \ + goto yybackup; \ + } \ + else \ + { \ + yyerror(YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (YYID(0)) +#define YYTERROR 1 +#define YYERRCODE 256 /* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. If N is 0, then set CURRENT to the empty location which ends @@ -666,136 +583,120 @@ while (YYID (0)) #define YYRHSLOC(Rhs, K) ((Rhs)[K]) #ifndef YYLLOC_DEFAULT -# define YYLLOC_DEFAULT(Current, Rhs, N) \ - do \ - if (YYID (N)) \ - { \ - (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ - (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ - (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ - (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ - } \ - else \ - { \ - (Current).first_line = (Current).last_line = \ - YYRHSLOC (Rhs, 0).last_line; \ - (Current).first_column = (Current).last_column = \ - YYRHSLOC (Rhs, 0).last_column; \ - } \ - while (YYID (0)) +#define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (YYID(N)) \ + { \ + (Current).first_line = YYRHSLOC(Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC(Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC(Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC(Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = YYRHSLOC(Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = YYRHSLOC(Rhs, 0).last_column; \ + } \ + while (YYID(0)) #endif - /* YY_LOCATION_PRINT -- Print the location on the stream. This macro was not mandated originally: define only if we know we won't break user code: when these are the locations we know. */ #ifndef YY_LOCATION_PRINT -# if YYLTYPE_IS_TRIVIAL -# define YY_LOCATION_PRINT(File, Loc) \ - fprintf (File, "%d.%d-%d.%d", \ - (Loc).first_line, (Loc).first_column, \ - (Loc).last_line, (Loc).last_column) -# else -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -# endif +#if YYLTYPE_IS_TRIVIAL +#define YY_LOCATION_PRINT(File, Loc) \ + fprintf(File, "%d.%d-%d.%d", (Loc).first_line, (Loc).first_column, (Loc).last_line, (Loc).last_column) +#else +#define YY_LOCATION_PRINT(File, Loc) ((void)0) +#endif #endif - /* YYLEX -- calling `yylex' with the right arguments. */ #ifdef YYLEX_PARAM -# define YYLEX yylex (YYLEX_PARAM) +#define YYLEX yylex(YYLEX_PARAM) #else -# define YYLEX yylex () +#define YYLEX yylex() #endif /* Enable debugging if requested. */ #if YYDEBUG -# ifndef YYFPRINTF -# include /* INFRINGES ON USER NAME SPACE */ -# define YYFPRINTF fprintf -# endif +#ifndef YYFPRINTF +#include /* INFRINGES ON USER NAME SPACE */ +#define YYFPRINTF fprintf +#endif -# define YYDPRINTF(Args) \ -do { \ - if (yydebug) \ - YYFPRINTF Args; \ -} while (YYID (0)) - -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ -do { \ - if (yydebug) \ - { \ - YYFPRINTF (stderr, "%s ", Title); \ - yy_symbol_print (stderr, \ - Type, Value); \ - YYFPRINTF (stderr, "\n"); \ - } \ -} while (YYID (0)) +#define YYDPRINTF(Args) \ + do \ + { \ + if (yydebug) \ + YYFPRINTF Args; \ + } while (YYID(0)) +#define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ + do \ + { \ + if (yydebug) \ + { \ + YYFPRINTF(stderr, "%s ", Title); \ + yy_symbol_print(stderr, Type, Value); \ + YYFPRINTF(stderr, "\n"); \ + } \ + } while (YYID(0)) /*--------------------------------. | Print this symbol on YYOUTPUT. | `--------------------------------*/ /*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_value_print (FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_symbol_value_print(FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) #else -static void -yy_symbol_value_print (yyoutput, yytype, yyvaluep) -FILE* yyoutput; +static void yy_symbol_value_print(yyoutput, yytype, yyvaluep) FILE* yyoutput; int yytype; YYSTYPE const* const yyvaluep; #endif { - if (!yyvaluep) - return; + if (!yyvaluep) + return; -# ifdef YYPRINT +#ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); + if (yytype < YYNTOKENS) + YYPRINT(yyoutput, yytoknum[yytype], *yyvaluep); -# else - YYUSE (yyoutput); -# endif +#else + YYUSE(yyoutput); +#endif - switch (yytype) - { - default: - break; - } + switch (yytype) + { + default: break; + } } - /*--------------------------------. | Print this symbol on YYOUTPUT. | `--------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_print (FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_symbol_print(FILE* yyoutput, int yytype, YYSTYPE const* const yyvaluep) #else -static void -yy_symbol_print (yyoutput, yytype, yyvaluep) -FILE* yyoutput; +static void yy_symbol_print(yyoutput, yytype, yyvaluep) FILE* yyoutput; int yytype; YYSTYPE const* const yyvaluep; #endif { - if (yytype < YYNTOKENS) - YYFPRINTF (yyoutput, "token %s (", yytname[yytype]); - else - YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]); + if (yytype < YYNTOKENS) + YYFPRINTF(yyoutput, "token %s (", yytname[yytype]); + else + YYFPRINTF(yyoutput, "nterm %s (", yytname[yytype]); - yy_symbol_value_print (yyoutput, yytype, yyvaluep); - YYFPRINTF (yyoutput, ")"); + yy_symbol_value_print(yyoutput, yytype, yyvaluep); + YYFPRINTF(yyoutput, ")"); } /*------------------------------------------------------------------. @@ -803,87 +704,76 @@ YYSTYPE const* const yyvaluep; | TOP (included). | `------------------------------------------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_stack_print (yytype_int16* yybottom, yytype_int16* yytop) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_stack_print(yytype_int16* yybottom, yytype_int16* yytop) #else -static void -yy_stack_print (yybottom, yytop) -yytype_int16* yybottom; +static void yy_stack_print(yybottom, yytop) yytype_int16* yybottom; yytype_int16* yytop; #endif { - YYFPRINTF (stderr, "Stack now"); + YYFPRINTF(stderr, "Stack now"); - for (; yybottom <= yytop; yybottom++) - { - int yybot = *yybottom; - YYFPRINTF (stderr, " %d", yybot); - } + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF(stderr, " %d", yybot); + } - YYFPRINTF (stderr, "\n"); + YYFPRINTF(stderr, "\n"); } -# define YY_STACK_PRINT(Bottom, Top) \ -do { \ - if (yydebug) \ - yy_stack_print ((Bottom), (Top)); \ -} while (YYID (0)) - +#define YY_STACK_PRINT(Bottom, Top) \ + do \ + { \ + if (yydebug) \ + yy_stack_print((Bottom), (Top)); \ + } while (YYID(0)) /*------------------------------------------------. | Report that the YYRULE is going to be reduced. | `------------------------------------------------*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_reduce_print (YYSTYPE* yyvsp, int yyrule) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yy_reduce_print(YYSTYPE* yyvsp, int yyrule) #else -static void -yy_reduce_print (yyvsp, yyrule) -YYSTYPE* yyvsp; +static void yy_reduce_print(yyvsp, yyrule) YYSTYPE* yyvsp; int yyrule; #endif { - int yynrhs = yyr2[yyrule]; - int yyi; - unsigned long int yylno = yyrline[yyrule]; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", - yyrule - 1, yylno); + int yynrhs = yyr2[yyrule]; + int yyi; + unsigned long int yylno = yyrline[yyrule]; + YYFPRINTF(stderr, "Reducing stack by rule %d (line %lu):\n", yyrule - 1, yylno); - /* The symbols being reduced. */ - for (yyi = 0; yyi < yynrhs; yyi++) - { - YYFPRINTF (stderr, " $%d = ", yyi + 1); - yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi], - &(yyvsp[(yyi + 1) - (yynrhs)]) - ); - YYFPRINTF (stderr, "\n"); - } + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF(stderr, " $%d = ", yyi + 1); + yy_symbol_print(stderr, yyrhs[yyprhs[yyrule] + yyi], &(yyvsp[(yyi + 1) - (yynrhs)])); + YYFPRINTF(stderr, "\n"); + } } -# define YY_REDUCE_PRINT(Rule) \ -do { \ - if (yydebug) \ - yy_reduce_print (yyvsp, Rule); \ -} while (YYID (0)) +#define YY_REDUCE_PRINT(Rule) \ + do \ + { \ + if (yydebug) \ + yy_reduce_print(yyvsp, Rule); \ + } while (YYID(0)) /* Nonzero means print parse trace. It is left uninitialized so that multiple parsers can coexist. */ int yydebug; #else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) -# define YY_STACK_PRINT(Bottom, Top) -# define YY_REDUCE_PRINT(Rule) +#define YYDPRINTF(Args) +#define YY_SYMBOL_PRINT(Title, Type, Value, Location) +#define YY_STACK_PRINT(Bottom, Top) +#define YY_REDUCE_PRINT(Rule) #endif /* !YYDEBUG */ - /* YYINITDEPTH -- initial size of the parser's stacks. */ -#ifndef YYINITDEPTH -# define YYINITDEPTH 200 +#ifndef YYINITDEPTH +#define YYINITDEPTH 200 #endif /* YYMAXDEPTH -- maximum size the stacks can grow to (effective only @@ -894,67 +784,57 @@ int yydebug; evaluated with infinite-precision integer arithmetic. */ #ifndef YYMAXDEPTH -# define YYMAXDEPTH 10000 +#define YYMAXDEPTH 10000 #endif - - #if YYERROR_VERBOSE -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else -/* Return the length of YYSTR. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static YYSIZE_T -yystrlen (const char* yystr) +#ifndef yystrlen +#if defined __GLIBC__ && defined _STRING_H +#define yystrlen strlen #else -static YYSIZE_T -yystrlen (yystr) -const char* yystr; +/* Return the length of YYSTR. */ +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static YYSIZE_T yystrlen(const char* yystr) +#else +static YYSIZE_T yystrlen(yystr) const char* yystr; #endif { - YYSIZE_T yylen; + YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; + for (yylen = 0; yystr[yylen]; yylen++) + continue; - return yylen; + return yylen; } -# endif -# endif +#endif +#endif -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else +#ifndef yystpcpy +#if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +#define yystpcpy stpcpy +#else /* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in YYDEST. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static char* -yystpcpy (char* yydest, const char* yysrc) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static char* yystpcpy(char* yydest, const char* yysrc) #else -static char* -yystpcpy (yydest, yysrc) -char* yydest; +static char* yystpcpy(yydest, yysrc) char* yydest; const char* yysrc; #endif { - char* yyd = yydest; - const char* yys = yysrc; + char* yyd = yydest; + const char* yys = yysrc; - while ((*yyd++ = *yys++) != '\0') - continue; + while ((*yyd++ = *yys++) != '\0') + continue; - return yyd - 1; + return yyd - 1; } -# endif -# endif +#endif +#endif -# ifndef yytnamerr +#ifndef yytnamerr /* Copy to YYRES the contents of YYSTR after stripping away unnecessary quotes and backslashes, so that it's suitable for yyerror. The heuristic is that double-quoting is unnecessary unless the string @@ -962,50 +842,47 @@ const char* yysrc; backslash-backslash). YYSTR is taken from yytname. If YYRES is null, do not copy; instead, return the length of what the result would have been. */ -static YYSIZE_T -yytnamerr (char* yyres, const char* yystr) +static YYSIZE_T yytnamerr(char* yyres, const char* yystr) { - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const* yyp = yystr; + if (*yystr == '"') + { + YYSIZE_T yyn = 0; + char const* yyp = yystr; - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': goto do_not_strip_quotes; - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; + /* Fall through. */ + default: + if (yyres) + yyres[yyn] = *yyp; - yyn++; - break; + yyn++; + break; - case '"': - if (yyres) - yyres[yyn] = '\0'; + case '"': + if (yyres) + yyres[yyn] = '\0'; - return yyn; - } + return yyn; + } -do_not_strip_quotes: - ; - } + do_not_strip_quotes:; + } - if (! yyres) - return yystrlen (yystr); + if (!yyres) + return yystrlen(yystr); - return yystpcpy (yyres, yystr) - yyres; + return yystpcpy(yyres, yystr) - yyres; } -# endif +#endif /* Copy into YYRESULT an error message about the unexpected token YYCHAR while in state YYSTATE. Return the number of bytes copied, @@ -1014,25 +891,27 @@ do_not_strip_quotes: copied. As a special case, return 0 if an ordinary "syntax error" message will do. Return YYSIZE_MAXIMUM if overflow occurs during size calculation. */ -static YYSIZE_T -yysyntax_error (char* yyresult, int yystate, int yychar) +static YYSIZE_T yysyntax_error(char* yyresult, int yystate, int yychar) { - int yyn = yypact[yystate]; + int yyn = yypact[yystate]; - if (! (YYPACT_NINF < yyn && yyn <= YYLAST)) - return 0; - else + if (!(YYPACT_NINF < yyn && yyn <= YYLAST)) + return 0; + else + { + int yytype = YYTRANSLATE(yychar); + YYSIZE_T yysize0 = yytnamerr(0, yytname[yytype]); + YYSIZE_T yysize = yysize0; + YYSIZE_T yysize1; + int yysize_overflow = 0; + enum { - int yytype = YYTRANSLATE (yychar); - YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]); - YYSIZE_T yysize = yysize0; - YYSIZE_T yysize1; - int yysize_overflow = 0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - char const* yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - int yyx; + YYERROR_VERBOSE_ARGS_MAXIMUM = 5 + }; + char const* yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; + int yyx; -# if 0 +#if 0 /* This is so xgettext sees the translatable formats that are constructed on the fly. */ YY_("syntax error, unexpected %s"); @@ -1040,277 +919,268 @@ yysyntax_error (char* yyresult, int yystate, int yychar) YY_("syntax error, unexpected %s, expecting %s or %s"); YY_("syntax error, unexpected %s, expecting %s or %s or %s"); YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); -# endif - char* yyfmt; - char const* yyf; - static char const yyunexpected[] = "syntax error, unexpected %s"; - static char const yyexpecting[] = ", expecting %s"; - static char const yyor[] = " or %s"; - char yyformat[sizeof yyunexpected - + sizeof yyexpecting - 1 - + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) - * (sizeof yyor - 1))]; - char const* yyprefix = yyexpecting; +#endif + char* yyfmt; + char const* yyf; + static char const yyunexpected[] = "syntax error, unexpected %s"; + static char const yyexpecting[] = ", expecting %s"; + static char const yyor[] = " or %s"; + char yyformat[sizeof yyunexpected + sizeof yyexpecting - 1 + + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) * (sizeof yyor - 1))]; + char const* yyprefix = yyexpecting; - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. */ - int yyxbegin = yyn < 0 ? -yyn : 0; + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. */ + int yyxbegin = yyn < 0 ? -yyn : 0; - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yycount = 1; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yycount = 1; - yyarg[0] = yytname[yytype]; - yyfmt = yystpcpy (yyformat, yyunexpected); + yyarg[0] = yytname[yytype]; + yyfmt = yystpcpy(yyformat, yyunexpected); - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - yyformat[sizeof yyunexpected - 1] = '\0'; - break; - } - - yyarg[yycount++] = yytname[yyx]; - yysize1 = yysize + yytnamerr (0, yytname[yyx]); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - yyfmt = yystpcpy (yyfmt, yyprefix); - yyprefix = yyor; - } - - yyf = YY_(yyformat); - yysize1 = yysize + yystrlen (yyf); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - - if (yysize_overflow) - return YYSIZE_MAXIMUM; - - if (yyresult) + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) + { + if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) { - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - char* yyp = yyresult; - int yyi = 0; - - while ((*yyp = *yyf) != '\0') - { - if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) - { - yyp += yytnamerr (yyp, yyarg[yyi++]); - yyf += 2; - } - else - { - yyp++; - yyf++; - } - } + yycount = 1; + yysize = yysize0; + yyformat[sizeof yyunexpected - 1] = '\0'; + break; } - return yysize; + yyarg[yycount++] = yytname[yyx]; + yysize1 = yysize + yytnamerr(0, yytname[yyx]); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + yyfmt = yystpcpy(yyfmt, yyprefix); + yyprefix = yyor; + } + + yyf = YY_(yyformat); + yysize1 = yysize + yystrlen(yyf); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + + if (yysize_overflow) + return YYSIZE_MAXIMUM; + + if (yyresult) + { + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + char* yyp = yyresult; + int yyi = 0; + + while ((*yyp = *yyf) != '\0') + { + if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) + { + yyp += yytnamerr(yyp, yyarg[yyi++]); + yyf += 2; + } + else + { + yyp++; + yyf++; + } + } } + + return yysize; + } } #endif /* YYERROR_VERBOSE */ - /*-----------------------------------------------. | Release the memory associated to this symbol. | `-----------------------------------------------*/ /*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yydestruct (const char* yymsg, int yytype, YYSTYPE* yyvaluep) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +static void yydestruct(const char* yymsg, int yytype, YYSTYPE* yyvaluep) #else -static void -yydestruct (yymsg, yytype, yyvaluep) -const char* yymsg; +static void yydestruct(yymsg, yytype, yyvaluep) const char* yymsg; int yytype; YYSTYPE* yyvaluep; #endif { - YYUSE (yyvaluep); + YYUSE(yyvaluep); - if (!yymsg) - yymsg = "Deleting"; + if (!yymsg) + yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + YY_SYMBOL_PRINT(yymsg, yytype, yyvaluep, yylocationp); - switch (yytype) + switch (yytype) + { + case 3: /* "CHAR_CONST" */ + + /* Line 1000 of yacc.c */ +#line 49 "qfeparser.ypp" { - case 3: /* "CHAR_CONST" */ + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ -#line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; - - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1138 "bison-win.cpp" - break; - case 4: /* "RELOP" */ + break; + case 4: /* "RELOP" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; + { + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1147 "bison-win.cpp" - break; - case 5: /* "LOGICOP" */ + break; + case 5: /* "LOGICOP" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; + { + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1156 "bison-win.cpp" - break; - case 10: /* "OBJNAME" */ + break; + case 10: /* "OBJNAME" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; + { + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1165 "bison-win.cpp" - break; - case 11: /* "INT_CONST" */ + break; + case 11: /* "INT_CONST" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; + { + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1174 "bison-win.cpp" - break; - case 16: /* "AS" */ + break; + case 16: /* "AS" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; + { + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1183 "bison-win.cpp" - break; - case 17: /* "FUNC" */ + break; + case 17: /* "FUNC" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 49 "qfeparser.ypp" - { - delete (yyvaluep->cp); - }; + { + delete (yyvaluep->cp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1192 "bison-win.cpp" - break; - case 27: /* "select_column_name" */ + break; + case 27: /* "select_column_name" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 50 "qfeparser.ypp" - { - delete (yyvaluep->rcp); - }; + { + delete (yyvaluep->rcp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1201 "bison-win.cpp" - break; - case 28: /* "func_col" */ + break; + case 28: /* "func_col" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 50 "qfeparser.ypp" - { - delete (yyvaluep->rcp); - }; + { + delete (yyvaluep->rcp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1210 "bison-win.cpp" - break; - case 29: /* "aliased_column_name" */ + break; + case 29: /* "aliased_column_name" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 50 "qfeparser.ypp" - { - delete (yyvaluep->rcp); - }; + { + delete (yyvaluep->rcp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1219 "bison-win.cpp" - break; - case 37: /* "obj_name" */ + break; + case 37: /* "obj_name" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 50 "qfeparser.ypp" - { - delete (yyvaluep->rcp); - }; + { + delete (yyvaluep->rcp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1228 "bison-win.cpp" - break; - case 39: /* "constant" */ + break; + case 39: /* "constant" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 51 "qfeparser.ypp" - { - delete (yyvaluep->cvp); - }; + { + delete (yyvaluep->cvp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1237 "bison-win.cpp" - break; - case 42: /* "groupby_item" */ + break; + case 42: /* "groupby_item" */ - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 50 "qfeparser.ypp" - { - delete (yyvaluep->rcp); - }; + { + delete (yyvaluep->rcp); + }; - /* Line 1000 of yacc.c */ + /* Line 1000 of yacc.c */ #line 1246 "bison-win.cpp" - break; + break; - default: - break; - } + default: break; + } } /* Prevent warnings from -Wmissing-prototypes. */ #ifdef YYPARSE_PARAM #if defined __STDC__ || defined __cplusplus -int yyparse (void* YYPARSE_PARAM); +int yyparse(void* YYPARSE_PARAM); #else -int yyparse (); +int yyparse(); #endif #else /* ! YYPARSE_PARAM */ #if defined __STDC__ || defined __cplusplus -int yyparse (void); +int yyparse(void); #else -int yyparse (); +int yyparse(); #endif #endif /* ! YYPARSE_PARAM */ - /* The lookahead symbol. */ int yychar; @@ -1320,869 +1190,843 @@ YYSTYPE yylval; /* Number of syntax errors so far. */ int yynerrs; - - /*-------------------------. | yyparse or yypush_parse. | `-------------------------*/ #ifdef YYPARSE_PARAM -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void* YYPARSE_PARAM) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +int yyparse(void* YYPARSE_PARAM) #else -int -yyparse (YYPARSE_PARAM) -void* YYPARSE_PARAM; +int yyparse(YYPARSE_PARAM) void* YYPARSE_PARAM; #endif #else /* ! YYPARSE_PARAM */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void) +#if (defined __STDC__ || defined __C99__FUNC__ || defined __cplusplus || defined _MSC_VER) +int yyparse(void) #else -int -yyparse () +int yyparse() #endif #endif { + int yystate; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus; + /* The stacks and their tools: + `yyss': related to states. + `yyvs': related to semantic values. - int yystate; - /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; + Refer to the stacks thru separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ - /* The stacks and their tools: - `yyss': related to states. - `yyvs': related to semantic values. + /* The state stack. */ + yytype_int16 yyssa[YYINITDEPTH]; + yytype_int16* yyss; + yytype_int16* yyssp; - Refer to the stacks thru separate pointers, to allow yyoverflow - to reallocate them elsewhere. */ + /* The semantic value stack. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE* yyvs; + YYSTYPE* yyvsp; - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16* yyss; - yytype_int16* yyssp; + YYSIZE_T yystacksize; - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE* yyvs; - YYSTYPE* yyvsp; - - YYSIZE_T yystacksize; - - int yyn; - int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken; - /* The variables used to return semantic value and location from the - action routines. */ - YYSTYPE yyval; + int yyn; + int yyresult; + /* Lookahead token as an internal (translated) token number. */ + int yytoken; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; #if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char* yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char* yymsg = yymsgbuf; + YYSIZE_T yymsg_alloc = sizeof yymsgbuf; #endif -#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) - /* The number of symbols on the RHS of the reduced rule. - Keep to zero when no symbol should be popped. */ - int yylen = 0; + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; - yytoken = 0; - yyss = yyssa; - yyvs = yyvsa; - yystacksize = YYINITDEPTH; + yytoken = 0; + yyss = yyssa; + yyvs = yyvsa; + yystacksize = YYINITDEPTH; - YYDPRINTF ((stderr, "Starting parse\n")); + YYDPRINTF((stderr, "Starting parse\n")); - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; - yychar = YYEMPTY; /* Cause a token to be read. */ + yystate = 0; + yyerrstatus = 0; + yynerrs = 0; + yychar = YYEMPTY; /* Cause a token to be read. */ - /* Initialize stack pointers. - Waste one element of value and location stack - so that they stay on the same level as the state stack. - The wasted elements are never initialized. */ - yyssp = yyss; - yyvsp = yyvs; + /* Initialize stack pointers. + Waste one element of value and location stack + so that they stay on the same level as the state stack. + The wasted elements are never initialized. */ + yyssp = yyss; + yyvsp = yyvs; - goto yysetstate; + goto yysetstate; - /*------------------------------------------------------------. - | yynewstate -- Push a new state, which is found in yystate. | - `------------------------------------------------------------*/ + /*------------------------------------------------------------. + | yynewstate -- Push a new state, which is found in yystate. | + `------------------------------------------------------------*/ yynewstate: - /* In all cases, when you get here, the value and location stacks - have just been pushed. So pushing a state here evens the stacks. */ - yyssp++; + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; yysetstate: - *yyssp = yystate; + *yyssp = yystate; - if (yyss + yystacksize - 1 <= yyssp) - { - /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; + if (yyss + yystacksize - 1 <= yyssp) + { + /* Get the current used size of the three stacks, in elements. */ + YYSIZE_T yysize = yyssp - yyss + 1; #ifdef yyoverflow - { - /* Give user a chance to reallocate the stack. Use copies of - these so that the &'s don't force the real ones into - memory. */ - YYSTYPE* yyvs1 = yyvs; - yytype_int16* yyss1 = yyss; + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + YYSTYPE* yyvs1 = yyvs; + yytype_int16* yyss1 = yyss; - /* Each stack pointer address is followed by the size of the - data in use in that stack, in bytes. This used to be a - conditional around just the two extra args, but that might - be undefined if yyoverflow is a macro. */ - yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), - &yystacksize); + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow(YY_("memory exhausted"), &yyss1, yysize * sizeof(*yyssp), &yyvs1, yysize * sizeof(*yyvsp), + &yystacksize); - yyss = yyss1; - yyvs = yyvs1; - } + yyss = yyss1; + yyvs = yyvs1; + } #else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE +#ifndef YYSTACK_RELOCATE + goto yyexhaustedlab; +#else + + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + goto yyexhaustedlab; + + yystacksize *= 2; + + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yytype_int16* yyss1 = yyss; + union yyalloc* yyptr = (union yyalloc*)YYSTACK_ALLOC(YYSTACK_BYTES(yystacksize)); + + if (!yyptr) goto yyexhaustedlab; -# else - /* Extend the stack our own way. */ - if (YYMAXDEPTH <= yystacksize) - goto yyexhaustedlab; + YYSTACK_RELOCATE(yyss_alloc, yyss); + YYSTACK_RELOCATE(yyvs_alloc, yyvs); +#undef YYSTACK_RELOCATE - yystacksize *= 2; - - if (YYMAXDEPTH < yystacksize) - yystacksize = YYMAXDEPTH; - - { - yytype_int16* yyss1 = yyss; - union yyalloc* yyptr = - (union yyalloc*) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); - - if (! yyptr) - goto yyexhaustedlab; - - YYSTACK_RELOCATE (yyss_alloc, yyss); - YYSTACK_RELOCATE (yyvs_alloc, yyvs); -# undef YYSTACK_RELOCATE - - if (yyss1 != yyssa) - YYSTACK_FREE (yyss1); - } -# endif + if (yyss1 != yyssa) + YYSTACK_FREE(yyss1); + } +#endif #endif /* no yyoverflow */ - yyssp = yyss + yysize - 1; - yyvsp = yyvs + yysize - 1; + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); + YYDPRINTF((stderr, "Stack size increased to %lu\n", (unsigned long int)yystacksize)); - if (yyss + yystacksize - 1 <= yyssp) - YYABORT; - } + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YYDPRINTF((stderr, "Entering state %d\n", yystate)); - if (yystate == YYFINAL) - YYACCEPT; + if (yystate == YYFINAL) + YYACCEPT; - goto yybackup; + goto yybackup; - /*-----------. - | yybackup. | - `-----------*/ + /*-----------. + | yybackup. | + `-----------*/ yybackup: - /* Do appropriate processing given the current state. Read a - lookahead token if we need one and don't already have one. */ + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ - /* First try to decide what to do without reference to lookahead token. */ - yyn = yypact[yystate]; + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; - if (yyn == YYPACT_NINF) - goto yydefault; + if (yyn == YYPACT_NINF) + goto yydefault; - /* Not known => get a lookahead token if don't already have one. */ + /* Not known => get a lookahead token if don't already have one. */ - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ - if (yychar == YYEMPTY) + /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ + if (yychar == YYEMPTY) + { + YYDPRINTF((stderr, "Reading a token: ")); + yychar = YYLEX; + } + + if (yychar <= YYEOF) + { + yychar = yytoken = YYEOF; + YYDPRINTF((stderr, "Now at end of input.\n")); + } + else + { + yytoken = YYTRANSLATE(yychar); + YY_SYMBOL_PRINT("Next token is", yytoken, &yylval, &yylloc); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + + yyn = yytable[yyn]; + + if (yyn <= 0) + { + if (yyn == 0 || yyn == YYTABLE_NINF) + goto yyerrlab; + + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT("Shifting", yytoken, &yylval, &yylloc); + + /* Discard the shifted token. */ + yychar = YYEMPTY; + + yystate = yyn; + *++yyvsp = yylval; + + goto yynewstate; + + /*-----------------------------------------------------------. + | yydefault -- do the default action for the current state. | + `-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + + if (yyn == 0) + goto yyerrlab; + + goto yyreduce; + + /*-----------------------------. + | yyreduce -- Do a reduction. | + `-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + `$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1 - yylen]; + + YY_REDUCE_PRINT(yyn); + + switch (yyn) + { + case 3: + + /* Line 1455 of yacc.c */ +#line 64 "qfeparser.ypp" { - YYDPRINTF ((stderr, "Reading a token: ")); - yychar = YYLEX; + execplan::SRCP rcp((yyvsp[(1) - (1)].rcp)); + ParserCSEP->returnedCols().push_back(rcp); + (yyval.cp) = 0; + ; } + break; + + case 4: + + /* Line 1455 of yacc.c */ +#line 70 "qfeparser.ypp" + { + execplan::SRCP rcp((yyvsp[(3) - (3)].rcp)); + ParserCSEP->returnedCols().push_back(rcp); + (yyval.cp) = 0; + ; + } + break; + + case 5: + + /* Line 1455 of yacc.c */ +#line 78 "qfeparser.ypp" + { + execplan::SimpleColumn* sc = dynamic_cast((yyvsp[(1) - (1)].rcp)); + + if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty()) + sc->schemaName(qfe::DefaultSchema); + + sc->setOID(); + sc->tableAlias(sc->tableName()); + sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString()); + (yyval.rcp) = (yyvsp[(1) - (1)].rcp); + ; + } + break; + + case 7: + + /* Line 1455 of yacc.c */ +#line 91 "qfeparser.ypp" + { + execplan::AggregateColumn* ag = new execplan::AggregateColumn( + execplan::AggregateColumn::agname2num(*((yyvsp[(1) - (4)].cp))), (yyvsp[(3) - (4)].rcp)); + delete (yyvsp[(1) - (4)].cp); + (yyval.rcp) = ag; + ; + } + break; + + case 9: + + /* Line 1455 of yacc.c */ +#line 102 "qfeparser.ypp" + { + (yyval.cp) = 0; + ; + } + break; + + case 10: + + /* Line 1455 of yacc.c */ +#line 104 "qfeparser.ypp" + { + delete (yyvsp[(2) - (2)].cp); + (yyval.cp) = (yyvsp[(1) - (2)].cp); + ; + } + break; + + case 11: + + /* Line 1455 of yacc.c */ +#line 109 "qfeparser.ypp" + { + delete (yyvsp[(2) - (2)].cp); + (yyval.cp) = (yyvsp[(1) - (2)].cp); + ; + } + break; + + case 12: + + /* Line 1455 of yacc.c */ +#line 116 "qfeparser.ypp" + { + (yyval.cp) = 0; + ; + } + break; + + case 15: + + /* Line 1455 of yacc.c */ +#line 125 "qfeparser.ypp" + { + execplan::SimpleColumn* sc = 0; + sc = dynamic_cast((yyvsp[(1) - (1)].rcp)); + execplan::CalpontSystemCatalog::TableAliasName tan; + tan.schema = sc->tableName(); + + if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty()) + tan.schema = qfe::DefaultSchema; + + if (qfe::DefaultSchema.empty()) + qfe::DefaultSchema = tan.schema; + + tan.table = sc->columnName(); + tan.alias = tan.table; + execplan::CalpontSelectExecutionPlan::TableList tl; + tl.push_back(tan); + ParserCSEP->tableList(tl); + delete sc; + (yyval.cp) = 0; + ; + } + break; + + case 19: + + /* Line 1455 of yacc.c */ +#line 152 "qfeparser.ypp" + { + pair cval = *((yyvsp[(3) - (3)].cvp)); + delete (yyvsp[(3) - (3)].cvp); + execplan::SimpleColumn* sc = 0; + sc = dynamic_cast((yyvsp[(1) - (3)].rcp)); + qfe::utils::updateParseTree(ParserCSC, ParserCSEP, sc, *((yyvsp[(2) - (3)].cp)), cval); + delete sc; + delete (yyvsp[(2) - (3)].cp); + (yyval.cp) = 0; + ; + } + break; + + case 20: + + /* Line 1455 of yacc.c */ +#line 164 "qfeparser.ypp" + { + // string logicop = *($2); + delete (yyvsp[(2) - (5)].cp); + pair cval = *((yyvsp[(5) - (5)].cvp)); + delete (yyvsp[(5) - (5)].cvp); + execplan::SimpleColumn* sc = 0; + sc = dynamic_cast((yyvsp[(3) - (5)].rcp)); + qfe::utils::updateParseTree(ParserCSC, ParserCSEP, sc, *((yyvsp[(4) - (5)].cp)), cval); + delete sc; + delete (yyvsp[(4) - (5)].cp); + (yyval.cp) = 0; + ; + } + break; + + case 21: + + /* Line 1455 of yacc.c */ +#line 180 "qfeparser.ypp" + { + // This is possibly a table name, but we shove it into a SimpleColumn. We'll + // fix this in the table_spec production + execplan::SimpleColumn* sc = + new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *((yyvsp[(1) - (1)].cp))); + sc->tableAlias("infinidb_unknown"); + sc->alias( + execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *((yyvsp[(1) - (1)].cp))).toString()); + // cerr << "inside parser: " << *sc << endl; + delete (yyvsp[(1) - (1)].cp); + (yyval.rcp) = sc; + ; + } + break; + + case 22: + + /* Line 1455 of yacc.c */ +#line 191 "qfeparser.ypp" + { + // This is possibly a table name, but we shove it into a SimpleColumn. We'll + // fix this in the table_spec production + execplan::SimpleColumn* sc = + new execplan::SimpleColumn("infinidb_unknown", *((yyvsp[(1) - (3)].cp)), *((yyvsp[(3) - (3)].cp))); + sc->tableAlias(*((yyvsp[(1) - (3)].cp))); + sc->alias(execplan::make_tcn("infinidb_unknown", *((yyvsp[(1) - (3)].cp)), *((yyvsp[(3) - (3)].cp))) + .toString()); + // cerr << "inside parser: " << *sc << endl; + delete (yyvsp[(1) - (3)].cp); + delete (yyvsp[(3) - (3)].cp); + (yyval.rcp) = sc; + ; + } + break; + + case 23: + + /* Line 1455 of yacc.c */ +#line 203 "qfeparser.ypp" + { + if (qfe::DefaultSchema.empty()) + qfe::DefaultSchema = *((yyvsp[(1) - (5)].cp)); + + execplan::SimpleColumn* sc = new execplan::SimpleColumn( + *((yyvsp[(1) - (5)].cp)), *((yyvsp[(3) - (5)].cp)), *((yyvsp[(5) - (5)].cp))); + sc->tableAlias(*((yyvsp[(3) - (5)].cp))); + sc->alias( + execplan::make_tcn(*((yyvsp[(1) - (5)].cp)), *((yyvsp[(3) - (5)].cp)), *((yyvsp[(5) - (5)].cp))) + .toString()); + // cerr << "inside parser: " << *sc << endl; + delete (yyvsp[(1) - (5)].cp); + delete (yyvsp[(3) - (5)].cp); + delete (yyvsp[(5) - (5)].cp); + (yyval.rcp) = sc; + ; + } + break; + + case 25: + + /* Line 1455 of yacc.c */ +#line 219 "qfeparser.ypp" + { + (yyval.cp) = (yyvsp[(2) - (3)].cp); + ; + } + break; + + case 26: + + /* Line 1455 of yacc.c */ +#line 225 "qfeparser.ypp" + { + pair* p = new pair(1, *((yyvsp[(1) - (1)].cp))); + delete (yyvsp[(1) - (1)].cp); + (yyval.cvp) = p; + ; + } + break; + + case 27: + + /* Line 1455 of yacc.c */ +#line 231 "qfeparser.ypp" + { + pair* p = new pair(0, *((yyvsp[(1) - (1)].cp))); + delete (yyvsp[(1) - (1)].cp); + (yyval.cvp) = p; + ; + } + break; + + case 29: + + /* Line 1455 of yacc.c */ +#line 240 "qfeparser.ypp" + { + YYERROR; + ; + } + break; + + case 31: + + /* Line 1455 of yacc.c */ +#line 245 "qfeparser.ypp" + { + delete (yyvsp[(3) - (3)].rcp); + ; + } + break; + + case 33: + + /* Line 1455 of yacc.c */ +#line 250 "qfeparser.ypp" + { + delete (yyvsp[(1) - (1)].cp); + (yyval.rcp) = 0; + ; + } + break; + + case 35: + + /* Line 1455 of yacc.c */ +#line 258 "qfeparser.ypp" + { + YYERROR; + ; + } + break; + + case 39: + + /* Line 1455 of yacc.c */ +#line 267 "qfeparser.ypp" + { + (yyval.cp) = 0; + ; + } + break; + + case 40: + + /* Line 1455 of yacc.c */ +#line 269 "qfeparser.ypp" + { + delete (yyvsp[(2) - (2)].cp); + YYERROR; + (yyval.cp) = 0; + ; + } + break; + + case 41: + + /* Line 1455 of yacc.c */ +#line 275 "qfeparser.ypp" + { + delete (yyvsp[(2) - (4)].cp); + delete (yyvsp[(4) - (4)].cp); + YYERROR; + (yyval.cp) = 0; + ; + } + break; + + /* Line 1455 of yacc.c */ +#line 1835 "bison-win.cpp" + + default: break; + } + + YY_SYMBOL_PRINT("-> $$ =", yyr1[yyn], &yyval, &yyloc); + + YYPOPSTACK(yylen); + yylen = 0; + YY_STACK_PRINT(yyss, yyssp); + + *++yyvsp = yyval; + + /* Now `shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + + yyn = yyr1[yyn]; + + yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; + + if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) + yystate = yytable[yystate]; + else + yystate = yydefgoto[yyn - YYNTOKENS]; + + goto yynewstate; + + /*------------------------------------. + | yyerrlab -- here on detecting error | + `------------------------------------*/ +yyerrlab: + + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; +#if !YYERROR_VERBOSE + yyerror(YY_("syntax error")); +#else + { + YYSIZE_T yysize = yysyntax_error(0, yystate, yychar); + + if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) + { + YYSIZE_T yyalloc = 2 * yysize; + + if (!(yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) + yyalloc = YYSTACK_ALLOC_MAXIMUM; + + if (yymsg != yymsgbuf) + YYSTACK_FREE(yymsg); + + yymsg = (char*)YYSTACK_ALLOC(yyalloc); + + if (yymsg) + yymsg_alloc = yyalloc; + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + } + } + + if (0 < yysize && yysize <= yymsg_alloc) + { + (void)yysyntax_error(yymsg, yystate, yychar); + yyerror(yymsg); + } + else + { + yyerror(YY_("syntax error")); + + if (yysize != 0) + goto yyexhaustedlab; + } + } +#endif + } + + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ if (yychar <= YYEOF) { - yychar = yytoken = YYEOF; - YYDPRINTF ((stderr, "Now at end of input.\n")); + /* Return failure if at end of input. */ + if (yychar == YYEOF) + YYABORT; } else { - yytoken = YYTRANSLATE (yychar); - YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); + yydestruct("Error: discarding", yytoken, &yylval); + yychar = YYEMPTY; } + } - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken; + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; - if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) - goto yydefault; - - yyn = yytable[yyn]; - - if (yyn <= 0) - { - if (yyn == 0 || yyn == YYTABLE_NINF) - goto yyerrlab; - - yyn = -yyn; - goto yyreduce; - } - - /* Count tokens shifted since error; after three, turn off error - status. */ - if (yyerrstatus) - yyerrstatus--; - - /* Shift the lookahead token. */ - YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - - yystate = yyn; - *++yyvsp = yylval; - - goto yynewstate; - - - /*-----------------------------------------------------------. - | yydefault -- do the default action for the current state. | - `-----------------------------------------------------------*/ -yydefault: - yyn = yydefact[yystate]; - - if (yyn == 0) - goto yyerrlab; - - goto yyreduce; - - - /*-----------------------------. - | yyreduce -- Do a reduction. | - `-----------------------------*/ -yyreduce: - /* yyn is the number of a rule to reduce with. */ - yylen = yyr2[yyn]; - - /* If YYLEN is nonzero, implement the default value of the action: - `$$ = $1'. - - Otherwise, the following line sets YYVAL to garbage. - This behavior is undocumented and Bison - users should not rely upon it. Assigning to YYVAL - unconditionally makes the parser a bit smaller, and it avoids a - GCC warning that YYVAL may be used uninitialized. */ - yyval = yyvsp[1 - yylen]; - - - YY_REDUCE_PRINT (yyn); - - switch (yyn) - { - case 3: - - /* Line 1455 of yacc.c */ -#line 64 "qfeparser.ypp" - { - execplan::SRCP rcp((yyvsp[(1) - (1)].rcp)); - ParserCSEP->returnedCols().push_back(rcp); - (yyval.cp) = 0; - ; - } - break; - - case 4: - - /* Line 1455 of yacc.c */ -#line 70 "qfeparser.ypp" - { - execplan::SRCP rcp((yyvsp[(3) - (3)].rcp)); - ParserCSEP->returnedCols().push_back(rcp); - (yyval.cp) = 0; - ; - } - break; - - case 5: - - /* Line 1455 of yacc.c */ -#line 78 "qfeparser.ypp" - { - execplan::SimpleColumn* sc = dynamic_cast((yyvsp[(1) - (1)].rcp)); - - if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty()) - sc->schemaName(qfe::DefaultSchema); - - sc->setOID(); - sc->tableAlias(sc->tableName()); - sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString()); - (yyval.rcp) = (yyvsp[(1) - (1)].rcp); - ; - } - break; - - case 7: - - /* Line 1455 of yacc.c */ -#line 91 "qfeparser.ypp" - { - execplan::AggregateColumn* ag = new execplan::AggregateColumn(execplan::AggregateColumn::agname2num(*((yyvsp[(1) - (4)].cp))), (yyvsp[(3) - (4)].rcp)); - delete (yyvsp[(1) - (4)].cp); - (yyval.rcp) = ag; - ; - } - break; - - case 9: - - /* Line 1455 of yacc.c */ -#line 102 "qfeparser.ypp" - { - (yyval.cp) = 0; ; - } - break; - - case 10: - - /* Line 1455 of yacc.c */ -#line 104 "qfeparser.ypp" - { - delete (yyvsp[(2) - (2)].cp); - (yyval.cp) = (yyvsp[(1) - (2)].cp); - ; - } - break; - - case 11: - - /* Line 1455 of yacc.c */ -#line 109 "qfeparser.ypp" - { - delete (yyvsp[(2) - (2)].cp); - (yyval.cp) = (yyvsp[(1) - (2)].cp); - ; - } - break; - - case 12: - - /* Line 1455 of yacc.c */ -#line 116 "qfeparser.ypp" - { - (yyval.cp) = 0; ; - } - break; - - case 15: - - /* Line 1455 of yacc.c */ -#line 125 "qfeparser.ypp" - { - execplan::SimpleColumn* sc = 0; - sc = dynamic_cast((yyvsp[(1) - (1)].rcp)); - execplan::CalpontSystemCatalog::TableAliasName tan; - tan.schema = sc->tableName(); - - if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty()) - tan.schema = qfe::DefaultSchema; - - if (qfe::DefaultSchema.empty()) - qfe::DefaultSchema = tan.schema; - - tan.table = sc->columnName(); - tan.alias = tan.table; - execplan::CalpontSelectExecutionPlan::TableList tl; - tl.push_back(tan); - ParserCSEP->tableList(tl); - delete sc; - (yyval.cp) = 0; - ; - } - break; - - case 19: - - /* Line 1455 of yacc.c */ -#line 152 "qfeparser.ypp" - { - pair cval = *((yyvsp[(3) - (3)].cvp)); - delete (yyvsp[(3) - (3)].cvp); - execplan::SimpleColumn* sc = 0; - sc = dynamic_cast((yyvsp[(1) - (3)].rcp)); - qfe::utils::updateParseTree(ParserCSC, ParserCSEP, - sc, *((yyvsp[(2) - (3)].cp)), cval); - delete sc; - delete (yyvsp[(2) - (3)].cp); - (yyval.cp) = 0; - ; - } - break; - - case 20: - - /* Line 1455 of yacc.c */ -#line 164 "qfeparser.ypp" - { - //string logicop = *($2); - delete (yyvsp[(2) - (5)].cp); - pair cval = *((yyvsp[(5) - (5)].cvp)); - delete (yyvsp[(5) - (5)].cvp); - execplan::SimpleColumn* sc = 0; - sc = dynamic_cast((yyvsp[(3) - (5)].rcp)); - qfe::utils::updateParseTree(ParserCSC, ParserCSEP, - sc, *((yyvsp[(4) - (5)].cp)), cval); - delete sc; - delete (yyvsp[(4) - (5)].cp); - (yyval.cp) = 0; - ; - } - break; - - case 21: - - /* Line 1455 of yacc.c */ -#line 180 "qfeparser.ypp" - { - //This is possibly a table name, but we shove it into a SimpleColumn. We'll - // fix this in the table_spec production - execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *((yyvsp[(1) - (1)].cp))); - sc->tableAlias("infinidb_unknown"); - sc->alias(execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *((yyvsp[(1) - (1)].cp))).toString()); - //cerr << "inside parser: " << *sc << endl; - delete (yyvsp[(1) - (1)].cp); - (yyval.rcp) = sc; - ; - } - break; - - case 22: - - /* Line 1455 of yacc.c */ -#line 191 "qfeparser.ypp" - { - //This is possibly a table name, but we shove it into a SimpleColumn. We'll - // fix this in the table_spec production - execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", *((yyvsp[(1) - (3)].cp)), *((yyvsp[(3) - (3)].cp))); - sc->tableAlias(*((yyvsp[(1) - (3)].cp))); - sc->alias(execplan::make_tcn("infinidb_unknown", *((yyvsp[(1) - (3)].cp)), *((yyvsp[(3) - (3)].cp))).toString()); - //cerr << "inside parser: " << *sc << endl; - delete (yyvsp[(1) - (3)].cp); - delete (yyvsp[(3) - (3)].cp); - (yyval.rcp) = sc; - ; - } - break; - - case 23: - - /* Line 1455 of yacc.c */ -#line 203 "qfeparser.ypp" - { - if (qfe::DefaultSchema.empty()) - qfe::DefaultSchema = *((yyvsp[(1) - (5)].cp)); - - execplan::SimpleColumn* sc = new execplan::SimpleColumn(*((yyvsp[(1) - (5)].cp)), *((yyvsp[(3) - (5)].cp)), *((yyvsp[(5) - (5)].cp))); - sc->tableAlias(*((yyvsp[(3) - (5)].cp))); - sc->alias(execplan::make_tcn(*((yyvsp[(1) - (5)].cp)), *((yyvsp[(3) - (5)].cp)), *((yyvsp[(5) - (5)].cp))).toString()); - //cerr << "inside parser: " << *sc << endl; - delete (yyvsp[(1) - (5)].cp); - delete (yyvsp[(3) - (5)].cp); - delete (yyvsp[(5) - (5)].cp); - (yyval.rcp) = sc; - ; - } - break; - - case 25: - - /* Line 1455 of yacc.c */ -#line 219 "qfeparser.ypp" - { - (yyval.cp) = (yyvsp[(2) - (3)].cp); - ; - } - break; - - case 26: - - /* Line 1455 of yacc.c */ -#line 225 "qfeparser.ypp" - { - pair* p = new pair(1, *((yyvsp[(1) - (1)].cp))); - delete (yyvsp[(1) - (1)].cp); - (yyval.cvp) = p; - ; - } - break; - - case 27: - - /* Line 1455 of yacc.c */ -#line 231 "qfeparser.ypp" - { - pair* p = new pair(0, *((yyvsp[(1) - (1)].cp))); - delete (yyvsp[(1) - (1)].cp); - (yyval.cvp) = p; - ; - } - break; - - case 29: - - /* Line 1455 of yacc.c */ -#line 240 "qfeparser.ypp" - { - YYERROR; ; - } - break; - - case 31: - - /* Line 1455 of yacc.c */ -#line 245 "qfeparser.ypp" - { - delete (yyvsp[(3) - (3)].rcp); ; - } - break; - - case 33: - - /* Line 1455 of yacc.c */ -#line 250 "qfeparser.ypp" - { - delete (yyvsp[(1) - (1)].cp); - (yyval.rcp) = 0; - ; - } - break; - - case 35: - - /* Line 1455 of yacc.c */ -#line 258 "qfeparser.ypp" - { - YYERROR; ; - } - break; - - case 39: - - /* Line 1455 of yacc.c */ -#line 267 "qfeparser.ypp" - { - (yyval.cp) = 0; ; - } - break; - - case 40: - - /* Line 1455 of yacc.c */ -#line 269 "qfeparser.ypp" - { - delete (yyvsp[(2) - (2)].cp); - YYERROR; - (yyval.cp) = 0; - ; - } - break; - - case 41: - - /* Line 1455 of yacc.c */ -#line 275 "qfeparser.ypp" - { - delete (yyvsp[(2) - (4)].cp); - delete (yyvsp[(4) - (4)].cp); - YYERROR; - (yyval.cp) = 0; - ; - } - break; - - - - /* Line 1455 of yacc.c */ -#line 1835 "bison-win.cpp" - - default: - break; - } - - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); - - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - - *++yyvsp = yyval; - - /* Now `shift' the result of the reduction. Determine what state - that goes to, based on the state we popped back to and the rule - number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; - - goto yynewstate; - - - /*------------------------------------. - | yyerrlab -- here on detecting error | - `------------------------------------*/ -yyerrlab: - - /* If not already recovering from an error, report this error. */ - if (!yyerrstatus) - { - ++yynerrs; -#if ! YYERROR_VERBOSE - yyerror (YY_("syntax error")); -#else - { - YYSIZE_T yysize = yysyntax_error (0, yystate, yychar); - - if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) - { - YYSIZE_T yyalloc = 2 * yysize; - - if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) - yyalloc = YYSTACK_ALLOC_MAXIMUM; - - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); - - yymsg = (char*) YYSTACK_ALLOC (yyalloc); - - if (yymsg) - yymsg_alloc = yyalloc; - else - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - } - } - - if (0 < yysize && yysize <= yymsg_alloc) - { - (void) yysyntax_error (yymsg, yystate, yychar); - yyerror (yymsg); - } - else - { - yyerror (YY_("syntax error")); - - if (yysize != 0) - goto yyexhaustedlab; - } - } -#endif - } - - - - if (yyerrstatus == 3) - { - /* If just tried and failed to reuse lookahead token after an - error, discard it. */ - - if (yychar <= YYEOF) - { - /* Return failure if at end of input. */ - if (yychar == YYEOF) - YYABORT; - } - else - { - yydestruct ("Error: discarding", - yytoken, &yylval); - yychar = YYEMPTY; - } - } - - /* Else will try to reuse lookahead token after shifting the error - token. */ - goto yyerrlab1; - - - /*---------------------------------------------------. - | yyerrorlab -- error raised explicitly by YYERROR. | - `---------------------------------------------------*/ + /*---------------------------------------------------. + | yyerrorlab -- error raised explicitly by YYERROR. | + `---------------------------------------------------*/ yyerrorlab: - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; + /* Pacify compilers like GCC when the user code never invokes + YYERROR and the label yyerrorlab therefore never appears in user + code. */ + if (/*CONSTCOND*/ 0) + goto yyerrorlab; - /* Do not reclaim the symbols of the rule which action triggered - this YYERROR. */ - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - yystate = *yyssp; - goto yyerrlab1; + /* Do not reclaim the symbols of the rule which action triggered + this YYERROR. */ + YYPOPSTACK(yylen); + yylen = 0; + YY_STACK_PRINT(yyss, yyssp); + yystate = *yyssp; + goto yyerrlab1; - - /*-------------------------------------------------------------. - | yyerrlab1 -- common code for both syntax error and YYERROR. | - `-------------------------------------------------------------*/ + /*-------------------------------------------------------------. + | yyerrlab1 -- common code for both syntax error and YYERROR. | + `-------------------------------------------------------------*/ yyerrlab1: - yyerrstatus = 3; /* Each real token shifted decrements this. */ + yyerrstatus = 3; /* Each real token shifted decrements this. */ - for (;;) + for (;;) + { + yyn = yypact[yystate]; + + if (yyn != YYPACT_NINF) { - yyn = yypact[yystate]; + yyn += YYTERROR; - if (yyn != YYPACT_NINF) - { - yyn += YYTERROR; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + { + yyn = yytable[yyn]; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) - { - yyn = yytable[yyn]; - - if (0 < yyn) - break; - } - } - - /* Pop the current state because it cannot handle the error token. */ - if (yyssp == yyss) - YYABORT; - - - yydestruct ("Error: popping", - yystos[yystate], yyvsp); - YYPOPSTACK (1); - yystate = *yyssp; - YY_STACK_PRINT (yyss, yyssp); + if (0 < yyn) + break; + } } - *++yyvsp = yylval; + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + yydestruct("Error: popping", yystos[yystate], yyvsp); + YYPOPSTACK(1); + yystate = *yyssp; + YY_STACK_PRINT(yyss, yyssp); + } - /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + *++yyvsp = yylval; - yystate = yyn; - goto yynewstate; + /* Shift the error token. */ + YY_SYMBOL_PRINT("Shifting", yystos[yyn], yyvsp, yylsp); + yystate = yyn; + goto yynewstate; - /*-------------------------------------. - | yyacceptlab -- YYACCEPT comes here. | - `-------------------------------------*/ + /*-------------------------------------. + | yyacceptlab -- YYACCEPT comes here. | + `-------------------------------------*/ yyacceptlab: - yyresult = 0; - goto yyreturn; + yyresult = 0; + goto yyreturn; - /*-----------------------------------. - | yyabortlab -- YYABORT comes here. | - `-----------------------------------*/ + /*-----------------------------------. + | yyabortlab -- YYABORT comes here. | + `-----------------------------------*/ yyabortlab: - yyresult = 1; - goto yyreturn; + yyresult = 1; + goto yyreturn; #if !defined(yyoverflow) || YYERROR_VERBOSE - /*-------------------------------------------------. - | yyexhaustedlab -- memory exhaustion comes here. | - `-------------------------------------------------*/ + /*-------------------------------------------------. + | yyexhaustedlab -- memory exhaustion comes here. | + `-------------------------------------------------*/ yyexhaustedlab: - yyerror (YY_("memory exhausted")); - yyresult = 2; - /* Fall through. */ + yyerror(YY_("memory exhausted")); + yyresult = 2; + /* Fall through. */ #endif yyreturn: - if (yychar != YYEMPTY) - yydestruct ("Cleanup: discarding lookahead", - yytoken, &yylval); + if (yychar != YYEMPTY) + yydestruct("Cleanup: discarding lookahead", yytoken, &yylval); - /* Do not reclaim the symbols of the rule which action triggered - this YYABORT or YYACCEPT. */ - YYPOPSTACK (yylen); - YY_STACK_PRINT (yyss, yyssp); + /* Do not reclaim the symbols of the rule which action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK(yylen); + YY_STACK_PRINT(yyss, yyssp); - while (yyssp != yyss) - { - yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp); - YYPOPSTACK (1); - } + while (yyssp != yyss) + { + yydestruct("Cleanup: popping", yystos[*yyssp], yyvsp); + YYPOPSTACK(1); + } #ifndef yyoverflow - if (yyss != yyssa) - YYSTACK_FREE (yyss); + if (yyss != yyssa) + YYSTACK_FREE(yyss); #endif #if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); + if (yymsg != yymsgbuf) + YYSTACK_FREE(yymsg); #endif - /* Make sure YYID is used. */ - return YYID (yyresult); + /* Make sure YYID is used. */ + return YYID(yyresult); } - - /* Line 1675 of yacc.c */ #line 283 "qfeparser.ypp" - int qfeerror(const string& s) { - extern int qfelineno; // defined and maintained in lex.c - extern char* qfetext; // defined and maintained in lex.c + extern int qfelineno; // defined and maintained in lex.c + extern char* qfetext; // defined and maintained in lex.c - cerr << "ERROR: " << s << " at symbol \"" << qfetext; - cerr << "\" on line " << qfelineno << endl; - return -1; + cerr << "ERROR: " << s << " at symbol \"" << qfetext; + cerr << "\" on line " << qfelineno << endl; + return -1; } int qfeerror(const char* s) { - return qfeerror(string(s)); + return qfeerror(string(s)); } string* newstr(const char* cp) { - string* strp; - strp = new string(cp); - return strp; + string* strp; + strp = new string(cp); + return strp; } - - diff --git a/tools/qfe/bison-win.h b/tools/qfe/bison-win.h index 904821ea9..4a66a0038 100644 --- a/tools/qfe/bison-win.h +++ b/tools/qfe/bison-win.h @@ -32,55 +32,47 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ - /* Tokens. */ #ifndef YYTOKENTYPE -# define YYTOKENTYPE +#define YYTOKENTYPE /* Put the tokens into the symbol table, so that GDB and other debuggers know about them. */ enum yytokentype { - CHAR_CONST = 258, - RELOP = 259, - LOGICOP = 260, - QFEP_SELECT = 261, - QFEP_FROM = 262, - QFEP_WHERE = 263, - GROUPBY = 264, - OBJNAME = 265, - INT_CONST = 266, - LIMIT = 267, - ORDERBY = 268, - ASC = 269, - DESC = 270, - AS = 271, - FUNC = 272 + CHAR_CONST = 258, + RELOP = 259, + LOGICOP = 260, + QFEP_SELECT = 261, + QFEP_FROM = 262, + QFEP_WHERE = 263, + GROUPBY = 264, + OBJNAME = 265, + INT_CONST = 266, + LIMIT = 267, + ORDERBY = 268, + ASC = 269, + DESC = 270, + AS = 271, + FUNC = 272 }; #endif - - -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +#if !defined YYSTYPE && !defined YYSTYPE_IS_DECLARED typedef union YYSTYPE { - - /* Line 1676 of yacc.c */ + /* Line 1676 of yacc.c */ #line 31 "qfeparser.ypp" - execplan::ReturnedColumn* rcp; - std::string* cp; - std::pair* cvp; + execplan::ReturnedColumn* rcp; + std::string* cp; + std::pair* cvp; - - - /* Line 1676 of yacc.c */ + /* Line 1676 of yacc.c */ #line 77 "bison-win.hpp" } YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define yystype YYSTYPE /* obsolescent; will be withdrawn */ -# define YYSTYPE_IS_DECLARED 1 +#define YYSTYPE_IS_TRIVIAL 1 +#define yystype YYSTYPE /* obsolescent; will be withdrawn */ +#define YYSTYPE_IS_DECLARED 1 #endif extern YYSTYPE qfelval; - - diff --git a/tools/qfe/cli.cpp b/tools/qfe/cli.cpp index 0931a3cdf..13ea438c4 100644 --- a/tools/qfe/cli.cpp +++ b/tools/qfe/cli.cpp @@ -35,152 +35,147 @@ namespace { void usage() { - cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] " << endl; - cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl; - cout << '\t' << "-p port connect on port instead of 9198" << endl; - cout << '\t' << "-c schema use schema as default instead of tpch1" << endl; - cout << '\t' << "-h display this help" << endl; -} + cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] " << endl; + cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl; + cout << '\t' << "-p port connect on port instead of 9198" << endl; + cout << '\t' << "-c schema use schema as default instead of tpch1" << endl; + cout << '\t' << "-h display this help" << endl; } +} // namespace int main(int argc, char** argv) { - opterr = 0; - int c; - string serverip("127.0.0.1"); - string schema("tpch1"); - short PortNo = 9198; + opterr = 0; + int c; + string serverip("127.0.0.1"); + string schema("tpch1"); + short PortNo = 9198; - while ((c = getopt(argc, argv, "s:c:p:h")) != -1) - switch (c) + while ((c = getopt(argc, argv, "s:c:p:h")) != -1) + switch (c) + { + case 's': serverip = optarg; break; + + case 'c': schema = optarg; break; + + case 'p': + PortNo = atoi(optarg); + + if (PortNo == 0) { - case 's': - serverip = optarg; - break; - - case 'c': - schema = optarg; - break; - - case 'p': - PortNo = atoi(optarg); - - if (PortNo == 0) - { - usage(); - return 1; - } - - break; - - case 'h': - case '?': - default: - usage(); - return 0; - break; + usage(); + return 1; } - if (argc - optind < 1) - { + break; + + case 'h': + case '?': + default: usage(); - return 1; + return 0; + break; } - string query(argv[optind + 0]); + if (argc - optind < 1) + { + usage(); + return 1; + } - SockType fd = -1; + string query(argv[optind + 0]); + + SockType fd = -1; #ifdef _MSC_VER - WSAData wsadata; - const WORD minVersion = MAKEWORD(2, 2); + WSAData wsadata; + const WORD minVersion = MAKEWORD(2, 2); - if (WSAStartup(minVersion, &wsadata) != 0) - cerr << "networking startup error: " << IDBSysErrorStr(WSAGetLastError()) << endl; + if (WSAStartup(minVersion, &wsadata) != 0) + cerr << "networking startup error: " << IDBSysErrorStr(WSAGetLastError()) << endl; #endif - fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - if (fd < 0) - { + if (fd < 0) + { #ifdef _MSC_VER - cerr << "socket create error: " << IDBSysErrorStr(WSAGetLastError()) << endl; + cerr << "socket create error: " << IDBSysErrorStr(WSAGetLastError()) << endl; #else - cerr << "socket create error: " << strerror(errno) << endl; + cerr << "socket create error: " << strerror(errno) << endl; #endif - return 1; - } + return 1; + } - int rc = 0; - struct sockaddr_in serv_addr; - struct addrinfo hints; - struct addrinfo* res = 0; - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = PF_INET; - hints.ai_socktype = SOCK_STREAM; - hints.ai_protocol = IPPROTO_TCP; - rc = getaddrinfo(serverip.c_str(), 0, &hints, &res); + int rc = 0; + struct sockaddr_in serv_addr; + struct addrinfo hints; + struct addrinfo* res = 0; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = PF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + rc = getaddrinfo(serverip.c_str(), 0, &hints, &res); - if (rc != 0) - { - cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl; - return 1; - } + if (rc != 0) + { + cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl; + return 1; + } - sockaddr_in* sain = 0; - sain = reinterpret_cast(res->ai_addr); - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin_family = PF_INET; - serv_addr.sin_addr = sain->sin_addr; - serv_addr.sin_port = htons(PortNo); - freeaddrinfo(res); + sockaddr_in* sain = 0; + sain = reinterpret_cast(res->ai_addr); + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = PF_INET; + serv_addr.sin_addr = sain->sin_addr; + serv_addr.sin_port = htons(PortNo); + freeaddrinfo(res); - rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr)); + rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr)); - if (rc < 0) - { + if (rc < 0) + { #ifdef _MSC_VER - cerr << "socket connect error: " << IDBSysErrorStr(WSAGetLastError()) << endl; - WSACleanup(); -#else - cerr << "socket connect error: " << strerror(errno) << endl; -#endif - return 1; - } - - socketio::writeString(fd, schema); - socketio::writeString(fd, query); - - uint32_t flag = 0; - string row; - row = socketio::readString(fd); - - if (row != "OK") - { - cerr << "query failed: " << row << endl; - goto bailout; - } - - row = socketio::readString(fd); - - while (!row.empty()) - { - cout << row << endl; - row = socketio::readString(fd); - } - - flag = 0; - SockWriteFcn(fd, &flag, 4); -bailout: - ::shutdown(fd, SHUT_RDWR); -#ifdef _MSC_VER - ::closesocket(fd); + cerr << "socket connect error: " << IDBSysErrorStr(WSAGetLastError()) << endl; WSACleanup(); #else - ::close(fd); + cerr << "socket connect error: " << strerror(errno) << endl; #endif - fd = -1; + return 1; + } - return 0; + socketio::writeString(fd, schema); + socketio::writeString(fd, query); + + uint32_t flag = 0; + string row; + row = socketio::readString(fd); + + if (row != "OK") + { + cerr << "query failed: " << row << endl; + goto bailout; + } + + row = socketio::readString(fd); + + while (!row.empty()) + { + cout << row << endl; + row = socketio::readString(fd); + } + + flag = 0; + SockWriteFcn(fd, &flag, 4); +bailout: + ::shutdown(fd, SHUT_RDWR); +#ifdef _MSC_VER + ::closesocket(fd); + WSACleanup(); +#else + ::close(fd); +#endif + fd = -1; + + return 0; } - diff --git a/tools/qfe/cseputils.cpp b/tools/qfe/cseputils.cpp index a34df0254..3584c8a03 100644 --- a/tools/qfe/cseputils.cpp +++ b/tools/qfe/cseputils.cpp @@ -12,143 +12,130 @@ using namespace execplan; #include "cseputils.h" - namespace qfe { extern string DefaultSchema; namespace utils { - ConstantColumn* createConstCol(const string& valstr) { - ConstantColumn* cc = new ConstantColumn(valstr); - cc->alias(valstr); - return cc; + ConstantColumn* cc = new ConstantColumn(valstr); + cc->alias(valstr); + return cc; } template ConstantColumn* createConstCol(const string& valstr, T val) { - ConstantColumn* cc = new ConstantColumn(valstr, val); - cc->alias(valstr); - return cc; + ConstantColumn* cc = new ConstantColumn(valstr, val); + cc->alias(valstr); + return cc; } -SimpleFilter* createSimpleFilter -( - boost::shared_ptr& csc, - const CalpontSystemCatalog::TableColName& tcn, - const string& opstr, - ConstantColumn* cc -) +SimpleFilter* createSimpleFilter(boost::shared_ptr& csc, + const CalpontSystemCatalog::TableColName& tcn, const string& opstr, + ConstantColumn* cc) { - SimpleFilter* lsf = new SimpleFilter(); + SimpleFilter* lsf = new SimpleFilter(); + + Operator* op = new Operator(); + op->data(opstr); + CalpontSystemCatalog::ColType ccct; + ccct = op->resultType(); + ccct.colDataType = cc->resultType().colDataType; + op->operationType(ccct); + + SOP sop(op); + lsf->op(sop); + + CalpontSystemCatalog::OID oid = csc->lookupOID(tcn); + CalpontSystemCatalog::ColType ct = csc->colType(oid); + + SimpleColumn* sc = new SimpleColumn(); + sc->schemaName(tcn.schema); + sc->tableName(tcn.table); + sc->tableAlias(tcn.table); + sc->columnName(tcn.column); + sc->oid(oid); + sc->resultType(ct); + sc->alias(tcn.toString()); + + lsf->lhs(sc); + lsf->rhs(cc); + + return lsf; +} + +void appendSimpleFilter(ParseTree*& ptree, SimpleFilter* filter) +{ + if (ptree->data() == 0) + { + // degenerate case, this filter goes at this node + ptree->data(filter); + } + else if (ptree->right() == 0 && ptree->left() == 0) + { + // this will be the case when there is a single node in the tree + // that contains a filter. Here we want to make the root node an + // 'and' operator, push the existing down to the lhs and make a + // new node for the new filter + ParseTree* newLhs = new ParseTree(ptree->data()); + ParseTree* newRhs = new ParseTree(filter); Operator* op = new Operator(); - op->data(opstr); - CalpontSystemCatalog::ColType ccct; - ccct = op->resultType(); - ccct.colDataType = cc->resultType().colDataType; - op->operationType(ccct); + op->data("and"); - SOP sop(op); - lsf->op(sop); + ptree->data(op); + ptree->left(newLhs); + ptree->right(newRhs); + } + else + { + // this will be the case once we have a tree with an 'and' at the + // root node, a filter in the lhs, and an arbitrary height tree + // with the same properties on the rhs. Because all operators + // are guaranteed to be and for now we simply insert a new rhs + // node and "push down" the existing tree + Operator* op = new Operator(); + op->data("and"); - CalpontSystemCatalog::OID oid = csc->lookupOID(tcn); - CalpontSystemCatalog::ColType ct = csc->colType(oid); - - SimpleColumn* sc = new SimpleColumn(); - sc->schemaName(tcn.schema); - sc->tableName(tcn.table); - sc->tableAlias(tcn.table); - sc->columnName(tcn.column); - sc->oid(oid); - sc->resultType(ct); - sc->alias(tcn.toString()); - - lsf->lhs(sc); - lsf->rhs(cc); - - return lsf; -} - -void appendSimpleFilter -( - ParseTree*& ptree, - SimpleFilter* filter -) -{ - if ( ptree->data() == 0 ) - { - // degenerate case, this filter goes at this node - ptree->data( filter ); - } - else if ( ptree->right() == 0 && ptree->left() == 0 ) - { - // this will be the case when there is a single node in the tree - // that contains a filter. Here we want to make the root node an - // 'and' operator, push the existing down to the lhs and make a - // new node for the new filter - ParseTree* newLhs = new ParseTree( ptree->data() ); - ParseTree* newRhs = new ParseTree( filter ); - - Operator* op = new Operator(); - op->data("and"); - - ptree->data( op ); - ptree->left( newLhs ); - ptree->right( newRhs ); - } - else - { - // this will be the case once we have a tree with an 'and' at the - // root node, a filter in the lhs, and an arbitrary height tree - // with the same properties on the rhs. Because all operators - // are guaranteed to be and for now we simply insert a new rhs - // node and "push down" the existing tree - Operator* op = new Operator(); - op->data("and"); - - ParseTree* newRhs = new ParseTree( op ); - newRhs->left( new ParseTree( filter ) ); - newRhs->right( ptree->right() ); - ptree->right( newRhs ); - } + ParseTree* newRhs = new ParseTree(op); + newRhs->left(new ParseTree(filter)); + newRhs->right(ptree->right()); + ptree->right(newRhs); + } } void updateParseTree(boost::shared_ptr& csc, - execplan::CalpontSelectExecutionPlan*& csep, - execplan::SimpleColumn* sc, + execplan::CalpontSelectExecutionPlan*& csep, execplan::SimpleColumn* sc, const std::string& relop, pair cval) { - execplan::ConstantColumn* cc = 0; + execplan::ConstantColumn* cc = 0; - if (cval.first == 0) - cc = createConstCol(cval.second, static_cast(atoll(cval.second.c_str()))); - else - cc = createConstCol(cval.second); + if (cval.first == 0) + cc = createConstCol(cval.second, static_cast(atoll(cval.second.c_str()))); + else + cc = createConstCol(cval.second); - if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty()) - sc->schemaName(DefaultSchema); + if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty()) + sc->schemaName(DefaultSchema); - execplan::SimpleFilter* sf = 0; - sf = createSimpleFilter(csc, - execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), - relop, cc); - execplan::ParseTree* ptp = 0; - ptp = csep->filters(); + execplan::SimpleFilter* sf = 0; + sf = createSimpleFilter(csc, execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), relop, + cc); + execplan::ParseTree* ptp = 0; + ptp = csep->filters(); - if (ptp == 0) - ptp = new execplan::ParseTree(); + if (ptp == 0) + ptp = new execplan::ParseTree(); - appendSimpleFilter(ptp, sf); - csep->filters(ptp); + appendSimpleFilter(ptp, sf); + csep->filters(ptp); } -//template instantiations +// template instantiations template ConstantColumn* createConstCol(const string& valstr, int64_t val); -} //namespace qfe::utils -} //namespace qfe - +} // namespace utils +} // namespace qfe diff --git a/tools/qfe/cseputils.h b/tools/qfe/cseputils.h index 5ac622757..2b8fa4d77 100644 --- a/tools/qfe/cseputils.h +++ b/tools/qfe/cseputils.h @@ -9,7 +9,7 @@ class ConstantColumn; class SimpleFilter; class ParseTree; class SimpleColumn; -} +} // namespace execplan #include "calpontsystemcatalog.h" namespace qfe @@ -21,26 +21,16 @@ execplan::ConstantColumn* createConstCol(const std::string& valstr); template execplan::ConstantColumn* createConstCol(const std::string& valstr, T val); -execplan::SimpleFilter* createSimpleFilter -( - boost::shared_ptr& csc, - const execplan::CalpontSystemCatalog::TableColName& tcn, - const std::string& opstr, - execplan::ConstantColumn* cc -); +execplan::SimpleFilter* createSimpleFilter(boost::shared_ptr& csc, + const execplan::CalpontSystemCatalog::TableColName& tcn, + const std::string& opstr, execplan::ConstantColumn* cc); -void appendSimpleFilter -( - execplan::ParseTree*& ptree, - execplan::SimpleFilter* filter -); +void appendSimpleFilter(execplan::ParseTree*& ptree, execplan::SimpleFilter* filter); void updateParseTree(boost::shared_ptr&, - execplan::CalpontSelectExecutionPlan*&, - execplan::SimpleColumn*, - const std::string&, pair); + execplan::CalpontSelectExecutionPlan*&, execplan::SimpleColumn*, const std::string&, + pair); -} //namespace qfe::utils -} //namespace qfe +} // namespace utils +} // namespace qfe #endif - diff --git a/tools/qfe/ddlstmts.cpp b/tools/qfe/ddlstmts.cpp index 1da2e30ff..d25ffd893 100644 --- a/tools/qfe/ddlstmts.cpp +++ b/tools/qfe/ddlstmts.cpp @@ -27,56 +27,54 @@ using namespace qfe; int processDDLStmt(const string& stmt, uint32_t sid) { - MessageQueueClient* mq = 0; - SqlParser parser; + MessageQueueClient* mq = 0; + SqlParser parser; - parser.setDefaultSchema(DefaultSchema); - parser.Parse(stmt.c_str()); + parser.setDefaultSchema(DefaultSchema); + parser.Parse(stmt.c_str()); - if (parser.Good()) - { - const ddlpackage::ParseTree& ptree = parser.GetParseTree(); - SqlStatement& ddlstmt = *ptree.fList[0]; - ddlstmt.fSessionID = sid; - ddlstmt.fSql = stmt; - ddlstmt.fOwner = DefaultSchema; - ByteStream bytestream; - bytestream << ddlstmt.fSessionID; - ddlstmt.serialize(bytestream); - mq = new MessageQueueClient("DDLProc"); - scoped_ptr smq(mq); - ByteStream::byte b = 0; - mq->write(bytestream); - bytestream = mq->read(); - bytestream >> b; - string emsg; - bytestream >> emsg; + if (parser.Good()) + { + const ddlpackage::ParseTree& ptree = parser.GetParseTree(); + SqlStatement& ddlstmt = *ptree.fList[0]; + ddlstmt.fSessionID = sid; + ddlstmt.fSql = stmt; + ddlstmt.fOwner = DefaultSchema; + ByteStream bytestream; + bytestream << ddlstmt.fSessionID; + ddlstmt.serialize(bytestream); + mq = new MessageQueueClient("DDLProc"); + scoped_ptr smq(mq); + ByteStream::byte b = 0; + mq->write(bytestream); + bytestream = mq->read(); + bytestream >> b; + string emsg; + bytestream >> emsg; - if (b != 0) - throw runtime_error(emsg); - } - else - { - throw runtime_error("syntax error"); - } + if (b != 0) + throw runtime_error(emsg); + } + else + { + throw runtime_error("syntax error"); + } - return 0; + return 0; } -} +} // namespace namespace qfe { - void processCreateStmt(const string& stmt, uint32_t sid) { - processDDLStmt(stmt, sid); + processDDLStmt(stmt, sid); } void processDropStmt(const string& stmt, uint32_t sid) { - processDDLStmt(stmt, sid); + processDDLStmt(stmt, sid); } -} //namespace qfe - +} // namespace qfe diff --git a/tools/qfe/ddlstmts.h b/tools/qfe/ddlstmts.h index c3f02a3ef..f86a5552b 100644 --- a/tools/qfe/ddlstmts.h +++ b/tools/qfe/ddlstmts.h @@ -8,7 +8,6 @@ namespace qfe { void processCreateStmt(const std::string&, uint32_t); void processDropStmt(const std::string&, uint32_t); -} +} // namespace qfe #endif - diff --git a/tools/qfe/lex-win.cpp b/tools/qfe/lex-win.cpp index 62245c7be..9d9f20901 100644 --- a/tools/qfe/lex-win.cpp +++ b/tools/qfe/lex-win.cpp @@ -29,7 +29,6 @@ #include - /* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */ #ifdef c_plusplus #ifndef __cplusplus @@ -37,7 +36,6 @@ #endif #endif - #ifdef __cplusplus #include @@ -49,19 +47,19 @@ /* The "const" storage-class-modifier is valid. */ #define YY_USE_CONST -#else /* ! __cplusplus */ +#else /* ! __cplusplus */ #if __STDC__ #define YY_USE_PROTOS #define YY_USE_CONST -#endif /* __STDC__ */ -#endif /* ! __cplusplus */ +#endif /* __STDC__ */ +#endif /* ! __cplusplus */ #ifdef __TURBOC__ -#pragma warn -rch -#pragma warn -use +#pragma warn - rch +#pragma warn - use #include #include #define YY_USE_CONST @@ -74,7 +72,6 @@ #define yyconst #endif - #ifdef YY_USE_PROTOS #define YY_PROTO(proto) proto #else @@ -89,7 +86,7 @@ * we want to instead treat it as an 8-bit unsigned char, hence the * double cast. */ -#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) +#define YY_SC_TO_UI(c) ((unsigned int)(unsigned char)c) /* Enter a start condition. This macro really ought to take a parameter, * but we do it the disgusting crufty way forced on us by the ()-less @@ -108,7 +105,7 @@ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin ) +#define YY_NEW_FILE yyrestart(yyin) #define YY_END_OF_BUFFER_CHAR 0 @@ -118,7 +115,7 @@ typedef struct yy_buffer_state* YY_BUFFER_STATE; extern int yyleng; -extern FILE* yyin, *yyout; +extern FILE *yyin, *yyout; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 @@ -140,18 +137,17 @@ extern FILE* yyin, *yyout; /* Return all but the first 'n' matched characters back to the input stream. */ -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - *yy_cp = yy_hold_char; \ - YY_RESTORE_YY_MORE_OFFSET \ - yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ - YY_DO_BEFORE_ACTION; /* set up yytext again */ \ - } \ - while ( 0 ) +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + *yy_cp = yy_hold_char; \ + YY_RESTORE_YY_MORE_OFFSET \ + yy_c_buf_p = yy_cp = yy_bp + n - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } while (0) -#define unput(c) yyunput( c, yytext_ptr ) +#define unput(c) yyunput(c, yytext_ptr) /* The following is because we cannot portably get our hands on size_t * (without autoconf's help, which isn't available because we want @@ -159,61 +155,60 @@ extern FILE* yyin, *yyout; */ typedef unsigned int yy_size_t; - struct yy_buffer_state { - FILE* yy_input_file; + FILE* yy_input_file; - char* yy_ch_buf; /* input buffer */ - char* yy_buf_pos; /* current position in input buffer */ + char* yy_ch_buf; /* input buffer */ + char* yy_buf_pos; /* current position in input buffer */ - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - yy_size_t yy_buf_size; + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + yy_size_t yy_buf_size; - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - int yy_n_chars; + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; - int yy_buffer_status; + int yy_buffer_status; #define YY_BUFFER_NEW 0 #define YY_BUFFER_NORMAL 1 - /* When an EOF's been seen but there's still some text to process - * then we mark the buffer as YY_EOF_PENDING, to indicate that we - * shouldn't try reading from the input source any more. We might - * still have a bunch of tokens to match, though, because of - * possible backing-up. - * - * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by - * just pointing yyin at a new input file. - */ + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ #define YY_BUFFER_EOF_PENDING 2 }; @@ -225,243 +220,160 @@ static YY_BUFFER_STATE yy_current_buffer = 0; */ #define YY_CURRENT_BUFFER yy_current_buffer - /* yy_hold_char holds the character lost when yytext is formed. */ static char yy_hold_char; -static int yy_n_chars; /* number of characters read into yy_ch_buf */ - +static int yy_n_chars; /* number of characters read into yy_ch_buf */ int yyleng; /* Points to current character in buffer. */ -static char* yy_c_buf_p = (char*) 0; -static int yy_init = 1; /* whether we need to initialize */ -static int yy_start = 0; /* start state number */ +static char* yy_c_buf_p = (char*)0; +static int yy_init = 1; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ /* Flag which is used to allow yywrap()'s to do buffer switches * instead of setting up a fresh yyin. A bit of a hack ... */ static int yy_did_buffer_switch_on_eof; -void yyrestart YY_PROTO(( FILE* input_file )); +void yyrestart YY_PROTO((FILE * input_file)); -void yy_switch_to_buffer YY_PROTO(( YY_BUFFER_STATE new_buffer )); -void yy_load_buffer_state YY_PROTO(( void )); -YY_BUFFER_STATE yy_create_buffer YY_PROTO(( FILE* file, int size )); -void yy_delete_buffer YY_PROTO(( YY_BUFFER_STATE b )); -void yy_init_buffer YY_PROTO(( YY_BUFFER_STATE b, FILE* file )); -void yy_flush_buffer YY_PROTO(( YY_BUFFER_STATE b )); -#define YY_FLUSH_BUFFER yy_flush_buffer( yy_current_buffer ) +void yy_switch_to_buffer YY_PROTO((YY_BUFFER_STATE new_buffer)); +void yy_load_buffer_state YY_PROTO((void)); +YY_BUFFER_STATE yy_create_buffer YY_PROTO((FILE * file, int size)); +void yy_delete_buffer YY_PROTO((YY_BUFFER_STATE b)); +void yy_init_buffer YY_PROTO((YY_BUFFER_STATE b, FILE* file)); +void yy_flush_buffer YY_PROTO((YY_BUFFER_STATE b)); +#define YY_FLUSH_BUFFER yy_flush_buffer(yy_current_buffer) -YY_BUFFER_STATE yy_scan_buffer YY_PROTO(( char* base, yy_size_t size )); -YY_BUFFER_STATE yy_scan_string YY_PROTO(( yyconst char* yy_str )); -YY_BUFFER_STATE yy_scan_bytes YY_PROTO(( yyconst char* bytes, int len )); +YY_BUFFER_STATE yy_scan_buffer YY_PROTO((char* base, yy_size_t size)); +YY_BUFFER_STATE yy_scan_string YY_PROTO((yyconst char* yy_str)); +YY_BUFFER_STATE yy_scan_bytes YY_PROTO((yyconst char* bytes, int len)); -static void* yy_flex_alloc YY_PROTO(( yy_size_t )); -static void* yy_flex_realloc YY_PROTO(( void*, yy_size_t )); -static void yy_flex_free YY_PROTO(( void* )); +static void* yy_flex_alloc YY_PROTO((yy_size_t)); +static void* yy_flex_realloc YY_PROTO((void*, yy_size_t)); +static void yy_flex_free YY_PROTO((void*)); #define yy_new_buffer yy_create_buffer -#define yy_set_interactive(is_interactive) \ - { \ - if ( ! yy_current_buffer ) \ - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ - yy_current_buffer->yy_is_interactive = is_interactive; \ - } +#define yy_set_interactive(is_interactive) \ + { \ + if (!yy_current_buffer) \ + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); \ + yy_current_buffer->yy_is_interactive = is_interactive; \ + } -#define yy_set_bol(at_bol) \ - { \ - if ( ! yy_current_buffer ) \ - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); \ - yy_current_buffer->yy_at_bol = at_bol; \ - } +#define yy_set_bol(at_bol) \ + { \ + if (!yy_current_buffer) \ + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); \ + yy_current_buffer->yy_at_bol = at_bol; \ + } #define YY_AT_BOL() (yy_current_buffer->yy_at_bol) - #define YY_USES_REJECT #define yywrap() 1 #define YY_SKIP_YYWRAP typedef unsigned char YY_CHAR; -FILE* yyin = (FILE*) 0, *yyout = (FILE*) 0; +FILE *yyin = (FILE*)0, *yyout = (FILE*)0; typedef int yy_state_type; extern int yylineno; int yylineno = 1; extern char* yytext; #define yytext_ptr yytext -static yy_state_type yy_get_previous_state YY_PROTO(( void )); -static yy_state_type yy_try_NUL_trans YY_PROTO(( yy_state_type current_state )); -static int yy_get_next_buffer YY_PROTO(( void )); -static void yy_fatal_error YY_PROTO(( yyconst char msg[] )); +static yy_state_type yy_get_previous_state YY_PROTO((void)); +static yy_state_type yy_try_NUL_trans YY_PROTO((yy_state_type current_state)); +static int yy_get_next_buffer YY_PROTO((void)); +static void yy_fatal_error YY_PROTO((yyconst char msg[])); /* Done after the current pattern has been matched and before the * corresponding action - sets up yytext. */ -#define YY_DO_BEFORE_ACTION \ - yytext_ptr = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ - yy_hold_char = *yy_cp; \ - *yy_cp = '\0'; \ - yy_c_buf_p = yy_cp; +#define YY_DO_BEFORE_ACTION \ + yytext_ptr = yy_bp; \ + yyleng = (int)(yy_cp - yy_bp); \ + yy_hold_char = *yy_cp; \ + *yy_cp = '\0'; \ + yy_c_buf_p = yy_cp; #define YY_NUM_RULES 30 #define YY_END_OF_BUFFER 31 -static yyconst short int yy_acclist[125] = -{ - 0, - 27, 27, 18, 18, 31, 29, 30, 27, 29, 30, - 28, 30, 17, 29, 30, 29, 30, 20, 29, 30, - 23, 29, 30, 26, 29, 30, 21, 29, 30, 16, - 29, 30, 16, 29, 30, 16, 29, 30, 16, 29, - 30, 16, 29, 30, 16, 29, 30, 16, 29, 30, - 16, 29, 30, 16, 29, 30, 16, 29, 30, 18, - 30, 19, 30, 27, 20, 24, 25, 22, 16, 16, - 11, 16, 16, 16, 16, 16, 16, 16, 16, 6, - 16, 16, 16, 16, 18, 5, 16, 9, 16, 15, - 16, 16, 16, 16, 16, 14, 16, 13, 16, 16, +static yyconst short int yy_acclist[125] = { + 0, 27, 27, 18, 18, 31, 29, 30, 27, 29, 30, 28, 30, 17, 29, 30, 29, 30, 20, 29, 30, 23, 29, 30, 26, 29, + 30, 21, 29, 30, 16, 29, 30, 16, 29, 30, 16, 29, 30, 16, 29, 30, 16, 29, 30, 16, 29, 30, 16, 29, 30, 16, + 29, 30, 16, 29, 30, 16, 29, 30, 18, 30, 19, 30, 27, 20, 24, 25, 22, 16, 16, 11, 16, 16, 16, 16, 16, 16, + 16, 16, 6, 16, 16, 16, 16, 18, 5, 16, 9, 16, 15, 16, 16, 16, 16, 16, 14, 16, 13, 16, 16, - 16, 12, 16, 16, 10, 16, 2, 16, 16, 16, - 16, 16, 16, 16, 7, 16, 16, 16, 3, 16, - 1, 16, 4, 8 -} ; + 16, 12, 16, 16, 10, 16, 2, 16, 16, 16, 16, 16, 16, 16, 7, 16, 16, 16, 3, 16, 1, 16, 4, 8}; -static yyconst short int yy_accept[81] = -{ - 0, - 1, 2, 3, 4, 5, 6, 8, 11, 13, 16, - 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, - 48, 51, 54, 57, 60, 62, 64, 65, 66, 67, - 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, - 79, 80, 82, 83, 84, 85, 86, 88, 90, 92, - 93, 94, 95, 96, 98, 100, 101, 102, 104, 105, - 107, 109, 110, 111, 112, 113, 114, 115, 117, 118, - 119, 121, 121, 121, 123, 123, 123, 124, 125, 125 -} ; +static yyconst short int yy_accept[81] = { + 0, 1, 2, 3, 4, 5, 6, 8, 11, 13, 16, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, + 48, 51, 54, 57, 60, 62, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, 79, + 80, 82, 83, 84, 85, 86, 88, 90, 92, 93, 94, 95, 96, 98, 100, 101, 102, 104, 105, 107, 109, + 110, 111, 112, 113, 114, 115, 117, 118, 119, 121, 121, 121, 123, 123, 123, 124, 125, 125}; -static yyconst int yy_ec[256] = -{ - 0, - 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 4, 1, 1, 1, 1, 1, 1, 5, 1, - 1, 1, 6, 1, 6, 1, 1, 7, 7, 7, - 7, 7, 7, 7, 7, 7, 7, 1, 1, 8, - 9, 10, 1, 1, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 11, 11, 21, 22, 23, 24, 25, - 11, 26, 27, 28, 29, 30, 31, 32, 33, 11, - 1, 1, 1, 1, 11, 1, 12, 13, 14, 15, +static yyconst int yy_ec[256] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 5, 1, 1, 1, 6, 1, 6, 1, 1, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 1, 1, 8, 9, 10, 1, 1, 12, 13, 14, 15, 16, 17, 18, 19, 20, 11, 11, 21, 22, + 23, 24, 25, 11, 26, 27, 28, 29, 30, 31, 32, 33, 11, 1, 1, 1, 1, 11, 1, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 11, 11, 21, 22, 23, - 24, 25, 11, 26, 27, 28, 29, 30, 31, 32, - 33, 11, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 16, 17, 18, 19, 20, 11, 11, 21, 22, 23, 24, 25, 11, 26, 27, 28, 29, 30, 31, 32, 33, 11, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 -} ; + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1}; -static yyconst int yy_meta[34] = -{ - 0, - 1, 1, 1, 1, 2, 1, 3, 1, 1, 1, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3 -} ; +static yyconst int yy_meta[34] = {0, 1, 1, 1, 1, 2, 1, 3, 1, 1, 1, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}; -static yyconst short int yy_base[83] = -{ - 0, - 0, 0, 92, 91, 95, 98, 32, 98, 98, 87, - 86, 28, 98, 83, 0, 12, 75, 64, 63, 68, - 28, 61, 25, 67, 0, 98, 41, 78, 98, 98, - 98, 0, 69, 69, 64, 54, 56, 55, 56, 45, - 53, 60, 53, 51, 56, 0, 0, 0, 0, 57, - 48, 40, 48, 0, 0, 51, 50, 0, 39, 0, - 0, 39, 35, 36, 47, 43, 49, 0, 48, 23, - 0, 37, 36, 0, 14, 13, 98, 98, 98, 54, - 41, 57 -} ; +static yyconst short int yy_base[83] = {0, 0, 0, 92, 91, 95, 98, 32, 98, 98, 87, 86, 28, 98, 83, 0, 12, + 75, 64, 63, 68, 28, 61, 25, 67, 0, 98, 41, 78, 98, 98, 98, 0, 69, + 69, 64, 54, 56, 55, 56, 45, 53, 60, 53, 51, 56, 0, 0, 0, 0, 57, + 48, 40, 48, 0, 0, 51, 50, 0, 39, 0, 0, 39, 35, 36, 47, 43, 49, + 0, 48, 23, 0, 37, 36, 0, 14, 13, 98, 98, 98, 54, 41, 57}; -static yyconst short int yy_def[83] = -{ - 0, - 79, 1, 80, 80, 79, 79, 79, 79, 79, 79, - 79, 79, 79, 79, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 82, 79, 79, 79, 79, 79, - 79, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 82, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 79, 79, 81, 79, 79, 79, 79, 0, 79, - 79, 79 -} ; +static yyconst short int yy_def[83] = {0, 79, 1, 80, 80, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 82, 79, 79, 79, 79, 79, 79, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 82, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 79, 79, 81, 79, 79, 79, 79, 0, 79, 79, 79}; -static yyconst short int yy_nxt[132] = -{ - 0, - 6, 7, 8, 7, 9, 10, 11, 12, 13, 14, - 15, 16, 15, 15, 17, 15, 18, 19, 15, 15, - 20, 21, 15, 22, 15, 15, 23, 15, 15, 15, - 24, 15, 15, 27, 33, 27, 29, 30, 34, 40, - 43, 35, 27, 32, 27, 78, 77, 41, 76, 75, - 74, 73, 72, 44, 25, 25, 25, 46, 71, 46, - 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, - 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, - 50, 49, 48, 47, 28, 45, 42, 39, 38, 37, - 36, 31, 28, 28, 79, 26, 26, 5, 79, 79, +static yyconst short int yy_nxt[132] = { + 0, 6, 7, 8, 7, 9, 10, 11, 12, 13, 14, 15, 16, 15, 15, 17, 15, 18, 19, 15, 15, 20, 21, 15, 22, 15, + 15, 23, 15, 15, 15, 24, 15, 15, 27, 33, 27, 29, 30, 34, 40, 43, 35, 27, 32, 27, 78, 77, 41, 76, 75, 74, + 73, 72, 44, 25, 25, 25, 46, 71, 46, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, + 53, 52, 51, 50, 49, 48, 47, 28, 45, 42, 39, 38, 37, 36, 31, 28, 28, 79, 26, 26, 5, 79, 79, - 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79 -} ; + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79}; -static yyconst short int yy_chk[132] = -{ - 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 7, 16, 7, 12, 12, 16, 21, - 23, 16, 27, 81, 27, 76, 75, 21, 73, 72, - 70, 69, 67, 23, 80, 80, 80, 82, 66, 82, - 65, 64, 63, 62, 59, 57, 56, 53, 52, 51, - 50, 45, 44, 43, 42, 41, 40, 39, 38, 37, - 36, 35, 34, 33, 28, 24, 22, 20, 19, 18, - 17, 14, 11, 10, 5, 4, 3, 79, 79, 79, +static yyconst short int yy_chk[132] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 7, 16, 7, 12, 12, 16, 21, 23, 16, 27, 81, 27, 76, 75, 21, 73, 72, 70, + 69, 67, 23, 80, 80, 80, 82, 66, 82, 65, 64, 63, 62, 59, 57, 56, 53, 52, 51, 50, 45, 44, 43, 42, 41, 40, + 39, 38, 37, 36, 35, 34, 33, 28, 24, 22, 20, 19, 18, 17, 14, 11, 10, 5, 4, 3, 79, 79, 79, - 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79 -} ; + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79}; static yy_state_type yy_state_buf[YY_BUF_SIZE + 2], *yy_state_ptr; static char* yy_full_match; static int yy_lp; -#define REJECT \ -{ \ -*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ \ -yy_cp = yy_full_match; /* restore poss. backed-over text */ \ -++yy_lp; \ -goto find_rule; \ -} +#define REJECT \ + { \ + *yy_cp = yy_hold_char; /* undo effects of setting up yytext */ \ + yy_cp = yy_full_match; /* restore poss. backed-over text */ \ + ++yy_lp; \ + goto find_rule; \ + } #define yymore() yymore_used_but_not_detected #define YY_MORE_ADJ 0 #define YY_RESTORE_YY_MORE_OFFSET @@ -493,29 +405,29 @@ extern std::string* newstr(const char* cp); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap YY_PROTO(( void )); +extern "C" int yywrap YY_PROTO((void)); #else -extern int yywrap YY_PROTO(( void )); +extern int yywrap YY_PROTO((void)); #endif #endif #ifndef YY_NO_UNPUT -static void yyunput YY_PROTO(( int c, char* buf_ptr )); +static void yyunput YY_PROTO((int c, char* buf_ptr)); #endif #ifndef yytext_ptr -static void yy_flex_strncpy YY_PROTO(( char*, yyconst char*, int )); +static void yy_flex_strncpy YY_PROTO((char*, yyconst char*, int)); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen YY_PROTO(( yyconst char* )); +static int yy_flex_strlen YY_PROTO((yyconst char*)); #endif #ifndef YY_NO_INPUT #ifdef __cplusplus -static int yyinput YY_PROTO(( void )); +static int yyinput YY_PROTO((void)); #else -static int input YY_PROTO(( void )); +static int input YY_PROTO((void)); #endif #endif @@ -524,13 +436,13 @@ static int yy_start_stack_ptr = 0; static int yy_start_stack_depth = 0; static int* yy_start_stack = 0; #ifndef YY_NO_PUSH_STATE -static void yy_push_state YY_PROTO(( int new_state )); +static void yy_push_state YY_PROTO((int new_state)); #endif #ifndef YY_NO_POP_STATE -static void yy_pop_state YY_PROTO(( void )); +static void yy_pop_state YY_PROTO((void)); #endif #ifndef YY_NO_TOP_STATE -static int yy_top_state YY_PROTO(( void )); +static int yy_top_state YY_PROTO((void)); #endif #else @@ -565,29 +477,27 @@ YY_MALLOC_DECL /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO (void) fwrite( yytext, yyleng, 1, yyout ) +#define ECHO (void)fwrite(yytext, yyleng, 1, yyout) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, * is returned in "result". */ #ifndef YY_INPUT -#define YY_INPUT(buf,result,max_size) \ - if ( yy_current_buffer->yy_is_interactive ) \ - { \ - int c = '*', n; \ - for ( n = 0; n < max_size && \ - (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ - buf[n] = (char) c; \ - if ( c == '\n' ) \ - buf[n++] = (char) c; \ - if ( c == EOF && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); \ - result = n; \ - } \ - else if ( ((result = fread( buf, 1, max_size, yyin )) == 0) \ - && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); +#define YY_INPUT(buf, result, max_size) \ + if (yy_current_buffer->yy_is_interactive) \ + { \ + int c = '*', n; \ + for (n = 0; n < max_size && (c = getc(yyin)) != EOF && c != '\n'; ++n) \ + buf[n] = (char)c; \ + if (c == '\n') \ + buf[n++] = (char)c; \ + if (c == EOF && ferror(yyin)) \ + YY_FATAL_ERROR("input in flex scanner failed"); \ + result = n; \ + } \ + else if (((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin)) \ + YY_FATAL_ERROR("input in flex scanner failed"); #endif /* No semi-colon after return; correct usage is to write "yyterminate();" - @@ -605,14 +515,14 @@ YY_MALLOC_DECL /* Report a fatal error. */ #ifndef YY_FATAL_ERROR -#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#define YY_FATAL_ERROR(msg) yy_fatal_error(msg) #endif /* Default declaration of generated scanner - a define so the user can * easily add parameters. */ #ifndef YY_DECL -#define YY_DECL int yylex YY_PROTO(( void )) +#define YY_DECL int yylex YY_PROTO((void)) #endif /* Code executed at the beginning of each rule, after yytext and yyleng @@ -627,434 +537,466 @@ YY_MALLOC_DECL #define YY_BREAK break; #endif -#define YY_RULE_SETUP \ - YY_USER_ACTION +#define YY_RULE_SETUP YY_USER_ACTION YY_DECL { - register yy_state_type yy_current_state; - register char* yy_cp, *yy_bp; - register int yy_act; + register yy_state_type yy_current_state; + register char *yy_cp, *yy_bp; + register int yy_act; #line 28 "qfelexer.lpp" - #line 635 "lex-win.cpp" - if ( yy_init ) - { - yy_init = 0; + if (yy_init) + { + yy_init = 0; #ifdef YY_USER_INIT - YY_USER_INIT; + YY_USER_INIT; #endif - if ( ! yy_start ) - yy_start = 1; /* first start state */ + if (!yy_start) + yy_start = 1; /* first start state */ - if ( ! yyin ) - yyin = stdin; + if (!yyin) + yyin = stdin; - if ( ! yyout ) - yyout = stdout; + if (!yyout) + yyout = stdout; - if ( ! yy_current_buffer ) - yy_current_buffer = - yy_create_buffer( yyin, YY_BUF_SIZE ); + if (!yy_current_buffer) + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); - yy_load_buffer_state(); + yy_load_buffer_state(); + } + + while (1) /* loops until end-of-file is reached */ + { + yy_cp = yy_c_buf_p; + + /* Support of yytext. */ + *yy_cp = yy_hold_char; + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = yy_start; + yy_state_ptr = yy_state_buf; + *yy_state_ptr++ = yy_current_state; + yy_match: + + do + { + register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; + + if (yy_current_state >= 80) + yy_c = yy_meta[(unsigned int)yy_c]; + } + + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + *yy_state_ptr++ = yy_current_state; + ++yy_cp; + } while (yy_base[yy_current_state] != 98); + + yy_find_action: + yy_current_state = *--yy_state_ptr; + yy_lp = yy_accept[yy_current_state]; + find_rule: /* we branch to this label when backing up */ + + for (;;) /* until we find what rule we matched */ + { + if (yy_lp && yy_lp < yy_accept[yy_current_state + 1]) + { + yy_act = yy_acclist[yy_lp]; + { + yy_full_match = yy_cp; + break; + } + } + + --yy_cp; + yy_current_state = *--yy_state_ptr; + yy_lp = yy_accept[yy_current_state]; } - while ( 1 ) /* loops until end-of-file is reached */ + YY_DO_BEFORE_ACTION; + + if (yy_act != YY_END_OF_BUFFER) { - yy_cp = yy_c_buf_p; + int yyl; - /* Support of yytext. */ - *yy_cp = yy_hold_char; + for (yyl = 0; yyl < yyleng; ++yyl) + if (yytext[yyl] == '\n') + ++yylineno; + } - /* yy_bp points to the position in yy_ch_buf of the start of - * the current run. - */ - yy_bp = yy_cp; + do_action: /* This label is used only to access EOF actions. */ - yy_current_state = yy_start; - yy_state_ptr = yy_state_buf; - *yy_state_ptr++ = yy_current_state; -yy_match: - - do - { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; - - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - - if ( yy_current_state >= 80 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - *yy_state_ptr++ = yy_current_state; - ++yy_cp; - } - while ( yy_base[yy_current_state] != 98 ); - -yy_find_action: - yy_current_state = *--yy_state_ptr; - yy_lp = yy_accept[yy_current_state]; -find_rule: /* we branch to this label when backing up */ - - for ( ; ; ) /* until we find what rule we matched */ - { - if ( yy_lp && yy_lp < yy_accept[yy_current_state + 1] ) - { - yy_act = yy_acclist[yy_lp]; - { - yy_full_match = yy_cp; - break; - } - } - - --yy_cp; - yy_current_state = *--yy_state_ptr; - yy_lp = yy_accept[yy_current_state]; - } - - YY_DO_BEFORE_ACTION; - - if ( yy_act != YY_END_OF_BUFFER ) - { - int yyl; - - for ( yyl = 0; yyl < yyleng; ++yyl ) - if ( yytext[yyl] == '\n' ) - ++yylineno; - } - -do_action: /* This label is used only to access EOF actions. */ - - - switch ( yy_act ) - { - /* beginning of action switch */ - case 1: - YY_RULE_SETUP + switch (yy_act) + { + /* beginning of action switch */ + case 1: YY_RULE_SETUP #line 30 "qfelexer.lpp" - { return QFEP_SELECT; } - YY_BREAK + { + return QFEP_SELECT; + } + YY_BREAK - case 2: - YY_RULE_SETUP + case 2: YY_RULE_SETUP #line 31 "qfelexer.lpp" - { return QFEP_FROM; } - YY_BREAK + { + return QFEP_FROM; + } + YY_BREAK - case 3: - YY_RULE_SETUP + case 3: YY_RULE_SETUP #line 32 "qfelexer.lpp" - { return QFEP_WHERE; } - YY_BREAK + { + return QFEP_WHERE; + } + YY_BREAK - case 4: - YY_RULE_SETUP + case 4: YY_RULE_SETUP #line 33 "qfelexer.lpp" - { return GROUPBY; } - YY_BREAK + { + return GROUPBY; + } + YY_BREAK - case 5: - YY_RULE_SETUP + case 5: YY_RULE_SETUP #line 34 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return LOGICOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return LOGICOP; + } + YY_BREAK - case 6: - YY_RULE_SETUP + case 6: YY_RULE_SETUP #line 35 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return LOGICOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return LOGICOP; + } + YY_BREAK - case 7: - YY_RULE_SETUP + case 7: YY_RULE_SETUP #line 36 "qfelexer.lpp" - { return LIMIT; } - YY_BREAK + { + return LIMIT; + } + YY_BREAK - case 8: - YY_RULE_SETUP + case 8: YY_RULE_SETUP #line 37 "qfelexer.lpp" - { return ORDERBY; } - YY_BREAK + { + return ORDERBY; + } + YY_BREAK - case 9: - YY_RULE_SETUP + case 9: YY_RULE_SETUP #line 38 "qfelexer.lpp" - { return ASC; } - YY_BREAK + { + return ASC; + } + YY_BREAK - case 10: - YY_RULE_SETUP + case 10: YY_RULE_SETUP #line 39 "qfelexer.lpp" - { return DESC; } - YY_BREAK + { + return DESC; + } + YY_BREAK - case 11: - YY_RULE_SETUP + case 11: YY_RULE_SETUP #line 40 "qfelexer.lpp" - { return AS; } - YY_BREAK + { + return AS; + } + YY_BREAK - case 12: - YY_RULE_SETUP + case 12: YY_RULE_SETUP #line 41 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return FUNC; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return FUNC; + } + YY_BREAK - case 13: - YY_RULE_SETUP + case 13: YY_RULE_SETUP #line 42 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return FUNC; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return FUNC; + } + YY_BREAK - case 14: - YY_RULE_SETUP + case 14: YY_RULE_SETUP #line 43 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return FUNC; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return FUNC; + } + YY_BREAK - case 15: - YY_RULE_SETUP + case 15: YY_RULE_SETUP #line 44 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return FUNC; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return FUNC; + } + YY_BREAK - case 16: - YY_RULE_SETUP + case 16: YY_RULE_SETUP #line 46 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return OBJNAME; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return OBJNAME; + } + YY_BREAK - case 17: - YY_RULE_SETUP + case 17: YY_RULE_SETUP #line 48 "qfelexer.lpp" - { BEGIN(CCONST); } - YY_BREAK + { + BEGIN(CCONST); + } + YY_BREAK - case 18: - YY_RULE_SETUP + case 18: YY_RULE_SETUP #line 49 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return CHAR_CONST; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return CHAR_CONST; + } + YY_BREAK - case 19: - YY_RULE_SETUP + case 19: YY_RULE_SETUP #line 50 "qfelexer.lpp" - { BEGIN(INITIAL); } - YY_BREAK + { + BEGIN(INITIAL); + } + YY_BREAK - case 20: - YY_RULE_SETUP + case 20: YY_RULE_SETUP #line 52 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return INT_CONST; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return INT_CONST; + } + YY_BREAK - case 21: - YY_RULE_SETUP + case 21: YY_RULE_SETUP #line 54 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return RELOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return RELOP; + } + YY_BREAK - case 22: - YY_RULE_SETUP + case 22: YY_RULE_SETUP #line 55 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return RELOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return RELOP; + } + YY_BREAK - case 23: - YY_RULE_SETUP + case 23: YY_RULE_SETUP #line 56 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return RELOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return RELOP; + } + YY_BREAK - case 24: - YY_RULE_SETUP + case 24: YY_RULE_SETUP #line 57 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return RELOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return RELOP; + } + YY_BREAK - case 25: - YY_RULE_SETUP + case 25: YY_RULE_SETUP #line 58 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return RELOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return RELOP; + } + YY_BREAK - case 26: - YY_RULE_SETUP + case 26: YY_RULE_SETUP #line 59 "qfelexer.lpp" - { qfelval.cp = newstr(qfetext); return RELOP; } - YY_BREAK + { + qfelval.cp = newstr(qfetext); + return RELOP; + } + YY_BREAK - case 27: - YY_RULE_SETUP + case 27: YY_RULE_SETUP #line 61 "qfelexer.lpp" - {} - YY_BREAK + { + } + YY_BREAK - case 28: - YY_RULE_SETUP + case 28: YY_RULE_SETUP #line 62 "qfelexer.lpp" - { qfelineno++; } - YY_BREAK + { + qfelineno++; + } + YY_BREAK - case 29: - YY_RULE_SETUP + case 29: YY_RULE_SETUP #line 64 "qfelexer.lpp" - { return qfetext[0]; } - YY_BREAK + { + return qfetext[0]; + } + YY_BREAK - case 30: - YY_RULE_SETUP + case 30: YY_RULE_SETUP #line 66 "qfelexer.lpp" - ECHO; - YY_BREAK + ECHO; + YY_BREAK #line 876 "lex-win.cpp" - case YY_STATE_EOF(INITIAL): - case YY_STATE_EOF(CCONST): - yyterminate(); + case YY_STATE_EOF(INITIAL): + case YY_STATE_EOF(CCONST): yyterminate(); - case YY_END_OF_BUFFER: + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int)(yy_cp - yytext_ptr) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = yy_hold_char; + YY_RESTORE_YY_MORE_OFFSET + + if (yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between yy_current_buffer and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + yy_n_chars = yy_current_buffer->yy_n_chars; + yy_current_buffer->yy_input_file = yyin; + yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if (yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars]) + { + /* This was really a NUL. */ + yy_state_type yy_next_state; + + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans(yy_current_state); + + yy_bp = yytext_ptr + YY_MORE_ADJ; + + if (yy_next_state) + { + /* Consume the NUL. */ + yy_cp = ++yy_c_buf_p; + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = yy_c_buf_p; + goto yy_find_action; + } + } + + else + switch (yy_get_next_buffer()) + { + case EOB_ACT_END_OF_FILE: { - /* Amount of text matched not including the EOB char. */ - int yy_amount_of_matched_text = (int) (yy_cp - yytext_ptr) - 1; + yy_did_buffer_switch_on_eof = 0; - /* Undo the effects of YY_DO_BEFORE_ACTION. */ - *yy_cp = yy_hold_char; - YY_RESTORE_YY_MORE_OFFSET - - if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_NEW ) - { - /* We're scanning a new file or input source. It's - * possible that this happened because the user - * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure - * consistency between yy_current_buffer and our - * globals. Here is the right place to do so, because - * this is the first action (other than possibly a - * back-up) that will match for the new input source. - */ - yy_n_chars = yy_current_buffer->yy_n_chars; - yy_current_buffer->yy_input_file = yyin; - yy_current_buffer->yy_buffer_status = YY_BUFFER_NORMAL; - } - - /* Note that here we test for yy_c_buf_p "<=" to the position - * of the first EOB in the buffer, since yy_c_buf_p will - * already have been incremented past the NUL character - * (since all states make transitions on EOB to the - * end-of-buffer state). Contrast this with the test - * in input(). + if (yywrap()) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. */ - if ( yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars] ) - { - /* This was really a NUL. */ - yy_state_type yy_next_state; + yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; - yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } - yy_current_state = yy_get_previous_state(); + else + { + if (!yy_did_buffer_switch_on_eof) + YY_NEW_FILE; + } - /* Okay, we're now positioned to make the NUL - * transition. We couldn't have - * yy_get_previous_state() go ahead and do it - * for us because it doesn't know how to deal - * with the possibility of jamming (and we don't - * want to build jamming into it because then it - * will run more slowly). - */ - - yy_next_state = yy_try_NUL_trans( yy_current_state ); - - yy_bp = yytext_ptr + YY_MORE_ADJ; - - if ( yy_next_state ) - { - /* Consume the NUL. */ - yy_cp = ++yy_c_buf_p; - yy_current_state = yy_next_state; - goto yy_match; - } - - else - { - yy_cp = yy_c_buf_p; - goto yy_find_action; - } - } - - else switch ( yy_get_next_buffer() ) - { - case EOB_ACT_END_OF_FILE: - { - yy_did_buffer_switch_on_eof = 0; - - if ( yywrap() ) - { - /* Note: because we've taken care in - * yy_get_next_buffer() to have set up - * yytext, we can now set up - * yy_c_buf_p so that if some total - * hoser (like flex itself) wants to - * call the scanner after we return the - * YY_NULL, it'll still work - another - * YY_NULL will get returned. - */ - yy_c_buf_p = yytext_ptr + YY_MORE_ADJ; - - yy_act = YY_STATE_EOF(YY_START); - goto do_action; - } - - else - { - if ( ! yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; - } - - break; - } - - case EOB_ACT_CONTINUE_SCAN: - yy_c_buf_p = - yytext_ptr + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state(); - - yy_cp = yy_c_buf_p; - yy_bp = yytext_ptr + YY_MORE_ADJ; - goto yy_match; - - case EOB_ACT_LAST_MATCH: - yy_c_buf_p = - &yy_current_buffer->yy_ch_buf[yy_n_chars]; - - yy_current_state = yy_get_previous_state(); - - yy_cp = yy_c_buf_p; - yy_bp = yytext_ptr + YY_MORE_ADJ; - goto yy_find_action; - } - - break; + break; } - default: - YY_FATAL_ERROR( - "fatal flex scanner internal error--no action found" ); - } /* end of action switch */ - } /* end of scanning one token */ -} /* end of yylex */ + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext_ptr + yy_amount_of_matched_text; + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + yy_c_buf_p = &yy_current_buffer->yy_ch_buf[yy_n_chars]; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext_ptr + YY_MORE_ADJ; + goto yy_find_action; + } + + break; + } + + default: YY_FATAL_ERROR("fatal flex scanner internal error--no action found"); + } /* end of action switch */ + } /* end of scanning one token */ +} /* end of yylex */ /* yy_get_next_buffer - try to read in a new buffer * @@ -1066,166 +1008,155 @@ do_action: /* This label is used only to access EOF actions. */ static int yy_get_next_buffer() { - register char* dest = yy_current_buffer->yy_ch_buf; - register char* source = yytext_ptr; - register int number_to_move, i; - int ret_val; + register char* dest = yy_current_buffer->yy_ch_buf; + register char* source = yytext_ptr; + register int number_to_move, i; + int ret_val; - if ( yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1] ) - YY_FATAL_ERROR( - "fatal flex scanner internal error--end of buffer missed" ); + if (yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1]) + YY_FATAL_ERROR("fatal flex scanner internal error--end of buffer missed"); - if ( yy_current_buffer->yy_fill_buffer == 0 ) + if (yy_current_buffer->yy_fill_buffer == 0) + { + /* Don't try to fill the buffer, so this is an EOF. */ + if (yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1) { - /* Don't try to fill the buffer, so this is an EOF. */ - if ( yy_c_buf_p - yytext_ptr - YY_MORE_ADJ == 1 ) - { - /* We matched a single character, the EOB, so - * treat this as a final EOF. - */ - return EOB_ACT_END_OF_FILE; - } - - else - { - /* We matched some text prior to the EOB, first - * process it. - */ - return EOB_ACT_LAST_MATCH; - } + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; } - /* Try to read more data. */ - - /* First move last chars to start of buffer. */ - number_to_move = (int) (yy_c_buf_p - yytext_ptr) - 1; - - for ( i = 0; i < number_to_move; ++i ) - *(dest++) = *(source++); - - if ( yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING ) - /* don't do the read, it's not guaranteed to return an EOF, - * just force an EOF - */ - yy_current_buffer->yy_n_chars = yy_n_chars = 0; - else { - int num_to_read = - yy_current_buffer->yy_buf_size - number_to_move - 1; + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } - while ( num_to_read <= 0 ) - { - /* Not enough room in the buffer - grow it. */ + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int)(yy_c_buf_p - yytext_ptr) - 1; + + for (i = 0; i < number_to_move; ++i) + *(dest++) = *(source++); + + if (yy_current_buffer->yy_buffer_status == YY_BUFFER_EOF_PENDING) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + yy_current_buffer->yy_n_chars = yy_n_chars = 0; + + else + { + int num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; + + while (num_to_read <= 0) + { + /* Not enough room in the buffer - grow it. */ #ifdef YY_USES_REJECT - YY_FATAL_ERROR( - "input buffer overflow, can't enlarge buffer because scanner uses REJECT" ); + YY_FATAL_ERROR("input buffer overflow, can't enlarge buffer because scanner uses REJECT"); #else - /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = yy_current_buffer; + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = yy_current_buffer; - int yy_c_buf_p_offset = - (int) (yy_c_buf_p - b->yy_ch_buf); + int yy_c_buf_p_offset = (int)(yy_c_buf_p - b->yy_ch_buf); - if ( b->yy_is_our_buffer ) - { - int new_size = b->yy_buf_size * 2; + if (b->yy_is_our_buffer) + { + int new_size = b->yy_buf_size * 2; - if ( new_size <= 0 ) - b->yy_buf_size += b->yy_buf_size / 8; - else - b->yy_buf_size *= 2; + if (new_size <= 0) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; - b->yy_ch_buf = (char*) - /* Include room in for 2 EOB chars. */ - yy_flex_realloc( (void*) b->yy_ch_buf, - b->yy_buf_size + 2 ); - } - else - /* Can't grow it, we don't own it. */ - b->yy_ch_buf = 0; + b->yy_ch_buf = (char*) + /* Include room in for 2 EOB chars. */ + yy_flex_realloc((void*)b->yy_ch_buf, b->yy_buf_size + 2); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = 0; - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( - "fatal error - scanner input buffer overflow" ); + if (!b->yy_ch_buf) + YY_FATAL_ERROR("fatal error - scanner input buffer overflow"); - yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; + yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; - num_to_read = yy_current_buffer->yy_buf_size - - number_to_move - 1; + num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; #endif - } - - if ( num_to_read > YY_READ_BUF_SIZE ) - num_to_read = YY_READ_BUF_SIZE; - - /* Read in more data. */ - YY_INPUT( (&yy_current_buffer->yy_ch_buf[number_to_move]), - yy_n_chars, num_to_read ); - - yy_current_buffer->yy_n_chars = yy_n_chars; } - if ( yy_n_chars == 0 ) - { - if ( number_to_move == YY_MORE_ADJ ) - { - ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin ); - } + if (num_to_read > YY_READ_BUF_SIZE) + num_to_read = YY_READ_BUF_SIZE; - else - { - ret_val = EOB_ACT_LAST_MATCH; - yy_current_buffer->yy_buffer_status = - YY_BUFFER_EOF_PENDING; - } + /* Read in more data. */ + YY_INPUT((&yy_current_buffer->yy_ch_buf[number_to_move]), yy_n_chars, num_to_read); + + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + if (yy_n_chars == 0) + { + if (number_to_move == YY_MORE_ADJ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart(yyin); } else - ret_val = EOB_ACT_CONTINUE_SCAN; + { + ret_val = EOB_ACT_LAST_MATCH; + yy_current_buffer->yy_buffer_status = YY_BUFFER_EOF_PENDING; + } + } - yy_n_chars += number_to_move; - yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; - yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; + else + ret_val = EOB_ACT_CONTINUE_SCAN; - yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + yy_n_chars += number_to_move; + yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; + yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; - return ret_val; + yytext_ptr = &yy_current_buffer->yy_ch_buf[0]; + + return ret_val; } - /* yy_get_previous_state - get the state just before the EOB char was reached */ static yy_state_type yy_get_previous_state() { - register yy_state_type yy_current_state; - register char* yy_cp; + register yy_state_type yy_current_state; + register char* yy_cp; - yy_current_state = yy_start; - yy_state_ptr = yy_state_buf; - *yy_state_ptr++ = yy_current_state; + yy_current_state = yy_start; + yy_state_ptr = yy_state_buf; + *yy_state_ptr++ = yy_current_state; - for ( yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp ) + for (yy_cp = yytext_ptr + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp) + { + register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + yy_current_state = (int)yy_def[yy_current_state]; - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - - if ( yy_current_state >= 80 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - *yy_state_ptr++ = yy_current_state; + if (yy_current_state >= 80) + yy_c = yy_meta[(unsigned int)yy_c]; } - return yy_current_state; -} + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + *yy_state_ptr++ = yy_current_state; + } + return yy_current_state; +} /* yy_try_NUL_trans - try to make a transition on the NUL character * @@ -1234,81 +1165,74 @@ static yy_state_type yy_get_previous_state() */ #ifdef YY_USE_PROTOS -static yy_state_type yy_try_NUL_trans( yy_state_type yy_current_state ) +static yy_state_type yy_try_NUL_trans(yy_state_type yy_current_state) #else -static yy_state_type yy_try_NUL_trans( yy_current_state ) -yy_state_type yy_current_state; +static yy_state_type yy_try_NUL_trans(yy_current_state) yy_state_type yy_current_state; #endif { - register int yy_is_jam; + register int yy_is_jam; - register YY_CHAR yy_c = 1; + register YY_CHAR yy_c = 1; - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; + while (yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = (int)yy_def[yy_current_state]; - if ( yy_current_state >= 80 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } + if (yy_current_state >= 80) + yy_c = yy_meta[(unsigned int)yy_c]; + } - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 79); + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int)yy_c]; + yy_is_jam = (yy_current_state == 79); - if ( ! yy_is_jam ) - *yy_state_ptr++ = yy_current_state; + if (!yy_is_jam) + *yy_state_ptr++ = yy_current_state; - return yy_is_jam ? 0 : yy_current_state; + return yy_is_jam ? 0 : yy_current_state; } - #ifndef YY_NO_UNPUT #ifdef YY_USE_PROTOS -static void yyunput( int c, register char* yy_bp ) +static void yyunput(int c, register char* yy_bp) #else -static void yyunput( c, yy_bp ) -int c; +static void yyunput(c, yy_bp) int c; register char* yy_bp; #endif { - register char* yy_cp = yy_c_buf_p; + register char* yy_cp = yy_c_buf_p; - /* undo effects of setting up yytext */ - *yy_cp = yy_hold_char; + /* undo effects of setting up yytext */ + *yy_cp = yy_hold_char; - if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) - { - /* need to shift things up to make room */ - /* +2 for EOB chars. */ - register int number_to_move = yy_n_chars + 2; - register char* dest = &yy_current_buffer->yy_ch_buf[ - yy_current_buffer->yy_buf_size + 2]; - register char* source = - &yy_current_buffer->yy_ch_buf[number_to_move]; + if (yy_cp < yy_current_buffer->yy_ch_buf + 2) + { + /* need to shift things up to make room */ + /* +2 for EOB chars. */ + register int number_to_move = yy_n_chars + 2; + register char* dest = &yy_current_buffer->yy_ch_buf[yy_current_buffer->yy_buf_size + 2]; + register char* source = &yy_current_buffer->yy_ch_buf[number_to_move]; - while ( source > yy_current_buffer->yy_ch_buf ) - *--dest = *--source; + while (source > yy_current_buffer->yy_ch_buf) + *--dest = *--source; - yy_cp += (int) (dest - source); - yy_bp += (int) (dest - source); - yy_current_buffer->yy_n_chars = - yy_n_chars = yy_current_buffer->yy_buf_size; + yy_cp += (int)(dest - source); + yy_bp += (int)(dest - source); + yy_current_buffer->yy_n_chars = yy_n_chars = yy_current_buffer->yy_buf_size; - if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) - YY_FATAL_ERROR( "flex scanner push-back overflow" ); - } + if (yy_cp < yy_current_buffer->yy_ch_buf + 2) + YY_FATAL_ERROR("flex scanner push-back overflow"); + } - *--yy_cp = (char) c; + *--yy_cp = (char)c; - if ( c == '\n' ) - --yylineno; + if (c == '\n') + --yylineno; - yytext_ptr = yy_bp; - yy_hold_char = *yy_cp; - yy_c_buf_p = yy_cp; + yytext_ptr = yy_bp; + yy_hold_char = *yy_cp; + yy_c_buf_p = yy_cp; } -#endif /* ifndef YY_NO_UNPUT */ - +#endif /* ifndef YY_NO_UNPUT */ #ifdef __cplusplus static int yyinput() @@ -1316,398 +1240,368 @@ static int yyinput() static int input() #endif { - int c; + int c; - *yy_c_buf_p = yy_hold_char; + *yy_c_buf_p = yy_hold_char; - if ( *yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) + if (*yy_c_buf_p == YY_END_OF_BUFFER_CHAR) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if (yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars]) + /* This was really a NUL. */ + *yy_c_buf_p = '\0'; + + else { - /* yy_c_buf_p now points to the character we want to return. - * If this occurs *before* the EOB characters, then it's a - * valid NUL; if not, then we've hit the end of the buffer. - */ - if ( yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars] ) - /* This was really a NUL. */ - *yy_c_buf_p = '\0'; + /* need more input */ + int offset = yy_c_buf_p - yytext_ptr; + ++yy_c_buf_p; - else + switch (yy_get_next_buffer()) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart(yyin); + + /* fall through */ + + case EOB_ACT_END_OF_FILE: { - /* need more input */ - int offset = yy_c_buf_p - yytext_ptr; - ++yy_c_buf_p; + if (yywrap()) + return EOF; - switch ( yy_get_next_buffer() ) - { - case EOB_ACT_LAST_MATCH: - /* This happens because yy_g_n_b() - * sees that we've accumulated a - * token and flags that we need to - * try matching the token before - * proceeding. But for input(), - * there's no matching to consider. - * So convert the EOB_ACT_LAST_MATCH - * to EOB_ACT_END_OF_FILE. - */ - - /* Reset buffer status. */ - yyrestart( yyin ); - - /* fall through */ - - case EOB_ACT_END_OF_FILE: - { - if ( yywrap() ) - return EOF; - - if ( ! yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; + if (!yy_did_buffer_switch_on_eof) + YY_NEW_FILE; #ifdef __cplusplus - return yyinput(); + return yyinput(); #else - return input(); + return input(); #endif - } - - case EOB_ACT_CONTINUE_SCAN: - yy_c_buf_p = yytext_ptr + offset; - break; - } } + + case EOB_ACT_CONTINUE_SCAN: yy_c_buf_p = yytext_ptr + offset; break; + } } + } - c = *(unsigned char*) yy_c_buf_p; /* cast for 8-bit char's */ - *yy_c_buf_p = '\0'; /* preserve yytext */ - yy_hold_char = *++yy_c_buf_p; + c = *(unsigned char*)yy_c_buf_p; /* cast for 8-bit char's */ + *yy_c_buf_p = '\0'; /* preserve yytext */ + yy_hold_char = *++yy_c_buf_p; - if ( c == '\n' ) - ++yylineno; + if (c == '\n') + ++yylineno; - return c; + return c; } - #ifdef YY_USE_PROTOS -void yyrestart( FILE* input_file ) +void yyrestart(FILE* input_file) #else -void yyrestart( input_file ) -FILE* input_file; +void yyrestart(input_file) FILE* input_file; #endif { - if ( ! yy_current_buffer ) - yy_current_buffer = yy_create_buffer( yyin, YY_BUF_SIZE ); + if (!yy_current_buffer) + yy_current_buffer = yy_create_buffer(yyin, YY_BUF_SIZE); - yy_init_buffer( yy_current_buffer, input_file ); - yy_load_buffer_state(); + yy_init_buffer(yy_current_buffer, input_file); + yy_load_buffer_state(); } - #ifdef YY_USE_PROTOS -void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +void yy_switch_to_buffer(YY_BUFFER_STATE new_buffer) #else -void yy_switch_to_buffer( new_buffer ) -YY_BUFFER_STATE new_buffer; +void yy_switch_to_buffer(new_buffer) YY_BUFFER_STATE new_buffer; #endif { - if ( yy_current_buffer == new_buffer ) - return; + if (yy_current_buffer == new_buffer) + return; - if ( yy_current_buffer ) - { - /* Flush out information for old buffer. */ - *yy_c_buf_p = yy_hold_char; - yy_current_buffer->yy_buf_pos = yy_c_buf_p; - yy_current_buffer->yy_n_chars = yy_n_chars; - } + if (yy_current_buffer) + { + /* Flush out information for old buffer. */ + *yy_c_buf_p = yy_hold_char; + yy_current_buffer->yy_buf_pos = yy_c_buf_p; + yy_current_buffer->yy_n_chars = yy_n_chars; + } - yy_current_buffer = new_buffer; - yy_load_buffer_state(); + yy_current_buffer = new_buffer; + yy_load_buffer_state(); - /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe - * to go ahead and always set it. - */ - yy_did_buffer_switch_on_eof = 1; + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yy_did_buffer_switch_on_eof = 1; } - #ifdef YY_USE_PROTOS -void yy_load_buffer_state( void ) +void yy_load_buffer_state(void) #else void yy_load_buffer_state() #endif { - yy_n_chars = yy_current_buffer->yy_n_chars; - yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; - yyin = yy_current_buffer->yy_input_file; - yy_hold_char = *yy_c_buf_p; + yy_n_chars = yy_current_buffer->yy_n_chars; + yytext_ptr = yy_c_buf_p = yy_current_buffer->yy_buf_pos; + yyin = yy_current_buffer->yy_input_file; + yy_hold_char = *yy_c_buf_p; } - #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_create_buffer( FILE* file, int size ) +YY_BUFFER_STATE yy_create_buffer(FILE* file, int size) #else -YY_BUFFER_STATE yy_create_buffer( file, size ) -FILE* file; +YY_BUFFER_STATE yy_create_buffer(file, size) FILE* file; int size; #endif { - YY_BUFFER_STATE b; + YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE)yy_flex_alloc(sizeof(struct yy_buffer_state)); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + if (!b) + YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - b->yy_buf_size = size; + b->yy_buf_size = size; - /* yy_ch_buf has to be 2 characters longer than the size given because - * we need to put in 2 end-of-buffer characters. - */ - b->yy_ch_buf = (char*) yy_flex_alloc( b->yy_buf_size + 2 ); + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char*)yy_flex_alloc(b->yy_buf_size + 2); - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + if (!b->yy_ch_buf) + YY_FATAL_ERROR("out of dynamic memory in yy_create_buffer()"); - b->yy_is_our_buffer = 1; + b->yy_is_our_buffer = 1; - yy_init_buffer( b, file ); + yy_init_buffer(b, file); - return b; + return b; } - #ifdef YY_USE_PROTOS -void yy_delete_buffer( YY_BUFFER_STATE b ) +void yy_delete_buffer(YY_BUFFER_STATE b) #else -void yy_delete_buffer( b ) -YY_BUFFER_STATE b; +void yy_delete_buffer(b) YY_BUFFER_STATE b; #endif { - if ( ! b ) - return; + if (!b) + return; - if ( b == yy_current_buffer ) - yy_current_buffer = (YY_BUFFER_STATE) 0; + if (b == yy_current_buffer) + yy_current_buffer = (YY_BUFFER_STATE)0; - if ( b->yy_is_our_buffer ) - yy_flex_free( (void*) b->yy_ch_buf ); + if (b->yy_is_our_buffer) + yy_flex_free((void*)b->yy_ch_buf); - yy_flex_free( (void*) b ); + yy_flex_free((void*)b); } - #ifndef YY_ALWAYS_INTERACTIVE #ifndef YY_NEVER_INTERACTIVE -extern int isatty YY_PROTO(( int )); +extern int isatty YY_PROTO((int)); #endif #endif #ifdef YY_USE_PROTOS -void yy_init_buffer( YY_BUFFER_STATE b, FILE* file ) +void yy_init_buffer(YY_BUFFER_STATE b, FILE* file) #else -void yy_init_buffer( b, file ) -YY_BUFFER_STATE b; +void yy_init_buffer(b, file) YY_BUFFER_STATE b; FILE* file; #endif - { - yy_flush_buffer( b ); + yy_flush_buffer(b); - b->yy_input_file = file; - b->yy_fill_buffer = 1; + b->yy_input_file = file; + b->yy_fill_buffer = 1; #if YY_ALWAYS_INTERACTIVE - b->yy_is_interactive = 1; + b->yy_is_interactive = 1; #else #if YY_NEVER_INTERACTIVE - b->yy_is_interactive = 0; + b->yy_is_interactive = 0; #else - b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; + b->yy_is_interactive = file ? (isatty(fileno(file)) > 0) : 0; #endif #endif } - #ifdef YY_USE_PROTOS -void yy_flush_buffer( YY_BUFFER_STATE b ) +void yy_flush_buffer(YY_BUFFER_STATE b) #else -void yy_flush_buffer( b ) -YY_BUFFER_STATE b; +void yy_flush_buffer(b) YY_BUFFER_STATE b; #endif { - if ( ! b ) - return; + if (!b) + return; - b->yy_n_chars = 0; + b->yy_n_chars = 0; - /* We always need two end-of-buffer characters. The first causes - * a transition to the end-of-buffer state. The second causes - * a jam in that state. - */ - b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; - b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; - b->yy_buf_pos = &b->yy_ch_buf[0]; + b->yy_buf_pos = &b->yy_ch_buf[0]; - b->yy_at_bol = 1; - b->yy_buffer_status = YY_BUFFER_NEW; + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; - if ( b == yy_current_buffer ) - yy_load_buffer_state(); + if (b == yy_current_buffer) + yy_load_buffer_state(); } - #ifndef YY_NO_SCAN_BUFFER #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_buffer( char* base, yy_size_t size ) +YY_BUFFER_STATE yy_scan_buffer(char* base, yy_size_t size) #else -YY_BUFFER_STATE yy_scan_buffer( base, size ) -char* base; +YY_BUFFER_STATE yy_scan_buffer(base, size) char* base; yy_size_t size; #endif { - YY_BUFFER_STATE b; + YY_BUFFER_STATE b; - if ( size < 2 || - base[size - 2] != YY_END_OF_BUFFER_CHAR || - base[size - 1] != YY_END_OF_BUFFER_CHAR ) - /* They forgot to leave room for the EOB's. */ - return 0; + if (size < 2 || base[size - 2] != YY_END_OF_BUFFER_CHAR || base[size - 1] != YY_END_OF_BUFFER_CHAR) + /* They forgot to leave room for the EOB's. */ + return 0; - b = (YY_BUFFER_STATE) yy_flex_alloc( sizeof( struct yy_buffer_state ) ); + b = (YY_BUFFER_STATE)yy_flex_alloc(sizeof(struct yy_buffer_state)); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + if (!b) + YY_FATAL_ERROR("out of dynamic memory in yy_scan_buffer()"); - b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ - b->yy_buf_pos = b->yy_ch_buf = base; - b->yy_is_our_buffer = 0; - b->yy_input_file = 0; - b->yy_n_chars = b->yy_buf_size; - b->yy_is_interactive = 0; - b->yy_at_bol = 1; - b->yy_fill_buffer = 0; - b->yy_buffer_status = YY_BUFFER_NEW; + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = 0; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b ); + yy_switch_to_buffer(b); - return b; + return b; } #endif - #ifndef YY_NO_SCAN_STRING #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_string( yyconst char* yy_str ) +YY_BUFFER_STATE yy_scan_string(yyconst char* yy_str) #else -YY_BUFFER_STATE yy_scan_string( yy_str ) -yyconst char* yy_str; +YY_BUFFER_STATE yy_scan_string(yy_str) yyconst char* yy_str; #endif { - int len; + int len; - for ( len = 0; yy_str[len]; ++len ) - ; + for (len = 0; yy_str[len]; ++len) + ; - return yy_scan_bytes( yy_str, len ); + return yy_scan_bytes(yy_str, len); } #endif - #ifndef YY_NO_SCAN_BYTES #ifdef YY_USE_PROTOS -YY_BUFFER_STATE yy_scan_bytes( yyconst char* bytes, int len ) +YY_BUFFER_STATE yy_scan_bytes(yyconst char* bytes, int len) #else -YY_BUFFER_STATE yy_scan_bytes( bytes, len ) -yyconst char* bytes; +YY_BUFFER_STATE yy_scan_bytes(bytes, len) yyconst char* bytes; int len; #endif { - YY_BUFFER_STATE b; - char* buf; - yy_size_t n; - int i; + YY_BUFFER_STATE b; + char* buf; + yy_size_t n; + int i; - /* Get memory for full buffer, including space for trailing EOB's. */ - n = len + 2; - buf = (char*) yy_flex_alloc( n ); + /* Get memory for full buffer, including space for trailing EOB's. */ + n = len + 2; + buf = (char*)yy_flex_alloc(n); - if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + if (!buf) + YY_FATAL_ERROR("out of dynamic memory in yy_scan_bytes()"); - for ( i = 0; i < len; ++i ) - buf[i] = bytes[i]; + for (i = 0; i < len; ++i) + buf[i] = bytes[i]; - buf[len] = buf[len + 1] = YY_END_OF_BUFFER_CHAR; + buf[len] = buf[len + 1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n ); + b = yy_scan_buffer(buf, n); - if ( ! b ) - YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + if (!b) + YY_FATAL_ERROR("bad buffer in yy_scan_bytes()"); - /* It's okay to grow etc. this buffer, and we should throw it - * away when we're done. - */ - b->yy_is_our_buffer = 1; + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; - return b; + return b; } #endif - #ifndef YY_NO_PUSH_STATE #ifdef YY_USE_PROTOS -static void yy_push_state( int new_state ) +static void yy_push_state(int new_state) #else -static void yy_push_state( new_state ) -int new_state; +static void yy_push_state(new_state) int new_state; #endif { - if ( yy_start_stack_ptr >= yy_start_stack_depth ) - { - yy_size_t new_size; + if (yy_start_stack_ptr >= yy_start_stack_depth) + { + yy_size_t new_size; - yy_start_stack_depth += YY_START_STACK_INCR; - new_size = yy_start_stack_depth * sizeof( int ); + yy_start_stack_depth += YY_START_STACK_INCR; + new_size = yy_start_stack_depth * sizeof(int); - if ( ! yy_start_stack ) - yy_start_stack = (int*) yy_flex_alloc( new_size ); + if (!yy_start_stack) + yy_start_stack = (int*)yy_flex_alloc(new_size); - else - yy_start_stack = (int*) yy_flex_realloc( - (void*) yy_start_stack, new_size ); + else + yy_start_stack = (int*)yy_flex_realloc((void*)yy_start_stack, new_size); - if ( ! yy_start_stack ) - YY_FATAL_ERROR( - "out of memory expanding start-condition stack" ); - } + if (!yy_start_stack) + YY_FATAL_ERROR("out of memory expanding start-condition stack"); + } - yy_start_stack[yy_start_stack_ptr++] = YY_START; + yy_start_stack[yy_start_stack_ptr++] = YY_START; - BEGIN(new_state); + BEGIN(new_state); } #endif - #ifndef YY_NO_POP_STATE static void yy_pop_state() { - if ( --yy_start_stack_ptr < 0 ) - YY_FATAL_ERROR( "start-condition stack underflow" ); + if (--yy_start_stack_ptr < 0) + YY_FATAL_ERROR("start-condition stack underflow"); - BEGIN(yy_start_stack[yy_start_stack_ptr]); + BEGIN(yy_start_stack[yy_start_stack_ptr]); } #endif - #ifndef YY_NO_TOP_STATE static int yy_top_state() { - return yy_start_stack[yy_start_stack_ptr - 1]; + return yy_start_stack[yy_start_stack_ptr - 1]; } #endif @@ -1716,114 +1610,103 @@ static int yy_top_state() #endif #ifdef YY_USE_PROTOS -static void yy_fatal_error( yyconst char msg[] ) +static void yy_fatal_error(yyconst char msg[]) #else -static void yy_fatal_error( msg ) -char msg[]; +static void yy_fatal_error(msg) char msg[]; #endif { - (void) fprintf( stderr, "%s\n", msg ); - exit( YY_EXIT_FAILURE ); + (void)fprintf(stderr, "%s\n", msg); + exit(YY_EXIT_FAILURE); } - - /* Redefine yyless() so it works in section 3 code. */ #undef yyless -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - yytext[yyleng] = yy_hold_char; \ - yy_c_buf_p = yytext + n; \ - yy_hold_char = *yy_c_buf_p; \ - *yy_c_buf_p = '\0'; \ - yyleng = n; \ - } \ - while ( 0 ) - +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yytext[yyleng] = yy_hold_char; \ + yy_c_buf_p = yytext + n; \ + yy_hold_char = *yy_c_buf_p; \ + *yy_c_buf_p = '\0'; \ + yyleng = n; \ + } while (0) /* Internal utility routines. */ #ifndef yytext_ptr #ifdef YY_USE_PROTOS -static void yy_flex_strncpy( char* s1, yyconst char* s2, int n ) +static void yy_flex_strncpy(char* s1, yyconst char* s2, int n) #else -static void yy_flex_strncpy( s1, s2, n ) -char* s1; +static void yy_flex_strncpy(s1, s2, n) char* s1; yyconst char* s2; int n; #endif { - register int i; + register int i; - for ( i = 0; i < n; ++i ) - s1[i] = s2[i]; + for (i = 0; i < n; ++i) + s1[i] = s2[i]; } #endif #ifdef YY_NEED_STRLEN #ifdef YY_USE_PROTOS -static int yy_flex_strlen( yyconst char* s ) +static int yy_flex_strlen(yyconst char* s) #else -static int yy_flex_strlen( s ) -yyconst char* s; +static int yy_flex_strlen(s) yyconst char* s; #endif { - register int n; + register int n; - for ( n = 0; s[n]; ++n ) - ; + for (n = 0; s[n]; ++n) + ; - return n; + return n; } #endif +#ifdef YY_USE_PROTOS +static void* yy_flex_alloc(yy_size_t size) +#else +static void* yy_flex_alloc(size) yy_size_t size; +#endif +{ + return (void*)malloc(size); +} #ifdef YY_USE_PROTOS -static void* yy_flex_alloc( yy_size_t size ) +static void* yy_flex_realloc(void* ptr, yy_size_t size) #else -static void* yy_flex_alloc( size ) +static void* yy_flex_realloc(ptr, size) void* ptr; yy_size_t size; #endif { - return (void*) malloc( size ); + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return (void*)realloc((char*)ptr, size); } #ifdef YY_USE_PROTOS -static void* yy_flex_realloc( void* ptr, yy_size_t size ) +static void yy_flex_free(void* ptr) #else -static void* yy_flex_realloc( ptr, size ) -void* ptr; -yy_size_t size; +static void yy_flex_free(ptr) void* ptr; #endif { - /* The cast to (char *) in the following accommodates both - * implementations that use char* generic pointers, and those - * that use void* generic pointers. It works with the latter - * because both ANSI C and C++ allow castless assignment from - * any pointer type to void*, and deal with argument conversions - * as though doing an assignment. - */ - return (void*) realloc( (char*) ptr, size ); -} - -#ifdef YY_USE_PROTOS -static void yy_flex_free( void* ptr ) -#else -static void yy_flex_free( ptr ) -void* ptr; -#endif -{ - free( ptr ); + free(ptr); } #if YY_MAIN int main() { - yylex(); - return 0; + yylex(); + return 0; } #endif #line 66 "qfelexer.lpp" diff --git a/tools/qfe/parsequery.cpp b/tools/qfe/parsequery.cpp index 54b676737..bda94df70 100644 --- a/tools/qfe/parsequery.cpp +++ b/tools/qfe/parsequery.cpp @@ -33,7 +33,7 @@ extern boost::shared_ptr ParserCSC; namespace { mutex ParserMutex; -} //anon namespace +} // namespace namespace qfe { @@ -41,58 +41,57 @@ extern string DefaultSchema; CalpontSelectExecutionPlan* parseQuery(const string& query, const uint32_t sid) { - //We're going to make parsing the query single-threaded for now. This makes it a lot - // easier to interface with the parser and doesn;t materially affect overall query - // performance (I think) - mutex::scoped_lock lk(ParserMutex); + // We're going to make parsing the query single-threaded for now. This makes it a lot + // easier to interface with the parser and doesn;t materially affect overall query + // performance (I think) + mutex::scoped_lock lk(ParserMutex); - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid); - CalpontSelectExecutionPlan* csep = 0; - csep = new CalpontSelectExecutionPlan(); - //we use an auto_ptr here with some trepidation. We only want auto delete on an execption. - //If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<> - //doesn't have an API to release ownership, so we use auto_ptr... - auto_ptr scsep(csep); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid); + CalpontSelectExecutionPlan* csep = 0; + csep = new CalpontSelectExecutionPlan(); + // we use an auto_ptr here with some trepidation. We only want auto delete on an execption. + // If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<> + // doesn't have an API to release ownership, so we use auto_ptr... + auto_ptr scsep(csep); - yy_buffer_state* ybs = 0; - ybs = qfe_scan_string(query.c_str()); + yy_buffer_state* ybs = 0; + ybs = qfe_scan_string(query.c_str()); - if (ybs != 0) - { - ParserCSEP = csep; - ParserCSC = csc; + if (ybs != 0) + { + ParserCSEP = csep; + ParserCSC = csc; - if (qfeparse() != 0) - throw runtime_error("syntax error"); + if (qfeparse() != 0) + throw runtime_error("syntax error"); - qfe_delete_buffer(ybs); - } - else - throw runtime_error("Internal parser memory error"); + qfe_delete_buffer(ybs); + } + else + throw runtime_error("Internal parser memory error"); - csep->data(query); + csep->data(query); - SessionManager sm; - TxnID txnID; - txnID = sm.getTxnID(sid); + SessionManager sm; + TxnID txnID; + txnID = sm.getTxnID(sid); - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } - QueryContext verID; - verID = sm.verID(); + QueryContext verID; + verID = sm.verID(); - csep->txnID(txnID.id); - csep->verID(verID); - csep->sessionID(sid); - - //cout << *csep << endl; - scsep.release(); - return csep; -} + csep->txnID(txnID.id); + csep->verID(verID); + csep->sessionID(sid); + // cout << *csep << endl; + scsep.release(); + return csep; } +} // namespace qfe diff --git a/tools/qfe/parsequery.h b/tools/qfe/parsequery.h index afd64580a..a2000aab5 100644 --- a/tools/qfe/parsequery.h +++ b/tools/qfe/parsequery.h @@ -8,7 +8,6 @@ namespace qfe { - execplan::CalpontSelectExecutionPlan* parseQuery(const std::string&, uint32_t); } diff --git a/tools/qfe/returnedrows.cpp b/tools/qfe/returnedrows.cpp index fc631993a..d833830a7 100644 --- a/tools/qfe/returnedrows.cpp +++ b/tools/qfe/returnedrows.cpp @@ -27,74 +27,71 @@ using namespace rowgroup; namespace qfe { - void processReturnedRows(MessageQueueClient* mqc, SockType fd) { - scoped_ptr cleaner(mqc); - SBS sbs; - sbs = mqc->read(); - //cerr << "got a bs of " << sbs->length() << " bytes" << endl; + scoped_ptr cleaner(mqc); + SBS sbs; + sbs = mqc->read(); + // cerr << "got a bs of " << sbs->length() << " bytes" << endl; - RowGroup rg; - rg.deserialize(*sbs); + RowGroup rg; + rg.deserialize(*sbs); - //cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl; - //cerr << rg.toString() << endl; + // cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl; + // cerr << rg.toString() << endl; - ByteStream bs; - ByteStream::quadbyte tableOID = 100; + ByteStream bs; + ByteStream::quadbyte tableOID = 100; + bs.reset(); + bs << tableOID; + mqc->write(bs); + + sbs = mqc->read(); + // cerr << "got a bs of " << sbs->length() << " bytes" << endl; + RGData rgd; + rgd.deserialize(*sbs, true); + rg.setData(&rgd); + // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl; + + socketio::writeString(fd, "OK"); + Row r; + + while (rg.getRowCount() > 0) + { + rg.initRow(&r); + rg.getRow(0, &r); + string csv; bs.reset(); - bs << tableOID; - mqc->write(bs); - sbs = mqc->read(); - //cerr << "got a bs of " << sbs->length() << " bytes" << endl; - RGData rgd; - rgd.deserialize(*sbs, true); - rg.setData(&rgd); - //cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl; - - socketio::writeString(fd, "OK"); - Row r; - - while (rg.getRowCount() > 0) + for (unsigned i = 0; i < rg.getRowCount(); i++) { - rg.initRow(&r); - rg.getRow(0, &r); - string csv; - bs.reset(); - - for (unsigned i = 0; i < rg.getRowCount(); i++) - { - csv = r.toCSV(); - bs << csv; - r.nextRow(); - } - - //cerr << "writing " << bs.length() << " bytes back to client" << endl; - SockWriteFcn(fd, bs.buf(), bs.length()); - - bs.reset(); - bs << tableOID; - mqc->write(bs); - - sbs = mqc->read(); - //cerr << "got a bs of " << sbs->length() << " bytes" << endl; - rgd.deserialize(*sbs, true); - rg.setData(&rgd); - //cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl; + csv = r.toCSV(); + bs << csv; + r.nextRow(); } - tableOID = 0; + // cerr << "writing " << bs.length() << " bytes back to client" << endl; + SockWriteFcn(fd, bs.buf(), bs.length()); + bs.reset(); bs << tableOID; mqc->write(bs); - //sync with the client on end-of-results - SockWriteFcn(fd, &tableOID, 4); - SockReadFcn(fd, &tableOID, 4); - -} + sbs = mqc->read(); + // cerr << "got a bs of " << sbs->length() << " bytes" << endl; + rgd.deserialize(*sbs, true); + rg.setData(&rgd); + // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl; + } + tableOID = 0; + bs.reset(); + bs << tableOID; + mqc->write(bs); + + // sync with the client on end-of-results + SockWriteFcn(fd, &tableOID, 4); + SockReadFcn(fd, &tableOID, 4); } +} // namespace qfe diff --git a/tools/qfe/returnedrows.h b/tools/qfe/returnedrows.h index ce4a4e520..384780adc 100644 --- a/tools/qfe/returnedrows.h +++ b/tools/qfe/returnedrows.h @@ -6,10 +6,8 @@ namespace qfe { - void processReturnedRows(messageqcpp::MessageQueueClient*, SockType); } #endif - diff --git a/tools/qfe/sendcsep.cpp b/tools/qfe/sendcsep.cpp index 3fb9c14e0..b720f16b4 100644 --- a/tools/qfe/sendcsep.cpp +++ b/tools/qfe/sendcsep.cpp @@ -14,42 +14,40 @@ using namespace messageqcpp; namespace qfe { - MessageQueueClient* sendCSEP(CalpontSelectExecutionPlan* csep) { - scoped_ptr cleaner(csep); + scoped_ptr cleaner(csep); - ByteStream bs; + ByteStream bs; - MessageQueueClient* mqc = 0; + MessageQueueClient* mqc = 0; - mqc = new MessageQueueClient("ExeMgr1"); - auto_ptr smqc(mqc); + mqc = new MessageQueueClient("ExeMgr1"); + auto_ptr smqc(mqc); - bs.reset(); - ByteStream::quadbyte wantTuples = 4; - bs << wantTuples; - mqc->write(bs); + bs.reset(); + ByteStream::quadbyte wantTuples = 4; + bs << wantTuples; + mqc->write(bs); - bs.reset(); - csep->serialize(bs); - mqc->write(bs); + bs.reset(); + csep->serialize(bs); + mqc->write(bs); - SBS sbs; - sbs = mqc->read(); - *sbs >> wantTuples; - //cerr << "got flag: " << wantTuples << endl; - string msg; - sbs = mqc->read(); - *sbs >> msg; - //cerr << "got msg: " << msg << endl; + SBS sbs; + sbs = mqc->read(); + *sbs >> wantTuples; + // cerr << "got flag: " << wantTuples << endl; + string msg; + sbs = mqc->read(); + *sbs >> msg; + // cerr << "got msg: " << msg << endl; - if (wantTuples != 0) - throw runtime_error(msg); - - smqc.release(); - return mqc; -} + if (wantTuples != 0) + throw runtime_error(msg); + smqc.release(); + return mqc; } +} // namespace qfe diff --git a/tools/qfe/sendcsep.h b/tools/qfe/sendcsep.h index 47ba28485..0dd7d385a 100644 --- a/tools/qfe/sendcsep.h +++ b/tools/qfe/sendcsep.h @@ -6,12 +6,10 @@ namespace qfe { - -//Takes ownership of the alloc'd ptr -//Returns an alloc'd mqc +// Takes ownership of the alloc'd ptr +// Returns an alloc'd mqc messageqcpp::MessageQueueClient* sendCSEP(execplan::CalpontSelectExecutionPlan*); -} +} // namespace qfe #endif - diff --git a/tools/qfe/server.cpp b/tools/qfe/server.cpp index e38a86ab5..018cf470a 100644 --- a/tools/qfe/server.cpp +++ b/tools/qfe/server.cpp @@ -16,7 +16,7 @@ MA 02110-1301, USA. */ /* - Protocol definition: + Protocol definition: */ #ifdef _MSC_VER @@ -65,7 +65,7 @@ using namespace boost; #include "socketio.h" #include "ddlstmts.h" -#include "exceptclasses.h" //brings in idbassert_s macro +#include "exceptclasses.h" //brings in idbassert_s macro #include "messagequeue.h" using namespace messageqcpp; @@ -90,89 +90,97 @@ using namespace qfe; enum StmtType { - UNKNOWN, - QUERY, - CREATE, - DROP, - SHOW, + UNKNOWN, + QUERY, + CREATE, + DROP, + SHOW, }; volatile uint32_t SystemSID; void log(const string& s) { - cerr << s << endl; + cerr << s << endl; } struct QueryMessage { - QueryMessage() : isValid(false) { } - ~QueryMessage() { } + QueryMessage() : isValid(false) + { + } + ~QueryMessage() + { + } - string toString() const; + string toString() const; - bool isValid; - string queryText; - string defaultSchema; + bool isValid; + string queryText; + string defaultSchema; }; string QueryMessage::toString() const { - ostringstream oss; - oss << "valid: " << boolalpha << isValid << ", " << - "queryText: " << queryText << ", " << - "defaultSchema: " << defaultSchema; - return oss.str(); + ostringstream oss; + oss << "valid: " << boolalpha << isValid << ", " + << "queryText: " << queryText << ", " + << "defaultSchema: " << defaultSchema; + return oss.str(); } ostream& operator<<(ostream& os, const QueryMessage& rhs) { - os << rhs.toString(); - return os; + os << rhs.toString(); + return os; } class ThreadFunc { -public: - ThreadFunc(SockType fd) : fFd(fd) { } - ~ThreadFunc() { } + public: + ThreadFunc(SockType fd) : fFd(fd) + { + } + ~ThreadFunc() + { + } - void run(); - void operator()() - { - run(); - } + void run(); + void operator()() + { + run(); + } -private: - ThreadFunc(const ThreadFunc& rhs); - ThreadFunc& operator=(const ThreadFunc& rhs); + private: + ThreadFunc(const ThreadFunc& rhs); + ThreadFunc& operator=(const ThreadFunc& rhs); - SockType fFd; + SockType fFd; }; bool serverInit() { #ifndef _MSC_VER - setsid(); + setsid(); - //Handle certain signals (we want these to return EINTR so we can throw) - //SIGPIPE - //I don't think we'll get any of these from init (except possibly HUP, but that's an indication - // of bad things anyway) - //SIGHUP? - //SIGUSR1? - //SIGUSR2? - //SIGPOLL? - struct sigaction sa; - memset(&sa, 0, sizeof(struct sigaction)); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, 0); - sigaction(SIGHUP, &sa, 0); - sigaction(SIGUSR1, &sa, 0); - sigaction(SIGUSR2, &sa, 0); + // Handle certain signals (we want these to return EINTR so we can throw) + // SIGPIPE + // I don't think we'll get any of these from init (except possibly HUP, but that's an indication + // of bad things anyway) + // SIGHUP? + // SIGUSR1? + // SIGUSR2? + // SIGPOLL? + struct sigaction sa; + memset(&sa, 0, sizeof(struct sigaction)); + sa.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sa, 0); + sigaction(SIGHUP, &sa, 0); + sigaction(SIGUSR1, &sa, 0); + sigaction(SIGUSR2, &sa, 0); #ifndef __FreeBSD__ - sigaction(SIGPOLL, &sa, 0); + sigaction(SIGPOLL, &sa, 0); #endif #if 0 int fd; @@ -187,259 +195,255 @@ bool serverInit() #endif #endif - return true; + return true; } SockType initListenSock(short portNo) { - SockType listenSock = -1; + SockType listenSock = -1; #ifdef _MSC_VER - WSAData wsadata; - const WORD minVersion = MAKEWORD(2, 2); - WSAStartup(minVersion, &wsadata); + WSAData wsadata; + const WORD minVersion = MAKEWORD(2, 2); + WSAStartup(minVersion, &wsadata); #endif - listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno)); - //if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno)); + listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno)); + // if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno)); #ifndef _MSC_VER - int optval = 1; - setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&optval), sizeof(optval)); + int optval = 1; + setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&optval), sizeof(optval)); #endif - int rc = 0; - struct sockaddr_in serv_addr; - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = INADDR_ANY; - serv_addr.sin_port = htons(portNo); - const int MaxTries = 5 * 60 / 10; - int tries = 0; + int rc = 0; + struct sockaddr_in serv_addr; + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = INADDR_ANY; + serv_addr.sin_port = htons(portNo); + const int MaxTries = 5 * 60 / 10; + int tries = 0; again: - rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr)); + rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr)); - if (rc < 0) - { + if (rc < 0) + { #ifdef _MSC_VER - int x = WSAGetLastError(); + int x = WSAGetLastError(); - if (x == WSAEADDRINUSE) + if (x == WSAEADDRINUSE) #else - if (errno == EADDRINUSE) + if (errno == EADDRINUSE) #endif - { - //cerr << "Addr in use..." << endl; - if (++tries >= MaxTries) - { - log("Waited too long for socket to bind...giving up"); - //cerr << "Waited too long for socket to bind...giving up" << endl; - exit(1); - } + { + // cerr << "Addr in use..." << endl; + if (++tries >= MaxTries) + { + log("Waited too long for socket to bind...giving up"); + // cerr << "Waited too long for socket to bind...giving up" << endl; + exit(1); + } - sleep(10); - goto again; - } - - idbassert_s(0, string("socket bind error: ") + strerror(errno)); - //throw runtime_error(string("socket bind error: ") + strerror(errno)); + sleep(10); + goto again; } - rc = listen(listenSock, 16); - idbassert_s(rc >= 0, string("socket listen error") + strerror(errno)); - //if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno)); + idbassert_s(0, string("socket bind error: ") + strerror(errno)); + // throw runtime_error(string("socket bind error: ") + strerror(errno)); + } - return listenSock; + rc = listen(listenSock, 16); + idbassert_s(rc >= 0, string("socket listen error") + strerror(errno)); + // if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno)); + + return listenSock; } QueryMessage getNextMsg(SockType fd) { - QueryMessage msg; + QueryMessage msg; - try - { - msg.defaultSchema = socketio::readString(fd); - msg.queryText = socketio::readString(fd); - msg.isValid = true; - } - catch (runtime_error& rex) - { - cerr << "re reading ctl msg: " << rex.what() << endl; - msg.queryText = ""; - } - catch (...) - { - cerr << "ex reading ctl msg" << endl; - msg.queryText = ""; - } + try + { + msg.defaultSchema = socketio::readString(fd); + msg.queryText = socketio::readString(fd); + msg.isValid = true; + } + catch (runtime_error& rex) + { + cerr << "re reading ctl msg: " << rex.what() << endl; + msg.queryText = ""; + } + catch (...) + { + cerr << "ex reading ctl msg" << endl; + msg.queryText = ""; + } - return msg; + return msg; } StmtType guessStatementType(const string& stmt) { - typedef boost::tokenizer > tokenizer; - char_separator sep; - tokenizer tokens(stmt, sep); - tokenizer::iterator tok_iter = tokens.begin(); - string first_word; - first_word = *tok_iter; - algorithm::to_lower(first_word); + typedef boost::tokenizer > tokenizer; + char_separator sep; + tokenizer tokens(stmt, sep); + tokenizer::iterator tok_iter = tokens.begin(); + string first_word; + first_word = *tok_iter; + algorithm::to_lower(first_word); - if (first_word == "select") - return QUERY; + if (first_word == "select") + return QUERY; - if (first_word == "create") - return CREATE; + if (first_word == "create") + return CREATE; - if (first_word == "drop") - return DROP; + if (first_word == "drop") + return DROP; - if (first_word == "show") - return SHOW; + if (first_word == "show") + return SHOW; - return UNKNOWN; + return UNKNOWN; } struct ScopedCleaner { - ScopedCleaner(SockType fd = -1) : fFd(fd) { } + ScopedCleaner(SockType fd = -1) : fFd(fd) + { + } #ifdef _MSC_VER - ~ScopedCleaner() - { - if (fFd >= 0) shutdown(fFd, SHUT_RDWR); + ~ScopedCleaner() + { + if (fFd >= 0) + shutdown(fFd, SHUT_RDWR); - closesocket(fFd); - } + closesocket(fFd); + } #else - ~ScopedCleaner() - { - if (fFd >= 0) shutdown(fFd, SHUT_RDWR); + ~ScopedCleaner() + { + if (fFd >= 0) + shutdown(fFd, SHUT_RDWR); - close(fFd); - } + close(fFd); + } #endif - SockType fFd; + SockType fFd; }; void ThreadFunc::run() { - QueryMessage m; - execplan::CalpontSelectExecutionPlan* csep = 0; - MessageQueueClient* msgqcl; + QueryMessage m; + execplan::CalpontSelectExecutionPlan* csep = 0; + MessageQueueClient* msgqcl; - ScopedCleaner cleaner(fFd); + ScopedCleaner cleaner(fFd); - uint32_t sid = 1; - sid = atomicops::atomicInc(&SystemSID); + uint32_t sid = 1; + sid = atomicops::atomicInc(&SystemSID); - try + try + { + m = getNextMsg(fFd); + + if (m.isValid) { - m = getNextMsg(fFd); + DefaultSchema = m.defaultSchema; + StmtType st = guessStatementType(m.queryText); - if (m.isValid) + switch (st) + { + case QUERY: + csep = parseQuery(m.queryText, sid); + // sendCSEP takes ownership of the ptr from parseQuery + msgqcl = sendCSEP(csep); + // processReturnedRows takes ownership of the ptr from sendCSEP + processReturnedRows(msgqcl, fFd); + break; + + case CREATE: processCreateStmt(m.queryText, sid); break; + + case DROP: processDropStmt(m.queryText, sid); break; + + case SHOW: { - DefaultSchema = m.defaultSchema; - StmtType st = guessStatementType(m.queryText); - - switch (st) - { - case QUERY: - csep = parseQuery(m.queryText, sid); - //sendCSEP takes ownership of the ptr from parseQuery - msgqcl = sendCSEP(csep); - //processReturnedRows takes ownership of the ptr from sendCSEP - processReturnedRows(msgqcl, fFd); - break; - - case CREATE: - processCreateStmt(m.queryText, sid); - break; - - case DROP: - processDropStmt(m.queryText, sid); - break; - - case SHOW: - { - ostringstream oss; - oss << "select calpontsys.systable.tablename from calpontsys.systable where " - "calpontsys.systable.schema='" << m.defaultSchema << "';"; - csep = parseQuery(oss.str(), sid); - msgqcl = sendCSEP(csep); - processReturnedRows(msgqcl, fFd); - break; - } - - default: - throw runtime_error("couldn't guess the statement type"); - break; - } + ostringstream oss; + oss << "select calpontsys.systable.tablename from calpontsys.systable where " + "calpontsys.systable.schema='" + << m.defaultSchema << "';"; + csep = parseQuery(oss.str(), sid); + msgqcl = sendCSEP(csep); + processReturnedRows(msgqcl, fFd); + break; } + + default: throw runtime_error("couldn't guess the statement type"); break; + } } - catch (std::exception& ex) - { - socketio::writeString(fFd, ex.what()); - throw; //in a multi-threaded server this will simply cause this thread to exit - } - catch (...) - { - socketio::writeString(fFd, "internal query processing error"); - throw; - } + } + catch (std::exception& ex) + { + socketio::writeString(fFd, ex.what()); + throw; // in a multi-threaded server this will simply cause this thread to exit + } + catch (...) + { + socketio::writeString(fFd, "internal query processing error"); + throw; + } } -} +} // namespace int main(int argc, char** argv) { - int c; - SockType listenSock; - short portNo; + int c; + SockType listenSock; + short portNo; - portNo = 0; - char* p = getenv("IDB_QFE_PORT"); + portNo = 0; + char* p = getenv("IDB_QFE_PORT"); - if (p && *p) - portNo = atoi(p); + if (p && *p) + portNo = atoi(p); - if (portNo <= 0) - portNo = 9198; + if (portNo <= 0) + portNo = 9198; #ifdef _MSC_VER - listenSock = INVALID_SOCKET; + listenSock = INVALID_SOCKET; #else - listenSock = -1; + listenSock = -1; #endif - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "p:")) != -1) - switch (c) - { - case 'p': - portNo = atoi(optarg); - break; - - case '?': - default: - break; - } - - if (!serverInit()) + while ((c = getopt(argc, argv, "p:")) != -1) + switch (c) { - log("Could not initialize the QFE Server!"); - cerr << "Could not initialize the QFE Server!" << endl; - return 1; + case 'p': portNo = atoi(optarg); break; + + case '?': + default: break; } - listenSock = initListenSock(portNo); + if (!serverInit()) + { + log("Could not initialize the QFE Server!"); + cerr << "Could not initialize the QFE Server!" << endl; + return 1; + } - SystemSID = 0; + listenSock = initListenSock(portNo); - for (;;) - { + SystemSID = 0; + + for (;;) + { #ifdef _MSC_VER - SOCKET querySock = INVALID_SOCKET; - querySock = accept(listenSock, 0, 0); - idbassert_s(querySock != INVALID_SOCKET, string("socket accept error: ") + strerror(errno)); + SOCKET querySock = INVALID_SOCKET; + querySock = accept(listenSock, 0, 0); + idbassert_s(querySock != INVALID_SOCKET, string("socket accept error: ") + strerror(errno)); #if 0 uint32_t sndbufsize; int sndbufsizelen = 4; @@ -476,38 +480,37 @@ int main(int argc, char** argv) } #endif - uint32_t sndbufsize = 512 * 1024; - setsockopt(querySock, SOL_SOCKET, SO_SNDBUF, (const char*)&sndbufsize, 4); + uint32_t sndbufsize = 512 * 1024; + setsockopt(querySock, SOL_SOCKET, SO_SNDBUF, (const char*)&sndbufsize, 4); #else - int querySock = -1; - querySock = accept(listenSock, 0, 0); - idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno)); + int querySock = -1; + querySock = accept(listenSock, 0, 0); + idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno)); #endif - //ThreadFunc now owns querySock and is responsible for cleaning it up - ThreadFunc tf(querySock); + // ThreadFunc now owns querySock and is responsible for cleaning it up + ThreadFunc tf(querySock); #ifdef SINGLE_THREADED - try - { - tf.run(); - } - catch (std::exception& ex) - { - cerr << "ThreadFunc run threw an exception: " << ex.what() << endl; - } - catch (...) - { - cerr << "ThreadFunc run threw an exception" << endl; - } - -#else - thread t(tf); -#endif + try + { + tf.run(); + } + catch (std::exception& ex) + { + cerr << "ThreadFunc run threw an exception: " << ex.what() << endl; + } + catch (...) + { + cerr << "ThreadFunc run threw an exception" << endl; } - return 0; +#else + thread t(tf); +#endif + } + + return 0; } // vim:ts=4 sw=4: - diff --git a/tools/qfe/socketio.cpp b/tools/qfe/socketio.cpp index c6ccf367c..654d5c283 100644 --- a/tools/qfe/socketio.cpp +++ b/tools/qfe/socketio.cpp @@ -33,188 +33,187 @@ namespace qfe { namespace socketio { - #ifndef _MSC_VER void readn(int fd, void* buf, const size_t wanted) { - size_t needed = wanted; - size_t sofar = 0; - char* p = static_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; + size_t needed = wanted; + size_t sofar = 0; + char* p = static_cast(buf); + ssize_t rrc = -1; + pollfd fds[1]; + int en = 0; - fds[0].fd = fd; - fds[0].events = POLLIN; + fds[0].fd = fd; + fds[0].events = POLLIN; - while (wanted > sofar) + while (wanted > sofar) + { + fds[0].revents = 0; + poll(fds, 1, -1); + errno = 0; + rrc = read(fd, (p + sofar), needed); + en = errno; + + if (rrc < 0) { - fds[0].revents = 0; - poll(fds, 1, -1); - errno = 0; - rrc = read(fd, (p + sofar), needed); - en = errno; + if (en == EAGAIN || en == EINTR || en == 512) + continue; - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR || en == 512) - continue; - - ostringstream oss; - oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - needed -= rrc; - sofar += rrc; + ostringstream oss; + oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")"; + idbassert_s(0, oss.str()); } + + needed -= rrc; + sofar += rrc; + } } size_t writen(int fd, const void* data, const size_t nbytes) { - size_t nleft; - ssize_t nwritten; - const char* bufp = static_cast(data); - nleft = nbytes; + size_t nleft; + ssize_t nwritten; + const char* bufp = static_cast(data); + nleft = nbytes; - while (nleft > 0) + while (nleft > 0) + { + // the O_NONBLOCK flag is not set, this is a blocking I/O. + if ((nwritten = ::write(fd, bufp, nleft)) < 0) { - // the O_NONBLOCK flag is not set, this is a blocking I/O. - if ((nwritten = ::write(fd, bufp, nleft)) < 0) - { - if (errno == EINTR) - nwritten = 0; - else - { - // save the error no first - int e = errno; - string errorMsg = "qfe: writen: write() error: "; - scoped_array buf(new char[80]); + if (errno == EINTR) + nwritten = 0; + else + { + // save the error no first + int e = errno; + string errorMsg = "qfe: writen: write() error: "; + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - errorMsg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + errorMsg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - errorMsg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + errorMsg += buf.get(); #endif - idbassert_s(0, errorMsg); - } - } - - nleft -= nwritten; - bufp += nwritten; + idbassert_s(0, errorMsg); + } } - return nbytes; + nleft -= nwritten; + bufp += nwritten; + } + + return nbytes; } #else const size_t MAX_RECV_BYTES = 64 * 1024; void reads(SOCKET fd, void* buf, const size_t wanted) { - size_t needed = wanted; - size_t sofar = 0; - char* p = reinterpret_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; + size_t needed = wanted; + size_t sofar = 0; + char* p = reinterpret_cast(buf); + ssize_t rrc = -1; + pollfd fds[1]; + int en = 0; - fds[0].fd = fd; - fds[0].events = POLLIN; + fds[0].fd = fd; + fds[0].events = POLLIN; - while (wanted > sofar) + while (wanted > sofar) + { + fds[0].revents = 0; + poll(fds, 1, -1); + errno = 0; + // Windows recv() can only read so much at a time... + int thisrecv = static_cast(std::min(needed, MAX_RECV_BYTES)); + rrc = ::recv(fd, (p + sofar), thisrecv, 0); + en = errno; + + if (rrc < 0) { - fds[0].revents = 0; - poll(fds, 1, -1); - errno = 0; - //Windows recv() can only read so much at a time... - int thisrecv = static_cast(std::min(needed, MAX_RECV_BYTES)); - rrc = ::recv(fd, (p + sofar), thisrecv, 0); - en = errno; + if (en == EAGAIN || en == EINTR) + continue; - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR) - continue; - - ostringstream oss; - oss << "qfe: reads: read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - needed -= rrc; - sofar += rrc; + ostringstream oss; + oss << "qfe: reads: read() returned " << rrc << " (" << strerror(en) << ")"; + idbassert_s(0, oss.str()); } + + needed -= rrc; + sofar += rrc; + } } size_t writes(SOCKET fd, const void* data, const size_t nbytes) { - size_t nleft; - ssize_t nwritten; - const char* bufp = static_cast(data); - nleft = nbytes; + size_t nleft; + ssize_t nwritten; + const char* bufp = static_cast(data); + nleft = nbytes; - while (nleft > 0) + while (nleft > 0) + { + int thissend = static_cast(std::min(nleft, MAX_RECV_BYTES)); + nwritten = ::send(fd, bufp, thissend, 0); + int en = errno; + + if (nwritten == SOCKET_ERROR) { - int thissend = static_cast(std::min(nleft, MAX_RECV_BYTES)); - nwritten = ::send(fd, bufp, thissend, 0); - int en = errno; + int wsaerrno = WSAGetLastError(); - if (nwritten == SOCKET_ERROR) - { - int wsaerrno = WSAGetLastError(); - - if (en == EINTR) - nwritten = 0; - else - { - ostringstream oss; - oss << "qfe: writes: send() returned " << nwritten << " (WSA: " << wsaerrno << ")"; - idbassert_s(0, oss.str()); - } - } - - nleft -= nwritten; - bufp += nwritten; + if (en == EINTR) + nwritten = 0; + else + { + ostringstream oss; + oss << "qfe: writes: send() returned " << nwritten << " (WSA: " << wsaerrno << ")"; + idbassert_s(0, oss.str()); + } } - return nbytes; + nleft -= nwritten; + bufp += nwritten; + } + + return nbytes; } #endif uint32_t readNumber32(SockType fd) { - uint32_t np; - SockReadFcn(fd, &np, 4); - return np; + uint32_t np; + SockReadFcn(fd, &np, 4); + return np; } string readString(SockType fd) { - string s; - uint32_t len = readNumber32(fd); + string s; + uint32_t len = readNumber32(fd); - if (len > 0) - { - scoped_array buf(new char[len + 1]); - SockReadFcn(fd, buf.get(), len); - buf[len] = 0; - s = buf.get(); - } + if (len > 0) + { + scoped_array buf(new char[len + 1]); + SockReadFcn(fd, buf.get(), len); + buf[len] = 0; + s = buf.get(); + } - return s; + return s; } void writeString(SockType fd, const string& data) { - uint32_t len = data.length(); - SockWriteFcn(fd, &len, 4); - SockWriteFcn(fd, data.c_str(), len); + uint32_t len = data.length(); + SockWriteFcn(fd, &len, 4); + SockWriteFcn(fd, data.c_str(), len); } -} //namespace qfe::socketio -} //namespace qfe +} // namespace socketio +} // namespace qfe diff --git a/tools/qfe/socketio.h b/tools/qfe/socketio.h index b673ab462..8787b4102 100644 --- a/tools/qfe/socketio.h +++ b/tools/qfe/socketio.h @@ -17,7 +17,6 @@ namespace qfe { namespace socketio { - #ifndef _MSC_VER void readn(int fd, void* buf, const size_t wanted); size_t writen(int fd, const void* data, const size_t nbytes); @@ -29,8 +28,7 @@ uint32_t readNumber32(SockType); std::string readString(SockType); void writeString(SockType, const std::string&); -} //namespace qfe::socketio -} //namespace qfe +} // namespace socketio +} // namespace qfe #endif - diff --git a/tools/qfe/socktype.h b/tools/qfe/socktype.h index ba832699b..d1eeb7850 100644 --- a/tools/qfe/socktype.h +++ b/tools/qfe/socktype.h @@ -18,4 +18,3 @@ typedef int SockType; #endif #endif - diff --git a/tools/rebuildEM/BRM_saves_em_system_tables_blob.h b/tools/rebuildEM/BRM_saves_em_system_tables_blob.h index d25eaa222..157dc0612 100644 --- a/tools/rebuildEM/BRM_saves_em_system_tables_blob.h +++ b/tools/rebuildEM/BRM_saves_em_system_tables_blob.h @@ -5,336 +5,239 @@ // TODO: This could be compressed by simple `run-length encoding` // algo. static const uint8_t BRM_saves_em_system_tables_blob[] = { - 0x20, 0x8b, 0xf7, 0x76, 0x29, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd1, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xea, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xd4, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xeb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xed, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xef, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x13, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x05, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x70, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x16, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xa0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xb0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0b, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0c, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xd4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0e, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0f, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x19, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x24, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1c, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x64, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x94, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xff, 0xff, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + 0x20, 0x8b, 0xf7, 0x76, 0x29, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xea, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xeb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xfd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x90, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xa0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xc4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0d, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x24, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x64, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xff, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; #endif diff --git a/tools/rebuildEM/main.cpp b/tools/rebuildEM/main.cpp index 3b82843e4..c59e8cb5f 100644 --- a/tools/rebuildEM/main.cpp +++ b/tools/rebuildEM/main.cpp @@ -31,130 +31,117 @@ using namespace RebuildExtentMap; static void usage(const string& pname) { - std::cout << "usage: " << pname << " [-vdhs]" << std::endl; - std::cout - << "rebuilds the extent map from the contents of the database file " - "system." - << std::endl; - std::cout - << " -v display verbose progress information. More v's, more debug" - << std::endl; - std::cout << " -d display what would be done--don't do it" << std::endl; - std::cout << " -h display this help text" << std::endl; - std::cout << " -s show extent map and quit" << std::endl; + std::cout << "usage: " << pname << " [-vdhs]" << std::endl; + std::cout << "rebuilds the extent map from the contents of the database file " + "system." + << std::endl; + std::cout << " -v display verbose progress information. More v's, more debug" << std::endl; + std::cout << " -d display what would be done--don't do it" << std::endl; + std::cout << " -h display this help text" << std::endl; + std::cout << " -s show extent map and quit" << std::endl; } int main(int argc, char** argv) { - int32_t option; - std::string pname(argv[0]); - bool showExtentMap = false; - bool verbose = false; - bool display = false; + int32_t option; + std::string pname(argv[0]); + bool showExtentMap = false; + bool verbose = false; + bool display = false; - while ((option = getopt(argc, argv, "vdhs")) != EOF) + while ((option = getopt(argc, argv, "vdhs")) != EOF) + { + switch (option) { - switch (option) - { - case 'v': - verbose = true; - break; + case 'v': verbose = true; break; - case 'd': - display = true; - break; + case 'd': display = true; break; - case 's': - showExtentMap = true; - break; + case 's': showExtentMap = true; break; - case 'h': - case '?': - default: - usage(pname); - return (option == 'h' ? 0 : 1); - break; - } + case 'h': + case '?': + default: + usage(pname); + return (option == 'h' ? 0 : 1); + break; } + } - EMReBuilder emReBuilder(verbose, display); - // Just show EM and quit. - if (showExtentMap) - { - emReBuilder.showExtentMap(); - return 0; - } - - // MCOL-4685 - std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. " - << std::endl; - std::cout << "Requirement: all DBRoots must be on this node. " << std::endl; - std::cout << "Do you want to continue Y/N? "; - std::string confirmation; - cin >> confirmation; - if (confirmation.size() == 0) - return 0; - - boost::algorithm::to_lower(confirmation); - if (!(confirmation == "y" || confirmation == "yes")) - return 0; - - auto* config = config::Config::makeConfig(); - - // Check for storage type. - const auto DBRootStorageType = config->getConfig("Installation", "DBRootStorageType"); - if (DBRootStorageType != "internal") - { - std::cout << "Only internal DBRootStorageType is supported, provided: " << DBRootStorageType - << std::endl; - return 0; - } - - const auto BRMSavesEM = - config->getConfig("SystemConfig", "DBRMRoot") + "_em"; - // Check for `BRM_saves_em` file presents. - // TODO: Should we add force option to remove file? - if (boost::filesystem::exists(BRMSavesEM)) - { - std::cout << BRMSavesEM << " file exists. " << std::endl; - std::cout << "Please note: this tool is only suitable in situations " - "where there is no `BRM_saves_em` file. " - << std::endl; - std::cout << "If `BRM_saves_em` " - "exists extent map will be restored from it. " - << std::endl; - std::cout << "Exiting. " << std::endl; - return 0; - } - - // Initialize system extents from the binary blob. - auto rc = emReBuilder.initializeSystemExtents(); - if (rc == -1) - { - std::cerr << "Cannot initialize system extents from binary blob." << std::endl; - std::cerr << "Exiting. " << std::endl; - return 1; - } - - // Make config from default path. - std::string count = config->getConfig("SystemConfig", "DBRootCount"); - // Read the number of DBRoots. - uint32_t dbRootCount = config->uFromText(count); - - // Iterate over DBRoots starting from the first one. - for (uint32_t dbRootNumber = 1; dbRootNumber <= dbRootCount; - ++dbRootNumber) - { - std::string dbRootName = "DBRoot" + std::to_string(dbRootNumber); - auto dbRootPath = config->getConfig("SystemConfig", dbRootName); - emReBuilder.setDBRoot(dbRootNumber); - emReBuilder.collectExtents(dbRootPath.c_str()); - emReBuilder.rebuildExtentMap(); - emReBuilder.clear(); - } - - // Save restored extent map. - emReBuilder.getEM().save(BRMSavesEM); - std::cout << "Completed." << std::endl; + EMReBuilder emReBuilder(verbose, display); + // Just show EM and quit. + if (showExtentMap) + { + emReBuilder.showExtentMap(); return 0; + } + + // MCOL-4685 + std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. " << std::endl; + std::cout << "Requirement: all DBRoots must be on this node. " << std::endl; + std::cout << "Do you want to continue Y/N? "; + std::string confirmation; + cin >> confirmation; + if (confirmation.size() == 0) + return 0; + + boost::algorithm::to_lower(confirmation); + if (!(confirmation == "y" || confirmation == "yes")) + return 0; + + auto* config = config::Config::makeConfig(); + + // Check for storage type. + const auto DBRootStorageType = config->getConfig("Installation", "DBRootStorageType"); + if (DBRootStorageType != "internal") + { + std::cout << "Only internal DBRootStorageType is supported, provided: " << DBRootStorageType << std::endl; + return 0; + } + + const auto BRMSavesEM = config->getConfig("SystemConfig", "DBRMRoot") + "_em"; + // Check for `BRM_saves_em` file presents. + // TODO: Should we add force option to remove file? + if (boost::filesystem::exists(BRMSavesEM)) + { + std::cout << BRMSavesEM << " file exists. " << std::endl; + std::cout << "Please note: this tool is only suitable in situations " + "where there is no `BRM_saves_em` file. " + << std::endl; + std::cout << "If `BRM_saves_em` " + "exists extent map will be restored from it. " + << std::endl; + std::cout << "Exiting. " << std::endl; + return 0; + } + + // Initialize system extents from the binary blob. + auto rc = emReBuilder.initializeSystemExtents(); + if (rc == -1) + { + std::cerr << "Cannot initialize system extents from binary blob." << std::endl; + std::cerr << "Exiting. " << std::endl; + return 1; + } + + // Make config from default path. + std::string count = config->getConfig("SystemConfig", "DBRootCount"); + // Read the number of DBRoots. + uint32_t dbRootCount = config->uFromText(count); + + // Iterate over DBRoots starting from the first one. + for (uint32_t dbRootNumber = 1; dbRootNumber <= dbRootCount; ++dbRootNumber) + { + std::string dbRootName = "DBRoot" + std::to_string(dbRootNumber); + auto dbRootPath = config->getConfig("SystemConfig", dbRootName); + emReBuilder.setDBRoot(dbRootNumber); + emReBuilder.collectExtents(dbRootPath.c_str()); + emReBuilder.rebuildExtentMap(); + emReBuilder.clear(); + } + + // Save restored extent map. + emReBuilder.getEM().save(BRMSavesEM); + std::cout << "Completed." << std::endl; + return 0; } diff --git a/tools/rebuildEM/rebuildEM.cpp b/tools/rebuildEM/rebuildEM.cpp index d74e7a942..8dc5954d5 100644 --- a/tools/rebuildEM/rebuildEM.cpp +++ b/tools/rebuildEM/rebuildEM.cpp @@ -36,395 +36,370 @@ using namespace idbdatafile; namespace RebuildExtentMap { - int32_t EMReBuilder::collectExtents(const string& dbRootPath) { - if (doVerbose()) - { - std::cout << "Collect extents for the DBRoot " << dbRootPath - << std::endl; - } + if (doVerbose()) + { + std::cout << "Collect extents for the DBRoot " << dbRootPath << std::endl; + } - for (boost::filesystem::recursive_directory_iterator dirIt(dbRootPath), - dirEnd; - dirIt != dirEnd; ++dirIt) - { - (void) collectExtent(dirIt->path().string()); - } + for (boost::filesystem::recursive_directory_iterator dirIt(dbRootPath), dirEnd; dirIt != dirEnd; ++dirIt) + { + (void)collectExtent(dirIt->path().string()); + } - return 0; + return 0; } int32_t EMReBuilder::collectExtent(const std::string& fullFileName) { - WriteEngine::FileOp fileOp; - uint32_t oid; - uint32_t partition; - uint32_t segment; - // Initialize oid, partition and segment from the given `fullFileName`. - auto rc = WriteEngine::Convertor::fileName2Oid(fullFileName, oid, - partition, segment); - if (rc != 0) - return rc; + WriteEngine::FileOp fileOp; + uint32_t oid; + uint32_t partition; + uint32_t segment; + // Initialize oid, partition and segment from the given `fullFileName`. + auto rc = WriteEngine::Convertor::fileName2Oid(fullFileName, oid, partition, segment); + if (rc != 0) + return rc; - // Open the given file. - std::unique_ptr dbFile(IDBDataFile::open( - IDBPolicy::getType(fullFileName, IDBPolicy::WRITEENG), - fullFileName.c_str(), "rb", 1)); + // Open the given file. + std::unique_ptr dbFile(IDBDataFile::open(IDBPolicy::getType(fullFileName, IDBPolicy::WRITEENG), + fullFileName.c_str(), "rb", 1)); - if (!dbFile) + if (!dbFile) + { + std::cerr << "Cannot open file " << fullFileName << std::endl; + return -1; + } + + rc = dbFile->seek(0, 0); + if (rc != 0) + { + std::cerr << "IDBDataFile::seek failed for the file " << fullFileName << std::endl; + return rc; + } + + // Read and verify header. + char fileHeader[compress::CompressInterface::HDR_BUF_LEN * 2]; + rc = fileOp.readHeaders(dbFile.get(), fileHeader); + if (rc != 0) + { + // FIXME: If the file was created without a compression, it does not + // have a header block, so header verification fails in this case, + // currently we skip it, because we cannot deduce needed data to create + // a column extent from the blob file. + // Skip fileID from system catalog. + if (doVerbose() && oid > 3000) { - std::cerr << "Cannot open file " << fullFileName << std::endl; - return -1; + std::cerr << "Cannot read file header from the file " << fullFileName + << ", probably this file was created without compression. " << std::endl; + } + return rc; + } + + if (doVerbose()) + { + std::cout << "Processing file: " << fullFileName << std::endl; + std::cout << "fileName2Oid: [OID: " << oid << ", partition: " << partition << ", segment: " << segment + << "] " << std::endl; + } + + // Read the `colDataType` and `colWidth` from the given header. + const auto versionNumber = compress::CompressInterface::getVersionNumber(fileHeader); + // Verify header number. + if (versionNumber < 3) + { + if (doVerbose()) + { + std::cerr << "File version " << versionNumber << " is not supported. " << std::endl; + } + return -1; + } + + auto colDataType = compress::CompressInterface::getColDataType(fileHeader); + auto colWidth = compress::CompressInterface::getColumnWidth(fileHeader); + auto blockCount = compress::CompressInterface::getBlockCount(fileHeader); + auto lbidCount = compress::CompressInterface::getLBIDCount(fileHeader); + auto compressionType = compress::CompressInterface::getCompressionType(fileHeader); + + if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED) + { + if (doVerbose()) + std::cout << "File header has invalid data. " << std::endl; + + return -1; + } + + auto isDict = isDictFile(colDataType, colWidth); + if (isDict) + colWidth = 8; + + if (doVerbose()) + { + std::cout << "Searching for HWM... " << std::endl; + std::cout << "Block count: " << blockCount << std::endl; + } + + uint64_t hwm = 0; + rc = searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, colDataType, colWidth, blockCount, isDict, + compressionType, hwm); + if (rc != 0) + return rc; + + if (doVerbose()) + std::cout << "HWM is: " << hwm << std::endl; + + const uint32_t extentMaxBlockCount = getEM().getExtentRows() * colWidth / BLOCK_SIZE; + // We found multiple extents per one segment file. + if (hwm >= extentMaxBlockCount) + { + for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex) + { + auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidIndex); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, /*hwm*/ 0, isDict); + extentMap.push_back(fileId); } - rc = dbFile->seek(0, 0); - if (rc != 0) - { - std::cerr << "IDBDataFile::seek failed for the file " << fullFileName - << std::endl; - return rc; - } - - // Read and verify header. - char fileHeader[compress::CompressInterface::HDR_BUF_LEN * 2]; - rc = fileOp.readHeaders(dbFile.get(), fileHeader); - if (rc != 0) - { - // FIXME: If the file was created without a compression, it does not - // have a header block, so header verification fails in this case, - // currently we skip it, because we cannot deduce needed data to create - // a column extent from the blob file. - // Skip fileID from system catalog. - if (doVerbose() && oid > 3000) - { - std::cerr << "Cannot read file header from the file " << fullFileName - << ", probably this file was created without compression. " << std::endl; - } - return rc; - } + // Last one has an actual HWM. + auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidCount - 1); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); + extentMap.push_back(fileId); if (doVerbose()) { - std::cout << "Processing file: " << fullFileName << std::endl; - std::cout << "fileName2Oid: [OID: " << oid - << ", partition: " << partition << ", segment: " << segment - << "] " << std::endl; + std::cout << "Found multiple extents per segment file " << std::endl; + std::cout << "FileId is collected " << fileId << std::endl; } - - // Read the `colDataType` and `colWidth` from the given header. - const auto versionNumber = - compress::CompressInterface::getVersionNumber(fileHeader); - // Verify header number. - if (versionNumber < 3) - { - if (doVerbose()) - { - std::cerr << "File version " << versionNumber - << " is not supported. " << std::endl; - } - return -1; - } - - auto colDataType = compress::CompressInterface::getColDataType(fileHeader); - auto colWidth = compress::CompressInterface::getColumnWidth(fileHeader); - auto blockCount = compress::CompressInterface::getBlockCount(fileHeader); - auto lbidCount = compress::CompressInterface::getLBIDCount(fileHeader); - auto compressionType = compress::CompressInterface::getCompressionType(fileHeader); - - if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED) - { - if (doVerbose()) - std::cout << "File header has invalid data. " << std::endl; - - return -1; - } - - auto isDict = isDictFile(colDataType, colWidth); - if (isDict) - colWidth = 8; + } + else + { + // One extent per segment file. + auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, 0); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); + extentMap.push_back(fileId); if (doVerbose()) - { - std::cout << "Searching for HWM... " << std::endl; - std::cout << "Block count: " << blockCount << std::endl; - } + std::cout << "FileId is collected " << fileId << std::endl; + } - uint64_t hwm = 0; - rc = searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, colDataType, colWidth, - blockCount, isDict, compressionType, hwm); - if (rc != 0) - return rc; - - if (doVerbose()) - std::cout << "HWM is: " << hwm << std::endl; - - const uint32_t extentMaxBlockCount = getEM().getExtentRows() * colWidth / BLOCK_SIZE; - // We found multiple extents per one segment file. - if (hwm >= extentMaxBlockCount) - { - for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex) - { - auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidIndex); - FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, /*hwm*/ 0, isDict); - extentMap.push_back(fileId); - } - - // Last one has an actual HWM. - auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidCount - 1); - FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); - extentMap.push_back(fileId); - - if (doVerbose()) - { - std::cout << "Found multiple extents per segment file " - << std::endl; - std::cout << "FileId is collected " << fileId << std::endl; - } - } - else - { - // One extent per segment file. - auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, 0); - FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); - extentMap.push_back(fileId); - - if (doVerbose()) - std::cout << "FileId is collected " << fileId << std::endl; - } - - return 0; + return 0; } int32_t EMReBuilder::rebuildExtentMap() { - if (doVerbose()) + if (doVerbose()) + { + std::cout << "Build extent map with size " << extentMap.size() << std::endl; + } + + // We have to restore extent map by restoring individual extent in order + // they were created. This is important part, otherwise we will get invalid + // extents for dictionary segment files and will not be able to access + // columns through dictionay segment files. + sort(extentMap.begin(), extentMap.end(), + [](const FileId& lhs, const FileId& rhs) { return lhs.lbid < rhs.lbid; }); + + for (const auto& fileId : extentMap) + { + uint32_t startBlockOffset; + int32_t allocdSize; + BRM::LBID_t lbid; + + if (!doDisplay()) { - std::cout << "Build extent map with size " << extentMap.size() - << std::endl; - } - - // We have to restore extent map by restoring individual extent in order - // they were created. This is important part, otherwise we will get invalid - // extents for dictionary segment files and will not be able to access - // columns through dictionay segment files. - sort(extentMap.begin(), extentMap.end(), - [](const FileId& lhs, const FileId& rhs) { - return lhs.lbid < rhs.lbid; - }); - - for (const auto& fileId : extentMap) - { - uint32_t startBlockOffset; - int32_t allocdSize; - BRM::LBID_t lbid; - - if (!doDisplay()) + try + { + if (fileId.isDict) { - try - { - if (fileId.isDict) - { - // Create a dictionary extent for the given oid, partition, - // segment, dbroot. - getEM().createDictStoreExtent( - fileId.oid, getDBRoot(), fileId.partition, - fileId.segment, lbid, allocdSize); - } - else - { - // Create a column extent for the given oid, partition, - // segment, dbroot and column width. - getEM().createColumnExtentExactFile( - fileId.oid, fileId.colWidth, getDBRoot(), - fileId.partition, fileId.segment, fileId.colDataType, - lbid, allocdSize, startBlockOffset); - } - } - // Could throw an logic_error exception. - catch (std::exception& e) - { - getEM().undoChanges(); - std::cerr << "Cannot create column extent: " << e.what() - << std::endl; - return -1; - } - - getEM().confirmChanges(); - if (doVerbose()) - { - std::cout << "Extent is created, allocated size " << allocdSize - << " actual LBID " << lbid << std::endl; - std::cout << "For " << fileId << std::endl; - } - - // This is important part, it sets a status for specific extent - // as 'available' that means we can use it. - if (doVerbose()) - { - std::cout << "Setting a HWM for " << fileId << std::endl; - } - try - { - getEM().setLocalHWM(fileId.oid, fileId.partition, - fileId.segment, fileId.hwm, false, true); - } - catch (std::exception& e) - { - getEM().undoChanges(); - std::cerr << "Cannot set local HWM: " << e.what() << std::endl; - return -1; - } - getEM().confirmChanges(); - } - } - return 0; -} - -int32_t EMReBuilder::searchHWMInSegmentFile( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, - uint64_t blockCount, bool isDict, uint32_t compressionType, uint64_t& hwm) -{ - std::unique_ptr chunkManagerWrapper; - try - { - if (isDict) - { - chunkManagerWrapper = std::unique_ptr( - new ChunkManagerWrapperDict(oid, dbRoot, partition, segment, - colDataType, colWidth, - compressionType)); + // Create a dictionary extent for the given oid, partition, + // segment, dbroot. + getEM().createDictStoreExtent(fileId.oid, getDBRoot(), fileId.partition, fileId.segment, lbid, + allocdSize); } else { - chunkManagerWrapper = std::unique_ptr( - new ChunkManagerWrapperColumn(oid, dbRoot, partition, segment, - colDataType, colWidth, - compressionType)); + // Create a column extent for the given oid, partition, + // segment, dbroot and column width. + getEM().createColumnExtentExactFile(fileId.oid, fileId.colWidth, getDBRoot(), fileId.partition, + fileId.segment, fileId.colDataType, lbid, allocdSize, + startBlockOffset); } - } - catch (...) - { - std::cerr << "Cannot create ChunkManagerWrapper" << std::endl; + } + // Could throw an logic_error exception. + catch (std::exception& e) + { + getEM().undoChanges(); + std::cerr << "Cannot create column extent: " << e.what() << std::endl; return -1; - } + } - hwm = 0; - // Starting from the end. - // Note: This solves problem related to `bulk` insertion. - // Currently it could start to insert values from any block into empty - // column. - for (int32_t currentBlock = blockCount - 1; currentBlock >= 0; - --currentBlock) + getEM().confirmChanges(); + if (doVerbose()) + { + std::cout << "Extent is created, allocated size " << allocdSize << " actual LBID " << lbid + << std::endl; + std::cout << "For " << fileId << std::endl; + } + + // This is important part, it sets a status for specific extent + // as 'available' that means we can use it. + if (doVerbose()) + { + std::cout << "Setting a HWM for " << fileId << std::endl; + } + try + { + getEM().setLocalHWM(fileId.oid, fileId.partition, fileId.segment, fileId.hwm, false, true); + } + catch (std::exception& e) + { + getEM().undoChanges(); + std::cerr << "Cannot set local HWM: " << e.what() << std::endl; + return -1; + } + getEM().confirmChanges(); + } + } + return 0; +} + +int32_t EMReBuilder::searchHWMInSegmentFile(uint32_t oid, uint32_t dbRoot, uint32_t partition, + uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth, uint64_t blockCount, bool isDict, + uint32_t compressionType, uint64_t& hwm) +{ + std::unique_ptr chunkManagerWrapper; + try + { + if (isDict) { - // Read the block associated to HWM. - // The uncompressed chunk size is 512 * 1024 * 8, so for `abbreviated` - // extent one chunk will hold all blocks, therefore we need to - // decompress chunk only once, the cached chunk will be used later. - chunkManagerWrapper->readBlock(currentBlock); - if (!chunkManagerWrapper->isEmptyBlock()) - { - hwm = currentBlock; - break; - } + chunkManagerWrapper = std::unique_ptr(new ChunkManagerWrapperDict( + oid, dbRoot, partition, segment, colDataType, colWidth, compressionType)); } + else + { + chunkManagerWrapper = std::unique_ptr(new ChunkManagerWrapperColumn( + oid, dbRoot, partition, segment, colDataType, colWidth, compressionType)); + } + } + catch (...) + { + std::cerr << "Cannot create ChunkManagerWrapper" << std::endl; + return -1; + } - return 0; + hwm = 0; + // Starting from the end. + // Note: This solves problem related to `bulk` insertion. + // Currently it could start to insert values from any block into empty + // column. + for (int32_t currentBlock = blockCount - 1; currentBlock >= 0; --currentBlock) + { + // Read the block associated to HWM. + // The uncompressed chunk size is 512 * 1024 * 8, so for `abbreviated` + // extent one chunk will hold all blocks, therefore we need to + // decompress chunk only once, the cached chunk will be used later. + chunkManagerWrapper->readBlock(currentBlock); + if (!chunkManagerWrapper->isEmptyBlock()) + { + hwm = currentBlock; + break; + } + } + + return 0; } void EMReBuilder::showExtentMap() { - std::cout << "range.start|range.size|fileId|blockOffset|HWM|partition|" - "segment|dbroot|width|status|hiVal|loVal|seqNum|isValid|" - << std::endl; - getEM().dumpTo(std::cout); + std::cout << "range.start|range.size|fileId|blockOffset|HWM|partition|" + "segment|dbroot|width|status|hiVal|loVal|seqNum|isValid|" + << std::endl; + getEM().dumpTo(std::cout); } // Check for hard-coded values which define dictionary file. -bool EMReBuilder::isDictFile( - execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t width) +bool EMReBuilder::isDictFile(execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t width) { - // Dictionary store extent has a width == 0. See more detais in - // `createDictStoreExtent` function. - return (colDataType == execplan::CalpontSystemCatalog::VARCHAR) && - (width == 0); + // Dictionary store extent has a width == 0. See more detais in + // `createDictStoreExtent` function. + return (colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (width == 0); } int32_t EMReBuilder::initializeSystemExtents() { - if (!doDisplay()) + if (!doDisplay()) + { + if (doVerbose()) { - if (doVerbose()) - { - std::cout << "Initialize system extents from the initial state" - << std::endl; - } - try - { - getEM().loadFromBinaryBlob(reinterpret_cast( - BRM_saves_em_system_tables_blob)); - } - catch (...) - { - std::cerr << "Cannot initialize system extents from binary blob " - << std::endl; - return -1; - } + std::cout << "Initialize system extents from the initial state" << std::endl; } - return 0; + try + { + getEM().loadFromBinaryBlob(reinterpret_cast(BRM_saves_em_system_tables_blob)); + } + catch (...) + { + std::cerr << "Cannot initialize system extents from binary blob " << std::endl; + return -1; + } + } + return 0; } -ChunkManagerWrapper::ChunkManagerWrapper( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth) - : oid(oid), dbRoot(dbRoot), partition(partition), segment(segment), - colDataType(colDataType), colWidth(colWidth), size(colWidth), - pFileOp(nullptr) +ChunkManagerWrapper::ChunkManagerWrapper(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth) + : oid(oid) + , dbRoot(dbRoot) + , partition(partition) + , segment(segment) + , colDataType(colDataType) + , colWidth(colWidth) + , size(colWidth) + , pFileOp(nullptr) { } int32_t ChunkManagerWrapper::readBlock(uint32_t blockNumber) { - auto rc = chunkManager.readBlock(pFile, blockData, blockNumber); - if (rc != 0) - return rc; - return 0; + auto rc = chunkManager.readBlock(pFile, blockData, blockNumber); + if (rc != 0) + return rc; + return 0; } -ChunkManagerWrapperColumn::ChunkManagerWrapperColumn( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, - uint32_t compressionType) - : ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, - colWidth) +ChunkManagerWrapperColumn::ChunkManagerWrapperColumn(uint32_t oid, uint32_t dbRoot, uint32_t partition, + uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth, uint32_t compressionType) + : ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, colWidth) { - pFileOp = std::unique_ptr( - new WriteEngine::ColumnOpCompress1(compressionType)); - chunkManager.fileOp(pFileOp.get()); - // Open compressed column segment file. We will read block by block - // from the compressed chunks. - pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, - colDataType, colWidth, fileName, - "rb", size, false, false); - if (!pFile) - { - throw std::bad_alloc(); - } + pFileOp = + std::unique_ptr(new WriteEngine::ColumnOpCompress1(compressionType)); + chunkManager.fileOp(pFileOp.get()); + // Open compressed column segment file. We will read block by block + // from the compressed chunks. + pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, colDataType, colWidth, fileName, + "rb", size, false, false); + if (!pFile) + { + throw std::bad_alloc(); + } - emptyValue = pFileOp->getEmptyRowValue(colDataType, colWidth); - midOffset = (WriteEngine::BYTE_PER_BLOCK / 2); - endOffset = (WriteEngine::BYTE_PER_BLOCK - colWidth); + emptyValue = pFileOp->getEmptyRowValue(colDataType, colWidth); + midOffset = (WriteEngine::BYTE_PER_BLOCK / 2); + endOffset = (WriteEngine::BYTE_PER_BLOCK - colWidth); } bool ChunkManagerWrapperColumn::isEmptyBlock() { - const uint8_t* beginValue = blockData; - const uint8_t* midValue = blockData + midOffset; - const uint8_t* endValue = blockData + endOffset; + const uint8_t* beginValue = blockData; + const uint8_t* midValue = blockData + midOffset; + const uint8_t* endValue = blockData + endOffset; - return isEmptyValue(beginValue) && isEmptyValue(midValue) && - isEmptyValue(endValue); + return isEmptyValue(beginValue) && isEmptyValue(midValue) && isEmptyValue(endValue); } // This function is copy pasted from `ColumnOp` file, unfortunately it's not @@ -434,65 +409,55 @@ bool ChunkManagerWrapperColumn::isEmptyBlock() // object. bool ChunkManagerWrapperColumn::isEmptyValue(const uint8_t* value) const { - switch (colWidth) - { - case 1: - return *(uint8_t*) value == *(uint8_t*) emptyValue; + switch (colWidth) + { + case 1: return *(uint8_t*)value == *(uint8_t*)emptyValue; - case 2: - return *(uint16_t*) value == *(uint16_t*) emptyValue; + case 2: return *(uint16_t*)value == *(uint16_t*)emptyValue; - case 4: - return *(uint32_t*) value == *(uint32_t*) emptyValue; + case 4: return *(uint32_t*)value == *(uint32_t*)emptyValue; - case 8: - return *(uint64_t*) value == *(uint64_t*) emptyValue; + case 8: return *(uint64_t*)value == *(uint64_t*)emptyValue; - case 16: - return *(uint128_t*) value == *(uint128_t*) emptyValue; - } + case 16: return *(uint128_t*)value == *(uint128_t*)emptyValue; + } - return false; + return false; } -ChunkManagerWrapperDict::ChunkManagerWrapperDict( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, - uint32_t compressionType) - : ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, - colWidth) +ChunkManagerWrapperDict::ChunkManagerWrapperDict(uint32_t oid, uint32_t dbRoot, uint32_t partition, + uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth, uint32_t compressionType) + : ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, colWidth) { - pFileOp = std::unique_ptr( - new WriteEngine::DctnryCompress1(compressionType)); - chunkManager.fileOp(pFileOp.get()); - // Open compressed dict segment file. - pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, - colDataType, colWidth, fileName, - "rb", size, false, true); - if (!pFile) - { - throw std::bad_alloc(); - } + pFileOp = std::unique_ptr(new WriteEngine::DctnryCompress1(compressionType)); + chunkManager.fileOp(pFileOp.get()); + // Open compressed dict segment file. + pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, colDataType, colWidth, fileName, + "rb", size, false, true); + if (!pFile) + { + throw std::bad_alloc(); + } - auto dictBlockHeaderSize = - WriteEngine::HDR_UNIT_SIZE + WriteEngine::NEXT_PTR_BYTES + - WriteEngine::HDR_UNIT_SIZE + WriteEngine::HDR_UNIT_SIZE; + auto dictBlockHeaderSize = WriteEngine::HDR_UNIT_SIZE + WriteEngine::NEXT_PTR_BYTES + + WriteEngine::HDR_UNIT_SIZE + WriteEngine::HDR_UNIT_SIZE; - emptyBlock = WriteEngine::BYTE_PER_BLOCK - dictBlockHeaderSize; + emptyBlock = WriteEngine::BYTE_PER_BLOCK - dictBlockHeaderSize; } bool ChunkManagerWrapperDict::isEmptyBlock() { - // Check that block is completely empty. - return (*(uint16_t*) blockData) == emptyBlock; + // Check that block is completely empty. + return (*(uint16_t*)blockData) == emptyBlock; } std::ostream& operator<<(std::ostream& os, const FileId& fileID) { - os << "[OID: " << fileID.oid << ", partition: " << fileID.partition - << ", segment: " << fileID.segment << ", col width: " << fileID.colWidth - << ", lbid:" << fileID.lbid << ", hwm: " << fileID.hwm - << ", isDict: " << fileID.isDict << "]"; - return os; + os << "[OID: " << fileID.oid << ", partition: " << fileID.partition << ", segment: " << fileID.segment + << ", col width: " << fileID.colWidth << ", lbid:" << fileID.lbid << ", hwm: " << fileID.hwm + << ", isDict: " << fileID.isDict << "]"; + return os; } -} // namespace RebuildExtentMap +} // namespace RebuildExtentMap diff --git a/tools/rebuildEM/rebuildEM.h b/tools/rebuildEM/rebuildEM.h index 481a2e102..6e2d11b63 100644 --- a/tools/rebuildEM/rebuildEM.h +++ b/tools/rebuildEM/rebuildEM.h @@ -40,192 +40,201 @@ namespace RebuildExtentMap // This struct represents a FileId. For internal purpose only. struct FileId { - FileId(uint32_t oid, uint32_t partition, uint32_t segment, - uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType, - int64_t lbid, uint64_t hwm, bool isDict) - : oid(oid), partition(partition), segment(segment), colWidth(colWidth), - colDataType(colDataType), lbid(lbid), hwm(hwm), isDict(isDict) - { - } + FileId(uint32_t oid, uint32_t partition, uint32_t segment, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType, int64_t lbid, uint64_t hwm, bool isDict) + : oid(oid) + , partition(partition) + , segment(segment) + , colWidth(colWidth) + , colDataType(colDataType) + , lbid(lbid) + , hwm(hwm) + , isDict(isDict) + { + } - uint32_t oid; - uint32_t partition; - uint32_t segment; - uint32_t colWidth; - execplan::CalpontSystemCatalog::ColDataType colDataType; - int64_t lbid; - uint64_t hwm; - bool isDict; + uint32_t oid; + uint32_t partition; + uint32_t segment; + uint32_t colWidth; + execplan::CalpontSystemCatalog::ColDataType colDataType; + int64_t lbid; + uint64_t hwm; + bool isDict; }; std::ostream& operator<<(std::ostream& os, const FileId& fileID); // This class represents extent map rebuilder. class EMReBuilder { - public: - EMReBuilder(bool verbose, bool display) - : verbose(verbose), display(display) - { - // Initalize plugins. - IDBPolicy::init(true, false, "", 0); - } - ~EMReBuilder() = default; + public: + EMReBuilder(bool verbose, bool display) : verbose(verbose), display(display) + { + // Initalize plugins. + IDBPolicy::init(true, false, "", 0); + } + ~EMReBuilder() = default; - // Collects extents from the given DBRoot path. - int32_t collectExtents(const std::string& dbRootPath); + // Collects extents from the given DBRoot path. + int32_t collectExtents(const std::string& dbRootPath); - // Clears collected extents. - void clear() { extentMap.clear(); } + // Clears collected extents. + void clear() + { + extentMap.clear(); + } - // Specifies whether we need verbose to output. - bool doVerbose() const { return verbose; } + // Specifies whether we need verbose to output. + bool doVerbose() const + { + return verbose; + } - // Specifies whether we need just display a pipeline, but not actually run - // it. - bool doDisplay() const { return display; } + // Specifies whether we need just display a pipeline, but not actually run + // it. + bool doDisplay() const + { + return display; + } - // Returns the number of current DBRoot. - uint32_t getDBRoot() const { return dbRoot; } + // Returns the number of current DBRoot. + uint32_t getDBRoot() const + { + return dbRoot; + } - // Retunrs a reference to `ExtentMap` object. - BRM::ExtentMap& getEM() { return em; } + // Retunrs a reference to `ExtentMap` object. + BRM::ExtentMap& getEM() + { + return em; + } - // Checks if the given data specifies a dictionary file. - static bool - isDictFile(execplan::CalpontSystemCatalog::ColDataType colDataType, - uint64_t width); + // Checks if the given data specifies a dictionary file. + static bool isDictFile(execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t width); - // Initializes system extents from the binary blob. - // This function solves the problem related to system segment files. - // Currently those files do not have file header, so we cannot - // get the data (like width, colType, lbid) to restore an extent for this - // particular segment file. The current approach is to keep a binary blob - // of initial state of the system extents. - // Returns -1 on error. - int32_t initializeSystemExtents(); + // Initializes system extents from the binary blob. + // This function solves the problem related to system segment files. + // Currently those files do not have file header, so we cannot + // get the data (like width, colType, lbid) to restore an extent for this + // particular segment file. The current approach is to keep a binary blob + // of initial state of the system extents. + // Returns -1 on error. + int32_t initializeSystemExtents(); - // Rebuilds extent map from the collected map. - int32_t rebuildExtentMap(); + // Rebuilds extent map from the collected map. + int32_t rebuildExtentMap(); - // Search HWM in the given segment file. - int32_t searchHWMInSegmentFile( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t width, uint64_t blocksCount, bool isDict, - uint32_t compressionType, uint64_t& hwm); + // Search HWM in the given segment file. + int32_t searchHWMInSegmentFile(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t width, + uint64_t blocksCount, bool isDict, uint32_t compressionType, uint64_t& hwm); - // Sets the dbroot to the given `number`. - void setDBRoot(uint32_t number) { dbRoot = number; } + // Sets the dbroot to the given `number`. + void setDBRoot(uint32_t number) + { + dbRoot = number; + } - // Shows the extent map. - void showExtentMap(); + // Shows the extent map. + void showExtentMap(); - private: - EMReBuilder(const EMReBuilder&) = delete; - EMReBuilder(EMReBuilder&&) = delete; - EMReBuilder& operator=(const EMReBuilder&) = delete; - EMReBuilder& operator=(EMReBuilder&&) = delete; + private: + EMReBuilder(const EMReBuilder&) = delete; + EMReBuilder(EMReBuilder&&) = delete; + EMReBuilder& operator=(const EMReBuilder&) = delete; + EMReBuilder& operator=(EMReBuilder&&) = delete; - // Collects the information for extent from the given file and stores - // it in `extentMap` set. - int32_t collectExtent(const std::string& fullFileName); - bool verbose; - bool display; - uint32_t dbRoot; - BRM::ExtentMap em; - std::vector extentMap; + // Collects the information for extent from the given file and stores + // it in `extentMap` set. + int32_t collectExtent(const std::string& fullFileName); + bool verbose; + bool display; + uint32_t dbRoot; + BRM::ExtentMap em; + std::vector extentMap; }; // The base class aroud `ChunkManager` to read and write decompressed blocks // from segment file. class ChunkManagerWrapper { - public: - ChunkManagerWrapper( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t colWidth); + public: + ChunkManagerWrapper(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth); - virtual ~ChunkManagerWrapper() = default; - ChunkManagerWrapper(const ChunkManagerWrapper& other) = delete; - ChunkManagerWrapper& operator=(const ChunkManagerWrapper& other) = delete; - ChunkManagerWrapper(ChunkManagerWrapper&& other) = delete; - ChunkManagerWrapper& operator=(ChunkManagerWrapper&& other) = delete; + virtual ~ChunkManagerWrapper() = default; + ChunkManagerWrapper(const ChunkManagerWrapper& other) = delete; + ChunkManagerWrapper& operator=(const ChunkManagerWrapper& other) = delete; + ChunkManagerWrapper(ChunkManagerWrapper&& other) = delete; + ChunkManagerWrapper& operator=(ChunkManagerWrapper&& other) = delete; - // Reads block, by given `blockNumber` from associated segment file and - // populates internal block buffer. - int32_t readBlock(uint32_t blockNumber); + // Reads block, by given `blockNumber` from associated segment file and + // populates internal block buffer. + int32_t readBlock(uint32_t blockNumber); - // Checks that last read block is empty. - virtual bool isEmptyBlock() = 0; + // Checks that last read block is empty. + virtual bool isEmptyBlock() = 0; - protected: - uint32_t oid; - uint32_t dbRoot; - uint32_t partition; - uint32_t segment; - execplan::CalpontSystemCatalog::ColDataType colDataType; - uint32_t colWidth; - int32_t size; - std::string fileName; - std::unique_ptr pFileOp; - // Note: We cannot clear this pointer directly, because - // `ChunkManager` closes this file for us, otherwise we will get double - // free error. - IDBDataFile* pFile; - WriteEngine::ChunkManager chunkManager; - uint8_t blockData[WriteEngine::BYTE_PER_BLOCK]; + protected: + uint32_t oid; + uint32_t dbRoot; + uint32_t partition; + uint32_t segment; + execplan::CalpontSystemCatalog::ColDataType colDataType; + uint32_t colWidth; + int32_t size; + std::string fileName; + std::unique_ptr pFileOp; + // Note: We cannot clear this pointer directly, because + // `ChunkManager` closes this file for us, otherwise we will get double + // free error. + IDBDataFile* pFile; + WriteEngine::ChunkManager chunkManager; + uint8_t blockData[WriteEngine::BYTE_PER_BLOCK]; }; // Class to read decompressed blocks from column segment files. class ChunkManagerWrapperColumn : public ChunkManagerWrapper { - public: - ChunkManagerWrapperColumn( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t colWidth, uint32_t compressionType); + public: + ChunkManagerWrapperColumn(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, + uint32_t compressionType); - ~ChunkManagerWrapperColumn() = default; - ChunkManagerWrapperColumn(const ChunkManagerWrapperColumn& other) = delete; - ChunkManagerWrapperColumn& - operator=(const ChunkManagerWrapperColumn& other) = delete; - ChunkManagerWrapperColumn(ChunkManagerWrapperColumn&& other) = delete; - ChunkManagerWrapperColumn& - operator=(ChunkManagerWrapperColumn&& other) = delete; + ~ChunkManagerWrapperColumn() = default; + ChunkManagerWrapperColumn(const ChunkManagerWrapperColumn& other) = delete; + ChunkManagerWrapperColumn& operator=(const ChunkManagerWrapperColumn& other) = delete; + ChunkManagerWrapperColumn(ChunkManagerWrapperColumn&& other) = delete; + ChunkManagerWrapperColumn& operator=(ChunkManagerWrapperColumn&& other) = delete; - bool isEmptyBlock() override; - bool isEmptyValue(const uint8_t* value) const; + bool isEmptyBlock() override; + bool isEmptyValue(const uint8_t* value) const; - private: - const uint8_t* emptyValue; - uint32_t midOffset; - uint32_t endOffset; + private: + const uint8_t* emptyValue; + uint32_t midOffset; + uint32_t endOffset; }; // Class to read decompressed blocks from dict segment files. class ChunkManagerWrapperDict : public ChunkManagerWrapper { - public: - ChunkManagerWrapperDict( - uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t colWidth, uint32_t compressionType); + public: + ChunkManagerWrapperDict(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, + uint32_t compressionType); - ~ChunkManagerWrapperDict() = default; - ChunkManagerWrapperDict(const ChunkManagerWrapperDict& other) = delete; - ChunkManagerWrapperDict& - operator=(const ChunkManagerWrapperDict& other) = delete; - ChunkManagerWrapperDict(ChunkManagerWrapperDict&& other) = delete; - ChunkManagerWrapperDict& - operator=(ChunkManagerWrapperDict&& other) = delete; + ~ChunkManagerWrapperDict() = default; + ChunkManagerWrapperDict(const ChunkManagerWrapperDict& other) = delete; + ChunkManagerWrapperDict& operator=(const ChunkManagerWrapperDict& other) = delete; + ChunkManagerWrapperDict(ChunkManagerWrapperDict&& other) = delete; + ChunkManagerWrapperDict& operator=(ChunkManagerWrapperDict&& other) = delete; - bool isEmptyBlock() override; + bool isEmptyBlock() override; - private: - uint32_t emptyBlock; + private: + uint32_t emptyBlock; }; -} // namespace RebuildExtentMap +} // namespace RebuildExtentMap #endif diff --git a/tools/rgprint/rgprint.cpp b/tools/rgprint/rgprint.cpp index 7f2f13cf0..49b91f59a 100644 --- a/tools/rgprint/rgprint.cpp +++ b/tools/rgprint/rgprint.cpp @@ -37,14 +37,16 @@ int main(int argc, char* argv[]) unsigned pid; void* agg; auto c = sscanf(p, "Agg-p%u-t%p-", &pid, &agg); - if (c == 2) { + if (c == 2) + { char fname[1024]; snprintf(fname, sizeof(fname), "META-p%u-t%p", pid, agg); rfd = open(fname, O_RDONLY); } if (rfd < 0) rfd = open("./META", O_RDONLY); - if (rfd >= 0) { + if (rfd >= 0) + { struct stat rst; fstat(rfd, &rst); messageqcpp::ByteStream rbs; @@ -56,17 +58,16 @@ int main(int argc, char* argv[]) rbs.advanceInputPtr(r); rg.deserialize(rbs); close(rfd); - } else { - std::vector pos{2, 6, 22, 30, 46, 54}; // ? - std::vector oids{3011, 3011, 3011, 3011, 3011}; // ? - std::vector keys{1, 1, 1, 1, 1}; // ? + } + else + { + std::vector pos{2, 6, 22, 30, 46, 54}; // ? + std::vector oids{3011, 3011, 3011, 3011, 3011}; // ? + std::vector keys{1, 1, 1, 1, 1}; // ? std::vector col_t{ - execplan::CalpontSystemCatalog::INT, - execplan::CalpontSystemCatalog::LONGDOUBLE, - execplan::CalpontSystemCatalog::UBIGINT, - execplan::CalpontSystemCatalog::LONGDOUBLE, - execplan::CalpontSystemCatalog::UBIGINT - }; + execplan::CalpontSystemCatalog::INT, execplan::CalpontSystemCatalog::LONGDOUBLE, + execplan::CalpontSystemCatalog::UBIGINT, execplan::CalpontSystemCatalog::LONGDOUBLE, + execplan::CalpontSystemCatalog::UBIGINT}; std::vector csN{8, 8, 8, 8, 8}; std::vector scale{0, 0, 0, 0, 0}; std::vector prec{10, 4294967295, 9999, 4294967295, 19}; diff --git a/tools/sendPlan/sendplan.cpp b/tools/sendPlan/sendplan.cpp index 3be62e74d..5374d8676 100644 --- a/tools/sendPlan/sendplan.cpp +++ b/tools/sendPlan/sendplan.cpp @@ -32,239 +32,224 @@ typedef CalpontSystemCatalog::OID OID; namespace { - bool vflg; void usage() { - cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl; - cout << "-v verbose output" << endl; - cout << "-t lvl set trace level to lvl" << endl; - cout << "-s sid set session id to sid" << endl; - cout << "-d display the query, but don't run it" << endl; - cout << "-B Bob's personal preferred output format" << endl; - cout << "-h display this help" << endl; + cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl; + cout << "-v verbose output" << endl; + cout << "-t lvl set trace level to lvl" << endl; + cout << "-s sid set session id to sid" << endl; + cout << "-d display the query, but don't run it" << endl; + cout << "-B Bob's personal preferred output format" << endl; + cout << "-h display this help" << endl; } double tm_diff(const struct timeval* st, const struct timeval* et) { - double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0; - double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0; + double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0; + double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0; - return (ed - sd); + return (ed - sd); } //...Extract the stats string to be printed and the runtime start time from //...the full stats string we receive from ExeMgr. -void parseStatsString( - const string& fullStatsString, - string& printStatsString, - struct timeval& queryRunStartTime) +void parseStatsString(const string& fullStatsString, string& printStatsString, + struct timeval& queryRunStartTime) { - string::size_type delimPos = fullStatsString.find ('|'); - printStatsString = fullStatsString.substr(0, delimPos); + string::size_type delimPos = fullStatsString.find('|'); + printStatsString = fullStatsString.substr(0, delimPos); - istringstream startTimeString( fullStatsString.substr(delimPos + 1) ); - startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec; + istringstream startTimeString(fullStatsString.substr(delimPos + 1)); + startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec; } -} +} // namespace int main(int argc, char** argv) { - vflg = false; - uint32_t tlvl = 0; - bool dflg = false; - int c; - int32_t sid = -1; - bool Bflg = false; + vflg = false; + uint32_t tlvl = 0; + bool dflg = false; + int c; + int32_t sid = -1; + bool Bflg = false; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF) - switch (c) - { - case 't': - tlvl = static_cast(strtoul(optarg, 0, 0)); - break; + while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF) + switch (c) + { + case 't': tlvl = static_cast(strtoul(optarg, 0, 0)); break; - case 'v': - vflg = true; - break; + case 'v': vflg = true; break; - case 'd': - dflg = true; - break; + case 'd': dflg = true; break; - case 's': - sid = static_cast(strtol(optarg, 0, 0)); - break; + case 's': sid = static_cast(strtol(optarg, 0, 0)); break; - case 'B': - Bflg = true; - break; + case 'B': Bflg = true; break; - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } + case 'h': + case '?': + default: + usage(); + return (c == 'h' ? 0 : 1); + break; + } + + if (dflg) + vflg = true; + + if ((argc - optind) < 1) + { + usage(); + return 1; + } + + ifstream inputf; + ByteStream bs; + ByteStream dbs; + ByteStream eoq; + ByteStream tbs; + ByteStream statsStream; + ByteStream::quadbyte q = 0; + eoq << q; + uint32_t sessionid; + time_t t; + SJLP jl; + DeliveredTableMap tm; + DeliveredTableMap::iterator iter; + DeliveredTableMap::iterator end; + CalpontSelectExecutionPlan csep; + struct timeval start_time; + struct timeval end_time; + + MessageQueueClient* mqc = 0; + + if (!dflg) + mqc = new MessageQueueClient("ExeMgr1"); + + if (sid == -1) + { + time(&t); + sessionid = static_cast(t); + } + else + { + sessionid = static_cast(sid); + } + + sessionid &= 0x7fffffff; + logging::ErrorCodes errorCodes; + + for (; optind < argc; optind++) + { + inputf.open(argv[optind]); + + if (!inputf.good()) + { + cerr << "error opening plan stream " << argv[optind] << endl; + return 1; + } + + bs.reset(); + inputf >> bs; + + inputf.close(); + + csep.unserialize(bs); + + csep.sessionID(sessionid); + SessionManager sm; + csep.verID(sm.verID()); + + csep.traceFlags(0); + ResourceManager* rm = ResourceManager::instance(); + jl = JobListFactory::makeJobList(&csep, rm); + csep.traceFlags(tlvl); + + if (vflg) + { + if (dflg) + cout << endl << "Query:" << endl; + else + { + cout << endl << "Session: " << sessionid << ", Sending Query"; + + if (Bflg) + cout << " (" << argv[optind] << ')'; + + cout << ':' << endl; + } + + if (!Bflg) + cout << csep.data() << endl << endl; + } if (dflg) - vflg = true; + continue; - if ((argc - optind) < 1) + try { - usage(); - return 1; - } + dbs.reset(); + csep.serialize(dbs); - ifstream inputf; - ByteStream bs; - ByteStream dbs; - ByteStream eoq; - ByteStream tbs; - ByteStream statsStream; - ByteStream::quadbyte q = 0; - eoq << q; - uint32_t sessionid; - time_t t; - SJLP jl; - DeliveredTableMap tm; - DeliveredTableMap::iterator iter; - DeliveredTableMap::iterator end; - CalpontSelectExecutionPlan csep; - struct timeval start_time; - struct timeval end_time; + gettimeofday(&start_time, 0); - MessageQueueClient* mqc = 0; + // try tuples first, but expect the worst... + bool expectTuples = false; + ByteStream tbs; + ByteStream::quadbyte tqb = 4; + tbs << tqb; + mqc->write(tbs); - if (!dflg) - mqc = new MessageQueueClient("ExeMgr1"); + // send the CSEP + mqc->write(dbs); - if (sid == -1) - { - time(&t); - sessionid = static_cast(t); - } - else - { - sessionid = static_cast(sid); - } + // read the response to the tuple request + tbs = mqc->read(); + idbassert(tbs.length() == 4); + tbs >> tqb; - sessionid &= 0x7fffffff; - logging::ErrorCodes errorCodes; + if (tqb == 4) + expectTuples = true; - for ( ; optind < argc; optind++) - { + if (!expectTuples) + cout << "Using TableBand I/F" << endl; + else + cout << "Using tuple I/F" << endl; - inputf.open(argv[optind]); + tm = jl->deliveredTables(); - if (!inputf.good()) + iter = tm.begin(); + end = tm.end(); + + OID toid; + uint64_t rowTot; + bool reported = false; + bool needRGCtor = true; + + while (iter != end) + { + toid = iter->first; + q = static_cast(toid); + tbs.reset(); + tbs << q; + mqc->write(tbs); + + ByteStream tbbs; + TableBand tb; + RowGroup rg; + rowTot = 0; + uint16_t status = 0; + TableBand::VBA::size_type rc; + ofstream out; + + for (;;) { - cerr << "error opening plan stream " << argv[optind] << endl; - return 1; - } - - bs.reset(); - inputf >> bs; - - inputf.close(); - - csep.unserialize(bs); - - csep.sessionID(sessionid); - SessionManager sm; - csep.verID(sm.verID()); - - csep.traceFlags(0); - ResourceManager* rm = ResourceManager::instance(); - jl = JobListFactory::makeJobList(&csep, rm); - csep.traceFlags(tlvl); - - if (vflg) - { - if (dflg) - cout << endl << "Query:" << endl; - else - { - cout << endl << "Session: " << sessionid << - ", Sending Query"; - - if (Bflg) - cout << " (" << argv[optind] << ')'; - - cout << ':' << endl; - } - - if (!Bflg) - cout << csep.data() << endl << endl; - } - - if (dflg) - continue; - - try - { - dbs.reset(); - csep.serialize(dbs); - - gettimeofday(&start_time, 0); - - //try tuples first, but expect the worst... - bool expectTuples = false; - ByteStream tbs; - ByteStream::quadbyte tqb = 4; - tbs << tqb; - mqc->write(tbs); - - //send the CSEP - mqc->write(dbs); - - //read the response to the tuple request - tbs = mqc->read(); - idbassert(tbs.length() == 4); - tbs >> tqb; - - if (tqb == 4) - expectTuples = true; - - if (!expectTuples) - cout << "Using TableBand I/F" << endl; - else - cout << "Using tuple I/F" << endl; - - tm = jl->deliveredTables(); - - iter = tm.begin(); - end = tm.end(); - - OID toid; - uint64_t rowTot; - bool reported = false; - bool needRGCtor = true; - - while (iter != end) - { - toid = iter->first; - q = static_cast(toid); - tbs.reset(); - tbs << q; - mqc->write(tbs); - - ByteStream tbbs; - TableBand tb; - RowGroup rg; - rowTot = 0; - uint16_t status = 0; - TableBand::VBA::size_type rc; - ofstream out; - - for (;;) - { - tbbs = mqc->read(); + tbbs = mqc->read(); #if 0 cout << tbbs.length() << endl; out.open("bs1.dat"); @@ -285,124 +270,124 @@ int main(int argc, char** argv) out.close(); #endif - if (tbbs.length()) - { - if (!expectTuples) - tb.unserialize(tbbs); - else - { - if (needRGCtor) - { - rg.deserialize(tbbs); - needRGCtor = false; - tbbs = mqc->read(); - } - - rg.setData((uint8_t*)tbbs.buf()); - } - } - else - { - //@bug 1346 - if (!status) - status = logging::makeJobListErr; - - break; - } - - if (!expectTuples) - { - rc = tb.getRowCount(); - status = tb.getStatus(); - } - else - { - rc = rg.getRowCount(); - status = rg.getStatus(); - - if (rc == 0) status = 0; - } - - if (rc == 0) - break; - - rowTot += rc; - } - - BatchPrimitive* step = dynamic_cast( iter->second.get() ); - - if (vflg && step) - { - cout << "For table " << step->tableName(); - - if (!Bflg) - cout << " " << toid; - - cout << ": read " << rowTot << " rows" << endl; - } - - if (status && !reported) - { - cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n"; - reported = true; - } - - if (!step && !reported) - { - cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n"; - reported = true; - } - - ++iter; - } - - if (vflg) + if (tbbs.length()) + { + if (!expectTuples) + tb.unserialize(tbbs); + else { - gettimeofday(&end_time, 0); - cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << - " secs" << endl; + if (needRGCtor) + { + rg.deserialize(tbbs); + needRGCtor = false; + tbbs = mqc->read(); + } - //...Ask for query stats through special table id of 3 - const OID TABLE_ID_TO_GET_QUERY_STATS = 3; - - if (!Bflg) - cout << "Retrieving stats..." << endl; - - toid = TABLE_ID_TO_GET_QUERY_STATS; - q = static_cast(toid); - statsStream.reset(); - statsStream << q; - mqc->write(statsStream); - - ByteStream bs_statsString; - bs_statsString = mqc->read(); - string statsString; - bs_statsString >> statsString; - - string printStatsString; - struct timeval startRunTime; - parseStatsString (statsString, printStatsString, startRunTime); - cout << printStatsString << "; QuerySetupTime-" << - tm_diff(&start_time, &startRunTime) << "secs" << endl; + rg.setData((uint8_t*)tbbs.buf()); } + } + else + { + //@bug 1346 + if (!status) + status = logging::makeJobListErr; - //...Close this query/session - mqc->write(eoq); - jl.reset(); + break; + } + + if (!expectTuples) + { + rc = tb.getRowCount(); + status = tb.getStatus(); + } + else + { + rc = rg.getRowCount(); + status = rg.getStatus(); + + if (rc == 0) + status = 0; + } + + if (rc == 0) + break; + + rowTot += rc; } - catch (const exception& ex) + + BatchPrimitive* step = dynamic_cast(iter->second.get()); + + if (vflg && step) { - cout << "### SendPlan caught an exception: " << ex.what() << endl; + cout << "For table " << step->tableName(); + + if (!Bflg) + cout << " " << toid; + + cout << ": read " << rowTot << " rows" << endl; } + + if (status && !reported) + { + cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n"; + reported = true; + } + + if (!step && !reported) + { + cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n"; + reported = true; + } + + ++iter; + } + + if (vflg) + { + gettimeofday(&end_time, 0); + cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs" + << endl; + + //...Ask for query stats through special table id of 3 + const OID TABLE_ID_TO_GET_QUERY_STATS = 3; + + if (!Bflg) + cout << "Retrieving stats..." << endl; + + toid = TABLE_ID_TO_GET_QUERY_STATS; + q = static_cast(toid); + statsStream.reset(); + statsStream << q; + mqc->write(statsStream); + + ByteStream bs_statsString; + bs_statsString = mqc->read(); + string statsString; + bs_statsString >> statsString; + + string printStatsString; + struct timeval startRunTime; + parseStatsString(statsString, printStatsString, startRunTime); + cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs" + << endl; + } + + //...Close this query/session + mqc->write(eoq); + jl.reset(); } + catch (const exception& ex) + { + cout << "### SendPlan caught an exception: " << ex.what() << endl; + } + } -// jl.reset(); - CalpontSystemCatalog::removeCalpontSystemCatalog( sessionid ); - config::Config::deleteInstanceMap(); + // jl.reset(); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionid); + config::Config::deleteInstanceMap(); - delete mqc; + delete mqc; - return 0; + return 0; } // vim:ts=4 sw=4: - diff --git a/tools/setConfig/main.cpp b/tools/setConfig/main.cpp index abaad2602..1c536167f 100644 --- a/tools/setConfig/main.cpp +++ b/tools/setConfig/main.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /***************************************************************************** -* $Id: main.cpp 210 2007-06-08 17:08:26Z rdempsey $ -* -*****************************************************************************/ + * $Id: main.cpp 210 2007-06-08 17:08:26Z rdempsey $ + * + *****************************************************************************/ #include #include @@ -34,127 +34,118 @@ using namespace oam; namespace { - void usage(const string& pname) { - cout << "usage: " << pname << " [-vdhx] [-c config_file] section param value" << endl << - " Updates configuration variable param in section section with value." << endl << - " -c config_file use config file config_file" << endl << - " -v display verbose information" << endl << - " -d don't perform misc checks and don't try to distribute the config file" << endl << - " after changes are made" << endl << - " -x delete the param from section (value is still required but ignored)" << endl << - " -h display this help text" << endl; + cout << "usage: " << pname << " [-vdhx] [-c config_file] section param value" << endl + << " Updates configuration variable param in section section with value." << endl + << " -c config_file use config file config_file" << endl + << " -v display verbose information" << endl + << " -d don't perform misc checks and don't try to distribute the config file" << endl + << " after changes are made" << endl + << " -x delete the param from section (value is still required but ignored)" << endl + << " -h display this help text" << endl; } -} +} // namespace int main(int argc, char** argv) { - int c; - string pname(argv[0]); - bool vflg = false; - bool dflg = false; - bool xflg = false; - string configFile; + int c; + string pname(argv[0]); + bool vflg = false; + bool dflg = false; + bool xflg = false; + string configFile; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "c:vdxh")) != EOF) - switch (c) - { - case 'v': - vflg = true; - break; - - case 'd': - dflg = true; - break; - - case 'c': - configFile = optarg; - break; - - case 'x': - xflg = true; - break; - - case 'h': - case '?': - default: - usage(pname); - return (c == 'h' ? 0 : 1); - break; - } - - if ((argc - optind) < 3) + while ((c = getopt(argc, argv, "c:vdxh")) != EOF) + switch (c) { + case 'v': vflg = true; break; + + case 'd': dflg = true; break; + + case 'c': configFile = optarg; break; + + case 'x': xflg = true; break; + + case 'h': + case '?': + default: usage(pname); - return 1; + return (c == 'h' ? 0 : 1); + break; } + if ((argc - optind) < 3) + { + usage(pname); + return 1; + } + #ifdef COMMUNITY_KEYRANGE - //No OAM in CE... - dflg = true; + // No OAM in CE... + dflg = true; #endif - Oam oam; - oamModuleInfo_t t; - bool parentOAMModuleFlag = true; - string parentOAMModule = " "; - int serverInstallType = oam::INSTALL_COMBINE_DM_UM_PM; + Oam oam; + oamModuleInfo_t t; + bool parentOAMModuleFlag = true; + string parentOAMModule = " "; + int serverInstallType = oam::INSTALL_COMBINE_DM_UM_PM; - //get local module info; validate running on Active Parent OAM Module - try + // get local module info; validate running on Active Parent OAM Module + try + { + t = oam.getModuleInfo(); + parentOAMModuleFlag = boost::get<4>(t); + parentOAMModule = boost::get<3>(t); + serverInstallType = boost::get<5>(t); + } + catch (exception&) + { + parentOAMModuleFlag = true; + } + + if (!dflg && !parentOAMModuleFlag) + { + cerr << "Exiting, mcsSetConfig can only be run on the Active " + "OAM Parent Module '" + << parentOAMModule << "'" << endl; + return 2; + } + + Config* cf; + + if (configFile.length() > 0) + cf = Config::makeConfig(configFile); + else + cf = Config::makeConfig(); + + if (vflg) + cout << "Using config file: " << cf->configFile() << endl; + + if (xflg) + cf->delConfig(argv[optind + 0], argv[optind + 1]); + else + { + cf->setConfig(argv[optind + 0], argv[optind + 1], argv[optind + 2]); + + if (strcmp(argv[optind + 1], "SystemName") == 0) { - t = oam.getModuleInfo(); - parentOAMModuleFlag = boost::get<4>(t); - parentOAMModule = boost::get<3>(t); - serverInstallType = boost::get<5>(t); - } - catch (exception&) - { - parentOAMModuleFlag = true; + oam.changeMyCnf("server_audit_syslog_info", argv[optind + 2]); } + } - if (!dflg && !parentOAMModuleFlag) - { - cerr << "Exiting, mcsSetConfig can only be run on the Active " - "OAM Parent Module '" << parentOAMModule << "'" << endl; - return 2; - } - - Config* cf; - - if (configFile.length() > 0) - cf = Config::makeConfig(configFile); - else - cf = Config::makeConfig(); - - if (vflg) - cout << "Using config file: " << cf->configFile() << endl; - - if (xflg) - cf->delConfig(argv[optind + 0], argv[optind + 1]); - else - { - cf->setConfig(argv[optind + 0], argv[optind + 1], argv[optind + 2]); - - if (strcmp(argv[optind + 1], "SystemName") == 0) - { - oam.changeMyCnf( "server_audit_syslog_info", argv[optind + 2] ); - } - } - - cf->write(); - - if (dflg || serverInstallType == oam::INSTALL_COMBINE_DM_UM_PM) - return 0; - - //get number of pms - string count = cf->getConfig("PrimitiveServers", "Count"); + cf->write(); + if (dflg || serverInstallType == oam::INSTALL_COMBINE_DM_UM_PM) return 0; + + // get number of pms + string count = cf->getConfig("PrimitiveServers", "Count"); + + return 0; } // vim:ts=4 sw=4: - diff --git a/tools/valprint/valprint.cpp b/tools/valprint/valprint.cpp index d1b91c613..2fcfd9079 100644 --- a/tools/valprint/valprint.cpp +++ b/tools/valprint/valprint.cpp @@ -26,25 +26,23 @@ using namespace config; int main(int argc, char** argv) { - int c; - opterr = 0; + int c; + opterr = 0; - while ((c = getopt(argc, argv, "")) != EOF) - switch (c) - { - case '?': - default: - break; - } - - string s; - - for (int i = optind; i < argc; i++) + while ((c = getopt(argc, argv, "")) != EOF) + switch (c) { - s = argv[i]; - cout << s << " = " << Config::fromText(s) << endl; + case '?': + default: break; } - return 0; -} + string s; + for (int i = optind; i < argc; i++) + { + s = argv[i]; + cout << s << " = " << Config::fromText(s) << endl; + } + + return 0; +} diff --git a/tools/viewtablelock/viewtablelock.cpp b/tools/viewtablelock/viewtablelock.cpp index 142c1ba8a..e83b3d8e7 100644 --- a/tools/viewtablelock/viewtablelock.cpp +++ b/tools/viewtablelock/viewtablelock.cpp @@ -16,9 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: viewtablelock.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -*/ - + * $Id: viewtablelock.cpp 2101 2013-01-21 14:12:52Z rdempsey $ + */ #include #include @@ -35,354 +34,320 @@ using namespace execplan; namespace { - //------------------------------------------------------------------------------ // Print command line usage //------------------------------------------------------------------------------ void usage() { - std::cout << "Usage: viewtablelock [schemaname tablename]" << std::endl - << " If schema/table are specified, then that table's lock " << - "information is displayed." << std::endl - << " If no schema/table are specified, then all table locks " - "are displayed." << std::endl; + std::cout << "Usage: viewtablelock [schemaname tablename]" << std::endl + << " If schema/table are specified, then that table's lock " + << "information is displayed." << std::endl + << " If no schema/table are specified, then all table locks " + "are displayed." + << std::endl; } //------------------------------------------------------------------------------ // Print table locks. // This function closely resembles CommandPackageProcessor::viewTableLock(). //------------------------------------------------------------------------------ -int printTableLocks( boost::shared_ptr systemCatalogPtr, - const std::string& tableNameIn, - const std::vector tableLocks ) +int printTableLocks(boost::shared_ptr systemCatalogPtr, const std::string& tableNameIn, + const std::vector tableLocks) { - if (tableLocks.size() == 1) - std::cout << " There is " << tableLocks.size() << - " table lock" << std::endl << std::endl; + if (tableLocks.size() == 1) + std::cout << " There is " << tableLocks.size() << " table lock" << std::endl << std::endl; + else + std::cout << " There are " << tableLocks.size() << " table locks" << std::endl << std::endl; + + std::string tableName(tableNameIn); + + // Make preliminary pass through the table locks in order to determine our + // output column widths based on the data. Min column widths are based on + // the width of the column heading (except for the 'state' column). + uint64_t maxLockID = 0; + uint32_t maxPID = 0; + int32_t maxSessionID = 0; + int32_t minSessionID = 0; + int32_t maxTxnID = 0; + + unsigned int tableNameColumnWidth = 5; // "Table" + unsigned int lockIDColumnWidth = 6; // "LockID" + unsigned int ownerColumnWidth = 7; // "Process" + unsigned int pidColumnWidth = 3; // "PID" + unsigned int sessionIDColumnWidth = 7; // "Session" + unsigned int txnIDColumnWidth = 3; // "Txn" + unsigned int createTimeColumnWidth = 12; // "CreationTime" + unsigned int pmColumnWidth = 7; // "DBRoots" + std::vector createTimes; + char cTimeBuffer[1024]; + std::vector tableNames; + + std::ostringstream errMsgs; + + for (unsigned int i = 0; i < tableLocks.size(); i++) + { + if (tableNameIn.empty()) + { + try + { + CalpontSystemCatalog::TableName tableNameStruct; + tableNameStruct = systemCatalogPtr->tableName(tableLocks[i].tableOID); + tableName = tableNameStruct.toString(); + } + catch (logging::IDBExcept&) + { + tableName.clear(); + + errMsgs << "Table with oid " << tableLocks[i].tableOID << " (Lock " << tableLocks[i].id << ")" + << " is not in systable." << std::endl; + } + catch (std::runtime_error& e) + { + tableName.clear(); + + errMsgs << "Error searching for table " << tableLocks[i].tableOID << " (Lock " << tableLocks[i].id + << ")" + << " in system catalog. " << e.what() << std::endl; + } + catch (...) + { + tableName.clear(); + + errMsgs << "Unknown error searching for table " << tableLocks[i].tableOID << " (Lock " + << tableLocks[i].id << ")" + << " in system catalog. " << std::endl; + } + } + + tableNames.push_back(tableName); + + tableNameColumnWidth = std::max(tableNameColumnWidth, static_cast(tableName.length())); + maxLockID = std::max(maxLockID, tableLocks[i].id); + ownerColumnWidth = + std::max(ownerColumnWidth, static_cast(tableLocks[i].ownerName.length())); + maxPID = std::max(maxPID, tableLocks[i].ownerPID); + maxSessionID = std::max(maxSessionID, tableLocks[i].ownerSessionID); + minSessionID = std::min(minSessionID, tableLocks[i].ownerSessionID); + maxTxnID = std::max(maxTxnID, tableLocks[i].ownerTxnID); + + ctime_r(&tableLocks[i].creationTime, cTimeBuffer); + cTimeBuffer[strlen(cTimeBuffer) - 1] = '\0'; // strip trailing '\n' + std::string cTimeStr(cTimeBuffer); + createTimeColumnWidth = std::max(createTimeColumnWidth, static_cast(cTimeStr.length())); + createTimes.push_back(cTimeStr); + + std::ostringstream pms; + + for (unsigned k = 0; k < tableLocks[i].dbrootList.size(); k++) + { + if (k > 0) + pms << ','; + + pms << tableLocks[i].dbrootList[k]; + } + + pmColumnWidth = std::max(pmColumnWidth, static_cast(pms.str().length())); + } + + tableNameColumnWidth += 2; + ownerColumnWidth += 2; + pmColumnWidth += 2; + createTimeColumnWidth += 2; + + std::ostringstream idString; + idString << maxLockID; + lockIDColumnWidth = std::max(lockIDColumnWidth, static_cast(idString.str().length())); + lockIDColumnWidth += 2; + + std::ostringstream pidString; + pidString << maxPID; + pidColumnWidth = std::max(pidColumnWidth, static_cast(pidString.str().length())); + pidColumnWidth += 2; + + const std::string sessionNoneStr("BulkLoad"); + std::ostringstream sessionString; + sessionString << maxSessionID; + sessionIDColumnWidth = + std::max(sessionIDColumnWidth, static_cast(sessionString.str().length())); + + if (minSessionID < 0) + sessionIDColumnWidth = std::max(sessionIDColumnWidth, static_cast(sessionNoneStr.length())); + + sessionIDColumnWidth += 2; + + const std::string txnNoneStr("n/a"); + std::ostringstream txnString; + txnString << maxTxnID; + txnIDColumnWidth = std::max(txnIDColumnWidth, static_cast(txnString.str().length())); + txnIDColumnWidth += 2; + + std::cout.setf(std::ios::left, std::ios::adjustfield); + std::cout << " " << std::setw(tableNameColumnWidth) << "Table" << std::setw(lockIDColumnWidth) << "LockID" + << std::setw(ownerColumnWidth) << "Process" << std::setw(pidColumnWidth) << "PID" + << std::setw(sessionIDColumnWidth) << "Session" << std::setw(txnIDColumnWidth) << "Txn" + << std::setw(createTimeColumnWidth) << "CreationTime" << std::setw(9) << "State" + << std::setw(pmColumnWidth) << "DBRoots" << std::endl; + + // Make second pass through the table locks to display our result. + for (unsigned idx = 0; idx < tableLocks.size(); idx++) + { + std::ostringstream pms; // dbroots now + + for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) + { + if (k > 0) + pms << ','; + + pms << tableLocks[idx].dbrootList[k]; + } + + std::cout << " " << std::setw(tableNameColumnWidth) << tableNames[idx] << std::setw(lockIDColumnWidth) + << tableLocks[idx].id << std::setw(ownerColumnWidth) << tableLocks[idx].ownerName + << std::setw(pidColumnWidth) << tableLocks[idx].ownerPID; + + // Log session ID, or "BulkLoad" if session is -1 + if (tableLocks[idx].ownerSessionID < 0) + std::cout << std::setw(sessionIDColumnWidth) << sessionNoneStr; else - std::cout << " There are " << tableLocks.size() << - " table locks" << std::endl << std::endl; + std::cout << std::setw(sessionIDColumnWidth) << tableLocks[idx].ownerSessionID; - std::string tableName(tableNameIn); + // Log txn ID, or "n/a" if txn is -1 + if (tableLocks[idx].ownerTxnID < 0) + std::cout << std::setw(txnIDColumnWidth) << txnNoneStr; + else + std::cout << std::setw(txnIDColumnWidth) << tableLocks[idx].ownerTxnID; - // Make preliminary pass through the table locks in order to determine our - // output column widths based on the data. Min column widths are based on - // the width of the column heading (except for the 'state' column). - uint64_t maxLockID = 0; - uint32_t maxPID = 0; - int32_t maxSessionID = 0; - int32_t minSessionID = 0; - int32_t maxTxnID = 0; + std::cout << std::setw(createTimeColumnWidth) << createTimes[idx] << std::setw(9) + << ((tableLocks[idx].state == BRM::LOADING) ? "LOADING" : "CLEANUP") << std::setw(pmColumnWidth) + << pms.str() << std::endl; + } - unsigned int tableNameColumnWidth = 5; // "Table" - unsigned int lockIDColumnWidth = 6; // "LockID" - unsigned int ownerColumnWidth = 7; // "Process" - unsigned int pidColumnWidth = 3; // "PID" - unsigned int sessionIDColumnWidth = 7; // "Session" - unsigned int txnIDColumnWidth = 3; // "Txn" - unsigned int createTimeColumnWidth = 12; // "CreationTime" - unsigned int pmColumnWidth = 7; // "DBRoots" - std::vector createTimes; - char cTimeBuffer[1024]; - std::vector tableNames; + if (!errMsgs.str().empty()) + std::cerr << std::endl << errMsgs.str() << std::endl; - std::ostringstream errMsgs; - - for (unsigned int i = 0; i < tableLocks.size(); i++) - { - if (tableNameIn.empty()) - { - try - { - CalpontSystemCatalog::TableName tableNameStruct; - tableNameStruct = systemCatalogPtr->tableName( - tableLocks[i].tableOID); - tableName = tableNameStruct.toString(); - } - catch ( logging::IDBExcept&) - { - tableName.clear(); - - errMsgs << "Table with oid " << tableLocks[i].tableOID << - " (Lock " << tableLocks[i].id << ")" << - " is not in systable." << std::endl; - } - catch (std::runtime_error& e) - { - tableName.clear(); - - errMsgs << - "Error searching for table " << tableLocks[i].tableOID << - " (Lock " << tableLocks[i].id << ")" << - " in system catalog. " << e.what() << std::endl; - } - catch (...) - { - tableName.clear(); - - errMsgs << - "Unknown error searching for table " << - tableLocks[i].tableOID << - " (Lock " << tableLocks[i].id << ")" << - " in system catalog. " << std::endl; - } - } - - tableNames.push_back( tableName ); - - tableNameColumnWidth = std::max(tableNameColumnWidth, - static_cast(tableName.length())); - maxLockID = std::max(maxLockID, tableLocks[i].id); - ownerColumnWidth = std::max(ownerColumnWidth, - static_cast(tableLocks[i].ownerName.length())); - maxPID = std::max(maxPID, tableLocks[i].ownerPID); - maxSessionID = std::max(maxSessionID, tableLocks[i].ownerSessionID); - minSessionID = std::min(minSessionID, tableLocks[i].ownerSessionID); - maxTxnID = std::max(maxTxnID, tableLocks[i].ownerTxnID); - - ctime_r( &tableLocks[i].creationTime, cTimeBuffer ); - cTimeBuffer[ strlen(cTimeBuffer) - 1 ] = '\0'; // strip trailing '\n' - std::string cTimeStr( cTimeBuffer ); - createTimeColumnWidth = std::max(createTimeColumnWidth, - static_cast(cTimeStr.length())); - createTimes.push_back( cTimeStr ); - - std::ostringstream pms; - - for (unsigned k = 0; k < tableLocks[i].dbrootList.size(); k++) - { - if (k > 0) - pms << ','; - - pms << tableLocks[i].dbrootList[k]; - } - - pmColumnWidth = std::max(pmColumnWidth, - static_cast(pms.str().length())); - } - - tableNameColumnWidth += 2; - ownerColumnWidth += 2; - pmColumnWidth += 2; - createTimeColumnWidth += 2; - - std::ostringstream idString; - idString << maxLockID; - lockIDColumnWidth = std::max(lockIDColumnWidth, - static_cast(idString.str().length())); - lockIDColumnWidth += 2; - - std::ostringstream pidString; - pidString << maxPID; - pidColumnWidth = std::max(pidColumnWidth, - static_cast(pidString.str().length())); - pidColumnWidth += 2; - - const std::string sessionNoneStr("BulkLoad"); - std::ostringstream sessionString; - sessionString << maxSessionID; - sessionIDColumnWidth = std::max(sessionIDColumnWidth, - static_cast(sessionString.str().length())); - - if (minSessionID < 0) - sessionIDColumnWidth = std::max(sessionIDColumnWidth, - static_cast(sessionNoneStr.length())); - - sessionIDColumnWidth += 2; - - const std::string txnNoneStr("n/a"); - std::ostringstream txnString; - txnString << maxTxnID; - txnIDColumnWidth = std::max(txnIDColumnWidth, - static_cast(txnString.str().length())); - txnIDColumnWidth += 2; - - std::cout.setf(std::ios::left, std::ios::adjustfield); - std::cout << " " << - std::setw(tableNameColumnWidth) << "Table" << - std::setw(lockIDColumnWidth) << "LockID" << - std::setw(ownerColumnWidth) << "Process" << - std::setw(pidColumnWidth) << "PID" << - std::setw(sessionIDColumnWidth) << "Session" << - std::setw(txnIDColumnWidth) << "Txn" << - std::setw(createTimeColumnWidth) << "CreationTime" << - std::setw(9) << "State" << - std::setw(pmColumnWidth) << "DBRoots" << std::endl; - - // Make second pass through the table locks to display our result. - for (unsigned idx = 0; idx < tableLocks.size(); idx++) - { - std::ostringstream pms; //dbroots now - - for (unsigned k = 0; k < tableLocks[idx].dbrootList.size(); k++) - { - if (k > 0) - pms << ','; - - pms << tableLocks[idx].dbrootList[k]; - } - - std::cout << " " << - std::setw(tableNameColumnWidth) << tableNames[idx] << - std::setw(lockIDColumnWidth) << tableLocks[idx].id << - std::setw(ownerColumnWidth) << tableLocks[idx].ownerName << - std::setw(pidColumnWidth) << tableLocks[idx].ownerPID; - - // Log session ID, or "BulkLoad" if session is -1 - if (tableLocks[idx].ownerSessionID < 0) - std::cout << std::setw(sessionIDColumnWidth) << sessionNoneStr; - else - std::cout << std::setw(sessionIDColumnWidth) << - tableLocks[idx].ownerSessionID; - - // Log txn ID, or "n/a" if txn is -1 - if (tableLocks[idx].ownerTxnID < 0) - std::cout << std::setw(txnIDColumnWidth) << txnNoneStr; - else - std::cout << std::setw(txnIDColumnWidth) << - tableLocks[idx].ownerTxnID; - - std::cout << - std::setw(createTimeColumnWidth) << - createTimes[idx] << - std::setw(9) << ((tableLocks[idx].state == BRM::LOADING) ? - "LOADING" : "CLEANUP") << - std::setw(pmColumnWidth) << pms.str() << std::endl; - } - - if (!errMsgs.str().empty()) - std::cerr << std::endl << errMsgs.str() << std::endl; - - return 0; + return 0; } -} +} // namespace //------------------------------------------------------------------------------ // Main entry point to this program //------------------------------------------------------------------------------ int main(int argc, char** argv) { - int c; + int c; - while ((c = getopt(argc, argv, "h")) != EOF) - { - switch (c) - { - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - } - - int nargs = argc - optind; - - if ((nargs > 2) || (nargs == 1)) + while ((c = getopt(argc, argv, "h")) != EOF) + { + switch (c) { + case 'h': + case '?': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } + } - BRM::DBRM dbrm; - std::vector tableLocks; + int nargs = argc - optind; + + if ((nargs > 2) || (nargs == 1)) + { + usage(); + return 1; + } + + BRM::DBRM dbrm; + std::vector tableLocks; + + try + { + tableLocks = dbrm.getAllTableLocks(); + } + catch (std::exception& ex) + { + std::cerr << "Error getting list of table locks: " << ex.what() << std::endl; + return 2; + } + + int rc = 0; + + if (nargs == 2) // List table lock information for a given table + { + std::string schema(argv[optind++]); + std::string table(argv[optind++]); + + // Get table oid + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(1); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = table; + CalpontSystemCatalog::ROPair roPair; try { - tableLocks = dbrm.getAllTableLocks(); + roPair = systemCatalogPtr->tableRID(tableName); } - catch (std::exception& ex) + catch (logging::IDBExcept& e) { - std::cerr << "Error getting list of table locks: " << ex.what() << - std::endl; - return 2; + std::cerr << e.what() << std::endl; + return 3; } - - int rc = 0; - - if (nargs == 2) // List table lock information for a given table + catch (std::runtime_error& e) { - std::string schema(argv[optind++]); - std::string table( argv[optind++]); - - // Get table oid - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(1); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = table; - CalpontSystemCatalog::ROPair roPair; - - try - { - roPair = systemCatalogPtr->tableRID( tableName ); - } - catch (logging::IDBExcept& e) - { - std::cerr << e.what() << std::endl; - return 3; - } - catch (std::runtime_error& e) - { - std::cerr << "Error searching for table in system catalog. " << - e.what() << std::endl; - return 4; - } - catch (...) - { - std::cerr << "Unknown error searching for table in system catalog." - << std::endl; - return 5; - } - - // Keep in mind the same table could have more than 1 lock - // (on different PMs), so we don't exit loop after "first" match. - std::vector matchingTableLocks; - - for (unsigned int i = 0; i < tableLocks.size(); i++) - { - if (roPair.objnum == (CalpontSystemCatalog::OID) - tableLocks[i].tableOID) - { - matchingTableLocks.push_back( tableLocks[i] ); - } - } - - if (matchingTableLocks.size() > 0) - { - std::string tableName(schema); - tableName += '.'; - tableName += table; - rc = printTableLocks( systemCatalogPtr, - tableName, matchingTableLocks ); - } - else - { - std::cout << " Table " << schema << "." << table << - " is not locked by any process. " << std::endl; - } + std::cerr << "Error searching for table in system catalog. " << e.what() << std::endl; + return 4; } - else // List table lock information for all table locks + catch (...) { - //All table lock info required - if (tableLocks.size() == 0) - { - std::cout << " No tables are locked in the database." << std::endl; - } - else - { - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - - std::string tableName; - rc = printTableLocks( systemCatalogPtr, - tableName, tableLocks ); - } + std::cerr << "Unknown error searching for table in system catalog." << std::endl; + return 5; } - return rc; + // Keep in mind the same table could have more than 1 lock + // (on different PMs), so we don't exit loop after "first" match. + std::vector matchingTableLocks; + + for (unsigned int i = 0; i < tableLocks.size(); i++) + { + if (roPair.objnum == (CalpontSystemCatalog::OID)tableLocks[i].tableOID) + { + matchingTableLocks.push_back(tableLocks[i]); + } + } + + if (matchingTableLocks.size() > 0) + { + std::string tableName(schema); + tableName += '.'; + tableName += table; + rc = printTableLocks(systemCatalogPtr, tableName, matchingTableLocks); + } + else + { + std::cout << " Table " << schema << "." << table << " is not locked by any process. " << std::endl; + } + } + else // List table lock information for all table locks + { + // All table lock info required + if (tableLocks.size() == 0) + { + std::cout << " No tables are locked in the database." << std::endl; + } + else + { + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + + std::string tableName; + rc = printTableLocks(systemCatalogPtr, tableName, tableLocks); + } + } + + return rc; } diff --git a/utils/batchloader/batchloader.cpp b/utils/batchloader/batchloader.cpp index c5fb6ab33..0e9bfe836 100644 --- a/utils/batchloader/batchloader.cpp +++ b/utils/batchloader/batchloader.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_dbrootextenttracker.cpp 3672 2012-03-26 12:31:27Z rdempsey $ -*/ + * $Id: we_dbrootextenttracker.cpp 3672 2012-03-26 12:31:27Z rdempsey $ + */ #include #include @@ -35,366 +35,368 @@ using namespace execplan; namespace batchloader { - //------------------------------------------------------------------------------ // ChooseStartLoadPM constructor // // Mutex lock not needed in this function as it is only called from main thread //------------------------------------------------------------------------------ -BatchLoader::BatchLoader ( uint32_t tableOid, - execplan::CalpontSystemCatalog::SCN sessionId, - std::vector& PMs ) +BatchLoader::BatchLoader(uint32_t tableOid, execplan::CalpontSystemCatalog::SCN sessionId, + std::vector& PMs) { - fFirstPm = 0; - fNextIdx = 0; - fPMs = PMs; - fSessionId = sessionId; - fTableOid = tableOid; - OamCache* oamcache = OamCache::makeOamCache(); - oam::OamCache::PMDbrootsMap_t systemPmDbrootMap = oamcache->getPMToDbrootsMap(); - std::map::iterator iter = systemPmDbrootMap->begin(); - //cout << "fPMs size is " << fPMs.size() << endl; - fPmDbrootMap.reset(new OamCache::PMDbrootsMap_t::element_type()); - fDbrootPMmap.reset(new map()); + fFirstPm = 0; + fNextIdx = 0; + fPMs = PMs; + fSessionId = sessionId; + fTableOid = tableOid; + OamCache* oamcache = OamCache::makeOamCache(); + oam::OamCache::PMDbrootsMap_t systemPmDbrootMap = oamcache->getPMToDbrootsMap(); + std::map::iterator iter = systemPmDbrootMap->begin(); + // cout << "fPMs size is " << fPMs.size() << endl; + fPmDbrootMap.reset(new OamCache::PMDbrootsMap_t::element_type()); + fDbrootPMmap.reset(new map()); - for (uint32_t i = 0; i < fPMs.size(); i++) + for (uint32_t i = 0; i < fPMs.size(); i++) + { + iter = systemPmDbrootMap->find(fPMs[i]); + + if (iter != systemPmDbrootMap->end()) { - iter = systemPmDbrootMap->find(fPMs[i]); - - if (iter != systemPmDbrootMap->end()) - { - fDbRoots.insert(fDbRoots.end(), (iter->second).begin(), (iter->second).end()); - (*fPmDbrootMap)[fPMs[i]] = iter->second; - } + fDbRoots.insert(fDbRoots.end(), (iter->second).begin(), (iter->second).end()); + (*fPmDbrootMap)[fPMs[i]] = iter->second; } + } - //Build dbroot to PM map - for (iter = fPmDbrootMap->begin(); iter != fPmDbrootMap->end(); iter++) + // Build dbroot to PM map + for (iter = fPmDbrootMap->begin(); iter != fPmDbrootMap->end(); iter++) + { + for (uint32_t i = 0; i < iter->second.size(); i++) { - for ( uint32_t i = 0; i < iter->second.size(); i++) - { - (*fDbrootPMmap)[iter->second[i]] = iter->first; - } + (*fDbrootPMmap)[iter->second[i]] = iter->first; } + } } //------------------------------------------------------------------------------ // Select the first PM to send the first batch of rows. /*Look up extent map to decide which dbroot to start - 1. If newly created table, starts from the pm where the abbreviated extent created. - 2. If the abbreviated extent has hwm > 0, and other PMs don't have any extent, start from next PM on the list. - 2.5If some DBRoots have extents and some don't the DBRoot with the fewest extents or blocks is chosen, - "unless" the partition 0, segment 0 extent is one of the HWM extents. In that case the partition 0, - segment 0 extent still takes precedence. - 3. If all PMs have extents, count the number of extents under each dbroot, find the dbroot which has the least extents to start - 4. If all dbroots have same number of extents, starts from the dbroot which has least number of blocks + 1. If newly created table, starts from the pm where the abbreviated extent created. + 2. If the abbreviated extent has hwm > 0, and other PMs don't have any extent, start from next PM on + the list. 2.5If some DBRoots have extents and some don't the DBRoot with the fewest extents or blocks is + chosen, "unless" the partition 0, segment 0 extent is one of the HWM extents. In that case the partition + 0, segment 0 extent still takes precedence. + 3. If all PMs have extents, count the number of extents under each dbroot, find the dbroot which has + the least extents to start + 4. If all dbroots have same number of extents, starts from the dbroot which has least number of blocks */ //------------------------------------------------------------------------------ -void BatchLoader::selectFirstPM ( uint32_t& PMId) +void BatchLoader::selectFirstPM(uint32_t& PMId) { - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionId); - //cout << "calling tableName for oid " << fTableOid << endl; - CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(fTableOid); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(fSessionId); + // cout << "calling tableName for oid " << fTableOid << endl; + CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(fTableOid); - CalpontSystemCatalog::RIDList ridList; - ridList = systemCatalogPtr->columnRIDs(aTableName, true); //exception will be handled in caller program. + CalpontSystemCatalog::RIDList ridList; + ridList = systemCatalogPtr->columnRIDs(aTableName, true); // exception will be handled in caller program. - scoped_ptr dbrmp(new DBRM()); + scoped_ptr dbrmp(new DBRM()); - //Build distVec, start from the PM where the table is created. If not in the PM list, 0 will be used. - uint16_t createdDbroot = 0; - int rc = 0; - std::vector allInfo (fPMs.size()); + // Build distVec, start from the PM where the table is created. If not in the PM list, 0 will be used. + uint16_t createdDbroot = 0; + int rc = 0; + std::vector allInfo(fPMs.size()); - for (unsigned i = 0; i < fPMs.size(); i++) + for (unsigned i = 0; i < fPMs.size(); i++) + { + rc = dbrmp->getDbRootHWMInfo((ridList[0].objnum), fPMs[i], allInfo[i]); + + if (rc != 0) //@Bug 4760 + break; + } + + if (rc != 0) + { + ostringstream oss; + oss << "There is no extent information for table " << aTableName.table; + throw std::runtime_error(oss.str()); + } + + uint32_t numDbroot = fDbRoots.size(); + + if (numDbroot == 0) + { + throw std::runtime_error("There are no dbroots found during selectFirstPM"); + } + + uint64_t* rootExtents = + (uint64_t*)alloca((numDbroot + 1) * sizeof(uint64_t)); // array of number extents for each dbroot + + for (unsigned i = 0; i < fDbRoots.size(); i++) + { + uint64_t numExtents; + dbrmp->getExtentCount_dbroot((ridList[0].objnum), fDbRoots[i], false, numExtents); + rootExtents[fDbRoots[i]] = numExtents; + } + + bool startDBRootSet = false; + uint64_t* rootBlocks = (uint64_t*)alloca( + (numDbroot + 1) * + sizeof(uint64_t)); // array of number of blocks for the last partition for each dbroot + // cout << "allInfo size is " << allInfo.size() << endl; + + //-------------------------------------------------------------------------- + // Load rootBlocks to carry total blocks for each DBRoot + // Set startDBRootSet if the partition 0 segment 0 extent is empty + // Set createdDbroot for the DBRoot that carries the partition 0 + // segment 0 extent (used to set the default PMId) + //-------------------------------------------------------------------------- + for (unsigned i = 0; i < allInfo.size(); i++) // All PMs + { + BRM::EmDbRootHWMInfo_v emDbRootHWMInfos = allInfo[i]; // one pm + + for (unsigned j = 0; j < emDbRootHWMInfos.size(); j++) { - rc = dbrmp->getDbRootHWMInfo((ridList[0].objnum), fPMs[i], allInfo[i]); + if (emDbRootHWMInfos[j].totalBlocks == 0) + { + // cout << "totalBlocks is 0" << endl; + continue; + } - if ( rc != 0 ) //@Bug 4760 - break; - } - - if ( rc != 0 ) - { - ostringstream oss; - oss << "There is no extent information for table " << aTableName.table; - throw std::runtime_error(oss.str()); - } - - uint32_t numDbroot = fDbRoots.size(); - - if (numDbroot == 0) - { - throw std::runtime_error("There are no dbroots found during selectFirstPM"); - } - - uint64_t* rootExtents = (uint64_t*)alloca((numDbroot + 1) * sizeof(uint64_t)); //array of number extents for each dbroot - - for (unsigned i = 0; i < fDbRoots.size(); i++) - { - uint64_t numExtents; - dbrmp->getExtentCount_dbroot((ridList[0].objnum), fDbRoots[i], - false, numExtents); - rootExtents[fDbRoots[i]] = numExtents; - } - - bool startDBRootSet = false; - uint64_t* rootBlocks = (uint64_t*)alloca((numDbroot + 1) * sizeof(uint64_t)); //array of number of blocks for the last partition for each dbroot - //cout << "allInfo size is " << allInfo.size() << endl; - - //-------------------------------------------------------------------------- - // Load rootBlocks to carry total blocks for each DBRoot - // Set startDBRootSet if the partition 0 segment 0 extent is empty - // Set createdDbroot for the DBRoot that carries the partition 0 - // segment 0 extent (used to set the default PMId) - //-------------------------------------------------------------------------- - for (unsigned i = 0; i < allInfo.size(); i++) //All PMs - { - BRM::EmDbRootHWMInfo_v emDbRootHWMInfos = allInfo[i]; //one pm - - for (unsigned j = 0; j < emDbRootHWMInfos.size(); j++) + //------------------------------------------------------------------ + // Ignore partition 0, segment 0 HWM extent if it is disabled + //------------------------------------------------------------------ + if ((emDbRootHWMInfos[j].partitionNum == 0) && (emDbRootHWMInfos[j].segmentNum == 0) && + (emDbRootHWMInfos[j].status != BRM::EXTENTOUTOFSERVICE)) + { + if (emDbRootHWMInfos[j].localHWM == 0) { - if (emDbRootHWMInfos[j].totalBlocks == 0) - { - //cout << "totalBlocks is 0" << endl; - continue; - } + // newly created table + // cout << " This is newly created table. PM id is " << PMId; + startDBRootSet = true; + createdDbroot = emDbRootHWMInfos[j].dbRoot; + break; + } + else + { + createdDbroot = emDbRootHWMInfos[j].dbRoot; + // cout << " and createdDbroot is " << createdDbroot << endl; + rootBlocks[emDbRootHWMInfos[j].dbRoot] = emDbRootHWMInfos[j].totalBlocks; + } + } + else + { + rootBlocks[emDbRootHWMInfos[j].dbRoot] = emDbRootHWMInfos[j].totalBlocks; + } + } - //------------------------------------------------------------------ - // Ignore partition 0, segment 0 HWM extent if it is disabled - //------------------------------------------------------------------ - if ((emDbRootHWMInfos[j].partitionNum == 0) && - (emDbRootHWMInfos[j].segmentNum == 0) && - (emDbRootHWMInfos[j].status != BRM::EXTENTOUTOFSERVICE)) - { - if (emDbRootHWMInfos[j].localHWM == 0) - { - //newly created table - //cout << " This is newly created table. PM id is " << PMId; - startDBRootSet = true; - createdDbroot = emDbRootHWMInfos[j].dbRoot; - break; - } - else - { - createdDbroot = emDbRootHWMInfos[j].dbRoot; - //cout << " and createdDbroot is " << createdDbroot << endl; - rootBlocks[emDbRootHWMInfos[j].dbRoot] = emDbRootHWMInfos[j].totalBlocks; - } - } - else - { - rootBlocks[emDbRootHWMInfos[j].dbRoot] = emDbRootHWMInfos[j].totalBlocks; - } + if (startDBRootSet) + break; + } + + //-------------------------------------------------------------------------- + // Set the default PMId to the PM with the partition 0 segment 0 extent + //-------------------------------------------------------------------------- + PMId = 0; + + if (createdDbroot != 0) + { + std::map::iterator iter = fDbrootPMmap->begin(); + + iter = fDbrootPMmap->find(createdDbroot); + + if (iter != fDbrootPMmap->end()) + PMId = iter->second; + } + + // This will build the batch distribution sequence + // cout << "Building BatchDistSeqVector with PMId " << PMId << endl; + buildBatchDistSeqVector(PMId); + + // cout << "startDBRootSet = " << startDBRootSet << endl; + bool allEqual = true; + bool allOtherDbrootEmpty = true; + + //-------------------------------------------------------------------------- + // startDBRootSet == false + // We don't have an empty partition 0, segment 0 extent to load; + // so evaluate more selection criteria. + //-------------------------------------------------------------------------- + if (!startDBRootSet) + { + std::vector rootsExtentsBlocks; + std::map::iterator iter; + + //---------------------------------------------------------------------- + // Load rootsExtentsBlocks to carry the number of extents and blocks + // for each DBRoot. + //---------------------------------------------------------------------- + for (unsigned j = 0; j < fPmDistSeq.size(); j++) + { + PMRootInfo aEntry; + aEntry.PMId = fPmDistSeq[j]; + iter = fPmDbrootMap->find(aEntry.PMId); + + for (unsigned k = 0; k < (iter->second).size(); k++) + { + RootExtentsBlocks aRootInfo; + aRootInfo.DBRoot = (iter->second)[k]; + aRootInfo.numExtents = rootExtents[aRootInfo.DBRoot]; + aRootInfo.numBlocks = rootBlocks[aRootInfo.DBRoot]; + // cout << "aRootInfo DBRoot:numExtents:numBlocks = " << + // aRootInfo.DBRoot<<":"< rootsExtentsBlocks[0].rootInfo[j].numExtents) + tmpLeastExtents = rootsExtentsBlocks[0].rootInfo[j].numExtents; + } + + for (unsigned i = 1; i < rootsExtentsBlocks.size(); i++) + { + uint32_t leastExtents = rootsExtentsBlocks[i].rootInfo[0].numExtents; + + for (unsigned j = 0; j < rootsExtentsBlocks[i].rootInfo.size(); j++) + { + if (leastExtents > rootsExtentsBlocks[i].rootInfo[j].numExtents) + leastExtents = rootsExtentsBlocks[i].rootInfo[j].numExtents; } - if (startDBRootSet) - break; - } - - //-------------------------------------------------------------------------- - // Set the default PMId to the PM with the partition 0 segment 0 extent - //-------------------------------------------------------------------------- - PMId = 0; - - if ( createdDbroot != 0) - { - std::map::iterator iter = fDbrootPMmap->begin(); - - iter = fDbrootPMmap->find(createdDbroot); - - if (iter != fDbrootPMmap->end()) - PMId = iter->second; - } - - // This will build the batch distribution sequence - //cout << "Building BatchDistSeqVector with PMId " << PMId << endl; - buildBatchDistSeqVector(PMId); - - //cout << "startDBRootSet = " << startDBRootSet << endl; - bool allEqual = true; - bool allOtherDbrootEmpty = true; - - //-------------------------------------------------------------------------- - // startDBRootSet == false - // We don't have an empty partition 0, segment 0 extent to load; - // so evaluate more selection criteria. - //-------------------------------------------------------------------------- - if (!startDBRootSet) - { - std::vector rootsExtentsBlocks; - std::map::iterator iter; - - //---------------------------------------------------------------------- - // Load rootsExtentsBlocks to carry the number of extents and blocks - // for each DBRoot. - //---------------------------------------------------------------------- - for (unsigned j = 0; j < fPmDistSeq.size(); j++) + if (leastExtents < tmpLeastExtents) { - PMRootInfo aEntry; - aEntry.PMId = fPmDistSeq[j]; - iter = fPmDbrootMap->find(aEntry.PMId); + tmpLeastExtents = leastExtents; + PMId = rootsExtentsBlocks[i].PMId; + allEqual = false; + } + else if (leastExtents > tmpLeastExtents) + allEqual = false; + } - for (unsigned k = 0; k < (iter->second).size(); k++) - { - RootExtentsBlocks aRootInfo; - aRootInfo.DBRoot = (iter->second)[k]; - aRootInfo.numExtents = rootExtents[aRootInfo.DBRoot]; - aRootInfo.numBlocks = rootBlocks[aRootInfo.DBRoot]; - //cout << "aRootInfo DBRoot:numExtents:numBlocks = " << aRootInfo.DBRoot<<":"< rootsExtentsBlocks[0].rootInfo[j].numBlocks) + tmpBloks = rootsExtentsBlocks[0].rootInfo[j].numBlocks; } - //cout << "rootsExtentsBlocks size is " << rootsExtentsBlocks.size() << " and allOtherDbrootEmpty is " << allOtherDbrootEmpty<< endl; - - //---------------------------------------------------------------------- - // See if all other DBRoots other than the createdDbroot have 0 extents - //---------------------------------------------------------------------- for (unsigned i = 1; i < rootsExtentsBlocks.size(); i++) { - if (!allOtherDbrootEmpty) - break; + uint32_t leastBlocks = rootsExtentsBlocks[i].rootInfo[0].numBlocks; - //cout << "createdDbroot is " << createdDbroot << endl; - if (i != createdDbroot) - { - for ( unsigned j = 0; j < rootsExtentsBlocks[i].rootInfo.size(); j++) - { - if (rootsExtentsBlocks[i].rootInfo[j].numExtents != 0) - { - //cout << "setting allOtherDbrootEmpty to false and i:j = " << i<<":"< rootsExtentsBlocks[0].rootInfo[j].numExtents) - tmpLeastExtents = rootsExtentsBlocks[0].rootInfo[j].numExtents; - } - - for (unsigned i = 1; i < rootsExtentsBlocks.size(); i++) - { - uint32_t leastExtents = rootsExtentsBlocks[i].rootInfo[0].numExtents; - - for ( unsigned j = 0; j < rootsExtentsBlocks[i].rootInfo.size(); j++) - { - if (leastExtents > rootsExtentsBlocks[i].rootInfo[j].numExtents) - leastExtents = rootsExtentsBlocks[i].rootInfo[j].numExtents; - } - - if (leastExtents < tmpLeastExtents) - { - tmpLeastExtents = leastExtents; - PMId = rootsExtentsBlocks[i].PMId; - allEqual = false; - } - else if (leastExtents > tmpLeastExtents) - allEqual = false; - - } - - //cout << "allEqual is " << allEqual << endl; - - //------------------------------------------------------------------ - // All DBRoots have the same number of extents. - // Select PM with DBRoot having the fewest number of blocks. - //------------------------------------------------------------------ - if (allEqual) //Find the dbroot which has least number of blocks - { - //cout << "All PMs have equal # of least extents" << endl; - uint32_t tmpBloks = rootsExtentsBlocks[0].rootInfo[0].numBlocks; - - PMId = rootsExtentsBlocks[0].PMId; - - //cout << "tmpBloks:PMId = " << tmpBloks <<":"< rootsExtentsBlocks[0].rootInfo[j].numBlocks) - tmpBloks = rootsExtentsBlocks[0].rootInfo[j].numBlocks; - } - - for (unsigned i = 1; i < rootsExtentsBlocks.size(); i++) - { - uint32_t leastBlocks = rootsExtentsBlocks[i].rootInfo[0].numBlocks; - - //cout << "leastBlocks = " << leastBlocks << endl; - for ( unsigned j = 0; j < rootsExtentsBlocks[i].rootInfo.size(); j++) - { - if (leastBlocks > rootsExtentsBlocks[i].rootInfo[j].numBlocks) - leastBlocks = rootsExtentsBlocks[i].rootInfo[j].numBlocks; - } - - if (leastBlocks < tmpBloks) - { - tmpBloks = leastBlocks; - //cout << "tmpBloks changed to " << tmpBloks << endl; - PMId = rootsExtentsBlocks[i].PMId; - //cout << "setting allEqual to false now" << endl; - allEqual = false; - } - } - } - } - } - //-------------------------------------------------------------------------- - // startDBRootSet == true - // We select the empty partition 0, segment 0 extent to load - //-------------------------------------------------------------------------- - else - { - allEqual = false; + } } + } + //-------------------------------------------------------------------------- + // startDBRootSet == true + // We select the empty partition 0, segment 0 extent to load + //-------------------------------------------------------------------------- + else + { + allEqual = false; + } + fFirstPm = PMId; + + if (!allOtherDbrootEmpty || (PMId == 0)) + { + prepareForSecondPM(); + // cout << "prepareForSecondPM is called. " << endl; + } + + if ((allEqual && (PMId == 0)) || allOtherDbrootEmpty) + { + PMId = selectNextPM(); fFirstPm = PMId; - - if (!allOtherDbrootEmpty || (PMId == 0)) - { - prepareForSecondPM(); - //cout << "prepareForSecondPM is called. " << endl; - } - - if ((allEqual && (PMId == 0)) || allOtherDbrootEmpty) - { - PMId = selectNextPM(); - fFirstPm = PMId; - //cout << "PMId is now " << PMId << endl; - } + // cout << "PMId is now " << PMId << endl; + } } //------------------------------------------------------------------------------ @@ -405,65 +407,64 @@ void BatchLoader::selectFirstPM ( uint32_t& PMId) //------------------------------------------------------------------------------ void BatchLoader::buildBatchDistSeqVector() { - fPmDistSeq.clear(); - BlIntVec aDbCntVec(fPMs.size()); + fPmDistSeq.clear(); + BlIntVec aDbCntVec(fPMs.size()); - std::map::iterator iter = fPmDbrootMap->begin(); + std::map::iterator iter = fPmDbrootMap->begin(); - for (uint32_t i = 0; i < fPMs.size(); i++) + for (uint32_t i = 0; i < fPMs.size(); i++) + { + iter = fPmDbrootMap->find(fPMs[i]); + + if ((iter != fPmDbrootMap->end()) && ((iter->second).begin() != (iter->second).end())) { - iter = fPmDbrootMap->find(fPMs[i]); + try + { + aDbCntVec[i] = (iter->second).size(); + // cout << "PM - "< StartPm) && (fPMs[i] <= aLast)) + aPms.push_back(fPMs[i]); + } + + // Add all the PMs with index less than "StartPm" + for (uint32_t i = 0; i < fPMs.size(); i++) + { + if ((fPMs[i] < StartPm) && (fPMs[i] >= aFirst)) + aPms.push_back(fPMs[i]); + } + } + + std::map::iterator iter = fPmDbrootMap->begin(); + + for (uint32_t i = 0; i < aPms.size(); i++) + { + iter = fPmDbrootMap->find(aPms[i]); + + if ((iter != fPmDbrootMap->end()) && ((iter->second).begin() != (iter->second).end())) + { + aDbCntVec[i] = (iter->second).size(); + // cout << "PM - "< StartPm) && (fPMs[i] <= aLast)) aPms.push_back(fPMs[i]); - } - - // Add all the PMs with index less than "StartPm" - for (uint32_t i = 0; i < fPMs.size(); i++) - { - if ((fPMs[i] < StartPm) && (fPMs[i] >= aFirst)) aPms.push_back(fPMs[i]); - } + if (aDbCntVec[i] > aMax) + { + aMax = aDbCntVec[i]; + aPmId = aPms[i]; + aRefIdx = i; + } } - - std::map::iterator iter = fPmDbrootMap->begin(); - - for (uint32_t i = 0; i < aPms.size(); i++) + if (aMax > 0) { - iter = fPmDbrootMap->find(aPms[i]); - - if ((iter != fPmDbrootMap->end()) && ((iter->second).begin() != (iter->second).end())) - { - aDbCntVec[i] = (iter->second).size(); - //cout << "PM - "<= fPmDistSeq.size()) fNextIdx = 0; //reset it + if (fNextIdx >= fPmDistSeq.size()) + fNextIdx = 0; // reset it - return fPmDistSeq[fNextIdx++]; + return fPmDistSeq[fNextIdx++]; } //------------------------------------------------------------------------------ void BatchLoader::reverseSequence() { - if (0 == fNextIdx) fNextIdx = fPmDistSeq.size() - 1; - else fNextIdx--; + if (0 == fNextIdx) + fNextIdx = fPmDistSeq.size() - 1; + else + fNextIdx--; } //------------------------------------------------------------------------------ void BatchLoader::prepareForSecondPM() { - // We loop thru by calling the selectNextPM() - // When we get the aSecPm == aFirstPM, we will break so that when - // we call the same function again when we come back get the next PM - if ((fFirstPm != 0) && (fPMs.size() > 1)) - { - unsigned int aSecPm = selectNextPM(); + // We loop thru by calling the selectNextPM() + // When we get the aSecPm == aFirstPM, we will break so that when + // we call the same function again when we come back get the next PM + if ((fFirstPm != 0) && (fPMs.size() > 1)) + { + unsigned int aSecPm = selectNextPM(); - while (fFirstPm != aSecPm ) aSecPm = selectNextPM(); - } + while (fFirstPm != aSecPm) + aSecPm = selectNextPM(); + } } -} // end of namespace +} // namespace batchloader diff --git a/utils/batchloader/batchloader.h b/utils/batchloader/batchloader.h index 56789b7f5..3f9e0e895 100644 --- a/utils/batchloader/batchloader.h +++ b/utils/batchloader/batchloader.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_dbrootextenttracker.h 3672 2012-03-26 12:31:27Z rdempsey $ -*/ + * $Id: we_dbrootextenttracker.h 3672 2012-03-26 12:31:27Z rdempsey $ + */ /** @file we_dbrootextenttracker.h * Contains classes to track the order of placement (rotation) of extents as @@ -34,100 +34,98 @@ namespace batchloader { - //------------------------------------------------------------------------------ /** @brief Class to find the PM id to send a batch of row */ //------------------------------------------------------------------------------ class BatchLoader { -public: + public: + /** @brief BatchLoader constructor + * @param tableOid table OID of interest. + * @param sessionId cpimport use 0 as session id. + * @param PMs vector Collection of PM ids. + */ + BatchLoader(uint32_t tableOid, execplan::CalpontSystemCatalog::SCN sessionId, std::vector& PMs); - /** @brief BatchLoader constructor - * @param tableOid table OID of interest. - * @param sessionId cpimport use 0 as session id. - * @param PMs vector Collection of PM ids. + /** + * @brief select the Next PM where batch data to be distributed. + * return the PM where next batch to be send. + * if an error occurs, 0 will be returned + */ + uint32_t selectNextPM(); + + /** + * @brief Move to previous Sequence in the array. + * This can be used when we find that we cannot use the PM right now + * and want to use it later. for example the queue is full of the + * current PM and we want to get the same PM Id when we call selectNextPM() + * next time also. + */ + void reverseSequence(); + + /* + * @brief After calling selectFirstPM(), if we need to keep continuing to + * the next PM in the list, we need to call this. If we just want to start + * distributing from dbroot 1 onwards, no need to call this function. + */ + void prepareForSecondPM(); + + struct RootExtentsBlocks + { + /** @brief the dbroot */ - BatchLoader ( uint32_t tableOid, execplan::CalpontSystemCatalog::SCN sessionId, std::vector& PMs); - - /** - * @brief select the Next PM where batch data to be distributed. - * return the PM where next batch to be send. - * if an error occurs, 0 will be returned + uint32_t DBRoot; + /** @brief the number of extents */ - uint32_t selectNextPM(); - - /** - * @brief Move to previous Sequence in the array. - * This can be used when we find that we cannot use the PM right now - * and want to use it later. for example the queue is full of the - * current PM and we want to get the same PM Id when we call selectNextPM() - * next time also. + uint64_t numExtents; + /** @brief the number of blocks in the last partition */ - void reverseSequence(); + uint64_t numBlocks; + }; - /* - * @brief After calling selectFirstPM(), if we need to keep continuing to - * the next PM in the list, we need to call this. If we just want to start - * distributing from dbroot 1 onwards, no need to call this function. + struct PMRootInfo + { + /** @brief the module id */ - void prepareForSecondPM(); - - struct RootExtentsBlocks - { - /** @brief the dbroot - */ - uint32_t DBRoot; - /** @brief the number of extents - */ - uint64_t numExtents; - /** @brief the number of blocks in the last partition - */ - uint64_t numBlocks; - }; - - struct PMRootInfo - { - /** @brief the module id - */ - uint32_t PMId; - /** @brief the dbroot info - */ - std::vector rootInfo; - }; - -private: - /** @brief Select the first PM to send the first batch of rows. - * @param startFromNextPM - if true, don't use the PMId. Instead use the next one in the sorted dmList. - * @param PMId - The PM id to send the first batch of rows if startFromNextPM is false. - * @return Returns 0 if success, else returns error code. + uint32_t PMId; + /** @brief the dbroot info */ - void selectFirstPM ( uint32_t& PMId); + std::vector rootInfo; + }; - /** @brief build the batch distribution sequence in a vector - * return void - */ - void buildBatchDistSeqVector(); + private: + /** @brief Select the first PM to send the first batch of rows. + * @param startFromNextPM - if true, don't use the PMId. Instead use the next one in the sorted dmList. + * @param PMId - The PM id to send the first batch of rows if startFromNextPM is false. + * @return Returns 0 if success, else returns error code. + */ + void selectFirstPM(uint32_t& PMId); - /** @brief build the batch distribution sequence in a vector - * return void - */ - void buildBatchDistSeqVector(uint32_t StartPm); + /** @brief build the batch distribution sequence in a vector + * return void + */ + void buildBatchDistSeqVector(); - typedef std::vector BlIntVec; - BlIntVec fPMs; - BlIntVec fDbRoots; - BlIntVec fPmDistSeq; - uint32_t fNextIdx; - uint32_t fFirstPm; - execplan::CalpontSystemCatalog::SCN fSessionId; - uint32_t fTableOid; - oam::OamCache::PMDbrootsMap_t fPmDbrootMap; - oam::OamCache::dbRootPMMap_t fDbrootPMmap; + /** @brief build the batch distribution sequence in a vector + * return void + */ + void buildBatchDistSeqVector(uint32_t StartPm); + + typedef std::vector BlIntVec; + BlIntVec fPMs; + BlIntVec fDbRoots; + BlIntVec fPmDistSeq; + uint32_t fNextIdx; + uint32_t fFirstPm; + execplan::CalpontSystemCatalog::SCN fSessionId; + uint32_t fTableOid; + oam::OamCache::PMDbrootsMap_t fPmDbrootMap; + oam::OamCache::dbRootPMMap_t fDbrootPMmap; }; -} //end of namespace +} // namespace batchloader #undef EXPORT -#endif // WE_DBROOTEXTENTTRACKER_H_ +#endif // WE_DBROOTEXTENTTRACKER_H_ diff --git a/utils/cacheutils/cacheutils.cpp b/utils/cacheutils/cacheutils.cpp index b53ef7875..ead2483ad 100644 --- a/utils/cacheutils/cacheutils.cpp +++ b/utils/cacheutils/cacheutils.cpp @@ -50,123 +50,126 @@ using namespace BRM; namespace { - -//Only one of the cacheutils fcns can run at a time +// Only one of the cacheutils fcns can run at a time boost::mutex CacheOpsMutex; -//This global is updated only w/ atomic ops +// This global is updated only w/ atomic ops volatile uint32_t MultiReturnCode; int32_t extractRespCode(const ByteStream& bs) { - if (bs.length() < (sizeof(ISMPacketHeader) + sizeof(int32_t))) - return 1; + if (bs.length() < (sizeof(ISMPacketHeader) + sizeof(int32_t))) + return 1; - const uint8_t* bytePtr = bs.buf(); - const ISMPacketHeader* hdrp = reinterpret_cast(bytePtr); + const uint8_t* bytePtr = bs.buf(); + const ISMPacketHeader* hdrp = reinterpret_cast(bytePtr); - if (hdrp->Command != CACHE_OP_RESULTS) - return 1; + if (hdrp->Command != CACHE_OP_RESULTS) + return 1; - const int32_t* resp = reinterpret_cast(bytePtr + sizeof(ISMPacketHeader)); - return *resp; + const int32_t* resp = reinterpret_cast(bytePtr + sizeof(ISMPacketHeader)); + return *resp; } class CacheOpThread { -public: - CacheOpThread(const string& svr, const ByteStream& outBs) : fServerName(svr), fOutBs(outBs) {} - ~CacheOpThread() {} - void operator()() + public: + CacheOpThread(const string& svr, const ByteStream& outBs) : fServerName(svr), fOutBs(outBs) + { + } + ~CacheOpThread() + { + } + void operator()() + { + struct timespec ts = {10, 0}; + int32_t rc = 0; + scoped_ptr cl(new MessageQueueClient(fServerName)); + + try { - struct timespec ts = { 10, 0 }; - int32_t rc = 0; - scoped_ptr cl(new MessageQueueClient(fServerName)); - - try - { - cl->write(fOutBs); - rc = extractRespCode(cl->read(&ts)); - } - catch (...) - { - rc = 1; - } - - if (rc != 0) - atomicops::atomicCAS(&MultiReturnCode, 0, 1); + cl->write(fOutBs); + rc = extractRespCode(cl->read(&ts)); + } + catch (...) + { + rc = 1; } -private: - //CacheOpThread(const CacheOpThread& rhs); - //CacheOpThread& operator=(const CacheOpThread& rhs); + if (rc != 0) + atomicops::atomicCAS(&MultiReturnCode, 0, 1); + } - string fServerName; - ByteStream fOutBs; + private: + // CacheOpThread(const CacheOpThread& rhs); + // CacheOpThread& operator=(const CacheOpThread& rhs); + + string fServerName; + ByteStream fOutBs; }; int sendToAll(const ByteStream& outBs) { - //Not thread-safe: external synchronization is needed! + // Not thread-safe: external synchronization is needed! - // added code here to flush any running primprocs that may be active - // TODO: we really only need to flush each unique PrimProc, but we can't tell from the - // config file which those are, so use the same logic as joblist::DistributedEngineComm - Config* cf = Config::makeConfig(); + // added code here to flush any running primprocs that may be active + // TODO: we really only need to flush each unique PrimProc, but we can't tell from the + // config file which those are, so use the same logic as joblist::DistributedEngineComm + Config* cf = Config::makeConfig(); - const string section = "PrimitiveServers"; - int cnt = static_cast(Config::fromText(cf->getConfig(section, "Count"))); + const string section = "PrimitiveServers"; + int cnt = static_cast(Config::fromText(cf->getConfig(section, "Count"))); - if (cnt <= 0) cnt = 1; + if (cnt <= 0) + cnt = 1; - thread_group tg; - int rc = 0; - MultiReturnCode = 0; + thread_group tg; + int rc = 0; + MultiReturnCode = 0; - for (int i = 0; i < cnt; i++) - { - ostringstream oss; - oss << "PMS" << (i + 1); - tg.create_thread(CacheOpThread(oss.str(), outBs)); - } + for (int i = 0; i < cnt; i++) + { + ostringstream oss; + oss << "PMS" << (i + 1); + tg.create_thread(CacheOpThread(oss.str(), outBs)); + } - tg.join_all(); + tg.join_all(); - if (MultiReturnCode != 0) - rc = -1; + if (MultiReturnCode != 0) + rc = -1; - return rc; + return rc; } -} +} // namespace namespace cacheutils { - /** * */ int flushPrimProcCache() { - boost::mutex::scoped_lock lk(CacheOpsMutex); + boost::mutex::scoped_lock lk(CacheOpsMutex); - try - { - const int msgsize = sizeof(ISMPacketHeader); - uint8_t msgbuf[msgsize]; - memset(msgbuf, 0, sizeof(ISMPacketHeader)); - ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); - hdrp->Command = CACHE_FLUSH; + try + { + const int msgsize = sizeof(ISMPacketHeader); + uint8_t msgbuf[msgsize]; + memset(msgbuf, 0, sizeof(ISMPacketHeader)); + ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); + hdrp->Command = CACHE_FLUSH; - ByteStream bs(msgbuf, msgsize); - int rc = sendToAll(bs); - return rc; - } - catch (...) - { - } + ByteStream bs(msgbuf, msgsize); + int rc = sendToAll(bs); + return rc; + } + catch (...) + { + } - return -1; + return -1; } /** @@ -174,180 +177,181 @@ int flushPrimProcCache() */ int flushPrimProcBlocks(const BRM::BlockList_t& list) { - if (list.empty()) return 0; + if (list.empty()) + return 0; - boost::mutex::scoped_lock lk(CacheOpsMutex); + boost::mutex::scoped_lock lk(CacheOpsMutex); #if defined(__LP64__) || defined(_WIN64) - if (list.size() > numeric_limits::max()) return -1; + if (list.size() > numeric_limits::max()) + return -1; #endif - try - { - const size_t msgsize = sizeof(ISMPacketHeader) + sizeof(uint32_t) + sizeof(LbidAtVer) * list.size(); - scoped_array msgbuf(new uint8_t[msgsize]); - memset(msgbuf.get(), 0, sizeof(ISMPacketHeader)); - ISMPacketHeader* hdrp = reinterpret_cast(msgbuf.get()); - hdrp->Command = CACHE_CLEAN_VSS; - uint32_t* cntp = reinterpret_cast(msgbuf.get() + sizeof(ISMPacketHeader)); - *cntp = static_cast(list.size()); - LbidAtVer* itemp = reinterpret_cast(msgbuf.get() + sizeof(ISMPacketHeader) + sizeof(uint32_t)); - BlockList_t::const_iterator iter = list.begin(); - BlockList_t::const_iterator end = list.end(); + try + { + const size_t msgsize = sizeof(ISMPacketHeader) + sizeof(uint32_t) + sizeof(LbidAtVer) * list.size(); + scoped_array msgbuf(new uint8_t[msgsize]); + memset(msgbuf.get(), 0, sizeof(ISMPacketHeader)); + ISMPacketHeader* hdrp = reinterpret_cast(msgbuf.get()); + hdrp->Command = CACHE_CLEAN_VSS; + uint32_t* cntp = reinterpret_cast(msgbuf.get() + sizeof(ISMPacketHeader)); + *cntp = static_cast(list.size()); + LbidAtVer* itemp = + reinterpret_cast(msgbuf.get() + sizeof(ISMPacketHeader) + sizeof(uint32_t)); + BlockList_t::const_iterator iter = list.begin(); + BlockList_t::const_iterator end = list.end(); - while (iter != end) - { - itemp->LBID = static_cast(iter->first); - itemp->Ver = static_cast(iter->second); - ++itemp; - ++iter; - } - - ByteStream bs(msgbuf.get(), msgsize); - int rc = sendToAll(bs); - return rc; - } - catch (...) + while (iter != end) { + itemp->LBID = static_cast(iter->first); + itemp->Ver = static_cast(iter->second); + ++itemp; + ++iter; } - return -1; + ByteStream bs(msgbuf.get(), msgsize); + int rc = sendToAll(bs); + return rc; + } + catch (...) + { + } + + return -1; } - int flushPrimProcAllverBlocks(const vector& list) { - if (list.empty()) return 0; + if (list.empty()) + return 0; - ByteStream bs(sizeof(ISMPacketHeader) + sizeof(uint32_t) + (sizeof(LBID_t) * list.size())); - ISMPacketHeader* hdr; - int rc; + ByteStream bs(sizeof(ISMPacketHeader) + sizeof(uint32_t) + (sizeof(LBID_t) * list.size())); + ISMPacketHeader* hdr; + int rc; - hdr = (ISMPacketHeader*) bs.getInputPtr(); - hdr->Command = FLUSH_ALL_VERSION; - bs.advanceInputPtr(sizeof(ISMPacketHeader)); - bs << (uint32_t) list.size(); - bs.append((uint8_t*) &list[0], sizeof(LBID_t) * list.size()); + hdr = (ISMPacketHeader*)bs.getInputPtr(); + hdr->Command = FLUSH_ALL_VERSION; + bs.advanceInputPtr(sizeof(ISMPacketHeader)); + bs << (uint32_t)list.size(); + bs.append((uint8_t*)&list[0], sizeof(LBID_t) * list.size()); - try - { - boost::mutex::scoped_lock lk(CacheOpsMutex); - rc = sendToAll(bs); - return rc; - } - catch (...) - { - } + try + { + boost::mutex::scoped_lock lk(CacheOpsMutex); + rc = sendToAll(bs); + return rc; + } + catch (...) + { + } - return -1; + return -1; } int flushOIDsFromCache(const vector& oids) { - /* Message format: - * ISMPacketHeader - * uint32_t - OID count - * uint32_t * - OID array - */ + /* Message format: + * ISMPacketHeader + * uint32_t - OID count + * uint32_t * - OID array + */ - boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t()); + boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t()); - ByteStream bs; - ISMPacketHeader ism; - uint32_t i; + ByteStream bs; + ISMPacketHeader ism; + uint32_t i; - void *ismp = static_cast(&ism); - memset(ismp, 0, sizeof(ISMPacketHeader)); - ism.Command = CACHE_FLUSH_BY_OID; - bs.load((uint8_t*) &ism, sizeof(ISMPacketHeader)); - bs << (uint32_t) oids.size(); + void* ismp = static_cast(&ism); + memset(ismp, 0, sizeof(ISMPacketHeader)); + ism.Command = CACHE_FLUSH_BY_OID; + bs.load((uint8_t*)&ism, sizeof(ISMPacketHeader)); + bs << (uint32_t)oids.size(); - for (i = 0; i < oids.size(); i++) - bs << (uint32_t) oids[i]; + for (i = 0; i < oids.size(); i++) + bs << (uint32_t)oids[i]; - lk.lock(); - return sendToAll(bs); + lk.lock(); + return sendToAll(bs); } int flushPartition(const std::vector& oids, set& partitionNums) { - /* Message format: - * ISMPacketHeader - * uint32_t - partition count - * LogicalPartition * - partitionNum - * uint32_t - OID count - * uint32_t * - OID array - */ + /* Message format: + * ISMPacketHeader + * uint32_t - partition count + * LogicalPartition * - partitionNum + * uint32_t - OID count + * uint32_t * - OID array + */ - boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t()); + boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t()); - ByteStream bs; - ISMPacketHeader ism; + ByteStream bs; + ISMPacketHeader ism; - void *ismp = static_cast(&ism); - memset(ismp, 0, sizeof(ISMPacketHeader)); - ism.Command = CACHE_FLUSH_PARTITION; - bs.load((uint8_t*) &ism, sizeof(ISMPacketHeader)); - serializeSet(bs, partitionNums); - serializeInlineVector(bs, oids); + void* ismp = static_cast(&ism); + memset(ismp, 0, sizeof(ISMPacketHeader)); + ism.Command = CACHE_FLUSH_PARTITION; + bs.load((uint8_t*)&ism, sizeof(ISMPacketHeader)); + serializeSet(bs, partitionNums); + serializeInlineVector(bs, oids); - lk.lock(); - return sendToAll(bs); + lk.lock(); + return sendToAll(bs); } - int dropPrimProcFdCache() { - const int msgsize = sizeof(ISMPacketHeader); - uint8_t msgbuf[msgsize]; - memset(msgbuf, 0, sizeof(ISMPacketHeader)); - ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); - hdrp->Command = CACHE_DROP_FDS; - ByteStream bs(msgbuf, msgsize); + const int msgsize = sizeof(ISMPacketHeader); + uint8_t msgbuf[msgsize]; + memset(msgbuf, 0, sizeof(ISMPacketHeader)); + ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); + hdrp->Command = CACHE_DROP_FDS; + ByteStream bs(msgbuf, msgsize); - try - { - boost::mutex::scoped_lock lk(CacheOpsMutex); - int rc = sendToAll(bs); - return rc; - } - catch (...) - { - } + try + { + boost::mutex::scoped_lock lk(CacheOpsMutex); + int rc = sendToAll(bs); + return rc; + } + catch (...) + { + } - return -1; + return -1; } int purgePrimProcFdCache(const std::vector files, const int pmId) { - const int msgsize = sizeof(ISMPacketHeader); - uint8_t msgbuf[msgsize]; - memset(msgbuf, 0, sizeof(ISMPacketHeader)); - ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); - hdrp->Command = CACHE_PURGE_FDS; - ByteStream bs(msgbuf, msgsize); - serializeInlineVector(bs, files); - int32_t rc = 0; + const int msgsize = sizeof(ISMPacketHeader); + uint8_t msgbuf[msgsize]; + memset(msgbuf, 0, sizeof(ISMPacketHeader)); + ISMPacketHeader* hdrp = reinterpret_cast(&msgbuf[0]); + hdrp->Command = CACHE_PURGE_FDS; + ByteStream bs(msgbuf, msgsize); + serializeInlineVector(bs, files); + int32_t rc = 0; - try - { - struct timespec ts = { 10, 0 }; - ostringstream oss; - oss << "PMS" << pmId; - scoped_ptr cl(new MessageQueueClient(oss.str())); - cl->write(bs); - rc = extractRespCode(cl->read(&ts)); - } - catch (...) - { - rc = -1; - } + try + { + struct timespec ts = {10, 0}; + ostringstream oss; + oss << "PMS" << pmId; + scoped_ptr cl(new MessageQueueClient(oss.str())); + cl->write(bs); + rc = extractRespCode(cl->read(&ts)); + } + catch (...) + { + rc = -1; + } - return rc; -} + return rc; } +} // namespace cacheutils // vim:ts=4 sw=4: - diff --git a/utils/cacheutils/cacheutils.h b/utils/cacheutils/cacheutils.h index 700970da5..13d01ef61 100644 --- a/utils/cacheutils/cacheutils.h +++ b/utils/cacheutils/cacheutils.h @@ -72,8 +72,7 @@ int dropPrimProcFdCache(); */ int purgePrimProcFdCache(const std::vector files, const int pmId); -} +} // namespace cacheutils // vim:ts=4 sw=4: #endif - diff --git a/utils/cloudio/SMComm.cpp b/utils/cloudio/SMComm.cpp index 59d93c78f..e2e9b3a93 100644 --- a/utils/cloudio/SMComm.cpp +++ b/utils/cloudio/SMComm.cpp @@ -23,258 +23,258 @@ using namespace messageqcpp; namespace { -idbdatafile::SMComm *instance = NULL; +idbdatafile::SMComm* instance = NULL; boost::mutex m; -}; +}; // namespace namespace idbdatafile { - -SMComm * SMComm::get() +SMComm* SMComm::get() { - if (instance) - return instance; - - boost::mutex::scoped_lock sl(m); - - if (instance) - return instance; - instance = new SMComm(); + if (instance) return instance; + + boost::mutex::scoped_lock sl(m); + + if (instance) + return instance; + instance = new SMComm(); + return instance; } // timesavers #define common_exit(bs1, bs2, retCode) \ - { \ - int l_errno = errno; \ - buffers.returnByteStream(bs1); \ - buffers.returnByteStream(bs2); \ - errno = l_errno; \ - return retCode; \ - } + { \ + int l_errno = errno; \ + buffers.returnByteStream(bs1); \ + buffers.returnByteStream(bs2); \ + errno = l_errno; \ + return retCode; \ + } // bs1 is the bytestream ptr with the command to SMComm. // bs2 is the bytestream pointer with the response from SMComm. // retCode is the var to store the return code in from the msg. // returns with the output pointer at the fcn-specific data #define check_for_error(bs1, bs2, retCode) \ - { \ - int l_errno; \ - *bs2 >> retCode; \ - if (retCode < 0) \ - { \ - *bs2 >> l_errno; \ - errno = l_errno; \ - common_exit(bs1, bs2, retCode); \ - } \ - else \ - errno = 0; \ - } - + { \ + int l_errno; \ + *bs2 >> retCode; \ + if (retCode < 0) \ + { \ + *bs2 >> l_errno; \ + errno = l_errno; \ + common_exit(bs1, bs2, retCode); \ + } \ + else \ + errno = 0; \ + } + SMComm::SMComm() { - char buf[4096]; - cwd = ::getcwd(buf, 4096); + char buf[4096]; + cwd = ::getcwd(buf, 4096); } SMComm::~SMComm() { } -string SMComm::getAbsFilename(const string &filename) +string SMComm::getAbsFilename(const string& filename) { - if (filename[0] == '/') - return filename; - else - return cwd + '/' + filename; + if (filename[0] == '/') + return filename; + else + return cwd + '/' + filename; } -int SMComm::open(const string &filename, const int mode, struct stat *statbuf) +int SMComm::open(const string& filename, const int mode, struct stat* statbuf) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); - - *command << (uint8_t) storagemanager::OPEN << mode << absfilename; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - - check_for_error(command, response, err); - memcpy(statbuf, response->buf(), sizeof(*statbuf)); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); + + *command << (uint8_t)storagemanager::OPEN << mode << absfilename; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + + check_for_error(command, response, err); + memcpy(statbuf, response->buf(), sizeof(*statbuf)); + common_exit(command, response, err); } -ssize_t SMComm::pread(const string &filename, void *buf, const size_t count, const off_t offset) +ssize_t SMComm::pread(const string& filename, void* buf, const size_t count, const off_t offset) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); - - *command << (uint8_t) storagemanager::READ << count << offset << absfilename; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); - - memcpy(buf, response->buf(), err); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); + + *command << (uint8_t)storagemanager::READ << count << offset << absfilename; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + + memcpy(buf, response->buf(), err); + common_exit(command, response, err); } -ssize_t SMComm::pwrite(const string &filename, const void *buf, const size_t count, const off_t offset) +ssize_t SMComm::pwrite(const string& filename, const void* buf, const size_t count, const off_t offset) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); - - *command << (uint8_t) storagemanager::WRITE << count << offset << absfilename; - command->needAtLeast(count); - uint8_t *cmdBuf = command->getInputPtr(); - memcpy(cmdBuf, buf, count); - command->advanceInputPtr(count); - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); + + *command << (uint8_t)storagemanager::WRITE << count << offset << absfilename; + command->needAtLeast(count); + uint8_t* cmdBuf = command->getInputPtr(); + memcpy(cmdBuf, buf, count); + command->advanceInputPtr(count); + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } -ssize_t SMComm::append(const string &filename, const void *buf, const size_t count) +ssize_t SMComm::append(const string& filename, const void* buf, const size_t count) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); - *command << (uint8_t) storagemanager::APPEND << count << absfilename; - command->needAtLeast(count); - uint8_t *cmdBuf = command->getInputPtr(); - memcpy(cmdBuf, buf, count); - command->advanceInputPtr(count); - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + *command << (uint8_t)storagemanager::APPEND << count << absfilename; + command->needAtLeast(count); + uint8_t* cmdBuf = command->getInputPtr(); + memcpy(cmdBuf, buf, count); + command->advanceInputPtr(count); + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } -int SMComm::unlink(const string &filename) +int SMComm::unlink(const string& filename) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); - - *command << (uint8_t) storagemanager::UNLINK << absfilename; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); + + *command << (uint8_t)storagemanager::UNLINK << absfilename; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } -int SMComm::stat(const string &filename, struct stat *statbuf) +int SMComm::stat(const string& filename, struct stat* statbuf) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); - - *command << (uint8_t) storagemanager::STAT << absfilename; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); - - memcpy(statbuf, response->buf(), sizeof(*statbuf)); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); + + *command << (uint8_t)storagemanager::STAT << absfilename; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + + memcpy(statbuf, response->buf(), sizeof(*statbuf)); + common_exit(command, response, err); } -int SMComm::truncate(const string &filename, const off64_t length) +int SMComm::truncate(const string& filename, const off64_t length) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename(getAbsFilename(filename)); - - *command << (uint8_t) storagemanager::TRUNCATE << length << absfilename; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename(getAbsFilename(filename)); + + *command << (uint8_t)storagemanager::TRUNCATE << length << absfilename; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } -int SMComm::listDirectory(const string &path, list *entries) +int SMComm::listDirectory(const string& path, list* entries) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string abspath(getAbsFilename(path)); - - *command << (uint8_t) storagemanager::LIST_DIRECTORY << abspath; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); - - uint32_t numElements; - string stmp; - entries->clear(); - *response >> numElements; - while (numElements > 0) { - *response >> stmp; - entries->push_back(stmp); - numElements--; - } + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string abspath(getAbsFilename(path)); + + *command << (uint8_t)storagemanager::LIST_DIRECTORY << abspath; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + + uint32_t numElements; + string stmp; + entries->clear(); + *response >> numElements; + while (numElements > 0) + { + *response >> stmp; + entries->push_back(stmp); + numElements--; + } + common_exit(command, response, err); } int SMComm::ping() { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - - *command << (uint8_t) storagemanager::PING; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + + *command << (uint8_t)storagemanager::PING; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } int SMComm::sync() { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; - *command << (uint8_t) storagemanager::SYNC; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + *command << (uint8_t)storagemanager::SYNC; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } -int SMComm::copyFile(const string &file1, const string &file2) +int SMComm::copyFile(const string& file1, const string& file2) { - ByteStream *command = buffers.getByteStream(); - ByteStream *response = buffers.getByteStream(); - ssize_t err; - string absfilename1(getAbsFilename(file1)); - string absfilename2(getAbsFilename(file2)); - - *command << (uint8_t) storagemanager::COPY << absfilename1 << absfilename2; - err = sockets.send_recv(*command, response); - if (err) - common_exit(command, response, err); - check_for_error(command, response, err); + ByteStream* command = buffers.getByteStream(); + ByteStream* response = buffers.getByteStream(); + ssize_t err; + string absfilename1(getAbsFilename(file1)); + string absfilename2(getAbsFilename(file2)); + + *command << (uint8_t)storagemanager::COPY << absfilename1 << absfilename2; + err = sockets.send_recv(*command, response); + if (err) common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); } -} +} // namespace idbdatafile diff --git a/utils/cloudio/SMComm.h b/utils/cloudio/SMComm.h index d7240e73a..bb46304bd 100644 --- a/utils/cloudio/SMComm.h +++ b/utils/cloudio/SMComm.h @@ -24,57 +24,56 @@ #include "bytestream.h" #include "bytestreampool.h" -namespace idbdatafile +namespace idbdatafile { - class SMComm : public boost::noncopyable { - public: - // This is a singleton. Get it with get() - static SMComm *get(); - - /* Open currently returns a stat struct so SMDataFile can set its initial position, otherwise - behaves how you'd think. */ - int open(const std::string &filename, const int mode, struct stat *statbuf); - - ssize_t pread(const std::string &filename, void *buf, const size_t count, const off_t offset); - - ssize_t pwrite(const std::string &filename, const void *buf, const size_t count, const off_t offset); - - /* append exists for cases where the file is open in append mode. A normal write won't work - because the file position/size may be out of date if there are multiple writers. */ - ssize_t append(const std::string &filename, const void *buf, const size_t count); - - int unlink(const std::string &filename); - - int stat(const std::string &filename, struct stat *statbuf); - - // added this one because it should be trivial to implement in SM, and prevents a large - // operation in SMDataFile. - int truncate(const std::string &filename, const off64_t length); - - int listDirectory(const std::string &path, std::list *entries); - - // health indicator. 0 = processes are talking to each other and SM has read/write access to - // the specified S3 bucket. Need to define specific error codes. - int ping(); - - int sync(); + public: + // This is a singleton. Get it with get() + static SMComm* get(); - int copyFile(const std::string &file1, const std::string &file2); - - virtual ~SMComm(); - - private: - SMComm(); - - std::string getAbsFilename(const std::string &filename); - - SocketPool sockets; - messageqcpp::ByteStreamPool buffers; - std::string cwd; + /* Open currently returns a stat struct so SMDataFile can set its initial position, otherwise + behaves how you'd think. */ + int open(const std::string& filename, const int mode, struct stat* statbuf); + + ssize_t pread(const std::string& filename, void* buf, const size_t count, const off_t offset); + + ssize_t pwrite(const std::string& filename, const void* buf, const size_t count, const off_t offset); + + /* append exists for cases where the file is open in append mode. A normal write won't work + because the file position/size may be out of date if there are multiple writers. */ + ssize_t append(const std::string& filename, const void* buf, const size_t count); + + int unlink(const std::string& filename); + + int stat(const std::string& filename, struct stat* statbuf); + + // added this one because it should be trivial to implement in SM, and prevents a large + // operation in SMDataFile. + int truncate(const std::string& filename, const off64_t length); + + int listDirectory(const std::string& path, std::list* entries); + + // health indicator. 0 = processes are talking to each other and SM has read/write access to + // the specified S3 bucket. Need to define specific error codes. + int ping(); + + int sync(); + + int copyFile(const std::string& file1, const std::string& file2); + + virtual ~SMComm(); + + private: + SMComm(); + + std::string getAbsFilename(const std::string& filename); + + SocketPool sockets; + messageqcpp::ByteStreamPool buffers; + std::string cwd; }; -} +} // namespace idbdatafile #endif diff --git a/utils/cloudio/SMDataFile.cpp b/utils/cloudio/SMDataFile.cpp index 8b9d1be83..84eab1ac9 100644 --- a/utils/cloudio/SMDataFile.cpp +++ b/utils/cloudio/SMDataFile.cpp @@ -14,7 +14,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - + #include #include "SMDataFile.h" @@ -23,131 +23,123 @@ using namespace std; namespace idbdatafile { - SMDataFile::~SMDataFile() { } -SMDataFile::SMDataFile(const char *name, int _openmode, const struct stat &_stat) : - IDBDataFile(name) +SMDataFile::SMDataFile(const char* name, int _openmode, const struct stat& _stat) : IDBDataFile(name) { - openmode = _openmode; - // the 'a' file open mode is the only one that starts at EOF - if ((openmode & O_APPEND) && !(openmode & O_RDWR)) - position = _stat.st_size; - else - position = 0; - comm = SMComm::get(); + openmode = _openmode; + // the 'a' file open mode is the only one that starts at EOF + if ((openmode & O_APPEND) && !(openmode & O_RDWR)) + position = _stat.st_size; + else + position = 0; + comm = SMComm::get(); } -ssize_t SMDataFile::pread(void *buf, off64_t offset, size_t count) +ssize_t SMDataFile::pread(void* buf, off64_t offset, size_t count) { - return comm->pread(name(), buf, count, offset); + return comm->pread(name(), buf, count, offset); } -ssize_t SMDataFile::read(void *buf, size_t count) +ssize_t SMDataFile::read(void* buf, size_t count) { - ssize_t ret = comm->pread(name(), buf, count, position); - if (ret < 0) - return ret; - position += ret; + ssize_t ret = comm->pread(name(), buf, count, position); + if (ret < 0) return ret; + position += ret; + return ret; } -ssize_t SMDataFile::write(const void *buf, size_t count) +ssize_t SMDataFile::write(const void* buf, size_t count) { - if (openmode & O_APPEND) - return comm->append(name(), buf, count); - ssize_t ret = comm->pwrite(name(), buf, count, position); - if (ret < 0) - return ret; - position += ret; + if (openmode & O_APPEND) + return comm->append(name(), buf, count); + ssize_t ret = comm->pwrite(name(), buf, count, position); + if (ret < 0) return ret; + position += ret; + return ret; } int SMDataFile::seek(off64_t offset, int whence) { - switch (whence) { - case SEEK_SET: - position = offset; - break; - case SEEK_CUR: - position += offset; - break; - case SEEK_END: - { - struct stat _stat; - int err = comm->stat(name(), &_stat); - if (err) - return err; - position = _stat.st_size + offset; - break; - } - default: - errno = EINVAL; - return -1; + switch (whence) + { + case SEEK_SET: position = offset; break; + case SEEK_CUR: position += offset; break; + case SEEK_END: + { + struct stat _stat; + int err = comm->stat(name(), &_stat); + if (err) + return err; + position = _stat.st_size + offset; + break; } - return 0; + default: errno = EINVAL; return -1; + } + return 0; } int SMDataFile::truncate(off64_t length) { - return comm->truncate(name(), length); + return comm->truncate(name(), length); } int SMDataFile::fallocate(int mode, off64_t offset, off64_t length) { - idbassert_s(mode == 0, "SMDataFile::fallocate() does not support mode != 0 right now."); - return comm->truncate(name(), offset + length); + idbassert_s(mode == 0, "SMDataFile::fallocate() does not support mode != 0 right now."); + return comm->truncate(name(), offset + length); } off64_t SMDataFile::size() { - struct stat _stat; - int err = comm->stat(name(), &_stat); - - if (err) - return err; - return _stat.st_size; + struct stat _stat; + int err = comm->stat(name(), &_stat); + + if (err) + return err; + return _stat.st_size; } off64_t SMDataFile::tell() { - return position; + return position; } int SMDataFile::flush() { - return 0; // writes are synchronous b/c of replication. If we allow asynchronous replication, - // then we need to implement a flush() cmd in SMComm. + return 0; // writes are synchronous b/c of replication. If we allow asynchronous replication, + // then we need to implement a flush() cmd in SMComm. } time_t SMDataFile::mtime() { - struct stat _stat; - int err = comm->stat(name(), &_stat); - - if (err) - return (time_t) err; - return _stat.st_mtime; + struct stat _stat; + int err = comm->stat(name(), &_stat); + + if (err) + return (time_t)err; + return _stat.st_mtime; } int SMDataFile::close() { - return 0; + return 0; } // constructor used for testing -SMDataFile::SMDataFile(const char *fname, int _openmode, size_t fake_size) - : IDBDataFile(fname) +SMDataFile::SMDataFile(const char* fname, int _openmode, size_t fake_size) : IDBDataFile(fname) { - openmode = _openmode; - // the 'a' file open mode is the only one that starts at EOF - if ((openmode & O_APPEND) && !(openmode & O_RDWR)) - position = fake_size; - else - position = 0; - comm = SMComm::get(); + openmode = _openmode; + // the 'a' file open mode is the only one that starts at EOF + if ((openmode & O_APPEND) && !(openmode & O_RDWR)) + position = fake_size; + else + position = 0; + comm = SMComm::get(); } -} +} // namespace idbdatafile diff --git a/utils/cloudio/SMDataFile.h b/utils/cloudio/SMDataFile.h index b83de0238..1d77edd6c 100644 --- a/utils/cloudio/SMDataFile.h +++ b/utils/cloudio/SMDataFile.h @@ -14,7 +14,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - + #ifndef SMDATAFILE_H_ #define SMDATAFILE_H_ @@ -23,39 +23,38 @@ #include "IDBDataFile.h" #include "SMComm.h" -namespace idbdatafile +namespace idbdatafile { - class SMDataFile : public IDBDataFile { - public: - virtual ~SMDataFile(); + public: + virtual ~SMDataFile(); - ssize_t pread(void* ptr, off64_t offset, size_t count); - ssize_t read(void* ptr, size_t count); - ssize_t write(const void* ptr, size_t count); - int seek(off64_t offset, int whence); - int truncate(off64_t length); - int fallocate(int mode, off64_t offset, off64_t length); - off64_t size(); - off64_t tell(); - int flush(); - time_t mtime(); - int close(); - - // for testing only - SMDataFile(const char *fname, int openmode, size_t fake_size); - - private: - SMDataFile(); - SMDataFile(const char *fname, int openmode, const struct stat &); - off64_t position; - int openmode; - SMComm *comm; - - friend class SMFileFactory; + ssize_t pread(void* ptr, off64_t offset, size_t count); + ssize_t read(void* ptr, size_t count); + ssize_t write(const void* ptr, size_t count); + int seek(off64_t offset, int whence); + int truncate(off64_t length); + int fallocate(int mode, off64_t offset, off64_t length); + off64_t size(); + off64_t tell(); + int flush(); + time_t mtime(); + int close(); + + // for testing only + SMDataFile(const char* fname, int openmode, size_t fake_size); + + private: + SMDataFile(); + SMDataFile(const char* fname, int openmode, const struct stat&); + off64_t position; + int openmode; + SMComm* comm; + + friend class SMFileFactory; }; -} +} // namespace idbdatafile #endif diff --git a/utils/cloudio/SMFileFactory.cpp b/utils/cloudio/SMFileFactory.cpp index d856d6e16..6582d5a93 100644 --- a/utils/cloudio/SMFileFactory.cpp +++ b/utils/cloudio/SMFileFactory.cpp @@ -14,7 +14,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - + #include #include #include "SMFileFactory.h" @@ -25,83 +25,82 @@ using namespace std; - -namespace idbdatafile { - -IDBDataFile* SMFileFactory::open(const char *filename, const char *mode, unsigned opts, unsigned colWidth) +namespace idbdatafile { - // TODO, test whether this breaks anything. - //if (opts & IDBDataFile::USE_TMPFILE) - // return new BufferedFile(filename, mode, opts); - - bool _read = false; - bool _write = false; - bool create = false; - bool truncate = false; - bool append = false; +IDBDataFile* SMFileFactory::open(const char* filename, const char* mode, unsigned opts, unsigned colWidth) +{ + // TODO, test whether this breaks anything. + // if (opts & IDBDataFile::USE_TMPFILE) + // return new BufferedFile(filename, mode, opts); - // strip 'b' chars from mode - char newmode[8] = {'\0'}; // there'd better not be 7 chars in the mode string - int i = 0; - for (const char *c = mode; *c != '\0' && i < 8; c++) - if (*c != 'b') - newmode[i++] = *c; - if (i == 8) { - errno = EINVAL; - return NULL; - } - - // parse the new mode string - if (newmode[0] == 'r') - { - _read = true; - if (newmode[1] == '+') - _write = true; - } - else if (newmode[0] == 'w') - { - _write = true; - truncate = true; - create = true; - if (newmode[1] == '+') - _read = true; - } - else if (newmode[0] == 'a') - { - _write = true; - create = true; - append = true; - if (newmode[1] == '+') - _read = true; - } - else - { - errno = EINVAL; - return NULL; - } - - // turn newmode into posix flags - uint posix_flags = 0; - if (_read && _write) - posix_flags |= O_RDWR; - else if (_read) - posix_flags |= O_RDONLY; - else if (_write) - posix_flags |= O_WRONLY; - - posix_flags |= (create ? O_CREAT : 0); - posix_flags |= (truncate ? O_TRUNC : 0); - posix_flags |= (append ? O_APPEND : 0); - - SMComm *comm = SMComm::get(); - struct stat _stat; - int err = comm->open(filename, posix_flags, &_stat); - if (err) - return NULL; - - SMDataFile *ret = new SMDataFile(filename, posix_flags, _stat); - return ret; + bool _read = false; + bool _write = false; + bool create = false; + bool truncate = false; + bool append = false; + + // strip 'b' chars from mode + char newmode[8] = {'\0'}; // there'd better not be 7 chars in the mode string + int i = 0; + for (const char* c = mode; *c != '\0' && i < 8; c++) + if (*c != 'b') + newmode[i++] = *c; + if (i == 8) + { + errno = EINVAL; + return NULL; + } + + // parse the new mode string + if (newmode[0] == 'r') + { + _read = true; + if (newmode[1] == '+') + _write = true; + } + else if (newmode[0] == 'w') + { + _write = true; + truncate = true; + create = true; + if (newmode[1] == '+') + _read = true; + } + else if (newmode[0] == 'a') + { + _write = true; + create = true; + append = true; + if (newmode[1] == '+') + _read = true; + } + else + { + errno = EINVAL; + return NULL; + } + + // turn newmode into posix flags + uint posix_flags = 0; + if (_read && _write) + posix_flags |= O_RDWR; + else if (_read) + posix_flags |= O_RDONLY; + else if (_write) + posix_flags |= O_WRONLY; + + posix_flags |= (create ? O_CREAT : 0); + posix_flags |= (truncate ? O_TRUNC : 0); + posix_flags |= (append ? O_APPEND : 0); + + SMComm* comm = SMComm::get(); + struct stat _stat; + int err = comm->open(filename, posix_flags, &_stat); + if (err) + return NULL; + + SMDataFile* ret = new SMDataFile(filename, posix_flags, _stat); + return ret; } - -} +} // namespace idbdatafile diff --git a/utils/cloudio/SMFileFactory.h b/utils/cloudio/SMFileFactory.h index 0ee0b7fba..961a85e54 100644 --- a/utils/cloudio/SMFileFactory.h +++ b/utils/cloudio/SMFileFactory.h @@ -23,12 +23,11 @@ namespace idbdatafile { - class SMFileFactory : public FileFactoryBase { - public: - IDBDataFile * open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); + public: + IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); }; -} +} // namespace idbdatafile #endif diff --git a/utils/cloudio/SMFileSystem.cpp b/utils/cloudio/SMFileSystem.cpp index fcd12b073..96fe25d37 100644 --- a/utils/cloudio/SMFileSystem.cpp +++ b/utils/cloudio/SMFileSystem.cpp @@ -24,95 +24,94 @@ using namespace std; namespace idbdatafile { - SMFileSystem::SMFileSystem() : IDBFileSystem(IDBFileSystem::CLOUD) { - SMComm::get(); // get SMComm running + SMComm::get(); // get SMComm running } SMFileSystem::~SMFileSystem() { } -int SMFileSystem::mkdir(const char *path) +int SMFileSystem::mkdir(const char* path) { - return 0; + return 0; } -off64_t SMFileSystem::size(const char *filename) const +off64_t SMFileSystem::size(const char* filename) const { - struct stat _stat; - - SMComm *smComm = SMComm::get(); - int err = smComm->stat(filename, &_stat); - if (err) - return err; - - return _stat.st_size; -} + struct stat _stat; -off64_t SMFileSystem::compressedSize(const char *filename) const -{ - // Yikes, punting on this one. - throw NotImplementedYet(__func__); -} - -int SMFileSystem::remove(const char *filename) -{ - SMComm *comm = SMComm::get(); - return comm->unlink(filename); -} - -int SMFileSystem::rename(const char *oldFile, const char *newFile) -{ - int err = copyFile(oldFile, newFile); - if (err) - return err; - err = this->remove(oldFile); + SMComm* smComm = SMComm::get(); + int err = smComm->stat(filename, &_stat); + if (err) return err; + + return _stat.st_size; } -bool SMFileSystem::exists(const char *filename) const +off64_t SMFileSystem::compressedSize(const char* filename) const { - struct stat _stat; - SMComm *comm = SMComm::get(); - - int err = comm->stat(filename, &_stat); - return (err == 0); + // Yikes, punting on this one. + throw NotImplementedYet(__func__); +} + +int SMFileSystem::remove(const char* filename) +{ + SMComm* comm = SMComm::get(); + return comm->unlink(filename); +} + +int SMFileSystem::rename(const char* oldFile, const char* newFile) +{ + int err = copyFile(oldFile, newFile); + if (err) + return err; + err = this->remove(oldFile); + return err; +} + +bool SMFileSystem::exists(const char* filename) const +{ + struct stat _stat; + SMComm* comm = SMComm::get(); + + int err = comm->stat(filename, &_stat); + return (err == 0); } int SMFileSystem::listDirectory(const char* pathname, std::list& contents) const { - SMComm *comm = SMComm::get(); - return comm->listDirectory(pathname, &contents); + SMComm* comm = SMComm::get(); + return comm->listDirectory(pathname, &contents); } -bool SMFileSystem::isDir(const char *path) const +bool SMFileSystem::isDir(const char* path) const { - SMComm *comm = SMComm::get(); - struct stat _stat; - - int err = comm->stat(path, &_stat); - if (err != 0) - return false; // reasonable to throw here? todo, look at what the other classes do. - return (_stat.st_mode & S_IFDIR); + SMComm* comm = SMComm::get(); + struct stat _stat; + + int err = comm->stat(path, &_stat); + if (err != 0) + return false; // reasonable to throw here? todo, look at what the other classes do. + return (_stat.st_mode & S_IFDIR); } -int SMFileSystem::copyFile(const char *src, const char *dest) const +int SMFileSystem::copyFile(const char* src, const char* dest) const { - SMComm *comm = SMComm::get(); - return comm->copyFile(src, dest); + SMComm* comm = SMComm::get(); + return comm->copyFile(src, dest); } - + bool SMFileSystem::filesystemIsUp() const { - SMComm *comm = SMComm::get(); - return (comm->ping() == 0); + SMComm* comm = SMComm::get(); + return (comm->ping() == 0); } bool SMFileSystem::filesystemSync() const { - SMComm *comm = SMComm::get(); - return (comm->sync() == 0); -} + SMComm* comm = SMComm::get(); + return (comm->sync() == 0); } +} // namespace idbdatafile diff --git a/utils/cloudio/SMFileSystem.h b/utils/cloudio/SMFileSystem.h index 4b136a2fd..0e60f5330 100644 --- a/utils/cloudio/SMFileSystem.h +++ b/utils/cloudio/SMFileSystem.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef SMFILESYSTEM_H_ #define SMFILESYSTEM_H_ @@ -26,28 +25,26 @@ namespace idbdatafile { - class SMFileSystem : public IDBFileSystem, boost::noncopyable { - public: - SMFileSystem(); - virtual ~SMFileSystem(); - - // why are some of these const and some not const in IDBFileSystem? - int mkdir(const char* pathname); - off64_t size(const char* path) const; - off64_t compressedSize(const char* path) const; - int remove(const char* pathname); - int rename(const char* oldpath, const char* newpath); - bool exists(const char* pathname) const; - int listDirectory(const char* pathname, std::list& contents) const; - bool isDir(const char* pathname) const; - int copyFile(const char* srcPath, const char* destPath) const; - bool filesystemIsUp() const; - bool filesystemSync() const; + public: + SMFileSystem(); + virtual ~SMFileSystem(); + // why are some of these const and some not const in IDBFileSystem? + int mkdir(const char* pathname); + off64_t size(const char* path) const; + off64_t compressedSize(const char* path) const; + int remove(const char* pathname); + int rename(const char* oldpath, const char* newpath); + bool exists(const char* pathname) const; + int listDirectory(const char* pathname, std::list& contents) const; + bool isDir(const char* pathname) const; + int copyFile(const char* srcPath, const char* destPath) const; + bool filesystemIsUp() const; + bool filesystemSync() const; }; -} +} // namespace idbdatafile #endif diff --git a/utils/cloudio/SocketPool.cpp b/utils/cloudio/SocketPool.cpp index 91ac7509a..f461e39f7 100644 --- a/utils/cloudio/SocketPool.cpp +++ b/utils/cloudio/SocketPool.cpp @@ -27,248 +27,249 @@ using namespace std; namespace { - void log(logging::LOG_TYPE whichLogFile, const string& msg) { - logging::Logger logger(12); //12 = configcpp - logger.logMessage(whichLogFile, msg, logging::LoggingID(12)); + logging::Logger logger(12); // 12 = configcpp + logger.logMessage(whichLogFile, msg, logging::LoggingID(12)); } -} +} // namespace namespace idbdatafile { - SocketPool::SocketPool() { - config::Config *config = config::Config::makeConfig(); // the most 'config' ever put into a single line of code? - string stmp; - int64_t itmp = 0; - - try + config::Config* config = + config::Config::makeConfig(); // the most 'config' ever put into a single line of code? + string stmp; + int64_t itmp = 0; + + try + { + stmp = config->getConfig("StorageManager", "MaxSockets"); + itmp = strtol(stmp.c_str(), NULL, 10); + if (itmp > 500 || itmp < 1) { - stmp = config->getConfig("StorageManager", "MaxSockets"); - itmp = strtol(stmp.c_str(), NULL, 10); - if (itmp > 500 || itmp < 1) - { - string errmsg = "SocketPool(): Got a bad value '" + stmp + "' for StorageManager/MaxSockets. Range is 1-500."; - log(logging::LOG_TYPE_CRITICAL, errmsg); - throw runtime_error(errmsg); - } - maxSockets = itmp; - } - catch (exception &e) - { - ostringstream os; - os << "SocketPool(): Using default of " << defaultSockets << "."; - log(logging::LOG_TYPE_CRITICAL, os.str()); - maxSockets = defaultSockets; + string errmsg = + "SocketPool(): Got a bad value '" + stmp + "' for StorageManager/MaxSockets. Range is 1-500."; + log(logging::LOG_TYPE_CRITICAL, errmsg); + throw runtime_error(errmsg); } + maxSockets = itmp; + } + catch (exception& e) + { + ostringstream os; + os << "SocketPool(): Using default of " << defaultSockets << "."; + log(logging::LOG_TYPE_CRITICAL, os.str()); + maxSockets = defaultSockets; + } } SocketPool::~SocketPool() { - boost::mutex::scoped_lock lock(mutex); + boost::mutex::scoped_lock lock(mutex); - for (uint i = 0; i < allSockets.size(); i++) - ::close(allSockets[i]); + for (uint i = 0; i < allSockets.size(); i++) + ::close(allSockets[i]); } -#define sm_check_error \ - if (err < 0) \ - { \ - char _smbuf[80]; \ - int l_errno = errno; \ - log(logging::LOG_TYPE_ERROR, string("SocketPool: got a network error: ") + strerror_r(l_errno, _smbuf, 80)); \ - remoteClosed(sock); \ - return -1; \ - } - -int SocketPool::send_recv(messageqcpp::ByteStream &in, messageqcpp::ByteStream *out) +#define sm_check_error \ + if (err < 0) \ + { \ + char _smbuf[80]; \ + int l_errno = errno; \ + log(logging::LOG_TYPE_ERROR, \ + string("SocketPool: got a network error: ") + strerror_r(l_errno, _smbuf, 80)); \ + remoteClosed(sock); \ + return -1; \ + } + +int SocketPool::send_recv(messageqcpp::ByteStream& in, messageqcpp::ByteStream* out) { - uint count = 0; - uint length = in.length(); - int sock = -1; - const uint8_t *inbuf = in.buf(); - ssize_t err = 0; - + uint count = 0; + uint length = in.length(); + int sock = -1; + const uint8_t* inbuf = in.buf(); + ssize_t err = 0; + retry: - int retries = 0; - while (sock < 0) + int retries = 0; + while (sock < 0) + { + sock = getSocket(); + if (sock < 0) { - sock = getSocket(); - if (sock < 0) - { - if (++retries < 10) - { - //log(logging::LOG_TYPE_ERROR, "SocketPool::send_recv(): retrying in 5 sec..."); - sleep(1); - } - else - { - errno = ECONNREFUSED; - return -1; - } - } - } - - storagemanager::sm_msg_header hdr; - hdr.type = storagemanager::SM_MSG_START; - hdr.payloadLen = length; - hdr.flags = 0; - //cout << "SP sending msg on sock " << sock << " with length = " << length << endl; - err = ::write(sock, &hdr, sizeof(hdr)); - if (err < 0 && errno == EPIPE) - { - log(logging::LOG_TYPE_WARNING, "SocketPool: remote connection is closed, getting a new one"); - remoteClosed(sock); - sock = -1; - goto retry; + if (++retries < 10) + { + // log(logging::LOG_TYPE_ERROR, "SocketPool::send_recv(): retrying in 5 sec..."); + sleep(1); + } + else + { + errno = ECONNREFUSED; + return -1; + } } + } + + storagemanager::sm_msg_header hdr; + hdr.type = storagemanager::SM_MSG_START; + hdr.payloadLen = length; + hdr.flags = 0; + // cout << "SP sending msg on sock " << sock << " with length = " << length << endl; + err = ::write(sock, &hdr, sizeof(hdr)); + if (err < 0 && errno == EPIPE) + { + log(logging::LOG_TYPE_WARNING, "SocketPool: remote connection is closed, getting a new one"); + remoteClosed(sock); + sock = -1; + goto retry; + } + sm_check_error; + while (count < length) + { + err = ::write(sock, &inbuf[count], length - count); sm_check_error; - while (count < length) + count += err; + in.advance(err); + } + // cout << "SP sent msg with length = " << length << endl; + + out->restart(); + uint8_t* outbuf; + uint8_t window[8192]; + uint remainingBytes = 0; + uint i; + storagemanager::sm_msg_header* resp = NULL; + + while (1) + { + // cout << "SP receiving msg on sock " << sock << endl; + // here remainingBytes means the # of bytes from the previous message + err = ::read(sock, &window[remainingBytes], 8192 - remainingBytes); + sm_check_error; + if (err == 0) { - err = ::write(sock, &inbuf[count], length-count); - sm_check_error; - count += err; - in.advance(err); + remoteClosed(sock); + // TODO, a retry loop + return -1; } - //cout << "SP sent msg with length = " << length << endl; - - out->restart(); - uint8_t *outbuf; - uint8_t window[8192]; - uint remainingBytes = 0; - uint i; - storagemanager::sm_msg_header *resp = NULL; - - while (1) + uint endOfData = remainingBytes + err; // for clarity + + // scan for the 8-byte header. If it is fragmented, move the fragment to the front of the buffer + // for the next iteration to handle. + + if (endOfData < storagemanager::SM_HEADER_LEN) { - //cout << "SP receiving msg on sock " << sock << endl; - // here remainingBytes means the # of bytes from the previous message - err = ::read(sock, &window[remainingBytes], 8192 - remainingBytes); - sm_check_error; - if (err == 0) - { - remoteClosed(sock); - // TODO, a retry loop - return -1; - } - uint endOfData = remainingBytes + err; // for clarity - - // scan for the 8-byte header. If it is fragmented, move the fragment to the front of the buffer - // for the next iteration to handle. - - if (endOfData < storagemanager::SM_HEADER_LEN) - { - remainingBytes = endOfData; - continue; - } - - - for (i = 0; i <= endOfData - storagemanager::SM_HEADER_LEN; i++) - { - if (*((uint *) &window[i]) == storagemanager::SM_MSG_START) - { - resp = (storagemanager::sm_msg_header *) &window[i]; - break; - } - } - - if (resp == NULL) // didn't find the header yet - { - remainingBytes = endOfData - i; - memmove(window, &window[i], remainingBytes); - } - else - { - // i == first byte of the header here - // copy the payload fragment we got into the output bytestream - uint startOfPayload = i + storagemanager::SM_HEADER_LEN; // for clarity - out->needAtLeast(resp->payloadLen); - outbuf = out->getInputPtr(); - if (resp->payloadLen < endOfData - startOfPayload) - cout << "SocketPool: warning! Probably got a bad length field! payload length = " << resp->payloadLen << - " endOfData = " << endOfData << " startOfPayload = " << startOfPayload << endl; - memcpy(outbuf, &window[startOfPayload], endOfData - startOfPayload); - remainingBytes = resp->payloadLen - (endOfData - startOfPayload); // remainingBytes is now the # of bytes left to read - out->advanceInputPtr(endOfData - startOfPayload); - break; // done looking for the header, can fill the output buffer directly now. - } + remainingBytes = endOfData; + continue; } - - // read the rest of the payload directly into the output bytestream - while (remainingBytes > 0) + + for (i = 0; i <= endOfData - storagemanager::SM_HEADER_LEN; i++) { - err = ::read(sock, &outbuf[resp->payloadLen - remainingBytes], remainingBytes); - sm_check_error; - remainingBytes -= err; - out->advanceInputPtr(err); + if (*((uint*)&window[i]) == storagemanager::SM_MSG_START) + { + resp = (storagemanager::sm_msg_header*)&window[i]; + break; + } } - returnSocket(sock); - return 0; + + if (resp == NULL) // didn't find the header yet + { + remainingBytes = endOfData - i; + memmove(window, &window[i], remainingBytes); + } + else + { + // i == first byte of the header here + // copy the payload fragment we got into the output bytestream + uint startOfPayload = i + storagemanager::SM_HEADER_LEN; // for clarity + out->needAtLeast(resp->payloadLen); + outbuf = out->getInputPtr(); + if (resp->payloadLen < endOfData - startOfPayload) + cout << "SocketPool: warning! Probably got a bad length field! payload length = " + << resp->payloadLen << " endOfData = " << endOfData << " startOfPayload = " << startOfPayload + << endl; + memcpy(outbuf, &window[startOfPayload], endOfData - startOfPayload); + remainingBytes = resp->payloadLen - + (endOfData - startOfPayload); // remainingBytes is now the # of bytes left to read + out->advanceInputPtr(endOfData - startOfPayload); + break; // done looking for the header, can fill the output buffer directly now. + } + } + + // read the rest of the payload directly into the output bytestream + while (remainingBytes > 0) + { + err = ::read(sock, &outbuf[resp->payloadLen - remainingBytes], remainingBytes); + sm_check_error; + remainingBytes -= err; + out->advanceInputPtr(err); + } + returnSocket(sock); + return 0; } int SocketPool::getSocket() { - boost::mutex::scoped_lock lock(mutex); - int clientSocket; - - if (freeSockets.size() == 0 && allSockets.size() < maxSockets) + boost::mutex::scoped_lock lock(mutex); + int clientSocket; + + if (freeSockets.size() == 0 && allSockets.size() < maxSockets) + { + // make a new connection + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null... + clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); + int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr)); + if (err >= 0) + allSockets.push_back(clientSocket); + else { - // make a new connection - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null... - clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0); - int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr)); - if (err >= 0) - allSockets.push_back(clientSocket); - else - { - int saved_errno = errno; - ostringstream os; - char buf[80]; - os << "SocketPool::getSocket() failed to connect; got '" << strerror_r(saved_errno, buf, 80) << "'"; - cout << os.str() << endl; - log(logging::LOG_TYPE_ERROR, os.str()); - close(clientSocket); - errno = saved_errno; - return -1; - } - return clientSocket; + int saved_errno = errno; + ostringstream os; + char buf[80]; + os << "SocketPool::getSocket() failed to connect; got '" << strerror_r(saved_errno, buf, 80) << "'"; + cout << os.str() << endl; + log(logging::LOG_TYPE_ERROR, os.str()); + close(clientSocket); + errno = saved_errno; + return -1; } - - // wait for a socket to become free - while (freeSockets.size() == 0) - socketAvailable.wait(lock); - - clientSocket = freeSockets.front(); - freeSockets.pop_front(); return clientSocket; + } + + // wait for a socket to become free + while (freeSockets.size() == 0) + socketAvailable.wait(lock); + + clientSocket = freeSockets.front(); + freeSockets.pop_front(); + return clientSocket; } - + void SocketPool::returnSocket(const int sock) { - boost::mutex::scoped_lock lock(mutex); - //cout << "returning socket " << sock << endl; - freeSockets.push_back(sock); - socketAvailable.notify_one(); + boost::mutex::scoped_lock lock(mutex); + // cout << "returning socket " << sock << endl; + freeSockets.push_back(sock); + socketAvailable.notify_one(); } void SocketPool::remoteClosed(const int sock) { - boost::mutex::scoped_lock lock(mutex); - //cout << "closing socket " << sock << endl; - ::close(sock); - for (vector::iterator i = allSockets.begin(); i != allSockets.end(); ++i) - if (*i == sock) - { - allSockets.erase(i, i+1); - break; - } + boost::mutex::scoped_lock lock(mutex); + // cout << "closing socket " << sock << endl; + ::close(sock); + for (vector::iterator i = allSockets.begin(); i != allSockets.end(); ++i) + if (*i == sock) + { + allSockets.erase(i, i + 1); + break; + } } - -} +} // namespace idbdatafile diff --git a/utils/cloudio/SocketPool.h b/utils/cloudio/SocketPool.h index 79d34011e..7e1bc9400 100644 --- a/utils/cloudio/SocketPool.h +++ b/utils/cloudio/SocketPool.h @@ -26,34 +26,31 @@ namespace idbdatafile { - /* This should be renamed; it's more than a pool, it also does the low-level communication. TBD. */ class SocketPool : public boost::noncopyable { - public: - SocketPool(); - - // the dtor will immediately close all sockets - virtual ~SocketPool(); - - // 0 = success, -1 = failure. Should this throw instead? - int send_recv(messageqcpp::ByteStream &to_send, messageqcpp::ByteStream *to_recv); + public: + SocketPool(); - private: - int getSocket(); - void returnSocket(const int sock); - void remoteClosed(const int sock); - - std::vector allSockets; - std::deque freeSockets; - boost::mutex mutex; - boost::condition_variable socketAvailable; - uint maxSockets; - static const uint defaultSockets = 20; + // the dtor will immediately close all sockets + virtual ~SocketPool(); + + // 0 = success, -1 = failure. Should this throw instead? + int send_recv(messageqcpp::ByteStream& to_send, messageqcpp::ByteStream* to_recv); + + private: + int getSocket(); + void returnSocket(const int sock); + void remoteClosed(const int sock); + + std::vector allSockets; + std::deque freeSockets; + boost::mutex mutex; + boost::condition_variable socketAvailable; + uint maxSockets; + static const uint defaultSockets = 20; }; -} - - +} // namespace idbdatafile #endif diff --git a/utils/cloudio/cloud_plugin.cpp b/utils/cloudio/cloud_plugin.cpp index 4236608d6..1e39a8dc5 100644 --- a/utils/cloudio/cloud_plugin.cpp +++ b/utils/cloudio/cloud_plugin.cpp @@ -22,18 +22,18 @@ using namespace idbdatafile; #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" #endif extern "C" { - FileFactoryEnt plugin_instance() - { - return FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem()); - } + FileFactoryEnt plugin_instance() + { + return FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem()); + } } #ifdef __clang__ - #pragma clang diagnostic pop +#pragma clang diagnostic pop #endif diff --git a/utils/cloudio/component_test.cpp b/utils/cloudio/component_test.cpp index 750f15206..4f69ecdbc 100644 --- a/utils/cloudio/component_test.cpp +++ b/utils/cloudio/component_test.cpp @@ -14,9 +14,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - - + #include #include #include @@ -44,218 +42,224 @@ using namespace std; volatile bool die = false; int errCode = 0; - + /* Right now this will just accept one connection & send a fixed error response */ void error_server_thread() { - struct sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); - - int server_socket = ::socket(AF_UNIX, SOCK_STREAM, 0); - assert(server_socket >= 0); - - int err = ::bind(server_socket, (struct sockaddr *) &addr, sizeof(addr)); - assert(err == 0); - - err = ::listen(server_socket, 1); - assert(err == 0); - - socklen_t addrlen; - struct sockaddr_un client_addr; - memset(&client_addr, 0, sizeof(client_addr)); - int client_socket = ::accept(server_socket, (struct sockaddr *) &client_addr, &addrlen); - assert(client_socket >= 0); - - //cout << "server thread got a connection" << endl; - - uint8_t buf[4096]; - uint32_t response[4] = { storagemanager::SM_MSG_START, 8, (uint32_t ) -1, EINVAL }; - uint remainingBytes = 0; - - while (!die) + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); + + int server_socket = ::socket(AF_UNIX, SOCK_STREAM, 0); + assert(server_socket >= 0); + + int err = ::bind(server_socket, (struct sockaddr*)&addr, sizeof(addr)); + assert(err == 0); + + err = ::listen(server_socket, 1); + assert(err == 0); + + socklen_t addrlen; + struct sockaddr_un client_addr; + memset(&client_addr, 0, sizeof(client_addr)); + int client_socket = ::accept(server_socket, (struct sockaddr*)&client_addr, &addrlen); + assert(client_socket >= 0); + + // cout << "server thread got a connection" << endl; + + uint8_t buf[4096]; + uint32_t response[4] = {storagemanager::SM_MSG_START, 8, (uint32_t)-1, EINVAL}; + uint remainingBytes = 0; + + while (!die) + { + /* This just scans for SM_MSG_START, and as it finds them it sends a generic error + response. */ + + err = ::recv(client_socket, &buf[remainingBytes], 4096 - remainingBytes, MSG_DONTWAIT); + if (err < 0) { - /* This just scans for SM_MSG_START, and as it finds them it sends a generic error - response. */ - - err = ::recv(client_socket, &buf[remainingBytes], 4096 - remainingBytes, MSG_DONTWAIT); - if (err < 0) - { - if (errno == EAGAIN || errno == EWOULDBLOCK) - { - //cout << "server looping..." << endl; - sleep(1); // who cares - continue; - } - else { - //char errbuf[80]; - //cout << "server thread got an error: " << strerror_r(errno, errbuf, 80) << endl; - close(client_socket); - errCode = -1; - die = true; - return; - } - } - - //cout << "server thread got some data" << endl; - uint endOfData = remainingBytes + err; - uint i; - if (endOfData < 4) - { - remainingBytes = endOfData; - continue; - } - for (i = 0; i <= endOfData - 4; i++) - { - if (*((uint32_t *) &buf[i]) == storagemanager::SM_MSG_START) { - //cout << "server thread found a msg start magic" << endl; - err = ::send(client_socket, response, 16, 0); - assert(err > 0); - } - } - memmove(buf, &buf[i], endOfData - i); // should be the trailing 3 bytes of the data - remainingBytes = endOfData - i; + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + // cout << "server looping..." << endl; + sleep(1); // who cares + continue; + } + else + { + // char errbuf[80]; + // cout << "server thread got an error: " << strerror_r(errno, errbuf, 80) << endl; + close(client_socket); + errCode = -1; + die = true; + return; + } } -} - + + // cout << "server thread got some data" << endl; + uint endOfData = remainingBytes + err; + uint i; + if (endOfData < 4) + { + remainingBytes = endOfData; + continue; + } + for (i = 0; i <= endOfData - 4; i++) + { + if (*((uint32_t*)&buf[i]) == storagemanager::SM_MSG_START) + { + // cout << "server thread found a msg start magic" << endl; + err = ::send(client_socket, response, 16, 0); + assert(err > 0); + } + } + memmove(buf, &buf[i], endOfData - i); // should be the trailing 3 bytes of the data + remainingBytes = endOfData - i; + } +} + int test1() { - /* Start a server thread that returns generic errors */ - boost::thread server_thread(error_server_thread); - - /* Instantiate each of SM subclasses, call each function, and verify the expected error response */ - int err; - SMFileFactory factory; - - cout << "open" << endl; - IDBDataFile *file = factory.open("dummy", "r", 0, 0); - assert(file == NULL && errno == EINVAL && !die); - - SMFileSystem filesystem; - - cout << "compressedSize" << endl; - bool gotException = false; - try { - filesystem.compressedSize("dummy"); - } - catch (NotImplementedYet &) { - gotException = true; - } - assert(gotException && !die); - - cout << "copyFile" << endl; - try { - filesystem.copyFile("dummy1", "dummy2"); - } - catch (NotImplementedYet &) { - gotException = true; - } - assert(gotException && !die); - - cout << "rename" << endl; - try { - filesystem.rename("dummy1", "dummy2"); - } - catch (NotImplementedYet &) { - gotException = true; - } - assert(gotException && !die); - - cout << "exists" << endl; - err = filesystem.exists("dummy"); - assert(!err && errno == EINVAL); - - cout << "filesystemisup" << endl; - err = filesystem.filesystemIsUp(); - assert(!err && errno == EINVAL && !die); - - cout << "isdir" << endl; - err = filesystem.isDir("dummy"); - assert(!err && errno == EINVAL && !die); - - cout << "listdirectory" << endl; - list filenames; - err = filesystem.listDirectory("dummy", filenames); - assert(err == -1 && errno == EINVAL && filenames.empty() && !die); - - cout << "remove" << endl; - err = filesystem.remove("dummy"); - assert(err == -1 && errno == EINVAL && !die); - - cout << "size" << endl; - err = filesystem.size("dummy"); - assert(err == -1 && errno == EINVAL && !die); - - cout << "mkdir" << endl; - err = filesystem.mkdir("dummy"); - assert(err == 0 && !die); - - cout << "datafile constructor" << endl; - SMDataFile f("dummy", O_RDONLY, 12345); - f = SMDataFile("dummy2", O_WRONLY, 123456); - f = SMDataFile("dummy2", O_RDWR | O_APPEND, 1234567); - - cout << "pread" << endl; - uint8_t buf[1024]; - err = f.pread(buf, 0, 1024); - assert(err == -1 && errno == EINVAL && !die); - - cout << "read" << endl; - err = f.read(buf, 1024); - assert(err == -1 && errno == EINVAL && !die); - - cout << "write" << endl; - err = f.write(buf, 1024); - assert(err == -1 && errno == EINVAL && !die); - - cout << "seek" << endl; - err = f.seek(1234, SEEK_SET); - assert(err == 0 && !die); - - err = f.seek(1234, SEEK_CUR); - assert(err == 0 && !die); - - err = f.seek(1234, SEEK_END); - assert(err == -1 && errno == EINVAL && !die); - - cout << "truncate" << endl; - err = f.truncate(1234); - assert(err == -1 && errno == EINVAL && !die); - - cout << "size" << endl; - err = f.size(); - assert(err == -1 && errno == EINVAL && !die); - - cout << "tell" << endl; - err = f.tell(); - assert(err == 2468); - - cout << "flush" << endl; - err = f.flush(); - assert(err == 0); - - cout << "mtime" << endl; - err = f.mtime(); - assert(err == -1 && errno == EINVAL && !die); - - cout << "close" << endl; - err = f.close(); - assert(err == 0); - - - // done, return errCode - die = true; - cout << "done, waiting for server thread to stop" << endl; - server_thread.join(); - return errCode; + /* Start a server thread that returns generic errors */ + boost::thread server_thread(error_server_thread); + + /* Instantiate each of SM subclasses, call each function, and verify the expected error response */ + int err; + SMFileFactory factory; + + cout << "open" << endl; + IDBDataFile* file = factory.open("dummy", "r", 0, 0); + assert(file == NULL && errno == EINVAL && !die); + + SMFileSystem filesystem; + + cout << "compressedSize" << endl; + bool gotException = false; + try + { + filesystem.compressedSize("dummy"); + } + catch (NotImplementedYet&) + { + gotException = true; + } + assert(gotException && !die); + + cout << "copyFile" << endl; + try + { + filesystem.copyFile("dummy1", "dummy2"); + } + catch (NotImplementedYet&) + { + gotException = true; + } + assert(gotException && !die); + + cout << "rename" << endl; + try + { + filesystem.rename("dummy1", "dummy2"); + } + catch (NotImplementedYet&) + { + gotException = true; + } + assert(gotException && !die); + + cout << "exists" << endl; + err = filesystem.exists("dummy"); + assert(!err && errno == EINVAL); + + cout << "filesystemisup" << endl; + err = filesystem.filesystemIsUp(); + assert(!err && errno == EINVAL && !die); + + cout << "isdir" << endl; + err = filesystem.isDir("dummy"); + assert(!err && errno == EINVAL && !die); + + cout << "listdirectory" << endl; + list filenames; + err = filesystem.listDirectory("dummy", filenames); + assert(err == -1 && errno == EINVAL && filenames.empty() && !die); + + cout << "remove" << endl; + err = filesystem.remove("dummy"); + assert(err == -1 && errno == EINVAL && !die); + + cout << "size" << endl; + err = filesystem.size("dummy"); + assert(err == -1 && errno == EINVAL && !die); + + cout << "mkdir" << endl; + err = filesystem.mkdir("dummy"); + assert(err == 0 && !die); + + cout << "datafile constructor" << endl; + SMDataFile f("dummy", O_RDONLY, 12345); + f = SMDataFile("dummy2", O_WRONLY, 123456); + f = SMDataFile("dummy2", O_RDWR | O_APPEND, 1234567); + + cout << "pread" << endl; + uint8_t buf[1024]; + err = f.pread(buf, 0, 1024); + assert(err == -1 && errno == EINVAL && !die); + + cout << "read" << endl; + err = f.read(buf, 1024); + assert(err == -1 && errno == EINVAL && !die); + + cout << "write" << endl; + err = f.write(buf, 1024); + assert(err == -1 && errno == EINVAL && !die); + + cout << "seek" << endl; + err = f.seek(1234, SEEK_SET); + assert(err == 0 && !die); + + err = f.seek(1234, SEEK_CUR); + assert(err == 0 && !die); + + err = f.seek(1234, SEEK_END); + assert(err == -1 && errno == EINVAL && !die); + + cout << "truncate" << endl; + err = f.truncate(1234); + assert(err == -1 && errno == EINVAL && !die); + + cout << "size" << endl; + err = f.size(); + assert(err == -1 && errno == EINVAL && !die); + + cout << "tell" << endl; + err = f.tell(); + assert(err == 2468); + + cout << "flush" << endl; + err = f.flush(); + assert(err == 0); + + cout << "mtime" << endl; + err = f.mtime(); + assert(err == -1 && errno == EINVAL && !die); + + cout << "close" << endl; + err = f.close(); + assert(err == 0); + + // done, return errCode + die = true; + cout << "done, waiting for server thread to stop" << endl; + server_thread.join(); + return errCode; } int main() { - int ret; - - ret = test1(); - return ret; + int ret; + + ret = test1(); + return ret; } - diff --git a/utils/cloudio/end_to_end_test.cpp b/utils/cloudio/end_to_end_test.cpp old mode 100755 new mode 100644 index 5a063a077..76bdab0c4 --- a/utils/cloudio/end_to_end_test.cpp +++ b/utils/cloudio/end_to_end_test.cpp @@ -14,9 +14,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - - - + #include #include #include @@ -47,216 +45,217 @@ int errCode = 0; void printResultPASS(int result) { - if (result == 0) - cout << " PASS" << endl; - else - cout << " FAIL" << endl; + if (result == 0) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; } void printResultFAIL(int result) { - if (result == 0) - cout << " FAIL" << endl; - else - cout << " PASS" << endl; + if (result == 0) + cout << " FAIL" << endl; + else + cout << " PASS" << endl; } void printResultPASS(bool result) { - if (result) - cout << " PASS" << endl; - else - cout << " FAIL" << endl; + if (result) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; } void printResultFAIL(bool result) { - if (result) - cout << " FAIL" << endl; - else - cout << " PASS" << endl; + if (result) + cout << " FAIL" << endl; + else + cout << " PASS" << endl; } void printResult(int result, int expected) { - if (result == expected) - cout << " PASS" << endl; - else - cout << " FAIL" << endl; + if (result == expected) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; } int testAll() { - /* Instantiate each of SM subclasses, call each function, and verify the expected error response */ - int err; - SMFileFactory factory; - list filenames; - SMFileSystem filesystem; - - cout << "compressedSize --- "; - bool gotException = false; - try { - filesystem.compressedSize("./testData/dummy"); - } - catch (NotImplementedYet &) { - gotException = true; - } - assert(gotException && !die); - cout << " N/A" << endl; + /* Instantiate each of SM subclasses, call each function, and verify the expected error response */ + int err; + SMFileFactory factory; + list filenames; + SMFileSystem filesystem; - cout << "copyFile dummy1->dummy2 --- "; - err = filesystem.copyFile("./testData/dummy1", "./testData/dummy2"); - printResultFAIL(err); + cout << "compressedSize --- "; + bool gotException = false; + try + { + filesystem.compressedSize("./testData/dummy"); + } + catch (NotImplementedYet&) + { + gotException = true; + } + assert(gotException && !die); + cout << " N/A" << endl; - cout << "copyFile end_to_end_test_file.start->readTest --- "; - err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/readTest"); - printResultPASS(err); + cout << "copyFile dummy1->dummy2 --- "; + err = filesystem.copyFile("./testData/dummy1", "./testData/dummy2"); + printResultFAIL(err); - cout << "copyFile end_to_end_test_file.start->dummy --- "; - err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/dummy"); - printResultPASS(err); - - cout << "rename dummy->dummyrename --- "; - err = filesystem.rename("./testData/dummy", "./testData/dummyrename"); - printResultPASS(err); + cout << "copyFile end_to_end_test_file.start->readTest --- "; + err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/readTest"); + printResultPASS(err); - cout << "exists dummyrename --- "; - bool exists = filesystem.exists("./testData/dummyrename"); - printResultPASS(exists); - cout << "exists DNE --- "; - bool doesNotExist = filesystem.exists("./testData/DNE"); - printResultFAIL(doesNotExist); - - cout << "filesystemisup --- "; - bool fsIsUp = filesystem.filesystemIsUp(); - printResultPASS(fsIsUp); - - cout << "isdir testData --- "; - bool isDir = filesystem.isDir("testData"); - printResultPASS(isDir); + cout << "copyFile end_to_end_test_file.start->dummy --- "; + err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/dummy"); + printResultPASS(err); - cout << "isdir readTest --- "; - bool isNotDir = filesystem.isDir("./testData/readTest"); - printResultFAIL(isNotDir); + cout << "rename dummy->dummyrename --- "; + err = filesystem.rename("./testData/dummy", "./testData/dummyrename"); + printResultPASS(err); - cout << "isdir DNE --- "; - isNotDir = filesystem.isDir("./testData/DNE"); - printResultFAIL(isNotDir); + cout << "exists dummyrename --- "; + bool exists = filesystem.exists("./testData/dummyrename"); + printResultPASS(exists); + cout << "exists DNE --- "; + bool doesNotExist = filesystem.exists("./testData/DNE"); + printResultFAIL(doesNotExist); - cout << "listdirectory testData --- "; - err = filesystem.listDirectory("testData", filenames); - printResultPASS(err); - - cout << "listdirectory dummyrename --- "; - err = filesystem.listDirectory("./testData/dummyrename", filenames); - printResultFAIL(err); + cout << "filesystemisup --- "; + bool fsIsUp = filesystem.filesystemIsUp(); + printResultPASS(fsIsUp); - cout << "listdirectory DNE --- "; - err = filesystem.listDirectory("./testData/DNE", filenames); - printResultFAIL(err); + cout << "isdir testData --- "; + bool isDir = filesystem.isDir("testData"); + printResultPASS(isDir); - cout << "remove dummyrename --- "; - err = filesystem.remove("./testData/dummyrename"); - printResultPASS(err); + cout << "isdir readTest --- "; + bool isNotDir = filesystem.isDir("./testData/readTest"); + printResultFAIL(isNotDir); - cout << "size readTest --- "; - err = filesystem.size("./testData/readTest"); - printResult(err,10940); + cout << "isdir DNE --- "; + isNotDir = filesystem.isDir("./testData/DNE"); + printResultFAIL(isNotDir); - cout << "size DNE --- "; - err = filesystem.size("./testData/DNE"); - printResultFAIL(err); + cout << "listdirectory testData --- "; + err = filesystem.listDirectory("testData", filenames); + printResultPASS(err); - cout << "open readTest r --- "; - IDBDataFile *fileR = factory.open("./testData/readTest", "r", 0, 0); - if (fileR) - cout << " PASS" << endl; - else - cout << " FAIL" << endl; + cout << "listdirectory dummyrename --- "; + err = filesystem.listDirectory("./testData/dummyrename", filenames); + printResultFAIL(err); - cout << "open readTest a --- "; - IDBDataFile *fileW = factory.open("./testData/readTest", "a", 0, 0); - if (fileW) - cout << " PASS" << endl; - else - cout << " FAIL" << endl; + cout << "listdirectory DNE --- "; + err = filesystem.listDirectory("./testData/DNE", filenames); + printResultFAIL(err); - cout << "open DNE --- "; - IDBDataFile *file2 = factory.open("./testData/DNE", "r", 0, 0); - if (file2) - cout << " FAIL" << endl; - else - cout << " PASS" << endl; + cout << "remove dummyrename --- "; + err = filesystem.remove("./testData/dummyrename"); + printResultPASS(err); - cout << "pread --- "; - uint8_t buf[2048]; - err = fileR->pread(buf, 0, 1024); - printResult(err,1024); + cout << "size readTest --- "; + err = filesystem.size("./testData/readTest"); + printResult(err, 10940); - cout << "read --- "; - err = fileR->read(buf, 1024); - printResult(err,1024); + cout << "size DNE --- "; + err = filesystem.size("./testData/DNE"); + printResultFAIL(err); - int newSize = 10940 + 1024; - cout << "write --- "; - err = fileW->write(buf, 1024); - printResult(err,1024); - cout << "size "<< newSize << " --- "; - err = fileW->size(); - cout << err; - printResult(err,newSize); + cout << "open readTest r --- "; + IDBDataFile* fileR = factory.open("./testData/readTest", "r", 0, 0); + if (fileR) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; - cout << "seek SEEK_SET --- "; - err = fileR->seek(1234, SEEK_SET); - printResultPASS(err); - cout << "tell 1234 --- "; - err = fileR->tell(); - cout << err; - printResult(err,1234); - cout << "seek SEEK_CUR --- "; - err = fileR->seek(1234, SEEK_CUR); - printResultPASS(err); - cout << "tell 2468 --- "; - err = fileR->tell(); - cout << err; - printResult(err,2468); - cout << "seek SEEK_END --- "; - err = fileR->seek(1234, SEEK_END); - printResultPASS(err); - cout << "tell 12174 --- "; - err = fileR->tell(); - cout << err; - printResult(err,13198); + cout << "open readTest a --- "; + IDBDataFile* fileW = factory.open("./testData/readTest", "a", 0, 0); + if (fileW) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; - cout << "truncate 1234 --- "; - err = fileR->truncate(1234); - printResultPASS(err); + cout << "open DNE --- "; + IDBDataFile* file2 = factory.open("./testData/DNE", "r", 0, 0); + if (file2) + cout << " FAIL" << endl; + else + cout << " PASS" << endl; - cout << "size 1234 --- "; - err = fileR->size(); - printResult(err,1234); + cout << "pread --- "; + uint8_t buf[2048]; + err = fileR->pread(buf, 0, 1024); + printResult(err, 1024); - cout << "mtime --- "; - err = fileR->mtime(); - if (err > 0) - cout << " PASS" << endl; - else - cout << " FAIL" << endl; + cout << "read --- "; + err = fileR->read(buf, 1024); + printResult(err, 1024); - cout << "remove readTest --- "; - err = filesystem.remove("./testData/readTest"); - printResultPASS(err); + int newSize = 10940 + 1024; + cout << "write --- "; + err = fileW->write(buf, 1024); + printResult(err, 1024); + cout << "size " << newSize << " --- "; + err = fileW->size(); + cout << err; + printResult(err, newSize); - cout << "done." << endl; - return errCode; + cout << "seek SEEK_SET --- "; + err = fileR->seek(1234, SEEK_SET); + printResultPASS(err); + cout << "tell 1234 --- "; + err = fileR->tell(); + cout << err; + printResult(err, 1234); + cout << "seek SEEK_CUR --- "; + err = fileR->seek(1234, SEEK_CUR); + printResultPASS(err); + cout << "tell 2468 --- "; + err = fileR->tell(); + cout << err; + printResult(err, 2468); + cout << "seek SEEK_END --- "; + err = fileR->seek(1234, SEEK_END); + printResultPASS(err); + cout << "tell 12174 --- "; + err = fileR->tell(); + cout << err; + printResult(err, 13198); + + cout << "truncate 1234 --- "; + err = fileR->truncate(1234); + printResultPASS(err); + + cout << "size 1234 --- "; + err = fileR->size(); + printResult(err, 1234); + + cout << "mtime --- "; + err = fileR->mtime(); + if (err > 0) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; + + cout << "remove readTest --- "; + err = filesystem.remove("./testData/readTest"); + printResultPASS(err); + + cout << "done." << endl; + return errCode; } int main() { - int ret; - - ret = testAll(); - //ret = testTest(); - return ret; + int ret; + + ret = testAll(); + // ret = testTest(); + return ret; } - diff --git a/utils/cloudio/sm_exceptions.h b/utils/cloudio/sm_exceptions.h index 206ce0c37..057c80bc5 100644 --- a/utils/cloudio/sm_exceptions.h +++ b/utils/cloudio/sm_exceptions.h @@ -22,18 +22,15 @@ namespace idbdatafile { - class NotImplementedYet : public std::logic_error { - public: - NotImplementedYet(const std::string &s); + public: + NotImplementedYet(const std::string& s); }; - - -NotImplementedYet::NotImplementedYet(const std::string &s) : - std::logic_error(s) + +NotImplementedYet::NotImplementedYet(const std::string& s) : std::logic_error(s) { } -} +} // namespace idbdatafile #endif diff --git a/utils/common/MonitorProcMem.cpp b/utils/common/MonitorProcMem.cpp index 320effcc9..1341dabbe 100644 --- a/utils/common/MonitorProcMem.cpp +++ b/utils/common/MonitorProcMem.cpp @@ -46,10 +46,9 @@ using namespace logging; namespace utils { - uint64_t MonitorProcMem::fMemTotal = 1; uint64_t MonitorProcMem::fMemFree = 0; -int MonitorProcMem::fMemPctCheck = 0; +int MonitorProcMem::fMemPctCheck = 0; //------------------------------------------------------------------------------ // Thread entry point function, that drives a thread to periodically (every @@ -58,31 +57,31 @@ int MonitorProcMem::fMemPctCheck = 0; //------------------------------------------------------------------------------ void MonitorProcMem::operator()() const { - utils::setThreadName("MonitorProcMem"); - while (1) + utils::setThreadName("MonitorProcMem"); + while (1) + { + if (fMaxPct > 0) { - if (fMaxPct > 0) - { - size_t pct = rss() * 100 / fMemTotal; + size_t pct = rss() * 100 / fMemTotal; - if (pct > fMaxPct) - { - cerr << "PrimProc: Too much memory allocated!" << endl; + if (pct > fMaxPct) + { + cerr << "PrimProc: Too much memory allocated!" << endl; - LoggingID logid(fSubsystemID); - logging::Message msg(logging::M0045); - logging::Message::Args args; - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_CRITICAL, msg, logid); - exit(1); - } - } - - fMemFree = cg.getFreeMemory(); - //calculateFreeMem(); - pause_(); + LoggingID logid(fSubsystemID); + logging::Message msg(logging::M0045); + logging::Message::Args args; + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_CRITICAL, msg, logid); + exit(1); + } } + + fMemFree = cg.getFreeMemory(); + // calculateFreeMem(); + pause_(); + } } //------------------------------------------------------------------------------ @@ -91,7 +90,7 @@ void MonitorProcMem::operator()() const //------------------------------------------------------------------------------ size_t MonitorProcMem::memTotal() const { - return cg.getTotalMemory(); + return cg.getTotalMemory(); } //------------------------------------------------------------------------------ @@ -100,83 +99,81 @@ size_t MonitorProcMem::memTotal() const //------------------------------------------------------------------------------ size_t MonitorProcMem::rss() const { - uint64_t rss; + uint64_t rss; #if defined(_MSC_VER) - HANDLE hProcess; - PROCESS_MEMORY_COUNTERS pmc; - hProcess = OpenProcess( PROCESS_QUERY_INFORMATION | - PROCESS_VM_READ, - FALSE, fPid ); + HANDLE hProcess; + PROCESS_MEMORY_COUNTERS pmc; + hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, fPid); - if (NULL == hProcess) - return 0; + if (NULL == hProcess) + return 0; - if ( GetProcessMemoryInfo( hProcess, &pmc, sizeof(pmc)) ) - rss = pmc.WorkingSetSize; - else - rss = 0; + if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) + rss = pmc.WorkingSetSize; + else + rss = 0; - CloseHandle( hProcess ); + CloseHandle(hProcess); #elif defined(__FreeBSD__) - ostringstream cmd; - cmd << "ps -a -o rss -p " << getpid() << " | tail +2"; - FILE* cmdPipe; - char input[80]; - cmdPipe = popen(cmd.str().c_str(), "r"); - input[0] = '\0'; - fgets(input, 80, cmdPipe); - input[79] = '\0'; - pclose(cmdPipe); - rss = atoi(input) * 1024LL; + ostringstream cmd; + cmd << "ps -a -o rss -p " << getpid() << " | tail +2"; + FILE* cmdPipe; + char input[80]; + cmdPipe = popen(cmd.str().c_str(), "r"); + input[0] = '\0'; + fgets(input, 80, cmdPipe); + input[79] = '\0'; + pclose(cmdPipe); + rss = atoi(input) * 1024LL; #else - ostringstream pstat; - pstat << "/proc/" << fPid << "/statm"; - ifstream in(pstat.str().c_str()); - size_t x; + ostringstream pstat; + pstat << "/proc/" << fPid << "/statm"; + ifstream in(pstat.str().c_str()); + size_t x; - in >> x; - in >> rss; + in >> x; + in >> rss; - //rss is now in pages, convert to bytes - rss *= fPageSize; + // rss is now in pages, convert to bytes + rss *= fPageSize; #endif - return static_cast(rss); + return static_cast(rss); } //------------------------------------------------------------------------------ // Stays in "sleep" state till fSleepSec seconds have elapsed. //------------------------------------------------------------------------------ -void MonitorProcMem::pause_( ) const +void MonitorProcMem::pause_() const { - struct timespec req; - struct timespec rem; + struct timespec req; + struct timespec rem; - req.tv_sec = fSleepSec; - req.tv_nsec = 0; + req.tv_sec = fSleepSec; + req.tv_nsec = 0; - rem.tv_sec = 0; - rem.tv_nsec = 0; + rem.tv_sec = 0; + rem.tv_nsec = 0; - while (1) - { + while (1) + { #ifdef _MSC_VER - Sleep(req.tv_sec * 1000); + Sleep(req.tv_sec * 1000); #else - if (nanosleep(&req, &rem) != 0) - { - if (rem.tv_sec > 0 || rem.tv_nsec > 0) - { - req = rem; - continue; - } - } + if (nanosleep(&req, &rem) != 0) + { + if (rem.tv_sec > 0 || rem.tv_nsec > 0) + { + req = rem; + continue; + } + } #endif - break; - } + break; + } } //------------------------------------------------------------------------------ @@ -184,7 +181,7 @@ void MonitorProcMem::pause_( ) const //------------------------------------------------------------------------------ unsigned MonitorProcMem::memUsedPct() { - return ((100 * (fMemTotal - fMemFree)) / fMemTotal); + return ((100 * (fMemTotal - fMemFree)) / fMemTotal); } //------------------------------------------------------------------------------ @@ -192,8 +189,8 @@ unsigned MonitorProcMem::memUsedPct() //------------------------------------------------------------------------------ bool MonitorProcMem::isMemAvailable(size_t memRequest) { - int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal); - return (memAvailPct < fMemPctCheck); + int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal); + return (memAvailPct < fMemPctCheck); } -} // end of namespace +} // namespace utils diff --git a/utils/common/MonitorProcMem.h b/utils/common/MonitorProcMem.h index 7a28c5971..e5da1d761 100644 --- a/utils/common/MonitorProcMem.h +++ b/utils/common/MonitorProcMem.h @@ -36,7 +36,6 @@ namespace utils { - /** @brief A class to monitor a process's memory usage. * * This class will monitor and terminate the process if @@ -44,87 +43,80 @@ namespace utils */ class MonitorProcMem { -public: - /** @brief MonitorProcMem constructor - * - * @param maxPct (in) maximum allowable memory usage - * @param memChk (in) monitor total system physical memory usage - * @param msgLog (in) message logger to log msg to - * @param sec (in) number of seconds between memory checks - */ - explicit MonitorProcMem(size_t maxPct, - size_t memChk, - uint32_t subsystemID, - unsigned sec = 1) : - fPid ( getpid() ), - fMaxPct ( maxPct ), - fSleepSec ( sec ), - fSubsystemID ( subsystemID ), - fPageSize ( getpagesize() ) - { - fMemPctCheck = memChk; - fMemTotal = memTotal(); - } + public: + /** @brief MonitorProcMem constructor + * + * @param maxPct (in) maximum allowable memory usage + * @param memChk (in) monitor total system physical memory usage + * @param msgLog (in) message logger to log msg to + * @param sec (in) number of seconds between memory checks + */ + explicit MonitorProcMem(size_t maxPct, size_t memChk, uint32_t subsystemID, unsigned sec = 1) + : fPid(getpid()), fMaxPct(maxPct), fSleepSec(sec), fSubsystemID(subsystemID), fPageSize(getpagesize()) + { + fMemPctCheck = memChk; + fMemTotal = memTotal(); + } - virtual ~MonitorProcMem() {}; + virtual ~MonitorProcMem(){}; - /** @brief Thread entry point - * - * Entry point for this thread that monitors memory usage. - */ - virtual void operator()() const; + /** @brief Thread entry point + * + * Entry point for this thread that monitors memory usage. + */ + virtual void operator()() const; - /* Accessor to return the free memory - * - */ - size_t getFreeMem() const - { - return fMemFree; - } + /* Accessor to return the free memory + * + */ + size_t getFreeMem() const + { + return fMemFree; + } - /* return the % of total memory used - * - */ - static unsigned memUsedPct(); + /* return the % of total memory used + * + */ + static unsigned memUsedPct(); - /* return true if memory usage is below configured limit - * - */ - static bool isMemAvailable(size_t memRequest = 0); + /* return true if memory usage is below configured limit + * + */ + static bool isMemAvailable(size_t memRequest = 0); -protected: - //Defaults are okay - //MonitorProcMem (const MonitorProcMem& rhs); - //MonitorProcMem& operator=(const MonitorProcMem& rhs); + protected: + // Defaults are okay + // MonitorProcMem (const MonitorProcMem& rhs); + // MonitorProcMem& operator=(const MonitorProcMem& rhs); - /** return the current process RSS size in MB - * - */ - size_t rss() const; + /** return the current process RSS size in MB + * + */ + size_t rss() const; - /* return the system RAM size in MB - * - */ - size_t memTotal() const; + /* return the system RAM size in MB + * + */ + size_t memTotal() const; - /* pause for fSleepSec seconds between memory usage checks - * - */ - void pause_() const; + /* pause for fSleepSec seconds between memory usage checks + * + */ + void pause_() const; - pid_t fPid; // process pid - size_t fMaxPct; // max allowable % memory use - unsigned fSleepSec; // sleep interval in seconds - uint32_t fSubsystemID; // Subsystem ID for Logger - int fPageSize; // page size for this host (in bytes) + pid_t fPid; // process pid + size_t fMaxPct; // max allowable % memory use + unsigned fSleepSec; // sleep interval in seconds + uint32_t fSubsystemID; // Subsystem ID for Logger + int fPageSize; // page size for this host (in bytes) - // @bug4507, monitor % of total used system memory - static uint64_t fMemTotal; - static uint64_t fMemFree; - static int fMemPctCheck; - mutable CGroupConfigurator cg; + // @bug4507, monitor % of total used system memory + static uint64_t fMemTotal; + static uint64_t fMemFree; + static int fMemPctCheck; + mutable CGroupConfigurator cg; }; -} +} // namespace utils #endif diff --git a/utils/common/any.hpp b/utils/common/any.hpp old mode 100755 new mode 100644 index 64b6e1bd4..e3429d7aa --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -18,290 +18,294 @@ namespace static_any { namespace anyimpl { - struct empty_any - { - }; +struct empty_any +{ +}; - struct base_any_policy - { - virtual void static_delete(void** x) = 0; - virtual void copy_from_value(void const* src, void** dest) = 0; - virtual void clone(void* const* src, void** dest) = 0; - virtual void move(void* const* src, void** dest) = 0; - virtual void* get_value(void** src) = 0; - virtual size_t get_size() = 0; - protected: - ~base_any_policy() = default; - }; +struct base_any_policy +{ + virtual void static_delete(void** x) = 0; + virtual void copy_from_value(void const* src, void** dest) = 0; + virtual void clone(void* const* src, void** dest) = 0; + virtual void move(void* const* src, void** dest) = 0; + virtual void* get_value(void** src) = 0; + virtual size_t get_size() = 0; - //inline base_any_policy::~base_any_policy() throw () {} + protected: + ~base_any_policy() = default; +}; - template - struct typed_base_any_policy : base_any_policy - { - virtual size_t get_size() - { - return sizeof(T); - } - protected: - ~typed_base_any_policy() = default; - }; +// inline base_any_policy::~base_any_policy() throw () {} - template - struct small_any_policy : typed_base_any_policy - { - virtual ~small_any_policy() = default; - virtual void static_delete(void** x) - { - *x = 0; - } - virtual void copy_from_value(void const* src, void** dest) - { - new(dest) T(*reinterpret_cast(src)); - } - virtual void clone(void* const* src, void** dest) - { - *dest = *src; - } - virtual void move(void* const* src, void** dest) - { - *dest = *src; - } - virtual void* get_value(void** src) - { - return reinterpret_cast(src); - } - }; +template +struct typed_base_any_policy : base_any_policy +{ + virtual size_t get_size() + { + return sizeof(T); + } - template - struct big_any_policy : typed_base_any_policy - { - virtual ~big_any_policy() = default; - virtual void static_delete(void** x) - { - if (*x) - delete(*reinterpret_cast(x)); - *x = NULL; - } - virtual void copy_from_value(void const* src, void** dest) - { - *dest = new T(*reinterpret_cast(src)); - } - virtual void clone(void* const* src, void** dest) - { - *dest = new T(**reinterpret_cast(src)); - } - virtual void move(void* const* src, void** dest) - { - (*reinterpret_cast(dest))->~T(); - **reinterpret_cast(dest) = **reinterpret_cast(src); - } - virtual void* get_value(void** src) - { - return *src; - } - }; + protected: + ~typed_base_any_policy() = default; +}; - template - struct choose_policy - { - typedef big_any_policy type; - }; +template +struct small_any_policy : typed_base_any_policy +{ + virtual ~small_any_policy() = default; + virtual void static_delete(void** x) + { + *x = 0; + } + virtual void copy_from_value(void const* src, void** dest) + { + new (dest) T(*reinterpret_cast(src)); + } + virtual void clone(void* const* src, void** dest) + { + *dest = *src; + } + virtual void move(void* const* src, void** dest) + { + *dest = *src; + } + virtual void* get_value(void** src) + { + return reinterpret_cast(src); + } +}; - template - struct choose_policy - { - typedef small_any_policy type; - }; +template +struct big_any_policy : typed_base_any_policy +{ + virtual ~big_any_policy() = default; + virtual void static_delete(void** x) + { + if (*x) + delete (*reinterpret_cast(x)); + *x = NULL; + } + virtual void copy_from_value(void const* src, void** dest) + { + *dest = new T(*reinterpret_cast(src)); + } + virtual void clone(void* const* src, void** dest) + { + *dest = new T(**reinterpret_cast(src)); + } + virtual void move(void* const* src, void** dest) + { + (*reinterpret_cast(dest))->~T(); + **reinterpret_cast(dest) = **reinterpret_cast(src); + } + virtual void* get_value(void** src) + { + return *src; + } +}; - struct any; +template +struct choose_policy +{ + typedef big_any_policy type; +}; - /// Choosing the policy for an any type is illegal, but should never happen. - /// This is designed to throw a compiler error. - template<> - struct choose_policy - { - typedef void type; - }; +template +struct choose_policy +{ + typedef small_any_policy type; +}; - /// Specializations for big types. -#define BIG_POLICY(TYPE) template<> struct \ - choose_policy { typedef big_any_policy type; }; +struct any; - BIG_POLICY(int128_t); - - /// Specializations for small types. -#define SMALL_POLICY(TYPE) template<> struct \ - choose_policy { typedef small_any_policy type; }; +/// Choosing the policy for an any type is illegal, but should never happen. +/// This is designed to throw a compiler error. +template <> +struct choose_policy +{ + typedef void type; +}; - SMALL_POLICY(char); - SMALL_POLICY(signed char); - SMALL_POLICY(unsigned char); - SMALL_POLICY(signed short); - SMALL_POLICY(unsigned short); - SMALL_POLICY(signed int); - SMALL_POLICY(unsigned int); - SMALL_POLICY(signed long); - SMALL_POLICY(unsigned long); - SMALL_POLICY(signed long long); - SMALL_POLICY(unsigned long long); - SMALL_POLICY(float); - SMALL_POLICY(double); - SMALL_POLICY(bool); +/// Specializations for big types. +#define BIG_POLICY(TYPE) \ + template <> \ + struct choose_policy \ + { \ + typedef big_any_policy type; \ + }; + +BIG_POLICY(int128_t); + +/// Specializations for small types. +#define SMALL_POLICY(TYPE) \ + template <> \ + struct choose_policy \ + { \ + typedef small_any_policy type; \ + }; + +SMALL_POLICY(char); +SMALL_POLICY(signed char); +SMALL_POLICY(unsigned char); +SMALL_POLICY(signed short); +SMALL_POLICY(unsigned short); +SMALL_POLICY(signed int); +SMALL_POLICY(unsigned int); +SMALL_POLICY(signed long); +SMALL_POLICY(unsigned long); +SMALL_POLICY(signed long long); +SMALL_POLICY(unsigned long long); +SMALL_POLICY(float); +SMALL_POLICY(double); +SMALL_POLICY(bool); #undef SMALL_POLICY - /// This function will return a different policy for each type. - template - base_any_policy* get_policy() - { - static typename choose_policy::type policy; - return &policy; - }; -} +/// This function will return a different policy for each type. +template +base_any_policy* get_policy() +{ + static typename choose_policy::type policy; + return &policy; +}; +} // namespace anyimpl class any { -private: - // fields - anyimpl::base_any_policy* policy; - void* object; + private: + // fields + anyimpl::base_any_policy* policy; + void* object; -public: - /// Initializing constructor. - template - any(const T& x) - : policy(anyimpl::get_policy()), object(NULL) + public: + /// Initializing constructor. + template + any(const T& x) : policy(anyimpl::get_policy()), object(NULL) + { + assign(x); + } + + /// Empty constructor. + any() : policy(anyimpl::get_policy()), object(NULL) + { + } + + /// Special initializing constructor for string literals. + any(const char* x) : policy(anyimpl::get_policy()), object(NULL) + { + assign(x); + } + + /// Copy constructor. + any(const any& x) : policy(anyimpl::get_policy()), object(NULL) + { + assign(x); + } + + /// Destructor. + ~any() + { + policy->static_delete(&object); + } + + /// Assignment function from another any. + any& assign(const any& x) + { + reset(); + policy = x.policy; + policy->clone(&x.object, &object); + return *this; + } + + /// Assignment function. + template + any& assign(const T& x) + { + reset(); + policy = anyimpl::get_policy(); + policy->copy_from_value(&x, &object); + return *this; + } + + /// Assignment operator. + template + any& operator=(const T& x) + { + return assign(x); + } + + /// Assignment operator, specialed for literal strings. + /// They have types like const char [6] which don't work as expected. + any& operator=(const char* x) + { + return assign(x); + } + + /// Less than operator for sorting + bool operator<(const any& x) const + { + if (policy == x.policy) { - assign(x); - } + void* p1 = const_cast(object); + void* p2 = const_cast(x.object); + return memcmp(policy->get_value(&p1), x.policy->get_value(&p2), policy->get_size()) < 0 ? 1 : 0; + } + return 0; + } - /// Empty constructor. - any() - : policy(anyimpl::get_policy()), object(NULL) + /// equal operator + bool operator==(const any& x) const + { + if (policy == x.policy) { + void* p1 = const_cast(object); + void* p2 = const_cast(x.object); + return memcmp(policy->get_value(&p1), x.policy->get_value(&p2), policy->get_size()) == 0 ? 1 : 0; } + return 0; + } - /// Special initializing constructor for string literals. - any(const char* x) - : policy(anyimpl::get_policy()), object(NULL) - { - assign(x); - } + /// Utility functions + uint8_t getHash() const + { + void* p1 = const_cast(object); + return *(uint64_t*)policy->get_value(&p1) % 4048; + } + any& swap(any& x) + { + std::swap(policy, x.policy); + std::swap(object, x.object); + return *this; + } - /// Copy constructor. - any(const any& x) - : policy(anyimpl::get_policy()), object(NULL) - { - assign(x); - } + /// Cast operator. You can only cast to the original type. + template + T& cast() + { + if (policy != anyimpl::get_policy()) + throw std::runtime_error("static_any: type mismatch in cast"); + T* r = reinterpret_cast(policy->get_value(&object)); + return *r; + } - /// Destructor. - ~any() - { - policy->static_delete(&object); - } + /// Returns true if the any contains no value. + bool empty() const + { + return policy == anyimpl::get_policy(); + } - /// Assignment function from another any. - any& assign(const any& x) - { - reset(); - policy = x.policy; - policy->clone(&x.object, &object); - return *this; - } + /// Frees any allocated memory, and sets the value to NULL. + void reset() + { + policy->static_delete(&object); + policy = anyimpl::get_policy(); + } - /// Assignment function. - template - any& assign(const T& x) - { - reset(); - policy = anyimpl::get_policy(); - policy->copy_from_value(&x, &object); - return *this; - } - - /// Assignment operator. - template - any& operator=(const T& x) { - return assign(x); - } - - /// Assignment operator, specialed for literal strings. - /// They have types like const char [6] which don't work as expected. - any& operator=(const char* x) { - return assign(x); - } - - /// Less than operator for sorting - bool operator<(const any& x) const - { - if (policy == x.policy) - { - void* p1 = const_cast(object); - void* p2 = const_cast(x.object); - return memcmp(policy->get_value(&p1), - x.policy->get_value(&p2), - policy->get_size()) < 0 ? 1 : 0; - } - return 0; - } - - /// equal operator - bool operator==(const any& x) const - { - if (policy == x.policy) - { - void* p1 = const_cast(object); - void* p2 = const_cast(x.object); - return memcmp(policy->get_value(&p1), - x.policy->get_value(&p2), - policy->get_size()) == 0 ? 1 : 0; - } - return 0; - } - - /// Utility functions - uint8_t getHash() const - { - void* p1 = const_cast(object); - return *(uint64_t*)policy->get_value(&p1) % 4048; - } - any& swap(any& x) - { - std::swap(policy, x.policy); - std::swap(object, x.object); - return *this; - } - - /// Cast operator. You can only cast to the original type. - template - T& cast() - { - if (policy != anyimpl::get_policy()) - throw std::runtime_error("static_any: type mismatch in cast"); - T* r = reinterpret_cast(policy->get_value(&object)); - return *r; - } - - /// Returns true if the any contains no value. - bool empty() const - { - return policy == anyimpl::get_policy(); - } - - /// Frees any allocated memory, and sets the value to NULL. - void reset() - { - policy->static_delete(&object); - policy = anyimpl::get_policy(); - } - - /// Returns true if the two types are the same. - bool compatible(const any& x) const - { - return policy == x.policy; - } + /// Returns true if the two types are the same. + bool compatible(const any& x) const + { + return policy == x.policy; + } }; -} +} // namespace static_any diff --git a/utils/common/atomicops.h b/utils/common/atomicops.h index 0f77d175e..d4b90bfb3 100644 --- a/utils/common/atomicops.h +++ b/utils/common/atomicops.h @@ -35,108 +35,97 @@ Boost has something in interprocess::ipcdetail, but it doesn't have 64-bit API's namespace atomicops { - -//Returns the resulting, incremented value +// Returns the resulting, incremented value template inline T atomicInc(volatile T* mem) { #ifdef _MSC_VER - switch (sizeof(T)) - { - case 4: - default: - return InterlockedIncrement(reinterpret_cast(mem)); + switch (sizeof(T)) + { + case 4: + default: return InterlockedIncrement(reinterpret_cast(mem)); - case 8: - return InterlockedIncrement64(reinterpret_cast(mem)); - } + case 8: return InterlockedIncrement64(reinterpret_cast(mem)); + } #else - return __sync_add_and_fetch(mem, 1); + return __sync_add_and_fetch(mem, 1); #endif } -//decrements, but returns the pre-decrement value +// decrements, but returns the pre-decrement value template inline T atomicDec(volatile T* mem) { #ifdef _MSC_VER - switch (sizeof(T)) - { - case 4: - default: - return InterlockedDecrement(reinterpret_cast(mem)) + 1; + switch (sizeof(T)) + { + case 4: + default: return InterlockedDecrement(reinterpret_cast(mem)) + 1; - case 8: - return InterlockedDecrement64(reinterpret_cast(mem)) + 1; - } + case 8: return InterlockedDecrement64(reinterpret_cast(mem)) + 1; + } #else - return __sync_fetch_and_add(mem, -1); + return __sync_fetch_and_add(mem, -1); #endif } -//Returns the resulting value (but doesn't need to yet) +// Returns the resulting value (but doesn't need to yet) template inline T atomicAdd(volatile T* mem, T val) { #ifdef _MSC_VER - switch (sizeof(T)) - { - case 4: - default: - InterlockedExchangeAdd(reinterpret_cast(mem), val); - break; + switch (sizeof(T)) + { + case 4: + default: InterlockedExchangeAdd(reinterpret_cast(mem), val); break; - case 8: - InterlockedExchangeAdd64(reinterpret_cast(mem), val); - break; - } + case 8: InterlockedExchangeAdd64(reinterpret_cast(mem), val); break; + } - return *mem; + return *mem; #else - return __sync_add_and_fetch(mem, val); + return __sync_add_and_fetch(mem, val); #endif } -//Returns the resulting value +// Returns the resulting value template inline T atomicSub(volatile T* mem, T val) { #ifdef _MSC_VER - switch (sizeof(T)) - { - case 4: - default: - InterlockedExchangeAdd(reinterpret_cast(mem), -(static_cast(val))); - break; + switch (sizeof(T)) + { + case 4: + default: InterlockedExchangeAdd(reinterpret_cast(mem), -(static_cast(val))); break; - case 8: - InterlockedExchangeAdd64(reinterpret_cast(mem), -(static_cast(val))); - break; - } + case 8: + InterlockedExchangeAdd64(reinterpret_cast(mem), -(static_cast(val))); + break; + } - return *mem; + return *mem; #else - return __sync_sub_and_fetch(mem, val); + return __sync_sub_and_fetch(mem, val); #endif } -//Implements a memory barrier +// Implements a memory barrier inline void atomicMb() { #ifdef _MSC_VER - MemoryBarrier(); + MemoryBarrier(); #else - __sync_synchronize(); + __sync_synchronize(); #endif } -//Returns true iff the CAS took place, that is +// Returns true iff the CAS took place, that is // if (*mem == comp) { // *mem = swap; // return true; @@ -148,20 +137,21 @@ inline bool atomicCAS(volatile T* mem, T comp, T swap) { #ifdef _MSC_VER - switch (sizeof(T)) - { - case 4: - default: - //The function returns the initial value of the mem parameter - return (InterlockedCompareExchange(reinterpret_cast(mem), swap, comp) == comp); + switch (sizeof(T)) + { + case 4: + default: + // The function returns the initial value of the mem parameter + return (InterlockedCompareExchange(reinterpret_cast(mem), swap, comp) == comp); - case 8: - return (InterlockedCompareExchange64(reinterpret_cast(mem), swap, comp) == comp); - } + case 8: + return (InterlockedCompareExchange64(reinterpret_cast(mem), swap, comp) == comp); + } #else - //If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is successful and swap was written. - return __sync_bool_compare_and_swap(mem, comp, swap); + // If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is + // successful and swap was written. + return __sync_bool_compare_and_swap(mem, comp, swap); #endif } @@ -170,33 +160,29 @@ template inline void atomicZero(volatile T* mem) { #ifdef _MSC_VER - - switch (sizeof(T)) - { - case 4: - default: - InterlockedXor(reinterpret_cast(mem),(static_cast(*mem))); - break; - - case 8: - InterlockedXor64(reinterpret_cast(mem),(static_cast(*mem))); - break; - } - #else - __sync_xor_and_fetch(mem, *mem); - #endif + + switch (sizeof(T)) + { + case 4: + default: InterlockedXor(reinterpret_cast(mem), (static_cast(*mem))); break; + + case 8: InterlockedXor64(reinterpret_cast(mem), (static_cast(*mem))); break; + } +#else + __sync_xor_and_fetch(mem, *mem); +#endif } -//Implements a scheduler yield +// Implements a scheduler yield inline void atomicYield() { #ifdef _MSC_VER - SwitchToThread(); + SwitchToThread(); #else - sched_yield(); + sched_yield(); #endif } -} +} // namespace atomicops #endif diff --git a/utils/common/branchpred.h b/utils/common/branchpred.h index f345fd9a9..a4b55262d 100644 --- a/utils/common/branchpred.h +++ b/utils/common/branchpred.h @@ -27,9 +27,8 @@ #endif #ifndef LIKELY -#define LIKELY(x) __builtin_expect((x),1) -#define UNLIKELY(x) __builtin_expect((x),0) +#define LIKELY(x) __builtin_expect((x), 1) +#define UNLIKELY(x) __builtin_expect((x), 0) #endif #endif - diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index 2be36b3a8..65b8b1139 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -30,58 +30,61 @@ using namespace boost; using namespace std; - // minor space-savers -#define RETURN_NO_GROUP(err) do { \ - if (!printedWarning) { \ - printedWarning = true; \ - ostringstream os; \ - os << "CGroup warning! The group " << cGroupName << " does not exist."; \ - cerr << os.str() << endl; \ - log(logging::LOG_TYPE_WARNING, os.str()); \ - } \ - return err; \ -} while (0) +#define RETURN_NO_GROUP(err) \ + do \ + { \ + if (!printedWarning) \ + { \ + printedWarning = true; \ + ostringstream os; \ + os << "CGroup warning! The group " << cGroupName << " does not exist."; \ + cerr << os.str() << endl; \ + log(logging::LOG_TYPE_WARNING, os.str()); \ + } \ + return err; \ + } while (0) -#define RETURN_READ_ERROR(err) do { \ - if (!printedWarning) { \ - printedWarning = true; \ - ostringstream os; \ - os << "CGroup warning! Could not read the file " << filename << "."; \ - cerr << os.str() << endl; \ - log(logging::LOG_TYPE_WARNING, os.str()); \ - } \ - return err; \ -} while (0) +#define RETURN_READ_ERROR(err) \ + do \ + { \ + if (!printedWarning) \ + { \ + printedWarning = true; \ + ostringstream os; \ + os << "CGroup warning! Could not read the file " << filename << "."; \ + cerr << os.str() << endl; \ + log(logging::LOG_TYPE_WARNING, os.str()); \ + } \ + return err; \ + } while (0) namespace { - void log(logging::LOG_TYPE whichLogFile, const string& msg) { - logging::Logger logger(12); //12 = configcpp - logger.logMessage(whichLogFile, msg, logging::LoggingID(12)); + logging::Logger logger(12); // 12 = configcpp + logger.logMessage(whichLogFile, msg, logging::LoggingID(12)); } -} +} // namespace namespace utils { - CGroupConfigurator::CGroupConfigurator() { - config = config::Config::makeConfig(); + config = config::Config::makeConfig(); - cGroupName = config->getConfig("SystemConfig", "CGroup"); + cGroupName = config->getConfig("SystemConfig", "CGroup"); - if (cGroupName.empty()) - cGroupDefined = false; - else - cGroupDefined = true; + if (cGroupName.empty()) + cGroupDefined = false; + else + cGroupDefined = true; - totalMemory = 0; - totalSwap = 0; - printedWarning = false; + totalMemory = 0; + totalSwap = 0; + printedWarning = false; } CGroupConfigurator::~CGroupConfigurator() @@ -90,445 +93,441 @@ CGroupConfigurator::~CGroupConfigurator() uint32_t CGroupConfigurator::getNumCoresFromCGroup() { - ostringstream filename_os; - filename_os << "/sys/fs/cgroup/cpuset/" << cGroupName << "/cpus"; - string filename = filename_os.str(); + ostringstream filename_os; + filename_os << "/sys/fs/cgroup/cpuset/" << cGroupName << "/cpus"; + string filename = filename_os.str(); - ifstream in(filename.c_str()); - string cpusString; - uint32_t cpus = 0; + ifstream in(filename.c_str()); + string cpusString; + uint32_t cpus = 0; - if (!in) - RETURN_NO_GROUP(0); + if (!in) + RETURN_NO_GROUP(0); - try + try + { + // Need to parse & count how many CPUs we have access to + in >> cpusString; + } + catch (...) + { + RETURN_READ_ERROR(0); + } + + // the file has comma-deliminted CPU ranges like "0-7,9,11-12". + size_t first = 0, last; + bool lastRange = false; + + while (!lastRange) + { + size_t dash; + string oneRange; + + last = cpusString.find(',', first); + + if (last == string::npos) { - - // Need to parse & count how many CPUs we have access to - in >> cpusString; + lastRange = true; + oneRange = cpusString.substr(first); } - catch (...) + else + oneRange = cpusString.substr(first, last - first - 1); + + if ((dash = oneRange.find('-')) == string::npos) // single-cpu range + cpus++; + else { - RETURN_READ_ERROR(0); + const char* data = oneRange.c_str(); + uint32_t firstCPU = strtol(data, NULL, 10); + uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10); + cpus += lastCPU - firstCPU + 1; } - // the file has comma-deliminted CPU ranges like "0-7,9,11-12". - size_t first = 0, last; - bool lastRange = false; + first = last + 1; + } - while (!lastRange) - { - size_t dash; - string oneRange; - - last = cpusString.find(',', first); - - if (last == string::npos) - { - lastRange = true; - oneRange = cpusString.substr(first); - } - else - oneRange = cpusString.substr(first, last - first - 1); - - if ((dash = oneRange.find('-')) == string::npos) // single-cpu range - cpus++; - else - { - const char* data = oneRange.c_str(); - uint32_t firstCPU = strtol(data, NULL, 10); - uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10); - cpus += lastCPU - firstCPU + 1; - } - - first = last + 1; - } - - //cout << "found " << cpus << " CPUS in the string " << cpusString << endl; - return cpus; + // cout << "found " << cpus << " CPUS in the string " << cpusString << endl; + return cpus; } uint32_t CGroupConfigurator::getNumCoresFromProc() { #ifdef _MSC_VER - SYSTEM_INFO siSysInfo; - GetSystemInfo(&siSysInfo); - return siSysInfo.dwNumberOfProcessors; + SYSTEM_INFO siSysInfo; + GetSystemInfo(&siSysInfo); + return siSysInfo.dwNumberOfProcessors; #else - uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN); + uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN); - return nc; + return nc; #endif } - - uint32_t CGroupConfigurator::getNumCores() { - /* - Detect if InfiniDB is in a C-Group - - get the group ID - If not, get the number of cores from /proc - */ - uint32_t ret; + /* + Detect if InfiniDB is in a C-Group + - get the group ID + If not, get the number of cores from /proc + */ + uint32_t ret; - if (!cGroupDefined) - ret = getNumCoresFromProc(); - else - { - ret = getNumCoresFromCGroup(); + if (!cGroupDefined) + ret = getNumCoresFromProc(); + else + { + ret = getNumCoresFromCGroup(); - if (ret == 0) - ret = getNumCoresFromProc(); - } + if (ret == 0) + ret = getNumCoresFromProc(); + } - //cout << "There are " << ret << " cores available" << endl; - return ret; + // cout << "There are " << ret << " cores available" << endl; + return ret; } uint64_t CGroupConfigurator::getTotalMemory() { - uint64_t ret; + uint64_t ret; - if (totalMemory != 0) - return totalMemory; - - if (!cGroupDefined) - ret = getTotalMemoryFromProc(); - else - { - ret = getTotalMemoryFromCGroup(); - - if (ret == 0) - ret = getTotalMemoryFromProc(); - } - - //cout << "Total mem available is " << ret << endl; - totalMemory = ret; + if (totalMemory != 0) return totalMemory; + + if (!cGroupDefined) + ret = getTotalMemoryFromProc(); + else + { + ret = getTotalMemoryFromCGroup(); + + if (ret == 0) + ret = getTotalMemoryFromProc(); + } + + // cout << "Total mem available is " << ret << endl; + totalMemory = ret; + return totalMemory; } uint64_t CGroupConfigurator::getTotalMemoryFromProc() { - size_t memTot; + size_t memTot; #if defined(_MSC_VER) - MEMORYSTATUSEX memStat; - memStat.dwLength = sizeof(memStat); + MEMORYSTATUSEX memStat; + memStat.dwLength = sizeof(memStat); - if (GlobalMemoryStatusEx(&memStat) == 0) - //FIXME: Assume 2GB? - memTot = 2 * 1024 * 1024; - else - { + if (GlobalMemoryStatusEx(&memStat) == 0) + // FIXME: Assume 2GB? + memTot = 2 * 1024 * 1024; + else + { #ifndef _WIN64 - memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); + memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); #endif - //We now have the total phys mem in bytes - memTot = memStat.ullTotalPhys / 1024; - } + // We now have the total phys mem in bytes + memTot = memStat.ullTotalPhys / 1024; + } #elif defined(__FreeBSD__) - string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'"); - FILE* cmdPipe; - char input[80]; - cmdPipe = popen(cmd.c_str(), "r"); - input[0] = '\0'; - fgets(input, 80, cmdPipe); - input[79] = '\0'; - pclose(cmdPipe); - memTot = atoi(input); + string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'"); + FILE* cmdPipe; + char input[80]; + cmdPipe = popen(cmd.c_str(), "r"); + input[0] = '\0'; + fgets(input, 80, cmdPipe); + input[79] = '\0'; + pclose(cmdPipe); + memTot = atoi(input); #else - ifstream in("/proc/meminfo"); - string x; + ifstream in("/proc/meminfo"); + string x; - in >> x; - in >> memTot; + in >> x; + in >> memTot; #endif - //memTot is now in KB, convert to bytes - memTot *= 1024; + // memTot is now in KB, convert to bytes + memTot *= 1024; - return memTot; + return memTot; } uint64_t CGroupConfigurator::getTotalMemoryFromCGroup() { - ifstream in; - uint64_t ret; - ostringstream os; - string filename; + ifstream in; + uint64_t ret; + ostringstream os; + string filename; - os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.limit_in_bytes"; - filename = os.str(); + os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.limit_in_bytes"; + filename = os.str(); - in.open(filename.c_str()); + in.open(filename.c_str()); - if (!in) - RETURN_NO_GROUP(0); + if (!in) + RETURN_NO_GROUP(0); - try - { - in >> ret; - } - catch (...) - { - RETURN_READ_ERROR(0); - } + try + { + in >> ret; + } + catch (...) + { + RETURN_READ_ERROR(0); + } - return ret; + return ret; } uint64_t CGroupConfigurator::getFreeMemory() { - uint64_t ret; + uint64_t ret; - if (!cGroupDefined) - ret = getFreeMemoryFromProc(); + if (!cGroupDefined) + ret = getFreeMemoryFromProc(); + else + { + uint64_t usage = getMemUsageFromCGroup(); + + if (usage == 0) + ret = getFreeMemoryFromProc(); else - { - uint64_t usage = getMemUsageFromCGroup(); + ret = getTotalMemory() - usage; + } - if (usage == 0) - ret = getFreeMemoryFromProc(); - else - ret = getTotalMemory() - usage; - } - - //cout << "free memory = " << ret << endl; - return ret; + // cout << "free memory = " << ret << endl; + return ret; } uint64_t CGroupConfigurator::getMemUsageFromCGroup() { - uint64_t ret = 0; - bool found = false; - char oneline[80]; + uint64_t ret = 0; + bool found = false; + char oneline[80]; - if (memUsageFilename.empty()) + if (memUsageFilename.empty()) + { + ostringstream filename; + filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat"; + memUsageFilename = filename.str(); + } + + ifstream in(memUsageFilename.c_str()); + string& filename = memUsageFilename; + + if (!in) + RETURN_NO_GROUP(0); + + try + { + while (in && !found) { - ostringstream filename; - filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat"; - memUsageFilename = filename.str(); + in.getline(oneline, 80); + + if (strncmp(oneline, "rss", 2) == 0) + { + ret = atoll(&oneline[3]); + found = true; + } } + } + catch (...) + { + RETURN_READ_ERROR(0); + } - ifstream in(memUsageFilename.c_str()); - string& filename = memUsageFilename; - - if (!in) - RETURN_NO_GROUP(0); - - try - { - while (in && !found) - { - in.getline(oneline, 80); - - if (strncmp(oneline, "rss", 2) == 0) - { - ret = atoll(&oneline[3]); - found = true; - } - } - } - catch (...) - { - RETURN_READ_ERROR(0); - } - - return ret; + return ret; } uint64_t CGroupConfigurator::getFreeMemoryFromProc() { - uint64_t memFree = 0; - uint64_t buffers = 0; - uint64_t cached = 0; - uint64_t memTotal = 0; - uint64_t memAvailable = 0; + uint64_t memFree = 0; + uint64_t buffers = 0; + uint64_t cached = 0; + uint64_t memTotal = 0; + uint64_t memAvailable = 0; #if defined(_MSC_VER) - MEMORYSTATUSEX memStat; - memStat.dwLength = sizeof(memStat); + MEMORYSTATUSEX memStat; + memStat.dwLength = sizeof(memStat); - if (GlobalMemoryStatusEx(&memStat)) - { - memAvailable = memStat.ullAvailPhys; + if (GlobalMemoryStatusEx(&memStat)) + { + memAvailable = memStat.ullAvailPhys; #ifndef _WIN64 - uint64_t tmp = getTotalMemoryFromProc(); + uint64_t tmp = getTotalMemoryFromProc(); - if (memFree > tmp) - memAvailable = tmp; + if (memFree > tmp) + memAvailable = tmp; #endif - } + } #elif defined(__FreeBSD__) - // FreeBSD is not supported, no optimization. - memAvailable = 0; + // FreeBSD is not supported, no optimization. + memAvailable = 0; #else - ifstream in("/proc/meminfo"); - string x; + ifstream in("/proc/meminfo"); + string x; - in >> x; // MemTotal: - in >> memTotal; - in >> x; // kB + in >> x; // MemTotal: + in >> memTotal; + in >> x; // kB - in >> x; // MemFree: - in >> memFree; - in >> x; // kB + in >> x; // MemFree: + in >> memFree; + in >> x; // kB - //check if available or buffers is passed - in >> x; + // check if available or buffers is passed + in >> x; - if ( x == "MemAvailable:") - { - in >> memAvailable; // MemAvailable - } - else - { - // centos 6 and older OSs - in >> buffers; - in >> x; // kB + if (x == "MemAvailable:") + { + in >> memAvailable; // MemAvailable + } + else + { + // centos 6 and older OSs + in >> buffers; + in >> x; // kB - in >> x; // Cached: - in >> cached; + in >> x; // Cached: + in >> cached; - memAvailable = memFree + buffers + cached; - } + memAvailable = memFree + buffers + cached; + } #endif - // amount available for application - memAvailable *= 1024; - return memAvailable; + // amount available for application + memAvailable *= 1024; + return memAvailable; } uint64_t CGroupConfigurator::getTotalSwapSpace() { - int64_t ret; + int64_t ret; - if (totalSwap != 0) - return totalSwap; + if (totalSwap != 0) + return totalSwap; - if (!cGroupDefined) - ret = getTotalSwapFromSysinfo(); + if (!cGroupDefined) + ret = getTotalSwapFromSysinfo(); + else + { + ret = getTotalMemAndSwapFromCGroup(); + + // if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case. + if (ret == -1 || ret == numeric_limits::max()) + ret = getTotalSwapFromSysinfo(); else - { - ret = getTotalMemAndSwapFromCGroup(); + ret -= getTotalMemory(); + } - // if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case. - if (ret == -1 || ret == numeric_limits::max()) - ret = getTotalSwapFromSysinfo(); - else - ret -= getTotalMemory(); - } - - //cout << "total swap=" << ret << endl; - totalSwap = ret; - return ret; + // cout << "total swap=" << ret << endl; + totalSwap = ret; + return ret; } uint64_t CGroupConfigurator::getTotalSwapFromSysinfo() { - struct sysinfo si; + struct sysinfo si; - sysinfo(&si); - return si.totalswap; + sysinfo(&si); + return si.totalswap; } int64_t CGroupConfigurator::getTotalMemAndSwapFromCGroup() { - int64_t ret; - ifstream in; - string filename; - ostringstream os; + int64_t ret; + ifstream in; + string filename; + ostringstream os; - os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes"; - filename = os.str(); - in.open(filename.c_str()); + os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes"; + filename = os.str(); + in.open(filename.c_str()); - if (!in) - RETURN_NO_GROUP(-1); + if (!in) + RETURN_NO_GROUP(-1); - try - { - in >> ret; - } - catch (...) - { - RETURN_READ_ERROR(-1); - } + try + { + in >> ret; + } + catch (...) + { + RETURN_READ_ERROR(-1); + } - return ret; + return ret; } uint64_t CGroupConfigurator::getSwapInUse() { - int64_t ret; + int64_t ret; - if (!cGroupDefined) - ret = getSwapInUseFromSysinfo(); - else - { - ret = getSwapInUseFromCGroup(); + if (!cGroupDefined) + ret = getSwapInUseFromSysinfo(); + else + { + ret = getSwapInUseFromCGroup(); - if (ret == -1) - ret = getSwapInUseFromSysinfo(); - } + if (ret == -1) + ret = getSwapInUseFromSysinfo(); + } - //cout << "current swap in use=" << ret << endl; - return ret; + // cout << "current swap in use=" << ret << endl; + return ret; } int64_t CGroupConfigurator::getSwapInUseFromCGroup() { - int64_t ret = -1; - ifstream in; - bool found = false; - char oneline[80]; + int64_t ret = -1; + ifstream in; + bool found = false; + char oneline[80]; - if (usedSwapFilename.empty()) + if (usedSwapFilename.empty()) + { + ostringstream os; + os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat"; + usedSwapFilename = os.str(); + } + + string& filename = usedSwapFilename; + in.open(filename.c_str()); + + if (!in) + RETURN_NO_GROUP(-1); + + try + { + while (in && !found) { - ostringstream os; - os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat"; - usedSwapFilename = os.str(); + in.getline(oneline, 80); + + if (strncmp(oneline, "swap", 4) == 0) + { + ret = atoll(&oneline[5]); + found = true; + } } + } + catch (...) + { + RETURN_READ_ERROR(-1); + } - string& filename = usedSwapFilename; - in.open(filename.c_str()); - - if (!in) - RETURN_NO_GROUP(-1); - - try - { - while (in && !found) - { - in.getline(oneline, 80); - - if (strncmp(oneline, "swap", 4) == 0) - { - ret = atoll(&oneline[5]); - found = true; - } - } - } - catch (...) - { - RETURN_READ_ERROR(-1); - } - - return ret; + return ret; } uint64_t CGroupConfigurator::getSwapInUseFromSysinfo() { - struct sysinfo si; + struct sysinfo si; - sysinfo(&si); - return si.totalswap - si.freeswap; + sysinfo(&si); + return si.totalswap - si.freeswap; } - -} +} // namespace utils diff --git a/utils/common/cgroupconfigurator.h b/utils/common/cgroupconfigurator.h index b7b5af20d..cf3bfae5c 100644 --- a/utils/common/cgroupconfigurator.h +++ b/utils/common/cgroupconfigurator.h @@ -26,51 +26,49 @@ namespace utils { - /* This class wraps a few methods for getting configuration variables that potentially come from a cgroup. Might move it to utils/configcpp, and/or change the name. */ class CGroupConfigurator { -public: - CGroupConfigurator(); - virtual ~CGroupConfigurator(); + public: + CGroupConfigurator(); + virtual ~CGroupConfigurator(); - uint32_t getNumCores(); - uint64_t getTotalMemory(); - uint64_t getFreeMemory(); - uint64_t getTotalSwapSpace(); - uint64_t getSwapInUse(); + uint32_t getNumCores(); + uint64_t getTotalMemory(); + uint64_t getFreeMemory(); + uint64_t getTotalSwapSpace(); + uint64_t getSwapInUse(); - bool usingCGroup() - { - return cGroupDefined; - } + bool usingCGroup() + { + return cGroupDefined; + } -private: - uint32_t getNumCoresFromProc(); - uint32_t getNumCoresFromCGroup(); - uint64_t getTotalMemoryFromProc(); - uint64_t getTotalMemoryFromCGroup(); - uint64_t getFreeMemoryFromProc(); - uint64_t getMemUsageFromCGroup(); - uint64_t getTotalSwapFromSysinfo(); - int64_t getTotalMemAndSwapFromCGroup(); - uint64_t getSwapInUseFromSysinfo(); - int64_t getSwapInUseFromCGroup(); + private: + uint32_t getNumCoresFromProc(); + uint32_t getNumCoresFromCGroup(); + uint64_t getTotalMemoryFromProc(); + uint64_t getTotalMemoryFromCGroup(); + uint64_t getFreeMemoryFromProc(); + uint64_t getMemUsageFromCGroup(); + uint64_t getTotalSwapFromSysinfo(); + int64_t getTotalMemAndSwapFromCGroup(); + uint64_t getSwapInUseFromSysinfo(); + int64_t getSwapInUseFromCGroup(); - std::string memUsageFilename; - std::string usedSwapFilename; - - std::string cGroupName; - bool cGroupDefined; - config::Config* config; - uint64_t totalMemory; - uint64_t totalSwap; - bool printedWarning; + std::string memUsageFilename; + std::string usedSwapFilename; + std::string cGroupName; + bool cGroupDefined; + config::Config* config; + uint64_t totalMemory; + uint64_t totalSwap; + bool printedWarning; }; -} +} // namespace utils -#endif // CGROUPCONFIGURATOR_H +#endif // CGROUPCONFIGURATOR_H diff --git a/utils/common/checks.h b/utils/common/checks.h index fb4a94e94..907933537 100644 --- a/utils/common/checks.h +++ b/utils/common/checks.h @@ -20,32 +20,36 @@ #include #include "mcs_int128.h" -namespace utils { +namespace utils +{ +template +typename std::enable_if::value || datatypes::is_uint128_t::value, bool>::type + is_nonnegative(T) +{ + return true; +}; template -typename std::enable_if::value || - datatypes::is_uint128_t::value, - bool>::type -is_nonnegative(T) { return true; }; +typename std::enable_if::value || datatypes::is_int128_t::value, bool>::type +is_nonnegative(T v) +{ + return v >= 0; +}; template -typename std::enable_if::value || - datatypes::is_int128_t::value, - bool>::type -is_nonnegative(T v) { return v >= 0; }; +typename std::enable_if::value || datatypes::is_uint128_t::value, bool>::type + is_negative(T) +{ + return false; +}; template -typename std::enable_if::value || - datatypes::is_uint128_t::value, - bool>::type -is_negative(T) { return false; }; +typename std::enable_if::value || datatypes::is_int128_t::value, bool>::type is_negative( + T v) +{ + return v < 0; +}; -template -typename std::enable_if::value || - datatypes::is_int128_t::value, - bool>::type -is_negative(T v) { return v < 0; }; +} // namespace utils -} // namespace utils - -#endif // UTILS_COMMON_CHECKS_H +#endif // UTILS_COMMON_CHECKS_H diff --git a/utils/common/collation.h b/utils/common/collation.h index 16ae79d07..36311e530 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -20,18 +20,18 @@ #if defined(PREFER_MY_CONFIG_H) - #if !defined(MY_CONFIG_H) - #error my_config.h was not included (but PREFER_MY_CONFIG_H was set) - #endif +#if !defined(MY_CONFIG_H) +#error my_config.h was not included (but PREFER_MY_CONFIG_H was set) +#endif - #include "mcsconfig_conflicting_defs_remember.h" - #include "mcsconfig_conflicting_defs_undef.h" +#include "mcsconfig_conflicting_defs_remember.h" +#include "mcsconfig_conflicting_defs_undef.h" #else - #if defined(MY_CONFIG_H) - #error my_config.h was included before mcsconfig.h (and PREFER_MY_CONFIG_H was not set) - #endif -#endif //PREFER_MY_CONFIG_H +#if defined(MY_CONFIG_H) +#error my_config.h was included before mcsconfig.h (and PREFER_MY_CONFIG_H was not set) +#endif +#endif // PREFER_MY_CONFIG_H #include "mcsconfig.h" @@ -49,11 +49,11 @@ #endif #ifndef TRUE -#define TRUE (1) +#define TRUE (1) #endif #ifndef DBUG_ASSERT -#define DBUG_ASSERT(x) idbassert(x) +#define DBUG_ASSERT(x) idbassert(x) #define DBUG_ASSERT_TEMPORARILY_DEFINED #endif @@ -73,21 +73,21 @@ typedef char my_bool; typedef unsigned char uchar; #if defined(__GNUC__) && !defined(_lint) -typedef char pchar; /* Mixed prototypes can take char */ -typedef char puchar; /* Mixed prototypes can take char */ -typedef char pbool; /* Mixed prototypes can take char */ -typedef short pshort; /* Mixed prototypes can take short int */ -typedef float pfloat; /* Mixed prototypes can take float */ +typedef char pchar; /* Mixed prototypes can take char */ +typedef char puchar; /* Mixed prototypes can take char */ +typedef char pbool; /* Mixed prototypes can take char */ +typedef short pshort; /* Mixed prototypes can take short int */ +typedef float pfloat; /* Mixed prototypes can take float */ #else -typedef int pchar; /* Mixed prototypes can't take char */ -typedef uint puchar; /* Mixed prototypes can't take char */ -typedef int pbool; /* Mixed prototypes can't take char */ -typedef int pshort; /* Mixed prototypes can't take short int */ -typedef double pfloat; /* Mixed prototypes can't take float */ +typedef int pchar; /* Mixed prototypes can't take char */ +typedef uint puchar; /* Mixed prototypes can't take char */ +typedef int pbool; /* Mixed prototypes can't take char */ +typedef int pshort; /* Mixed prototypes can't take short int */ +typedef double pfloat; /* Mixed prototypes can't take float */ #endif typedef const struct charset_info_st CHARSET_INFO; -extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO *default_charset_info; +extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO* default_charset_info; #include "m_ctype.h" @@ -98,128 +98,116 @@ extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO *default_charset_info; #undef DBUG_ASSERT #endif - #if defined(PREFER_MY_CONFIG_H) - #include "mcsconfig_conflicting_defs_restore.h" +#include "mcsconfig_conflicting_defs_restore.h" #endif - namespace datatypes { - class MariaDBHasher { - ulong mPart1; - ulong mPart2; -public: - MariaDBHasher() - :mPart1(1), mPart2(4) - { } - MariaDBHasher & add(CHARSET_INFO * cs, const char *str, size_t length) - { - cs->hash_sort((const uchar *) str, length, &mPart1, &mPart2); - return *this; - } - MariaDBHasher & add(CHARSET_INFO *cs, const utils::ConstString &str) - { - return add(cs, str.str(), str.length()); - } - uint32_t finalize() const - { - return (uint32_t)mPart1; - } -}; + ulong mPart1; + ulong mPart2; + public: + MariaDBHasher() : mPart1(1), mPart2(4) + { + } + MariaDBHasher& add(CHARSET_INFO* cs, const char* str, size_t length) + { + cs->hash_sort((const uchar*)str, length, &mPart1, &mPart2); + return *this; + } + MariaDBHasher& add(CHARSET_INFO* cs, const utils::ConstString& str) + { + return add(cs, str.str(), str.length()); + } + uint32_t finalize() const + { + return (uint32_t)mPart1; + } +}; // A reference to MariaDB CHARSET_INFO. class Charset { -protected: - const struct charset_info_st * mCharset; -public: - Charset(CHARSET_INFO & cs) :mCharset(&cs) { } - Charset(CHARSET_INFO *cs) - :mCharset(cs ? cs : &my_charset_bin) - { } - Charset(uint32_t charsetNumber); - CHARSET_INFO & getCharset() const { return *mCharset; } - uint32_t hash(const char *data, uint64_t len) const - { - return MariaDBHasher().add(mCharset, data, len).finalize(); - } - bool eq(const std::string & str1, const std::string & str2) const - { - return mCharset->strnncollsp(str1.data(), str1.length(), - str2.data(), str2.length()) == 0; - } - int strnncollsp(const utils::ConstString &str1, - const utils::ConstString &str2) const - { - return mCharset->strnncollsp(str1.str(), str1.length(), - str2.str(), str2.length()); - } - int strnncollsp(const char *str1, size_t length1, - const char *str2, size_t length2) const - { - return mCharset->strnncollsp(str1, length1, str2, length2); - } - int strnncollsp(const unsigned char *str1, size_t length1, - const unsigned char *str2, size_t length2) const - { - return mCharset->strnncollsp((const char *) str1, length1, - (const char *) str2, length2); - } - bool test_if_important_data(const char *str, const char *end) const - { - if (mCharset->state & MY_CS_NOPAD) - return str < end; - return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end; + protected: + const struct charset_info_st* mCharset; - } - bool like(bool neg, - const utils::ConstString &subject, - const utils::ConstString &pattern) const - { - bool res= !mCharset->wildcmp(subject.str(), subject.end(), - pattern.str(), pattern.end(), - '\\','_','%'); - return neg ? !res : res; - } + public: + Charset(CHARSET_INFO& cs) : mCharset(&cs) + { + } + Charset(CHARSET_INFO* cs) : mCharset(cs ? cs : &my_charset_bin) + { + } + Charset(uint32_t charsetNumber); + CHARSET_INFO& getCharset() const + { + return *mCharset; + } + uint32_t hash(const char* data, uint64_t len) const + { + return MariaDBHasher().add(mCharset, data, len).finalize(); + } + bool eq(const std::string& str1, const std::string& str2) const + { + return mCharset->strnncollsp(str1.data(), str1.length(), str2.data(), str2.length()) == 0; + } + int strnncollsp(const utils::ConstString& str1, const utils::ConstString& str2) const + { + return mCharset->strnncollsp(str1.str(), str1.length(), str2.str(), str2.length()); + } + int strnncollsp(const char* str1, size_t length1, const char* str2, size_t length2) const + { + return mCharset->strnncollsp(str1, length1, str2, length2); + } + int strnncollsp(const unsigned char* str1, size_t length1, const unsigned char* str2, size_t length2) const + { + return mCharset->strnncollsp((const char*)str1, length1, (const char*)str2, length2); + } + bool test_if_important_data(const char* str, const char* end) const + { + if (mCharset->state & MY_CS_NOPAD) + return str < end; + return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end; + } + bool like(bool neg, const utils::ConstString& subject, const utils::ConstString& pattern) const + { + bool res = !mCharset->wildcmp(subject.str(), subject.end(), pattern.str(), pattern.end(), '\\', '_', '%'); + return neg ? !res : res; + } }; - -class CollationAwareHasher: public Charset +class CollationAwareHasher : public Charset { -public: - CollationAwareHasher(const Charset &cs) - :Charset(cs) - { } - inline uint32_t operator()(const std::string& s) const - { - return operator()(s.data(), s.length()); - } - inline uint32_t operator()(const char* data, uint64_t len) const - { - return Charset::hash(data, len); - } + public: + CollationAwareHasher(const Charset& cs) : Charset(cs) + { + } + inline uint32_t operator()(const std::string& s) const + { + return operator()(s.data(), s.length()); + } + inline uint32_t operator()(const char* data, uint64_t len) const + { + return Charset::hash(data, len); + } }; - -class CollationAwareComparator: public Charset +class CollationAwareComparator : public Charset { -public: - CollationAwareComparator(const Charset &cs) - :Charset(cs) - { } - bool operator()(const std::string & str1, const std::string & str2) const - { - return Charset::eq(str1, str2); - } + public: + CollationAwareComparator(const Charset& cs) : Charset(cs) + { + } + bool operator()(const std::string& str1, const std::string& str2) const + { + return Charset::eq(str1, str2); + } }; - - -} // end of namespace datatypes +} // end of namespace datatypes #endif diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 9d02a7312..4db3a0232 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2020 MariaDB Corporation +/* Copyright (C) 2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -22,71 +22,63 @@ namespace utils { - const uint8_t MAXLEGACYWIDTH = 8ULL; - const uint8_t MAXCOLUMNWIDTH = 16ULL; - - inline bool isWide(uint8_t width) - { - return width > MAXLEGACYWIDTH; - } - - inline bool isNarrow(uint8_t width) - { - return width <= MAXLEGACYWIDTH; - } - - /** @brief Map a DECIMAL precision to data width in bytes */ - inline uint8_t widthByPrecision(unsigned p) - { - if (LIKELY(p > 18 && p < 39)) - return 16; - - switch (p) - { - case 1: - case 2: - return 1; - - case 3: - case 4: - return 2; - - case 5: - case 6: - case 7: - case 8: - case 9: - return 4; - - case 10: - case 11: - case 12: - case 13: - case 14: - case 15: - case 16: - case 17: - case 18: - return 8; - - default: - return 16; - } - } - - inline uint8_t precisionByWidth(unsigned w) - { - switch(w) - { - case 16: - return 38; - // In case we will support decimals that spans 32 bytes. - default: - return 65; - } - } - +const uint8_t MAXLEGACYWIDTH = 8ULL; +const uint8_t MAXCOLUMNWIDTH = 16ULL; +inline bool isWide(uint8_t width) +{ + return width > MAXLEGACYWIDTH; } -#endif // UTILS_COLWIDTH_H +inline bool isNarrow(uint8_t width) +{ + return width <= MAXLEGACYWIDTH; +} + +/** @brief Map a DECIMAL precision to data width in bytes */ +inline uint8_t widthByPrecision(unsigned p) +{ + if (LIKELY(p > 18 && p < 39)) + return 16; + + switch (p) + { + case 1: + case 2: return 1; + + case 3: + case 4: return 2; + + case 5: + case 6: + case 7: + case 8: + case 9: return 4; + + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: return 8; + + default: return 16; + } +} + +inline uint8_t precisionByWidth(unsigned w) +{ + switch (w) + { + case 16: return 38; + // In case we will support decimals that spans 32 bytes. + default: return 65; + } +} + +} // namespace utils + +#endif // UTILS_COLWIDTH_H diff --git a/utils/common/conststring.h b/utils/common/conststring.h index 685900aac..14dc63101 100644 --- a/utils/common/conststring.h +++ b/utils/common/conststring.h @@ -15,29 +15,36 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef MARIADB_CONSTSTRING_H #define MARIADB_CONSTSTRING_H - namespace utils { - class ConstString { -protected: - const char *mStr; + protected: + const char* mStr; size_t mLength; -public: - ConstString(const char *str, size_t length) - :mStr(str), mLength(length) - { } - explicit ConstString(const std::string &str) - :mStr(str.data()), mLength(str.length()) - { } - const char *str() const { return mStr; } - const char *end() const { return mStr + mLength; } - size_t length() const { return mLength; } + + public: + ConstString(const char* str, size_t length) : mStr(str), mLength(length) + { + } + explicit ConstString(const std::string& str) : mStr(str.data()), mLength(str.length()) + { + } + const char* str() const + { + return mStr; + } + const char* end() const + { + return mStr + mLength; + } + size_t length() const + { + return mLength; + } std::string toString() const { return std::string(mStr, mLength); @@ -46,19 +53,19 @@ public: { return mLength == 1 && mStr[0] == ch; } - bool eq(const ConstString &rhs) const + bool eq(const ConstString& rhs) const { return mLength == rhs.mLength && !memcmp(mStr, rhs.mStr, mLength); } - ConstString & rtrimZero() + ConstString& rtrimZero() { - for ( ; mLength && mStr[mLength - 1] == '\0'; mLength--) - { } + for (; mLength && mStr[mLength - 1] == '\0'; mLength--) + { + } return *this; } }; +} // namespace utils -} // namespace utils - -#endif // MARIADB_CONSTSTRING_H +#endif // MARIADB_CONSTSTRING_H diff --git a/utils/common/crashtrace.cpp b/utils/common/crashtrace.cpp index 5e8bd3c2f..e40d97aa0 100644 --- a/utils/common/crashtrace.cpp +++ b/utils/common/crashtrace.cpp @@ -27,26 +27,26 @@ void fatalHandler(int sig) { - char filename[128]; - void* addrs[128]; - snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid()); - FILE* logfile = fopen(filename, "w"); - char s[30]; - struct tm tim; - time_t now; - now = time(NULL); - tim = *(localtime(&now)); - strftime(s, 30, "%F %T", &tim); - fprintf(logfile, "Date/time: %s\n", s); - fprintf(logfile, "Signal: %d\n\n", sig); - fflush(logfile); - int fd = fileno(logfile); - int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0])); - backtrace_symbols_fd(addrs, count, fd); - fclose(logfile); - struct sigaction sigact; - memset(&sigact, 0, sizeof(sigact)); - sigact.sa_handler = SIG_DFL; - sigaction(sig, &sigact, NULL); - raise(sig); + char filename[128]; + void* addrs[128]; + snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid()); + FILE* logfile = fopen(filename, "w"); + char s[30]; + struct tm tim; + time_t now; + now = time(NULL); + tim = *(localtime(&now)); + strftime(s, 30, "%F %T", &tim); + fprintf(logfile, "Date/time: %s\n", s); + fprintf(logfile, "Signal: %d\n\n", sig); + fflush(logfile); + int fd = fileno(logfile); + int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0])); + backtrace_symbols_fd(addrs, count, fd); + fclose(logfile); + struct sigaction sigact; + memset(&sigact, 0, sizeof(sigact)); + sigact.sa_handler = SIG_DFL; + sigaction(sig, &sigact, NULL); + raise(sig); } diff --git a/utils/common/fixedallocator.cpp b/utils/common/fixedallocator.cpp index 5bb8bc7c4..2799acdb0 100644 --- a/utils/common/fixedallocator.cpp +++ b/utils/common/fixedallocator.cpp @@ -16,11 +16,11 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ -//This is one of the first files we compile, check the compiler... +// This is one of the first files we compile, check the compiler... #if defined(__GNUC__) #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1) #error "This is a very old GCC, and it's probably not going to work." @@ -42,80 +42,78 @@ using namespace boost; namespace utils { - FixedAllocator::FixedAllocator(const FixedAllocator& f) { - elementCount = f.elementCount; - elementSize = f.elementSize; - tmpSpace = f.tmpSpace; - capacityRemaining = 0; - currentlyStored = 0; - useLock = f.useLock; - lock = false; - + elementCount = f.elementCount; + elementSize = f.elementSize; + tmpSpace = f.tmpSpace; + capacityRemaining = 0; + currentlyStored = 0; + useLock = f.useLock; + lock = false; } FixedAllocator& FixedAllocator::operator=(const FixedAllocator& f) { - elementCount = f.elementCount; - elementSize = f.elementSize; - tmpSpace = f.tmpSpace; - useLock = f.useLock; - lock = false; - deallocateAll(); - return *this; + elementCount = f.elementCount; + elementSize = f.elementSize; + tmpSpace = f.tmpSpace; + useLock = f.useLock; + lock = false; + deallocateAll(); + return *this; } void FixedAllocator::setUseLock(bool useIt) { - useLock = useIt; + useLock = useIt; } void FixedAllocator::setAllocSize(uint allocSize) { - elementSize = allocSize; + elementSize = allocSize; } void FixedAllocator::newBlock() { - shared_array next; + shared_array next; - capacityRemaining = elementCount * elementSize; + capacityRemaining = elementCount * elementSize; - if (!tmpSpace || mem.size() == 0) - { - next.reset(new uint8_t[elementCount * elementSize]); - mem.push_back(next); - nextAlloc = next.get(); - } - else - { - currentlyStored = 0; - nextAlloc = mem.front().get(); - } + if (!tmpSpace || mem.size() == 0) + { + next.reset(new uint8_t[elementCount * elementSize]); + mem.push_back(next); + nextAlloc = next.get(); + } + else + { + currentlyStored = 0; + nextAlloc = mem.front().get(); + } } void FixedAllocator::truncateBy(uint32_t amt) { - if (useLock) - getSpinlock(lock); - nextAlloc -= amt; - capacityRemaining += amt; - currentlyStored -= amt; - if (useLock) - releaseSpinlock(lock); + if (useLock) + getSpinlock(lock); + nextAlloc -= amt; + capacityRemaining += amt; + currentlyStored -= amt; + if (useLock) + releaseSpinlock(lock); } void FixedAllocator::deallocateAll() { - mem.clear(); - currentlyStored = 0; - capacityRemaining = 0; + mem.clear(); + currentlyStored = 0; + capacityRemaining = 0; } uint64_t FixedAllocator::getMemUsage() const { - return (mem.size() * elementCount * elementSize); + return (mem.size() * elementCount * elementSize); } -} +} // namespace utils diff --git a/utils/common/fixedallocator.h b/utils/common/fixedallocator.h index 9e716a133..66b9269c8 100644 --- a/utils/common/fixedallocator.h +++ b/utils/common/fixedallocator.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ /* This allocator is for frequent small allocations that all get deallocated at once. It allocates large blocks of memory from the system and distributes 'allocsize' @@ -49,94 +49,102 @@ namespace utils { - class FixedAllocator { -public: - EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize + public: + EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize - EXPORT FixedAllocator() : - capacityRemaining(0), - elementCount(DEFAULT_NUM_ELEMENTS), - elementSize(0), - currentlyStored(0), - tmpSpace(false), - nextAlloc(0), - useLock(false), - lock(false) {} - EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false, - unsigned long numElements = DEFAULT_NUM_ELEMENTS) : - capacityRemaining(0), - elementCount(numElements), - elementSize(allocSize), - currentlyStored(0), - tmpSpace(isTmpSpace), - nextAlloc(0), - useLock(false), - lock(false) {} - EXPORT FixedAllocator(const FixedAllocator&); - EXPORT FixedAllocator& operator=(const FixedAllocator&); - virtual ~FixedAllocator() {} + EXPORT FixedAllocator() + : capacityRemaining(0) + , elementCount(DEFAULT_NUM_ELEMENTS) + , elementSize(0) + , currentlyStored(0) + , tmpSpace(false) + , nextAlloc(0) + , useLock(false) + , lock(false) + { + } + EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false, + unsigned long numElements = DEFAULT_NUM_ELEMENTS) + : capacityRemaining(0) + , elementCount(numElements) + , elementSize(allocSize) + , currentlyStored(0) + , tmpSpace(isTmpSpace) + , nextAlloc(0) + , useLock(false) + , lock(false) + { + } + EXPORT FixedAllocator(const FixedAllocator&); + EXPORT FixedAllocator& operator=(const FixedAllocator&); + virtual ~FixedAllocator() + { + } - EXPORT void* allocate(); - EXPORT void* allocate(uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead) - EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution - void deallocate() { } // does nothing - EXPORT void deallocateAll(); // drops all memory in use - EXPORT uint64_t getMemUsage() const; - void setUseLock(bool); - void setAllocSize(uint); + EXPORT void* allocate(); + EXPORT void* allocate( + uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead) + EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution + void deallocate() + { + } // does nothing + EXPORT void deallocateAll(); // drops all memory in use + EXPORT uint64_t getMemUsage() const; + void setUseLock(bool); + void setAllocSize(uint); -private: - void newBlock(); + private: + void newBlock(); - std::vector > mem; - unsigned long capacityRemaining; - uint64_t elementCount; - unsigned long elementSize; - uint64_t currentlyStored; - bool tmpSpace; - uint8_t* nextAlloc; - bool useLock; - std::atomic lock; + std::vector > mem; + unsigned long capacityRemaining; + uint64_t elementCount; + unsigned long elementSize; + uint64_t currentlyStored; + bool tmpSpace; + uint8_t* nextAlloc; + bool useLock; + std::atomic lock; }; inline void* FixedAllocator::allocate() { - void* ret; + void* ret; - if (useLock) - getSpinlock(lock); - if (capacityRemaining < elementSize) - newBlock(); + if (useLock) + getSpinlock(lock); + if (capacityRemaining < elementSize) + newBlock(); - ret = nextAlloc; - nextAlloc += elementSize; - capacityRemaining -= elementSize; - currentlyStored += elementSize; - if (useLock) - releaseSpinlock(lock); - return ret; + ret = nextAlloc; + nextAlloc += elementSize; + capacityRemaining -= elementSize; + currentlyStored += elementSize; + if (useLock) + releaseSpinlock(lock); + return ret; } inline void* FixedAllocator::allocate(uint32_t len) { - void* ret; + void* ret; - if (useLock) - getSpinlock(lock); - if (capacityRemaining < len) - newBlock(); + if (useLock) + getSpinlock(lock); + if (capacityRemaining < len) + newBlock(); - ret = nextAlloc; - nextAlloc += len; - capacityRemaining -= len; - currentlyStored += len; - if (useLock) - releaseSpinlock(lock); - return ret; + ret = nextAlloc; + nextAlloc += len; + capacityRemaining -= len; + currentlyStored += len; + if (useLock) + releaseSpinlock(lock); + return ret; } #undef EXPORT -} // namespace +} // namespace utils #endif diff --git a/utils/common/genericparser.h b/utils/common/genericparser.h index 2e23d9568..87d72f744 100644 --- a/utils/common/genericparser.h +++ b/utils/common/genericparser.h @@ -15,33 +15,30 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef GENERICPARSER_H #define GENERICPARSER_H #include "conststring.h" - namespace genericparser { - using utils::ConstString; - class Tokenizer { -protected: - const char *mStr; - const char *mEnd; -public: - explicit Tokenizer(const char *str, size_t length) - :mStr(str), mEnd(str + length) - { } + protected: + const char* mStr; + const char* mEnd; + + public: + explicit Tokenizer(const char* str, size_t length) : mStr(str), mEnd(str + length) + { + } size_t length() const { return mEnd - mStr; } - const char *ptr() const + const char* ptr() const { return mStr; } @@ -66,18 +63,20 @@ public: { if (!isSpace()) return ConstString(nullptr, 0); - const char *start = mStr; - for ( ; isSpace() ; mStr++) - { } + const char* start = mStr; + for (; isSpace(); mStr++) + { + } return ConstString(start, mStr - start); } ConstString tokenDigits() { if (!isDigit()) return ConstString(nullptr, 0); - const char *start = mStr; - for ( ; isDigit() ; mStr++) - { } + const char* start = mStr; + for (; isDigit(); mStr++) + { + } return ConstString(start, mStr - start); } ConstString tokenChar(char chr) @@ -94,19 +93,19 @@ public: } }; - -class Parser: public Tokenizer +class Parser : public Tokenizer { -protected: + protected: bool mSyntaxError; -public: - explicit Parser(const char *str, size_t length) - :Tokenizer(str, length), mSyntaxError(false) - { } - explicit Parser(const std::string & str) - :Parser(str.data(), str.length()) - { } - Parser & skipLeadingSpaces() + + public: + explicit Parser(const char* str, size_t length) : Tokenizer(str, length), mSyntaxError(false) + { + } + explicit Parser(const std::string& str) : Parser(str.data(), str.length()) + { + } + Parser& skipLeadingSpaces() { tokenSpaces(); return *this; @@ -120,7 +119,7 @@ public: mSyntaxError = true; return false; } - const char *tokStart() const + const char* tokStart() const { return mStr; } @@ -129,16 +128,14 @@ public: return ConstString(mStr, 0); } - // A helper class template to set the parser syntax error // if A returned isNull() after parsing. - template - class SetSyntaxErrorOnNull :public A + template + class SetSyntaxErrorOnNull : public A { - public: - SetSyntaxErrorOnNull(Parser *p) - :A(p) + public: + SetSyntaxErrorOnNull(Parser* p) : A(p) { if (A::isNull()) p->setSyntaxError(); @@ -147,15 +144,14 @@ public: // A helper class template for a rule in the form: := [ ] - template - class Opt: public A + template + class Opt : public A { - public: - explicit Opt(const A &rhs) - :A(rhs) - { } - explicit Opt(Parser *p) - :A(p) + public: + explicit Opt(const A& rhs) : A(rhs) + { + } + explicit Opt(Parser* p) : A(p) { if (A::isNull() && !p->syntaxError()) A::operator=(A::empty(p)); @@ -170,7 +166,6 @@ public: // M - mandatory - this part is required during parse time // O - optional - this part is optional during parse time - // A helper class template for a rule in the form: := // i.e. both parts are mandatory at parse time // The value of is not important, and is created @@ -179,53 +174,52 @@ public: // Example: // - template - class UD2MM: public B + template + class UD2MM : public B { - public: - explicit UD2MM(Parser *p) - :B(A(p).isNull() ? B() :SetSyntaxErrorOnNull(p)) - { } - explicit UD2MM(const B & b) - :B(b) - { } - explicit UD2MM() - :B() - { } - bool isNull() const { return B::isNull(); } + public: + explicit UD2MM(Parser* p) : B(A(p).isNull() ? B() : SetSyntaxErrorOnNull(p)) + { + } + explicit UD2MM(const B& b) : B(b) + { + } + explicit UD2MM() : B() + { + } + bool isNull() const + { + return B::isNull(); + } }; // A helper class template for a rule in the form: := // i.e. both parts are mandatory at parse time. - template - class DD2MM: public A, - public B + template + class DD2MM : public A, public B { - public: + public: // Sets syntax error if was not followed by - explicit DD2MM(Parser *p) - :A(p), - B(A::isNull() ? B() : SetSyntaxErrorOnNull(p)) - { } - explicit DD2MM(const A & a, const B &b) - :A(b), B(b) - { } + explicit DD2MM(Parser* p) : A(p), B(A::isNull() ? B() : SetSyntaxErrorOnNull(p)) + { + } + explicit DD2MM(const A& a, const B& b) : A(b), B(b) + { + } }; // A helper class template for a rule in the form: := [ ] // i.e. is mandatory, is optional at parse time. - template - class DD2MO: public A, - public B + template + class DD2MO : public A, public B { - public: - explicit DD2MO(Parser *p) - :A(p), - B(A::isNull() ? B() : B(p)) - { } - explicit DD2MO(const A &a, const B &b) - :A(a), B(b) - { } + public: + explicit DD2MO(Parser* p) : A(p), B(A::isNull() ? B() : B(p)) + { + } + explicit DD2MO(const A& a, const B& b) : A(a), B(b) + { + } }; // A helper class template for a rule in the form: := [ ] @@ -233,73 +227,68 @@ public: // The value of is not important and is not included // into the target class, e.g. // [ ] - template - class UD2MO: public B + template + class UD2MO : public B { - public: - explicit UD2MO(Parser *p) - :B(A(p).isNull() ? B() : B(p)) - { } - explicit UD2MO(const B &b) - :B(b) - { } - explicit UD2MO() - :B() - { } + public: + explicit UD2MO(Parser* p) : B(A(p).isNull() ? B() : B(p)) + { + } + explicit UD2MO(const B& b) : B(b) + { + } + explicit UD2MO() : B() + { + } }; - // A helper class template for a rule in the form: := [ ] // i.e. is mandatory, is optional at parse time. // The result class derives from "A". // The result class puts "B" as a member. - template - class DM2MO: public A + template + class DM2MO : public A { - protected: + protected: B mB; - public: - explicit DM2MO(Parser *p) - :A(p), - mB(A::isNull() ? B() : B(p)) - { } - }; + public: + explicit DM2MO(Parser* p) : A(p), mB(A::isNull() ? B() : B(p)) + { + } + }; // A helper class template for a rule in the form: := [ ] // i.e. is optional, is mandatory at parse time. - template - class DD2OM: public Opt, - public B + template + class DD2OM : public Opt, public B { - public: - explicit DD2OM(Parser *p) - :Opt(p), B(p) + public: + explicit DD2OM(Parser* p) : Opt(p), B(p) { if (B::isNull()) p->setSyntaxError(); } - explicit DD2OM() - :Opt(A()) - { } - explicit DD2OM(const A & a, const B & b) - :Opt(a), B(b) - { } + explicit DD2OM() : Opt(A()) + { + } + explicit DD2OM(const A& a, const B& b) : Opt(a), B(b) + { + } }; // A helper class template for a rule in the form: := a | b - template - class Choice2: public Container + template + class Choice2 : public Container { - public: - explicit Choice2(const A & a) - :Container(a) - { } - explicit Choice2(const B & b) - :Container(b) - { } - explicit Choice2(Parser *p) - :Container(A(p)) + public: + explicit Choice2(const A& a) : Container(a) + { + } + explicit Choice2(const B& b) : Container(b) + { + } + explicit Choice2(Parser* p) : Container(A(p)) { if (Container::isNull() && !p->syntaxError()) *this = Choice2(B(p)); @@ -307,7 +296,6 @@ public: }; }; +} // namespace genericparser -} // namespace genericparser - -#endif // GENERICPARSER_H +#endif // GENERICPARSER_H diff --git a/utils/common/hasher.h b/utils/common/hasher.h index cd2ef210e..c13ab7633 100644 --- a/utils/common/hasher.h +++ b/utils/common/hasher.h @@ -39,331 +39,331 @@ namespace utils inline uint32_t rotl32(uint32_t x, int8_t r) { - return (x << r) | (x >> (32 - r)); + return (x << r) | (x >> (32 - r)); } inline uint32_t fmix(uint32_t h) { - h ^= h >> 16; - h *= 0x85ebca6b; - h ^= h >> 13; - h *= 0xc2b2ae35; - h ^= h >> 16; + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; - return h; + return h; } inline uint64_t fmix(uint64_t k) { - k ^= k >> 33; - k *= 0xff51afd7ed558ccdULL; - k ^= k >> 33; - k *= 0xc4ceb9fe1a85ec53ULL; - k ^= k >> 33; + k ^= k >> 33; + k *= 0xff51afd7ed558ccdULL; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53ULL; + k ^= k >> 33; - return k; + return k; } inline uint64_t rotl64(uint64_t x, int8_t r) { - return (x << r) | (x >> (64 - r)); + return (x << r) | (x >> (64 - r)); } class Hasher { -public: - inline uint32_t operator()(const std::string& s) const + public: + inline uint32_t operator()(const std::string& s) const + { + return operator()(s.data(), s.length()); + } + + inline uint32_t operator()(const char* data, uint64_t len) const + { + const int nblocks = len / 4; + + uint32_t h1 = 0; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4); + + for (int i = -nblocks; i; i++) { - return operator()(s.data(), s.length()); + uint32_t k1 = blocks[i]; + + k1 *= c1; + k1 = rotl32(k1, 15); + k1 *= c2; + + h1 ^= k1; + h1 = rotl32(h1, 13); + h1 = h1 * 5 + 0xe6546b64; } - inline uint32_t operator()(const char* data, uint64_t len) const + //---------- + // tail + + const uint8_t* tail = (const uint8_t*)(data + nblocks * 4); + + uint32_t k1 = 0; + + switch (len & 3) { - const int nblocks = len / 4; + case 3: + k1 ^= tail[2] << 16; + /* fall through */ + case 2: + k1 ^= tail[1] << 8; + /* fall through */ + case 1: + k1 ^= tail[0]; + k1 *= c1; + k1 = rotl32(k1, 15); + k1 *= c2; + h1 ^= k1; + }; - uint32_t h1 = 0; + //---------- + // finalization - const uint32_t c1 = 0xcc9e2d51; - const uint32_t c2 = 0x1b873593; + h1 ^= len; - //---------- - // body + h1 = fmix(h1); - const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4); - - for (int i = -nblocks; i; i++) - { - uint32_t k1 = blocks[i]; - - k1 *= c1; - k1 = rotl32(k1, 15); - k1 *= c2; - - h1 ^= k1; - h1 = rotl32(h1, 13); - h1 = h1 * 5 + 0xe6546b64; - } - - //---------- - // tail - - const uint8_t* tail = (const uint8_t*) (data + nblocks * 4); - - uint32_t k1 = 0; - - switch (len & 3) - { - case 3: - k1 ^= tail[2] << 16; - /* fall through */ - case 2: - k1 ^= tail[1] << 8; - /* fall through */ - case 1: - k1 ^= tail[0]; - k1 *= c1; - k1 = rotl32(k1, 15); - k1 *= c2; - h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; - - h1 = fmix(h1); - - return h1; - } + return h1; + } }; class Hasher_r { -public: - inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const + public: + inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const + { + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4); + + for (int i = -nblocks; i; i++) { - const int nblocks = len / 4; + uint32_t k1 = blocks[i]; - uint32_t h1 = seed; + k1 *= c1; + k1 = rotl32(k1, 15); + k1 *= c2; - const uint32_t c1 = 0xcc9e2d51; - const uint32_t c2 = 0x1b873593; - - //---------- - // body - - const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4); - - for (int i = -nblocks; i; i++) - { - uint32_t k1 = blocks[i]; - - k1 *= c1; - k1 = rotl32(k1, 15); - k1 *= c2; - - h1 ^= k1; - h1 = rotl32(h1, 13); - h1 = h1 * 5 + 0xe6546b64; - } - - //---------- - // tail - - const uint8_t* tail = (const uint8_t*) (data + nblocks * 4); - - uint32_t k1 = 0; - - switch (len & 3) - { - case 3: - k1 ^= tail[2] << 16; - /* FALLTHRU */ - - case 2: - k1 ^= tail[1] << 8; - /* FALLTHRU */ - - case 1: - k1 ^= tail[0]; - k1 *= c1; - k1 = rotl32(k1, 15); - k1 *= c2; - h1 ^= k1; - }; - - return h1; + h1 ^= k1; + h1 = rotl32(h1, 13); + h1 = h1 * 5 + 0xe6546b64; } - inline uint32_t finalize(uint32_t seed, uint32_t len) const + //---------- + // tail + + const uint8_t* tail = (const uint8_t*)(data + nblocks * 4); + + uint32_t k1 = 0; + + switch (len & 3) { - seed ^= len; - seed = fmix(seed); - return seed; - } + case 3: + k1 ^= tail[2] << 16; + /* FALLTHRU */ + + case 2: + k1 ^= tail[1] << 8; + /* FALLTHRU */ + + case 1: + k1 ^= tail[0]; + k1 *= c1; + k1 = rotl32(k1, 15); + k1 *= c2; + h1 ^= k1; + }; + + return h1; + } + + inline uint32_t finalize(uint32_t seed, uint32_t len) const + { + seed ^= len; + seed = fmix(seed); + return seed; + } }; class Hasher128 { -public: - inline uint64_t operator()(const char* data, uint64_t len) const + public: + inline uint64_t operator()(const char* data, uint64_t len) const + { + const int nblocks = len / 16; + + uint64_t h1 = 0; + uint64_t h2 = 0; + + const uint64_t c1 = 0x87c37b91114253d5ULL; + const uint64_t c2 = 0x4cf5ad432745937fULL; + + //---------- + // body + + const uint64_t* blocks = (const uint64_t*)(data); + + for (int i = 0; i < nblocks; i++) { - const int nblocks = len / 16; + uint64_t k1 = blocks[i * 2 + 0]; + uint64_t k2 = blocks[i * 2 + 1]; - uint64_t h1 = 0; - uint64_t h2 = 0; + k1 *= c1; + k1 = rotl64(k1, 31); + k1 *= c2; + h1 ^= k1; - const uint64_t c1 = 0x87c37b91114253d5ULL; - const uint64_t c2 = 0x4cf5ad432745937fULL; + h1 = rotl64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; - //---------- - // body + k2 *= c2; + k2 = rotl64(k2, 33); + k2 *= c1; + h2 ^= k2; - const uint64_t* blocks = (const uint64_t*) (data); - - for (int i = 0; i < nblocks; i++) - { - uint64_t k1 = blocks[i * 2 + 0]; - uint64_t k2 = blocks[i * 2 + 1]; - - k1 *= c1; - k1 = rotl64(k1, 31); - k1 *= c2; - h1 ^= k1; - - h1 = rotl64(h1, 27); - h1 += h2; - h1 = h1 * 5 + 0x52dce729; - - k2 *= c2; - k2 = rotl64(k2, 33); - k2 *= c1; - h2 ^= k2; - - h2 = rotl64(h2, 31); - h2 += h1; - h2 = h2 * 5 + 0x38495ab5; - } - - //---------- - // tail - - const uint8_t* tail = (const uint8_t*) (data + nblocks * 16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch (len & 15) - { - case 15: - k2 ^= uint64_t(tail[14]) << 48; - /* FALLTHRU */ - - case 14: - k2 ^= uint64_t(tail[13]) << 40; - /* FALLTHRU */ - - case 13: - k2 ^= uint64_t(tail[12]) << 32; - /* FALLTHRU */ - - case 12: - k2 ^= uint64_t(tail[11]) << 24; - /* FALLTHRU */ - - case 11: - k2 ^= uint64_t(tail[10]) << 16; - /* FALLTHRU */ - - case 10: - k2 ^= uint64_t(tail[9]) << 8; - /* FALLTHRU */ - - case 9: - k2 ^= uint64_t(tail[8]) << 0; - k2 *= c2; - k2 = rotl64(k2, 33); - k2 *= c1; - h2 ^= k2; - /* FALLTHRU */ - - case 8: - k1 ^= uint64_t(tail[7]) << 56; - /* FALLTHRU */ - - case 7: - k1 ^= uint64_t(tail[6]) << 48; - /* FALLTHRU */ - - case 6: - k1 ^= uint64_t(tail[5]) << 40; - /* FALLTHRU */ - - case 5: - k1 ^= uint64_t(tail[4]) << 32; - /* FALLTHRU */ - - case 4: - k1 ^= uint64_t(tail[3]) << 24; - /* FALLTHRU */ - - case 3: - k1 ^= uint64_t(tail[2]) << 16; - /* FALLTHRU */ - - case 2: - k1 ^= uint64_t(tail[1]) << 8; - /* FALLTHRU */ - - case 1: - k1 ^= uint64_t(tail[0]) << 0; - k1 *= c1; - k1 = rotl64(k1, 31); - k1 *= c2; - h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; - - h2 ^= len; - - h1 += h2; - - h2 += h1; - - h1 = fmix(h1); - - h2 = fmix(h2); - - h1 += h2; - - h2 += h1; - - return h1; + h2 = rotl64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; } + + //---------- + // tail + + const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (len & 15) + { + case 15: + k2 ^= uint64_t(tail[14]) << 48; + /* FALLTHRU */ + + case 14: + k2 ^= uint64_t(tail[13]) << 40; + /* FALLTHRU */ + + case 13: + k2 ^= uint64_t(tail[12]) << 32; + /* FALLTHRU */ + + case 12: + k2 ^= uint64_t(tail[11]) << 24; + /* FALLTHRU */ + + case 11: + k2 ^= uint64_t(tail[10]) << 16; + /* FALLTHRU */ + + case 10: + k2 ^= uint64_t(tail[9]) << 8; + /* FALLTHRU */ + + case 9: + k2 ^= uint64_t(tail[8]) << 0; + k2 *= c2; + k2 = rotl64(k2, 33); + k2 *= c1; + h2 ^= k2; + /* FALLTHRU */ + + case 8: + k1 ^= uint64_t(tail[7]) << 56; + /* FALLTHRU */ + + case 7: + k1 ^= uint64_t(tail[6]) << 48; + /* FALLTHRU */ + + case 6: + k1 ^= uint64_t(tail[5]) << 40; + /* FALLTHRU */ + + case 5: + k1 ^= uint64_t(tail[4]) << 32; + /* FALLTHRU */ + + case 4: + k1 ^= uint64_t(tail[3]) << 24; + /* FALLTHRU */ + + case 3: + k1 ^= uint64_t(tail[2]) << 16; + /* FALLTHRU */ + + case 2: + k1 ^= uint64_t(tail[1]) << 8; + /* FALLTHRU */ + + case 1: + k1 ^= uint64_t(tail[0]) << 0; + k1 *= c1; + k1 = rotl64(k1, 31); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h2 ^= len; + + h1 += h2; + + h2 += h1; + + h1 = fmix(h1); + + h2 = fmix(h2); + + h1 += h2; + + h2 += h1; + + return h1; + } }; // TODO a copy of these classes also exists in primitiveprocessor.h; consolidate class Hash128 { -public: - inline size_t operator()(const int128_t i) const - { - return (uint64_t)((int64_t) i); - } + public: + inline size_t operator()(const int128_t i) const + { + return (uint64_t)((int64_t)i); + } }; class Equal128 { -public: - inline bool operator()(const int128_t f1, const int128_t f2) const - { - return f1 == f2; - } + public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } }; //------------------------------------------------------------------------------ @@ -373,17 +373,19 @@ public: //------------------------------------------------------------------------------ class TupleHasher { -public: - TupleHasher(uint32_t len) : fHashLen(len) {} - inline uint64_t operator()(const uint8_t* hashKey) const - { - return fHasher(reinterpret_cast(hashKey), fHashLen); - } -private: - Hasher fHasher; - uint32_t fHashLen; -}; + public: + TupleHasher(uint32_t len) : fHashLen(len) + { + } + inline uint64_t operator()(const uint8_t* hashKey) const + { + return fHasher(reinterpret_cast(hashKey), fHashLen); + } + private: + Hasher fHasher; + uint32_t fHashLen; +}; //------------------------------------------------------------------------------ /** @brief class TupleComparator @@ -392,17 +394,19 @@ private: //------------------------------------------------------------------------------ class TupleComparator { -public: - TupleComparator(uint32_t len) : fCmpLen(len) {} - inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const - { - return (memcmp(hashKey1, hashKey2, fCmpLen) == 0); - } -private: - uint32_t fCmpLen; + public: + TupleComparator(uint32_t len) : fCmpLen(len) + { + } + inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const + { + return (memcmp(hashKey1, hashKey2, fCmpLen) == 0); + } + + private: + uint32_t fCmpLen; }; - -} +} // namespace utils #endif // UTILS_HASHER_H diff --git a/utils/common/hashfamily.h b/utils/common/hashfamily.h index b324ea856..6733beaf8 100644 --- a/utils/common/hashfamily.h +++ b/utils/common/hashfamily.h @@ -23,35 +23,33 @@ namespace utils { - class HashFamily { - public: - HashFamily(const utils::Hasher_r& h, - const uint64_t intermediateHash, - const uint64_t len, - const datatypes::MariaDBHasher& hM) : mHasher(h), - mMariaDBHasher(hM), - mHasher_rHash(intermediateHash), - mHasher_rLen(len) - { } + public: + HashFamily(const utils::Hasher_r& h, const uint64_t intermediateHash, const uint64_t len, + const datatypes::MariaDBHasher& hM) + : mHasher(h), mMariaDBHasher(hM), mHasher_rHash(intermediateHash), mHasher_rLen(len) + { + } - // Algorithm, seed and factor are taken from this discussion - // https://stackoverflow.com/questions/1646807/quick-and-simple-hash-code-combinations - inline uint64_t finalize() const - { - return (seed * factor + mHasher.finalize(mHasher_rHash, mHasher_rLen)) * factor + mMariaDBHasher.finalize(); - } - private: - constexpr static uint64_t seed = 1009ULL; - constexpr static uint64_t factor = 9176ULL; + // Algorithm, seed and factor are taken from this discussion + // https://stackoverflow.com/questions/1646807/quick-and-simple-hash-code-combinations + inline uint64_t finalize() const + { + return (seed * factor + mHasher.finalize(mHasher_rHash, mHasher_rLen)) * factor + + mMariaDBHasher.finalize(); + } - const utils::Hasher_r& mHasher; - const datatypes::MariaDBHasher& mMariaDBHasher; - const uint64_t mHasher_rHash; - const uint32_t mHasher_rLen; + private: + constexpr static uint64_t seed = 1009ULL; + constexpr static uint64_t factor = 9176ULL; + + const utils::Hasher_r& mHasher; + const datatypes::MariaDBHasher& mMariaDBHasher; + const uint64_t mHasher_rHash; + const uint32_t mHasher_rLen; }; -} +} // namespace utils #endif // vim:ts=2 sw=2: diff --git a/utils/common/mcs_basic_types.h b/utils/common/mcs_basic_types.h index 131fcdc68..2e1caf32b 100644 --- a/utils/common/mcs_basic_types.h +++ b/utils/common/mcs_basic_types.h @@ -14,15 +14,13 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ + MA 02110-1301, USA. +*/ #ifndef MCS_BASIC_TYPES_H_INCLUDED #define MCS_BASIC_TYPES_H_INCLUDED - using int128_t = __int128; using uint128_t = unsigned __int128; - -#endif // MCS_BASIC_TYPES_H_INCLUDED +#endif // MCS_BASIC_TYPES_H_INCLUDED // vim:ts=2 sw=2: diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 46ccc5abd..da2fd76e6 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -16,7 +16,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include "nullvaluemanip.h" #include @@ -25,234 +24,180 @@ using namespace execplan; namespace utils { - uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) { - switch (t) + switch (t) + { + case CalpontSystemCatalog::TINYINT: return joblist::TINYINTNULL; + + case CalpontSystemCatalog::SMALLINT: return joblist::SMALLINTNULL; + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return joblist::INTNULL; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return joblist::FLOATNULL; + + case CalpontSystemCatalog::DATE: return joblist::DATENULL; + + case CalpontSystemCatalog::BIGINT: return joblist::BIGINTNULL; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL; + + case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + + case CalpontSystemCatalog::TIMESTAMP: return joblist::TIMESTAMPNULL; + + case CalpontSystemCatalog::TIME: return joblist::TIMENULL; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::STRINT: { - case CalpontSystemCatalog::TINYINT: - return joblist::TINYINTNULL; + switch (colWidth) + { + case 1: return joblist::CHAR1NULL; - case CalpontSystemCatalog::SMALLINT: - return joblist::SMALLINTNULL; + case 2: return joblist::CHAR2NULL; - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return joblist::INTNULL; + case 3: + case 4: return joblist::CHAR4NULL; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return joblist::FLOATNULL; + case 5: + case 6: + case 7: + case 8: return joblist::CHAR8NULL; - case CalpontSystemCatalog::DATE: - return joblist::DATENULL; + default: throw logic_error("getNullValue() Can't return the NULL string"); + } - case CalpontSystemCatalog::BIGINT: - return joblist::BIGINTNULL; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return joblist::DOUBLENULL; - - case CalpontSystemCatalog::DATETIME: - return joblist::DATETIMENULL; - - case CalpontSystemCatalog::TIMESTAMP: - return joblist::TIMESTAMPNULL; - - case CalpontSystemCatalog::TIME: - return joblist::TIMENULL; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::STRINT: - { - switch (colWidth) - { - case 1: - return joblist::CHAR1NULL; - - case 2: - return joblist::CHAR2NULL; - - case 3: - case 4: - return joblist::CHAR4NULL; - - case 5: - case 6: - case 7: - case 8: - return joblist::CHAR8NULL; - - default: - throw logic_error("getNullValue() Can't return the NULL string"); - } - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - switch (colWidth) - { - case 1 : - return joblist::TINYINTNULL; - - case 2 : - return joblist::SMALLINTNULL; - - case 4 : - return joblist::INTNULL; - - default: - return joblist::BIGINTNULL; - } - - break; - } - - case CalpontSystemCatalog::UTINYINT: - return joblist::UTINYINTNULL; - - case CalpontSystemCatalog::USMALLINT: - return joblist::USMALLINTNULL; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return joblist::UINTNULL; - - case CalpontSystemCatalog::UBIGINT: - return joblist::UBIGINTNULL; - - case CalpontSystemCatalog::VARBINARY: - default: - ostringstream os; - os << "getNullValue(): got bad column type (" << t << - "). Width=" << colWidth << endl; - throw logic_error(os.str()); + break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + switch (colWidth) + { + case 1: return joblist::TINYINTNULL; + + case 2: return joblist::SMALLINTNULL; + + case 4: return joblist::INTNULL; + + default: return joblist::BIGINTNULL; + } + + break; + } + + case CalpontSystemCatalog::UTINYINT: return joblist::UTINYINTNULL; + + case CalpontSystemCatalog::USMALLINT: return joblist::USMALLINTNULL; + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return joblist::UINTNULL; + + case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTNULL; + + case CalpontSystemCatalog::VARBINARY: + default: + ostringstream os; + os << "getNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl; + throw logic_error(os.str()); + } } int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) { - switch (t) + switch (t) + { + case CalpontSystemCatalog::TINYINT: return (int64_t)((int8_t)joblist::TINYINTNULL); + + case CalpontSystemCatalog::SMALLINT: return (int64_t)((int16_t)joblist::SMALLINTNULL); + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: return (int64_t)((int32_t)joblist::INTNULL); + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: return (int64_t)((int32_t)joblist::FLOATNULL); + + case CalpontSystemCatalog::DATE: return (int64_t)((int32_t)joblist::DATENULL); + + case CalpontSystemCatalog::BIGINT: return joblist::BIGINTNULL; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL; + + case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + + case CalpontSystemCatalog::TIMESTAMP: return joblist::TIMESTAMPNULL; + + case CalpontSystemCatalog::TIME: return joblist::TIMENULL; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::STRINT: { - case CalpontSystemCatalog::TINYINT: - return (int64_t) ((int8_t) joblist::TINYINTNULL); + switch (colWidth) + { + case 1: return (int64_t)((int8_t)joblist::CHAR1NULL); - case CalpontSystemCatalog::SMALLINT: - return (int64_t) ((int16_t) joblist::SMALLINTNULL); + case 2: return (int64_t)((int16_t)joblist::CHAR2NULL); - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return (int64_t) ((int32_t) joblist::INTNULL); + case 3: + case 4: return (int64_t)((int32_t)joblist::CHAR4NULL); - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (int64_t) ((int32_t) joblist::FLOATNULL); + case 5: + case 6: + case 7: + case 8: return joblist::CHAR8NULL; - case CalpontSystemCatalog::DATE: - return (int64_t) ((int32_t) joblist::DATENULL); - - case CalpontSystemCatalog::BIGINT: - return joblist::BIGINTNULL; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return joblist::DOUBLENULL; - - case CalpontSystemCatalog::DATETIME: - return joblist::DATETIMENULL; - - case CalpontSystemCatalog::TIMESTAMP: - return joblist::TIMESTAMPNULL; - - case CalpontSystemCatalog::TIME: - return joblist::TIMENULL; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::STRINT: - { - switch (colWidth) - { - case 1: - return (int64_t) ((int8_t) joblist::CHAR1NULL); - - case 2: - return (int64_t) ((int16_t) joblist::CHAR2NULL); - - case 3: - case 4: - return (int64_t) ((int32_t) joblist::CHAR4NULL); - - case 5: - case 6: - case 7: - case 8: - return joblist::CHAR8NULL; - - default: - throw logic_error("getSignedNullValue() Can't return the NULL string"); - } - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - switch (colWidth) - { - case 1 : - return (int64_t) ((int8_t) joblist::TINYINTNULL); - - case 2 : - return (int64_t) ((int16_t) joblist::SMALLINTNULL); - - case 4 : - return (int64_t) ((int32_t) joblist::INTNULL); - - case 8: - return joblist::BIGINTNULL; - - default: - ostringstream os; - os << "getSignedNullValue(): got bad column width (" << t << - "). Width=" << colWidth << endl; - throw logic_error(os.str()); - } - - break; - } - - case CalpontSystemCatalog::UTINYINT: - return (int64_t) ((int8_t) joblist::UTINYINTNULL); - - case CalpontSystemCatalog::USMALLINT: - return (int64_t) ((int16_t) joblist::USMALLINTNULL); - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return (int64_t) ((int32_t) joblist::UINTNULL); - - case CalpontSystemCatalog::UBIGINT: - return (int64_t)joblist::UBIGINTNULL; - - case CalpontSystemCatalog::LONGDOUBLE: - return (int64_t)joblist::LONGDOUBLENULL; - - case CalpontSystemCatalog::VARBINARY: - default: - ostringstream os; - os << "getSignedNullValue(): got bad column type (" << t << - "). Width=" << colWidth << endl; - throw logic_error(os.str()); + default: throw logic_error("getSignedNullValue() Can't return the NULL string"); + } + break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + switch (colWidth) + { + case 1: return (int64_t)((int8_t)joblist::TINYINTNULL); + + case 2: return (int64_t)((int16_t)joblist::SMALLINTNULL); + + case 4: return (int64_t)((int32_t)joblist::INTNULL); + + case 8: return joblist::BIGINTNULL; + + default: + ostringstream os; + os << "getSignedNullValue(): got bad column width (" << t << "). Width=" << colWidth << endl; + throw logic_error(os.str()); + } + + break; + } + + case CalpontSystemCatalog::UTINYINT: return (int64_t)((int8_t)joblist::UTINYINTNULL); + + case CalpontSystemCatalog::USMALLINT: return (int64_t)((int16_t)joblist::USMALLINTNULL); + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return (int64_t)((int32_t)joblist::UINTNULL); + + case CalpontSystemCatalog::UBIGINT: return (int64_t)joblist::UBIGINTNULL; + + case CalpontSystemCatalog::LONGDOUBLE: return (int64_t)joblist::LONGDOUBLENULL; + + case CalpontSystemCatalog::VARBINARY: + default: + ostringstream os; + os << "getSignedNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl; + throw logic_error(os.str()); + } } - -} +} // namespace utils diff --git a/utils/common/nullvaluemanip.h b/utils/common/nullvaluemanip.h index 307665e7c..956973c98 100644 --- a/utils/common/nullvaluemanip.h +++ b/utils/common/nullvaluemanip.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef NULLVALUEMANIP_H #define NULLVALUEMANIP_H @@ -23,13 +22,11 @@ namespace utils { - // returns the NULL value for our 'numeric' types including short strings. // The width is only relevant for long string columns. uint64_t getNullValue(execplan::CalpontSystemCatalog::ColDataType, uint32_t colWidth = 0); int64_t getSignedNullValue(execplan::CalpontSystemCatalog::ColDataType, uint32_t colWidth = 0); +} // namespace utils -} - -#endif // NULLVALUEMANIP_H +#endif // NULLVALUEMANIP_H diff --git a/utils/common/pipe.h b/utils/common/pipe.h index 427f17ac0..d7cf301d3 100644 --- a/utils/common/pipe.h +++ b/utils/common/pipe.h @@ -24,11 +24,12 @@ class Pipe { int fd[2]; -public: + + public: Pipe() { - fd[0]= 0; - fd[1]= 0; + fd[0] = 0; + fd[1] = 0; } ~Pipe() { @@ -46,35 +47,35 @@ public: { return ::pipe(fd) == -1; } - bool init() // TODO: remove this + bool init() // TODO: remove this { return open(); } - ssize_t read(char *str, size_t nbytes) + ssize_t read(char* str, size_t nbytes) { return ::read(fd[0], str, nbytes); } - ssize_t readtm(const struct timeval &tv, void *buf, size_t nbytes) + ssize_t readtm(const struct timeval& tv, void* buf, size_t nbytes) { - fd_set rfds; - FD_ZERO(&rfds); - FD_SET(fd[0], &rfds); - struct timeval tmptv = tv; - int retval = select(fd[0] + 1, &rfds, NULL, NULL, &tmptv); - if (retval == -1) - return -1; - if (!retval) - { - errno= ETIME; - return -1; - } - return ::read(fd[0], buf, nbytes); + fd_set rfds; + FD_ZERO(&rfds); + FD_SET(fd[0], &rfds); + struct timeval tmptv = tv; + int retval = select(fd[0] + 1, &rfds, NULL, NULL, &tmptv); + if (retval == -1) + return -1; + if (!retval) + { + errno = ETIME; + return -1; + } + return ::read(fd[0], buf, nbytes); } - ssize_t write(const char *str, size_t nbytes) + ssize_t write(const char* str, size_t nbytes) { return ::write(fd[1], str, nbytes); } - ssize_t write(const std::string &str) + ssize_t write(const std::string& str) { return write(str.data(), str.length()); } @@ -83,15 +84,14 @@ public: if (fd[0]) { ::close(fd[0]); - fd[0]= 0; + fd[0] = 0; } if (fd[1]) { ::close(fd[1]); - fd[1]= 0; + fd[1] = 0; } } }; - -#endif // PIPE_H_INCLUDED +#endif // PIPE_H_INCLUDED diff --git a/utils/common/poolallocator.cpp b/utils/common/poolallocator.cpp index 684e7049c..84e0e6b25 100644 --- a/utils/common/poolallocator.cpp +++ b/utils/common/poolallocator.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ #include //#define NDEBUG @@ -31,72 +31,71 @@ using namespace boost; namespace utils { - PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v) { - allocSize = v.allocSize; - tmpSpace = v.tmpSpace; - useLock = v.useLock; - deallocateAll(); - return *this; + allocSize = v.allocSize; + tmpSpace = v.tmpSpace; + useLock = v.useLock; + deallocateAll(); + return *this; } void PoolAllocator::deallocateAll() { - capacityRemaining = 0; - nextAlloc = NULL; - memUsage = 0; - mem.clear(); - oob.clear(); + capacityRemaining = 0; + nextAlloc = NULL; + memUsage = 0; + mem.clear(); + oob.clear(); } void PoolAllocator::newBlock() { - shared_array next; + shared_array next; - capacityRemaining = allocSize; + capacityRemaining = allocSize; - if (!tmpSpace || mem.size() == 0) - { - next.reset(new uint8_t[allocSize]); - mem.push_back(next); - nextAlloc = next.get(); - } - else - nextAlloc = mem.front().get(); + if (!tmpSpace || mem.size() == 0) + { + next.reset(new uint8_t[allocSize]); + mem.push_back(next); + nextAlloc = next.get(); + } + else + nextAlloc = mem.front().get(); } -void * PoolAllocator::allocOOB(uint64_t size) +void* PoolAllocator::allocOOB(uint64_t size) { - OOBMemInfo memInfo; + OOBMemInfo memInfo; - memUsage += size; - memInfo.mem.reset(new uint8_t[size]); - memInfo.size = size; - void *ret = (void*) memInfo.mem.get(); - oob[ret] = memInfo; - return ret; + memUsage += size; + memInfo.mem.reset(new uint8_t[size]); + memInfo.size = size; + void* ret = (void*)memInfo.mem.get(); + oob[ret] = memInfo; + return ret; } void PoolAllocator::deallocate(void* p) { - bool _false = false; - if (useLock) - while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) - _false = false; - OutOfBandMap::iterator it = oob.find(p); + bool _false = false; + if (useLock) + while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) + _false = false; + OutOfBandMap::iterator it = oob.find(p); - if (it == oob.end()) - { - if (useLock) - lock.store(false, std::memory_order_release); - return; - } - - memUsage -= it->second.size; - oob.erase(it); + if (it == oob.end()) + { if (useLock) - lock.store(false, std::memory_order_release); + lock.store(false, std::memory_order_release); + return; + } + + memUsage -= it->second.size; + oob.erase(it); + if (useLock) + lock.store(false, std::memory_order_release); } -} +} // namespace utils diff --git a/utils/common/poolallocator.h b/utils/common/poolallocator.h index d02db5d55..b0f4dcf12 100644 --- a/utils/common/poolallocator.h +++ b/utils/common/poolallocator.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ /* This allocator is an attempt to consolidate small allocations and deallocations to boost performance and reduce mem fragmentation. */ @@ -35,101 +35,107 @@ namespace utils { - class PoolAllocator { -public: - static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages + public: + static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages - explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false, bool _useLock = false) : - allocSize(windowSize), - tmpSpace(isTmpSpace), - capacityRemaining(0), - memUsage(0), - nextAlloc(0), - useLock(_useLock), - lock(false) { } - PoolAllocator(const PoolAllocator& p) : - allocSize(p.allocSize), - tmpSpace(p.tmpSpace), - capacityRemaining(0), - memUsage(0), - nextAlloc(0), - useLock(p.useLock), - lock(false) { } - virtual ~PoolAllocator() {} + explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false, + bool _useLock = false) + : allocSize(windowSize) + , tmpSpace(isTmpSpace) + , capacityRemaining(0) + , memUsage(0) + , nextAlloc(0) + , useLock(_useLock) + , lock(false) + { + } + PoolAllocator(const PoolAllocator& p) + : allocSize(p.allocSize) + , tmpSpace(p.tmpSpace) + , capacityRemaining(0) + , memUsage(0) + , nextAlloc(0) + , useLock(p.useLock) + , lock(false) + { + } + virtual ~PoolAllocator() + { + } - PoolAllocator& operator=(const PoolAllocator&); + PoolAllocator& operator=(const PoolAllocator&); - void* allocate(uint64_t size); - void deallocate(void* p); - void deallocateAll(); + void* allocate(uint64_t size); + void deallocate(void* p); + void deallocateAll(); - inline uint64_t getMemUsage() const - { - return memUsage; - } - unsigned getWindowSize() const - { - return allocSize; - } + inline uint64_t getMemUsage() const + { + return memUsage; + } + unsigned getWindowSize() const + { + return allocSize; + } - void setUseLock(bool ul) - { - useLock = ul; - } + void setUseLock(bool ul) + { + useLock = ul; + } -private: - void newBlock(); - void *allocOOB(uint64_t size); + private: + void newBlock(); + void* allocOOB(uint64_t size); - unsigned allocSize; - std::vector > mem; - bool tmpSpace; - unsigned capacityRemaining; - uint64_t memUsage; - uint8_t* nextAlloc; - bool useLock; - std::atomic lock; + unsigned allocSize; + std::vector > mem; + bool tmpSpace; + unsigned capacityRemaining; + uint64_t memUsage; + uint8_t* nextAlloc; + bool useLock; + std::atomic lock; - struct OOBMemInfo - { - boost::shared_array mem; - uint64_t size; - }; - typedef std::map OutOfBandMap; - OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd + struct OOBMemInfo + { + boost::shared_array mem; + uint64_t size; + }; + typedef std::map OutOfBandMap; + OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd }; inline void* PoolAllocator::allocate(uint64_t size) { - void *ret; - bool _false = false; + void* ret; + bool _false = false; + if (useLock) + while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) + _false = false; + + if (size > allocSize) + { + ret = allocOOB(size); if (useLock) - while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) - _false = false; - - if (size > allocSize) - { - ret = allocOOB(size); - if (useLock) - lock.store(false, std::memory_order_release); - return ret; - } - - if (size > capacityRemaining) - newBlock(); - - ret = (void*) nextAlloc; - nextAlloc += size; - capacityRemaining -= size; - memUsage += size; - if (useLock) - lock.store(false, std::memory_order_release); + lock.store(false, std::memory_order_release); return ret; + } + + if (size > capacityRemaining) + newBlock(); + + ret = (void*)nextAlloc; + nextAlloc += size; + capacityRemaining -= size; + memUsage += size; + if (useLock) + lock.store(false, std::memory_order_release); + return ret; } -} +} // namespace utils #endif diff --git a/utils/common/robin_hood.h b/utils/common/robin_hood.h index 5304ca53c..3cf3e0fae 100644 --- a/utils/common/robin_hood.h +++ b/utils/common/robin_hood.h @@ -34,61 +34,64 @@ #define ROBIN_HOOD_H_INCLUDED // see https://semver.org/ -#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes -#define ROBIN_HOOD_VERSION_MINOR 10 // for adding functionality in a backwards-compatible manner -#define ROBIN_HOOD_VERSION_PATCH 0 // for backwards-compatible bug fixes +#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes +#define ROBIN_HOOD_VERSION_MINOR 10 // for adding functionality in a backwards-compatible manner +#define ROBIN_HOOD_VERSION_PATCH 0 // for backwards-compatible bug fixes #include #include #include #include -#include // only to support hash of smart pointers +#include // only to support hash of smart pointers #include #include #include #include #if __cplusplus >= 201703L -# include +#include #endif // #define ROBIN_HOOD_LOG_ENABLED #ifdef ROBIN_HOOD_LOG_ENABLED -# include -# define ROBIN_HOOD_LOG(...) \ - std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; +#include +#define ROBIN_HOOD_LOG(...) std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; #else -# define ROBIN_HOOD_LOG(x) +#define ROBIN_HOOD_LOG(x) #endif // #define ROBIN_HOOD_TRACE_ENABLED #ifdef ROBIN_HOOD_TRACE_ENABLED -# include -# define ROBIN_HOOD_TRACE(...) \ - std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; +#include +#define ROBIN_HOOD_TRACE(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; #else -# define ROBIN_HOOD_TRACE(x) +#define ROBIN_HOOD_TRACE(x) #endif // #define ROBIN_HOOD_COUNT_ENABLED #ifdef ROBIN_HOOD_COUNT_ENABLED -# include -# define ROBIN_HOOD_COUNT(x) ++counts().x; -namespace robin_hood { -struct Counts { - uint64_t shiftUp{}; - uint64_t shiftDown{}; +#include +#define ROBIN_HOOD_COUNT(x) ++counts().x; +namespace robin_hood +{ +struct Counts +{ + uint64_t shiftUp{}; + uint64_t shiftDown{}; }; -inline std::ostream& operator<<(std::ostream& os, Counts const& c) { - return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; +inline std::ostream& operator<<(std::ostream& os, Counts const& c) +{ + return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; } -static Counts& counts() { - static Counts counts{}; - return counts; +static Counts& counts() +{ + static Counts counts{}; + return counts; } -} // namespace robin_hood +} // namespace robin_hood #else -# define ROBIN_HOOD_COUNT(x) +#define ROBIN_HOOD_COUNT(x) #endif // all non-argument macros should use this facility. See @@ -100,104 +103,103 @@ static Counts& counts() { // bitness #if SIZE_MAX == UINT32_MAX -# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32 +#define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32 #elif SIZE_MAX == UINT64_MAX -# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64 +#define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64 #else -# error Unsupported bitness +#error Unsupported bitness #endif // endianess #ifdef _MSC_VER -# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1 -# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0 +#define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1 +#define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0 #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \ - (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #endif // inline #ifdef _MSC_VER -# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline) +#define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline) #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline)) +#define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline)) #endif // exceptions #if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0 +#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0 #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1 +#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1 #endif // count leading/trailing bits #if !defined(ROBIN_HOOD_DISABLE_INTRINSICS) -# ifdef _MSC_VER -# if ROBIN_HOOD(BITNESS) == 32 -# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward -# else -# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 -# endif -# include -# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) -# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ - [](size_t mask) noexcept -> int { \ - unsigned long index; \ - return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) \ - : ROBIN_HOOD(BITNESS); \ - }(x) -# else -# if ROBIN_HOOD(BITNESS) == 32 -# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl -# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl -# else -# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll -# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll -# endif -# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) -# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) -# endif +#ifdef _MSC_VER +#if ROBIN_HOOD(BITNESS) == 32 +#define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward +#else +#define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 +#endif +#include +#pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) +#define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ + [](size_t mask) noexcept -> int \ + { \ + unsigned long index; \ + return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) : ROBIN_HOOD(BITNESS); \ + }(x) +#else +#if ROBIN_HOOD(BITNESS) == 32 +#define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl +#define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl +#else +#define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll +#define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll +#endif +#define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) +#define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) +#endif #endif // fallthrough -#ifndef __has_cpp_attribute // For backwards compatibility -# define __has_cpp_attribute(x) 0 +#ifndef __has_cpp_attribute // For backwards compatibility +#define __has_cpp_attribute(x) 0 #endif #if __has_cpp_attribute(clang::fallthrough) -# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]] +#define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]] #elif __has_cpp_attribute(gnu::fallthrough) -# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]] +#define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]] #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() +#define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() #endif // likely/unlikely #ifdef _MSC_VER -# define ROBIN_HOOD_LIKELY(condition) condition -# define ROBIN_HOOD_UNLIKELY(condition) condition +#define ROBIN_HOOD_LIKELY(condition) condition +#define ROBIN_HOOD_UNLIKELY(condition) condition #else -# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1) -# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) +#define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1) +#define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) #endif // detect if native wchar_t type is availiable in MSVC #ifdef _MSC_VER -# ifdef _NATIVE_WCHAR_T_DEFINED -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 -# else -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0 -# endif +#ifdef _NATIVE_WCHAR_T_DEFINED +#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0 +#endif +#else +#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 #endif // workaround missing "is_trivially_copyable" in g++ < 5.0 // See https://stackoverflow.com/a/31798726/48181 #if defined(__GNUC__) && __GNUC__ < 5 -# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +#define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) #else -# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value +#define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value #endif // helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html @@ -208,73 +210,81 @@ static Counts& counts() { #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) -# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]] +#define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]] #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() +#define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() #endif -namespace robin_hood { - +namespace robin_hood +{ #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) -# define ROBIN_HOOD_STD std +#define ROBIN_HOOD_STD std #else // c++11 compatibility layer -namespace ROBIN_HOOD_STD { +namespace ROBIN_HOOD_STD +{ template -struct alignment_of - : std::integral_constant::type)> {}; +struct alignment_of : std::integral_constant::type)> +{ +}; template -class integer_sequence { -public: - using value_type = T; - static_assert(std::is_integral::value, "not integral type"); - static constexpr std::size_t size() noexcept { - return sizeof...(Ints); - } +class integer_sequence +{ + public: + using value_type = T; + static_assert(std::is_integral::value, "not integral type"); + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } }; template using index_sequence = integer_sequence; -namespace detail_ { +namespace detail_ +{ template -struct IntSeqImpl { - using TValue = T; - static_assert(std::is_integral::value, "not integral type"); - static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)"); +struct IntSeqImpl +{ + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)"); - template - struct IntSeqCombiner; + template + struct IntSeqCombiner; - template - struct IntSeqCombiner, integer_sequence> { - using TResult = integer_sequence; - }; + template + struct IntSeqCombiner, integer_sequence> + { + using TResult = integer_sequence; + }; - using TResult = - typename IntSeqCombiner::TResult, - typename IntSeqImpl::TResult>::TResult; + using TResult = typename IntSeqCombiner< + typename IntSeqImpl::TResult, + typename IntSeqImpl::TResult>::TResult; }; template -struct IntSeqImpl { - using TValue = T; - static_assert(std::is_integral::value, "not integral type"); - static_assert(Begin >= 0, "unexpected argument (Begin<0)"); - using TResult = integer_sequence; +struct IntSeqImpl +{ + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence; }; template -struct IntSeqImpl { - using TValue = T; - static_assert(std::is_integral::value, "not integral type"); - static_assert(Begin >= 0, "unexpected argument (Begin<0)"); - using TResult = integer_sequence; +struct IntSeqImpl +{ + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence; }; -} // namespace detail_ +} // namespace detail_ template using make_integer_sequence = typename detail_::IntSeqImpl::TResult; @@ -285,12 +295,12 @@ using make_index_sequence = make_integer_sequence; template using index_sequence_for = make_index_sequence; -} // namespace ROBIN_HOOD_STD +} // namespace ROBIN_HOOD_STD #endif -namespace detail { - +namespace detail +{ // make sure we static_cast to the correct type for hash_int #if ROBIN_HOOD(BITNESS) == 64 using SizeT = uint64_t; @@ -299,21 +309,24 @@ using SizeT = uint32_t; #endif template -T rotr(T x, unsigned k) { - return (x >> k) | (x << (8U * sizeof(T) - k)); +T rotr(T x, unsigned k) +{ + return (x >> k) | (x << (8U * sizeof(T) - k)); } // This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to // 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with // care! template -inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept { - return reinterpret_cast(ptr); +inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept +{ + return reinterpret_cast(ptr); } template -inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept { - return reinterpret_cast(ptr); +inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept +{ + return reinterpret_cast(ptr); } // make sure this is not inlined as it is slow and dramatically enlarges code, thus making other @@ -321,208 +334,229 @@ inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept { template [[noreturn]] ROBIN_HOOD(NOINLINE) #if ROBIN_HOOD(HAS_EXCEPTIONS) - void doThrow(Args&&... args) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) - throw E(std::forward(args)...); + void doThrow(Args&&... args) +{ + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) + throw E(std::forward(args)...); } #else - void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) { - abort(); + void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) +{ + abort(); } #endif template -T* assertNotNull(T* t, Args&&... args) { - if (ROBIN_HOOD_UNLIKELY(nullptr == t)) { - doThrow(std::forward(args)...); - } - return t; +T* assertNotNull(T* t, Args&&... args) +{ + if (ROBIN_HOOD_UNLIKELY(nullptr == t)) + { + doThrow(std::forward(args)...); + } + return t; } template -inline T unaligned_load(void const* ptr) noexcept { - // using memcpy so we don't get into unaligned load problems. - // compiler should optimize this very well anyways. - T t; - std::memcpy(&t, ptr, sizeof(T)); - return t; +inline T unaligned_load(void const* ptr) noexcept +{ + // using memcpy so we don't get into unaligned load problems. + // compiler should optimize this very well anyways. + T t; + std::memcpy(&t, ptr, sizeof(T)); + return t; } // Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor, // and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a // pointer. template -class BulkPoolAllocator { -public: - BulkPoolAllocator() noexcept = default; +class BulkPoolAllocator +{ + public: + BulkPoolAllocator() noexcept = default; - // does not copy anything, just creates a new allocator. - BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept - : mHead(nullptr) - , mListForFree(nullptr) {} + // does not copy anything, just creates a new allocator. + BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept + : mHead(nullptr), mListForFree(nullptr) + { + } - BulkPoolAllocator(BulkPoolAllocator&& o) noexcept - : mHead(o.mHead) - , mListForFree(o.mListForFree) { - o.mListForFree = nullptr; - o.mHead = nullptr; + BulkPoolAllocator(BulkPoolAllocator&& o) noexcept : mHead(o.mHead), mListForFree(o.mListForFree) + { + o.mListForFree = nullptr; + o.mHead = nullptr; + } + + BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept + { + reset(); + mHead = o.mHead; + mListForFree = o.mListForFree; + o.mListForFree = nullptr; + o.mHead = nullptr; + return *this; + } + + BulkPoolAllocator& + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept + { + // does not do anything + return *this; + } + + ~BulkPoolAllocator() noexcept + { + reset(); + } + + // Deallocates all allocated memory. + void reset() noexcept + { + while (mListForFree) + { + T* tmp = *mListForFree; + ROBIN_HOOD_LOG("std::free") + std::free(mListForFree); + mListForFree = reinterpret_cast_no_cast_align_warning(tmp); + } + mHead = nullptr; + } + + // allocates, but does NOT initialize. Use in-place new constructor, e.g. + // T* obj = pool.allocate(); + // ::new (static_cast(obj)) T(); + T* allocate() + { + T* tmp = mHead; + if (!tmp) + { + tmp = performAllocation(); } - BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept { - reset(); - mHead = o.mHead; - mListForFree = o.mListForFree; - o.mListForFree = nullptr; - o.mHead = nullptr; - return *this; + mHead = *reinterpret_cast_no_cast_align_warning(tmp); + return tmp; + } + + // does not actually deallocate but puts it in store. + // make sure you have already called the destructor! e.g. with + // obj->~T(); + // pool.deallocate(obj); + void deallocate(T* obj) noexcept + { + *reinterpret_cast_no_cast_align_warning(obj) = mHead; + mHead = obj; + } + + // Adds an already allocated block of memory to the allocator. This allocator is from now on + // responsible for freeing the data (with free()). If the provided data is not large enough to + // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor. + void addOrFree(void* ptr, const size_t numBytes) noexcept + { + // calculate number of available elements in ptr + if (numBytes < ALIGNMENT + ALIGNED_SIZE) + { + // not enough data for at least one element. Free and return. + ROBIN_HOOD_LOG("std::free") + std::free(ptr); + } + else + { + ROBIN_HOOD_LOG("add to buffer") + add(ptr, numBytes); + } + } + + void swap(BulkPoolAllocator& other) noexcept + { + using std::swap; + swap(mHead, other.mHead); + swap(mListForFree, other.mListForFree); + } + + private: + // iterates the list of allocated memory to calculate how many to alloc next. + // Recalculating this each time saves us a size_t member. + // This ignores the fact that memory blocks might have been added manually with addOrFree. In + // practice, this should not matter much. + ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept + { + auto tmp = mListForFree; + size_t numAllocs = MinNumAllocs; + + while (numAllocs * 2 <= MaxNumAllocs && tmp) + { + auto x = reinterpret_cast(tmp); + tmp = *x; + numAllocs *= 2; } - BulkPoolAllocator& - // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) - operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept { - // does not do anything - return *this; + return numAllocs; + } + + // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree(). + void add(void* ptr, const size_t numBytes) noexcept + { + const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE; + + auto data = reinterpret_cast(ptr); + + // link free list + auto x = reinterpret_cast(data); + *x = mListForFree; + mListForFree = data; + + // create linked list for newly allocated data + auto* const headT = reinterpret_cast_no_cast_align_warning(reinterpret_cast(ptr) + ALIGNMENT); + + auto* const head = reinterpret_cast(headT); + + // Visual Studio compiler automatically unrolls this loop, which is pretty cool + for (size_t i = 0; i < numElements; ++i) + { + *reinterpret_cast_no_cast_align_warning(head + i * ALIGNED_SIZE) = + head + (i + 1) * ALIGNED_SIZE; } - ~BulkPoolAllocator() noexcept { - reset(); - } + // last one points to 0 + *reinterpret_cast_no_cast_align_warning(head + (numElements - 1) * ALIGNED_SIZE) = mHead; + mHead = headT; + } - // Deallocates all allocated memory. - void reset() noexcept { - while (mListForFree) { - T* tmp = *mListForFree; - ROBIN_HOOD_LOG("std::free") - std::free(mListForFree); - mListForFree = reinterpret_cast_no_cast_align_warning(tmp); - } - mHead = nullptr; - } + // Called when no memory is available (mHead == 0). + // Don't inline this slow path. + ROBIN_HOOD(NOINLINE) T* performAllocation() + { + size_t const numElementsToAlloc = calcNumElementsToAlloc(); - // allocates, but does NOT initialize. Use in-place new constructor, e.g. - // T* obj = pool.allocate(); - // ::new (static_cast(obj)) T(); - T* allocate() { - T* tmp = mHead; - if (!tmp) { - tmp = performAllocation(); - } + // alloc new memory: [prev |T, T, ... T] + size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; + ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE << " * " + << numElementsToAlloc) + add(assertNotNull(std::malloc(bytes)), bytes); + return mHead; + } - mHead = *reinterpret_cast_no_cast_align_warning(tmp); - return tmp; - } - - // does not actually deallocate but puts it in store. - // make sure you have already called the destructor! e.g. with - // obj->~T(); - // pool.deallocate(obj); - void deallocate(T* obj) noexcept { - *reinterpret_cast_no_cast_align_warning(obj) = mHead; - mHead = obj; - } - - // Adds an already allocated block of memory to the allocator. This allocator is from now on - // responsible for freeing the data (with free()). If the provided data is not large enough to - // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor. - void addOrFree(void* ptr, const size_t numBytes) noexcept { - // calculate number of available elements in ptr - if (numBytes < ALIGNMENT + ALIGNED_SIZE) { - // not enough data for at least one element. Free and return. - ROBIN_HOOD_LOG("std::free") - std::free(ptr); - } else { - ROBIN_HOOD_LOG("add to buffer") - add(ptr, numBytes); - } - } - - void swap(BulkPoolAllocator& other) noexcept { - using std::swap; - swap(mHead, other.mHead); - swap(mListForFree, other.mListForFree); - } - -private: - // iterates the list of allocated memory to calculate how many to alloc next. - // Recalculating this each time saves us a size_t member. - // This ignores the fact that memory blocks might have been added manually with addOrFree. In - // practice, this should not matter much. - ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept { - auto tmp = mListForFree; - size_t numAllocs = MinNumAllocs; - - while (numAllocs * 2 <= MaxNumAllocs && tmp) { - auto x = reinterpret_cast(tmp); - tmp = *x; - numAllocs *= 2; - } - - return numAllocs; - } - - // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree(). - void add(void* ptr, const size_t numBytes) noexcept { - const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE; - - auto data = reinterpret_cast(ptr); - - // link free list - auto x = reinterpret_cast(data); - *x = mListForFree; - mListForFree = data; - - // create linked list for newly allocated data - auto* const headT = - reinterpret_cast_no_cast_align_warning(reinterpret_cast(ptr) + ALIGNMENT); - - auto* const head = reinterpret_cast(headT); - - // Visual Studio compiler automatically unrolls this loop, which is pretty cool - for (size_t i = 0; i < numElements; ++i) { - *reinterpret_cast_no_cast_align_warning(head + i * ALIGNED_SIZE) = - head + (i + 1) * ALIGNED_SIZE; - } - - // last one points to 0 - *reinterpret_cast_no_cast_align_warning(head + (numElements - 1) * ALIGNED_SIZE) = - mHead; - mHead = headT; - } - - // Called when no memory is available (mHead == 0). - // Don't inline this slow path. - ROBIN_HOOD(NOINLINE) T* performAllocation() { - size_t const numElementsToAlloc = calcNumElementsToAlloc(); - - // alloc new memory: [prev |T, T, ... T] - size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; - ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE - << " * " << numElementsToAlloc) - add(assertNotNull(std::malloc(bytes)), bytes); - return mHead; - } - - // enforce byte alignment of the T's + // enforce byte alignment of the T's #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) - static constexpr size_t ALIGNMENT = - (std::max)(std::alignment_of::value, std::alignment_of::value); + static constexpr size_t ALIGNMENT = (std::max)(std::alignment_of::value, std::alignment_of::value); #else - static const size_t ALIGNMENT = - (ROBIN_HOOD_STD::alignment_of::value > ROBIN_HOOD_STD::alignment_of::value) - ? ROBIN_HOOD_STD::alignment_of::value - : +ROBIN_HOOD_STD::alignment_of::value; // the + is for walkarround + static const size_t ALIGNMENT = + (ROBIN_HOOD_STD::alignment_of::value > ROBIN_HOOD_STD::alignment_of::value) + ? ROBIN_HOOD_STD::alignment_of::value + : +ROBIN_HOOD_STD::alignment_of::value; // the + is for walkarround #endif - static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT; + static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT; - static_assert(MinNumAllocs >= 1, "MinNumAllocs"); - static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs"); - static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE"); - static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod"); - static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT"); + static_assert(MinNumAllocs >= 1, "MinNumAllocs"); + static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs"); + static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE"); + static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod"); + static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT"); - T* mHead{nullptr}; - T** mListForFree{nullptr}; + T* mHead{nullptr}; + T** mListForFree{nullptr}; }; template @@ -530,291 +564,327 @@ struct NodeAllocator; // dummy allocator that does nothing template -struct NodeAllocator { - - // we are not using the data, so just free it. - void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept { - ROBIN_HOOD_LOG("std::free") - std::free(ptr); - } +struct NodeAllocator +{ + // we are not using the data, so just free it. + void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept + { + ROBIN_HOOD_LOG("std::free") + std::free(ptr); + } }; template -struct NodeAllocator : public BulkPoolAllocator {}; +struct NodeAllocator : public BulkPoolAllocator +{ +}; // dummy hash, unsed as mixer when robin_hood::hash is already used template -struct identity_hash { - constexpr size_t operator()(T const& obj) const noexcept { - return static_cast(obj); - } +struct identity_hash +{ + constexpr size_t operator()(T const& obj) const noexcept + { + return static_cast(obj); + } }; // c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making // my own here. -namespace swappable { +namespace swappable +{ #if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17) using std::swap; template -struct nothrow { - static const bool value = noexcept(swap(std::declval(), std::declval())); +struct nothrow +{ + static const bool value = noexcept(swap(std::declval(), std::declval())); }; #else template -struct nothrow { - static const bool value = std::is_nothrow_swappable::value; +struct nothrow +{ + static const bool value = std::is_nothrow_swappable::value; }; #endif -} // namespace swappable +} // namespace swappable -} // namespace detail +} // namespace detail -struct is_transparent_tag {}; +struct is_transparent_tag +{ +}; // A custom pair implementation is used in the map because std::pair is not is_trivially_copyable, // which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is // also tested. template -struct pair { - using first_type = T1; - using second_type = T2; +struct pair +{ + using first_type = T1; + using second_type = T2; - template ::value && - std::is_default_constructible::value>::type> - constexpr pair() noexcept(noexcept(U1()) && noexcept(U2())) - : first() - , second() {} + template ::value && + std::is_default_constructible::value>::type> + constexpr pair() noexcept(noexcept(U1()) && noexcept(U2())) : first(), second() + { + } - // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. - explicit constexpr pair(std::pair const& o) noexcept( - noexcept(T1(std::declval())) && noexcept(T2(std::declval()))) - : first(o.first) - , second(o.second) {} + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair const& o) noexcept( + noexcept(T1(std::declval())) && noexcept(T2(std::declval()))) + : first(o.first), second(o.second) + { + } - // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. - explicit constexpr pair(std::pair&& o) noexcept(noexcept( - T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) - : first(std::move(o.first)) - , second(std::move(o.second)) {} + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair&& o) noexcept( + noexcept(T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) + : first(std::move(o.first)), second(std::move(o.second)) + { + } - constexpr pair(T1&& a, T2&& b) noexcept(noexcept( - T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) - : first(std::move(a)) - , second(std::move(b)) {} + constexpr pair(T1&& a, T2&& b) noexcept( + noexcept(T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) + : first(std::move(a)), second(std::move(b)) + { + } - template - constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward( - std::declval()))) && noexcept(T2(std::forward(std::declval())))) - : first(std::forward(a)) - , second(std::forward(b)) {} + template + constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward(std::declval()))) && noexcept( + T2(std::forward(std::declval())))) + : first(std::forward(a)), second(std::forward(b)) + { + } - template - constexpr pair( - std::piecewise_construct_t /*unused*/, std::tuple a, - std::tuple b) noexcept(noexcept(pair(std::declval&>(), - std::declval&>(), - ROBIN_HOOD_STD::index_sequence_for(), - ROBIN_HOOD_STD::index_sequence_for()))) - : pair(a, b, ROBIN_HOOD_STD::index_sequence_for(), - ROBIN_HOOD_STD::index_sequence_for()) {} + template + constexpr pair(std::piecewise_construct_t /*unused*/, std::tuple a, + std::tuple b) noexcept(noexcept(pair(std::declval&>(), + std::declval&>(), + ROBIN_HOOD_STD::index_sequence_for(), + ROBIN_HOOD_STD::index_sequence_for()))) + : pair(a, b, ROBIN_HOOD_STD::index_sequence_for(), ROBIN_HOOD_STD::index_sequence_for()) + { + } - // constructor called from the std::piecewise_construct_t ctor - template - pair(std::tuple& a, std::tuple& b, ROBIN_HOOD_STD::index_sequence /*unused*/, ROBIN_HOOD_STD::index_sequence /*unused*/) noexcept( - noexcept(T1(std::forward(std::get( - std::declval&>()))...)) && noexcept(T2(std:: - forward(std::get( - std::declval&>()))...))) - : first(std::forward(std::get(a))...) - , second(std::forward(std::get(b))...) { - // make visual studio compiler happy about warning about unused a & b. - // Visual studio's pair implementation disables warning 4100. - (void)a; - (void)b; - } + // constructor called from the std::piecewise_construct_t ctor + template + pair( + std::tuple& a, std::tuple& b, ROBIN_HOOD_STD::index_sequence /*unused*/, + ROBIN_HOOD_STD::index_sequence< + I2...> /*unused*/) noexcept(noexcept(T1(std:: + forward(std::get( + std::declval&>()))...)) && noexcept(T2(std:: + forward< + U2>(std::get< + I2>( + std::declval&>()))...))) + : first(std::forward(std::get(a))...), second(std::forward(std::get(b))...) + { + // make visual studio compiler happy about warning about unused a & b. + // Visual studio's pair implementation disables warning 4100. + (void)a; + (void)b; + } - void swap(pair& o) noexcept((detail::swappable::nothrow::value) && - (detail::swappable::nothrow::value)) { - using std::swap; - swap(first, o.first); - swap(second, o.second); - } + void swap(pair& o) noexcept((detail::swappable::nothrow::value) && + (detail::swappable::nothrow::value)) + { + using std::swap; + swap(first, o.first); + swap(second, o.second); + } - T1 first; // NOLINT(misc-non-private-member-variables-in-classes) - T2 second; // NOLINT(misc-non-private-member-variables-in-classes) + T1 first; // NOLINT(misc-non-private-member-variables-in-classes) + T2 second; // NOLINT(misc-non-private-member-variables-in-classes) }; template inline void swap(pair& a, pair& b) noexcept( - noexcept(std::declval&>().swap(std::declval&>()))) { - a.swap(b); + noexcept(std::declval&>().swap(std::declval&>()))) +{ + a.swap(b); } template -inline constexpr bool operator==(pair const& x, pair const& y) { - return (x.first == y.first) && (x.second == y.second); +inline constexpr bool operator==(pair const& x, pair const& y) +{ + return (x.first == y.first) && (x.second == y.second); } template -inline constexpr bool operator!=(pair const& x, pair const& y) { - return !(x == y); +inline constexpr bool operator!=(pair const& x, pair const& y) +{ + return !(x == y); } template -inline constexpr bool operator<(pair const& x, pair const& y) noexcept(noexcept( - std::declval() < std::declval()) && noexcept(std::declval() < - std::declval())) { - return x.first < y.first || (!(y.first < x.first) && x.second < y.second); +inline constexpr bool operator<(pair const& x, pair const& y) noexcept( + noexcept(std::declval() < std::declval()) && noexcept(std::declval() < + std::declval())) +{ + return x.first < y.first || (!(y.first < x.first) && x.second < y.second); } template -inline constexpr bool operator>(pair const& x, pair const& y) { - return y < x; +inline constexpr bool operator>(pair const& x, pair const& y) +{ + return y < x; } template -inline constexpr bool operator<=(pair const& x, pair const& y) { - return !(x > y); +inline constexpr bool operator<=(pair const& x, pair const& y) +{ + return !(x > y); } template -inline constexpr bool operator>=(pair const& x, pair const& y) { - return !(x < y); +inline constexpr bool operator>=(pair const& x, pair const& y) +{ + return !(x < y); } -inline size_t hash_bytes(void const* ptr, size_t len) noexcept { - static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); - static constexpr uint64_t seed = UINT64_C(0xe17a1465); - static constexpr unsigned int r = 47; +inline size_t hash_bytes(void const* ptr, size_t len) noexcept +{ + static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); + static constexpr uint64_t seed = UINT64_C(0xe17a1465); + static constexpr unsigned int r = 47; - auto const* const data64 = static_cast(ptr); - uint64_t h = seed ^ (len * m); + auto const* const data64 = static_cast(ptr); + uint64_t h = seed ^ (len * m); - size_t const n_blocks = len / 8; - for (size_t i = 0; i < n_blocks; ++i) { - auto k = detail::unaligned_load(data64 + i); + size_t const n_blocks = len / 8; + for (size_t i = 0; i < n_blocks; ++i) + { + auto k = detail::unaligned_load(data64 + i); - k *= m; - k ^= k >> r; - k *= m; + k *= m; + k ^= k >> r; + k *= m; - h ^= k; - h *= m; - } - - auto const* const data8 = reinterpret_cast(data64 + n_blocks); - switch (len & 7U) { - case 7: - h ^= static_cast(data8[6]) << 48U; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - case 6: - h ^= static_cast(data8[5]) << 40U; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - case 5: - h ^= static_cast(data8[4]) << 32U; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - case 4: - h ^= static_cast(data8[3]) << 24U; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - case 3: - h ^= static_cast(data8[2]) << 16U; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - case 2: - h ^= static_cast(data8[1]) << 8U; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - case 1: - h ^= static_cast(data8[0]); - h *= m; - ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH - default: - break; - } - - h ^= h >> r; + h ^= k; h *= m; - h ^= h >> r; - return static_cast(h); + } + + auto const* const data8 = reinterpret_cast(data64 + n_blocks); + switch (len & 7U) + { + case 7: h ^= static_cast(data8[6]) << 48U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 6: h ^= static_cast(data8[5]) << 40U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 5: h ^= static_cast(data8[4]) << 32U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 4: h ^= static_cast(data8[3]) << 24U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 3: h ^= static_cast(data8[2]) << 16U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 2: h ^= static_cast(data8[1]) << 8U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 1: + h ^= static_cast(data8[0]); + h *= m; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + default: break; + } + + h ^= h >> r; + h *= m; + h ^= h >> r; + return static_cast(h); } -inline size_t hash_int(uint64_t x) noexcept { - // inspired by lemire's strongly universal hashing - // https://lemire.me/blog/2018/08/15/fast-strongly-universal-64-bit-hashing-everywhere/ - // - // Instead of shifts, we use rotations so we don't lose any bits. - // - // Added a final multiplcation with a constant for more mixing. It is most important that - // the lower bits are well mixed. - auto h1 = x * UINT64_C(0xA24BAED4963EE407); - auto h2 = detail::rotr(x, 32U) * UINT64_C(0x9FB21C651E98DF25); - auto h = detail::rotr(h1 + h2, 32U); - return static_cast(h); +inline size_t hash_int(uint64_t x) noexcept +{ + // inspired by lemire's strongly universal hashing + // https://lemire.me/blog/2018/08/15/fast-strongly-universal-64-bit-hashing-everywhere/ + // + // Instead of shifts, we use rotations so we don't lose any bits. + // + // Added a final multiplcation with a constant for more mixing. It is most important that + // the lower bits are well mixed. + auto h1 = x * UINT64_C(0xA24BAED4963EE407); + auto h2 = detail::rotr(x, 32U) * UINT64_C(0x9FB21C651E98DF25); + auto h = detail::rotr(h1 + h2, 32U); + return static_cast(h); } // A thin wrapper around std::hash, performing an additional simple mixing step of the result. template -struct hash : public std::hash { - size_t operator()(T const& obj) const - noexcept(noexcept(std::declval>().operator()(std::declval()))) { - // call base hash - auto result = std::hash::operator()(obj); - // return mixed of that, to be save against identity has - return hash_int(static_cast(result)); - } +struct hash : public std::hash +{ + size_t operator()(T const& obj) const + noexcept(noexcept(std::declval>().operator()(std::declval()))) + { + // call base hash + auto result = std::hash::operator()(obj); + // return mixed of that, to be save against identity has + return hash_int(static_cast(result)); + } }; template -struct hash> { - size_t operator()(std::basic_string const& str) const noexcept { - return hash_bytes(str.data(), sizeof(CharT) * str.size()); - } +struct hash> +{ + size_t operator()(std::basic_string const& str) const noexcept + { + return hash_bytes(str.data(), sizeof(CharT) * str.size()); + } }; #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) template -struct hash> { - size_t operator()(std::basic_string_view const& sv) const noexcept { - return hash_bytes(sv.data(), sizeof(CharT) * sv.size()); - } +struct hash> +{ + size_t operator()(std::basic_string_view const& sv) const noexcept + { + return hash_bytes(sv.data(), sizeof(CharT) * sv.size()); + } }; #endif template -struct hash { - size_t operator()(T* ptr) const noexcept { - return hash_int(reinterpret_cast(ptr)); - } +struct hash +{ + size_t operator()(T* ptr) const noexcept + { + return hash_int(reinterpret_cast(ptr)); + } }; template -struct hash> { - size_t operator()(std::unique_ptr const& ptr) const noexcept { - return hash_int(reinterpret_cast(ptr.get())); - } +struct hash> +{ + size_t operator()(std::unique_ptr const& ptr) const noexcept + { + return hash_int(reinterpret_cast(ptr.get())); + } }; template -struct hash> { - size_t operator()(std::shared_ptr const& ptr) const noexcept { - return hash_int(reinterpret_cast(ptr.get())); - } +struct hash> +{ + size_t operator()(std::shared_ptr const& ptr) const noexcept + { + return hash_int(reinterpret_cast(ptr.get())); + } }; template -struct hash::value>::type> { - size_t operator()(Enum e) const noexcept { - using Underlying = typename std::underlying_type::type; - return hash{}(static_cast(e)); - } +struct hash::value>::type> +{ + size_t operator()(Enum e) const noexcept + { + using Underlying = typename std::underlying_type::type; + return hash{}(static_cast(e)); + } }; -#define ROBIN_HOOD_HASH_INT(T) \ - template <> \ - struct hash { \ - size_t operator()(T const& obj) const noexcept { \ - return hash_int(static_cast(obj)); \ - } \ - } +#define ROBIN_HOOD_HASH_INT(T) \ + template <> \ + struct hash \ + { \ + size_t operator()(T const& obj) const noexcept \ + { \ + return hash_int(static_cast(obj)); \ + } \ + } #if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wuseless-cast" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuseless-cast" #endif // see https://en.cppreference.com/w/cpp/utility/hash ROBIN_HOOD_HASH_INT(bool); @@ -835,36 +905,44 @@ ROBIN_HOOD_HASH_INT(long long); ROBIN_HOOD_HASH_INT(unsigned long); ROBIN_HOOD_HASH_INT(unsigned long long); #if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif -namespace detail { - +namespace detail +{ template -struct void_type { - using type = void; +struct void_type +{ + using type = void; }; template -struct has_is_transparent : public std::false_type {}; +struct has_is_transparent : public std::false_type +{ +}; template -struct has_is_transparent::type> - : public std::true_type {}; +struct has_is_transparent::type> : public std::true_type +{ +}; // using wrapper classes for hash and key_equal prevents the diamond problem when the same type // is used. see https://stackoverflow.com/a/28771920/48181 template -struct WrapHash : public T { - WrapHash() = default; - explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval()))) - : T(o) {} +struct WrapHash : public T +{ + WrapHash() = default; + explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval()))) : T(o) + { + } }; template -struct WrapKeyEqual : public T { - WrapKeyEqual() = default; - explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval()))) - : T(o) {} +struct WrapKeyEqual : public T +{ + WrapKeyEqual() = default; + explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval()))) : T(o) + { + } }; // A highly optimized hashmap implementation, using the Robin Hood algorithm. @@ -893,1544 +971,1752 @@ struct WrapKeyEqual : public T { // According to STL, order of templates has effect on throughput. That's why I've moved the // boolean to the front. // https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ -template -class Table - : public WrapHash, - public WrapKeyEqual, - detail::NodeAllocator< - typename std::conditional< - std::is_void::value, Key, - robin_hood::pair::type, T>>::type, - 4, 16384, IsFlat> { -public: - static constexpr bool is_flat = IsFlat; - static constexpr bool is_map = !std::is_void::value; - static constexpr bool is_set = !is_map; - static constexpr bool is_transparent = - has_is_transparent::value && has_is_transparent::value; +template +class Table : public WrapHash, + public WrapKeyEqual, + detail::NodeAllocator< + typename std::conditional< + std::is_void::value, Key, + robin_hood::pair::type, T>>::type, + 4, 16384, IsFlat> +{ + public: + static constexpr bool is_flat = IsFlat; + static constexpr bool is_map = !std::is_void::value; + static constexpr bool is_set = !is_map; + static constexpr bool is_transparent = + has_is_transparent::value && has_is_transparent::value; - using key_type = Key; - using mapped_type = T; - using value_type = typename std::conditional< - is_set, Key, - robin_hood::pair::type, T>>::type; - using size_type = size_t; - using hasher = Hash; - using key_equal = KeyEqual; - using Self = Table; + using key_type = Key; + using mapped_type = T; + using value_type = typename std::conditional< + is_set, Key, robin_hood::pair::type, T>>::type; + using size_type = size_t; + using hasher = Hash; + using key_equal = KeyEqual; + using Self = Table; -private: - static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, - "MaxLoadFactor100 needs to be >10 && < 100"); + private: + static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, "MaxLoadFactor100 needs to be >10 && < 100"); - using WHash = WrapHash; - using WKeyEqual = WrapKeyEqual; + using WHash = WrapHash; + using WKeyEqual = WrapKeyEqual; - // configuration defaults + // configuration defaults - // make sure we have 8 elements, needed to quickly rehash mInfo - static constexpr size_t InitialNumElements = sizeof(uint64_t); - static constexpr uint32_t InitialInfoNumBits = 5; - static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; - static constexpr size_t InfoMask = InitialInfoInc - 1U; - static constexpr uint8_t InitialInfoHashShift = 0; - using DataPool = detail::NodeAllocator; + // make sure we have 8 elements, needed to quickly rehash mInfo + static constexpr size_t InitialNumElements = sizeof(uint64_t); + static constexpr uint32_t InitialInfoNumBits = 5; + static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; + static constexpr size_t InfoMask = InitialInfoInc - 1U; + static constexpr uint8_t InitialInfoHashShift = 0; + using DataPool = detail::NodeAllocator; - // type needs to be wider than uint8_t. - using InfoType = uint32_t; + // type needs to be wider than uint8_t. + using InfoType = uint32_t; - // DataNode //////////////////////////////////////////////////////// + // DataNode //////////////////////////////////////////////////////// - // Primary template for the data node. We have special implementations for small and big - // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these - // on the heap so swap merely swaps a pointer. - template - class DataNode {}; + // Primary template for the data node. We have special implementations for small and big + // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these + // on the heap so swap merely swaps a pointer. + template + class DataNode + { + }; - // Small: just allocate on the stack. - template - class DataNode final { - public: - template - explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept( - noexcept(value_type(std::forward(args)...))) - : mData(std::forward(args)...) {} - - DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept( - std::is_nothrow_move_constructible::value) - : mData(std::move(n.mData)) {} - - // doesn't do anything - void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {} - void destroyDoNotDeallocate() noexcept {} - - value_type const* operator->() const noexcept { - return &mData; - } - value_type* operator->() noexcept { - return &mData; - } - - const value_type& operator*() const noexcept { - return mData; - } - - value_type& operator*() noexcept { - return mData; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getFirst() noexcept { - return mData.first; - } - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getFirst() noexcept { - return mData; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type - getFirst() const noexcept { - return mData.first; - } - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getFirst() const noexcept { - return mData; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getSecond() noexcept { - return mData.second; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getSecond() const noexcept { - return mData.second; - } - - void swap(DataNode& o) noexcept( - noexcept(std::declval().swap(std::declval()))) { - mData.swap(o.mData); - } - - private: - value_type mData; - }; - - // big object: allocate on heap. - template - class DataNode { - public: - template - explicit DataNode(M& map, Args&&... args) - : mData(map.allocate()) { - ::new (static_cast(mData)) value_type(std::forward(args)...); - } - - DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept - : mData(std::move(n.mData)) {} - - void destroy(M& map) noexcept { - // don't deallocate, just put it into list of datapool. - mData->~value_type(); - map.deallocate(mData); - } - - void destroyDoNotDeallocate() noexcept { - mData->~value_type(); - } - - value_type const* operator->() const noexcept { - return mData; - } - - value_type* operator->() noexcept { - return mData; - } - - const value_type& operator*() const { - return *mData; - } - - value_type& operator*() { - return *mData; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getFirst() noexcept { - return mData->first; - } - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getFirst() noexcept { - return *mData; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type - getFirst() const noexcept { - return mData->first; - } - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getFirst() const noexcept { - return *mData; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getSecond() noexcept { - return mData->second; - } - - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::type getSecond() const noexcept { - return mData->second; - } - - void swap(DataNode& o) noexcept { - using std::swap; - swap(mData, o.mData); - } - - private: - value_type* mData; - }; - - using Node = DataNode; - - // helpers for doInsert: extract first entry (only const required) - ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept { - return n.getFirst(); + // Small: just allocate on the stack. + template + class DataNode final + { + public: + template + explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, + Args&&... args) noexcept(noexcept(value_type(std::forward(args)...))) + : mData(std::forward(args)...) + { } - // in case we have void mapped_type, we are not using a pair, thus we just route k through. - // No need to disable this because it's just not used if not applicable. - ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept { - return k; + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, + DataNode&& n) noexcept(std::is_nothrow_move_constructible::value) + : mData(std::move(n.mData)) + { } - // in case we have non-void mapped_type, we have a standard robin_hood::pair - template + // doesn't do anything + void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept + { + } + void destroyDoNotDeallocate() noexcept + { + } + + value_type const* operator->() const noexcept + { + return &mData; + } + value_type* operator->() noexcept + { + return &mData; + } + + const value_type& operator*() const noexcept + { + return mData; + } + + value_type& operator*() noexcept + { + return mData; + } + + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, key_type const&>::type - getFirstConst(value_type const& vt) const noexcept { - return vt.first; + typename std::enable_if::type getFirst() noexcept + { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept + { + return mData; } - // Cloner ////////////////////////////////////////////////////////// + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept + { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept + { + return mData; + } - template - struct Cloner; + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept + { + return mData.second; + } - // fast path: Just copy data, without allocating anything. - template - struct Cloner { - void operator()(M const& source, M& target) const { - auto const* const src = reinterpret_cast(source.mKeyVals); - auto* tgt = reinterpret_cast(target.mKeyVals); - auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); - std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept + { + return mData.second; + } + + void swap(DataNode& o) noexcept( + noexcept(std::declval().swap(std::declval()))) + { + mData.swap(o.mData); + } + + private: + value_type mData; + }; + + // big object: allocate on heap. + template + class DataNode + { + public: + template + explicit DataNode(M& map, Args&&... args) : mData(map.allocate()) + { + ::new (static_cast(mData)) value_type(std::forward(args)...); + } + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept + : mData(std::move(n.mData)) + { + } + + void destroy(M& map) noexcept + { + // don't deallocate, just put it into list of datapool. + mData->~value_type(); + map.deallocate(mData); + } + + void destroyDoNotDeallocate() noexcept + { + mData->~value_type(); + } + + value_type const* operator->() const noexcept + { + return mData; + } + + value_type* operator->() noexcept + { + return mData; + } + + const value_type& operator*() const + { + return *mData; + } + + value_type& operator*() + { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept + { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept + { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept + { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept + { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept + { + return mData->second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept + { + return mData->second; + } + + void swap(DataNode& o) noexcept + { + using std::swap; + swap(mData, o.mData); + } + + private: + value_type* mData; + }; + + using Node = DataNode; + + // helpers for doInsert: extract first entry (only const required) + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept + { + return n.getFirst(); + } + + // in case we have void mapped_type, we are not using a pair, thus we just route k through. + // No need to disable this because it's just not used if not applicable. + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept + { + return k; + } + + // in case we have non-void mapped_type, we have a standard robin_hood::pair + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, key_type const&>::type + getFirstConst(value_type const& vt) const noexcept + { + return vt.first; + } + + // Cloner ////////////////////////////////////////////////////////// + + template + struct Cloner; + + // fast path: Just copy data, without allocating anything. + template + struct Cloner + { + void operator()(M const& source, M& target) const + { + auto const* const src = reinterpret_cast(source.mKeyVals); + auto* tgt = reinterpret_cast(target.mKeyVals); + auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); + std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); + } + }; + + template + struct Cloner + { + void operator()(M const& s, M& t) const + { + auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); + std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); + + for (size_t i = 0; i < numElementsWithBuffer; ++i) + { + if (t.mInfo[i]) + { + ::new (static_cast(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); } - }; + } + } + }; - template - struct Cloner { - void operator()(M const& s, M& t) const { - auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); - std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); + // Destroyer /////////////////////////////////////////////////////// - for (size_t i = 0; i < numElementsWithBuffer; ++i) { - if (t.mInfo[i]) { - ::new (static_cast(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); - } - } + template + struct Destroyer + { + }; + + template + struct Destroyer + { + void nodes(M& m) const noexcept + { + m.mNumElements = 0; + } + + void nodesDoNotDeallocate(M& m) const noexcept + { + m.mNumElements = 0; + } + }; + + template + struct Destroyer + { + void nodes(M& m) const noexcept + { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) + { + if (0 != m.mInfo[idx]) + { + Node& n = m.mKeyVals[idx]; + n.destroy(m); + n.~Node(); } - }; + } + } - // Destroyer /////////////////////////////////////////////////////// - - template - struct Destroyer {}; - - template - struct Destroyer { - void nodes(M& m) const noexcept { - m.mNumElements = 0; + void nodesDoNotDeallocate(M& m) const noexcept + { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) + { + if (0 != m.mInfo[idx]) + { + Node& n = m.mKeyVals[idx]; + n.destroyDoNotDeallocate(); + n.~Node(); } + } + } + }; - void nodesDoNotDeallocate(M& m) const noexcept { - m.mNumElements = 0; - } - }; + // Iter //////////////////////////////////////////////////////////// - template - struct Destroyer { - void nodes(M& m) const noexcept { - m.mNumElements = 0; - // clear also resets mInfo to 0, that's sometimes not necessary. - auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + struct fast_forward_tag + { + }; - for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { - if (0 != m.mInfo[idx]) { - Node& n = m.mKeyVals[idx]; - n.destroy(m); - n.~Node(); - } - } - } + // generic iterator for both const_iterator and iterator. + template + // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions) + class Iter + { + private: + using NodePtr = typename std::conditional::type; - void nodesDoNotDeallocate(M& m) const noexcept { - m.mNumElements = 0; - // clear also resets mInfo to 0, that's sometimes not necessary. - auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); - for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { - if (0 != m.mInfo[idx]) { - Node& n = m.mKeyVals[idx]; - n.destroyDoNotDeallocate(); - n.~Node(); - } - } - } - }; + public: + using difference_type = std::ptrdiff_t; + using value_type = typename Self::value_type; + using reference = typename std::conditional::type; + using pointer = typename std::conditional::type; + using iterator_category = std::forward_iterator_tag; - // Iter //////////////////////////////////////////////////////////// + // default constructed iterator can be compared to itself, but WON'T return true when + // compared to end(). + Iter() = default; - struct fast_forward_tag {}; + // Rule of zero: nothing specified. The conversion constructor is only enabled for + // iterator to const_iterator, so it doesn't accidentally work as a copy ctor. - // generic iterator for both const_iterator and iterator. - template - // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions) - class Iter { - private: - using NodePtr = typename std::conditional::type; + // Conversion constructor from iterator to const_iterator. + template ::type> + // NOLINTNEXTLINE(hicpp-explicit-conversions) + Iter(Iter const& other) noexcept : mKeyVals(other.mKeyVals), mInfo(other.mInfo) + { + } - public: - using difference_type = std::ptrdiff_t; - using value_type = typename Self::value_type; - using reference = typename std::conditional::type; - using pointer = typename std::conditional::type; - using iterator_category = std::forward_iterator_tag; + Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept : mKeyVals(valPtr), mInfo(infoPtr) + { + } - // default constructed iterator can be compared to itself, but WON'T return true when - // compared to end(). - Iter() = default; + Iter(NodePtr valPtr, uint8_t const* infoPtr, fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept + : mKeyVals(valPtr), mInfo(infoPtr) + { + fastForward(); + } - // Rule of zero: nothing specified. The conversion constructor is only enabled for - // iterator to const_iterator, so it doesn't accidentally work as a copy ctor. + template ::type> + Iter& operator=(Iter const& other) noexcept + { + mKeyVals = other.mKeyVals; + mInfo = other.mInfo; + return *this; + } - // Conversion constructor from iterator to const_iterator. - template ::type> - // NOLINTNEXTLINE(hicpp-explicit-conversions) - Iter(Iter const& other) noexcept - : mKeyVals(other.mKeyVals) - , mInfo(other.mInfo) {} + // prefix increment. Undefined behavior if we are at end()! + Iter& operator++() noexcept + { + mInfo++; + mKeyVals++; + fastForward(); + return *this; + } - Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept - : mKeyVals(valPtr) - , mInfo(infoPtr) {} + Iter operator++(int) noexcept + { + Iter tmp = *this; + ++(*this); + return tmp; + } - Iter(NodePtr valPtr, uint8_t const* infoPtr, - fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept - : mKeyVals(valPtr) - , mInfo(infoPtr) { - fastForward(); - } + reference operator*() const + { + return **mKeyVals; + } - template ::type> - Iter& operator=(Iter const& other) noexcept { - mKeyVals = other.mKeyVals; - mInfo = other.mInfo; - return *this; - } + pointer operator->() const + { + return &**mKeyVals; + } - // prefix increment. Undefined behavior if we are at end()! - Iter& operator++() noexcept { - mInfo++; - mKeyVals++; - fastForward(); - return *this; - } + template + bool operator==(Iter const& o) const noexcept + { + return mKeyVals == o.mKeyVals; + } - Iter operator++(int) noexcept { - Iter tmp = *this; - ++(*this); - return tmp; - } + template + bool operator!=(Iter const& o) const noexcept + { + return mKeyVals != o.mKeyVals; + } - reference operator*() const { - return **mKeyVals; - } - - pointer operator->() const { - return &**mKeyVals; - } - - template - bool operator==(Iter const& o) const noexcept { - return mKeyVals == o.mKeyVals; - } - - template - bool operator!=(Iter const& o) const noexcept { - return mKeyVals != o.mKeyVals; - } - - private: - // fast forward to the next non-free info byte - // I've tried a few variants that don't depend on intrinsics, but unfortunately they are - // quite a bit slower than this one. So I've reverted that change again. See map_benchmark. - void fastForward() noexcept { - size_t n = 0; - while (0U == (n = detail::unaligned_load(mInfo))) { - mInfo += sizeof(size_t); - mKeyVals += sizeof(size_t); - } + private: + // fast forward to the next non-free info byte + // I've tried a few variants that don't depend on intrinsics, but unfortunately they are + // quite a bit slower than this one. So I've reverted that change again. See map_benchmark. + void fastForward() noexcept + { + size_t n = 0; + while (0U == (n = detail::unaligned_load(mInfo))) + { + mInfo += sizeof(size_t); + mKeyVals += sizeof(size_t); + } #if defined(ROBIN_HOOD_DISABLE_INTRINSICS) - // we know for certain that within the next 8 bytes we'll find a non-zero one. - if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { - mInfo += 4; - mKeyVals += 4; - } - if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { - mInfo += 2; - mKeyVals += 2; - } - if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) { - mInfo += 1; - mKeyVals += 1; - } + // we know for certain that within the next 8 bytes we'll find a non-zero one. + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) + { + mInfo += 4; + mKeyVals += 4; + } + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) + { + mInfo += 2; + mKeyVals += 2; + } + if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) + { + mInfo += 1; + mKeyVals += 1; + } #else -# if ROBIN_HOOD(LITTLE_ENDIAN) - auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; -# else - auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; -# endif - mInfo += inc; - mKeyVals += inc; +#if ROBIN_HOOD(LITTLE_ENDIAN) + auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; +#else + auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; +#endif + mInfo += inc; + mKeyVals += inc; #endif - } - - friend class Table; - NodePtr mKeyVals{nullptr}; - uint8_t const* mInfo{nullptr}; - }; - - //////////////////////////////////////////////////////////////////// - - // highly performance relevant code. - // Lower bits are used for indexing into the array (2^n size) - // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. - template - void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const { - // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as - // an additional mixing step. This serves as a bad hash prevention, if the given data is - // badly mixed. - using Mix = - typename std::conditional, hasher>::value, - ::robin_hood::detail::identity_hash, - ::robin_hood::hash>::type; - - // the lower InitialInfoNumBits are reserved for info. - auto h = Mix{}(WHash::operator()(key)); - *info = mInfoInc + static_cast((h & InfoMask) >> mInfoHashShift); - *idx = (h >> InitialInfoNumBits) & mMask; } - // forwards the index by one, wrapping around at the end - void next(InfoType* info, size_t* idx) const noexcept { - *idx = *idx + 1; - *info += mInfoInc; + friend class Table; + NodePtr mKeyVals{nullptr}; + uint8_t const* mInfo{nullptr}; + }; + + //////////////////////////////////////////////////////////////////// + + // highly performance relevant code. + // Lower bits are used for indexing into the array (2^n size) + // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. + template + void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const + { + // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as + // an additional mixing step. This serves as a bad hash prevention, if the given data is + // badly mixed. + using Mix = typename std::conditional, hasher>::value, + ::robin_hood::detail::identity_hash, + ::robin_hood::hash>::type; + + // the lower InitialInfoNumBits are reserved for info. + auto h = Mix{}(WHash::operator()(key)); + *info = mInfoInc + static_cast((h & InfoMask) >> mInfoHashShift); + *idx = (h >> InitialInfoNumBits) & mMask; + } + + // forwards the index by one, wrapping around at the end + void next(InfoType* info, size_t* idx) const noexcept + { + *idx = *idx + 1; + *info += mInfoInc; + } + + void nextWhileLess(InfoType* info, size_t* idx) const noexcept + { + // unrolling this by hand did not bring any speedups. + while (*info < mInfo[*idx]) + { + next(info, idx); + } + } + + // Shift everything up by one element. Tries to move stuff around. + void shiftUp(size_t startIdx, + size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable::value) + { + auto idx = startIdx; + ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); + while (--idx != insertion_idx) + { + mKeyVals[idx] = std::move(mKeyVals[idx - 1]); } - void nextWhileLess(InfoType* info, size_t* idx) const noexcept { - // unrolling this by hand did not bring any speedups. - while (*info < mInfo[*idx]) { - next(info, idx); - } + idx = startIdx; + while (idx != insertion_idx) + { + ROBIN_HOOD_COUNT(shiftUp) + mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); + if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) + { + mMaxNumElementsAllowed = 0; + } + --idx; + } + } + + void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable::value) + { + // until we find one that is either empty or has zero offset. + // TODO(martinus) we don't need to move everything, just the last one for the same + // bucket. + mKeyVals[idx].destroy(*this); + + // until we find one that is either empty or has zero offset. + while (mInfo[idx + 1] >= 2 * mInfoInc) + { + ROBIN_HOOD_COUNT(shiftDown) + mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); + mKeyVals[idx] = std::move(mKeyVals[idx + 1]); + ++idx; } - // Shift everything up by one element. Tries to move stuff around. - void - shiftUp(size_t startIdx, - size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable::value) { - auto idx = startIdx; - ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); - while (--idx != insertion_idx) { - mKeyVals[idx] = std::move(mKeyVals[idx - 1]); - } + mInfo[idx] = 0; + // don't destroy, we've moved it + // mKeyVals[idx].destroy(*this); + mKeyVals[idx].~Node(); + } - idx = startIdx; - while (idx != insertion_idx) { - ROBIN_HOOD_COUNT(shiftUp) - mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); - if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { - mMaxNumElementsAllowed = 0; - } - --idx; - } + // copy of find(), except that it returns iterator instead of const_iterator. + template + ROBIN_HOOD(NODISCARD) + size_t findIdx(Other const& key) const + { + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + + do + { + // unrolling this twice gives a bit of a speedup. More unrolling did not help. + if (info == mInfo[idx] && ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) + { + return idx; + } + next(&info, &idx); + if (info == mInfo[idx] && ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) + { + return idx; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found! + return mMask == 0 ? 0 + : static_cast( + std::distance(mKeyVals, reinterpret_cast_no_cast_align_warning(mInfo))); + } + + void cloneData(const Table& o) + { + Cloner()(o, *this); + } + + // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. + // @return index where the element was created + size_t insert_move(Node&& keyval) + { + // we don't retry, fail if overflowing + // don't need to check max num elements + if (0 == mMaxNumElementsAllowed && !try_increase_info()) + { + throwOverflowError(); // impossible to reach LCOV_EXCL_LINE } - void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable::value) { - // until we find one that is either empty or has zero offset. - // TODO(martinus) we don't need to move everything, just the last one for the same - // bucket. - mKeyVals[idx].destroy(*this); + size_t idx{}; + InfoType info{}; + keyToIdx(keyval.getFirst(), &idx, &info); - // until we find one that is either empty or has zero offset. - while (mInfo[idx + 1] >= 2 * mInfoInc) { - ROBIN_HOOD_COUNT(shiftDown) - mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); - mKeyVals[idx] = std::move(mKeyVals[idx + 1]); - ++idx; - } - - mInfo[idx] = 0; - // don't destroy, we've moved it - // mKeyVals[idx].destroy(*this); - mKeyVals[idx].~Node(); + // skip forward. Use <= because we are certain that the element is not there. + while (info <= mInfo[idx]) + { + idx = idx + 1; + info += mInfoInc; } - // copy of find(), except that it returns iterator instead of const_iterator. - template - ROBIN_HOOD(NODISCARD) - size_t findIdx(Other const& key) const { - size_t idx{}; - InfoType info{}; - keyToIdx(key, &idx, &info); - - do { - // unrolling this twice gives a bit of a speedup. More unrolling did not help. - if (info == mInfo[idx] && - ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { - return idx; - } - next(&info, &idx); - if (info == mInfo[idx] && - ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { - return idx; - } - next(&info, &idx); - } while (info <= mInfo[idx]); - - // nothing found! - return mMask == 0 ? 0 - : static_cast(std::distance( - mKeyVals, reinterpret_cast_no_cast_align_warning(mInfo))); + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = static_cast(info); + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) + { + mMaxNumElementsAllowed = 0; } - void cloneData(const Table& o) { - Cloner()(o, *this); + // find an empty spot + while (0 != mInfo[idx]) + { + next(&info, &idx); } - // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. - // @return index where the element was created - size_t insert_move(Node&& keyval) { - // we don't retry, fail if overflowing - // don't need to check max num elements - if (0 == mMaxNumElementsAllowed && !try_increase_info()) { - throwOverflowError(); // impossible to reach LCOV_EXCL_LINE - } - - size_t idx{}; - InfoType info{}; - keyToIdx(keyval.getFirst(), &idx, &info); - - // skip forward. Use <= because we are certain that the element is not there. - while (info <= mInfo[idx]) { - idx = idx + 1; - info += mInfoInc; - } - - // key not found, so we are now exactly where we want to insert it. - auto const insertion_idx = idx; - auto const insertion_info = static_cast(info); - if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { - mMaxNumElementsAllowed = 0; - } - - // find an empty spot - while (0 != mInfo[idx]) { - next(&info, &idx); - } - - auto& l = mKeyVals[insertion_idx]; - if (idx == insertion_idx) { - ::new (static_cast(&l)) Node(std::move(keyval)); - } else { - shiftUp(idx, insertion_idx); - l = std::move(keyval); - } - - // put at empty spot - mInfo[insertion_idx] = insertion_info; - - ++mNumElements; - return insertion_idx; + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) + { + ::new (static_cast(&l)) Node(std::move(keyval)); + } + else + { + shiftUp(idx, insertion_idx); + l = std::move(keyval); } -public: - using iterator = Iter; - using const_iterator = Iter; + // put at empty spot + mInfo[insertion_idx] = insertion_info; - Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual())) - : WHash() - , WKeyEqual() { - ROBIN_HOOD_TRACE(this) + ++mNumElements; + return insertion_idx; + } + + public: + using iterator = Iter; + using const_iterator = Iter; + + Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual())) : WHash(), WKeyEqual() + { + ROBIN_HOOD_TRACE(this) + } + + // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. + // This tremendously speeds up ctor & dtor of a map that never receives an element. The + // penalty is payed at the first insert, and not before. Lookup of this empty map works + // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the + // standard, but we can ignore it. + explicit Table(size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) + : WHash(h), WKeyEqual(equal) + { + ROBIN_HOOD_TRACE(this) + } + + template + Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) + : WHash(h), WKeyEqual(equal) + { + ROBIN_HOOD_TRACE(this) + insert(first, last); + } + + Table(std::initializer_list initlist, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, + const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) + : WHash(h), WKeyEqual(equal) + { + ROBIN_HOOD_TRACE(this) + insert(initlist.begin(), initlist.end()); + } + + Table(Table&& o) noexcept + : WHash(std::move(static_cast(o))) + , WKeyEqual(std::move(static_cast(o))) + , DataPool(std::move(static_cast(o))) + { + ROBIN_HOOD_TRACE(this) + if (o.mMask) + { + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + // set other's mask to 0 so its destructor won't do anything + o.init(); } + } - // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. - // This tremendously speeds up ctor & dtor of a map that never receives an element. The - // penalty is payed at the first insert, and not before. Lookup of this empty map works - // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the - // standard, but we can ignore it. - explicit Table( - size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{}, - const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) - : WHash(h) - , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this) - } - - template - Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, - const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) - : WHash(h) - , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this) - insert(first, last); - } - - Table(std::initializer_list initlist, - size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, - const KeyEqual& equal = KeyEqual{}) - : WHash(h) - , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this) - insert(initlist.begin(), initlist.end()); - } - - Table(Table&& o) noexcept - : WHash(std::move(static_cast(o))) - , WKeyEqual(std::move(static_cast(o))) - , DataPool(std::move(static_cast(o))) { - ROBIN_HOOD_TRACE(this) - if (o.mMask) { - mKeyVals = std::move(o.mKeyVals); - mInfo = std::move(o.mInfo); - mNumElements = std::move(o.mNumElements); - mMask = std::move(o.mMask); - mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); - mInfoInc = std::move(o.mInfoInc); - mInfoHashShift = std::move(o.mInfoHashShift); - // set other's mask to 0 so its destructor won't do anything - o.init(); - } - } - - Table& operator=(Table&& o) noexcept { - ROBIN_HOOD_TRACE(this) - if (&o != this) { - if (o.mMask) { - // only move stuff if the other map actually has some data - destroy(); - mKeyVals = std::move(o.mKeyVals); - mInfo = std::move(o.mInfo); - mNumElements = std::move(o.mNumElements); - mMask = std::move(o.mMask); - mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); - mInfoInc = std::move(o.mInfoInc); - mInfoHashShift = std::move(o.mInfoHashShift); - WHash::operator=(std::move(static_cast(o))); - WKeyEqual::operator=(std::move(static_cast(o))); - DataPool::operator=(std::move(static_cast(o))); - - o.init(); - - } else { - // nothing in the other map => just clear us. - clear(); - } - } - return *this; - } - - Table(const Table& o) - : WHash(static_cast(o)) - , WKeyEqual(static_cast(o)) - , DataPool(static_cast(o)) { - ROBIN_HOOD_TRACE(this) - if (!o.empty()) { - // not empty: create an exact copy. it is also possible to just iterate through all - // elements and insert them, but copying is probably faster. - - auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); - auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); - - ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" - << numElementsWithBuffer << ")") - mKeyVals = static_cast( - detail::assertNotNull(std::malloc(numBytesTotal))); - // no need for calloc because clonData does memcpy - mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); - mNumElements = o.mNumElements; - mMask = o.mMask; - mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; - mInfoInc = o.mInfoInc; - mInfoHashShift = o.mInfoHashShift; - cloneData(o); - } - } - - // Creates a copy of the given map. Copy constructor of each entry is used. - // Not sure why clang-tidy thinks this doesn't handle self assignment, it does - // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) - Table& operator=(Table const& o) { - ROBIN_HOOD_TRACE(this) - if (&o == this) { - // prevent assigning of itself - return *this; - } - - // we keep using the old allocator and not assign the new one, because we want to keep - // the memory available. when it is the same size. - if (o.empty()) { - if (0 == mMask) { - // nothing to do, we are empty too - return *this; - } - - // not empty: destroy what we have there - // clear also resets mInfo to 0, that's sometimes not necessary. - destroy(); - init(); - WHash::operator=(static_cast(o)); - WKeyEqual::operator=(static_cast(o)); - DataPool::operator=(static_cast(o)); - - return *this; - } - - // clean up old stuff - Destroyer::value>{}.nodes(*this); - - if (mMask != o.mMask) { - // no luck: we don't have the same array size allocated, so we need to realloc. - if (0 != mMask) { - // only deallocate if we actually have data! - ROBIN_HOOD_LOG("std::free") - std::free(mKeyVals); - } - - auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); - auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); - ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" - << numElementsWithBuffer << ")") - mKeyVals = static_cast( - detail::assertNotNull(std::malloc(numBytesTotal))); - - // no need for calloc here because cloneData performs a memcpy. - mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); - // sentinel is set in cloneData - } - WHash::operator=(static_cast(o)); - WKeyEqual::operator=(static_cast(o)); - DataPool::operator=(static_cast(o)); - mNumElements = o.mNumElements; - mMask = o.mMask; - mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; - mInfoInc = o.mInfoInc; - mInfoHashShift = o.mInfoHashShift; - cloneData(o); - - return *this; - } - - // Swaps everything between the two maps. - void swap(Table& o) { - ROBIN_HOOD_TRACE(this) - using std::swap; - swap(o, *this); - } - - // Clears all data, without resizing. - void clear() { - ROBIN_HOOD_TRACE(this) - if (empty()) { - // don't do anything! also important because we don't want to write to - // DummyInfoByte::b, even though we would just write 0 to it. - return; - } - - Destroyer::value>{}.nodes(*this); - - auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); - // clear everything, then set the sentinel again - uint8_t const z = 0; - std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); - mInfo[numElementsWithBuffer] = 1; - - mInfoInc = InitialInfoInc; - mInfoHashShift = InitialInfoHashShift; - } - - // Destroys the map and all it's contents. - ~Table() { - ROBIN_HOOD_TRACE(this) + Table& operator=(Table&& o) noexcept + { + ROBIN_HOOD_TRACE(this) + if (&o != this) + { + if (o.mMask) + { + // only move stuff if the other map actually has some data destroy(); + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + WHash::operator=(std::move(static_cast(o))); + WKeyEqual::operator=(std::move(static_cast(o))); + DataPool::operator=(std::move(static_cast(o))); + + o.init(); + } + else + { + // nothing in the other map => just clear us. + clear(); + } + } + return *this; + } + + Table(const Table& o) + : WHash(static_cast(o)) + , WKeyEqual(static_cast(o)) + , DataPool(static_cast(o)) + { + ROBIN_HOOD_TRACE(this) + if (!o.empty()) + { + // not empty: create an exact copy. it is also possible to just iterate through all + // elements and insert them, but copying is probably faster. + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + + ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer + << ")") + mKeyVals = static_cast(detail::assertNotNull(std::malloc(numBytesTotal))); + // no need for calloc because clonData does memcpy + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + } + } + + // Creates a copy of the given map. Copy constructor of each entry is used. + // Not sure why clang-tidy thinks this doesn't handle self assignment, it does + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + Table& operator=(Table const& o) + { + ROBIN_HOOD_TRACE(this) + if (&o == this) + { + // prevent assigning of itself + return *this; } - // Checks if both tables contain the same entries. Order is irrelevant. - bool operator==(const Table& other) const { - ROBIN_HOOD_TRACE(this) - if (other.size() != size()) { - return false; - } - for (auto const& otherEntry : other) { - if (!has(otherEntry)) { - return false; - } - } + // we keep using the old allocator and not assign the new one, because we want to keep + // the memory available. when it is the same size. + if (o.empty()) + { + if (0 == mMask) + { + // nothing to do, we are empty too + return *this; + } - return true; + // not empty: destroy what we have there + // clear also resets mInfo to 0, that's sometimes not necessary. + destroy(); + init(); + WHash::operator=(static_cast(o)); + WKeyEqual::operator=(static_cast(o)); + DataPool::operator=(static_cast(o)); + + return *this; } - bool operator!=(const Table& other) const { - ROBIN_HOOD_TRACE(this) - return !operator==(other); + // clean up old stuff + Destroyer::value>{}.nodes(*this); + + if (mMask != o.mMask) + { + // no luck: we don't have the same array size allocated, so we need to realloc. + if (0 != mMask) + { + // only deallocate if we actually have data! + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer + << ")") + mKeyVals = static_cast(detail::assertNotNull(std::malloc(numBytesTotal))); + + // no need for calloc here because cloneData performs a memcpy. + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + // sentinel is set in cloneData + } + WHash::operator=(static_cast(o)); + WKeyEqual::operator=(static_cast(o)); + DataPool::operator=(static_cast(o)); + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + + return *this; + } + + // Swaps everything between the two maps. + void swap(Table& o) + { + ROBIN_HOOD_TRACE(this) + using std::swap; + swap(o, *this); + } + + // Clears all data, without resizing. + void clear() + { + ROBIN_HOOD_TRACE(this) + if (empty()) + { + // don't do anything! also important because we don't want to write to + // DummyInfoByte::b, even though we would just write 0 to it. + return; } - template - typename std::enable_if::value, Q&>::type operator[](const key_type& key) { - ROBIN_HOOD_TRACE(this) - return doCreateByKey(key); + Destroyer::value>{}.nodes(*this); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + // clear everything, then set the sentinel again + uint8_t const z = 0; + std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // Destroys the map and all it's contents. + ~Table() + { + ROBIN_HOOD_TRACE(this) + destroy(); + } + + // Checks if both tables contain the same entries. Order is irrelevant. + bool operator==(const Table& other) const + { + ROBIN_HOOD_TRACE(this) + if (other.size() != size()) + { + return false; + } + for (auto const& otherEntry : other) + { + if (!has(otherEntry)) + { + return false; + } } - template - typename std::enable_if::value, Q&>::type operator[](key_type&& key) { - ROBIN_HOOD_TRACE(this) - return doCreateByKey(std::move(key)); + return true; + } + + bool operator!=(const Table& other) const + { + ROBIN_HOOD_TRACE(this) + return !operator==(other); + } + + template + typename std::enable_if::value, Q&>::type operator[](const key_type& key) + { + ROBIN_HOOD_TRACE(this) + return doCreateByKey(key); + } + + template + typename std::enable_if::value, Q&>::type operator[](key_type&& key) + { + ROBIN_HOOD_TRACE(this) + return doCreateByKey(std::move(key)); + } + + template + void insert(Iter first, Iter last) + { + for (; first != last; ++first) + { + // value_type ctor needed because this might be called with std::pair's + insert(value_type(*first)); + } + } + + template + std::pair emplace(Args&&... args) + { + ROBIN_HOOD_TRACE(this) + Node n{*this, std::forward(args)...}; + auto r = doInsert(std::move(n)); + if (!r.second) + { + // insertion not possible: destroy node + // NOLINTNEXTLINE(bugprone-use-after-move) + n.destroy(*this); + } + return r; + } + + template + std::pair try_emplace(const key_type& key, Args&&... args) + { + return try_emplace_impl(key, std::forward(args)...); + } + + template + std::pair try_emplace(key_type&& key, Args&&... args) + { + return try_emplace_impl(std::move(key), std::forward(args)...); + } + + template + std::pair try_emplace(const_iterator hint, const key_type& key, Args&&... args) + { + (void)hint; + return try_emplace_impl(key, std::forward(args)...); + } + + template + std::pair try_emplace(const_iterator hint, key_type&& key, Args&&... args) + { + (void)hint; + return try_emplace_impl(std::move(key), std::forward(args)...); + } + + template + std::pair insert_or_assign(const key_type& key, Mapped&& obj) + { + return insert_or_assign_impl(key, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type&& key, Mapped&& obj) + { + return insert_or_assign_impl(std::move(key), std::forward(obj)); + } + + template + std::pair insert_or_assign(const_iterator hint, const key_type& key, Mapped&& obj) + { + (void)hint; + return insert_or_assign_impl(key, std::forward(obj)); + } + + template + std::pair insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) + { + (void)hint; + return insert_or_assign_impl(std::move(key), std::forward(obj)); + } + + std::pair insert(const value_type& keyval) + { + ROBIN_HOOD_TRACE(this) + return doInsert(keyval); + } + + std::pair insert(value_type&& keyval) + { + return doInsert(std::move(keyval)); + } + + // Returns 1 if key is found, 0 otherwise. + size_t count(const key_type& key) const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) + { + return 1; + } + return 0; + } + + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type count(const OtherKey& key) const + { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) + { + return 1; + } + return 0; + } + + bool contains(const key_type& key) const + { // NOLINT(modernize-use-nodiscard) + return 1U == count(key); + } + + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type contains(const OtherKey& key) const + { + return 1U == count(key); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q&>::type at(key_type const& key) + { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) + { + doThrow("key not found"); + } + return kv->getSecond(); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q const&>::type at(key_type const& key) const + { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) + { + doThrow("key not found"); + } + return kv->getSecond(); + } + + const_iterator find(const key_type& key) const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const + { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type // NOLINT(modernize-use-nodiscard) + find(const OtherKey& key) const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator find(const key_type& key) + { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + iterator find(const OtherKey& key, is_transparent_tag /*unused*/) + { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type find(const OtherKey& key) + { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator begin() + { + ROBIN_HOOD_TRACE(this) + if (empty()) + { + return end(); + } + return iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + const_iterator begin() const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return cbegin(); + } + const_iterator cbegin() const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + if (empty()) + { + return cend(); + } + return const_iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + + iterator end() + { + ROBIN_HOOD_TRACE(this) + // no need to supply valid info pointer: end() must not be dereferenced, and only node + // pointer is compared. + return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; + } + const_iterator end() const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return cend(); + } + const_iterator cend() const + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; + } + + iterator erase(const_iterator pos) + { + ROBIN_HOOD_TRACE(this) + // its safe to perform const cast here + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); + } + + // Erases element at pos, returns iterator to the next element. + iterator erase(iterator pos) + { + ROBIN_HOOD_TRACE(this) + // we assume that pos always points to a valid entry, and not end(). + auto const idx = static_cast(pos.mKeyVals - mKeyVals); + + shiftDown(idx); + --mNumElements; + + if (*pos.mInfo) + { + // we've backward shifted, return this again + return pos; } - template - void insert(Iter first, Iter last) { - for (; first != last; ++first) { - // value_type ctor needed because this might be called with std::pair's - insert(value_type(*first)); - } - } + // no backward shift, return next element + return ++pos; + } - template - std::pair emplace(Args&&... args) { - ROBIN_HOOD_TRACE(this) - Node n{*this, std::forward(args)...}; - auto r = doInsert(std::move(n)); - if (!r.second) { - // insertion not possible: destroy node - // NOLINTNEXTLINE(bugprone-use-after-move) - n.destroy(*this); - } - return r; - } - - template - std::pair try_emplace(const key_type& key, Args&&... args) { - return try_emplace_impl(key, std::forward(args)...); - } - - template - std::pair try_emplace(key_type&& key, Args&&... args) { - return try_emplace_impl(std::move(key), std::forward(args)...); - } - - template - std::pair try_emplace(const_iterator hint, const key_type& key, - Args&&... args) { - (void)hint; - return try_emplace_impl(key, std::forward(args)...); - } - - template - std::pair try_emplace(const_iterator hint, key_type&& key, Args&&... args) { - (void)hint; - return try_emplace_impl(std::move(key), std::forward(args)...); - } - - template - std::pair insert_or_assign(const key_type& key, Mapped&& obj) { - return insert_or_assign_impl(key, std::forward(obj)); - } - - template - std::pair insert_or_assign(key_type&& key, Mapped&& obj) { - return insert_or_assign_impl(std::move(key), std::forward(obj)); - } - - template - std::pair insert_or_assign(const_iterator hint, const key_type& key, - Mapped&& obj) { - (void)hint; - return insert_or_assign_impl(key, std::forward(obj)); - } - - template - std::pair insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) { - (void)hint; - return insert_or_assign_impl(std::move(key), std::forward(obj)); - } - - std::pair insert(const value_type& keyval) { - ROBIN_HOOD_TRACE(this) - return doInsert(keyval); - } - - std::pair insert(value_type&& keyval) { - return doInsert(std::move(keyval)); - } - - // Returns 1 if key is found, 0 otherwise. - size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - auto kv = mKeyVals + findIdx(key); - if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { - return 1; - } - return 0; - } - - template - // NOLINTNEXTLINE(modernize-use-nodiscard) - typename std::enable_if::type count(const OtherKey& key) const { - ROBIN_HOOD_TRACE(this) - auto kv = mKeyVals + findIdx(key); - if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { - return 1; - } - return 0; - } - - bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - return 1U == count(key); - } - - template - // NOLINTNEXTLINE(modernize-use-nodiscard) - typename std::enable_if::type contains(const OtherKey& key) const { - return 1U == count(key); - } - - // Returns a reference to the value found for key. - // Throws std::out_of_range if element cannot be found - template - // NOLINTNEXTLINE(modernize-use-nodiscard) - typename std::enable_if::value, Q&>::type at(key_type const& key) { - ROBIN_HOOD_TRACE(this) - auto kv = mKeyVals + findIdx(key); - if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { - doThrow("key not found"); - } - return kv->getSecond(); - } - - // Returns a reference to the value found for key. - // Throws std::out_of_range if element cannot be found - template - // NOLINTNEXTLINE(modernize-use-nodiscard) - typename std::enable_if::value, Q const&>::type at(key_type const& key) const { - ROBIN_HOOD_TRACE(this) - auto kv = mKeyVals + findIdx(key); - if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { - doThrow("key not found"); - } - return kv->getSecond(); - } - - const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - const size_t idx = findIdx(key); - return const_iterator{mKeyVals + idx, mInfo + idx}; - } - - template - const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const { - ROBIN_HOOD_TRACE(this) - const size_t idx = findIdx(key); - return const_iterator{mKeyVals + idx, mInfo + idx}; - } - - template - typename std::enable_if::type // NOLINT(modernize-use-nodiscard) - find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - const size_t idx = findIdx(key); - return const_iterator{mKeyVals + idx, mInfo + idx}; - } - - iterator find(const key_type& key) { - ROBIN_HOOD_TRACE(this) - const size_t idx = findIdx(key); - return iterator{mKeyVals + idx, mInfo + idx}; - } - - template - iterator find(const OtherKey& key, is_transparent_tag /*unused*/) { - ROBIN_HOOD_TRACE(this) - const size_t idx = findIdx(key); - return iterator{mKeyVals + idx, mInfo + idx}; - } - - template - typename std::enable_if::type find(const OtherKey& key) { - ROBIN_HOOD_TRACE(this) - const size_t idx = findIdx(key); - return iterator{mKeyVals + idx, mInfo + idx}; - } - - iterator begin() { - ROBIN_HOOD_TRACE(this) - if (empty()) { - return end(); - } - return iterator(mKeyVals, mInfo, fast_forward_tag{}); - } - const_iterator begin() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return cbegin(); - } - const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - if (empty()) { - return cend(); - } - return const_iterator(mKeyVals, mInfo, fast_forward_tag{}); - } - - iterator end() { - ROBIN_HOOD_TRACE(this) - // no need to supply valid info pointer: end() must not be dereferenced, and only node - // pointer is compared. - return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; - } - const_iterator end() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return cend(); - } - const_iterator cend() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; - } - - iterator erase(const_iterator pos) { - ROBIN_HOOD_TRACE(this) - // its safe to perform const cast here - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) - return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); - } - - // Erases element at pos, returns iterator to the next element. - iterator erase(iterator pos) { - ROBIN_HOOD_TRACE(this) - // we assume that pos always points to a valid entry, and not end(). - auto const idx = static_cast(pos.mKeyVals - mKeyVals); + size_t erase(const key_type& key) + { + ROBIN_HOOD_TRACE(this) + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + // check while info matches with the source idx + do + { + if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) + { shiftDown(idx); --mNumElements; + return 1; + } + next(&info, &idx); + } while (info <= mInfo[idx]); - if (*pos.mInfo) { - // we've backward shifted, return this again - return pos; - } + // nothing found to delete + return 0; + } - // no backward shift, return next element - return ++pos; + // reserves space for the specified number of elements. Makes sure the old data fits. + // exactly the same as reserve(c). + void rehash(size_t c) + { + // forces a reserve + reserve(c, true); + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // Exactly the same as rehash(c). Use rehash(0) to shrink to fit. + void reserve(size_t c) + { + // reserve, but don't force rehash + reserve(c, false); + } + + // If possible reallocates the map to a smaller one. This frees the underlying table. + // Does not do anything if load_factor is too large for decreasing the table's size. + void compact() + { + ROBIN_HOOD_TRACE(this) + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) + { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) + { + throwOverflowError(); } - size_t erase(const key_type& key) { - ROBIN_HOOD_TRACE(this) - size_t idx{}; - InfoType info{}; - keyToIdx(key, &idx, &info); + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") - // check while info matches with the source idx - do { - if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { - shiftDown(idx); - --mNumElements; - return 1; - } - next(&info, &idx); - } while (info <= mInfo[idx]); + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (newSize < mMask + 1) + { + rehashPowerOfTwo(newSize, true); + } + } - // nothing found to delete - return 0; + size_type size() const noexcept + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return mNumElements; + } + + size_type max_size() const noexcept + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return static_cast(-1); + } + + ROBIN_HOOD(NODISCARD) bool empty() const noexcept + { + ROBIN_HOOD_TRACE(this) + return 0 == mNumElements; + } + + float max_load_factor() const noexcept + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return MaxLoadFactor100 / 100.0F; + } + + // Average number of elements per bucket. Since we allow only 1 per bucket + float load_factor() const noexcept + { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return static_cast(size()) / static_cast(mMask + 1); + } + + ROBIN_HOOD(NODISCARD) size_t mask() const noexcept + { + ROBIN_HOOD_TRACE(this) + return mMask; + } + + ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept + { + if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits::max)() / 100)) + { + return maxElements * MaxLoadFactor100 / 100; } - // reserves space for the specified number of elements. Makes sure the old data fits. - // exactly the same as reserve(c). - void rehash(size_t c) { - // forces a reserve - reserve(c, true); - } + // we might be a bit inprecise, but since maxElements is quite large that doesn't matter + return (maxElements / 100) * MaxLoadFactor100; + } - // reserves space for the specified number of elements. Makes sure the old data fits. - // Exactly the same as rehash(c). Use rehash(0) to shrink to fit. - void reserve(size_t c) { - // reserve, but don't force rehash - reserve(c, false); - } + ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept + { + // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load + // 64bit types. + return numElements + sizeof(uint64_t); + } - // If possible reallocates the map to a smaller one. This frees the underlying table. - // Does not do anything if load_factor is too large for decreasing the table's size. - void compact() { - ROBIN_HOOD_TRACE(this) - auto newSize = InitialNumElements; - while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) { - newSize *= 2; - } - if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { - throwOverflowError(); - } + ROBIN_HOOD(NODISCARD) + size_t calcNumElementsWithBuffer(size_t numElements) const noexcept + { + auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); + return numElements + (std::min)(maxNumElementsAllowed, (static_cast(0xFF))); + } - ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") - - // only actually do anything when the new size is bigger than the old one. This prevents to - // continuously allocate for each reserve() call. - if (newSize < mMask + 1) { - rehashPowerOfTwo(newSize, true); - } - } - - size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return mNumElements; - } - - size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return static_cast(-1); - } - - ROBIN_HOOD(NODISCARD) bool empty() const noexcept { - ROBIN_HOOD_TRACE(this) - return 0 == mNumElements; - } - - float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return MaxLoadFactor100 / 100.0F; - } - - // Average number of elements per bucket. Since we allow only 1 per bucket - float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this) - return static_cast(size()) / static_cast(mMask + 1); - } - - ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { - ROBIN_HOOD_TRACE(this) - return mMask; - } - - ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept { - if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits::max)() / 100)) { - return maxElements * MaxLoadFactor100 / 100; - } - - // we might be a bit inprecise, but since maxElements is quite large that doesn't matter - return (maxElements / 100) * MaxLoadFactor100; - } - - ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept { - // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load - // 64bit types. - return numElements + sizeof(uint64_t); - } - - ROBIN_HOOD(NODISCARD) - size_t calcNumElementsWithBuffer(size_t numElements) const noexcept { - auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); - return numElements + (std::min)(maxNumElementsAllowed, (static_cast(0xFF))); - } - - // calculation only allowed for 2^n values - ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const { + // calculation only allowed for 2^n values + ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const + { #if ROBIN_HOOD(BITNESS) == 64 - return numElements * sizeof(Node) + calcNumBytesInfo(numElements); + return numElements * sizeof(Node) + calcNumBytesInfo(numElements); #else - // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows. - auto const ne = static_cast(numElements); - auto const s = static_cast(sizeof(Node)); - auto const infos = static_cast(calcNumBytesInfo(numElements)); + // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows. + auto const ne = static_cast(numElements); + auto const s = static_cast(sizeof(Node)); + auto const infos = static_cast(calcNumBytesInfo(numElements)); - auto const total64 = ne * s + infos; - auto const total = static_cast(total64); + auto const total64 = ne * s + infos; + auto const total = static_cast(total64); - if (ROBIN_HOOD_UNLIKELY(static_cast(total) != total64)) { - throwOverflowError(); - } - return total; + if (ROBIN_HOOD_UNLIKELY(static_cast(total) != total64)) + { + throwOverflowError(); + } + return total; #endif + } + + private: + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const + { + ROBIN_HOOD_TRACE(this) + auto it = find(e.first); + return it != end() && it->second == e.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const + { + ROBIN_HOOD_TRACE(this) + return find(e) != end(); + } + + void reserve(size_t c, bool forceRehash) + { + ROBIN_HOOD_TRACE(this) + auto const minElementsAllowed = (std::max)(c, mNumElements); + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) + { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) + { + throwOverflowError(); } -private: - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, bool>::type has(const value_type& e) const { - ROBIN_HOOD_TRACE(this) - auto it = find(e.first); - return it != end() && it->second == e.second; - } + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") - template - ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, bool>::type has(const value_type& e) const { - ROBIN_HOOD_TRACE(this) - return find(e) != end(); + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (forceRehash || newSize > mMask + 1) + { + rehashPowerOfTwo(newSize, false); } + } - void reserve(size_t c, bool forceRehash) { - ROBIN_HOOD_TRACE(this) - auto const minElementsAllowed = (std::max)(c, mNumElements); - auto newSize = InitialNumElements; - while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { - newSize *= 2; - } - if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { - throwOverflowError(); + // reserves space for at least the specified number of elements. + // only works if numBuckets if power of two + void rehashPowerOfTwo(size_t numBuckets, bool forceFree) + { + ROBIN_HOOD_TRACE(this) + + Node* const oldKeyVals = mKeyVals; + uint8_t const* const oldInfo = mInfo; + + const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + // resize operation: move stuff + init_data(numBuckets); + if (oldMaxElementsWithBuffer > 1) + { + for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) + { + if (oldInfo[i] != 0) + { + insert_move(std::move(oldKeyVals[i])); + // destroy the node but DON'T destroy the data. + oldKeyVals[i].~Node(); } + } - ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") - - // only actually do anything when the new size is bigger than the old one. This prevents to - // continuously allocate for each reserve() call. - if (forceRehash || newSize > mMask + 1) { - rehashPowerOfTwo(newSize, false); + // this check is not necessary as it's guarded by the previous if, but it helps silence + // g++'s overeager "attempt to free a non-heap object 'map' + // [-Werror=free-nonheap-object]" warning. + if (oldKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) + { + // don't destroy old data: put it into the pool instead + if (forceFree) + { + std::free(oldKeyVals); } + else + { + DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); + } + } } + } - // reserves space for at least the specified number of elements. - // only works if numBuckets if power of two - void rehashPowerOfTwo(size_t numBuckets, bool forceFree) { - ROBIN_HOOD_TRACE(this) - - Node* const oldKeyVals = mKeyVals; - uint8_t const* const oldInfo = mInfo; - - const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); - - // resize operation: move stuff - init_data(numBuckets); - if (oldMaxElementsWithBuffer > 1) { - for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) { - if (oldInfo[i] != 0) { - insert_move(std::move(oldKeyVals[i])); - // destroy the node but DON'T destroy the data. - oldKeyVals[i].~Node(); - } - } - - // this check is not necessary as it's guarded by the previous if, but it helps silence - // g++'s overeager "attempt to free a non-heap object 'map' - // [-Werror=free-nonheap-object]" warning. - if (oldKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { - // don't destroy old data: put it into the pool instead - if (forceFree) { - std::free(oldKeyVals); - } else { - DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); - } - } - } - } - - ROBIN_HOOD(NOINLINE) void throwOverflowError() const { + ROBIN_HOOD(NOINLINE) void throwOverflowError() const + { #if ROBIN_HOOD(HAS_EXCEPTIONS) - throw std::overflow_error("robin_hood::map overflow"); + throw std::overflow_error("robin_hood::map overflow"); #else - abort(); + abort(); #endif + } + + template + std::pair try_emplace_impl(OtherKey&& key, Args&&... args) + { + ROBIN_HOOD_TRACE(this) + auto it = find(key); + if (it == end()) + { + return emplace(std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); } + return {it, false}; + } - template - std::pair try_emplace_impl(OtherKey&& key, Args&&... args) { - ROBIN_HOOD_TRACE(this) - auto it = find(key); - if (it == end()) { - return emplace(std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), - std::forward_as_tuple(std::forward(args)...)); - } - return {it, false}; + template + std::pair insert_or_assign_impl(OtherKey&& key, Mapped&& obj) + { + ROBIN_HOOD_TRACE(this) + auto it = find(key); + if (it == end()) + { + return emplace(std::forward(key), std::forward(obj)); } + it->second = std::forward(obj); + return {it, false}; + } - template - std::pair insert_or_assign_impl(OtherKey&& key, Mapped&& obj) { - ROBIN_HOOD_TRACE(this) - auto it = find(key); - if (it == end()) { - return emplace(std::forward(key), std::forward(obj)); + void init_data(size_t max_elements) + { + mNumElements = 0; + mMask = max_elements - 1; + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); + + // calloc also zeroes everything + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer << ")") + mKeyVals = reinterpret_cast(detail::assertNotNull(std::calloc(1, numBytesTotal))); + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + + // set sentinel + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + template + typename std::enable_if::value, Q&>::type doCreateByKey(Arg&& key) + { + while (true) + { + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + nextWhileLess(&info, &idx); + + // while we potentially have a match. Can't do a do-while here because when mInfo is + // 0 we don't want to skip forward + while (info == mInfo[idx]) + { + if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) + { + // key already exists, do not insert. + return mKeyVals[idx].getSecond(); } - it->second = std::forward(obj); - return {it, false}; - } + next(&info, &idx); + } - void init_data(size_t max_elements) { - mNumElements = 0; - mMask = max_elements - 1; - mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); + // unlikely that this evaluates to true + if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) + { + increase_size(); + continue; + } - auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); - - // calloc also zeroes everything - auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); - ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" - << numElementsWithBuffer << ")") - mKeyVals = reinterpret_cast( - detail::assertNotNull(std::calloc(1, numBytesTotal))); - mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); - - // set sentinel - mInfo[numElementsWithBuffer] = 1; - - mInfoInc = InitialInfoInc; - mInfoHashShift = InitialInfoHashShift; - } - - template - typename std::enable_if::value, Q&>::type doCreateByKey(Arg&& key) { - while (true) { - size_t idx{}; - InfoType info{}; - keyToIdx(key, &idx, &info); - nextWhileLess(&info, &idx); - - // while we potentially have a match. Can't do a do-while here because when mInfo is - // 0 we don't want to skip forward - while (info == mInfo[idx]) { - if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { - // key already exists, do not insert. - return mKeyVals[idx].getSecond(); - } - next(&info, &idx); - } - - // unlikely that this evaluates to true - if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { - increase_size(); - continue; - } - - // key not found, so we are now exactly where we want to insert it. - auto const insertion_idx = idx; - auto const insertion_info = info; - if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { - mMaxNumElementsAllowed = 0; - } - - // find an empty spot - while (0 != mInfo[idx]) { - next(&info, &idx); - } - - auto& l = mKeyVals[insertion_idx]; - if (idx == insertion_idx) { - // put at empty spot. This forwards all arguments into the node where the object - // is constructed exactly where it is needed. - ::new (static_cast(&l)) - Node(*this, std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); - } else { - shiftUp(idx, insertion_idx); - l = Node(*this, std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); - } - - // mKeyVals[idx].getFirst() = std::move(key); - mInfo[insertion_idx] = static_cast(insertion_info); - - ++mNumElements; - return mKeyVals[insertion_idx].getSecond(); - } - } - - // This is exactly the same code as operator[], except for the return values - template - std::pair doInsert(Arg&& keyval) { - while (true) { - size_t idx{}; - InfoType info{}; - keyToIdx(getFirstConst(keyval), &idx, &info); - nextWhileLess(&info, &idx); - - // while we potentially have a match - while (info == mInfo[idx]) { - if (WKeyEqual::operator()(getFirstConst(keyval), mKeyVals[idx].getFirst())) { - // key already exists, do NOT insert. - // see http://en.cppreference.com/w/cpp/container/unordered_map/insert - return std::make_pair(iterator(mKeyVals + idx, mInfo + idx), - false); - } - next(&info, &idx); - } - - // unlikely that this evaluates to true - if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { - increase_size(); - continue; - } - - // key not found, so we are now exactly where we want to insert it. - auto const insertion_idx = idx; - auto const insertion_info = info; - if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { - mMaxNumElementsAllowed = 0; - } - - // find an empty spot - while (0 != mInfo[idx]) { - next(&info, &idx); - } - - auto& l = mKeyVals[insertion_idx]; - if (idx == insertion_idx) { - ::new (static_cast(&l)) Node(*this, std::forward(keyval)); - } else { - shiftUp(idx, insertion_idx); - l = Node(*this, std::forward(keyval)); - } - - // put at empty spot - mInfo[insertion_idx] = static_cast(insertion_info); - - ++mNumElements; - return std::make_pair(iterator(mKeyVals + insertion_idx, mInfo + insertion_idx), true); - } - } - - bool try_increase_info() { - ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements - << ", maxNumElementsAllowed=" - << calcMaxNumElementsAllowed(mMask + 1)) - if (mInfoInc <= 2) { - // need to be > 2 so that shift works (otherwise undefined behavior!) - return false; - } - // we got space left, try to make info smaller - mInfoInc = static_cast(mInfoInc >> 1U); - - // remove one bit of the hash, leaving more space for the distance info. - // This is extremely fast because we can operate on 8 bytes at once. - ++mInfoHashShift; - auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); - - for (size_t i = 0; i < numElementsWithBuffer; i += 8) { - auto val = unaligned_load(mInfo + i); - val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); - std::memcpy(mInfo + i, &val, sizeof(val)); - } - // update sentinel, which might have been cleared out! - mInfo[numElementsWithBuffer] = 1; - - mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); - return true; - } - - void increase_size() { - // nothing allocated yet? just allocate InitialNumElements - if (0 == mMask) { - init_data(InitialNumElements); - return; - } - - auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); - if (mNumElements < maxNumElementsAllowed && try_increase_info()) { - return; - } - - ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" - << maxNumElementsAllowed << ", load=" - << (static_cast(mNumElements) * 100.0 / - (static_cast(mMask) + 1))) - // it seems we have a really bad hash function! don't try to resize again - if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { - throwOverflowError(); - } - - rehashPowerOfTwo((mMask + 1) * 2, false); - } - - void destroy() { - if (0 == mMask) { - // don't deallocate! - return; - } - - Destroyer::value>{} - .nodesDoNotDeallocate(*this); - - // This protection against not deleting mMask shouldn't be needed as it's sufficiently - // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise - // reports a compile error: attempt to free a non-heap object 'fm' - // [-Werror=free-nonheap-object] - if (mKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { - ROBIN_HOOD_LOG("std::free") - std::free(mKeyVals); - } - } - - void init() noexcept { - mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); - mInfo = reinterpret_cast(&mMask); - mNumElements = 0; - mMask = 0; + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = info; + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) + { mMaxNumElementsAllowed = 0; - mInfoInc = InitialInfoInc; - mInfoHashShift = InitialInfoHashShift; + } + + // find an empty spot + while (0 != mInfo[idx]) + { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) + { + // put at empty spot. This forwards all arguments into the node where the object + // is constructed exactly where it is needed. + ::new (static_cast(&l)) + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple()); + } + else + { + shiftUp(idx, insertion_idx); + l = Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple()); + } + + // mKeyVals[idx].getFirst() = std::move(key); + mInfo[insertion_idx] = static_cast(insertion_info); + + ++mNumElements; + return mKeyVals[insertion_idx].getSecond(); + } + } + + // This is exactly the same code as operator[], except for the return values + template + std::pair doInsert(Arg&& keyval) + { + while (true) + { + size_t idx{}; + InfoType info{}; + keyToIdx(getFirstConst(keyval), &idx, &info); + nextWhileLess(&info, &idx); + + // while we potentially have a match + while (info == mInfo[idx]) + { + if (WKeyEqual::operator()(getFirstConst(keyval), mKeyVals[idx].getFirst())) + { + // key already exists, do NOT insert. + // see http://en.cppreference.com/w/cpp/container/unordered_map/insert + return std::make_pair(iterator(mKeyVals + idx, mInfo + idx), false); + } + next(&info, &idx); + } + + // unlikely that this evaluates to true + if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) + { + increase_size(); + continue; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = info; + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) + { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) + { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) + { + ::new (static_cast(&l)) Node(*this, std::forward(keyval)); + } + else + { + shiftUp(idx, insertion_idx); + l = Node(*this, std::forward(keyval)); + } + + // put at empty spot + mInfo[insertion_idx] = static_cast(insertion_info); + + ++mNumElements; + return std::make_pair(iterator(mKeyVals + insertion_idx, mInfo + insertion_idx), true); + } + } + + bool try_increase_info() + { + ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements + << ", maxNumElementsAllowed=" << calcMaxNumElementsAllowed(mMask + 1)) + if (mInfoInc <= 2) + { + // need to be > 2 so that shift works (otherwise undefined behavior!) + return false; + } + // we got space left, try to make info smaller + mInfoInc = static_cast(mInfoInc >> 1U); + + // remove one bit of the hash, leaving more space for the distance info. + // This is extremely fast because we can operate on 8 bytes at once. + ++mInfoHashShift; + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + for (size_t i = 0; i < numElementsWithBuffer; i += 8) + { + auto val = unaligned_load(mInfo + i); + val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); + std::memcpy(mInfo + i, &val, sizeof(val)); + } + // update sentinel, which might have been cleared out! + mInfo[numElementsWithBuffer] = 1; + + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + return true; + } + + void increase_size() + { + // nothing allocated yet? just allocate InitialNumElements + if (0 == mMask) + { + init_data(InitialNumElements); + return; } - // members are sorted so no padding occurs - Node* mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); // 8 byte 8 - uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 16 - size_t mNumElements = 0; // 8 byte 24 - size_t mMask = 0; // 8 byte 32 - size_t mMaxNumElementsAllowed = 0; // 8 byte 40 - InfoType mInfoInc = InitialInfoInc; // 4 byte 44 - InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48 - // 16 byte 56 if NodeAllocator + auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + if (mNumElements < maxNumElementsAllowed && try_increase_info()) + { + return; + } + + ROBIN_HOOD_LOG( + "mNumElements=" << mNumElements << ", maxNumElementsAllowed=" << maxNumElementsAllowed << ", load=" + << (static_cast(mNumElements) * 100.0 / (static_cast(mMask) + 1))) + // it seems we have a really bad hash function! don't try to resize again + if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) + { + throwOverflowError(); + } + + rehashPowerOfTwo((mMask + 1) * 2, false); + } + + void destroy() + { + if (0 == mMask) + { + // don't deallocate! + return; + } + + Destroyer::value>{}.nodesDoNotDeallocate(*this); + + // This protection against not deleting mMask shouldn't be needed as it's sufficiently + // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise + // reports a compile error: attempt to free a non-heap object 'fm' + // [-Werror=free-nonheap-object] + if (mKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) + { + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } + } + + void init() noexcept + { + mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); + mInfo = reinterpret_cast(&mMask); + mNumElements = 0; + mMask = 0; + mMaxNumElementsAllowed = 0; + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // members are sorted so no padding occurs + Node* mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); // 8 byte 8 + uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 16 + size_t mNumElements = 0; // 8 byte 24 + size_t mMask = 0; // 8 byte 32 + size_t mMaxNumElementsAllowed = 0; // 8 byte 40 + InfoType mInfoInc = InitialInfoInc; // 4 byte 44 + InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48 + // 16 byte 56 if NodeAllocator }; -} // namespace detail +} // namespace detail // map -template , - typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> using unordered_flat_map = detail::Table; -template , - typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> using unordered_node_map = detail::Table; -template , - typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> -using unordered_map = - detail::Table) <= sizeof(size_t) * 6 && - std::is_nothrow_move_constructible>::value && - std::is_nothrow_move_assignable>::value, - MaxLoadFactor100, Key, T, Hash, KeyEqual>; +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_map = detail::Table) <= sizeof(size_t) * 6 && + std::is_nothrow_move_constructible>::value && + std::is_nothrow_move_assignable>::value, + MaxLoadFactor100, Key, T, Hash, KeyEqual>; // set @@ -2444,11 +2730,11 @@ using unordered_node_set = detail::Table, typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> -using unordered_set = detail::Table::value && - std::is_nothrow_move_assignable::value, - MaxLoadFactor100, Key, void, Hash, KeyEqual>; +using unordered_set = + detail::Table::value && + std::is_nothrow_move_assignable::value, + MaxLoadFactor100, Key, void, Hash, KeyEqual>; -} // namespace robin_hood +} // namespace robin_hood #endif diff --git a/utils/common/service.h b/utils/common/service.h index 041fd1eda..f3f68b25a 100644 --- a/utils/common/service.h +++ b/utils/common/service.h @@ -21,96 +21,96 @@ #include #include "pipe.h" - class Service { -protected: - // The service name, for logging - const std::string m_name; - // The pipe to send messages from the child to the parent - Pipe m_pipe; + protected: + // The service name, for logging + const std::string m_name; + // The pipe to send messages from the child to the parent + Pipe m_pipe; + static void common_signal_handler_CHLD(int sig) + { + } - static void common_signal_handler_CHLD(int sig) - { } + void InitCommonSignalHandlers() + { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = common_signal_handler_CHLD; + sigaction(SIGCHLD, &sa, NULL); + } - void InitCommonSignalHandlers() + int RunForking() + { + int err; + InitCommonSignalHandlers(); + + if (m_pipe.open() || (err = fork()) < 0) { - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = common_signal_handler_CHLD; - sigaction(SIGCHLD, &sa, NULL); + // Pipe or fork failed + LogErrno(); + return 1; } - int RunForking() + if (err > 0) // Parent + return Parent(); + + return Child(); + } + + virtual int Parent() + { + char str[100]; + // Read the message from the child + ssize_t nbytes = m_pipe.readtm({120, 0}, str, sizeof(str)); + if (nbytes >= 0) { - int err; - InitCommonSignalHandlers(); - - if (m_pipe.open() || (err = fork()) < 0) - { - // Pipe or fork failed - LogErrno(); - return 1; - } - - if (err > 0) // Parent - return Parent(); - - return Child(); + ParentLogChildMessage(std::string(str, nbytes)); } - - virtual int Parent() + else { - char str[100]; - // Read the message from the child - ssize_t nbytes= m_pipe.readtm({120,0}, str, sizeof(str)); - if (nbytes >= 0) - { - ParentLogChildMessage(std::string(str, nbytes)); - } - else - { - // read() failed - LogErrno(); - return 1; - } - return 0; + // read() failed + LogErrno(); + return 1; } - + return 0; + } -public: - Service(const std::string &name) - :m_name(name) - { } - virtual ~Service() { } + public: + Service(const std::string& name) : m_name(name) + { + } + virtual ~Service() + { + } - void NotifyServiceStarted() + void NotifyServiceStarted() + { + if (m_pipe.is_open_for_write()) { - if (m_pipe.is_open_for_write()) - { - std::ostringstream str; - str << m_name << " main process has started"; - m_pipe.write(str.str()); - } + std::ostringstream str; + str << m_name << " main process has started"; + m_pipe.write(str.str()); } + } - void NotifyServiceInitializationFailed() + void NotifyServiceInitializationFailed() + { + if (m_pipe.is_open_for_write()) { - if (m_pipe.is_open_for_write()) - { - std::ostringstream str; - str << m_name << " main process initialization failed"; - m_pipe.write(str.str()); - } + std::ostringstream str; + str << m_name << " main process initialization failed"; + m_pipe.write(str.str()); } + } - // Used by both Parent and Child to log errors - virtual void LogErrno()= 0; - // Used by Parent to log an initialization notification message from child - virtual void ParentLogChildMessage(const std::string &str)= 0; - // The main service process job - virtual int Child()= 0; + // Used by both Parent and Child to log errors + virtual void LogErrno() = 0; + // Used by Parent to log an initialization notification message from child + virtual void ParentLogChildMessage(const std::string& str) = 0; + // The main service process job + virtual int Child() = 0; }; -#endif // SERVICE_H_INCLUDED +#endif // SERVICE_H_INCLUDED diff --git a/utils/common/simd_sse.h b/utils/common/simd_sse.h index f7411dc4f..9815029c7 100644 --- a/utils/common/simd_sse.h +++ b/utils/common/simd_sse.h @@ -23,880 +23,876 @@ #include #ifdef __OPTIMIZE__ - #include - #include - #define MCS_FORCE_INLINE __attribute__((__always_inline__)) +#include +#include +#define MCS_FORCE_INLINE __attribute__((__always_inline__)) #else - #define __OPTIMIZE__ - #include - #include - #undef __OPTIMIZE__ - #define MCS_FORCE_INLINE inline +#define __OPTIMIZE__ +#include +#include +#undef __OPTIMIZE__ +#define MCS_FORCE_INLINE inline #endif #include // Column filtering is dispatched 4-way based on the column type, // which defines implementation of comparison operations for the column values -enum ENUM_KIND {KIND_DEFAULT, // compared as signed integers - KIND_UNSIGNED, // compared as unsigned integers - KIND_FLOAT, // compared as floating-point numbers - KIND_TEXT}; // whitespace-trimmed and then compared as signed integers +enum ENUM_KIND +{ + KIND_DEFAULT, // compared as signed integers + KIND_UNSIGNED, // compared as unsigned integers + KIND_FLOAT, // compared as floating-point numbers + KIND_TEXT +}; // whitespace-trimmed and then compared as signed integers namespace simd { - using vi128_t = __m128i; - using vi128f_t = __m128; - using vi128d_t = __m128d; - using int128_t = __int128; - using MT = uint16_t; - // These ugly wrappers are used to allow to use __m128* as template class parameter argument - struct vi128_wr - { - __m128i v; - }; +using vi128_t = __m128i; +using vi128f_t = __m128; +using vi128d_t = __m128d; +using int128_t = __int128; +using MT = uint16_t; +// These ugly wrappers are used to allow to use __m128* as template class parameter argument +struct vi128_wr +{ + __m128i v; +}; - struct vi128f_wr - { - __m128 v; - }; +struct vi128f_wr +{ + __m128 v; +}; - struct vi128d_wr - { - __m128d v; - }; +struct vi128d_wr +{ + __m128d v; +}; - template - struct IntegralToSIMD; +template +struct IntegralToSIMD; - template - struct IntegralToSIMD::type> - { - using type = vi128d_wr; - }; +template +struct IntegralToSIMD::type> +{ + using type = vi128d_wr; +}; - template - struct IntegralToSIMD::type> - { - using type = vi128f_wr; - }; +template +struct IntegralToSIMD::type> +{ + using type = vi128f_wr; +}; - template - struct IntegralToSIMD::type> - { - using type = vi128_wr; - }; +template +struct IntegralToSIMD::type> +{ + using type = vi128_wr; +}; - template - struct StorageToFiltering; +template +struct StorageToFiltering; - template - struct StorageToFiltering::type> - { - using type = double; - }; +template +struct StorageToFiltering::type> +{ + using type = double; +}; - template - struct StorageToFiltering::type> - { - using type = float; - }; +template +struct StorageToFiltering::type> +{ + using type = float; +}; - template - struct StorageToFiltering::type> - { - using type = T; - }; +template +struct StorageToFiltering::type> +{ + using type = T; +}; - template - class SimdFilterProcessor; +template +class SimdFilterProcessor; - // Dummy class that captures all impossible cases, e.g. integer vector as VT and flot as CHECK_T. - template - class SimdFilterProcessor +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if<(std::is_same::value && sizeof(CHECK_T) == 16) || - (std::is_same::value && !std::is_same::value && !std::is_same::value)>::type> + (std::is_same::value && !std::is_same::value && + !std::is_same::value)>::type> +{ + // This is a dummy class that is not currently used. + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) { - // This is a dummy class that is not currently used. - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using T = typename datatypes::WidthToSIntegralType::type; - using SimdWrapperType = vi128_wr; - using SimdType = vi128_t; - using FilterType = T; - using StorageType = T; - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - return loadValue(fill); - } + return loadValue(fill); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_loadu_si128(reinterpret_cast(&fill)); - } + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_loadu_si128(reinterpret_cast(&fill)); + } - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_si128(reinterpret_cast(from)); - } + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } - MCS_FORCE_INLINE MT cmpDummy(SimdType& x, SimdType& y) - { - return 0xFFFF; - } - // Compare - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT cmpDummy(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_epi8(vmask); - } + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - return cmpDummy(x, y); - } + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpDummy(x, y); + } - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_si128(); - } + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } - // store - MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) - { - _mm_maskmoveu_si128(x, vmask, dst); - } + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_si128(reinterpret_cast(dst), x); - } - }; + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; - template - class SimdFilterProcessor +class SimdFilterProcessor< + VT, T, typename std::enable_if::value && std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using FilterType = T; + using NullEmptySimdType = vi128_t; + using SimdWrapperType = simd::vi128d_wr; + using SimdType = simd::vi128d_t; + using StorageSimdType = simd::vi128_t; + using StorageType = typename datatypes::WidthToSIntegralType::type; + using StorageVecProcType = SimdFilterProcessor; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) { - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using FilterType = T; - using NullEmptySimdType = vi128_t; - using SimdWrapperType = simd::vi128d_wr; - using SimdType = simd::vi128d_t; - using StorageSimdType = simd::vi128_t; - using StorageType = typename datatypes::WidthToSIntegralType::type; - using StorageVecProcType = SimdFilterProcessor; - // Mask calculation for int and float types differs. - // See corresponding intrinsics algos for details. - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - StorageVecProcType nullEmptyProcessor; - // This spec borrows the expr from u-/int64 based proceesor class. - return (SimdType) nullEmptyProcessor.loadValue(fill); - } + StorageVecProcType nullEmptyProcessor; + // This spec borrows the expr from u-/int64 based proceesor class. + return (SimdType)nullEmptyProcessor.loadValue(fill); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_set1_pd(fill); - } - - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_pd(reinterpret_cast(from)); - } - - // Compare - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpeq_pd(x, y)); - } - - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpge_pd(x,y)); - } - - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpgt_pd(x, y)); - } - - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmple_pd(x, y)); - } - - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmplt_pd(x, y)); - } - - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpneq_pd(x, y)); - } - - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } - - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } - - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_pd(vmask); - } - - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - StorageVecProcType nullEmptyProcessor; - NullEmptySimdType* xAsIntVecPtr = reinterpret_cast(&x); - NullEmptySimdType* yAsIntVecPtr = reinterpret_cast(&y); - // This spec borrows the expr from u-/int64 based proceesor class. - return nullEmptyProcessor.cmpNe(*xAsIntVecPtr, *yAsIntVecPtr); - } - - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - StorageVecProcType nullEmptyProcessor; - - NullEmptySimdType* xAsIntVecPtr - = reinterpret_cast(&x); - NullEmptySimdType* yAsIntVecPtr - = reinterpret_cast(&y); - // This spec borrows the expr from u-/int64 based proceesor class. - return nullEmptyProcessor.cmpEq(*xAsIntVecPtr, *yAsIntVecPtr); - } - - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_pd(); - } - - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_pd(reinterpret_cast(dst), x); - } - }; - - template - class SimdFilterProcessor::value && std::is_same::value>::type> + MCS_FORCE_INLINE SimdType loadValue(const T fill) { - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using FilterType = T; - using NullEmptySimdType = vi128_t; - using SimdWrapperType = vi128f_wr; - using SimdType = vi128f_t; - using StorageSimdType = simd::vi128_t; - using StorageType = typename datatypes::WidthToSIntegralType::type; - using StorageVecProcType = SimdFilterProcessor; - // Mask calculation for int and float types differs. - // See corresponding intrinsics algos for details. - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - StorageVecProcType nullEmptyProcessor; - // This spec borrows the expr from u-/int64 based proceesor class. - return (SimdType) nullEmptyProcessor.loadValue(fill); - } + return _mm_set1_pd(fill); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_set1_ps(fill); - } - - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_ps(reinterpret_cast(from)); - } - - // Compare - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpeq_ps(x, y)); - } - - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType) _mm_cmpge_ps(x,y)); - } - - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpgt_ps(x, y)); - } - - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmple_ps(x, y)); - } - - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmplt_ps(x, y)); - } - - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8((StorageSimdType)_mm_cmpneq_ps(x, y)); - } - - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } - - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } - - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_ps(vmask); - } - - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - StorageVecProcType nullEmptyProcessor; - - NullEmptySimdType* xAsIntVecPtr - = reinterpret_cast(&x); - NullEmptySimdType* yAsIntVecPtr - = reinterpret_cast(&y); - // This spec borrows the expr from u-/int64 based proceesor class. - return nullEmptyProcessor.cmpNe(*xAsIntVecPtr, *yAsIntVecPtr); - } - - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - StorageVecProcType nullEmptyProcessor; - - NullEmptySimdType* xAsIntVecPtr - = reinterpret_cast(&x); - NullEmptySimdType* yAsIntVecPtr - = reinterpret_cast(&y); - // This spec borrows the expr from u-/int64 based proceesor class. - return nullEmptyProcessor.cmpEq(*xAsIntVecPtr, *yAsIntVecPtr); - } - - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_ps(); - } - - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_ps(reinterpret_cast(dst), x); - } - }; - - template - class SimdFilterProcessor::value && - sizeof(CHECK_T) == 8 && !std::is_same::value>::type> + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) { - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using T = typename datatypes::WidthToSIntegralType::type; - using SimdWrapperType = vi128_wr; - using SimdType = vi128_t; - using FilterType = T; - using StorageType = T; - // Mask calculation for int and float types differs. - // See corresponding intrinsics algos for details. - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - return loadValue(fill); - } + return _mm_loadu_pd(reinterpret_cast(from)); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_set_epi64x(fill, fill); - } - - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_si128(reinterpret_cast(from)); - } - - // Compare - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_or_si128(_mm_cmpgt_epi64(x, y),_mm_cmpeq_epi64(x, y))); - } - - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpgt_epi64(x, y)); - } - - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)); - } - - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return cmpGt(x, y) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return cmpNe(x, y) ^ cmpGt(x, y); - } - - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } - - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } - - - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_epi8(vmask); - } - - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_si128(); - } - - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - return cmpNe(x, y); - } - - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - return cmpEq(x, y); - } - - // store - MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) - { - _mm_maskmoveu_si128(x, vmask, dst); - } - - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_si128(reinterpret_cast(dst), x); - } - }; - - template - class SimdFilterProcessor::value && - sizeof(CHECK_T) == 4 && !std::is_same::value>::type> + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) { - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using T = typename datatypes::WidthToSIntegralType::type; - using SimdWrapperType = vi128_wr; - using SimdType = vi128_t; - using FilterType = T; - using StorageType = T; - // Mask calculation for int and float types differs. - // See corresponding intrinsics algos for details. - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - return loadValue(fill); - } + return _mm_movemask_epi8((StorageSimdType)_mm_cmpeq_pd(x, y)); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_set1_epi32(fill); - } - - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_si128(reinterpret_cast(from)); - } - - // Compare - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)); - } - - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return cmpLt(x, y) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpgt_epi32(x, y)); - } - - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return cmpGt(x, y) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmplt_epi32(x, y)); - } - - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } - - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } - - - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_epi8(vmask); - } - - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - return cmpNe(x, y); - } - - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - return cmpEq(x, y); - } - - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_si128(); - } - - // store - MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) - { - _mm_maskmoveu_si128(x, vmask, dst); - } - - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_si128(reinterpret_cast(dst), x); - } - }; - - template - class SimdFilterProcessor::value && sizeof(CHECK_T) == 2>::type> + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) { - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using T = typename datatypes::WidthToSIntegralType::type; - using SimdWrapperType = simd::vi128_wr; - using SimdType = simd::vi128_t; - using FilterType = T; - using StorageType = T; - // Mask calculation for int and float types differs. - // See corresponding intrinsics algos for details. - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - return loadValue(fill); - } + return _mm_movemask_epi8((StorageSimdType)_mm_cmpge_pd(x, y)); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_set1_epi16(fill); - } - - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_si128(reinterpret_cast(from)); - } - - // Compare - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)); - } - - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return cmpLt(x, y) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpgt_epi16(x, y)); - } - - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return cmpGt(x, y) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmplt_epi16(x, y)); - } - - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF; - } - - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } - - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } - - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_epi8(vmask); - } - - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - return cmpNe(x, y); - } - - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - return cmpEq(x, y); - } - - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_si128(); - } - - // store - MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) - { - _mm_maskmoveu_si128(x, vmask, dst); - } - - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_si128(reinterpret_cast(dst), x); - } - }; - - template - class SimdFilterProcessor::value && sizeof(CHECK_T) == 1>::type> + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) { - public: - constexpr static const uint16_t vecByteSize = 16U; - constexpr static const uint16_t vecBitSize = 128U; - using T = typename datatypes::WidthToSIntegralType::type; - using SimdWrapperType = vi128_wr; - using SimdType = vi128_t; - using FilterType = T; - using StorageType = T; - // Mask calculation for int and float types differs. - // See corresponding intrinsics algos for details. - constexpr static const uint16_t FilterMaskStep = sizeof(T); - // Load value - MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) - { - return loadValue(fill); - } + return _mm_movemask_epi8((StorageSimdType)_mm_cmpgt_pd(x, y)); + } - MCS_FORCE_INLINE SimdType loadValue(const T fill) - { - return _mm_set1_epi8(fill); - } + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmple_pd(x, y)); + } - // Load from - MCS_FORCE_INLINE SimdType loadFrom(const char* from) - { - return _mm_loadu_si128(reinterpret_cast(from)); - } + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmplt_pd(x, y)); + } - // Compare - MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)); - } + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmpneq_pd(x, y)); + } - MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) - { - return cmpLt(x, y) ^ 0xFFFF; - } + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } - MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y)); - } + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } - MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) - { - return cmpGt(x, y) ^ 0xFFFF; - } + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_pd(vmask); + } - MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmplt_epi8(x, y)); - } + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + StorageVecProcType nullEmptyProcessor; + NullEmptySimdType* xAsIntVecPtr = reinterpret_cast(&x); + NullEmptySimdType* yAsIntVecPtr = reinterpret_cast(&y); + // This spec borrows the expr from u-/int64 based proceesor class. + return nullEmptyProcessor.cmpNe(*xAsIntVecPtr, *yAsIntVecPtr); + } - MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) - { - return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF; - } + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + StorageVecProcType nullEmptyProcessor; - MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) - { - return 0; - } + NullEmptySimdType* xAsIntVecPtr = reinterpret_cast(&x); + NullEmptySimdType* yAsIntVecPtr = reinterpret_cast(&y); + // This spec borrows the expr from u-/int64 based proceesor class. + return nullEmptyProcessor.cmpEq(*xAsIntVecPtr, *yAsIntVecPtr); + } - MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) - { - return 0xFFFF; - } + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_pd(); + } - // permute -/* TODO Available in AVX-512 - MCS_FORCE_INLINE SimdType perm8Bits(SimdType& x, SimdType& idx) - { - return _mm_permutexvar_epi8(x, idx); - } -*/ - // misc - MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) - { - return _mm_movemask_epi8(vmask); - } + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_pd(reinterpret_cast(dst), x); + } +}; - MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) - { - return cmpNe(x, y); - } +template +class SimdFilterProcessor< + VT, T, typename std::enable_if::value && std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using FilterType = T; + using NullEmptySimdType = vi128_t; + using SimdWrapperType = vi128f_wr; + using SimdType = vi128f_t; + using StorageSimdType = simd::vi128_t; + using StorageType = typename datatypes::WidthToSIntegralType::type; + using StorageVecProcType = SimdFilterProcessor; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + StorageVecProcType nullEmptyProcessor; + // This spec borrows the expr from u-/int64 based proceesor class. + return (SimdType)nullEmptyProcessor.loadValue(fill); + } - MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) - { - return cmpEq(x, y); - } + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_ps(fill); + } - MCS_FORCE_INLINE SimdType setToZero() - { - return _mm_setzero_si128(); - } + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_ps(reinterpret_cast(from)); + } - // store - MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) - { - _mm_maskmoveu_si128(x, vmask, dst); - } + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmpeq_ps(x, y)); + } - MCS_FORCE_INLINE void store(char* dst, SimdType& x) - { - _mm_storeu_si128(reinterpret_cast(dst), x); - } - }; + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmpge_ps(x, y)); + } -} // end of simd + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmpgt_ps(x, y)); + } -#endif // if defined(__x86_64__ ) + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmple_ps(x, y)); + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmplt_ps(x, y)); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8((StorageSimdType)_mm_cmpneq_ps(x, y)); + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_ps(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + StorageVecProcType nullEmptyProcessor; + + NullEmptySimdType* xAsIntVecPtr = reinterpret_cast(&x); + NullEmptySimdType* yAsIntVecPtr = reinterpret_cast(&y); + // This spec borrows the expr from u-/int64 based proceesor class. + return nullEmptyProcessor.cmpNe(*xAsIntVecPtr, *yAsIntVecPtr); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + StorageVecProcType nullEmptyProcessor; + + NullEmptySimdType* xAsIntVecPtr = reinterpret_cast(&x); + NullEmptySimdType* yAsIntVecPtr = reinterpret_cast(&y); + // This spec borrows the expr from u-/int64 based proceesor class. + return nullEmptyProcessor.cmpEq(*xAsIntVecPtr, *yAsIntVecPtr); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_ps(); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_ps(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor::value && sizeof(CHECK_T) == 8 && + !std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set_epi64x(fill, fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_or_si128(_mm_cmpgt_epi64(x, y), _mm_cmpeq_epi64(x, y))); + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpgt_epi64(x, y)); + } + + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpNe(x, y) ^ cmpGt(x, y); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor::value && sizeof(CHECK_T) == 4 && + !std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi32(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpLt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpgt_epi32(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmplt_epi32(x, y)); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if::value && sizeof(CHECK_T) == 2>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = simd::vi128_wr; + using SimdType = simd::vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi16(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpLt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpgt_epi16(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmplt_epi16(x, y)); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if::value && sizeof(CHECK_T) == 1>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi8(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpLt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmplt_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // permute + /* TODO Available in AVX-512 + MCS_FORCE_INLINE SimdType perm8Bits(SimdType& x, SimdType& idx) + { + return _mm_permutexvar_epi8(x, idx); + } + */ + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +} // namespace simd + +#endif // if defined(__x86_64__ ) // vim:ts=2 sw=2: diff --git a/utils/common/simpleallocator.h b/utils/common/simpleallocator.h index 987c57387..851f3cf34 100644 --- a/utils/common/simpleallocator.h +++ b/utils/common/simpleallocator.h @@ -38,11 +38,11 @@ namespace utils { - // A specialized allocator for std::tr1::unordered_multimap based joiner // or std::tr1::unordered_map based aggregation. // User shall initialize a pool and pass it to allocator, release the pool when map is done. -template class SimpleAllocator; +template +class SimpleAllocator; // this pool is best for node size of 3*sizeof(int64). // map nodes are taken from fixed size blocks, and control hash tables are from ::new. @@ -52,209 +52,213 @@ template class SimpleAllocator; #define OPT_NODE_UNITS 10 class SimplePool { -public: - SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0) {} - ~SimplePool() - { - reset(); - } + public: + SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0) + { + } + ~SimplePool() + { + reset(); + } - inline void* allocate(size_t n, const void* = 0); - inline void deallocate(void* p, size_t n); - inline size_t max_size() const throw(); - inline uint64_t getMemUsage() const; + inline void* allocate(size_t n, const void* = 0); + inline void deallocate(void* p, size_t n); + inline size_t max_size() const throw(); + inline uint64_t getMemUsage() const; -private: - static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240; + private: + static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240; - inline void reset(); - inline void allocateNewChunk(); + inline void reset(); + inline void allocateNewChunk(); - // MemUnit stores a pointer to next unit before allocated, and T after allocated. - union MemUnit - { - MemUnit* fNext; - uint64_t fData; - }* fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit + // MemUnit stores a pointer to next unit before allocated, and T after allocated. + union MemUnit + { + MemUnit* fNext; + uint64_t fData; + } * fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit - std::list fBlockList; - uint64_t fTableMemSize; + std::list fBlockList; + uint64_t fTableMemSize; - static const size_t fUnitSize = sizeof(MemUnit); - static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS; - static const size_t fChunkSize = fUnitSize * fUnitPerChunk; + static const size_t fUnitSize = sizeof(MemUnit); + static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS; + static const size_t fChunkSize = fUnitSize * fUnitPerChunk; }; - -template +template class SimpleAllocator { -public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T* pointer; - typedef const T* const_pointer; - typedef T& reference; - typedef const T& const_reference; - typedef T value_type; - template struct rebind - { - typedef SimpleAllocator other; - }; + public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + typedef T value_type; + template + struct rebind + { + typedef SimpleAllocator other; + }; - SimpleAllocator() throw() {} - SimpleAllocator(std::shared_ptr pool) throw() - { - fPool = pool; - } - SimpleAllocator(const SimpleAllocator& alloc) - { - fPool = alloc.fPool; - } - template SimpleAllocator(const SimpleAllocator& alloc) - { - fPool = alloc.fPool; - } + SimpleAllocator() throw() + { + } + SimpleAllocator(std::shared_ptr pool) throw() + { + fPool = pool; + } + SimpleAllocator(const SimpleAllocator& alloc) + { + fPool = alloc.fPool; + } + template + SimpleAllocator(const SimpleAllocator& alloc) + { + fPool = alloc.fPool; + } - ~SimpleAllocator() throw() { } + ~SimpleAllocator() throw() + { + } - inline pointer address(reference x) const - { - return &x; - } - inline const_pointer address(const_reference x) const - { - return &x; - } + inline pointer address(reference x) const + { + return &x; + } + inline const_pointer address(const_reference x) const + { + return &x; + } - inline pointer allocate(size_type n, const void* = 0) - { - return static_cast(fPool->allocate(n * sizeof(T))); - } - inline void deallocate(pointer p, size_type n) - { - fPool->deallocate(p, n * sizeof(T)); - } + inline pointer allocate(size_type n, const void* = 0) + { + return static_cast(fPool->allocate(n * sizeof(T))); + } + inline void deallocate(pointer p, size_type n) + { + fPool->deallocate(p, n * sizeof(T)); + } #ifdef _MSC_VER - //The MSVC STL library really needs this to return a big number... - size_type max_size() const throw() - { - return std::numeric_limits::max(); - } + // The MSVC STL library really needs this to return a big number... + size_type max_size() const throw() + { + return std::numeric_limits::max(); + } #else - inline size_type max_size() const throw() - { - return fPool->max_size() / sizeof(T); - } + inline size_type max_size() const throw() + { + return fPool->max_size() / sizeof(T); + } #endif - inline void construct(pointer ptr, const T& val) - { - new ((void*)ptr) T(val); - } - inline void destroy(pointer ptr) - { - ptr->T::~T(); - } + inline void construct(pointer ptr, const T& val) + { + new ((void*)ptr) T(val); + } + inline void destroy(pointer ptr) + { + ptr->T::~T(); + } - inline void setPool(SimplePool* pool) - { - fPool.reset(pool); - } + inline void setPool(SimplePool* pool) + { + fPool.reset(pool); + } - std::shared_ptr fPool; + std::shared_ptr fPool; }; - // inlines inline void* SimplePool::allocate(size_t n, const void* dur) { - // make sure the block allocated is on unit boundary - size_t unitCount = n / fUnitSize; + // make sure the block allocated is on unit boundary + size_t unitCount = n / fUnitSize; - if ((n % fUnitSize) != 0) - unitCount += 1; + if ((n % fUnitSize) != 0) + unitCount += 1; - // if for control table, let new allocator handle it. - if (unitCount > OPT_NODE_UNITS) + // if for control table, let new allocator handle it. + if (unitCount > OPT_NODE_UNITS) + { + fTableMemSize += n; + return new uint8_t[n]; + } + + // allocate node + MemUnit* curr = fNext; + + do + { + if (curr == NULL) { - fTableMemSize += n; - return new uint8_t[n]; + allocateNewChunk(); + curr = fNext; } - // allocate node - MemUnit* curr = fNext; + fNext = curr + unitCount; - do - { - if (curr == NULL) - { - allocateNewChunk(); - curr = fNext; - } + if (fNext > fEnd) + curr = NULL; + } while (!curr); - fNext = curr + unitCount; - - if (fNext > fEnd) - curr = NULL; - } - while (!curr); - - return curr; + return curr; } inline void SimplePool::deallocate(void* p, size_t n) { - // only delete the old control table, which is allocated by new allocator. - if (n > fMaxNodeSize) - { - fTableMemSize -= n; - delete [] (static_cast(p)); - } + // only delete the old control table, which is allocated by new allocator. + if (n > fMaxNodeSize) + { + fTableMemSize -= n; + delete[](static_cast(p)); + } } inline size_t SimplePool::max_size() const throw() { - return fUnitSize * fUnitPerChunk; + return fUnitSize * fUnitPerChunk; } inline uint64_t SimplePool::getMemUsage() const { - return fTableMemSize + fBlockList.size() * fChunkSize + - // add list overhead, element type is a pointer, and - // lists store a next pointer. - fBlockList.size() * 2 * sizeof(void*); + return fTableMemSize + fBlockList.size() * fChunkSize + + // add list overhead, element type is a pointer, and + // lists store a next pointer. + fBlockList.size() * 2 * sizeof(void*); } inline void SimplePool::reset() { - for (std::list::iterator i = fBlockList.begin(); i != fBlockList.end(); i++) - delete [] (*i); + for (std::list::iterator i = fBlockList.begin(); i != fBlockList.end(); i++) + delete[](*i); - fNext = NULL; - fEnd = NULL; + fNext = NULL; + fEnd = NULL; } inline void SimplePool::allocateNewChunk() { - MemUnit* chunk = new MemUnit[fUnitPerChunk]; - fBlockList.push_back(chunk); - fNext = chunk; - fEnd = chunk + fUnitPerChunk; + MemUnit* chunk = new MemUnit[fUnitPerChunk]; + fBlockList.push_back(chunk); + fNext = chunk; + fEnd = chunk + fUnitPerChunk; } template inline bool operator==(const SimpleAllocator&, const SimpleAllocator&) { - return true; + return true; } template inline bool operator!=(const SimpleAllocator&, const SimpleAllocator&) { - return false; -} + return false; } +} // namespace utils #endif // UTILS_SIMPLEALLOCATOR_H - diff --git a/utils/common/spinlock.h b/utils/common/spinlock.h index 1dc84391f..2549230c7 100644 --- a/utils/common/spinlock.h +++ b/utils/common/spinlock.h @@ -4,24 +4,23 @@ namespace utils { - -inline void getSpinlock(std::atomic &lock) +inline void getSpinlock(std::atomic& lock) { - bool _false = false; - while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) - _false = false; + bool _false = false; + while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) + _false = false; } -inline bool trySpinlock(std::atomic &lock) +inline bool trySpinlock(std::atomic& lock) { - bool _false = false; - bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire); - return ret; + bool _false = false; + bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire); + return ret; } -inline void releaseSpinlock(std::atomic &lock) +inline void releaseSpinlock(std::atomic& lock) { - lock.store(false, std::memory_order_release); + lock.store(false, std::memory_order_release); } -} +} // namespace utils diff --git a/utils/common/statistics.cpp b/utils/common/statistics.cpp index 7a4e9b867..229ca9556 100644 --- a/utils/common/statistics.cpp +++ b/utils/common/statistics.cpp @@ -35,405 +35,407 @@ using ColumnsCache = std::vector>; StatisticsManager* StatisticsManager::instance() { - static StatisticsManager* sm = new StatisticsManager(); - return sm; + static StatisticsManager* sm = new StatisticsManager(); + return sm; } void StatisticsManager::analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace) { - std::lock_guard lock(mut); - auto rowCount = rowGroup.getRowCount(); - const auto columnCount = rowGroup.getColumnCount(); - if (!rowCount || !columnCount) - return; + std::lock_guard lock(mut); + auto rowCount = rowGroup.getRowCount(); + const auto columnCount = rowGroup.getColumnCount(); + if (!rowCount || !columnCount) + return; - auto& oids = rowGroup.getOIDs(); + auto& oids = rowGroup.getOIDs(); - rowgroup::Row r; - rowGroup.initRow(&r); - rowGroup.getRow(0, &r); + rowgroup::Row r; + rowGroup.initRow(&r); + rowGroup.getRow(0, &r); - ColumnsCache columns(columnCount, std::unordered_set()); - // Init key types. - for (uint32_t index = 0; index < columnCount; ++index) - keyTypes[oids[index]] = KeyType::PK; + ColumnsCache columns(columnCount, std::unordered_set()); + // Init key types. + for (uint32_t index = 0; index < columnCount; ++index) + keyTypes[oids[index]] = KeyType::PK; - const uint32_t maxRowCount = 4096; - // TODO: We should read just couple of blocks from columns, not all data, but this requires - // more deep refactoring of column commands. - rowCount = std::min(rowCount, maxRowCount); - // This is strange, it's a CS but I'm processing data as row by row, how to fix it? - for (uint32_t i = 0; i < rowCount; ++i) + const uint32_t maxRowCount = 4096; + // TODO: We should read just couple of blocks from columns, not all data, but this requires + // more deep refactoring of column commands. + rowCount = std::min(rowCount, maxRowCount); + // This is strange, it's a CS but I'm processing data as row by row, how to fix it? + for (uint32_t i = 0; i < rowCount; ++i) + { + for (uint32_t j = 0; j < columnCount; ++j) { - for (uint32_t j = 0; j < columnCount; ++j) - { - if (r.isNullValue(j) || columns[j].count(r.getIntField(j))) - keyTypes[oids[j]] = KeyType::FK; - else - columns[j].insert(r.getIntField(j)); - } - r.nextRow(); + if (r.isNullValue(j) || columns[j].count(r.getIntField(j))) + keyTypes[oids[j]] = KeyType::FK; + else + columns[j].insert(r.getIntField(j)); } + r.nextRow(); + } - if (trace) - output(StatisticsType::PK_FK); + if (trace) + output(StatisticsType::PK_FK); } void StatisticsManager::output(StatisticsType statisticsType) { - if (statisticsType == StatisticsType::PK_FK) - { - std::cout << "Columns count: " << keyTypes.size() << std::endl; - for (const auto& p : keyTypes) - std::cout << p.first << " " << (int) p.second << std::endl; - } + if (statisticsType == StatisticsType::PK_FK) + { + std::cout << "Columns count: " << keyTypes.size() << std::endl; + for (const auto& p : keyTypes) + std::cout << p.first << " " << (int)p.second << std::endl; + } } // Someday it will be a virtual method, based on statistics type we processing. std::unique_ptr StatisticsManager::convertStatsToDataStream(uint64_t& dataStreamSize) { - // Number of pairs. - uint64_t count = keyTypes.size(); - // count, [[uid, keyType], ... ] - dataStreamSize = sizeof(uint64_t) + count * (sizeof(uint32_t) + sizeof(KeyType)); + // Number of pairs. + uint64_t count = keyTypes.size(); + // count, [[uid, keyType], ... ] + dataStreamSize = sizeof(uint64_t) + count * (sizeof(uint32_t) + sizeof(KeyType)); - // Allocate memory for data stream. - std::unique_ptr dataStreamSmartPtr(new char[dataStreamSize]); - auto* dataStream = dataStreamSmartPtr.get(); - // Initialize the data stream. - uint64_t offset = 0; - std::memcpy(dataStream, reinterpret_cast(&count), sizeof(uint64_t)); - offset += sizeof(uint64_t); + // Allocate memory for data stream. + std::unique_ptr dataStreamSmartPtr(new char[dataStreamSize]); + auto* dataStream = dataStreamSmartPtr.get(); + // Initialize the data stream. + uint64_t offset = 0; + std::memcpy(dataStream, reinterpret_cast(&count), sizeof(uint64_t)); + offset += sizeof(uint64_t); - // For each pair [oid, key type]. - for (const auto& p : keyTypes) - { - uint32_t oid = p.first; - std::memcpy(&dataStream[offset], reinterpret_cast(&oid), sizeof(uint32_t)); - offset += sizeof(uint32_t); + // For each pair [oid, key type]. + for (const auto& p : keyTypes) + { + uint32_t oid = p.first; + std::memcpy(&dataStream[offset], reinterpret_cast(&oid), sizeof(uint32_t)); + offset += sizeof(uint32_t); - KeyType keyType = p.second; - std::memcpy(&dataStream[offset], reinterpret_cast(&keyType), sizeof(KeyType)); - offset += sizeof(KeyType); - } + KeyType keyType = p.second; + std::memcpy(&dataStream[offset], reinterpret_cast(&keyType), sizeof(KeyType)); + offset += sizeof(KeyType); + } - return dataStreamSmartPtr; + return dataStreamSmartPtr; } void StatisticsManager::saveToFile() { - std::lock_guard lock(mut); + std::lock_guard lock(mut); - const char* fileName = statsFile.c_str(); - std::unique_ptr out( - IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "wb", 1)); + const char* fileName = statsFile.c_str(); + std::unique_ptr out( + IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "wb", 1)); - if (!out) - { - BRM::log_errno("StatisticsManager::saveToFile(): open"); - throw ios_base::failure("StatisticsManager::saveToFile(): open failed."); - } + if (!out) + { + BRM::log_errno("StatisticsManager::saveToFile(): open"); + throw ios_base::failure("StatisticsManager::saveToFile(): open failed."); + } - // Compute hash. - uint64_t dataStreamSize = 0; - std::unique_ptr dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize); - utils::Hasher128 hasher; + // Compute hash. + uint64_t dataStreamSize = 0; + std::unique_ptr dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize); + utils::Hasher128 hasher; - // Prepare a statistics file header. - const uint32_t headerSize = sizeof(StatisticsFileHeader); - StatisticsFileHeader fileHeader; - std::memset(&fileHeader, 0, headerSize); - fileHeader.version = version; - fileHeader.epoch = epoch; - fileHeader.dataSize = dataStreamSize; - // Compute hash from the data. - fileHeader.dataHash = hasher(dataStreamSmartPtr.get(), dataStreamSize); + // Prepare a statistics file header. + const uint32_t headerSize = sizeof(StatisticsFileHeader); + StatisticsFileHeader fileHeader; + std::memset(&fileHeader, 0, headerSize); + fileHeader.version = version; + fileHeader.epoch = epoch; + fileHeader.dataSize = dataStreamSize; + // Compute hash from the data. + fileHeader.dataHash = hasher(dataStreamSmartPtr.get(), dataStreamSize); - // Write statistics file header. - uint64_t size = out->write(reinterpret_cast(&fileHeader), headerSize); - if (size != headerSize) - { - auto rc = IDBPolicy::remove(fileName); - if (rc == -1) - std::cerr << "Cannot remove file " << fileName << std::endl; + // Write statistics file header. + uint64_t size = out->write(reinterpret_cast(&fileHeader), headerSize); + if (size != headerSize) + { + auto rc = IDBPolicy::remove(fileName); + if (rc == -1) + std::cerr << "Cannot remove file " << fileName << std::endl; - throw ios_base::failure("StatisticsManager::saveToFile(): write failed. "); - } + throw ios_base::failure("StatisticsManager::saveToFile(): write failed. "); + } - // Write data. - size = out->write(dataStreamSmartPtr.get(), dataStreamSize); - if (size != dataStreamSize) - { - auto rc = IDBPolicy::remove(fileName); - if (rc == -1) - std::cerr << "Cannot remove file " << fileName << std::endl; + // Write data. + size = out->write(dataStreamSmartPtr.get(), dataStreamSize); + if (size != dataStreamSize) + { + auto rc = IDBPolicy::remove(fileName); + if (rc == -1) + std::cerr << "Cannot remove file " << fileName << std::endl; - throw ios_base::failure("StatisticsManager::saveToFile(): write failed. "); - } + throw ios_base::failure("StatisticsManager::saveToFile(): write failed. "); + } } void StatisticsManager::loadFromFile() { - std::lock_guard lock(mut); - // Check that stats file does exist. - if (!boost::filesystem::exists(statsFile)) - return; + std::lock_guard lock(mut); + // Check that stats file does exist. + if (!boost::filesystem::exists(statsFile)) + return; - const char* fileName = statsFile.c_str(); - std::unique_ptr in( - IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1)); + const char* fileName = statsFile.c_str(); + std::unique_ptr in( + IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1)); - if (!in) - { - BRM::log_errno("StatisticsManager::loadFromFile(): open"); - throw ios_base::failure( - "StatisticsManager::loadFromFile(): open failed. Check the error log."); - } + if (!in) + { + BRM::log_errno("StatisticsManager::loadFromFile(): open"); + throw ios_base::failure("StatisticsManager::loadFromFile(): open failed. Check the error log."); + } - // Read the file header. - StatisticsFileHeader fileHeader; - const uint32_t headerSize = sizeof(StatisticsFileHeader); - int64_t size = in->read(reinterpret_cast(&fileHeader), headerSize); - if (size != headerSize) - throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. "); + // Read the file header. + StatisticsFileHeader fileHeader; + const uint32_t headerSize = sizeof(StatisticsFileHeader); + int64_t size = in->read(reinterpret_cast(&fileHeader), headerSize); + if (size != headerSize) + throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. "); - // Initialize fields from the file header. - version = fileHeader.version; - epoch = fileHeader.epoch; - const auto dataHash = fileHeader.dataHash; - const auto dataStreamSize = fileHeader.dataSize; + // Initialize fields from the file header. + version = fileHeader.version; + epoch = fileHeader.epoch; + const auto dataHash = fileHeader.dataHash; + const auto dataStreamSize = fileHeader.dataSize; - // Allocate the memory for the file data. - std::unique_ptr dataStreamSmartPtr(new char[dataStreamSize]); - auto* dataStream = dataStreamSmartPtr.get(); + // Allocate the memory for the file data. + std::unique_ptr dataStreamSmartPtr(new char[dataStreamSize]); + auto* dataStream = dataStreamSmartPtr.get(); + + // Read the data. + uint64_t dataOffset = 0; + auto sizeToRead = dataStreamSize; + size = in->read(dataStream, sizeToRead); + sizeToRead -= size; + dataOffset += size; + + while (sizeToRead > 0) + { + size = in->read(dataStream + dataOffset, sizeToRead); + if (size < 0) + throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. "); - // Read the data. - uint64_t dataOffset = 0; - auto sizeToRead = dataStreamSize; - size = in->read(dataStream, sizeToRead); sizeToRead -= size; dataOffset += size; + } - while (sizeToRead > 0) - { - size = in->read(dataStream + dataOffset, sizeToRead); - if (size < 0) - throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. "); + utils::Hasher128 hasher; + auto computedDataHash = hasher(dataStream, dataStreamSize); + if (dataHash != computedDataHash) + throw ios_base::failure("StatisticsManager::loadFromFile(): invalid file hash. "); - sizeToRead -= size; - dataOffset += size; - } + uint64_t count = 0; + std::memcpy(reinterpret_cast(&count), dataStream, sizeof(uint64_t)); + uint64_t offset = sizeof(uint64_t); - utils::Hasher128 hasher; - auto computedDataHash = hasher(dataStream, dataStreamSize); - if (dataHash != computedDataHash) - throw ios_base::failure("StatisticsManager::loadFromFile(): invalid file hash. "); - - uint64_t count = 0; - std::memcpy(reinterpret_cast(&count), dataStream, sizeof(uint64_t)); - uint64_t offset = sizeof(uint64_t); - - // For each pair. - for (uint64_t i = 0; i < count; ++i) - { - uint32_t oid; - KeyType keyType; - std::memcpy(reinterpret_cast(&oid), &dataStream[offset], sizeof(uint32_t)); - offset += sizeof(uint32_t); - std::memcpy(reinterpret_cast(&keyType), &dataStream[offset], sizeof(KeyType)); - offset += sizeof(KeyType); - // Insert pair. - keyTypes[oid] = keyType; - } + // For each pair. + for (uint64_t i = 0; i < count; ++i) + { + uint32_t oid; + KeyType keyType; + std::memcpy(reinterpret_cast(&oid), &dataStream[offset], sizeof(uint32_t)); + offset += sizeof(uint32_t); + std::memcpy(reinterpret_cast(&keyType), &dataStream[offset], sizeof(KeyType)); + offset += sizeof(KeyType); + // Insert pair. + keyTypes[oid] = keyType; + } } uint64_t StatisticsManager::computeHashFromStats() { - utils::Hasher128 hasher; - uint64_t dataStreamSize = 0; - std::unique_ptr dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize); - return hasher(dataStreamSmartPtr.get(), dataStreamSize); + utils::Hasher128 hasher; + uint64_t dataStreamSize = 0; + std::unique_ptr dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize); + return hasher(dataStreamSmartPtr.get(), dataStreamSize); } void StatisticsManager::serialize(messageqcpp::ByteStream& bs) { - uint64_t count = keyTypes.size(); - bs << version; - bs << epoch; - bs << count; + uint64_t count = keyTypes.size(); + bs << version; + bs << epoch; + bs << count; - for (const auto& keyType : keyTypes) - { - bs << keyType.first; - bs << (uint32_t) keyType.second; - } + for (const auto& keyType : keyTypes) + { + bs << keyType.first; + bs << (uint32_t)keyType.second; + } } void StatisticsManager::unserialize(messageqcpp::ByteStream& bs) { - uint64_t count; - bs >> version; - bs >> epoch; - bs >> count; + uint64_t count; + bs >> version; + bs >> epoch; + bs >> count; - for (uint32_t i = 0; i < count; ++i) - { - uint32_t oid, keyType; - bs >> oid; - bs >> keyType; - keyTypes[oid] = static_cast(keyType); - } + for (uint32_t i = 0; i < count; ++i) + { + uint32_t oid, keyType; + bs >> oid; + bs >> keyType; + keyTypes[oid] = static_cast(keyType); + } } -bool StatisticsManager::hasKey(uint32_t oid) { return keyTypes.count(oid) > 0 ? true : false; } +bool StatisticsManager::hasKey(uint32_t oid) +{ + return keyTypes.count(oid) > 0 ? true : false; +} -KeyType StatisticsManager::getKeyType(uint32_t oid) { return keyTypes[oid]; } +KeyType StatisticsManager::getKeyType(uint32_t oid) +{ + return keyTypes[oid]; +} StatisticsDistributor* StatisticsDistributor::instance() { - static StatisticsDistributor* sd = new StatisticsDistributor(); - return sd; + static StatisticsDistributor* sd = new StatisticsDistributor(); + return sd; } void StatisticsDistributor::distributeStatistics() { - countClients(); + countClients(); + { + std::lock_guard lock(mut); + // No clients. + if (clientsCount == 0) + return; + +#ifdef DEBUG_STATISTICS + std::cout << "Distribute statistics from ExeMgr(Server) to ExeMgr(Clients) " << std::endl; +#endif + + messageqcpp::ByteStream msg, statsHash, statsBs; + // Current hash. + statsHash << statistics::StatisticsManager::instance()->computeHashFromStats(); + // Statistics. + statistics::StatisticsManager::instance()->serialize(statsBs); + + for (uint32_t i = 0; i < clientsCount; ++i) { - std::lock_guard lock(mut); - // No clients. - if (clientsCount == 0) - return; + try + { + messageqcpp::ByteStream::quadbyte qb = ANALYZE_TABLE_REC_STATS; + msg << qb; + + auto exeMgrID = "ExeMgr" + std::to_string(i + 2); + // Create a client. + std::unique_ptr exemgrClient( + new messageqcpp::MessageQueueClient(exeMgrID)); #ifdef DEBUG_STATISTICS - std::cout << "Distribute statistics from ExeMgr(Server) to ExeMgr(Clients) " << std::endl; + std::cout << "Try to connect to " << exeMgrID << std::endl; #endif - - messageqcpp::ByteStream msg, statsHash, statsBs; - // Current hash. - statsHash << statistics::StatisticsManager::instance()->computeHashFromStats(); - // Statistics. - statistics::StatisticsManager::instance()->serialize(statsBs); - - for (uint32_t i = 0; i < clientsCount; ++i) + // Try to connect to the client. + if (!exemgrClient->connect()) { - try - { - messageqcpp::ByteStream::quadbyte qb = ANALYZE_TABLE_REC_STATS; - msg << qb; - - auto exeMgrID = "ExeMgr" + std::to_string(i + 2); - // Create a client. - std::unique_ptr exemgrClient( - new messageqcpp::MessageQueueClient(exeMgrID)); - + msg.restart(); #ifdef DEBUG_STATISTICS - std::cout << "Try to connect to " << exeMgrID << std::endl; + std::cout << "Unable to connect to " << exeMgrID << std::endl; #endif - // Try to connect to the client. - if (!exemgrClient->connect()) - { - msg.restart(); -#ifdef DEBUG_STATISTICS - std::cout << "Unable to connect to " << exeMgrID << std::endl; -#endif - continue; - } - -#ifdef DEBUG_STATISTICS - std::cout - << "Write flag ANALYZE_TABLE_REC_STATS from ExeMgr(Server) to ExeMgr(Clients) " - << std::endl; -#endif - // Write a flag to client ExeMgr. - exemgrClient->write(msg); - -#ifdef DEBUG_STATISTICS - std::cout << "Write statistics hash from ExeMgr(Server) to ExeMgr(Clients) " - << std::endl; -#endif - // Write a hash of the stats. - exemgrClient->write(statsHash); - - // Read the state from Client. - msg.restart(); - msg = exemgrClient->read(); - msg >> qb; - - // Do not need a stats. - if (qb == ANALYZE_TABLE_SUCCESS) - { - msg.restart(); - continue; - } - -#ifdef DEBUG_STATISTICS - std::cout << "Write statistics bytestream from ExeMgr(Server) to ExeMgr(Clients) " - << std::endl; -#endif - // Write a statistics to client ExeMgr. - exemgrClient->write(statsBs); - - // Read the flag back from the client ExeMgr. - msg.restart(); - msg = exemgrClient->read(); - - if (msg.length() == 0) - throw runtime_error("Lost conection to ExeMgr."); -#ifdef DEBUG_STATISTICS - std::cout << "Read flag on ExeMgr(Server) from ExeMgr(Client) " << std::endl; -#endif - msg.restart(); - } - catch (std::exception& e) - { - msg.restart(); - std::cerr << "distributeStatistics() failed with error: " << e.what() << std::endl; - } - catch (...) - { - msg.restart(); - std::cerr << "distributeStatistics() failed with unknown error." << std::endl; - } + continue; } + +#ifdef DEBUG_STATISTICS + std::cout << "Write flag ANALYZE_TABLE_REC_STATS from ExeMgr(Server) to ExeMgr(Clients) " + << std::endl; +#endif + // Write a flag to client ExeMgr. + exemgrClient->write(msg); + +#ifdef DEBUG_STATISTICS + std::cout << "Write statistics hash from ExeMgr(Server) to ExeMgr(Clients) " << std::endl; +#endif + // Write a hash of the stats. + exemgrClient->write(statsHash); + + // Read the state from Client. + msg.restart(); + msg = exemgrClient->read(); + msg >> qb; + + // Do not need a stats. + if (qb == ANALYZE_TABLE_SUCCESS) + { + msg.restart(); + continue; + } + +#ifdef DEBUG_STATISTICS + std::cout << "Write statistics bytestream from ExeMgr(Server) to ExeMgr(Clients) " << std::endl; +#endif + // Write a statistics to client ExeMgr. + exemgrClient->write(statsBs); + + // Read the flag back from the client ExeMgr. + msg.restart(); + msg = exemgrClient->read(); + + if (msg.length() == 0) + throw runtime_error("Lost conection to ExeMgr."); +#ifdef DEBUG_STATISTICS + std::cout << "Read flag on ExeMgr(Server) from ExeMgr(Client) " << std::endl; +#endif + msg.restart(); + } + catch (std::exception& e) + { + msg.restart(); + std::cerr << "distributeStatistics() failed with error: " << e.what() << std::endl; + } + catch (...) + { + msg.restart(); + std::cerr << "distributeStatistics() failed with unknown error." << std::endl; + } } + } } void StatisticsDistributor::countClients() { #ifdef DEBUG_STATISTICS - std::cout << "count clients to distribute statistics " << std::endl; + std::cout << "count clients to distribute statistics " << std::endl; #endif - auto* config = config::Config::makeConfig(); - // Starting from the ExeMgr2, since the Server starts on the ExeMgr1. - std::atomic exeMgrNumber(2); + auto* config = config::Config::makeConfig(); + // Starting from the ExeMgr2, since the Server starts on the ExeMgr1. + std::atomic exeMgrNumber(2); - try + try + { + while (true) { - while (true) - { - auto exeMgrID = "ExeMgr" + std::to_string(exeMgrNumber); - auto exeMgrIP = config->getConfig(exeMgrID, "IPAddr"); - if (exeMgrIP == "") - break; + auto exeMgrID = "ExeMgr" + std::to_string(exeMgrNumber); + auto exeMgrIP = config->getConfig(exeMgrID, "IPAddr"); + if (exeMgrIP == "") + break; #ifdef DEBUG_STATISTICS - std::cout << "Client: " << exeMgrID << std::endl; + std::cout << "Client: " << exeMgrID << std::endl; #endif - ++exeMgrNumber; - } - } - catch (std::exception& e) - { - std::cerr << "countClients() failed with error: " << e.what() << std::endl; - } - catch (...) - { - std::cerr << "countClients() failed with unknown error: "; + ++exeMgrNumber; } + } + catch (std::exception& e) + { + std::cerr << "countClients() failed with error: " << e.what() << std::endl; + } + catch (...) + { + std::cerr << "countClients() failed with unknown error: "; + } - clientsCount = exeMgrNumber - 2; + clientsCount = exeMgrNumber - 2; #ifdef DEBUG_STATISTICS - std::cout << "Number of clients: " << clientsCount << std::endl; + std::cout << "Number of clients: " << clientsCount << std::endl; #endif } -} // namespace statistics +} // namespace statistics diff --git a/utils/common/statistics.h b/utils/common/statistics.h index 2c7dbe95b..4c639d2a0 100644 --- a/utils/common/statistics.h +++ b/utils/common/statistics.h @@ -38,31 +38,30 @@ using namespace idbdatafile; namespace statistics { - // Represents a column key type: // PK - primary key. // FK - foreign key. enum class KeyType : uint32_t { - PK, - FK + PK, + FK }; // Rerpresents types of statistics CS supports. enum class StatisticsType : uint32_t { - // A special statistics type, made to solve circular inner join problem. - PK_FK + // A special statistics type, made to solve circular inner join problem. + PK_FK }; // Represetns a header for the statistics file. struct StatisticsFileHeader { - uint64_t version; - uint64_t epoch; - uint64_t dataHash; - uint64_t dataSize; - uint8_t offset[1024]; + uint64_t version; + uint64_t epoch; + uint64_t dataHash; + uint64_t dataSize; + uint8_t offset[1024]; }; // This class is responsible for processing and storing statistics. @@ -70,57 +69,65 @@ struct StatisticsFileHeader // the updated statistics into the special file. class StatisticsManager { - public: - // Returns the instance of this class, static initialization happens only once. - static StatisticsManager* instance(); - // Analyzes the given `rowGroup` by processing it row by row and searching for foreign key. - void analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace); - // Ouputs stats to out stream. - void output(StatisticsType statisticsType = StatisticsType::PK_FK); - // Saves stats to the file. - void saveToFile(); - // Loads stats from the file. - void loadFromFile(); - void incEpoch() { ++epoch; } - // Serialize stats to the given `bs`. - void serialize(messageqcpp::ByteStream& bs); - // Unserialize stats from the given `bs`. - void unserialize(messageqcpp::ByteStream& bs); - // Computes hash from the current statistics data. - uint64_t computeHashFromStats(); - // Checks whether statistics is available for the given `oid`. - bool hasKey(uint32_t oid); - // Returns a KeyType for the given `oid`. - KeyType getKeyType(uint32_t oid); + public: + // Returns the instance of this class, static initialization happens only once. + static StatisticsManager* instance(); + // Analyzes the given `rowGroup` by processing it row by row and searching for foreign key. + void analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace); + // Ouputs stats to out stream. + void output(StatisticsType statisticsType = StatisticsType::PK_FK); + // Saves stats to the file. + void saveToFile(); + // Loads stats from the file. + void loadFromFile(); + void incEpoch() + { + ++epoch; + } + // Serialize stats to the given `bs`. + void serialize(messageqcpp::ByteStream& bs); + // Unserialize stats from the given `bs`. + void unserialize(messageqcpp::ByteStream& bs); + // Computes hash from the current statistics data. + uint64_t computeHashFromStats(); + // Checks whether statistics is available for the given `oid`. + bool hasKey(uint32_t oid); + // Returns a KeyType for the given `oid`. + KeyType getKeyType(uint32_t oid); - private: - std::map keyTypes; - StatisticsManager() : epoch(0), version(1) { IDBPolicy::init(true, false, "", 0); } - std::unique_ptr convertStatsToDataStream(uint64_t& dataStreamSize); + private: + std::map keyTypes; + StatisticsManager() : epoch(0), version(1) + { + IDBPolicy::init(true, false, "", 0); + } + std::unique_ptr convertStatsToDataStream(uint64_t& dataStreamSize); - std::mutex mut; - uint32_t epoch; - uint32_t version; - std::string statsFile = "/var/lib/columnstore/local/statistics"; + std::mutex mut; + uint32_t epoch; + uint32_t version; + std::string statsFile = "/var/lib/columnstore/local/statistics"; }; // This class is responsible for distributing the statistics across all `ExeMgr` in a cluster. class StatisticsDistributor { - public: - // Returns the instance of this class, static initialization happens only once. - static StatisticsDistributor* instance(); + public: + // Returns the instance of this class, static initialization happens only once. + static StatisticsDistributor* instance(); - // Distribute stats across all `ExeMgr` in cluster by connecting to them using config file. - void distributeStatistics(); + // Distribute stats across all `ExeMgr` in cluster by connecting to them using config file. + void distributeStatistics(); - private: - StatisticsDistributor() : clientsCount(0) {} - // Count the number of clients by reading config file and evaluating `ExeMgr` fields. - void countClients(); - uint32_t clientsCount; - std::mutex mut; + private: + StatisticsDistributor() : clientsCount(0) + { + } + // Count the number of clients by reading config file and evaluating `ExeMgr` fields. + void countClients(); + uint32_t clientsCount; + std::mutex mut; }; -} // namespace statistics +} // namespace statistics #endif diff --git a/utils/common/stlpoolallocator.h b/utils/common/stlpoolallocator.h index 650d4e79f..ed2bd2229 100644 --- a/utils/common/stlpoolallocator.h +++ b/utils/common/stlpoolallocator.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ /* Makes PoolAllocator STL-compliant */ @@ -34,153 +34,153 @@ namespace utils { - /* If using the pool allocator with a boost smart ptr, use an instance of this as the deleter. */ struct BoostPoolDeallocator { - inline void operator()(void* ptr) { }; + inline void operator()(void* ptr){}; }; /* This is an STL-compliant wrapper for PoolAllocator + an optimization for containers * that aren't entirely node based (ex: vectors and hash tables) */ -template +template class STLPoolAllocator { -public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T* pointer; - typedef const T* const_pointer; - typedef T& reference; - typedef const T& const_reference; - typedef T value_type; - template struct rebind - { - typedef STLPoolAllocator other; - }; + public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + typedef T value_type; + template + struct rebind + { + typedef STLPoolAllocator other; + }; - STLPoolAllocator() throw(); - STLPoolAllocator(const STLPoolAllocator&) throw(); - STLPoolAllocator(uint32_t capacity) throw(); - template STLPoolAllocator(const STLPoolAllocator&) throw(); - ~STLPoolAllocator(); + STLPoolAllocator() throw(); + STLPoolAllocator(const STLPoolAllocator&) throw(); + STLPoolAllocator(uint32_t capacity) throw(); + template + STLPoolAllocator(const STLPoolAllocator&) throw(); + ~STLPoolAllocator(); - STLPoolAllocator& operator=(const STLPoolAllocator&); + STLPoolAllocator& operator=(const STLPoolAllocator&); - void usePoolAllocator(boost::shared_ptr b); - boost::shared_ptr getPoolAllocator(); + void usePoolAllocator(boost::shared_ptr b); + boost::shared_ptr getPoolAllocator(); - pointer allocate(size_type, const void* hint = 0); - void deallocate(pointer p, size_type n); - size_type max_size() const throw(); - inline uint64_t getMemUsage() const - { - return pa->getMemUsage(); - } + pointer allocate(size_type, const void* hint = 0); + void deallocate(pointer p, size_type n); + size_type max_size() const throw(); + inline uint64_t getMemUsage() const + { + return pa->getMemUsage(); + } - void construct(pointer p, const T& val); - void destroy(pointer p); + void construct(pointer p, const T& val); + void destroy(pointer p); - static const uint32_t DEFAULT_SIZE = 32768 * sizeof(T); + static const uint32_t DEFAULT_SIZE = 32768 * sizeof(T); - boost::shared_ptr pa; + boost::shared_ptr pa; }; -template +template STLPoolAllocator::STLPoolAllocator() throw() { - pa.reset(new PoolAllocator(DEFAULT_SIZE)); + pa.reset(new PoolAllocator(DEFAULT_SIZE)); } -template +template STLPoolAllocator::STLPoolAllocator(const STLPoolAllocator& s) throw() { - pa = s.pa; + pa = s.pa; } -template +template STLPoolAllocator::STLPoolAllocator(uint32_t capacity) throw() { - pa.reset(new PoolAllocator(capacity)); + pa.reset(new PoolAllocator(capacity)); } -template -template +template +template STLPoolAllocator::STLPoolAllocator(const STLPoolAllocator& s) throw() { - pa = s.pa; + pa = s.pa; } -template +template STLPoolAllocator::~STLPoolAllocator() { } -template +template void STLPoolAllocator::usePoolAllocator(boost::shared_ptr p) { - pa = p; + pa = p; } -template +template boost::shared_ptr STLPoolAllocator::getPoolAllocator() { - return pa; + return pa; } -template -typename STLPoolAllocator::pointer -STLPoolAllocator::allocate(typename STLPoolAllocator::size_type s, - typename std::allocator::const_pointer hint) +template +typename STLPoolAllocator::pointer STLPoolAllocator::allocate( + typename STLPoolAllocator::size_type s, typename std::allocator::const_pointer hint) { - return (pointer) pa->allocate(s * sizeof(T)); + return (pointer)pa->allocate(s * sizeof(T)); } -template +template void STLPoolAllocator::deallocate(typename STLPoolAllocator::pointer p, typename STLPoolAllocator::size_type n) { - pa->deallocate((void*) p); + pa->deallocate((void*)p); } -template +template typename STLPoolAllocator::size_type STLPoolAllocator::max_size() const throw() { - return std::numeric_limits::max() / sizeof(T); + return std::numeric_limits::max() / sizeof(T); } -template +template void STLPoolAllocator::construct(typename STLPoolAllocator::pointer p, const T& val) { - new ((void*)p) T(val); + new ((void*)p) T(val); } -template +template void STLPoolAllocator::destroy(typename STLPoolAllocator::pointer p) { - p->T::~T(); + p->T::~T(); } -template +template STLPoolAllocator& STLPoolAllocator::operator=(const STLPoolAllocator& c) { - pa = c.pa; - return *this; + pa = c.pa; + return *this; } -template +template bool operator==(const STLPoolAllocator&, const STLPoolAllocator&) { - return true; + return true; } -template +template bool operator!=(const STLPoolAllocator&, const STLPoolAllocator&) { - return false; + return false; } -} +} // namespace utils #endif diff --git a/utils/common/syncstream.h b/utils/common/syncstream.h index 001c02dbb..ad851ad29 100644 --- a/utils/common/syncstream.h +++ b/utils/common/syncstream.h @@ -30,28 +30,28 @@ void copyStream(istream& iss, ostream& oss) { - string line; - getline(iss, line); - while (iss.good()) - { - oss << line << endl; - getline(iss, line); - } + string line; + getline(iss, line); + while (iss.good()) + { + oss << line << endl; + getline(iss, line); + } } main() { - FILE* ifp; - FILE* ofp; + FILE* ifp; + FILE* ofp; - ... + ... - isyncstream iss(ifp); - osyncstream oss(ofp); + isyncstream iss(ifp); + osyncstream oss(ofp); - copyStream(iss, oss); + copyStream(iss, oss); - ... + ... } */ @@ -63,90 +63,98 @@ main() namespace syncstream { - /** A streambuf implementation for C stdio FILE* streams. * * Adapted from http://www.drdobbs.com/184401305 */ class syncbuf : public std::streambuf { -public: - /** ctor */ - syncbuf(FILE* f) : std::streambuf(), fptr(f) {} + public: + /** ctor */ + syncbuf(FILE* f) : std::streambuf(), fptr(f) + { + } -protected: - /** Write character in the case of overflow */ - virtual int overflow(int c = EOF) - { - return (c != EOF ? fputc(c, fptr) : EOF); - } - /** Get character in the case of overflow */ - virtual int underflow() - { - int c = getc(fptr); + protected: + /** Write character in the case of overflow */ + virtual int overflow(int c = EOF) + { + return (c != EOF ? fputc(c, fptr) : EOF); + } + /** Get character in the case of overflow */ + virtual int underflow() + { + int c = getc(fptr); - if (c != EOF) - ungetc(c, fptr); + if (c != EOF) + ungetc(c, fptr); - return c; - } - /** Get character in the case of overflow and advance get pointer */ - virtual int uflow() - { - return getc(fptr); - } - /** put character back in the case of backup underflow */ - virtual int pbackfail(int c = EOF) - { - return (c != EOF ? ungetc(c, fptr) : EOF); - } - /** Synchronize stream buffer */ - virtual int sync() - { - return fflush(fptr); - } + return c; + } + /** Get character in the case of overflow and advance get pointer */ + virtual int uflow() + { + return getc(fptr); + } + /** put character back in the case of backup underflow */ + virtual int pbackfail(int c = EOF) + { + return (c != EOF ? ungetc(c, fptr) : EOF); + } + /** Synchronize stream buffer */ + virtual int sync() + { + return fflush(fptr); + } -private: - FILE* fptr; + private: + FILE* fptr; }; /** An istream adaptor for input FILE* streams */ class isyncstream : public std::istream { -public: - /** ctor */ - isyncstream() : istream(&buf), buf(0) {} - /** ctor */ - isyncstream(FILE* fptr) : istream(&buf), buf(fptr) {} - /** const streambuf accessor */ - const syncbuf* rdbuf() const - { - return &buf; - } + public: + /** ctor */ + isyncstream() : istream(&buf), buf(0) + { + } + /** ctor */ + isyncstream(FILE* fptr) : istream(&buf), buf(fptr) + { + } + /** const streambuf accessor */ + const syncbuf* rdbuf() const + { + return &buf; + } -private: - syncbuf buf; + private: + syncbuf buf; }; /** An ostream adaptor for output FILE* streams */ class osyncstream : public std::ostream { -public: - /** ctor */ - osyncstream() : ostream(&buf), buf(0) {} - /** ctor */ - osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) {} - /** const streambuf accessor */ - const syncbuf* rdbuf() const - { - return &buf; - } + public: + /** ctor */ + osyncstream() : ostream(&buf), buf(0) + { + } + /** ctor */ + osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) + { + } + /** const streambuf accessor */ + const syncbuf* rdbuf() const + { + return &buf; + } -private: - syncbuf buf; + private: + syncbuf buf; }; -} +} // namespace syncstream #endif - diff --git a/utils/common/threadnaming.cpp b/utils/common/threadnaming.cpp index 204b84474..3183bc80f 100644 --- a/utils/common/threadnaming.cpp +++ b/utils/common/threadnaming.cpp @@ -20,15 +20,15 @@ namespace utils { - void setThreadName(const char *threadName) - { - prctl(PR_SET_NAME, threadName, 0, 0, 0); - } +void setThreadName(const char* threadName) +{ + prctl(PR_SET_NAME, threadName, 0, 0, 0); +} - std::string getThreadName() - { - char buf[32]; - prctl(PR_GET_NAME, buf, 0, 0, 0); - return std::string(buf); - } -} // end of namespace +std::string getThreadName() +{ + char buf[32]; + prctl(PR_GET_NAME, buf, 0, 0, 0); + return std::string(buf); +} +} // namespace utils diff --git a/utils/common/threadnaming.h b/utils/common/threadnaming.h index 486cd168a..31f489d10 100644 --- a/utils/common/threadnaming.h +++ b/utils/common/threadnaming.h @@ -21,7 +21,7 @@ namespace utils { - void setThreadName(const char *threadName); - std::string getThreadName(); -} // end of namespace +void setThreadName(const char* threadName); +std::string getThreadName(); +} // namespace utils #endif diff --git a/utils/common/utils_utf8.cpp b/utils/common/utils_utf8.cpp index cb23bb6a8..659bc61ac 100644 --- a/utils/common/utils_utf8.cpp +++ b/utils/common/utils_utf8.cpp @@ -17,21 +17,16 @@ #include "mariadb_my_sys.h" - namespace datatypes { - -static inline CHARSET_INFO & get_charset_or_bin(int32_t charsetNumber) +static inline CHARSET_INFO& get_charset_or_bin(int32_t charsetNumber) { - CHARSET_INFO *cs= get_charset(charsetNumber, MYF(MY_WME)); - return cs ? *cs : my_charset_bin; + CHARSET_INFO* cs = get_charset(charsetNumber, MYF(MY_WME)); + return cs ? *cs : my_charset_bin; } - -Charset::Charset(uint32_t charsetNumber) - :mCharset(&get_charset_or_bin(charsetNumber)) +Charset::Charset(uint32_t charsetNumber) : mCharset(&get_charset_or_bin(charsetNumber)) { } - -} +} // namespace datatypes diff --git a/utils/common/utils_utf8.h b/utils/common/utils_utf8.h index 75123e460..dff8c1423 100644 --- a/utils/common/utils_utf8.h +++ b/utils/common/utils_utf8.h @@ -18,12 +18,9 @@ // $Id$ - #ifndef _UTILS_UTF8_H_ #define _UTILS_UTF8_H_ - - #include #if defined(_MSC_VER) #include @@ -41,121 +38,118 @@ // Change the name from utf8. Even change the file name to something resembling char helper namespace utf8 { - const int MAX_UTF8_BYTES_PER_CHAR = 4; // BUG 5241 // Infinidb specific mbstowcs(). This will handle both windows and unix platforms // Params dest and max should have enough length to accomodate NULL -inline -size_t idb_mbstowcs(wchar_t* dest, const char* src, size_t max) +inline size_t idb_mbstowcs(wchar_t* dest, const char* src, size_t max) { #ifdef _MSC_VER - // 4th param (-1) denotes to convert till hit NULL char - // if 6th param max = 0, will return the required buffer size - size_t strwclen = MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)max); - // decrement the count of NULL; will become -1 on failure - return --strwclen; + // 4th param (-1) denotes to convert till hit NULL char + // if 6th param max = 0, will return the required buffer size + size_t strwclen = MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)max); + // decrement the count of NULL; will become -1 on failure + return --strwclen; #else - return mbstowcs(dest, src, max); + return mbstowcs(dest, src, max); #endif } // BUG 5241 // Infinidb specific wcstombs(). This will handle both windows and unix platforms // Params dest and max should have enough length to accomodate NULL -inline -size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max) +inline size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max) { #ifdef _MSC_VER - // 4th param (-1) denotes to convert till hit NULL char - //if 6th param max = 0, will return the required buffer size - size_t strmblen = WideCharToMultiByte( CP_UTF8, 0, src, -1, dest, (int)max, NULL, NULL); - // decrement the count of NULL; will become -1 on failure - return --strmblen; + // 4th param (-1) denotes to convert till hit NULL char + // if 6th param max = 0, will return the required buffer size + size_t strmblen = WideCharToMultiByte(CP_UTF8, 0, src, -1, dest, (int)max, NULL, NULL); + // decrement the count of NULL; will become -1 on failure + return --strmblen; #else - return wcstombs(dest, src, max); + return wcstombs(dest, src, max); #endif } - // convert UTF-8 string to wstring -inline -std::wstring utf8_to_wstring (const std::string& str) +inline std::wstring utf8_to_wstring(const std::string& str) { - size_t bufsize = str.length() + 1; + size_t bufsize = str.length() + 1; - // Convert to wide characters. Do all further work in wide characters - wchar_t* wcbuf = new wchar_t[bufsize]; - // Passing +1 so that windows is happy to see extra position to place NULL - size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1); + // Convert to wide characters. Do all further work in wide characters + wchar_t* wcbuf = new wchar_t[bufsize]; + // Passing +1 so that windows is happy to see extra position to place NULL + size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1); - // if result is -1 it means bad characters which may happen if locale is wrong. - // return an empty string - if ( strwclen == static_cast(-1) ) - strwclen = 0; + // if result is -1 it means bad characters which may happen if locale is wrong. + // return an empty string + if (strwclen == static_cast(-1)) + strwclen = 0; - std::wstring ret(wcbuf, strwclen); + std::wstring ret(wcbuf, strwclen); - delete [] wcbuf; - return ret; + delete[] wcbuf; + return ret; } - // convert wstring to UTF-8 string -inline -std::string wstring_to_utf8 (const std::wstring& str) +inline std::string wstring_to_utf8(const std::wstring& str) { - char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1]; - // Passing +1 so that windows is happy to see extra position to place NULL - size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1); + char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1]; + // Passing +1 so that windows is happy to see extra position to place NULL + size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1); - // if result is -1 it means bad characters which may happen if locale is wrong. - // return an empty string - if ( strmblen == static_cast(-1) ) - strmblen = 0; + // if result is -1 it means bad characters which may happen if locale is wrong. + // return an empty string + if (strmblen == static_cast(-1)) + strmblen = 0; - std::string ret(outbuf, strmblen); + std::string ret(outbuf, strmblen); - delete [] outbuf; - return ret; + delete[] outbuf; + return ret; } -inline -uint8_t utf8_truncate_point(const char* input, size_t length) +inline uint8_t utf8_truncate_point(const char* input, size_t length) { - // Find the beginning of a multibyte char to truncate at and return the - // number of bytes to truncate1` - if (length < 3) - { - return 0; - } - - const unsigned char* b = (const unsigned char*)(input) + length - 3; - - if (b[2] & 0x80) - { - // First byte in a new multi-byte sequence - if (b[2] & 0x40) return 1; - // 3 byte sequence - else if ((b[1] & 0xe0) == 0xe0) return 2; - // 4 byte sequence - else if ((b[0] & 0xf0) == 0xf0) return 3; - } - + // Find the beginning of a multibyte char to truncate at and return the + // number of bytes to truncate1` + if (length < 3) + { return 0; + } + + const unsigned char* b = (const unsigned char*)(input) + length - 3; + + if (b[2] & 0x80) + { + // First byte in a new multi-byte sequence + if (b[2] & 0x40) + return 1; + // 3 byte sequence + else if ((b[1] & 0xe0) == 0xe0) + return 2; + // 4 byte sequence + else if ((b[0] & 0xf0) == 0xf0) + return 3; + } + + return 0; } int mcs_strcoll(const char* str1, const char* str2, const uint32_t charsetNumber); -int mcs_strcoll(const char* str1, const uint32_t l1, const char* str2, const uint32_t l2, const uint32_t charsetNumber); +int mcs_strcoll(const char* str1, const uint32_t l1, const char* str2, const uint32_t l2, + const uint32_t charsetNumber); int mcs_strcoll(const std::string* str1, const std::string* str2, const uint32_t charsetNumber); int mcs_strcoll(const std::string& str1, const std::string& str2, const uint32_t charsetNumber); int mcs_strcollsp(const char* str1, const char* str2, const uint32_t charsetNumber); -int mcs_strcollsp(const char* str1, uint32_t l1, const char* str2, const uint32_t l2, const uint32_t charsetNumber); +int mcs_strcollsp(const char* str1, uint32_t l1, const char* str2, const uint32_t l2, + const uint32_t charsetNumber); int mcs_strcollsp(const std::string* str1, const std::string* str2, const uint32_t charsetNumber); int mcs_strcollsp(const std::string& str1, const std::string& str2, const uint32_t charsetNumber); -} //namespace utf8 +} // namespace utf8 #endif diff --git a/utils/common/vlarray.h b/utils/common/vlarray.h index c803046c0..665e45c91 100644 --- a/utils/common/vlarray.h +++ b/utils/common/vlarray.h @@ -17,84 +17,111 @@ #ifndef UTILS_COMMON_VLARRAY_H #define UTILS_COMMON_VLARRAY_H -namespace utils { - -template +namespace utils +{ +template class VLArray { -public: - VLArray(size_t sz) : - sz(sz), - stack_storage(NULL), - dyn_storage(NULL), - ptr(NULL) + public: + VLArray(size_t sz) : sz(sz), stack_storage(NULL), dyn_storage(NULL), ptr(NULL) + { + if (sz > SIZE) { - if (sz > SIZE) { - dyn_storage = new T[sz]; - ptr = dyn_storage; - } else { - stack_storage = new (stack) T[sz]; - ptr = stack_storage; - } + dyn_storage = new T[sz]; + ptr = dyn_storage; } - - VLArray(size_t sz, const T& initval) : VLArray(sz) + else { - for (size_t i= 0; i < sz; ++i) - ptr[i]= initval; + stack_storage = new (stack) T[sz]; + ptr = stack_storage; } + } - VLArray(const VLArray&) = delete; - VLArray(VLArray&&) = delete; - VLArray& operator=(const VLArray&) = delete; - VLArray& operator=(VLArray&&) = delete; + VLArray(size_t sz, const T& initval) : VLArray(sz) + { + for (size_t i = 0; i < sz; ++i) + ptr[i] = initval; + } - ~VLArray() { - if (dyn_storage) { - delete [] dyn_storage; - } else { - // we cannot use `delete [] stack_storage` here so call d-tors explicitly - if (!std::is_trivially_destructible::value) { - for (size_t i = 0; i < sz; ++i) - stack_storage[i].~T(); - } - } + VLArray(const VLArray&) = delete; + VLArray(VLArray&&) = delete; + VLArray& operator=(const VLArray&) = delete; + VLArray& operator=(VLArray&&) = delete; + + ~VLArray() + { + if (dyn_storage) + { + delete[] dyn_storage; } - - size_t size() const { return sz; } - const T* data() const { return ptr; } - T* data() { return ptr; } - - const T& operator[](size_t i) const { return ptr[i]; } - T& operator[](size_t i) { return ptr[i]; } - - const T& at(size_t i) const { - if (i >= sz) { - throw std::out_of_range("index out of range: " + std::to_string(i) + - " >= size " + std::to_string(sz)); - } - return ptr[i]; + else + { + // we cannot use `delete [] stack_storage` here so call d-tors explicitly + if (!std::is_trivially_destructible::value) + { + for (size_t i = 0; i < sz; ++i) + stack_storage[i].~T(); + } } + } - T& at(size_t i) { - if (i >= sz) { - throw std::out_of_range("index out of range: " + std::to_string(i) + - " >= size " + std::to_string(sz)); - } - return ptr[i]; + size_t size() const + { + return sz; + } + const T* data() const + { + return ptr; + } + T* data() + { + return ptr; + } + + const T& operator[](size_t i) const + { + return ptr[i]; + } + T& operator[](size_t i) + { + return ptr[i]; + } + + const T& at(size_t i) const + { + if (i >= sz) + { + throw std::out_of_range("index out of range: " + std::to_string(i) + " >= size " + std::to_string(sz)); } + return ptr[i]; + } - operator const T* () const { return ptr; } - operator T* () { return ptr; } + T& at(size_t i) + { + if (i >= sz) + { + throw std::out_of_range("index out of range: " + std::to_string(i) + " >= size " + std::to_string(sz)); + } + return ptr[i]; + } -private: - const size_t sz; - alignas(T) char stack[SIZE * sizeof(T)]; - T* stack_storage; - T* dyn_storage; - T* ptr; + operator const T*() const + { + return ptr; + } + operator T*() + { + return ptr; + } + + private: + const size_t sz; + alignas(T) char stack[SIZE * sizeof(T)]; + T* stack_storage; + T* dyn_storage; + T* ptr; }; -} // namespace utils +} // namespace utils -#endif // UTILS_COMMON_VLARRAY_H +#endif // UTILS_COMMON_VLARRAY_H diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index ae70d275c..fb5da354a 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -23,27 +23,27 @@ namespace utils { - const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; - const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; +const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; +const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; - inline void int128Max(int128_t& val) - { - uint64_t* ptr = reinterpret_cast(&val); - ptr[0] = 0xFFFFFFFFFFFFFFFF; - ptr[1] = 0x7FFFFFFFFFFFFFFF; - } - - inline void int128Min(int128_t& val) - { - val = int128_t(0x8000000000000000LL) << 64; - } - - inline void uint128Max(uint128_t& val) - { - uint64_t* ptr = reinterpret_cast(&val); - ptr[0] = 0xFFFFFFFFFFFFFFFF; - ptr[1] = 0xFFFFFFFFFFFFFFFF; - } +inline void int128Max(int128_t& val) +{ + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0xFFFFFFFFFFFFFFFF; + ptr[1] = 0x7FFFFFFFFFFFFFFF; } -#endif // WIDE_DECIMAL_UTILS_H +inline void int128Min(int128_t& val) +{ + val = int128_t(0x8000000000000000LL) << 64; +} + +inline void uint128Max(uint128_t& val) +{ + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0xFFFFFFFFFFFFFFFF; + ptr[1] = 0xFFFFFFFFFFFFFFFF; +} +} // namespace utils + +#endif // WIDE_DECIMAL_UTILS_H diff --git a/utils/compress/idbcompress.cpp b/utils/compress/idbcompress.cpp index f25520f6e..cffe8d268 100644 --- a/utils/compress/idbcompress.cpp +++ b/utils/compress/idbcompress.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: idbcompress.cpp 3907 2013-06-18 13:32:46Z dcathey $ -* -******************************************************************************************/ + * $Id: idbcompress.cpp 3907 2013-06-18 13:32:46Z dcathey $ + * + ******************************************************************************************/ #include #include #include @@ -34,8 +34,9 @@ using namespace std; #include "lz4.h" #else // Taken from lz4.h. -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) \ + ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize) / 255) + 16) #endif #define IDBCOMP_DLLEXPORT @@ -47,7 +48,6 @@ namespace const uint64_t MAGIC_NUMBER = 0xfdc119a384d0778eULL; const uint64_t VERSION_NUM3 = 3; - // version 1.1 of the chunk data has a short header // QuickLZ compressed data never has the high bit set on the first byte const int SIG_OFFSET = 0; @@ -60,164 +60,159 @@ const uint32_t LBID_MAX_SIZE = 10; struct CompressedDBFileHeader { - uint64_t fMagicNumber; - uint64_t fVersionNum; - uint64_t fCompressionType; - uint64_t fHeaderSize; - uint64_t fBlockCount; - uint64_t fColumnWidth; - execplan::CalpontSystemCatalog::ColDataType fColDataType; - uint64_t fLBIDCount; - uint64_t fLBIDS[LBID_MAX_SIZE]; + uint64_t fMagicNumber; + uint64_t fVersionNum; + uint64_t fCompressionType; + uint64_t fHeaderSize; + uint64_t fBlockCount; + uint64_t fColumnWidth; + execplan::CalpontSystemCatalog::ColDataType fColDataType; + uint64_t fLBIDCount; + uint64_t fLBIDS[LBID_MAX_SIZE]; }; // Make the header to be 4K, regardless number of fields being defined/used in header. union CompressedDBFileHeaderBlock { - CompressedDBFileHeader fHeader; - char fDummy[compress::CompressInterface::HDR_BUF_LEN]; + CompressedDBFileHeader fHeader; + char fDummy[compress::CompressInterface::HDR_BUF_LEN]; }; -void initCompressedDBFileHeader( - void* hdrBuf, uint32_t columnWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType, - int compressionType, int hdrSize) +void initCompressedDBFileHeader(void* hdrBuf, uint32_t columnWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType, int compressionType, + int hdrSize) { - CompressedDBFileHeaderBlock* hdr = reinterpret_cast(hdrBuf); - hdr->fHeader.fMagicNumber = MAGIC_NUMBER; - hdr->fHeader.fVersionNum = VERSION_NUM3; - hdr->fHeader.fCompressionType = compressionType; - hdr->fHeader.fBlockCount = 0; - hdr->fHeader.fHeaderSize = hdrSize; - hdr->fHeader.fColumnWidth = columnWidth; - hdr->fHeader.fColDataType = colDataType; - hdr->fHeader.fLBIDCount = 0; - std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS)); + CompressedDBFileHeaderBlock* hdr = reinterpret_cast(hdrBuf); + hdr->fHeader.fMagicNumber = MAGIC_NUMBER; + hdr->fHeader.fVersionNum = VERSION_NUM3; + hdr->fHeader.fCompressionType = compressionType; + hdr->fHeader.fBlockCount = 0; + hdr->fHeader.fHeaderSize = hdrSize; + hdr->fHeader.fColumnWidth = columnWidth; + hdr->fHeader.fColDataType = colDataType; + hdr->fHeader.fLBIDCount = 0; + std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS)); } -} // namespace - +} // namespace namespace compress { #ifndef SKIP_IDB_COMPRESSION -CompressInterface::CompressInterface(unsigned int numUserPaddingBytes) : - fNumUserPaddingBytes(numUserPaddingBytes) -{ } +CompressInterface::CompressInterface(unsigned int numUserPaddingBytes) + : fNumUserPaddingBytes(numUserPaddingBytes) +{ +} /* V1 is really only available for decompression, we kill any DDL using V1 by hand. * Maybe should have a new api, isDecompressionAvail() ? Any request to compress * using V1 will silently be changed to V2. -*/ + */ /*static*/ bool CompressInterface::isCompressionAvail(int compressionType) { - return ((compressionType == 0) || (compressionType == 1) || - (compressionType == 2) || (compressionType == 3)); + return ((compressionType == 0) || (compressionType == 1) || (compressionType == 2) || + (compressionType == 3)); } size_t CompressInterface::getMaxCompressedSizeGeneric(size_t inLen) { - return std::max(snappy::MaxCompressedLength(inLen), - LZ4_COMPRESSBOUND(inLen)) + - HEADER_SIZE; + return std::max(snappy::MaxCompressedLength(inLen), LZ4_COMPRESSBOUND(inLen)) + HEADER_SIZE; } //------------------------------------------------------------------------------ // Compress a block of data //------------------------------------------------------------------------------ -int CompressInterface::compressBlock(const char* in, const size_t inLen, - unsigned char* out, size_t& outLen) const +int CompressInterface::compressBlock(const char* in, const size_t inLen, unsigned char* out, + size_t& outLen) const { - size_t snaplen = 0; - utils::Hasher128 hasher; + size_t snaplen = 0; + utils::Hasher128 hasher; - // loose input checking. - if (outLen < maxCompressedSize(inLen)) - { - cerr << "got outLen = " << outLen << " for inLen = " << inLen - << ", needed " << (maxCompressedSize(inLen)) << endl; - return ERR_BADOUTSIZE; - } + // loose input checking. + if (outLen < maxCompressedSize(inLen)) + { + cerr << "got outLen = " << outLen << " for inLen = " << inLen << ", needed " << (maxCompressedSize(inLen)) + << endl; + return ERR_BADOUTSIZE; + } - auto rc = compress(in, inLen, reinterpret_cast(&out[HEADER_SIZE]), - &outLen); - if (rc != ERR_OK) - { - return rc; - } + auto rc = compress(in, inLen, reinterpret_cast(&out[HEADER_SIZE]), &outLen); + if (rc != ERR_OK) + { + return rc; + } - snaplen = outLen; - uint8_t* signature = (uint8_t*) &out[SIG_OFFSET]; - uint32_t* checksum = (uint32_t*) &out[CHECKSUM_OFFSET]; - uint32_t* len = (uint32_t*) &out[LEN_OFFSET]; - *signature = getChunkMagicNumber(); - *checksum = hasher((char*) &out[HEADER_SIZE], snaplen); - *len = snaplen; + snaplen = outLen; + uint8_t* signature = (uint8_t*)&out[SIG_OFFSET]; + uint32_t* checksum = (uint32_t*)&out[CHECKSUM_OFFSET]; + uint32_t* len = (uint32_t*)&out[LEN_OFFSET]; + *signature = getChunkMagicNumber(); + *checksum = hasher((char*)&out[HEADER_SIZE], snaplen); + *len = snaplen; - //cerr << "cb: " << inLen << '/' << outLen << '/' << (snappy::MaxCompressedLength(inLen) + HEADER_SIZE) << - // " : " << (snaplen + HEADER_SIZE) << endl; + // cerr << "cb: " << inLen << '/' << outLen << '/' << (snappy::MaxCompressedLength(inLen) + HEADER_SIZE) << + // " : " << (snaplen + HEADER_SIZE) << endl; - outLen = snaplen + HEADER_SIZE; + outLen = snaplen + HEADER_SIZE; - return ERR_OK; + return ERR_OK; } //------------------------------------------------------------------------------ // Decompress a block of data //------------------------------------------------------------------------------ -int CompressInterface::uncompressBlock(const char* in, const size_t inLen, - unsigned char* out, +int CompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out, size_t& outLen) const { - uint32_t realChecksum; - uint32_t storedChecksum; - uint32_t storedLen; - uint8_t storedMagic; - utils::Hasher128 hasher; - auto tmpOutLen = outLen; - outLen = 0; + uint32_t realChecksum; + uint32_t storedChecksum; + uint32_t storedLen; + uint8_t storedMagic; + utils::Hasher128 hasher; + auto tmpOutLen = outLen; + outLen = 0; - if (inLen < 1) - return ERR_BADINPUT; + if (inLen < 1) + return ERR_BADINPUT; - storedMagic = *((uint8_t*) &in[SIG_OFFSET]); + storedMagic = *((uint8_t*)&in[SIG_OFFSET]); - if (storedMagic == getChunkMagicNumber()) + if (storedMagic == getChunkMagicNumber()) + { + if (inLen < HEADER_SIZE) + return ERR_BADINPUT; + + storedChecksum = *((uint32_t*)&in[CHECKSUM_OFFSET]); + storedLen = *((uint32_t*)(&in[LEN_OFFSET])); + + if (inLen < storedLen + HEADER_SIZE) + return ERR_BADINPUT; + + realChecksum = hasher(&in[HEADER_SIZE], storedLen); + + if (storedChecksum != realChecksum) + return ERR_CHECKSUM; + + auto rc = uncompress(&in[HEADER_SIZE], storedLen, reinterpret_cast(out), &tmpOutLen); + if (rc != ERR_OK) { - if (inLen < HEADER_SIZE) - return ERR_BADINPUT; - - storedChecksum = *((uint32_t*) &in[CHECKSUM_OFFSET]); - storedLen = *((uint32_t*) (&in[LEN_OFFSET])); - - if (inLen < storedLen + HEADER_SIZE) - return ERR_BADINPUT; - - realChecksum = hasher(&in[HEADER_SIZE], storedLen); - - if (storedChecksum != realChecksum) - return ERR_CHECKSUM; - - auto rc = uncompress(&in[HEADER_SIZE], storedLen, reinterpret_cast(out), &tmpOutLen); - if (rc != ERR_OK) - { - cerr << "uncompressBlock failed!" << endl; - return ERR_DECOMPRESS; - } - - outLen = tmpOutLen; - } - else - { - // v1 compression or bad header - return ERR_BADINPUT; + cerr << "uncompressBlock failed!" << endl; + return ERR_DECOMPRESS; } - //cerr << "ub: " << inLen << " : " << outLen << endl; + outLen = tmpOutLen; + } + else + { + // v1 compression or bad header + return ERR_BADINPUT; + } - return ERR_OK; + // cerr << "ub: " << inLen << " : " << outLen << endl; + + return ERR_OK; } //------------------------------------------------------------------------------ @@ -225,42 +220,41 @@ int CompressInterface::uncompressBlock(const char* in, const size_t inLen, //------------------------------------------------------------------------------ int CompressInterface::verifyHdr(const void* hdrBuf) { - const CompressedDBFileHeader* hdr = reinterpret_cast(hdrBuf); + const CompressedDBFileHeader* hdr = reinterpret_cast(hdrBuf); - if (hdr->fMagicNumber != MAGIC_NUMBER) - return -1; + if (hdr->fMagicNumber != MAGIC_NUMBER) + return -1; - if (!isCompressionAvail(hdr->fCompressionType)) - return -2; + if (!isCompressionAvail(hdr->fCompressionType)) + return -2; - return 0; + return 0; } //------------------------------------------------------------------------------ // Extract compression pointer information out of the pointer buffer that is // passed in. ptrBuf points to the pointer section of the compression hdr. //------------------------------------------------------------------------------ -int CompressInterface::getPtrList(const char* ptrBuf, const int ptrBufSize, - CompChunkPtrList& chunkPtrs) +int CompressInterface::getPtrList(const char* ptrBuf, const int ptrBufSize, CompChunkPtrList& chunkPtrs) { - int rc = 0; - chunkPtrs.clear(); + int rc = 0; + chunkPtrs.clear(); - const uint64_t* ptrs = reinterpret_cast(ptrBuf); - const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t); + const uint64_t* ptrs = reinterpret_cast(ptrBuf); + const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t); - for (unsigned int i = 0; (i < NUM_PTRS) && (rc == 0); i++) - { - if (ptrs[i + 1] == 0) // 0 offset means end of data - break; + for (unsigned int i = 0; (i < NUM_PTRS) && (rc == 0); i++) + { + if (ptrs[i + 1] == 0) // 0 offset means end of data + break; - if (ptrs[i + 1] > ptrs[i]) - chunkPtrs.push_back(make_pair( ptrs[i], (ptrs[i + 1] - ptrs[i]))); - else - rc = -1; - } + if (ptrs[i + 1] > ptrs[i]) + chunkPtrs.push_back(make_pair(ptrs[i], (ptrs[i + 1] - ptrs[i]))); + else + rc = -1; + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -269,31 +263,30 @@ int CompressInterface::getPtrList(const char* ptrBuf, const int ptrBufSize, // one for the file header, and one for the list of pointers. // Wrapper of above method for backward compatibility. //------------------------------------------------------------------------------ -int CompressInterface::getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs ) +int CompressInterface::getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs) { - return getPtrList(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN, chunkPtrs); + return getPtrList(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN, chunkPtrs); } //------------------------------------------------------------------------------ // Count the number of chunk pointers in the pointer header(s) //------------------------------------------------------------------------------ -unsigned int CompressInterface::getPtrCount(const char* ptrBuf, - const int ptrBufSize) +unsigned int CompressInterface::getPtrCount(const char* ptrBuf, const int ptrBufSize) { - unsigned int chunkCount = 0; + unsigned int chunkCount = 0; - const uint64_t* ptrs = reinterpret_cast(ptrBuf); - const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t); + const uint64_t* ptrs = reinterpret_cast(ptrBuf); + const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t); - for (unsigned int i = 0; i < NUM_PTRS; i++) - { - if (ptrs[i + 1] == 0) // 0 offset means end of data - break; + for (unsigned int i = 0; i < NUM_PTRS; i++) + { + if (ptrs[i + 1] == 0) // 0 offset means end of data + break; - chunkCount++; - } + chunkCount++; + } - return chunkCount; + return chunkCount; } //------------------------------------------------------------------------------ @@ -304,22 +297,21 @@ unsigned int CompressInterface::getPtrCount(const char* ptrBuf, //------------------------------------------------------------------------------ unsigned int CompressInterface::getPtrCount(const char* hdrBuf) { - return getPtrCount(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN); + return getPtrCount(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN); } //------------------------------------------------------------------------------ // Store list of compression pointers into the specified header. //------------------------------------------------------------------------------ -void CompressInterface::storePtrs(const std::vector& ptrs, - void* ptrBuf, int ptrSectionSize) +void CompressInterface::storePtrs(const std::vector& ptrs, void* ptrBuf, int ptrSectionSize) { - memset((ptrBuf), 0, ptrSectionSize); // reset the pointer section to 0 - uint64_t* hdrPtrs = reinterpret_cast(ptrBuf); + memset((ptrBuf), 0, ptrSectionSize); // reset the pointer section to 0 + uint64_t* hdrPtrs = reinterpret_cast(ptrBuf); - for (unsigned i = 0; i < ptrs.size(); i++) - { - hdrPtrs[i] = ptrs[i]; - } + for (unsigned i = 0; i < ptrs.size(); i++) + { + hdrPtrs[i] = ptrs[i]; + } } //------------------------------------------------------------------------------ @@ -327,34 +319,29 @@ void CompressInterface::storePtrs(const std::vector& ptrs, //------------------------------------------------------------------------------ void CompressInterface::storePtrs(const std::vector& ptrs, void* ptrBuf) { - storePtrs(ptrs, reinterpret_cast(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN); + storePtrs(ptrs, reinterpret_cast(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN); } //------------------------------------------------------------------------------ // Initialize the header blocks to be written at the start of a dictionary file. //------------------------------------------------------------------------------ -void CompressInterface::initHdr( - void* hdrBuf, void* ptrBuf, uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType columnType, - int compressionType, int hdrSize) +void CompressInterface::initHdr(void* hdrBuf, void* ptrBuf, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType, + int hdrSize) { - memset(hdrBuf, 0, HDR_BUF_LEN); - memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN); - initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType, - hdrSize); + memset(hdrBuf, 0, HDR_BUF_LEN); + memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN); + initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType, hdrSize); } //------------------------------------------------------------------------------ // Initialize the header blocks to be written at the start of a column file. //------------------------------------------------------------------------------ -void CompressInterface::initHdr( - void* hdrBuf, uint32_t columnWidth, - execplan::CalpontSystemCatalog::ColDataType columnType, - int compressionType) +void CompressInterface::initHdr(void* hdrBuf, uint32_t columnWidth, + execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType) { - memset(hdrBuf, 0, HDR_BUF_LEN * 2); - initCompressedDBFileHeader(hdrBuf, columnWidth, columnType, - compressionType, HDR_BUF_LEN * 2); + memset(hdrBuf, 0, HDR_BUF_LEN * 2); + initCompressedDBFileHeader(hdrBuf, columnWidth, columnType, compressionType, HDR_BUF_LEN * 2); } //------------------------------------------------------------------------------ @@ -362,8 +349,7 @@ void CompressInterface::initHdr( //------------------------------------------------------------------------------ uint64_t CompressInterface::getVersionNumber(const void* hdrBuf) { - return ( - reinterpret_cast(hdrBuf)->fVersionNum); + return (reinterpret_cast(hdrBuf)->fVersionNum); } //------------------------------------------------------------------------------ @@ -371,7 +357,7 @@ uint64_t CompressInterface::getVersionNumber(const void* hdrBuf) //------------------------------------------------------------------------------ void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count) { - reinterpret_cast(hdrBuf)->fBlockCount = count; + reinterpret_cast(hdrBuf)->fBlockCount = count; } //------------------------------------------------------------------------------ @@ -379,7 +365,7 @@ void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count) //------------------------------------------------------------------------------ uint64_t CompressInterface::getBlockCount(const void* hdrBuf) { - return (reinterpret_cast(hdrBuf)->fBlockCount); + return (reinterpret_cast(hdrBuf)->fBlockCount); } //------------------------------------------------------------------------------ @@ -387,8 +373,7 @@ uint64_t CompressInterface::getBlockCount(const void* hdrBuf) //------------------------------------------------------------------------------ uint64_t CompressInterface::getCompressionType(const void* hdrBuf) { - return (reinterpret_cast(hdrBuf) - ->fCompressionType); + return (reinterpret_cast(hdrBuf)->fCompressionType); } //------------------------------------------------------------------------------ @@ -396,7 +381,7 @@ uint64_t CompressInterface::getCompressionType(const void* hdrBuf) //------------------------------------------------------------------------------ void CompressInterface::setHdrSize(void* hdrBuf, uint64_t size) { - reinterpret_cast(hdrBuf)->fHeaderSize = size; + reinterpret_cast(hdrBuf)->fHeaderSize = size; } //------------------------------------------------------------------------------ @@ -404,17 +389,15 @@ void CompressInterface::setHdrSize(void* hdrBuf, uint64_t size) //------------------------------------------------------------------------------ uint64_t CompressInterface::getHdrSize(const void* hdrBuf) { - return (reinterpret_cast(hdrBuf)->fHeaderSize); + return (reinterpret_cast(hdrBuf)->fHeaderSize); } //------------------------------------------------------------------------------ // Get column type //----------------------------------------------------------------------------- -execplan::CalpontSystemCatalog::ColDataType -CompressInterface::getColDataType(const void* hdrBuf) +execplan::CalpontSystemCatalog::ColDataType CompressInterface::getColDataType(const void* hdrBuf) { - return ( - reinterpret_cast(hdrBuf)->fColDataType); + return (reinterpret_cast(hdrBuf)->fColDataType); } //------------------------------------------------------------------------------ @@ -422,8 +405,7 @@ CompressInterface::getColDataType(const void* hdrBuf) //------------------------------------------------------------------------------ uint64_t CompressInterface::getColumnWidth(const void* hdrBuf) { - return ( - reinterpret_cast(hdrBuf)->fColumnWidth); + return (reinterpret_cast(hdrBuf)->fColumnWidth); } //------------------------------------------------------------------------------ @@ -431,9 +413,9 @@ uint64_t CompressInterface::getColumnWidth(const void* hdrBuf) //------------------------------------------------------------------------------ uint64_t CompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index) { - if (index < LBID_MAX_SIZE) - return (reinterpret_cast(hdrBuf)->fLBIDS[index]); - return 0; + if (index < LBID_MAX_SIZE) + return (reinterpret_cast(hdrBuf)->fLBIDS[index]); + return 0; } //------------------------------------------------------------------------------ @@ -441,12 +423,12 @@ uint64_t CompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index) //------------------------------------------------------------------------------ void CompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) { - if (lbid && index < LBID_MAX_SIZE) - { - reinterpret_cast(hdrBuf)->fLBIDS[index] = lbid; - reinterpret_cast(hdrBuf)->fLBIDCount = - std::max(index + 1, reinterpret_cast(hdrBuf)->fLBIDCount); - } + if (lbid && index < LBID_MAX_SIZE) + { + reinterpret_cast(hdrBuf)->fLBIDS[index] = lbid; + reinterpret_cast(hdrBuf)->fLBIDCount = + std::max(index + 1, reinterpret_cast(hdrBuf)->fLBIDCount); + } } //------------------------------------------------------------------------------ @@ -454,25 +436,24 @@ void CompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t ind //------------------------------------------------------------------------------ uint64_t CompressInterface::getLBIDCount(void* hdrBuf) { - return reinterpret_cast(hdrBuf)->fLBIDCount; + return reinterpret_cast(hdrBuf)->fLBIDCount; } //------------------------------------------------------------------------------ // Calculates the chunk and block offset within the chunk for the specified // block number. //------------------------------------------------------------------------------ -void CompressInterface::locateBlock(unsigned int block, - unsigned int& chunkIndex, +void CompressInterface::locateBlock(unsigned int block, unsigned int& chunkIndex, unsigned int& blockOffsetWithinChunk) const { - const uint64_t BUFLEN = UNCOMPRESSED_INBUF_LEN; + const uint64_t BUFLEN = UNCOMPRESSED_INBUF_LEN; - uint64_t byteOffset = (uint64_t)block * BLOCK_SIZE; - uint64_t chunk = byteOffset / BUFLEN; - uint64_t blockInChunk = (byteOffset % BUFLEN) / BLOCK_SIZE; + uint64_t byteOffset = (uint64_t)block * BLOCK_SIZE; + uint64_t chunk = byteOffset / BUFLEN; + uint64_t blockInChunk = (byteOffset % BUFLEN) / BLOCK_SIZE; - chunkIndex = chunk; - blockOffsetWithinChunk = blockInChunk; + chunkIndex = chunk; + blockOffsetWithinChunk = blockInChunk; } //------------------------------------------------------------------------------ @@ -480,238 +461,215 @@ void CompressInterface::locateBlock(unsigned int block, // also expand to allow for user requested padding. Lastly, initialize padding // bytes to 0. //------------------------------------------------------------------------------ -int CompressInterface::padCompressedChunks(unsigned char* buf, size_t& len, - unsigned int maxLen) const +int CompressInterface::padCompressedChunks(unsigned char* buf, size_t& len, unsigned int maxLen) const { - int nPaddingBytes = 0; - int nRem = len % COMPRESSED_CHUNK_INCREMENT_SIZE; + int nPaddingBytes = 0; + int nRem = len % COMPRESSED_CHUNK_INCREMENT_SIZE; - if (nRem != 0) - { - nPaddingBytes = COMPRESSED_CHUNK_INCREMENT_SIZE - nRem; - } + if (nRem != 0) + { + nPaddingBytes = COMPRESSED_CHUNK_INCREMENT_SIZE - nRem; + } - nPaddingBytes = nPaddingBytes + fNumUserPaddingBytes; + nPaddingBytes = nPaddingBytes + fNumUserPaddingBytes; - if (nPaddingBytes > 0) - { - if ((len + nPaddingBytes) > maxLen) - return -1; + if (nPaddingBytes > 0) + { + if ((len + nPaddingBytes) > maxLen) + return -1; - memset(buf + len, 0, nPaddingBytes); - len = len + nPaddingBytes; - } + memset(buf + len, 0, nPaddingBytes); + len = len + nPaddingBytes; + } - return 0; + return 0; } // Snappy CompressInterfaceSnappy::CompressInterfaceSnappy(uint32_t numUserPaddingBytes) - : CompressInterface(numUserPaddingBytes) + : CompressInterface(numUserPaddingBytes) { } -int32_t CompressInterfaceSnappy::compress(const char* in, size_t inLen, - char* out, size_t* outLen) const +int32_t CompressInterfaceSnappy::compress(const char* in, size_t inLen, char* out, size_t* outLen) const { - snappy::RawCompress(in, inLen, out, outLen); + snappy::RawCompress(in, inLen, out, outLen); #ifdef DEBUG_COMPRESSION - std::cout << "Snappy::compress: inLen " << inLen << ", outLen " << *outLen - << std::endl; + std::cout << "Snappy::compress: inLen " << inLen << ", outLen " << *outLen << std::endl; #endif - return ERR_OK; + return ERR_OK; } -int32_t CompressInterfaceSnappy::uncompress(const char* in, size_t inLen, - char* out, size_t* outLen) const +int32_t CompressInterfaceSnappy::uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const { - size_t realOutLen = 0; - auto rc = snappy::GetUncompressedLength(in, inLen, &realOutLen); + size_t realOutLen = 0; + auto rc = snappy::GetUncompressedLength(in, inLen, &realOutLen); - if (!rc || realOutLen > *outLen) - { - cerr << "snappy::GetUncompressedLength failed. InLen: " << inLen - << ", outLen: " << *outLen << ", realOutLen: " << realOutLen - << endl; - return ERR_DECOMPRESS; - } + if (!rc || realOutLen > *outLen) + { + cerr << "snappy::GetUncompressedLength failed. InLen: " << inLen << ", outLen: " << *outLen + << ", realOutLen: " << realOutLen << endl; + return ERR_DECOMPRESS; + } - rc = snappy::RawUncompress(in, inLen, out); + rc = snappy::RawUncompress(in, inLen, out); - if (!rc) - { - cerr << "snappy::RawUnompress failed. InLen: " << inLen - << ", outLen: " << *outLen << endl; - return ERR_DECOMPRESS; - } + if (!rc) + { + cerr << "snappy::RawUnompress failed. InLen: " << inLen << ", outLen: " << *outLen << endl; + return ERR_DECOMPRESS; + } #ifdef DEBUG_COMPRESSION - std::cout << "Snappy::uncompress: inLen " << inLen << ", outLen " - << *outLen << std::endl; + std::cout << "Snappy::uncompress: inLen " << inLen << ", outLen " << *outLen << std::endl; #endif - *outLen = realOutLen; + *outLen = realOutLen; - return ERR_OK; + return ERR_OK; } size_t CompressInterfaceSnappy::maxCompressedSize(size_t uncompSize) const { - return (snappy::MaxCompressedLength(uncompSize) + HEADER_SIZE); + return (snappy::MaxCompressedLength(uncompSize) + HEADER_SIZE); } -bool CompressInterfaceSnappy::getUncompressedSize(char* in, size_t inLen, - size_t* outLen) const +bool CompressInterfaceSnappy::getUncompressedSize(char* in, size_t inLen, size_t* outLen) const { - return snappy::GetUncompressedLength(in, inLen, outLen); + return snappy::GetUncompressedLength(in, inLen, outLen); } uint8_t CompressInterfaceSnappy::getChunkMagicNumber() const { - return CHUNK_MAGIC_SNAPPY; + return CHUNK_MAGIC_SNAPPY; } // LZ4 CompressInterfaceLZ4::CompressInterfaceLZ4(uint32_t numUserPaddingBytes) - : CompressInterface(numUserPaddingBytes) + : CompressInterface(numUserPaddingBytes) { } -int32_t CompressInterfaceLZ4::compress(const char* in, size_t inLen, char* out, - size_t* outLen) const +int32_t CompressInterfaceLZ4::compress(const char* in, size_t inLen, char* out, size_t* outLen) const { #ifdef HAVE_LZ4 - auto compressedLen = LZ4_compress_default(in, out, inLen, *outLen); + auto compressedLen = LZ4_compress_default(in, out, inLen, *outLen); - if (!compressedLen) - { - cerr << "LZ_compress_default failed. InLen: " << inLen - << ", compressedLen: " << compressedLen << endl; - return ERR_COMPRESS; - } - -#ifdef DEBUG_COMPRESSION - std::cout << "LZ4::compress: inLen " << inLen << ", comressedLen " - << compressedLen << std::endl; -#endif - - *outLen = compressedLen; - return ERR_OK; -#else + if (!compressedLen) + { + cerr << "LZ_compress_default failed. InLen: " << inLen << ", compressedLen: " << compressedLen << endl; return ERR_COMPRESS; + } + +#ifdef DEBUG_COMPRESSION + std::cout << "LZ4::compress: inLen " << inLen << ", comressedLen " << compressedLen << std::endl; +#endif + + *outLen = compressedLen; + return ERR_OK; +#else + return ERR_COMPRESS; #endif } -int32_t CompressInterfaceLZ4::uncompress(const char* in, size_t inLen, - char* out, size_t* outLen) const +int32_t CompressInterfaceLZ4::uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const { #ifdef HAVE_LZ4 - auto decompressedLen = LZ4_decompress_safe(in, out, inLen, *outLen); + auto decompressedLen = LZ4_decompress_safe(in, out, inLen, *outLen); - if (decompressedLen < 0) - { - cerr << "LZ_decompress_safe failed with error code " << decompressedLen - << endl; - cerr << "InLen: " << inLen << ", outLen: " << *outLen << endl; - return ERR_DECOMPRESS; - } + if (decompressedLen < 0) + { + cerr << "LZ_decompress_safe failed with error code " << decompressedLen << endl; + cerr << "InLen: " << inLen << ", outLen: " << *outLen << endl; + return ERR_DECOMPRESS; + } - *outLen = decompressedLen; + *outLen = decompressedLen; #ifdef DEBUG_COMPRESSION - std::cout << "LZ4::uncompress: inLen " << inLen << ", outLen " << *outLen - << std::endl; + std::cout << "LZ4::uncompress: inLen " << inLen << ", outLen " << *outLen << std::endl; #endif - return ERR_OK; + return ERR_OK; #else - return ERR_DECOMPRESS; + return ERR_DECOMPRESS; #endif } size_t CompressInterfaceLZ4::maxCompressedSize(size_t uncompSize) const { - return (LZ4_COMPRESSBOUND(uncompSize) + HEADER_SIZE); + return (LZ4_COMPRESSBOUND(uncompSize) + HEADER_SIZE); } -bool CompressInterfaceLZ4::getUncompressedSize(char* in, size_t inLen, - size_t* outLen) const +bool CompressInterfaceLZ4::getUncompressedSize(char* in, size_t inLen, size_t* outLen) const { - // LZ4 does not have such function. - idbassert(false); - return false; + // LZ4 does not have such function. + idbassert(false); + return false; } uint8_t CompressInterfaceLZ4::getChunkMagicNumber() const { - return CHUNK_MAGIC_LZ4; + return CHUNK_MAGIC_LZ4; } -CompressInterface* getCompressInterfaceByType(uint32_t compressionType, - uint32_t numUserPaddingBytes) +CompressInterface* getCompressInterfaceByType(uint32_t compressionType, uint32_t numUserPaddingBytes) { - switch (compressionType) - { + switch (compressionType) + { case 1: - case 2: - return new CompressInterfaceSnappy(numUserPaddingBytes); - case 3: - return new CompressInterfaceLZ4(numUserPaddingBytes); - } + case 2: return new CompressInterfaceSnappy(numUserPaddingBytes); + case 3: return new CompressInterfaceLZ4(numUserPaddingBytes); + } - return nullptr; + return nullptr; } CompressInterface* getCompressInterfaceByName(const std::string& compressionName, uint32_t numUserPaddingBytes) { - if (compressionName == "SNAPPY") - return new CompressInterfaceSnappy(numUserPaddingBytes); - else if (compressionName == "LZ4") - return new CompressInterfaceLZ4(numUserPaddingBytes); - return nullptr; + if (compressionName == "SNAPPY") + return new CompressInterfaceSnappy(numUserPaddingBytes); + else if (compressionName == "LZ4") + return new CompressInterfaceLZ4(numUserPaddingBytes); + return nullptr; } void initializeCompressorPool( - std::unordered_map>& - compressorPool, + std::unordered_map>& compressorPool, uint32_t numUserPaddingBytes) { - compressorPool = { - make_pair(2, std::shared_ptr( - new CompressInterfaceSnappy(numUserPaddingBytes))), - make_pair(3, std::shared_ptr( - new CompressInterfaceLZ4(numUserPaddingBytes)))}; + compressorPool = { + make_pair(2, std::shared_ptr(new CompressInterfaceSnappy(numUserPaddingBytes))), + make_pair(3, std::shared_ptr(new CompressInterfaceLZ4(numUserPaddingBytes)))}; } std::shared_ptr getCompressorByType( - std::unordered_map>& - compressorPool, + std::unordered_map>& compressorPool, uint32_t compressionType) { - switch (compressionType) - { + switch (compressionType) + { case 1: case 2: - if (!compressorPool.count(2)) - { - return nullptr; - } - return compressorPool[2]; + if (!compressorPool.count(2)) + { + return nullptr; + } + return compressorPool[2]; case 3: - if (!compressorPool.count(3)) - { - return nullptr; - } - return compressorPool[3]; - } + if (!compressorPool.count(3)) + { + return nullptr; + } + return compressorPool[3]; + } - return nullptr; + return nullptr; } #endif -} // namespace compress +} // namespace compress // vim:ts=4 sw=4: - diff --git a/utils/compress/idbcompress.h b/utils/compress/idbcompress.h index 23c02f966..92863c680 100644 --- a/utils/compress/idbcompress.h +++ b/utils/compress/idbcompress.h @@ -38,356 +38,332 @@ namespace compress { - typedef std::pair CompChunkPtr; typedef std::vector CompChunkPtrList; class CompressInterface { -public: - static const unsigned int HDR_BUF_LEN = 4096; - static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8; - static const uint32_t COMPRESSED_CHUNK_INCREMENT_SIZE = 8192; + public: + static const unsigned int HDR_BUF_LEN = 4096; + static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8; + static const uint32_t COMPRESSED_CHUNK_INCREMENT_SIZE = 8192; - // error codes from uncompressBlock() - static const int ERR_OK = 0; - static const int ERR_CHECKSUM = -1; - static const int ERR_DECOMPRESS = -2; - static const int ERR_BADINPUT = -3; - static const int ERR_BADOUTSIZE = -4; - static const int ERR_COMPRESS = -5; + // error codes from uncompressBlock() + static const int ERR_OK = 0; + static const int ERR_CHECKSUM = -1; + static const int ERR_DECOMPRESS = -2; + static const int ERR_BADINPUT = -3; + static const int ERR_BADOUTSIZE = -4; + static const int ERR_COMPRESS = -5; - /** - * When CompressInterface object is being used to compress a chunk, this - * construct can be used to specify the padding added by padCompressedChunks - */ - EXPORT explicit CompressInterface(unsigned int numUserPaddingBytes = 0); + /** + * When CompressInterface object is being used to compress a chunk, this + * construct can be used to specify the padding added by padCompressedChunks + */ + EXPORT explicit CompressInterface(unsigned int numUserPaddingBytes = 0); - /** - * dtor - */ - EXPORT virtual ~CompressInterface() = default; + /** + * dtor + */ + EXPORT virtual ~CompressInterface() = default; - /** - * see if the algo is available in this lib - */ - EXPORT static bool isCompressionAvail(int compressionType = 0); + /** + * see if the algo is available in this lib + */ + EXPORT static bool isCompressionAvail(int compressionType = 0); - /** - * Returns the maximum compressed size from all available compression - * types. - */ - EXPORT static size_t getMaxCompressedSizeGeneric(size_t inLen); + /** + * Returns the maximum compressed size from all available compression + * types. + */ + EXPORT static size_t getMaxCompressedSizeGeneric(size_t inLen); - /** - * Compresses specified "in" buffer of length "inLen" bytes. - * Compressed data and size are returned in "out" and "outLen". - * "out" should be sized using maxCompressedSize() to allow for incompressible data. - * Returns 0 if success. - */ + /** + * Compresses specified "in" buffer of length "inLen" bytes. + * Compressed data and size are returned in "out" and "outLen". + * "out" should be sized using maxCompressedSize() to allow for incompressible data. + * Returns 0 if success. + */ - EXPORT int compressBlock(const char* in, const size_t inLen, - unsigned char* out, size_t& outLen) const; + EXPORT int compressBlock(const char* in, const size_t inLen, unsigned char* out, size_t& outLen) const; - /** - * outLen must be initialized with the size of the out buffer before calling uncompressBlock. - * On return, outLen will have the number of bytes used in out. - */ - EXPORT int uncompressBlock(const char* in, const size_t inLen, - unsigned char* out, size_t& outLen) const; + /** + * outLen must be initialized with the size of the out buffer before calling uncompressBlock. + * On return, outLen will have the number of bytes used in out. + */ + EXPORT int uncompressBlock(const char* in, const size_t inLen, unsigned char* out, size_t& outLen) const; - /** - * This fcn wraps whatever compression algorithm we're using at the time, and - * is not specific to blocks on disk. - */ - EXPORT virtual int compress(const char* in, size_t inLen, char* out, - size_t* outLen) const = 0; + /** + * This fcn wraps whatever compression algorithm we're using at the time, and + * is not specific to blocks on disk. + */ + EXPORT virtual int compress(const char* in, size_t inLen, char* out, size_t* outLen) const = 0; - /** - * This fcn wraps whatever compression algorithm we're using at the time, and - * is not specific to blocks on disk. The caller needs to make sure out is big - * enough to contain the output by using getUncompressedSize(). - */ - EXPORT virtual int uncompress(const char* in, size_t inLen, char* out, - size_t* outLen) const = 0; + /** + * This fcn wraps whatever compression algorithm we're using at the time, and + * is not specific to blocks on disk. The caller needs to make sure out is big + * enough to contain the output by using getUncompressedSize(). + */ + EXPORT virtual int uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const = 0; - /** - * Initialize header buffer at start of compressed db file. - * - * @warning hdrBuf must be at least HDR_BUF_LEN bytes - * @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes - */ - EXPORT static void - initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht, - execplan::CalpontSystemCatalog::ColDataType columnType, - int compressionType, int hdrSize); - /** - * Initialize header buffer at start of compressed db file. - * - * @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes - */ - EXPORT static void - initHdr(void* hdrBuf, uint32_t columnWidth, - execplan::CalpontSystemCatalog::ColDataType columnType, - int compressionType); + /** + * Initialize header buffer at start of compressed db file. + * + * @warning hdrBuf must be at least HDR_BUF_LEN bytes + * @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes + */ + EXPORT static void initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht, + execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType, + int hdrSize); + /** + * Initialize header buffer at start of compressed db file. + * + * @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes + */ + EXPORT static void initHdr(void* hdrBuf, uint32_t columnWidth, + execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType); - /** - * Verify the passed in buffer contains a compressed db file header. - */ - EXPORT static int verifyHdr(const void* hdrBuf); + /** + * Verify the passed in buffer contains a compressed db file header. + */ + EXPORT static int verifyHdr(const void* hdrBuf); - /** - * Extracts list of compression pointers from the specified ptr buffer. - * ptrBuf points to the pointer section taken from the headers. - * chunkPtrs is a vector of offset, size pairs for the compressed chunks. - * Returns 0 if success. - */ - EXPORT static int getPtrList(const char* ptrBuf, const int ptrBufSize, - CompChunkPtrList& chunkPtrs); + /** + * Extracts list of compression pointers from the specified ptr buffer. + * ptrBuf points to the pointer section taken from the headers. + * chunkPtrs is a vector of offset, size pairs for the compressed chunks. + * Returns 0 if success. + */ + EXPORT static int getPtrList(const char* ptrBuf, const int ptrBufSize, CompChunkPtrList& chunkPtrs); - /** - * Extracts list of compression pointers from the specified header. - * hdrBuf points to start of 2 buffer headers from compressed db file. - * Overloaded for backward compatibility. For none dictionary columns. - * Note: the pointer passed in is the beginning of the header, - * not the pointer section as above. - */ - EXPORT static int getPtrList(const char* hdrBuf, - CompChunkPtrList& chunkPtrs); + /** + * Extracts list of compression pointers from the specified header. + * hdrBuf points to start of 2 buffer headers from compressed db file. + * Overloaded for backward compatibility. For none dictionary columns. + * Note: the pointer passed in is the beginning of the header, + * not the pointer section as above. + */ + EXPORT static int getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs); - /** - * Return the number of chunk pointers contained in the specified ptr buffer. - * ptrBuf points to the pointer section taken from the headers. - */ - EXPORT static unsigned int getPtrCount(const char* ptrBuf, - const int ptrBufSize); + /** + * Return the number of chunk pointers contained in the specified ptr buffer. + * ptrBuf points to the pointer section taken from the headers. + */ + EXPORT static unsigned int getPtrCount(const char* ptrBuf, const int ptrBufSize); - /** - * Return the number of chunk pointers contained in the specified header. - * hdrBuf points to start of 2 buffer headers from compressed db file. - * For non-dictionary columns. - */ - EXPORT static unsigned int getPtrCount(const char* hdrBuf); + /** + * Return the number of chunk pointers contained in the specified header. + * hdrBuf points to start of 2 buffer headers from compressed db file. + * For non-dictionary columns. + */ + EXPORT static unsigned int getPtrCount(const char* hdrBuf); - /** - * Store vector of pointers into the specified buffer header's pointer section. - */ - EXPORT static void storePtrs(const std::vector& ptrs, - void* hdrBuf, int ptrSectionSize); + /** + * Store vector of pointers into the specified buffer header's pointer section. + */ + EXPORT static void storePtrs(const std::vector& ptrs, void* hdrBuf, int ptrSectionSize); - /** - * Store vector of pointers into the specified buffer header. - * Overloaded for backward compatibility. For none dictionary columns. - * Note: the pointer passed in is the beginning of the header, - * not the pointer section as above. - */ - EXPORT static void storePtrs(const std::vector& ptrs, - void* hdrBuf); + /** + * Store vector of pointers into the specified buffer header. + * Overloaded for backward compatibility. For none dictionary columns. + * Note: the pointer passed in is the beginning of the header, + * not the pointer section as above. + */ + EXPORT static void storePtrs(const std::vector& ptrs, void* hdrBuf); - /** - * Calculates the chunk, and the block offset within the chunk, for the - * specified block number. - */ - EXPORT void locateBlock(unsigned int block, unsigned int& chunkIndex, - unsigned int& blockOffsetWithinChunk) const; + /** + * Calculates the chunk, and the block offset within the chunk, for the + * specified block number. + */ + EXPORT void locateBlock(unsigned int block, unsigned int& chunkIndex, + unsigned int& blockOffsetWithinChunk) const; - /** - * Pads the specified compressed chunk to the nearest compressed chunk - * increment, by padding buf with 0's, and updating len accordingly. - * maxLen is the maximum size for buf. nonzero return code means the - * result output buffer length is > than maxLen. - */ - EXPORT int padCompressedChunks(unsigned char* buf, size_t& len, - unsigned int maxLen) const; + /** + * Pads the specified compressed chunk to the nearest compressed chunk + * increment, by padding buf with 0's, and updating len accordingly. + * maxLen is the maximum size for buf. nonzero return code means the + * result output buffer length is > than maxLen. + */ + EXPORT int padCompressedChunks(unsigned char* buf, size_t& len, unsigned int maxLen) const; - /* - * Mutator methods for the block count in the file - */ - /** - * getVersionNumber - */ - EXPORT static uint64_t getVersionNumber(const void* hdrBuf); + /* + * Mutator methods for the block count in the file + */ + /** + * getVersionNumber + */ + EXPORT static uint64_t getVersionNumber(const void* hdrBuf); - /** - * setBlockCount - */ - EXPORT static void setBlockCount(void* hdrBuf, uint64_t count); + /** + * setBlockCount + */ + EXPORT static void setBlockCount(void* hdrBuf, uint64_t count); - /** - * getBlockCount - */ - EXPORT static uint64_t getBlockCount(const void* hdrBuf); + /** + * getBlockCount + */ + EXPORT static uint64_t getBlockCount(const void* hdrBuf); - /** - * getCompressionType - */ - EXPORT static uint64_t getCompressionType(const void* hdrBuf); + /** + * getCompressionType + */ + EXPORT static uint64_t getCompressionType(const void* hdrBuf); - /* - * Mutator methods for the overall header size - */ - /** - * setHdrSize - */ - EXPORT static void setHdrSize(void* hdrBuf, uint64_t size); + /* + * Mutator methods for the overall header size + */ + /** + * setHdrSize + */ + EXPORT static void setHdrSize(void* hdrBuf, uint64_t size); - /** - * getHdrSize - */ - EXPORT static uint64_t getHdrSize(const void* hdrBuf); + /** + * getHdrSize + */ + EXPORT static uint64_t getHdrSize(const void* hdrBuf); - /** - * getColumnType - */ - EXPORT static execplan::CalpontSystemCatalog::ColDataType - getColDataType(const void* hdrBuf); + /** + * getColumnType + */ + EXPORT static execplan::CalpontSystemCatalog::ColDataType getColDataType(const void* hdrBuf); - /** - * getColumnWidth - */ - EXPORT static uint64_t getColumnWidth(const void* hdrBuf); + /** + * getColumnWidth + */ + EXPORT static uint64_t getColumnWidth(const void* hdrBuf); - /** - * getLBIDByIndex - */ - EXPORT static uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index); + /** + * getLBIDByIndex + */ + EXPORT static uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index); - /** - * setLBIDByIndex - */ - EXPORT static void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index); + /** + * setLBIDByIndex + */ + EXPORT static void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index); - /** - * getLBIDCount - */ - EXPORT static uint64_t getLBIDCount(void* hdrBuf); + /** + * getLBIDCount + */ + EXPORT static uint64_t getLBIDCount(void* hdrBuf); - /** - * Mutator methods for the user padding bytes - */ - /** - * set numUserPaddingBytes - */ - EXPORT void numUserPaddingBytes(uint64_t num) - { - fNumUserPaddingBytes = num; - } + /** + * Mutator methods for the user padding bytes + */ + /** + * set numUserPaddingBytes + */ + EXPORT void numUserPaddingBytes(uint64_t num) + { + fNumUserPaddingBytes = num; + } - /** - * get numUserPaddingBytes - */ - EXPORT uint64_t numUserPaddingBytes() const - { - return fNumUserPaddingBytes; - } + /** + * get numUserPaddingBytes + */ + EXPORT uint64_t numUserPaddingBytes() const + { + return fNumUserPaddingBytes; + } - /** - * Given an input, uncompressed block, what's the maximum possible output, - * compressed size? - */ - EXPORT virtual size_t maxCompressedSize(size_t uncompSize) const = 0; + /** + * Given an input, uncompressed block, what's the maximum possible output, + * compressed size? + */ + EXPORT virtual size_t maxCompressedSize(size_t uncompSize) const = 0; - /** - * Given a compressed block, returns the uncompressed size in outLen. - * Returns false on error, true on success. - */ - EXPORT virtual bool getUncompressedSize(char* in, size_t inLen, - size_t* outLen) const = 0; + /** + * Given a compressed block, returns the uncompressed size in outLen. + * Returns false on error, true on success. + */ + EXPORT virtual bool getUncompressedSize(char* in, size_t inLen, size_t* outLen) const = 0; - protected: - virtual uint8_t getChunkMagicNumber() const = 0; + protected: + virtual uint8_t getChunkMagicNumber() const = 0; - private: - //defaults okay - //CompressInterface(const CompressInterface& rhs); - //CompressInterface& operator=(const CompressInterface& rhs); + private: + // defaults okay + // CompressInterface(const CompressInterface& rhs); + // CompressInterface& operator=(const CompressInterface& rhs); - unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks + unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks }; class CompressInterfaceSnappy : public CompressInterface { - public: - EXPORT CompressInterfaceSnappy(uint32_t numUserPaddingBytes = 0); - EXPORT ~CompressInterfaceSnappy() = default; - /** - * Compress the given block using snappy compression API. - */ - EXPORT int32_t compress(const char* in, size_t inLen, char* out, - size_t* outLen) const override; - /** - * Uncompress the given block using snappy compression API. - */ - EXPORT int32_t uncompress(const char* in, size_t inLen, char* out, - size_t* outLen) const override; - /** - * Get max compressed size for the given `uncompSize` value using snappy - * compression API. - */ - EXPORT size_t maxCompressedSize(size_t uncompSize) const override; + public: + EXPORT CompressInterfaceSnappy(uint32_t numUserPaddingBytes = 0); + EXPORT ~CompressInterfaceSnappy() = default; + /** + * Compress the given block using snappy compression API. + */ + EXPORT int32_t compress(const char* in, size_t inLen, char* out, size_t* outLen) const override; + /** + * Uncompress the given block using snappy compression API. + */ + EXPORT int32_t uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const override; + /** + * Get max compressed size for the given `uncompSize` value using snappy + * compression API. + */ + EXPORT size_t maxCompressedSize(size_t uncompSize) const override; - /** - * Get uncompressed size for the given block using snappy - * compression API. - */ - EXPORT - bool getUncompressedSize(char* in, size_t inLen, - size_t* outLen) const override; + /** + * Get uncompressed size for the given block using snappy + * compression API. + */ + EXPORT + bool getUncompressedSize(char* in, size_t inLen, size_t* outLen) const override; - protected: - uint8_t getChunkMagicNumber() const override; + protected: + uint8_t getChunkMagicNumber() const override; - private: - const uint8_t CHUNK_MAGIC_SNAPPY = 0xfd; + private: + const uint8_t CHUNK_MAGIC_SNAPPY = 0xfd; }; class CompressInterfaceLZ4 : public CompressInterface { - public: - EXPORT CompressInterfaceLZ4(uint32_t numUserPaddingBytes = 0); - EXPORT ~CompressInterfaceLZ4() = default; - /** - * Compress the given block using LZ4 compression API. - */ - EXPORT int32_t compress(const char* in, size_t inLen, char* out, - size_t* outLen) const override; - /** - * Uncompress the given block using LZ4 compression API. - */ - EXPORT int32_t uncompress(const char* in, size_t inLen, char* out, - size_t* outLen) const override; - /** - * Get max compressed size for the given `uncompSize` value using LZ4 - * compression API. - */ - EXPORT size_t maxCompressedSize(size_t uncompSize) const override; + public: + EXPORT CompressInterfaceLZ4(uint32_t numUserPaddingBytes = 0); + EXPORT ~CompressInterfaceLZ4() = default; + /** + * Compress the given block using LZ4 compression API. + */ + EXPORT int32_t compress(const char* in, size_t inLen, char* out, size_t* outLen) const override; + /** + * Uncompress the given block using LZ4 compression API. + */ + EXPORT int32_t uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const override; + /** + * Get max compressed size for the given `uncompSize` value using LZ4 + * compression API. + */ + EXPORT size_t maxCompressedSize(size_t uncompSize) const override; - /** - * Get uncompressed size for the given block using LZ4 - * compression API. - */ - EXPORT - bool getUncompressedSize(char* in, size_t inLen, - size_t* outLen) const override; + /** + * Get uncompressed size for the given block using LZ4 + * compression API. + */ + EXPORT + bool getUncompressedSize(char* in, size_t inLen, size_t* outLen) const override; - protected: - uint8_t getChunkMagicNumber() const override; + protected: + uint8_t getChunkMagicNumber() const override; - private: - const uint8_t CHUNK_MAGIC_LZ4 = 0xfc; + private: + const uint8_t CHUNK_MAGIC_LZ4 = 0xfc; }; -using CompressorPool = - std::unordered_map>; +using CompressorPool = std::unordered_map>; /** * Returns a pointer to the appropriate compression interface based on * `compressionType`. `compressionType` must be greater than 0. * Note: caller is responsible for memory deallocation. */ -EXPORT CompressInterface* -getCompressInterfaceByType(uint32_t compressionType, - uint32_t numUserPaddingBytes = 0); +EXPORT CompressInterface* getCompressInterfaceByType(uint32_t compressionType, + uint32_t numUserPaddingBytes = 0); /** * Returns a pointer to the appropriate compression interface based on @@ -401,122 +377,154 @@ EXPORT CompressInterface* getCompressInterfaceByName(const std::string& compress * Initializes a given `unordered_map` with all available compression * interfaces. */ -EXPORT void initializeCompressorPool(CompressorPool& compressorPool, - uint32_t numUserPaddingBytes = 0); +EXPORT void initializeCompressorPool(CompressorPool& compressorPool, uint32_t numUserPaddingBytes = 0); /** * Returns a `shared_ptr` to the appropriate compression interface. */ -EXPORT std::shared_ptr -getCompressorByType(CompressorPool& compressorPool, uint32_t compressionType); +EXPORT std::shared_ptr getCompressorByType(CompressorPool& compressorPool, + uint32_t compressionType); #ifdef SKIP_IDB_COMPRESSION -inline CompressInterface::CompressInterface(unsigned int /*numUserPaddingBytes*/) {} +inline CompressInterface::CompressInterface(unsigned int /*numUserPaddingBytes*/) +{ +} inline bool CompressInterface::isCompressionAvail(int c) { - return (c == 0); + return (c == 0); } inline int CompressInterface::compressBlock(const char*, const size_t, unsigned char*, size_t&) const { - return -1; + return -1; } -inline int CompressInterface::uncompressBlock(const char* in, - const size_t inLen, - unsigned char* out, +inline int CompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out, size_t& outLen) const { - return -1; + return -1; +} +inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) +{ } -inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) {} inline int CompressInterface::verifyHdr(const void*) { - return -1; + return -1; +} +inline void CompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, + int, int) +{ +} +inline void CompressInterface::initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, + int) const +{ } -inline void CompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int, int) {} -inline void CompressInterface::initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) const {} inline int CompressInterface::getPtrList(const char*, const int, CompChunkPtrList&) { - return -1; + return -1; } inline unsigned int CompressInterface::getPtrCount(const char*, const int) { - return 0; + return 0; } inline unsigned int CompressInterface::getPtrCount(const char*) { - return 0; + return 0; } -inline void CompressInterface::storePtrs(const std::vector&, void*, int) {} -inline void CompressInterface::storePtrs(const std::vector&, void*) {} -inline void -CompressInterface::locateBlock(unsigned int block, unsigned int& chunkIndex, - unsigned int& blockOffsetWithinChunk) const +inline void CompressInterface::storePtrs(const std::vector&, void*, int) { } -inline int CompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len, unsigned int maxLen) const +inline void CompressInterface::storePtrs(const std::vector&, void*) { - return -1; +} +inline void CompressInterface::locateBlock(unsigned int block, unsigned int& chunkIndex, + unsigned int& blockOffsetWithinChunk) const +{ +} +inline int CompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len, + unsigned int maxLen) const +{ + return -1; } inline uint64_t CompressInterface::getVersionNumber(const void* hdrBuf) { - return 0; + return 0; +} +inline void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count) +{ } -inline void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count) {} inline uint64_t CompressInterface::getBlockCount(const void* hdrBuf) { - return 0; + return 0; } inline uint64_t CompressInterface::getCompressionType(const void* hdrBuf) { - return 0; + return 0; } -inline execplan::CalpontSystemCatalog::ColDataType -CompressInterface::getColDataType(const void* hdrBuf) +inline execplan::CalpontSystemCatalog::ColDataType CompressInterface::getColDataType(const void* hdrBuf) { - return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED; + return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED; } inline uint64_t CompressInterface::getColumnWidth(const void* hdrBuf) const { - return 0; + return 0; +} +inline uint64_t getLBID0(const void* hdrBuf) +{ + return 0; +} +void setLBID0(void* hdrBuf, uint64_t lbid) +{ +} +inline uint64_t getLBID1(const void* hdrBuf) +{ + return 0; +} +void setLBID1(void* hdrBuf, uint64_t lbid) +{ +} +inline void CompressInterface::setHdrSize(void*, uint64_t) +{ } -inline uint64_t getLBID0(const void* hdrBuf) { return 0; } -void setLBID0(void* hdrBuf, uint64_t lbid) {} -inline uint64_t getLBID1(const void* hdrBuf) { return 0; } -void setLBID1(void* hdrBuf, uint64_t lbid) {} -inline void CompressInterface::setHdrSize(void*, uint64_t) {} inline uint64_t CompressInterface::getHdrSize(const void*) { - return 0; + return 0; } CompressInterfaceSnappy::CompressInterfaceSnappy(uint32_t numUserPaddingBytes) - : CompressInterface(numUserPaddingBytes) + : CompressInterface(numUserPaddingBytes) { } -inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const { return 0; } +inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const +{ + return 0; +} inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize) { - return uncompSize; + return uncompSize; } inline uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint32_t index) const { - return 0; + return 0; +} +void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const +{ +} +void IDBCompressInterface::getLBIDCount(void* hdrBuf) const +{ } -void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const {} -void IDBCompressInterface::getLBIDCount(void* hdrBuf) const {} inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen) { - return false; + return false; } -uint8_t getChunkMagicNumber() const { return 0; } -CompressInterface* getCompressInterfaceByType(uint32_t compressionType, - uint32_t numUserPaddingBytes) +uint8_t getChunkMagicNumber() const { - return nullptr; + return 0; +} +CompressInterface* getCompressInterfaceByType(uint32_t compressionType, uint32_t numUserPaddingBytes) +{ + return nullptr; } #endif -} +} // namespace compress #undef EXPORT #endif - diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 99691e36f..a7194ef7d 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: configcpp.cpp 3899 2013-06-17 20:54:10Z rdempsey $ -* -******************************************************************************************/ + * $Id: configcpp.cpp 3899 2013-06-17 20:54:10Z rdempsey $ + * + ******************************************************************************************/ #include "mcsconfig.h" #include @@ -79,586 +79,577 @@ boost::mutex Config::fWriteXmlLock; Config* Config::makeConfig(const string& cf) { - return makeConfig(cf.c_str()); + return makeConfig(cf.c_str()); } Config* Config::makeConfig(const char* cf) { - boost::mutex::scoped_lock lk(fInstanceMapMutex); + boost::mutex::scoped_lock lk(fInstanceMapMutex); - static string defaultFilePath; + static string defaultFilePath; - if (cf == 0 || *cf == 0) + if (cf == 0 || *cf == 0) + { + fs::path configFilePath; + configFilePath = fs::path(MCSSYSCONFDIR) / fs::path("columnstore") / defaultCalpontConfigFile; + defaultFilePath = configFilePath.string(); + + if (fInstanceMap.find(defaultFilePath) == fInstanceMap.end()) { - fs::path configFilePath; - configFilePath = fs::path(MCSSYSCONFDIR) / fs::path("columnstore") / defaultCalpontConfigFile; - defaultFilePath = configFilePath.string(); - - if (fInstanceMap.find(defaultFilePath) == fInstanceMap.end()) - { - Config* instance = new Config(defaultFilePath); - fInstanceMap[defaultFilePath] = instance; - } - - return fInstanceMap[defaultFilePath]; + Config* instance = new Config(defaultFilePath); + fInstanceMap[defaultFilePath] = instance; } - string configFile(cf); + return fInstanceMap[defaultFilePath]; + } - if (fInstanceMap.find(configFile) == fInstanceMap.end()) - { - Config* instance = new Config(configFile); - fInstanceMap[configFile] = instance; - } + string configFile(cf); - return fInstanceMap[configFile]; + if (fInstanceMap.find(configFile) == fInstanceMap.end()) + { + Config* instance = new Config(configFile); + fInstanceMap[configFile] = instance; + } + + return fInstanceMap[configFile]; } -Config::Config(const string& configFile) : - fDoc(0), fConfigFile(configFile), fMtime(0), fParser() +Config::Config(const string& configFile) : fDoc(0), fConfigFile(configFile), fMtime(0), fParser() { - int i = 0; - for ( ; i < 2 ; i++ ) - { - if (access(fConfigFile.c_str(), R_OK) == 0) - break; - sleep (1); - } + int i = 0; + for (; i < 2; i++) + { + if (access(fConfigFile.c_str(), R_OK) == 0) + break; + sleep(1); + } - if ( i == 2 ) - throw runtime_error("Config::Config: error accessing config file " + fConfigFile); + if (i == 2) + throw runtime_error("Config::Config: error accessing config file " + fConfigFile); - struct stat statbuf; + struct stat statbuf; - if (stat(configFile.c_str(), &statbuf) == 0) - fMtime = statbuf.st_mtime; + if (stat(configFile.c_str(), &statbuf) == 0) + fMtime = statbuf.st_mtime; - parseDoc(); + parseDoc(); } Config::~Config() { - if (fDoc != 0) - closeConfig(); + if (fDoc != 0) + closeConfig(); } void Config::parseDoc(void) { - struct flock fl; - int fd; + struct flock fl; + int fd; - memset(&fl, 0, sizeof(fl)); - fl.l_type = F_RDLCK; // read lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; //lock whole file + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; // lock whole file - // lock file if exist - if ((fd = open(fConfigFile.c_str(), O_RDONLY)) >= 0) + // lock file if exist + if ((fd = open(fConfigFile.c_str(), O_RDONLY)) >= 0) + { + if (fcntl(fd, F_SETLKW, &fl) != 0) { - if (fcntl(fd, F_SETLKW, &fl) != 0) - { - ostringstream oss; - oss << "Config::parseDoc: error locking file " << - fConfigFile << - ": " << - strerror(errno) << - ", proceding anyway."; - cerr << oss.str() << endl; - } - - fXmlLock.lock(); - fDoc = xmlParseFile(fConfigFile.c_str()); - fXmlLock.unlock(); - - fl.l_type = F_UNLCK; //unlock - fcntl(fd, F_SETLK, &fl); - - close(fd); - } - else - { - ostringstream oss; - oss << "Config::parseDoc: error opening file " << - fConfigFile << - ": " << - strerror(errno); - throw runtime_error(oss.str()); + ostringstream oss; + oss << "Config::parseDoc: error locking file " << fConfigFile << ": " << strerror(errno) + << ", proceding anyway."; + cerr << oss.str() << endl; } + fXmlLock.lock(); + fDoc = xmlParseFile(fConfigFile.c_str()); + fXmlLock.unlock(); - if (fDoc == 0 ) - { - throw runtime_error("Config::parseDoc: error parsing config file " + fConfigFile); - } + fl.l_type = F_UNLCK; // unlock + fcntl(fd, F_SETLK, &fl); - xmlNodePtr cur = xmlDocGetRootElement(fDoc); + close(fd); + } + else + { + ostringstream oss; + oss << "Config::parseDoc: error opening file " << fConfigFile << ": " << strerror(errno); + throw runtime_error(oss.str()); + } - if (cur == NULL) - { - xmlFreeDoc(fDoc); - fDoc = 0; - throw runtime_error("Config::parseDoc: error parsing config file " + fConfigFile); - } + if (fDoc == 0) + { + throw runtime_error("Config::parseDoc: error parsing config file " + fConfigFile); + } - if (xmlStrcmp(cur->name, (const xmlChar*)"Columnstore")) - { - xmlFreeDoc(fDoc); - fDoc = 0; - throw runtime_error("Config::parseDoc: error parsing config file " + fConfigFile); - } + xmlNodePtr cur = xmlDocGetRootElement(fDoc); - return; + if (cur == NULL) + { + xmlFreeDoc(fDoc); + fDoc = 0; + throw runtime_error("Config::parseDoc: error parsing config file " + fConfigFile); + } + + if (xmlStrcmp(cur->name, (const xmlChar*)"Columnstore")) + { + xmlFreeDoc(fDoc); + fDoc = 0; + throw runtime_error("Config::parseDoc: error parsing config file " + fConfigFile); + } + + return; } void Config::closeConfig(void) { - xmlFreeDoc(fDoc); - fDoc = 0; + xmlFreeDoc(fDoc); + fDoc = 0; } const string Config::getConfig(const string& section, const string& name) { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - if (section.length() == 0 || name.length() == 0) - throw invalid_argument("Config::getConfig: both section and name must have a length"); + if (section.length() == 0 || name.length() == 0) + throw invalid_argument("Config::getConfig: both section and name must have a length"); - if (fDoc == 0) + if (fDoc == 0) + { + throw runtime_error("Config::getConfig: no XML document!"); + } + + struct stat statbuf; + + if (stat(fConfigFile.c_str(), &statbuf) == 0) + { + if (statbuf.st_mtime != fMtime) { - throw runtime_error("Config::getConfig: no XML document!"); + closeConfig(); + fMtime = statbuf.st_mtime; + parseDoc(); } + } - struct stat statbuf; - - if (stat(fConfigFile.c_str(), &statbuf) == 0) - { - if (statbuf.st_mtime != fMtime) - { - closeConfig(); - fMtime = statbuf.st_mtime; - parseDoc(); - } - } - - return fParser.getConfig(fDoc, section, name); + return fParser.getConfig(fDoc, section, name); } void Config::getConfig(const string& section, const string& name, vector& values) { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - if (section.length() == 0) - throw invalid_argument("Config::getConfig: section must have a length"); + if (section.length() == 0) + throw invalid_argument("Config::getConfig: section must have a length"); - if (fDoc == 0) - throw runtime_error("Config::getConfig: no XML document!"); + if (fDoc == 0) + throw runtime_error("Config::getConfig: no XML document!"); - struct stat statbuf; + struct stat statbuf; - if (stat(fConfigFile.c_str(), &statbuf) == 0) + if (stat(fConfigFile.c_str(), &statbuf) == 0) + { + if (statbuf.st_mtime != fMtime) { - if (statbuf.st_mtime != fMtime) - { - closeConfig(); - fMtime = statbuf.st_mtime; - parseDoc(); - } + closeConfig(); + fMtime = statbuf.st_mtime; + parseDoc(); } + } - fParser.getConfig(fDoc, section, name, values); + fParser.getConfig(fDoc, section, name, values); } void Config::setConfig(const string& section, const string& name, const string& value) { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - if (section.length() == 0 || name.length() == 0 ) - throw invalid_argument("Config::setConfig: all of section and name must have a length"); + if (section.length() == 0 || name.length() == 0) + throw invalid_argument("Config::setConfig: all of section and name must have a length"); - if (fDoc == 0) + if (fDoc == 0) + { + throw runtime_error("Config::setConfig: no XML document!"); + } + + struct stat statbuf; + + memset(&statbuf, 0, sizeof(statbuf)); + + if (stat(fConfigFile.c_str(), &statbuf) == 0) + { + if (statbuf.st_mtime != fMtime) { - throw runtime_error("Config::setConfig: no XML document!"); + closeConfig(); + fMtime = statbuf.st_mtime; + parseDoc(); } + } - struct stat statbuf; - - memset(&statbuf, 0, sizeof(statbuf)); - - if (stat(fConfigFile.c_str(), &statbuf) == 0) - { - if (statbuf.st_mtime != fMtime) - { - closeConfig(); - fMtime = statbuf.st_mtime; - parseDoc(); - } - } - - fParser.setConfig(fDoc, section, name, value); - return; + fParser.setConfig(fDoc, section, name, value); + return; } void Config::delConfig(const string& section, const string& name) { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - if (section.length() == 0 || name.length() == 0) - throw invalid_argument("Config::delConfig: both section and name must have a length"); + if (section.length() == 0 || name.length() == 0) + throw invalid_argument("Config::delConfig: both section and name must have a length"); - if (fDoc == 0) + if (fDoc == 0) + { + throw runtime_error("Config::delConfig: no XML document!"); + } + + struct stat statbuf; + + if (stat(fConfigFile.c_str(), &statbuf) == 0) + { + if (statbuf.st_mtime != fMtime) { - throw runtime_error("Config::delConfig: no XML document!"); + closeConfig(); + fMtime = statbuf.st_mtime; + parseDoc(); } + } - struct stat statbuf; - - if (stat(fConfigFile.c_str(), &statbuf) == 0) - { - if (statbuf.st_mtime != fMtime) - { - closeConfig(); - fMtime = statbuf.st_mtime; - parseDoc(); - } - } - - fParser.delConfig(fDoc, section, name); - return; + fParser.delConfig(fDoc, section, name); + return; } void Config::writeConfig(const string& configFile) const { - boost::recursive_mutex::scoped_lock lk(fLock); - FILE* fi; + boost::recursive_mutex::scoped_lock lk(fLock); + FILE* fi; - if (fDoc == 0) - throw runtime_error("Config::writeConfig: no XML document!"); + if (fDoc == 0) + throw runtime_error("Config::writeConfig: no XML document!"); #ifdef _MSC_VER - fs::path configFilePth(configFile); - fs::path outFilePth(configFilePth); - outFilePth.replace_extension("temp"); + fs::path configFilePth(configFile); + fs::path outFilePth(configFilePth); + outFilePth.replace_extension("temp"); - if ((fi = fopen(outFilePth.string().c_str(), "wt")) == NULL) - throw runtime_error("Config::writeConfig: error opening config file for write " + outFilePth.string()); + if ((fi = fopen(outFilePth.string().c_str(), "wt")) == NULL) + throw runtime_error("Config::writeConfig: error opening config file for write " + outFilePth.string()); - int rc = -1; + int rc = -1; + rc = xmlDocDump(fi, fDoc); + + if (rc < 0) + { + throw runtime_error("Config::writeConfig: error writing config file " + outFilePth.string()); + } + + fclose(fi); + + if (fs::exists(configFilePth)) + fs::remove(configFilePth); + + fs::rename(outFilePth, configFilePth); +#else + + const fs::path defaultCalpontConfigFileTemp("Columnstore.xml.temp"); + const fs::path saveCalpontConfigFileTemp("Columnstore.xml.columnstoreSave"); + const fs::path tmpCalpontConfigFileTemp("Columnstore.xml.temp1"); + + fs::path etcdir = fs::path(MCSSYSCONFDIR) / fs::path("columnstore"); + + fs::path dcf = etcdir / fs::path(defaultCalpontConfigFile); + fs::path dcft = etcdir / fs::path(defaultCalpontConfigFileTemp); + fs::path scft = etcdir / fs::path(saveCalpontConfigFileTemp); + fs::path tcft = etcdir / fs::path(tmpCalpontConfigFileTemp); + + // perform a temp write first if Columnstore.xml file to prevent possible corruption + if (configFile == dcf) + { + if (exists(dcft)) + fs::remove(dcft); + + if ((fi = fopen(dcft.string().c_str(), "w+")) == NULL) + throw runtime_error("Config::writeConfig: error writing config file " + configFile); + + int rc; rc = xmlDocDump(fi, fDoc); if (rc < 0) { - throw runtime_error("Config::writeConfig: error writing config file " + outFilePth.string()); + throw runtime_error("Config::writeConfig: error writing config file " + configFile); + // cout << "xmlDocDump " << rc << " " << errno << endl; } fclose(fi); - if (fs::exists(configFilePth)) - fs::remove(configFilePth); - - fs::rename(outFilePth, configFilePth); -#else - - const fs::path defaultCalpontConfigFileTemp("Columnstore.xml.temp"); - const fs::path saveCalpontConfigFileTemp("Columnstore.xml.columnstoreSave"); - const fs::path tmpCalpontConfigFileTemp("Columnstore.xml.temp1"); - - fs::path etcdir = fs::path(MCSSYSCONFDIR) / fs::path("columnstore"); - - fs::path dcf = etcdir / fs::path(defaultCalpontConfigFile); - fs::path dcft = etcdir / fs::path(defaultCalpontConfigFileTemp); - fs::path scft = etcdir / fs::path(saveCalpontConfigFileTemp); - fs::path tcft = etcdir / fs::path(tmpCalpontConfigFileTemp); - - //perform a temp write first if Columnstore.xml file to prevent possible corruption - if ( configFile == dcf ) + // check temp file + try { + Config* c1 = makeConfig(dcft.string().c_str()); - if (exists(dcft)) fs::remove(dcft); + string value; + value = c1->getConfig("SystemConfig", "SystemName"); - if ((fi = fopen(dcft.string().c_str(), "w+")) == NULL) - throw runtime_error("Config::writeConfig: error writing config file " + configFile); + // good read, save copy, copy temp file tp tmp then to Columnstore.xml + // move to get around a 'same file error' in mv command + try + { + if (exists(scft)) + fs::remove(scft); + } + catch (fs::filesystem_error&) + { + } - int rc; - rc = xmlDocDump(fi, fDoc); + fs::copy_file(dcf, scft, fs::copy_option::overwrite_if_exists); - if ( rc < 0) - { - throw runtime_error("Config::writeConfig: error writing config file " + configFile); - //cout << "xmlDocDump " << rc << " " << errno << endl; - } + try + { + fs::permissions(scft, fs::add_perms | fs::owner_read | fs::owner_write | fs::group_read | + fs::group_write | fs::others_read | fs::others_write); + } + catch (fs::filesystem_error&) + { + } - fclose(fi); + if (exists(tcft)) + fs::remove(tcft); - //check temp file - try - { - Config* c1 = makeConfig(dcft.string().c_str()); + fs::rename(dcft, tcft); - string value; - value = c1->getConfig("SystemConfig", "SystemName"); + if (exists(dcf)) + fs::remove(dcf); - //good read, save copy, copy temp file tp tmp then to Columnstore.xml - //move to get around a 'same file error' in mv command - try - { - if (exists(scft)) fs::remove(scft); - } - catch (fs::filesystem_error&) { } - - fs::copy_file(dcf, scft, fs::copy_option::overwrite_if_exists); - - try - { - fs::permissions(scft, fs::add_perms | fs::owner_read | fs::owner_write | - fs::group_read | fs::group_write | - fs::others_read | fs::others_write); - } - catch (fs::filesystem_error&) { } - - if (exists(tcft)) fs::remove(tcft); - - fs::rename(dcft, tcft); - - if (exists(dcf)) fs::remove(dcf); - - fs::rename(tcft, dcf); - } - catch (...) - { - throw runtime_error("Config::writeConfig: error writing config file " + configFile); - } + fs::rename(tcft, dcf); } - else + catch (...) { - // non Columnstore.xml, perform update - if ((fi = fopen(configFile.c_str(), "w")) == NULL) - throw runtime_error("Config::writeConfig: error writing config file " + configFile); - - xmlDocDump(fi, fDoc); - - fclose(fi); + throw runtime_error("Config::writeConfig: error writing config file " + configFile); } + } + else + { + // non Columnstore.xml, perform update + if ((fi = fopen(configFile.c_str(), "w")) == NULL) + throw runtime_error("Config::writeConfig: error writing config file " + configFile); + + xmlDocDump(fi, fDoc); + + fclose(fi); + } #endif - return; + return; } void Config::write(void) const { - boost::mutex::scoped_lock lk(fWriteXmlLock); + boost::mutex::scoped_lock lk(fWriteXmlLock); #ifdef _MSC_VER - writeConfig(fConfigFile); + writeConfig(fConfigFile); #else - write(fConfigFile); + write(fConfigFile); #endif } void Config::write(const string& configFile) const { - struct flock fl; - int fd; + struct flock fl; + int fd; - fl.l_type = F_WRLCK; // write lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - fl.l_pid = getpid(); + fl.l_type = F_WRLCK; // write lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = getpid(); - // lock file if it exists - if ((fd = open(configFile.c_str(), O_WRONLY)) >= 0) + // lock file if it exists + if ((fd = open(configFile.c_str(), O_WRONLY)) >= 0) + { + if (fcntl(fd, F_SETLKW, &fl) == -1) + throw runtime_error("Config::write: file lock error " + configFile); + + try { - if (fcntl(fd, F_SETLKW, &fl) == -1) - throw runtime_error("Config::write: file lock error " + configFile); - - - try - { - writeConfig(configFile); - } - catch (...) - { - fl.l_type = F_UNLCK; //unlock - - if (fcntl(fd, F_SETLK, &fl) == -1) - throw runtime_error("Config::write: file unlock error after exception in writeConfig " + configFile); - - throw; - } - - fl.l_type = F_UNLCK; //unlock - - if (fcntl(fd, F_SETLK, &fl) == -1) - throw runtime_error("Config::write: file unlock error " + configFile); - - close(fd); + writeConfig(configFile); } - else + catch (...) { - writeConfig(configFile); + fl.l_type = F_UNLCK; // unlock + + if (fcntl(fd, F_SETLK, &fl) == -1) + throw runtime_error("Config::write: file unlock error after exception in writeConfig " + configFile); + + throw; } + + fl.l_type = F_UNLCK; // unlock + + if (fcntl(fd, F_SETLK, &fl) == -1) + throw runtime_error("Config::write: file unlock error " + configFile); + + close(fd); + } + else + { + writeConfig(configFile); + } } void Config::writeConfigFile(messageqcpp::ByteStream msg) const { - struct flock fl; - int fd; + struct flock fl; + int fd; - //get config file name being udated - string fileName; - msg >> fileName; + // get config file name being udated + string fileName; + msg >> fileName; - fl.l_type = F_WRLCK; // write lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - fl.l_pid = getpid(); + fl.l_type = F_WRLCK; // write lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = getpid(); - // lock file if it exists - if ((fd = open(fileName.c_str(), O_WRONLY)) >= 0) - { - if (fcntl(fd, F_SETLKW, &fl) == -1) - throw runtime_error("Config::write: file lock error " + fileName); + // lock file if it exists + if ((fd = open(fileName.c_str(), O_WRONLY)) >= 0) + { + if (fcntl(fd, F_SETLKW, &fl) == -1) + throw runtime_error("Config::write: file lock error " + fileName); - ofstream out(fileName.c_str()); - out << msg; + ofstream out(fileName.c_str()); + out << msg; - fl.l_type = F_UNLCK; //unlock + fl.l_type = F_UNLCK; // unlock - if (fcntl(fd, F_SETLK, &fl) == -1) - throw runtime_error("Config::write: file unlock error " + fileName); + if (fcntl(fd, F_SETLK, &fl) == -1) + throw runtime_error("Config::write: file unlock error " + fileName); - close(fd); - } - else - { - ofstream out(fileName.c_str()); - out << msg; - } + close(fd); + } + else + { + ofstream out(fileName.c_str()); + out << msg; + } } - /* static */ void Config::deleteInstanceMap() { - boost::mutex::scoped_lock lk(fInstanceMapMutex); + boost::mutex::scoped_lock lk(fInstanceMapMutex); - for (Config::configMap_t::iterator iter = fInstanceMap.begin(); - iter != fInstanceMap.end(); ++iter) - { - Config* instance = iter->second; - delete instance; - } + for (Config::configMap_t::iterator iter = fInstanceMap.begin(); iter != fInstanceMap.end(); ++iter) + { + Config* instance = iter->second; + delete instance; + } - fInstanceMap.clear(); + fInstanceMap.clear(); } /* static */ int64_t Config::fromText(const std::string& text) { - if (text.length() == 0) return 0; + if (text.length() == 0) + return 0; - int64_t val = 0; - char* ctext = static_cast(alloca(text.length() + 1)); - strcpy(ctext, text.c_str()); - char* cptr; + int64_t val = 0; + char* ctext = static_cast(alloca(text.length() + 1)); + strcpy(ctext, text.c_str()); + char* cptr; - val = strtoll(ctext, &cptr, 0); + val = strtoll(ctext, &cptr, 0); - switch (*cptr) - { - case 'T': - case 't': - val *= 1024; + switch (*cptr) + { + case 'T': + case 't': val *= 1024; - /* fallthru */ - case 'G': - case 'g': - val *= 1024; + /* fallthru */ + case 'G': + case 'g': val *= 1024; - /* fallthru */ - case 'M': - case 'm': - val *= 1024; + /* fallthru */ + case 'M': + case 'm': val *= 1024; - /* fallthru */ - case 'K': - case 'k': - val *= 1024; + /* fallthru */ + case 'K': + case 'k': val *= 1024; - /* fallthru */ - case '\0': - break; + /* fallthru */ + case '\0': break; - default: - ostringstream oss; - oss << "Invalid character '" << *cptr << "' found in numeric parameter '" << text << - "'. Since this will not do what you want it is fatal." << endl; - throw runtime_error(oss.str()); - break; - } + default: + ostringstream oss; + oss << "Invalid character '" << *cptr << "' found in numeric parameter '" << text + << "'. Since this will not do what you want it is fatal." << endl; + throw runtime_error(oss.str()); + break; + } - return val; + return val; } time_t Config::getCurrentMTime() { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - struct stat statbuf; + struct stat statbuf; - if (stat(fConfigFile.c_str(), &statbuf) == 0) - return statbuf.st_mtime; - else - return 0; + if (stat(fConfigFile.c_str(), &statbuf) == 0) + return statbuf.st_mtime; + else + return 0; } const vector Config::enumConfig() { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - if (fDoc == 0) + if (fDoc == 0) + { + throw runtime_error("Config::getConfig: no XML document!"); + } + + struct stat statbuf; + + if (stat(fConfigFile.c_str(), &statbuf) == 0) + { + if (statbuf.st_mtime != fMtime) { - throw runtime_error("Config::getConfig: no XML document!"); + closeConfig(); + fMtime = statbuf.st_mtime; + parseDoc(); } + } - struct stat statbuf; - - if (stat(fConfigFile.c_str(), &statbuf) == 0) - { - if (statbuf.st_mtime != fMtime) - { - closeConfig(); - fMtime = statbuf.st_mtime; - parseDoc(); - } - } - - return fParser.enumConfig(fDoc); + return fParser.enumConfig(fDoc); } const vector Config::enumSection(const string& section) { - boost::recursive_mutex::scoped_lock lk(fLock); + boost::recursive_mutex::scoped_lock lk(fLock); - if (fDoc == 0) + if (fDoc == 0) + { + throw runtime_error("Config::getConfig: no XML document!"); + } + + struct stat statbuf; + + if (stat(fConfigFile.c_str(), &statbuf) == 0) + { + if (statbuf.st_mtime != fMtime) { - throw runtime_error("Config::getConfig: no XML document!"); + closeConfig(); + fMtime = statbuf.st_mtime; + parseDoc(); } + } - struct stat statbuf; - - if (stat(fConfigFile.c_str(), &statbuf) == 0) - { - if (statbuf.st_mtime != fMtime) - { - closeConfig(); - fMtime = statbuf.st_mtime; - parseDoc(); - } - } - - return fParser.enumSection(fDoc, section); + return fParser.enumSection(fDoc, section); } std::string Config::getTempFileDir(Config::TempDirPurpose what) { @@ -670,15 +661,12 @@ std::string Config::getTempFileDir(Config::TempDirPurpose what) prefix.append("/"); switch (what) { - case TempDirPurpose::Joins: - return prefix.append("joins/"); - case TempDirPurpose::Aggregates: - return prefix.append("aggregates/"); + case TempDirPurpose::Joins: return prefix.append("joins/"); + case TempDirPurpose::Aggregates: return prefix.append("aggregates/"); } // NOTREACHED return {}; } -} //namespace config +} // namespace config // vim:ts=4 sw=4: - diff --git a/utils/configcpp/configcpp.h b/utils/configcpp/configcpp.h index 726287c3f..f573f292a 100644 --- a/utils/configcpp/configcpp.h +++ b/utils/configcpp/configcpp.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: configcpp.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -******************************************************************************************/ + * $Id: configcpp.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -39,7 +39,6 @@ namespace messageqcpp { - class ByteStream; } @@ -52,7 +51,6 @@ class ByteStream; namespace config { - /** @brief a config file I/F class * * This is a singleton pattern: you must use the makeConfig() factory method to get a @@ -62,191 +60,189 @@ namespace config */ class Config { -public: - /** @brief Config factory method - * - * Creates a singleton Config object - */ - EXPORT static Config* makeConfig(const char* cf = 0); + public: + /** @brief Config factory method + * + * Creates a singleton Config object + */ + EXPORT static Config* makeConfig(const char* cf = 0); - /** @brief Config factory method - * - * Creates a singleton Config object - */ - EXPORT static Config* makeConfig(const std::string& cf); + /** @brief Config factory method + * + * Creates a singleton Config object + */ + EXPORT static Config* makeConfig(const std::string& cf); - /** @brief dtor - */ - EXPORT virtual ~Config(); + /** @brief dtor + */ + EXPORT virtual ~Config(); - /** @brief get name's value from section - * - * get name's value from section in the current config file. - * @param section the name of the config file section to search - * @param name the param name whose value is to be returned - */ - EXPORT const std::string getConfig(const std::string& section, const std::string& name); + /** @brief get name's value from section + * + * get name's value from section in the current config file. + * @param section the name of the config file section to search + * @param name the param name whose value is to be returned + */ + EXPORT const std::string getConfig(const std::string& section, const std::string& name); - /** @brief get all name's values from a section - * - * get name's values from section in the current config file. - * @param section the name of the config file section to search - * @param name the param name whose value is to be returned - * @param values the values in the section are returned in this vector - */ - EXPORT void getConfig(const std::string& section, const std::string& name, - std::vector& values); + /** @brief get all name's values from a section + * + * get name's values from section in the current config file. + * @param section the name of the config file section to search + * @param name the param name whose value is to be returned + * @param values the values in the section are returned in this vector + */ + EXPORT void getConfig(const std::string& section, const std::string& name, + std::vector& values); - /** @brief set name's value in section - * - * set name's value in section in the current config file. - * @param section the name of the config file section to update - * @param name the param name whose value is to be updated - * @param value the param value - */ - EXPORT void setConfig(const std::string& section, const std::string& name, const std::string& value); + /** @brief set name's value in section + * + * set name's value in section in the current config file. + * @param section the name of the config file section to update + * @param name the param name whose value is to be updated + * @param value the param value + */ + EXPORT void setConfig(const std::string& section, const std::string& name, const std::string& value); - /** @brief delete name from section - * - * delete name from section in the current config file. - * @param section the name of the config file section to search - * @param name the param name whose entry is to be deleted - * @note if you delete the last param from a section, the section will still remain - */ - EXPORT void delConfig(const std::string& section, const std::string& name); + /** @brief delete name from section + * + * delete name from section in the current config file. + * @param section the name of the config file section to search + * @param name the param name whose entry is to be deleted + * @note if you delete the last param from a section, the section will still remain + */ + EXPORT void delConfig(const std::string& section, const std::string& name); - /** @brief write the config file back out to disk - * - * write the config file back out to disk using the current filename. - */ - EXPORT void write(void) const; + /** @brief write the config file back out to disk + * + * write the config file back out to disk using the current filename. + */ + EXPORT void write(void) const; - /** @brief write the config file back out to disk as fileName - * - * write the config file out to disk as a new file fileName. Does not affect the current - * config filename. - */ - EXPORT void write(const std::string& fileName) const; + /** @brief write the config file back out to disk as fileName + * + * write the config file out to disk as a new file fileName. Does not affect the current + * config filename. + */ + EXPORT void write(const std::string& fileName) const; - /** @brief write a stream copy of config file to disk - * - * write a stream copy of config file to disk. used to distributed mass updates to system nodes - * - */ - EXPORT void writeConfigFile(messageqcpp::ByteStream msg) const; + /** @brief write a stream copy of config file to disk + * + * write a stream copy of config file to disk. used to distributed mass updates to system nodes + * + */ + EXPORT void writeConfigFile(messageqcpp::ByteStream msg) const; - /** @brief return the name of this config file - * - * return the name of this config file. - */ - EXPORT inline const std::string& configFile() const - { - return fConfigFile; - } + /** @brief return the name of this config file + * + * return the name of this config file. + */ + EXPORT inline const std::string& configFile() const + { + return fConfigFile; + } - /** @brief delete all config file instances - * - * deletes \b all config file maps - */ - EXPORT static void deleteInstanceMap(); + /** @brief delete all config file instances + * + * deletes \b all config file maps + */ + EXPORT static void deleteInstanceMap(); - /** @brief parse config file numerics - * - * Convert human-friendly number formats to machine-friendly. Handle suffixes 'K', 'M', 'G'. - * Handle decimal, hex and octal notation in the same way as the C compiler. - * Ignore any 'B' following [KMG]. Ignore case in suffixes. - * An empty string or an unparseable string returns 0. - * Return a signed numeric value. - */ - EXPORT static int64_t fromText(const std::string& text); + /** @brief parse config file numerics + * + * Convert human-friendly number formats to machine-friendly. Handle suffixes 'K', 'M', 'G'. + * Handle decimal, hex and octal notation in the same way as the C compiler. + * Ignore any 'B' following [KMG]. Ignore case in suffixes. + * An empty string or an unparseable string returns 0. + * Return a signed numeric value. + */ + EXPORT static int64_t fromText(const std::string& text); - /** @brief parse config file numerics - * - * Return an unsigned numeric value. - */ - EXPORT static inline uint64_t uFromText(const std::string& text) - { - return static_cast(fromText(text)); - } + /** @brief parse config file numerics + * + * Return an unsigned numeric value. + */ + EXPORT static inline uint64_t uFromText(const std::string& text) + { + return static_cast(fromText(text)); + } - /** @brief Used externally to check whether there has been a config change without loading everything - * - */ - inline time_t getLastMTime() const - { - return fMtime; - } + /** @brief Used externally to check whether there has been a config change without loading everything + * + */ + inline time_t getLastMTime() const + { + return fMtime; + } - /** @brief Used externally to check whether there has been a config change without loading everything - * - */ - EXPORT time_t getCurrentMTime(); + /** @brief Used externally to check whether there has been a config change without loading everything + * + */ + EXPORT time_t getCurrentMTime(); - /** @brief Enumerate all the sections in the config file - * - */ - EXPORT const std::vector enumConfig(); + /** @brief Enumerate all the sections in the config file + * + */ + EXPORT const std::vector enumConfig(); - /** @brief Enumerate all the names in a section in the config file - * - */ - EXPORT const std::vector enumSection(const std::string& section); + /** @brief Enumerate all the names in a section in the config file + * + */ + EXPORT const std::vector enumSection(const std::string& section); - enum class TempDirPurpose - { - Joins, ///< disk joins - Aggregates ///< disk-based aggregation - }; - /** @brief Return temporaru directory path for the specified purpose */ - EXPORT std::string getTempFileDir(TempDirPurpose what); + enum class TempDirPurpose + { + Joins, ///< disk joins + Aggregates ///< disk-based aggregation + }; + /** @brief Return temporaru directory path for the specified purpose */ + EXPORT std::string getTempFileDir(TempDirPurpose what); -protected: - /** @brief parse the XML file - * - */ - void parseDoc(void); + protected: + /** @brief parse the XML file + * + */ + void parseDoc(void); - /** @brief write the XML tree to disk - * - */ - EXPORT void writeConfig(const std::string& fileName) const; + /** @brief write the XML tree to disk + * + */ + EXPORT void writeConfig(const std::string& fileName) const; - /** @brief stop processing this XML file - * - */ - void closeConfig(void); + /** @brief stop processing this XML file + * + */ + void closeConfig(void); -private: - typedef std::map configMap_t; + private: + typedef std::map configMap_t; - /* - */ - Config(const Config& rhs); - /* - */ - Config& operator=(const Config& rhs); + /* + */ + Config(const Config& rhs); + /* + */ + Config& operator=(const Config& rhs); - /** @brief ctor with config file specified - */ - Config(const std::string& configFile); + /** @brief ctor with config file specified + */ + Config(const std::string& configFile); - static configMap_t fInstanceMap; - static boost::mutex fInstanceMapMutex; - static boost::mutex fXmlLock; - static boost::mutex fWriteXmlLock; - - xmlDocPtr fDoc; - const std::string fConfigFile; - time_t fMtime; - mutable boost::recursive_mutex fLock; - XMLParser fParser; + static configMap_t fInstanceMap; + static boost::mutex fInstanceMapMutex; + static boost::mutex fXmlLock; + static boost::mutex fWriteXmlLock; + xmlDocPtr fDoc; + const std::string fConfigFile; + time_t fMtime; + mutable boost::recursive_mutex fLock; + XMLParser fParser; }; -} +} // namespace config #undef EXPORT #endif // vim:ts=4 sw=4: - diff --git a/utils/configcpp/configstream.cpp b/utils/configcpp/configstream.cpp index 2ad5aea96..059f690ed 100644 --- a/utils/configcpp/configstream.cpp +++ b/utils/configcpp/configstream.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ #include "mcsconfig.h" #include @@ -34,39 +34,34 @@ using namespace messageqcpp; namespace config { - -ConfigStream::ConfigStream(const ByteStream& bs) : - fParser() +ConfigStream::ConfigStream(const ByteStream& bs) : fParser() { - init(reinterpret_cast(bs.buf())); + init(reinterpret_cast(bs.buf())); } -ConfigStream::ConfigStream(const string& str) : - fParser() +ConfigStream::ConfigStream(const string& str) : fParser() { - init(reinterpret_cast(str.c_str())); + init(reinterpret_cast(str.c_str())); } -ConfigStream::ConfigStream(const char* cptr) : - fParser() +ConfigStream::ConfigStream(const char* cptr) : fParser() { - init(reinterpret_cast(cptr)); + init(reinterpret_cast(cptr)); } ConfigStream::~ConfigStream() { - if (fDoc != NULL) - xmlFreeDoc(fDoc); + if (fDoc != NULL) + xmlFreeDoc(fDoc); } void ConfigStream::init(const xmlChar* xp) { - fDoc = xmlParseDoc(xp); + fDoc = xmlParseDoc(xp); - if (fDoc == NULL) - throw runtime_error("ConfigStream::ConfigStream: bad XML stream"); + if (fDoc == NULL) + throw runtime_error("ConfigStream::ConfigStream: bad XML stream"); } -} //namespace +} // namespace config // vim:ts=4 sw=4: - diff --git a/utils/configcpp/configstream.h b/utils/configcpp/configstream.h index fd6ffcba5..b42c65061 100644 --- a/utils/configcpp/configstream.h +++ b/utils/configcpp/configstream.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ /** * @file */ @@ -34,35 +34,33 @@ namespace config { - /** @brief a config ByteStream I/F class * */ class ConfigStream { -public: - ConfigStream(const messageqcpp::ByteStream& bs); - ConfigStream(const std::string& str); - ConfigStream(const char* cptr); - ~ConfigStream(); + public: + ConfigStream(const messageqcpp::ByteStream& bs); + ConfigStream(const std::string& str); + ConfigStream(const char* cptr); + ~ConfigStream(); - const std::string getConfig(const std::string& section, const std::string& name) const - { - return fParser.getConfig(fDoc, section, name); - } + const std::string getConfig(const std::string& section, const std::string& name) const + { + return fParser.getConfig(fDoc, section, name); + } -private: - ConfigStream(const ConfigStream& rhs); - ConfigStream& operator=(const ConfigStream& rhs); + private: + ConfigStream(const ConfigStream& rhs); + ConfigStream& operator=(const ConfigStream& rhs); - void init(const xmlChar* xp); + void init(const xmlChar* xp); - XMLParser fParser; - xmlDocPtr fDoc; + XMLParser fParser; + xmlDocPtr fDoc; }; -} //namespace +} // namespace config #endif // vim:ts=4 sw=4: - diff --git a/utils/configcpp/md5/md32_common.h b/utils/configcpp/md5/md32_common.h index cd458f2b5..197a12b47 100644 --- a/utils/configcpp/md5/md32_common.h +++ b/utils/configcpp/md5/md32_common.h @@ -142,135 +142,146 @@ */ #undef ROTATE #ifndef PEDANTIC -# if defined(_MSC_VER) || defined(__ICC) -# define ROTATE(a,n) _lrotl(a,n) -# elif defined(__MWERKS__) -# if defined(__POWERPC__) -# define ROTATE(a,n) __rlwinm(a,n,0,31) -# elif defined(__MC68K__) +#if defined(_MSC_VER) || defined(__ICC) +#define ROTATE(a, n) _lrotl(a, n) +#elif defined(__MWERKS__) +#if defined(__POWERPC__) +#define ROTATE(a, n) __rlwinm(a, n, 0, 31) +#elif defined(__MC68K__) /* Motorola specific tweak. */ -# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) ) -# else -# define ROTATE(a,n) __rol(a,n) -# endif -# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) +#define ROTATE(a, n) (n < 24 ? __rol(a, n) : __ror(a, 32 - n)) +#else +#define ROTATE(a, n) __rol(a, n) +#endif +#elif defined(__GNUC__) && __GNUC__ >= 2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) /* * Some GNU C inline assembler templates. Note that these are * rotates by *constant* number of bits! But that's exactly * what we need here... * */ -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "roll %1,%0" \ - : "=r"(ret) \ - : "I"(n), "0"((unsigned int)(a)) \ - : "cc"); \ - ret; \ - }) -# elif defined(_ARCH_PPC) || defined(_ARCH_PPC64) || \ - defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "rlwinm %0,%1,%2,0,31" \ - : "=r"(ret) \ - : "r"(a), "I"(n)); \ - ret; \ - }) -# elif defined(__s390x__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ("rll %0,%1,%2" \ - : "=r"(ret) \ - : "r"(a), "I"(n)); \ - ret; \ - }) -# endif -# endif +#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) +#define ROTATE(a, n) \ + ( \ + { \ + register unsigned int ret; \ + asm("roll %1,%0" : "=r"(ret) : "I"(n), "0"((unsigned int)(a)) : "cc"); \ + ret; \ + }) +#elif defined(_ARCH_PPC) || defined(_ARCH_PPC64) || defined(__powerpc) || defined(__ppc__) || \ + defined(__powerpc64__) +#define ROTATE(a, n) \ + ( \ + { \ + register unsigned int ret; \ + asm("rlwinm %0,%1,%2,0,31" : "=r"(ret) : "r"(a), "I"(n)); \ + ret; \ + }) +#elif defined(__s390x__) +#define ROTATE(a, n) \ + ( \ + { \ + register unsigned int ret; \ + asm("rll %0,%1,%2" : "=r"(ret) : "r"(a), "I"(n)); \ + ret; \ + }) +#endif +#endif #endif /* PEDANTIC */ #ifndef ROTATE -#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) +#define ROTATE(a, n) (((a) << (n)) | (((a)&0xffffffff) >> (32 - (n)))) #endif #if defined(DATA_ORDER_IS_BIG_ENDIAN) #ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if ((defined(__i386) || defined(__i386__)) && !defined(I386_ONLY)) || \ - (defined(__x86_64) || defined(__x86_64__)) -# if !defined(B_ENDIAN) +#if defined(__GNUC__) && __GNUC__ >= 2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) +#if ((defined(__i386) || defined(__i386__)) && !defined(I386_ONLY)) || \ + (defined(__x86_64) || defined(__x86_64__)) +#if !defined(B_ENDIAN) /* * This gives ~30-40% performance improvement in SHA-256 compiled * with gcc [on P4]. Well, first macro to be frank. We can pull * this trick on x86* platforms only, because these CPUs can fetch * unaligned data without raising an exception. */ -# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - (c)+=4; (l)=r; }) -# define HOST_l2c(l,c) ({ unsigned int r=(l); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - *((unsigned int *)(c))=r; (c)+=4; r; }) -# endif -# endif -# endif +#define HOST_c2l(c, l) \ + ( \ + { \ + unsigned int r = *((const unsigned int*)(c)); \ + asm("bswapl %0" : "=r"(r) : "0"(r)); \ + (c) += 4; \ + (l) = r; \ + }) +#define HOST_l2c(l, c) \ + ( \ + { \ + unsigned int r = (l); \ + asm("bswapl %0" : "=r"(r) : "0"(r)); \ + *((unsigned int*)(c)) = r; \ + (c) += 4; \ + r; \ + }) +#endif +#endif +#endif #endif #if defined(__s390__) || defined(__s390x__) -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, (l)) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, (l)) +#define HOST_c2l(c, l) ((l) = *((const unsigned int*)(c)), (c) += 4, (l)) +#define HOST_l2c(l, c) (*((unsigned int*)(c)) = (l), (c) += 4, (l)) #endif #ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++))) ), \ - l) +#define HOST_c2l(c, l) \ + (l = (((unsigned long)(*((c)++))) << 24), l |= (((unsigned long)(*((c)++))) << 16), \ + l |= (((unsigned long)(*((c)++))) << 8), l |= (((unsigned long)(*((c)++)))), l) #endif #ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff), \ - l) +#define HOST_l2c(l, c) \ + (*((c)++) = (unsigned char)(((l) >> 24) & 0xff), *((c)++) = (unsigned char)(((l) >> 16) & 0xff), \ + *((c)++) = (unsigned char)(((l) >> 8) & 0xff), *((c)++) = (unsigned char)(((l)) & 0xff), l) #endif #elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) #ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__s390x__) -# define HOST_c2l(c,l) ({ asm ("lrv %0,%1" \ - :"=d"(l) :"m"(*(const unsigned int *)(c)));\ - (c)+=4; (l); }) -# define HOST_l2c(l,c) ({ asm ("strv %1,%0" \ - :"=m"(*(unsigned int *)(c)) :"d"(l));\ - (c)+=4; (l); }) -# endif -# endif +#if defined(__GNUC__) && __GNUC__ >= 2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) +#if defined(__s390x__) +#define HOST_c2l(c, l) \ + ( \ + { \ + asm("lrv %0,%1" : "=d"(l) : "m"(*(const unsigned int*)(c))); \ + (c) += 4; \ + (l); \ + }) +#define HOST_l2c(l, c) \ + ( \ + { \ + asm("strv %1,%0" : "=m"(*(unsigned int*)(c)) : "d"(l)); \ + (c) += 4; \ + (l); \ + }) +#endif +#endif #endif #if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) -# ifndef B_ENDIAN +#ifndef B_ENDIAN /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) -# endif +#define HOST_c2l(c, l) ((l) = *((const unsigned int*)(c)), (c) += 4, l) +#define HOST_l2c(l, c) (*((unsigned int*)(c)) = (l), (c) += 4, l) +#endif #endif #ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<<24), \ - l) +#define HOST_c2l(c, l) \ + (l = (((unsigned long)(*((c)++)))), l |= (((unsigned long)(*((c)++))) << 8), \ + l |= (((unsigned long)(*((c)++))) << 16), l |= (((unsigned long)(*((c)++))) << 24), l) #endif #ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24)&0xff), \ - l) +#define HOST_l2c(l, c) \ + (*((c)++) = (unsigned char)(((l)) & 0xff), *((c)++) = (unsigned char)(((l) >> 8) & 0xff), \ + *((c)++) = (unsigned char)(((l) >> 16) & 0xff), *((c)++) = (unsigned char)(((l) >> 24) & 0xff), l) #endif #endif @@ -279,113 +290,112 @@ * Time for some action:-) */ -int HASH_UPDATE (HASH_CTX* c, const void* data_, size_t len) +int HASH_UPDATE(HASH_CTX* c, const void* data_, size_t len) { - const unsigned char* data = data_; - unsigned char* p; - HASH_LONG l; - size_t n; - - if (len == 0) return 1; - - l = (c->Nl + (((HASH_LONG)len) << 3)) & 0xffffffffUL; - - /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to - * Wei Dai for pointing it out. */ - if (l < c->Nl) /* overflow */ - c->Nh++; - - c->Nh += (HASH_LONG)(len >> 29); /* might cause compiler warning on 16-bit */ - c->Nl = l; - - n = c->num; - - if (n != 0) - { - p = (unsigned char*)c->data; - - if (len >= HASH_CBLOCK || len + n >= HASH_CBLOCK) - { - memcpy (p + n, data, HASH_CBLOCK - n); - HASH_BLOCK_DATA_ORDER (c, p, 1); - n = HASH_CBLOCK - n; - data += n; - len -= n; - c->num = 0; - memset (p, 0, HASH_CBLOCK); /* keep it zeroed */ - } - else - { - memcpy (p + n, data, len); - c->num += (unsigned int)len; - return 1; - } - } - - n = len / HASH_CBLOCK; - - if (n > 0) - { - HASH_BLOCK_DATA_ORDER (c, data, n); - n *= HASH_CBLOCK; - data += n; - len -= n; - } - - if (len != 0) - { - p = (unsigned char*)c->data; - c->num = (unsigned int)len; - memcpy (p, data, len); - } + const unsigned char* data = data_; + unsigned char* p; + HASH_LONG l; + size_t n; + if (len == 0) return 1; -} + l = (c->Nl + (((HASH_LONG)len) << 3)) & 0xffffffffUL; -void HASH_TRANSFORM (HASH_CTX* c, const unsigned char* data) -{ - HASH_BLOCK_DATA_ORDER (c, data, 1); -} + /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to + * Wei Dai for pointing it out. */ + if (l < c->Nl) /* overflow */ + c->Nh++; + c->Nh += (HASH_LONG)(len >> 29); /* might cause compiler warning on 16-bit */ + c->Nl = l; -int HASH_FINAL (unsigned char* md, HASH_CTX* c) -{ - unsigned char* p = (unsigned char*)c->data; - size_t n = c->num; + n = c->num; - p[n] = 0x80; /* there is always room for one */ - n++; + if (n != 0) + { + p = (unsigned char*)c->data; - if (n > (HASH_CBLOCK - 8)) + if (len >= HASH_CBLOCK || len + n >= HASH_CBLOCK) { - memset (p + n, 0, HASH_CBLOCK - n); - n = 0; - HASH_BLOCK_DATA_ORDER (c, p, 1); + memcpy(p + n, data, HASH_CBLOCK - n); + HASH_BLOCK_DATA_ORDER(c, p, 1); + n = HASH_CBLOCK - n; + data += n; + len -= n; + c->num = 0; + memset(p, 0, HASH_CBLOCK); /* keep it zeroed */ } + else + { + memcpy(p + n, data, len); + c->num += (unsigned int)len; + return 1; + } + } - memset (p + n, 0, HASH_CBLOCK - 8 - n); + n = len / HASH_CBLOCK; - p += HASH_CBLOCK - 8; -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - (void)HOST_l2c(c->Nh, p); - (void)HOST_l2c(c->Nl, p); + if (n > 0) + { + HASH_BLOCK_DATA_ORDER(c, data, n); + n *= HASH_CBLOCK; + data += n; + len -= n; + } + + if (len != 0) + { + p = (unsigned char*)c->data; + c->num = (unsigned int)len; + memcpy(p, data, len); + } + + return 1; +} + +void HASH_TRANSFORM(HASH_CTX* c, const unsigned char* data) +{ + HASH_BLOCK_DATA_ORDER(c, data, 1); +} + +int HASH_FINAL(unsigned char* md, HASH_CTX* c) +{ + unsigned char* p = (unsigned char*)c->data; + size_t n = c->num; + + p[n] = 0x80; /* there is always room for one */ + n++; + + if (n > (HASH_CBLOCK - 8)) + { + memset(p + n, 0, HASH_CBLOCK - n); + n = 0; + HASH_BLOCK_DATA_ORDER(c, p, 1); + } + + memset(p + n, 0, HASH_CBLOCK - 8 - n); + + p += HASH_CBLOCK - 8; +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + (void)HOST_l2c(c->Nh, p); + (void)HOST_l2c(c->Nl, p); #elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - (void)HOST_l2c(c->Nl, p); - (void)HOST_l2c(c->Nh, p); + (void)HOST_l2c(c->Nl, p); + (void)HOST_l2c(c->Nh, p); #endif - p -= HASH_CBLOCK; - HASH_BLOCK_DATA_ORDER (c, p, 1); - c->num = 0; - memset (p, 0, HASH_CBLOCK); + p -= HASH_CBLOCK; + HASH_BLOCK_DATA_ORDER(c, p, 1); + c->num = 0; + memset(p, 0, HASH_CBLOCK); #ifndef HASH_MAKE_STRING #error "HASH_MAKE_STRING must be defined!" #else - HASH_MAKE_STRING(c, md); + HASH_MAKE_STRING(c, md); #endif - return 1; + return 1; } #ifndef MD32_REG_T diff --git a/utils/configcpp/md5/md5.h b/utils/configcpp/md5/md5.h index b12bf1a18..77418469c 100644 --- a/utils/configcpp/md5/md5.h +++ b/utils/configcpp/md5/md5.h @@ -61,20 +61,21 @@ #include -#ifdef __cplusplus -extern "C" { +#ifdef __cplusplus +extern "C" +{ #endif #ifdef OPENSSL_NO_MD5 #error MD5 is disabled. #endif -/* - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - * ! MD5_LONG has to be at least 32 bits wide. If it's wider, then ! - * ! MD5_LONG_LOG2 has to be defined along. ! - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - */ + /* + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + * ! MD5_LONG has to be at least 32 bits wide. If it's wider, then ! + * ! MD5_LONG_LOG2 has to be defined along. ! + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + */ #if defined(__LP32__) #define MD5_LONG unsigned long @@ -92,24 +93,24 @@ extern "C" { #define MD5_LONG unsigned int #endif -#define MD5_CBLOCK 64 -#define MD5_LBLOCK (MD5_CBLOCK/4) +#define MD5_CBLOCK 64 +#define MD5_LBLOCK (MD5_CBLOCK / 4) #define MD5_DIGEST_LENGTH 16 -typedef struct MD5state_st -{ + typedef struct MD5state_st + { MD5_LONG A, B, C, D; MD5_LONG Nl, Nh; MD5_LONG data[MD5_LBLOCK]; unsigned int num; -} MD5_CTX; + } MD5_CTX; -int MD5_Init(MD5_CTX* c); -int MD5_Update(MD5_CTX* c, const void* data, size_t len); -int MD5_Final(unsigned char* md, MD5_CTX* c); -unsigned char* MD5(const unsigned char* d, size_t n, unsigned char* md); -void MD5_Transform(MD5_CTX* c, const unsigned char* b); -#ifdef __cplusplus + int MD5_Init(MD5_CTX* c); + int MD5_Update(MD5_CTX* c, const void* data, size_t len); + int MD5_Final(unsigned char* md, MD5_CTX* c); + unsigned char* MD5(const unsigned char* d, size_t n, unsigned char* md); + void MD5_Transform(MD5_CTX* c, const unsigned char* b); +#ifdef __cplusplus } #endif diff --git a/utils/configcpp/md5/md5_locl.h b/utils/configcpp/md5/md5_locl.h index c26c39fc3..f42074a39 100644 --- a/utils/configcpp/md5/md5_locl.h +++ b/utils/configcpp/md5/md5_locl.h @@ -68,32 +68,38 @@ #endif #ifdef MD5_ASM -# if defined(__i386) || defined(__i386__) || defined(_M_IX86) || defined(__INTEL__) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64) -# define md5_block_data_order md5_block_asm_data_order -# elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64) -# define md5_block_data_order md5_block_asm_data_order -# endif +#if defined(__i386) || defined(__i386__) || defined(_M_IX86) || defined(__INTEL__) || defined(__x86_64) || \ + defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64) +#define md5_block_data_order md5_block_asm_data_order +#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64) +#define md5_block_data_order md5_block_asm_data_order +#endif #endif -void md5_block_data_order (MD5_CTX* c, const void* p, size_t num); +void md5_block_data_order(MD5_CTX* c, const void* p, size_t num); #define DATA_ORDER_IS_LITTLE_ENDIAN -#define HASH_LONG MD5_LONG -#define HASH_CTX MD5_CTX -#define HASH_CBLOCK MD5_CBLOCK -#define HASH_UPDATE MD5_Update -#define HASH_TRANSFORM MD5_Transform -#define HASH_FINAL MD5_Final -#define HASH_MAKE_STRING(c,s) do { \ - unsigned long ll; \ - ll=(c)->A; HOST_l2c(ll,(s)); \ - ll=(c)->B; HOST_l2c(ll,(s)); \ - ll=(c)->C; HOST_l2c(ll,(s)); \ - ll=(c)->D; HOST_l2c(ll,(s)); \ - } while (0) -#define HASH_BLOCK_DATA_ORDER md5_block_data_order +#define HASH_LONG MD5_LONG +#define HASH_CTX MD5_CTX +#define HASH_CBLOCK MD5_CBLOCK +#define HASH_UPDATE MD5_Update +#define HASH_TRANSFORM MD5_Transform +#define HASH_FINAL MD5_Final +#define HASH_MAKE_STRING(c, s) \ + do \ + { \ + unsigned long ll; \ + ll = (c)->A; \ + HOST_l2c(ll, (s)); \ + ll = (c)->B; \ + HOST_l2c(ll, (s)); \ + ll = (c)->C; \ + HOST_l2c(ll, (s)); \ + ll = (c)->D; \ + HOST_l2c(ll, (s)); \ + } while (0) +#define HASH_BLOCK_DATA_ORDER md5_block_data_order #include "md32_common.h" @@ -106,27 +112,35 @@ void md5_block_data_order (MD5_CTX* c, const void* p, size_t num); * simplified to the code below. Wei attributes these optimizations * to Peter Gutmann's SHS code, and he attributes it to Rich Schroeppel. */ -#define F(b,c,d) ((((c) ^ (d)) & (b)) ^ (d)) -#define G(b,c,d) ((((b) ^ (c)) & (d)) ^ (c)) -#define H(b,c,d) ((b) ^ (c) ^ (d)) -#define I(b,c,d) (((~(d)) | (b)) ^ (c)) +#define F(b, c, d) ((((c) ^ (d)) & (b)) ^ (d)) +#define G(b, c, d) ((((b) ^ (c)) & (d)) ^ (c)) +#define H(b, c, d) ((b) ^ (c) ^ (d)) +#define I(b, c, d) (((~(d)) | (b)) ^ (c)) -#define R0(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+F((b),(c),(d))); \ - a=ROTATE(a,s); \ - a+=b; };\ +#define R0(a, b, c, d, k, s, t) \ + { \ + a += ((k) + (t) + F((b), (c), (d))); \ + a = ROTATE(a, s); \ + a += b; \ + }; -#define R1(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+G((b),(c),(d))); \ - a=ROTATE(a,s); \ - a+=b; }; +#define R1(a, b, c, d, k, s, t) \ + { \ + a += ((k) + (t) + G((b), (c), (d))); \ + a = ROTATE(a, s); \ + a += b; \ + }; -#define R2(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+H((b),(c),(d))); \ - a=ROTATE(a,s); \ - a+=b; }; +#define R2(a, b, c, d, k, s, t) \ + { \ + a += ((k) + (t) + H((b), (c), (d))); \ + a = ROTATE(a, s); \ + a += b; \ + }; -#define R3(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+I((b),(c),(d))); \ - a=ROTATE(a,s); \ - a+=b; }; +#define R3(a, b, c, d, k, s, t) \ + { \ + a += ((k) + (t) + I((b), (c), (d))); \ + a = ROTATE(a, s); \ + a += b; \ + }; diff --git a/utils/configcpp/resource.h b/utils/configcpp/resource.h index 95ce1419e..148f1ec8e 100644 --- a/utils/configcpp/resource.h +++ b/utils/configcpp/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/utils/configcpp/tdriver.cpp b/utils/configcpp/tdriver.cpp index 229c595b3..62d5e5948 100644 --- a/utils/configcpp/tdriver.cpp +++ b/utils/configcpp/tdriver.cpp @@ -39,340 +39,333 @@ using namespace config; class ConfigFileTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ConfigFileTest); - CPPUNIT_TEST_SUITE( ConfigFileTest ); + CPPUNIT_TEST(test1); - CPPUNIT_TEST( test1 ); + CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error); + CPPUNIT_TEST(test3); + CPPUNIT_TEST(test4); + CPPUNIT_TEST_EXCEPTION(test5, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(test6, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(test7, std::invalid_argument); + CPPUNIT_TEST_EXCEPTION(test8, std::invalid_argument); + CPPUNIT_TEST(test9); + CPPUNIT_TEST(test10); + CPPUNIT_TEST(test11); + CPPUNIT_TEST(test12); + CPPUNIT_TEST_EXCEPTION(test13_1, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(test13_2, std::runtime_error); + CPPUNIT_TEST(test14); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_EXCEPTION( test2, std::runtime_error ); - CPPUNIT_TEST( test3 ); - CPPUNIT_TEST( test4 ); - CPPUNIT_TEST_EXCEPTION( test5, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( test6, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( test7, std::invalid_argument ); - CPPUNIT_TEST_EXCEPTION( test8, std::invalid_argument ); - CPPUNIT_TEST( test9 ); - CPPUNIT_TEST( test10 ); - CPPUNIT_TEST( test11 ); - CPPUNIT_TEST( test12 ); - CPPUNIT_TEST_EXCEPTION( test13_1, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( test13_2, std::runtime_error ); - CPPUNIT_TEST( test14 ); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: -public: - void setUp() + void tearDown() + { + } + + void test1() + { + Config* c1 = Config::makeConfig("./Columnstore.xml"); + string value; + value = c1->getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); + + value = c1->getConfig("Message", "xName"); + CPPUNIT_ASSERT(value.size() == 0); + Config::deleteInstanceMap(); + } + + void test2() + { + Config* c1 = Config::makeConfig("./yadayada.xml"); + string value; + + value = c1->getConfig("Message", "Name"); + CPPUNIT_ASSERT(value.size() == 0); + + value = c1->getConfig("Message", "xName"); + CPPUNIT_ASSERT(value.size() == 0); + + Config::deleteInstanceMap(); + } + + void test3() + { + Config* c1; + string value; + + for (int i = 0; i < 1000; i++) { + c1 = Config::makeConfig("./Columnstore.xml"); + value = c1->getConfig("Message", "Name"); + assert(value == "Message"); } - void tearDown() + Config::deleteInstanceMap(); + } + + void test4() + { + Config* c1 = Config::makeConfig("./Columnstore.xml"); + string value; + + value = c1->getConfig("SystemConfig", "SystemVersion"); + c1->setConfig("SystemConfig", "SystemVersion", "2.2.versionversionversion"); + value = c1->getConfig("SystemConfig", "SystemVersion"); + CPPUNIT_ASSERT(value == "2.2.versionversionversion"); + + ::unlink("./Columnstore.xml.new"); + c1->write("./Columnstore.xml.new"); + + value = c1->getConfig("SystemConfig", "SystemVersion"); + CPPUNIT_ASSERT(value == "2.2.versionversionversion"); + + c1->setConfig("SystemConfig", "SystemVersion1", "V1.x"); + value = c1->getConfig("SystemConfig", "SystemVersion1"); + CPPUNIT_ASSERT(value == "V1.x"); + + c1->setConfig("SystemConfig1", "SystemVersion1", "Vx.x"); + value = c1->getConfig("SystemConfig1", "SystemVersion1"); + CPPUNIT_ASSERT(value == "Vx.x"); + + c1->write("./Columnstore.xml.new"); + Config* c2 = Config::makeConfig("./Columnstore.xml.new"); + value = c2->getConfig("SystemConfig1", "SystemVersion1"); + CPPUNIT_ASSERT(value == "Vx.x"); + c2->setConfig("SystemConfig", "SystemVersion1", "V1.1"); + value = c2->getConfig("SystemConfig", "SystemVersion1"); + CPPUNIT_ASSERT(value == "V1.1"); + c2->write(); + Config::deleteInstanceMap(); + } + + void test5() + { + Config* c1 = Config::makeConfig("./Columnstore.xml"); + c1->write("/cantwritethis"); + Config::deleteInstanceMap(); + } + + void test6() + { + Config* c1 = Config::makeConfig("./XColumnstore.xml"); + // compiler warning...we won't actually get here + c1 = 0; + } + + void test7() + { + Config* c1 = Config::makeConfig("./Columnstore.xml"); + string s; + string n; + string v; + c1->setConfig(s, n, v); + Config::deleteInstanceMap(); + } + + void test8() + { + Config* c1 = Config::makeConfig("./Columnstore.xml"); + string s; + string n; + string v; + v = c1->getConfig(s, n); + Config::deleteInstanceMap(); + } + + void test9() + { + string value; + + Config* c1 = Config::makeConfig("./Columnstore.xml"); + Config* c2 = Config::makeConfig("./Columnstore.xml.new"); + + value = c1->getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); + + value = c2->getConfig("SystemConfig", "SystemVersion1"); + CPPUNIT_ASSERT(value == "V1.1"); + Config::deleteInstanceMap(); + } + + void test10() + { + string value; + + setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); + Config* c1 = Config::makeConfig(); + + value = c1->getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); + Config::deleteInstanceMap(); + } + + void test11() + { + string value; + struct stat stat_buf; + struct utimbuf utime_buf; + + CPPUNIT_ASSERT(stat("./Columnstore.xml.new", &stat_buf) == 0); + + Config* c1 = Config::makeConfig("./Columnstore.xml.new"); + + value = c1->getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); + + utime_buf.actime = utime_buf.modtime = stat_buf.st_mtime + 1; + CPPUNIT_ASSERT(utime("./Columnstore.xml.new", &utime_buf) == 0); + + value = c1->getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); + Config::deleteInstanceMap(); + } + + void test12() + { + string value; + int64_t ival; + uint64_t uval; + + value = "10"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == 10); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == 10); + + value = "0x10"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == 0x10); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == 0x10); + + value = "010"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == 010); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == 010); + + value = "-10"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == -10); + + value = "10K"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024)); + + value = "10k"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024)); + + value = "10M"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); + + value = "10m"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); + + value = "10G"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL)); + + value = "10g"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL)); + + value = "10MB"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); + + value = "0x7afafafafafafafa"; + ival = Config::fromText(value); + CPPUNIT_ASSERT(ival == 0x7afafafafafafafaLL); + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == 0x7afafafafafafafaULL); + + value = "-0x7afafafafafafafa"; + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == 0x8505050505050506ULL); + + value = "-1"; + uval = Config::uFromText(value); + CPPUNIT_ASSERT(uval == 0xffffffffffffffffULL); + } + + void test13_1() + { + string value; + int64_t ival; + + value = "2.2MB"; // invalid char causes throw + ival = Config::fromText(value); + } + + void test13_2() + { + string value; + int64_t ival; + + value = "10,000"; // invalid char causes throw + ival = Config::fromText(value); + } + + void test14() + { + ByteStream bs; + ifstream ifs("./Columnstore.xml"); + ifs >> bs; + string id("."); + string value; { + ConfigStream cs(bs, id); + value = cs.getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); } - - void test1() + string bss(reinterpret_cast(bs.buf()), bs.length()); { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string value; - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - - value = c1->getConfig("Message", "xName"); - CPPUNIT_ASSERT(value.size() == 0); - Config::deleteInstanceMap(); + ConfigStream cs(bss, id); + value = cs.getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); } - - void test2() { - Config* c1 = Config::makeConfig("./yadayada.xml"); - string value; - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value.size() == 0); - - value = c1->getConfig("Message", "xName"); - CPPUNIT_ASSERT(value.size() == 0); - - Config::deleteInstanceMap(); + ConfigStream cs(bss.c_str(), id); + value = cs.getConfig("Message", "Name"); + CPPUNIT_ASSERT(value == "Message"); } - - void test3() - { - Config* c1; - string value; - - for (int i = 0; i < 1000; i++) - { - c1 = Config::makeConfig("./Columnstore.xml"); - value = c1->getConfig("Message", "Name"); - assert(value == "Message"); - } - - Config::deleteInstanceMap(); - } - - void test4() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string value; - - value = c1->getConfig("SystemConfig", "SystemVersion"); - c1->setConfig("SystemConfig", "SystemVersion", "2.2.versionversionversion"); - value = c1->getConfig("SystemConfig", "SystemVersion"); - CPPUNIT_ASSERT(value == "2.2.versionversionversion"); - - ::unlink("./Columnstore.xml.new"); - c1->write("./Columnstore.xml.new"); - - value = c1->getConfig("SystemConfig", "SystemVersion"); - CPPUNIT_ASSERT(value == "2.2.versionversionversion"); - - c1->setConfig("SystemConfig", "SystemVersion1", "V1.x"); - value = c1->getConfig("SystemConfig", "SystemVersion1"); - CPPUNIT_ASSERT(value == "V1.x"); - - c1->setConfig("SystemConfig1", "SystemVersion1", "Vx.x"); - value = c1->getConfig("SystemConfig1", "SystemVersion1"); - CPPUNIT_ASSERT(value == "Vx.x"); - - c1->write("./Columnstore.xml.new"); - Config* c2 = Config::makeConfig("./Columnstore.xml.new"); - value = c2->getConfig("SystemConfig1", "SystemVersion1"); - CPPUNIT_ASSERT(value == "Vx.x"); - c2->setConfig("SystemConfig", "SystemVersion1", "V1.1"); - value = c2->getConfig("SystemConfig", "SystemVersion1"); - CPPUNIT_ASSERT(value == "V1.1"); - c2->write(); - Config::deleteInstanceMap(); - } - - void test5() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - c1->write("/cantwritethis"); - Config::deleteInstanceMap(); - } - - void test6() - { - Config* c1 = Config::makeConfig("./XColumnstore.xml"); - // compiler warning...we won't actually get here - c1 = 0; - } - - void test7() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string s; - string n; - string v; - c1->setConfig(s, n, v); - Config::deleteInstanceMap(); - } - - void test8() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string s; - string n; - string v; - v = c1->getConfig(s, n); - Config::deleteInstanceMap(); - } - - void test9() - { - string value; - - Config* c1 = Config::makeConfig("./Columnstore.xml"); - Config* c2 = Config::makeConfig("./Columnstore.xml.new"); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - - value = c2->getConfig("SystemConfig", "SystemVersion1"); - CPPUNIT_ASSERT(value == "V1.1"); - Config::deleteInstanceMap(); - } - - void test10() - { - string value; - - setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); - Config* c1 = Config::makeConfig(); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - Config::deleteInstanceMap(); - - } - - void test11() - { - string value; - struct stat stat_buf; - struct utimbuf utime_buf; - - CPPUNIT_ASSERT(stat("./Columnstore.xml.new", &stat_buf) == 0); - - Config* c1 = Config::makeConfig("./Columnstore.xml.new"); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - - utime_buf.actime = utime_buf.modtime = stat_buf.st_mtime + 1; - CPPUNIT_ASSERT(utime("./Columnstore.xml.new", &utime_buf) == 0); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - Config::deleteInstanceMap(); - } - - void test12() - { - string value; - int64_t ival; - uint64_t uval; - - value = "10"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 10); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 10); - - value = "0x10"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 0x10); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0x10); - - value = "010"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 010); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 010); - - value = "-10"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == -10); - - value = "10K"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024)); - - value = "10k"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024)); - - value = "10M"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); - - value = "10m"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); - - value = "10G"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL)); - - value = "10g"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL)); - - value = "10MB"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); - - value = "0x7afafafafafafafa"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 0x7afafafafafafafaLL); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0x7afafafafafafafaULL); - - value = "-0x7afafafafafafafa"; - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0x8505050505050506ULL); - - value = "-1"; - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0xffffffffffffffffULL); - - } - - void test13_1() - { - string value; - int64_t ival; - - value = "2.2MB"; //invalid char causes throw - ival = Config::fromText(value); - } - - void test13_2() - { - string value; - int64_t ival; - - value = "10,000"; //invalid char causes throw - ival = Config::fromText(value); - } - - void test14() - { - ByteStream bs; - ifstream ifs("./Columnstore.xml"); - ifs >> bs; - string id("."); - string value; - { - ConfigStream cs(bs, id); - value = cs.getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - } - string bss(reinterpret_cast(bs.buf()), bs.length()); - { - ConfigStream cs(bss, id); - value = cs.getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - } - { - ConfigStream cs(bss.c_str(), id); - value = cs.getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - } - } - + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ConfigFileTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(ConfigFileTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - - diff --git a/utils/configcpp/tdriver2.cpp b/utils/configcpp/tdriver2.cpp index 57a33a85f..e652ac045 100644 --- a/utils/configcpp/tdriver2.cpp +++ b/utils/configcpp/tdriver2.cpp @@ -40,92 +40,88 @@ const string cf("./woparms.dat"); class WOConfigFileTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(WOConfigFileTest); - CPPUNIT_TEST_SUITE( WOConfigFileTest ); + CPPUNIT_TEST(test1); + CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(test3, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(test4, std::runtime_error); + CPPUNIT_TEST(test5); - CPPUNIT_TEST( test1 ); - CPPUNIT_TEST_EXCEPTION( test2, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( test3, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( test4, std::runtime_error ); - CPPUNIT_TEST( test5 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + unlink(cf.c_str()); + } -private: + void tearDown() + { + unlink(cf.c_str()); + } -public: - void setUp() - { - unlink(cf.c_str()); - } + void test1() + { + WriteOnceConfig woc(cf); + CPPUNIT_ASSERT(woc.owns("PrimitiveServers", "LBID_Shift")); + CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount")); + CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot")); - void tearDown() - { - unlink(cf.c_str()); - } + CPPUNIT_ASSERT(!woc.owns("dummy", "dummy")); - void test1() - { - WriteOnceConfig woc(cf); - CPPUNIT_ASSERT(woc.owns("PrimitiveServers", "LBID_Shift")); - CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount")); - CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot")); + int vali; - CPPUNIT_ASSERT(!woc.owns("dummy", "dummy")); + vali = Config::fromText(woc.getConfig("PrimitiveServers", "LBID_Shift")); + CPPUNIT_ASSERT(vali == 13); - int vali; + woc.setConfig("SystemConfig", "DBRootCount", "10"); + vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount")); + CPPUNIT_ASSERT(vali == 10); - vali = Config::fromText(woc.getConfig("PrimitiveServers", "LBID_Shift")); - CPPUNIT_ASSERT(vali == 13); + WriteOnceConfig woc2(cf.c_str()); + vali = Config::fromText(woc2.getConfig("SystemConfig", "DBRootCount")); + CPPUNIT_ASSERT(vali == 10); + } - woc.setConfig("SystemConfig", "DBRootCount", "10"); - vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount")); - CPPUNIT_ASSERT(vali == 10); + void test2() + { + WriteOnceConfig woc(cf); + woc.getConfig("dummy", "dummy"); + } - WriteOnceConfig woc2(cf.c_str()); - vali = Config::fromText(woc2.getConfig("SystemConfig", "DBRootCount")); - CPPUNIT_ASSERT(vali == 10); - } + void test3() + { + WriteOnceConfig woc(cf); + woc.setConfig("dummy", "dummy", "100"); + } - void test2() - { - WriteOnceConfig woc(cf); - woc.getConfig("dummy", "dummy"); - } - - void test3() - { - WriteOnceConfig woc(cf); - woc.setConfig("dummy", "dummy", "100"); - } - - void test4() - { - WriteOnceConfig woc(cf); - woc.setConfig("SystemConfig", "DBRootCount", "10"); - woc.setConfig("SystemConfig", "DBRootCount", "11"); - } - - void test5() - { - WriteOnceConfig woc(cf); - woc.setConfig("SystemConfig", "DBRootCount", "10"); - woc.setConfig("SystemConfig", "DBRootCount", "11", true); - } + void test4() + { + WriteOnceConfig woc(cf); + woc.setConfig("SystemConfig", "DBRootCount", "10"); + woc.setConfig("SystemConfig", "DBRootCount", "11"); + } + void test5() + { + WriteOnceConfig woc(cf); + woc.setConfig("SystemConfig", "DBRootCount", "10"); + woc.setConfig("SystemConfig", "DBRootCount", "11", true); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( WOConfigFileTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(WOConfigFileTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - diff --git a/utils/configcpp/writeonce.h b/utils/configcpp/writeonce.h index 8ee1e7364..93231a853 100644 --- a/utils/configcpp/writeonce.h +++ b/utils/configcpp/writeonce.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: writeonce.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -******************************************************************************************/ + * $Id: writeonce.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -38,107 +38,106 @@ namespace config { - /** @brief a write-once config file I/F class * * This class handles write-once config items */ class WriteOnceConfig { -public: - /** @brief ctor - * - */ - explicit WriteOnceConfig(const char* cf = 0); + public: + /** @brief ctor + * + */ + explicit WriteOnceConfig(const char* cf = 0); - /** @brief ctor - * - */ - explicit WriteOnceConfig(const std::string& cf) : - fConfigFileName(cf) - { - setup(); - } + /** @brief ctor + * + */ + explicit WriteOnceConfig(const std::string& cf) : fConfigFileName(cf) + { + setup(); + } - /** @brief dtor - * - */ - virtual ~WriteOnceConfig() {} + /** @brief dtor + * + */ + virtual ~WriteOnceConfig() + { + } - /** @brief check if this class owns parm - * - */ - bool owns(const std::string& section, const std::string& name) const - { - return (fEntryMap.find(std::string(section + "." + name)) != fEntryMap.end()); - } + /** @brief check if this class owns parm + * + */ + bool owns(const std::string& section, const std::string& name) const + { + return (fEntryMap.find(std::string(section + "." + name)) != fEntryMap.end()); + } - /** @brief set parm to value - * - * If you attempt to set a value more than once, and force is false, this will throw a runtime_error. - */ - void setConfig(const std::string& section, const std::string& name, const std::string& value, bool force = false); + /** @brief set parm to value + * + * If you attempt to set a value more than once, and force is false, this will throw a runtime_error. + */ + void setConfig(const std::string& section, const std::string& name, const std::string& value, + bool force = false); - /** @brief get value of parm - * - */ - const std::string getConfig(const std::string& section, const std::string& name) const; + /** @brief get value of parm + * + */ + const std::string getConfig(const std::string& section, const std::string& name) const; -protected: - /** @brief load from file - * - */ - messageqcpp::ByteStream load(); + protected: + /** @brief load from file + * + */ + messageqcpp::ByteStream load(); - /** @brief save to file - * - */ - void save(messageqcpp::ByteStream& ibs) const; + /** @brief save to file + * + */ + void save(messageqcpp::ByteStream& ibs) const; - /** @brief serialize to ByteStream - * - */ - virtual void serialize(messageqcpp::ByteStream& obs) const; + /** @brief serialize to ByteStream + * + */ + virtual void serialize(messageqcpp::ByteStream& obs) const; - /** @brief load from ByteStream - * - */ - virtual void unserialize(messageqcpp::ByteStream& ibs); + /** @brief load from ByteStream + * + */ + virtual void unserialize(messageqcpp::ByteStream& ibs); -private: - typedef std::pair ConfigItem_t; - typedef std::tr1::unordered_map EntryMap_t; + private: + typedef std::pair ConfigItem_t; + typedef std::tr1::unordered_map EntryMap_t; - static const uint32_t WriteOnceConfigVersion = 1; + static const uint32_t WriteOnceConfigVersion = 1; - //defaults okay - //WriteOnceConfig(const WriteOnceConfig& rhs); - //WriteOnceConfig& operator=(const WriteOnceConfig& rhs); + // defaults okay + // WriteOnceConfig(const WriteOnceConfig& rhs); + // WriteOnceConfig& operator=(const WriteOnceConfig& rhs); - /** @brief ctor helper - * - */ - void setup(); + /** @brief ctor helper + * + */ + void setup(); - /** @brief setup defaults when file doesn't exist - * - */ - void initializeDefaults(); + /** @brief setup defaults when file doesn't exist + * + */ + void initializeDefaults(); - EntryMap_t fEntryMap; + EntryMap_t fEntryMap; - std::string fConfigFileName; - - ConfigItem_t fLBID_Shift; - ConfigItem_t fDBRootCount; - ConfigItem_t fDBRMRoot; - ConfigItem_t fSharedMemoryTmpFile1; - ConfigItem_t fTxnIDFile; - ConfigItem_t fSharedMemoryTmpFile2; + std::string fConfigFileName; + ConfigItem_t fLBID_Shift; + ConfigItem_t fDBRootCount; + ConfigItem_t fDBRMRoot; + ConfigItem_t fSharedMemoryTmpFile1; + ConfigItem_t fTxnIDFile; + ConfigItem_t fSharedMemoryTmpFile2; }; -} +} // namespace config #endif - diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index 7b16af9d5..03da1f959 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ #include "mcsconfig.h" #include @@ -33,257 +33,256 @@ using namespace std; namespace config { - const string XMLParser::getConfig(const xmlDocPtr doc, const string& section, const string& name) const { - string res; - xmlNodePtr cur1 = xmlDocGetRootElement(doc); + string res; + xmlNodePtr cur1 = xmlDocGetRootElement(doc); - if (cur1 == NULL) - throw runtime_error("XMLParser::getConfig: error accessing XML root"); + if (cur1 == NULL) + throw runtime_error("XMLParser::getConfig: error accessing XML root"); - cur1 = cur1->xmlChildrenNode; + cur1 = cur1->xmlChildrenNode; - while (cur1 != NULL) + while (cur1 != NULL) + { + string cur1name = (const char*)cur1->name; + + if ((boost::iequals(cur1name, section))) { - string cur1name = (const char*)cur1->name; + xmlNodePtr cur2 = cur1->xmlChildrenNode; - if ((boost::iequals(cur1name, section))) + while (cur2 != NULL) + { + string cur2name = (const char*)cur2->name; + + if ((boost::iequals(cur2name, name))) { - xmlNodePtr cur2 = cur1->xmlChildrenNode; + xmlNodePtr cur3 = cur2->xmlChildrenNode; - while (cur2 != NULL) - { - string cur2name = (const char*)cur2->name; + if (cur3) + res = (const char*)cur3->content; - if ((boost::iequals(cur2name, name))) - { - xmlNodePtr cur3 = cur2->xmlChildrenNode; - - if (cur3) - res = (const char*)cur3->content; - - return res; - } - - cur2 = cur2->next; - } + return res; } - cur1 = cur1->next; + cur2 = cur2->next; + } } - // maybe nullstr if not found - return res; + cur1 = cur1->next; + } + + // maybe nullstr if not found + return res; } -void XMLParser::getConfig(const xmlDocPtr doc, const string& section, const string& name, vector& values) const +void XMLParser::getConfig(const xmlDocPtr doc, const string& section, const string& name, + vector& values) const { - string res; + string res; - xmlNodePtr cur1 = xmlDocGetRootElement(doc); + xmlNodePtr cur1 = xmlDocGetRootElement(doc); - if (cur1 == NULL) - throw runtime_error("XMLParser::getConfig: error accessing XML root"); + if (cur1 == NULL) + throw runtime_error("XMLParser::getConfig: error accessing XML root"); - cur1 = cur1->xmlChildrenNode; + cur1 = cur1->xmlChildrenNode; - while (cur1 != NULL) + while (cur1 != NULL) + { + string cur1name = (const char*)cur1->name; + + if ((boost::iequals(cur1name, section))) { - string cur1name = (const char*)cur1->name; + xmlNodePtr cur2 = cur1->xmlChildrenNode; - if ((boost::iequals(cur1name, section))) + while (cur2 != NULL) + { + string cur2name = (const char*)cur2->name; + + if ((boost::iequals(cur2name, name))) { - xmlNodePtr cur2 = cur1->xmlChildrenNode; + res.clear(); + xmlNodePtr cur3 = cur2->xmlChildrenNode; - while (cur2 != NULL) - { - string cur2name = (const char*)cur2->name; + if (cur3) + res = (const char*)cur3->content; - if ((boost::iequals(cur2name, name))) - { - res.clear(); - xmlNodePtr cur3 = cur2->xmlChildrenNode; - - if (cur3) - res = (const char*)cur3->content; - - values.push_back(res); - } - - cur2 = cur2->next; - } + values.push_back(res); } - cur1 = cur1->next; + cur2 = cur2->next; + } } + + cur1 = cur1->next; + } } void XMLParser::setConfig(xmlDocPtr doc, const string& section, const string& name, const string& value) { - xmlNodePtr cur1 = xmlDocGetRootElement(doc); + xmlNodePtr cur1 = xmlDocGetRootElement(doc); - if (cur1 == NULL) - throw runtime_error("XMLParser::setConfig: error accessing XML root"); + if (cur1 == NULL) + throw runtime_error("XMLParser::setConfig: error accessing XML root"); - xmlNodePtr cur2; + xmlNodePtr cur2; - cur1 = cur1->xmlChildrenNode; + cur1 = cur1->xmlChildrenNode; - while (cur1 != NULL) + while (cur1 != NULL) + { + string cur1name = (const char*)cur1->name; + + if (boost::iequals(cur1name, section)) { - string cur1name = (const char*)cur1->name; + cur2 = cur1->xmlChildrenNode; - if (boost::iequals(cur1name, section)) + while (cur2 != NULL) + { + string cur2name = (const char*)cur2->name; + + if (boost::iequals(cur2name, name)) { - cur2 = cur1->xmlChildrenNode; + xmlNodePtr cur3 = cur2->xmlChildrenNode; - while (cur2 != NULL) - { - string cur2name = (const char*)cur2->name; + if (cur3 == NULL) + { + xmlAddChild(cur2, xmlNewText((const xmlChar*)"\t")); + cur3 = cur2->xmlChildrenNode; + } + else + { + xmlFree(cur3->content); + } - if (boost::iequals(cur2name, name)) - { - xmlNodePtr cur3 = cur2->xmlChildrenNode; - - if (cur3 == NULL) - { - xmlAddChild(cur2, xmlNewText((const xmlChar*)"\t")); - cur3 = cur2->xmlChildrenNode; - } - else - { - xmlFree(cur3->content); - } - - cur3->content = xmlStrdup((const xmlChar*)value.c_str()); - return; - } - - cur2 = cur2->next; - } - - // We found the section, but not the name, so we need to add a new node here - xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t")); - xmlNewTextChild(cur1, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str()); - xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n\t")); - return; + cur3->content = xmlStrdup((const xmlChar*)value.c_str()); + return; } - cur1 = cur1->next; + cur2 = cur2->next; + } + + // We found the section, but not the name, so we need to add a new node here + xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t")); + xmlNewTextChild(cur1, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str()); + xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n\t")); + return; } - // We did not find the section, so we need to add it and the name here - cur1 = xmlDocGetRootElement(doc); - xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t")); - cur2 = xmlNewChild(cur1, NULL, (const xmlChar*)section.c_str(), NULL); - xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t\t")); - xmlNewTextChild(cur2, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str()); - xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t")); - xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n")); + cur1 = cur1->next; + } - return; + // We did not find the section, so we need to add it and the name here + cur1 = xmlDocGetRootElement(doc); + xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t")); + cur2 = xmlNewChild(cur1, NULL, (const xmlChar*)section.c_str(), NULL); + xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t\t")); + xmlNewTextChild(cur2, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str()); + xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t")); + xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n")); + + return; } void XMLParser::delConfig(xmlDocPtr doc, const string& section, const string& name) { - string res; + string res; - xmlNodePtr cur1 = xmlDocGetRootElement(doc); + xmlNodePtr cur1 = xmlDocGetRootElement(doc); - if (cur1 == NULL) - throw runtime_error("XMLParser::delConfig: error accessing XML root"); + if (cur1 == NULL) + throw runtime_error("XMLParser::delConfig: error accessing XML root"); - cur1 = cur1->xmlChildrenNode; + cur1 = cur1->xmlChildrenNode; - while (cur1 != NULL) + while (cur1 != NULL) + { + string cur1name = (const char*)cur1->name; + + if ((boost::iequals(cur1name, section))) { - string cur1name = (const char*)cur1->name; + xmlNodePtr cur2 = cur1->xmlChildrenNode; - if ((boost::iequals(cur1name, section))) + while (cur2 != NULL) + { + xmlNodePtr tmp = cur2; + cur2 = cur2->next; + string tmpname = (const char*)tmp->name; + + if ((boost::iequals(tmpname, name))) { - xmlNodePtr cur2 = cur1->xmlChildrenNode; - - while (cur2 != NULL) - { - xmlNodePtr tmp = cur2; - cur2 = cur2->next; - string tmpname = (const char*)tmp->name; - - if ((boost::iequals(tmpname, name))) - { - xmlUnlinkNode(tmp); - xmlFreeNode(tmp); - } - } + xmlUnlinkNode(tmp); + xmlFreeNode(tmp); } - - cur1 = cur1->next; + } } - return; + cur1 = cur1->next; + } + + return; } const vector XMLParser::enumConfig(const xmlDocPtr doc) const { - vector resv; - string res; + vector resv; + string res; - xmlNodePtr cur1 = xmlDocGetRootElement(doc); + xmlNodePtr cur1 = xmlDocGetRootElement(doc); - if (cur1 == NULL) - throw runtime_error("XMLParser::getConfig: error accessing XML root"); + if (cur1 == NULL) + throw runtime_error("XMLParser::getConfig: error accessing XML root"); - cur1 = cur1->xmlChildrenNode; + cur1 = cur1->xmlChildrenNode; - while (cur1 != NULL) - { - res = reinterpret_cast(cur1->name); + while (cur1 != NULL) + { + res = reinterpret_cast(cur1->name); - if (res != "text" && res != "comment") - resv.push_back(res); + if (res != "text" && res != "comment") + resv.push_back(res); - cur1 = cur1->next; - } + cur1 = cur1->next; + } - return resv; + return resv; } const vector XMLParser::enumSection(const xmlDocPtr doc, const string& section) const { - vector resv; - string res; + vector resv; + string res; - xmlNodePtr cur1 = xmlDocGetRootElement(doc); + xmlNodePtr cur1 = xmlDocGetRootElement(doc); - if (cur1 == NULL) - throw runtime_error("XMLParser::getConfig: error accessing XML root"); + if (cur1 == NULL) + throw runtime_error("XMLParser::getConfig: error accessing XML root"); - cur1 = cur1->xmlChildrenNode; + cur1 = cur1->xmlChildrenNode; - while (cur1 != NULL) + while (cur1 != NULL) + { + if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str()))) { - if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str()))) - { - xmlNodePtr cur2 = cur1->xmlChildrenNode; + xmlNodePtr cur2 = cur1->xmlChildrenNode; - while (cur2 != NULL) - { - res = reinterpret_cast(cur2->name); + while (cur2 != NULL) + { + res = reinterpret_cast(cur2->name); - if (res != "text" && res != "comment") - resv.push_back(res); + if (res != "text" && res != "comment") + resv.push_back(res); - cur2 = cur2->next; - } - } - - cur1 = cur1->next; + cur2 = cur2->next; + } } - return resv; + cur1 = cur1->next; + } + + return resv; } -} //namespace +} // namespace config // vim:ts=4 sw=4: - diff --git a/utils/configcpp/xmlparser.h b/utils/configcpp/xmlparser.h index 35fec9f21..48467d249 100644 --- a/utils/configcpp/xmlparser.h +++ b/utils/configcpp/xmlparser.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ /** * @file */ @@ -31,38 +31,38 @@ namespace config { - /** class XMLParser */ class XMLParser { -public: - XMLParser() { } - ~XMLParser() { } + public: + XMLParser() + { + } + ~XMLParser() + { + } - const std::string getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name) const; + const std::string getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name) const; - void getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name, - std::vector& values) const; + void getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name, + std::vector& values) const; - void setConfig(xmlDocPtr doc, const std::string& section, const std::string& name, - const std::string& value); + void setConfig(xmlDocPtr doc, const std::string& section, const std::string& name, + const std::string& value); - void delConfig(xmlDocPtr doc, const std::string& section, const std::string& name); + void delConfig(xmlDocPtr doc, const std::string& section, const std::string& name); - const std::vector enumConfig(const xmlDocPtr doc) const; - - const std::vector enumSection(const xmlDocPtr doc, const std::string& section) const; - -private: - //defaults okay - //XMLParser(const XMLParser& rhs); - //XMLParser& operator=(const XMLParser& rhs); + const std::vector enumConfig(const xmlDocPtr doc) const; + const std::vector enumSection(const xmlDocPtr doc, const std::string& section) const; + private: + // defaults okay + // XMLParser(const XMLParser& rhs); + // XMLParser& operator=(const XMLParser& rhs); }; -} //namespace +} // namespace config #endif // vim:ts=4 sw=4: - diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 27332ca01..5d5e9d346 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: dataconvert.cpp 3901 2013-06-17 20:59:13Z rdempsey $ -* -* -****************************************************************************/ + * $Id: dataconvert.cpp 3901 2013-06-17 20:59:13Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -49,215 +49,206 @@ typedef uint32_t ulong; using namespace logging; - namespace { - -const int64_t columnstore_precision[19] = -{ - 0, - 9, - 99, - 999, - 9999, - 99999, - 999999, - 9999999, - 99999999, - 999999999, - 9999999999LL, - 99999999999LL, - 999999999999LL, - 9999999999999LL, - 99999999999999LL, - 999999999999999LL, - 9999999999999999LL, - 99999999999999999LL, - 999999999999999999LL -}; +const int64_t columnstore_precision[19] = {0, + 9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999LL, + 99999999999LL, + 999999999999LL, + 9999999999999LL, + 99999999999999LL, + 999999999999999LL, + 9999999999999999LL, + 99999999999999999LL, + 999999999999999999LL}; template -bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) +bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&)) { - std::istringstream iss(s); - return !(iss >> f >> t).fail(); + std::istringstream iss(s); + return !(iss >> f >> t).fail(); } -bool number_value ( const string& data ) +bool number_value(const string& data) { - for (unsigned int i = 0; i < strlen(data.c_str()); i++) + for (unsigned int i = 0; i < strlen(data.c_str()); i++) + { + if (data[i] > '9' || data[i] < '0') { - if (data[i] > '9' || data[i] < '0') - { - if (data[i] != '+' && data[i] != '-' && data[i] != '.' && data[i] != ' ' && - data[i] != 'E' && data[i] != 'e') - { - throw QueryDataExcept("value is not numerical.", formatErr); - } - } + if (data[i] != '+' && data[i] != '-' && data[i] != '.' && data[i] != ' ' && data[i] != 'E' && + data[i] != 'e') + { + throw QueryDataExcept("value is not numerical.", formatErr); + } } + } - return true; + return true; } -} // namespace anon +} // namespace namespace dataconvert { - // LE stands for Little Endian uint32_t getUInt32LE(const char* ptr) { - return reinterpret_cast(ptr)[0]; + return reinterpret_cast(ptr)[0]; } int32_t getSInt32LE(const char* ptr) { - return reinterpret_cast(ptr)[0]; + return reinterpret_cast(ptr)[0]; } uint64_t getUInt64LE(const char* ptr) { - return reinterpret_cast(ptr)[0]; + return reinterpret_cast(ptr)[0]; } int64_t getSInt64LE(const char* ptr) { - return reinterpret_cast(ptr)[0]; + return reinterpret_cast(ptr)[0]; } template -void number_int_value(const string& data, - cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd& ct, - bool& pushwarning, - bool noRoundup, - T& intVal, bool* saturate) +void number_int_value(const string& data, cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, + bool noRoundup, T& intVal, bool* saturate) { - // copy of the original input - string valStr(data); + // copy of the original input + string valStr(data); - // in case, the values are in parentheses - string::size_type x = valStr.find('('); - string::size_type y = valStr.find(')'); + // in case, the values are in parentheses + string::size_type x = valStr.find('('); + string::size_type y = valStr.find(')'); - while (x < string::npos) + while (x < string::npos) + { + // erase y first + if (y == string::npos) + throw QueryDataExcept("'(' is not matched.", formatErr); + + valStr.erase(y, 1); + valStr.erase(x, 1); + x = valStr.find('('); + y = valStr.find(')'); + } + + if (y != string::npos) + throw QueryDataExcept("')' is not matched.", formatErr); + + if (boost::iequals(valStr, "true")) + { + intVal = 1; + return; + } + if (boost::iequals(valStr, "false")) + { + intVal = 0; + return; + } + + // convert to fixed-point notation if input is in scientific notation + if (valStr.find('E') < string::npos || valStr.find('e') < string::npos) + { + size_t epos = valStr.find('E'); + + if (epos == string::npos) + epos = valStr.find('e'); + + // get the coefficient + string coef = valStr.substr(0, epos); + // get the exponent + string exp = valStr.substr(epos + 1); + bool overflow = false; + T exponent = dataconvert::string_to_ll(exp, overflow); + + // if the exponent can not be held in 64 or 128 bits, not supported or saturated. + if (overflow) + throw QueryDataExcept("value is invalid.", formatErr); + + // find the optional "." point + size_t dpos = coef.find('.'); + + if (dpos != string::npos) { - // erase y first - if (y == string::npos) - throw QueryDataExcept("'(' is not matched.", formatErr); - - valStr.erase(y, 1); - valStr.erase(x, 1); - x = valStr.find('('); - y = valStr.find(')'); + // move "." to the end by mutiply 10 ** (# of fraction digits) + coef.erase(dpos, 1); + exponent -= coef.length() - dpos; } - if (y != string::npos) - throw QueryDataExcept("')' is not matched.", formatErr); - - if (boost::iequals(valStr, "true")) + if (exponent >= 0) { - intVal = 1; - return; + coef.resize(coef.length() + exponent, '0'); } - if (boost::iequals(valStr, "false")) + else { - intVal = 0; - return; + size_t bpos = coef.find_first_of("0123456789"); + size_t epos = coef.length(); + size_t mpos = -exponent; + dpos = epos - mpos; + int64_t padding = (int64_t)mpos - (int64_t)(epos - bpos); + + if (padding > 0) + { + coef.insert(bpos, padding, '0'); + dpos = bpos; + } + + coef.insert(dpos, "."); } - // convert to fixed-point notation if input is in scientific notation - if (valStr.find('E') < string::npos || valStr.find('e') < string::npos) + valStr = coef; + } + + // apply the scale + if (ct.scale != 0) + { + uint64_t scale = (uint64_t)(ct.scale < 0) ? (-ct.scale) : (ct.scale); + size_t dpos = valStr.find('.'); + string intPart = valStr.substr(0, dpos); + string leftStr; + + if (ct.scale > 0) { - size_t epos = valStr.find('E'); + if (dpos != string::npos) + { + // decimal point exist, prepare "#scale" digits in fraction part + ++dpos; + string frnStr = valStr.substr(dpos, scale); - if (epos == string::npos) - epos = valStr.find('e'); + if (frnStr.length() < scale) + frnStr.resize(scale, '0'); // padding digit 0, not null. - // get the coefficient - string coef = valStr.substr(0, epos); - // get the exponent - string exp = valStr.substr(epos + 1); - bool overflow = false; - T exponent = dataconvert::string_to_ll(exp, overflow); - - // if the exponent can not be held in 64 or 128 bits, not supported or saturated. - if (overflow) - throw QueryDataExcept("value is invalid.", formatErr); - - // find the optional "." point - size_t dpos = coef.find('.'); - - if (dpos != string::npos) - { - // move "." to the end by mutiply 10 ** (# of fraction digits) - coef.erase(dpos, 1); - exponent -= coef.length() - dpos; - } - - if (exponent >= 0) - { - coef.resize(coef.length() + exponent, '0'); - } - else - { - size_t bpos = coef.find_first_of("0123456789"); - size_t epos = coef.length(); - size_t mpos = -exponent; - dpos = epos - mpos; - int64_t padding = (int64_t)mpos - (int64_t)(epos - bpos); - - if (padding > 0) - { - coef.insert(bpos, padding, '0'); - dpos = bpos; - } - - coef.insert(dpos, "."); - } - - valStr = coef; + // effectly shift "#scale" digits to left. + intPart += frnStr; + leftStr = valStr.substr(dpos); + leftStr.erase(0, scale); + } + else + { + // no decimal point, shift "#scale" digits to left. + intPart.resize(intPart.length() + scale, '0'); // padding digit 0, not null. + } } - - // apply the scale - if (ct.scale != 0) + else // if (ct.scale < 0) -- in ct.scale != 0 block { - uint64_t scale = (uint64_t) (ct.scale < 0) ? (-ct.scale) : (ct.scale); - size_t dpos = valStr.find('.'); - string intPart = valStr.substr(0, dpos); - string leftStr; - - if (ct.scale > 0) - { - if (dpos != string::npos) - { - // decimal point exist, prepare "#scale" digits in fraction part - ++dpos; - string frnStr = valStr.substr(dpos, scale); - - if (frnStr.length() < scale) - frnStr.resize(scale, '0'); // padding digit 0, not null. - - // effectly shift "#scale" digits to left. - intPart += frnStr; - leftStr = valStr.substr(dpos); - leftStr.erase(0, scale); - } - else - { - // no decimal point, shift "#scale" digits to left. - intPart.resize(intPart.length() + scale, '0'); // padding digit 0, not null. - } - } - else // if (ct.scale < 0) -- in ct.scale != 0 block - { - if (dpos != string::npos) - { - // decimal point exist, get the fraction part - ++dpos; - leftStr = valStr.substr(dpos); - } + if (dpos != string::npos) + { + // decimal point exist, get the fraction part + ++dpos; + leftStr = valStr.substr(dpos); + } // add above to keep the old behavior, to comply with tdriver // uncomment code below to support negative scale @@ -276,447 +267,434 @@ void number_int_value(const string& data, leftStr = intPart.substr(intPart.length() - scale) + leftStr; intPart.erase(intPart.length() - scale, scale); #endif - } - - valStr = intPart; - - if (leftStr.length() > 0) - valStr += "." + leftStr; } - // now, convert to long long int - string intStr(valStr); - string frnStr = ""; - size_t dp = valStr.find('.'); - int roundup = 0; + valStr = intPart; - if (dp != string::npos) + if (leftStr.length() > 0) + valStr += "." + leftStr; + } + + // now, convert to long long int + string intStr(valStr); + string frnStr = ""; + size_t dp = valStr.find('.'); + int roundup = 0; + + if (dp != string::npos) + { + // Check if need round up + int frac1 = dataconvert::string_to_ll(valStr.substr(dp + 1, 1), pushwarning); + + if ((!noRoundup) && frac1 >= 5) + roundup = 1; + + intStr.erase(dp); + frnStr = valStr.substr(dp + 1); + + if (intStr.length() == 0) + intStr = "0"; + else if ((intStr.length() == 1) && ((intStr[0] == '+') || (intStr[0] == '-'))) { - //Check if need round up - int frac1 = dataconvert::string_to_ll(valStr.substr(dp + 1, 1), pushwarning); - - if ((!noRoundup) && frac1 >= 5) - roundup = 1; - - intStr.erase(dp); - frnStr = valStr.substr(dp + 1); - - if ( intStr.length() == 0 ) - intStr = "0"; - else if (( intStr.length() == 1 ) && ( (intStr[0] == '+') || (intStr[0] == '-') ) ) - { - intStr.insert( 1, 1, '0'); - } + intStr.insert(1, 1, '0'); } + } - intVal = dataconvert::string_to_ll(intStr, pushwarning); - //@Bug 3350 negative value round up. - intVal += intVal >= 0 ? roundup : -roundup; - bool dummy = false; - T frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ll(frnStr, dummy) : 0; + intVal = dataconvert::string_to_ll(intStr, pushwarning); + //@Bug 3350 negative value round up. + intVal += intVal >= 0 ? roundup : -roundup; + bool dummy = false; + T frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ll(frnStr, dummy) : 0; - if (frnVal != 0) + if (frnVal != 0) + pushwarning = true; + + switch (typeCode) + { + case datatypes::SystemCatalog::TINYINT: + if (intVal < MIN_TINYINT) + { + intVal = MIN_TINYINT; pushwarning = true; - switch (typeCode) + if (saturate) + *saturate = true; + } + else if (intVal > MAX_TINYINT) + { + intVal = MAX_TINYINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + + break; + + case datatypes::SystemCatalog::SMALLINT: + if (intVal < MIN_SMALLINT) + { + intVal = MIN_SMALLINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + else if (intVal > MAX_SMALLINT) + { + intVal = MAX_SMALLINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + + break; + + case datatypes::SystemCatalog::MEDINT: + if (intVal < MIN_MEDINT) + { + intVal = MIN_MEDINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + else if (intVal > MAX_MEDINT) + { + intVal = MAX_MEDINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + + break; + + case datatypes::SystemCatalog::INT: + if (intVal < MIN_INT) + { + intVal = MIN_INT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + else if (intVal > MAX_INT) + { + intVal = MAX_INT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + + break; + + case datatypes::SystemCatalog::BIGINT: + if (intVal < MIN_BIGINT) + { + intVal = MIN_BIGINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + + break; + + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (LIKELY(ct.colWidth == 16)) + { + int128_t tmp; + utils::int128Min(tmp); + if (intVal < tmp + 2) // + 2 for NULL and EMPTY values + { + intVal = tmp + 2; + pushwarning = true; + + if (saturate) + *saturate = true; + } + } + else if (ct.colWidth == 8) + { + if (intVal < MIN_BIGINT) + { + intVal = MIN_BIGINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + } + else if (ct.colWidth == 4) + { + if (intVal < MIN_INT) + { + intVal = MIN_INT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + else if (intVal > MAX_INT) + { + intVal = MAX_INT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + } + else if (ct.colWidth == 2) + { + if (intVal < MIN_SMALLINT) + { + intVal = MIN_SMALLINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + else if (intVal > MAX_SMALLINT) + { + intVal = MAX_SMALLINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + } + else if (ct.colWidth == 1) + { + if (intVal < MIN_TINYINT) + { + intVal = MIN_TINYINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + else if (intVal > MAX_TINYINT) + { + intVal = MAX_TINYINT; + pushwarning = true; + + if (saturate) + *saturate = true; + } + } + + break; + + default: break; + } + + // @ bug 3285 make sure the value is in precision range for decimal data type + if ((typeCode == datatypes::SystemCatalog::DECIMAL) || (typeCode == datatypes::SystemCatalog::UDECIMAL) || + (ct.scale > 0)) + { + T rangeUp, rangeLow; + + if (ct.precision < 19) { - case datatypes::SystemCatalog::TINYINT: - if (intVal < MIN_TINYINT) - { - intVal = MIN_TINYINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_TINYINT) - { - intVal = MAX_TINYINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - - break; - - case datatypes::SystemCatalog::SMALLINT: - if (intVal < MIN_SMALLINT) - { - intVal = MIN_SMALLINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_SMALLINT) - { - intVal = MAX_SMALLINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - - break; - - case datatypes::SystemCatalog::MEDINT: - if (intVal < MIN_MEDINT) - { - intVal = MIN_MEDINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_MEDINT) - { - intVal = MAX_MEDINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - - break; - - case datatypes::SystemCatalog::INT: - if (intVal < MIN_INT) - { - intVal = MIN_INT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_INT) - { - intVal = MAX_INT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - - break; - - case datatypes::SystemCatalog::BIGINT: - if (intVal < MIN_BIGINT) - { - intVal = MIN_BIGINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - - break; - - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::UDECIMAL: - if (LIKELY(ct.colWidth == 16)) - { - int128_t tmp; - utils::int128Min(tmp); - if (intVal < tmp + 2) // + 2 for NULL and EMPTY values - { - intVal = tmp + 2; - pushwarning = true; - - if (saturate) - *saturate = true; - } - } - else if (ct.colWidth == 8) - { - if (intVal < MIN_BIGINT) - { - intVal = MIN_BIGINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - } - else if (ct.colWidth == 4) - { - if (intVal < MIN_INT) - { - intVal = MIN_INT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_INT) - { - intVal = MAX_INT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - } - else if (ct.colWidth == 2) - { - if (intVal < MIN_SMALLINT) - { - intVal = MIN_SMALLINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_SMALLINT) - { - intVal = MAX_SMALLINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - } - else if (ct.colWidth == 1) - { - if (intVal < MIN_TINYINT) - { - intVal = MIN_TINYINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal > MAX_TINYINT) - { - intVal = MAX_TINYINT; - pushwarning = true; - - if (saturate) - *saturate = true; - } - } - - break; - - default: - break; + rangeUp = (T)columnstore_precision[ct.precision]; + } + else + { + bool dummy = false; + char* ep = NULL; + rangeUp = (T)dataconvert::strtoll128(columnstore_big_precision[ct.precision - 19].c_str(), dummy, &ep); } - // @ bug 3285 make sure the value is in precision range for decimal data type - if ( (typeCode == datatypes::SystemCatalog::DECIMAL) || - (typeCode == datatypes::SystemCatalog::UDECIMAL) || - (ct.scale > 0)) + rangeLow = -rangeUp; + + if (intVal > rangeUp) { - T rangeUp, rangeLow; + intVal = rangeUp; + pushwarning = true; - if (ct.precision < 19) - { - rangeUp = (T) columnstore_precision[ct.precision]; - } - else - { - bool dummy = false; - char *ep = NULL; - rangeUp = (T) dataconvert::strtoll128(columnstore_big_precision[ct.precision - 19].c_str(), dummy, &ep); - } - - rangeLow = -rangeUp; - - if (intVal > rangeUp) - { - intVal = rangeUp; - pushwarning = true; - - if (saturate) - *saturate = true; - } - else if (intVal < rangeLow) - { - intVal = rangeLow; - pushwarning = true; - - if (saturate) - *saturate = true; - } + if (saturate) + *saturate = true; } + else if (intVal < rangeLow) + { + intVal = rangeLow; + pushwarning = true; + + if (saturate) + *saturate = true; + } + } } // Explicit template instantiation -template -void number_int_value(const std::string& data, - cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd& ct, - bool& pushwarning, - bool noRoundup, - int64_t& intVal, bool* saturate); +template void number_int_value(const std::string& data, cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, bool noRoundup, int64_t& intVal, bool* saturate); -template -void number_int_value(const std::string& data, - cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd& ct, - bool& pushwarning, - bool noRoundup, - int128_t& intVal, bool* saturate); +template void number_int_value(const std::string& data, cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, bool noRoundup, int128_t& intVal, bool* saturate); -uint64_t number_uint_value(const string& data, - cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd& ct, - bool& pushwarning, - bool noRoundup) +uint64_t number_uint_value(const string& data, cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, + bool noRoundup) { - // copy of the original input - string valStr(data); + // copy of the original input + string valStr(data); - // in case, the values are in parentheses - string::size_type x = valStr.find('('); - string::size_type y = valStr.find(')'); + // in case, the values are in parentheses + string::size_type x = valStr.find('('); + string::size_type y = valStr.find(')'); - while (x < string::npos) + while (x < string::npos) + { + // erase y first + if (y == string::npos) + throw QueryDataExcept("'(' is not matched.", formatErr); + + valStr.erase(y, 1); + valStr.erase(x, 1); + x = valStr.find('('); + y = valStr.find(')'); + } + + if (y != string::npos) + throw QueryDataExcept("')' is not matched.", formatErr); + + // convert to fixed-point notation if input is in scientific notation + if (valStr.find('E') < string::npos || valStr.find('e') < string::npos) + { + size_t epos = valStr.find('E'); + + if (epos == string::npos) + epos = valStr.find('e'); + + // get the coefficient + string coef = valStr.substr(0, epos); + // get the exponent + string exp = valStr.substr(epos + 1); + bool overflow = false; + int64_t exponent = dataconvert::string_to_ll(exp, overflow); + + // if the exponent can not be held in 64-bit, not supported or saturated. + if (overflow) + throw QueryDataExcept("value is invalid.", formatErr); + + // find the optional "." point + size_t dpos = coef.find('.'); + + if (dpos != string::npos) { - // erase y first - if (y == string::npos) - throw QueryDataExcept("'(' is not matched.", formatErr); - - valStr.erase(y, 1); - valStr.erase(x, 1); - x = valStr.find('('); - y = valStr.find(')'); + // move "." to the end by mutiply 10 ** (# of fraction digits) + coef.erase(dpos, 1); + exponent -= coef.length() - dpos; } - if (y != string::npos) - throw QueryDataExcept("')' is not matched.", formatErr); - - // convert to fixed-point notation if input is in scientific notation - if (valStr.find('E') < string::npos || valStr.find('e') < string::npos) + if (exponent >= 0) { - size_t epos = valStr.find('E'); + coef.resize(coef.length() + exponent, '0'); + } + else + { + size_t bpos = coef.find_first_of("0123456789"); + size_t epos = coef.length(); + size_t mpos = -exponent; + dpos = epos - mpos; + int64_t padding = (int64_t)mpos - (int64_t)(epos - bpos); - if (epos == string::npos) - epos = valStr.find('e'); + if (padding > 0) + { + coef.insert(bpos, padding, '0'); + dpos = bpos; + } - // get the coefficient - string coef = valStr.substr(0, epos); - // get the exponent - string exp = valStr.substr(epos + 1); - bool overflow = false; - int64_t exponent = dataconvert::string_to_ll(exp, overflow); - - // if the exponent can not be held in 64-bit, not supported or saturated. - if (overflow) - throw QueryDataExcept("value is invalid.", formatErr); - - // find the optional "." point - size_t dpos = coef.find('.'); - - if (dpos != string::npos) - { - // move "." to the end by mutiply 10 ** (# of fraction digits) - coef.erase(dpos, 1); - exponent -= coef.length() - dpos; - } - - if (exponent >= 0) - { - coef.resize(coef.length() + exponent, '0'); - } - else - { - size_t bpos = coef.find_first_of("0123456789"); - size_t epos = coef.length(); - size_t mpos = -exponent; - dpos = epos - mpos; - int64_t padding = (int64_t)mpos - (int64_t)(epos - bpos); - - if (padding > 0) - { - coef.insert(bpos, padding, '0'); - dpos = bpos; - } - - coef.insert(dpos, "."); - } - - valStr = coef; + coef.insert(dpos, "."); } - // now, convert to uint64_t - string intStr(valStr); - string frnStr = ""; - size_t dp = valStr.find('.'); + valStr = coef; + } - if (dp != string::npos) + // now, convert to uint64_t + string intStr(valStr); + string frnStr = ""; + size_t dp = valStr.find('.'); + + if (dp != string::npos) + { + intStr.erase(dp); + frnStr = valStr.substr(dp + 1); + + if (intStr.length() == 0) + intStr = "0"; + else if ((intStr.length() == 1) && ((intStr[0] == '+') || (intStr[0] == '-'))) { - intStr.erase(dp); - frnStr = valStr.substr(dp + 1); - - if ( intStr.length() == 0 ) - intStr = "0"; - else if (( intStr.length() == 1 ) && ( (intStr[0] == '+') || (intStr[0] == '-') ) ) - { - intStr.insert( 1, 1, '0'); - } + intStr.insert(1, 1, '0'); } + } - uint64_t uintVal = dataconvert::string_to_ull(intStr, pushwarning); + uint64_t uintVal = dataconvert::string_to_ull(intStr, pushwarning); - bool dummy = false; - uint64_t frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ull(frnStr, dummy) : 0; + bool dummy = false; + uint64_t frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ull(frnStr, dummy) : 0; - if (frnVal != 0) + if (frnVal != 0) + pushwarning = true; + + switch (typeCode) + { + case datatypes::SystemCatalog::UTINYINT: + if (uintVal > MAX_UTINYINT) + { + uintVal = MAX_UTINYINT; pushwarning = true; + } - switch (typeCode) - { - case datatypes::SystemCatalog::UTINYINT: - if (uintVal > MAX_UTINYINT) - { - uintVal = MAX_UTINYINT; - pushwarning = true; - } + break; - break; + case datatypes::SystemCatalog::USMALLINT: + if (uintVal > MAX_USMALLINT) + { + uintVal = MAX_USMALLINT; + pushwarning = true; + } - case datatypes::SystemCatalog::USMALLINT: - if (uintVal > MAX_USMALLINT) - { - uintVal = MAX_USMALLINT; - pushwarning = true; - } + break; - break; + case datatypes::SystemCatalog::UMEDINT: + if (uintVal > MAX_UMEDINT) + { + uintVal = MAX_UMEDINT; + pushwarning = true; + } - case datatypes::SystemCatalog::UMEDINT: - if (uintVal > MAX_UMEDINT) - { - uintVal = MAX_UMEDINT; - pushwarning = true; - } + break; - break; + case datatypes::SystemCatalog::UINT: + if (uintVal > MAX_UINT) + { + uintVal = MAX_UINT; + pushwarning = true; + } - case datatypes::SystemCatalog::UINT: - if (uintVal > MAX_UINT) - { - uintVal = MAX_UINT; - pushwarning = true; - } + break; - break; + case datatypes::SystemCatalog::UBIGINT: + if (uintVal > MAX_UBIGINT) + { + uintVal = MAX_UBIGINT; + pushwarning = true; + } - case datatypes::SystemCatalog::UBIGINT: - if (uintVal > MAX_UBIGINT) - { - uintVal = MAX_UBIGINT; - pushwarning = true; - } + break; - break; + default: break; + } - default: - break; - } - - return uintVal; + return uintVal; } /** @@ -729,2311 +707,2261 @@ uint64_t number_uint_value(const string& data, * It's up to the caller to figure out whether an error occurred based on * their definition of an error and how many characters were read */ -uint32_t readDecimal( const char*& str, int32_t& value, uint32_t max = 0 ) +uint32_t readDecimal(const char*& str, int32_t& value, uint32_t max = 0) { - value = 0; - uint32_t numread = 0; + value = 0; + uint32_t numread = 0; - while ( (!max || numread < max) && *str && isdigit(*str) ) - { - value = value * 10 + ((*str) - '0'); - ++numread; - ++str; - } + while ((!max || numread < max) && *str && isdigit(*str)) + { + value = value * 10 + ((*str) - '0'); + ++numread; + ++str; + } - return numread; + return numread; } -bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate ) +bool mysql_str_to_datetime(const string& input, DateTime& output, bool& isDate) { - /** - * First we are going to identify the stop/start of the date portion. - * The rules are: - * - Date portion must come before anything else - * - Date portion may only contain numbers and '-' - * - Date portion ends with ' ', 'T', or '\0' - * - Date portion always starts with Year - * - Without date separators ('-'): - * YYMMDD - * YYYYMMDD - * - With date separators there are no specific field length - * requirements - */ - int32_t datesepct = 0; - uint32_t dtend = 0; + /** + * First we are going to identify the stop/start of the date portion. + * The rules are: + * - Date portion must come before anything else + * - Date portion may only contain numbers and '-' + * - Date portion ends with ' ', 'T', or '\0' + * - Date portion always starts with Year + * - Without date separators ('-'): + * YYMMDD + * YYYYMMDD + * - With date separators there are no specific field length + * requirements + */ + int32_t datesepct = 0; + uint32_t dtend = 0; - for ( ; dtend < input.length(); ++dtend ) + for (; dtend < input.length(); ++dtend) + { + char c = input[dtend]; + + if (isdigit(c)) { - char c = input[dtend]; - - if ( isdigit( c ) ) - { - continue; - } -// else if( dtend != 0 && c == '-' ) - else if ( dtend != 0 && ispunct(c) ) - { - ++datesepct; - } - else if ( c == 'T' || c == ' ' ) - { - break; - } - else - { - // some other character showed up - output.reset(); - return false; - } + continue; } - - int32_t year = -1; - int32_t mon = -1; - int32_t day = -1; - const char* ptr = input.c_str(); - - if ( datesepct == 0 ) + // else if( dtend != 0 && c == '-' ) + else if (dtend != 0 && ispunct(c)) { - if ( dtend == 6 || dtend == 12 ) - { - readDecimal(ptr, year, 2); - readDecimal(ptr, mon, 2); - readDecimal(ptr, day, 2); - year += 2000; - - if ( year > 2069 ) - year -= 100; - - if ( dtend == 12 ) - dtend -= 6; - } - else if ( dtend == 8 || dtend == 14 ) - { - readDecimal(ptr, year, 4); - readDecimal(ptr, mon, 2); - readDecimal(ptr, day, 2); - - if ( dtend == 14 ) - dtend -= 6; - } - else - { - output.reset(); - return false; - } + ++datesepct; } - else if ( datesepct == 2 ) + else if (c == 'T' || c == ' ') { - uint32_t numread = readDecimal(ptr, year); - - if ( numread == 2 ) - { - // special handling if we read a 2-byte year - year += 2000; - - if ( year > 2069 ) - year -= 100; - } - - ++ptr; // skip one separator - readDecimal(ptr, mon); - ++ptr; // skip one separator - readDecimal(ptr, day); // skip two separators + break; } else { - output.reset(); - return false; + // some other character showed up + output.reset(); + return false; } + } - if (!isDateValid(day, mon, year)) + int32_t year = -1; + int32_t mon = -1; + int32_t day = -1; + const char* ptr = input.c_str(); + + if (datesepct == 0) + { + if (dtend == 6 || dtend == 12) { - output.reset(); - return false; + readDecimal(ptr, year, 2); + readDecimal(ptr, mon, 2); + readDecimal(ptr, day, 2); + year += 2000; + + if (year > 2069) + year -= 100; + + if (dtend == 12) + dtend -= 6; } - - output.year = year; - output.month = mon; - output.day = day; - - /** - * Now we need to deal with the time portion. - * The rules are: - * - Time portion may be empty - * - Time portion may start with 'T' - * - Time portion always ends with '\0' - * - Time portion always starts with hour - * - Without time separators (':'): - * HHMMSS - * - All Times can end with option .[microseconds] - * - With time separators there are no specific field length - * requirements - */ - while ( input[dtend] == ' ' && dtend < input.length() ) + else if (dtend == 8 || dtend == 14) { - ++dtend; - } + readDecimal(ptr, year, 4); + readDecimal(ptr, mon, 2); + readDecimal(ptr, day, 2); - if ( dtend == input.length() ) - { - isDate = true; - return true; - } - - uint32_t timesep_ct = 0; - bool has_usec = false; - uint32_t len_before_msec = 0; - uint32_t tmstart = ( input[dtend] == ' ' || input[dtend] == 'T' ) ? dtend + 1 : dtend; - uint32_t tmend = tmstart; - - for ( ; tmend < input.length(); ++tmend ) - { - char c = input[tmend]; - - if ( isdigit( c ) ) - { - // digits always ok - continue; - } -// else if( c == ':' ) -// { -// timesep_ct++; -// } -// else if( c == '.' ) -// { -// len_before_msec = ( tmend - tmstart ); -// has_usec = true; -// } - else if ( ispunct(c) ) - { - if ( c == '.' && timesep_ct == 2 ) - { - len_before_msec = ( tmend - tmstart ); - has_usec = true; - } - else - { - timesep_ct++; - } - } - else - { - // some other character showed up - output.reset(); - return false; - } - } - - if ( !len_before_msec ) - len_before_msec = ( tmend - tmstart ); - - int32_t hour = -1; - int32_t min = 0; - int32_t sec = 0; - int32_t usec = 0; - const char* tstart = input.c_str() + tmstart; - - if ( timesep_ct == 2 ) - { - readDecimal(tstart, hour); - ++tstart; // skip one separator - readDecimal(tstart, min); - ++tstart; // skip one separator - readDecimal(tstart, sec); - } - else if ( timesep_ct == 1 ) - { - readDecimal(tstart, hour); - ++tstart; // skip one separator - readDecimal(tstart, min); - } - else if ( timesep_ct == 0 && len_before_msec == 6 ) - { - readDecimal(tstart, hour, 2); - readDecimal(tstart, min, 2); - readDecimal(tstart, sec, 2); - } - else if ( timesep_ct == 0 && len_before_msec == 4 ) - { - readDecimal(tstart, hour, 2); - readDecimal(tstart, min, 2); - } - else if ( timesep_ct == 0 && len_before_msec == 2 ) - { - readDecimal(tstart, hour, 2); + if (dtend == 14) + dtend -= 6; } else { - output.reset(); - return false; + output.reset(); + return false; } + } + else if (datesepct == 2) + { + uint32_t numread = readDecimal(ptr, year); - if ( has_usec ) + if (numread == 2) { - ++tstart; // skip '.' character. We could error check if we wanted to - uint32_t numread = readDecimal(tstart, usec); + // special handling if we read a 2-byte year + year += 2000; - if ( numread > 6 || numread < 1 ) - { - // don't allow more than 6 digits when specifying microseconds - output.reset(); - return false; - } - - // usec have to be scaled up so that it always represents microseconds - for ( int i = numread; i < 6; i++ ) - usec *= 10; + if (year > 2069) + year -= 100; } - if ( !isDateTimeValid( hour, min, sec, usec ) ) - { - output.reset(); - return false; - } + ++ptr; // skip one separator + readDecimal(ptr, mon); + ++ptr; // skip one separator + readDecimal(ptr, day); // skip two separators + } + else + { + output.reset(); + return false; + } - output.hour = hour; - output.minute = min; - output.second = sec; - output.msecond = usec; - isDate = false; + if (!isDateValid(day, mon, year)) + { + output.reset(); + return false; + } + + output.year = year; + output.month = mon; + output.day = day; + + /** + * Now we need to deal with the time portion. + * The rules are: + * - Time portion may be empty + * - Time portion may start with 'T' + * - Time portion always ends with '\0' + * - Time portion always starts with hour + * - Without time separators (':'): + * HHMMSS + * - All Times can end with option .[microseconds] + * - With time separators there are no specific field length + * requirements + */ + while (input[dtend] == ' ' && dtend < input.length()) + { + ++dtend; + } + + if (dtend == input.length()) + { + isDate = true; return true; -} + } -bool mysql_str_to_time( const string& input, Time& output, long decimals ) -{ - uint32_t dtend = 0; - bool isNeg = false; + uint32_t timesep_ct = 0; + bool has_usec = false; + uint32_t len_before_msec = 0; + uint32_t tmstart = (input[dtend] == ' ' || input[dtend] == 'T') ? dtend + 1 : dtend; + uint32_t tmend = tmstart; - /** - * We need to deal with the time portion. - * The rules are: - * - Time portion always ends with '\0' - * - Time portion always starts with hour - * - Without time separators (':'): - * HHMMSS - * - All Times can end with option .[microseconds] - * - With time separators there are no specific field length - * requirements - */ - while ( input[dtend] == ' ' && dtend < input.length() ) + for (; tmend < input.length(); ++tmend) + { + char c = input[tmend]; + + if (isdigit(c)) { - ++dtend; + // digits always ok + continue; } - - if ( dtend == input.length() ) + // else if( c == ':' ) + // { + // timesep_ct++; + // } + // else if( c == '.' ) + // { + // len_before_msec = ( tmend - tmstart ); + // has_usec = true; + // } + else if (ispunct(c)) { - return false; - } - - uint32_t timesep_ct = 0; - bool has_usec = false; - uint32_t len_before_msec = 0; - uint32_t tmstart = dtend; - uint32_t tmend = tmstart; - - for ( ; tmend < input.length(); ++tmend ) - { - char c = input[tmend]; - - if ( isdigit( c ) ) - { - // digits always ok - continue; - } -// else if( c == ':' ) -// { -// timesep_ct++; -// } -// else if( c == '.' ) -// { -// len_before_msec = ( tmend - tmstart ); -// has_usec = true; -// } - else if ( ispunct(c) ) - { - if ( c == '.' && timesep_ct == 2 ) - { - len_before_msec = ( tmend - tmstart ); - has_usec = true; - } - else if (c == '-' && (tmend == tmstart)) - { - isNeg = true; - ++tmstart; - } - else - { - timesep_ct++; - } - } - else - { - // some other character showed up - output.reset(); - return false; - } - } - - if ( !len_before_msec ) - len_before_msec = ( tmend - tmstart ); - - int32_t hour = -1; - int32_t min = 0; - int32_t sec = 0; - int32_t usec = 0; - const char* tstart = input.c_str() + tmstart; - - if ( timesep_ct == 2 ) - { - readDecimal(tstart, hour); - ++tstart; // skip one separator - readDecimal(tstart, min); - ++tstart; // skip one separator - readDecimal(tstart, sec); - } - else if ( timesep_ct == 1 ) - { - readDecimal(tstart, hour); - ++tstart; // skip one separator - readDecimal(tstart, min); - } - else if ( timesep_ct == 0 && len_before_msec == 6 ) - { - readDecimal(tstart, hour, 2); - readDecimal(tstart, min, 2); - readDecimal(tstart, sec, 2); - } - else if ( timesep_ct == 0 && len_before_msec == 4 ) - { - readDecimal(tstart, hour, 2); - readDecimal(tstart, min, 2); - } - else if ( timesep_ct == 0 && len_before_msec == 2 ) - { - readDecimal(tstart, hour, 2); + if (c == '.' && timesep_ct == 2) + { + len_before_msec = (tmend - tmstart); + has_usec = true; + } + else + { + timesep_ct++; + } } else { - output.reset(); - return false; + // some other character showed up + output.reset(); + return false; } + } - if ( has_usec ) + if (!len_before_msec) + len_before_msec = (tmend - tmstart); + + int32_t hour = -1; + int32_t min = 0; + int32_t sec = 0; + int32_t usec = 0; + const char* tstart = input.c_str() + tmstart; + + if (timesep_ct == 2) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + ++tstart; // skip one separator + readDecimal(tstart, sec); + } + else if (timesep_ct == 1) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + } + else if (timesep_ct == 0 && len_before_msec == 6) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + readDecimal(tstart, sec, 2); + } + else if (timesep_ct == 0 && len_before_msec == 4) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + } + else if (timesep_ct == 0 && len_before_msec == 2) + { + readDecimal(tstart, hour, 2); + } + else + { + output.reset(); + return false; + } + + if (has_usec) + { + ++tstart; // skip '.' character. We could error check if we wanted to + uint32_t numread = readDecimal(tstart, usec); + + if (numread > 6 || numread < 1) { - ++tstart; // skip '.' character. We could error check if we wanted to - uint32_t numread = readDecimal(tstart, usec); - - if ( numread > 6 || numread < 1 ) - { - // don't allow more than 6 digits when specifying microseconds - output.reset(); - return false; - } - - // usec have to be scaled up so that it always represents microseconds - for ( int i = numread; i < 6; i++ ) - usec *= 10; + // don't allow more than 6 digits when specifying microseconds + output.reset(); + return false; } - if ( !isTimeValid( hour, min, sec, usec ) ) - { - // Emulate MariaDB's time saturation - // TODO: msec saturation - if ((hour > 838) && !isNeg) - { - output.hour = 838; - output.minute = 59; - output.second = 59; - output.msecond = exp10(decimals) - 1; - output.is_neg = 0; - } - else if ((hour < -838) || ((hour > 838) && isNeg)) - { - output.hour = -838; - output.minute = 59; - output.second = 59; - output.msecond = exp10(decimals) - 1; - output.is_neg = 1; - } - // If neither of the above match then we return a 0 time - else - { - output.reset(); - } + // usec have to be scaled up so that it always represents microseconds + for (int i = numread; i < 6; i++) + usec *= 10; + } - return false; - } + if (!isDateTimeValid(hour, min, sec, usec)) + { + output.reset(); + return false; + } - output.hour = isNeg ? 0 - hour : hour; - output.minute = min; - output.second = sec; - output.msecond = usec; - output.is_neg = isNeg; - return true; + output.hour = hour; + output.minute = min; + output.second = sec; + output.msecond = usec; + isDate = false; + return true; } -bool stringToDateStruct( const string& data, Date& date ) +bool mysql_str_to_time(const string& input, Time& output, long decimals) { - bool isDate; - DateTime dt; + uint32_t dtend = 0; + bool isNeg = false; - if ( !mysql_str_to_datetime( data, dt, isDate )) - return false; + /** + * We need to deal with the time portion. + * The rules are: + * - Time portion always ends with '\0' + * - Time portion always starts with hour + * - Without time separators (':'): + * HHMMSS + * - All Times can end with option .[microseconds] + * - With time separators there are no specific field length + * requirements + */ + while (input[dtend] == ' ' && dtend < input.length()) + { + ++dtend; + } - date.year = dt.year; - date.month = dt.month; - date.day = dt.day; - return true; + if (dtend == input.length()) + { + return false; + } + + uint32_t timesep_ct = 0; + bool has_usec = false; + uint32_t len_before_msec = 0; + uint32_t tmstart = dtend; + uint32_t tmend = tmstart; + + for (; tmend < input.length(); ++tmend) + { + char c = input[tmend]; + + if (isdigit(c)) + { + // digits always ok + continue; + } + // else if( c == ':' ) + // { + // timesep_ct++; + // } + // else if( c == '.' ) + // { + // len_before_msec = ( tmend - tmstart ); + // has_usec = true; + // } + else if (ispunct(c)) + { + if (c == '.' && timesep_ct == 2) + { + len_before_msec = (tmend - tmstart); + has_usec = true; + } + else if (c == '-' && (tmend == tmstart)) + { + isNeg = true; + ++tmstart; + } + else + { + timesep_ct++; + } + } + else + { + // some other character showed up + output.reset(); + return false; + } + } + + if (!len_before_msec) + len_before_msec = (tmend - tmstart); + + int32_t hour = -1; + int32_t min = 0; + int32_t sec = 0; + int32_t usec = 0; + const char* tstart = input.c_str() + tmstart; + + if (timesep_ct == 2) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + ++tstart; // skip one separator + readDecimal(tstart, sec); + } + else if (timesep_ct == 1) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + } + else if (timesep_ct == 0 && len_before_msec == 6) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + readDecimal(tstart, sec, 2); + } + else if (timesep_ct == 0 && len_before_msec == 4) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + } + else if (timesep_ct == 0 && len_before_msec == 2) + { + readDecimal(tstart, hour, 2); + } + else + { + output.reset(); + return false; + } + + if (has_usec) + { + ++tstart; // skip '.' character. We could error check if we wanted to + uint32_t numread = readDecimal(tstart, usec); + + if (numread > 6 || numread < 1) + { + // don't allow more than 6 digits when specifying microseconds + output.reset(); + return false; + } + + // usec have to be scaled up so that it always represents microseconds + for (int i = numread; i < 6; i++) + usec *= 10; + } + + if (!isTimeValid(hour, min, sec, usec)) + { + // Emulate MariaDB's time saturation + // TODO: msec saturation + if ((hour > 838) && !isNeg) + { + output.hour = 838; + output.minute = 59; + output.second = 59; + output.msecond = exp10(decimals) - 1; + output.is_neg = 0; + } + else if ((hour < -838) || ((hour > 838) && isNeg)) + { + output.hour = -838; + output.minute = 59; + output.second = 59; + output.msecond = exp10(decimals) - 1; + output.is_neg = 1; + } + // If neither of the above match then we return a 0 time + else + { + output.reset(); + } + + return false; + } + + output.hour = isNeg ? 0 - hour : hour; + output.minute = min; + output.second = sec; + output.msecond = usec; + output.is_neg = isNeg; + return true; +} + +bool stringToDateStruct(const string& data, Date& date) +{ + bool isDate; + DateTime dt; + + if (!mysql_str_to_datetime(data, dt, isDate)) + return false; + + date.year = dt.year; + date.month = dt.month; + date.day = dt.day; + return true; } bool stringToDatetimeStruct(const string& data, DateTime& dtime, bool* date) { - bool isDate; + bool isDate; - if ( !mysql_str_to_datetime( data, dtime, isDate ) ) - return false; + if (!mysql_str_to_datetime(data, dtime, isDate)) + return false; - if ( isDate ) - { - if (date) - *date = true; + if (isDate) + { + if (date) + *date = true; - dtime.hour = 0; - dtime.minute = 0; - dtime.second = 0; - dtime.msecond = 0; - } + dtime.hour = 0; + dtime.minute = 0; + dtime.second = 0; + dtime.msecond = 0; + } - return true; + return true; } bool stringToTimeStruct(const string& data, Time& dtime, long decimals) { - if ( !mysql_str_to_time( data, dtime, decimals ) ) - return false; + if (!mysql_str_to_time(data, dtime, decimals)) + return false; - return true; + return true; } bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const string& timeZone) { - // special handling for 0000-00-00 00:00:00 - // "0" is sent by the server when checking for default value - // in the DDL. This is equivalent of 0000-00-00 00:00:00 - if (data.substr(0, 19) == "0000-00-00 00:00:00" || data == "0") - { - timeStamp.second = 0; - timeStamp.msecond = 0; - return true; - } - - // for alter table add column timestamp, - // if the table is non-empty, then columnstore will apply - // default value to populate the new column - if (data == "current_timestamp() ON UPDATE current_timestamp()") - { - struct timeval tv; - gettimeofday(&tv, 0); - timeStamp.second = tv.tv_sec; - timeStamp.msecond = tv.tv_usec; - return true; - } - - bool isDate; - - DateTime dtime; - - if ( !mysql_str_to_datetime( data, dtime, isDate ) ) - { - timeStamp.reset(); - return false; - } - - if ( isDate ) - { - dtime.hour = 0; - dtime.minute = 0; - dtime.second = 0; - dtime.msecond = 0; - } - - MySQLTime m_time; - m_time.year = dtime.year; - m_time.month = dtime.month; - m_time.day = dtime.day; - m_time.hour = dtime.hour; - m_time.minute = dtime.minute; - m_time.second = dtime.second; - m_time.second_part = dtime.msecond; - - bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone, isValid); - - if (!isValid) - { - timeStamp.reset(); - return false; - } - - timeStamp.second = seconds; - timeStamp.msecond = m_time.second_part; - + // special handling for 0000-00-00 00:00:00 + // "0" is sent by the server when checking for default value + // in the DDL. This is equivalent of 0000-00-00 00:00:00 + if (data.substr(0, 19) == "0000-00-00 00:00:00" || data == "0") + { + timeStamp.second = 0; + timeStamp.msecond = 0; return true; + } + // for alter table add column timestamp, + // if the table is non-empty, then columnstore will apply + // default value to populate the new column + if (data == "current_timestamp() ON UPDATE current_timestamp()") + { + struct timeval tv; + gettimeofday(&tv, 0); + timeStamp.second = tv.tv_sec; + timeStamp.msecond = tv.tv_usec; + return true; + } + + bool isDate; + + DateTime dtime; + + if (!mysql_str_to_datetime(data, dtime, isDate)) + { + timeStamp.reset(); + return false; + } + + if (isDate) + { + dtime.hour = 0; + dtime.minute = 0; + dtime.second = 0; + dtime.msecond = 0; + } + + MySQLTime m_time; + m_time.year = dtime.year; + m_time.month = dtime.month; + m_time.day = dtime.day; + m_time.hour = dtime.hour; + m_time.minute = dtime.minute; + m_time.second = dtime.second; + m_time.second_part = dtime.msecond; + + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone, isValid); + + if (!isValid) + { + timeStamp.reset(); + return false; + } + + timeStamp.second = seconds; + timeStamp.msecond = m_time.second_part; + + return true; } -boost::any -DataConvert::StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const datatypes::ConvertFromStringParam &prm, - const std::string& dataOrig, - bool& pushWarning) +boost::any DataConvert::StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam& prm, const std::string& dataOrig, + bool& pushWarning) { - std::string data(dataOrig); - unsigned int x = data.find("("); + std::string data(dataOrig); + unsigned int x = data.find("("); - if (x <= data.length()) + if (x <= data.length()) + { + data.replace(x, 1, " "); + } + + x = data.find(")"); + + if (x <= data.length()) + { + data.replace(x, 1, " "); + } + + int64_t tmp = 0; + + number_int_value(data, datatypes::SystemCatalog::BIT, colType, pushWarning, prm.noRoundup(), tmp); + + if (tmp) + { + bool bitvalue; + + if (from_string(bitvalue, data, std::dec)) { - data.replace ( x, 1, " "); - } - - x = data.find(")"); - - if (x <= data.length()) - { - data.replace (x, 1, " "); - } - - int64_t tmp = 0; - - number_int_value (data, datatypes::SystemCatalog::BIT, colType, pushWarning, prm.noRoundup(), tmp); - - if (tmp) - { - bool bitvalue; - - if (from_string(bitvalue, data, std::dec )) - { - boost::any value = bitvalue; - return value; - } - else - { - throw QueryDataExcept("range, valid value or conversion error on BIT type.", formatErr); - } - } - return boost::any(); -} - - -boost::any -DataConvert::StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const datatypes::ConvertFromStringParam &prm, - const std::string& data, bool& pushWarning) -{ - const cscDataType typeCode= datatypes::SystemCatalog::DECIMAL; - if (LIKELY(colType.colWidth == 16)) - { - int128_t val128; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); - boost::any value = (int128_t) val128; - return value; - } - else if (colType.colWidth == 8) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (long long) val64; - return value; - } - else if (colType.colWidth == 4) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (int) val64; - return value; - } - else if (colType.colWidth == 2) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (short) val64; - return value; - } - else if (colType.colWidth == 1) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - boost::any value = (char) val64; - return value; - } - //else if (colType.colWidth == 32) - // value = data; - return boost::any(); -} - - -boost::any -DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const datatypes::ConvertFromStringParam &prm, - const std::string& data, bool& pushWarning) -{ - const cscDataType typeCode= datatypes::SystemCatalog::UDECIMAL; - - // UDECIMAL numbers may not be negative - if (LIKELY(colType.colWidth == 16)) - { - int128_t val128; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); - - if (val128 < 0 && - !datatypes::Decimal::isWideDecimalNullValue(val128) && - !datatypes::Decimal::isWideDecimalEmptyValue(val128)) - { - val128 = 0; - pushWarning = true; - } - - boost::any value = val128; - return value; - } - else if (colType.colWidth == 8) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - long long ival = static_cast(val64); - - if (ival < 0 && - ival != static_cast(joblist::BIGINTEMPTYROW) && - ival != static_cast(joblist::BIGINTNULL)) - { - ival = 0; - pushWarning = true; - } - - boost::any value = ival; - return value; - } - else if (colType.colWidth == 4) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - int ival = static_cast(val64); - - if (ival < 0 && - ival != static_cast(joblist::INTEMPTYROW) && - ival != static_cast(joblist::INTNULL)) - { - ival = 0; - pushWarning = true; - } - - boost::any value = ival; - return value; - } - else if (colType.colWidth == 2) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - short ival = (short) val64; - - if (ival < 0 && - ival != static_cast(joblist::SMALLINTEMPTYROW) && - ival != static_cast(joblist::SMALLINTNULL)) - { - ival = 0; - pushWarning = true; - } - - boost::any value = ival; - return value; - } - else if (colType.colWidth == 1) - { - int64_t val64; - number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - char ival = (char) val64; - - if (ival < 0 && - ival != static_cast(joblist::TINYINTEMPTYROW) && - ival != static_cast(joblist::TINYINTNULL)) - { - ival = 0; - pushWarning = true; - } - - boost::any value = ival; - return value; - } - return boost::any(); -} - - -boost::any -DataConvert::StringToFloat(cscDataType typeCode, - const std::string& dataOrig, - bool& pushWarning) -{ - boost::any value; - std::string data(dataOrig); - - string::size_type x = data.find('('); - - if (x < string::npos) - data.erase(x, 1); - - x = data.find(')'); - - if (x < string::npos) - data.erase(x, 1); - - if ( number_value ( data ) ) - { - float floatvalue; - errno = 0; -#ifdef _MSC_VER - double dval = strtod(data.c_str(), 0); - - if (dval > MAX_FLOAT) - { - pushWarning = true; - floatvalue = MAX_FLOAT; - } - else if (dval < MIN_FLOAT) - { - pushWarning = true; - floatvalue = MIN_FLOAT; - } - else - { - floatvalue = (float)dval; - } -#else - floatvalue = strtof(data.c_str(), 0); -#endif - - if (errno == ERANGE) - { - pushWarning = true; -#ifdef _MSC_VER - - if ( abs(floatvalue) == HUGE_VAL ) -#else - if ( abs(floatvalue) == HUGE_VALF ) -#endif - { - if ( floatvalue > 0 ) - floatvalue = MAX_FLOAT; - else - floatvalue = MIN_FLOAT; - } - else - floatvalue = 0; - } - - if (floatvalue < 0.0 && - typeCode == datatypes::SystemCatalog::UFLOAT && - floatvalue != static_cast(joblist::FLOATEMPTYROW) && - floatvalue != static_cast(joblist::FLOATNULL)) - { - value = 0.0; // QQ: should it assign floatvalue? - pushWarning = true; - } - - value = floatvalue; - } - else - throw QueryDataExcept("range, valid value or conversion error on FLOAT type.", formatErr); - return value; -} - - - -boost::any -DataConvert::StringToDouble(cscDataType typeCode, - const std::string& dataOrig, - bool& pushWarning) -{ - boost::any value; - std::string data(dataOrig); - - string::size_type x = data.find('('); - - if (x < string::npos) - data.erase(x, 1); - - x = data.find(')'); - - if (x < string::npos) - data.erase(x, 1); - - if ( number_value ( data ) ) - { - double doublevalue; - errno = 0; - doublevalue = strtod(data.c_str(), 0); - - if (errno == ERANGE) - { - pushWarning = true; -#ifdef _MSC_VER - - if ( abs(doublevalue) == HUGE_VAL ) -#else - if ( abs(doublevalue) == HUGE_VALL ) -#endif - { - if ( doublevalue > 0 ) - value = MAX_DOUBLE; - else - value = MIN_DOUBLE; - } - else - value = 0; - } - else - value = doublevalue; - - if (doublevalue < 0.0 && - typeCode == datatypes::SystemCatalog::UDOUBLE && - doublevalue != static_cast(joblist::DOUBLEEMPTYROW) && - doublevalue != static_cast(joblist::DOUBLENULL)) - { - doublevalue = 0.0; // QQ: should it assign "value" ? - pushWarning = true; - } + boost::any value = bitvalue; + return value; } else { - throw QueryDataExcept("range, valid value or conversion error on DOUBLE type.", formatErr); + throw QueryDataExcept("range, valid value or conversion error on BIT type.", formatErr); } + } + return boost::any(); +} + +boost::any DataConvert::StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) +{ + const cscDataType typeCode = datatypes::SystemCatalog::DECIMAL; + if (LIKELY(colType.colWidth == 16)) + { + int128_t val128; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); + boost::any value = (int128_t)val128; return value; + } + else if (colType.colWidth == 8) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (long long)val64; + return value; + } + else if (colType.colWidth == 4) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int)val64; + return value; + } + else if (colType.colWidth == 2) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (short)val64; + return value; + } + else if (colType.colWidth == 1) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (char)val64; + return value; + } + // else if (colType.colWidth == 32) + // value = data; + return boost::any(); } +boost::any DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) +{ + const cscDataType typeCode = datatypes::SystemCatalog::UDECIMAL; -boost::any -DataConvert::StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const std::string& dataOrig, - bool& pushWarning) + // UDECIMAL numbers may not be negative + if (LIKELY(colType.colWidth == 16)) + { + int128_t val128; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); + + if (val128 < 0 && !datatypes::Decimal::isWideDecimalNullValue(val128) && + !datatypes::Decimal::isWideDecimalEmptyValue(val128)) + { + val128 = 0; + pushWarning = true; + } + + boost::any value = val128; + return value; + } + else if (colType.colWidth == 8) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + long long ival = static_cast(val64); + + if (ival < 0 && ival != static_cast(joblist::BIGINTEMPTYROW) && + ival != static_cast(joblist::BIGINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 4) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + int ival = static_cast(val64); + + if (ival < 0 && ival != static_cast(joblist::INTEMPTYROW) && + ival != static_cast(joblist::INTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 2) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + short ival = (short)val64; + + if (ival < 0 && ival != static_cast(joblist::SMALLINTEMPTYROW) && + ival != static_cast(joblist::SMALLINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 1) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + char ival = (char)val64; + + if (ival < 0 && ival != static_cast(joblist::TINYINTEMPTYROW) && + ival != static_cast(joblist::TINYINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + return boost::any(); +} + +boost::any DataConvert::StringToFloat(cscDataType typeCode, const std::string& dataOrig, bool& pushWarning) +{ + boost::any value; + std::string data(dataOrig); + + string::size_type x = data.find('('); + + if (x < string::npos) + data.erase(x, 1); + + x = data.find(')'); + + if (x < string::npos) + data.erase(x, 1); + + if (number_value(data)) + { + float floatvalue; + errno = 0; +#ifdef _MSC_VER + double dval = strtod(data.c_str(), 0); + + if (dval > MAX_FLOAT) + { + pushWarning = true; + floatvalue = MAX_FLOAT; + } + else if (dval < MIN_FLOAT) + { + pushWarning = true; + floatvalue = MIN_FLOAT; + } + else + { + floatvalue = (float)dval; + } +#else + floatvalue = strtof(data.c_str(), 0); +#endif + + if (errno == ERANGE) + { + pushWarning = true; +#ifdef _MSC_VER + + if (abs(floatvalue) == HUGE_VAL) +#else + if (abs(floatvalue) == HUGE_VALF) +#endif + { + if (floatvalue > 0) + floatvalue = MAX_FLOAT; + else + floatvalue = MIN_FLOAT; + } + else + floatvalue = 0; + } + + if (floatvalue < 0.0 && typeCode == datatypes::SystemCatalog::UFLOAT && + floatvalue != static_cast(joblist::FLOATEMPTYROW) && + floatvalue != static_cast(joblist::FLOATNULL)) + { + value = 0.0; // QQ: should it assign floatvalue? + pushWarning = true; + } + + value = floatvalue; + } + else + throw QueryDataExcept("range, valid value or conversion error on FLOAT type.", formatErr); + return value; +} + +boost::any DataConvert::StringToDouble(cscDataType typeCode, const std::string& dataOrig, bool& pushWarning) +{ + boost::any value; + std::string data(dataOrig); + + string::size_type x = data.find('('); + + if (x < string::npos) + data.erase(x, 1); + + x = data.find(')'); + + if (x < string::npos) + data.erase(x, 1); + + if (number_value(data)) + { + double doublevalue; + errno = 0; + doublevalue = strtod(data.c_str(), 0); + + if (errno == ERANGE) + { + pushWarning = true; +#ifdef _MSC_VER + + if (abs(doublevalue) == HUGE_VAL) +#else + if (abs(doublevalue) == HUGE_VALL) +#endif + { + if (doublevalue > 0) + value = MAX_DOUBLE; + else + value = MIN_DOUBLE; + } + else + value = 0; + } + else + value = doublevalue; + + if (doublevalue < 0.0 && typeCode == datatypes::SystemCatalog::UDOUBLE && + doublevalue != static_cast(joblist::DOUBLEEMPTYROW) && + doublevalue != static_cast(joblist::DOUBLENULL)) + { + doublevalue = 0.0; // QQ: should it assign "value" ? + pushWarning = true; + } + } + else + { + throw QueryDataExcept("range, valid value or conversion error on DOUBLE type.", formatErr); + } + return value; +} + +boost::any DataConvert::StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& dataOrig, bool& pushWarning) { - std::string data(dataOrig); - //check data length - if ( data.length() > (unsigned int)colType.colWidth ) - { - // TODO: charsetNumber should be moved to TypeStdAttributes ASAP - const execplan::CalpontSystemCatalog::ColType &colType2= - static_cast(colType); - datatypes::Charset cs(colType2.charsetNumber); - const char *newEnd = data.data() + colType.colWidth; - const char *origEnd = data.data() + data.length(); - pushWarning = cs.test_if_important_data(newEnd, origEnd); - data = data.substr(0, colType.colWidth); - boost::any value = data; - return value; - } - if ( (unsigned int)colType.colWidth > data.length()) - { - //Pad null character to the string - data.resize(colType.colWidth, 0); - } + std::string data(dataOrig); + // check data length + if (data.length() > (unsigned int)colType.colWidth) + { + // TODO: charsetNumber should be moved to TypeStdAttributes ASAP + const execplan::CalpontSystemCatalog::ColType& colType2 = + static_cast(colType); + datatypes::Charset cs(colType2.charsetNumber); + const char* newEnd = data.data() + colType.colWidth; + const char* origEnd = data.data() + data.length(); + pushWarning = cs.test_if_important_data(newEnd, origEnd); + data = data.substr(0, colType.colWidth); boost::any value = data; return value; + } + if ((unsigned int)colType.colWidth > data.length()) + { + // Pad null character to the string + data.resize(colType.colWidth, 0); + } + boost::any value = data; + return value; } - -boost::any -DataConvert::StringToDate(const std::string& data, bool& pushWarning) +boost::any DataConvert::StringToDate(const std::string& data, bool& pushWarning) { - Date aDay; + Date aDay; - if (stringToDateStruct(data, aDay)) - { - boost::any value = getUInt32LE((const char*) &aDay); - return value; - } - boost::any value = (uint32_t) 0; + if (stringToDateStruct(data, aDay)) + { + boost::any value = getUInt32LE((const char*)&aDay); + return value; + } + boost::any value = (uint32_t)0; + pushWarning = true; + return value; +} + +boost::any DataConvert::StringToDatetime(const std::string& data, bool& pushWarning) +{ + DateTime aDatetime; + + if (stringToDatetimeStruct(data, aDatetime, 0)) // QQ: why 0? + { + boost::any value = getUInt64LE((const char*)&aDatetime); + return value; + } + boost::any value = (uint64_t)0; + pushWarning = true; + return value; +} + +boost::any DataConvert::StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& data, bool& pushWarning) +{ + Time aTime; + + if (!stringToTimeStruct(data, aTime, colType.precision)) + { pushWarning = true; - return value; + } + + boost::any value = getSInt64LE((const char*)&aTime); + return value; } - -boost::any -DataConvert::StringToDatetime(const std::string& data, bool& pushWarning) +boost::any DataConvert::StringToTimestamp(const datatypes::ConvertFromStringParam& prm, + const std::string& data, bool& pushWarning) { - DateTime aDatetime; + TimeStamp aTimestamp; - if (stringToDatetimeStruct(data, aDatetime, 0)) // QQ: why 0? - { - boost::any value = getUInt64LE((const char*) &aDatetime); - return value; - } - boost::any value = (uint64_t) 0; + if (!stringToTimestampStruct(data, aTimestamp, prm.timeZone())) + { pushWarning = true; - return value; + } + + boost::any value = getUInt64LE((const char*)&aTimestamp); + return value; } - -boost::any -DataConvert::StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const std::string& data, - bool& pushWarning) -{ - Time aTime; - - if (!stringToTimeStruct(data, aTime, colType.precision)) - { - pushWarning = true; - } - - boost::any value = getSInt64LE((const char*) &aTime); - return value; -} - - -boost::any -DataConvert::StringToTimestamp(const datatypes::ConvertFromStringParam &prm, - const std::string& data, - bool& pushWarning) -{ - TimeStamp aTimestamp; - - if (!stringToTimestampStruct(data, aTimestamp, prm.timeZone())) - { - pushWarning = true; - } - - boost::any value = getUInt64LE((const char*) &aTimestamp); - return value; -} - - //------------------------------------------------------------------------------ // Convert date string to binary date. Used by BulkLoad. //------------------------------------------------------------------------------ -int32_t DataConvert::convertColumnDate( - const char* dataOrg, - CalpontDateTimeFormat dateFormat, - int& status, - unsigned int dataOrgLen ) +int32_t DataConvert::convertColumnDate(const char* dataOrg, CalpontDateTimeFormat dateFormat, int& status, + unsigned int dataOrgLen) { - status = 0; - const char* p; - p = dataOrg; - char fld[10]; - int32_t value = 0; - - if ( dateFormat != CALPONTDATE_ENUM ) - { - status = -1; - return value; - } - - // @bug 5787: allow for leading blanks - unsigned int dataLen = dataOrgLen; - - if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) - { - unsigned nblanks = 0; - - for (unsigned nn = 0; nn < dataOrgLen; nn++) - { - if (dataOrg[nn] == ' ') - nblanks++; - else - break; - } - - p = dataOrg + nblanks; - dataLen = dataOrgLen - nblanks; - } - - if ( dataLen < 10) - { - status = -1; - return value; - } - - int inYear, inMonth, inDay; - memcpy( fld, p, 4); - fld[4] = '\0'; - - inYear = strtol(fld, 0, 10); - - memcpy( fld, p + 5, 2); - fld[2] = '\0'; - - inMonth = strtol(fld, 0, 10); - - memcpy( fld, p + 8, 2); - fld[2] = '\0'; - - inDay = strtol(fld, 0, 10); - - if ( isDateValid (inDay, inMonth, inYear)) - { - Date aDay; - aDay.year = inYear; - aDay.month = inMonth; - aDay.day = inDay; - memcpy( &value, &aDay, 4); - } - else - { - status = -1; - } + status = 0; + const char* p; + p = dataOrg; + char fld[10]; + int32_t value = 0; + if (dateFormat != CALPONTDATE_ENUM) + { + status = -1; return value; + } + + // @bug 5787: allow for leading blanks + unsigned int dataLen = dataOrgLen; + + if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) + { + unsigned nblanks = 0; + + for (unsigned nn = 0; nn < dataOrgLen; nn++) + { + if (dataOrg[nn] == ' ') + nblanks++; + else + break; + } + + p = dataOrg + nblanks; + dataLen = dataOrgLen - nblanks; + } + + if (dataLen < 10) + { + status = -1; + return value; + } + + int inYear, inMonth, inDay; + memcpy(fld, p, 4); + fld[4] = '\0'; + + inYear = strtol(fld, 0, 10); + + memcpy(fld, p + 5, 2); + fld[2] = '\0'; + + inMonth = strtol(fld, 0, 10); + + memcpy(fld, p + 8, 2); + fld[2] = '\0'; + + inDay = strtol(fld, 0, 10); + + if (isDateValid(inDay, inMonth, inYear)) + { + Date aDay; + aDay.year = inYear; + aDay.month = inMonth; + aDay.day = inDay; + memcpy(&value, &aDay, 4); + } + else + { + status = -1; + } + + return value; } //------------------------------------------------------------------------------ // Verify that specified date is valid //------------------------------------------------------------------------------ -bool DataConvert::isColumnDateValid( int32_t date ) +bool DataConvert::isColumnDateValid(int32_t date) { - Date d; - void* dp = static_cast(&d); - memcpy(dp, &date, sizeof(int32_t)); - return (isDateValid(d.day, d.month, d.year)); + Date d; + void* dp = static_cast(&d); + memcpy(dp, &date, sizeof(int32_t)); + return (isDateValid(d.day, d.month, d.year)); } //------------------------------------------------------------------------------ // Convert date/time string to binary date/time. Used by BulkLoad. //------------------------------------------------------------------------------ -int64_t DataConvert::convertColumnDatetime( - const char* dataOrg, - CalpontDateTimeFormat datetimeFormat, - int& status, - unsigned int dataOrgLen ) +int64_t DataConvert::convertColumnDatetime(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen) { - status = 0; - const char* p; - p = dataOrg; - char fld[10]; - int64_t value = 0; + status = 0; + const char* p; + p = dataOrg; + char fld[10]; + int64_t value = 0; - if ( datetimeFormat != CALPONTDATETIME_ENUM ) + if (datetimeFormat != CALPONTDATETIME_ENUM) + { + status = -1; + return value; + } + + // @bug 5787: allow for leading blanks + unsigned int dataLen = dataOrgLen; + + if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) + { + unsigned nblanks = 0; + + for (unsigned nn = 0; nn < dataOrgLen; nn++) { - status = -1; - return value; + if (dataOrg[nn] == ' ') + nblanks++; + else + break; } - // @bug 5787: allow for leading blanks - unsigned int dataLen = dataOrgLen; + p = dataOrg + nblanks; + dataLen = dataOrgLen - nblanks; + } - if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) + if (dataLen < 10) + { + status = -1; + return value; + } + + int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + memcpy(fld, p, 4); + fld[4] = '\0'; + + inYear = strtol(fld, 0, 10); + + memcpy(fld, p + 5, 2); + fld[2] = '\0'; + + inMonth = strtol(fld, 0, 10); + + memcpy(fld, p + 8, 2); + fld[2] = '\0'; + + inDay = strtol(fld, 0, 10); + + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + + if (dataLen > 12) + { + // For backwards compatability we still allow leading blank + if ((!isdigit(p[11]) && (p[11] != ' ')) || !isdigit(p[12])) { - unsigned nblanks = 0; - - for (unsigned nn = 0; nn < dataOrgLen; nn++) - { - if (dataOrg[nn] == ' ') - nblanks++; - else - break; - } - - p = dataOrg + nblanks; - dataLen = dataOrgLen - nblanks; + status = -1; + return value; } - if ( dataLen < 10) - { - status = -1; - return value; - } - - int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; - memcpy( fld, p, 4); - fld[4] = '\0'; - - inYear = strtol(fld, 0, 10); - - memcpy( fld, p + 5, 2); + memcpy(fld, p + 11, 2); fld[2] = '\0'; - inMonth = strtol(fld, 0, 10); + inHour = strtol(fld, 0, 10); - memcpy( fld, p + 8, 2); - fld[2] = '\0'; - - inDay = strtol(fld, 0, 10); - - inHour = 0; - inMinute = 0; - inSecond = 0; - inMicrosecond = 0; - - if (dataLen > 12) + if (dataLen > 15) { - // For backwards compatability we still allow leading blank - if ((!isdigit(p[11]) && (p[11] != ' ')) || - !isdigit(p[12])) + if (!isdigit(p[14]) || !isdigit(p[15])) + { + status = -1; + return value; + } + + memcpy(fld, p + 14, 2); + fld[2] = '\0'; + + inMinute = strtol(fld, 0, 10); + + if (dataLen > 18) + { + if (!isdigit(p[17]) || !isdigit(p[18])) { - status = -1; - return value; + status = -1; + return value; } - memcpy( fld, p + 11, 2); + memcpy(fld, p + 17, 2); fld[2] = '\0'; - inHour = strtol(fld, 0, 10); + inSecond = strtol(fld, 0, 10); - if (dataLen > 15) + if (dataLen > 20) { - if (!isdigit(p[14]) || !isdigit(p[15])) - { - status = -1; - return value; - } + unsigned int microFldLen = dataLen - 20; - memcpy( fld, p + 14, 2); - fld[2] = '\0'; + if (microFldLen > (sizeof(fld) - 1)) + microFldLen = sizeof(fld) - 1; - inMinute = strtol(fld, 0, 10); - - if (dataLen > 18) - { - if (!isdigit(p[17]) || !isdigit(p[18])) - { - status = -1; - return value; - } - - memcpy( fld, p + 17, 2); - fld[2] = '\0'; - - inSecond = strtol(fld, 0, 10); - - if (dataLen > 20) - { - unsigned int microFldLen = dataLen - 20; - - if (microFldLen > (sizeof(fld) - 1)) - microFldLen = sizeof(fld) - 1; - - memcpy( fld, p + 20, microFldLen); - fld[microFldLen] = '\0'; - inMicrosecond = strtol(fld, 0, 10); - } - } + memcpy(fld, p + 20, microFldLen); + fld[microFldLen] = '\0'; + inMicrosecond = strtol(fld, 0, 10); } + } } + } - if ( isDateValid (inDay, inMonth, inYear) && - isDateTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) - { - DateTime aDatetime; - aDatetime.year = inYear; - aDatetime.month = inMonth; - aDatetime.day = inDay; - aDatetime.hour = inHour; - aDatetime.minute = inMinute; - aDatetime.second = inSecond; - aDatetime.msecond = inMicrosecond; + if (isDateValid(inDay, inMonth, inYear) && isDateTimeValid(inHour, inMinute, inSecond, inMicrosecond)) + { + DateTime aDatetime; + aDatetime.year = inYear; + aDatetime.month = inMonth; + aDatetime.day = inDay; + aDatetime.hour = inHour; + aDatetime.minute = inMinute; + aDatetime.second = inSecond; + aDatetime.msecond = inMicrosecond; - memcpy( &value, &aDatetime, 8); - } - else - { - status = -1; - } + memcpy(&value, &aDatetime, 8); + } + else + { + status = -1; + } - return value; + return value; } //------------------------------------------------------------------------------ // Convert timestamp string to binary timestamp. Used by BulkLoad. // Most of this code is taken from DataConvert::convertColumnDatetime //------------------------------------------------------------------------------ -int64_t DataConvert::convertColumnTimestamp( - const char* dataOrg, - CalpontDateTimeFormat datetimeFormat, - int& status, - unsigned int dataOrgLen, - const std::string& timeZone ) +int64_t DataConvert::convertColumnTimestamp(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen, const std::string& timeZone) { - char tmbuf[64]; - std::string dataOrgTemp = dataOrg; - if (dataOrgTemp.substr(0, 19) == "0000-00-00 00:00:00") + char tmbuf[64]; + std::string dataOrgTemp = dataOrg; + if (dataOrgTemp.substr(0, 19) == "0000-00-00 00:00:00") + { + return 0; + } + + // this is the default value of the first timestamp field in a table, + // which is stored in the system catalog + if (strcmp(dataOrg, "current_timestamp() ON UPDATE current_timestamp()") == 0) + { + struct timeval tv; + gettimeofday(&tv, 0); + MySQLTime time; + gmtSecToMySQLTime(tv.tv_sec, time, timeZone); + sprintf(tmbuf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, + time.minute, time.second, tv.tv_usec); + dataOrg = tmbuf; + dataOrgLen = strlen(tmbuf); + } + + status = 0; + const char* p; + p = dataOrg; + char fld[10]; + int64_t value = 0; + + if (datetimeFormat != CALPONTDATETIME_ENUM) + { + status = -1; + return value; + } + + unsigned int dataLen = dataOrgLen; + + if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) + { + unsigned nblanks = 0; + + for (unsigned nn = 0; nn < dataOrgLen; nn++) { - return 0; + if (dataOrg[nn] == ' ') + nblanks++; + else + break; } - // this is the default value of the first timestamp field in a table, - // which is stored in the system catalog - if (strcmp(dataOrg, "current_timestamp() ON UPDATE current_timestamp()") == 0) + p = dataOrg + nblanks; + dataLen = dataOrgLen - nblanks; + } + + if (dataLen < 10) + { + status = -1; + return value; + } + + int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + memcpy(fld, p, 4); + fld[4] = '\0'; + + inYear = strtol(fld, 0, 10); + + memcpy(fld, p + 5, 2); + fld[2] = '\0'; + + inMonth = strtol(fld, 0, 10); + + memcpy(fld, p + 8, 2); + fld[2] = '\0'; + + inDay = strtol(fld, 0, 10); + + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + + if (dataLen > 12) + { + // For backwards compatability we still allow leading blank + if ((!isdigit(p[11]) && (p[11] != ' ')) || !isdigit(p[12])) { - struct timeval tv; - gettimeofday(&tv, 0); - MySQLTime time; - gmtSecToMySQLTime(tv.tv_sec, time, timeZone); - sprintf(tmbuf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, time.minute, time.second, tv.tv_usec); - dataOrg = tmbuf; - dataOrgLen = strlen(tmbuf); + status = -1; + return value; } - status = 0; - const char* p; - p = dataOrg; - char fld[10]; - int64_t value = 0; - - if ( datetimeFormat != CALPONTDATETIME_ENUM ) - { - status = -1; - return value; - } - - unsigned int dataLen = dataOrgLen; - - if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) - { - unsigned nblanks = 0; - - for (unsigned nn = 0; nn < dataOrgLen; nn++) - { - if (dataOrg[nn] == ' ') - nblanks++; - else - break; - } - - p = dataOrg + nblanks; - dataLen = dataOrgLen - nblanks; - } - - if ( dataLen < 10) - { - status = -1; - return value; - } - - int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; - memcpy( fld, p, 4); - fld[4] = '\0'; - - inYear = strtol(fld, 0, 10); - - memcpy( fld, p + 5, 2); + memcpy(fld, p + 11, 2); fld[2] = '\0'; - inMonth = strtol(fld, 0, 10); + inHour = strtol(fld, 0, 10); - memcpy( fld, p + 8, 2); - fld[2] = '\0'; - - inDay = strtol(fld, 0, 10); - - inHour = 0; - inMinute = 0; - inSecond = 0; - inMicrosecond = 0; - - if (dataLen > 12) + if (dataLen > 15) { - // For backwards compatability we still allow leading blank - if ((!isdigit(p[11]) && (p[11] != ' ')) || - !isdigit(p[12])) + if (!isdigit(p[14]) || !isdigit(p[15])) + { + status = -1; + return value; + } + + memcpy(fld, p + 14, 2); + fld[2] = '\0'; + + inMinute = strtol(fld, 0, 10); + + if (dataLen > 18) + { + if (!isdigit(p[17]) || !isdigit(p[18])) { - status = -1; - return value; + status = -1; + return value; } - memcpy( fld, p + 11, 2); + memcpy(fld, p + 17, 2); fld[2] = '\0'; - inHour = strtol(fld, 0, 10); + inSecond = strtol(fld, 0, 10); - if (dataLen > 15) + if (dataLen > 20) { - if (!isdigit(p[14]) || !isdigit(p[15])) - { - status = -1; - return value; - } + unsigned int microFldLen = dataLen - 20; - memcpy( fld, p + 14, 2); - fld[2] = '\0'; + if (microFldLen > (sizeof(fld) - 1)) + microFldLen = sizeof(fld) - 1; - inMinute = strtol(fld, 0, 10); - - if (dataLen > 18) - { - if (!isdigit(p[17]) || !isdigit(p[18])) - { - status = -1; - return value; - } - - memcpy( fld, p + 17, 2); - fld[2] = '\0'; - - inSecond = strtol(fld, 0, 10); - - if (dataLen > 20) - { - unsigned int microFldLen = dataLen - 20; - - if (microFldLen > (sizeof(fld) - 1)) - microFldLen = sizeof(fld) - 1; - - memcpy( fld, p + 20, microFldLen); - fld[microFldLen] = '\0'; - inMicrosecond = strtol(fld, 0, 10); - } - } + memcpy(fld, p + 20, microFldLen); + fld[microFldLen] = '\0'; + inMicrosecond = strtol(fld, 0, 10); } + } } + } - if ( isDateValid (inDay, inMonth, inYear) && - isDateTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) + if (isDateValid(inDay, inMonth, inYear) && isDateTimeValid(inHour, inMinute, inSecond, inMicrosecond)) + { + MySQLTime m_time; + m_time.year = inYear; + m_time.month = inMonth; + m_time.day = inDay; + m_time.hour = inHour; + m_time.minute = inMinute; + m_time.second = inSecond; + m_time.second_part = inMicrosecond; + + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone, isValid); + + if (!isValid) { - MySQLTime m_time; - m_time.year = inYear; - m_time.month = inMonth; - m_time.day = inDay; - m_time.hour = inHour; - m_time.minute = inMinute; - m_time.second = inSecond; - m_time.second_part = inMicrosecond; - - bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone, isValid); - - if (!isValid) - { - status = -1; - return value; - } - - TimeStamp timestamp; - timestamp.second = seconds; - timestamp.msecond = m_time.second_part; - - memcpy( &value, ×tamp, 8 ); - } - else - { - status = -1; + status = -1; + return value; } - return value; + TimeStamp timestamp; + timestamp.second = seconds; + timestamp.msecond = m_time.second_part; + + memcpy(&value, ×tamp, 8); + } + else + { + status = -1; + } + + return value; } //------------------------------------------------------------------------------ // Convert time string to binary time. Used by BulkLoad. // Most of this is taken from str_to_time in sql-common/my_time.c //------------------------------------------------------------------------------ -int64_t DataConvert::convertColumnTime( - const char* dataOrg, - CalpontDateTimeFormat datetimeFormat, - int& status, - unsigned int dataOrgLen ) +int64_t DataConvert::convertColumnTime(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, int& status, + unsigned int dataOrgLen) { - status = 0; - char* p; - char* retp = NULL; - char* savePoint = NULL; - p = const_cast(dataOrg); - int64_t value = 0; - int inHour, inMinute, inSecond, inMicrosecond; - inHour = 0; - inMinute = 0; - inSecond = 0; - inMicrosecond = 0; - bool isNeg = false; - - if ( datetimeFormat != CALPONTTIME_ENUM ) - { - status = -1; - return value; - } - - if (dataOrgLen == 0) - { - return value; - } - - if (dataOrgLen < 3) - { - // Not enough chars to be a time - status = -1; - return value; - } - - if (p[0] == '-') - { - isNeg = true; - } - - errno = 0; - - p = strtok_r(p, ":.", &savePoint); - inHour = strtol(p, &retp, 10); - - if (errno || !retp) - { - status = -1; - return value; - } - - p = strtok_r(NULL, ":.", &savePoint); - - if (p == NULL) - { - status = -1; - return value; - } - - inMinute = strtol(p, &retp, 10); - - if (errno || !retp) - { - status = -1; - return value; - } - - p = strtok_r(NULL, ":.", &savePoint); - - if (p == NULL) - { - status = -1; - return value; - } - - inSecond = strtol(p, &retp, 10); - - if (errno || !retp) - { - status = -1; - return value; - } - - p = strtok_r(NULL, ":.", &savePoint); - - if (p != NULL) - { - inMicrosecond = strtol(p, &retp, 10); - - if (errno || !retp) - { - status = -1; - return value; - } - } - - if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) - { - Time atime; - atime.hour = inHour; - atime.minute = inMinute; - atime.second = inSecond; - atime.msecond = inMicrosecond; - atime.is_neg = isNeg; - - memcpy( &value, &atime, 8); - } - else - { - // Emulate MariaDB's time saturation - if (inHour > 838) - { - Time atime; - atime.hour = 838; - atime.minute = 59; - atime.second = 59; - atime.msecond = 999999; - atime.is_neg = false; - memcpy( &value, &atime, 8); - } - else if (inHour < -838) - { - Time atime; - atime.hour = -838; - atime.minute = 59; - atime.second = 59; - atime.msecond = 999999; - atime.is_neg = false; - memcpy( &value, &atime, 8); - } - - // If neither of the above match then we return a 0 time - - status = -1; - } + status = 0; + char* p; + char* retp = NULL; + char* savePoint = NULL; + p = const_cast(dataOrg); + int64_t value = 0; + int inHour, inMinute, inSecond, inMicrosecond; + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + bool isNeg = false; + if (datetimeFormat != CALPONTTIME_ENUM) + { + status = -1; return value; + } + if (dataOrgLen == 0) + { + return value; + } + + if (dataOrgLen < 3) + { + // Not enough chars to be a time + status = -1; + return value; + } + + if (p[0] == '-') + { + isNeg = true; + } + + errno = 0; + + p = strtok_r(p, ":.", &savePoint); + inHour = strtol(p, &retp, 10); + + if (errno || !retp) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p == NULL) + { + status = -1; + return value; + } + + inMinute = strtol(p, &retp, 10); + + if (errno || !retp) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p == NULL) + { + status = -1; + return value; + } + + inSecond = strtol(p, &retp, 10); + + if (errno || !retp) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p != NULL) + { + inMicrosecond = strtol(p, &retp, 10); + + if (errno || !retp) + { + status = -1; + return value; + } + } + + if (isTimeValid(inHour, inMinute, inSecond, inMicrosecond)) + { + Time atime; + atime.hour = inHour; + atime.minute = inMinute; + atime.second = inSecond; + atime.msecond = inMicrosecond; + atime.is_neg = isNeg; + + memcpy(&value, &atime, 8); + } + else + { + // Emulate MariaDB's time saturation + if (inHour > 838) + { + Time atime; + atime.hour = 838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + atime.is_neg = false; + memcpy(&value, &atime, 8); + } + else if (inHour < -838) + { + Time atime; + atime.hour = -838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + atime.is_neg = false; + memcpy(&value, &atime, 8); + } + + // If neither of the above match then we return a 0 time + + status = -1; + } + + return value; } - //------------------------------------------------------------------------------ // Verify that specified datetime is valid //------------------------------------------------------------------------------ -bool DataConvert::isColumnDateTimeValid( int64_t dateTime ) +bool DataConvert::isColumnDateTimeValid(int64_t dateTime) { - DateTime dt; - void* dtp = static_cast(&dt); - memcpy(dtp, &dateTime, sizeof(uint64_t)); + DateTime dt; + void* dtp = static_cast(&dt); + memcpy(dtp, &dateTime, sizeof(uint64_t)); - if (isDateValid(dt.day, dt.month, dt.year)) - return isDateTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); + if (isDateValid(dt.day, dt.month, dt.year)) + return isDateTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); - return false; + return false; } -bool DataConvert::isColumnTimeValid( int64_t time ) +bool DataConvert::isColumnTimeValid(int64_t time) { - Time dt; - void* dtp = static_cast(&dt); - memcpy(dtp, &time, sizeof(uint64_t)); + Time dt; + void* dtp = static_cast(&dt); + memcpy(dtp, &time, sizeof(uint64_t)); - return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); + return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); } -bool DataConvert::isColumnTimeStampValid( int64_t timeStamp ) +bool DataConvert::isColumnTimeStampValid(int64_t timeStamp) { - TimeStamp dt; - void* dtp = static_cast(&dt); - memcpy(dtp, &timeStamp, sizeof(uint64_t)); + TimeStamp dt; + void* dtp = static_cast(&dt); + memcpy(dtp, &timeStamp, sizeof(uint64_t)); - return isTimestampValid(dt.second, dt.msecond); + return isTimestampValid(dt.second, dt.msecond); } -std::string DataConvert::dateToString( int datevalue ) +std::string DataConvert::dateToString(int datevalue) { - // @bug 4703 abandon multiple ostringstream's for conversion - Date d(datevalue); - const int DATETOSTRING_LEN = 12; // YYYY-MM-DD\0 - char buf[DATETOSTRING_LEN]; + // @bug 4703 abandon multiple ostringstream's for conversion + Date d(datevalue); + const int DATETOSTRING_LEN = 12; // YYYY-MM-DD\0 + char buf[DATETOSTRING_LEN]; - sprintf(buf, "%04d-%02d-%02d", d.year, d.month, d.day); - return buf; + sprintf(buf, "%04d-%02d-%02d", d.year, d.month, d.day); + return buf; } -std::string DataConvert::datetimeToString( long long datetimevalue, long decimals ) +std::string DataConvert::datetimeToString(long long datetimevalue, long decimals) { - // 10 is default which means we don't need microseconds - if (decimals > 6 || decimals < 0) - { - decimals = 0; - } + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } - // @bug 4703 abandon multiple ostringstream's for conversion - DateTime dt(datetimevalue); - const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 - char buf[DATETIMETOSTRING_LEN]; + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 + char buf[DATETIMETOSTRING_LEN]; - sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); - if (dt.msecond && decimals) - { - // Pad start with zeros - sprintf(buf + strlen(buf), ".%0*d", (int)decimals, dt.msecond); - } + if (dt.msecond && decimals) + { + // Pad start with zeros + sprintf(buf + strlen(buf), ".%0*d", (int)decimals, dt.msecond); + } - return buf; + return buf; } -std::string DataConvert::timestampToString( long long timestampvalue, const std::string& timezone, long decimals ) +std::string DataConvert::timestampToString(long long timestampvalue, const std::string& timezone, + long decimals) { - // 10 is default which means we don't need microseconds - if (decimals > 6 || decimals < 0) - { - decimals = 0; - } + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } - TimeStamp timestamp(timestampvalue); - int64_t seconds = timestamp.second; + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; - MySQLTime time; - gmtSecToMySQLTime(seconds, time, timezone); + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); - const int TIMESTAMPTOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 - char buf[TIMESTAMPTOSTRING_LEN]; + const int TIMESTAMPTOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 + char buf[TIMESTAMPTOSTRING_LEN]; - sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", time.year, time.month, time.day, time.hour, time.minute, time.second); + sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", time.year, time.month, time.day, time.hour, time.minute, + time.second); - if (timestamp.msecond && decimals) - { - // Pad start with zeros - sprintf(buf + strlen(buf), ".%0*d", (int)decimals, timestamp.msecond); - } + if (timestamp.msecond && decimals) + { + // Pad start with zeros + sprintf(buf + strlen(buf), ".%0*d", (int)decimals, timestamp.msecond); + } - return buf; + return buf; } -std::string DataConvert::timeToString( long long timevalue, long decimals ) +std::string DataConvert::timeToString(long long timevalue, long decimals) { - // 10 is default which means we don't need microseconds - if (decimals > 6 || decimals < 0) - { - decimals = 0; - } + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } - // @bug 4703 abandon multiple ostringstream's for conversion - Time dt(timevalue); - const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 - char buf[TIMETOSTRING_LEN]; - char* outbuf = buf; + // @bug 4703 abandon multiple ostringstream's for conversion + Time dt(timevalue); + const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 + char buf[TIMETOSTRING_LEN]; + char* outbuf = buf; - if ((dt.hour >= 0) && dt.is_neg) - { - outbuf[0] = '-'; - outbuf++; - } + if ((dt.hour >= 0) && dt.is_neg) + { + outbuf[0] = '-'; + outbuf++; + } - sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); - if (dt.msecond && decimals) - { - // Pad start with zeros - sprintf(buf + strlen(buf), ".%0*d", (int)decimals, dt.msecond); - } + if (dt.msecond && decimals) + { + // Pad start with zeros + sprintf(buf + strlen(buf), ".%0*d", (int)decimals, dt.msecond); + } - return buf; + return buf; } -std::string DataConvert::dateToString1( int datevalue ) +std::string DataConvert::dateToString1(int datevalue) { - // @bug 4703 abandon multiple ostringstream's for conversion - Date d(datevalue); - const int DATETOSTRING1_LEN = 10; // YYYYMMDD\0 - char buf[DATETOSTRING1_LEN]; + // @bug 4703 abandon multiple ostringstream's for conversion + Date d(datevalue); + const int DATETOSTRING1_LEN = 10; // YYYYMMDD\0 + char buf[DATETOSTRING1_LEN]; - sprintf(buf, "%04d%02d%02d", d.year, d.month, d.day); - return buf; + sprintf(buf, "%04d%02d%02d", d.year, d.month, d.day); + return buf; } -std::string DataConvert::datetimeToString1( long long datetimevalue ) +std::string DataConvert::datetimeToString1(long long datetimevalue) { - // @bug 4703 abandon multiple ostringstream's for conversion - DateTime dt(datetimevalue); - // Interesting, gcc 7 says the sprintf below generates between 21 and 23 bytes of output. - const int DATETIMETOSTRING1_LEN = 23; // YYYYMMDDHHMMSSmmmmmm\0 - char buf[DATETIMETOSTRING1_LEN]; + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + // Interesting, gcc 7 says the sprintf below generates between 21 and 23 bytes of output. + const int DATETIMETOSTRING1_LEN = 23; // YYYYMMDDHHMMSSmmmmmm\0 + char buf[DATETIMETOSTRING1_LEN]; - sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond); - return buf; + sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, + dt.msecond); + return buf; } -std::string DataConvert::timestampToString1( long long timestampvalue, const std::string& timezone ) +std::string DataConvert::timestampToString1(long long timestampvalue, const std::string& timezone) { - const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 - char buf[TIMESTAMPTOSTRING1_LEN]; + const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 + char buf[TIMESTAMPTOSTRING1_LEN]; - TimeStamp timestamp(timestampvalue); - int64_t seconds = timestamp.second; - MySQLTime time; - gmtSecToMySQLTime(seconds, time, timezone); + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); - sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", time.year, time.month, time.day, time.hour, time.minute, time.second, timestamp.msecond); - return buf; + sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", time.year, time.month, time.day, time.hour, time.minute, + time.second, timestamp.msecond); + return buf; } -std::string DataConvert::timeToString1( long long datetimevalue ) +std::string DataConvert::timeToString1(long long datetimevalue) { - // @bug 4703 abandon multiple ostringstream's for conversion - DateTime dt(datetimevalue); - const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 - char buf[TIMETOSTRING1_LEN]; + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 + char buf[TIMETOSTRING1_LEN]; - char* outbuf = buf; + char* outbuf = buf; - sprintf(outbuf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); - return buf; + sprintf(outbuf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + return buf; } int64_t DataConvert::dateToInt(const string& date) { - return stringToDate(date); + return stringToDate(date); } int64_t DataConvert::datetimeToInt(const string& datetime) { - return stringToDatetime(datetime); + return stringToDatetime(datetime); } int64_t DataConvert::timestampToInt(const string& timestamp, const string& timeZone) { - return stringToTimestamp(timestamp, timeZone); + return stringToTimestamp(timestamp, timeZone); } int64_t DataConvert::timeToInt(const string& time) { - return stringToTime(time); + return stringToTime(time); } int64_t DataConvert::stringToDate(const string& data) { - Date aDay; + Date aDay; - if ( stringToDateStruct( data, aDay ) ) - { - uint32_t temp = getUInt32LE((const char*) &aDay); - return ((temp & 0xFFFFFFC0) | 0x3E); - } - else - return -1; + if (stringToDateStruct(data, aDay)) + { + uint32_t temp = getUInt32LE((const char*)&aDay); + return ((temp & 0xFFFFFFC0) | 0x3E); + } + else + return -1; } int64_t DataConvert::stringToDatetime(const string& data, bool* date) { - DateTime dtime; + DateTime dtime; - if ( stringToDatetimeStruct( data, dtime, date ) ) - return getUInt64LE((const char*) &dtime); - else - return -1; + if (stringToDatetimeStruct(data, dtime, date)) + return getUInt64LE((const char*)&dtime); + else + return -1; } int64_t DataConvert::stringToTimestamp(const string& data, const string& timeZone) { - TimeStamp aTimestamp; + TimeStamp aTimestamp; - if ( stringToTimestampStruct( data, aTimestamp, timeZone ) ) - return getUInt64LE((const char*) &aTimestamp); - else - return -1; + if (stringToTimestampStruct(data, aTimestamp, timeZone)) + return getUInt64LE((const char*)&aTimestamp); + else + return -1; } /* This is really painful and expensive b/c it seems the input is not normalized or sanitized. That should really be done on ingestion. */ int64_t DataConvert::intToDate(int64_t data) { - char buf[21] = {0}; - Date aday; + char buf[21] = {0}; + Date aday; - if (data == 0) - { - aday.year = 0; - aday.month = 0; - aday.day = 0; - return getUInt32LE((const char*) &aday); - } + if (data == 0) + { + aday.year = 0; + aday.month = 0; + aday.day = 0; + return getUInt32LE((const char*)&aday); + } - // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # - // into 15 bytes, however, that appears to be intentional. + // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # + // into 15 bytes, however, that appears to be intentional. #if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" - snprintf( buf, 15, "%llu", (long long unsigned int)data); + snprintf(buf, 15, "%llu", (long long unsigned int)data); #pragma GCC diagnostic pop #else - snprintf( buf, 15, "%llu", (long long unsigned int)data); + snprintf(buf, 15, "%llu", (long long unsigned int)data); #endif - string year, month, day, hour, min, sec, msec; - int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0; + string year, month, day, hour, min, sec, msec; + int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0; - switch (strlen(buf)) - { - case 14: - year = string(buf, 4); - month = string(buf + 4, 2); - day = string(buf + 6, 2); - hour = string(buf + 8, 2); - min = string(buf + 10, 2); - sec = string(buf + 12, 2); - msec = string(buf + 14, 6); - break; + switch (strlen(buf)) + { + case 14: + year = string(buf, 4); + month = string(buf + 4, 2); + day = string(buf + 6, 2); + hour = string(buf + 8, 2); + min = string(buf + 10, 2); + sec = string(buf + 12, 2); + msec = string(buf + 14, 6); + break; - case 12: - year = string(buf, 2); - month = string(buf + 2, 2); - day = string(buf + 4, 2); - hour = string(buf + 6, 2); - min = string(buf + 8, 2); - sec = string(buf + 10, 2); - msec = string(buf + 12, 6); - break; + case 12: + year = string(buf, 2); + month = string(buf + 2, 2); + day = string(buf + 4, 2); + hour = string(buf + 6, 2); + min = string(buf + 8, 2); + sec = string(buf + 10, 2); + msec = string(buf + 12, 6); + break; - case 10: - month = string(buf, 2); - day = string(buf + 2, 2); - hour = string(buf + 4, 2); - min = string(buf + 6, 2); - sec = string(buf + 8, 2); - msec = string(buf + 10, 6); - break; + case 10: + month = string(buf, 2); + day = string(buf + 2, 2); + hour = string(buf + 4, 2); + min = string(buf + 6, 2); + sec = string(buf + 8, 2); + msec = string(buf + 10, 6); + break; - case 9: - month = string(buf, 1); - day = string(buf + 1, 2); - hour = string(buf + 3, 2); - min = string(buf + 5, 2); - sec = string(buf + 7, 2); - msec = string(buf + 9, 6); - break; + case 9: + month = string(buf, 1); + day = string(buf + 1, 2); + hour = string(buf + 3, 2); + min = string(buf + 5, 2); + sec = string(buf + 7, 2); + msec = string(buf + 9, 6); + break; - case 8: - year = string(buf, 4); - month = string(buf + 4, 2); - day = string(buf + 6, 2); - break; + case 8: + year = string(buf, 4); + month = string(buf + 4, 2); + day = string(buf + 6, 2); + break; - case 6: - year = string(buf, 2); - month = string(buf + 2, 2); - day = string(buf + 4, 2); - break; + case 6: + year = string(buf, 2); + month = string(buf + 2, 2); + day = string(buf + 4, 2); + break; - case 4: - month = string(buf, 2); - day = string(buf + 2, 2); - break; + case 4: + month = string(buf, 2); + day = string(buf + 2, 2); + break; - case 3: - month = string(buf, 1); - day = string(buf + 1, 2); - break; + case 3: + month = string(buf, 1); + day = string(buf + 1, 2); + break; - default: - return -1; - } + default: return -1; + } - if (year.empty()) - { - // MMDD format. assume current year - time_t calender_time; - struct tm todays_date; - calender_time = time(NULL); - localtime_r(&calender_time, &todays_date); - y = todays_date.tm_year + 1900; - } - else - { - y = atoi(year.c_str()); - } + if (year.empty()) + { + // MMDD format. assume current year + time_t calender_time; + struct tm todays_date; + calender_time = time(NULL); + localtime_r(&calender_time, &todays_date); + y = todays_date.tm_year + 1900; + } + else + { + y = atoi(year.c_str()); + } - m = atoi(month.c_str()); - d = atoi(day.c_str()); - h = atoi(hour.c_str()); - minute = atoi(min.c_str()); - s = atoi(sec.c_str()); - ms = atoi(msec.c_str()); + m = atoi(month.c_str()); + d = atoi(day.c_str()); + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + ms = atoi(msec.c_str()); - //if (!isDateValid(d, m, y)) - // return -1; - if (!isDateValid(d, m, y) || !isDateTimeValid(h, minute, s, ms)) - return -1; + // if (!isDateValid(d, m, y)) + // return -1; + if (!isDateValid(d, m, y) || !isDateTimeValid(h, minute, s, ms)) + return -1; - aday.year = y; - aday.month = m; - aday.day = d; - return getUInt32LE((const char*) &aday); + aday.year = y; + aday.month = m; + aday.day = d; + return getUInt32LE((const char*)&aday); } /* This is really painful and expensive b/c it seems the input is not normalized or sanitized. That should really be done on ingestion. */ int64_t DataConvert::intToDatetime(int64_t data, bool* date) { - bool isDate = false; - char buf[21] = {0}; - DateTime adaytime; + bool isDate = false; + char buf[21] = {0}; + DateTime adaytime; - if (data == 0) - { - adaytime.year = 0; - adaytime.month = 0; - adaytime.day = 0; - adaytime.hour = 0; - adaytime.minute = 0; - adaytime.second = 0; - adaytime.msecond = 0; + if (data == 0) + { + adaytime.year = 0; + adaytime.month = 0; + adaytime.day = 0; + adaytime.hour = 0; + adaytime.minute = 0; + adaytime.second = 0; + adaytime.msecond = 0; - if (date) - *date = true; + if (date) + *date = true; - return getUInt64LE((const char*) &adaytime); - } + return getUInt64LE((const char*)&adaytime); + } - // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # - // into 15 bytes, however, that appears to be intentional. + // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # + // into 15 bytes, however, that appears to be intentional. #if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" - snprintf( buf, 15, "%llu", (long long unsigned int)data); + snprintf(buf, 15, "%llu", (long long unsigned int)data); #pragma GCC diagnostic pop #else - snprintf( buf, 15, "%llu", (long long unsigned int)data); + snprintf(buf, 15, "%llu", (long long unsigned int)data); #endif - //string date = buf; - string year, month, day, hour, min, sec, msec; - int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0; + // string date = buf; + string year, month, day, hour, min, sec, msec; + int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0; - switch (strlen(buf)) + switch (strlen(buf)) + { + case 14: + year = string(buf, 4); + month = string(buf + 4, 2); + day = string(buf + 6, 2); + hour = string(buf + 8, 2); + min = string(buf + 10, 2); + sec = string(buf + 12, 2); + break; + + case 12: + year = string(buf, 2); + month = string(buf + 2, 2); + day = string(buf + 4, 2); + hour = string(buf + 6, 2); + min = string(buf + 8, 2); + sec = string(buf + 10, 2); + break; + + case 10: + month = string(buf, 2); + day = string(buf + 2, 2); + hour = string(buf + 4, 2); + min = string(buf + 6, 2); + sec = string(buf + 8, 2); + break; + + case 9: + month = string(buf, 1); + day = string(buf + 1, 2); + hour = string(buf + 3, 2); + min = string(buf + 5, 2); + sec = string(buf + 7, 2); + break; + + case 8: + year = string(buf, 4); + month = string(buf + 4, 2); + day = string(buf + 6, 2); + isDate = true; + break; + + case 6: + year = string(buf, 2); + month = string(buf + 2, 2); + day = string(buf + 4, 2); + isDate = true; + break; + + case 4: + month = string(buf, 2); + day = string(buf + 2, 2); + break; + + case 3: + month = string(buf, 1); + day = string(buf + 1, 2); + isDate = true; + break; + + default: return -1; + } + + if (year.empty()) + { + // MMDD format. assume current year + time_t calender_time; + struct tm todays_date; + calender_time = time(NULL); + localtime_r(&calender_time, &todays_date); + y = todays_date.tm_year + 1900; + } + else + { + y = atoi(year.c_str()); + + // special handling for 2-byte year + if (year.length() == 2) { - case 14: - year = string(buf, 4); - month = string(buf + 4, 2); - day = string(buf + 6, 2); - hour = string(buf + 8, 2); - min = string(buf + 10, 2); - sec = string(buf + 12, 2); - break; - - case 12: - year = string(buf, 2); - month = string(buf + 2, 2); - day = string(buf + 4, 2); - hour = string(buf + 6, 2); - min = string(buf + 8, 2); - sec = string(buf + 10, 2); - break; - - case 10: - month = string(buf, 2); - day = string(buf + 2, 2); - hour = string(buf + 4, 2); - min = string(buf + 6, 2); - sec = string(buf + 8, 2); - break; - - case 9: - month = string(buf, 1); - day = string(buf + 1, 2); - hour = string(buf + 3, 2); - min = string(buf + 5, 2); - sec = string(buf + 7, 2); - break; - - case 8: - year = string(buf, 4); - month = string(buf + 4, 2); - day = string(buf + 6, 2); - isDate = true; - break; - - case 6: - year = string(buf, 2); - month = string(buf + 2, 2); - day = string(buf + 4, 2); - isDate = true; - break; - - case 4: - month = string(buf, 2); - day = string(buf + 2, 2); - break; - - case 3: - month = string(buf, 1); - day = string(buf + 1, 2); - isDate = true; - break; - - default: - return -1; + y += 2000; + if (y > 2069) + y -= 100; } + } - if (year.empty()) - { - // MMDD format. assume current year - time_t calender_time; - struct tm todays_date; - calender_time = time(NULL); - localtime_r(&calender_time, &todays_date); - y = todays_date.tm_year + 1900; - } - else - { - y = atoi(year.c_str()); + m = atoi(month.c_str()); + d = atoi(day.c_str()); + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + ms = 0; - // special handling for 2-byte year - if (year.length() == 2) - { - y += 2000; - if (y > 2069) - y -= 100; - } - } + if (!isDateValid(d, m, y) || !isDateTimeValid(h, minute, s, ms)) + return -1; - m = atoi(month.c_str()); - d = atoi(day.c_str()); - h = atoi(hour.c_str()); - minute = atoi(min.c_str()); - s = atoi(sec.c_str()); - ms = 0; + adaytime.year = y; + adaytime.month = m; + adaytime.day = d; + adaytime.hour = h; + adaytime.minute = minute; + adaytime.second = s; + adaytime.msecond = ms; - if (!isDateValid(d, m, y) || !isDateTimeValid(h, minute, s, ms)) - return -1; + if (date) + *date = isDate; - adaytime.year = y; - adaytime.month = m; - adaytime.day = d; - adaytime.hour = h; - adaytime.minute = minute; - adaytime.second = s; - adaytime.msecond = ms; - - if (date) - *date = isDate; - - return getUInt64LE((const char*) &adaytime); + return getUInt64LE((const char*)&adaytime); } /* This is really painful and expensive b/c it seems the input is not normalized or sanitized. That should really be done on ingestion. */ int64_t DataConvert::intToTime(int64_t data, bool fromString) { - char buf[21] = {0}; - char* bufread = buf; - Time atime; - bool isNeg = false; + char buf[21] = {0}; + char* bufread = buf; + Time atime; + bool isNeg = false; - if (data == 0) - { - atime.hour = 0; - atime.minute = 0; - atime.second = 0; - atime.msecond = 0; - atime.is_neg = 0; + if (data == 0) + { + atime.hour = 0; + atime.minute = 0; + atime.second = 0; + atime.msecond = 0; + atime.is_neg = 0; - return getSInt64LE((const char*) &atime); - } + return getSInt64LE((const char*)&atime); + } - // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # - // into 15 bytes, however, that appears to be intentional. + // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # + // into 15 bytes, however, that appears to be intentional. #if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" - snprintf( buf, 15, "%lld", (long long int)data); + snprintf(buf, 15, "%lld", (long long int)data); #pragma GCC diagnostic pop #else - snprintf( buf, 15, "%lld", (long long int)data); + snprintf(buf, 15, "%lld", (long long int)data); #endif - //string date = buf; - string hour, min, sec, msec; - int64_t h = 0, minute = 0, s = 0, ms = 0; + // string date = buf; + string hour, min, sec, msec; + int64_t h = 0, minute = 0, s = 0, ms = 0; - if (bufread[0] == '-') - { - isNeg = true; - bufread++; - } + if (bufread[0] == '-') + { + isNeg = true; + bufread++; + } - bool zero = false; + bool zero = false; - switch (strlen(bufread)) - { - // A full datetime - case 14: - hour = string(buf + 8, 2); - min = string(buf + 10, 2); - sec = string(buf + 12, 2); - break; + switch (strlen(bufread)) + { + // A full datetime + case 14: + hour = string(buf + 8, 2); + min = string(buf + 10, 2); + sec = string(buf + 12, 2); + break; - // Date so this is all 0 - case 8: - zero = true; - break; + // Date so this is all 0 + case 8: zero = true; break; - case 7: - hour = string(bufread, 3); - min = string(bufread + 3, 2); - sec = string(bufread + 5, 2); - break; + case 7: + hour = string(bufread, 3); + min = string(bufread + 3, 2); + sec = string(bufread + 5, 2); + break; - case 6: - hour = string(bufread, 2); - min = string(bufread + 2, 2); - sec = string(bufread + 4, 2); - break; + case 6: + hour = string(bufread, 2); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + break; - case 5: - hour = string(bufread, 1); - min = string(bufread + 1, 2); - sec = string(bufread + 3, 2); - break; + case 5: + hour = string(bufread, 1); + min = string(bufread + 1, 2); + sec = string(bufread + 3, 2); + break; - case 4: - min = string(bufread, 2); - sec = string(bufread + 2, 2); - break; + case 4: + min = string(bufread, 2); + sec = string(bufread + 2, 2); + break; - case 3: - min = string(bufread, 1); - sec = string(bufread + 1, 2); - break; + case 3: + min = string(bufread, 1); + sec = string(bufread + 1, 2); + break; - case 2: - sec = string(bufread, 2); - break; + case 2: sec = string(bufread, 2); break; - case 1: - sec = string(bufread, 1); - break; + case 1: sec = string(bufread, 1); break; - default: - return -1; - } + default: return -1; + } - if (!zero) - { - h = atoi(hour.c_str()); - minute = atoi(min.c_str()); - s = atoi(sec.c_str()); - } - else if (fromString) - { - // Saturate fromString - h = 838; - minute = 59; - s = 59; - ms = 999999; - } + if (!zero) + { + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + } + else if (fromString) + { + // Saturate fromString + h = 838; + minute = 59; + s = 59; + ms = 999999; + } - if (!isTimeValid(h, minute, s, 0)) - return -1; + if (!isTimeValid(h, minute, s, 0)) + return -1; - atime.hour = h; - atime.minute = minute; - atime.second = s; - atime.msecond = ms; - atime.is_neg = isNeg; + atime.hour = h; + atime.minute = minute; + atime.second = s; + atime.msecond = ms; + atime.is_neg = isNeg; - return getSInt64LE((const char*) &atime); + return getSInt64LE((const char*)&atime); } int64_t DataConvert::stringToTime(const string& data) { - // MySQL supported time value format 'D HHH:MM:SS.fraction' - // -34 <= D <= 34 - // -838 <= H <= 838 - uint64_t min = 0, sec = 0, msec = 0; - int64_t day = -1, hour = 0; - bool isNeg = false; - bool hasDate = false; - string time, hms, ms; - char* end = NULL; + // MySQL supported time value format 'D HHH:MM:SS.fraction' + // -34 <= D <= 34 + // -838 <= H <= 838 + uint64_t min = 0, sec = 0, msec = 0; + int64_t day = -1, hour = 0; + bool isNeg = false; + bool hasDate = false; + string time, hms, ms; + char* end = NULL; + size_t pos = data.find("-"); - size_t pos = data.find("-"); + if (pos != string::npos) + { + isNeg = true; + } - if (pos != string::npos) + if (data.substr(pos + 1, data.length() - pos - 1).find("-") != string::npos) + { + // A second dash, this has a date + hasDate = true; + isNeg = false; + } + // Day + pos = data.find(" "); + + if (pos != string::npos) + { + if (!hasDate) { - isNeg = true; + day = strtol(data.substr(0, pos).c_str(), &end, 10); + + if (*end != '\0') + return -1; + + hour = day * 24; + day = -1; } + time = data.substr(pos + 1, data.length() - pos - 1); + } + else + { + time = data; + } - if (data.substr(pos+1, data.length()-pos-1).find("-") != string::npos) + if (time.find(":") == string::npos) + { + if (hasDate) { - // A second dash, this has a date - hasDate = true; - isNeg = false; - } - // Day - pos = data.find(" "); - - if (pos != string::npos) - { - if (!hasDate) - { - day = strtol(data.substr(0, pos).c_str(), &end, 10); - - if (*end != '\0') - return -1; - - hour = day * 24; - day = -1; - } - time = data.substr(pos + 1, data.length() - pos - 1); + // Has dashes, no colons. This is just a date! + // Or the length < 6 (MariaDB returns NULL) + return -1; } else { - time = data; + // This is an int time + return intToTime(atoll(time.c_str()), true); } + } - if (time.find(":") == string::npos) - { - if (hasDate) - { - // Has dashes, no colons. This is just a date! - // Or the length < 6 (MariaDB returns NULL) - return -1; - } - else - { - // This is an int time - return intToTime(atoll(time.c_str()), true); - } - } + // Fraction + pos = time.find("."); + if (pos != string::npos) + { + msec = strtoll(time.substr(pos + 1, time.length() - pos - 1).c_str(), 0, 10); + hms = time.substr(0, pos); + } + else + { + hms = time; + } - // Fraction - pos = time.find("."); + // HHH:MM:SS + pos = hms.find(":"); - if (pos != string::npos) - { - msec = strtoll(time.substr(pos + 1, time.length() - pos - 1).c_str(), 0, 10); - hms = time.substr(0, pos); - } + if (pos == string::npos) + { + if (hour >= 0) + hour += atoi(hms.c_str()); else - { - hms = time; - } - - // HHH:MM:SS - pos = hms.find(":"); - - if (pos == string::npos) - { - if (hour >= 0) - hour += atoi(hms.c_str()); - else - hour -= atoi(hms.c_str()); - } + hour -= atoi(hms.c_str()); + } + else + { + if (hour >= 0) + hour += atoi(hms.substr(0, pos).c_str()); else - { - if (hour >= 0) - hour += atoi(hms.substr(0, pos).c_str()); - else - hour -= atoi(hms.substr(0, pos).c_str()); + hour -= atoi(hms.substr(0, pos).c_str()); - ms = hms.substr(pos + 1, hms.length() - pos - 1); - } + ms = hms.substr(pos + 1, hms.length() - pos - 1); + } - // MM:SS - pos = ms.find(":"); + // MM:SS + pos = ms.find(":"); - if (pos != string::npos) - { - min = atoi(ms.substr(0, pos).c_str()); - sec = atoi(ms.substr(pos + 1, ms.length() - pos - 1).c_str()); - } - else - { - min = atoi(ms.c_str()); - } + if (pos != string::npos) + { + min = atoi(ms.substr(0, pos).c_str()); + sec = atoi(ms.substr(pos + 1, ms.length() - pos - 1).c_str()); + } + else + { + min = atoi(ms.c_str()); + } - Time atime; - atime.day = day; - atime.hour = hour; - atime.minute = min; - atime.second = sec; - atime.msecond = msec; - atime.is_neg = isNeg; - return getSInt64LE((const char*) &atime); + Time atime; + atime.day = day; + atime.hour = hour; + atime.minute = min; + atime.second = sec; + atime.msecond = msec; + atime.is_neg = isNeg; + return getSInt64LE((const char*)&atime); } - -void -DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, - const datatypes::SystemCatalog::TypeHolderStd &type) +void DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd& unionedType, + const datatypes::SystemCatalog::TypeHolderStd& type) { - // limited support for VARBINARY, no implicit conversion. - if (type.colDataType == datatypes::SystemCatalog::VARBINARY || - unionedType.colDataType == datatypes::SystemCatalog::VARBINARY) - { - if (type.colDataType != unionedType.colDataType || - type.colWidth != unionedType.colWidth) - throw runtime_error("VARBINARY in UNION must be the same width."); - } + // limited support for VARBINARY, no implicit conversion. + if (type.colDataType == datatypes::SystemCatalog::VARBINARY || + unionedType.colDataType == datatypes::SystemCatalog::VARBINARY) + { + if (type.colDataType != unionedType.colDataType || type.colWidth != unionedType.colWidth) + throw runtime_error("VARBINARY in UNION must be the same width."); + } - switch (type.colDataType) + switch (type.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: { + switch (unionedType.colDataType) + { case datatypes::SystemCatalog::TINYINT: case datatypes::SystemCatalog::SMALLINT: case datatypes::SystemCatalog::MEDINT: @@ -3046,441 +2974,419 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione case datatypes::SystemCatalog::UINT: case datatypes::SystemCatalog::UBIGINT: case datatypes::SystemCatalog::UDECIMAL: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: - unionedType.precision = std::max(type.precision, unionedType.precision); - unionedType.scale = std::max(type.scale, unionedType.scale); - - if (datatypes::Decimal::isWideDecimalTypeByPrecision(unionedType.precision)) - { - unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; - unionedType.colWidth = datatypes::MAXDECIMALWIDTH; - break; - } - - if (type.colWidth > unionedType.colWidth) - { - unionedType.colDataType = type.colDataType; - unionedType.colWidth = type.colWidth; - } - - // If same size but different signedness - if (type.colWidth == unionedType.colWidth && - ((!isUnsigned(unionedType.colDataType) && isUnsigned(type.colDataType)) || - (!isUnsigned(type.colDataType) && isUnsigned(unionedType.colDataType)))) - { - unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; - unionedType.colWidth = datatypes::Decimal::isWideDecimalTypeByPrecision(unionedType.precision) ? - datatypes::MAXDECIMALWIDTH : - datatypes::MAXLEGACYWIDTH; - } - - if (type.colDataType == datatypes::SystemCatalog::DECIMAL || type.colDataType == datatypes::SystemCatalog::UDECIMAL) - { - unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; - } - - break; - - case datatypes::SystemCatalog::DATE: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.colWidth = 20; - break; - - case datatypes::SystemCatalog::TIME: - case datatypes::SystemCatalog::DATETIME: - case datatypes::SystemCatalog::TIMESTAMP: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.colWidth = 26; - break; - - case datatypes::SystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; - - break; - - case datatypes::SystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; - - break; - - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::LONGDOUBLE: - default: - break; - } + unionedType.precision = std::max(type.precision, unionedType.precision); + unionedType.scale = std::max(type.scale, unionedType.scale); + if (datatypes::Decimal::isWideDecimalTypeByPrecision(unionedType.precision)) + { + unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; + unionedType.colWidth = datatypes::MAXDECIMALWIDTH; break; - } + } + + if (type.colWidth > unionedType.colWidth) + { + unionedType.colDataType = type.colDataType; + unionedType.colWidth = type.colWidth; + } + + // If same size but different signedness + if (type.colWidth == unionedType.colWidth && + ((!isUnsigned(unionedType.colDataType) && isUnsigned(type.colDataType)) || + (!isUnsigned(type.colDataType) && isUnsigned(unionedType.colDataType)))) + { + unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; + unionedType.colWidth = datatypes::Decimal::isWideDecimalTypeByPrecision(unionedType.precision) + ? datatypes::MAXDECIMALWIDTH + : datatypes::MAXLEGACYWIDTH; + } + + if (type.colDataType == datatypes::SystemCatalog::DECIMAL || + type.colDataType == datatypes::SystemCatalog::UDECIMAL) + { + unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; + } + + break; case datatypes::SystemCatalog::DATE: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::LONGDOUBLE: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; - - case datatypes::SystemCatalog::CHAR: - if (unionedType.colWidth < 10) - unionedType.colWidth = 10; - - break; - - case datatypes::SystemCatalog::VARCHAR: - if (unionedType.colWidth < 11) - unionedType.colWidth = 11; - - break; - - case datatypes::SystemCatalog::DATE: - case datatypes::SystemCatalog::DATETIME: - case datatypes::SystemCatalog::TIMESTAMP: - case datatypes::SystemCatalog::TIME: - default: - break; - } - - break; - } + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.colWidth = 20; + break; + case datatypes::SystemCatalog::TIME: case datatypes::SystemCatalog::DATETIME: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::TIME: - case datatypes::SystemCatalog::LONGDOUBLE: - case datatypes::SystemCatalog::TIMESTAMP: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case datatypes::SystemCatalog::DATE: - unionedType.colDataType = datatypes::SystemCatalog::DATETIME; - unionedType.colWidth = type.colWidth; - break; - - case datatypes::SystemCatalog::CHAR: - if (unionedType.colWidth < 26) - unionedType.colWidth = 26; - - break; - - case datatypes::SystemCatalog::VARCHAR: - if (unionedType.colWidth < 27) - unionedType.colWidth = 27; - - break; - - case datatypes::SystemCatalog::DATETIME: - default: - break; - } - - break; - } - case datatypes::SystemCatalog::TIMESTAMP: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::TIME: - case datatypes::SystemCatalog::DATETIME: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.colWidth = 26; + break; - case datatypes::SystemCatalog::DATE: - unionedType.colDataType = datatypes::SystemCatalog::TIMESTAMP; - unionedType.colWidth = type.colWidth; - break; + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) + unionedType.colWidth = 20; - case datatypes::SystemCatalog::CHAR: - if (unionedType.colWidth < 26) - unionedType.colWidth = 26; + break; - break; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; - case datatypes::SystemCatalog::VARCHAR: - if (unionedType.colWidth < 27) - unionedType.colWidth = 27; - - break; - - case datatypes::SystemCatalog::TIMESTAMP: - default: - break; - } - - break; - } + break; case datatypes::SystemCatalog::FLOAT: case datatypes::SystemCatalog::DOUBLE: case datatypes::SystemCatalog::UFLOAT: case datatypes::SystemCatalog::UDOUBLE: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::DATE: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; - - case datatypes::SystemCatalog::DATETIME: - case datatypes::SystemCatalog::TIMESTAMP: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case datatypes::SystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; - - break; - - case datatypes::SystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; - - break; - - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; - unionedType.scale = 0; - unionedType.colWidth = sizeof(double); - break; - - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::UDECIMAL: - if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) - { - unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; - unionedType.scale = 0; - unionedType.colWidth = sizeof(double); - } - break; - - default: - break; - } - - break; - } - case datatypes::SystemCatalog::LONGDOUBLE: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::DATE: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; + default: break; + } - case datatypes::SystemCatalog::DATETIME: - unionedType.colDataType = datatypes::SystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; + break; + } - case datatypes::SystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; + case datatypes::SystemCatalog::DATE: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; - break; + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 10) + unionedType.colWidth = 10; - case datatypes::SystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; + break; - break; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 11) + unionedType.colWidth = 11; - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::LONGDOUBLE: - unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; - unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; - unionedType.colWidth = sizeof(long double); - unionedType.precision = -1; - break; + break; - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::UDECIMAL: - if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) - { - unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; - unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; - unionedType.colWidth = sizeof(long double); - unionedType.precision = -1; - } - break; + case datatypes::SystemCatalog::DATE: + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + case datatypes::SystemCatalog::TIME: + default: break; + } - default: - break; - } + break; + } - break; - } + case datatypes::SystemCatalog::DATETIME: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::LONGDOUBLE: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::DATETIME; + unionedType.colWidth = type.colWidth; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 26) + unionedType.colWidth = 26; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 27) + unionedType.colWidth = 27; + + break; + + case datatypes::SystemCatalog::DATETIME: + default: break; + } + + break; + } + + case datatypes::SystemCatalog::TIMESTAMP: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::DATETIME: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::TIMESTAMP; + unionedType.colWidth = type.colWidth; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 26) + unionedType.colWidth = 26; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 27) + unionedType.colWidth = 27; + + break; + + case datatypes::SystemCatalog::TIMESTAMP: + default: break; + } + + break; + } + + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) + unionedType.colWidth = 20; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; + + break; + + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; + unionedType.scale = 0; + unionedType.colWidth = sizeof(double); + break; + + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) + { + unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; + unionedType.scale = 0; + unionedType.colWidth = sizeof(double); + } + break; + + default: break; + } + + break; + } + + case datatypes::SystemCatalog::LONGDOUBLE: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::DATETIME: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) + unionedType.colWidth = 20; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; + + break; + + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + unionedType.colWidth = sizeof(long double); + unionedType.precision = -1; + break; + + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) + { + unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + unionedType.colWidth = sizeof(long double); + unionedType.precision = -1; + } + break; + + default: break; + } + + break; + } + + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.scale = 0; + unionedType.colWidth = (type.colWidth > 20) ? type.colWidth : 20; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colWidth = (type.colWidth > 10) ? type.colWidth : 10; + break; + + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colWidth = (type.colWidth > 26) ? type.colWidth : 26; + break; case datatypes::SystemCatalog::CHAR: case datatypes::SystemCatalog::VARCHAR: - { - switch (unionedType.colDataType) - { - case datatypes::SystemCatalog::TINYINT: - case datatypes::SystemCatalog::SMALLINT: - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UTINYINT: - case datatypes::SystemCatalog::USMALLINT: - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: - case datatypes::SystemCatalog::UFLOAT: - case datatypes::SystemCatalog::UDOUBLE: - case datatypes::SystemCatalog::LONGDOUBLE: - unionedType.scale = 0; - unionedType.colWidth = (type.colWidth > 20) ? type.colWidth : 20; - break; - case datatypes::SystemCatalog::DATE: - unionedType.colWidth = (type.colWidth > 10) ? type.colWidth : 10; - break; + // VARCHAR will fit in CHAR of the same width + if (unionedType.colWidth < type.colWidth) + unionedType.colWidth = type.colWidth; - case datatypes::SystemCatalog::DATETIME: - case datatypes::SystemCatalog::TIMESTAMP: - unionedType.colWidth = (type.colWidth > 26) ? type.colWidth : 26; - break; + break; - case datatypes::SystemCatalog::CHAR: - case datatypes::SystemCatalog::VARCHAR: + default: break; + } - // VARCHAR will fit in CHAR of the same width - if (unionedType.colWidth < type.colWidth) - unionedType.colWidth = type.colWidth; + // MariaDB bug 651. Setting to CHAR broke union in subquery + unionedType.colDataType = datatypes::SystemCatalog::VARCHAR; + break; + } - break; - - default: - break; - } - - // MariaDB bug 651. Setting to CHAR broke union in subquery - unionedType.colDataType = datatypes::SystemCatalog::VARCHAR; - break; - } - - default: - { - break; - } - } // switch + default: + { + break; + } + } // switch } -} // namespace dataconvert +} // namespace dataconvert // vim:ts=4 sw=4: - diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 123c90f65..ec716d80d 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: dataconvert.h 3693 2013-04-05 16:11:30Z chao $ -* -* -****************************************************************************/ + * $Id: dataconvert.h 3693 2013-04-05 16:11:30Z chao $ + * + * + ****************************************************************************/ /** @file */ #ifndef DATACONVERT_H @@ -52,44 +52,43 @@ #include "exceptclasses.h" #include "common/branchpred.h" - #include "bytestream.h" #include "errorids.h" // remove this block if the htonll is defined in library #ifdef __linux__ #include -#if __BYTE_ORDER == __BIG_ENDIAN // 4312 +#if __BYTE_ORDER == __BIG_ENDIAN // 4312 inline uint64_t htonll(uint64_t n) { - return n; + return n; } #elif __BYTE_ORDER == __LITTLE_ENDIAN // 1234 inline uint64_t htonll(uint64_t n) { - return ((((uint64_t) htonl(n & 0xFFFFFFFFLLU)) << 32) | (htonl((n & 0xFFFFFFFF00000000LLU) >> 32))); + return ((((uint64_t)htonl(n & 0xFFFFFFFFLLU)) << 32) | (htonl((n & 0xFFFFFFFF00000000LLU) >> 32))); } -#else // __BYTE_ORDER == __PDP_ENDIAN 3412 +#else // __BYTE_ORDER == __PDP_ENDIAN 3412 inline uint64_t htonll(uint64_t n); // don't know 34127856 or 78563412, hope never be required to support this byte order. #endif -#else //!__linux__ +#else //!__linux__ #if _MSC_VER < 1600 -//Assume we're on little-endian +// Assume we're on little-endian inline uint64_t htonll(uint64_t n) { - return ((((uint64_t) htonl(n & 0xFFFFFFFFULL)) << 32) | (htonl((n & 0xFFFFFFFF00000000ULL) >> 32))); + return ((((uint64_t)htonl(n & 0xFFFFFFFFULL)) << 32) | (htonl((n & 0xFFFFFFFF00000000ULL) >> 32))); } -#endif //_MSC_VER -#endif //__linux__ +#endif //_MSC_VER +#endif //__linux__ -#define LEAPS_THRU_END_OF(y) ((y) / 4 - (y) / 100 + (y) / 400) +#define LEAPS_THRU_END_OF(y) ((y) / 4 - (y) / 100 + (y) / 400) #define isleap(y) (((y) % 4) == 0 && (((y) % 100) != 0 || ((y) % 400) == 0)) // this method evalutes the uint64 that stores a char[] to expected value inline uint64_t uint64ToStr(uint64_t n) { - return htonll(n); + return htonll(n); } using cscDataType = datatypes::SystemCatalog::ColDataType; @@ -100,52 +99,46 @@ using cscDataType = datatypes::SystemCatalog::ColDataType; #define EXPORT #endif -const int64_t IDB_pow[19] = -{ - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000LL, - 100000000000LL, - 1000000000000LL, - 10000000000000LL, - 100000000000000LL, - 1000000000000000LL, - 10000000000000000LL, - 100000000000000000LL, - 1000000000000000000LL -}; +const int64_t IDB_pow[19] = {1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; -const std::string columnstore_big_precision[20] = -{ - "9999999999999999999", - "99999999999999999999", - "999999999999999999999", - "9999999999999999999999", - "99999999999999999999999", - "999999999999999999999999", - "9999999999999999999999999", - "99999999999999999999999999", - "999999999999999999999999999", - "9999999999999999999999999999", - "99999999999999999999999999999", - "999999999999999999999999999999", - "9999999999999999999999999999999", - "99999999999999999999999999999999", - "999999999999999999999999999999999", - "9999999999999999999999999999999999", - "99999999999999999999999999999999999", - "999999999999999999999999999999999999", - "9999999999999999999999999999999999999", - "99999999999999999999999999999999999999" -}; +const std::string columnstore_big_precision[20] = {"9999999999999999999", + "99999999999999999999", + "999999999999999999999", + "9999999999999999999999", + "99999999999999999999999", + "999999999999999999999999", + "9999999999999999999999999", + "99999999999999999999999999", + "999999999999999999999999999", + "9999999999999999999999999999", + "99999999999999999999999999999", + "999999999999999999999999999999", + "9999999999999999999999999999999", + "99999999999999999999999999999999", + "999999999999999999999999999999999", + "9999999999999999999999999999999999", + "99999999999999999999999999999999999", + "999999999999999999999999999999999999", + "9999999999999999999999999999999999999", + "99999999999999999999999999999999999999"}; const int32_t SECS_PER_MIN = 60; const int32_t MINS_PER_HOUR = 60; @@ -162,15 +155,13 @@ const int32_t MIN_TIMESTAMP_YEAR = 1969; const int32_t MAX_TIMESTAMP_VALUE = (1ULL << 31) - 1; const int32_t MIN_TIMESTAMP_VALUE = 0; - namespace dataconvert { - enum CalpontDateTimeFormat { - CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" - CALPONTDATETIME_ENUM = 2, // date format is: "YYYY-MM-DD HH:MI:SS" - CALPONTTIME_ENUM = 3 + CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" + CALPONTDATETIME_ENUM = 2, // date format is: "YYYY-MM-DD HH:MI:SS" + CALPONTTIME_ENUM = 3 }; /** @brief a structure that represents a timestamp in broken down @@ -178,24 +169,24 @@ enum CalpontDateTimeFormat */ struct MySQLTime { - unsigned int year, month, day, hour, minute, second; - unsigned long second_part; - CalpontDateTimeFormat time_type; - void reset() - { - year = month = day = 0; - hour = minute = second = second_part = 0; - time_type = CALPONTDATETIME_ENUM; - } + unsigned int year, month, day, hour, minute, second; + unsigned long second_part; + CalpontDateTimeFormat time_type; + void reset() + { + year = month = day = 0; + hour = minute = second = second_part = 0; + time_type = CALPONTDATETIME_ENUM; + } }; /* Structure describing local time type (e.g. Moscow summer time (MSD)) */ typedef struct ttinfo { - long tt_gmtoff; // Offset from UTC in seconds + long tt_gmtoff; // Offset from UTC in seconds uint tt_isdst; // Is daylight saving time or not. Used to set tm_isdst #ifdef ABBR_ARE_USED - uint tt_abbrind; // Index of start of abbreviation for this time type. + uint tt_abbrind; // Index of start of abbreviation for this time type. #endif /* We don't use tt_ttisstd and tt_ttisgmt members of original elsie-code @@ -206,8 +197,8 @@ typedef struct ttinfo /* Structure describing leap-second corrections. */ typedef struct lsinfo { - int64_t ls_trans; // Transition time - long ls_corr; // Correction to apply + int64_t ls_trans; // Transition time + long ls_corr; // Correction to apply } LS_INFO; /* @@ -217,7 +208,7 @@ typedef struct lsinfo */ typedef struct revtinfo { - long rt_offset; // Offset of local time from UTC in seconds + long rt_offset; // Offset of local time from UTC in seconds uint rt_type; // Type of period 0 - Normal period. 1 - Spring time-gap } REVT_INFO; @@ -233,70 +224,70 @@ typedef struct st_time_zone_info uint charcnt; // Number of characters used for abbreviations uint revcnt; // Number of transition descr. for TIME->my_time_t conversion /* The following are dynamical arrays are allocated in MEM_ROOT */ - int64_t *ats; // Times of transitions between time types - unsigned char *types; // Local time types for transitions - TRAN_TYPE_INFO *ttis; // Local time types descriptions + int64_t* ats; // Times of transitions between time types + unsigned char* types; // Local time types for transitions + TRAN_TYPE_INFO* ttis; // Local time types descriptions #ifdef ABBR_ARE_USED /* Storage for local time types abbreviations. They are stored as ASCIIZ */ - char *chars; + char* chars; #endif /* Leap seconds corrections descriptions, this array is shared by all time zones who use leap seconds. */ - LS_INFO *lsis; + LS_INFO* lsis; /* Starting points and descriptions of shifted my_time_t (my_time_t + offset) ranges on which shifted my_time_t -> my_time_t mapping is linear or undefined. Used for tm -> my_time_t conversion. */ - int64_t *revts; - REVT_INFO *revtis; + int64_t* revts; + REVT_INFO* revtis; /* Time type which is used for times smaller than first transition or if there are no transitions at all. */ - TRAN_TYPE_INFO *fallback_tti; + TRAN_TYPE_INFO* fallback_tti; } TIME_ZONE_INFO; inline void serializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* tz) { - bs << (uint) tz->leapcnt; - bs << (uint) tz->timecnt; - bs << (uint) tz->typecnt; - bs << (uint) tz->charcnt; - bs << (uint) tz->revcnt; + bs << (uint)tz->leapcnt; + bs << (uint)tz->timecnt; + bs << (uint)tz->typecnt; + bs << (uint)tz->charcnt; + bs << (uint)tz->revcnt; - // going to put size in front of these dynamically sized arrays - // and use deserializeInlineVector on the other side - bs << (uint64_t) tz->timecnt; - bs.append((uint8_t*)tz->ats,(tz->timecnt*sizeof(int64_t))); - bs << (uint64_t) tz->timecnt; - bs.append((uint8_t*)tz->types,tz->timecnt); - bs << (uint64_t) tz->typecnt; - bs.append((uint8_t*)tz->ttis,(tz->typecnt*sizeof(TRAN_TYPE_INFO))); + // going to put size in front of these dynamically sized arrays + // and use deserializeInlineVector on the other side + bs << (uint64_t)tz->timecnt; + bs.append((uint8_t*)tz->ats, (tz->timecnt * sizeof(int64_t))); + bs << (uint64_t)tz->timecnt; + bs.append((uint8_t*)tz->types, tz->timecnt); + bs << (uint64_t)tz->typecnt; + bs.append((uint8_t*)tz->ttis, (tz->typecnt * sizeof(TRAN_TYPE_INFO))); #ifdef ABBR_ARE_USED - bs << (uint) tz->charcnt; - bs.append((uint8_t*)tz->chars,tz->charcnt); + bs << (uint)tz->charcnt; + bs.append((uint8_t*)tz->chars, tz->charcnt); #endif - bs << (uint64_t) tz->leapcnt; - bs.append((uint8_t*)tz->lsis,(tz->leapcnt*sizeof(LS_INFO))); - bs << (uint64_t) (tz->revcnt + 1); - bs.append((uint8_t*)tz->revts,((tz->revcnt + 1)*sizeof(int64_t))); - bs << (uint64_t) tz->revcnt; - bs.append((uint8_t*)tz->revtis,(tz->revcnt*sizeof(REVT_INFO))); - bs << (uint64_t) tz->typecnt; - bs.append((uint8_t*)tz->fallback_tti,(tz->typecnt*sizeof(TRAN_TYPE_INFO))); + bs << (uint64_t)tz->leapcnt; + bs.append((uint8_t*)tz->lsis, (tz->leapcnt * sizeof(LS_INFO))); + bs << (uint64_t)(tz->revcnt + 1); + bs.append((uint8_t*)tz->revts, ((tz->revcnt + 1) * sizeof(int64_t))); + bs << (uint64_t)tz->revcnt; + bs.append((uint8_t*)tz->revtis, (tz->revcnt * sizeof(REVT_INFO))); + bs << (uint64_t)tz->typecnt; + bs.append((uint8_t*)tz->fallback_tti, (tz->typecnt * sizeof(TRAN_TYPE_INFO))); }; inline void unserializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* tz) { - bs >> (uint&)tz->leapcnt; - bs >> (uint&)tz->timecnt; - bs >> (uint&)tz->typecnt; - bs >> (uint&)tz->charcnt; - bs >> (uint&)tz->revcnt; + bs >> (uint&)tz->leapcnt; + bs >> (uint&)tz->timecnt; + bs >> (uint&)tz->typecnt; + bs >> (uint&)tz->charcnt; + bs >> (uint&)tz->revcnt; }; /** @@ -304,200 +295,185 @@ inline void unserializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds * Most of this code is taken from tztime.cc:str_to_offset */ -inline -bool timeZoneToOffset(const char *str, std::string::size_type length, long *offset) +inline bool timeZoneToOffset(const char* str, std::string::size_type length, long* offset) { - const char *end = str + length; - bool negative; - unsigned long number_tmp; - long offset_tmp; + const char* end = str + length; + bool negative; + unsigned long number_tmp; + long offset_tmp; - if (length < 4) - return 1; + if (length < 4) + return 1; - if (*str == '+') - negative = 0; - else if (*str == '-') - negative = 1; - else - return 1; + if (*str == '+') + negative = 0; + else if (*str == '-') + negative = 1; + else + return 1; + str++; + + number_tmp = 0; + + while (str < end && isdigit(*str)) + { + number_tmp = number_tmp * 10 + *str - '0'; str++; + } - number_tmp = 0; + if (str + 1 >= end || *str != ':') + return 1; + str++; - while (str < end && isdigit(*str)) - { - number_tmp = number_tmp * 10 + *str - '0'; - str++; - } + offset_tmp = number_tmp * 60L; + number_tmp = 0; - if (str + 1 >= end || *str != ':') - return 1; + while (str < end && isdigit(*str)) + { + number_tmp = number_tmp * 10 + *str - '0'; str++; + } - offset_tmp = number_tmp * 60L; - number_tmp = 0; + if (str != end) + return 1; - while (str < end && isdigit(*str)) - { - number_tmp = number_tmp * 10 + *str - '0'; - str++; - } + offset_tmp = (offset_tmp + number_tmp) * 60L; - if (str != end) - return 1; + if (negative) + offset_tmp = -offset_tmp; - offset_tmp = (offset_tmp + number_tmp) * 60L; + /* + Check if offset is in range prescribed by standard + (from -12:59 to 13:00). + */ - if (negative) - offset_tmp = -offset_tmp; + if (number_tmp > 59 || offset_tmp < -13 * 3600L + 1 || offset_tmp > 13 * 3600L) + return 1; - /* - Check if offset is in range prescribed by standard - (from -12:59 to 13:00). - */ + *offset = offset_tmp; - if (number_tmp > 59 || offset_tmp < -13 * 3600L + 1 || - offset_tmp > 13 * 3600L) - return 1; - - *offset = offset_tmp; - - return 0; + return 0; } -const int32_t year_lengths[2] = -{ - DAYS_PER_NYEAR, DAYS_PER_LYEAR -}; +const int32_t year_lengths[2] = {DAYS_PER_NYEAR, DAYS_PER_LYEAR}; -const unsigned int mon_lengths[2][MONS_PER_YEAR]= -{ - { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, - { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } -}; +const unsigned int mon_lengths[2][MONS_PER_YEAR] = {{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, + {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}}; -const unsigned int mon_starts[2][MONS_PER_YEAR]= -{ - { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }, - { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335 } -}; +const unsigned int mon_starts[2][MONS_PER_YEAR] = {{0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}, + {0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335}}; inline int32_t leapsThruEndOf(int32_t year) { - return (year / 4 - year / 100 + year / 400); + return (year / 4 - year / 100 + year / 400); } -inline bool isLeapYear ( int year) +inline bool isLeapYear(int year) { - if ( year % 400 == 0 ) - return true; + if (year % 400 == 0) + return true; - if ( ( year % 4 == 0 ) && ( year % 100 != 0 ) ) - return true; + if ((year % 4 == 0) && (year % 100 != 0)) + return true; - return false; + return false; } static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; inline uint32_t getDaysInMonth(uint32_t month, int year) { - if (month < 1 || month > 12) - return 0; + if (month < 1 || month > 12) + return 0; - uint32_t days = daysInMonth[month - 1]; + uint32_t days = daysInMonth[month - 1]; - if ((month == 2) && isLeapYear(year)) - days++; + if ((month == 2) && isLeapYear(year)) + days++; - return days; + return days; } -inline -bool isDateValid ( int day, int month, int year) +inline bool isDateValid(int day, int month, int year) { - bool valid = true; + bool valid = true; - if ( day == 0 && month == 0 && year == 0 ) - { - return true; - } + if (day == 0 && month == 0 && year == 0) + { + return true; + } - int daycheck = getDaysInMonth( month, year ); + int daycheck = getDaysInMonth(month, year); - if ( ( year < 1000 ) || ( year > 9999 ) ) - valid = false; - else if ( month < 1 || month > 12 ) - valid = false; - else if ( day < 1 || day > daycheck ) - valid = false; + if ((year < 1000) || (year > 9999)) + valid = false; + else if (month < 1 || month > 12) + valid = false; + else if (day < 1 || day > daycheck) + valid = false; - return ( valid ); + return (valid); } -inline -bool isDateTimeValid ( int hour, int minute, int second, int microSecond) +inline bool isDateTimeValid(int hour, int minute, int second, int microSecond) { - bool valid = false; + bool valid = false; - if ( hour >= 0 && hour <= 24 ) + if (hour >= 0 && hour <= 24) + { + if (minute >= 0 && minute < 60) { - if ( minute >= 0 && minute < 60 ) + if (second >= 0 && second < 60) + { + if (microSecond >= 0 && microSecond <= 999999) { - if ( second >= 0 && second < 60 ) - { - if ( microSecond >= 0 && microSecond <= 999999 ) - { - valid = true; - } - } + valid = true; } + } } + } - return valid; + return valid; } -inline -bool isTimeValid ( int hour, int minute, int second, int microSecond) +inline bool isTimeValid(int hour, int minute, int second, int microSecond) { - bool valid = false; + bool valid = false; - if ( hour >= -838 && hour <= 838 ) + if (hour >= -838 && hour <= 838) + { + if (minute >= 0 && minute < 60) { - if ( minute >= 0 && minute < 60 ) + if (second >= 0 && second < 60) + { + if (microSecond >= 0 && microSecond <= 999999) { - if ( second >= 0 && second < 60 ) - { - if ( microSecond >= 0 && microSecond <= 999999 ) - { - valid = true; - } - } + valid = true; } + } } + } - return valid; + return valid; } -inline -bool isTimestampValid ( uint64_t second, uint64_t microsecond ) +inline bool isTimestampValid(uint64_t second, uint64_t microsecond) { - bool valid = false; + bool valid = false; - // MariaDB server currently sets the upper limit on timestamp to - // 0x7FFFFFFF. So enforce the same restriction here. - // TODO: We however store the seconds portion of the timestamp in - // 44 bits, so change this limit when the server supports higher values. - if ( second <= MAX_TIMESTAMP_VALUE ) + // MariaDB server currently sets the upper limit on timestamp to + // 0x7FFFFFFF. So enforce the same restriction here. + // TODO: We however store the seconds portion of the timestamp in + // 44 bits, so change this limit when the server supports higher values. + if (second <= MAX_TIMESTAMP_VALUE) + { + if (microsecond <= 999999) { - if ( microsecond <= 999999 ) - { - valid = true; - } + valid = true; } + } - return valid; + return valid; } /** @@ -511,89 +487,86 @@ bool isTimestampValid ( uint64_t second, uint64_t microsecond ) * @param timeZone a string with the server timezone of the machine * which initiated the query */ -inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, - const std::string& timeZone) +inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, const std::string& timeZone) { - if (seconds == 0) + if (seconds == 0) + { + time.reset(); + return; + } + + if (timeZone == "SYSTEM") + { + struct tm tmp_tm; + time_t tmp_t = (time_t)seconds; + localtime_r(&tmp_t, &tmp_tm); + time.second_part = 0; + time.year = (int)((tmp_tm.tm_year + 1900) % 10000); + time.month = (int)tmp_tm.tm_mon + 1; + time.day = (int)tmp_tm.tm_mday; + time.hour = (int)tmp_tm.tm_hour; + time.minute = (int)tmp_tm.tm_min; + time.second = (int)tmp_tm.tm_sec; + time.time_type = CALPONTDATETIME_ENUM; + if (time.second == 60 || time.second == 61) + time.second = 59; + } + else + { + long offset; + if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) { - time.reset(); - return; + time.reset(); + return; } - if (timeZone == "SYSTEM") + int64_t days; + int32_t rem; + int32_t y; + int32_t yleap; + const unsigned int* ip; + + days = (int64_t)(seconds / SECS_PER_DAY); + rem = (int32_t)(seconds % SECS_PER_DAY); + + rem += offset; + while (rem < 0) { - struct tm tmp_tm; - time_t tmp_t = (time_t)seconds; - localtime_r(&tmp_t, &tmp_tm); - time.second_part = 0; - time.year = (int) ((tmp_tm.tm_year + 1900) % 10000); - time.month = (int) tmp_tm.tm_mon + 1; - time.day = (int) tmp_tm.tm_mday; - time.hour = (int) tmp_tm.tm_hour; - time.minute = (int) tmp_tm.tm_min; - time.second = (int) tmp_tm.tm_sec; - time.time_type = CALPONTDATETIME_ENUM; - if (time.second == 60 || time.second == 61) - time.second = 59; + rem += SECS_PER_DAY; + days--; } - else + while (rem >= SECS_PER_DAY) { - long offset; - if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - time.reset(); - return; - } - - int64_t days; - int32_t rem; - int32_t y; - int32_t yleap; - const unsigned int *ip; - - days = (int64_t) (seconds / SECS_PER_DAY); - rem = (int32_t) (seconds % SECS_PER_DAY); - - rem += offset; - while (rem < 0) - { - rem += SECS_PER_DAY; - days--; - } - while (rem >= SECS_PER_DAY) - { - rem -= SECS_PER_DAY; - days++; - } - time.hour = (unsigned int) (rem / SECS_PER_HOUR); - rem = rem % SECS_PER_HOUR; - time.minute = (unsigned int) (rem / SECS_PER_MIN); - time.second = (unsigned int) (rem % SECS_PER_MIN); - - y = EPOCH_YEAR; - while (days < 0 || days >= (int64_t) (year_lengths[yleap = isLeapYear(y)])) - { - int32_t newy; - - newy = y + days / DAYS_PER_NYEAR; - if (days < 0) - newy--; - days -= (newy - y) * DAYS_PER_NYEAR + - leapsThruEndOf(newy - 1) - - leapsThruEndOf(y - 1); - y = newy; - } - time.year = y; - - ip = mon_lengths[yleap]; - for (time.month = 0; days >= (int64_t) ip[time.month]; time.month++) - days -= (int64_t) ip[time.month]; - time.month++; - time.day = (unsigned int) (days + 1); - - time.second_part = 0; - time.time_type = CALPONTDATETIME_ENUM; + rem -= SECS_PER_DAY; + days++; } + time.hour = (unsigned int)(rem / SECS_PER_HOUR); + rem = rem % SECS_PER_HOUR; + time.minute = (unsigned int)(rem / SECS_PER_MIN); + time.second = (unsigned int)(rem % SECS_PER_MIN); + + y = EPOCH_YEAR; + while (days < 0 || days >= (int64_t)(year_lengths[yleap = isLeapYear(y)])) + { + int32_t newy; + + newy = y + days / DAYS_PER_NYEAR; + if (days < 0) + newy--; + days -= (newy - y) * DAYS_PER_NYEAR + leapsThruEndOf(newy - 1) - leapsThruEndOf(y - 1); + y = newy; + } + time.year = y; + + ip = mon_lengths[yleap]; + for (time.month = 0; days >= (int64_t)ip[time.month]; time.month++) + days -= (int64_t)ip[time.month]; + time.month++; + time.day = (unsigned int)(days + 1); + + time.second_part = 0; + time.time_type = CALPONTDATETIME_ENUM; + } } /** @@ -611,43 +584,40 @@ inline bool validateTimestampRange(const MySQLTime& t) return true; } -inline -int64_t secSinceEpoch(int year, int month, int day, int hour, int min, int sec) +inline int64_t secSinceEpoch(int year, int month, int day, int hour, int min, int sec) { - int64_t days = (year - EPOCH_YEAR) * DAYS_PER_NYEAR + - leapsThruEndOf(year - 1) - - leapsThruEndOf(EPOCH_YEAR - 1); - days += mon_starts[isLeapYear(year)][month - 1]; - days += day - 1; + int64_t days = + (year - EPOCH_YEAR) * DAYS_PER_NYEAR + leapsThruEndOf(year - 1) - leapsThruEndOf(EPOCH_YEAR - 1); + days += mon_starts[isLeapYear(year)][month - 1]; + days += day - 1; - return ((days * HOURS_PER_DAY + hour) * MINS_PER_HOUR + min) * - SECS_PER_MIN + sec; + return ((days * HOURS_PER_DAY + hour) * MINS_PER_HOUR + min) * SECS_PER_MIN + sec; } // This is duplicate of funchelpers.h:calc_mysql_daynr, // with one additional function parameter -inline uint32_t calc_mysql_daynr( uint32_t year, uint32_t month, uint32_t day, bool& isValid ) +inline uint32_t calc_mysql_daynr(uint32_t year, uint32_t month, uint32_t day, bool& isValid) { - int temp; - int y = year; - long delsum; + int temp; + int y = year; + long delsum; - if ( !isDateValid( day, month, year ) ) - { - isValid = false; - return 0; - } + if (!isDateValid(day, month, year)) + { + isValid = false; + return 0; + } - delsum = (long) (365 * y + 31 * ((int) month - 1) + (int) day); + delsum = (long)(365 * y + 31 * ((int)month - 1) + (int)day); - if (month <= 2) - y--; - else - delsum -= (long) ((int) month * 4 + 23) / 10; + if (month <= 2) + y--; + else + delsum -= (long)((int)month * 4 + 23) / 10; - temp = (int) ((y / 100 + 1) * 3) / 4; + temp = (int)((y / 100 + 1) * 3) / 4; - return delsum + (int) y / 4 - temp; + return delsum + (int)y / 4 - temp; } /** @@ -658,113 +628,109 @@ inline uint32_t calc_mysql_daynr( uint32_t year, uint32_t month, uint32_t day, b @param timeZone a string with the server timezone of the machine which initiated the query */ -inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, - const std::string& timeZone, bool& isValid) +inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, const std::string& timeZone, bool& isValid) { - int64_t seconds; - - if (!validateTimestampRange(time)) - { - isValid = false; - return 0; - } - - if (timeZone == "SYSTEM") - { - // This is mirror of code in func_unix_timestamp.cpp - uint32_t loop; - time_t tmp_t = 0; - int shift = 0; - struct tm* l_time, tm_tmp; - int64_t diff; - localtime_r(&tmp_t, &tm_tmp); - // Get the system timezone offset at 0 seconds since epoch - int64_t my_time_zone = tm_tmp.tm_gmtoff; - int day = time.day; - - if ((time.year == MAX_TIMESTAMP_YEAR) && (time.month == 1) && (day > 4)) - { - day -= 2; - shift = 2; - } - - tmp_t = (time_t)(((calc_mysql_daynr(time.year, time.month, day, isValid) - - 719528) * 86400L + (int64_t)time.hour * 3600L + - (int64_t)(time.minute * 60 + time.second)) - (time_t)my_time_zone); - if (!isValid) - return 0; - - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - - for (loop = 0; loop < 2 && (time.hour != (uint32_t) l_time->tm_hour || - time.minute != (uint32_t) l_time->tm_min || - time.second != (uint32_t)l_time->tm_sec); loop++) - { - int days = day - l_time->tm_mday; - - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; - - diff = (3600L * (int64_t) (days * 24 + ((int) time.hour - (int) l_time->tm_hour)) + - (int64_t) (60 * ((int) time.minute - (int) l_time->tm_min)) + - (int64_t) ((int) time.second - (int) l_time->tm_sec)); - tmp_t += (time_t) diff; - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - } - - if (loop == 2 && time.hour != (uint32_t)l_time->tm_hour) - { - int days = day - l_time->tm_mday; - - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; - - diff = (3600L * (int64_t) (days * 24 + ((int) time.hour - (int) l_time->tm_hour)) + - (int64_t) (60 * ((int) time.minute - (int) l_time->tm_min)) + - (int64_t) ((int) time.second - (int) l_time->tm_sec)); - - if (diff == 3600) - tmp_t += 3600 - time.minute * 60 - time.second; /* Move to next hour */ - else if (diff == -3600) - tmp_t -= time.minute * 60 + time.second; /* Move to previous hour */ - } - - - /* shift back, if we were dealing with boundary dates */ - tmp_t += shift * 86400L; - - seconds = (int64_t)tmp_t; - } - else - { - long offset; - if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - isValid = false; - return -1; - } - seconds = secSinceEpoch(time.year, time.month, time.day, - time.hour, time.minute, time.second) - offset; - } - /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ - if (seconds >= MIN_TIMESTAMP_VALUE && seconds <= MAX_TIMESTAMP_VALUE) - return seconds; + int64_t seconds; + if (!validateTimestampRange(time)) + { isValid = false; return 0; + } + if (timeZone == "SYSTEM") + { + // This is mirror of code in func_unix_timestamp.cpp + uint32_t loop; + time_t tmp_t = 0; + int shift = 0; + struct tm *l_time, tm_tmp; + int64_t diff; + localtime_r(&tmp_t, &tm_tmp); + // Get the system timezone offset at 0 seconds since epoch + int64_t my_time_zone = tm_tmp.tm_gmtoff; + int day = time.day; + + if ((time.year == MAX_TIMESTAMP_YEAR) && (time.month == 1) && (day > 4)) + { + day -= 2; + shift = 2; + } + + tmp_t = (time_t)(((calc_mysql_daynr(time.year, time.month, day, isValid) - 719528) * 86400L + + (int64_t)time.hour * 3600L + (int64_t)(time.minute * 60 + time.second)) - + (time_t)my_time_zone); + if (!isValid) + return 0; + + localtime_r(&tmp_t, &tm_tmp); + l_time = &tm_tmp; + + for (loop = 0; + loop < 2 && (time.hour != (uint32_t)l_time->tm_hour || time.minute != (uint32_t)l_time->tm_min || + time.second != (uint32_t)l_time->tm_sec); + loop++) + { + int days = day - l_time->tm_mday; + + if (days < -1) + days = 1; /* Month has wrapped */ + else if (days > 1) + days = -1; + + diff = (3600L * (int64_t)(days * 24 + ((int)time.hour - (int)l_time->tm_hour)) + + (int64_t)(60 * ((int)time.minute - (int)l_time->tm_min)) + + (int64_t)((int)time.second - (int)l_time->tm_sec)); + tmp_t += (time_t)diff; + localtime_r(&tmp_t, &tm_tmp); + l_time = &tm_tmp; + } + + if (loop == 2 && time.hour != (uint32_t)l_time->tm_hour) + { + int days = day - l_time->tm_mday; + + if (days < -1) + days = 1; /* Month has wrapped */ + else if (days > 1) + days = -1; + + diff = (3600L * (int64_t)(days * 24 + ((int)time.hour - (int)l_time->tm_hour)) + + (int64_t)(60 * ((int)time.minute - (int)l_time->tm_min)) + + (int64_t)((int)time.second - (int)l_time->tm_sec)); + + if (diff == 3600) + tmp_t += 3600 - time.minute * 60 - time.second; /* Move to next hour */ + else if (diff == -3600) + tmp_t -= time.minute * 60 + time.second; /* Move to previous hour */ + } + + /* shift back, if we were dealing with boundary dates */ + tmp_t += shift * 86400L; + + seconds = (int64_t)tmp_t; + } + else + { + long offset; + if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + isValid = false; + return -1; + } + seconds = secSinceEpoch(time.year, time.month, time.day, time.hour, time.minute, time.second) - offset; + } + /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ + if (seconds >= MIN_TIMESTAMP_VALUE && seconds <= MAX_TIMESTAMP_VALUE) + return seconds; + + isValid = false; + return 0; } - -inline uint find_time_range(int64_t t, const int64_t *range_boundaries, uint higher_bound) +inline uint find_time_range(int64_t t, const int64_t* range_boundaries, uint higher_bound) { - uint i, lower_bound= 0; + uint i, lower_bound = 0; /* Function will work without this assertion but result would be meaningless. @@ -780,36 +746,36 @@ inline uint find_time_range(int64_t t, const int64_t *range_boundaries, uint hig while (higher_bound - lower_bound > 1) { - i= (lower_bound + higher_bound) >> 1; + i = (lower_bound + higher_bound) >> 1; if (range_boundaries[i] <= t) - lower_bound= i; + lower_bound = i; else - higher_bound= i; + higher_bound = i; } return lower_bound; } -inline int64_t TIME_to_gmt_sec(const MySQLTime& time, const TIME_ZONE_INFO *sp, uint32_t* error_code) +inline int64_t TIME_to_gmt_sec(const MySQLTime& time, const TIME_ZONE_INFO* sp, uint32_t* error_code) { int64_t local_t; uint saved_seconds; uint i; - int shift= 0; - //DBUG_ENTER("TIME_to_gmt_sec"); + int shift = 0; + // DBUG_ENTER("TIME_to_gmt_sec"); if (!validateTimestampRange(time)) { //*error_code= ER_WARN_DATA_OUT_OF_RANGE; - //DBUG_RETURN(0); - *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; - return 0; + // DBUG_RETURN(0); + *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; + return 0; } /* We need this for correct leap seconds handling */ if (time.second < SECS_PER_MIN) - saved_seconds= 0; + saved_seconds = 0; else - saved_seconds= time.second; + saved_seconds = time.second; /* NOTE: to convert full my_time_t range we do a shift of the @@ -830,13 +796,11 @@ inline int64_t TIME_to_gmt_sec(const MySQLTime& time, const TIME_ZONE_INFO *sp, want this value to be a positive number, so we shift only dates > 4.01.2038 (to avoid owerflow). */ - shift= 2; + shift = 2; } - - local_t= secSinceEpoch(time.year, time.month, (time.day - shift), - time.hour, time.minute, - saved_seconds ? 0 : time.second); + local_t = secSinceEpoch(time.year, time.month, (time.day - shift), time.hour, time.minute, + saved_seconds ? 0 : time.second); /* We have at least one range */ idbassert(sp->revcnt >= 1); @@ -847,12 +811,12 @@ inline int64_t TIME_to_gmt_sec(const MySQLTime& time, const TIME_ZONE_INFO *sp, This means that source time can't be represented as my_time_t due to limited my_time_t range. */ - *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; - return 0; + *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; + return 0; } /* binary search for our range */ - i= find_time_range(local_t, sp->revts, sp->revcnt); + i = find_time_range(local_t, sp->revts, sp->revcnt); /* As there are no offset switches at the end of TIMESTAMP range, @@ -861,13 +825,13 @@ inline int64_t TIME_to_gmt_sec(const MySQLTime& time, const TIME_ZONE_INFO *sp, */ if (shift) { - if (local_t > (int64_t) (MAX_TIMESTAMP_VALUE - shift * SECS_PER_DAY + - sp->revtis[i].rt_offset - saved_seconds)) + if (local_t > + (int64_t)(MAX_TIMESTAMP_VALUE - shift * SECS_PER_DAY + sp->revtis[i].rt_offset - saved_seconds)) { - *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; - return 0; /* my_time_t overflow */ + *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; + return 0; /* my_time_t overflow */ } - local_t+= shift * SECS_PER_DAY; + local_t += shift * SECS_PER_DAY; } if (sp->revtis[i].rt_type) @@ -879,22 +843,22 @@ inline int64_t TIME_to_gmt_sec(const MySQLTime& time, const TIME_ZONE_INFO *sp, beginning of the gap. */ //*error_code= ER_WARN_INVALID_TIMESTAMP; - local_t= sp->revts[i] - sp->revtis[i].rt_offset + saved_seconds; + local_t = sp->revts[i] - sp->revtis[i].rt_offset + saved_seconds; } else - local_t= local_t - sp->revtis[i].rt_offset + saved_seconds; + local_t = local_t - sp->revtis[i].rt_offset + saved_seconds; /* check for TIMESTAMP_MAX_VALUE was already done above */ if (local_t < MIN_TIMESTAMP_VALUE) { - *error_code= logging::ERR_FUNC_OUT_OF_RANGE_RESULT; - return 0; + *error_code = logging::ERR_FUNC_OUT_OF_RANGE_RESULT; + return 0; } return local_t; } -static const TRAN_TYPE_INFO* find_transition_type(int64_t t, const TIME_ZONE_INFO *sp) +static const TRAN_TYPE_INFO* find_transition_type(int64_t t, const TIME_ZONE_INFO* sp) { if ((sp->timecnt == 0 || t < sp->ats[0])) { @@ -913,25 +877,25 @@ static const TRAN_TYPE_INFO* find_transition_type(int64_t t, const TIME_ZONE_INF return &(sp->ttis[sp->types[find_time_range(t, sp->ats, sp->timecnt)]]); } -static void sec_to_TIME(MySQLTime * tmp, int64_t t, long offset) +static void sec_to_TIME(MySQLTime* tmp, int64_t t, long offset) { long days; long rem; int y; int yleap; - const uint *ip; + const uint* ip; - days= (long) (t / SECS_PER_DAY); - rem= (long) (t % SECS_PER_DAY); + days = (long)(t / SECS_PER_DAY); + rem = (long)(t % SECS_PER_DAY); /* We do this as separate step after dividing t, because this allows us handle times near my_time_t bounds without overflows. */ - rem+= offset; + rem += offset; while (rem < 0) { - rem+= SECS_PER_DAY; + rem += SECS_PER_DAY; days--; } while (rem >= SECS_PER_DAY) @@ -939,48 +903,46 @@ static void sec_to_TIME(MySQLTime * tmp, int64_t t, long offset) rem -= SECS_PER_DAY; days++; } - tmp->hour= (uint)(rem / SECS_PER_HOUR); - rem= rem % SECS_PER_HOUR; - tmp->minute= (uint)(rem / SECS_PER_MIN); + tmp->hour = (uint)(rem / SECS_PER_HOUR); + rem = rem % SECS_PER_HOUR; + tmp->minute = (uint)(rem / SECS_PER_MIN); /* A positive leap second requires a special representation. This uses "... ??:59:60" et seq. */ - tmp->second= (uint)(rem % SECS_PER_MIN); + tmp->second = (uint)(rem % SECS_PER_MIN); - y= EPOCH_YEAR; - while (days < 0 || days >= (long)year_lengths[yleap= isleap(y)]) + y = EPOCH_YEAR; + while (days < 0 || days >= (long)year_lengths[yleap = isleap(y)]) { int newy; - newy= y + days / DAYS_PER_NYEAR; + newy = y + days / DAYS_PER_NYEAR; if (days < 0) newy--; - days-= (newy - y) * DAYS_PER_NYEAR + - LEAPS_THRU_END_OF(newy - 1) - - LEAPS_THRU_END_OF(y - 1); - y= newy; + days -= (newy - y) * DAYS_PER_NYEAR + LEAPS_THRU_END_OF(newy - 1) - LEAPS_THRU_END_OF(y - 1); + y = newy; } - tmp->year= y; + tmp->year = y; - ip= mon_lengths[yleap]; - for (tmp->month= 0; days >= (long) ip[tmp->month]; tmp->month++) - days= days - (long) ip[tmp->month]; + ip = mon_lengths[yleap]; + for (tmp->month = 0; days >= (long)ip[tmp->month]; tmp->month++) + days = days - (long)ip[tmp->month]; tmp->month++; - tmp->day= (uint)(days + 1); + tmp->day = (uint)(days + 1); /* filling MySQL specific MYSQL_TIME members */ - tmp->second_part= 0; - tmp->time_type= CALPONTDATETIME_ENUM; + tmp->second_part = 0; + tmp->time_type = CALPONTDATETIME_ENUM; } -inline void gmt_sec_to_TIME(MySQLTime *tmp, int64_t sec_in_utc, const TIME_ZONE_INFO *sp) +inline void gmt_sec_to_TIME(MySQLTime* tmp, int64_t sec_in_utc, const TIME_ZONE_INFO* sp) { - const TRAN_TYPE_INFO *ttisp; - const LS_INFO *lp; - long corr= 0; - int hit= 0; - int i; + const TRAN_TYPE_INFO* ttisp; + const LS_INFO* lp; + long corr = 0; + int hit = 0; + int i; /* Find proper transition (and its local time type) for our sec_in_utc value. @@ -988,7 +950,7 @@ inline void gmt_sec_to_TIME(MySQLTime *tmp, int64_t sec_in_utc, const TIME_ZONE_ which very close to glibc's code. No wonder since they obviously use the same base and all steps are sensible. */ - ttisp= find_transition_type(sec_in_utc, sp); + ttisp = find_transition_type(sec_in_utc, sp); /* Let us find leap correction for our sec_in_utc value and number of extra @@ -997,19 +959,17 @@ inline void gmt_sec_to_TIME(MySQLTime *tmp, int64_t sec_in_utc, const TIME_ZONE_ leap seconds, and even in case when we have such time zone there won't be many iterations (we have about 22 corrections at this moment (2004)). */ - for ( i= sp->leapcnt; i-- > 0; ) + for (i = sp->leapcnt; i-- > 0;) { - lp= &sp->lsis[i]; + lp = &sp->lsis[i]; if (sec_in_utc >= lp->ls_trans) { if (sec_in_utc == lp->ls_trans) { - hit= ((i == 0 && lp->ls_corr > 0) || - lp->ls_corr > sp->lsis[i - 1].ls_corr); + hit = ((i == 0 && lp->ls_corr > 0) || lp->ls_corr > sp->lsis[i - 1].ls_corr); if (hit) { - while (i > 0 && - sp->lsis[i].ls_trans == sp->lsis[i - 1].ls_trans + 1 && + while (i > 0 && sp->lsis[i].ls_trans == sp->lsis[i - 1].ls_trans + 1 && sp->lsis[i].ls_corr == sp->lsis[i - 1].ls_corr + 1) { hit++; @@ -1017,85 +977,96 @@ inline void gmt_sec_to_TIME(MySQLTime *tmp, int64_t sec_in_utc, const TIME_ZONE_ } } } - corr= lp->ls_corr; + corr = lp->ls_corr; break; } } sec_to_TIME(tmp, sec_in_utc, ttisp->tt_gmtoff - corr); - tmp->second+= hit; + tmp->second += hit; } /** @brief a structure to hold a date */ struct Date { - unsigned spare : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; - // NULL column value = 0xFFFFFFFE - Date( ) : - spare(0x3E), day(0x3F), month(0xF), year(0xFFFF) {} - // Construct a Date from a 64 bit integer Calpont date. - Date(uint64_t val) : - spare(0x3E), day((val >> 6) & 077), month((val >> 12) & 0xF), year((val >> 16)) {} - // Construct using passed in parameters, no value checking - Date(unsigned y, unsigned m, unsigned d) : spare(0x3E), day(d), month(m), year(y) {} + unsigned spare : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; + // NULL column value = 0xFFFFFFFE + Date() : spare(0x3E), day(0x3F), month(0xF), year(0xFFFF) + { + } + // Construct a Date from a 64 bit integer Calpont date. + Date(uint64_t val) : spare(0x3E), day((val >> 6) & 077), month((val >> 12) & 0xF), year((val >> 16)) + { + } + // Construct using passed in parameters, no value checking + Date(unsigned y, unsigned m, unsigned d) : spare(0x3E), day(d), month(m), year(y) + { + } - int32_t convertToMySQLint() const; + int32_t convertToMySQLint() const; }; -inline -int32_t Date::convertToMySQLint() const +inline int32_t Date::convertToMySQLint() const { - return (int32_t) (year * 10000) + (month * 100) + day; + return (int32_t)(year * 10000) + (month * 100) + day; } /** @brief a structure to hold a datetime */ struct DateTime { - unsigned msecond : 20; - unsigned second : 6; - unsigned minute : 6; - unsigned hour : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; - // NULL column value = 0xFFFFFFFFFFFFFFFE - DateTime( ) : - msecond(0xFFFFE), second(0x3F), minute(0x3F), hour(0x3F), day(0x3F), month(0xF), year(0xFFFF) {} - // Construct a DateTime from a 64 bit integer Calpont datetime. - DateTime(uint64_t val) : - msecond(val & 0xFFFFF), second((val >> 20) & 077), minute((val >> 26) & 077), - hour((val >> 32) & 077), day((val >> 38) & 077), month((val >> 44) & 0xF), - year(val >> 48) {} - // Construct using passed in parameters, no value checking - DateTime(unsigned y, unsigned m, unsigned d, unsigned h, unsigned min, unsigned sec, unsigned msec) : - msecond(msec), second(sec), minute(min), hour(h), day(d), month(m), year(y) {} + unsigned msecond : 20; + unsigned second : 6; + unsigned minute : 6; + unsigned hour : 6; + unsigned day : 6; + unsigned month : 4; + unsigned year : 16; + // NULL column value = 0xFFFFFFFFFFFFFFFE + DateTime() : msecond(0xFFFFE), second(0x3F), minute(0x3F), hour(0x3F), day(0x3F), month(0xF), year(0xFFFF) + { + } + // Construct a DateTime from a 64 bit integer Calpont datetime. + DateTime(uint64_t val) + : msecond(val & 0xFFFFF) + , second((val >> 20) & 077) + , minute((val >> 26) & 077) + , hour((val >> 32) & 077) + , day((val >> 38) & 077) + , month((val >> 44) & 0xF) + , year(val >> 48) + { + } + // Construct using passed in parameters, no value checking + DateTime(unsigned y, unsigned m, unsigned d, unsigned h, unsigned min, unsigned sec, unsigned msec) + : msecond(msec), second(sec), minute(min), hour(h), day(d), month(m), year(y) + { + } - int64_t convertToMySQLint() const; - void reset(); + int64_t convertToMySQLint() const; + void reset(); }; -inline -int64_t DateTime::convertToMySQLint() const +inline int64_t DateTime::convertToMySQLint() const { - return (int64_t) (year * 10000000000LL) + (month * 100000000) + (day * 1000000) + (hour * 10000) + (minute * 100) + second; + return (int64_t)(year * 10000000000LL) + (month * 100000000) + (day * 1000000) + (hour * 10000) + + (minute * 100) + second; } -inline -void DateTime::reset() +inline void DateTime::reset() { - msecond = 0xFFFFE; - second = 0x3F; - minute = 0x3F; - hour = 0x3F; - day = 0x3F; - month = 0xF; - year = 0xFFFF; + msecond = 0xFFFFE; + second = 0x3F; + minute = 0x3F; + hour = 0x3F; + day = 0x3F; + month = 0xF; + year = 0xFFFF; } /** @brief a structure to hold a time @@ -1103,686 +1074,638 @@ void DateTime::reset() */ struct Time { - signed msecond : 24; - signed second : 8; - signed minute : 8; - signed hour : 12; - signed day : 11; - signed is_neg : 1; + signed msecond : 24; + signed second : 8; + signed minute : 8; + signed hour : 12; + signed day : 11; + signed is_neg : 1; - // NULL column value = 0xFFFFFFFFFFFFFFFE - Time() : msecond (-2), - second (-1), - minute (-1), - hour (-1), - day (-1), - is_neg (0b1) - {} + // NULL column value = 0xFFFFFFFFFFFFFFFE + Time() : msecond(-2), second(-1), minute(-1), hour(-1), day(-1), is_neg(0b1) + { + } - // Construct a Time from a 64 bit integer InfiniDB time. - Time(int64_t val) : - msecond(val & 0xffffff), - second((val >> 24) & 0xff), - minute((val >> 32) & 0xff), - hour((val >> 40) & 0xfff), - day((val >> 52) & 0x7ff), - is_neg(val >> 63) - {} + // Construct a Time from a 64 bit integer InfiniDB time. + Time(int64_t val) + : msecond(val & 0xffffff) + , second((val >> 24) & 0xff) + , minute((val >> 32) & 0xff) + , hour((val >> 40) & 0xfff) + , day((val >> 52) & 0x7ff) + , is_neg(val >> 63) + { + } - Time(signed d, signed h, signed min, signed sec, signed msec, bool neg) : - msecond(msec), second(sec), minute(min), hour(h), day(d), is_neg(neg) - { - if (h < 0) - is_neg = 0b1; - } + Time(signed d, signed h, signed min, signed sec, signed msec, bool neg) + : msecond(msec), second(sec), minute(min), hour(h), day(d), is_neg(neg) + { + if (h < 0) + is_neg = 0b1; + } - int64_t convertToMySQLint() const; - void reset(); + int64_t convertToMySQLint() const; + void reset(); }; -inline -void Time::reset() +inline void Time::reset() { - msecond = -2; - second = -1; - minute = -1; - hour = -1; - is_neg = 0b1; - day = -1; + msecond = -2; + second = -1; + minute = -1; + hour = -1; + is_neg = 0b1; + day = -1; } -inline -int64_t Time::convertToMySQLint() const +inline int64_t Time::convertToMySQLint() const { - if ((hour >= 0) && is_neg) - { - return (int64_t) ((hour * 10000) + (minute * 100) + second) * -1; - } - else if (hour >= 0) - { - return (int64_t) (hour * 10000) + (minute * 100) + second; - } - else - { - return (int64_t) (hour * 10000) - (minute * 100) - second; - } + if ((hour >= 0) && is_neg) + { + return (int64_t)((hour * 10000) + (minute * 100) + second) * -1; + } + else if (hour >= 0) + { + return (int64_t)(hour * 10000) + (minute * 100) + second; + } + else + { + return (int64_t)(hour * 10000) - (minute * 100) - second; + } } /** @brief a structure to hold a timestamp */ struct TimeStamp { - unsigned msecond : 20; - unsigned long long second : 44; - // NULL column value = 0xFFFFFFFFFFFFFFFE - TimeStamp( ) : - msecond(0xFFFFE), second(0xFFFFFFFFFFF) {} - // Construct a TimeStamp from a 64 bit integer Calpont timestamp. - TimeStamp(uint64_t val) : - msecond(val & 0xFFFFF), second(val >> 20) {} - TimeStamp(unsigned msec, unsigned long long sec) : - msecond(msec), second(sec) {} + unsigned msecond : 20; + unsigned long long second : 44; + // NULL column value = 0xFFFFFFFFFFFFFFFE + TimeStamp() : msecond(0xFFFFE), second(0xFFFFFFFFFFF) + { + } + // Construct a TimeStamp from a 64 bit integer Calpont timestamp. + TimeStamp(uint64_t val) : msecond(val & 0xFFFFF), second(val >> 20) + { + } + TimeStamp(unsigned msec, unsigned long long sec) : msecond(msec), second(sec) + { + } - int64_t convertToMySQLint(const std::string& timeZone) const; - void reset(); + int64_t convertToMySQLint(const std::string& timeZone) const; + void reset(); }; -inline -int64_t TimeStamp::convertToMySQLint(const std::string& timeZone) const +inline int64_t TimeStamp::convertToMySQLint(const std::string& timeZone) const { - const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 - char buf[TIMESTAMPTOSTRING1_LEN]; + const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 + char buf[TIMESTAMPTOSTRING1_LEN]; - MySQLTime time; - gmtSecToMySQLTime(second, time, timeZone); + MySQLTime time; + gmtSecToMySQLTime(second, time, timeZone); - sprintf(buf, "%04d%02d%02d%02d%02d%02d", time.year, time.month, time.day, time.hour, time.minute, time.second); + sprintf(buf, "%04d%02d%02d%02d%02d%02d", time.year, time.month, time.day, time.hour, time.minute, + time.second); - return (int64_t) atoll(buf); + return (int64_t)atoll(buf); } -inline -void TimeStamp::reset() +inline void TimeStamp::reset() { - msecond = 0xFFFFE; - second = 0xFFFFFFFFFFF; + msecond = 0xFFFFE; + second = 0xFFFFFFFFFFF; } -template -inline T string_to_ll( const std::string& data, bool& bSaturate ) +template +inline T string_to_ll(const std::string& data, bool& bSaturate) { - // This function doesn't take into consideration our special values - // for NULL and EMPTY when setting the saturation point. Should it? - char* ep = NULL; - const char* str = data.c_str(); - errno = 0; - int64_t value = strtoll(str, &ep, 10); + // This function doesn't take into consideration our special values + // for NULL and EMPTY when setting the saturation point. Should it? + char* ep = NULL; + const char* str = data.c_str(); + errno = 0; + int64_t value = strtoll(str, &ep, 10); - // (no digits) || (more chars) || (other errors & value = 0) - if ((ep == str) || (*ep != '\0') || (errno != 0 && value == 0)) - throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); + // (no digits) || (more chars) || (other errors & value = 0) + if ((ep == str) || (*ep != '\0') || (errno != 0 && value == 0)) + throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); - if (errno == ERANGE && (value == std::numeric_limits::max() || value == std::numeric_limits::min())) - bSaturate = true; + if (errno == ERANGE && + (value == std::numeric_limits::max() || value == std::numeric_limits::min())) + bSaturate = true; - return value; + return value; } -inline -uint64_t string_to_ull( const std::string& data, bool& bSaturate ) +inline uint64_t string_to_ull(const std::string& data, bool& bSaturate) { - // This function doesn't take into consideration our special values - // for NULL and EMPTY when setting the saturation point. Should it? - char* ep = NULL; - const char* str = data.c_str(); - errno = 0; + // This function doesn't take into consideration our special values + // for NULL and EMPTY when setting the saturation point. Should it? + char* ep = NULL; + const char* str = data.c_str(); + errno = 0; - // check for negative number. saturate to 0; - if (data.find('-') != data.npos) - { - bSaturate = true; - return 0; - } + // check for negative number. saturate to 0; + if (data.find('-') != data.npos) + { + bSaturate = true; + return 0; + } - uint64_t value = strtoull(str, &ep, 10); + uint64_t value = strtoull(str, &ep, 10); - // (no digits) || (more chars) || (other errors & value = 0) - if ((ep == str) || (*ep != '\0') || (errno != 0 && value == 0)) - throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); + // (no digits) || (more chars) || (other errors & value = 0) + if ((ep == str) || (*ep != '\0') || (errno != 0 && value == 0)) + throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); - if (errno == ERANGE && (value == std::numeric_limits::max())) - bSaturate = true; + if (errno == ERANGE && (value == std::numeric_limits::max())) + bSaturate = true; - return value; + return value; } template -void number_int_value(const std::string& data, - cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd &ct, - bool& pushwarning, - bool noRoundup, - T& intVal, bool* saturate = 0); +void number_int_value(const std::string& data, cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, + bool noRoundup, T& intVal, bool* saturate = 0); -uint64_t number_uint_value(const string& data, - cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd& ct, - bool& pushwarning, - bool noRoundup); +uint64_t number_uint_value(const string& data, cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, + bool noRoundup); /** @brief DataConvert is a component for converting string data to Calpont format - */ + */ class DataConvert { -public: + public: + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string dateToString(int datevalue); + static inline void dateToString(int datevalue, char* buf, unsigned int buflen); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string dateToString( int datevalue ); - static inline void dateToString( int datevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string datetimeToString(long long datetimevalue, long decimals = 0); + static inline void datetimeToString(long long datetimevalue, char* buf, unsigned int buflen, + long decimals = 0); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); - static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timestampToString(long long timestampvalue, const std::string& timezone, + long decimals = 0); + static inline void timestampToString(long long timestampvalue, char* buf, unsigned int buflen, + const std::string& timezone, long decimals = 0); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string timestampToString( long long timestampvalue, const std::string& timezone, long decimals = 0 ); - static inline void timestampToString( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone, long decimals = 0 ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString(long long timevalue, long decimals = 0); + static inline void timeToString(long long timevalue, char* buf, unsigned int buflen, long decimals = 0); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); - static inline void timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals = 0); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string dateToString1(int datevalue); + static inline void dateToString1(int datevalue, char* buf, unsigned int buflen); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string dateToString1( int datevalue ); - static inline void dateToString1( int datevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string datetimeToString1(long long datetimevalue); + static inline void datetimeToString1(long long datetimevalue, char* buf, unsigned int buflen); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string datetimeToString1( long long datetimevalue ); - static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timestampToString1(long long timestampvalue, const std::string& timezone); + static inline void timestampToString1(long long timestampvalue, char* buf, unsigned int buflen, + const std::string& timezone); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string timestampToString1( long long timestampvalue, const std::string& timezone ); - static inline void timestampToString1( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString1(long long timevalue); + static inline void timeToString1(long long timevalue, char* buf, unsigned int buflen); - /** - * @brief convert a columns data from native format to a string - * - * @param type the columns database type - * @param data the columns string representation of it's data - */ - EXPORT static std::string timeToString1( long long timevalue ); - static inline void timeToString1( long long timevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a date column data, represnted as a string, to it's native + * format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param dateFormat the format the date value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int32_t convertColumnDate(const char* dataOrg, CalpontDateTimeFormat dateFormat, int& status, + unsigned int dataOrgLen); - /** - * @brief convert a date column data, represnted as a string, to it's native - * format. This function is for bulkload to use. - * - * @param type the columns data type - * @param dataOrig the columns string representation of it's data - * @param dateFormat the format the date value in - * @param status 0 - success, -1 - fail - * @param dataOrgLen length specification of dataOrg - */ - EXPORT static int32_t convertColumnDate( const char* dataOrg, - CalpontDateTimeFormat dateFormat, - int& status, unsigned int dataOrgLen ); + /** + * @brief Is specified date valid; used by binary bulk load + */ + EXPORT static bool isColumnDateValid(int32_t date); - /** - * @brief Is specified date valid; used by binary bulk load - */ - EXPORT static bool isColumnDateValid( int32_t date ); + /** + * @brief convert a datetime column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param datetimeFormat the format the date value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int64_t convertColumnDatetime(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen); - /** - * @brief convert a datetime column data, represented as a string, - * to it's native format. This function is for bulkload to use. - * - * @param type the columns data type - * @param dataOrig the columns string representation of it's data - * @param datetimeFormat the format the date value in - * @param status 0 - success, -1 - fail - * @param dataOrgLen length specification of dataOrg - */ - EXPORT static int64_t convertColumnDatetime( const char* dataOrg, - CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen ); + /** + * @brief convert a timestamp column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param dataOrg the columns string representation of it's data + * @param datetimeFormat the format the date value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + * @param timeZone the timezone used for conversion to native format + */ + EXPORT static int64_t convertColumnTimestamp(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen, + const std::string& timeZone); - /** - * @brief convert a timestamp column data, represented as a string, - * to it's native format. This function is for bulkload to use. - * - * @param dataOrg the columns string representation of it's data - * @param datetimeFormat the format the date value in - * @param status 0 - success, -1 - fail - * @param dataOrgLen length specification of dataOrg - * @param timeZone the timezone used for conversion to native format - */ - EXPORT static int64_t convertColumnTimestamp( const char* dataOrg, - CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen, - const std::string& timeZone ); + /** + * @brief convert a time column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param timeFormat the format the time value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int64_t convertColumnTime(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen); - /** - * @brief convert a time column data, represented as a string, - * to it's native format. This function is for bulkload to use. - * - * @param type the columns data type - * @param dataOrig the columns string representation of it's data - * @param timeFormat the format the time value in - * @param status 0 - success, -1 - fail - * @param dataOrgLen length specification of dataOrg - */ - EXPORT static int64_t convertColumnTime( const char* dataOrg, - CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen ); + /** + * @brief Is specified datetime valid; used by binary bulk load + */ + EXPORT static bool isColumnDateTimeValid(int64_t dateTime); + EXPORT static bool isColumnTimeValid(int64_t time); + EXPORT static bool isColumnTimeStampValid(int64_t timeStamp); - /** - * @brief Is specified datetime valid; used by binary bulk load - */ - EXPORT static bool isColumnDateTimeValid( int64_t dateTime ); - EXPORT static bool isColumnTimeValid( int64_t time ); - EXPORT static bool isColumnTimeStampValid( int64_t timeStamp ); + static inline void trimWhitespace(int64_t& charData); - static inline void trimWhitespace(int64_t& charData); + // convert string to date + EXPORT static int64_t stringToDate(const std::string& data); + // convert string to datetime + EXPORT static int64_t stringToDatetime(const std::string& data, bool* isDate = NULL); + // convert string to timestamp + EXPORT static int64_t stringToTimestamp(const std::string& data, const std::string& timeZone); + // convert integer to date + EXPORT static int64_t intToDate(int64_t data); + // convert integer to datetime + EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL); + // convert integer to date + EXPORT static int64_t intToTime(int64_t data, bool fromString = false); + // convert string to date. alias to stringToDate + EXPORT static int64_t dateToInt(const std::string& date); + // convert string to datetime. alias to datetimeToInt + EXPORT static int64_t datetimeToInt(const std::string& datetime); + EXPORT static int64_t timestampToInt(const std::string& timestamp, const std::string& timeZone); + EXPORT static int64_t timeToInt(const std::string& time); + EXPORT static int64_t stringToTime(const std::string& data); + // bug4388, union type conversion + EXPORT static void joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd& unionedType, + const datatypes::SystemCatalog::TypeHolderStd& type); - // convert string to date - EXPORT static int64_t stringToDate(const std::string& data); - // convert string to datetime - EXPORT static int64_t stringToDatetime(const std::string& data, bool* isDate = NULL); - // convert string to timestamp - EXPORT static int64_t stringToTimestamp(const std::string& data, const std::string& timeZone); - // convert integer to date - EXPORT static int64_t intToDate(int64_t data); - // convert integer to datetime - EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL); - // convert integer to date - EXPORT static int64_t intToTime(int64_t data, bool fromString = false); - // convert string to date. alias to stringToDate - EXPORT static int64_t dateToInt(const std::string& date); - // convert string to datetime. alias to datetimeToInt - EXPORT static int64_t datetimeToInt(const std::string& datetime); - EXPORT static int64_t timestampToInt(const std::string& timestamp, const std::string& timeZone); - EXPORT static int64_t timeToInt(const std::string& time); - EXPORT static int64_t stringToTime (const std::string& data); - // bug4388, union type conversion - EXPORT static void joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, - const datatypes::SystemCatalog::TypeHolderStd &type); + static boost::any StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam& prm, const std::string& dataOrig, + bool& pushWarning); - static boost::any StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const datatypes::ConvertFromStringParam &prm, - const std::string& dataOrig, - bool& pushWarning); - - static boost::any StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const datatypes::ConvertFromStringParam &prm, - const std::string& data, - bool& pushWarning); - - static boost::any StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const datatypes::ConvertFromStringParam &prm, - const std::string& data, - bool& pushWarning); - - static boost::any StringToFloat(cscDataType typeCode, - const std::string& dataOrig, - bool& pushWarning); - - static boost::any StringToDouble(cscDataType typeCode, - const std::string& dataOrig, + static boost::any StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam& prm, const std::string& data, bool& pushWarning); - static boost::any StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const std::string& dataOrig, + static boost::any StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam& prm, const std::string& data, bool& pushWarning); - static boost::any StringToDate(const std::string& data, - bool& pushWarning); + static boost::any StringToFloat(cscDataType typeCode, const std::string& dataOrig, bool& pushWarning); - static boost::any StringToDatetime(const std::string& data, - bool& pushWarning); + static boost::any StringToDouble(cscDataType typeCode, const std::string& dataOrig, bool& pushWarning); - static boost::any StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, - const std::string& data, - bool& pushWarning); + static boost::any StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& dataOrig, bool& pushWarning); - static boost::any StringToTimestamp(const datatypes::ConvertFromStringParam &prm, - const std::string& data, - bool& pushWarning); + static boost::any StringToDate(const std::string& data, bool& pushWarning); + + static boost::any StringToDatetime(const std::string& data, bool& pushWarning); + + static boost::any StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& data, bool& pushWarning); + + static boost::any StringToTimestamp(const datatypes::ConvertFromStringParam& prm, const std::string& data, + bool& pushWarning); }; -inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int buflen) +inline void DataConvert::dateToString(int datevalue, char* buf, unsigned int buflen) { - snprintf( buf, buflen, "%04d-%02d-%02d", - (unsigned)((datevalue >> 16) & 0xffff), - (unsigned)((datevalue >> 12) & 0xf), - (unsigned)((datevalue >> 6) & 0x3f) - ); + snprintf(buf, buflen, "%04d-%02d-%02d", (unsigned)((datevalue >> 16) & 0xffff), + (unsigned)((datevalue >> 12) & 0xf), (unsigned)((datevalue >> 6) & 0x3f)); } -inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals ) +inline void DataConvert::datetimeToString(long long datetimevalue, char* buf, unsigned int buflen, + long decimals) { - // 10 is default which means we don't need microseconds - if (decimals > 6 || decimals < 0) - { - decimals = 0; - } + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } - int msec = 0; + int msec = 0; - if ((datetimevalue & 0xfffff) > 0) - { - msec = (unsigned)((datetimevalue) & 0xfffff); - } + if ((datetimevalue & 0xfffff) > 0) + { + msec = (unsigned)((datetimevalue)&0xfffff); + } - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f) - ); + snprintf(buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f)); - if (msec || decimals) - { - snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, msec); - } + if (msec || decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, msec); + } } -inline void DataConvert::timestampToString( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone, long decimals ) +inline void DataConvert::timestampToString(long long timestampvalue, char* buf, unsigned int buflen, + const std::string& timezone, long decimals) { - // 10 is default which means we don't need microseconds - if (decimals > 6 || decimals < 0) - { - decimals = 0; - } + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } - TimeStamp timestamp(timestampvalue); - int64_t seconds = timestamp.second; + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; - MySQLTime time; - gmtSecToMySQLTime(seconds, time, timezone); + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", - time.year, time.month, time.day, - time.hour, time.minute, time.second - ); + snprintf(buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", time.year, time.month, time.day, time.hour, + time.minute, time.second); - if (timestamp.msecond || decimals) - { - snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, timestamp.msecond); - } + if (timestamp.msecond || decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, timestamp.msecond); + } } -inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals ) +inline void DataConvert::timeToString(long long timevalue, char* buf, unsigned int buflen, long decimals) { - // 10 is default which means we don't need microseconds - if (decimals > 6 || decimals < 0) - { - decimals = 0; - } + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } - // Handle negative correctly - int hour = 0, msec = 0; + // Handle negative correctly + int hour = 0, msec = 0; - if ((timevalue >> 40) & 0x800) - { - hour = 0xfffff000; - } + if ((timevalue >> 40) & 0x800) + { + hour = 0xfffff000; + } - hour |= ((timevalue >> 40) & 0xfff); + hour |= ((timevalue >> 40) & 0xfff); - if ((timevalue & 0xffffff) > 0) - { - msec = (unsigned)((timevalue) & 0xffffff); - } + if ((timevalue & 0xffffff) > 0) + { + msec = (unsigned)((timevalue)&0xffffff); + } - if ((hour >= 0) && (timevalue >> 63)) - { - buf[0] = '-'; - buf++; - buflen--; - } + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } - snprintf( buf, buflen, "%02d:%02d:%02d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 24) & 0xff) - ); + snprintf(buf, buflen, "%02d:%02d:%02d", hour, (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff)); - if (msec || decimals) - { - // Pad start with zeros - snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, msec); - } + if (msec || decimals) + { + // Pad start with zeros + snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, msec); + } } -inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) +inline void DataConvert::dateToString1(int datevalue, char* buf, unsigned int buflen) { - snprintf( buf, buflen, "%04d%02d%02d", - (unsigned)((datevalue >> 16) & 0xffff), - (unsigned)((datevalue >> 12) & 0xf), - (unsigned)((datevalue >> 6) & 0x3f) - ); + snprintf(buf, buflen, "%04d%02d%02d", (unsigned)((datevalue >> 16) & 0xffff), + (unsigned)((datevalue >> 12) & 0xf), (unsigned)((datevalue >> 6) & 0x3f)); } -inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ) +inline void DataConvert::datetimeToString1(long long datetimevalue, char* buf, unsigned int buflen) { - snprintf( buf, buflen, "%04d%02d%02d%02d%02d%02d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f) - ); + snprintf(buf, buflen, "%04d%02d%02d%02d%02d%02d", (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f)); } -inline void DataConvert::timestampToString1( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone ) +inline void DataConvert::timestampToString1(long long timestampvalue, char* buf, unsigned int buflen, + const std::string& timezone) { - TimeStamp timestamp(timestampvalue); - int64_t seconds = timestamp.second; + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; - MySQLTime time; - gmtSecToMySQLTime(seconds, time, timezone); + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); - snprintf( buf, buflen, "%04d%02d%02d%02d%02d%02d", - time.year, time.month, time.day, - time.hour, time.minute, time.second - ); + snprintf(buf, buflen, "%04d%02d%02d%02d%02d%02d", time.year, time.month, time.day, time.hour, time.minute, + time.second); } -inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) +inline void DataConvert::timeToString1(long long timevalue, char* buf, unsigned int buflen) { - // Handle negative correctly - int hour = 0; + // Handle negative correctly + int hour = 0; - if ((timevalue >> 40) & 0x800) - { - hour = 0xfffff000; - } + if ((timevalue >> 40) & 0x800) + { + hour = 0xfffff000; + } - hour |= ((timevalue >> 40) & 0xfff); + hour |= ((timevalue >> 40) & 0xfff); - if ((hour >= 0) && (timevalue >> 63)) - { - buf[0] = '-'; - buf++; - buflen--; - } - // this snprintf call causes a compiler warning b/c buffer size is less - // then maximum string size. + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + // this snprintf call causes a compiler warning b/c buffer size is less + // then maximum string size. #if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" - snprintf( buf, buflen, "%02d%02d%02d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 14) & 0xff) - ); + snprintf(buf, buflen, "%02d%02d%02d", hour, (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 14) & 0xff)); #pragma GCC diagnostic pop #else - snprintf( buf, buflen, "%02d%02d%02d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 14) & 0xff) - ); + snprintf(buf, buflen, "%02d%02d%02d", hour, (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 14) & 0xff)); #endif } inline void DataConvert::trimWhitespace(int64_t& charData) { - // Trims whitespace characters off non-dict character data - char* ch_data = (char*) &charData; + // Trims whitespace characters off non-dict character data + char* ch_data = (char*)&charData; - for (int8_t i = 7; i > 0; i--) - { - if (ch_data[i] == ' ' || ch_data[i] == '\0') - ch_data[i] = '\0'; - else - break; - } + for (int8_t i = 7; i > 0; i--) + { + if (ch_data[i] == ' ' || ch_data[i] == '\0') + ch_data[i] = '\0'; + else + break; + } } - inline int128_t add128(int128_t a, int128_t b) { - return a + b; + return a + b; } inline int128_t subtract128(int128_t a, int128_t b) { - return a - b; + return a - b; } inline bool lessThan128(int128_t a, int128_t b) { - return a < b; + return a < b; } inline bool greaterThan128(int128_t a, int128_t b) { - return a > b; + return a > b; } // Naive int128_t version of strtoll inline int128_t strtoll128(const char* data, bool& saturate, char** ep) { - int128_t res = 0; - - if (*data == '\0') - { - if (ep) - *ep = (char*)data; - return res; - } - - // skip leading whitespace characters - while (*data != '\0' && - (*data == ' ' || *data == '\t' || *data == '\n')) - data++; - - int128_t (*op)(int128_t, int128_t); - op = add128; - bool (*compare)(int128_t, int128_t); - compare = lessThan128; - - // check the -ve sign - bool is_neg = false; - if (*data == '-') - { - is_neg = true; - op = subtract128; - compare = greaterThan128; - data++; - } - - int128_t tmp; - - for (; *data != '\0' && isdigit(*data); data++) - { - tmp = op(res*10, *data - '0'); - - if (UNLIKELY(compare(tmp, res))) - { - saturate = true; - - if (is_neg) - utils::int128Min(res); - else - utils::int128Max(res); - - while (*data != '\0' && isdigit(*data)) - data++; - - if (ep) - *ep = (char*)data; - - return res; - } - - res = tmp; - } + int128_t res = 0; + if (*data == '\0') + { if (ep) + *ep = (char*)data; + return res; + } + + // skip leading whitespace characters + while (*data != '\0' && (*data == ' ' || *data == '\t' || *data == '\n')) + data++; + + int128_t (*op)(int128_t, int128_t); + op = add128; + bool (*compare)(int128_t, int128_t); + compare = lessThan128; + + // check the -ve sign + bool is_neg = false; + if (*data == '-') + { + is_neg = true; + op = subtract128; + compare = greaterThan128; + data++; + } + + int128_t tmp; + + for (; *data != '\0' && isdigit(*data); data++) + { + tmp = op(res * 10, *data - '0'); + + if (UNLIKELY(compare(tmp, res))) + { + saturate = true; + + if (is_neg) + utils::int128Min(res); + else + utils::int128Max(res); + + while (*data != '\0' && isdigit(*data)) + data++; + + if (ep) *ep = (char*)data; - return res; + return res; + } + + res = tmp; + } + + if (ep) + *ep = (char*)data; + + return res; } -template<> -inline int128_t string_to_ll ( const std::string& data, bool& bSaturate ) +template <> +inline int128_t string_to_ll(const std::string& data, bool& bSaturate) { - // This function doesn't take into consideration our special values - // for NULL and EMPTY when setting the saturation point. Should it? - char* ep = NULL; - const char* str = data.c_str(); - int128_t value = strtoll128(str, bSaturate, &ep); + // This function doesn't take into consideration our special values + // for NULL and EMPTY when setting the saturation point. Should it? + char* ep = NULL; + const char* str = data.c_str(); + int128_t value = strtoll128(str, bSaturate, &ep); - // (no digits) || (more chars) - if ((ep == str) || (*ep != '\0')) - throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); + // (no digits) || (more chars) + if ((ep == str) || (*ep != '\0')) + throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); - return value; + return value; } -} // namespace dataconvert +} // namespace dataconvert #undef EXPORT -#endif //DATACONVERT_H - +#endif // DATACONVERT_H diff --git a/utils/dataconvert/tdriver.cpp b/utils/dataconvert/tdriver.cpp index 6640a6a6e..6d011d676 100644 --- a/utils/dataconvert/tdriver.cpp +++ b/utils/dataconvert/tdriver.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /** -* $Id: tdriver.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -*/ + * $Id: tdriver.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + */ #include #include @@ -37,1395 +37,1339 @@ using namespace dataconvert; class DataConvertTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DataConvertTest); + + CPPUNIT_TEST(dc1); + CPPUNIT_TEST(dc2); + CPPUNIT_TEST(dc3); + CPPUNIT_TEST(dc4); + CPPUNIT_TEST(dc5); + CPPUNIT_TEST(dc6); + CPPUNIT_TEST(dc7); + CPPUNIT_TEST(dc8); + CPPUNIT_TEST(dc_datetest); + CPPUNIT_TEST(dc_datetostrtest); + CPPUNIT_TEST(dc_datetostr1test); + CPPUNIT_TEST(dc_datetomysqlinttest); + CPPUNIT_TEST(dc_datetest_perf); + CPPUNIT_TEST(dc_datetimetest); + CPPUNIT_TEST(dc_datetimetostrtest); + CPPUNIT_TEST(dc_datetimetostr1test); + CPPUNIT_TEST(dc_datetimetest_perf); + CPPUNIT_TEST(dc_datetimetomysqlinttest); + CPPUNIT_TEST(dc_datevalidtest); + CPPUNIT_TEST(dc9); + CPPUNIT_TEST(dc10); + CPPUNIT_TEST_SUITE_END(); + + private: + DataConvert converter; + boost::any anyval; + CalpontSystemCatalog::ColType ct; + string data; + + public: + void setUp() + { + } + + void tearDown() + { + } + + /* from calpontsystemcatalog.h + struct ColType + { + int colWidth; + ConstraintType constraintType; + ColDataType colDataType; + DictOID ddn; + boost::any defaultValue; + int colPosition; // temporally put here. may need to have ColInfo struct later + int scale; //number after decimal points + int precision; + }; + colWidth + constraintType + colDataType + defaultValue + scale + precision + */ + + // mid-range positive integers + void dc1() + { + ct.colWidth = 1; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::TINYINT; + ct.scale = -1; + ct.precision = 2; + data = "12"; + bool pushWarning; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(char)); + char tinyintval = any_cast(anyval); + CPPUNIT_ASSERT(tinyintval == 12); + + ct.colWidth = 2; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::SMALLINT; + ct.scale = -1; + ct.precision = 4; + data = "1234"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + int16_t smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 1234); + + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = -1; + ct.precision = 9; + data = "12345678"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + int32_t intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 12345678); + + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::MEDINT; + ct.scale = -1; + ct.precision = 9; + data = "12345678"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 12345678); + + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = -1; + ct.precision = 18; + data = "1234567890123456"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123456LL); + } + + // mid-range negative integers + void dc2() + { + ct.colWidth = 1; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::TINYINT; + ct.scale = -1; + ct.precision = 2; + data = "-12"; + bool pushWarning; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(char)); + char tinyintval = any_cast(anyval); + CPPUNIT_ASSERT(tinyintval == -12); + + ct.colWidth = 2; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::SMALLINT; + ct.scale = -1; + ct.precision = 4; + data = "-1234"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + int16_t smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -1234); + + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = -1; + ct.precision = 9; + data = "-12345678"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + int32_t intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -12345678); + + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::MEDINT; + ct.scale = -1; + ct.precision = 9; + data = "-12345678"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -12345678); + + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = -1; + ct.precision = 18; + data = "-1234567890123456"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -1234567890123456LL); + } + + // limit positive integers + void dc3() + { + ct.colWidth = 1; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::TINYINT; + ct.scale = -1; + ct.precision = 3; + data = "127"; + bool pushWarning; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(char)); + char tinyintval = any_cast(anyval); + CPPUNIT_ASSERT(tinyintval == 127); + + ct.colWidth = 2; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::SMALLINT; + ct.scale = -1; + ct.precision = 5; + data = "32767"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + int16_t smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 32767); + + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = -1; + ct.precision = 10; + data = "2147483647"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + int32_t intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 2147483647); + + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::MEDINT; + ct.scale = -1; + ct.precision = 10; + data = "2147483647"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 2147483647); + + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = 0; + ct.precision = 18; + data = "9223372036854775807"; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 9223372036854775807LL); + } + + // limit negative integers + void dc4() + { + ct.colWidth = 1; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::TINYINT; + ct.scale = -1; + ct.precision = 3; + data = "-126"; + bool pushWarning; + + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(char)); + char tinyintval = any_cast(anyval); + CPPUNIT_ASSERT(tinyintval == -126); + + ct.colWidth = 2; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::SMALLINT; + ct.scale = -1; + ct.precision = 4; + data = "-32766"; - CPPUNIT_TEST_SUITE( DataConvertTest ); - - CPPUNIT_TEST( dc1 ); - CPPUNIT_TEST( dc2 ); - CPPUNIT_TEST( dc3 ); - CPPUNIT_TEST( dc4 ); - CPPUNIT_TEST( dc5 ); - CPPUNIT_TEST( dc6 ); - CPPUNIT_TEST( dc7 ); - CPPUNIT_TEST( dc8 ); - CPPUNIT_TEST( dc_datetest ); - CPPUNIT_TEST( dc_datetostrtest ); - CPPUNIT_TEST( dc_datetostr1test ); - CPPUNIT_TEST( dc_datetomysqlinttest ); - CPPUNIT_TEST( dc_datetest_perf ); - CPPUNIT_TEST( dc_datetimetest ); - CPPUNIT_TEST( dc_datetimetostrtest ); - CPPUNIT_TEST( dc_datetimetostr1test ); - CPPUNIT_TEST( dc_datetimetest_perf ); - CPPUNIT_TEST( dc_datetimetomysqlinttest ); - CPPUNIT_TEST( dc_datevalidtest ); - CPPUNIT_TEST( dc9 ); - CPPUNIT_TEST( dc10 ); - CPPUNIT_TEST_SUITE_END(); - -private: - DataConvert converter; - boost::any anyval; - CalpontSystemCatalog::ColType ct; - string data; - -public: - void setUp() - { - } - - void tearDown() - { - } - - /* from calpontsystemcatalog.h - struct ColType - { - int colWidth; - ConstraintType constraintType; - ColDataType colDataType; - DictOID ddn; - boost::any defaultValue; - int colPosition; // temporally put here. may need to have ColInfo struct later - int scale; //number after decimal points - int precision; - }; - colWidth - constraintType - colDataType - defaultValue - scale - precision - */ - -// mid-range positive integers - void dc1() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 2; - data = "12"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == 12); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 4; - data = "1234"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 1234); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 9; - data = "12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 12345678); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 9; - data = "12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 12345678); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = -1; - ct.precision = 18; - data = "1234567890123456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - - } - -// mid-range negative integers - void dc2() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 2; - data = "-12"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == -12); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 4; - data = "-1234"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -1234); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 9; - data = "-12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -12345678); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 9; - data = "-12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -12345678); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = -1; - ct.precision = 18; - data = "-1234567890123456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123456LL); - - } - -// limit positive integers - void dc3() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 3; - data = "127"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == 127); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 5; - data = "32767"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 32767); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 10; - data = "2147483647"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 2147483647); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 10; - data = "2147483647"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 2147483647); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 0; - ct.precision = 18; - data = "9223372036854775807"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 9223372036854775807LL); - - } - -// limit negative integers - void dc4() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 3; - data = "-126"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); + + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + int16_t smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -32766); - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == -126); + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = -1; + ct.precision = 9; + data = "-2147483646"; - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 4; - data = "-32766"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + int32_t intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -2147483646); - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -32766); + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::MEDINT; + ct.scale = -1; + ct.precision = 9; + data = "-2147483646"; - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 9; - data = "-2147483646"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -2147483646); - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -2147483646); + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = -1; + ct.precision = 18; + data = "-9223372036854775806"; - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 9; - data = "-2147483646"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -9223372036854775806LL); + } - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -2147483646); + // mid-range positive decimals (2 fixed points) + void dc5() + { + ct.colWidth = 2; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::SMALLINT; + ct.scale = 2; + ct.precision = 4; + bool pushWarning; - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = -1; - ct.precision = 18; - data = "-9223372036854775806"; + data = "12.34"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -9223372036854775806LL); + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + int16_t smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 1234); - } + data = "2.345"; -// mid-range positive decimals (2 fixed points) - void dc5() - { - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = 2; - ct.precision = 4; - bool pushWarning; + anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - data = "12.34"; + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 234); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "(2.34)"; - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 1234); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "2.345"; + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 234); - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); + data = "2.3"; - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 234); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "(2.34)"; + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 230); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "2."; - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 234); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "2.3"; + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 200); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "2"; - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 230); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "2."; + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == 200); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = 6; + ct.precision = 9; + data = "0.000481"; - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 200); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "2"; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + int32_t intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 481); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = 2; + ct.precision = 9; + data = "1234567.89"; - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 200); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = 6; - ct.precision = 9; - data = "0.000481"; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 123456789); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "34567.8999"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 481); + anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = 2; - ct.precision = 9; - data = "1234567.89"; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 3456789); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "(34567.89)"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 123456789); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "34567.8999"; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 3456789); - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); + data = "34567.8"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456789); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "(34567.89)"; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 3456780); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "34567."; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456789); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "34567.8"; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 3456700); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "34567"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456780); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "34567."; + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == 3456700); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = 2; + ct.precision = 18; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456700); + data = "12345678901234.56"; - data = "34567"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456700); + data = "45678901234.56093"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 4567890123456LL); - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 2; - ct.precision = 18; + data = "(45678901234.56)"; - data = "12345678901234.56"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 4567890123456LL); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); + data = "12345678901234.5"; - data = "45678901234.56093"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123450LL); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 4567890123456LL); + data = "12345678901234."; - data = "(45678901234.56)"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123400LL); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 4567890123456LL); + data = "12345678901234"; - data = "12345678901234.5"; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123400LL); + } - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123450LL); + // mid-range positive decimals (fixed-point) + void dc6() + { + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = 1; + ct.precision = 18; + bool pushWarning; - data = "12345678901234."; + data = "123456789012345.6"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123400LL); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - data = "12345678901234"; + ct.scale = 3; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "1234567890123.456"; - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123400LL); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - } + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123456LL); -// mid-range positive decimals (fixed-point) - void dc6() - { - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 1; - ct.precision = 18; - bool pushWarning; + ct.scale = 4; - data = "123456789012345.6"; + data = "123456789012.3456"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == 1234567890123456LL); + } - ct.scale = 3; + // mid-range negative decimals (2 fixed points) + void dc7() + { + ct.colWidth = 2; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::SMALLINT; + ct.scale = 2; + ct.precision = 4; + bool pushWarning; - data = "1234567890123.456"; + data = "-12.34"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + int16_t smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -1234); - ct.scale = 4; + data = "-2.345"; - data = "123456789012.3456"; + anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -234); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); + data = "(-2.34)"; - } + anyval = converter.convertColumnData(ct, data, pushWarning, false); -// mid-range negative decimals (2 fixed points) - void dc7() - { - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = 2; - ct.precision = 4; - bool pushWarning; + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -234); - data = "-12.34"; + data = "-2.3"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -1234); + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -230); - data = "-2.345"; + data = "-2."; - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -234); + CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); + smallintval = any_cast(anyval); + CPPUNIT_ASSERT(smallintval == -200); - data = "(-2.34)"; + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::INT; + ct.scale = 2; + ct.precision = 9; + data = "-1234567.89"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -234); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + int32_t intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -123456789); - data = "-2.3"; + data = "-34567.8999"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -230); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -3456789); - data = "-2."; + data = "(-34567.89)"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -200); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -3456789); - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = 2; - ct.precision = 9; - data = "-1234567.89"; + data = "-34567.8"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -123456789); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -3456780); - data = "-34567.8999"; + data = "-34567."; - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456789); + CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); + intval = any_cast(anyval); + CPPUNIT_ASSERT(intval == -3456700); - data = "(-34567.89)"; + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.scale = 2; + ct.precision = 18; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "-12345678901234.56"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456789); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "-34567.8"; + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + int64_t bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -1234567890123456LL); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "-45678901234.56093"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456780); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "-34567."; + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -4567890123456LL); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "(-45678901234.56)"; - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456700); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 2; - ct.precision = 18; + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -4567890123456LL); - data = "-12345678901234.56"; + data = "-12345678901234.5"; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123456LL); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -1234567890123450LL); - data = "-45678901234.56093"; + data = "-12345678901234."; - anyval = converter.convertColumnData(ct, data, pushWarning, false); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -4567890123456LL); + CPPUNIT_ASSERT(anyval.type() == typeid(long long)); + bigintval = static_cast(any_cast(anyval)); + CPPUNIT_ASSERT(bigintval == -1234567890123400LL); + } - data = "(-45678901234.56)"; + void dc8() + { + data = "2007-05-11"; + int status = 0; + uint32_t anyval; + anyval = converter.convertColumnDate(data.c_str(), CALPONTDATE_ENUM, status, data.length()); + CPPUNIT_ASSERT(anyval == 131551998); + std::string backToString = converter.dateToString(anyval); + CPPUNIT_ASSERT(backToString == data); + } - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -4567890123456LL); - - data = "-12345678901234.5"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123450LL); - - data = "-12345678901234."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123400LL); - - } - - void dc8() - { - data = "2007-05-11"; - int status = 0; - uint32_t anyval; - anyval = converter.convertColumnDate( data.c_str(), CALPONTDATE_ENUM, status, data.length()); - CPPUNIT_ASSERT(anyval == 131551998); - std::string backToString = converter.dateToString(anyval); - CPPUNIT_ASSERT( backToString == data ); - } - - void show_date_debugs( const Date& d1 ) - { - cout << "d1.spare = " << d1.spare << endl; - cout << "d1.day = " << d1.day << endl; - cout << "d1.month = " << d1.month << endl; - cout << "d1.year = " << d1.year << endl; - } + void show_date_debugs(const Date& d1) + { + cout << "d1.spare = " << d1.spare << endl; + cout << "d1.day = " << d1.day << endl; + cout << "d1.month = " << d1.month << endl; + cout << "d1.year = " << d1.year << endl; + } #include #include - /* Subtract the `struct timeval' values X and Y, - storing the result in RESULT. - Return 1 if the difference is negative, otherwise 0. */ + /* Subtract the `struct timeval' values X and Y, + storing the result in RESULT. + Return 1 if the difference is negative, otherwise 0. */ - int - timeval_subtract (struct timeval* result, struct timeval* x, struct timeval* y) + int timeval_subtract(struct timeval* result, struct timeval* x, struct timeval* y) + { + /* Perform the carry for the later subtraction by updating y. */ + if (x->tv_usec < y->tv_usec) { - /* Perform the carry for the later subtraction by updating y. */ - if (x->tv_usec < y->tv_usec) - { - int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; - y->tv_usec -= 1000000 * nsec; - y->tv_sec += nsec; - } - - if (x->tv_usec - y->tv_usec > 1000000) - { - int nsec = (x->tv_usec - y->tv_usec) / 1000000; - y->tv_usec += 1000000 * nsec; - y->tv_sec -= nsec; - } - - /* Compute the time remaining to wait. - tv_usec is certainly positive. */ - result->tv_sec = x->tv_sec - y->tv_sec; - result->tv_usec = x->tv_usec - y->tv_usec; - - /* Return 1 if result is negative. */ - return x->tv_sec < y->tv_sec; + int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; + y->tv_usec -= 1000000 * nsec; + y->tv_sec += nsec; } - void dc_datetest_perf() + if (x->tv_usec - y->tv_usec > 1000000) { - data = "2007-05-11"; - - struct timeval tv1, tv2, delta; - const int iters = 10000000; - - // test the performance of the default date conversion with full format support - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.stringToDate(data); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the default stringToDate method" << endl; - unsigned long total_usec = delta.tv_sec * 1000000 + delta.tv_usec; - CPPUNIT_ASSERT( total_usec < 3000000 ); - - // now run the performance of the "fast path" date conversions to check relative performance - int status = 0; - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.convertColumnDate( data.c_str(), CALPONTDATE_ENUM, status, data.length()); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the \"fastpath\" convertColumnDate method" << endl; + int nsec = (x->tv_usec - y->tv_usec) / 1000000; + y->tv_usec += 1000000 * nsec; + y->tv_sec -= nsec; } - void dc_datetimetest_perf() + /* Compute the time remaining to wait. + tv_usec is certainly positive. */ + result->tv_sec = x->tv_sec - y->tv_sec; + result->tv_usec = x->tv_usec - y->tv_usec; + + /* Return 1 if result is negative. */ + return x->tv_sec < y->tv_sec; + } + + void dc_datetest_perf() + { + data = "2007-05-11"; + + struct timeval tv1, tv2, delta; + const int iters = 10000000; + + // test the performance of the default date conversion with full format support + gettimeofday(&tv1, 0); + + for (int i = 0; i < iters; i++) { - data = "2007-05-11 10:30:45"; - - struct timeval tv1, tv2, delta; - const int iters = 10000000; - - // test the performance of the default datetime conversion with full format support - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.stringToDatetime(data); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the default stringToDatetime method" << endl; - unsigned long total_usec = delta.tv_sec * 1000000 + delta.tv_usec; - CPPUNIT_ASSERT( total_usec < 3000000 ); - - // now run the performance of the "fast path" datetime conversions to check relative performance - int status = 0; - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.convertColumnDatetime( data.c_str(), CALPONTDATETIME_ENUM, status, data.length()); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the \"fastpath\" convertColumnDatetime method" << endl; + converter.stringToDate(data); } - void dc_datetest() + gettimeofday(&tv2, 0); + + timeval_subtract(&delta, &tv2, &tv1); + cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" + << " using the default stringToDate method" << endl; + unsigned long total_usec = delta.tv_sec * 1000000 + delta.tv_usec; + CPPUNIT_ASSERT(total_usec < 3000000); + + // now run the performance of the "fast path" date conversions to check relative performance + int status = 0; + gettimeofday(&tv1, 0); + + for (int i = 0; i < iters; i++) { - struct DateCheck - { - const char* str; - Date date; - }; - - DateCheck date_tests[] = - { - { "0000-00-00", Date() }, // bogus date - { "2000-01-01", Date(2000, 1, 1) }, - { "2000-00-01", Date() }, // 0 not a valid monht - { "2000-Jan-01", Date() }, // no conversion from month names - { "2000-January-01", Date() }, // no conversion from month names - { "2000-Jac-01", Date() }, // no conversion from month names - { "2000-jan-01", Date() }, // no conversion from month names - { "20000101", Date(2000, 1, 1) }, - { "2000", Date() }, // no month or day - { "200001", Date() }, // no day - { "000101", Date(2000, 1, 1) }, - { "01-01-2000", Date() }, // can not swap field order - //{ "100-10-1", Date(100,10,1) }, // boost min date is 1400-1-1 - { "10-100-1", Date() }, // invalid month=100 - //{ "5-02-28", Date(5,2,28) }, // boost min date is 1400-1-1 - { "2000-02-28", Date(2000, 2, 28) }, - { "2000-02-29", Date(2000, 2, 29) }, - { "2004-02-29", Date(2004, 2, 29) }, - { "2003-12-31", Date(2003, 12, 31) }, - { "2003-12-32", Date() }, // no Dec. 32nd - { "03-11-30", Date(2003, 11, 30) }, - { "07-07-31", Date(2007, 7, 31) }, - { "2005-06-23 ", Date(2005, 6, 23) }, - { "2005-06-23 tomorrow", Date() }, // no trailing text - { "2005-06-23tomorrow ", Date() }, // no trailing text - { "2006-011-29 ", Date(2006, 11, 29) }, - { "2006-11-029 ", Date(2006, 11, 29) }, - { "10000-01-02 ", Date() }, - { "1999-0102 ", Date() }, - }; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - uint32_t val = converter.stringToDate(date_tests[i].str); - bool check = (((uint32_t)(val & 0xFFFFFFC0)) == ((*(reinterpret_cast (&date_tests[i].date))) & 0xFFFFFFC0)); - - if (!check) - { - printf("For input \"%s\", check 0x%08x vs 0x%08x\n", date_tests[i].str, (val & 0xFFFFFFC0), (*(reinterpret_cast (&date_tests[i].date))) & 0xFFFFFFC0); - } - - CPPUNIT_ASSERT( check ); - } + converter.convertColumnDate(data.c_str(), CALPONTDATE_ENUM, status, data.length()); } - void dc_datetostrtest() + gettimeofday(&tv2, 0); + + timeval_subtract(&delta, &tv2, &tv1); + cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" + << " using the \"fastpath\" convertColumnDate method" << endl; + } + + void dc_datetimetest_perf() + { + data = "2007-05-11 10:30:45"; + + struct timeval tv1, tv2, delta; + const int iters = 10000000; + + // test the performance of the default datetime conversion with full format support + gettimeofday(&tv1, 0); + + for (int i = 0; i < iters; i++) { - struct DateCheck - { - const char* str; - Date date; - }; - - DateCheck date_tests[] = - { - { "2000-01-01", Date(2000, 1, 1) }, - { "0100-10-01", Date(100, 10, 1) }, - { "0005-02-28", Date(5, 2, 28) }, - { "2000-02-28", Date(2000, 2, 28) }, - { "2000-02-29", Date(2000, 2, 29) }, - { "2004-02-29", Date(2004, 2, 29) }, - { "2003-12-31", Date(2003, 12, 31) }, - { "2003-11-30", Date(2003, 11, 30) }, - { "2007-07-31", Date(2007, 7, 31) }, - { "2005-06-23", Date(2005, 6, 23) }, - { "2006-11-29", Date(2006, 11, 29) } - }; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - string val = converter.dateToString(*(reinterpret_cast(&date_tests[i].date))); - - bool check = (val == date_tests[i].str); - - if (!check) - { - cout << "\"" << date_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT( check ); - } + converter.stringToDatetime(data); } - void dc_datetostr1test() + gettimeofday(&tv2, 0); + + timeval_subtract(&delta, &tv2, &tv1); + cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" + << " using the default stringToDatetime method" << endl; + unsigned long total_usec = delta.tv_sec * 1000000 + delta.tv_usec; + CPPUNIT_ASSERT(total_usec < 3000000); + + // now run the performance of the "fast path" datetime conversions to check relative performance + int status = 0; + gettimeofday(&tv1, 0); + + for (int i = 0; i < iters; i++) { - struct DateCheck - { - const char* str; - Date date; - }; - - DateCheck date_tests[] = - { - { "20000101", Date(2000, 1, 1) }, - { "01001001", Date(100, 10, 1) }, - { "00050228", Date(5, 2, 28) }, - { "20000228", Date(2000, 2, 28) }, - { "20000229", Date(2000, 2, 29) }, - { "20040229", Date(2004, 2, 29) }, - { "20031231", Date(2003, 12, 31) }, - { "20031130", Date(2003, 11, 30) }, - { "20070731", Date(2007, 7, 31) }, - { "20050623", Date(2005, 6, 23) }, - { "20061129", Date(2006, 11, 29) } - }; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - string val = converter.dateToString1(*(reinterpret_cast(&date_tests[i].date))); - - bool check = (val == date_tests[i].str); - - if (!check) - { - cout << "\"" << date_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT( check ); - } + converter.convertColumnDatetime(data.c_str(), CALPONTDATETIME_ENUM, status, data.length()); } - void dc_datetomysqlinttest() + gettimeofday(&tv2, 0); + + timeval_subtract(&delta, &tv2, &tv1); + cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" + << " using the \"fastpath\" convertColumnDatetime method" << endl; + } + + void dc_datetest() + { + struct DateCheck { - struct DateCheck - { - int32_t intval; - Date date; - }; + const char* str; + Date date; + }; - DateCheck date_tests[] = - { - { 20000101, Date(2000, 1, 1) }, - { 1001001, Date(100, 10, 1) }, - { 50228, Date(5, 2, 28) }, - { 20000228, Date(2000, 2, 28) }, - { 20000229, Date(2000, 2, 29) }, - { 20040229, Date(2004, 2, 29) }, - { 20031231, Date(2003, 12, 31) }, - { 20031130, Date(2003, 11, 30) }, - { 20070731, Date(2007, 7, 31) }, - { 20050623, Date(2005, 6, 23) }, - { 20061129, Date(2006, 11, 29) } - }; + DateCheck date_tests[] = { + {"0000-00-00", Date()}, // bogus date + {"2000-01-01", Date(2000, 1, 1)}, + {"2000-00-01", Date()}, // 0 not a valid monht + {"2000-Jan-01", Date()}, // no conversion from month names + {"2000-January-01", Date()}, // no conversion from month names + {"2000-Jac-01", Date()}, // no conversion from month names + {"2000-jan-01", Date()}, // no conversion from month names + {"20000101", Date(2000, 1, 1)}, + {"2000", Date()}, // no month or day + {"200001", Date()}, // no day + {"000101", Date(2000, 1, 1)}, + {"01-01-2000", Date()}, // can not swap field order + //{ "100-10-1", Date(100,10,1) }, // boost min date is 1400-1-1 + {"10-100-1", Date()}, // invalid month=100 + //{ "5-02-28", Date(5,2,28) }, // boost min date is 1400-1-1 + {"2000-02-28", Date(2000, 2, 28)}, + {"2000-02-29", Date(2000, 2, 29)}, + {"2004-02-29", Date(2004, 2, 29)}, + {"2003-12-31", Date(2003, 12, 31)}, + {"2003-12-32", Date()}, // no Dec. 32nd + {"03-11-30", Date(2003, 11, 30)}, + {"07-07-31", Date(2007, 7, 31)}, + {"2005-06-23 ", Date(2005, 6, 23)}, + {"2005-06-23 tomorrow", Date()}, // no trailing text + {"2005-06-23tomorrow ", Date()}, // no trailing text + {"2006-011-29 ", Date(2006, 11, 29)}, + {"2006-11-029 ", Date(2006, 11, 29)}, + {"10000-01-02 ", Date()}, + {"1999-0102 ", Date()}, + }; - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - int64_t val = date_tests[i].date.convertToMySQLint(); + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) + { + uint32_t val = converter.stringToDate(date_tests[i].str); + bool check = (((uint32_t)(val & 0xFFFFFFC0)) == + ((*(reinterpret_cast(&date_tests[i].date))) & 0xFFFFFFC0)); - bool check = (val == date_tests[i].intval); + if (!check) + { + printf("For input \"%s\", check 0x%08x vs 0x%08x\n", date_tests[i].str, (val & 0xFFFFFFC0), + (*(reinterpret_cast(&date_tests[i].date))) & 0xFFFFFFC0); + } - if (!check) - { - cout << val << " != " << date_tests[i].intval << endl; - } + CPPUNIT_ASSERT(check); + } + } - CPPUNIT_ASSERT( check ); - } + void dc_datetostrtest() + { + struct DateCheck + { + const char* str; + Date date; + }; + + DateCheck date_tests[] = {{"2000-01-01", Date(2000, 1, 1)}, {"0100-10-01", Date(100, 10, 1)}, + {"0005-02-28", Date(5, 2, 28)}, {"2000-02-28", Date(2000, 2, 28)}, + {"2000-02-29", Date(2000, 2, 29)}, {"2004-02-29", Date(2004, 2, 29)}, + {"2003-12-31", Date(2003, 12, 31)}, {"2003-11-30", Date(2003, 11, 30)}, + {"2007-07-31", Date(2007, 7, 31)}, {"2005-06-23", Date(2005, 6, 23)}, + {"2006-11-29", Date(2006, 11, 29)}}; + + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) + { + string val = converter.dateToString(*(reinterpret_cast(&date_tests[i].date))); + + bool check = (val == date_tests[i].str); + + if (!check) + { + cout << "\"" << date_tests[i].str << "\" != \"" << val << endl; + } + + CPPUNIT_ASSERT(check); + } + } + + void dc_datetostr1test() + { + struct DateCheck + { + const char* str; + Date date; + }; + + DateCheck date_tests[] = { + {"20000101", Date(2000, 1, 1)}, {"01001001", Date(100, 10, 1)}, {"00050228", Date(5, 2, 28)}, + {"20000228", Date(2000, 2, 28)}, {"20000229", Date(2000, 2, 29)}, {"20040229", Date(2004, 2, 29)}, + {"20031231", Date(2003, 12, 31)}, {"20031130", Date(2003, 11, 30)}, {"20070731", Date(2007, 7, 31)}, + {"20050623", Date(2005, 6, 23)}, {"20061129", Date(2006, 11, 29)}}; + + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) + { + string val = converter.dateToString1(*(reinterpret_cast(&date_tests[i].date))); + + bool check = (val == date_tests[i].str); + + if (!check) + { + cout << "\"" << date_tests[i].str << "\" != \"" << val << endl; + } + + CPPUNIT_ASSERT(check); + } + } + + void dc_datetomysqlinttest() + { + struct DateCheck + { + int32_t intval; + Date date; + }; + + DateCheck date_tests[] = { + {20000101, Date(2000, 1, 1)}, {1001001, Date(100, 10, 1)}, {50228, Date(5, 2, 28)}, + {20000228, Date(2000, 2, 28)}, {20000229, Date(2000, 2, 29)}, {20040229, Date(2004, 2, 29)}, + {20031231, Date(2003, 12, 31)}, {20031130, Date(2003, 11, 30)}, {20070731, Date(2007, 7, 31)}, + {20050623, Date(2005, 6, 23)}, {20061129, Date(2006, 11, 29)}}; + + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) + { + int64_t val = date_tests[i].date.convertToMySQLint(); + + bool check = (val == date_tests[i].intval); + + if (!check) + { + cout << val << " != " << date_tests[i].intval << endl; + } + + CPPUNIT_ASSERT(check); + } + } + + void show_datetime_debugs(const DateTime& d1) + { + cout << "d1.day = " << setw(10) << d1.day << endl; + cout << "d1.month = " << setw(10) << d1.month << endl; + cout << "d1.year = " << setw(10) << d1.year << endl; + cout << "d1.hour = " << setw(10) << d1.hour << endl; + cout << "d1.minute = " << setw(10) << d1.minute << endl; + cout << "d1.second = " << setw(10) << d1.second << endl; + cout << "d1.msecond = " << setw(10) << d1.msecond << endl; + } + + void dc_datetimetest() + { + struct DateTimeCheck + { + const char* str; + DateTime dtime; + }; + + DateTimeCheck dtime_tests[] = { + {"2010-03-31 10:30:45.000PM", DateTime()}, // AM/PM not supported + {"2010-03-31 10:30:45.000 PM", DateTime()}, // AM/PM not supported + {"2010-03-31 10:30:45PM", DateTime()}, // AM/PM not allowed without microsecs + {"2010-03-31 10:30:45AM", DateTime()}, // AM/PM not allowed without microsecs + {"2010-03-31 10:30:45 PM", DateTime()}, // AM/PM not allowed without microsecs + {"2008-01-01 10:00", DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {"2008-01-01 10:30:02", DateTime(2008, 1, 1, 10, 30, 2, 0)}, + {"2008-01-01 10:45:01.1111", DateTime(2008, 1, 1, 10, 45, 1, 111100)}, + {"2008-01-01 10:45:02.222222", DateTime(2008, 1, 1, 10, 45, 2, 222222)}, + {"2008-01-01 10:45:02.2222223", DateTime()}, // microsecs field too long + {"2008-01-01 10", DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {"200903311030", DateTime()}, // Interprets as YYMMDDHHMMSS so hour(31) is invalid + {"20090331103017", DateTime(2009, 03, 31, 10, 30, 17, 0)}, + // { "2009-04-28 11.34.55", DateTime(2009,4,28,11,34,55,0) }, // no more support for arbitrary + // separators + {"20100228T134501", DateTime(2010, 02, 28, 13, 45, 1, 0)}, + {"10:45:00 2010-01-31", DateTime()}, // can't switch time/date order + {"0000-00-00 00:00:00", DateTime()}, // invalid date/time + {"2009-12-31 23:59:56", DateTime(2009, 12, 31, 23, 59, 56, 0)} // invalid date/time + }; + + for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) + { + u_int64_t val = converter.stringToDatetime(dtime_tests[i].str); + + // this is a little screwy, but the way things are working is that stringToDateTime + // returns a -1 when the stringToDatetime operation fails. The default value for + // datetime, though is -2, so if we get back the error value then we need to mask + // off the LSB + if (val == 0xFFFFFFFFFFFFFFFF) + val = (val & 0xFFFFFFFFFFFFFFFE); + + bool check = (val == *(reinterpret_cast(&dtime_tests[i].dtime))); + + if (!check) + { + printf("For input \"%s\", check 0x%016lx vs 0x%016lx\n", dtime_tests[i].str, val, + *(reinterpret_cast(&dtime_tests[i].dtime))); + } + + if (!check) + show_datetime_debugs(DateTime(val)); + + CPPUNIT_ASSERT(check); + } + } + + void dc_datetimetostrtest() + { + struct DateTimeCheck + { + const char* str; + DateTime dtime; + }; + + DateTimeCheck dtime_tests[] = { + {"2008-01-01 10:00:00", DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {"2008-01-01 10:30:02", DateTime(2008, 1, 1, 10, 30, 2, 0)}, + {"2008-01-01 10:45:01", DateTime(2008, 1, 1, 10, 45, 1, 111100)}, + {"2008-01-01 10:45:02", DateTime(2008, 1, 1, 10, 45, 2, 222222)}, + {"2008-01-01 10:00:00", DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {"2009-03-31 10:30:17", DateTime(2009, 03, 31, 10, 30, 17, 0)}, + {"2009-04-28 11:34:55", DateTime(2009, 4, 28, 11, 34, 55, 0)}, + {"2010-02-28 13:45:01", DateTime(2010, 02, 28, 13, 45, 1, 0)}, + {"2009-12-31 23:59:56", DateTime(2009, 12, 31, 23, 59, 56, 0)} // invalid date/time + }; + + for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) + { + string val = converter.datetimeToString(*(reinterpret_cast(&dtime_tests[i].dtime))); + + bool check = (val == dtime_tests[i].str); + + if (!check) + { + cout << "\"" << dtime_tests[i].str << "\" != \"" << val << endl; + } + + CPPUNIT_ASSERT(check); + } + } + + void dc_datetimetostr1test() + { + struct DateTimeCheck + { + const char* str; + DateTime dtime; + }; + + DateTimeCheck dtime_tests[] = {{"20080101100000000000", DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {"20080101103002000000", DateTime(2008, 1, 1, 10, 30, 2, 0)}, + {"20080101104501111100", DateTime(2008, 1, 1, 10, 45, 1, 111100)}, + {"20080101104502222222", DateTime(2008, 1, 1, 10, 45, 2, 222222)}, + {"20080101100000000000", DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {"20090331103017000000", DateTime(2009, 03, 31, 10, 30, 17, 0)}, + {"20090428113455000000", DateTime(2009, 4, 28, 11, 34, 55, 0)}, + {"20100228134501000000", DateTime(2010, 02, 28, 13, 45, 1, 0)}, + {"20091231235956000000", DateTime(2009, 12, 31, 23, 59, 56, 0)}}; + + for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) + { + string val = converter.datetimeToString1(*(reinterpret_cast(&dtime_tests[i].dtime))); + + bool check = (val == dtime_tests[i].str); + + if (!check) + { + cout << "\"" << dtime_tests[i].str << "\" != \"" << val << endl; + } + + CPPUNIT_ASSERT(check); + } + } + + void dc_datetimetomysqlinttest() + { + struct DateTimeCheck + { + int64_t intval; + DateTime dtime; + }; + + DateTimeCheck dtime_tests[] = {{20080101100000, DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {20080101103002, DateTime(2008, 1, 1, 10, 30, 2, 0)}, + {20080101104501, DateTime(2008, 1, 1, 10, 45, 1, 111100)}, + {20080101104502, DateTime(2008, 1, 1, 10, 45, 2, 222222)}, + {20080101100000, DateTime(2008, 1, 1, 10, 0, 0, 0)}, + {20090331103017, DateTime(2009, 03, 31, 10, 30, 17, 0)}, + {20090428113455, DateTime(2009, 4, 28, 11, 34, 55, 0)}, + {20100228134501, DateTime(2010, 02, 28, 13, 45, 1, 0)}, + {20091231235956, DateTime(2009, 12, 31, 23, 59, 56, 0)}}; + + for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) + { + int64_t val = dtime_tests[i].dtime.convertToMySQLint(); + + bool check = (val == dtime_tests[i].intval); + + if (!check) + { + cout << val << " != " << dtime_tests[i].intval << endl; + } + + CPPUNIT_ASSERT(check); + } + } + + void check_date(int year, int month, int day, bool valid, bool boost_only = false) + { + bool idbval = isDateValid(day, month, year); + + bool boostval; + + try + { + boost::gregorian::date d(year, month, day); + boostval = true; + } + catch (...) + { + boostval = false; } - void show_datetime_debugs( const DateTime& d1 ) + bool check = boost_only ? (idbval == boostval) : (idbval == valid && ((boostval == idbval) || boostval)); + + if (!check) { - cout << "d1.day = " << setw(10) << d1.day << endl; - cout << "d1.month = " << setw(10) << d1.month << endl; - cout << "d1.year = " << setw(10) << d1.year << endl; - cout << "d1.hour = " << setw(10) << d1.hour << endl; - cout << "d1.minute = " << setw(10) << d1.minute << endl; - cout << "d1.second = " << setw(10) << d1.second << endl; - cout << "d1.msecond = " << setw(10) << d1.msecond << endl; + cout << "(" << year << "," << month << "," << day << "), expected:" << valid << ", idb=" << idbval + << ", boost=" << boostval << endl; } - void dc_datetimetest() + CPPUNIT_ASSERT(check); + } + + void dc_datevalidtest() + { + struct Check { - struct DateTimeCheck - { - const char* str; - DateTime dtime; - }; + int year; + int month; + int day; + bool valid; + }; - DateTimeCheck dtime_tests[] = - { - { "2010-03-31 10:30:45.000PM", DateTime() }, // AM/PM not supported - { "2010-03-31 10:30:45.000 PM", DateTime() }, // AM/PM not supported - { "2010-03-31 10:30:45PM", DateTime() }, // AM/PM not allowed without microsecs - { "2010-03-31 10:30:45AM", DateTime() }, // AM/PM not allowed without microsecs - { "2010-03-31 10:30:45 PM", DateTime() }, // AM/PM not allowed without microsecs - { "2008-01-01 10:00", DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { "2008-01-01 10:30:02", DateTime(2008, 1, 1, 10, 30, 2, 0) }, - { "2008-01-01 10:45:01.1111", DateTime(2008, 1, 1, 10, 45, 1, 111100) }, - { "2008-01-01 10:45:02.222222", DateTime(2008, 1, 1, 10, 45, 2, 222222) }, - { "2008-01-01 10:45:02.2222223", DateTime() }, // microsecs field too long - { "2008-01-01 10", DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { "200903311030", DateTime() }, // Interprets as YYMMDDHHMMSS so hour(31) is invalid - { "20090331103017", DateTime(2009, 03, 31, 10, 30, 17, 0) }, - // { "2009-04-28 11.34.55", DateTime(2009,4,28,11,34,55,0) }, // no more support for arbitrary separators - { "20100228T134501", DateTime(2010, 02, 28, 13, 45, 1, 0) }, - { "10:45:00 2010-01-31", DateTime() }, // can't switch time/date order - { "0000-00-00 00:00:00", DateTime() }, // invalid date/time - { "2009-12-31 23:59:56", DateTime(2009, 12, 31, 23, 59, 56, 0) } // invalid date/time - }; + Check tests[] = { + {2000, 2, 29, true}, {2001, 2, 29, false}, {1900, 2, 29, false}, {1900, 2, 28, true}, + {1399, 12, 31, false}, {10000, 1, 1, false}, {2000, 0, 29, false}, {2000, 13, 29, false}, + {2000, 1, 0, false}, {2000, 1, 32, false}, + }; - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - u_int64_t val = converter.stringToDatetime(dtime_tests[i].str); - - // this is a little screwy, but the way things are working is that stringToDateTime - // returns a -1 when the stringToDatetime operation fails. The default value for - // datetime, though is -2, so if we get back the error value then we need to mask - // off the LSB - if ( val == 0xFFFFFFFFFFFFFFFF ) - val = (val & 0xFFFFFFFFFFFFFFFE); - - bool check = ( val == *(reinterpret_cast(&dtime_tests[i].dtime)) ); - - if (!check) - { - printf("For input \"%s\", check 0x%016lx vs 0x%016lx\n", dtime_tests[i].str, val, *(reinterpret_cast(&dtime_tests[i].dtime))); - } - - if (!check) - show_datetime_debugs( DateTime(val) ); - - CPPUNIT_ASSERT( check ); - } + // predefined tests shown above + for (unsigned i = 0; i < sizeof(tests) / sizeof(Check); i++) + { + check_date(tests[i].year, tests[i].month, tests[i].day, tests[i].valid); } - void dc_datetimetostrtest() + // randomly generated date tests + for (unsigned i = 0; i < 1000000; i++) { - struct DateTimeCheck - { - const char* str; - DateTime dtime; - }; + int year = random() % 9000 + 1200; // some spillage on either side - DateTimeCheck dtime_tests[] = - { - { "2008-01-01 10:00:00", DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { "2008-01-01 10:30:02", DateTime(2008, 1, 1, 10, 30, 2, 0) }, - { "2008-01-01 10:45:01", DateTime(2008, 1, 1, 10, 45, 1, 111100) }, - { "2008-01-01 10:45:02", DateTime(2008, 1, 1, 10, 45, 2, 222222) }, - { "2008-01-01 10:00:00", DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { "2009-03-31 10:30:17", DateTime(2009, 03, 31, 10, 30, 17, 0) }, - { "2009-04-28 11:34:55", DateTime(2009, 4, 28, 11, 34, 55, 0) }, - { "2010-02-28 13:45:01", DateTime(2010, 02, 28, 13, 45, 1, 0) }, - { "2009-12-31 23:59:56", DateTime(2009, 12, 31, 23, 59, 56, 0) } // invalid date/time - }; + if (year == 10000) + --year; // year 10000 is not good because boost calls it good but we don't want to - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - string val = converter.datetimeToString(*(reinterpret_cast(&dtime_tests[i].dtime))); - - bool check = (val == dtime_tests[i].str); - - if (!check) - { - cout << "\"" << dtime_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT( check ); - } + int month = random() % 14; // some spillage on either side + int day = random() % 33; // some spillage on either side + check_date(year, month, day, true, true); } + } - void dc_datetimetostr1test() - { - struct DateTimeCheck - { - const char* str; - DateTime dtime; - }; + // Testing equality of floating point numbers is not good practice, rather make sure + // the conversion is within a tolerance (usually +/- 1 LSD). + inline bool inTolerance(double d, double a, double t) + { + return ((d > (a - t)) && (d < (a + t))); + } - DateTimeCheck dtime_tests[] = - { - { "20080101100000000000", DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { "20080101103002000000", DateTime(2008, 1, 1, 10, 30, 2, 0) }, - { "20080101104501111100", DateTime(2008, 1, 1, 10, 45, 1, 111100) }, - { "20080101104502222222", DateTime(2008, 1, 1, 10, 45, 2, 222222) }, - { "20080101100000000000", DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { "20090331103017000000", DateTime(2009, 03, 31, 10, 30, 17, 0) }, - { "20090428113455000000", DateTime(2009, 4, 28, 11, 34, 55, 0) }, - { "20100228134501000000", DateTime(2010, 02, 28, 13, 45, 1, 0) }, - { "20091231235956000000", DateTime(2009, 12, 31, 23, 59, 56, 0) } - }; + // Float tests 6 digits of accuracy + void dc9() + { + ct.colWidth = 4; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::FLOAT; + ct.scale = 0; + ct.precision = 4; + bool pushWarning; - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - string val = converter.datetimeToString1(*(reinterpret_cast(&dtime_tests[i].dtime))); + data = "0.123456"; - bool check = (val == dtime_tests[i].str); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - if (!check) - { - cout << "\"" << dtime_tests[i].str << "\" != \"" << val << endl; - } + CPPUNIT_ASSERT(anyval.type() == typeid(float)); + float floatval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(floatval, 0.123456, 0.000001)); - CPPUNIT_ASSERT( check ); - } - } + data = "3456.01"; - void dc_datetimetomysqlinttest() - { - struct DateTimeCheck - { - int64_t intval; - DateTime dtime; - }; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - DateTimeCheck dtime_tests[] = - { - { 20080101100000, DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { 20080101103002, DateTime(2008, 1, 1, 10, 30, 2, 0) }, - { 20080101104501, DateTime(2008, 1, 1, 10, 45, 1, 111100) }, - { 20080101104502, DateTime(2008, 1, 1, 10, 45, 2, 222222) }, - { 20080101100000, DateTime(2008, 1, 1, 10, 0, 0, 0) }, - { 20090331103017, DateTime(2009, 03, 31, 10, 30, 17, 0) }, - { 20090428113455, DateTime(2009, 4, 28, 11, 34, 55, 0) }, - { 20100228134501, DateTime(2010, 02, 28, 13, 45, 1, 0) }, - { 20091231235956, DateTime(2009, 12, 31, 23, 59, 56, 0) } - }; + CPPUNIT_ASSERT(anyval.type() == typeid(float)); + floatval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(floatval, 3456.01, 0.01)); - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - int64_t val = dtime_tests[i].dtime.convertToMySQLint(); + data = "(3456.01)"; - bool check = (val == dtime_tests[i].intval); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - if (!check) - { - cout << val << " != " << dtime_tests[i].intval << endl; - } + CPPUNIT_ASSERT(anyval.type() == typeid(float)); + floatval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(floatval, 3456.01, 0.01)); - CPPUNIT_ASSERT( check ); - } - } + data = "6.02214E+23"; - void check_date( int year, int month, int day, bool valid, bool boost_only = false ) - { - bool idbval = isDateValid( day, month, year ); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - bool boostval; + CPPUNIT_ASSERT(anyval.type() == typeid(float)); + floatval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(floatval, 6.02214E+23, 0.00001E+23)); - try - { - boost::gregorian::date d( year, month, day ); - boostval = true; - } - catch (...) - { - boostval = false; - } + data = "1.60217E-19"; - bool check = boost_only ? (idbval == boostval) : - (idbval == valid && ((boostval == idbval) || boostval)); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - if (!check) - { - cout << "(" << year << "," << month << "," << day << "), expected:" - << valid << ", idb=" << idbval << ", boost=" << boostval << endl; - } + CPPUNIT_ASSERT(anyval.type() == typeid(float)); + floatval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(floatval, 1.60217E-19, 0.00001E-19)); + } - CPPUNIT_ASSERT( check ); - } + // Double tests 15 digits of accuracy + void dc10() + { + ct.colWidth = 8; + ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.scale = 0; + ct.precision = 4; + bool pushWarning; - void dc_datevalidtest() - { - struct Check - { - int year; - int month; - int day; - bool valid; - }; + data = "0.123456789012345"; - Check tests[] = - { - { 2000, 2, 29, true }, - { 2001, 2, 29, false }, - { 1900, 2, 29, false }, - { 1900, 2, 28, true }, - { 1399, 12, 31, false }, - { 10000, 1, 1, false }, - { 2000, 0, 29, false }, - { 2000, 13, 29, false }, - { 2000, 1, 0, false }, - { 2000, 1, 32, false }, - }; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - // predefined tests shown above - for (unsigned i = 0; i < sizeof(tests) / sizeof(Check); i++) - { - check_date( tests[i].year, tests[i].month, tests[i].day, tests[i].valid ); - } + CPPUNIT_ASSERT(anyval.type() == typeid(double)); + double doubleval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(doubleval, 0.123456789012345, 0.000000000000001)); - // randomly generated date tests - for (unsigned i = 0; i < 1000000; i++) - { - int year = random() % 9000 + 1200; // some spillage on either side + data = "123456.000000001"; - if ( year == 10000 ) - --year; // year 10000 is not good because boost calls it good but we don't want to + anyval = converter.convertColumnData(ct, data, pushWarning, false); - int month = random() % 14; // some spillage on either side - int day = random() % 33; // some spillage on either side - check_date( year, month, day, true, true ); - } - } + CPPUNIT_ASSERT(anyval.type() == typeid(double)); + doubleval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(doubleval, 123456.000000001, 0.000000001)); -// Testing equality of floating point numbers is not good practice, rather make sure -// the conversion is within a tolerance (usually +/- 1 LSD). - inline bool inTolerance(double d, double a, double t) - { - return ((d > (a - t)) && (d < (a + t))); - } + data = "(123456.000000001)"; -// Float tests 6 digits of accuracy - void dc9() - { - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::FLOAT; - ct.scale = 0; - ct.precision = 4; - bool pushWarning; + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "0.123456"; + CPPUNIT_ASSERT(anyval.type() == typeid(double)); + doubleval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(doubleval, 123456.000000001, 0.000000001)); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "6.02214179000000E+23"; - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - float floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 0.123456, 0.000001)); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "3456.01"; + CPPUNIT_ASSERT(anyval.type() == typeid(double)); + doubleval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(doubleval, 6.02214179000000E+23, 0.00000000000001E+23)); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "1.60217653140000E-19"; - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 3456.01, 0.01)); + anyval = converter.convertColumnData(ct, data, pushWarning, false); - data = "(3456.01)"; + CPPUNIT_ASSERT(anyval.type() == typeid(double)); + doubleval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(doubleval, 1.60217653140000E-19, 0.00000000000001E-19)); - anyval = converter.convertColumnData(ct, data, pushWarning, false); + data = "3.14159265358979"; - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 3456.01, 0.01)); - - data = "6.02214E+23"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 6.02214E+23, 0.00001E+23)); - - data = "1.60217E-19"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 1.60217E-19, 0.00001E-19)); - - } - -// Double tests 15 digits of accuracy - void dc10() - { - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.scale = 0; - ct.precision = 4; - bool pushWarning; - - data = "0.123456789012345"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - double doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 0.123456789012345, 0.000000000000001)); - - data = "123456.000000001"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 123456.000000001, 0.000000001)); - - data = "(123456.000000001)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 123456.000000001, 0.000000001)); - - data = "6.02214179000000E+23"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 6.02214179000000E+23, 0.00000000000001E+23)); - - data = "1.60217653140000E-19"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 1.60217653140000E-19, 0.00000000000001E-19)); - - data = "3.14159265358979"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 3.14159265358979, 0.00000000000001)); - - } + anyval = converter.convertColumnData(ct, data, pushWarning, false); + CPPUNIT_ASSERT(anyval.type() == typeid(double)); + doubleval = any_cast(anyval); + CPPUNIT_ASSERT(inTolerance(doubleval, 3.14159265358979, 0.00000000000001)); + } }; - -CPPUNIT_TEST_SUITE_REGISTRATION( DataConvertTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DataConvertTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - diff --git a/utils/ddlcleanup/ddlcleanuputil.cpp b/utils/ddlcleanup/ddlcleanuputil.cpp index edab99340..a9defdf45 100644 --- a/utils/ddlcleanup/ddlcleanuputil.cpp +++ b/utils/ddlcleanup/ddlcleanuputil.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: ddlcleanuputil.cpp 3904 2013-06-18 12:22:51Z rdempsey $ -*/ + * $Id: ddlcleanuputil.cpp 3904 2013-06-18 12:22:51Z rdempsey $ + */ #include #include @@ -65,206 +65,209 @@ using namespace logging; namespace ddlcleanuputil { - int ddl_cleanup() { - scoped_ptr dbrmp(new DBRM()); - DDLPackageProcessor ddlpackageprocessor(dbrmp.get()); - DDLPackageProcessor::TableLogInfo tableLogs; - int rc = 0; - uint64_t uniqueId = dbrmp->getUnique64(); - logging::LoggingID lid(20); // This is running in the DMLProc space, so we use DML's lid - logging::MessageLog ml(lid); + scoped_ptr dbrmp(new DBRM()); + DDLPackageProcessor ddlpackageprocessor(dbrmp.get()); + DDLPackageProcessor::TableLogInfo tableLogs; + int rc = 0; + uint64_t uniqueId = dbrmp->getUnique64(); + logging::LoggingID lid(20); // This is running in the DMLProc space, so we use DML's lid + logging::MessageLog ml(lid); - //Get the logs information back first. - try + // Get the logs information back first. + try + { + ddlpackageprocessor.fetchLogFile(tableLogs, uniqueId); + } + catch (runtime_error& ex) + { + // Log to err.log + ostringstream oss; + oss << "DDLProc cannot get clean up information from DDL log files due to " << ex.what(); + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logErrorMessage(message1); + } + + DDLPackageProcessor::TableLogInfo::const_iterator it; + + for (it = tableLogs.begin(); it != tableLogs.end(); it++) + { + ostringstream oss; + oss << "DDLCleanup is cleaning table with oid " << it->first; + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logInfoMessage(message1); + DDLPackageProcessor::LogInfo aLogInfo = it->second; + + switch (aLogInfo.fileType) { - ddlpackageprocessor.fetchLogFile(tableLogs, uniqueId); - } - catch (runtime_error& ex) - { - //Log to err.log - ostringstream oss; - oss << "DDLProc cannot get clean up information from DDL log files due to " << ex.what(); - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logErrorMessage( message1 ); - } - - DDLPackageProcessor::TableLogInfo::const_iterator it ; - - for (it = tableLogs.begin(); it != tableLogs.end(); it++) - { - ostringstream oss; - oss << "DDLCleanup is cleaning table with oid " << it->first; - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logInfoMessage( message1 ); - DDLPackageProcessor::LogInfo aLogInfo = it->second; - - switch (aLogInfo.fileType) + case DDLPackageProcessor::DROPTABLE_LOG: + { + try { - case DDLPackageProcessor::DROPTABLE_LOG: - { - try - { - ddlpackageprocessor.flushPrimprocCache ( aLogInfo.oids ); - ddlpackageprocessor.removeExtents( aLogInfo.oids ); - ddlpackageprocessor.removeFiles( uniqueId, aLogInfo.oids ); - //delete log file - ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPTABLE_LOG, it->first, uniqueId); - ostringstream oss; - oss << "DDLProc has cleaned up drop table left over for table " << it->first; - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logInfoMessage( message1 ); - } - catch (runtime_error& ex) - { - ostringstream oss; - oss << "DDLProc cannot clean up drop table for table " << it->first << " due to " << ex.what(); - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logErrorMessage( message1 ); - } - - break; - } - - case DDLPackageProcessor::DROPPART_LOG: - { - string emsg; - rc = dbrmp->markPartitionForDeletion( aLogInfo.oids, aLogInfo.partitionNums, emsg); - - if (( rc != 0 ) && ( rc != BRM::ERR_PARTITION_DISABLED )) - { - continue; - } - - try - { - ddlpackageprocessor.removePartitionFiles( aLogInfo.oids, aLogInfo.partitionNums, uniqueId ); - cacheutils::flushPartition( aLogInfo.oids, aLogInfo.partitionNums ); - emsg.clear(); - rc = dbrmp->deletePartition( aLogInfo.oids, aLogInfo.partitionNums, emsg); - ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPPART_LOG, it->first, uniqueId); - ostringstream oss; - oss << "DDLProc has cleaned up drop partitions left over for table " << it->first; - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logInfoMessage( message1 ); - } - catch (runtime_error& ex) - { - ostringstream oss; - oss << "DDLProc cannot clean up drop partitions for table " << it->first << " due to " << ex.what(); - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logErrorMessage( message1 ); - } - - break; - } - - case DDLPackageProcessor::TRUNCATE_LOG: - { - rc = dbrmp->markAllPartitionForDeletion( aLogInfo.oids); - - if (rc != 0) //Log a message to err.log - { - string errMsg; - BRM::errString(rc, errMsg); - ostringstream oss; - oss << "DDLProc didn't clean up files for truncate table with oid " << it->first << " due to " << errMsg; - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logErrorMessage( message1 ); - continue; - } - - try - { - ddlpackageprocessor.removeFiles(uniqueId, aLogInfo.oids); - rc = cacheutils::flushOIDsFromCache(aLogInfo.oids); - ddlpackageprocessor.removeExtents( aLogInfo.oids ); - //create a new sets of files. Just find a dbroot according th the number of tables in syscat-1. - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(1); - int tableCount = systemCatalogPtr->getTableCount(); - int dbRootCnt = 1; - int useDBRoot = 1; - string DBRootCount = config::Config::makeConfig()->getConfig("SystemConfig", "DBRootCount"); - - if (DBRootCount.length() != 0) - dbRootCnt = static_cast(config::Config::fromText(DBRootCount)); - - useDBRoot = ((tableCount - 1) % dbRootCnt) + 1; - - //Create all column and dictionary files - CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(it->first); - ddlpackageprocessor.createFiles(aTableName, useDBRoot, uniqueId, static_cast(aLogInfo.oids.size())); - ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::TRUNCATE_LOG, it->first, uniqueId); - ostringstream oss; - oss << "DDLProc has cleaned up truncate table left over for table " << it->first << " and the table lock is released."; - logging::Message::Args args1; - logging::Message message1(2); - args1.add(oss.str()); - message1.format( args1 ); - ml.logInfoMessage( message1 ); - } - catch (std::exception& ex) - { - logging::Message::Args args1; - logging::Message message1(2); - ostringstream oss; - oss << "DDLProc didn't clean up truncate table left over for table with oid " << it->first << " due to " << ex.what(); - args1.add(oss.str()); - message1.format( args1 ); - ml.logErrorMessage( message1 ); - continue; - } - - break; - } - - default: - break; + ddlpackageprocessor.flushPrimprocCache(aLogInfo.oids); + ddlpackageprocessor.removeExtents(aLogInfo.oids); + ddlpackageprocessor.removeFiles(uniqueId, aLogInfo.oids); + // delete log file + ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPTABLE_LOG, it->first, uniqueId); + ostringstream oss; + oss << "DDLProc has cleaned up drop table left over for table " << it->first; + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logInfoMessage(message1); } - } - - //unlock the table - std::vector tableLocks; - tableLocks = dbrmp->getAllTableLocks(); - - for (unsigned idx = 0; idx < tableLocks.size(); idx++) - { - if (tableLocks[idx].ownerName == "DDLProc") + catch (runtime_error& ex) { - try - { - (void)dbrmp->releaseTableLock(tableLocks[idx].id); - } - catch ( ... ) {} + ostringstream oss; + oss << "DDLProc cannot clean up drop table for table " << it->first << " due to " << ex.what(); + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logErrorMessage(message1); } - } - return 0; + break; + } + + case DDLPackageProcessor::DROPPART_LOG: + { + string emsg; + rc = dbrmp->markPartitionForDeletion(aLogInfo.oids, aLogInfo.partitionNums, emsg); + + if ((rc != 0) && (rc != BRM::ERR_PARTITION_DISABLED)) + { + continue; + } + + try + { + ddlpackageprocessor.removePartitionFiles(aLogInfo.oids, aLogInfo.partitionNums, uniqueId); + cacheutils::flushPartition(aLogInfo.oids, aLogInfo.partitionNums); + emsg.clear(); + rc = dbrmp->deletePartition(aLogInfo.oids, aLogInfo.partitionNums, emsg); + ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPPART_LOG, it->first, uniqueId); + ostringstream oss; + oss << "DDLProc has cleaned up drop partitions left over for table " << it->first; + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logInfoMessage(message1); + } + catch (runtime_error& ex) + { + ostringstream oss; + oss << "DDLProc cannot clean up drop partitions for table " << it->first << " due to " << ex.what(); + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logErrorMessage(message1); + } + + break; + } + + case DDLPackageProcessor::TRUNCATE_LOG: + { + rc = dbrmp->markAllPartitionForDeletion(aLogInfo.oids); + + if (rc != 0) // Log a message to err.log + { + string errMsg; + BRM::errString(rc, errMsg); + ostringstream oss; + oss << "DDLProc didn't clean up files for truncate table with oid " << it->first << " due to " + << errMsg; + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logErrorMessage(message1); + continue; + } + + try + { + ddlpackageprocessor.removeFiles(uniqueId, aLogInfo.oids); + rc = cacheutils::flushOIDsFromCache(aLogInfo.oids); + ddlpackageprocessor.removeExtents(aLogInfo.oids); + // create a new sets of files. Just find a dbroot according th the number of tables in syscat-1. + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(1); + int tableCount = systemCatalogPtr->getTableCount(); + int dbRootCnt = 1; + int useDBRoot = 1; + string DBRootCount = config::Config::makeConfig()->getConfig("SystemConfig", "DBRootCount"); + + if (DBRootCount.length() != 0) + dbRootCnt = static_cast(config::Config::fromText(DBRootCount)); + + useDBRoot = ((tableCount - 1) % dbRootCnt) + 1; + + // Create all column and dictionary files + CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(it->first); + ddlpackageprocessor.createFiles(aTableName, useDBRoot, uniqueId, + static_cast(aLogInfo.oids.size())); + ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::TRUNCATE_LOG, it->first, uniqueId); + ostringstream oss; + oss << "DDLProc has cleaned up truncate table left over for table " << it->first + << " and the table lock is released."; + logging::Message::Args args1; + logging::Message message1(2); + args1.add(oss.str()); + message1.format(args1); + ml.logInfoMessage(message1); + } + catch (std::exception& ex) + { + logging::Message::Args args1; + logging::Message message1(2); + ostringstream oss; + oss << "DDLProc didn't clean up truncate table left over for table with oid " << it->first + << " due to " << ex.what(); + args1.add(oss.str()); + message1.format(args1); + ml.logErrorMessage(message1); + continue; + } + + break; + } + + default: break; + } + } + + // unlock the table + std::vector tableLocks; + tableLocks = dbrmp->getAllTableLocks(); + + for (unsigned idx = 0; idx < tableLocks.size(); idx++) + { + if (tableLocks[idx].ownerName == "DDLProc") + { + try + { + (void)dbrmp->releaseTableLock(tableLocks[idx].id); + } + catch (...) + { + } + } + } + + return 0; } -} //namespace ddlcleanuputil +} // namespace ddlcleanuputil // vim:ts=4 sw=4: - diff --git a/utils/ddlcleanup/ddlcleanuputil.h b/utils/ddlcleanup/ddlcleanuputil.h index 7642e7aa4..7eebdf156 100644 --- a/utils/ddlcleanup/ddlcleanuputil.h +++ b/utils/ddlcleanup/ddlcleanuputil.h @@ -22,7 +22,6 @@ #ifndef DDLCLEANUPUTIL_H__ #define DDLCLEANUPUTIL_H__ - /** * block cache utilities */ @@ -33,4 +32,3 @@ int ddl_cleanup(); // vim:ts=4 sw=4: #endif - diff --git a/utils/funcexp/func_abs.cpp b/utils/funcexp/func_abs.cpp index 10227b4e9..64ac5d014 100644 --- a/utils/funcexp/func_abs.cpp +++ b/utils/funcexp/func_abs.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_abs.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_abs.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,61 +36,47 @@ using namespace rowgroup; namespace funcexp { -CalpontSystemCatalog::ColType Func_abs::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_abs::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -int64_t Func_abs::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int64_t Func_abs::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - // null value is indicated by isNull - return llabs(parm[0]->data()->getIntVal(row, isNull)); + // null value is indicated by isNull + return llabs(parm[0]->data()->getIntVal(row, isNull)); } -uint64_t Func_abs::getUintVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +uint64_t Func_abs::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - // null value is indicated by isNull - return parm[0]->data()->getIntVal(row, isNull); + // null value is indicated by isNull + return parm[0]->data()->getIntVal(row, isNull); } -IDB_Decimal Func_abs::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +IDB_Decimal Func_abs::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - if (parm[0]->data()->resultType().isWideDecimalType()) - d.s128Value = (d.s128Value < 0) ? -d.s128Value : d.s128Value; - else - d.value = llabs(d.value); - return d; + if (parm[0]->data()->resultType().isWideDecimalType()) + d.s128Value = (d.s128Value < 0) ? -d.s128Value : d.s128Value; + else + d.value = llabs(d.value); + return d; } - -double Func_abs::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_abs::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - return fabs(parm[0]->data()->getDoubleVal(row, isNull)); + return fabs(parm[0]->data()->getDoubleVal(row, isNull)); } -long double Func_abs::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +long double Func_abs::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - return fabsl(parm[0]->data()->getLongDoubleVal(row, isNull)); + return fabsl(parm[0]->data()->getLongDoubleVal(row, isNull)); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index f5f501949..09b1d2d0c 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_add_time.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_add_time.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,283 +38,269 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_add_time::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_add_time::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_add_time::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_add_time::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIME) - { - return getTimeIntVal(row, parm, isNull, op_ct); - } - else - { - return getDatetimeIntVal(row, parm, isNull, op_ct); - } + if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIME) + { + return getTimeIntVal(row, parm, isNull, op_ct); + } + else + { + return getDatetimeIntVal(row, parm, isNull, op_ct); + } } -string Func_add_time::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_add_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return intToString(getIntVal(row, parm, isNull, ct)); + return intToString(getIntVal(row, parm, isNull, ct)); } -int32_t Func_add_time::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_add_time::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E); + return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E); } -int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - if (parm[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) - return getTimestampIntVal(row, parm, isNull, ct); + if (parm[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) + return getTimestampIntVal(row, parm, isNull, ct); - int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - if (isNull) - return -1; + if (isNull) + return -1; - // Adding a zero date to a time is always NULL - if (val1 == 0) - { - isNull = true; - return -1; - } + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } - const string& val2 = parm[1]->data()->getStrVal(row, isNull); - int sign = parm[2]->data()->getIntVal(row, isNull); - DateTime dt1; - dt1.year = (val1 >> 48) & 0xffff; - dt1.month = (val1 >> 44) & 0xf; - dt1.day = (val1 >> 38) & 0x3f; - dt1.hour = (val1 >> 32) & 0x3f; - dt1.minute = (val1 >> 26) & 0x3f; - dt1.second = (val1 >> 20) & 0x3f; - dt1.msecond = val1 & 0xfffff; + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + DateTime dt1; + dt1.year = (val1 >> 48) & 0xffff; + dt1.month = (val1 >> 44) & 0xf; + dt1.day = (val1 >> 38) & 0x3f; + dt1.hour = (val1 >> 32) & 0x3f; + dt1.minute = (val1 >> 26) & 0x3f; + dt1.second = (val1 >> 20) & 0x3f; + dt1.msecond = val1 & 0xfffff; - int64_t time = DataConvert::stringToTime(val2); + int64_t time = DataConvert::stringToTime(val2); - if (time == -1) - { - isNull = true; - return -1; - } + if (time == -1) + { + isNull = true; + return -1; + } - Time t2 = *(reinterpret_cast(&time)); + Time t2 = *(reinterpret_cast(&time)); - // MySQL TIME type range '-838:59:59' and '838:59:59' - if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) - { - isNull = true; - return -1; - } + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } - int val_sign = 1; + int val_sign = 1; - if (t2.hour < 0) - { - val_sign = -1; - } + if (t2.hour < 0) + { + val_sign = -1; + } - if (abs(t2.hour) > 838) - { - t2.hour = 838; - t2.minute = 59; - t2.second = 59; - t2.msecond = 999999; - } + if (abs(t2.hour) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } - if (val_sign * sign < 0) - { - t2.hour = -abs(t2.hour); - t2.minute = -abs(t2.minute); - t2.second = -abs(t2.second); - t2.msecond = -abs(t2.msecond); - } - else - { - t2.hour = abs(t2.hour); - t2.minute = abs(t2.minute); - t2.second = abs(t2.second); - t2.msecond = abs(t2.msecond); - } + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } - t2.day = 0; + t2.day = 0; - return addTime(dt1, t2); + return addTime(dt1, t2); } -int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); + int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - if (isNull) - return -1; + if (isNull) + return -1; - // Adding a zero date to a time is always NULL - if (val1 == 0) - { - isNull = true; - return -1; - } + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } - const string& val2 = parm[1]->data()->getStrVal(row, isNull); - int sign = parm[2]->data()->getIntVal(row, isNull); - DateTime dt1; - TimeStamp timestamp(val1); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - dt1.year = m_time.year; - dt1.month = m_time.month; - dt1.day = m_time.day; - dt1.hour = m_time.hour; - dt1.minute = m_time.minute; - dt1.second = m_time.second; - dt1.msecond = timestamp.msecond; + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + DateTime dt1; + TimeStamp timestamp(val1); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + dt1.year = m_time.year; + dt1.month = m_time.month; + dt1.day = m_time.day; + dt1.hour = m_time.hour; + dt1.minute = m_time.minute; + dt1.second = m_time.second; + dt1.msecond = timestamp.msecond; - int64_t time = DataConvert::stringToTime(val2); + int64_t time = DataConvert::stringToTime(val2); - if (time == -1) - { - isNull = true; - return -1; - } + if (time == -1) + { + isNull = true; + return -1; + } - Time t2 = *(reinterpret_cast(&time)); + Time t2 = *(reinterpret_cast(&time)); - // MySQL TIME type range '-838:59:59' and '838:59:59' - if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) - { - isNull = true; - return -1; - } + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } - int val_sign = 1; + int val_sign = 1; - if (t2.hour < 0) - { - val_sign = -1; - } + if (t2.hour < 0) + { + val_sign = -1; + } - if (abs(t2.hour) > 838) - { - t2.hour = 838; - t2.minute = 59; - t2.second = 59; - t2.msecond = 999999; - } + if (abs(t2.hour) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } - if (val_sign * sign < 0) - { - t2.hour = -abs(t2.hour); - t2.minute = -abs(t2.minute); - t2.second = -abs(t2.second); - t2.msecond = -abs(t2.msecond); - } - else - { - t2.hour = abs(t2.hour); - t2.minute = abs(t2.minute); - t2.second = abs(t2.second); - t2.msecond = abs(t2.msecond); - } + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } - t2.day = 0; + t2.day = 0; - return addTime(dt1, t2); + return addTime(dt1, t2); } -int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); - if (isNull) - return -1; + if (isNull) + return -1; - const string& val2 = parm[1]->data()->getStrVal(row, isNull); - int sign = parm[2]->data()->getIntVal(row, isNull); - Time dt1; - dt1.day = 0; - dt1.is_neg = val1 >> 63; - dt1.hour = (val1 >> 40) & 0xfff; - dt1.minute = (val1 >> 32) & 0xff; - dt1.second = (val1 >> 24) & 0xff; - dt1.msecond = val1 & 0xffffff; + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + Time dt1; + dt1.day = 0; + dt1.is_neg = val1 >> 63; + dt1.hour = (val1 >> 40) & 0xfff; + dt1.minute = (val1 >> 32) & 0xff; + dt1.second = (val1 >> 24) & 0xff; + dt1.msecond = val1 & 0xffffff; - int64_t time = DataConvert::stringToTime(val2); + int64_t time = DataConvert::stringToTime(val2); - if (time == -1) - { - isNull = true; - return -1; - } + if (time == -1) + { + isNull = true; + return -1; + } - Time t2 = *(reinterpret_cast(&time)); + Time t2 = *(reinterpret_cast(&time)); - // MySQL TIME type range '-838:59:59' and '838:59:59' - if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) - { - isNull = true; - return -1; - } + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } - int val_sign = 1; + int val_sign = 1; - if (t2.hour < 0) - { - val_sign = -1; - } + if (t2.hour < 0) + { + val_sign = -1; + } - if (abs(t2.hour) > 838) - { - t2.hour = 838; - t2.minute = 59; - t2.second = 59; - t2.msecond = 999999; - } + if (abs(t2.hour) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } - t2.day = 0; + t2.day = 0; - if (val_sign * sign < 0) - { - t2.hour = -abs(t2.hour); - t2.minute = -abs(t2.minute); - t2.second = -abs(t2.second); - t2.msecond = -abs(t2.msecond); - } - else - { - t2.hour = abs(t2.hour); - t2.minute = abs(t2.minute); - t2.second = abs(t2.second); - t2.msecond = abs(t2.msecond); - } + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } - return addTime(dt1, t2); + return addTime(dt1, t2); } - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ascii.cpp b/utils/funcexp/func_ascii.cpp index 1900c0633..eb93f6011 100644 --- a/utils/funcexp/func_ascii.cpp +++ b/utils/funcexp/func_ascii.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_ascii.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_ascii.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,24 +38,22 @@ using namespace dataconvert; namespace funcexp { -CalpontSystemCatalog::ColType Func_ascii::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_ascii::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_ascii::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_ascii::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - const string& str = parm[0]->data()->getStrVal(row, isNull); + const string& str = parm[0]->data()->getStrVal(row, isNull); - if (str.empty()) - return 0; + if (str.empty()) + return 0; - return (unsigned char)str[0]; + return (unsigned char)str[0]; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index b5632b94b..3bd2b907b 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_between.cpp 3954 2013-07-08 16:30:15Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_between.cpp 3954 2013-07-08 16:30:15Z bpaul $ + * + * + ****************************************************************************/ #include #include @@ -38,366 +38,342 @@ using namespace execplan; #include "idberrorinfo.h" #include "errorids.h" - using namespace logging; namespace { -template +template inline bool numericGE(result_t op1, result_t op2) { - return op1 >= op2; + return op1 >= op2; } -template +template inline bool numericLE(result_t op1, result_t op2) { - return op1 <= op2; + return op1 <= op2; } -inline bool strGE(CHARSET_INFO &cs, const string& op1, const string& op2) +inline bool strGE(CHARSET_INFO& cs, const string& op1, const string& op2) { - return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) >= 0; + return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) >= 0; } -inline bool strLE(CHARSET_INFO &cs, const string& op1, const string& op2) +inline bool strLE(CHARSET_INFO& cs, const string& op1, const string& op2) { - return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) <= 0; + return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) <= 0; } -inline bool getBool(rowgroup::Row& row, - funcexp::FunctionParm& pm, - bool& isNull, - CalpontSystemCatalog::ColType& ct, - bool notBetween) +inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, + CalpontSystemCatalog::ColType& ct, bool notBetween) { - switch (ct.colDataType) + switch (ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - int64_t val = pm[0]->data()->getIntVal(row, isNull); + int64_t val = pm[0]->data()->getIntVal(row, isNull); - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getIntVal(row, isNull)) && !isNull) - return true; + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getIntVal(row, isNull)) && !isNull) + return true; - isNull = false; - return (!numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull); - } + isNull = false; + return (!numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull); + } - return !isNull && - numericGE(val, pm[1]->data()->getIntVal(row, isNull)) && - numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t val = pm[0]->data()->getUintVal(row, isNull); - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getUintVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getUintVal(row, isNull)) && - numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t val = pm[0]->data()->getDateIntVal(row, isNull); - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getDateIntVal(row, isNull)) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) && - numericLE(val, pm[2]->data()->getDateIntVal(row, isNull)); - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t val = pm[0]->data()->getDatetimeIntVal(row, isNull); - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) && - numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull); - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && - numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)); - } - - case execplan::CalpontSystemCatalog::TIME: - { - // Shift out unused day for compare - int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12; - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && - numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12); - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - double val = pm[0]->data()->getDoubleVal(row, isNull); - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getDoubleVal(row, isNull)) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) && - numericLE(val, pm[2]->data()->getDoubleVal(row, isNull)); - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double val = pm[0]->data()->getLongDoubleVal(row, isNull); - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull)) && !isNull); - } - - return !isNull && - numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) && - numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull)); - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - return false; - - if (notBetween) - { - if (!numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)) && !isNull); - } - - return numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && - numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)); - } - - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& val = pm[0]->data()->getStrVal(row, isNull); - CHARSET_INFO &cs = datatypes::Charset(ct.charsetNumber).getCharset(); - - if (notBetween) - { - if (!strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) && !isNull) - return true; - - isNull = false; - return (!strLE(cs, val, pm[2]->data()->getStrVal(row, isNull)) && !isNull); - } - - return !isNull && - strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) && - strLE(cs, val, pm[2]->data()->getStrVal(row, isNull)); - } - - default: - { - std::ostringstream oss; - oss << "between: datatype of " << execplan::colDataTypeToString(ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return !isNull && numericGE(val, pm[1]->data()->getIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull; } + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t val = pm[0]->data()->getUintVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getUintVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getUintVal(row, isNull)) && + numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t val = pm[0]->data()->getDateIntVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getDateIntVal(row, isNull)) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getDateIntVal(row, isNull)); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t val = pm[0]->data()->getDatetimeIntVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)); + } + + case execplan::CalpontSystemCatalog::TIME: + { + // Shift out unused day for compare + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12; + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12); + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + double val = pm[0]->data()->getDoubleVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getDoubleVal(row, isNull)) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) && + numericLE(val, pm[2]->data()->getDoubleVal(row, isNull)); + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double val = pm[0]->data()->getLongDoubleVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull)) && !isNull); + } + + return !isNull && numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) && + numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull)); + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + return false; + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)) && !isNull); + } + + return numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && + numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)); + } + + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& val = pm[0]->data()->getStrVal(row, isNull); + CHARSET_INFO& cs = datatypes::Charset(ct.charsetNumber).getCharset(); + + if (notBetween) + { + if (!strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!strLE(cs, val, pm[2]->data()->getStrVal(row, isNull)) && !isNull); + } + + return !isNull && strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) && + strLE(cs, val, pm[2]->data()->getStrVal(row, isNull)); + } + + default: + { + std::ostringstream oss; + oss << "between: datatype of " << execplan::colDataTypeToString(ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } -} +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_between::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - PredicateOperator op; - CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType(); - //op.operationType(fp[0]->data()->resultType()); - bool allString = true; + PredicateOperator op; + CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType(); + // op.operationType(fp[0]->data()->resultType()); + bool allString = true; + + for (uint32_t i = 1; i < fp.size(); i++) + { + // op.setOpType(op.operationType(), fp[i]->data()->resultType()); + op.setOpType(ct, fp[i]->data()->resultType()); + ct = op.operationType(); + + if ((fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || + ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIMESTAMP || ct.colDataType == CalpontSystemCatalog::TIME) + { + allString = false; + } + } + + if (allString) + { + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + } + + else if (op.operationType().colDataType == CalpontSystemCatalog::DATETIME) + { + ConstantColumn* cc = NULL; for (uint32_t i = 1; i < fp.size(); i++) { - //op.setOpType(op.operationType(), fp[i]->data()->resultType()); - op.setOpType(ct, fp[i]->data()->resultType()); - ct = op.operationType(); + cc = dynamic_cast(fp[i]->data()); - if ((fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || - ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME || - ct.colDataType == CalpontSystemCatalog::TIMESTAMP || - ct.colDataType == CalpontSystemCatalog::TIME) - { - allString = false; - } + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); + cc->result(result); + } } + } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) + { + ConstantColumn* cc = NULL; - if (allString) + for (uint32_t i = 1; i < fp.size(); i++) { - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - } + cc = dynamic_cast(fp[i]->data()); - else if (op.operationType().colDataType == CalpontSystemCatalog::DATETIME) + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone()); + cc->result(result); + } + } + } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) { - ConstantColumn* cc = NULL; + cc = dynamic_cast(fp[i]->data()); - for (uint32_t i = 1; i < fp.size(); i++) - { - cc = dynamic_cast(fp[i]->data()); - - if (cc) - { - Result result = cc->result(); - result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); - cc->result(result); - } - } + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + cc->result(result); + } } - else if (op.operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) - { - ConstantColumn* cc = NULL; + } - for (uint32_t i = 1; i < fp.size(); i++) - { - cc = dynamic_cast(fp[i]->data()); - - if (cc) - { - Result result = cc->result(); - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone()); - cc->result(result); - } - } - } - else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) - { - ConstantColumn* cc = NULL; - - for (uint32_t i = 1; i < fp.size(); i++) - { - cc = dynamic_cast(fp[i]->data()); - - if (cc) - { - Result result = cc->result(); - result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); - cc->result(result); - } - } - } - - - return ct; + return ct; } -bool Func_between::getBoolVal(rowgroup::Row& row, - FunctionParm& pm, - bool& isNull, +bool Func_between::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, false) && !isNull; + return getBool(row, pm, isNull, ct, false) && !isNull; } - - - - -CalpontSystemCatalog::ColType Func_notbetween::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_notbetween::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - PredicateOperator* op = new PredicateOperator(); - CalpontSystemCatalog::ColType ct; - op->setOpType(fp[0]->data()->resultType(), fp[1]->data()->resultType()); - op->setOpType(op->resultType(), fp[2]->data()->resultType()); - return op->operationType(); + PredicateOperator* op = new PredicateOperator(); + CalpontSystemCatalog::ColType ct; + op->setOpType(fp[0]->data()->resultType(), fp[1]->data()->resultType()); + op->setOpType(op->resultType(), fp[2]->data()->resultType()); + return op->operationType(); } -bool Func_notbetween::getBoolVal(rowgroup::Row& row, - FunctionParm& pm, - bool& isNull, +bool Func_notbetween::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, true) && !isNull; + return getBool(row, pm, isNull, ct, true) && !isNull; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 69264d197..76d09b9bd 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_bitwise.cpp 3616 2013-03-04 14:56:29Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_bitwise.cpp 3616 2013-03-04 14:56:29Z rdempsey $ + * + * + ****************************************************************************/ #include using namespace std; @@ -49,52 +49,44 @@ namespace { using namespace funcexp; - -bool validateBitOperandTypeOrError(execplan::FunctionColumn &col, - const Func & func, - uint argno) +bool validateBitOperandTypeOrError(execplan::FunctionColumn& col, const Func& func, uint argno) { - auto & type = col.functionParms()[argno]->data()->resultType(); - if (type.canReturnXInt64()) - return false; - func.raiseIllegalParameterDataTypeError(type); - return true; + auto& type = col.functionParms()[argno]->data()->resultType(); + if (type.canReturnXInt64()) + return false; + func.raiseIllegalParameterDataTypeError(type); + return true; } - -template -datatypes::TUInt64Null ConvertToBitOperand(const T &val) +template +datatypes::TUInt64Null ConvertToBitOperand(const T& val) { - if (val > static_cast(UINT64_MAX)) - return datatypes::TUInt64Null(UINT64_MAX); - if (val >= 0) - return datatypes::TUInt64Null(static_cast(val)); - if (val < static_cast(INT64_MIN)) - return datatypes::TUInt64Null(static_cast(INT64_MAX)+1); - return datatypes::TUInt64Null((uint64_t) (int64_t) val); + if (val > static_cast(UINT64_MAX)) + return datatypes::TUInt64Null(UINT64_MAX); + if (val >= 0) + return datatypes::TUInt64Null(static_cast(val)); + if (val < static_cast(INT64_MIN)) + return datatypes::TUInt64Null(static_cast(INT64_MAX) + 1); + return datatypes::TUInt64Null((uint64_t)(int64_t)val); } - -static -datatypes::TUInt64Null DecimalToBitOperand(Row& row, - const execplan::SPTP& parm, - const funcexp::Func& thisFunc) +static datatypes::TUInt64Null DecimalToBitOperand(Row& row, const execplan::SPTP& parm, + const funcexp::Func& thisFunc) { - bool tmpIsNull = false; - datatypes::Decimal d = parm->data()->getDecimalVal(row, tmpIsNull); - if (tmpIsNull) - return datatypes::TUInt64Null(); + bool tmpIsNull = false; + datatypes::Decimal d = parm->data()->getDecimalVal(row, tmpIsNull); + if (tmpIsNull) + return datatypes::TUInt64Null(); - if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - int128_t val = d.getPosNegRoundedIntegralPart(0).getValue(); - return ConvertToBitOperand(val); - } + if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t val = d.getPosNegRoundedIntegralPart(0).getValue(); + return ConvertToBitOperand(val); + } - return datatypes::TUInt64Null((uint64_t) d.decimal64ToSInt64Round()); + return datatypes::TUInt64Null((uint64_t)d.decimal64ToSInt64Round()); } - // Functions TypeHolderStd::canReturnXInt64() and GenericToBitOperand() // should be splitted eventually to virtual methods in TypeHandler. // @@ -112,460 +104,392 @@ datatypes::TUInt64Null DecimalToBitOperand(Row& row, // here. This same method is potentially useful in other methods // and could be extracted into a utility class with its own header // if that is the case - this is left as future exercise -datatypes::TUInt64Null GenericToBitOperand( - Row& row, - const execplan::SPTP& parm, - const funcexp::Func& thisFunc, - bool temporalRounding) +datatypes::TUInt64Null GenericToBitOperand(Row& row, const execplan::SPTP& parm, + const funcexp::Func& thisFunc, bool temporalRounding) { - switch (parm->data()->resultType().colDataType) + switch (parm->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - datatypes::TSInt64Null tmp= parm->data()->toTSInt64Null(row); - return tmp.isNull() ? datatypes::TUInt64Null() : - datatypes::TUInt64Null((uint64_t) (int64_t) tmp); - } - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::UFLOAT: - { - bool tmpIsNull = false; - double val = parm->data()->getDoubleVal(row, tmpIsNull); - return tmpIsNull ? datatypes::TUInt64Null() : - ConvertToBitOperand(round(val)); - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - return parm->data()->toTUInt64Null(row); - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - bool tmpIsNull = false; - const string& str = parm->data()->getStrVal(row, tmpIsNull); - if (tmpIsNull) - return datatypes::TUInt64Null(); - - datatypes::DataCondition cnverr; - literal::Converter cnv(str, cnverr); - cnv.normalize(); - return cnv.negative() ? - datatypes::TUInt64Null((uint64_t)cnv.toPackedSDecimal(0, cnverr)) : - datatypes::TUInt64Null(cnv.toPackedUDecimal(0, cnverr)); - } - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - return DecimalToBitOperand(row, parm, thisFunc); - - case execplan::CalpontSystemCatalog::DATE: - { - bool tmpIsNull = false; - int32_t time = parm->data()->getDateIntVal(row, tmpIsNull); - if (tmpIsNull) - return datatypes::TUInt64Null(); - - int64_t value = Date(time).convertToMySQLint(); - return datatypes::TUInt64Null((uint64_t) value); - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - bool tmpIsNull = false; - int64_t time = parm->data()->getDatetimeIntVal(row, tmpIsNull); - if (tmpIsNull) - return datatypes::TUInt64Null(); - - // @bug 4703 - missing year when convering to int - DateTime dt(time); - int64_t value = dt.convertToMySQLint(); - if (temporalRounding && dt.msecond >= 500000) - value++; - return datatypes::TUInt64Null((uint64_t) value); - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - bool tmpIsNull = false; - int64_t time = parm->data()->getTimestampIntVal(row, tmpIsNull); - if (tmpIsNull) - return datatypes::TUInt64Null(); - - TimeStamp dt(time); - int64_t value = dt.convertToMySQLint(thisFunc.timeZone()); - if (temporalRounding && dt.msecond >= 500000) - value++; - return datatypes::TUInt64Null((uint64_t) value); - } - - case execplan::CalpontSystemCatalog::TIME: - { - bool tmpIsNull = false; - int64_t time = parm->data()->getTimeIntVal(row, tmpIsNull); - - Time dt(time); - int64_t value = dt.convertToMySQLint(); - if (temporalRounding && dt.msecond >= 500000) - value < 0 ? value-- : value++; - return datatypes::TUInt64Null((uint64_t) value); - } - - default: - idbassert(0); // Not possible: checked during the preparation stage. - break; + datatypes::TSInt64Null tmp = parm->data()->toTSInt64Null(row); + return tmp.isNull() ? datatypes::TUInt64Null() : datatypes::TUInt64Null((uint64_t)(int64_t)tmp); + } + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::UFLOAT: + { + bool tmpIsNull = false; + double val = parm->data()->getDoubleVal(row, tmpIsNull); + return tmpIsNull ? datatypes::TUInt64Null() : ConvertToBitOperand(round(val)); } - return datatypes::TUInt64Null(); + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: return parm->data()->toTUInt64Null(row); + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + bool tmpIsNull = false; + const string& str = parm->data()->getStrVal(row, tmpIsNull); + if (tmpIsNull) + return datatypes::TUInt64Null(); + + datatypes::DataCondition cnverr; + literal::Converter cnv(str, cnverr); + cnv.normalize(); + return cnv.negative() ? datatypes::TUInt64Null((uint64_t)cnv.toPackedSDecimal(0, cnverr)) + : datatypes::TUInt64Null(cnv.toPackedUDecimal(0, cnverr)); + } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: return DecimalToBitOperand(row, parm, thisFunc); + + case execplan::CalpontSystemCatalog::DATE: + { + bool tmpIsNull = false; + int32_t time = parm->data()->getDateIntVal(row, tmpIsNull); + if (tmpIsNull) + return datatypes::TUInt64Null(); + + int64_t value = Date(time).convertToMySQLint(); + return datatypes::TUInt64Null((uint64_t)value); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + bool tmpIsNull = false; + int64_t time = parm->data()->getDatetimeIntVal(row, tmpIsNull); + if (tmpIsNull) + return datatypes::TUInt64Null(); + + // @bug 4703 - missing year when convering to int + DateTime dt(time); + int64_t value = dt.convertToMySQLint(); + if (temporalRounding && dt.msecond >= 500000) + value++; + return datatypes::TUInt64Null((uint64_t)value); + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + bool tmpIsNull = false; + int64_t time = parm->data()->getTimestampIntVal(row, tmpIsNull); + if (tmpIsNull) + return datatypes::TUInt64Null(); + + TimeStamp dt(time); + int64_t value = dt.convertToMySQLint(thisFunc.timeZone()); + if (temporalRounding && dt.msecond >= 500000) + value++; + return datatypes::TUInt64Null((uint64_t)value); + } + + case execplan::CalpontSystemCatalog::TIME: + { + bool tmpIsNull = false; + int64_t time = parm->data()->getTimeIntVal(row, tmpIsNull); + + Time dt(time); + int64_t value = dt.convertToMySQLint(); + if (temporalRounding && dt.msecond >= 500000) + value < 0 ? value-- : value++; + return datatypes::TUInt64Null((uint64_t)value); + } + + default: + idbassert(0); // Not possible: checked during the preparation stage. + break; + } + + return datatypes::TUInt64Null(); } -} +} // namespace namespace funcexp { - - -class BitOperandGeneric: public datatypes::TUInt64Null +class BitOperandGeneric : public datatypes::TUInt64Null { -public: - BitOperandGeneric() { } - BitOperandGeneric(Row& row, - const execplan::SPTP& parm, - const funcexp::Func& thisFunc) - :TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true)) - { } + public: + BitOperandGeneric() + { + } + BitOperandGeneric(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true)) + { + } }; - // The shift amount operand in MariaDB does not round temporal values // when sql_mode=TIME_FRAC_ROUND is not set. -class BitOperandGenericShiftAmount: public datatypes::TUInt64Null +class BitOperandGenericShiftAmount : public datatypes::TUInt64Null { -public: - BitOperandGenericShiftAmount() { } - BitOperandGenericShiftAmount(Row& row, - const execplan::SPTP& parm, - const funcexp::Func& thisFunc) - :TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false)) - { } + public: + BitOperandGenericShiftAmount() + { + } + BitOperandGenericShiftAmount(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false)) + { + } }; - - // A functor to return NULL as a bitwise operation result. // Used when an unexpected argument count // is encounteded during the preparation step. -class Func_bitwise_null: public Func_BitOp +class Func_bitwise_null : public Func_BitOp { -public: - Func_bitwise_null(): Func_BitOp("bitwise") { } - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - isNull = true; - return 0; - } + public: + Func_bitwise_null() : Func_BitOp("bitwise") + { + } + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + isNull = true; + return 0; + } }; - -bool Func_BitOp::validateArgCount(execplan::FunctionColumn &col, uint expected) const +bool Func_BitOp::validateArgCount(execplan::FunctionColumn& col, uint expected) const { - static Func_bitwise_null return_null; - if (col.functionParms().size() == expected) - return false; - col.setFunctor(&return_null); - return true; + static Func_bitwise_null return_null; + if (col.functionParms().size() == expected) + return false; + col.setFunctor(&return_null); + return true; } - -void Func_BitOp::setFunctorByParm(execplan::FunctionColumn &col, - const execplan::SPTP& parm, - Func_Int & return_uint64_from_uint64, - Func_Int & return_uint64_from_sint64, - Func_Int & return_uint64_generic) const +void Func_BitOp::setFunctorByParm(execplan::FunctionColumn& col, const execplan::SPTP& parm, + Func_Int& return_uint64_from_uint64, Func_Int& return_uint64_from_sint64, + Func_Int& return_uint64_generic) const { - if (parm->data()->resultType().isUnsignedInteger()) - col.setFunctor(&return_uint64_from_uint64); - else if (parm->data()->resultType().isSignedInteger()) - col.setFunctor(&return_uint64_from_sint64); - else - col.setFunctor(&return_uint64_generic); -} - - -bool Func_BitOp::fixForBitShift(execplan::FunctionColumn &col, - Func_Int & return_uint64_from_uint64, - Func_Int & return_uint64_from_sint64, - Func_Int & return_uint64_generic) const -{ - if (validateArgCount(col, 2)) - return false; - // The functor detection is done using functionParms()[0] only. - // This is how MariaDB performs it. - setFunctorByParm(col, col.functionParms()[0], - return_uint64_from_uint64, - return_uint64_from_sint64, - return_uint64_generic); - return validateBitOperandTypeOrError(col, *this, 0) || - validateBitOperandTypeOrError(col, *this, 1); -} - - -bool Func_BitOp::fixForBitOp2(execplan::FunctionColumn &col, - Func_Int & return_uint64_from_uint64_uint64, - Func_Int & return_uint64_from_sint64_sint64, - Func_Int & return_uint64_generic) const -{ - if (validateArgCount(col, 2)) - return false; - - if (col.functionParms()[0]->data()->resultType().isUnsignedInteger() && - col.functionParms()[1]->data()->resultType().isUnsignedInteger()) - { - col.setFunctor(&return_uint64_from_uint64_uint64); - return false; - } - if (col.functionParms()[0]->data()->resultType().isSignedInteger() && - col.functionParms()[1]->data()->resultType().isSignedInteger()) - { - col.setFunctor(&return_uint64_from_sint64_sint64); - return false; - } + if (parm->data()->resultType().isUnsignedInteger()) + col.setFunctor(&return_uint64_from_uint64); + else if (parm->data()->resultType().isSignedInteger()) + col.setFunctor(&return_uint64_from_sint64); + else col.setFunctor(&return_uint64_generic); - return validateBitOperandTypeOrError(col, *this, 0) || - validateBitOperandTypeOrError(col, *this, 1); } +bool Func_BitOp::fixForBitShift(execplan::FunctionColumn& col, Func_Int& return_uint64_from_uint64, + Func_Int& return_uint64_from_sint64, Func_Int& return_uint64_generic) const +{ + if (validateArgCount(col, 2)) + return false; + // The functor detection is done using functionParms()[0] only. + // This is how MariaDB performs it. + setFunctorByParm(col, col.functionParms()[0], return_uint64_from_uint64, return_uint64_from_sint64, + return_uint64_generic); + return validateBitOperandTypeOrError(col, *this, 0) || validateBitOperandTypeOrError(col, *this, 1); +} + +bool Func_BitOp::fixForBitOp2(execplan::FunctionColumn& col, Func_Int& return_uint64_from_uint64_uint64, + Func_Int& return_uint64_from_sint64_sint64, + Func_Int& return_uint64_generic) const +{ + if (validateArgCount(col, 2)) + return false; + + if (col.functionParms()[0]->data()->resultType().isUnsignedInteger() && + col.functionParms()[1]->data()->resultType().isUnsignedInteger()) + { + col.setFunctor(&return_uint64_from_uint64_uint64); + return false; + } + if (col.functionParms()[0]->data()->resultType().isSignedInteger() && + col.functionParms()[1]->data()->resultType().isSignedInteger()) + { + col.setFunctor(&return_uint64_from_sint64_sint64); + return false; + } + col.setFunctor(&return_uint64_generic); + return validateBitOperandTypeOrError(col, *this, 0) || validateBitOperandTypeOrError(col, *this, 1); +} // // BITAND // - -template -class Func_bitand_return_uint64: public Func_bitand +template +class Func_bitand_return_uint64 : public Func_bitand { -public: - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - idbassert(parm.size() == 2); - Arg2Lazy args(row, parm, *this); - return (int64_t) (args.a & args.b).nullSafeValue(isNull); - } + public: + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + idbassert(parm.size() == 2); + Arg2Lazy args(row, parm, *this); + return (int64_t)(args.a & args.b).nullSafeValue(isNull); + } }; - -bool Func_bitand::fix(execplan::FunctionColumn &col) const +bool Func_bitand::fix(execplan::FunctionColumn& col) const { - static Func_bitand_return_uint64 return_uint64_from_uint64_uint64; - static Func_bitand_return_uint64 return_uint64_from_sint64_sint64; - static Func_bitand_return_uint64 return_uint64_generic; - return fixForBitOp2(col, return_uint64_from_uint64_uint64, - return_uint64_from_sint64_sint64, - return_uint64_generic); + static Func_bitand_return_uint64 return_uint64_from_uint64_uint64; + static Func_bitand_return_uint64 return_uint64_from_sint64_sint64; + static Func_bitand_return_uint64 return_uint64_generic; + return fixForBitOp2(col, return_uint64_from_uint64_uint64, return_uint64_from_sint64_sint64, + return_uint64_generic); } - // // LEFT SHIFT // - -template -class Func_leftshift_return_uint64: public Func_leftshift +template +class Func_leftshift_return_uint64 : public Func_leftshift { -public: - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); - return (int64_t) args.a.MariaDBShiftLeft(args.b).nullSafeValue(isNull); - } + public: + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + idbassert(parm.size() == 2); + Arg2Eager args(row, parm, *this); + return (int64_t)args.a.MariaDBShiftLeft(args.b).nullSafeValue(isNull); + } }; - -bool Func_leftshift::fix(execplan::FunctionColumn &col) const +bool Func_leftshift::fix(execplan::FunctionColumn& col) const { - static Func_leftshift_return_uint64 return_uint64_from_uint64; - static Func_leftshift_return_uint64 return_uint64_from_sint64; - static Func_leftshift_return_uint64 return_uint64_generic; - return fixForBitShift(col, return_uint64_from_uint64, - return_uint64_from_sint64, - return_uint64_generic); + static Func_leftshift_return_uint64 return_uint64_from_uint64; + static Func_leftshift_return_uint64 return_uint64_from_sint64; + static Func_leftshift_return_uint64 return_uint64_generic; + return fixForBitShift(col, return_uint64_from_uint64, return_uint64_from_sint64, return_uint64_generic); } - // // RIGHT SHIFT // - -template -class Func_rightshift_return_uint64: public Func_rightshift +template +class Func_rightshift_return_uint64 : public Func_rightshift { -public: - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); - return (int64_t) args.a.MariaDBShiftRight(args.b).nullSafeValue(isNull); - } + public: + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + idbassert(parm.size() == 2); + Arg2Eager args(row, parm, *this); + return (int64_t)args.a.MariaDBShiftRight(args.b).nullSafeValue(isNull); + } }; - -bool Func_rightshift::fix(execplan::FunctionColumn &col) const +bool Func_rightshift::fix(execplan::FunctionColumn& col) const { - static Func_rightshift_return_uint64 return_uint64_from_uint64; - static Func_rightshift_return_uint64 return_uint64_from_sint64; - static Func_rightshift_return_uint64 return_uint64_generic; - return fixForBitShift(col, return_uint64_from_uint64, - return_uint64_from_sint64, - return_uint64_generic); + static Func_rightshift_return_uint64 return_uint64_from_uint64; + static Func_rightshift_return_uint64 return_uint64_from_sint64; + static Func_rightshift_return_uint64 return_uint64_generic; + return fixForBitShift(col, return_uint64_from_uint64, return_uint64_from_sint64, return_uint64_generic); } - // // BIT OR // - -uint64_t Func_bitor::getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +uint64_t Func_bitor::getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return static_cast(getIntVal(row, fp, isNull, op_ct)); + return static_cast(getIntVal(row, fp, isNull, op_ct)); } - -template -class Func_bitor_return_uint64: public Func_bitor +template +class Func_bitor_return_uint64 : public Func_bitor { -public: - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - idbassert(parm.size() == 2); - Arg2Lazy args(row, parm, *this); - return (int64_t) (args.a | args.b).nullSafeValue(isNull); - } + public: + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + idbassert(parm.size() == 2); + Arg2Lazy args(row, parm, *this); + return (int64_t)(args.a | args.b).nullSafeValue(isNull); + } }; - -bool Func_bitor::fix(execplan::FunctionColumn &col) const +bool Func_bitor::fix(execplan::FunctionColumn& col) const { - static Func_bitor_return_uint64 return_uint64_from_uint64_uint64; - static Func_bitor_return_uint64 return_uint64_from_sint64_sint64; - static Func_bitor_return_uint64 return_uint64_generic; - return fixForBitOp2(col, return_uint64_from_uint64_uint64, - return_uint64_from_sint64_sint64, - return_uint64_generic); + static Func_bitor_return_uint64 return_uint64_from_uint64_uint64; + static Func_bitor_return_uint64 return_uint64_from_sint64_sint64; + static Func_bitor_return_uint64 return_uint64_generic; + return fixForBitOp2(col, return_uint64_from_uint64_uint64, return_uint64_from_sint64_sint64, + return_uint64_generic); } - // // BIT XOR // - -template -class Func_bitxor_return_uint64: public Func_bitxor +template +class Func_bitxor_return_uint64 : public Func_bitxor { -public: - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); - return (int64_t) (args.a ^ args.b).nullSafeValue(isNull); - } + public: + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + idbassert(parm.size() == 2); + Arg2Eager args(row, parm, *this); + return (int64_t)(args.a ^ args.b).nullSafeValue(isNull); + } }; - -bool Func_bitxor::fix(execplan::FunctionColumn &col) const +bool Func_bitxor::fix(execplan::FunctionColumn& col) const { - static Func_bitxor_return_uint64 return_uint64_from_uint64_uint64; - static Func_bitxor_return_uint64 return_uint64_from_sint64_sint64; - static Func_bitxor_return_uint64 return_uint64_generic; - return fixForBitOp2(col, return_uint64_from_uint64_uint64, - return_uint64_from_sint64_sint64, - return_uint64_generic); + static Func_bitxor_return_uint64 return_uint64_from_uint64_uint64; + static Func_bitxor_return_uint64 return_uint64_from_sint64_sint64; + static Func_bitxor_return_uint64 return_uint64_generic; + return fixForBitOp2(col, return_uint64_from_uint64_uint64, return_uint64_from_sint64_sint64, + return_uint64_generic); } - // // BIT COUNT // - inline int64_t bitCount(uint64_t val) { - // Refer to Hacker's Delight Chapter 5 - // for the bit counting algo used here - val = val - ((val >> 1) & 0x5555555555555555); - val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); - val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; - val = val + (val >> 8); - val = val + (val >> 16); - val = val + (val >> 32); + // Refer to Hacker's Delight Chapter 5 + // for the bit counting algo used here + val = val - ((val >> 1) & 0x5555555555555555); + val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); + val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; + val = val + (val >> 8); + val = val + (val >> 16); + val = val + (val >> 32); - return (int64_t)(val & 0x000000000000007F); + return (int64_t)(val & 0x000000000000007F); } - -template -class Func_bit_count_return_uint64: public Func_bit_count +template +class Func_bit_count_return_uint64 : public Func_bit_count { -public: - int64_t getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) override - { - idbassert(parm.size() == 1); - return bitCount((uint64_t) TA(row, parm[0], *this).nullSafeValue(isNull)); - } + public: + int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) override + { + idbassert(parm.size() == 1); + return bitCount((uint64_t)TA(row, parm[0], *this).nullSafeValue(isNull)); + } }; - -bool Func_bit_count::fix(execplan::FunctionColumn &col) const +bool Func_bit_count::fix(execplan::FunctionColumn& col) const { - static Func_bit_count_return_uint64 return_uint64_from_uint64; - static Func_bit_count_return_uint64 return_uint64_from_sint64; - static Func_bit_count_return_uint64 return_uint64_generic; - if (validateArgCount(col, 1)) - return false; - setFunctorByParm(col, col.functionParms()[0], - return_uint64_from_uint64, - return_uint64_from_sint64, - return_uint64_generic); - return validateBitOperandTypeOrError(col, *this, 0); + static Func_bit_count_return_uint64 return_uint64_from_uint64; + static Func_bit_count_return_uint64 return_uint64_from_sint64; + static Func_bit_count_return_uint64 return_uint64_generic; + if (validateArgCount(col, 1)) + return false; + setFunctorByParm(col, col.functionParms()[0], return_uint64_from_uint64, return_uint64_from_sint64, + return_uint64_generic); + return validateBitOperandTypeOrError(col, *this, 0); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 40f55e1fd..e8bfbd0f4 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_case.cpp 3954 2013-07-08 16:30:15Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_case.cpp 3954 2013-07-08 16:30:15Z bpaul $ + * + * + ****************************************************************************/ #include //#define NDEBUG @@ -42,421 +42,411 @@ using namespace logging; using namespace funcexp; - namespace { using namespace funcexp; -inline uint64_t simple_case_cmp(Row& row, - FunctionParm& parm, - bool& isNull, +inline uint64_t simple_case_cmp(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = 0; // index to the parm list - uint64_t n = 0; // remove expression from count of expression_i + result_i - uint64_t hasElse = (parm.size() - 1) % 2; // if 1, then ELSE exist - uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; - bool foundIt = false; + uint64_t i = 0; // index to the parm list + uint64_t n = 0; // remove expression from count of expression_i + result_i + uint64_t hasElse = (parm.size() - 1) % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; + bool foundIt = false; - switch (operationColType.colDataType) + switch (operationColType.colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DATE: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DATE: + int64_t ev = parm[n]->data()->getIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) { - int64_t ev = parm[n]->data()->getIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t ev = parm[n]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getTimestampIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t ev = parm[n]->data()->getUintVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARCHAR: - { - const string& ev = parm[n]->data()->getStrVal(row, isNull); - if (isNull) - break; - CHARSET_INFO* cs = parm[n]->data()->resultType().getCharset(); - - for (i = 1; i <= whereCount; i++) - { - //BUG 5362 - const string& p1 = parm[i]->data()->getStrVal(row, isNull); - if (isNull) - break; - if (cs->strnncoll(ev.c_str(), ev.length(), p1.c_str(), p1.length()) == 0) - { - foundIt = true; - break; - } - } - - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal ev = parm[n]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double ev = parm[n]->data()->getDoubleVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float ev = parm[n]->data()->getFloatVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double ev = parm[n]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getLongDoubleVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - default: - { - std::ostringstream oss; - oss << "case: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - } - - if (!foundIt && !hasElse) - isNull = true; - else if (!foundIt && hasElse && !isNull) - { - i = parm.size() - 1; - } - else if (isNull && hasElse) - // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL - // if so and we have else condition we need to use it by setting i = else - { - i = parm.size() - 1; - isNull = false; - } - - if (foundIt) - { - i += whereCount; - } - - return i; -} - - -inline uint64_t searched_case_cmp(Row& row, - FunctionParm& parm, - bool& isNull) -{ - uint64_t i = 0; // index to the parm list - uint64_t hasElse = parm.size() % 2; // if 1, then ELSE exist - uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; - bool foundIt = false; - - for (i = 0; i < whereCount; i++) - { - if (parm[i]->getBoolVal(row, isNull)) - { - foundIt = true; - break; - } - } - - isNull = false; - - if (!foundIt && !hasElse) - isNull = true; - else if (!foundIt && hasElse) - { - i = parm.size() - 1; - } - - if (foundIt) - { - i += whereCount; - } - - return i; -} - - -CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, - CalpontSystemCatalog::ColType& resultType, - bool simpleCase) -{ - uint64_t simple = simpleCase ? 1 : 0; - bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist - - uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); - uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; - - bool allStringO = true; - bool allStringR = true; - - FunctionParm::size_type l = fp.size() - 1; // last fp index - idbassert(fp[l]->data()); - CalpontSystemCatalog::ColType oct = fp[l]->data()->resultType(); - CalpontSystemCatalog::ColType rct = resultType; - bool operation = true; - - for (uint64_t i = 0; i <= parmCount; i++) - { - // for SimpleCase, we return the type of the case expression, - // which will always be in position 0. - if (i == 0 && simpleCase) - { - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) - { - PredicateOperator op; - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } - - i += 1; - } - - // operation or result type - operation = ((i > 0+simple) && (i <= whereCount)); - - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) - { - // this is not a string column - PredicateOperator op; - - if (operation) - { - if (!simpleCase) - { - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } - } - - // If any parm is of string type, the result type should be string. (same as if) - else if (rct.colDataType != CalpontSystemCatalog::CHAR && - rct.colDataType != CalpontSystemCatalog::TEXT && - rct.colDataType != CalpontSystemCatalog::VARCHAR) - { - op.setOpType(rct, fp[i]->data()->resultType()); - allStringR = false; - rct = op.operationType(); - } + foundIt = true; + break; } else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) { - // this is a string - // If any parm is of string type, the result type should be string. (same as if) - allStringR = true; + foundIt = true; + break; } + else + isNull = false; + } + + break; } - if (allStringO) + case execplan::CalpontSystemCatalog::TIMESTAMP: { - oct.colDataType = CalpontSystemCatalog::VARCHAR; - oct.colWidth = 255; + int64_t ev = parm[n]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimestampIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; } - if (allStringR) + case execplan::CalpontSystemCatalog::TIME: { - rct.colDataType = CalpontSystemCatalog::VARCHAR; - rct.colWidth = 255; + int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; } - if (rct.scale != 0 && rct.colDataType == CalpontSystemCatalog::BIGINT) - rct.colDataType = CalpontSystemCatalog::DECIMAL; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t ev = parm[n]->data()->getUintVal(row, isNull); - if (oct.scale != 0 && oct.colDataType == CalpontSystemCatalog::BIGINT) - oct.colDataType = CalpontSystemCatalog::DECIMAL; + if (isNull) + break; - resultType = rct; - return oct; + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARCHAR: + { + const string& ev = parm[n]->data()->getStrVal(row, isNull); + if (isNull) + break; + CHARSET_INFO* cs = parm[n]->data()->resultType().getCharset(); + + for (i = 1; i <= whereCount; i++) + { + // BUG 5362 + const string& p1 = parm[i]->data()->getStrVal(row, isNull); + if (isNull) + break; + if (cs->strnncoll(ev.c_str(), ev.length(), p1.c_str(), p1.length()) == 0) + { + foundIt = true; + break; + } + } + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal ev = parm[n]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double ev = parm[n]->data()->getDoubleVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float ev = parm[n]->data()->getFloatVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double ev = parm[n]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getLongDoubleVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + default: + { + std::ostringstream oss; + oss << "case: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + if (!foundIt && !hasElse) + isNull = true; + else if (!foundIt && hasElse && !isNull) + { + i = parm.size() - 1; + } + else if (isNull && hasElse) + // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL + // if so and we have else condition we need to use it by setting i = else + { + i = parm.size() - 1; + isNull = false; + } + + if (foundIt) + { + i += whereCount; + } + + return i; } +inline uint64_t searched_case_cmp(Row& row, FunctionParm& parm, bool& isNull) +{ + uint64_t i = 0; // index to the parm list + uint64_t hasElse = parm.size() % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; + bool foundIt = false; + + for (i = 0; i < whereCount; i++) + { + if (parm[i]->getBoolVal(row, isNull)) + { + foundIt = true; + break; + } + } + + isNull = false; + + if (!foundIt && !hasElse) + isNull = true; + else if (!foundIt && hasElse) + { + i = parm.size() - 1; + } + + if (foundIt) + { + i += whereCount; + } + + return i; } +CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, + bool simpleCase) +{ + uint64_t simple = simpleCase ? 1 : 0; + bool hasElse = (((fp.size() - simple) % 2) != 0); // if 1, then ELSE exist + + uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; + + bool allStringO = true; + bool allStringR = true; + + FunctionParm::size_type l = fp.size() - 1; // last fp index + idbassert(fp[l]->data()); + CalpontSystemCatalog::ColType oct = fp[l]->data()->resultType(); + CalpontSystemCatalog::ColType rct = resultType; + bool operation = true; + + for (uint64_t i = 0; i <= parmCount; i++) + { + // for SimpleCase, we return the type of the case expression, + // which will always be in position 0. + if (i == 0 && simpleCase) + { + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + { + PredicateOperator op; + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } + + i += 1; + } + + // operation or result type + operation = ((i > 0 + simple) && (i <= whereCount)); + + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + { + // this is not a string column + PredicateOperator op; + + if (operation) + { + if (!simpleCase) + { + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } + } + + // If any parm is of string type, the result type should be string. (same as if) + else if (rct.colDataType != CalpontSystemCatalog::CHAR && + rct.colDataType != CalpontSystemCatalog::TEXT && + rct.colDataType != CalpontSystemCatalog::VARCHAR) + { + op.setOpType(rct, fp[i]->data()->resultType()); + allStringR = false; + rct = op.operationType(); + } + } + else + { + // this is a string + // If any parm is of string type, the result type should be string. (same as if) + allStringR = true; + } + } + + if (allStringO) + { + oct.colDataType = CalpontSystemCatalog::VARCHAR; + oct.colWidth = 255; + } + + if (allStringR) + { + rct.colDataType = CalpontSystemCatalog::VARCHAR; + rct.colWidth = 255; + } + + if (rct.scale != 0 && rct.colDataType == CalpontSystemCatalog::BIGINT) + rct.colDataType = CalpontSystemCatalog::DECIMAL; + + if (oct.scale != 0 && oct.colDataType == CalpontSystemCatalog::BIGINT) + oct.colDataType = CalpontSystemCatalog::DECIMAL; + + resultType = rct; + return oct; +} + +} // namespace + namespace funcexp { - // simple CASE: // SELECT CASE ("expression") // WHEN "condition1" THEN "result1" @@ -470,158 +460,129 @@ namespace funcexp // // Note that this order changed in 10.2.14, see MCOL-1341 -CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return caseOperationType(fp, resultType, true); + return caseOperationType(fp, resultType, true); } - -bool Func_simple_case::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, +bool Func_simple_case::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return false; + if (isNull) + return false; - ParseTree* lop = parm[i]->left(); - ParseTree* rop = parm[i]->right(); - if (lop && rop) - { - return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); - } + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); + if (lop && rop) + { + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); + } - return parm[i]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } - -int64_t Func_simple_case::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_simple_case::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return joblist::BIGINTNULL; + if (isNull) + return joblist::BIGINTNULL; - return parm[i]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } - -string Func_simple_case::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_simple_case::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return string(""); + if (isNull) + return string(""); - return parm[i]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } - -IDB_Decimal Func_simple_case::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +IDB_Decimal Func_simple_case::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return IDB_Decimal(); // need a null value for IDB_Decimal?? + if (isNull) + return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } - -double Func_simple_case::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_simple_case::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return doubleNullVal(); + if (isNull) + return doubleNullVal(); - return parm[i]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } -long double Func_simple_case::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +long double Func_simple_case::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return doubleNullVal(); + if (isNull) + return doubleNullVal(); - return parm[i]->data()->getLongDoubleVal(row, isNull); + return parm[i]->data()->getLongDoubleVal(row, isNull); } - -int32_t Func_simple_case::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_simple_case::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::DATENULL; + if (isNull) + return joblist::DATENULL; - return parm[i]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } - -int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::DATETIMENULL; + if (isNull) + return joblist::DATETIMENULL; - return parm[i]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDatetimeIntVal(row, isNull); } - -int64_t Func_simple_case::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_simple_case::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::TIMESTAMPNULL; + if (isNull) + return joblist::TIMESTAMPNULL; - return parm[i]->data()->getTimestampIntVal(row, isNull); + return parm[i]->data()->getTimestampIntVal(row, isNull); } - -int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::TIMENULL; + if (isNull) + return joblist::TIMENULL; - return parm[i]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getTimeIntVal(row, isNull); } - // searched CASE: // SELECT CASE // WHEN "boolean_expression1" THEN "result1" @@ -635,157 +596,131 @@ int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, // // Note that this order changed in 10.2.14, see MCOL-1341 -CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type not used by this functor. - // return fp[1]->data()->resultType(); - return caseOperationType(fp, resultType, false); + // operation type not used by this functor. + // return fp[1]->data()->resultType(); + return caseOperationType(fp, resultType, false); } -bool Func_searched_case::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, +bool Func_searched_case::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - uint64_t i = searched_case_cmp(row, parm, isNull); + uint64_t i = searched_case_cmp(row, parm, isNull); - if (isNull) - return false; + if (isNull) + return false; - ParseTree* lop = parm[i]->left(); - ParseTree* rop = parm[i]->right(); + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); - if (lop && rop) - { - return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); - } + if (lop && rop) + { + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); + } - return parm[i]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } -int64_t Func_searched_case::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_searched_case::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - uint64_t i = searched_case_cmp(row, parm, isNull); + uint64_t i = searched_case_cmp(row, parm, isNull); - if (isNull) - return joblist::BIGINTNULL; + if (isNull) + return joblist::BIGINTNULL; - return parm[i]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } - -string Func_searched_case::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_searched_case::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - uint64_t i = searched_case_cmp(row, parm, isNull); + uint64_t i = searched_case_cmp(row, parm, isNull); - if (isNull) - return string(""); + if (isNull) + return string(""); - return parm[i]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } - -IDB_Decimal Func_searched_case::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +IDB_Decimal Func_searched_case::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - uint64_t i = searched_case_cmp(row, parm, isNull); + uint64_t i = searched_case_cmp(row, parm, isNull); - if (isNull) - return IDB_Decimal(); // need a null value for IDB_Decimal?? + if (isNull) + return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } - -double Func_searched_case::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_searched_case::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - uint64_t i = searched_case_cmp(row, parm, isNull); + uint64_t i = searched_case_cmp(row, parm, isNull); - if (isNull) - return doubleNullVal(); + if (isNull) + return doubleNullVal(); - return parm[i]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } -long double Func_searched_case::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +long double Func_searched_case::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - uint64_t i = searched_case_cmp(row, parm, isNull); + uint64_t i = searched_case_cmp(row, parm, isNull); - if (isNull) - return longDoubleNullVal(); + if (isNull) + return longDoubleNullVal(); - return parm[i]->data()->getLongDoubleVal(row, isNull); + return parm[i]->data()->getLongDoubleVal(row, isNull); } - -int32_t Func_searched_case::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int32_t Func_searched_case::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::DATENULL; + if (isNull) + return joblist::DATENULL; - return parm[i]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } - -int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::DATETIMENULL; + if (isNull) + return joblist::DATETIMENULL; - return parm[i]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDatetimeIntVal(row, isNull); } - -int64_t Func_searched_case::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_searched_case::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::TIMESTAMPNULL; + if (isNull) + return joblist::TIMESTAMPNULL; - return parm[i]->data()->getTimestampIntVal(row, isNull); + return parm[i]->data()->getTimestampIntVal(row, isNull); } - -int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::TIMENULL; + if (isNull) + return joblist::TIMENULL; - return parm[i]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getTimeIntVal(row, isNull); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 9b35a00b5..b3917b2a5 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -18,7 +18,6 @@ // $Id: func_cast.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ - #include using namespace std; @@ -52,1610 +51,1502 @@ struct lconv* convData = localeconv(); namespace funcexp { - // Why isn't "return resultType" the base default behavior? -CalpontSystemCatalog::ColType Func_cast_signed::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_signed::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -CalpontSystemCatalog::ColType Func_cast_unsigned::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_unsigned::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -CalpontSystemCatalog::ColType Func_cast_char::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_char::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -CalpontSystemCatalog::ColType Func_cast_date::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_date::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -CalpontSystemCatalog::ColType Func_cast_datetime::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_datetime::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -CalpontSystemCatalog::ColType Func_cast_decimal::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_decimal::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -CalpontSystemCatalog::ColType Func_cast_double::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_cast_double::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } // // Func_cast_signed // -int64_t Func_cast_signed::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_cast_signed::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - return (int64_t) parm[0]->data()->getIntVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - return (int64_t) parm[0]->data()->getUintVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull)); - return d.toMCSSInt64Round(); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - datatypes::TLongDouble d(parm[0]->data()->getLongDoubleVal(row, isNull)); - return d.toMCSSInt64Round(); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& value = parm[0]->data()->getStrVal(row, isNull); - - if (isNull) - { - isNull = true; - return 0; - } - - return atoll(value.c_str()); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - return parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int64_t time = parm[0]->data()->getDateIntVal(row, isNull); - - Date d(time); - return d.convertToMySQLint(); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t time = parm[0]->data()->getDatetimeIntVal(row, isNull); - - // @bug 4703 need to include year - DateTime dt(time); - return dt.convertToMySQLint(); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); - - TimeStamp dt(time); - return dt.convertToMySQLint(timeZone()); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); - - Time dt(time); - return dt.convertToMySQLint(); - } - break; - - default: - { - std::ostringstream oss; - oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return (int64_t)parm[0]->data()->getIntVal(row, isNull); } + break; - return 0; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + return (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull)); + return d.toMCSSInt64Round(); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + datatypes::TLongDouble d(parm[0]->data()->getLongDoubleVal(row, isNull)); + return d.toMCSSInt64Round(); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& value = parm[0]->data()->getStrVal(row, isNull); + + if (isNull) + { + isNull = true; + return 0; + } + + return atoll(value.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + return parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int64_t time = parm[0]->data()->getDateIntVal(row, isNull); + + Date d(time); + return d.convertToMySQLint(); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t time = parm[0]->data()->getDatetimeIntVal(row, isNull); + + // @bug 4703 need to include year + DateTime dt(time); + return dt.convertToMySQLint(); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); + + TimeStamp dt(time); + return dt.convertToMySQLint(timeZone()); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + + default: + { + std::ostringstream oss; + oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return 0; } - // // Func_cast_unsigned // -uint64_t Func_cast_unsigned::getUintVal(Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_cast_unsigned::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - return (int64_t) parm[0]->data()->getUintVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - return parm[0]->data()->getUintVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull)); - return d.toMCSUInt64Round(); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - datatypes::TLongDouble d(parm[0]->data()->getLongDoubleVal(row, isNull)); - return d.toMCSUInt64Round(); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& value = parm[0]->data()->getStrVal(row, isNull); - - if (isNull) - { - isNull = true; - return 0; - } - - uint64_t ret = strtoul(value.c_str(), 0, 0); - return ret; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - return d.toUInt64Round(); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int64_t time = parm[0]->data()->getDateIntVal(row, isNull); - - Date d(time); - return d.convertToMySQLint(); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t time = parm[0]->data()->getDatetimeIntVal(row, isNull); - - // @bug 4703 need to include year - DateTime dt(time); - return dt.convertToMySQLint(); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); - - TimeStamp dt(time); - return dt.convertToMySQLint(timeZone()); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); - - Time dt(time); - return dt.convertToMySQLint(); - } - break; - - default: - { - std::ostringstream oss; - oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return (int64_t)parm[0]->data()->getUintVal(row, isNull); } + break; - return 0; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + return parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull)); + return d.toMCSUInt64Round(); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + datatypes::TLongDouble d(parm[0]->data()->getLongDoubleVal(row, isNull)); + return d.toMCSUInt64Round(); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& value = parm[0]->data()->getStrVal(row, isNull); + + if (isNull) + { + isNull = true; + return 0; + } + + uint64_t ret = strtoul(value.c_str(), 0, 0); + return ret; + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + return d.toUInt64Round(); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int64_t time = parm[0]->data()->getDateIntVal(row, isNull); + + Date d(time); + return d.convertToMySQLint(); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t time = parm[0]->data()->getDatetimeIntVal(row, isNull); + + // @bug 4703 need to include year + DateTime dt(time); + return dt.convertToMySQLint(); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); + + TimeStamp dt(time); + return dt.convertToMySQLint(timeZone()); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + + default: + { + std::ostringstream oss; + oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return 0; } - // // Func_cast_char // -string Func_cast_char::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_cast_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { + // check for convert with 1 arg, return the argument + if (parm.size() == 1) + return parm[0]->data()->getStrVal(row, isNull); + ; - // check for convert with 1 arg, return the argument - if ( parm.size() == 1 ) - return parm[0]->data()->getStrVal(row, isNull);; + int64_t length = parm[1]->data()->getIntVal(row, isNull); - int64_t length = parm[1]->data()->getIntVal(row, isNull); + // @bug3488, a dummy parm is appended even the optional N is not present. + if (length < 0) + return parm[0]->data()->getStrVal(row, isNull); + ; - // @bug3488, a dummy parm is appended even the optional N is not present. - if ( length < 0 ) - return parm[0]->data()->getStrVal(row, isNull);; - - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - return helpers::intToString(parm[0]->data()->getIntVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - return helpers::uintToString(parm[0]->data()->getUintVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - return helpers::doubleToString(parm[0]->data()->getDoubleVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - return helpers::longDoubleToString(parm[0]->data()->getLongDoubleVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - return doubleToString(parm[0]->data()->getFloatVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& value = parm[0]->data()->getStrVal(row, isNull); - - if (isNull) - { - isNull = true; - return value; - } - - return value.substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - - if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - return d.toString(true).substr(0, length); - else - return d.toString().substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - return dataconvert::DataConvert::dateToString(parm[0]->data()->getDateIntVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - return dataconvert::DataConvert::datetimeToString(parm[0]->data()->getDatetimeIntVal(row, isNull)).substr(0, length); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()).substr(0, length); - } - break; - - default: - { - std::ostringstream oss; - oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return helpers::intToString(parm[0]->data()->getIntVal(row, isNull)).substr(0, length); } -} + break; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + return helpers::uintToString(parm[0]->data()->getUintVal(row, isNull)).substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + return helpers::doubleToString(parm[0]->data()->getDoubleVal(row, isNull)).substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + return helpers::longDoubleToString(parm[0]->data()->getLongDoubleVal(row, isNull)).substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + return doubleToString(parm[0]->data()->getFloatVal(row, isNull)).substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& value = parm[0]->data()->getStrVal(row, isNull); + + if (isNull) + { + isNull = true; + return value; + } + + return value.substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + return d.toString(true).substr(0, length); + else + return d.toString().substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + return dataconvert::DataConvert::dateToString(parm[0]->data()->getDateIntVal(row, isNull)) + .substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + return dataconvert::DataConvert::datetimeToString(parm[0]->data()->getDatetimeIntVal(row, isNull)) + .substr(0, length); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), + timeZone()) + .substr(0, length); + } + break; + + default: + { + std::ostringstream oss; + oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // Func_cast_date // -int64_t Func_cast_date::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_cast_date::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - if (operationColType.colDataType == execplan::CalpontSystemCatalog::DATE) - return Func_cast_date::getDateIntVal(row, - parm, - isNull, - operationColType); - - return Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (operationColType.colDataType == execplan::CalpontSystemCatalog::DATE) + return Func_cast_date::getDateIntVal(row, parm, isNull, operationColType); + return Func_cast_date::getDatetimeIntVal(row, parm, isNull, operationColType); } -string Func_cast_date::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_cast_date::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - int64_t value; - - if (operationColType.colDataType == execplan::CalpontSystemCatalog::DATE) - { - value = Func_cast_date::getDateIntVal(row, - parm, - isNull, - operationColType); - char buf[30] = {'\0'}; - dataconvert::DataConvert::dateToString(value, buf, sizeof(buf)); - return string(buf); - } - - value = Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + int64_t value; + if (operationColType.colDataType == execplan::CalpontSystemCatalog::DATE) + { + value = Func_cast_date::getDateIntVal(row, parm, isNull, operationColType); char buf[30] = {'\0'}; - dataconvert::DataConvert::datetimeToString(value, buf, sizeof(buf)); + dataconvert::DataConvert::dateToString(value, buf, sizeof(buf)); return string(buf); + } + + value = Func_cast_date::getDatetimeIntVal(row, parm, isNull, operationColType); + + char buf[30] = {'\0'}; + dataconvert::DataConvert::datetimeToString(value, buf, sizeof(buf)); + return string(buf); } -IDB_Decimal Func_cast_date::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +IDB_Decimal Func_cast_date::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal decimal; + IDB_Decimal decimal; - if (parm[0]->data()->resultType().isWideDecimalType()) - decimal.s128Value = Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); - else - decimal.value = Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (parm[0]->data()->resultType().isWideDecimalType()) + decimal.s128Value = Func_cast_date::getDatetimeIntVal(row, parm, isNull, operationColType); + else + decimal.value = Func_cast_date::getDatetimeIntVal(row, parm, isNull, operationColType); - return decimal; + return decimal; } -double Func_cast_date::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_cast_date::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return (double) Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + return (double)Func_cast_date::getDatetimeIntVal(row, parm, isNull, operationColType); } -long double Func_cast_date::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +long double Func_cast_date::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - return (long double) Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + return (long double)Func_cast_date::getDatetimeIntVal(row, parm, isNull, operationColType); } - -int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t val; + int64_t val; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - val = dataconvert::DataConvert::intToDate(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDate(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; - else - return val; + if (val == -1) + isNull = true; + else + return val; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - val = dataconvert::DataConvert::intToDate(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDate(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - { - return parm[0]->data()->getDateIntVal(row, isNull); - } - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - string value = dataconvert::DataConvert::timestampToString(val1, timeZone()); - value = value.substr(0, 10); - return dataconvert::DataConvert::stringToDate(value); - } - case execplan::CalpontSystemCatalog::TIME: - { - int64_t val1; - string value = ""; - DateTime aDateTime = static_cast(nowDatetime()); - Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - if ((aTime.hour < 0) || (aTime.is_neg)) - { - aTime.hour = -abs(aTime.hour); - aTime.minute = -abs(aTime.minute); - aTime.second = -abs(aTime.second); - aTime.msecond = -abs(aTime.msecond); - } - - aDateTime.hour = 0; - aDateTime.minute = 0; - aDateTime.second = 0; - aDateTime.msecond = 0; - val1 = addTime(aDateTime, aTime); - value = dataconvert::DataConvert::datetimeToString(val1); - value = value.substr(0, 10); - return dataconvert::DataConvert::stringToDate(value); - break; - } - - - - default: - { - isNull = true; - } + break; } - return 0; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + val = dataconvert::DataConvert::intToDate(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDate(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getDateIntVal(row, isNull); + } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); + string value = dataconvert::DataConvert::timestampToString(val1, timeZone()); + value = value.substr(0, 10); + return dataconvert::DataConvert::stringToDate(value); + } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val1; + string value = ""; + DateTime aDateTime = static_cast(nowDatetime()); + Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + if ((aTime.hour < 0) || (aTime.is_neg)) + { + aTime.hour = -abs(aTime.hour); + aTime.minute = -abs(aTime.minute); + aTime.second = -abs(aTime.second); + aTime.msecond = -abs(aTime.msecond); + } + + aDateTime.hour = 0; + aDateTime.minute = 0; + aDateTime.second = 0; + aDateTime.msecond = 0; + val1 = addTime(aDateTime, aTime); + value = dataconvert::DataConvert::datetimeToString(val1); + value = value.substr(0, 10); + return dataconvert::DataConvert::stringToDate(value); + break; + } + + default: + { + isNull = true; + } + } + + return 0; } -int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType) +int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) { - int64_t val; + int64_t val; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; - else - return val; + if (val == -1) + isNull = true; + else + return val; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale != 0) - { - isNull = true; - break; - } - else - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - return parm[0]->data()->getDatetimeIntVal(row, isNull); - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - // @bug 4703 eliminated unnecessary conversion from datetime to string and back - // need to zero out the time portions since we are casting to date - DateTime val1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - val1.hour = 0; - val1.minute = 0; - val1.second = 0; - val1.msecond = 0; - return *(reinterpret_cast(&val1)); - } - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - DateTime dt; - dt.year = m_time.year; - dt.month = m_time.month; - dt.day = m_time.day; - dt.hour = 0; - dt.minute = 0; - dt.second = 0; - dt.msecond = 0; - return *(reinterpret_cast(&dt)); - } - case CalpontSystemCatalog::TIME: - { - DateTime aDateTime = static_cast(nowDatetime()); - Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - if ((aTime.hour < 0) || (aTime.is_neg)) - { - aTime.hour = -abs(aTime.hour); - aTime.minute = -abs(aTime.minute); - aTime.second = -abs(aTime.second); - aTime.msecond = -abs(aTime.msecond); - } - - aDateTime.hour = 0; - aDateTime.minute = 0; - aDateTime.second = 0; - aDateTime.msecond = 0; - val = addTime(aDateTime, aTime); - return val; - } - - - default: - { - isNull = true; - } + break; } - return 0; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (parm[0]->data()->resultType().scale != 0) + { + isNull = true; + break; + } + else + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getDatetimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + // @bug 4703 eliminated unnecessary conversion from datetime to string and back + // need to zero out the time portions since we are casting to date + DateTime val1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + val1.hour = 0; + val1.minute = 0; + val1.second = 0; + val1.msecond = 0; + return *(reinterpret_cast(&val1)); + } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + return *(reinterpret_cast(&dt)); + } + case CalpontSystemCatalog::TIME: + { + DateTime aDateTime = static_cast(nowDatetime()); + Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + if ((aTime.hour < 0) || (aTime.is_neg)) + { + aTime.hour = -abs(aTime.hour); + aTime.minute = -abs(aTime.minute); + aTime.second = -abs(aTime.second); + aTime.msecond = -abs(aTime.msecond); + } + + aDateTime.hour = 0; + aDateTime.minute = 0; + aDateTime.second = 0; + aDateTime.msecond = 0; + val = addTime(aDateTime, aTime); + return val; + } + + default: + { + isNull = true; + } + } + + return 0; } // // Func_cast_datetime // -int64_t Func_cast_datetime::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_cast_datetime::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); - + return Func_cast_datetime::getDatetimeIntVal(row, parm, isNull, operationColType); } -string Func_cast_datetime::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_cast_datetime::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - int64_t value = Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + int64_t value = Func_cast_datetime::getDatetimeIntVal(row, parm, isNull, operationColType); - char buf[30] = {'\0'}; - dataconvert::DataConvert::datetimeToString(value, buf, sizeof(buf)); - return string(buf); + char buf[30] = {'\0'}; + dataconvert::DataConvert::datetimeToString(value, buf, sizeof(buf)); + return string(buf); } -IDB_Decimal Func_cast_datetime::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +IDB_Decimal Func_cast_datetime::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal decimal; + IDB_Decimal decimal; - if (parm[0]->data()->resultType().isWideDecimalType()) - decimal.s128Value = Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); - else - decimal.value = Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (parm[0]->data()->resultType().isWideDecimalType()) + decimal.s128Value = Func_cast_datetime::getDatetimeIntVal(row, parm, isNull, operationColType); + else + decimal.value = Func_cast_datetime::getDatetimeIntVal(row, parm, isNull, operationColType); - return decimal; + return decimal; } -double Func_cast_datetime::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_cast_datetime::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return (double) Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + return (double)Func_cast_datetime::getDatetimeIntVal(row, parm, isNull, operationColType); } -long double Func_cast_datetime::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +long double Func_cast_datetime::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - return (long double) Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + return (long double)Func_cast_datetime::getDatetimeIntVal(row, parm, isNull, operationColType); } -int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType) +int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) { - int64_t val; + int64_t val; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; - else - return val; + if (val == -1) + isNull = true; + else + return val; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - return parm[0]->data()->getDatetimeIntVal(row, isNull); - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - return parm[0]->data()->getDatetimeIntVal(row, isNull); - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - DateTime dt; - dt.year = m_time.year; - dt.month = m_time.month; - dt.day = m_time.day; - dt.hour = m_time.hour; - dt.minute = m_time.minute; - dt.second = m_time.second; - dt.msecond = timestamp.msecond; - return *(reinterpret_cast(&dt)); - } - - case CalpontSystemCatalog::TIME: - { - DateTime aDateTime = static_cast(nowDatetime()); - Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aDateTime.hour = 0; - aDateTime.minute = 0; - aDateTime.second = 0; - aDateTime.msecond = 0; - if ((aTime.hour < 0) || (aTime.is_neg)) - { - aTime.hour = -abs(aTime.hour); - aTime.minute = -abs(aTime.minute); - aTime.second = -abs(aTime.second); - aTime.msecond = -abs(aTime.msecond); - } - aTime.day = 0; - return addTime(aDateTime, aTime); - break; - } - - default: - { - isNull = true; - } + break; } - return -1; -} - - -int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType) -{ - int64_t val; - - switch (parm[0]->data()->resultType().colDataType) + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; - else - return val; + if (val == -1) + isNull = true; + else + return val; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - else - return val; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - return parm[0]->data()->getTimeIntVal(row, isNull); - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - return parm[0]->data()->getTimeIntVal(row, isNull); - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - Time time; - time.hour = m_time.hour; - time.minute = m_time.minute; - time.second = m_time.second; - time.is_neg = 0; - time.day = 0; - time.msecond = 0; - return *(reinterpret_cast(&time)); - } - - default: - { - isNull = true; - } + break; } - return -1; + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getDatetimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getDatetimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = m_time.hour; + dt.minute = m_time.minute; + dt.second = m_time.second; + dt.msecond = timestamp.msecond; + return *(reinterpret_cast(&dt)); + } + + case CalpontSystemCatalog::TIME: + { + DateTime aDateTime = static_cast(nowDatetime()); + Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aDateTime.hour = 0; + aDateTime.minute = 0; + aDateTime.second = 0; + aDateTime.msecond = 0; + if ((aTime.hour < 0) || (aTime.is_neg)) + { + aTime.hour = -abs(aTime.hour); + aTime.minute = -abs(aTime.minute); + aTime.second = -abs(aTime.second); + aTime.msecond = -abs(aTime.msecond); + } + aTime.day = 0; + return addTime(aDateTime, aTime); + break; + } + + default: + { + isNull = true; + } + } + + return -1; } +int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) +{ + int64_t val; + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + Time time; + time.hour = m_time.hour; + time.minute = m_time.minute; + time.second = m_time.second; + time.is_neg = 0; + time.day = 0; + time.msecond = 0; + return *(reinterpret_cast(&time)); + } + + default: + { + isNull = true; + } + } + + return -1; +} // // Func_cast_decimal // -int64_t Func_cast_decimal::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_cast_decimal::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal decimal = Func_cast_decimal::getDecimalVal(row, - parm, - isNull, - operationColType); - return decimal.toSInt64Round(); + IDB_Decimal decimal = Func_cast_decimal::getDecimalVal(row, parm, isNull, operationColType); + return decimal.toSInt64Round(); } - -string Func_cast_decimal::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_cast_decimal::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal decimal = Func_cast_decimal::getDecimalVal(row, - parm, - isNull, - operationColType); - if (operationColType.isWideDecimalType()) - return decimal.toString(true); - else - return decimal.toString(); + IDB_Decimal decimal = Func_cast_decimal::getDecimalVal(row, parm, isNull, operationColType); + if (operationColType.isWideDecimalType()) + return decimal.toString(true); + else + return decimal.toString(); } - -IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal decimal; + IDB_Decimal decimal; - int32_t decimals = parm[1]->data()->getIntVal(row, isNull); - int64_t max_length = parm[2]->data()->getIntVal(row, isNull); + int32_t decimals = parm[1]->data()->getIntVal(row, isNull); + int64_t max_length = parm[2]->data()->getIntVal(row, isNull); - if (max_length > datatypes::INT128MAXPRECISION || max_length <= 0) - max_length = datatypes::INT128MAXPRECISION; + if (max_length > datatypes::INT128MAXPRECISION || max_length <= 0) + max_length = datatypes::INT128MAXPRECISION; - decimal.precision = max_length; + decimal.precision = max_length; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: + if (decimal.isTSInt128ByPrecision()) + { + bool dummy = false; + char* ep = NULL; + int128_t max_number_decimal = + dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + decimal.s128Value = parm[0]->data()->getIntVal(row, isNull); + decimal.scale = 0; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + int128_t value = decimal.s128Value * scaleDivisor; + + if (value > max_number_decimal) { - if (decimal.isTSInt128ByPrecision()) - { - bool dummy = false; - char *ep = NULL; - int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - decimal.s128Value = parm[0]->data()->getIntVal(row, isNull); - decimal.scale = 0; - int128_t scaleDivisor; - datatypes::getScaleDivisor(scaleDivisor, decimals); - int128_t value = decimal.s128Value * scaleDivisor; - - if ( value > max_number_decimal ) - { - decimal.s128Value = max_number_decimal; - decimal.scale = decimals; - } - else if ( value < -max_number_decimal ) - { - decimal.s128Value = -max_number_decimal; - decimal.scale = decimals; - } - } - else - { - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.value = parm[0]->data()->getIntVal(row, isNull); - decimal.scale = 0; - int64_t value = decimal.value * helpers::powerOf10_c[decimals]; - - if ( value > max_number_decimal ) - { - decimal.value = max_number_decimal; - decimal.scale = decimals; - } - else if ( value < -max_number_decimal ) - { - decimal.value = -max_number_decimal; - decimal.scale = decimals; - } - } + decimal.s128Value = max_number_decimal; + decimal.scale = decimals; } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: + else if (value < -max_number_decimal) { - if (decimal.isTSInt128ByPrecision()) - { - bool dummy = false; - char *ep = NULL; - int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - - uint128_t uval = parm[0]->data()->getUintVal(row, isNull); - - if (uval > (uint128_t)datatypes::Decimal::maxInt128) - { - uval = datatypes::Decimal::maxInt128; - } - - decimal.s128Value = uval; - decimal.scale = 0; - int128_t scaleDivisor; - datatypes::getScaleDivisor(scaleDivisor, decimals); - int128_t value = decimal.s128Value * scaleDivisor; - - if ( value > max_number_decimal ) - { - decimal.s128Value = max_number_decimal; - decimal.scale = decimals; - } - } - else - { - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - - uint64_t uval = parm[0]->data()->getUintVal(row, isNull); - - if (uval > (uint64_t)numeric_limits::max()) - { - uval = numeric_limits::max(); - } - - decimal.value = uval; - decimal.scale = 0; - int64_t value = decimal.value * helpers::powerOf10_c[decimals]; - - if ( value > max_number_decimal ) - { - decimal.value = max_number_decimal; - decimal.scale = decimals; - } - } + decimal.s128Value = -max_number_decimal; + decimal.scale = decimals; } - break; + } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + decimal.value = parm[0]->data()->getIntVal(row, isNull); + decimal.scale = 0; + int64_t value = decimal.value * helpers::powerOf10_c[decimals]; - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: + if (value > max_number_decimal) { - if (decimal.isTSInt128ByPrecision()) - { - bool dummy = false; - char *ep = NULL; - int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - - float128_t value = parm[0]->data()->getDoubleVal(row, isNull); - - int128_t scaleDivisor; - datatypes::getScaleDivisor(scaleDivisor, decimals); - - if (value > 0) - decimal.s128Value = (int128_t) (value * scaleDivisor + 0.5); - else if (value < 0) - decimal.s128Value = (int128_t) (value * scaleDivisor - 0.5); - else - decimal.s128Value = 0; - - decimal.scale = decimals; - - if ( value > max_number_decimal ) - decimal.s128Value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.s128Value = -max_number_decimal; - } - else - { - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - - double value = parm[0]->data()->getDoubleVal(row, isNull); - - if (value > 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); - else if (value < 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); - else - decimal.value = 0; - - decimal.scale = decimals; - - if ( value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.value = -max_number_decimal; - } + decimal.value = max_number_decimal; + decimal.scale = decimals; } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: + else if (value < -max_number_decimal) { - if (decimal.isTSInt128ByPrecision()) - { - bool dummy = false; - char *ep = NULL; - int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - - float128_t value = parm[0]->data()->getLongDoubleVal(row, isNull); - - int128_t scaleDivisor; - datatypes::getScaleDivisor(scaleDivisor, decimals); - - if (value > 0) - decimal.s128Value = (int128_t) (value * scaleDivisor + 0.5); - else if (value < 0) - decimal.s128Value = (int128_t) (value * scaleDivisor - 0.5); - else - decimal.s128Value = 0; - - decimal.scale = decimals; - - if ( value > max_number_decimal ) - decimal.s128Value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.s128Value = -max_number_decimal; - } - else - { - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (value > 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); - else if (value < 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); - else - decimal.value = 0; - - decimal.scale = decimals; - - if ( value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.value = -max_number_decimal; - } + decimal.value = -max_number_decimal; + decimal.scale = decimals; } - break; + } + } + break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + if (decimal.isTSInt128ByPrecision()) + { + bool dummy = false; + char* ep = NULL; + int128_t max_number_decimal = + dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + + uint128_t uval = parm[0]->data()->getUintVal(row, isNull); + + if (uval > (uint128_t)datatypes::Decimal::maxInt128) { - if (decimal.isTSInt128ByPrecision()) - { - bool dummy = false; - char *ep = NULL; - int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - - decimal = parm[0]->data()->getDecimalVal(row, isNull); - - int128_t scaleDivisor; - datatypes::getScaleDivisor(scaleDivisor, abs(decimals - decimal.scale)); - - if (!decimal.isTSInt128ByPrecision()) - decimal.s128Value = decimal.value; - - decimal.precision = max_length; - - if (scaleDivisor > 1) - { - if (decimals > decimal.scale) - decimal.s128Value *= scaleDivisor; - else - decimal.s128Value = (int128_t)(decimal.s128Value > 0 ? - (float128_t)decimal.s128Value / scaleDivisor + 0.5 : - (float128_t)decimal.s128Value / scaleDivisor - 0.5); - } - - decimal.scale = decimals; - - if ( decimal.s128Value > max_number_decimal ) - decimal.s128Value = max_number_decimal; - else if ( decimal.s128Value < -max_number_decimal ) - decimal.s128Value = -max_number_decimal; - } - else - { - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - - decimal = parm[0]->data()->getDecimalVal(row, isNull); - - if (decimal.isTSInt128ByPrecision()) - { - if ( decimal.s128Value > (int128_t) max_number_decimal ) - decimal.value = max_number_decimal; - else if ( decimal.s128Value < (int128_t) -max_number_decimal ) - decimal.value = -max_number_decimal; - else - decimal.value = decimal.s128Value; - } - - decimal.precision = max_length; - - if (decimals > decimal.scale) - decimal.value *= helpers::powerOf10_c[decimals - decimal.scale]; - else - decimal.value = (int64_t)(decimal.value > 0 ? - (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] + 0.5 : - (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] - 0.5); - - decimal.scale = decimals; - - if ( decimal.value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( decimal.value < -max_number_decimal ) - decimal.value = -max_number_decimal; - } - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& strValue = parm[0]->data()->getStrVal(row, isNull); - if (strValue.empty()) - { - isNull = true; - return IDB_Decimal(); // need a null value for IDB_Decimal?? - } - datatypes::DataCondition convError; - return IDB_Decimal(strValue.data(), strValue.length(), convError, decimals, max_length); + uval = datatypes::Decimal::maxInt128; } - break; + decimal.s128Value = uval; + decimal.scale = 0; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + int128_t value = decimal.s128Value * scaleDivisor; - case execplan::CalpontSystemCatalog::DATE: + if (value > max_number_decimal) { - int32_t s = 0; - - string value = dataconvert::DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); - int32_t x = atol(value.c_str()); - - if (!isNull) - { - if (decimal.isTSInt128ByPrecision()) - decimal.s128Value = x; - else - decimal.value = x; - - decimal.scale = s; - } + decimal.s128Value = max_number_decimal; + decimal.scale = decimals; } - break; + } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - case execplan::CalpontSystemCatalog::DATETIME: + uint64_t uval = parm[0]->data()->getUintVal(row, isNull); + + if (uval > (uint64_t)numeric_limits::max()) { - int32_t s = 0; - - string value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - - //strip off micro seconds - string date = value.substr(0, 14); - - int64_t x = atoll(date.c_str()); - - if (!isNull) - { - if (decimal.isTSInt128ByPrecision()) - decimal.s128Value = x; - else - decimal.value = x; - - decimal.scale = s; - } + uval = numeric_limits::max(); } - break; - case execplan::CalpontSystemCatalog::TIMESTAMP: + decimal.value = uval; + decimal.scale = 0; + int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + + if (value > max_number_decimal) { - int32_t s = 0; - - string value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - - //strip off micro seconds - string date = value.substr(0, 14); - - int64_t x = atoll(date.c_str()); - - if (!isNull) - { - if (max_length > decimal.isTSInt128ByPrecision()) - decimal.s128Value = x; - else - decimal.value = x; - - decimal.scale = s; - } + decimal.value = max_number_decimal; + decimal.scale = decimals; } - break; + } + } + break; - case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + if (decimal.isTSInt128ByPrecision()) + { + bool dummy = false; + char* ep = NULL; + int128_t max_number_decimal = + dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + + float128_t value = parm[0]->data()->getDoubleVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (value > 0) + decimal.s128Value = (int128_t)(value * scaleDivisor + 0.5); + else if (value < 0) + decimal.s128Value = (int128_t)(value * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + decimal.scale = decimals; + + if (value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + + double value = parm[0]->data()->getDoubleVal(row, isNull); + + if (value > 0) + decimal.value = (int64_t)(value * helpers::powerOf10_c[decimals] + 0.5); + else if (value < 0) + decimal.value = (int64_t)(value * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + decimal.scale = decimals; + + if (value > max_number_decimal) + decimal.value = max_number_decimal; + else if (value < -max_number_decimal) + decimal.value = -max_number_decimal; + } + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + if (decimal.isTSInt128ByPrecision()) + { + bool dummy = false; + char* ep = NULL; + int128_t max_number_decimal = + dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + + float128_t value = parm[0]->data()->getLongDoubleVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (value > 0) + decimal.s128Value = (int128_t)(value * scaleDivisor + 0.5); + else if (value < 0) + decimal.s128Value = (int128_t)(value * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + decimal.scale = decimals; + + if (value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (value > 0) + decimal.value = (int64_t)(value * helpers::powerOf10_c[decimals] + 0.5); + else if (value < 0) + decimal.value = (int64_t)(value * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + decimal.scale = decimals; + + if (value > max_number_decimal) + decimal.value = max_number_decimal; + else if (value < -max_number_decimal) + decimal.value = -max_number_decimal; + } + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (decimal.isTSInt128ByPrecision()) + { + bool dummy = false; + char* ep = NULL; + int128_t max_number_decimal = + dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + + decimal = parm[0]->data()->getDecimalVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, abs(decimals - decimal.scale)); + + if (!decimal.isTSInt128ByPrecision()) + decimal.s128Value = decimal.value; + + decimal.precision = max_length; + + if (scaleDivisor > 1) { - int32_t s = 0; - - string value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); - - //strip off micro seconds - string date = value.substr(0, 14); - - int64_t x = atoll(date.c_str()); - - if (!isNull) - { - if (decimal.isTSInt128ByPrecision()) - decimal.s128Value = x; - else - decimal.value = x; - - decimal.scale = s; - } + if (decimals > decimal.scale) + decimal.s128Value *= scaleDivisor; + else + decimal.s128Value = + (int128_t)(decimal.s128Value > 0 ? (float128_t)decimal.s128Value / scaleDivisor + 0.5 + : (float128_t)decimal.s128Value / scaleDivisor - 0.5); } - break; - default: + decimal.scale = decimals; + + if (decimal.s128Value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (decimal.s128Value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + + decimal = parm[0]->data()->getDecimalVal(row, isNull); + + if (decimal.isTSInt128ByPrecision()) { - std::ostringstream oss; - oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + if (decimal.s128Value > (int128_t)max_number_decimal) + decimal.value = max_number_decimal; + else if (decimal.s128Value < (int128_t)-max_number_decimal) + decimal.value = -max_number_decimal; + else + decimal.value = decimal.s128Value; } + + decimal.precision = max_length; + + if (decimals > decimal.scale) + decimal.value *= helpers::powerOf10_c[decimals - decimal.scale]; + else + decimal.value = + (int64_t)(decimal.value > 0 + ? (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] + 0.5 + : (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] - 0.5); + + decimal.scale = decimals; + + if (decimal.value > max_number_decimal) + decimal.value = max_number_decimal; + else if (decimal.value < -max_number_decimal) + decimal.value = -max_number_decimal; + } + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& strValue = parm[0]->data()->getStrVal(row, isNull); + if (strValue.empty()) + { + isNull = true; + return IDB_Decimal(); // need a null value for IDB_Decimal?? + } + datatypes::DataCondition convError; + return IDB_Decimal(strValue.data(), strValue.length(), convError, decimals, max_length); } - return decimal; + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + int32_t x = atol(value.c_str()); + + if (!isNull) + { + if (decimal.isTSInt128ByPrecision()) + decimal.s128Value = x; + else + decimal.value = x; + + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int32_t s = 0; + + string value = + dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + // strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + if (decimal.isTSInt128ByPrecision()) + decimal.s128Value = x; + else + decimal.value = x; + + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timestampToString1( + parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + + // strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + if (max_length > decimal.isTSInt128ByPrecision()) + decimal.s128Value = x; + else + decimal.value = x; + + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + if (decimal.isTSInt128ByPrecision()) + decimal.s128Value = x; + else + decimal.value = x; + + decimal.scale = s; + } + } + break; + + default: + { + std::ostringstream oss; + oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return decimal; } -double Func_cast_decimal::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_cast_decimal::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal decimal = Func_cast_decimal::getDecimalVal(row, - parm, - isNull, - operationColType); + IDB_Decimal decimal = Func_cast_decimal::getDecimalVal(row, parm, isNull, operationColType); - // TODO MCOL-641 This could deliver wrong result b/c wide DECIMAL might have - // p <= INT64MAXPRECISION || p > INT128MAXPRECISION - if (decimal.isTSInt128ByPrecision()) - { - return static_cast(decimal); - } + // TODO MCOL-641 This could deliver wrong result b/c wide DECIMAL might have + // p <= INT64MAXPRECISION || p > INT128MAXPRECISION + if (decimal.isTSInt128ByPrecision()) + { + return static_cast(decimal); + } - return decimal.decimal64ToXFloat(); + return decimal.decimal64ToXFloat(); } - // // Func_cast_double // -int64_t Func_cast_double::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_cast_double::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { + double dblval = Func_cast_double::getDoubleVal(row, parm, isNull, operationColType); - double dblval = Func_cast_double::getDoubleVal(row, - parm, - isNull, - operationColType); - - return (int64_t) dblval; + return (int64_t)dblval; } - -string Func_cast_double::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_cast_double::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - double dblval = Func_cast_double::getDoubleVal(row, - parm, - isNull, - operationColType); + double dblval = Func_cast_double::getDoubleVal(row, parm, isNull, operationColType); - std::string value = helpers::doubleToString(dblval); - - return value; + std::string value = helpers::doubleToString(dblval); + return value; } - -double Func_cast_double::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_cast_double::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - double dblval; + double dblval; - // TODO: Here onwards - switch (parm[0]->data()->resultType().colDataType) + // TODO: Here onwards + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t intval = parm[0]->data()->getIntVal(row, isNull); - dblval = (double) intval; - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - - // strip off micro seconds - str = str.substr(0, 14); - - dblval = atof(str.c_str()); - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t uintval = parm[0]->data()->getUintVal(row, isNull); - dblval = (double) uintval; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - dblval = parm[0]->data()->getDoubleVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - dblval = static_cast(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - - if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - dblval = static_cast(decimal); - } - else - { - dblval = decimal.decimal64ToXFloat(); - } - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& strValue = parm[0]->data()->getStrVal(row, isNull); - - dblval = strtod(strValue.c_str(), NULL); - } - break; - - default: - { - std::ostringstream oss; - oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + int64_t intval = parm[0]->data()->getIntVal(row, isNull); + dblval = (double)intval; } + break; - return dblval; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + + // strip off micro seconds + str = str.substr(0, 14); + + dblval = atof(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t uintval = parm[0]->data()->getUintVal(row, isNull); + dblval = (double)uintval; + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + dblval = parm[0]->data()->getDoubleVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + dblval = static_cast(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + dblval = static_cast(decimal); + } + else + { + dblval = decimal.decimal64ToXFloat(); + } + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& strValue = parm[0]->data()->getStrVal(row, isNull); + + dblval = strtod(strValue.c_str(), NULL); + } + break; + + default: + { + std::ostringstream oss; + oss << "cast: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return dblval; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 41ca82536..f0c016847 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_ceil.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_ceil.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -44,585 +44,558 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_ceil::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_ceil::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return fp[0]->data()->resultType(); + return fp[0]->data()->resultType(); } - -int64_t Func_ceil::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +int64_t Func_ceil::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t ret = 0; + int64_t ret = 0; - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - { - ret = parm[0]->data()->getIntVal(row, isNull); - } - break; - - // ceil(decimal(X,Y)) leads to this path if X, Y allows to - // downcast to INT otherwise Func_ceil::getDecimalVal() is called - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (op_ct.scale == 0) - { - ret = parm[0]->data()->getIntVal(row, isNull); - break; - } - - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - // negative scale is not supported by CNX yet - if (d.scale > 0) - { - if (d.scale > datatypes::INT128MAXPRECISION) - { - std::ostringstream oss; - oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) d.scale << " is beyond supported scale"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - - ret = d.toSInt64Ceil(); - } - } - break; - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - { - ret = (int64_t)(parm[0]->data()->getUintVal(row, isNull)); - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - ret = (int64_t) ceil(parm[0]->data()->getDoubleVal(row, isNull)); - } - break; - - case CalpontSystemCatalog::LONGDOUBLE: - { - ret = (int64_t) ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - { - const string& str = parm[0]->data()->getStrVal(row, isNull); - - if (!isNull) - ret = (int64_t) ceil(strtod(str.c_str(), 0)); - } - break; - - case CalpontSystemCatalog::DATE: - { - // For some reason, MDB doesn't return this as a date, - // but datetime is returned as a datetime. Expect - // this to change in the future. - Date d (parm[0]->data()->getDateIntVal(row, isNull)); - if (!isNull) - ret = d.convertToMySQLint(); - } - break; - - case CalpontSystemCatalog::DATETIME: - { - ret = parm[0]->data()->getDatetimeIntVal(row, isNull); - } - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - ret = parm[0]->data()->getTimestampIntVal(row, isNull); - } - break; - - case CalpontSystemCatalog::TIME: - { - ret = parm[0]->data()->getTimeIntVal(row, isNull); - } - break; - - default: - { - std::ostringstream oss; - oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) - << " is not supported"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + ret = parm[0]->data()->getIntVal(row, isNull); } + break; - return ret; + // ceil(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_ceil::getDecimalVal() is called + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (op_ct.scale == 0) + { + ret = parm[0]->data()->getIntVal(row, isNull); + break; + } + + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (d.scale > 0) + { + if (d.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) << " with scale " + << (int)d.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + ret = d.toSInt64Ceil(); + } + } + break; + + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + { + ret = (int64_t)(parm[0]->data()->getUintVal(row, isNull)); + } + break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + ret = (int64_t)ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case CalpontSystemCatalog::LONGDOUBLE: + { + ret = (int64_t)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret = (int64_t)ceil(strtod(str.c_str(), 0)); + } + break; + + case CalpontSystemCatalog::DATE: + { + // For some reason, MDB doesn't return this as a date, + // but datetime is returned as a datetime. Expect + // this to change in the future. + Date d(parm[0]->data()->getDateIntVal(row, isNull)); + if (!isNull) + ret = d.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::DATETIME: + { + ret = parm[0]->data()->getDatetimeIntVal(row, isNull); + } + break; + + case CalpontSystemCatalog::TIMESTAMP: + { + ret = parm[0]->data()->getTimestampIntVal(row, isNull); + } + break; + + case CalpontSystemCatalog::TIME: + { + ret = parm[0]->data()->getTimeIntVal(row, isNull); + } + break; + + default: + { + std::ostringstream oss; + oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) << " is not supported"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return ret; } -uint64_t Func_ceil::getUintVal(Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_ceil::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t ret = 0; + uint64_t ret = 0; - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - { - ret = (uint64_t)parm[0]->data()->getIntVal(row, isNull); - } - break; - - // ceil(decimal(X,Y)) leads to this path if X, Y allows to - // downcast to INT otherwise Func_ceil::getDecimalVal() is called - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (op_ct.scale == 0) - { - ret = parm[0]->data()->getIntVal(row, isNull); - break; - } - - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - // negative scale is not supported by CNX yet - if (d.scale > 0) - { - if (d.scale > datatypes::INT128MAXPRECISION) - { - std::ostringstream oss; - oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) d.scale << " is beyond supported scale"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - - ret = d.toUInt64Ceil(); - } - } - break; - - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - { - ret = (parm[0]->data()->getUintVal(row, isNull)); - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - ret = (uint64_t) ceil(parm[0]->data()->getDoubleVal(row, isNull)); - } - break; - - case CalpontSystemCatalog::LONGDOUBLE: - { - ret = (uint64_t) ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - { - const string& str = parm[0]->data()->getStrVal(row, isNull); - - if (!isNull) - ret = (uint64_t) ceil(strtod(str.c_str(), 0)); - } - break; - - case CalpontSystemCatalog::DATE: - { - // For some reason, MDB doesn't return this as a date, - // but datetime is returned as a datetime. Expect - // this to change in the future. - Date d (parm[0]->data()->getDateIntVal(row, isNull)); - if (!isNull) - ret = d.convertToMySQLint(); - } - break; - - case CalpontSystemCatalog::DATETIME: - { - ret = parm[0]->data()->getDatetimeIntVal(row, isNull); - } - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - ret = parm[0]->data()->getTimestampIntVal(row, isNull); - } - break; - - case CalpontSystemCatalog::TIME: - { - ret = parm[0]->data()->getTimeIntVal(row, isNull); - } - break; - - default: - { - std::ostringstream oss; - oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) - << " is not supported"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + ret = (uint64_t)parm[0]->data()->getIntVal(row, isNull); } + break; - return ret; + // ceil(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_ceil::getDecimalVal() is called + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (op_ct.scale == 0) + { + ret = parm[0]->data()->getIntVal(row, isNull); + break; + } + + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (d.scale > 0) + { + if (d.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) << " with scale " + << (int)d.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + ret = d.toUInt64Ceil(); + } + } + break; + + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + { + ret = (parm[0]->data()->getUintVal(row, isNull)); + } + break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + ret = (uint64_t)ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case CalpontSystemCatalog::LONGDOUBLE: + { + ret = (uint64_t)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret = (uint64_t)ceil(strtod(str.c_str(), 0)); + } + break; + + case CalpontSystemCatalog::DATE: + { + // For some reason, MDB doesn't return this as a date, + // but datetime is returned as a datetime. Expect + // this to change in the future. + Date d(parm[0]->data()->getDateIntVal(row, isNull)); + if (!isNull) + ret = d.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::DATETIME: + { + ret = parm[0]->data()->getDatetimeIntVal(row, isNull); + } + break; + + case CalpontSystemCatalog::TIMESTAMP: + { + ret = parm[0]->data()->getTimestampIntVal(row, isNull); + } + break; + + case CalpontSystemCatalog::TIME: + { + ret = parm[0]->data()->getTimeIntVal(row, isNull); + } + break; + + default: + { + std::ostringstream oss; + oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) << " is not supported"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return ret; } - -double Func_ceil::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_ceil::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - double ret = 0.0; + double ret = 0.0; - if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || - op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || - op_ct.colDataType == CalpontSystemCatalog::FLOAT || - op_ct.colDataType == CalpontSystemCatalog::UFLOAT) - { - ret = ceil(parm[0]->data()->getDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || - op_ct.colDataType == CalpontSystemCatalog::CHAR || - op_ct.colDataType == CalpontSystemCatalog::TEXT) - { - const string& str = parm[0]->data()->getStrVal(row, isNull); + if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || + op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || + op_ct.colDataType == CalpontSystemCatalog::FLOAT || op_ct.colDataType == CalpontSystemCatalog::UFLOAT) + { + ret = ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || + op_ct.colDataType == CalpontSystemCatalog::CHAR || op_ct.colDataType == CalpontSystemCatalog::TEXT) + { + const string& str = parm[0]->data()->getStrVal(row, isNull); - if (!isNull) - ret = ceil(strtod(str.c_str(), 0)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) - { - ret = (double)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || - op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) - { - IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + if (!isNull) + ret = ceil(strtod(str.c_str(), 0)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + ret = (double)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); - if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - ret = static_cast(tmp.toTSInt128()); - } - else - { - ret = (double) tmp.value; - } + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = static_cast(tmp.toTSInt128()); } else { - if (isUnsigned(op_ct.colDataType)) - { - ret = (double) getUintVal(row, parm, isNull, op_ct); - } - else - { - ret = (double) getIntVal(row, parm, isNull, op_ct); - } + ret = (double)tmp.value; } - - return ret; -} - -long double Func_ceil::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) -{ - long double ret = 0.0; - - if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + } + else + { + if (isUnsigned(op_ct.colDataType)) { - ret = ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || - op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || - op_ct.colDataType == CalpontSystemCatalog::FLOAT || - op_ct.colDataType == CalpontSystemCatalog::UFLOAT) - { - ret = ceil(parm[0]->data()->getDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || - op_ct.colDataType == CalpontSystemCatalog::CHAR || - op_ct.colDataType == CalpontSystemCatalog::TEXT) - { - const string& str = parm[0]->data()->getStrVal(row, isNull); - - if (!isNull) - ret = ceil(strtod(str.c_str(), 0)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || - op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) - { - IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); - - if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - ret = static_cast(tmp.toTSInt128()); - } - else - { - ret = (long double) tmp.value; - } + ret = (double)getUintVal(row, parm, isNull, op_ct); } else { - if (isUnsigned(op_ct.colDataType)) - { - ret = (double) getUintVal(row, parm, isNull, op_ct); - } - else - { - ret = (double) getIntVal(row, parm, isNull, op_ct); - } + ret = (double)getIntVal(row, parm, isNull, op_ct); } + } - return ret; + return ret; } - -string Func_ceil::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +long double Func_ceil::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - char tmp[512] = {'\0'}; + long double ret = 0.0; - if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || - op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || - op_ct.colDataType == CalpontSystemCatalog::FLOAT || - op_ct.colDataType == CalpontSystemCatalog::UFLOAT || - op_ct.colDataType == CalpontSystemCatalog::VARCHAR || - op_ct.colDataType == CalpontSystemCatalog::CHAR || - op_ct.colDataType == CalpontSystemCatalog::TEXT) + if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + ret = ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || + op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || + op_ct.colDataType == CalpontSystemCatalog::FLOAT || + op_ct.colDataType == CalpontSystemCatalog::UFLOAT) + { + ret = ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || + op_ct.colDataType == CalpontSystemCatalog::CHAR || op_ct.colDataType == CalpontSystemCatalog::TEXT) + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret = ceil(strtod(str.c_str(), 0)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) { - snprintf(tmp, 511, "%f", getDoubleVal(row, parm, isNull, op_ct)); - - // remove the decimals in the oss string. - char* d = tmp; - - while ((*d != '.') && (*d != '\0')) - d++; - - *d = '\0'; + ret = static_cast(tmp.toTSInt128()); } - else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + else { - snprintf(tmp, 511, "%Lf", getLongDoubleVal(row, parm, isNull, op_ct)); - - // remove the decimals in the oss string. - char* d = tmp; - - while ((*d != '.') && (*d != '\0')) - d++; - - *d = '\0'; + ret = (long double)tmp.value; } - else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || - op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + } + else + { + if (isUnsigned(op_ct.colDataType)) { - IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); - - if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - return d.toString(true); - } - else - { - return d.toString(); - } + ret = (double)getUintVal(row, parm, isNull, op_ct); } - else if (isUnsigned(op_ct.colDataType)) + else { + ret = (double)getIntVal(row, parm, isNull, op_ct); + } + } + + return ret; +} + +string Func_ceil::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) +{ + char tmp[512] = {'\0'}; + + if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || + op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || + op_ct.colDataType == CalpontSystemCatalog::FLOAT || op_ct.colDataType == CalpontSystemCatalog::UFLOAT || + op_ct.colDataType == CalpontSystemCatalog::VARCHAR || op_ct.colDataType == CalpontSystemCatalog::CHAR || + op_ct.colDataType == CalpontSystemCatalog::TEXT) + { + snprintf(tmp, 511, "%f", getDoubleVal(row, parm, isNull, op_ct)); + + // remove the decimals in the oss string. + char* d = tmp; + + while ((*d != '.') && (*d != '\0')) + d++; + + *d = '\0'; + } + else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + snprintf(tmp, 511, "%Lf", getLongDoubleVal(row, parm, isNull, op_ct)); + + // remove the decimals in the oss string. + char* d = tmp; + + while ((*d != '.') && (*d != '\0')) + d++; + + *d = '\0'; + } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + return d.toString(true); + } + else + { + return d.toString(); + } + } + else if (isUnsigned(op_ct.colDataType)) + { #ifndef __LP64__ - snprintf(tmp, 511, "%llu", getUintVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%llu", getUintVal(row, parm, isNull, op_ct)); #else - snprintf(tmp, 511, "%lu", getUintVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%lu", getUintVal(row, parm, isNull, op_ct)); #endif - } - else - { + } + else + { #ifndef __LP64__ - snprintf(tmp, 511, "%lld", getIntVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%lld", getIntVal(row, parm, isNull, op_ct)); #else - snprintf(tmp, 511, "%ld", getIntVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%ld", getIntVal(row, parm, isNull, op_ct)); #endif - } + } - return string(tmp); + return string(tmp); } - -IDB_Decimal Func_ceil::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +IDB_Decimal Func_ceil::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - // Questionable approach. I believe we should use pointer here - // and call an appropriate ctor - IDB_Decimal ret; + // Questionable approach. I believe we should use pointer here + // and call an appropriate ctor + IDB_Decimal ret; - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - ret.value = parm[0]->data()->getIntVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - ret = parm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - // negative scale is not supported by CNX yet - if (ret.scale > 0) - { - if (ret.scale > datatypes::INT128MAXPRECISION) - { - std::ostringstream oss; - oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) ret.scale << " is beyond supported scale"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - return ret.ceil(); - } - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - ret.value = (int64_t) ceil(parm[0]->data()->getDoubleVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - ret.value = (int64_t) ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& str = parm[0]->data()->getStrVal(row, isNull); - - if (!isNull) - ret.value = (int64_t) ceil(strtod(str.c_str(), 0)); - } - break; - - case CalpontSystemCatalog::DATE: - { - Date d (parm[0]->data()->getDateIntVal(row, isNull)); - - if (!isNull) - ret.value = d.convertToMySQLint(); - } - break; - - case CalpontSystemCatalog::DATETIME: - { - DateTime dt(parm[0]->data()->getDatetimeIntVal(row, isNull)); - - if (!isNull) - ret.value = dt.convertToMySQLint(); - } - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); - - if (!isNull) - ret.value = dt.convertToMySQLint(timeZone()); - } - break; - - case CalpontSystemCatalog::TIME: - { - Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); - - if (!isNull) - ret.value = dt.convertToMySQLint(); - } - break; - - default: - { - std::ostringstream oss; - oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) - << " is not supported"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + ret.value = parm[0]->data()->getIntVal(row, isNull); } + break; - ret.scale = 0; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull); - return ret; + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (ret.scale > 0) + { + if (ret.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) << " with scale " + << (int)ret.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + return ret.ceil(); + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret.value = (int64_t)ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret.value = (int64_t)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret.value = (int64_t)ceil(strtod(str.c_str(), 0)); + } + break; + + case CalpontSystemCatalog::DATE: + { + Date d(parm[0]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + ret.value = d.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::DATETIME: + { + DateTime dt(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(timeZone()); + } + break; + + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(); + } + break; + + default: + { + std::ostringstream oss; + oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) << " is not supported"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + ret.scale = 0; + + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index 46c79d82b..7ead2b947 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_char.cpp 3931 2013-06-21 20:17:28Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_char.cpp 3931 2013-06-21 20:17:28Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,148 +38,144 @@ using namespace rowgroup; #include "errorids.h" using namespace logging; -#include // min_intXstore() +#include // min_intXstore() #include "vlarray.h" namespace { - inline size_t getChar(int32_t num, char*& buf) { - char tmp[4]; - size_t numBytes = 0; - if (num & 0xFF000000L) - { - mi_int4store(tmp, num); - numBytes = 4; - } - else if (num & 0xFF0000L) - { - mi_int3store(tmp, num); - numBytes = 3; - } - else if (num & 0xFF00L) - { - mi_int2store(tmp, num); - numBytes = 2; - } - else - { - *((int8_t*)buf) = num; - ++ buf; - return 1; - } - memcpy(buf, tmp, numBytes); - buf += numBytes; - return numBytes; + char tmp[4]; + size_t numBytes = 0; + if (num & 0xFF000000L) + { + mi_int4store(tmp, num); + numBytes = 4; + } + else if (num & 0xFF0000L) + { + mi_int3store(tmp, num); + numBytes = 3; + } + else if (num & 0xFF00L) + { + mi_int2store(tmp, num); + numBytes = 2; + } + else + { + *((int8_t*)buf) = num; + ++buf; + return 1; + } + memcpy(buf, tmp, numBytes); + buf += numBytes; + return numBytes; } -} +} // namespace namespace funcexp { -CalpontSystemCatalog::ColType Func_char::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_char::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -string Func_char::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +string Func_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - const int BUF_SIZE = 4 * parm.size(); - utils::VLArray buf(BUF_SIZE); - buf[0]= 0; - char* pBuf = buf; - CHARSET_INFO* cs = ct.getCharset(); - int32_t value = 0; - int32_t numBytes = 0; - for (uint32_t i = 0; i < parm.size(); ++i) + const int BUF_SIZE = 4 * parm.size(); + utils::VLArray buf(BUF_SIZE); + buf[0] = 0; + char* pBuf = buf; + CHARSET_INFO* cs = ct.getCharset(); + int32_t value = 0; + int32_t numBytes = 0; + for (uint32_t i = 0; i < parm.size(); ++i) + { + ReturnedColumn* rc = (ReturnedColumn*)parm[i]->data(); + + switch (rc->resultType().colDataType) { - ReturnedColumn* rc = (ReturnedColumn*)parm[i]->data(); - - switch (rc->resultType().colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - value = rc->getIntVal(row, isNull); - } - break; + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + value = rc->getIntVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - double vf = std::round(rc->getDoubleVal(row, isNull)); - value = (int32_t)vf; - } - break; + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + double vf = std::round(rc->getDoubleVal(row, isNull)); + value = (int32_t)vf; + } + break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - value = static_cast(rc->getDecimalVal(row, isNull).toSInt64Round()); - } - break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + value = static_cast(rc->getDecimalVal(row, isNull).toSInt64Round()); + } + break; - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - continue; // Dates are ignored - } - break; + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + continue; // Dates are ignored + } + break; - default: - { - value = 0; - } - } - - if (isNull) - continue; - - numBytes += getChar(value, pBuf); + default: + { + value = 0; + } } - isNull = false; - /* Check whether we got a well-formed string */ - MY_STRCOPY_STATUS status; - int32_t actualBytes = cs->well_formed_char_length(buf, buf + numBytes, numBytes, &status); + if (isNull) + continue; - if (UNLIKELY(actualBytes < numBytes)) - { - numBytes = actualBytes; - ostringstream os; - os << "Invalid character string for " << cs->cs_name.str << ": value = " << hex << buf + actualBytes; - logging::Message::Args args; - logging::Message message(9); - args.add(os.str()); - logging::LoggingID logid(28); // Shows as PrimProc, which may not be correct in all cases - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - // TODO: push warning to client - } - std::string ret(buf, numBytes); - return ret; + numBytes += getChar(value, pBuf); + } + + isNull = false; + /* Check whether we got a well-formed string */ + MY_STRCOPY_STATUS status; + int32_t actualBytes = cs->well_formed_char_length(buf, buf + numBytes, numBytes, &status); + + if (UNLIKELY(actualBytes < numBytes)) + { + numBytes = actualBytes; + ostringstream os; + os << "Invalid character string for " << cs->cs_name.str << ": value = " << hex << buf + actualBytes; + logging::Message::Args args; + logging::Message message(9); + args.add(os.str()); + logging::LoggingID logid(28); // Shows as PrimProc, which may not be correct in all cases + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + // TODO: push warning to client + } + std::string ret(buf, numBytes); + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index bb322241a..749914188 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_char_length.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_char_length.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,89 +38,87 @@ using namespace execplan; #include "idberrorinfo.h" #include "errorids.h" - using namespace logging; namespace funcexp { - -CalpontSystemCatalog::ColType Func_char_length::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_char_length::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return fp[0]->data()->resultType(); + return fp[0]->data()->resultType(); } -int64_t Func_char_length::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; + CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; - switch (type) + switch (type) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - const string& tstr = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - return 0; - const char* b = tstr.c_str(); - const char* e = tstr.c_str() + tstr.length(); - return (int64_t)parm[0]->data()->resultType().getCharset()->numchars(b, e); - } - - case execplan::CalpontSystemCatalog::DATE: - { - string date = dataconvert::DataConvert::dateToString(parm[0]->data()->getDateIntVal(row, isNull)); - return (int64_t)date.size(); - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - string date = dataconvert::DataConvert::datetimeToString(parm[0]->data()->getDatetimeIntVal(row, isNull)); - return (int64_t)date.size(); - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - return (int64_t)date.size(); - } - - case execplan::CalpontSystemCatalog::TIME: - { - string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); - return (int64_t)date.size(); - } - - default: - { - std::ostringstream oss; - oss << "char_length: datatype of " << execplan::colDataTypeToString(type); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + const string& tstr = parm[0]->data()->getStrVal(row, isNull); + if (isNull) + return 0; + const char* b = tstr.c_str(); + const char* e = tstr.c_str() + tstr.length(); + return (int64_t)parm[0]->data()->resultType().getCharset()->numchars(b, e); } - return 0; + case execplan::CalpontSystemCatalog::DATE: + { + string date = dataconvert::DataConvert::dateToString(parm[0]->data()->getDateIntVal(row, isNull)); + return (int64_t)date.size(); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + string date = + dataconvert::DataConvert::datetimeToString(parm[0]->data()->getDatetimeIntVal(row, isNull)); + return (int64_t)date.size(); + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string date = dataconvert::DataConvert::timestampToString( + parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + return (int64_t)date.size(); + } + + case execplan::CalpontSystemCatalog::TIME: + { + string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); + return (int64_t)date.size(); + } + + default: + { + std::ostringstream oss; + oss << "char_length: datatype of " << execplan::colDataTypeToString(type); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return 0; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 1f46a15b3..ebabde050 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_coalesce.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_coalesce.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -39,229 +39,209 @@ using namespace dataconvert; namespace funcexp { -CalpontSystemCatalog::ColType Func_coalesce::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_coalesce::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_coalesce::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_coalesce::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; + int64_t val = 0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getIntVal(row, isNull); + + if (isNull) { - val = parm[i]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return val; + isNull = false; + continue; } - isNull = true; return val; + } + + isNull = true; + return val; } -string Func_coalesce::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_coalesce::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - string val; + string val; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getStrVal(row, isNull); + + if (isNull) { - val = parm[i]->data()->getStrVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return val; + isNull = false; + continue; } - isNull = true; - return ""; + return val; + } + + isNull = true; + return ""; } -int32_t Func_coalesce::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_coalesce::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t val = 0; + int64_t val = 0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getDateIntVal(row, isNull); + + if (isNull) { - val = parm[i]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return val; + isNull = false; + continue; } - isNull = true; return val; + } + + isNull = true; + return val; } -int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - int64_t val = 0; + int64_t val = 0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) { - val = parm[i]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return val; + isNull = false; + continue; } - isNull = true; return val; + } + + isNull = true; + return val; } -int64_t Func_coalesce::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_coalesce::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - int64_t val = 0; + int64_t val = 0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimestampIntVal(row, isNull); + + if (isNull) { - val = parm[i]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return val; + isNull = false; + continue; } - isNull = true; return val; + } + + isNull = true; + return val; } -int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t val = 0; + int64_t val = 0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimeIntVal(row, isNull); + + if (isNull) { - val = parm[i]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return val; + isNull = false; + continue; } - isNull = true; return val; + } + + isNull = true; + return val; } -double Func_coalesce::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_coalesce::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - double d = 0.0; + double d = 0.0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + d = parm[i]->data()->getDoubleVal(row, isNull); + + if (isNull) { - d = parm[i]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return d; + isNull = false; + continue; } - isNull = true; return d; + } + + isNull = true; + return d; } - -long double Func_coalesce::getLongDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +long double Func_coalesce::getLongDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - long double d = 0.0; + long double d = 0.0; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + d = parm[i]->data()->getLongDoubleVal(row, isNull); + + if (isNull) { - d = parm[i]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return d; + isNull = false; + continue; } - isNull = true; return d; + } + + isNull = true; + return d; } - -execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - IDB_Decimal d; + IDB_Decimal d; - for (uint32_t i = 0; i < parm.size(); i++) + for (uint32_t i = 0; i < parm.size(); i++) + { + d = parm[i]->data()->getDecimalVal(row, isNull); + + if (isNull) { - d = parm[i]->data()->getDecimalVal(row, isNull); - - if (isNull) - { - isNull = false; - continue; - } - - return d; + isNull = false; + continue; } - isNull = true; return d; + } + + isNull = true; + return d; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat.cpp b/utils/funcexp/func_concat.cpp index 74a43ca28..73b2a2b2d 100644 --- a/utils/funcexp/func_concat.cpp +++ b/utils/funcexp/func_concat.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_concat.cpp 3918 2013-06-19 13:54:34Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_concat.cpp 3918 2013-06-19 13:54:34Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -36,37 +36,33 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_concat::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_concat::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - // Returns the string that results from concatenating the arguments. // concat() returns NULL if any argument is NULL. // -string Func_concat::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +string Func_concat::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - string ret; - string tmp; - stringValue(parm[0], row, isNull, ret); - - // TODO: do a better job of cutting down the number re-allocations. - // look at Item_func_concat::realloc_result for ideas and use - // std::string:resize() appropriatly. - for ( unsigned int id = 1 ; id < parm.size() ; id++) - { - stringValue(parm[id], row, isNull, tmp); - ret.append(tmp); - } + string ret; + string tmp; + stringValue(parm[0], row, isNull, ret); - return ret; + // TODO: do a better job of cutting down the number re-allocations. + // look at Item_func_concat::realloc_result for ideas and use + // std::string:resize() appropriatly. + for (unsigned int id = 1; id < parm.size(); id++) + { + stringValue(parm[id], row, isNull, tmp); + ret.append(tmp); + } + + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat_oracle.cpp b/utils/funcexp/func_concat_oracle.cpp index 06f7ed254..28713a107 100644 --- a/utils/funcexp/func_concat_oracle.cpp +++ b/utils/funcexp/func_concat_oracle.cpp @@ -30,48 +30,45 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_concat_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_concat_oracle::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - // Returns the string that results from concatenating the arguments. // concat_oracle() returns NULL all arguments are NULL. // single arguments null is replaced by "". // -string Func_concat_oracle::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +string Func_concat_oracle::getStrVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - string ret; - string tmp; - stringValue(parm[0], row, isNull, ret); - // Oracle Mode should replace NULL with "" unless all values are NULL + string ret; + string tmp; + stringValue(parm[0], row, isNull, ret); + // Oracle Mode should replace NULL with "" unless all values are NULL + if (isNull) + { + ret = ""; + isNull = false; + } + // TODO: do a better job of cutting down the number re-allocations. + // look at Item_func_concat::realloc_result for ideas and use + // std::string:resize() appropriatly. + for (unsigned int id = 1; id < parm.size(); id++) + { + stringValue(parm[id], row, isNull, tmp); if (isNull) { - ret = ""; - isNull = false; - } - // TODO: do a better job of cutting down the number re-allocations. - // look at Item_func_concat::realloc_result for ideas and use - // std::string:resize() appropriatly. - for ( unsigned int id = 1 ; id < parm.size() ; id++) - { - stringValue(parm[id], row, isNull, tmp); - if (isNull) - { - tmp = ""; - isNull = false; - } - ret.append(tmp); + tmp = ""; + isNull = false; } + ret.append(tmp); + } - return ret; + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat_ws.cpp b/utils/funcexp/func_concat_ws.cpp index eb89c1b31..2c07c5b0f 100644 --- a/utils/funcexp/func_concat_ws.cpp +++ b/utils/funcexp/func_concat_ws.cpp @@ -16,42 +16,38 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_concat_ws.cpp 3714 2013-04-18 16:29:25Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_concat_ws.cpp 3714 2013-04-18 16:29:25Z bpaul $ + * + * + ****************************************************************************/ #include using namespace std; #include "functor_str.h" #include "functioncolumn.h" -#include "utils_utf8.h" // idb_mbstowcs() +#include "utils_utf8.h" // idb_mbstowcs() using namespace execplan; #include "rowgroup.h" using namespace rowgroup; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_concat_ws::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_concat_ws::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -string Func_concat_ws::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_concat_ws::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& type) { - string delim; - stringValue(parm[0], row, isNull, delim); - if (isNull) - return ""; + string delim; + stringValue(parm[0], row, isNull, delim); + if (isNull) + return ""; // TODO: I don't think we need wide chars here. // Concatenation works without see Server implementation. @@ -97,33 +93,32 @@ string Func_concat_ws::getStrVal(Row& row, delete [] wcbuf; return ret; #endif - string str; - string tmp; - for ( uint32_t i = 1 ; i < parm.size() ; i++) + string str; + string tmp; + for (uint32_t i = 1; i < parm.size(); i++) + { + stringValue(parm[i], row, isNull, tmp); + if (isNull) { - stringValue(parm[i], row, isNull, tmp); - if (isNull) - { - isNull = false; - continue; - } - - if (!str.empty()) - str += delim; - - // TODO: Work on string reallocation. Use std::string::resize() to - // grab larger chunks in some intellegent manner. - str += tmp; + isNull = false; + continue; } - if (str.empty()) - isNull = true; - else - isNull = false; + if (!str.empty()) + str += delim; - return str; + // TODO: Work on string reallocation. Use std::string::resize() to + // grab larger chunks in some intellegent manner. + str += tmp; + } + + if (str.empty()) + isNull = true; + else + isNull = false; + + return str; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_conv.cpp b/utils/funcexp/func_conv.cpp index 3afc13790..5f73f5ea4 100644 --- a/utils/funcexp/func_conv.cpp +++ b/utils/funcexp/func_conv.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_conv.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_conv.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,92 +38,93 @@ namespace { int64_t convStrToNum(const string& str, int base, bool unsignedFlag) { - int negative; - uint64_t cutoff, cutlim, i, j, save; - int overflow; + int negative; + uint64_t cutoff, cutlim, i, j, save; + int overflow; - // to skip the leading spaces. - for (i = 0; i < str.length() && str.c_str()[i] == ' '; i++) - {} + // to skip the leading spaces. + for (i = 0; i < str.length() && str.c_str()[i] == ' '; i++) + { + } - if (i == str.length()) - { - return 0L; - } + if (i == str.length()) + { + return 0L; + } - if (str.c_str()[i] == '-') - { - negative = 1; - ++i; - } - else if (str.c_str()[i] == '+') - { - negative = 0; - ++i; - } + if (str.c_str()[i] == '-') + { + negative = 1; + ++i; + } + else if (str.c_str()[i] == '+') + { + negative = 0; + ++i; + } + else + negative = 0; + + save = i; + + cutoff = (~(uint64_t)0) / (uint64_t)base; + cutlim = (uint32_t)((~(uint64_t)0) % (uint64_t)base); + + overflow = 0; + j = 0; + + for (; i < str.length(); i++) + { + unsigned char c = str.c_str()[i]; + + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'A' && c <= 'Z') + c = c - 'A' + 10; + else if (c >= 'a' && c <= 'z') + c = c - 'a' + 10; else - negative = 0; + break; - save = i; + if (c >= base) + break; - cutoff = (~(uint64_t) 0) / (uint64_t) base; - cutlim = (uint32_t) ((~(uint64_t) 0) % (uint64_t) base); - - overflow = 0; - j = 0; - - for (; i < str.length(); i++) + if (j > cutoff || (j == cutoff && c > cutlim)) + overflow = 1; + else { - unsigned char c = str.c_str()[i]; - - if (c >= '0' && c <= '9') - c -= '0'; - else if (c >= 'A' && c <= 'Z') - c = c - 'A' + 10; - else if (c >= 'a' && c <= 'z') - c = c - 'a' + 10; - else - break; - - if (c >= base) - break; - - if (j > cutoff || (j == cutoff && c > cutlim)) - overflow = 1; - else - { - j *= (uint64_t) base; - j += c; - } + j *= (uint64_t)base; + j += c; } + } - if (i == save) - return 0L; + if (i == save) + return 0L; - if (!unsignedFlag) + if (!unsignedFlag) + { + if (negative) { - if (negative) - { - if (j > (uint64_t) numeric_limits::min()) - overflow = 1; - } - else if (j > (uint64_t) numeric_limits::max()) - { - overflow = 1; - } + if (j > (uint64_t)numeric_limits::min()) + overflow = 1; } - - if (overflow) + else if (j > (uint64_t)numeric_limits::max()) { - if (unsignedFlag) - return (~(uint64_t) 0); - - return negative ? numeric_limits::min() : numeric_limits::max(); + overflow = 1; } + } - return (negative ? -((int64_t) j) : (int64_t) j); -} + if (overflow) + { + if (unsignedFlag) + return (~(uint64_t)0); + + return negative ? numeric_limits::min() : numeric_limits::max(); + } + + return (negative ? -((int64_t)j) : (int64_t)j); } +} // namespace namespace funcexp { @@ -131,61 +132,61 @@ namespace helpers { const char* convNumToStr(int64_t val, char* dst, int radix) { - if (radix == 16 || radix == -16) - sprintf(dst, "%llX", (long long)val); + if (radix == 16 || radix == -16) + sprintf(dst, "%llX", (long long)val); - else if (radix == 8 || radix == -8) - sprintf(dst, "%llo", (long long)val); + else if (radix == 8 || radix == -8) + sprintf(dst, "%llo", (long long)val); - else if (radix == 10) + else if (radix == 10) + { + sprintf(dst, "%llu", (unsigned long long)val); + } + else if (radix == -10) + sprintf(dst, "%lld", (long long)val); + + else if (radix == 2 || radix == -2) + { + char tmp[65]; + char* ptr = &tmp[64]; + *ptr-- = 0; + + for (int i = 0; i < 64; i++) { - sprintf(dst, "%llu", (unsigned long long)val); - } - else if (radix == -10) - sprintf(dst, "%lld", (long long)val); + if (val & 1) + *ptr-- = '1'; + else + *ptr-- = '0'; - else if (radix == 2 || radix == -2) + val >>= 1; + } + + ptr = strchr(tmp, '1'); + + if (ptr == 0) + strcpy(dst, &tmp[63]); + else + strcpy(dst, ptr); + } + else if (radix == 4 || radix == -4) + { + char tmp[33]; + char* ptr = &tmp[32]; + *ptr-- = 0; + + for (int i = 0; i < 32; i++) { - char tmp[65]; - char* ptr = &tmp[64]; - *ptr-- = 0; - - for (int i = 0; i < 64; i++) - { - if (val & 1) - *ptr-- = '1'; - else - *ptr-- = '0'; - - val >>= 1; - } - - ptr = strchr(tmp, '1'); - - if (ptr == 0) - strcpy(dst, &tmp[63]); - else - strcpy(dst, ptr); + *ptr-- = '0' + (val & 3); + val >>= 2; } - else if (radix == 4 || radix == -4) - { - char tmp[33]; - char* ptr = &tmp[32]; - *ptr-- = 0; - for (int i = 0; i < 32; i++) - { - *ptr-- = '0' + (val & 3); - val >>= 2; - } + ptr = strpbrk(tmp, "123"); - ptr = strpbrk(tmp, "123"); - - if (ptr == 0) - strcpy(dst, &tmp[31]); - else - strcpy(dst, ptr); - } + if (ptr == 0) + strcpy(dst, &tmp[31]); + else + strcpy(dst, ptr); + } #if 0 else if (radix == 8 || radix == -8) @@ -234,76 +235,73 @@ const char* convNumToStr(int64_t val, char* dst, int radix) } #endif - else if (radix == 32 || radix == -32) + else if (radix == 32 || radix == -32) + { + char tmp[14]; + char* ptr = &tmp[13]; + *ptr-- = 0; + + for (int i = 0; i < 13; i++) { - char tmp[14]; - char* ptr = &tmp[13]; - *ptr-- = 0; + int v = val & 0x1f; - for (int i = 0; i < 13; i++) - { - int v = val & 0x1f; + if (v > 9) + *ptr-- = 'A' + v - 10; + else + *ptr-- = '0' + v; - if (v > 9) - *ptr-- = 'A' + v - 10; - else - *ptr-- = '0' + v; - - val >>= 5; - } - - ptr = strpbrk(tmp, "123456789ABCDEFGHIJKLMNOPQRSTUV"); - - if (ptr == 0) - strcpy(dst, &tmp[12]); - else - strcpy(dst, ptr); + val >>= 5; } + + ptr = strpbrk(tmp, "123456789ABCDEFGHIJKLMNOPQRSTUV"); + + if (ptr == 0) + strcpy(dst, &tmp[12]); else - *dst = 0; + strcpy(dst, ptr); + } + else + *dst = 0; - return dst; + return dst; } -} //namespace funcexp::helpers +} // namespace helpers -CalpontSystemCatalog::ColType Func_conv::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_conv::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -string Func_conv::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_conv::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - const string& res = parm[0]->data()->getStrVal(row, isNull); - string str; - char ans[65]; - int64_t dec; - int64_t from_base = parm[1]->data()->getIntVal(row, isNull); - int64_t to_base = parm[2]->data()->getIntVal(row, isNull); + const string& res = parm[0]->data()->getStrVal(row, isNull); + string str; + char ans[65]; + int64_t dec; + int64_t from_base = parm[1]->data()->getIntVal(row, isNull); + int64_t to_base = parm[2]->data()->getIntVal(row, isNull); - if (isNull || abs(static_cast(to_base)) > 36 || abs(static_cast(to_base)) < 2 || - abs(static_cast(from_base)) > 36 || abs(static_cast(from_base)) < 2 || !(res.length())) - { - isNull = true; - return ""; - } + if (isNull || abs(static_cast(to_base)) > 36 || abs(static_cast(to_base)) < 2 || + abs(static_cast(from_base)) > 36 || abs(static_cast(from_base)) < 2 || !(res.length())) + { + isNull = true; + return ""; + } - if (from_base < 0) - dec = convStrToNum(res, -from_base, false); - else - dec = (int64_t) convStrToNum( res, from_base, true); + if (from_base < 0) + dec = convStrToNum(res, -from_base, false); + else + dec = (int64_t)convStrToNum(res, from_base, true); - str = helpers::convNumToStr(dec, ans, to_base); + str = helpers::convNumToStr(dec, ans, to_base); - isNull = str.empty(); + isNull = str.empty(); - return str; + return str; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_convert_tz.cpp b/utils/funcexp/func_convert_tz.cpp index c5264a1ff..424b7494c 100644 --- a/utils/funcexp/func_convert_tz.cpp +++ b/utils/funcexp/func_convert_tz.cpp @@ -40,192 +40,180 @@ using namespace logging; namespace funcexp { - // need my_tz_find in dataconvert -CalpontSystemCatalog::ColType Func_convert_tz::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_convert_tz::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_convert_tz::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +int64_t Func_convert_tz::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::intToDatetime(getIntVal(row, parm, isNull, ct)); + return dataconvert::DataConvert::intToDatetime(getIntVal(row, parm, isNull, ct)); } -int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - messageqcpp::ByteStream bs; - bool bFromTz = false; - bool bToTz = false; - dataconvert::TIME_ZONE_INFO tzinfo; - std::vector ats; - std::vector types; - std::vector ttis; + messageqcpp::ByteStream bs; + bool bFromTz = false; + bool bToTz = false; + dataconvert::TIME_ZONE_INFO tzinfo; + std::vector ats; + std::vector types; + std::vector ttis; #ifdef ABBR_ARE_USED - std::vector chars; + std::vector chars; #endif - std::vector lsis; - std::vector revts; - std::vector revtis; - std::vector fallback_tti; + std::vector lsis; + std::vector revts; + std::vector revtis; + std::vector fallback_tti; - int64_t seconds = 0; - uint32_t err_code = 0; - int64_t datetime_start = parm[0]->data()->getDatetimeIntVal(row, isNull); - int64_t rtn = 0; + int64_t seconds = 0; + uint32_t err_code = 0; + int64_t datetime_start = parm[0]->data()->getDatetimeIntVal(row, isNull); + int64_t rtn = 0; - if (isNull) - return 0; -// Adding a zero date to a time is always NULL - if (datetime_start == 0) + if (isNull) + return 0; + // Adding a zero date to a time is always NULL + if (datetime_start == 0) + { + isNull = true; + return -1; + } + + const string& from_tz = parm[1]->data()->getStrVal(row, isNull); + if (isNull) + { + return 0; + } + const string& to_tz = parm[2]->data()->getStrVal(row, isNull); + if (isNull) + { + return 0; + } + + cout << "from " << from_tz << endl; + cout << "to " << to_tz << endl; + + const string& serialized_from_tzinfo = parm[3]->data()->getStrVal(row, isNull); + const string& serialized_to_tzinfo = parm[4]->data()->getStrVal(row, isNull); + + if (!serialized_from_tzinfo.empty()) + { + bs.append((uint8_t*)serialized_from_tzinfo.c_str(), serialized_from_tzinfo.length()); + dataconvert::unserializeTimezoneInfo(bs, &tzinfo); + deserializeInlineVector(bs, ats); + tzinfo.ats = ats.data(); + deserializeInlineVector(bs, types); + tzinfo.types = types.data(); + deserializeInlineVector(bs, ttis); + tzinfo.ttis = ttis.data(); +#ifdef ABBR_ARE_USED + deserializeInlineVector(bs_fromTZ, chars); + tzinfo.chars = chars.data(); +#endif + deserializeInlineVector(bs, lsis); + tzinfo.lsis = lsis.data(); + deserializeInlineVector(bs, revts); + tzinfo.revts = revts.data(); + deserializeInlineVector(bs, revtis); + tzinfo.revtis = revtis.data(); + deserializeInlineVector(bs, fallback_tti); + tzinfo.fallback_tti = fallback_tti.data(); + bFromTz = true; + } + + dataconvert::DateTime datetime(datetime_start); + if (!dataconvert::DataConvert::isColumnDateTimeValid(datetime_start)) + return datetime_start; + + // int64_t result_datetime; + bool valid = true; + MySQLTime my_time_tmp, my_start_time; + my_time_tmp.reset(); + my_start_time.year = datetime.year; + my_start_time.month = datetime.month; + my_start_time.day = datetime.day; + my_start_time.hour = datetime.hour; + my_start_time.minute = datetime.minute; + my_start_time.second = datetime.second; + my_start_time.second_part = datetime.msecond; + + if (bFromTz) + { + seconds = dataconvert::TIME_to_gmt_sec(my_start_time, &tzinfo, &err_code); + if (err_code) { + return datetime.convertToMySQLint(); + } + } + else + { + seconds = dataconvert::mySQLTimeToGmtSec(my_start_time, from_tz, valid); + if (!valid) + { + if (seconds != 0) isNull = true; - return -1; + return datetime.convertToMySQLint(); } + } - const string& from_tz = parm[1]->data()->getStrVal(row, isNull); - if (isNull) - { - return 0; - } - const string& to_tz = parm[2]->data()->getStrVal(row, isNull); - if (isNull) - { - return 0; - } - - cout << "from " << from_tz << endl; - cout << "to " << to_tz << endl; - - const string& serialized_from_tzinfo = parm[3]->data()->getStrVal(row, isNull); - const string& serialized_to_tzinfo = parm[4]->data()->getStrVal(row, isNull); - - if (!serialized_from_tzinfo.empty()) - { - bs.append((uint8_t*)serialized_from_tzinfo.c_str(),serialized_from_tzinfo.length()); - dataconvert::unserializeTimezoneInfo(bs,&tzinfo); - deserializeInlineVector(bs, ats); - tzinfo.ats = ats.data(); - deserializeInlineVector(bs, types); - tzinfo.types = types.data(); - deserializeInlineVector(bs, ttis); - tzinfo.ttis = ttis.data(); + if (!serialized_to_tzinfo.empty()) + { + bs.reset(); + bs.append((uint8_t*)serialized_to_tzinfo.c_str(), serialized_to_tzinfo.length()); + dataconvert::unserializeTimezoneInfo(bs, &tzinfo); + deserializeInlineVector(bs, ats); + tzinfo.ats = ats.data(); + deserializeInlineVector(bs, types); + tzinfo.types = types.data(); + deserializeInlineVector(bs, ttis); + tzinfo.ttis = ttis.data(); #ifdef ABBR_ARE_USED - deserializeInlineVector(bs_fromTZ, chars); - tzinfo.chars = chars.data(); + deserializeInlineVector(bs_fromTZ, chars); + tzinfo.chars = chars.data(); #endif - deserializeInlineVector(bs, lsis); - tzinfo.lsis = lsis.data(); - deserializeInlineVector(bs, revts); - tzinfo.revts = revts.data(); - deserializeInlineVector(bs, revtis); - tzinfo.revtis = revtis.data(); - deserializeInlineVector(bs, fallback_tti); - tzinfo.fallback_tti = fallback_tti.data(); - bFromTz = true; - } + deserializeInlineVector(bs, lsis); + tzinfo.lsis = lsis.data(); + deserializeInlineVector(bs, revts); + tzinfo.revts = revts.data(); + deserializeInlineVector(bs, revtis); + tzinfo.revtis = revtis.data(); + deserializeInlineVector(bs, fallback_tti); + tzinfo.fallback_tti = fallback_tti.data(); + bToTz = true; + } + if (bToTz) + { + dataconvert::gmt_sec_to_TIME(&my_time_tmp, seconds, &tzinfo); + if (my_time_tmp.second == 60 || my_time_tmp.second == 61) + my_time_tmp.second = 59; + } + else + { + dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz); + } - dataconvert::DateTime datetime(datetime_start); - if (!dataconvert::DataConvert::isColumnDateTimeValid( datetime_start )) - return datetime_start; + dataconvert::DateTime result_datetime(my_time_tmp.year, my_time_tmp.month, my_time_tmp.day, + my_time_tmp.hour, my_time_tmp.minute, my_time_tmp.second, + my_time_tmp.second_part); - //int64_t result_datetime; - bool valid = true; - MySQLTime my_time_tmp,my_start_time; - my_time_tmp.reset(); - my_start_time.year = datetime.year; - my_start_time.month = datetime.month; - my_start_time.day = datetime.day; - my_start_time.hour = datetime.hour; - my_start_time.minute = datetime.minute; - my_start_time.second = datetime.second; - my_start_time.second_part = datetime.msecond; + if ((rtn = result_datetime.convertToMySQLint()) == 0) + isNull = true; - if (bFromTz) - { - seconds = dataconvert::TIME_to_gmt_sec(my_start_time,&tzinfo,&err_code); - if (err_code) - { - return datetime.convertToMySQLint(); - } - } - else - { - seconds = dataconvert::mySQLTimeToGmtSec(my_start_time,from_tz,valid); - if (!valid) - { - if (seconds != 0) - isNull = true; - return datetime.convertToMySQLint(); - } - } - - if (!serialized_to_tzinfo.empty()) - { - bs.reset(); - bs.append((uint8_t*)serialized_to_tzinfo.c_str(),serialized_to_tzinfo.length()); - dataconvert::unserializeTimezoneInfo(bs,&tzinfo); - deserializeInlineVector(bs, ats); - tzinfo.ats = ats.data(); - deserializeInlineVector(bs, types); - tzinfo.types = types.data(); - deserializeInlineVector(bs, ttis); - tzinfo.ttis = ttis.data(); -#ifdef ABBR_ARE_USED - deserializeInlineVector(bs_fromTZ, chars); - tzinfo.chars = chars.data(); -#endif - deserializeInlineVector(bs, lsis); - tzinfo.lsis = lsis.data(); - deserializeInlineVector(bs, revts); - tzinfo.revts = revts.data(); - deserializeInlineVector(bs, revtis); - tzinfo.revtis = revtis.data(); - deserializeInlineVector(bs, fallback_tti); - tzinfo.fallback_tti = fallback_tti.data(); - bToTz = true; - } - if (bToTz) - { - dataconvert::gmt_sec_to_TIME(&my_time_tmp, seconds, &tzinfo); - if (my_time_tmp.second == 60 || my_time_tmp.second == 61) - my_time_tmp.second= 59; - } - else - { - dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz); - } - - dataconvert::DateTime result_datetime(my_time_tmp.year, - my_time_tmp.month, - my_time_tmp.day, - my_time_tmp.hour, - my_time_tmp.minute, - my_time_tmp.second, - my_time_tmp.second_part); - - if ((rtn = result_datetime.convertToMySQLint()) == 0) - isNull = true; - - return rtn; + return rtn; } - -string Func_convert_tz::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +string Func_convert_tz::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct)); + return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct)); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_crc32.cpp b/utils/funcexp/func_crc32.cpp index 24f1e580f..8d728c111 100644 --- a/utils/funcexp/func_crc32.cpp +++ b/utils/funcexp/func_crc32.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_crc32.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_crc32.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,37 +38,30 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_crc32::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_crc32::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_crc32::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_crc32::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - unsigned crc = 0; - switch (parm.size()) - { - default: - isNull = true; - return 0; - case 1: - break; - case 2: - crc = static_cast(parm[0]->data()->getIntVal(row, isNull)); - if (isNull) - return 0; - } - const string& b = parm[parm.size() - 1]->data()->getStrVal(row, isNull); - if (isNull) + unsigned crc = 0; + switch (parm.size()) + { + default: isNull = true; return 0; + case 1: break; + case 2: + crc = static_cast(parm[0]->data()->getIntVal(row, isNull)); + if (isNull) return 0; - return crc32(crc, reinterpret_cast(b.data()), b.size()); + } + const string& b = parm[parm.size() - 1]->data()->getStrVal(row, isNull); + if (isNull) + return 0; + return crc32(crc, reinterpret_cast(b.data()), b.size()); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 76a50a90f..8ebbd2250 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_date.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_date.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,122 +38,115 @@ using namespace execplan; #include "errorids.h" using namespace logging; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_date::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_date::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_date::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; + CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; - string value = ""; + string value = ""; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (type) + switch (type) + { + case execplan::CalpontSystemCatalog::DATE: { - case execplan::CalpontSystemCatalog::DATE: - { - return parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - value = dataconvert::DataConvert::datetimeToString(val1); - value = value.substr(0, 10); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - value = dataconvert::DataConvert::timestampToString(val1, timeZone()); - value = value.substr(0, 10); - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - { - int64_t val; - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - aDateTime.hour = 0; - aDateTime.minute = 0; - aDateTime.second = 0; - aDateTime.msecond = 0; - if ((aTime.hour < 0) || (aTime.is_neg)) - { - aTime.hour = -abs(aTime.hour); - aTime.minute = -abs(aTime.minute); - aTime.second = -abs(aTime.second); - aTime.msecond = -abs(aTime.msecond); - } - val = addTime(aDateTime, aTime); - value = dataconvert::DataConvert::datetimeToString(val); - value = value.substr(0, 10); - break; - } - - - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - isNull = true; - return 0; - } - break; - - default: - { - std::ostringstream oss; - oss << "date: datatype of " << execplan::colDataTypeToString(type); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return parm[0]->data()->getDatetimeIntVal(row, isNull); + break; } - return dataconvert::DataConvert::datetimeToInt(value); + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + value = dataconvert::DataConvert::datetimeToString(val1); + value = value.substr(0, 10); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); + value = dataconvert::DataConvert::timestampToString(val1, timeZone()); + value = value.substr(0, 10); + break; + } + + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + { + int64_t val; + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + aDateTime.hour = 0; + aDateTime.minute = 0; + aDateTime.second = 0; + aDateTime.msecond = 0; + if ((aTime.hour < 0) || (aTime.is_neg)) + { + aTime.hour = -abs(aTime.hour); + aTime.minute = -abs(aTime.minute); + aTime.second = -abs(aTime.second); + aTime.msecond = -abs(aTime.msecond); + } + val = addTime(aDateTime, aTime); + value = dataconvert::DataConvert::datetimeToString(val); + value = value.substr(0, 10); + break; + } + + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + isNull = true; + return 0; + } + break; + + default: + { + std::ostringstream oss; + oss << "date: datatype of " << execplan::colDataTypeToString(type); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return dataconvert::DataConvert::datetimeToInt(value); } -string Func_date::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - const string& val = parm[0]->data()->getStrVal(row, isNull); + const string& val = parm[0]->data()->getStrVal(row, isNull); - return val.substr(0, 10); + return val.substr(0, 10); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 8c0e547fa..c98d54581 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_date_add.cpp 3905 2013-06-18 12:27:32Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_date_add.cpp 3905 2013-06-18 12:27:32Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -44,807 +44,776 @@ namespace funcexp { namespace helpers { - -uint64_t dateAdd( uint64_t time, const string& expr, IntervalColumn::interval_type unit, bool dateType, OpType funcType ) +uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_type unit, bool dateType, + OpType funcType) { - int array[10]; - int64_t array2[10]; + int array[10]; + int64_t array2[10]; - int month_length[13] = { 0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + int month_length[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - int32_t year = 0, - month = 0, - day = 0, - hour = 0, - monthSave = 0; + int32_t year = 0, month = 0, day = 0, hour = 0, monthSave = 0; - int64_t min = 0, - sec = 0, - msec = 0; + int64_t min = 0, sec = 0, msec = 0; - int32_t yearAdd = 0, - monthAdd = 0, - dayAdd = 0, - hourAdd = 0; + int32_t yearAdd = 0, monthAdd = 0, dayAdd = 0, hourAdd = 0; - int64_t minAdd = 0, - secAdd = 0, - msecAdd = 0; + int64_t minAdd = 0, secAdd = 0, msecAdd = 0; + if (dateType) + { + year = (uint32_t)((time >> 16) & 0xffff); + month = (uint32_t)((time >> 12) & 0xf); + day = (uint32_t)((time >> 6) & 0x3f); + } + else + { + year = (uint32_t)((time >> 48) & 0xffff); + month = (uint32_t)((time >> 44) & 0xf); + day = (uint32_t)((time >> 38) & 0x3f); + hour = (uint32_t)((time >> 32) & 0x3f); + min = (uint32_t)((time >> 26) & 0x3f); + sec = (uint32_t)((time >> 20) & 0x3f); + msec = (uint32_t)((time & 0xfffff)); + } - if (dateType) + monthSave = month; + + int index = -1; + + switch (unit) + { + case IntervalColumn::INTERVAL_MINUTE: + case IntervalColumn::INTERVAL_SECOND: + case IntervalColumn::INTERVAL_MICROSECOND: index = getNumbers(expr, array2, funcType); break; + + default: index = getNumbers(expr, array, funcType); break; + }; + + if (index <= 0) + throw runtime_error("expression type is not supported"); + + switch (unit) + { + case IntervalColumn::INTERVAL_YEAR: { - year = (uint32_t)((time >> 16) & 0xffff); - month = (uint32_t)((time >> 12) & 0xf); - day = (uint32_t)((time >> 6) & 0x3f); - } - else - { - year = (uint32_t)((time >> 48) & 0xffff); - month = (uint32_t)((time >> 44) & 0xf); - day = (uint32_t)((time >> 38) & 0x3f); - hour = (uint32_t)((time >> 32) & 0x3f); - min = (uint32_t)((time >> 26) & 0x3f); - sec = (uint32_t)((time >> 20) & 0x3f); - msec = (uint32_t)((time & 0xfffff)); + yearAdd = array[0]; + break; } - monthSave = month; - - int index = -1; - - switch ( unit ) + case IntervalColumn::INTERVAL_QUARTER: { - case IntervalColumn::INTERVAL_MINUTE: - case IntervalColumn::INTERVAL_SECOND: - case IntervalColumn::INTERVAL_MICROSECOND: - index = getNumbers( expr, array2, funcType); - break; - - default: - index = getNumbers( expr, array, funcType); - break; - }; - - - if ( index <= 0 ) - throw runtime_error("expression type is not supported"); - - switch ( unit ) - { - case IntervalColumn::INTERVAL_YEAR: - { - yearAdd = array[0]; - break; - } - - case IntervalColumn::INTERVAL_QUARTER: - { - monthAdd = array[0] * 3; - break; - } - - case IntervalColumn::INTERVAL_MONTH: - { - monthAdd = array[0]; - break; - } - - case IntervalColumn::INTERVAL_WEEK: - { - dayAdd = array[0] * 7; - break; - } - - case IntervalColumn::INTERVAL_DAY: - { - dayAdd = array[0]; - break; - } - - case IntervalColumn::INTERVAL_HOUR: - { - hourAdd = array[0]; - break; - } - - case IntervalColumn::INTERVAL_MINUTE: - { - minAdd = array2[0]; - break; - } - - case IntervalColumn::INTERVAL_SECOND: - { - secAdd = array2[0]; - break; - } - - case IntervalColumn::INTERVAL_MICROSECOND: - { - msecAdd = array2[0]; - break; - } - - case IntervalColumn::INTERVAL_YEAR_MONTH: - { - if ( index > 2 ) - { - return 0; - } - - if ( index == 1 ) - monthAdd = array[0]; - else - { - yearAdd = array[0]; - monthAdd = array[1]; - } - - break; - } - - case IntervalColumn::INTERVAL_DAY_HOUR: - { - if ( index > 2 ) - { - return 0; - } - - if ( index == 1 ) - hourAdd = array[0]; - else - { - dayAdd = array[0]; - hourAdd = array[1]; - } - - break; - } - - case IntervalColumn::INTERVAL_DAY_MINUTE: - { - if ( index > 3 ) - { - return 0; - } - - if ( index == 1 ) - minAdd = array[0]; - else - { - if ( index == 2 ) - { - hourAdd = array[0]; - minAdd = array[1]; - } - else - { - dayAdd = array[0]; - hourAdd = array[1]; - minAdd = array[2]; - } - } - - break; - } - - case IntervalColumn::INTERVAL_DAY_SECOND: - { - if ( index > 4 ) - { - return 0; - } - - if ( index == 1 ) - secAdd = array[0]; - else - { - if ( index == 2 ) - { - minAdd = array[0]; - secAdd = array[1]; - } - else - { - if ( index == 3 ) - { - hourAdd = array[0]; - minAdd = array[1]; - secAdd = array[2]; - } - else - { - dayAdd = array[0]; - hourAdd = array[1]; - minAdd = array[2]; - secAdd = array[3]; - } - } - } - - break; - } - - case IntervalColumn::INTERVAL_HOUR_MINUTE: - { - if ( index > 2 ) - { - return 0; - } - - if ( index == 1 ) - minAdd = array[0]; - else - { - hourAdd = array[0]; - minAdd = array[1]; - } - - break; - } - - case IntervalColumn::INTERVAL_HOUR_SECOND: - { - if ( index > 3 ) - { - return 0; - } - - if ( index == 1 ) - secAdd = array[0]; - else - { - if ( index == 2 ) - { - minAdd = array[0]; - secAdd = array[1]; - } - else - { - hourAdd = array[0]; - minAdd = array[1]; - secAdd = array[2]; - } - } - - break; - } - - case IntervalColumn::INTERVAL_MINUTE_SECOND: - { - if ( index > 2 ) - { - return 0; - } - - if ( index == 1 ) - secAdd = array[0]; - else - { - minAdd = array[0]; - secAdd = array[1]; - } - - break; - } - - case IntervalColumn::INTERVAL_DAY_MICROSECOND: - { - if ( index > 5 ) - { - return 0; - } - - if ( index == 1 ) - msecAdd = array[0]; - else - { - if ( index == 2 ) - { - secAdd = array[0]; - msecAdd = array[1]; - } - else - { - if ( index == 3 ) - { - minAdd = array[0]; - secAdd = array[1]; - msecAdd = array[2]; - } - else - { - if ( index == 4 ) - { - hourAdd = array[0]; - minAdd = array[1]; - secAdd = array[2]; - msecAdd = array[3]; - } - else - { - dayAdd = array[0]; - hourAdd = array[1]; - minAdd = array[2]; - secAdd = array[3]; - msecAdd = array[4]; - } - } - } - } - - break; - } - - case IntervalColumn::INTERVAL_HOUR_MICROSECOND: - { - if ( index > 4 ) - { - return 0; - } - - if ( index == 1 ) - msecAdd = array[0]; - else - { - if ( index == 2 ) - { - secAdd = array[0]; - msecAdd = array[1]; - } - else - { - if ( index == 3 ) - { - minAdd = array[0]; - secAdd = array[1]; - msecAdd = array[2]; - } - else - { - hourAdd = array[0]; - minAdd = array[1]; - secAdd = array[2]; - msecAdd = array[3]; - } - } - } - - break; - } - - case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: - { - if ( index > 3 ) - { - return 0; - } - - if ( index == 1 ) - msecAdd = array[0]; - else - { - if ( index == 2 ) - { - secAdd = array[0]; - msecAdd = array[1]; - } - else - { - minAdd = array[0]; - secAdd = array[1]; - msecAdd = array[2]; - } - } - - break; - } - - case IntervalColumn::INTERVAL_SECOND_MICROSECOND: - { - if ( index > 2 ) - { - return 0; - } - - if ( index == 1 ) - msecAdd = array[0]; - else - { - secAdd = array[0]; - msecAdd = array[1]; - } - - break; - } - - default: - // should be impossible to get here since we checked the presence - // in the map, but just to be safe... - throw runtime_error("unit type is not supported"); - }; - - - // calulate new date - - year += yearAdd; - - month += monthAdd; - - day += dayAdd; - - hour += hourAdd; - - min += minAdd; - - sec += secAdd; - - msec += msecAdd; - - if ( msec > 999999 ) - { - int64_t secs = msec / 1000000; - sec += secs; - msec = msec - ( secs * 1000000 ); + monthAdd = array[0] * 3; + break; } - if ( msec < 0 ) + case IntervalColumn::INTERVAL_MONTH: { - int64_t secs = 1 + (-msec / 1000000); - sec -= secs; - msec = msec + ( secs * 1000000 ); + monthAdd = array[0]; + break; } - if ( sec > 59 ) + case IntervalColumn::INTERVAL_WEEK: { - int64_t mins = sec / 60; - min += mins; - sec = sec - ( mins * 60 ); + dayAdd = array[0] * 7; + break; } - if ( sec < 0 ) + case IntervalColumn::INTERVAL_DAY: { - int64_t mins = 0; - - if ( ( sec + (-sec / 60) * 60 ) == 0 && - sec != 0) - mins = (-sec / 60); - else - mins = 1 + (-sec / 60); - - min -= mins; - sec = sec + ( mins * 60 ); - - if ( sec >= 60 ) - sec = 0; + dayAdd = array[0]; + break; } - if ( min > 59 ) + case IntervalColumn::INTERVAL_HOUR: { - int hours = min / 60; - hour += hours; - min = min - ( hours * 60 ); + hourAdd = array[0]; + break; } - if ( min < 0 ) + case IntervalColumn::INTERVAL_MINUTE: { - int hours = 0; - - if ( ( min + (-min / 60) * 60 ) == 0 && - min != 0) - hours = (-min / 60); - else - hours = 1 + (-min / 60); - - hour -= hours; - min = min + ( hours * 60 ); - - if ( min >= 60 ) - min = 0; + minAdd = array2[0]; + break; } - if ( hour > 23 ) + case IntervalColumn::INTERVAL_SECOND: { - int days = hour / 24; - day += days; - hour = hour - ( days * 24 ); + secAdd = array2[0]; + break; } - if ( hour < 0 ) + case IntervalColumn::INTERVAL_MICROSECOND: { - int days = 0; - - if ( ( hour + (-hour / 24) * 24 ) == 0 && - hour != 0) - days = (-hour / 24); - else - days = 1 + (-hour / 24); - - day -= days; - hour = hour + ( days * 24 ); - - if ( hour >= 24 ) - hour = 0; + msecAdd = array2[0]; + break; } - int tmpYear = year; - - if ( isLeapYear(tmpYear) ) - month_length[2] = 29; - else - month_length[2] = 28; - - if ( day > 0 ) - { - while (true) - { - int years = (monthSave - 1) / 12; - -// tmpYear += years; - if ( isLeapYear(tmpYear) ) - month_length[2] = 29; - else - month_length[2] = 28; - - int tmpMonth = monthSave - ( years * 12 ); - - if ( day <= month_length[tmpMonth] ) - break; - - month++; - day = day - month_length[tmpMonth]; - monthSave++; - - if ( monthSave > 12 ) - { - monthSave = 1; - tmpYear++; - } - } - } - else - { - while (true) - { - if ( day > 0 ) - break; - - if ( -day < month_length[monthSave] ) - { - month--; - monthSave--; - - if ( monthSave == 0 ) - { - monthSave = 12; - tmpYear--; - } - - if (monthSave == 2) - { - // if ( dataconvert::DataConvert::isLeapYear(year) ) - if ( isLeapYear(tmpYear) ) - month_length[2] = 29; - } - - if (day < 1 ) - day = month_length[monthSave] + day; - - // BUG 5448 - changed from '==' to '<=' - if ( day <= 0 ) - { - month--; - monthSave--; - - if ( monthSave == 0 ) - { - monthSave = 12; - tmpYear--; - } - - if (monthSave == 2) - { - // if ( dataconvert::DataConvert::isLeapYear(year) ) - if ( isLeapYear(tmpYear) ) - month_length[2] = 29; - } - - day = day + month_length[monthSave]; - } - - break; - } - - month--; - monthSave--; - - if ( monthSave == 0 ) - { - monthSave = 12; - tmpYear--; - - if ( isLeapYear(tmpYear) ) - month_length[2] = 29; - else - month_length[2] = 28; - } - - day = day + month_length[monthSave]; - } - } - - - if ( month > 12 ) - { - int years = (month - 1) / 12; - year += years; - month = month - ( years * 12 ); - } - - if ( month < 1 ) - { - int years = 1 + ((-month) / 12); - year -= years; - month = month + ( years * 12 ); - } - - if ( isLeapYear(year) ) - month_length[2] = 29; - else - month_length[2] = 28; - - if ( day > month_length[month] ) - day = month_length[month]; - - if ( year < 1000 || year > 9999 ) + case IntervalColumn::INTERVAL_YEAR_MONTH: { + if (index > 2) + { return 0; + } + + if (index == 1) + monthAdd = array[0]; + else + { + yearAdd = array[0]; + monthAdd = array[1]; + } + + break; } + case IntervalColumn::INTERVAL_DAY_HOUR: + { + if (index > 2) + { + return 0; + } - uint64_t value; - dataconvert::DateTime aDatetime; - aDatetime.year = year; - aDatetime.month = month; - aDatetime.day = day; - aDatetime.hour = hour; - aDatetime.minute = min; - aDatetime.second = sec; - aDatetime.msecond = msec; - value = *(reinterpret_cast(&aDatetime)); + if (index == 1) + hourAdd = array[0]; + else + { + dayAdd = array[0]; + hourAdd = array[1]; + } - return value; + break; + } + case IntervalColumn::INTERVAL_DAY_MINUTE: + { + if (index > 3) + { + return 0; + } + + if (index == 1) + minAdd = array[0]; + else + { + if (index == 2) + { + hourAdd = array[0]; + minAdd = array[1]; + } + else + { + dayAdd = array[0]; + hourAdd = array[1]; + minAdd = array[2]; + } + } + + break; + } + + case IntervalColumn::INTERVAL_DAY_SECOND: + { + if (index > 4) + { + return 0; + } + + if (index == 1) + secAdd = array[0]; + else + { + if (index == 2) + { + minAdd = array[0]; + secAdd = array[1]; + } + else + { + if (index == 3) + { + hourAdd = array[0]; + minAdd = array[1]; + secAdd = array[2]; + } + else + { + dayAdd = array[0]; + hourAdd = array[1]; + minAdd = array[2]; + secAdd = array[3]; + } + } + } + + break; + } + + case IntervalColumn::INTERVAL_HOUR_MINUTE: + { + if (index > 2) + { + return 0; + } + + if (index == 1) + minAdd = array[0]; + else + { + hourAdd = array[0]; + minAdd = array[1]; + } + + break; + } + + case IntervalColumn::INTERVAL_HOUR_SECOND: + { + if (index > 3) + { + return 0; + } + + if (index == 1) + secAdd = array[0]; + else + { + if (index == 2) + { + minAdd = array[0]; + secAdd = array[1]; + } + else + { + hourAdd = array[0]; + minAdd = array[1]; + secAdd = array[2]; + } + } + + break; + } + + case IntervalColumn::INTERVAL_MINUTE_SECOND: + { + if (index > 2) + { + return 0; + } + + if (index == 1) + secAdd = array[0]; + else + { + minAdd = array[0]; + secAdd = array[1]; + } + + break; + } + + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + { + if (index > 5) + { + return 0; + } + + if (index == 1) + msecAdd = array[0]; + else + { + if (index == 2) + { + secAdd = array[0]; + msecAdd = array[1]; + } + else + { + if (index == 3) + { + minAdd = array[0]; + secAdd = array[1]; + msecAdd = array[2]; + } + else + { + if (index == 4) + { + hourAdd = array[0]; + minAdd = array[1]; + secAdd = array[2]; + msecAdd = array[3]; + } + else + { + dayAdd = array[0]; + hourAdd = array[1]; + minAdd = array[2]; + secAdd = array[3]; + msecAdd = array[4]; + } + } + } + } + + break; + } + + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + { + if (index > 4) + { + return 0; + } + + if (index == 1) + msecAdd = array[0]; + else + { + if (index == 2) + { + secAdd = array[0]; + msecAdd = array[1]; + } + else + { + if (index == 3) + { + minAdd = array[0]; + secAdd = array[1]; + msecAdd = array[2]; + } + else + { + hourAdd = array[0]; + minAdd = array[1]; + secAdd = array[2]; + msecAdd = array[3]; + } + } + } + + break; + } + + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: + { + if (index > 3) + { + return 0; + } + + if (index == 1) + msecAdd = array[0]; + else + { + if (index == 2) + { + secAdd = array[0]; + msecAdd = array[1]; + } + else + { + minAdd = array[0]; + secAdd = array[1]; + msecAdd = array[2]; + } + } + + break; + } + + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: + { + if (index > 2) + { + return 0; + } + + if (index == 1) + msecAdd = array[0]; + else + { + secAdd = array[0]; + msecAdd = array[1]; + } + + break; + } + + default: + // should be impossible to get here since we checked the presence + // in the map, but just to be safe... + throw runtime_error("unit type is not supported"); + }; + + // calulate new date + + year += yearAdd; + + month += monthAdd; + + day += dayAdd; + + hour += hourAdd; + + min += minAdd; + + sec += secAdd; + + msec += msecAdd; + + if (msec > 999999) + { + int64_t secs = msec / 1000000; + sec += secs; + msec = msec - (secs * 1000000); + } + + if (msec < 0) + { + int64_t secs = 1 + (-msec / 1000000); + sec -= secs; + msec = msec + (secs * 1000000); + } + + if (sec > 59) + { + int64_t mins = sec / 60; + min += mins; + sec = sec - (mins * 60); + } + + if (sec < 0) + { + int64_t mins = 0; + + if ((sec + (-sec / 60) * 60) == 0 && sec != 0) + mins = (-sec / 60); + else + mins = 1 + (-sec / 60); + + min -= mins; + sec = sec + (mins * 60); + + if (sec >= 60) + sec = 0; + } + + if (min > 59) + { + int hours = min / 60; + hour += hours; + min = min - (hours * 60); + } + + if (min < 0) + { + int hours = 0; + + if ((min + (-min / 60) * 60) == 0 && min != 0) + hours = (-min / 60); + else + hours = 1 + (-min / 60); + + hour -= hours; + min = min + (hours * 60); + + if (min >= 60) + min = 0; + } + + if (hour > 23) + { + int days = hour / 24; + day += days; + hour = hour - (days * 24); + } + + if (hour < 0) + { + int days = 0; + + if ((hour + (-hour / 24) * 24) == 0 && hour != 0) + days = (-hour / 24); + else + days = 1 + (-hour / 24); + + day -= days; + hour = hour + (days * 24); + + if (hour >= 24) + hour = 0; + } + + int tmpYear = year; + + if (isLeapYear(tmpYear)) + month_length[2] = 29; + else + month_length[2] = 28; + + if (day > 0) + { + while (true) + { + int years = (monthSave - 1) / 12; + + // tmpYear += years; + if (isLeapYear(tmpYear)) + month_length[2] = 29; + else + month_length[2] = 28; + + int tmpMonth = monthSave - (years * 12); + + if (day <= month_length[tmpMonth]) + break; + + month++; + day = day - month_length[tmpMonth]; + monthSave++; + + if (monthSave > 12) + { + monthSave = 1; + tmpYear++; + } + } + } + else + { + while (true) + { + if (day > 0) + break; + + if (-day < month_length[monthSave]) + { + month--; + monthSave--; + + if (monthSave == 0) + { + monthSave = 12; + tmpYear--; + } + + if (monthSave == 2) + { + // if ( dataconvert::DataConvert::isLeapYear(year) ) + if (isLeapYear(tmpYear)) + month_length[2] = 29; + } + + if (day < 1) + day = month_length[monthSave] + day; + + // BUG 5448 - changed from '==' to '<=' + if (day <= 0) + { + month--; + monthSave--; + + if (monthSave == 0) + { + monthSave = 12; + tmpYear--; + } + + if (monthSave == 2) + { + // if ( dataconvert::DataConvert::isLeapYear(year) ) + if (isLeapYear(tmpYear)) + month_length[2] = 29; + } + + day = day + month_length[monthSave]; + } + + break; + } + + month--; + monthSave--; + + if (monthSave == 0) + { + monthSave = 12; + tmpYear--; + + if (isLeapYear(tmpYear)) + month_length[2] = 29; + else + month_length[2] = 28; + } + + day = day + month_length[monthSave]; + } + } + + if (month > 12) + { + int years = (month - 1) / 12; + year += years; + month = month - (years * 12); + } + + if (month < 1) + { + int years = 1 + ((-month) / 12); + year -= years; + month = month + (years * 12); + } + + if (isLeapYear(year)) + month_length[2] = 29; + else + month_length[2] = 28; + + if (day > month_length[month]) + day = month_length[month]; + + if (year < 1000 || year > 9999) + { + return 0; + } + + uint64_t value; + dataconvert::DateTime aDatetime; + aDatetime.year = year; + aDatetime.month = month; + aDatetime.day = day; + aDatetime.hour = hour; + aDatetime.minute = min; + aDatetime.second = sec; + aDatetime.msecond = msec; + value = *(reinterpret_cast(&aDatetime)); + + return value; } -} // namespace funcexp::helpers +} // namespace helpers -CalpontSystemCatalog::ColType Func_date_add::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_date_add::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - resultType.colDataType = CalpontSystemCatalog::DATETIME; - resultType.colWidth = 8; + resultType.colDataType = CalpontSystemCatalog::DATETIME; + resultType.colWidth = 8; - return resultType; + return resultType; } - -int64_t Func_date_add::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_date_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t val = 0; - bool dateType = false; + int64_t val = 0; + bool dateType = false; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - else - isNull = true; - - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - val = parm[0]->data()->getDateIntVal(row, isNull); - dateType = true; - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - DateTime dt; - dt.year = m_time.year; - dt.month = m_time.month; - dt.day = m_time.day; - dt.hour = m_time.hour; - dt.minute = m_time.minute; - dt.second = m_time.second; - dt.msecond = timestamp.msecond; - val = *(reinterpret_cast(&dt)); - break; - } - - default: - { - isNull = true; - } + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + break; } - if (isNull || val == -1) + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { + if (parm[0]->data()->resultType().scale == 0) + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + else isNull = true; - return 0; + + break; } - IntervalColumn::interval_type unit = static_cast(parm[2]->data()->getIntVal(row, isNull)); - OpType funcType = OP_ADD; - ConstantColumn* constCol = dynamic_cast(parm[3]->data()); - execplan::CalpontSystemCatalog::ColType ct3 = parm[3]->data()->resultType(); + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + break; + } - if ((ct3.colDataType == execplan::CalpontSystemCatalog::CHAR || - ct3.colDataType == execplan::CalpontSystemCatalog::TEXT || - ct3.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && - constCol != NULL && constCol->constval().compare("SUB") == 0) - funcType = OP_SUB; - else - funcType = static_cast(parm[3]->data()->getIntVal(row, isNull)); + case execplan::CalpontSystemCatalog::DATE: + { + val = parm[0]->data()->getDateIntVal(row, isNull); + dateType = true; + break; + } - uint64_t value = helpers::dateAdd( - val, parm[1]->data()->getStrVal(row, isNull), unit, dateType, funcType); + case execplan::CalpontSystemCatalog::DATETIME: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } - if ( value == 0 ) - isNull = true; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = m_time.hour; + dt.minute = m_time.minute; + dt.second = m_time.second; + dt.msecond = timestamp.msecond; + val = *(reinterpret_cast(&dt)); + break; + } -// if (dateType) -// value = (((value >> 32) & 0xFFFFFFC0) | 0x3E); + default: + { + isNull = true; + } + } - return value; + if (isNull || val == -1) + { + isNull = true; + return 0; + } + + IntervalColumn::interval_type unit = + static_cast(parm[2]->data()->getIntVal(row, isNull)); + OpType funcType = OP_ADD; + ConstantColumn* constCol = dynamic_cast(parm[3]->data()); + execplan::CalpontSystemCatalog::ColType ct3 = parm[3]->data()->resultType(); + + if ((ct3.colDataType == execplan::CalpontSystemCatalog::CHAR || + ct3.colDataType == execplan::CalpontSystemCatalog::TEXT || + ct3.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && + constCol != NULL && constCol->constval().compare("SUB") == 0) + funcType = OP_SUB; + else + funcType = static_cast(parm[3]->data()->getIntVal(row, isNull)); + + uint64_t value = helpers::dateAdd(val, parm[1]->data()->getStrVal(row, isNull), unit, dateType, funcType); + + if (value == 0) + isNull = true; + + // if (dateType) + // value = (((value >> 32) & 0xFFFFFFC0) | 0x3E); + + return value; } - -string Func_date_add::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_date_add::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct)); - + return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct)); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 88a1ed99f..0aff75d9e 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_date_format.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_date_format.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -40,394 +40,380 @@ namespace helpers { const string IDB_date_format(const DateTime& dt, const string& format) { - // assume 256 is enough. assume not allowing incomplete date - char buf[256]; - char* ptr = buf; - uint32_t weekday = 0; - uint32_t dayval = 0; - uint32_t weekval = 0; - uint32_t weekyear = 0; + // assume 256 is enough. assume not allowing incomplete date + char buf[256]; + char* ptr = buf; + uint32_t weekday = 0; + uint32_t dayval = 0; + uint32_t weekval = 0; + uint32_t weekyear = 0; - for (uint32_t i = 0; i < format.length(); i++) + for (uint32_t i = 0; i < format.length(); i++) + { + if (format[i] != '%') + *ptr++ = format[i]; + else { - if (format[i] != '%') - *ptr++ = format[i]; - else - { - i++; + i++; - switch (format[i]) - { - case 'M': - sprintf(ptr, "%s", helpers::monthFullNames[dt.month].c_str()); - ptr += helpers::monthFullNames[dt.month].length(); - break; + switch (format[i]) + { + case 'M': + sprintf(ptr, "%s", helpers::monthFullNames[dt.month].c_str()); + ptr += helpers::monthFullNames[dt.month].length(); + break; - case 'b': - sprintf(ptr, "%s", helpers::monthAbNames[dt.month].c_str()); - ptr += helpers::monthAbNames[dt.month].length(); - break; + case 'b': + sprintf(ptr, "%s", helpers::monthAbNames[dt.month].c_str()); + ptr += helpers::monthAbNames[dt.month].length(); + break; - case 'W': - weekday = helpers::calc_mysql_weekday( dt.year, dt.month, dt.day, false); - sprintf(ptr, "%s", helpers::weekdayFullNames[weekday].c_str()); - ptr += helpers::weekdayFullNames[weekday].length(); - break; + case 'W': + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false); + sprintf(ptr, "%s", helpers::weekdayFullNames[weekday].c_str()); + ptr += helpers::weekdayFullNames[weekday].length(); + break; - case 'w': - weekday = helpers::calc_mysql_weekday( dt.year, dt.month, dt.day, true); - sprintf(ptr, "%01d", weekday); - ptr += 1; - break; + case 'w': + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, true); + sprintf(ptr, "%01d", weekday); + ptr += 1; + break; - case 'a': - weekday = helpers::calc_mysql_weekday( dt.year, dt.month, dt.day, false); - sprintf(ptr, "%s", helpers::weekdayAbNames[weekday].c_str()); - ptr += helpers::weekdayAbNames[weekday].length(); - break; + case 'a': + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false); + sprintf(ptr, "%s", helpers::weekdayAbNames[weekday].c_str()); + ptr += helpers::weekdayAbNames[weekday].length(); + break; - case 'D': - sprintf(ptr, "%s", helpers::dayOfMonth[dt.day].c_str()); - ptr += helpers::dayOfMonth[dt.day].length(); - break; + case 'D': + sprintf(ptr, "%s", helpers::dayOfMonth[dt.day].c_str()); + ptr += helpers::dayOfMonth[dt.day].length(); + break; - case 'Y': - sprintf(ptr, "%04d", dt.year); - ptr += 4; - break; + case 'Y': + sprintf(ptr, "%04d", dt.year); + ptr += 4; + break; - case 'y': - sprintf(ptr, "%02d", dt.year % 100); - ptr += 2; - break; + case 'y': + sprintf(ptr, "%02d", dt.year % 100); + ptr += 2; + break; - case 'm': - sprintf(ptr, "%02d", dt.month); - ptr += 2; - break; + case 'm': + sprintf(ptr, "%02d", dt.month); + ptr += 2; + break; - case 'c': - sprintf(ptr, "%d", dt.month); - ptr = ptr + (dt.month >= 10 ? 2 : 1); - break; + case 'c': + sprintf(ptr, "%d", dt.month); + ptr = ptr + (dt.month >= 10 ? 2 : 1); + break; - case 'd': - sprintf(ptr, "%02d", dt.day); - ptr += 2; - break; + case 'd': + sprintf(ptr, "%02d", dt.day); + ptr += 2; + break; - case 'e': - sprintf(ptr, "%d", dt.day); - ptr = ptr + (dt.day >= 10 ? 2 : 1); - break; + case 'e': + sprintf(ptr, "%d", dt.day); + ptr = ptr + (dt.day >= 10 ? 2 : 1); + break; - case 'f': - sprintf(ptr, "%06d", dt.msecond); - ptr += 6; - break; + case 'f': + sprintf(ptr, "%06d", dt.msecond); + ptr += 6; + break; - case 'H': - sprintf(ptr, "%02d", dt.hour); - ptr += 2; - break; + case 'H': + sprintf(ptr, "%02d", dt.hour); + ptr += 2; + break; - case 'h': - case 'I': - sprintf(ptr, "%02d", (dt.hour % 24 + 11) % 12 + 1); - ptr += 2; - break; + case 'h': + case 'I': + sprintf(ptr, "%02d", (dt.hour % 24 + 11) % 12 + 1); + ptr += 2; + break; - case 'i': /* minutes */ - sprintf(ptr, "%02d", dt.minute); - ptr += 2; - break; + case 'i': /* minutes */ + sprintf(ptr, "%02d", dt.minute); + ptr += 2; + break; - case 'j': - dayval = helpers::calc_mysql_daynr( dt.year, dt.month, dt.day ) - - helpers::calc_mysql_daynr( dt.year, 1, 1 ) + 1; - sprintf(ptr, "%03d", dayval); - ptr += 3; - break; + case 'j': + dayval = helpers::calc_mysql_daynr(dt.year, dt.month, dt.day) - + helpers::calc_mysql_daynr(dt.year, 1, 1) + 1; + sprintf(ptr, "%03d", dayval); + ptr += 3; + break; - case 'k': - sprintf(ptr, "%d", dt.hour); - ptr += (dt.hour >= 10 ? 2 : 1); - break; + case 'k': + sprintf(ptr, "%d", dt.hour); + ptr += (dt.hour >= 10 ? 2 : 1); + break; - case 'l': - sprintf(ptr, "%d", (dt.hour % 24 + 11) % 12 + 1); - ptr += ((dt.hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1); - break; + case 'l': + sprintf(ptr, "%d", (dt.hour % 24 + 11) % 12 + 1); + ptr += ((dt.hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1); + break; - case 'p': - sprintf(ptr, "%s", (dt.hour % 24 < 12 ? "AM" : "PM")); - ptr += 2; - break; + case 'p': + sprintf(ptr, "%s", (dt.hour % 24 < 12 ? "AM" : "PM")); + ptr += 2; + break; - case 'r': - sprintf(ptr, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), - (dt.hour + 11) % 12 + 1, dt.minute, dt.second); - ptr += 11; - break; + case 'r': + sprintf(ptr, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), + (dt.hour + 11) % 12 + 1, dt.minute, dt.second); + ptr += 11; + break; - case 'S': - case 's': - sprintf(ptr, "%02d", dt.second); - ptr += 2; - break; + case 'S': + case 's': + sprintf(ptr, "%02d", dt.second); + ptr += 2; + break; - case 'T': - sprintf (ptr, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); - ptr += 8; - break; + case 'T': + sprintf(ptr, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + ptr += 8; + break; - case 'U': - weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, 0); - sprintf(ptr, "%02d", weekval); - ptr += 2; - break; + case 'U': + weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, 0); + sprintf(ptr, "%02d", weekval); + ptr += 2; + break; - case 'V': - weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, - helpers::WEEK_NO_ZERO ); - sprintf(ptr, "%02d", weekval); - ptr += 2; - break; + case 'V': + weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO); + sprintf(ptr, "%02d", weekval); + ptr += 2; + break; - case 'u': - weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, - helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); - sprintf(ptr, "%02d", weekval); - ptr += 2; - break; + case 'u': + weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, + helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); + sprintf(ptr, "%02d", weekval); + ptr += 2; + break; - case 'v': - weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, - helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); - sprintf(ptr, "%02d", weekval); - ptr += 2; - break; + case 'v': + weekval = helpers::calc_mysql_week( + dt.year, dt.month, dt.day, + helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); + sprintf(ptr, "%02d", weekval); + ptr += 2; + break; - case 'x': - helpers::calc_mysql_week( dt.year, dt.month, dt.day, - helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS, &weekyear); - sprintf(ptr, "%04d", weekyear); - ptr += 4; - break; + case 'x': + helpers::calc_mysql_week( + dt.year, dt.month, dt.day, + helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS, &weekyear); + sprintf(ptr, "%04d", weekyear); + ptr += 4; + break; - case 'X': - helpers::calc_mysql_week( dt.year, dt.month, dt.day, - helpers::WEEK_NO_ZERO, &weekyear); - sprintf(ptr, "%04d", weekyear); - ptr += 4; - break; + case 'X': + helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO, &weekyear); + sprintf(ptr, "%04d", weekyear); + ptr += 4; + break; - default: - *ptr++ = format[i]; - } - } + default: *ptr++ = format[i]; + } } + } - *ptr = 0; - return string(buf); -} + *ptr = 0; + return string(buf); } +} // namespace helpers -CalpontSystemCatalog::ColType Func_date_format::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_date_format::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + return ct; } -string Func_date_format::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - int64_t val = 0; - DateTime dt = 0; + int64_t val = 0; + DateTime dt = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + dt.year = (uint32_t)((val >> 16) & 0xffff); + dt.month = (uint32_t)((val >> 12) & 0xf); + dt.day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + dt.year = (uint32_t)((val >> 48) & 0xffff); + dt.month = (uint32_t)((val >> 44) & 0xf); + dt.day = (uint32_t)((val >> 38) & 0x3f); + dt.hour = (uint32_t)((val >> 32) & 0x3f); + dt.minute = (uint32_t)((val >> 26) & 0x3f); + dt.second = (uint32_t)((val >> 20) & 0x3f); + dt.msecond = (uint32_t)((val & 0xfffff)); + break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - dt.year = (uint32_t)((val >> 16) & 0xffff); - dt.month = (uint32_t)((val >> 12) & 0xf); - dt.day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - dt.year = (uint32_t)((val >> 48) & 0xffff); - dt.month = (uint32_t)((val >> 44) & 0xf); - dt.day = (uint32_t)((val >> 38) & 0x3f); - dt.hour = (uint32_t)((val >> 32) & 0x3f); - dt.minute = (uint32_t)((val >> 26) & 0x3f); - dt.second = (uint32_t)((val >> 20) & 0x3f); - dt.msecond = (uint32_t)((val & 0xfffff)); - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - val = parm[0]->data()->getTimestampIntVal(row, isNull); - TimeStamp timestamp(val); - int64_t seconds = timestamp.second; - MySQLTime time; - gmtSecToMySQLTime(seconds, time, timeZone()); - dt.year = time.year; - dt.month = time.month; - dt.day = time.day; - dt.hour = time.hour; - dt.minute = time.minute; - dt.second = time.second; - dt.msecond = timestamp.msecond; - break; - } - - case CalpontSystemCatalog::TIME: - { - DateTime aDateTime = static_cast(nowDatetime()); - Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - aDateTime.hour = 0; - aDateTime.minute = 0; - aDateTime.second = 0; - aDateTime.msecond = 0; - if ((aTime.hour < 0) || (aTime.is_neg)) - { - aTime.hour = -abs(aTime.hour); - aTime.minute = -abs(aTime.minute); - aTime.second = -abs(aTime.second); - aTime.msecond = -abs(aTime.msecond); - } - val = addTime(aDateTime, aTime); - dt.year = (uint32_t)((val >> 48) & 0xffff); - dt.month = (uint32_t)((val >> 44) & 0xf); - dt.day = (uint32_t)((val >> 38) & 0x3f); - dt.hour = (uint32_t)((val >> 32) & 0x3f); - dt.minute = (uint32_t)((val >> 26) & 0x3f); - dt.second = (uint32_t)((val >> 20) & 0x3f); - dt.msecond = (uint32_t)((val & 0xfffff)); - break; - } - - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return ""; - } - else - { - dt.year = (uint32_t)((val >> 48) & 0xffff); - dt.month = (uint32_t)((val >> 44) & 0xf); - dt.day = (uint32_t)((val >> 38) & 0x3f); - dt.hour = (uint32_t)((val >> 32) & 0x3f); - dt.minute = (uint32_t)((val >> 26) & 0x3f); - dt.second = (uint32_t)((val >> 20) & 0x3f); - dt.msecond = (uint32_t)((val & 0xfffff)); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return ""; - } - else - { - dt.year = (uint32_t)((val >> 48) & 0xffff); - dt.month = (uint32_t)((val >> 44) & 0xf); - dt.day = (uint32_t)((val >> 38) & 0x3f); - dt.hour = (uint32_t)((val >> 32) & 0x3f); - dt.minute = (uint32_t)((val >> 26) & 0x3f); - dt.second = (uint32_t)((val >> 20) & 0x3f); - dt.msecond = (uint32_t)((val & 0xfffff)); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return ""; - } - else - { - dt.year = (uint32_t)((val >> 48) & 0xffff); - dt.month = (uint32_t)((val >> 44) & 0xf); - dt.day = (uint32_t)((val >> 38) & 0x3f); - dt.hour = (uint32_t)((val >> 32) & 0x3f); - dt.minute = (uint32_t)((val >> 26) & 0x3f); - dt.second = (uint32_t)((val >> 20) & 0x3f); - dt.msecond = (uint32_t)((val & 0xfffff)); - } - } - else - { - isNull = true; - return ""; - } - - break; - - default: - isNull = true; - return ""; + val = parm[0]->data()->getTimestampIntVal(row, isNull); + TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timeZone()); + dt.year = time.year; + dt.month = time.month; + dt.day = time.day; + dt.hour = time.hour; + dt.minute = time.minute; + dt.second = time.second; + dt.msecond = timestamp.msecond; + break; } - const string& format = parm[1]->data()->getStrVal(row, isNull); + case CalpontSystemCatalog::TIME: + { + DateTime aDateTime = static_cast(nowDatetime()); + Time aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + aDateTime.hour = 0; + aDateTime.minute = 0; + aDateTime.second = 0; + aDateTime.msecond = 0; + if ((aTime.hour < 0) || (aTime.is_neg)) + { + aTime.hour = -abs(aTime.hour); + aTime.minute = -abs(aTime.minute); + aTime.second = -abs(aTime.second); + aTime.msecond = -abs(aTime.msecond); + } + val = addTime(aDateTime, aTime); + dt.year = (uint32_t)((val >> 48) & 0xffff); + dt.month = (uint32_t)((val >> 44) & 0xf); + dt.day = (uint32_t)((val >> 38) & 0x3f); + dt.hour = (uint32_t)((val >> 32) & 0x3f); + dt.minute = (uint32_t)((val >> 26) & 0x3f); + dt.second = (uint32_t)((val >> 20) & 0x3f); + dt.msecond = (uint32_t)((val & 0xfffff)); + break; + } - return helpers::IDB_date_format(dt, format); + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return ""; + } + else + { + dt.year = (uint32_t)((val >> 48) & 0xffff); + dt.month = (uint32_t)((val >> 44) & 0xf); + dt.day = (uint32_t)((val >> 38) & 0x3f); + dt.hour = (uint32_t)((val >> 32) & 0x3f); + dt.minute = (uint32_t)((val >> 26) & 0x3f); + dt.second = (uint32_t)((val >> 20) & 0x3f); + dt.msecond = (uint32_t)((val & 0xfffff)); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return ""; + } + else + { + dt.year = (uint32_t)((val >> 48) & 0xffff); + dt.month = (uint32_t)((val >> 44) & 0xf); + dt.day = (uint32_t)((val >> 38) & 0x3f); + dt.hour = (uint32_t)((val >> 32) & 0x3f); + dt.minute = (uint32_t)((val >> 26) & 0x3f); + dt.second = (uint32_t)((val >> 20) & 0x3f); + dt.msecond = (uint32_t)((val & 0xfffff)); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return ""; + } + else + { + dt.year = (uint32_t)((val >> 48) & 0xffff); + dt.month = (uint32_t)((val >> 44) & 0xf); + dt.day = (uint32_t)((val >> 38) & 0x3f); + dt.hour = (uint32_t)((val >> 32) & 0x3f); + dt.minute = (uint32_t)((val >> 26) & 0x3f); + dt.second = (uint32_t)((val >> 20) & 0x3f); + dt.msecond = (uint32_t)((val & 0xfffff)); + } + } + else + { + isNull = true; + return ""; + } + + break; + + default: isNull = true; return ""; + } + + const string& format = parm[1]->data()->getStrVal(row, isNull); + + return helpers::IDB_date_format(dt, format); } - -int32_t Func_date_format::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_date_format::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::dateToInt(getStrVal(row, parm, isNull, ct)); + return dataconvert::DataConvert::dateToInt(getStrVal(row, parm, isNull, ct)); } - -int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); + return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } -int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index 9b7def395..29baf4466 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_day.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_day.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,119 +35,112 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_day::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_day::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_day::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; + int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + return (uint32_t)((val >> 6) & 0x3f); + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + return (uint32_t)((val >> 38) & 0x3f); + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - return (uint32_t)((val >> 6) & 0x3f); - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - return (uint32_t)((val >> 38) & 0x3f); - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - return m_time.day; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - return (uint32_t)((val >> 38) & 0x3f); - break; - - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + return m_time.day; } - return -1; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + return (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + } + + break; + + default: isNull = true; return -1; + } + + return -1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 7a076435b..9a94882c6 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_dayname.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_dayname.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,156 +36,147 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_dayname::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_dayname::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_dayname::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_dayname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint32_t year = 0; - uint32_t month = 0; - uint32_t day = 0; - int64_t val = 0; - int32_t dayofweek = 0; + uint32_t year = 0; + uint32_t month = 0; + uint32_t day = 0; + int64_t val = 0; + int32_t dayofweek = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - val = parm[0]->data()->getIntVal(row, isNull); - dataconvert::TimeStamp timestamp(val); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); - year = time.year; - month = time.month; - day = time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } - - break; - - default: - isNull = true; - return -1; + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + year = time.year; + month = time.month; + day = time.day; + break; } - dayofweek = helpers::calc_mysql_weekday( year, month, day, false); - return dayofweek; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } + + break; + + default: isNull = true; return -1; + } + + dayofweek = helpers::calc_mysql_weekday(year, month, day, false); + return dayofweek; } - -string Func_dayname::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_dayname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int32_t weekday = getIntVal(row, parm, isNull, op_ct); + int32_t weekday = getIntVal(row, parm, isNull, op_ct); - if (weekday == -1) - return ""; + if (weekday == -1) + return ""; - return helpers::weekdayFullNames[weekday]; + return helpers::weekdayFullNames[weekday]; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index d0b28d22e..64f04bcf3 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_dayofweek.cpp 3616 2013-03-04 14:56:29Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_dayofweek.cpp 3616 2013-03-04 14:56:29Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,139 +36,134 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_dayofweek::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_dayofweek::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - uint32_t year = 0; - uint32_t month = 0; - uint32_t day = 0; - int64_t val = 0; + uint32_t year = 0; + uint32_t month = 0; + uint32_t day = 0; + int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - val = parm[0]->data()->getIntVal(row, isNull); - dataconvert::TimeStamp timestamp(val); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); - year = time.year; - month = time.month; - day = time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } - - break; - - default: - isNull = true; - return -1; + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + year = time.year; + month = time.month; + day = time.day; + break; } - return helpers::calc_mysql_weekday(year, month, day, true) + 1; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } + + break; + + default: isNull = true; return -1; + } + + return helpers::calc_mysql_weekday(year, month, day, true) + 1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index cf9c37814..487635c7c 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_dayofyear.cpp 3616 2013-03-04 14:56:29Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_dayofyear.cpp 3616 2013-03-04 14:56:29Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,134 +36,128 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_dayofyear::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_dayofyear::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - uint32_t year = 0; - uint32_t month = 0; - uint32_t day = 0; - int64_t val = 0; + uint32_t year = 0; + uint32_t month = 0; + uint32_t day = 0; + int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - year = m_time.year; - month = m_time.month; - day = m_time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; } - return helpers::calc_mysql_daynr(year, month, day) - - helpers::calc_mysql_daynr(year, 1, 1) + 1; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + + break; + + default: isNull = true; return -1; + } + + return helpers::calc_mysql_daynr(year, month, day) - helpers::calc_mysql_daynr(year, 1, 1) + 1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_decode.cpp b/utils/funcexp/func_decode.cpp index 14b8e9e88..1adbfd407 100644 --- a/utils/funcexp/func_decode.cpp +++ b/utils/funcexp/func_decode.cpp @@ -27,63 +27,61 @@ using namespace execplan; namespace funcexp { - -void Func_decode::hash_password(ulong *result, const char *password, uint password_len) +void Func_decode::hash_password(ulong* result, const char* password, uint password_len) { - ulong nr=1345345333L, add=7, nr2=0x12345671L; + ulong nr = 1345345333L, add = 7, nr2 = 0x12345671L; ulong tmp; - const char *password_end= password + password_len; + const char* password_end = password + password_len; for (; password < password_end; password++) { if (*password == ' ' || *password == '\t') continue; - tmp= (ulong) (unsigned char) *password; - nr^= (((nr & 63)+add)*tmp)+ (nr << 8); - nr2+=(nr2 << 8) ^ nr; - add+=tmp; + tmp = (ulong)(unsigned char)*password; + nr ^= (((nr & 63) + add) * tmp) + (nr << 8); + nr2 += (nr2 << 8) ^ nr; + add += tmp; } - result[0]=nr & (((ulong) 1L << 31) -1L); - result[1]=nr2 & (((ulong) 1L << 31) -1L); + result[0] = nr & (((ulong)1L << 31) - 1L); + result[1] = nr2 & (((ulong)1L << 31) - 1L); } -CalpontSystemCatalog::ColType Func_decode::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_decode::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_decode::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +string Func_decode::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - const string& str = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - { - return ""; - } - const string& password = parm[1]->data()->getStrVal(row, isNull); - if (isNull) - { - return ""; - } + const string& str = parm[0]->data()->getStrVal(row, isNull); + if (isNull) + { + return ""; + } + const string& password = parm[1]->data()->getStrVal(row, isNull); + if (isNull) + { + return ""; + } - int nStrLen = str.length(); - int nPassLen = password.length(); - utils::VLArray res(nStrLen + 1); - memset(res.data(),0,nStrLen+1); + int nStrLen = str.length(); + int nPassLen = password.length(); + utils::VLArray res(nStrLen + 1); + memset(res.data(), 0, nStrLen + 1); - if (!fSeeded) - { - hash_password(fSeeds, password.c_str(), nPassLen); - sql_crypt.init(fSeeds); - fSeeded = true; - } + if (!fSeeded) + { + hash_password(fSeeds, password.c_str(), nPassLen); + sql_crypt.init(fSeeds); + fSeeded = true; + } - memcpy(res.data(),str.c_str(),nStrLen); - sql_crypt.decode(res.data(),nStrLen); - sql_crypt.reinit(); + memcpy(res.data(), str.c_str(), nStrLen); + sql_crypt.decode(res.data(), nStrLen); + sql_crypt.reinit(); - return res.data(); + return res.data(); } -} +} // namespace funcexp diff --git a/utils/funcexp/func_decode_oracle.cpp b/utils/funcexp/func_decode_oracle.cpp index 2caf2fa15..0e53e16d8 100644 --- a/utils/funcexp/func_decode_oracle.cpp +++ b/utils/funcexp/func_decode_oracle.cpp @@ -35,385 +35,378 @@ using namespace logging; using namespace funcexp; - namespace { using namespace funcexp; -inline uint64_t simple_case_cmp(Row& row, - FunctionParm& parm, - bool& isNull, +inline uint64_t simple_case_cmp(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = 0; // index to the parm list - uint64_t n = 0; // remove expression from count of expression_i + result_i - uint64_t hasElse = (parm.size() - 1) % 2; // if 1, then ELSE exist - uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; - bool foundIt = false; + uint64_t i = 0; // index to the parm list + uint64_t n = 0; // remove expression from count of expression_i + result_i + uint64_t hasElse = (parm.size() - 1) % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; + bool foundIt = false; - switch (operationColType.colDataType) + switch (operationColType.colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DATE: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DATE: + int64_t ev = parm[n]->data()->getIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) { - int64_t ev = parm[n]->data()->getIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t ev = parm[n]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getTimestampIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t ev = parm[n]->data()->getUintVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARCHAR: - { - const string& ev = parm[n]->data()->getStrVal(row, isNull); - if (isNull) - break; - CHARSET_INFO* cs = parm[n]->data()->resultType().getCharset(); - - for (i = 1; i <= whereCount; i++) - { - //BUG 5362 - const string& p1 = parm[i]->data()->getStrVal(row, isNull); - if (isNull) - break; - if (cs->strnncoll(ev.c_str(), ev.length(), p1.c_str(), p1.length()) == 0) - { - foundIt = true; - break; - } - } - - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal ev = parm[n]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double ev = parm[n]->data()->getDoubleVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float ev = parm[n]->data()->getFloatVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double ev = parm[n]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - break; - - for (i = 1; i <= whereCount; i++) - { - if (ev == parm[i]->data()->getLongDoubleVal(row, isNull) && !isNull) - { - foundIt = true; - break; - } - else - isNull = false; - } - - break; - } - - default: - { - std::ostringstream oss; - oss << "case: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - } - - if (!foundIt && !hasElse) - isNull = true; - else if (!foundIt && hasElse && !isNull) - { - i = parm.size() - 1; - } - else if (isNull && hasElse) - // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL - // if so and we have else condition we need to use it by setting i = else - { - i = parm.size() - 1; - isNull = false; - } - - if (foundIt) - { - i += whereCount; - } - - return i; -} - -CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, - CalpontSystemCatalog::ColType& resultType, - bool simpleCase) -{ - uint64_t simple = simpleCase ? 1 : 0; - bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist - - uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); - uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; - - bool allStringO = true; - bool allStringR = true; - - FunctionParm::size_type l = fp.size() - 1; // last fp index - idbassert(fp[l]->data()); - CalpontSystemCatalog::ColType oct = fp[l]->data()->resultType(); - CalpontSystemCatalog::ColType rct = resultType; - bool operation = true; - - for (uint64_t i = 0; i <= parmCount; i++) - { - // for SimpleCase, we return the type of the case expression, - // which will always be in position 0. - if (i == 0 && simpleCase) - { - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) - { - PredicateOperator op; - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } - - i += 1; - } - - // operation or result type - operation = ((i > 0+simple) && (i <= whereCount)); - - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) - { - // this is not a string column - PredicateOperator op; - - if (operation) - { - if (!simpleCase) - { - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } - } - - // If any parm is of string type, the result type should be string. (same as if) - else if (rct.colDataType != CalpontSystemCatalog::CHAR && - rct.colDataType != CalpontSystemCatalog::TEXT && - rct.colDataType != CalpontSystemCatalog::VARCHAR) - { - op.setOpType(rct, fp[i]->data()->resultType()); - allStringR = false; - rct = op.operationType(); - } + foundIt = true; + break; } else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) { - // this is a string - // If any parm is of string type, the result type should be string. (same as if) - allStringR = true; + foundIt = true; + break; } + else + isNull = false; + } + + break; } - if (allStringO) + case execplan::CalpontSystemCatalog::TIMESTAMP: { - oct.colDataType = CalpontSystemCatalog::VARCHAR; - oct.colWidth = 255; + int64_t ev = parm[n]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimestampIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; } - if (allStringR) + case execplan::CalpontSystemCatalog::TIME: { - rct.colDataType = CalpontSystemCatalog::VARCHAR; - rct.colWidth = 255; + int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; } - if (rct.scale != 0 && rct.colDataType == CalpontSystemCatalog::BIGINT) - rct.colDataType = CalpontSystemCatalog::DECIMAL; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t ev = parm[n]->data()->getUintVal(row, isNull); - if (oct.scale != 0 && oct.colDataType == CalpontSystemCatalog::BIGINT) - oct.colDataType = CalpontSystemCatalog::DECIMAL; + if (isNull) + break; - resultType = rct; - return oct; + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARCHAR: + { + const string& ev = parm[n]->data()->getStrVal(row, isNull); + if (isNull) + break; + CHARSET_INFO* cs = parm[n]->data()->resultType().getCharset(); + + for (i = 1; i <= whereCount; i++) + { + // BUG 5362 + const string& p1 = parm[i]->data()->getStrVal(row, isNull); + if (isNull) + break; + if (cs->strnncoll(ev.c_str(), ev.length(), p1.c_str(), p1.length()) == 0) + { + foundIt = true; + break; + } + } + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal ev = parm[n]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double ev = parm[n]->data()->getDoubleVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float ev = parm[n]->data()->getFloatVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double ev = parm[n]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getLongDoubleVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + default: + { + std::ostringstream oss; + oss << "case: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + if (!foundIt && !hasElse) + isNull = true; + else if (!foundIt && hasElse && !isNull) + { + i = parm.size() - 1; + } + else if (isNull && hasElse) + // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL + // if so and we have else condition we need to use it by setting i = else + { + i = parm.size() - 1; + isNull = false; + } + + if (foundIt) + { + i += whereCount; + } + + return i; } +CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, + bool simpleCase) +{ + uint64_t simple = simpleCase ? 1 : 0; + bool hasElse = (((fp.size() - simple) % 2) != 0); // if 1, then ELSE exist + + uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; + + bool allStringO = true; + bool allStringR = true; + + FunctionParm::size_type l = fp.size() - 1; // last fp index + idbassert(fp[l]->data()); + CalpontSystemCatalog::ColType oct = fp[l]->data()->resultType(); + CalpontSystemCatalog::ColType rct = resultType; + bool operation = true; + + for (uint64_t i = 0; i <= parmCount; i++) + { + // for SimpleCase, we return the type of the case expression, + // which will always be in position 0. + if (i == 0 && simpleCase) + { + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + { + PredicateOperator op; + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } + + i += 1; + } + + // operation or result type + operation = ((i > 0 + simple) && (i <= whereCount)); + + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + { + // this is not a string column + PredicateOperator op; + + if (operation) + { + if (!simpleCase) + { + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } + } + + // If any parm is of string type, the result type should be string. (same as if) + else if (rct.colDataType != CalpontSystemCatalog::CHAR && + rct.colDataType != CalpontSystemCatalog::TEXT && + rct.colDataType != CalpontSystemCatalog::VARCHAR) + { + op.setOpType(rct, fp[i]->data()->resultType()); + allStringR = false; + rct = op.operationType(); + } + } + else + { + // this is a string + // If any parm is of string type, the result type should be string. (same as if) + allStringR = true; + } + } + + if (allStringO) + { + oct.colDataType = CalpontSystemCatalog::VARCHAR; + oct.colWidth = 255; + } + + if (allStringR) + { + rct.colDataType = CalpontSystemCatalog::VARCHAR; + rct.colWidth = 255; + } + + if (rct.scale != 0 && rct.colDataType == CalpontSystemCatalog::BIGINT) + rct.colDataType = CalpontSystemCatalog::DECIMAL; + + if (oct.scale != 0 && oct.colDataType == CalpontSystemCatalog::BIGINT) + oct.colDataType = CalpontSystemCatalog::DECIMAL; + + resultType = rct; + return oct; } +} // namespace namespace funcexp { - // simple CASE: // SELECT CASE ("expression") // WHEN "condition1" THEN "result1" @@ -427,157 +420,127 @@ namespace funcexp // // Note that this order changed in 10.2.14, see MCOL-1341 -CalpontSystemCatalog::ColType Func_decode_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_decode_oracle::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return caseOperationType(fp, resultType, true); + return caseOperationType(fp, resultType, true); } - -bool Func_decode_oracle::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) -{ - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - - if (isNull) - return false; - - ParseTree* lop = parm[i]->left(); - ParseTree* rop = parm[i]->right(); - if (lop && rop) - { - return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); - } - - return parm[i]->data()->getBoolVal(row, isNull); -} - - -int64_t Func_decode_oracle::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +bool Func_decode_oracle::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return joblist::BIGINTNULL; + if (isNull) + return false; - return parm[i]->data()->getIntVal(row, isNull); + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); + if (lop && rop) + { + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); + } + + return parm[i]->data()->getBoolVal(row, isNull); } - -string Func_decode_oracle::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) -{ - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - - if (isNull) - return string(""); - - return parm[i]->data()->getStrVal(row, isNull); -} - - -IDB_Decimal Func_decode_oracle::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) -{ - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - - if (isNull) - return IDB_Decimal(); // need a null value for IDB_Decimal?? - - return parm[i]->data()->getDecimalVal(row, isNull); -} - - -double Func_decode_oracle::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_decode_oracle::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return doubleNullVal(); + if (isNull) + return joblist::BIGINTNULL; - return parm[i]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } -long double Func_decode_oracle::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +string Func_decode_oracle::getStrVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return doubleNullVal(); + if (isNull) + return string(""); - return parm[i]->data()->getLongDoubleVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } - -int32_t Func_decode_oracle::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +IDB_Decimal Func_decode_oracle::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return joblist::DATENULL; + if (isNull) + return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } - -int64_t Func_decode_oracle::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +double Func_decode_oracle::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return joblist::DATETIMENULL; + if (isNull) + return doubleNullVal(); - return parm[i]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } - -int64_t Func_decode_oracle::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +long double Func_decode_oracle::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); - if (isNull) - return joblist::TIMESTAMPNULL; + if (isNull) + return doubleNullVal(); - return parm[i]->data()->getTimestampIntVal(row, isNull); + return parm[i]->data()->getLongDoubleVal(row, isNull); } - -int64_t Func_decode_oracle::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int32_t Func_decode_oracle::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); - if (isNull) - return joblist::TIMENULL; + if (isNull) + return joblist::DATENULL; - return parm[i]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } +int64_t Func_decode_oracle::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + if (isNull) + return joblist::DATETIMENULL; + return parm[i]->data()->getDatetimeIntVal(row, isNull); } + +int64_t Func_decode_oracle::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMESTAMPNULL; + + return parm[i]->data()->getTimestampIntVal(row, isNull); +} + +int64_t Func_decode_oracle::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i]->data()->getTimeIntVal(row, isNull); +} + +} // namespace funcexp diff --git a/utils/funcexp/func_div.cpp b/utils/funcexp/func_div.cpp index b82cfaa4b..7169992bb 100644 --- a/utils/funcexp/func_div.cpp +++ b/utils/funcexp/func_div.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_div.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_div.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,27 +38,25 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_div::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_div::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_div::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_div::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - double val1 = parm[0]->data()->getDoubleVal(row, isNull); - double val2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (val2 == 0 || val2 == NAN) - { - isNull = true; - return 0; - } - // MCOL-179 InnoDB doesn't round or convert to int before dividing. - return static_cast(val1 / val2); + double val1 = parm[0]->data()->getDoubleVal(row, isNull); + double val2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (val2 == 0 || val2 == NAN) + { + isNull = true; + return 0; + } + // MCOL-179 InnoDB doesn't round or convert to int before dividing. + return static_cast(val1 / val2); #if 0 int64_t int_val2 = (int64_t)(val2 > 0 ? val2 + 0.5 : val2 - 0.5); @@ -79,45 +77,35 @@ int64_t Func_div::getIntVal(rowgroup::Row& row, } return int_val1 / int_val2; -#endif +#endif } - -uint64_t Func_div::getUintVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_div::getUintVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t val1 = parm[0]->data()->getUintVal(row, isNull); - uint64_t val2 = parm[1]->data()->getUintVal(row, isNull); + uint64_t val1 = parm[0]->data()->getUintVal(row, isNull); + uint64_t val2 = parm[1]->data()->getUintVal(row, isNull); - if (val2 == 0) - { - isNull = true; - return 0; - } + if (val2 == 0) + { + isNull = true; + return 0; + } - return val1 / val2; + return val1 / val2; } - -double Func_div::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_div::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - return getIntVal(row, parm, isNull, ct); + return getIntVal(row, parm, isNull, ct); } - -string Func_div::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_div::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return intToString(getIntVal(row, parm, isNull, ct)); + return intToString(getIntVal(row, parm, isNull, ct)); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index 125340f29..22446095f 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_elt.cpp 2665 2011-06-01 20:42:52Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_elt.cpp 2665 2011-06-01 20:42:52Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -38,68 +38,62 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_elt::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_elt::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_elt::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_elt::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - uint64_t number = 0; + uint64_t number = 0; - //get number - switch (parm[0]->data()->resultType().colDataType) + // get number + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - number = (int64_t) value; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - number = static_cast(parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round()); - break; - } - - default: - isNull = true; - return ""; + double value = parm[0]->data()->getDoubleVal(row, isNull); + number = (int64_t)value; + break; } - if (number < 1) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - isNull = true; - return ""; + number = static_cast(parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round()); + break; } - if (number > parm.size() - 1 ) - { - isNull = true; - return ""; - } + default: isNull = true; return ""; + } - std::string ret; - stringValue(parm[number], row, isNull, ret); - return ret; + if (number < 1) + { + isNull = true; + return ""; + } + if (number > parm.size() - 1) + { + isNull = true; + return ""; + } + + std::string ret; + stringValue(parm[number], row, isNull, ret); + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_encode.cpp b/utils/funcexp/func_encode.cpp index 5d6d1f05f..dcfc40e83 100644 --- a/utils/funcexp/func_encode.cpp +++ b/utils/funcexp/func_encode.cpp @@ -28,66 +28,61 @@ using namespace execplan; namespace funcexp { - -void Func_encode::hash_password(ulong *result, const char *password, uint password_len) +void Func_encode::hash_password(ulong* result, const char* password, uint password_len) { - ulong nr=1345345333L, add=7, nr2=0x12345671L; + ulong nr = 1345345333L, add = 7, nr2 = 0x12345671L; ulong tmp; - const char *password_end= password + password_len; + const char* password_end = password + password_len; for (; password < password_end; password++) { if (*password == ' ' || *password == '\t') continue; - tmp= (ulong) (unsigned char) *password; - nr^= (((nr & 63)+add)*tmp)+ (nr << 8); - nr2+=(nr2 << 8) ^ nr; - add+=tmp; + tmp = (ulong)(unsigned char)*password; + nr ^= (((nr & 63) + add) * tmp) + (nr << 8); + nr2 += (nr2 << 8) ^ nr; + add += tmp; } - result[0]=nr & (((ulong) 1L << 31) -1L); - result[1]=nr2 & (((ulong) 1L << 31) -1L); + result[0] = nr & (((ulong)1L << 31) - 1L); + result[1] = nr2 & (((ulong)1L << 31) - 1L); } -CalpontSystemCatalog::ColType Func_encode::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_encode::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_encode::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +string Func_encode::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { + const string& str = parm[0]->data()->getStrVal(row, isNull); + if (isNull) + { + return ""; + } + const string& password = parm[1]->data()->getStrVal(row, isNull); + if (isNull) + { + return ""; + } + int nStrLen = str.length(); + int nPassLen = password.length(); + utils::VLArray res(nStrLen + 1); + memset(res.data(), 0, nStrLen + 1); - const string& str = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - { - return ""; - } - const string& password = parm[1]->data()->getStrVal(row, isNull); - if (isNull) - { - return ""; - } + if (!fSeeded) + { + hash_password(fSeeds, password.c_str(), nPassLen); + sql_crypt.init(fSeeds); + fSeeded = true; + } - int nStrLen = str.length(); - int nPassLen = password.length(); - utils::VLArray res(nStrLen + 1); - memset(res.data(),0,nStrLen+1); + memcpy(res.data(), str.c_str(), nStrLen); + sql_crypt.encode(res.data(), nStrLen); + sql_crypt.reinit(); - - if (!fSeeded) - { - hash_password(fSeeds, password.c_str(), nPassLen); - sql_crypt.init(fSeeds); - fSeeded = true; - } - - memcpy(res.data(),str.c_str(),nStrLen); - sql_crypt.encode(res.data(),nStrLen); - sql_crypt.reinit(); - - return res.data(); + return res.data(); } -} +} // namespace funcexp diff --git a/utils/funcexp/func_exp.cpp b/utils/funcexp/func_exp.cpp index 217102daf..bab2fc926 100644 --- a/utils/funcexp/func_exp.cpp +++ b/utils/funcexp/func_exp.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_exp.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_exp.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -41,79 +41,73 @@ using namespace logging; namespace funcexp { - -CalpontSystemCatalog::ColType Func_exp::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_exp::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_exp::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_exp::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - // null value is indicated by isNull - double x = parm[0]->data()->getDoubleVal(row, isNull); - double ret = 0.0; + // null value is indicated by isNull + double x = parm[0]->data()->getDoubleVal(row, isNull); + double ret = 0.0; - if (!isNull) + if (!isNull) + { + errno = 0; + ret = exp(x); + + if (errno == ERANGE) // display NULL for out range value { - errno = 0; - ret = exp(x); - - if (errno == ERANGE) // display NULL for out range value - { - if (x > 0) - { - isNull = true; - Message::Args args; - args.add("exp"); - args.add(x); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - else - ret = 0.0; - } + if (x > 0) + { + isNull = true; + Message::Args args; + args.add("exp"); + args.add(x); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + else + ret = 0.0; } + } - return ret; + return ret; } -long double Func_exp::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +long double Func_exp::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - // null value is indicated by isNull - long double x = parm[0]->data()->getLongDoubleVal(row, isNull); - long double ret = 0.0; + // null value is indicated by isNull + long double x = parm[0]->data()->getLongDoubleVal(row, isNull); + long double ret = 0.0; - if (!isNull) + if (!isNull) + { + errno = 0; + ret = expl(x); + + if (errno == ERANGE) // display NULL for out range value { - errno = 0; - ret = expl(x); - - if (errno == ERANGE) // display NULL for out range value - { - if (x > 0) - { - isNull = true; - Message::Args args; - args.add("exp"); - args.add((double)x); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - else - ret = 0.0; - } + if (x > 0) + { + isNull = true; + Message::Args args; + args.add("exp"); + args.add((double)x); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + else + ret = 0.0; } + } - return ret; + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index abb45049d..066151aa3 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_extract.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_extract.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,270 +38,220 @@ namespace { using namespace funcexp; -long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateType ) +long long dateGet(uint64_t time, IntervalColumn::interval_type unit, bool dateType) { - uint64_t year = 0, - month = 0, - day = 0, - hour = 0, - min = 0, - sec = 0, - msec = 0; + uint64_t year = 0, month = 0, day = 0, hour = 0, min = 0, sec = 0, msec = 0; - if (dateType) - { - year = (uint32_t)((time >> 16) & 0xffff); - month = (uint32_t)((time >> 12) & 0xf); - day = (uint32_t)((time >> 6) & 0x3f); - } - else - { - year = (uint32_t)((time >> 48) & 0xffff); - month = (uint32_t)((time >> 44) & 0xf); - day = (uint32_t)((time >> 38) & 0x3f); - hour = (uint32_t)((time >> 32) & 0x3f); - min = (uint32_t)((time >> 26) & 0x3f); - sec = (uint32_t)((time >> 20) & 0x3f); - msec = (uint32_t)((time & 0xfffff)); - } + if (dateType) + { + year = (uint32_t)((time >> 16) & 0xffff); + month = (uint32_t)((time >> 12) & 0xf); + day = (uint32_t)((time >> 6) & 0x3f); + } + else + { + year = (uint32_t)((time >> 48) & 0xffff); + month = (uint32_t)((time >> 44) & 0xf); + day = (uint32_t)((time >> 38) & 0x3f); + hour = (uint32_t)((time >> 32) & 0x3f); + min = (uint32_t)((time >> 26) & 0x3f); + sec = (uint32_t)((time >> 20) & 0x3f); + msec = (uint32_t)((time & 0xfffff)); + } - switch ( unit ) - { - case IntervalColumn::INTERVAL_YEAR: - return year; + switch (unit) + { + case IntervalColumn::INTERVAL_YEAR: return year; - case IntervalColumn::INTERVAL_MONTH: - return month; + case IntervalColumn::INTERVAL_MONTH: return month; - case IntervalColumn::INTERVAL_DAY: - return day; + case IntervalColumn::INTERVAL_DAY: return day; - case IntervalColumn::INTERVAL_HOUR: - return hour; + case IntervalColumn::INTERVAL_HOUR: return hour; - case IntervalColumn::INTERVAL_MINUTE: - return min; + case IntervalColumn::INTERVAL_MINUTE: return min; - case IntervalColumn::INTERVAL_SECOND: - return sec; + case IntervalColumn::INTERVAL_SECOND: return sec; - case IntervalColumn::INTERVAL_MICROSECOND: - return msec; + case IntervalColumn::INTERVAL_MICROSECOND: return msec; - case IntervalColumn::INTERVAL_QUARTER: - return month / 4 + 1; + case IntervalColumn::INTERVAL_QUARTER: return month / 4 + 1; - case IntervalColumn::INTERVAL_WEEK: - return helpers::calc_mysql_week(year, month, day, 0); + case IntervalColumn::INTERVAL_WEEK: return helpers::calc_mysql_week(year, month, day, 0); - case IntervalColumn::INTERVAL_YEAR_MONTH: - return (year * 100) + month; + case IntervalColumn::INTERVAL_YEAR_MONTH: return (year * 100) + month; - case IntervalColumn::INTERVAL_DAY_HOUR: - return (day * 100) + hour; + case IntervalColumn::INTERVAL_DAY_HOUR: return (day * 100) + hour; - case IntervalColumn::INTERVAL_DAY_MINUTE: - return (day * 10000) + (hour * 100) + min; + case IntervalColumn::INTERVAL_DAY_MINUTE: return (day * 10000) + (hour * 100) + min; - case IntervalColumn::INTERVAL_DAY_SECOND: - return (day * 1000000) + (hour * 10000) + (min * 100) + sec; + case IntervalColumn::INTERVAL_DAY_SECOND: return (day * 1000000) + (hour * 10000) + (min * 100) + sec; - case IntervalColumn::INTERVAL_HOUR_MINUTE: - return (hour * 100) + min; + case IntervalColumn::INTERVAL_HOUR_MINUTE: return (hour * 100) + min; - case IntervalColumn::INTERVAL_HOUR_SECOND: - return (hour * 10000) + (min * 100) + sec; + case IntervalColumn::INTERVAL_HOUR_SECOND: return (hour * 10000) + (min * 100) + sec; - case IntervalColumn::INTERVAL_MINUTE_SECOND: - return (min * 100) + sec; + case IntervalColumn::INTERVAL_MINUTE_SECOND: return (min * 100) + sec; - case IntervalColumn::INTERVAL_DAY_MICROSECOND: - return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; - case IntervalColumn::INTERVAL_HOUR_MICROSECOND: - return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; - case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: - return (((min * 100) + sec) * 1000000) + msec; + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: return (((min * 100) + sec) * 1000000) + msec; - case IntervalColumn::INTERVAL_SECOND_MICROSECOND: - return (sec * 1000000) + msec; + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: return (sec * 1000000) + msec; - default: - throw runtime_error(std::string("unit type is not supported: ") + std::to_string(unit)); - }; + default: throw runtime_error(std::string("unit type is not supported: ") + std::to_string(unit)); + }; } -long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) +long long timeGet(uint64_t time, IntervalColumn::interval_type unit) { - int32_t hour = 0, - min = 0, - sec = 0, - msec = 0, - day = 0; + int32_t hour = 0, min = 0, sec = 0, msec = 0, day = 0; - min = (int32_t)((time >> 32) & 0xff); - sec = (int32_t)((time >> 24) & 0xff); - msec = (int32_t)((time & 0xfffff)); + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); - // If negative, mask so it doesn't turn positive - int64_t mask = 0; + // If negative, mask so it doesn't turn positive + int64_t mask = 0; - if ((time >> 40) & 0x800) - mask = 0xfffffffffffff000; + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; - hour = mask | ((time >> 40) & 0xfff); + hour = mask | ((time >> 40) & 0xfff); - if ((hour >= 0) && (time >> 63)) - hour *= -1; + if ((hour >= 0) && (time >> 63)) + hour *= -1; - // Always positive! - day = abs(hour / 24); + // Always positive! + day = abs(hour / 24); - switch ( unit ) - { - case IntervalColumn::INTERVAL_YEAR: - case IntervalColumn::INTERVAL_MONTH: - return 0; + switch (unit) + { + case IntervalColumn::INTERVAL_YEAR: + case IntervalColumn::INTERVAL_MONTH: return 0; - case IntervalColumn::INTERVAL_DAY: - return day; + case IntervalColumn::INTERVAL_DAY: return day; - case IntervalColumn::INTERVAL_HOUR: - return hour; + case IntervalColumn::INTERVAL_HOUR: return hour; - case IntervalColumn::INTERVAL_MINUTE: - return min; + case IntervalColumn::INTERVAL_MINUTE: return min; - case IntervalColumn::INTERVAL_SECOND: - return sec; + case IntervalColumn::INTERVAL_SECOND: return sec; - case IntervalColumn::INTERVAL_MICROSECOND: - return msec; + case IntervalColumn::INTERVAL_MICROSECOND: return msec; - case IntervalColumn::INTERVAL_QUARTER: - case IntervalColumn::INTERVAL_WEEK: - case IntervalColumn::INTERVAL_YEAR_MONTH: - return 0; + case IntervalColumn::INTERVAL_QUARTER: + case IntervalColumn::INTERVAL_WEEK: + case IntervalColumn::INTERVAL_YEAR_MONTH: return 0; - case IntervalColumn::INTERVAL_DAY_HOUR: - return (day * 100) + hour; + case IntervalColumn::INTERVAL_DAY_HOUR: return (day * 100) + hour; - case IntervalColumn::INTERVAL_DAY_MINUTE: - return (day * 10000) + (hour * 100) + min; + case IntervalColumn::INTERVAL_DAY_MINUTE: return (day * 10000) + (hour * 100) + min; - case IntervalColumn::INTERVAL_DAY_SECOND: - return (day * 1000000) + (hour * 10000) + (min * 100) + sec; + case IntervalColumn::INTERVAL_DAY_SECOND: return (day * 1000000) + (hour * 10000) + (min * 100) + sec; - case IntervalColumn::INTERVAL_HOUR_MINUTE: - return (hour * 100) + min; + case IntervalColumn::INTERVAL_HOUR_MINUTE: return (hour * 100) + min; - case IntervalColumn::INTERVAL_HOUR_SECOND: - return (hour * 10000) + (min * 100) + sec; + case IntervalColumn::INTERVAL_HOUR_SECOND: return (hour * 10000) + (min * 100) + sec; - case IntervalColumn::INTERVAL_MINUTE_SECOND: - return (min * 100) + sec; + case IntervalColumn::INTERVAL_MINUTE_SECOND: return (min * 100) + sec; - case IntervalColumn::INTERVAL_DAY_MICROSECOND: - return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; - case IntervalColumn::INTERVAL_HOUR_MICROSECOND: - return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; - case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: - return (((min * 100) + sec) * 1000000) + msec; + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: return (((min * 100) + sec) * 1000000) + msec; - case IntervalColumn::INTERVAL_SECOND_MICROSECOND: - return (sec * 1000000) + msec; - - default: - throw runtime_error(std::string("unit type is not supported: ") + std::to_string(unit)); - }; }; -} + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: return (sec * 1000000) + msec; + default: throw runtime_error(std::string("unit type is not supported: ") + std::to_string(unit)); + }; +}; +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_extract::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_extract::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_extract::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_extract::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - IntervalColumn::interval_type unit = static_cast(parm[1]->data()->getIntVal(row, isNull)); - uint64_t time; - bool isTime = false; + IntervalColumn::interval_type unit = + static_cast(parm[1]->data()->getIntVal(row, isNull)); + uint64_t time; + bool isTime = false; - //@bug4678 handle conversion from non date/datetime datatype - switch (parm[0]->data()->resultType().colDataType) + //@bug4678 handle conversion from non date/datetime datatype + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - time = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - dataconvert::DateTime dt; - dt.year = m_time.year; - dt.month = m_time.month; - dt.day = m_time.day; - dt.hour = m_time.hour; - dt.minute = m_time.minute; - dt.second = m_time.second; - dt.msecond = timestamp.msecond; - time = *(reinterpret_cast(&dt)); - break; - } - - case CalpontSystemCatalog::TIME: - time = parm[0]->data()->getTimeIntVal(row, isNull); - isTime = true; - break; - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - { - const string& val = parm[0]->data()->getStrVal(row, isNull); - time = dataconvert::DataConvert::stringToDatetime(val); - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::SMALLINT: - { - int64_t val = parm[0]->data()->getIntVal(row, isNull); - time = dataconvert::DataConvert::intToDatetime(val); - break; - } - - default: - time = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = m_time.hour; + dt.minute = m_time.minute; + dt.second = m_time.second; + dt.msecond = timestamp.msecond; + time = *(reinterpret_cast(&dt)); + break; } - long long value; + case CalpontSystemCatalog::TIME: + time = parm[0]->data()->getTimeIntVal(row, isNull); + isTime = true; + break; - if (isTime) - value = timeGet( time, unit ); - else - value = dateGet( time, unit, false ); + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + { + const string& val = parm[0]->data()->getStrVal(row, isNull); + time = dataconvert::DataConvert::stringToDatetime(val); + break; + } - return value; + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::SMALLINT: + { + int64_t val = parm[0]->data()->getIntVal(row, isNull); + time = dataconvert::DataConvert::intToDatetime(val); + break; + } + + default: time = parm[0]->data()->getIntVal(row, isNull); + } + + long long value; + + if (isTime) + value = timeGet(time, unit); + else + value = dateGet(time, unit, false); + + return value; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_find_in_set.cpp b/utils/funcexp/func_find_in_set.cpp index a272e0bc3..94c1243dc 100644 --- a/utils/funcexp/func_find_in_set.cpp +++ b/utils/funcexp/func_find_in_set.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_find_in_set.cpp 2675 2011-06-22 04:58:07Z chao $ -* -* -****************************************************************************/ + * $Id: func_find_in_set.cpp 2675 2011-06-22 04:58:07Z chao $ + * + * + ****************************************************************************/ #include #include @@ -42,103 +42,89 @@ using namespace execplan; #include "errorids.h" using namespace logging; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_find_in_set::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_find_in_set::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_find_in_set::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_find_in_set::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - const string& searchStr = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - return 0; - - const string& setString = parm[1]->data()->getStrVal(row, isNull); - if (isNull) - return 0; - - if (searchStr.find(",") != string::npos) - return 0; - - if (setString.length() < searchStr.length()) - return 0; - - CHARSET_INFO *cs= op_ct.getCharset(); - - my_wc_t wc= 0; - const char *str_begin = setString.c_str(); - const char *str_end = setString.c_str(); - const char *real_end = str_end + setString.length(); - const char *find_str = searchStr.c_str(); - size_t find_str_len = searchStr.length(); - int position = 0; - static const char separator=','; - while (1) - { - int symbol_len; - if ((symbol_len= cs->mb_wc(&wc, (uchar*) str_end, - (uchar*) real_end)) > 0) - { - const char *substr_end = str_end + symbol_len; - bool is_last_item= (substr_end == real_end); - bool is_separator = (wc == (my_wc_t) separator); - if (is_separator || is_last_item) - { - position++; - if (is_last_item && !is_separator) - str_end = substr_end; - if (!cs->strnncoll(str_begin, (size_t) (str_end - str_begin), - find_str, find_str_len)) - return (int64_t) position; - else - str_begin = substr_end; - } - str_end = substr_end; - } - else if (str_end - str_begin == 0 && - find_str_len == 0 && - wc == (my_wc_t) separator) - return (longlong) ++position; - else - return 0; - } + const string& searchStr = parm[0]->data()->getStrVal(row, isNull); + if (isNull) return 0; + + const string& setString = parm[1]->data()->getStrVal(row, isNull); + if (isNull) + return 0; + + if (searchStr.find(",") != string::npos) + return 0; + + if (setString.length() < searchStr.length()) + return 0; + + CHARSET_INFO* cs = op_ct.getCharset(); + + my_wc_t wc = 0; + const char* str_begin = setString.c_str(); + const char* str_end = setString.c_str(); + const char* real_end = str_end + setString.length(); + const char* find_str = searchStr.c_str(); + size_t find_str_len = searchStr.length(); + int position = 0; + static const char separator = ','; + while (1) + { + int symbol_len; + if ((symbol_len = cs->mb_wc(&wc, (uchar*)str_end, (uchar*)real_end)) > 0) + { + const char* substr_end = str_end + symbol_len; + bool is_last_item = (substr_end == real_end); + bool is_separator = (wc == (my_wc_t)separator); + if (is_separator || is_last_item) + { + position++; + if (is_last_item && !is_separator) + str_end = substr_end; + if (!cs->strnncoll(str_begin, (size_t)(str_end - str_begin), find_str, find_str_len)) + return (int64_t)position; + else + str_begin = substr_end; + } + str_end = substr_end; + } + else if (str_end - str_begin == 0 && find_str_len == 0 && wc == (my_wc_t)separator) + return (longlong)++position; + else + return 0; + } + return 0; } -double Func_find_in_set::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_find_in_set::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - return (double)getIntVal(row, parm, isNull, ct); + return (double)getIntVal(row, parm, isNull, ct); } - -string Func_find_in_set::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_find_in_set::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return intToString(getIntVal(row, parm, isNull, ct)); + return intToString(getIntVal(row, parm, isNull, ct)); } -execplan::IDB_Decimal Func_find_in_set::getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +execplan::IDB_Decimal Func_find_in_set::getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal decimal; - decimal.value = getIntVal(row, fp, isNull, op_ct); - decimal.scale = op_ct.scale; - return decimal; + IDB_Decimal decimal; + decimal.value = getIntVal(row, fp, isNull, op_ct); + decimal.scale = op_ct.scale; + return decimal; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index dde7d04ba..c212d19c0 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -18,7 +18,6 @@ // $Id: func_floor.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ - #include #include #include @@ -41,546 +40,519 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_floor::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_floor::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return fp[0]->data()->resultType(); + return fp[0]->data()->resultType(); } - -int64_t Func_floor::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_floor::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t ret = 0; + int64_t ret = 0; - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - ret = parm[0]->data()->getIntVal(row, isNull); - } - break; + ret = parm[0]->data()->getIntVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - ret = (int64_t)parm[0]->data()->getUintVal(row, isNull); - } - break; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - ret = (int64_t) floor(parm[0]->data()->getDoubleVal(row, isNull)); - } - break; + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret = (int64_t)floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - ret = (int64_t) floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret = (int64_t)floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& str = parm[0]->data()->getStrVal(row, isNull); + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); - if (!isNull) - ret = (int64_t) floor(strtod(str.c_str(), 0)); - } - break; + if (!isNull) + ret = (int64_t)floor(strtod(str.c_str(), 0)); + } + break; - case execplan::CalpontSystemCatalog::DATE: - { - // For some reason, MDB doesn't return this as a date, - // but datetime is returned as a datetime. Expect - // this to change in the future. - Date d (parm[0]->data()->getDateIntVal(row, isNull)); - if (!isNull) - ret = d.convertToMySQLint(); - } - break; + case execplan::CalpontSystemCatalog::DATE: + { + // For some reason, MDB doesn't return this as a date, + // but datetime is returned as a datetime. Expect + // this to change in the future. + Date d(parm[0]->data()->getDateIntVal(row, isNull)); + if (!isNull) + ret = d.convertToMySQLint(); + } + break; - case execplan::CalpontSystemCatalog::DATETIME: - { - ret = parm[0]->data()->getDatetimeIntVal(row, isNull); + case execplan::CalpontSystemCatalog::DATETIME: + { + ret = parm[0]->data()->getDatetimeIntVal(row, isNull); + } + break; - } - break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + ret = parm[0]->data()->getTimestampIntVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - ret = parm[0]->data()->getTimestampIntVal(row, isNull); - } - break; + case execplan::CalpontSystemCatalog::TIME: + { + ret = parm[0]->data()->getTimeIntVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::TIME: - { - ret = parm[0]->data()->getTimeIntVal(row, isNull); - } - break; - - // floor(decimal(X,Y)) leads to this path if X, Y allows to - // downcast to INT otherwise Func_floor::getDecimalVal() is called - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - ret = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Floor(); - break; - } - - default: - { - std::ostringstream oss; - oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + // floor(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_floor::getDecimalVal() is called + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Floor(); + break; } - return ret; + default: + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return ret; } - -uint64_t Func_floor::getUintVal(Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_floor::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t ret = 0; + int64_t ret = 0; - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - ret = parm[0]->data()->getIntVal(row, isNull); - } - break; + ret = parm[0]->data()->getIntVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - ret = (int64_t)parm[0]->data()->getUintVal(row, isNull); - } - break; + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - ret = (uint64_t)floor(parm[0]->data()->getDoubleVal(row, isNull)); - } - break; + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret = (uint64_t)floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - ret = (uint64_t) floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret = (uint64_t)floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& str = parm[0]->data()->getStrVal(row, isNull); + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); - if (!isNull) - ret = (uint64_t)floor(strtod(str.c_str(), 0)); - } - break; + if (!isNull) + ret = (uint64_t)floor(strtod(str.c_str(), 0)); + } + break; - case execplan::CalpontSystemCatalog::DATE: - { - string str = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + case execplan::CalpontSystemCatalog::DATE: + { + string str = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); - if (!isNull) - ret = strtoull(str.c_str(), NULL, 10); - } - break; + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; - case execplan::CalpontSystemCatalog::DATETIME: - { - string str = - DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + case execplan::CalpontSystemCatalog::DATETIME: + { + string str = DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - // strip off micro seconds - str = str.substr(0, 14); + // strip off micro seconds + str = str.substr(0, 14); - if (!isNull) - ret = strtoull(str.c_str(), NULL, 10); - } - break; + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - // strip off micro seconds - str = str.substr(0, 14); + // strip off micro seconds + str = str.substr(0, 14); - if (!isNull) - ret = strtoull(str.c_str(), NULL, 10); - } - break; + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; - case execplan::CalpontSystemCatalog::TIME: - { - string str = - DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + case execplan::CalpontSystemCatalog::TIME: + { + string str = DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); - // strip off micro seconds - str = str.substr(0, 14); + // strip off micro seconds + str = str.substr(0, 14); - if (!isNull) - ret = strtoull(str.c_str(), NULL, 10); - } - break; + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - ret = parm[0]->data()->getDecimalVal(row, isNull).toUInt64Floor(); - break; - } - - default: - { - std::ostringstream oss; - oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull).toUInt64Floor(); + break; } - return ret; + default: + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return ret; } - -double Func_floor::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_floor::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - double ret = 0.0; + double ret = 0.0; - if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || - op_ct.colDataType == CalpontSystemCatalog::FLOAT) - { - ret = floor(parm[0]->data()->getDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) - { - ret = floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || - op_ct.colDataType == CalpontSystemCatalog::CHAR || - op_ct.colDataType == CalpontSystemCatalog::TEXT) - { - const string& str = parm[0]->data()->getStrVal(row, isNull); + if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || op_ct.colDataType == CalpontSystemCatalog::FLOAT) + { + ret = floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + ret = floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || + op_ct.colDataType == CalpontSystemCatalog::CHAR || op_ct.colDataType == CalpontSystemCatalog::TEXT) + { + const string& str = parm[0]->data()->getStrVal(row, isNull); - if (!isNull) - ret = floor(strtod(str.c_str(), 0)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || - op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) - { - IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + if (!isNull) + ret = floor(strtod(str.c_str(), 0)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); - if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - ret = static_cast(tmp.toTSInt128()); - } - else - { - ret = (double) tmp.value; - } + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = static_cast(tmp.toTSInt128()); } else { - ret = (double) getIntVal(row, parm, isNull, op_ct); + ret = (double)tmp.value; } + } + else + { + ret = (double)getIntVal(row, parm, isNull, op_ct); + } - return ret; + return ret; } -long double Func_floor::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +long double Func_floor::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - long double ret = 0.0; + long double ret = 0.0; - if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE || - op_ct.colDataType == CalpontSystemCatalog::FLOAT) - { - ret = floor(parm[0]->data()->getDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) - { - ret = floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || - op_ct.colDataType == CalpontSystemCatalog::CHAR || - op_ct.colDataType == CalpontSystemCatalog::TEXT) - { - const string& str = parm[0]->data()->getStrVal(row, isNull); + if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE || + op_ct.colDataType == CalpontSystemCatalog::FLOAT) + { + ret = floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + ret = floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::VARCHAR || + op_ct.colDataType == CalpontSystemCatalog::CHAR || op_ct.colDataType == CalpontSystemCatalog::TEXT) + { + const string& str = parm[0]->data()->getStrVal(row, isNull); - if (!isNull) - ret = floor(strtod(str.c_str(), 0)); - } - else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || - op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) - { - IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + if (!isNull) + ret = floor(strtod(str.c_str(), 0)); + } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); - if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - ret = static_cast(tmp.toTSInt128()); - } - else - { - ret = (long double) tmp.value; - } + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = static_cast(tmp.toTSInt128()); } else { - ret = (long double) getIntVal(row, parm, isNull, op_ct); + ret = (long double)tmp.value; } + } + else + { + ret = (long double)getIntVal(row, parm, isNull, op_ct); + } - return ret; + return ret; } -string Func_floor::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +string Func_floor::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - char tmp[512] = {'\0'}; + char tmp[512] = {'\0'}; - if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || - op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || - op_ct.colDataType == CalpontSystemCatalog::FLOAT || - op_ct.colDataType == CalpontSystemCatalog::UFLOAT || - op_ct.colDataType == CalpontSystemCatalog::VARCHAR || - op_ct.colDataType == CalpontSystemCatalog::CHAR || - op_ct.colDataType == CalpontSystemCatalog::TEXT) + if (op_ct.colDataType == CalpontSystemCatalog::DOUBLE || + op_ct.colDataType == CalpontSystemCatalog::UDOUBLE || + op_ct.colDataType == CalpontSystemCatalog::FLOAT || op_ct.colDataType == CalpontSystemCatalog::UFLOAT || + op_ct.colDataType == CalpontSystemCatalog::VARCHAR || op_ct.colDataType == CalpontSystemCatalog::CHAR || + op_ct.colDataType == CalpontSystemCatalog::TEXT) + { + snprintf(tmp, 511, "%f", getDoubleVal(row, parm, isNull, op_ct)); + + // remove the decimals in the oss string. + char* d = tmp; + + while ((*d != '.') && (*d != '\0')) + d++; + + *d = '\0'; + } + if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + { + snprintf(tmp, 511, "%Lf", getLongDoubleVal(row, parm, isNull, op_ct)); + + // remove the decimals in the oss string. + char* d = tmp; + + while ((*d != '.') && (*d != '\0')) + d++; + + *d = '\0'; + } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) { - snprintf(tmp, 511, "%f", getDoubleVal(row, parm, isNull, op_ct)); - - // remove the decimals in the oss string. - char* d = tmp; - - while ((*d != '.') && (*d != '\0')) - d++; - - *d = '\0'; + return d.toString(true); } - if (op_ct.colDataType == CalpontSystemCatalog::LONGDOUBLE) + else { - snprintf(tmp, 511, "%Lf", getLongDoubleVal(row, parm, isNull, op_ct)); - - // remove the decimals in the oss string. - char* d = tmp; - - while ((*d != '.') && (*d != '\0')) - d++; - - *d = '\0'; + return d.toString(); } - else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || - op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) - { - IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); - - if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) - { - return d.toString(true); - } - else - { - return d.toString(); - } - } - else if (isUnsigned(op_ct.colDataType)) - { + } + else if (isUnsigned(op_ct.colDataType)) + { #ifndef __LP64__ - snprintf(tmp, 511, "%llu", getUintVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%llu", getUintVal(row, parm, isNull, op_ct)); #else - snprintf(tmp, 511, "%lu", getUintVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%lu", getUintVal(row, parm, isNull, op_ct)); #endif - } - else - { + } + else + { #ifndef __LP64__ - snprintf(tmp, 511, "%lld", getIntVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%lld", getIntVal(row, parm, isNull, op_ct)); #else - snprintf(tmp, 511, "%ld", getIntVal(row, parm, isNull, op_ct)); + snprintf(tmp, 511, "%ld", getIntVal(row, parm, isNull, op_ct)); #endif - } + } - return string(tmp); + return string(tmp); } -IDB_Decimal Func_floor::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +IDB_Decimal Func_floor::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal ret; + IDB_Decimal ret; - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - ret.value = parm[0]->data()->getIntVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - ret = parm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - // negative scale is not supported by CNX yet - if (ret.scale > 0) - { - if (ret.scale > datatypes::INT128MAXPRECISION) - { - std::ostringstream oss; - oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) ret.scale << " is beyond supported scale"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - - return ret.floor(); - } - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - ret.value = (int64_t) floor(parm[0]->data()->getDoubleVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - ret.value = (int64_t) floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& str = parm[0]->data()->getStrVal(row, isNull); - - if (!isNull) - ret.value = (int64_t) floor(strtod(str.c_str(), 0)); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - string str = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); - - if (!isNull) - ret.value = atoll(str.c_str()); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - string str = - DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - - // strip off micro seconds - str = str.substr(0, 14); - - if (!isNull) - ret.value = atoll(str.c_str()); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - - // strip off micro seconds - str = str.substr(0, 14); - - if (!isNull) - ret.value = atoll(str.c_str()); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - string str = - DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); - - // strip off micro seconds - str = str.substr(0, 14); - - if (!isNull) - ret.value = atoll(str.c_str()); - } - break; - - default: - { - std::ostringstream oss; - oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + ret.value = parm[0]->data()->getIntVal(row, isNull); } + break; - ret.scale = 0; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull); - return ret; + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (ret.scale > 0) + { + if (ret.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) << " with scale " + << (int)ret.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + return ret.floor(); + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret.value = (int64_t)floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret.value = (int64_t)floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret.value = (int64_t)floor(strtod(str.c_str(), 0)); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + string str = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + string str = DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + string str = DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + default: + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + ret.scale = 0; + + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_days.cpp b/utils/funcexp/func_from_days.cpp index bbd8da01c..e9f45b421 100644 --- a/utils/funcexp/func_from_days.cpp +++ b/utils/funcexp/func_from_days.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_from_days.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_from_days.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -37,56 +37,47 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_from_days::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_from_days::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_from_days::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_from_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return getDatetimeIntVal(row, parm, isNull, op_ct); + return getDatetimeIntVal(row, parm, isNull, op_ct); } -string Func_from_days::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_from_days::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return intToString(getIntVal(row, parm, isNull, ct)); + return intToString(getIntVal(row, parm, isNull, ct)); } -int32_t Func_from_days::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_from_days::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E); + return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E); } -int64_t Func_from_days::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_from_days::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - double val1 = parm[0]->data()->getDoubleVal(row, isNull); - int64_t daynr = (int64_t)(val1 > 0 ? val1 + 0.5 : val1 - 0.5); + double val1 = parm[0]->data()->getDoubleVal(row, isNull); + int64_t daynr = (int64_t)(val1 > 0 ? val1 + 0.5 : val1 - 0.5); - DateTime aDaytime; - helpers::get_date_from_mysql_daynr( daynr, aDaytime ); + DateTime aDaytime; + helpers::get_date_from_mysql_daynr(daynr, aDaytime); - // to be safe - aDaytime.hour = 0; - aDaytime.minute = 0; - aDaytime.second = 0; - aDaytime.msecond = 0; + // to be safe + aDaytime.hour = 0; + aDaytime.minute = 0; + aDaytime.second = 0; + aDaytime.msecond = 0; - return (*(reinterpret_cast (&aDaytime))); + return (*(reinterpret_cast(&aDaytime))); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 287a95315..cca4083be 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_from_unixtime.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_from_unixtime.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -36,202 +36,180 @@ using namespace execplan; #include "dataconvert.h" using namespace dataconvert; - namespace { using namespace funcexp; -DateTime getDateTime(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull) +DateTime getDateTime(rowgroup::Row& row, FunctionParm& parm, bool& isNull) { - int64_t val = 0; - uint32_t msec = 0; + int64_t val = 0; + uint32_t msec = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DOUBLE: { - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - double fracpart, intpart; - fracpart = modf(value, &intpart); - val = (int64_t)intpart; - msec = (uint32_t)(fracpart * IDB_pow[parm[0]->data()->resultType().scale]); - break; - } - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal dec = parm[0]->data()->getDecimalVal(row, isNull); + double value = parm[0]->data()->getDoubleVal(row, isNull); + double fracpart, intpart; + fracpart = modf(value, &intpart); + val = (int64_t)intpart; + msec = (uint32_t)(fracpart * IDB_pow[parm[0]->data()->resultType().scale]); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal dec = parm[0]->data()->getDecimalVal(row, isNull); - if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - auto integralAndFractional = dec.getIntegralAndFractional(); - val = static_cast(integralAndFractional.first); - msec = static_cast(integralAndFractional.second); - } - else - { - val = dec.value / IDB_pow[dec.scale]; - msec = dec.value % IDB_pow[dec.scale]; - } - break; - } - - default: - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + auto integralAndFractional = dec.getIntegralAndFractional(); + val = static_cast(integralAndFractional.first); + msec = static_cast(integralAndFractional.second); + } + else + { + val = dec.value / IDB_pow[dec.scale]; + msec = dec.value % IDB_pow[dec.scale]; + } + break; } - if (val < 0 || val > helpers::TIMESTAMP_MAX_VALUE) - return 0; + default: val = parm[0]->data()->getDatetimeIntVal(row, isNull); + } - DateTime dt; + if (val < 0 || val > helpers::TIMESTAMP_MAX_VALUE) + return 0; - struct tm tmp_tm; - time_t tmp_t = (time_t)val; - localtime_r(&tmp_t, &tmp_tm); + DateTime dt; - //to->neg=0; - dt.year = (int64_t) ((tmp_tm.tm_year + 1900) % 10000); - dt.month = (int64_t) tmp_tm.tm_mon + 1; - dt.day = (int64_t) tmp_tm.tm_mday; - dt.hour = (int64_t) tmp_tm.tm_hour; - dt.minute = (int64_t) tmp_tm.tm_min; - dt.second = (int64_t) tmp_tm.tm_sec; - dt.msecond = msec; - return dt; -} + struct tm tmp_tm; + time_t tmp_t = (time_t)val; + localtime_r(&tmp_t, &tmp_tm); + + // to->neg=0; + dt.year = (int64_t)((tmp_tm.tm_year + 1900) % 10000); + dt.month = (int64_t)tmp_tm.tm_mon + 1; + dt.day = (int64_t)tmp_tm.tm_mday; + dt.hour = (int64_t)tmp_tm.tm_hour; + dt.minute = (int64_t)tmp_tm.tm_min; + dt.second = (int64_t)tmp_tm.tm_sec; + dt.msecond = msec; + return dt; } +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_from_unixtime::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_from_unixtime::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + return ct; } -string Func_from_unixtime::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_from_unixtime::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - DateTime dt = getDateTime(row, parm, isNull); + DateTime dt = getDateTime(row, parm, isNull); - if (*reinterpret_cast(&dt) == 0) - { - isNull = true; - return ""; - } + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return ""; + } - if (parm.size() == 2) - { - const string& format = parm[1]->data()->getStrVal(row, isNull); - return helpers::IDB_date_format(dt, format); - } + if (parm.size() == 2) + { + const string& format = parm[1]->data()->getStrVal(row, isNull); + return helpers::IDB_date_format(dt, format); + } - char buf[256] = {0}; - DataConvert::datetimeToString(*(reinterpret_cast(&dt)), buf, 255); - return string(buf, 255); + char buf[256] = {0}; + DataConvert::datetimeToString(*(reinterpret_cast(&dt)), buf, 255); + return string(buf, 255); } -int32_t Func_from_unixtime::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int32_t Func_from_unixtime::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E); + return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E); } -int64_t Func_from_unixtime::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_from_unixtime::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - DateTime dt = getDateTime(row, parm, isNull); + DateTime dt = getDateTime(row, parm, isNull); - if (*reinterpret_cast(&dt) == 0) - { - isNull = true; - return 0; - } + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } - return *reinterpret_cast(&dt); + return *reinterpret_cast(&dt); } -int64_t Func_from_unixtime::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_from_unixtime::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - DateTime dt = getDateTime(row, parm, isNull); + DateTime dt = getDateTime(row, parm, isNull); - if (*reinterpret_cast(&dt) == 0) - { - isNull = true; - return 0; - } + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } - return *reinterpret_cast(&dt); + return *reinterpret_cast(&dt); } -int64_t Func_from_unixtime::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_from_unixtime::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { + DateTime dt = getDateTime(row, parm, isNull); + + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } + + char buf[32]; // actual string guaranteed to be 22 + snprintf(buf, 32, "%04d%02d%02d%02d%02d%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + return atoll(buf); +} + +double Func_from_unixtime::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + if (parm.size() == 1) + { DateTime dt = getDateTime(row, parm, isNull); if (*reinterpret_cast(&dt) == 0) { - isNull = true; - return 0; + isNull = true; + return 0; } char buf[32]; // actual string guaranteed to be 22 - snprintf( buf, 32, "%04d%02d%02d%02d%02d%02d", - dt.year, dt.month, dt.day, dt.hour, - dt.minute, dt.second ); - return atoll(buf); + snprintf(buf, 32, "%04d%02d%02d%02d%02d%02d.%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, + dt.second, dt.msecond); + return atof(buf); + } + + return (double)atoi(getStrVal(row, parm, isNull, ct).c_str()); } -double Func_from_unixtime::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +long double Func_from_unixtime::getLongDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - if (parm.size() == 1) - { - DateTime dt = getDateTime(row, parm, isNull); - - if (*reinterpret_cast(&dt) == 0) - { - isNull = true; - return 0; - } - - char buf[32]; // actual string guaranteed to be 22 - snprintf( buf, 32, "%04d%02d%02d%02d%02d%02d.%06d", - dt.year, dt.month, dt.day, dt.hour, - dt.minute, dt.second, dt.msecond ); - return atof(buf); - } - - return (double) atoi(getStrVal(row, parm, isNull, ct).c_str()); + return (long double)getDoubleVal(row, parm, isNull, ct); } -long double Func_from_unixtime::getLongDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) -{ - return (long double) getDoubleVal(row, parm, isNull, ct); -} - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_get_format.cpp b/utils/funcexp/func_get_format.cpp index 076f3f36f..d902c0873 100644 --- a/utils/funcexp/func_get_format.cpp +++ b/utils/funcexp/func_get_format.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_get_format.cpp 2665 2011-06-01 20:42:52Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_get_format.cpp 2665 2011-06-01 20:42:52Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,88 +36,75 @@ using namespace dataconvert; class to_upper { -public: - char operator() (char c) const // notice the return type - { - return toupper(c); - } + public: + char operator()(char c) const // notice the return type + { + return toupper(c); + } }; - namespace funcexp { +string known_date_time_formats[5][4] = {{"USA", "%m.%d.%Y", "%Y-%m-%d %H.%i.%s", "%h:%i:%s %p"}, + {"JIS", "%Y-%m-%d", "%Y-%m-%d %H:%i:%s", "%H:%i:%s"}, + {"ISO", "%Y-%m-%d", "%Y-%m-%d %H:%i:%s", "%H:%i:%s"}, + {"EUR", "%d.%m.%Y", "%Y-%m-%d %H.%i.%s", "%H.%i.%s"}, + {"INTERNAL", "%Y%m%d", "%Y%m%d%H%i%s", "%H%i%s"}}; -string known_date_time_formats[5][4] = -{ - {"USA", "%m.%d.%Y", "%Y-%m-%d %H.%i.%s", "%h:%i:%s %p" }, - {"JIS", "%Y-%m-%d", "%Y-%m-%d %H:%i:%s", "%H:%i:%s" }, - {"ISO", "%Y-%m-%d", "%Y-%m-%d %H:%i:%s", "%H:%i:%s" }, - {"EUR", "%d.%m.%Y", "%Y-%m-%d %H.%i.%s", "%H.%i.%s" }, - {"INTERNAL", "%Y%m%d", "%Y%m%d%H%i%s", "%H%i%s" } -}; +string know_types[3] = {"DATE", "DATETIME", "TIME"}; -string know_types[3] = +CalpontSystemCatalog::ColType Func_get_format::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - "DATE", "DATETIME", "TIME" -}; - -CalpontSystemCatalog::ColType Func_get_format::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) -{ - return resultType; + return resultType; } -string Func_get_format::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_get_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - // parm[0] -- format - // parm[1] -- type - string format = parm[0]->data()->getStrVal(row, isNull); - - if (isNull) - return ""; - - transform (format.begin(), format.end(), format.begin(), to_upper()); - - string type = parm[1]->data()->getStrVal(row, isNull); - - if (isNull) - return ""; - - transform (type.begin(), type.end(), type.begin(), to_upper()); - - int itype = 0; - - for ( ; itype < 3 ; itype++ ) - { - if ( know_types[itype] == type ) - break; - } - - // check for match - if ( itype == 3 ) - return ""; - - for ( int i = 0 ; i < 5 ; i ++ ) - { - if ( known_date_time_formats[i][0] == format ) - { - switch (itype) - { - case 0: - return known_date_time_formats[i][2]; - break; - - default: - return ""; - } - } - } + // parm[0] -- format + // parm[1] -- type + string format = parm[0]->data()->getStrVal(row, isNull); + if (isNull) return ""; + + transform(format.begin(), format.end(), format.begin(), to_upper()); + + string type = parm[1]->data()->getStrVal(row, isNull); + + if (isNull) + return ""; + + transform(type.begin(), type.end(), type.begin(), to_upper()); + + int itype = 0; + + for (; itype < 3; itype++) + { + if (know_types[itype] == type) + break; + } + + // check for match + if (itype == 3) + return ""; + + for (int i = 0; i < 5; i++) + { + if (known_date_time_formats[i][0] == format) + { + switch (itype) + { + case 0: return known_date_time_formats[i][2]; break; + + default: return ""; + } + } + } + + return ""; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 5a1085a19..e8562be20 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_greatest.cpp 3954 2013-07-08 16:30:15Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_greatest.cpp 3954 2013-07-08 16:30:15Z bpaul $ + * + * + ****************************************************************************/ #include #include @@ -39,227 +39,205 @@ using namespace joblist; using namespace funcexp; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_greatest::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_greatest::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - //return fp[0]->data()->resultType(); - return resultType; + // operation type is not used by this functor + // return fp[0]->data()->resultType(); + return resultType; } -int64_t Func_greatest::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_greatest::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - double str = fp[0]->data()->getDoubleVal(row, isNull); + double str = fp[0]->data()->getDoubleVal(row, isNull); - double greatestStr = str; + double greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - double str1 = fp[i]->data()->getDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + double str1 = fp[i]->data()->getDoubleVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - uint64_t tmp = (uint64_t)greatestStr; - return (int64_t) tmp; + uint64_t tmp = (uint64_t)greatestStr; + return (int64_t)tmp; } -uint64_t Func_greatest::getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +uint64_t Func_greatest::getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - double str = fp[0]->data()->getDoubleVal(row, isNull); + double str = fp[0]->data()->getDoubleVal(row, isNull); - double greatestStr = str; + double greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - double str1 = fp[i]->data()->getDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + double str1 = fp[i]->data()->getDoubleVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return (uint64_t) greatestStr; + return (uint64_t)greatestStr; } -double Func_greatest::getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +double Func_greatest::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - double str = fp[0]->data()->getDoubleVal(row, isNull); + double str = fp[0]->data()->getDoubleVal(row, isNull); - double greatestStr = str; + double greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - double str1 = fp[i]->data()->getDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + double str1 = fp[i]->data()->getDoubleVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return (double) greatestStr; + return (double)greatestStr; } -long double Func_greatest::getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +long double Func_greatest::getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - long double str = fp[0]->data()->getLongDoubleVal(row, isNull); + long double str = fp[0]->data()->getLongDoubleVal(row, isNull); - long double greatestStr = str; + long double greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - long double str1 = fp[i]->data()->getLongDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + long double str1 = fp[i]->data()->getLongDoubleVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return greatestStr; + return greatestStr; } -std::string Func_greatest::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_greatest::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - const string& str = fp[0]->data()->getStrVal(row, isNull); - CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); + const string& str = fp[0]->data()->getStrVal(row, isNull); + CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); - string greatestStr = str; + string greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) + for (uint32_t i = 1; i < fp.size(); i++) + { + const string& str1 = fp[i]->data()->getStrVal(row, isNull); + + if (cs->strnncoll(greatestStr.c_str(), greatestStr.length(), str1.c_str(), str1.length()) < 0) { - const string& str1 = fp[i]->data()->getStrVal(row, isNull); - - if (cs->strnncoll(greatestStr.c_str(), greatestStr.length(), str1.c_str(), str1.length()) < 0) - { - greatestStr = str1; - } + greatestStr = str1; } + } - return greatestStr; + return greatestStr; } -IDB_Decimal Func_greatest::getDecimalVal(Row& row, - FunctionParm& fp, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +IDB_Decimal Func_greatest::getDecimalVal(Row& row, FunctionParm& fp, bool& isNull, + CalpontSystemCatalog::ColType& ct) { -// double str = fp[0]->data()->getDoubleVal(row, isNull); - IDB_Decimal str = fp[0]->data()->getDecimalVal(row, isNull); + // double str = fp[0]->data()->getDoubleVal(row, isNull); + IDB_Decimal str = fp[0]->data()->getDecimalVal(row, isNull); - IDB_Decimal greatestStr = str; + IDB_Decimal greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - IDB_Decimal str1 = fp[i]->data()->getDecimalVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + IDB_Decimal str1 = fp[i]->data()->getDecimalVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return greatestStr; + return greatestStr; } -int32_t Func_greatest::getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int32_t Func_greatest::getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - int32_t str = fp[0]->data()->getDateIntVal(row, isNull); + int32_t str = fp[0]->data()->getDateIntVal(row, isNull); - int32_t greatestStr = str; + int32_t greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - int32_t str1 = fp[i]->data()->getDateIntVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + int32_t str1 = fp[i]->data()->getDateIntVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return greatestStr; + return greatestStr; } -int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - int64_t str = fp[0]->data()->getDatetimeIntVal(row, isNull); + int64_t str = fp[0]->data()->getDatetimeIntVal(row, isNull); - int64_t greatestStr = str; + int64_t greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - int64_t str1 = fp[i]->data()->getDatetimeIntVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getDatetimeIntVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return greatestStr; + return greatestStr; } -int64_t Func_greatest::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +int64_t Func_greatest::getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - int64_t str = fp[0]->data()->getTimestampIntVal(row, isNull); + int64_t str = fp[0]->data()->getTimestampIntVal(row, isNull); - int64_t greatestStr = str; + int64_t greatestStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - int64_t str1 = fp[i]->data()->getTimestampIntVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimestampIntVal(row, isNull); - if ( greatestStr < str1 ) - greatestStr = str1; - } + if (greatestStr < str1) + greatestStr = str1; + } - return greatestStr; + return greatestStr; } -int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - // Strip off unused day - int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); + // Strip off unused day + int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t str = greatestStr << 12; + int64_t str = greatestStr << 12; - for (uint32_t i = 1; i < fp.size(); i++) + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; + + if (str < str2) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); - int64_t str2 = str1 << 12; - - if ( str < str2 ) - { - greatestStr = str1; - str = str2; - } + greatestStr = str1; + str = str2; } + } - return greatestStr; + return greatestStr; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index e4177a947..133eca923 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_hex.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_hex.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -44,103 +44,100 @@ char digit_upper[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; void octet2hex(char* to, const char* str, uint32_t len) { - const char* str_end = str + len; + const char* str_end = str + len; - for (; str != str_end; ++str) - { - *to++ = digit_upper[((uint8_t) * str) >> 4]; - *to++ = digit_upper[((uint8_t) * str) & 0x0F]; - } + for (; str != str_end; ++str) + { + *to++ = digit_upper[((uint8_t)*str) >> 4]; + *to++ = digit_upper[((uint8_t)*str) & 0x0F]; + } - *to = '\0'; + *to = '\0'; } -CalpontSystemCatalog::ColType Func_hex::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_hex::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_hex::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_hex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - string retval; - uint64_t dec; - char ans[65]; + string retval; + uint64_t dec; + char ans[65]; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::TIME: { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::TIME: - { - const string& arg = parm[0]->data()->getStrVal(row, isNull); - scoped_array hexPtr(new char[strlen(arg.c_str()) * 2 + 1]); - octet2hex(hexPtr.get(), arg.c_str(), strlen(arg.c_str())); - return string(hexPtr.get(), strlen(arg.c_str()) * 2); - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - /* Return hex of unsigned longlong value */ - double val = parm[0]->data()->getDoubleVal(row, isNull); - - if ((val <= (double) numeric_limits::min()) || - (val >= (double) numeric_limits::max())) - dec = ~(int64_t) 0; - else - dec = (uint64_t) (val + (val > 0 ? 0.5 : -0.5)); - - retval = helpers::convNumToStr(dec, ans, 16); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - char buf[256]; - long double val = parm[0]->data()->getLongDoubleVal(row, isNull); - -#ifdef _MSC_VER - sprintf(buf, "%llA", val); - -#else - sprintf(buf, "%LA", val); -#endif - retval = buf; - break; - } - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - { - const string& arg = parm[0]->data()->getStrVal(row, isNull); - uint64_t hexLen = arg.size() * 2; - scoped_array hexPtr(new char[hexLen + 1]); // "+ 1" for the last \0 - octet2hex(hexPtr.get(), arg.data(), arg.size()); - return string(hexPtr.get(), hexLen); - } - - default: - { - dec = (uint64_t)parm[0]->data()->getIntVal(row, isNull); - retval = helpers::convNumToStr(dec, ans, 16); - - if (retval.length() > (uint32_t)ct.colWidth) - retval = retval.substr(retval.length() - ct.colWidth, ct.colWidth); - } + const string& arg = parm[0]->data()->getStrVal(row, isNull); + scoped_array hexPtr(new char[strlen(arg.c_str()) * 2 + 1]); + octet2hex(hexPtr.get(), arg.c_str(), strlen(arg.c_str())); + return string(hexPtr.get(), strlen(arg.c_str()) * 2); } - return retval; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + /* Return hex of unsigned longlong value */ + double val = parm[0]->data()->getDoubleVal(row, isNull); + + if ((val <= (double)numeric_limits::min()) || (val >= (double)numeric_limits::max())) + dec = ~(int64_t)0; + else + dec = (uint64_t)(val + (val > 0 ? 0.5 : -0.5)); + + retval = helpers::convNumToStr(dec, ans, 16); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + char buf[256]; + long double val = parm[0]->data()->getLongDoubleVal(row, isNull); + +#ifdef _MSC_VER + sprintf(buf, "%llA", val); + +#else + sprintf(buf, "%LA", val); +#endif + retval = buf; + break; + } + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + { + const string& arg = parm[0]->data()->getStrVal(row, isNull); + uint64_t hexLen = arg.size() * 2; + scoped_array hexPtr(new char[hexLen + 1]); // "+ 1" for the last \0 + octet2hex(hexPtr.get(), arg.data(), arg.size()); + return string(hexPtr.get(), hexLen); + } + + default: + { + dec = (uint64_t)parm[0]->data()->getIntVal(row, isNull); + retval = helpers::convNumToStr(dec, ans, 16); + + if (retval.length() > (uint32_t)ct.colWidth) + retval = retval.substr(retval.length() - ct.colWidth, ct.colWidth); + } + } + + return retval; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index af949a393..b9ef7f823 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_hour.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_hour.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,129 +35,125 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_hour::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_hour::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_hour::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_hour::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; - bool isTime = false; + int64_t val = 0; + bool isTime = false; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; + if (val == -1) + isNull = true; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - } - else - { - isNull = true; - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - { - isNull = true; - } - /* fall through */ - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - return m_time.hour; - } - - case execplan::CalpontSystemCatalog::TIME: - { - isTime = true; - val = parm[0]->data()->getTimeIntVal(row, isNull); - break; - } - - default: - { - isNull = true; - } + break; } - if (isNull) - return -1; - - if (isTime) + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - // HOUR() is always positive in MariaDB, even for negative time - int64_t mask = 0; + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if ((val >> 40) & 0x800) - mask = 0xfffffffffffff000; + if (val == -1) + isNull = true; + } + else + { + isNull = true; + } - val = abs(mask | ((val >> 40) & 0xfff)); - } - else - { - val = (val >> 32) & 0x3f; + break; } - return val; + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + { + isNull = true; + } + /* fall through */ + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + return m_time.hour; + } + + case execplan::CalpontSystemCatalog::TIME: + { + isTime = true; + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + + default: + { + isNull = true; + } + } + + if (isNull) + return -1; + + if (isTime) + { + // HOUR() is always positive in MariaDB, even for negative time + int64_t mask = 0; + + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + val = abs(mask | ((val >> 40) & 0xfff)); + } + else + { + val = (val >> 32) & 0x3f; + } + + return val; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_idbpartition.cpp b/utils/funcexp/func_idbpartition.cpp index 330801c6a..3b71cdd97 100644 --- a/utils/funcexp/func_idbpartition.cpp +++ b/utils/funcexp/func_idbpartition.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id$ -* -* -****************************************************************************/ + * $Id$ + * + * + ****************************************************************************/ #include #include @@ -43,28 +43,24 @@ using namespace logging; namespace funcexp { -CalpontSystemCatalog::ColType Func_idbpartition::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_idbpartition::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // all integer - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - return ct; + // all integer + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + return ct; } -string Func_idbpartition::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_idbpartition::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - LogicalPartition part( - parm[0]->data()->getIntVal(row, isNull), - parm[1]->data()->getIntVal(row, isNull), - parm[2]->data()->getIntVal(row, isNull)); + LogicalPartition part(parm[0]->data()->getIntVal(row, isNull), parm[1]->data()->getIntVal(row, isNull), + parm[2]->data()->getIntVal(row, isNull)); - return part.toString(); + return part.toString(); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 168383b93..3e208dc7d 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_if.cpp 3871 2013-06-07 16:25:01Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_if.cpp 3871 2013-06-07 16:25:01Z bpaul $ + * + * + ****************************************************************************/ #include using namespace std; @@ -35,245 +35,201 @@ using namespace rowgroup; namespace { - bool boolVal(SPTP& parm, Row& row, const string& timeZone) { - bool ret = true; - bool isNull = false; // Keep it local. We don't want to mess with the global one here. + bool ret = true; + bool isNull = false; // Keep it local. We don't want to mess with the global one here. - try + try + { + ret = parm->getBoolVal(row, isNull) && !isNull; + } + catch (logging::NotImplementedExcept&) + { + switch (parm->data()->resultType().colDataType) { - ret = parm->getBoolVal(row, isNull) && !isNull; - } - catch (logging::NotImplementedExcept&) - { - switch (parm->data()->resultType().colDataType) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - ret = (atoi((char*)(parm->data()->getStrVal(timeZone).c_str())) != 0); - break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - ret = (parm->data()->getFloatVal(row, isNull) != 0); - break; - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - ret = (parm->data()->getDoubleVal(row, isNull) != 0); - break; - case CalpontSystemCatalog::LONGDOUBLE: - ret = (parm->data()->getLongDoubleVal(row, isNull) != 0); - break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - ret = (parm->data()->getDecimalVal(row, isNull).s128Value != 0); - else - ret = (parm->data()->getDecimalVal(row, isNull).value != 0); - break; - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - default: - ret = (parm->data()->getIntVal(row, isNull) != 0); - break; - } + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + ret = (atoi((char*)(parm->data()->getStrVal(timeZone).c_str())) != 0); + break; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: ret = (parm->data()->getFloatVal(row, isNull) != 0); break; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: ret = (parm->data()->getDoubleVal(row, isNull) != 0); break; + case CalpontSystemCatalog::LONGDOUBLE: ret = (parm->data()->getLongDoubleVal(row, isNull) != 0); break; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + ret = (parm->data()->getDecimalVal(row, isNull).s128Value != 0); + else + ret = (parm->data()->getDecimalVal(row, isNull).value != 0); + break; + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + default: ret = (parm->data()->getIntVal(row, isNull) != 0); break; } + } - return ret; -} - + return ret; } +} // namespace namespace funcexp { - // IF(expression1, expression12, expression3) // // if parm order: // expression1 expression2 expression3 // -CalpontSystemCatalog::ColType Func_if::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_if::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - // The result type given by the connector may not be right if there's derived table (MySQL bug?) - // We want to double check on our own. - // If any parm is of string type, the result type should be string. - if (fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TEXT || - fp[2]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || - fp[2]->data()->resultType().colDataType == CalpontSystemCatalog::TEXT || - fp[2]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - resultType = ct; - return ct; - } - - CalpontSystemCatalog::ColType ct = fp[1]->data()->resultType(); - PredicateOperator op; - op.setOpType(ct, fp[2]->data()->resultType()); - ct = op.operationType(); + // operation type is not used by this functor + // The result type given by the connector may not be right if there's derived table (MySQL bug?) + // We want to double check on our own. + // If any parm is of string type, the result type should be string. + if (fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TEXT || + fp[2]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || + fp[2]->data()->resultType().colDataType == CalpontSystemCatalog::TEXT || + fp[2]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; resultType = ct; return ct; + } + + CalpontSystemCatalog::ColType ct = fp[1]->data()->resultType(); + PredicateOperator op; + op.setOpType(ct, fp[2]->data()->resultType()); + ct = op.operationType(); + resultType = ct; + return ct; } - -int64_t Func_if::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getIntVal(row, isNull); - } - else - { - return parm[2]->data()->getIntVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getIntVal(row, isNull); + } + else + { + return parm[2]->data()->getIntVal(row, isNull); + } } - -string Func_if::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getStrVal(row, isNull); - } - else - { - return parm[2]->data()->getStrVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getStrVal(row, isNull); + } + else + { + return parm[2]->data()->getStrVal(row, isNull); + } } - -IDB_Decimal Func_if::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getDecimalVal(row, isNull); - } - else - { - return parm[2]->data()->getDecimalVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getDecimalVal(row, isNull); + } + else + { + return parm[2]->data()->getDecimalVal(row, isNull); + } } - -double Func_if::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getDoubleVal(row, isNull); - } - else - { - return parm[2]->data()->getDoubleVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getDoubleVal(row, isNull); + } + else + { + return parm[2]->data()->getDoubleVal(row, isNull); + } } -long double Func_if::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +long double Func_if::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getLongDoubleVal(row, isNull); - } - else - { - return parm[2]->data()->getLongDoubleVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getLongDoubleVal(row, isNull); + } + else + { + return parm[2]->data()->getLongDoubleVal(row, isNull); + } } - -int32_t Func_if::getDateIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getDateIntVal(row, isNull); - } - else - { - return parm[2]->data()->getDateIntVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getDateIntVal(row, isNull); + } + else + { + return parm[2]->data()->getDateIntVal(row, isNull); + } } - -int64_t Func_if::getDatetimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getDatetimeIntVal(row, isNull); - } - else - { - return parm[2]->data()->getDatetimeIntVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getDatetimeIntVal(row, isNull); + } + else + { + return parm[2]->data()->getDatetimeIntVal(row, isNull); + } } - -int64_t Func_if::getTimestampIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_if::getTimestampIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getTimestampIntVal(row, isNull); - } - else - { - return parm[2]->data()->getTimestampIntVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getTimestampIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimestampIntVal(row, isNull); + } } - -int64_t Func_if::getTimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, timeZone())) - { - return parm[1]->data()->getTimeIntVal(row, isNull); - } - else - { - return parm[2]->data()->getTimeIntVal(row, isNull); - } + if (boolVal(parm[0], row, timeZone())) + { + return parm[1]->data()->getTimeIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimeIntVal(row, isNull); + } } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index e479ba69c..52dae2556 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_ifnull.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_ifnull.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,7 +34,6 @@ using namespace rowgroup; namespace funcexp { - // IFNULL(expression1, expression12) // // if parm order: @@ -44,196 +43,166 @@ namespace funcexp // otherwise it returns expression2. // -CalpontSystemCatalog::ColType Func_ifnull::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_ifnull::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -int64_t Func_ifnull::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int64_t Func_ifnull::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return 0; + if (isNull) + return 0; - int64_t r = parm[0]->data()->getIntVal(row, isNull); + int64_t r = parm[0]->data()->getIntVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getIntVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getIntVal(row, isNull); + } - return r; + return r; } - -string Func_ifnull::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +string Func_ifnull::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return string(); + if (isNull) + return string(); - const string& r = parm[0]->data()->getStrVal(row, isNull); + const string& r = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getStrVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getStrVal(row, isNull); + } - return r; + return r; } - -IDB_Decimal Func_ifnull::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +IDB_Decimal Func_ifnull::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return IDB_Decimal(); + if (isNull) + return IDB_Decimal(); - IDB_Decimal r = parm[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal r = parm[0]->data()->getDecimalVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getDecimalVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getDecimalVal(row, isNull); + } - return r; + return r; } - -double Func_ifnull::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_ifnull::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return 0.0; + if (isNull) + return 0.0; - double r = parm[0]->data()->getDoubleVal(row, isNull); + double r = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getDoubleVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getDoubleVal(row, isNull); + } - return r; + return r; } -long double Func_ifnull::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +long double Func_ifnull::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - if (isNull) - return 0.0; + if (isNull) + return 0.0; - long double r = parm[0]->data()->getLongDoubleVal(row, isNull); + long double r = parm[0]->data()->getLongDoubleVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getLongDoubleVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getLongDoubleVal(row, isNull); + } - return r; + return r; } - -int32_t Func_ifnull::getDateIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int32_t Func_ifnull::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return 0; + if (isNull) + return 0; - int64_t r = parm[0]->data()->getDateIntVal(row, isNull); + int64_t r = parm[0]->data()->getDateIntVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getDateIntVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getDateIntVal(row, isNull); + } - return r; + return r; } -int64_t Func_ifnull::getDatetimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_ifnull::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return 0; + if (isNull) + return 0; - int64_t r = parm[0]->data()->getDatetimeIntVal(row, isNull); + int64_t r = parm[0]->data()->getDatetimeIntVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getDatetimeIntVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getDatetimeIntVal(row, isNull); + } - return r; + return r; } -int64_t Func_ifnull::getTimestampIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_ifnull::getTimestampIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return 0; + if (isNull) + return 0; - int64_t r = parm[0]->data()->getTimestampIntVal(row, isNull); + int64_t r = parm[0]->data()->getTimestampIntVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getTimestampIntVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimestampIntVal(row, isNull); + } - return r; + return r; } -int64_t Func_ifnull::getTimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +int64_t Func_ifnull::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (isNull) - return 0; + if (isNull) + return 0; - int64_t r = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t r = parm[0]->data()->getTimeIntVal(row, isNull); - if (isNull) - { - isNull = false; - return parm[1]->data()->getTimeIntVal(row, isNull); - } + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimeIntVal(row, isNull); + } - return r; + return r; } -bool Func_ifnull::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t ret = getIntVal(row, parm, isNull, ct); - return (ret == 0 ? false : true); + int64_t ret = getIntVal(row, parm, isNull, ct); + return (ret == 0 ? false : true); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index ad5270805..89ed89593 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_in.cpp 3954 2013-07-08 16:30:15Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_in.cpp 3954 2013-07-08 16:30:15Z bpaul $ + * + * + ****************************************************************************/ #include #include @@ -43,359 +43,347 @@ using namespace logging; using namespace funcexp; - namespace { -template +template inline bool numericEQ(result_t op1, result_t op2) { - return op1 == op2; + return op1 == op2; } -inline bool getBoolForIn(rowgroup::Row& row, - funcexp::FunctionParm& pm, - bool& isNull, - CalpontSystemCatalog::ColType& ct, - bool isNotIn) +inline bool getBoolForIn(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, + CalpontSystemCatalog::ColType& ct, bool isNotIn) { - IDB_Decimal d; // to be removed; + IDB_Decimal d; // to be removed; - switch (ct.colDataType) + switch (ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - int64_t val = pm[0]->data()->getIntVal(row, isNull); + int64_t val = pm[0]->data()->getIntVal(row, isNull); - if (isNull) - return false; + if (isNull) + return false; - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; - if (val == pm[i]->data()->getIntVal(row, isNull) && !isNull ) - return true; + if (val == pm[i]->data()->getIntVal(row, isNull) && !isNull) + return true; - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } - return false; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t val = pm[0]->data()->getUintVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if (val == pm[i]->data()->getUintVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t val = pm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getDateIntVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t val = pm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getTimestampIntVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getTimeIntVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - double val = pm[0]->data()->getDoubleVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getDoubleVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double val = pm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getLongDoubleVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - return false; - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - - if ( val == pm[i]->data()->getDecimalVal(row, isNull) && !isNull ) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& val = pm[0]->data()->getStrVal(row, isNull); - if (isNull) - return false; - - CHARSET_INFO* cs = pm[0]->data()->resultType().getCharset(); - - for (uint32_t i = 1; i < pm.size(); i++) - { - isNull = false; - const string& str1 = pm[i]->data()->getStrVal(row, isNull); - if (cs->strnncoll(val.c_str(), val.length(), str1.c_str(), str1.length()) == 0 && !isNull) - return true; - - if (isNull && isNotIn) - return true; // will be reversed to false by the caller - } - - return false; - } - - default: - { - std::ostringstream oss; - oss << "regexo: datatype of " << execplan::colDataTypeToString(ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return false; } + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t val = pm[0]->data()->getUintVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getUintVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t val = pm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getDateIntVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t val = pm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getTimestampIntVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + double val = pm[0]->data()->getDoubleVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getDoubleVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double val = pm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getLongDoubleVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if (val == pm[i]->data()->getDecimalVal(row, isNull) && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& val = pm[0]->data()->getStrVal(row, isNull); + if (isNull) + return false; + + CHARSET_INFO* cs = pm[0]->data()->resultType().getCharset(); + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + const string& str1 = pm[i]->data()->getStrVal(row, isNull); + if (cs->strnncoll(val.c_str(), val.length(), str1.c_str(), str1.length()) == 0 && !isNull) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + + default: + { + std::ostringstream oss; + oss << "regexo: datatype of " << execplan::colDataTypeToString(ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } -} +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_in::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_in::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - PredicateOperator op; - CalpontSystemCatalog::ColType ct; + PredicateOperator op; + CalpontSystemCatalog::ColType ct; - // @bug 4230. Initialize ct to be the first argument. - if (!fp.empty()) - ct = fp[0]->data()->resultType(); + // @bug 4230. Initialize ct to be the first argument. + if (!fp.empty()) + ct = fp[0]->data()->resultType(); - bool allString = true; + bool allString = true; - for (uint32_t i = 0; i < fp.size(); i++) + for (uint32_t i = 0; i < fp.size(); i++) + { + // op.setOpType(op.operationType(), fp[i]->data()->resultType()); + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT) { - //op.setOpType(op.operationType(), fp[i]->data()->resultType()); - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT) - { - allString = false; - op.setOpType(ct, fp[i]->data()->resultType()); - ct = op.operationType(); - } + allString = false; + op.setOpType(ct, fp[i]->data()->resultType()); + ct = op.operationType(); } + } - if (allString) + if (allString) + { + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + } + + // convert date const value according to the compare type here. + if (op.operationType().colDataType == CalpontSystemCatalog::DATE) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) { - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - } + cc = dynamic_cast(fp[i]->data()); - // convert date const value according to the compare type here. - if (op.operationType().colDataType == CalpontSystemCatalog::DATE) + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::dateToInt(result.strVal); + cc->result(result); + } + } + } + else if (op.operationType().colDataType == CalpontSystemCatalog::DATETIME) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) { - ConstantColumn* cc = NULL; + cc = dynamic_cast(fp[i]->data()); - for (uint32_t i = 1; i < fp.size(); i++) - { - cc = dynamic_cast(fp[i]->data()); - - if (cc) - { - Result result = cc->result(); - result.intVal = dataconvert::DataConvert::dateToInt(result.strVal); - cc->result(result); - } - } + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); + cc->result(result); + } } - else if (op.operationType().colDataType == CalpontSystemCatalog::DATETIME) - { - ConstantColumn* cc = NULL; + } - for (uint32_t i = 1; i < fp.size(); i++) - { - cc = dynamic_cast(fp[i]->data()); - - if (cc) - { - Result result = cc->result(); - result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); - cc->result(result); - } - } - } - - return ct; + return ct; } -bool Func_in::getBoolVal(rowgroup::Row& row, - FunctionParm& pm, - bool& isNull, +bool Func_in::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBoolForIn(row, pm, isNull, ct, false) && !isNull; + return getBoolForIn(row, pm, isNull, ct, false) && !isNull; } - - - - -CalpontSystemCatalog::ColType Func_notin::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_notin::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - PredicateOperator* op = new PredicateOperator(); - CalpontSystemCatalog::ColType ct; - op->setOpType(fp[0]->data()->resultType(), fp[1]->data()->resultType()); - return op->operationType(); + PredicateOperator* op = new PredicateOperator(); + CalpontSystemCatalog::ColType ct; + op->setOpType(fp[0]->data()->resultType(), fp[1]->data()->resultType()); + return op->operationType(); } -bool Func_notin::getBoolVal(rowgroup::Row& row, - FunctionParm& pm, - bool& isNull, +bool Func_notin::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return (!getBoolForIn(row, pm, isNull, ct, true) && !isNull); + return (!getBoolForIn(row, pm, isNull, ct, true) && !isNull); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index 7ac21cd77..77c560cad 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_inet_aton.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_inet_aton.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include "functor_real.h" @@ -31,68 +31,62 @@ namespace funcexp { - //------------------------------------------------------------------------------ // Return input argument type. // See mcs_add in udfsdk.h for explanation of this function. //------------------------------------------------------------------------------ execplan::CalpontSystemCatalog::ColType Func_inet_aton::operationType( - FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType) + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) { - return fp[0]->data()->resultType(); // input type + return fp[0]->data()->resultType(); // input type } //------------------------------------------------------------------------------ // Return IP address as a long long int value. // SELECT ... WHERE inet_aton(ipstring) = 11111111 will call getIntVal() //------------------------------------------------------------------------------ -int64_t Func_inet_aton::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_inet_aton::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_aton::getIntVal" << std::endl; + // std::cout << "In Func_inet_aton::getIntVal" << std::endl; - int64_t iValue = joblist::NULL_INT64; + int64_t iValue = joblist::NULL_INT64; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton(sValue, isNull); if (!isNull) - { - int64_t iVal = convertAton( sValue, isNull ); + iValue = iVal; + } - if (!isNull) - iValue = iVal; - } - - return iValue; + return iValue; } //------------------------------------------------------------------------------ // Return IP address as a double value. // SELECT ... WHERE inet_aton(ipstring) = '11111111' will call getDoubleVal() //------------------------------------------------------------------------------ -double Func_inet_aton::getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +double Func_inet_aton::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_aton::getDoubleVal" << std::endl; + // std::cout << "In Func_inet_aton::getDoubleVal" << std::endl; - double dValue = doubleNullVal(); + double dValue = doubleNullVal(); - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iValue = convertAton(sValue, isNull); if (!isNull) - { - int64_t iValue = convertAton( sValue, isNull ); + dValue = iValue; + } - if (!isNull) - dValue = iValue; - } - - return dValue; + return dValue; } //------------------------------------------------------------------------------ @@ -103,21 +97,19 @@ double Func_inet_aton::getDoubleVal(rowgroup::Row& row, // a syntactically valid one. // Don't know if this function will ever be called. //------------------------------------------------------------------------------ -std::string Func_inet_aton::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_inet_aton::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_aton::getStrVal" << std::endl; + // std::cout << "In Func_inet_aton::getStrVal" << std::endl; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); - if (!isNull) - { - convertAton( sValue, isNull ); // ignore return value - } + if (!isNull) + { + convertAton(sValue, isNull); // ignore return value + } - return sValue; + return sValue; } //------------------------------------------------------------------------------ @@ -125,63 +117,59 @@ std::string Func_inet_aton::getStrVal(rowgroup::Row& row, // getBoolVal() makes no sense for inet_aton() but we will implement anyway. // Don't know if this function will ever be called. //------------------------------------------------------------------------------ -bool Func_inet_aton::getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +bool Func_inet_aton::getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - bool bValue = false; + bool bValue = false; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); - if (!isNull) - { - int64_t iVal = convertAton( sValue, isNull ); + if (!isNull) + { + int64_t iVal = convertAton(sValue, isNull); - if ((!isNull) && (iVal != 0)) - bValue = true; - } + if ((!isNull) && (iVal != 0)) + bValue = true; + } - return bValue; + return bValue; } //------------------------------------------------------------------------------ // Return IP address as a decimal value. // SELECT ... WHERE inet_aton(ipstring) = 11111111. will call getDecimalVal() //------------------------------------------------------------------------------ -execplan::IDB_Decimal Func_inet_aton::getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +execplan::IDB_Decimal Func_inet_aton::getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - execplan::CalpontSystemCatalog::ColType colType = fp[0]->data()->resultType(); + execplan::CalpontSystemCatalog::ColType colType = fp[0]->data()->resultType(); - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); - if (!datatypes::Decimal::isWideDecimalTypeByPrecision(colType.precision)) + if (!datatypes::Decimal::isWideDecimalTypeByPrecision(colType.precision)) + { + if (!isNull) { - if (!isNull) - { - int64_t iValue = convertAton( sValue, isNull ); + int64_t iValue = convertAton(sValue, isNull); - if (!isNull) - return execplan::IDB_Decimal( iValue, colType.scale, colType.precision ); - } - - return execplan::IDB_Decimal( joblist::NULL_INT64, colType.scale, colType.precision ); + if (!isNull) + return execplan::IDB_Decimal(iValue, colType.scale, colType.precision); } - else + + return execplan::IDB_Decimal(joblist::NULL_INT64, colType.scale, colType.precision); + } + else + { + if (!isNull) { - if (!isNull) - { - int64_t iValue = convertAton( sValue, isNull ); + int64_t iValue = convertAton(sValue, isNull); - if (!isNull) - return execplan::IDB_Decimal( 0, colType.scale, colType.precision, (int128_t) iValue ); - } - - return execplan::IDB_Decimal( 0, colType.scale, colType.precision, datatypes::Decimal128Null ); + if (!isNull) + return execplan::IDB_Decimal(0, colType.scale, colType.precision, (int128_t)iValue); } + + return execplan::IDB_Decimal(0, colType.scale, colType.precision, datatypes::Decimal128Null); + } } //------------------------------------------------------------------------------ @@ -189,24 +177,22 @@ execplan::IDB_Decimal Func_inet_aton::getDecimalVal(rowgroup::Row& row, // getDateIntVal() makes no sense for inet_aton() but we will implement anyway. // Don't know if this function will ever be called. //------------------------------------------------------------------------------ -int32_t Func_inet_aton::getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int32_t Func_inet_aton::getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int32_t iValue = joblist::DATENULL; + int32_t iValue = joblist::DATENULL; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton(sValue, isNull); if (!isNull) - { - int64_t iVal = convertAton( sValue, isNull ); + iValue = iVal; + } - if (!isNull) - iValue = iVal; - } - - return iValue; + return iValue; } //------------------------------------------------------------------------------ @@ -215,64 +201,58 @@ int32_t Func_inet_aton::getDateIntVal(rowgroup::Row& row, // anyway. // Don't know if this function will ever be called. //------------------------------------------------------------------------------ -int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t iValue = joblist::DATETIMENULL; + int64_t iValue = joblist::DATETIMENULL; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton(sValue, isNull); if (!isNull) - { - int64_t iVal = convertAton( sValue, isNull ); + iValue = iVal; + } - if (!isNull) - iValue = iVal; - } - - return iValue; + return iValue; } -int64_t Func_inet_aton::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_inet_aton::getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t iValue = joblist::TIMESTAMPNULL; + int64_t iValue = joblist::TIMESTAMPNULL; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton(sValue, isNull); if (!isNull) - { - int64_t iVal = convertAton( sValue, isNull ); + iValue = iVal; + } - if (!isNull) - iValue = iVal; - } - - return iValue; + return iValue; } -int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t iValue = joblist::TIMENULL; + int64_t iValue = joblist::TIMENULL; - const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton(sValue, isNull); if (!isNull) - { - int64_t iVal = convertAton( sValue, isNull ); + iValue = iVal; + } - if (!isNull) - iValue = iVal; - } - - return iValue; + return iValue; } //------------------------------------------------------------------------------ @@ -280,83 +260,79 @@ int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, // isNull is set to true if the IP address string has invalid content. // Source code based on MySQL source (Item_func_inet_aton() in item_func.cc). //------------------------------------------------------------------------------ -int64_t Func_inet_aton::convertAton( - const std::string& ipString, - bool& isNull ) +int64_t Func_inet_aton::convertAton(const std::string& ipString, bool& isNull) { - char c = '.'; - int dot_count = 0; - unsigned int byte_result = 0; - unsigned long long result = 0; + char c = '.'; + int dot_count = 0; + unsigned int byte_result = 0; + unsigned long long result = 0; - const char* p = ipString.c_str(); - const char* end = p + ipString.length(); + const char* p = ipString.c_str(); + const char* end = p + ipString.length(); - // Loop through bytes in the IP address string - while (p < end) + // Loop through bytes in the IP address string + while (p < end) + { + c = *p++; + + int digit = (int)(c - '0'); // Assume ascii + + if (digit >= 0 && digit <= 9) { - c = *p++; + // Add the next digit from the string to byte_result + if ((byte_result = byte_result * 10 + digit) > 255) + { + // Wrong address + isNull = true; + return 0; + } + } + // Detect end of one portion of the IP address. + // Shift current result over 8 bits, and add next byte (byte_result) + else if (c == '.') + { + dot_count++; + result = (result << 8) + (unsigned long long)byte_result; + byte_result = 0; + } + // Exit loop if/when we encounter end of string for fixed length column, + // that is padded with '\0' at the end. + else if (c == '\0') + { + break; + } + else + { + // Invalid character + isNull = true; + return 0; + } + } - int digit = (int) (c - '0'); // Assume ascii + if (c != '.') // IP number can't end on '.' + { + // + // Handle short-forms addresses according to standard. Examples: + // 127 -> 0.0.0.127 + // 127.1 -> 127.0.0.1 + // 127.2.1 -> 127.2.0.1 + // + switch (dot_count) + { + case 1: result <<= 8; /* Fall through */ - if (digit >= 0 && digit <= 9) - { - // Add the next digit from the string to byte_result - if ((byte_result = byte_result * 10 + digit) > 255) - { - // Wrong address - isNull = true; - return 0; - } - } - // Detect end of one portion of the IP address. - // Shift current result over 8 bits, and add next byte (byte_result) - else if (c == '.') - { - dot_count++; - result = (result << 8) + (unsigned long long) byte_result; - byte_result = 0; - } - // Exit loop if/when we encounter end of string for fixed length column, - // that is padded with '\0' at the end. - else if (c == '\0') - { - break; - } - else - { - // Invalid character - isNull = true; - return 0; - } + case 2: result <<= 8; /* Fall through */ } - if (c != '.') // IP number can't end on '.' - { - // - // Handle short-forms addresses according to standard. Examples: - // 127 -> 0.0.0.127 - // 127.1 -> 127.0.0.1 - // 127.2.1 -> 127.2.0.1 - // - switch (dot_count) - { - case 1: - result <<= 8; /* Fall through */ + // std::cout << "aton: " << + // (result << 8) + (unsigned long long) byte_result << std::endl; - case 2: - result <<= 8; /* Fall through */ - } + return (result << 8) + (unsigned long long)byte_result; + } -// std::cout << "aton: " << -// (result << 8) + (unsigned long long) byte_result << std::endl; - - return (result << 8) + (unsigned long long) byte_result; - } - - // Invalid IP address ended in '.' - isNull = true; - return 0; + // Invalid IP address ended in '.' + isNull = true; + return 0; } -} +} // namespace funcexp diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index ea528ceb6..d3ac55dd9 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -16,16 +16,16 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_inet_ntoa.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_inet_ntoa.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #ifndef _MSC_VER #include #include #endif -#include // included when debugging +#include // included when debugging #include #include @@ -38,7 +38,6 @@ namespace funcexp { - //------------------------------------------------------------------------------ // The only accessor function that appears to make sense in conjunction with // inet_ntoa() is getStrVal(). I tested in MySQL (outside infinidb), and @@ -57,10 +56,9 @@ namespace funcexp // See mcs_add in udfsdk.h for explanation of this function. //------------------------------------------------------------------------------ execplan::CalpontSystemCatalog::ColType Func_inet_ntoa::operationType( - FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType) + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) { - return fp[0]->data()->resultType(); // input type + return fp[0]->data()->resultType(); // input type } //------------------------------------------------------------------------------ @@ -68,34 +66,32 @@ execplan::CalpontSystemCatalog::ColType Func_inet_ntoa::operationType( // Not sure this is ever called, but emulated getDoubleVal() implementation, // to be safe. (See getDoubleVal() description) //------------------------------------------------------------------------------ -int64_t Func_inet_ntoa::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_inet_ntoa::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getIntVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getIntVal" << std::endl; - std::string sValue = getStrVal( row, fp, isNull, op_ct ); - int64_t iValue = joblist::NULL_INT64; + std::string sValue = getStrVal(row, fp, isNull, op_ct); + int64_t iValue = joblist::NULL_INT64; - if ( !isNull ) + if (!isNull) + { + unsigned int newLength = sValue.length(); + std::string::size_type dot1 = sValue.find('.'); + + if (dot1 != std::string::npos) { - unsigned int newLength = sValue.length(); - std::string::size_type dot1 = sValue.find('.'); - - if (dot1 != std::string::npos) - { - newLength = dot1; - } - - if (newLength != sValue.length()) - sValue.resize(newLength); - - std::istringstream iss( sValue ); - iss >> iValue; + newLength = dot1; } - return iValue; + if (newLength != sValue.length()) + sValue.resize(newLength); + + std::istringstream iss(sValue); + iss >> iValue; + } + + return iValue; } //------------------------------------------------------------------------------ @@ -103,205 +99,183 @@ int64_t Func_inet_ntoa::getIntVal(rowgroup::Row& row, // SELECT ... WHERE inet_ntoa(ipstring) = 1.1 will call getDoubleVal() // SELECT ... WHERE inet_ntoa(ipstring) = 1 will also call getDoubleVal() //------------------------------------------------------------------------------ -double Func_inet_ntoa::getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +double Func_inet_ntoa::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getDoubleVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getDoubleVal" << std::endl; - std::string sValue = getStrVal( row, fp, isNull, op_ct ); - double dValue = doubleNullVal(); + std::string sValue = getStrVal(row, fp, isNull, op_ct); + double dValue = doubleNullVal(); - if ( !isNull ) + if (!isNull) + { + unsigned int newLength = sValue.length(); + std::string::size_type dot1 = sValue.find('.'); + + if ((dot1 != std::string::npos) && (sValue.length() > dot1 + 1)) { - unsigned int newLength = sValue.length(); - std::string::size_type dot1 = sValue.find('.'); + std::string::size_type dot2 = sValue.find('.', dot1 + 1); - if ((dot1 != std::string::npos) && (sValue.length() > dot1 + 1)) - { - std::string::size_type dot2 = sValue.find('.', dot1 + 1); - - if (dot2 != std::string::npos) - { - newLength = dot2; - } - } - - if (newLength != sValue.length()) - sValue.resize(newLength); - - std::istringstream iss( sValue ); - iss >> dValue; + if (dot2 != std::string::npos) + { + newLength = dot2; + } } - return dValue; + if (newLength != sValue.length()) + sValue.resize(newLength); + + std::istringstream iss(sValue); + iss >> dValue; + } + + return dValue; } //------------------------------------------------------------------------------ // Return IP address as a string value. // This is the get function that makes sense to use. //------------------------------------------------------------------------------ -std::string Func_inet_ntoa::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_inet_ntoa::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getStrVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getStrVal" << std::endl; - std::string sValue; + std::string sValue; - int64_t iValue = 0; + int64_t iValue = 0; - // @bug 3628 reopened: get double and round up, if necessary; - // else just get integer value - if ((fp[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DECIMAL) || - (fp[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::UDECIMAL) || - (fp[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::FLOAT) || - (fp[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DOUBLE)) - { - double d = fp[0]->data()->getDoubleVal(row, isNull); + // @bug 3628 reopened: get double and round up, if necessary; + // else just get integer value + if ((fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) || + (fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL) || + (fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::FLOAT) || + (fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DOUBLE)) + { + double d = fp[0]->data()->getDoubleVal(row, isNull); - if (d >= 0.0) - iValue = (int64_t)(d + 0.5); - else - iValue = (int64_t)(d - 0.5); - } + if (d >= 0.0) + iValue = (int64_t)(d + 0.5); else - { - iValue = fp[0]->data()->getIntVal(row, isNull); - } + iValue = (int64_t)(d - 0.5); + } + else + { + iValue = fp[0]->data()->getIntVal(row, isNull); + } - if (!isNull) - { - // @bug 3628 reopened: add check for out of range values - if ((iValue < 0) || (iValue > UINT_MAX)) - isNull = true; - else - convertNtoa( iValue, sValue ); - } + if (!isNull) + { + // @bug 3628 reopened: add check for out of range values + if ((iValue < 0) || (iValue > UINT_MAX)) + isNull = true; + else + convertNtoa(iValue, sValue); + } - return sValue; + return sValue; } //------------------------------------------------------------------------------ // Return IP address as a boolean. // N/A so returning null. See explanation at the top of this source file. //------------------------------------------------------------------------------ -bool Func_inet_ntoa::getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +bool Func_inet_ntoa::getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getBoolVal" << std::endl; - bool bValue = false; - isNull = true; + // std::cout << "In Func_inet_ntoa::getBoolVal" << std::endl; + bool bValue = false; + isNull = true; - return bValue; + return bValue; } //------------------------------------------------------------------------------ // Return IP address as a decimal value. // N/A so returning null. See explanation at the top of this source file. //------------------------------------------------------------------------------ -execplan::IDB_Decimal Func_inet_ntoa::getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +execplan::IDB_Decimal Func_inet_ntoa::getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { -// IDB_Decimal dValue = fp[0]->data()->getDecimalVal(row, isNull); - execplan::IDB_Decimal dValue ( joblist::NULL_INT64, 0, 0 ); - isNull = true; + // IDB_Decimal dValue = fp[0]->data()->getDecimalVal(row, isNull); + execplan::IDB_Decimal dValue(joblist::NULL_INT64, 0, 0); + isNull = true; - return dValue; + return dValue; } //------------------------------------------------------------------------------ // Return IP address as a date. // N/A so returning null. See explanation at the top of this source file. //------------------------------------------------------------------------------ -int32_t Func_inet_ntoa::getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int32_t Func_inet_ntoa::getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getDateIntVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getDateIntVal" << std::endl; -// int32_t iValue = fp[0]->data()->getDateIntVal(row, isNull); - int32_t iValue = joblist::DATENULL; - isNull = true; + // int32_t iValue = fp[0]->data()->getDateIntVal(row, isNull); + int32_t iValue = joblist::DATENULL; + isNull = true; - return iValue; + return iValue; } //------------------------------------------------------------------------------ // Return IP address as a date/time. // N/A so returning null. See explanation at the top of this source file. //------------------------------------------------------------------------------ -int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getDatetimeVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getDatetimeVal" << std::endl; -// int64t iValue = fp[0]->data()->getDatetimeIntVal(row, isNull); - int64_t iValue = joblist::DATETIMENULL; - isNull = true; + // int64t iValue = fp[0]->data()->getDatetimeIntVal(row, isNull); + int64_t iValue = joblist::DATETIMENULL; + isNull = true; - return iValue; + return iValue; } -int64_t Func_inet_ntoa::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +int64_t Func_inet_ntoa::getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getTimestampVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getTimestampVal" << std::endl; -// int64t iValue = fp[0]->data()->getTimestampIntVal(row, isNull); - int64_t iValue = joblist::TIMESTAMPNULL; - isNull = true; + // int64t iValue = fp[0]->data()->getTimestampIntVal(row, isNull); + int64_t iValue = joblist::TIMESTAMPNULL; + isNull = true; - return iValue; + return iValue; } -int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; + // std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; -// int64t iValue = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t iValue = joblist::TIMENULL; - isNull = true; + // int64t iValue = fp[0]->data()->getTimeIntVal(row, isNull); + int64_t iValue = joblist::TIMENULL; + isNull = true; - return iValue; + return iValue; } //------------------------------------------------------------------------------ // Convert an integer IP address to its equivalent IP address string. // Source code based on MySQL source (Item_func_inet_ntoa() in item_strfunc.cc). //------------------------------------------------------------------------------ -void Func_inet_ntoa::convertNtoa( - int64_t ipNum, - std::string& ipString ) +void Func_inet_ntoa::convertNtoa(int64_t ipNum, std::string& ipString) { - struct sockaddr_in sa; - sa.sin_addr.s_addr = htonl(ipNum); + struct sockaddr_in sa; + sa.sin_addr.s_addr = htonl(ipNum); - // now get it back and print it + // now get it back and print it #ifdef _MSC_VER - ipString = inet_ntoa(sa.sin_addr); + ipString = inet_ntoa(sa.sin_addr); #else - char str[INET_ADDRSTRLEN]; - ipString = inet_ntop(AF_INET, &(sa.sin_addr), str, INET_ADDRSTRLEN); + char str[INET_ADDRSTRLEN]; + ipString = inet_ntop(AF_INET, &(sa.sin_addr), str, INET_ADDRSTRLEN); #endif } -} +} // namespace funcexp diff --git a/utils/funcexp/func_insert.cpp b/utils/funcexp/func_insert.cpp index a9d707d34..236752e96 100644 --- a/utils/funcexp/func_insert.cpp +++ b/utils/funcexp/func_insert.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_insert.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_insert.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include using namespace std; @@ -37,74 +37,69 @@ using namespace joblist; #include "utf8.h" using namespace utf8; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_insert::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_insert::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -std::string Func_insert::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_insert::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType&) { - string src; - string tnewstr; - int64_t start, length; + string src; + string tnewstr; + int64_t start, length; - stringValue(fp[0], row, isNull, src); - if (isNull) - return ""; + stringValue(fp[0], row, isNull, src); + if (isNull) + return ""; - stringValue(fp[3], row, isNull, tnewstr); - if (isNull) - return ""; + stringValue(fp[3], row, isNull, tnewstr); + if (isNull) + return ""; - start = fp[1]->data()->getIntVal(row, isNull); - if (isNull) - return ""; - start--; // Because SQL syntax is 1 based and we want 0 based. + start = fp[1]->data()->getIntVal(row, isNull); + if (isNull) + return ""; + start--; // Because SQL syntax is 1 based and we want 0 based. - length = fp[2]->data()->getIntVal(row, isNull); - if (isNull) - return ""; + length = fp[2]->data()->getIntVal(row, isNull); + if (isNull) + return ""; - CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); + CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); - // binLen represents the number of bytes - int64_t binLen = static_cast(src.length()); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen is number of characters - int64_t strLen = cs->numchars(pos, end); - - // Return the original string if start isn't within the string. - if ((start < 0) || start >= strLen) - return src; + // binLen represents the number of bytes + int64_t binLen = static_cast(src.length()); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen is number of characters + int64_t strLen = cs->numchars(pos, end); - if ((length < 0) || (length > strLen)) - length = strLen; + // Return the original string if start isn't within the string. + if ((start < 0) || start >= strLen) + return src; - // Convert start and length from characters to bytes. - start = cs->charpos(pos, end, start); - length = cs->charpos(pos+start, end, length); + if ((length < 0) || (length > strLen)) + length = strLen; - string out; - out.reserve(binLen - length + tnewstr.length() + 1); + // Convert start and length from characters to bytes. + start = cs->charpos(pos, end, start); + length = cs->charpos(pos + start, end, length); - out.append(src.c_str(), start); - out.append(tnewstr.c_str(), tnewstr.length()); - if (binLen - start - length > 0) - out.append(src.c_str() + start + length, binLen - start - length); + string out; + out.reserve(binLen - length + tnewstr.length() + 1); - return out; + out.append(src.c_str(), start); + out.append(tnewstr.c_str(), tnewstr.length()); + if (binLen - start - length > 0) + out.append(src.c_str() + start + length, binLen - start - length); + + return out; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_instr.cpp b/utils/funcexp/func_instr.cpp index 3b89aa453..ab0379cce 100644 --- a/utils/funcexp/func_instr.cpp +++ b/utils/funcexp/func_instr.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_instr.cpp 3645 2013-03-19 13:10:24Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_instr.cpp 3645 2013-03-19 13:10:24Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -33,60 +33,56 @@ using namespace execplan; namespace funcexp { -CalpontSystemCatalog::ColType Func_instr::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_instr::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + return ct; } -int64_t Func_instr::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_instr::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& colType) { - int64_t start = 0; - int64_t start0= 0; - my_match_t match; - - const std::string& str = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - return 0; - const char* s1 = str.c_str(); - uint32_t l1 = (uint32_t)str.length(); - - const std::string& substr =parm[1]->data()->getStrVal(row, isNull); - if (isNull) - return 0; + int64_t start = 0; + int64_t start0 = 0; + my_match_t match; - const char* s2 = substr.c_str(); - uint32_t l2 = (uint32_t)substr.length(); - if (l2 < 1) - return start + 1; + const std::string& str = parm[0]->data()->getStrVal(row, isNull); + if (isNull) + return 0; + const char* s1 = str.c_str(); + uint32_t l1 = (uint32_t)str.length(); - CHARSET_INFO* cs = colType.getCharset(); - - if (parm.size() == 3) - { - start0 = start = parm[2]->data()->getIntVal(row, isNull) - 1; - - if ((start < 0) || (start > l1)) - return 0; - - start = (int64_t)cs->charpos(s1, s1+l1, start); // adjust start for multi-byte + const std::string& substr = parm[1]->data()->getStrVal(row, isNull); + if (isNull) + return 0; - if (start + l2 > l1) // Substring is longer than str at pos. - return 0; - } - - if (!cs->instr(s1+start, l1-start, - s2, l2, - &match, 1)) - return 0; - return (int64_t)match.mb_len + start0 + 1; + const char* s2 = substr.c_str(); + uint32_t l2 = (uint32_t)substr.length(); + if (l2 < 1) + return start + 1; + + CHARSET_INFO* cs = colType.getCharset(); + + if (parm.size() == 3) + { + start0 = start = parm[2]->data()->getIntVal(row, isNull) - 1; + + if ((start < 0) || (start > l1)) + return 0; + + start = (int64_t)cs->charpos(s1, s1 + l1, start); // adjust start for multi-byte + + if (start + l2 > l1) // Substring is longer than str at pos. + return 0; + } + + if (!cs->instr(s1 + start, l1 - start, s2, l2, &match, 1)) + return 0; + return (int64_t)match.mb_len + start0 + 1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_isnull.cpp b/utils/funcexp/func_isnull.cpp index ffda2415d..48337b3ec 100644 --- a/utils/funcexp/func_isnull.cpp +++ b/utils/funcexp/func_isnull.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_isnull.cpp 3648 2013-03-19 21:33:52Z dhall $ -* -* -****************************************************************************/ + * $Id: func_isnull.cpp 3648 2013-03-19 21:33:52Z dhall $ + * + * + ****************************************************************************/ #include #include @@ -37,13 +37,12 @@ using namespace rowgroup; namespace funcexp { - -CalpontSystemCatalog::ColType Func_isnull::operationType (FunctionParm& fp, - CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_isnull::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type of idb_isnull should be the same as the argument type - assert (fp.size() == 1); - return fp[0]->data()->resultType(); + // operation type of idb_isnull should be the same as the argument type + assert(fp.size() == 1); + return fp[0]->data()->resultType(); } /** @@ -51,44 +50,33 @@ CalpontSystemCatalog::ColType Func_isnull::operationType (FunctionParm& fp, * * This would be the most commonly called API for idb_isnull function */ -bool Func_isnull::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +bool Func_isnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - switch (op_ct.colDataType) - { - // For the purpose of this function, one does not need to get the value of - // the argument. One only need to know if the argument is NULL. The passed - // in parameter isNull will be set if the parameter is evaluated NULL. - // Please note that before this function returns, isNull should be set to - // false, otherwise the result of the function would be considered NULL, - // which is not possible for idb_isnull(). - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - parm[0]->data()->getDecimalVal(row, isNull); - break; + switch (op_ct.colDataType) + { + // For the purpose of this function, one does not need to get the value of + // the argument. One only need to know if the argument is NULL. The passed + // in parameter isNull will be set if the parameter is evaluated NULL. + // Please note that before this function returns, isNull should be set to + // false, otherwise the result of the function would be considered NULL, + // which is not possible for idb_isnull(). + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: parm[0]->data()->getDecimalVal(row, isNull); break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - parm[0]->data()->getStrVal(row, isNull); - break; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: parm[0]->data()->getStrVal(row, isNull); break; - case CalpontSystemCatalog::LONGDOUBLE: - parm[0]->data()->getLongDoubleVal(row, isNull); - break; + case CalpontSystemCatalog::LONGDOUBLE: parm[0]->data()->getLongDoubleVal(row, isNull); break; - default: - parm[0]->data()->getIntVal(row, isNull); - } + default: parm[0]->data()->getIntVal(row, isNull); + } - bool ret = isNull; - // It's important to reset isNull indicator. - isNull = false; - return (fIsNotNull ? !ret : ret); + bool ret = isNull; + // It's important to reset isNull indicator. + isNull = false; + return (fIsNotNull ? !ret : ret); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index b20e8502f..9040c47d2 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_last_day.cpp 2477 2011-05-11 16:07:35Z chao $ -* -* -****************************************************************************/ + * $Id: func_last_day.cpp 2477 2011-05-11 16:07:35Z chao $ + * + * + ****************************************************************************/ #include #include @@ -37,168 +37,162 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_last_day::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_last_day::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_last_day::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_last_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint32_t year = 0; - uint32_t month = 0; - uint32_t day = 0; - int64_t val = 0; - DateTime aDateTime; - Time aTime; + uint32_t year = 0; + uint32_t month = 0; + uint32_t day = 0; + int64_t val = 0; + DateTime aDateTime; + Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - year = m_time.year; - month = m_time.month; - day = m_time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } - - break; - - default: - isNull = true; - return -1; + TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; } - uint32_t lastday = day; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; - if (isLeapYear(year) && (month == 2)) - { - lastday = 29; - } - else if ( month == 2) - { - lastday = 28; - } - else - { - lastday = daysInMonth[month - 1]; - } + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - if (day > lastday) - { + if (val == -1) + { isNull = true; return -1; - } + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } - dataconvert::DateTime aDay; - aDay.year = year; - aDay.month = month; - aDay.day = lastday; - aDay.hour = 0; - aDay.minute = 0; - aDay.second = 0; - aDay.msecond = 0; - val = *(reinterpret_cast(&aDay)); - return val; + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } + + break; + + default: isNull = true; return -1; + } + + uint32_t lastday = day; + + if (isLeapYear(year) && (month == 2)) + { + lastday = 29; + } + else if (month == 2) + { + lastday = 28; + } + else + { + lastday = daysInMonth[month - 1]; + } + + if (day > lastday) + { + isNull = true; + return -1; + } + + dataconvert::DateTime aDay; + aDay.year = year; + aDay.month = month; + aDay.day = lastday; + aDay.hour = 0; + aDay.minute = 0; + aDay.second = 0; + aDay.msecond = 0; + val = *(reinterpret_cast(&aDay)); + return val; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_lcase.cpp b/utils/funcexp/func_lcase.cpp index 53e4bc5b1..7de396e47 100644 --- a/utils/funcexp/func_lcase.cpp +++ b/utils/funcexp/func_lcase.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_lcase.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_lcase.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,39 +34,34 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_lcase::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_lcase::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -std::string Func_lcase::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_lcase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& colType) { - const string& tstr = fp[0]->data()->getStrVal(row, isNull); + const string& tstr = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; + if (isNull) + return ""; - CHARSET_INFO* cs = colType.getCharset(); - uint64_t inLen = tstr.length(); - uint64_t bufLen= inLen * cs->casedn_multiply; - char* outBuf = new char[bufLen]; - - uint64_t outLen = cs->casedn(tstr.c_str(), inLen, outBuf, bufLen); + CHARSET_INFO* cs = colType.getCharset(); + uint64_t inLen = tstr.length(); + uint64_t bufLen = inLen * cs->casedn_multiply; + char* outBuf = new char[bufLen]; - string ret = string(outBuf, outLen); - delete [] outBuf; - return ret; + uint64_t outLen = cs->casedn(tstr.c_str(), inLen, outBuf, bufLen); + + string ret = string(outBuf, outLen); + delete[] outBuf; + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index 3fe002d29..add78cfa7 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_least.cpp 3954 2013-07-08 16:30:15Z bpaul $ -* -* -****************************************************************************/ + * $Id: func_least.cpp 3954 2013-07-08 16:30:15Z bpaul $ + * + * + ****************************************************************************/ #include #include @@ -39,204 +39,183 @@ using namespace joblist; using namespace funcexp; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_least::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_least::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - //return fp[0]->data()->resultType(); - return resultType; + // operation type is not used by this functor + // return fp[0]->data()->resultType(); + return resultType; } -int64_t Func_least::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_least::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - double str = fp[0]->data()->getDoubleVal(row, isNull); + double str = fp[0]->data()->getDoubleVal(row, isNull); - double leastStr = str; + double leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - double str1 = fp[i]->data()->getDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + double str1 = fp[i]->data()->getDoubleVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return (int64_t) leastStr; + return (int64_t)leastStr; } -double Func_least::getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +double Func_least::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - double str = fp[0]->data()->getDoubleVal(row, isNull); + double str = fp[0]->data()->getDoubleVal(row, isNull); - double leastStr = str; + double leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - double str1 = fp[i]->data()->getDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + double str1 = fp[i]->data()->getDoubleVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return (double) leastStr; + return (double)leastStr; } -long double Func_least::getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +long double Func_least::getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - long double str = fp[0]->data()->getLongDoubleVal(row, isNull); + long double str = fp[0]->data()->getLongDoubleVal(row, isNull); - long double leastStr = str; + long double leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - long double str1 = fp[i]->data()->getLongDoubleVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + long double str1 = fp[i]->data()->getLongDoubleVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return leastStr; + return leastStr; } -std::string Func_least::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_least::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - string leastStr = fp[0]->data()->getStrVal(row, isNull); - CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); + string leastStr = fp[0]->data()->getStrVal(row, isNull); + CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); - for (uint32_t i = 1; i < fp.size(); i++) + for (uint32_t i = 1; i < fp.size(); i++) + { + const string& str1 = fp[i]->data()->getStrVal(row, isNull); + + if (cs->strnncoll(leastStr.c_str(), leastStr.length(), str1.c_str(), str1.length()) > 0) { - const string& str1 = fp[i]->data()->getStrVal(row, isNull); - - if (cs->strnncoll(leastStr.c_str(), leastStr.length(), str1.c_str(), str1.length()) > 0) - { - leastStr = str1; - } + leastStr = str1; } + } - return leastStr; + return leastStr; } -IDB_Decimal Func_least::getDecimalVal(Row& row, - FunctionParm& fp, - bool& isNull, +IDB_Decimal Func_least::getDecimalVal(Row& row, FunctionParm& fp, bool& isNull, CalpontSystemCatalog::ColType& ct) { - IDB_Decimal str = fp[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal str = fp[0]->data()->getDecimalVal(row, isNull); - IDB_Decimal leastStr = str; + IDB_Decimal leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - IDB_Decimal str1 = fp[i]->data()->getDecimalVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + IDB_Decimal str1 = fp[i]->data()->getDecimalVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return leastStr; + return leastStr; } -int32_t Func_least::getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int32_t Func_least::getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int32_t str = fp[0]->data()->getDateIntVal(row, isNull); + int32_t str = fp[0]->data()->getDateIntVal(row, isNull); - int32_t leastStr = str; + int32_t leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - int32_t str1 = fp[i]->data()->getDateIntVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + int32_t str1 = fp[i]->data()->getDateIntVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return leastStr; + return leastStr; } -int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t str = fp[0]->data()->getDatetimeIntVal(row, isNull); + int64_t str = fp[0]->data()->getDatetimeIntVal(row, isNull); - int64_t leastStr = str; + int64_t leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - int64_t str1 = fp[i]->data()->getDatetimeIntVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getDatetimeIntVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return leastStr; + return leastStr; } -int64_t Func_least::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_least::getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t str = fp[0]->data()->getTimestampIntVal(row, isNull); + int64_t str = fp[0]->data()->getTimestampIntVal(row, isNull); - int64_t leastStr = str; + int64_t leastStr = str; - for (uint32_t i = 1; i < fp.size(); i++) - { - int64_t str1 = fp[i]->data()->getTimestampIntVal(row, isNull); + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimestampIntVal(row, isNull); - if ( leastStr > str1 ) - leastStr = str1; - } + if (leastStr > str1) + leastStr = str1; + } - return leastStr; + return leastStr; } -int64_t Func_least::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_least::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - // Strip off unused day - int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); + // Strip off unused day + int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t str = leastStr << 12; + int64_t str = leastStr << 12; - for (uint32_t i = 1; i < fp.size(); i++) + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; + + if (str > str2) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); - int64_t str2 = str1 << 12; - - if ( str > str2 ) - { - leastStr = str1; - str = str2; - } + leastStr = str1; + str = str2; } + } - return leastStr; + return leastStr; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_left.cpp b/utils/funcexp/func_left.cpp index ae81a6b41..a90d5f46c 100644 --- a/utils/funcexp/func_left.cpp +++ b/utils/funcexp/func_left.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_left.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_left.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -33,51 +33,44 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_left::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_left::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_left::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_left::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; - size_t trimLength = fp[1]->data()->getUintVal(row, isNull); - if (isNull || trimLength <= 0) - return ""; + size_t trimLength = fp[1]->data()->getUintVal(row, isNull); + if (isNull || trimLength <= 0) + return ""; - size_t charPos; + size_t charPos; - if ((binLen <= trimLength) || - (binLen <= (charPos= cs->charpos(pos, end, trimLength)))) - { - return src; - } + if ((binLen <= trimLength) || (binLen <= (charPos = cs->charpos(pos, end, trimLength)))) + { + return src; + } - std::string ret(pos, charPos); - return ret; + std::string ret(pos, charPos); + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_length.cpp b/utils/funcexp/func_length.cpp index 89900f680..f63e2f1bb 100644 --- a/utils/funcexp/func_length.cpp +++ b/utils/funcexp/func_length.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_length.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_length.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -33,29 +33,26 @@ using namespace execplan; #include "rowgroup.h" - namespace funcexp { -CalpontSystemCatalog::ColType Func_length::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_length::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + return ct; } -int64_t Func_length::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_length::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, CalpontSystemCatalog::ColType&) { - if ((fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARBINARY) || - (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::BLOB)) - return fp[0]->data()->getStrVal(row, isNull).length(); + if ((fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARBINARY) || + (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::BLOB)) + return fp[0]->data()->getStrVal(row, isNull).length(); - return strlen(fp[0]->data()->getStrVal(row, isNull).c_str()); + return strlen(fp[0]->data()->getStrVal(row, isNull).c_str()); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_lpad.cpp b/utils/funcexp/func_lpad.cpp index 1022f3b24..0ad0bdc25 100644 --- a/utils/funcexp/func_lpad.cpp +++ b/utils/funcexp/func_lpad.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_lpad.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_lpad.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include "errorids.h" #include @@ -41,95 +41,90 @@ namespace funcexp { const string Func_lpad::fPad = " "; - -CalpontSystemCatalog::ColType Func_lpad::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_lpad::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_lpad::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_lpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // In the case where someone entered pad length as a quoted string, - // it may be interpreted by columnstore to be an actual string - // and stored in fResult.int as a htonl of that string, - // However fResult.double is always correct, so we'll use that. - size_t padLength = (size_t)fp[1]->data()->getDoubleVal(row, isNull); - if (isNull || padLength <= 0) - return ""; - if (padLength > (size_t)INT_MAX32) - padLength = (size_t)INT_MAX32; - - if (padLength < strLen) - { - binLen = cs->charpos(pos, end, padLength); - std::string ret(pos, binLen); - return ret; - } + // In the case where someone entered pad length as a quoted string, + // it may be interpreted by columnstore to be an actual string + // and stored in fResult.int as a htonl of that string, + // However fResult.double is always correct, so we'll use that. + size_t padLength = (size_t)fp[1]->data()->getDoubleVal(row, isNull); + if (isNull || padLength <= 0) + return ""; + if (padLength > (size_t)INT_MAX32) + padLength = (size_t)INT_MAX32; - // The pad characters. - const string* pad = &fPad; // Defaults to space - if (fp.size() > 2) - { - pad = &fp[2]->data()->getStrVal(row, isNull); - } - // binPLen represents the number of bytes in pad - size_t binPLen = pad->length(); - const char* posP = pad->c_str(); - // plen = the number of characters in pad - size_t plen = cs->numchars(posP, posP+binPLen); - if (plen == 0) - return src; - - size_t byteCount = (padLength+1) * cs->mbmaxlen; // absolute maximun number of bytes - char* buf = new char[byteCount]; - char* pBuf = buf; - - padLength -= strLen; - byteCount = 0; - - while (padLength >= plen) - { - memcpy(pBuf, posP, binPLen); - padLength -= plen; - byteCount += binPLen; - pBuf += binPLen; - } - // Sometimes, in a case with multi-char pad, we need to add a partial pad - if (padLength > 0) - { - size_t partialSize = cs->charpos(posP, posP+binPLen, padLength); - memcpy(pBuf, posP, partialSize); - byteCount += partialSize; - pBuf += partialSize; - } - memcpy(pBuf, pos, binLen); - byteCount += binLen; - - std::string ret(buf, byteCount); - delete [] buf; + if (padLength < strLen) + { + binLen = cs->charpos(pos, end, padLength); + std::string ret(pos, binLen); return ret; + } + + // The pad characters. + const string* pad = &fPad; // Defaults to space + if (fp.size() > 2) + { + pad = &fp[2]->data()->getStrVal(row, isNull); + } + // binPLen represents the number of bytes in pad + size_t binPLen = pad->length(); + const char* posP = pad->c_str(); + // plen = the number of characters in pad + size_t plen = cs->numchars(posP, posP + binPLen); + if (plen == 0) + return src; + + size_t byteCount = (padLength + 1) * cs->mbmaxlen; // absolute maximun number of bytes + char* buf = new char[byteCount]; + char* pBuf = buf; + + padLength -= strLen; + byteCount = 0; + + while (padLength >= plen) + { + memcpy(pBuf, posP, binPLen); + padLength -= plen; + byteCount += binPLen; + pBuf += binPLen; + } + // Sometimes, in a case with multi-char pad, we need to add a partial pad + if (padLength > 0) + { + size_t partialSize = cs->charpos(posP, posP + binPLen, padLength); + memcpy(pBuf, posP, partialSize); + byteCount += partialSize; + pBuf += partialSize; + } + memcpy(pBuf, pos, binLen); + byteCount += binLen; + + std::string ret(buf, byteCount); + delete[] buf; + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_ltrim.cpp b/utils/funcexp/func_ltrim.cpp index f7ef9a38d..39fd7165f 100644 --- a/utils/funcexp/func_ltrim.cpp +++ b/utils/funcexp/func_ltrim.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_ltrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_ltrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,68 +34,63 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_ltrim::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_ltrim::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_ltrim::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& type) +std::string Func_ltrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // The trim characters. - const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); - // binTLen represents the number of bytes in trim - size_t binTLen = trim.length(); - const char* posT = trim.c_str(); - // strTLen = the number of characters in trim - size_t strTLen = cs->numchars(posT, posT+binTLen); - if (strTLen == 0 || strTLen > strLen) - return src; + // The trim characters. + const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); + // binTLen represents the number of bytes in trim + size_t binTLen = trim.length(); + const char* posT = trim.c_str(); + // strTLen = the number of characters in trim + size_t strTLen = cs->numchars(posT, posT + binTLen); + if (strTLen == 0 || strTLen > strLen) + return src; - if (binTLen == 1) + if (binTLen == 1) + { + // If the trim string is 1 byte, don't waste cpu for memcmp + while (pos < end && *pos == *posT) { - // If the trim string is 1 byte, don't waste cpu for memcmp - while (pos < end && *pos == *posT) - { - ++pos; - --binLen; - } + ++pos; + --binLen; } - else + } + else + { + while (pos + binTLen <= end && memcmp(pos, posT, binTLen) == 0) { - while (pos+binTLen <= end && memcmp(pos,posT,binTLen) == 0) - { - pos += binTLen; - binLen -= binTLen; - } + pos += binTLen; + binLen -= binTLen; } - // Turn back to a string - std::string ret(pos, binLen); - return ret; + } + // Turn back to a string + std::string ret(pos, binLen); + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_ltrim_oracle.cpp b/utils/funcexp/func_ltrim_oracle.cpp index 3a726c29e..234808e74 100644 --- a/utils/funcexp/func_ltrim_oracle.cpp +++ b/utils/funcexp/func_ltrim_oracle.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_ltrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_ltrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,72 +34,67 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_ltrim_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_ltrim_oracle::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_ltrim_oracle::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& type) +std::string Func_ltrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // The trim characters. - const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); - // binTLen represents the number of bytes in trim - size_t binTLen = trim.length(); - const char* posT = trim.c_str(); - // strTLen = the number of characters in trim - size_t strTLen = cs->numchars(posT, posT+binTLen); - if (strTLen == 0 || strTLen > strLen) - return src; + // The trim characters. + const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); + // binTLen represents the number of bytes in trim + size_t binTLen = trim.length(); + const char* posT = trim.c_str(); + // strTLen = the number of characters in trim + size_t strTLen = cs->numchars(posT, posT + binTLen); + if (strTLen == 0 || strTLen > strLen) + return src; - if (binTLen == 1) + if (binTLen == 1) + { + // If the trim string is 1 byte, don't waste cpu for memcmp + while (pos < end && *pos == *posT) { - // If the trim string is 1 byte, don't waste cpu for memcmp - while (pos < end && *pos == *posT) - { - ++pos; - --binLen; - } + ++pos; + --binLen; } - else + } + else + { + while (pos + binTLen <= end && memcmp(pos, posT, binTLen) == 0) { - while (pos+binTLen <= end && memcmp(pos,posT,binTLen) == 0) - { - pos += binTLen; - binLen -= binTLen; - } + pos += binTLen; + binLen -= binTLen; } - // Turn back to a string - std::string ret(pos, binLen); - if (binLen == 0) - { - isNull = true; - } - return ret; + } + // Turn back to a string + std::string ret(pos, binLen); + if (binLen == 0) + { + isNull = true; + } + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 8d36954f8..0418508d3 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_makedate.cpp 2665 2011-06-01 20:42:52Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_makedate.cpp 2665 2011-06-01 20:42:52Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -39,169 +39,155 @@ using namespace funcexp; namespace { -uint64_t makedate(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull) +uint64_t makedate(rowgroup::Row& row, FunctionParm& parm, bool& isNull) { - int64_t year = 0; - string dayofyear; + int64_t year = 0; + string dayofyear; - //get year - switch (parm[0]->data()->resultType().colDataType) + // get year + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - year = (int64_t) value; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - year = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); - break; - } - - default: - isNull = true; - return 0; + double value = parm[0]->data()->getDoubleVal(row, isNull); + year = (int64_t)value; + break; } - if (year < 70) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - year = 2000 + year; + year = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); + break; } - else if (year < 100) - { - year = 1900 + year; - } - else if (year < 1000 || year > 9999) + + default: isNull = true; return 0; + } + + if (year < 70) + { + year = 2000 + year; + } + else if (year < 100) + { + year = 1900 + year; + } + else if (year < 1000 || year > 9999) + { + isNull = true; + return 0; + } + + // get dayofyear + switch (parm[1]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: { + dayofyear = parm[1]->data()->getStrVal(row, isNull); + + if (atoi(dayofyear.c_str()) < 1) + { isNull = true; return 0; + } + + break; } - //get dayofyear - switch (parm[1]->data()->resultType().colDataType) - { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - dayofyear = parm[1]->data()->getStrVal(row, isNull); - - if (atoi(dayofyear.c_str()) < 1) - { - isNull = true; - return 0; - } - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - int64_t tmp = parm[1]->data()->getDecimalVal(row, isNull).toSInt64Round(); - if (tmp < 1) - { - isNull = true; - return 0; - } - dayofyear = helpers::intToString(tmp); - break; - } - - case CalpontSystemCatalog::TIME: - { - std::ostringstream ss; - char buf[9]; - uint64_t aTime = parm[1]->data()->getTimeIntVal(row, isNull); - DataConvert::timeToString1(aTime, buf, 9); - dayofyear = buf; - break; - } - - default: - isNull = true; - return 0; - } - - if (atoi(dayofyear.c_str()) == 0) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { + int64_t tmp = parm[1]->data()->getDecimalVal(row, isNull).toSInt64Round(); + if (tmp < 1) + { isNull = true; return 0; + } + dayofyear = helpers::intToString(tmp); + break; } - // convert the year to a date in our internal format, then subtract - // one since we are about to add the day of year back in - Date d(year, 1, 1); - //@Bug 5232. spare bit is set, cannot use regular substraction - d.day -= 1; - //uint64_t intDate = ((*(reinterpret_cast (&d))) & 0xFFFFFFC) - 1; - uint64_t intDate = *(reinterpret_cast (&d)); - - uint64_t value = helpers::dateAdd( intDate, dayofyear, IntervalColumn::INTERVAL_DAY, true, OP_ADD ); - - if ( value == 0 ) + case CalpontSystemCatalog::TIME: { - isNull = true; + std::ostringstream ss; + char buf[9]; + uint64_t aTime = parm[1]->data()->getTimeIntVal(row, isNull); + DataConvert::timeToString1(aTime, buf, 9); + dayofyear = buf; + break; } - return value; -} - + default: isNull = true; return 0; + } + + if (atoi(dayofyear.c_str()) == 0) + { + isNull = true; + return 0; + } + + // convert the year to a date in our internal format, then subtract + // one since we are about to add the day of year back in + Date d(year, 1, 1); + //@Bug 5232. spare bit is set, cannot use regular substraction + d.day -= 1; + // uint64_t intDate = ((*(reinterpret_cast (&d))) & 0xFFFFFFC) - 1; + uint64_t intDate = *(reinterpret_cast(&d)); + + uint64_t value = helpers::dateAdd(intDate, dayofyear, IntervalColumn::INTERVAL_DAY, true, OP_ADD); + + if (value == 0) + { + isNull = true; + } + + return value; } +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_makedate::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_makedate::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_makedate::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_makedate::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - return makedate(row, parm, isNull); + return makedate(row, parm, isNull); } - -string Func_makedate::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_makedate::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - uint64_t value = makedate(row, parm, isNull); + uint64_t value = makedate(row, parm, isNull); - if (isNull) - return ""; + if (isNull) + return ""; - return dataconvert::DataConvert::dateToString(value); + return dataconvert::DataConvert::dateToString(value); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index e6d339666..6d0a9d903 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_maketime.cpp 2665 2011-06-01 20:42:52Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_maketime.cpp 2665 2011-06-01 20:42:52Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -37,147 +37,138 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_maketime::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_maketime::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_maketime::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_maketime::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - int64_t hour = 0; - int64_t min = 0; - int64_t sec = 0; + int64_t hour = 0; + int64_t min = 0; + int64_t sec = 0; - //get hour - switch (parm[0]->data()->resultType().colDataType) + // get hour + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - hour = (int64_t) value; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - hour = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); - break; - } - - default: - isNull = true; - return ""; + double value = parm[0]->data()->getDoubleVal(row, isNull); + hour = (int64_t)value; + break; } - //get minute - switch (parm[1]->data()->resultType().colDataType) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - double value = parm[1]->data()->getDoubleVal(row, isNull); - min = (int64_t) value; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - min = parm[1]->data()->getDecimalVal(row, isNull).toSInt64Round(); - break; - } - - default: - isNull = true; - return ""; + hour = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); + break; } - if (min < 0 || min > 59) + default: isNull = true; return ""; + } + + // get minute + switch (parm[1]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: { - isNull = true; - return ""; + double value = parm[1]->data()->getDoubleVal(row, isNull); + min = (int64_t)value; + break; } - //get second - switch (parm[2]->data()->resultType().colDataType) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - double value = parm[2]->data()->getDoubleVal(row, isNull); - sec = (int64_t) value; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - sec = parm[2]->data()->getDecimalVal(row, isNull).toSInt64Round(); - break; - } - - default: - isNull = true; - return ""; + min = parm[1]->data()->getDecimalVal(row, isNull).toSInt64Round(); + break; } - if (sec < 0 || sec > 59) + default: isNull = true; return ""; + } + + if (min < 0 || min > 59) + { + isNull = true; + return ""; + } + + // get second + switch (parm[2]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: { - isNull = true; - return ""; + double value = parm[2]->data()->getDoubleVal(row, isNull); + sec = (int64_t)value; + break; } - if (hour > 838) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - hour = 838; - min = 59; - sec = 59; + sec = parm[2]->data()->getDecimalVal(row, isNull).toSInt64Round(); + break; } - if (hour < -838) - { - hour = -838; - min = 59; - sec = 59; - } + default: isNull = true; return ""; + } - // in worst case hour is 4 characters (3 digits + '-') so max - // string length is 11 (4:2:2 + '\0') - char buf[11]; - snprintf(buf, 11, "%02d:%02d:%02d", (int) hour, (int) min, (int) sec); - return buf; + if (sec < 0 || sec > 59) + { + isNull = true; + return ""; + } + + if (hour > 838) + { + hour = 838; + min = 59; + sec = 59; + } + + if (hour < -838) + { + hour = -838; + min = 59; + sec = 59; + } + + // in worst case hour is 4 characters (3 digits + '-') so max + // string length is 11 (4:2:2 + '\0') + char buf[11]; + snprintf(buf, 11, "%02d:%02d:%02d", (int)hour, (int)min, (int)sec); + return buf; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index a64e8fd75..5f822d539 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -17,15 +17,15 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_math.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_math.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include #include -#define _USE_MATH_DEFINES // MSC: enable math defines +#define _USE_MATH_DEFINES // MSC: enable math defines #include #include using namespace std; @@ -57,2278 +57,2223 @@ namespace { inline double degrees(double radian) { - return (radian * 180.0 / M_PI); + return (radian * 180.0 / M_PI); } inline double radians(double degree) { - return (degree * M_PI / 180.0); + return (degree * M_PI / 180.0); } #if defined(_MSC_VER) && MSC_VER < 1800 inline double log2(double x) { - return (log(x) / M_LN2); + return (log(x) / M_LN2); } #endif -} +} // namespace namespace funcexp { - // // acos // -CalpontSystemCatalog::ColType Func_acos::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_acos::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_acos::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_acos::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } - return acos(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return acos(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return acos((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return acos((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return acos((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return acos((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "acos: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return acos(value); } + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "acos: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } - // // asin // -CalpontSystemCatalog::ColType Func_asin::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_asin::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_asin::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_asin::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } - return asin(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by isNull - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return asin(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return asin((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return asin((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return asin((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull || (value < -1.0 || value > 1.0)) - { - isNull = true; - return doubleNullVal(); - } - - return asin((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "asin: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return asin(value); } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by isNull + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "asin: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } // // atan // -CalpontSystemCatalog::ColType Func_atan::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_atan::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_atan::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_atan::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return atan2(value, value2); - } - - return atan(value); + isNull = true; + return doubleNullVal(); } - break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by isNull - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + return atan2(value, value2); + } - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - if (parm.size() > 1 ) - { - long double value2 = parm[1]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return atan2(value, value2); - } - return atan(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return atan2(value, value2); - } - - return atan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return atan2(value, value2); - } - - return atan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return atan2(value, value2); - } - - return atan((double)value); - } - break; - - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return atan2(value, value2); - } - - return atan((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "atan: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return atan(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by isNull + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + if (parm.size() > 1) + { + long double value2 = parm[1]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + return atan(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "atan: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // cos // -CalpontSystemCatalog::ColType Func_cos::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_cos::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_cos::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_cos::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = true; - return doubleNullVal(); - } + if (isNull) + { + isNull = true; + return doubleNullVal(); + } - return cos(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by isNull - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return cos(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return cos((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return cos((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return cos((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return cos((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "cos: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return cos(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by isNull + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "cos: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // cot // -CalpontSystemCatalog::ColType Func_cot::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_cot::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_cot::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_cot::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (value == 0) - { - Message::Args args; - args.add("cot"); - args.add(value); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add(value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - if (isNull) - { - isNull = true; - return doubleNullVal(); - } + if (isNull) + { + isNull = true; + return doubleNullVal(); + } - return 1.0 / tan(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (value == 0) - { - Message::Args args; - args.add("cot"); - args.add((double)value); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return 1.0 / tan(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (value == 0) - { - Message::Args args; - args.add("cot"); - args.add(value); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return 1.0 / tan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (value == 0) - { - Message::Args args; - args.add("cot"); - args.add((uint64_t)value); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return 1.0 / tan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (value == 0) - { - Message::Args args; - args.add("cot"); - args.add((uint64_t)value); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return 1.0 / tan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (value == 0) - { - Message::Args args; - args.add("cot"); - args.add((uint64_t)value); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return 1.0 / tan((double)value); - } - break; - - - default: - { - std::ostringstream oss; - oss << "cot: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return 1.0 / tan(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((double)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add(value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "cot: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // log // -CalpontSystemCatalog::ColType Func_log::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_log::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_log::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_log::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0)) { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull || (value2 <= 0.0 || value == 1.0) ) - { - isNull = true; - return doubleNullVal(); - } - - return log(value2) / log(value); - } - - return log(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - long double value2 = parm[1]->data()->getLongDoubleVal(row, isNull); - - if (isNull || (value2 <= 0.0 || value == 1.0) ) - { - isNull = true; - return doubleNullVal(); - } - - return log(value2) / log(value); - } - - return log(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull || (value2 <= 0.0 || value == 1.0) ) - { - isNull = true; - return doubleNullVal(); - } - - return log(value2) / log((double)value); - } - - return log((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull || (value2 <= 0.0 || value == 1.0) ) - { - isNull = true; - return doubleNullVal(); - } - - return log(value2) / log((double)value); - } - - return log((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull || (value2 <= 0.0 || value == 1.0) ) - { - isNull = true; - return doubleNullVal(); - } - - return log(value2) / log((double)value); - } - - return log((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - if (parm.size() > 1 ) - { - double value2 = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull || (value2 <= 0.0 || value == 1.0) ) - { - isNull = true; - return doubleNullVal(); - } - - return log(value2) / log((double)value); - } - - return log((double)value); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - isNull = true; - return doubleNullVal(); + isNull = true; + return doubleNullVal(); } - default: - { - std::ostringstream oss; - oss << "log: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return log(value2) / log(value); + } + + return log(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + long double value2 = parm[1]->data()->getLongDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log(value); + } + + return log(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + isNull = true; + return doubleNullVal(); + } + + default: + { + std::ostringstream oss; + oss << "log: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // log2 // -CalpontSystemCatalog::ColType Func_log2::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_log2::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_log2::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_log2::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } - return log2(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log2(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log2(value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log2(value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log2(value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log2(value); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - isNull = true; - return doubleNullVal(); - } - - default: - { - std::ostringstream oss; - oss << "log2: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return log2(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + isNull = true; + return doubleNullVal(); + } + + default: + { + std::ostringstream oss; + oss << "log2: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // log10 // -CalpontSystemCatalog::ColType Func_log10::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_log10::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_log10::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_log10::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } - return log10(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log10(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log10((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log10((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log10((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull || value <= 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return log10((double)value); - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - isNull = true; - return doubleNullVal(); - } - - default: - { - std::ostringstream oss; - oss << "log10: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return log10(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + isNull = true; + return doubleNullVal(); + } + + default: + { + std::ostringstream oss; + oss << "log10: datatype of " + << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // sin // -CalpontSystemCatalog::ColType Func_sin::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_sin::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_sin::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_sin::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = true; - return doubleNullVal(); - } + if (isNull) + { + isNull = true; + return doubleNullVal(); + } - return sin(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return sin(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return sin((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return sin((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return sin((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return sin((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "sin: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return sin(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "sin: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // sqrt // -CalpontSystemCatalog::ColType Func_sqrt::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_sqrt::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_sqrt::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_sqrt::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull || value < 0) - { - isNull = true; - return doubleNullVal(); - } + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } - return sqrt(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull || value < 0.0) - { - isNull = true; - return doubleNullVal(); - } - - return sqrt(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull || value < 0) - { - isNull = true; - return doubleNullVal(); - } - - return sqrt((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull || value < 0) - { - isNull = true; - return doubleNullVal(); - } - - return sqrt((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull || value < 0) - { - isNull = true; - return doubleNullVal(); - } - - return sqrt((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull || value < 0) - { - isNull = true; - return doubleNullVal(); - } - - return sqrt((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "sqrt: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return sqrt(value); } -} + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull || value < 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "sqrt: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } +} // // tan // -CalpontSystemCatalog::ColType Func_tan::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_tan::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_tan::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_tan::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = true; - return doubleNullVal(); - } + if (isNull) + { + isNull = true; + return doubleNullVal(); + } - return tan(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by NaN - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return tan(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return tan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return tan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return tan((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return tan((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "tan: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return tan(value); } + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by NaN + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "tan: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } - // // format // -CalpontSystemCatalog::ColType Func_format::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_format::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -string Func_format::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_format::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - string value; - int scale = 0; + string value; + int scale = 0; - if (parm.size() > 1 ) - scale = parm[1]->data()->getIntVal(row, isNull); + if (parm.size() > 1) + scale = parm[1]->data()->getIntVal(row, isNull); - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - value = parm[0]->data()->getStrVal(row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - double rawValue = parm[0]->data()->getDoubleVal(row, isNull); - - // roundup - if (scale < 0) scale = 0; - - if (rawValue >= 0) - rawValue += 0.5 / pow(10.0, scale); - else - rawValue -= 0.5 / pow(10.0, scale); - - // double's can be *really* long to print out. Max mysql - // is e308 so allow for 308 + 36 decimal places minimum. - char buf[384]; - snprintf(buf, 384, "%0.36f", rawValue); - value = buf; - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - value = dataconvert::DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - - // This is an unacceptable way of doing rounding - //perform rouding if needed - if ( scale < 0 ) - scale = 0; - - if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - if ( scale < decimal.scale ) - { - int64_t d = 0; - int128_t p = 1; - - if (!isNull && parm.size() > 1) - { - d = scale; - - if (!isNull) - helpers::decimalPlaceDec(d, p, decimal.scale); - } - - if (isNull) - break; - - int128_t x = decimal.s128Value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int128_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) - { - if (x >= 0) - x += h; - else - x -= h; - - if (p != 0) - x = x / p; - else - x = 0; - } - else - { - x = 0; - } - } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.s128Value = x; - } - value = decimal.toString(true); - } - else - { - if ( scale < decimal.scale ) - { - int64_t d = 0; - int64_t p = 1; - - if (!isNull && parm.size() > 1) - { - d = scale; - - if (!isNull) - helpers::decimalPlaceDec(d, p, decimal.scale); - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int64_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) - { - if (x >= 0) - x += h; - else - x -= h; - - if (p != 0) - x = x / p; - else - x = 0; - } - else - { - x = 0; - } - } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.value = x; - } - value = decimal.toString(); - } - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double rawValue = parm[0]->data()->getLongDoubleVal(row, isNull); - - // roundup - if (scale < 0) scale = 0; - - if (rawValue >= 0) - rawValue += 0.5 / pow(10.0, scale); - else - rawValue -= 0.5 / pow(10.0, scale); - - // double's can be *really* long to print out. Max mysql - // is e308 so allow for 308 + 36 decimal places minimum. - char buf[384]; - snprintf(buf, 384, "%0.36Lf", rawValue); - value = buf; - } - break; - - default: - { - std::ostringstream oss; - oss << "format: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + value = parm[0]->data()->getStrVal(row, isNull); } + break; - if ( scale <= 0 ) + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - // strip off any scale value - string::size_type pos = value.find ('.', 0); - - if (pos != string::npos) - value = value.substr(0, pos); + double rawValue = parm[0]->data()->getDoubleVal(row, isNull); + // roundup + if (scale < 0) scale = 0; + + if (rawValue >= 0) + rawValue += 0.5 / pow(10.0, scale); + else + rawValue -= 0.5 / pow(10.0, scale); + + // double's can be *really* long to print out. Max mysql + // is e308 so allow for 308 + 36 decimal places minimum. + char buf[384]; + snprintf(buf, 384, "%0.36f", rawValue); + value = buf; } - else + break; + + case execplan::CalpontSystemCatalog::DATE: { - // add or adjust scale value - int pad = scale; - string::size_type pos = value.find ('.', 0); - - if (pos == string::npos) - value = value.append("."); - else - { - //if value begins with '.', prefix with a '0' - if (pos == 0) - { - value = "0" + value; - pos++; - } - - //remove part of scale value, if needed - value = value.substr(0, pos + scale + 1); - - pad = scale - ( value.size() - pos - 1 ); - } - - // pad extra with '0' - if (*(value.data()) != '#') - { - for ( int i = 0 ; i < pad ; i++ ) - { - value = value.append("0"); - } - } - - scale++; + value = dataconvert::DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); } + break; - int comma = value.size() - scale; - int end = 0; - string::size_type pos = value.find ('-', 0); + case execplan::CalpontSystemCatalog::DATETIME: + { + value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), + timeZone()); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); + + // This is an unacceptable way of doing rounding + // perform rouding if needed + if (scale < 0) + scale = 0; + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (scale < decimal.scale) + { + int64_t d = 0; + int128_t p = 1; + + if (!isNull && parm.size() > 1) + { + d = scale; + + if (!isNull) + helpers::decimalPlaceDec(d, p, decimal.scale); + } + + if (isNull) + break; + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int128_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; + } + value = decimal.toString(true); + } + else + { + if (scale < decimal.scale) + { + int64_t d = 0; + int64_t p = 1; + + if (!isNull && parm.size() > 1) + { + d = scale; + + if (!isNull) + helpers::decimalPlaceDec(d, p, decimal.scale); + } + + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int64_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.value = x; + } + value = decimal.toString(); + } + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double rawValue = parm[0]->data()->getLongDoubleVal(row, isNull); + + // roundup + if (scale < 0) + scale = 0; + + if (rawValue >= 0) + rawValue += 0.5 / pow(10.0, scale); + else + rawValue -= 0.5 / pow(10.0, scale); + + // double's can be *really* long to print out. Max mysql + // is e308 so allow for 308 + 36 decimal places minimum. + char buf[384]; + snprintf(buf, 384, "%0.36Lf", rawValue); + value = buf; + } + break; + + default: + { + std::ostringstream oss; + oss << "format: datatype of " + << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + if (scale <= 0) + { + // strip off any scale value + string::size_type pos = value.find('.', 0); if (pos != string::npos) - end = 1; + value = value.substr(0, pos); - while ((comma -= 3) > end) + scale = 0; + } + else + { + // add or adjust scale value + int pad = scale; + string::size_type pos = value.find('.', 0); + + if (pos == string::npos) + value = value.append("."); + else { - value.insert(comma, ","); + // if value begins with '.', prefix with a '0' + if (pos == 0) + { + value = "0" + value; + pos++; + } + + // remove part of scale value, if needed + value = value.substr(0, pos + scale + 1); + + pad = scale - (value.size() - pos - 1); } - return value; -} + // pad extra with '0' + if (*(value.data()) != '#') + { + for (int i = 0; i < pad; i++) + { + value = value.append("0"); + } + } + scale++; + } + + int comma = value.size() - scale; + int end = 0; + string::size_type pos = value.find('-', 0); + + if (pos != string::npos) + end = 1; + + while ((comma -= 3) > end) + { + value.insert(comma, ","); + } + + return value; +} // // radians // -CalpontSystemCatalog::ColType Func_radians::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_radians::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_radians::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_radians::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = true; - return doubleNullVal(); - } + if (isNull) + { + isNull = true; + return doubleNullVal(); + } - return radians(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by isNull - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return radians(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return radians((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return radians((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return radians((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return radians((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "radians: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return radians(value); } + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by isNull + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "radians: datatype of " + << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } // // degrees // -CalpontSystemCatalog::ColType Func_degrees::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_degrees::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_degrees::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_degrees::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - // null value is indicated by isNull - double value = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double value = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = true; - return doubleNullVal(); - } + if (isNull) + { + isNull = true; + return doubleNullVal(); + } - return degrees(value); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - // null value is indicated by isNull - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return degrees(value); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t value = parm[0]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return degrees((double)value); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return degrees((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return degrees((double)value); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = true; - return doubleNullVal(); - } - - return degrees((double)value); - } - break; - - default: - { - std::ostringstream oss; - oss << "radians: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + return degrees(value); } + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + // null value is indicated by isNull + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees(value); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t value = parm[0]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t value = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + + default: + { + std::ostringstream oss; + oss << "radians: datatype of " + << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } } - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_md5.cpp b/utils/funcexp/func_md5.cpp index 9780c389d..6ef058e84 100644 --- a/utils/funcexp/func_md5.cpp +++ b/utils/funcexp/func_md5.cpp @@ -16,16 +16,15 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_md5.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_md5.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include #include #include - #include #include #include @@ -76,98 +75,95 @@ namespace // documentation and/or software. ///////////////////////////////////////////////////////////////////////// -typedef unsigned int uint4; +typedef unsigned int uint4; typedef unsigned short int uint2; -typedef unsigned char uchar; +typedef unsigned char uchar; char* PrintMD5(uchar md5Digest[16]); char* MD5String(const char* szString); -//char* MD5File(char* szFilename); +// char* MD5File(char* szFilename); class md5 { -// Methods -public: - md5() - { - Init(); - } - void Init(); - void Update(uchar* chInput, uint4 nInputLen); - void Finalize(); - uchar* Digest() - { - return m_Digest; - } + // Methods + public: + md5() + { + Init(); + } + void Init(); + void Update(uchar* chInput, uint4 nInputLen); + void Finalize(); + uchar* Digest() + { + return m_Digest; + } -private: + private: + void Transform(uchar* block); + void Encode(uchar* dest, uint4* src, uint4 nLength); + void Decode(uint4* dest, uchar* src, uint4 nLength); - void Transform(uchar* block); - void Encode(uchar* dest, uint4* src, uint4 nLength); - void Decode(uint4* dest, uchar* src, uint4 nLength); + inline uint4 rotate_left(uint4 x, uint4 n) + { + return ((x << n) | (x >> (32 - n))); + } + inline uint4 F(uint4 x, uint4 y, uint4 z) + { + return ((x & y) | (~x & z)); + } - inline uint4 rotate_left(uint4 x, uint4 n) - { - return ((x << n) | (x >> (32 - n))); - } + inline uint4 G(uint4 x, uint4 y, uint4 z) + { + return ((x & z) | (y & ~z)); + } - inline uint4 F(uint4 x, uint4 y, uint4 z) - { - return ((x & y) | (~x & z)); - } + inline uint4 H(uint4 x, uint4 y, uint4 z) + { + return (x ^ y ^ z); + } - inline uint4 G(uint4 x, uint4 y, uint4 z) - { - return ((x & z) | (y & ~z)); - } + inline uint4 I(uint4 x, uint4 y, uint4 z) + { + return (y ^ (x | ~z)); + } - inline uint4 H(uint4 x, uint4 y, uint4 z) - { - return (x ^ y ^ z); - } + inline void FF(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) + { + a += F(b, c, d) + x + ac; + a = rotate_left(a, s); + a += b; + } - inline uint4 I(uint4 x, uint4 y, uint4 z) - { - return (y ^ (x | ~z)); - } + inline void GG(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) + { + a += G(b, c, d) + x + ac; + a = rotate_left(a, s); + a += b; + } - inline void FF(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) - { - a += F(b, c, d) + x + ac; - a = rotate_left(a, s); - a += b; - } + inline void HH(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) + { + a += H(b, c, d) + x + ac; + a = rotate_left(a, s); + a += b; + } - inline void GG(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) - { - a += G(b, c, d) + x + ac; - a = rotate_left(a, s); - a += b; - } - - inline void HH(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) - { - a += H(b, c, d) + x + ac; - a = rotate_left(a, s); - a += b; - } - - inline void II(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) - { - a += I(b, c, d) + x + ac; - a = rotate_left(a, s); - a += b; - } - -// Data -private: - uint4 m_State[4]; - uint4 m_Count[2]; - uchar m_Buffer[64]; - uchar m_Digest[16]; - uchar m_Finalized; + inline void II(uint4& a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) + { + a += I(b, c, d) + x + ac; + a = rotate_left(a, s); + a += b; + } + // Data + private: + uint4 m_State[4]; + uint4 m_Count[2]; + uchar m_Buffer[64]; + uchar m_Digest[16]; + uchar m_Finalized; }; ///////////////////////////////////////////////////////////////////////// @@ -205,12 +201,9 @@ private: // documentation and/or software. ///////////////////////////////////////////////////////////////////////// -static unsigned char PADDING[64] = -{ - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; +static unsigned char PADDING[64] = {0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; #define S11 7 #define S12 12 @@ -229,40 +222,38 @@ static unsigned char PADDING[64] = #define S43 15 #define S44 21 - // PrintMD5: Converts a completed md5 digest into a char* string. char* PrintMD5(uchar md5Digest[16]) { - char chBuffer[256]; - char chEach[10]; - int nCount; - size_t chEachSize = 0; + char chBuffer[256]; + char chEach[10]; + int nCount; + size_t chEachSize = 0; - memset(chBuffer, 0, 256); - memset(chEach, 0, 10); + memset(chBuffer, 0, 256); + memset(chEach, 0, 10); - for (nCount = 0; nCount < 16; nCount++) - { - sprintf(chEach, "%02x", md5Digest[nCount]); - chEachSize = sizeof(chEach); - strncat(chBuffer, chEach, chEachSize); - } + for (nCount = 0; nCount < 16; nCount++) + { + sprintf(chEach, "%02x", md5Digest[nCount]); + chEachSize = sizeof(chEach); + strncat(chBuffer, chEach, chEachSize); + } - return strdup(chBuffer); + return strdup(chBuffer); } // MD5String: Performs the MD5 algorithm on a char* string, returning // the results as a char*. char* MD5String(const char* szString) { - int nLen = strlen(szString); - md5 alg; + int nLen = strlen(szString); + md5 alg; - alg.Update((unsigned char*)szString, (unsigned int)nLen); - alg.Finalize(); - - return PrintMD5(alg.Digest()); + alg.Update((unsigned char*)szString, (unsigned int)nLen); + alg.Finalize(); + return PrintMD5(alg.Digest()); } // this fcn isn't used, so commenting it @@ -305,12 +296,12 @@ char* MD5File(char* szFilename) // Initializes a new context. void md5::Init() { - memset(m_Count, 0, 2 * sizeof(uint4)); + memset(m_Count, 0, 2 * sizeof(uint4)); - m_State[0] = 0x67452301; - m_State[1] = 0xefcdab89; - m_State[2] = 0x98badcfe; - m_State[3] = 0x10325476; + m_State[0] = 0x67452301; + m_State[1] = 0xefcdab89; + m_State[2] = 0x98badcfe; + m_State[3] = 0x10325476; } // md5::Update @@ -319,35 +310,35 @@ void md5::Init() // context. void md5::Update(uchar* chInput, uint4 nInputLen) { - uint4 i, index, partLen; + uint4 i, index, partLen; - // Compute number of bytes mod 64 - index = (unsigned int)((m_Count[0] >> 3) & 0x3F); + // Compute number of bytes mod 64 + index = (unsigned int)((m_Count[0] >> 3) & 0x3F); - // Update number of bits - if ((m_Count[0] += (nInputLen << 3)) < (nInputLen << 3)) - m_Count[1]++; + // Update number of bits + if ((m_Count[0] += (nInputLen << 3)) < (nInputLen << 3)) + m_Count[1]++; - m_Count[1] += (nInputLen >> 29); + m_Count[1] += (nInputLen >> 29); - partLen = 64 - index; + partLen = 64 - index; - // Transform as many times as possible. - if (nInputLen >= partLen) - { - memcpy( &m_Buffer[index], chInput, partLen ); - Transform(m_Buffer); + // Transform as many times as possible. + if (nInputLen >= partLen) + { + memcpy(&m_Buffer[index], chInput, partLen); + Transform(m_Buffer); - for (i = partLen; i + 63 < nInputLen; i += 64) - Transform(&chInput[i]); + for (i = partLen; i + 63 < nInputLen; i += 64) + Transform(&chInput[i]); - index = 0; - } - else - i = 0; + index = 0; + } + else + i = 0; - // Buffer remaining input - memcpy( &m_Buffer[index], &chInput[i], nInputLen - i ); + // Buffer remaining input + memcpy(&m_Buffer[index], &chInput[i], nInputLen - i); } // md5::Finalize @@ -355,114 +346,114 @@ void md5::Update(uchar* chInput, uint4 nInputLen) // the message digest and zeroizing the context. void md5::Finalize() { - uchar bits[8]; - uint4 index, padLen; + uchar bits[8]; + uint4 index, padLen; - // Save number of bits - Encode (bits, m_Count, 8); + // Save number of bits + Encode(bits, m_Count, 8); - // Pad out to 56 mod 64 - index = (unsigned int)((m_Count[0] >> 3) & 0x3f); - padLen = (index < 56) ? (56 - index) : (120 - index); - Update(PADDING, padLen); + // Pad out to 56 mod 64 + index = (unsigned int)((m_Count[0] >> 3) & 0x3f); + padLen = (index < 56) ? (56 - index) : (120 - index); + Update(PADDING, padLen); - // Append length (before padding) - Update (bits, 8); + // Append length (before padding) + Update(bits, 8); - // Store state in digest - Encode (m_Digest, m_State, 16); + // Store state in digest + Encode(m_Digest, m_State, 16); - memset(m_Count, 0, 2 * sizeof(uint4)); - memset(m_State, 0, 4 * sizeof(uint4)); - memset(m_Buffer, 0, 64 * sizeof(uchar)); + memset(m_Count, 0, 2 * sizeof(uint4)); + memset(m_State, 0, 4 * sizeof(uint4)); + memset(m_Buffer, 0, 64 * sizeof(uchar)); } // md5::Transform // MD5 basic transformation. Transforms state based on block. -void md5::Transform (uchar* block) +void md5::Transform(uchar* block) { - uint4 a = m_State[0], b = m_State[1], c = m_State[2], d = m_State[3], x[16]; + uint4 a = m_State[0], b = m_State[1], c = m_State[2], d = m_State[3], x[16]; - Decode (x, block, 64); + Decode(x, block, 64); - // Round 1 - FF (a, b, c, d, x[ 0], S11, 0xd76aa478); - FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); - FF (c, d, a, b, x[ 2], S13, 0x242070db); - FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); - FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); - FF (d, a, b, c, x[ 5], S12, 0x4787c62a); - FF (c, d, a, b, x[ 6], S13, 0xa8304613); - FF (b, c, d, a, x[ 7], S14, 0xfd469501); - FF (a, b, c, d, x[ 8], S11, 0x698098d8); - FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); - FF (c, d, a, b, x[10], S13, 0xffff5bb1); - FF (b, c, d, a, x[11], S14, 0x895cd7be); - FF (a, b, c, d, x[12], S11, 0x6b901122); - FF (d, a, b, c, x[13], S12, 0xfd987193); - FF (c, d, a, b, x[14], S13, 0xa679438e); - FF (b, c, d, a, x[15], S14, 0x49b40821); + // Round 1 + FF(a, b, c, d, x[0], S11, 0xd76aa478); + FF(d, a, b, c, x[1], S12, 0xe8c7b756); + FF(c, d, a, b, x[2], S13, 0x242070db); + FF(b, c, d, a, x[3], S14, 0xc1bdceee); + FF(a, b, c, d, x[4], S11, 0xf57c0faf); + FF(d, a, b, c, x[5], S12, 0x4787c62a); + FF(c, d, a, b, x[6], S13, 0xa8304613); + FF(b, c, d, a, x[7], S14, 0xfd469501); + FF(a, b, c, d, x[8], S11, 0x698098d8); + FF(d, a, b, c, x[9], S12, 0x8b44f7af); + FF(c, d, a, b, x[10], S13, 0xffff5bb1); + FF(b, c, d, a, x[11], S14, 0x895cd7be); + FF(a, b, c, d, x[12], S11, 0x6b901122); + FF(d, a, b, c, x[13], S12, 0xfd987193); + FF(c, d, a, b, x[14], S13, 0xa679438e); + FF(b, c, d, a, x[15], S14, 0x49b40821); -// Round 2 - GG (a, b, c, d, x[ 1], S21, 0xf61e2562); - GG (d, a, b, c, x[ 6], S22, 0xc040b340); - GG (c, d, a, b, x[11], S23, 0x265e5a51); - GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); - GG (a, b, c, d, x[ 5], S21, 0xd62f105d); - GG (d, a, b, c, x[10], S22, 0x2441453); - GG (c, d, a, b, x[15], S23, 0xd8a1e681); - GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); - GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); - GG (d, a, b, c, x[14], S22, 0xc33707d6); - GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); - GG (b, c, d, a, x[ 8], S24, 0x455a14ed); - GG (a, b, c, d, x[13], S21, 0xa9e3e905); - GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); - GG (c, d, a, b, x[ 7], S23, 0x676f02d9); - GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); + // Round 2 + GG(a, b, c, d, x[1], S21, 0xf61e2562); + GG(d, a, b, c, x[6], S22, 0xc040b340); + GG(c, d, a, b, x[11], S23, 0x265e5a51); + GG(b, c, d, a, x[0], S24, 0xe9b6c7aa); + GG(a, b, c, d, x[5], S21, 0xd62f105d); + GG(d, a, b, c, x[10], S22, 0x2441453); + GG(c, d, a, b, x[15], S23, 0xd8a1e681); + GG(b, c, d, a, x[4], S24, 0xe7d3fbc8); + GG(a, b, c, d, x[9], S21, 0x21e1cde6); + GG(d, a, b, c, x[14], S22, 0xc33707d6); + GG(c, d, a, b, x[3], S23, 0xf4d50d87); + GG(b, c, d, a, x[8], S24, 0x455a14ed); + GG(a, b, c, d, x[13], S21, 0xa9e3e905); + GG(d, a, b, c, x[2], S22, 0xfcefa3f8); + GG(c, d, a, b, x[7], S23, 0x676f02d9); + GG(b, c, d, a, x[12], S24, 0x8d2a4c8a); - // Round 3 - HH (a, b, c, d, x[ 5], S31, 0xfffa3942); - HH (d, a, b, c, x[ 8], S32, 0x8771f681); - HH (c, d, a, b, x[11], S33, 0x6d9d6122); - HH (b, c, d, a, x[14], S34, 0xfde5380c); - HH (a, b, c, d, x[ 1], S31, 0xa4beea44); - HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); - HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); - HH (b, c, d, a, x[10], S34, 0xbebfbc70); - HH (a, b, c, d, x[13], S31, 0x289b7ec6); - HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); - HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); - HH (b, c, d, a, x[ 6], S34, 0x4881d05); - HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); - HH (d, a, b, c, x[12], S32, 0xe6db99e5); - HH (c, d, a, b, x[15], S33, 0x1fa27cf8); - HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); + // Round 3 + HH(a, b, c, d, x[5], S31, 0xfffa3942); + HH(d, a, b, c, x[8], S32, 0x8771f681); + HH(c, d, a, b, x[11], S33, 0x6d9d6122); + HH(b, c, d, a, x[14], S34, 0xfde5380c); + HH(a, b, c, d, x[1], S31, 0xa4beea44); + HH(d, a, b, c, x[4], S32, 0x4bdecfa9); + HH(c, d, a, b, x[7], S33, 0xf6bb4b60); + HH(b, c, d, a, x[10], S34, 0xbebfbc70); + HH(a, b, c, d, x[13], S31, 0x289b7ec6); + HH(d, a, b, c, x[0], S32, 0xeaa127fa); + HH(c, d, a, b, x[3], S33, 0xd4ef3085); + HH(b, c, d, a, x[6], S34, 0x4881d05); + HH(a, b, c, d, x[9], S31, 0xd9d4d039); + HH(d, a, b, c, x[12], S32, 0xe6db99e5); + HH(c, d, a, b, x[15], S33, 0x1fa27cf8); + HH(b, c, d, a, x[2], S34, 0xc4ac5665); - // Round 4 - II (a, b, c, d, x[ 0], S41, 0xf4292244); - II (d, a, b, c, x[ 7], S42, 0x432aff97); - II (c, d, a, b, x[14], S43, 0xab9423a7); - II (b, c, d, a, x[ 5], S44, 0xfc93a039); - II (a, b, c, d, x[12], S41, 0x655b59c3); - II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); - II (c, d, a, b, x[10], S43, 0xffeff47d); - II (b, c, d, a, x[ 1], S44, 0x85845dd1); - II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); - II (d, a, b, c, x[15], S42, 0xfe2ce6e0); - II (c, d, a, b, x[ 6], S43, 0xa3014314); - II (b, c, d, a, x[13], S44, 0x4e0811a1); - II (a, b, c, d, x[ 4], S41, 0xf7537e82); - II (d, a, b, c, x[11], S42, 0xbd3af235); - II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); - II (b, c, d, a, x[ 9], S44, 0xeb86d391); + // Round 4 + II(a, b, c, d, x[0], S41, 0xf4292244); + II(d, a, b, c, x[7], S42, 0x432aff97); + II(c, d, a, b, x[14], S43, 0xab9423a7); + II(b, c, d, a, x[5], S44, 0xfc93a039); + II(a, b, c, d, x[12], S41, 0x655b59c3); + II(d, a, b, c, x[3], S42, 0x8f0ccc92); + II(c, d, a, b, x[10], S43, 0xffeff47d); + II(b, c, d, a, x[1], S44, 0x85845dd1); + II(a, b, c, d, x[8], S41, 0x6fa87e4f); + II(d, a, b, c, x[15], S42, 0xfe2ce6e0); + II(c, d, a, b, x[6], S43, 0xa3014314); + II(b, c, d, a, x[13], S44, 0x4e0811a1); + II(a, b, c, d, x[4], S41, 0xf7537e82); + II(d, a, b, c, x[11], S42, 0xbd3af235); + II(c, d, a, b, x[2], S43, 0x2ad7d2bb); + II(b, c, d, a, x[9], S44, 0xeb86d391); - m_State[0] += a; - m_State[1] += b; - m_State[2] += c; - m_State[3] += d; + m_State[0] += a; + m_State[1] += b; + m_State[2] += c; + m_State[3] += d; - memset(x, 0, sizeof(x)); + memset(x, 0, sizeof(x)); } // md5::Encode @@ -470,17 +461,17 @@ void md5::Transform (uchar* block) // a multiple of 4. void md5::Encode(uchar* dest, uint4* src, uint4 nLength) { - uint4 i, j; + uint4 i, j; - idbassert(nLength % 4 == 0); + idbassert(nLength % 4 == 0); - for (i = 0, j = 0; j < nLength; i++, j += 4) - { - dest[j] = (uchar)(src[i] & 0xff); - dest[j + 1] = (uchar)((src[i] >> 8) & 0xff); - dest[j + 2] = (uchar)((src[i] >> 16) & 0xff); - dest[j + 3] = (uchar)((src[i] >> 24) & 0xff); - } + for (i = 0, j = 0; j < nLength; i++, j += 4) + { + dest[j] = (uchar)(src[i] & 0xff); + dest[j + 1] = (uchar)((src[i] >> 8) & 0xff); + dest[j + 2] = (uchar)((src[i] >> 16) & 0xff); + dest[j + 3] = (uchar)((src[i] >> 24) & 0xff); + } } // md5::Decode @@ -488,42 +479,37 @@ void md5::Encode(uchar* dest, uint4* src, uint4 nLength) // a multiple of 4. void md5::Decode(uint4* dest, uchar* src, uint4 nLength) { - uint4 i, j; + uint4 i, j; - idbassert(nLength % 4 == 0); - - for (i = 0, j = 0; j < nLength; i++, j += 4) - { - dest[i] = ((uint4)src[j]) | (((uint4)src[j + 1]) << 8) | - (((uint4)src[j + 2]) << 16) | (((uint4)src[j + 3]) << 24); - } -} + idbassert(nLength % 4 == 0); + for (i = 0, j = 0; j < nLength; i++, j += 4) + { + dest[i] = ((uint4)src[j]) | (((uint4)src[j + 1]) << 8) | (((uint4)src[j + 2]) << 16) | + (((uint4)src[j + 3]) << 24); + } } +} // namespace namespace funcexp { -CalpontSystemCatalog::ColType Func_md5::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_md5::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -string Func_md5::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_md5::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - const string& arg = parm[0]->data()->getStrVal(row, isNull); - return MD5String(arg.c_str()); + const string& arg = parm[0]->data()->getStrVal(row, isNull); + return MD5String(arg.c_str()); - //return str; + // return str; } - -} // namespace funcexp - +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index 18b1cdabb..aca1e3aea 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_microsecond.cpp 2477 2011-05-11 16:07:35Z chao $ -* -* -****************************************************************************/ + * $Id: func_microsecond.cpp 2477 2011-05-11 16:07:35Z chao $ + * + * + ****************************************************************************/ #include #include @@ -36,111 +36,105 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_microsecond::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_microsecond::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_microsecond::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_microsecond::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; - uint32_t microSecond = 0; + int64_t val = 0; + uint32_t microSecond = 0; - switch (parm[0]->data()->resultType().colDataType) - { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - microSecond = 0; - break; + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = 0; + break; - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - microSecond = (uint32_t)((val & 0xfffff)); - break; + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xfffff)); + break; - case CalpontSystemCatalog::TIMESTAMP: - val = parm[0]->data()->getIntVal(row, isNull); - microSecond = (uint32_t)((val & 0xfffff)); - break; + case CalpontSystemCatalog::TIMESTAMP: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xfffff)); + break; - case CalpontSystemCatalog::TIME: - val = parm[0]->data()->getIntVal(row, isNull); - microSecond = (uint32_t)((val & 0xffffff)); - break; + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xffffff)); + break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - if (val == -1) - { - isNull = true; - return -1; - } - else - { - microSecond = (uint32_t)((val & 0xfffff)); - } + if (val == -1) + { + isNull = true; + return -1; + } + else + { + microSecond = (uint32_t)((val & 0xfffff)); + } - break; + break; - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - { - isNull = true; - return -1; - } - else - { - microSecond = (uint32_t)((val & 0xfffff)); - } + if (val == -1) + { + isNull = true; + return -1; + } + else + { + microSecond = (uint32_t)((val & 0xfffff)); + } - break; + break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - { - isNull = true; - return -1; - } - else - { - microSecond = (uint32_t)((val & 0xfffff)); - } - } - else - { - isNull = true; - return -1; - } + if (val == -1) + { + isNull = true; + return -1; + } + else + { + microSecond = (uint32_t)((val & 0xfffff)); + } + } + else + { + isNull = true; + return -1; + } - break; + break; - default: - isNull = true; - return -1; - } + default: isNull = true; return -1; + } - return microSecond; + return microSecond; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index 672008048..4a5f594a7 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_minute.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_minute.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,110 +35,106 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_minute::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_minute::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_minute::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_minute::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; + int64_t val = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; + if (val == -1) + isNull = true; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - } - else - { - isNull = true; - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - { - isNull = true; - } - /* fall through */ - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - return m_time.minute; - } - - default: - { - isNull = true; - } + break; } - if (isNull) - return -1; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if ( val < 1000000000 ) - return 0; + if (val == -1) + isNull = true; + } + else + { + isNull = true; + } - return (unsigned)((val >> 26) & 0x3f); + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + { + isNull = true; + } + /* fall through */ + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + return m_time.minute; + } + + default: + { + isNull = true; + } + } + + if (isNull) + return -1; + + if (val < 1000000000) + return 0; + + return (unsigned)((val >> 26) & 0x3f); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index 250740510..bf33a6491 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_mod.cpp 3616 2013-03-04 14:56:29Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_mod.cpp 3616 2013-03-04 14:56:29Z rdempsey $ + * + * + ****************************************************************************/ #include using namespace std; @@ -39,517 +39,496 @@ using namespace rowgroup; #include "errorids.h" using namespace logging; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_mod::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_mod::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -IDB_Decimal Func_mod::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +IDB_Decimal Func_mod::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - if ( parm.size() < 2 ) + if (parm.size() < 2) + { + isNull = true; + return IDB_Decimal(); + } + + if (parm[0]->data()->resultType().isWideDecimalType() || parm[1]->data()->resultType().isWideDecimalType()) + { + IDB_Decimal div = parm[1]->data()->getDecimalVal(row, isNull); + int128_t divInt = + (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) ? div.s128Value : div.value; + + if (divInt == 0) { - isNull = true; - return IDB_Decimal(); - } - - if (parm[0]->data()->resultType().isWideDecimalType() || - parm[1]->data()->resultType().isWideDecimalType()) - { - IDB_Decimal div = parm[1]->data()->getDecimalVal(row, isNull); - int128_t divInt = (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) ? div.s128Value : div.value; - - if (divInt == 0) - { - isNull = true; - return IDB_Decimal(); - } - - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - - // two special cases: both Decimals has no scale - // or divisor has no scale - if (!div.isScaled()) - { - return IDB_Decimal(d % div.toTSInt128(), - d.scale, - datatypes::INT128MAXPRECISION); - } - // float division - else - { - int128_t dividendInt = (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) ? d.s128Value : d.value; - - - float128_t divF, dividendF; - - int128_t scaleDivisor; - - datatypes::getScaleDivisor(scaleDivisor, div.scale); - divF = (float128_t) divInt / scaleDivisor; - - datatypes::getScaleDivisor(scaleDivisor, d.scale); - dividendF = (float128_t) dividendInt / scaleDivisor; - - float128_t mod = datatypes::TFloat128::fmodq(dividendF, divF) * scaleDivisor; - - return IDB_Decimal(datatypes::TSInt128((int128_t) mod), - d.scale, - datatypes::INT128MAXPRECISION); - } - } - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) - { - isNull = true; - return IDB_Decimal(); + isNull = true; + return IDB_Decimal(); } IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - int lefto = d.value % (int)pow(10.0, d.scale); - int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; - // It is misterious but precision is set to 0! - return IDB_Decimal(mod, d.scale, 0); + // two special cases: both Decimals has no scale + // or divisor has no scale + if (!div.isScaled()) + { + return IDB_Decimal(d % div.toTSInt128(), d.scale, datatypes::INT128MAXPRECISION); + } + // float division + else + { + int128_t dividendInt = + (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) ? d.s128Value : d.value; + + float128_t divF, dividendF; + + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, div.scale); + divF = (float128_t)divInt / scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + dividendF = (float128_t)dividendInt / scaleDivisor; + + float128_t mod = datatypes::TFloat128::fmodq(dividendF, divF) * scaleDivisor; + + return IDB_Decimal(datatypes::TSInt128((int128_t)mod), d.scale, datatypes::INT128MAXPRECISION); + } + } + int64_t div = parm[1]->data()->getIntVal(row, isNull); + + if (div == 0) + { + isNull = true; + return IDB_Decimal(); + } + + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + int64_t value = d.value / pow(10.0, d.scale); + int lefto = d.value % (int)pow(10.0, d.scale); + + int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; + // It is misterious but precision is set to 0! + return IDB_Decimal(mod, d.scale, 0); } - -double Func_mod::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_mod::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - if ( parm.size() < 2 ) + if (parm.size() < 2) + { + isNull = true; + return 0; + } + + int64_t div = parm[1]->data()->getIntVal(row, isNull); + + if (div == 0) + { + isNull = true; + return 0; + } + + double mod = 0; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - isNull = true; - return 0; + int64_t value = parm[0]->data()->getIntVal(row, isNull); + + mod = value % div; + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); + uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); + + mod = uvalue % udiv; + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double value = parm[0]->data()->getDoubleVal(row, isNull); + + mod = fmod(value, div); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + mod = (double)fmodl(value, div); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float value = parm[0]->data()->getFloatVal(row, isNull); + + mod = fmod(value, div); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + mod = doDecimal(parm, div, row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + double value = parm[0]->data()->getDoubleVal(row, isNull); + mod = fmod(value, div); + break; } - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) + default: { - isNull = true; - return 0; + std::ostringstream oss; + oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } + } - double mod = 0; - - switch (parm[0]->data()->resultType().colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - int64_t value = parm[0]->data()->getIntVal(row, isNull); - - mod = value % div; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); - uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); - - mod = uvalue % udiv; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - mod = (double)fmodl(value, div); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float value = parm[0]->data()->getFloatVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - mod = doDecimal(parm, div, row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - mod = fmod(value, div); - break; - } - - default: - { - std::ostringstream oss; - oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - } - - return mod; + return mod; } -long double Func_mod::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) +long double Func_mod::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { - if ( parm.size() < 2 ) + if (parm.size() < 2) + { + isNull = true; + return 0; + } + + int64_t div = parm[1]->data()->getIntVal(row, isNull); + + if (div == 0) + { + isNull = true; + return 0; + } + + long double mod = 0; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - isNull = true; - return 0; + int64_t value = parm[0]->data()->getIntVal(row, isNull); + + mod = value % div; + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); + uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); + + mod = uvalue % udiv; + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double value = parm[0]->data()->getDoubleVal(row, isNull); + + mod = fmod(value, div); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float value = parm[0]->data()->getFloatVal(row, isNull); + + mod = fmod(value, div); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + mod = fmodl(value, div); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + mod = doDecimal(parm, div, row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + double value = parm[0]->data()->getDoubleVal(row, isNull); + mod = fmod(value, div); + break; } - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) + default: { - isNull = true; - return 0; + std::ostringstream oss; + oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } + } - long double mod = 0; - - switch (parm[0]->data()->resultType().colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - int64_t value = parm[0]->data()->getIntVal(row, isNull); - - mod = value % div; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); - uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); - - mod = uvalue % udiv; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float value = parm[0]->data()->getFloatVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - mod = fmodl(value, div); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - mod = doDecimal(parm, div, row, isNull); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - mod = fmod(value, div); - break; - } - - default: - { - std::ostringstream oss; - oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - } - - return mod; + return mod; } -int64_t Func_mod::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_mod::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - if ( parm.size() < 2 ) + if (parm.size() < 2) + { + isNull = true; + return 0; + } + + int64_t div = parm[1]->data()->getIntVal(row, isNull); + + if (div == 0) + { + isNull = true; + return 0; + } + + int64_t mod = 0; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARCHAR: { - isNull = true; - return 0; + int64_t value = parm[0]->data()->getIntVal(row, isNull); + + mod = value % div; } + break; - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - isNull = true; - return 0; + uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); + uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); + + mod = uvalue % udiv; } + break; - int64_t mod = 0; - - switch (parm[0]->data()->resultType().colDataType) + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARCHAR: - { - int64_t value = parm[0]->data()->getIntVal(row, isNull); + double value = parm[0]->data()->getDoubleVal(row, isNull); - mod = value % div; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); - uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); - - mod = uvalue % udiv; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float value = parm[0]->data()->getFloatVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - mod = fmodl(value, div); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - mod = doDecimal(parm, div, row, isNull); - } - break; - - default: - { - std::ostringstream oss; - oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + mod = fmod(value, div); } + break; - return mod; + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float value = parm[0]->data()->getFloatVal(row, isNull); + + mod = fmod(value, div); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + mod = fmodl(value, div); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + mod = doDecimal(parm, div, row, isNull); + } + break; + + default: + { + std::ostringstream oss; + oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return mod; } -uint64_t Func_mod::getUIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_mod::getUIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - if ( parm.size() < 2 ) + if (parm.size() < 2) + { + isNull = true; + return 0; + } + + int64_t div = parm[1]->data()->getIntVal(row, isNull); + + if (div == 0) + { + isNull = true; + return 0; + } + + uint64_t mod = 0; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARCHAR: { - isNull = true; - return 0; + int64_t value = parm[0]->data()->getIntVal(row, isNull); + + mod = value % div; } + break; - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - isNull = true; - return 0; + uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); + uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); + + mod = uvalue % udiv; } + break; - uint64_t mod = 0; - - switch (parm[0]->data()->resultType().colDataType) + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARCHAR: - { - int64_t value = parm[0]->data()->getIntVal(row, isNull); + double value = parm[0]->data()->getDoubleVal(row, isNull); - mod = value % div; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t udiv = parm[1]->data()->getIntVal(row, isNull); - uint64_t uvalue = parm[0]->data()->getUintVal(row, isNull); - - mod = uvalue % udiv; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double value = parm[0]->data()->getDoubleVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float value = parm[0]->data()->getFloatVal(row, isNull); - - mod = fmod(value, div); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - - mod = fmodl(value, div); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - mod = doDecimal(parm, div, row, isNull); - } - break; - - default: - { - std::ostringstream oss; - oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + mod = fmod(value, div); } + break; - return mod; + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float value = parm[0]->data()->getFloatVal(row, isNull); + + mod = fmod(value, div); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + + mod = fmodl(value, div); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + mod = doDecimal(parm, div, row, isNull); + } + break; + + default: + { + std::ostringstream oss; + oss << "mod: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return mod; } -std::string Func_mod::getStrVal(Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_mod::getStrVal(Row& row, FunctionParm& fp, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - if ( fp.size() < 2 ) - { - isNull = true; - return std::string(); - } + if (fp.size() < 2) + { + isNull = true; + return std::string(); + } - switch (fp[0]->data()->resultType().colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - return intToString(getIntVal(row, fp, isNull, op_ct)); - break; + switch (fp[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + return intToString(getIntVal(row, fp, isNull, op_ct)); + break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - return longDoubleToString(getLongDoubleVal(row, fp, isNull, op_ct)); - break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + return longDoubleToString(getLongDoubleVal(row, fp, isNull, op_ct)); + break; - default: - return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); - break; - } + default: return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); break; + } } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index f9cbfddd0..3f9136a04 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_month.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_month.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,117 +35,111 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_month::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_month::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_month::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_month::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + int64_t val = 0; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + return (unsigned)((val >> 12) & 0xf); + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + return (unsigned)((val >> 44) & 0xf); + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - return (unsigned)((val >> 12) & 0xf); - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - return (unsigned)((val >> 44) & 0xf); - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - return m_time.month; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - return (unsigned)((val >> 44) & 0xf); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 44) & 0xf); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 44) & 0xf); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 44) & 0xf); - } - } - else - { - isNull = true; - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + return m_time.month; } - return -1; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + return (unsigned)((val >> 44) & 0xf); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 44) & 0xf); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 44) & 0xf); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 44) & 0xf); + } + } + else + { + isNull = true; + } + + break; + + default: isNull = true; return -1; + } + + return -1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 8bc35a0c8..3f76db8ca 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_monthname.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_monthname.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,178 +36,159 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_monthname::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_monthname::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_monthname::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_monthname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int32_t month = getIntVal(row, parm, isNull, op_ct); + int32_t month = getIntVal(row, parm, isNull, op_ct); - if (month == -1) - return ""; + if (month == -1) + return ""; - return helpers::monthFullNames[month]; + return helpers::monthFullNames[month]; } -int32_t Func_monthname::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_monthname::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - uint32_t val = getIntVal(row, parm, isNull, ct); - return val; + uint32_t val = getIntVal(row, parm, isNull, ct); + return val; } -int64_t Func_monthname::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_monthname::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - uint32_t val = getIntVal(row, parm, isNull, ct); - return val; + uint32_t val = getIntVal(row, parm, isNull, ct); + return val; } -int64_t Func_monthname::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_monthname::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - uint32_t val = getIntVal(row, parm, isNull, ct); - return val; + uint32_t val = getIntVal(row, parm, isNull, ct); + return val; } -int64_t Func_monthname::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + int64_t val = 0; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + return (unsigned)((val >> 12) & 0xf); + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + return (unsigned)((val >> 44) & 0xf); + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - return (unsigned)((val >> 12) & 0xf); - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - return (unsigned)((val >> 44) & 0xf); - - case CalpontSystemCatalog::TIMESTAMP: - { - val = parm[0]->data()->getIntVal(row, isNull); - dataconvert::TimeStamp timestamp(val); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); - return time.month; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - return (unsigned)((val >> 44) & 0xf); - break; - - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 44) & 0xf); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 44) & 0xf); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 44) & 0xf); - } - } - - break; - - default: - isNull = true; - return -1; + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + return time.month; } - return -1; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + return (unsigned)((val >> 44) & 0xf); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 44) & 0xf); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 44) & 0xf); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 44) & 0xf); + } + } + + break; + + default: isNull = true; return -1; + } + + return -1; } -double Func_monthname::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_monthname::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return 0; + return 0; } -execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal d; + IDB_Decimal d; - if (fp[0]->data()->resultType().isWideDecimalType()) - d.s128Value = getIntVal(row, fp, isNull, op_ct); - else - d.value = getIntVal(row, fp, isNull, op_ct); - d.scale = 0; - return d; + if (fp[0]->data()->resultType().isWideDecimalType()) + d.s128Value = getIntVal(row, fp, isNull, op_ct); + else + d.value = getIntVal(row, fp, isNull, op_ct); + d.scale = 0; + return d; } - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 5fb5062b0..77c07c674 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_nullif.cpp 2485 2011-04-05 19:01:13Z zzhu $ -* -* -****************************************************************************/ + * $Id: func_nullif.cpp 2485 2011-04-05 19:01:13Z zzhu $ + * + * + ****************************************************************************/ #include #include @@ -44,1013 +44,981 @@ using namespace dataconvert; #include "funchelpers.h" using namespace funcexp; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_nullif::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_nullif::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_nullif::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_nullif::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t exp1 = parm[0]->data()->getIntVal(row, isNull); - int64_t exp2 = 0; + int64_t exp1 = parm[0]->data()->getIntVal(row, isNull); + int64_t exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); + exp2 = parm[1]->data()->getIntVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - exp2 = parm[1]->data()->getUintVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME) - { - // NULLIF arg0 is DATETIME, arg1 is DATE, - // Upgrade arg1 to datetime - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - } - else - { - // NULLIF arg0 is not DATETIME (might be date), arg1 is not DATE - // Use arg1 as DATE - exp2 = parm[1]->data()->getDateIntVal(row, isNull); - } - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - - if ((parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATE) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME)) - - { - // NULLIF arg0 is DATE, arg1 is DATETIME, - // Upgrade arg1 to datetime - // When comparing exp1 as a Date, we can't simply promote. We have - // to be careful of the return value in case of not null return. - int64_t exp1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - - if ( exp1 == exp2 ) - { - isNull = true; - return 0; - } - - // since exp1 here is inside the block, when we leave the block, the - // original (Date) value is restored. - } - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); - - if ((parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::TIMESTAMP) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME)) - - { - // NULLIF arg0 is DATETIME, arg1 is TIMESTAMP, - // Upgrade arg0 to timestamp - // When comparing exp1 as a Date, we can't simply promote. We have - // to be careful of the return value in case of not null return. - int64_t exp1 = parm[0]->data()->getTimestampIntVal(row, isNull); - - if ( exp1 == exp2 ) - { - isNull = true; - return 0; - } - - // since exp1 here is inside the block, when we leave the block, the - // original (Date) value is restored. - } - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - exp2 = parm[1]->data()->getTimeIntVal(row, isNull); - - if ((parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::TIME)) - { - // NULLIF arg0 is DATETIME, arg1 is TIME, - // Upgrade arg1 to time - // When comparing exp1 as a Date, we can't simply promote. We have - // to be careful of the return value in case of not null return. - int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); - - if ( exp1 == exp2 ) - { - isNull = true; - return 0; - } - - // since exp1 here is inside the block, when we leave the block, the - // original (Date) value is restored. - } - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - default: - { - isNull = true; - } - } - - if ( exp1 == exp2 ) - { - isNull = true; - return 0; - } - - return exp1; -} - -uint64_t Func_nullif::getUintVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) -{ - uint64_t exp1 = parm[0]->data()->getUintVal(row, isNull); - uint64_t exp2 = 0; - - switch (parm[1]->data()->resultType().colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - int64_t iexp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull || iexp2 < 0) - { - isNull = false; - return exp1; - } - - exp2 = iexp2; - break; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - exp2 = parm[1]->data()->getUintVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - exp2 = parm[1]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - exp2 = parm[1]->data()->getTimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - default: - { - isNull = true; - } - } - - if ( exp1 == exp2 ) - { - isNull = true; - return 0; - } - - return exp1; -} - -string Func_nullif::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) -{ - string exp1 = parm[0]->data()->getStrVal(row, isNull); - CHARSET_INFO* cs = parm[0]->data()->resultType().getCharset(); - - if (isNull) - { - isNull = false; - return ""; - } - - string exp2 = parm[1]->data()->getStrVal(row, isNull); - - if (isNull) - { + if (isNull) + { isNull = false; return exp1; + } + + break; } - int datatype0 = parm[0]->data()->resultType().colDataType; - int datatype1 = parm[1]->data()->resultType().colDataType; - - if ( datatype0 == execplan::CalpontSystemCatalog::DATE && - (datatype1 == execplan::CalpontSystemCatalog::DATETIME || - datatype1 == execplan::CalpontSystemCatalog::TIMESTAMP)) + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - exp1 = exp1 + " 00:00:00"; + exp2 = parm[1]->data()->getUintVal(row, isNull); + break; } - if ( datatype1 == execplan::CalpontSystemCatalog::DATE && - (datatype0 == execplan::CalpontSystemCatalog::DATETIME || - datatype0 == execplan::CalpontSystemCatalog::TIMESTAMP)) + case execplan::CalpontSystemCatalog::DATE: { - exp2 = exp2 + " 00:00:00"; - } + if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATETIME) + { + // NULLIF arg0 is DATETIME, arg1 is DATE, + // Upgrade arg1 to datetime + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + } + else + { + // NULLIF arg0 is not DATETIME (might be date), arg1 is not DATE + // Use arg1 as DATE + exp2 = parm[1]->data()->getDateIntVal(row, isNull); + } - if (cs->strnncoll(exp1.c_str(), exp1.length(), exp2.c_str(), exp2.length()) == 0) + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: { - isNull = true; - return ""; + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + + if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATE) || + (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATETIME)) + + { + // NULLIF arg0 is DATE, arg1 is DATETIME, + // Upgrade arg1 to datetime + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; } - return parm[0]->data()->getStrVal(row, isNull); + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) || + (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATETIME)) + + { + // NULLIF arg0 is DATETIME, arg1 is TIMESTAMP, + // Upgrade arg0 to timestamp + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATETIME) || + (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIME)) + { + // NULLIF arg0 is DATETIME, arg1 is TIME, + // Upgrade arg1 to time + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } -int32_t Func_nullif::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_nullif::getUintVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t exp1 = parm[0]->data()->getUintVal(row, isNull); + uint64_t exp2 = 0; + + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + int64_t iexp2 = parm[1]->data()->getIntVal(row, isNull); + + if (isNull || iexp2 < 0) + { + isNull = false; + return exp1; + } + + exp2 = iexp2; + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + exp2 = parm[1]->data()->getUintVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + exp2 = parm[1]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; +} + +string Func_nullif::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + string exp1 = parm[0]->data()->getStrVal(row, isNull); + CHARSET_INFO* cs = parm[0]->data()->resultType().getCharset(); + + if (isNull) + { + isNull = false; + return ""; + } + + string exp2 = parm[1]->data()->getStrVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + int datatype0 = parm[0]->data()->resultType().colDataType; + int datatype1 = parm[1]->data()->resultType().colDataType; + + if (datatype0 == execplan::CalpontSystemCatalog::DATE && + (datatype1 == execplan::CalpontSystemCatalog::DATETIME || + datatype1 == execplan::CalpontSystemCatalog::TIMESTAMP)) + { + exp1 = exp1 + " 00:00:00"; + } + + if (datatype1 == execplan::CalpontSystemCatalog::DATE && + (datatype0 == execplan::CalpontSystemCatalog::DATETIME || + datatype0 == execplan::CalpontSystemCatalog::TIMESTAMP)) + { + exp2 = exp2 + " 00:00:00"; + } + + if (cs->strnncoll(exp1.c_str(), exp1.length(), exp2.c_str(), exp2.length()) == 0) + { + isNull = true; + return ""; + } + + return parm[0]->data()->getStrVal(row, isNull); +} + +int32_t Func_nullif::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t exp1 = parm[0]->data()->getDateIntVal(row, isNull); - int64_t exp2 = 0; + int64_t exp1 = parm[0]->data()->getDateIntVal(row, isNull); + int64_t exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); + exp2 = parm[1]->data()->getIntVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - exp2 = parm[1]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - default: - { - isNull = true; - } + break; } - if ( exp1 == exp2 ) + case execplan::CalpontSystemCatalog::DATE: { - isNull = true; - return 0; + exp2 = parm[1]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; } - return exp1; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } -int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t exp1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - int64_t exp2 = 0; + int64_t exp1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + int64_t exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DATE: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DATE: - { - // Upgrade to datetime for proper comparison - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + // Upgrade to datetime for proper comparison + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - break; + break; + } + break; - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - - default: - { - isNull = true; - } + break; } - if ( exp1 == exp2 ) + default: { - isNull = true; - return 0; + isNull = true; } + } - return exp1; + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } -int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); - int64_t exp2 = 0; + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - - default: - { - isNull = false; - return exp1; - } + break; } - if ( exp1 == exp2 ) + default: { - isNull = true; - return 0; + isNull = false; + return exp1; } + } - return exp1; + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } -int64_t Func_nullif::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_nullif::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t exp1 = parm[0]->data()->getTimestampIntVal(row, isNull); - int64_t exp2 = 0; + int64_t exp1 = parm[0]->data()->getTimestampIntVal(row, isNull); + int64_t exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - - default: - { - isNull = false; - return exp1; - } + break; } - if ( exp1 == exp2 ) + default: { - isNull = true; - return 0; + isNull = false; + return exp1; } + } - return exp1; + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } - -double Func_nullif::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_nullif::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - double exp1 = parm[0]->data()->getDoubleVal(row, isNull); - double exp2 = 0; + double exp1 = parm[0]->data()->getDoubleVal(row, isNull); + double exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getDoubleVal(row, isNull); + exp2 = parm[1]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - exp2 = (double)parm[1]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - exp2 = parm[1]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - { - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - default: - { - isNull = true; - } + break; } - if ( exp1 == exp2 ) + case execplan::CalpontSystemCatalog::LONGDOUBLE: { - isNull = true; - return 0; + exp2 = (double)parm[1]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; } - return exp1; + case execplan::CalpontSystemCatalog::DATE: + { + exp2 = parm[1]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + break; + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } - -long double Func_nullif::getLongDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +long double Func_nullif::getLongDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - long double exp1 = parm[0]->data()->getLongDoubleVal(row, isNull); - long double exp2 = 0; + long double exp1 = parm[0]->data()->getLongDoubleVal(row, isNull); + long double exp2 = 0; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getDoubleVal(row, isNull); + exp2 = parm[1]->data()->getDoubleVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } + if (isNull) + { + isNull = false; + return exp1; + } - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - exp2 = parm[1]->data()->getLongDoubleVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - exp2 = parm[1]->data()->getDateIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - { - exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - - default: - { - isNull = true; - } + break; } - if ( exp1 == exp2 ) + case execplan::CalpontSystemCatalog::LONGDOUBLE: { - isNull = true; - return 0; + exp2 = parm[1]->data()->getLongDoubleVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; } - return exp1; + case execplan::CalpontSystemCatalog::DATE: + { + exp2 = parm[1]->data()->getDateIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + break; + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if (exp1 == exp2) + { + isNull = true; + return 0; + } + + return exp1; } - -execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal exp1 = parm[0]->data()->getDecimalVal(row, isNull); - IDB_Decimal exp2; + IDB_Decimal exp1 = parm[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal exp2; - switch (parm[1]->data()->resultType().colDataType) + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - exp2 = parm[1]->data()->getDecimalVal(row, isNull); + exp2 = parm[1]->data()->getDecimalVal(row, isNull); - if (isNull) - { - isNull = false; - return exp1; - } - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - int64_t value = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - exp2.value = value; - exp2.scale = 0; - } - /* fall through */ - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - double value = parm[1]->data()->getDoubleVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - exp2.value = (int64_t) value; - exp2.scale = 0; - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t s = 0; - int64_t x = 0; - - string value = DataConvert::dateToString1(parm[1]->data()->getDateIntVal(row, isNull)); - - if (!isNull) - { - x = atoll(value.c_str()); - - if ( s > 11 ) - s = 0; - - if ( s > 0 ) - { - x *= helpers::powerOf10_c[s]; - } - else if (s < 0) - { - s = -s; - - if ( s >= (int32_t) value.size() ) - { - x = 0; - } - else - { - x /= helpers::powerOf10_c[s]; - x *= helpers::powerOf10_c[s]; - } - - s = 0; - } - } - else - { - isNull = false; - return exp1; - } - - exp2.value = x; - exp2.scale = s; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int32_t s = 0; - int64_t x = 0; - string value; - - if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), timeZone()); - else - value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); - - if (!isNull) - { - //strip off micro seconds - value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); - - if ( s > 5 ) - s = 0; - - if ( s > 0 ) - { - x *= helpers::powerOf10_c[s]; - } - else if (s < 0) - { - s = -s; - - if ( s >= (int32_t) value.size() ) - { - x = 0; - } - else - { - x /= helpers::powerOf10_c[s]; - x *= helpers::powerOf10_c[s]; - } - - s = 0; - } - } - else - { - isNull = false; - return exp1; - } - - exp2.value = x; - exp2.scale = s; - } - break; - - default: - { - std::ostringstream oss; - oss << "truncate: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + if (isNull) + { + isNull = false; + return exp1; + } } + break; - if ( exp1 == exp2 ) + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - isNull = true; - IDB_Decimal decimal; - decimal.value = 0; - decimal.scale = 0; - return decimal; - } + int64_t value = parm[1]->data()->getIntVal(row, isNull); - return exp1; + if (isNull) + { + isNull = false; + return exp1; + } + + exp2.value = value; + exp2.scale = 0; + } + /* fall through */ + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + double value = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + exp2.value = (int64_t)value; + exp2.scale = 0; + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t s = 0; + int64_t x = 0; + + string value = DataConvert::dateToString1(parm[1]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + { + x = atoll(value.c_str()); + + if (s > 11) + s = 0; + + if (s > 0) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if (s >= (int32_t)value.size()) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + else + { + isNull = false; + return exp1; + } + + exp2.value = x; + exp2.scale = s; + } + break; + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int32_t s = 0; + int64_t x = 0; + string value; + + if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), timeZone()); + else + value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); + + if (!isNull) + { + // strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if (s > 5) + s = 0; + + if (s > 0) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if (s >= (int32_t)value.size()) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + else + { + isNull = false; + return exp1; + } + + exp2.value = x; + exp2.scale = s; + } + break; + + default: + { + std::ostringstream oss; + oss << "truncate: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + if (exp1 == exp2) + { + isNull = true; + IDB_Decimal decimal; + decimal.value = 0; + decimal.scale = 0; + return decimal; + } + + return exp1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_period_add.cpp b/utils/funcexp/func_period_add.cpp index a07b8ee23..23b964151 100644 --- a/utils/funcexp/func_period_add.cpp +++ b/utils/funcexp/func_period_add.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_period_add.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_period_add.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,66 +36,62 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_period_add::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_period_add::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_period_add::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_period_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t period = parm[0]->data()->getIntVal(row, isNull); + int64_t period = parm[0]->data()->getIntVal(row, isNull); - if ( period < 10000 ) + if (period < 10000) + { + // get first 2 digits of year + time_t now; + now = time(NULL); + struct tm tm; + localtime_r(&now, &tm); + char timestamp[10]; + strftime(timestamp, 10, "%Y", &tm); + string Syear = timestamp; + Syear = Syear.substr(0, 2); + int topyear = atoi(Syear.c_str()); + period = (topyear * 10000) + period; + } + + int64_t year = period / 100; + + int64_t month = period - (year * 100); + + int64_t months = parm[1]->data()->getIntVal(row, isNull); + + int64_t yearsAdd = months / 12; + + int64_t monthsAdd = months - (yearsAdd * 12); + + year = year + yearsAdd; + month = month + monthsAdd; + + if (month > 12) + { + year++; + month = month - 12; + } + else + { + if (month < 1) { - //get first 2 digits of year - time_t now; - now = time(NULL); - struct tm tm; - localtime_r(&now, &tm); - char timestamp[10]; - strftime (timestamp, 10, "%Y", &tm); - string Syear = timestamp; - Syear = Syear.substr(0, 2); - int topyear = atoi(Syear.c_str()); - period = (topyear * 10000) + period; + year--; + month = month + 12; } + } - int64_t year = period / 100; - - int64_t month = period - (year * 100); - - int64_t months = parm[1]->data()->getIntVal(row, isNull); - - int64_t yearsAdd = months / 12; - - int64_t monthsAdd = months - (yearsAdd * 12) ; - - year = year + yearsAdd; - month = month + monthsAdd; - - if ( month > 12 ) - { - year++; - month = month - 12; - } - else - { - if ( month < 1 ) - { - year--; - month = month + 12; - } - } - - - int64_t value = (year * 100) + month; - return value; + int64_t value = (year * 100) + month; + return value; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index dcf922494..470ab60c9 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_period_diff.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_period_diff.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -40,85 +40,77 @@ namespace funcexp inline uint64_t convert_period_to_month(uint64_t period) { - uint64_t a, b; + uint64_t a, b; - if (period == 0 || period > 999912) - return 0L; + if (period == 0 || period > 999912) + return 0L; - if ((a = period / 100) < YY_PART_YEAR) - a += 2000; - else if (a < 100) - a += 1900; + if ((a = period / 100) < YY_PART_YEAR) + a += 2000; + else if (a < 100) + a += 1900; - b = period % 100; - return a * 12 + b - 1; + b = period % 100; + return a * 12 + b - 1; } - -int64_t getArgSInt64Val(rowgroup::Row& row, TreeNode *exp, bool& isNull) +int64_t getArgSInt64Val(rowgroup::Row& row, TreeNode* exp, bool& isNull) { - switch (exp->resultType().colDataType) + switch (exp->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: return exp->getIntVal(row, isNull); + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - return exp->getIntVal(row, isNull); - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal d = exp->getDecimalVal(row, isNull); - return d.toSInt64Round(); - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - return atoi(exp->getStrVal(row, isNull).c_str()); - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - { - datatypes::TDouble d(exp->getDoubleVal(row, isNull)); - return d.toMCSSInt64Round(); - } - - default: - isNull = true; + IDB_Decimal d = exp->getDecimalVal(row, isNull); + return d.toSInt64Round(); } - return 0; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: return atoi(exp->getStrVal(row, isNull).c_str()); + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + { + datatypes::TDouble d(exp->getDoubleVal(row, isNull)); + return d.toMCSSInt64Round(); + } + + default: isNull = true; + } + return 0; } - -CalpontSystemCatalog::ColType Func_period_diff::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_period_diff::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_period_diff::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_period_diff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t period1 = (uint64_t) getArgSInt64Val(row, parm[0]->data(), isNull); + uint64_t period1 = (uint64_t)getArgSInt64Val(row, parm[0]->data(), isNull); - if (isNull) - return 0; + if (isNull) + return 0; - uint64_t period2 = (uint64_t) getArgSInt64Val(row, parm[1]->data(), isNull); + uint64_t period2 = (uint64_t)getArgSInt64Val(row, parm[1]->data(), isNull); - if (isNull) - return 0; + if (isNull) + return 0; - return convert_period_to_month(period1) - convert_period_to_month(period2); + return convert_period_to_month(period1) - convert_period_to_month(period2); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_pow.cpp b/utils/funcexp/func_pow.cpp index 4e07c8410..734b50a22 100644 --- a/utils/funcexp/func_pow.cpp +++ b/utils/funcexp/func_pow.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_pow.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_pow.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -45,89 +45,80 @@ using namespace logging; namespace funcexp { - -CalpontSystemCatalog::ColType Func_pow::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_pow::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -double Func_pow::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +double Func_pow::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - // null value is indicated by isNull - double base = parm[0]->data()->getDoubleVal(row, isNull); + // null value is indicated by isNull + double base = parm[0]->data()->getDoubleVal(row, isNull); + + if (!isNull) + { + double exponent = parm[1]->data()->getDoubleVal(row, isNull); if (!isNull) { - double exponent = parm[1]->data()->getDoubleVal(row, isNull); + errno = 0; + double x = pow(base, exponent); - if (!isNull) - { - errno = 0; - double x = pow(base, exponent); + // @bug3490, 4461, rule out domain error, pole error and overflow range error. + if (!isfinite(x)) + { + isNull = true; + Message::Args args; + args.add("pow"); + args.add(base); + args.add(exponent); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - // @bug3490, 4461, rule out domain error, pole error and overflow range error. - if (!isfinite(x)) - { - isNull = true; - Message::Args args; - args.add("pow"); - args.add(base); - args.add(exponent); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - - return x; - } + return x; } + } - return 0.0; + return 0.0; } -long double Func_pow::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) +long double Func_pow::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType&) { - // null value is indicated by isNull - long double base = parm[0]->data()->getLongDoubleVal(row, isNull); + // null value is indicated by isNull + long double base = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (!isNull) + { + // Should this be long double? Not sure on usage. + double exponent = parm[1]->data()->getDoubleVal(row, isNull); if (!isNull) { - // Should this be long double? Not sure on usage. - double exponent = parm[1]->data()->getDoubleVal(row, isNull); + errno = 0; + long double x = powl(base, (long double)exponent); - if (!isNull) - { - errno = 0; - long double x = powl(base, (long double)exponent); + // @bug3490, 4461, rule out domain error, pole error and overflow range error. + if (!isfinite(x)) + { + isNull = true; + Message::Args args; + args.add("pow"); + args.add((double)base); + args.add(exponent); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - // @bug3490, 4461, rule out domain error, pole error and overflow range error. - if (!isfinite(x)) - { - isNull = true; - Message::Args args; - args.add("pow"); - args.add((double)base); - args.add(exponent); - unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } - - - return x; - } + return x; } + } - return 0.0; + return 0.0; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index 35b9d93f4..c4417497f 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_quarter.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_quarter.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,112 +36,107 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_quarter::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_quarter::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_quarter::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_quarter::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - // try to cast to date/datetime - int64_t val = 0, month = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + // try to cast to date/datetime + int64_t val = 0, month = 0; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + month = (val >> 12) & 0xf; + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + month = (val >> 44) & 0xf; + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - month = (val >> 12) & 0xf; - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - month = (val >> 44) & 0xf; - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - month = m_time.month; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - month = (uint32_t)((val >> 44) & 0xf); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - else - month = (val >> 44) & 0xf; - - break; - } - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - else - month = (val >> 44) & 0xf; - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDate(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - else - month = (val >> 12) & 0xf; - - break; - } - else - { - isNull = true; - break; - } - } - - default: - isNull = true; + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + month = m_time.month; + break; } - return (int64_t)((month + 2) / 3); + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + month = (uint32_t)((val >> 44) & 0xf); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + month = (val >> 44) & 0xf; + + break; + } + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + month = (val >> 44) & 0xf; + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDate(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + month = (val >> 12) & 0xf; + + break; + } + else + { + isNull = true; + break; + } + } + + default: isNull = true; + } + + return (int64_t)((month + 2) / 3); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_quote.cpp b/utils/funcexp/func_quote.cpp index 2bf82870f..05ce4cecf 100644 --- a/utils/funcexp/func_quote.cpp +++ b/utils/funcexp/func_quote.cpp @@ -26,63 +26,58 @@ using namespace rowgroup; namespace funcexp { - -CalpontSystemCatalog::ColType Func_quote::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_quote::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -std::string Func_quote::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_quote::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - string str; + string str; - stringValue(fp[0], row, isNull, str); + stringValue(fp[0], row, isNull, str); - if (isNull) + if (isNull) + { + isNull = false; + return "NULL"; + } + + if (str.empty()) + return "NULL"; + + string result; + result.reserve((str.size() * 1.3) + 2); + + result.push_back('\''); + + for (uint64_t i = 0; i < str.size(); i++) + { + switch (str[i]) { - isNull = false; - return "NULL"; + case 0: + result.push_back('\\'); + result.push_back('0'); + break; + case '\032': + result.push_back('\\'); + result.push_back('Z'); + break; + case '\'': + case '\\': + result.push_back('\\'); + result.push_back(str[i]); + break; + default: result.push_back(str[i]); break; } + } - if (str.empty()) - return "NULL"; + result.push_back('\''); - string result; - result.reserve((str.size() * 1.3) + 2); - - result.push_back('\''); - - for (uint64_t i = 0; i < str.size(); i++) - { - switch(str[i]) - { - case 0: - result.push_back('\\'); - result.push_back('0'); - break; - case '\032': - result.push_back('\\'); - result.push_back('Z'); - break; - case '\'': - case '\\': - result.push_back('\\'); - result.push_back(str[i]); - break; - default: - result.push_back(str[i]); - break; - } - } - - result.push_back('\''); - - return result; + return result; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_rand.cpp b/utils/funcexp/func_rand.cpp index 3c2f2a7f8..223949bbf 100644 --- a/utils/funcexp/func_rand.cpp +++ b/utils/funcexp/func_rand.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_rand.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_rand.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,7 +35,6 @@ using namespace execplan; #include "dataconvert.h" - namespace funcexp { uint64_t maxValue = 0x3FFFFFFFL; @@ -45,61 +44,58 @@ uint64_t maxValue = 0x3FFFFFFFL; */ double Func_rand::getRand() { - uint64_t fSeed1_save = fSeed1; - fSeed1 = (fSeed1 * 3 + fSeed2) % maxValue; + uint64_t fSeed1_save = fSeed1; + fSeed1 = (fSeed1 * 3 + fSeed2) % maxValue; - // prevent the seed to repeat itself. e.g. seed1 = 1073741790; seed2 = 66; - if (fSeed1_save == fSeed1) - fSeed1 += 23; + // prevent the seed to repeat itself. e.g. seed1 = 1073741790; seed2 = 66; + if (fSeed1_save == fSeed1) + fSeed1 += 23; - fSeed2 = (fSeed1 + fSeed2 + 33) % maxValue; + fSeed2 = (fSeed1 + fSeed2 + 33) % maxValue; - return (((double) fSeed1) / (double)maxValue); + return (((double)fSeed1) / (double)maxValue); } -CalpontSystemCatalog::ColType Func_rand::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_rand::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -double Func_rand::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_rand::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - // NOTE: this function needs to use 32bit ints otherwise it will break for negative values - uint32_t seedParm = 0; + // NOTE: this function needs to use 32bit ints otherwise it will break for negative values + uint32_t seedParm = 0; - // rand with parameter. if the parm is constanct, then a column is attached for fetching - if (parm.size() == 1 || parm.size() == 2) + // rand with parameter. if the parm is constanct, then a column is attached for fetching + if (parm.size() == 1 || parm.size() == 2) + { + execplan::ConstantColumn* cc = dynamic_cast(parm[0].get()->data()); + + if (!fSeedSet || !cc) { - execplan::ConstantColumn* cc = dynamic_cast(parm[0].get()->data()); - - if (!fSeedSet || !cc) - { - /* Copied from item_func.cpp */ - seedParm = parm[0]->data()->getIntVal(row, isNull); - fSeed1 = (uint32_t)(seedParm * 0x10001L + 55555555L); - fSeed2 = (uint32_t)(seedParm * 0x10000001L); - fSeedSet = true; - } + /* Copied from item_func.cpp */ + seedParm = parm[0]->data()->getIntVal(row, isNull); + fSeed1 = (uint32_t)(seedParm * 0x10001L + 55555555L); + fSeed2 = (uint32_t)(seedParm * 0x10000001L); + fSeedSet = true; } - // rand without parameter. thd->rand are passed in. The 3rd is a simple column for fetching - else + } + // rand without parameter. thd->rand are passed in. The 3rd is a simple column for fetching + else + { + idbassert(parm.size() == 3); + if (fSeedSet) { - idbassert(parm.size() == 3); - if (fSeedSet) - { - fSeed1 = parm[0]->data()->getIntVal(row, isNull); - fSeed2 = parm[1]->data()->getIntVal(row, isNull); - fSeedSet = true; - } + fSeed1 = parm[0]->data()->getIntVal(row, isNull); + fSeed2 = parm[1]->data()->getIntVal(row, isNull); + fSeedSet = true; } + } - return getRand(); + return getRand(); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 0fa61b0a0..f918aa7b4 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_regexp.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_regexp.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -47,213 +47,205 @@ using namespace logging; namespace { -inline bool getBool(rowgroup::Row& row, - funcexp::FunctionParm& pm, - bool& isNull, - CalpontSystemCatalog::ColType& ct, - const string& timeZone) +inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, + CalpontSystemCatalog::ColType& ct, const string& timeZone) { + string expr; + string pattern; - string expr; - string pattern; - - switch (pm[0]->data()->resultType().colDataType) + switch (pm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - expr = pm[0]->data()->getStrVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - expr = dataconvert::DataConvert::dateToString(pm[0]->data()->getDateIntVal(row, isNull)); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - expr = dataconvert::DataConvert::datetimeToString(pm[0]->data()->getDatetimeIntVal(row, isNull)); - //strip off micro seconds - expr = expr.substr(0, 19); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - expr = dataconvert::DataConvert::timestampToString(pm[0]->data()->getTimestampIntVal(row, isNull), timeZone); - //strip off micro seconds - expr = expr.substr(0, 19); - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); - //strip off micro seconds - expr = expr.substr(0, 19); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal d = pm[0]->data()->getDecimalVal(row, isNull); - - if (pm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - expr = d.toString(true); - } - else - { - expr = d.toString(); - } - - break; - } - - default: - { - std::ostringstream oss; - oss << "regexp: datatype of " << execplan::colDataTypeToString(ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + expr = pm[0]->data()->getStrVal(row, isNull); + break; } - switch (pm[1]->data()->resultType().colDataType) + case execplan::CalpontSystemCatalog::DATE: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - pattern = pm[1]->data()->getStrVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - pattern = dataconvert::DataConvert::dateToString(pm[1]->data()->getDateIntVal(row, isNull)); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - pattern = dataconvert::DataConvert::datetimeToString(pm[1]->data()->getDatetimeIntVal(row, isNull)); - //strip off micro seconds - pattern = pattern.substr(0, 19); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - pattern = dataconvert::DataConvert::timestampToString(pm[1]->data()->getTimestampIntVal(row, isNull), timeZone); - //strip off micro seconds - pattern = pattern.substr(0, 19); - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); - //strip off micro seconds - pattern = pattern.substr(0, 19); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal d = pm[1]->data()->getDecimalVal(row, isNull); - - if (pm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - pattern = d.toString(true); - } - else - { - pattern = d.toString(); - } - break; - } - - default: - { - std::ostringstream oss; - oss << "regexp: datatype of " << execplan::colDataTypeToString(ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + expr = dataconvert::DataConvert::dateToString(pm[0]->data()->getDateIntVal(row, isNull)); + break; } + case execplan::CalpontSystemCatalog::DATETIME: + { + expr = dataconvert::DataConvert::datetimeToString(pm[0]->data()->getDatetimeIntVal(row, isNull)); + // strip off micro seconds + expr = expr.substr(0, 19); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + expr = dataconvert::DataConvert::timestampToString(pm[0]->data()->getTimestampIntVal(row, isNull), + timeZone); + // strip off micro seconds + expr = expr.substr(0, 19); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); + // strip off micro seconds + expr = expr.substr(0, 19); + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal d = pm[0]->data()->getDecimalVal(row, isNull); + + if (pm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + expr = d.toString(true); + } + else + { + expr = d.toString(); + } + + break; + } + + default: + { + std::ostringstream oss; + oss << "regexp: datatype of " << execplan::colDataTypeToString(ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + switch (pm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + pattern = pm[1]->data()->getStrVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + pattern = dataconvert::DataConvert::dateToString(pm[1]->data()->getDateIntVal(row, isNull)); + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + pattern = dataconvert::DataConvert::datetimeToString(pm[1]->data()->getDatetimeIntVal(row, isNull)); + // strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + pattern = dataconvert::DataConvert::timestampToString(pm[1]->data()->getTimestampIntVal(row, isNull), + timeZone); + // strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); + // strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal d = pm[1]->data()->getDecimalVal(row, isNull); + + if (pm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + pattern = d.toString(true); + } + else + { + pattern = d.toString(); + } + break; + } + + default: + { + std::ostringstream oss; + oss << "regexp: datatype of " << execplan::colDataTypeToString(ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } #ifdef __linux__ - regex_t re; + regex_t re; - regcomp(&re, pattern.c_str(), REG_EXTENDED | REG_NOSUB ); + regcomp(&re, pattern.c_str(), REG_EXTENDED | REG_NOSUB); - int res = regexec(&re, expr.c_str(), 0, NULL, 0); - regfree(&re); + int res = regexec(&re, expr.c_str(), 0, NULL, 0); + regfree(&re); - if (res == 0) - return true; - else - return false; + if (res == 0) + return true; + else + return false; #else - regex pat(pattern.c_str()); - return regex_search(expr.c_str(), pat); + regex pat(pattern.c_str()); + return regex_search(expr.c_str(), pat); #endif - - } -} +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_regexp::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_regexp::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -bool Func_regexp::getBoolVal(rowgroup::Row& row, - FunctionParm& pm, - bool& isNull, +bool Func_regexp::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, timeZone()) && !isNull; + return getBool(row, pm, isNull, ct, timeZone()) && !isNull; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_repeat.cpp b/utils/funcexp/func_repeat.cpp index 98c3dbd3d..6b0e6ace0 100644 --- a/utils/funcexp/func_repeat.cpp +++ b/utils/funcexp/func_repeat.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_repeat.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_repeat.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -38,60 +38,56 @@ using namespace joblist; namespace funcexp { - -CalpontSystemCatalog::ColType Func_repeat::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_repeat::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - //return fp[0]->data()->resultType(); - return resultType; + // operation type is not used by this functor + // return fp[0]->data()->resultType(); + return resultType; } -std::string Func_repeat::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_repeat::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - string str; + string str; - stringValue(fp[0], row, isNull, str); + stringValue(fp[0], row, isNull, str); - if (str.empty() || str == "") - return ""; + if (str.empty() || str == "") + return ""; - int count = fp[1]->data()->getIntVal(row, isNull); + int count = fp[1]->data()->getIntVal(row, isNull); - if (isNull) - return ""; + if (isNull) + return ""; - if ( count < 1 ) - return ""; + if (count < 1) + return ""; - //calculate size of buffer to allocate + // calculate size of buffer to allocate - int size = str.length() * count; + int size = str.length() * count; - //allocate memory - char* result = new char[size + 1]; + // allocate memory + char* result = new char[size + 1]; - if (result == NULL) - { - return ""; - } + if (result == NULL) + { + return ""; + } - memset( (char*) result, 0, size); + memset((char*)result, 0, size); - for ( int i = 0 ; i < count ; i ++ ) - { - if (strcat(result, str.c_str()) == NULL) //questionable check - return ""; - } + for (int i = 0; i < count; i++) + { + if (strcat(result, str.c_str()) == NULL) // questionable check + return ""; + } - std::string res(result); - delete [] result; - return res; + std::string res(result); + delete[] result; + return res; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_replace.cpp b/utils/funcexp/func_replace.cpp index 20299f027..05e0fe359 100644 --- a/utils/funcexp/func_replace.cpp +++ b/utils/funcexp/func_replace.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_replace.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_replace.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,150 +34,143 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_replace::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_replace::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_replace::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_replace::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - CHARSET_INFO* cs = ct.getCharset(); + CHARSET_INFO* cs = ct.getCharset(); - const string& str = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - size_t strLen = str.length(); - - const string& fromstr = fp[1]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (fromstr.length() == 0) - return str; - size_t fromLen = fromstr.length(); - - const string& tostr = fp[2]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - size_t toLen = tostr.length(); + const string& str = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + size_t strLen = str.length(); - bool binaryCmp = (cs->state & MY_CS_BINSORT) || !cs->use_mb(); - string newstr; - size_t pos = 0; - if (binaryCmp) + const string& fromstr = fp[1]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (fromstr.length() == 0) + return str; + size_t fromLen = fromstr.length(); + + const string& tostr = fp[2]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + size_t toLen = tostr.length(); + + bool binaryCmp = (cs->state & MY_CS_BINSORT) || !cs->use_mb(); + string newstr; + size_t pos = 0; + if (binaryCmp) + { + // Count the number of fromstr in strend so we can reserve buffer space. + int count = 0; + do { - // Count the number of fromstr in strend so we can reserve buffer space. - int count = 0; - do - { - ++count; - pos = str.find(fromstr, pos + fromLen); - } - while (pos != string::npos); - - newstr.reserve(strLen + (count * ((int)toLen - (int)fromLen)) + 1); - - uint32_t i = 0; - pos = str.find(fromstr); - if (pos == string::npos) - return str; - // Move the stuff into newstr - do - { - if (pos > i) - newstr = newstr + str.substr(i, pos - i); + ++count; + pos = str.find(fromstr, pos + fromLen); + } while (pos != string::npos); - newstr = newstr + tostr; - i = pos + fromLen; - pos = str.find(fromstr, i); - } - while (pos != string::npos); - - newstr = newstr + str.substr(i, string::npos); - } - else + newstr.reserve(strLen + (count * ((int)toLen - (int)fromLen)) + 1); + + uint32_t i = 0; + pos = str.find(fromstr); + if (pos == string::npos) + return str; + // Move the stuff into newstr + do { - // UTF - const char* src = str.c_str(); - const char* srcEnd = src + strLen; - const char* srchEnd = srcEnd - fromLen + 1; - const char* from = fromstr.c_str(); - const char* fromEnd = from + fromLen; - const char* to = tostr.c_str(); - const char* ptr = src; - char *i,*j; - size_t count = 10; // Some arbitray number to reserve some space to start. - int growlen = (int)toLen - (int)fromLen; - growlen = growlen < 1 ? 1 : growlen; - growlen *= count; - newstr.reserve(strLen + (count * growlen) + 1); - size_t maxsize = newstr.capacity(); - uint32_t l; + if (pos > i) + newstr = newstr + str.substr(i, pos - i); - // We don't know where byte patterns might match so - // we start at the beginning of the string and move forward - // one character at a time until we find a match. Then we can - // move the src bytes and add in the to bytes,then try again. - while (ptr < srchEnd) + newstr = newstr + tostr; + i = pos + fromLen; + pos = str.find(fromstr, i); + } while (pos != string::npos); + + newstr = newstr + str.substr(i, string::npos); + } + else + { + // UTF + const char* src = str.c_str(); + const char* srcEnd = src + strLen; + const char* srchEnd = srcEnd - fromLen + 1; + const char* from = fromstr.c_str(); + const char* fromEnd = from + fromLen; + const char* to = tostr.c_str(); + const char* ptr = src; + char *i, *j; + size_t count = 10; // Some arbitray number to reserve some space to start. + int growlen = (int)toLen - (int)fromLen; + growlen = growlen < 1 ? 1 : growlen; + growlen *= count; + newstr.reserve(strLen + (count * growlen) + 1); + size_t maxsize = newstr.capacity(); + uint32_t l; + + // We don't know where byte patterns might match so + // we start at the beginning of the string and move forward + // one character at a time until we find a match. Then we can + // move the src bytes and add in the to bytes,then try again. + while (ptr < srchEnd) + { + bool found = false; + if (*ptr == *from) // If the first byte matches, maybe we have a match + { + // Do a byte by byte compare of src at that spot against from + i = const_cast(ptr) + 1; + j = const_cast(from) + 1; + found = true; + while (j != fromEnd) { - bool found = false; - if (*ptr == *from) // If the first byte matches, maybe we have a match - { - // Do a byte by byte compare of src at that spot against from - i = const_cast(ptr) + 1; - j = const_cast(from) + 1; - found = true; - while (j != fromEnd) - { - if (*i++ != *j++) - { - found = false; - break; - } - } - } - if (found) - { - if (ptr < i) - { - int mvsize = ptr - src; - if (newstr.length() + mvsize + toLen > maxsize) - { - // We need a re-alloc - newstr.reserve(maxsize + growlen); - maxsize = newstr.capacity(); - growlen *= 2; - } - newstr.append(src, ptr - src); - src += mvsize + fromLen; - ptr = src; - } - newstr.append(to, toLen); - } - else - { - // move to the next character - if ((l = my_ismbchar(cs, ptr, srcEnd))) // returns the number of bytes in the leading char or zero if one byte - ptr += l; - else - ++ptr; - } + if (*i++ != *j++) + { + found = false; + break; + } } - // Copy in the trailing src chars. - newstr.append(src, srcEnd - src); + } + if (found) + { + if (ptr < i) + { + int mvsize = ptr - src; + if (newstr.length() + mvsize + toLen > maxsize) + { + // We need a re-alloc + newstr.reserve(maxsize + growlen); + maxsize = newstr.capacity(); + growlen *= 2; + } + newstr.append(src, ptr - src); + src += mvsize + fromLen; + ptr = src; + } + newstr.append(to, toLen); + } + else + { + // move to the next character + if ((l = my_ismbchar(cs, ptr, + srcEnd))) // returns the number of bytes in the leading char or zero if one byte + ptr += l; + else + ++ptr; + } } - return newstr; + // Copy in the trailing src chars. + newstr.append(src, srcEnd - src); + } + return newstr; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_replace_oracle.cpp b/utils/funcexp/func_replace_oracle.cpp index 79c09e12d..b8eb7f6ef 100644 --- a/utils/funcexp/func_replace_oracle.cpp +++ b/utils/funcexp/func_replace_oracle.cpp @@ -28,150 +28,143 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_replace_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_replace_oracle::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_replace_oracle::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +std::string Func_replace_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - CHARSET_INFO* cs = ct.getCharset(); + CHARSET_INFO* cs = ct.getCharset(); - const string& str = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - size_t strLen = str.length(); - - const string& fromstr = fp[1]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (fromstr.length() == 0) - return str; - size_t fromLen = fromstr.length(); - - const string& tostr = fp[2]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - size_t toLen = tostr.length(); + const string& str = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + size_t strLen = str.length(); - bool binaryCmp = (cs->state & MY_CS_BINSORT) || !cs->use_mb(); - string newstr; - size_t pos = 0; - if (binaryCmp) + const string& fromstr = fp[1]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (fromstr.length() == 0) + return str; + size_t fromLen = fromstr.length(); + + const string& tostr = fp[2]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + size_t toLen = tostr.length(); + + bool binaryCmp = (cs->state & MY_CS_BINSORT) || !cs->use_mb(); + string newstr; + size_t pos = 0; + if (binaryCmp) + { + // Count the number of fromstr in strend so we can reserve buffer space. + int count = 0; + do { - // Count the number of fromstr in strend so we can reserve buffer space. - int count = 0; - do - { - ++count; - pos = str.find(fromstr, pos + fromLen); - } - while (pos != string::npos); - - newstr.reserve(strLen + (count * ((int)toLen - (int)fromLen)) + 1); - - uint32_t i = 0; - pos = str.find(fromstr); - if (pos == string::npos) - return str; - // Move the stuff into newstr - do - { - if (pos > i) - newstr = newstr + str.substr(i, pos - i); + ++count; + pos = str.find(fromstr, pos + fromLen); + } while (pos != string::npos); - newstr = newstr + tostr; - i = pos + fromLen; - pos = str.find(fromstr, i); - } - while (pos != string::npos); - - newstr = newstr + str.substr(i, string::npos); - } - else + newstr.reserve(strLen + (count * ((int)toLen - (int)fromLen)) + 1); + + uint32_t i = 0; + pos = str.find(fromstr); + if (pos == string::npos) + return str; + // Move the stuff into newstr + do { - // UTF - const char* src = str.c_str(); - const char* srcEnd = src + strLen; - const char* srchEnd = srcEnd - fromLen + 1; - const char* from = fromstr.c_str(); - const char* fromEnd = from + fromLen; - const char* to = tostr.c_str(); - const char* ptr = src; - char *i,*j; - size_t count = 10; // Some arbitray number to reserve some space to start. - int growlen = (int)toLen - (int)fromLen; - growlen = growlen < 1 ? 1 : growlen; - growlen *= count; - newstr.reserve(strLen + (count * growlen) + 1); - size_t maxsize = newstr.capacity(); - uint32_t l; + if (pos > i) + newstr = newstr + str.substr(i, pos - i); - // We don't know where byte patterns might match so - // we start at the beginning of the string and move forward - // one character at a time until we find a match. Then we can - // move the src bytes and add in the to bytes,then try again. - while (ptr < srchEnd) + newstr = newstr + tostr; + i = pos + fromLen; + pos = str.find(fromstr, i); + } while (pos != string::npos); + + newstr = newstr + str.substr(i, string::npos); + } + else + { + // UTF + const char* src = str.c_str(); + const char* srcEnd = src + strLen; + const char* srchEnd = srcEnd - fromLen + 1; + const char* from = fromstr.c_str(); + const char* fromEnd = from + fromLen; + const char* to = tostr.c_str(); + const char* ptr = src; + char *i, *j; + size_t count = 10; // Some arbitray number to reserve some space to start. + int growlen = (int)toLen - (int)fromLen; + growlen = growlen < 1 ? 1 : growlen; + growlen *= count; + newstr.reserve(strLen + (count * growlen) + 1); + size_t maxsize = newstr.capacity(); + uint32_t l; + + // We don't know where byte patterns might match so + // we start at the beginning of the string and move forward + // one character at a time until we find a match. Then we can + // move the src bytes and add in the to bytes,then try again. + while (ptr < srchEnd) + { + bool found = false; + if (*ptr == *from) // If the first byte matches, maybe we have a match + { + // Do a byte by byte compare of src at that spot against from + i = const_cast(ptr) + 1; + j = const_cast(from) + 1; + found = true; + while (j != fromEnd) { - bool found = false; - if (*ptr == *from) // If the first byte matches, maybe we have a match - { - // Do a byte by byte compare of src at that spot against from - i = const_cast(ptr) + 1; - j = const_cast(from) + 1; - found = true; - while (j != fromEnd) - { - if (*i++ != *j++) - { - found = false; - break; - } - } - } - if (found) - { - if (ptr < i) - { - int mvsize = ptr - src; - if (newstr.length() + mvsize + toLen > maxsize) - { - // We need a re-alloc - newstr.reserve(maxsize + growlen); - maxsize = newstr.capacity(); - growlen *= 2; - } - newstr.append(src, ptr - src); - src += mvsize + fromLen; - ptr = src; - } - newstr.append(to, toLen); - } - else - { - // move to the next character - if ((l = my_ismbchar(cs, ptr, srcEnd))) // returns the number of bytes in the leading char or zero if one byte - ptr += l; - else - ++ptr; - } + if (*i++ != *j++) + { + found = false; + break; + } } - // Copy in the trailing src chars. - newstr.append(src, srcEnd - src); + } + if (found) + { + if (ptr < i) + { + int mvsize = ptr - src; + if (newstr.length() + mvsize + toLen > maxsize) + { + // We need a re-alloc + newstr.reserve(maxsize + growlen); + maxsize = newstr.capacity(); + growlen *= 2; + } + newstr.append(src, ptr - src); + src += mvsize + fromLen; + ptr = src; + } + newstr.append(to, toLen); + } + else + { + // move to the next character + if ((l = my_ismbchar(cs, ptr, + srcEnd))) // returns the number of bytes in the leading char or zero if one byte + ptr += l; + else + ++ptr; + } } - return newstr; + // Copy in the trailing src chars. + newstr.append(src, srcEnd - src); + } + return newstr; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_reverse.cpp b/utils/funcexp/func_reverse.cpp index 9749c0a83..2b383ee7f 100644 --- a/utils/funcexp/func_reverse.cpp +++ b/utils/funcexp/func_reverse.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_reverse.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_reverse.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,68 +34,63 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_reverse::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_reverse::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -std::string Func_reverse::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_reverse::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); + CHARSET_INFO* cs = type.getCharset(); - string str; - stringValue(fp[0], row, isNull, str); - if (isNull) - return ""; - if (str.empty() || str.length() == 0) - return str; - // binLen represents the number of bytes in str - size_t binLen = str.length(); - const char* pos = str.c_str(); - const char* end = pos + binLen; + string str; + stringValue(fp[0], row, isNull, str); + if (isNull) + return ""; + if (str.empty() || str.length() == 0) + return str; + // binLen represents the number of bytes in str + size_t binLen = str.length(); + const char* pos = str.c_str(); + const char* end = pos + binLen; - char* pbuf = new char[binLen + 1]; - pbuf[binLen] = 0; - char* tmp = pbuf + binLen; - - if (cs->use_mb()) // uses multi-byte characters + char* pbuf = new char[binLen + 1]; + pbuf[binLen] = 0; + char* tmp = pbuf + binLen; + + if (cs->use_mb()) // uses multi-byte characters + { + uint32 l; + while (pos < end) { - uint32 l; - while (pos < end) - { - if ((l = my_ismbchar(cs, pos, end))) - { - tmp -= l; - idbassert(tmp >= pbuf); - memcpy(tmp, pos, l); - pos += l; - } - else - { - *--tmp= *pos++; - } - } - } - else - { - while (pos < end) - *--tmp= *pos++; + if ((l = my_ismbchar(cs, pos, end))) + { + tmp -= l; + idbassert(tmp >= pbuf); + memcpy(tmp, pos, l); + pos += l; + } + else + { + *--tmp = *pos++; + } } + } + else + { + while (pos < end) + *--tmp = *pos++; + } - string rstr = pbuf; - delete [] pbuf; - return rstr; + string rstr = pbuf; + delete[] pbuf; + return rstr; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_right.cpp b/utils/funcexp/func_right.cpp index 6c96882ce..4615256e5 100644 --- a/utils/funcexp/func_right.cpp +++ b/utils/funcexp/func_right.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_right.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_right.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,47 +34,43 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_right::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_right::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_right::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_right::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; - size_t trimLength = fp[1]->data()->getUintVal(row, isNull); - if (isNull || trimLength <= 0) - return ""; + size_t trimLength = fp[1]->data()->getUintVal(row, isNull); + if (isNull || trimLength <= 0) + return ""; - size_t start = cs->numchars(pos, end); // Here, start is number of characters in src - if (start <= trimLength) - return src; - start = cs->charpos(pos, end, start - trimLength); // Here, start becomes number of bytes into src to start copying + size_t start = cs->numchars(pos, end); // Here, start is number of characters in src + if (start <= trimLength) + return src; + start = cs->charpos(pos, end, + start - trimLength); // Here, start becomes number of bytes into src to start copying - std::string ret(pos+start, binLen-start); - return ret; + std::string ret(pos + start, binLen - start); + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index bb4131b18..293159570 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_round.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_round.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -43,694 +43,671 @@ using namespace logging; namespace { - using namespace funcexp; // P should either be double or long double template inline void decimalPlaceDouble(FunctionParm& fp, int64_t& s, P& p, Row& row, bool& isNull) { - s = fp[1]->data()->getIntVal(row, isNull); - int64_t d = s; + s = fp[1]->data()->getIntVal(row, isNull); + int64_t d = s; - if (isNull) - return; + if (isNull) + return; - int64_t i = (d >= 0) ? d : (-d); - int64_t r = 1; + int64_t i = (d >= 0) ? d : (-d); + int64_t r = 1; - while (i--) - r *= 10; + while (i--) + r *= 10; - if (d >= 0) - p = (P) r; - else - p = 1.0 / ((P) r); + if (d >= 0) + p = (P)r; + else + p = 1.0 / ((P)r); } -} +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_round::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_round::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - if (resultType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) + if (resultType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) + { + CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType(); + + switch (ct.colDataType) { - CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType(); + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (resultType.scale > ct.scale) + (resultType).scale = ct.scale; - switch (ct.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (resultType.scale > ct.scale) - (resultType).scale = ct.scale; + break; + } - break; - } - - default: - { - break; - } - } + default: + { + break; + } } + } - return fp[0]->data()->resultType(); + return fp[0]->data()->resultType(); } - // round(X), round(X, D) // -int64_t Func_round::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_round::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (!op_ct.isWideDecimalType()) + if (!op_ct.isWideDecimalType()) + { + if (x.scale > 0) { - if (x.scale > 0) - { - while (x.scale-- > 0) - x.value /= 10; - } - else - { - while (x.scale++ < 0) - x.value *= 10; - } - - return x.value; + while (x.scale-- > 0) + x.value /= 10; } else { - return static_cast(x.getIntegralPart()); + while (x.scale++ < 0) + x.value *= 10; } + + return x.value; + } + else + { + return static_cast(x.getIntegralPart()); + } } - -uint64_t Func_round::getUintVal(Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_round::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t x; - if (UNLIKELY(op_ct.colDataType == execplan::CalpontSystemCatalog::DATE)) + uint64_t x; + if (UNLIKELY(op_ct.colDataType == execplan::CalpontSystemCatalog::DATE)) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + x = static_cast(d.value); + } + else + { + x = parm[0]->data()->getUintVal(row, isNull); + } + + return x; +} + +double Func_round::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + if (execplan::CalpontSystemCatalog::DOUBLE == op_ct.colDataType || + execplan::CalpontSystemCatalog::FLOAT == op_ct.colDataType) + { + int64_t d = 0; + double p = 1; + + if (parm.size() > 1) // round(X, D) + decimalPlaceDouble(parm, d, p, row, isNull); + + if (isNull) + return 0.0; + + double x = parm[0]->data()->getDoubleVal(row, isNull); + + if (!isNull) { - IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); - x = static_cast(d.value); - } - else - { - x = parm[0]->data()->getUintVal(row, isNull); + x *= p; + + if (x >= 0) + x = floor(x + 0.5); + else + x = ceil(x - 0.5); + + if (p != 0.0) + x /= p; + else + x = 0.0; } return x; + } + + if (isUnsigned(op_ct.colDataType)) + { + return getUintVal(row, parm, isNull, op_ct); + } + + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + + if (isNull) + return 0.0; + + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = static_cast(x.toTSInt128()); + + if (x.scale > 0) + { + while (x.scale-- > 0) + d /= 10.0; + } + else + { + while (x.scale++ < 0) + d *= 10.0; + } + + return d; } - -double Func_round::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +long double Func_round::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - if (execplan::CalpontSystemCatalog::DOUBLE == op_ct.colDataType || - execplan::CalpontSystemCatalog::FLOAT == op_ct.colDataType) - { - int64_t d = 0; - double p = 1; + if (execplan::CalpontSystemCatalog::LONGDOUBLE == op_ct.colDataType || + execplan::CalpontSystemCatalog::DOUBLE == op_ct.colDataType || + execplan::CalpontSystemCatalog::FLOAT == op_ct.colDataType) + { + int64_t d = 0; + long double p = 1; - if (parm.size() > 1) // round(X, D) - decimalPlaceDouble(parm, d, p, row, isNull); - - if (isNull) - return 0.0; - - double x = parm[0]->data()->getDoubleVal(row, isNull); - - if (!isNull) - { - x *= p; - - if (x >= 0) - x = floor(x + 0.5); - else - x = ceil(x - 0.5); - - if (p != 0.0) - x /= p; - else - x = 0.0; - } - - return x; - } - - if (isUnsigned(op_ct.colDataType)) - { - return getUintVal(row, parm, isNull, op_ct); - } - - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + if (parm.size() > 1) // round(X, D) + decimalPlaceDouble(parm, d, p, row, isNull); if (isNull) - return 0.0; + return 0.0; - double d; + long double x = parm[0]->data()->getLongDoubleVal(row, isNull); - if (!op_ct.isWideDecimalType()) - d = x.value; - else - d = static_cast(x.toTSInt128()); - - if (x.scale > 0) + if (!isNull) { - while (x.scale-- > 0) - d /= 10.0; - } - else - { - while (x.scale++ < 0) - d *= 10.0; + x *= p; + + if (x >= 0) + x = floor(x + 0.5); + else + x = ceil(x - 0.5); + + if (p != 0.0) + x /= p; + else + x = 0.0; } - return d; + return x; + } + + if (isUnsigned(op_ct.colDataType)) + { + return getUintVal(row, parm, isNull, op_ct); + } + + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + + if (isNull) + return 0.0; + + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = static_cast(x.toTSInt128()); + + if (x.scale > 0) + { + while (x.scale-- > 0) + d /= 10.0; + } + else + { + while (x.scale++ < 0) + d *= 10.0; + } + + return d; } -long double Func_round::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - if (execplan::CalpontSystemCatalog::LONGDOUBLE == op_ct.colDataType || - execplan::CalpontSystemCatalog::DOUBLE == op_ct.colDataType || - execplan::CalpontSystemCatalog::FLOAT == op_ct.colDataType) - { - int64_t d = 0; - long double p = 1; + IDB_Decimal decimal; - if (parm.size() > 1) // round(X, D) - decimalPlaceDouble(parm, d, p, row, isNull); + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + int64_t d = 0; + decimal = parm[0]->data()->getDecimalVal(row, isNull); + + if (!op_ct.isWideDecimalType()) + { + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int64_t p = 1; + + if (!isNull && parm.size() > 1) // round(X, D) + { + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } if (isNull) - return 0.0; + break; - long double x = parm[0]->data()->getLongDoubleVal(row, isNull); + int64_t x = decimal.value; - if (!isNull) + if (d > 0) { - x *= p; + x = x * p; + } + else if (d < 0) + { + int64_t h = p / 2; // 0.5 + if ((x >= h) || (x <= -h)) + { if (x >= 0) - x = floor(x + 0.5); + x += h; else - x = ceil(x - 0.5); + x -= h; - if (p != 0.0) - x /= p; + if (p != 0) + x = x / p; else - x = 0.0; + x = 0; + } + else + { + x = 0; + } } - return x; - } + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; - if (isUnsigned(op_ct.colDataType)) + while (++decimal.scale < 0); + } + + decimal.value = x; + } + else + { + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int128_t p = 1; + + if (!isNull && parm.size() > 1) // round(X, D) + { + int128_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + if (d < -datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int128_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - return getUintVal(row, parm, isNull, op_ct); + uint64_t x = parm[0]->data()->getUintVal(row, isNull); + + if (x > (uint64_t)helpers::maxNumber_c[18]) + { + x = helpers::maxNumber_c[18]; + } + + decimal.value = x; + decimal.scale = 0; } + break; - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - - if (isNull) - return 0.0; - - double d; - - if (!op_ct.isWideDecimalType()) - d = x.value; - else - d = static_cast(x.toTSInt128()); - - if (x.scale > 0) + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - while (x.scale-- > 0) - d /= 10.0; - } - else - { - while (x.scale++ < 0) - d *= 10.0; - } + int64_t s = 0; + double p = 1; - return d; + if (parm.size() > 1) // round(X, D) + decimalPlaceDouble(parm, s, p, row, isNull); + + if (isNull) + break; + + double x = parm[0]->data()->getDoubleVal(row, isNull); + + if (!isNull) + { + x *= p; + + if (x >= 0) + x = floor(x + 0.5); + else + x = ceil(x - 0.5); + + decimal.value = (int64_t)x; + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + int64_t s = 0; + long double p = 1; + + if (parm.size() > 1) // round(X, D) + decimalPlaceDouble(parm, s, p, row, isNull); + + if (isNull) + break; + + long double x = parm[0]->data()->getDoubleVal(row, isNull); + + if (!isNull) + { + x *= p; + + if (x >= 0) + x = floor(x + 0.5); + else + x = ceil(x - 0.5); + + decimal.value = (int64_t)x; + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + + if (parm.size() > 1) // round(X, D) + { + s = parm[1]->data()->getIntVal(row, isNull); + + if (s > 0) + { + s = 0; // Dates don't have digits after int part + } + else + { + if (-s >= (int32_t)value.size()) + value = "0"; + else + { + // check to see if last digit needs to be rounded up + int firstcutdigit = atoi(value.substr(value.size() + s, 1).c_str()); + value = value.substr(0, value.size() + s); + int lastdigit = atoi(value.substr(value.size() - 1, 1).c_str()); + + if (firstcutdigit > 5) + { + lastdigit++; + string lastStr = intToString(lastdigit); + value = value.substr(0, value.size() - 1) + lastStr; + } + + s = -s; + + for (int i = 0; i < s; i++) + { + value = value + "0"; + } + } + + s = 0; + } + } + + int64_t x = atoll(value.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int32_t s = 0; + + string value; + if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), + timeZone()); + else + value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + // strip off micro seconds + value = value.substr(0, 14); + + if (parm.size() > 1) // round(X, D) + { + s = parm[1]->data()->getIntVal(row, isNull); + + if (s > 5) + s = 0; + + if (s > 0) + { + for (int i = 0; i < s; i++) + { + value = value + "0"; + } + } + else + { + if (-s >= (int32_t)value.size()) + value = "0"; + else + { + // check to see if last digit needs to be rounded up + int firstcutdigit = atoi(value.substr(value.size() + s, 1).c_str()); + value = value.substr(0, value.size() + s); + int lastdigit = atoi(value.substr(value.size() - 1, 1).c_str()); + + if (firstcutdigit > 5) + { + lastdigit++; + string lastStr = intToString(lastdigit); + value = value.substr(0, value.size() - 1) + lastStr; + } + + s = -s; + + for (int i = 0; i < s; i++) + { + value = value + "0"; + } + } + + s = 0; + } + } + + int64_t x = atoll(value.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + + default: + { + std::ostringstream oss; + oss << "round: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return decimal; } -IDB_Decimal Func_round::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_round::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) +{ + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + int64_t e = (x.scale < 0) ? (-x.scale) : x.scale; + int64_t p = 1; + + while (e-- > 0) + p *= 10; + + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + if (x.scale != 0) + { + if (x.scale > 0 && x.scale < 19) + { + x.value /= IDB_pow[x.scale]; + } + else if (x.scale < 0 && x.scale > -19) + { + x.value *= IDB_pow[-x.scale]; // may overflow + } + else if (x.scale > 0) + { + x.value = 0; + } + else // overflow may need throw exception + { + int64_t e = -x.scale % 18; + x.value *= IDB_pow[e]; + e = -x.scale - e; + + while (e > 0) + { + x.value *= IDB_pow[18]; + e -= 18; + } + } + + x.scale = 0; + } + + break; + + default: break; + } + + if (!op_ct.isWideDecimalType()) + return x.toString(); + else + return x.toString(true); +} + +int64_t Func_round::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal decimal; - - switch (op_ct.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - int64_t d = 0; - decimal = parm[0]->data()->getDecimalVal(row, isNull); - - if (!op_ct.isWideDecimalType()) - { - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int64_t p = 1; - - if (!isNull && parm.size() > 1) // round(X, D) - { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int64_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) - { - if (x >= 0) - x += h; - else - x -= h; - - if (p != 0) - x = x / p; - else - x = 0; - } - else - { - x = 0; - } - } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.value = x; - } - else - { - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int128_t p = 1; - - if (!isNull && parm.size() > 1) // round(X, D) - { - int128_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - if (d < -datatypes::INT128MAXPRECISION) - { - decimal.s128Value = 0; - break; - } - - int128_t x = decimal.s128Value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int128_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) - { - if (x >= 0) - x += h; - else - x -= h; - - if (p != 0) - x = x / p; - else - x = 0; - } - else - { - x = 0; - } - } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.s128Value = x; - } - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint64_t x = parm[0]->data()->getUintVal(row, isNull); - - if (x > (uint64_t)helpers::maxNumber_c[18]) - { - x = helpers::maxNumber_c[18]; - } - - decimal.value = x; - decimal.scale = 0; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - int64_t s = 0; - double p = 1; - - if (parm.size() > 1) // round(X, D) - decimalPlaceDouble(parm, s, p, row, isNull); - - if (isNull) - break; - - double x = parm[0]->data()->getDoubleVal(row, isNull); - - if (!isNull) - { - x *= p; - - if (x >= 0) - x = floor(x + 0.5); - else - x = ceil(x - 0.5); - - decimal.value = (int64_t) x; - decimal.scale = s; - } - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - int64_t s = 0; - long double p = 1; - - if (parm.size() > 1) // round(X, D) - decimalPlaceDouble(parm, s, p, row, isNull); - - if (isNull) - break; - - long double x = parm[0]->data()->getDoubleVal(row, isNull); - - if (!isNull) - { - x *= p; - - if (x >= 0) - x = floor(x + 0.5); - else - x = ceil(x - 0.5); - - decimal.value = (int64_t) x; - decimal.scale = s; - } - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t s = 0; - - string value = dataconvert::DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); - - if (parm.size() > 1) // round(X, D) - { - s = parm[1]->data()->getIntVal(row, isNull); - - if ( s > 0 ) - { - s = 0; // Dates don't have digits after int part - } - else - { - if ( -s >= (int32_t) value.size() ) - value = "0"; - else - { - //check to see if last digit needs to be rounded up - int firstcutdigit = atoi(value.substr(value.size() + s, 1).c_str()); - value = value.substr(0, value.size() + s); - int lastdigit = atoi(value.substr(value.size() - 1, 1).c_str()); - - if ( firstcutdigit > 5 ) - { - lastdigit++; - string lastStr = intToString(lastdigit); - value = value.substr(0, value.size() - 1) + lastStr; - } - - s = -s; - - for ( int i = 0 ; i < s ; i++) - { - value = value + "0"; - } - } - - s = 0; - } - } - - int64_t x = atoll(value.c_str()); - - if (!isNull) - { - decimal.value = x; - decimal.scale = s; - } - } - break; - - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int32_t s = 0; - - string value; - if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - else - value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - - //strip off micro seconds - value = value.substr(0, 14); - - if (parm.size() > 1) // round(X, D) - { - s = parm[1]->data()->getIntVal(row, isNull); - - if ( s > 5 ) - s = 0; - - if ( s > 0 ) - { - for ( int i = 0 ; i < s ; i++) - { - value = value + "0"; - } - } - else - { - if ( -s >= (int32_t) value.size() ) - value = "0"; - else - { - //check to see if last digit needs to be rounded up - int firstcutdigit = atoi(value.substr(value.size() + s, 1).c_str()); - value = value.substr(0, value.size() + s); - int lastdigit = atoi(value.substr(value.size() - 1, 1).c_str()); - - if ( firstcutdigit > 5 ) - { - lastdigit++; - string lastStr = intToString(lastdigit); - value = value.substr(0, value.size() - 1) + lastStr; - } - - s = -s; - - for ( int i = 0 ; i < s ; i++) - { - value = value + "0"; - } - } - - s = 0; - } - } - - int64_t x = atoll(value.c_str()); - - if (!isNull) - { - decimal.value = x; - decimal.scale = s; - } - } - break; - - - default: - { - std::ostringstream oss; - oss << "round: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - } - - return decimal; + return parm[0]->data()->getIntVal(row, isNull); } - -string Func_round::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) -{ - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - int64_t e = (x.scale < 0) ? (-x.scale) : x.scale; - int64_t p = 1; - - while (e-- > 0) - p *= 10; - - switch (op_ct.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - if (x.scale != 0) - { - if (x.scale > 0 && x.scale < 19) - { - x.value /= IDB_pow[x.scale]; - } - else if (x.scale < 0 && x.scale > -19) - { - x.value *= IDB_pow[-x.scale]; // may overflow - } - else if (x.scale > 0) - { - x.value = 0; - } - else // overflow may need throw exception - { - int64_t e = -x.scale % 18; - x.value *= IDB_pow[e]; - e = -x.scale - e; - - while (e > 0) - { - x.value *= IDB_pow[18]; - e -= 18; - } - } - - x.scale = 0; - } - - break; - - default: - break; - } - - if (!op_ct.isWideDecimalType()) - return x.toString(); - else - return x.toString(true); -} - - -int64_t Func_round::getDatetimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) -{ - return parm[0]->data()->getIntVal(row, isNull); -} - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rpad.cpp b/utils/funcexp/func_rpad.cpp index 483121369..fba47bfae 100644 --- a/utils/funcexp/func_rpad.cpp +++ b/utils/funcexp/func_rpad.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_rpad.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_rpad.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include "errorids.h" #include using namespace std; @@ -40,95 +40,91 @@ namespace funcexp { const string Func_rpad::fPad = " "; -CalpontSystemCatalog::ColType Func_rpad::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_rpad::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_rpad::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_rpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // In the case where someone entered pad length as a quoted string, - // it may be interpreted by columnstore to be an actual string - // and stored in fResult.int as a htonl of that string, - // However fResult.double is always correct, so we'll use that. - size_t padLength = (size_t)fp[1]->data()->getDoubleVal(row, isNull); - if (isNull || padLength <= 0) - return ""; - if (padLength > (size_t)INT_MAX32) - padLength = (size_t)INT_MAX32; - - if (padLength < strLen) - { - binLen = cs->charpos(pos, end, padLength); - std::string ret(pos, binLen); - return ret; - } + // In the case where someone entered pad length as a quoted string, + // it may be interpreted by columnstore to be an actual string + // and stored in fResult.int as a htonl of that string, + // However fResult.double is always correct, so we'll use that. + size_t padLength = (size_t)fp[1]->data()->getDoubleVal(row, isNull); + if (isNull || padLength <= 0) + return ""; + if (padLength > (size_t)INT_MAX32) + padLength = (size_t)INT_MAX32; - // The pad characters. - const string* pad = &fPad; - if (fp.size() > 2) - { - pad = &fp[2]->data()->getStrVal(row, isNull); - } - // binPLen represents the number of bytes in pad - size_t binPLen = pad->length(); - const char* posP = pad->c_str(); - // plen = the number of characters in pad - size_t plen = cs->numchars(posP, posP+binPLen); - if (plen == 0) - return src; - - size_t byteCount = (padLength+1) * cs->mbmaxlen; // absolute maximun number of bytes - char* buf = new char[byteCount]; - char* pBuf = buf; - - byteCount = 0; - - memcpy(pBuf, pos, binLen); - byteCount += binLen; - padLength -= strLen; - pBuf += binLen; - - while (padLength >= plen) - { - memcpy(pBuf, posP, binPLen); - padLength -= plen; - byteCount += binPLen; - pBuf += binPLen; - } - // Sometimes, in a case with multi-char pad, we need to add a partial pad - if (padLength > 0) - { - size_t partialSize = cs->charpos(posP, posP+plen, padLength); - memcpy(pBuf, posP, partialSize); - byteCount += partialSize; - } - - std::string ret(buf, byteCount); - delete [] buf; + if (padLength < strLen) + { + binLen = cs->charpos(pos, end, padLength); + std::string ret(pos, binLen); return ret; + } + + // The pad characters. + const string* pad = &fPad; + if (fp.size() > 2) + { + pad = &fp[2]->data()->getStrVal(row, isNull); + } + // binPLen represents the number of bytes in pad + size_t binPLen = pad->length(); + const char* posP = pad->c_str(); + // plen = the number of characters in pad + size_t plen = cs->numchars(posP, posP + binPLen); + if (plen == 0) + return src; + + size_t byteCount = (padLength + 1) * cs->mbmaxlen; // absolute maximun number of bytes + char* buf = new char[byteCount]; + char* pBuf = buf; + + byteCount = 0; + + memcpy(pBuf, pos, binLen); + byteCount += binLen; + padLength -= strLen; + pBuf += binLen; + + while (padLength >= plen) + { + memcpy(pBuf, posP, binPLen); + padLength -= plen; + byteCount += binPLen; + pBuf += binPLen; + } + // Sometimes, in a case with multi-char pad, we need to add a partial pad + if (padLength > 0) + { + size_t partialSize = cs->charpos(posP, posP + plen, padLength); + memcpy(pBuf, posP, partialSize); + byteCount += partialSize; + } + + std::string ret(buf, byteCount); + delete[] buf; + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_rtrim.cpp b/utils/funcexp/func_rtrim.cpp index 70233dc8c..426ec8712 100644 --- a/utils/funcexp/func_rtrim.cpp +++ b/utils/funcexp/func_rtrim.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_rtrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_rtrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,130 +34,127 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_rtrim::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_rtrim::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_rtrim::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& type) +std::string Func_rtrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // The trim characters. - const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); - // binTLen represents the number of bytes in trim - size_t binTLen = trim.length(); - const char* posT = trim.c_str(); - // strTLen = the number of characters in trim - size_t strTLen = cs->numchars(posT, posT+binTLen); - if (strTLen == 0 || strTLen > strLen) - return src; + // The trim characters. + const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); + // binTLen represents the number of bytes in trim + size_t binTLen = trim.length(); + const char* posT = trim.c_str(); + // strTLen = the number of characters in trim + size_t strTLen = cs->numchars(posT, posT + binTLen); + if (strTLen == 0 || strTLen > strLen) + return src; - if (binTLen == 1) + if (binTLen == 1) + { + const char* ptr = pos; + if (cs->use_mb()) // This is a multi-byte charset { - const char* ptr = pos; - if (cs->use_mb()) // This is a multi-byte charset + const char* p = pos; + uint32 l; + // Multibyte characters in the string give us alignment problems + // What we do here is skip past any multibyte characters. Whn + // don with this loop, ptr is pointing to a singlebyte char that + // is after all multibyte chars in the string, or to end. + while (ptr < end) + { + if ((l = my_ismbchar(cs, ptr, + end))) // returns the number of bytes in the leading char or zero if one byte { - const char* p = pos; - uint32 l; - // Multibyte characters in the string give us alignment problems - // What we do here is skip past any multibyte characters. Whn - // don with this loop, ptr is pointing to a singlebyte char that - // is after all multibyte chars in the string, or to end. - while (ptr < end) - { - if ((l = my_ismbchar(cs, ptr, end))) // returns the number of bytes in the leading char or zero if one byte - { - ptr += l; - p = ptr; - } - else - { - ++ptr; - } - } - ptr = p; - } - while (ptr < end && end[-1] == *posT) - { - --end; - --binLen; - } - } - else - { - // An uncommon case where the space character is > 1 byte - if (cs->use_mb()) // This is a multi-byte charset - { - // The problem is that the byte pattern at the end could - // match memcmp, but not be correct since the first byte compared - // may actually be a second or later byte from a previous char. - - // We start at the beginning of the string and move forward - // one character at a time until we reach the end. Then we can - // safely compare and remove on character. Then back to the beginning - // and try again. - while (end - binTLen >= pos) - { - const char* p = pos; - uint32 l; - while (p + binTLen < end) - { - if ((l = my_ismbchar(cs, p, end))) // returns the number of bytes in the leading char or zero if one byte - p += l; - else - ++p; - } - if (p + binTLen == end && memcmp(p,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } - else - { - break; // We've run out of places to look - } - } + ptr += l; + p = ptr; } else { - // This implies we have a single byte charset and a multibyte - // space character. - // Should never get here, since rtrim only trims space characters - // Included for completeness. - while (end-binTLen >= pos && memcmp(end-binTLen,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } + ++ptr; } + } + ptr = p; } - // Turn back to a string - std::string ret(pos, binLen); - return ret; + while (ptr < end && end[-1] == *posT) + { + --end; + --binLen; + } + } + else + { + // An uncommon case where the space character is > 1 byte + if (cs->use_mb()) // This is a multi-byte charset + { + // The problem is that the byte pattern at the end could + // match memcmp, but not be correct since the first byte compared + // may actually be a second or later byte from a previous char. + + // We start at the beginning of the string and move forward + // one character at a time until we reach the end. Then we can + // safely compare and remove on character. Then back to the beginning + // and try again. + while (end - binTLen >= pos) + { + const char* p = pos; + uint32 l; + while (p + binTLen < end) + { + if ((l = my_ismbchar(cs, p, + end))) // returns the number of bytes in the leading char or zero if one byte + p += l; + else + ++p; + } + if (p + binTLen == end && memcmp(p, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + else + { + break; // We've run out of places to look + } + } + } + else + { + // This implies we have a single byte charset and a multibyte + // space character. + // Should never get here, since rtrim only trims space characters + // Included for completeness. + while (end - binTLen >= pos && memcmp(end - binTLen, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + } + } + // Turn back to a string + std::string ret(pos, binLen); + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_rtrim_oracle.cpp b/utils/funcexp/func_rtrim_oracle.cpp index f3b104c39..1b47d7ddf 100644 --- a/utils/funcexp/func_rtrim_oracle.cpp +++ b/utils/funcexp/func_rtrim_oracle.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_rtrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_rtrim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,134 +34,131 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_rtrim_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_rtrim_oracle::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_rtrim_oracle::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& type) +std::string Func_rtrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // The trim characters. - const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); - // binTLen represents the number of bytes in trim - size_t binTLen = trim.length(); - const char* posT = trim.c_str(); - // strTLen = the number of characters in trim - size_t strTLen = cs->numchars(posT, posT+binTLen); - if (strTLen == 0 || strTLen > strLen) - return src; + // The trim characters. + const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); + // binTLen represents the number of bytes in trim + size_t binTLen = trim.length(); + const char* posT = trim.c_str(); + // strTLen = the number of characters in trim + size_t strTLen = cs->numchars(posT, posT + binTLen); + if (strTLen == 0 || strTLen > strLen) + return src; - if (binTLen == 1) + if (binTLen == 1) + { + const char* ptr = pos; + if (cs->use_mb()) // This is a multi-byte charset { - const char* ptr = pos; - if (cs->use_mb()) // This is a multi-byte charset + const char* p = pos; + uint32 l; + // Multibyte characters in the string give us alignment problems + // What we do here is skip past any multibyte characters. Whn + // don with this loop, ptr is pointing to a singlebyte char that + // is after all multibyte chars in the string, or to end. + while (ptr < end) + { + if ((l = my_ismbchar(cs, ptr, + end))) // returns the number of bytes in the leading char or zero if one byte { - const char* p = pos; - uint32 l; - // Multibyte characters in the string give us alignment problems - // What we do here is skip past any multibyte characters. Whn - // don with this loop, ptr is pointing to a singlebyte char that - // is after all multibyte chars in the string, or to end. - while (ptr < end) - { - if ((l = my_ismbchar(cs, ptr, end))) // returns the number of bytes in the leading char or zero if one byte - { - ptr += l; - p = ptr; - } - else - { - ++ptr; - } - } - ptr = p; - } - while (ptr < end && end[-1] == *posT) - { - --end; - --binLen; - } - } - else - { - // An uncommon case where the space character is > 1 byte - if (cs->use_mb()) // This is a multi-byte charset - { - // The problem is that the byte pattern at the end could - // match memcmp, but not be correct since the first byte compared - // may actually be a second or later byte from a previous char. - - // We start at the beginning of the string and move forward - // one character at a time until we reach the end. Then we can - // safely compare and remove on character. Then back to the beginning - // and try again. - while (end - binTLen >= pos) - { - const char* p = pos; - uint32 l; - while (p + binTLen < end) - { - if ((l = my_ismbchar(cs, p, end))) // returns the number of bytes in the leading char or zero if one byte - p += l; - else - ++p; - } - if (p + binTLen == end && memcmp(p,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } - else - { - break; // We've run out of places to look - } - } + ptr += l; + p = ptr; } else { - // This implies we have a single byte charset and a multibyte - // space character. - // Should never get here, since rtrim only trims space characters - // Included for completeness. - while (end-binTLen >= pos && memcmp(end-binTLen,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } + ++ptr; } + } + ptr = p; } - // Turn back to a string - std::string ret(pos, binLen); - if (binLen == 0) + while (ptr < end && end[-1] == *posT) { - isNull = true; + --end; + --binLen; } - return ret; + } + else + { + // An uncommon case where the space character is > 1 byte + if (cs->use_mb()) // This is a multi-byte charset + { + // The problem is that the byte pattern at the end could + // match memcmp, but not be correct since the first byte compared + // may actually be a second or later byte from a previous char. + + // We start at the beginning of the string and move forward + // one character at a time until we reach the end. Then we can + // safely compare and remove on character. Then back to the beginning + // and try again. + while (end - binTLen >= pos) + { + const char* p = pos; + uint32 l; + while (p + binTLen < end) + { + if ((l = my_ismbchar(cs, p, + end))) // returns the number of bytes in the leading char or zero if one byte + p += l; + else + ++p; + } + if (p + binTLen == end && memcmp(p, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + else + { + break; // We've run out of places to look + } + } + } + else + { + // This implies we have a single byte charset and a multibyte + // space character. + // Should never get here, since rtrim only trims space characters + // Included for completeness. + while (end - binTLen >= pos && memcmp(end - binTLen, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + } + } + // Turn back to a string + std::string ret(pos, binLen); + if (binLen == 0) + { + isNull = true; + } + return ret; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_sec_to_time.cpp b/utils/funcexp/func_sec_to_time.cpp index 58fb9d2e3..d3b4af675 100644 --- a/utils/funcexp/func_sec_to_time.cpp +++ b/utils/funcexp/func_sec_to_time.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_sec_to_time.cpp 2477 2011-05-12 16:07:35Z chao $ -* -* -****************************************************************************/ + * $Id: func_sec_to_time.cpp 2477 2011-05-12 16:07:35Z chao $ + * + * + ****************************************************************************/ #include #include @@ -41,204 +41,194 @@ using namespace execplan; using namespace logging; namespace funcexp { - -CalpontSystemCatalog::ColType Func_sec_to_time::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_sec_to_time::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_sec_to_time::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_sec_to_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { + int64_t val = 0; + CalpontSystemCatalog::ColType curCt = parm[0]->data()->resultType(); - int64_t val = 0; - CalpontSystemCatalog::ColType curCt = parm[0]->data()->resultType(); - - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - val = parm[0]->data()->getIntVal(row, isNull); - } - break; + val = parm[0]->data()->getIntVal(row, isNull); + } + break; - case execplan::CalpontSystemCatalog::DOUBLE: - { - datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull)); - val = d.toMCSSInt64Round(); - break; - } - case execplan::CalpontSystemCatalog::FLOAT: - { - datatypes::TDouble d(parm[0]->data()->getFloatVal(row, isNull)); - val = d.toMCSSInt64Round(); - } - break; + case execplan::CalpontSystemCatalog::DOUBLE: + { + datatypes::TDouble d(parm[0]->data()->getDoubleVal(row, isNull)); + val = d.toMCSSInt64Round(); + break; + } + case execplan::CalpontSystemCatalog::FLOAT: + { + datatypes::TDouble d(parm[0]->data()->getFloatVal(row, isNull)); + val = d.toMCSSInt64Round(); + } + break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - val = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); - break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + val = parm[0]->data()->getDecimalVal(row, isNull).toSInt64Round(); + break; - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = parm[0]->data()->getIntVal(row, isNull); + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = parm[0]->data()->getIntVal(row, isNull); - break; - } - - default: - { - std::ostringstream oss; - oss << "sec_to_time: datatype of " << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + break; } - int64_t posVal = llabs(val); + default: + { + std::ostringstream oss; + oss << "sec_to_time: datatype of " + << execplan::colDataTypeToString(parm[0]->data()->resultType().colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } - if (val > 3020399) - return ("838:59:59"); + int64_t posVal = llabs(val); - if (val < -3020399) - return ("-838:59:59"); + if (val > 3020399) + return ("838:59:59"); - //Format the time - uint32_t hour = 0; - uint32_t minute = 0; - uint32_t second = 0; + if (val < -3020399) + return ("-838:59:59"); - hour = posVal / 3600; - minute = (posVal - (hour * 3600)) / 60; - second = posVal - (hour * 3600) - (minute * 60); + // Format the time + uint32_t hour = 0; + uint32_t minute = 0; + uint32_t second = 0; - const char* minus = "-"; - const char* nominus = ""; + hour = posVal / 3600; + minute = (posVal - (hour * 3600)) / 60; + second = posVal - (hour * 3600) - (minute * 60); - const char* signstr = (val < 0) ? minus : nominus; + const char* minus = "-"; + const char* nominus = ""; - char buf[32]; // actual string either 9 or 10 characters - snprintf(buf, 32, "%s%02d:%02d:%02d", signstr, hour, minute, second); - return buf; + const char* signstr = (val < 0) ? minus : nominus; + + char buf[32]; // actual string either 9 or 10 characters + snprintf(buf, 32, "%s%02d:%02d:%02d", signstr, hour, minute, second); + return buf; } - -int64_t Func_sec_to_time::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_sec_to_time::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = parm[0]->data()->getIntVal(row, isNull); + int64_t val = parm[0]->data()->getIntVal(row, isNull); - if (val > 3020399) - val = 8385959; - else if (val < -3020399) - val = 4286581337LL; - else + if (val > 3020399) + val = 8385959; + else if (val < -3020399) + val = 4286581337LL; + else + { + string time = getStrVal(row, parm, isNull, op_ct); + size_t x = time.find(":"); + + while (x < string::npos) { - string time = getStrVal(row, parm, isNull, op_ct); - size_t x = time.find(":"); - - while (x < string::npos) - { - time.erase(x, 1); - x = time.find(":"); - } - - char* ep = NULL; - const char* str = time.c_str(); - errno = 0; - val = strtoll(str, &ep, 10); + time.erase(x, 1); + x = time.find(":"); } - return val; + char* ep = NULL; + const char* str = time.c_str(); + errno = 0; + val = strtoll(str, &ep, 10); + } + + return val; } -double Func_sec_to_time::getDoubleVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +double Func_sec_to_time::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { + double val = parm[0]->data()->getDoubleVal(row, isNull); - double val = parm[0]->data()->getDoubleVal(row, isNull); + if (val > 3020399) + val = 8385959; + else if (val < -3020399) + val = 4286581337LL; + else + { + string time = getStrVal(row, parm, isNull, op_ct); + size_t x = time.find(":"); - if (val > 3020399) - val = 8385959; - else if (val < -3020399) - val = 4286581337LL; - else + while (x < string::npos) { - string time = getStrVal(row, parm, isNull, op_ct); - size_t x = time.find(":"); - - while (x < string::npos) - { - time.erase(x, 1); - x = time.find(":"); - } - - char* ep = NULL; - const char* str = time.c_str(); - errno = 0; - val = (double)strtoll(str, &ep, 10); + time.erase(x, 1); + x = time.find(":"); } - return val; + char* ep = NULL; + const char* str = time.c_str(); + errno = 0; + val = (double)strtoll(str, &ep, 10); + } + + return val; } -execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) +execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal d; + IDB_Decimal d; - int64_t val = parm[0]->data()->getIntVal(row, isNull); + int64_t val = parm[0]->data()->getIntVal(row, isNull); - int64_t tmpVal; + int64_t tmpVal; - if (val > 3020399) - tmpVal = 8385959; - else if (val < -3020399) - tmpVal = 4286581337LL; - else + if (val > 3020399) + tmpVal = 8385959; + else if (val < -3020399) + tmpVal = 4286581337LL; + else + { + string time = getStrVal(row, parm, isNull, op_ct); + size_t x = time.find(":"); + + while (x < string::npos) { - string time = getStrVal(row, parm, isNull, op_ct); - size_t x = time.find(":"); - - while (x < string::npos) - { - time.erase(x, 1); - x = time.find(":"); - } - - char* ep = NULL; - const char* str = time.c_str(); - errno = 0; - tmpVal = strtoll(str, &ep, 10); + time.erase(x, 1); + x = time.find(":"); } - if (parm[0]->data()->resultType().isWideDecimalType()) - d.s128Value = tmpVal; - else - d.value = tmpVal; + char* ep = NULL; + const char* str = time.c_str(); + errno = 0; + tmpVal = strtoll(str, &ep, 10); + } - d.scale = 0; - return d; + if (parm[0]->data()->resultType().isWideDecimalType()) + d.s128Value = tmpVal; + else + d.value = tmpVal; + + d.scale = 0; + return d; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index ff09bf320..e07d29176 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_second.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_second.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,117 +35,113 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_second::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_second::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_second::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_second::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; + int64_t val = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; + if (val == -1) + isNull = true; - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - } - else - { - isNull = true; - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - { - isNull = true; - } - /* fall through */ - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - return m_time.second; - } - - case execplan::CalpontSystemCatalog::TIME: - { - val = parm[0]->data()->getTimeIntVal(row, isNull); - return (uint32_t)((val >> 24) & 0xff); - break; - } - - default: - { - isNull = true; - break; - } + break; } - if (isNull) - return -1; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - if ( val < 1000000000 ) - return 0; + if (val == -1) + isNull = true; + } + else + { + isNull = true; + } - return (uint32_t)((val >> 20) & 0x3f); + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + { + isNull = true; + } + /* fall through */ + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + return m_time.second; + } + + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + return (uint32_t)((val >> 24) & 0xff); + break; + } + + default: + { + isNull = true; + break; + } + } + + if (isNull) + return -1; + + if (val < 1000000000) + return 0; + + return (uint32_t)((val >> 20) & 0x3f); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sha.cpp b/utils/funcexp/func_sha.cpp index 769475b44..bc74a0a2a 100644 --- a/utils/funcexp/func_sha.cpp +++ b/utils/funcexp/func_sha.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id$ -* -* -****************************************************************************/ + * $Id$ + * + * + ****************************************************************************/ #include #include @@ -86,7 +86,7 @@ namespace funcexp */ SHA1::SHA1() { - Reset(); + Reset(); } /* @@ -106,7 +106,7 @@ SHA1::SHA1() */ SHA1::~SHA1() { - // The destructor does nothing + // The destructor does nothing } /* @@ -127,18 +127,18 @@ SHA1::~SHA1() */ void SHA1::Reset() { - Length_Low = 0; - Length_High = 0; - Message_Block_Index = 0; + Length_Low = 0; + Length_High = 0; + Message_Block_Index = 0; - H[0] = 0x67452301; - H[1] = 0xEFCDAB89; - H[2] = 0x98BADCFE; - H[3] = 0x10325476; - H[4] = 0xC3D2E1F0; + H[0] = 0x67452301; + H[1] = 0xEFCDAB89; + H[2] = 0x98BADCFE; + H[3] = 0x10325476; + H[4] = 0xC3D2E1F0; - Computed = false; - Corrupted = false; + Computed = false; + Corrupted = false; } /* @@ -161,25 +161,25 @@ void SHA1::Reset() */ bool SHA1::Result(unsigned* message_digest_array) { - int i; // Counter + int i; // Counter - if (Corrupted) - { - return false; - } + if (Corrupted) + { + return false; + } - if (!Computed) - { - PadMessage(); - Computed = true; - } + if (!Computed) + { + PadMessage(); + Computed = true; + } - for (i = 0; i < 5; i++) - { - message_digest_array[i] = H[i]; - } + for (i = 0; i < 5; i++) + { + message_digest_array[i] = H[i]; + } - return true; + return true; } /* @@ -200,45 +200,44 @@ bool SHA1::Result(unsigned* message_digest_array) * Comments: * */ -void SHA1::Input( const unsigned char* message_array, - unsigned length) +void SHA1::Input(const unsigned char* message_array, unsigned length) { - if (!length) + if (!length) + { + return; + } + + if (Computed || Corrupted) + { + Corrupted = true; + return; + } + + while (length-- && !Corrupted) + { + Message_Block[Message_Block_Index++] = (*message_array & 0xFF); + + Length_Low += 8; + Length_Low &= 0xFFFFFFFF; // Force it to 32 bits + + if (Length_Low == 0) { - return; + Length_High++; + Length_High &= 0xFFFFFFFF; // Force it to 32 bits + + if (Length_High == 0) + { + Corrupted = true; // Message is too long + } } - if (Computed || Corrupted) + if (Message_Block_Index == 64) { - Corrupted = true; - return; + ProcessMessageBlock(); } - while (length-- && !Corrupted) - { - Message_Block[Message_Block_Index++] = (*message_array & 0xFF); - - Length_Low += 8; - Length_Low &= 0xFFFFFFFF; // Force it to 32 bits - - if (Length_Low == 0) - { - Length_High++; - Length_High &= 0xFFFFFFFF; // Force it to 32 bits - - if (Length_High == 0) - { - Corrupted = true; // Message is too long - } - } - - if (Message_Block_Index == 64) - { - ProcessMessageBlock(); - } - - message_array++; - } + message_array++; + } } /* @@ -261,10 +260,9 @@ void SHA1::Input( const unsigned char* message_array, * Comments: * */ -void SHA1::Input( const char* message_array, - unsigned length) +void SHA1::Input(const char* message_array, unsigned length) { - Input((unsigned char*) message_array, length); + Input((unsigned char*)message_array, length); } /* @@ -285,7 +283,7 @@ void SHA1::Input( const char* message_array, */ void SHA1::Input(unsigned char message_element) { - Input(&message_element, 1); + Input(&message_element, 1); } /* @@ -306,7 +304,7 @@ void SHA1::Input(unsigned char message_element) */ void SHA1::Input(char message_element) { - Input((unsigned char*) &message_element, 1); + Input((unsigned char*)&message_element, 1); } /* @@ -329,15 +327,15 @@ void SHA1::Input(char message_element) */ SHA1& SHA1::operator<<(const char* message_array) { - const char* p = message_array; + const char* p = message_array; - while (*p) - { - Input(*p); - p++; - } + while (*p) + { + Input(*p); + p++; + } - return *this; + return *this; } /* @@ -360,15 +358,15 @@ SHA1& SHA1::operator<<(const char* message_array) */ SHA1& SHA1::operator<<(const unsigned char* message_array) { - const unsigned char* p = message_array; + const unsigned char* p = message_array; - while (*p) - { - Input(*p); - p++; - } + while (*p) + { + Input(*p); + p++; + } - return *this; + return *this; } /* @@ -390,9 +388,9 @@ SHA1& SHA1::operator<<(const unsigned char* message_array) */ SHA1& SHA1::operator<<(const char message_element) { - Input((unsigned char*) &message_element, 1); + Input((unsigned char*)&message_element, 1); - return *this; + return *this; } /* @@ -414,9 +412,9 @@ SHA1& SHA1::operator<<(const char message_element) */ SHA1& SHA1::operator<<(const unsigned char message_element) { - Input(&message_element, 1); + Input(&message_element, 1); - return *this; + return *this; } /* @@ -440,92 +438,86 @@ SHA1& SHA1::operator<<(const unsigned char message_element) */ void SHA1::ProcessMessageBlock() { - const unsigned K[] = // Constants defined for SHA-1 - { - 0x5A827999, - 0x6ED9EBA1, - 0x8F1BBCDC, - 0xCA62C1D6 - }; - int t; // Loop counter - unsigned temp; // Temporary word value - unsigned W[80]; // Word sequence - unsigned A, B, C, D, E; // Word buffers + const unsigned K[] = // Constants defined for SHA-1 + {0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6}; + int t; // Loop counter + unsigned temp; // Temporary word value + unsigned W[80]; // Word sequence + unsigned A, B, C, D, E; // Word buffers - /* - * Initialize the first 16 words in the array W - */ - for (t = 0; t < 16; t++) - { - W[t] = ((unsigned) Message_Block[t * 4]) << 24; - W[t] |= ((unsigned) Message_Block[t * 4 + 1]) << 16; - W[t] |= ((unsigned) Message_Block[t * 4 + 2]) << 8; - W[t] |= ((unsigned) Message_Block[t * 4 + 3]); - } + /* + * Initialize the first 16 words in the array W + */ + for (t = 0; t < 16; t++) + { + W[t] = ((unsigned)Message_Block[t * 4]) << 24; + W[t] |= ((unsigned)Message_Block[t * 4 + 1]) << 16; + W[t] |= ((unsigned)Message_Block[t * 4 + 2]) << 8; + W[t] |= ((unsigned)Message_Block[t * 4 + 3]); + } - for (t = 16; t < 80; t++) - { - W[t] = CircularShift(1, W[t - 3] ^ W[t - 8] ^ W[t - 14] ^ W[t - 16]); - } + for (t = 16; t < 80; t++) + { + W[t] = CircularShift(1, W[t - 3] ^ W[t - 8] ^ W[t - 14] ^ W[t - 16]); + } - A = H[0]; - B = H[1]; - C = H[2]; - D = H[3]; - E = H[4]; + A = H[0]; + B = H[1]; + C = H[2]; + D = H[3]; + E = H[4]; - for (t = 0; t < 20; t++) - { - temp = CircularShift(5, A) + ((B & C) | ((~B) & D)) + E + W[t] + K[0]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = CircularShift(30, B); - B = A; - A = temp; - } + for (t = 0; t < 20; t++) + { + temp = CircularShift(5, A) + ((B & C) | ((~B) & D)) + E + W[t] + K[0]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = CircularShift(30, B); + B = A; + A = temp; + } - for (t = 20; t < 40; t++) - { - temp = CircularShift(5, A) + (B ^ C ^ D) + E + W[t] + K[1]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = CircularShift(30, B); - B = A; - A = temp; - } + for (t = 20; t < 40; t++) + { + temp = CircularShift(5, A) + (B ^ C ^ D) + E + W[t] + K[1]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = CircularShift(30, B); + B = A; + A = temp; + } - for (t = 40; t < 60; t++) - { - temp = CircularShift(5, A) + - ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = CircularShift(30, B); - B = A; - A = temp; - } + for (t = 40; t < 60; t++) + { + temp = CircularShift(5, A) + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = CircularShift(30, B); + B = A; + A = temp; + } - for (t = 60; t < 80; t++) - { - temp = CircularShift(5, A) + (B ^ C ^ D) + E + W[t] + K[3]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = CircularShift(30, B); - B = A; - A = temp; - } + for (t = 60; t < 80; t++) + { + temp = CircularShift(5, A) + (B ^ C ^ D) + E + W[t] + K[3]; + temp &= 0xFFFFFFFF; + E = D; + D = C; + C = CircularShift(30, B); + B = A; + A = temp; + } - H[0] = (H[0] + A) & 0xFFFFFFFF; - H[1] = (H[1] + B) & 0xFFFFFFFF; - H[2] = (H[2] + C) & 0xFFFFFFFF; - H[3] = (H[3] + D) & 0xFFFFFFFF; - H[4] = (H[4] + E) & 0xFFFFFFFF; + H[0] = (H[0] + A) & 0xFFFFFFFF; + H[1] = (H[1] + B) & 0xFFFFFFFF; + H[2] = (H[2] + C) & 0xFFFFFFFF; + H[3] = (H[3] + D) & 0xFFFFFFFF; + H[4] = (H[4] + E) & 0xFFFFFFFF; - Message_Block_Index = 0; + Message_Block_Index = 0; } /* @@ -551,53 +543,51 @@ void SHA1::ProcessMessageBlock() */ void SHA1::PadMessage() { - /* - * Check to see if the current message block is too small to hold - * the initial padding bits and length. If so, we will pad the - * block, process it, and then continue padding into a second block. - */ - if (Message_Block_Index > 55) + /* + * Check to see if the current message block is too small to hold + * the initial padding bits and length. If so, we will pad the + * block, process it, and then continue padding into a second block. + */ + if (Message_Block_Index > 55) + { + Message_Block[Message_Block_Index++] = 0x80; + + while (Message_Block_Index < 64) { - Message_Block[Message_Block_Index++] = 0x80; - - while (Message_Block_Index < 64) - { - Message_Block[Message_Block_Index++] = 0; - } - - ProcessMessageBlock(); - - while (Message_Block_Index < 56) - { - Message_Block[Message_Block_Index++] = 0; - } + Message_Block[Message_Block_Index++] = 0; } - else - { - Message_Block[Message_Block_Index++] = 0x80; - - while (Message_Block_Index < 56) - { - Message_Block[Message_Block_Index++] = 0; - } - - } - - /* - * Store the message length as the last 8 octets - */ - Message_Block[56] = (Length_High >> 24) & 0xFF; - Message_Block[57] = (Length_High >> 16) & 0xFF; - Message_Block[58] = (Length_High >> 8) & 0xFF; - Message_Block[59] = (Length_High) & 0xFF; - Message_Block[60] = (Length_Low >> 24) & 0xFF; - Message_Block[61] = (Length_Low >> 16) & 0xFF; - Message_Block[62] = (Length_Low >> 8) & 0xFF; - Message_Block[63] = (Length_Low) & 0xFF; ProcessMessageBlock(); -} + while (Message_Block_Index < 56) + { + Message_Block[Message_Block_Index++] = 0; + } + } + else + { + Message_Block[Message_Block_Index++] = 0x80; + + while (Message_Block_Index < 56) + { + Message_Block[Message_Block_Index++] = 0; + } + } + + /* + * Store the message length as the last 8 octets + */ + Message_Block[56] = (Length_High >> 24) & 0xFF; + Message_Block[57] = (Length_High >> 16) & 0xFF; + Message_Block[58] = (Length_High >> 8) & 0xFF; + Message_Block[59] = (Length_High)&0xFF; + Message_Block[60] = (Length_Low >> 24) & 0xFF; + Message_Block[61] = (Length_Low >> 16) & 0xFF; + Message_Block[62] = (Length_Low >> 8) & 0xFF; + Message_Block[63] = (Length_Low)&0xFF; + + ProcessMessageBlock(); +} /* * CircularShift @@ -619,52 +609,50 @@ void SHA1::PadMessage() */ unsigned SHA1::CircularShift(int bits, unsigned word) { - return ((word << bits) & 0xFFFFFFFF) | ((word & 0xFFFFFFFF) >> (32 - bits)); + return ((word << bits) & 0xFFFFFFFF) | ((word & 0xFFFFFFFF) >> (32 - bits)); } /** Definition of class Func_sha */ -CalpontSystemCatalog::ColType Func_sha::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_sha::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_sha::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_sha::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - SHA1 sha; - uint32_t message_digest[5]; + SHA1 sha; + uint32_t message_digest[5]; - // Input is always treated as sring - sha.Reset(); - sha << parm[0]->data()->getStrVal(row, isNull).c_str(); + // Input is always treated as sring + sha.Reset(); + sha << parm[0]->data()->getStrVal(row, isNull).c_str(); - // can not compute + // can not compute #ifdef _MSC_VER - // This cast is probably portable, but we'll leave it for Windows only for now... - if (!sha.Result(reinterpret_cast(message_digest))) + // This cast is probably portable, but we'll leave it for Windows only for now... + if (!sha.Result(reinterpret_cast(message_digest))) #else - if (!sha.Result(message_digest)) + if (!sha.Result(message_digest)) #endif - { - isNull = true; - return ""; - } + { + isNull = true; + return ""; + } - // result length is always 40+1 - char result[41]; - snprintf(result, 41, "%08x", message_digest[0]); - snprintf(result + 8, 41 - 8, "%08x", message_digest[1]); - snprintf(result + 16, 41 - 16, "%08x", message_digest[2]); - snprintf(result + 24, 41 - 24, "%08x", message_digest[3]); - snprintf(result + 32, 41 - 32, "%08x", message_digest[4]); - result[40] = 0; - return result; + // result length is always 40+1 + char result[41]; + snprintf(result, 41, "%08x", message_digest[0]); + snprintf(result + 8, 41 - 8, "%08x", message_digest[1]); + snprintf(result + 16, 41 - 16, "%08x", message_digest[2]); + snprintf(result + 24, 41 - 24, "%08x", message_digest[3]); + snprintf(result + 32, 41 - 32, "%08x", message_digest[4]); + result[40] = 0; + return result; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sign.cpp b/utils/funcexp/func_sign.cpp index ba999e06a..a0cece67a 100644 --- a/utils/funcexp/func_sign.cpp +++ b/utils/funcexp/func_sign.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_sign.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_sign.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -33,45 +33,42 @@ using namespace execplan; namespace funcexp { -CalpontSystemCatalog::ColType Func_sign::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_sign::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + return ct; } -int64_t Func_sign::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_sign::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - double val = parm[0]->data()->getDoubleVal(row, isNull); + double val = parm[0]->data()->getDoubleVal(row, isNull); - if (isNull) - return 0; - else if (val > 0) - return 1; - else if (val < 0) - return -1; - else - return 0; + if (isNull) + return 0; + else if (val > 0) + return 1; + else if (val < 0) + return -1; + else + return 0; } -string Func_sign::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_sign::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t sign = getIntVal(row, parm, isNull, op_ct); + int64_t sign = getIntVal(row, parm, isNull, op_ct); - if (sign > 0) - return string("1"); - else if (sign < 0) - return string("-1"); - else - return string("0"); + if (sign > 0) + return string("1"); + else if (sign < 0) + return string("-1"); + else + return string("0"); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_space.cpp b/utils/funcexp/func_space.cpp index 7b70a78da..93e765a56 100644 --- a/utils/funcexp/func_space.cpp +++ b/utils/funcexp/func_space.cpp @@ -26,41 +26,35 @@ using namespace rowgroup; namespace funcexp { - -CalpontSystemCatalog::ColType Func_space::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_space::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -std::string Func_space::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_space::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { + CalpontSystemCatalog::ColDataType ct = fp[0]->data()->resultType().colDataType; - CalpontSystemCatalog::ColDataType ct = fp[0]->data()->resultType().colDataType; + // Int representation of temporal types can be a very large value, + // so exit early if this is the case + if (ct == CalpontSystemCatalog::DATE || ct == CalpontSystemCatalog::DATETIME || + ct == CalpontSystemCatalog::TIMESTAMP || ct == CalpontSystemCatalog::TIME) + { + isNull = true; + return ""; + } - // Int representation of temporal types can be a very large value, - // so exit early if this is the case - if (ct == CalpontSystemCatalog::DATE || - ct == CalpontSystemCatalog::DATETIME || - ct == CalpontSystemCatalog::TIMESTAMP || - ct == CalpontSystemCatalog::TIME) - { - isNull = true; - return ""; - } + int64_t count = fp[0]->data()->getIntVal(row, isNull); - int64_t count = fp[0]->data()->getIntVal(row, isNull); + if (isNull || count < 1) + return ""; - if (isNull || count < 1) - return ""; + string result(count, ' '); - string result(count, ' '); - - return result; + return result; } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index 3a4c165ac..ff1f53356 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_str_to_date.cpp 2578 2011-05-12 16:26:55Z chao $ -* -* -****************************************************************************/ + * $Id: func_str_to_date.cpp 2578 2011-05-12 16:26:55Z chao $ + * + * + ****************************************************************************/ #include #include @@ -41,245 +41,227 @@ namespace { using namespace funcexp; -dataconvert::DateTime getDateTime (rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct, - const string& timeZone) +dataconvert::DateTime getDateTime(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct, const string& timeZone) { - TimeExtractor extractor; - dataconvert::DateTime dateTime; - dateTime.year = 0; - dateTime.month = 0; - dateTime.day = 0; - dateTime.hour = 0; - dateTime.minute = 0; - dateTime.second = 0; - dateTime.msecond = 0; - int64_t val = 0; - string valStr; - const string& formatStr = parm[1]->data()->getStrVal(row, isNull); - int rc = 0; + TimeExtractor extractor; + dataconvert::DateTime dateTime; + dateTime.year = 0; + dateTime.month = 0; + dateTime.day = 0; + dateTime.hour = 0; + dateTime.minute = 0; + dateTime.second = 0; + dateTime.msecond = 0; + int64_t val = 0; + string valStr; + const string& formatStr = parm[1]->data()->getStrVal(row, isNull); + int rc = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: { - case CalpontSystemCatalog::DATE: - { - val = parm[0]->data()->getIntVal(row, isNull); - valStr = dataconvert::DataConvert::dateToString (val); - rc = extractor.extractTime (valStr, formatStr, dateTime); + val = parm[0]->data()->getIntVal(row, isNull); + valStr = dataconvert::DataConvert::dateToString(val); + rc = extractor.extractTime(valStr, formatStr, dateTime); - if ( rc < 0) - { - isNull = true; - return -1; - } + if (rc < 0) + { + isNull = true; + return -1; + } - break; - } - - case CalpontSystemCatalog::DATETIME: - { - val = parm[0]->data()->getIntVal(row, isNull); - valStr = dataconvert::DataConvert::datetimeToString (val); - rc = extractor.extractTime (valStr, formatStr, dateTime); - - if ( rc < 0) - { - isNull = true; - return -1; - } - - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - val = parm[0]->data()->getIntVal(row, isNull); - valStr = dataconvert::DataConvert::timestampToString (val, timeZone); - rc = extractor.extractTime (valStr, formatStr, dateTime); - - if ( rc < 0) - { - isNull = true; - return -1; - } - - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - { - const string& valref = parm[0]->data()->getStrVal(row, isNull); - //decode with provided format - rc = extractor.extractTime (valref, formatStr, dateTime); - - if ( rc < 0) - { - isNull = true; - return -1; - } - - break; - } - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - { - val = parm[0]->data()->getIntVal(row, isNull); - //decode with provided format - rc = extractor.extractTime (helpers::intToString(val), formatStr, dateTime); - - if ( rc < 0) - { - isNull = true; - return -1; - } - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - { - val = parm[0]->data()->getIntVal(row, isNull); - - //decode with provided format - rc = extractor.extractTime (helpers::intToString(val), formatStr, dateTime); - - if ( rc < 0) - { - isNull = true; - return -1; - } - } - else - { - isNull = true; - return -1; - } - - break; - } - - default: - isNull = true; - return -1; + break; } - return dateTime; + case CalpontSystemCatalog::DATETIME: + { + val = parm[0]->data()->getIntVal(row, isNull); + valStr = dataconvert::DataConvert::datetimeToString(val); + rc = extractor.extractTime(valStr, formatStr, dateTime); + + if (rc < 0) + { + isNull = true; + return -1; + } + + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getIntVal(row, isNull); + valStr = dataconvert::DataConvert::timestampToString(val, timeZone); + rc = extractor.extractTime(valStr, formatStr, dateTime); + + if (rc < 0) + { + isNull = true; + return -1; + } + + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + const string& valref = parm[0]->data()->getStrVal(row, isNull); + // decode with provided format + rc = extractor.extractTime(valref, formatStr, dateTime); + + if (rc < 0) + { + isNull = true; + return -1; + } + + break; + } + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + { + val = parm[0]->data()->getIntVal(row, isNull); + // decode with provided format + rc = extractor.extractTime(helpers::intToString(val), formatStr, dateTime); + + if (rc < 0) + { + isNull = true; + return -1; + } + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (parm[0]->data()->resultType().scale == 0) + { + val = parm[0]->data()->getIntVal(row, isNull); + + // decode with provided format + rc = extractor.extractTime(helpers::intToString(val), formatStr, dateTime); + + if (rc < 0) + { + isNull = true; + return -1; + } + } + else + { + isNull = true; + return -1; + } + + break; + } + + default: isNull = true; return -1; + } + + return dateTime; } -} +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_str_to_date::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_str_to_date::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_str_to_date::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_str_to_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); - string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*) &dateTime)); - return convertedDate; + dataconvert::DateTime dateTime; + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*)&dateTime)); + return convertedDate; } -int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); - int64_t time = *(reinterpret_cast(&dateTime)); - return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); + dataconvert::DateTime dateTime; + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + int64_t time = *(reinterpret_cast(&dateTime)); + return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); } -int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); - int64_t time = *(reinterpret_cast(&dateTime)); - return time; + dataconvert::DateTime dateTime; + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + int64_t time = *(reinterpret_cast(&dateTime)); + return time; } -int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); - dataconvert::TimeStamp timestamp; - dataconvert::MySQLTime m_time; - m_time.year = dateTime.year; - m_time.month = dateTime.month; - m_time.day = dateTime.day; - m_time.hour = dateTime.hour; - m_time.minute = dateTime.minute; - m_time.second = dateTime.second; - bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone(), isValid); - if (!isValid) - { - timestamp = -1; - isNull = true; - } - else - { - timestamp.second = seconds; - timestamp.msecond = dateTime.msecond; - } - int64_t time = *(reinterpret_cast(×tamp)); - return time; + dataconvert::DateTime dateTime; + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dataconvert::TimeStamp timestamp; + dataconvert::MySQLTime m_time; + m_time.year = dateTime.year; + m_time.month = dateTime.month; + m_time.day = dateTime.day; + m_time.hour = dateTime.hour; + m_time.minute = dateTime.minute; + m_time.second = dateTime.second; + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone(), isValid); + if (!isValid) + { + timestamp = -1; + isNull = true; + } + else + { + timestamp.second = seconds; + timestamp.msecond = dateTime.msecond; + } + int64_t time = *(reinterpret_cast(×tamp)); + return time; } -int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + dataconvert::DateTime dateTime; + dataconvert::Time retTime; + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + retTime.day = 0; + retTime.is_neg = false; + retTime.hour = dateTime.hour; + retTime.minute = dateTime.minute; + retTime.second = dateTime.second; + retTime.msecond = dateTime.msecond; + int64_t time = *(reinterpret_cast(&retTime)); + return time; +} + +int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - dataconvert::DateTime dateTime; - dataconvert::Time retTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); - retTime.day = 0; - retTime.is_neg = false; - retTime.hour = dateTime.hour; - retTime.minute = dateTime.minute; - retTime.second = dateTime.second; - retTime.msecond = dateTime.msecond; - int64_t time = *(reinterpret_cast(&retTime)); - return time; + dataconvert::DateTime dateTime; + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + int64_t time = *(reinterpret_cast(&dateTime)); + return time; } -int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) -{ - dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); - int64_t time = *(reinterpret_cast(&dateTime)); - return time; -} - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_strcmp.cpp b/utils/funcexp/func_strcmp.cpp index e3a28aedf..a16c68cc4 100644 --- a/utils/funcexp/func_strcmp.cpp +++ b/utils/funcexp/func_strcmp.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_strcmp.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_strcmp.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -38,48 +38,40 @@ using namespace joblist; using namespace funcexp; - - namespace funcexp { - -CalpontSystemCatalog::ColType Func_strcmp::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_strcmp::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - //return fp[0]->data()->resultType(); - return resultType; + // operation type is not used by this functor + // return fp[0]->data()->resultType(); + return resultType; } -int64_t Func_strcmp::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_strcmp::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); - const string& str = fp[0]->data()->getStrVal(row, isNull); - const string& str1 = fp[1]->data()->getStrVal(row, isNull); + CHARSET_INFO* cs = fp[0]->data()->resultType().getCharset(); + const string& str = fp[0]->data()->getStrVal(row, isNull); + const string& str1 = fp[1]->data()->getStrVal(row, isNull); - int ret = cs->strnncollsp(str.c_str(), str.length(), str1.c_str(), str1.length()); - // mysql's strcmp returns only -1, 0, and 1 - return (ret < 0 ? -1 : (ret > 0 ? 1 : 0)); + int ret = cs->strnncollsp(str.c_str(), str.length(), str1.c_str(), str1.length()); + // mysql's strcmp returns only -1, 0, and 1 + return (ret < 0 ? -1 : (ret > 0 ? 1 : 0)); } - -std::string Func_strcmp::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_strcmp::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - int64_t val = getIntVal(row, fp, isNull, type); + int64_t val = getIntVal(row, fp, isNull, type); - if (val > 0) - return string("1"); - else if (val < 0) - return string("-1"); - else - return string("0"); + if (val > 0) + return string("1"); + else if (val < 0) + return string("-1"); + else + return string("0"); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_substr.cpp b/utils/funcexp/func_substr.cpp index 469b28c74..cb45acc0b 100644 --- a/utils/funcexp/func_substr.cpp +++ b/utils/funcexp/func_substr.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_substr.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_substr.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,74 +34,68 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_substr::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_substr::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_substr::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_substr::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - CHARSET_INFO* cs = ct.getCharset(); + CHARSET_INFO* cs = ct.getCharset(); - const string& str = fp[0]->data()->getStrVal(row, isNull); + const string& str = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + int64_t strLen = str.length(); + const char* strptr = str.c_str(); + const char* strend = strptr + strLen; + uint32_t strChars = cs->numchars(strptr, strend); + + int64_t start = fp[1]->data()->getIntVal(row, isNull) - 1; + if (isNull) + return ""; + if (start < -1) // negative pos, beginning from end + start += strChars + 1; + if (start < 0 || strChars <= start) + { + return ""; + } + + int64_t length; + if (fp.size() == 3) + { + length = fp[2]->data()->getIntVal(row, isNull); if (isNull) - return ""; - int64_t strLen = str.length(); - const char* strptr = str.c_str(); - const char* strend = strptr + strLen; - uint32_t strChars = cs->numchars(strptr, strend); - - int64_t start = fp[1]->data()->getIntVal(row, isNull) - 1; - if (isNull) - return ""; - if (start < -1) // negative pos, beginning from end - start += strChars + 1; - if (start < 0 || strChars <= start) - { - return ""; - } + return ""; + if (length < 1) + return ""; + } + else + { + length = strChars - start; + } - int64_t length; - if (fp.size() == 3) - { - length = fp[2]->data()->getIntVal(row, isNull); - if (isNull) - return ""; - if (length < 1) - return ""; - } - else - { - length = strChars - start; - } + // start is now number of chars into str to start the substring + // We convert it to number of bytes: + start = cs->charpos(strptr, strend, start); + // Convert length to bytes as well + length = cs->charpos(strptr + start, strend, length); + if ((start < 0) || (start + 1 > strLen)) + return ""; - // start is now number of chars into str to start the substring - // We convert it to number of bytes: - start = cs->charpos(strptr, strend, start); - // Convert length to bytes as well - length= cs->charpos(strptr + start, strend, length); - if ((start < 0) || (start + 1 > strLen)) - return ""; + if (start == 0 && strLen == length) + return str; - if (start == 0 && strLen == length) - return str; + length = std::min(length, strLen - start); - length= std::min(length, strLen - start); - - std::string ret(strptr + start, length); - return ret; + std::string ret(strptr + start, length); + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_substring_index.cpp b/utils/funcexp/func_substring_index.cpp index 83fd44063..ffae25e5e 100644 --- a/utils/funcexp/func_substring_index.cpp +++ b/utils/funcexp/func_substring_index.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_substring_index.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_substring_index.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,179 +34,175 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_substring_index::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_substring_index::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -std::string Func_substring_index::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) +std::string Func_substring_index::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { - CHARSET_INFO* cs = ct.getCharset(); + CHARSET_INFO* cs = ct.getCharset(); - const string& str = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - int64_t strLen = str.length(); - - const string& delimstr = fp[1]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - int64_t delimLen = delimstr.length(); - - int64_t count = fp[2]->data()->getIntVal(row, isNull); - if (isNull) - return ""; + const string& str = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + int64_t strLen = str.length(); - if (strLen == 0 || delimLen == 0 || count == 0) - return ""; + const string& delimstr = fp[1]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + int64_t delimLen = delimstr.length(); - if (count > strLen) - return str; + int64_t count = fp[2]->data()->getIntVal(row, isNull); + if (isNull) + return ""; - if ((count < 0) && ((count * -1) > strLen)) - return str; + if (strLen == 0 || delimLen == 0 || count == 0) + return ""; - bool binaryCmp = (cs->state & MY_CS_BINSORT) || !cs->use_mb(); - std::string value; // Only used if !use_mb() - - if (!binaryCmp) // Charset supports multibyte characters + if (count > strLen) + return str; + + if ((count < 0) && ((count * -1) > strLen)) + return str; + + bool binaryCmp = (cs->state & MY_CS_BINSORT) || !cs->use_mb(); + std::string value; // Only used if !use_mb() + + if (!binaryCmp) // Charset supports multibyte characters + { + const char* src = str.c_str(); + const char* srcEnd = src + strLen; + const char* srchEnd = srcEnd - delimLen + 1; + const char* delim = delimstr.c_str(); + const char* delimEnd = delim + delimLen; + char* ptr = const_cast(src); + char *i, *j; + uint32_t l; + int32_t n = 0, c = count, pass; + // For count > 0, this loop goes once. + // For count < 0, it goes twice + for (pass = (count > 0 ? 1 : 0); pass < 2; ++pass) { - const char* src = str.c_str(); - const char* srcEnd = src + strLen; - const char* srchEnd = srcEnd - delimLen + 1; - const char* delim = delimstr.c_str(); - const char* delimEnd = delim + delimLen; - char* ptr = const_cast(src); - char *i,*j; - uint32_t l; - int32_t n = 0, c = count, pass; - // For count > 0, this loop goes once. - // For count < 0, it goes twice - for (pass = (count > 0 ? 1 : 0); pass<2; ++pass) + while (ptr < srchEnd) + { + bool found = false; + if (*ptr == *delim) // If the first byte matches, maybe we have a match { - while (ptr < srchEnd) + // Do a byte by byte compare of src at that spot against delim + i = ptr + 1; + j = const_cast(delim) + 1; + found = true; + while (j != delimEnd) + { + if (*i++ != *j++) { - bool found = false; - if (*ptr == *delim) // If the first byte matches, maybe we have a match - { - // Do a byte by byte compare of src at that spot against delim - i = ptr + 1; - j = const_cast(delim) + 1; - found = true; - while (j != delimEnd) - { - if (*i++ != *j++) - { - found = false; - break; - } - } - } - if (found) - { - if (pass==0) - ++n; - else if (!--c) - break; - - ptr += delimLen; - continue; - } - else - { - // move to the next character - if ((l = my_ismbchar(cs, ptr, srcEnd))) // returns the number of bytes in the leading char or zero if one byte - ptr += l; - else - ++ptr; - } - } - if (pass == 0) /* count<0 */ - { - c += n + 1; - if (c <= 0) - { - return str; // not found, return the original string - } - // Go back and do a second pass - ptr = const_cast(src); - } - else - { - if (c) - { - return str; // not found, return the original string - } + found = false; + break; } + } } - - if ( count > 0) /* return left part */ + if (found) { - std::string ret(src, ptr - src); - return ret; - } - else /* return right part */ - { - ptr+= delimLen; - std::string ret(ptr, srcEnd - ptr); - return ret; - } - } - else - { - if (count > 0) - { - int pointer = 0; - int64_t end = strLen; - for ( int64_t i = 0 ; i < count ; i ++ ) - { - string::size_type pos = str.find(delimstr, pointer); + if (pass == 0) + ++n; + else if (!--c) + break; - if (pos != string::npos) - pointer = pos + 1; - - end = pos; - } - - value = str.substr(0, end); + ptr += delimLen; + continue; } else { - count = -count; - int pointer = strLen; - int start = 0; - - for ( int64_t i = 0 ; i < count ; i ++ ) - { - string::size_type pos = str.rfind(delimstr, pointer); - - if (pos != string::npos) - { - if ( count > strLen ) - return ""; - - pointer = pos - 1; - start = pos + 1; - } - else - start = 0; - } - - value = str.substr(start, strLen); + // move to the next character + if ((l = my_ismbchar( + cs, ptr, srcEnd))) // returns the number of bytes in the leading char or zero if one byte + ptr += l; + else + ++ptr; } + } + if (pass == 0) /* count<0 */ + { + c += n + 1; + if (c <= 0) + { + return str; // not found, return the original string + } + // Go back and do a second pass + ptr = const_cast(src); + } + else + { + if (c) + { + return str; // not found, return the original string + } + } } - return value; + + if (count > 0) /* return left part */ + { + std::string ret(src, ptr - src); + return ret; + } + else /* return right part */ + { + ptr += delimLen; + std::string ret(ptr, srcEnd - ptr); + return ret; + } + } + else + { + if (count > 0) + { + int pointer = 0; + int64_t end = strLen; + for (int64_t i = 0; i < count; i++) + { + string::size_type pos = str.find(delimstr, pointer); + + if (pos != string::npos) + pointer = pos + 1; + + end = pos; + } + + value = str.substr(0, end); + } + else + { + count = -count; + int pointer = strLen; + int start = 0; + + for (int64_t i = 0; i < count; i++) + { + string::size_type pos = str.rfind(delimstr, pointer); + + if (pos != string::npos) + { + if (count > strLen) + return ""; + + pointer = pos - 1; + start = pos + 1; + } + else + start = 0; + } + + value = str.substr(start, strLen); + } + } + return value; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index 0019b3110..b30e66c53 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: Func_sysdate.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: Func_sysdate.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -42,81 +42,67 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_sysdate::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_sysdate::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_sysdate::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_sysdate::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - struct tm tmp_tm; - time_t now; - now = time(NULL); - localtime_r(&now, &tmp_tm); + struct tm tmp_tm; + time_t now; + now = time(NULL); + localtime_r(&now, &tmp_tm); - dataconvert::DateTime aDatetime; - aDatetime.year = (tmp_tm.tm_year + 1900) % 10000; - aDatetime.month = tmp_tm.tm_mon + 1; - aDatetime.day = tmp_tm.tm_mday; - aDatetime.hour = tmp_tm.tm_hour; - aDatetime.minute = tmp_tm.tm_min; - aDatetime.second = tmp_tm.tm_sec; - aDatetime.msecond = 0; - return *(reinterpret_cast(&aDatetime)); + dataconvert::DateTime aDatetime; + aDatetime.year = (tmp_tm.tm_year + 1900) % 10000; + aDatetime.month = tmp_tm.tm_mon + 1; + aDatetime.day = tmp_tm.tm_mday; + aDatetime.hour = tmp_tm.tm_hour; + aDatetime.minute = tmp_tm.tm_min; + aDatetime.second = tmp_tm.tm_sec; + aDatetime.msecond = 0; + return *(reinterpret_cast(&aDatetime)); } -string Func_sysdate::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_sysdate::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - time_t now; - now = time(NULL); - struct tm tm; - localtime_r(&now, &tm); + time_t now; + now = time(NULL); + struct tm tm; + localtime_r(&now, &tm); - char timestamp[80]; - strftime (timestamp, 80, "%Y-%m-%d %H:%M:%S", &tm); - return timestamp; + char timestamp[80]; + strftime(timestamp, 80, "%Y-%m-%d %H:%M:%S", &tm); + return timestamp; } -int32_t Func_sysdate::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int32_t Func_sysdate::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return (((getIntVal(row, parm, isNull, operationColType) >> 32) & 0xFFFFFFC0) | 0x3E); + return (((getIntVal(row, parm, isNull, operationColType) >> 32) & 0xFFFFFFC0) | 0x3E); } -int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return getIntVal(row, parm, isNull, operationColType); + return getIntVal(row, parm, isNull, operationColType); } -int64_t Func_sysdate::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_sysdate::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return getIntVal(row, parm, isNull, operationColType); + return getIntVal(row, parm, isNull, operationColType); } -int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - return getIntVal(row, parm, isNull, operationColType); + return getIntVal(row, parm, isNull, operationColType); } - - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 89d95b021..a859c4843 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_time.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_time.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,146 +35,138 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_time::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_time::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + return ct; } -string Func_time::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - int64_t val = 0; + int64_t val = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); - if (val == -1) - isNull = true; + if (val == -1) + isNull = true; - //else - // return *(reinterpret_cast(&val)); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - isNull = true; - - //else - // return *(reinterpret_cast(&val)); - } - else - { - isNull = true; - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - { - isNull = true; - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - isNull = true; - - //else - // return *(reinterpret_cast(&val)); - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - val = parm[0]->data()->getTimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - val = parm[0]->data()->getTimeIntVal(row, isNull); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - dataconvert::Time time; - time.hour = m_time.hour; - time.minute = m_time.minute; - time.second = m_time.second; - time.is_neg = 0; - time.day = 0; - time.msecond = 0; - val = *(reinterpret_cast(&time)); - break; - } - - default: - { - isNull = true; - } + // else + // return *(reinterpret_cast(&val)); + break; } - if (isNull) - return ""; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); - char buf[30] = {'\0'}; - dataconvert::DataConvert::timeToString(val, buf, sizeof(buf)); - string time(buf); - return time; + if (val == -1) + isNull = true; + + // else + // return *(reinterpret_cast(&val)); + } + else + { + isNull = true; + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + { + isNull = true; + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + + // else + // return *(reinterpret_cast(&val)); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::Time time; + time.hour = m_time.hour; + time.minute = m_time.minute; + time.second = m_time.second; + time.is_neg = 0; + time.day = 0; + time.msecond = 0; + val = *(reinterpret_cast(&time)); + break; + } + + default: + { + isNull = true; + } + } + + if (isNull) + return ""; + + char buf[30] = {'\0'}; + dataconvert::DataConvert::timeToString(val, buf, sizeof(buf)); + string time(buf); + return time; } -int64_t Func_time::getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +int64_t Func_time::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return dataconvert::DataConvert::timeToInt(getStrVal(row, fp, isNull, op_ct)); + return dataconvert::DataConvert::timeToInt(getStrVal(row, fp, isNull, op_ct)); } -double Func_time::getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +double Func_time::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - // convert time value to int followiing mysql. e.g. 23:34:34 => 233434 - int64_t datetimevalue = dataconvert::DataConvert::stringToDatetime(fp[0]->data()->getStrVal(row, isNull)); - return ((unsigned)((datetimevalue >> 32) & 0x3f)) * 10000 + - ((unsigned)((datetimevalue >> 26) & 0x3f)) * 100 + - (unsigned)((datetimevalue >> 20) & 0x3f); + // convert time value to int followiing mysql. e.g. 23:34:34 => 233434 + int64_t datetimevalue = dataconvert::DataConvert::stringToDatetime(fp[0]->data()->getStrVal(row, isNull)); + return ((unsigned)((datetimevalue >> 32) & 0x3f)) * 10000 + + ((unsigned)((datetimevalue >> 26) & 0x3f)) * 100 + (unsigned)((datetimevalue >> 20) & 0x3f); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index c77e46934..5b971b958 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_time_format.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_time_format.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,208 +35,198 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_time_format::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_time_format::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = 255; - return ct; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = 255; + return ct; } -string Func_time_format::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - // assume 256 is enough. assume not allowing incomplete date - char buf[256]; - int64_t val = 0; - uint32_t hour = 0, - min = 0, - sec = 0, - msec = 0; + // assume 256 is enough. assume not allowing incomplete date + char buf[256]; + int64_t val = 0; + uint32_t hour = 0, min = 0, sec = 0, msec = 0; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + isNull = true; + return ""; + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + hour = (uint32_t)((val >> 32) & 0x3f); + min = (uint32_t)((val >> 26) & 0x3f); + sec = (uint32_t)((val >> 20) & 0x3f); + msec = (uint32_t)((val & 0xfffff)); + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - isNull = true; - return ""; - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); - msec = (uint32_t)((val & 0xfffff)); - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - hour = m_time.hour; - min = m_time.minute; - sec = m_time.second; - msec = timestamp.msecond; - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return ""; - } - else - { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); - msec = (uint32_t)((val & 0xfffff)); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return ""; - } - else - { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); - msec = (uint32_t)((val & 0xfffff)); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return ""; - } - else - { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); - msec = (uint32_t)((val & 0xfffff)); - } - } - else - { - isNull = true; - return ""; - } - - break; - - default: - isNull = true; - return ""; + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + hour = m_time.hour; + min = m_time.minute; + sec = m_time.second; + msec = timestamp.msecond; + break; } - const string& format = parm[1]->data()->getStrVal(row, isNull); + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - char* ptr = buf; + if (val == -1) + { + isNull = true; + return ""; + } + else + { + hour = (uint32_t)((val >> 32) & 0x3f); + min = (uint32_t)((val >> 26) & 0x3f); + sec = (uint32_t)((val >> 20) & 0x3f); + msec = (uint32_t)((val & 0xfffff)); + } - for (uint32_t i = 0; i < format.length(); i++) - { - if (format[i] != '%') - *ptr++ = format[i]; + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return ""; + } + else + { + hour = (uint32_t)((val >> 32) & 0x3f); + min = (uint32_t)((val >> 26) & 0x3f); + sec = (uint32_t)((val >> 20) & 0x3f); + msec = (uint32_t)((val & 0xfffff)); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return ""; + } else { - i++; - - switch (format[i]) - { - case 'f': - sprintf(ptr, "%06d", msec); - ptr += 6; - break; - - case 'H': - sprintf(ptr, "%02d", hour); - ptr += 2; - break; - - case 'h': - case 'I': - sprintf(ptr, "%02d", (hour % 24 + 11) % 12 + 1); - ptr += 2; - break; - - case 'i': /* minutes */ - sprintf(ptr, "%02d", min); - ptr += 2; - break; - - case 'k': - sprintf(ptr, "%d", hour); - ptr += (hour >= 10 ? 2 : 1); - break; - - case 'l': - sprintf(ptr, "%d", (hour % 24 + 11) % 12 + 1); - ptr += ((hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1); - break; - - case 'p': - sprintf(ptr, "%s", (hour % 24 < 12 ? "AM" : "PM")); - ptr += 2; - break; - - case 'r': - sprintf(ptr, (hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), - (hour + 11) % 12 + 1, min, sec); - ptr += 11; - break; - - case 'S': - case 's': - sprintf(ptr, "%02d", sec); - ptr += 2; - break; - - case 'T': - sprintf (ptr, "%02d:%02d:%02d", hour, min, sec); - ptr += 8; - break; - - default: - isNull = true; - return ""; - } + hour = (uint32_t)((val >> 32) & 0x3f); + min = (uint32_t)((val >> 26) & 0x3f); + sec = (uint32_t)((val >> 20) & 0x3f); + msec = (uint32_t)((val & 0xfffff)); } - } + } + else + { + isNull = true; + return ""; + } - *ptr = 0; - return string(buf); + break; + + default: isNull = true; return ""; + } + + const string& format = parm[1]->data()->getStrVal(row, isNull); + + char* ptr = buf; + + for (uint32_t i = 0; i < format.length(); i++) + { + if (format[i] != '%') + *ptr++ = format[i]; + else + { + i++; + + switch (format[i]) + { + case 'f': + sprintf(ptr, "%06d", msec); + ptr += 6; + break; + + case 'H': + sprintf(ptr, "%02d", hour); + ptr += 2; + break; + + case 'h': + case 'I': + sprintf(ptr, "%02d", (hour % 24 + 11) % 12 + 1); + ptr += 2; + break; + + case 'i': /* minutes */ + sprintf(ptr, "%02d", min); + ptr += 2; + break; + + case 'k': + sprintf(ptr, "%d", hour); + ptr += (hour >= 10 ? 2 : 1); + break; + + case 'l': + sprintf(ptr, "%d", (hour % 24 + 11) % 12 + 1); + ptr += ((hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1); + break; + + case 'p': + sprintf(ptr, "%s", (hour % 24 < 12 ? "AM" : "PM")); + ptr += 2; + break; + + case 'r': + sprintf(ptr, (hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), (hour + 11) % 12 + 1, + min, sec); + ptr += 11; + break; + + case 'S': + case 's': + sprintf(ptr, "%02d", sec); + ptr += 2; + break; + + case 'T': + sprintf(ptr, "%02d:%02d:%02d", hour, min, sec); + ptr += 8; + break; + + default: isNull = true; return ""; + } + } + } + + *ptr = 0; + return string(buf); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index b8ab898b8..d6b5de7cf 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_time_to_sec.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_time_to_sec.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,172 +36,163 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_time_to_sec::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_time_to_sec::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - // assume 256 is enough. assume not allowing incomplete date - int32_t hour = 0, - min = 0, - sec = 0; - bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' + // assume 256 is enough. assume not allowing incomplete date + int32_t hour = 0, min = 0, sec = 0; + bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' - int64_t val = 0; - int64_t mask = 0; - dataconvert::Time tval; + int64_t val = 0; + int64_t mask = 0; + dataconvert::Time tval; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: return 0; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); + break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - return 0; + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + hour = time.hour; + min = time.minute; + sec = time.second; + break; + } - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - hour = (int32_t)((val >> 32) & 0x3f); - min = (int32_t)((val >> 26) & 0x3f); - sec = (int32_t)((val >> 20) & 0x3f); - break; + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getTimeIntVal(row, isNull); - case CalpontSystemCatalog::TIMESTAMP: + // If negative, mask so it doesn't turn positive + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + bIsNegative = val >> 63; + hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + + if ((hour >= 0) && bIsNegative) + hour *= -1; + else + bIsNegative = false; + + min = (int32_t)((val >> 32) & 0xff); + sec = (int32_t)((val >> 24) & 0xff); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + std::string strVal = parm[0]->data()->getStrVal(row, isNull); + + if (strVal[0] == '-') + { + bIsNegative = true; + strVal.replace(0, 1, 1, ' '); + } + + val = dataconvert::DataConvert::stringToTime(strVal); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + tval = *(reinterpret_cast(&val)); + hour = (uint32_t)(tval.hour); + min = (uint32_t)(tval.minute); + sec = (uint32_t)(tval.second); + } + } + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) { - val = parm[0]->data()->getIntVal(row, isNull); - dataconvert::TimeStamp timestamp(val); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); - hour = time.hour; - min = time.minute; - sec = time.second; - break; + isNull = true; + return -1; } - - case CalpontSystemCatalog::TIME: - val = parm[0]->data()->getTimeIntVal(row, isNull); - - // If negative, mask so it doesn't turn positive - if ((val >> 40) & 0x800) - mask = 0xfffffffffffff000; - - bIsNegative = val >> 63; - hour = (int32_t)(mask | ((val >> 40) & 0xfff)); - - if ((hour >= 0) && bIsNegative) - hour *= -1; - else - bIsNegative = false; - - min = (int32_t)((val >> 32) & 0xff); - sec = (int32_t)((val >> 24) & 0xff); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: + else { - std::string strVal = parm[0]->data()->getStrVal(row, isNull); - - if (strVal[0] == '-') - { - bIsNegative = true; - strVal.replace(0, 1, 1, ' '); - } - - val = dataconvert::DataConvert::stringToTime(strVal); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - tval = *(reinterpret_cast(&val)); - hour = (uint32_t)(tval.hour); - min = (uint32_t)(tval.minute); - sec = (uint32_t)(tval.second); - } + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } - break; + } + else + { + isNull = true; + return -1; + } - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + break; - if (val == -1) - { - isNull = true; - return -1; - } - else - { - hour = (int32_t)((val >> 32) & 0x3f); - min = (int32_t)((val >> 26) & 0x3f); - sec = (int32_t)((val >> 20) & 0x3f); - } + default: isNull = true; return -1; + } - break; + int64_t rtn; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + if (hour < 0) + { + rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; + } + else + { + rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + } - if (val == -1) - { - isNull = true; - return -1; - } - else - { - hour = (int32_t)((val >> 32) & 0x3f); - min = (int32_t)((val >> 26) & 0x3f); - sec = (int32_t)((val >> 20) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } + if (bIsNegative) + { + rtn *= -1; + } - break; - - default: - isNull = true; - return -1; - } - - int64_t rtn; - - if (hour < 0) - { - rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; - } - else - { - rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; - } - - if (bIsNegative) - { - rtn *= -1; - } - - return rtn; + return rtn; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 184200528..2477943ab 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_timediff.cpp 3696 2013-04-05 18:07:21Z dhall $ -* -* -****************************************************************************/ + * $Id: func_timediff.cpp 3696 2013-04-05 18:07:21Z dhall $ + * + * + ****************************************************************************/ #include #include @@ -44,328 +44,314 @@ namespace funcexp { namespace helpers { - -const string timediff( int64_t time1, int64_t time2, bool isDateTime) +const string timediff(int64_t time1, int64_t time2, bool isDateTime) { - long long seconds; - long long microseconds; - int l_sign = 1; + long long seconds; + long long microseconds; + int l_sign = 1; - if ( (time1 < 0 && time2 >= 0) || - (time2 < 0 && time1 >= 0) ) - l_sign = -l_sign; + if ((time1 < 0 && time2 >= 0) || (time2 < 0 && time1 >= 0)) + l_sign = -l_sign; - if ( time1 > time2 ) - helpers::calc_time_diff(time1, time2, l_sign, &seconds, µseconds, isDateTime); - else - helpers::calc_time_diff(time2, time1, l_sign, &seconds, µseconds, isDateTime); + if (time1 > time2) + helpers::calc_time_diff(time1, time2, l_sign, &seconds, µseconds, isDateTime); + else + helpers::calc_time_diff(time2, time1, l_sign, &seconds, µseconds, isDateTime); - long t_seconds; - int hour = seconds / 3600L; - t_seconds = seconds % 3600L; - int minute = t_seconds / 60L; - int second = t_seconds % 60L; + long t_seconds; + int hour = seconds / 3600L; + t_seconds = seconds % 3600L; + int minute = t_seconds / 60L; + int second = t_seconds % 60L; - // Bug 5099: Standardize to mysql behavior. No timediff may be > 838:59:59 - if (hour > 838) - { - hour = 838; - minute = 59; - second = 59; - } + // Bug 5099: Standardize to mysql behavior. No timediff may be > 838:59:59 + if (hour > 838) + { + hour = 838; + minute = 59; + second = 59; + } - int sign = 0; + int sign = 0; - if ( time1 < time2 ) - sign = 1; + if (time1 < time2) + sign = 1; - char buf[256]; - char* ptr = buf; - string time3; + char buf[256]; + char* ptr = buf; + string time3; - if ( microseconds == 0 ) - sprintf (ptr, "%s%02d:%02d:%02d", sign ? "-" : "", hour, minute, second); - else - sprintf (ptr, "%s%02d:%02d:%02d:%06lld", sign ? "-" : "", hour, minute, second, microseconds); + if (microseconds == 0) + sprintf(ptr, "%s%02d:%02d:%02d", sign ? "-" : "", hour, minute, second); + else + sprintf(ptr, "%s%02d:%02d:%02d:%06lld", sign ? "-" : "", hour, minute, second, microseconds); - time3 = ptr; + time3 = ptr; - return time3; -} + return time3; } +} // namespace helpers -bool treatIntAsDatetime(const std::string &text) +bool treatIntAsDatetime(const std::string& text) { - // min used when converting into to datetime is YYYYMMDD - // note: time diffing an int perceived to be in the format YYMMDD is actually treated as HHMMSS. Same functionality as MDB - bool isNeg = text.find("-") == 0; - return (text.length() > 8) || (text.length() >= 8 && !isNeg); + // min used when converting into to datetime is YYYYMMDD + // note: time diffing an int perceived to be in the format YYMMDD is actually treated as HHMMSS. Same + // functionality as MDB + bool isNeg = text.find("-") == 0; + return (text.length() > 8) || (text.length() >= 8 && !isNeg); } -CalpontSystemCatalog::ColType Func_timediff::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_timediff::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -string Func_timediff::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int16_t type1 = parm[0]->data()->resultType().colDataType; - int16_t type2 = parm[1]->data()->resultType().colDataType; + int16_t type1 = parm[0]->data()->resultType().colDataType; + int16_t type2 = parm[1]->data()->resultType().colDataType; - int64_t val1 = -1, val2 = -1; - bool isDate1 = false, isDate2 = false; - bool isTime1 = false, isTime2 = false; - std::string text; + int64_t val1 = -1, val2 = -1; + bool isDate1 = false, isDate2 = false; + bool isTime1 = false, isTime2 = false; + std::string text; - switch (type1) - { - case execplan::CalpontSystemCatalog::DATE: - val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - isDate1 = true; - break; + switch (type1) + { + case execplan::CalpontSystemCatalog::DATE: + val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + isDate1 = true; + break; - case execplan::CalpontSystemCatalog::TIME: - isTime1 = true; - /* fall through */ - case execplan::CalpontSystemCatalog::DATETIME: - // Diff between time and datetime returns NULL in MariaDB - if ((type2 == execplan::CalpontSystemCatalog::TIME || - type2 == execplan::CalpontSystemCatalog::DATETIME) && - type1 != type2) - { - isNull = true; - break; - } - val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - if (type1 != type2) - { - isNull = true; - break; - } - int64_t temp = parm[0]->data()->getTimestampIntVal(row, isNull); - dataconvert::TimeStamp timestamp(temp); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); - dataconvert::DateTime dt; - dt.year = time.year; - dt.month = time.month; - dt.day = time.day; - dt.hour = time.hour; - dt.minute = time.minute; - dt.second = time.second; - dt.msecond = timestamp.msecond; - val1 = (int64_t) *(reinterpret_cast(&dt)); - - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - text = parm[0]->data()->getStrVal(row, isNull); - - if (text.length() >= 12) // datetime has length at least 12 (YYMMDDHHMMSS), convert others to time - { - val1 = dataconvert::DataConvert::stringToDatetime(text, &isDate1); - } - else - { - val1 = dataconvert::DataConvert::stringToTime(text); - isTime1 = true; - } - break; - - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - text = parm[0]->data()->getStrVal(row, isNull); - if (treatIntAsDatetime(text)) - val1 = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull), &isDate1); - else - { - val1 = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); - isTime1 = true; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale != 0) - { - isNull = true; - break; - } - else - { - text = parm[0]->data()->getStrVal(row, isNull); - if (treatIntAsDatetime(text)) - val1 = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull), &isDate1); - else - { - val1 = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); - isTime1 = true; - } - break; - } - - default: - isNull = true; - } - - switch (type2) - { - case execplan::CalpontSystemCatalog::DATE: - val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - isDate2 = true; - break; - - case execplan::CalpontSystemCatalog::TIME: - isTime2 = true; - /* fall through */ - case execplan::CalpontSystemCatalog::DATETIME: - val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int64_t temp = parm[1]->data()->getTimestampIntVal(row, isNull); - dataconvert::TimeStamp timestamp(temp); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); - dataconvert::DateTime dt; - dt.year = time.year; - dt.month = time.month; - dt.day = time.day; - dt.hour = time.hour; - dt.minute = time.minute; - dt.second = time.second; - dt.msecond = timestamp.msecond; - val2 = (int64_t) *(reinterpret_cast(&dt)); - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - text = parm[1]->data()->getStrVal(row, isNull); - if (text.length() >= 12) // datetime has length at least 12 (YYMMDDHHMMSS), convert others to time - { - val2 = dataconvert::DataConvert::stringToDatetime(text, &isDate2); - } - else - { - val2 = dataconvert::DataConvert::stringToTime(text); - isTime2 = true; - } - break; - - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - text = parm[1]->data()->getStrVal(row, isNull); - if (treatIntAsDatetime(text)) - val2 = dataconvert::DataConvert::intToDatetime(parm[1]->data()->getIntVal(row, isNull), &isDate2); - else - { - val2 = dataconvert::DataConvert::intToTime(parm[1]->data()->getIntVal(row, isNull)); - isTime2 = true; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - if (parm[1]->data()->resultType().scale != 0) - { - isNull = true; - break; - } - else - { - text = parm[1]->data()->getStrVal(row, isNull); - if (treatIntAsDatetime(text)) - val2 = dataconvert::DataConvert::intToDatetime(parm[1]->data()->getIntVal(row, isNull), &isDate2); - else - { - val2 = dataconvert::DataConvert::intToTime(parm[1]->data()->getIntVal(row, isNull)); - isTime2 = true; - } - break; - } - - default: - isNull = true; - } - - if (isNull || val1 == -1 || val2 == -1) - { + case execplan::CalpontSystemCatalog::TIME: + isTime1 = true; + /* fall through */ + case execplan::CalpontSystemCatalog::DATETIME: + // Diff between time and datetime returns NULL in MariaDB + if ((type2 == execplan::CalpontSystemCatalog::TIME || + type2 == execplan::CalpontSystemCatalog::DATETIME) && + type1 != type2) + { isNull = true; - return ""; + break; + } + val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if (type1 != type2) + { + isNull = true; + break; + } + int64_t temp = parm[0]->data()->getTimestampIntVal(row, isNull); + dataconvert::TimeStamp timestamp(temp); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::DateTime dt; + dt.year = time.year; + dt.month = time.month; + dt.day = time.day; + dt.hour = time.hour; + dt.minute = time.minute; + dt.second = time.second; + dt.msecond = timestamp.msecond; + val1 = (int64_t) * (reinterpret_cast(&dt)); + + break; } - // both date format or both datetime format. Diff between time and datetime returns NULL in MariaDB - if ((isDate1 && isDate2) || ((!isDate1 && !isDate2) && (isTime1 == isTime2))) - return helpers::timediff( val1, val2, !isTime1); + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + text = parm[0]->data()->getStrVal(row, isNull); + if (text.length() >= 12) // datetime has length at least 12 (YYMMDDHHMMSS), convert others to time + { + val1 = dataconvert::DataConvert::stringToDatetime(text, &isDate1); + } + else + { + val1 = dataconvert::DataConvert::stringToTime(text); + isTime1 = true; + } + break; + + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + text = parm[0]->data()->getStrVal(row, isNull); + if (treatIntAsDatetime(text)) + val1 = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull), &isDate1); + else + { + val1 = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + isTime1 = true; + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale != 0) + { + isNull = true; + break; + } + else + { + text = parm[0]->data()->getStrVal(row, isNull); + if (treatIntAsDatetime(text)) + val1 = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull), &isDate1); + else + { + val1 = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + isTime1 = true; + } + break; + } + + default: isNull = true; + } + + switch (type2) + { + case execplan::CalpontSystemCatalog::DATE: + val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + isDate2 = true; + break; + + case execplan::CalpontSystemCatalog::TIME: + isTime2 = true; + /* fall through */ + case execplan::CalpontSystemCatalog::DATETIME: + val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t temp = parm[1]->data()->getTimestampIntVal(row, isNull); + dataconvert::TimeStamp timestamp(temp); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::DateTime dt; + dt.year = time.year; + dt.month = time.month; + dt.day = time.day; + dt.hour = time.hour; + dt.minute = time.minute; + dt.second = time.second; + dt.msecond = timestamp.msecond; + val2 = (int64_t) * (reinterpret_cast(&dt)); + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + text = parm[1]->data()->getStrVal(row, isNull); + if (text.length() >= 12) // datetime has length at least 12 (YYMMDDHHMMSS), convert others to time + { + val2 = dataconvert::DataConvert::stringToDatetime(text, &isDate2); + } + else + { + val2 = dataconvert::DataConvert::stringToTime(text); + isTime2 = true; + } + break; + + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + text = parm[1]->data()->getStrVal(row, isNull); + if (treatIntAsDatetime(text)) + val2 = dataconvert::DataConvert::intToDatetime(parm[1]->data()->getIntVal(row, isNull), &isDate2); + else + { + val2 = dataconvert::DataConvert::intToTime(parm[1]->data()->getIntVal(row, isNull)); + isTime2 = true; + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + if (parm[1]->data()->resultType().scale != 0) + { + isNull = true; + break; + } + else + { + text = parm[1]->data()->getStrVal(row, isNull); + if (treatIntAsDatetime(text)) + val2 = dataconvert::DataConvert::intToDatetime(parm[1]->data()->getIntVal(row, isNull), &isDate2); + else + { + val2 = dataconvert::DataConvert::intToTime(parm[1]->data()->getIntVal(row, isNull)); + isTime2 = true; + } + break; + } + + default: isNull = true; + } + + if (isNull || val1 == -1 || val2 == -1) + { isNull = true; return ""; + } + + // both date format or both datetime format. Diff between time and datetime returns NULL in MariaDB + if ((isDate1 && isDate2) || ((!isDate1 && !isDate2) && (isTime1 == isTime2))) + return helpers::timediff(val1, val2, !isTime1); + + isNull = true; + return ""; } -int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); + return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } -int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); } -int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); + return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); } -int64_t Func_timediff::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_timediff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - // @bug 3585 - //return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); - return strtoll(getStrVal(row, parm, isNull, ct).c_str(), 0, 10); + // @bug 3585 + // return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); + return strtoll(getStrVal(row, parm, isNull, ct).c_str(), 0, 10); } -double Func_timediff::getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +double Func_timediff::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - // @bug 3585 - //return (double)dataconvert::DataConvert::datetimeToInt(getStrVal(row, fp, isNull, op_ct)); - return atof(getStrVal(row, fp, isNull, op_ct).c_str()); + // @bug 3585 + // return (double)dataconvert::DataConvert::datetimeToInt(getStrVal(row, fp, isNull, op_ct)); + return atof(getStrVal(row, fp, isNull, op_ct).c_str()); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 3972554d3..83024c069 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_timestampdiff.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_timestampdiff.cpp 3921 2013-06-19 18:59:56Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -39,172 +39,161 @@ using namespace dataconvert; namespace funcexp { - -CalpontSystemCatalog::ColType Func_timestampdiff::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_timestampdiff::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val1, val2; - DateTime dt1, dt2; - if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + int64_t val1, val2; + DateTime dt1, dt2; + if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + dt1.year = m_time.year; + dt1.month = m_time.month; + dt1.day = m_time.day; + dt1.hour = m_time.hour; + dt1.minute = m_time.minute; + dt1.second = m_time.second; + dt1.msecond = timestamp.msecond; + val1 = *(reinterpret_cast(&dt1)); + } + else + { + val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + dt1.year = (val1 >> 48) & 0xffff; + dt1.month = (val1 >> 44) & 0xf; + dt1.day = (val1 >> 38) & 0x3f; + dt1.hour = (val1 >> 32) & 0x3f; + dt1.minute = (val1 >> 26) & 0x3f; + dt1.second = (val1 >> 20) & 0x3f; + dt1.msecond = val1 & 0xfffff; + } + + if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, timeZone()); + dt2.year = m_time.year; + dt2.month = m_time.month; + dt2.day = m_time.day; + dt2.hour = m_time.hour; + dt2.minute = m_time.minute; + dt2.second = m_time.second; + dt2.msecond = timestamp.msecond; + val2 = *(reinterpret_cast(&dt2)); + } + else + { + val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + dt2.year = (val2 >> 48) & 0xffff; + dt2.month = (val2 >> 44) & 0xf; + dt2.day = (val2 >> 38) & 0x3f; + dt2.hour = (val2 >> 32) & 0x3f; + dt2.minute = (val2 >> 26) & 0x3f; + dt2.second = (val2 >> 20) & 0x3f; + dt2.msecond = val2 & 0xfffff; + } + + IntervalColumn::interval_type unit = + static_cast(parm[2]->data()->getIntVal(row, isNull)); + + int64_t diff = 0; + + // unit: MICROSECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, or YEAR + int64_t monthdiff = + ((int64_t)dt2.month - (int64_t)dt1.month) + ((int64_t)dt2.year - (int64_t)dt1.year) * 12; + + if (unit == IntervalColumn::INTERVAL_YEAR) + { + diff = monthdiff / 12; + } + else if (unit == IntervalColumn::INTERVAL_MONTH) + { + diff = monthdiff; + + if (dt2.day < dt1.day && monthdiff > 0) + diff = monthdiff - 1; + else if (dt1.day < dt2.day && monthdiff < 0) + diff = monthdiff + 1; + } + else if (unit == IntervalColumn::INTERVAL_QUARTER) + { + diff = monthdiff / 3; + int daydiff = monthdiff % 3; + + if (daydiff == 0) { - TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - dt1.year = m_time.year; - dt1.month = m_time.month; - dt1.day = m_time.day; - dt1.hour = m_time.hour; - dt1.minute = m_time.minute; - dt1.second = m_time.second; - dt1.msecond = timestamp.msecond; - val1 = *(reinterpret_cast(&dt1)); - } - else - { - val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - dt1.year = (val1 >> 48) & 0xffff; - dt1.month = (val1 >> 44) & 0xf; - dt1.day = (val1 >> 38) & 0x3f; - dt1.hour = (val1 >> 32) & 0x3f; - dt1.minute = (val1 >> 26) & 0x3f; - dt1.second = (val1 >> 20) & 0x3f; - dt1.msecond = val1 & 0xfffff; + if (dt2.day < dt1.day && monthdiff > 0) + diff--; + else if (dt1.day < dt2.day && monthdiff < 0) + diff++; } + } + else + { + int64_t seconds = 0, mseconds = 0; + int l_sign = 1; + int l_sign3; - if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - { - TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); - dt2.year = m_time.year; - dt2.month = m_time.month; - dt2.day = m_time.day; - dt2.hour = m_time.hour; - dt2.minute = m_time.minute; - dt2.second = m_time.second; - dt2.msecond = timestamp.msecond; - val2 = *(reinterpret_cast(&dt2)); - } - else - { - val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - dt2.year = (val2 >> 48) & 0xffff; - dt2.month = (val2 >> 44) & 0xf; - dt2.day = (val2 >> 38) & 0x3f; - dt2.hour = (val2 >> 32) & 0x3f; - dt2.minute = (val2 >> 26) & 0x3f; - dt2.second = (val2 >> 20) & 0x3f; - dt2.msecond = val2 & 0xfffff; - } + l_sign3 = helpers::calc_time_diff(val2, val1, l_sign, (long long*)&seconds, (long long*)&mseconds); + l_sign3 = (l_sign3 == 0 ? 1 : -1); - IntervalColumn::interval_type unit = static_cast(parm[2]->data()->getIntVal(row, isNull)); + if (unit == IntervalColumn::INTERVAL_SECOND) + diff = l_sign3 * seconds; + else if (unit == IntervalColumn::INTERVAL_MICROSECOND) + diff = l_sign3 * (seconds * 1000000L + mseconds); + else if (unit == IntervalColumn::INTERVAL_MINUTE) + diff = l_sign3 * (seconds / 60L); + else if (unit == IntervalColumn::INTERVAL_HOUR) + diff = l_sign3 * (seconds / 3600L); + else if (unit == IntervalColumn::INTERVAL_DAY) + diff = l_sign3 * (seconds / (24L * 3600L)); + else if (unit == IntervalColumn::INTERVAL_WEEK) + diff = l_sign3 * (seconds / (24L * 3600L) / 7L); + } - int64_t diff = 0; - - // unit: MICROSECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, or YEAR - int64_t monthdiff = ((int64_t)dt2.month - (int64_t)dt1.month) + - ((int64_t)dt2.year - (int64_t)dt1.year) * 12; - - if (unit == IntervalColumn::INTERVAL_YEAR) - { - diff = monthdiff / 12; - } - else if (unit == IntervalColumn::INTERVAL_MONTH) - { - diff = monthdiff; - - if (dt2.day < dt1.day && monthdiff > 0) - diff = monthdiff - 1; - else if (dt1.day < dt2.day && monthdiff < 0) - diff = monthdiff + 1; - } - else if (unit == IntervalColumn::INTERVAL_QUARTER) - { - diff = monthdiff / 3; - int daydiff = monthdiff % 3; - - if (daydiff == 0) - { - if (dt2.day < dt1.day && monthdiff > 0) - diff --; - else if (dt1.day < dt2.day && monthdiff < 0) - diff ++; - } - } - else - { - int64_t seconds = 0, mseconds = 0; - int l_sign = 1; - int l_sign3; - - l_sign3 = helpers::calc_time_diff(val2, val1, l_sign, (long long*)&seconds, (long long*)&mseconds); - l_sign3 = (l_sign3 == 0 ? 1 : -1); - - if (unit == IntervalColumn::INTERVAL_SECOND) - diff = l_sign3 * seconds; - else if (unit == IntervalColumn::INTERVAL_MICROSECOND) - diff = l_sign3 * (seconds * 1000000L + mseconds); - else if (unit == IntervalColumn::INTERVAL_MINUTE) - diff = l_sign3 * (seconds / 60L); - else if (unit == IntervalColumn::INTERVAL_HOUR) - diff = l_sign3 * (seconds / 3600L); - else if (unit == IntervalColumn::INTERVAL_DAY) - diff = l_sign3 * (seconds / (24L * 3600L)); - else if (unit == IntervalColumn::INTERVAL_WEEK) - diff = l_sign3 * (seconds / (24L * 3600L) / 7L); - } - - return diff; + return diff; } -string Func_timestampdiff::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_timestampdiff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return intToString(getIntVal(row, parm, isNull, ct)); + return intToString(getIntVal(row, parm, isNull, ct)); } -int32_t Func_timestampdiff::getDateIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int32_t Func_timestampdiff::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return getIntVal(row, parm, isNull, ct); + return getIntVal(row, parm, isNull, ct); } -int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return getIntVal(row, parm, isNull, ct); + return getIntVal(row, parm, isNull, ct); } -int64_t Func_timestampdiff::getTimestampIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_timestampdiff::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return getIntVal(row, parm, isNull, ct); + return getIntVal(row, parm, isNull, ct); } -int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) +int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - return getIntVal(row, parm, isNull, ct); + return getIntVal(row, parm, isNull, ct); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index 26a993d1a..79fc165b4 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_to_days.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_to_days.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -39,121 +39,116 @@ using namespace execplan; #include "errorids.h" using namespace logging; - namespace funcexp { - -CalpontSystemCatalog::ColType Func_to_days::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_to_days::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_to_days::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_to_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; + CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; - uint32_t year = 0, - month = 0, - day = 0; + uint32_t year = 0, month = 0, day = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (type) + switch (type) + { + case execplan::CalpontSystemCatalog::DATE: { - case execplan::CalpontSystemCatalog::DATE: - { - int32_t val = parm[0]->data()->getDateIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - return helpers::calc_mysql_daynr(year, month, day); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int64_t val = parm[0]->data()->getDatetimeIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - - return helpers::calc_mysql_daynr(year, month, day); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - year = m_time.year; - month = m_time.month; - day = m_time.day; - return helpers::calc_mysql_daynr(year, month, day); - } - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - { - int64_t val; - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aDateTime.hour = 0; - aDateTime.minute = 0; - aDateTime.second = 0; - aDateTime.msecond = 0; - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - return helpers::calc_mysql_daynr(year, month, day); - break; - } - - case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - const string& value = parm[0]->data()->getStrVal(row, isNull); - int64_t val = 0; - - if ( value.size() == 10 ) - { - // date type - val = dataconvert::DataConvert::dateToInt(value); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - } - else - { - // datetime type - val = dataconvert::DataConvert::datetimeToInt(value); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - return helpers::calc_mysql_daynr(year, month, day); - break; - } - - default: - { - std::ostringstream oss; - oss << "to_days: datatype of " << execplan::colDataTypeToString(type);; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } + int32_t val = parm[0]->data()->getDateIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + return helpers::calc_mysql_daynr(year, month, day); + break; } - return 0; + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t val = parm[0]->data()->getDatetimeIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + + return helpers::calc_mysql_daynr(year, month, day); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + year = m_time.year; + month = m_time.month; + day = m_time.day; + return helpers::calc_mysql_daynr(year, month, day); + } + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + { + int64_t val; + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aDateTime.hour = 0; + aDateTime.minute = 0; + aDateTime.second = 0; + aDateTime.msecond = 0; + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + return helpers::calc_mysql_daynr(year, month, day); + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR' + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& value = parm[0]->data()->getStrVal(row, isNull); + int64_t val = 0; + + if (value.size() == 10) + { + // date type + val = dataconvert::DataConvert::dateToInt(value); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + } + else + { + // datetime type + val = dataconvert::DataConvert::datetimeToInt(value); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + return helpers::calc_mysql_daynr(year, month, day); + break; + } + + default: + { + std::ostringstream oss; + oss << "to_days: datatype of " << execplan::colDataTypeToString(type); + ; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return 0; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_trim.cpp b/utils/funcexp/func_trim.cpp index f8175a4be..636afe44e 100644 --- a/utils/funcexp/func_trim.cpp +++ b/utils/funcexp/func_trim.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_trim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_trim.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,141 +34,138 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { -CalpontSystemCatalog::ColType Func_trim::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_trim::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_trim::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_trim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // The trim characters. - const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); - // binTLen represents the number of bytes in trim - size_t binTLen = trim.length(); - const char* posT = trim.c_str(); - // strTLen = the number of characters in trim - size_t strTLen = cs->numchars(posT, posT+binTLen); - if (strTLen == 0 || strTLen > strLen) - return src; + // The trim characters. + const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); + // binTLen represents the number of bytes in trim + size_t binTLen = trim.length(); + const char* posT = trim.c_str(); + // strTLen = the number of characters in trim + size_t strTLen = cs->numchars(posT, posT + binTLen); + if (strTLen == 0 || strTLen > strLen) + return src; - if (binTLen == 1) + if (binTLen == 1) + { + // If the trim string is 1 byte, don't waste cpu for memcmp + // Trim leading + while (pos < end && *pos == *posT) { - // If the trim string is 1 byte, don't waste cpu for memcmp - // Trim leading - while (pos < end && *pos == *posT) - { - ++pos; - --binLen; - } - // Trim trailing - const char* ptr = pos; - if (cs->use_mb()) // This is a multi-byte charset - { - const char* p = pos; - uint32 l; - // Multibyte characters in the string give us alignment problems - // What we do here is skip past any multibyte characters. Whn - // don with this loop, ptr is pointing to a singlebyte char that - // is after all multibyte chars in the string, or to end. - while (ptr < end) - { - if ((l = my_ismbchar(cs, ptr, end))) // returns the number of bytes in the leading char or zero if one byte - { - ptr += l; - p = ptr; - } - else - { - ++ptr; - } - } - ptr = p; - } - while (ptr < end && end[-1] == *posT) - { - --end; - --binLen; - } + ++pos; + --binLen; } - else + // Trim trailing + const char* ptr = pos; + if (cs->use_mb()) // This is a multi-byte charset { - // Trim leading is easy - while (pos+binTLen <= end && memcmp(pos,posT,binTLen) == 0) + const char* p = pos; + uint32 l; + // Multibyte characters in the string give us alignment problems + // What we do here is skip past any multibyte characters. Whn + // don with this loop, ptr is pointing to a singlebyte char that + // is after all multibyte chars in the string, or to end. + while (ptr < end) + { + if ((l = my_ismbchar(cs, ptr, + end))) // returns the number of bytes in the leading char or zero if one byte { - pos += binTLen; - binLen -= binTLen; - } - - // Trim trailing - if (cs->use_mb()) // This is a multi-byte charset - { - // The problem is that the byte pattern at the end could - // match memcmp, but not be correct since the first byte compared - // may actually be a second or later byte from a previous char. - - // We start at the beginning of the string and move forward - // one character at a time until we reach the end. Then we can - // safely compare and remove one character. Then back to the beginning - // and try again. - while (end - binTLen >= pos) - { - const char* p = pos; - uint32_t l; - while (p + binTLen < end) - { - if ((l = my_ismbchar(cs, p, end))) // returns the number of bytes in the leading char or zero if one byte - p += l; - else - ++p; - } - if (p + binTLen == end && memcmp(p,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } - else - { - break; // We've run out of places to look - } - } + ptr += l; + p = ptr; } else { - while (end-binTLen >= pos && memcmp(end-binTLen,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } + ++ptr; } + } + ptr = p; } - // Turn back to a string - std::string ret(pos, binLen); - return ret; + while (ptr < end && end[-1] == *posT) + { + --end; + --binLen; + } + } + else + { + // Trim leading is easy + while (pos + binTLen <= end && memcmp(pos, posT, binTLen) == 0) + { + pos += binTLen; + binLen -= binTLen; + } + + // Trim trailing + if (cs->use_mb()) // This is a multi-byte charset + { + // The problem is that the byte pattern at the end could + // match memcmp, but not be correct since the first byte compared + // may actually be a second or later byte from a previous char. + + // We start at the beginning of the string and move forward + // one character at a time until we reach the end. Then we can + // safely compare and remove one character. Then back to the beginning + // and try again. + while (end - binTLen >= pos) + { + const char* p = pos; + uint32_t l; + while (p + binTLen < end) + { + if ((l = my_ismbchar(cs, p, + end))) // returns the number of bytes in the leading char or zero if one byte + p += l; + else + ++p; + } + if (p + binTLen == end && memcmp(p, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + else + { + break; // We've run out of places to look + } + } + } + else + { + while (end - binTLen >= pos && memcmp(end - binTLen, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + } + } + // Turn back to a string + std::string ret(pos, binLen); + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_trim_oracle.cpp b/utils/funcexp/func_trim_oracle.cpp index 56f2c1c4b..5e1226c58 100644 --- a/utils/funcexp/func_trim_oracle.cpp +++ b/utils/funcexp/func_trim_oracle.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include using namespace std; @@ -29,145 +28,142 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - namespace funcexp { -CalpontSystemCatalog::ColType Func_trim_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_trim_oracle::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } - -std::string Func_trim_oracle::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& type) +std::string Func_trim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& type) { - CHARSET_INFO* cs = type.getCharset(); - // The original string - const string& src = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; - if (src.empty() || src.length() == 0) - return src; - // binLen represents the number of bytes in src - size_t binLen = src.length(); - const char* pos = src.c_str(); - const char* end = pos + binLen; - // strLen = the number of characters in src - size_t strLen = cs->numchars(pos, end); + CHARSET_INFO* cs = type.getCharset(); + // The original string + const string& src = fp[0]->data()->getStrVal(row, isNull); + if (isNull) + return ""; + if (src.empty() || src.length() == 0) + return src; + // binLen represents the number of bytes in src + size_t binLen = src.length(); + const char* pos = src.c_str(); + const char* end = pos + binLen; + // strLen = the number of characters in src + size_t strLen = cs->numchars(pos, end); - // The trim characters. - const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); - // binTLen represents the number of bytes in trim - size_t binTLen = trim.length(); - const char* posT = trim.c_str(); - // strTLen = the number of characters in trim - size_t strTLen = cs->numchars(posT, posT+binTLen); - if (strTLen == 0 || strTLen > strLen) - return src; + // The trim characters. + const string& trim = (fp.size() > 1 ? fp[1]->data()->getStrVal(row, isNull) : " "); + // binTLen represents the number of bytes in trim + size_t binTLen = trim.length(); + const char* posT = trim.c_str(); + // strTLen = the number of characters in trim + size_t strTLen = cs->numchars(posT, posT + binTLen); + if (strTLen == 0 || strTLen > strLen) + return src; - if (binTLen == 1) + if (binTLen == 1) + { + // If the trim string is 1 byte, don't waste cpu for memcmp + // Trim leading + while (pos < end && *pos == *posT) { - // If the trim string is 1 byte, don't waste cpu for memcmp - // Trim leading - while (pos < end && *pos == *posT) - { - ++pos; - --binLen; - } - // Trim trailing - const char* ptr = pos; - if (cs->use_mb()) // This is a multi-byte charset - { - const char* p = pos; - uint32 l; - // Multibyte characters in the string give us alignment problems - // What we do here is skip past any multibyte characters. Whn - // don with this loop, ptr is pointing to a singlebyte char that - // is after all multibyte chars in the string, or to end. - while (ptr < end) - { - if ((l = my_ismbchar(cs, ptr, end))) // returns the number of bytes in the leading char or zero if one byte - { - ptr += l; - p = ptr; - } - else - { - ++ptr; - } - } - ptr = p; - } - while (ptr < end && end[-1] == *posT) - { - --end; - --binLen; - } + ++pos; + --binLen; } - else + // Trim trailing + const char* ptr = pos; + if (cs->use_mb()) // This is a multi-byte charset { - // Trim leading is easy - while (pos+binTLen <= end && memcmp(pos,posT,binTLen) == 0) + const char* p = pos; + uint32 l; + // Multibyte characters in the string give us alignment problems + // What we do here is skip past any multibyte characters. Whn + // don with this loop, ptr is pointing to a singlebyte char that + // is after all multibyte chars in the string, or to end. + while (ptr < end) + { + if ((l = my_ismbchar(cs, ptr, + end))) // returns the number of bytes in the leading char or zero if one byte { - pos += binTLen; - binLen -= binTLen; - } - - // Trim trailing - if (cs->use_mb()) // This is a multi-byte charset - { - // The problem is that the byte pattern at the end could - // match memcmp, but not be correct since the first byte compared - // may actually be a second or later byte from a previous char. - - // We start at the beginning of the string and move forward - // one character at a time until we reach the end. Then we can - // safely compare and remove one character. Then back to the beginning - // and try again. - while (end - binTLen >= pos) - { - const char* p = pos; - uint32_t l; - while (p + binTLen < end) - { - if ((l = my_ismbchar(cs, p, end))) // returns the number of bytes in the leading char or zero if one byte - p += l; - else - ++p; - } - if (p + binTLen == end && memcmp(p,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } - else - { - break; // We've run out of places to look - } - } + ptr += l; + p = ptr; } else { - while (end-binTLen >= pos && memcmp(end-binTLen,posT,binTLen) == 0) - { - end -= binTLen; - binLen -= binTLen; - } + ++ptr; } + } + ptr = p; } - // Turn back to a string - std::string ret(pos, binLen); - if (binLen == 0) + while (ptr < end && end[-1] == *posT) { - isNull = true; + --end; + --binLen; } - return ret; + } + else + { + // Trim leading is easy + while (pos + binTLen <= end && memcmp(pos, posT, binTLen) == 0) + { + pos += binTLen; + binLen -= binTLen; + } + + // Trim trailing + if (cs->use_mb()) // This is a multi-byte charset + { + // The problem is that the byte pattern at the end could + // match memcmp, but not be correct since the first byte compared + // may actually be a second or later byte from a previous char. + + // We start at the beginning of the string and move forward + // one character at a time until we reach the end. Then we can + // safely compare and remove one character. Then back to the beginning + // and try again. + while (end - binTLen >= pos) + { + const char* p = pos; + uint32_t l; + while (p + binTLen < end) + { + if ((l = my_ismbchar(cs, p, + end))) // returns the number of bytes in the leading char or zero if one byte + p += l; + else + ++p; + } + if (p + binTLen == end && memcmp(p, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + else + { + break; // We've run out of places to look + } + } + } + else + { + while (end - binTLen >= pos && memcmp(end - binTLen, posT, binTLen) == 0) + { + end -= binTLen; + binLen -= binTLen; + } + } + } + // Turn back to a string + std::string ret(pos, binLen); + if (binLen == 0) + { + isNull = true; + } + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 8718fc832..f9b7826d0 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -40,700 +40,679 @@ using namespace dataconvert; namespace { - using namespace funcexp; // P should be double or long double template inline void decimalPlaceDouble(FunctionParm& fp, int64_t& s, P& p, Row& row, bool& isNull) { - s = fp[1]->data()->getIntVal(row, isNull); - int64_t d = s; + s = fp[1]->data()->getIntVal(row, isNull); + int64_t d = s; - if (isNull) - return; + if (isNull) + return; - int64_t i = (d >= 0) ? d : (-d); - int64_t r = 1; + int64_t i = (d >= 0) ? d : (-d); + int64_t r = 1; - while (i--) - r *= 10; + while (i--) + r *= 10; - if (d >= 0) - p = (P) r; - else - p = 1.0 / ((P) r); + if (d >= 0) + p = (P)r; + else + p = 1.0 / ((P)r); } -} +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_truncate::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_truncate::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - if (resultType.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - resultType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + if (resultType.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + resultType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType(); + + switch (ct.colDataType) { - CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType(); + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (resultType.scale > ct.scale) + (resultType).scale = ct.scale; - switch (ct.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (resultType.scale > ct.scale) - (resultType).scale = ct.scale; + break; + } - break; - } - - default: - { - break; - } - } + default: + { + break; + } } + } - return fp[0]->data()->resultType(); + return fp[0]->data()->resultType(); } - // truncate(X, D) // -int64_t Func_truncate::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_truncate::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (!op_ct.isWideDecimalType()) + if (!op_ct.isWideDecimalType()) + { + if (x.scale > 0) { - if (x.scale > 0) - { - while (x.scale-- > 0) - x.value /= 10; - } - else - { - while (x.scale++ < 0) - x.value *= 10; - } - - return x.value; + while (x.scale-- > 0) + x.value /= 10; } else { - return static_cast(x.getIntegralPart()); + while (x.scale++ < 0) + x.value *= 10; } + + return x.value; + } + else + { + return static_cast(x.getIntegralPart()); + } } - -uint64_t Func_truncate::getUintVal(Row& row, - FunctionParm& parm, - bool& isNull, +uint64_t Func_truncate::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t val = parm[0]->data()->getUintVal(row, isNull); - - if (isNull) - return val; - - int64_t d = parm[1]->data()->getIntVal(row, isNull); - - if (isNull || d >= 0) - return val; - - uint64_t p = 1; - int64_t i = (-d); - - // Handle overflow since p can't have more than 19 0's - if (i >= 20) - { - val = 0; - } - else - { - while (i--) - p *= 10; - - val /= p; - val *= p; - } + uint64_t val = parm[0]->data()->getUintVal(row, isNull); + if (isNull) return val; + + int64_t d = parm[1]->data()->getIntVal(row, isNull); + + if (isNull || d >= 0) + return val; + + uint64_t p = 1; + int64_t i = (-d); + + // Handle overflow since p can't have more than 19 0's + if (i >= 20) + { + val = 0; + } + else + { + while (i--) + p *= 10; + + val /= p; + val *= p; + } + + return val; } - -double Func_truncate::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double Func_truncate::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - if (execplan::CalpontSystemCatalog::DOUBLE == op_ct.colDataType || - execplan::CalpontSystemCatalog::FLOAT == op_ct.colDataType) + if (execplan::CalpontSystemCatalog::DOUBLE == op_ct.colDataType || + execplan::CalpontSystemCatalog::FLOAT == op_ct.colDataType) + { + int64_t d = 0; + double p = 1; + decimalPlaceDouble(parm, d, p, row, isNull); + + if (isNull) + return 0.0; + + double x = parm[0]->data()->getDoubleVal(row, isNull); + + if (!isNull) { - int64_t d = 0; - double p = 1; - decimalPlaceDouble(parm, d, p, row, isNull); + x *= p; - if (isNull) - return 0.0; + if (x > 0) + x = floor(x); + else + x = ceil(x); - double x = parm[0]->data()->getDoubleVal(row, isNull); + if (p != 0.0) + x /= p; + else + x = 0.0; + } + + return x; + } + + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + + if (isNull) + return 0.0; + + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = static_cast(x.toTSInt128()); + + if (x.scale > 0) + { + while (x.scale-- > 0) + d /= 10.0; + } + else + { + while (x.scale++ < 0) + d *= 10.0; + } + + return d; +} + +long double Func_truncate::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + if (execplan::CalpontSystemCatalog::LONGDOUBLE == op_ct.colDataType) + { + int64_t d = 0; + long double p = 1; + decimalPlaceDouble(parm, d, p, row, isNull); + + if (isNull) + return 0.0; + + long double x = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (!isNull) + { + x *= p; + + if (x > 0) + x = floor(x); + else + x = ceil(x); + + if (p != 0.0) + x /= p; + else + x = 0.0; + } + + return x; + } + + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + + if (isNull) + return 0.0; + + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = static_cast(x.toTSInt128()); + + if (x.scale > 0) + { + while (x.scale-- > 0) + d /= 10.0; + } + else + { + while (x.scale++ < 0) + d *= 10.0; + } + + return d; +} + +IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + IDB_Decimal decimal; + + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + int64_t d = 0; + decimal = parm[0]->data()->getDecimalVal(row, isNull); + + if (!op_ct.isWideDecimalType()) + { + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int64_t p = 1; if (!isNull) { - x *= p; + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); - if (x > 0) - x = floor(x); - else - x = ceil(x); + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); - if (p != 0.0) - x /= p; - else - x = 0.0; + p = nvp; } - return x; - } - - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - - if (isNull) - return 0.0; - - double d; - - if (!op_ct.isWideDecimalType()) - d = x.value; - else - d = static_cast(x.toTSInt128()); - - if (x.scale > 0) - { - while (x.scale-- > 0) - d /= 10.0; - } - else - { - while (x.scale++ < 0) - d *= 10.0; - } - - return d; -} - -long double Func_truncate::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) -{ - if (execplan::CalpontSystemCatalog::LONGDOUBLE == op_ct.colDataType) - { - int64_t d = 0; - long double p = 1; - decimalPlaceDouble(parm, d, p, row, isNull); - if (isNull) - return 0.0; + break; - long double x = parm[0]->data()->getLongDoubleVal(row, isNull); + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + if ((x >= p) || (x <= -p)) + { + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.value = x; + } + else + { + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int128_t p = 1; if (!isNull) { - x *= p; + int128_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); - if (x > 0) - x = floor(x); - else - x = ceil(x); + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); - if (p != 0.0) - x /= p; - else - x = 0.0; + p = nvp; } - return x; + if (isNull) + break; + + if (d < -datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + if ((x >= p) || (x <= -p)) + { + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; + } } + break; - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - - if (isNull) - return 0.0; - - double d; - - if (!op_ct.isWideDecimalType()) - d = x.value; - else - d = static_cast(x.toTSInt128()); - - if (x.scale > 0) + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: { - while (x.scale-- > 0) - d /= 10.0; - } - else - { - while (x.scale++ < 0) - d *= 10.0; - } + int64_t s = 0; + double p = 1; + decimalPlaceDouble(parm, s, p, row, isNull); - return d; + if (isNull) + break; + + double x = parm[0]->data()->getDoubleVal(row, isNull); + + if (!isNull) + { + x *= p; + decimal.value = (int64_t)x; + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + int64_t s = 0; + long double p = 1; + decimalPlaceDouble(parm, s, p, row, isNull); + + if (isNull) + break; + + long double x = parm[0]->data()->getLongDoubleVal(row, isNull); + + if (!isNull) + { + x *= p; + decimal.value = (int64_t)x; + decimal.scale = s; + } + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + int32_t s = 0; + int64_t x = 0; + + string value = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + x = atoll(value.c_str()); + + if (s > 11) + s = 0; + + if (s > 0) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if (s >= (int32_t)value.size()) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + int32_t s = 0; + int64_t x = 0; + + string value = DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + // strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if (s > 5) + s = 0; + + if (s > 0) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if (s >= (int32_t)value.size()) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int32_t s = 0; + int64_t x = 0; + + string value = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + // strip off micro seconds + value = value.substr(0, 14); + x = atoll(value.c_str()); + + if (s > 5) + s = 0; + + if (s > 0) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if (s >= (int32_t)value.size()) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + int64_t x = 0; + + string value = DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + // strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if (s > 5) + s = 0; + + if (s > 0) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if (s >= (int32_t)value.size()) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; + + default: + { + std::ostringstream oss; + oss << "truncate: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + return decimal; } -IDB_Decimal Func_truncate::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) -{ - IDB_Decimal decimal; - - switch (op_ct.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - int64_t d = 0; - decimal = parm[0]->data()->getDecimalVal(row, isNull); - - if (!op_ct.isWideDecimalType()) - { - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int64_t p = 1; - - if (!isNull) - { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - if ((x >= p) || (x <= -p)) - { - if (p != 0) - x = x / p; - else - x = 0; - } - else - { - x = 0; - } - } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.value = x; - } - else - { - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int128_t p = 1; - - if (!isNull) - { - int128_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - if (d < -datatypes::INT128MAXPRECISION) - { - decimal.s128Value = 0; - break; - } - - int128_t x = decimal.s128Value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - if ((x >= p) || (x <= -p)) - { - if (p != 0) - x = x / p; - else - x = 0; - } - else - { - x = 0; - } - } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.s128Value = x; - } - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - int64_t s = 0; - double p = 1; - decimalPlaceDouble(parm, s, p, row, isNull); - - if (isNull) - break; - - double x = parm[0]->data()->getDoubleVal(row, isNull); - - if (!isNull) - { - x *= p; - decimal.value = (int64_t) x; - decimal.scale = s; - } - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - int64_t s = 0; - long double p = 1; - decimalPlaceDouble(parm, s, p, row, isNull); - - if (isNull) - break; - - long double x = parm[0]->data()->getLongDoubleVal(row, isNull); - - if (!isNull) - { - x *= p; - decimal.value = (int64_t) x; - decimal.scale = s; - } - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int32_t s = 0; - int64_t x = 0; - - string value = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); - - s = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - { - x = atoll(value.c_str()); - - if ( s > 11 ) - s = 0; - - if ( s > 0 ) - { - x *= helpers::powerOf10_c[s]; - } - else if (s < 0) - { - s = -s; - - if ( s >= (int32_t) value.size() ) - { - x = 0; - } - else - { - x /= helpers::powerOf10_c[s]; - x *= helpers::powerOf10_c[s]; - } - - s = 0; - } - } - - decimal.value = x; - decimal.scale = s; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - int32_t s = 0; - int64_t x = 0; - - string value = - DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); - - s = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - { - //strip off micro seconds - value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); - - if ( s > 5 ) - s = 0; - - if ( s > 0 ) - { - x *= helpers::powerOf10_c[s]; - } - else if (s < 0) - { - s = -s; - - if ( s >= (int32_t) value.size() ) - { - x = 0; - } - else - { - x /= helpers::powerOf10_c[s]; - x *= helpers::powerOf10_c[s]; - } - - s = 0; - } - } - - decimal.value = x; - decimal.scale = s; - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int32_t s = 0; - int64_t x = 0; - - string value = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); - - s = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - { - //strip off micro seconds - value = value.substr(0, 14); - x = atoll(value.c_str()); - - if ( s > 5 ) - s = 0; - - if ( s > 0 ) - { - x *= helpers::powerOf10_c[s]; - } - else if (s < 0) - { - s = -s; - - if ( s >= (int32_t) value.size() ) - { - x = 0; - } - else - { - x /= helpers::powerOf10_c[s]; - x *= helpers::powerOf10_c[s]; - } - - s = 0; - } - } - - decimal.value = x; - decimal.scale = s; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - int32_t s = 0; - int64_t x = 0; - - string value = - DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); - - s = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - { - //strip off micro seconds - value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); - - if ( s > 5 ) - s = 0; - - if ( s > 0 ) - { - x *= helpers::powerOf10_c[s]; - } - else if (s < 0) - { - s = -s; - - if ( s >= (int32_t) value.size() ) - { - x = 0; - } - else - { - x /= helpers::powerOf10_c[s]; - x *= helpers::powerOf10_c[s]; - } - - s = 0; - } - } - - decimal.value = x; - decimal.scale = s; - } - break; - - default: - { - std::ostringstream oss; - oss << "truncate: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - } - - return decimal; -} - - -string Func_truncate::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, +string Func_truncate::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - switch (op_ct.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - if (x.scale != 0) - { - if (x.scale > 0 && x.scale < 19) - { - x.value /= IDB_pow[x.scale]; - } - else if (x.scale < 0 && x.scale > -19) - { - x.value *= IDB_pow[-x.scale]; // may overflow - } - else if (x.scale > 0) - { - x.value = 0; - } - else // overflow may need throw exception - { - int64_t e = -x.scale % 18; - x.value *= IDB_pow[e]; - e = -x.scale - e; + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + if (x.scale != 0) + { + if (x.scale > 0 && x.scale < 19) + { + x.value /= IDB_pow[x.scale]; + } + else if (x.scale < 0 && x.scale > -19) + { + x.value *= IDB_pow[-x.scale]; // may overflow + } + else if (x.scale > 0) + { + x.value = 0; + } + else // overflow may need throw exception + { + int64_t e = -x.scale % 18; + x.value *= IDB_pow[e]; + e = -x.scale - e; - while (e > 0) - { - x.value *= IDB_pow[18]; - e -= 18; - } - } + while (e > 0) + { + x.value *= IDB_pow[18]; + e -= 18; + } + } - x.scale = 0; - } + x.scale = 0; + } - break; + break; - default: - break; - } + default: break; + } - if (!op_ct.isWideDecimalType()) - return x.toString(); - else - return x.toString(true); + if (!op_ct.isWideDecimalType()) + return x.toString(); + else + return x.toString(true); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ucase.cpp b/utils/funcexp/func_ucase.cpp index 66157f3b2..4d78fb8b2 100644 --- a/utils/funcexp/func_ucase.cpp +++ b/utils/funcexp/func_ucase.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_ucase.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_ucase.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include using namespace std; @@ -34,47 +34,43 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; - class to_upper { -public: - char operator() (char c) const // notice the return type - { - return toupper(c); - } + public: + char operator()(char c) const // notice the return type + { + return toupper(c); + } }; namespace funcexp { -CalpontSystemCatalog::ColType Func_ucase::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType Func_ucase::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type is not used by this functor - return fp[0]->data()->resultType(); + // operation type is not used by this functor + return fp[0]->data()->resultType(); } -std::string Func_ucase::getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, +std::string Func_ucase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& colType) { - const string& tstr = fp[0]->data()->getStrVal(row, isNull); + const string& tstr = fp[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; + if (isNull) + return ""; - CHARSET_INFO* cs = colType.getCharset(); - uint64_t inLen = tstr.length(); - uint64_t bufLen= inLen * cs->caseup_multiply; - char* outBuf = new char[bufLen]; - - uint64_t outLen = cs->caseup(tstr.c_str(), inLen, outBuf, bufLen); + CHARSET_INFO* cs = colType.getCharset(); + uint64_t inLen = tstr.length(); + uint64_t bufLen = inLen * cs->caseup_multiply; + char* outBuf = new char[bufLen]; - string ret = string(outBuf, outLen); - delete [] outBuf; - return ret; + uint64_t outLen = cs->caseup(tstr.c_str(), inLen, outBuf, bufLen); + + string ret = string(outBuf, outLen); + delete[] outBuf; + return ret; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - diff --git a/utils/funcexp/func_unhex.cpp b/utils/funcexp/func_unhex.cpp index 2019e24ab..38da2cb2d 100644 --- a/utils/funcexp/func_unhex.cpp +++ b/utils/funcexp/func_unhex.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_unhex.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ -* -* -****************************************************************************/ + * $Id: func_unhex.cpp 3923 2013-06-19 21:43:06Z bwilkinson $ + * + * + ****************************************************************************/ #include #include @@ -38,82 +38,78 @@ namespace { inline int hex_to_int(char c, bool& isNull) { - if (c <= '9' && c >= '0') - return c - '0'; + if (c <= '9' && c >= '0') + return c - '0'; - c |= 32; + c |= 32; - if (c <= 'f' && c >= 'a') - return c - 'a' + 10; + if (c <= 'f' && c >= 'a') + return c - 'a' + 10; - isNull = true; - return -1; + isNull = true; + return -1; } -inline string cleanup(char *to) +inline string cleanup(char* to) { - delete[] to; - return ""; -} + delete[] to; + return ""; } +} // namespace namespace funcexp { - -CalpontSystemCatalog::ColType Func_unhex::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_unhex::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -string Func_unhex::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_unhex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - const string& from = parm[0]->data()->getStrVal(row, isNull); + const string& from = parm[0]->data()->getStrVal(row, isNull); - if (isNull) - return ""; + if (isNull) + return ""; - char* to = new char[2 + from.size() / 2]; + char* to = new char[2 + from.size() / 2]; - uint64_t from_pos = 0, to_pos = 0; - int hex_char = 0; + uint64_t from_pos = 0, to_pos = 0; + int hex_char = 0; - if (from.size() % 2) - { - hex_char = hex_to_int(from[from_pos++], isNull); + if (from.size() % 2) + { + hex_char = hex_to_int(from[from_pos++], isNull); - if (hex_char == -1) - return cleanup(to); + if (hex_char == -1) + return cleanup(to); - to[to_pos++] = hex_char; - } + to[to_pos++] = hex_char; + } - for (; from_pos < from.size(); from_pos += 2) - { - hex_char = hex_to_int(from[from_pos], isNull) << 4; + for (; from_pos < from.size(); from_pos += 2) + { + hex_char = hex_to_int(from[from_pos], isNull) << 4; - if (hex_char == -1) - return cleanup(to); + if (hex_char == -1) + return cleanup(to); - to[to_pos] = hex_char; - hex_char = hex_to_int(from[from_pos + 1], isNull); + to[to_pos] = hex_char; + hex_char = hex_to_int(from[from_pos + 1], isNull); - if (hex_char == -1) - return cleanup(to); + if (hex_char == -1) + return cleanup(to); - to[to_pos++] |= hex_char; - } + to[to_pos++] |= hex_char; + } - to[to_pos] = 0; - string tmp = string(to); - delete[] to; + to[to_pos] = 0; + string tmp = string(to); + delete[] to; - return tmp; + return tmp; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_unix_timestamp.cpp b/utils/funcexp/func_unix_timestamp.cpp index 4c43eaebb..d5b65fe25 100644 --- a/utils/funcexp/func_unix_timestamp.cpp +++ b/utils/funcexp/func_unix_timestamp.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_unix_timestamp.cpp 2521 2011-05-02 19:36:52Z zzhu $ -* -* -****************************************************************************/ + * $Id: func_unix_timestamp.cpp 2521 2011-05-02 19:36:52Z zzhu $ + * + * + ****************************************************************************/ #include #include @@ -35,209 +35,192 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_unix_timestamp::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_unix_timestamp::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - int64_t val = parm[0]->data()->getIntVal(row, isNull); + int64_t val = parm[0]->data()->getIntVal(row, isNull); - if (isNull) //no paramter, return current unix_timestamp - { - // get current time in seconds - time_t cal; - time (&cal); - return (int64_t) cal; - } + if (isNull) // no paramter, return current unix_timestamp + { + // get current time in seconds + time_t cal; + time(&cal); + return (int64_t)cal; + } - uint32_t year = 0, - month = 0, - day = 0, - hour = 0, - min = 0, - sec = 0; + uint32_t year = 0, month = 0, day = 0, hour = 0, min = 0, sec = 0; + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; - switch (parm[0]->data()->resultType().colDataType) - { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + hour = (uint32_t)((val >> 32) & 0x3f); + min = (uint32_t)((val >> 26) & 0x3f); + sec = (uint32_t)((val >> 20) & 0x3f); + break; - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); - break; + case CalpontSystemCatalog::TIMESTAMP: + val = parm[0]->data()->getIntVal(row, isNull); + // TimeStamp timeStamp(val); + return ((val >> 20) & 0xFFFFFFFFFFFULL); + break; - case CalpontSystemCatalog::TIMESTAMP: - val = parm[0]->data()->getIntVal(row, isNull); - //TimeStamp timeStamp(val); - return ((val >> 20) & 0xFFFFFFFFFFFULL); - break; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } + break; - break; + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } + break; - break; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } + break; - break; + default: isNull = true; return -1; + } - default: - isNull = true; - return -1; - } + // same algorithm as my_time.c:my_system_gmt_sec + uint32_t loop; + time_t tmp_t = 0; + int shift = 0; + struct tm *l_time, tm_tmp; + int64_t diff, my_time_zone = parm[1]->data()->getIntVal(row, isNull); - // same algorithm as my_time.c:my_system_gmt_sec - uint32_t loop; - time_t tmp_t = 0; - int shift = 0; - struct tm* l_time, tm_tmp; - int64_t diff, my_time_zone = parm[1]->data()->getIntVal(row, isNull); + if ((year == helpers::TIMESTAMP_MAX_YEAR) && (month == 1) && (day > 4)) + { + day -= 2; + shift = 2; + } - if ((year == helpers::TIMESTAMP_MAX_YEAR) && (month == 1) && (day > 4)) - { - day -= 2; - shift = 2; - } + tmp_t = (time_t)(((helpers::calc_mysql_daynr(year, month, day) - 719528) * 86400L + (int64_t)hour * 3600L + + (int64_t)(min * 60 + sec)) - + (time_t)my_time_zone); - tmp_t = (time_t)(((helpers::calc_mysql_daynr(year, month, day) - - 719528) * 86400L + (int64_t)hour * 3600L + - (int64_t)(min * 60 + sec)) - (time_t)my_time_zone); + localtime_r(&tmp_t, &tm_tmp); + l_time = &tm_tmp; + for (loop = 0; loop < 2 && (hour != (uint32_t)l_time->tm_hour || min != (uint32_t)l_time->tm_min || + sec != (uint32_t)l_time->tm_sec); + loop++) + { + int days = day - l_time->tm_mday; + + if (days < -1) + days = 1; /* Month has wrapped */ + else if (days > 1) + days = -1; + + diff = (3600L * (int64_t)(days * 24 + ((int)hour - (int)l_time->tm_hour)) + + (int64_t)(60 * ((int)min - (int)l_time->tm_min)) + (int64_t)((int)sec - (int)l_time->tm_sec)); + tmp_t += (time_t)diff; localtime_r(&tmp_t, &tm_tmp); l_time = &tm_tmp; + } - for (loop = 0; loop < 2 && (hour != (uint32_t) l_time->tm_hour || min != (uint32_t) l_time->tm_min || - sec != (uint32_t)l_time->tm_sec); loop++) - { - int days = day - l_time->tm_mday; + if (loop == 2 && hour != (uint32_t)l_time->tm_hour) + { + int days = day - l_time->tm_mday; - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; + if (days < -1) + days = 1; /* Month has wrapped */ + else if (days > 1) + days = -1; - diff = (3600L * (int64_t) (days * 24 + ((int) hour - (int) l_time->tm_hour)) + - (int64_t) (60 * ((int) min - (int) l_time->tm_min)) + - (int64_t) ((int) sec - (int) l_time->tm_sec)); - tmp_t += (time_t) diff; - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - } + diff = (3600L * (int64_t)(days * 24 + ((int)hour - (int)l_time->tm_hour)) + + (int64_t)(60 * ((int)min - (int)l_time->tm_min)) + (int64_t)((int)sec - (int)l_time->tm_sec)); - if (loop == 2 && hour != (uint32_t)l_time->tm_hour) - { - int days = day - l_time->tm_mday; + if (diff == 3600) + tmp_t += 3600 - min * 60 - sec; /* Move to next hour */ + else if (diff == -3600) + tmp_t -= min * 60 + sec; /* Move to previous hour */ + } - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; + /* shift back, if we were dealing with boundary dates */ + tmp_t += shift * 86400L; - diff = (3600L * (int64_t) (days * 24 + ((int) hour - (int) l_time->tm_hour)) + - (int64_t) (60 * ((int) min - (int) l_time->tm_min)) + - (int64_t) ((int) sec - (int) l_time->tm_sec)); + /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ + if ((tmp_t < helpers::TIMESTAMP_MIN_VALUE) || (tmp_t > helpers::TIMESTAMP_MAX_VALUE)) + tmp_t = 0; - if (diff == 3600) - tmp_t += 3600 - min * 60 - sec; /* Move to next hour */ - else if (diff == -3600) - tmp_t -= min * 60 + sec; /* Move to previous hour */ - } - - - /* shift back, if we were dealing with boundary dates */ - tmp_t += shift * 86400L; - - /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ - if ((tmp_t < helpers::TIMESTAMP_MIN_VALUE) || (tmp_t > helpers::TIMESTAMP_MAX_VALUE)) - tmp_t = 0; - - return (int64_t)tmp_t; + return (int64_t)tmp_t; } - -string Func_unix_timestamp::getStrVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +string Func_unix_timestamp::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct)); - + return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct)); } -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index 874e95b42..f0fe91afc 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_week.cpp 3616 2013-03-04 14:56:29Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_week.cpp 3616 2013-03-04 14:56:29Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,145 +36,137 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_week::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_week::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } -int64_t Func_week::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_week::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint32_t year = 0, - month = 0, - day = 0; + uint32_t year = 0, month = 0, day = 0; - int64_t val = 0; - int16_t mode = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + int64_t val = 0; + int16_t mode = 0; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - if (parm.size() > 1) // mode value - mode = parm[1]->data()->getIntVal(row, isNull); + if (parm.size() > 1) // mode value + mode = parm[1]->data()->getIntVal(row, isNull); - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - year = m_time.year; - month = m_time.month; - day = m_time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; } - int week = helpers::calc_mysql_week(year, month, day, - helpers::convert_mysql_mode_to_modeflags(mode)); + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; - return week; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } + + break; + + default: isNull = true; return -1; + } + + int week = helpers::calc_mysql_week(year, month, day, helpers::convert_mysql_mode_to_modeflags(mode)); + + return week; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index e8b59e620..a6fbbcceb 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_weekday.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_weekday.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,138 +36,132 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_weekday::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_weekday::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_weekday::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint32_t year = 0; - uint32_t month = 0; - uint32_t day = 0; - int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + uint32_t year = 0; + uint32_t month = 0; + uint32_t day = 0; + int64_t val = 0; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - year = m_time.year; - month = m_time.month; - day = m_time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; } - return helpers::calc_mysql_weekday(year, month, day, false); + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } + + break; + + default: isNull = true; return -1; + } + + return helpers::calc_mysql_weekday(year, month, day, false); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 68f60ac20..0b64c2534 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_year.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_year.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -35,115 +35,109 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_year::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_year::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_year::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_year::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val = 0; - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + int64_t val = 0; + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + return (unsigned)((val >> 16) & 0xffff); + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + return (unsigned)((val >> 48) & 0xffff); + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - return (unsigned)((val >> 16) & 0xffff); - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - return (unsigned)((val >> 48) & 0xffff); - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - return m_time.year; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - return (unsigned)((val >> 48) & 0xffff); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 48) & 0xffff); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 48) & 0xffff); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - return (unsigned)((val >> 48) & 0xffff); - } - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + return m_time.year; } - return -1; + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + return (unsigned)((val >> 48) & 0xffff); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 48) & 0xffff); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::DOUBLE: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 48) & 0xffff); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + return (unsigned)((val >> 48) & 0xffff); + } + } + + break; + + default: isNull = true; return -1; + } + + return -1; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 07e111c22..3e59dd120 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: func_yearweek.cpp 2477 2011-04-01 16:07:35Z rdempsey $ -* -* -****************************************************************************/ + * $Id: func_yearweek.cpp 2477 2011-04-01 16:07:35Z rdempsey $ + * + * + ****************************************************************************/ #include #include @@ -36,150 +36,141 @@ using namespace execplan; namespace funcexp { - -CalpontSystemCatalog::ColType Func_yearweek::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +CalpontSystemCatalog::ColType Func_yearweek::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - return resultType; + return resultType; } - -int64_t Func_yearweek::getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, +int64_t Func_yearweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint32_t year = 0, - month = 0, - day = 0; + uint32_t year = 0, month = 0, day = 0; - int64_t val = 0; - int16_t mode = 0; // default to 2 - dataconvert::DateTime aDateTime; - dataconvert::Time aTime; + int64_t val = 0; + int16_t mode = 0; // default to 2 + dataconvert::DateTime aDateTime; + dataconvert::Time aTime; - if (parm.size() > 1) // mode value - mode = parm[1]->data()->getIntVal(row, isNull); + if (parm.size() > 1) // mode value + mode = parm[1]->data()->getIntVal(row, isNull); -//cout << parm.size() << " " << mode << endl; + // cout << parm.size() << " " << mode << endl; - switch (parm[0]->data()->resultType().colDataType) + switch (parm[0]->data()->resultType().colDataType) + { + case CalpontSystemCatalog::DATE: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 16) & 0xffff); + month = (uint32_t)((val >> 12) & 0xf); + day = (uint32_t)((val >> 6) & 0x3f); + break; + + case CalpontSystemCatalog::DATETIME: + val = parm[0]->data()->getIntVal(row, isNull); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: { - case CalpontSystemCatalog::DATE: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 16) & 0xffff); - month = (uint32_t)((val >> 12) & 0xf); - day = (uint32_t)((val >> 6) & 0x3f); - break; - - case CalpontSystemCatalog::DATETIME: - val = parm[0]->data()->getIntVal(row, isNull); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); - int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); - year = m_time.year; - month = m_time.month; - day = m_time.day; - break; - } - - // Time adds to now() and then gets value - case CalpontSystemCatalog::TIME: - aDateTime = static_cast(nowDatetime()); - aTime = parm[0]->data()->getTimeIntVal(row, isNull); - aTime.day = 0; - val = addTime(aDateTime, aTime); - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARCHAR: - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::INT: - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (parm[0]->data()->resultType().scale == 0) - { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); - - if (val == -1) - { - isNull = true; - return -1; - } - else - { - year = (uint32_t)((val >> 48) & 0xffff); - month = (uint32_t)((val >> 44) & 0xf); - day = (uint32_t)((val >> 38) & 0x3f); - } - } - else - { - isNull = true; - return -1; - } - - break; - - default: - isNull = true; - return -1; + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; } - uint32_t lyear = 0; - int week = helpers::calc_mysql_week(year, month, day, - (helpers::convert_mysql_mode_to_modeflags(mode) | helpers::WEEK_NO_ZERO), - &lyear); + // Time adds to now() and then gets value + case CalpontSystemCatalog::TIME: + aDateTime = static_cast(nowDatetime()); + aTime = parm[0]->data()->getTimeIntVal(row, isNull); + aTime.day = 0; + val = addTime(aDateTime, aTime); + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + break; - return (lyear * 100) + week; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::INT: + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + + break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (parm[0]->data()->resultType().scale == 0) + { + val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + { + isNull = true; + return -1; + } + else + { + year = (uint32_t)((val >> 48) & 0xffff); + month = (uint32_t)((val >> 44) & 0xf); + day = (uint32_t)((val >> 38) & 0x3f); + } + } + else + { + isNull = true; + return -1; + } + + break; + + default: isNull = true; return -1; + } + + uint32_t lyear = 0; + int week = helpers::calc_mysql_week( + year, month, day, (helpers::convert_mysql_mode_to_modeflags(mode) | helpers::WEEK_NO_ZERO), &lyear); + + return (lyear * 100) + week; } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 5f5881fb4..703f40ea8 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: funcexp.cpp 3954 2013-07-08 16:30:15Z bpaul $ -* -* -****************************************************************************/ + * $Id: funcexp.cpp 3954 2013-07-08 16:30:15Z bpaul $ + * + * + ****************************************************************************/ #include @@ -46,7 +46,6 @@ using namespace joblist; namespace funcexp { - /* static */ FuncExp* FuncExp::fInstance = 0; @@ -55,468 +54,464 @@ boost::mutex FuncExp::fInstanceMutex; FuncExp* FuncExp::instance() { - boost::mutex::scoped_lock lk(fInstanceMutex); + boost::mutex::scoped_lock lk(fInstanceMutex); - if (!fInstance) - fInstance = new FuncExp(); + if (!fInstance) + fInstance = new FuncExp(); - return fInstance; + return fInstance; } - FuncExp::FuncExp() { - fFuncMap["<<"] = new Func_leftshift(); - fFuncMap[">>"] = new Func_rightshift(); - fFuncMap["|"] = new Func_bitor(); - fFuncMap["^"] = new Func_bitxor(); - fFuncMap["&"] = new Func_bitand(); - fFuncMap["abs"] = new Func_abs(); - fFuncMap["acos"] = new Func_acos(); - fFuncMap["add_time"] = new Func_add_time(); - fFuncMap["addtime"] = new Func_add_time(); - fFuncMap["subtime"] = new Func_add_time(); - fFuncMap["asin"] = new Func_asin(); - fFuncMap["ascii"] = new Func_ascii(); - fFuncMap["atan"] = new Func_atan(); - fFuncMap["atan2"] = new Func_atan(); - fFuncMap["between"] = new Func_between(); - fFuncMap["bit_count"] = new Func_bit_count(); - fFuncMap["case_searched"] = new Func_searched_case(); - fFuncMap["case_simple"] = new Func_simple_case(); - fFuncMap["cast_as_signed"] = new Func_cast_signed(); //dlh - fFuncMap["cast_as_unsigned"] = new Func_cast_unsigned(); //dch - fFuncMap["cast_as_char"] = new Func_cast_char(); //dlh - fFuncMap["cast_as_date"] = new Func_cast_date(); //dlh - fFuncMap["cast_as_datetime"] = new Func_cast_datetime(); //dlh - fFuncMap["decimal_typecast"] = new Func_cast_decimal(); //dlh - fFuncMap["double_typecast"] = new Func_cast_double(); - fFuncMap["ceil"] = new Func_ceil(); //dlh - fFuncMap["ceiling"] = new Func_ceil(); //dlh - fFuncMap["char"] = new Func_char(); //dlh - fFuncMap["char_length"] = new Func_char_length(); //dlh - fFuncMap["character_length"] = new Func_char_length(); //dlh - fFuncMap["coalesce"] = new Func_coalesce(); - fFuncMap["concat_operator_oracle"] = new Func_concat_oracle(); - fFuncMap["concat"] = new Func_concat(); - fFuncMap["concat_ws"] = new Func_concat_ws(); - fFuncMap["conv"] = new Func_conv(); - fFuncMap["cos"] = new Func_cos(); - fFuncMap["cot"] = new Func_cot(); - fFuncMap["convert"] = new Func_cast_char(); //dlh - fFuncMap["convert_tz"] = new Func_convert_tz(); //BT - fFuncMap["crc32"] = new Func_crc32(); - fFuncMap["date_add_interval"] = new Func_date_add(); //dlh - fFuncMap["date_format"] = new Func_date_format(); - fFuncMap["day"] = new Func_day(); //dlh - fFuncMap["dayname"] = new Func_dayname(); - fFuncMap["dayofmonth"] = new Func_day(); //dlh - fFuncMap["dayofweek"] = new Func_dayofweek(); //dlh - fFuncMap["dayofyear"] = new Func_dayofyear(); //dlh - fFuncMap["decode"] = new Func_decode(); // BT - fFuncMap["decode_oracle"] = new Func_decode_oracle(); // BT - fFuncMap["degrees"] = new Func_degrees(); - fFuncMap["DIV"] = new Func_div(); // MySQL use upper case for this function name - fFuncMap["elt"] = new Func_elt(); - fFuncMap["encode"] = new Func_encode(); // BT - fFuncMap["exp"] = new Func_exp(); - fFuncMap["extract"] = new Func_extract(); //dlh - fFuncMap["find_in_set"] = new Func_find_in_set(); - fFuncMap["floor"] = new Func_floor(); //dlh - fFuncMap["format"] = new Func_format(); //dlh - fFuncMap["from_days"] = new Func_from_days(); - fFuncMap["from_unixtime"] = new Func_from_unixtime(); - fFuncMap["get_format"] = new Func_get_format(); //dlh - fFuncMap["greatest"] = new Func_greatest(); //dlh - fFuncMap["hex"] = new Func_hex(); - fFuncMap["hour"] = new Func_hour(); //dlh - fFuncMap["idbpartition"] = new Func_idbpartition(); // pseudo column - fFuncMap["if"] = new Func_if(); - fFuncMap["ifnull"] = new Func_ifnull(); - fFuncMap["in"] = new Func_in(); - fFuncMap[" IN "] = new Func_in(); - fFuncMap["inet_aton"] = new Func_inet_aton(); - fFuncMap["inet_ntoa"] = new Func_inet_ntoa(); - fFuncMap["insert"] = new Func_insert(); - fFuncMap["instr"] = new Func_instr(); - fFuncMap["isnull"] = new Func_isnull(false); - fFuncMap["isnotnull"] = new Func_isnull(true); - fFuncMap["istrue"] = new Func_IsTrue(); - fFuncMap["isnottrue"] = new Func_IsNotTrue(); - fFuncMap["isfalse"] = new Func_IsFalse(); - fFuncMap["isnotfalse"] = new Func_IsNotFalse(); - fFuncMap["last_day"] = new Func_last_day(); - fFuncMap["lcase"] = new Func_lcase(); //dlh - fFuncMap["least"] = new Func_least(); //dlh - fFuncMap["left"] = new Func_left(); //dlh - fFuncMap["length"] = new Func_length(); - fFuncMap["octet_length"] = new Func_length(); // MariaDB 10.3 - fFuncMap["ln"] = new Func_log(); - fFuncMap["locate"] = new Func_instr(); - fFuncMap["log"] = new Func_log(); - fFuncMap["log2"] = new Func_log2(); - fFuncMap["log10"] = new Func_log10(); - fFuncMap["lower"] = new Func_lcase(); //dlh - fFuncMap["lpad"] = new Func_lpad(); //dlh - fFuncMap["ltrim"] = new Func_ltrim(); //dlh - fFuncMap["ltrim_oracle"] = new Func_ltrim_oracle(); //dlh - fFuncMap["makedate"] = new Func_makedate(); - fFuncMap["maketime"] = new Func_maketime(); - fFuncMap["microsecond"] = new Func_microsecond(); - fFuncMap["minute"] = new Func_minute(); //dlh - fFuncMap["mod"] = new Func_mod(); //dlh - fFuncMap["MOD"] = new Func_mod(); // MariaDB 10.3 - fFuncMap["%"] = new Func_mod(); //dlh - fFuncMap["md5"] = new Func_md5(); - fFuncMap["mid"] = new Func_substr(); - fFuncMap["month"] = new Func_month(); //dlh - fFuncMap["monthname"] = new Func_monthname(); - fFuncMap["notin"] = new Func_notin(); - fFuncMap["not IN "] = new Func_notin(); - fFuncMap["notbetween"] = new Func_notbetween(); - fFuncMap["nullif"] = new Func_nullif(); - fFuncMap["period_add"] = new Func_period_add(); //dlh - fFuncMap["period_diff"] = new Func_period_diff(); //dlh - fFuncMap["position"] = new Func_instr(); //dlh - fFuncMap["pow"] = new Func_pow(); - fFuncMap["power"] = new Func_pow(); - fFuncMap["quote"] = new Func_quote(); - fFuncMap["quarter"] = new Func_quarter(); - fFuncMap["radians"] = new Func_radians(); //dlh - fFuncMap["rand"] = new Func_rand(); - fFuncMap["regexp"] = new Func_regexp(); //dlh - fFuncMap["repeat"] = new Func_repeat(); //dlh - fFuncMap["replace"] = new Func_replace(); //dlh - fFuncMap["replace_oracle"] = new Func_replace_oracle(); //dlh - fFuncMap["reverse"] = new Func_reverse(); //dlh - fFuncMap["right"] = new Func_right(); //dlh - fFuncMap["round"] = new Func_round(); - fFuncMap["rpad"] = new Func_rpad(); //dlh - fFuncMap["rtrim"] = new Func_rtrim(); //dlh - fFuncMap["rtrim_oracle"] = new Func_rtrim_oracle(); //dlh - fFuncMap["second"] = new Func_second(); //dlh - fFuncMap["sec_to_time"] = new Func_sec_to_time(); - fFuncMap["sha"] = new Func_sha(); - fFuncMap["sha1"] = new Func_sha(); - fFuncMap["sign"] = new Func_sign(); - fFuncMap["sin"] = new Func_sin(); - fFuncMap["space"] = new Func_space(); - fFuncMap["sqrt"] = new Func_sqrt(); - fFuncMap["str_to_date"] = new Func_str_to_date(); - fFuncMap["strcmp"] = new Func_strcmp(); - fFuncMap["substr"] = new Func_substr(); - fFuncMap["substring"] = new Func_substr(); //dlh - fFuncMap["substring_index"] = new Func_substring_index(); //dlh - fFuncMap["sysdate"] = new Func_sysdate(); //dlhFuncExp - fFuncMap["cast_as_time"] = new Func_time(); //dlh - fFuncMap["tan"] = new Func_tan(); - fFuncMap["timediff"] = new Func_timediff(); //dlh - fFuncMap["timestampdiff"] = new Func_timestampdiff(); - fFuncMap["time_format"] = new Func_time_format(); //dlh - fFuncMap["time_to_sec"] = new Func_time_to_sec(); //dlh - fFuncMap["to_days"] = new Func_to_days(); //dlh - fFuncMap["trim"] = new Func_trim(); //dlh - fFuncMap["trim_oracle"] = new Func_trim_oracle(); //dlh - fFuncMap["truncate"] = new Func_truncate(); //dlh - fFuncMap["ucase"] = new Func_ucase(); //dlh - fFuncMap["unhex"] = new Func_unhex(); - fFuncMap["unix_timestamp"] = new Func_unix_timestamp(); - fFuncMap["upper"] = new Func_ucase(); //dlh - fFuncMap["week"] = new Func_week(); //dlh - fFuncMap["weekday"] = new Func_weekday(); - fFuncMap["weekofyear"] = new Func_week(); //dlh - fFuncMap["year"] = new Func_year(); //dlh - fFuncMap["yearweek"] = new Func_yearweek(); //dlh + fFuncMap["<<"] = new Func_leftshift(); + fFuncMap[">>"] = new Func_rightshift(); + fFuncMap["|"] = new Func_bitor(); + fFuncMap["^"] = new Func_bitxor(); + fFuncMap["&"] = new Func_bitand(); + fFuncMap["abs"] = new Func_abs(); + fFuncMap["acos"] = new Func_acos(); + fFuncMap["add_time"] = new Func_add_time(); + fFuncMap["addtime"] = new Func_add_time(); + fFuncMap["subtime"] = new Func_add_time(); + fFuncMap["asin"] = new Func_asin(); + fFuncMap["ascii"] = new Func_ascii(); + fFuncMap["atan"] = new Func_atan(); + fFuncMap["atan2"] = new Func_atan(); + fFuncMap["between"] = new Func_between(); + fFuncMap["bit_count"] = new Func_bit_count(); + fFuncMap["case_searched"] = new Func_searched_case(); + fFuncMap["case_simple"] = new Func_simple_case(); + fFuncMap["cast_as_signed"] = new Func_cast_signed(); // dlh + fFuncMap["cast_as_unsigned"] = new Func_cast_unsigned(); // dch + fFuncMap["cast_as_char"] = new Func_cast_char(); // dlh + fFuncMap["cast_as_date"] = new Func_cast_date(); // dlh + fFuncMap["cast_as_datetime"] = new Func_cast_datetime(); // dlh + fFuncMap["decimal_typecast"] = new Func_cast_decimal(); // dlh + fFuncMap["double_typecast"] = new Func_cast_double(); + fFuncMap["ceil"] = new Func_ceil(); // dlh + fFuncMap["ceiling"] = new Func_ceil(); // dlh + fFuncMap["char"] = new Func_char(); // dlh + fFuncMap["char_length"] = new Func_char_length(); // dlh + fFuncMap["character_length"] = new Func_char_length(); // dlh + fFuncMap["coalesce"] = new Func_coalesce(); + fFuncMap["concat_operator_oracle"] = new Func_concat_oracle(); + fFuncMap["concat"] = new Func_concat(); + fFuncMap["concat_ws"] = new Func_concat_ws(); + fFuncMap["conv"] = new Func_conv(); + fFuncMap["cos"] = new Func_cos(); + fFuncMap["cot"] = new Func_cot(); + fFuncMap["convert"] = new Func_cast_char(); // dlh + fFuncMap["convert_tz"] = new Func_convert_tz(); // BT + fFuncMap["crc32"] = new Func_crc32(); + fFuncMap["date_add_interval"] = new Func_date_add(); // dlh + fFuncMap["date_format"] = new Func_date_format(); + fFuncMap["day"] = new Func_day(); // dlh + fFuncMap["dayname"] = new Func_dayname(); + fFuncMap["dayofmonth"] = new Func_day(); // dlh + fFuncMap["dayofweek"] = new Func_dayofweek(); // dlh + fFuncMap["dayofyear"] = new Func_dayofyear(); // dlh + fFuncMap["decode"] = new Func_decode(); // BT + fFuncMap["decode_oracle"] = new Func_decode_oracle(); // BT + fFuncMap["degrees"] = new Func_degrees(); + fFuncMap["DIV"] = new Func_div(); // MySQL use upper case for this function name + fFuncMap["elt"] = new Func_elt(); + fFuncMap["encode"] = new Func_encode(); // BT + fFuncMap["exp"] = new Func_exp(); + fFuncMap["extract"] = new Func_extract(); // dlh + fFuncMap["find_in_set"] = new Func_find_in_set(); + fFuncMap["floor"] = new Func_floor(); // dlh + fFuncMap["format"] = new Func_format(); // dlh + fFuncMap["from_days"] = new Func_from_days(); + fFuncMap["from_unixtime"] = new Func_from_unixtime(); + fFuncMap["get_format"] = new Func_get_format(); // dlh + fFuncMap["greatest"] = new Func_greatest(); // dlh + fFuncMap["hex"] = new Func_hex(); + fFuncMap["hour"] = new Func_hour(); // dlh + fFuncMap["idbpartition"] = new Func_idbpartition(); // pseudo column + fFuncMap["if"] = new Func_if(); + fFuncMap["ifnull"] = new Func_ifnull(); + fFuncMap["in"] = new Func_in(); + fFuncMap[" IN "] = new Func_in(); + fFuncMap["inet_aton"] = new Func_inet_aton(); + fFuncMap["inet_ntoa"] = new Func_inet_ntoa(); + fFuncMap["insert"] = new Func_insert(); + fFuncMap["instr"] = new Func_instr(); + fFuncMap["isnull"] = new Func_isnull(false); + fFuncMap["isnotnull"] = new Func_isnull(true); + fFuncMap["istrue"] = new Func_IsTrue(); + fFuncMap["isnottrue"] = new Func_IsNotTrue(); + fFuncMap["isfalse"] = new Func_IsFalse(); + fFuncMap["isnotfalse"] = new Func_IsNotFalse(); + fFuncMap["last_day"] = new Func_last_day(); + fFuncMap["lcase"] = new Func_lcase(); // dlh + fFuncMap["least"] = new Func_least(); // dlh + fFuncMap["left"] = new Func_left(); // dlh + fFuncMap["length"] = new Func_length(); + fFuncMap["octet_length"] = new Func_length(); // MariaDB 10.3 + fFuncMap["ln"] = new Func_log(); + fFuncMap["locate"] = new Func_instr(); + fFuncMap["log"] = new Func_log(); + fFuncMap["log2"] = new Func_log2(); + fFuncMap["log10"] = new Func_log10(); + fFuncMap["lower"] = new Func_lcase(); // dlh + fFuncMap["lpad"] = new Func_lpad(); // dlh + fFuncMap["ltrim"] = new Func_ltrim(); // dlh + fFuncMap["ltrim_oracle"] = new Func_ltrim_oracle(); // dlh + fFuncMap["makedate"] = new Func_makedate(); + fFuncMap["maketime"] = new Func_maketime(); + fFuncMap["microsecond"] = new Func_microsecond(); + fFuncMap["minute"] = new Func_minute(); // dlh + fFuncMap["mod"] = new Func_mod(); // dlh + fFuncMap["MOD"] = new Func_mod(); // MariaDB 10.3 + fFuncMap["%"] = new Func_mod(); // dlh + fFuncMap["md5"] = new Func_md5(); + fFuncMap["mid"] = new Func_substr(); + fFuncMap["month"] = new Func_month(); // dlh + fFuncMap["monthname"] = new Func_monthname(); + fFuncMap["notin"] = new Func_notin(); + fFuncMap["not IN "] = new Func_notin(); + fFuncMap["notbetween"] = new Func_notbetween(); + fFuncMap["nullif"] = new Func_nullif(); + fFuncMap["period_add"] = new Func_period_add(); // dlh + fFuncMap["period_diff"] = new Func_period_diff(); // dlh + fFuncMap["position"] = new Func_instr(); // dlh + fFuncMap["pow"] = new Func_pow(); + fFuncMap["power"] = new Func_pow(); + fFuncMap["quote"] = new Func_quote(); + fFuncMap["quarter"] = new Func_quarter(); + fFuncMap["radians"] = new Func_radians(); // dlh + fFuncMap["rand"] = new Func_rand(); + fFuncMap["regexp"] = new Func_regexp(); // dlh + fFuncMap["repeat"] = new Func_repeat(); // dlh + fFuncMap["replace"] = new Func_replace(); // dlh + fFuncMap["replace_oracle"] = new Func_replace_oracle(); // dlh + fFuncMap["reverse"] = new Func_reverse(); // dlh + fFuncMap["right"] = new Func_right(); // dlh + fFuncMap["round"] = new Func_round(); + fFuncMap["rpad"] = new Func_rpad(); // dlh + fFuncMap["rtrim"] = new Func_rtrim(); // dlh + fFuncMap["rtrim_oracle"] = new Func_rtrim_oracle(); // dlh + fFuncMap["second"] = new Func_second(); // dlh + fFuncMap["sec_to_time"] = new Func_sec_to_time(); + fFuncMap["sha"] = new Func_sha(); + fFuncMap["sha1"] = new Func_sha(); + fFuncMap["sign"] = new Func_sign(); + fFuncMap["sin"] = new Func_sin(); + fFuncMap["space"] = new Func_space(); + fFuncMap["sqrt"] = new Func_sqrt(); + fFuncMap["str_to_date"] = new Func_str_to_date(); + fFuncMap["strcmp"] = new Func_strcmp(); + fFuncMap["substr"] = new Func_substr(); + fFuncMap["substring"] = new Func_substr(); // dlh + fFuncMap["substring_index"] = new Func_substring_index(); // dlh + fFuncMap["sysdate"] = new Func_sysdate(); // dlhFuncExp + fFuncMap["cast_as_time"] = new Func_time(); // dlh + fFuncMap["tan"] = new Func_tan(); + fFuncMap["timediff"] = new Func_timediff(); // dlh + fFuncMap["timestampdiff"] = new Func_timestampdiff(); + fFuncMap["time_format"] = new Func_time_format(); // dlh + fFuncMap["time_to_sec"] = new Func_time_to_sec(); // dlh + fFuncMap["to_days"] = new Func_to_days(); // dlh + fFuncMap["trim"] = new Func_trim(); // dlh + fFuncMap["trim_oracle"] = new Func_trim_oracle(); // dlh + fFuncMap["truncate"] = new Func_truncate(); // dlh + fFuncMap["ucase"] = new Func_ucase(); // dlh + fFuncMap["unhex"] = new Func_unhex(); + fFuncMap["unix_timestamp"] = new Func_unix_timestamp(); + fFuncMap["upper"] = new Func_ucase(); // dlh + fFuncMap["week"] = new Func_week(); // dlh + fFuncMap["weekday"] = new Func_weekday(); + fFuncMap["weekofyear"] = new Func_week(); // dlh + fFuncMap["year"] = new Func_year(); // dlh + fFuncMap["yearweek"] = new Func_yearweek(); // dlh #ifndef SKIP_UDF - udfsdk::UDFSDK sdk; - FuncMap sdkfm = sdk.UDFMap(); - FuncMap::const_iterator iter = sdkfm.begin(); - FuncMap::iterator funcMapIter; + udfsdk::UDFSDK sdk; + FuncMap sdkfm = sdk.UDFMap(); + FuncMap::const_iterator iter = sdkfm.begin(); + FuncMap::iterator funcMapIter; - for (; iter != sdkfm.end(); ++iter) + for (; iter != sdkfm.end(); ++iter) + { + // add sdkfm to fFuncMap + funcMapIter = fFuncMap.find(iter->first); + + if (funcMapIter != fFuncMap.end()) { - //add sdkfm to fFuncMap - funcMapIter = fFuncMap.find(iter->first); - - if (funcMapIter != fFuncMap.end()) - { - // silently ignore the udf function? log it? - } - else - { - fFuncMap[iter->first] = iter->second; - } + // silently ignore the udf function? log it? } + else + { + fFuncMap[iter->first] = iter->second; + } + } #endif } Func* FuncExp::getFunctor(std::string& funcName) { - FuncMap::iterator iter = fFuncMap.find(funcName); + FuncMap::iterator iter = fFuncMap.find(funcName); - if (iter == fFuncMap.end()) - return NULL; - else - return (*iter).second; + if (iter == fFuncMap.end()) + return NULL; + else + return (*iter).second; } void FuncExp::evaluate(rowgroup::Row& row, std::vector& expression) { - bool isNull; + bool isNull; - for (uint32_t i = 0; i < expression.size(); i++) + for (uint32_t i = 0; i < expression.size(); i++) + { + isNull = false; + + switch (expression[i]->resultType().colDataType) { - isNull = false; + case CalpontSystemCatalog::DATE: + { + int64_t val = expression[i]->getIntVal(row, isNull); - switch (expression[i]->resultType().colDataType) + // @bug6061, workaround date_add always return datetime for both date and datetime + if (val & 0xFFFFFFFF00000000) + val = (((val >> 32) & 0xFFFFFFC0) | 0x3E); + + if (isNull) + row.setUintField<4>(DATENULL, expression[i]->outputIndex()); + else + row.setUintField<4>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::DATETIME: + { + int64_t val = expression[i]->getDatetimeIntVal(row, isNull); + + if (isNull) + row.setUintField<8>(DATETIMENULL, expression[i]->outputIndex()); + else + row.setUintField<8>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + int64_t val = expression[i]->getTimestampIntVal(row, isNull); + + if (isNull) + row.setUintField<8>(TIMESTAMPNULL, expression[i]->outputIndex()); + else + row.setUintField<8>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::TIME: + { + int64_t val = expression[i]->getTimeIntVal(row, isNull); + + if (isNull) + row.setIntField<8>(TIMENULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + + // TODO: might not be right thing for BLOB + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + const std::string& val = expression[i]->getStrVal(row, isNull); + + if (isNull) + row.setStringField(CPNULLSTRMARK, expression[i]->outputIndex()); + else + row.setStringField(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::BIGINT: + { + int64_t val = expression[i]->getIntVal(row, isNull); + + if (isNull) + row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::UBIGINT: + { + uint64_t val = expression[i]->getUintVal(row, isNull); + + if (isNull) + row.setUintField<8>(UBIGINTNULL, expression[i]->outputIndex()); + else + row.setUintField<8>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + { + int64_t val = expression[i]->getIntVal(row, isNull); + + if (isNull) + row.setIntField<4>(INTNULL, expression[i]->outputIndex()); + else + row.setIntField<4>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + { + uint64_t val = expression[i]->getUintVal(row, isNull); + + if (isNull) + row.setUintField<4>(UINTNULL, expression[i]->outputIndex()); + else + row.setUintField<4>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::SMALLINT: + { + int64_t val = expression[i]->getIntVal(row, isNull); + + if (isNull) + row.setIntField<2>(SMALLINTNULL, expression[i]->outputIndex()); + else + row.setIntField<2>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::USMALLINT: + { + uint64_t val = expression[i]->getUintVal(row, isNull); + + if (isNull) + row.setUintField<2>(USMALLINTNULL, expression[i]->outputIndex()); + else + row.setUintField<2>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::TINYINT: + { + int64_t val = expression[i]->getIntVal(row, isNull); + + if (isNull) + row.setIntField<1>(TINYINTNULL, expression[i]->outputIndex()); + else + row.setIntField<1>(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::UTINYINT: + { + uint64_t val = expression[i]->getUintVal(row, isNull); + + if (isNull) + row.setUintField<1>(UTINYINTNULL, expression[i]->outputIndex()); + else + row.setUintField<1>(val, expression[i]->outputIndex()); + + break; + } + + // In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double val = expression[i]->getDoubleVal(row, isNull); + + if (isNull) + row.setIntField<8>(DOUBLENULL, expression[i]->outputIndex()); + else + row.setDoubleField(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float val = expression[i]->getFloatVal(row, isNull); + + if (isNull) + row.setIntField<4>(FLOATNULL, expression[i]->outputIndex()); + else + row.setFloatField(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double val = expression[i]->getLongDoubleVal(row, isNull); + + if (isNull) + row.setLongDoubleField(LONGDOUBLENULL, expression[i]->outputIndex()); + else + row.setLongDoubleField(val, expression[i]->outputIndex()); + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); + + if (expression[i]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - case CalpontSystemCatalog::DATE: - { - int64_t val = expression[i]->getIntVal(row, isNull); - - // @bug6061, workaround date_add always return datetime for both date and datetime - if (val & 0xFFFFFFFF00000000) - val = (((val >> 32) & 0xFFFFFFC0) | 0x3E); - - if (isNull) - row.setUintField<4>(DATENULL, expression[i]->outputIndex()); - else - row.setUintField<4>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::DATETIME: - { - int64_t val = expression[i]->getDatetimeIntVal(row, isNull); - - if (isNull) - row.setUintField<8>(DATETIMENULL, expression[i]->outputIndex()); - else - row.setUintField<8>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - int64_t val = expression[i]->getTimestampIntVal(row, isNull); - - if (isNull) - row.setUintField<8>(TIMESTAMPNULL, expression[i]->outputIndex()); - else - row.setUintField<8>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::TIME: - { - int64_t val = expression[i]->getTimeIntVal(row, isNull); - - if (isNull) - row.setIntField<8>(TIMENULL, expression[i]->outputIndex()); - else - row.setIntField<8>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - - // TODO: might not be right thing for BLOB - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - const std::string& val = expression[i]->getStrVal(row, isNull); - - if (isNull) - row.setStringField(CPNULLSTRMARK, expression[i]->outputIndex()); - else - row.setStringField(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - int64_t val = expression[i]->getIntVal(row, isNull); - - if (isNull) - row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); - else - row.setIntField<8>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - uint64_t val = expression[i]->getUintVal(row, isNull); - - if (isNull) - row.setUintField<8>(UBIGINTNULL, expression[i]->outputIndex()); - else - row.setUintField<8>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - { - int64_t val = expression[i]->getIntVal(row, isNull); - - if (isNull) - row.setIntField<4>(INTNULL, expression[i]->outputIndex()); - else - row.setIntField<4>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - uint64_t val = expression[i]->getUintVal(row, isNull); - - if (isNull) - row.setUintField<4>(UINTNULL, expression[i]->outputIndex()); - else - row.setUintField<4>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - int64_t val = expression[i]->getIntVal(row, isNull); - - if (isNull) - row.setIntField<2>(SMALLINTNULL, expression[i]->outputIndex()); - else - row.setIntField<2>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - uint64_t val = expression[i]->getUintVal(row, isNull); - - if (isNull) - row.setUintField<2>(USMALLINTNULL, expression[i]->outputIndex()); - else - row.setUintField<2>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::TINYINT: - { - int64_t val = expression[i]->getIntVal(row, isNull); - - if (isNull) - row.setIntField<1>(TINYINTNULL, expression[i]->outputIndex()); - else - row.setIntField<1>(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - uint64_t val = expression[i]->getUintVal(row, isNull); - - if (isNull) - row.setUintField<1>(UTINYINTNULL, expression[i]->outputIndex()); - else - row.setUintField<1>(val, expression[i]->outputIndex()); - - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double val = expression[i]->getDoubleVal(row, isNull); - - if (isNull) - row.setIntField<8>(DOUBLENULL, expression[i]->outputIndex()); - else - row.setDoubleField(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float val = expression[i]->getFloatVal(row, isNull); - - if (isNull) - row.setIntField<4>(FLOATNULL, expression[i]->outputIndex()); - else - row.setFloatField(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double val = expression[i]->getLongDoubleVal(row, isNull); - - if (isNull) - row.setLongDoubleField(LONGDOUBLENULL, expression[i]->outputIndex()); - else - row.setLongDoubleField(val, expression[i]->outputIndex()); - - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - - if (expression[i]->resultType().colWidth - == datatypes::MAXDECIMALWIDTH) - { - if (isNull) - { - row.setBinaryField_offset( - const_cast(&datatypes::Decimal128Null), - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); - } - else - { - row.setBinaryField_offset(&val.s128Value, - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); - } - } - else - { - if (isNull) - row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); - else - row.setIntField<8>(val.value, expression[i]->outputIndex()); - } - - break; - } - - default: // treat as int64 - { - throw std::runtime_error("funcexp::evaluate(): non support datatype to set field."); - } + if (isNull) + { + row.setBinaryField_offset(const_cast(&datatypes::Decimal128Null), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } + else + { + row.setBinaryField_offset(&val.s128Value, expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } } + else + { + if (isNull) + row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val.value, expression[i]->outputIndex()); + } + + break; + } + + default: // treat as int64 + { + throw std::runtime_error("funcexp::evaluate(): non support datatype to set field."); + } } + } } -} +} // namespace funcexp diff --git a/utils/funcexp/funcexp.h b/utils/funcexp/funcexp.h index 5275632b2..8d39a294a 100644 --- a/utils/funcexp/funcexp.h +++ b/utils/funcexp/funcexp.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: funcexp.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -****************************************************************************/ + * $Id: funcexp.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ****************************************************************************/ /** @file */ #ifndef FUNCEXP_H @@ -42,7 +42,7 @@ namespace execplan { class FunctionColumn; class ArithmeticColumn; -} +} // namespace execplan namespace funcexp { @@ -51,72 +51,70 @@ class Func; typedef std::tr1::unordered_map FuncMap; /** @brief FuncExp is a component for evaluate function and expression filters - */ + */ class FuncExp { -public: + public: + /** Singleton pattern */ + static FuncExp* instance(); - /** Singleton pattern */ - static FuncExp* instance(); + /******************************************************************** + * Row based evaluation APIs + ********************************************************************/ - /******************************************************************** - * Row based evaluation APIs - ********************************************************************/ + /** @brief evaluate a filter stack on row. used for F&E on the where clause + * + * @param row input row that contains all the columns in the filter stack + * @param filters parsetree of filters to evaluate + * @return boolean of whether or not the row passed evaluation + */ + inline bool evaluate(rowgroup::Row& row, execplan::ParseTree* filters); - /** @brief evaluate a filter stack on row. used for F&E on the where clause - * - * @param row input row that contains all the columns in the filter stack - * @param filters parsetree of filters to evaluate - * @return boolean of whether or not the row passed evaluation - */ - inline bool evaluate(rowgroup::Row& row, execplan::ParseTree* filters); + /** @brief evaluate a filter stack on rowgroup + * + * @param row input rowgroup that contains all the columns in the filter stack + * @param filters parse tree of filters to evaluate. The failed rows are removed from the rowgroup + */ + inline void evaluate(rowgroup::RowGroup& rowgroup, execplan::ParseTree* filters); - /** @brief evaluate a filter stack on rowgroup - * - * @param row input rowgroup that contains all the columns in the filter stack - * @param filters parse tree of filters to evaluate. The failed rows are removed from the rowgroup - */ - inline void evaluate(rowgroup::RowGroup& rowgroup, execplan::ParseTree* filters); + /** @brief evaluate a F&E column on row. used for F&E on the select and group by clause + * + * @param row input row that contains all the columns in all the expressions + * @param expressions vector of F&Es that needs evaluation. The results are filled on the row. + */ + void evaluate(rowgroup::Row& row, std::vector& expressions); - /** @brief evaluate a F&E column on row. used for F&E on the select and group by clause - * - * @param row input row that contains all the columns in all the expressions - * @param expressions vector of F&Es that needs evaluation. The results are filled on the row. - */ - void evaluate(rowgroup::Row& row, std::vector& expressions); + /** @brief evaluate a F&E column on rowgroup. used for F&E on the select and group by clause + * + * @param row input rowgroup that contains all the columns in all the expressions + * @param expressions vector of F&Es that needs evaluation. The results are filled on each row. + */ + inline void evaluate(rowgroup::RowGroup& rowgroup, std::vector& expressions); - /** @brief evaluate a F&E column on rowgroup. used for F&E on the select and group by clause - * - * @param row input rowgroup that contains all the columns in all the expressions - * @param expressions vector of F&Es that needs evaluation. The results are filled on each row. - */ - inline void evaluate(rowgroup::RowGroup& rowgroup, std::vector& expressions); + /** @brief get functor from functor map + * + * @param funcName function name + * @return functor pointer. If non-support function, return NULL + */ + Func* getFunctor(std::string& funcName); - /** @brief get functor from functor map - * - * @param funcName function name - * @return functor pointer. If non-support function, return NULL - */ - Func* getFunctor(std::string& funcName); - -private: - static FuncExp* fInstance; - static boost::mutex fInstanceMutex; - FuncMap fFuncMap; - FuncExp(); + private: + static FuncExp* fInstance; + static boost::mutex fInstanceMutex; + FuncMap fFuncMap; + FuncExp(); }; -inline bool FuncExp::evaluate( rowgroup::Row& row, execplan::ParseTree* filters ) +inline bool FuncExp::evaluate(rowgroup::Row& row, execplan::ParseTree* filters) { - bool isNull = false; - return (filters->getBoolVal(row, isNull)); + bool isNull = false; + return (filters->getBoolVal(row, isNull)); } -inline void FuncExp::evaluate( rowgroup::RowGroup& rowgroup, execplan::ParseTree* filters ) +inline void FuncExp::evaluate(rowgroup::RowGroup& rowgroup, execplan::ParseTree* filters) { } -} +} // namespace funcexp #endif - diff --git a/utils/funcexp/funcexpwrapper.cpp b/utils/funcexp/funcexpwrapper.cpp index 8bf1adf4f..9b400f4ca 100644 --- a/utils/funcexp/funcexpwrapper.cpp +++ b/utils/funcexp/funcexpwrapper.cpp @@ -38,100 +38,99 @@ using namespace execplan; namespace funcexp { - FuncExpWrapper::FuncExpWrapper() { - fe = FuncExp::instance(); + fe = FuncExp::instance(); } FuncExpWrapper::FuncExpWrapper(const FuncExpWrapper& f) { - uint32_t i; + uint32_t i; - fe = FuncExp::instance(); + fe = FuncExp::instance(); - filters.resize(f.filters.size()); + filters.resize(f.filters.size()); - for (i = 0; i < f.filters.size(); i++) - filters[i].reset(new ParseTree(*(f.filters[i]))); + for (i = 0; i < f.filters.size(); i++) + filters[i].reset(new ParseTree(*(f.filters[i]))); - rcs.resize(f.rcs.size()); + rcs.resize(f.rcs.size()); - for (i = 0; i < f.rcs.size(); i++) - rcs[i].reset(f.rcs[i]->clone()); + for (i = 0; i < f.rcs.size(); i++) + rcs[i].reset(f.rcs[i]->clone()); } FuncExpWrapper::~FuncExpWrapper() -{ } +{ +} void FuncExpWrapper::operator=(const FuncExpWrapper& f) { - uint32_t i; + uint32_t i; - filters.resize(f.filters.size()); + filters.resize(f.filters.size()); - for (i = 0; i < f.filters.size(); i++) - filters[i].reset(new ParseTree(*(f.filters[i]))); + for (i = 0; i < f.filters.size(); i++) + filters[i].reset(new ParseTree(*(f.filters[i]))); - rcs.resize(f.rcs.size()); - - for (i = 0; i < f.rcs.size(); i++) - rcs[i].reset(f.rcs[i]->clone()); + rcs.resize(f.rcs.size()); + for (i = 0; i < f.rcs.size(); i++) + rcs[i].reset(f.rcs[i]->clone()); } void FuncExpWrapper::serialize(ByteStream& bs) const { - uint32_t i; + uint32_t i; - bs << (uint32_t) filters.size(); - bs << (uint32_t) rcs.size(); + bs << (uint32_t)filters.size(); + bs << (uint32_t)rcs.size(); - for (i = 0; i < filters.size(); i++) - ObjectReader::writeParseTree(filters[i].get(), bs); + for (i = 0; i < filters.size(); i++) + ObjectReader::writeParseTree(filters[i].get(), bs); - for (i = 0; i < rcs.size(); i++) - rcs[i]->serialize(bs); + for (i = 0; i < rcs.size(); i++) + rcs[i]->serialize(bs); } void FuncExpWrapper::deserialize(ByteStream& bs) { - uint32_t fCount, rcsCount, i; + uint32_t fCount, rcsCount, i; - bs >> fCount; - bs >> rcsCount; + bs >> fCount; + bs >> rcsCount; - for (i = 0; i < fCount; i++) - filters.push_back(boost::shared_ptr(ObjectReader::createParseTree(bs))); + for (i = 0; i < fCount; i++) + filters.push_back(boost::shared_ptr(ObjectReader::createParseTree(bs))); - for (i = 0; i < rcsCount; i++) - { - ReturnedColumn* rc = (ReturnedColumn*) ObjectReader::createTreeNode(bs); - rcs.push_back(boost::shared_ptr(rc)); - } + for (i = 0; i < rcsCount; i++) + { + ReturnedColumn* rc = (ReturnedColumn*)ObjectReader::createTreeNode(bs); + rcs.push_back(boost::shared_ptr(rc)); + } } bool FuncExpWrapper::evaluate(Row* r) { - uint32_t i; + uint32_t i; - for (i = 0; i < filters.size(); i++) - if (!fe->evaluate(*r, filters[i].get())) - return false; + for (i = 0; i < filters.size(); i++) + if (!fe->evaluate(*r, filters[i].get())) + return false; - fe->evaluate(*r, rcs); + fe->evaluate(*r, rcs); - return true; + return true; } void FuncExpWrapper::addFilter(const boost::shared_ptr& f) { - filters.push_back(f); + filters.push_back(f); } void FuncExpWrapper::addReturnedColumn(const boost::shared_ptr& rc) { - rcs.push_back(rc); + rcs.push_back(rc); } -}; +}; // namespace funcexp diff --git a/utils/funcexp/funcexpwrapper.h b/utils/funcexp/funcexpwrapper.h index 6a310bfd9..967ee1658 100644 --- a/utils/funcexp/funcexpwrapper.h +++ b/utils/funcexp/funcexpwrapper.h @@ -41,44 +41,43 @@ namespace funcexp { - /** @brief FuncExpWrapper class - */ + */ class FuncExpWrapper : public messageqcpp::Serializeable { -public: - FuncExpWrapper(); - FuncExpWrapper(const FuncExpWrapper&); - virtual ~FuncExpWrapper(); + public: + FuncExpWrapper(); + FuncExpWrapper(const FuncExpWrapper&); + virtual ~FuncExpWrapper(); - void operator=(const FuncExpWrapper&); + void operator=(const FuncExpWrapper&); - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&); + void serialize(messageqcpp::ByteStream&) const; + void deserialize(messageqcpp::ByteStream&); - bool evaluate(rowgroup::Row*); - inline bool evaluateFilter(uint32_t num, rowgroup::Row* r); - inline uint32_t getFilterCount() const; + bool evaluate(rowgroup::Row*); + inline bool evaluateFilter(uint32_t num, rowgroup::Row* r); + inline uint32_t getFilterCount() const; - void addFilter(const boost::shared_ptr&); - void addReturnedColumn(const boost::shared_ptr&); + void addFilter(const boost::shared_ptr&); + void addReturnedColumn(const boost::shared_ptr&); -private: - std::vector > filters; - std::vector > rcs; - FuncExp* fe; + private: + std::vector > filters; + std::vector > rcs; + FuncExp* fe; }; inline bool FuncExpWrapper::evaluateFilter(uint32_t num, rowgroup::Row* r) { - return fe->evaluate(*r, filters[num].get()); + return fe->evaluate(*r, filters[num].get()); } inline uint32_t FuncExpWrapper::getFilterCount() const { - return filters.size(); + return filters.size(); } -} +} // namespace funcexp #endif diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 3317a5f69..1fe7bcf6e 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -48,220 +48,158 @@ namespace funcexp { namespace helpers { - // 10 ** i -const int64_t powerOf10_c[] = -{ - 1ll, - 10ll, - 100ll, - 1000ll, - 10000ll, - 100000ll, - 1000000ll, - 10000000ll, - 100000000ll, - 1000000000ll, - 10000000000ll, - 100000000000ll, - 1000000000000ll, - 10000000000000ll, - 100000000000000ll, - 1000000000000000ll, - 10000000000000000ll, - 100000000000000000ll, - 1000000000000000000ll -}; - +const int64_t powerOf10_c[] = {1ll, + 10ll, + 100ll, + 1000ll, + 10000ll, + 100000ll, + 1000000ll, + 10000000ll, + 100000000ll, + 1000000000ll, + 10000000000ll, + 100000000000ll, + 1000000000000ll, + 10000000000000ll, + 100000000000000ll, + 1000000000000000ll, + 10000000000000000ll, + 100000000000000000ll, + 1000000000000000000ll}; // max integer number of i digits -const int64_t maxNumber_c[] = -{ - 0ll, - 9ll, - 99ll, - 999ll, - 9999ll, - 99999ll, - 999999ll, - 9999999ll, - 99999999ll, - 999999999ll, - 9999999999ll, - 99999999999ll, - 999999999999ll, - 9999999999999ll, - 99999999999999ll, - 999999999999999ll, - 9999999999999999ll, - 99999999999999999ll, - 999999999999999999ll -}; - +const int64_t maxNumber_c[] = {0ll, + 9ll, + 99ll, + 999ll, + 9999ll, + 99999ll, + 999999ll, + 9999999ll, + 99999999ll, + 999999999ll, + 9999999999ll, + 99999999999ll, + 999999999999ll, + 9999999999999ll, + 99999999999999ll, + 999999999999999ll, + 9999999999999999ll, + 99999999999999999ll, + 999999999999999999ll}; const uint32_t TIMESTAMP_MAX_YEAR = 2038; const uint32_t TIMESTAMP_MIN_YEAR = (1970 - 1); const int TIMESTAMP_MIN_VALUE = 1; const int64_t TIMESTAMP_MAX_VALUE = 0x7FFFFFFFL; -const unsigned long long MAX_NEGATIVE_NUMBER = 0x8000000000000000ULL; -const long long LONGLONG_MIN = 0x8000000000000000LL; +const unsigned long long MAX_NEGATIVE_NUMBER = 0x8000000000000000ULL; +const long long LONGLONG_MIN = 0x8000000000000000LL; const int INIT_CNT = 9; -const unsigned long LFACTOR = 1000000000; +const unsigned long LFACTOR = 1000000000; const unsigned long long LFACTOR1 = 10000000000ULL; const unsigned long long LFACTOR2 = 100000000000ULL; -const unsigned long long ulonglong_max = ~(unsigned long long) 0; +const unsigned long long ulonglong_max = ~(unsigned long long)0; -static std::string monthFullNames[13] = -{ - "NON_VALID", - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December" -}; +static std::string monthFullNames[13] = {"NON_VALID", "January", "February", "March", "April", + "May", "June", "July", "August", "September", + "October", "November", "December"}; -static std::string monthAbNames[13] = -{ - "NON_VALID", - "Jan", - "Feb", - "Mar", - "Apr", - "May", - "Jun", - "Jul", - "Aug", - "Sep", - "Oct", - "Nov", - "Dec" -}; +static std::string monthAbNames[13] = {"NON_VALID", "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; -static std::string weekdayFullNames[8] = -{ - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday", - "Saturday", - "Sunday" -}; +static std::string weekdayFullNames[8] = {"Monday", "Tuesday", "Wednesday", "Thursday", + "Friday", "Saturday", "Sunday"}; -static std::string weekdayAbNames[8] = -{ - "Mon", - "Tue", - "Wed", - "Thu", - "Fri", - "Sat", - "Sun" -}; +static std::string weekdayAbNames[8] = {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"}; -static std::string dayOfMonth[32] = -{ - "0th", "1st", "2nd", "3rd", "4th", - "5th", "6th", "7th", "8th", "9th", - "10th", "11th", "12th", "13th", "14th", - "15th", "16th", "17th", "18th", "19th", "20th", - "21st", "22nd", "23rd", "24th", "25th", - "26th", "27th", "28th", "29th", "30th", - "31st" -}; +static std::string dayOfMonth[32] = {"0th", "1st", "2nd", "3rd", "4th", "5th", "6th", "7th", + "8th", "9th", "10th", "11th", "12th", "13th", "14th", "15th", + "16th", "17th", "18th", "19th", "20th", "21st", "22nd", "23rd", + "24th", "25th", "26th", "27th", "28th", "29th", "30th", "31st"}; static uint8_t days_in_month[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; // Given a date, calculate the number of days since year 0 // This is a mirror of calc_daynr, at a later date we should use my_time.h -inline uint32_t calc_mysql_daynr( uint32_t year, uint32_t month, uint32_t day ) +inline uint32_t calc_mysql_daynr(uint32_t year, uint32_t month, uint32_t day) { - int temp; - int y = year; - long delsum; + int temp; + int y = year; + long delsum; - if ( !dataconvert::isDateValid( day, month, year ) ) - return 0; + if (!dataconvert::isDateValid(day, month, year)) + return 0; - delsum = (long) (365 * y + 31 * ((int) month - 1) + (int) day); + delsum = (long)(365 * y + 31 * ((int)month - 1) + (int)day); - if (month <= 2) - y--; - else - delsum -= (long) ((int) month * 4 + 23) / 10; + if (month <= 2) + y--; + else + delsum -= (long)((int)month * 4 + 23) / 10; - temp = (int) ((y / 100 + 1) * 3) / 4; + temp = (int)((y / 100 + 1) * 3) / 4; - return delsum + (int) y / 4 - temp; + return delsum + (int)y / 4 - temp; } // used by get_date_from_mysql_daynr() and calc_mysql_week() inline uint32_t calc_mysql_days_in_year(uint32_t year) { - return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? - 366 : 365); + return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? 366 : 365); } // convert from a MySQL day number (offset from year 0) to a date // This is a mirror of get_date_from_daynr, at a later date we should use sql_time.h inline void get_date_from_mysql_daynr(long daynr, dataconvert::DateTime& dateTime) { - uint32_t year, temp, leap_day, day_of_year, days_in_year; - uint8_t* month_pos; - uint32_t ret_year, ret_month, ret_day; - const int MAX_DAY_NUMBER = 3652424; + uint32_t year, temp, leap_day, day_of_year, days_in_year; + uint8_t* month_pos; + uint32_t ret_year, ret_month, ret_day; + const int MAX_DAY_NUMBER = 3652424; + if (daynr < 366 || daynr > MAX_DAY_NUMBER) + { + dateTime.year = dateTime.month = dateTime.day = 0; + return; + } - if (daynr < 366 || daynr > MAX_DAY_NUMBER) + year = (uint32_t)(daynr * 100 / 36525L); + temp = (((year - 1) / 100 + 1) * 3) / 4; + day_of_year = (uint)(daynr - (long)year * 365L) - (year - 1) / 4 + temp; + + while (day_of_year > (days_in_year = calc_mysql_days_in_year(year))) + { + day_of_year -= days_in_year; + (year)++; + } + + leap_day = 0; + + if (days_in_year == 366) + { + if (day_of_year > 31 + 28) { - dateTime.year = dateTime.month = dateTime.day = 0; - return; + day_of_year--; + + if (day_of_year == 31 + 28) + leap_day = 1; /* Handle leapyears leapday */ } + } - year = (uint32_t) (daynr * 100 / 36525L); - temp = (((year - 1) / 100 + 1) * 3) / 4; - day_of_year = (uint) (daynr - (long) year * 365L) - (year - 1) / 4 + temp; + ret_month = 1; - while (day_of_year > (days_in_year = calc_mysql_days_in_year(year))) - { - day_of_year -= days_in_year; - (year)++; - } + for (month_pos = days_in_month; day_of_year > (uint32_t)*month_pos; + day_of_year -= *(month_pos++), (ret_month)++) + ; - leap_day = 0; + ret_year = year; + ret_day = day_of_year + leap_day; - if (days_in_year == 366) - { - if (day_of_year > 31 + 28) - { - day_of_year--; - - if (day_of_year == 31 + 28) - leap_day = 1; /* Handle leapyears leapday */ - } - } - - ret_month = 1; - - for (month_pos = days_in_month ; - day_of_year > (uint32_t) *month_pos ; - day_of_year -= *(month_pos++), (ret_month)++) - ; - - ret_year = year; - ret_day = day_of_year + leap_day; - - dateTime.year = ret_year; - dateTime.month = ret_month; - dateTime.day = ret_day; + dateTime.year = ret_year; + dateTime.month = ret_month; + dateTime.day = ret_day; } // Returns the weekday index for a given date: @@ -270,29 +208,28 @@ inline void get_date_from_mysql_daynr(long daynr, dataconvert::DateTime& dateTim // else: // 0 = Monday, 1 = Tuesday, ..., 6 = Sunday // This is a mirror of calc_weekday, at a later date we should use sql_time.h -inline uint32_t calc_mysql_weekday( uint32_t year, uint32_t month, uint32_t day, bool sundayFirst ) +inline uint32_t calc_mysql_weekday(uint32_t year, uint32_t month, uint32_t day, bool sundayFirst) { - if ( !dataconvert::isDateValid( day, month, year ) ) - return 0; - - uint32_t daynr = calc_mysql_daynr(year, month, day); - return ((int) ((daynr + 5L + (sundayFirst ? 1L : 0L)) % 7)); + if (!dataconvert::isDateValid(day, month, year)) + return 0; + uint32_t daynr = calc_mysql_daynr(year, month, day); + return ((int)((daynr + 5L + (sundayFirst ? 1L : 0L)) % 7)); } // Flags for calc_mysql_week -const uint32_t WEEK_MONDAY_FIRST = 1; -const uint32_t WEEK_NO_ZERO = 2; +const uint32_t WEEK_MONDAY_FIRST = 1; +const uint32_t WEEK_NO_ZERO = 2; const uint32_t WEEK_GT_THREE_DAYS = 4; // Takes a MySQL WEEK() function mode setting and converts to a bitmask // used by calc_mysql_week() mirror of MariaDB's week_mode() -inline int16_t convert_mysql_mode_to_modeflags( int16_t mode ) +inline int16_t convert_mysql_mode_to_modeflags(int16_t mode) { - if (!(mode & WEEK_MONDAY_FIRST)) - mode ^= WEEK_GT_THREE_DAYS; + if (!(mode & WEEK_MONDAY_FIRST)) + mode ^= WEEK_GT_THREE_DAYS; - return mode; + return mode; } // Returns a week index conforming to the MySQL WEEK() function. Note @@ -305,492 +242,464 @@ inline int16_t convert_mysql_mode_to_modeflags( int16_t mode ) // than the year of the input date // // This is a mirror of calc_week, at a later date we should use sql_time.h -inline uint32_t calc_mysql_week( uint32_t year, uint32_t month, uint32_t day, - int16_t modeflags, - uint32_t* weekyear = 0 ) +inline uint32_t calc_mysql_week(uint32_t year, uint32_t month, uint32_t day, int16_t modeflags, + uint32_t* weekyear = 0) { - // need to make sure that the date is valid - if ( !dataconvert::isDateValid( day, month, year ) ) - return 0; + // need to make sure that the date is valid + if (!dataconvert::isDateValid(day, month, year)) + return 0; + uint32_t days; + uint32_t daynr = calc_mysql_daynr(year, month, day); + uint32_t first_daynr = calc_mysql_daynr(year, 1, 1); + bool monday_first = modeflags & WEEK_MONDAY_FIRST; + bool week_year = modeflags & WEEK_NO_ZERO; + bool first_weekday = modeflags & WEEK_GT_THREE_DAYS; - uint32_t days; - uint32_t daynr = calc_mysql_daynr(year, month, day); - uint32_t first_daynr = calc_mysql_daynr(year, 1, 1); - bool monday_first = modeflags & WEEK_MONDAY_FIRST; - bool week_year = modeflags & WEEK_NO_ZERO; - bool first_weekday = modeflags & WEEK_GT_THREE_DAYS; + uint32_t weekday = calc_mysql_weekday(year, 1, 1, !monday_first); - uint32_t weekday = calc_mysql_weekday(year, 1, 1, !monday_first); + if (weekyear) + { + *weekyear = year; + } + + if (month == 1 && day <= 7 - weekday) + { + if (!week_year && ((first_weekday && weekday != 0) || (!first_weekday && weekday >= 4))) + return 0; + + week_year = 1; if (weekyear) { - *weekyear = year; + (*weekyear)--; } - if (month == 1 && day <= 7 - weekday) + year--; + first_daynr -= (days = calc_mysql_days_in_year(year)); + weekday = (weekday + 53 * 7 - days) % 7; + } + + if ((first_weekday && weekday != 0) || (!first_weekday && weekday >= 4)) + days = daynr - (first_daynr + (7 - weekday)); + else + days = daynr - (first_daynr - weekday); + + if (week_year && days >= 52 * 7) + { + weekday = (weekday + calc_mysql_days_in_year(year)) % 7; + + if ((!first_weekday && weekday < 4) || (first_weekday && weekday == 0)) { - if (!week_year && - ((first_weekday && weekday != 0) || - (!first_weekday && weekday >= 4))) - return 0; + if (weekyear) + { + (*weekyear)++; + } - week_year = 1; - - if (weekyear) - { - (*weekyear)--; - } - - year--; - first_daynr -= (days = calc_mysql_days_in_year(year)); - weekday = (weekday + 53 * 7 - days) % 7; + return 1; } + } - if ((first_weekday && weekday != 0) || - (!first_weekday && weekday >= 4)) - days = daynr - (first_daynr + (7 - weekday)); - else - days = daynr - (first_daynr - weekday); - - if (week_year && days >= 52 * 7) - { - weekday = (weekday + calc_mysql_days_in_year(year)) % 7; - - if ((!first_weekday && weekday < 4) || - (first_weekday && weekday == 0)) - { - if (weekyear) - { - (*weekyear)++; - } - - return 1; - } - } - - return days / 7 + 1; + return days / 7 + 1; } -inline bool calc_time_diff(int64_t time1, int64_t time2, int l_sign, long long* seconds_out, long long* microseconds_out, bool isDateTime = true) +inline bool calc_time_diff(int64_t time1, int64_t time2, int l_sign, long long* seconds_out, + long long* microseconds_out, bool isDateTime = true) { - int64_t days; - bool neg; - int64_t microseconds; + int64_t days; + bool neg; + int64_t microseconds; - uint64_t year1 = 0, - month1 = 0, - day1 = 0, - hour1 = 0, - min1 = 0, - sec1 = 0, - msec1 = 0; + uint64_t year1 = 0, month1 = 0, day1 = 0, hour1 = 0, min1 = 0, sec1 = 0, msec1 = 0; - uint64_t year2 = 0, - month2 = 0, - day2 = 0, - hour2 = 0, - min2 = 0, - sec2 = 0, - msec2 = 0; + uint64_t year2 = 0, month2 = 0, day2 = 0, hour2 = 0, min2 = 0, sec2 = 0, msec2 = 0; - if (isDateTime) - { - year1 = (uint32_t)((time1 >> 48) & 0xffff); - month1 = (uint32_t)((time1 >> 44) & 0xf); - day1 = (uint32_t)((time1 >> 38) & 0x3f); - hour1 = (uint32_t)((time1 >> 32) & 0x3f); - min1 = (uint32_t)((time1 >> 26) & 0x3f); - sec1 = (uint32_t)((time1 >> 20) & 0x3f); - msec1 = (uint32_t)((time1 & 0xfffff)); + if (isDateTime) + { + year1 = (uint32_t)((time1 >> 48) & 0xffff); + month1 = (uint32_t)((time1 >> 44) & 0xf); + day1 = (uint32_t)((time1 >> 38) & 0x3f); + hour1 = (uint32_t)((time1 >> 32) & 0x3f); + min1 = (uint32_t)((time1 >> 26) & 0x3f); + sec1 = (uint32_t)((time1 >> 20) & 0x3f); + msec1 = (uint32_t)((time1 & 0xfffff)); - year2 = (uint32_t)((time2 >> 48) & 0xffff); - month2 = (uint32_t)((time2 >> 44) & 0xf); - day2 = (uint32_t)((time2 >> 38) & 0x3f); - hour2 = (uint32_t)((time2 >> 32) & 0x3f); - min2 = (uint32_t)((time2 >> 26) & 0x3f); - sec2 = (uint32_t)((time2 >> 20) & 0x3f); - msec2 = (uint32_t)(time2 & 0xfffff); - } - else - { - year1 = 0; - month1 = 0; - day1 = (time1 >> 52) & 0x7ff; - hour1 = (time1 >> 40) & 0xfff; - min1 = (time1 >> 32) & 0xff; - sec1 = (time1 >> 24) & 0xff; - msec1 = time1 & 0xffffff; + year2 = (uint32_t)((time2 >> 48) & 0xffff); + month2 = (uint32_t)((time2 >> 44) & 0xf); + day2 = (uint32_t)((time2 >> 38) & 0x3f); + hour2 = (uint32_t)((time2 >> 32) & 0x3f); + min2 = (uint32_t)((time2 >> 26) & 0x3f); + sec2 = (uint32_t)((time2 >> 20) & 0x3f); + msec2 = (uint32_t)(time2 & 0xfffff); + } + else + { + year1 = 0; + month1 = 0; + day1 = (time1 >> 52) & 0x7ff; + hour1 = (time1 >> 40) & 0xfff; + min1 = (time1 >> 32) & 0xff; + sec1 = (time1 >> 24) & 0xff; + msec1 = time1 & 0xffffff; - year2 = 0; - month2 = 0; - day2 = (time2 >> 52) & 0x7ff; - hour2 = (time2 >> 40) & 0xfff; - min2 = (time2 >> 32) & 0xff; - sec2 = (time2 >> 24) & 0xff; - msec2 = time2 & 0xffffff; - } + year2 = 0; + month2 = 0; + day2 = (time2 >> 52) & 0x7ff; + hour2 = (time2 >> 40) & 0xfff; + min2 = (time2 >> 32) & 0xff; + sec2 = (time2 >> 24) & 0xff; + msec2 = time2 & 0xffffff; + } - days = calc_mysql_daynr(year1, month1, day1); + days = calc_mysql_daynr(year1, month1, day1); - days -= l_sign * calc_mysql_daynr(year2, month2, day2); + days -= l_sign * calc_mysql_daynr(year2, month2, day2); - microseconds = ((long long)days * (long)(86400) + - (long long)(hour1 * 3600L + - min1 * 60L + - sec1) - - l_sign * (long long)(hour2 * 3600L + - min2 * 60L + - sec2)) * (long long)(1000000) + - (long long)msec1 - - l_sign * (long long)msec2; + microseconds = ((long long)days * (long)(86400) + (long long)(hour1 * 3600L + min1 * 60L + sec1) - + l_sign * (long long)(hour2 * 3600L + min2 * 60L + sec2)) * + (long long)(1000000) + + (long long)msec1 - l_sign * (long long)msec2; - neg = 0; + neg = 0; - if (microseconds < 0) - { - microseconds = -microseconds; - neg = 1; - } + if (microseconds < 0) + { + microseconds = -microseconds; + neg = 1; + } - *seconds_out = microseconds / 1000000L; - *microseconds_out = (long long) (microseconds % 1000000L); - return neg; + *seconds_out = microseconds / 1000000L; + *microseconds_out = (long long)(microseconds % 1000000L); + return neg; } -inline int power ( int16_t a ) +inline int power(int16_t a) { - int b = 1; + int b = 1; - for ( int i = 0 ; i < a ; i++ ) - { - b = b * 10; - } + for (int i = 0; i < a; i++) + { + b = b * 10; + } - return b; + return b; } template inline void decimalPlaceDec(int64_t& d, T& p, int8_t& s) { - // find new scale if D < s - if (d < s) - { - int64_t t = s; - s = d; // the new scale - d -= t; - int64_t i = (d >= 0) ? d : (-d); + // find new scale if D < s + if (d < s) + { + int64_t t = s; + s = d; // the new scale + d -= t; + int64_t i = (d >= 0) ? d : (-d); - while (i--) - p *= 10; - } - else - { - d = s; - } + while (i--) + p *= 10; + } + else + { + d = s; + } } -inline uint32_t convertMonth (std::string month) +inline uint32_t convertMonth(std::string month) { - uint32_t value = 0; + uint32_t value = 0; - boost::algorithm::to_lower(month); + boost::algorithm::to_lower(month); - if ( month == "jan" || month == "january" ) - { - value = 1; - } - else if ( month == "feb" || month == "february" ) - { - value = 2; - } - else if ( month == "mar" || month == "march" ) - { - value = 3; - } - else if ( month == "apr" || month == "april" ) - { - value = 4; - } - else if ( month == "may" ) - { - value = 5; - } - else if ( month == "jun" || month == "june" ) - { - value = 6; - } - else if ( month == "jul" || month == "july" ) - { - value = 7; - } - else if ( month == "aug" || month == "august" ) - { - value = 8; - } - else if ( month == "sep" || month == "september" ) - { - value = 9; - } - else if ( month == "oct" || month == "october" ) - { - value = 10; - } - else if ( month == "nov" || month == "november" ) - { - value = 11; - } - else if ( month == "dec" || month == "december" ) - { - value = 12; - } - else - { - value = 0; - } + if (month == "jan" || month == "january") + { + value = 1; + } + else if (month == "feb" || month == "february") + { + value = 2; + } + else if (month == "mar" || month == "march") + { + value = 3; + } + else if (month == "apr" || month == "april") + { + value = 4; + } + else if (month == "may") + { + value = 5; + } + else if (month == "jun" || month == "june") + { + value = 6; + } + else if (month == "jul" || month == "july") + { + value = 7; + } + else if (month == "aug" || month == "august") + { + value = 8; + } + else if (month == "sep" || month == "september") + { + value = 9; + } + else if (month == "oct" || month == "october") + { + value = 10; + } + else if (month == "nov" || month == "november") + { + value = 11; + } + else if (month == "dec" || month == "december") + { + value = 12; + } + else + { + value = 0; + } - return value; + return value; } class to_lower { -public: - char operator() (char c) const // notice the return type - { - return tolower(c); - } + public: + char operator()(char c) const // notice the return type + { + return tolower(c); + } }; - -inline int getNumbers( const std::string& expr, int64_t* array, execplan::OpType funcType) +inline int getNumbers(const std::string& expr, int64_t* array, execplan::OpType funcType) { - int index = 0; + int index = 0; - int funcNeg = 1; + int funcNeg = 1; - if ( funcType == execplan::OP_SUB ) - funcNeg = -1; + if (funcType == execplan::OP_SUB) + funcNeg = -1; - if ( expr.size() == 0 ) - return 0; + if (expr.size() == 0) + return 0; - // @bug 4703 reworked this code to avoid use of incrementally - // built string to hold temporary values while - // scanning expr for numbers. This function is now - // covered by a unit test in tdriver.cpp - bool foundNumber = false; - int64_t number = 0; - int neg = 1; + // @bug 4703 reworked this code to avoid use of incrementally + // built string to hold temporary values while + // scanning expr for numbers. This function is now + // covered by a unit test in tdriver.cpp + bool foundNumber = false; + int64_t number = 0; + int neg = 1; - for ( unsigned int i = 0 ; i < expr.size() ; i++ ) + for (unsigned int i = 0; i < expr.size(); i++) + { + char value = expr[i]; + + if ((value >= '0' && value <= '9')) { - char value = expr[i]; - - if ( (value >= '0' && value <= '9') ) - { - foundNumber = true; - number = ( number * 10 ) + ( value - '0' ); - } - else if ( value == '-' && !foundNumber ) - { - neg = -1; - } - else if ( value == '-') - { - // this is actually an error condition - it means that - // input came in with something like NN-NN (i.e. a dash - // between two numbers. To match prior code we will - // return the number up to the dash and just return - array[index] = number * funcNeg * neg; - index++; - - return index; - } - else - { - if ( foundNumber ) - { - array[index] = number * funcNeg * neg; - number = 0; - neg = 1; - index++; - - if ( index > 9 ) - return index; - } - } + foundNumber = true; + number = (number * 10) + (value - '0'); } - - if ( foundNumber ) + else if (value == '-' && !foundNumber) { - array[index] = number * funcNeg * neg; - index++; + neg = -1; } - - return index; -} - - -inline int getNumbers( const std::string& expr, int* array, execplan::OpType funcType) -{ - int index = 0; - - int funcNeg = 1; - - if ( funcType == execplan::OP_SUB ) - funcNeg = -1; - - if ( expr.size() == 0 ) - return 0; - - // @bug 4703 reworked this code to avoid use of incrementally - // built string to hold temporary values while - // scanning expr for numbers. This function is now - // covered by a unit test in tdriver.cpp - bool foundNumber = false; - int number = 0; - int neg = 1; - - for ( unsigned int i = 0 ; i < expr.size() ; i++ ) + else if (value == '-') { - char value = expr[i]; + // this is actually an error condition - it means that + // input came in with something like NN-NN (i.e. a dash + // between two numbers. To match prior code we will + // return the number up to the dash and just return + array[index] = number * funcNeg * neg; + index++; - if ( (value >= '0' && value <= '9') ) - { - foundNumber = true; - number = ( number * 10 ) + ( value - '0' ); - } - else if ( value == '-' && !foundNumber ) - { - neg = -1; - } - else if ( value == '-') - { - // this is actually an error condition - it means that - // input came in with something like NN-NN (i.e. a dash - // between two numbers. To match prior code we will - // return the number up to the dash and just return - array[index] = number * funcNeg * neg; - index++; - - return index; - } - else - { - if ( foundNumber ) - { - array[index] = number * funcNeg * neg; - number = 0; - neg = 1; - index++; - - if ( index > 9 ) - return index; - } - } - } - - if ( foundNumber ) - { - array[index] = number * funcNeg * neg; - index++; - } - - return index; -} - -inline int dayOfWeek(std::string day) //Sunday = 0 -{ - int value = -1; - boost::to_lower(day); - - if ( day == "sunday" || day == "sun" ) - { - value = 0; - } - else if ( day == "monday" || day == "mon" ) - { - value = 1; - } - else if ( day == "tuesday" || day == "tue" ) - { - value = 2; - } - else if ( day == "wednesday" || day == "wed" ) - { - value = 3; - } - else if ( day == "thursday" || day == "thu" ) - { - value = 4; - } - else if ( day == "friday" || day == "fri" ) - { - value = 5; - } - else if ( day == "saturday" || day == "sat" ) - { - value = 6; + return index; } else { - value = -1; + if (foundNumber) + { + array[index] = number * funcNeg * neg; + number = 0; + neg = 1; + index++; + + if (index > 9) + return index; + } } + } - return value; + if (foundNumber) + { + array[index] = number * funcNeg * neg; + index++; + } + return index; } -inline -string intToString(int64_t i) +inline int getNumbers(const std::string& expr, int* array, execplan::OpType funcType) { - char buf[32]; + int index = 0; + + int funcNeg = 1; + + if (funcType == execplan::OP_SUB) + funcNeg = -1; + + if (expr.size() == 0) + return 0; + + // @bug 4703 reworked this code to avoid use of incrementally + // built string to hold temporary values while + // scanning expr for numbers. This function is now + // covered by a unit test in tdriver.cpp + bool foundNumber = false; + int number = 0; + int neg = 1; + + for (unsigned int i = 0; i < expr.size(); i++) + { + char value = expr[i]; + + if ((value >= '0' && value <= '9')) + { + foundNumber = true; + number = (number * 10) + (value - '0'); + } + else if (value == '-' && !foundNumber) + { + neg = -1; + } + else if (value == '-') + { + // this is actually an error condition - it means that + // input came in with something like NN-NN (i.e. a dash + // between two numbers. To match prior code we will + // return the number up to the dash and just return + array[index] = number * funcNeg * neg; + index++; + + return index; + } + else + { + if (foundNumber) + { + array[index] = number * funcNeg * neg; + number = 0; + neg = 1; + index++; + + if (index > 9) + return index; + } + } + } + + if (foundNumber) + { + array[index] = number * funcNeg * neg; + index++; + } + + return index; +} + +inline int dayOfWeek(std::string day) // Sunday = 0 +{ + int value = -1; + boost::to_lower(day); + + if (day == "sunday" || day == "sun") + { + value = 0; + } + else if (day == "monday" || day == "mon") + { + value = 1; + } + else if (day == "tuesday" || day == "tue") + { + value = 2; + } + else if (day == "wednesday" || day == "wed") + { + value = 3; + } + else if (day == "thursday" || day == "thu") + { + value = 4; + } + else if (day == "friday" || day == "fri") + { + value = 5; + } + else if (day == "saturday" || day == "sat") + { + value = 6; + } + else + { + value = -1; + } + + return value; +} + +inline string intToString(int64_t i) +{ + char buf[32]; #ifndef _MSC_VER - snprintf(buf, 32, "%" PRId64 "", i); + snprintf(buf, 32, "%" PRId64 "", i); #else - snprintf(buf, 32, "%lld", i); + snprintf(buf, 32, "%lld", i); #endif - return buf; + return buf; } -inline -string uintToString(uint64_t i) +inline string uintToString(uint64_t i) { - char buf[32]; + char buf[32]; #ifndef _MSC_VER - snprintf(buf, 32, "%" PRIu64 "", i); + snprintf(buf, 32, "%" PRIu64 "", i); #else - snprintf(buf, 32, "%llu", i); + snprintf(buf, 32, "%llu", i); #endif - return buf; + return buf; } -inline -string doubleToString(double d) +inline string doubleToString(double d) { - // double's can be *really* long to print out. Max mysql - // is e308 so allow for 308 + 36 decimal places minimum. - char buf[384]; - snprintf(buf, 384, "%f", d); - return buf; + // double's can be *really* long to print out. Max mysql + // is e308 so allow for 308 + 36 decimal places minimum. + char buf[384]; + snprintf(buf, 384, "%f", d); + return buf; } -inline -string longDoubleToString(long double ld) +inline string longDoubleToString(long double ld) { - // long double's can be *really* long to print out. Max mysql - // is e308 so allow for 308 + 36 decimal places minimum. - char buf[384]; - snprintf(buf, 384, "%Lf", ld); - return buf; + // long double's can be *really* long to print out. Max mysql + // is e308 so allow for 308 + 36 decimal places minimum. + char buf[384]; + snprintf(buf, 384, "%Lf", ld); + return buf; } -uint64_t dateAdd( uint64_t time, const std::string& expr, execplan::IntervalColumn::interval_type unit, bool dateType, execplan::OpType funcType ); +uint64_t dateAdd(uint64_t time, const std::string& expr, execplan::IntervalColumn::interval_type unit, + bool dateType, execplan::OpType funcType); const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&); const std::string timediff(int64_t, int64_t, bool isDateTime = true); const char* convNumToStr(int64_t, char*, int); -} //namespace funcexp::helpers -} //namespace funcexp +} // namespace helpers +} // namespace funcexp #endif diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 2696eaf1c..ce83903cb 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /**************************************************************************** -* $Id: functor.cpp 3898 2013-06-17 20:41:05Z rdempsey $ -* -* -****************************************************************************/ + * $Id: functor.cpp 3898 2013-06-17 20:41:05Z rdempsey $ + * + * + ****************************************************************************/ #ifndef _MSC_VER #ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS @@ -50,328 +50,315 @@ using namespace funcexp; namespace funcexp { - void Func::init() { - uint32_t fni = joblist::FLOATNULL; - float* fp = reinterpret_cast(&fni); - fFloatNullVal = *fp; + uint32_t fni = joblist::FLOATNULL; + float* fp = reinterpret_cast(&fni); + fFloatNullVal = *fp; - uint64_t dni = joblist::DOUBLENULL; - double* dp = reinterpret_cast(&dni); - fDoubleNullVal = *dp; + uint64_t dni = joblist::DOUBLENULL; + double* dp = reinterpret_cast(&dni); + fDoubleNullVal = *dp; - fLongDoubleNullVal = joblist::LONGDOUBLENULL; + fLongDoubleNullVal = joblist::LONGDOUBLENULL; } - Func::Func() { - init(); + init(); } - Func::Func(const string& funcName) : fFuncName(funcName) { - init(); + init(); } - uint32_t Func::stringToDate(const string str) { - int64_t ret = DataConvert::stringToDate(str); + int64_t ret = DataConvert::stringToDate(str); - if (ret == -1) - { - Message::Args args; - args.add("date"); - args.add(str); - unsigned errcode = ERR_INCORRECT_VALUE; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } + if (ret == -1) + { + Message::Args args; + args.add("date"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - return ret; + return ret; } - uint64_t Func::stringToDatetime(const string str) { - int64_t ret = DataConvert::stringToDatetime(str); + int64_t ret = DataConvert::stringToDatetime(str); - if (ret == -1) - { - Message::Args args; - args.add("datetime"); - args.add(str); - unsigned errcode = ERR_INCORRECT_VALUE; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } + if (ret == -1) + { + Message::Args args; + args.add("datetime"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - return ret; + return ret; } uint64_t Func::stringToTimestamp(const string str) { - int64_t ret = DataConvert::stringToTimestamp(str, timeZone()); + int64_t ret = DataConvert::stringToTimestamp(str, timeZone()); - if (ret == -1) - { - Message::Args args; - args.add("timestamp"); - args.add(str); - unsigned errcode = ERR_INCORRECT_VALUE; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } + if (ret == -1) + { + Message::Args args; + args.add("timestamp"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - return ret; + return ret; } int64_t Func::stringToTime(const string str) { - int64_t ret = DataConvert::stringToTime(str); + int64_t ret = DataConvert::stringToTime(str); - if (ret == -1) - { - Message::Args args; - args.add("time"); - args.add(str); - unsigned errcode = ERR_INCORRECT_VALUE; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); - } + if (ret == -1) + { + Message::Args args; + args.add("time"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } - return ret; + return ret; } uint32_t Func::intToDate(int64_t i) { - if ((uint64_t) i > 0xFFFFFFFFL) - return ((((uint32_t) (i >> 32)) & 0xFFFFFFC0L) | 0x3E); + if ((uint64_t)i > 0xFFFFFFFFL) + return ((((uint32_t)(i >> 32)) & 0xFFFFFFC0L) | 0x3E); - return i; + return i; } - uint64_t Func::intToDatetime(int64_t i) { - if ((uint64_t) i < 0xFFFFFFFFL) - return (i << 32); + if ((uint64_t)i < 0xFFFFFFFFL) + return (i << 32); - return i; + return i; } uint64_t Func::intToTimestamp(int64_t i) { - return i; + return i; } int64_t Func::intToTime(int64_t i) { - // Don't think we need to do anything here? - return i; + // Don't think we need to do anything here? + return i; } int64_t Func::nowDatetime() { - DateTime result; - boost::posix_time::ptime now = boost::posix_time::microsec_clock::local_time(); - result.year = now.date().year(); - result.month = now.date().month(); - result.day = now.date().day(); - result.hour = now.time_of_day().hours(); - result.minute = now.time_of_day().minutes(); - result.second = now.time_of_day().seconds(); - result.msecond = now.time_of_day().total_microseconds(); + DateTime result; + boost::posix_time::ptime now = boost::posix_time::microsec_clock::local_time(); + result.year = now.date().year(); + result.month = now.date().month(); + result.day = now.date().day(); + result.hour = now.time_of_day().hours(); + result.minute = now.time_of_day().minutes(); + result.second = now.time_of_day().seconds(); + result.msecond = now.time_of_day().total_microseconds(); - return (int64_t) * (reinterpret_cast(&result)); + return (int64_t) * (reinterpret_cast(&result)); } int64_t Func::addTime(DateTime& dt1, Time& dt2) { - DateTime dt; - dt.year = 0; - dt.month = 0; - dt.day = 0; - dt.hour = 0; - dt.minute = 0; - dt.second = 0; - dt.msecond = 0; + DateTime dt; + dt.year = 0; + dt.month = 0; + dt.day = 0; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; - int64_t month, day, hour, min, sec, msec, tmp; - msec = (signed)(dt1.msecond + dt2.msecond); - dt.msecond = tmp = msec % 1000000; + int64_t month, day, hour, min, sec, msec, tmp; + msec = (signed)(dt1.msecond + dt2.msecond); + dt.msecond = tmp = msec % 1000000; - if (tmp < 0) + if (tmp < 0) + { + dt.msecond = tmp + 1000000; + dt2.second--; + } + + sec = (signed)(dt1.second + dt2.second + msec / 1000000); + dt.second = tmp = sec % 60; + + if (tmp < 0) + { + dt.second = tmp + 60; + dt2.minute--; + } + + min = (signed)(dt1.minute + dt2.minute + sec / 60); + dt.minute = tmp = min % 60; + + if (tmp < 0) + { + dt.minute = tmp + 60; + dt2.hour--; + } + + hour = (signed)(dt1.hour + dt2.hour + min / 60); + + if ((hour < 0) || (hour > 23)) + { + dt2.day = hour / 24; + hour = hour % 24; + } + + if (hour < 0) + { + dt.hour = hour + 24; + dt2.day--; + } + else + { + dt.hour = hour; + } + + day = (signed)(dt1.day + dt2.day); + + if (isLeapYear(dt1.year) && dt1.month == 2) + day--; + + month = dt1.month; + int addyear = 0; + + if (day <= 0) + { + int monthSave = month; + + while (day <= 0) { - dt.msecond = tmp + 1000000; - dt2.second--; + month = (month == 1 ? 12 : month - 1); + + for (; day <= 0 && month > 0; month--) + day += getDaysInMonth(month, dt1.year); + + month++; + // month=12; } - sec = (signed)(dt1.second + dt2.second + msec / 1000000); - dt.second = tmp = sec % 60; + if (month > monthSave) + addyear--; + } + else + { + int monthSave = month; - if (tmp < 0) + while (day > getDaysInMonth(month, dt1.year)) { - dt.second = tmp + 60; - dt2.minute--; + for (; day > getDaysInMonth(month, dt1.year) && month <= 12; month++) + day -= getDaysInMonth(month, dt1.year); + + if (month > 12) + month = 1; } - min = (signed)(dt1.minute + dt2.minute + sec / 60); - dt.minute = tmp = min % 60; + if (month < monthSave) + addyear++; + } - if (tmp < 0) - { - dt.minute = tmp + 60; - dt2.hour--; - } + dt.day = day; + dt.month = month; + dt.year = dt1.year + addyear; - hour = (signed)(dt1.hour + dt2.hour + min / 60); - - if ((hour < 0) || (hour > 23)) - { - dt2.day = hour / 24; - hour = hour % 24; - } - - if (hour < 0) - { - dt.hour = hour + 24; - dt2.day--; - } - else - { - dt.hour = hour; - } - - day = (signed)(dt1.day + dt2.day); - - - if (isLeapYear(dt1.year) && dt1.month == 2) - day--; - - month = dt1.month; - int addyear = 0; - - if (day <= 0) - { - int monthSave = month; - - while (day <= 0) - { - month = (month == 1 ? 12 : month - 1); - - for (; day <= 0 && month > 0; month--) - day += getDaysInMonth(month, dt1.year); - - month++; -// month=12; - } - - if ( month > monthSave ) - addyear--; - } - else - { - int monthSave = month; - - while (day > getDaysInMonth(month, dt1.year)) - { - for (; day > getDaysInMonth(month, dt1.year) && month <= 12; month++) - day -= getDaysInMonth(month, dt1.year); - - if (month > 12) - month = 1; - } - - if ( month < monthSave ) - addyear++; - } - - dt.day = day; - dt.month = month; - dt.year = dt1.year + addyear; - - return *(reinterpret_cast(&dt)); + return *(reinterpret_cast(&dt)); } int64_t Func::addTime(Time& dt1, Time& dt2) { - Time dt; - dt.is_neg = false; - dt.hour = 0; - dt.minute = 0; - dt.second = 0; - dt.msecond = 0; + Time dt; + dt.is_neg = false; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; - int64_t min, sec, msec, tmp; - msec = (signed)(dt1.msecond + dt2.msecond); - dt.msecond = tmp = msec % 1000000; + int64_t min, sec, msec, tmp; + msec = (signed)(dt1.msecond + dt2.msecond); + dt.msecond = tmp = msec % 1000000; - if (tmp < 0) - { - dt.msecond = tmp + 1000000; - dt2.second--; - } + if (tmp < 0) + { + dt.msecond = tmp + 1000000; + dt2.second--; + } - sec = (signed)(dt1.second + dt2.second + msec / 1000000); - dt.second = tmp = sec % 60; + sec = (signed)(dt1.second + dt2.second + msec / 1000000); + dt.second = tmp = sec % 60; - if (tmp < 0) - { - dt.second = tmp + 60; - dt2.minute--; - } + if (tmp < 0) + { + dt.second = tmp + 60; + dt2.minute--; + } - min = (signed)(dt1.minute + dt2.minute + sec / 60); - dt.minute = tmp = min % 60; + min = (signed)(dt1.minute + dt2.minute + sec / 60); + dt.minute = tmp = min % 60; - if (tmp < 0) - { - dt.minute = tmp + 60; - dt2.hour--; - } + if (tmp < 0) + { + dt.minute = tmp + 60; + dt2.hour--; + } - dt.hour = tmp = (signed)(dt1.hour + dt2.hour + min / 60); + dt.hour = tmp = (signed)(dt1.hour + dt2.hour + min / 60); - // Saturation - if (tmp > 838) - { - dt.hour = 838; - dt.minute = 59; - dt.second = 59; - dt.msecond = 999999; - } - else if (tmp < -838) - { - dt.is_neg = true; - dt.hour = -838; - dt.minute = 59; - dt.second = 59; - dt.msecond = 999999; - } + // Saturation + if (tmp > 838) + { + dt.hour = 838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + else if (tmp < -838) + { + dt.is_neg = true; + dt.hour = -838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } - return *(reinterpret_cast(&dt)); + return *(reinterpret_cast(&dt)); } - string Func::intToString(int64_t i) { - return helpers::intToString(i); + return helpers::intToString(i); } - string Func::doubleToString(double d) { - return helpers::doubleToString(d); + return helpers::doubleToString(d); } string Func::longDoubleToString(long double ld) { - return helpers::longDoubleToString(ld); + return helpers::longDoubleToString(ld); } - -} // namespace funcexp +} // namespace funcexp // vim:ts=4 sw=4: - - - diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 9027ab351..cccfb00d5 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -47,251 +47,218 @@ namespace execplan { class FunctionColumn; extern const std::string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt); -} +} // namespace execplan namespace funcexp { -//typedef std::vector FunctionParm; +// typedef std::vector FunctionParm; typedef std::vector FunctionParm; - /** @brief Func class - */ + */ class Func { -public: - Func(); - Func(const std::string& funcName); - virtual ~Func() {} + public: + Func(); + Func(const std::string& funcName); + virtual ~Func() + { + } - const std::string funcName() const - { - return fFuncName; - } - void funcName(const std::string funcName) - { - fFuncName = funcName; - } + const std::string funcName() const + { + return fFuncName; + } + void funcName(const std::string funcName) + { + fFuncName = funcName; + } - const std::string timeZone() const - { - std::unique_lock l(tzMutex); - return fTimeZone; - } - void timeZone(const std::string timeZone) - { - std::unique_lock l(tzMutex); - fTimeZone = timeZone; - } + const std::string timeZone() const + { + std::unique_lock l(tzMutex); + return fTimeZone; + } + void timeZone(const std::string timeZone) + { + std::unique_lock l(tzMutex); + fTimeZone = timeZone; + } - void raiseIllegalParameterDataTypeError(const execplan::CalpontSystemCatalog::ColType& colType) const - { - std::ostringstream oss; - oss << "Illegal parameter data type " - << execplan::colDataTypeToString(colType.colDataType) - << " for operation " - << funcName(); - throw logging::IDBExcept(oss.str(), logging::ERR_DATATYPE_NOT_SUPPORT); - } + void raiseIllegalParameterDataTypeError(const execplan::CalpontSystemCatalog::ColType& colType) const + { + std::ostringstream oss; + oss << "Illegal parameter data type " << execplan::colDataTypeToString(colType.colDataType) + << " for operation " << funcName(); + throw logging::IDBExcept(oss.str(), logging::ERR_DATATYPE_NOT_SUPPORT); + } - virtual bool fix(execplan::FunctionColumn &col) const - { - return false; - } + virtual bool fix(execplan::FunctionColumn& col) const + { + return false; + } - virtual execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) = 0; + virtual execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) = 0; - virtual int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + virtual int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + + virtual uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return static_cast(getIntVal(row, fp, isNull, op_ct)); + } + + virtual double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - virtual uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + virtual long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + + virtual std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + + virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return execplan::IDB_Decimal(getIntVal(row, fp, isNull, op_ct), 0, 0); + } + + virtual int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return static_cast(getIntVal(row, fp, isNull, op_ct)); - } + { + return intToDate(getIntVal(row, fp, isNull, op_ct)); + } - virtual double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToDatetime(getIntVal(row, fp, isNull, op_ct)); + } - virtual long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + virtual int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTimestamp(getIntVal(row, fp, isNull, op_ct)); + } - virtual std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + virtual int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTime(getIntVal(row, fp, isNull, op_ct)); + } - virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return execplan::IDB_Decimal(getIntVal(row, fp, isNull, op_ct), 0, 0); - } + virtual bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + int64_t intVal = getIntVal(row, fp, isNull, op_ct); + return (!isNull && intVal); + } - virtual int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return intToDate(getIntVal(row, fp, isNull, op_ct)); - } - - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return intToDatetime(getIntVal(row, fp, isNull, op_ct)); - } - - virtual int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return intToTimestamp(getIntVal(row, fp, isNull, op_ct)); - } - - virtual int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return intToTime(getIntVal(row, fp, isNull, op_ct)); - } - - virtual bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + virtual float getFloatVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - int64_t intVal = getIntVal(row, fp, isNull, op_ct); - return (!isNull && intVal); - } + { + return getDoubleVal(row, fp, isNull, op_ct); + } - virtual float getFloatVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return getDoubleVal(row, fp, isNull, op_ct); - } + float floatNullVal() const + { + return fFloatNullVal; + } + double doubleNullVal() const + { + return fDoubleNullVal; + } + long double longDoubleNullVal() const + { + return fLongDoubleNullVal; + } - float floatNullVal() const - { - return fFloatNullVal; - } - double doubleNullVal() const - { - return fDoubleNullVal; - } - long double longDoubleNullVal() const - { - return fLongDoubleNullVal; - } + protected: + virtual uint32_t stringToDate(std::string); + virtual uint64_t stringToDatetime(std::string); + virtual uint64_t stringToTimestamp(std::string); + virtual int64_t stringToTime(std::string); + virtual uint32_t intToDate(int64_t); + virtual uint64_t intToDatetime(int64_t); + virtual uint64_t intToTimestamp(int64_t); + virtual int64_t intToTime(int64_t); -protected: - virtual uint32_t stringToDate(std::string); - virtual uint64_t stringToDatetime(std::string); - virtual uint64_t stringToTimestamp(std::string); - virtual int64_t stringToTime(std::string); + virtual std::string intToString(int64_t); + virtual std::string doubleToString(double); + virtual std::string longDoubleToString(long double); - virtual uint32_t intToDate(int64_t); - virtual uint64_t intToDatetime(int64_t); - virtual uint64_t intToTimestamp(int64_t); - virtual int64_t intToTime(int64_t); + virtual int64_t nowDatetime(); + virtual int64_t addTime(dataconvert::DateTime& dt1, dataconvert::Time& dt2); + virtual int64_t addTime(dataconvert::Time& dt1, dataconvert::Time& dt2); - virtual std::string intToString(int64_t); - virtual std::string doubleToString(double); - virtual std::string longDoubleToString(long double); + std::string fFuncName; - virtual int64_t nowDatetime(); - virtual int64_t addTime(dataconvert::DateTime& dt1, dataconvert::Time& dt2); - virtual int64_t addTime(dataconvert::Time& dt1, dataconvert::Time& dt2); + private: + // defaults okay + // Func(const Func& rhs); + // Func& operator=(const Func& rhs); - std::string fFuncName; + void init(); -private: - //defaults okay - //Func(const Func& rhs); - //Func& operator=(const Func& rhs); + float fFloatNullVal; + double fDoubleNullVal; + long double fLongDoubleNullVal; - void init(); - - float fFloatNullVal; - double fDoubleNullVal; - long double fLongDoubleNullVal; - - std::string fTimeZone; - mutable std::mutex tzMutex; + std::string fTimeZone; + mutable std::mutex tzMutex; }; - -class ParmTSInt64: public datatypes::TSInt64Null +class ParmTSInt64 : public datatypes::TSInt64Null { -public: - ParmTSInt64() { } - ParmTSInt64(rowgroup::Row& row, - const execplan::SPTP& parm, - const funcexp::Func& thisFunc) - :TSInt64Null(parm->data()->toTSInt64Null(row)) - { } + public: + ParmTSInt64() + { + } + ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + : TSInt64Null(parm->data()->toTSInt64Null(row)) + { + } }; - -class ParmTUInt64: public datatypes::TUInt64Null +class ParmTUInt64 : public datatypes::TUInt64Null { -public: - ParmTUInt64() { } - ParmTUInt64(rowgroup::Row& row, - const execplan::SPTP& parm, - const funcexp::Func& thisFunc) - :TUInt64Null(parm->data()->toTUInt64Null(row)) - { } + public: + ParmTUInt64() + { + } + ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + : TUInt64Null(parm->data()->toTUInt64Null(row)) + { + } }; - -template class Arg2Lazy +template +class Arg2Lazy { -public: - TA a; - TB b; - Arg2Lazy(rowgroup::Row& row, - FunctionParm& parm, - const Func& thisFunc) - :a(row, parm[0], thisFunc), - b(a.isNull() ? TB() : TB(row, parm[1], thisFunc)) - { } + public: + TA a; + TB b; + Arg2Lazy(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc) + : a(row, parm[0], thisFunc), b(a.isNull() ? TB() : TB(row, parm[1], thisFunc)) + { + } }; - -template class Arg2Eager +template +class Arg2Eager { -public: - TA a; - TB b; - Arg2Eager(rowgroup::Row& row, - FunctionParm& parm, - const Func& thisFunc) - :a(row, parm[0], thisFunc), - b(row, parm[1], thisFunc) - { } + public: + TA a; + TB b; + Arg2Eager(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc) + : a(row, parm[0], thisFunc), b(row, parm[1], thisFunc) + { + } }; - -} +} // namespace funcexp #endif - - - diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index c0568f463..e8ae5de6a 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -25,585 +25,448 @@ #include "functor.h" - namespace funcexp { - /** @brief Func_All class - * For function that can retun all supported types. - * Must implement getIntVal(), getDoubleVal(), and getStrVal() - * Implement any other methods that behave differently from the default. - * Note: implementation based on result type will be more efficient. - */ + * For function that can retun all supported types. + * Must implement getIntVal(), getDoubleVal(), and getStrVal() + * Implement any other methods that behave differently from the default. + * Note: implementation based on result type will be more efficient. + */ class Func_All : public Func { -public: - Func_All() {} - Func_All(const std::string& funcName) : Func(funcName) {} - virtual ~Func_All() {} + public: + Func_All() + { + } + Func_All(const std::string& funcName) : Func(funcName) + { + } + virtual ~Func_All() + { + } - /* - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + /* + int64_t getIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + double getDoubleVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - */ + std::string getStrVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + */ }; - /** @brief Func_simple_case class - */ + */ class Func_simple_case : public Func_All { -public: - Func_simple_case(): Func_All("case_simple") {} - virtual ~Func_simple_case() {} + public: + Func_simple_case() : Func_All("case_simple") + { + } + virtual ~Func_simple_case() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; class Func_decode_oracle : public Func_All { -public: - Func_decode_oracle(): Func_All("decode_oracle") {} - virtual ~Func_decode_oracle() {} + public: + Func_decode_oracle() : Func_All("decode_oracle") + { + } + virtual ~Func_decode_oracle() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_searched_case class - */ + */ class Func_searched_case : public Func_All { -public: - Func_searched_case(): Func_All("case_searched") {} - virtual ~Func_searched_case() {} + public: + Func_searched_case() : Func_All("case_searched") + { + } + virtual ~Func_searched_case() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_if class - */ + */ class Func_if : public Func_All { -public: - Func_if() : Func_All("Func_if") {} - virtual ~Func_if() {} + public: + Func_if() : Func_All("Func_if") + { + } + virtual ~Func_if() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_ifnull class - */ + */ class Func_ifnull : public Func_All { -public: - Func_ifnull() : Func_All("ifnull") {} - virtual ~Func_ifnull() {} + public: + Func_ifnull() : Func_All("ifnull") + { + } + virtual ~Func_ifnull() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_greatest class - */ + */ class Func_greatest : public Func_All { -public: - Func_greatest() : Func_All("greatest") {} - virtual ~Func_greatest() {} + public: + Func_greatest() : Func_All("greatest") + { + } + virtual ~Func_greatest() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_least class - */ + */ class Func_least : public Func_All { -public: - Func_least() : Func_All("least") {} - virtual ~Func_least() {} + public: + Func_least() : Func_All("least") + { + } + virtual ~Func_least() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_coalesce class - */ + */ class Func_coalesce : public Func_All { -public: - Func_coalesce() : Func_All("coalesce") {} - virtual ~Func_coalesce() {} + public: + Func_coalesce() : Func_All("coalesce") + { + } + virtual ~Func_coalesce() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_nullif class - */ + */ class Func_nullif : public Func_All { -public: - Func_nullif() : Func_All("nullif") {} - virtual ~Func_nullif() {} + public: + Func_nullif() : Func_All("nullif") + { + } + virtual ~Func_nullif() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - -} +} // namespace funcexp #endif diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index 7467f57d4..76cd0feb2 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -23,307 +23,320 @@ #ifndef FUNCTOR_BOOL_H #define FUNCTOR_BOOL_H - #include "functor.h" - namespace funcexp { - /** @brief Func_Bool class - * For function that returns a boolean result. - * Must implement getBoolVal() - * Implement any other methods that behave differently from the default. - */ + * For function that returns a boolean result. + * Must implement getBoolVal() + * Implement any other methods that behave differently from the default. + */ class Func_Bool : public Func { -public: - Func_Bool() {} - Func_Bool(const std::string& funcName) : Func(funcName) {} - virtual ~Func_Bool() {} + public: + Func_Bool() + { + } + Func_Bool(const std::string& funcName) : Func(funcName) + { + } + virtual ~Func_Bool() + { + } - /* - virtual bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - */ + /* + virtual bool getBoolVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + = 0; + */ - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return (getBoolVal(row, fp, isNull, op_ct) ? 1 : 0); + } + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (getBoolVal(row, fp, isNull, op_ct) ? 1 : 0); - } + { + return (getBoolVal(row, fp, isNull, op_ct) ? 1.0 : 0.0); + } - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (getBoolVal(row, fp, isNull, op_ct) ? 1.0 : 0.0); - } - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (getBoolVal(row, fp, isNull, op_ct) ? 1.0 : 0.0); - } - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (getBoolVal(row, fp, isNull, op_ct) ? "1" : "0"); - } - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return execplan::IDB_Decimal(getIntVal(row, fp, isNull, op_ct), 0, 0); - } - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - isNull = true; - return 0; - } - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - isNull = true; - return 0; - } - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - isNull = true; - return 0; - } + { + return (getBoolVal(row, fp, isNull, op_ct) ? 1.0 : 0.0); + } - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - isNull = true; - return 0; - } + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return (getBoolVal(row, fp, isNull, op_ct) ? "1" : "0"); + } + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return execplan::IDB_Decimal(getIntVal(row, fp, isNull, op_ct), 0, 0); + } + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } }; - /** @brief Func_between class - */ + */ class Func_between : public Func_Bool { -public: - Func_between() : Func_Bool("between") {} - virtual ~Func_between() {} + public: + Func_between() : Func_Bool("between") + { + } + virtual ~Func_between() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_notbetween class - */ + */ class Func_notbetween : public Func_Bool { -public: - Func_notbetween() : Func_Bool("notbetween") {} - virtual ~Func_notbetween() {} + public: + Func_notbetween() : Func_Bool("notbetween") + { + } + virtual ~Func_notbetween() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_in class - */ + */ class Func_in : public Func_Bool { -public: - Func_in() : Func_Bool("in") {} - virtual ~Func_in() {} + public: + Func_in() : Func_Bool("in") + { + } + virtual ~Func_in() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_notin class - */ + */ class Func_notin : public Func_Bool { -public: - Func_notin() : Func_Bool("notin") {} - virtual ~Func_notin() {} + public: + Func_notin() : Func_Bool("notin") + { + } + virtual ~Func_notin() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_regexp class - */ + */ class Func_regexp : public Func_Bool { -public: - Func_regexp() : Func_Bool("regexp") {} - virtual ~Func_regexp() {} + public: + Func_regexp() : Func_Bool("regexp") + { + } + virtual ~Func_regexp() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_isnull class - */ + */ class Func_isnull : public Func_Bool { -public: - /* - * Constructor. Pass the function name to the base constructor. - */ - Func_isnull(): fIsNotNull(false) {} - Func_isnull(bool isnotnull) : fIsNotNull(isnotnull) {} - /* - * Destructor. isnull does not need to do anything here to clean up. - */ - virtual ~Func_isnull() {} + public: + /* + * Constructor. Pass the function name to the base constructor. + */ + Func_isnull() : fIsNotNull(false) + { + } + Func_isnull(bool isnotnull) : fIsNotNull(isnotnull) + { + } + /* + * Destructor. isnull does not need to do anything here to clean up. + */ + virtual ~Func_isnull() + { + } - /** - * Decide on the function's operation type - */ - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + /** + * Decide on the function's operation type + */ + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); -private: - bool fIsNotNull; + private: + bool fIsNotNull; }; - /** @brief Func_Truth class - */ + */ class Func_Truth : public Func_Bool { -public: - Func_Truth(const std::string& funcName, bool a_value, bool a_affirmative) : - Func_Bool(funcName), value(a_value), affirmative(a_affirmative) {} + public: + Func_Truth(const std::string& funcName, bool a_value, bool a_affirmative) + : Func_Bool(funcName), value(a_value), affirmative(a_affirmative) + { + } - virtual ~Func_Truth() {} + virtual ~Func_Truth() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) - { - assert (fp.size() == 1); - return fp[0]->data()->resultType(); - } + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType) + { + assert(fp.size() == 1); + return fp[0]->data()->resultType(); + } - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - bool val = fp[0]->data()->getBoolVal(row, isNull); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + bool val = fp[0]->data()->getBoolVal(row, isNull); - /* - NULL val IS {TRUE, FALSE} --> FALSE - NULL val IS NOT {TRUE, FALSE} --> TRUE - {TRUE, FALSE} val IS {TRUE, FALSE} value --> val == value - {TRUE, FALSE} val IS NOT {TRUE, FALSE} value --> val != value - These cases can be reduced to the following bitwise operation. - */ - bool ret = (isNull & (!affirmative)) | ((!isNull) & (affirmative ^ (value ^ val))); + /* + NULL val IS {TRUE, FALSE} --> FALSE + NULL val IS NOT {TRUE, FALSE} --> TRUE + {TRUE, FALSE} val IS {TRUE, FALSE} value --> val == value + {TRUE, FALSE} val IS NOT {TRUE, FALSE} value --> val != value + These cases can be reduced to the following bitwise operation. + */ + bool ret = (isNull & (!affirmative)) | ((!isNull) & (affirmative ^ (value ^ val))); - isNull = false; + isNull = false; - return ret; - } + return ret; + } -private: - const bool value, affirmative; + private: + const bool value, affirmative; }; - /** @brief Func_IsTrue class - */ + */ class Func_IsTrue : public Func_Truth { -public: - Func_IsTrue() : Func_Truth("istrue", true, true) {} - ~Func_IsTrue() {} + public: + Func_IsTrue() : Func_Truth("istrue", true, true) + { + } + ~Func_IsTrue() + { + } }; - /** @brief Func_IsNotTrue class - */ + */ class Func_IsNotTrue : public Func_Truth { -public: - Func_IsNotTrue() : Func_Truth("isnottrue", true, false) {} - ~Func_IsNotTrue() {} + public: + Func_IsNotTrue() : Func_Truth("isnottrue", true, false) + { + } + ~Func_IsNotTrue() + { + } }; - /** @brief Func_IsFalse class - */ + */ class Func_IsFalse : public Func_Truth { -public: - Func_IsFalse() : Func_Truth("isfalse", false, true) {} - ~Func_IsFalse() {} + public: + Func_IsFalse() : Func_Truth("isfalse", false, true) + { + } + ~Func_IsFalse() + { + } }; - /** @brief Func_IsNotFalse class - */ + */ class Func_IsNotFalse : public Func_Truth { -public: - Func_IsNotFalse() : Func_Truth("isnotfalse", false, false) {} - ~Func_IsNotFalse() {} + public: + Func_IsNotFalse() : Func_Truth("isnotfalse", false, false) + { + } + ~Func_IsNotFalse() + { + } }; - -} +} // namespace funcexp #endif diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index c9d144ac3..890b3a41a 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -25,559 +25,480 @@ #include "functor.h" - namespace funcexp { - /** @brief Func_Dtm class - * For function that returns a date or datetime result. - * Must implement getIntVal() and getStrVal() - * Implement any other methods that behave differently from the default. - * Note: need to pay attention to the difference between - * getIntVal and getDateIntVal/getDatetimeIntVal. - */ + * For function that returns a date or datetime result. + * Must implement getIntVal() and getStrVal() + * Implement any other methods that behave differently from the default. + * Note: need to pay attention to the difference between + * getIntVal and getDateIntVal/getDatetimeIntVal. + */ class Func_Dtm : public Func { -public: - Func_Dtm() {} - Func_Dtm(const std::string& funcName) : Func(funcName) {} - virtual ~Func_Dtm() {} + public: + Func_Dtm() + { + } + Func_Dtm(const std::string& funcName) : Func(funcName) + { + } + virtual ~Func_Dtm() + { + } - /* - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - */ + /* + int64_t getIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + */ - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (double) getIntVal(row, fp, isNull, op_ct); - } + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return (double)getIntVal(row, fp, isNull, op_ct); + } - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (long double) getIntVal(row, fp, isNull, op_ct); - } + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return (long double)getIntVal(row, fp, isNull, op_ct); + } - /* - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - */ + /* + std::string getStrVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + */ }; - /** @brief Func_date class - */ + */ class Func_date : public Func_Dtm { -public: - Func_date() : Func_Dtm("date") {} - virtual ~Func_date() {} + public: + Func_date() : Func_Dtm("date") + { + } + virtual ~Func_date() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_time class - */ + */ class Func_time : public Func_Dtm { -public: - Func_time() : Func_Dtm("time") {} - virtual ~Func_time() {} + public: + Func_time() : Func_Dtm("time") + { + } + virtual ~Func_time() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_timediff class - */ + */ class Func_timediff : public Func_Dtm { -public: - Func_timediff() : Func_Dtm("timediff") {} - virtual ~Func_timediff() {} + public: + Func_timediff() : Func_Dtm("timediff") + { + } + virtual ~Func_timediff() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); -}; + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_date_add class - */ + */ class Func_date_add : public Func_Dtm { -public: - Func_date_add() : Func_Dtm("date_add") {} - virtual ~Func_date_add() {} + public: + Func_date_add() : Func_Dtm("date_add") + { + } + virtual ~Func_date_add() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_cast_date class - */ + */ class Func_cast_date : public Func_Dtm { -public: - Func_cast_date() : Func_Dtm("cast_date") {} - virtual ~Func_cast_date() {} + public: + Func_cast_date() : Func_Dtm("cast_date") + { + } + virtual ~Func_cast_date() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_cast_datetime class - */ + */ class Func_cast_datetime : public Func_Dtm { -public: - Func_cast_datetime() : Func_Dtm("cast_datetime") {} - virtual ~Func_cast_datetime() {} + public: + Func_cast_datetime() : Func_Dtm("cast_datetime") + { + } + virtual ~Func_cast_datetime() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - }; - /** @brief Func_from_days class - */ + */ class Func_from_days : public Func_Dtm { -public: - Func_from_days() : Func_Dtm("from_days") {} - virtual ~Func_from_days() {} + public: + Func_from_days() : Func_Dtm("from_days") + { + } + virtual ~Func_from_days() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_add_time class - */ + */ class Func_add_time : public Func_Dtm { -public: - Func_add_time() : Func_Dtm("add_time") {} - virtual ~Func_add_time() {} + public: + Func_add_time() : Func_Dtm("add_time") + { + } + virtual ~Func_add_time() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_timestampdiff class - */ + */ class Func_timestampdiff : public Func_Dtm { -public: - Func_timestampdiff() : Func_Dtm("timestamp_diff") {} - virtual ~Func_timestampdiff() {} + public: + Func_timestampdiff() : Func_Dtm("timestamp_diff") + { + } + virtual ~Func_timestampdiff() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_sysdate class - */ + */ class Func_sysdate : public Func_Dtm { -public: - Func_sysdate() : Func_Dtm("sysdate") {} - virtual ~Func_sysdate() {} + public: + Func_sysdate() : Func_Dtm("sysdate") + { + } + virtual ~Func_sysdate() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_from_unixtime - */ + */ class Func_from_unixtime : public Func_Dtm { -public: - Func_from_unixtime() : Func_Dtm("from_unixtime") {} - virtual ~Func_from_unixtime() {} - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + public: + Func_from_unixtime() : Func_Dtm("from_unixtime") + { + } + virtual ~Func_from_unixtime() + { + } + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class - */ + */ class Func_str_to_date : public Func_Dtm { -public: - Func_str_to_date() : Func_Dtm("str_to_date") {} - virtual ~Func_str_to_date() {} + public: + Func_str_to_date() : Func_Dtm("str_to_date") + { + } + virtual ~Func_str_to_date() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_makedate class - */ + */ class Func_makedate : public Func_Dtm { -public: - Func_makedate() : Func_Dtm("makedate") {} - virtual ~Func_makedate() {} + public: + Func_makedate() : Func_Dtm("makedate") + { + } + virtual ~Func_makedate() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; -class Func_convert_tz: public Func_Dtm +class Func_convert_tz : public Func_Dtm { -public: - Func_convert_tz() : Func_Dtm("convert_tz") {} - virtual ~Func_convert_tz() {} + public: + Func_convert_tz() : Func_Dtm("convert_tz") + { + } + virtual ~Func_convert_tz() + { + } - //bool from_tz_cached, to_tz_cached; - //Time_zone *from_tz, *to_tz; + // bool from_tz_cached, to_tz_cached; + // Time_zone *from_tz, *to_tz; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; -} +} // namespace funcexp #endif diff --git a/utils/funcexp/functor_export.h b/utils/funcexp/functor_export.h index 270ecd208..7ec43b7a1 100644 --- a/utils/funcexp/functor_export.h +++ b/utils/funcexp/functor_export.h @@ -23,7 +23,6 @@ #ifndef FUNCTOR_EXPORT_H #define FUNCTOR_EXPORT_H - #include "functor.h" #include @@ -31,55 +30,50 @@ namespace funcexp { - /** @brief Func_rand class - * This function is exported, so visible to FunctionColumn in dbcon. - */ + * This function is exported, so visible to FunctionColumn in dbcon. + */ class Func_rand : public Func { -public: - Func_rand() : Func("rand"), fSeed1(0), fSeed2(0), fSeedSet(false) {} - virtual ~Func_rand() {} + public: + Func_rand() : Func("rand"), fSeed1(0), fSeed2(0), fSeedSet(false) + { + } + virtual ~Func_rand() + { + } - double getRand(); - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + double getRand(); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return ((int64_t) getDoubleVal(row, fp, isNull, op_ct)); - } + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return ((int64_t)getDoubleVal(row, fp, isNull, op_ct)); + } - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return (long double)getDoubleVal(row, fp, isNull, op_ct); + } + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (long double)getDoubleVal(row, fp, isNull, op_ct); - } + { + return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); + } - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); - } - -private: - uint64_t fSeed1; - uint64_t fSeed2; - bool fSeedSet; + private: + uint64_t fSeed1; + uint64_t fSeed2; + bool fSeedSet; }; - -} +} // namespace funcexp #endif diff --git a/utils/funcexp/functor_int.h b/utils/funcexp/functor_int.h index 99b24c656..56a7bbf7e 100644 --- a/utils/funcexp/functor_int.h +++ b/utils/funcexp/functor_int.h @@ -25,731 +25,785 @@ #include "functor.h" - namespace funcexp { - /** @brief Func_Int class - * For function that returns a integer result. - * Must implement getIntVal() - * Implement any other methods that behave differently from the default. - */ + * For function that returns a integer result. + * Must implement getIntVal() + * Implement any other methods that behave differently from the default. + */ class Func_Int : public Func { -public: - Func_Int() {} - Func_Int(const std::string& funcName) : Func(funcName) {} - virtual ~Func_Int() {} + public: + Func_Int() + { + } + Func_Int(const std::string& funcName) : Func(funcName) + { + } + virtual ~Func_Int() + { + } - /* - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) = 0; - */ + /* + int64_t getIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) = 0; + */ - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return ((double)getIntVal(row, fp, isNull, op_ct)); + } + + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return ((long double)getIntVal(row, fp, isNull, op_ct)); + } + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return ((double) getIntVal(row, fp, isNull, op_ct)); - } - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return ((long double) getIntVal(row, fp, isNull, op_ct)); - } - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return intToString(getIntVal(row, fp, isNull, op_ct)); - } - + { + return intToString(getIntVal(row, fp, isNull, op_ct)); + } }; - class Func_BitOp : public Func_Int { -public: - Func_BitOp(const std::string& funcName) : Func_Int(funcName) {} - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override - { - return resultType; - } - bool validateArgCount(execplan::FunctionColumn &col, uint expected) const; - void setFunctorByParm(execplan::FunctionColumn & col, - const execplan::SPTP& parm, - Func_Int & return_uint64_from_uint64, - Func_Int & return_uint64_from_sint64, - Func_Int & return_uint64_from_generic) const; - // Fix for << and >> - bool fixForBitShift(execplan::FunctionColumn & col, - Func_Int & return_uint64_from_uint64, - Func_Int & return_uint64_from_sint64, - Func_Int & return_uint64_from_generic) const; - // Fix for & | ^ - bool fixForBitOp2(execplan::FunctionColumn & col, - Func_Int & return_uint64_from_uint64_uint64, - Func_Int & return_uint64_from_sint64_sint64, - Func_Int & return_uint64_from_generic_generic) const; + public: + Func_BitOp(const std::string& funcName) : Func_Int(funcName) + { + } + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override + { + return resultType; + } + bool validateArgCount(execplan::FunctionColumn& col, uint expected) const; + void setFunctorByParm(execplan::FunctionColumn& col, const execplan::SPTP& parm, + Func_Int& return_uint64_from_uint64, Func_Int& return_uint64_from_sint64, + Func_Int& return_uint64_from_generic) const; + // Fix for << and >> + bool fixForBitShift(execplan::FunctionColumn& col, Func_Int& return_uint64_from_uint64, + Func_Int& return_uint64_from_sint64, Func_Int& return_uint64_from_generic) const; + // Fix for & | ^ + bool fixForBitOp2(execplan::FunctionColumn& col, Func_Int& return_uint64_from_uint64_uint64, + Func_Int& return_uint64_from_sint64_sint64, + Func_Int& return_uint64_from_generic_generic) const; - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) override - { - isNull = true; - return 0; - } + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override + { + isNull = true; + return 0; + } }; - /** @brief Func_instr class - */ + */ class Func_instr : public Func_Int { -public: - Func_instr() : Func_Int("instr") {} - virtual ~Func_instr() {} + public: + Func_instr() : Func_Int("instr") + { + } + virtual ~Func_instr() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_length class - */ + */ class Func_length : public Func_Int { -public: - Func_length() : Func_Int("length") {} - virtual ~Func_length() {} + public: + Func_length() : Func_Int("length") + { + } + virtual ~Func_length() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_sign class - */ + */ class Func_sign : public Func_Int { -public: - Func_sign() : Func_Int("sign") {} - virtual ~Func_sign() {} + public: + Func_sign() : Func_Int("sign") + { + } + virtual ~Func_sign() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_day class - */ + */ class Func_day : public Func_Int { -public: - Func_day() : Func_Int("day") {} - virtual ~Func_day() {} + public: + Func_day() : Func_Int("day") + { + } + virtual ~Func_day() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_minute class - */ + */ class Func_minute : public Func_Int { -public: - Func_minute() : Func_Int("minute") {} - virtual ~Func_minute() {} + public: + Func_minute() : Func_Int("minute") + { + } + virtual ~Func_minute() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_month class - */ + */ class Func_month : public Func_Int { -public: - Func_month() : Func_Int("month") {} - virtual ~Func_month() {} + public: + Func_month() : Func_Int("month") + { + } + virtual ~Func_month() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_week class - */ + */ class Func_week : public Func_Int { -public: - Func_week() : Func_Int("week") {} - virtual ~Func_week() {} + public: + Func_week() : Func_Int("week") + { + } + virtual ~Func_week() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_year class - */ + */ class Func_year : public Func_Int { -public: - Func_year() : Func_Int("year") {} - virtual ~Func_year() {} + public: + Func_year() : Func_Int("year") + { + } + virtual ~Func_year() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_to_days class - */ + */ class Func_to_days : public Func_Int { -public: - Func_to_days() : Func_Int("to_days") {} - virtual ~Func_to_days() {} + public: + Func_to_days() : Func_Int("to_days") + { + } + virtual ~Func_to_days() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_char_length class - */ + */ class Func_char_length : public Func_Int { -public: - Func_char_length() : Func_Int("length") {} - virtual ~Func_char_length() {} + public: + Func_char_length() : Func_Int("length") + { + } + virtual ~Func_char_length() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_extract class - */ + */ class Func_extract : public Func_Int { -public: - Func_extract() : Func_Int("extract") {} - virtual ~Func_extract() {} + public: + Func_extract() : Func_Int("extract") + { + } + virtual ~Func_extract() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_cast_signed class - */ + */ class Func_cast_signed : public Func_Int { -public: - Func_cast_signed() : Func_Int("cast_signed") {} - virtual ~Func_cast_signed() {} + public: + Func_cast_signed() : Func_Int("cast_signed") + { + } + virtual ~Func_cast_signed() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_cast_unsigned class - */ + */ class Func_cast_unsigned : public Func_Int { -public: - Func_cast_unsigned() : Func_Int("cast_unsigned") {} - virtual ~Func_cast_unsigned() {} + public: + Func_cast_unsigned() : Func_Int("cast_unsigned") + { + } + virtual ~Func_cast_unsigned() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return (int64_t)(getUintVal(row, fp, isNull, op_ct)); - } + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return (int64_t)(getUintVal(row, fp, isNull, op_ct)); + } - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_bitand class - */ + */ class Func_bitand : public Func_BitOp { -public: - Func_bitand() : Func_BitOp("bitand") {} - virtual ~Func_bitand() {} - bool fix(execplan::FunctionColumn &col) const override; + public: + Func_bitand() : Func_BitOp("bitand") + { + } + virtual ~Func_bitand() + { + } + bool fix(execplan::FunctionColumn& col) const override; }; - /** @brief Func_bitor class - */ + */ class Func_bitor : public Func_BitOp { -public: - Func_bitor() : Func_BitOp("bitor") {} - virtual ~Func_bitor() {} + public: + Func_bitor() : Func_BitOp("bitor") + { + } + virtual ~Func_bitor() + { + } - bool fix(execplan::FunctionColumn &col) const override; + bool fix(execplan::FunctionColumn& col) const override; - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) override; + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; - /** @brief Func_bitxor class - */ + */ class Func_bitxor : public Func_BitOp { -public: - Func_bitxor() : Func_BitOp("bitxor") {} - virtual ~Func_bitxor() {} - bool fix(execplan::FunctionColumn &col) const override; + public: + Func_bitxor() : Func_BitOp("bitxor") + { + } + virtual ~Func_bitxor() + { + } + bool fix(execplan::FunctionColumn& col) const override; }; - /** @brief Func_bit_count class - */ + */ class Func_bit_count : public Func_BitOp { -public: - Func_bit_count() : Func_BitOp("bit_count") {} - virtual ~Func_bit_count() {} - bool fix(execplan::FunctionColumn &col) const override; + public: + Func_bit_count() : Func_BitOp("bit_count") + { + } + virtual ~Func_bit_count() + { + } + bool fix(execplan::FunctionColumn& col) const override; }; - /** @brief Func_hour class - */ + */ class Func_hour : public Func_Int { -public: - Func_hour() : Func_Int("hour") {} - virtual ~Func_hour() {} + public: + Func_hour() : Func_Int("hour") + { + } + virtual ~Func_hour() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_second class - */ + */ class Func_second : public Func_Int { -public: - Func_second() : Func_Int("second") {} - virtual ~Func_second() {} + public: + Func_second() : Func_Int("second") + { + } + virtual ~Func_second() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_dayofweek class - */ + */ class Func_dayofweek : public Func_Int { -public: - Func_dayofweek() : Func_Int("dayofweek") {} - virtual ~Func_dayofweek() {} + public: + Func_dayofweek() : Func_Int("dayofweek") + { + } + virtual ~Func_dayofweek() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_dayofyear class - */ + */ class Func_dayofyear : public Func_Int { -public: - Func_dayofyear() : Func_Int("dayofyear") {} - virtual ~Func_dayofyear() {} + public: + Func_dayofyear() : Func_Int("dayofyear") + { + } + virtual ~Func_dayofyear() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_leftshift class - */ + */ class Func_leftshift : public Func_BitOp { -public: - Func_leftshift() : Func_BitOp("leftshift") {} - virtual ~Func_leftshift() {} - bool fix(execplan::FunctionColumn &col) const override; + public: + Func_leftshift() : Func_BitOp("leftshift") + { + } + virtual ~Func_leftshift() + { + } + bool fix(execplan::FunctionColumn& col) const override; }; - /** @brief Func_rightshift class - */ + */ class Func_rightshift : public Func_BitOp { -public: - Func_rightshift() : Func_BitOp("rightshift") {} - virtual ~Func_rightshift() {} - bool fix(execplan::FunctionColumn &col) const override; + public: + Func_rightshift() : Func_BitOp("rightshift") + { + } + virtual ~Func_rightshift() + { + } + bool fix(execplan::FunctionColumn& col) const override; }; - /** @brief Func_quarter class - */ + */ class Func_quarter : public Func_Int { -public: - Func_quarter() : Func_Int("quarter") {} - virtual ~Func_quarter() {} + public: + Func_quarter() : Func_Int("quarter") + { + } + virtual ~Func_quarter() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_ascii class - */ + */ class Func_ascii : public Func_Int { -public: - Func_ascii() : Func_Int("ascii") {} - virtual ~Func_ascii() {} + public: + Func_ascii() : Func_Int("ascii") + { + } + virtual ~Func_ascii() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_dayname class - */ + */ class Func_dayname : public Func_Int { -public: - Func_dayname() : Func_Int("dayname") {} - virtual ~Func_dayname() {} + public: + Func_dayname() : Func_Int("dayname") + { + } + virtual ~Func_dayname() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_weekday class - */ + */ class Func_weekday : public Func_Int { -public: - Func_weekday() : Func_Int("weekday") {} - virtual ~Func_weekday() {} + public: + Func_weekday() : Func_Int("weekday") + { + } + virtual ~Func_weekday() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_yearweek class - */ + */ class Func_yearweek : public Func_Int { -public: - Func_yearweek() : Func_Int("yearweek") {} - virtual ~Func_yearweek() {} + public: + Func_yearweek() : Func_Int("yearweek") + { + } + virtual ~Func_yearweek() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_last_day class - */ + */ class Func_last_day : public Func_Int { -public: - Func_last_day() : Func_Int("last_day") {} - virtual ~Func_last_day() {} + public: + Func_last_day() : Func_Int("last_day") + { + } + virtual ~Func_last_day() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_time_to_sec class - */ + */ class Func_time_to_sec : public Func_Int { -public: - Func_time_to_sec() : Func_Int("time_to_sec") {} - virtual ~Func_time_to_sec() {} + public: + Func_time_to_sec() : Func_Int("time_to_sec") + { + } + virtual ~Func_time_to_sec() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_microsecond class - */ + */ class Func_microsecond : public Func_Int { -public: - Func_microsecond() : Func_Int("microsecond") {} - virtual ~Func_microsecond() {} + public: + Func_microsecond() : Func_Int("microsecond") + { + } + virtual ~Func_microsecond() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_crc32 class - */ + */ class Func_crc32 : public Func_Int { -public: - Func_crc32() : Func_Int("crc32") {} - virtual ~Func_crc32() {} + public: + Func_crc32() : Func_Int("crc32") + { + } + virtual ~Func_crc32() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_period_add class - */ + */ class Func_period_add : public Func_Int { -public: - Func_period_add() : Func_Int("period_add") {} - virtual ~Func_period_add() {} + public: + Func_period_add() : Func_Int("period_add") + { + } + virtual ~Func_period_add() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_period_diff class - */ + */ class Func_period_diff : public Func_Int { -public: - Func_period_diff() : Func_Int("period_diff") {} - virtual ~Func_period_diff() {} + public: + Func_period_diff() : Func_Int("period_diff") + { + } + virtual ~Func_period_diff() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_strcmp class - */ + */ class Func_strcmp : public Func_Int { -public: - Func_strcmp() : Func_Int("strcmp") {} - virtual ~Func_strcmp() {} + public: + Func_strcmp() : Func_Int("strcmp") + { + } + virtual ~Func_strcmp() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_unix_timestamp class - */ + */ class Func_unix_timestamp : public Func_Int { -public: - Func_unix_timestamp() : Func_Int("unix_timestamp") {} - virtual ~Func_unix_timestamp() {} + public: + Func_unix_timestamp() : Func_Int("unix_timestamp") + { + } + virtual ~Func_unix_timestamp() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_strcmp class - */ + */ class Func_find_in_set : public Func_Int { -public: - Func_find_in_set() : Func_Int("find_in_set") {} - virtual ~Func_find_in_set() {} + public: + Func_find_in_set() : Func_Int("find_in_set") + { + } + virtual ~Func_find_in_set() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; -} +} // namespace funcexp #endif diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 968f400fe..515737068 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -23,757 +23,701 @@ #ifndef FUNCTOR_REAL_H #define FUNCTOR_REAL_H - #include "functor.h" - namespace funcexp { - /** @brief Func_Real class - * For function that returns a int/decimal/double result. - * Must implement getDoubleVal() - * Implement any other methods that behave differently from the default. - * Note: getIntVal is a good candidate to be implemented, too. - */ + * For function that returns a int/decimal/double result. + * Must implement getDoubleVal() + * Implement any other methods that behave differently from the default. + * Note: getIntVal is a good candidate to be implemented, too. + */ class Func_Real : public Func { -public: - Func_Real() {} - Func_Real(const std::string& funcName) : Func(funcName) {} - virtual ~Func_Real() {} + public: + Func_Real() + { + } + Func_Real(const std::string& funcName) : Func(funcName) + { + } + virtual ~Func_Real() + { + } - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return ((int64_t)getDoubleVal(row, fp, isNull, op_ct)); + } + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return ((int64_t) getDoubleVal(row, fp, isNull, op_ct)); - } + { + return ((uint64_t)getDoubleVal(row, fp, isNull, op_ct)); + } - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return ((long double)getDoubleVal(row, fp, isNull, op_ct)); + } + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return ((uint64_t) getDoubleVal(row, fp, isNull, op_ct)); - } - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return ((long double) getDoubleVal(row, fp, isNull, op_ct)); - } - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); - } + { + return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); + } }; - /** @brief Func_abs class - */ + */ class Func_abs : public Func_Real { -public: - Func_abs(): Func_Real("abs") {} - virtual ~Func_abs() {} + public: + Func_abs() : Func_Real("abs") + { + } + virtual ~Func_abs() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_exp class - */ + */ class Func_exp : public Func_Real { -public: - Func_exp() : Func_Real("exp") {} - virtual ~Func_exp() {} + public: + Func_exp() : Func_Real("exp") + { + } + virtual ~Func_exp() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_pow class - */ + */ class Func_pow : public Func_Real { -public: - Func_pow() : Func_Real("pow") {} - virtual ~Func_pow() {} + public: + Func_pow() : Func_Real("pow") + { + } + virtual ~Func_pow() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_round class - */ + */ class Func_round : public Func_Real { -public: - Func_round() : Func_Real("round") {} - virtual ~Func_round() {} + public: + Func_round() : Func_Real("round") + { + } + virtual ~Func_round() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_truncate class - */ + */ class Func_truncate : public Func_Real { -public: - Func_truncate() : Func_Real("truncate") {} - virtual ~Func_truncate() {} + public: + Func_truncate() : Func_Real("truncate") + { + } + virtual ~Func_truncate() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_ceil class - */ + */ class Func_ceil : public Func_Real { -public: - Func_ceil() : Func_Real("ceil") {} - virtual ~Func_ceil() {} + public: + Func_ceil() : Func_Real("ceil") + { + } + virtual ~Func_ceil() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_floor class - */ + */ class Func_floor : public Func_Real { -public: - Func_floor() : Func_Real("floor") {} - virtual ~Func_floor() {} + public: + Func_floor() : Func_Real("floor") + { + } + virtual ~Func_floor() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_cast_decimal class - */ + */ class Func_cast_decimal : public Func_Real { -public: - Func_cast_decimal() : Func_Real("cast_decimal") {} - virtual ~Func_cast_decimal() {} + public: + Func_cast_decimal() : Func_Real("cast_decimal") + { + } + virtual ~Func_cast_decimal() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_cast_double class - */ + */ class Func_cast_double : public Func_Real { -public: - Func_cast_double() : Func_Real("cast_double") {} - virtual ~Func_cast_double() {} + public: + Func_cast_double() : Func_Real("cast_double") + { + } + virtual ~Func_cast_double() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_mod class - */ + */ class Func_mod : public Func_Real { -public: - Func_mod() : Func_Real("mod") {} - virtual ~Func_mod() {} + public: + Func_mod() : Func_Real("mod") + { + } + virtual ~Func_mod() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType); - - uint64_t getUIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -private: - template - ModType doDecimal(const FunctionParm& parm, - const int64_t div, - rowgroup::Row& row, - bool isNull) - { - execplan::IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - - if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - { - if (!d.isScaled()) - { - return d.toTSInt128() % div; - } - else - { - auto intAndFract = d.getIntegralAndDividedFractional(); - - datatypes::TSInt128 integralRemainder = intAndFract.first % div; - return static_cast(integralRemainder.toTFloat128() + - intAndFract.second); - - } - } - int64_t value = d.value / pow(10.0, d.scale); - return value % div; - } - -}; - - -/** @brief Func_acos class - */ -class Func_acos : public Func_Real -{ -public: - Func_acos() : Func_Real("acos") {} - virtual ~Func_acos() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_asin class - */ -class Func_asin : public Func_Real -{ -public: - Func_asin() : Func_Real("asin") {} - virtual ~Func_asin() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_atan class - */ -class Func_atan : public Func_Real -{ -public: - Func_atan() : Func_Real("atan") {} - virtual ~Func_atan() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_cos class - */ -class Func_cos : public Func_Real -{ -public: - Func_cos() : Func_Real("cos") {} - virtual ~Func_cos() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_cot class - */ -class Func_cot : public Func_Real -{ -public: - Func_cot() : Func_Real("cot") {} - virtual ~Func_cot() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_log class - */ -class Func_log : public Func_Real -{ -public: - Func_log() : Func_Real("log") {} - virtual ~Func_log() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_log2 class - */ -class Func_log2 : public Func_Real -{ -public: - Func_log2() : Func_Real("log2") {} - virtual ~Func_log2() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_log10 class - */ -class Func_log10 : public Func_Real -{ -public: - Func_log10() : Func_Real("log10") {} - virtual ~Func_log10() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_sin class - */ -class Func_sin : public Func_Real -{ -public: - Func_sin() : Func_Real("sin") {} - virtual ~Func_sin() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_sqrt class - */ -class Func_sqrt : public Func_Real -{ -public: - Func_sqrt() : Func_Real("sqrt") {} - virtual ~Func_sqrt() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_tan class - */ -class Func_tan : public Func_Real -{ -public: - Func_tan() : Func_Real("tan") {} - virtual ~Func_tan() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_radians class - */ -class Func_radians : public Func_Real -{ -public: - Func_radians() : Func_Real("radians") {} - virtual ~Func_radians() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_div class - */ -class Func_div : public Func_Real -{ -public: - Func_div() : Func_Real("DIV") {} - virtual ~Func_div() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - uint64_t getUintVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_inet_aton class to convert ascii IP address to big-endian - * (network ordered) int - */ -class Func_inet_aton : public Func_Real -{ -public: - Func_inet_aton() : Func_Real("inet_aton") {} - virtual ~Func_inet_aton() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType); -private: - int64_t convertAton(const std::string& ipString, bool& isNull); + uint64_t getUIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: + template + ModType doDecimal(const FunctionParm& parm, const int64_t div, rowgroup::Row& row, bool isNull) + { + execplan::IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (!d.isScaled()) + { + return d.toTSInt128() % div; + } + else + { + auto intAndFract = d.getIntegralAndDividedFractional(); + + datatypes::TSInt128 integralRemainder = intAndFract.first % div; + return static_cast(integralRemainder.toTFloat128() + intAndFract.second); + } + } + int64_t value = d.value / pow(10.0, d.scale); + return value % div; + } +}; + +/** @brief Func_acos class + */ +class Func_acos : public Func_Real +{ + public: + Func_acos() : Func_Real("acos") + { + } + virtual ~Func_acos() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_asin class + */ +class Func_asin : public Func_Real +{ + public: + Func_asin() : Func_Real("asin") + { + } + virtual ~Func_asin() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_atan class + */ +class Func_atan : public Func_Real +{ + public: + Func_atan() : Func_Real("atan") + { + } + virtual ~Func_atan() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_cos class + */ +class Func_cos : public Func_Real +{ + public: + Func_cos() : Func_Real("cos") + { + } + virtual ~Func_cos() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_cot class + */ +class Func_cot : public Func_Real +{ + public: + Func_cot() : Func_Real("cot") + { + } + virtual ~Func_cot() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_log class + */ +class Func_log : public Func_Real +{ + public: + Func_log() : Func_Real("log") + { + } + virtual ~Func_log() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_log2 class + */ +class Func_log2 : public Func_Real +{ + public: + Func_log2() : Func_Real("log2") + { + } + virtual ~Func_log2() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_log10 class + */ +class Func_log10 : public Func_Real +{ + public: + Func_log10() : Func_Real("log10") + { + } + virtual ~Func_log10() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_sin class + */ +class Func_sin : public Func_Real +{ + public: + Func_sin() : Func_Real("sin") + { + } + virtual ~Func_sin() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_sqrt class + */ +class Func_sqrt : public Func_Real +{ + public: + Func_sqrt() : Func_Real("sqrt") + { + } + virtual ~Func_sqrt() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_tan class + */ +class Func_tan : public Func_Real +{ + public: + Func_tan() : Func_Real("tan") + { + } + virtual ~Func_tan() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_radians class + */ +class Func_radians : public Func_Real +{ + public: + Func_radians() : Func_Real("radians") + { + } + virtual ~Func_radians() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_div class + */ +class Func_div : public Func_Real +{ + public: + Func_div() : Func_Real("DIV") + { + } + virtual ~Func_div() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_inet_aton class to convert ascii IP address to big-endian + * (network ordered) int + */ +class Func_inet_aton : public Func_Real +{ + public: + Func_inet_aton() : Func_Real("inet_aton") + { + } + virtual ~Func_inet_aton() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: + int64_t convertAton(const std::string& ipString, bool& isNull); }; /** @brief Func_degrees class - */ + */ class Func_degrees : public Func_Real { -public: - Func_degrees() : Func_Real("degrees") {} - virtual ~Func_degrees() {} + public: + Func_degrees() : Func_Real("degrees") + { + } + virtual ~Func_degrees() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - - -} +} // namespace funcexp #endif diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index e6abccf5c..8cf42f2b4 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -28,42 +28,41 @@ namespace funcexp { - /** @brief Func_Str class - * For function that returns a string result. - * Must implement getStrVal() - * Implement any other methods that behave differently from the default. - */ + * For function that returns a string result. + * Must implement getStrVal() + * Implement any other methods that behave differently from the default. + */ class Func_Str : public Func { -public: - Func_Str() {} - Func_Str(const std::string& funcName) : Func(funcName) {} - virtual ~Func_Str() {} + public: + Func_Str() + { + } + Func_Str(const std::string& funcName) : Func(funcName) + { + } + virtual ~Func_Str() + { + } - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return atoll(getStrVal(row, fp, isNull, op_ct).c_str()); + } + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) - { - return atoll(getStrVal(row, fp, isNull, op_ct).c_str()); - } + { + return strtod(getStrVal(row, fp, isNull, op_ct).c_str(), NULL); + } - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return strtod(getStrVal(row, fp, isNull, op_ct).c_str(), NULL); - } - - long double getLongDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - return strtold(getStrVal(row, fp, isNull, op_ct).c_str(), NULL); - } + long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return strtold(getStrVal(row, fp, isNull, op_ct).c_str(), NULL); + } #if 0 std::string getStrVal(rowgroup::Row& row, @@ -72,953 +71,995 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct) = 0; #endif - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return execplan::IDB_Decimal(atoll(getStrVal(row, fp, isNull, op_ct).c_str()), 0, 0); + } + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToDate(str)); + } + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToDatetime(str)); + } + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTimestamp(str)); + } + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTime(str)); + } + + protected: + void stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull, std::string& fFloatStr) + { + // Bug3788, use the shorter of fixed or scientific notation for floating point values. + // [ the default format in treenode.h is fixed-point notation ] + char buf[20]; + long double floatVal; + int exponent; + long double base; + + switch (fp->data()->resultType().colDataType) { - return execplan::IDB_Decimal(atoll(getStrVal(row, fp, isNull, op_ct).c_str()), 0, 0); + case execplan::CalpontSystemCatalog::LONGDOUBLE: + floatVal = fp->data()->getLongDoubleVal(row, isNull); + break; + + case execplan::CalpontSystemCatalog::DOUBLE: floatVal = fp->data()->getDoubleVal(row, isNull); break; + + case execplan::CalpontSystemCatalog::FLOAT: floatVal = fp->data()->getFloatVal(row, isNull); break; + + default: + fFloatStr = fp->data()->getStrVal(row, isNull); + return; + break; } - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + exponent = (int)floor(log10(fabsl(floatVal))); + base = floatVal * pow(10, -1.0 * exponent); + + if (std::isnan(exponent) || std::isnan(base)) { - std::string str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToDate(str)); + snprintf(buf, 20, "%Lf", floatVal); + fFloatStr = execplan::removeTrailing0(buf, 20); } - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + else { - std::string str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToDatetime(str)); - } - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - std::string str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToTimestamp(str)); - } - - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) - { - std::string str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToTime(str)); - } - -protected: - void stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull, std::string& fFloatStr) - { - // Bug3788, use the shorter of fixed or scientific notation for floating point values. - // [ the default format in treenode.h is fixed-point notation ] - char buf[20]; - long double floatVal; - int exponent; - long double base; - - switch (fp->data()->resultType().colDataType) - { - case execplan::CalpontSystemCatalog::LONGDOUBLE: - floatVal = fp->data()->getLongDoubleVal(row, isNull); - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - floatVal = fp->data()->getDoubleVal(row, isNull); - break; - - case execplan::CalpontSystemCatalog::FLOAT: - floatVal = fp->data()->getFloatVal(row, isNull); - break; - - default: - fFloatStr = fp->data()->getStrVal(row, isNull); - return; - break; - } - - exponent = (int)floor(log10( fabsl(floatVal))); - base = floatVal * pow(10, -1.0 * exponent); - - if (std::isnan(exponent) || std::isnan(base)) - { - snprintf(buf, 20, "%Lf", floatVal); - fFloatStr = execplan::removeTrailing0(buf, 20); - } - else - { - snprintf(buf, 20, "%.5Lf", base); - fFloatStr = execplan::removeTrailing0(buf, 20); - snprintf(buf, 20, "e%02d", exponent); - fFloatStr += buf; - } - + snprintf(buf, 20, "%.5Lf", base); + fFloatStr = execplan::removeTrailing0(buf, 20); + snprintf(buf, 20, "e%02d", exponent); + fFloatStr += buf; } + } }; - /** @brief Func_concat class - */ + */ class Func_concat : public Func_Str { -public: - Func_concat() : Func_Str("concat") {} - virtual ~Func_concat() {} + public: + Func_concat() : Func_Str("concat") + { + } + virtual ~Func_concat() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_concat_oracle class - */ + */ class Func_concat_oracle : public Func_Str { -public: - Func_concat_oracle() : Func_Str("concat_operator_oracle") {} - virtual ~Func_concat_oracle() {} + public: + Func_concat_oracle() : Func_Str("concat_operator_oracle") + { + } + virtual ~Func_concat_oracle() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_substr class - */ + */ class Func_substr : public Func_Str { -public: - Func_substr() : Func_Str("substr") {} - virtual ~Func_substr() {} + public: + Func_substr() : Func_Str("substr") + { + } + virtual ~Func_substr() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_date_format class - */ + */ class Func_date_format : public Func_Str { -public: - Func_date_format() : Func_Str("date_format") {} - virtual ~Func_date_format() {} + public: + Func_date_format() : Func_Str("date_format") + { + } + virtual ~Func_date_format() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_lcase class - */ + */ class Func_lcase : public Func_Str { -public: - Func_lcase() : Func_Str("lcase") {} - virtual ~Func_lcase() {} + public: + Func_lcase() : Func_Str("lcase") + { + } + virtual ~Func_lcase() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_ucase class - */ + */ class Func_ucase : public Func_Str { -public: - Func_ucase() : Func_Str("ucase") {} - virtual ~Func_ucase() {} + public: + Func_ucase() : Func_Str("ucase") + { + } + virtual ~Func_ucase() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_left class - */ + */ class Func_left : public Func_Str { -public: - Func_left() : Func_Str("left") {} - virtual ~Func_left() {} + public: + Func_left() : Func_Str("left") + { + } + virtual ~Func_left() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_ltrim class - */ + */ class Func_ltrim : public Func_Str { -public: - Func_ltrim() : Func_Str("ltrim") {} - virtual ~Func_ltrim() {} + public: + Func_ltrim() : Func_Str("ltrim") + { + } + virtual ~Func_ltrim() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_rtrim class - */ + */ class Func_rtrim : public Func_Str { -public: - Func_rtrim() : Func_Str("rtrim") {} - virtual ~Func_rtrim() {} + public: + Func_rtrim() : Func_Str("rtrim") + { + } + virtual ~Func_rtrim() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_trim class - */ + */ class Func_trim : public Func_Str { -public: - Func_trim() : Func_Str("trim") {} - virtual ~Func_trim() {} + public: + Func_trim() : Func_Str("trim") + { + } + virtual ~Func_trim() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_ltrim class - */ + */ class Func_ltrim_oracle : public Func_Str { -public: - Func_ltrim_oracle() : Func_Str("ltrim_oracle") {} - virtual ~Func_ltrim_oracle() {} + public: + Func_ltrim_oracle() : Func_Str("ltrim_oracle") + { + } + virtual ~Func_ltrim_oracle() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_rtrim class - */ + */ class Func_rtrim_oracle : public Func_Str { -public: - Func_rtrim_oracle() : Func_Str("rtrim_oracle") {} - virtual ~Func_rtrim_oracle() {} + public: + Func_rtrim_oracle() : Func_Str("rtrim_oracle") + { + } + virtual ~Func_rtrim_oracle() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_trim class - */ + */ class Func_trim_oracle : public Func_Str { -public: - Func_trim_oracle() : Func_Str("trim_oracle") {} - virtual ~Func_trim_oracle() {} + public: + Func_trim_oracle() : Func_Str("trim_oracle") + { + } + virtual ~Func_trim_oracle() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_lpad class - */ + */ class Func_lpad : public Func_Str { - static const std::string fPad; -public: - Func_lpad() : Func_Str("lpad") {} - virtual ~Func_lpad() {} + static const std::string fPad; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + public: + Func_lpad() : Func_Str("lpad") + { + } + virtual ~Func_lpad() + { + } - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_rpad class - */ + */ class Func_rpad : public Func_Str { - static const std::string fPad; -public: - Func_rpad() : Func_Str("rpad") {} - virtual ~Func_rpad() {} + static const std::string fPad; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + public: + Func_rpad() : Func_Str("rpad") + { + } + virtual ~Func_rpad() + { + } - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_replace class - */ + */ class Func_replace : public Func_Str { -public: - Func_replace() : Func_Str("replace") {} - virtual ~Func_replace() {} + public: + Func_replace() : Func_Str("replace") + { + } + virtual ~Func_replace() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; class Func_replace_oracle : public Func_Str { -public: - Func_replace_oracle() : Func_Str("replace_oracle") {} - virtual ~Func_replace_oracle() {} + public: + Func_replace_oracle() : Func_Str("replace_oracle") + { + } + virtual ~Func_replace_oracle() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_right class - */ + */ class Func_right : public Func_Str { -public: - Func_right() : Func_Str("right") {} - virtual ~Func_right() {} + public: + Func_right() : Func_Str("right") + { + } + virtual ~Func_right() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_char class - */ + */ class Func_char : public Func_Str { -public: - Func_char() : Func_Str("char") {} - virtual ~Func_char() {} + public: + Func_char() : Func_Str("char") + { + } + virtual ~Func_char() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_cast_char class - */ + */ class Func_cast_char : public Func_Str { -public: - Func_cast_char() : Func_Str("char") {} - virtual ~Func_cast_char() {} + public: + Func_cast_char() : Func_Str("char") + { + } + virtual ~Func_cast_char() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_format class - */ + */ class Func_format : public Func_Str { -public: - Func_format() : Func_Str("format") {} - virtual ~Func_format() {} + public: + Func_format() : Func_Str("format") + { + } + virtual ~Func_format() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_conv class - */ + */ class Func_conv : public Func_Str { -public: - Func_conv() : Func_Str("conv") {} - virtual ~Func_conv() {} + public: + Func_conv() : Func_Str("conv") + { + } + virtual ~Func_conv() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_md5 class - */ + */ class Func_md5 : public Func_Str { -public: - Func_md5() : Func_Str("md5") {} - virtual ~Func_md5() {} + public: + Func_md5() : Func_Str("md5") + { + } + virtual ~Func_md5() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_unhex class - */ + */ class Func_unhex : public Func_Str { -public: - Func_unhex() : Func_Str("unhex") {} - virtual ~Func_unhex() {} + public: + Func_unhex() : Func_Str("unhex") + { + } + virtual ~Func_unhex() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_concat_ws class - */ + */ class Func_concat_ws : public Func_Str { -public: - Func_concat_ws() : Func_Str("concat_ws") {} - virtual ~Func_concat_ws() {} + public: + Func_concat_ws() : Func_Str("concat_ws") + { + } + virtual ~Func_concat_ws() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_monthname class - */ + */ class Func_monthname : public Func_Str { -public: - Func_monthname() : Func_Str("monthname") {} - virtual ~Func_monthname() {} + public: + Func_monthname() : Func_Str("monthname") + { + } + virtual ~Func_monthname() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_time_format class - */ -class Func_time_format : public Func_Str -{ -public: - Func_time_format() : Func_Str("time_format") {} - virtual ~Func_time_format() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_sec_to_time class - */ -class Func_sec_to_time : public Func_Str -{ -public: - Func_sec_to_time() : Func_Str("sec_to_time") {} - virtual ~Func_sec_to_time() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_substring_index class - */ -class Func_substring_index : public Func_Str -{ -public: - Func_substring_index() : Func_Str("substring_index") {} - virtual ~Func_substring_index() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_hex class - */ -class Func_hex : public Func_Str -{ -public: - Func_hex() : Func_Str("hex") {} - virtual ~Func_hex() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_repeat class - */ -class Func_repeat : public Func_Str -{ -public: - Func_repeat() : Func_Str("repeat") {} - virtual ~Func_repeat() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -}; - - -/** @brief Func_inet_ntoa class to convert big-indian (network ordered) int to - * an ascii IP address - */ -class Func_inet_ntoa : public Func_Str -{ -public: - Func_inet_ntoa() : Func_Str("inet_ntoa") {} - virtual ~Func_inet_ntoa() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - int64_t getIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - double getDoubleVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - bool getBoolVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int32_t getDateIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getDatetimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimestampIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); -private: - void convertNtoa(int64_t ipNum, std::string& ipString); + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; +/** @brief Func_time_format class + */ +class Func_time_format : public Func_Str +{ + public: + Func_time_format() : Func_Str("time_format") + { + } + virtual ~Func_time_format() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_sec_to_time class + */ +class Func_sec_to_time : public Func_Str +{ + public: + Func_sec_to_time() : Func_Str("sec_to_time") + { + } + virtual ~Func_sec_to_time() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_substring_index class + */ +class Func_substring_index : public Func_Str +{ + public: + Func_substring_index() : Func_Str("substring_index") + { + } + virtual ~Func_substring_index() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_hex class + */ +class Func_hex : public Func_Str +{ + public: + Func_hex() : Func_Str("hex") + { + } + virtual ~Func_hex() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_repeat class + */ +class Func_repeat : public Func_Str +{ + public: + Func_repeat() : Func_Str("repeat") + { + } + virtual ~Func_repeat() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + +/** @brief Func_inet_ntoa class to convert big-indian (network ordered) int to + * an ascii IP address + */ +class Func_inet_ntoa : public Func_Str +{ + public: + Func_inet_ntoa() : Func_Str("inet_ntoa") + { + } + virtual ~Func_inet_ntoa() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: + void convertNtoa(int64_t ipNum, std::string& ipString); +}; /** @brief Func_reverse class - */ + */ class Func_reverse : public Func_Str { -public: - Func_reverse() : Func_Str("reverse") {} - virtual ~Func_reverse() {} + public: + Func_reverse() : Func_Str("reverse") + { + } + virtual ~Func_reverse() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_insert class - */ + */ class Func_insert : public Func_Str { -public: - Func_insert() : Func_Str("insert") {} - virtual ~Func_insert() {} + public: + Func_insert() : Func_Str("insert") + { + } + virtual ~Func_insert() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_maketime class - */ + */ class Func_maketime : public Func_Str { -public: - Func_maketime() : Func_Str("maketime") {} - virtual ~Func_maketime() {} + public: + Func_maketime() : Func_Str("maketime") + { + } + virtual ~Func_maketime() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_get_format class - */ + */ class Func_get_format : public Func_Str { -public: - Func_get_format() : Func_Str("get_format") {} - virtual ~Func_get_format() {} + public: + Func_get_format() : Func_Str("get_format") + { + } + virtual ~Func_get_format() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_elt class - */ + */ class Func_elt : public Func_Str { -public: - Func_elt() : Func_Str("elt") {} - virtual ~Func_elt() {} + public: + Func_elt() : Func_Str("elt") + { + } + virtual ~Func_elt() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_sha class - */ + */ class Func_sha : public Func_Str { -public: - Func_sha() : Func_Str("sha") {} - virtual ~Func_sha() {} + public: + Func_sha() : Func_Str("sha") + { + } + virtual ~Func_sha() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_idbpartition class - */ + */ class Func_idbpartition : public Func_Str { -public: - Func_idbpartition() : Func_Str("idbpartition") {} - virtual ~Func_idbpartition() {} + public: + Func_idbpartition() : Func_Str("idbpartition") + { + } + virtual ~Func_idbpartition() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_space class - */ + */ class Func_space : public Func_Str { -public: - Func_space() : Func_Str("space") {} - virtual ~Func_space() {} + public: + Func_space() : Func_Str("space") + { + } + virtual ~Func_space() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; - /** @brief Func_quote class - */ + */ class Func_quote : public Func_Str { -public: - Func_quote() : Func_Str("quote") {} - virtual ~Func_quote() {} + public: + Func_quote() : Func_Str("quote") + { + } + virtual ~Func_quote() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_encode class - */ + */ class Func_encode : public Func_Str { -public: - Func_encode() : Func_Str("encode") , fSeeded(false) , fSeeds{0,0} {} - virtual ~Func_encode() {} + public: + Func_encode() : Func_Str("encode"), fSeeded(false), fSeeds{0, 0} + { + } + virtual ~Func_encode() + { + } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - void resetSeed() - { - fSeeded = false; - fSeeds[0] = 0; - fSeeds[1] = 0; - } -private: - void hash_password(ulong *result, const char *password, uint password_len); - bool fSeeded; - SQL_CRYPT sql_crypt; - ulong fSeeds[2]; -}; + void resetSeed() + { + fSeeded = false; + fSeeds[0] = 0; + fSeeds[1] = 0; + } -/** @brief Func_encode class - */ - -class Func_decode : public Func_Str -{ -public: - Func_decode() : Func_Str("decode") , fSeeded(false) , fSeeds{0,0} {} - virtual ~Func_decode() {} - - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); - - std::string getStrVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - void resetSeed() - { - fSeeded = false; - fSeeds[0] = 0; - fSeeds[1] = 0; - } -private: - void hash_password(ulong *result, const char *password, uint password_len); + private: + void hash_password(ulong* result, const char* password, uint password_len); bool fSeeded; SQL_CRYPT sql_crypt; ulong fSeeds[2]; }; -} +/** @brief Func_encode class + */ + +class Func_decode : public Func_Str +{ + public: + Func_decode() : Func_Str("decode"), fSeeded(false), fSeeds{0, 0} + { + } + virtual ~Func_decode() + { + } + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); + + std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + void resetSeed() + { + fSeeded = false; + fSeeds[0] = 0; + fSeeds[1] = 0; + } + + private: + void hash_password(ulong* result, const char* password, uint password_len); + bool fSeeded; + SQL_CRYPT sql_crypt; + ulong fSeeds[2]; +}; + +} // namespace funcexp #endif diff --git a/utils/funcexp/sha.h b/utils/funcexp/sha.h index 936147e91..ef4d5158f 100644 --- a/utils/funcexp/sha.h +++ b/utils/funcexp/sha.h @@ -53,61 +53,59 @@ namespace funcexp { class SHA1 { -public: - SHA1(); - virtual ~SHA1(); + public: + SHA1(); + virtual ~SHA1(); - /* - * Re-initialize the class - */ - void Reset(); + /* + * Re-initialize the class + */ + void Reset(); - /* - * Returns the message digest - */ - bool Result(unsigned* message_digest_array); + /* + * Returns the message digest + */ + bool Result(unsigned* message_digest_array); - /* - * Provide input to SHA1 - */ - void Input( const unsigned char* message_array, - unsigned length); - void Input( const char* message_array, - unsigned length); - void Input(unsigned char message_element); - void Input(char message_element); - SHA1& operator<<(const char* message_array); - SHA1& operator<<(const unsigned char* message_array); - SHA1& operator<<(const char message_element); - SHA1& operator<<(const unsigned char message_element); + /* + * Provide input to SHA1 + */ + void Input(const unsigned char* message_array, unsigned length); + void Input(const char* message_array, unsigned length); + void Input(unsigned char message_element); + void Input(char message_element); + SHA1& operator<<(const char* message_array); + SHA1& operator<<(const unsigned char* message_array); + SHA1& operator<<(const char message_element); + SHA1& operator<<(const unsigned char message_element); -private: - /* - * Process the next 512 bits of the message - */ - void ProcessMessageBlock(); + private: + /* + * Process the next 512 bits of the message + */ + void ProcessMessageBlock(); - /* - * Pads the current message block to 512 bits - */ - void PadMessage(); + /* + * Pads the current message block to 512 bits + */ + void PadMessage(); - /* - * Performs a circular left shift operation - */ - inline unsigned CircularShift(int bits, unsigned word); + /* + * Performs a circular left shift operation + */ + inline unsigned CircularShift(int bits, unsigned word); - unsigned H[5]; // Message digest buffers - unsigned Length_Low; // Message length in bits - unsigned Length_High; // Message length in bits + unsigned H[5]; // Message digest buffers + unsigned Length_Low; // Message length in bits + unsigned Length_High; // Message length in bits - unsigned char Message_Block[64]; // 512-bit message blocks - int Message_Block_Index; // Index into message block array + unsigned char Message_Block[64]; // 512-bit message blocks + int Message_Block_Index; // Index into message block array - bool Computed; // Is the digest computed? - bool Corrupted; // Is the message digest corruped? + bool Computed; // Is the digest computed? + bool Corrupted; // Is the message digest corruped? }; -} //namespace +} // namespace funcexp #endif diff --git a/utils/funcexp/sql_crypt.cpp b/utils/funcexp/sql_crypt.cpp index 24dd9392c..c16aac670 100644 --- a/utils/funcexp/sql_crypt.cpp +++ b/utils/funcexp/sql_crypt.cpp @@ -6,51 +6,47 @@ using namespace std; namespace funcexp { - -void SQL_CRYPT::init(ulong *rand_nr) +void SQL_CRYPT::init(ulong* rand_nr) { uint i; - my_rnd_init(&rand,rand_nr[0],rand_nr[1]); + my_rnd_init(&rand, rand_nr[0], rand_nr[1]); - for (i=0 ; i<=255; i++) - decode_buff[i]= (char) i; + for (i = 0; i <= 255; i++) + decode_buff[i] = (char)i; - for (i=0 ; i<= 255 ; i++) + for (i = 0; i <= 255; i++) { - int idx= (uint) (my_rnd(&rand)*255.0); - char a= decode_buff[idx]; - decode_buff[idx]= decode_buff[i]; - decode_buff[+i]=a; + int idx = (uint)(my_rnd(&rand) * 255.0); + char a = decode_buff[idx]; + decode_buff[idx] = decode_buff[i]; + decode_buff[+i] = a; } - for (i=0 ; i <= 255 ; i++) - encode_buff[(unsigned char) decode_buff[i]]=i; - org_rand=rand; - shift=0; + for (i = 0; i <= 255; i++) + encode_buff[(unsigned char)decode_buff[i]] = i; + org_rand = rand; + shift = 0; } - -void SQL_CRYPT::encode(char *str,uint length) +void SQL_CRYPT::encode(char* str, uint length) { - for (uint i=0; i < length; i++) + for (uint i = 0; i < length; i++) { - shift^=(uint) (my_rnd(&rand)*255.0); - uint idx= (uint) (unsigned char) str[0]; - *str++ = (char) ((unsigned char) encode_buff[idx] ^ shift); - shift^= idx; + shift ^= (uint)(my_rnd(&rand) * 255.0); + uint idx = (uint)(unsigned char)str[0]; + *str++ = (char)((unsigned char)encode_buff[idx] ^ shift); + shift ^= idx; } } - -void SQL_CRYPT::decode(char *str,uint length) +void SQL_CRYPT::decode(char* str, uint length) { - for (uint i=0; i < length; i++) + for (uint i = 0; i < length; i++) { - shift^=(uint) (my_rnd(&rand)*255.0); - uint idx= (uint) ((unsigned char) str[0] ^ shift); + shift ^= (uint)(my_rnd(&rand) * 255.0); + uint idx = (uint)((unsigned char)str[0] ^ shift); *str = decode_buff[idx]; - shift^= (uint) (unsigned char) *str++; + shift ^= (uint)(unsigned char)*str++; } } -} - +} // namespace funcexp diff --git a/utils/funcexp/sql_crypt.h b/utils/funcexp/sql_crypt.h index dbcba424a..941750be6 100644 --- a/utils/funcexp/sql_crypt.h +++ b/utils/funcexp/sql_crypt.h @@ -4,8 +4,10 @@ //#include "my_global.h" /* Macros to make switching between C and C++ mode easier */ #ifdef __cplusplus -#define C_MODE_START extern "C" { -#define C_MODE_END } +#define C_MODE_START \ + extern "C" \ + { +#define C_MODE_END } #else #define C_MODE_START #define C_MODE_END @@ -14,27 +16,33 @@ namespace funcexp { - class SQL_CRYPT { - struct my_rnd_struct rand,org_rand; - char decode_buff[256],encode_buff[256]; + struct my_rnd_struct rand, org_rand; + char decode_buff[256], encode_buff[256]; uint shift; public: - SQL_CRYPT() {} - SQL_CRYPT(ulong *seed) + SQL_CRYPT() + { + } + SQL_CRYPT(ulong* seed) { init(seed); } - ~SQL_CRYPT() {} - void init(ulong *seed); - void reinit() { shift=0; rand=org_rand; } - void encode(char *str, uint length); - void decode(char *str, uint length); + ~SQL_CRYPT() + { + } + void init(ulong* seed); + void reinit() + { + shift = 0; + rand = org_rand; + } + void encode(char* str, uint length); + void decode(char* str, uint length); }; -} - +} // namespace funcexp #endif /* _SQL_CRYPT_H_ */ diff --git a/utils/funcexp/tdriver.cpp b/utils/funcexp/tdriver.cpp index 6774f8b39..8ab2fe614 100644 --- a/utils/funcexp/tdriver.cpp +++ b/utils/funcexp/tdriver.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /** -* $Id$ -*/ + * $Id$ + */ #include #include @@ -39,347 +39,330 @@ using namespace execplan; class FuncExpTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(FuncExpTest); - CPPUNIT_TEST_SUITE( FuncExpTest ); + CPPUNIT_TEST(fe_getnumbertest); + CPPUNIT_TEST(fe_strtodatetest); + CPPUNIT_TEST(fe_dateaddtest); + CPPUNIT_TEST(fe_daynametest); + CPPUNIT_TEST(fe_fromunixtimetest); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST( fe_getnumbertest ); - CPPUNIT_TEST( fe_strtodatetest ); - CPPUNIT_TEST( fe_dateaddtest ); - CPPUNIT_TEST( fe_daynametest ); - CPPUNIT_TEST( fe_fromunixtimetest ); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void setUp() + { + } -private: + void tearDown() + { + } -public: - void setUp() + void show_datetime_debugs(const string& nm, const DateTime& d1) + { + cout << nm << ".day = " << setw(10) << d1.day << endl; + cout << nm << ".month = " << setw(10) << d1.month << endl; + cout << nm << ".year = " << setw(10) << d1.year << endl; + cout << nm << ".hour = " << setw(10) << d1.hour << endl; + cout << nm << ".minute = " << setw(10) << d1.minute << endl; + cout << nm << ".second = " << setw(10) << d1.second << endl; + cout << nm << ".msecond = " << setw(10) << d1.msecond << endl; + } + + void fe_strtodatetest() + { + struct DateCheck { + const char* inputstr; + const char* formatstr; + DateTime date; + }; + + DateCheck date_tests[] = { + {"2009", "%Y", DateTime(2009, 0, 0, 0, 0, 0, 0)}, + {" 2009", "%Y", DateTime(2009, 0, 0, 0, 0, 0, 0)}, + {" 2009", " %Y", DateTime(2009, 0, 0, 0, 0, 0, 0)}, + {"jan", "%b", DateTime(0, 1, 0, 0, 0, 0, 0)}, + {"2009feb", "%Y%b", DateTime(2009, 2, 0, 0, 0, 0, 0)}, + {" 2009 ApR", " %Y %b", DateTime(2009, 4, 0, 0, 0, 0, 0)}, + {"200910", "%Y%m", DateTime(2009, 10, 0, 0, 0, 0, 0)}, + {" 2009 10", " %Y %m", DateTime(2009, 10, 0, 0, 0, 0, 0)}, + {"200910", "%Y%c", DateTime(2009, 10, 0, 0, 0, 0, 0)}, + {" 2009 10", " %Y %c", DateTime(2009, 10, 0, 0, 0, 0, 0)}, + {"01,5,2013", "%d,%m,%Y", DateTime(2013, 5, 1, 0, 0, 0, 0)}, + {"01,5,2013", "%e,%m,%Y", DateTime(2013, 5, 1, 0, 0, 0, 0)}, + {"7,5th,2012", "%m,%D,%Y", DateTime(2012, 7, 5, 0, 0, 0, 0)}, + {"a09:30:17", "a%h:%i", DateTime(0, 0, 0, 9, 30, 0, 0)}, + {"a09:30:17", "%h:%i", DateTime(0, 0, 0, 0, 0, 0, 0)}, + {"9:30.170", "%h:%i.%f", DateTime(0, 0, 0, 9, 30, 0, 170000)}, + {"178546 17", "%f %H", DateTime(0, 0, 0, 17, 0, 0, 178546)}, + {"2009march415", "%Y%b", DateTime(2009, 3, 0, 0, 0, 0, 0)}, + {" 2009 July", " %Y %b", DateTime(2009, 7, 0, 0, 0, 0, 0)}, + {"09 pm", "%h %p", DateTime(0, 0, 0, 21, 0, 0, 0)}, + {"09:13:14pm", "%r", DateTime(0, 0, 0, 9, 13, 14, 0)}, + {"12:13:14am", "%r", DateTime(0, 0, 0, 12, 13, 14, 0)}, + {"92", " %Y", DateTime(1992, 0, 0, 0, 0, 0, 0)}, + {"9", " %Y", DateTime(2009, 0, 0, 0, 0, 0, 0)}, + {"2013 31", "%Y %j", DateTime(2013, 1, 31, 0, 0, 0, 0)}, + {"2013 1 3", "%Y %U %w", DateTime(2013, 1, 9, 0, 0, 0, 0)}, + {"2013 1 3", "%Y %u %w", DateTime(2013, 1, 9, 0, 0, 0, 0)}, + {"2007 1 3", "%Y %U %w", DateTime(2007, 1, 10, 0, 0, 0, 0)}, + {"2007 1 3", "%Y %u %w", DateTime(2007, 1, 3, 0, 0, 0, 0)}, + {"2013 1 3", "%X %V %w", DateTime(2013, 1, 9, 0, 0, 0, 0)}, + {"2013 1 3", "%X %v %w", DateTime(2013, 1, 9, 0, 0, 0, 0)}, + {"2007 1 3", "%X %V %w", DateTime(2007, 1, 10, 0, 0, 0, 0)}, + {"2007 1 3", "%X %v %w", DateTime(2007, 1, 3, 0, 0, 0, 0)}, + {"33:30.170", "%h:%i.%f", DateTime(0, 0, 0, 0, 0, 0, 0)}, + {"01,25,2013", "%d,%m,%Y", DateTime(0, 0, 0, 0, 0, 0, 0)}, + {"%09:30:17", "%%%h:%i:%s", DateTime(0, 0, 0, 9, 30, 17, 0)}, + {"2009 foobar", "%Y", DateTime(2009, 0, 0, 0, 0, 0, 0)}, + {"2009-12-31", "%h:%i:%s %p", DateTime(0, 0, 0, 0, 0, 0, 0)}, + {"12:00:01 AM", "%h:%i:%s %p", DateTime(0, 0, 0, 0, 0, 1, 0)}, + {"12:00:01 PM", "%h:%i:%s %p", DateTime(0, 0, 0, 12, 0, 1, 0)}, + {"11:00:01 PM", "%h:%i:%s %p", DateTime(0, 0, 0, 23, 0, 1, 0)}, + {"11:00:01 AM", "%h:%i:%s %p", DateTime(0, 0, 0, 11, 0, 1, 0)}, + {"10000-01-02", "%Y:%m:%d", DateTime(0, 0, 0, 0, 0, 0, 0)}, + }; + + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) + { + DateTime dt; + dt.year = 0; + dt.month = 0; + dt.day = 0; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + + TimeExtractor extractor; + int ret = extractor.extractTime(date_tests[i].inputstr, date_tests[i].formatstr, dt); + + if (ret != 0) + cout << "Extractor reported error for " << date_tests[i].inputstr << "," << date_tests[i].formatstr + << endl; + + bool check = + ((*(reinterpret_cast(&date_tests[i].date))) == ((*(reinterpret_cast(&dt))))); + + if (!check) + { + printf("For input \"%s\", format \"%s\", check 0x%016lx vs 0x%016lx\n", date_tests[i].inputstr, + date_tests[i].formatstr, *(reinterpret_cast(&dt)), + *(reinterpret_cast(&date_tests[i].date))); + show_datetime_debugs("check", dt); + show_datetime_debugs("ref", date_tests[i].date); + } + + CPPUNIT_ASSERT(check); } + } - void tearDown() + void fe_getnumbertest() + { + const int MAX_VALS = 5; + struct Check { - } + const char* str; + int numvals; + int vals[MAX_VALS]; + }; - void show_datetime_debugs( const string& nm, const DateTime& d1 ) + Check tests[] = { + {" ", 0, {}}, + {"0", 1, {0}}, + {"0.0", 2, {0, 0}}, + {"0:0.0", 3, {0, 0, 0}}, + {"-0", 1, {0}}, + {"0000-000", 1, {0}}, + {"-2", 1, {-2}}, + {"2223", 1, {2223}}, + {"37", 1, {37}}, + {"393", 1, {393}}, + {"39-3", 1, {39}}, + {"17-", 1, {17}}, + {"10.2303", 2, {10, 2303}}, + {"20:10.2303", 3, {20, 10, 2303}}, + {"17:37", 2, {17, 37}}, + }; + + int a1[MAX_VALS] = {0xdeadbeef}; + + for (unsigned i = 0; i < sizeof(tests) / sizeof(Check); i++) { - cout << nm << ".day = " << setw(10) << d1.day << endl; - cout << nm << ".month = " << setw(10) << d1.month << endl; - cout << nm << ".year = " << setw(10) << d1.year << endl; - cout << nm << ".hour = " << setw(10) << d1.hour << endl; - cout << nm << ".minute = " << setw(10) << d1.minute << endl; - cout << nm << ".second = " << setw(10) << d1.second << endl; - cout << nm << ".msecond = " << setw(10) << d1.msecond << endl; - } + int rc1 = helpers::getNumbers(tests[i].str, a1, execplan::OP_ADD); - void fe_strtodatetest() - { - struct DateCheck + cout << "For input \"" << tests[i].str << "\", numbers = " << rc1 << ",["; + + bool check = true; + + if (rc1 != tests[i].numvals) + check = false; + + for (int j = 0; j < rc1; ++j) + { + if (j > 0) + cout << ','; + + cout << a1[j]; + + if (a1[j] != tests[i].vals[j]) { - const char* inputstr; - const char* formatstr; - DateTime date; - }; - - DateCheck date_tests[] = - { - { "2009", "%Y", DateTime(2009, 0, 0, 0, 0, 0, 0) }, - { " 2009", "%Y", DateTime(2009, 0, 0, 0, 0, 0, 0) }, - { " 2009", " %Y", DateTime(2009, 0, 0, 0, 0, 0, 0) }, - { "jan", "%b", DateTime(0, 1, 0, 0, 0, 0, 0) }, - { "2009feb", "%Y%b", DateTime(2009, 2, 0, 0, 0, 0, 0) }, - { " 2009 ApR", " %Y %b", DateTime(2009, 4, 0, 0, 0, 0, 0) }, - { "200910", "%Y%m", DateTime(2009, 10, 0, 0, 0, 0, 0) }, - { " 2009 10", " %Y %m", DateTime(2009, 10, 0, 0, 0, 0, 0) }, - { "200910", "%Y%c", DateTime(2009, 10, 0, 0, 0, 0, 0) }, - { " 2009 10", " %Y %c", DateTime(2009, 10, 0, 0, 0, 0, 0) }, - { "01,5,2013", "%d,%m,%Y", DateTime(2013, 5, 1, 0, 0, 0, 0) }, - { "01,5,2013", "%e,%m,%Y", DateTime(2013, 5, 1, 0, 0, 0, 0) }, - { "7,5th,2012", "%m,%D,%Y", DateTime(2012, 7, 5, 0, 0, 0, 0) }, - { "a09:30:17", "a%h:%i", DateTime(0, 0, 0, 9, 30, 0, 0) }, - { "a09:30:17", "%h:%i", DateTime(0, 0, 0, 0, 0, 0, 0) }, - { "9:30.170", "%h:%i.%f", DateTime(0, 0, 0, 9, 30, 0, 170000) }, - { "178546 17", "%f %H", DateTime(0, 0, 0, 17, 0, 0, 178546) }, - { "2009march415", "%Y%b", DateTime(2009, 3, 0, 0, 0, 0, 0) }, - { " 2009 July", " %Y %b", DateTime(2009, 7, 0, 0, 0, 0, 0) }, - { "09 pm", "%h %p", DateTime(0, 0, 0, 21, 0, 0, 0) }, - { "09:13:14pm", "%r", DateTime(0, 0, 0, 9, 13, 14, 0) }, - { "12:13:14am", "%r", DateTime(0, 0, 0, 12, 13, 14, 0) }, - { "92", " %Y", DateTime(1992, 0, 0, 0, 0, 0, 0) }, - { "9", " %Y", DateTime(2009, 0, 0, 0, 0, 0, 0) }, - { "2013 31", "%Y %j", DateTime(2013, 1, 31, 0, 0, 0, 0) }, - { "2013 1 3", "%Y %U %w", DateTime(2013, 1, 9, 0, 0, 0, 0) }, - { "2013 1 3", "%Y %u %w", DateTime(2013, 1, 9, 0, 0, 0, 0) }, - { "2007 1 3", "%Y %U %w", DateTime(2007, 1, 10, 0, 0, 0, 0) }, - { "2007 1 3", "%Y %u %w", DateTime(2007, 1, 3, 0, 0, 0, 0) }, - { "2013 1 3", "%X %V %w", DateTime(2013, 1, 9, 0, 0, 0, 0) }, - { "2013 1 3", "%X %v %w", DateTime(2013, 1, 9, 0, 0, 0, 0) }, - { "2007 1 3", "%X %V %w", DateTime(2007, 1, 10, 0, 0, 0, 0) }, - { "2007 1 3", "%X %v %w", DateTime(2007, 1, 3, 0, 0, 0, 0) }, - { "33:30.170", "%h:%i.%f", DateTime(0, 0, 0, 0, 0, 0, 0) }, - { "01,25,2013", "%d,%m,%Y", DateTime(0, 0, 0, 0, 0, 0, 0) }, - { "%09:30:17", "%%%h:%i:%s", DateTime(0, 0, 0, 9, 30, 17, 0) }, - { "2009 foobar", "%Y", DateTime(2009, 0, 0, 0, 0, 0, 0) }, - { "2009-12-31", "%h:%i:%s %p", DateTime(0, 0, 0, 0, 0, 0, 0) }, - { "12:00:01 AM", "%h:%i:%s %p", DateTime(0, 0, 0, 0, 0, 1, 0) }, - { "12:00:01 PM", "%h:%i:%s %p", DateTime(0, 0, 0, 12, 0, 1, 0) }, - { "11:00:01 PM", "%h:%i:%s %p", DateTime(0, 0, 0, 23, 0, 1, 0) }, - { "11:00:01 AM", "%h:%i:%s %p", DateTime(0, 0, 0, 11, 0, 1, 0) }, - { "10000-01-02", "%Y:%m:%d", DateTime(0, 0, 0, 0, 0, 0, 0) }, - }; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - DateTime dt; - dt.year = 0; - dt.month = 0; - dt.day = 0; - dt.hour = 0; - dt.minute = 0; - dt.second = 0; - dt.msecond = 0; - - TimeExtractor extractor; - int ret = extractor.extractTime(date_tests[i].inputstr, date_tests[i].formatstr, dt); - - if ( ret != 0 ) - cout << "Extractor reported error for " - << date_tests[i].inputstr << "," << date_tests[i].formatstr << endl; - - bool check = ((*(reinterpret_cast (&date_tests[i].date))) == - ((*(reinterpret_cast (&dt))))); - - if (!check) - { - printf("For input \"%s\", format \"%s\", check 0x%016lx vs 0x%016lx\n", - date_tests[i].inputstr, - date_tests[i].formatstr, - *(reinterpret_cast (&dt)), - *(reinterpret_cast(&date_tests[i].date)) ); - show_datetime_debugs("check", dt); - show_datetime_debugs("ref", date_tests[i].date); - } - - CPPUNIT_ASSERT( check ); + check = false; } + } + cout << ']' << endl; + + CPPUNIT_ASSERT(check); } + } - void fe_getnumbertest() + void fe_dateaddtest() + { + struct DateCheck { - const int MAX_VALS = 5; - struct Check - { - const char* str; - int numvals; - int vals[MAX_VALS]; - }; + const char* intervalstr; + IntervalColumn::interval_type unit; + OpType funcType; + DateTime date; + DateTime ref; + }; - Check tests[] = - { - { " ", 0, {} }, - { "0", 1, {0} }, - { "0.0", 2, {0, 0} }, - { "0:0.0", 3, {0, 0, 0} }, - { "-0", 1, {0} }, - { "0000-000", 1, {0} }, - { "-2", 1, {-2} }, - { "2223", 1, {2223} }, - { "37", 1, {37} }, - { "393", 1, {393} }, - { "39-3", 1, {39} }, - { "17-", 1, {17} }, - { "10.2303", 2, {10, 2303} }, - { "20:10.2303", 3, {20, 10, 2303} }, - { "17:37", 2, {17, 37} }, - }; + DateCheck date_tests[] = { + {"111111:22", IntervalColumn::INTERVAL_YEAR_MONTH, execplan::OP_ADD, + DateTime(1997, 01, 01, 0, 0, 0, 0), DateTime(0, 0, 0, 0, 0, 0, 0)}, + {"-30", IntervalColumn::INTERVAL_DAY, execplan::OP_ADD, DateTime(2013, 3, 1, 0, 0, 0, 0), + DateTime(2013, 1, 30, 0, 0, 0, 0)}, + }; - int a1[MAX_VALS] = { 0xdeadbeef }; - - for (unsigned i = 0; i < sizeof(tests) / sizeof(Check); i++) - { - int rc1 = helpers::getNumbers(tests[i].str, a1, execplan::OP_ADD); - - cout << "For input \"" << tests[i].str << "\", numbers = " << rc1 << ",["; - - bool check = true; - - if (rc1 != tests[i].numvals) - check = false; - - for (int j = 0; j < rc1; ++j) - { - if (j > 0) - cout << ','; - - cout << a1[j]; - - if (a1[j] != tests[i].vals[j]) - { - check = false; - } - } - - cout << ']' << endl; - - CPPUNIT_ASSERT( check ); - } - } - - void fe_dateaddtest() + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) { - struct DateCheck - { - const char* intervalstr; - IntervalColumn::interval_type unit; - OpType funcType; - DateTime date; - DateTime ref; - }; + DateTime dt; - DateCheck date_tests[] = - { - { "111111:22", IntervalColumn::INTERVAL_YEAR_MONTH, execplan::OP_ADD, DateTime(1997, 01, 01, 0, 0, 0, 0), DateTime(0, 0, 0, 0, 0, 0, 0) }, - { "-30", IntervalColumn::INTERVAL_DAY, execplan::OP_ADD, DateTime(2013, 3, 1, 0, 0, 0, 0), DateTime(2013, 1, 30, 0, 0, 0, 0) }, - }; + uint64_t timeval = *(reinterpret_cast(&date_tests[i].date)); + uint64_t value = helpers::dateAdd(timeval, date_tests[i].intervalstr, date_tests[i].unit, false, + date_tests[i].funcType); - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - DateTime dt; + *(reinterpret_cast(&dt)) = value; - uint64_t timeval = *(reinterpret_cast (&date_tests[i].date)); - uint64_t value = helpers::dateAdd( timeval, date_tests[i].intervalstr, date_tests[i].unit, false, date_tests[i].funcType ); + bool check = (value == *(reinterpret_cast(&date_tests[i].ref))); - *(reinterpret_cast (&dt)) = value; + if (!check) + { + printf("For interval \"%s\", check 0x%016lx vs 0x%016lx\n", date_tests[i].intervalstr, + *(reinterpret_cast(&dt)), *(reinterpret_cast(&date_tests[i].ref))); + show_datetime_debugs("check", dt); + show_datetime_debugs("ref", date_tests[i].ref); + } - bool check = ( value == *(reinterpret_cast (&date_tests[i].ref))); - - if (!check) - { - printf("For interval \"%s\", check 0x%016lx vs 0x%016lx\n", - date_tests[i].intervalstr, - *(reinterpret_cast (&dt)), - *(reinterpret_cast(&date_tests[i].ref)) ); - show_datetime_debugs("check", dt); - show_datetime_debugs("ref", date_tests[i].ref); - } - - CPPUNIT_ASSERT( check ); - } + CPPUNIT_ASSERT(check); } + } - void fe_daynametest() + void fe_daynametest() + { + struct DateCheck { - struct DateCheck - { - DateTime date; - const char* dayname; - }; + DateTime date; + const char* dayname; + }; - DateCheck date_tests[] = - { - { DateTime(1997, 01, 01, 0, 0, 0, 0), "Wednesday" }, - }; + DateCheck date_tests[] = { + {DateTime(1997, 01, 01, 0, 0, 0, 0), "Wednesday"}, + }; - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - boost::gregorian::date d(date_tests[i].date.year, date_tests[i].date.month, date_tests[i].date.day); - uint32_t dayofweek = helpers::calc_mysql_weekday(date_tests[i].date.year, date_tests[i].date.month, date_tests[i].date.day, false); - - bool check = ( strcmp( helpers::weekdayFullNames[dayofweek].c_str(), date_tests[i].dayname ) == 0 ); - - if (!check) - { - printf("For date %s, check %s vs %s\n", - to_simple_string(d).c_str(), - helpers::weekdayFullNames[dayofweek].c_str(), - date_tests[i].dayname ); - } - - CPPUNIT_ASSERT( check ); - } - } - - DateTime getDateTime(int64_t val) + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) { - if (val < 0 || val > helpers::TIMESTAMP_MAX_VALUE) - return 0; + boost::gregorian::date d(date_tests[i].date.year, date_tests[i].date.month, date_tests[i].date.day); + uint32_t dayofweek = helpers::calc_mysql_weekday(date_tests[i].date.year, date_tests[i].date.month, + date_tests[i].date.day, false); - DateTime dt; + bool check = (strcmp(helpers::weekdayFullNames[dayofweek].c_str(), date_tests[i].dayname) == 0); - struct tm tmp_tm; - time_t tmp_t = (time_t)val; - localtime_r(&tmp_t, &tmp_tm); + if (!check) + { + printf("For date %s, check %s vs %s\n", to_simple_string(d).c_str(), + helpers::weekdayFullNames[dayofweek].c_str(), date_tests[i].dayname); + } - //to->neg=0; - dt.year = (int64_t) ((tmp_tm.tm_year + 1900) % 10000); - dt.month = (int64_t) tmp_tm.tm_mon + 1; - dt.day = (int64_t) tmp_tm.tm_mday; - dt.hour = (int64_t) tmp_tm.tm_hour; - dt.minute = (int64_t) tmp_tm.tm_min; - dt.second = (int64_t) tmp_tm.tm_sec; - dt.msecond = 0; - return dt; + CPPUNIT_ASSERT(check); } + } - void fe_fromunixtimetest() + DateTime getDateTime(int64_t val) + { + if (val < 0 || val > helpers::TIMESTAMP_MAX_VALUE) + return 0; + + DateTime dt; + + struct tm tmp_tm; + time_t tmp_t = (time_t)val; + localtime_r(&tmp_t, &tmp_tm); + + // to->neg=0; + dt.year = (int64_t)((tmp_tm.tm_year + 1900) % 10000); + dt.month = (int64_t)tmp_tm.tm_mon + 1; + dt.day = (int64_t)tmp_tm.tm_mday; + dt.hour = (int64_t)tmp_tm.tm_hour; + dt.minute = (int64_t)tmp_tm.tm_min; + dt.second = (int64_t)tmp_tm.tm_sec; + dt.msecond = 0; + return dt; + } + + void fe_fromunixtimetest() + { + struct DateCheck { - struct DateCheck - { - int64_t ts; - const char* formatstr; - const char* refstr; - }; + int64_t ts; + const char* formatstr; + const char* refstr; + }; - DateCheck date_tests[] = - { - { -26, "%H:%i:%s", "NULL" }, - { 26, "%H:%i:%s", "18:00:26" }, - { 0, "%H:%i:%s", "18:00:00" }, - }; + DateCheck date_tests[] = { + {-26, "%H:%i:%s", "NULL"}, + {26, "%H:%i:%s", "18:00:26"}, + {0, "%H:%i:%s", "18:00:00"}, + }; - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - bool check = false; - string ret; - DateTime dt = getDateTime(date_tests[i].ts); + for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) + { + bool check = false; + string ret; + DateTime dt = getDateTime(date_tests[i].ts); - if (*reinterpret_cast(&dt) == 0) - { - check = ( strcmp( date_tests[i].refstr, "NULL" ) == 0 ); - } - else - { - ret = helpers::IDB_date_format(dt, date_tests[i].formatstr); - check = ( strcmp( date_tests[i].refstr, ret.c_str() ) == 0 ); - } + if (*reinterpret_cast(&dt) == 0) + { + check = (strcmp(date_tests[i].refstr, "NULL") == 0); + } + else + { + ret = helpers::IDB_date_format(dt, date_tests[i].formatstr); + check = (strcmp(date_tests[i].refstr, ret.c_str()) == 0); + } - if (!check) - { - printf("For timestamp %ld, check %s vs %s\n", - date_tests[i].ts, - ret.c_str(), - date_tests[i].refstr ); - } + if (!check) + { + printf("For timestamp %ld, check %s vs %s\n", date_tests[i].ts, ret.c_str(), date_tests[i].refstr); + } - CPPUNIT_ASSERT( check ); - } + CPPUNIT_ASSERT(check); } - + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( FuncExpTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(FuncExpTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - - diff --git a/utils/funcexp/timeextract.h b/utils/funcexp/timeextract.h index 192069f58..5e0fbc792 100644 --- a/utils/funcexp/timeextract.h +++ b/utils/funcexp/timeextract.h @@ -21,502 +21,499 @@ namespace funcexp { - class TimeExtractor { -private: - int32_t dayOfWeek; - int32_t dayOfYear; - int32_t weekOfYear; - bool sundayFirst; + private: + int32_t dayOfWeek; + int32_t dayOfYear; + int32_t weekOfYear; + bool sundayFirst; -public: - TimeExtractor() : - dayOfWeek( -1 ), - dayOfYear( -1 ), - weekOfYear( -1 ), - sundayFirst( false ) {;} + public: + TimeExtractor() : dayOfWeek(-1), dayOfYear(-1), weekOfYear(-1), sundayFirst(false) + { + ; + } - /** - * extractTime is an implementation that matches MySQL behavior for the - * STR_TO_DATE() function. See MySQL documentation for details. - * - * Returns 0 on success, -1 on failure. On failure the DateTime is - * reset to 0s in all fields. - */ - int extractTime (const std::string& valStr, const std::string& formatStr, dataconvert::DateTime& dateTime) + /** + * extractTime is an implementation that matches MySQL behavior for the + * STR_TO_DATE() function. See MySQL documentation for details. + * + * Returns 0 on success, -1 on failure. On failure the DateTime is + * reset to 0s in all fields. + */ + int extractTime(const std::string& valStr, const std::string& formatStr, dataconvert::DateTime& dateTime) + { + uint32_t fcur = 0; + uint32_t vcur = 0; + + while (fcur != formatStr.length()) { - uint32_t fcur = 0; - uint32_t vcur = 0; - - while ( fcur != formatStr.length() ) - { - if ( !handleNextToken(valStr, vcur, formatStr, fcur, dateTime) ) - return returnError( dateTime ); - } - - if ( dayOfYear > 0 ) - { - // they set day of year - we also need to make sure there is a year to work with - if ( !dataconvert::isDateValid( 1, 1, dateTime.year ) ) - return returnError( dateTime ); - - helpers::get_date_from_mysql_daynr( helpers::calc_mysql_daynr( dateTime.year, 1, 1 ) + dayOfYear - 1, dateTime ); - } - else if ( weekOfYear > 0 ) - { - if ( dayOfWeek < 0 || !dataconvert::isDateValid( 1, 1, dateTime.year ) ) - return returnError( dateTime ); - - uint32_t yearfirst = helpers::calc_mysql_daynr(dateTime.year, 1, 1); - // figure out which day of week Jan-01 is - uint32_t firstweekday = helpers::calc_mysql_weekday( dateTime.year, 1, 1, sundayFirst ); - - // calculate the offset to the first week starting day - uint32_t firstoffset = firstweekday ? ( 7 - firstweekday ) : 0; - - firstoffset += ( ( weekOfYear - 1) * 7 ) + dayOfWeek - ( sundayFirst ? 0 : 1 ); - yearfirst += firstoffset; - helpers::get_date_from_mysql_daynr(yearfirst, dateTime); - } - - if ( !dataconvert::isDateTimeValid( dateTime.hour, dateTime.minute, dateTime.second, dateTime.msecond ) ) - return returnError( dateTime ); - - return 0; + if (!handleNextToken(valStr, vcur, formatStr, fcur, dateTime)) + return returnError(dateTime); } -private: - int returnError( dataconvert::DateTime& dateTime ) + if (dayOfYear > 0) { - (*(reinterpret_cast (&dateTime))) = (uint64_t) 0; - return -1; + // they set day of year - we also need to make sure there is a year to work with + if (!dataconvert::isDateValid(1, 1, dateTime.year)) + return returnError(dateTime); + + helpers::get_date_from_mysql_daynr(helpers::calc_mysql_daynr(dateTime.year, 1, 1) + dayOfYear - 1, + dateTime); + } + else if (weekOfYear > 0) + { + if (dayOfWeek < 0 || !dataconvert::isDateValid(1, 1, dateTime.year)) + return returnError(dateTime); + + uint32_t yearfirst = helpers::calc_mysql_daynr(dateTime.year, 1, 1); + // figure out which day of week Jan-01 is + uint32_t firstweekday = helpers::calc_mysql_weekday(dateTime.year, 1, 1, sundayFirst); + + // calculate the offset to the first week starting day + uint32_t firstoffset = firstweekday ? (7 - firstweekday) : 0; + + firstoffset += ((weekOfYear - 1) * 7) + dayOfWeek - (sundayFirst ? 0 : 1); + yearfirst += firstoffset; + helpers::get_date_from_mysql_daynr(yearfirst, dateTime); } - bool scanDecimalVal(const char* nptr, const char** endptr, int32_t& value) + if (!dataconvert::isDateTimeValid(dateTime.hour, dateTime.minute, dateTime.second, dateTime.msecond)) + return returnError(dateTime); + + return 0; + } + + private: + int returnError(dataconvert::DateTime& dateTime) + { + (*(reinterpret_cast(&dateTime))) = (uint64_t)0; + return -1; + } + + bool scanDecimalVal(const char* nptr, const char** endptr, int32_t& value) + { + value = 0; + const char* p = nptr; + + while (p < *endptr && isdigit(*p)) { - value = 0; - const char* p = nptr; - - while ( p < *endptr && isdigit( *p ) ) - { - value = value * 10 + ((*p) - '0'); - ++p; - } - - *endptr = p; - return (*endptr != nptr); + value = value * 10 + ((*p) - '0'); + ++p; } - bool handleNextToken(const std::string& valStr, uint32_t& vptr, const std::string& formatStr, uint32_t& fptr, dataconvert::DateTime& dateTime) + *endptr = p; + return (*endptr != nptr); + } + + bool handleNextToken(const std::string& valStr, uint32_t& vptr, const std::string& formatStr, + uint32_t& fptr, dataconvert::DateTime& dateTime) + { + // advance both strings to the first non-whitespace character + while (valStr[vptr] == ' ' && vptr < valStr.length()) + ++vptr; + + bool vend = (vptr == valStr.length()); + + while (formatStr[fptr] == ' ' && fptr < formatStr.length()) + ++fptr; + + bool fend = (fptr == formatStr.length()); + + if (vend && fend) { - // advance both strings to the first non-whitespace character - while ( valStr[vptr] == ' ' && vptr < valStr.length() ) - ++vptr; + // apparent trailing whitespace + return true; + } + else if (!vend && !fend) + { + if (formatStr[fptr] == '%') + { + // has to be at least one more character in format string + if (fptr >= formatStr.length() - 1) + return false; - bool vend = (vptr == valStr.length()); + fptr++; // skip over % - while ( formatStr[fptr] == ' ' && fptr < formatStr.length() ) - ++fptr; - - bool fend = (fptr == formatStr.length()); - - if ( vend && fend ) + // check for special case of %% + if (formatStr[fptr] == '%') { - // apparent trailing whitespace - return true; - } - else if ( !vend && !fend ) - { - if ( formatStr[fptr] == '%' ) - { - // has to be at least one more character in format string - if ( fptr >= formatStr.length() - 1) - return false; - - fptr++; // skip over % - - // check for special case of %% - if ( formatStr[fptr] == '%' ) - { - bool ret = formatStr[fptr] == valStr[vptr]; - ++fptr; - ++vptr; - return ret; - } - else - { - char field = formatStr[fptr]; - ++fptr; // also skip the format code - bool ret = handleField( field, valStr, vptr, dateTime ); - return ret; - } - } - else - { - bool ret = formatStr[fptr] == valStr[vptr]; - ++fptr; - ++vptr; - return ret; - } + bool ret = formatStr[fptr] == valStr[vptr]; + ++fptr; + ++vptr; + return ret; } else { - // one string finish before the other one - not good - return false; + char field = formatStr[fptr]; + ++fptr; // also skip the format code + bool ret = handleField(field, valStr, vptr, dateTime); + return ret; } + } + else + { + bool ret = formatStr[fptr] == valStr[vptr]; + ++fptr; + ++vptr; + return ret; + } } - - bool handleField(char field, const std::string& valStr, uint32_t& vptr, dataconvert::DateTime& dateTime) + else { - int32_t value; - const char* valptr = valStr.c_str() + vptr; + // one string finish before the other one - not good + return false; + } + } - switch ( field ) + bool handleField(char field, const std::string& valStr, uint32_t& vptr, dataconvert::DateTime& dateTime) + { + int32_t value; + const char* valptr = valStr.c_str() + vptr; + + switch (field) + { + case 'a': + { + // weekday abbreviations are always exactly 3 characters + std::string weekday_str(valStr, vptr, 3); + vptr += 3; + dayOfWeek = helpers::dayOfWeek(weekday_str); + + if (dayOfWeek < 0) { - case 'a': - { - // weekday abbreviations are always exactly 3 characters - std::string weekday_str( valStr, vptr, 3 ); - vptr += 3; - dayOfWeek = helpers::dayOfWeek(weekday_str); - - if ( dayOfWeek < 0 ) - { - return false; - } - - break; - } - - case 'b': - { - // month abbreviations are always exactly 3 characters - std::string month_str( valStr, vptr, 3 ); - vptr += 3; - value = helpers::convertMonth(month_str); - - if ( value < 0 ) - { - return false; - } - else - { - dateTime.month = value; - } - - break; - } - - case 'c': - case 'm': - { - // Month, numeric (0..12) - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - - // we have to do range checking on the month value here because - // dateTime will arbitrarily truncate to 4 bits and may turn a - // bad value into a good one - if ( value < 0 || value > 12 ) - return false; - - dateTime.month = value; - break; - } - - case 'D': - case 'd': - case 'e': - { - // %D - Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - // %d - Day of the month, numeric (00..31) - // %e - Day of the month, numeric (0..31) - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - - // now also skip suffix if required - always 2 characters - if ( field == 'D') - vptr += 2; - - // we have to do range checking on the month value here because - // dateTime will arbitrarily truncate to 6 bits and may turn a - // bad value into a good one - if ( value < 0 || value > 31 ) - return false; - - dateTime.day = value; - break; - } - - case 'f': - { - // Microseconds (000000..999999) - const char* vend = valptr + min(6, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - - for ( int i = (vend - valptr); i < 6; ++i ) - value = value * 10; - - dateTime.msecond = value; - break; - } - - case 'H': - case 'k': - { - // Hour (00..23) - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - dateTime.hour = value; - break; - } - - case 'h': - case 'I': - case 'l': - { - // Hour (01..12) - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - dateTime.hour = value; - break; - } - - case 'i': - { - // Minutes, numeric (00..59) - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - dateTime.minute = value; - break; - } - - case 'j': - { - // Day of year (001..366) - const char* vend = valptr + min(3, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - - if ( value < 1 || value > 366 ) - return false; - - dayOfYear = value; - break; - } - - case 'M': - { - // Month name (January..December) - // look for the first non-alphabetic character - size_t endpos = vptr; - - while ( tolower(valStr[endpos]) >= 'a' && tolower(valStr[endpos]) <= 'z' && endpos < valStr.length() ) - ++endpos; - - std::string month_str( valStr, vptr, endpos ); - vptr += month_str.length(); - value = helpers::convertMonth(month_str); - - if ( value < 0 ) - { - return false; - } - else - { - dateTime.month = value; - } - - break; - } - - case 'p': - { - // AM or PM - if ( tolower(valStr[vptr]) == 'p' && tolower(valStr[vptr + 1]) == 'm' ) - { - if ( dateTime.hour < 12 ) - dateTime.hour += 12; - } - else if ( tolower(valStr[vptr]) == 'a' && tolower(valStr[vptr + 1]) == 'm' ) - { - if ( dateTime.hour == 12 ) - dateTime.hour = 0; - } - else - { - vptr += 2; - return false; - } - - vptr += 2; - break; - } - - case 'r': - case 'T': - { - // Time, 12-hour (hh:mm:ss followed by AM or PM) - // Time, 24-hour (hh:mm:ss) - int32_t hour = -1; - int32_t min = -1; - int32_t sec = -1; - int32_t numread; - int32_t sscanf_ck; - - if ( ( sscanf_ck = sscanf( valptr, "%2d:%2d:%2d%n", &hour, &min, &sec, &numread) ) != 3 ) - { - return false; - } - - valptr += numread; - vptr += numread; - - dateTime.hour = hour; - dateTime.minute = min; - dateTime.second = sec; - break; - } - - case 'S': - case 's': - { - // Seconds (00..59) - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - dateTime.second = value; - break; - } - - case 'U': - case 'u': - case 'V': - case 'v': - { - // %U - Week (00..53), where Sunday is the first day of the week - // %u - Week (00..53), where Monday is the first day of the week - // %V - Week (01..53), where Sunday is the first day of the week; used with %X - // %v - Week (01..53), where Monday is the first day of the week; used with %x - sundayFirst = (isupper(field) != 0); - const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - weekOfYear = value; - break; - } - - case 'X': - case 'x': - case 'Y': - case 'y': - { - // %X - Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - // %x - Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v - // %Y - Year, numeric, four digits - // %y - Year, numeric (two digits) - sundayFirst = ( field == 'X' ); - int minFieldWidth = ( field == 'y' ? 2 : 4 ); - const char* vend = valptr + min( minFieldWidth, (int)(valStr.length() - vptr) ); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - - if ( (vend - valptr) <= 2 ) - { - // regardless of whether field was supposed to be 4 characters. - // If we read two then apply year 2000 handling - value += 2000; - - if ( value > 2069 ) - value -= 100; - } - - dateTime.year = value; - break; - } - - case 'W': - { - // Weekday name (Sunday..Saturday) - // look for the first non-alphabetic character - size_t endpos = vptr; - - while ( tolower(valStr[endpos]) >= 'a' && tolower(valStr[endpos]) <= 'z' && endpos < valStr.length() ) - ++endpos; - - std::string weekday_str( valStr, vptr, endpos ); - vptr += weekday_str.length(); - value = helpers::dayOfWeek(weekday_str); - - if ( value < 0 ) - { - return false; - } - else - { - dayOfWeek = value; - } - - break; - } - - case 'w': - { - // Day of the week (0=Sunday..6=Saturday) - const char* vend = valptr + min(1, (int)(valStr.length() - vptr)); - - if ( !scanDecimalVal( valptr, &vend, value ) ) - return false; - - vptr += (vend - valptr); - dayOfWeek = value; - break; - } - - default: - return false; + return false; } - return true; + break; + } + + case 'b': + { + // month abbreviations are always exactly 3 characters + std::string month_str(valStr, vptr, 3); + vptr += 3; + value = helpers::convertMonth(month_str); + + if (value < 0) + { + return false; + } + else + { + dateTime.month = value; + } + + break; + } + + case 'c': + case 'm': + { + // Month, numeric (0..12) + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + + // we have to do range checking on the month value here because + // dateTime will arbitrarily truncate to 4 bits and may turn a + // bad value into a good one + if (value < 0 || value > 12) + return false; + + dateTime.month = value; + break; + } + + case 'D': + case 'd': + case 'e': + { + // %D - Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) + // %d - Day of the month, numeric (00..31) + // %e - Day of the month, numeric (0..31) + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + + // now also skip suffix if required - always 2 characters + if (field == 'D') + vptr += 2; + + // we have to do range checking on the month value here because + // dateTime will arbitrarily truncate to 6 bits and may turn a + // bad value into a good one + if (value < 0 || value > 31) + return false; + + dateTime.day = value; + break; + } + + case 'f': + { + // Microseconds (000000..999999) + const char* vend = valptr + min(6, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + + for (int i = (vend - valptr); i < 6; ++i) + value = value * 10; + + dateTime.msecond = value; + break; + } + + case 'H': + case 'k': + { + // Hour (00..23) + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + dateTime.hour = value; + break; + } + + case 'h': + case 'I': + case 'l': + { + // Hour (01..12) + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + dateTime.hour = value; + break; + } + + case 'i': + { + // Minutes, numeric (00..59) + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + dateTime.minute = value; + break; + } + + case 'j': + { + // Day of year (001..366) + const char* vend = valptr + min(3, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + + if (value < 1 || value > 366) + return false; + + dayOfYear = value; + break; + } + + case 'M': + { + // Month name (January..December) + // look for the first non-alphabetic character + size_t endpos = vptr; + + while (tolower(valStr[endpos]) >= 'a' && tolower(valStr[endpos]) <= 'z' && endpos < valStr.length()) + ++endpos; + + std::string month_str(valStr, vptr, endpos); + vptr += month_str.length(); + value = helpers::convertMonth(month_str); + + if (value < 0) + { + return false; + } + else + { + dateTime.month = value; + } + + break; + } + + case 'p': + { + // AM or PM + if (tolower(valStr[vptr]) == 'p' && tolower(valStr[vptr + 1]) == 'm') + { + if (dateTime.hour < 12) + dateTime.hour += 12; + } + else if (tolower(valStr[vptr]) == 'a' && tolower(valStr[vptr + 1]) == 'm') + { + if (dateTime.hour == 12) + dateTime.hour = 0; + } + else + { + vptr += 2; + return false; + } + + vptr += 2; + break; + } + + case 'r': + case 'T': + { + // Time, 12-hour (hh:mm:ss followed by AM or PM) + // Time, 24-hour (hh:mm:ss) + int32_t hour = -1; + int32_t min = -1; + int32_t sec = -1; + int32_t numread; + int32_t sscanf_ck; + + if ((sscanf_ck = sscanf(valptr, "%2d:%2d:%2d%n", &hour, &min, &sec, &numread)) != 3) + { + return false; + } + + valptr += numread; + vptr += numread; + + dateTime.hour = hour; + dateTime.minute = min; + dateTime.second = sec; + break; + } + + case 'S': + case 's': + { + // Seconds (00..59) + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + dateTime.second = value; + break; + } + + case 'U': + case 'u': + case 'V': + case 'v': + { + // %U - Week (00..53), where Sunday is the first day of the week + // %u - Week (00..53), where Monday is the first day of the week + // %V - Week (01..53), where Sunday is the first day of the week; used with %X + // %v - Week (01..53), where Monday is the first day of the week; used with %x + sundayFirst = (isupper(field) != 0); + const char* vend = valptr + min(2, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + weekOfYear = value; + break; + } + + case 'X': + case 'x': + case 'Y': + case 'y': + { + // %X - Year for the week where Sunday is the first day of the week, numeric, four digits; used with + // %V %x - Year for the week, where Monday is the first day of the week, numeric, four digits; used + // with %v %Y - Year, numeric, four digits %y - Year, numeric (two digits) + sundayFirst = (field == 'X'); + int minFieldWidth = (field == 'y' ? 2 : 4); + const char* vend = valptr + min(minFieldWidth, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + + if ((vend - valptr) <= 2) + { + // regardless of whether field was supposed to be 4 characters. + // If we read two then apply year 2000 handling + value += 2000; + + if (value > 2069) + value -= 100; + } + + dateTime.year = value; + break; + } + + case 'W': + { + // Weekday name (Sunday..Saturday) + // look for the first non-alphabetic character + size_t endpos = vptr; + + while (tolower(valStr[endpos]) >= 'a' && tolower(valStr[endpos]) <= 'z' && endpos < valStr.length()) + ++endpos; + + std::string weekday_str(valStr, vptr, endpos); + vptr += weekday_str.length(); + value = helpers::dayOfWeek(weekday_str); + + if (value < 0) + { + return false; + } + else + { + dayOfWeek = value; + } + + break; + } + + case 'w': + { + // Day of the week (0=Sunday..6=Saturday) + const char* vend = valptr + min(1, (int)(valStr.length() - vptr)); + + if (!scanDecimalVal(valptr, &vend, value)) + return false; + + vptr += (vend - valptr); + dayOfWeek = value; + break; + } + + default: return false; } + return true; + } }; -} +} // namespace funcexp diff --git a/utils/funcexp/utf8.h b/utils/funcexp/utf8.h index 82b13f59f..13bb7953a 100644 --- a/utils/funcexp/utf8.h +++ b/utils/funcexp/utf8.h @@ -24,11 +24,10 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 #define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 #include "utf8/checked.h" #include "utf8/unchecked.h" -#endif // header guard +#endif // header guard diff --git a/utils/funcexp/utf8/checked.h b/utils/funcexp/utf8/checked.h index b18862d9e..259d3acf4 100644 --- a/utils/funcexp/utf8/checked.h +++ b/utils/funcexp/utf8/checked.h @@ -24,7 +24,6 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 #define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 @@ -41,56 +40,65 @@ class exception : public ::std::exception // Exceptions that may be thrown from the library functions. class invalid_code_point : public exception { - uint32_t cp; -public: - invalid_code_point(uint32_t cp) : cp(cp) {} - virtual const char* what() const throw() - { - return "Invalid code point"; - } - uint32_t code_point() const - { - return cp; - } + uint32_t cp; + + public: + invalid_code_point(uint32_t cp) : cp(cp) + { + } + virtual const char* what() const throw() + { + return "Invalid code point"; + } + uint32_t code_point() const + { + return cp; + } }; class invalid_utf8 : public exception { - uint8_t u8; -public: - invalid_utf8 (uint8_t u) : u8(u) {} - virtual const char* what() const throw() - { - return "Invalid UTF-8"; - } - uint8_t utf8_octet() const - { - return u8; - } + uint8_t u8; + + public: + invalid_utf8(uint8_t u) : u8(u) + { + } + virtual const char* what() const throw() + { + return "Invalid UTF-8"; + } + uint8_t utf8_octet() const + { + return u8; + } }; class invalid_utf16 : public exception { - uint16_t u16; -public: - invalid_utf16 (uint16_t u) : u16(u) {} - virtual const char* what() const throw() - { - return "Invalid UTF-16"; - } - uint16_t utf16_word() const - { - return u16; - } + uint16_t u16; + + public: + invalid_utf16(uint16_t u) : u16(u) + { + } + virtual const char* what() const throw() + { + return "Invalid UTF-16"; + } + uint16_t utf16_word() const + { + return u16; + } }; class not_enough_room : public exception { -public: - virtual const char* what() const throw() - { - return "Not enough space"; - } + public: + virtual const char* what() const throw() + { + return "Not enough space"; + } }; /// The library API - functions intended to be called by the users @@ -98,297 +106,292 @@ public: template octet_iterator append(uint32_t cp, octet_iterator result) { - if (!utf8::internal::is_code_point_valid(cp)) - throw invalid_code_point(cp); + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); - if (cp < 0x80) // one octet - *(result++) = static_cast(cp); - else if (cp < 0x800) // two octets - { - *(result++) = static_cast((cp >> 6) | 0xc0); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) // three octets - { - *(result++) = static_cast((cp >> 12) | 0xe0); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else // four octets - { - *(result++) = static_cast((cp >> 18) | 0xf0); - *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) // two octets + { + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) // three octets + { + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else // four octets + { + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } - return result; + return result; } template -output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) +output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, + uint32_t replacement) { - while (start != end) + while (start != end) + { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + + switch (err_code) { - octet_iterator sequence_start = start; - internal::utf_error err_code = utf8::internal::validate_next(start, end); + case internal::UTF8_OK: + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; - switch (err_code) - { - case internal::UTF8_OK : - for (octet_iterator it = sequence_start; it != start; ++it) - *out++ = *it; + break; - break; + case internal::NOT_ENOUGH_ROOM: throw not_enough_room(); - case internal::NOT_ENOUGH_ROOM: - throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append(replacement, out); + ++start; + break; - case internal::INVALID_LEAD: - out = utf8::append (replacement, out); - ++start; - break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append(replacement, out); + ++start; - case internal::INCOMPLETE_SEQUENCE: - case internal::OVERLONG_SEQUENCE: - case internal::INVALID_CODE_POINT: - out = utf8::append (replacement, out); - ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; - // just one replacement mark for the sequence - while (start != end && utf8::internal::is_trail(*start)) - ++start; - - break; - } + break; } + } - return out; + return out; } template inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) { - static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); - return utf8::replace_invalid(start, end, out, replacement_marker); + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); } template uint32_t next(octet_iterator& it, octet_iterator end) { - uint32_t cp = 0; - internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); - switch (err_code) - { - case internal::UTF8_OK : - break; + switch (err_code) + { + case internal::UTF8_OK: break; - case internal::NOT_ENOUGH_ROOM : - throw not_enough_room(); + case internal::NOT_ENOUGH_ROOM: throw not_enough_room(); - case internal::INVALID_LEAD : - case internal::INCOMPLETE_SEQUENCE : - case internal::OVERLONG_SEQUENCE : - throw invalid_utf8(*it); + case internal::INVALID_LEAD: + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: throw invalid_utf8(*it); - case internal::INVALID_CODE_POINT : - throw invalid_code_point(cp); - } + case internal::INVALID_CODE_POINT: throw invalid_code_point(cp); + } - return cp; + return cp; } template uint32_t peek_next(octet_iterator it, octet_iterator end) { - return utf8::next(it, end); + return utf8::next(it, end); } template uint32_t prior(octet_iterator& it, octet_iterator start) { - // can't do much if it == start + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) if (it == start) - throw not_enough_room(); + throw invalid_utf8(*it); // error - no lead byte in the sequence - octet_iterator end = it; - - // Go back until we hit either a lead octet or start - while (utf8::internal::is_trail(*(--it))) - if (it == start) - throw invalid_utf8(*it); // error - no lead byte in the sequence - - return utf8::peek_next(it, end); + return utf8::peek_next(it, end); } /// Deprecated in versions that include "prior" template uint32_t previous(octet_iterator& it, octet_iterator pass_start) { - octet_iterator end = it; + octet_iterator end = it; - while (utf8::internal::is_trail(*(--it))) - if (it == pass_start) - throw invalid_utf8(*it); // error - no lead byte in the sequence + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence - octet_iterator temp = it; - return utf8::next(temp, end); + octet_iterator temp = it; + return utf8::next(temp, end); } template -void advance (octet_iterator& it, distance_type n, octet_iterator end) +void advance(octet_iterator& it, distance_type n, octet_iterator end) { - for (distance_type i = 0; i < n; ++i) - utf8::next(it, end); + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); } template -typename std::iterator_traits::difference_type -distance (octet_iterator first, octet_iterator last) +typename std::iterator_traits::difference_type distance(octet_iterator first, + octet_iterator last) { - typename std::iterator_traits::difference_type dist; + typename std::iterator_traits::difference_type dist; - for (dist = 0; first < last; ++dist) - utf8::next(first, last); + for (dist = 0; first < last; ++dist) + utf8::next(first, last); - return dist; + return dist; } template -octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) +octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result) { - while (start != end) + while (start != end) + { + uint32_t cp = utf8::internal::mask16(*start++); + + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { - uint32_t cp = utf8::internal::mask16(*start++); + if (start != end) + { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); - // Take care of surrogate pairs first - if (utf8::internal::is_lead_surrogate(cp)) - { - if (start != end) - { - uint32_t trail_surrogate = utf8::internal::mask16(*start++); - - if (utf8::internal::is_trail_surrogate(trail_surrogate)) - cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; - else - throw invalid_utf16(static_cast(trail_surrogate)); - } - else - throw invalid_utf16(static_cast(cp)); - - } - // Lone trail surrogate - else if (utf8::internal::is_trail_surrogate(cp)) - throw invalid_utf16(static_cast(cp)); - - result = utf8::append(cp, result); - } - - return result; -} - -template -u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) -{ - while (start != end) - { - uint32_t cp = utf8::next(start, end); - - if (cp > 0xffff) //make a surrogate pair - { - *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); - } + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; else - *result++ = static_cast(cp); + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); - return result; + result = utf8::append(cp, result); + } + + return result; +} + +template +u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result) +{ + while (start != end) + { + uint32_t cp = utf8::next(start, end); + + if (cp > 0xffff) // make a surrogate pair + { + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + + return result; } template -octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) +octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result) { - while (start != end) - result = utf8::append(*(start++), result); + while (start != end) + result = utf8::append(*(start++), result); - return result; + return result; } template -u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) +u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result) { - while (start != end) - (*result++) = utf8::next(start, end); + while (start != end) + (*result++) = utf8::next(start, end); - return result; + return result; } // The iterator class template -class iterator : public std::iterator +class iterator : public std::iterator { - octet_iterator it; - octet_iterator range_start; - octet_iterator range_end; -public: - iterator () {} - explicit iterator (const octet_iterator& octet_it, - const octet_iterator& range_start, - const octet_iterator& range_end) : - it(octet_it), range_start(range_start), range_end(range_end) - { - if (it < range_start || it > range_end) - throw std::out_of_range("Invalid utf-8 iterator position"); - } - // the default "big three" are OK - octet_iterator base () const - { - return it; - } - uint32_t operator * () const - { - octet_iterator temp = it; - return utf8::next(temp, range_end); - } - bool operator == (const iterator& rhs) const - { - if (range_start != rhs.range_start || range_end != rhs.range_end) - throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; - return (it == rhs.it); - } - bool operator != (const iterator& rhs) const - { - return !(operator == (rhs)); - } - iterator& operator ++ () - { - utf8::next(it, range_end); - return *this; - } - iterator operator ++ (int) - { - iterator temp = *this; - utf8::next(it, range_end); - return temp; - } - iterator& operator -- () - { - utf8::prior(it, range_start); - return *this; - } - iterator operator -- (int) - { - iterator temp = *this; - utf8::prior(it, range_start); - return temp; - } -}; // class iterator + public: + iterator() + { + } + explicit iterator(const octet_iterator& octet_it, const octet_iterator& range_start, + const octet_iterator& range_end) + : it(octet_it), range_start(range_start), range_end(range_end) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base() const + { + return it; + } + uint32_t operator*() const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator==(const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); -} // namespace utf8 - -#endif //header guard + return (it == rhs.it); + } + bool operator!=(const iterator& rhs) const + { + return !(operator==(rhs)); + } + iterator& operator++() + { + utf8::next(it, range_end); + return *this; + } + iterator operator++(int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator--() + { + utf8::prior(it, range_start); + return *this; + } + iterator operator--(int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } +}; // class iterator +} // namespace utf8 +#endif // header guard diff --git a/utils/funcexp/utf8/core.h b/utils/funcexp/utf8/core.h index 478eef134..0aa100c91 100644 --- a/utils/funcexp/utf8/core.h +++ b/utils/funcexp/utf8/core.h @@ -24,7 +24,6 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 #define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 @@ -39,180 +38,192 @@ namespace internal // Unicode constants // Leading (high) surrogates: 0xd800 - 0xdbff // Trailing (low) surrogates: 0xdc00 - 0xdfff -const uint16_t LEAD_SURROGATE_MIN = 0xd800u; -const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; +const uint16_t LEAD_SURROGATE_MIN = 0xd800u; +const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; -const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); -const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; +const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); +const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; // Maximum valid value for a Unicode code point -const uint32_t CODE_POINT_MAX = 0x0010ffffu; +const uint32_t CODE_POINT_MAX = 0x0010ffffu; -template +template inline uint8_t mask8(octet_type oc) { - return static_cast(0xff & oc); + return static_cast(0xff & oc); } -template +template inline uint16_t mask16(u16_type oc) { - return static_cast(0xffff & oc); + return static_cast(0xffff & oc); } -template +template inline bool is_trail(octet_type oc) { - return ((utf8::internal::mask8(oc) >> 6) == 0x2); + return ((utf8::internal::mask8(oc) >> 6) == 0x2); } template inline bool is_lead_surrogate(u16 cp) { - return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); } template inline bool is_trail_surrogate(u16 cp) { - return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); } template inline bool is_surrogate(u16 cp) { - return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); } template inline bool is_code_point_valid(u32 cp) { - return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); } template -inline typename std::iterator_traits::difference_type -sequence_length(octet_iterator lead_it) +inline typename std::iterator_traits::difference_type sequence_length(octet_iterator lead_it) { - uint8_t lead = utf8::internal::mask8(*lead_it); + uint8_t lead = utf8::internal::mask8(*lead_it); - if (lead < 0x80) - return 1; - else if ((lead >> 5) == 0x6) - return 2; - else if ((lead >> 4) == 0xe) - return 3; - else if ((lead >> 3) == 0x1e) - return 4; - else - return 0; + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; } template inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) { - if (cp < 0x80) - { - if (length != 1) - return true; - } - else if (cp < 0x800) - { - if (length != 2) - return true; - } - else if (cp < 0x10000) - { - if (length != 3) - return true; - } + if (cp < 0x80) + { + if (length != 1) + return true; + } + else if (cp < 0x800) + { + if (length != 2) + return true; + } + else if (cp < 0x10000) + { + if (length != 3) + return true; + } - return false; + return false; } -enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; +enum utf_error +{ + UTF8_OK, + NOT_ENOUGH_ROOM, + INVALID_LEAD, + INCOMPLETE_SEQUENCE, + OVERLONG_SEQUENCE, + INVALID_CODE_POINT +}; /// Helper for get_sequence_x template utf_error increase_safely(octet_iterator& it, octet_iterator end) { - if (++it == end) - return NOT_ENOUGH_ROOM; + if (++it == end) + return NOT_ENOUGH_ROOM; - if (!utf8::internal::is_trail(*it)) - return INCOMPLETE_SEQUENCE; + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; - return UTF8_OK; + return UTF8_OK; } -#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} +#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) \ + { \ + utf_error ret = increase_safely(IT, END); \ + if (ret != UTF8_OK) \ + return ret; \ + } /// get_sequence_x functions decode utf-8 sequences of the length x template utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) { - if (it == end) - return NOT_ENOUGH_ROOM; + if (it == end) + return NOT_ENOUGH_ROOM; - code_point = utf8::internal::mask8(*it); + code_point = utf8::internal::mask8(*it); - return UTF8_OK; + return UTF8_OK; } template utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) { - if (it == end) - return NOT_ENOUGH_ROOM; + if (it == end) + return NOT_ENOUGH_ROOM; - code_point = utf8::internal::mask8(*it); + code_point = utf8::internal::mask8(*it); - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); - return UTF8_OK; + return UTF8_OK; } template utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) { - if (it == end) - return NOT_ENOUGH_ROOM; + if (it == end) + return NOT_ENOUGH_ROOM; - code_point = utf8::internal::mask8(*it); + code_point = utf8::internal::mask8(*it); - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - code_point += (*it) & 0x3f; + code_point += (*it) & 0x3f; - return UTF8_OK; + return UTF8_OK; } template utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) { - if (it == end) - return NOT_ENOUGH_ROOM; + if (it == end) + return NOT_ENOUGH_ROOM; - code_point = utf8::internal::mask8(*it); + code_point = utf8::internal::mask8(*it); - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - code_point += (*it) & 0x3f; + code_point += (*it) & 0x3f; - return UTF8_OK; + return UTF8_OK; } #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR @@ -220,72 +231,63 @@ utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_ template utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) { - // Save the original value of it so we can go back in case of failure - // Of course, it does not make much sense with i.e. stream iterators - octet_iterator original_it = it; + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; - uint32_t cp = 0; - // Determine the sequence length based on the lead octet - typedef typename std::iterator_traits::difference_type octet_difference_type; - const octet_difference_type length = utf8::internal::sequence_length(it); + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); - // Get trail octets and calculate the code point - utf_error err = UTF8_OK; + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; - switch (length) + switch (length) + { + case 0: return INVALID_LEAD; + + case 1: err = utf8::internal::get_sequence_1(it, end, cp); break; + + case 2: err = utf8::internal::get_sequence_2(it, end, cp); break; + + case 3: err = utf8::internal::get_sequence_3(it, end, cp); break; + + case 4: err = utf8::internal::get_sequence_4(it, end, cp); break; + } + + if (err == UTF8_OK) + { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { - case 0: - return INVALID_LEAD; - - case 1: - err = utf8::internal::get_sequence_1(it, end, cp); - break; - - case 2: - err = utf8::internal::get_sequence_2(it, end, cp); - break; - - case 3: - err = utf8::internal::get_sequence_3(it, end, cp); - break; - - case 4: - err = utf8::internal::get_sequence_4(it, end, cp); - break; + if (!utf8::internal::is_overlong_sequence(cp, length)) + { + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; } + else + err = INVALID_CODE_POINT; + } - if (err == UTF8_OK) - { - // Decoding succeeded. Now, security checks... - if (utf8::internal::is_code_point_valid(cp)) - { - if (!utf8::internal::is_overlong_sequence(cp, length)) - { - // Passed! Return here. - code_point = cp; - ++it; - return UTF8_OK; - } - else - err = OVERLONG_SEQUENCE; - } - else - err = INVALID_CODE_POINT; - } - - // Failure branch - restore the original value of the iterator - it = original_it; - return err; + // Failure branch - restore the original value of the iterator + it = original_it; + return err; } template inline utf_error validate_next(octet_iterator& it, octet_iterator end) { - uint32_t ignored; - return utf8::internal::validate_next(it, end, ignored); + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); } -} // namespace internal +} // namespace internal /// The library API - functions intended to be called by the users @@ -295,47 +297,40 @@ const uint8_t bom[] = {0xef, 0xbb, 0xbf}; template octet_iterator find_invalid(octet_iterator start, octet_iterator end) { - octet_iterator result = start; + octet_iterator result = start; - while (result != end) - { - utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + while (result != end) + { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); - if (err_code != internal::UTF8_OK) - return result; - } + if (err_code != internal::UTF8_OK) + return result; + } - return result; + return result; } template inline bool is_valid(octet_iterator start, octet_iterator end) { - return (utf8::find_invalid(start, end) == end); + return (utf8::find_invalid(start, end) == end); } template -inline bool starts_with_bom (octet_iterator it, octet_iterator end) +inline bool starts_with_bom(octet_iterator it, octet_iterator end) { - return ( - ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && - ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && - ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) - ); + return (((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2])); } -//Deprecated in release 2.3 +// Deprecated in release 2.3 template -inline bool is_bom (octet_iterator it) +inline bool is_bom(octet_iterator it) { - return ( - (utf8::internal::mask8(*it++)) == bom[0] && - (utf8::internal::mask8(*it++)) == bom[1] && - (utf8::internal::mask8(*it)) == bom[2] - ); + return ((utf8::internal::mask8(*it++)) == bom[0] && (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2]); } -} // namespace utf8 - -#endif // header guard - +} // namespace utf8 +#endif // header guard diff --git a/utils/funcexp/utf8/unchecked.h b/utils/funcexp/utf8/unchecked.h index cb76dbdda..8f50e0e5f 100644 --- a/utils/funcexp/utf8/unchecked.h +++ b/utils/funcexp/utf8/unchecked.h @@ -24,7 +24,6 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 #define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 @@ -37,218 +36,221 @@ namespace unchecked template octet_iterator append(uint32_t cp, octet_iterator result) { - if (cp < 0x80) // one octet - *(result++) = static_cast(cp); - else if (cp < 0x800) // two octets - { - *(result++) = static_cast((cp >> 6) | 0xc0); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) // three octets - { - *(result++) = static_cast((cp >> 12) | 0xe0); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else // four octets - { - *(result++) = static_cast((cp >> 18) | 0xf0); - *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) // two octets + { + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) // three octets + { + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else // four octets + { + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } - return result; + return result; } template uint32_t next(octet_iterator& it) { - uint32_t cp = utf8::internal::mask8(*it); - typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); - switch (length) - { - case 1: - break; + switch (length) + { + case 1: break; - case 2: - it++; - cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); - break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; - case 3: - ++it; - cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); - ++it; - cp += (*it) & 0x3f; - break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; - case 4: - ++it; - cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); - ++it; - cp += (utf8::internal::mask8(*it) << 6) & 0xfff; - ++it; - cp += (*it) & 0x3f; - break; - } + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } - ++it; - return cp; + ++it; + return cp; } template uint32_t peek_next(octet_iterator it) { - return utf8::unchecked::next(it); + return utf8::unchecked::next(it); } template uint32_t prior(octet_iterator& it) { - while (utf8::internal::is_trail(*(--it))) ; + while (utf8::internal::is_trail(*(--it))) + ; - octet_iterator temp = it; - return utf8::unchecked::next(temp); + octet_iterator temp = it; + return utf8::unchecked::next(temp); } // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) template inline uint32_t previous(octet_iterator& it) { - return utf8::unchecked::prior(it); + return utf8::unchecked::prior(it); } template -void advance (octet_iterator& it, distance_type n) +void advance(octet_iterator& it, distance_type n) { - for (distance_type i = 0; i < n; ++i) - utf8::unchecked::next(it); + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); } template -typename std::iterator_traits::difference_type -distance (octet_iterator first, octet_iterator last) +typename std::iterator_traits::difference_type distance(octet_iterator first, + octet_iterator last) { - typename std::iterator_traits::difference_type dist; + typename std::iterator_traits::difference_type dist; - for (dist = 0; first < last; ++dist) - utf8::unchecked::next(first); + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); - return dist; + return dist; } template -octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) +octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result) { - while (start != end) + while (start != end) + { + uint32_t cp = utf8::internal::mask16(*start++); + + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { - uint32_t cp = utf8::internal::mask16(*start++); - - // Take care of surrogate pairs first - if (utf8::internal::is_lead_surrogate(cp)) - { - uint32_t trail_surrogate = utf8::internal::mask16(*start++); - cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; - } - - result = utf8::unchecked::append(cp, result); + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; } - return result; + result = utf8::unchecked::append(cp, result); + } + + return result; } template -u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) +u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result) { - while (start < end) + while (start < end) + { + uint32_t cp = utf8::unchecked::next(start); + + if (cp > 0xffff) // make a surrogate pair { - uint32_t cp = utf8::unchecked::next(start); - - if (cp > 0xffff) //make a surrogate pair - { - *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); - } - else - *result++ = static_cast(cp); + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); } + else + *result++ = static_cast(cp); + } - return result; + return result; } template -octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) +octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result) { - while (start != end) - result = utf8::unchecked::append(*(start++), result); + while (start != end) + result = utf8::unchecked::append(*(start++), result); - return result; + return result; } template -u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) +u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result) { - while (start < end) - (*result++) = utf8::unchecked::next(start); + while (start < end) + (*result++) = utf8::unchecked::next(start); - return result; + return result; } // The iterator class template -class iterator : public std::iterator +class iterator : public std::iterator { - octet_iterator it; -public: - iterator () {} - explicit iterator (const octet_iterator& octet_it): it(octet_it) {} - // the default "big three" are OK - octet_iterator base () const - { - return it; - } - uint32_t operator * () const - { - octet_iterator temp = it; - return utf8::unchecked::next(temp); - } - bool operator == (const iterator& rhs) const - { - return (it == rhs.it); - } - bool operator != (const iterator& rhs) const - { - return !(operator == (rhs)); - } - iterator& operator ++ () - { - ::std::advance(it, utf8::internal::sequence_length(it)); - return *this; - } - iterator operator ++ (int) - { - iterator temp = *this; - ::std::advance(it, utf8::internal::sequence_length(it)); - return temp; - } - iterator& operator -- () - { - utf8::unchecked::prior(it); - return *this; - } - iterator operator -- (int) - { - iterator temp = *this; - utf8::unchecked::prior(it); - return temp; - } -}; // class iterator + octet_iterator it; -} // namespace utf8::unchecked -} // namespace utf8 + public: + iterator() + { + } + explicit iterator(const octet_iterator& octet_it) : it(octet_it) + { + } + // the default "big three" are OK + octet_iterator base() const + { + return it; + } + uint32_t operator*() const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator==(const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator!=(const iterator& rhs) const + { + return !(operator==(rhs)); + } + iterator& operator++() + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator++(int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator--() + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator--(int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } +}; // class iterator +} // namespace unchecked +} // namespace utf8 -#endif // header guard - +#endif // header guard diff --git a/utils/idbdatafile/BufferedFile.cpp b/utils/idbdatafile/BufferedFile.cpp index 05973ed23..6d7c58347 100644 --- a/utils/idbdatafile/BufferedFile.cpp +++ b/utils/idbdatafile/BufferedFile.cpp @@ -33,250 +33,247 @@ using namespace std; namespace idbdatafile { - -BufferedFile::BufferedFile(const char* fname, const char* mode, unsigned opts) : - IDBDataFile( fname ), - m_fp( 0 ), - m_buffer( 0 ) +BufferedFile::BufferedFile(const char* fname, const char* mode, unsigned opts) + : IDBDataFile(fname), m_fp(0), m_buffer(0) { - m_fp = fopen(fname, mode); + m_fp = fopen(fname, mode); - if ( m_fp == NULL ) - { - throw std::runtime_error("unable to open Buffered file "); - } + if (m_fp == NULL) + { + throw std::runtime_error("unable to open Buffered file "); + } - applyOptions( opts ); + applyOptions(opts); } -void BufferedFile::applyOptions( unsigned opts ) +void BufferedFile::applyOptions(unsigned opts) { - if ( opts & IDBDataFile::USE_VBUF ) - { - const int DEFAULT_BUFSIZ = 1 * 1024 * 1024; - m_buffer = new char[DEFAULT_BUFSIZ]; - setvbuf(m_fp, m_buffer, _IOFBF, DEFAULT_BUFSIZ); - } - else if ( opts & IDBDataFile::USE_NOVBUF ) - { - setvbuf(m_fp, NULL, _IONBF, 0); - } + if (opts & IDBDataFile::USE_VBUF) + { + const int DEFAULT_BUFSIZ = 1 * 1024 * 1024; + m_buffer = new char[DEFAULT_BUFSIZ]; + setvbuf(m_fp, m_buffer, _IOFBF, DEFAULT_BUFSIZ); + } + else if (opts & IDBDataFile::USE_NOVBUF) + { + setvbuf(m_fp, NULL, _IONBF, 0); + } } BufferedFile::~BufferedFile() { - close(); - m_fp = 0; - delete [] m_buffer; + close(); + m_fp = 0; + delete[] m_buffer; } ssize_t BufferedFile::pread(void* ptr, off64_t offset, size_t count) { - ssize_t ret = 0; - int savedErrno; - ssize_t curpos = tell(); + ssize_t ret = 0; + int savedErrno; + ssize_t curpos = tell(); - seek(offset, SEEK_SET); - ret = read(ptr, count); - savedErrno = errno; - seek(curpos, SEEK_SET); + seek(offset, SEEK_SET); + ret = read(ptr, count); + savedErrno = errno; + seek(curpos, SEEK_SET); - if ( IDBLogger::isEnabled() ) - IDBLogger::logRW("pread", m_fname, this, offset, count, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logRW("pread", m_fname, this, offset, count, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } ssize_t BufferedFile::read(void* ptr, size_t count) { - ssize_t ret = 0; - ssize_t offset = tell(); - int savedErrno = -1; - size_t progress = 0; - uint8_t* ptr8 = (uint8_t*) ptr; + ssize_t ret = 0; + ssize_t offset = tell(); + int savedErrno = -1; + size_t progress = 0; + uint8_t* ptr8 = (uint8_t*)ptr; - while (progress < count) + while (progress < count) + { + ret = fread(ptr8 + progress, 1, count - progress, m_fp); + savedErrno = errno; + + if (ret <= 0) { - ret = fread(ptr8 + progress, 1, count - progress, m_fp); - savedErrno = errno; - - if (ret <= 0) - { - if (ferror(m_fp)) - { - errno = savedErrno; - return -1; - } - else if (feof(m_fp)) - return progress; - } - - progress += ret; + if (ferror(m_fp)) + { + errno = savedErrno; + return -1; + } + else if (feof(m_fp)) + return progress; } - if ( IDBLogger::isEnabled() ) - IDBLogger::logRW("read", m_fname, this, offset, count, progress); + progress += ret; + } - errno = savedErrno; - return progress; + if (IDBLogger::isEnabled()) + IDBLogger::logRW("read", m_fname, this, offset, count, progress); + + errno = savedErrno; + return progress; } ssize_t BufferedFile::write(const void* ptr, size_t count) { - ssize_t ret = 0; - off64_t offset = tell(); - int savedErrno = 0; - size_t progress = 0; - uint8_t* ptr8 = (uint8_t*) ptr; + ssize_t ret = 0; + off64_t offset = tell(); + int savedErrno = 0; + size_t progress = 0; + uint8_t* ptr8 = (uint8_t*)ptr; - while (progress < count) + while (progress < count) + { + ret = fwrite(ptr8 + progress, 1, count - progress, m_fp); + savedErrno = errno; + + if (ret <= 0 && ferror(m_fp)) { - ret = fwrite(ptr8 + progress, 1, count - progress, m_fp); - savedErrno = errno; - - if (ret <= 0 && ferror(m_fp)) - { - errno = savedErrno; - return -1; - } - else if (ret > 0) - progress += ret; - - // can fwrite() continually return 0 with no error? + errno = savedErrno; + return -1; } + else if (ret > 0) + progress += ret; - if ( IDBLogger::isEnabled() ) - IDBLogger::logRW("write", m_fname, this, offset, count, progress); + // can fwrite() continually return 0 with no error? + } - errno = savedErrno; - return progress; + if (IDBLogger::isEnabled()) + IDBLogger::logRW("write", m_fname, this, offset, count, progress); + + errno = savedErrno; + return progress; } int BufferedFile::seek(off64_t offset, int whence) { - int ret = 0; - int savedErrno; + int ret = 0; + int savedErrno; #ifdef _MSC_VER - ret = _fseeki64(m_fp, offset, whence); + ret = _fseeki64(m_fp, offset, whence); #else - ret = fseek(m_fp, offset, whence); + ret = fseek(m_fp, offset, whence); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logSeek(m_fname, this, offset, whence, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logSeek(m_fname, this, offset, whence, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } int BufferedFile::truncate(off64_t length) { - int ret = 0; - int savedErrno; + int ret = 0; + int savedErrno; #ifdef _MSC_VER - ret = _chsize_s(_fileno(m_fp), length); + ret = _chsize_s(_fileno(m_fp), length); #else - ret = ftruncate(fileno(m_fp), length); + ret = ftruncate(fileno(m_fp), length); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logTruncate(m_fname, this, length, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logTruncate(m_fname, this, length, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } off64_t BufferedFile::size() { #ifdef _MSC_VER - return _filelengthi64(fileno(m_fp)); // Interestingly, implemented as fseek/ftell in the windows crt + return _filelengthi64(fileno(m_fp)); // Interestingly, implemented as fseek/ftell in the windows crt #else - // going to calculate size 2 ways - first, via seek - off64_t length = -1; - off64_t here; + // going to calculate size 2 ways - first, via seek + off64_t length = -1; + off64_t here; - flockfile(m_fp); + flockfile(m_fp); - try + try + { + if ((here = ftell(m_fp)) > -1) { - if ((here = ftell(m_fp)) > -1) - { - if (fseek(m_fp, 0, SEEK_END) > -1) - { - length = ftell(m_fp); - fseek(m_fp, here, SEEK_SET); - } - } - - funlockfile(m_fp); - } - catch (...) - { - funlockfile(m_fp); + if (fseek(m_fp, 0, SEEK_END) > -1) + { + length = ftell(m_fp); + fseek(m_fp, here, SEEK_SET); + } } - return length; + funlockfile(m_fp); + } + catch (...) + { + funlockfile(m_fp); + } + + return length; #endif } off64_t BufferedFile::tell() { #ifdef _MSC_VER - return _ftelli64(m_fp); + return _ftelli64(m_fp); #else - return ftell(m_fp); + return ftell(m_fp); #endif } int BufferedFile::flush() { - int rc = fflush(m_fp); - int savedErrno = errno; + int rc = fflush(m_fp); + int savedErrno = errno; - if ( rc == 0 ) - { + if (rc == 0) + { #ifdef _MSC_VER - rc = _commit(_fileno(m_fp)); + rc = _commit(_fileno(m_fp)); #else - rc = fsync( fileno( m_fp ) ); + rc = fsync(fileno(m_fp)); #endif - savedErrno = errno; - } + savedErrno = errno; + } - if ( IDBLogger::isEnabled() ) - IDBLogger::logNoArg(m_fname, this, "flush", rc); + if (IDBLogger::isEnabled()) + IDBLogger::logNoArg(m_fname, this, "flush", rc); - errno = savedErrno; - return rc; + errno = savedErrno; + return rc; } time_t BufferedFile::mtime() { - time_t ret = 0; - struct stat statbuf; + time_t ret = 0; + struct stat statbuf; - if (::fstat(fileno(m_fp), &statbuf) == 0) - ret = statbuf.st_mtime; - else - ret = (time_t) - 1; + if (::fstat(fileno(m_fp), &statbuf) == 0) + ret = statbuf.st_mtime; + else + ret = (time_t)-1; - return ret; + return ret; } int BufferedFile::close() { - int ret = fclose(m_fp); - int savedErrno = errno; + int ret = fclose(m_fp); + int savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logNoArg(m_fname, this, "close", ret); + if (IDBLogger::isEnabled()) + IDBLogger::logNoArg(m_fname, this, "close", ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } /** @@ -287,19 +284,19 @@ int BufferedFile::close() */ int BufferedFile::fallocate(int mode, off64_t offset, off64_t length) { - int ret = 0; - int savedErrno = 0; + int ret = 0; + int savedErrno = 0; - ret = ::fallocate( fileno(m_fp), mode, offset, length ); - savedErrno = errno; + ret = ::fallocate(fileno(m_fp), mode, offset, length); + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - { - IDBLogger::logNoArg(m_fname, this, "fallocate", errno); - } + if (IDBLogger::isEnabled()) + { + IDBLogger::logNoArg(m_fname, this, "fallocate", errno); + } - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/BufferedFile.h b/utils/idbdatafile/BufferedFile.h index 3e6ae040e..09646b448 100644 --- a/utils/idbdatafile/BufferedFile.h +++ b/utils/idbdatafile/BufferedFile.h @@ -29,7 +29,6 @@ typedef void* HANDLE; namespace idbdatafile { - /** * BufferedFile implements the IDBDataFile for I/O to a C library FILE* * (via fopen, fwrite, fread, etc.). See IDBDataFile.h for more documentation @@ -37,31 +36,31 @@ namespace idbdatafile */ class BufferedFile : public IDBDataFile, boost::noncopyable { -public: - BufferedFile(const char* fname, const char* mode, unsigned opts); - /* virtual */ ~BufferedFile(); + public: + BufferedFile(const char* fname, const char* mode, unsigned opts); + /* virtual */ ~BufferedFile(); - /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count); - /* virtual */ ssize_t read(void* ptr, size_t count); - /* virtual */ ssize_t write(const void* ptr, size_t count); - /* virtual */ int seek(off64_t offset, int whence); - /* virtual */ int truncate(off64_t length); - /* virtual */ off64_t size(); - /* virtual */ off64_t tell(); - /* virtual */ int flush(); - /* virtual */ time_t mtime(); - /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); + /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count); + /* virtual */ ssize_t read(void* ptr, size_t count); + /* virtual */ ssize_t write(const void* ptr, size_t count); + /* virtual */ int seek(off64_t offset, int whence); + /* virtual */ int truncate(off64_t length); + /* virtual */ off64_t size(); + /* virtual */ off64_t tell(); + /* virtual */ int flush(); + /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); -protected: - /* virtual */ - int close(); + protected: + /* virtual */ + int close(); -private: - void applyOptions( unsigned opts ); + private: + void applyOptions(unsigned opts); - FILE* m_fp; - char* m_buffer; + FILE* m_fp; + char* m_buffer; }; -} +} // namespace idbdatafile #endif /* BUFFEREDFILE_H_ */ diff --git a/utils/idbdatafile/BufferedFileFactory.h b/utils/idbdatafile/BufferedFileFactory.h index b2d675e4b..a9fd4b740 100644 --- a/utils/idbdatafile/BufferedFileFactory.h +++ b/utils/idbdatafile/BufferedFileFactory.h @@ -24,19 +24,18 @@ namespace idbdatafile { - class BufferedFileFactory : public FileFactoryBase { -public: - /* virtual */ - IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); + public: + /* virtual */ + IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); }; -inline -IDBDataFile* BufferedFileFactory::open(const char* fname, const char* mode, unsigned opts, unsigned /*defsize*/) +inline IDBDataFile* BufferedFileFactory::open(const char* fname, const char* mode, unsigned opts, + unsigned /*defsize*/) { - return new BufferedFile(fname, mode, opts); + return new BufferedFile(fname, mode, opts); } -} +} // namespace idbdatafile #endif /* BUFFEREDFILEFACTORY_H_ */ diff --git a/utils/idbdatafile/FileFactoryBase.h b/utils/idbdatafile/FileFactoryBase.h index d1572ca05..8bc1998fc 100644 --- a/utils/idbdatafile/FileFactoryBase.h +++ b/utils/idbdatafile/FileFactoryBase.h @@ -20,15 +20,17 @@ namespace idbdatafile { - class IDBDataFile; class FileFactoryBase { -public: - virtual ~FileFactoryBase() {;} - virtual IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth) = 0; + public: + virtual ~FileFactoryBase() + { + ; + } + virtual IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth) = 0; }; -} +} // namespace idbdatafile #endif /* FILEFACTORYBASE_H_ */ diff --git a/utils/idbdatafile/IDBDataFile.cpp b/utils/idbdatafile/IDBDataFile.cpp index 128442a0c..2c53f3933 100644 --- a/utils/idbdatafile/IDBDataFile.cpp +++ b/utils/idbdatafile/IDBDataFile.cpp @@ -31,29 +31,29 @@ using namespace std; namespace idbdatafile { - -IDBDataFile* IDBDataFile::open(Types type, const char* fname, const char* mode, unsigned opts, unsigned colWidth) +IDBDataFile* IDBDataFile::open(Types type, const char* fname, const char* mode, unsigned opts, + unsigned colWidth) { - IDBDataFile* ret = 0; + IDBDataFile* ret = 0; - try - { - ret = IDBFactory::open(type, fname, mode, opts, colWidth); - } - catch (std::exception& e) - { - std::ostringstream oss; - oss << "Failed to open file: " << fname << ", exception: " << e.what(); - IDBLogger::syslog(oss.str(), logging::LOG_TYPE_DEBUG); + try + { + ret = IDBFactory::open(type, fname, mode, opts, colWidth); + } + catch (std::exception& e) + { + std::ostringstream oss; + oss << "Failed to open file: " << fname << ", exception: " << e.what(); + IDBLogger::syslog(oss.str(), logging::LOG_TYPE_DEBUG); - if ( IDBLogger::isEnabled() ) - IDBLogger::logNoArg( fname, 0, e.what(), 0 ); - } + if (IDBLogger::isEnabled()) + IDBLogger::logNoArg(fname, 0, e.what(), 0); + } - if ( IDBLogger::isEnabled() ) - IDBLogger::logOpen( type, fname, mode, opts, ret ); + if (IDBLogger::isEnabled()) + IDBLogger::logOpen(type, fname, mode, opts, ret); - return ret; + return ret; } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/IDBDataFile.h b/utils/idbdatafile/IDBDataFile.h index 7559b26f6..1747189e5 100644 --- a/utils/idbdatafile/IDBDataFile.h +++ b/utils/idbdatafile/IDBDataFile.h @@ -21,18 +21,17 @@ #include #include #include "largefile64.h" -#include +#include #include "stdint.h" #include #ifdef _MSC_VER #undef tell #else -#define INVALID_HANDLE_VALUE 0 // Defined already in MSC +#define INVALID_HANDLE_VALUE 0 // Defined already in MSC #endif namespace idbdatafile { - /** * IDBDataFile is a combination abstract base class and factory. The purpose * is to encapsulate different underlying storage types from the InfiniDB @@ -44,207 +43,202 @@ namespace idbdatafile */ class IDBDataFile { -public: - /** - * The TYPE enum defines the supported underlying storage types - */ - enum Types - { - UNKNOWN = 0x0000, - BUFFERED = 0x0001, - UNBUFFERED = 0x0002, - HDFS = 0x0003, - CLOUD = 0x0004, - }; + public: + /** + * The TYPE enum defines the supported underlying storage types + */ + enum Types + { + UNKNOWN = 0x0000, + BUFFERED = 0x0001, + UNBUFFERED = 0x0002, + HDFS = 0x0003, + CLOUD = 0x0004, + }; - /** - * The Options enum defines elements of a bitmask that can be passed into - * the open method for controlling specific options relating to the - * particular file type in question. If an option is unsupported by the - * particular type the open() method will fail. - * USE_ODIRECT - requests the O_DIRECT flag to be set, only for - * unbuffered files - * USE_VBUF - requests that a non-default buffer is use for a - * buffered file. For now, the size of the buffer is - * hard-coded to match DEFAULT_BUFSIZ in writeengine - * (1*1024*1024) - * USE_NOVBUF - requests that buffering is disabled when using a - * buffered type file - * USE_TMPFILE - requests to output the membuffer to a temp file i/o - * overwrite the original file - */ - enum Options - { - USE_ODIRECT = 0x0001, - USE_VBUF = 0x0002, - USE_NOVBUF = 0x0004, - USE_TMPFILE = 0x0008 - }; + /** + * The Options enum defines elements of a bitmask that can be passed into + * the open method for controlling specific options relating to the + * particular file type in question. If an option is unsupported by the + * particular type the open() method will fail. + * USE_ODIRECT - requests the O_DIRECT flag to be set, only for + * unbuffered files + * USE_VBUF - requests that a non-default buffer is use for a + * buffered file. For now, the size of the buffer is + * hard-coded to match DEFAULT_BUFSIZ in writeengine + * (1*1024*1024) + * USE_NOVBUF - requests that buffering is disabled when using a + * buffered type file + * USE_TMPFILE - requests to output the membuffer to a temp file i/o + * overwrite the original file + */ + enum Options + { + USE_ODIRECT = 0x0001, + USE_VBUF = 0x0002, + USE_NOVBUF = 0x0004, + USE_TMPFILE = 0x0008 + }; - /** - * DEFSIZE is used as the starting size for the memory buffer of read/write - * HDFS files. Thes files are entirely buffered in memory and modified - * there since HDFS does not support any file updates. This value was - * engineered to accomodate the standard default extent for a 4-byte column - * - */ - static const int DEFSIZE = 33562624; - /** - * EXTENTSIZE is used with the passed in col width the starting size for the - * memory buffer of read/write HDFS files. Thes files are entirely buffered - * in memory and modified there since HDFS does not support any file updates. - */ - static const int EXTENTSIZE = 8390656; + /** + * DEFSIZE is used as the starting size for the memory buffer of read/write + * HDFS files. Thes files are entirely buffered in memory and modified + * there since HDFS does not support any file updates. This value was + * engineered to accomodate the standard default extent for a 4-byte column + * + */ + static const int DEFSIZE = 33562624; + /** + * EXTENTSIZE is used with the passed in col width the starting size for the + * memory buffer of read/write HDFS files. Thes files are entirely buffered + * in memory and modified there since HDFS does not support any file updates. + */ + static const int EXTENTSIZE = 8390656; - /** - * This is an alternate factory method that accepts a typical mode - * string (ala fopen). Note that in general IDBDataFile only recognizes - * a subset of the possible mode string that fopen supports however - * at present time, the mode string will be passed through directly to - * the BufferedFile constructor and through to fopen. The eventual goal - * is not complete support but rather only that necessary for Infinidb so - * this will be reconsidered. - */ - static IDBDataFile* open(Types type, const char* fname, const char* mode, unsigned opts, unsigned colWidth = 4); + /** + * This is an alternate factory method that accepts a typical mode + * string (ala fopen). Note that in general IDBDataFile only recognizes + * a subset of the possible mode string that fopen supports however + * at present time, the mode string will be passed through directly to + * the BufferedFile constructor and through to fopen. The eventual goal + * is not complete support but rather only that necessary for Infinidb so + * this will be reconsidered. + */ + static IDBDataFile* open(Types type, const char* fname, const char* mode, unsigned opts, + unsigned colWidth = 4); - /** - * Library users should assume that the destructor closes the file. - * This actually occurs in the derived clasess - hence the protected - * member close() here - */ - virtual ~IDBDataFile(); + /** + * Library users should assume that the destructor closes the file. + * This actually occurs in the derived clasess - hence the protected + * member close() here + */ + virtual ~IDBDataFile(); - /** - * This returns the name of the file. - */ - const std::string& name() const; + /** + * This returns the name of the file. + */ + const std::string& name() const; - /** - * This is a positional read method similar to kernel style pread - * or fseek followed by read for C-library FILE*. Return value - * is the number of bytes read. - */ - virtual ssize_t pread(void* ptr, off64_t offset, size_t count) = 0; + /** + * This is a positional read method similar to kernel style pread + * or fseek followed by read for C-library FILE*. Return value + * is the number of bytes read. + */ + virtual ssize_t pread(void* ptr, off64_t offset, size_t count) = 0; - /** - * This is a read method similar to kernel style read or C library - * fread(). Return value is the number of bytes read. - */ - virtual ssize_t read(void* ptr, size_t count) = 0; + /** + * This is a read method similar to kernel style read or C library + * fread(). Return value is the number of bytes read. + */ + virtual ssize_t read(void* ptr, size_t count) = 0; - /** - * The write() call semantics match the standard library. There is - * no positional write and further there is no lseek or equivalent so - * all writing must therefore occur sequentially - either from the - * beginning of a file (open via WRITE) or the end of a file (open via - * APPEND). Return value is the number of bytes written. - */ - virtual ssize_t write(const void* ptr, size_t count) = 0; + /** + * The write() call semantics match the standard library. There is + * no positional write and further there is no lseek or equivalent so + * all writing must therefore occur sequentially - either from the + * beginning of a file (open via WRITE) or the end of a file (open via + * APPEND). Return value is the number of bytes written. + */ + virtual ssize_t write(const void* ptr, size_t count) = 0; - /** - * The seek() method is equivalent to the lseek() and fseek() functions. - * The whence parameter accepts SEEK_SET, SEEK_CUR, SEEK_END just as - * those functions do. Note that not all file systems support this - * operation - ex. HDFS will not support it for files opened for writing - * Returns 0 on success, -1 on error - */ - virtual int seek(off64_t offset, int whence) = 0; + /** + * The seek() method is equivalent to the lseek() and fseek() functions. + * The whence parameter accepts SEEK_SET, SEEK_CUR, SEEK_END just as + * those functions do. Note that not all file systems support this + * operation - ex. HDFS will not support it for files opened for writing + * Returns 0 on success, -1 on error + */ + virtual int seek(off64_t offset, int whence) = 0; - /** - * The truncate() method is equivalent to the ftruncate method. Note - * that not all file types support this operation - ex. HDFS files opened - * or write or append do not, but HDFS files opened for modification do. - * Returns 0 on success, -1 on error. - */ - virtual int truncate(off64_t length) = 0; + /** + * The truncate() method is equivalent to the ftruncate method. Note + * that not all file types support this operation - ex. HDFS files opened + * or write or append do not, but HDFS files opened for modification do. + * Returns 0 on success, -1 on error. + */ + virtual int truncate(off64_t length) = 0; - /** - * The size() method returns the size of the file in a manner consistent - * with the underlying filesystem. Note that this method will always - * return the correct size from the perspective of the open file handle, - * thus depending on the semantics of the underlying file system an - * external view of size may differ (ex. if writing buffered i/o before - * a flush/sync or if writing an open HDFS file). Returns -1 on error. - */ - virtual off64_t size() = 0; + /** + * The size() method returns the size of the file in a manner consistent + * with the underlying filesystem. Note that this method will always + * return the correct size from the perspective of the open file handle, + * thus depending on the semantics of the underlying file system an + * external view of size may differ (ex. if writing buffered i/o before + * a flush/sync or if writing an open HDFS file). Returns -1 on error. + */ + virtual off64_t size() = 0; - /** - * The tell() call returns the current offset in the file. This is - * similar to lseek with 0 offset in the standard library and ftell - * for buffered FILE *s. - */ - virtual off64_t tell() = 0; + /** + * The tell() call returns the current offset in the file. This is + * similar to lseek with 0 offset in the standard library and ftell + * for buffered FILE *s. + */ + virtual off64_t tell() = 0; - /** - * The flush() method instructs the file to write any buffered contents - * to disk. Where relevant this method will include a call to fsync - */ - virtual int flush() = 0; + /** + * The flush() method instructs the file to write any buffered contents + * to disk. Where relevant this method will include a call to fsync + */ + virtual int flush() = 0; - /** - * The mtime() method returns the modification time of the file in - * seconds. Returns -1 on error. - */ - virtual time_t mtime() = 0; + /** + * The mtime() method returns the modification time of the file in + * seconds. Returns -1 on error. + */ + virtual time_t mtime() = 0; - /** - * The fallocate() method preallocates disk space cheaper then - * sequential write. fallocate() is supported by a limited number - * of FSes.This method is implemented for Un-/BufferedFile classes - * only. - * Returns -1 on error. - */ - virtual int fallocate(int mode, off64_t offset, off64_t length) = 0; + /** + * The fallocate() method preallocates disk space cheaper then + * sequential write. fallocate() is supported by a limited number + * of FSes.This method is implemented for Un-/BufferedFile classes + * only. + * Returns -1 on error. + */ + virtual int fallocate(int mode, off64_t offset, off64_t length) = 0; - int colWidth() - { - return m_fColWidth; - } + int colWidth() + { + return m_fColWidth; + } -protected: - /** - * Constructor - takes the filename to be stored in a member variable - * for logging purposes - */ - IDBDataFile( const char* fname ); + protected: + /** + * Constructor - takes the filename to be stored in a member variable + * for logging purposes + */ + IDBDataFile(const char* fname); - /** - * The close() method closes the file. It is defined as protected - * because the preference is for close() to happen automatically during - * deletion of the object - */ - virtual int close() = 0; + /** + * The close() method closes the file. It is defined as protected + * because the preference is for close() to happen automatically during + * deletion of the object + */ + virtual int close() = 0; - /** - * file name - */ - std::string m_fname; - - /** - * Column width. If not applicable, defaults to 4. - * We use this to determine the initial size of hdfs ram buffers. - */ - int m_fColWidth; + /** + * file name + */ + std::string m_fname; + /** + * Column width. If not applicable, defaults to 4. + * We use this to determine the initial size of hdfs ram buffers. + */ + int m_fColWidth; }; -inline -IDBDataFile::IDBDataFile( const char* fname ) : - m_fname( fname ), - m_fColWidth( 4 ) +inline IDBDataFile::IDBDataFile(const char* fname) : m_fname(fname), m_fColWidth(4) { } -inline -IDBDataFile::~IDBDataFile() +inline IDBDataFile::~IDBDataFile() { } -inline -const std::string& IDBDataFile::name() const +inline const std::string& IDBDataFile::name() const { - return m_fname; + return m_fname; } -} +} // namespace idbdatafile #endif /* IDBDATAFILE_H_ */ diff --git a/utils/idbdatafile/IDBFactory.cpp b/utils/idbdatafile/IDBFactory.cpp index 1956dbf61..873c7eaa5 100644 --- a/utils/idbdatafile/IDBFactory.cpp +++ b/utils/idbdatafile/IDBFactory.cpp @@ -42,94 +42,97 @@ boost::mutex fac_guard; namespace idbdatafile { - IDBFactory::FactoryMap IDBFactory::s_plugins; bool IDBFactory::installDefaultPlugins() { - // protect these methods since we are changing our static data structure - boost::mutex::scoped_lock lock( fac_guard ); + // protect these methods since we are changing our static data structure + boost::mutex::scoped_lock lock(fac_guard); - s_plugins[IDBDataFile::BUFFERED] = FileFactoryEnt(IDBDataFile::BUFFERED, "buffered", new BufferedFileFactory(), new PosixFileSystem()); - s_plugins[IDBDataFile::UNBUFFERED] = FileFactoryEnt(IDBDataFile::UNBUFFERED, "unbuffered", new UnbufferedFileFactory(), new PosixFileSystem()); - - // TODO: use the installPlugin fcn below instead of declaring this statically, then remove the dependency - // IDBDatafile -> cloudio - //s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem()); - - return false; + s_plugins[IDBDataFile::BUFFERED] = + FileFactoryEnt(IDBDataFile::BUFFERED, "buffered", new BufferedFileFactory(), new PosixFileSystem()); + s_plugins[IDBDataFile::UNBUFFERED] = FileFactoryEnt(IDBDataFile::UNBUFFERED, "unbuffered", + new UnbufferedFileFactory(), new PosixFileSystem()); + + // TODO: use the installPlugin fcn below instead of declaring this statically, then remove the dependency + // IDBDatafile -> cloudio + // s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new + // SMFileSystem()); + + return false; } bool IDBFactory::installPlugin(const std::string& plugin) { #ifdef _MSC_VER - ostringstream oss; - oss << "InfiniDB for Windows does not support plugins: plugin = " << plugin; - throw std::runtime_error(oss.str()); + ostringstream oss; + oss << "InfiniDB for Windows does not support plugins: plugin = " << plugin; + throw std::runtime_error(oss.str()); #else - // protect these methods since we are changing our static data structure - boost::mutex::scoped_lock lock( fac_guard ); + // protect these methods since we are changing our static data structure + boost::mutex::scoped_lock lock(fac_guard); - void* handle = dlopen(plugin.c_str(), RTLD_LAZY); - - if ( handle == NULL ) - { - std::ostringstream oss; - oss << "IDBFactory::installPlugin: dlopen for " << plugin << " failed: " << dlerror(); - IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); - return false; - } - - void* functor = dlsym(handle, "plugin_instance"); - - if ( functor == NULL ) - { - std::ostringstream oss; - oss << "IDBFactory::installPlugin: dlsym for plugin_instance() failed. plugin " << plugin << dlerror(); - IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); - return false; - } - - FileFactoryEnt ent = (*(FileFactoryEntryFunc)functor)(); - s_plugins[ent.type] = ent; + void* handle = dlopen(plugin.c_str(), RTLD_LAZY); + if (handle == NULL) + { std::ostringstream oss; - oss << "IDBFactory::installPlugin: installed filesystem plugin " << plugin; - IDBLogger::syslog(oss.str(), logging::LOG_TYPE_DEBUG); - return true; + oss << "IDBFactory::installPlugin: dlopen for " << plugin << " failed: " << dlerror(); + IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); + return false; + } + + void* functor = dlsym(handle, "plugin_instance"); + + if (functor == NULL) + { + std::ostringstream oss; + oss << "IDBFactory::installPlugin: dlsym for plugin_instance() failed. plugin " << plugin << dlerror(); + IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); + return false; + } + + FileFactoryEnt ent = (*(FileFactoryEntryFunc)functor)(); + s_plugins[ent.type] = ent; + + std::ostringstream oss; + oss << "IDBFactory::installPlugin: installed filesystem plugin " << plugin; + IDBLogger::syslog(oss.str(), logging::LOG_TYPE_DEBUG); + return true; #endif } vector IDBFactory::listPlugins() { - vector ret; - for (FactoryMap::iterator it = s_plugins.begin(); it != s_plugins.end(); ++it) - ret.push_back(it->first); - return ret; + vector ret; + for (FactoryMap::iterator it = s_plugins.begin(); it != s_plugins.end(); ++it) + ret.push_back(it->first); + return ret; } -IDBDataFile* IDBFactory::open(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, unsigned colWidth) +IDBDataFile* IDBFactory::open(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, + unsigned colWidth) { - if ( s_plugins.find(type) == s_plugins.end() ) - { - ostringstream oss; - oss << "Cannot find factory plugin type " << (int) type << " to open file: " << fname; - throw std::runtime_error(oss.str()); - } + if (s_plugins.find(type) == s_plugins.end()) + { + ostringstream oss; + oss << "Cannot find factory plugin type " << (int)type << " to open file: " << fname; + throw std::runtime_error(oss.str()); + } - return s_plugins[type].factory->open(fname, mode, opts, colWidth); + return s_plugins[type].factory->open(fname, mode, opts, colWidth); } IDBFileSystem& IDBFactory::getFs(IDBDataFile::Types type) { - if ( s_plugins.find(type) == s_plugins.end() ) - { - ostringstream oss; - oss << "Cannot find filesystem for plugin type " << (int) type; - throw std::runtime_error(oss.str()); - } + if (s_plugins.find(type) == s_plugins.end()) + { + ostringstream oss; + oss << "Cannot find filesystem for plugin type " << (int)type; + throw std::runtime_error(oss.str()); + } - return *(s_plugins[type].filesystem); + return *(s_plugins[type].filesystem); } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/IDBFactory.h b/utils/idbdatafile/IDBFactory.h index 83e29e0a8..4ddfbbf94 100644 --- a/utils/idbdatafile/IDBFactory.h +++ b/utils/idbdatafile/IDBFactory.h @@ -27,32 +27,26 @@ namespace idbdatafile { - class FileFactoryBase; class IDBFileSystem; struct FileFactoryEnt { - FileFactoryEnt() : - type(IDBDataFile::UNKNOWN), - name("unknown"), - factory(0), - filesystem(0) {;} + FileFactoryEnt() : type(IDBDataFile::UNKNOWN), name("unknown"), factory(0), filesystem(0) + { + ; + } - FileFactoryEnt( - IDBDataFile::Types t, - const std::string& n, - FileFactoryBase* f, - IDBFileSystem* fs) : - type(t), - name(n), - factory(f), - filesystem(fs) {;} + FileFactoryEnt(IDBDataFile::Types t, const std::string& n, FileFactoryBase* f, IDBFileSystem* fs) + : type(t), name(n), factory(f), filesystem(fs) + { + ; + } - IDBDataFile::Types type; - std::string name; - FileFactoryBase* factory; - IDBFileSystem* filesystem; + IDBDataFile::Types type; + std::string name; + FileFactoryBase* factory; + IDBFileSystem* filesystem; }; typedef FileFactoryEnt (*FileFactoryEntryFunc)(); @@ -66,62 +60,62 @@ typedef FileFactoryEnt (*FileFactoryEntryFunc)(); */ class IDBFactory { -public: - /** - * This method installs the default plugs for Buffered and Unbuffered files. - * It is called automatically from the IDBPolicy::init() body so that clients - * don't have to worry about it. - */ - static bool installDefaultPlugins(); + public: + /** + * This method installs the default plugs for Buffered and Unbuffered files. + * It is called automatically from the IDBPolicy::init() body so that clients + * don't have to worry about it. + */ + static bool installDefaultPlugins(); - /** - * This method installs a dynamic plugin. The plugin argument must refer - * to a ".so" file that exposes an extern "C" functions: - * FileFactoryEnt plugin_instance() - */ - static bool installPlugin(const std::string& plugin); + /** + * This method installs a dynamic plugin. The plugin argument must refer + * to a ".so" file that exposes an extern "C" functions: + * FileFactoryEnt plugin_instance() + */ + static bool installPlugin(const std::string& plugin); - /** - * This method lists the loaded plugins by type. - */ - static std::vector listPlugins(); - - /** - * This method calls the Factory for the specified type - */ - static IDBDataFile* open(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, unsigned colWidth); + /** + * This method lists the loaded plugins by type. + */ + static std::vector listPlugins(); - /** - * This retrieves the IDBFileSystem for the specified type - */ - static IDBFileSystem& getFs(IDBDataFile::Types type); + /** + * This method calls the Factory for the specified type + */ + static IDBDataFile* open(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, + unsigned colWidth); - /** - * This retrieves the IDBFileSystem for the specified type - */ - static const std::string& name(IDBDataFile::Types type); - -private: - typedef std::map FactoryMap; - typedef FactoryMap::const_iterator FactoryMapCIter; + /** + * This retrieves the IDBFileSystem for the specified type + */ + static IDBFileSystem& getFs(IDBDataFile::Types type); - static FactoryMap s_plugins; + /** + * This retrieves the IDBFileSystem for the specified type + */ + static const std::string& name(IDBDataFile::Types type); - IDBFactory(); - virtual ~IDBFactory(); + private: + typedef std::map FactoryMap; + typedef FactoryMap::const_iterator FactoryMapCIter; + + static FactoryMap s_plugins; + + IDBFactory(); + virtual ~IDBFactory(); }; -inline -const std::string& IDBFactory::name(IDBDataFile::Types type) +inline const std::string& IDBFactory::name(IDBDataFile::Types type) { - if ( s_plugins.find(type) == s_plugins.end() ) - { - throw std::runtime_error("unknown plugin type in IDBFactory::name"); - } + if (s_plugins.find(type) == s_plugins.end()) + { + throw std::runtime_error("unknown plugin type in IDBFactory::name"); + } - return s_plugins[type].name; + return s_plugins[type].name; } -} +} // namespace idbdatafile #endif /* IDBFACTORY_H_ */ diff --git a/utils/idbdatafile/IDBFileSystem.cpp b/utils/idbdatafile/IDBFileSystem.cpp index f3c145d25..e2d8c9c80 100644 --- a/utils/idbdatafile/IDBFileSystem.cpp +++ b/utils/idbdatafile/IDBFileSystem.cpp @@ -19,9 +19,7 @@ namespace idbdatafile { - -IDBFileSystem::IDBFileSystem( Types type ) : - m_type(type) +IDBFileSystem::IDBFileSystem(Types type) : m_type(type) { } @@ -29,4 +27,4 @@ IDBFileSystem::~IDBFileSystem() { } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/IDBFileSystem.h b/utils/idbdatafile/IDBFileSystem.h index 98459e9f1..4ca4a2855 100644 --- a/utils/idbdatafile/IDBFileSystem.h +++ b/utils/idbdatafile/IDBFileSystem.h @@ -25,7 +25,6 @@ namespace idbdatafile { - /** * IDBFileSystem is a combination abstract base class and factory. The * purpose is to encapsulate access to various file system functions @@ -33,137 +32,132 @@ namespace idbdatafile */ class IDBFileSystem { -public: - /** - * The TYPE enum defines the supported underlying filesystem types - */ - enum Types - { - UNKNOWN = 0x00, - POSIX = 0x0001, - HDFS = 0x0002, - CLOUD = 0x0003 - }; + public: + /** + * The TYPE enum defines the supported underlying filesystem types + */ + enum Types + { + UNKNOWN = 0x00, + POSIX = 0x0001, + HDFS = 0x0002, + CLOUD = 0x0003 + }; - /** - * Destructor - */ - virtual ~IDBFileSystem(); + /** + * Destructor + */ + virtual ~IDBFileSystem(); - /** - * getFs() returns a reference to the IDBFileSystem instance for the - * specified type - */ - static IDBFileSystem& getFs(IDBDataFile::Types type); + /** + * getFs() returns a reference to the IDBFileSystem instance for the + * specified type + */ + static IDBFileSystem& getFs(IDBDataFile::Types type); - /** - * mkdir() creates a directory specified by pathname including any - * parent directories if needed. - * Returns 0 on success, -1 on error - */ - virtual int mkdir(const char* pathname) = 0; + /** + * mkdir() creates a directory specified by pathname including any + * parent directories if needed. + * Returns 0 on success, -1 on error + */ + virtual int mkdir(const char* pathname) = 0; - /** - * remove() removes a path from the filesystem. It will handle - * both files and directories (including non-empty directories). Note - * that a call to remove of a path that does not exist will return as - * success. - * - * Returns 0 on success, -1 on error - */ - virtual int remove(const char* pathname) = 0; + /** + * remove() removes a path from the filesystem. It will handle + * both files and directories (including non-empty directories). Note + * that a call to remove of a path that does not exist will return as + * success. + * + * Returns 0 on success, -1 on error + */ + virtual int remove(const char* pathname) = 0; - /** - * rename() names a file, moving it between directories if required. - * Returns 0 on success, -1 on error - */ - virtual int rename(const char* oldpath, const char* newpath) = 0; + /** + * rename() names a file, moving it between directories if required. + * Returns 0 on success, -1 on error + */ + virtual int rename(const char* oldpath, const char* newpath) = 0; - /** - * size() returns the size of the file specified by path. - * Returns the size on success, -1 on error - */ - virtual off64_t size(const char* path) const = 0; + /** + * size() returns the size of the file specified by path. + * Returns the size on success, -1 on error + */ + virtual off64_t size(const char* path) const = 0; - /** - * compressedSize() returns the decompressed size of the file - * speicified by path. - * Returns the size on success, -1 on error - */ - virtual off64_t compressedSize(const char* path) const = 0; + /** + * compressedSize() returns the decompressed size of the file + * speicified by path. + * Returns the size on success, -1 on error + */ + virtual off64_t compressedSize(const char* path) const = 0; - /** - * exists() checks for the existence of a particular path. - * Returns true if exists, false otherwise. - */ - virtual bool exists(const char* pathname) const = 0; + /** + * exists() checks for the existence of a particular path. + * Returns true if exists, false otherwise. + */ + virtual bool exists(const char* pathname) const = 0; - /** - * listDirectory() returns the contents of the directory as a - * list of strings. For now, no indication of whether each - * item is a file or directory - both files and subdirectories - * will be returned. - * Returns 0 on success. -1 on error (incl if directory does - * not exist. - */ + /** + * listDirectory() returns the contents of the directory as a + * list of strings. For now, no indication of whether each + * item is a file or directory - both files and subdirectories + * will be returned. + * Returns 0 on success. -1 on error (incl if directory does + * not exist. + */ - virtual int listDirectory(const char* pathname, std::list& contents) const = 0; + virtual int listDirectory(const char* pathname, std::list& contents) const = 0; - /** - * isDir() returns whether or not the path is a directory - */ - virtual bool isDir(const char* pathname) const = 0; + /** + * isDir() returns whether or not the path is a directory + */ + virtual bool isDir(const char* pathname) const = 0; - /** - * copyfile() copies the source file to the destination file - * Returns 0 on success. -1 on error. - */ - virtual int copyFile(const char* srcPath, const char* destPath) const = 0; + /** + * copyfile() copies the source file to the destination file + * Returns 0 on success. -1 on error. + */ + virtual int copyFile(const char* srcPath, const char* destPath) const = 0; - /** - * isFsUp() checks if the filesystem is up - * Returns 0 on success. -1 on error. - */ - virtual bool filesystemIsUp() const - { - return true; - } + /** + * isFsUp() checks if the filesystem is up + * Returns 0 on success. -1 on error. + */ + virtual bool filesystemIsUp() const + { + return true; + } - virtual bool filesystemSync() const - { - return true; - } + virtual bool filesystemSync() const + { + return true; + } - /** - * chown() changes the owner of the object on the FS - * Returns 0 on success. -1 on error. - */ - virtual int chown(const char* objectName, - const uid_t p_uid, - const gid_t p_pid, - int& funcErrno) const - { - return 0; - } + /** + * chown() changes the owner of the object on the FS + * Returns 0 on success. -1 on error. + */ + virtual int chown(const char* objectName, const uid_t p_uid, const gid_t p_pid, int& funcErrno) const + { + return 0; + } + protected: + IDBFileSystem(Types type); -protected: - IDBFileSystem( Types type ); - -private: - Types m_type; + private: + Types m_type; }; /** * getFs() returns a reference to the IDBFileSystem instance for the * specified type */ -inline -IDBFileSystem& IDBFileSystem::getFs(IDBDataFile::Types type) +inline IDBFileSystem& IDBFileSystem::getFs(IDBDataFile::Types type) { - return IDBFactory::getFs(type); + return IDBFactory::getFs(type); } -} +} // namespace idbdatafile #endif /* IDBFILESYSTEM_H_ */ diff --git a/utils/idbdatafile/IDBLogger.cpp b/utils/idbdatafile/IDBLogger.cpp index 11ce65029..37d4cb3c4 100644 --- a/utils/idbdatafile/IDBLogger.cpp +++ b/utils/idbdatafile/IDBLogger.cpp @@ -41,193 +41,188 @@ using namespace logging; namespace idbdatafile { - bool IDBLogger::s_enabled = false; /* * Log format is: * [timestamp] , [filename/pathname] , [ptr] , [operation] , [op-args1] , [op-args2] , [ret] , [backtrace] */ -void IDBLogger::writeLog( const std::string& logmsg ) +void IDBLogger::writeLog(const std::string& logmsg) { #ifdef _MSC_VER - DWORD pid = GetCurrentProcessId(); - DWORD threadid = GetCurrentThreadId(); + DWORD pid = GetCurrentProcessId(); + DWORD threadid = GetCurrentThreadId(); #else - pid_t pid = getpid(); - pthread_t threadid = pthread_self(); + pid_t pid = getpid(); + pthread_t threadid = pthread_self(); #endif - ostringstream fname; - - string tmpDir = startup::StartUp::tmpDir(); + ostringstream fname; - fname << tmpDir + "/idbdf-log-" << pid << "-" << threadid << ".csv"; + string tmpDir = startup::StartUp::tmpDir(); - ofstream output; - output.open( fname.str().c_str(), ios::out | ios::app ); + fname << tmpDir + "/idbdf-log-" << pid << "-" << threadid << ".csv"; - char fmt[64], buf[64]; - struct timeval tv; - struct tm tm; + ofstream output; + output.open(fname.str().c_str(), ios::out | ios::app); - gettimeofday(&tv, NULL); + char fmt[64], buf[64]; + struct timeval tv; + struct tm tm; + + gettimeofday(&tv, NULL); #ifdef _MSC_VER - errno_t p = 0; - time_t t = (time_t)tv.tv_sec; - p = localtime_s(&tm, &t); + errno_t p = 0; + time_t t = (time_t)tv.tv_sec; + p = localtime_s(&tm, &t); - if (p != 0) - memset(&tm, 0, sizeof(tm)); + if (p != 0) + memset(&tm, 0, sizeof(tm)); #else - localtime_r(&tv.tv_sec, &tm); + localtime_r(&tv.tv_sec, &tm); #endif - strftime(fmt, sizeof fmt, "\'%Y-%m-%d %H:%M:%S.%%06u", &tm); - snprintf(buf, sizeof buf, fmt, tv.tv_usec); + strftime(fmt, sizeof fmt, "\'%Y-%m-%d %H:%M:%S.%%06u", &tm); + snprintf(buf, sizeof buf, fmt, tv.tv_usec); - output << buf << "," << logmsg << "," << get_backtrace(3, 4) << endl; - output.close(); + output << buf << "," << logmsg << "," << get_backtrace(3, 4) << endl; + output.close(); } -void IDBLogger::logOpen(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, IDBDataFile* ret) +void IDBLogger::logOpen(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, + IDBDataFile* ret) { - ostringstream logmsg; - logmsg << fname << ",,open,type="; - logmsg << (( type == IDBDataFile::UNBUFFERED ) ? "unbuffered" : ( type == IDBDataFile::HDFS ) ? "hdfs" : "buffered" ); - logmsg << ";mode=" << mode; - logmsg << ";opts=" << opts << ",,"; - logmsg << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << fname << ",,open,type="; + logmsg << ((type == IDBDataFile::UNBUFFERED) ? "unbuffered" + : (type == IDBDataFile::HDFS) ? "hdfs" + : "buffered"); + logmsg << ";mode=" << mode; + logmsg << ";opts=" << opts << ",,"; + logmsg << ret; + writeLog(logmsg.str()); } void IDBLogger::logNoArg(const std::string& fname, const IDBDataFile* ptr, const char* op, int ret) { - ostringstream logmsg; - logmsg << fname << "," << ptr << "," << op << ",,," << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << fname << "," << ptr << "," << op << ",,," << ret; + writeLog(logmsg.str()); } -void IDBLogger::logRW(const char* op, const std::string& fname, const IDBDataFile* ptr, size_t offset, size_t count, size_t bytesRead) +void IDBLogger::logRW(const char* op, const std::string& fname, const IDBDataFile* ptr, size_t offset, + size_t count, size_t bytesRead) { - ostringstream logmsg; - logmsg << fname << "," << ptr << "," << op << "," << offset << "," << count << "," << bytesRead; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << fname << "," << ptr << "," << op << "," << offset << "," << count << "," << bytesRead; + writeLog(logmsg.str()); } void IDBLogger::logSeek(const std::string& fname, const IDBDataFile* ptr, off64_t offset, int whence, int ret) { - ostringstream logmsg; - logmsg << fname << "," << ptr << ",seek," << offset << "," << whence << "," << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << fname << "," << ptr << ",seek," << offset << "," << whence << "," << ret; + writeLog(logmsg.str()); } void IDBLogger::logTruncate(const std::string& fname, const IDBDataFile* ptr, off64_t length, int ret) { - ostringstream logmsg; - logmsg << fname << "," << ptr << ",truncate," << length << ",," << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << fname << "," << ptr << ",truncate," << length << ",," << ret; + writeLog(logmsg.str()); } void IDBLogger::logSize(const std::string& fname, const IDBDataFile* ptr, long long ret) { - ostringstream logmsg; - logmsg << fname << "," << ptr << ",size,,," << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << fname << "," << ptr << ",size,,," << ret; + writeLog(logmsg.str()); } -void IDBLogger::logFSop(IDBFileSystem::Types type, const char* op, const char* pathname, const IDBFileSystem* ptr, long long ret) +void IDBLogger::logFSop(IDBFileSystem::Types type, const char* op, const char* pathname, + const IDBFileSystem* ptr, long long ret) { - ostringstream logmsg; - logmsg << pathname << "," << ptr << "," << op << ","; - logmsg << "type=" << ( type == IDBFileSystem::POSIX ? "posix" : "hdfs" ) << ","; - logmsg << "," << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << pathname << "," << ptr << "," << op << ","; + logmsg << "type=" << (type == IDBFileSystem::POSIX ? "posix" : "hdfs") << ","; + logmsg << "," << ret; + writeLog(logmsg.str()); } -void IDBLogger::logFSop2(IDBFileSystem::Types type, const char* op, const char* oldpath, const char* newpath, const IDBFileSystem* ptr, long long ret) +void IDBLogger::logFSop2(IDBFileSystem::Types type, const char* op, const char* oldpath, const char* newpath, + const IDBFileSystem* ptr, long long ret) { - ostringstream logmsg; - logmsg << oldpath << "," << ptr << "," << op << ","; - logmsg << "type=" << ( type == IDBFileSystem::POSIX ? "posix" : "hdfs" ) << ","; - logmsg << newpath; - logmsg << "," << ret; - writeLog( logmsg.str() ); + ostringstream logmsg; + logmsg << oldpath << "," << ptr << "," << op << ","; + logmsg << "type=" << (type == IDBFileSystem::POSIX ? "posix" : "hdfs") << ","; + logmsg << newpath; + logmsg << "," << ret; + writeLog(logmsg.str()); } -string IDBLogger::get_backtrace (int to_skip, int num_to_show) +string IDBLogger::get_backtrace(int to_skip, int num_to_show) { - string retval; + string retval; #if !defined(_MSC_VER) && !defined(SKIP_UNWIND) - char name[256]; - unw_cursor_t cursor; - unw_context_t uc; - unw_word_t ip, sp, offp; + char name[256]; + unw_cursor_t cursor; + unw_context_t uc; + unw_word_t ip, sp, offp; - unw_getcontext (&uc); - unw_init_local (&cursor, &uc); + unw_getcontext(&uc); + unw_init_local(&cursor, &uc); - int frame_ct = 0; + int frame_ct = 0; - while (unw_step(&cursor) > 0 && frame_ct < (to_skip + num_to_show)) + while (unw_step(&cursor) > 0 && frame_ct < (to_skip + num_to_show)) + { + ++frame_ct; + + if (frame_ct > to_skip) { - ++frame_ct; + name[0] = '\0'; + unw_get_proc_name(&cursor, name, 256, &offp); + unw_get_reg(&cursor, UNW_REG_IP, &ip); + unw_get_reg(&cursor, UNW_REG_SP, &sp); - if ( frame_ct > to_skip ) - { - name[0] = '\0'; - unw_get_proc_name (&cursor, name, 256, &offp); - unw_get_reg (&cursor, UNW_REG_IP, &ip); - unw_get_reg (&cursor, UNW_REG_SP, &sp); + if (retval.length()) + retval = retval + ","; - if ( retval.length() ) - retval = retval + ","; + char* dem_name_ptr = cplus_demangle(name, 0); - char* dem_name_ptr = cplus_demangle(name, 0); - -// replace with this for full function prototype names including parms -// char* dem_name_ptr = cplus_demangle(name,DMGL_PARAMS | DMGL_ANSI); - if ( dem_name_ptr ) - retval = retval + "\"" + dem_name_ptr + "\""; - else - retval = retval + "\"" + name + "\""; - } + // replace with this for full function prototype names including parms + // char* dem_name_ptr = cplus_demangle(name,DMGL_PARAMS | DMGL_ANSI); + if (dem_name_ptr) + retval = retval + "\"" + dem_name_ptr + "\""; + else + retval = retval + "\"" + name + "\""; } + } #endif - return retval; + return retval; } void IDBLogger::syslog(const std::string& msg, logging::LOG_TYPE level) { - logging::Message::Args args; - logging::Message message(2); - args.add(msg); - message.format(args); - logging::LoggingID lid(35); - logging::MessageLog ml(lid); + logging::Message::Args args; + logging::Message message(2); + args.add(msg); + message.format(args); + logging::LoggingID lid(35); + logging::MessageLog ml(lid); - switch (level) - { - case LOG_TYPE_DEBUG: - ml.logDebugMessage(message); - break; + switch (level) + { + case LOG_TYPE_DEBUG: ml.logDebugMessage(message); break; - case LOG_TYPE_INFO: - ml.logInfoMessage(message); - break; + case LOG_TYPE_INFO: ml.logInfoMessage(message); break; - case LOG_TYPE_WARNING: - ml.logWarningMessage(message); - break; + case LOG_TYPE_WARNING: ml.logWarningMessage(message); break; - case LOG_TYPE_ERROR: - ml.logErrorMessage(message); - break; + case LOG_TYPE_ERROR: ml.logErrorMessage(message); break; - case LOG_TYPE_CRITICAL: - ml.logCriticalMessage(message); - break; - } + case LOG_TYPE_CRITICAL: ml.logCriticalMessage(message); break; + } } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/IDBLogger.h b/utils/idbdatafile/IDBLogger.h index 842ffecb3..b200c9147 100644 --- a/utils/idbdatafile/IDBLogger.h +++ b/utils/idbdatafile/IDBLogger.h @@ -25,48 +25,49 @@ namespace idbdatafile { - class IDBLogger { -public: - static void enable(bool enabled); - static bool isEnabled(); + public: + static void enable(bool enabled); + static bool isEnabled(); - static void logOpen(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, IDBDataFile* ret); - static void logNoArg(const std::string& fname, const IDBDataFile* ptr, const char* op, int ret); - static void logRW(const char* op, const std::string& fname, const IDBDataFile* ptr, size_t offset, size_t count, size_t bytesRead); - static void logSeek(const std::string& fname, const IDBDataFile* ptr, off64_t offset, int whence, int ret); - static void logTruncate(const std::string& fname, const IDBDataFile* ptr, off64_t length, int ret); - static void logSize(const std::string& fname, const IDBDataFile* ptr, long long ret); + static void logOpen(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, + IDBDataFile* ret); + static void logNoArg(const std::string& fname, const IDBDataFile* ptr, const char* op, int ret); + static void logRW(const char* op, const std::string& fname, const IDBDataFile* ptr, size_t offset, + size_t count, size_t bytesRead); + static void logSeek(const std::string& fname, const IDBDataFile* ptr, off64_t offset, int whence, int ret); + static void logTruncate(const std::string& fname, const IDBDataFile* ptr, off64_t length, int ret); + static void logSize(const std::string& fname, const IDBDataFile* ptr, long long ret); - static void logFSop(IDBFileSystem::Types type, const char* op, const char* pathname, const IDBFileSystem* ptr, long long ret); - static void logFSop2(IDBFileSystem::Types type, const char* op, const char* oldpath, const char* newpath, const IDBFileSystem* ptr, long long ret); + static void logFSop(IDBFileSystem::Types type, const char* op, const char* pathname, + const IDBFileSystem* ptr, long long ret); + static void logFSop2(IDBFileSystem::Types type, const char* op, const char* oldpath, const char* newpath, + const IDBFileSystem* ptr, long long ret); - static void syslog(const std::string& msg, logging::LOG_TYPE level); + static void syslog(const std::string& msg, logging::LOG_TYPE level); -private: - static void writeLog( const std::string& logmsg ); + private: + static void writeLog(const std::string& logmsg); - // prevent anyone from instantiating - IDBLogger(); + // prevent anyone from instantiating + IDBLogger(); - static std::string get_backtrace (int to_skip, int num_to_show); + static std::string get_backtrace(int to_skip, int num_to_show); - static bool s_enabled; + static bool s_enabled; }; -inline -void IDBLogger::enable(bool enabled) +inline void IDBLogger::enable(bool enabled) { - s_enabled = enabled; + s_enabled = enabled; } -inline -bool IDBLogger::isEnabled() +inline bool IDBLogger::isEnabled() { - return s_enabled; + return s_enabled; } -} +} // namespace idbdatafile #endif /* IDBLOGGER_H_ */ diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 8210ce93f..01d3f65f0 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -22,7 +22,7 @@ #include #include -#include "configcpp.h" // for Config +#include "configcpp.h" // for Config #include "oamcache.h" #include "liboamcpp.h" #include "IDBPolicy.h" @@ -38,12 +38,10 @@ #include "installdir.h" #include "vlarray.h" - using namespace std; namespace idbdatafile { - bool IDBPolicy::s_usehdfs = false; bool IDBPolicy::s_usecloud = false; bool IDBPolicy::s_bUseRdwrMemBuffer = false; @@ -53,107 +51,110 @@ bool IDBPolicy::s_configed = false; boost::mutex IDBPolicy::s_mutex; std::vector IDBPolicy::s_PreallocSpace; -void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize ) +void IDBPolicy::init(bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, + int64_t hdfsRdwrBufferMaxSize) { - IDBFactory::installDefaultPlugins(); + IDBFactory::installDefaultPlugins(); - IDBLogger::enable( bEnableLogging ); + IDBLogger::enable(bEnableLogging); - s_bUseRdwrMemBuffer = bUseRdwrMemBuffer; - s_hdfsRdwrBufferMaxSize = hdfsRdwrBufferMaxSize; - s_hdfsRdwrScratch = hdfsRdwrScratch; + s_bUseRdwrMemBuffer = bUseRdwrMemBuffer; + s_hdfsRdwrBufferMaxSize = hdfsRdwrBufferMaxSize; + s_hdfsRdwrScratch = hdfsRdwrScratch; - // Create our scratch directory - if ( hdfsRdwrScratch.length() > 0 ) + // Create our scratch directory + if (hdfsRdwrScratch.length() > 0) + { + // TODO-check to make sure this directory has sufficient space, whatever that means. + + boost::filesystem::path tmpfilepath(hdfsRdwrScratch); + + if (boost::filesystem::exists(tmpfilepath)) { - // TODO-check to make sure this directory has sufficient space, whatever that means. - - boost::filesystem::path tmpfilepath( hdfsRdwrScratch ); - - if (boost::filesystem::exists(tmpfilepath)) - { - if (!boost::filesystem::is_directory(tmpfilepath) && useHdfs()) - { - // We found a file named what we want our scratch directory to be named. - // Major issue, since we can't assume anything about where to put our tmp files - ostringstream oss; - oss << "IDBPolicy::init: scratch diretory setting " << hdfsRdwrScratch.c_str() - << " exists as a file. Can't create hdfs buffer files."; - throw runtime_error(oss.str()); - } - } - else - { - cout << tmpfilepath << endl; - bool itWorked = false; - - try - { - itWorked = boost::filesystem::create_directories(tmpfilepath); - } - catch (...) - { } - if (!itWorked) - { - // We failed to create the scratch directory - ostringstream oss; - oss << "IDBPolicy::init: failed to create hdfs scratch directory " - << hdfsRdwrScratch.c_str() << ". Can't create hdfs buffer files."; - throw runtime_error(oss.str()); - } - } + if (!boost::filesystem::is_directory(tmpfilepath) && useHdfs()) + { + // We found a file named what we want our scratch directory to be named. + // Major issue, since we can't assume anything about where to put our tmp files + ostringstream oss; + oss << "IDBPolicy::init: scratch diretory setting " << hdfsRdwrScratch.c_str() + << " exists as a file. Can't create hdfs buffer files."; + throw runtime_error(oss.str()); + } } + else + { + cout << tmpfilepath << endl; + bool itWorked = false; + + try + { + itWorked = boost::filesystem::create_directories(tmpfilepath); + } + catch (...) + { + } + if (!itWorked) + { + // We failed to create the scratch directory + ostringstream oss; + oss << "IDBPolicy::init: failed to create hdfs scratch directory " << hdfsRdwrScratch.c_str() + << ". Can't create hdfs buffer files."; + throw runtime_error(oss.str()); + } + } + } } bool IDBPolicy::installPlugin(const std::string& plugin) { - bool ret = IDBFactory::installPlugin(plugin); + bool ret = IDBFactory::installPlugin(plugin); - vector plugins = IDBFactory::listPlugins(); - for (uint i = 0; i < plugins.size(); i++) - if (plugins[i] == IDBDataFile::HDFS) - s_usehdfs = true; - else if (plugins[i] == IDBDataFile::CLOUD) - s_usecloud = true; + vector plugins = IDBFactory::listPlugins(); + for (uint i = 0; i < plugins.size(); i++) + if (plugins[i] == IDBDataFile::HDFS) + s_usehdfs = true; + else if (plugins[i] == IDBDataFile::CLOUD) + s_usecloud = true; - return ret; + return ret; } -bool IDBPolicy::isLocalFile( const std::string& path ) +bool IDBPolicy::isLocalFile(const std::string& path) { - boost::filesystem::path filepath( path ); + boost::filesystem::path filepath(path); #ifdef _MSC_VER - size_t strmblen = utf8::idb_wcstombs(0, filepath.extension().c_str(), 0) + 1; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); - strmblen = utf8::idb_wcstombs(outbuf, filepath.extension().c_str(), strmblen); - string fileExt(outbuf, strmblen); + size_t strmblen = utf8::idb_wcstombs(0, filepath.extension().c_str(), 0) + 1; + char* outbuf = (char*)alloca(strmblen * sizeof(char)); + strmblen = utf8::idb_wcstombs(outbuf, filepath.extension().c_str(), strmblen); + string fileExt(outbuf, strmblen); #else - //string fileExt = filepath.extension().c_str(); + // string fileExt = filepath.extension().c_str(); #endif - bool isXml = filepath.extension() == ".xml"; - //bool isDbrm = path.find("dbrm") != string::npos; // StorageManager: debatable whether dbrm files should go in the cloud - bool isVb = filepath.filename() == "versionbuffer.cdf"; - bool isScratch = path.find(s_hdfsRdwrScratch) == 0; + bool isXml = filepath.extension() == ".xml"; + // bool isDbrm = path.find("dbrm") != string::npos; // StorageManager: debatable whether dbrm files should + // go in the cloud + bool isVb = filepath.filename() == "versionbuffer.cdf"; + bool isScratch = path.find(s_hdfsRdwrScratch) == 0; - return isXml || isVb || isScratch; - //return isXml || isDbrm || isVb || isScratch; + return isXml || isVb || isScratch; + // return isXml || isDbrm || isVb || isScratch; } -IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt ) +IDBDataFile::Types IDBPolicy::getType(const std::string& path, Contexts ctxt) { - bool isLocal = isLocalFile( path ); + bool isLocal = isLocalFile(path); + + if (isLocal || (!useHdfs() && !useCloud())) + if (ctxt == PRIMPROC) + return IDBDataFile::UNBUFFERED; + else + return IDBDataFile::BUFFERED; + else if (useHdfs()) + return IDBDataFile::HDFS; + else if (useCloud()) + return IDBDataFile::CLOUD; + throw runtime_error("IDBPolicy: No appropriate data file type"); - if (isLocal || (!useHdfs() && !useCloud())) - if (ctxt == PRIMPROC) - return IDBDataFile::UNBUFFERED; - else - return IDBDataFile::BUFFERED; - else if (useHdfs()) - return IDBDataFile::HDFS; - else if (useCloud()) - return IDBDataFile::CLOUD; - throw runtime_error("IDBPolicy: No appropriate data file type"); - #if 0 if ( ctxt == PRIMPROC ) { @@ -172,147 +173,144 @@ IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt ) #endif } -IDBFileSystem& IDBPolicy::getFs( const std::string& path ) +IDBFileSystem& IDBPolicy::getFs(const std::string& path) { - // for now context doesn't actually matter so just pass PRIMPROC - // later the whole logic around file -> filesystem type mapping - // needs to be data driven - return IDBFactory::getFs( getType( path, PRIMPROC) ); + // for now context doesn't actually matter so just pass PRIMPROC + // later the whole logic around file -> filesystem type mapping + // needs to be data driven + return IDBFactory::getFs(getType(path, PRIMPROC)); } // ported from we_config.cpp void IDBPolicy::configIDBPolicy() { - // make sure this is done once. - boost::mutex::scoped_lock lk(s_mutex); + // make sure this is done once. + boost::mutex::scoped_lock lk(s_mutex); - if (s_configed) - return; + if (s_configed) + return; - config::Config* cf = config::Config::makeConfig(); + config::Config* cf = config::Config::makeConfig(); - //-------------------------------------------------------------------------- - // IDBDataFile logging - //-------------------------------------------------------------------------- - bool idblog = false; - string idblogstr = cf->getConfig("SystemConfig", "DataFileLog"); + //-------------------------------------------------------------------------- + // IDBDataFile logging + //-------------------------------------------------------------------------- + bool idblog = false; + string idblogstr = cf->getConfig("SystemConfig", "DataFileLog"); - // Must be faster. - if ( idblogstr.size() == 2 - && ( idblogstr[0] == 'O' || idblogstr[0] == 'o' ) - && ( idblogstr[1] == 'N' || idblogstr[1] == 'n' )) + // Must be faster. + if (idblogstr.size() == 2 && (idblogstr[0] == 'O' || idblogstr[0] == 'o') && + (idblogstr[1] == 'N' || idblogstr[1] == 'n')) + { + idblog = true; + } + + //-------------------------------------------------------------------------- + // Optional File System Plugin - if a HDFS type plugin is loaded + // then the system will use HDFS for all IDB data files + //-------------------------------------------------------------------------- + string fsplugin = cf->getConfig("SystemConfig", "DataFilePlugin"); + + if (fsplugin.length() != 0) + { + IDBPolicy::installPlugin(fsplugin); + } + + //-------------------------------------------------------------------------- + // HDFS file buffering + //-------------------------------------------------------------------------- + // Maximum amount of memory to use for hdfs buffering. + bool bUseRdwrMemBuffer = true; // If true, use in-memory buffering, else use file buffering + int64_t hdfsRdwrBufferMaxSize = 0; + string strBufferMaxSize = cf->getConfig("SystemConfig", "hdfsRdwrBufferMaxSize"); + + // Default is use membuf with no maximum size unless changed by config file. + if (strBufferMaxSize.length() > 0) + { + hdfsRdwrBufferMaxSize = static_cast(cf->uFromText(strBufferMaxSize)); + + if (hdfsRdwrBufferMaxSize == 0) { - idblog = true; + // If we're given a size of 0, turn off membuffering. + bUseRdwrMemBuffer = false; } + } - //-------------------------------------------------------------------------- - // Optional File System Plugin - if a HDFS type plugin is loaded - // then the system will use HDFS for all IDB data files - //-------------------------------------------------------------------------- - string fsplugin = cf->getConfig("SystemConfig", "DataFilePlugin"); + // Directory in which to place file buffer temporary files. + string tmpDir = startup::StartUp::tmpDir(); - if ( fsplugin.length() != 0 ) + string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); + string hdfsRdwrScratch = tmpDir + scratch; + + // MCOL-498. Use DBRootX.PreallocSpace to disable + // dbroots file space preallocation. + // The feature is used in the FileOp code and enabled by default. + char configSectionPref[] = "DBRoot"; + int confSectionLen = sizeof(configSectionPref) + oam::MAX_MODULE_ID_SIZE; + utils::VLArray configSection(confSectionLen); + + IDBPolicy::init(idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize); + s_configed = true; + + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + int PMId = oamcache->getLocalPMId(); + + oam::OamCache::PMDbrootsMap_t pmDbrootsMap; + pmDbrootsMap.reset(new oam::OamCache::PMDbrootsMap_t::element_type()); + oam::systemStorageInfo_t t; + oam::Oam oamInst; + t = oamInst.getStorageConfig(); + + oam::DeviceDBRootList moduledbrootlist = boost::get<2>(t); + oam::DeviceDBRootList::iterator pt = moduledbrootlist.begin(); + + while (pt != moduledbrootlist.end() && (*pt).DeviceID != PMId) + { + pt++; + continue; + } + + // CS could return here b/c it initialised this singleton and + // there is no DBRootX sections in XML. + if (pt == moduledbrootlist.end()) + { + return; + } + + oam::DBRootConfigList& dbRootVec = (*pt).dbrootConfigList; + s_PreallocSpace.reserve(dbRootVec.size() >> 1); + { + int rc; + oam::DBRootConfigList::iterator dbRootIter = dbRootVec.begin(); + for (; dbRootIter != dbRootVec.end(); dbRootIter++) { - IDBPolicy::installPlugin(fsplugin); - } + ::memset(configSection.data() + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE); + rc = snprintf(configSection.data(), confSectionLen, "%s%d", configSectionPref, *dbRootIter); + // gcc 8.2 warnings + if (rc < 0 || rc >= confSectionLen) + { + ostringstream oss; + oss << "IDBPolicy::configIDBPolicy: failed to parse DBRootX section."; + throw runtime_error(oss.str()); + } + string setting = cf->getConfig(configSection.data(), "PreallocSpace"); - //-------------------------------------------------------------------------- - // HDFS file buffering - //-------------------------------------------------------------------------- - // Maximum amount of memory to use for hdfs buffering. - bool bUseRdwrMemBuffer = true; // If true, use in-memory buffering, else use file buffering - int64_t hdfsRdwrBufferMaxSize = 0; - string strBufferMaxSize = cf->getConfig("SystemConfig", "hdfsRdwrBufferMaxSize"); - - // Default is use membuf with no maximum size unless changed by config file. - if (strBufferMaxSize.length() > 0) - { - hdfsRdwrBufferMaxSize = static_cast(cf->uFromText(strBufferMaxSize)); - - if ( hdfsRdwrBufferMaxSize == 0 ) + if (setting.length() != 0) + { + if (setting.size() == 2 && (setting[0] == 'O' || setting[0] == 'o') && + (setting[1] == 'N' || setting[1] == 'n')) { - // If we're given a size of 0, turn off membuffering. - bUseRdwrMemBuffer = false; - } - } - - // Directory in which to place file buffer temporary files. - string tmpDir = startup::StartUp::tmpDir(); - - string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); - string hdfsRdwrScratch = tmpDir + scratch; - - // MCOL-498. Use DBRootX.PreallocSpace to disable - // dbroots file space preallocation. - // The feature is used in the FileOp code and enabled by default. - char configSectionPref[] = "DBRoot"; - int confSectionLen = sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE; - utils::VLArray configSection(confSectionLen); - - IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); - s_configed = true; - - oam::OamCache* oamcache = oam::OamCache::makeOamCache(); - int PMId = oamcache->getLocalPMId(); - - oam::OamCache::PMDbrootsMap_t pmDbrootsMap; - pmDbrootsMap.reset(new oam::OamCache::PMDbrootsMap_t::element_type()); - oam::systemStorageInfo_t t; - oam::Oam oamInst; - t = oamInst.getStorageConfig(); - - oam::DeviceDBRootList moduledbrootlist = boost::get<2>(t); - oam::DeviceDBRootList::iterator pt = moduledbrootlist.begin(); - - while ( pt != moduledbrootlist.end() && (*pt).DeviceID != PMId) - { - pt++; - continue; - } - - // CS could return here b/c it initialised this singleton and - // there is no DBRootX sections in XML. - if (pt == moduledbrootlist.end()) - { - return; - } - - oam::DBRootConfigList &dbRootVec = (*pt).dbrootConfigList; - s_PreallocSpace.reserve(dbRootVec.size()>>1); - { - int rc; - oam::DBRootConfigList::iterator dbRootIter = dbRootVec.begin(); - for(; dbRootIter != dbRootVec.end(); dbRootIter++) - { - ::memset(configSection.data() + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE); - rc = snprintf(configSection.data(), confSectionLen, "%s%d", configSectionPref, *dbRootIter); - // gcc 8.2 warnings - if ( rc < 0 || rc >= confSectionLen) - { - ostringstream oss; - oss << "IDBPolicy::configIDBPolicy: failed to parse DBRootX section."; - throw runtime_error(oss.str()); - } - string setting = cf->getConfig(configSection.data(), "PreallocSpace"); - - if ( setting.length() != 0 ) - { - if ( setting.size() == 2 - && ( setting[0] == 'O' || setting[0] == 'o' ) - && ( setting[1] == 'N' || setting[1] == 'n' ) - ) - { - // int into uint16_t implicit conversion - s_PreallocSpace.push_back(*dbRootIter); - } - } + // int into uint16_t implicit conversion + s_PreallocSpace.push_back(*dbRootIter); } + } } + } } void IDBPolicy::enablePreallocSpace(uint16_t dbRoot) { - s_PreallocSpace.push_back(dbRoot); + s_PreallocSpace.push_back(dbRoot); } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 11a534da2..117f1e8d1 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -30,7 +30,6 @@ namespace idbdatafile { - /** * IDBPolicy is a static class that is used to manage the interaction * between the rest of infinidb and the idbdatafile subsystem. It supports @@ -39,218 +38,202 @@ namespace idbdatafile */ class IDBPolicy { -public: - /** - * Define the contexts recognized by IDBPolicy. The same file may be - * opened as a different type depending on the context (for ex. data - * files are buffered in WriteEngine and Unbuffered in PrimProc). - */ - enum Contexts - { - PRIMPROC, - WRITEENG - }; + public: + /** + * Define the contexts recognized by IDBPolicy. The same file may be + * opened as a different type depending on the context (for ex. data + * files are buffered in WriteEngine and Unbuffered in PrimProc). + */ + enum Contexts + { + PRIMPROC, + WRITEENG + }; - /** - * Config the IDBPolicy based on the Columnstore.xml - */ - static void configIDBPolicy(); + /** + * Config the IDBPolicy based on the Columnstore.xml + */ + static void configIDBPolicy(); + /** + * Initialize the IDBDataFile subsystem. This should be called once + * by the main thread of the application prior to any other use of the + * library. + * bEnableLogging -- for debug only. + * bUseRdwrMemBuffer -- If true, use Memory Buffered files (class HdfsRdwrMemBuffer) until + * hdfsRdwrBufferMaxSize memory is used, then switch to file buffering until mem used is below + * hdfsRdwrBufferMaxSize. hdfsRdwrScratch -- where to store file-buffered HDFS files (class + * HdfsRdwrFileBuffer) - it should name a writable directory with sufficient space to store all needed + * buffers (size-TBD?). hdfsRdwrBufferMaxSize -- When RdwrMemBuffers get this big, switch to file buffers . + */ + static void init(bool bEnableLogging, bool bUseRdwrMemBuffer, const std::string& hdfsRdwrScratch, + int64_t hdfsRdwrBufferMaxSize); - /** - * Initialize the IDBDataFile subsystem. This should be called once - * by the main thread of the application prior to any other use of the - * library. - * bEnableLogging -- for debug only. - * bUseRdwrMemBuffer -- If true, use Memory Buffered files (class HdfsRdwrMemBuffer) until hdfsRdwrBufferMaxSize - * memory is used, then switch to file buffering until mem used is below hdfsRdwrBufferMaxSize. - * hdfsRdwrScratch -- where to store file-buffered HDFS files (class HdfsRdwrFileBuffer) - it should name - * a writable directory with sufficient space to store all needed buffers (size-TBD?). - * hdfsRdwrBufferMaxSize -- When RdwrMemBuffers get this big, switch to file buffers . - */ - static void init( bool bEnableLogging, bool bUseRdwrMemBuffer, const std::string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize ); + /** + * Load a new filetype plugin. Return value indicates success(true) + * or failure (false) + */ + static bool installPlugin(const std::string& plugin); - /** - * Load a new filetype plugin. Return value indicates success(true) - * or failure (false) - */ - static bool installPlugin(const std::string& plugin); + /** + * Accessor method that returns whether or not HDFS is enabled + */ + static bool useHdfs(); - /** - * Accessor method that returns whether or not HDFS is enabled - */ - static bool useHdfs(); + /** + * Accessor method that returns whether or not cloud IO is enabled + */ + static bool useCloud(); - /** - * Accessor method that returns whether or not cloud IO is enabled - */ - static bool useCloud(); + /** + * Checks for disk space preallocation feature status for a dbroot + */ + static bool PreallocSpaceDisabled(uint16_t dbRoot); - /** - * Checks for disk space preallocation feature status for a dbroot - */ - static bool PreallocSpaceDisabled(uint16_t dbRoot); + /** + * Accessor method that returns whether to use HDFS memory buffers + */ + static bool useRdwrMemBuffer(); - /** - * Accessor method that returns whether to use HDFS memory buffers - */ - static bool useRdwrMemBuffer(); + /** + * Accessor method that returns the max amount of mem buff to use before switching to file buffering + */ + static size_t hdfsRdwrBufferMaxSize(); - /** - * Accessor method that returns the max amount of mem buff to use before switching to file buffering - */ - static size_t hdfsRdwrBufferMaxSize(); + /** + * Accessor method that returns the directory to place our temp hdfs buffer files + */ + static const std::string& hdfsRdwrScratch(); - /** - * Accessor method that returns the directory to place our temp hdfs buffer files - */ - static const std::string& hdfsRdwrScratch(); + /** + * getType() returns the proper IDBDataFile::Types for a file given a + * path for the path and a context in which it will be interacted with. + */ + static IDBDataFile::Types getType(const std::string& path, Contexts ctxt); - /** - * getType() returns the proper IDBDataFile::Types for a file given a - * path for the path and a context in which it will be interacted with. - */ - static IDBDataFile::Types getType( const std::string& path, Contexts ctxt ); + /** + * getFs() returns a reference to the proper IDBFileSystem instance that + * is able to interact with the file/directory specified by path + */ + static IDBFileSystem& getFs(const std::string& path); - /** - * getFs() returns a reference to the proper IDBFileSystem instance that - * is able to interact with the file/directory specified by path - */ - static IDBFileSystem& getFs( const std::string& path ); + /** + * These are convenience functions that simplify the syntax required to + * make a filesystem call. Each of the calls determines the proper + * FileSystem to reference for the specified path and then performs the + * actual call. Documentation for these will not be duplicated here - + * please see IDBFileSystem.h. + */ + static int mkdir(const char* pathname); + static off64_t size(const char* path); + static off64_t compressedSize(const char* path); + static int remove(const char* pathname); + static int rename(const char* oldpath, const char* newpath); + static bool exists(const char* pathname); + static int listDirectory(const char* pathname, std::list& contents); + static bool isDir(const char* pathname); + static int copyFile(const char* srcPath, const char* destPath); + /** + * This is used in WE shared components Unit Tests + */ + static void enablePreallocSpace(uint16_t dbRoot); - /** - * These are convenience functions that simplify the syntax required to - * make a filesystem call. Each of the calls determines the proper - * FileSystem to reference for the specified path and then performs the - * actual call. Documentation for these will not be duplicated here - - * please see IDBFileSystem.h. - */ - static int mkdir(const char* pathname); - static off64_t size(const char* path); - static off64_t compressedSize(const char* path); - static int remove(const char* pathname); - static int rename(const char* oldpath, const char* newpath); - static bool exists(const char* pathname); - static int listDirectory(const char* pathname, std::list& contents); - static bool isDir(const char* pathname); - static int copyFile(const char* srcPath, const char* destPath); - /** - * This is used in WE shared components Unit Tests - */ - static void enablePreallocSpace(uint16_t dbRoot); + private: + /** + * don't allow this class to be constructed. It exposes a purely + * static interface + */ + IDBPolicy(); -private: - /** - * don't allow this class to be constructed. It exposes a purely - * static interface - */ - IDBPolicy(); + static bool isLocalFile(const std::string& path); - static bool isLocalFile( const std::string& path ); - - static bool s_usehdfs; - static bool s_usecloud; - static std::vector s_PreallocSpace; - static bool s_bUseRdwrMemBuffer; - static std::string s_hdfsRdwrScratch; - static int64_t s_hdfsRdwrBufferMaxSize; - static bool s_configed; - static boost::mutex s_mutex; + static bool s_usehdfs; + static bool s_usecloud; + static std::vector s_PreallocSpace; + static bool s_bUseRdwrMemBuffer; + static std::string s_hdfsRdwrScratch; + static int64_t s_hdfsRdwrBufferMaxSize; + static bool s_configed; + static boost::mutex s_mutex; }; -inline -const std::string& IDBPolicy::hdfsRdwrScratch() +inline const std::string& IDBPolicy::hdfsRdwrScratch() { - return s_hdfsRdwrScratch; + return s_hdfsRdwrScratch; } -inline -bool IDBPolicy::useHdfs() +inline bool IDBPolicy::useHdfs() { - return s_usehdfs; + return s_usehdfs; } -inline -bool IDBPolicy::useCloud() +inline bool IDBPolicy::useCloud() { - return s_usecloud; + return s_usecloud; } // MCOL-498 Looking for dbRoot in the List set in configIDBPolicy. -inline -bool IDBPolicy::PreallocSpaceDisabled(uint16_t dbRoot) +inline bool IDBPolicy::PreallocSpaceDisabled(uint16_t dbRoot) { - std::vector::iterator dbRootIter = - find(s_PreallocSpace.begin(), s_PreallocSpace.end(), dbRoot); - return dbRootIter == s_PreallocSpace.end(); + std::vector::iterator dbRootIter = find(s_PreallocSpace.begin(), s_PreallocSpace.end(), dbRoot); + return dbRootIter == s_PreallocSpace.end(); } -inline -bool IDBPolicy::useRdwrMemBuffer() +inline bool IDBPolicy::useRdwrMemBuffer() { - return s_bUseRdwrMemBuffer; + return s_bUseRdwrMemBuffer; } -inline -size_t IDBPolicy::hdfsRdwrBufferMaxSize() +inline size_t IDBPolicy::hdfsRdwrBufferMaxSize() { - return s_hdfsRdwrBufferMaxSize; + return s_hdfsRdwrBufferMaxSize; } -inline -int IDBPolicy::mkdir(const char* pathname) +inline int IDBPolicy::mkdir(const char* pathname) { - return IDBPolicy::getFs( pathname ).mkdir( pathname ); + return IDBPolicy::getFs(pathname).mkdir(pathname); } -inline -off64_t IDBPolicy::size(const char* path) +inline off64_t IDBPolicy::size(const char* path) { - return IDBPolicy::getFs( path ).size( path ); + return IDBPolicy::getFs(path).size(path); } -inline -off64_t IDBPolicy::compressedSize(const char* path) +inline off64_t IDBPolicy::compressedSize(const char* path) { - return IDBPolicy::getFs( path ).compressedSize( path ); + return IDBPolicy::getFs(path).compressedSize(path); } -inline -int IDBPolicy::remove(const char* pathname) +inline int IDBPolicy::remove(const char* pathname) { - return IDBPolicy::getFs( pathname ).remove( pathname ); + return IDBPolicy::getFs(pathname).remove(pathname); } -inline -int IDBPolicy::rename(const char* oldpath, const char* newpath) +inline int IDBPolicy::rename(const char* oldpath, const char* newpath) { - return IDBPolicy::getFs( oldpath ).rename( oldpath, newpath ); + return IDBPolicy::getFs(oldpath).rename(oldpath, newpath); } -inline -bool IDBPolicy::exists(const char* pathname) +inline bool IDBPolicy::exists(const char* pathname) { - return IDBPolicy::getFs( pathname ).exists( pathname ); + return IDBPolicy::getFs(pathname).exists(pathname); } -inline -int IDBPolicy::listDirectory(const char* pathname, std::list& contents) +inline int IDBPolicy::listDirectory(const char* pathname, std::list& contents) { - return IDBPolicy::getFs( pathname ).listDirectory( pathname, contents ); + return IDBPolicy::getFs(pathname).listDirectory(pathname, contents); } -inline -bool IDBPolicy::isDir(const char* pathname) +inline bool IDBPolicy::isDir(const char* pathname) { - return IDBPolicy::getFs( pathname ).isDir( pathname ); + return IDBPolicy::getFs(pathname).isDir(pathname); } -inline -int IDBPolicy::copyFile(const char* srcPath, const char* destPath) +inline int IDBPolicy::copyFile(const char* srcPath, const char* destPath) { - return IDBPolicy::getFs( srcPath ).copyFile( srcPath, destPath ); + return IDBPolicy::getFs(srcPath).copyFile(srcPath, destPath); } -} +} // namespace idbdatafile #endif /* IDBPOLICY_H_ */ diff --git a/utils/idbdatafile/PosixFileSystem.cpp b/utils/idbdatafile/PosixFileSystem.cpp index b9bb9644e..bd3e1c4b2 100644 --- a/utils/idbdatafile/PosixFileSystem.cpp +++ b/utils/idbdatafile/PosixFileSystem.cpp @@ -35,9 +35,7 @@ using namespace std; namespace idbdatafile { - -PosixFileSystem::PosixFileSystem() : - IDBFileSystem( IDBFileSystem::POSIX ) +PosixFileSystem::PosixFileSystem() : IDBFileSystem(IDBFileSystem::POSIX) { } @@ -47,275 +45,265 @@ PosixFileSystem::~PosixFileSystem() int PosixFileSystem::mkdir(const char* pathname) { - int ret = 0; - boost::filesystem::path pathDir(pathname); + int ret = 0; + boost::filesystem::path pathDir(pathname); - try - { - boost::filesystem::create_directories(pathDir); - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Failed to create directories: " << pathDir << ", exception: " << ex.what() << endl; - IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); - ret = -1; - } + try + { + boost::filesystem::create_directories(pathDir); + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Failed to create directories: " << pathDir << ", exception: " << ex.what() << endl; + IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); + ret = -1; + } - if ( IDBLogger::isEnabled() ) - IDBLogger::logFSop( POSIX, "mkdir", pathname, this, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logFSop(POSIX, "mkdir", pathname, this, ret); - return ret; + return ret; } int PosixFileSystem::remove(const char* pathname) { - int ret = 0; - boost::filesystem::path dirPath(pathname); + int ret = 0; + boost::filesystem::path dirPath(pathname); - try - { - boost::filesystem::remove_all(dirPath); - } + try + { + boost::filesystem::remove_all(dirPath); + } #ifdef _MSC_VER - catch (std::exception& ex) - { - // FIXME: alas, Windows cannot delete a file that is in use :-( - std::string reason(ex.what()); - std::string ignore("The directory is not empty"); + catch (std::exception& ex) + { + // FIXME: alas, Windows cannot delete a file that is in use :-( + std::string reason(ex.what()); + std::string ignore("The directory is not empty"); - if (reason.find(ignore) != std::string::npos) - (void)0; - else - ret = -1; - } + if (reason.find(ignore) != std::string::npos) + (void)0; + else + ret = -1; + } #endif - catch (...) - { - // TODO Log this - ret = -1; - } + catch (...) + { + // TODO Log this + ret = -1; + } - if ( IDBLogger::isEnabled() ) - IDBLogger::logFSop( POSIX, "remove", pathname, this, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logFSop(POSIX, "remove", pathname, this, ret); - return ret; + return ret; } int PosixFileSystem::rename(const char* oldpath, const char* newpath) { - // should this use Boost?? - int ret = ::rename(oldpath, newpath); - int savedErrno = errno; + // should this use Boost?? + int ret = ::rename(oldpath, newpath); + int savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logFSop2( POSIX, "rename", oldpath, newpath, this, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logFSop2(POSIX, "rename", oldpath, newpath, this, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } off64_t PosixFileSystem::size(const char* path) const { - // should this use Boost?? - struct stat statBuf; - int rc = ::stat( path, &statBuf ); - int savedErrno = errno; - off64_t ret = ((rc == 0) ? statBuf.st_size : -1); + // should this use Boost?? + struct stat statBuf; + int rc = ::stat(path, &statBuf); + int savedErrno = errno; + off64_t ret = ((rc == 0) ? statBuf.st_size : -1); - if ( IDBLogger::isEnabled() ) - IDBLogger::logFSop( POSIX, "fs:size", path, this, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logFSop(POSIX, "fs:size", path, this, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } -size_t readFillBuffer( - idbdatafile::IDBDataFile* pFile, - char* buffer, - size_t bytesReq) +size_t readFillBuffer(idbdatafile::IDBDataFile* pFile, char* buffer, size_t bytesReq) { - char* pBuf = buffer; - ssize_t nBytes; - size_t bytesToRead = bytesReq; - size_t totalBytesRead = 0; + char* pBuf = buffer; + ssize_t nBytes; + size_t bytesToRead = bytesReq; + size_t totalBytesRead = 0; - while (1) - { - nBytes = pFile->read(pBuf, bytesToRead); + while (1) + { + nBytes = pFile->read(pBuf, bytesToRead); - if (nBytes > 0) - totalBytesRead += nBytes; - else - break; + if (nBytes > 0) + totalBytesRead += nBytes; + else + break; - if ((size_t)nBytes == bytesToRead) - break; + if ((size_t)nBytes == bytesToRead) + break; - pBuf += nBytes; - bytesToRead = bytesToRead - (size_t)nBytes; - } + pBuf += nBytes; + bytesToRead = bytesToRead - (size_t)nBytes; + } - return totalBytesRead; + return totalBytesRead; } off64_t PosixFileSystem::compressedSize(const char* path) const { - IDBDataFile* pFile = NULL; - size_t nBytes; - off64_t dataSize = 0; + IDBDataFile* pFile = NULL; + size_t nBytes; + off64_t dataSize = 0; - try + try + { + pFile = IDBDataFile::open(IDBDataFile::BUFFERED, path, "r", 0); + + if (!pFile) { - pFile = IDBDataFile::open(IDBDataFile::BUFFERED, path, "r", 0); - - if (!pFile) - { - return -1; - } - - char hdr1[compress::CompressInterface::HDR_BUF_LEN]; - nBytes = readFillBuffer( pFile, hdr1, compress::CompressInterface::HDR_BUF_LEN); - - if ( nBytes != compress::CompressInterface::HDR_BUF_LEN ) - { - delete pFile; - return -1; - } - - // Verify we are a compressed file - if (compress::CompressInterface::verifyHdr(hdr1) < 0) - { - delete pFile; - return -1; - } - - int64_t ptrSecSize = compress::CompressInterface::getHdrSize(hdr1) - - compress::CompressInterface::HDR_BUF_LEN; - char* hdr2 = new char[ptrSecSize]; - nBytes = readFillBuffer( pFile, hdr2, ptrSecSize); - - if ( (int64_t)nBytes != ptrSecSize ) - { - delete[] hdr2; - delete pFile; - return -1; - } - - compress::CompChunkPtrList chunkPtrs; - int rc = compress::CompressInterface::getPtrList(hdr2, ptrSecSize, - chunkPtrs); - delete[] hdr2; - - if (rc != 0) - { - delete pFile; - return -1; - } - - unsigned k = chunkPtrs.size(); - - // last header's offset + length will be the data bytes - if (k < 1) - { - delete pFile; - return -1; - } - - dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second; - delete pFile; - return dataSize; + return -1; } - catch (...) + + char hdr1[compress::CompressInterface::HDR_BUF_LEN]; + nBytes = readFillBuffer(pFile, hdr1, compress::CompressInterface::HDR_BUF_LEN); + + if (nBytes != compress::CompressInterface::HDR_BUF_LEN) { - delete pFile; - return -1; + delete pFile; + return -1; } + + // Verify we are a compressed file + if (compress::CompressInterface::verifyHdr(hdr1) < 0) + { + delete pFile; + return -1; + } + + int64_t ptrSecSize = + compress::CompressInterface::getHdrSize(hdr1) - compress::CompressInterface::HDR_BUF_LEN; + char* hdr2 = new char[ptrSecSize]; + nBytes = readFillBuffer(pFile, hdr2, ptrSecSize); + + if ((int64_t)nBytes != ptrSecSize) + { + delete[] hdr2; + delete pFile; + return -1; + } + + compress::CompChunkPtrList chunkPtrs; + int rc = compress::CompressInterface::getPtrList(hdr2, ptrSecSize, chunkPtrs); + delete[] hdr2; + + if (rc != 0) + { + delete pFile; + return -1; + } + + unsigned k = chunkPtrs.size(); + + // last header's offset + length will be the data bytes + if (k < 1) + { + delete pFile; + return -1; + } + + dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second; + delete pFile; + return dataSize; + } + catch (...) + { + delete pFile; + return -1; + } } bool PosixFileSystem::exists(const char* pathname) const { - boost::filesystem::path dirPath(pathname); - return boost::filesystem::exists( dirPath ); + boost::filesystem::path dirPath(pathname); + return boost::filesystem::exists(dirPath); } int PosixFileSystem::listDirectory(const char* pathname, std::list& contents) const { - int ret = 0; + int ret = 0; - // clear the return list - contents.erase( contents.begin(), contents.end() ); + // clear the return list + contents.erase(contents.begin(), contents.end()); - try + try + { + boost::filesystem::path dirPath(pathname); + boost::filesystem::directory_iterator end_itr; // create EOD marker + + // Loop through all the files in the specified directory + for (boost::filesystem::directory_iterator itr(dirPath); itr != end_itr; ++itr) { - boost::filesystem::path dirPath( pathname ); - boost::filesystem::directory_iterator end_itr; // create EOD marker - - // Loop through all the files in the specified directory - for ( boost::filesystem::directory_iterator itr( dirPath ); - itr != end_itr; - ++itr ) - { - contents.push_back( itr->path().filename().generic_string() ); - } - } - catch (std::exception &) - { - ret = -1; + contents.push_back(itr->path().filename().generic_string()); } + } + catch (std::exception&) + { + ret = -1; + } - return ret; + return ret; } bool PosixFileSystem::isDir(const char* pathname) const { - boost::filesystem::path dirPath(pathname); - return boost::filesystem::is_directory( dirPath ); + boost::filesystem::path dirPath(pathname); + return boost::filesystem::is_directory(dirPath); } int PosixFileSystem::copyFile(const char* srcPath, const char* destPath) const { - int ret = 0; + int ret = 0; - try - { - boost::filesystem::path inPath(srcPath); - boost::filesystem::path outPath(destPath); + try + { + boost::filesystem::path inPath(srcPath); + boost::filesystem::path outPath(destPath); - boost::filesystem::copy_file(inPath, outPath); - } + boost::filesystem::copy_file(inPath, outPath); + } #if BOOST_VERSION >= 105200 - catch (boost::filesystem::filesystem_error& ex) + catch (boost::filesystem::filesystem_error& ex) #else - catch (boost::filesystem::basic_filesystem_error& ex) + catch (boost::filesystem::basic_filesystem_error& ex) #endif - { - std::ostringstream oss; - oss << "Failed to copy file: " << srcPath << " to " << destPath << - ", exception: " << ex.what() << endl; - IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); - ret = -1; - } + { + std::ostringstream oss; + oss << "Failed to copy file: " << srcPath << " to " << destPath << ", exception: " << ex.what() << endl; + IDBLogger::syslog(oss.str(), logging::LOG_TYPE_ERROR); + ret = -1; + } - if ( IDBLogger::isEnabled() ) - IDBLogger::logFSop2( POSIX, "copyFile", srcPath, destPath, this, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logFSop2(POSIX, "copyFile", srcPath, destPath, this, ret); - return ret; + return ret; } -int PosixFileSystem::chown(const char* objectName, - const uid_t p_uid, - const gid_t p_gid, - int& funcErrno) const +int PosixFileSystem::chown(const char* objectName, const uid_t p_uid, const gid_t p_gid, int& funcErrno) const { - int ret = 0; - errno = 0; - if ((ret = ::chown(objectName, p_uid, p_gid))) - funcErrno = errno; - return ret; + int ret = 0; + errno = 0; + if ((ret = ::chown(objectName, p_uid, p_gid))) + funcErrno = errno; + return ret; } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/PosixFileSystem.h b/utils/idbdatafile/PosixFileSystem.h index ab976d8bc..c1f4e5ff7 100644 --- a/utils/idbdatafile/PosixFileSystem.h +++ b/utils/idbdatafile/PosixFileSystem.h @@ -22,28 +22,24 @@ namespace idbdatafile { - class PosixFileSystem : public IDBFileSystem { -public: - PosixFileSystem(); - ~PosixFileSystem(); + public: + PosixFileSystem(); + ~PosixFileSystem(); - int mkdir(const char* pathname) override; - off64_t size(const char* path) const override; - off64_t compressedSize(const char* path) const override; - int remove(const char* pathname) override; - int rename(const char* oldpath, const char* newpath) override; - bool exists(const char* pathname) const override; - int listDirectory(const char* pathname, std::list& contents) const override; - bool isDir(const char* pathname) const override; - int copyFile(const char* srcPath, const char* destPath) const override; - int chown(const char* objectName, - const uid_t p_uid, - const gid_t p_pid, - int& funcErrno) const override; + int mkdir(const char* pathname) override; + off64_t size(const char* path) const override; + off64_t compressedSize(const char* path) const override; + int remove(const char* pathname) override; + int rename(const char* oldpath, const char* newpath) override; + bool exists(const char* pathname) const override; + int listDirectory(const char* pathname, std::list& contents) const override; + bool isDir(const char* pathname) const override; + int copyFile(const char* srcPath, const char* destPath) const override; + int chown(const char* objectName, const uid_t p_uid, const gid_t p_pid, int& funcErrno) const override; }; -} +} // namespace idbdatafile #endif /* POSIXFILESYSTEM_H_ */ diff --git a/utils/idbdatafile/UnbufferedFile.cpp b/utils/idbdatafile/UnbufferedFile.cpp index 5286c69b4..07a7630bd 100644 --- a/utils/idbdatafile/UnbufferedFile.cpp +++ b/utils/idbdatafile/UnbufferedFile.cpp @@ -28,305 +28,303 @@ using namespace std; namespace idbdatafile { - -UnbufferedFile::UnbufferedFile(const char* fname, const char* mode, unsigned opts) : - IDBDataFile( fname ) +UnbufferedFile::UnbufferedFile(const char* fname, const char* mode, unsigned opts) : IDBDataFile(fname) { #ifdef _MSC_VER - int createFlags; - int flags = modeStrToFlags(mode, createFlags); - m_fd = CreateFile(fname, flags, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, - createFlags, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, 0); + int createFlags; + int flags = modeStrToFlags(mode, createFlags); + m_fd = CreateFile(fname, flags, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, createFlags, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, 0); - if ( m_fd == INVALID_HANDLE_VALUE ) - { - throw std::runtime_error("unable to open Unbuffered file "); - } + if (m_fd == INVALID_HANDLE_VALUE) + { + throw std::runtime_error("unable to open Unbuffered file "); + } #else - int flags = modeStrToFlags(mode); + int flags = modeStrToFlags(mode); - if ( flags == -1 ) - { - ostringstream oss; - oss << "Error opening file - unsupported mode " << mode; - throw std::runtime_error(oss.str()); - } + if (flags == -1) + { + ostringstream oss; + oss << "Error opening file - unsupported mode " << mode; + throw std::runtime_error(oss.str()); + } - // special flags we set by convention due to InfiniDB usage - flags |= O_LARGEFILE | O_NOATIME; + // special flags we set by convention due to InfiniDB usage + flags |= O_LARGEFILE | O_NOATIME; - if ( opts & IDBDataFile::USE_ODIRECT ) - { - flags |= O_DIRECT; - } + if (opts & IDBDataFile::USE_ODIRECT) + { + flags |= O_DIRECT; + } - m_fd = ::open(fname, flags, S_IRWXU); + m_fd = ::open(fname, flags, S_IRWXU); - if ( m_fd == -1 ) - { - m_fd = INVALID_HANDLE_VALUE; - throw std::runtime_error("unable to open Unbuffered file "); - } + if (m_fd == -1) + { + m_fd = INVALID_HANDLE_VALUE; + throw std::runtime_error("unable to open Unbuffered file "); + } #endif } UnbufferedFile::~UnbufferedFile() { - close(); + close(); } ssize_t UnbufferedFile::pread(void* ptr, off64_t offset, size_t count) { - ssize_t ret; - int savedErrno; + ssize_t ret; + int savedErrno; - if (m_fd == INVALID_HANDLE_VALUE) - return -1; + if (m_fd == INVALID_HANDLE_VALUE) + return -1; #ifdef _MSC_VER - OVERLAPPED ovl; - memset(&ovl, 0, sizeof(ovl)); - DWORD mtest = (DWORD)offset; - ovl.Offset = (DWORD)(offset & 0x00000000FFFFFFFF); - ovl.OffsetHigh = (DWORD)(offset >> 32); - DWORD bytesRead; + OVERLAPPED ovl; + memset(&ovl, 0, sizeof(ovl)); + DWORD mtest = (DWORD)offset; + ovl.Offset = (DWORD)(offset & 0x00000000FFFFFFFF); + ovl.OffsetHigh = (DWORD)(offset >> 32); + DWORD bytesRead; - if (ReadFile(m_fd, ptr, (DWORD)count, &bytesRead, &ovl)) - ret = bytesRead; - else - ret = -1; + if (ReadFile(m_fd, ptr, (DWORD)count, &bytesRead, &ovl)) + ret = bytesRead; + else + ret = -1; - savedErrno = errno; + savedErrno = errno; #else - ret = ::pread(m_fd, ptr, count, offset); - savedErrno = errno; + ret = ::pread(m_fd, ptr, count, offset); + savedErrno = errno; #endif - if ( IDBLogger::isEnabled() ) - IDBLogger::logRW("pread", m_fname, this, offset, count, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logRW("pread", m_fname, this, offset, count, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } ssize_t UnbufferedFile::read(void* ptr, size_t count) { - ssize_t ret = 0; - ssize_t offset = tell(); - int savedErrno; + ssize_t ret = 0; + ssize_t offset = tell(); + int savedErrno; #ifdef _MSC_VER - DWORD bytesRead; + DWORD bytesRead; - if (ReadFile(m_fd, ptr, (DWORD)count, &bytesRead, NULL)) - ret = bytesRead; - else - ret = -1; + if (ReadFile(m_fd, ptr, (DWORD)count, &bytesRead, NULL)) + ret = bytesRead; + else + ret = -1; #else - ret = ::read(m_fd, ptr, count); + ret = ::read(m_fd, ptr, count); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logRW("read", m_fname, this, offset, count, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logRW("read", m_fname, this, offset, count, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } ssize_t UnbufferedFile::write(const void* ptr, size_t count) { - ssize_t ret = 0; - ssize_t offset = tell(); - int savedErrno; + ssize_t ret = 0; + ssize_t offset = tell(); + int savedErrno; #ifdef _MSC_VER - DWORD bytesWritten; + DWORD bytesWritten; - if (WriteFile(m_fd, ptr, (DWORD)count, &bytesWritten, NULL)) - ret = bytesWritten; - else - ret = -1; + if (WriteFile(m_fd, ptr, (DWORD)count, &bytesWritten, NULL)) + ret = bytesWritten; + else + ret = -1; #else - ret = ::write(m_fd, ptr, count); + ret = ::write(m_fd, ptr, count); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logRW("write", m_fname, this, offset, count, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logRW("write", m_fname, this, offset, count, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } int UnbufferedFile::seek(off64_t offset, int whence) { - int ret; - int savedErrno; + int ret; + int savedErrno; #ifdef _MSC_VER - LONG lDistanceToMove = LONG(offset & 0x00000000FFFFFFFF); - LONG lDistanceToMoveHigh = LONG(offset >> 32); - ret = SetFilePointer(m_fd, lDistanceToMove, &lDistanceToMoveHigh, whence); + LONG lDistanceToMove = LONG(offset & 0x00000000FFFFFFFF); + LONG lDistanceToMoveHigh = LONG(offset >> 32); + ret = SetFilePointer(m_fd, lDistanceToMove, &lDistanceToMoveHigh, whence); #else - ret = (lseek(m_fd, offset, whence) >= 0) ? 0 : -1; + ret = (lseek(m_fd, offset, whence) >= 0) ? 0 : -1; #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logSeek(m_fname, this, offset, whence, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logSeek(m_fname, this, offset, whence, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } int UnbufferedFile::truncate(off64_t length) { - int ret; - int savedErrno; + int ret; + int savedErrno; #ifdef _MSC_VER - LONG lDistanceToMove = LONG(length & 0x00000000FFFFFFFF); - LONG lDistanceToMoveHigh = LONG(length >> 32); - ret = SetFilePointer(m_fd, lDistanceToMove, &lDistanceToMoveHigh, SEEK_SET); + LONG lDistanceToMove = LONG(length & 0x00000000FFFFFFFF); + LONG lDistanceToMoveHigh = LONG(length >> 32); + ret = SetFilePointer(m_fd, lDistanceToMove, &lDistanceToMoveHigh, SEEK_SET); - if (ret > 0) - ret = SetEndOfFile(m_fd); + if (ret > 0) + ret = SetEndOfFile(m_fd); #else - ret = ftruncate(m_fd, length); + ret = ftruncate(m_fd, length); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logTruncate(m_fname, this, length, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logTruncate(m_fname, this, length, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } off64_t UnbufferedFile::size() { - off64_t ret = 0; - int savedErrno; + off64_t ret = 0; + int savedErrno; #ifdef _MSC_VER - DWORD hi = 0; - DWORD lo = GetFileSize(m_fd, &hi); - ret = off64_t(((uint64_t)hi) << 32) | lo; + DWORD hi = 0; + DWORD lo = GetFileSize(m_fd, &hi); + ret = off64_t(((uint64_t)hi) << 32) | lo; #else - struct stat statBuf; - int rc = ::fstat( m_fd, &statBuf ); - ret = ((rc == 0) ? statBuf.st_size : -1); + struct stat statBuf; + int rc = ::fstat(m_fd, &statBuf); + ret = ((rc == 0) ? statBuf.st_size : -1); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logSize(m_fname, this, ret); + if (IDBLogger::isEnabled()) + IDBLogger::logSize(m_fname, this, ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } off64_t UnbufferedFile::tell() { - off64_t ret; + off64_t ret; #ifdef _MSC_VER - LARGE_INTEGER wRet; - LARGE_INTEGER dist; - dist.QuadPart = 0; - SetFilePointerEx(m_fd, dist, &wRet, FILE_CURRENT); - ret = wRet.QuadPart; + LARGE_INTEGER wRet; + LARGE_INTEGER dist; + dist.QuadPart = 0; + SetFilePointerEx(m_fd, dist, &wRet, FILE_CURRENT); + ret = wRet.QuadPart; #else - ret = lseek(m_fd, 0, SEEK_CUR); + ret = lseek(m_fd, 0, SEEK_CUR); #endif - return ret; + return ret; } int UnbufferedFile::flush() { - int ret; - int savedErrno; + int ret; + int savedErrno; #ifdef _MSC_VER - ret = FlushFileBuffers(m_fd); + ret = FlushFileBuffers(m_fd); - // In this case for Windows, ret is the reverse of Linux - if (ret == 0) - ret = -1; - else - ret = 0; + // In this case for Windows, ret is the reverse of Linux + if (ret == 0) + ret = -1; + else + ret = 0; #else - ret = fsync( m_fd ); + ret = fsync(m_fd); #endif - savedErrno = errno; + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - IDBLogger::logNoArg(m_fname, this, "flush", ret); + if (IDBLogger::isEnabled()) + IDBLogger::logNoArg(m_fname, this, "flush", ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } time_t UnbufferedFile::mtime() { - time_t ret = 0; + time_t ret = 0; #ifdef _MSC_VER - BY_HANDLE_FILE_INFORMATION info; + BY_HANDLE_FILE_INFORMATION info; - if (GetFileInformationByHandle(m_fd, &info)) - { - ret = time_t((uint64_t)info.ftLastAccessTime.dwHighDateTime << 32) | info.ftLastAccessTime.dwLowDateTime; - } - else - { - ret = (time_t) - 1; - } + if (GetFileInformationByHandle(m_fd, &info)) + { + ret = time_t((uint64_t)info.ftLastAccessTime.dwHighDateTime << 32) | info.ftLastAccessTime.dwLowDateTime; + } + else + { + ret = (time_t)-1; + } #else - struct stat statbuf; + struct stat statbuf; - if (::fstat(m_fd, &statbuf) == 0) - ret = statbuf.st_mtime; - else - ret = (time_t) - 1; + if (::fstat(m_fd, &statbuf) == 0) + ret = statbuf.st_mtime; + else + ret = (time_t)-1; #endif - return ret; + return ret; } int UnbufferedFile::close() { - int ret = -1; - int savedErrno = EINVAL; // corresponds to INVALID_HANDLE_VALUE + int ret = -1; + int savedErrno = EINVAL; // corresponds to INVALID_HANDLE_VALUE - if (m_fd != INVALID_HANDLE_VALUE) - { + if (m_fd != INVALID_HANDLE_VALUE) + { #ifdef _MSC_VER - ret = CloseHandle(m_fd); + ret = CloseHandle(m_fd); - // In this case for Windows, ret is the reverse of Linux - if (ret == 0) - ret = -1; - else - ret = 0; + // In this case for Windows, ret is the reverse of Linux + if (ret == 0) + ret = -1; + else + ret = 0; #else - ret = ::close(m_fd); + ret = ::close(m_fd); #endif - savedErrno = errno; - } + savedErrno = errno; + } - if ( IDBLogger::isEnabled() ) - IDBLogger::logNoArg(m_fname, this, "close", ret); + if (IDBLogger::isEnabled()) + IDBLogger::logNoArg(m_fname, this, "close", ret); - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } /** @@ -337,19 +335,19 @@ int UnbufferedFile::close() */ int UnbufferedFile::fallocate(int mode, off64_t offset, off64_t length) { - int ret = 0; - int savedErrno = 0; + int ret = 0; + int savedErrno = 0; - ret = ::fallocate( m_fd, mode, offset, length ); - savedErrno = errno; + ret = ::fallocate(m_fd, mode, offset, length); + savedErrno = errno; - if ( IDBLogger::isEnabled() ) - { - IDBLogger::logNoArg(m_fname, this, "fallocate", errno); - } + if (IDBLogger::isEnabled()) + { + IDBLogger::logNoArg(m_fname, this, "fallocate", errno); + } - errno = savedErrno; - return ret; + errno = savedErrno; + return ret; } -} +} // namespace idbdatafile diff --git a/utils/idbdatafile/UnbufferedFile.h b/utils/idbdatafile/UnbufferedFile.h index 6564c59d7..61e41688c 100644 --- a/utils/idbdatafile/UnbufferedFile.h +++ b/utils/idbdatafile/UnbufferedFile.h @@ -26,7 +26,6 @@ namespace idbdatafile { - /** * UnbufferedFile implements the IDBDataFile for I/O to a standand kernel * fd (via open, write, read, etc.). See IDBDataFile.h for more documentation @@ -34,33 +33,33 @@ namespace idbdatafile */ class UnbufferedFile : public IDBDataFile, boost::noncopyable { -public: - UnbufferedFile(const char* fname, const char* mode, unsigned opts); - /* virtual */ ~UnbufferedFile(); + public: + UnbufferedFile(const char* fname, const char* mode, unsigned opts); + /* virtual */ ~UnbufferedFile(); - /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count); - /* virtual */ ssize_t read(void* ptr, size_t count); - /* virtual */ ssize_t write(const void* ptr, size_t count); - /* virtual */ int seek(off64_t offset, int whence); - /* virtual */ int truncate(off64_t length); - /* virtual */ off64_t size(); - /* virtual */ off64_t tell(); - /* virtual */ int flush(); - /* virtual */ time_t mtime(); - /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); + /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count); + /* virtual */ ssize_t read(void* ptr, size_t count); + /* virtual */ ssize_t write(const void* ptr, size_t count); + /* virtual */ int seek(off64_t offset, int whence); + /* virtual */ int truncate(off64_t length); + /* virtual */ off64_t size(); + /* virtual */ off64_t tell(); + /* virtual */ int flush(); + /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); -protected: - /* virtual */ - int close(); + protected: + /* virtual */ + int close(); -private: + private: #ifdef _MSC_VER - HANDLE m_fd; + HANDLE m_fd; #else - int m_fd; + int m_fd; #endif }; -} +} // namespace idbdatafile #endif /* UNBUFFEREDFILE_H_ */ diff --git a/utils/idbdatafile/UnbufferedFileFactory.h b/utils/idbdatafile/UnbufferedFileFactory.h index 3e2d59856..de35abcc8 100644 --- a/utils/idbdatafile/UnbufferedFileFactory.h +++ b/utils/idbdatafile/UnbufferedFileFactory.h @@ -24,19 +24,18 @@ namespace idbdatafile { - class UnbufferedFileFactory : public FileFactoryBase { -public: - /* virtual */ - IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); + public: + /* virtual */ + IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); }; -inline -IDBDataFile* UnbufferedFileFactory::open(const char* fname, const char* mode, unsigned opts, unsigned /*colWidth*/) +inline IDBDataFile* UnbufferedFileFactory::open(const char* fname, const char* mode, unsigned opts, + unsigned /*colWidth*/) { - return new UnbufferedFile(fname, mode, opts); + return new UnbufferedFile(fname, mode, opts); } -} +} // namespace idbdatafile #endif /* UNBUFFEREDFILEFACTORY_H_ */ diff --git a/utils/idbdatafile/largefile64.h b/utils/idbdatafile/largefile64.h index 6c919804e..aba000b51 100644 --- a/utils/idbdatafile/largefile64.h +++ b/utils/idbdatafile/largefile64.h @@ -35,12 +35,12 @@ #ifdef __FreeBSD__ typedef int64_t off64_t; -# ifndef O_LARGEFILE -# define O_LARGEFILE 0 -# endif -# ifndef O_NOATIME -# define O_NOATIME 0 -# endif +#ifndef O_LARGEFILE +#define O_LARGEFILE 0 +#endif +#ifndef O_NOATIME +#define O_NOATIME 0 +#endif #endif #endif diff --git a/utils/idbdatafile/tdriver.cpp b/utils/idbdatafile/tdriver.cpp index 44abb0b68..42b468737 100644 --- a/utils/idbdatafile/tdriver.cpp +++ b/utils/idbdatafile/tdriver.cpp @@ -44,1345 +44,1344 @@ size_t BLK_SIZE = 2048; const size_t MAX_BLK_SIZE = 1048576; size_t MAX_BLOCK = 1024; -int timeval_subtract (struct timeval* result, struct timeval* x, struct timeval* y) +int timeval_subtract(struct timeval* result, struct timeval* x, struct timeval* y) { - /* Perform the carry for the later subtraction by updating y. */ - if (x->tv_usec < y->tv_usec) - { - int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; - y->tv_usec -= 1000000 * nsec; - y->tv_sec += nsec; - } + /* Perform the carry for the later subtraction by updating y. */ + if (x->tv_usec < y->tv_usec) + { + int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; + y->tv_usec -= 1000000 * nsec; + y->tv_sec += nsec; + } - if (x->tv_usec - y->tv_usec > 1000000) - { - int nsec = (y->tv_usec - x->tv_usec) / 1000000; - y->tv_usec += 1000000 * nsec; - y->tv_sec -= nsec; - } + if (x->tv_usec - y->tv_usec > 1000000) + { + int nsec = (y->tv_usec - x->tv_usec) / 1000000; + y->tv_usec += 1000000 * nsec; + y->tv_sec -= nsec; + } - /* Compute the time remaining to wait. - tv_usec is certainly positive. */ - result->tv_sec = x->tv_sec - y->tv_sec; - result->tv_usec = x->tv_usec - y->tv_usec; + /* Compute the time remaining to wait. + tv_usec is certainly positive. */ + result->tv_sec = x->tv_sec - y->tv_sec; + result->tv_usec = x->tv_usec - y->tv_usec; - /* Return 1 if result is negative. */ - return x->tv_sec < y->tv_sec; + /* Return 1 if result is negative. */ + return x->tv_sec < y->tv_sec; } void usage() { - cerr << "usage: fophdfs [adpsrh:b:t:cvuen:l:m:] " << endl; - cerr << " -d use O_DIRECT" << endl; - cerr << " -p flush before read" << endl; - cerr << " -s sync on write" << endl; - cerr << " -r reopen file for read" << endl; - cerr << " -b block size in bytes" << endl; - cerr << " -d delay before read" << endl; - cerr << " -c close file after each write" << endl; - cerr << " -v use setvbuf to disable buffering" << endl; - cerr << " -n number of blocks to write/read" << endl; - cerr << " -t number of concurrent test threads" << endl; - cerr << " -l do an HDFS large read-write file test" << endl; - cerr << " -h load an HDFS plugin (and run HDFS tests)" << endl; - cerr << " -m set max size for HDFS mem buffer, default is unlimited" << endl; + cerr << "usage: fophdfs [adpsrh:b:t:cvuen:l:m:] " << endl; + cerr << " -d use O_DIRECT" << endl; + cerr << " -p flush before read" << endl; + cerr << " -s sync on write" << endl; + cerr << " -r reopen file for read" << endl; + cerr << " -b block size in bytes" << endl; + cerr << " -d delay before read" << endl; + cerr << " -c close file after each write" << endl; + cerr << " -v use setvbuf to disable buffering" << endl; + cerr << " -n number of blocks to write/read" << endl; + cerr << " -t number of concurrent test threads" << endl; + cerr << " -l do an HDFS large read-write file test" << endl; + cerr << " -h load an HDFS plugin (and run HDFS tests)" << endl; + cerr << " -m set max size for HDFS mem buffer, default is unlimited" << endl; } struct foptest_opts { - bool usedirect; - bool preflush; - bool synconwrite; - bool reopen; - bool closeonwrite; - bool usevbuf; - int numthreads; - int numblocks; - IDBDataFile::Types filetype; - string largeFile; - string pluginFile; - bool useHdfs; - unsigned hdfsMaxMem; + bool usedirect; + bool preflush; + bool synconwrite; + bool reopen; + bool closeonwrite; + bool usevbuf; + int numthreads; + int numblocks; + IDBDataFile::Types filetype; + string largeFile; + string pluginFile; + bool useHdfs; + unsigned hdfsMaxMem; - foptest_opts() : - usedirect(false), - preflush(false), - synconwrite(false), - reopen(false), - closeonwrite(false), - usevbuf(false), - numthreads(1), - numblocks(64), - filetype(IDBDataFile::UNBUFFERED), - largeFile(""), - pluginFile(""), - useHdfs(false), - hdfsMaxMem(0) - {;} + foptest_opts() + : usedirect(false) + , preflush(false) + , synconwrite(false) + , reopen(false) + , closeonwrite(false) + , usevbuf(false) + , numthreads(1) + , numblocks(64) + , filetype(IDBDataFile::UNBUFFERED) + , largeFile("") + , pluginFile("") + , useHdfs(false) + , hdfsMaxMem(0) + { + ; + } }; class TestRunner { -public: - TestRunner(int id, const foptest_opts& opts); - ~TestRunner(); + public: + TestRunner(int id, const foptest_opts& opts); + ~TestRunner(); - bool runTest( IDBDataFile::Types filetype, unsigned open_opts ); + bool runTest(IDBDataFile::Types filetype, unsigned open_opts); - bool fillDefault(); - bool doBlock(unsigned int blocknum, unsigned char tag, unsigned int count); - bool writeBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag); - bool writeBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count); - bool readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag); - bool readBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count); + bool fillDefault(); + bool doBlock(unsigned int blocknum, unsigned char tag, unsigned int count); + bool writeBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag); + bool writeBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count); + bool readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag); + bool readBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count); - const foptest_opts& runOpts() const - { - return m_opts; - } + const foptest_opts& runOpts() const + { + return m_opts; + } - enum LogLevel - { - INFO, - ERROR - }; - void logMsg( LogLevel level, const string& msg, bool bold = false ); + enum LogLevel + { + INFO, + ERROR + }; + void logMsg(LogLevel level, const string& msg, bool bold = false); -private: - bool readTest( IDBDataFile::Types filetype ); - friend void thread_func2( TestRunner& trun ); - bool randomReadTest(); - bool openByModeStrTest(); - bool writeTest( IDBDataFile::Types filetype ); - bool truncateTest( IDBDataFile::Types filetype ); - bool renameTest( IDBDataFile::Types filetype ); - bool copyTest( IDBDataFile::Types filetype ); - bool rdwrTest( IDBDataFile::Types filetype ); - bool hdfsRdwrExhaustTest(); - bool concurrencyTest( IDBDataFile::Types filetype ); - bool tellTest( IDBDataFile::Types filetype ); - bool openWF( bool reopen = false ); - bool openRF(); - bool flushTest(IDBDataFile::Types); - bool seekTest(IDBDataFile::Types); - bool listDirTest(IDBDataFile::Types, const string&); + private: + bool readTest(IDBDataFile::Types filetype); + friend void thread_func2(TestRunner& trun); + bool randomReadTest(); + bool openByModeStrTest(); + bool writeTest(IDBDataFile::Types filetype); + bool truncateTest(IDBDataFile::Types filetype); + bool renameTest(IDBDataFile::Types filetype); + bool copyTest(IDBDataFile::Types filetype); + bool rdwrTest(IDBDataFile::Types filetype); + bool hdfsRdwrExhaustTest(); + bool concurrencyTest(IDBDataFile::Types filetype); + bool tellTest(IDBDataFile::Types filetype); + bool openWF(bool reopen = false); + bool openRF(); + bool flushTest(IDBDataFile::Types); + bool seekTest(IDBDataFile::Types); + bool listDirTest(IDBDataFile::Types, const string&); - void reset(); + void reset(); - string m_fname; - foptest_opts m_opts; - unsigned char m_defbuf[MAX_BLK_SIZE]; - IDBDataFile* m_file; - unsigned m_open_opts; - int m_id; - static boost::mutex m_guard; + string m_fname; + foptest_opts m_opts; + unsigned char m_defbuf[MAX_BLK_SIZE]; + IDBDataFile* m_file; + unsigned m_open_opts; + int m_id; + static boost::mutex m_guard; }; boost::mutex TestRunner::m_guard; -void thread_func2( TestRunner& trun ) +void thread_func2(TestRunner& trun) { - trun.randomReadTest(); + trun.randomReadTest(); } -TestRunner::TestRunner(int id, const foptest_opts& opts) : - m_opts(opts), - m_file(NULL), - m_open_opts(0), - m_id(id) +TestRunner::TestRunner(int id, const foptest_opts& opts) + : m_opts(opts), m_file(NULL), m_open_opts(0), m_id(id) { - for ( unsigned i = 0; i < BLK_SIZE; i++ ) - { - m_defbuf[i] = 0xfe; - } + for (unsigned i = 0; i < BLK_SIZE; i++) + { + m_defbuf[i] = 0xfe; + } } void TestRunner::reset() { - delete m_file; - m_file = 0; + delete m_file; + m_file = 0; } struct TestCleanup { - TestCleanup( IDBDataFile::Types type, const string& dir, const string& file ) : - m_fs( IDBFileSystem::getFs( type ) ), - m_dir(dir), - mm_file(file) {} + TestCleanup(IDBDataFile::Types type, const string& dir, const string& file) + : m_fs(IDBFileSystem::getFs(type)), m_dir(dir), mm_file(file) + { + } - ~TestCleanup() - { - assert( m_fs.exists( mm_file.c_str() ) ); - m_fs.remove(mm_file.c_str()); - assert( !m_fs.exists( mm_file.c_str() ) ); - assert( m_fs.exists( m_dir.c_str() ) ); - assert( m_fs.isDir( m_dir.c_str() )); - m_fs.remove(m_dir.c_str()); - assert( !m_fs.exists( m_dir.c_str() ) ); - } + ~TestCleanup() + { + assert(m_fs.exists(mm_file.c_str())); + m_fs.remove(mm_file.c_str()); + assert(!m_fs.exists(mm_file.c_str())); + assert(m_fs.exists(m_dir.c_str())); + assert(m_fs.isDir(m_dir.c_str())); + m_fs.remove(m_dir.c_str()); + assert(!m_fs.exists(m_dir.c_str())); + } - IDBFileSystem& m_fs; - string m_dir; - string mm_file; + IDBFileSystem& m_fs; + string m_dir; + string mm_file; }; -bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts ) +bool TestRunner::runTest(IDBDataFile::Types filetype, unsigned open_opts) { - m_open_opts = open_opts; - ostringstream infostr; - string typeString = ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" : - ( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) ); - infostr << "Running test for file type " << typeString; - logMsg( INFO, infostr.str() ); + m_open_opts = open_opts; + ostringstream infostr; + string typeString = + (filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" + : (filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS")); + infostr << "Running test for file type " << typeString; + logMsg(INFO, infostr.str()); - if (filetype != IDBDataFile::UNBUFFERED && filetype != IDBDataFile::BUFFERED) + if (filetype != IDBDataFile::UNBUFFERED && filetype != IDBDataFile::BUFFERED) + { + if (!IDBPolicy::installPlugin(m_opts.pluginFile)) { - if ( !IDBPolicy::installPlugin(m_opts.pluginFile) ) - { - cout << "ERROR: unable to install HDFS plugin!" << endl; - return -1; - } + cout << "ERROR: unable to install HDFS plugin!" << endl; + return -1; } + } - IDBFileSystem& fs = IDBFileSystem::getFs( filetype ); + IDBFileSystem& fs = IDBFileSystem::getFs(filetype); - // build the file name we are going to use - ostringstream oss; - // embed pid so that this is a new directory path - oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id << "/Calpont/data"; - // we need to make sure this directory doesn't already exist - // todo-this only works non-HDFS - string cmd = "rm -rf " + oss.str(); - system(cmd.c_str()); - string dir = oss.str(); - m_fname = dir + "/foobar"; + // build the file name we are going to use + ostringstream oss; + // embed pid so that this is a new directory path + oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id << "/Calpont/data"; + // we need to make sure this directory doesn't already exist + // todo-this only works non-HDFS + string cmd = "rm -rf " + oss.str(); + system(cmd.c_str()); + string dir = oss.str(); + m_fname = dir + "/foobar"; - // instantiate this here so that we always clean up files we created no matter - // where we exit the function from - boost::scoped_ptr cleanup( new TestCleanup( filetype, dir, m_fname ) ); + // instantiate this here so that we always clean up files we created no matter + // where we exit the function from + boost::scoped_ptr cleanup(new TestCleanup(filetype, dir, m_fname)); - // HDFS will automatically create parent directories when opening a file so these - // tests around mkdir are irrelevant. - if ( filetype != IDBDataFile::HDFS ) - { - // we expect this to fail because the parent directory should not exist - reset(); - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts); - - if ( m_file ) - { - ostringstream errstr; - errstr << "open for writing of path " << m_fname << " should not succeed!"; - logMsg( ERROR, errstr.str() ); - return false; - } - - // now create the path - if (fs.mkdir(dir.c_str())) - { - ostringstream errstr; - errstr << "open mkdir of " << dir << " failed!"; - logMsg( ERROR, errstr.str() ); - return false; - } - } - - bool returnval = true; - - if ( returnval ) - { - returnval = writeTest( filetype ); - } - - // going to check the size two different ways - once through the file and - // once through the file system. - unsigned fsize = m_file->size(); - - if ( fsize != m_opts.numblocks * BLK_SIZE ) - { - ostringstream errstr; - errstr << "bad file size from file " << fsize << " != " << m_opts.numblocks* BLK_SIZE << "!"; - logMsg( ERROR, errstr.str() ); - return false; - } - - // this deletes and closes the file - required to get accurate file size with - // buffered IO and in HDFS from the file system + // HDFS will automatically create parent directories when opening a file so these + // tests around mkdir are irrelevant. + if (filetype != IDBDataFile::HDFS) + { + // we expect this to fail because the parent directory should not exist reset(); - fsize = fs.size(m_fname.c_str()); + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts); - if ( fsize != m_opts.numblocks * BLK_SIZE ) + if (m_file) { - ostringstream errstr; - errstr << "bad file size from fs " << fsize << " != " << m_opts.numblocks* BLK_SIZE << "!"; - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "open for writing of path " << m_fname << " should not succeed!"; + logMsg(ERROR, errstr.str()); + return false; } - if ( returnval ) + // now create the path + if (fs.mkdir(dir.c_str())) { - returnval = tellTest( filetype ); + ostringstream errstr; + errstr << "open mkdir of " << dir << " failed!"; + logMsg(ERROR, errstr.str()); + return false; } + } - if ( returnval ) - { - returnval = readTest( filetype ); - } + bool returnval = true; - if ( returnval && (filetype == IDBDataFile::UNBUFFERED || filetype == IDBDataFile::HDFS )) - { - returnval = concurrencyTest( filetype ); - } + if (returnval) + { + returnval = writeTest(filetype); + } - if ( returnval ) - { - reset(); - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + // going to check the size two different ways - once through the file and + // once through the file system. + unsigned fsize = m_file->size(); - if ( !m_file ) - { - ostringstream errstr; - errstr << "Unable to open " << m_fname << " for reading"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (fsize != m_opts.numblocks * BLK_SIZE) + { + ostringstream errstr; + errstr << "bad file size from file " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!"; + logMsg(ERROR, errstr.str()); + return false; + } - if ( m_opts.numthreads > 1) - { - boost::thread_group thread_group; + // this deletes and closes the file - required to get accurate file size with + // buffered IO and in HDFS from the file system + reset(); + fsize = fs.size(m_fname.c_str()); - for ( int i = 0; i < m_opts.numthreads; ++i ) - { - thread_group.create_thread(boost::bind(thread_func2, boost::ref(*this))); - } + if (fsize != m_opts.numblocks * BLK_SIZE) + { + ostringstream errstr; + errstr << "bad file size from fs " << fsize << " != " << m_opts.numblocks * BLK_SIZE << "!"; + logMsg(ERROR, errstr.str()); + return false; + } - thread_group.join_all(); - } + if (returnval) + { + returnval = tellTest(filetype); + } - returnval = randomReadTest(); - } + if (returnval) + { + returnval = readTest(filetype); + } - if ( returnval ) - { - returnval = rdwrTest( filetype ); - } - - if ( returnval && filetype == IDBDataFile::BUFFERED ) - { - returnval = openByModeStrTest(); - } - - if ( returnval ) - { - returnval = truncateTest( filetype ); - } - - if ( returnval ) - { - returnval = renameTest( filetype ); - } - - if ( returnval ) - { - returnval = copyTest( filetype ); - } - - if ( returnval && filetype == IDBDataFile::HDFS ) - { - returnval = hdfsRdwrExhaustTest(); - } - - if ( returnval ) - { - returnval = flushTest( filetype ); - } - - if ( returnval ) - { - returnval = seekTest( filetype ); - } - - if ( returnval ) - { - returnval = listDirTest( filetype, dir ); - } - - if ( returnval ) - logMsg( INFO, typeString + " tests passed!", true ); - - reset(); - return returnval; -} - -bool TestRunner::readTest( IDBDataFile::Types filetype ) -{ - logMsg( INFO, "readTest" ); + if (returnval && (filetype == IDBDataFile::UNBUFFERED || filetype == IDBDataFile::HDFS)) + { + returnval = concurrencyTest(filetype); + } + if (returnval) + { reset(); m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); - if ( !m_file ) + if (!m_file) { - ostringstream errstr; - errstr << "Unable to open " << m_fname << " for reading"; - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "Unable to open " << m_fname << " for reading"; + logMsg(ERROR, errstr.str()); + return false; } - // check the mtime method. - time_t now = time(0); - time_t mtime = m_file->mtime(); - assert( (now - mtime) <= 3 ); - - struct timeval starttime, endtime, timediff; - gettimeofday(&starttime, 0x0); - - for ( int i = 0; i < m_opts.numblocks; ++i) + if (m_opts.numthreads > 1) { - if ( !readBlock(i, m_defbuf, i) ) - return false; + boost::thread_group thread_group; + + for (int i = 0; i < m_opts.numthreads; ++i) + { + thread_group.create_thread(boost::bind(thread_func2, boost::ref(*this))); + } + + thread_group.join_all(); } - gettimeofday(&endtime, 0x0); - timeval_subtract(&timediff, &endtime, &starttime); - float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001); + returnval = randomReadTest(); + } - ostringstream infostr; - infostr << "Read " << m_opts.numblocks* BLK_SIZE << " bytes in " << secs << " secs, "; - infostr << "Throughput = " << setprecision(3) << ((m_opts.numblocks * BLK_SIZE) / 1000000.0) / secs << "MB/sec"; - logMsg( INFO, infostr.str() ); + if (returnval) + { + returnval = rdwrTest(filetype); + } - return true; + if (returnval && filetype == IDBDataFile::BUFFERED) + { + returnval = openByModeStrTest(); + } + + if (returnval) + { + returnval = truncateTest(filetype); + } + + if (returnval) + { + returnval = renameTest(filetype); + } + + if (returnval) + { + returnval = copyTest(filetype); + } + + if (returnval && filetype == IDBDataFile::HDFS) + { + returnval = hdfsRdwrExhaustTest(); + } + + if (returnval) + { + returnval = flushTest(filetype); + } + + if (returnval) + { + returnval = seekTest(filetype); + } + + if (returnval) + { + returnval = listDirTest(filetype, dir); + } + + if (returnval) + logMsg(INFO, typeString + " tests passed!", true); + + reset(); + return returnval; +} + +bool TestRunner::readTest(IDBDataFile::Types filetype) +{ + logMsg(INFO, "readTest"); + + reset(); + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + + if (!m_file) + { + ostringstream errstr; + errstr << "Unable to open " << m_fname << " for reading"; + logMsg(ERROR, errstr.str()); + return false; + } + + // check the mtime method. + time_t now = time(0); + time_t mtime = m_file->mtime(); + assert((now - mtime) <= 3); + + struct timeval starttime, endtime, timediff; + gettimeofday(&starttime, 0x0); + + for (int i = 0; i < m_opts.numblocks; ++i) + { + if (!readBlock(i, m_defbuf, i)) + return false; + } + + gettimeofday(&endtime, 0x0); + timeval_subtract(&timediff, &endtime, &starttime); + float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001); + + ostringstream infostr; + infostr << "Read " << m_opts.numblocks * BLK_SIZE << " bytes in " << secs << " secs, "; + infostr << "Throughput = " << setprecision(3) << ((m_opts.numblocks * BLK_SIZE) / 1000000.0) / secs + << "MB/sec"; + logMsg(INFO, infostr.str()); + + return true; } bool TestRunner::randomReadTest() { - logMsg( INFO, "randomReadTest" ); + logMsg(INFO, "randomReadTest"); - struct drand48_data d48data; - srand48_r(pthread_self(), &d48data); + struct drand48_data d48data; + srand48_r(pthread_self(), &d48data); - for ( int i = 0; i < 10; ++i ) - { - long int blk_num; - lrand48_r( &d48data, &blk_num); - blk_num = blk_num % m_opts.numblocks; + for (int i = 0; i < 10; ++i) + { + long int blk_num; + lrand48_r(&d48data, &blk_num); + blk_num = blk_num % m_opts.numblocks; - unsigned char readbuf[4]; - assert( m_file->pread( readbuf, blk_num * BLK_SIZE, 4 ) == 4 ); - assert( readbuf[0] == (unsigned char) blk_num ); - } + unsigned char readbuf[4]; + assert(m_file->pread(readbuf, blk_num * BLK_SIZE, 4) == 4); + assert(readbuf[0] == (unsigned char)blk_num); + } - return true; + return true; } bool TestRunner::openByModeStrTest() { - logMsg( INFO, "openByModeStrTest" ); + logMsg(INFO, "openByModeStrTest"); - // in this test we want to check the alternate open modes available for buffered i/o - // this test is only run if we are doing buffered I/O and expects it is run after the - // write test to guarantee the file is there - reset(); - m_file = IDBDataFile::open(IDBDataFile::BUFFERED, m_fname.c_str(), "r+b", m_open_opts); + // in this test we want to check the alternate open modes available for buffered i/o + // this test is only run if we are doing buffered I/O and expects it is run after the + // write test to guarantee the file is there + reset(); + m_file = IDBDataFile::open(IDBDataFile::BUFFERED, m_fname.c_str(), "r+b", m_open_opts); - if ( !m_file ) - { - ostringstream errstr; - errstr << "Unable to open " << m_fname << " for read/write"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (!m_file) + { + ostringstream errstr; + errstr << "Unable to open " << m_fname << " for read/write"; + logMsg(ERROR, errstr.str()); + return false; + } - // keep this fairly simple - read a block then write a block + // keep this fairly simple - read a block then write a block - ssize_t readct = m_file->read(m_defbuf, BLK_SIZE); + ssize_t readct = m_file->read(m_defbuf, BLK_SIZE); - if ( (size_t) readct != BLK_SIZE ) - { - ostringstream errstr; - errstr << "Only read " << readct << " bytes, expected 4"; - logMsg( ERROR, errstr.str() ); - return false; - } + if ((size_t)readct != BLK_SIZE) + { + ostringstream errstr; + errstr << "Only read " << readct << " bytes, expected 4"; + logMsg(ERROR, errstr.str()); + return false; + } - if ( m_defbuf[0] != (unsigned char) 0 ) - { - ostringstream errstr; - errstr << "Data error - expected " << 0 << ", read " << (int) m_defbuf[0]; - logMsg( ERROR, errstr.str() ); - return false; - } + if (m_defbuf[0] != (unsigned char)0) + { + ostringstream errstr; + errstr << "Data error - expected " << 0 << ", read " << (int)m_defbuf[0]; + logMsg(ERROR, errstr.str()); + return false; + } - // we should be at block 1 - long filepos = m_file->tell(); + // we should be at block 1 + long filepos = m_file->tell(); - if ( filepos != long(BLK_SIZE) ) - { - ostringstream errstr; - errstr << "File position not at correct block, " << filepos << " != " << BLK_SIZE; - logMsg( ERROR, errstr.str() ); - return false; - } + if (filepos != long(BLK_SIZE)) + { + ostringstream errstr; + errstr << "File position not at correct block, " << filepos << " != " << BLK_SIZE; + logMsg(ERROR, errstr.str()); + return false; + } - m_defbuf[0] = 1; - ssize_t bytes_written = m_file->write(m_defbuf, BLK_SIZE); + m_defbuf[0] = 1; + ssize_t bytes_written = m_file->write(m_defbuf, BLK_SIZE); - if ( (size_t) bytes_written != BLK_SIZE ) - { - ostringstream errstr; - errstr << "Only wrote " << bytes_written << " bytes, expected 4"; - logMsg( ERROR, errstr.str() ); - return false; - } + if ((size_t)bytes_written != BLK_SIZE) + { + ostringstream errstr; + errstr << "Only wrote " << bytes_written << " bytes, expected 4"; + logMsg(ERROR, errstr.str()); + return false; + } - return true; + return true; } -bool TestRunner::writeTest( IDBDataFile::Types filetype ) +bool TestRunner::writeTest(IDBDataFile::Types filetype) { - logMsg( INFO, "writeTest" ); + logMsg(INFO, "writeTest"); - reset(); - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts); + reset(); + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts); - if ( !m_file ) - { - ostringstream errstr; - errstr << "Unable to open " << m_fname << " for writing"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (!m_file) + { + ostringstream errstr; + errstr << "Unable to open " << m_fname << " for writing"; + logMsg(ERROR, errstr.str()); + return false; + } - struct timeval starttime, endtime, timediff; + struct timeval starttime, endtime, timediff; - gettimeofday(&starttime, 0x0); + gettimeofday(&starttime, 0x0); - for ( int i = 0; i < m_opts.numblocks; ++i) - { - if ( !writeBlock(i, m_defbuf, i) ) - return false; - } + for (int i = 0; i < m_opts.numblocks; ++i) + { + if (!writeBlock(i, m_defbuf, i)) + return false; + } - gettimeofday(&endtime, 0x0); - timeval_subtract(&timediff, &endtime, &starttime); - float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001); + gettimeofday(&endtime, 0x0); + timeval_subtract(&timediff, &endtime, &starttime); + float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001); - ostringstream infostr; - infostr << "Wrote " << m_opts.numblocks* BLK_SIZE << " bytes in " << secs << " secs, "; - infostr << "Throughput = " << setprecision(3) << ((m_opts.numblocks * BLK_SIZE) / 1000000.0) / secs << "MB/sec"; - logMsg( INFO, infostr.str() ); + ostringstream infostr; + infostr << "Wrote " << m_opts.numblocks * BLK_SIZE << " bytes in " << secs << " secs, "; + infostr << "Throughput = " << setprecision(3) << ((m_opts.numblocks * BLK_SIZE) / 1000000.0) / secs + << "MB/sec"; + logMsg(INFO, infostr.str()); - return true; + return true; } -bool TestRunner::truncateTest( IDBDataFile::Types filetype ) +bool TestRunner::truncateTest(IDBDataFile::Types filetype) { - logMsg( INFO, "truncateTest" ); + logMsg(INFO, "truncateTest"); - reset(); - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "a", m_open_opts); + reset(); + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "a", m_open_opts); - if ( !m_file ) - { - ostringstream errstr; - errstr << "Unable to open " << m_fname << " for writing"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (!m_file) + { + ostringstream errstr; + errstr << "Unable to open " << m_fname << " for writing"; + logMsg(ERROR, errstr.str()); + return false; + } - // choose a random block to truncate at - struct drand48_data d48data; - srand48_r(0xdeadbeef, &d48data); + // choose a random block to truncate at + struct drand48_data d48data; + srand48_r(0xdeadbeef, &d48data); + long int blk_num; + lrand48_r(&d48data, &blk_num); + blk_num = blk_num % m_opts.numblocks; + + // always leave at least one block + if (blk_num == 0) + blk_num = 1; + + int rc = m_file->truncate(blk_num * BLK_SIZE); + + if ((filetype != IDBDataFile::HDFS) && rc) + { + logMsg(ERROR, "truncate failed!"); + return false; + } + else if ((filetype == IDBDataFile::HDFS) && !rc) + { + logMsg(ERROR, "truncate is supposed to fail for HDFS files!"); + return false; + } + else if ((filetype == IDBDataFile::HDFS)) + { + // this is the "success" case for HDFS we didn't expect to truncate so reset blk_num + blk_num = m_opts.numblocks; + } + + off64_t fsize = m_file->size(); + + if (fsize != (off64_t)(blk_num * BLK_SIZE)) + { + ostringstream errstr; + errstr << "wrong file size after truncate, " << fsize << " != " << blk_num * BLK_SIZE; + logMsg(ERROR, errstr.str()); + return false; + } + + return true; +} + +bool TestRunner::renameTest(IDBDataFile::Types type) +{ + logMsg(INFO, "renameTest"); + + // assume this test is run after the write test so that the file m_fname exists + reset(); + IDBFileSystem& fs = IDBFileSystem::getFs(type); + + // get the size before we move for compare purposes. + off64_t fsize_orig = fs.size(m_fname.c_str()); + + // choose a path in a different directory that we know already exists + // and make it specific to our thread... + ostringstream oss; + // embed pid so that this is a new directory path + oss << "/tmp/renametest-" << getpid() << "-" << m_id; + string newpath = oss.str(); + + // rename it + int rc = fs.rename(m_fname.c_str(), newpath.c_str()); + + if (rc != 0) + { + ostringstream errstr; + errstr << "rename failed, " << strerror(errno); + logMsg(ERROR, errstr.str()); + return false; + } + + // now check if oldpath exists using size method + off64_t fsize = fs.size(m_fname.c_str()); + + if (fsize != -1) + { + ostringstream errstr; + errstr << "old file still exists, size = " << fsize; + logMsg(ERROR, errstr.str()); + return false; + } + + // now check if newpath exists using size method + fsize = fs.size(newpath.c_str()); + + if (fsize != fsize_orig) + { + ostringstream errstr; + errstr << "new file and old file sizes differ, " << fsize << "!=" << fsize_orig; + logMsg(ERROR, errstr.str()); + return false; + } + + // everything looks good - put it back + // rename it + rc = fs.rename(newpath.c_str(), m_fname.c_str()); + + if (rc != 0) + { + ostringstream errstr; + errstr << "final rename failed, " << strerror(errno); + logMsg(ERROR, errstr.str()); + return false; + } + + // now a negative test case. Try to rename a file that does not exist + assert(fs.rename("a-bogus-file-name", newpath.c_str()) == -1); + + return true; +} + +bool TestRunner::copyTest(IDBDataFile::Types type) +{ + // assume this test is run after the write test so that the file m_fname exists + reset(); + IDBFileSystem& fs = IDBFileSystem::getFs(type); + + // get the size before we copy for compare purposes. + off64_t fsize_orig = fs.size(m_fname.c_str()); + + // choose a path in a different directory that we know already exists + // and make it specific to our thread... + ostringstream oss; + // embed pid so that this is a new directory path + oss << "/tmp/copytest-" << getpid() << "-" << m_id; + string newpath = oss.str(); + + // copy it + int rc = fs.copyFile(m_fname.c_str(), newpath.c_str()); + + if (rc != 0) + { + ostringstream errstr; + errstr << "copy failed, " << strerror(errno); + logMsg(ERROR, errstr.str()); + return false; + } + + // now check if newpath exists using size method + off64_t fsize = fs.size(newpath.c_str()); + + if (fsize != fsize_orig) + { + ostringstream errstr; + errstr << "new file and old file sizes differ, " << fsize << "!=" << fsize_orig; + logMsg(ERROR, errstr.str()); + return false; + } + + // everything looks good - delete the copy + rc = fs.remove(newpath.c_str()); + + if (rc != 0) + { + ostringstream errstr; + errstr << "file delete failed (after copy), " << strerror(errno); + logMsg(ERROR, errstr.str()); + return false; + } + + // now a negative test case. Try to copy a file that does not exist + assert(fs.copyFile("a-bogus-file-name", newpath.c_str()) == -1); + + return true; +} + +bool TestRunner::rdwrTest(IDBDataFile::Types filetype) +{ + logMsg(INFO, "rdwrTest"); + + reset(); + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); + + if (!m_file) + { + ostringstream errstr; + errstr << "Unable to open " << m_fname << " for reading"; + logMsg(ERROR, errstr.str()); + return false; + } + + struct drand48_data d48data; + + srand48_r(0xdeadbeef, &d48data); + + // we will write to 5 random blocks and then come back and + // verify the contents and undo them + int blocks_to_touch = min(5, m_opts.numblocks); + + vector touched; + + for (int i = 0; i < blocks_to_touch; ++i) + { long int blk_num; - lrand48_r( &d48data, &blk_num); - blk_num = blk_num % m_opts.numblocks; + // we need to make sure all the blocks we touch are unique or + // the pattern used by this test won't work + bool found = false; - // always leave at least one block - if ( blk_num == 0 ) - blk_num = 1; + while (!found) + { + lrand48_r(&d48data, &blk_num); + blk_num = blk_num % m_opts.numblocks; + vector::iterator pos = find(touched.begin(), touched.end(), blk_num); - int rc = m_file->truncate(blk_num * BLK_SIZE); - - if ( (filetype != IDBDataFile::HDFS) && rc) - { - logMsg( ERROR, "truncate failed!" ); - return false; - } - else if ( (filetype == IDBDataFile::HDFS) && !rc ) - { - logMsg( ERROR, "truncate is supposed to fail for HDFS files!" ); - return false; - } - else if ( (filetype == IDBDataFile::HDFS) ) - { - // this is the "success" case for HDFS we didn't expect to truncate so reset blk_num - blk_num = m_opts.numblocks; + if (pos == touched.end()) + found = true; } - off64_t fsize = m_file->size(); - - if ( fsize != (off64_t) (blk_num * BLK_SIZE)) + if (m_file->seek(blk_num * BLK_SIZE, SEEK_SET)) { - ostringstream errstr; - errstr << "wrong file size after truncate, " << fsize << " != " << blk_num* BLK_SIZE; - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "failed to seek block " << blk_num; + logMsg(ERROR, errstr.str()); + return false; } - return true; -} + unsigned char writeval = 0xb0; + ssize_t writect = m_file->write(&writeval, 1); -bool TestRunner::renameTest( IDBDataFile::Types type ) -{ - logMsg( INFO, "renameTest" ); - - // assume this test is run after the write test so that the file m_fname exists - reset(); - IDBFileSystem& fs = IDBFileSystem::getFs( type ); - - // get the size before we move for compare purposes. - off64_t fsize_orig = fs.size( m_fname.c_str() ); - - // choose a path in a different directory that we know already exists - // and make it specific to our thread... - ostringstream oss; - // embed pid so that this is a new directory path - oss << "/tmp/renametest-" << getpid() << "-" << m_id; - string newpath = oss.str(); - - // rename it - int rc = fs.rename( m_fname.c_str(), newpath.c_str() ); - - if ( rc != 0 ) + if (writect != 1) { - ostringstream errstr; - errstr << "rename failed, " << strerror( errno ); - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "Only wrote " << writect << " bytes, expected 1"; + logMsg(ERROR, errstr.str()); + return false; } - // now check if oldpath exists using size method - off64_t fsize = fs.size( m_fname.c_str() ); + touched.push_back(blk_num); + } - if ( fsize != -1 ) + m_file->flush(); + + for (int i = 0; i < (int)touched.size(); ++i) + { + unsigned char readbuf; + ssize_t readct = m_file->pread(&readbuf, touched[i] * BLK_SIZE, 1); + + if (readct != 1 || readbuf != (unsigned char)0xb0) { - ostringstream errstr; - errstr << "old file still exists, size = " << fsize; - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "Error reading expected value, readct=" << readct << " bytes, value" << (int)readbuf; + logMsg(ERROR, errstr.str()); + return false; } - // now check if newpath exists using size method - fsize = fs.size( newpath.c_str() ); + readbuf = touched[i]; - if ( fsize != fsize_orig ) + if (m_file->seek(-1, SEEK_CUR)) { - ostringstream errstr; - errstr << "new file and old file sizes differ, " << fsize << "!=" << fsize_orig; - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "failed to seek -1"; + logMsg(ERROR, errstr.str()); + return false; } - // everything looks good - put it back - // rename it - rc = fs.rename( newpath.c_str(), m_fname.c_str() ); + ssize_t writect = m_file->write(&readbuf, 1); - if ( rc != 0 ) + if (writect != 1) { - ostringstream errstr; - errstr << "final rename failed, " << strerror( errno ); - logMsg( ERROR, errstr.str() ); - return false; + ostringstream errstr; + errstr << "Only wrote " << writect << " bytes, expected 1"; + logMsg(ERROR, errstr.str()); + return false; } + } - // now a negative test case. Try to rename a file that does not exist - assert( fs.rename( "a-bogus-file-name", newpath.c_str() ) == -1 ); - - return true; -} - -bool TestRunner::copyTest( IDBDataFile::Types type ) -{ - // assume this test is run after the write test so that the file m_fname exists - reset(); - IDBFileSystem& fs = IDBFileSystem::getFs( type ); - - // get the size before we copy for compare purposes. - off64_t fsize_orig = fs.size( m_fname.c_str() ); - - // choose a path in a different directory that we know already exists - // and make it specific to our thread... - ostringstream oss; - // embed pid so that this is a new directory path - oss << "/tmp/copytest-" << getpid() << "-" << m_id; - string newpath = oss.str(); - - // copy it - int rc = fs.copyFile( m_fname.c_str(), newpath.c_str() ); - - if ( rc != 0 ) - { - ostringstream errstr; - errstr << "copy failed, " << strerror( errno ); - logMsg( ERROR, errstr.str() ); - return false; - } - - // now check if newpath exists using size method - off64_t fsize = fs.size( newpath.c_str() ); - - if ( fsize != fsize_orig ) - { - ostringstream errstr; - errstr << "new file and old file sizes differ, " << fsize << "!=" << fsize_orig; - logMsg( ERROR, errstr.str() ); - return false; - } - - // everything looks good - delete the copy - rc = fs.remove( newpath.c_str() ); - - if ( rc != 0 ) - { - ostringstream errstr; - errstr << "file delete failed (after copy), " << strerror( errno ); - logMsg( ERROR, errstr.str() ); - return false; - } - - // now a negative test case. Try to copy a file that does not exist - assert( fs.copyFile( "a-bogus-file-name", newpath.c_str() ) == -1 ); - - return true; -} - -bool TestRunner::rdwrTest( IDBDataFile::Types filetype ) -{ - logMsg( INFO, "rdwrTest" ); - - reset(); - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); - - if ( !m_file ) - { - ostringstream errstr; - errstr << "Unable to open " << m_fname << " for reading"; - logMsg( ERROR, errstr.str() ); - return false; - } - - struct drand48_data d48data; - - srand48_r(0xdeadbeef, &d48data); - - // we will write to 5 random blocks and then come back and - // verify the contents and undo them - int blocks_to_touch = min( 5, m_opts.numblocks); - - vector touched; - - for ( int i = 0; i < blocks_to_touch; ++i ) - { - long int blk_num; - // we need to make sure all the blocks we touch are unique or - // the pattern used by this test won't work - bool found = false; - - while (!found) - { - lrand48_r( &d48data, &blk_num); - blk_num = blk_num % m_opts.numblocks; - vector::iterator pos = find( touched.begin(), touched.end(), blk_num ); - - if ( pos == touched.end()) - found = true; - } - - if ( m_file->seek(blk_num * BLK_SIZE, SEEK_SET) ) - { - ostringstream errstr; - errstr << "failed to seek block " << blk_num; - logMsg( ERROR, errstr.str() ); - return false; - } - - unsigned char writeval = 0xb0; - ssize_t writect = m_file->write(&writeval, 1); - - if ( writect != 1 ) - { - ostringstream errstr; - errstr << "Only wrote " << writect << " bytes, expected 1"; - logMsg( ERROR, errstr.str() ); - return false; - } - - touched.push_back(blk_num); - } - - m_file->flush(); - - for ( int i = 0; i < (int) touched.size(); ++i ) - { - unsigned char readbuf; - ssize_t readct = m_file->pread(&readbuf, touched[i] * BLK_SIZE, 1); - - if ( readct != 1 || readbuf != (unsigned char) 0xb0 ) - { - ostringstream errstr; - errstr << "Error reading expected value, readct=" << readct << " bytes, value" << (int) readbuf; - logMsg( ERROR, errstr.str() ); - return false; - } - - readbuf = touched[i]; - - if ( m_file->seek(-1, SEEK_CUR) ) - { - ostringstream errstr; - errstr << "failed to seek -1"; - logMsg( ERROR, errstr.str() ); - return false; - } - - ssize_t writect = m_file->write(&readbuf, 1); - - if ( writect != 1 ) - { - ostringstream errstr; - errstr << "Only wrote " << writect << " bytes, expected 1"; - logMsg( ERROR, errstr.str() ); - return false; - } - } - - return true; + return true; } bool TestRunner::hdfsRdwrExhaustTest() { - logMsg( INFO, "hdfsRdwrExhaustTest" ); + logMsg(INFO, "hdfsRdwrExhaustTest"); - // this is going to be a self-contained test that attempts to test - // all logic inherent in HdfsRdwr + // this is going to be a self-contained test that attempts to test + // all logic inherent in HdfsRdwr - // choose a new filename that is specific to our thread - ostringstream oss; - // embed pid so that this is a new directory path - oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id; - string newpath = oss.str(); + // choose a new filename that is specific to our thread + ostringstream oss; + // embed pid so that this is a new directory path + oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id; + string newpath = oss.str(); - // open a file with arbitrarily small buffer - IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8); - assert( file ); + // open a file with arbitrarily small buffer + IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8); + assert(file); - // check various empty file conditions - assert( file->size() == 0 ); - assert( file->tell() == 0 ); - assert( file->seek(-1, SEEK_CUR) == -1); - assert( file->seek(0, SEEK_SET) == 0); - unsigned char buf[4]; - assert( file->read(buf, 4) == 0); + // check various empty file conditions + assert(file->size() == 0); + assert(file->tell() == 0); + assert(file->seek(-1, SEEK_CUR) == -1); + assert(file->seek(0, SEEK_SET) == 0); + unsigned char buf[4]; + assert(file->read(buf, 4) == 0); - // write some data - buf[0] = 0xde; - buf[1] = 0xad; - buf[2] = 0xbe; - buf[3] = 0xef; - assert( file->write(buf, 4) == 4); - assert( file->size() == 4 ); - assert( file->tell() == 4 ); - assert( file->truncate(-1) == -1 ); + // write some data + buf[0] = 0xde; + buf[1] = 0xad; + buf[2] = 0xbe; + buf[3] = 0xef; + assert(file->write(buf, 4) == 4); + assert(file->size() == 4); + assert(file->tell() == 4); + assert(file->truncate(-1) == -1); - // now make file empty again - assert( file->truncate(0) == 0 ); - assert( file->size() == 0 ); - assert( file->seek(0, SEEK_SET) == 0); - assert( file->tell() == 0 ); - assert( file->read(buf, 4) == 0); + // now make file empty again + assert(file->truncate(0) == 0); + assert(file->size() == 0); + assert(file->seek(0, SEEK_SET) == 0); + assert(file->tell() == 0); + assert(file->read(buf, 4) == 0); - // write data again, this time exactly up to allocated size - assert( file->write(buf, 4) == 4); - assert( file->write(buf, 4) == 4); - assert( file->size() == 8 ); - assert( file->tell() == 8 ); + // write data again, this time exactly up to allocated size + assert(file->write(buf, 4) == 4); + assert(file->write(buf, 4) == 4); + assert(file->size() == 8); + assert(file->tell() == 8); - // truncate back to 4 - assert( file->truncate(4) == 0 ); - assert( file->size() == 4 ); - assert( file->seek(4, SEEK_SET) == 0); - assert( file->tell() == 4 ); + // truncate back to 4 + assert(file->truncate(4) == 0); + assert(file->size() == 4); + assert(file->seek(4, SEEK_SET) == 0); + assert(file->tell() == 4); - // now trigger a buffer reallocation - assert( file->write(buf, 4) == 4); - assert( file->write(buf, 4) == 4); - assert( file->size() == 12 ); + // now trigger a buffer reallocation + assert(file->write(buf, 4) == 4); + assert(file->write(buf, 4) == 4); + assert(file->size() == 12); - // now delete and close. - delete file; + // now delete and close. + delete file; - // check the file size through the file system - IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS ); - assert( fs.size( newpath.c_str() ) == 12); + // check the file size through the file system + IDBFileSystem& fs = IDBFileSystem::getFs(IDBDataFile::HDFS); + assert(fs.size(newpath.c_str()) == 12); - // open again - the file is bigger than the default buffer so it triggers alternate - // logic in the constructor - file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8); - assert( file ); - assert( file->size() == 12); - unsigned char newbuf[4]; - assert( file->pread(newbuf, 4, 4) == 4); - assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe && newbuf[3] == 0xef); - delete file; + // open again - the file is bigger than the default buffer so it triggers alternate + // logic in the constructor + file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8); + assert(file); + assert(file->size() == 12); + unsigned char newbuf[4]; + assert(file->pread(newbuf, 4, 4) == 4); + assert(newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe && newbuf[3] == 0xef); + delete file; - fs.remove(newpath.c_str()); + fs.remove(newpath.c_str()); - if ( m_opts.largeFile.length() ) - { - file = IDBDataFile::open(IDBDataFile::HDFS, m_opts.largeFile.c_str(), "r+", 0); - assert( file ); - } + if (m_opts.largeFile.length()) + { + file = IDBDataFile::open(IDBDataFile::HDFS, m_opts.largeFile.c_str(), "r+", 0); + assert(file); + } - return true; + return true; } -bool TestRunner::concurrencyTest( IDBDataFile::Types filetype ) +bool TestRunner::concurrencyTest(IDBDataFile::Types filetype) { - logMsg( INFO, "concurrencyTest" ); + logMsg(INFO, "concurrencyTest"); - reset(); + reset(); - // ok - scenario is a reader opens the file then a reader/writer opens - // the same file and updates something, then the reader tries to read - // again. - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); - assert(m_file); + // ok - scenario is a reader opens the file then a reader/writer opens + // the same file and updates something, then the reader tries to read + // again. + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + assert(m_file); - // read a few blocks - assert( readBlock(0, m_defbuf, 0) ); - assert( readBlock(1, m_defbuf, 1) ); + // read a few blocks + assert(readBlock(0, m_defbuf, 0)); + assert(readBlock(1, m_defbuf, 1)); - // open the same file for read/write - IDBDataFile* rdwrFile = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); - assert(rdwrFile); + // open the same file for read/write + IDBDataFile* rdwrFile = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); + assert(rdwrFile); - // read a block, write a block - assert( rdwrFile->pread(m_defbuf, 0, BLK_SIZE) == (ssize_t) BLK_SIZE ); - assert( m_defbuf[0] == 0 ); - m_defbuf[0] = 95; - assert( rdwrFile->seek(0, 0) == 0 ); - assert( rdwrFile->write(m_defbuf, BLK_SIZE) ); + // read a block, write a block + assert(rdwrFile->pread(m_defbuf, 0, BLK_SIZE) == (ssize_t)BLK_SIZE); + assert(m_defbuf[0] == 0); + m_defbuf[0] = 95; + assert(rdwrFile->seek(0, 0) == 0); + assert(rdwrFile->write(m_defbuf, BLK_SIZE)); - // close file - delete rdwrFile; + // close file + delete rdwrFile; - // this 5 seconds is important in the HDFS case because it gives HDFS - // time to age off (or whatever) the blocks for the original file that - // have now been rewritten. The value was determined experimentally - - // 3 secs works fine most times but not all. If HDFS hasn't aged out - // the blocks then the read will return the old data - if ( filetype == IDBDataFile::HDFS ) - sleep(10); + // this 5 seconds is important in the HDFS case because it gives HDFS + // time to age off (or whatever) the blocks for the original file that + // have now been rewritten. The value was determined experimentally - + // 3 secs works fine most times but not all. If HDFS hasn't aged out + // the blocks then the read will return the old data + if (filetype == IDBDataFile::HDFS) + sleep(10); - // go back to the reader and make sure he got the new value, then close - assert( readBlock(0, m_defbuf, 95) ); - delete m_file; + // go back to the reader and make sure he got the new value, then close + assert(readBlock(0, m_defbuf, 95)); + delete m_file; - // now put block 0 back the way it was - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); - assert(m_file); + // now put block 0 back the way it was + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts); + assert(m_file); - assert( writeBlock(0, m_defbuf, 0) ); + assert(writeBlock(0, m_defbuf, 0)); - return true; + return true; } -bool TestRunner::tellTest( IDBDataFile::Types filetype ) +bool TestRunner::tellTest(IDBDataFile::Types filetype) { - logMsg( INFO, "tellTest" ); + logMsg(INFO, "tellTest"); - reset(); + reset(); - // scenario: reader opens file, seeks somewhere and tells where it is. - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); - assert(m_file); + // scenario: reader opens file, seeks somewhere and tells where it is. + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + assert(m_file); - // read a few blocks - assert( readBlock(0, m_defbuf, 0) ); - assert( readBlock(1, m_defbuf, 1) ); + // read a few blocks + assert(readBlock(0, m_defbuf, 0)); + assert(readBlock(1, m_defbuf, 1)); - if ( m_file->seek(BLK_SIZE, SEEK_SET) ) - { - ostringstream errstr; - errstr << "tellTest: failed to seek block"; - logMsg( ERROR, errstr.str() ); - return false; - } - - off64_t filepos = m_file->tell(); - - if ( filepos != off64_t(BLK_SIZE) ) - { - ostringstream errstr; - errstr << "tellTest: File position not at correct block, " << filepos << " != " << BLK_SIZE; - logMsg( ERROR, errstr.str() ); - return false; - } - - return true; -} - -bool TestRunner::flushTest( IDBDataFile::Types filetype ) -{ - logMsg( INFO, "flushTest" ); - - reset(); - - string scratch = "/tmp/rdwr_scratch" + m_fname; // scratch file name if exists - boost::filesystem::remove(scratch); - IDBPolicy::remove(m_fname.c_str()); - - // scenario: writer opens the file, writes 8 bytes, flush; - // reader opens the file, reads 8 bytes, verifys the data, then closes file; - // writer writes 8M bytes (force to buffered file) if -m option used correctly; - // reader opens the file, verifys the file size and content, then closes the file; - // writer closes the file. + if (m_file->seek(BLK_SIZE, SEEK_SET)) + { ostringstream errstr; - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts, /*default:4*/ 1); + errstr << "tellTest: failed to seek block"; + logMsg(ERROR, errstr.str()); + return false; + } - if (!m_file) - { - errstr << "flushTest: open " << m_fname.c_str() << " for write failed"; - logMsg( ERROR, errstr.str() ); - return false; - } + off64_t filepos = m_file->tell(); - // write 8 "1" through mem buff - const char w1[] = "11111111"; - ssize_t bytes = 0; + if (filepos != off64_t(BLK_SIZE)) + { + ostringstream errstr; + errstr << "tellTest: File position not at correct block, " << filepos << " != " << BLK_SIZE; + logMsg(ERROR, errstr.str()); + return false; + } - if ((bytes = m_file->write(w1, 8)) != 8) - { - errstr << "flushTest: write count = 8, return = " << bytes; - logMsg( ERROR, errstr.str() ); - return false; - } - - // for HDFS, force writing out to disk. - m_file->flush(); - - if (!IDBPolicy::exists(m_fname.c_str())) - { - errstr << "flushTest: " << m_fname.c_str() << " does not exist"; - logMsg( ERROR, errstr.str() ); - return false; - } - - if (filetype == IDBDataFile::HDFS && boost::filesystem::exists(scratch)) - { - errstr << "flushTest: " << scratch << " exists after 1st write"; - logMsg( ERROR, errstr.str() ); - return false; - } - - // read from file in "r" mode - IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1); - - if (!file) - { - errstr << "flushTest: 1st open " << m_fname.c_str() << " to read failed"; - logMsg( ERROR, errstr.str() ); - return false; - } - - char r1[9] = {0}; - ssize_t n1 = file->pread(r1, 0, 8); - - if (n1 != 8 || strncmp(r1, w1, 8) != 0) - { - errstr << "flushTest: read " << n1 << " != 8 OR " << r1 << "!= 11111111"; - logMsg( ERROR, errstr.str() ); - return false; - } - - delete file; - file = NULL; - - // write 8M "2", switched to file buffer if max size for mem buffer is small. - //char w2[] = {[0 ... 8*1024*1024] = '2'}; - ssize_t m9 = 9 * 1024 * 1024; // must be large than EXTENTSIZE = 8390656 to swith to file buffer - boost::scoped_array w2(new char[m9]); - memset(w2.get(), '2', m9); - m_file->write(w2.get(), m9); - m_file->flush(); - - // check file size - if (IDBPolicy::size(m_fname.c_str()) != 8 + m9) - { - errstr << "flushTest: size of " << m_fname.c_str() << " is " - << IDBPolicy::size(m_fname.c_str()) << ", expecting " << (8 + m9); - logMsg( ERROR, errstr.str() ); - return false; - } - - if (filetype == IDBDataFile::HDFS && - !boost::filesystem::exists(scratch) && - m_opts.hdfsMaxMem < m9) - { - errstr << "flushTest: " << scratch << " does not exist after 2nd write"; - logMsg( ERROR, errstr.str() ); - return false; - } - - // 2nd read - file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1); - - if (!file) - { - errstr << "flushTest: 2nd open " << m_fname.c_str() << " to read failed"; - logMsg( ERROR, errstr.str() ); - return false; - } - - //char r2[9*1024*1024 + 8 + 1] = {0}; - boost::scoped_array r2(new char[9 * 1024 * 1024 + 8 + 1]); - memset(r2.get(), 0, m9 + 9); - ssize_t n2 = file->pread(r2.get(), 0, m9 + 8); - - if (n2 != (m9 + 8) || strncmp(r2.get(), w1, 8) != 0 || memcmp(r2.get() + 8, w2.get(), m9) != 0) - { - errstr << "flushTest: 2nd read " << m_fname.c_str() << " failed" << endl - << " return value: " << n2 << " bytes -- " << r2; // need hex dump? - logMsg( ERROR, errstr.str() ); - return false; - } - - delete file; - file = NULL; - - delete m_file; - m_file = NULL; - - return true; + return true; } -bool TestRunner::seekTest( IDBDataFile::Types filetype ) +bool TestRunner::flushTest(IDBDataFile::Types filetype) { - logMsg( INFO, "seekTest" ); + logMsg(INFO, "flushTest"); - reset(); + reset(); - // scenario: writer opens the file with w+, writes 8 bytes, seek to 4 from 0, write 4 bytes - // reader opens the file with r, verify size and contents, - // writer seeks 4 bytes beyond EOF, write 4 bytes, and close the file, - // reader rewinds, verify size and contents, and close the file. - ostringstream errstr; - m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts); + string scratch = "/tmp/rdwr_scratch" + m_fname; // scratch file name if exists + boost::filesystem::remove(scratch); + IDBPolicy::remove(m_fname.c_str()); - if (!m_file) - { - errstr << "seekTest: open " << m_fname.c_str() << " for write failed"; - logMsg( ERROR, errstr.str() ); - return false; - } + // scenario: writer opens the file, writes 8 bytes, flush; + // reader opens the file, reads 8 bytes, verifys the data, then closes file; + // writer writes 8M bytes (force to buffered file) if -m option used correctly; + // reader opens the file, verifys the file size and content, then closes the file; + // writer closes the file. + ostringstream errstr; + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts, /*default:4*/ 1); - // write 8 "1" through mem buff - const char w1[] = "11111111"; - ssize_t bytes = 0; + if (!m_file) + { + errstr << "flushTest: open " << m_fname.c_str() << " for write failed"; + logMsg(ERROR, errstr.str()); + return false; + } - if ((bytes = m_file->write(w1, 8)) != 8) - { - errstr << "seekTest: write1 count = 8, return = " << bytes; - logMsg( ERROR, errstr.str() ); - return false; - } + // write 8 "1" through mem buff + const char w1[] = "11111111"; + ssize_t bytes = 0; - if (m_file->seek(4, SEEK_SET) != 0) - { - errstr << "seekTest: seek(4, SEEK_SET) failed"; - logMsg( ERROR, errstr.str() ); - return false; - } + if ((bytes = m_file->write(w1, 8)) != 8) + { + errstr << "flushTest: write count = 8, return = " << bytes; + logMsg(ERROR, errstr.str()); + return false; + } - const char w2[] = "2222"; + // for HDFS, force writing out to disk. + m_file->flush(); - if ((bytes = m_file->write(w2, 4)) != 4) - { - errstr << "seekTest: write2 count = 4, return = " << bytes; - logMsg( ERROR, errstr.str() ); - return false; - } + if (!IDBPolicy::exists(m_fname.c_str())) + { + errstr << "flushTest: " << m_fname.c_str() << " does not exist"; + logMsg(ERROR, errstr.str()); + return false; + } - // for HDFS, force writing out to disk. - m_file->flush(); + if (filetype == IDBDataFile::HDFS && boost::filesystem::exists(scratch)) + { + errstr << "flushTest: " << scratch << " exists after 1st write"; + logMsg(ERROR, errstr.str()); + return false; + } - // read from file in "r" mode - IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + // read from file in "r" mode + IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1); - if (!file) - { - errstr << "seekTest: 1st open " << m_fname.c_str() << " to read failed"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (!file) + { + errstr << "flushTest: 1st open " << m_fname.c_str() << " to read failed"; + logMsg(ERROR, errstr.str()); + return false; + } - char r1[9] = {0}; - ssize_t n1 = file->pread(r1, 0, 8); + char r1[9] = {0}; + ssize_t n1 = file->pread(r1, 0, 8); - if (IDBPolicy::size(m_fname.c_str()) != 8) - { - errstr << "seekTest: size of " << m_fname.c_str() << " is " - << IDBPolicy::size(m_fname.c_str()) << ", expecting 8"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (n1 != 8 || strncmp(r1, w1, 8) != 0) + { + errstr << "flushTest: read " << n1 << " != 8 OR " << r1 << "!= 11111111"; + logMsg(ERROR, errstr.str()); + return false; + } - if (n1 != 8 || strncmp(r1, w1, 4) != 0 || strncmp(r1 + 4, w2, 4) != 0) - { - errstr << "seekTest: read " << n1 << " != 8 OR " << r1 << "!= 11112222"; - logMsg( ERROR, errstr.str() ); - return false; - } + delete file; + file = NULL; - // now seek beyond the eof, and write 4 bytes. - const char w3[] = "3333"; + // write 8M "2", switched to file buffer if max size for mem buffer is small. + // char w2[] = {[0 ... 8*1024*1024] = '2'}; + ssize_t m9 = 9 * 1024 * 1024; // must be large than EXTENTSIZE = 8390656 to swith to file buffer + boost::scoped_array w2(new char[m9]); + memset(w2.get(), '2', m9); + m_file->write(w2.get(), m9); + m_file->flush(); - if (m_file->seek(4, SEEK_END) != 0) - { - errstr << "seekTest: seek(4, SEEK_END) failed"; - logMsg( ERROR, errstr.str() ); - return false; - } + // check file size + if (IDBPolicy::size(m_fname.c_str()) != 8 + m9) + { + errstr << "flushTest: size of " << m_fname.c_str() << " is " << IDBPolicy::size(m_fname.c_str()) + << ", expecting " << (8 + m9); + logMsg(ERROR, errstr.str()); + return false; + } - if ((bytes = m_file->write(w3, 4)) != 4) - { - errstr << "seekTest: write3 count = 4, return = " << bytes; - logMsg( ERROR, errstr.str() ); - return false; - } + if (filetype == IDBDataFile::HDFS && !boost::filesystem::exists(scratch) && m_opts.hdfsMaxMem < m9) + { + errstr << "flushTest: " << scratch << " does not exist after 2nd write"; + logMsg(ERROR, errstr.str()); + return false; + } - m_file->flush(); + // 2nd read + file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts, 1); - delete m_file; - m_file = NULL; + if (!file) + { + errstr << "flushTest: 2nd open " << m_fname.c_str() << " to read failed"; + logMsg(ERROR, errstr.str()); + return false; + } - // check file size - if (IDBPolicy::size(m_fname.c_str()) != 16) - { - errstr << "seekTest: size of " << m_fname.c_str() << " is " - << IDBPolicy::size(m_fname.c_str()) << ", expecting 16"; - logMsg( ERROR, errstr.str() ); - return false; - } + // char r2[9*1024*1024 + 8 + 1] = {0}; + boost::scoped_array r2(new char[9 * 1024 * 1024 + 8 + 1]); + memset(r2.get(), 0, m9 + 9); + ssize_t n2 = file->pread(r2.get(), 0, m9 + 8); - // 2nd read - file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + if (n2 != (m9 + 8) || strncmp(r2.get(), w1, 8) != 0 || memcmp(r2.get() + 8, w2.get(), m9) != 0) + { + errstr << "flushTest: 2nd read " << m_fname.c_str() << " failed" << endl + << " return value: " << n2 << " bytes -- " << r2; // need hex dump? + logMsg(ERROR, errstr.str()); + return false; + } - if (!file) - { - errstr << "seekTest: 2nd open " << m_fname.c_str() << " to read failed"; - logMsg( ERROR, errstr.str() ); - return false; - } + delete file; + file = NULL; - char r2[17] = {0}; - const char pd[4] = {0}; // padding - ssize_t n2 = file->pread(r2, 0, 16); + delete m_file; + m_file = NULL; - if (n2 != 16 || - strncmp(r2, w1, 4) != 0 || memcmp(r2 + 4, w2, 4) != 0 || - strncmp(r2 + 8, pd, 4) != 0 || memcmp(r2 + 12, w3, 4) != 0) - { - errstr << "seekTest: 2nd read " << m_fname.c_str() << " failed" << endl - << " return value: " << n2 << " bytes -- " << r2; // need hex dump? - logMsg( ERROR, errstr.str() ); - return false; - } - - delete file; - file = NULL; - - return true; + return true; } -bool TestRunner::listDirTest( IDBDataFile::Types filetype, const string& dir ) +bool TestRunner::seekTest(IDBDataFile::Types filetype) { - logMsg( INFO, "listDirTest" ); + logMsg(INFO, "seekTest"); - IDBFileSystem& fs = IDBFileSystem::getFs( filetype ); - ostringstream errstr; - string fname2 = m_fname + "2"; - string fname3 = m_fname + "3"; + reset(); - IDBDataFile* file2 = IDBDataFile::open(filetype, fname2.c_str(), "w", m_open_opts); + // scenario: writer opens the file with w+, writes 8 bytes, seek to 4 from 0, write 4 bytes + // reader opens the file with r, verify size and contents, + // writer seeks 4 bytes beyond EOF, write 4 bytes, and close the file, + // reader rewinds, verify size and contents, and close the file. + ostringstream errstr; + m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w+", m_open_opts); - if (file2) - delete file2; + if (!m_file) + { + errstr << "seekTest: open " << m_fname.c_str() << " for write failed"; + logMsg(ERROR, errstr.str()); + return false; + } - IDBDataFile* file3 = IDBDataFile::open(filetype, fname3.c_str(), "w", m_open_opts); + // write 8 "1" through mem buff + const char w1[] = "11111111"; + ssize_t bytes = 0; - if (file3) - delete file3; + if ((bytes = m_file->write(w1, 8)) != 8) + { + errstr << "seekTest: write1 count = 8, return = " << bytes; + logMsg(ERROR, errstr.str()); + return false; + } - list dircontents; + if (m_file->seek(4, SEEK_SET) != 0) + { + errstr << "seekTest: seek(4, SEEK_SET) failed"; + logMsg(ERROR, errstr.str()); + return false; + } - if (fs.listDirectory( dir.c_str(), dircontents ) != 0) - { - errstr << "Error calling listDirectory"; - logMsg( ERROR, errstr.str() ); - return false; - } + const char w2[] = "2222"; - ostringstream ldstr; - ldstr << "Listed directory " << dir << ":"; - list::iterator iend = dircontents.end(); - bool foobarFound = false; - bool foobar2Found = false; - bool foobar3Found = false; + if ((bytes = m_file->write(w2, 4)) != 4) + { + errstr << "seekTest: write2 count = 4, return = " << bytes; + logMsg(ERROR, errstr.str()); + return false; + } - for ( list::iterator i = dircontents.begin(); i != iend; ++i ) - { - ldstr << (*i) << ","; + // for HDFS, force writing out to disk. + m_file->flush(); - if ((*i) == "foobar") - foobarFound = true; - else if ((*i) == "foobar2") - foobar2Found = true; - else if ((*i) == "foobar3") - foobar3Found = true; - } + // read from file in "r" mode + IDBDataFile* file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); - logMsg( INFO, ldstr.str() ); + if (!file) + { + errstr << "seekTest: 1st open " << m_fname.c_str() << " to read failed"; + logMsg(ERROR, errstr.str()); + return false; + } - if (dircontents.size() != 3) - { - errstr << "listDirectory not returning 3 file names"; - logMsg( ERROR, errstr.str() ); - return false; - } + char r1[9] = {0}; + ssize_t n1 = file->pread(r1, 0, 8); - if ((!foobarFound || !foobar2Found || !foobar3Found)) - { - errstr << "listDirectory returning incorrect file names"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (IDBPolicy::size(m_fname.c_str()) != 8) + { + errstr << "seekTest: size of " << m_fname.c_str() << " is " << IDBPolicy::size(m_fname.c_str()) + << ", expecting 8"; + logMsg(ERROR, errstr.str()); + return false; + } - // now check a bogus path and make sure it returns -1 - if (fs.listDirectory( "/this-is-a-bogus-directory", dircontents ) != -1) - { - errstr << "listDirectory not failing a call for a bogus directory"; - logMsg( ERROR, errstr.str() ); - return false; - } + if (n1 != 8 || strncmp(r1, w1, 4) != 0 || strncmp(r1 + 4, w2, 4) != 0) + { + errstr << "seekTest: read " << n1 << " != 8 OR " << r1 << "!= 11112222"; + logMsg(ERROR, errstr.str()); + return false; + } - assert( fs.remove( "/this-is-a-bogus-directory" ) == 0 ); - assert( !fs.isDir( "/this-is-a-bogus-directory" )); - assert( !fs.isDir( m_fname.c_str() )); + // now seek beyond the eof, and write 4 bytes. + const char w3[] = "3333"; - return true; + if (m_file->seek(4, SEEK_END) != 0) + { + errstr << "seekTest: seek(4, SEEK_END) failed"; + logMsg(ERROR, errstr.str()); + return false; + } + + if ((bytes = m_file->write(w3, 4)) != 4) + { + errstr << "seekTest: write3 count = 4, return = " << bytes; + logMsg(ERROR, errstr.str()); + return false; + } + + m_file->flush(); + + delete m_file; + m_file = NULL; + + // check file size + if (IDBPolicy::size(m_fname.c_str()) != 16) + { + errstr << "seekTest: size of " << m_fname.c_str() << " is " << IDBPolicy::size(m_fname.c_str()) + << ", expecting 16"; + logMsg(ERROR, errstr.str()); + return false; + } + + // 2nd read + file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts); + + if (!file) + { + errstr << "seekTest: 2nd open " << m_fname.c_str() << " to read failed"; + logMsg(ERROR, errstr.str()); + return false; + } + + char r2[17] = {0}; + const char pd[4] = {0}; // padding + ssize_t n2 = file->pread(r2, 0, 16); + + if (n2 != 16 || strncmp(r2, w1, 4) != 0 || memcmp(r2 + 4, w2, 4) != 0 || strncmp(r2 + 8, pd, 4) != 0 || + memcmp(r2 + 12, w3, 4) != 0) + { + errstr << "seekTest: 2nd read " << m_fname.c_str() << " failed" << endl + << " return value: " << n2 << " bytes -- " << r2; // need hex dump? + logMsg(ERROR, errstr.str()); + return false; + } + + delete file; + file = NULL; + + return true; +} + +bool TestRunner::listDirTest(IDBDataFile::Types filetype, const string& dir) +{ + logMsg(INFO, "listDirTest"); + + IDBFileSystem& fs = IDBFileSystem::getFs(filetype); + ostringstream errstr; + string fname2 = m_fname + "2"; + string fname3 = m_fname + "3"; + + IDBDataFile* file2 = IDBDataFile::open(filetype, fname2.c_str(), "w", m_open_opts); + + if (file2) + delete file2; + + IDBDataFile* file3 = IDBDataFile::open(filetype, fname3.c_str(), "w", m_open_opts); + + if (file3) + delete file3; + + list dircontents; + + if (fs.listDirectory(dir.c_str(), dircontents) != 0) + { + errstr << "Error calling listDirectory"; + logMsg(ERROR, errstr.str()); + return false; + } + + ostringstream ldstr; + ldstr << "Listed directory " << dir << ":"; + list::iterator iend = dircontents.end(); + bool foobarFound = false; + bool foobar2Found = false; + bool foobar3Found = false; + + for (list::iterator i = dircontents.begin(); i != iend; ++i) + { + ldstr << (*i) << ","; + + if ((*i) == "foobar") + foobarFound = true; + else if ((*i) == "foobar2") + foobar2Found = true; + else if ((*i) == "foobar3") + foobar3Found = true; + } + + logMsg(INFO, ldstr.str()); + + if (dircontents.size() != 3) + { + errstr << "listDirectory not returning 3 file names"; + logMsg(ERROR, errstr.str()); + return false; + } + + if ((!foobarFound || !foobar2Found || !foobar3Found)) + { + errstr << "listDirectory returning incorrect file names"; + logMsg(ERROR, errstr.str()); + return false; + } + + // now check a bogus path and make sure it returns -1 + if (fs.listDirectory("/this-is-a-bogus-directory", dircontents) != -1) + { + errstr << "listDirectory not failing a call for a bogus directory"; + logMsg(ERROR, errstr.str()); + return false; + } + + assert(fs.remove("/this-is-a-bogus-directory") == 0); + assert(!fs.isDir("/this-is-a-bogus-directory")); + assert(!fs.isDir(m_fname.c_str())); + + return true; } TestRunner::~TestRunner() @@ -1391,228 +1390,229 @@ TestRunner::~TestRunner() bool TestRunner::fillDefault() { - return true; + return true; } bool TestRunner::writeBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag) { - buf[0] = tag; - ssize_t rc = m_file->write(buf, BLK_SIZE); + buf[0] = tag; + ssize_t rc = m_file->write(buf, BLK_SIZE); - if ((size_t) rc != BLK_SIZE) - { - ostringstream errstr; - errstr << "writeBlock failed for block " << blocknum << ", wrote " << rc << " bytes, expecting " << BLK_SIZE; - logMsg( ERROR, errstr.str() ); - return false; - } + if ((size_t)rc != BLK_SIZE) + { + ostringstream errstr; + errstr << "writeBlock failed for block " << blocknum << ", wrote " << rc << " bytes, expecting " + << BLK_SIZE; + logMsg(ERROR, errstr.str()); + return false; + } - return true; + return true; } bool TestRunner::writeBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count) { - for (unsigned i = 0; i < count; ++i) - { - writeBlock(blocknum + i, buf, tag + i); - } + for (unsigned i = 0; i < count; ++i) + { + writeBlock(blocknum + i, buf, tag + i); + } - // should actually aggregate return values but using messages for now - return true; + // should actually aggregate return values but using messages for now + return true; } bool TestRunner::readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag) { - ssize_t rc = m_file->pread(buf, blocknum * BLK_SIZE, BLK_SIZE); - //cout << "DEBUG: read " << rc << " bytes at offset " << blocknum * BLK_SIZE << endl; + ssize_t rc = m_file->pread(buf, blocknum * BLK_SIZE, BLK_SIZE); + // cout << "DEBUG: read " << rc << " bytes at offset " << blocknum * BLK_SIZE << endl; - if ((size_t) rc != BLK_SIZE) - { - ostringstream errstr; - errstr << "readBlock failed for block " << blocknum << ", read " << rc << " bytes, expecting " << BLK_SIZE; - logMsg( ERROR, errstr.str() ); - return false; - } - else if (tag != buf[0]) - { - ostringstream errstr; - errstr << "read tag 0x" << setw(2) << hex << setfill('0') << (int) buf[0] << " at block " << dec - << blocknum << ", expected 0x" << (int) tag; - logMsg( ERROR, errstr.str() ); - return false; - } + if ((size_t)rc != BLK_SIZE) + { + ostringstream errstr; + errstr << "readBlock failed for block " << blocknum << ", read " << rc << " bytes, expecting " + << BLK_SIZE; + logMsg(ERROR, errstr.str()); + return false; + } + else if (tag != buf[0]) + { + ostringstream errstr; + errstr << "read tag 0x" << setw(2) << hex << setfill('0') << (int)buf[0] << " at block " << dec + << blocknum << ", expected 0x" << (int)tag; + logMsg(ERROR, errstr.str()); + return false; + } - return true; + return true; } bool TestRunner::readBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count) { - for (unsigned i = 0; i < count; ++i) - { - readBlock(blocknum + i, buf, (unsigned char)(tag + i)); - } + for (unsigned i = 0; i < count; ++i) + { + readBlock(blocknum + i, buf, (unsigned char)(tag + i)); + } - // should actually aggregate return values but using messages for now - return true; + // should actually aggregate return values but using messages for now + return true; } bool TestRunner::doBlock(unsigned int blocknum, unsigned char tag, unsigned int count) { - writeBlocks(blocknum, m_defbuf, tag, count); - m_file->flush(); + writeBlocks(blocknum, m_defbuf, tag, count); + m_file->flush(); - if ( m_opts.closeonwrite ) - { - delete m_file; - // will have to cache type somewhere later - m_file = IDBDataFile::open( IDBDataFile::HDFS, m_fname.c_str(), "a", m_open_opts ); + if (m_opts.closeonwrite) + { + delete m_file; + // will have to cache type somewhere later + m_file = IDBDataFile::open(IDBDataFile::HDFS, m_fname.c_str(), "a", m_open_opts); - if ( !m_file ) - return false; - } + if (!m_file) + return false; + } - if ( m_opts.reopen ) - { - delete m_file; - // will have to cache type somewhere later - m_file = IDBDataFile::open( IDBDataFile::HDFS, m_fname.c_str(), "r", m_open_opts ); + if (m_opts.reopen) + { + delete m_file; + // will have to cache type somewhere later + m_file = IDBDataFile::open(IDBDataFile::HDFS, m_fname.c_str(), "r", m_open_opts); - if ( !m_file ) - return false; - } + if (!m_file) + return false; + } - unsigned char buf[BLK_SIZE]; - return readBlocks(blocknum, buf, tag, count); + unsigned char buf[BLK_SIZE]; + return readBlocks(blocknum, buf, tag, count); } -void TestRunner::logMsg( LogLevel level, const string& msg, bool bold) +void TestRunner::logMsg(LogLevel level, const string& msg, bool bold) { - boost::mutex::scoped_lock lock( m_guard ); + boost::mutex::scoped_lock lock(m_guard); - if (bold) cout << "\033[0;1m"; + if (bold) + cout << "\033[0;1m"; - cout << "Test-" << m_id << ":" << ((level == INFO) ? "INFO:" : "ERROR:") << msg << endl; + cout << "Test-" << m_id << ":" << ((level == INFO) ? "INFO:" : "ERROR:") << msg << endl; - if (bold) cout << "\033[0;39m"; + if (bold) + cout << "\033[0;39m"; } -void thread_func( TestRunner& trun ) +void thread_func(TestRunner& trun) { - bool ret = true; - // todo-add some capability to use USE_ODIRECT or not - ret = ret && trun.runTest( IDBDataFile::UNBUFFERED, 0 ); - // todo-add some capability to use USE_VBUF or not - ret = ret && trun.runTest( IDBDataFile::BUFFERED, 0 ); + bool ret = true; + // todo-add some capability to use USE_ODIRECT or not + ret = ret && trun.runTest(IDBDataFile::UNBUFFERED, 0); + // todo-add some capability to use USE_VBUF or not + ret = ret && trun.runTest(IDBDataFile::BUFFERED, 0); - if ( trun.runOpts().useHdfs ) - ret = ret && trun.runTest( IDBDataFile::HDFS, 0 ); + if (trun.runOpts().useHdfs) + ret = ret && trun.runTest(IDBDataFile::HDFS, 0); - - trun.logMsg( TestRunner::INFO, string(ret ? "A" : "NOT a") + "ll tests passed!\n", true ); + trun.logMsg(TestRunner::INFO, string(ret ? "A" : "NOT a") + "ll tests passed!\n", true); } int main(int argc, char** argv) { - foptest_opts opts; + foptest_opts opts; - int c; + int c; - while ((c = getopt (argc, argv, "adpsrh:b:t:cvuen:l:m:")) != -1) - switch (c) + while ((c = getopt(argc, argv, "adpsrh:b:t:cvuen:l:m:")) != -1) + switch (c) + { + case 'b': + BLK_SIZE = atoi(optarg); + + if (BLK_SIZE > MAX_BLK_SIZE) { - case 'b': - BLK_SIZE = atoi(optarg); - - if ( BLK_SIZE > MAX_BLK_SIZE ) - { - BLK_SIZE = MAX_BLK_SIZE; - cout << "WARNING: block size exceeds max, using " << BLK_SIZE << endl; - } - else - { - cout << "INFO: using BLK_SIZE " << BLK_SIZE << endl; - } - - break; - - case 'c': - opts.closeonwrite = true; - cout << "INFO: will close on write" << endl; - break; - - case 'd': - opts.usedirect = 1; - cout << "INFO: will open with O_DIRECT" << endl; - break; - - case 'p': - opts.preflush = 1; - cout << "INFO: will use read pre-flush" << endl; - break; - - case 's': - opts.synconwrite = 1; - cout << "INFO: will fsync after write" << endl; - break; - - case 'r': - opts.reopen = 1; - cout << "INFO: will reopen before each read" << endl; - break; - - case 't': - opts.numthreads = atoi(optarg); - cout << "INFO: will start " << opts.numthreads << " test threads" << endl; - break; - - case 'v': - opts.usevbuf = true; - cout << "INFO: will use setvbuf to disable buffering" << endl; - break; - - case 'n': - opts.numblocks = atoi(optarg); - cout << "INFO: will operate on " << opts.numblocks << " blocks" << endl; - break; - - case 'l': - opts.largeFile = optarg; - cout << "INFO: will run a large HDFS RDWR test on " << opts.largeFile << endl; - break; - - case 'h': - opts.pluginFile = optarg; - cout << "INFO: will load HDFS plugin " << optarg << endl; - break; - - case 'm': - opts.hdfsMaxMem = atoi(optarg); - - if (opts.hdfsMaxMem) - cout << "INFO: will set hdfsRdwrBufferMaxSize to " << opts.hdfsMaxMem << endl; - else - cout << "INFO: will set hdfsRdwrBufferMaxSize to unlimited" << endl; - - break; - - default: - usage(); - return 1; + BLK_SIZE = MAX_BLK_SIZE; + cout << "WARNING: block size exceeds max, using " << BLK_SIZE << endl; + } + else + { + cout << "INFO: using BLK_SIZE " << BLK_SIZE << endl; } - // init the library with logging enabled - IDBPolicy::init( true, true, "/tmp/rdwr_scratch", opts.hdfsMaxMem); + break; - if ( opts.pluginFile.length() ) - opts.useHdfs = true; + case 'c': + opts.closeonwrite = true; + cout << "INFO: will close on write" << endl; + break; - //vector - boost::thread_group thread_group; + case 'd': + opts.usedirect = 1; + cout << "INFO: will open with O_DIRECT" << endl; + break; - for ( int i = 0; i < opts.numthreads; ++i ) - { - thread_group.create_thread(boost::bind(thread_func, boost::ref(* (new TestRunner(i, opts))))); + case 'p': + opts.preflush = 1; + cout << "INFO: will use read pre-flush" << endl; + break; + + case 's': + opts.synconwrite = 1; + cout << "INFO: will fsync after write" << endl; + break; + + case 'r': + opts.reopen = 1; + cout << "INFO: will reopen before each read" << endl; + break; + + case 't': + opts.numthreads = atoi(optarg); + cout << "INFO: will start " << opts.numthreads << " test threads" << endl; + break; + + case 'v': + opts.usevbuf = true; + cout << "INFO: will use setvbuf to disable buffering" << endl; + break; + + case 'n': + opts.numblocks = atoi(optarg); + cout << "INFO: will operate on " << opts.numblocks << " blocks" << endl; + break; + + case 'l': + opts.largeFile = optarg; + cout << "INFO: will run a large HDFS RDWR test on " << opts.largeFile << endl; + break; + + case 'h': + opts.pluginFile = optarg; + cout << "INFO: will load HDFS plugin " << optarg << endl; + break; + + case 'm': + opts.hdfsMaxMem = atoi(optarg); + + if (opts.hdfsMaxMem) + cout << "INFO: will set hdfsRdwrBufferMaxSize to " << opts.hdfsMaxMem << endl; + else + cout << "INFO: will set hdfsRdwrBufferMaxSize to unlimited" << endl; + + break; + + default: usage(); return 1; } - thread_group.join_all(); + // init the library with logging enabled + IDBPolicy::init(true, true, "/tmp/rdwr_scratch", opts.hdfsMaxMem); + + if (opts.pluginFile.length()) + opts.useHdfs = true; + + // vector + boost::thread_group thread_group; + + for (int i = 0; i < opts.numthreads; ++i) + { + thread_group.create_thread(boost::bind(thread_func, boost::ref(*(new TestRunner(i, opts))))); + } + + thread_group.join_all(); } diff --git a/utils/idbdatafile/tdriver1.cpp b/utils/idbdatafile/tdriver1.cpp index 32a1349bd..2c82adb8e 100644 --- a/utils/idbdatafile/tdriver1.cpp +++ b/utils/idbdatafile/tdriver1.cpp @@ -7,7 +7,7 @@ using namespace std; int main(int argc, char** argv) { - void* dl = 0; + void* dl = 0; #if 0 dl = dlopen("./libjvm.so", RTLD_NOW); @@ -28,11 +28,10 @@ int main(int argc, char** argv) } #endif - bool ret = false; - ret = idbdatafile::IDBPolicy::installPlugin("hdfs-20.so"); + bool ret = false; + ret = idbdatafile::IDBPolicy::installPlugin("hdfs-20.so"); - cout << "ret = " << boolalpha << ret << endl; + cout << "ret = " << boolalpha << ret << endl; - return 0; + return 0; } - diff --git a/utils/idbdatafile/utility.h b/utils/idbdatafile/utility.h index 15f66c2a9..274352ae3 100644 --- a/utils/idbdatafile/utility.h +++ b/utils/idbdatafile/utility.h @@ -25,96 +25,93 @@ namespace idbdatafile { - /** * Take an fopen() style mode string and return the corresponding flags * ala open() */ -inline -int modeStrToFlags(const char* mode) +inline int modeStrToFlags(const char* mode) { - std::string modestr = mode; + std::string modestr = mode; - // this could easily be migrated to a lookup table if performance - // ever became a concern, but for now this is fine. - if ( modestr == "r" || modestr == "rb" ) - { - return O_RDONLY; - } - else if ( modestr == "r+" || modestr == "r+b" ) - { - return O_RDWR; - } - else if ( modestr == "w" || modestr == "wb" ) - { - return O_WRONLY | O_CREAT | O_TRUNC; - } - else if ( modestr == "w+" || modestr == "w+b" ) - { - return O_RDWR | O_CREAT | O_TRUNC; - } - else if ( modestr == "a" || modestr == "ab" ) - { - return O_WRONLY | O_CREAT | O_APPEND; - } - else if ( modestr == "a+" || modestr == "a+b" ) - { - return O_RDWR | O_CREAT | O_APPEND; - } - else - { - // error - return -1; - } + // this could easily be migrated to a lookup table if performance + // ever became a concern, but for now this is fine. + if (modestr == "r" || modestr == "rb") + { + return O_RDONLY; + } + else if (modestr == "r+" || modestr == "r+b") + { + return O_RDWR; + } + else if (modestr == "w" || modestr == "wb") + { + return O_WRONLY | O_CREAT | O_TRUNC; + } + else if (modestr == "w+" || modestr == "w+b") + { + return O_RDWR | O_CREAT | O_TRUNC; + } + else if (modestr == "a" || modestr == "ab") + { + return O_WRONLY | O_CREAT | O_APPEND; + } + else if (modestr == "a+" || modestr == "a+b") + { + return O_RDWR | O_CREAT | O_APPEND; + } + else + { + // error + return -1; + } } #ifdef _MSC_VER -inline -int modeStrToFlags(const char* mode, int& createflags) +inline int modeStrToFlags(const char* mode, int& createflags) { - std::string modestr = mode; - createflags = 0; + std::string modestr = mode; + createflags = 0; - // this could easily be migrated to a lookup table if performance - // ever became a concern, but for now this is fine. - if ( modestr == "r" || modestr == "rb" ) - { - createflags = OPEN_EXISTING; - return GENERIC_READ; - } - else if ( modestr == "r+" || modestr == "r+b" ) - { - createflags = OPEN_EXISTING; - return GENERIC_READ | GENERIC_WRITE; - } - else if ( modestr == "w" || modestr == "wb" ) - { - createflags = CREATE_ALWAYS; - return GENERIC_WRITE; - } - else if ( modestr == "w+" || modestr == "w+b" ) - { - createflags = CREATE_ALWAYS; - return GENERIC_READ | GENERIC_WRITE; - } - else if ( modestr == "a" || modestr == "ab" ) - { - createflags = OPEN_ALWAYS; - return FILE_APPEND_DATA; - } - else if ( modestr == "a+" || modestr == "a+b" ) - { - createflags = OPEN_ALWAYS; - return GENERIC_READ | FILE_APPEND_DATA; - } - else - { - // error - return -1; - } + // this could easily be migrated to a lookup table if performance + // ever became a concern, but for now this is fine. + if (modestr == "r" || modestr == "rb") + { + createflags = OPEN_EXISTING; + return GENERIC_READ; + } + else if (modestr == "r+" || modestr == "r+b") + { + createflags = OPEN_EXISTING; + return GENERIC_READ | GENERIC_WRITE; + } + else if (modestr == "w" || modestr == "wb") + { + createflags = CREATE_ALWAYS; + return GENERIC_WRITE; + } + else if (modestr == "w+" || modestr == "w+b") + { + createflags = CREATE_ALWAYS; + return GENERIC_READ | GENERIC_WRITE; + } + else if (modestr == "a" || modestr == "ab") + { + createflags = OPEN_ALWAYS; + return FILE_APPEND_DATA; + } + else if (modestr == "a+" || modestr == "a+b") + { + createflags = OPEN_ALWAYS; + return GENERIC_READ | FILE_APPEND_DATA; + } + else + { + // error + return -1; + } } #endif -} +} // namespace idbdatafile #endif /* UTILITY_H_ */ diff --git a/utils/joiner/btree.h b/utils/joiner/btree.h index 1a7c94bcb..1890d27c0 100644 --- a/utils/joiner/btree.h +++ b/utils/joiner/btree.h @@ -121,7 +121,6 @@ namespace btree { - // Inside a btree method, if we just call swap(), it will choose the // btree::swap method, which we don't want. And we can't say ::swap // because then MSVC won't pickup any std::swap() implementations. We @@ -132,21 +131,21 @@ namespace btree template inline void btree_swap_helper(T& a, T& b) { - using std::swap; - swap(a, b); + using std::swap; + swap(a, b); } // A template helper used to select A or B based on a condition. -template +template struct if_ { - typedef A type; + typedef A type; }; -template +template struct if_ { - typedef B type; + typedef B type; }; // Types small_ and big_ are promise that sizeof(small_) < sizeof(big_) @@ -154,7 +153,7 @@ typedef char small_; struct big_ { - char dummy[2]; + char dummy[2]; }; // A compile-time assertion. @@ -163,8 +162,7 @@ struct CompileAssert { }; -#define COMPILE_ASSERT(expr, msg) \ - typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] +#define COMPILE_ASSERT(expr, msg) typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] // A helper type used to indicate that a key-compare-to functor has been // provided. A user can specify a key-compare-to functor by doing: @@ -185,8 +183,7 @@ struct btree_key_compare_to_tag // A helper class that indicates if the Compare parameter is derived from // btree_key_compare_to_tag. template -struct btree_is_key_compare_to - : public std::is_convertible +struct btree_is_key_compare_to : public std::is_convertible { }; @@ -201,40 +198,51 @@ struct btree_is_key_compare_to template struct btree_key_compare_to_adapter : Compare { - btree_key_compare_to_adapter() { } - btree_key_compare_to_adapter(const Compare& c) : Compare(c) { } - btree_key_compare_to_adapter(const btree_key_compare_to_adapter& c) - : Compare(c) - { - } + btree_key_compare_to_adapter() + { + } + btree_key_compare_to_adapter(const Compare& c) : Compare(c) + { + } + btree_key_compare_to_adapter(const btree_key_compare_to_adapter& c) : Compare(c) + { + } }; template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag +struct btree_key_compare_to_adapter > : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const std::less&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const std::string& a, const std::string& b) const - { - return a.compare(b); - } + btree_key_compare_to_adapter() + { + } + btree_key_compare_to_adapter(const std::less&) + { + } + btree_key_compare_to_adapter(const btree_key_compare_to_adapter >&) + { + } + int operator()(const std::string& a, const std::string& b) const + { + return a.compare(b); + } }; template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag +struct btree_key_compare_to_adapter > : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const std::greater&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const std::string& a, const std::string& b) const - { - return b.compare(a); - } + btree_key_compare_to_adapter() + { + } + btree_key_compare_to_adapter(const std::greater&) + { + } + btree_key_compare_to_adapter(const btree_key_compare_to_adapter >&) + { + } + int operator()(const std::string& a, const std::string& b) const + { + return b.compare(a); + } }; // A helper class that allows a compare-to functor to behave like a plain @@ -243,17 +251,21 @@ struct btree_key_compare_to_adapter > template struct btree_key_comparer { - btree_key_comparer() {} - btree_key_comparer(Compare c) : comp(c) {} - static bool bool_compare(const Compare& comp, const Key& x, const Key& y) - { - return comp(x, y); - } - bool operator()(const Key& x, const Key& y) const - { - return bool_compare(comp, x, y); - } - Compare comp; + btree_key_comparer() + { + } + btree_key_comparer(Compare c) : comp(c) + { + } + static bool bool_compare(const Compare& comp, const Key& x, const Key& y) + { + return comp(x, y); + } + bool operator()(const Key& x, const Key& y) const + { + return bool_compare(comp, x, y); + } + Compare comp; }; // A specialization of btree_key_comparer when a compare-to functor is @@ -262,17 +274,21 @@ struct btree_key_comparer template struct btree_key_comparer { - btree_key_comparer() {} - btree_key_comparer(Compare c) : comp(c) {} - static bool bool_compare(const Compare& comp, const Key& x, const Key& y) - { - return comp(x, y) < 0; - } - bool operator()(const Key& x, const Key& y) const - { - return bool_compare(comp, x, y); - } - Compare comp; + btree_key_comparer() + { + } + btree_key_comparer(Compare c) : comp(c) + { + } + static bool bool_compare(const Compare& comp, const Key& x, const Key& y) + { + return comp(x, y) < 0; + } + bool operator()(const Key& x, const Key& y) const + { + return bool_compare(comp, x, y); + } + Compare comp; }; // A helper function to compare to keys using the specified compare @@ -280,198 +296,189 @@ struct btree_key_comparer // depending on whether we have a compare-to functor or not (which depends on // whether Compare is derived from btree_key_compare_to_tag). template -bool btree_compare_keys( - const Compare& comp, const Key& x, const Key& y) +bool btree_compare_keys(const Compare& comp, const Key& x, const Key& y) { - typedef btree_key_comparer::value> key_comparer; - return key_comparer::bool_compare(comp, x, y); + typedef btree_key_comparer::value> key_comparer; + return key_comparer::bool_compare(comp, x, y); } -template +template struct btree_common_params { - // If Compare is derived from btree_key_compare_to_tag then use it as the - // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will - // fall-back to Compare if we don't have an appropriate specialization. - typedef typename if_ < - btree_is_key_compare_to::value, - Compare, btree_key_compare_to_adapter >::type key_compare; - // A type which indicates if we have a key-compare-to functor or a plain old - // key-compare functor. - typedef btree_is_key_compare_to is_key_compare_to; + // If Compare is derived from btree_key_compare_to_tag then use it as the + // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will + // fall-back to Compare if we don't have an appropriate specialization. + typedef typename if_::value, Compare, + btree_key_compare_to_adapter >::type key_compare; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + typedef btree_is_key_compare_to is_key_compare_to; - typedef Alloc allocator_type; - typedef Key key_type; - typedef ssize_t size_type; - typedef ptrdiff_t difference_type; + typedef Alloc allocator_type; + typedef Key key_type; + typedef ssize_t size_type; + typedef ptrdiff_t difference_type; - enum - { - kTargetNodeSize = TargetNodeSize, + enum + { + kTargetNodeSize = TargetNodeSize, - // Available space for values. This is largest for leaf nodes, - // which has overhead no fewer than two pointers. - kNodeValueSpace = TargetNodeSize - 2 * sizeof(void*), - }; + // Available space for values. This is largest for leaf nodes, + // which has overhead no fewer than two pointers. + kNodeValueSpace = TargetNodeSize - 2 * sizeof(void*), + }; - // This is an integral type large enough to hold as many - // ValueSize-values as will fit a node of TargetNodeSize bytes. - typedef typename if_ < - (kNodeValueSpace / ValueSize) >= 256, - uint16_t, - uint8_t >::type node_count_type; + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + typedef typename if_<(kNodeValueSpace / ValueSize) >= 256, uint16_t, uint8_t>::type node_count_type; }; // A parameters structure for holding the type parameters for a btree_map. -template +template struct btree_map_params - : public btree_common_params < Key, Compare, Alloc, TargetNodeSize, - sizeof(Key) + sizeof(Data) > + : public btree_common_params +{ + typedef Data data_type; + typedef Data mapped_type; + typedef std::pair value_type; + typedef std::pair mutable_value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + + enum { - typedef Data data_type; - typedef Data mapped_type; - typedef std::pair value_type; - typedef std::pair mutable_value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - - enum -{ kValueSize = sizeof(Key) + sizeof(data_type), -}; + }; -static const Key& key(const value_type& x) -{ + static const Key& key(const value_type& x) + { return x.first; -} -static const Key& key(const mutable_value_type& x) -{ + } + static const Key& key(const mutable_value_type& x) + { return x.first; -} -static void swap(mutable_value_type* a, mutable_value_type* b) -{ + } + static void swap(mutable_value_type* a, mutable_value_type* b) + { btree_swap_helper(a->first, b->first); btree_swap_helper(a->second, b->second); -} - }; + } +}; // A parameters structure for holding the type parameters for a btree_set. template -struct btree_set_params - : public btree_common_params +struct btree_set_params : public btree_common_params +{ + typedef std::false_type data_type; + typedef std::false_type mapped_type; + typedef Key value_type; + typedef value_type mutable_value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + + enum { - typedef std::false_type data_type; - typedef std::false_type mapped_type; - typedef Key value_type; - typedef value_type mutable_value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - - enum -{ kValueSize = sizeof(Key), -}; - -static const Key& key(const value_type& x) -{ - return x; -} -static void swap(mutable_value_type* a, mutable_value_type* b) -{ - btree_swap_helper(*a, *b); -} }; + static const Key& key(const value_type& x) + { + return x; + } + static void swap(mutable_value_type* a, mutable_value_type* b) + { + btree_swap_helper(*a, *b); + } +}; + // An adapter class that converts a lower-bound compare into an upper-bound // compare. template struct btree_upper_bound_adapter : public Compare { - btree_upper_bound_adapter(Compare c) : Compare(c) {} - bool operator()(const Key& a, const Key& b) const - { - return !static_cast(*this)(b, a); - } + btree_upper_bound_adapter(Compare c) : Compare(c) + { + } + bool operator()(const Key& a, const Key& b) const + { + return !static_cast(*this)(b, a); + } }; template struct btree_upper_bound_compare_to_adapter : public CompareTo { - btree_upper_bound_compare_to_adapter(CompareTo c) : CompareTo(c) {} - int operator()(const Key& a, const Key& b) const - { - return static_cast(*this)(b, a); - } + btree_upper_bound_compare_to_adapter(CompareTo c) : CompareTo(c) + { + } + int operator()(const Key& a, const Key& b) const + { + return static_cast(*this)(b, a); + } }; // Dispatch helper class for using linear search with plain compare. template struct btree_linear_search_plain_compare { - static int lower_bound(const K& k, const N& n, Compare comp) - { - return n.linear_search_plain_compare(k, 0, n.count(), comp); - } - static int upper_bound(const K& k, const N& n, Compare comp) - { - typedef btree_upper_bound_adapter upper_compare; - return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); - } + static int lower_bound(const K& k, const N& n, Compare comp) + { + return n.linear_search_plain_compare(k, 0, n.count(), comp); + } + static int upper_bound(const K& k, const N& n, Compare comp) + { + typedef btree_upper_bound_adapter upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } }; // Dispatch helper class for using linear search with compare-to template struct btree_linear_search_compare_to { - static int lower_bound(const K& k, const N& n, CompareTo comp) - { - return n.linear_search_compare_to(k, 0, n.count(), comp); - } - static int upper_bound(const K& k, const N& n, CompareTo comp) - { - typedef btree_upper_bound_adapter > upper_compare; - return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); - } + static int lower_bound(const K& k, const N& n, CompareTo comp) + { + return n.linear_search_compare_to(k, 0, n.count(), comp); + } + static int upper_bound(const K& k, const N& n, CompareTo comp) + { + typedef btree_upper_bound_adapter > upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } }; // Dispatch helper class for using binary search with plain compare. template struct btree_binary_search_plain_compare { - static int lower_bound(const K& k, const N& n, Compare comp) - { - return n.binary_search_plain_compare(k, 0, n.count(), comp); - } - static int upper_bound(const K& k, const N& n, Compare comp) - { - typedef btree_upper_bound_adapter upper_compare; - return n.binary_search_plain_compare(k, 0, n.count(), upper_compare(comp)); - } + static int lower_bound(const K& k, const N& n, Compare comp) + { + return n.binary_search_plain_compare(k, 0, n.count(), comp); + } + static int upper_bound(const K& k, const N& n, Compare comp) + { + typedef btree_upper_bound_adapter upper_compare; + return n.binary_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } }; // Dispatch helper class for using binary search with compare-to. template struct btree_binary_search_compare_to { - static int lower_bound(const K& k, const N& n, CompareTo comp) - { - return n.binary_search_compare_to(k, 0, n.count(), CompareTo()); - } - static int upper_bound(const K& k, const N& n, CompareTo comp) - { - typedef btree_upper_bound_adapter > upper_compare; - return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); - } + static int lower_bound(const K& k, const N& n, CompareTo comp) + { + return n.binary_search_compare_to(k, 0, n.count(), CompareTo()); + } + static int upper_bound(const K& k, const N& n, CompareTo comp) + { + typedef btree_upper_bound_adapter > upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } }; // A node in the btree holding. The same node type is used for both internal @@ -480,2026 +487,1954 @@ struct btree_binary_search_compare_to template class btree_node { -public: - typedef Params params_type; - typedef btree_node self_type; - typedef typename Params::key_type key_type; - typedef typename Params::data_type data_type; - typedef typename Params::value_type value_type; - typedef typename Params::mutable_value_type mutable_value_type; - typedef typename Params::pointer pointer; - typedef typename Params::const_pointer const_pointer; - typedef typename Params::reference reference; - typedef typename Params::const_reference const_reference; - typedef typename Params::key_compare key_compare; - typedef typename Params::size_type size_type; - typedef typename Params::difference_type difference_type; - // Typedefs for the various types of node searches. - typedef btree_linear_search_plain_compare < - key_type, self_type, key_compare > linear_search_plain_compare_type; - typedef btree_linear_search_compare_to < - key_type, self_type, key_compare > linear_search_compare_to_type; - typedef btree_binary_search_plain_compare < - key_type, self_type, key_compare > binary_search_plain_compare_type; - typedef btree_binary_search_compare_to < - key_type, self_type, key_compare > binary_search_compare_to_type; - // If we have a valid key-compare-to type, use linear_search_compare_to, - // otherwise use linear_search_plain_compare. - typedef typename if_ < - Params::is_key_compare_to::value, - linear_search_compare_to_type, - linear_search_plain_compare_type >::type linear_search_type; - // If we have a valid key-compare-to type, use binary_search_compare_to, - // otherwise use binary_search_plain_compare. - typedef typename if_ < - Params::is_key_compare_to::value, - binary_search_compare_to_type, - binary_search_plain_compare_type >::type binary_search_type; - // If the key is an integral or floating point type, use linear search which - // is faster than binary search for such types. Might be wise to also - // configure linear search based on node-size. - typedef typename if_ < - std::is_integral::value || - std::is_floating_point::value, - linear_search_type, binary_search_type >::type search_type; + public: + typedef Params params_type; + typedef btree_node self_type; + typedef typename Params::key_type key_type; + typedef typename Params::data_type data_type; + typedef typename Params::value_type value_type; + typedef typename Params::mutable_value_type mutable_value_type; + typedef typename Params::pointer pointer; + typedef typename Params::const_pointer const_pointer; + typedef typename Params::reference reference; + typedef typename Params::const_reference const_reference; + typedef typename Params::key_compare key_compare; + typedef typename Params::size_type size_type; + typedef typename Params::difference_type difference_type; + // Typedefs for the various types of node searches. + typedef btree_linear_search_plain_compare + linear_search_plain_compare_type; + typedef btree_linear_search_compare_to linear_search_compare_to_type; + typedef btree_binary_search_plain_compare + binary_search_plain_compare_type; + typedef btree_binary_search_compare_to binary_search_compare_to_type; + // If we have a valid key-compare-to type, use linear_search_compare_to, + // otherwise use linear_search_plain_compare. + typedef typename if_::type linear_search_type; + // If we have a valid key-compare-to type, use binary_search_compare_to, + // otherwise use binary_search_plain_compare. + typedef typename if_::type binary_search_type; + // If the key is an integral or floating point type, use linear search which + // is faster than binary search for such types. Might be wise to also + // configure linear search based on node-size. + typedef typename if_::value || std::is_floating_point::value, + linear_search_type, binary_search_type>::type search_type; - struct base_fields + struct base_fields + { + typedef typename Params::node_count_type field_type; + + // A boolean indicating whether the node is a leaf or not. + bool leaf; + // The position of the node in the node's parent. + field_type position; + // The maximum number of values the node can hold. + field_type max_count; + // The count of the number of values in the node. + field_type count; + // A pointer to the node's parent. + btree_node* parent; + }; + + enum + { + kValueSize = params_type::kValueSize, + kTargetNodeSize = params_type::kTargetNodeSize, + + // Compute how many values we can fit onto a leaf node. + kNodeTargetValues = (kTargetNodeSize - sizeof(base_fields)) / kValueSize, + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + kExactMatch = 1 << 30, + kMatchMask = kExactMatch - 1, + }; + + struct leaf_fields : public base_fields + { + // The array of values. Only the first count of these values have been + // constructed and are valid. + mutable_value_type values[kNodeValues]; + }; + + struct internal_fields : public leaf_fields + { + // The array of child pointers. The keys in children_[i] are all less than + // key(i). The keys in children_[i + 1] are all greater than key(i). There + // are always count + 1 children. + btree_node* children[kNodeValues + 1]; + }; + + struct root_fields : public internal_fields + { + btree_node* rightmost; + size_type size; + }; + + public: + // Getter/setter for whether this is a leaf node or not. This value doesn't + // change after the node is created. + bool leaf() const + { + return fields_.leaf; + } + + // Getter for the position of this node in its parent. + int position() const + { + return fields_.position; + } + void set_position(int v) + { + fields_.position = v; + } + + // Getter/setter for the number of values stored in this node. + int count() const + { + return fields_.count; + } + void set_count(int v) + { + fields_.count = v; + } + int max_count() const + { + return fields_.max_count; + } + + // Getter for the parent of this node. + btree_node* parent() const + { + return fields_.parent; + } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const + { + return parent()->leaf(); + } + void make_root() + { + assert(parent()->is_root()); + fields_.parent = fields_.parent->parent(); + } + + // Getter for the rightmost root node field. Only valid on the root node. + btree_node* rightmost() const + { + return fields_.rightmost; + } + btree_node** mutable_rightmost() + { + return &fields_.rightmost; + } + + // Getter for the size root node field. Only valid on the root node. + size_type size() const + { + return fields_.size; + } + size_type* mutable_size() + { + return &fields_.size; + } + + // Getters for the key/value at position i in the node. + const key_type& key(int i) const + { + return params_type::key(fields_.values[i]); + } + reference value(int i) + { + return reinterpret_cast(fields_.values[i]); + } + const_reference value(int i) const + { + return reinterpret_cast(fields_.values[i]); + } + mutable_value_type* mutable_value(int i) + { + return &fields_.values[i]; + } + + // Swap value i in this node with value j in node x. + void value_swap(int i, btree_node* x, int j) + { + params_type::swap(mutable_value(i), x->mutable_value(j)); + } + + // Getters/setter for the child at position i in the node. + btree_node* child(int i) const + { + return fields_.children[i]; + } + btree_node** mutable_child(int i) + { + return &fields_.children[i]; + } + void set_child(int i, btree_node* c) + { + *mutable_child(i) = c; + c->fields_.parent = this; + c->fields_.position = i; + } + + // Returns the position of the first value whose key is not less than k. + template + int lower_bound(const key_type& k, const Compare& comp) const + { + return search_type::lower_bound(k, *this, comp); + } + // Returns the position of the first value whose key is greater than k. + template + int upper_bound(const key_type& k, const Compare& comp) const + { + return search_type::upper_bound(k, *this, comp); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template + int linear_search_plain_compare(const key_type& k, int s, int e, const Compare& comp) const + { + while (s < e) { - typedef typename Params::node_count_type field_type; + if (!btree_compare_keys(comp, key(s), k)) + { + break; + } - // A boolean indicating whether the node is a leaf or not. - bool leaf; - // The position of the node in the node's parent. - field_type position; - // The maximum number of values the node can hold. - field_type max_count; - // The count of the number of values in the node. - field_type count; - // A pointer to the node's parent. - btree_node* parent; - }; - - enum - { - kValueSize = params_type::kValueSize, - kTargetNodeSize = params_type::kTargetNodeSize, - - // Compute how many values we can fit onto a leaf node. - kNodeTargetValues = (kTargetNodeSize - sizeof(base_fields)) / kValueSize, - // We need a minimum of 3 values per internal node in order to perform - // splitting (1 value for the two nodes involved in the split and 1 value - // propagated to the parent as the delimiter for the split). - kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, - - kExactMatch = 1 << 30, - kMatchMask = kExactMatch - 1, - }; - - struct leaf_fields : public base_fields - { - // The array of values. Only the first count of these values have been - // constructed and are valid. - mutable_value_type values[kNodeValues]; - }; - - struct internal_fields : public leaf_fields - { - // The array of child pointers. The keys in children_[i] are all less than - // key(i). The keys in children_[i + 1] are all greater than key(i). There - // are always count + 1 children. - btree_node* children[kNodeValues + 1]; - }; - - struct root_fields : public internal_fields - { - btree_node* rightmost; - size_type size; - }; - -public: - // Getter/setter for whether this is a leaf node or not. This value doesn't - // change after the node is created. - bool leaf() const - { - return fields_.leaf; + ++s; } - // Getter for the position of this node in its parent. - int position() const + return s; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template + int linear_search_compare_to(const key_type& k, int s, int e, const Compare& comp) const + { + while (s < e) { - return fields_.position; - } - void set_position(int v) - { - fields_.position = v; + int c = comp(key(s), k); + + if (c == 0) + { + return s | kExactMatch; + } + else if (c > 0) + { + break; + } + + ++s; } - // Getter/setter for the number of values stored in this node. - int count() const + return s; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template + int binary_search_plain_compare(const key_type& k, int s, int e, const Compare& comp) const + { + while (s != e) { - return fields_.count; - } - void set_count(int v) - { - fields_.count = v; - } - int max_count() const - { - return fields_.max_count; + int mid = (s + e) / 2; + + if (btree_compare_keys(comp, key(mid), k)) + { + s = mid + 1; + } + else + { + e = mid; + } } - // Getter for the parent of this node. - btree_node* parent() const + return s; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template + int binary_search_compare_to(const key_type& k, int s, int e, const CompareTo& comp) const + { + while (s != e) { - return fields_.parent; - } - // Getter for whether the node is the root of the tree. The parent of the - // root of the tree is the leftmost node in the tree which is guaranteed to - // be a leaf. - bool is_root() const - { - return parent()->leaf(); - } - void make_root() - { - assert(parent()->is_root()); - fields_.parent = fields_.parent->parent(); + int mid = (s + e) / 2; + int c = comp(key(mid), k); + + if (c < 0) + { + s = mid + 1; + } + else if (c > 0) + { + e = mid; + } + else + { + // Need to return the first value whose key is not less than k, which + // requires continuing the binary search. Note that we are guaranteed + // that the result is an exact match because if "key(mid-1) < k" the + // call to binary_search_compare_to() will return "mid". + s = binary_search_compare_to(k, s, mid, comp); + return s | kExactMatch; + } } - // Getter for the rightmost root node field. Only valid on the root node. - btree_node* rightmost() const + return s; + } + + // Inserts the value x at position i, shifting all existing values and + // children at positions >= i to the right by 1. + void insert_value(int i, const value_type& x); + + // Removes the value at position i, shifting all existing values and children + // at positions > i to the left by 1. + void remove_value(int i); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(btree_node* sibling, int to_move); + void rebalance_left_to_right(btree_node* sibling, int to_move); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(btree_node* sibling, int insert_position); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself. + void merge(btree_node* sibling); + + // Swap the contents of "this" and "src". + void swap(btree_node* src); + + // Node allocation/deletion routines. + static btree_node* init_leaf(leaf_fields* f, btree_node* parent, int max_count) + { + btree_node* n = reinterpret_cast(f); + f->leaf = 1; + f->position = 0; + f->max_count = max_count; + f->count = 0; + f->parent = parent; + + if (!NDEBUG) { - return fields_.rightmost; - } - btree_node** mutable_rightmost() - { - return &fields_.rightmost; + memset(&f->values, 0, max_count * sizeof(value_type)); } - // Getter for the size root node field. Only valid on the root node. - size_type size() const + return n; + } + static btree_node* init_internal(internal_fields* f, btree_node* parent) + { + btree_node* n = init_leaf(f, parent, kNodeValues); + f->leaf = 0; + + if (!NDEBUG) { - return fields_.size; - } - size_type* mutable_size() - { - return &fields_.size; + memset(f->children, 0, sizeof(f->children)); } - // Getters for the key/value at position i in the node. - const key_type& key(int i) const + return n; + } + static btree_node* init_root(root_fields* f, btree_node* parent) + { + btree_node* n = init_internal(f, parent); + f->rightmost = parent; + f->size = parent->count(); + return n; + } + void destroy() + { + for (int i = 0; i < count(); ++i) { - return params_type::key(fields_.values[i]); - } - reference value(int i) - { - return reinterpret_cast(fields_.values[i]); - } - const_reference value(int i) const - { - return reinterpret_cast(fields_.values[i]); - } - mutable_value_type* mutable_value(int i) - { - return &fields_.values[i]; + value_destroy(i); } + } - // Swap value i in this node with value j in node x. - void value_swap(int i, btree_node* x, int j) - { - params_type::swap(mutable_value(i), x->mutable_value(j)); - } + private: + void value_init(int i) + { + new (&fields_.values[i]) mutable_value_type; + } + void value_init(int i, const value_type& x) + { + new (&fields_.values[i]) mutable_value_type(x); + } + void value_destroy(int i) + { + fields_.values[i].~mutable_value_type(); + } - // Getters/setter for the child at position i in the node. - btree_node* child(int i) const - { - return fields_.children[i]; - } - btree_node** mutable_child(int i) - { - return &fields_.children[i]; - } - void set_child(int i, btree_node* c) - { - *mutable_child(i) = c; - c->fields_.parent = this; - c->fields_.position = i; - } + private: + root_fields fields_; - // Returns the position of the first value whose key is not less than k. - template - int lower_bound(const key_type& k, const Compare& comp) const - { - return search_type::lower_bound(k, *this, comp); - } - // Returns the position of the first value whose key is greater than k. - template - int upper_bound(const key_type& k, const Compare& comp) const - { - return search_type::upper_bound(k, *this, comp); - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using plain compare. - template - int linear_search_plain_compare( - const key_type& k, int s, int e, const Compare& comp) const - { - while (s < e) - { - if (!btree_compare_keys(comp, key(s), k)) - { - break; - } - - ++s; - } - - return s; - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using compare-to. - template - int linear_search_compare_to( - const key_type& k, int s, int e, const Compare& comp) const - { - while (s < e) - { - int c = comp(key(s), k); - - if (c == 0) - { - return s | kExactMatch; - } - else if (c > 0) - { - break; - } - - ++s; - } - - return s; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using plain compare. - template - int binary_search_plain_compare( - const key_type& k, int s, int e, const Compare& comp) const - { - while (s != e) - { - int mid = (s + e) / 2; - - if (btree_compare_keys(comp, key(mid), k)) - { - s = mid + 1; - } - else - { - e = mid; - } - } - - return s; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using compare-to. - template - int binary_search_compare_to( - const key_type& k, int s, int e, const CompareTo& comp) const - { - while (s != e) - { - int mid = (s + e) / 2; - int c = comp(key(mid), k); - - if (c < 0) - { - s = mid + 1; - } - else if (c > 0) - { - e = mid; - } - else - { - // Need to return the first value whose key is not less than k, which - // requires continuing the binary search. Note that we are guaranteed - // that the result is an exact match because if "key(mid-1) < k" the - // call to binary_search_compare_to() will return "mid". - s = binary_search_compare_to(k, s, mid, comp); - return s | kExactMatch; - } - } - - return s; - } - - // Inserts the value x at position i, shifting all existing values and - // children at positions >= i to the right by 1. - void insert_value(int i, const value_type& x); - - // Removes the value at position i, shifting all existing values and children - // at positions > i to the left by 1. - void remove_value(int i); - - // Rebalances a node with its right sibling. - void rebalance_right_to_left(btree_node* sibling, int to_move); - void rebalance_left_to_right(btree_node* sibling, int to_move); - - // Splits a node, moving a portion of the node's values to its right sibling. - void split(btree_node* sibling, int insert_position); - - // Merges a node with its right sibling, moving all of the values and the - // delimiting key in the parent node onto itself. - void merge(btree_node* sibling); - - // Swap the contents of "this" and "src". - void swap(btree_node* src); - - // Node allocation/deletion routines. - static btree_node* init_leaf( - leaf_fields* f, btree_node* parent, int max_count) - { - btree_node* n = reinterpret_cast(f); - f->leaf = 1; - f->position = 0; - f->max_count = max_count; - f->count = 0; - f->parent = parent; - - if (!NDEBUG) - { - memset(&f->values, 0, max_count * sizeof(value_type)); - } - - return n; - } - static btree_node* init_internal(internal_fields* f, btree_node* parent) - { - btree_node* n = init_leaf(f, parent, kNodeValues); - f->leaf = 0; - - if (!NDEBUG) - { - memset(f->children, 0, sizeof(f->children)); - } - - return n; - } - static btree_node* init_root(root_fields* f, btree_node* parent) - { - btree_node* n = init_internal(f, parent); - f->rightmost = parent; - f->size = parent->count(); - return n; - } - void destroy() - { - for (int i = 0; i < count(); ++i) - { - value_destroy(i); - } - } - -private: - void value_init(int i) - { - new (&fields_.values[i]) mutable_value_type; - } - void value_init(int i, const value_type& x) - { - new (&fields_.values[i]) mutable_value_type(x); - } - void value_destroy(int i) - { - fields_.values[i].~mutable_value_type(); - } - -private: - root_fields fields_; - -private: - btree_node(const btree_node&); - void operator=(const btree_node&); + private: + btree_node(const btree_node&); + void operator=(const btree_node&); }; template struct btree_iterator { - typedef typename Node::key_type key_type; - typedef typename Node::size_type size_type; - typedef typename Node::difference_type difference_type; - typedef typename Node::params_type params_type; + typedef typename Node::key_type key_type; + typedef typename Node::size_type size_type; + typedef typename Node::difference_type difference_type; + typedef typename Node::params_type params_type; - typedef Node node_type; - typedef typename std::remove_const::type normal_node; - typedef const Node const_node; - typedef typename params_type::value_type value_type; - typedef typename params_type::pointer normal_pointer; - typedef typename params_type::reference normal_reference; - typedef typename params_type::const_pointer const_pointer; - typedef typename params_type::const_reference const_reference; + typedef Node node_type; + typedef typename std::remove_const::type normal_node; + typedef const Node const_node; + typedef typename params_type::value_type value_type; + typedef typename params_type::pointer normal_pointer; + typedef typename params_type::reference normal_reference; + typedef typename params_type::const_pointer const_pointer; + typedef typename params_type::const_reference const_reference; - typedef Pointer pointer; - typedef Reference reference; - typedef std::bidirectional_iterator_tag iterator_category; + typedef Pointer pointer; + typedef Reference reference; + typedef std::bidirectional_iterator_tag iterator_category; - typedef btree_iterator < - normal_node, normal_reference, normal_pointer > iterator; - typedef btree_iterator < - const_node, const_reference, const_pointer > const_iterator; - typedef btree_iterator self_type; + typedef btree_iterator iterator; + typedef btree_iterator const_iterator; + typedef btree_iterator self_type; - btree_iterator() - : node(NULL), - position(-1) - { - } - btree_iterator(Node* n, int p) - : node(n), - position(p) - { - } - btree_iterator(const iterator& x) - : node(x.node), - position(x.position) + btree_iterator() : node(NULL), position(-1) + { + } + btree_iterator(Node* n, int p) : node(n), position(p) + { + } + btree_iterator(const iterator& x) : node(x.node), position(x.position) + { + } + + // Increment/decrement the iterator. + void increment() + { + if (node->leaf() && ++position < node->count()) { + return; } - // Increment/decrement the iterator. - void increment() - { - if (node->leaf() && ++position < node->count()) - { - return; - } + increment_slow(); + } + void increment_by(int count); + void increment_slow(); - increment_slow(); - } - void increment_by(int count); - void increment_slow(); - - void decrement() + void decrement() + { + if (node->leaf() && --position >= 0) { - if (node->leaf() && --position >= 0) - { - return; - } - - decrement_slow(); - } - void decrement_slow(); - - bool operator==(const const_iterator& x) const - { - return node == x.node && position == x.position; - } - bool operator!=(const const_iterator& x) const - { - return node != x.node || position != x.position; + return; } - // Accessors for the key/value the iterator is pointing at. - const key_type& key() const - { - return node->key(position); - } - reference operator*() const - { - return node->value(position); - } - pointer operator->() const - { - return &node->value(position); - } + decrement_slow(); + } + void decrement_slow(); - self_type& operator++() - { - increment(); - return *this; - } - self_type& operator--() - { - decrement(); - return *this; - } - self_type operator++(int) - { - self_type tmp = *this; - ++*this; - return tmp; - } - self_type operator--(int) - { - self_type tmp = *this; - --*this; - return tmp; - } + bool operator==(const const_iterator& x) const + { + return node == x.node && position == x.position; + } + bool operator!=(const const_iterator& x) const + { + return node != x.node || position != x.position; + } - // The node in the tree the iterator is pointing at. - Node* node; - // The position within the node of the tree the iterator is pointing at. - int position; + // Accessors for the key/value the iterator is pointing at. + const key_type& key() const + { + return node->key(position); + } + reference operator*() const + { + return node->value(position); + } + pointer operator->() const + { + return &node->value(position); + } + + self_type& operator++() + { + increment(); + return *this; + } + self_type& operator--() + { + decrement(); + return *this; + } + self_type operator++(int) + { + self_type tmp = *this; + ++*this; + return tmp; + } + self_type operator--(int) + { + self_type tmp = *this; + --*this; + return tmp; + } + + // The node in the tree the iterator is pointing at. + Node* node; + // The position within the node of the tree the iterator is pointing at. + int position; }; // Dispatch helper class for using btree::internal_locate with plain compare. struct btree_internal_locate_plain_compare { - template - static std::pair dispatch(const K& k, const T& t, Iter iter) - { - return t.internal_locate_plain_compare(k, iter); - } + template + static std::pair dispatch(const K& k, const T& t, Iter iter) + { + return t.internal_locate_plain_compare(k, iter); + } }; // Dispatch helper class for using btree::internal_locate with compare-to. struct btree_internal_locate_compare_to { - template - static std::pair dispatch(const K& k, const T& t, Iter iter) - { - return t.internal_locate_compare_to(k, iter); - } + template + static std::pair dispatch(const K& k, const T& t, Iter iter) + { + return t.internal_locate_compare_to(k, iter); + } }; template class btree : public Params::key_compare { - typedef btree self_type; - typedef btree_node node_type; - typedef typename node_type::base_fields base_fields; - typedef typename node_type::leaf_fields leaf_fields; - typedef typename node_type::internal_fields internal_fields; - typedef typename node_type::root_fields root_fields; - typedef typename Params::is_key_compare_to is_key_compare_to; + typedef btree self_type; + typedef btree_node node_type; + typedef typename node_type::base_fields base_fields; + typedef typename node_type::leaf_fields leaf_fields; + typedef typename node_type::internal_fields internal_fields; + typedef typename node_type::root_fields root_fields; + typedef typename Params::is_key_compare_to is_key_compare_to; - friend class btree_internal_locate_plain_compare; - friend class btree_internal_locate_compare_to; - typedef typename if_ < - is_key_compare_to::value, - btree_internal_locate_compare_to, - btree_internal_locate_plain_compare >::type internal_locate_type; + friend class btree_internal_locate_plain_compare; + friend class btree_internal_locate_compare_to; + typedef typename if_::type internal_locate_type; - enum + enum + { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + kValueSize = node_type::kValueSize, + kExactMatch = node_type::kExactMatch, + kMatchMask = node_type::kMatchMask, + }; + + // A helper class to get the empty base class optimization for 0-size + // allocators. Base is internal_allocator_type. + // (e.g. empty_base_handle). If Base is + // 0-size, the compiler doesn't have to reserve any space for it and + // sizeof(empty_base_handle) will simply be sizeof(Data). Google [empty base + // class optimization] for more details. + template + struct empty_base_handle : public Base + { + empty_base_handle(const Base& b, const Data& d) : Base(b), data(d) { - kNodeValues = node_type::kNodeValues, - kMinNodeValues = kNodeValues / 2, - kValueSize = node_type::kValueSize, - kExactMatch = node_type::kExactMatch, - kMatchMask = node_type::kMatchMask, - }; + } + Data data; + }; - // A helper class to get the empty base class optimization for 0-size - // allocators. Base is internal_allocator_type. - // (e.g. empty_base_handle). If Base is - // 0-size, the compiler doesn't have to reserve any space for it and - // sizeof(empty_base_handle) will simply be sizeof(Data). Google [empty base - // class optimization] for more details. - template - struct empty_base_handle : public Base + struct node_stats + { + node_stats(ssize_t l, ssize_t i) : leaf_nodes(l), internal_nodes(i) { - empty_base_handle(const Base& b, const Data& d) - : Base(b), - data(d) + } + + node_stats& operator+=(const node_stats& x) + { + leaf_nodes += x.leaf_nodes; + internal_nodes += x.internal_nodes; + return *this; + } + + ssize_t leaf_nodes; + ssize_t internal_nodes; + }; + + public: + typedef Params params_type; + typedef typename Params::key_type key_type; + typedef typename Params::data_type data_type; + typedef typename Params::mapped_type mapped_type; + typedef typename Params::value_type value_type; + typedef typename Params::key_compare key_compare; + typedef typename Params::pointer pointer; + typedef typename Params::const_pointer const_pointer; + typedef typename Params::reference reference; + typedef typename Params::const_reference const_reference; + typedef typename Params::size_type size_type; + typedef typename Params::difference_type difference_type; + typedef btree_iterator iterator; + typedef typename iterator::const_iterator const_iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + typedef typename Params::allocator_type allocator_type; + typedef typename allocator_type::template rebind::other internal_allocator_type; + + public: + // Default constructor. + btree(const key_compare& comp, const allocator_type& alloc) : key_compare(comp), root_(alloc, NULL) + { + } + + // Copy constructor. + btree(const self_type& x) : key_compare(x.key_comp()), root_(x.internal_allocator(), NULL) + { + assign(x); + } + + // Destructor. + ~btree() + { + clear(); + } + + // Iterator routines. + iterator begin() + { + return iterator(leftmost(), 0); + } + const_iterator begin() const + { + return const_iterator(leftmost(), 0); + } + iterator end() + { + return iterator(rightmost(), rightmost() ? rightmost()->count() : 0); + } + const_iterator end() const + { + return const_iterator(rightmost(), rightmost() ? rightmost()->count() : 0); + } + reverse_iterator rbegin() + { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const + { + return const_reverse_iterator(end()); + } + reverse_iterator rend() + { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const + { + return const_reverse_iterator(begin()); + } + + // Finds the first element whose key is not less than key. + iterator lower_bound(const key_type& key) + { + return internal_end(internal_lower_bound(key, iterator(root(), 0))); + } + const_iterator lower_bound(const key_type& key) const + { + return internal_end(internal_lower_bound(key, const_iterator(root(), 0))); + } + + // Finds the first element whose key is greater than key. + iterator upper_bound(const key_type& key) + { + return internal_end(internal_upper_bound(key, iterator(root(), 0))); + } + const_iterator upper_bound(const key_type& key) const + { + return internal_end(internal_upper_bound(key, const_iterator(root(), 0))); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member pair of + // the pair is equal to upper_bound(key). + std::pair equal_range(const key_type& key) + { + return std::make_pair(lower_bound(key), upper_bound(key)); + } + std::pair equal_range(const key_type& key) const + { + return std::make_pair(lower_bound(key), upper_bound(key)); + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. The + // ValuePointer type is used to avoid instatiating the value unless the key + // is being inserted. Value is not dereferenced if the key already exists in + // the btree. See btree_map::operator[]. + template + std::pair insert_unique(const key_type& key, ValuePointer value) + { + if (empty()) + { + *mutable_root() = new_leaf_root_node(1); + } + + std::pair res = internal_locate(key, iterator(root(), 0)); + iterator& iter = res.first; + + if (res.second == kExactMatch) + { + // The key already exists in the tree, do nothing. + return std::make_pair(internal_last(iter), false); + } + else if (!res.second) + { + iterator last = internal_last(iter); + + if (last.node && !compare_keys(key, last.key())) + { + // The key already exists in the tree, do nothing. + return std::make_pair(last, false); + } + } + + return std::make_pair(internal_insert(iter, *value), true); + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + std::pair insert_unique(const value_type& v) + { + return insert_unique(params_type::key(v), &v); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique(v) were made. + iterator insert_unique(iterator position, const value_type& v) + { + if (!empty()) + { + const key_type& key = params_type::key(v); + + if (position == end() || compare_keys(key, position.key())) + { + iterator prev = position; + + if (position == begin() || compare_keys((--prev).key(), key)) { + // prev.key() < key < position.key() + return internal_insert(position, v); } - Data data; - }; + } + else if (compare_keys(position.key(), key)) + { + iterator next = position; + ++next; - struct node_stats - { - node_stats(ssize_t l, ssize_t i) - : leaf_nodes(l), - internal_nodes(i) + if (next == end() || compare_keys(key, next.key())) { + // position.key() < key < next.key() + return internal_insert(next, v); } + } + else + { + // position.key() == key + return position; + } + } - node_stats& operator+=(const node_stats& x) + return insert_unique(v).first; + } + + // Insert a range of values into the btree. + template + void insert_unique(InputIterator b, InputIterator e) + { + for (; b != e; ++b) + { + insert_unique(end(), *b); + } + } + + // Inserts a value into the btree. The ValuePointer type is used to avoid + // instatiating the value unless the key is being inserted. Value is not + // dereferenced if the key already exists in the btree. See + // btree_map::operator[]. + template + iterator insert_multi(const key_type& key, ValuePointer value) + { + if (empty()) + { + *mutable_root() = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key, iterator(root(), 0)); + + if (!iter.node) + { + iter = end(); + } + + return internal_insert(iter, *value); + } + + // Inserts a value into the btree. + iterator insert_multi(const value_type& v) + { + return insert_multi(params_type::key(v), &v); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + iterator insert_multi(iterator position, const value_type& v) + { + if (!empty()) + { + const key_type& key = params_type::key(v); + + if (position == end() || !compare_keys(position.key(), key)) + { + iterator prev = position; + + if (position == begin() || !compare_keys(key, (--prev).key())) { - leaf_nodes += x.leaf_nodes; - internal_nodes += x.internal_nodes; - return *this; + // prev.key() <= key <= position.key() + return internal_insert(position, v); } + } + else + { + iterator next = position; + ++next; - ssize_t leaf_nodes; - ssize_t internal_nodes; - }; - -public: - typedef Params params_type; - typedef typename Params::key_type key_type; - typedef typename Params::data_type data_type; - typedef typename Params::mapped_type mapped_type; - typedef typename Params::value_type value_type; - typedef typename Params::key_compare key_compare; - typedef typename Params::pointer pointer; - typedef typename Params::const_pointer const_pointer; - typedef typename Params::reference reference; - typedef typename Params::const_reference const_reference; - typedef typename Params::size_type size_type; - typedef typename Params::difference_type difference_type; - typedef btree_iterator iterator; - typedef typename iterator::const_iterator const_iterator; - typedef std::reverse_iterator const_reverse_iterator; - typedef std::reverse_iterator reverse_iterator; - - typedef typename Params::allocator_type allocator_type; - typedef typename allocator_type::template rebind::other - internal_allocator_type; - -public: - // Default constructor. - btree(const key_compare& comp, const allocator_type& alloc) - : key_compare(comp), - root_(alloc, NULL) - { + if (next == end() || !compare_keys(next.key(), key)) + { + // position.key() < key <= next.key() + return internal_insert(next, v); + } + } } - // Copy constructor. - btree(const self_type& x) - : key_compare(x.key_comp()), - root_(x.internal_allocator(), NULL) + return insert_multi(v); + } + + // Insert a range of values into the btree. + template + void insert_multi(InputIterator b, InputIterator e) + { + for (; b != e; ++b) { - assign(x); + insert_multi(end(), *b); + } + } + + void assign(const self_type& x) + { + clear(); + + *mutable_key_comp() = x.key_comp(); + *mutable_internal_allocator() = x.internal_allocator(); + + // Assignment can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + for (const_iterator iter = x.begin(); iter != x.end(); ++iter) + { + if (empty()) + { + insert_multi(*iter); + } + else + { + // If the btree is not empty, we can just insert the new value at the end + // of the tree! + internal_insert(end(), *iter); + } + } + } + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(iterator iter) + { + bool internal_delete = false; + + if (!iter.node->leaf()) + { + // Deletion of a value on an internal node. Swap the key with the largest + // value of our left child. This is easy, we just decrement iter. + iterator tmp_iter(iter--); + assert(iter.node->leaf()); + assert(!compare_keys(tmp_iter.key(), iter.key())); + iter.node->value_swap(iter.position, tmp_iter.node, tmp_iter.position); + internal_delete = true; + --*mutable_size(); + } + else if (!root()->leaf()) + { + --*mutable_size(); } - // Destructor. - ~btree() - { - clear(); - } + // Delete the key from the leaf. + iter.node->remove_value(iter.position); - // Iterator routines. - iterator begin() - { - return iterator(leftmost(), 0); - } - const_iterator begin() const - { - return const_iterator(leftmost(), 0); - } - iterator end() - { - return iterator(rightmost(), rightmost() ? rightmost()->count() : 0); - } - const_iterator end() const - { - return const_iterator(rightmost(), rightmost() ? rightmost()->count() : 0); - } - reverse_iterator rbegin() - { - return reverse_iterator(end()); - } - const_reverse_iterator rbegin() const - { - return const_reverse_iterator(end()); - } - reverse_iterator rend() - { - return reverse_iterator(begin()); - } - const_reverse_iterator rend() const - { - return const_reverse_iterator(begin()); - } + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. - // Finds the first element whose key is not less than key. - iterator lower_bound(const key_type& key) - { - return internal_end( - internal_lower_bound(key, iterator(root(), 0))); - } - const_iterator lower_bound(const key_type& key) const - { - return internal_end( - internal_lower_bound(key, const_iterator(root(), 0))); - } + // Merge/rebalance as we walk back up the tree. + iterator res(iter); - // Finds the first element whose key is greater than key. - iterator upper_bound(const key_type& key) + for (;;) { - return internal_end( - internal_upper_bound(key, iterator(root(), 0))); - } - const_iterator upper_bound(const key_type& key) const - { - return internal_end( - internal_upper_bound(key, const_iterator(root(), 0))); - } + if (iter.node == root()) + { + try_shrink(); - // Finds the range of values which compare equal to key. The first member of - // the returned pair is equal to lower_bound(key). The second member pair of - // the pair is equal to upper_bound(key). - std::pair equal_range(const key_type& key) - { - return std::make_pair(lower_bound(key), upper_bound(key)); - } - std::pair equal_range(const key_type& key) const - { - return std::make_pair(lower_bound(key), upper_bound(key)); - } - - // Inserts a value into the btree only if it does not already exist. The - // boolean return value indicates whether insertion succeeded or failed. The - // ValuePointer type is used to avoid instatiating the value unless the key - // is being inserted. Value is not dereferenced if the key already exists in - // the btree. See btree_map::operator[]. - template - std::pair insert_unique(const key_type& key, ValuePointer value) - { if (empty()) { - *mutable_root() = new_leaf_root_node(1); + return end(); } - std::pair res = internal_locate(key, iterator(root(), 0)); - iterator& iter = res.first; + break; + } - if (res.second == kExactMatch) - { - // The key already exists in the tree, do nothing. - return std::make_pair(internal_last(iter), false); - } - else if (!res.second) - { - iterator last = internal_last(iter); + if (iter.node->count() >= kMinNodeValues) + { + break; + } - if (last.node && !compare_keys(key, last.key())) - { - // The key already exists in the tree, do nothing. - return std::make_pair(last, false); - } - } + bool merged = try_merge_or_rebalance(&iter); - return std::make_pair(internal_insert(iter, *value), true); + if (iter.node->leaf()) + { + res = iter; + } + + if (!merged) + { + break; + } + + iter.node = iter.node->parent(); } - // Inserts a value into the btree only if it does not already exist. The - // boolean return value indicates whether insertion succeeded or failed. - std::pair insert_unique(const value_type& v) + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->count()) { - return insert_unique(params_type::key(v), &v); + res.position = res.node->count() - 1; + ++res; } - // Insert with hint. Check to see if the value should be placed immediately - // before position in the tree. If it does, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_unique(v) were made. - iterator insert_unique(iterator position, const value_type& v) + // If we erased from an internal node, advance the iterator. + if (internal_delete) { - if (!empty()) - { - const key_type& key = params_type::key(v); - - if (position == end() || compare_keys(key, position.key())) - { - iterator prev = position; - - if (position == begin() || compare_keys((--prev).key(), key)) - { - // prev.key() < key < position.key() - return internal_insert(position, v); - } - } - else if (compare_keys(position.key(), key)) - { - iterator next = position; - ++next; - - if (next == end() || compare_keys(key, next.key())) - { - // position.key() < key < next.key() - return internal_insert(next, v); - } - } - else - { - // position.key() == key - return position; - } - } - - return insert_unique(v).first; + ++res; } - // Insert a range of values into the btree. - template - void insert_unique(InputIterator b, InputIterator e) + return res; + } + + // Erases range. Returns the number of keys erased. + int erase(iterator begin, iterator end) + { + int count = distance(begin, end); + + for (int i = 0; i < count; i++) { - for (; b != e; ++b) - { - insert_unique(end(), *b); - } + begin = erase(begin); } - // Inserts a value into the btree. The ValuePointer type is used to avoid - // instatiating the value unless the key is being inserted. Value is not - // dereferenced if the key already exists in the btree. See - // btree_map::operator[]. - template - iterator insert_multi(const key_type& key, ValuePointer value) + return count; + } + + // Erases the specified key from the btree. Returns 1 if an element was + // erased and 0 otherwise. + int erase_unique(const key_type& key) + { + iterator iter = internal_find_unique(key, iterator(root(), 0)); + + if (!iter.node) { - if (empty()) - { - *mutable_root() = new_leaf_root_node(1); - } - - iterator iter = internal_upper_bound(key, iterator(root(), 0)); - - if (!iter.node) - { - iter = end(); - } - - return internal_insert(iter, *value); + // The key doesn't exist in the tree, return nothing done. + return 0; } - // Inserts a value into the btree. - iterator insert_multi(const value_type& v) + erase(iter); + return 1; + } + + // Erases all of the entries matching the specified key from the + // btree. Returns the number of elements erased. + int erase_multi(const key_type& key) + { + iterator begin = internal_lower_bound(key, iterator(root(), 0)); + + if (!begin.node) { - return insert_multi(params_type::key(v), &v); + // The key doesn't exist in the tree, return nothing done. + return 0; } - // Insert with hint. Check to see if the value should be placed immediately - // before position in the tree. If it does, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_multi(v) were made. - iterator insert_multi(iterator position, const value_type& v) - { - if (!empty()) - { - const key_type& key = params_type::key(v); + // Delete all of the keys between begin and upper_bound(key). + iterator end = internal_end(internal_upper_bound(key, iterator(root(), 0))); + return erase(begin, end); + } - if (position == end() || !compare_keys(position.key(), key)) + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + iterator find_unique(const key_type& key) + { + return internal_end(internal_find_unique(key, iterator(root(), 0))); + } + const_iterator find_unique(const key_type& key) const + { + return internal_end(internal_find_unique(key, const_iterator(root(), 0))); + } + iterator find_multi(const key_type& key) + { + return internal_end(internal_find_multi(key, iterator(root(), 0))); + } + const_iterator find_multi(const key_type& key) const + { + return internal_end(internal_find_multi(key, const_iterator(root(), 0))); + } + + // Returns a count of the number of times the key appears in the btree. + size_type count_unique(const key_type& key) const + { + const_iterator begin = internal_find_unique(key, const_iterator(root(), 0)); + + if (!begin.node) + { + // The key doesn't exist in the tree. + return 0; + } + + return 1; + } + // Returns a count of the number of times the key appears in the btree. + size_type count_multi(const key_type& key) const + { + return distance(lower_bound(key), upper_bound(key)); + } + + // Clear the btree, deleting all of the values it contains. + void clear() + { + if (root() != NULL) + { + internal_clear(root()); + } + + *mutable_root() = NULL; + } + + // Swap the contents of *this and x. + void swap(self_type& x) + { + std::swap(static_cast(*this), static_cast(x)); + std::swap(root_, x.root_); + } + + // Assign the contents of x to *this. + self_type& operator=(const self_type& x) + { + if (&x == this) + { + // Don't copy onto ourselves. + return *this; + } + + assign(x); + return *this; + } + + key_compare* mutable_key_comp() + { + return this; + } + const key_compare& key_comp() const + { + return *this; + } + bool compare_keys(const key_type& x, const key_type& y) const + { + return btree_compare_keys(key_comp(), x, y); + } + + // Dump the btree to the specified ostream. Requires that operator<< is + // defined for Key and Value. + void dump(std::ostream& os) const + { + if (root() != NULL) + { + internal_dump(os, root(), 0); + } + } + + // Verifies the structure of the btree. + void verify() const + { + if (root() != NULL) + { + assert(size() == internal_verify(root(), NULL, NULL)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost() == (--const_iterator(root(), root()->count())).node); + assert(leftmost()->leaf()); + assert(rightmost()->leaf()); + } + else + { + assert(size() == 0); + assert(leftmost() == NULL); + assert(rightmost() == NULL); + } + } + + // Size routines. Note that empty() is slightly faster than doing size()==0. + size_type size() const + { + if (empty()) + return 0; + + if (root()->leaf()) + return root()->count(); + + return root()->size(); + } + size_type max_size() const + { + return std::numeric_limits::max(); + } + bool empty() const + { + return root() == NULL; + } + + // The height of the btree. An empty tree will have height 0. + size_type height() const + { + size_type h = 0; + + if (root()) + { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type* n = root(); + + do + { + ++h; + n = n->parent(); + } while (n != root()); + } + + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const + { + return internal_stats(root()).leaf_nodes; + } + size_type internal_nodes() const + { + return internal_stats(root()).internal_nodes; + } + size_type nodes() const + { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const + { + node_stats stats = internal_stats(root()); + + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) + { + return sizeof(*this) + sizeof(base_fields) + root()->max_count() * sizeof(value_type); + } + else + { + return sizeof(*this) + sizeof(root_fields) - sizeof(internal_fields) + + stats.leaf_nodes * sizeof(leaf_fields) + stats.internal_nodes * sizeof(internal_fields); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() + { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return sizeof(leaf_fields) / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + double fullness() const + { + return double(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + double overhead() const + { + if (empty()) + { + return 0.0; + } + + return (bytes_used() - size() * kValueSize) / double(size()); + } + + private: + // Internal accessor routines. + node_type* root() + { + return root_.data; + } + const node_type* root() const + { + return root_.data; + } + node_type** mutable_root() + { + return &root_.data; + } + + // The rightmost node is stored in the root node. + node_type* rightmost() + { + return (!root() || root()->leaf()) ? root() : root()->rightmost(); + } + const node_type* rightmost() const + { + return (!root() || root()->leaf()) ? root() : root()->rightmost(); + } + node_type** mutable_rightmost() + { + return root()->mutable_rightmost(); + } + + // The leftmost node is stored as the parent of the root node. + node_type* leftmost() + { + return root() ? root()->parent() : NULL; + } + const node_type* leftmost() const + { + return root() ? root()->parent() : NULL; + } + + // The size of the tree is stored in the root node. + size_type* mutable_size() + { + return root()->mutable_size(); + } + + // Allocator routines. + internal_allocator_type* mutable_internal_allocator() + { + return static_cast(&root_); + } + const internal_allocator_type& internal_allocator() const + { + return *static_cast(&root_); + } + + // Node creation/deletion routines. + node_type* new_internal_node(node_type* parent) + { + internal_fields* p = + reinterpret_cast(mutable_internal_allocator()->allocate(sizeof(internal_fields))); + return node_type::init_internal(p, parent); + } + node_type* new_internal_root_node() + { + root_fields* p = + reinterpret_cast(mutable_internal_allocator()->allocate(sizeof(root_fields))); + return node_type::init_root(p, root()->parent()); + } + node_type* new_leaf_node(node_type* parent) + { + leaf_fields* p = + reinterpret_cast(mutable_internal_allocator()->allocate(sizeof(leaf_fields))); + return node_type::init_leaf(p, parent, kNodeValues); + } + node_type* new_leaf_root_node(int max_count) + { + leaf_fields* p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(base_fields) + max_count * sizeof(value_type))); + return node_type::init_leaf(p, reinterpret_cast(p), max_count); + } + void delete_internal_node(node_type* node) + { + node->destroy(); + assert(node != root()); + mutable_internal_allocator()->deallocate(reinterpret_cast(node), sizeof(internal_fields)); + } + void delete_internal_root_node() + { + root()->destroy(); + mutable_internal_allocator()->deallocate(reinterpret_cast(root()), sizeof(root_fields)); + } + void delete_leaf_node(node_type* node) + { + node->destroy(); + mutable_internal_allocator()->deallocate(reinterpret_cast(node), + sizeof(base_fields) + node->max_count() * sizeof(value_type)); + } + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator* iter) + { + node_type*& node = iter->node; + int& insert_position = iter->position; + assert(node->count() == node->max_count()); + + // First try to make room on the node by rebalancing. + node_type* parent = node->parent(); + + if (node != root()) + { + if (node->position() > 0) + { + // Try rebalancing with our left sibling. + node_type* left = parent->child(node->position() - 1); + + if (left->count() < left->max_count()) + { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = (left->max_count() - left->count()) / (1 + (insert_position < left->max_count())); + to_move = std::max(1, to_move); + + if (((insert_position - to_move) >= 0) || ((left->count() + to_move) < left->max_count())) + { + left->rebalance_right_to_left(node, to_move); + + assert(node->max_count() - node->count() == to_move); + insert_position = insert_position - to_move; + + if (insert_position < 0) { - iterator prev = position; - - if (position == begin() || !compare_keys(key, (--prev).key())) - { - // prev.key() <= key <= position.key() - return internal_insert(position, v); - } - } - else - { - iterator next = position; - ++next; - - if (next == end() || !compare_keys(next.key(), key)) - { - // position.key() < key <= next.key() - return internal_insert(next, v); - } - } - } - - return insert_multi(v); - } - - // Insert a range of values into the btree. - template - void insert_multi(InputIterator b, InputIterator e) - { - for (; b != e; ++b) - { - insert_multi(end(), *b); - } - } - - void assign(const self_type& x) - { - clear(); - - *mutable_key_comp() = x.key_comp(); - *mutable_internal_allocator() = x.internal_allocator(); - - // Assignment can avoid key comparisons because we know the order of the - // values is the same order we'll store them in. - for (const_iterator iter = x.begin(); iter != x.end(); ++iter) - { - if (empty()) - { - insert_multi(*iter); - } - else - { - // If the btree is not empty, we can just insert the new value at the end - // of the tree! - internal_insert(end(), *iter); - } - } - } - - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - iterator erase(iterator iter) - { - bool internal_delete = false; - - if (!iter.node->leaf()) - { - // Deletion of a value on an internal node. Swap the key with the largest - // value of our left child. This is easy, we just decrement iter. - iterator tmp_iter(iter--); - assert(iter.node->leaf()); - assert(!compare_keys(tmp_iter.key(), iter.key())); - iter.node->value_swap(iter.position, tmp_iter.node, tmp_iter.position); - internal_delete = true; - --*mutable_size(); - } - else if (!root()->leaf()) - { - --*mutable_size(); - } - - // Delete the key from the leaf. - iter.node->remove_value(iter.position); - - // We want to return the next value after the one we just erased. If we - // erased from an internal node (internal_delete == true), then the next - // value is ++(++iter). If we erased from a leaf node (internal_delete == - // false) then the next value is ++iter. Note that ++iter may point to an - // internal node and the value in the internal node may move to a leaf node - // (iter.node) when rebalancing is performed at the leaf level. - - // Merge/rebalance as we walk back up the tree. - iterator res(iter); - - for (;;) - { - if (iter.node == root()) - { - try_shrink(); - - if (empty()) - { - return end(); - } - - break; + insert_position = insert_position + left->count() + 1; + node = left; } - if (iter.node->count() >= kMinNodeValues) - { - break; - } - - bool merged = try_merge_or_rebalance(&iter); - - if (iter.node->leaf()) - { - res = iter; - } - - if (!merged) - { - break; - } - - iter.node = iter.node->parent(); - } - - // Adjust our return value. If we're pointing at the end of a node, advance - // the iterator. - if (res.position == res.node->count()) - { - res.position = res.node->count() - 1; - ++res; - } - - // If we erased from an internal node, advance the iterator. - if (internal_delete) - { - ++res; - } - - return res; - } - - // Erases range. Returns the number of keys erased. - int erase(iterator begin, iterator end) - { - int count = distance(begin, end); - - for (int i = 0; i < count; i++) - { - begin = erase(begin); - } - - return count; - } - - // Erases the specified key from the btree. Returns 1 if an element was - // erased and 0 otherwise. - int erase_unique(const key_type& key) - { - iterator iter = internal_find_unique(key, iterator(root(), 0)); - - if (!iter.node) - { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - - erase(iter); - return 1; - } - - // Erases all of the entries matching the specified key from the - // btree. Returns the number of elements erased. - int erase_multi(const key_type& key) - { - iterator begin = internal_lower_bound(key, iterator(root(), 0)); - - if (!begin.node) - { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - - // Delete all of the keys between begin and upper_bound(key). - iterator end = internal_end( - internal_upper_bound(key, iterator(root(), 0))); - return erase(begin, end); - } - - // Finds the iterator corresponding to a key or returns end() if the key is - // not present. - iterator find_unique(const key_type& key) - { - return internal_end( - internal_find_unique(key, iterator(root(), 0))); - } - const_iterator find_unique(const key_type& key) const - { - return internal_end( - internal_find_unique(key, const_iterator(root(), 0))); - } - iterator find_multi(const key_type& key) - { - return internal_end( - internal_find_multi(key, iterator(root(), 0))); - } - const_iterator find_multi(const key_type& key) const - { - return internal_end( - internal_find_multi(key, const_iterator(root(), 0))); - } - - // Returns a count of the number of times the key appears in the btree. - size_type count_unique(const key_type& key) const - { - const_iterator begin = internal_find_unique( - key, const_iterator(root(), 0)); - - if (!begin.node) - { - // The key doesn't exist in the tree. - return 0; - } - - return 1; - } - // Returns a count of the number of times the key appears in the btree. - size_type count_multi(const key_type& key) const - { - return distance(lower_bound(key), upper_bound(key)); - } - - // Clear the btree, deleting all of the values it contains. - void clear() - { - if (root() != NULL) - { - internal_clear(root()); - } - - *mutable_root() = NULL; - } - - // Swap the contents of *this and x. - void swap(self_type& x) - { - std::swap(static_cast(*this), static_cast(x)); - std::swap(root_, x.root_); - } - - // Assign the contents of x to *this. - self_type& operator=(const self_type& x) - { - if (&x == this) - { - // Don't copy onto ourselves. - return *this; - } - - assign(x); - return *this; - } - - key_compare* mutable_key_comp() - { - return this; - } - const key_compare& key_comp() const - { - return *this; - } - bool compare_keys(const key_type& x, const key_type& y) const - { - return btree_compare_keys(key_comp(), x, y); - } - - // Dump the btree to the specified ostream. Requires that operator<< is - // defined for Key and Value. - void dump(std::ostream& os) const - { - if (root() != NULL) - { - internal_dump(os, root(), 0); - } - } - - // Verifies the structure of the btree. - void verify() const - { - if (root() != NULL) - { - assert(size() == internal_verify(root(), NULL, NULL)); - assert(leftmost() == (++const_iterator(root(), -1)).node); - assert(rightmost() == (--const_iterator(root(), root()->count())).node); - assert(leftmost()->leaf()); - assert(rightmost()->leaf()); - } - else - { - assert(size() == 0); - assert(leftmost() == NULL); - assert(rightmost() == NULL); - } - } - - // Size routines. Note that empty() is slightly faster than doing size()==0. - size_type size() const - { - if (empty()) return 0; - - if (root()->leaf()) return root()->count(); - - return root()->size(); - } - size_type max_size() const - { - return std::numeric_limits::max(); - } - bool empty() const - { - return root() == NULL; - } - - // The height of the btree. An empty tree will have height 0. - size_type height() const - { - size_type h = 0; - - if (root()) - { - // Count the length of the chain from the leftmost node up to the - // root. We actually count from the root back around to the level below - // the root, but the calculation is the same because of the circularity - // of that traversal. - const node_type* n = root(); - - do - { - ++h; - n = n->parent(); - } - while (n != root()); - } - - return h; - } - - // The number of internal, leaf and total nodes used by the btree. - size_type leaf_nodes() const - { - return internal_stats(root()).leaf_nodes; - } - size_type internal_nodes() const - { - return internal_stats(root()).internal_nodes; - } - size_type nodes() const - { - node_stats stats = internal_stats(root()); - return stats.leaf_nodes + stats.internal_nodes; - } - - // The total number of bytes used by the btree. - size_type bytes_used() const - { - node_stats stats = internal_stats(root()); - - if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) - { - return sizeof(*this) + - sizeof(base_fields) + root()->max_count() * sizeof(value_type); - } - else - { - return sizeof(*this) + - sizeof(root_fields) - sizeof(internal_fields) + - stats.leaf_nodes * sizeof(leaf_fields) + - stats.internal_nodes * sizeof(internal_fields); - } - } - - // The average number of bytes used per value stored in the btree. - static double average_bytes_per_value() - { - // Returns the number of bytes per value on a leaf node that is 75% - // full. Experimentally, this matches up nicely with the computed number of - // bytes per value in trees that had their values inserted in random order. - return sizeof(leaf_fields) / (kNodeValues * 0.75); - } - - // The fullness of the btree. Computed as the number of elements in the btree - // divided by the maximum number of elements a tree with the current number - // of nodes could hold. A value of 1 indicates perfect space - // utilization. Smaller values indicate space wastage. - double fullness() const - { - return double(size()) / (nodes() * kNodeValues); - } - // The overhead of the btree structure in bytes per node. Computed as the - // total number of bytes used by the btree minus the number of bytes used for - // storing elements divided by the number of elements. - double overhead() const - { - if (empty()) - { - return 0.0; - } - - return (bytes_used() - size() * kValueSize) / double(size()); - } - -private: - // Internal accessor routines. - node_type* root() - { - return root_.data; - } - const node_type* root() const - { - return root_.data; - } - node_type** mutable_root() - { - return &root_.data; - } - - // The rightmost node is stored in the root node. - node_type* rightmost() - { - return (!root() || root()->leaf()) ? root() : root()->rightmost(); - } - const node_type* rightmost() const - { - return (!root() || root()->leaf()) ? root() : root()->rightmost(); - } - node_type** mutable_rightmost() - { - return root()->mutable_rightmost(); - } - - // The leftmost node is stored as the parent of the root node. - node_type* leftmost() - { - return root() ? root()->parent() : NULL; - } - const node_type* leftmost() const - { - return root() ? root()->parent() : NULL; - } - - // The size of the tree is stored in the root node. - size_type* mutable_size() - { - return root()->mutable_size(); - } - - // Allocator routines. - internal_allocator_type* mutable_internal_allocator() - { - return static_cast(&root_); - } - const internal_allocator_type& internal_allocator() const - { - return *static_cast(&root_); - } - - // Node creation/deletion routines. - node_type* new_internal_node(node_type* parent) - { - internal_fields* p = reinterpret_cast( - mutable_internal_allocator()->allocate(sizeof(internal_fields))); - return node_type::init_internal(p, parent); - } - node_type* new_internal_root_node() - { - root_fields* p = reinterpret_cast( - mutable_internal_allocator()->allocate(sizeof(root_fields))); - return node_type::init_root(p, root()->parent()); - } - node_type* new_leaf_node(node_type* parent) - { - leaf_fields* p = reinterpret_cast( - mutable_internal_allocator()->allocate(sizeof(leaf_fields))); - return node_type::init_leaf(p, parent, kNodeValues); - } - node_type* new_leaf_root_node(int max_count) - { - leaf_fields* p = reinterpret_cast( - mutable_internal_allocator()->allocate( - sizeof(base_fields) + max_count * sizeof(value_type))); - return node_type::init_leaf(p, reinterpret_cast(p), max_count); - } - void delete_internal_node(node_type* node) - { - node->destroy(); - assert(node != root()); - mutable_internal_allocator()->deallocate( - reinterpret_cast(node), sizeof(internal_fields)); - } - void delete_internal_root_node() - { - root()->destroy(); - mutable_internal_allocator()->deallocate( - reinterpret_cast(root()), sizeof(root_fields)); - } - void delete_leaf_node(node_type* node) - { - node->destroy(); - mutable_internal_allocator()->deallocate( - reinterpret_cast(node), - sizeof(base_fields) + node->max_count() * sizeof(value_type)); - } - - // Rebalances or splits the node iter points to. - void rebalance_or_split(iterator* iter) - { - node_type*& node = iter->node; - int& insert_position = iter->position; - assert(node->count() == node->max_count()); - - // First try to make room on the node by rebalancing. - node_type* parent = node->parent(); - - if (node != root()) - { - if (node->position() > 0) - { - // Try rebalancing with our left sibling. - node_type* left = parent->child(node->position() - 1); - - if (left->count() < left->max_count()) - { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the end of the right node then we bias rebalancing to - // fill up the left node. - int to_move = (left->max_count() - left->count()) / - (1 + (insert_position < left->max_count())); - to_move = std::max(1, to_move); - - if (((insert_position - to_move) >= 0) || - ((left->count() + to_move) < left->max_count())) - { - left->rebalance_right_to_left(node, to_move); - - assert(node->max_count() - node->count() == to_move); - insert_position = insert_position - to_move; - - if (insert_position < 0) - { - insert_position = insert_position + left->count() + 1; - node = left; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - if (node->position() < parent->count()) - { - // Try rebalancing with our right sibling. - node_type* right = parent->child(node->position() + 1); - - if (right->count() < right->max_count()) - { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the beginning of the left node then we bias rebalancing - // to fill up the right node. - int to_move = (right->max_count() - right->count()) / - (1 + (insert_position > 0)); - to_move = std::max(1, to_move); - - if ((insert_position <= (node->count() - to_move)) || - ((right->count() + to_move) < right->max_count())) - { - node->rebalance_left_to_right(right, to_move); - - if (insert_position > node->count()) - { - insert_position = insert_position - node->count() - 1; - node = right; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - // Rebalancing failed, make sure there is room on the parent node for a new - // value. - if (parent->count() == parent->max_count()) - { - iterator parent_iter(node->parent(), node->position()); - rebalance_or_split(&parent_iter); - } - } - else - { - // Rebalancing not possible because this is the root node. - if (root()->leaf()) - { - // The root node is currently a leaf node: create a new root node and set - // the current root node as the child of the new root. - parent = new_internal_root_node(); - parent->set_child(0, root()); - *mutable_root() = parent; - assert(*mutable_rightmost() == parent->child(0)); - } - else - { - // The root node is an internal node. We do not want to create a new root - // node because the root node is special and holds the size of the tree - // and a pointer to the rightmost node. So we create a new internal node - // and move all of the items on the current root into the new node. - parent = new_internal_node(parent); - parent->set_child(0, parent); - parent->swap(root()); - node = parent; - } - } - - // Split the node. - node_type* split_node; - - if (node->leaf()) - { - split_node = new_leaf_node(parent); - node->split(split_node, insert_position); - - if (rightmost() == node) - { - *mutable_rightmost() = split_node; - } - } - else - { - split_node = new_internal_node(parent); - node->split(split_node, insert_position); - } - - if (insert_position > node->count()) - { - insert_position = insert_position - node->count() - 1; - node = split_node; - } - } - - // Merges the values of left, right and the delimiting key on their parent - // onto left, removing the delimiting key and deleting right. - void merge_nodes(node_type* left, node_type* right) - { - left->merge(right); - - if (right->leaf()) - { - if (rightmost() == right) - { - *mutable_rightmost() = left; - } - - delete_leaf_node(right); - } - else - { - delete_internal_node(right); - } - } - - // Tries to merge node with its left or right sibling, and failing that, - // rebalance with its left or right sibling. Returns true if a merge - // occurred, at which point it is no longer valid to access node. Returns - // false if no merging took place. - bool try_merge_or_rebalance(iterator* iter) - { - node_type* parent = iter->node->parent(); - - if (iter->node->position() > 0) - { - // Try merging with our left sibling. - node_type* left = parent->child(iter->node->position() - 1); - - if ((1 + left->count() + iter->node->count()) <= left->max_count()) - { - iter->position += 1 + left->count(); - merge_nodes(left, iter->node); - iter->node = left; - return true; - } - } - - if (iter->node->position() < parent->count()) - { - // Try merging with our right sibling. - node_type* right = parent->child(iter->node->position() + 1); - - if ((1 + iter->node->count() + right->count()) <= right->max_count()) - { - merge_nodes(iter->node, right); - return true; - } - - // Try rebalancing with our right sibling. We don't perform rebalancing if - // we deleted the first element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the front of the tree. - if ((right->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position > 0))) - { - int to_move = (right->count() - iter->node->count()) / 2; - to_move = std::min(to_move, right->count() - 1); - iter->node->rebalance_right_to_left(right, to_move); - return false; - } - } - - if (iter->node->position() > 0) - { - // Try rebalancing with our left sibling. We don't perform rebalancing if - // we deleted the last element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the back of the tree. - node_type* left = parent->child(iter->node->position() - 1); - - if ((left->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position < iter->node->count()))) - { - int to_move = (left->count() - iter->node->count()) / 2; - to_move = std::min(to_move, left->count() - 1); - left->rebalance_left_to_right(iter->node, to_move); - iter->position += to_move; - return false; - } - } - - return false; - } - - // Tries to shrink the height of the tree by 1. - void try_shrink() - { - if (root()->count() > 0) - { + assert(node->count() < node->max_count()); return; + } } + } - // Deleted the last item on the root node, shrink the height of the tree. - if (root()->leaf()) - { - assert(size() == 0); - delete_leaf_node(root()); - *mutable_root() = NULL; - } - else - { - node_type* child = root()->child(0); + if (node->position() < parent->count()) + { + // Try rebalancing with our right sibling. + node_type* right = parent->child(node->position() + 1); - if (child->leaf()) + if (right->count() < right->max_count()) + { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = (right->max_count() - right->count()) / (1 + (insert_position > 0)); + to_move = std::max(1, to_move); + + if ((insert_position <= (node->count() - to_move)) || + ((right->count() + to_move) < right->max_count())) + { + node->rebalance_left_to_right(right, to_move); + + if (insert_position > node->count()) { - // The child is a leaf node so simply make it the root node in the tree. - child->make_root(); - delete_internal_root_node(); - *mutable_root() = child; - } - else - { - // The child is an internal node. We want to keep the existing root node - // so we move all of the values from the child node into the existing - // (empty) root node. - child->swap(root()); - delete_internal_node(child); + insert_position = insert_position - node->count() - 1; + node = right; } + + assert(node->count() < node->max_count()); + return; + } } + } + + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + if (parent->count() == parent->max_count()) + { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } } - - iterator internal_end(iterator iter) + else { - return iter.node ? iter : end(); + // Rebalancing not possible because this is the root node. + if (root()->leaf()) + { + // The root node is currently a leaf node: create a new root node and set + // the current root node as the child of the new root. + parent = new_internal_root_node(); + parent->set_child(0, root()); + *mutable_root() = parent; + assert(*mutable_rightmost() == parent->child(0)); + } + else + { + // The root node is an internal node. We do not want to create a new root + // node because the root node is special and holds the size of the tree + // and a pointer to the rightmost node. So we create a new internal node + // and move all of the items on the current root into the new node. + parent = new_internal_node(parent); + parent->set_child(0, parent); + parent->swap(root()); + node = parent; + } } - const_iterator internal_end(const_iterator iter) const + + // Split the node. + node_type* split_node; + + if (node->leaf()) { - return iter.node ? iter : end(); - } + split_node = new_leaf_node(parent); + node->split(split_node, insert_position); - // Inserts a value into the btree immediately before iter. Requires that - // key(v) <= iter.key() and (--iter).key() <= key(v). - iterator internal_insert(iterator iter, const value_type& v) + if (rightmost() == node) + { + *mutable_rightmost() = split_node; + } + } + else { - if (!iter.node->leaf()) - { - // We can't insert on an internal node. Instead, we'll insert after the - // previous value which is guaranteed to be on a leaf node. - --iter; - ++iter.position; - } - - if (iter.node->count() == iter.node->max_count()) - { - // Make room in the leaf for the new item. - if (iter.node->max_count() < kNodeValues) - { - // Insertion into the root where the root is smaller that the full node - // size. Simply grow the size of the root node. - assert(iter.node == root()); - iter.node = new_leaf_root_node( - std::min(kNodeValues, 2 * iter.node->max_count())); - iter.node->swap(root()); - delete_leaf_node(root()); - *mutable_root() = iter.node; - } - else - { - rebalance_or_split(&iter); - ++*mutable_size(); - } - } - else if (!root()->leaf()) - { - ++*mutable_size(); - } - - iter.node->insert_value(iter.position, v); - return iter; + split_node = new_internal_node(parent); + node->split(split_node, insert_position); } - // Returns an iterator pointing to the first value >= the value "iter" is - // pointing at. Note that "iter" might be pointing to an invalid location as - // iter.position == iter.node->count(). This routine simply moves iter up in - // the tree to a valid location. - template - static IterType internal_last(IterType iter) + if (insert_position > node->count()) { - while (iter.node && iter.position == iter.node->count()) - { - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - - if (iter.node->leaf()) - { - iter.node = NULL; - } - } - - return iter; + insert_position = insert_position - node->count() - 1; + node = split_node; } + } - // Returns an iterator pointing to the leaf position at which key would - // reside in the tree. We provide 2 versions of internal_locate. The first - // version (internal_locate_plain_compare) always returns 0 for the second - // field of the pair. The second version (internal_locate_compare_to) is for - // the key-compare-to specialization and returns either kExactMatch (if the - // key was found in the tree) or -kExactMatch (if it wasn't) in the second - // field of the pair. The compare_to specialization allows the caller to - // avoid a subsequent comparison to determine if an exact match was made, - // speeding up string keys. - template - std::pair internal_locate( - const key_type& key, IterType iter) const + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type* left, node_type* right) + { + left->merge(right); + + if (right->leaf()) { - return internal_locate_type::dispatch(key, *this, iter); - } + if (rightmost() == right) + { + *mutable_rightmost() = left; + } - template - std::pair internal_locate_plain_compare( - const key_type& key, IterType iter) const + delete_leaf_node(right); + } + else { - for (;;) - { - iter.position = iter.node->lower_bound(key, key_comp()); - - if (iter.node->leaf()) - { - break; - } - - iter.node = iter.node->child(iter.position); - } - - return std::make_pair(iter, 0); + delete_internal_node(right); } + } - template - std::pair internal_locate_compare_to( - const key_type& key, IterType iter) const + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator* iter) + { + node_type* parent = iter->node->parent(); + + if (iter->node->position() > 0) { - for (;;) - { - int res = iter.node->lower_bound(key, key_comp()); - iter.position = res & kMatchMask; + // Try merging with our left sibling. + node_type* left = parent->child(iter->node->position() - 1); - if (res & kExactMatch) - { - return std::make_pair(iter, static_cast(kExactMatch)); - } - - if (iter.node->leaf()) - { - break; - } - - iter.node = iter.node->child(iter.position); - } - - return std::make_pair(iter, -kExactMatch); + if ((1 + left->count() + iter->node->count()) <= left->max_count()) + { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } } - // Internal routine which implements lower_bound(). - template - IterType internal_lower_bound( - const key_type& key, IterType iter) const + if (iter->node->position() < parent->count()) { - if (iter.node) - { - for (;;) - { - iter.position = - iter.node->lower_bound(key, key_comp()) & kMatchMask; + // Try merging with our right sibling. + node_type* right = parent->child(iter->node->position() + 1); - if (iter.node->leaf()) - { - break; - } + if ((1 + iter->node->count() + right->count()) <= right->max_count()) + { + merge_nodes(iter->node, right); + return true; + } - iter.node = iter.node->child(iter.position); - } - - iter = internal_last(iter); - } - - return iter; + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if ((right->count() > kMinNodeValues) && ((iter->node->count() == 0) || (iter->position > 0))) + { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = std::min(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(right, to_move); + return false; + } } - // Internal routine which implements upper_bound(). - template - IterType internal_upper_bound( - const key_type& key, IterType iter) const + if (iter->node->position() > 0) { - if (iter.node) - { - for (;;) - { - iter.position = iter.node->upper_bound(key, key_comp()); + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type* left = parent->child(iter->node->position() - 1); - if (iter.node->leaf()) - { - break; - } - - iter.node = iter.node->child(iter.position); - } - - iter = internal_last(iter); - } - - return iter; + if ((left->count() > kMinNodeValues) && + ((iter->node->count() == 0) || (iter->position < iter->node->count()))) + { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = std::min(to_move, left->count() - 1); + left->rebalance_left_to_right(iter->node, to_move); + iter->position += to_move; + return false; + } } - // Internal routine which implements find_unique(). - template - IterType internal_find_unique( - const key_type& key, IterType iter) const + return false; + } + + // Tries to shrink the height of the tree by 1. + void try_shrink() + { + if (root()->count() > 0) { - if (iter.node) - { - std::pair res = internal_locate(key, iter); - - if (res.second == kExactMatch) - { - return res.first; - } - - if (!res.second) - { - iter = internal_last(res.first); - - if (iter.node && !compare_keys(key, iter.key())) - { - return iter; - } - } - } - - return IterType(NULL, 0); + return; } - // Internal routine which implements find_multi(). - template - IterType internal_find_multi( - const key_type& key, IterType iter) const + // Deleted the last item on the root node, shrink the height of the tree. + if (root()->leaf()) { - if (iter.node) - { - iter = internal_lower_bound(key, iter); - - if (iter.node) - { - iter = internal_last(iter); - - if (iter.node && !compare_keys(key, iter.key())) - { - return iter; - } - } - } - - return IterType(NULL, 0); + assert(size() == 0); + delete_leaf_node(root()); + *mutable_root() = NULL; } - - // Deletes a node and all of its children. - void internal_clear(node_type* node) + else { - if (!node->leaf()) - { - for (int i = 0; i <= node->count(); ++i) - { - internal_clear(node->child(i)); - } + node_type* child = root()->child(0); - if (node == root()) - { - delete_internal_root_node(); - } - else - { - delete_internal_node(node); - } - } - else - { - delete_leaf_node(node); - } + if (child->leaf()) + { + // The child is a leaf node so simply make it the root node in the tree. + child->make_root(); + delete_internal_root_node(); + *mutable_root() = child; + } + else + { + // The child is an internal node. We want to keep the existing root node + // so we move all of the values from the child node into the existing + // (empty) root node. + child->swap(root()); + delete_internal_node(child); + } } + } - // Dumps a node and all of its children to the specified ostream. - void internal_dump(std::ostream& os, const node_type* node, int level) const + iterator internal_end(iterator iter) + { + return iter.node ? iter : end(); + } + const_iterator internal_end(const_iterator iter) const + { + return iter.node ? iter : end(); + } + + // Inserts a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + iterator internal_insert(iterator iter, const value_type& v) + { + if (!iter.node->leaf()) { - for (int i = 0; i < node->count(); ++i) - { - if (!node->leaf()) - { - internal_dump(os, node->child(i), level + 1); - } - - for (int j = 0; j < level; ++j) - { - os << " "; - } - - os << node->key(i) << " [" << level << "]\n"; - } - - if (!node->leaf()) - { - internal_dump(os, node->child(node->count()), level + 1); - } + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; } - // Verifies the tree structure of node. - int internal_verify(const node_type* node, - const key_type* lo, const key_type* hi) const + if (iter.node->count() == iter.node->max_count()) { - assert(node->count() > 0); - assert(node->count() <= node->max_count()); - - if (lo) - { - assert(!compare_keys(node->key(0), *lo)); - } - - if (hi) - { - assert(!compare_keys(*hi, node->key(node->count() - 1))); - } - - for (int i = 1; i < node->count(); ++i) - { - assert(!compare_keys(node->key(i), node->key(i - 1))); - } - - int count = node->count(); - - if (!node->leaf()) - { - for (int i = 0; i <= node->count(); ++i) - { - assert(node->child(i) != NULL); - assert(node->child(i)->parent() == node); - assert(node->child(i)->position() == i); - count += internal_verify( - node->child(i), - (i == 0) ? lo : &node->key(i - 1), - (i == node->count()) ? hi : &node->key(i)); - } - } - - return count; + // Make room in the leaf for the new item. + if (iter.node->max_count() < kNodeValues) + { + // Insertion into the root where the root is smaller that the full node + // size. Simply grow the size of the root node. + assert(iter.node == root()); + iter.node = new_leaf_root_node(std::min(kNodeValues, 2 * iter.node->max_count())); + iter.node->swap(root()); + delete_leaf_node(root()); + *mutable_root() = iter.node; + } + else + { + rebalance_or_split(&iter); + ++*mutable_size(); + } } - - node_stats internal_stats(const node_type* node) const + else if (!root()->leaf()) { - if (!node) - { - return node_stats(0, 0); - } - - if (node->leaf()) - { - return node_stats(1, 0); - } - - node_stats res(0, 1); - - for (int i = 0; i <= node->count(); ++i) - { - res += internal_stats(node->child(i)); - } - - return res; + ++*mutable_size(); } -private: - empty_base_handle root_; + iter.node->insert_value(iter.position, v); + return iter; + } -private: - // A never instantiated helper function that returns big_ if we have a - // key-compare-to functor or if R is bool and small_ otherwise. - template - static typename if_ < - if_, - std::is_same >::type::value, - big_, small_ >::type key_compare_checker(R); + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location as + // iter.position == iter.node->count(). This routine simply moves iter up in + // the tree to a valid location. + template + static IterType internal_last(IterType iter) + { + while (iter.node && iter.position == iter.node->count()) + { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); - // A never instantiated helper function that returns the key comparison - // functor. - static key_compare key_compare_helper(); + if (iter.node->leaf()) + { + iter.node = NULL; + } + } - // Verify that key_compare returns a bool. This is similar to the way - // is_convertible in base/type_traits.h works. Note that key_compare_checker - // is never actually invoked. The compiler will select which - // key_compare_checker() to instantiate and then figure out the size of the - // return type of key_compare_checker() at compile time which we then check - // against the sizeof of big_. - COMPILE_ASSERT( - sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == - sizeof(big_), - key_comparison_function_must_return_bool); + return iter; + } - // Note: We insist on kTargetValues, which is computed from - // Params::kTargetNodeSize, must fit the base_fields::field_type. - COMPILE_ASSERT(kNodeValues < - (1 << (8 * sizeof(typename base_fields::field_type))), - target_node_size_too_large); + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree. We provide 2 versions of internal_locate. The first + // version (internal_locate_plain_compare) always returns 0 for the second + // field of the pair. The second version (internal_locate_compare_to) is for + // the key-compare-to specialization and returns either kExactMatch (if the + // key was found in the tree) or -kExactMatch (if it wasn't) in the second + // field of the pair. The compare_to specialization allows the caller to + // avoid a subsequent comparison to determine if an exact match was made, + // speeding up string keys. + template + std::pair internal_locate(const key_type& key, IterType iter) const + { + return internal_locate_type::dispatch(key, *this, iter); + } - // Test the assumption made in setting kNodeValueSpace. - COMPILE_ASSERT(sizeof(base_fields) >= 2 * sizeof(void*), - node_space_assumption_incorrect); + template + std::pair internal_locate_plain_compare(const key_type& key, IterType iter) const + { + for (;;) + { + iter.position = iter.node->lower_bound(key, key_comp()); + + if (iter.node->leaf()) + { + break; + } + + iter.node = iter.node->child(iter.position); + } + + return std::make_pair(iter, 0); + } + + template + std::pair internal_locate_compare_to(const key_type& key, IterType iter) const + { + for (;;) + { + int res = iter.node->lower_bound(key, key_comp()); + iter.position = res & kMatchMask; + + if (res & kExactMatch) + { + return std::make_pair(iter, static_cast(kExactMatch)); + } + + if (iter.node->leaf()) + { + break; + } + + iter.node = iter.node->child(iter.position); + } + + return std::make_pair(iter, -kExactMatch); + } + + // Internal routine which implements lower_bound(). + template + IterType internal_lower_bound(const key_type& key, IterType iter) const + { + if (iter.node) + { + for (;;) + { + iter.position = iter.node->lower_bound(key, key_comp()) & kMatchMask; + + if (iter.node->leaf()) + { + break; + } + + iter.node = iter.node->child(iter.position); + } + + iter = internal_last(iter); + } + + return iter; + } + + // Internal routine which implements upper_bound(). + template + IterType internal_upper_bound(const key_type& key, IterType iter) const + { + if (iter.node) + { + for (;;) + { + iter.position = iter.node->upper_bound(key, key_comp()); + + if (iter.node->leaf()) + { + break; + } + + iter.node = iter.node->child(iter.position); + } + + iter = internal_last(iter); + } + + return iter; + } + + // Internal routine which implements find_unique(). + template + IterType internal_find_unique(const key_type& key, IterType iter) const + { + if (iter.node) + { + std::pair res = internal_locate(key, iter); + + if (res.second == kExactMatch) + { + return res.first; + } + + if (!res.second) + { + iter = internal_last(res.first); + + if (iter.node && !compare_keys(key, iter.key())) + { + return iter; + } + } + } + + return IterType(NULL, 0); + } + + // Internal routine which implements find_multi(). + template + IterType internal_find_multi(const key_type& key, IterType iter) const + { + if (iter.node) + { + iter = internal_lower_bound(key, iter); + + if (iter.node) + { + iter = internal_last(iter); + + if (iter.node && !compare_keys(key, iter.key())) + { + return iter; + } + } + } + + return IterType(NULL, 0); + } + + // Deletes a node and all of its children. + void internal_clear(node_type* node) + { + if (!node->leaf()) + { + for (int i = 0; i <= node->count(); ++i) + { + internal_clear(node->child(i)); + } + + if (node == root()) + { + delete_internal_root_node(); + } + else + { + delete_internal_node(node); + } + } + else + { + delete_leaf_node(node); + } + } + + // Dumps a node and all of its children to the specified ostream. + void internal_dump(std::ostream& os, const node_type* node, int level) const + { + for (int i = 0; i < node->count(); ++i) + { + if (!node->leaf()) + { + internal_dump(os, node->child(i), level + 1); + } + + for (int j = 0; j < level; ++j) + { + os << " "; + } + + os << node->key(i) << " [" << level << "]\n"; + } + + if (!node->leaf()) + { + internal_dump(os, node->child(node->count()), level + 1); + } + } + + // Verifies the tree structure of node. + int internal_verify(const node_type* node, const key_type* lo, const key_type* hi) const + { + assert(node->count() > 0); + assert(node->count() <= node->max_count()); + + if (lo) + { + assert(!compare_keys(node->key(0), *lo)); + } + + if (hi) + { + assert(!compare_keys(*hi, node->key(node->count() - 1))); + } + + for (int i = 1; i < node->count(); ++i) + { + assert(!compare_keys(node->key(i), node->key(i - 1))); + } + + int count = node->count(); + + if (!node->leaf()) + { + for (int i = 0; i <= node->count(); ++i) + { + assert(node->child(i) != NULL); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); + count += internal_verify(node->child(i), (i == 0) ? lo : &node->key(i - 1), + (i == node->count()) ? hi : &node->key(i)); + } + } + + return count; + } + + node_stats internal_stats(const node_type* node) const + { + if (!node) + { + return node_stats(0, 0); + } + + if (node->leaf()) + { + return node_stats(1, 0); + } + + node_stats res(0, 1); + + for (int i = 0; i <= node->count(); ++i) + { + res += internal_stats(node->child(i)); + } + + return res; + } + + private: + empty_base_handle root_; + + private: + // A never instantiated helper function that returns big_ if we have a + // key-compare-to functor or if R is bool and small_ otherwise. + template + static + typename if_, std::is_same >::type::value, + big_, small_>::type key_compare_checker(R); + + // A never instantiated helper function that returns the key comparison + // functor. + static key_compare key_compare_helper(); + + // Verify that key_compare returns a bool. This is similar to the way + // is_convertible in base/type_traits.h works. Note that key_compare_checker + // is never actually invoked. The compiler will select which + // key_compare_checker() to instantiate and then figure out the size of the + // return type of key_compare_checker() at compile time which we then check + // against the sizeof of big_. + COMPILE_ASSERT(sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == sizeof(big_), + key_comparison_function_must_return_bool); + + // Note: We insist on kTargetValues, which is computed from + // Params::kTargetNodeSize, must fit the base_fields::field_type. + COMPILE_ASSERT(kNodeValues < (1 << (8 * sizeof(typename base_fields::field_type))), + target_node_size_too_large); + + // Test the assumption made in setting kNodeValueSpace. + COMPILE_ASSERT(sizeof(base_fields) >= 2 * sizeof(void*), node_space_assumption_incorrect); }; //// @@ -2507,309 +2442,309 @@ private: template inline void btree_node

::insert_value(int i, const value_type& x) { - assert(i <= count()); - value_init(count(), x); + assert(i <= count()); + value_init(count(), x); + + for (int j = count(); j > i; --j) + { + value_swap(j, this, j - 1); + } + + set_count(count() + 1); + + if (!leaf()) + { + ++i; for (int j = count(); j > i; --j) { - value_swap(j, this, j - 1); + *mutable_child(j) = child(j - 1); + child(j)->set_position(j); } - set_count(count() + 1); - - if (!leaf()) - { - ++i; - - for (int j = count(); j > i; --j) - { - *mutable_child(j) = child(j - 1); - child(j)->set_position(j); - } - - *mutable_child(i) = NULL; - } + *mutable_child(i) = NULL; + } } template inline void btree_node

::remove_value(int i) { - if (!leaf()) + if (!leaf()) + { + assert(child(i + 1)->count() == 0); + + for (int j = i + 1; j < count(); ++j) { - assert(child(i + 1)->count() == 0); - - for (int j = i + 1; j < count(); ++j) - { - *mutable_child(j) = child(j + 1); - child(j)->set_position(j); - } - - *mutable_child(count()) = NULL; + *mutable_child(j) = child(j + 1); + child(j)->set_position(j); } - set_count(count() - 1); + *mutable_child(count()) = NULL; + } - for (; i < count(); ++i) - { - value_swap(i, this, i + 1); - } + set_count(count() - 1); - value_destroy(i); + for (; i < count(); ++i) + { + value_swap(i, this, i + 1); + } + + value_destroy(i); } template void btree_node

::rebalance_right_to_left(btree_node* src, int to_move) { - assert(parent() == src->parent()); - assert(position() + 1 == src->position()); - assert(src->count() >= count()); - assert(to_move >= 1); - assert(to_move <= src->count()); + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); + assert(src->count() >= count()); + assert(to_move >= 1); + assert(to_move <= src->count()); - // Make room in the left node for the new values. + // Make room in the left node for the new values. + for (int i = 0; i < to_move; ++i) + { + value_init(i + count()); + } + + // Move the delimiting value to the left node and the new delimiting value + // from the right node. + value_swap(count(), parent(), position()); + parent()->value_swap(position(), src, to_move - 1); + + // Move the values from the right to the left node. + for (int i = 1; i < to_move; ++i) + { + value_swap(count() + i, src, i - 1); + } + + // Shift the values in the right node to their correct position. + for (int i = to_move; i < src->count(); ++i) + { + src->value_swap(i - to_move, src, i); + } + + for (int i = 1; i <= to_move; ++i) + { + src->value_destroy(src->count() - i); + } + + if (!leaf()) + { + // Move the child pointers from the right to the left node. for (int i = 0; i < to_move; ++i) { - value_init(i + count()); + set_child(1 + count() + i, src->child(i)); } - // Move the delimiting value to the left node and the new delimiting value - // from the right node. - value_swap(count(), parent(), position()); - parent()->value_swap(position(), src, to_move - 1); - - // Move the values from the right to the left node. - for (int i = 1; i < to_move; ++i) + for (int i = 0; i <= src->count() - to_move; ++i) { - value_swap(count() + i, src, i - 1); + assert(i + to_move <= src->max_count()); + src->set_child(i, src->child(i + to_move)); + *src->mutable_child(i + to_move) = NULL; } + } - // Shift the values in the right node to their correct position. - for (int i = to_move; i < src->count(); ++i) - { - src->value_swap(i - to_move, src, i); - } - - for (int i = 1; i <= to_move; ++i) - { - src->value_destroy(src->count() - i); - } - - if (!leaf()) - { - // Move the child pointers from the right to the left node. - for (int i = 0; i < to_move; ++i) - { - set_child(1 + count() + i, src->child(i)); - } - - for (int i = 0; i <= src->count() - to_move; ++i) - { - assert(i + to_move <= src->max_count()); - src->set_child(i, src->child(i + to_move)); - *src->mutable_child(i + to_move) = NULL; - } - } - - // Fixup the counts on the src and dest nodes. - set_count(count() + to_move); - src->set_count(src->count() - to_move); + // Fixup the counts on the src and dest nodes. + set_count(count() + to_move); + src->set_count(src->count() - to_move); } template void btree_node

::rebalance_left_to_right(btree_node* dest, int to_move) { - assert(parent() == dest->parent()); - assert(position() + 1 == dest->position()); - assert(count() >= dest->count()); - assert(to_move >= 1); - assert(to_move <= count()); + assert(parent() == dest->parent()); + assert(position() + 1 == dest->position()); + assert(count() >= dest->count()); + assert(to_move >= 1); + assert(to_move <= count()); - // Make room in the right node for the new values. - for (int i = 0; i < to_move; ++i) + // Make room in the right node for the new values. + for (int i = 0; i < to_move; ++i) + { + dest->value_init(i + dest->count()); + } + + for (int i = dest->count() - 1; i >= 0; --i) + { + dest->value_swap(i, dest, i + to_move); + } + + // Move the delimiting value to the right node and the new delimiting value + // from the left node. + dest->value_swap(to_move - 1, parent(), position()); + parent()->value_swap(position(), this, count() - to_move); + value_destroy(count() - to_move); + + // Move the values from the left to the right node. + for (int i = 1; i < to_move; ++i) + { + value_swap(count() - to_move + i, dest, i - 1); + value_destroy(count() - to_move + i); + } + + if (!leaf()) + { + // Move the child pointers from the left to the right node. + for (int i = dest->count(); i >= 0; --i) { - dest->value_init(i + dest->count()); + dest->set_child(i + to_move, dest->child(i)); + *dest->mutable_child(i) = NULL; } - for (int i = dest->count() - 1; i >= 0; --i) + for (int i = 1; i <= to_move; ++i) { - dest->value_swap(i, dest, i + to_move); + dest->set_child(i - 1, child(count() - to_move + i)); + *mutable_child(count() - to_move + i) = NULL; } + } - // Move the delimiting value to the right node and the new delimiting value - // from the left node. - dest->value_swap(to_move - 1, parent(), position()); - parent()->value_swap(position(), this, count() - to_move); - value_destroy(count() - to_move); - - // Move the values from the left to the right node. - for (int i = 1; i < to_move; ++i) - { - value_swap(count() - to_move + i, dest, i - 1); - value_destroy(count() - to_move + i); - } - - if (!leaf()) - { - // Move the child pointers from the left to the right node. - for (int i = dest->count(); i >= 0; --i) - { - dest->set_child(i + to_move, dest->child(i)); - *dest->mutable_child(i) = NULL; - } - - for (int i = 1; i <= to_move; ++i) - { - dest->set_child(i - 1, child(count() - to_move + i)); - *mutable_child(count() - to_move + i) = NULL; - } - } - - // Fixup the counts on the src and dest nodes. - set_count(count() - to_move); - dest->set_count(dest->count() + to_move); + // Fixup the counts on the src and dest nodes. + set_count(count() - to_move); + dest->set_count(dest->count() + to_move); } template void btree_node

::split(btree_node* dest, int insert_position) { - assert(dest->count() == 0); + assert(dest->count() == 0); - // We bias the split based on the position being inserted. If we're - // inserting at the beginning of the left node then bias the split to put - // more values on the right node. If we're inserting at the end of the - // right node then bias the split to put more values on the left node. - if (insert_position == 0) - { - dest->set_count(count() - 1); - } - else if (insert_position == max_count()) - { - dest->set_count(0); - } - else - { - dest->set_count(count() / 2); - } + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == 0) + { + dest->set_count(count() - 1); + } + else if (insert_position == max_count()) + { + dest->set_count(0); + } + else + { + dest->set_count(count() / 2); + } - set_count(count() - dest->count()); - assert(count() >= 1); + set_count(count() - dest->count()); + assert(count() >= 1); - // Move values from the left sibling to the right sibling. - for (int i = 0; i < dest->count(); ++i) + // Move values from the left sibling to the right sibling. + for (int i = 0; i < dest->count(); ++i) + { + dest->value_init(i); + value_swap(count() + i, dest, i); + value_destroy(count() + i); + } + + // The split key is the largest value in the left sibling. + set_count(count() - 1); + parent()->insert_value(position(), value_type()); + value_swap(count(), parent(), position()); + value_destroy(count()); + parent()->set_child(position() + 1, dest); + + if (!leaf()) + { + for (int i = 0; i <= dest->count(); ++i) { - dest->value_init(i); - value_swap(count() + i, dest, i); - value_destroy(count() + i); - } - - // The split key is the largest value in the left sibling. - set_count(count() - 1); - parent()->insert_value(position(), value_type()); - value_swap(count(), parent(), position()); - value_destroy(count()); - parent()->set_child(position() + 1, dest); - - if (!leaf()) - { - for (int i = 0; i <= dest->count(); ++i) - { - assert(child(count() + i + 1) != NULL); - dest->set_child(i, child(count() + i + 1)); - *mutable_child(count() + i + 1) = NULL; - } + assert(child(count() + i + 1) != NULL); + dest->set_child(i, child(count() + i + 1)); + *mutable_child(count() + i + 1) = NULL; } + } } template void btree_node

::merge(btree_node* src) { - assert(parent() == src->parent()); - assert(position() + 1 == src->position()); + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); - // Move the delimiting value to the left node. - value_init(count()); - value_swap(count(), parent(), position()); + // Move the delimiting value to the left node. + value_init(count()); + value_swap(count(), parent(), position()); - // Move the values from the right to the left node. - for (int i = 0; i < src->count(); ++i) + // Move the values from the right to the left node. + for (int i = 0; i < src->count(); ++i) + { + value_init(1 + count() + i); + value_swap(1 + count() + i, src, i); + src->value_destroy(i); + } + + if (!leaf()) + { + // Move the child pointers from the right to the left node. + for (int i = 0; i <= src->count(); ++i) { - value_init(1 + count() + i); - value_swap(1 + count() + i, src, i); - src->value_destroy(i); + set_child(1 + count() + i, src->child(i)); + *src->mutable_child(i) = NULL; } + } - if (!leaf()) - { - // Move the child pointers from the right to the left node. - for (int i = 0; i <= src->count(); ++i) - { - set_child(1 + count() + i, src->child(i)); - *src->mutable_child(i) = NULL; - } - } + // Fixup the counts on the src and dest nodes. + set_count(1 + count() + src->count()); + src->set_count(0); - // Fixup the counts on the src and dest nodes. - set_count(1 + count() + src->count()); - src->set_count(0); - - // Remove the value on the parent node. - parent()->remove_value(position()); + // Remove the value on the parent node. + parent()->remove_value(position()); } template void btree_node

::swap(btree_node* x) { - assert(leaf() == x->leaf()); + assert(leaf() == x->leaf()); - // Swap the values. - for (int i = count(); i < x->count(); ++i) + // Swap the values. + for (int i = count(); i < x->count(); ++i) + { + value_init(i); + } + + for (int i = x->count(); i < count(); ++i) + { + x->value_init(i); + } + + int n = std::max(count(), x->count()); + + for (int i = 0; i < n; ++i) + { + value_swap(i, x, i); + } + + for (int i = count(); i < x->count(); ++i) + { + x->value_destroy(i); + } + + for (int i = x->count(); i < count(); ++i) + { + value_destroy(i); + } + + if (!leaf()) + { + // Swap the child pointers. + for (int i = 0; i <= n; ++i) { - value_init(i); + btree_swap_helper(*mutable_child(i), *x->mutable_child(i)); } - for (int i = x->count(); i < count(); ++i) + for (int i = 0; i <= count(); ++i) { - x->value_init(i); + x->child(i)->fields_.parent = x; } - int n = std::max(count(), x->count()); - - for (int i = 0; i < n; ++i) + for (int i = 0; i <= x->count(); ++i) { - value_swap(i, x, i); + child(i)->fields_.parent = this; } + } - for (int i = count(); i < x->count(); ++i) - { - x->value_destroy(i); - } - - for (int i = x->count(); i < count(); ++i) - { - value_destroy(i); - } - - if (!leaf()) - { - // Swap the child pointers. - for (int i = 0; i <= n; ++i) - { - btree_swap_helper(*mutable_child(i), *x->mutable_child(i)); - } - - for (int i = 0; i <= count(); ++i) - { - x->child(i)->fields_.parent = x; - } - - for (int i = 0; i <= x->count(); ++i) - { - child(i)->fields_.parent = this; - } - } - - // Swap the counts. - btree_swap_helper(fields_.count, x->fields_.count); + // Swap the counts. + btree_swap_helper(fields_.count, x->fields_.count); } //// @@ -2817,96 +2752,96 @@ void btree_node

::swap(btree_node* x) template void btree_iterator::increment_slow() { - if (node->leaf()) + if (node->leaf()) + { + assert(position >= node->count()); + self_type save(*this); + + while (position == node->count() && !node->is_root()) { - assert(position >= node->count()); - self_type save(*this); - - while (position == node->count() && !node->is_root()) - { - assert(node->parent()->child(node->position()) == node); - position = node->position(); - node = node->parent(); - } - - if (position == node->count()) - { - *this = save; - } + assert(node->parent()->child(node->position()) == node); + position = node->position(); + node = node->parent(); } - else + + if (position == node->count()) { - assert(position < node->count()); - node = node->child(position + 1); - - while (!node->leaf()) - { - node = node->child(0); - } - - position = 0; + *this = save; } + } + else + { + assert(position < node->count()); + node = node->child(position + 1); + + while (!node->leaf()) + { + node = node->child(0); + } + + position = 0; + } } template void btree_iterator::increment_by(int count) { - while (count > 0) + while (count > 0) + { + if (node->leaf()) { - if (node->leaf()) - { - int rest = node->count() - position; - position += std::min(rest, count); - count = count - rest; + int rest = node->count() - position; + position += std::min(rest, count); + count = count - rest; - if (position < node->count()) - { - return; - } - } - else - { - --count; - } - - increment_slow(); + if (position < node->count()) + { + return; + } } + else + { + --count; + } + + increment_slow(); + } } template void btree_iterator::decrement_slow() { - if (node->leaf()) + if (node->leaf()) + { + assert(position <= -1); + self_type save(*this); + + while (position < 0 && !node->is_root()) { - assert(position <= -1); - self_type save(*this); - - while (position < 0 && !node->is_root()) - { - assert(node->parent()->child(node->position()) == node); - position = node->position() - 1; - node = node->parent(); - } - - if (position < 0) - { - *this = save; - } + assert(node->parent()->child(node->position()) == node); + position = node->position() - 1; + node = node->parent(); } - else + + if (position < 0) { - assert(position >= 0); - node = node->child(position); - - while (!node->leaf()) - { - node = node->child(node->count()); - } - - position = node->count() - 1; + *this = save; } + } + else + { + assert(position >= 0); + node = node->child(position); + + while (!node->leaf()) + { + node = node->child(node->count()); + } + + position = node->count() - 1; + } } -} // namespace btree +} // namespace btree #endif // UTIL_BTREE_BTREE_H__ diff --git a/utils/joiner/btree_container.h b/utils/joiner/btree_container.h index 72874401e..cde4ab863 100644 --- a/utils/joiner/btree_container.h +++ b/utils/joiner/btree_container.h @@ -22,439 +22,427 @@ namespace btree { - // A common base class for btree_set, btree_map, btree_multiset and // btree_multimap. template class btree_container { - typedef btree_container self_type; + typedef btree_container self_type; -public: - typedef typename Tree::params_type params_type; - typedef typename Tree::key_type key_type; - typedef typename Tree::value_type value_type; - typedef typename Tree::key_compare key_compare; - typedef typename Tree::allocator_type allocator_type; - typedef typename Tree::pointer pointer; - typedef typename Tree::const_pointer const_pointer; - typedef typename Tree::reference reference; - typedef typename Tree::const_reference const_reference; - typedef typename Tree::size_type size_type; - typedef typename Tree::difference_type difference_type; - typedef typename Tree::iterator iterator; - typedef typename Tree::const_iterator const_iterator; - typedef typename Tree::reverse_iterator reverse_iterator; - typedef typename Tree::const_reverse_iterator const_reverse_iterator; + public: + typedef typename Tree::params_type params_type; + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::pointer pointer; + typedef typename Tree::const_pointer const_pointer; + typedef typename Tree::reference reference; + typedef typename Tree::const_reference const_reference; + typedef typename Tree::size_type size_type; + typedef typename Tree::difference_type difference_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + typedef typename Tree::reverse_iterator reverse_iterator; + typedef typename Tree::const_reverse_iterator const_reverse_iterator; -public: - // Default constructor. - btree_container(const key_compare& comp, const allocator_type& alloc) - : tree_(comp, alloc) + public: + // Default constructor. + btree_container(const key_compare& comp, const allocator_type& alloc) : tree_(comp, alloc) + { + } + + // Copy constructor. + btree_container(const self_type& x) : tree_(x.tree_) + { + } + + // Iterator routines. + iterator begin() + { + return tree_.begin(); + } + const_iterator begin() const + { + return tree_.begin(); + } + iterator end() + { + return tree_.end(); + } + const_iterator end() const + { + return tree_.end(); + } + reverse_iterator rbegin() + { + return tree_.rbegin(); + } + const_reverse_iterator rbegin() const + { + return tree_.rbegin(); + } + reverse_iterator rend() + { + return tree_.rend(); + } + const_reverse_iterator rend() const + { + return tree_.rend(); + } + + // Lookup routines. + iterator lower_bound(const key_type& key) + { + return tree_.lower_bound(key); + } + const_iterator lower_bound(const key_type& key) const + { + return tree_.lower_bound(key); + } + iterator upper_bound(const key_type& key) + { + return tree_.upper_bound(key); + } + const_iterator upper_bound(const key_type& key) const + { + return tree_.upper_bound(key); + } + std::pair equal_range(const key_type& key) + { + return tree_.equal_range(key); + } + std::pair equal_range(const key_type& key) const + { + return tree_.equal_range(key); + } + + // Utility routines. + void clear() + { + tree_.clear(); + } + void swap(self_type& x) + { + tree_.swap(x.tree_); + } + void dump(std::ostream& os) const + { + tree_.dump(os); + } + void verify() const + { + tree_.verify(); + } + + // Size routines. + size_type size() const + { + return tree_.size(); + } + size_type max_size() const + { + return tree_.max_size(); + } + bool empty() const + { + return tree_.empty(); + } + size_type height() const + { + return tree_.height(); + } + size_type internal_nodes() const + { + return tree_.internal_nodes(); + } + size_type leaf_nodes() const + { + return tree_.leaf_nodes(); + } + size_type nodes() const + { + return tree_.nodes(); + } + size_type bytes_used() const + { + return tree_.bytes_used(); + } + static double average_bytes_per_value() + { + return Tree::average_bytes_per_value(); + } + double fullness() const + { + return tree_.fullness(); + } + double overhead() const + { + return tree_.overhead(); + } + + bool operator==(const self_type& x) const + { + if (size() != x.size()) { + return false; } - // Copy constructor. - btree_container(const self_type& x) - : tree_(x.tree_) + for (const_iterator i = begin(), xi = x.begin(); i != end(); ++i, ++xi) { + if (*i != *xi) + { + return false; + } } - // Iterator routines. - iterator begin() - { - return tree_.begin(); - } - const_iterator begin() const - { - return tree_.begin(); - } - iterator end() - { - return tree_.end(); - } - const_iterator end() const - { - return tree_.end(); - } - reverse_iterator rbegin() - { - return tree_.rbegin(); - } - const_reverse_iterator rbegin() const - { - return tree_.rbegin(); - } - reverse_iterator rend() - { - return tree_.rend(); - } - const_reverse_iterator rend() const - { - return tree_.rend(); - } + return true; + } - // Lookup routines. - iterator lower_bound(const key_type& key) - { - return tree_.lower_bound(key); - } - const_iterator lower_bound(const key_type& key) const - { - return tree_.lower_bound(key); - } - iterator upper_bound(const key_type& key) - { - return tree_.upper_bound(key); - } - const_iterator upper_bound(const key_type& key) const - { - return tree_.upper_bound(key); - } - std::pair equal_range(const key_type& key) - { - return tree_.equal_range(key); - } - std::pair equal_range(const key_type& key) const - { - return tree_.equal_range(key); - } + bool operator!=(const self_type& other) const + { + return !operator==(other); + } - // Utility routines. - void clear() - { - tree_.clear(); - } - void swap(self_type& x) - { - tree_.swap(x.tree_); - } - void dump(std::ostream& os) const - { - tree_.dump(os); - } - void verify() const - { - tree_.verify(); - } - - // Size routines. - size_type size() const - { - return tree_.size(); - } - size_type max_size() const - { - return tree_.max_size(); - } - bool empty() const - { - return tree_.empty(); - } - size_type height() const - { - return tree_.height(); - } - size_type internal_nodes() const - { - return tree_.internal_nodes(); - } - size_type leaf_nodes() const - { - return tree_.leaf_nodes(); - } - size_type nodes() const - { - return tree_.nodes(); - } - size_type bytes_used() const - { - return tree_.bytes_used(); - } - static double average_bytes_per_value() - { - return Tree::average_bytes_per_value(); - } - double fullness() const - { - return tree_.fullness(); - } - double overhead() const - { - return tree_.overhead(); - } - - bool operator==(const self_type& x) const - { - if (size() != x.size()) - { - return false; - } - - for (const_iterator i = begin(), xi = x.begin(); i != end(); ++i, ++xi) - { - if (*i != *xi) - { - return false; - } - } - - return true; - } - - bool operator!=(const self_type& other) const - { - return !operator==(other); - } - - -protected: - Tree tree_; + protected: + Tree tree_; }; template inline std::ostream& operator<<(std::ostream& os, const btree_container& b) { - b.dump(os); - return os; + b.dump(os); + return os; } // A common base class for btree_set and safe_btree_set. template class btree_unique_container : public btree_container { - typedef btree_unique_container self_type; - typedef btree_container super_type; + typedef btree_unique_container self_type; + typedef btree_container super_type; -public: - typedef typename Tree::key_type key_type; - typedef typename Tree::value_type value_type; - typedef typename Tree::size_type size_type; - typedef typename Tree::key_compare key_compare; - typedef typename Tree::allocator_type allocator_type; - typedef typename Tree::iterator iterator; - typedef typename Tree::const_iterator const_iterator; + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::size_type size_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; -public: - // Default constructor. - btree_unique_container(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - } + public: + // Default constructor. + btree_unique_container(const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } - // Copy constructor. - btree_unique_container(const self_type& x) - : super_type(x) - { - } + // Copy constructor. + btree_unique_container(const self_type& x) : super_type(x) + { + } - // Range constructor. - template - btree_unique_container(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - insert(b, e); - } + // Range constructor. + template + btree_unique_container(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + insert(b, e); + } - // Lookup routines. - iterator find(const key_type& key) - { - return this->tree_.find_unique(key); - } - const_iterator find(const key_type& key) const - { - return this->tree_.find_unique(key); - } - size_type count(const key_type& key) const - { - return this->tree_.count_unique(key); - } + // Lookup routines. + iterator find(const key_type& key) + { + return this->tree_.find_unique(key); + } + const_iterator find(const key_type& key) const + { + return this->tree_.find_unique(key); + } + size_type count(const key_type& key) const + { + return this->tree_.count_unique(key); + } - // Insertion routines. - std::pair insert(const value_type& x) - { - return this->tree_.insert_unique(x); - } - iterator insert(iterator position, const value_type& x) - { - return this->tree_.insert_unique(position, x); - } - template - void insert(InputIterator b, InputIterator e) - { - this->tree_.insert_unique(b, e); - } + // Insertion routines. + std::pair insert(const value_type& x) + { + return this->tree_.insert_unique(x); + } + iterator insert(iterator position, const value_type& x) + { + return this->tree_.insert_unique(position, x); + } + template + void insert(InputIterator b, InputIterator e) + { + this->tree_.insert_unique(b, e); + } - // Deletion routines. - int erase(const key_type& key) - { - return this->tree_.erase_unique(key); - } - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - iterator erase(const iterator& iter) - { - return this->tree_.erase(iter); - } - void erase(const iterator& first, const iterator& last) - { - this->tree_.erase(first, last); - } + // Deletion routines. + int erase(const key_type& key) + { + return this->tree_.erase_unique(key); + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const iterator& iter) + { + return this->tree_.erase(iter); + } + void erase(const iterator& first, const iterator& last) + { + this->tree_.erase(first, last); + } }; // A common base class for btree_map and safe_btree_map. template class btree_map_container : public btree_unique_container { - typedef btree_map_container self_type; - typedef btree_unique_container super_type; + typedef btree_map_container self_type; + typedef btree_unique_container super_type; -public: - typedef typename Tree::key_type key_type; - typedef typename Tree::data_type data_type; - typedef typename Tree::value_type value_type; - typedef typename Tree::mapped_type mapped_type; - typedef typename Tree::key_compare key_compare; - typedef typename Tree::allocator_type allocator_type; + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::data_type data_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::mapped_type mapped_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; -private: - // A pointer-like object which only generates its value when - // dereferenced. Used by operator[] to avoid constructing an empty data_type - // if the key already exists in the map. - struct generate_value - { - generate_value(const key_type& k) - : key(k) - { - } - value_type operator*() const - { - return std::make_pair(key, data_type()); - } - const key_type& key; - }; - -public: - // Default constructor. - btree_map_container(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) + private: + // A pointer-like object which only generates its value when + // dereferenced. Used by operator[] to avoid constructing an empty data_type + // if the key already exists in the map. + struct generate_value + { + generate_value(const key_type& k) : key(k) { } - - // Copy constructor. - btree_map_container(const self_type& x) - : super_type(x) + value_type operator*() const { + return std::make_pair(key, data_type()); } + const key_type& key; + }; - // Range constructor. - template - btree_map_container(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(b, e, comp, alloc) - { - } + public: + // Default constructor. + btree_map_container(const key_compare& comp = key_compare(), const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } - // Insertion routines. - data_type& operator[](const key_type& key) - { - return this->tree_.insert_unique(key, generate_value(key)).first->second; - } + // Copy constructor. + btree_map_container(const self_type& x) : super_type(x) + { + } + + // Range constructor. + template + btree_map_container(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(b, e, comp, alloc) + { + } + + // Insertion routines. + data_type& operator[](const key_type& key) + { + return this->tree_.insert_unique(key, generate_value(key)).first->second; + } }; // A common base class for btree_multiset and btree_multimap. template class btree_multi_container : public btree_container { - typedef btree_multi_container self_type; - typedef btree_container super_type; + typedef btree_multi_container self_type; + typedef btree_container super_type; -public: - typedef typename Tree::key_type key_type; - typedef typename Tree::value_type value_type; - typedef typename Tree::size_type size_type; - typedef typename Tree::key_compare key_compare; - typedef typename Tree::allocator_type allocator_type; - typedef typename Tree::iterator iterator; - typedef typename Tree::const_iterator const_iterator; + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::size_type size_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; -public: - // Default constructor. - btree_multi_container(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - } + public: + // Default constructor. + btree_multi_container(const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } - // Copy constructor. - btree_multi_container(const self_type& x) - : super_type(x) - { - } + // Copy constructor. + btree_multi_container(const self_type& x) : super_type(x) + { + } - // Range constructor. - template - btree_multi_container(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - insert(b, e); - } + // Range constructor. + template + btree_multi_container(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + insert(b, e); + } - // Lookup routines. - iterator find(const key_type& key) - { - return this->tree_.find_multi(key); - } - const_iterator find(const key_type& key) const - { - return this->tree_.find_multi(key); - } - size_type count(const key_type& key) const - { - return this->tree_.count_multi(key); - } + // Lookup routines. + iterator find(const key_type& key) + { + return this->tree_.find_multi(key); + } + const_iterator find(const key_type& key) const + { + return this->tree_.find_multi(key); + } + size_type count(const key_type& key) const + { + return this->tree_.count_multi(key); + } - // Insertion routines. - iterator insert(const value_type& x) - { - return this->tree_.insert_multi(x); - } - iterator insert(iterator position, const value_type& x) - { - return this->tree_.insert_multi(position, x); - } - template - void insert(InputIterator b, InputIterator e) - { - this->tree_.insert_multi(b, e); - } + // Insertion routines. + iterator insert(const value_type& x) + { + return this->tree_.insert_multi(x); + } + iterator insert(iterator position, const value_type& x) + { + return this->tree_.insert_multi(position, x); + } + template + void insert(InputIterator b, InputIterator e) + { + this->tree_.insert_multi(b, e); + } - // Deletion routines. - int erase(const key_type& key) - { - return this->tree_.erase_multi(key); - } - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - iterator erase(const iterator& iter) - { - return this->tree_.erase(iter); - } - void erase(const iterator& first, const iterator& last) - { - this->tree_.erase(first, last); - } + // Deletion routines. + int erase(const key_type& key) + { + return this->tree_.erase_multi(key); + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const iterator& iter) + { + return this->tree_.erase(iter); + } + void erase(const iterator& first, const iterator& last) + { + this->tree_.erase(first, last); + } }; -} // namespace btree +} // namespace btree #endif // UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/utils/joiner/btree_map.h b/utils/joiner/btree_map.h index 41b06e679..4480c616e 100644 --- a/utils/joiner/btree_map.h +++ b/utils/joiner/btree_map.h @@ -33,109 +33,92 @@ namespace btree { - // The btree_map class is needed mainly for its constructors. -template , - typename Alloc = std::allocator >, - int TargetNodeSize = 256> -class btree_map : public btree_map_container < - btree > > +template , + typename Alloc = std::allocator >, int TargetNodeSize = 256> +class btree_map + : public btree_map_container > > { + typedef btree_map self_type; + typedef btree_map_params params_type; + typedef btree btree_type; + typedef btree_map_container super_type; - typedef btree_map self_type; - typedef btree_map_params < - Key, Value, Compare, Alloc, TargetNodeSize > params_type; - typedef btree btree_type; - typedef btree_map_container super_type; + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; -public: - typedef typename btree_type::key_compare key_compare; - typedef typename btree_type::allocator_type allocator_type; + public: + // Default constructor. + btree_map(const key_compare& comp = key_compare(), const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } -public: - // Default constructor. - btree_map(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - } + // Copy constructor. + btree_map(const self_type& x) : super_type(x) + { + } - // Copy constructor. - btree_map(const self_type& x) - : super_type(x) - { - } - - // Range constructor. - template - btree_map(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(b, e, comp, alloc) - { - } + // Range constructor. + template + btree_map(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(b, e, comp, alloc) + { + } }; template -inline void swap(btree_map& x, - btree_map& y) +inline void swap(btree_map& x, btree_map& y) { - x.swap(y); + x.swap(y); } // The btree_multimap class is needed mainly for its constructors. -template , - typename Alloc = std::allocator >, - int TargetNodeSize = 256> -class btree_multimap : public btree_multi_container < - btree > > +template , + typename Alloc = std::allocator >, int TargetNodeSize = 256> +class btree_multimap + : public btree_multi_container > > { + typedef btree_multimap self_type; + typedef btree_map_params params_type; + typedef btree btree_type; + typedef btree_multi_container super_type; - typedef btree_multimap self_type; - typedef btree_map_params < - Key, Value, Compare, Alloc, TargetNodeSize > params_type; - typedef btree btree_type; - typedef btree_multi_container super_type; + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + typedef typename btree_type::data_type data_type; + typedef typename btree_type::mapped_type mapped_type; -public: - typedef typename btree_type::key_compare key_compare; - typedef typename btree_type::allocator_type allocator_type; - typedef typename btree_type::data_type data_type; - typedef typename btree_type::mapped_type mapped_type; + public: + // Default constructor. + btree_multimap(const key_compare& comp = key_compare(), const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } -public: - // Default constructor. - btree_multimap(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - } + // Copy constructor. + btree_multimap(const self_type& x) : super_type(x) + { + } - // Copy constructor. - btree_multimap(const self_type& x) - : super_type(x) - { - } - - // Range constructor. - template - btree_multimap(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(b, e, comp, alloc) - { - } + // Range constructor. + template + btree_multimap(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(b, e, comp, alloc) + { + } }; template -inline void swap(btree_multimap& x, - btree_multimap& y) +inline void swap(btree_multimap& x, btree_multimap& y) { - x.swap(y); + x.swap(y); } -} // namespace btree +} // namespace btree #endif // UTIL_BTREE_BTREE_MAP_H__ diff --git a/utils/joiner/btree_set.h b/utils/joiner/btree_set.h index bb149b243..dcabc91a3 100644 --- a/utils/joiner/btree_set.h +++ b/utils/joiner/btree_set.h @@ -29,104 +29,90 @@ namespace btree { - // The btree_set class is needed mainly for its constructors. -template , - typename Alloc = std::allocator, +template , typename Alloc = std::allocator, int TargetNodeSize = 256> -class btree_set : public btree_unique_container < - btree > > +class btree_set + : public btree_unique_container > > { + typedef btree_set self_type; + typedef btree_set_params params_type; + typedef btree btree_type; + typedef btree_unique_container super_type; - typedef btree_set self_type; - typedef btree_set_params params_type; - typedef btree btree_type; - typedef btree_unique_container super_type; + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; -public: - typedef typename btree_type::key_compare key_compare; - typedef typename btree_type::allocator_type allocator_type; + public: + // Default constructor. + btree_set(const key_compare& comp = key_compare(), const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } -public: - // Default constructor. - btree_set(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - } + // Copy constructor. + btree_set(const self_type& x) : super_type(x) + { + } - // Copy constructor. - btree_set(const self_type& x) - : super_type(x) - { - } - - // Range constructor. - template - btree_set(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(b, e, comp, alloc) - { - } + // Range constructor. + template + btree_set(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(b, e, comp, alloc) + { + } }; template inline void swap(btree_set& x, btree_set& y) { - x.swap(y); + x.swap(y); } // The btree_multiset class is needed mainly for its constructors. -template , - typename Alloc = std::allocator, +template , typename Alloc = std::allocator, int TargetNodeSize = 256> -class btree_multiset : public btree_multi_container < - btree > > +class btree_multiset + : public btree_multi_container > > { + typedef btree_multiset self_type; + typedef btree_set_params params_type; + typedef btree btree_type; + typedef btree_multi_container super_type; - typedef btree_multiset self_type; - typedef btree_set_params params_type; - typedef btree btree_type; - typedef btree_multi_container super_type; + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; -public: - typedef typename btree_type::key_compare key_compare; - typedef typename btree_type::allocator_type allocator_type; + public: + // Default constructor. + btree_multiset(const key_compare& comp = key_compare(), const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) + { + } -public: - // Default constructor. - btree_multiset(const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(comp, alloc) - { - } + // Copy constructor. + btree_multiset(const self_type& x) : super_type(x) + { + } - // Copy constructor. - btree_multiset(const self_type& x) - : super_type(x) - { - } - - // Range constructor. - template - btree_multiset(InputIterator b, InputIterator e, - const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : super_type(b, e, comp, alloc) - { - } + // Range constructor. + template + btree_multiset(InputIterator b, InputIterator e, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(b, e, comp, alloc) + { + } }; template -inline void swap(btree_multiset& x, - btree_multiset& y) +inline void swap(btree_multiset& x, btree_multiset& y) { - x.swap(y); + x.swap(y); } -} // namespace btree +} // namespace btree #endif // UTIL_BTREE_BTREE_SET_H__ diff --git a/utils/joiner/joiner.cpp b/utils/joiner/joiner.cpp index b252bfc75..f62115d72 100644 --- a/utils/joiner/joiner.cpp +++ b/utils/joiner/joiner.cpp @@ -24,65 +24,66 @@ using namespace utils; namespace joiner { - Joiner::Joiner(bool ia) : _includeAll(ia), _inPM(false), _pool(new SimplePool) { - SimpleAllocator > alloc(_pool); - h.reset(new hash_t(10, hash_t::hasher(), hash_t::key_equal(), alloc)); -// cout << "Joiner()\n"; + SimpleAllocator > alloc(_pool); + h.reset(new hash_t(10, hash_t::hasher(), hash_t::key_equal(), alloc)); + // cout << "Joiner()\n"; } Joiner::Joiner() -{ } +{ +} Joiner::Joiner(const Joiner& j) -{ } +{ +} Joiner& Joiner::operator=(const Joiner& j) { - return *this; + return *this; } Joiner::~Joiner() { -// cout << "~Joiner()\n"; - // get rid of the hash table first - h.reset(); -// delete _pool; -// _pool = NULL; + // cout << "~Joiner()\n"; + // get rid of the hash table first + h.reset(); + // delete _pool; + // _pool = NULL; } boost::shared_ptr > Joiner::getSortedMatches() { - boost::shared_ptr > ret; - iterator it; + boost::shared_ptr > ret; + iterator it; - ret.reset(new vector()); + ret.reset(new vector()); - for (it = begin(); it != end(); ++it) - if (it->second & MSB) - ret->push_back(ElementType(it->second & ~MSB, it->first)); + for (it = begin(); it != end(); ++it) + if (it->second & MSB) + ret->push_back(ElementType(it->second & ~MSB, it->first)); - sort::iterator>(ret->begin(), ret->end()); - return ret; + sort::iterator>(ret->begin(), ret->end()); + return ret; } boost::shared_ptr > Joiner::getSmallSide() { - boost::shared_ptr > ret; - iterator it; + boost::shared_ptr > ret; + iterator it; - ret.reset(new vector()); + ret.reset(new vector()); - for (it = begin(); it != end(); ++it) - ret->push_back(ElementType(it->second & ~MSB, it->first)); + for (it = begin(); it != end(); ++it) + ret->push_back(ElementType(it->second & ~MSB, it->first)); - return ret; + return ret; } void Joiner::doneInserting() { - //sort here if the data structure is a vector + // sort here if the data structure is a vector } -} +} // namespace joiner diff --git a/utils/joiner/joiner.h b/utils/joiner/joiner.h index e374de3ab..47fe48dca 100644 --- a/utils/joiner/joiner.h +++ b/utils/joiner/joiner.h @@ -39,18 +39,16 @@ namespace std { namespace tr1 { -template<> -struct hash - : public std::unary_function +template <> +struct hash : public std::unary_function { - std::size_t - operator()(long long unsigned int val) const - { - return static_cast(val); - } + std::size_t operator()(long long unsigned int val) const + { + return static_cast(val); + } }; -} -} +} // namespace tr1 +} // namespace std #endif #endif #endif @@ -59,152 +57,153 @@ struct hash #include "../joblist/elementtype.h" #undef NO_DATALISTS - namespace joiner { - /* There has to be a better name for this. Not used ATM. */ struct MatchedET { - MatchedET() { } - MatchedET(const joblist::ElementType& et) : e(et) { } - joblist::ElementType e; -// bool matched; // Might need this, might not + MatchedET() + { + } + MatchedET(const joblist::ElementType& et) : e(et) + { + } + joblist::ElementType e; + // bool matched; // Might need this, might not - inline bool operator<(const MatchedET& c) const - { - return e.second < c.e.second; - } + inline bool operator<(const MatchedET& c) const + { + return e.second < c.e.second; + } }; - class Joiner { -public: -// typedef std::tr1::unordered_multimap hash_t; - typedef std::tr1::unordered_multimap, std::equal_to, - utils::SimpleAllocator > > hash_t; + public: + // typedef std::tr1::unordered_multimap hash_t; + typedef std::tr1::unordered_multimap, std::equal_to, + utils::SimpleAllocator > > + hash_t; - typedef hash_t::iterator iterator; + typedef hash_t::iterator iterator; - Joiner(bool bIncludeAll); - virtual ~Joiner(); + Joiner(bool bIncludeAll); + virtual ~Joiner(); - // elements are stored as - inline iterator begin() + // elements are stored as + inline iterator begin() + { + return h->begin(); + } + inline iterator end() + { + return h->end(); + } + inline size_t size() + { + return h->size(); + } + inline void insert(const joblist::ElementType& e) + { + h->insert(std::pair(e.second, e.first)); + } + void doneInserting(); + boost::shared_ptr > getSmallSide(); + boost::shared_ptr > getSortedMatches(); + + /* Used by the UM */ + inline bool match(const joblist::ElementType& large) + { + std::pair range; + iterator it = h->find(large.second); + + if (it == h->end()) + return _includeAll; + else if (it->second & MSB) + return true; + else { - return h->begin(); + range = h->equal_range(large.second); + + for (; range.first != range.second; ++range.first) + range.first->second |= MSB; + + return true; } - inline iterator end() + } + + inline void mark(const joblist::ElementType& large) + { + std::pair range; + + range = h->equal_range(large.second); + + for (; range.first != range.second; ++range.first) + range.first->second |= MSB; + } + + /* Used by the PM */ + inline bool getNewMatches(const uint64_t value, std::vector* newMatches) + { + std::pair range; + iterator it = h->find(value); + + if (it == h->end()) + return _includeAll; + else if (it->second & MSB) + return true; + else { - return h->end(); + newMatches->push_back(joblist::ElementType(it->second | MSB, value)); + range = h->equal_range(value); + + for (; range.first != range.second; ++range.first) + range.first->second |= MSB; + + return true; } - inline size_t size() - { - return h->size(); - } - inline void insert(const joblist::ElementType& e) - { - h->insert(std::pair(e.second, e.first)); - } - void doneInserting(); - boost::shared_ptr > getSmallSide(); - boost::shared_ptr > getSortedMatches(); + } - /* Used by the UM */ - inline bool match(const joblist::ElementType& large) - { - std::pair range; - iterator it = h->find(large.second); + inline bool inPM() + { + return _inPM; + } + void inPM(bool b) + { + _inPM = b; + } + inline bool inUM() + { + return !_inPM; + } + void inUM(bool b) + { + _inPM = !b; + } + bool includeAll() + { + return _includeAll; + } - if (it == h->end()) - return _includeAll; - else if (it->second & MSB) - return true; - else - { - range = h->equal_range(large.second); + uint64_t getMemUsage() + { + return (_pool ? _pool->getMemUsage() : 0); + } - for ( ; range.first != range.second; ++range.first) - range.first->second |= MSB; + static const uint64_t MSB = 0x8000000000000000ULL; - return true; - } - } + protected: + Joiner(); + Joiner(const Joiner&); + Joiner& operator=(const Joiner&); - inline void mark(const joblist::ElementType& large) - { - std::pair range; - - range = h->equal_range(large.second); - - for ( ; range.first != range.second; ++range.first) - range.first->second |= MSB; - } - - /* Used by the PM */ - inline bool getNewMatches(const uint64_t value, - std::vector* newMatches) - { - std::pair range; - iterator it = h->find(value); - - if (it == h->end()) - return _includeAll; - else if (it->second & MSB) - return true; - else - { - newMatches->push_back( - joblist::ElementType(it->second | MSB, value)); - range = h->equal_range(value); - - for ( ; range.first != range.second; ++range.first) - range.first->second |= MSB; - - return true; - } - } - - inline bool inPM() - { - return _inPM; - } - void inPM(bool b) - { - _inPM = b; - } - inline bool inUM() - { - return !_inPM; - } - void inUM(bool b) - { - _inPM = !b; - } - bool includeAll() - { - return _includeAll; - } - - uint64_t getMemUsage() - { - return (_pool ? _pool->getMemUsage() : 0); - } - - static const uint64_t MSB = 0x8000000000000000ULL; -protected: - Joiner(); - Joiner(const Joiner&); - Joiner& operator=(const Joiner&); -private: - boost::shared_ptr h; - bool _includeAll; - bool _inPM; // true -> should execute on the PM, false -> UM - boost::shared_ptr _pool; // pool for the table and nodes + private: + boost::shared_ptr h; + bool _includeAll; + bool _inPM; // true -> should execute on the PM, false -> UM + boost::shared_ptr _pool; // pool for the table and nodes }; -} +} // namespace joiner #endif diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index 8b1badfb4..5fa2ec706 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -16,7 +16,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#define _CRT_RAND_S //for win rand_s +#define _CRT_RAND_S // for win rand_s #include #include #include "configcpp.h" @@ -37,856 +37,864 @@ namespace // returns a value between 0 and RAND_MAX (on Linux this is the max signed int) int rand_r(unsigned int* seedp) { - unsigned int randval = 0; - rand_s(&randval); //returns 0 if okay, 0<=randval& smallKeys, - const vector& largeKeys, - bool typeless, - bool antiWMN, - bool hasFEFilter, - uint64_t totalUMMemory, - uint64_t partitionSize) : - smallRG(sRG), largeRG(lRG), - smallKeyCols(smallKeys), largeKeyCols(largeKeys), typelessJoin(typeless), - nextPartitionToReturn(0), htSizeEstimate(0), htTargetSize(partitionSize), rootNode(true), - antiWithMatchNulls(antiWMN), needsAllNullRows(hasFEFilter), gotNullRow(false), - totalBytesRead(0), totalBytesWritten(0), maxLargeSize(0), maxSmallSize(0), - nextSmallOffset(0), nextLargeOffset(0) +JoinPartition::JoinPartition(const RowGroup& lRG, const RowGroup& sRG, const vector& smallKeys, + const vector& largeKeys, bool typeless, bool antiWMN, bool hasFEFilter, + uint64_t totalUMMemory, uint64_t partitionSize) + : smallRG(sRG) + , largeRG(lRG) + , smallKeyCols(smallKeys) + , largeKeyCols(largeKeys) + , typelessJoin(typeless) + , nextPartitionToReturn(0) + , htSizeEstimate(0) + , htTargetSize(partitionSize) + , rootNode(true) + , antiWithMatchNulls(antiWMN) + , needsAllNullRows(hasFEFilter) + , gotNullRow(false) + , totalBytesRead(0) + , totalBytesWritten(0) + , maxLargeSize(0) + , maxSmallSize(0) + , nextSmallOffset(0) + , nextLargeOffset(0) { - config::Config* config = config::Config::makeConfig(); - string cfgTxt; - smallSizeOnDisk = largeSizeOnDisk = 0; + config::Config* config = config::Config::makeConfig(); + string cfgTxt; + smallSizeOnDisk = largeSizeOnDisk = 0; - /* Debugging, rand() is used to simulate failures - time_t t = time(NULL); - srand(t); - */ + /* Debugging, rand() is used to simulate failures + time_t t = time(NULL); + srand(t); + */ + cfgTxt = config->getConfig("HashJoin", "TempFileCompression"); - cfgTxt = config->getConfig("HashJoin", "TempFileCompression"); + if (cfgTxt == "n" || cfgTxt == "N") + useCompression = false; + else + useCompression = true; - if (cfgTxt == "n" || cfgTxt == "N") - useCompression = false; - else - useCompression = true; + fileMode = false; + uniqueID = atomicops::atomicInc(&uniqueNums); + uint32_t tmp = uniqueID; + hashSeed = rand_r(&tmp); + hashSeed = hasher((char*)&hashSeed, sizeof(hashSeed), uniqueID); + hashSeed = hasher.finalize(hashSeed, sizeof(hashSeed)); - fileMode = false; - uniqueID = atomicops::atomicInc(&uniqueNums); - uint32_t tmp = uniqueID; - hashSeed = rand_r(&tmp); - hashSeed = hasher((char*) &hashSeed, sizeof(hashSeed), uniqueID); - hashSeed = hasher.finalize(hashSeed, sizeof(hashSeed)); + // start with initial capacity = 2 * totalUMMemory + bucketCount = (totalUMMemory * 2) / htTargetSize + 1; - // start with initial capacity = 2 * totalUMMemory - bucketCount = (totalUMMemory * 2) / htTargetSize + 1; + largeRG.initRow(&largeRow); + smallRG.initRow(&smallRow); - largeRG.initRow(&largeRow); - smallRG.initRow(&smallRow); + buckets.reserve(bucketCount); - buckets.reserve(bucketCount); + string compressionType; + try + { + compressionType = config->getConfig("HashJoin", "TempFileCompressionType"); + } + catch (...) + { + } - string compressionType; - try - { - compressionType = - config->getConfig("HashJoin", "TempFileCompressionType"); - } catch (...) {} + if (compressionType == "LZ4") + { + compressor.reset(new compress::CompressInterfaceLZ4()); + } + else + { + compressor.reset(new compress::CompressInterfaceSnappy()); + } - if (compressionType == "LZ4") - { - compressor.reset(new compress::CompressInterfaceLZ4()); - } - else - { - compressor.reset(new compress::CompressInterfaceSnappy()); - } - - for (uint32_t i = 0; i < bucketCount; i++) - buckets.push_back(boost::shared_ptr(new JoinPartition(*this, false))); - - + for (uint32_t i = 0; i < bucketCount; i++) + buckets.push_back(boost::shared_ptr(new JoinPartition(*this, false))); } /* Ctor used by JoinPartition on expansion, creates JP's in filemode */ -JoinPartition::JoinPartition(const JoinPartition& jp, bool splitMode) : - smallRG(jp.smallRG), largeRG(jp.largeRG), - smallKeyCols(jp.smallKeyCols), largeKeyCols(jp.largeKeyCols), - typelessJoin(jp.typelessJoin), bucketCount(jp.bucketCount), - smallRow(jp.smallRow), largeRow(jp.largeRow), - nextPartitionToReturn(0), htSizeEstimate(0), htTargetSize(jp.htTargetSize), - rootNode(false), antiWithMatchNulls(jp.antiWithMatchNulls), - needsAllNullRows(jp.needsAllNullRows), gotNullRow(false), - useCompression(jp.useCompression), totalBytesRead(0), - totalBytesWritten(0), maxLargeSize(0), maxSmallSize(0), - nextSmallOffset(0), nextLargeOffset(0) +JoinPartition::JoinPartition(const JoinPartition& jp, bool splitMode) + : smallRG(jp.smallRG) + , largeRG(jp.largeRG) + , smallKeyCols(jp.smallKeyCols) + , largeKeyCols(jp.largeKeyCols) + , typelessJoin(jp.typelessJoin) + , bucketCount(jp.bucketCount) + , smallRow(jp.smallRow) + , largeRow(jp.largeRow) + , nextPartitionToReturn(0) + , htSizeEstimate(0) + , htTargetSize(jp.htTargetSize) + , rootNode(false) + , antiWithMatchNulls(jp.antiWithMatchNulls) + , needsAllNullRows(jp.needsAllNullRows) + , gotNullRow(false) + , useCompression(jp.useCompression) + , totalBytesRead(0) + , totalBytesWritten(0) + , maxLargeSize(0) + , maxSmallSize(0) + , nextSmallOffset(0) + , nextLargeOffset(0) { - boost::posix_time::ptime t; - ostringstream os; + boost::posix_time::ptime t; + ostringstream os; - fileMode = true; - // tuning issue: with the defaults, each 100MB bucket would split s.t. the children - // could store another 4GB total. Given a good hash and evenly distributed data, - // the first level of expansion would happen for all JPs at once, giving a total - // capacity of (4GB * 40) = 160GB, when actual usage at that point is a little over 4GB. - // Instead, each will double in size, giving a capacity of 8GB -> 16 -> 32, and so on. -// bucketCount = jp.bucketCount; - bucketCount = 2; - config::Config* config = config::Config::makeConfig(); - filenamePrefix = config->getTempFileDir(config::Config::TempDirPurpose::Joins); + fileMode = true; + // tuning issue: with the defaults, each 100MB bucket would split s.t. the children + // could store another 4GB total. Given a good hash and evenly distributed data, + // the first level of expansion would happen for all JPs at once, giving a total + // capacity of (4GB * 40) = 160GB, when actual usage at that point is a little over 4GB. + // Instead, each will double in size, giving a capacity of 8GB -> 16 -> 32, and so on. + // bucketCount = jp.bucketCount; + bucketCount = 2; + config::Config* config = config::Config::makeConfig(); + filenamePrefix = config->getTempFileDir(config::Config::TempDirPurpose::Joins); - filenamePrefix += "/Columnstore-join-data-"; + filenamePrefix += "/Columnstore-join-data-"; - uniqueID = atomicops::atomicInc(&uniqueNums); - uint32_t tmp = uniqueID; - hashSeed = rand_r(&tmp); - hashSeed = hasher((char*) &hashSeed, sizeof(hashSeed), uniqueID); - hashSeed = hasher.finalize(hashSeed, sizeof(hashSeed)); + uniqueID = atomicops::atomicInc(&uniqueNums); + uint32_t tmp = uniqueID; + hashSeed = rand_r(&tmp); + hashSeed = hasher((char*)&hashSeed, sizeof(hashSeed), uniqueID); + hashSeed = hasher.finalize(hashSeed, sizeof(hashSeed)); - os << filenamePrefix << uniqueID; - filenamePrefix = os.str(); - smallFilename = filenamePrefix + "-small"; - largeFilename = filenamePrefix + "-large"; + os << filenamePrefix << uniqueID; + filenamePrefix = os.str(); + smallFilename = filenamePrefix + "-small"; + largeFilename = filenamePrefix + "-large"; - smallSizeOnDisk = largeSizeOnDisk = 0; + smallSizeOnDisk = largeSizeOnDisk = 0; - buffer.reinit(smallRG); - smallRG.setData(&buffer); - smallRG.resetRowGroup(0); - smallRG.getRow(0, &smallRow); + buffer.reinit(smallRG); + smallRG.setData(&buffer); + smallRG.resetRowGroup(0); + smallRG.getRow(0, &smallRow); - compressor = jp.compressor; + compressor = jp.compressor; } - JoinPartition::~JoinPartition() { - if (fileMode) - { - smallFile.close(); - largeFile.close(); - boost::filesystem::remove(smallFilename); - boost::filesystem::remove(largeFilename); - } + if (fileMode) + { + smallFile.close(); + largeFile.close(); + boost::filesystem::remove(smallFilename); + boost::filesystem::remove(largeFilename); + } } int64_t JoinPartition::insertSmallSideRGData(RGData& rgData) { - int64_t ret; - ret = processSmallBuffer(rgData); + int64_t ret; + ret = processSmallBuffer(rgData); - return ret; + return ret; } int64_t JoinPartition::insertSmallSideRGData(vector& rgData) { - int64_t ret = 0; + int64_t ret = 0; - // this iterates over the vector backward to free mem asap - while (rgData.size() > 0) - { - ret += insertSmallSideRGData(rgData.back()); - rgData.pop_back(); - } + // this iterates over the vector backward to free mem asap + while (rgData.size() > 0) + { + ret += insertSmallSideRGData(rgData.back()); + rgData.pop_back(); + } - return ret; + return ret; } int64_t JoinPartition::insertSmallSideRow(const Row& row) { - int64_t ret = 0; + int64_t ret = 0; - copyRow(row, &smallRow); - smallRG.incRowCount(); + copyRow(row, &smallRow); + smallRG.incRowCount(); - if (smallRG.getRowCount() == 8192) - ret = processSmallBuffer(); - else - smallRow.nextRow(); + if (smallRG.getRowCount() == 8192) + ret = processSmallBuffer(); + else + smallRow.nextRow(); - return ret; + return ret; } int64_t JoinPartition::insertLargeSideRGData(RGData& rgData) { - int64_t ret; + int64_t ret; - ret = processLargeBuffer(rgData); - return ret; + ret = processLargeBuffer(rgData); + return ret; } int64_t JoinPartition::insertLargeSideRow(const Row& row) { - int64_t ret = 0; + int64_t ret = 0; - copyRow(row, &largeRow); - largeRG.incRowCount(); + copyRow(row, &largeRow); + largeRG.incRowCount(); - if (largeRG.getRowCount() == 8192) - ret = processLargeBuffer(); - else - largeRow.nextRow(); + if (largeRG.getRowCount() == 8192) + ret = processLargeBuffer(); + else + largeRow.nextRow(); - return ret; + return ret; } int64_t JoinPartition::doneInsertingSmallData() { - /* - flush buffers to leaf nodes - config for large-side insertion - */ - int64_t ret = 0; - int64_t leafNodeIncrement; + /* + flush buffers to leaf nodes + config for large-side insertion + */ + int64_t ret = 0; + int64_t leafNodeIncrement; - /* flushing doesn't apply to the root node b/c it inserts entire RGs at once */ - if (!rootNode) - ret = processSmallBuffer(); + /* flushing doesn't apply to the root node b/c it inserts entire RGs at once */ + if (!rootNode) + ret = processSmallBuffer(); - if (!fileMode) - for (int i = 0; i < (int) buckets.size(); i++) - { - leafNodeIncrement = buckets[i]->doneInsertingSmallData(); - ret += leafNodeIncrement; - smallSizeOnDisk += leafNodeIncrement; - } - - //else - // cout << uniqueID << " htsizeestimate = " << htSizeEstimate << endl; - - if (!rootNode) + if (!fileMode) + for (int i = 0; i < (int)buckets.size(); i++) { - buffer.reinit(largeRG); - largeRG.setData(&buffer); - largeRG.resetRowGroup(0); - largeRG.getRow(0, &largeRow); + leafNodeIncrement = buckets[i]->doneInsertingSmallData(); + ret += leafNodeIncrement; + smallSizeOnDisk += leafNodeIncrement; } - if (maxSmallSize < smallSizeOnDisk) - maxSmallSize = smallSizeOnDisk; + // else + // cout << uniqueID << " htsizeestimate = " << htSizeEstimate << endl; - return ret; + if (!rootNode) + { + buffer.reinit(largeRG); + largeRG.setData(&buffer); + largeRG.resetRowGroup(0); + largeRG.getRow(0, &largeRow); + } + + if (maxSmallSize < smallSizeOnDisk) + maxSmallSize = smallSizeOnDisk; + + return ret; } int64_t JoinPartition::doneInsertingLargeData() { - /* - flush buffers to leaf nodes - */ + /* + flush buffers to leaf nodes + */ - int64_t ret = 0; - int64_t leafNodeIncrement; + int64_t ret = 0; + int64_t leafNodeIncrement; - /* flushing doesn't apply to the root node b/c it inserts entire RGs at once */ - if (!rootNode) - ret = processLargeBuffer(); + /* flushing doesn't apply to the root node b/c it inserts entire RGs at once */ + if (!rootNode) + ret = processLargeBuffer(); - if (!fileMode) - for (int i = 0; i < (int) buckets.size(); i++) - { - leafNodeIncrement = buckets[i]->doneInsertingLargeData(); - ret += leafNodeIncrement; - largeSizeOnDisk += leafNodeIncrement; - } + if (!fileMode) + for (int i = 0; i < (int)buckets.size(); i++) + { + leafNodeIncrement = buckets[i]->doneInsertingLargeData(); + ret += leafNodeIncrement; + largeSizeOnDisk += leafNodeIncrement; + } - if (maxLargeSize < largeSizeOnDisk) - maxLargeSize = largeSizeOnDisk; + if (maxLargeSize < largeSizeOnDisk) + maxLargeSize = largeSizeOnDisk; - return ret; + return ret; } int64_t JoinPartition::convertToSplitMode() { - int i, j; - ByteStream bs; - RGData rgData; - uint32_t hash; - uint64_t tmp; - int64_t ret = -(int64_t)smallSizeOnDisk; // smallFile gets deleted - boost::scoped_array rowDist(new uint32_t[bucketCount]); - uint32_t rowCount = 0; + int i, j; + ByteStream bs; + RGData rgData; + uint32_t hash; + uint64_t tmp; + int64_t ret = -(int64_t)smallSizeOnDisk; // smallFile gets deleted + boost::scoped_array rowDist(new uint32_t[bucketCount]); + uint32_t rowCount = 0; - memset(rowDist.get(), 0, sizeof(uint32_t) * bucketCount); - fileMode = false; - htSizeEstimate = 0; - smallSizeOnDisk = 0; - buckets.reserve(bucketCount); + memset(rowDist.get(), 0, sizeof(uint32_t) * bucketCount); + fileMode = false; + htSizeEstimate = 0; + smallSizeOnDisk = 0; + buckets.reserve(bucketCount); - for (i = 0; i < (int) bucketCount; i++) - buckets.push_back(boost::shared_ptr(new JoinPartition(*this, false))); + for (i = 0; i < (int)bucketCount; i++) + buckets.push_back(boost::shared_ptr(new JoinPartition(*this, false))); - RowGroup& rg = smallRG; - Row& row = smallRow; - nextSmallOffset = 0; + RowGroup& rg = smallRG; + Row& row = smallRow; + nextSmallOffset = 0; - while (1) + while (1) + { + readByteStream(0, &bs); + + if (bs.length() == 0) + break; + + rgData.deserialize(bs); + rg.setData(&rgData); + + for (j = 0; j < (int)rg.getRowCount(); j++) { - readByteStream(0, &bs); + rg.getRow(j, &row); - if (bs.length() == 0) - break; - - rgData.deserialize(bs); - rg.setData(&rgData); - - for (j = 0; j < (int) rg.getRowCount(); j++) + if (antiWithMatchNulls && hasNullJoinColumn(row)) + { + if (needsAllNullRows || !gotNullRow) { - rg.getRow(j, &row); + for (j = 0; j < (int)bucketCount; j++) + ret += buckets[j]->insertSmallSideRow(row); - if (antiWithMatchNulls && hasNullJoinColumn(row)) - { - if (needsAllNullRows || !gotNullRow) - { - for (j = 0; j < (int) bucketCount; j++) - ret += buckets[j]->insertSmallSideRow(row); - - gotNullRow = true; - } - - continue; - } - - if (typelessJoin) - hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount; - else - { - if (UNLIKELY(row.isUnsigned(smallKeyCols[0]))) - tmp = row.getUintField(smallKeyCols[0]); - else - tmp = row.getIntField(smallKeyCols[0]); - - hash = hasher((char*) &tmp, 8, hashSeed); - hash = hasher.finalize(hash, 8) % bucketCount; - } - - rowCount++; - rowDist[hash]++; - ret += buckets[hash]->insertSmallSideRow(row); + gotNullRow = true; } + + continue; + } + + if (typelessJoin) + hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount; + else + { + if (UNLIKELY(row.isUnsigned(smallKeyCols[0]))) + tmp = row.getUintField(smallKeyCols[0]); + else + tmp = row.getIntField(smallKeyCols[0]); + + hash = hasher((char*)&tmp, 8, hashSeed); + hash = hasher.finalize(hash, 8) % bucketCount; + } + + rowCount++; + rowDist[hash]++; + ret += buckets[hash]->insertSmallSideRow(row); } + } - boost::filesystem::remove(smallFilename); - smallFilename.clear(); + boost::filesystem::remove(smallFilename); + smallFilename.clear(); - for (i = 0; i < (int) bucketCount; i++) - if (rowDist[i] == rowCount) - throw IDBExcept("All rows hashed to the same bucket", ERR_DBJ_DATA_DISTRIBUTION); + for (i = 0; i < (int)bucketCount; i++) + if (rowDist[i] == rowCount) + throw IDBExcept("All rows hashed to the same bucket", ERR_DBJ_DATA_DISTRIBUTION); - rg.setData(&buffer); - rg.resetRowGroup(0); - rg.getRow(0, &row); + rg.setData(&buffer); + rg.resetRowGroup(0); + rg.getRow(0, &row); - return ret; + return ret; } /* either forwards the specified buffer to the next level of JP's or writes it to a file, returns the # of bytes written to the file */ - int64_t JoinPartition::processSmallBuffer() { - int64_t ret; + int64_t ret; - ret = processSmallBuffer(buffer); - smallRG.resetRowGroup(0); - smallRG.getRow(0, &smallRow); - return ret; + ret = processSmallBuffer(buffer); + smallRG.resetRowGroup(0); + smallRG.getRow(0, &smallRow); + return ret; } - int64_t JoinPartition::processSmallBuffer(RGData& rgData) { - RowGroup& rg = smallRG; - Row& row = smallRow; - int64_t ret = 0; + RowGroup& rg = smallRG; + Row& row = smallRow; + int64_t ret = 0; - rg.setData(&rgData); - //if (rootNode) - //cout << "smallside RGData: " << rg.toString() << endl; + rg.setData(&rgData); + // if (rootNode) + // cout << "smallside RGData: " << rg.toString() << endl; - if (fileMode) + if (fileMode) + { + ByteStream bs; + rg.serializeRGData(bs); + // cout << "writing RGData: " << rg.toString() << endl; + + ret = writeByteStream(0, bs); + // cout << "wrote " << ret << " bytes" << endl; + + /* Check whether this partition is now too big -> convert to split mode. + + The current estimate is based on 100M 4-byte rows = 4GB. The total size is + the amount stored in RowGroups in mem + the size of the hash table. The RowGroups + in that case use 600MB, so 3.4GB is used by the hash table. 3.4GB/100M rows = 34 bytes/row + */ + htSizeEstimate += rg.getDataSize() + (34 * rg.getRowCount()); + + if (htSizeEstimate > htTargetSize) + ret += convertToSplitMode(); + + // cout << "wrote some data, returning " << ret << endl; + } + else + { + uint64_t hash, tmp; + int i, j; + + for (i = 0; i < (int)rg.getRowCount(); i++) { - ByteStream bs; - rg.serializeRGData(bs); - //cout << "writing RGData: " << rg.toString() << endl; + rg.getRow(i, &row); - ret = writeByteStream(0, bs); - //cout << "wrote " << ret << " bytes" << endl; - - /* Check whether this partition is now too big -> convert to split mode. - - The current estimate is based on 100M 4-byte rows = 4GB. The total size is - the amount stored in RowGroups in mem + the size of the hash table. The RowGroups - in that case use 600MB, so 3.4GB is used by the hash table. 3.4GB/100M rows = 34 bytes/row - */ - htSizeEstimate += rg.getDataSize() + (34 * rg.getRowCount()); - - if (htSizeEstimate > htTargetSize) - ret += convertToSplitMode(); - - //cout << "wrote some data, returning " << ret << endl; - } - else - { - uint64_t hash, tmp; - int i, j; - - for (i = 0; i < (int) rg.getRowCount(); i++) + if (antiWithMatchNulls && hasNullJoinColumn(row)) + { + if (needsAllNullRows || !gotNullRow) { - rg.getRow(i, &row); + for (j = 0; j < (int)bucketCount; j++) + ret += buckets[j]->insertSmallSideRow(row); - if (antiWithMatchNulls && hasNullJoinColumn(row)) - { - if (needsAllNullRows || !gotNullRow) - { - for (j = 0; j < (int) bucketCount; j++) - ret += buckets[j]->insertSmallSideRow(row); - - gotNullRow = true; - } - - continue; - } - - if (typelessJoin) - hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount; - else - { - if (UNLIKELY(row.isUnsigned(smallKeyCols[0]))) - tmp = row.getUintField(smallKeyCols[0]); - else - tmp = row.getIntField(smallKeyCols[0]); - - hash = hasher((char*) &tmp, 8, hashSeed); - hash = hasher.finalize(hash, 8) % bucketCount; - } - - //cout << "hashing smallside row: " << row.toString() << endl; - ret += buckets[hash]->insertSmallSideRow(row); + gotNullRow = true; } - //cout << "distributed rows, returning " << ret << endl; + continue; + } + + if (typelessJoin) + hash = getHashOfTypelessKey(row, smallKeyCols, hashSeed) % bucketCount; + else + { + if (UNLIKELY(row.isUnsigned(smallKeyCols[0]))) + tmp = row.getUintField(smallKeyCols[0]); + else + tmp = row.getIntField(smallKeyCols[0]); + + hash = hasher((char*)&tmp, 8, hashSeed); + hash = hasher.finalize(hash, 8) % bucketCount; + } + + // cout << "hashing smallside row: " << row.toString() << endl; + ret += buckets[hash]->insertSmallSideRow(row); } - smallSizeOnDisk += ret; - return ret; -} + // cout << "distributed rows, returning " << ret << endl; + } + smallSizeOnDisk += ret; + return ret; +} /* the difference between processSmall & processLarge is mostly the names of variables being small* -> large*, template? */ int64_t JoinPartition::processLargeBuffer() { - int64_t ret; + int64_t ret; - ret = processLargeBuffer(buffer); - largeRG.resetRowGroup(0); - largeRG.getRow(0, &largeRow); - return ret; + ret = processLargeBuffer(buffer); + largeRG.resetRowGroup(0); + largeRG.getRow(0, &largeRow); + return ret; } int64_t JoinPartition::processLargeBuffer(RGData& rgData) { - RowGroup& rg = largeRG; - Row& row = largeRow; - int64_t ret = 0; - int i, j; + RowGroup& rg = largeRG; + Row& row = largeRow; + int64_t ret = 0; + int i, j; - rg.setData(&rgData); + rg.setData(&rgData); - //if (rootNode) - // cout << "largeside RGData: " << rg.toString() << endl; + // if (rootNode) + // cout << "largeside RGData: " << rg.toString() << endl; - /* Need to fail a query with an anti join, an FE filter, and a NULL row on the - large side b/c it needs to be joined with the entire small side table. */ - if (antiWithMatchNulls && needsAllNullRows) + /* Need to fail a query with an anti join, an FE filter, and a NULL row on the + large side b/c it needs to be joined with the entire small side table. */ + if (antiWithMatchNulls && needsAllNullRows) + { + rg.getRow(0, &row); + + for (i = 0; i < (int)rg.getRowCount(); i++, row.nextRow()) { - rg.getRow(0, &row); - - for (i = 0; i < (int) rg.getRowCount(); i++, row.nextRow()) - { - for (j = 0; j < (int) largeKeyCols.size(); j++) - { - if (row.isNullValue(largeKeyCols[j])) - throw QueryDataExcept("", ERR_DBJ_ANTI_NULL); - } - } + for (j = 0; j < (int)largeKeyCols.size(); j++) + { + if (row.isNullValue(largeKeyCols[j])) + throw QueryDataExcept("", ERR_DBJ_ANTI_NULL); + } } + } - if (fileMode) + if (fileMode) + { + ByteStream bs; + rg.serializeRGData(bs); + // cout << "writing large RGData: " << rg.toString() << endl; + ret = writeByteStream(1, bs); + // cout << "wrote " << ret << " bytes" << endl; + } + else + { + uint64_t hash, tmp; + int i; + + for (i = 0; i < (int)rg.getRowCount(); i++) { - ByteStream bs; - rg.serializeRGData(bs); - //cout << "writing large RGData: " << rg.toString() << endl; - ret = writeByteStream(1, bs); - //cout << "wrote " << ret << " bytes" << endl; + rg.getRow(i, &row); + + if (typelessJoin) + hash = getHashOfTypelessKey(row, largeKeyCols, hashSeed) % bucketCount; + else + { + if (UNLIKELY(row.isUnsigned(largeKeyCols[0]))) + tmp = row.getUintField(largeKeyCols[0]); + else + tmp = row.getIntField(largeKeyCols[0]); + + hash = hasher((char*)&tmp, 8, hashSeed); + hash = hasher.finalize(hash, 8) % bucketCount; + } + + // cout << "large side hashing row: " << row.toString() << endl; + ret += buckets[hash]->insertLargeSideRow(row); } - else - { - uint64_t hash, tmp; - int i; + } - for (i = 0; i < (int) rg.getRowCount(); i++) - { - rg.getRow(i, &row); - - if (typelessJoin) - hash = getHashOfTypelessKey(row, largeKeyCols, hashSeed) % bucketCount; - else - { - if (UNLIKELY(row.isUnsigned(largeKeyCols[0]))) - tmp = row.getUintField(largeKeyCols[0]); - else - tmp = row.getIntField(largeKeyCols[0]); - - hash = hasher((char*) &tmp, 8, hashSeed); - hash = hasher.finalize(hash, 8) % bucketCount; - } - - //cout << "large side hashing row: " << row.toString() << endl; - ret += buckets[hash]->insertLargeSideRow(row); - } - } - - largeSizeOnDisk += ret; - return ret; + largeSizeOnDisk += ret; + return ret; } bool JoinPartition::getNextPartition(vector* smallData, uint64_t* partitionID, JoinPartition** jp) { + if (fileMode) + { + ByteStream bs; + RGData rgData; - if (fileMode) + if (nextPartitionToReturn > 0) + return false; + + // cout << "reading the small side" << endl; + nextSmallOffset = 0; + + while (1) { - ByteStream bs; - RGData rgData; + readByteStream(0, &bs); - if (nextPartitionToReturn > 0) - return false; + if (bs.length() == 0) + break; - //cout << "reading the small side" << endl; - nextSmallOffset = 0; - - while (1) - { - readByteStream(0, &bs); - - if (bs.length() == 0) - break; - - rgData.deserialize(bs); - //smallRG.setData(&rgData); - //cout << "read a smallRG with " << smallRG.getRowCount() << " rows" << endl; - smallData->push_back(rgData); - } - - nextPartitionToReturn = 1; - *partitionID = uniqueID; - *jp = this; - return true; + rgData.deserialize(bs); + // smallRG.setData(&rgData); + // cout << "read a smallRG with " << smallRG.getRowCount() << " rows" << endl; + smallData->push_back(rgData); } - bool ret = false; + nextPartitionToReturn = 1; + *partitionID = uniqueID; + *jp = this; + return true; + } - while (!ret && nextPartitionToReturn < bucketCount) - { - ret = buckets[nextPartitionToReturn]->getNextPartition(smallData, partitionID, jp); + bool ret = false; - if (!ret) - nextPartitionToReturn++; - } + while (!ret && nextPartitionToReturn < bucketCount) + { + ret = buckets[nextPartitionToReturn]->getNextPartition(smallData, partitionID, jp); - return ret; + if (!ret) + nextPartitionToReturn++; + } + + return ret; } boost::shared_ptr JoinPartition::getNextLargeRGData() { - boost::shared_ptr ret; + boost::shared_ptr ret; - ByteStream bs; - readByteStream(1, &bs); + ByteStream bs; + readByteStream(1, &bs); - if (bs.length() != 0) - { - ret.reset(new RGData()); - ret->deserialize(bs); - } - else - { - boost::filesystem::remove(largeFilename); - largeSizeOnDisk = 0; - } + if (bs.length() != 0) + { + ret.reset(new RGData()); + ret->deserialize(bs); + } + else + { + boost::filesystem::remove(largeFilename); + largeSizeOnDisk = 0; + } - return ret; + return ret; } bool JoinPartition::hasNullJoinColumn(Row& r) { - for (uint32_t i = 0; i < smallKeyCols.size(); i++) - { - if (r.isNullValue(smallKeyCols[i])) - return true; - } + for (uint32_t i = 0; i < smallKeyCols.size(); i++) + { + if (r.isNullValue(smallKeyCols[i])) + return true; + } - return false; + return false; } void JoinPartition::initForProcessing() { - int i; + int i; - nextPartitionToReturn = 0; + nextPartitionToReturn = 0; - if (!fileMode) - for (i = 0; i < (int) bucketCount; i++) - buckets[i]->initForProcessing(); - else - nextLargeOffset = 0; + if (!fileMode) + for (i = 0; i < (int)bucketCount; i++) + buckets[i]->initForProcessing(); + else + nextLargeOffset = 0; } void JoinPartition::initForLargeSideFeed() { - int i; + int i; - if (!rootNode) - { - buffer.reinit(largeRG); - largeRG.setData(&buffer); - largeRG.resetRowGroup(0); - largeRG.getRow(0, &largeRow); - } + if (!rootNode) + { + buffer.reinit(largeRG); + largeRG.setData(&buffer); + largeRG.resetRowGroup(0); + largeRG.getRow(0, &largeRow); + } - largeSizeOnDisk = 0; + largeSizeOnDisk = 0; - if (fileMode) - nextLargeOffset = 0; - else - for (i = 0; i < (int) bucketCount; i++) - buckets[i]->initForLargeSideFeed(); + if (fileMode) + nextLargeOffset = 0; + else + for (i = 0; i < (int)bucketCount; i++) + buckets[i]->initForLargeSideFeed(); } void JoinPartition::saveSmallSidePartition(vector& rgData) { - //cout << "JP: saving partition: " << id << endl; - htSizeEstimate = 0; - smallSizeOnDisk = 0; - nextSmallOffset = 0; - boost::filesystem::remove(smallFilename); - insertSmallSideRGData(rgData); - doneInsertingSmallData(); + // cout << "JP: saving partition: " << id << endl; + htSizeEstimate = 0; + smallSizeOnDisk = 0; + nextSmallOffset = 0; + boost::filesystem::remove(smallFilename); + insertSmallSideRGData(rgData); + doneInsertingSmallData(); } void JoinPartition::readByteStream(int which, ByteStream* bs) { - size_t& offset = (which == 0 ? nextSmallOffset : nextLargeOffset); - fstream& fs = (which == 0 ? smallFile : largeFile); - const char* filename = (which == 0 ? smallFilename.c_str() : largeFilename.c_str()); + size_t& offset = (which == 0 ? nextSmallOffset : nextLargeOffset); + fstream& fs = (which == 0 ? smallFile : largeFile); + const char* filename = (which == 0 ? smallFilename.c_str() : largeFilename.c_str()); - size_t len; + size_t len; - bs->restart(); + bs->restart(); - fs.open(filename, ios::binary | ios::in); - int saveErrno = errno; + fs.open(filename, ios::binary | ios::in); + int saveErrno = errno; - if (!fs) + if (!fs) + { + fs.close(); + ostringstream os; + os << "Disk join could not open file (read access) " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); + } + + fs.seekg(offset); + fs.read((char*)&len, sizeof(len)); + + saveErrno = errno; + + if (!fs) + { + if (fs.eof()) { - fs.close(); - ostringstream os; - os << "Disk join could not open file (read access) " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); + fs.close(); + return; } + else + { + fs.close(); + ostringstream os; + os << "Disk join could not read file " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); + } + } - fs.seekg(offset); - fs.read((char*) &len, sizeof(len)); + idbassert(len != 0); + totalBytesRead += sizeof(len); + if (!useCompression) + { + bs->needAtLeast(len); + fs.read((char*)bs->getInputPtr(), len); saveErrno = errno; if (!fs) { - if (fs.eof()) - { - fs.close(); - return; - } - else - { - fs.close(); - ostringstream os; - os << "Disk join could not read file " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); - } + fs.close(); + ostringstream os; + os << "Disk join could not read file " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } - idbassert(len != 0); - totalBytesRead += sizeof(len); + totalBytesRead += len; + bs->advanceInputPtr(len); + } + else + { + size_t uncompressedSize; + fs.read((char*)&uncompressedSize, sizeof(uncompressedSize)); - if (!useCompression) + boost::scoped_array buf(new char[len]); + + fs.read(buf.get(), len); + saveErrno = errno; + + if (!fs || !uncompressedSize) { - bs->needAtLeast(len); - fs.read((char*) bs->getInputPtr(), len); - saveErrno = errno; - - if (!fs) - { - fs.close(); - ostringstream os; - os << "Disk join could not read file " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); - } - - totalBytesRead += len; - bs->advanceInputPtr(len); - } - else - { - size_t uncompressedSize; - fs.read((char*) &uncompressedSize, sizeof(uncompressedSize)); - - boost::scoped_array buf(new char[len]); - - fs.read(buf.get(), len); - saveErrno = errno; - - if (!fs || !uncompressedSize) - { - fs.close(); - ostringstream os; - os << "Disk join could not read file " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); - } - - totalBytesRead += len; - bs->needAtLeast(uncompressedSize); - compressor->uncompress(buf.get(), len, (char*) bs->getInputPtr(), - &uncompressedSize); - bs->advanceInputPtr(uncompressedSize); + fs.close(); + ostringstream os; + os << "Disk join could not read file " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } - offset = fs.tellg(); - fs.close(); + totalBytesRead += len; + bs->needAtLeast(uncompressedSize); + compressor->uncompress(buf.get(), len, (char*)bs->getInputPtr(), &uncompressedSize); + bs->advanceInputPtr(uncompressedSize); + } + + offset = fs.tellg(); + fs.close(); } uint64_t JoinPartition::writeByteStream(int which, ByteStream& bs) { - size_t& offset = (which == 0 ? nextSmallOffset : nextLargeOffset); - fstream& fs = (which == 0 ? smallFile : largeFile); - const char* filename = (which == 0 ? smallFilename.c_str() : largeFilename.c_str()); + size_t& offset = (which == 0 ? nextSmallOffset : nextLargeOffset); + fstream& fs = (which == 0 ? smallFile : largeFile); + const char* filename = (which == 0 ? smallFilename.c_str() : largeFilename.c_str()); - fs.open(filename, ios::binary | ios::out | ios::app); - int saveErrno = errno; + fs.open(filename, ios::binary | ios::out | ios::app); + int saveErrno = errno; + + if (!fs) + { + fs.close(); + ostringstream os; + os << "Disk join could not open file (write access) " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); + } + + uint64_t ret = 0; + size_t len = bs.length(); + idbassert(len != 0); + + fs.seekp(offset); + + if (!useCompression) + { + ret = len + 4; + fs.write((char*)&len, sizeof(len)); + fs.write((char*)bs.buf(), len); + saveErrno = errno; if (!fs) { - fs.close(); - ostringstream os; - os << "Disk join could not open file (write access) " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); + fs.close(); + ostringstream os; + os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } - uint64_t ret = 0; - size_t len = bs.length(); - idbassert(len != 0); + totalBytesWritten += sizeof(len) + len; + } + else + { + size_t maxSize = compressor->maxCompressedSize(len); + size_t actualSize = maxSize; + boost::scoped_array compressed(new uint8_t[maxSize]); - fs.seekp(offset); + compressor->compress((char*)bs.buf(), len, (char*)compressed.get(), &actualSize); + ret = actualSize + 4 + 8; // sizeof (size_t) == 8. Why 4? + fs.write((char*)&actualSize, sizeof(actualSize)); + // Save uncompressed len. + fs.write((char*)&len, sizeof(len)); + fs.write((char*)compressed.get(), actualSize); + saveErrno = errno; - if (!useCompression) + if (!fs) { - ret = len + 4; - fs.write((char*) &len, sizeof(len)); - fs.write((char*) bs.buf(), len); - saveErrno = errno; - - if (!fs) - { - fs.close(); - ostringstream os; - os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); - } - - totalBytesWritten += sizeof(len) + len; - } - else - { - size_t maxSize = compressor->maxCompressedSize(len); - size_t actualSize = maxSize; - boost::scoped_array compressed(new uint8_t[maxSize]); - - compressor->compress((char*) bs.buf(), len, (char*) compressed.get(), &actualSize); - ret = actualSize + 4 + 8; // sizeof (size_t) == 8. Why 4? - fs.write((char*) &actualSize, sizeof(actualSize)); - // Save uncompressed len. - fs.write((char*) &len, sizeof(len)); - fs.write((char*) compressed.get(), actualSize); - saveErrno = errno; - - if (!fs) - { - fs.close(); - ostringstream os; - os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; - throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); - } - - totalBytesWritten += sizeof(actualSize) + actualSize; + fs.close(); + ostringstream os; + os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl; + throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR); } - bs.advance(len); + totalBytesWritten += sizeof(actualSize) + actualSize; + } - offset = fs.tellp(); - fs.close(); - return ret; + bs.advance(len); + + offset = fs.tellp(); + fs.close(); + return ret; } uint64_t JoinPartition::getBytesRead() { - uint64_t ret; + uint64_t ret; - if (fileMode) - return totalBytesRead; + if (fileMode) + return totalBytesRead; - ret = totalBytesRead; + ret = totalBytesRead; - for (int i = 0; i < (int) bucketCount; i++) - ret += buckets[i]->getBytesRead(); + for (int i = 0; i < (int)bucketCount; i++) + ret += buckets[i]->getBytesRead(); - return ret; + return ret; } uint64_t JoinPartition::getBytesWritten() { - uint64_t ret; + uint64_t ret; - if (fileMode) - return totalBytesWritten; + if (fileMode) + return totalBytesWritten; - ret = totalBytesWritten; + ret = totalBytesWritten; - for (int i = 0; i < (int) bucketCount; i++) - ret += buckets[i]->getBytesWritten(); + for (int i = 0; i < (int)bucketCount; i++) + ret += buckets[i]->getBytesWritten(); - return ret; + return ret; } - -} // namespace - +} // namespace joiner diff --git a/utils/joiner/joinpartition.h b/utils/joiner/joinpartition.h index 9c33d8e28..20a852992 100644 --- a/utils/joiner/joinpartition.h +++ b/utils/joiner/joinpartition.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #ifndef JOINPARTITION_H #define JOINPARTITION_H @@ -28,156 +27,145 @@ namespace joiner { - class JoinPartition { -public: - JoinPartition(); - JoinPartition(const rowgroup::RowGroup& largeRG, - const rowgroup::RowGroup& smallRG, - const std::vector& smallkeyCols, - const std::vector& largeKeyCols, - bool typeless, - bool isAntiWithMatchNulls, - bool hasFEFilter, - uint64_t totalUMMemory, - uint64_t partitionSize); - JoinPartition(const JoinPartition&, bool splitMode); + public: + JoinPartition(); + JoinPartition(const rowgroup::RowGroup& largeRG, const rowgroup::RowGroup& smallRG, + const std::vector& smallkeyCols, const std::vector& largeKeyCols, + bool typeless, bool isAntiWithMatchNulls, bool hasFEFilter, uint64_t totalUMMemory, + uint64_t partitionSize); + JoinPartition(const JoinPartition&, bool splitMode); - virtual ~JoinPartition(); + virtual ~JoinPartition(); - // For now, the root node will use the RGData interface, the branches & leaves use - // only the Row interface. - int64_t insertSmallSideRow(const rowgroup::Row& row); - int64_t insertSmallSideRGData(rowgroup::RGData&); - // note, the vector version of this fcn frees the input RGDatas as it goes - int64_t insertSmallSideRGData(std::vector&); - int64_t doneInsertingSmallData(); - int64_t insertLargeSideRGData(rowgroup::RGData&); - int64_t insertLargeSideRow(const rowgroup::Row& row); - int64_t doneInsertingLargeData(); + // For now, the root node will use the RGData interface, the branches & leaves use + // only the Row interface. + int64_t insertSmallSideRow(const rowgroup::Row& row); + int64_t insertSmallSideRGData(rowgroup::RGData&); + // note, the vector version of this fcn frees the input RGDatas as it goes + int64_t insertSmallSideRGData(std::vector&); + int64_t doneInsertingSmallData(); + int64_t insertLargeSideRGData(rowgroup::RGData&); + int64_t insertLargeSideRow(const rowgroup::Row& row); + int64_t doneInsertingLargeData(); - /* Returns true if there are more partitions to fetch, false otherwise */ - bool getNextPartition(std::vector* smallData, uint64_t* partitionID, - JoinPartition** jp); + /* Returns true if there are more partitions to fetch, false otherwise */ + bool getNextPartition(std::vector* smallData, uint64_t* partitionID, JoinPartition** jp); - boost::shared_ptr getNextLargeRGData(); + boost::shared_ptr getNextLargeRGData(); - /* It's important to follow the sequence of operations to maintain the correct - internal state. Right now it doesn't check that you the programmer are doing things - right, it'll likely fail queries or crash if you do things wrong. - This should be made simpler at some point. + /* It's important to follow the sequence of operations to maintain the correct + internal state. Right now it doesn't check that you the programmer are doing things + right, it'll likely fail queries or crash if you do things wrong. + This should be made simpler at some point. - On construction, the JP is config'd for small-side reading. - After that's done, call doneInsertingSmallData() and initForLargeSideFeed(). - Then, insert the large-side data. When done, call doneInsertingLargeData() - and initForProcessing(). - In the processing phase, use getNextPartition() and getNextLargeRGData() - to get the data back out. After processing all partitions, if it's necessary - to process more iterations of the large side, call initForProcessing() again, and - continue as before. - */ + On construction, the JP is config'd for small-side reading. + After that's done, call doneInsertingSmallData() and initForLargeSideFeed(). + Then, insert the large-side data. When done, call doneInsertingLargeData() + and initForProcessing(). + In the processing phase, use getNextPartition() and getNextLargeRGData() + to get the data back out. After processing all partitions, if it's necessary + to process more iterations of the large side, call initForProcessing() again, and + continue as before. + */ + /* Call this before reading into the large side */ + void initForLargeSideFeed(); + /* Call this between large-side insertion & join processing */ + void initForProcessing(); + /* Small outer joins need to retain some state after each large-side iteration */ + void saveSmallSidePartition(std::vector& rgdata); + /* each JP instance stores the sizes of every JP instance below it, so root node has the total. */ + int64_t getCurrentDiskUsage() + { + return smallSizeOnDisk + largeSizeOnDisk; + } + int64_t getSmallSideDiskUsage() + { + return smallSizeOnDisk; + } + int64_t getLargeSideDiskUsage() + { + return largeSizeOnDisk; + } - /* Call this before reading into the large side */ - void initForLargeSideFeed(); - /* Call this between large-side insertion & join processing */ - void initForProcessing(); - /* Small outer joins need to retain some state after each large-side iteration */ - void saveSmallSidePartition(std::vector& rgdata); + uint64_t getBytesRead(); + uint64_t getBytesWritten(); + uint64_t getMaxLargeSize() + { + return maxLargeSize; + } + uint64_t getMaxSmallSize() + { + return maxSmallSize; + } - /* each JP instance stores the sizes of every JP instance below it, so root node has the total. */ - int64_t getCurrentDiskUsage() - { - return smallSizeOnDisk + largeSizeOnDisk; - } - int64_t getSmallSideDiskUsage() - { - return smallSizeOnDisk; - } - int64_t getLargeSideDiskUsage() - { - return largeSizeOnDisk; - } + protected: + private: + void initBuffers(); + int64_t convertToSplitMode(); + int64_t processSmallBuffer(); + int64_t processLargeBuffer(); - uint64_t getBytesRead(); - uint64_t getBytesWritten(); - uint64_t getMaxLargeSize() - { - return maxLargeSize; - } - uint64_t getMaxSmallSize() - { - return maxSmallSize; - } + int64_t processSmallBuffer(rowgroup::RGData&); + int64_t processLargeBuffer(rowgroup::RGData&); -protected: -private: - void initBuffers(); - int64_t convertToSplitMode(); - int64_t processSmallBuffer(); - int64_t processLargeBuffer(); + rowgroup::RowGroup smallRG; + rowgroup::RowGroup largeRG; + std::vector smallKeyCols; + std::vector largeKeyCols; + bool typelessJoin; + uint32_t hashSeed; + std::vector > buckets; + uint32_t bucketCount; // = TotalUMMem / htTargetSize - int64_t processSmallBuffer(rowgroup::RGData&); - int64_t processLargeBuffer(rowgroup::RGData&); + bool fileMode; + std::fstream smallFile; + std::fstream largeFile; + std::string filenamePrefix; + std::string smallFilename; + std::string largeFilename; + rowgroup::RGData buffer; + rowgroup::Row smallRow; + rowgroup::Row largeRow; + uint32_t nextPartitionToReturn; + uint64_t htSizeEstimate; + uint64_t htTargetSize; + uint64_t uniqueID; + uint64_t smallSizeOnDisk; + uint64_t largeSizeOnDisk; + utils::Hasher_r hasher; + bool rootNode; - rowgroup::RowGroup smallRG; - rowgroup::RowGroup largeRG; - std::vector smallKeyCols; - std::vector largeKeyCols; - bool typelessJoin; - uint32_t hashSeed; - std::vector > buckets; - uint32_t bucketCount; // = TotalUMMem / htTargetSize + /* Not-in antijoin hack. A small-side row with a null join column has to go into every partition or + into one always resident partition (TBD). - bool fileMode; - std::fstream smallFile; - std::fstream largeFile; - std::string filenamePrefix; - std::string smallFilename; - std::string largeFilename; - rowgroup::RGData buffer; - rowgroup::Row smallRow; - rowgroup::Row largeRow; - uint32_t nextPartitionToReturn; - uint64_t htSizeEstimate; - uint64_t htTargetSize; - uint64_t uniqueID; - uint64_t smallSizeOnDisk; - uint64_t largeSizeOnDisk; - utils::Hasher_r hasher; - bool rootNode; + If an F&E filter exists, it needs all null rows, if not, it only needs one. */ + bool antiWithMatchNulls; + bool needsAllNullRows; + bool gotNullRow; + bool hasNullJoinColumn(rowgroup::Row&); - /* Not-in antijoin hack. A small-side row with a null join column has to go into every partition or - into one always resident partition (TBD). + // which = 0 -> smallFile, which = 1 -> largeFile + void readByteStream(int which, messageqcpp::ByteStream* bs); + uint64_t writeByteStream(int which, messageqcpp::ByteStream& bs); - If an F&E filter exists, it needs all null rows, if not, it only needs one. */ - bool antiWithMatchNulls; - bool needsAllNullRows; - bool gotNullRow; - bool hasNullJoinColumn(rowgroup::Row&); + /* Compression support */ + bool useCompression; + std::shared_ptr compressor; + /* TBD: do the reading/writing in one thread, compression/decompression in another */ - // which = 0 -> smallFile, which = 1 -> largeFile - void readByteStream(int which, messageqcpp::ByteStream* bs); - uint64_t writeByteStream(int which, messageqcpp::ByteStream& bs); + /* Some stats for reporting */ + uint64_t totalBytesRead, totalBytesWritten; + uint64_t maxLargeSize, maxSmallSize; - /* Compression support */ - bool useCompression; - std::shared_ptr compressor; - /* TBD: do the reading/writing in one thread, compression/decompression in another */ - - /* Some stats for reporting */ - uint64_t totalBytesRead, totalBytesWritten; - uint64_t maxLargeSize, maxSmallSize; - - /* file descriptor reduction */ - size_t nextSmallOffset; - size_t nextLargeOffset; + /* file descriptor reduction */ + size_t nextSmallOffset; + size_t nextLargeOffset; }; +} // namespace joiner - -} - -#endif // JOINPARTITION_H +#endif // JOINPARTITION_H diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index acb6f06f6..fc5a5052f 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -31,7 +31,6 @@ #include "spinlock.h" #include "vlarray.h" - using namespace std; using namespace rowgroup; using namespace utils; @@ -40,1888 +39,1888 @@ using namespace joblist; namespace joiner { - // Typed joiner ctor -TupleJoiner::TupleJoiner( - const rowgroup::RowGroup& smallInput, - const rowgroup::RowGroup& largeInput, - uint32_t smallJoinColumn, - uint32_t largeJoinColumn, - JoinType jt, - threadpool::ThreadPool *jsThreadPool) : - smallRG(smallInput), largeRG(largeInput), joinAlg(INSERTING), joinType(jt), - threadCount(1), typelessJoin(false), bSignedUnsignedJoin(false), uniqueLimit(100), finished(false), - jobstepThreadPool(jsThreadPool), _convertToDiskJoin(false) +TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, + uint32_t smallJoinColumn, uint32_t largeJoinColumn, JoinType jt, + threadpool::ThreadPool* jsThreadPool) + : smallRG(smallInput) + , largeRG(largeInput) + , joinAlg(INSERTING) + , joinType(jt) + , threadCount(1) + , typelessJoin(false) + , bSignedUnsignedJoin(false) + , uniqueLimit(100) + , finished(false) + , jobstepThreadPool(jsThreadPool) + , _convertToDiskJoin(false) { - uint i; + uint i; - getBucketCount(); - m_bucketLocks.reset(new boost::mutex[bucketCount]); + getBucketCount(); + m_bucketLocks.reset(new boost::mutex[bucketCount]); - if (smallRG.getColTypes()[smallJoinColumn] == CalpontSystemCatalog::LONGDOUBLE) + if (smallRG.getColTypes()[smallJoinColumn] == CalpontSystemCatalog::LONGDOUBLE) + { + ld.reset(new boost::scoped_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) { - ld.reset(new boost::scoped_ptr[bucketCount]); - _pool.reset(new boost::shared_ptr[bucketCount]); - for (i = 0; i < bucketCount; i++) - { - STLPoolAllocator > alloc; - _pool[i] = alloc.getPoolAllocator(); - ld[i].reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); - } + STLPoolAllocator> alloc; + _pool[i] = alloc.getPoolAllocator(); + ld[i].reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); } - else if (smallRG.usesStringTable()) + } + else if (smallRG.usesStringTable()) + { + sth.reset(new boost::scoped_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) { - sth.reset(new boost::scoped_ptr[bucketCount]); - _pool.reset(new boost::shared_ptr[bucketCount]); - for (i = 0; i < bucketCount; i++) - { - STLPoolAllocator > alloc; - _pool[i] = alloc.getPoolAllocator(); - sth[i].reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); - } + STLPoolAllocator> alloc; + _pool[i] = alloc.getPoolAllocator(); + sth[i].reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); + } + } + else + { + h.reset(new boost::scoped_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) + { + STLPoolAllocator> alloc; + _pool[i] = alloc.getPoolAllocator(); + h[i].reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); + } + } + + smallRG.initRow(&smallNullRow); + + if (smallOuterJoin() || largeOuterJoin() || semiJoin() || antiJoin()) + { + smallNullMemory = RGData(smallRG, 1); + smallRG.setData(&smallNullMemory); + smallRG.getRow(0, &smallNullRow); + smallNullRow.initToNull(); + } + + smallKeyColumns.push_back(smallJoinColumn); + largeKeyColumns.push_back(largeJoinColumn); + discreteValues.reset(new bool[1]); + cpValues.reset(new vector[1]); + discreteValues[0] = false; + + if (smallRG.isUnsigned(smallKeyColumns[0])) + { + if (datatypes::isWideDecimalType(smallRG.getColType(smallKeyColumns[0]), + smallRG.getColumnWidth(smallKeyColumns[0]))) + { + cpValues[0].push_back((int128_t)-1); + cpValues[0].push_back(0); } else { - h.reset(new boost::scoped_ptr[bucketCount]); - _pool.reset(new boost::shared_ptr[bucketCount]); - for (i = 0; i < bucketCount; i++) - { - STLPoolAllocator > alloc; - _pool[i] = alloc.getPoolAllocator(); - h[i].reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); - } + cpValues[0].push_back((int128_t)numeric_limits::max()); + cpValues[0].push_back(0); } - - smallRG.initRow(&smallNullRow); - - if (smallOuterJoin() || largeOuterJoin() || semiJoin() || antiJoin()) + } + else + { + if (datatypes::isWideDecimalType(smallRG.getColType(smallKeyColumns[0]), + smallRG.getColumnWidth(smallKeyColumns[0]))) { - smallNullMemory = RGData(smallRG, 1); - smallRG.setData(&smallNullMemory); - smallRG.getRow(0, &smallNullRow); - smallNullRow.initToNull(); - } - - smallKeyColumns.push_back(smallJoinColumn); - largeKeyColumns.push_back(largeJoinColumn); - discreteValues.reset(new bool[1]); - cpValues.reset(new vector[1]); - discreteValues[0] = false; - - if (smallRG.isUnsigned(smallKeyColumns[0])) - { - if (datatypes::isWideDecimalType( - smallRG.getColType(smallKeyColumns[0]), - smallRG.getColumnWidth(smallKeyColumns[0]))) - { - cpValues[0].push_back((int128_t) -1); - cpValues[0].push_back(0); - } - else - { - cpValues[0].push_back((int128_t) numeric_limits::max()); - cpValues[0].push_back(0); - } + cpValues[0].push_back(utils::maxInt128); + cpValues[0].push_back(utils::minInt128); } else { - if (datatypes::isWideDecimalType( - smallRG.getColType(smallKeyColumns[0]), - smallRG.getColumnWidth(smallKeyColumns[0]))) - { - cpValues[0].push_back(utils::maxInt128); - cpValues[0].push_back(utils::minInt128); - } - else - { - cpValues[0].push_back((int128_t) numeric_limits::max()); - cpValues[0].push_back((int128_t) numeric_limits::min()); - } + cpValues[0].push_back((int128_t)numeric_limits::max()); + cpValues[0].push_back((int128_t)numeric_limits::min()); } + } - if (smallRG.isUnsigned(smallJoinColumn) != largeRG.isUnsigned(largeJoinColumn)) - bSignedUnsignedJoin = true; + if (smallRG.isUnsigned(smallJoinColumn) != largeRG.isUnsigned(largeJoinColumn)) + bSignedUnsignedJoin = true; - nullValueForJoinColumn = smallNullRow.getSignedNullValue(smallJoinColumn); + nullValueForJoinColumn = smallNullRow.getSignedNullValue(smallJoinColumn); } // Typeless joiner ctor -TupleJoiner::TupleJoiner( - const rowgroup::RowGroup& smallInput, - const rowgroup::RowGroup& largeInput, - const vector& smallJoinColumns, - const vector& largeJoinColumns, - JoinType jt, - threadpool::ThreadPool *jsThreadPool) : - smallRG(smallInput), largeRG(largeInput), joinAlg(INSERTING), - joinType(jt), threadCount(1), typelessJoin(true), - smallKeyColumns(smallJoinColumns), largeKeyColumns(largeJoinColumns), - bSignedUnsignedJoin(false), uniqueLimit(100), finished(false), - jobstepThreadPool(jsThreadPool), _convertToDiskJoin(false) +TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, + const vector& smallJoinColumns, const vector& largeJoinColumns, + JoinType jt, threadpool::ThreadPool* jsThreadPool) + : smallRG(smallInput) + , largeRG(largeInput) + , joinAlg(INSERTING) + , joinType(jt) + , threadCount(1) + , typelessJoin(true) + , smallKeyColumns(smallJoinColumns) + , largeKeyColumns(largeJoinColumns) + , bSignedUnsignedJoin(false) + , uniqueLimit(100) + , finished(false) + , jobstepThreadPool(jsThreadPool) + , _convertToDiskJoin(false) { - uint i; + uint i; - getBucketCount(); + getBucketCount(); - _pool.reset(new boost::shared_ptr[bucketCount]); - ht.reset(new boost::scoped_ptr[bucketCount]); - for (i = 0; i < bucketCount; i++) + _pool.reset(new boost::shared_ptr[bucketCount]); + ht.reset(new boost::scoped_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) + { + STLPoolAllocator> alloc; + _pool[i] = alloc.getPoolAllocator(); + ht[i].reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); + } + m_bucketLocks.reset(new boost::mutex[bucketCount]); + + smallRG.initRow(&smallNullRow); + + if (smallOuterJoin() || largeOuterJoin() || semiJoin() || antiJoin()) + { + smallNullMemory = RGData(smallRG, 1); + smallRG.setData(&smallNullMemory); + smallRG.getRow(0, &smallNullRow); + smallNullRow.initToNull(); + } + + keyLength = calculateKeyLength(smallKeyColumns, smallRG, &largeKeyColumns, &largeRG); + + discreteValues.reset(new bool[smallKeyColumns.size()]); + cpValues.reset(new vector[smallKeyColumns.size()]); + + for (i = 0; i < smallKeyColumns.size(); ++i) + { + uint32_t smallKeyColumnsIdx = smallKeyColumns[i]; + auto smallSideColType = smallRG.getColTypes()[smallKeyColumnsIdx]; + // Set bSignedUnsignedJoin if one or more join columns are signed to unsigned compares. + if (smallRG.isUnsigned(smallKeyColumnsIdx) != largeRG.isUnsigned(largeKeyColumns[i])) { - STLPoolAllocator > alloc; - _pool[i] = alloc.getPoolAllocator(); - ht[i].reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); - } - m_bucketLocks.reset(new boost::mutex[bucketCount]); - - smallRG.initRow(&smallNullRow); - - if (smallOuterJoin() || largeOuterJoin() || semiJoin() || antiJoin()) - { - smallNullMemory = RGData(smallRG, 1); - smallRG.setData(&smallNullMemory); - smallRG.getRow(0, &smallNullRow); - smallNullRow.initToNull(); + bSignedUnsignedJoin = true; } - keyLength = calculateKeyLength(smallKeyColumns, smallRG, &largeKeyColumns, &largeRG); - - discreteValues.reset(new bool[smallKeyColumns.size()]); - cpValues.reset(new vector[smallKeyColumns.size()]); - - for (i = 0; i < smallKeyColumns.size(); ++i) + discreteValues[i] = false; + if (isUnsigned(smallSideColType)) { - uint32_t smallKeyColumnsIdx = smallKeyColumns[i]; - auto smallSideColType = smallRG.getColTypes()[smallKeyColumnsIdx]; - // Set bSignedUnsignedJoin if one or more join columns are signed to unsigned compares. - if (smallRG.isUnsigned(smallKeyColumnsIdx) != largeRG.isUnsigned(largeKeyColumns[i])) - { - bSignedUnsignedJoin = true; - } - - discreteValues[i] = false; - if (isUnsigned(smallSideColType)) - { - cpValues[i].push_back((int128_t) numeric_limits::max()); - cpValues[i].push_back(0); - } - else - { - if (datatypes::isWideDecimalType(smallSideColType, - smallRG.getColumnWidth(smallKeyColumnsIdx))) - { - cpValues[i].push_back(utils::maxInt128); - cpValues[i].push_back(utils::minInt128); - } - else - { - cpValues[i].push_back(numeric_limits::max()); - cpValues[i].push_back(numeric_limits::min()); - } - } + cpValues[i].push_back((int128_t)numeric_limits::max()); + cpValues[i].push_back(0); } + else + { + if (datatypes::isWideDecimalType(smallSideColType, smallRG.getColumnWidth(smallKeyColumnsIdx))) + { + cpValues[i].push_back(utils::maxInt128); + cpValues[i].push_back(utils::minInt128); + } + else + { + cpValues[i].push_back(numeric_limits::max()); + cpValues[i].push_back(numeric_limits::min()); + } + } + } - // note, 'numcores' is implied by tuplehashjoin on calls to insertRGData(). - // TODO: make it explicit to avoid future confusion. - storedKeyAlloc.reset(new FixedAllocator[numCores]); - for (i = 0; i < (uint) numCores; i++) - storedKeyAlloc[i].setAllocSize(keyLength); + // note, 'numcores' is implied by tuplehashjoin on calls to insertRGData(). + // TODO: make it explicit to avoid future confusion. + storedKeyAlloc.reset(new FixedAllocator[numCores]); + for (i = 0; i < (uint)numCores; i++) + storedKeyAlloc[i].setAllocSize(keyLength); } -TupleJoiner::TupleJoiner() { } +TupleJoiner::TupleJoiner() +{ +} TupleJoiner::TupleJoiner(const TupleJoiner& j) { - throw runtime_error("TupleJoiner(TupleJoiner) shouldn't be called."); + throw runtime_error("TupleJoiner(TupleJoiner) shouldn't be called."); } TupleJoiner& TupleJoiner::operator=(const TupleJoiner& j) { - throw runtime_error("TupleJoiner::operator=() shouldn't be called."); - return *this; + throw runtime_error("TupleJoiner::operator=() shouldn't be called."); + return *this; } TupleJoiner::~TupleJoiner() { - smallNullMemory = RGData(); + smallNullMemory = RGData(); } bool TupleJoiner::operator<(const TupleJoiner& tj) const { - return size() < tj.size(); + return size() < tj.size(); } void TupleJoiner::getBucketCount() { - // get the # of cores, round up to nearest power of 2 - // make the bucket mask - numCores = sysconf(_SC_NPROCESSORS_ONLN); - if (numCores <= 0) - numCores = 8; - bucketCount = (numCores == 1 ? 1 : (1 << (32 - __builtin_clz(numCores - 1)))); - bucketMask = bucketCount - 1; + // get the # of cores, round up to nearest power of 2 + // make the bucket mask + numCores = sysconf(_SC_NPROCESSORS_ONLN); + if (numCores <= 0) + numCores = 8; + bucketCount = (numCores == 1 ? 1 : (1 << (32 - __builtin_clz(numCores - 1)))); + bucketMask = bucketCount - 1; } -template -void TupleJoiner::bucketsToTables(buckets_t *buckets, hash_table_t *tables) +template +void TupleJoiner::bucketsToTables(buckets_t* buckets, hash_table_t* tables) { - uint i; + uint i; - bool done = false, wasProductive; - while (!done) + bool done = false, wasProductive; + while (!done) + { + done = true; + wasProductive = false; + for (i = 0; i < bucketCount; i++) { - done = true; - wasProductive = false; - for (i = 0; i < bucketCount; i++) - { - if (buckets[i].empty()) - continue; - bool gotIt = m_bucketLocks[i].try_lock(); - if (!gotIt) - { - done = false; - continue; - } - for (auto &element : buckets[i]) - tables[i]->insert(element); - m_bucketLocks[i].unlock(); - wasProductive = true; - buckets[i].clear(); - } - if (!done && !wasProductive) - ::usleep(1000 * numCores); + if (buckets[i].empty()) + continue; + bool gotIt = m_bucketLocks[i].try_lock(); + if (!gotIt) + { + done = false; + continue; + } + for (auto& element : buckets[i]) + tables[i]->insert(element); + m_bucketLocks[i].unlock(); + wasProductive = true; + buckets[i].clear(); } + if (!done && !wasProductive) + ::usleep(1000 * numCores); + } } -void TupleJoiner::um_insertTypeless(uint threadID, uint rowCount, Row &r) +void TupleJoiner::um_insertTypeless(uint threadID, uint rowCount, Row& r) { - utils::VLArray td(rowCount); - utils::VLArray > > v(bucketCount); - uint i; - FixedAllocator *alloc = &storedKeyAlloc[threadID]; + utils::VLArray td(rowCount); + utils::VLArray>> v(bucketCount); + uint i; + FixedAllocator* alloc = &storedKeyAlloc[threadID]; - for (i = 0; i < rowCount; i++, r.nextRow()) - { - td[i] = makeTypelessKey(r, smallKeyColumns, keyLength, alloc, - largeRG, largeKeyColumns); - if (td[i].len == 0) - continue; - uint bucket = bucketPicker((char *) td[i].data, td[i].len, bpSeed) & bucketMask; - v[bucket].push_back(pair(td[i], r.getPointer())); - } - bucketsToTables(&v[0], ht.get()); + for (i = 0; i < rowCount; i++, r.nextRow()) + { + td[i] = makeTypelessKey(r, smallKeyColumns, keyLength, alloc, largeRG, largeKeyColumns); + if (td[i].len == 0) + continue; + uint bucket = bucketPicker((char*)td[i].data, td[i].len, bpSeed) & bucketMask; + v[bucket].push_back(pair(td[i], r.getPointer())); + } + bucketsToTables(&v[0], ht.get()); } -void TupleJoiner::um_insertLongDouble(uint rowCount, Row &r) +void TupleJoiner::um_insertLongDouble(uint rowCount, Row& r) { - utils::VLArray > > v(bucketCount); - uint i; - uint smallKeyColumn = smallKeyColumns[0]; + utils::VLArray>> v(bucketCount); + uint i; + uint smallKeyColumn = smallKeyColumns[0]; - for (i = 0; i < rowCount; i++, r.nextRow()) - { - long double smallKey = r.getLongDoubleField(smallKeyColumn); - uint bucket = bucketPicker((char *) &smallKey, 10, bpSeed) & bucketMask; // change if we decide to support windows again - if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) - v[bucket].push_back(pair(joblist::LONGDOUBLENULL, r.getPointer())); - else - v[bucket].push_back(pair(smallKey, r.getPointer())); - } - bucketsToTables(&v[0], ld.get()); -} - -void TupleJoiner::um_insertInlineRows(uint rowCount, Row &r) -{ - uint i; - int64_t smallKey; - utils::VLArray > > v(bucketCount); - uint smallKeyColumn = smallKeyColumns[0]; - - for (i = 0; i < rowCount; i++, r.nextRow()) - { - if (!r.isUnsigned(smallKeyColumn)) - smallKey = r.getIntField(smallKeyColumn); - else - smallKey = (int64_t) r.getUintField(smallKeyColumn); - uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; - if (UNLIKELY(smallKey == nullValueForJoinColumn)) - v[bucket].push_back(pair(getJoinNullValue(), r.getData())); - else - v[bucket].push_back(pair(smallKey, r.getData())); - } - bucketsToTables(&v[0], h.get()); -} - -void TupleJoiner::um_insertStringTable(uint rowCount, Row &r) -{ - int64_t smallKey; - uint i; - utils::VLArray > > v(bucketCount); - uint smallKeyColumn = smallKeyColumns[0]; - - for (i = 0; i < rowCount; i++, r.nextRow()) - { - if (!r.isUnsigned(smallKeyColumn)) - smallKey = r.getIntField(smallKeyColumn); - else - smallKey = (int64_t) r.getUintField(smallKeyColumn); - uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; - if (UNLIKELY(smallKey == nullValueForJoinColumn)) - v[bucket].push_back(pair(getJoinNullValue(), r.getPointer())); - else - v[bucket].push_back(pair(smallKey, r.getPointer())); - } - bucketsToTables(&v[0], sth.get()); -} - -void TupleJoiner::insertRGData(RowGroup &rg, uint threadID) -{ - uint i, rowCount; - Row r; - - rg.initRow(&r); - rowCount = rg.getRowCount(); - - rg.getRow(0, &r); - m_cpValuesLock.lock(); - for (i = 0; i < rowCount; i++, r.nextRow()) - { - updateCPData(r); - r.zeroRid(); - } - m_cpValuesLock.unlock(); - rg.getRow(0, &r); - - if (joinAlg == UM) - { - if (typelessJoin) - um_insertTypeless(threadID, rowCount, r); - else if (r.getColType(smallKeyColumns[0]) == execplan::CalpontSystemCatalog::LONGDOUBLE) - um_insertLongDouble(rowCount, r); - else if (!smallRG.usesStringTable()) - um_insertInlineRows(rowCount, r); - else - um_insertStringTable(rowCount, r); - } + for (i = 0; i < rowCount; i++, r.nextRow()) + { + long double smallKey = r.getLongDoubleField(smallKeyColumn); + uint bucket = bucketPicker((char*)&smallKey, 10, bpSeed) & + bucketMask; // change if we decide to support windows again + if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) + v[bucket].push_back(pair(joblist::LONGDOUBLENULL, r.getPointer())); else - { - // while in PM-join mode, inserting is single-threaded - for (i = 0; i < rowCount; i++, r.nextRow()) - rows.push_back(r.getPointer()); - } + v[bucket].push_back(pair(smallKey, r.getPointer())); + } + bucketsToTables(&v[0], ld.get()); +} + +void TupleJoiner::um_insertInlineRows(uint rowCount, Row& r) +{ + uint i; + int64_t smallKey; + utils::VLArray>> v(bucketCount); + uint smallKeyColumn = smallKeyColumns[0]; + + for (i = 0; i < rowCount; i++, r.nextRow()) + { + if (!r.isUnsigned(smallKeyColumn)) + smallKey = r.getIntField(smallKeyColumn); + else + smallKey = (int64_t)r.getUintField(smallKeyColumn); + uint bucket = bucketPicker((char*)&smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + v[bucket].push_back(pair(getJoinNullValue(), r.getData())); + else + v[bucket].push_back(pair(smallKey, r.getData())); + } + bucketsToTables(&v[0], h.get()); +} + +void TupleJoiner::um_insertStringTable(uint rowCount, Row& r) +{ + int64_t smallKey; + uint i; + utils::VLArray>> v(bucketCount); + uint smallKeyColumn = smallKeyColumns[0]; + + for (i = 0; i < rowCount; i++, r.nextRow()) + { + if (!r.isUnsigned(smallKeyColumn)) + smallKey = r.getIntField(smallKeyColumn); + else + smallKey = (int64_t)r.getUintField(smallKeyColumn); + uint bucket = bucketPicker((char*)&smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + v[bucket].push_back(pair(getJoinNullValue(), r.getPointer())); + else + v[bucket].push_back(pair(smallKey, r.getPointer())); + } + bucketsToTables(&v[0], sth.get()); +} + +void TupleJoiner::insertRGData(RowGroup& rg, uint threadID) +{ + uint i, rowCount; + Row r; + + rg.initRow(&r); + rowCount = rg.getRowCount(); + + rg.getRow(0, &r); + m_cpValuesLock.lock(); + for (i = 0; i < rowCount; i++, r.nextRow()) + { + updateCPData(r); + r.zeroRid(); + } + m_cpValuesLock.unlock(); + rg.getRow(0, &r); + + if (joinAlg == UM) + { + if (typelessJoin) + um_insertTypeless(threadID, rowCount, r); + else if (r.getColType(smallKeyColumns[0]) == execplan::CalpontSystemCatalog::LONGDOUBLE) + um_insertLongDouble(rowCount, r); + else if (!smallRG.usesStringTable()) + um_insertInlineRows(rowCount, r); + else + um_insertStringTable(rowCount, r); + } + else + { + // while in PM-join mode, inserting is single-threaded + for (i = 0; i < rowCount; i++, r.nextRow()) + rows.push_back(r.getPointer()); + } } void TupleJoiner::insert(Row& r, bool zeroTheRid) { - /* when doing a disk-based join, only the first iteration on the large side - will 'zeroTheRid'. The successive iterations will need it unchanged. */ - if (zeroTheRid) - r.zeroRid(); + /* when doing a disk-based join, only the first iteration on the large side + will 'zeroTheRid'. The successive iterations will need it unchanged. */ + if (zeroTheRid) + r.zeroRid(); - updateCPData(r); + updateCPData(r); - if (joinAlg == UM) + if (joinAlg == UM) + { + if (typelessJoin) { - if (typelessJoin) - { - TypelessData td = makeTypelessKey(r, smallKeyColumns, keyLength, &storedKeyAlloc[0], - largeRG, largeKeyColumns); - if (td.len > 0) - { - uint bucket = bucketPicker((char *) td.data, td.len, bpSeed) & bucketMask; - ht[bucket]->insert(pair(td, r.getPointer())); - } - } - else if (r.getColType(smallKeyColumns[0]) == execplan::CalpontSystemCatalog::LONGDOUBLE) - { - long double smallKey = r.getLongDoubleField(smallKeyColumns[0]); - uint bucket = bucketPicker((char *) &smallKey, 10, bpSeed) & bucketMask; // change if we decide to support windows again - if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) - ld[bucket]->insert(pair(joblist::LONGDOUBLENULL, r.getPointer())); - else - ld[bucket]->insert(pair(smallKey, r.getPointer())); - } - else if (!smallRG.usesStringTable()) - { - int64_t smallKey; + TypelessData td = + makeTypelessKey(r, smallKeyColumns, keyLength, &storedKeyAlloc[0], largeRG, largeKeyColumns); + if (td.len > 0) + { + uint bucket = bucketPicker((char*)td.data, td.len, bpSeed) & bucketMask; + ht[bucket]->insert(pair(td, r.getPointer())); + } + } + else if (r.getColType(smallKeyColumns[0]) == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + long double smallKey = r.getLongDoubleField(smallKeyColumns[0]); + uint bucket = bucketPicker((char*)&smallKey, 10, bpSeed) & + bucketMask; // change if we decide to support windows again + if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) + ld[bucket]->insert(pair(joblist::LONGDOUBLENULL, r.getPointer())); + else + ld[bucket]->insert(pair(smallKey, r.getPointer())); + } + else if (!smallRG.usesStringTable()) + { + int64_t smallKey; - if (!r.isUnsigned(smallKeyColumns[0])) - smallKey = r.getIntField(smallKeyColumns[0]); - else - smallKey = (int64_t) r.getUintField(smallKeyColumns[0]); - uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; - if (UNLIKELY(smallKey == nullValueForJoinColumn)) - h[bucket]->insert(pair(getJoinNullValue(), r.getData())); - else - h[bucket]->insert(pair(smallKey, r.getData())); // Normal path for integers - } - else - { - int64_t smallKey; - - if (!r.isUnsigned(smallKeyColumns[0])) - smallKey = r.getIntField(smallKeyColumns[0]); - else - smallKey = (int64_t) r.getUintField(smallKeyColumns[0]); - uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; - if (UNLIKELY(smallKey == nullValueForJoinColumn)) - sth[bucket]->insert(pair(getJoinNullValue(), r.getPointer())); - else - sth[bucket]->insert(pair(smallKey, r.getPointer())); - } + if (!r.isUnsigned(smallKeyColumns[0])) + smallKey = r.getIntField(smallKeyColumns[0]); + else + smallKey = (int64_t)r.getUintField(smallKeyColumns[0]); + uint bucket = bucketPicker((char*)&smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + h[bucket]->insert(pair(getJoinNullValue(), r.getData())); + else + h[bucket]->insert(pair(smallKey, r.getData())); // Normal path for integers } else - rows.push_back(r.getPointer()); + { + int64_t smallKey; + + if (!r.isUnsigned(smallKeyColumns[0])) + smallKey = r.getIntField(smallKeyColumns[0]); + else + smallKey = (int64_t)r.getUintField(smallKeyColumns[0]); + uint bucket = bucketPicker((char*)&smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + sth[bucket]->insert(pair(getJoinNullValue(), r.getPointer())); + else + sth[bucket]->insert(pair(smallKey, r.getPointer())); + } + } + else + rows.push_back(r.getPointer()); } void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uint32_t threadID, vector* matches) { - uint32_t i; - bool isNull = hasNullJoinColumn(largeSideRow); - matches->clear(); + uint32_t i; + bool isNull = hasNullJoinColumn(largeSideRow); + matches->clear(); - if (inPM()) + if (inPM()) + { + vector& v = pmJoinResults[threadID][largeRowIndex]; + uint32_t size = v.size(); + + for (i = 0; i < size; i++) + if (v[i] < rows.size()) + matches->push_back(rows[v[i]]); + + if (UNLIKELY((semiJoin() || antiJoin()) && matches->size() == 0)) + matches->push_back(smallNullRow.getPointer()); + } + else if (LIKELY(!isNull)) + { + if (UNLIKELY(typelessJoin)) { - vector& v = pmJoinResults[threadID][largeRowIndex]; - uint32_t size = v.size(); + TypelessData largeKey; + thIterator it; + pair range; - for (i = 0; i < size; i++) - if (v[i] < rows.size()) - matches->push_back(rows[v[i]]); + largeKey = makeTypelessKey(largeSideRow, largeKeyColumns, keyLength, &tmpKeyAlloc[threadID], smallRG, + smallKeyColumns); + if (largeKey.len == 0) + return; - if (UNLIKELY((semiJoin() || antiJoin()) && matches->size() == 0)) - matches->push_back(smallNullRow.getPointer()); + uint bucket = bucketPicker((char*)largeKey.data, largeKey.len, bpSeed) & bucketMask; + range = ht[bucket]->equal_range(largeKey); + + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) + return; + + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); } - else if (LIKELY(!isNull)) + else if (largeSideRow.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE && ld) { - if (UNLIKELY(typelessJoin)) - { - TypelessData largeKey; - thIterator it; - pair range; + // This is a compare of two long double + long double largeKey; + ldIterator it; + pair range; + Row r; - largeKey = makeTypelessKey(largeSideRow, largeKeyColumns, keyLength, &tmpKeyAlloc[threadID], smallRG, smallKeyColumns); - if (largeKey.len == 0) - return; + largeKey = largeSideRow.getLongDoubleField(largeKeyColumns[0]); + uint bucket = bucketPicker((char*)&largeKey, 10, bpSeed) & bucketMask; + range = ld[bucket]->equal_range(largeKey); - uint bucket = bucketPicker((char *) largeKey.data, largeKey.len, bpSeed) & bucketMask; - range = ht[bucket]->equal_range(largeKey); - - if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) - return; - - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } - else if (largeSideRow.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE - && ld) - { - // This is a compare of two long double - long double largeKey; - ldIterator it; - pair range; - Row r; - - largeKey = largeSideRow.getLongDoubleField(largeKeyColumns[0]); - uint bucket = bucketPicker((char *) &largeKey, 10, bpSeed) & bucketMask; - range = ld[bucket]->equal_range(largeKey); - - if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) - return; - for (; range.first != range.second; ++range.first) - { - matches->push_back(range.first->second); - } - } - else if (!smallRG.usesStringTable()) - { - int64_t largeKey; - - if (largeSideRow.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - { - largeKey = (int64_t)largeSideRow.getLongDoubleField(largeKeyColumns[0]); - } - else if (largeSideRow.isUnsigned(largeKeyColumns[0])) - { - largeKey = (int64_t)largeSideRow.getUintField(largeKeyColumns[0]); - } - else - { - largeKey = largeSideRow.getIntField(largeKeyColumns[0]); - } - - if (ld) - { - // Compare against long double - long double ldKey = largeKey; - uint bucket = bucketPicker((char *) &ldKey, 10, bpSeed) & bucketMask; - auto range = ld[bucket]->equal_range(ldKey); - - if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) - return; - - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } - else - { - uint bucket = bucketPicker((char *) &largeKey, sizeof(largeKey), bpSeed) & bucketMask; - auto range = h[bucket]->equal_range(largeKey); - - if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) - return; - - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } - } - else - { - int64_t largeKey = largeSideRow.getIntField(largeKeyColumns[0]); - uint bucket = bucketPicker((char *) &largeKey, sizeof(largeKey), bpSeed) & bucketMask; - auto range = sth[bucket]->equal_range(largeKey); - - if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) - return; - - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) + return; + for (; range.first != range.second; ++range.first) + { + matches->push_back(range.first->second); + } } - - if (UNLIKELY(largeOuterJoin() && matches->size() == 0)) + else if (!smallRG.usesStringTable()) { - //cout << "Matched the NULL row: " << smallNullRow.toString() << endl; - matches->push_back(smallNullRow.getPointer()); - } + int64_t largeKey; - if (UNLIKELY(inUM() && (joinType & MATCHNULLS) && !isNull && !typelessJoin)) + if (largeSideRow.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + { + largeKey = (int64_t)largeSideRow.getLongDoubleField(largeKeyColumns[0]); + } + else if (largeSideRow.isUnsigned(largeKeyColumns[0])) + { + largeKey = (int64_t)largeSideRow.getUintField(largeKeyColumns[0]); + } + else + { + largeKey = largeSideRow.getIntField(largeKeyColumns[0]); + } + + if (ld) + { + // Compare against long double + long double ldKey = largeKey; + uint bucket = bucketPicker((char*)&ldKey, 10, bpSeed) & bucketMask; + auto range = ld[bucket]->equal_range(ldKey); + + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) + return; + + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); + } + else + { + uint bucket = bucketPicker((char*)&largeKey, sizeof(largeKey), bpSeed) & bucketMask; + auto range = h[bucket]->equal_range(largeKey); + + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) + return; + + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); + } + } + else { - if (smallRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - { - uint bucket = bucketPicker((char *) &(joblist::LONGDOUBLENULL), - sizeof(joblist::LONGDOUBLENULL), bpSeed) & bucketMask; - pair range = ld[bucket]->equal_range(joblist::LONGDOUBLENULL); + int64_t largeKey = largeSideRow.getIntField(largeKeyColumns[0]); + uint bucket = bucketPicker((char*)&largeKey, sizeof(largeKey), bpSeed) & bucketMask; + auto range = sth[bucket]->equal_range(largeKey); - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } - else if (!smallRG.usesStringTable()) - { - auto nullVal = getJoinNullValue(); - uint bucket = bucketPicker((char *) &nullVal, sizeof(nullVal), bpSeed) & bucketMask; - pair range = h[bucket]->equal_range(nullVal); + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) + return; - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } - else - { - auto nullVal = getJoinNullValue(); - uint bucket = bucketPicker((char *) &nullVal, sizeof(nullVal), bpSeed) & bucketMask; - pair range = sth[bucket]->equal_range(nullVal); - - for (; range.first != range.second; ++range.first) - matches->push_back(range.first->second); - } + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); } + } - /* Bug 3524. For 'not in' queries this matches everything. - */ - if (UNLIKELY(inUM() && isNull && antiJoin() && (joinType & MATCHNULLS))) + if (UNLIKELY(largeOuterJoin() && matches->size() == 0)) + { + // cout << "Matched the NULL row: " << smallNullRow.toString() << endl; + matches->push_back(smallNullRow.getPointer()); + } + + if (UNLIKELY(inUM() && (joinType & MATCHNULLS) && !isNull && !typelessJoin)) + { + if (smallRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) { - if (!typelessJoin) - { - if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - { - ldIterator it; + uint bucket = bucketPicker((char*)&(joblist::LONGDOUBLENULL), sizeof(joblist::LONGDOUBLENULL), bpSeed) & + bucketMask; + pair range = ld[bucket]->equal_range(joblist::LONGDOUBLENULL); - for (uint i = 0; i < bucketCount; i++) - for (it = ld[i]->begin(); it != ld[i]->end(); ++it) - matches->push_back(it->second); - } - else if (!smallRG.usesStringTable()) - { - iterator it; - - for (uint i = 0; i < bucketCount; i++) - for (it = h[i]->begin(); it != h[i]->end(); ++it) - matches->push_back(it->second); - } - else - { - sthash_t::iterator it; - - for (uint i = 0; i < bucketCount; i++) - for (it = sth[i]->begin(); it != sth[i]->end(); ++it) - matches->push_back(it->second); - } - } - else - { - thIterator it; - - for (uint i = 0; i < bucketCount; i++) - for (it = ht[i]->begin(); it != ht[i]->end(); ++it) - matches->push_back(it->second); - } + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); } + else if (!smallRG.usesStringTable()) + { + auto nullVal = getJoinNullValue(); + uint bucket = bucketPicker((char*)&nullVal, sizeof(nullVal), bpSeed) & bucketMask; + pair range = h[bucket]->equal_range(nullVal); + + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); + } + else + { + auto nullVal = getJoinNullValue(); + uint bucket = bucketPicker((char*)&nullVal, sizeof(nullVal), bpSeed) & bucketMask; + pair range = sth[bucket]->equal_range(nullVal); + + for (; range.first != range.second; ++range.first) + matches->push_back(range.first->second); + } + } + + /* Bug 3524. For 'not in' queries this matches everything. + */ + if (UNLIKELY(inUM() && isNull && antiJoin() && (joinType & MATCHNULLS))) + { + if (!typelessJoin) + { + if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + { + ldIterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = ld[i]->begin(); it != ld[i]->end(); ++it) + matches->push_back(it->second); + } + else if (!smallRG.usesStringTable()) + { + iterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = h[i]->begin(); it != h[i]->end(); ++it) + matches->push_back(it->second); + } + else + { + sthash_t::iterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = sth[i]->begin(); it != sth[i]->end(); ++it) + matches->push_back(it->second); + } + } + else + { + thIterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = ht[i]->begin(); it != ht[i]->end(); ++it) + matches->push_back(it->second); + } + } } void TupleJoiner::doneInserting() { - - // a minor textual cleanup + // a minor textual cleanup #ifdef TJ_DEBUG -#define CHECKSIZE \ - if (uniquer.size() > uniqueLimit) { \ - cout << "too many discrete values\n"; \ - return; \ - } +#define CHECKSIZE \ + if (uniquer.size() > uniqueLimit) \ + { \ + cout << "too many discrete values\n"; \ + return; \ + } #else -#define CHECKSIZE \ - if (uniquer.size() > uniqueLimit) \ - return; +#define CHECKSIZE \ + if (uniquer.size() > uniqueLimit) \ + return; #endif - uint32_t col; + uint32_t col; - /* Put together the discrete values for the runtime casual partitioning restriction */ + /* Put together the discrete values for the runtime casual partitioning restriction */ - finished = true; + finished = true; - for (col = 0; col < smallKeyColumns.size(); col++) + for (col = 0; col < smallKeyColumns.size(); col++) + { + typedef std::tr1::unordered_set unordered_set_int128; + unordered_set_int128 uniquer; + unordered_set_int128::iterator uit; + sthash_t::iterator sthit; + hash_t::iterator hit; + ldhash_t::iterator ldit; + typelesshash_t::iterator thit; + uint32_t i, pmpos = 0, rowCount; + Row smallRow; + auto smallSideColIdx = smallKeyColumns[col]; + auto smallSideColType = smallRG.getColType(smallSideColIdx); + + smallRG.initRow(&smallRow); + + if (smallRow.isCharType(smallSideColIdx)) + continue; + + rowCount = size(); + + uint bucket = 0; + if (joinAlg == PM) + pmpos = 0; + else if (typelessJoin) + thit = ht[bucket]->begin(); + else if (isLongDouble(smallRG.getColType(smallKeyColumns[0]))) + ldit = ld[bucket]->begin(); + else if (!smallRG.usesStringTable()) + hit = h[bucket]->begin(); + else + sthit = sth[bucket]->begin(); + + for (i = 0; i < rowCount; i++) { - typedef std::tr1::unordered_set unordered_set_int128; - unordered_set_int128 uniquer; - unordered_set_int128::iterator uit; - sthash_t::iterator sthit; - hash_t::iterator hit; - ldhash_t::iterator ldit; - typelesshash_t::iterator thit; - uint32_t i, pmpos = 0, rowCount; - Row smallRow; - auto smallSideColIdx = smallKeyColumns[col]; - auto smallSideColType = smallRG.getColType(smallSideColIdx); + if (joinAlg == PM) + smallRow.setPointer(rows[pmpos++]); + else if (typelessJoin) + { + while (thit == ht[bucket]->end()) + thit = ht[++bucket]->begin(); + smallRow.setPointer(thit->second); + ++thit; + } + else if (isLongDouble(smallSideColType)) + { + while (ldit == ld[bucket]->end()) + ldit = ld[++bucket]->begin(); + smallRow.setPointer(ldit->second); + ++ldit; + } + else if (!smallRG.usesStringTable()) + { + while (hit == h[bucket]->end()) + hit = h[++bucket]->begin(); + smallRow.setPointer(hit->second); + ++hit; + } + else + { + while (sthit == sth[bucket]->end()) + sthit = sth[++bucket]->begin(); + smallRow.setPointer(sthit->second); + ++sthit; + } - smallRG.initRow(&smallRow); - - if (smallRow.isCharType(smallSideColIdx)) - continue; - - rowCount = size(); - - uint bucket = 0; - if (joinAlg == PM) - pmpos = 0; - else if (typelessJoin) - thit = ht[bucket]->begin(); - else if (isLongDouble(smallRG.getColType(smallKeyColumns[0]))) - ldit = ld[bucket]->begin(); - else if (!smallRG.usesStringTable()) - hit = h[bucket]->begin(); - else - sthit = sth[bucket]->begin(); - - for (i = 0; i < rowCount; i++) + if (isLongDouble(smallSideColType)) + { + double dval = (double)roundl(smallRow.getLongDoubleField(smallSideColIdx)); + switch (largeRG.getColType(largeKeyColumns[col])) { - if (joinAlg == PM) - smallRow.setPointer(rows[pmpos++]); - else if (typelessJoin) - { - while (thit == ht[bucket]->end()) - thit = ht[++bucket]->begin(); - smallRow.setPointer(thit->second); - ++thit; - } - else if (isLongDouble(smallSideColType)) - { - while (ldit == ld[bucket]->end()) - ldit = ld[++bucket]->begin(); - smallRow.setPointer(ldit->second); - ++ldit; - } - else if (!smallRG.usesStringTable()) - { - while (hit == h[bucket]->end()) - hit = h[++bucket]->begin(); - smallRow.setPointer(hit->second); - ++hit; - } - else - { - while (sthit == sth[bucket]->end()) - sthit = sth[++bucket]->begin(); - smallRow.setPointer(sthit->second); - ++sthit; - } - - if (isLongDouble(smallSideColType)) - { - double dval = (double)roundl(smallRow.getLongDoubleField(smallSideColIdx)); - switch (largeRG.getColType(largeKeyColumns[col])) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - uniquer.insert(*(int64_t*)&dval); - break; - } - default: - { - uniquer.insert((int64_t)dval); - } - } - } - else if (datatypes::isWideDecimalType(smallSideColType, - smallRow.getColumnWidth(smallSideColIdx))) - { - uniquer.insert(smallRow.getTSInt128Field(smallSideColIdx).getValue()); - } - else if (smallRow.isUnsigned(smallSideColIdx)) - { - uniquer.insert((int64_t)smallRow.getUintField(smallSideColIdx)); - } - else - { - uniquer.insert(smallRow.getIntField(smallSideColIdx)); - } - - CHECKSIZE; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + uniquer.insert(*(int64_t*)&dval); + break; + } + default: + { + uniquer.insert((int64_t)dval); + } } + } + else if (datatypes::isWideDecimalType(smallSideColType, smallRow.getColumnWidth(smallSideColIdx))) + { + uniquer.insert(smallRow.getTSInt128Field(smallSideColIdx).getValue()); + } + else if (smallRow.isUnsigned(smallSideColIdx)) + { + uniquer.insert((int64_t)smallRow.getUintField(smallSideColIdx)); + } + else + { + uniquer.insert(smallRow.getIntField(smallSideColIdx)); + } - discreteValues[col] = true; - cpValues[col].clear(); + CHECKSIZE; + } + + discreteValues[col] = true; + cpValues[col].clear(); #ifdef TJ_DEBUG - cout << "inserting " << uniquer.size() << " discrete values\n"; + cout << "inserting " << uniquer.size() << " discrete values\n"; #endif - for (uit = uniquer.begin(); uit != uniquer.end(); ++uit) - cpValues[col].push_back(*uit); - } + for (uit = uniquer.begin(); uit != uniquer.end(); ++uit) + cpValues[col].push_back(*uit); + } } void TupleJoiner::setInPM() { - joinAlg = PM; + joinAlg = PM; } void TupleJoiner::umJoinConvert(size_t begin, size_t end) { - Row smallRow; - smallRG.initRow(&smallRow); + Row smallRow; + smallRG.initRow(&smallRow); - while (begin < end) - { - smallRow.setPointer(rows[begin++]); - insert(smallRow); - } + while (begin < end) + { + smallRow.setPointer(rows[begin++]); + insert(smallRow); + } } void TupleJoiner::setInUM() { + vector empty; + Row smallRow; + uint32_t i, size; + + if (joinAlg == UM) + return; + + joinAlg = UM; + size = rows.size(); + size_t chunkSize = + ((size / numCores) + 1 < 50000 ? 50000 + : (size / numCores) + 1); // don't start a thread to process < 50k rows + + utils::VLArray jobs(numCores); + i = 0; + for (size_t firstRow = 0; i < (uint)numCores && firstRow < size; i++, firstRow += chunkSize) + jobs[i] = jobstepThreadPool->invoke( + [this, firstRow, chunkSize, size] + { this->umJoinConvert(firstRow, (firstRow + chunkSize < size ? firstRow + chunkSize : size)); }); + + for (uint j = 0; j < i; j++) + jobstepThreadPool->join(jobs[j]); + +#ifdef TJ_DEBUG + cout << "done\n"; +#endif + rows.swap(empty); + + if (typelessJoin) + { + tmpKeyAlloc.reset(new FixedAllocator[threadCount]); + + for (i = 0; i < threadCount; i++) + tmpKeyAlloc[i] = FixedAllocator(keyLength, true); + } +} + +void TupleJoiner::umJoinConvert(uint threadID, vector& rgs, size_t begin, size_t end) +{ + RowGroup l_smallRG(smallRG); + + while (begin < end) + { + l_smallRG.setData(&(rgs[begin++])); + insertRGData(l_smallRG, threadID); + } +} + +void TupleJoiner::setInUM(vector& rgs) +{ + Row smallRow; + uint32_t i, size; + + if (joinAlg == UM) + return; + + { // don't need rows anymore, free the mem vector empty; - Row smallRow; - uint32_t i, size; - - if (joinAlg == UM) - return; - - joinAlg = UM; - size = rows.size(); - size_t chunkSize = ((size / numCores) + 1 < 50000 ? 50000 : (size / numCores) + 1); // don't start a thread to process < 50k rows - - utils::VLArray jobs(numCores); - i = 0; - for (size_t firstRow = 0; i < (uint) numCores && firstRow < size; i++, firstRow += chunkSize) - jobs[i] = jobstepThreadPool->invoke([this, firstRow, chunkSize, size] { - this->umJoinConvert(firstRow, (firstRow + chunkSize < size ? firstRow + chunkSize : size)); - } ); - - for (uint j = 0; j < i; j++) - jobstepThreadPool->join(jobs[j]); - -#ifdef TJ_DEBUG - cout << "done\n"; -#endif rows.swap(empty); + } - if (typelessJoin) - { - tmpKeyAlloc.reset(new FixedAllocator[threadCount]); + joinAlg = UM; + size = rgs.size(); + size_t chunkSize = + ((size / numCores) + 1 < 10 ? 10 : (size / numCores) + 1); // don't issue jobs for < 10 rowgroups - for (i = 0; i < threadCount; i++) - tmpKeyAlloc[i] = FixedAllocator(keyLength, true); - } -} + utils::VLArray jobs(numCores); + i = 0; + for (size_t firstRow = 0; i < (uint)numCores && firstRow < size; i++, firstRow += chunkSize) + jobs[i] = jobstepThreadPool->invoke( + [this, firstRow, chunkSize, size, i, &rgs] { + this->umJoinConvert(i, rgs, firstRow, (firstRow + chunkSize < size ? firstRow + chunkSize : size)); + }); -void TupleJoiner::umJoinConvert(uint threadID, vector &rgs, size_t begin, size_t end) -{ - RowGroup l_smallRG(smallRG); - - while (begin < end) - { - l_smallRG.setData(&(rgs[begin++])); - insertRGData(l_smallRG, threadID); - } -} - -void TupleJoiner::setInUM(vector &rgs) -{ - Row smallRow; - uint32_t i, size; - - if (joinAlg == UM) - return; - - { // don't need rows anymore, free the mem - vector empty; - rows.swap(empty); - } - - joinAlg = UM; - size = rgs.size(); - size_t chunkSize = ((size / numCores) + 1 < 10 ? 10 : (size / numCores) + 1); // don't issue jobs for < 10 rowgroups - - utils::VLArray jobs(numCores); - i = 0; - for (size_t firstRow = 0; i < (uint) numCores && firstRow < size; i++, firstRow += chunkSize) - jobs[i] = jobstepThreadPool->invoke([this, firstRow, chunkSize, size, i, &rgs] { - this->umJoinConvert(i, rgs, firstRow, (firstRow + chunkSize < size ? firstRow + chunkSize : size)); - } ); - - for (uint j = 0; j < i; j++) - jobstepThreadPool->join(jobs[j]); + for (uint j = 0; j < i; j++) + jobstepThreadPool->join(jobs[j]); #ifdef TJ_DEBUG - cout << "done\n"; + cout << "done\n"; #endif - if (typelessJoin) - { - tmpKeyAlloc.reset(new FixedAllocator[threadCount]); + if (typelessJoin) + { + tmpKeyAlloc.reset(new FixedAllocator[threadCount]); - for (i = 0; i < threadCount; i++) - tmpKeyAlloc[i] = FixedAllocator(keyLength, true); - } + for (i = 0; i < threadCount; i++) + tmpKeyAlloc[i] = FixedAllocator(keyLength, true); + } } void TupleJoiner::setPMJoinResults(boost::shared_array> jr, uint32_t threadID) { - pmJoinResults[threadID] = jr; + pmJoinResults[threadID] = jr; } void TupleJoiner::markMatches(uint32_t threadID, uint32_t rowCount) { - boost::shared_array > matches = pmJoinResults[threadID]; - uint32_t i, j; + boost::shared_array> matches = pmJoinResults[threadID]; + uint32_t i, j; - for (i = 0; i < rowCount; i++) - for (j = 0; j < matches[i].size(); j++) - { - if (matches[i][j] < rows.size()) - { - smallRow[threadID].setPointer(rows[matches[i][j]]); - smallRow[threadID].markRow(); - } - } + for (i = 0; i < rowCount; i++) + for (j = 0; j < matches[i].size(); j++) + { + if (matches[i][j] < rows.size()) + { + smallRow[threadID].setPointer(rows[matches[i][j]]); + smallRow[threadID].markRow(); + } + } } void TupleJoiner::markMatches(uint32_t threadID, const vector& matches) { - uint32_t rowCount = matches.size(); - uint32_t i; + uint32_t rowCount = matches.size(); + uint32_t i; - for (i = 0; i < rowCount; i++) - { - smallRow[threadID].setPointer(matches[i]); - smallRow[threadID].markRow(); - } + for (i = 0; i < rowCount; i++) + { + smallRow[threadID].setPointer(matches[i]); + smallRow[threadID].markRow(); + } } -boost::shared_array > TupleJoiner::getPMJoinArrays(uint32_t threadID) +boost::shared_array> TupleJoiner::getPMJoinArrays(uint32_t threadID) { - return pmJoinResults[threadID]; + return pmJoinResults[threadID]; } void TupleJoiner::setThreadCount(uint32_t cnt) { - threadCount = cnt; - pmJoinResults.reset(new boost::shared_array>[cnt]); - smallRow.reset(new Row[cnt]); + threadCount = cnt; + pmJoinResults.reset(new boost::shared_array>[cnt]); + smallRow.reset(new Row[cnt]); + + for (uint32_t i = 0; i < cnt; i++) + smallRG.initRow(&smallRow[i]); + + if (typelessJoin) + { + tmpKeyAlloc.reset(new FixedAllocator[threadCount]); + + for (uint32_t i = 0; i < threadCount; i++) + tmpKeyAlloc[i] = FixedAllocator(keyLength, true); + } + + if (fe) + { + fes.reset(new funcexp::FuncExpWrapper[cnt]); for (uint32_t i = 0; i < cnt; i++) - smallRG.initRow(&smallRow[i]); - - if (typelessJoin) - { - tmpKeyAlloc.reset(new FixedAllocator[threadCount]); - - for (uint32_t i = 0; i < threadCount; i++) - tmpKeyAlloc[i] = FixedAllocator(keyLength, true); - } - - if (fe) - { - fes.reset(new funcexp::FuncExpWrapper[cnt]); - - for (uint32_t i = 0; i < cnt; i++) - fes[i] = *fe; - } + fes[i] = *fe; + } } void TupleJoiner::getUnmarkedRows(vector* out) { - Row smallR; + Row smallR; - smallRG.initRow(&smallR); - out->clear(); + smallRG.initRow(&smallR); + out->clear(); - if (inPM()) + if (inPM()) + { + uint32_t i, size; + + size = rows.size(); + + for (i = 0; i < size; i++) { - uint32_t i, size; + smallR.setPointer(rows[i]); - size = rows.size(); + if (!smallR.isMarked()) + out->push_back(rows[i]); + } + } + else + { + if (typelessJoin) + { + typelesshash_t::iterator it; - for (i = 0; i < size; i++) + for (uint i = 0; i < bucketCount; i++) + for (it = ht[i]->begin(); it != ht[i]->end(); ++it) { - smallR.setPointer(rows[i]); + smallR.setPointer(it->second); - if (!smallR.isMarked()) - out->push_back(rows[i]); + if (!smallR.isMarked()) + out->push_back(it->second); + } + } + else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + { + ldIterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = ld[i]->begin(); it != ld[i]->end(); ++it) + { + smallR.setPointer(it->second); + + if (!smallR.isMarked()) + out->push_back(it->second); + } + } + else if (!smallRG.usesStringTable()) + { + iterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = h[i]->begin(); it != h[i]->end(); ++it) + { + smallR.setPointer(it->second); + + if (!smallR.isMarked()) + out->push_back(it->second); } } else { - if (typelessJoin) + sthash_t::iterator it; + + for (uint i = 0; i < bucketCount; i++) + for (it = sth[i]->begin(); it != sth[i]->end(); ++it) { - typelesshash_t::iterator it; + smallR.setPointer(it->second); - for (uint i = 0; i < bucketCount; i++) - for (it = ht[i]->begin(); it != ht[i]->end(); ++it) - { - smallR.setPointer(it->second); - - if (!smallR.isMarked()) - out->push_back(it->second); - } - } - else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - { - ldIterator it; - - for (uint i = 0; i < bucketCount; i++) - for (it = ld[i]->begin(); it != ld[i]->end(); ++it) - { - smallR.setPointer(it->second); - - if (!smallR.isMarked()) - out->push_back(it->second); - } - } - else if (!smallRG.usesStringTable()) - { - iterator it; - - for (uint i = 0; i < bucketCount; i++) - for (it = h[i]->begin(); it != h[i]->end(); ++it) - { - smallR.setPointer(it->second); - - if (!smallR.isMarked()) - out->push_back(it->second); - } - } - else - { - sthash_t::iterator it; - - for (uint i = 0; i < bucketCount; i++) - for (it = sth[i]->begin(); it != sth[i]->end(); ++it) - { - smallR.setPointer(it->second); - - if (!smallR.isMarked()) - out->push_back(it->second); - } + if (!smallR.isMarked()) + out->push_back(it->second); } } + } } uint64_t TupleJoiner::getMemUsage() const { - if (inUM() && typelessJoin) - { - size_t ret = 0; - for (uint i = 0; i < bucketCount; i++) - ret += _pool[i]->getMemUsage(); - for (int i = 0; i < numCores; i++) - ret += storedKeyAlloc[i].getMemUsage(); - return ret; - } - else if (inUM()) - { - size_t ret = 0; - for (uint i = 0; i < bucketCount; i++) - ret += _pool[i]->getMemUsage(); - return ret; - } - else - return (rows.size() * sizeof(Row::Pointer)); + if (inUM() && typelessJoin) + { + size_t ret = 0; + for (uint i = 0; i < bucketCount; i++) + ret += _pool[i]->getMemUsage(); + for (int i = 0; i < numCores; i++) + ret += storedKeyAlloc[i].getMemUsage(); + return ret; + } + else if (inUM()) + { + size_t ret = 0; + for (uint i = 0; i < bucketCount; i++) + ret += _pool[i]->getMemUsage(); + return ret; + } + else + return (rows.size() * sizeof(Row::Pointer)); } void TupleJoiner::setFcnExpFilter(boost::shared_ptr pt) { - fe = pt; + fe = pt; - if (fe) - joinType |= WITHFCNEXP; - else - joinType &= ~WITHFCNEXP; + if (fe) + joinType |= WITHFCNEXP; + else + joinType &= ~WITHFCNEXP; } void TupleJoiner::updateCPData(const Row& r) { - uint32_t col; + uint32_t col; - if (antiJoin() || largeOuterJoin()) - return; + if (antiJoin() || largeOuterJoin()) + return; - for (col = 0; col < smallKeyColumns.size(); col++) + for (col = 0; col < smallKeyColumns.size(); col++) + { + auto colIdx = smallKeyColumns[col]; + if (r.isLongString(colIdx)) + continue; + + auto &min = cpValues[col][0], &max = cpValues[col][1]; + + if (r.isCharType(colIdx)) { - auto colIdx = smallKeyColumns[col]; - if (r.isLongString(colIdx)) - continue; + datatypes::Charset cs(r.getCharset(colIdx)); + int64_t val = r.getIntField(colIdx); - auto& min = cpValues[col][0], &max = cpValues[col][1]; + if (datatypes::TCharShort::strnncollsp(cs, val, min, r.getColumnWidth(smallKeyColumns[col])) < 0 || + ((int64_t)min) == numeric_limits::max()) + { + min = val; + } - if (r.isCharType(colIdx)) - { - datatypes::Charset cs(r.getCharset(colIdx)); - int64_t val = r.getIntField(colIdx); - - if (datatypes::TCharShort::strnncollsp(cs, val, min, r.getColumnWidth(smallKeyColumns[col])) < 0 || - ((int64_t) min) == numeric_limits::max()) - { - min = val; - } - - if (datatypes::TCharShort::strnncollsp(cs, val, max, r.getColumnWidth(smallKeyColumns[col])) > 0 || - ((int64_t) max) == numeric_limits::min()) - { - max = val; - } - } - else if (r.isUnsigned(colIdx)) - { - uint128_t uval; - - if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) - { - double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); - switch (largeRG.getColType(largeKeyColumns[col])) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - uval = *(uint64_t*)&dval; - break; - } - default: - { - uval = (uint64_t)dval; - } - } - } - else if (datatypes::isWideDecimalType( - r.getColType(colIdx), - r.getColumnWidth(colIdx))) - { - - uval = r.getTSInt128Field(colIdx).getValue(); - } - else - { - uval = r.getUintField(colIdx); - } - - if (uval > static_cast(max)) - max = static_cast(uval); - - if (uval < static_cast(min)) - min = static_cast(uval); - } - else - { - int128_t val = 0; - - if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) - { - double dval = (double)roundl(r.getLongDoubleField(colIdx)); - switch (largeRG.getColType(largeKeyColumns[col])) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - val = *(int64_t*)&dval; - break; - } - default: - { - val = (int64_t)dval; - } - } - } - else if (datatypes::isWideDecimalType( - r.getColType(colIdx), - r.getColumnWidth(colIdx))) - { - val = r.getTSInt128Field(colIdx).getValue(); - } - else - { - val = r.getIntField(colIdx); - } - - if (val > max) - max = val; - - if (val < min) - min = val; - } + if (datatypes::TCharShort::strnncollsp(cs, val, max, r.getColumnWidth(smallKeyColumns[col])) > 0 || + ((int64_t)max) == numeric_limits::min()) + { + max = val; + } } + else if (r.isUnsigned(colIdx)) + { + uint128_t uval; + + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) + { + double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); + switch (largeRG.getColType(largeKeyColumns[col])) + { + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + uval = *(uint64_t*)&dval; + break; + } + default: + { + uval = (uint64_t)dval; + } + } + } + else if (datatypes::isWideDecimalType(r.getColType(colIdx), r.getColumnWidth(colIdx))) + { + uval = r.getTSInt128Field(colIdx).getValue(); + } + else + { + uval = r.getUintField(colIdx); + } + + if (uval > static_cast(max)) + max = static_cast(uval); + + if (uval < static_cast(min)) + min = static_cast(uval); + } + else + { + int128_t val = 0; + + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) + { + double dval = (double)roundl(r.getLongDoubleField(colIdx)); + switch (largeRG.getColType(largeKeyColumns[col])) + { + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + val = *(int64_t*)&dval; + break; + } + default: + { + val = (int64_t)dval; + } + } + } + else if (datatypes::isWideDecimalType(r.getColType(colIdx), r.getColumnWidth(colIdx))) + { + val = r.getTSInt128Field(colIdx).getValue(); + } + else + { + val = r.getIntField(colIdx); + } + + if (val > max) + max = val; + + if (val < min) + min = val; + } + } } size_t TupleJoiner::size() const { - if (joinAlg == UM || joinAlg == INSERTING) - { - size_t ret = 0; - for (uint i = 0; i < bucketCount; i++) - if (UNLIKELY(typelessJoin)) - ret += ht[i]->size(); - else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - ret += ld[i]->size(); - else if (!smallRG.usesStringTable()) - ret += h[i]->size(); - else - ret += sth[i]->size(); - return ret; - } + if (joinAlg == UM || joinAlg == INSERTING) + { + size_t ret = 0; + for (uint i = 0; i < bucketCount; i++) + if (UNLIKELY(typelessJoin)) + ret += ht[i]->size(); + else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + ret += ld[i]->size(); + else if (!smallRG.usesStringTable()) + ret += h[i]->size(); + else + ret += sth[i]->size(); + return ret; + } - return rows.size(); + return rows.size(); } - class TypelessDataStringEncoder { - const uint8_t* mStr; - uint32_t mLength; -public: - TypelessDataStringEncoder(const uint8_t *str, uint32_t length) - :mStr(str), mLength(length) - { } - TypelessDataStringEncoder(const utils::ConstString &str) - :mStr((const uint8_t*) str.str()), mLength(str.length()) - { } - bool store(uint8_t* to, uint32_t& off, uint32_t keylen) const + const uint8_t* mStr; + uint32_t mLength; + + public: + TypelessDataStringEncoder(const uint8_t* str, uint32_t length) : mStr(str), mLength(length) + { + } + TypelessDataStringEncoder(const utils::ConstString& str) + : mStr((const uint8_t*)str.str()), mLength(str.length()) + { + } + bool store(uint8_t* to, uint32_t& off, uint32_t keylen) const + { + if (mLength > 0xFFFF) // We encode length into two bytes below { - if (mLength > 0xFFFF) // We encode length into two bytes below - { - throw runtime_error("Cannot join strings greater than 64KB"); - } - - if (off + mLength + 2 > keylen) - return true; - - to[off++]= mLength / 0xFF; - to[off++]= mLength % 0xFF; - /* - QQ: perhaps now when we put length, - we don't need to stop at '\0' bytes any more. - If so, the loop below can be replace to memcpy(). - */ - for (uint32_t j = 0; j < mLength && mStr[j] != 0; j++) - { - if (off >= keylen) - return true; - to[off++] = mStr[j]; - } - - return false; + throw runtime_error("Cannot join strings greater than 64KB"); } + + if (off + mLength + 2 > keylen) + return true; + + to[off++] = mLength / 0xFF; + to[off++] = mLength % 0xFF; + /* + QQ: perhaps now when we put length, + we don't need to stop at '\0' bytes any more. + If so, the loop below can be replace to memcpy(). + */ + for (uint32_t j = 0; j < mLength && mStr[j] != 0; j++) + { + if (off >= keylen) + return true; + to[off++] = mStr[j]; + } + + return false; + } }; class WideDecimalKeyConverter { - const Row* mR; - uint64_t convertedValue; - const uint32_t mKeyColId; - uint16_t width; - public: - WideDecimalKeyConverter(const Row& r, - const uint32_t keyColId): mR(&r), - mKeyColId(keyColId), - width(datatypes::MAXDECIMALWIDTH) - { } - bool isConvertedToSmallSideType() const { return width == datatypes::MAXLEGACYWIDTH; } - int64_t getConvertedTInt64() const { return (int64_t)convertedValue; } - // Returns true if the value doesn't fit into allowed range for a type. - template - bool numericRangeCheckAndConvert(const AT& value) - { - if (value > AT(std::numeric_limits::max()) || - value < AT(std::numeric_limits::min())) - return true; + const Row* mR; + uint64_t convertedValue; + const uint32_t mKeyColId; + uint16_t width; - convertedValue = (uint64_t) static_cast(value); - return false; - } - // As of MCS 6.x there is an asumption MCS can't join having - // INTEGER and non-INTEGER potentially fractional keys, - // e.g. BIGINT to DECIMAL(38,1). It can only join BIGINT to DECIMAL(38). - // convert() checks if wide-DECIMAL overflows INTEGER type range - // and sets internal width to 0 if it is. If not width is set to 8 - // and convertedValue is casted to INTEGER type. - // This convert() is called in EM to cast smallSide TypelessData - // if the key columns has a skew, e.g. INT to DECIMAL(38). - inline WideDecimalKeyConverter& - convert(const bool otherSideIsIntOrNarrow, - const execplan::CalpontSystemCatalog::ColDataType otherSideType) - { - if (otherSideIsIntOrNarrow) - { - datatypes::TSInt128 integralPart = mR->getTSInt128Field(mKeyColId); + public: + WideDecimalKeyConverter(const Row& r, const uint32_t keyColId) + : mR(&r), mKeyColId(keyColId), width(datatypes::MAXDECIMALWIDTH) + { + } + bool isConvertedToSmallSideType() const + { + return width == datatypes::MAXLEGACYWIDTH; + } + int64_t getConvertedTInt64() const + { + return (int64_t)convertedValue; + } + // Returns true if the value doesn't fit into allowed range for a type. + template + bool numericRangeCheckAndConvert(const AT& value) + { + if (value > AT(std::numeric_limits::max()) || value < AT(std::numeric_limits::min())) + return true; - bool isUnsigned = datatypes::isUnsigned(otherSideType); - if (isUnsigned) - { - width = (numericRangeCheckAndConvert(integralPart)) ? 0 : datatypes::MAXLEGACYWIDTH; - return *this; - } - width = (numericRangeCheckAndConvert(integralPart)) ? 0 : datatypes::MAXLEGACYWIDTH; - } + convertedValue = (uint64_t) static_cast(value); + return false; + } + // As of MCS 6.x there is an asumption MCS can't join having + // INTEGER and non-INTEGER potentially fractional keys, + // e.g. BIGINT to DECIMAL(38,1). It can only join BIGINT to DECIMAL(38). + // convert() checks if wide-DECIMAL overflows INTEGER type range + // and sets internal width to 0 if it is. If not width is set to 8 + // and convertedValue is casted to INTEGER type. + // This convert() is called in EM to cast smallSide TypelessData + // if the key columns has a skew, e.g. INT to DECIMAL(38). + inline WideDecimalKeyConverter& convert(const bool otherSideIsIntOrNarrow, + const execplan::CalpontSystemCatalog::ColDataType otherSideType) + { + if (otherSideIsIntOrNarrow) + { + datatypes::TSInt128 integralPart = mR->getTSInt128Field(mKeyColId); + + bool isUnsigned = datatypes::isUnsigned(otherSideType); + if (isUnsigned) + { + width = (numericRangeCheckAndConvert(integralPart)) ? 0 : datatypes::MAXLEGACYWIDTH; return *this; + } + width = (numericRangeCheckAndConvert(integralPart)) ? 0 : datatypes::MAXLEGACYWIDTH; } - // Stores the value that might had been converted. - inline bool store(TypelessData& typelessData, - uint32_t& off, - const uint32_t keylen) const + return *this; + } + // Stores the value that might had been converted. + inline bool store(TypelessData& typelessData, uint32_t& off, const uint32_t keylen) const + { + // A note from convert() if there is otherSide column type range + // overflow so store() returns TD with len=0. This tells EM to skip this + // key b/c it won't match at PP. This happens it is possible to skip + // smallSide TD but can't to do the same with largeSide b/c of OUTER joins. + if (!width) { - // A note from convert() if there is otherSide column type range - // overflow so store() returns TD with len=0. This tells EM to skip this - // key b/c it won't match at PP. This happens it is possible to skip - // smallSide TD but can't to do the same with largeSide b/c of OUTER joins. - if (!width) - { - typelessData.len = 0; - return true; - } - if (off + width > keylen) - return true; - switch (width) - { - case datatypes::MAXDECIMALWIDTH: - { - mR->storeInt128FieldIntoPtr(mKeyColId, &typelessData.data[off]); - break; - } - default: - { - datatypes::TUInt64(convertedValue).store(&typelessData.data[off]); - } - } - off += width; - return false; + typelessData.len = 0; + return true; } + if (off + width > keylen) + return true; + switch (width) + { + case datatypes::MAXDECIMALWIDTH: + { + mR->storeInt128FieldIntoPtr(mKeyColId, &typelessData.data[off]); + break; + } + default: + { + datatypes::TUInt64(convertedValue).store(&typelessData.data[off]); + } + } + off += width; + return false; + } }; // smallSideColWidths is non-nullptr valid pointer only // if there is a skew b/w small and large side columns widths. -uint32 TypelessData::hash(const RowGroup& r, - const std::vector& keyCols, +uint32 TypelessData::hash(const RowGroup& r, const std::vector& keyCols, const std::vector* smallSideKeyColumnsIds, const rowgroup::RowGroup* smallSideRG) const { - // This part is for largeSide hashing using Row at PP. - if (!isSmallSide()) + // This part is for largeSide hashing using Row at PP. + if (!isSmallSide()) + { + return mRowPtr->hashTypeless(keyCols, smallSideKeyColumnsIds, + (smallSideRG) ? &smallSideRG->getColWidths() : nullptr); + } + // This part is for smallSide hashing at PP. + TypelessDataDecoder decoder(*this); + datatypes::MariaDBHasher hasher; + for (auto keyColId : keyCols) + { + switch (r.getColTypes()[keyColId]) { - return mRowPtr->hashTypeless(keyCols, - smallSideKeyColumnsIds, - (smallSideRG) ? &smallSideRG->getColWidths() : nullptr); - } - // This part is for smallSide hashing at PP. - TypelessDataDecoder decoder(*this); - datatypes::MariaDBHasher hasher; - for (auto keyColId: keyCols) - { - switch (r.getColTypes()[keyColId]) + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + { + CHARSET_INFO* cs = const_cast(r).getCharset(keyColId); + hasher.add(cs, decoder.scanString()); + break; + } + case CalpontSystemCatalog::DECIMAL: + { + const uint32_t width = std::max(r.getColWidths()[keyColId], datatypes::MAXLEGACYWIDTH); + if (isSmallSideWithSkewedData() || width == datatypes::MAXLEGACYWIDTH) { - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - { - CHARSET_INFO *cs= const_cast(r).getCharset(keyColId); - hasher.add(cs, decoder.scanString()); - break; - } - case CalpontSystemCatalog::DECIMAL: - { - const uint32_t width = std::max(r.getColWidths()[keyColId], datatypes::MAXLEGACYWIDTH); - if (isSmallSideWithSkewedData() || width == datatypes::MAXLEGACYWIDTH) - { - int64_t val = decoder.scanTInt64(); - hasher.add(&my_charset_bin, reinterpret_cast(&val), datatypes::MAXLEGACYWIDTH); - } - else - hasher.add(&my_charset_bin, decoder.scanGeneric(width)); - break; - } - default: - { - hasher.add(&my_charset_bin, decoder.scanGeneric(datatypes::MAXLEGACYWIDTH)); - break; - } + int64_t val = decoder.scanTInt64(); + hasher.add(&my_charset_bin, reinterpret_cast(&val), datatypes::MAXLEGACYWIDTH); } + else + hasher.add(&my_charset_bin, decoder.scanGeneric(width)); + break; + } + default: + { + hasher.add(&my_charset_bin, decoder.scanGeneric(datatypes::MAXLEGACYWIDTH)); + break; + } } - return hasher.finalize(); + } + return hasher.finalize(); } // this is smallSide, Row represents largeSide record. -int TypelessData::cmpToRow(const RowGroup& r, - const std::vector& keyCols, - const rowgroup::Row &row, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) const +int TypelessData::cmpToRow(const RowGroup& r, const std::vector& keyCols, const rowgroup::Row& row, + const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) const { - TypelessDataDecoder a(*this); + TypelessDataDecoder a(*this); - for (uint32_t i = 0; i < keyCols.size(); i++) + for (uint32_t i = 0; i < keyCols.size(); i++) + { + auto largeSideKeyColRowIdx = keyCols[i]; + switch (r.getColType(largeSideKeyColRowIdx)) { - auto largeSideKeyColRowIdx = keyCols[i]; - switch (r.getColType(largeSideKeyColRowIdx)) + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + { + datatypes::Charset cs(*const_cast(r).getCharset(largeSideKeyColRowIdx)); + ConstString ta = a.scanString(); + ConstString tb = row.getConstString(largeSideKeyColRowIdx); + if (int rc = cs.strnncollsp(ta, tb)) + return rc; + break; + } + case CalpontSystemCatalog::DECIMAL: + { + auto largeSideWidth = row.getColumnWidth(largeSideKeyColRowIdx); + // First branch processes skewed JOIN, e.g. INT to DECIMAL(38) + // else branch processes decimal with common width at both small- and largeSide. + if (isSmallSideWithSkewedData() && + largeSideWidth != smallSideRG->getColumnWidth(smallSideKeyColumnsIds->operator[](i))) { - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - { - datatypes::Charset cs(*const_cast(r).getCharset(largeSideKeyColRowIdx)); - ConstString ta = a.scanString(); - ConstString tb = row.getConstString(largeSideKeyColRowIdx); - if (int rc= cs.strnncollsp(ta, tb)) - return rc; - break; - } - case CalpontSystemCatalog::DECIMAL: - { - auto largeSideWidth = row.getColumnWidth(largeSideKeyColRowIdx); - // First branch processes skewed JOIN, e.g. INT to DECIMAL(38) - // else branch processes decimal with common width at both small- and largeSide. - if (isSmallSideWithSkewedData() && - largeSideWidth != smallSideRG->getColumnWidth(smallSideKeyColumnsIds->operator[](i))) - { - if (largeSideWidth == datatypes::MAXLEGACYWIDTH) - { - if (int rc = a.scanTInt64() != row.getIntField(largeSideKeyColRowIdx)) - return rc; - } - else - { - WideDecimalKeyConverter cv(row, largeSideKeyColRowIdx); - if (!cv.convert(true, - smallSideRG->getColType(smallSideKeyColumnsIds->operator[](i))) - .isConvertedToSmallSideType()) - return 1; - if (int rc = a.scanTInt64() != cv.getConvertedTInt64()) - return rc; - } - } - else - { - // There is an assumption that both sides here are equal and are either 8 or 16 bytes. - if (largeSideWidth == datatypes::MAXDECIMALWIDTH) - { - if (int rc = a.scanTInt128() != row.getTSInt128Field(largeSideKeyColRowIdx)) - return rc; - } - else - { - if (int rc = a.scanTInt64() != row.getIntField(largeSideKeyColRowIdx)) - return rc; - } - } - break; - } - default: - { - ConstString ta = a.scanGeneric(datatypes::MAXLEGACYWIDTH); - if (r.isUnsigned(largeSideKeyColRowIdx)) - { - uint64_t tb = row.getUintField(largeSideKeyColRowIdx); - if (int rc = memcmp(ta.str(), &tb , datatypes::MAXLEGACYWIDTH)) - return rc; - } - else - { - int64_t tb = row.getIntField(largeSideKeyColRowIdx); - if (int rc = memcmp(ta.str(), &tb , datatypes::MAXLEGACYWIDTH)) - return rc; - } - break; - } - } - } - return 0; // Equal -} - -int TypelessData::cmp(const RowGroup& r, const std::vector& keyCols, - const TypelessData &da, const TypelessData &db, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) -{ - idbassert(da.isSmallSide() || db.isSmallSide()); - if (!da.isSmallSide() && db.isSmallSide()) - return -db.cmpToRow(r, keyCols, da.mRowPtr[0], smallSideKeyColumnsIds, smallSideRG); - if (da.isSmallSide() && !db.isSmallSide()) - return da.cmpToRow(r, keyCols, db.mRowPtr[0], smallSideKeyColumnsIds, smallSideRG); - - // This case happens in BPP::addToJoiner when it populates the final - // hashmap with multiple smallSide TDs from temp hashmaps. - idbassert(da.isSmallSide() && db.isSmallSide()); - - TypelessDataDecoder a(da); - TypelessDataDecoder b(db); - - for (uint32_t i = 0; i < keyCols.size(); ++i) - { - auto keyColIdx = keyCols[i]; - switch (r.getColTypes()[keyColIdx]) - { - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::TEXT: - { - datatypes::Charset cs(*const_cast(r).getCharset(keyColIdx)); - ConstString ta = a.scanString(); - ConstString tb = b.scanString(); - if (int rc= cs.strnncollsp(ta, tb)) - return rc; - break; - } - case CalpontSystemCatalog::DECIMAL: - { - auto largeSideWidth = r.getColumnWidth(keyColIdx); - // First and second branches processes skewed JOIN, e.g. INT to DECIMAL(38) - // Third processes decimal with common width at both small- and largeSide. - auto width = (da.isSmallSideWithSkewedData() && - largeSideWidth != smallSideRG->getColumnWidth(smallSideKeyColumnsIds->operator[](i))) ? datatypes::MAXLEGACYWIDTH : std::max(r.getColWidths()[keyColIdx], datatypes::MAXLEGACYWIDTH); - ConstString ta = a.scanGeneric(width); - ConstString tb = b.scanGeneric(width); - if (int rc= memcmp(ta.str(), tb.str(), width)) - return rc; - break; - } - default: - { - ConstString ta = a.scanGeneric(datatypes::MAXLEGACYWIDTH); - ConstString tb = b.scanGeneric(datatypes::MAXLEGACYWIDTH); - idbassert(ta.length() == tb.length()); - // It is impossible to join signed to unsigned types now - // but there is a potential error, e.g. uint64 vs negative int64. - if (int rc= memcmp(ta.str(), tb.str() , ta.length())) - return rc; - break; - } - } - } - return 0; // Equal -} - - - -// Called in joblist code to produce SmallSide TypelessData to be sent to PP. -TypelessData makeTypelessKey(const Row& r, const vector& keyCols, - uint32_t keylen, FixedAllocator* fa, - const rowgroup::RowGroup& otherSideRG, - const std::vector& otherKeyCols) -{ - TypelessData ret; - uint32_t off = 0, i; - execplan::CalpontSystemCatalog::ColDataType type; - - ret.data = (uint8_t*) fa->allocate(); - idbassert(keyCols.size() == otherKeyCols.size()); - - for (i = 0; i < keyCols.size(); i++) - { - type = r.getColTypes()[keyCols[i]]; - - if (datatypes::isCharType(type)) - { - // this is a string, copy a normalized version - const utils::ConstString str = r.getConstString(keyCols[i]); - if (TypelessDataStringEncoder(str).store(ret.data, off, keylen)) - goto toolong; - } - else if (datatypes::isWideDecimalType(type, r.getColumnWidth(keyCols[i]))) - { - bool otherSideIsIntOrNarrow = otherSideRG.getColumnWidth(otherKeyCols[i]) <= datatypes::MAXLEGACYWIDTH; - // useless if otherSideIsInt is false - auto otherSideType = (otherSideIsIntOrNarrow) ? otherSideRG.getColType(otherKeyCols[i]) - : datatypes::SystemCatalog::UNDEFINED; - if (WideDecimalKeyConverter(r, keyCols[i]).convert(otherSideIsIntOrNarrow, otherSideType) - .store(ret, off, keylen)) - { - goto toolong; - } - } - else if (datatypes::isLongDouble(type)) - { - if (off + sizeof(long double) > keylen) - goto toolong; - // Small side is a long double. Since CS can't store larger than DOUBLE, - // we need to convert to whatever type large side is -- double or int64 - long double keyld = r.getLongDoubleField(keyCols[i]); - switch (otherSideRG.getColType(otherKeyCols[i])) - { - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - if (off + 8 > keylen) - goto toolong; - if (keyld > MAX_DOUBLE || keyld < MIN_DOUBLE) - { - ret.len = 0; - return ret; - } - else - { - double d = (double)keyld; - *((int64_t*) &ret.data[off]) = *(int64_t*)&d; - } - break; - } - case CalpontSystemCatalog::LONGDOUBLE: - { - if (off + sizeof(long double) > keylen) - goto toolong; - *((long double*) &ret.data[off]) = keyld; - off += sizeof(long double); - break; - } - default: - { - if (off + 8 > keylen) - goto toolong; - if (r.isUnsigned(keyCols[i]) && keyld > MAX_UBIGINT) - { - ret.len = 0; - return ret; - } - else if (keyld > MAX_BIGINT || keyld < MIN_BIGINT) - { - ret.len = 0; - return ret; - } - else - { - *((int64_t*) &ret.data[off]) = (int64_t)keyld; - off += 8; - } - break; - } - } - } - else if (r.isUnsigned(keyCols[i])) - { - if (off + 8 > keylen) - goto toolong; - *((uint64_t*) &ret.data[off]) = r.getUintField(keyCols[i]); - off += 8; + if (largeSideWidth == datatypes::MAXLEGACYWIDTH) + { + if (int rc = a.scanTInt64() != row.getIntField(largeSideKeyColRowIdx)) + return rc; + } + else + { + WideDecimalKeyConverter cv(row, largeSideKeyColRowIdx); + if (!cv.convert(true, smallSideRG->getColType(smallSideKeyColumnsIds->operator[](i))) + .isConvertedToSmallSideType()) + return 1; + if (int rc = a.scanTInt64() != cv.getConvertedTInt64()) + return rc; + } } else { - if (off + 8 > keylen) - goto toolong; - *((int64_t*) &ret.data[off]) = r.getIntField(keyCols[i]); - off += 8; + // There is an assumption that both sides here are equal and are either 8 or 16 bytes. + if (largeSideWidth == datatypes::MAXDECIMALWIDTH) + { + if (int rc = a.scanTInt128() != row.getTSInt128Field(largeSideKeyColRowIdx)) + return rc; + } + else + { + if (int rc = a.scanTInt64() != row.getIntField(largeSideKeyColRowIdx)) + return rc; + } } + break; + } + default: + { + ConstString ta = a.scanGeneric(datatypes::MAXLEGACYWIDTH); + if (r.isUnsigned(largeSideKeyColRowIdx)) + { + uint64_t tb = row.getUintField(largeSideKeyColRowIdx); + if (int rc = memcmp(ta.str(), &tb, datatypes::MAXLEGACYWIDTH)) + return rc; + } + else + { + int64_t tb = row.getIntField(largeSideKeyColRowIdx); + if (int rc = memcmp(ta.str(), &tb, datatypes::MAXLEGACYWIDTH)) + return rc; + } + break; + } } + } + return 0; // Equal +} - ret.len = off; - fa->truncateBy(keylen - off); - return ret; +int TypelessData::cmp(const RowGroup& r, const std::vector& keyCols, const TypelessData& da, + const TypelessData& db, const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) +{ + idbassert(da.isSmallSide() || db.isSmallSide()); + if (!da.isSmallSide() && db.isSmallSide()) + return -db.cmpToRow(r, keyCols, da.mRowPtr[0], smallSideKeyColumnsIds, smallSideRG); + if (da.isSmallSide() && !db.isSmallSide()) + return da.cmpToRow(r, keyCols, db.mRowPtr[0], smallSideKeyColumnsIds, smallSideRG); + + // This case happens in BPP::addToJoiner when it populates the final + // hashmap with multiple smallSide TDs from temp hashmaps. + idbassert(da.isSmallSide() && db.isSmallSide()); + + TypelessDataDecoder a(da); + TypelessDataDecoder b(db); + + for (uint32_t i = 0; i < keyCols.size(); ++i) + { + auto keyColIdx = keyCols[i]; + switch (r.getColTypes()[keyColIdx]) + { + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + { + datatypes::Charset cs(*const_cast(r).getCharset(keyColIdx)); + ConstString ta = a.scanString(); + ConstString tb = b.scanString(); + if (int rc = cs.strnncollsp(ta, tb)) + return rc; + break; + } + case CalpontSystemCatalog::DECIMAL: + { + auto largeSideWidth = r.getColumnWidth(keyColIdx); + // First and second branches processes skewed JOIN, e.g. INT to DECIMAL(38) + // Third processes decimal with common width at both small- and largeSide. + auto width = (da.isSmallSideWithSkewedData() && + largeSideWidth != smallSideRG->getColumnWidth(smallSideKeyColumnsIds->operator[](i))) + ? datatypes::MAXLEGACYWIDTH + : std::max(r.getColWidths()[keyColIdx], datatypes::MAXLEGACYWIDTH); + ConstString ta = a.scanGeneric(width); + ConstString tb = b.scanGeneric(width); + if (int rc = memcmp(ta.str(), tb.str(), width)) + return rc; + break; + } + default: + { + ConstString ta = a.scanGeneric(datatypes::MAXLEGACYWIDTH); + ConstString tb = b.scanGeneric(datatypes::MAXLEGACYWIDTH); + idbassert(ta.length() == tb.length()); + // It is impossible to join signed to unsigned types now + // but there is a potential error, e.g. uint64 vs negative int64. + if (int rc = memcmp(ta.str(), tb.str(), ta.length())) + return rc; + break; + } + } + } + return 0; // Equal +} + +// Called in joblist code to produce SmallSide TypelessData to be sent to PP. +TypelessData makeTypelessKey(const Row& r, const vector& keyCols, uint32_t keylen, + FixedAllocator* fa, const rowgroup::RowGroup& otherSideRG, + const std::vector& otherKeyCols) +{ + TypelessData ret; + uint32_t off = 0, i; + execplan::CalpontSystemCatalog::ColDataType type; + + ret.data = (uint8_t*)fa->allocate(); + idbassert(keyCols.size() == otherKeyCols.size()); + + for (i = 0; i < keyCols.size(); i++) + { + type = r.getColTypes()[keyCols[i]]; + + if (datatypes::isCharType(type)) + { + // this is a string, copy a normalized version + const utils::ConstString str = r.getConstString(keyCols[i]); + if (TypelessDataStringEncoder(str).store(ret.data, off, keylen)) + goto toolong; + } + else if (datatypes::isWideDecimalType(type, r.getColumnWidth(keyCols[i]))) + { + bool otherSideIsIntOrNarrow = otherSideRG.getColumnWidth(otherKeyCols[i]) <= datatypes::MAXLEGACYWIDTH; + // useless if otherSideIsInt is false + auto otherSideType = (otherSideIsIntOrNarrow) ? otherSideRG.getColType(otherKeyCols[i]) + : datatypes::SystemCatalog::UNDEFINED; + if (WideDecimalKeyConverter(r, keyCols[i]) + .convert(otherSideIsIntOrNarrow, otherSideType) + .store(ret, off, keylen)) + { + goto toolong; + } + } + else if (datatypes::isLongDouble(type)) + { + if (off + sizeof(long double) > keylen) + goto toolong; + // Small side is a long double. Since CS can't store larger than DOUBLE, + // we need to convert to whatever type large side is -- double or int64 + long double keyld = r.getLongDoubleField(keyCols[i]); + switch (otherSideRG.getColType(otherKeyCols[i])) + { + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + if (off + 8 > keylen) + goto toolong; + if (keyld > MAX_DOUBLE || keyld < MIN_DOUBLE) + { + ret.len = 0; + return ret; + } + else + { + double d = (double)keyld; + *((int64_t*)&ret.data[off]) = *(int64_t*)&d; + } + break; + } + case CalpontSystemCatalog::LONGDOUBLE: + { + if (off + sizeof(long double) > keylen) + goto toolong; + *((long double*)&ret.data[off]) = keyld; + off += sizeof(long double); + break; + } + default: + { + if (off + 8 > keylen) + goto toolong; + if (r.isUnsigned(keyCols[i]) && keyld > MAX_UBIGINT) + { + ret.len = 0; + return ret; + } + else if (keyld > MAX_BIGINT || keyld < MIN_BIGINT) + { + ret.len = 0; + return ret; + } + else + { + *((int64_t*)&ret.data[off]) = (int64_t)keyld; + off += 8; + } + break; + } + } + } + else if (r.isUnsigned(keyCols[i])) + { + if (off + 8 > keylen) + goto toolong; + *((uint64_t*)&ret.data[off]) = r.getUintField(keyCols[i]); + off += 8; + } + else + { + if (off + 8 > keylen) + goto toolong; + *((int64_t*)&ret.data[off]) = r.getIntField(keyCols[i]); + off += 8; + } + } + + ret.len = off; + fa->truncateBy(keylen - off); + return ret; toolong: - fa->truncateBy(keylen); - ret.len = 0; - return ret; + fa->truncateBy(keylen); + ret.len = 0; + return ret; } // The method is used by disk-based JOIN and it is not collation or wide DECIMAL aware. uint64_t getHashOfTypelessKey(const Row& r, const vector& keyCols, uint32_t seed) { - Hasher_r hasher; - uint64_t ret = seed, tmp; - uint32_t i; - uint32_t width = 0; - char nullChar = '\0'; - execplan::CalpontSystemCatalog::ColDataType type; + Hasher_r hasher; + uint64_t ret = seed, tmp; + uint32_t i; + uint32_t width = 0; + char nullChar = '\0'; + execplan::CalpontSystemCatalog::ColDataType type; - for (i = 0; i < keyCols.size(); i++) + for (i = 0; i < keyCols.size(); i++) + { + type = r.getColTypes()[keyCols[i]]; + + if (type == CalpontSystemCatalog::VARCHAR || type == CalpontSystemCatalog::CHAR || + type == CalpontSystemCatalog::TEXT) { - type = r.getColTypes()[keyCols[i]]; - - if (type == CalpontSystemCatalog::VARCHAR || - type == CalpontSystemCatalog::CHAR || - type == CalpontSystemCatalog::TEXT) - { - // this is a string, copy a normalized version - const utils::ConstString str = r.getConstString(keyCols[i]); - ret = hasher(str.str(), str.length(), ret); - ret = hasher(&nullChar, 1, ret); - width += str.length() + 1; - } - else if (r.getColType(keyCols[i]) == CalpontSystemCatalog::LONGDOUBLE) - { - long double tmp = r.getLongDoubleField(keyCols[i]); - ret = hasher((char*) &tmp, sizeof(long double), ret); - width += sizeof(long double); - } - else - if (r.isUnsigned(keyCols[i])) - { - tmp = r.getUintField(keyCols[i]); - ret = hasher((char*) &tmp, 8, ret); - width += 8; - } - else - { - tmp = r.getIntField(keyCols[i]); - ret = hasher((char*) &tmp, 8, ret); - width += 8; - } + // this is a string, copy a normalized version + const utils::ConstString str = r.getConstString(keyCols[i]); + ret = hasher(str.str(), str.length(), ret); + ret = hasher(&nullChar, 1, ret); + width += str.length() + 1; } + else if (r.getColType(keyCols[i]) == CalpontSystemCatalog::LONGDOUBLE) + { + long double tmp = r.getLongDoubleField(keyCols[i]); + ret = hasher((char*)&tmp, sizeof(long double), ret); + width += sizeof(long double); + } + else if (r.isUnsigned(keyCols[i])) + { + tmp = r.getUintField(keyCols[i]); + ret = hasher((char*)&tmp, 8, ret); + width += 8; + } + else + { + tmp = r.getIntField(keyCols[i]); + ret = hasher((char*)&tmp, 8, ret); + width += 8; + } + } - ret = hasher.finalize(ret, width); - return ret; + ret = hasher.finalize(ret, width); + return ret; } string TypelessData::toString() const { - uint32_t i; - ostringstream os; + uint32_t i; + ostringstream os; - os << hex; + os << hex; - for (i = 0; i < len; i++) - { - os << (uint32_t) data[i] << " "; - } + for (i = 0; i < len; i++) + { + os << (uint32_t)data[i] << " "; + } - os << dec; - return os.str(); + os << dec; + return os.str(); } void TypelessData::serialize(messageqcpp::ByteStream& b) const { - b << len; - b.append(data, len); - // Flags are not send b/c they are locally significant now. + b << len; + b.append(data, len); + // Flags are not send b/c they are locally significant now. } void TypelessData::deserialize(messageqcpp::ByteStream& b, utils::PoolAllocator& fa) { - b >> len; - data = (uint8_t*) fa.allocate(len); - memcpy(data, b.buf(), len); - b.advance(len); + b >> len; + data = (uint8_t*)fa.allocate(len); + memcpy(data, b.buf(), len); + b.advance(len); } bool TupleJoiner::hasNullJoinColumn(const Row& r) const { - uint64_t key; + uint64_t key; - for (uint32_t i = 0; i < largeKeyColumns.size(); i++) + for (uint32_t i = 0; i < largeKeyColumns.size(); i++) + { + if (r.isNullValue(largeKeyColumns[i])) + return true; + + if (UNLIKELY(bSignedUnsignedJoin)) { - if (r.isNullValue(largeKeyColumns[i])) - return true; + // BUG 5628 If this is a signed/unsigned join column and the sign bit is set on either + // side, then this row should not compare. Treat as NULL to prevent compare, even if + // the bit patterns match. + if (smallRG.isUnsigned(smallKeyColumns[i]) != largeRG.isUnsigned(largeKeyColumns[i])) + { + if (r.isUnsigned(largeKeyColumns[i])) + key = r.getUintField(largeKeyColumns[i]); // Does not propogate sign bit + else + key = r.getIntField(largeKeyColumns[i]); // Propogates sign bit - if (UNLIKELY(bSignedUnsignedJoin)) + if (key & 0x8000000000000000ULL) { - // BUG 5628 If this is a signed/unsigned join column and the sign bit is set on either - // side, then this row should not compare. Treat as NULL to prevent compare, even if - // the bit patterns match. - if (smallRG.isUnsigned(smallKeyColumns[i]) != largeRG.isUnsigned(largeKeyColumns[i])) - { - if (r.isUnsigned(largeKeyColumns[i])) - key = r.getUintField(largeKeyColumns[i]); // Does not propogate sign bit - else - key = r.getIntField(largeKeyColumns[i]); // Propogates sign bit - - if (key & 0x8000000000000000ULL) - { - return true; - } - } + return true; } + } } + } - return false; + return false; } string TupleJoiner::getTableName() const { - return tableName; + return tableName; } void TupleJoiner::setTableName(const string& tname) { - tableName = tname; + tableName = tname; } /* Disk based join support */ void TupleJoiner::clearData() { - _pool.reset(new boost::shared_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + if (typelessJoin) + ht.reset(new boost::scoped_ptr[bucketCount]); + else if (smallRG.getColTypes()[smallKeyColumns[0]] == CalpontSystemCatalog::LONGDOUBLE) + ld.reset(new boost::scoped_ptr[bucketCount]); + else if (smallRG.usesStringTable()) + sth.reset(new boost::scoped_ptr[bucketCount]); + else + h.reset(new boost::scoped_ptr[bucketCount]); + + for (uint i = 0; i < bucketCount; i++) + { + STLPoolAllocator> alloc; + _pool[i] = alloc.getPoolAllocator(); if (typelessJoin) - ht.reset(new boost::scoped_ptr[bucketCount]); + ht[i].reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); else if (smallRG.getColTypes()[smallKeyColumns[0]] == CalpontSystemCatalog::LONGDOUBLE) - ld.reset(new boost::scoped_ptr[bucketCount]); + ld[i].reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); else if (smallRG.usesStringTable()) - sth.reset(new boost::scoped_ptr[bucketCount]); + sth[i].reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); else - h.reset(new boost::scoped_ptr[bucketCount]); + h[i].reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); + } - for (uint i = 0; i < bucketCount; i++) - { - STLPoolAllocator > alloc; - _pool[i] = alloc.getPoolAllocator(); - if (typelessJoin) - ht[i].reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); - else if (smallRG.getColTypes()[smallKeyColumns[0]] == CalpontSystemCatalog::LONGDOUBLE) - ld[i].reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); - else if (smallRG.usesStringTable()) - sth[i].reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); - else - h[i].reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); - } - - std::vector empty; - rows.swap(empty); - finished = false; + std::vector empty; + rows.swap(empty); + finished = false; } boost::shared_ptr TupleJoiner::copyForDiskJoin() { - boost::shared_ptr ret(new TupleJoiner()); + boost::shared_ptr ret(new TupleJoiner()); - ret->smallRG = smallRG; - ret->largeRG = largeRG; - ret->smallNullMemory = smallNullMemory; - ret->smallNullRow = smallNullRow; - ret->joinType = joinType; - ret->tableName = tableName; - ret->typelessJoin = typelessJoin; - ret->smallKeyColumns = smallKeyColumns; - ret->largeKeyColumns = largeKeyColumns; - ret->keyLength = keyLength; - ret->bSignedUnsignedJoin = bSignedUnsignedJoin; - ret->fe = fe; + ret->smallRG = smallRG; + ret->largeRG = largeRG; + ret->smallNullMemory = smallNullMemory; + ret->smallNullRow = smallNullRow; + ret->joinType = joinType; + ret->tableName = tableName; + ret->typelessJoin = typelessJoin; + ret->smallKeyColumns = smallKeyColumns; + ret->largeKeyColumns = largeKeyColumns; + ret->keyLength = keyLength; + ret->bSignedUnsignedJoin = bSignedUnsignedJoin; + ret->fe = fe; - ret->nullValueForJoinColumn = nullValueForJoinColumn; - ret->uniqueLimit = uniqueLimit; + ret->nullValueForJoinColumn = nullValueForJoinColumn; + ret->uniqueLimit = uniqueLimit; - ret->discreteValues.reset(new bool[smallKeyColumns.size()]); - ret->cpValues.reset(new vector[smallKeyColumns.size()]); + ret->discreteValues.reset(new bool[smallKeyColumns.size()]); + ret->cpValues.reset(new vector[smallKeyColumns.size()]); - for (uint32_t i = 0; i < smallKeyColumns.size(); i++) + for (uint32_t i = 0; i < smallKeyColumns.size(); i++) + { + ret->discreteValues[i] = false; + if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - ret->discreteValues[i] = false; - if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) - { - if (datatypes::isWideDecimalType( - smallRG.getColType(smallKeyColumns[i]), - smallRG.getColumnWidth(smallKeyColumns[i]))) - { - ret->cpValues[i].push_back((int128_t) -1); - ret->cpValues[i].push_back(0); - } - else - { - ret->cpValues[i].push_back((int128_t) numeric_limits::max()); - ret->cpValues[i].push_back(0); - } - } - else - { - if (datatypes::isWideDecimalType( - smallRG.getColType(smallKeyColumns[i]), - smallRG.getColumnWidth(smallKeyColumns[i]))) - { - ret->cpValues[i].push_back(utils::maxInt128); - ret->cpValues[i].push_back(utils::minInt128); - } - else - { - ret->cpValues[i].push_back(numeric_limits::max()); - ret->cpValues[i].push_back(numeric_limits::min()); - } - } + if (datatypes::isWideDecimalType(smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + ret->cpValues[i].push_back((int128_t)-1); + ret->cpValues[i].push_back(0); + } + else + { + ret->cpValues[i].push_back((int128_t)numeric_limits::max()); + ret->cpValues[i].push_back(0); + } } - - if (typelessJoin) + else { - ret->storedKeyAlloc.reset(new FixedAllocator[numCores]); - for (int i = 0; i < numCores; i++) - ret->storedKeyAlloc[i].setAllocSize(keyLength); + if (datatypes::isWideDecimalType(smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + ret->cpValues[i].push_back(utils::maxInt128); + ret->cpValues[i].push_back(utils::minInt128); + } + else + { + ret->cpValues[i].push_back(numeric_limits::max()); + ret->cpValues[i].push_back(numeric_limits::min()); + } } + } - ret->numCores = numCores; - ret->bucketCount = bucketCount; - ret->bucketMask = bucketMask; - ret->jobstepThreadPool = jobstepThreadPool; + if (typelessJoin) + { + ret->storedKeyAlloc.reset(new FixedAllocator[numCores]); + for (int i = 0; i < numCores; i++) + ret->storedKeyAlloc[i].setAllocSize(keyLength); + } - ret->setThreadCount(1); - ret->clearData(); - ret->setInUM(); - return ret; + ret->numCores = numCores; + ret->bucketCount = bucketCount; + ret->bucketMask = bucketMask; + ret->jobstepThreadPool = jobstepThreadPool; + + ret->setThreadCount(1); + ret->clearData(); + ret->setInUM(); + return ret; } // Used for Typeless JOIN to detect if there is a JOIN when largeSide is wide-DECIMAL and // smallSide is a smaller data type, e.g. INT or narrow-DECIMAL. bool TupleJoiner::joinHasSkewedKeyColumn() { - std::vector::const_iterator largeSideKeyColumnsIter = getLargeKeyColumns().begin(); - std::vector::const_iterator smallSideKeyColumnsIter = getSmallKeyColumns().begin(); - idbassert(getLargeKeyColumns().size() == getSmallKeyColumns().size()); - while (largeSideKeyColumnsIter != getLargeKeyColumns().end()) + std::vector::const_iterator largeSideKeyColumnsIter = getLargeKeyColumns().begin(); + std::vector::const_iterator smallSideKeyColumnsIter = getSmallKeyColumns().begin(); + idbassert(getLargeKeyColumns().size() == getSmallKeyColumns().size()); + while (largeSideKeyColumnsIter != getLargeKeyColumns().end()) + { + auto smallSideColumnWidth = smallRG.getColumnWidth(*smallSideKeyColumnsIter); + auto largeSideColumnWidth = largeRG.getColumnWidth(*largeSideKeyColumnsIter); + bool widthIsDifferent = smallSideColumnWidth != largeSideColumnWidth; + if (widthIsDifferent && + (datatypes::isWideDecimalType(smallRG.getColTypes()[*smallSideKeyColumnsIter], + smallSideColumnWidth) || + datatypes::isWideDecimalType(largeRG.getColTypes()[*largeSideKeyColumnsIter], largeSideColumnWidth))) { - auto smallSideColumnWidth = smallRG.getColumnWidth(*smallSideKeyColumnsIter); - auto largeSideColumnWidth = largeRG.getColumnWidth(*largeSideKeyColumnsIter); - bool widthIsDifferent = smallSideColumnWidth != largeSideColumnWidth; - if (widthIsDifferent && (datatypes::isWideDecimalType(smallRG.getColTypes()[*smallSideKeyColumnsIter], smallSideColumnWidth) || - datatypes::isWideDecimalType(largeRG.getColTypes()[*largeSideKeyColumnsIter], largeSideColumnWidth))) - { - return true; - } - ++largeSideKeyColumnsIter; - ++smallSideKeyColumnsIter; + return true; } - return false; + ++largeSideKeyColumnsIter; + ++smallSideKeyColumnsIter; + } + return false; } void TupleJoiner::setConvertToDiskJoin() { - _convertToDiskJoin = true; + _convertToDiskJoin = true; } // The method is made to reuse the code from Typeless TupleJoiner ctor. @@ -1936,47 +1935,46 @@ uint32_t calculateKeyLength(const std::vector& aKeyColumnsIds, const std::vector* aLargeKeyColumnsIds, const rowgroup::RowGroup* aLargeRowGroup) { - uint32_t keyLength = 0; - for (size_t keyColumnIdx = 0; keyColumnIdx < aKeyColumnsIds.size(); ++keyColumnIdx) + uint32_t keyLength = 0; + for (size_t keyColumnIdx = 0; keyColumnIdx < aKeyColumnsIds.size(); ++keyColumnIdx) + { + auto smallSideKeyColumnId = aKeyColumnsIds[keyColumnIdx]; + auto largeSideKeyColumnId = (aLargeRowGroup) ? aLargeKeyColumnsIds->operator[](keyColumnIdx) + : std::numeric_limits::max(); + const auto& smallKeyColumnType = aSmallRowGroup.getColTypes()[smallSideKeyColumnId]; + // Not used if aLargeRowGroup is 0 that happens in PrimProc. + const auto& largeKeyColumntype = (aLargeRowGroup) ? aLargeRowGroup->getColTypes()[largeSideKeyColumnId] + : datatypes::SystemCatalog::UNDEFINED; + if (datatypes::isCharType(smallKeyColumnType)) { - auto smallSideKeyColumnId = aKeyColumnsIds[keyColumnIdx]; - auto largeSideKeyColumnId = (aLargeRowGroup) - ? aLargeKeyColumnsIds->operator[](keyColumnIdx) - : std::numeric_limits::max(); - const auto& smallKeyColumnType = aSmallRowGroup.getColTypes()[smallSideKeyColumnId]; - // Not used if aLargeRowGroup is 0 that happens in PrimProc. - const auto& largeKeyColumntype = (aLargeRowGroup) ? aLargeRowGroup->getColTypes()[largeSideKeyColumnId] - : datatypes::SystemCatalog::UNDEFINED; - if (datatypes::isCharType(smallKeyColumnType)) - { - keyLength += aSmallRowGroup.getColumnWidth(smallSideKeyColumnId) + 2; // +2 for encoded length + keyLength += aSmallRowGroup.getColumnWidth(smallSideKeyColumnId) + 2; // +2 for encoded length - // MCOL-698: if we don't do this LONGTEXT allocates 32TB RAM - if (keyLength > 65536) - return 65536; - } - else if (datatypes::isLongDouble(smallKeyColumnType)) - { - keyLength += sizeof(long double); - } - else if (datatypes::isWideDecimalType(smallKeyColumnType, - aSmallRowGroup.getColumnWidth(smallSideKeyColumnId))) - { - keyLength += (aLargeRowGroup && - !datatypes::isWideDecimalType(largeKeyColumntype, - aLargeRowGroup->getColumnWidth(smallSideKeyColumnId))) - ? datatypes::MAXLEGACYWIDTH // Small=Wide, Large=Narrow/xINT - : datatypes::MAXDECIMALWIDTH; // Small=Wide, Large=Wide - } - else - // The branch covers all datatypes left including skewed DECIMAL JOIN case - // Small=Wide, Large=Narrow - { - keyLength += datatypes::MAXLEGACYWIDTH; - } + // MCOL-698: if we don't do this LONGTEXT allocates 32TB RAM + if (keyLength > 65536) + return 65536; } + else if (datatypes::isLongDouble(smallKeyColumnType)) + { + keyLength += sizeof(long double); + } + else if (datatypes::isWideDecimalType(smallKeyColumnType, + aSmallRowGroup.getColumnWidth(smallSideKeyColumnId))) + { + keyLength += (aLargeRowGroup && + !datatypes::isWideDecimalType(largeKeyColumntype, + aLargeRowGroup->getColumnWidth(smallSideKeyColumnId))) + ? datatypes::MAXLEGACYWIDTH // Small=Wide, Large=Narrow/xINT + : datatypes::MAXDECIMALWIDTH; // Small=Wide, Large=Wide + } + else + // The branch covers all datatypes left including skewed DECIMAL JOIN case + // Small=Wide, Large=Narrow + { + keyLength += datatypes::MAXLEGACYWIDTH; + } + } - return keyLength; + return keyLength; } -}; +}; // namespace joiner diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index 9106be3d0..a69583815 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -44,144 +44,142 @@ namespace joiner { - -uint32_t calculateKeyLength(const std::vector& aKeyColumnsIds, - const rowgroup::RowGroup& aRowGroup, +uint32_t calculateKeyLength(const std::vector& aKeyColumnsIds, const rowgroup::RowGroup& aRowGroup, const std::vector* aLargeKeyColumnsIds = nullptr, const rowgroup::RowGroup* aLargeRowGroup = nullptr); -constexpr uint8_t IS_SMALLSIDE = 0x01; // SmallSide of a JOIN w/o a skew in key columns widths -constexpr uint8_t IS_SMALLSIDE_SKEWED = 0x02; // SmallSide of a JOIN with a skew in key cols widths +constexpr uint8_t IS_SMALLSIDE = 0x01; // SmallSide of a JOIN w/o a skew in key columns widths +constexpr uint8_t IS_SMALLSIDE_SKEWED = 0x02; // SmallSide of a JOIN with a skew in key cols widths class TypelessDataDecoder; class TypelessData { -public: - union { - uint8_t* data; - const rowgroup::Row *mRowPtr; - }; - uint32_t len; - // The flags are locally significant in PP now so serialize doesn't send it over the wire. - uint32_t mFlags; + public: + union + { + uint8_t* data; + const rowgroup::Row* mRowPtr; + }; + uint32_t len; + // The flags are locally significant in PP now so serialize doesn't send it over the wire. + uint32_t mFlags; - TypelessData() : data(nullptr), len(0), mFlags(0) { } - TypelessData(const rowgroup::Row *rowPtr) : mRowPtr(rowPtr), len(0), mFlags(0) { } - TypelessData(messageqcpp::ByteStream& bs, utils::PoolAllocator& memAllocator) : data(nullptr), len(0), mFlags(0) - { - deserialize(bs, memAllocator); - } - inline bool operator==(const TypelessData&) const; - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&, utils::FixedAllocator&); - void deserialize(messageqcpp::ByteStream&, utils::PoolAllocator&); - std::string toString() const; - uint32_t hash(const rowgroup::RowGroup&, - const std::vector& keyCols, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) const; - static int cmp(const rowgroup::RowGroup&, - const std::vector& keyCols, - const TypelessData &a, - const TypelessData &b, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG); - int cmpToRow(const rowgroup::RowGroup& r, - const std::vector& keyCols, - const rowgroup::Row &row, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) const; - inline void setSmallSide() - { - mFlags |= IS_SMALLSIDE; - } - inline void setSmallSideWithSkewedData() - { - mFlags |= IS_SMALLSIDE_SKEWED; - } - inline bool isSmallSide() const - { - return mFlags & (IS_SMALLSIDE_SKEWED | IS_SMALLSIDE); - } - inline bool isSmallSideWithSkewedData() const - { - return mFlags & IS_SMALLSIDE_SKEWED; - } + TypelessData() : data(nullptr), len(0), mFlags(0) + { + } + TypelessData(const rowgroup::Row* rowPtr) : mRowPtr(rowPtr), len(0), mFlags(0) + { + } + TypelessData(messageqcpp::ByteStream& bs, utils::PoolAllocator& memAllocator) + : data(nullptr), len(0), mFlags(0) + { + deserialize(bs, memAllocator); + } + inline bool operator==(const TypelessData&) const; + void serialize(messageqcpp::ByteStream&) const; + void deserialize(messageqcpp::ByteStream&, utils::FixedAllocator&); + void deserialize(messageqcpp::ByteStream&, utils::PoolAllocator&); + std::string toString() const; + uint32_t hash(const rowgroup::RowGroup&, const std::vector& keyCols, + const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) const; + static int cmp(const rowgroup::RowGroup&, const std::vector& keyCols, const TypelessData& a, + const TypelessData& b, const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG); + int cmpToRow(const rowgroup::RowGroup& r, const std::vector& keyCols, const rowgroup::Row& row, + const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) const; + inline void setSmallSide() + { + mFlags |= IS_SMALLSIDE; + } + inline void setSmallSideWithSkewedData() + { + mFlags |= IS_SMALLSIDE_SKEWED; + } + inline bool isSmallSide() const + { + return mFlags & (IS_SMALLSIDE_SKEWED | IS_SMALLSIDE); + } + inline bool isSmallSideWithSkewedData() const + { + return mFlags & IS_SMALLSIDE_SKEWED; + } }; // This operator is used in EM only so it doesn't support TD cmp operation // using Row pointers. inline bool TypelessData::operator==(const TypelessData& t) const { - if (len != t.len) - return false; + if (len != t.len) + return false; - if (len == 0) // special value to force mismatches - return false; + if (len == 0) // special value to force mismatches + return false; - return (memcmp(data, t.data, len) == 0); + return (memcmp(data, t.data, len) == 0); } class TypelessDataDecoder { - const uint8_t *mPtr; - const uint8_t *mEnd; - void checkAvailableData(uint32_t nbytes) const - { - if (mPtr + nbytes > mEnd) - throw runtime_error("TypelessData is too short"); - } -public: - TypelessDataDecoder(const uint8_t* ptr, size_t length) - :mPtr(ptr), mEnd(ptr + length) - { } - TypelessDataDecoder(const TypelessData &data) - :TypelessDataDecoder(data.data, data.len) - { } - utils::ConstString scanGeneric(uint32_t length) - { - checkAvailableData(length); - utils::ConstString res((const char *) mPtr, length); - mPtr += length; - return res; - } - uint32_t scanStringLength() - { - checkAvailableData(2); - uint32_t res = ((uint32_t) mPtr[0]) * 255 + mPtr[1]; - mPtr += 2; - return res; - } - utils::ConstString scanString() - { - return scanGeneric(scanStringLength()); - } - int64_t scanTInt64() - { - checkAvailableData(sizeof(int64_t)); - int64_t res = *reinterpret_cast(mPtr); - mPtr += sizeof(int64_t); - return res; - } - datatypes::TSInt128 scanTInt128() - { - checkAvailableData(datatypes::MAXDECIMALWIDTH); - datatypes::TSInt128 res(mPtr); - mPtr += datatypes::MAXDECIMALWIDTH; - return res; - } + const uint8_t* mPtr; + const uint8_t* mEnd; + void checkAvailableData(uint32_t nbytes) const + { + if (mPtr + nbytes > mEnd) + throw runtime_error("TypelessData is too short"); + } + public: + TypelessDataDecoder(const uint8_t* ptr, size_t length) : mPtr(ptr), mEnd(ptr + length) + { + } + TypelessDataDecoder(const TypelessData& data) : TypelessDataDecoder(data.data, data.len) + { + } + utils::ConstString scanGeneric(uint32_t length) + { + checkAvailableData(length); + utils::ConstString res((const char*)mPtr, length); + mPtr += length; + return res; + } + uint32_t scanStringLength() + { + checkAvailableData(2); + uint32_t res = ((uint32_t)mPtr[0]) * 255 + mPtr[1]; + mPtr += 2; + return res; + } + utils::ConstString scanString() + { + return scanGeneric(scanStringLength()); + } + int64_t scanTInt64() + { + checkAvailableData(sizeof(int64_t)); + int64_t res = *reinterpret_cast(mPtr); + mPtr += sizeof(int64_t); + return res; + } + datatypes::TSInt128 scanTInt128() + { + checkAvailableData(datatypes::MAXDECIMALWIDTH); + datatypes::TSInt128 res(mPtr); + mPtr += datatypes::MAXDECIMALWIDTH; + return res; + } }; // Comparator for long double in the hash class LongDoubleEq { -public: - LongDoubleEq(){}; - inline bool operator()(const long double& pos1, const long double& pos2) const - { - return pos1 == pos2; - } + public: + LongDoubleEq(){}; + inline bool operator()(const long double& pos1, const long double& pos2) const + { + return pos1 == pos2; + } }; /* This function makes the keys for string & compound joins. The length of the @@ -189,396 +187,389 @@ public: * signifying that it shouldn't match anything. */ // MCOL-1822 SUM/AVG as long double: pass in RG and col so we can determine type conversion -extern TypelessData makeTypelessKey(const rowgroup::Row&, - const std::vector&, uint32_t keylen, utils::FixedAllocator* fa, - const rowgroup::RowGroup&, const std::vector&); -extern uint64_t getHashOfTypelessKey(const rowgroup::Row&, const std::vector&, - uint32_t seed = 0); +extern TypelessData makeTypelessKey(const rowgroup::Row&, const std::vector&, uint32_t keylen, + utils::FixedAllocator* fa, const rowgroup::RowGroup&, + const std::vector&); +extern uint64_t getHashOfTypelessKey(const rowgroup::Row&, const std::vector&, uint32_t seed = 0); class TypelessDataStructure { -public: - const rowgroup::RowGroup *mRowGroup; - const std::vector *mMap; - const std::vector *mSmallSideKeyColumnsIds; - const rowgroup::RowGroup *mSmallSideRG; - TypelessDataStructure(const rowgroup::RowGroup *rg, - const std::vector *map, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) - :mRowGroup(rg), - mMap(map), - mSmallSideKeyColumnsIds(smallSideKeyColumnsIds), - mSmallSideRG(smallSideRG) - { } + public: + const rowgroup::RowGroup* mRowGroup; + const std::vector* mMap; + const std::vector* mSmallSideKeyColumnsIds; + const rowgroup::RowGroup* mSmallSideRG; + TypelessDataStructure(const rowgroup::RowGroup* rg, const std::vector* map, + const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) + : mRowGroup(rg), mMap(map), mSmallSideKeyColumnsIds(smallSideKeyColumnsIds), mSmallSideRG(smallSideRG) + { + } }; - class TupleJoiner { -public: - struct hasher + public: + struct hasher + { + inline size_t operator()(int64_t val) const { - inline size_t operator()(int64_t val) const - { - return fHasher((char*) &val, 8); - } - inline size_t operator()(uint64_t val) const - { - return fHasher((char*) &val, 8); - } - inline size_t operator()(const TypelessData& e) const - { - return fHasher((char*) e.data, e.len); - } - inline size_t operator()(long double val) const - { - if (sizeof(long double) == 8) // Probably just MSC, but you never know. - { - return fHasher((char*) &val, sizeof(long double)); - } - else - { - // For Linux x86_64, long double is stored in 128 bits, but only 80 are significant - return fHasher((char*) &val, 10); - } - } - - private: - utils::Hasher fHasher; - }; - - - struct TypelessDataHasher: public TypelessDataStructure - { - TypelessDataHasher(const rowgroup::RowGroup *rg, - const std::vector *map, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) - :TypelessDataStructure(rg, map, smallSideKeyColumnsIds, smallSideRG) - { } - inline size_t operator()(const TypelessData& e) const - { - return e.hash(*mRowGroup, *mMap, mSmallSideKeyColumnsIds, mSmallSideRG); - } - }; - - struct TypelessDataComparator: public TypelessDataStructure - { - public: - TypelessDataComparator(const rowgroup::RowGroup *rg, - const std::vector *map, - const std::vector *smallSideKeyColumnsIds, - const rowgroup::RowGroup *smallSideRG) - :TypelessDataStructure(rg, map, smallSideKeyColumnsIds, smallSideRG) - { } - bool operator()(const TypelessData& a, const TypelessData& b) const - { - return !TypelessData::cmp(*mRowGroup, *mMap, a, b, mSmallSideKeyColumnsIds, mSmallSideRG); - } - }; - - - /* ctor to use for numeric join */ - TupleJoiner( - const rowgroup::RowGroup& smallInput, - const rowgroup::RowGroup& largeInput, - uint32_t smallJoinColumn, - uint32_t largeJoinColumn, - joblist::JoinType jt, - threadpool::ThreadPool *jsThreadPool); - - /* ctor to use for string & compound join */ - TupleJoiner( - const rowgroup::RowGroup& smallInput, - const rowgroup::RowGroup& largeInput, - const std::vector& smallJoinColumns, - const std::vector& largeJoinColumns, - joblist::JoinType jt, - threadpool::ThreadPool *jsThreadPool); - - ~TupleJoiner(); - - size_t size() const; - void insert(rowgroup::Row& r, bool zeroTheRid = true); // not thread-safe - void insertRGData(rowgroup::RowGroup &rg, uint threadID); - void doneInserting(); - - /* match() returns the small-side rows that match the large-side row. - On a UM join, it uses largeSideRow, - on a PM join, it uses index and threadID. - */ - void match(rowgroup::Row& largeSideRow, uint32_t index, uint32_t threadID, - std::vector* matches); - - /* On a PM left outer join + aggregation, the result is already complete. - No need to match, just mark. - */ - void markMatches(uint32_t threadID, uint32_t rowCount); - - /* For small outer joins, this is how matches are marked now. */ - void markMatches(uint32_t threadID, const std::vector& matches); - - /* Some accessors */ - inline bool inPM() const - { - return joinAlg == PM; + return fHasher((char*)&val, 8); } - inline bool inUM() const + inline size_t operator()(uint64_t val) const { - return joinAlg == UM; + return fHasher((char*)&val, 8); } - inline bool onDisk() const + inline size_t operator()(const TypelessData& e) const { - return _convertToDiskJoin; + return fHasher((char*)e.data, e.len); } - void setInPM(); - void setInUM(std::vector &rgs); - void umJoinConvert(uint threadID, std::vector &rgs, size_t begin, size_t end); - - // TODO: these are currently in use by edge cases, ex, converting to disk - // join. Would be nice to make those cases use the rgdata variants - // above. - void setInUM(); - void umJoinConvert(size_t begin, size_t end); - - void setThreadCount(uint32_t cnt); - void setPMJoinResults(boost::shared_array >, - uint32_t threadID); - boost::shared_array > getPMJoinArrays(uint32_t threadID); - std::vector* getSmallSide() + inline size_t operator()(long double val) const { - return &rows; - } - inline bool smallOuterJoin() - { - return ((joinType & joblist::SMALLOUTER) != 0); - } - inline bool largeOuterJoin() - { - return ((joinType & joblist::LARGEOUTER) != 0); - } - inline bool innerJoin() - { - return joinType == joblist::INNER; - } - inline bool fullOuterJoin() - { - return (smallOuterJoin() && largeOuterJoin()); - } - inline joblist::JoinType getJoinType() - { - return joinType; - } - inline const rowgroup::RowGroup& getSmallRG() - { - return smallRG; - } - inline const rowgroup::RowGroup& getLargeRG() - { - return largeRG; - } - inline uint32_t getSmallKeyColumn() - { - return smallKeyColumns[0]; - } - inline uint32_t getLargeKeyColumn() - { - return largeKeyColumns[0]; - } - bool hasNullJoinColumn(const rowgroup::Row& largeRow) const; - void getUnmarkedRows(std::vector* out); - std::string getTableName() const; - void setTableName(const std::string& tname); - - /* To allow sorting */ - bool operator<(const TupleJoiner&) const; - - uint64_t getMemUsage() const; - - /* Typeless join interface */ - inline bool isTypelessJoin() - { - return typelessJoin; - } - inline bool isSignedUnsignedJoin() - { - return bSignedUnsignedJoin; - } - inline const std::vector& getSmallKeyColumns() - { - return smallKeyColumns; - } - inline const std::vector& getLargeKeyColumns() - { - return largeKeyColumns; - } - inline uint32_t getKeyLength() - { - return keyLength; + if (sizeof(long double) == 8) // Probably just MSC, but you never know. + { + return fHasher((char*)&val, sizeof(long double)); + } + else + { + // For Linux x86_64, long double is stored in 128 bits, but only 80 are significant + return fHasher((char*)&val, 10); + } } - /* Runtime casual partitioning support */ - inline const boost::scoped_array& discreteCPValues() + private: + utils::Hasher fHasher; + }; + + struct TypelessDataHasher : public TypelessDataStructure + { + TypelessDataHasher(const rowgroup::RowGroup* rg, const std::vector* map, + const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) + : TypelessDataStructure(rg, map, smallSideKeyColumnsIds, smallSideRG) { - return discreteValues; } - inline const boost::scoped_array >& getCPData() + inline size_t operator()(const TypelessData& e) const { - return cpValues; + return e.hash(*mRowGroup, *mMap, mSmallSideKeyColumnsIds, mSmallSideRG); } - inline void setUniqueLimit(uint32_t limit) + }; + + struct TypelessDataComparator : public TypelessDataStructure + { + public: + TypelessDataComparator(const rowgroup::RowGroup* rg, const std::vector* map, + const std::vector* smallSideKeyColumnsIds, + const rowgroup::RowGroup* smallSideRG) + : TypelessDataStructure(rg, map, smallSideKeyColumnsIds, smallSideRG) { - uniqueLimit = limit; } - - /* Semi-join interface */ - inline bool semiJoin() + bool operator()(const TypelessData& a, const TypelessData& b) const { - return ((joinType & joblist::SEMI) != 0); - } - inline bool antiJoin() - { - return ((joinType & joblist::ANTI) != 0); - } - inline bool scalar() - { - return ((joinType & joblist::SCALAR) != 0); - } - inline bool matchnulls() - { - return ((joinType & joblist::MATCHNULLS) != 0); - } - inline bool hasFEFilter() - { - return fe.get(); - } - inline boost::shared_ptr getFcnExpFilter() - { - return fe; - } - void setFcnExpFilter(boost::shared_ptr fe); - inline bool evaluateFilter(rowgroup::Row& r, uint32_t index) - { - return fes[index].evaluate(&r); - } - inline uint64_t getJoinNullValue() - { - return joblist::BIGINTNULL; // a normalized NULL value - } - inline uint64_t smallNullValue() - { - return nullValueForJoinColumn; + return !TypelessData::cmp(*mRowGroup, *mMap, a, b, mSmallSideKeyColumnsIds, mSmallSideRG); } + }; - // Wide-DECIMAL JOIN - bool joinHasSkewedKeyColumn(); - inline const vector& getSmallSideColumnsWidths() const - { - return smallRG.getColWidths(); - } - // Disk-based join support - void clearData(); - boost::shared_ptr copyForDiskJoin(); - bool isFinished() - { - return finished; - } - void setConvertToDiskJoin(); + /* ctor to use for numeric join */ + TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, + uint32_t smallJoinColumn, uint32_t largeJoinColumn, joblist::JoinType jt, + threadpool::ThreadPool* jsThreadPool); -private: - typedef std::tr1::unordered_multimap, - utils::STLPoolAllocator > > hash_t; - typedef std::tr1::unordered_multimap, - utils::STLPoolAllocator > > sthash_t; - typedef std::tr1::unordered_multimap, - utils::STLPoolAllocator > > typelesshash_t; - // MCOL-1822 Add support for Long Double AVG/SUM small side - typedef std::tr1::unordered_multimap > > ldhash_t; + /* ctor to use for string & compound join */ + TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, + const std::vector& smallJoinColumns, const std::vector& largeJoinColumns, + joblist::JoinType jt, threadpool::ThreadPool* jsThreadPool); - typedef hash_t::iterator iterator; - typedef typelesshash_t::iterator thIterator; - typedef ldhash_t::iterator ldIterator; + ~TupleJoiner(); - TupleJoiner(); - TupleJoiner(const TupleJoiner&); - TupleJoiner& operator=(const TupleJoiner&); - void getBucketCount(); + size_t size() const; + void insert(rowgroup::Row& r, bool zeroTheRid = true); // not thread-safe + void insertRGData(rowgroup::RowGroup& rg, uint threadID); + void doneInserting(); - rowgroup::RGData smallNullMemory; + /* match() returns the small-side rows that match the large-side row. + On a UM join, it uses largeSideRow, + on a PM join, it uses index and threadID. + */ + void match(rowgroup::Row& largeSideRow, uint32_t index, uint32_t threadID, + std::vector* matches); - boost::scoped_array > h; // used for UM joins on ints - boost::scoped_array > sth; // used for UM join on ints where the backing table uses a string table - boost::scoped_array > ld; // used for UM join on long double - std::vector rows; // used for PM join + /* On a PM left outer join + aggregation, the result is already complete. + No need to match, just mark. + */ + void markMatches(uint32_t threadID, uint32_t rowCount); - /* This struct is rough. The BPP-JL stores the parsed results for - the logical block being processed. There are X threads at once, so - up to X logical blocks being processed. For each of those there's a vector - of matches. Each match is an index into 'rows'. */ - boost::shared_array > > pmJoinResults; - rowgroup::RowGroup smallRG, largeRG; - boost::scoped_array smallRow; - //boost::shared_array smallNullMemory; - rowgroup::Row smallNullRow; + /* For small outer joins, this is how matches are marked now. */ + void markMatches(uint32_t threadID, const std::vector& matches); - enum JoinAlg - { - INSERTING, - PM, - UM, - LARGE - }; - JoinAlg joinAlg; - joblist::JoinType joinType; - boost::shared_array > _pool; // pools for the table and nodes - uint32_t threadCount; - std::string tableName; + /* Some accessors */ + inline bool inPM() const + { + return joinAlg == PM; + } + inline bool inUM() const + { + return joinAlg == UM; + } + inline bool onDisk() const + { + return _convertToDiskJoin; + } + void setInPM(); + void setInUM(std::vector& rgs); + void umJoinConvert(uint threadID, std::vector& rgs, size_t begin, size_t end); - /* vars, & fcns for typeless join */ - bool typelessJoin; - std::vector smallKeyColumns, largeKeyColumns; - boost::scoped_array > ht; // used for UM join on strings - uint32_t keyLength; - boost::scoped_array storedKeyAlloc; - boost::scoped_array tmpKeyAlloc; - bool bSignedUnsignedJoin; // Set if we have a signed vs unsigned compare in a join. When not set, we can save checking for the signed bit. + // TODO: these are currently in use by edge cases, ex, converting to disk + // join. Would be nice to make those cases use the rgdata variants + // above. + void setInUM(); + void umJoinConvert(size_t begin, size_t end); - /* semi-join vars & fcns */ - boost::shared_ptr fe; - boost::scoped_array fes; // holds X copies of fe, one per thread - // this var is only used to normalize the NULL values for single-column joins, - // will have to change when/if we need to support that for compound or string joins - int64_t nullValueForJoinColumn; + void setThreadCount(uint32_t cnt); + void setPMJoinResults(boost::shared_array >, uint32_t threadID); + boost::shared_array > getPMJoinArrays(uint32_t threadID); + std::vector* getSmallSide() + { + return &rows; + } + inline bool smallOuterJoin() + { + return ((joinType & joblist::SMALLOUTER) != 0); + } + inline bool largeOuterJoin() + { + return ((joinType & joblist::LARGEOUTER) != 0); + } + inline bool innerJoin() + { + return joinType == joblist::INNER; + } + inline bool fullOuterJoin() + { + return (smallOuterJoin() && largeOuterJoin()); + } + inline joblist::JoinType getJoinType() + { + return joinType; + } + inline const rowgroup::RowGroup& getSmallRG() + { + return smallRG; + } + inline const rowgroup::RowGroup& getLargeRG() + { + return largeRG; + } + inline uint32_t getSmallKeyColumn() + { + return smallKeyColumns[0]; + } + inline uint32_t getLargeKeyColumn() + { + return largeKeyColumns[0]; + } + bool hasNullJoinColumn(const rowgroup::Row& largeRow) const; + void getUnmarkedRows(std::vector* out); + std::string getTableName() const; + void setTableName(const std::string& tname); - /* Runtime casual partitioning support */ - void updateCPData(const rowgroup::Row& r); - boost::scoped_array discreteValues; - boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max - uint32_t uniqueLimit; - bool finished; + /* To allow sorting */ + bool operator<(const TupleJoiner&) const; - // multithreaded UM hash table construction - int numCores; - uint bucketCount; - uint bucketMask; - boost::scoped_array m_bucketLocks; - boost::mutex m_typelessLock, m_cpValuesLock; - utils::Hasher_r bucketPicker; - const uint32_t bpSeed = 0x4545e1d7; // an arbitrary random # - threadpool::ThreadPool *jobstepThreadPool; - void um_insertTypeless(uint threadID, uint rowcount, rowgroup::Row &r); - void um_insertLongDouble(uint rowcount, rowgroup::Row &r); - void um_insertInlineRows(uint rowcount, rowgroup::Row &r); - void um_insertStringTable(uint rowcount, rowgroup::Row &r); + uint64_t getMemUsage() const; - template - void bucketsToTables(buckets_t *, hash_table_t *); + /* Typeless join interface */ + inline bool isTypelessJoin() + { + return typelessJoin; + } + inline bool isSignedUnsignedJoin() + { + return bSignedUnsignedJoin; + } + inline const std::vector& getSmallKeyColumns() + { + return smallKeyColumns; + } + inline const std::vector& getLargeKeyColumns() + { + return largeKeyColumns; + } + inline uint32_t getKeyLength() + { + return keyLength; + } - bool _convertToDiskJoin; + /* Runtime casual partitioning support */ + inline const boost::scoped_array& discreteCPValues() + { + return discreteValues; + } + inline const boost::scoped_array >& getCPData() + { + return cpValues; + } + inline void setUniqueLimit(uint32_t limit) + { + uniqueLimit = limit; + } + + /* Semi-join interface */ + inline bool semiJoin() + { + return ((joinType & joblist::SEMI) != 0); + } + inline bool antiJoin() + { + return ((joinType & joblist::ANTI) != 0); + } + inline bool scalar() + { + return ((joinType & joblist::SCALAR) != 0); + } + inline bool matchnulls() + { + return ((joinType & joblist::MATCHNULLS) != 0); + } + inline bool hasFEFilter() + { + return fe.get(); + } + inline boost::shared_ptr getFcnExpFilter() + { + return fe; + } + void setFcnExpFilter(boost::shared_ptr fe); + inline bool evaluateFilter(rowgroup::Row& r, uint32_t index) + { + return fes[index].evaluate(&r); + } + inline uint64_t getJoinNullValue() + { + return joblist::BIGINTNULL; // a normalized NULL value + } + inline uint64_t smallNullValue() + { + return nullValueForJoinColumn; + } + + // Wide-DECIMAL JOIN + bool joinHasSkewedKeyColumn(); + inline const vector& getSmallSideColumnsWidths() const + { + return smallRG.getColWidths(); + } + // Disk-based join support + void clearData(); + boost::shared_ptr copyForDiskJoin(); + bool isFinished() + { + return finished; + } + void setConvertToDiskJoin(); + + private: + typedef std::tr1::unordered_multimap, + utils::STLPoolAllocator > > + hash_t; + typedef std::tr1::unordered_multimap< + int64_t, rowgroup::Row::Pointer, hasher, std::equal_to, + utils::STLPoolAllocator > > + sthash_t; + typedef std::tr1::unordered_multimap< + TypelessData, rowgroup::Row::Pointer, hasher, std::equal_to, + utils::STLPoolAllocator > > + typelesshash_t; + // MCOL-1822 Add support for Long Double AVG/SUM small side + typedef std::tr1::unordered_multimap< + long double, rowgroup::Row::Pointer, hasher, LongDoubleEq, + utils::STLPoolAllocator > > + ldhash_t; + + typedef hash_t::iterator iterator; + typedef typelesshash_t::iterator thIterator; + typedef ldhash_t::iterator ldIterator; + + TupleJoiner(); + TupleJoiner(const TupleJoiner&); + TupleJoiner& operator=(const TupleJoiner&); + void getBucketCount(); + + rowgroup::RGData smallNullMemory; + + boost::scoped_array > h; // used for UM joins on ints + boost::scoped_array > + sth; // used for UM join on ints where the backing table uses a string table + boost::scoped_array > ld; // used for UM join on long double + std::vector rows; // used for PM join + + /* This struct is rough. The BPP-JL stores the parsed results for + the logical block being processed. There are X threads at once, so + up to X logical blocks being processed. For each of those there's a vector + of matches. Each match is an index into 'rows'. */ + boost::shared_array > > pmJoinResults; + rowgroup::RowGroup smallRG, largeRG; + boost::scoped_array smallRow; + // boost::shared_array smallNullMemory; + rowgroup::Row smallNullRow; + + enum JoinAlg + { + INSERTING, + PM, + UM, + LARGE + }; + JoinAlg joinAlg; + joblist::JoinType joinType; + boost::shared_array > _pool; // pools for the table and nodes + uint32_t threadCount; + std::string tableName; + + /* vars, & fcns for typeless join */ + bool typelessJoin; + std::vector smallKeyColumns, largeKeyColumns; + boost::scoped_array > ht; // used for UM join on strings + uint32_t keyLength; + boost::scoped_array storedKeyAlloc; + boost::scoped_array tmpKeyAlloc; + bool bSignedUnsignedJoin; // Set if we have a signed vs unsigned compare in a join. When not set, we can + // save checking for the signed bit. + + /* semi-join vars & fcns */ + boost::shared_ptr fe; + boost::scoped_array fes; // holds X copies of fe, one per thread + // this var is only used to normalize the NULL values for single-column joins, + // will have to change when/if we need to support that for compound or string joins + int64_t nullValueForJoinColumn; + + /* Runtime casual partitioning support */ + void updateCPData(const rowgroup::Row& r); + boost::scoped_array discreteValues; + boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max + uint32_t uniqueLimit; + bool finished; + + // multithreaded UM hash table construction + int numCores; + uint bucketCount; + uint bucketMask; + boost::scoped_array m_bucketLocks; + boost::mutex m_typelessLock, m_cpValuesLock; + utils::Hasher_r bucketPicker; + const uint32_t bpSeed = 0x4545e1d7; // an arbitrary random # + threadpool::ThreadPool* jobstepThreadPool; + void um_insertTypeless(uint threadID, uint rowcount, rowgroup::Row& r); + void um_insertLongDouble(uint rowcount, rowgroup::Row& r); + void um_insertInlineRows(uint rowcount, rowgroup::Row& r); + void um_insertStringTable(uint rowcount, rowgroup::Row& r); + + template + void bucketsToTables(buckets_t*, hash_table_t*); + + bool _convertToDiskJoin; }; -} +} // namespace joiner #endif diff --git a/utils/libmysql_client/libmysql_client.cpp b/utils/libmysql_client/libmysql_client.cpp index 71cf6ec3d..54c4bfa16 100644 --- a/utils/libmysql_client/libmysql_client.cpp +++ b/utils/libmysql_client/libmysql_client.cpp @@ -30,112 +30,107 @@ using namespace std; namespace utils { - LibMySQL::LibMySQL() : fCon(NULL), fRes(NULL) { } - LibMySQL::~LibMySQL() { - if (fRes) - { - mysql_free_result(fRes); - } + if (fRes) + { + mysql_free_result(fRes); + } - fRes = NULL; + fRes = NULL; - if (fCon) - { - mysql_close(fCon); - } + if (fCon) + { + mysql_close(fCon); + } - fCon = NULL; + fCon = NULL; } - int LibMySQL::init(const char* h, unsigned int p, const char* u, const char* w, const char* d) { - int ret = 0; + int ret = 0; - fCon = mysql_init(NULL); + fCon = mysql_init(NULL); - config::Config* cf = config::Config::makeConfig(); - const string TLSCA = cf->getConfig("CrossEngineSupport", "TLSCA"); - const string TLSClientCert = cf->getConfig("CrossEngineSupport", "TLSClientCert"); - const string TLSClientKey = cf->getConfig("CrossEngineSupport", "TLSClientKey"); + config::Config* cf = config::Config::makeConfig(); + const string TLSCA = cf->getConfig("CrossEngineSupport", "TLSCA"); + const string TLSClientCert = cf->getConfig("CrossEngineSupport", "TLSClientCert"); + const string TLSClientKey = cf->getConfig("CrossEngineSupport", "TLSClientKey"); - if (!(TLSCA.empty() || TLSClientCert.empty() || TLSClientKey.empty())) + if (!(TLSCA.empty() || TLSClientCert.empty() || TLSClientKey.empty())) + { + mysql_ssl_set(fCon, TLSClientKey.c_str(), TLSClientCert.c_str(), TLSCA.c_str(), NULL, NULL); + } + + if (fCon != NULL) + { + unsigned int tcp_option = MYSQL_PROTOCOL_TCP; + mysql_options(fCon, MYSQL_OPT_PROTOCOL, &tcp_option); + + if (mysql_real_connect(fCon, h, u, w, d, p, NULL, 0) == NULL) { - mysql_ssl_set(fCon, TLSClientKey.c_str(), TLSClientCert.c_str(), - TLSCA.c_str(), NULL, NULL); - } - - if (fCon != NULL) - { - unsigned int tcp_option = MYSQL_PROTOCOL_TCP; - mysql_options(fCon, MYSQL_OPT_PROTOCOL, &tcp_option); - - if (mysql_real_connect(fCon, h, u, w, d, p, NULL, 0) == NULL) - { - fErrStr = "fatal error running mysql_real_connect() in libmysql_client lib"; - ret = mysql_errno(fCon); - } - else - { - mysql_set_character_set(fCon, "utf8"); - } + fErrStr = "fatal error running mysql_real_connect() in libmysql_client lib"; + ret = mysql_errno(fCon); } else { - fErrStr = "fatal error running mysql_init() in libmysql_client lib"; - ret = -1; + mysql_set_character_set(fCon, "utf8"); } + } + else + { + fErrStr = "fatal error running mysql_init() in libmysql_client lib"; + ret = -1; + } - return ret; + return ret; } - int LibMySQL::run(const char* query, bool resultExpected) { - int ret = 0; - - if (mysql_real_query(fCon, query, strlen(query)) != 0) - { - fErrStr = "fatal error runing mysql_real_query() in libmysql_client lib"; - ret = -1; - return ret; - } - - fRes = mysql_use_result(fCon); - - if (fRes == NULL && resultExpected) - { - fErrStr = "fatal error running mysql_use_result() or empty result set in libmysql_client lib"; - ret = -1; - } + int ret = 0; + if (mysql_real_query(fCon, query, strlen(query)) != 0) + { + fErrStr = "fatal error runing mysql_real_query() in libmysql_client lib"; + ret = -1; return ret; + } + + fRes = mysql_use_result(fCon); + + if (fRes == NULL && resultExpected) + { + fErrStr = "fatal error running mysql_use_result() or empty result set in libmysql_client lib"; + ret = -1; + } + + return ret; } void LibMySQL::handleMySqlError(const char* errStr, int errCode) { - ostringstream oss; + ostringstream oss; - if (getErrno()) - { - oss << errStr << " (" << getErrno() << ")"; - oss << " (" << getErrorMsg() << ")"; - } - else - { - oss << errStr << " (" << errCode << ")"; - oss << " (unknown)"; - } + if (getErrno()) + { + oss << errStr << " (" << getErrno() << ")"; + oss << " (" << getErrorMsg() << ")"; + } + else + { + oss << errStr << " (" << errCode << ")"; + oss << " (unknown)"; + } - throw logging::IDBExcept(oss.str(), logging::ERR_CROSS_ENGINE_CONNECT); + throw logging::IDBExcept(oss.str(), logging::ERR_CROSS_ENGINE_CONNECT); - return; + return; } -} // namespace +} // namespace utils diff --git a/utils/libmysql_client/libmysql_client.h b/utils/libmysql_client/libmysql_client.h index e7437c83c..4207a7054 100644 --- a/utils/libmysql_client/libmysql_client.h +++ b/utils/libmysql_client/libmysql_client.h @@ -25,72 +25,69 @@ namespace utils { - class LibMySQL { -public: - LibMySQL(); - ~LibMySQL(); + public: + LibMySQL(); + ~LibMySQL(); - // init: host port username passwd db - int init(const char*, unsigned int, const char*, const char*, const char*); + // init: host port username passwd db + int init(const char*, unsigned int, const char*, const char*, const char*); - // run the query - int run(const char* q, bool resultExpected = true); + // run the query + int run(const char* q, bool resultExpected = true); - void handleMySqlError(const char*, int); + void handleMySqlError(const char*, int); - MYSQL* getMySqlCon() - { - return fCon; - } - int getFieldCount() - { - return mysql_num_fields(fRes); - } - int getRowCount() - { - return mysql_num_rows(fRes); - } - char** nextRow() - { - char** row = mysql_fetch_row(fRes); - fieldLengths = mysql_fetch_lengths(fRes); - fFields = mysql_fetch_fields(fRes); - return row; - } - long getFieldLength(int field) - { - return fieldLengths[field]; - } - MYSQL_FIELD* getField(int field) - { - return &fFields[field]; - } - const std::string& getError() - { - return fErrStr; - } - unsigned int getErrno() - { - return mysql_errno(fCon); - } - const char* getErrorMsg() - { - return mysql_error(fCon); - } + MYSQL* getMySqlCon() + { + return fCon; + } + int getFieldCount() + { + return mysql_num_fields(fRes); + } + int getRowCount() + { + return mysql_num_rows(fRes); + } + char** nextRow() + { + char** row = mysql_fetch_row(fRes); + fieldLengths = mysql_fetch_lengths(fRes); + fFields = mysql_fetch_fields(fRes); + return row; + } + long getFieldLength(int field) + { + return fieldLengths[field]; + } + MYSQL_FIELD* getField(int field) + { + return &fFields[field]; + } + const std::string& getError() + { + return fErrStr; + } + unsigned int getErrno() + { + return mysql_errno(fCon); + } + const char* getErrorMsg() + { + return mysql_error(fCon); + } -private: - MYSQL* fCon; - MYSQL_RES* fRes; - MYSQL_FIELD* fFields; - std::string fErrStr; - unsigned long* fieldLengths; + private: + MYSQL* fCon; + MYSQL_RES* fRes; + MYSQL_FIELD* fFields; + std::string fErrStr; + unsigned long* fieldLengths; }; -} // namespace - - +} // namespace utils #endif // UTILS_LIBMYSQL_CL_H diff --git a/utils/loggingcpp/errorcodes.cpp b/utils/loggingcpp/errorcodes.cpp index 4b4196800..e52a5e32d 100644 --- a/utils/loggingcpp/errorcodes.cpp +++ b/utils/loggingcpp/errorcodes.cpp @@ -28,72 +28,79 @@ using namespace std; namespace logging { - -ErrorCodes::ErrorCodes(): fErrorCodes(), fPreamble("An unexpected condition within the query caused an internal processing error within Columnstore. Please check the log files for more details. Additional Information: ") +ErrorCodes::ErrorCodes() + : fErrorCodes() + , fPreamble( + "An unexpected condition within the query caused an internal processing error within Columnstore. " + "Please check the log files for more details. Additional Information: ") { - fErrorCodes[batchPrimitiveStepErr] = "error in BatchPrimitiveStep."; - fErrorCodes[tupleBPSErr] = "error in TupleBPS."; - fErrorCodes[batchPrimitiveStepLargeDataListFileErr] = "error in BatchPrimitiveStep LargeDataList File handling."; - fErrorCodes[bucketReuseStepErr] = "error in BucketReuseStep."; - fErrorCodes[bucketReuseStepLargeDataListFileErr] = "error in bucketReuseStep LargeDataList File handling."; - fErrorCodes[aggregateFilterStepErr] = "error in AggregateFilterStep."; - fErrorCodes[filterStepErr] = "error in FilterStep."; - fErrorCodes[functionStepErr] = "error in FunctionStep."; - fErrorCodes[hashJoinStepErr] = "error in HashJoinStep."; - fErrorCodes[hashJoinStepLargeDataListFileErr] = "error in HashJoinStep LargeDataList File handling."; - fErrorCodes[largeHashJoinErr] = "error in LargeHashJoin."; - fErrorCodes[largeHashJoinLargeDataListFileErr] = "error in LargeHashJoin LargeDataList File handling."; - fErrorCodes[stringHashJoinStepErr] = "error in StringHashJoinStep."; - fErrorCodes[stringHashJoinStepLargeDataListFileErr] = "error in StringHashJoinStep LargeDataList File handling."; - fErrorCodes[tupleHashJoinTooBigErr] = "error in TupleHashJoin: join is too big."; - fErrorCodes[threadResourceErr] = "error in ExeMgr: too many threads on the system."; - fErrorCodes[pDictionaryScanErr] = "error in pDictionaryScan."; - fErrorCodes[pDictionaryScanLargeDataListFileErr] = "error in pDictionaryScan LargeDataList File handling."; - fErrorCodes[pIdxListErr] = "error in pIdxList."; - fErrorCodes[pIdxWalkErr] = "error in pIdxWalk."; - fErrorCodes[pnlJoinErr] = "error in PNLJoinErr."; - fErrorCodes[reduceStepErr] = "error in ReduceStep."; - fErrorCodes[reduceStepLargeDataListFileErr] = "error in ReduceStep LargeDataList File handling."; - fErrorCodes[unionStepErr] = "error in UnionStep."; - fErrorCodes[unionStepLargeDataListFileErr] = "error in UnionStep LargeDataList File handling."; - fErrorCodes[unionStepTooBigErr] = "the union required too much memory."; - fErrorCodes[tupleAggregateStepErr] = "error in TupleAggregateStep."; - fErrorCodes[tupleConstantStepErr] = "error in TupleConstantStep."; - fErrorCodes[tupleHavingStepErr] = "error in TupleHavingStep."; - fErrorCodes[aggregateResourceErr] = "Memory required to perform aggregation exceeds the RowAggregation/MaxMemory setting."; - fErrorCodes[makeJobListErr] = "error in MakeJobList."; - fErrorCodes[aggregateFuncErr] = "unsupported aggregation function."; - fErrorCodes[aggregateDataErr] = "aggregation data overflow."; - fErrorCodes[batchPrimitiveProcessorErr] = "error in BatchPrimitiveProcessor."; - fErrorCodes[bppSeederErr] = "error in bppSeeder."; - fErrorCodes[primitiveServerErr] = "error in PrimitiveServer."; - fErrorCodes[projectResultErr] = "error in BatchPrimitiveProcessor projectResult. Please check crit.log for more details."; - fErrorCodes[hwmRangeSizeErr] = "error in PrimitiveServer load block with HWM. Please check crit.log for more details."; - fErrorCodes[formatErr] = "format mismatch."; - fErrorCodes[dataTypeErr] = "data type unknown."; - fErrorCodes[incompatJoinCols] = "incompatible column types specified for join condition."; - fErrorCodes[incompatFilterCols] = "incompatible column types specified for filter condition."; + fErrorCodes[batchPrimitiveStepErr] = "error in BatchPrimitiveStep."; + fErrorCodes[tupleBPSErr] = "error in TupleBPS."; + fErrorCodes[batchPrimitiveStepLargeDataListFileErr] = + "error in BatchPrimitiveStep LargeDataList File handling."; + fErrorCodes[bucketReuseStepErr] = "error in BucketReuseStep."; + fErrorCodes[bucketReuseStepLargeDataListFileErr] = "error in bucketReuseStep LargeDataList File handling."; + fErrorCodes[aggregateFilterStepErr] = "error in AggregateFilterStep."; + fErrorCodes[filterStepErr] = "error in FilterStep."; + fErrorCodes[functionStepErr] = "error in FunctionStep."; + fErrorCodes[hashJoinStepErr] = "error in HashJoinStep."; + fErrorCodes[hashJoinStepLargeDataListFileErr] = "error in HashJoinStep LargeDataList File handling."; + fErrorCodes[largeHashJoinErr] = "error in LargeHashJoin."; + fErrorCodes[largeHashJoinLargeDataListFileErr] = "error in LargeHashJoin LargeDataList File handling."; + fErrorCodes[stringHashJoinStepErr] = "error in StringHashJoinStep."; + fErrorCodes[stringHashJoinStepLargeDataListFileErr] = + "error in StringHashJoinStep LargeDataList File handling."; + fErrorCodes[tupleHashJoinTooBigErr] = "error in TupleHashJoin: join is too big."; + fErrorCodes[threadResourceErr] = "error in ExeMgr: too many threads on the system."; + fErrorCodes[pDictionaryScanErr] = "error in pDictionaryScan."; + fErrorCodes[pDictionaryScanLargeDataListFileErr] = "error in pDictionaryScan LargeDataList File handling."; + fErrorCodes[pIdxListErr] = "error in pIdxList."; + fErrorCodes[pIdxWalkErr] = "error in pIdxWalk."; + fErrorCodes[pnlJoinErr] = "error in PNLJoinErr."; + fErrorCodes[reduceStepErr] = "error in ReduceStep."; + fErrorCodes[reduceStepLargeDataListFileErr] = "error in ReduceStep LargeDataList File handling."; + fErrorCodes[unionStepErr] = "error in UnionStep."; + fErrorCodes[unionStepLargeDataListFileErr] = "error in UnionStep LargeDataList File handling."; + fErrorCodes[unionStepTooBigErr] = "the union required too much memory."; + fErrorCodes[tupleAggregateStepErr] = "error in TupleAggregateStep."; + fErrorCodes[tupleConstantStepErr] = "error in TupleConstantStep."; + fErrorCodes[tupleHavingStepErr] = "error in TupleHavingStep."; + fErrorCodes[aggregateResourceErr] = + "Memory required to perform aggregation exceeds the RowAggregation/MaxMemory setting."; + fErrorCodes[makeJobListErr] = "error in MakeJobList."; + fErrorCodes[aggregateFuncErr] = "unsupported aggregation function."; + fErrorCodes[aggregateDataErr] = "aggregation data overflow."; + fErrorCodes[batchPrimitiveProcessorErr] = "error in BatchPrimitiveProcessor."; + fErrorCodes[bppSeederErr] = "error in bppSeeder."; + fErrorCodes[primitiveServerErr] = "error in PrimitiveServer."; + fErrorCodes[projectResultErr] = + "error in BatchPrimitiveProcessor projectResult. Please check crit.log for more details."; + fErrorCodes[hwmRangeSizeErr] = + "error in PrimitiveServer load block with HWM. Please check crit.log for more details."; + fErrorCodes[formatErr] = "format mismatch."; + fErrorCodes[dataTypeErr] = "data type unknown."; + fErrorCodes[incompatJoinCols] = "incompatible column types specified for join condition."; + fErrorCodes[incompatFilterCols] = "incompatible column types specified for filter condition."; } string ErrorCodes::errorString(uint16_t code) const { - CodeMap::const_iterator iter; - CodeMap::key_type key = static_cast(code); - CodeMap::mapped_type msg; + CodeMap::const_iterator iter; + CodeMap::key_type key = static_cast(code); + CodeMap::mapped_type msg; - iter = fErrorCodes.find(key); + iter = fErrorCodes.find(key); - if (iter == fErrorCodes.end()) - { - msg = "was an unknown internal error."; - } - else - { - msg = iter->second; - } + if (iter == fErrorCodes.end()) + { + msg = "was an unknown internal error."; + } + else + { + msg = iter->second; + } - return (fPreamble + msg); + return (fPreamble + msg); } -} //namespace logging +} // namespace logging // vim:ts=4 sw=4: - diff --git a/utils/loggingcpp/errorcodes.h b/utils/loggingcpp/errorcodes.h index e270ece65..e00baff82 100644 --- a/utils/loggingcpp/errorcodes.h +++ b/utils/loggingcpp/errorcodes.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: errorcodes.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: errorcodes.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef LOGGING_ERRORCODES_H #define LOGGING_ERRORCODES_H @@ -31,71 +31,70 @@ namespace logging { - enum ErrorCodeValues { - batchPrimitiveStepErr = 1, - tupleBPSErr, - batchPrimitiveStepLargeDataListFileErr, - bucketReuseStepErr, - bucketReuseStepLargeDataListFileErr, - aggregateFilterStepErr, - filterStepErr, - functionStepErr, - hashJoinStepErr, - hashJoinStepLargeDataListFileErr, - largeHashJoinErr, - largeHashJoinLargeDataListFileErr, - stringHashJoinStepErr, - stringHashJoinStepLargeDataListFileErr, - tupleHashJoinTooBigErr, - threadResourceErr, - pDictionaryScanErr, - pDictionaryScanLargeDataListFileErr, - pIdxListErr, - pIdxWalkErr, - pnlJoinErr, - reduceStepErr, - reduceStepLargeDataListFileErr, - unionStepErr, - unionStepLargeDataListFileErr, - unionStepTooBigErr, - tupleAggregateStepErr, - tupleConstantStepErr, - tupleHavingStepErr, - makeJobListErr, - aggregateFuncErr, - aggregateDataErr, -//don't use 100, same as SQL_NOT_FOUND - batchPrimitiveProcessorErr = 101, - bppSeederErr, - primitiveServerErr, - projectResultErr, - hwmRangeSizeErr, -// user input data error - formatErr = 201, - dataTypeErr, - incompatJoinCols, - incompatFilterCols, - aggregateResourceErr, - statisticsJobListEmpty = 301 + batchPrimitiveStepErr = 1, + tupleBPSErr, + batchPrimitiveStepLargeDataListFileErr, + bucketReuseStepErr, + bucketReuseStepLargeDataListFileErr, + aggregateFilterStepErr, + filterStepErr, + functionStepErr, + hashJoinStepErr, + hashJoinStepLargeDataListFileErr, + largeHashJoinErr, + largeHashJoinLargeDataListFileErr, + stringHashJoinStepErr, + stringHashJoinStepLargeDataListFileErr, + tupleHashJoinTooBigErr, + threadResourceErr, + pDictionaryScanErr, + pDictionaryScanLargeDataListFileErr, + pIdxListErr, + pIdxWalkErr, + pnlJoinErr, + reduceStepErr, + reduceStepLargeDataListFileErr, + unionStepErr, + unionStepLargeDataListFileErr, + unionStepTooBigErr, + tupleAggregateStepErr, + tupleConstantStepErr, + tupleHavingStepErr, + makeJobListErr, + aggregateFuncErr, + aggregateDataErr, + // don't use 100, same as SQL_NOT_FOUND + batchPrimitiveProcessorErr = 101, + bppSeederErr, + primitiveServerErr, + projectResultErr, + hwmRangeSizeErr, + // user input data error + formatErr = 201, + dataTypeErr, + incompatJoinCols, + incompatFilterCols, + aggregateResourceErr, + statisticsJobListEmpty = 301 }; struct ErrorCodes { - ErrorCodes(); - std::string errorString(uint16_t code) const; -private: - typedef std::map CodeMap; + ErrorCodes(); + std::string errorString(uint16_t code) const; - //defaults okay - //ErrorCodes(const ErrorCodes& rhs); - //ErrorCodes& operator=(const ErrorCodes& rhs); + private: + typedef std::map CodeMap; - CodeMap fErrorCodes; - const std::string fPreamble; + // defaults okay + // ErrorCodes(const ErrorCodes& rhs); + // ErrorCodes& operator=(const ErrorCodes& rhs); + + CodeMap fErrorCodes; + const std::string fPreamble; }; -} -#endif //LOGGING_ERRORCODES_H - +} // namespace logging +#endif // LOGGING_ERRORCODES_H diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index 3fd77dc8a..92aebbbbc 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -37,121 +37,136 @@ namespace logging { - /** @brief specific error exception class -* -* -*/ + * + * + */ class IDBExcept : public std::runtime_error { -public: - IDBExcept(uint16_t code) : - std::runtime_error(IDBErrorInfo::instance()->errorMsg(code)), fErrCode(code) {} - IDBExcept(uint16_t code, const Message::Args& args) : - std::runtime_error(IDBErrorInfo::instance()->errorMsg(code, args)), fErrCode(code) {} - IDBExcept(const std::string& msg, uint16_t code) : - std::runtime_error(msg), fErrCode(code) { } - void errorCode(uint16_t code) - { - fErrCode = code; - } - uint16_t errorCode() const - { - return fErrCode; - } -protected: - unsigned fErrCode; + public: + IDBExcept(uint16_t code) : std::runtime_error(IDBErrorInfo::instance()->errorMsg(code)), fErrCode(code) + { + } + IDBExcept(uint16_t code, const Message::Args& args) + : std::runtime_error(IDBErrorInfo::instance()->errorMsg(code, args)), fErrCode(code) + { + } + IDBExcept(const std::string& msg, uint16_t code) : std::runtime_error(msg), fErrCode(code) + { + } + void errorCode(uint16_t code) + { + fErrCode = code; + } + uint16_t errorCode() const + { + return fErrCode; + } + + protected: + unsigned fErrCode; }; class DictionaryBufferOverflow : public IDBExcept { -public: - DictionaryBufferOverflow() : IDBExcept(ERR_DICTBUFFER_OVERFLOW) - { } + public: + DictionaryBufferOverflow() : IDBExcept(ERR_DICTBUFFER_OVERFLOW) + { + } }; class LargeDataListExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - explicit LargeDataListExcept(const std::string& msg) : std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + explicit LargeDataListExcept(const std::string& msg) : std::runtime_error(msg) + { + } - virtual ~LargeDataListExcept() throw() { } + virtual ~LargeDataListExcept() throw() + { + } }; - /** @brief specific error exception class for query data -* @bug 1155 -* -*/ + * @bug 1155 + * + */ class QueryDataExcept : public IDBExcept { -public: - /** Takes a character string describing the error. */ - QueryDataExcept(const std::string& msg, uint16_t code) : - IDBExcept(msg, code) { } + public: + /** Takes a character string describing the error. */ + QueryDataExcept(const std::string& msg, uint16_t code) : IDBExcept(msg, code) + { + } - virtual ~QueryDataExcept() throw() { } -private: - //defaults okay - //QueryDataExcept(const QueryDataExcept& rhs); - //QueryDataExcept& operator=(const QueryDataExcept& rhs); + virtual ~QueryDataExcept() throw() + { + } + + private: + // defaults okay + // QueryDataExcept(const QueryDataExcept& rhs); + // QueryDataExcept& operator=(const QueryDataExcept& rhs); }; /** @brief specific error exception class for VBBM Version Buffer overflow -* @bug 1949 -* -*/ + * @bug 1949 + * + */ class VBBMBufferOverFlowExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - VBBMBufferOverFlowExcept(const std::string& msg) : - std::runtime_error(msg), fErrorCode(8) { } - //ERR_VBBM_OVERFLOW = 8 defined in brmtypes.h - int8_t errorCode() const - { - return fErrorCode; - } + public: + /** Takes a character string describing the error. */ + VBBMBufferOverFlowExcept(const std::string& msg) : std::runtime_error(msg), fErrorCode(8) + { + } + // ERR_VBBM_OVERFLOW = 8 defined in brmtypes.h + int8_t errorCode() const + { + return fErrorCode; + } -private: - int8_t fErrorCode; + private: + int8_t fErrorCode; }; /** @brief specific error exception class for VBBM Version Buffer overflow -* @bug 1949 -* -*/ + * @bug 1949 + * + */ class PrimitiveColumnProjectResultExcept : public QueryDataExcept { -public: - /** Takes a character string describing the error. */ - PrimitiveColumnProjectResultExcept(const std::string& msg) : - QueryDataExcept(msg, projectResultErr) { } + public: + /** Takes a character string describing the error. */ + PrimitiveColumnProjectResultExcept(const std::string& msg) : QueryDataExcept(msg, projectResultErr) + { + } }; /** @brief specific error exception class for PrimProc invalid HWM -* @bug 2173 -* -*/ + * @bug 2173 + * + */ class InvalidRangeHWMExcept : public QueryDataExcept { -public: - /** Takes a character string describing the error. */ - InvalidRangeHWMExcept(const std::string& msg) : - QueryDataExcept(msg, hwmRangeSizeErr) { } + public: + /** Takes a character string describing the error. */ + InvalidRangeHWMExcept(const std::string& msg) : QueryDataExcept(msg, hwmRangeSizeErr) + { + } }; - /** @brief Exception for F&E framework to throw * Invalid Operation Exception */ class InvalidOperationExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - InvalidOperationExcept(const std::string& msg) : - std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + InvalidOperationExcept(const std::string& msg) : std::runtime_error(msg) + { + } }; /** @brief Exception for F&E framework to throw @@ -159,10 +174,11 @@ public: */ class InvalidConversionExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - InvalidConversionExcept(const std::string& msg) : - std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + InvalidConversionExcept(const std::string& msg) : std::runtime_error(msg) + { + } }; /** @brief Exception for F&E framework -- function evaluation to throw @@ -170,10 +186,11 @@ public: */ class InvalidArgumentExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - InvalidArgumentExcept(const std::string& msg) : - std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + InvalidArgumentExcept(const std::string& msg) : std::runtime_error(msg) + { + } }; /** @brief Exception for F&E framework -- function evaluation to throw @@ -181,10 +198,11 @@ public: */ class NotImplementedExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - NotImplementedExcept(const std::string& msg) : - std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + NotImplementedExcept(const std::string& msg) : std::runtime_error(msg) + { + } }; /** @brief Exception for F&E framework to throw on op overflow @@ -192,91 +210,102 @@ public: */ class OperationOverflowExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - OperationOverflowExcept(const std::string& msg) : - std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + OperationOverflowExcept(const std::string& msg) : std::runtime_error(msg) + { + } }; /** @brief specific error exception class for getSysData in Calpontsystemcatalog. -* @bug 2574 -* -*/ + * @bug 2574 + * + */ class NoTableExcept : public std::runtime_error { -public: - /** Takes a character string describing the error. */ - NoTableExcept(const std::string& msg) : - std::runtime_error(msg) { } + public: + /** Takes a character string describing the error. */ + NoTableExcept(const std::string& msg) : std::runtime_error(msg) + { + } }; class MoreThan1RowExcept : public IDBExcept { -public: - MoreThan1RowExcept() : - IDBExcept(ERR_MORE_THAN_1_ROW) - { } + public: + MoreThan1RowExcept() : IDBExcept(ERR_MORE_THAN_1_ROW) + { + } }; class CorrelateFailExcept : public IDBExcept { -public: - CorrelateFailExcept() : - IDBExcept(ERR_CORRELATE_FAIL) - { } + public: + CorrelateFailExcept() : IDBExcept(ERR_CORRELATE_FAIL) + { + } }; class DBRMException : public std::runtime_error { -public: - DBRMException(const std::string& emsg) : - std::runtime_error(emsg) {} + public: + DBRMException(const std::string& emsg) : std::runtime_error(emsg) + { + } }; class ProtocolError : public std::logic_error { -public: - ProtocolError(const std::string& emsg) : std::logic_error(emsg) { } + public: + ProtocolError(const std::string& emsg) : std::logic_error(emsg) + { + } }; #ifndef __STRING #define __STRING(x) #x #endif -#define idbassert(x) do { \ - if (!(x)) { \ - std::ostringstream os; \ -\ - os << __FILE__ << "@" << __LINE__ << ": assertion \'" << __STRING(x) << "\' failed"; \ - std::cerr << os.str() << std::endl; \ - logging::MessageLog logger((logging::LoggingID())); \ - logging::Message message; \ - logging::Message::Args args; \ -\ - args.add(os.str()); \ - message.format(args); \ - logger.logErrorMessage(message); \ - throw logging::IDBExcept(logging::ERR_ASSERTION_FAILURE); \ - } \ -} while (0) +#define idbassert(x) \ + do \ + { \ + if (!(x)) \ + { \ + std::ostringstream os; \ + \ + os << __FILE__ << "@" << __LINE__ << ": assertion \'" << __STRING(x) << "\' failed"; \ + std::cerr << os.str() << std::endl; \ + logging::MessageLog logger((logging::LoggingID())); \ + logging::Message message; \ + logging::Message::Args args; \ + \ + args.add(os.str()); \ + message.format(args); \ + logger.logErrorMessage(message); \ + throw logging::IDBExcept(logging::ERR_ASSERTION_FAILURE); \ + } \ + } while (0) -#define idbassert_s(x, s) do { \ - if (!(x)) { \ - std::ostringstream os; \ -\ - os << __FILE__ << "@" << __LINE__ << ": assertion \'" << __STRING(x) << "\' failed. Error msg \'" << s << "\'"; \ - std::cerr << os.str() << std::endl; \ - logging::MessageLog logger((logging::LoggingID())); \ - logging::Message message; \ - logging::Message::Args args; \ -\ - args.add(os.str()); \ - message.format(args); \ - logger.logErrorMessage(message); \ - throw logging::IDBExcept(logging::ERR_ASSERTION_FAILURE); \ - } \ -} while (0) - -} +#define idbassert_s(x, s) \ + do \ + { \ + if (!(x)) \ + { \ + std::ostringstream os; \ + \ + os << __FILE__ << "@" << __LINE__ << ": assertion \'" << __STRING(x) << "\' failed. Error msg \'" \ + << s << "\'"; \ + std::cerr << os.str() << std::endl; \ + logging::MessageLog logger((logging::LoggingID())); \ + logging::Message message; \ + logging::Message::Args args; \ + \ + args.add(os.str()); \ + message.format(args); \ + logger.logErrorMessage(message); \ + throw logging::IDBExcept(logging::ERR_ASSERTION_FAILURE); \ + } \ + } while (0) +} // namespace logging #endif // vim:ts=4 sw=4: diff --git a/utils/loggingcpp/idberrorinfo.cpp b/utils/loggingcpp/idberrorinfo.cpp index 18940dcb8..2b0054881 100644 --- a/utils/loggingcpp/idberrorinfo.cpp +++ b/utils/loggingcpp/idberrorinfo.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: idberrorinfo.cpp 3626 2013-03-11 15:36:08Z xlou $ -* -******************************************************************************************/ + * $Id: idberrorinfo.cpp 3626 2013-03-11 15:36:08Z xlou $ + * + ******************************************************************************************/ #include #include #include @@ -45,69 +45,68 @@ using namespace config; namespace logging { - IDBErrorInfo* IDBErrorInfo::fInstance = 0; boost::mutex mx; IDBErrorInfo* IDBErrorInfo::instance() { - boost::mutex::scoped_lock lk(mx); + boost::mutex::scoped_lock lk(mx); - if (!fInstance) - fInstance = new IDBErrorInfo(); + if (!fInstance) + fInstance = new IDBErrorInfo(); - return fInstance; + return fInstance; } IDBErrorInfo::IDBErrorInfo() { - Config* cf = Config::makeConfig(); - string configFile(cf->getConfig("SystemConfig", "ErrorMessageFile")); + Config* cf = Config::makeConfig(); + string configFile(cf->getConfig("SystemConfig", "ErrorMessageFile")); - if (configFile.length() == 0) - configFile = std::string(MCSSYSCONFDIR) + "/columnstore/ErrorMessage.txt"; + if (configFile.length() == 0) + configFile = std::string(MCSSYSCONFDIR) + "/columnstore/ErrorMessage.txt"; - ifstream msgFile(configFile.c_str()); + ifstream msgFile(configFile.c_str()); - while (msgFile.good()) + while (msgFile.good()) + { + stringbuf* sb = new stringbuf; + msgFile.get(*sb); + string m = sb->str(); + delete sb; + + if (m.length() > 0 && m[0] != '#') { - stringbuf* sb = new stringbuf; - msgFile.get(*sb); - string m = sb->str(); - delete sb; + typedef boost::tokenizer > tokenizer; + boost::char_separator sep("\t"); + tokenizer tokens(m, sep); + tokenizer::iterator tok_iter = tokens.begin(); - if (m.length() > 0 && m[0] != '#') + if (tok_iter != tokens.end()) + { + int msgid = atoi(tok_iter->c_str()); + ++tok_iter; + + if (tok_iter != tokens.end()) { - typedef boost::tokenizer > tokenizer; - boost::char_separator sep("\t"); - tokenizer tokens(m, sep); - tokenizer::iterator tok_iter = tokens.begin(); + ++tok_iter; - if (tok_iter != tokens.end()) - { - int msgid = atoi(tok_iter->c_str()); - ++tok_iter; - - if (tok_iter != tokens.end()) - { - ++tok_iter; - - if (tok_iter != tokens.end()) - { - string msgtext = *tok_iter; - fErrMap[msgid] = msgtext; - } - } - } + if (tok_iter != tokens.end()) + { + string msgtext = *tok_iter; + fErrMap[msgid] = msgtext; + } } - - ios_base::iostate st = msgFile.rdstate(); - - if ((st & ios_base::failbit) && !(st & ios_base::eofbit)) - msgFile.clear(); - - (void)msgFile.get(); + } } + + ios_base::iostate st = msgFile.rdstate(); + + if ((st & ios_base::failbit) && !(st & ios_base::eofbit)) + msgFile.clear(); + + (void)msgFile.get(); + } } IDBErrorInfo::~IDBErrorInfo() @@ -116,104 +115,100 @@ IDBErrorInfo::~IDBErrorInfo() string IDBErrorInfo::errorMsg(const unsigned eid, const Message::Args& args) { - string errMsg = lookupError(eid); - format(errMsg, args); - return errMsg; + string errMsg = lookupError(eid); + format(errMsg, args); + return errMsg; } string IDBErrorInfo::errorMsg(const unsigned eid) { - string errMsg = lookupError(eid); - Message::Args args; // empty args - format(errMsg, args); - return errMsg; + string errMsg = lookupError(eid); + Message::Args args; // empty args + format(errMsg, args); + return errMsg; } string IDBErrorInfo::errorMsg(const unsigned eid, int i) { - string errMsg = lookupError(eid); - Message::Args args; - args.add(i); - format(errMsg, args); - return errMsg; + string errMsg = lookupError(eid); + Message::Args args; + args.add(i); + format(errMsg, args); + return errMsg; } string IDBErrorInfo::errorMsg(const unsigned eid, const string& s) { - string errMsg = lookupError(eid); - Message::Args args; - args.add(s); - format(errMsg, args); - return errMsg; + string errMsg = lookupError(eid); + Message::Args args; + args.add(s); + format(errMsg, args); + return errMsg; } -string IDBErrorInfo::logError(const logging::LOG_TYPE logLevel, - const logging::LoggingID logid, - const unsigned eid, - const logging::Message::Args& args) +string IDBErrorInfo::logError(const logging::LOG_TYPE logLevel, const logging::LoggingID logid, + const unsigned eid, const logging::Message::Args& args) { - Logger logger(logid.fSubsysID); - Message message(errorMsg(eid, args)); - return logger.logMessage(logLevel, message, logid); + Logger logger(logid.fSubsysID); + Message message(errorMsg(eid, args)); + return logger.logMessage(logLevel, message, logid); } void IDBErrorInfo::format(string& errMsg, const Message::Args& args) { - Message::Args::AnyVec::const_iterator iter = args.args().begin(); - Message::Args::AnyVec::const_iterator end = args.args().end(); + Message::Args::AnyVec::const_iterator iter = args.args().begin(); + Message::Args::AnyVec::const_iterator end = args.args().end(); - boost::format fmt(errMsg); - fmt.exceptions(boost::io::no_error_bits); + boost::format fmt(errMsg); + fmt.exceptions(boost::io::no_error_bits); - while (iter != end) + while (iter != end) + { + if (iter->type() == typeid(long)) { - if (iter->type() == typeid(long)) - { - long l = any_cast(*iter); - fmt % l; - } - else if (iter->type() == typeid(uint64_t)) - { - uint64_t u64 = any_cast(*iter); - fmt % u64; - } - else if (iter->type() == typeid(double)) - { - double d = any_cast(*iter); - fmt % d; - } - else if (iter->type() == typeid(string)) - { - string s = any_cast(*iter); - fmt % s; - } - else - { - throw logic_error("IDBErrorInfo::format: unexpected type in argslist"); - } - - ++iter; + long l = any_cast(*iter); + fmt % l; + } + else if (iter->type() == typeid(uint64_t)) + { + uint64_t u64 = any_cast(*iter); + fmt % u64; + } + else if (iter->type() == typeid(double)) + { + double d = any_cast(*iter); + fmt % d; + } + else if (iter->type() == typeid(string)) + { + string s = any_cast(*iter); + fmt % s; + } + else + { + throw logic_error("IDBErrorInfo::format: unexpected type in argslist"); } - errMsg = fmt.str(); + ++iter; + } + + errMsg = fmt.str(); } /* static */ string IDBErrorInfo::lookupError(const unsigned eid) { - string msgstr; - ErrorMap::const_iterator iter = fErrMap.find(eid); + string msgstr; + ErrorMap::const_iterator iter = fErrMap.find(eid); - if (iter == fErrMap.end()) - msgstr = "Unknown Error %1% %2% %3% %4% %5%"; - else - msgstr = iter->second; + if (iter == fErrMap.end()) + msgstr = "Unknown Error %1% %2% %3% %4% %5%"; + else + msgstr = iter->second; - ostringstream oss; - oss << "MCS-" << setw(4) << setfill('0') << eid << ": " << msgstr; - return oss.str(); + ostringstream oss; + oss << "MCS-" << setw(4) << setfill('0') << eid << ": " << msgstr; + return oss.str(); } -} - - +} // namespace logging diff --git a/utils/loggingcpp/idberrorinfo.h b/utils/loggingcpp/idberrorinfo.h index f28fe1c2b..124a1ca8a 100644 --- a/utils/loggingcpp/idberrorinfo.h +++ b/utils/loggingcpp/idberrorinfo.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: idberrorinfo.h 3626 2013-03-11 15:36:08Z xlou $ -* -******************************************************************************************/ + * $Id: idberrorinfo.h 3626 2013-03-11 15:36:08Z xlou $ + * + ******************************************************************************************/ /** * @file */ @@ -33,7 +33,6 @@ namespace logging { - /** @brief an IDB error info class * * Contains a error message map for looking up and constructing formatted message @@ -43,26 +42,24 @@ typedef std::map ErrorMap; class IDBErrorInfo { -public: - static IDBErrorInfo* instance(); - std::string errorMsg(const unsigned eid, const Message::Args& args); - std::string errorMsg(const unsigned eid); - std::string errorMsg(const unsigned eid, int i); - std::string errorMsg(const unsigned eid, const std::string& s); - std::string logError(const logging::LOG_TYPE logLevel, - const LoggingID logid, - const unsigned eid, - const Message::Args& args); - ~IDBErrorInfo(); + public: + static IDBErrorInfo* instance(); + std::string errorMsg(const unsigned eid, const Message::Args& args); + std::string errorMsg(const unsigned eid); + std::string errorMsg(const unsigned eid, int i); + std::string errorMsg(const unsigned eid, const std::string& s); + std::string logError(const logging::LOG_TYPE logLevel, const LoggingID logid, const unsigned eid, + const Message::Args& args); + ~IDBErrorInfo(); -private: - static IDBErrorInfo* fInstance; - ErrorMap fErrMap; - IDBErrorInfo(); - void format(std::string& messageFormat, const Message::Args& args); - std::string lookupError(const unsigned eid); + private: + static IDBErrorInfo* fInstance; + ErrorMap fErrMap; + IDBErrorInfo(); + void format(std::string& messageFormat, const Message::Args& args); + std::string lookupError(const unsigned eid); }; -}//namespace logging +} // namespace logging #endif diff --git a/utils/loggingcpp/logger.cpp b/utils/loggingcpp/logger.cpp index 1f3c5ad3f..1efbf14be 100644 --- a/utils/loggingcpp/logger.cpp +++ b/utils/loggingcpp/logger.cpp @@ -28,86 +28,73 @@ using namespace boost; namespace logging { - -Logger::Logger(unsigned subsys) : - fMl1(LoggingID(subsys)) +Logger::Logger(unsigned subsys) : fMl1(LoggingID(subsys)) { } const string Logger::logMessage(LOG_TYPE logLevel, Message::MessageID mid, const Message::Args& args, const LoggingID& logInfo) { - MsgMap::mapped_type msg; - MsgMap::const_iterator msgIter = fMsgMap.find(mid); + MsgMap::mapped_type msg; + MsgMap::const_iterator msgIter = fMsgMap.find(mid); - //Default message if specified # not found - if (msgIter == fMsgMap.end()) - msg = Message(M0000); - else - msg = msgIter->second; + // Default message if specified # not found + if (msgIter == fMsgMap.end()) + msg = Message(M0000); + else + msg = msgIter->second; - msg.reset(); - msg.format(args); + msg.reset(); + msg.format(args); - return logMessage(logLevel, msg, logInfo); - /* - boost::mutex::scoped_lock lk(fLogLock); - fMl1.logData(logInfo); + return logMessage(logLevel, msg, logInfo); + /* + boost::mutex::scoped_lock lk(fLogLock); + fMl1.logData(logInfo); - switch (logLevel) - { - case LOG_TYPE_DEBUG: - default: - fMl1.logDebugMessage(msg); - break; - case LOG_TYPE_INFO: - fMl1.logInfoMessage(msg); - break; - case LOG_TYPE_WARNING: - fMl1.logWarningMessage(msg); - break; - case LOG_TYPE_ERROR: - fMl1.logErrorMessage(msg); - break; - case LOG_TYPE_CRITICAL: - fMl1.logCriticalMessage(msg); - break; - } + switch (logLevel) + { + case LOG_TYPE_DEBUG: + default: + fMl1.logDebugMessage(msg); + break; + case LOG_TYPE_INFO: + fMl1.logInfoMessage(msg); + break; + case LOG_TYPE_WARNING: + fMl1.logWarningMessage(msg); + break; + case LOG_TYPE_ERROR: + fMl1.logErrorMessage(msg); + break; + case LOG_TYPE_CRITICAL: + fMl1.logCriticalMessage(msg); + break; + } - return msg.msg();*/ + return msg.msg();*/ } const std::string Logger::logMessage(LOG_TYPE logLevel, const Message& msg, const LoggingID& logInfo) { - boost::mutex::scoped_lock lk(fLogLock); - fMl1.logData(logInfo); + boost::mutex::scoped_lock lk(fLogLock); + fMl1.logData(logInfo); - switch (logLevel) - { - case LOG_TYPE_DEBUG: - default: - fMl1.logDebugMessage(msg); - break; + switch (logLevel) + { + case LOG_TYPE_DEBUG: + default: fMl1.logDebugMessage(msg); break; - case LOG_TYPE_INFO: - fMl1.logInfoMessage(msg); - break; + case LOG_TYPE_INFO: fMl1.logInfoMessage(msg); break; - case LOG_TYPE_WARNING: - fMl1.logWarningMessage(msg); - break; + case LOG_TYPE_WARNING: fMl1.logWarningMessage(msg); break; - case LOG_TYPE_ERROR: - fMl1.logErrorMessage(msg); - break; + case LOG_TYPE_ERROR: fMl1.logErrorMessage(msg); break; - case LOG_TYPE_CRITICAL: - fMl1.logCriticalMessage(msg); - break; - } - - return msg.msg(); -} + case LOG_TYPE_CRITICAL: fMl1.logCriticalMessage(msg); break; + } + return msg.msg(); } +} // namespace logging diff --git a/utils/loggingcpp/logger.h b/utils/loggingcpp/logger.h index f640f7d6b..a431667bc 100644 --- a/utils/loggingcpp/logger.h +++ b/utils/loggingcpp/logger.h @@ -35,62 +35,60 @@ namespace logging { - typedef std::map MsgMap; /** @brief message log wrapper class */ class Logger { -public: - /** @brief ctor - * - * @param subsys the subsystem id - */ - explicit Logger(unsigned subsys); + public: + /** @brief ctor + * + * @param subsys the subsystem id + */ + explicit Logger(unsigned subsys); - /** @brief log a message - * - * Log a message at a certain debug level - */ - const std::string logMessage(LOG_TYPE logLevel, Message::MessageID mid, - const Message::Args& args, const LoggingID& logInfo); + /** @brief log a message + * + * Log a message at a certain debug level + */ + const std::string logMessage(LOG_TYPE logLevel, Message::MessageID mid, const Message::Args& args, + const LoggingID& logInfo); - /** @brief log a formated message - * - * For the error framework to use - */ - const std::string logMessage(LOG_TYPE logLevel, const Message& message, const LoggingID& logInfo); + /** @brief log a formated message + * + * For the error framework to use + */ + const std::string logMessage(LOG_TYPE logLevel, const Message& message, const LoggingID& logInfo); - /** @brief set the message map for this logger - * - * This method sets the message map for this logger. You need to call this (once and only once) before - * using the class. The map needs to be constructed before and given to this class. - */ - void msgMap(const MsgMap& msgMap) - { - fMsgMap = msgMap; - } + /** @brief set the message map for this logger + * + * This method sets the message map for this logger. You need to call this (once and only once) before + * using the class. The map needs to be constructed before and given to this class. + */ + void msgMap(const MsgMap& msgMap) + { + fMsgMap = msgMap; + } - /** @brief get the message map from this logger - */ - const MsgMap& msgMap() const - { - return fMsgMap; - } + /** @brief get the message map from this logger + */ + const MsgMap& msgMap() const + { + return fMsgMap; + } -private: - // not copyable (because of the mutex) - Logger(const Logger& rhs); - Logger& operator=(const Logger& rhs); + private: + // not copyable (because of the mutex) + Logger(const Logger& rhs); + Logger& operator=(const Logger& rhs); - MsgMap fMsgMap; - MessageLog fMl1; - boost::mutex fLogLock; + MsgMap fMsgMap; + MessageLog fMl1; + boost::mutex fLogLock; }; typedef boost::shared_ptr SPL; -} +} // namespace logging #endif - diff --git a/utils/loggingcpp/loggingid.h b/utils/loggingcpp/loggingid.h index 0b47ebcfd..b7c656c80 100644 --- a/utils/loggingcpp/loggingid.h +++ b/utils/loggingcpp/loggingid.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: loggingid.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -******************************************************************************************/ + * $Id: loggingid.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -29,25 +29,25 @@ namespace logging { - /** @brief a logging context structure * */ struct LoggingID { - /** @brief LoggingID ctor - * - */ - explicit LoggingID(unsigned subsysID = 0, unsigned sessionID = 0, unsigned txnID = 0, unsigned ThdID = 0) - : fSubsysID(subsysID), fSessionID(sessionID), fTxnID(txnID), fThdID(ThdID) - {} + /** @brief LoggingID ctor + * + */ + explicit LoggingID(unsigned subsysID = 0, unsigned sessionID = 0, unsigned txnID = 0, unsigned ThdID = 0) + : fSubsysID(subsysID), fSessionID(sessionID), fTxnID(txnID), fThdID(ThdID) + { + } - unsigned fSubsysID; /// subsystem ID - unsigned fSessionID; /// session ID - unsigned fTxnID; /// transaction ID - unsigned fThdID; /// thread ID + unsigned fSubsysID; /// subsystem ID + unsigned fSessionID; /// session ID + unsigned fTxnID; /// transaction ID + unsigned fThdID; /// thread ID }; -} +} // namespace logging #endif diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 3f7ba1274..316923f6f 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: message.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -******************************************************************************************/ + * $Id: message.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + ******************************************************************************************/ #include #include #include @@ -43,7 +43,6 @@ using namespace config; namespace { - boost::mutex mx; bool catalogLoaded = false; @@ -53,182 +52,179 @@ CatMap catmap; void loadCatalog() { - Config* cf = Config::makeConfig(); - string configFile(cf->getConfig("MessageLog", "MessageLogFile")); + Config* cf = Config::makeConfig(); + string configFile(cf->getConfig("MessageLog", "MessageLogFile")); - if (configFile.length() == 0) - configFile = std::string(MCSSYSCONFDIR) + "/columnstore/MessageFile.txt"; + if (configFile.length() == 0) + configFile = std::string(MCSSYSCONFDIR) + "/columnstore/MessageFile.txt"; - ifstream msgFile(configFile.c_str()); + ifstream msgFile(configFile.c_str()); - while (msgFile.good()) + while (msgFile.good()) + { + stringbuf* sb = new stringbuf; + msgFile.get(*sb); + string m = sb->str(); + delete sb; + + if (m.length() > 0 && m[0] != '#') { - stringbuf* sb = new stringbuf; - msgFile.get(*sb); - string m = sb->str(); - delete sb; + typedef boost::tokenizer > tokenizer; + boost::char_separator sep("\t"); + tokenizer tokens(m, sep); + tokenizer::iterator tok_iter = tokens.begin(); - if (m.length() > 0 && m[0] != '#') + if (tok_iter != tokens.end()) + { + int msgid = atoi(tok_iter->c_str()); + ++tok_iter; + + if (tok_iter != tokens.end()) { - typedef boost::tokenizer > tokenizer; - boost::char_separator sep("\t"); - tokenizer tokens(m, sep); - tokenizer::iterator tok_iter = tokens.begin(); - - if (tok_iter != tokens.end()) - { - int msgid = atoi(tok_iter->c_str()); - ++tok_iter; - - if (tok_iter != tokens.end()) - { - string msgtext = *tok_iter; - catmap[msgid] = msgtext; - } - } + string msgtext = *tok_iter; + catmap[msgid] = msgtext; } - - ios_base::iostate st = msgFile.rdstate(); - - if ((st & ios_base::failbit) && !(st & ios_base::eofbit)) - msgFile.clear(); - - (void)msgFile.get(); + } } + + ios_base::iostate st = msgFile.rdstate(); + + if ((st & ios_base::failbit) && !(st & ios_base::eofbit)) + msgFile.clear(); + + (void)msgFile.get(); + } } -} +} // namespace namespace logging { - -Message::Message(const MessageID msgid) : - fMsgID(msgid), fMsg(lookupMessage(msgid)), fConfig(Config::makeConfig()) +Message::Message(const MessageID msgid) + : fMsgID(msgid), fMsg(lookupMessage(msgid)), fConfig(Config::makeConfig()) { } -Message::Message(const string msg): - fMsgID(0), fMsg(msg), fConfig(Config::makeConfig()) +Message::Message(const string msg) : fMsgID(0), fMsg(msg), fConfig(Config::makeConfig()) { } void Message::swap(Message& rhs) { - std::swap(fMsgID, rhs.fMsgID); - std::swap(fMsg, rhs.fMsg); - std::swap(fConfig, rhs.fConfig); + std::swap(fMsgID, rhs.fMsgID); + std::swap(fMsg, rhs.fMsg); + std::swap(fConfig, rhs.fConfig); } void Message::Args::add(int i) { - fArgs.push_back(long(i)); + fArgs.push_back(long(i)); } void Message::Args::add(uint64_t u64) { - fArgs.push_back(u64); + fArgs.push_back(u64); } void Message::Args::add(const string& s) { - fArgs.push_back(s); + fArgs.push_back(s); } void Message::Args::add(double d) { - fArgs.push_back(d); + fArgs.push_back(d); } void Message::Args::reset() { - fArgs.clear(); + fArgs.clear(); } void Message::format(const Args& args) { - Args::AnyVec::const_iterator iter = args.args().begin(); - Args::AnyVec::const_iterator end = args.args().end(); + Args::AnyVec::const_iterator iter = args.args().begin(); + Args::AnyVec::const_iterator end = args.args().end(); - boost::format fmt(fMsg); - fmt.exceptions(boost::io::no_error_bits); + boost::format fmt(fMsg); + fmt.exceptions(boost::io::no_error_bits); - while (iter != end) + while (iter != end) + { + if (iter->type() == typeid(long)) { - if (iter->type() == typeid(long)) - { - long l = any_cast(*iter); - fmt % l; - } - else if (iter->type() == typeid(uint64_t)) - { - uint64_t u64 = any_cast(*iter); - fmt % u64; - } - else if (iter->type() == typeid(double)) - { - double d = any_cast(*iter); - fmt % d; - } - else if (iter->type() == typeid(string)) - { - string s = any_cast(*iter); - fmt % s; - } - else - { - throw logic_error("Message::format: unexpected type in argslist"); - } - - ++iter; + long l = any_cast(*iter); + fmt % l; + } + else if (iter->type() == typeid(uint64_t)) + { + uint64_t u64 = any_cast(*iter); + fmt % u64; + } + else if (iter->type() == typeid(double)) + { + double d = any_cast(*iter); + fmt % d; + } + else if (iter->type() == typeid(string)) + { + string s = any_cast(*iter); + fmt % s; + } + else + { + throw logic_error("Message::format: unexpected type in argslist"); } - fMsg = fmt.str(); + ++iter; + } + + fMsg = fmt.str(); } /* static */ const string Message::lookupMessage(const MessageID& msgid) { + if (!catalogLoaded) + { + boost::mutex::scoped_lock lock(mx); + if (!catalogLoaded) { - boost::mutex::scoped_lock lock(mx); - - if (!catalogLoaded) - { - loadCatalog(); - catalogLoaded = true; - } + loadCatalog(); + catalogLoaded = true; } + } - string msgstr; - CatMap::const_iterator iter = catmap.find(msgid); + string msgstr; + CatMap::const_iterator iter = catmap.find(msgid); + + if (iter == catmap.end()) + { + iter = catmap.find(0); if (iter == catmap.end()) { - iter = catmap.find(0); - - if (iter == catmap.end()) - { - msgstr = "%1% %2% %3% %4% %5%"; - } - else - { - msgstr = iter->second; - } + msgstr = "%1% %2% %3% %4% %5%"; } else { - msgstr = iter->second; + msgstr = iter->second; } + } + else + { + msgstr = iter->second; + } - ostringstream oss; - oss << "CAL" << setw(4) << setfill('0') << msgid << ": " << msgstr; - return oss.str(); + ostringstream oss; + oss << "CAL" << setw(4) << setfill('0') << msgid << ": " << msgstr; + return oss.str(); } void Message::reset() { - fMsg = lookupMessage(fMsgID); -} - + fMsg = lookupMessage(fMsgID); } +} // namespace logging diff --git a/utils/loggingcpp/messagelog.cpp b/utils/loggingcpp/messagelog.cpp index 8f5727908..b5ffed3e3 100644 --- a/utils/loggingcpp/messagelog.cpp +++ b/utils/loggingcpp/messagelog.cpp @@ -17,9 +17,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: messagelog.cpp 3658 2013-03-22 20:26:02Z chao $ -* -******************************************************************************************/ + * $Id: messagelog.cpp 3658 2013-03-22 20:26:02Z chao $ + * + ******************************************************************************************/ #include #include #include @@ -46,69 +46,64 @@ using namespace logging; /* * This list matches SubsystemIDs.txt, but has names matching Process Config DB */ -const vector SubsystemID = ba::list_of - ("Calpont") // id = 0 default - ("ddljoblist") // id = 1 - ("ddlpackage") // id = 2 - ("dmlpackage") // id = 3 - ("execplan") // id = 4 - ("joblist") // id = 5 - ("resultset") // id = 6 - ("mcsadmin") // id = 7 - ("oamcpp") // id = 8 - ("ServerMonitor") // id = 9 - ("traphandler") // id = 10 - ("alarmmanager") // id = 11 - ("configcpp") // id = 12 - ("loggingcpp") // id = 13 - ("messageqcpp") // id = 14 - ("DDLProc") // id = 15 - ("ExeMgr") // id = 16 - ("ProcessManager") // id = 17 - ("ProcessMonitor") // id = 18 - ("writeengine") // id = 19 - ("DMLProc") // id = 20 - ("dmlpackageproc") // id = 21 - ("threadpool") // id = 22 - ("ddlpackageproc") // id = 23 - ("dbcon") // id = 24 - ("DiskManager") // id = 25 - ("RouteMsg") // id = 26 - ("SQLBuffMgr") // id = 27 - ("PrimProc") // id = 28 - ("controllernode") // id = 29 - ("workernode") // id = 30 - ("messagequeue") // id = 31 - ("writeengineserver")//id = 32 - ("writeenginesplit")// id = 33 - ("cpimport.bin") // id = 34 - ("IDBFile") // id = 35 - ; +const vector SubsystemID = ba::list_of("Calpont") // id = 0 default + ("ddljoblist") // id = 1 + ("ddlpackage") // id = 2 + ("dmlpackage") // id = 3 + ("execplan") // id = 4 + ("joblist") // id = 5 + ("resultset") // id = 6 + ("mcsadmin") // id = 7 + ("oamcpp") // id = 8 + ("ServerMonitor") // id = 9 + ("traphandler") // id = 10 + ("alarmmanager") // id = 11 + ("configcpp") // id = 12 + ("loggingcpp") // id = 13 + ("messageqcpp") // id = 14 + ("DDLProc") // id = 15 + ("ExeMgr") // id = 16 + ("ProcessManager") // id = 17 + ("ProcessMonitor") // id = 18 + ("writeengine") // id = 19 + ("DMLProc") // id = 20 + ("dmlpackageproc") // id = 21 + ("threadpool") // id = 22 + ("ddlpackageproc") // id = 23 + ("dbcon") // id = 24 + ("DiskManager") // id = 25 + ("RouteMsg") // id = 26 + ("SQLBuffMgr") // id = 27 + ("PrimProc") // id = 28 + ("controllernode") // id = 29 + ("workernode") // id = 30 + ("messagequeue") // id = 31 + ("writeengineserver") // id = 32 + ("writeenginesplit") // id = 33 + ("cpimport.bin") // id = 34 + ("IDBFile") // id = 35 + ; string timestr() { - struct tm tm; - struct timeval tv; - gettimeofday(&tv, 0); + struct tm tm; + struct timeval tv; + gettimeofday(&tv, 0); #ifdef _MSC_VER - errno_t p = 0; - time_t t = (time_t)tv.tv_sec; - p = localtime_s(&tm, &t); + errno_t p = 0; + time_t t = (time_t)tv.tv_sec; + p = localtime_s(&tm, &t); - if (p != 0) - memset(&tm, 0, sizeof(tm)); + if (p != 0) + memset(&tm, 0, sizeof(tm)); #else - localtime_r(&tv.tv_sec, &tm); + localtime_r(&tv.tv_sec, &tm); #endif - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_sec - << '.' - << setw(6) << tv.tv_usec - ; - return oss.str(); + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_sec << '.' << setw(6) << tv.tv_usec; + return oss.str(); } void closeLog() @@ -117,162 +112,156 @@ void closeLog() void openLog(unsigned subsystemid, int localLogNum) { - if (subsystemid >= SubsystemID.size()) - subsystemid = 0; + if (subsystemid >= SubsystemID.size()) + subsystemid = 0; } const string escape_pct(const string& in) { - string out(in); - string::size_type pos; - pos = out.find('%', 0); + string out(in); + string::size_type pos; + pos = out.find('%', 0); - while (pos != string::npos) - { - out.replace(pos, 1, "%%"); - pos = out.find('%', pos + 2); - } + while (pos != string::npos) + { + out.replace(pos, 1, "%%"); + pos = out.find('%', pos + 2); + } - return out; + return out; } -} +} // namespace namespace logging { - -MessageLog::MessageLog(const LoggingID& initData, int localLogNum) : - fLogData(initData), fFacility(localLogNum) +MessageLog::MessageLog(const LoggingID& initData, int localLogNum) + : fLogData(initData), fFacility(localLogNum) { - openLog(fLogData.fSubsysID, fFacility); + openLog(fLogData.fSubsysID, fFacility); } void MessageLog::logData(const LoggingID& logData) { - if (fLogData.fSubsysID != logData.fSubsysID) - { - closeLog(); - openLog(logData.fSubsysID, fFacility); - } + if (fLogData.fSubsysID != logData.fSubsysID) + { + closeLog(); + openLog(logData.fSubsysID, fFacility); + } - fLogData = logData; + fLogData = logData; } MessageLog::~MessageLog() { - closeLog(); + closeLog(); } const string MessageLog::format(const Message& msg, const char prefix) { - ostringstream oss; - oss << timestr() << " |" - << fLogData.fSessionID - << '|' << fLogData.fTxnID - << '|' << fLogData.fThdID - << "| " - << prefix << ' ' << setw(2) << setfill('0') << fLogData.fSubsysID - << ' ' << msg.msg() - ; - return escape_pct(oss.str()); + ostringstream oss; + oss << timestr() << " |" << fLogData.fSessionID << '|' << fLogData.fTxnID << '|' << fLogData.fThdID << "| " + << prefix << ' ' << setw(2) << setfill('0') << fLogData.fSubsysID << ' ' << msg.msg(); + return escape_pct(oss.str()); } void MessageLog::logDebugMessage(const Message& msg) { - ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_CONS, fFacility); - ::syslog(LOG_DEBUG, "%s", format(msg, 'D').c_str()); - ::closelog(); + ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_CONS, fFacility); + ::syslog(LOG_DEBUG, "%s", format(msg, 'D').c_str()); + ::closelog(); } void MessageLog::logInfoMessage(const Message& msg) { - ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_CONS, fFacility); - ::syslog(LOG_INFO, "%s", format(msg, 'I').c_str()); - ::closelog(); + ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_CONS, fFacility); + ::syslog(LOG_INFO, "%s", format(msg, 'I').c_str()); + ::closelog(); } void MessageLog::logWarningMessage(const Message& msg) { - ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_CONS, fFacility); - ::syslog(LOG_WARNING, "%s", format(msg, 'W').c_str()); - ::closelog(); + ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_CONS, fFacility); + ::syslog(LOG_WARNING, "%s", format(msg, 'W').c_str()); + ::closelog(); } void MessageLog::logErrorMessage(const Message& msg) { - // @bug 24 use 'E' instead of 'S' - ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_PERROR | LOG_CONS, fFacility); - ::syslog(LOG_ERR, "%s", format(msg, 'E').c_str()); - ::closelog(); + // @bug 24 use 'E' instead of 'S' + ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_PERROR | LOG_CONS, fFacility); + ::syslog(LOG_ERR, "%s", format(msg, 'E').c_str()); + ::closelog(); } void MessageLog::logCriticalMessage(const Message& msg) { - ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_PERROR | LOG_CONS, fFacility); - ::syslog(LOG_CRIT, "%s", format(msg, 'C').c_str()); - ::closelog(); + ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID | LOG_PERROR | LOG_CONS, fFacility); + ::syslog(LOG_CRIT, "%s", format(msg, 'C').c_str()); + ::closelog(); } -//Bug 5218. comment out the following functions to alleviate issue where dml messages show up in crit.log. This +// Bug 5218. comment out the following functions to alleviate issue where dml messages show up in crit.log. +// This // will turn off data_mods.log. void logDML(unsigned sessionId, unsigned txnId, const string& statement, const string& owner) { - /* logging::Message::Args args; + /* logging::Message::Args args; - unsigned subsystemId = 20; // DMLProc - unsigned threadId = 0; // 0 for now + unsigned subsystemId = 20; // DMLProc + unsigned threadId = 0; // 0 for now - logging::LoggingID loggingId(subsystemId, sessionId, txnId, threadId); - logging::MessageLog messageLog(loggingId, LOG_LOCAL2); - logging::Message m(M0017); - args.add("|" + owner + "|" + statement); - m.format(args); - messageLog.logCriticalMessage(m); */ + logging::LoggingID loggingId(subsystemId, sessionId, txnId, threadId); + logging::MessageLog messageLog(loggingId, LOG_LOCAL2); + logging::Message m(M0017); + args.add("|" + owner + "|" + statement); + m.format(args); + messageLog.logCriticalMessage(m); */ } void logDDL(unsigned sessionId, unsigned txnId, const string& statement, const string& owner) { - /* - logging::Message::Args args; + /* + logging::Message::Args args; - unsigned subsystemId = 15; // DDLProc - unsigned threadId = 0; // 0 for now + unsigned subsystemId = 15; // DDLProc + unsigned threadId = 0; // 0 for now - logging::LoggingID loggingId(subsystemId, sessionId, txnId, threadId); - logging::MessageLog messageLog(loggingId, LOG_LOCAL2); - logging::Message m(M0018); - args.add("|" + owner + "|" + statement); - m.format(args); - messageLog.logCriticalMessage(m); */ + logging::LoggingID loggingId(subsystemId, sessionId, txnId, threadId); + logging::MessageLog messageLog(loggingId, LOG_LOCAL2); + logging::Message m(M0018); + args.add("|" + owner + "|" + statement); + m.format(args); + messageLog.logCriticalMessage(m); */ } void logCommand(unsigned sessionId, unsigned txnId, const string& statement) { - /* logging::Message::Args args; + /* logging::Message::Args args; - unsigned subsystemId = 20; // DMLProc - unsigned threadId = 0; // 0 for now + unsigned subsystemId = 20; // DMLProc + unsigned threadId = 0; // 0 for now - logging::LoggingID loggingId(subsystemId, sessionId, txnId, threadId); - logging::MessageLog messageLog(loggingId, LOG_LOCAL2); - logging::Message m(M0019); - args.add("|" + statement); - m.format(args); - messageLog.logCriticalMessage(m); */ + logging::LoggingID loggingId(subsystemId, sessionId, txnId, threadId); + logging::MessageLog messageLog(loggingId, LOG_LOCAL2); + logging::Message m(M0019); + args.add("|" + statement); + m.format(args); + messageLog.logCriticalMessage(m); */ } void logEventToDataLog(unsigned messageId, const string& messageText) { - /* logging::Message::Args args; + /* logging::Message::Args args; - unsigned subsystemId = 20; // DMLProc - unsigned threadId = 0; // 0 for now + unsigned subsystemId = 20; // DMLProc + unsigned threadId = 0; // 0 for now - logging::LoggingID loggingId(subsystemId, 0, 0, threadId); - logging::MessageLog messageLog(loggingId, LOG_LOCAL2); - logging::Message m(messageId); - args.add(messageText); - m.format(args); - messageLog.logCriticalMessage(m); */ + logging::LoggingID loggingId(subsystemId, 0, 0, threadId); + logging::MessageLog messageLog(loggingId, LOG_LOCAL2); + logging::Message m(messageId); + args.add(messageText); + m.format(args); + messageLog.logCriticalMessage(m); */ } -} //namespace logging +} // namespace logging diff --git a/utils/loggingcpp/messagelog.h b/utils/loggingcpp/messagelog.h index 2e2955e9e..4385657ef 100644 --- a/utils/loggingcpp/messagelog.h +++ b/utils/loggingcpp/messagelog.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: messagelog.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -******************************************************************************************/ + * $Id: messagelog.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -33,19 +33,17 @@ class MessageLoggingTest; namespace logging { - /** @brief Log Types -*/ + */ enum LOG_TYPE { - LOG_TYPE_DEBUG, // 0 = debug - LOG_TYPE_INFO, // 1 = info - LOG_TYPE_WARNING, // 2 = warning - LOG_TYPE_ERROR, // 3 = error - LOG_TYPE_CRITICAL // 4 = critical + LOG_TYPE_DEBUG, // 0 = debug + LOG_TYPE_INFO, // 1 = info + LOG_TYPE_WARNING, // 2 = warning + LOG_TYPE_ERROR, // 3 = error + LOG_TYPE_CRITICAL // 4 = critical }; - class Message; /** @brief a message log class @@ -53,105 +51,104 @@ class Message; */ class MessageLog { -public: - /** @brief ctor - * @param initData the LogginID object. - * @param facility openLog will be called using this as the third parm. - */ - MessageLog(const LoggingID& initData, int facility = LOG_LOCAL1); + public: + /** @brief ctor + * @param initData the LogginID object. + * @param facility openLog will be called using this as the third parm. + */ + MessageLog(const LoggingID& initData, int facility = LOG_LOCAL1); - /** @brief dtor - * - */ - ~MessageLog(); + /** @brief dtor + * + */ + ~MessageLog(); - /** @brief log a debug message - * - * @param msg the message to log - */ - void logDebugMessage(const Message& msg); + /** @brief log a debug message + * + * @param msg the message to log + */ + void logDebugMessage(const Message& msg); - /** @brief log an info message - * - * @param msg the message to log - */ - void logInfoMessage(const Message& msg); + /** @brief log an info message + * + * @param msg the message to log + */ + void logInfoMessage(const Message& msg); - /** @brief log a warning message - * - * @param msg the message to log - */ - void logWarningMessage(const Message& msg); + /** @brief log a warning message + * + * @param msg the message to log + */ + void logWarningMessage(const Message& msg); - /** @brief log an error message - * - * @param msg the message to log - */ - void logErrorMessage(const Message& msg); + /** @brief log an error message + * + * @param msg the message to log + */ + void logErrorMessage(const Message& msg); - /** @brief log a serious message - * - * @param msg the message to log - * @note this is the same as calling logErrorMessage() - * @bug 24 add logErrorMessage() and make logSeriousMessage() call it - */ - void logSeriousMessage(const Message& msg) - { - logErrorMessage(msg); - } + /** @brief log a serious message + * + * @param msg the message to log + * @note this is the same as calling logErrorMessage() + * @bug 24 add logErrorMessage() and make logSeriousMessage() call it + */ + void logSeriousMessage(const Message& msg) + { + logErrorMessage(msg); + } - /** @brief log a critial message - * - * @param msg the message to log - */ - void logCriticalMessage(const Message& msg); + /** @brief log a critial message + * + * @param msg the message to log + */ + void logCriticalMessage(const Message& msg); - /** @brief LoggingID mutator - * - * @param logData the new LoggingID. If the subsystem id is changed, the syslog connection will be closed - * and reopened. - */ - void logData(const LoggingID& logData); + /** @brief LoggingID mutator + * + * @param logData the new LoggingID. If the subsystem id is changed, the syslog connection will be closed + * and reopened. + */ + void logData(const LoggingID& logData); - /** @brief LoggingID accessor - * - */ - const LoggingID& logData() const - { - return fLogData; - } + /** @brief LoggingID accessor + * + */ + const LoggingID& logData() const + { + return fLogData; + } - friend class ::MessageLoggingTest; + friend class ::MessageLoggingTest; -protected: + protected: + const std::string format(const Message& msg, const char prefix = 'U'); - const std::string format(const Message& msg, const char prefix = 'U'); + private: + // defaults okay + // MessageLog(const MessageLog& rhs); + // MessageLog& operator=(const MessageLog& rhs); -private: - //defaults okay - //MessageLog(const MessageLog& rhs); - //MessageLog& operator=(const MessageLog& rhs); - - LoggingID fLogData; /// the logging context data - int fFacility; /// the syslog facility number + LoggingID fLogData; /// the logging context data + int fFacility; /// the syslog facility number }; /** @brief logs the DML statement using a Syslog critical message to LOG_LOCAL2. -*/ + */ void logDML(unsigned sessionId, unsigned txnId, const std::string& statement, const std::string& owner); /** @brief logs the DDL statement using a Syslog critical message to LOG_LOCAL2. -*/ + */ void logDDL(unsigned sessionId, unsigned txnId, const std::string& statement, const std::string& owner); /** @brief logs the commit or rollback statement using a Syslog critical message to LOG_LOCAL2. -*/ + */ void logCommand(unsigned sessionId, unsigned txnId, const std::string& statement); /** @brief logs the event using a Syslog critical message to LOG_LOCAL2. Used for messages 19..24. -*/ + */ void logEventToDataLog(unsigned messageId, const std::string& messageText); -} +} // namespace logging #endif diff --git a/utils/loggingcpp/messageobj.h b/utils/loggingcpp/messageobj.h index 735dd847b..0c0031013 100644 --- a/utils/loggingcpp/messageobj.h +++ b/utils/loggingcpp/messageobj.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: messageobj.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -******************************************************************************************/ + * $Id: messageobj.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + ******************************************************************************************/ /** * @file */ @@ -40,127 +40,125 @@ class MessageLoggingTest; namespace logging { - /** @brief a message class * * Contains a message to be logged */ class Message { -public: - /** @brief the args to a formatted message - */ - class Args + public: + /** @brief the args to a formatted message + */ + class Args + { + public: + /** @brief the args vector + */ + typedef std::vector AnyVec; + + // default ctors & dtors okay + + /** @brief add an int arg to the message + */ + void add(int i); + + /** @brief add an unsigned 64 bit int arg to the message + */ + void add(uint64_t i); + + /** @brief add a float arg to the message + */ + void add(double d); + + /** @brief add a string arg to the message + */ + void add(const std::string& s); + + /** @brief args accessor + */ + const AnyVec& args() const { - public: - /** @brief the args vector - */ - typedef std::vector AnyVec; - - // default ctors & dtors okay - - /** @brief add an int arg to the message - */ - void add(int i); - - /** @brief add an unsigned 64 bit int arg to the message - */ - void add(uint64_t i); - - /** @brief add a float arg to the message - */ - void add(double d); - - /** @brief add a string arg to the message - */ - void add(const std::string& s); - - /** @brief args accessor - */ - const AnyVec& args() const - { - return fArgs; - } - - /** @brief reset the args list - * - */ - void reset(); - - friend class ::MessageLoggingTest; - - private: - AnyVec fArgs; /// the args vector - }; - - /** @brief the MessageID type - */ - typedef unsigned MessageID; - - /** @brief default ctor - */ - explicit Message(const MessageID msgid = 0); - - /** @brief ctor turn a message string to a Message - * - * For error handling framework use to log error - */ - Message(const std::string msg); - - /** @brief format message with args - */ - void format(const Args& args); - - /** @brief msg accessor - */ - const std::string& msg() const - { - return fMsg; + return fArgs; } - /** @brief swap - */ - void swap(Message& rhs); - - /** @brief lookup the message format string - * - * looks up the message format string for msgid. Returns a reasonable - * default if one can't be found. - */ - static const std::string lookupMessage(const MessageID& msgid); - - /** @brief reset the formated string - * - * The original, unformated string is restored, and this Message - * is then ready for a new call to format(). - */ + /** @brief reset the args list + * + */ void reset(); - /** @brief msgID accessor - */ - const MessageID& msgID() const - { - return fMsgID; - } - friend class ::MessageLoggingTest; -private: - //defaults okay - //Message(const Message& rhs); - //Message& operator=(const Message& rhs); + private: + AnyVec fArgs; /// the args vector + }; - MessageID fMsgID; /// the msgID - std::string fMsg; /// the formated or unformated message - config::Config* fConfig; /// config file ptr + /** @brief the MessageID type + */ + typedef unsigned MessageID; + + /** @brief default ctor + */ + explicit Message(const MessageID msgid = 0); + + /** @brief ctor turn a message string to a Message + * + * For error handling framework use to log error + */ + Message(const std::string msg); + + /** @brief format message with args + */ + void format(const Args& args); + + /** @brief msg accessor + */ + const std::string& msg() const + { + return fMsg; + } + + /** @brief swap + */ + void swap(Message& rhs); + + /** @brief lookup the message format string + * + * looks up the message format string for msgid. Returns a reasonable + * default if one can't be found. + */ + static const std::string lookupMessage(const MessageID& msgid); + + /** @brief reset the formated string + * + * The original, unformated string is restored, and this Message + * is then ready for a new call to format(). + */ + void reset(); + + /** @brief msgID accessor + */ + const MessageID& msgID() const + { + return fMsgID; + } + + friend class ::MessageLoggingTest; + + private: + // defaults okay + // Message(const Message& rhs); + // Message& operator=(const Message& rhs); + + MessageID fMsgID; /// the msgID + std::string fMsg; /// the formated or unformated message + config::Config* fConfig; /// config file ptr }; inline void swap(logging::Message& lhs, logging::Message& rhs) { - lhs.swap(rhs); + lhs.swap(rhs); } -}//namespace logging - +} // namespace logging #endif diff --git a/utils/loggingcpp/sqllogger.cpp b/utils/loggingcpp/sqllogger.cpp index 7ea6ccaba..88702bae5 100644 --- a/utils/loggingcpp/sqllogger.cpp +++ b/utils/loggingcpp/sqllogger.cpp @@ -31,47 +31,51 @@ using namespace std; namespace logging { - const unsigned LogDefaultMsg = M0000; const unsigned LogStartSql = M0041; const unsigned LogEndSql = M0042; -//We can't use a member Logger here. When used with DML and DDL, the syslog gets closed by calls to DMLLog and DDLLog. +// We can't use a member Logger here. When used with DML and DDL, the syslog gets closed by calls to DMLLog +// and DDLLog. SQLLogger::SQLLogger(const std::string sql, unsigned subsys, unsigned session, unsigned txn, unsigned thread) - : fLogId(subsys, session, txn, thread), fLog(!sql.empty()) + : fLogId(subsys, session, txn, thread), fLog(!sql.empty()) { - makeMsgMap(); + makeMsgMap(); - if (fLog) logMessage(LOG_TYPE_DEBUG, sql, LogStartSql); + if (fLog) + logMessage(LOG_TYPE_DEBUG, sql, LogStartSql); } SQLLogger::SQLLogger(const std::string sql, const LoggingID& logId) - : fLogId(logId.fSubsysID, logId.fSessionID, logId.fTxnID, logId.fThdID), fLog(!sql.empty()) + : fLogId(logId.fSubsysID, logId.fSessionID, logId.fTxnID, logId.fThdID), fLog(!sql.empty()) { - makeMsgMap(); + makeMsgMap(); - if (fLog) logMessage(LOG_TYPE_DEBUG, sql, LogStartSql); + if (fLog) + logMessage(LOG_TYPE_DEBUG, sql, LogStartSql); } void SQLLogger::makeMsgMap() { - fMsgMap[LogDefaultMsg] = Message(LogDefaultMsg); - fMsgMap[LogStartSql] = Message(LogStartSql); - fMsgMap[LogEndSql] = Message(LogEndSql); + fMsgMap[LogDefaultMsg] = Message(LogDefaultMsg); + fMsgMap[LogStartSql] = Message(LogStartSql); + fMsgMap[LogEndSql] = Message(LogEndSql); } SQLLogger::~SQLLogger() { - if (fLog) logMessage(LOG_TYPE_DEBUG, "", LogEndSql); + if (fLog) + logMessage(LOG_TYPE_DEBUG, "", LogEndSql); } -std::string SQLLogger::logMessage(logging::LOG_TYPE logLevel, const std::string& msg, logging::Message::MessageID mid ) +std::string SQLLogger::logMessage(logging::LOG_TYPE logLevel, const std::string& msg, + logging::Message::MessageID mid) { - logging::Message::Args args; - args.add(msg); - Logger logger(fLogId.fSubsysID); - logger.msgMap(fMsgMap); - return logger.logMessage(logLevel, mid, args, fLogId); + logging::Message::Args args; + args.add(msg); + Logger logger(fLogId.fSubsysID); + logger.msgMap(fMsgMap); + return logger.logMessage(logLevel, mid, args, fLogId); } -} +} // namespace logging diff --git a/utils/loggingcpp/sqllogger.h b/utils/loggingcpp/sqllogger.h index 0af43fee2..5cdfd22d5 100644 --- a/utils/loggingcpp/sqllogger.h +++ b/utils/loggingcpp/sqllogger.h @@ -40,25 +40,25 @@ extern const unsigned LogStartSql; extern const unsigned LogEndSql; /** @brief writes sql start with sql statement in constructor and sql end in destructor in debug.log - if the sql text is not empty + if the sql text is not empty */ class SQLLogger { -public: - SQLLogger(const std::string sql, unsigned subsys, unsigned session, unsigned txn = 0, unsigned thread = 0); - SQLLogger(const std::string sql, const LoggingID& logId); + public: + SQLLogger(const std::string sql, unsigned subsys, unsigned session, unsigned txn = 0, unsigned thread = 0); + SQLLogger(const std::string sql, const LoggingID& logId); - ~SQLLogger(); + ~SQLLogger(); - std::string logMessage(logging::LOG_TYPE logLevel, const std::string& msg, logging::Message::MessageID mid = LogDefaultMsg ); + std::string logMessage(logging::LOG_TYPE logLevel, const std::string& msg, + logging::Message::MessageID mid = LogDefaultMsg); -private: - void makeMsgMap(); - MsgMap fMsgMap; - LoggingID fLogId; - bool fLog; + private: + void makeMsgMap(); + MsgMap fMsgMap; + LoggingID fLogId; + bool fLog; }; -} +} // namespace logging #endif - diff --git a/utils/loggingcpp/stopwatch.cpp b/utils/loggingcpp/stopwatch.cpp index 9a53bc2c8..fccf49278 100644 --- a/utils/loggingcpp/stopwatch.cpp +++ b/utils/loggingcpp/stopwatch.cpp @@ -75,172 +75,167 @@ using namespace std; namespace logging { - void StopWatch::stop(const string& message) { - stop(message, 1); + stop(message, 1); } bool StopWatch::stop(const string& message, const int limit) { - gettimeofday(&fTvLast, 0); - fOpenCalls--; - bool found = false; - uint32_t idx = 0; + gettimeofday(&fTvLast, 0); + fOpenCalls--; + bool found = false; + uint32_t idx = 0; - for (uint32_t i = 0; i < fProcessStats.size(); i++) + for (uint32_t i = 0; i < fProcessStats.size(); i++) + { + if (fProcessStats[i].fProcess == message) { - if (fProcessStats[i].fProcess == message) - { - idx = i; - found = true; - break; - } + idx = i; + found = true; + break; } + } - if (!found) - { - //throw std::runtime_error("StopWatch::stop " + message + " called without calling start first."); - std::cerr << "StopWatch receiving STOP for unknown event: " << message << std::endl; - return false; - } - - fProcessStats[idx].processStop(); - - if (fProcessStats[idx].fStopCount >= limit) - return true; - + if (!found) + { + // throw std::runtime_error("StopWatch::stop " + message + " called without calling start first."); + std::cerr << "StopWatch receiving STOP for unknown event: " << message << std::endl; return false; + } + + fProcessStats[idx].processStop(); + + if (fProcessStats[idx].fStopCount >= limit) + return true; + + return false; } void StopWatch::start(const string& message) { - fOpenCalls++; - gettimeofday(&fTvLast, 0); - bool found = false; - uint32_t idx = 0; - ProcessStats processStats; + fOpenCalls++; + gettimeofday(&fTvLast, 0); + bool found = false; + uint32_t idx = 0; + ProcessStats processStats; - if (!fStarted) + if (!fStarted) + { + fStarted = true; + gettimeofday(&fTvStart, 0); + } + + for (uint32_t i = 0; i < fProcessStats.size(); i++) + { + if (fProcessStats[i].fProcess == message) { - fStarted = true; - gettimeofday(&fTvStart, 0); + idx = i; + found = true; + break; } + } - for (uint32_t i = 0; i < fProcessStats.size(); i++) - { - if (fProcessStats[i].fProcess == message) - { - idx = i; - found = true; - break; - } - } + if (!found) + { + fProcessStats.push_back(processStats); + idx = fProcessStats.size() - 1; + } - if (!found) - { - fProcessStats.push_back(processStats); - idx = fProcessStats.size() - 1; - } - - fProcessStats[idx].fProcess = message; - fProcessStats[idx].processStart(); + fProcessStats[idx].fProcess = message; + fProcessStats[idx].processStart(); } void StopWatch::finish() { - ostringstream oss; + ostringstream oss; - oss << endl; - oss << "Seconds Percentage Calls Description" << endl; + oss << endl; + oss << "Seconds Percentage Calls Description" << endl; - // total seconds elapsed - double totalSeconds = 1.0; + // total seconds elapsed + double totalSeconds = 1.0; - // Add a last entry into the vector for total. - ProcessStats total; - total.fProcess = "Total"; + // Add a last entry into the vector for total. + ProcessStats total; + total.fProcess = "Total"; - if (fProcessStats.size() > 0) + if (fProcessStats.size() > 0) + { + // Calculate the total seconds elapsed. + totalSeconds = + (fTvLast.tv_sec + (fTvLast.tv_usec / 1000000.0)) - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); + total.fTotalSeconds = totalSeconds; + total.fStartCount = 1; + } + else + { + total.fTotalSeconds = 0.0; + total.fStartCount = 0; + } + + fProcessStats.push_back(total); + + for (uint32_t i = 0; i < fProcessStats.size(); i++) + { + if (i == (fProcessStats.size() - 1)) { - // Calculate the total seconds elapsed. - totalSeconds = - (fTvLast.tv_sec + (fTvLast.tv_usec / 1000000.0)) - - (fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0)); - total.fTotalSeconds = totalSeconds; - total.fStartCount = 1; + oss << endl; } + + // Seconds. + string seconds; + ostringstream ossTemp; + ossTemp << fProcessStats[i].fTotalSeconds; + seconds = ossTemp.str(); + seconds.resize(11, ' '); + oss << seconds << " "; + + // Percentage. + string percentage; + ossTemp.str(""); // clear the stream. + ossTemp << (fProcessStats[i].fTotalSeconds / totalSeconds) * 100.0; + percentage = ossTemp.str(); + percentage.resize(11, ' '); + oss << percentage << "% "; + + // Times Initiated. + ossTemp.str(""); // clear the stream. + ossTemp << fProcessStats[i].fStartCount; + string timesInitiated = ossTemp.str(); + timesInitiated.resize(10, ' '); + oss << timesInitiated << " "; + + // Description. + if (fId >= 0) + oss << fId << ": " << fProcessStats[i].fProcess << endl; else - { - total.fTotalSeconds = 0.0; - total.fStartCount = 0; - } + oss << fProcessStats[i].fProcess << endl; + } - fProcessStats.push_back(total); + if (fOutputToFile) + { + ofstream profLog; + profLog.open(fLogFile.c_str(), std::ios::app); - for (uint32_t i = 0; i < fProcessStats.size(); i++) - { + // Output the date and time. + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + profLog << endl << timeString; - if (i == (fProcessStats.size() - 1)) - { - oss << endl; - } - - // Seconds. - string seconds; - ostringstream ossTemp; - ossTemp << fProcessStats[i].fTotalSeconds; - seconds = ossTemp.str(); - seconds.resize(11, ' '); - oss << seconds << " "; - - // Percentage. - string percentage; - ossTemp.str(""); // clear the stream. - ossTemp << (fProcessStats[i].fTotalSeconds / totalSeconds) * 100.0; - percentage = ossTemp.str(); - percentage.resize(11, ' '); - oss << percentage << "% "; - - // Times Initiated. - ossTemp.str(""); // clear the stream. - ossTemp << fProcessStats[i].fStartCount; - string timesInitiated = ossTemp.str(); - timesInitiated.resize(10, ' '); - oss << timesInitiated << " "; - - // Description. - if (fId >= 0) - oss << fId << ": " << fProcessStats[i].fProcess << endl; - else - oss << fProcessStats[i].fProcess << endl; - } - - if (fOutputToFile) - { - - ofstream profLog; - profLog.open(fLogFile.c_str(), std::ios::app); - - // Output the date and time. - time_t t = time(0); - char timeString[50]; - ctime_r(&t, timeString); - timeString[ strlen(timeString) - 1 ] = '\0'; - profLog << endl << timeString; - - // Output the stopwatch info. - profLog << oss.str(); - } - else - { - cout << oss.str(); - } - - // Clear everything out. - fStarted = false; - fProcessStats.clear(); + // Output the stopwatch info. + profLog << oss.str(); + } + else + { + cout << oss.str(); + } + // Clear everything out. + fStarted = false; + fProcessStats.clear(); } -} // end of logging namespace +} // namespace logging diff --git a/utils/loggingcpp/stopwatch.h b/utils/loggingcpp/stopwatch.h index aa26a36ab..1190f2e77 100644 --- a/utils/loggingcpp/stopwatch.h +++ b/utils/loggingcpp/stopwatch.h @@ -74,68 +74,67 @@ namespace logging { - class StopWatch { -public: - void start(const std::string& message); - bool stop(const std::string& message, const int limit); - void stop(const std::string& message); - void finish(); + public: + void start(const std::string& message); + bool stop(const std::string& message, const int limit); + void stop(const std::string& message); + void finish(); - bool isActive() + bool isActive() + { + return fOpenCalls > 0; + } + StopWatch() : fStarted(false), fId(-1), fOpenCalls(0), fOutputToFile(false), fLogFile(""){}; + StopWatch(int id) : fStarted(false), fId(id), fOpenCalls(0), fOutputToFile(false), fLogFile(""){}; + StopWatch(const std::string& fileName) + : fStarted(false), fId(-1), fOpenCalls(0), fOutputToFile(true), fLogFile(fileName) + { + } + struct ::timeval fTvLast; + int getId() + { + return fId; + } + + private: + class ProcessStats + { + public: + std::string fProcess; + struct timeval fTvProcessStarted; + double fTotalSeconds; + int64_t fStartCount; + int64_t fStopCount; + + ProcessStats() : fProcess(""), fTotalSeconds(0.0), fStartCount(0), fStopCount(0){}; + + void processStart() { - return fOpenCalls > 0; - } - StopWatch() : fStarted(false), fId(-1), fOpenCalls(0), fOutputToFile(false), fLogFile("") {}; - StopWatch(int id) : fStarted(false), fId(id), fOpenCalls(0), fOutputToFile(false), fLogFile("") {}; - StopWatch(const std::string& fileName) : fStarted(false), fId(-1), fOpenCalls(0), fOutputToFile(true), fLogFile(fileName) {} - struct ::timeval fTvLast; - int getId() - { - return fId; + gettimeofday(&fTvProcessStarted, 0); + fStartCount++; } -private: - class ProcessStats + void processStop() { - public: + struct timeval tvStop; + gettimeofday(&tvStop, 0); + fStopCount++; + fTotalSeconds += (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - + (fTvProcessStarted.tv_sec + (fTvProcessStarted.tv_usec / 1000000.0)); + } + }; - std::string fProcess; - struct timeval fTvProcessStarted; - double fTotalSeconds; - int64_t fStartCount; - int64_t fStopCount; - - ProcessStats() : fProcess(""), fTotalSeconds(0.0), fStartCount(0), fStopCount(0) {}; - - void processStart() - { - gettimeofday(&fTvProcessStarted, 0); - fStartCount++; - } - - void processStop() - { - struct timeval tvStop; - gettimeofday(&tvStop, 0); - fStopCount++; - fTotalSeconds += - (tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) - - (fTvProcessStarted.tv_sec + (fTvProcessStarted.tv_usec / 1000000.0)); - - } - }; - - struct timeval fTvStart; - std::vector fProcessStats; - bool fStarted; - int fId; - int fOpenCalls; - bool fOutputToFile; - std::string fLogFile; + struct timeval fTvStart; + std::vector fProcessStats; + bool fStarted; + int fId; + int fOpenCalls; + bool fOutputToFile; + std::string fLogFile; }; -} // end of logging namespace +} // namespace logging -#endif // STOPWATCH_H +#endif // STOPWATCH_H diff --git a/utils/loggingcpp/tdriver.cpp b/utils/loggingcpp/tdriver.cpp index 3e1342afa..8fab12711 100644 --- a/utils/loggingcpp/tdriver.cpp +++ b/utils/loggingcpp/tdriver.cpp @@ -30,348 +30,343 @@ using namespace config; class MessageLoggingTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(MessageLoggingTest); - CPPUNIT_TEST_SUITE( MessageLoggingTest ); + CPPUNIT_TEST(m1); + CPPUNIT_TEST(m2); + CPPUNIT_TEST(m3); + CPPUNIT_TEST(m4); + CPPUNIT_TEST(m5); - CPPUNIT_TEST( m1 ); - CPPUNIT_TEST( m2 ); - CPPUNIT_TEST( m3 ); - CPPUNIT_TEST( m4 ); - CPPUNIT_TEST( m5 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + LoggingID lid; -private: - LoggingID lid; + public: + void setUp() + { + setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); + } -public: - void setUp() + void tearDown() + { + } + + void m1() + { + Message::Args args; + args.add("hello"); + args.add("world"); + args.add(123); + args.add(1234.55); + Message m(100); + m.fMsg = "%1% %2% %3% %4%"; + m.format(args); + // CPPUNIT_ASSERT(m.msg() == "hello world 123 1234.55"); + m.reset(); + m.fMsg = "%1% %2% %3% %4%"; + args.reset(); + m.format(args); + // CPPUNIT_ASSERT(m.msg() == " "); + LoggingID lid(7); + MessageLog ml(lid); + m.reset(); + args.add("hello"); + args.add("world"); + args.add(123); + args.add(1234.55); + m.format(args); + ml.logDebugMessage(m); + + args.reset(); + args.add("begin CEP generation"); + m.reset(); + m.format(args); + ml.logInfoMessage(m); + + args.reset(); + args.add("end CEP generation"); + m.reset(); + m.format(args); + ml.logInfoMessage(m); + + args.reset(); + args.add("something took too long"); + m.reset(); + m.format(args); + ml.logWarningMessage(m); + + args.reset(); + args.add("something seriously took too long"); + m.reset(); + m.format(args); + ml.logSeriousMessage(m); + ml.logErrorMessage(m); + + args.reset(); + args.add("something critical took too long"); + m.reset(); + m.format(args); + ml.logCriticalMessage(m); + + LoggingID lid1; + MessageLog ml1(lid1); + args.reset(); + m.reset(); + args.add("subsystem 0 = Calpont test"); + m.format(args); + ml1.logDebugMessage(m); + + LoggingID lid2(1000); + MessageLog ml2(lid2); + args.reset(); + m.reset(); + args.add("subsystem above MAX = Calpont test"); + m.format(args); + ml2.logDebugMessage(m); + + LoggingID lid3(7); + MessageLog ml3(lid3); + args.reset(); + m.reset(); + args.add("subsystem 7 = mcsadmin test"); + m.format(args); + ml3.logDebugMessage(m); + Config::deleteInstanceMap(); + } + + void m2() + { + Message m1(100); + CPPUNIT_ASSERT(m1.msgID() == 100); + Message m2(10); + CPPUNIT_ASSERT(m2.msgID() == 10); + m2 = m1; + CPPUNIT_ASSERT(m2.msgID() == 100); + Message m3(m2); + CPPUNIT_ASSERT(m3.msgID() == m2.msgID()); + Message m4(99); + Message m5(199); + CPPUNIT_ASSERT(m4.msgID() == 99); + CPPUNIT_ASSERT(m5.msgID() == 199); + m4.swap(m5); + CPPUNIT_ASSERT(m5.msgID() == 99); + CPPUNIT_ASSERT(m4.msgID() == 199); + Config::deleteInstanceMap(); + } + + void m3() + { + LoggingID lid1(1, 2, 3, 4); + MessageLog ml1(lid1); + CPPUNIT_ASSERT(ml1.fLogData.fSubsysID == 1); + + LoggingID lid2(10, 20, 30, 40); + MessageLog ml2(lid2); + CPPUNIT_ASSERT(ml2.fLogData.fSubsysID == 10); + + ml2 = ml1; + CPPUNIT_ASSERT(ml2.fLogData.fSubsysID == 1); + + MessageLog ml3(ml2); + CPPUNIT_ASSERT(ml3.fLogData.fSubsysID == 1); + Config::deleteInstanceMap(); + } + + void m4() + { + LoggingID lid1(100, 200, 300, 400); + MessageLog ml1(lid1); + Message::Args args; + Message* m; + args.add("hello"); + args.add("world"); + args.add(123); + args.add(1234.55); + + for (int i = 0; i < 4; i++) { - setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); + m = new Message(i); + m->format(args); + ml1.logDebugMessage(*m); + delete m; } - void tearDown() + Config::deleteInstanceMap(); + } + + //------------------------------------------------------------------------------ + // This method is intended to test the messages used to profile db performance. + // The method also provides an example on how to use these log messages. + // Test can be verified by viewing /var/log/mariadb/columnstore/debug.log. + // + // Message types are: + // + // 26 Start Transaction + // 27 End Transaction + // 28 Start Statement + // 29 End Statement + // 30 Start Step + // 31 End Step + // 32 I/O Reads + // + // The messages should be logged in the following way: + // + // 1. The application should log a StartTransaction message at the beginning + // of a database transaction. + // 2. When the application begins processing a statement, the StartStatement + // message should be logged. + // 3. As each primitive step is executed, it's start time should be recorded + // by logging a StartStep message. + // 4. During the execution of a step, 1 or more I/O Read messages should be + // logged to record the I/O block count used in accessing each object. If + // necessary, more than 1 message can be logged for the same object and + // step. In these cases, the script that post-processes the syslog will + // add up the block counts for the same object and step. + // 5. Upon completion of each primitive step, an EndStep message should be + // logged. If multiple steps are executing in parallel, the EndStep msg + // should be logged as each step completes. After all these stpes are + // completed and logged, if needed, a new set of StartStep messages can + // be logged (for the same statement), for a new set of parallel steps. + // 6. Upon completion of each statement, an EndStatement msg should be logged. + // 7. Upon completion of each transaction, an EndTransaction msg should + // be logged. + // + // Some possible enhancements to simplify this profile logging for the + // application programmer: + // + // 1. Define enum or const ints in a common header file to represent the + // list of valid subSystem ids. + // + // 2. Add set of helper methods to MessageLog class to reduce the + // amount of work for the application. For example a method like: + // + // void logStartStatement ( int statement, int ver, string SQL ); + // + // would allow the application to log a StartStatement by simply doing; + // + // Message msgStartStatement ( M0028 ); + // int statementId = 11; + // int versionId = 22; + // string sql ("SELECT column1, column2 FROM table1 WHERE ... + // msgStartStatement.logStartStatement ( statementId, versionId, sql); + // + // 3. Could also do something similar to #2 except instead of adding helper + // methods to MessageLog, we could add specialized classes that derive + // from MessageLog or contain a MessageLog. One advantage of doing it + // this way is that a derived class like MessageLogStatement "could" be + // implemented to log the StartStatement, and its desctructor could then + // log the EndStatement automatically, on behalf of the application. + // + //------------------------------------------------------------------------------ + void m5() + { + int subSystem = 5; // joblist subSystem + int session = 100; + int transaction = 1; + int thread = 0; + + LoggingID lid1(subSystem, session, transaction, thread); + MessageLog msgLog(lid1); + + Message::Args args; + + // Log the start time of a transaction + Message msgStartTrans(M0026); + msgStartTrans.format(args); + msgLog.logDebugMessage(msgStartTrans); + + // Log the start of execution time for a SQL statement + Message msgStartStatement(M0028); + int statementId = 11; + int versionId = 22; + string sql("SELECT column1, column2 FROM table1 WHERE column1 = 345"); + args.reset(); + args.add(statementId); + args.add(versionId); + args.add(sql); + msgStartStatement.format(args); + msgLog.logDebugMessage(msgStartStatement); + + const string stepNames[] = {"steponeA", "steptwoA", "stepthreeA", "stepfourB", "stepfiveB", "stepsixB"}; + + // To process this SQL statement, simulate executing 2 job steps, + // with each job step consisting of of 3 parallel primitive steps + for (int jobStep = 0; jobStep < 2; jobStep++) { + int primStep1 = jobStep * 3; + + // Log 3 parallel steps starting to execute + for (int i = primStep1; i < (primStep1 + 3); i++) + { + Message msgStartStep(M0030); // Start Step + int stepId = i + 1; + string stepName = stepNames[i]; + args.reset(); + args.add(statementId); + args.add(stepId); + args.add(stepName); + msgStartStep.format(args); + msgLog.logDebugMessage(msgStartStep); + } + + // Record I/O block count for 0 or more objects per step; + // for this example we just record I/O for 1 object per step. + // Then log the completion of each step. + for (int i = primStep1; i < (primStep1 + 3); i++) + { + Message msgBlockCount(M0032); // I/O block count + int stepId = i + 1; + int objectId = stepId * 20; + int phyCount = stepId * 30; + int logCount = phyCount + 5; + args.reset(); + args.add(statementId); + args.add(stepId); + args.add(objectId); + args.add(phyCount); + args.add(logCount); + msgBlockCount.format(args); + msgLog.logDebugMessage(msgBlockCount); + + Message msgEndStep(M0031); // End Step + args.reset(); + args.add(statementId); + args.add(stepId); + msgEndStep.format(args); + msgLog.logDebugMessage(msgEndStep); + } } - void m1() - { - Message::Args args; - args.add("hello"); - args.add("world"); - args.add(123); - args.add(1234.55); - Message m(100); - m.fMsg = "%1% %2% %3% %4%"; - m.format(args); - //CPPUNIT_ASSERT(m.msg() == "hello world 123 1234.55"); - m.reset(); - m.fMsg = "%1% %2% %3% %4%"; - args.reset(); - m.format(args); - //CPPUNIT_ASSERT(m.msg() == " "); - LoggingID lid(7); - MessageLog ml(lid); - m.reset(); - args.add("hello"); - args.add("world"); - args.add(123); - args.add(1234.55); - m.format(args); - ml.logDebugMessage(m); + // Log the completion time of the SQL statement + Message msgEndStatement(M0029); + args.reset(); + args.add(statementId); + msgEndStatement.format(args); + msgLog.logDebugMessage(msgEndStatement); - args.reset(); - args.add("begin CEP generation"); - m.reset(); - m.format(args); - ml.logInfoMessage(m); + // Log the completion time of the transaction + Message msgEndTrans(M0027); + args.reset(); + args.add(string("COMMIT")); + msgEndTrans.format(args); + msgLog.logDebugMessage(msgEndTrans); + Config::deleteInstanceMap(); + } - args.reset(); - args.add("end CEP generation"); - m.reset(); - m.format(args); - ml.logInfoMessage(m); +}; // end of CppUnit::TestFixture class - args.reset(); - args.add("something took too long"); - m.reset(); - m.format(args); - ml.logWarningMessage(m); - - args.reset(); - args.add("something seriously took too long"); - m.reset(); - m.format(args); - ml.logSeriousMessage(m); - ml.logErrorMessage(m); - - args.reset(); - args.add("something critical took too long"); - m.reset(); - m.format(args); - ml.logCriticalMessage(m); - - LoggingID lid1; - MessageLog ml1(lid1); - args.reset(); - m.reset(); - args.add("subsystem 0 = Calpont test"); - m.format(args); - ml1.logDebugMessage(m); - - LoggingID lid2(1000); - MessageLog ml2(lid2); - args.reset(); - m.reset(); - args.add("subsystem above MAX = Calpont test"); - m.format(args); - ml2.logDebugMessage(m); - - LoggingID lid3(7); - MessageLog ml3(lid3); - args.reset(); - m.reset(); - args.add("subsystem 7 = mcsadmin test"); - m.format(args); - ml3.logDebugMessage(m); - Config::deleteInstanceMap(); - - } - - void m2() - { - Message m1(100); - CPPUNIT_ASSERT(m1.msgID() == 100); - Message m2(10); - CPPUNIT_ASSERT(m2.msgID() == 10); - m2 = m1; - CPPUNIT_ASSERT(m2.msgID() == 100); - Message m3(m2); - CPPUNIT_ASSERT(m3.msgID() == m2.msgID()); - Message m4(99); - Message m5(199); - CPPUNIT_ASSERT(m4.msgID() == 99); - CPPUNIT_ASSERT(m5.msgID() == 199); - m4.swap(m5); - CPPUNIT_ASSERT(m5.msgID() == 99); - CPPUNIT_ASSERT(m4.msgID() == 199); - Config::deleteInstanceMap(); - } - - void m3() - { - LoggingID lid1(1, 2, 3, 4); - MessageLog ml1(lid1); - CPPUNIT_ASSERT(ml1.fLogData.fSubsysID == 1); - - LoggingID lid2(10, 20, 30, 40); - MessageLog ml2(lid2); - CPPUNIT_ASSERT(ml2.fLogData.fSubsysID == 10); - - ml2 = ml1; - CPPUNIT_ASSERT(ml2.fLogData.fSubsysID == 1); - - MessageLog ml3(ml2); - CPPUNIT_ASSERT(ml3.fLogData.fSubsysID == 1); - Config::deleteInstanceMap(); - } - - void m4() - { - LoggingID lid1(100, 200, 300, 400); - MessageLog ml1(lid1); - Message::Args args; - Message* m; - args.add("hello"); - args.add("world"); - args.add(123); - args.add(1234.55); - - for (int i = 0; i < 4; i++) - { - m = new Message(i); - m->format(args); - ml1.logDebugMessage(*m); - delete m; - } - - Config::deleteInstanceMap(); - } - -//------------------------------------------------------------------------------ -// This method is intended to test the messages used to profile db performance. -// The method also provides an example on how to use these log messages. -// Test can be verified by viewing /var/log/mariadb/columnstore/debug.log. -// -// Message types are: -// -// 26 Start Transaction -// 27 End Transaction -// 28 Start Statement -// 29 End Statement -// 30 Start Step -// 31 End Step -// 32 I/O Reads -// -// The messages should be logged in the following way: -// -// 1. The application should log a StartTransaction message at the beginning -// of a database transaction. -// 2. When the application begins processing a statement, the StartStatement -// message should be logged. -// 3. As each primitive step is executed, it's start time should be recorded -// by logging a StartStep message. -// 4. During the execution of a step, 1 or more I/O Read messages should be -// logged to record the I/O block count used in accessing each object. If -// necessary, more than 1 message can be logged for the same object and -// step. In these cases, the script that post-processes the syslog will -// add up the block counts for the same object and step. -// 5. Upon completion of each primitive step, an EndStep message should be -// logged. If multiple steps are executing in parallel, the EndStep msg -// should be logged as each step completes. After all these stpes are -// completed and logged, if needed, a new set of StartStep messages can -// be logged (for the same statement), for a new set of parallel steps. -// 6. Upon completion of each statement, an EndStatement msg should be logged. -// 7. Upon completion of each transaction, an EndTransaction msg should -// be logged. -// -// Some possible enhancements to simplify this profile logging for the -// application programmer: -// -// 1. Define enum or const ints in a common header file to represent the -// list of valid subSystem ids. -// -// 2. Add set of helper methods to MessageLog class to reduce the -// amount of work for the application. For example a method like: -// -// void logStartStatement ( int statement, int ver, string SQL ); -// -// would allow the application to log a StartStatement by simply doing; -// -// Message msgStartStatement ( M0028 ); -// int statementId = 11; -// int versionId = 22; -// string sql ("SELECT column1, column2 FROM table1 WHERE ... -// msgStartStatement.logStartStatement ( statementId, versionId, sql); -// -// 3. Could also do something similar to #2 except instead of adding helper -// methods to MessageLog, we could add specialized classes that derive -// from MessageLog or contain a MessageLog. One advantage of doing it -// this way is that a derived class like MessageLogStatement "could" be -// implemented to log the StartStatement, and its desctructor could then -// log the EndStatement automatically, on behalf of the application. -// -//------------------------------------------------------------------------------ - void m5() - { - int subSystem = 5; // joblist subSystem - int session = 100; - int transaction = 1; - int thread = 0; - - LoggingID lid1 ( subSystem, session, transaction, thread ); - MessageLog msgLog ( lid1 ); - - Message::Args args; - - // Log the start time of a transaction - Message msgStartTrans ( M0026 ); - msgStartTrans.format ( args ); - msgLog.logDebugMessage ( msgStartTrans ); - - // Log the start of execution time for a SQL statement - Message msgStartStatement ( M0028 ); - int statementId = 11; - int versionId = 22; - string sql ("SELECT column1, column2 FROM table1 WHERE column1 = 345"); - args.reset ( ); - args.add ( statementId ); - args.add ( versionId ); - args.add ( sql ); - msgStartStatement.format ( args ); - msgLog.logDebugMessage ( msgStartStatement ); - - const string stepNames[] = { "steponeA", "steptwoA", "stepthreeA", - "stepfourB", "stepfiveB", "stepsixB" - }; - - // To process this SQL statement, simulate executing 2 job steps, - // with each job step consisting of of 3 parallel primitive steps - for (int jobStep = 0; jobStep < 2; jobStep++) - { - int primStep1 = jobStep * 3; - - // Log 3 parallel steps starting to execute - for (int i = primStep1; i < (primStep1 + 3); i++) - { - Message msgStartStep ( M0030 ); // Start Step - int stepId = i + 1; - string stepName = stepNames[i]; - args.reset ( ); - args.add ( statementId ); - args.add ( stepId ); - args.add ( stepName ); - msgStartStep.format ( args ); - msgLog.logDebugMessage ( msgStartStep ); - } - - // Record I/O block count for 0 or more objects per step; - // for this example we just record I/O for 1 object per step. - // Then log the completion of each step. - for (int i = primStep1; i < (primStep1 + 3); i++) - { - Message msgBlockCount ( M0032 ); // I/O block count - int stepId = i + 1; - int objectId = stepId * 20; - int phyCount = stepId * 30; - int logCount = phyCount + 5; - args.reset ( ); - args.add ( statementId ); - args.add ( stepId ); - args.add ( objectId ); - args.add ( phyCount ); - args.add ( logCount ); - msgBlockCount.format ( args ); - msgLog.logDebugMessage ( msgBlockCount ); - - Message msgEndStep ( M0031 ); // End Step - args.reset ( ); - args.add ( statementId ); - args.add ( stepId ); - msgEndStep.format ( args ); - msgLog.logDebugMessage ( msgEndStep ); - } - } - - // Log the completion time of the SQL statement - Message msgEndStatement ( M0029 ); - args.reset ( ); - args.add ( statementId ); - msgEndStatement.format ( args ); - msgLog.logDebugMessage ( msgEndStatement ); - - // Log the completion time of the transaction - Message msgEndTrans ( M0027 ); - args.reset ( ); - args.add ( string("COMMIT") ); - msgEndTrans.format ( args ); - msgLog.logDebugMessage ( msgEndTrans ); - Config::deleteInstanceMap(); - } - -}; // end of CppUnit::TestFixture class - -CPPUNIT_TEST_SUITE_REGISTRATION( MessageLoggingTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(MessageLoggingTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 4e1dbd14d..e65102df3 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -43,740 +43,739 @@ using namespace boost; namespace messageqcpp { - /* Copies only the data left to be read */ void ByteStream::doCopy(const ByteStream& rhs) { - uint32_t rlen = rhs.length(); + uint32_t rlen = rhs.length(); - if (fMaxLen < rlen) - { - delete [] fBuf; - fBuf = new uint8_t[rlen + ISSOverhead]; - fMaxLen = rlen; - } + if (fMaxLen < rlen) + { + delete[] fBuf; + fBuf = new uint8_t[rlen + ISSOverhead]; + fMaxLen = rlen; + } - memcpy(fBuf + ISSOverhead, rhs.fCurOutPtr, rlen); - fCurInPtr = fBuf + ISSOverhead + rlen; - fCurOutPtr = fBuf + ISSOverhead; - // Copy `longStrings` as well. - longStrings = rhs.longStrings; + memcpy(fBuf + ISSOverhead, rhs.fCurOutPtr, rlen); + fCurInPtr = fBuf + ISSOverhead + rlen; + fCurOutPtr = fBuf + ISSOverhead; + // Copy `longStrings` as well. + longStrings = rhs.longStrings; } -ByteStream::ByteStream(const ByteStream& rhs) : - fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) +ByteStream::ByteStream(const ByteStream& rhs) : fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) { - //don't need to copy an empty ByteStream - if (rhs.fBuf) - doCopy(rhs); + // don't need to copy an empty ByteStream + if (rhs.fBuf) + doCopy(rhs); } -ByteStream::ByteStream(const SBS& rhs) : - fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) +ByteStream::ByteStream(const SBS& rhs) : fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) { - if (rhs->fBuf) - doCopy(*rhs); + if (rhs->fBuf) + doCopy(*rhs); } ByteStream& ByteStream::operator=(const ByteStream& rhs) { - if (this != &rhs) + if (this != &rhs) + { + if (rhs.fBuf) + doCopy(rhs); + else { - if (rhs.fBuf) - doCopy(rhs); - else - { - delete [] fBuf; - fBuf = fCurInPtr = fCurOutPtr = 0; - fMaxLen = 0; - // Clear `longStrings`. - longStrings.clear(); - } + delete[] fBuf; + fBuf = fCurInPtr = fCurOutPtr = 0; + fMaxLen = 0; + // Clear `longStrings`. + longStrings.clear(); } + } - return *this; + return *this; } -ByteStream::ByteStream(uint32_t initSize) : - fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) +ByteStream::ByteStream(uint32_t initSize) : fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) { - if (initSize > 0) growBuf(initSize); + if (initSize > 0) + growBuf(initSize); } void ByteStream::add(const uint8_t b) { - if (fBuf == 0 || (static_cast(fCurInPtr - fBuf) == fMaxLen + ISSOverhead)) - growBuf(); + if (fBuf == 0 || (static_cast(fCurInPtr - fBuf) == fMaxLen + ISSOverhead)) + growBuf(); - *fCurInPtr++ = b; + *fCurInPtr++ = b; } void ByteStream::growBuf(uint32_t toSize) { - if (fBuf == 0) - { - if (toSize == 0) - toSize = BlockSize; - else - toSize = ((toSize + BlockSize - 1) / BlockSize) * BlockSize; - - fBuf = new uint8_t[toSize + ISSOverhead]; -#ifdef ZERO_ON_NEW - memset(fBuf, 0, (toSize + ISSOverhead)); -#endif - fMaxLen = toSize; - fCurInPtr = - fCurOutPtr = fBuf + ISSOverhead; - } + if (fBuf == 0) + { + if (toSize == 0) + toSize = BlockSize; else - { - if (toSize == 0) - toSize = fMaxLen + BlockSize; - else - toSize = ((toSize + BlockSize - 1) / BlockSize) * BlockSize; + toSize = ((toSize + BlockSize - 1) / BlockSize) * BlockSize; - if (toSize <= fMaxLen) - return; - - // Make sure we at least double the allocation - toSize = std::max(toSize, fMaxLen * 2); - - uint8_t* t = new uint8_t[toSize + ISSOverhead]; - uint32_t curOutOff = fCurOutPtr - fBuf; - uint32_t curInOff = fCurInPtr - fBuf; - memcpy(t, fBuf, fCurInPtr - fBuf); + fBuf = new uint8_t[toSize + ISSOverhead]; #ifdef ZERO_ON_NEW - memset(t + (fCurInPtr - fBuf), 0, (toSize + ISSOverhead) - (fCurInPtr - fBuf)); + memset(fBuf, 0, (toSize + ISSOverhead)); #endif - delete [] fBuf; - fBuf = t; - fMaxLen = toSize; - fCurInPtr = fBuf + curInOff; - fCurOutPtr = fBuf + curOutOff; - } + fMaxLen = toSize; + fCurInPtr = fCurOutPtr = fBuf + ISSOverhead; + } + else + { + if (toSize == 0) + toSize = fMaxLen + BlockSize; + else + toSize = ((toSize + BlockSize - 1) / BlockSize) * BlockSize; + + if (toSize <= fMaxLen) + return; + + // Make sure we at least double the allocation + toSize = std::max(toSize, fMaxLen * 2); + + uint8_t* t = new uint8_t[toSize + ISSOverhead]; + uint32_t curOutOff = fCurOutPtr - fBuf; + uint32_t curInOff = fCurInPtr - fBuf; + memcpy(t, fBuf, fCurInPtr - fBuf); +#ifdef ZERO_ON_NEW + memset(t + (fCurInPtr - fBuf), 0, (toSize + ISSOverhead) - (fCurInPtr - fBuf)); +#endif + delete[] fBuf; + fBuf = t; + fMaxLen = toSize; + fCurInPtr = fBuf + curInOff; + fCurOutPtr = fBuf + curOutOff; + } } -std::vector>& ByteStream::getLongStrings() { return longStrings; } +std::vector>& ByteStream::getLongStrings() +{ + return longStrings; +} const std::vector>& ByteStream::getLongStrings() const { - return longStrings; + return longStrings; } void ByteStream::setLongStrings(const std::vector>& other) { - longStrings = other; + longStrings = other; } ByteStream& ByteStream::operator<<(const int8_t b) { - if (fBuf == 0 || (fCurInPtr - fBuf + 1U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 1U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((int8_t*) fCurInPtr) = b; - fCurInPtr += 1; + *((int8_t*)fCurInPtr) = b; + fCurInPtr += 1; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const uint8_t b) { - add(b); + add(b); - return *this; + return *this; } ByteStream& ByteStream::operator<<(const int16_t d) { - if (fBuf == 0 || (fCurInPtr - fBuf + 2U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 2U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((int16_t*) fCurInPtr) = d; - fCurInPtr += 2; + *((int16_t*)fCurInPtr) = d; + fCurInPtr += 2; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const uint16_t d) { - if (fBuf == 0 || (fCurInPtr - fBuf + 2U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 2U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((uint16_t*) fCurInPtr) = d; - fCurInPtr += 2; + *((uint16_t*)fCurInPtr) = d; + fCurInPtr += 2; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const int32_t q) { - if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((int32_t*) fCurInPtr) = q; - fCurInPtr += 4; + *((int32_t*)fCurInPtr) = q; + fCurInPtr += 4; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const uint32_t q) { - if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((uint32_t*) fCurInPtr) = q; - fCurInPtr += 4; + *((uint32_t*)fCurInPtr) = q; + fCurInPtr += 4; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const int64_t o) { - if (fBuf == 0 || (fCurInPtr - fBuf + 8U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 8U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((int64_t*) fCurInPtr) = o; - fCurInPtr += 8; + *((int64_t*)fCurInPtr) = o; + fCurInPtr += 8; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const uint64_t o) { - if (fBuf == 0 || (fCurInPtr - fBuf + 8U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 8U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((uint64_t*) fCurInPtr) = o; - fCurInPtr += 8; + *((uint64_t*)fCurInPtr) = o; + fCurInPtr += 8; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const uint128_t& o) { - if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); - datatypes::TSInt128::storeUnaligned(fCurInPtr, o); - fCurInPtr += 16; - return *this; + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + datatypes::TSInt128::storeUnaligned(fCurInPtr, o); + fCurInPtr += 16; + return *this; } ByteStream& ByteStream::operator<<(const int128_t& o) { - if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); - datatypes::TSInt128::storeUnaligned(fCurInPtr, o); - fCurInPtr += 16; - return *this; + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + datatypes::TSInt128::storeUnaligned(fCurInPtr, o); + fCurInPtr += 16; + return *this; } ByteStream& ByteStream::operator<<(const string& s) { - int32_t len = s.size(); + int32_t len = s.size(); - *this << len; + *this << len; #if DEBUG_DUMP_STRINGS_LESS_THAN > 0 - if (len < DEBUG_DUMP_STRINGS_LESS_THAN) + if (len < DEBUG_DUMP_STRINGS_LESS_THAN) + { + cerr << "bs: appending string len " << len << ": "; + + for (size_t i = 0; i < len; i++) { - cerr << "bs: appending string len " << len << ": "; + char xxx = s.c_str()[i]; - for (size_t i = 0; i < len; i++) - { - char xxx = s.c_str()[i]; - - if (isprint(xxx)) cerr << xxx << ' '; - else cerr << "0x" << hex << ((unsigned)xxx & 0xff) << dec << ' '; - } - - cerr << endl; + if (isprint(xxx)) + cerr << xxx << ' '; + else + cerr << "0x" << hex << ((unsigned)xxx & 0xff) << dec << ' '; } -#endif - append(reinterpret_cast(s.c_str()), len); + cerr << endl; + } - return *this; +#endif + append(reinterpret_cast(s.c_str()), len); + + return *this; } ByteStream& ByteStream::operator>>(int8_t& b) { - peek(b); - fCurOutPtr++; - return *this; + peek(b); + fCurOutPtr++; + return *this; } ByteStream& ByteStream::operator>>(uint8_t& b) { - peek(b); - fCurOutPtr++; - return *this; + peek(b); + fCurOutPtr++; + return *this; } ByteStream& ByteStream::operator>>(int16_t& d) { - peek(d); - fCurOutPtr += 2; - return *this; + peek(d); + fCurOutPtr += 2; + return *this; } ByteStream& ByteStream::operator>>(uint16_t& d) { - peek(d); - fCurOutPtr += 2; - return *this; + peek(d); + fCurOutPtr += 2; + return *this; } ByteStream& ByteStream::operator>>(int32_t& q) { - peek(q); - fCurOutPtr += 4; - return *this; + peek(q); + fCurOutPtr += 4; + return *this; } ByteStream& ByteStream::operator>>(uint32_t& q) { - peek(q); - fCurOutPtr += 4; - return *this; + peek(q); + fCurOutPtr += 4; + return *this; } ByteStream& ByteStream::operator>>(int64_t& o) { - peek(o); - fCurOutPtr += 8; - return *this; + peek(o); + fCurOutPtr += 8; + return *this; } ByteStream& ByteStream::operator>>(uint64_t& o) { - peek(o); - fCurOutPtr += 8; - return *this; + peek(o); + fCurOutPtr += 8; + return *this; } ByteStream& ByteStream::operator>>(uint128_t& o) { - peek(o); - fCurOutPtr += 16; - return *this; + peek(o); + fCurOutPtr += 16; + return *this; } ByteStream& ByteStream::operator>>(int128_t& o) { - peek(o); - fCurOutPtr += 16; - return *this; + peek(o); + fCurOutPtr += 16; + return *this; } ByteStream& ByteStream::operator>>(string& s) { - peek(s); - fCurOutPtr += 4 + s.length(); - return *this; + peek(s); + fCurOutPtr += 4 + s.length(); + return *this; } ByteStream& ByteStream::operator>>(uint8_t*& bpr) { - peek(bpr); - restart(); - return *this; + peek(bpr); + restart(); + return *this; } void ByteStream::peek(int8_t& b) const { - if (length() < 1) - throw underflow_error("ByteStream::peek(int8_t): not enough data in stream to fill datatype"); + if (length() < 1) + throw underflow_error("ByteStream::peek(int8_t): not enough data in stream to fill datatype"); - b = *fCurOutPtr; + b = *fCurOutPtr; } void ByteStream::peek(uint8_t& b) const { - if (length() < 1) - throw underflow_error("ByteStream::peek(uint8_t): not enough data in stream to fill datatype"); + if (length() < 1) + throw underflow_error("ByteStream::peek(uint8_t): not enough data in stream to fill datatype"); - b = *((int8_t*)fCurOutPtr); + b = *((int8_t*)fCurOutPtr); } void ByteStream::peek(int16_t& d) const { - if (length() < 2) - throw underflow_error("ByteStream>int16_t: not enough data in stream to fill datatype"); + if (length() < 2) + throw underflow_error("ByteStream>int16_t: not enough data in stream to fill datatype"); - d = *((int16_t*) fCurOutPtr); + d = *((int16_t*)fCurOutPtr); } void ByteStream::peek(uint16_t& d) const { - if (length() < 2) - throw underflow_error("ByteStream>uint16_t: not enough data in stream to fill datatype"); + if (length() < 2) + throw underflow_error("ByteStream>uint16_t: not enough data in stream to fill datatype"); - d = *((uint16_t*) fCurOutPtr); + d = *((uint16_t*)fCurOutPtr); } void ByteStream::peek(int32_t& q) const { - if (length() < 4) - throw underflow_error("ByteStream>int32_t: not enough data in stream to fill datatype"); + if (length() < 4) + throw underflow_error("ByteStream>int32_t: not enough data in stream to fill datatype"); - q = *((int32_t*) fCurOutPtr); + q = *((int32_t*)fCurOutPtr); } void ByteStream::peek(uint32_t& q) const { - if (length() < 4) - throw underflow_error("ByteStream>uint32_t: not enough data in stream to fill datatype"); + if (length() < 4) + throw underflow_error("ByteStream>uint32_t: not enough data in stream to fill datatype"); - q = *((uint32_t*) fCurOutPtr); + q = *((uint32_t*)fCurOutPtr); } void ByteStream::peek(int64_t& o) const { + if (length() < 8) + throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); - if (length() < 8) - throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); - - o = *((int64_t*) fCurOutPtr); + o = *((int64_t*)fCurOutPtr); } void ByteStream::peek(uint64_t& o) const { + if (length() < 8) + throw underflow_error("ByteStream>uint64_t: not enough data in stream to fill datatype"); - if (length() < 8) - throw underflow_error("ByteStream>uint64_t: not enough data in stream to fill datatype"); - - o = *((uint64_t*) fCurOutPtr); + o = *((uint64_t*)fCurOutPtr); } void ByteStream::peek(uint128_t& o) const { - if (length() < 16) - throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); - datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); + if (length() < 16) + throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); + datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); } void ByteStream::peek(int128_t& o) const { - if (length() < 16) - throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); - datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); + if (length() < 16) + throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); + datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); } void ByteStream::peek(string& s) const { - int32_t len; + int32_t len; - peek(len); + peek(len); #if DEBUG_DUMP_STRINGS_LESS_THAN > 0 - if (len < DEBUG_DUMP_STRINGS_LESS_THAN) + if (len < DEBUG_DUMP_STRINGS_LESS_THAN) + { + cerr << "bs: reading string len " << len << ": "; + + for (size_t i = 0; i < len; i++) { - cerr << "bs: reading string len " << len << ": "; + char xxx = fCurOutPtr[4 + i]; - for (size_t i = 0; i < len; i++) - { - char xxx = fCurOutPtr[4 + i]; - - if (isprint(xxx)) cerr << xxx << ' '; - else cerr << "0x" << hex << ((unsigned)xxx & 0xff) << dec << ' '; - } - - cerr << endl; + if (isprint(xxx)) + cerr << xxx << ' '; + else + cerr << "0x" << hex << ((unsigned)xxx & 0xff) << dec << ' '; } + cerr << endl; + } + #endif - if (len < 0) - throw logging::ProtocolError("expected a string"); + if (len < 0) + throw logging::ProtocolError("expected a string"); - //we know len >= 0 by now... - if (length() < static_cast(len + 4)) - { + // we know len >= 0 by now... + if (length() < static_cast(len + 4)) + { #if DEBUG_DUMP_STRINGS_LESS_THAN > 0 - cerr << "bs: wanted " << len + 4 << " bytes, but there are only " << length() << " remaining" << endl; + cerr << "bs: wanted " << len + 4 << " bytes, but there are only " << length() << " remaining" << endl; #endif - // "put back" the qbyte we just read for strong exception guarantee - throw underflow_error("ByteStream>string: not enough data in stream to fill datatype"); - } + // "put back" the qbyte we just read for strong exception guarantee + throw underflow_error("ByteStream>string: not enough data in stream to fill datatype"); + } - s.assign((char*) &fCurOutPtr[4], len); + s.assign((char*)&fCurOutPtr[4], len); } void ByteStream::load(const uint8_t* bp, uint32_t len) { - // Do all the stuff that could throw an exception first - if (bp == 0 && len != 0) - throw invalid_argument("ByteStream::load: bp cannot equal 0 when len is not equal to 0"); + // Do all the stuff that could throw an exception first + if (bp == 0 && len != 0) + throw invalid_argument("ByteStream::load: bp cannot equal 0 when len is not equal to 0"); - uint32_t newMaxLen = (len + BlockSize - 1) / BlockSize * BlockSize; + uint32_t newMaxLen = (len + BlockSize - 1) / BlockSize * BlockSize; - if (len > fMaxLen) - { - delete [] fBuf; - fBuf = new uint8_t[newMaxLen + ISSOverhead]; - fMaxLen = newMaxLen; - } + if (len > fMaxLen) + { + delete[] fBuf; + fBuf = new uint8_t[newMaxLen + ISSOverhead]; + fMaxLen = newMaxLen; + } - memcpy(fBuf + ISSOverhead, bp, len); - fCurOutPtr = fBuf + ISSOverhead; - fCurInPtr = fBuf + len + ISSOverhead; + memcpy(fBuf + ISSOverhead, bp, len); + fCurOutPtr = fBuf + ISSOverhead; + fCurInPtr = fBuf + len + ISSOverhead; } void ByteStream::append(const uint8_t* bp, uint32_t len) { - if (len == 0) - return; + if (len == 0) + return; - if (bp == 0) - throw invalid_argument("ByteStream::append: bp cannot equal 0 when len is not equal to 0"); + if (bp == 0) + throw invalid_argument("ByteStream::append: bp cannot equal 0 when len is not equal to 0"); - uint32_t newSize = static_cast(fCurInPtr - fBuf + len); + uint32_t newSize = static_cast(fCurInPtr - fBuf + len); - if (fBuf == 0 || (newSize > fMaxLen)) - growBuf(newSize); + if (fBuf == 0 || (newSize > fMaxLen)) + growBuf(newSize); - memcpy(fCurInPtr, bp, len); - fCurInPtr += len; + memcpy(fCurInPtr, bp, len); + fCurInPtr += len; } void ByteStream::swap(ByteStream& rhs) { - std::swap(fBuf, rhs.fBuf); - std::swap(fCurInPtr, rhs.fCurInPtr); - std::swap(fCurOutPtr, rhs.fCurOutPtr); - std::swap(fMaxLen, rhs.fMaxLen); - std::swap(longStrings, rhs.longStrings); + std::swap(fBuf, rhs.fBuf); + std::swap(fCurInPtr, rhs.fCurInPtr); + std::swap(fCurOutPtr, rhs.fCurOutPtr); + std::swap(fMaxLen, rhs.fMaxLen); + std::swap(longStrings, rhs.longStrings); } ifstream& operator>>(ifstream& ifs, ByteStream& bs) { - int ifs_len; - ifs.seekg(0, ios::end); - ifs_len = ifs.tellg(); - ifs.seekg(0, ios::beg); - boost::scoped_array buf(new char[ifs_len]); - ifs.read(buf.get(), ifs_len); - bs.append(reinterpret_cast(buf.get()), ifs_len); - return ifs; + int ifs_len; + ifs.seekg(0, ios::end); + ifs_len = ifs.tellg(); + ifs.seekg(0, ios::beg); + boost::scoped_array buf(new char[ifs_len]); + ifs.read(buf.get(), ifs_len); + bs.append(reinterpret_cast(buf.get()), ifs_len); + return ifs; } bool ByteStream::operator==(const ByteStream& b) const { - if (b.length() != length()) - return false; + if (b.length() != length()) + return false; - if (memcmp(fCurOutPtr, b.fCurOutPtr, length()) != 0) + if (memcmp(fCurOutPtr, b.fCurOutPtr, length()) != 0) + return false; + + // Check the `longString` sizes. + if (longStrings.size() != b.longStrings.size()) + return false; + + // For each `longString`. + for (uint32_t i = 0, e = b.longStrings.size(); i < e; ++i) + { + const auto* leftMemChunk = reinterpret_cast(longStrings[i].get()); + const auto* rightMemChunk = reinterpret_cast(b.longStrings[i].get()); + if (leftMemChunk == nullptr || rightMemChunk == nullptr) return false; - // Check the `longString` sizes. - if (longStrings.size() != b.longStrings.size()) - return false; + const uint32_t leftSize = leftMemChunk->currentSize; + const uint32_t rightSize = rightMemChunk->currentSize; + if (leftSize != rightSize) + return false; - // For each `longString`. - for (uint32_t i = 0, e = b.longStrings.size(); i < e; ++i) - { - const auto* leftMemChunk = reinterpret_cast(longStrings[i].get()); - const auto* rightMemChunk = reinterpret_cast(b.longStrings[i].get()); - if (leftMemChunk == nullptr || rightMemChunk == nullptr) - return false; + if (memcmp(leftMemChunk->data, rightMemChunk->data, leftSize) != 0) + return false; + } - const uint32_t leftSize = leftMemChunk->currentSize; - const uint32_t rightSize = rightMemChunk->currentSize; - if (leftSize != rightSize) - return false; - - if (memcmp(leftMemChunk->data, rightMemChunk->data, leftSize) != 0) - return false; - } - - return true; + return true; } bool ByteStream::operator!=(const ByteStream& b) const { - return !(*this == b); + return !(*this == b); } /* Serializeable interface */ void ByteStream::serialize(ByteStream& bs) const { - bs << length(); - bs.append(buf(), length()); + bs << length(); + bs.append(buf(), length()); } void ByteStream::deserialize(ByteStream& bs) { - uint32_t len; + uint32_t len; - restart(); - bs >> len; - load(bs.buf(), len); - bs.advance(len); + restart(); + bs >> len; + load(bs.buf(), len); + bs.advance(len); } void ByteStream::needAtLeast(size_t amount) { - size_t currentSpace; + size_t currentSpace; - currentSpace = fMaxLen - (fCurInPtr - (fBuf + ISSOverhead)); + currentSpace = fMaxLen - (fCurInPtr - (fBuf + ISSOverhead)); - if (currentSpace < amount) - growBuf(fMaxLen + amount); + if (currentSpace < amount) + growBuf(fMaxLen + amount); } #ifdef _MSC_VER #if BOOST_VERSION < 104500 ByteStream& ByteStream::operator<<(const uint32_t ui) { - if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - uint32_t q = ui; - *((uint32_t*) fCurInPtr) = q; - fCurInPtr += 4; + uint32_t q = ui; + *((uint32_t*)fCurInPtr) = q; + fCurInPtr += 4; - return *this; + return *this; } ByteStream& ByteStream::operator>>(uint32_t& ui) { - uint32_t q; - peek(q); - fCurOutPtr += 4; - ui = q; - return *this; + uint32_t q; + peek(q); + fCurOutPtr += 4; + ui = q; + return *this; } #endif #endif ByteStream& ByteStream::operator<<(const ByteStream& bs) { - uint32_t len = bs.length(); + uint32_t len = bs.length(); - *this << len; + *this << len; - append(bs.buf(), len); + append(bs.buf(), len); - return *this; + return *this; } ByteStream& ByteStream::operator>>(ByteStream& bs) { - peek(bs); - fCurOutPtr += 4 + bs.length(); - return *this; + peek(bs); + fCurOutPtr += 4 + bs.length(); + return *this; } void ByteStream::peek(ByteStream& bs) const { - uint32_t len; + uint32_t len; - peek(len); + peek(len); - if (length() < len) - throw underflow_error("ByteStream>ByteStream: not enough data in stream to fill datatype"); + if (length() < len) + throw underflow_error("ByteStream>ByteStream: not enough data in stream to fill datatype"); - bs.load(&fCurOutPtr[4], len); + bs.load(&fCurOutPtr[4], len); } ByteStream& ByteStream::operator<<(const uuid& u) { - append(reinterpret_cast(&u.data[0]), uuids::uuid::static_size()); - return *this; + append(reinterpret_cast(&u.data[0]), uuids::uuid::static_size()); + return *this; } ByteStream& ByteStream::operator>>(uuid& u) { - peek(u); - fCurOutPtr += uuids::uuid::static_size(); - return *this; + peek(u); + fCurOutPtr += uuids::uuid::static_size(); + return *this; } void ByteStream::peek(uuid& u) const { - if (length() < uuids::uuid::static_size()) - throw underflow_error("ByteStream>uuid: not enough data in stream to fill datatype"); + if (length() < uuids::uuid::static_size()) + throw underflow_error("ByteStream>uuid: not enough data in stream to fill datatype"); - memcpy(&u.data[0], fCurOutPtr, uuids::uuid::static_size()); + memcpy(&u.data[0], fCurOutPtr, uuids::uuid::static_size()); } ByteStream& ByteStream::operator<<(const float f) { - int sz = sizeof(float); + int sz = sizeof(float); - if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((float*) fCurInPtr) = f; - fCurInPtr += sz; + *((float*)fCurInPtr) = f; + fCurInPtr += sz; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const double d) { - int sz = sizeof(double); + int sz = sizeof(double); - if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((double*) fCurInPtr) = d; - fCurInPtr += sz; + *((double*)fCurInPtr) = d; + fCurInPtr += sz; - return *this; + return *this; } ByteStream& ByteStream::operator<<(const long double d) { - int sz = sizeof(long double); + int sz = sizeof(long double); - if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); + if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); - *((long double*) fCurInPtr) = d; - fCurInPtr += sz; + *((long double*)fCurInPtr) = d; + fCurInPtr += sz; - return *this; + return *this; } ByteStream& ByteStream::operator>>(float& f) { - peek(f); - fCurOutPtr += sizeof(float); - return *this; + peek(f); + fCurOutPtr += sizeof(float); + return *this; } ByteStream& ByteStream::operator>>(double& d) { - peek(d); - fCurOutPtr += sizeof(double); - return *this; + peek(d); + fCurOutPtr += sizeof(double); + return *this; } ByteStream& ByteStream::operator>>(long double& d) { - peek(d); - fCurOutPtr += sizeof(long double); - return *this; + peek(d); + fCurOutPtr += sizeof(long double); + return *this; } void ByteStream::peek(float& f) const { - if (length() < sizeof(float)) - throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); + if (length() < sizeof(float)) + throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); - f = *((float*) fCurOutPtr); + f = *((float*)fCurOutPtr); } void ByteStream::peek(double& d) const { - if (length() < sizeof(double)) - throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); + if (length() < sizeof(double)) + throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); - d = *((double*) fCurOutPtr); + d = *((double*)fCurOutPtr); } void ByteStream::peek(long double& d) const { - if (length() < sizeof(long double)) - throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); + if (length() < sizeof(long double)) + throw underflow_error("ByteStream>int64_t: not enough data in stream to fill datatype"); - d = *((long double*) fCurOutPtr); + d = *((long double*)fCurOutPtr); } - -}//namespace messageqcpp - +} // namespace messageqcpp diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 153d22d3b..eff0cc82c 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -47,10 +47,8 @@ class ByteStreamTestSuite; #define EXPORT #endif - namespace messageqcpp { - typedef boost::shared_ptr SBS; /** @@ -70,442 +68,456 @@ typedef boost::shared_ptr SBS; */ class ByteStream : public Serializeable { -public: - // We now use the standard Linux types of uint8_t, uint16_t, etc. - // These are kept around for backward compatibility - typedef uint8_t byte; - typedef uint16_t doublebyte; - typedef uint32_t quadbyte; - typedef uint64_t octbyte; - typedef int128_t hexbyte; - typedef boost::uuids::uuid uuid; + public: + // We now use the standard Linux types of uint8_t, uint16_t, etc. + // These are kept around for backward compatibility + typedef uint8_t byte; + typedef uint16_t doublebyte; + typedef uint32_t quadbyte; + typedef uint64_t octbyte; + typedef int128_t hexbyte; + typedef boost::uuids::uuid uuid; - /** - * default ctor - */ - EXPORT explicit ByteStream(uint32_t initSize = 8192); // multiples of pagesize are best - /** - * ctor with a uint8_t array and len initializer - */ - inline ByteStream(const uint8_t* bp, const uint32_t len); - /** - * copy ctor - */ - EXPORT ByteStream(const ByteStream& rhs); - EXPORT ByteStream(const SBS& rhs); - /** - * assign op - */ - EXPORT ByteStream& operator=(const ByteStream& rhs); - /** - * assign op - */ - inline ByteStream& operator=(const SBS& rhs); - /** - * dtor - */ - inline virtual ~ByteStream(); + /** + * default ctor + */ + EXPORT explicit ByteStream(uint32_t initSize = 8192); // multiples of pagesize are best + /** + * ctor with a uint8_t array and len initializer + */ + inline ByteStream(const uint8_t* bp, const uint32_t len); + /** + * copy ctor + */ + EXPORT ByteStream(const ByteStream& rhs); + EXPORT ByteStream(const SBS& rhs); + /** + * assign op + */ + EXPORT ByteStream& operator=(const ByteStream& rhs); + /** + * assign op + */ + inline ByteStream& operator=(const SBS& rhs); + /** + * dtor + */ + inline virtual ~ByteStream(); - /** - * push a int8_t onto the end of the stream - */ - EXPORT ByteStream& operator<<(const int8_t b); - /** - * push a uint8_t onto the end of the stream - */ - EXPORT ByteStream& operator<<(const uint8_t b); - /** - * push a int16_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const int16_t d); - /** - * push a uint16_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const uint16_t d); - /** - * push a int32_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const int32_t q); - /** - * push a uint32_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const uint32_t q); + /** + * push a int8_t onto the end of the stream + */ + EXPORT ByteStream& operator<<(const int8_t b); + /** + * push a uint8_t onto the end of the stream + */ + EXPORT ByteStream& operator<<(const uint8_t b); + /** + * push a int16_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int16_t d); + /** + * push a uint16_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const uint16_t d); + /** + * push a int32_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int32_t q); + /** + * push a uint32_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const uint32_t q); #ifdef _MSC_VER #if BOOST_VERSION < 104500 - //These 2 are to make MS VC++ w/ boost < 1.45 happy - // TODO: Do we still need these? - EXPORT ByteStream& operator<<(const uint32_t ui); + // These 2 are to make MS VC++ w/ boost < 1.45 happy + // TODO: Do we still need these? + EXPORT ByteStream& operator<<(const uint32_t ui); - EXPORT ByteStream& operator>>(uint32_t& ui); + EXPORT ByteStream& operator>>(uint32_t& ui); #endif #endif - /** - * push an int64_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const int64_t o); - /** - * push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const uint64_t o); - /** - * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const int128_t& o); + /** + * push an int64_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int64_t o); + /** + * push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const uint64_t o); + /** + * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int128_t& o); - /** - * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const uint128_t& o); - /** - * push a float onto the end of the stream. The byte order is - * whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const float f); - /** - * push a double onto the end of the stream. The byte order is - * whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const double d); - /** - * push a long double onto the end of the stream. The byte - * order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const long double d); - /** - * push a std::string onto the end of the stream. - */ - EXPORT ByteStream& operator<<(const std::string& s); - /** - * push an arbitrary class onto the end of the stream. - */ - inline ByteStream& operator<<(const Serializeable& s); - /** - * push a ByteStream onto the end of the stream. - */ - EXPORT ByteStream& operator<<(const ByteStream& bs); - /** - * push a UUID onto the end of the stream. - */ - EXPORT ByteStream& operator<<(const uuid& u); + /** + * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const uint128_t& o); + /** + * push a float onto the end of the stream. The byte order is + * whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const float f); + /** + * push a double onto the end of the stream. The byte order is + * whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const double d); + /** + * push a long double onto the end of the stream. The byte + * order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const long double d); + /** + * push a std::string onto the end of the stream. + */ + EXPORT ByteStream& operator<<(const std::string& s); + /** + * push an arbitrary class onto the end of the stream. + */ + inline ByteStream& operator<<(const Serializeable& s); + /** + * push a ByteStream onto the end of the stream. + */ + EXPORT ByteStream& operator<<(const ByteStream& bs); + /** + * push a UUID onto the end of the stream. + */ + EXPORT ByteStream& operator<<(const uuid& u); - /** - * extract a int8_t from the front of the stream. - */ - EXPORT ByteStream& operator>>(int8_t& b); - /** - * extract a uint8_t from the front of the stream. - */ - EXPORT ByteStream& operator>>(uint8_t& b); - /** - * extract a int16_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(int16_t& d); - /** - * extract a uint16_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(uint16_t& d); - /** - * extract a int32_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(int32_t& q); - /** - * extract a uint32_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(uint32_t& q); - /** - * extract an int64_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(int64_t& o); - /** - * extract an uint64_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(uint64_t& o); - /** - * extract an int128_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(int128_t& o); - /** - * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(uint128_t& o); - /** - * extract a float from the front of the stream. The byte - * order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(float& f); - /** - * extract a double from the front of the stream. The byte - * order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(double& d); - /** - * extract a long double from the front of the stream. The byte - * order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(long double& d); - /** - * extract a std::string from the front of the stream. - */ - EXPORT ByteStream& operator>>(std::string& s); - /** - * write the current stream into b. The ByteStream will be empty after this operation. - * @warning the caller is responsible for making sure b is big enough to hold all the data (perhaps by - * calling length()). - */ - EXPORT ByteStream& operator>>(uint8_t*& b); - /** - * extract an arbitrary object from the front of the stream. - */ - inline ByteStream& operator>>(Serializeable& s); - /** - * extract a ByteStream from the front of the stream. - */ - EXPORT ByteStream& operator>>(ByteStream& bs); - /** - * extract a UUID from the front of the stream. - */ - EXPORT ByteStream& operator>>(uuid& u); + /** + * extract a int8_t from the front of the stream. + */ + EXPORT ByteStream& operator>>(int8_t& b); + /** + * extract a uint8_t from the front of the stream. + */ + EXPORT ByteStream& operator>>(uint8_t& b); + /** + * extract a int16_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int16_t& d); + /** + * extract a uint16_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(uint16_t& d); + /** + * extract a int32_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int32_t& q); + /** + * extract a uint32_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(uint32_t& q); + /** + * extract an int64_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int64_t& o); + /** + * extract an uint64_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(uint64_t& o); + /** + * extract an int128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int128_t& o); + /** + * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order + *is. + */ + EXPORT ByteStream& operator>>(uint128_t& o); + /** + * extract a float from the front of the stream. The byte + * order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(float& f); + /** + * extract a double from the front of the stream. The byte + * order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(double& d); + /** + * extract a long double from the front of the stream. The byte + * order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(long double& d); + /** + * extract a std::string from the front of the stream. + */ + EXPORT ByteStream& operator>>(std::string& s); + /** + * write the current stream into b. The ByteStream will be empty after this operation. + * @warning the caller is responsible for making sure b is big enough to hold all the data (perhaps by + * calling length()). + */ + EXPORT ByteStream& operator>>(uint8_t*& b); + /** + * extract an arbitrary object from the front of the stream. + */ + inline ByteStream& operator>>(Serializeable& s); + /** + * extract a ByteStream from the front of the stream. + */ + EXPORT ByteStream& operator>>(ByteStream& bs); + /** + * extract a UUID from the front of the stream. + */ + EXPORT ByteStream& operator>>(uuid& u); - /** - * Peek at a int8_t from the front of the stream. - */ - EXPORT void peek(int8_t& b) const; - /** - * Peek at a uint8_t from the front of the stream. - */ - EXPORT void peek(uint8_t& b) const; - /** - * Peek at a int16_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(int16_t& d) const; - /** - * Peek at a uint16_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(uint16_t& d) const; - /** - * Peek at a int32_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(int32_t& q) const; - /** - * Peek at a uint32_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(uint32_t& q) const; - /** - * Peek at an int64_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(int64_t& o) const; - /** - * Peek at an uint64_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(uint64_t& o) const; - /** - * Peek at an int128_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(int128_t& o) const; - /** - * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(uint128_t& o) const; - /** - * Peek at a float from the front of the stream. The byte order - * is whatever the native byte order is. - */ - EXPORT void peek(float& f) const; - /** - * Peek at a double from the front of the stream. The byte - * order is whatever the native byte order is. - */ - EXPORT void peek(double& f) const; - /** - * Peek at a long double from the front of the stream. The byte - * order is whatever the native byte order is. - */ - EXPORT void peek(long double& f) const; - /** - * Peek at a std::string from the front of the stream. - */ - EXPORT void peek(std::string& s) const; - /** - * Peek at the whole ByteStream buffer. - * @warning the caller is responsible for making sure b is big enough to hold all the data (perhaps by - * calling length()). - */ - inline void peek(uint8_t*& b) const; - /** - * Peek at a ByteStream from the front of the stream. - */ - EXPORT void peek(ByteStream& bs) const; - /** - * Peek at a UUID from the front of the stream. - */ - EXPORT void peek(uuid& u) const; + /** + * Peek at a int8_t from the front of the stream. + */ + EXPORT void peek(int8_t& b) const; + /** + * Peek at a uint8_t from the front of the stream. + */ + EXPORT void peek(uint8_t& b) const; + /** + * Peek at a int16_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int16_t& d) const; + /** + * Peek at a uint16_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(uint16_t& d) const; + /** + * Peek at a int32_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int32_t& q) const; + /** + * Peek at a uint32_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(uint32_t& q) const; + /** + * Peek at an int64_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int64_t& o) const; + /** + * Peek at an uint64_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(uint64_t& o) const; + /** + * Peek at an int128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int128_t& o) const; + /** + * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order + *is. + */ + EXPORT void peek(uint128_t& o) const; + /** + * Peek at a float from the front of the stream. The byte order + * is whatever the native byte order is. + */ + EXPORT void peek(float& f) const; + /** + * Peek at a double from the front of the stream. The byte + * order is whatever the native byte order is. + */ + EXPORT void peek(double& f) const; + /** + * Peek at a long double from the front of the stream. The byte + * order is whatever the native byte order is. + */ + EXPORT void peek(long double& f) const; + /** + * Peek at a std::string from the front of the stream. + */ + EXPORT void peek(std::string& s) const; + /** + * Peek at the whole ByteStream buffer. + * @warning the caller is responsible for making sure b is big enough to hold all the data (perhaps by + * calling length()). + */ + inline void peek(uint8_t*& b) const; + /** + * Peek at a ByteStream from the front of the stream. + */ + EXPORT void peek(ByteStream& bs) const; + /** + * Peek at a UUID from the front of the stream. + */ + EXPORT void peek(uuid& u) const; - /** - * load the stream from an array. Clears out any previous data. - */ - EXPORT void load(const uint8_t* bp, uint32_t len); + /** + * load the stream from an array. Clears out any previous data. + */ + EXPORT void load(const uint8_t* bp, uint32_t len); - /** - * append bytes to the end of the stream. - */ - EXPORT void append(const uint8_t* bp, uint32_t len); + /** + * append bytes to the end of the stream. + */ + EXPORT void append(const uint8_t* bp, uint32_t len); - /** - * equality check on buffer contents. - */ - EXPORT bool operator==(const ByteStream& b) const; + /** + * equality check on buffer contents. + */ + EXPORT bool operator==(const ByteStream& b) const; - /** - * inequality check on buffer contents. - */ - EXPORT bool operator!=(const ByteStream& b) const; + /** + * inequality check on buffer contents. + */ + EXPORT bool operator!=(const ByteStream& b) const; - /** - * these fcns are an alternative to code like {build a msg, bs.load or bs.append}. - * This will let us build the msg directly in the BS buffer. - */ - EXPORT void needAtLeast(size_t amount); - inline uint8_t* getInputPtr(); - inline void advanceInputPtr(size_t amount); + /** + * these fcns are an alternative to code like {build a msg, bs.load or bs.append}. + * This will let us build the msg directly in the BS buffer. + */ + EXPORT void needAtLeast(size_t amount); + inline uint8_t* getInputPtr(); + inline void advanceInputPtr(size_t amount); - /** - * returns a const pointer to the current head of the queue. If you use it for - * raw access, you might want to advance the current head. - */ - inline const uint8_t* buf() const; + /** + * returns a const pointer to the current head of the queue. If you use it for + * raw access, you might want to advance the current head. + */ + inline const uint8_t* buf() const; - /** - * returns a pointer to the current head of the queue. If you use it for - * raw access, you might want to advance the current head. - */ - inline uint8_t* buf(); + /** + * returns a pointer to the current head of the queue. If you use it for + * raw access, you might want to advance the current head. + */ + inline uint8_t* buf(); - /** - * advance the output ptr without having to extract bytes - * @warning be careful advancing near 4GB! - */ - inline void advance(uint32_t amt); + /** + * advance the output ptr without having to extract bytes + * @warning be careful advancing near 4GB! + */ + inline void advance(uint32_t amt); - /** - * returns the length of the queue (in bytes) - * @warning do not attempt to make a ByteStream bigger than 4GB! - */ - inline uint32_t length() const; - inline bool empty() const; + /** + * returns the length of the queue (in bytes) + * @warning do not attempt to make a ByteStream bigger than 4GB! + */ + inline uint32_t length() const; + inline bool empty() const; - /** - * returns the length of the queue, including header overhead (in bytes) - */ - inline uint32_t lengthWithHdrOverhead() const; + /** + * returns the length of the queue, including header overhead (in bytes) + */ + inline uint32_t lengthWithHdrOverhead() const; - /** - * clears the stream. Releases any current stream and sets all pointers to 0. The state of the object - * is identical to its state immediately after default construction. - */ - inline void reset(); + /** + * clears the stream. Releases any current stream and sets all pointers to 0. The state of the object + * is identical to its state immediately after default construction. + */ + inline void reset(); - /** - * operator+= - */ - inline ByteStream& operator+=(const ByteStream& rhs); + /** + * operator+= + */ + inline ByteStream& operator+=(const ByteStream& rhs); - /** - * swap this ByteStream with another ByteStream - */ - EXPORT void swap(ByteStream& rhs); + /** + * swap this ByteStream with another ByteStream + */ + EXPORT void swap(ByteStream& rhs); - /** - * reset the input & output pointers to the beginning - */ - inline void restart(); + /** + * reset the input & output pointers to the beginning + */ + inline void restart(); - /** - * Move the input pointer back to the beginning so the contents - * can be read again. - */ - inline void rewind(); + /** + * Move the input pointer back to the beginning so the contents + * can be read again. + */ + inline void rewind(); - /** - * Get the allocated size of the buffer. - */ - inline uint32_t getBufferSize() const; - - /** - * Serializeable interface - */ - EXPORT void serialize(ByteStream& bs) const; + /** + * Get the allocated size of the buffer. + */ + inline uint32_t getBufferSize() const; - /** - * Serializeable interface - */ - EXPORT void deserialize(ByteStream& bs); - - /** - * memory allocation chunk size - */ - EXPORT static const uint32_t BlockSize = 4096; + /** + * Serializeable interface + */ + EXPORT void serialize(ByteStream& bs) const; - /** size of the space we want in front of the data */ - EXPORT static const uint32_t ISSOverhead = - 3 * sizeof(uint32_t); // space for the BS magic & length & number of long strings. + /** + * Serializeable interface + */ + EXPORT void deserialize(ByteStream& bs); - // Methods to get and set `long strings`. - EXPORT std::vector>& getLongStrings(); - EXPORT const std::vector>& getLongStrings() const; - EXPORT void setLongStrings(const std::vector>& other); + /** + * memory allocation chunk size + */ + EXPORT static const uint32_t BlockSize = 4096; - friend class ::ByteStreamTestSuite; + /** size of the space we want in front of the data */ + EXPORT static const uint32_t ISSOverhead = + 3 * sizeof(uint32_t); // space for the BS magic & length & number of long strings. -protected: - /** - * pushes one uint8_t onto the end of the stream - */ - void add(const uint8_t b); - /** - * adds another BlockSize bytes to the internal buffer - */ - void growBuf(uint32_t toSize = 0); - /** - * handles member copying from one ByteStream to another - */ - void doCopy(const ByteStream& rhs); + // Methods to get and set `long strings`. + EXPORT std::vector>& getLongStrings(); + EXPORT const std::vector>& getLongStrings() const; + EXPORT void setLongStrings(const std::vector>& other); -private: - // Put struct `MemChunk` declaration here, to avoid circular dependency. - struct MemChunk - { - uint32_t currentSize; - uint32_t capacity; - uint8_t data[]; - }; + friend class ::ByteStreamTestSuite; - uint8_t* fBuf; ///the start of the allocated buffer - uint8_t* fCurInPtr; //the point in fBuf where data is inserted next - uint8_t* fCurOutPtr; //the point in fBuf where data is extracted from next - uint32_t fMaxLen; //how big fBuf is currently - // Stores `long strings`. - std::vector> longStrings; + protected: + /** + * pushes one uint8_t onto the end of the stream + */ + void add(const uint8_t b); + /** + * adds another BlockSize bytes to the internal buffer + */ + void growBuf(uint32_t toSize = 0); + /** + * handles member copying from one ByteStream to another + */ + void doCopy(const ByteStream& rhs); + + private: + // Put struct `MemChunk` declaration here, to avoid circular dependency. + struct MemChunk + { + uint32_t currentSize; + uint32_t capacity; + uint8_t data[]; + }; + + uint8_t* fBuf; /// the start of the allocated buffer + uint8_t* fCurInPtr; // the point in fBuf where data is inserted next + uint8_t* fCurOutPtr; // the point in fBuf where data is extracted from next + uint32_t fMaxLen; // how big fBuf is currently + // Stores `long strings`. + std::vector> longStrings; }; -template +template struct _ByteStreamType { - typedef T type; + typedef T type; }; template -struct ByteStreamType: _ByteStreamType { }; +struct ByteStreamType : _ByteStreamType +{ +}; template <> -struct ByteStreamType<1>: _ByteStreamType<1, ByteStream::byte> { }; +struct ByteStreamType<1> : _ByteStreamType<1, ByteStream::byte> +{ +}; template <> -struct ByteStreamType<2>: _ByteStreamType<2, ByteStream::doublebyte> { }; +struct ByteStreamType<2> : _ByteStreamType<2, ByteStream::doublebyte> +{ +}; template <> -struct ByteStreamType<4>: _ByteStreamType<4, ByteStream::quadbyte> { }; +struct ByteStreamType<4> : _ByteStreamType<4, ByteStream::quadbyte> +{ +}; template <> -struct ByteStreamType<8>: _ByteStreamType<8, ByteStream::octbyte> { }; +struct ByteStreamType<8> : _ByteStreamType<8, ByteStream::octbyte> +{ +}; template <> -struct ByteStreamType<16>: _ByteStreamType<16, ByteStream::hexbyte> { }; +struct ByteStreamType<16> : _ByteStreamType<16, ByteStream::hexbyte> +{ +}; // type descriptors to let ByteStream point out protocol errors, WIP static const uint8_t BS_UINT8 = 0; @@ -523,99 +535,99 @@ static const uint8_t BS_UUID = 11; inline ByteStream::ByteStream(const uint8_t* bp, const uint32_t len) : fBuf(0), fMaxLen(0) { - load(bp, len); + load(bp, len); } inline ByteStream::~ByteStream() { - delete [] fBuf; + delete[] fBuf; } inline const uint8_t* ByteStream::buf() const { - return fCurOutPtr; + return fCurOutPtr; } inline uint8_t* ByteStream::buf() { - return fCurOutPtr; + return fCurOutPtr; } inline uint32_t ByteStream::length() const { - return (uint32_t)(fCurInPtr - fCurOutPtr); + return (uint32_t)(fCurInPtr - fCurOutPtr); } inline bool ByteStream::empty() const { - return (length() == 0); + return (length() == 0); } inline uint32_t ByteStream::lengthWithHdrOverhead() const { - return (length() + ISSOverhead); + return (length() + ISSOverhead); } inline void ByteStream::reset() { - delete [] fBuf; - fMaxLen = 0; - fCurInPtr = fCurOutPtr = fBuf = 0; + delete[] fBuf; + fMaxLen = 0; + fCurInPtr = fCurOutPtr = fBuf = 0; } inline void ByteStream::restart() { - fCurInPtr = fCurOutPtr = fBuf + ISSOverhead; + fCurInPtr = fCurOutPtr = fBuf + ISSOverhead; } inline void ByteStream::rewind() { - fCurOutPtr = fBuf + ISSOverhead; + fCurOutPtr = fBuf + ISSOverhead; } inline void ByteStream::advance(uint32_t adv) { - //fCurOutPtr is always >= fBuf, so fCurOutPtr - fBuf is >= 0, and this difference is always <= 32 bits - //there is an edge condition not detected here: if fCurOutPtr - fBuf is nearly 4GB and you try to - //advance by a lot, you will wrap over, so be warned! - if (adv > length()) - throw std::length_error("ByteStream: advanced beyond the end of the buffer"); + // fCurOutPtr is always >= fBuf, so fCurOutPtr - fBuf is >= 0, and this difference is always <= 32 bits + // there is an edge condition not detected here: if fCurOutPtr - fBuf is nearly 4GB and you try to + // advance by a lot, you will wrap over, so be warned! + if (adv > length()) + throw std::length_error("ByteStream: advanced beyond the end of the buffer"); - fCurOutPtr += adv; + fCurOutPtr += adv; } inline uint8_t* ByteStream::getInputPtr() { - return fCurInPtr; + return fCurInPtr; } inline void ByteStream::advanceInputPtr(size_t amount) { - fCurInPtr += amount; + fCurInPtr += amount; } inline void ByteStream::peek(uint8_t*& bpr) const { - memcpy(bpr, fCurOutPtr, length()); + memcpy(bpr, fCurOutPtr, length()); } inline ByteStream& ByteStream::operator+=(const ByteStream& rhs) { - append(rhs.buf(), rhs.length()); - return *this; + append(rhs.buf(), rhs.length()); + return *this; } inline ByteStream operator+(const ByteStream& lhs, const ByteStream& rhs) { - ByteStream temp(lhs); - return temp += rhs; + ByteStream temp(lhs); + return temp += rhs; } inline ByteStream& ByteStream::operator>>(Serializeable& s) { - s.deserialize(*this); - return *this; + s.deserialize(*this); + return *this; } inline ByteStream& ByteStream::operator<<(const Serializeable& s) { - s.serialize(*this); - return *this; + s.serialize(*this); + return *this; } inline ByteStream& ByteStream::operator=(const SBS& rhs) { - *this = *rhs; - return *this; + *this = *rhs; + return *this; } inline uint32_t ByteStream::getBufferSize() const { - return fMaxLen; + return fMaxLen; } /** @@ -623,7 +635,7 @@ inline uint32_t ByteStream::getBufferSize() const */ inline std::ostream& operator<<(std::ostream& os, const ByteStream& bs) { - return os.write(reinterpret_cast(bs.buf()), bs.length()); + return os.write(reinterpret_cast(bs.buf()), bs.length()); } /** @@ -632,113 +644,113 @@ inline std::ostream& operator<<(std::ostream& os, const ByteStream& bs) EXPORT std::ifstream& operator>>(std::ifstream& os, ByteStream& bs); /// Generic method to export a vector of T's that implement Serializeable -template +template void serializeVector(ByteStream& bs, const std::vector& v) { - typename std::vector::const_iterator it; - uint64_t size; + typename std::vector::const_iterator it; + uint64_t size; - size = v.size(); - bs << size; + size = v.size(); + bs << size; - for (it = v.begin(); it != v.end(); it++) - bs << *it; + for (it = v.begin(); it != v.end(); it++) + bs << *it; } /// Generic method to deserialize a vector of T's that implement Serializeable -template +template void deserializeVector(ByteStream& bs, std::vector& v) { - uint32_t i; - T tmp; - uint64_t size; + uint32_t i; + T tmp; + uint64_t size; - v.clear(); - bs >> size; + v.clear(); + bs >> size; - for (i = 0; i < size; i++) - { - bs >> tmp; - v.push_back(tmp); - } + for (i = 0; i < size; i++) + { + bs >> tmp; + v.push_back(tmp); + } } #ifdef _MSC_VER -//Until the API is fixed to be 64-bit clean... -#pragma warning (push) -#pragma warning (disable : 4267) +// Until the API is fixed to be 64-bit clean... +#pragma warning(push) +#pragma warning(disable : 4267) #endif -template +template void serializeInlineVector(ByteStream& bs, const std::vector& v) { - uint64_t size = v.size(); - bs << size; + uint64_t size = v.size(); + bs << size; - if (size > 0) - bs.append((const uint8_t*) & (v[0]), sizeof(T) * size); + if (size > 0) + bs.append((const uint8_t*)&(v[0]), sizeof(T) * size); } inline void serializeVector(ByteStream& bs, const std::vector& v) { - serializeInlineVector(bs, v); + serializeInlineVector(bs, v); } -template +template void deserializeInlineVector(ByteStream& bs, std::vector& v) { - uint64_t size; - const uint8_t* buf; + uint64_t size; + const uint8_t* buf; - v.clear(); - bs >> size; + v.clear(); + bs >> size; - if (size > 0) - { - v.resize(size); - buf = bs.buf(); - memcpy(&(v[0]), buf, sizeof(T) * size); - bs.advance(sizeof(T) * size); - } + if (size > 0) + { + v.resize(size); + buf = bs.buf(); + memcpy(&(v[0]), buf, sizeof(T) * size); + bs.advance(sizeof(T) * size); + } } #ifdef _MSC_VER -#pragma warning (pop) +#pragma warning(pop) #endif inline void deserializeVector(ByteStream& bs, std::vector& v) { - deserializeInlineVector(bs, v); + deserializeInlineVector(bs, v); } /// Generic method to serialize a set of T's that implement Serializeable -template +template void serializeSet(ByteStream& bs, const std::set& s) { - uint64_t size = s.size(); - bs << size; - typename std::set::const_iterator it; + uint64_t size = s.size(); + bs << size; + typename std::set::const_iterator it; - for (it = s.begin(); it != s.end(); ++it) - bs << *it; + for (it = s.begin(); it != s.end(); ++it) + bs << *it; } /// Generic method to deserialize a set of T's that implement Serializeable -template +template void deserializeSet(ByteStream& bs, std::set& s) { - uint32_t i; - T tmp; - uint64_t size; + uint32_t i; + T tmp; + uint64_t size; - s.clear(); - bs >> size; + s.clear(); + bs >> size; - for (i = 0; i < size; i++) - { - bs >> tmp; - s.insert(tmp); - } + for (i = 0; i < size; i++) + { + bs >> tmp; + s.insert(tmp); + } } /* template<> @@ -747,19 +759,20 @@ struct ByteStream::_ByteStreamType<1, ByteStream::byte>> typedef ByteStream::byte type; }*/ -}//namespace messageqcpp +} // namespace messageqcpp namespace std { /** total specialization of std::swap * */ -template<> inline void swap(messageqcpp::ByteStream& lhs, messageqcpp::ByteStream& rhs) +template <> +inline void swap(messageqcpp::ByteStream& lhs, messageqcpp::ByteStream& rhs) { - lhs.swap(rhs); + lhs.swap(rhs); } -}//namespace std +} // namespace std #undef EXPORT -#endif //MESSAGEQCPP_BYTESTREAM_H +#endif // MESSAGEQCPP_BYTESTREAM_H diff --git a/utils/messageqcpp/bytestreampool.cpp b/utils/messageqcpp/bytestreampool.cpp index 3039ea7b0..81946492e 100644 --- a/utils/messageqcpp/bytestreampool.cpp +++ b/utils/messageqcpp/bytestreampool.cpp @@ -14,70 +14,70 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - + #include "bytestreampool.h" namespace messageqcpp { - ByteStreamPool::ByteStreamPool() { - maxBufferSize = 1 << 20; // 1MB - maxFreeBuffers = 10; + maxBufferSize = 1 << 20; // 1MB + maxFreeBuffers = 10; } ByteStreamPool::ByteStreamPool(uint largeBufferSize) { - maxBufferSize = largeBufferSize; - maxFreeBuffers = 10; + maxBufferSize = largeBufferSize; + maxFreeBuffers = 10; } ByteStreamPool::ByteStreamPool(uint largeBufferSize, uint freeBufferLimit) { - maxBufferSize = largeBufferSize; - maxFreeBuffers = freeBufferLimit; + maxBufferSize = largeBufferSize; + maxFreeBuffers = freeBufferLimit; } ByteStreamPool::~ByteStreamPool() { - while (!freeByteStreams.empty()) - { - ByteStream *next = freeByteStreams.front(); - freeByteStreams.pop_front(); - delete next; - } + while (!freeByteStreams.empty()) + { + ByteStream* next = freeByteStreams.front(); + freeByteStreams.pop_front(); + delete next; + } } -ByteStream * ByteStreamPool::getByteStream() +ByteStream* ByteStreamPool::getByteStream() { + boost::mutex::scoped_lock s(mutex); + ByteStream* ret; + + if (!freeByteStreams.empty()) + { + ret = freeByteStreams.front(); + freeByteStreams.pop_front(); + } + else + ret = new ByteStream(); + + return ret; +} + +void ByteStreamPool::returnByteStream(ByteStream* bs) +{ + if (bs->getBufferSize() > maxBufferSize) + delete bs; + else + { boost::mutex::scoped_lock s(mutex); - ByteStream *ret; - - if (!freeByteStreams.empty()) - { - ret = freeByteStreams.front(); - freeByteStreams.pop_front(); - } + if (freeByteStreams.size() > maxFreeBuffers) + delete bs; else - ret = new ByteStream(); - - return ret; -} - -void ByteStreamPool::returnByteStream(ByteStream *bs) -{ - if (bs->getBufferSize() > maxBufferSize) - delete bs; - else { - boost::mutex::scoped_lock s(mutex); - if (freeByteStreams.size() > maxFreeBuffers) - delete bs; - else { - bs->restart(); - freeByteStreams.push_back(bs); - } + bs->restart(); + freeByteStreams.push_back(bs); } + } } -} +} // namespace messageqcpp diff --git a/utils/messageqcpp/bytestreampool.h b/utils/messageqcpp/bytestreampool.h index 443964b16..aa8e552a3 100644 --- a/utils/messageqcpp/bytestreampool.h +++ b/utils/messageqcpp/bytestreampool.h @@ -14,13 +14,13 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - + #ifndef BYTESTREAMPOOL_H_ #define BYTESTREAMPOOL_H_ /* This class defines a pool of bytestreams to improve BS reuse, reducing the need to use the dynamic allocator. It allocates as many BS's as needed, -and automatically disposes of buffers that are 'large' to reduce mem usage. +and automatically disposes of buffers that are 'large' to reduce mem usage. Initially, 'large' is defined as 1MB. */ @@ -30,25 +30,25 @@ Initially, 'large' is defined as 1MB. namespace messageqcpp { - class ByteStreamPool { - public: - ByteStreamPool(); - ByteStreamPool(uint largeBufferSize); // BS's above largeBufferSize get deallocated on return to the pool - ByteStreamPool(uint largeBufferSize, uint freeBufferLimit); // freeBufferLimit is the max # of BSs to reserve - virtual ~ByteStreamPool(); - - ByteStream * getByteStream(); - void returnByteStream(ByteStream *); - - private: - std::deque freeByteStreams; - boost::mutex mutex; - uint maxBufferSize; - uint maxFreeBuffers; + public: + ByteStreamPool(); + ByteStreamPool(uint largeBufferSize); // BS's above largeBufferSize get deallocated on return to the pool + ByteStreamPool(uint largeBufferSize, + uint freeBufferLimit); // freeBufferLimit is the max # of BSs to reserve + virtual ~ByteStreamPool(); + + ByteStream* getByteStream(); + void returnByteStream(ByteStream*); + + private: + std::deque freeByteStreams; + boost::mutex mutex; + uint maxBufferSize; + uint maxFreeBuffers; }; -} +} // namespace messageqcpp #endif diff --git a/utils/messageqcpp/cli.cpp b/utils/messageqcpp/cli.cpp index 55dff7c66..c67731df7 100644 --- a/utils/messageqcpp/cli.cpp +++ b/utils/messageqcpp/cli.cpp @@ -10,31 +10,30 @@ using namespace config; int main(int argc, char** argv) { - Config* cf = Config::makeConfig("./Columnstore.xml"); - MessageQueueClient mqc("server1", cf); + Config* cf = Config::makeConfig("./Columnstore.xml"); + MessageQueueClient mqc("server1", cf); - ByteStream obs; - string msg("Hello, world!"); - ByteStream ibs; - uint32_t qb; + ByteStream obs; + string msg("Hello, world!"); + ByteStream ibs; + uint32_t qb; - for (int i = 0; i < 10; i++) + for (int i = 0; i < 10; i++) + { + obs.restart(); + obs << msg; + cout << "writing " << obs.length() << " bytes to " << mqc.addr2String() << endl; + mqc.write(obs); + ibs = mqc.read(); + ibs >> qb; + + if (qb != 0) { - obs.restart(); - obs << msg; - cout << "writing " << obs.length() << " bytes to " << mqc.addr2String() << endl; - mqc.write(obs); - ibs = mqc.read(); - ibs >> qb; - - if (qb != 0) - { - string emsg("server did not ack message!"); - cerr << emsg << endl; - throw runtime_error(emsg); - } + string emsg("server did not ack message!"); + cerr << emsg << endl; + throw runtime_error(emsg); } + } - return 0; + return 0; } - diff --git a/utils/messageqcpp/compressed_iss.cpp b/utils/messageqcpp/compressed_iss.cpp index 585b3a3f5..4eba6405f 100644 --- a/utils/messageqcpp/compressed_iss.cpp +++ b/utils/messageqcpp/compressed_iss.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* -***********************************************************************/ + * $Id$ + * + * + ***********************************************************************/ #include "mcsconfig.h" #include @@ -59,99 +59,99 @@ using namespace compress; namespace messageqcpp { - CompressedInetStreamSocket::CompressedInetStreamSocket() { - config::Config* config = config::Config::makeConfig(); - string val; - string compressionType; + config::Config* config = config::Config::makeConfig(); + string val; + string compressionType; - try - { - val = config->getConfig("NetworkCompression", "Enabled"); - } - catch (...) { } + try + { + val = config->getConfig("NetworkCompression", "Enabled"); + } + catch (...) + { + } - if (val == "" || val == "Y") - useCompression = true; - else - useCompression = false; + if (val == "" || val == "Y") + useCompression = true; + else + useCompression = false; - try - { - compressionType = - config->getConfig("NetworkCompression", "NetworkCompression"); - } - catch (...) { } + try + { + compressionType = config->getConfig("NetworkCompression", "NetworkCompression"); + } + catch (...) + { + } - auto* compressInterface = compress::getCompressInterfaceByName(compressionType); - if (!compressInterface) - compressInterface = new compress::CompressInterfaceSnappy(); + auto* compressInterface = compress::getCompressInterfaceByName(compressionType); + if (!compressInterface) + compressInterface = new compress::CompressInterfaceSnappy(); - alg.reset(compressInterface); + alg.reset(compressInterface); } Socket* CompressedInetStreamSocket::clone() const { - return new CompressedInetStreamSocket(*this); + return new CompressedInetStreamSocket(*this); } const SBS CompressedInetStreamSocket::read(const struct timespec* timeout, bool* isTimeOut, - Stats* stats) const + Stats* stats) const { - SBS readBS, ret; - size_t uncompressedSize; + SBS readBS, ret; + size_t uncompressedSize; - readBS = InetStreamSocket::read(timeout, isTimeOut, stats); + readBS = InetStreamSocket::read(timeout, isTimeOut, stats); - if (readBS->length() == 0 || fMagicBuffer == BYTESTREAM_MAGIC) - return readBS; + if (readBS->length() == 0 || fMagicBuffer == BYTESTREAM_MAGIC) + return readBS; - // Read stored len, first 4 bytes. - uint32_t storedLen = *(uint32_t*) readBS->buf(); + // Read stored len, first 4 bytes. + uint32_t storedLen = *(uint32_t*)readBS->buf(); - if (!storedLen) - return SBS(new ByteStream(0)); + if (!storedLen) + return SBS(new ByteStream(0)); - uncompressedSize = storedLen; - ret.reset(new ByteStream(uncompressedSize)); + uncompressedSize = storedLen; + ret.reset(new ByteStream(uncompressedSize)); - alg->uncompress((char*) readBS->buf() + HEADER_SIZE, - readBS->length() - HEADER_SIZE, (char*) ret->getInputPtr(), - &uncompressedSize); + alg->uncompress((char*)readBS->buf() + HEADER_SIZE, readBS->length() - HEADER_SIZE, + (char*)ret->getInputPtr(), &uncompressedSize); - ret->advanceInputPtr(uncompressedSize); + ret->advanceInputPtr(uncompressedSize); - return ret; + return ret; } void CompressedInetStreamSocket::write(const ByteStream& msg, Stats* stats) { - size_t len = msg.length(); + size_t len = msg.length(); - if (useCompression && (len > 512)) - { - size_t outLen = alg->maxCompressedSize(len) + HEADER_SIZE; - ByteStream smsg(outLen); + if (useCompression && (len > 512)) + { + size_t outLen = alg->maxCompressedSize(len) + HEADER_SIZE; + ByteStream smsg(outLen); - alg->compress((char*) msg.buf(), len, - (char*) smsg.getInputPtr() + HEADER_SIZE, &outLen); - // Save original len. - *(uint32_t*) smsg.getInputPtr() = len; - smsg.advanceInputPtr(outLen + HEADER_SIZE); + alg->compress((char*)msg.buf(), len, (char*)smsg.getInputPtr() + HEADER_SIZE, &outLen); + // Save original len. + *(uint32_t*)smsg.getInputPtr() = len; + smsg.advanceInputPtr(outLen + HEADER_SIZE); - if (outLen < len) - do_write(smsg, COMPRESSED_BYTESTREAM_MAGIC, stats); - else - InetStreamSocket::write(msg, stats); - } + if (outLen < len) + do_write(smsg, COMPRESSED_BYTESTREAM_MAGIC, stats); else - InetStreamSocket::write(msg, stats); + InetStreamSocket::write(msg, stats); + } + else + InetStreamSocket::write(msg, stats); } void CompressedInetStreamSocket::write(SBS msg, Stats* stats) { - write(*msg, stats); + write(*msg, stats); } /* this was cut & pasted from InetStreamSocket; @@ -159,34 +159,33 @@ void CompressedInetStreamSocket::write(SBS msg, Stats* stats) */ const IOSocket CompressedInetStreamSocket::accept(const struct timespec* timeout) { - int clientfd; - long msecs = 0; + int clientfd; + long msecs = 0; - struct pollfd pfd[1]; - pfd[0].fd = socketParms().sd(); - pfd[0].events = POLLIN; + struct pollfd pfd[1]; + pfd[0].fd = socketParms().sd(); + pfd[0].events = POLLIN; - if (timeout != 0) - { - msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; + if (timeout != 0) + { + msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; - if (poll(pfd, 1, msecs) != 1 || (pfd[0].revents & POLLIN) == 0 || - pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL)) - return IOSocket(new CompressedInetStreamSocket()); - } + if (poll(pfd, 1, msecs) != 1 || (pfd[0].revents & POLLIN) == 0 || + pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL)) + return IOSocket(new CompressedInetStreamSocket()); + } - struct sockaddr sa; + struct sockaddr sa; - socklen_t sl = sizeof(sa); + socklen_t sl = sizeof(sa); - int e; + int e; - do - { - clientfd = ::accept(socketParms().sd(), &sa, &sl); - e = errno; - } - while (clientfd < 0 && (e == EINTR || + do + { + clientfd = ::accept(socketParms().sd(), &sa, &sl); + e = errno; + } while (clientfd < 0 && (e == EINTR || #ifdef ERESTART e == ERESTART || #endif @@ -195,90 +194,86 @@ const IOSocket CompressedInetStreamSocket::accept(const struct timespec* timeout #endif false)); - if (clientfd < 0) - { - string msg = "CompressedInetStreamSocket::accept: accept() error: "; - scoped_array buf(new char[80]); + if (clientfd < 0) + { + string msg = "CompressedInetStreamSocket::accept: accept() error: "; + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - msg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + msg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - msg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + msg += buf.get(); #endif - throw runtime_error(msg); - } + throw runtime_error(msg); + } - if (fSyncProto) + if (fSyncProto) + { + /* send a byte to artificially synchronize with connect() on the remote */ + char b = 'A'; + int ret; + + ret = ::send(clientfd, &b, 1, 0); + e = errno; + + if (ret < 0) { - /* send a byte to artificially synchronize with connect() on the remote */ - char b = 'A'; - int ret; - - ret = ::send(clientfd, &b, 1, 0); - e = errno; - - if (ret < 0) - { - ostringstream os; - char blah[80]; + ostringstream os; + char blah[80]; #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, blah, 80)) != 0) - os << "CompressedInetStreamSocket::accept sync: " << p; + if ((p = strerror_r(e, blah, 80)) != 0) + os << "CompressedInetStreamSocket::accept sync: " << p; #else - int p; + int p; - if ((p = strerror_r(e, blah, 80)) == 0) - os << "CompressedInetStreamSocket::accept sync: " << blah; + if ((p = strerror_r(e, blah, 80)) == 0) + os << "CompressedInetStreamSocket::accept sync: " << blah; #endif - ::close(clientfd); - throw runtime_error(os.str()); - } - else if (ret == 0) - { - ::close(clientfd); - throw runtime_error("CompressedInetStreamSocket::accept sync: got unexpected error code"); - } + ::close(clientfd); + throw runtime_error(os.str()); } + else if (ret == 0) + { + ::close(clientfd); + throw runtime_error("CompressedInetStreamSocket::accept sync: got unexpected error code"); + } + } - CompressedInetStreamSocket* ciss = new CompressedInetStreamSocket(); - IOSocket ios; - sockaddr_in* sin = (sockaddr_in*) &sa; + CompressedInetStreamSocket* ciss = new CompressedInetStreamSocket(); + IOSocket ios; + sockaddr_in* sin = (sockaddr_in*)&sa; - if ((sin->sin_addr.s_addr == fSa.sin_addr.s_addr) || - sin->sin_addr.s_addr == inet_addr("127.0.0.1")) - ciss->useCompression = false; + if ((sin->sin_addr.s_addr == fSa.sin_addr.s_addr) || sin->sin_addr.s_addr == inet_addr("127.0.0.1")) + ciss->useCompression = false; - ios.setSocketImpl(ciss); - SocketParms sp; - sp = ios.socketParms(); - sp.sd(clientfd); - ios.socketParms(sp); - ios.sa(&sa); - return ios; + ios.setSocketImpl(ciss); + SocketParms sp; + sp = ios.socketParms(); + sp.sd(clientfd); + ios.socketParms(sp); + ios.sa(&sa); + return ios; } void CompressedInetStreamSocket::connect(const sockaddr* serv_addr) { - sockaddr_in* sin = (sockaddr_in*) serv_addr; + sockaddr_in* sin = (sockaddr_in*)serv_addr; - if (sin->sin_addr.s_addr == fSa.sin_addr.s_addr || - sin->sin_addr.s_addr == inet_addr("127.0.0.1")) - useCompression = false; + if (sin->sin_addr.s_addr == fSa.sin_addr.s_addr || sin->sin_addr.s_addr == inet_addr("127.0.0.1")) + useCompression = false; - InetStreamSocket::connect(serv_addr); + InetStreamSocket::connect(serv_addr); } - -} //namespace messageqcpp - +} // namespace messageqcpp diff --git a/utils/messageqcpp/compressed_iss.h b/utils/messageqcpp/compressed_iss.h index 2514195da..72c622dbc 100644 --- a/utils/messageqcpp/compressed_iss.h +++ b/utils/messageqcpp/compressed_iss.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* -***********************************************************************/ + * $Id$ + * + * + ***********************************************************************/ /** @file */ #ifndef MESSAGEQCPP_COMPRESSED_ISS_H #define MESSAGEQCPP_COMPRESSED_ISS_H @@ -37,29 +37,29 @@ namespace messageqcpp { - class CompressedInetStreamSocket : public InetStreamSocket { -public: - CompressedInetStreamSocket(); - CompressedInetStreamSocket(const CompressedInetStreamSocket&) = default; - virtual ~CompressedInetStreamSocket(){}; + public: + CompressedInetStreamSocket(); + CompressedInetStreamSocket(const CompressedInetStreamSocket&) = default; + virtual ~CompressedInetStreamSocket(){}; - using InetStreamSocket::operator=; - virtual Socket* clone() const; - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, - Stats* stats = NULL) const; - virtual void write(const ByteStream& msg, Stats* stats = NULL); - virtual void write(SBS msg, Stats* stats = NULL); - virtual const IOSocket accept(const struct timespec* timeout); - virtual void connect(const sockaddr* addr); -private: - std::shared_ptr alg; - bool useCompression; - static const uint32_t HEADER_SIZE = 4; + using InetStreamSocket::operator=; + virtual Socket* clone() const; + virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, + Stats* stats = NULL) const; + virtual void write(const ByteStream& msg, Stats* stats = NULL); + virtual void write(SBS msg, Stats* stats = NULL); + virtual const IOSocket accept(const struct timespec* timeout); + virtual void connect(const sockaddr* addr); + + private: + std::shared_ptr alg; + bool useCompression; + static const uint32_t HEADER_SIZE = 4; }; -} //namespace messageqcpp +} // namespace messageqcpp #undef EXPORT diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index 67900b0b3..81ed82f6d 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -108,7 +108,7 @@ using messageqcpp::ByteStream; // that it is not supposed to "leak" out into the user space. But we are // sometimes seeing "unknown error 512" error msgs in response to calls to // read(), so adding logic to retry after ERESTARTSYS the way we do for EINTR. -//const int KERR_ERESTARTSYS = 512; +// const int KERR_ERESTARTSYS = 512; #ifdef _MSC_VER const int MaxSendPacketSize = 64 * 1024; @@ -116,43 +116,39 @@ const int MaxSendPacketSize = 64 * 1024; int in_cksum(unsigned short* buf, int sz) { - int nleft = sz; - int sum = 0; - unsigned short* w = buf; - unsigned short ans = 0; + int nleft = sz; + int sum = 0; + unsigned short* w = buf; + unsigned short ans = 0; - while (nleft > 1) - { - sum += *w++; - nleft -= 2; - } + while (nleft > 1) + { + sum += *w++; + nleft -= 2; + } - if (nleft == 1) - { - *(unsigned char*)(&ans) = *(unsigned char*)w; - sum += ans; - } + if (nleft == 1) + { + *(unsigned char*)(&ans) = *(unsigned char*)w; + sum += ans; + } - sum = (sum >> 16) + (sum & 0xFFFF); - sum += (sum >> 16); - ans = ~sum; - return ans; + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + ans = ~sum; + return ans; } -} //namespace anon +} // namespace namespace messageqcpp { - -InetStreamSocket::InetStreamSocket(size_t blocksize) : - fSocketParms(PF_INET, SOCK_STREAM, IPPROTO_TCP), - fBlocksize(blocksize), - fSyncProto(true), - fMagicBuffer(0) +InetStreamSocket::InetStreamSocket(size_t blocksize) + : fSocketParms(PF_INET, SOCK_STREAM, IPPROTO_TCP), fBlocksize(blocksize), fSyncProto(true), fMagicBuffer(0) { - memset(&fSa, 0, sizeof(fSa)); - fConnectionTimeout.tv_sec = 20; - fConnectionTimeout.tv_nsec = 0; + memset(&fSa, 0, sizeof(fSa)); + fConnectionTimeout.tv_sec = 20; + fConnectionTimeout.tv_nsec = 0; } InetStreamSocket::~InetStreamSocket() @@ -161,144 +157,145 @@ InetStreamSocket::~InetStreamSocket() void InetStreamSocket::open() { - int bufferSize; - int ret; - socklen_t bufferSizeSize; + int bufferSize; + int ret; + socklen_t bufferSizeSize; - if (isOpen()) - throw logic_error("InetStreamSocket::open: socket is already open"); + if (isOpen()) + throw logic_error("InetStreamSocket::open: socket is already open"); - int sd; - sd = ::socket(fSocketParms.domain(), fSocketParms.type(), fSocketParms.protocol()); - int e = errno; + int sd; + sd = ::socket(fSocketParms.domain(), fSocketParms.type(), fSocketParms.protocol()); + int e = errno; - if (sd < 0) - { + if (sd < 0) + { #ifdef _MSC_VER - int wsaError = WSAGetLastError(); + int wsaError = WSAGetLastError(); - if (wsaError == WSANOTINITIALISED) + if (wsaError == WSANOTINITIALISED) + { + WSAData wsadata; + const WORD minVersion = MAKEWORD(2, 2); + + if (WSAStartup(minVersion, &wsadata) == 0) + { + if (wsadata.wVersion == minVersion) { - WSAData wsadata; - const WORD minVersion = MAKEWORD(2, 2); + sd = ::socket(fSocketParms.domain(), fSocketParms.type(), fSocketParms.protocol()); + e = errno; - if (WSAStartup(minVersion, &wsadata) == 0) - { - if (wsadata.wVersion == minVersion) - { - sd = ::socket(fSocketParms.domain(), fSocketParms.type(), fSocketParms.protocol()); - e = errno; - - if (sd >= 0) goto setopts; - } - - //Didn't get the required min version, error out - } - - //WSAStartup failed, continue to report error + if (sd >= 0) + goto setopts; } -#endif - string msg = "InetStreamSocket::open: socket() error: "; - scoped_array buf(new char[80]); -#if STRERROR_R_CHAR_P - const char* p; + // Didn't get the required min version, error out + } - if ((p = strerror_r(e, buf.get(), 80)) != 0) - msg += p; + // WSAStartup failed, continue to report error + } + +#endif + string msg = "InetStreamSocket::open: socket() error: "; + scoped_array buf(new char[80]); +#if STRERROR_R_CHAR_P + const char* p; + + if ((p = strerror_r(e, buf.get(), 80)) != 0) + msg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - msg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + msg += buf.get(); #endif - throw runtime_error(msg); - } + throw runtime_error(msg); + } #ifdef _MSC_VER setopts: #endif - /* XXXPAT: If we have latency problems again, try these... - bufferSizeSize = 4; - bufferSize = 512000; - setsockopt(sd, SOL_SOCKET, SO_SNDBUF, &bufferSize, bufferSizeSize); - bufferSize = 512000; - setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &bufferSize, bufferSizeSize); - bufferSize = 1; - setsockopt(sd, SOL_SOCKET, SO_RCVLOWAT, &bufferSize, bufferSizeSize); - setsockopt(sd, SOL_SOCKET, SO_SNDLOWAT, &bufferSize, bufferSizeSize); - */ - bufferSize = 1; - bufferSizeSize = 4; - ret = setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (const char*)&bufferSize, bufferSizeSize); + /* XXXPAT: If we have latency problems again, try these... + bufferSizeSize = 4; + bufferSize = 512000; + setsockopt(sd, SOL_SOCKET, SO_SNDBUF, &bufferSize, bufferSizeSize); + bufferSize = 512000; + setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &bufferSize, bufferSizeSize); + bufferSize = 1; + setsockopt(sd, SOL_SOCKET, SO_RCVLOWAT, &bufferSize, bufferSizeSize); + setsockopt(sd, SOL_SOCKET, SO_SNDLOWAT, &bufferSize, bufferSizeSize); + */ + bufferSize = 1; + bufferSizeSize = 4; + ret = setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (const char*)&bufferSize, bufferSizeSize); - if (ret < 0) - { - perror("setsockopt"); - exit(1); - } + if (ret < 0) + { + perror("setsockopt"); + exit(1); + } - bufferSize = 1; - ret = setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (const char*)&bufferSize, bufferSizeSize); + bufferSize = 1; + ret = setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (const char*)&bufferSize, bufferSizeSize); - if (ret < 0) - { - perror("setsockopt"); - exit(1); - } + if (ret < 0) + { + perror("setsockopt"); + exit(1); + } - fSocketParms.sd(sd); + fSocketParms.sd(sd); } void InetStreamSocket::close() { - if (isOpen()) - { - ::shutdown(fSocketParms.sd(), SHUT_RDWR); + if (isOpen()) + { + ::shutdown(fSocketParms.sd(), SHUT_RDWR); #ifdef _MSC_VER - ::closesocket(fSocketParms.sd()); + ::closesocket(fSocketParms.sd()); #else - ::close(fSocketParms.sd()); + ::close(fSocketParms.sd()); #endif - fSocketParms.sd(-1); - } + fSocketParms.sd(-1); + } } // needs to be in sync with clone() void InetStreamSocket::doCopy(const InetStreamSocket& rhs) { - fBlocksize = rhs.fBlocksize; - fSocketParms = rhs.fSocketParms; - fSa = rhs.fSa; - fConnectionTimeout = rhs.fConnectionTimeout; - fSyncProto = rhs.fSyncProto; + fBlocksize = rhs.fBlocksize; + fSocketParms = rhs.fSocketParms; + fSa = rhs.fSa; + fConnectionTimeout = rhs.fConnectionTimeout; + fSyncProto = rhs.fSyncProto; } // needs to be in sync with doCopy() Socket* InetStreamSocket::clone() const { - InetStreamSocket* iss = new InetStreamSocket(fBlocksize); - iss->fSocketParms = fSocketParms; - iss->fSa = fSa; - iss->fConnectionTimeout = fConnectionTimeout; - iss->fSyncProto = fSyncProto; - return iss; + InetStreamSocket* iss = new InetStreamSocket(fBlocksize); + iss->fSocketParms = fSocketParms; + iss->fSa = fSa; + iss->fConnectionTimeout = fConnectionTimeout; + iss->fSyncProto = fSyncProto; + return iss; } InetStreamSocket::InetStreamSocket(const InetStreamSocket& rhs) { - doCopy(rhs); + doCopy(rhs); } InetStreamSocket& InetStreamSocket::operator=(const InetStreamSocket& rhs) { - if (this != &rhs) - doCopy(rhs); + if (this != &rhs) + doCopy(rhs); - return *this; + return *this; } /* The caller needs to know when/if the remote closes the connection or sends data. @@ -306,511 +303,504 @@ InetStreamSocket& InetStreamSocket::operator=(const InetStreamSocket& rhs) */ int InetStreamSocket::pollConnection(int connectionNum, long msecs) { - struct pollfd pfd[1]; - int err; + struct pollfd pfd[1]; + int err; retry: - memset(&pfd, 0, sizeof(struct pollfd)); - pfd[0].fd = connectionNum; - pfd[0].events = POLLIN; - err = poll(pfd, 1, msecs); + memset(&pfd, 0, sizeof(struct pollfd)); + pfd[0].fd = connectionNum; + pfd[0].events = POLLIN; + err = poll(pfd, 1, msecs); - if (err < 0) - { - int e = errno; + if (err < 0) + { + int e = errno; - if (e == EINTR || e == KERR_ERESTARTSYS) - goto retry; - } + if (e == EINTR || e == KERR_ERESTARTSYS) + goto retry; + } - // Linux doesn't set POLLHUP, need add'l check for data or EOF - if (pfd[0].revents & POLLIN) - { - char buf; - err = ::recv(connectionNum, &buf, 1, MSG_PEEK); + // Linux doesn't set POLLHUP, need add'l check for data or EOF + if (pfd[0].revents & POLLIN) + { + char buf; + err = ::recv(connectionNum, &buf, 1, MSG_PEEK); - if (err == 0) - return 2; - else if (err == 1) // there is in fact data to read - return 1; - else - return 3; - } + if (err == 0) + return 2; + else if (err == 1) // there is in fact data to read + return 1; + else + return 3; + } - if (err == 0) // timeout - return 0; + if (err == 0) // timeout + return 0; - return 3; // catch-all error code + return 3; // catch-all error code } /* returns true when the next thing in the stream is the beginning of a new ByteStream object. */ bool InetStreamSocket::readToMagic(long msecs, bool* isTimeOut, Stats* stats) const { - int err; - struct pollfd pfd[1]; - uint8_t* magicBuffer8; + int err; + struct pollfd pfd[1]; + uint8_t* magicBuffer8; - fMagicBuffer = 0; - magicBuffer8 = reinterpret_cast(&fMagicBuffer); - pfd[0].fd = fSocketParms.sd(); - pfd[0].events = POLLIN; + fMagicBuffer = 0; + magicBuffer8 = reinterpret_cast(&fMagicBuffer); + pfd[0].fd = fSocketParms.sd(); + pfd[0].events = POLLIN; - while ((fMagicBuffer != BYTESTREAM_MAGIC) && (fMagicBuffer != COMPRESSED_BYTESTREAM_MAGIC)) + while ((fMagicBuffer != BYTESTREAM_MAGIC) && (fMagicBuffer != COMPRESSED_BYTESTREAM_MAGIC)) + { + if (msecs >= 0) { + pfd[0].revents = 0; - if (msecs >= 0) + err = poll(pfd, 1, msecs); + + if (err < 0) + { + int e = errno; + + if (e == EINTR) { - pfd[0].revents = 0; - - err = poll(pfd, 1, msecs); - - if (err < 0) - { - int e = errno; - - if (e == EINTR) - { - continue; - } - - if (e == KERR_ERESTARTSYS) - { - logIoError("InetStreamSocket::readToMagic(): I/O error1", e); - continue; - } - - ostringstream oss; - oss << "InetStreamSocket::readToMagic(): I/O error1: " << - strerror(e); - throw runtime_error(oss.str()); - } - - if (pfd[0].revents & (POLLHUP | POLLNVAL | POLLERR)) - { - ostringstream oss; - oss << "InetStreamSocket::readToMagic(): I/O error1: rc-" << - err << "; poll signal interrupt ( "; - - if (pfd[0].revents & POLLHUP) - oss << "POLLHUP "; - - if (pfd[0].revents & POLLNVAL) - oss << "POLLNVAL "; - - if (pfd[0].revents & POLLERR) - oss << "POLLERR "; - - oss << ")"; - throw runtime_error(oss.str()); - } - - if (err == 0) // timeout - { - if (isTimeOut) - *isTimeOut = true; - - return false; - } + continue; } - fMagicBuffer = fMagicBuffer >> 8; -retry: -#ifdef _MSC_VER - err = ::recv(fSocketParms.sd(), (char*)&magicBuffer8[3], 1, 0); -#else - err = ::read(fSocketParms.sd(), &magicBuffer8[3], 1); -#endif - - if (err < 0) + if (e == KERR_ERESTARTSYS) { - int e = errno; -#ifdef _MSC_VER - - if (WSAGetLastError() == WSAECONNRESET) - { - //throw runtime_error("connection reset by peer"); - if (msecs < 0) return false; - else throw SocketClosed("InetStreamSocket::readToMagic: Remote is closed"); - } - -#endif - - if (e == EINTR) - { - goto retry; - } - - if (e == KERR_ERESTARTSYS) - { - logIoError("InetStreamSocket::readToMagic(): I/O error2.0", e); - goto retry; - } - - ostringstream oss; - oss << "InetStreamSocket::readToMagic(): I/O error2.1: " << - "err = " << err << " e = " << e << -#ifdef _MSC_VER - " WSA error = " << WSAGetLastError() << -#endif - ": " << strerror(e); - throw runtime_error(oss.str()); + logIoError("InetStreamSocket::readToMagic(): I/O error1", e); + continue; } - // EOF. If no timeout was specified, ByteStream() gets returned to the caller. - // If one was, throw SocketClosed. - if (err == 0) // EOF. if a timeout was specified, ByteStream() - { - if (msecs < 0) - return false; - else - throw SocketClosed("InetStreamSocket::readToMagic: Remote is closed"); - } + ostringstream oss; + oss << "InetStreamSocket::readToMagic(): I/O error1: " << strerror(e); + throw runtime_error(oss.str()); + } - if (stats) - stats->dataRecvd(1); + if (pfd[0].revents & (POLLHUP | POLLNVAL | POLLERR)) + { + ostringstream oss; + oss << "InetStreamSocket::readToMagic(): I/O error1: rc-" << err << "; poll signal interrupt ( "; + + if (pfd[0].revents & POLLHUP) + oss << "POLLHUP "; + + if (pfd[0].revents & POLLNVAL) + oss << "POLLNVAL "; + + if (pfd[0].revents & POLLERR) + oss << "POLLERR "; + + oss << ")"; + throw runtime_error(oss.str()); + } + + if (err == 0) // timeout + { + if (isTimeOut) + *isTimeOut = true; + + return false; + } } - return true; -} - -bool InetStreamSocket::readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, - const size_t numberOfBytes, - const struct ::timespec* timeout, bool* isTimeOut, - Stats* stats, int64_t msecs) const -{ - size_t bytesRead = 0; - while (bytesRead < numberOfBytes) - { - ssize_t currentBytesRead; - int err; - - if (timeout != NULL) - { - pfd[0].revents = 0; - err = poll(pfd, 1, msecs); - - if (err < 0 || pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL)) - { - ostringstream oss; - oss << "InetStreamSocket::read: I/O error1: " << - strerror(errno); - throw runtime_error(oss.str()); - } - - if (err == 0) // timeout - { - if (isTimeOut) - *isTimeOut = true; - - logIoError("InetStreamSocket::read: timeout during first poll", 0); - return false; - } - } - + fMagicBuffer = fMagicBuffer >> 8; + retry: #ifdef _MSC_VER - currentBytesRead = - ::recv(fSocketParms.sd(), (char*) (buffer + bytesRead), - std::min(numberOfBytes - bytesRead, reinterpret_cast(MaxSendPacketSize)); - readAmoumt, 0); + err = ::recv(fSocketParms.sd(), (char*)&magicBuffer8[3], 1, 0); #else - currentBytesRead = ::read(fSocketParms.sd(), buffer + bytesRead, numberOfBytes - bytesRead); + err = ::read(fSocketParms.sd(), &magicBuffer8[3], 1); #endif - if (currentBytesRead == 0) - { - if (timeout == NULL) - { - logIoError("InetStreamSocket::read: timeout during first read", 0); - return false; - } - else - throw SocketClosed("InetStreamSocket::read: Remote is closed"); - } + if (err < 0) + { + int e = errno; +#ifdef _MSC_VER - if (currentBytesRead < 0) - { - err = errno; + if (WSAGetLastError() == WSAECONNRESET) + { + // throw runtime_error("connection reset by peer"); + if (msecs < 0) + return false; + else + throw SocketClosed("InetStreamSocket::readToMagic: Remote is closed"); + } - if (err == EINTR) - continue; +#endif - if (err == KERR_ERESTARTSYS) - { - logIoError("InetStreamSocket::read: I/O error2", err); - continue; - } + if (e == EINTR) + { + goto retry; + } - ostringstream oss; - oss << "InetStreamSocket::read: I/O error2: " << strerror(err); - throw runtime_error(oss.str()); - } + if (e == KERR_ERESTARTSYS) + { + logIoError("InetStreamSocket::readToMagic(): I/O error2.0", e); + goto retry; + } - bytesRead += currentBytesRead; + ostringstream oss; + oss << "InetStreamSocket::readToMagic(): I/O error2.1: " + << "err = " << err << " e = " << e << +#ifdef _MSC_VER + " WSA error = " << WSAGetLastError() << +#endif + ": " << strerror(e); + throw runtime_error(oss.str()); + } + + // EOF. If no timeout was specified, ByteStream() gets returned to the caller. + // If one was, throw SocketClosed. + if (err == 0) // EOF. if a timeout was specified, ByteStream() + { + if (msecs < 0) + return false; + else + throw SocketClosed("InetStreamSocket::readToMagic: Remote is closed"); } if (stats) - stats->dataRecvd(bytesRead); + stats->dataRecvd(1); + } - return true; + return true; } -const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeOut, - Stats* stats) const +bool InetStreamSocket::readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, const size_t numberOfBytes, + const struct ::timespec* timeout, bool* isTimeOut, Stats* stats, + int64_t msecs) const { - int64_t msecs = -1; + size_t bytesRead = 0; + while (bytesRead < numberOfBytes) + { + ssize_t currentBytesRead; + int err; - struct pollfd pfd[1]; - pfd[0].fd = fSocketParms.sd(); - pfd[0].events = POLLIN; - - if (timeout != 0) - msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; - - if (readToMagic(msecs, isTimeOut, stats) == false) // indicates a timeout or EOF + if (timeout != NULL) { - // MCOL-480 The connector calls with timeout in a loop so that - // it can check a killed flag. This means that for a long running query, - // the following fills the warning log. - // if (isTimeOut && *isTimeOut) - // { - // logIoError("InetStreamSocket::read: timeout during readToMagic", 0); - // } - return SBS(new ByteStream(0)); + pfd[0].revents = 0; + err = poll(pfd, 1, msecs); + + if (err < 0 || pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL)) + { + ostringstream oss; + oss << "InetStreamSocket::read: I/O error1: " << strerror(errno); + throw runtime_error(oss.str()); + } + + if (err == 0) // timeout + { + if (isTimeOut) + *isTimeOut = true; + + logIoError("InetStreamSocket::read: timeout during first poll", 0); + return false; + } } - // we need to read the 4-byte message length first. - uint32_t msglen; - if (!readFixedSizeData(pfd, reinterpret_cast(&msglen), sizeof(msglen), timeout, - isTimeOut, stats, msecs)) - return SBS(new ByteStream(0)); +#ifdef _MSC_VER + currentBytesRead = + ::recv(fSocketParms.sd(), (char*)(buffer + bytesRead), + std::min(numberOfBytes - bytesRead, reinterpret_cast(MaxSendPacketSize)); + readAmoumt, 0); +#else + currentBytesRead = ::read(fSocketParms.sd(), buffer + bytesRead, numberOfBytes - bytesRead); +#endif - // Read the number of the `long strings`. - uint32_t longStringSize; - if (!readFixedSizeData(pfd, reinterpret_cast(&longStringSize), sizeof(longStringSize), - timeout, isTimeOut, stats, msecs)) - return SBS(new ByteStream(0)); - - // Read the actual data of the `ByteStream`. - SBS res(new ByteStream(msglen)); - if (!readFixedSizeData(pfd, res->getInputPtr(), msglen, timeout, isTimeOut, stats, msecs)) - return SBS(new ByteStream(0)); - res->advanceInputPtr(msglen); - - std::vector> longStrings; - try + if (currentBytesRead == 0) { - for (uint32_t i = 0; i < longStringSize; ++i) - { - // Read `MemChunk`. - rowgroup::StringStore::MemChunk memChunk; - if (!readFixedSizeData(pfd, reinterpret_cast(&memChunk), - sizeof(rowgroup::StringStore::MemChunk), timeout, isTimeOut, - stats, msecs)) - return SBS(new ByteStream(0)); - - // Allocate new memory for the `long string`. - boost::shared_array longString( - new uint8_t[sizeof(rowgroup::StringStore::MemChunk) + memChunk.currentSize]); - - uint8_t* longStringData = longString.get(); - // Initialize memchunk with `current size` and `capacity`. - auto* memChunkPointer = - reinterpret_cast(longStringData); - memChunkPointer->currentSize = memChunk.currentSize; - memChunkPointer->capacity = memChunk.capacity; - - // Read the `long string`. - if (!readFixedSizeData(pfd, memChunkPointer->data, memChunkPointer->currentSize, - timeout, isTimeOut, stats, msecs)) - return SBS(new ByteStream(0)); - - longStrings.push_back(longString); - } - } - catch (std::bad_alloc& exception) - { - logIoError("InetStreamSocket::read: error during read for 'long strings' - 'bad_alloc'", 0); - return SBS(new ByteStream(0)); - } - catch (std::exception& exception) - { - std::string errorMsg = "InetStreamSocket::read: error during read for 'long strings' "; - errorMsg += exception.what(); - throw runtime_error(errorMsg); + if (timeout == NULL) + { + logIoError("InetStreamSocket::read: timeout during first read", 0); + return false; + } + else + throw SocketClosed("InetStreamSocket::read: Remote is closed"); } - res->setLongStrings(longStrings); - return res; + if (currentBytesRead < 0) + { + err = errno; + + if (err == EINTR) + continue; + + if (err == KERR_ERESTARTSYS) + { + logIoError("InetStreamSocket::read: I/O error2", err); + continue; + } + + ostringstream oss; + oss << "InetStreamSocket::read: I/O error2: " << strerror(err); + throw runtime_error(oss.str()); + } + + bytesRead += currentBytesRead; + } + + if (stats) + stats->dataRecvd(bytesRead); + + return true; +} + +const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeOut, Stats* stats) const +{ + int64_t msecs = -1; + + struct pollfd pfd[1]; + pfd[0].fd = fSocketParms.sd(); + pfd[0].events = POLLIN; + + if (timeout != 0) + msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; + + if (readToMagic(msecs, isTimeOut, stats) == false) // indicates a timeout or EOF + { + // MCOL-480 The connector calls with timeout in a loop so that + // it can check a killed flag. This means that for a long running query, + // the following fills the warning log. + // if (isTimeOut && *isTimeOut) + // { + // logIoError("InetStreamSocket::read: timeout during readToMagic", 0); + // } + return SBS(new ByteStream(0)); + } + + // we need to read the 4-byte message length first. + uint32_t msglen; + if (!readFixedSizeData(pfd, reinterpret_cast(&msglen), sizeof(msglen), timeout, isTimeOut, stats, + msecs)) + return SBS(new ByteStream(0)); + + // Read the number of the `long strings`. + uint32_t longStringSize; + if (!readFixedSizeData(pfd, reinterpret_cast(&longStringSize), sizeof(longStringSize), timeout, + isTimeOut, stats, msecs)) + return SBS(new ByteStream(0)); + + // Read the actual data of the `ByteStream`. + SBS res(new ByteStream(msglen)); + if (!readFixedSizeData(pfd, res->getInputPtr(), msglen, timeout, isTimeOut, stats, msecs)) + return SBS(new ByteStream(0)); + res->advanceInputPtr(msglen); + + std::vector> longStrings; + try + { + for (uint32_t i = 0; i < longStringSize; ++i) + { + // Read `MemChunk`. + rowgroup::StringStore::MemChunk memChunk; + if (!readFixedSizeData(pfd, reinterpret_cast(&memChunk), + sizeof(rowgroup::StringStore::MemChunk), timeout, isTimeOut, stats, msecs)) + return SBS(new ByteStream(0)); + + // Allocate new memory for the `long string`. + boost::shared_array longString( + new uint8_t[sizeof(rowgroup::StringStore::MemChunk) + memChunk.currentSize]); + + uint8_t* longStringData = longString.get(); + // Initialize memchunk with `current size` and `capacity`. + auto* memChunkPointer = reinterpret_cast(longStringData); + memChunkPointer->currentSize = memChunk.currentSize; + memChunkPointer->capacity = memChunk.capacity; + + // Read the `long string`. + if (!readFixedSizeData(pfd, memChunkPointer->data, memChunkPointer->currentSize, timeout, isTimeOut, + stats, msecs)) + return SBS(new ByteStream(0)); + + longStrings.push_back(longString); + } + } + catch (std::bad_alloc& exception) + { + logIoError("InetStreamSocket::read: error during read for 'long strings' - 'bad_alloc'", 0); + return SBS(new ByteStream(0)); + } + catch (std::exception& exception) + { + std::string errorMsg = "InetStreamSocket::read: error during read for 'long strings' "; + errorMsg += exception.what(); + throw runtime_error(errorMsg); + } + + res->setLongStrings(longStrings); + return res; } /* -* The protocol here is that we write the length of the ByteStream first, then the bytes. On the -* read side, we reverse it. -*/ + * The protocol here is that we write the length of the ByteStream first, then the bytes. On the + * read side, we reverse it. + */ void InetStreamSocket::write(SBS msg, Stats* stats) { - write(*msg, stats); + write(*msg, stats); } void InetStreamSocket::do_write(const ByteStream& msg, uint32_t whichMagic, Stats* stats) const { - uint32_t msglen = msg.length(); - uint32_t magic = whichMagic; - uint32_t* realBuf; + uint32_t msglen = msg.length(); + uint32_t magic = whichMagic; + uint32_t* realBuf; - if (msglen == 0) return; + if (msglen == 0) + return; - const auto& longStrings = msg.getLongStrings(); - /* buf.fCurOutPtr points to the data to send; ByteStream guarantees that there - are at least 12 bytes before that for the magic & length fields */ - realBuf = (uint32_t*)msg.buf(); - realBuf -= 3; - realBuf[0] = magic; - realBuf[1] = msglen; - realBuf[2] = longStrings.size(); + const auto& longStrings = msg.getLongStrings(); + /* buf.fCurOutPtr points to the data to send; ByteStream guarantees that there + are at least 12 bytes before that for the magic & length fields */ + realBuf = (uint32_t*)msg.buf(); + realBuf -= 3; + realBuf[0] = magic; + realBuf[1] = msglen; + realBuf[2] = longStrings.size(); - try + try + { + auto bytesToWrite = sizeof(msglen) + sizeof(magic) + sizeof(uint32_t) + msglen; + written(fSocketParms.sd(), (const uint8_t*)realBuf, bytesToWrite); + + for (const auto& longString : longStrings) { - auto bytesToWrite = sizeof(msglen) + sizeof(magic) + sizeof(uint32_t) + msglen; - written(fSocketParms.sd(), (const uint8_t*) realBuf, bytesToWrite); - - for (const auto& longString : longStrings) - { - const rowgroup::StringStore::MemChunk* memChunk = - reinterpret_cast(longString.get()); - const auto writeSize = memChunk->currentSize + sizeof(rowgroup::StringStore::MemChunk); - written(fSocketParms.sd(), (const uint8_t*) longString.get(), writeSize); - // For stats. - bytesToWrite += writeSize; - } - - if (stats) - stats->dataSent(bytesToWrite); - } - catch (std::exception& ex) - { - string errorMsg(ex.what()); - errorMsg += " -- write from " + toString(); - throw runtime_error(errorMsg); + const rowgroup::StringStore::MemChunk* memChunk = + reinterpret_cast(longString.get()); + const auto writeSize = memChunk->currentSize + sizeof(rowgroup::StringStore::MemChunk); + written(fSocketParms.sd(), (const uint8_t*)longString.get(), writeSize); + // For stats. + bytesToWrite += writeSize; } + + if (stats) + stats->dataSent(bytesToWrite); + } + catch (std::exception& ex) + { + string errorMsg(ex.what()); + errorMsg += " -- write from " + toString(); + throw runtime_error(errorMsg); + } } void InetStreamSocket::write(const ByteStream& msg, Stats* stats) { - do_write(msg, BYTESTREAM_MAGIC, stats); + do_write(msg, BYTESTREAM_MAGIC, stats); } void InetStreamSocket::write_raw(const ByteStream& msg, Stats* stats) const { - uint32_t msglen = msg.length(); + uint32_t msglen = msg.length(); - if (msglen == 0) return; + if (msglen == 0) + return; - try - { - written(fSocketParms.sd(), msg.buf(), msglen); - } - catch (std::exception& ex) - { - string errorMsg(ex.what()); - errorMsg += " -- write_raw from " + toString(); - throw runtime_error(errorMsg); - } + try + { + written(fSocketParms.sd(), msg.buf(), msglen); + } + catch (std::exception& ex) + { + string errorMsg(ex.what()); + errorMsg += " -- write_raw from " + toString(); + throw runtime_error(errorMsg); + } - if (stats) - stats->dataSent(msglen); + if (stats) + stats->dataSent(msglen); } void InetStreamSocket::bind(const sockaddr* serv_addr) { - memcpy(&fSa, serv_addr, sizeof(sockaddr_in)); + memcpy(&fSa, serv_addr, sizeof(sockaddr_in)); - if (::bind(fSocketParms.sd(), serv_addr, sizeof(sockaddr_in)) != 0) - { - int e = errno; - string msg = "InetStreamSocket::bind: bind() error: "; - scoped_array buf(new char[80]); + if (::bind(fSocketParms.sd(), serv_addr, sizeof(sockaddr_in)) != 0) + { + int e = errno; + string msg = "InetStreamSocket::bind: bind() error: "; + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - msg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + msg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - msg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + msg += buf.get(); #endif - throw runtime_error(msg); - } - + throw runtime_error(msg); + } } void InetStreamSocket::listen(int backlog) { #ifndef _MSC_VER - fcntl(socketParms().sd(), F_SETFD, fcntl(socketParms().sd(), F_GETFD) | FD_CLOEXEC); + fcntl(socketParms().sd(), F_SETFD, fcntl(socketParms().sd(), F_GETFD) | FD_CLOEXEC); #endif - if (::listen(socketParms().sd(), backlog) != 0) - { - int e = errno; - string msg = "InetStreamSocket::listen: listen() error: "; - scoped_array buf(new char[80]); + if (::listen(socketParms().sd(), backlog) != 0) + { + int e = errno; + string msg = "InetStreamSocket::listen: listen() error: "; + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - msg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + msg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - msg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + msg += buf.get(); #endif - throw runtime_error(msg); - } - + throw runtime_error(msg); + } } const IOSocket InetStreamSocket::accept(const struct timespec* timeout) { - int clientfd; - long msecs = 0; + int clientfd; + long msecs = 0; - IOSocket ios(new InetStreamSocket(fBlocksize)); + IOSocket ios(new InetStreamSocket(fBlocksize)); - struct pollfd pfd[1]; - pfd[0].fd = socketParms().sd(); - pfd[0].events = POLLIN; + struct pollfd pfd[1]; + pfd[0].fd = socketParms().sd(); + pfd[0].events = POLLIN; - if (timeout != 0) - { - msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; + if (timeout != 0) + { + msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; - if (poll(pfd, 1, msecs) != 1 || (pfd[0].revents & POLLIN) == 0 || - pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL)) - return ios; - } + if (poll(pfd, 1, msecs) != 1 || (pfd[0].revents & POLLIN) == 0 || + pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL)) + return ios; + } - struct sockaddr sa; + struct sockaddr sa; - socklen_t sl = sizeof(sa); + socklen_t sl = sizeof(sa); - int e; + int e; - do - { - clientfd = ::accept(socketParms().sd(), &sa, &sl); - e = errno; - } - while (clientfd < 0 && (e == EINTR || + do + { + clientfd = ::accept(socketParms().sd(), &sa, &sl); + e = errno; + } while (clientfd < 0 && (e == EINTR || #ifdef ERESTART e == ERESTART || #endif @@ -819,187 +809,185 @@ const IOSocket InetStreamSocket::accept(const struct timespec* timeout) #endif false)); - if (clientfd < 0) - { - string msg = "InetStreamSocket::accept: accept() error: "; - scoped_array buf(new char[80]); + if (clientfd < 0) + { + string msg = "InetStreamSocket::accept: accept() error: "; + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - msg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + msg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - msg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + msg += buf.get(); #endif - throw runtime_error(msg); - } + throw runtime_error(msg); + } - if (fSyncProto) + if (fSyncProto) + { + /* send a byte to artificially synchronize with connect() on the remote */ + char b = 'A'; + int ret; + + ret = ::send(clientfd, &b, 1, 0); + e = errno; + + if (ret < 0) { - /* send a byte to artificially synchronize with connect() on the remote */ - char b = 'A'; - int ret; - - ret = ::send(clientfd, &b, 1, 0); - e = errno; - - if (ret < 0) - { - ostringstream os; - char blah[80]; + ostringstream os; + char blah[80]; #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, blah, 80)) != 0) - os << "InetStreamSocket::accept sync: " << p; + if ((p = strerror_r(e, blah, 80)) != 0) + os << "InetStreamSocket::accept sync: " << p; #else - int p; + int p; - if ((p = strerror_r(e, blah, 80)) == 0) - os << "InetStreamSocket::accept sync: " << blah; + if ((p = strerror_r(e, blah, 80)) == 0) + os << "InetStreamSocket::accept sync: " << blah; #endif - ::close(clientfd); - throw runtime_error(os.str()); - } - else if (ret == 0) - { - ::close(clientfd); - throw runtime_error("InetStreamSocket::accept sync: got unexpected error code"); - } + ::close(clientfd); + throw runtime_error(os.str()); } + else if (ret == 0) + { + ::close(clientfd); + throw runtime_error("InetStreamSocket::accept sync: got unexpected error code"); + } + } - SocketParms sp; - sp = ios.socketParms(); - sp.sd(clientfd); - ios.socketParms(sp); - ios.sa(&sa); - return ios; - + SocketParms sp; + sp = ios.socketParms(); + sp.sd(clientfd); + ios.socketParms(sp); + ios.sa(&sa); + return ios; } void InetStreamSocket::connect(const sockaddr* serv_addr) { - memcpy(&fSa, serv_addr, sizeof(sockaddr_in)); + memcpy(&fSa, serv_addr, sizeof(sockaddr_in)); - if (::connect(socketParms().sd(), serv_addr, sizeof(sockaddr_in))) - { - int e = errno; - string msg = "InetStreamSocket::connect: connect() error: "; + if (::connect(socketParms().sd(), serv_addr, sizeof(sockaddr_in))) + { + int e = errno; + string msg = "InetStreamSocket::connect: connect() error: "; #ifdef _MSC_VER - char m[80]; - int x = WSAGetLastError(); + char m[80]; + int x = WSAGetLastError(); - if (x == WSAECONNREFUSED) - strcpy(m, "connection refused"); - else - sprintf(m, "%d 0x%x", x, x); + if (x == WSAECONNREFUSED) + strcpy(m, "connection refused"); + else + sprintf(m, "%d 0x%x", x, x); - msg += m; + msg += m; #else - scoped_array buf(new char[80]); + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - msg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + msg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - msg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + msg += buf.get(); #endif #endif - msg += " to: " + toString(); - throw runtime_error(msg); - } + msg += " to: " + toString(); + throw runtime_error(msg); + } - if (!fSyncProto) - return; + if (!fSyncProto) + return; - /* read a byte to artificially synchronize with accept() on the remote */ - int ret = -1; - int e = EBADF; - struct pollfd pfd; + /* read a byte to artificially synchronize with accept() on the remote */ + int ret = -1; + int e = EBADF; + struct pollfd pfd; - long msecs = fConnectionTimeout.tv_sec * 1000 + fConnectionTimeout.tv_nsec / 1000000; + long msecs = fConnectionTimeout.tv_sec * 1000 + fConnectionTimeout.tv_nsec / 1000000; - do - { - pfd.fd = socketParms().sd(); - pfd.revents = 0; - pfd.events = POLLIN; - ret = poll(&pfd, 1, msecs); - e = errno; - } - while (ret == -1 && e == EINTR && !(pfd.revents & (POLLERR | POLLHUP | POLLNVAL))); + do + { + pfd.fd = socketParms().sd(); + pfd.revents = 0; + pfd.events = POLLIN; + ret = poll(&pfd, 1, msecs); + e = errno; + } while (ret == -1 && e == EINTR && !(pfd.revents & (POLLERR | POLLHUP | POLLNVAL))); - // success - if (ret == 1) - { + // success + if (ret == 1) + { #ifdef _MSC_VER - char buf = '\0'; - (void)::recv(socketParms().sd(), &buf, 1, 0); + char buf = '\0'; + (void)::recv(socketParms().sd(), &buf, 1, 0); #else #if defined(__GNUC__) && __GNUC__ >= 5 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer + char buf = '\0'; + ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer #pragma GCC diagnostic pop #else - char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#endif // pragma + char buf = '\0'; + ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer +#endif // pragma #endif - return; - } + return; + } - /* handle the various errors */ - if (ret == 0) - throw runtime_error("InetStreamSocket::connect: connection timed out"); - else if (ret == -1 && e != EINTR) - { - ostringstream os; - char blah[80]; + /* handle the various errors */ + if (ret == 0) + throw runtime_error("InetStreamSocket::connect: connection timed out"); + else if (ret == -1 && e != EINTR) + { + ostringstream os; + char blah[80]; #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, blah, 80)) != 0) - os << "InetStreamSocket::connect: " << p; + if ((p = strerror_r(e, blah, 80)) != 0) + os << "InetStreamSocket::connect: " << p; #else - int p; + int p; - if ((p = strerror_r(e, blah, 80)) == 0) - os << "InetStreamSocket::connect: " << blah; + if ((p = strerror_r(e, blah, 80)) == 0) + os << "InetStreamSocket::connect: " << blah; #endif - throw runtime_error(os.str()); - } - else - throw runtime_error("InetStreamSocket::connect: unknown connection error"); + throw runtime_error(os.str()); + } + else + throw runtime_error("InetStreamSocket::connect: unknown connection error"); } const string InetStreamSocket::toString() const { - ostringstream oss; - char buf[INET_ADDRSTRLEN]; - const SocketParms& sp = fSocketParms; - oss << "InetStreamSocket: sd: " << sp.sd() << + ostringstream oss; + char buf[INET_ADDRSTRLEN]; + const SocketParms& sp = fSocketParms; + oss << "InetStreamSocket: sd: " << sp.sd() << #ifndef _MSC_VER - " inet: " << inet_ntop(AF_INET, &fSa.sin_addr, buf, INET_ADDRSTRLEN) << + " inet: " << inet_ntop(AF_INET, &fSa.sin_addr, buf, INET_ADDRSTRLEN) << #endif - " port: " << ntohs(fSa.sin_port); - return oss.str(); + " port: " << ntohs(fSa.sin_port); + return oss.str(); } // @@ -1009,250 +997,250 @@ const string InetStreamSocket::toString() const // void InetStreamSocket::logIoError(const char* errMsg, int errNum) const { - logging::Logger logger(31); - logging::Message::Args args; - logging::LoggingID li(31); - args.add(errMsg); - args.add(strerror(errNum)); - args.add(toString()); + logging::Logger logger(31); + logging::Message::Args args; + logging::LoggingID li(31); + args.add(errMsg); + args.add(strerror(errNum)); + args.add(toString()); - logging::MsgMap msgMap; - msgMap[logging::M0071] = logging::Message( logging::M0071 ); - logger.msgMap(msgMap); + logging::MsgMap msgMap; + msgMap[logging::M0071] = logging::Message(logging::M0071); + logger.msgMap(msgMap); - logger.logMessage(logging::LOG_TYPE_WARNING, logging::M0071, args, li); + logger.logMessage(logging::LOG_TYPE_WARNING, logging::M0071, args, li); } ssize_t InetStreamSocket::written(int fd, const uint8_t* ptr, size_t nbytes) const { - size_t nleft; - ssize_t nwritten; - const char* bufp; + size_t nleft; + ssize_t nwritten; + const char* bufp; - nleft = nbytes; - bufp = reinterpret_cast(ptr); + nleft = nbytes; + bufp = reinterpret_cast(ptr); - while (nleft > 0) - { - // the O_NONBLOCK flag is not set, this is a blocking I/O. + while (nleft > 0) + { + // the O_NONBLOCK flag is not set, this is a blocking I/O. #ifdef _MSC_VER - int writeAmount = std::min((int)nleft, MaxSendPacketSize); + int writeAmount = std::min((int)nleft, MaxSendPacketSize); - if ((nwritten = ::send(fd, bufp, writeAmount, 0)) < 0) + if ((nwritten = ::send(fd, bufp, writeAmount, 0)) < 0) #else - if ((nwritten = ::write(fd, bufp, nleft)) < 0) + if ((nwritten = ::write(fd, bufp, nleft)) < 0) #endif - { - if (errno == EINTR) - nwritten = 0; - else - { - // save the error no first - int e = errno; - string errorMsg = "InetStreamSocket::write error: "; - scoped_array buf(new char[80]); + { + if (errno == EINTR) + nwritten = 0; + else + { + // save the error no first + int e = errno; + string errorMsg = "InetStreamSocket::write error: "; + scoped_array buf(new char[80]); #if STRERROR_R_CHAR_P - const char* p; + const char* p; - if ((p = strerror_r(e, buf.get(), 80)) != 0) - errorMsg += p; + if ((p = strerror_r(e, buf.get(), 80)) != 0) + errorMsg += p; #else - int p; + int p; - if ((p = strerror_r(e, buf.get(), 80)) == 0) - errorMsg += buf.get(); + if ((p = strerror_r(e, buf.get(), 80)) == 0) + errorMsg += buf.get(); #endif - throw runtime_error(errorMsg); - } - } - - nleft -= nwritten; - bufp += nwritten; + throw runtime_error(errorMsg); + } } - return nbytes; + nleft -= nwritten; + bufp += nwritten; + } + + return nbytes; } const string InetStreamSocket::addr2String() const { - string s; + string s; #ifdef _MSC_VER - //This is documented to be thread-safe in Windows - s = inet_ntoa(fSa.sin_addr); + // This is documented to be thread-safe in Windows + s = inet_ntoa(fSa.sin_addr); #else - char dst[INET_ADDRSTRLEN]; - s = inet_ntop(AF_INET, &fSa.sin_addr, dst, INET_ADDRSTRLEN); + char dst[INET_ADDRSTRLEN]; + s = inet_ntop(AF_INET, &fSa.sin_addr, dst, INET_ADDRSTRLEN); #endif - return s; + return s; } bool InetStreamSocket::isSameAddr(const Socket* rhs) const { - const InetStreamSocket* issp = dynamic_cast(rhs); + const InetStreamSocket* issp = dynamic_cast(rhs); - if (!issp) return false; + if (!issp) + return false; - return (fSa.sin_addr.s_addr == issp->fSa.sin_addr.s_addr); + return (fSa.sin_addr.s_addr == issp->fSa.sin_addr.s_addr); } /*static*/ int InetStreamSocket::ping(const std::string& ipaddr, const struct timespec* timeout) { - sockaddr_in pingaddr; - memset(&pingaddr, 0, sizeof(pingaddr)); + sockaddr_in pingaddr; + memset(&pingaddr, 0, sizeof(pingaddr)); - if (inet_aton(ipaddr.c_str(), &pingaddr.sin_addr) == 0) - return -1; + if (inet_aton(ipaddr.c_str(), &pingaddr.sin_addr) == 0) + return -1; - long msecs = 30 * 1000; + long msecs = 30 * 1000; - if (timeout) - msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; + if (timeout) + msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; #ifndef _MSC_VER - int pingsock; - pingsock = ::socket(PF_INET, SOCK_RAW, IPPROTO_ICMP); + int pingsock; + pingsock = ::socket(PF_INET, SOCK_RAW, IPPROTO_ICMP); - if (pingsock < 0) - return -1; + if (pingsock < 0) + return -1; - ssize_t len = 0; - size_t pktlen = 0; - const size_t PktSize = 1024; - char pkt[PktSize]; - memset(pkt, 0, PktSize); - struct icmp* pingPktPtr = reinterpret_cast(pkt); + ssize_t len = 0; + size_t pktlen = 0; + const size_t PktSize = 1024; + char pkt[PktSize]; + memset(pkt, 0, PktSize); + struct icmp* pingPktPtr = reinterpret_cast(pkt); - pingPktPtr->icmp_type = ICMP_ECHO; - pingPktPtr->icmp_cksum = in_cksum(reinterpret_cast(pkt), PktSize); + pingPktPtr->icmp_type = ICMP_ECHO; + pingPktPtr->icmp_cksum = in_cksum(reinterpret_cast(pkt), PktSize); - pktlen = 56 + ICMP_MINLEN; - len = ::sendto(pingsock, pkt, pktlen, 0, reinterpret_cast(&pingaddr), - sizeof(pingaddr)); - - if (len < 0 || static_cast(len) != pktlen) - { - ::close(pingsock); - return -1; - } - - memset(pkt, 0, PktSize); - pktlen = PktSize; - - int pollrc = 0; - pollrc = pollConnection(pingsock, msecs); - - if (pollrc != 1) - { - ::close(pingsock); - return -1; - } - - len = ::recvfrom(pingsock, pkt, pktlen, 0, 0, 0); - - if (len < 76) - { - ::close(pingsock); - return -1; - } - - struct ip* iphdr = reinterpret_cast(pkt); - - pingPktPtr = reinterpret_cast(pkt + (iphdr->ip_hl << 2)); - - if (pingPktPtr->icmp_type != ICMP_ECHOREPLY) - { - ::close(pingsock); - return -1; - } + pktlen = 56 + ICMP_MINLEN; + len = ::sendto(pingsock, pkt, pktlen, 0, reinterpret_cast(&pingaddr), + sizeof(pingaddr)); + if (len < 0 || static_cast(len) != pktlen) + { ::close(pingsock); + return -1; + } -#else //Windows version - HANDLE icmpFile; - icmpFile = IcmpCreateFile(); + memset(pkt, 0, PktSize); + pktlen = PktSize; - if (icmpFile == INVALID_HANDLE_VALUE) - return -1; + int pollrc = 0; + pollrc = pollConnection(pingsock, msecs); - DWORD ret; - const size_t PingPktSize = 1024; - char rqd[PingPktSize]; - WORD rqs = PingPktSize; - char rpd[PingPktSize]; - DWORD rps = PingPktSize; + if (pollrc != 1) + { + ::close(pingsock); + return -1; + } - ZeroMemory(rqd, PingPktSize); - ZeroMemory(rpd, PingPktSize); + len = ::recvfrom(pingsock, pkt, pktlen, 0, 0, 0); - rqs = 64; + if (len < 76) + { + ::close(pingsock); + return -1; + } - ret = IcmpSendEcho(icmpFile, pingaddr.sin_addr.s_addr, rqd, rqs, 0, rpd, rps, msecs); + struct ip* iphdr = reinterpret_cast(pkt); - if (ret <= 0) - { - IcmpCloseHandle(icmpFile); - return -1; - } + pingPktPtr = reinterpret_cast(pkt + (iphdr->ip_hl << 2)); - PICMP_ECHO_REPLY echoReply = (PICMP_ECHO_REPLY)rpd; + if (pingPktPtr->icmp_type != ICMP_ECHOREPLY) + { + ::close(pingsock); + return -1; + } - if (echoReply->Status != IP_SUCCESS) - { - IcmpCloseHandle(icmpFile); - return -1; - } + ::close(pingsock); +#else // Windows version + HANDLE icmpFile; + icmpFile = IcmpCreateFile(); + + if (icmpFile == INVALID_HANDLE_VALUE) + return -1; + + DWORD ret; + const size_t PingPktSize = 1024; + char rqd[PingPktSize]; + WORD rqs = PingPktSize; + char rpd[PingPktSize]; + DWORD rps = PingPktSize; + + ZeroMemory(rqd, PingPktSize); + ZeroMemory(rpd, PingPktSize); + + rqs = 64; + + ret = IcmpSendEcho(icmpFile, pingaddr.sin_addr.s_addr, rqd, rqs, 0, rpd, rps, msecs); + + if (ret <= 0) + { IcmpCloseHandle(icmpFile); + return -1; + } + + PICMP_ECHO_REPLY echoReply = (PICMP_ECHO_REPLY)rpd; + + if (echoReply->Status != IP_SUCCESS) + { + IcmpCloseHandle(icmpFile); + return -1; + } + + IcmpCloseHandle(icmpFile); #endif - return 0; + return 0; } bool InetStreamSocket::isConnected() const { - int error = 0; - socklen_t len = sizeof(error); - int retval = getsockopt(fSocketParms.sd(), SOL_SOCKET, SO_ERROR, &error, &len); + int error = 0; + socklen_t len = sizeof(error); + int retval = getsockopt(fSocketParms.sd(), SOL_SOCKET, SO_ERROR, &error, &len); - if (error || retval) - return false; + if (error || retval) + return false; - struct pollfd pfd[1]; - pfd[0].fd = fSocketParms.sd(); - pfd[0].events = POLLIN; - pfd[0].revents = 0; + struct pollfd pfd[1]; + pfd[0].fd = fSocketParms.sd(); + pfd[0].events = POLLIN; + pfd[0].revents = 0; - error = poll(pfd, 1, 0); + error = poll(pfd, 1, 0); - if ((error < 0) || (pfd[0].revents & (POLLHUP | POLLNVAL | POLLERR))) - { - return false; - } + if ((error < 0) || (pfd[0].revents & (POLLHUP | POLLNVAL | POLLERR))) + { + return false; + } - return true; + return true; } bool InetStreamSocket::hasData() const { - int count; - char buf[1]; - ssize_t retval; - ioctl(fSocketParms.sd(), FIONREAD, &count); - - if (count) - return true; - - // EAGAIN | EWOULDBLOCK means the socket is clear. Anything else is data or error - retval = recv(fSocketParms.sd(), buf, 1, MSG_DONTWAIT); - - if (retval & (EAGAIN | EWOULDBLOCK)) - return false; + int count; + char buf[1]; + ssize_t retval; + ioctl(fSocketParms.sd(), FIONREAD, &count); + if (count) return true; + + // EAGAIN | EWOULDBLOCK means the socket is clear. Anything else is data or error + retval = recv(fSocketParms.sd(), buf, 1, MSG_DONTWAIT); + + if (retval & (EAGAIN | EWOULDBLOCK)) + return false; + + return true; } -} //namespace messageqcpp - +} // namespace messageqcpp diff --git a/utils/messageqcpp/inetstreamsocket.h b/utils/messageqcpp/inetstreamsocket.h index 81538177e..c29e50247 100644 --- a/utils/messageqcpp/inetstreamsocket.h +++ b/utils/messageqcpp/inetstreamsocket.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: inetstreamsocket.h 3632 2013-03-13 18:08:46Z pleblanc $ -* -* -***********************************************************************/ + * $Id: inetstreamsocket.h 3632 2013-03-13 18:08:46Z pleblanc $ + * + * + ***********************************************************************/ /** @file */ #ifndef MESSAGEQCPP_INETSTREAMSOCKET_H #define MESSAGEQCPP_INETSTREAMSOCKET_H @@ -56,223 +56,224 @@ const uint32_t COMPRESSED_BYTESTREAM_MAGIC = 0x14fbc138; */ class InetStreamSocket : public Socket { -public: - /** ctor - * - */ - explicit InetStreamSocket(size_t blocksize = ByteStream::BlockSize); + public: + /** ctor + * + */ + explicit InetStreamSocket(size_t blocksize = ByteStream::BlockSize); - /** dtor - * - */ - virtual ~InetStreamSocket(); + /** dtor + * + */ + virtual ~InetStreamSocket(); - /** copy ctor - * - */ - InetStreamSocket(const InetStreamSocket& rhs); + /** copy ctor + * + */ + InetStreamSocket(const InetStreamSocket& rhs); - /** assign op - * - */ - virtual InetStreamSocket& operator=(const InetStreamSocket& rhs); + /** assign op + * + */ + virtual InetStreamSocket& operator=(const InetStreamSocket& rhs); - /** fSocket mutator - * - */ - inline virtual void socketParms(const SocketParms& socket); + /** fSocket mutator + * + */ + inline virtual void socketParms(const SocketParms& socket); - /** fSocket accessor - * - */ - inline virtual const SocketParms socketParms() const; + /** fSocket accessor + * + */ + inline virtual const SocketParms socketParms() const; - /** sockaddr mutator - * - */ - inline virtual void sa(const sockaddr* sa); + /** sockaddr mutator + * + */ + inline virtual void sa(const sockaddr* sa); - /** call socket() to get a sd - * - */ - virtual void open(); + /** call socket() to get a sd + * + */ + virtual void open(); - /** close the sd - * - */ - virtual void close(); + /** close the sd + * + */ + virtual void close(); - /** test if this socket is open - * - */ - inline virtual bool isOpen() const; + /** test if this socket is open + * + */ + inline virtual bool isOpen() const; - /** read a message from the socket - * - * wait for and return a message from the socket. The deafult timeout waits forever. Note that - * eventhough struct timespec has nanosecond resolution, this method only has milisecond resolution. - * @warning If you specify a timeout, the stream can be corrupted in certain - * extreme circumstances. The circumstance: receiving a portion of the message - * followed by a timeout. If the rest of the message is ever received, it - * will be misinterpreted by the following read(). Symptom: The caller will - * receive an incomplete ByteStream - * (do try-catch around all ">>" operations to detect underflow). Mitigation: - * the caller should not perform another read(). Caller should close the connection. - * The behavior will be unpredictable and possibly fatal. - * @note A fix is being reviewed but this is low-priority. - */ - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, Stats* stats = NULL) const; + /** read a message from the socket + * + * wait for and return a message from the socket. The deafult timeout waits forever. Note that + * eventhough struct timespec has nanosecond resolution, this method only has milisecond resolution. + * @warning If you specify a timeout, the stream can be corrupted in certain + * extreme circumstances. The circumstance: receiving a portion of the message + * followed by a timeout. If the rest of the message is ever received, it + * will be misinterpreted by the following read(). Symptom: The caller will + * receive an incomplete ByteStream + * (do try-catch around all ">>" operations to detect underflow). Mitigation: + * the caller should not perform another read(). Caller should close the connection. + * The behavior will be unpredictable and possibly fatal. + * @note A fix is being reviewed but this is low-priority. + */ + virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, + Stats* stats = NULL) const; - /** write a message to the socket - * - * write a message to the socket - */ - virtual void write(const ByteStream& msg, Stats* stats = NULL); - virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const; + /** write a message to the socket + * + * write a message to the socket + */ + virtual void write(const ByteStream& msg, Stats* stats = NULL); + virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const; - /** this version of write takes ownership of the bytestream - */ - virtual void write(SBS msg, Stats* stats = NULL); + /** this version of write takes ownership of the bytestream + */ + virtual void write(SBS msg, Stats* stats = NULL); - /** bind to a port - * - */ - virtual void bind(const sockaddr* serv_addr); + /** bind to a port + * + */ + virtual void bind(const sockaddr* serv_addr); - /** listen for connections - * - */ - virtual void listen(int backlog = 5); + /** listen for connections + * + */ + virtual void listen(int backlog = 5); - /** return an (accepted) IOSocket ready for I/O - * - */ - virtual const IOSocket accept(const struct timespec* timeout = 0); + /** return an (accepted) IOSocket ready for I/O + * + */ + virtual const IOSocket accept(const struct timespec* timeout = 0); - /** connect to a server socket - * - */ - virtual void connect(const sockaddr* serv_addr); + /** connect to a server socket + * + */ + virtual void connect(const sockaddr* serv_addr); - /** dynamically allocate a copy of this object - * - */ - virtual Socket* clone() const; + /** dynamically allocate a copy of this object + * + */ + virtual Socket* clone() const; - /** get a string rep of the object - * - */ - virtual const std::string toString() const; + /** get a string rep of the object + * + */ + virtual const std::string toString() const; - /** set the connection timeout (in ms) - * - */ - virtual void connectionTimeout(const struct ::timespec* timeout) - { - if (timeout) fConnectionTimeout = *timeout; - } + /** set the connection timeout (in ms) + * + */ + virtual void connectionTimeout(const struct ::timespec* timeout) + { + if (timeout) + fConnectionTimeout = *timeout; + } - /** set the connection protocol to be synchronous - * - */ - virtual void syncProto(bool use) - { - fSyncProto = use; - } + /** set the connection protocol to be synchronous + * + */ + virtual void syncProto(bool use) + { + fSyncProto = use; + } - int getConnectionNum() const - { - return fSocketParms.sd(); - } + int getConnectionNum() const + { + return fSocketParms.sd(); + } - /* The caller needs to know when/if the remote closes the connection or sends data. - * Returns 0 on timeout, 1 if there is data to read, 2 if the connection was dropped. - * On error 3 is returned. - */ - static int pollConnection(int connectionNum, long msecs); + /* The caller needs to know when/if the remote closes the connection or sends data. + * Returns 0 on timeout, 1 if there is data to read, 2 if the connection was dropped. + * On error 3 is returned. + */ + static int pollConnection(int connectionNum, long msecs); - /** return the address as a string - * - */ - virtual const std::string addr2String() const; + /** return the address as a string + * + */ + virtual const std::string addr2String() const; - /** compare 2 addresses - * - */ - virtual bool isSameAddr(const Socket* rhs) const; + /** compare 2 addresses + * + */ + virtual bool isSameAddr(const Socket* rhs) const; - /** ping an ip address - * - */ - EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout = 0); + /** ping an ip address + * + */ + EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout = 0); - // Check if we are still connected - virtual bool isConnected() const; + // Check if we are still connected + virtual bool isConnected() const; - // Check if the socket still has data pending + // Check if the socket still has data pending - virtual bool hasData() const; + virtual bool hasData() const; - /* - * allow test suite access to private data for OOB test - */ - friend class ::MessageQTestSuite; + /* + * allow test suite access to private data for OOB test + */ + friend class ::MessageQTestSuite; -protected: - static const int KERR_ERESTARTSYS = 512; + protected: + static const int KERR_ERESTARTSYS = 512; - void logIoError(const char* errMsg, int errNum) const; + void logIoError(const char* errMsg, int errNum) const; - /** Empty the stream up to the beginning of the next ByteStream. - * - * Reads until the beginning of the next ByteStream is found. - * @param msecs An optional timeout value. - * @param residual Pass in an array of at least 8 bytes, on return it will contain - * the first bytes of the stream. - * @param reslen On return, it will contain the # of bytes in residual. - * @return true if the next byte in the stream is the beginning of a ByteStream, - * false otherwise. - */ - virtual bool readToMagic(long msecs, bool* isTimeOut, Stats* stats) const; + /** Empty the stream up to the beginning of the next ByteStream. + * + * Reads until the beginning of the next ByteStream is found. + * @param msecs An optional timeout value. + * @param residual Pass in an array of at least 8 bytes, on return it will contain + * the first bytes of the stream. + * @param reslen On return, it will contain the # of bytes in residual. + * @return true if the next byte in the stream is the beginning of a ByteStream, + * false otherwise. + */ + virtual bool readToMagic(long msecs, bool* isTimeOut, Stats* stats) const; - void do_write(const ByteStream& msg, uint32_t magic, Stats* stats = NULL) const; - ssize_t written(int fd, const uint8_t* ptr, size_t nbytes) const; - bool readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, const size_t numberOfBytes, - const struct ::timespec* timeout, bool* isTimeOut, Stats* stats, - int64_t msec) const; + void do_write(const ByteStream& msg, uint32_t magic, Stats* stats = NULL) const; + ssize_t written(int fd, const uint8_t* ptr, size_t nbytes) const; + bool readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, const size_t numberOfBytes, + const struct ::timespec* timeout, bool* isTimeOut, Stats* stats, int64_t msec) const; - SocketParms fSocketParms; /// The socket parms - size_t fBlocksize; - sockaddr_in fSa; + SocketParms fSocketParms; /// The socket parms + size_t fBlocksize; + sockaddr_in fSa; - // how long to wait for a connect() call to complete (in ms) - struct ::timespec fConnectionTimeout; + // how long to wait for a connect() call to complete (in ms) + struct ::timespec fConnectionTimeout; - // use sync proto - bool fSyncProto; + // use sync proto + bool fSyncProto; - /// The buffer used to scan for the ByteStream magic in the stream. - mutable uint32_t fMagicBuffer; + /// The buffer used to scan for the ByteStream magic in the stream. + mutable uint32_t fMagicBuffer; -private: - void doCopy(const InetStreamSocket& rhs); + private: + void doCopy(const InetStreamSocket& rhs); }; inline bool InetStreamSocket::isOpen() const { - return (fSocketParms.sd() >= 0); + return (fSocketParms.sd() >= 0); } inline const SocketParms InetStreamSocket::socketParms() const { - return fSocketParms; + return fSocketParms; } inline void InetStreamSocket::socketParms(const SocketParms& socketParms) { - fSocketParms = socketParms; + fSocketParms = socketParms; } inline void InetStreamSocket::sa(const sockaddr* sa) { - memcpy(&fSa, sa, sizeof(sockaddr_in)); + memcpy(&fSa, sa, sizeof(sockaddr_in)); } /** @@ -280,13 +281,12 @@ inline void InetStreamSocket::sa(const sockaddr* sa) */ inline std::ostream& operator<<(std::ostream& os, const InetStreamSocket& rhs) { - os << rhs.toString(); - return os; + os << rhs.toString(); + return os; } -} //namespace messageqcpp +} // namespace messageqcpp #undef EXPORT -#endif //MESSAGEQCPP_INETSTREAMSOCKET_H - +#endif // MESSAGEQCPP_INETSTREAMSOCKET_H diff --git a/utils/messageqcpp/iosocket.cpp b/utils/messageqcpp/iosocket.cpp index 971f71f38..16bedf54f 100644 --- a/utils/messageqcpp/iosocket.cpp +++ b/utils/messageqcpp/iosocket.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: iosocket.cpp 3632 2013-03-13 18:08:46Z pleblanc $ -* -* -***********************************************************************/ + * $Id: iosocket.cpp 3632 2013-03-13 18:08:46Z pleblanc $ + * + * + ***********************************************************************/ #include #include #include @@ -42,58 +42,55 @@ using namespace std; namespace messageqcpp { - -IOSocket::IOSocket(Socket* socket) : - fSocket(socket), sockID(0) +IOSocket::IOSocket(Socket* socket) : fSocket(socket), sockID(0) { - memset(&fSa, 0, sizeof(fSa)); + memset(&fSa, 0, sizeof(fSa)); } IOSocket::~IOSocket() { - delete fSocket; + delete fSocket; } void IOSocket::doCopy(const IOSocket& rhs) { - fSocket = rhs.fSocket->clone(); - fSa = rhs.fSa; - sockID = rhs.sockID; + fSocket = rhs.fSocket->clone(); + fSa = rhs.fSa; + sockID = rhs.sockID; } IOSocket::IOSocket(const IOSocket& rhs) { - doCopy(rhs); + doCopy(rhs); } IOSocket& IOSocket::operator=(const IOSocket& rhs) { - if (this != &rhs) - { - delete fSocket; - doCopy(rhs); - } + if (this != &rhs) + { + delete fSocket; + doCopy(rhs); + } - return *this; + return *this; } const string IOSocket::toString() const { #ifdef NOSSTREAM - return "IOSocket"; + return "IOSocket"; #else - ostringstream oss; - char buf[INET_ADDRSTRLEN]; - SocketParms sp = fSocket->socketParms(); - const sockaddr_in* sinp = reinterpret_cast(&fSa); - oss << "IOSocket: sd: " << sp.sd() << + ostringstream oss; + char buf[INET_ADDRSTRLEN]; + SocketParms sp = fSocket->socketParms(); + const sockaddr_in* sinp = reinterpret_cast(&fSa); + oss << "IOSocket: sd: " << sp.sd() << #ifndef _MSC_VER - " inet: " << inet_ntop(AF_INET, &sinp->sin_addr, buf, INET_ADDRSTRLEN) << + " inet: " << inet_ntop(AF_INET, &sinp->sin_addr, buf, INET_ADDRSTRLEN) << #endif - " port: " << ntohs(sinp->sin_port); - return oss.str(); + " port: " << ntohs(sinp->sin_port); + return oss.str(); #endif } -} //namespace messageqcpp - +} // namespace messageqcpp diff --git a/utils/messageqcpp/iosocket.h b/utils/messageqcpp/iosocket.h index 6ec946c81..21dce5573 100644 --- a/utils/messageqcpp/iosocket.h +++ b/utils/messageqcpp/iosocket.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: iosocket.h 3632 2013-03-13 18:08:46Z pleblanc $ -* -* -***********************************************************************/ + * $Id: iosocket.h 3632 2013-03-13 18:08:46Z pleblanc $ + * + * + ***********************************************************************/ /** @file */ #ifndef MESSAGEQCPP_IOSOCKET_H #define MESSAGEQCPP_IOSOCKET_H @@ -61,238 +61,237 @@ class ServerSocket; */ class IOSocket { -public: + public: + /** ctor + * + */ + EXPORT explicit IOSocket(Socket* socket = 0); - /** ctor - * - */ - EXPORT explicit IOSocket(Socket* socket = 0); + /** copy ctor + * + */ + EXPORT IOSocket(const IOSocket& rhs); - /** copy ctor - * - */ - EXPORT IOSocket(const IOSocket& rhs); + /** assign op + * + */ + EXPORT IOSocket& operator=(const IOSocket& rhs); - /** assign op - * - */ - EXPORT IOSocket& operator=(const IOSocket& rhs); + /** dtor + * + */ + EXPORT virtual ~IOSocket(); - /** dtor - * - */ - EXPORT virtual ~IOSocket(); + /** read a ByteStream from this socket + * + * This socket needs to be connected first. Will throw runtime_error on I/O error. Caller should + * call close() method if exception is thrown. + */ + virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, + Stats* stats = NULL) const; - /** read a ByteStream from this socket - * - * This socket needs to be connected first. Will throw runtime_error on I/O error. Caller should - * call close() method if exception is thrown. - */ - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, Stats* stats = NULL) const; + /** write a ByteStream to this socket + * + * This socket needs to be connected first. Will throw runtime_error on I/O error. Caller should + * call close() method if exception is thrown. + */ + EXPORT virtual void write(const ByteStream& msg, Stats* stats = NULL) const; + EXPORT virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const; + EXPORT virtual void write(SBS msg, Stats* stats = NULL) const; - /** write a ByteStream to this socket - * - * This socket needs to be connected first. Will throw runtime_error on I/O error. Caller should - * call close() method if exception is thrown. - */ - EXPORT virtual void write(const ByteStream& msg, Stats* stats = NULL) const; - EXPORT virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const; - EXPORT virtual void write(SBS msg, Stats* stats = NULL) const; + /** access the sockaddr member + */ + inline virtual const sockaddr sa() const; - /** access the sockaddr member - */ - inline virtual const sockaddr sa() const; + /** modify the sockaddr member + */ + inline virtual void sa(const sockaddr* sa); - /** modify the sockaddr member - */ - inline virtual void sa(const sockaddr* sa); + /** open the socket + * + */ + inline virtual void open(); - /** open the socket - * - */ - inline virtual void open(); + /** close the socket + * + */ + inline virtual void close(); - /** close the socket - * - */ - inline virtual void close(); + /** test if the socket is open + * + */ + inline virtual bool isOpen() const; - /** test if the socket is open - * - */ - inline virtual bool isOpen() const; + /** get the socket params + * + */ + inline virtual const SocketParms socketParms() const; - /** get the socket params - * - */ - inline virtual const SocketParms socketParms() const; + /** set the socket params + * + */ + inline virtual void socketParms(const SocketParms& socketParms); - /** set the socket params - * - */ - inline virtual void socketParms(const SocketParms& socketParms); + /** set the socket implementation + * + * Install a socket implementation that meets the Socket interface + */ + EXPORT virtual void setSocketImpl(Socket* socket); - /** set the socket implementation - * - * Install a socket implementation that meets the Socket interface - */ - EXPORT virtual void setSocketImpl(Socket* socket); + /** get a string rep of the IOSocket + * + */ + EXPORT virtual const std::string toString() const; - /** get a string rep of the IOSocket - * - */ - EXPORT virtual const std::string toString() const; + /** syncProto() forwarder for inherited classes + * + */ + EXPORT virtual void syncProto(bool use) + { + fSocket->syncProto(use); + } - /** syncProto() forwarder for inherited classes - * - */ - EXPORT virtual void syncProto(bool use) - { - fSocket->syncProto(use); - } + EXPORT virtual int getConnectionNum() const; - EXPORT virtual int getConnectionNum() const; + // Debug + EXPORT void setSockID(uint32_t id) + { + sockID = id; + } + EXPORT uint32_t getSockID() + { + return sockID; + } + /* + * allow test suite access to private data for OOB test + */ + /** + * @brief return the address as a string + */ + virtual const std::string addr2String() const + { + return fSocket->addr2String(); + } - // Debug - EXPORT void setSockID(uint32_t id) - { - sockID = id; - } - EXPORT uint32_t getSockID() - { - return sockID; - } - /* - * allow test suite access to private data for OOB test - */ - /** - * @brief return the address as a string - */ - virtual const std::string addr2String() const - { - return fSocket->addr2String(); - } + /** + * @brief compare 2 addresses + */ + virtual bool isSameAddr(const IOSocket* rhs) const + { + return fSocket->isSameAddr(rhs->fSocket); + } - /** - * @brief compare 2 addresses - */ - virtual bool isSameAddr(const IOSocket* rhs) const - { - return fSocket->isSameAddr(rhs->fSocket); - } + /** connect() forwarder for inherited classes + * + */ + virtual void connect(const struct sockaddr* serv_addr) + { + fSocket->connect(serv_addr); + } - /** connect() forwarder for inherited classes - * - */ - virtual void connect(const struct sockaddr* serv_addr) - { - fSocket->connect(serv_addr); - } + /** connectionTimeout() forwarder for inherited classes + * + */ + virtual void connectionTimeout(const struct timespec* timeout) + { + fSocket->connectionTimeout(timeout); + } - /** connectionTimeout() forwarder for inherited classes - * - */ - virtual void connectionTimeout(const struct timespec* timeout) - { - fSocket->connectionTimeout(timeout); - } + inline virtual bool isConnected() const; + inline virtual bool hasData() const; + inline bool hasSocketDescriptor() const; - inline virtual bool isConnected() const; - inline virtual bool hasData() const; - inline bool hasSocketDescriptor() const; + friend class ::MessageQTestSuite; - friend class ::MessageQTestSuite; + protected: + private: + void doCopy(const IOSocket& rhs); -protected: - - -private: - void doCopy(const IOSocket& rhs); - - Socket* fSocket; - sockaddr fSa; - uint32_t sockID; // For debug purposes + Socket* fSocket; + sockaddr fSa; + uint32_t sockID; // For debug purposes }; - inline const sockaddr IOSocket::sa() const { - return fSa; + return fSa; } inline bool IOSocket::hasSocketDescriptor() const { - return fSocket && utils::is_nonnegative(fSocket->socketParms().sd()); + return fSocket && utils::is_nonnegative(fSocket->socketParms().sd()); } inline void IOSocket::sa(const sockaddr* sa) { - fSa = *sa; + fSa = *sa; - if (fSocket) - fSocket->sa( sa ); + if (fSocket) + fSocket->sa(sa); } inline void IOSocket::open() { - idbassert(fSocket); - fSocket->open(); + idbassert(fSocket); + fSocket->open(); } -//RJD: changing close() to simply bail on null fSocket. I'm not really sure what's best here, but this is probably +// RJD: changing close() to simply bail on null fSocket. I'm not really sure what's best here, but this is +// probably // better that asserting... inline void IOSocket::close() { - if (fSocket) fSocket->close(); + if (fSocket) + fSocket->close(); } inline bool IOSocket::isOpen() const { - return (fSocket && fSocket->isOpen()); + return (fSocket && fSocket->isOpen()); } inline const SBS IOSocket::read(const struct timespec* timeout, bool* isTimeOut, Stats* stats) const { - idbassert(fSocket); - return fSocket->read(timeout, isTimeOut, stats); + idbassert(fSocket); + return fSocket->read(timeout, isTimeOut, stats); } inline void IOSocket::write(const ByteStream& msg, Stats* stats) const { - idbassert(fSocket); - fSocket->write(msg, stats); + idbassert(fSocket); + fSocket->write(msg, stats); } inline void IOSocket::write_raw(const ByteStream& msg, Stats* stats) const { - idbassert(fSocket); - fSocket->write_raw(msg, stats); + idbassert(fSocket); + fSocket->write_raw(msg, stats); } inline void IOSocket::write(SBS msg, Stats* stats) const { - idbassert(fSocket); - fSocket->write(msg, stats); + idbassert(fSocket); + fSocket->write(msg, stats); } inline const SocketParms IOSocket::socketParms() const { - idbassert(fSocket); - return fSocket->socketParms(); + idbassert(fSocket); + return fSocket->socketParms(); } inline void IOSocket::socketParms(const SocketParms& socketParms) { - idbassert(fSocket); - fSocket->socketParms(socketParms); + idbassert(fSocket); + fSocket->socketParms(socketParms); } inline void IOSocket::setSocketImpl(Socket* socket) { - delete fSocket; - fSocket = socket; + delete fSocket; + fSocket = socket; } inline int IOSocket::getConnectionNum() const { - return fSocket->getConnectionNum(); + return fSocket->getConnectionNum(); } inline bool IOSocket::isConnected() const { - return fSocket->isConnected(); + return fSocket->isConnected(); } inline bool IOSocket::hasData() const { - return fSocket->hasData(); + return fSocket->hasData(); } /** @@ -300,13 +299,12 @@ inline bool IOSocket::hasData() const */ inline std::ostream& operator<<(std::ostream& os, const IOSocket& rhs) { - os << rhs.toString(); - return os; + os << rhs.toString(); + return os; } -} //namespace messageqcpp +} // namespace messageqcpp #undef EXPORT -#endif //MESSAGEQCPP_IOSOCKET_H - +#endif // MESSAGEQCPP_IOSOCKET_H diff --git a/utils/messageqcpp/messagequeue.cpp b/utils/messageqcpp/messagequeue.cpp index 4bd3e794a..e168949dc 100644 --- a/utils/messageqcpp/messagequeue.cpp +++ b/utils/messageqcpp/messagequeue.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: messagequeue.cpp 3632 2013-03-13 18:08:46Z pleblanc $ -* -* -***********************************************************************/ + * $Id: messagequeue.cpp 3632 2013-03-13 18:08:46Z pleblanc $ + * + * + ***********************************************************************/ #include #include #include @@ -55,327 +55,323 @@ using namespace config; namespace messageqcpp { // Aux function to try to resolve supplied identifier to fill struct addrinfo. -struct sockaddr* hostnameResolver(const std::string& dnOrIp, - const uint16_t port, - logging::Logger& aLogger, +struct sockaddr* hostnameResolver(const std::string& dnOrIp, const uint16_t port, logging::Logger& aLogger, struct sockaddr* sockAddrPtr) { - struct addrinfo hints; - struct addrinfo *servinfo; - int rc = 0; + struct addrinfo hints; + struct addrinfo* servinfo; + int rc = 0; - memset(&hints, 0, sizeof hints); - // ATM We support IPv4 only. - hints.ai_family = AF_INET; - hints.ai_socktype = SOCK_STREAM; + memset(&hints, 0, sizeof hints); + // ATM We support IPv4 only. + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; - if( !(rc = getaddrinfo(dnOrIp.c_str(), nullptr, &hints, &servinfo)) ) - { - memset(sockAddrPtr, 0, sizeof(*sockAddrPtr)); - sockaddr_in* sinp = reinterpret_cast(sockAddrPtr); - *sinp = *reinterpret_cast(servinfo->ai_addr); - sinp->sin_port = htons(port); - freeaddrinfo(servinfo); - } - else - { - string msg = "messageqcpp::hostnameResolver "; - msg.append(gai_strerror(rc)); - logging::Message::Args args; - logging::LoggingID li(31); - args.add(msg); - aLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li); - } - return sockAddrPtr; + if (!(rc = getaddrinfo(dnOrIp.c_str(), nullptr, &hints, &servinfo))) + { + memset(sockAddrPtr, 0, sizeof(*sockAddrPtr)); + sockaddr_in* sinp = reinterpret_cast(sockAddrPtr); + *sinp = *reinterpret_cast(servinfo->ai_addr); + sinp->sin_port = htons(port); + freeaddrinfo(servinfo); + } + else + { + string msg = "messageqcpp::hostnameResolver "; + msg.append(gai_strerror(rc)); + logging::Message::Args args; + logging::LoggingID li(31); + args.add(msg); + aLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li); + } + return sockAddrPtr; } void MessageQueueServer::setup(size_t blocksize, int backlog, bool syncProto) { - string thisEndPortStr; + string thisEndPortStr; - thisEndPortStr = fConfig->getConfig(fThisEnd, "Port"); - uint16_t port; + thisEndPortStr = fConfig->getConfig(fThisEnd, "Port"); + uint16_t port; - if (thisEndPortStr.length() == 0 || (port = static_cast(strtol(thisEndPortStr.c_str(), 0, 0))) == 0) - { - string msg = "MessageQueueServer::MessageQueueServer: config error: Invalid/Missing Port " - "attribute for " + fThisEnd; - throw runtime_error(msg); - } + if (thisEndPortStr.length() == 0 || + (port = static_cast(strtol(thisEndPortStr.c_str(), 0, 0))) == 0) + { + string msg = + "MessageQueueServer::MessageQueueServer: config error: Invalid/Missing Port " + "attribute for " + + fThisEnd; + throw runtime_error(msg); + } - in_addr listenAddr; - listenAddr.s_addr = INADDR_ANY; - string listenAddrStr = fConfig->getConfig(fThisEnd, "ListenAddr"); + in_addr listenAddr; + listenAddr.s_addr = INADDR_ANY; + string listenAddrStr = fConfig->getConfig(fThisEnd, "ListenAddr"); - if (listenAddrStr.length() > 0) - { - struct in_addr la; + if (listenAddrStr.length() > 0) + { + struct in_addr la; - if (inet_aton(listenAddrStr.c_str(), &la) != 0) - listenAddr = la; - } + if (inet_aton(listenAddrStr.c_str(), &la) != 0) + listenAddr = la; + } - memset(&fServ_addr, 0, sizeof(fServ_addr)); - sockaddr_in* sinp = reinterpret_cast(&fServ_addr); - sinp->sin_family = AF_INET; - sinp->sin_addr.s_addr = listenAddr.s_addr; - sinp->sin_port = htons(port); + memset(&fServ_addr, 0, sizeof(fServ_addr)); + sockaddr_in* sinp = reinterpret_cast(&fServ_addr); + sinp->sin_family = AF_INET; + sinp->sin_addr.s_addr = listenAddr.s_addr; + sinp->sin_port = htons(port); #ifdef SKIP_IDB_COMPRESSION - fListenSock.setSocketImpl(new InetStreamSocket(blocksize)); + fListenSock.setSocketImpl(new InetStreamSocket(blocksize)); #else - fListenSock.setSocketImpl(new CompressedInetStreamSocket()); + fListenSock.setSocketImpl(new CompressedInetStreamSocket()); #endif - fListenSock.syncProto(syncProto); - fListenSock.open(); - fListenSock.bind(&fServ_addr); - fListenSock.listen(backlog); + fListenSock.syncProto(syncProto); + fListenSock.open(); + fListenSock.bind(&fServ_addr); + fListenSock.listen(backlog); #ifdef SKIP_IDB_COMPRESSION - fClientSock.setSocketImpl(new InetStreamSocket(blocksize)); + fClientSock.setSocketImpl(new InetStreamSocket(blocksize)); #else - fClientSock.setSocketImpl(new CompressedInetStreamSocket()); + fClientSock.setSocketImpl(new CompressedInetStreamSocket()); #endif - fClientSock.syncProto(syncProto); + fClientSock.syncProto(syncProto); } -MessageQueueServer::MessageQueueServer(const string& thisEnd, const string& config, - size_t blocksize, int backlog, bool syncProto) : - fThisEnd(thisEnd), - fConfig(Config::makeConfig(config)), - fLogger(31) +MessageQueueServer::MessageQueueServer(const string& thisEnd, const string& config, size_t blocksize, + int backlog, bool syncProto) + : fThisEnd(thisEnd), fConfig(Config::makeConfig(config)), fLogger(31) { - setup(blocksize, backlog, syncProto); + setup(blocksize, backlog, syncProto); } -MessageQueueServer::MessageQueueServer(const string& thisEnd, Config* config, - size_t blocksize, int backlog, bool syncProto) : - fThisEnd(thisEnd), - fConfig(config), - fLogger(31) +MessageQueueServer::MessageQueueServer(const string& thisEnd, Config* config, size_t blocksize, int backlog, + bool syncProto) + : fThisEnd(thisEnd), fConfig(config), fLogger(31) { - if (fConfig == 0) - fConfig = Config::makeConfig(); + if (fConfig == 0) + fConfig = Config::makeConfig(); - setup(blocksize, backlog, syncProto); + setup(blocksize, backlog, syncProto); } MessageQueueServer::~MessageQueueServer() { - fClientSock.close(); - fListenSock.close(); + fClientSock.close(); + fListenSock.close(); } const IOSocket MessageQueueServer::accept(const struct timespec* timeout) const { - return fListenSock.accept(timeout); + return fListenSock.accept(timeout); } void MessageQueueServer::syncProto(bool use) { - fListenSock.syncProto(use); - fClientSock.syncProto(use); + fListenSock.syncProto(use); + fClientSock.syncProto(use); } MessageQueueClient::~MessageQueueClient() { - fClientSock.close(); + fClientSock.close(); } void MessageQueueClient::shutdown() { - fClientSock.close(); + fClientSock.close(); } std::pair getAddressAndPort(config::Config* config, const std::string& fOtherEnd) { - std::string otherEndDnOrIPStr = config->getConfig(fOtherEnd, "IPAddr"); - std::string otherEndPortStr = config->getConfig(fOtherEnd, "Port"); - uint16_t port = otherEndPortStr.length() > 0 - ? static_cast(strtol(otherEndPortStr.c_str(), 0, 0)) - : 0; + std::string otherEndDnOrIPStr = config->getConfig(fOtherEnd, "IPAddr"); + std::string otherEndPortStr = config->getConfig(fOtherEnd, "Port"); + uint16_t port = + otherEndPortStr.length() > 0 ? static_cast(strtol(otherEndPortStr.c_str(), 0, 0)) : 0; - if (otherEndDnOrIPStr == "unassigned") - return {"0.0.0.0", port}; + if (otherEndDnOrIPStr == "unassigned") + return {"0.0.0.0", port}; - if (otherEndDnOrIPStr.empty()) - return {"127.0.0.1", port}; + if (otherEndDnOrIPStr.empty()) + return {"127.0.0.1", port}; - return {otherEndDnOrIPStr, port}; + return {otherEndDnOrIPStr, port}; } void MessageQueueClient::setup(bool syncProto) { - auto addressAndPort = getAddressAndPort(fConfig, fOtherEnd); - if (!addressAndPort.second) - { - string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute"; - throw runtime_error(msg); - } + auto addressAndPort = getAddressAndPort(fConfig, fOtherEnd); + if (!addressAndPort.second) + { + string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute"; + throw runtime_error(msg); + } #ifdef SKIP_IDB_COMPRESSION - fClientSock.setSocketImpl(new InetStreamSocket()); + fClientSock.setSocketImpl(new InetStreamSocket()); #else - fClientSock.setSocketImpl(new CompressedInetStreamSocket()); + fClientSock.setSocketImpl(new CompressedInetStreamSocket()); #endif - fClientSock.syncProto(syncProto); - fClientSock.sa(hostnameResolver(addressAndPort.first, addressAndPort.second, fLogger, &fServ_addr)); + fClientSock.syncProto(syncProto); + fClientSock.sa(hostnameResolver(addressAndPort.first, addressAndPort.second, fLogger, &fServ_addr)); } -MessageQueueClient::MessageQueueClient(const string& otherEnd, const string& config, bool syncProto) : - fOtherEnd(otherEnd), fConfig(Config::makeConfig(config)), fLogger(31), fIsAvailable(true) +MessageQueueClient::MessageQueueClient(const string& otherEnd, const string& config, bool syncProto) + : fOtherEnd(otherEnd), fConfig(Config::makeConfig(config)), fLogger(31), fIsAvailable(true) { - setup(syncProto); + setup(syncProto); } -MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, bool syncProto) : - fOtherEnd(otherEnd), fConfig(config), fLogger(31), fIsAvailable(true) +MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, bool syncProto) + : fOtherEnd(otherEnd), fConfig(config), fLogger(31), fIsAvailable(true) { - if (fConfig == 0) - fConfig = Config::makeConfig(); + if (fConfig == 0) + fConfig = Config::makeConfig(); - setup(syncProto); + setup(syncProto); } -MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) : - fLogger(31), fIsAvailable(true) +MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) + : fLogger(31), fIsAvailable(true) { #ifdef SKIP_IDB_COMPRESSION - fClientSock.setSocketImpl(new InetStreamSocket()); + fClientSock.setSocketImpl(new InetStreamSocket()); #else - fClientSock.setSocketImpl(new CompressedInetStreamSocket()); + fClientSock.setSocketImpl(new CompressedInetStreamSocket()); #endif - fClientSock.syncProto(syncProto); - fClientSock.sa(hostnameResolver(dnOrIp, port, fLogger, &fServ_addr)); + fClientSock.syncProto(syncProto); + fClientSock.sa(hostnameResolver(dnOrIp, port, fLogger, &fServ_addr)); } const SBS MessageQueueClient::read(const struct timespec* timeout, bool* isTimeOut, Stats* stats) const { - if (!fClientSock.isOpen()) - { - fClientSock.open(); - - try - { - fClientSock.connect(&fServ_addr); - } - catch (...) - { - fClientSock.close(); - throw; - } - } - - SBS res; + if (!fClientSock.isOpen()) + { + fClientSock.open(); try { - res = fClientSock.read(timeout, isTimeOut, stats); + fClientSock.connect(&fServ_addr); } - catch (runtime_error& re) + catch (...) { - // This is an I/O error from IOSocket::read() -// cerr << "MessageQueueClient::read: close socket for " << re.what() << endl; - logging::Message::Args args; - logging::LoggingID li(31); - args.add("Client read close socket for"); - args.add(re.what()); - fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); - fClientSock.close(); - throw; - } - catch (SocketClosed& e) - { -// cerr << "MessageQueueClient::read: close socket for " << e.what() << endl; - logging::Message::Args args; - logging::LoggingID li(31); - args.add("Client read close socket for"); - args.add(e.what()); - fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); - fClientSock.close(); - throw; + fClientSock.close(); + throw; } + } - return res; + SBS res; + + try + { + res = fClientSock.read(timeout, isTimeOut, stats); + } + catch (runtime_error& re) + { + // This is an I/O error from IOSocket::read() + // cerr << "MessageQueueClient::read: close socket for " << re.what() << endl; + logging::Message::Args args; + logging::LoggingID li(31); + args.add("Client read close socket for"); + args.add(re.what()); + fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); + fClientSock.close(); + throw; + } + catch (SocketClosed& e) + { + // cerr << "MessageQueueClient::read: close socket for " << e.what() << endl; + logging::Message::Args args; + logging::LoggingID li(31); + args.add("Client read close socket for"); + args.add(e.what()); + fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); + fClientSock.close(); + throw; + } + + return res; } void MessageQueueClient::write(const ByteStream& msg, const struct timespec* timeout, Stats* stats) const { - if (!fClientSock.isOpen()) - { - fClientSock.open(); - - try - { - fClientSock.connectionTimeout(timeout); - fClientSock.connect(&fServ_addr); - } - catch (...) - { - fClientSock.close(); - throw; - } - } + if (!fClientSock.isOpen()) + { + fClientSock.open(); try { - fClientSock.write(msg, stats); + fClientSock.connectionTimeout(timeout); + fClientSock.connect(&fServ_addr); } - catch (runtime_error& e) + catch (...) { - try - { - ostringstream oss; - oss << "MessageQueueClient::write: error writing " << msg.length() << " bytes to " - << fClientSock << ". Socket error was " << e.what() << endl; -// cerr << oss.str() << endl; - logging::Message::Args args; - logging::LoggingID li(31); - args.add(oss.str()); - fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); - } - catch (...) - { - } - - fClientSock.close(); - throw; + fClientSock.close(); + throw; } + } + + try + { + fClientSock.write(msg, stats); + } + catch (runtime_error& e) + { + try + { + ostringstream oss; + oss << "MessageQueueClient::write: error writing " << msg.length() << " bytes to " << fClientSock + << ". Socket error was " << e.what() << endl; + // cerr << oss.str() << endl; + logging::Message::Args args; + logging::LoggingID li(31); + args.add(oss.str()); + fLogger.logMessage(logging::LOG_TYPE_WARNING, logging::M0000, args, li); + } + catch (...) + { + } + + fClientSock.close(); + throw; + } } bool MessageQueueClient::connect() const { - if (!fClientSock.isOpen()) - { - fClientSock.open(); + if (!fClientSock.isOpen()) + { + fClientSock.open(); + try + { + fClientSock.connect(&fServ_addr); + } + catch (runtime_error& re) + { + string what = re.what(); + + if (what.find("Connection refused") != string::npos) + { try { - fClientSock.connect(&fServ_addr); - } - catch (runtime_error& re) - { - string what = re.what(); - - if (what.find("Connection refused") != string::npos) - { - try - { - fClientSock.close(); - } - catch (...) - { - } - } - else - throw; + fClientSock.close(); } catch (...) { - throw; } + } + else + throw; } + catch (...) + { + throw; + } + } - return fClientSock.isOpen(); + return fClientSock.isOpen(); } -}//namespace messageqcpp +} // namespace messageqcpp diff --git a/utils/messageqcpp/messagequeue.h b/utils/messageqcpp/messagequeue.h index 14ca00883..7d9cba826 100644 --- a/utils/messageqcpp/messagequeue.h +++ b/utils/messageqcpp/messagequeue.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: messagequeue.h 3632 2013-03-13 18:08:46Z pleblanc $ -* -* -******************************************************************************************/ + * $Id: messagequeue.h 3632 2013-03-13 18:08:46Z pleblanc $ + * + * + ******************************************************************************************/ /** @file */ #ifndef MESSAGEQCPP_MESSAGEQUEUE_H #define MESSAGEQCPP_MESSAGEQUEUE_H @@ -55,7 +55,6 @@ class MessageQTestSuite; namespace messageqcpp { - using AddrAndPortPair = std::pair; // utility f-s // Extracts a pair of address and port from the XML configuration. @@ -82,76 +81,77 @@ AddrAndPortPair getAddressAndPort(config::Config* config, const std::string& fOt */ class MessageQueueServer { -public: - /** - * @brief construct a server queue for thisEnd - * - * construct a server queue for thisEnd. Optionally specify a Config object to use. - */ - EXPORT explicit MessageQueueServer(const std::string& thisEnd, config::Config* config = 0, - size_t blocksize = ByteStream::BlockSize, int backlog = 5, bool syncProto = true); + public: + /** + * @brief construct a server queue for thisEnd + * + * construct a server queue for thisEnd. Optionally specify a Config object to use. + */ + EXPORT explicit MessageQueueServer(const std::string& thisEnd, config::Config* config = 0, + size_t blocksize = ByteStream::BlockSize, int backlog = 5, + bool syncProto = true); - /** - * @brief construct a server queue for thisEnd - * - * construct a server queue for thisEnd, specifying the name of a config file to use. - */ - EXPORT MessageQueueServer(const std::string& thisEnd, const std::string& config, - size_t blocksize = ByteStream::BlockSize, int backlog = 5, bool syncProto = true); + /** + * @brief construct a server queue for thisEnd + * + * construct a server queue for thisEnd, specifying the name of a config file to use. + */ + EXPORT MessageQueueServer(const std::string& thisEnd, const std::string& config, + size_t blocksize = ByteStream::BlockSize, int backlog = 5, bool syncProto = true); - /** - * @brief destructor - */ - EXPORT ~MessageQueueServer(); -// - /** - * @brief wait for a connection and return an IOSocket - * - * This method can be used by a main thread to wait for an incoming connection. The IOSocket - * that is returned can be passed to a thread to handle the socket connection. The main thread - * is then free to wait again for another connection. The IOSocket is already open and ready for - * read() and/or write(). The caller is responsible for calling close() when it is done. - */ - EXPORT const IOSocket accept(const struct timespec* timeout = 0) const; + /** + * @brief destructor + */ + EXPORT ~MessageQueueServer(); + // + /** + * @brief wait for a connection and return an IOSocket + * + * This method can be used by a main thread to wait for an incoming connection. The IOSocket + * that is returned can be passed to a thread to handle the socket connection. The main thread + * is then free to wait again for another connection. The IOSocket is already open and ready for + * read() and/or write(). The caller is responsible for calling close() when it is done. + */ + EXPORT const IOSocket accept(const struct timespec* timeout = 0) const; - /** - * @brief get a mutable pointer to the client IOSocket - */ - inline IOSocket& clientSock() const; + /** + * @brief get a mutable pointer to the client IOSocket + */ + inline IOSocket& clientSock() const; - /** - * @brief set the sync proto - */ - EXPORT void syncProto(bool use); + /** + * @brief set the sync proto + */ + EXPORT void syncProto(bool use); - /** - * allow test suite access to private data for OOB test - */ - friend class ::MessageQTestSuite; + /** + * allow test suite access to private data for OOB test + */ + friend class ::MessageQTestSuite; -private: - /** copy ctor - * - */ - MessageQueueServer(const MessageQueueServer& rhs); + private: + /** copy ctor + * + */ + MessageQueueServer(const MessageQueueServer& rhs); - /** assign op - * - */ - MessageQueueServer& operator=(const MessageQueueServer& rhs); + /** assign op + * + */ + MessageQueueServer& operator=(const MessageQueueServer& rhs); - /** ctor helper - * - */ - void setup(size_t blocksize, int backlog, bool syncProto); + /** ctor helper + * + */ + void setup(size_t blocksize, int backlog, bool syncProto); - std::string fThisEnd; /// the process name for this process - struct sockaddr fServ_addr; /// the addr of the server (may be this process) - config::Config* fConfig; /// config file has the IP addrs and port numbers - mutable ServerSocket fListenSock; /// the socket the server listens on for new connections - mutable IOSocket fClientSock; /// the socket connected to a client + std::string fThisEnd; /// the process name for this process + struct sockaddr fServ_addr; /// the addr of the server (may be this process) + config::Config* fConfig; /// config file has the IP addrs and port numbers + mutable ServerSocket fListenSock; /// the socket the server listens on for new connections + mutable IOSocket fClientSock; /// the socket connected to a client - mutable logging::Logger fLogger; + mutable logging::Logger fLogger; }; /** @@ -170,171 +170,172 @@ private: */ class MessageQueueClient { -public: - /** - * @brief construct a queue to otherEnd - * - * construct a queue from this process to otherEnd. Optionally specify a Config object to use. - */ - EXPORT explicit MessageQueueClient(const std::string& otherEnd, config::Config* config = 0, bool syncProto = true); + public: + /** + * @brief construct a queue to otherEnd + * + * construct a queue from this process to otherEnd. Optionally specify a Config object to use. + */ + EXPORT explicit MessageQueueClient(const std::string& otherEnd, config::Config* config = 0, + bool syncProto = true); - /** - * @brief construct a queue to otherEnd - * - * construct a queue from this process to otherEnd, specifying the name of a config file to use. - */ - EXPORT explicit MessageQueueClient(const std::string& otherEnd, const std::string& config, bool syncProto = true); + /** + * @brief construct a queue to otherEnd + * + * construct a queue from this process to otherEnd, specifying the name of a config file to use. + */ + EXPORT explicit MessageQueueClient(const std::string& otherEnd, const std::string& config, + bool syncProto = true); - /** - * @brief construct a queue to otherEnd - * - * construct a queue from this process to otherEnd on the given IP and Port. - */ - EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto=true); + /** + * @brief construct a queue to otherEnd + * + * construct a queue from this process to otherEnd on the given IP and Port. + */ + EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto = true); + /** + * @brief destructor + * + * calls shutdown() method. + */ + EXPORT ~MessageQueueClient(); - /** - * @brief destructor - * - * calls shutdown() method. - */ - EXPORT ~MessageQueueClient(); + /** + * @brief read a message from the queue + * + * wait for and return a message from otherEnd. The deafult timeout waits forever. Note that + * eventhough struct timespec has nanosecond resolution, this method only has milisecond resolution. + */ + EXPORT const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, + Stats* stats = NULL) const; - /** - * @brief read a message from the queue - * - * wait for and return a message from otherEnd. The deafult timeout waits forever. Note that - * eventhough struct timespec has nanosecond resolution, this method only has milisecond resolution. - */ - EXPORT const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, Stats* stats = NULL) const; + /** + * @brief write a message to the queue + * + * write a message to otherEnd. If the socket is not open, the timeout parm (in ms) will be used + * to establish a sync connection w/ the server + */ + EXPORT void write(const ByteStream& msg, const struct timespec* timeout = 0, Stats* stats = NULL) const; - /** - * @brief write a message to the queue - * - * write a message to otherEnd. If the socket is not open, the timeout parm (in ms) will be used - * to establish a sync connection w/ the server - */ - EXPORT void write(const ByteStream& msg, const struct timespec* timeout = 0, Stats* stats = NULL) const; + /** + * @brief shutdown the connection to the server + * + * indicate to the class that the user is done with the socket + * and the other class methods won't be used. + */ + EXPORT void shutdown(); - /** - * @brief shutdown the connection to the server - * - * indicate to the class that the user is done with the socket - * and the other class methods won't be used. - */ - EXPORT void shutdown(); + /** + * @brief connect to the server. Returns true if connection was successful. + * + * read() and write() automatically connect, but this method can be used to verify a server is listening + * before that. + */ + EXPORT bool connect() const; - /** - * @brief connect to the server. Returns true if connection was successful. - * - * read() and write() automatically connect, but this method can be used to verify a server is listening - * before that. - */ - EXPORT bool connect() const; + /** + * @brief accessors and mutators + */ + EXPORT const sockaddr serv_addr() const + { + return fServ_addr; + } + EXPORT const std::string otherEnd() const + { + return fOtherEnd; + } + EXPORT bool isAvailable() const + { + return fIsAvailable; + } + EXPORT void isAvailable(const bool isAvailable) + { + fIsAvailable = isAvailable; + } + EXPORT const std::string moduleName() const + { + return fModuleName; + } + EXPORT void moduleName(const std::string& moduleName) + { + fModuleName = moduleName; + } - /** - * @brief accessors and mutators - */ - EXPORT const sockaddr serv_addr() const - { - return fServ_addr; - } - EXPORT const std::string otherEnd() const - { - return fOtherEnd; - } - EXPORT bool isAvailable() const - { - return fIsAvailable; - } - EXPORT void isAvailable (const bool isAvailable) - { - fIsAvailable = isAvailable; - } - EXPORT const std::string moduleName() const - { - return fModuleName; - } - EXPORT void moduleName(const std::string& moduleName) - { - fModuleName = moduleName; - } + /** + * @brief set the sync proto + */ + inline void syncProto(bool use); - /** - * @brief set the sync proto - */ - inline void syncProto(bool use); + /** + * @brief return the address as a string + */ + inline const std::string addr2String() const; - /** - * @brief return the address as a string - */ - inline const std::string addr2String() const; + /** + * @brief compare the addresses of 2 MessageQueueClient + */ + inline bool isSameAddr(const MessageQueueClient& rhs) const; - /** - * @brief compare the addresses of 2 MessageQueueClient - */ - inline bool isSameAddr(const MessageQueueClient& rhs) const; + bool isConnected() + { + return fClientSock.isConnected(); + } - bool isConnected() - { - return fClientSock.isConnected(); - } + bool hasData() + { + return fClientSock.hasData(); + } + /* + * allow test suite access to private data for OOB test + */ + friend class ::MessageQTestSuite; - bool hasData() - { - return fClientSock.hasData(); - } - /* - * allow test suite access to private data for OOB test - */ - friend class ::MessageQTestSuite; + private: + /** copy ctor + * + */ + MessageQueueClient(const MessageQueueClient& rhs); -private: - /** copy ctor - * - */ - MessageQueueClient(const MessageQueueClient& rhs); + /** assign op + * + */ + MessageQueueClient& operator=(const MessageQueueClient& rhs); - /** assign op - * - */ - MessageQueueClient& operator=(const MessageQueueClient& rhs); + /** ctor helper + * + */ + void setup(bool syncProto); - /** ctor helper - * - */ - void setup(bool syncProto); - - std::string fOtherEnd; /// the process name for this process - struct sockaddr fServ_addr; /// the addr of the server (may be this process) - config::Config* fConfig; /// config file has the IP addrs and port numbers - mutable IOSocket fClientSock; /// the socket to communicate with the server - mutable logging::Logger fLogger; - bool fIsAvailable; - std::string fModuleName; + std::string fOtherEnd; /// the process name for this process + struct sockaddr fServ_addr; /// the addr of the server (may be this process) + config::Config* fConfig; /// config file has the IP addrs and port numbers + mutable IOSocket fClientSock; /// the socket to communicate with the server + mutable logging::Logger fLogger; + bool fIsAvailable; + std::string fModuleName; }; inline IOSocket& MessageQueueServer::clientSock() const { - return fClientSock; + return fClientSock; } inline const std::string MessageQueueClient::addr2String() const { - return fClientSock.addr2String(); + return fClientSock.addr2String(); } inline bool MessageQueueClient::isSameAddr(const MessageQueueClient& rhs) const { - return fClientSock.isSameAddr(&rhs.fClientSock); + return fClientSock.isSameAddr(&rhs.fClientSock); } inline void MessageQueueClient::syncProto(bool use) { - fClientSock.syncProto(use); + fClientSock.syncProto(use); } -} +} // namespace messageqcpp #undef EXPORT -#endif //MESSAGEQCPP_MESSAGEQUEUE_H +#endif // MESSAGEQCPP_MESSAGEQUEUE_H // vim:ts=4 sw=4: - diff --git a/utils/messageqcpp/messagequeuepool.cpp b/utils/messageqcpp/messagequeuepool.cpp index f3cbcb2bc..19667f7d6 100644 --- a/utils/messageqcpp/messagequeuepool.cpp +++ b/utils/messageqcpp/messagequeuepool.cpp @@ -24,7 +24,6 @@ namespace messageqcpp { - boost::mutex queueMutex; // Make linker happy std::multimap MessageQueueClientPool::clientMap; @@ -34,177 +33,175 @@ std::multimap MessageQueueClientPool::clientMap; static uint64_t TimeSpecToSeconds(struct timespec* ts) { - return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000; + return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000; } -MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &dnOrIp, uint64_t port) +MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port) { - boost::mutex::scoped_lock lock(queueMutex); + boost::mutex::scoped_lock lock(queueMutex); - std::ostringstream oss; - oss << dnOrIp << "_" << port; - std::string searchString = oss.str(); + std::ostringstream oss; + oss << dnOrIp << "_" << port; + std::string searchString = oss.str(); - MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(searchString); + MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(searchString); - // We found one, return it - if (returnClient != NULL) - { - return returnClient; - } + // We found one, return it + if (returnClient != NULL) + { + return returnClient; + } - // We didn't find one, create new one - ClientObject* newClientObject = new ClientObject(); - struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - uint64_t nowSeconds = TimeSpecToSeconds(&now); + // We didn't find one, create new one + ClientObject* newClientObject = new ClientObject(); + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + uint64_t nowSeconds = TimeSpecToSeconds(&now); - newClientObject->client = new MessageQueueClient(dnOrIp, port); - newClientObject->inUse = true; - newClientObject->lastUsed = nowSeconds; - clientMap.insert(std::pair(searchString, newClientObject)); - return newClientObject->client; + newClientObject->client = new MessageQueueClient(dnOrIp, port); + newClientObject->inUse = true; + newClientObject->lastUsed = nowSeconds; + clientMap.insert(std::pair(searchString, newClientObject)); + return newClientObject->client; } MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module) { - boost::mutex::scoped_lock lock(queueMutex); + boost::mutex::scoped_lock lock(queueMutex); - MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module); + MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module); - // We found one, return it - if (returnClient != NULL) - { - return returnClient; - } + // We found one, return it + if (returnClient != NULL) + { + return returnClient; + } - // We didn't find one, create new one - ClientObject* newClientObject = new ClientObject(); - struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - uint64_t nowSeconds = TimeSpecToSeconds(&now); + // We didn't find one, create new one + ClientObject* newClientObject = new ClientObject(); + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + uint64_t nowSeconds = TimeSpecToSeconds(&now); - newClientObject->client = new MessageQueueClient(module); - newClientObject->inUse = true; - newClientObject->lastUsed = nowSeconds; - clientMap.insert(std::pair(module, newClientObject)); - return newClientObject->client; + newClientObject->client = new MessageQueueClient(module); + newClientObject->inUse = true; + newClientObject->lastUsed = nowSeconds; + clientMap.insert(std::pair(module, newClientObject)); + return newClientObject->client; } MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search) { - struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - uint64_t nowSeconds = TimeSpecToSeconds(&now); - MessageQueueClient* returnClient = NULL; + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + uint64_t nowSeconds = TimeSpecToSeconds(&now); + MessageQueueClient* returnClient = NULL; - std::multimap::iterator it = clientMap.begin(); + std::multimap::iterator it = clientMap.begin(); - // Scan pool - while (it != clientMap.end()) + // Scan pool + while (it != clientMap.end()) + { + ClientObject* clientObject = it->second; + uint64_t elapsedTime = nowSeconds - clientObject->lastUsed; + + // If connection hasn't been used for MAX_IDLE_TIME we probably don't need it so drop it + // Don't drop in use connections that have been in use a long time + if ((elapsedTime >= MAX_IDLE_TIME) && (!clientObject->inUse)) { - ClientObject* clientObject = it->second; - uint64_t elapsedTime = nowSeconds - clientObject->lastUsed; - - // If connection hasn't been used for MAX_IDLE_TIME we probably don't need it so drop it - // Don't drop in use connections that have been in use a long time - if ((elapsedTime >= MAX_IDLE_TIME) && (!clientObject->inUse)) - { - delete clientObject->client; - delete clientObject; - // Do this so we don't invalidate current interator - std::multimap::iterator toDelete = it; - it++; - clientMap.erase(toDelete); - continue; - } - - if (!clientObject->inUse) - { - MessageQueueClient* client = clientObject->client; - - // If the unused socket isn't connected or has data pending read, destroy it - if (!client->isConnected() || client->hasData()) - { - delete client; - delete clientObject; - // Do this so we don't invalidate current interator - std::multimap::iterator toDelete = it; - it++; - clientMap.erase(toDelete); - continue; - } - - } - - // If connection matches store it for later, but keep scanning the pool for more timeout prunes - if (it->first.compare(search) == 0) - { - if ((returnClient == NULL) && (!clientObject->inUse)) - { - returnClient = clientObject->client; - clientObject->inUse = true; - return returnClient; - } - } - - it++; + delete clientObject->client; + delete clientObject; + // Do this so we don't invalidate current interator + std::multimap::iterator toDelete = it; + it++; + clientMap.erase(toDelete); + continue; } - return NULL; + if (!clientObject->inUse) + { + MessageQueueClient* client = clientObject->client; + + // If the unused socket isn't connected or has data pending read, destroy it + if (!client->isConnected() || client->hasData()) + { + delete client; + delete clientObject; + // Do this so we don't invalidate current interator + std::multimap::iterator toDelete = it; + it++; + clientMap.erase(toDelete); + continue; + } + } + + // If connection matches store it for later, but keep scanning the pool for more timeout prunes + if (it->first.compare(search) == 0) + { + if ((returnClient == NULL) && (!clientObject->inUse)) + { + returnClient = clientObject->client; + clientObject->inUse = true; + return returnClient; + } + } + + it++; + } + + return NULL; } void MessageQueueClientPool::releaseInstance(MessageQueueClient* client) { - // Scan pool for pointer and release - // Set the last used and mark as not in use + // Scan pool for pointer and release + // Set the last used and mark as not in use - if (client == NULL) - return; + if (client == NULL) + return; - boost::mutex::scoped_lock lock(queueMutex); - std::multimap::iterator it = clientMap.begin(); + boost::mutex::scoped_lock lock(queueMutex); + std::multimap::iterator it = clientMap.begin(); - while (it != clientMap.end()) + while (it != clientMap.end()) + { + if (it->second->client == client) { - if (it->second->client == client) - { - struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - uint64_t nowSeconds = TimeSpecToSeconds(&now); - it->second->inUse = false; - it->second->lastUsed = nowSeconds; - return; - } - - it++; + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + uint64_t nowSeconds = TimeSpecToSeconds(&now); + it->second->inUse = false; + it->second->lastUsed = nowSeconds; + return; } + + it++; + } } // WriteEngine needs this as it forces connections closed and can't reuse. Also good for connection errors void MessageQueueClientPool::deleteInstance(MessageQueueClient* client) { - // Scan pool for pointer and delete - // Set the last used and mark as not in use + // Scan pool for pointer and delete + // Set the last used and mark as not in use - if (client == NULL) - return; + if (client == NULL) + return; - boost::mutex::scoped_lock lock(queueMutex); - std::multimap::iterator it = clientMap.begin(); + boost::mutex::scoped_lock lock(queueMutex); + std::multimap::iterator it = clientMap.begin(); - while (it != clientMap.end()) + while (it != clientMap.end()) + { + if (it->second->client == client) { - if (it->second->client == client) - { - delete it->second->client; - delete it->second; - clientMap.erase(it); - return; - } - - it++; + delete it->second->client; + delete it->second; + clientMap.erase(it); + return; } + it++; + } } -} +} // namespace messageqcpp diff --git a/utils/messageqcpp/messagequeuepool.h b/utils/messageqcpp/messagequeuepool.h index 553328e7b..111de2038 100644 --- a/utils/messageqcpp/messagequeuepool.h +++ b/utils/messageqcpp/messagequeuepool.h @@ -23,36 +23,32 @@ namespace messageqcpp { - - struct ClientObject { - MessageQueueClient* client; - uint64_t lastUsed; - bool inUse; + MessageQueueClient* client; + uint64_t lastUsed; + bool inUse; - ClientObject() : - client(NULL), - lastUsed(0), - inUse(false) - {} + ClientObject() : client(NULL), lastUsed(0), inUse(false) + { + } }; class MessageQueueClientPool { -public: - static MessageQueueClient* getInstance(const std::string& module); - static MessageQueueClient *getInstance(const std::string &dnOrIp, uint64_t port); - static void releaseInstance(MessageQueueClient* client); - static void deleteInstance(MessageQueueClient* client); - static MessageQueueClient* findInPool(const std::string& search); + public: + static MessageQueueClient* getInstance(const std::string& module); + static MessageQueueClient* getInstance(const std::string& dnOrIp, uint64_t port); + static void releaseInstance(MessageQueueClient* client); + static void deleteInstance(MessageQueueClient* client); + static MessageQueueClient* findInPool(const std::string& search); -private: - MessageQueueClientPool() { }; - ~MessageQueueClientPool() { }; + private: + MessageQueueClientPool(){}; + ~MessageQueueClientPool(){}; - static std::multimap clientMap; + static std::multimap clientMap; }; -} -#endif //MESSAGEQCPP_MESSAGEQUEUECLIENT_H +} // namespace messageqcpp +#endif // MESSAGEQCPP_MESSAGEQUEUECLIENT_H diff --git a/utils/messageqcpp/serializeable.h b/utils/messageqcpp/serializeable.h index 2137f04ed..179c2fa3d 100644 --- a/utils/messageqcpp/serializeable.h +++ b/utils/messageqcpp/serializeable.h @@ -29,33 +29,32 @@ namespace messageqcpp { - class ByteStream; /** This is an abstract class that defines the interface ByteStream will use to serialize and deserialize your class. - To serialize an object, do 'ByteStream << object' - To deserialize an object, instantiate one of its type and do 'ByteStream >> object' + To serialize an object, do 'ByteStream << object' + To deserialize an object, instantiate one of its type and do 'ByteStream >> object' */ class Serializeable { -public: - /** dtor - * - */ - virtual ~Serializeable() { }; - /** serialize interface - * - */ - virtual void serialize(ByteStream&) const = 0; - /** deserialize interface - * - */ - virtual void deserialize(ByteStream&) = 0; + public: + /** dtor + * + */ + virtual ~Serializeable(){}; + /** serialize interface + * + */ + virtual void serialize(ByteStream&) const = 0; + /** deserialize interface + * + */ + virtual void deserialize(ByteStream&) = 0; }; -} +} // namespace messageqcpp #endif diff --git a/utils/messageqcpp/serversocket.h b/utils/messageqcpp/serversocket.h index 8e9c0979b..233110029 100644 --- a/utils/messageqcpp/serversocket.h +++ b/utils/messageqcpp/serversocket.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: serversocket.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: serversocket.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef MESSAGEQCPP_SERVERSOCKET_H #define MESSAGEQCPP_SERVERSOCKET_H @@ -47,129 +47,129 @@ class SocketParms; */ class ServerSocket { -public: + public: + /** ctor + * + */ + explicit ServerSocket(Socket* socket = 0) : fSocket(socket) + { + } - /** ctor - * - */ - explicit ServerSocket(Socket* socket = 0) : fSocket(socket) {} + /** dtor + * + */ + virtual ~ServerSocket() + { + delete fSocket; + } - /** dtor - * - */ - virtual ~ServerSocket() - { - delete fSocket; - } + /** bind to an port + * + * bind this ServerSocket to the address/port specified in serv_addr + */ + inline virtual void bind(const struct sockaddr* serv_addr); - /** bind to an port - * - * bind this ServerSocket to the address/port specified in serv_addr - */ - inline virtual void bind(const struct sockaddr* serv_addr); + /** setup to listen for incoming connections + * + */ + inline virtual void listen(int backlog = 5); - /** setup to listen for incoming connections - * - */ - inline virtual void listen(int backlog = 5); + /** accept an incoming connection + * + * accepts a new incoming connection and returns an IOSocket to communicate over + */ + inline virtual const IOSocket accept(const struct timespec* timeout = 0); - /** accept an incoming connection - * - * accepts a new incoming connection and returns an IOSocket to communicate over - */ - inline virtual const IOSocket accept(const struct timespec* timeout = 0); + /** open the socket + * + */ + inline virtual void open(); - /** open the socket - * - */ - inline virtual void open(); + /** close the socket + * + */ + inline virtual void close(); - /** close the socket - * - */ - inline virtual void close(); + /** test if the socket is open + * + */ + inline virtual bool isOpen() const; - /** test if the socket is open - * - */ - inline virtual bool isOpen() const; + /** get the socket params + * + */ + inline virtual const SocketParms socketParms() const; - /** get the socket params - * - */ - inline virtual const SocketParms socketParms() const; + /** set the socket params + * + */ + inline virtual void socketParms(const SocketParms& socketParms); - /** set the socket params - * - */ - inline virtual void socketParms(const SocketParms& socketParms); + /** set the socket implementation + * + * Install a socket implementation that meets the Socket interface + */ + inline virtual void setSocketImpl(Socket* socket); - /** set the socket implementation - * - * Install a socket implementation that meets the Socket interface - */ - inline virtual void setSocketImpl(Socket* socket); + /** set the socket sync proto + * + */ + inline virtual void syncProto(bool use); - /** set the socket sync proto - * - */ - inline virtual void syncProto(bool use); + /* + * allow test suite access to private data for OOB test + */ + friend class ::MessageQTestSuite; - /* - * allow test suite access to private data for OOB test - */ - friend class ::MessageQTestSuite; + private: + ServerSocket(const ServerSocket& rhs); + ServerSocket& operator=(const ServerSocket& rhs); -private: - ServerSocket(const ServerSocket& rhs); - ServerSocket& operator=(const ServerSocket& rhs); - - Socket* fSocket; + Socket* fSocket; }; inline void ServerSocket::bind(const struct sockaddr* serv_addr) { - fSocket->bind(serv_addr); + fSocket->bind(serv_addr); } inline void ServerSocket::listen(int backlog) { - fSocket->listen(backlog); + fSocket->listen(backlog); } inline const IOSocket ServerSocket::accept(const struct timespec* timeout) { - return fSocket->accept(timeout); + return fSocket->accept(timeout); } inline void ServerSocket::open() { - fSocket->open(); + fSocket->open(); } inline void ServerSocket::close() { - fSocket->close(); + fSocket->close(); } inline bool ServerSocket::isOpen() const { - return fSocket->isOpen(); + return fSocket->isOpen(); } inline const SocketParms ServerSocket::socketParms() const { - return fSocket->socketParms(); + return fSocket->socketParms(); } inline void ServerSocket::socketParms(const SocketParms& socketParms) { - fSocket->socketParms(socketParms); + fSocket->socketParms(socketParms); } inline void ServerSocket::setSocketImpl(Socket* socket) { - delete fSocket; - fSocket = socket; + delete fSocket; + fSocket = socket; } inline void ServerSocket::syncProto(bool use) { - fSocket->syncProto(use); + fSocket->syncProto(use); } -} //namespace messageqcpp - -#endif //MESSAGEQCPP_SERVERSOCKET_H +} // namespace messageqcpp +#endif // MESSAGEQCPP_SERVERSOCKET_H diff --git a/utils/messageqcpp/socket.h b/utils/messageqcpp/socket.h index 41c5e4628..60d5f34ff 100644 --- a/utils/messageqcpp/socket.h +++ b/utils/messageqcpp/socket.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: socket.h 3633 2013-03-13 20:50:23Z pleblanc $ -* -* -***********************************************************************/ + * $Id: socket.h 3633 2013-03-13 20:50:23Z pleblanc $ + * + * + ***********************************************************************/ /** @file */ #ifndef MESSAGEQCPP_SOCKET_H #define MESSAGEQCPP_SOCKET_H @@ -37,34 +37,37 @@ namespace messageqcpp class IOSocket; class SocketParms; - // Might want to expand on this / derive from it to include things like uncompressed // data size, etc... class Stats { -public: - Stats() : data_sent(0), data_recvd(0) - { } - virtual ~Stats() { } - virtual uint64_t dataSent() - { - return data_sent; - } - virtual uint64_t dataRecvd() - { - return data_recvd; - } - virtual void dataSent(uint64_t amt) - { - data_sent += amt; - } - virtual void dataRecvd(uint64_t amt) - { - data_recvd += amt; - } -private: - uint64_t data_sent; - uint64_t data_recvd; + public: + Stats() : data_sent(0), data_recvd(0) + { + } + virtual ~Stats() + { + } + virtual uint64_t dataSent() + { + return data_sent; + } + virtual uint64_t dataRecvd() + { + return data_recvd; + } + virtual void dataSent(uint64_t amt) + { + data_sent += amt; + } + virtual void dataRecvd(uint64_t amt) + { + data_recvd += amt; + } + + private: + uint64_t data_sent; + uint64_t data_recvd; }; /** an abstract socket class interface @@ -72,117 +75,116 @@ private: */ class Socket { -public: - /** dtor - * - */ - virtual ~Socket() {} + public: + /** dtor + * + */ + virtual ~Socket() + { + } - /** open the socket - * - */ - virtual void open() = 0; + /** open the socket + * + */ + virtual void open() = 0; - /** read a message from the socket - * - * wait for and return a message from the socket. The deafult timeout waits forever. Note that - * eventhough struct timespec has nanosecond resolution, this method only has millisecond resolution. - */ - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, Stats* stats = NULL) const = 0; + /** read a message from the socket + * + * wait for and return a message from the socket. The deafult timeout waits forever. Note that + * eventhough struct timespec has nanosecond resolution, this method only has millisecond resolution. + */ + virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, + Stats* stats = NULL) const = 0; - /** write a message to the socket - * - * write a message to the socket - */ - virtual void write(const ByteStream& msg, Stats* stats = NULL) = 0; - virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const = 0; - virtual void write(SBS msg, Stats* stats = NULL) = 0; + /** write a message to the socket + * + * write a message to the socket + */ + virtual void write(const ByteStream& msg, Stats* stats = NULL) = 0; + virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const = 0; + virtual void write(SBS msg, Stats* stats = NULL) = 0; - /** close the socket - * - */ - virtual void close() = 0; + /** close the socket + * + */ + virtual void close() = 0; - /** bind to a port - * - */ - virtual void bind(const struct sockaddr* serv_addr) = 0; + /** bind to a port + * + */ + virtual void bind(const struct sockaddr* serv_addr) = 0; - /** listen for connections - * - */ - virtual void listen(int backlog = 5) = 0; + /** listen for connections + * + */ + virtual void listen(int backlog = 5) = 0; - /** return an (accepted) IOSocket ready for I/O - * - */ - virtual const IOSocket accept(const struct timespec* timeout = 0) = 0; + /** return an (accepted) IOSocket ready for I/O + * + */ + virtual const IOSocket accept(const struct timespec* timeout = 0) = 0; - /** connect to a server socket - * - */ - virtual void connect(const sockaddr* serv_addr) = 0; + /** connect to a server socket + * + */ + virtual void connect(const sockaddr* serv_addr) = 0; - /** test if this socket is open - * - */ - virtual bool isOpen() const = 0; + /** test if this socket is open + * + */ + virtual bool isOpen() const = 0; - /** get the SocketParms - * - */ - virtual const SocketParms socketParms() const = 0; + /** get the SocketParms + * + */ + virtual const SocketParms socketParms() const = 0; - /** set the SocketParms - * - */ - virtual void socketParms(const SocketParms& socketParms) = 0; + /** set the SocketParms + * + */ + virtual void socketParms(const SocketParms& socketParms) = 0; - /** set the sockaddr struct - * - */ - virtual void sa(const sockaddr* sa) = 0; + /** set the sockaddr struct + * + */ + virtual void sa(const sockaddr* sa) = 0; - /** dynamically allocate a copy of this object - * - */ - virtual Socket* clone() const = 0; + /** dynamically allocate a copy of this object + * + */ + virtual Socket* clone() const = 0; - /** set the connection timeout (in ms) - * - */ - virtual void connectionTimeout(const struct ::timespec* timeout) = 0; + /** set the connection timeout (in ms) + * + */ + virtual void connectionTimeout(const struct ::timespec* timeout) = 0; - /** set the connection protocol to be synchronous - * - */ - virtual void syncProto(bool use) = 0; + /** set the connection protocol to be synchronous + * + */ + virtual void syncProto(bool use) = 0; - virtual int getConnectionNum() const = 0; + virtual int getConnectionNum() const = 0; - /** return the address as a string - * - */ - virtual const std::string addr2String() const = 0; + /** return the address as a string + * + */ + virtual const std::string addr2String() const = 0; - /** compare 2 addresses - * - */ - virtual bool isSameAddr(const Socket* rhs) const = 0; + /** compare 2 addresses + * + */ + virtual bool isSameAddr(const Socket* rhs) const = 0; - virtual bool isConnected() const = 0; - virtual bool hasData() const = 0; - - /* - * allow test suite access to private data for OOB test - */ - friend class ::MessageQTestSuite; + virtual bool isConnected() const = 0; + virtual bool hasData() const = 0; + /* + * allow test suite access to private data for OOB test + */ + friend class ::MessageQTestSuite; }; +} // namespace messageqcpp - -} //namespace messageqcpp - -#endif //MESSAGEQCPP_SOCKET_H - +#endif // MESSAGEQCPP_SOCKET_H diff --git a/utils/messageqcpp/socketclosed.h b/utils/messageqcpp/socketclosed.h index c2993edcd..62250e65b 100644 --- a/utils/messageqcpp/socketclosed.h +++ b/utils/messageqcpp/socketclosed.h @@ -30,32 +30,32 @@ namespace messageqcpp { - /** @brief A closed socket exception class -* -* Some sort of activity has been requested on a closed socket -*/ + * + * Some sort of activity has been requested on a closed socket + */ class SocketClosed : public std::exception { - std::string _M_msg; + std::string _M_msg; -public: - /** Takes a character string describing the error. */ - explicit - SocketClosed(const std::string& __arg) : _M_msg(__arg) { } + public: + /** Takes a character string describing the error. */ + explicit SocketClosed(const std::string& __arg) : _M_msg(__arg) + { + } - virtual - ~SocketClosed() throw() { } + virtual ~SocketClosed() throw() + { + } - /** Returns a C-style character string describing the general cause of - * the current error (the same string passed to the ctor). */ - virtual const char* - what() const throw() - { - return _M_msg.c_str(); - } + /** Returns a C-style character string describing the general cause of + * the current error (the same string passed to the ctor). */ + virtual const char* what() const throw() + { + return _M_msg.c_str(); + } }; -} +} // namespace messageqcpp #endif diff --git a/utils/messageqcpp/socketparms.cpp b/utils/messageqcpp/socketparms.cpp index 731a0522b..722bdfe11 100644 --- a/utils/messageqcpp/socketparms.cpp +++ b/utils/messageqcpp/socketparms.cpp @@ -16,18 +16,17 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: socketparms.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: socketparms.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ***********************************************************************/ #include "socketparms.h" namespace messageqcpp { - -SocketParms::SocketParms(int domain, int type, int protocol) : - fSd(-1), fDomain(domain), fType(type), fProtocol(protocol) +SocketParms::SocketParms(int domain, int type, int protocol) + : fSd(-1), fDomain(domain), fType(type), fProtocol(protocol) { } @@ -37,26 +36,25 @@ SocketParms::~SocketParms() void SocketParms::doCopy(const SocketParms& rhs) { - fSd = rhs.fSd; - fDomain = rhs.fDomain; - fType = rhs.fType; - fProtocol = rhs.fProtocol; + fSd = rhs.fSd; + fDomain = rhs.fDomain; + fType = rhs.fType; + fProtocol = rhs.fProtocol; } SocketParms::SocketParms(const SocketParms& rhs) { - doCopy(rhs); + doCopy(rhs); } SocketParms& SocketParms::operator=(const SocketParms& rhs) { - if (this != &rhs) - { - doCopy(rhs); - } - - return *this; -} + if (this != &rhs) + { + doCopy(rhs); + } + return *this; } +} // namespace messageqcpp diff --git a/utils/messageqcpp/socketparms.h b/utils/messageqcpp/socketparms.h index 9fe4cbadf..29d5a74f5 100644 --- a/utils/messageqcpp/socketparms.h +++ b/utils/messageqcpp/socketparms.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: socketparms.h 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: socketparms.h 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ***********************************************************************/ /** @file */ #ifndef MESSAGEQCPP_SOCKETPARMS_H #define MESSAGEQCPP_SOCKETPARMS_H @@ -28,130 +28,128 @@ class MessageQTestSuite; namespace messageqcpp { - /** a simple socket parameters class * */ class SocketParms { -public: - /** ctor - * - */ - explicit SocketParms(int domain = -1, int type = -1, int protocol = -1); + public: + /** ctor + * + */ + explicit SocketParms(int domain = -1, int type = -1, int protocol = -1); - /** dtor - * - */ - virtual ~SocketParms(); + /** dtor + * + */ + virtual ~SocketParms(); - /** copy ctor - * - */ - SocketParms(const SocketParms& rhs); + /** copy ctor + * + */ + SocketParms(const SocketParms& rhs); - /** assign op - * - */ - SocketParms& operator=(const SocketParms& rhs); + /** assign op + * + */ + SocketParms& operator=(const SocketParms& rhs); - /** accessor - * - */ - inline int sd() const; + /** accessor + * + */ + inline int sd() const; - /** accessor - * - */ - inline int domain() const; + /** accessor + * + */ + inline int domain() const; - /** accessor - * - */ - inline int type() const; + /** accessor + * + */ + inline int type() const; - /** accessor - * - */ - inline int protocol() const; + /** accessor + * + */ + inline int protocol() const; - /** mutator - * - */ - inline void sd(int sd); + /** mutator + * + */ + inline void sd(int sd); - /** mutator - * - */ - inline void domain(int domain); + /** mutator + * + */ + inline void domain(int domain); - /** mutator - * - */ - inline void type(int type); + /** mutator + * + */ + inline void type(int type); - /** mutator - * - */ - inline void protocol(int protocol); + /** mutator + * + */ + inline void protocol(int protocol); - /** isOpen test - * - */ - inline bool isOpen() const; + /** isOpen test + * + */ + inline bool isOpen() const; - /* - * allow test suite access to private data for OOB test - */ - friend class ::MessageQTestSuite; + /* + * allow test suite access to private data for OOB test + */ + friend class ::MessageQTestSuite; -private: - void doCopy(const SocketParms& rhs); + private: + void doCopy(const SocketParms& rhs); - int fSd; /// the socket descriptor - int fDomain; /// the socket domain - int fType; /// the socket type - int fProtocol; /// the socket protocol + int fSd; /// the socket descriptor + int fDomain; /// the socket domain + int fType; /// the socket type + int fProtocol; /// the socket protocol }; inline int SocketParms::sd() const { - return fSd; + return fSd; } inline int SocketParms::domain() const { - return fDomain; + return fDomain; } inline int SocketParms::type() const { - return fType; + return fType; } inline int SocketParms::protocol() const { - return fProtocol; + return fProtocol; } inline bool SocketParms::isOpen() const { - return (fSd >= 0); + return (fSd >= 0); } inline void SocketParms::sd(int sd) { - fSd = sd; + fSd = sd; } inline void SocketParms::domain(int domain) { - fDomain = domain; + fDomain = domain; } inline void SocketParms::type(int type) { - fType = type; + fType = type; } inline void SocketParms::protocol(int protocol) { - fProtocol = protocol; + fProtocol = protocol; } -} //namespace messageqcpp - -#endif //MESSAGEQCPP_SOCKETPARMS_H +} // namespace messageqcpp +#endif // MESSAGEQCPP_SOCKETPARMS_H diff --git a/utils/messageqcpp/srv.cpp b/utils/messageqcpp/srv.cpp index ccc919c27..f646428bf 100644 --- a/utils/messageqcpp/srv.cpp +++ b/utils/messageqcpp/srv.cpp @@ -9,33 +9,32 @@ using namespace config; int main(int argc, char** argv) { - Config* cf = Config::makeConfig("./Columnstore.xml"); - MessageQueueServer mqs("server1", cf); + Config* cf = Config::makeConfig("./Columnstore.xml"); + MessageQueueServer mqs("server1", cf); - cout << "server ready..." << endl; + cout << "server ready..." << endl; - IOSocket ios; - ByteStream ibs; - ByteStream obs; - uint32_t qb = 0; + IOSocket ios; + ByteStream ibs; + ByteStream obs; + uint32_t qb = 0; - while (1) + while (1) + { + ios = mqs.accept(); + ibs = ios.read(); + + while (ibs.length() > 0) { - ios = mqs.accept(); - ibs = ios.read(); - - while (ibs.length() > 0) - { - cout << "read " << ibs.length() << " bytes from " << ios << endl; - obs.restart(); - obs << qb; - ios.write(obs); - ibs = ios.read(); - } - - ios.close(); + cout << "read " << ibs.length() << " bytes from " << ios << endl; + obs.restart(); + obs << qb; + ios.write(obs); + ibs = ios.read(); } - return 0; -} + ios.close(); + } + return 0; +} diff --git a/utils/messageqcpp/tdriver.cpp b/utils/messageqcpp/tdriver.cpp index 90d240cf3..5f2004257 100644 --- a/utils/messageqcpp/tdriver.cpp +++ b/utils/messageqcpp/tdriver.cpp @@ -40,890 +40,890 @@ using namespace config; class ByteStreamTestSuite : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ByteStreamTestSuite); - CPPUNIT_TEST_SUITE( ByteStreamTestSuite ); + CPPUNIT_TEST(bs_1); + CPPUNIT_TEST(bs_1_1); + CPPUNIT_TEST(bs_1_2); + CPPUNIT_TEST(bs_2); + CPPUNIT_TEST(bs_3); + CPPUNIT_TEST(bs_4); + CPPUNIT_TEST_EXCEPTION(bs_5_1, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_2, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_3, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_4, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_5, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_6, std::underflow_error); + CPPUNIT_TEST(bs_6); + CPPUNIT_TEST(bs_7); + CPPUNIT_TEST(bs_8); + CPPUNIT_TEST_EXCEPTION(bs_9, std::underflow_error); + CPPUNIT_TEST(bs_10); + // CPPUNIT_TEST( bs_11 ); + CPPUNIT_TEST(bs_12); + CPPUNIT_TEST(bs_13); + CPPUNIT_TEST(bs_14); + CPPUNIT_TEST(bs_15); + CPPUNIT_TEST(bs_16); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST( bs_1 ); - CPPUNIT_TEST( bs_1_1 ); - CPPUNIT_TEST( bs_1_2 ); - CPPUNIT_TEST( bs_2 ); - CPPUNIT_TEST( bs_3 ); - CPPUNIT_TEST( bs_4 ); - CPPUNIT_TEST_EXCEPTION( bs_5_1, std::underflow_error ); - CPPUNIT_TEST_EXCEPTION( bs_5_2, std::underflow_error ); - CPPUNIT_TEST_EXCEPTION( bs_5_3, std::underflow_error ); - CPPUNIT_TEST_EXCEPTION( bs_5_4, std::underflow_error ); - CPPUNIT_TEST_EXCEPTION( bs_5_5, std::underflow_error ); - CPPUNIT_TEST_EXCEPTION( bs_5_6, std::underflow_error ); - CPPUNIT_TEST( bs_6 ); - CPPUNIT_TEST( bs_7 ); - CPPUNIT_TEST( bs_8 ); - CPPUNIT_TEST_EXCEPTION( bs_9, std::underflow_error ); - CPPUNIT_TEST( bs_10 ); -// CPPUNIT_TEST( bs_11 ); - CPPUNIT_TEST( bs_12 ); - CPPUNIT_TEST( bs_13 ); - CPPUNIT_TEST( bs_14 ); - CPPUNIT_TEST( bs_15 ); - CPPUNIT_TEST( bs_16 ); - CPPUNIT_TEST_SUITE_END(); + private: + ByteStream::byte b; + ByteStream::doublebyte d; + ByteStream::quadbyte q; + ByteStream::octbyte o; -private: - ByteStream::byte b; - ByteStream::doublebyte d; - ByteStream::quadbyte q; - ByteStream::octbyte o; + uint8_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; - uint8_t u8; - uint16_t u16; - uint32_t u32; - uint64_t u64; - int8_t i8; - int16_t i16; - int32_t i32; - int64_t i64; + ByteStream bs; + ByteStream bs1; - ByteStream bs; - ByteStream bs1; + ByteStream::byte* bap; + ByteStream::byte* bap1; - ByteStream::byte* bap; - ByteStream::byte* bap1; + int len; - int len; + public: + void setUp() + { + bs.reset(); + bs1.reset(); + bap = 0; + bap1 = 0; + } -public: - void setUp() + void tearDown() + { + bs.reset(); + bs1.reset(); + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + } + + void bs_1() + { + bs.reset(); + + o = 0xdeadbeefbadc0ffeLL; + bs << o; + CPPUNIT_ASSERT(bs.length() == 8); + o = 0; + bs >> o; + CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 0); + + q = 0xdeadbeef; + bs << q; + CPPUNIT_ASSERT(bs.length() == 4); + q = 0; + bs >> q; + CPPUNIT_ASSERT(q == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 0); + + d = 0xf00f; + bs << d; + CPPUNIT_ASSERT(bs.length() == 2); + d = 0; + bs >> d; + CPPUNIT_ASSERT(d == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 0); + + b = 0x0f; + bs << b; + CPPUNIT_ASSERT(bs.length() == 1); + b = 0; + bs >> b; + CPPUNIT_ASSERT(b == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + + o = 0xdeadbeefbadc0ffeLL; + bs << o; + CPPUNIT_ASSERT(bs.length() == 8); + o = 0; + + q = 0xdeadbeef; + bs << q; + CPPUNIT_ASSERT(bs.length() == 12); + q = 0; + + d = 0xf00f; + bs << d; + CPPUNIT_ASSERT(bs.length() == 14); + d = 0; + + b = 0x0f; + bs << b; + CPPUNIT_ASSERT(bs.length() == 15); + b = 0; + + bs >> o; + CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 7); + bs >> q; + CPPUNIT_ASSERT(q == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 3); + bs >> d; + CPPUNIT_ASSERT(d == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 1); + bs >> b; + CPPUNIT_ASSERT(b == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_1_1() + { + bs.reset(); + + o = 0xdeadbeefbadc0ffeLL; + bs << o; + CPPUNIT_ASSERT(bs.length() == 8); + o = 0; + + q = 0xdeadbeef; + bs << q; + CPPUNIT_ASSERT(bs.length() == 12); + q = 0; + + d = 0xf00f; + bs << d; + CPPUNIT_ASSERT(bs.length() == 14); + d = 0; + + b = 0x0f; + bs << b; + CPPUNIT_ASSERT(bs.length() == 15); + b = 0; + + ByteStream bbs1; + bbs1 << bs; + CPPUNIT_ASSERT(bbs1.length() == bs.length() + 4); + bs.reset(); + bbs1 >> bs; + CPPUNIT_ASSERT(bbs1.length() == 0); + CPPUNIT_ASSERT(bs.length() == 15); + + bs >> o; + CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 7); + bs >> q; + CPPUNIT_ASSERT(q == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 3); + bs >> d; + CPPUNIT_ASSERT(d == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 1); + bs >> b; + CPPUNIT_ASSERT(b == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_1_2() + { + bs.reset(); + + i64 = -2401053089477160962; + bs << i64; + CPPUNIT_ASSERT(bs.length() == 8); + i64 = 0; + + i32 = -559038737; + bs << i32; + CPPUNIT_ASSERT(bs.length() == 12); + i32 = 0; + + i16 = -4081; + bs << i16; + CPPUNIT_ASSERT(bs.length() == 14); + i16 = 0; + + i8 = 15; + bs << i8; + CPPUNIT_ASSERT(bs.length() == 15); + i8 = 0; + + bs >> i64; + CPPUNIT_ASSERT(i64 == -2401053089477160962); + CPPUNIT_ASSERT(bs.length() == 7); + + bs >> i32; + CPPUNIT_ASSERT(i32 == -559038737); + CPPUNIT_ASSERT(bs.length() == 3); + + bs >> i16; + CPPUNIT_ASSERT(i16 == -4081); + CPPUNIT_ASSERT(bs.length() == 1); + + bs >> i8; + CPPUNIT_ASSERT(i8 == 15); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_2() + { + int i; + + bs.reset(); + srand(time(0)); + + for (i = 0; i < 10240; i++) { - bs.reset(); - bs1.reset(); - bap = 0; - bap1 = 0; + bs << (uint32_t)rand(); } - void tearDown() + bs1 = bs; + + uint32_t q1; + + for (i = 0; i < 10240; i++) { - bs.reset(); - bs1.reset(); - delete [] bap; - bap = 0; - delete [] bap1; - bap1 = 0; + bs >> u32; + bs1 >> q1; + CPPUNIT_ASSERT(u32 == q1); } - void bs_1() + bs.reset(); + bs1.reset(); + } + + void bs_3() + { + uint8_t ba[1024] = { + 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, + }; + + bs.load(ba, 8); + CPPUNIT_ASSERT(bs.length() == 8); + bs >> u8; + CPPUNIT_ASSERT(b == 0x12); + bs >> u16; + CPPUNIT_ASSERT(d == 0x5634); + bs >> u32; + CPPUNIT_ASSERT(q == 0xdebc9a78); + + CPPUNIT_ASSERT(bs.length() == 1); + + bs.reset(); + CPPUNIT_ASSERT(bs.length() == 0); + + bs.load(ba, 8); + len = bs.length(); + CPPUNIT_ASSERT(len == 8); + bap = new ByteStream::byte[len]; + // bs >> bap; + memcpy(bap, bs.buf(), len); + CPPUNIT_ASSERT(memcmp(ba, bap, len) == 0); + delete[] bap; + bap = 0; + + bs.reset(); + + for (u32 = 0; u32 < 20480; u32++) { - - bs.reset(); - - o = 0xdeadbeefbadc0ffeLL; - bs << o; - CPPUNIT_ASSERT(bs.length() == 8); - o = 0; - bs >> o; - CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); - CPPUNIT_ASSERT(bs.length() == 0); - - q = 0xdeadbeef; - bs << q; - CPPUNIT_ASSERT(bs.length() == 4); - q = 0; - bs >> q; - CPPUNIT_ASSERT(q == 0xdeadbeef); - CPPUNIT_ASSERT(bs.length() == 0); - - d = 0xf00f; - bs << d; - CPPUNIT_ASSERT(bs.length() == 2); - d = 0; - bs >> d; - CPPUNIT_ASSERT(d == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 0); - - b = 0x0f; - bs << b; - CPPUNIT_ASSERT(bs.length() == 1); - b = 0; - bs >> b; - CPPUNIT_ASSERT(b == 0x0f); - CPPUNIT_ASSERT(bs.length() == 0); - - o = 0xdeadbeefbadc0ffeLL; - bs << o; - CPPUNIT_ASSERT(bs.length() == 8); - o = 0; - - q = 0xdeadbeef; - bs << q; - CPPUNIT_ASSERT(bs.length() == 12); - q = 0; - - d = 0xf00f; - bs << d; - CPPUNIT_ASSERT(bs.length() == 14); - d = 0; - - b = 0x0f; - bs << b; - CPPUNIT_ASSERT(bs.length() == 15); - b = 0; - - bs >> o; - CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); - CPPUNIT_ASSERT(bs.length() == 7); - bs >> q; - CPPUNIT_ASSERT(q == 0xdeadbeef); - CPPUNIT_ASSERT(bs.length() == 3); - bs >> d; - CPPUNIT_ASSERT(d == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 1); - bs >> b; - CPPUNIT_ASSERT(b == 0x0f); - CPPUNIT_ASSERT(bs.length() == 0); + bs << u32; } - void bs_1_1() + len = bs.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(u32))); + bap = new ByteStream::byte[len]; + // bs >> bap; + memcpy(bap, bs.buf(), len); + + bs.reset(); + + for (u32 = 0; u32 < 20480; u32++) { - - bs.reset(); - - o = 0xdeadbeefbadc0ffeLL; - bs << o; - CPPUNIT_ASSERT(bs.length() == 8); - o = 0; - - q = 0xdeadbeef; - bs << q; - CPPUNIT_ASSERT(bs.length() == 12); - q = 0; - - d = 0xf00f; - bs << d; - CPPUNIT_ASSERT(bs.length() == 14); - d = 0; - - b = 0x0f; - bs << b; - CPPUNIT_ASSERT(bs.length() == 15); - b = 0; - - ByteStream bbs1; - bbs1 << bs; - CPPUNIT_ASSERT(bbs1.length() == bs.length() + 4); - bs.reset(); - bbs1 >> bs; - CPPUNIT_ASSERT(bbs1.length() == 0); - CPPUNIT_ASSERT(bs.length() == 15); - - bs >> o; - CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); - CPPUNIT_ASSERT(bs.length() == 7); - bs >> q; - CPPUNIT_ASSERT(q == 0xdeadbeef); - CPPUNIT_ASSERT(bs.length() == 3); - bs >> d; - CPPUNIT_ASSERT(d == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 1); - bs >> b; - CPPUNIT_ASSERT(b == 0x0f); - CPPUNIT_ASSERT(bs.length() == 0); + bs << u32; } - void bs_1_2() + len = bs.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(q))); + bap1 = new ByteStream::byte[len]; + // bs >> bap1; + memcpy(bap1, bs.buf(), len); + + CPPUNIT_ASSERT(memcmp(bap1, bap, len) == 0); + + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + bs.reset(); + } + void bs_4() + { + for (i32 = 0; i32 < 20480; i32++) { - - bs.reset(); - - i64 = -2401053089477160962; - bs << i64; - CPPUNIT_ASSERT(bs.length() == 8); - i64 = 0; - - i32 = -559038737; - bs << i32; - CPPUNIT_ASSERT(bs.length() == 12); - i32 = 0; - - i16 = -4081; - bs << i16; - CPPUNIT_ASSERT(bs.length() == 14); - i16 = 0; - - i8 = 15; - bs << i8; - CPPUNIT_ASSERT(bs.length() == 15); - i8 = 0; - - bs >> i64; - CPPUNIT_ASSERT(i64 == -2401053089477160962); - CPPUNIT_ASSERT(bs.length() == 7); - - bs >> i32; - CPPUNIT_ASSERT(i32 == -559038737); - CPPUNIT_ASSERT(bs.length() == 3); - - bs >> i16; - CPPUNIT_ASSERT(i16 == -4081); - CPPUNIT_ASSERT(bs.length() == 1); - - bs >> i8; - CPPUNIT_ASSERT(i8 == 15); - CPPUNIT_ASSERT(bs.length() == 0); + bs << i32; } - void bs_2() + ByteStream bs2(bs); + len = bs2.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(i32))); + bap = new ByteStream::byte[len]; + // bs2 >> bap; + memcpy(bap, bs2.buf(), len); + + bs1 = bs2; + len = bs1.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(i32))); + bap1 = new ByteStream::byte[len]; + // bs1 >> bap1; + memcpy(bap1, bs1.buf(), len); + + CPPUNIT_ASSERT(memcmp(bap1, bap, len) == 0); + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + bs.reset(); + bs1.reset(); + bs2.reset(); + } + + void bs_5_1() + { + bs.reset(); + + u8 = 0x0f; + bs << u8; + + for (;;) + bs >> u32; + } + + void bs_5_2() + { + bs.reset(); + + u8 = 0x0f; + bs << u8; + + for (;;) + bs >> u16; + } + + void bs_5_3() + { + bs.reset(); + + u8 = 0x0f; + bs << u8; + + for (;;) + bs >> u8; + } + + void bs_5_4() + { + bs.reset(); + + i8 = 0x0f; + bs << i8; + + for (;;) + bs >> i32; + } + + void bs_5_5() + { + bs.reset(); + + i8 = 0x0f; + bs << i8; + + for (;;) + bs >> i16; + } + + void bs_5_6() + { + bs.reset(); + + i8 = 0x0f; + bs << i8; + + for (;;) + bs >> i8; + } + + void bs_6() + { + u8 = 0x1a; + bs << u8; + u8 = 0x2b; + bs << u8; + u8 = 0x3c; + bs << u8; + + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x1a); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x2b); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x3c); + + bs.reset(); + + u8 = 12; + bs << u8; + u8 = 3; + bs << u8; + u8 = 0; + bs << u8; + u8 = 2; + bs << u8; + + ByteStream bs3(bs); + + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 12); + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 3); + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 0); + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 2); + } + + void bs_7() + { + size_t i; + + bs.reset(); + bap = new ByteStream::byte[ByteStream::BlockSize * 2]; + ByteStream::byte* bapp; + + for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0xa5; + + bs.append(bap, ByteStream::BlockSize); + CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 1)); + + for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0x5a; + + bs.append(bap, ByteStream::BlockSize); + CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 2)); + + for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize * 2; bapp++, i++) + *bapp = 0x55; + + bs.append(bap, ByteStream::BlockSize * 2); + CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 4)); + delete[] bap; + bap = new ByteStream::byte[bs.length()]; + // bs >> bap; + memcpy(bap, bs.buf(), bs.length()); + bap1 = new ByteStream::byte[bs.length()]; + + for (bapp = &bap1[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0xa5; + + for (i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0x5a; + + for (i = 0; i < ByteStream::BlockSize * 2; bapp++, i++) + *bapp = 0x55; + + CPPUNIT_ASSERT(memcmp(bap, bap1, bs.length()) == 0); + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + } + + void bs_8() + { + bs.reset(); + string s; + s = "This is a test"; + bs << s; + string s1; + bs >> s1; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == 0); + + ifstream ifs; + ifs.open("./tdriver.cpp"); + int ifs_len; + ifs.seekg(0, ios::end); + ifs_len = ifs.tellg(); + ifs.seekg(0, ios::beg); + boost::scoped_array buf(new char[ifs_len + 1]); + ifs.read(buf.get(), ifs_len); + buf[ifs_len] = 0; + ifs.close(); + bs.reset(); + s = buf.get(); + bs << s; + bs >> s1; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == 0); + + u8 = 0xa5; + bs << u8; + u16 = 0x5aa5; + bs << u16; + u32 = 0xdeadbeef; + bs << u32; + bs << s; + s += s1; + bs << s; + s += s1; + bs << s; + bs << u32; + bs << u16; + bs << u8; + + bs >> u8; + CPPUNIT_ASSERT(u8 == 0xa5); + bs >> u16; + CPPUNIT_ASSERT(u16 == 0x5aa5); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> s; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(s.length() == (s1.length() * 1)); + bs >> s; + CPPUNIT_ASSERT(s.length() == (s1.length() * 2)); + bs >> s; + CPPUNIT_ASSERT(s.length() == (s1.length() * 3)); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> u16; + CPPUNIT_ASSERT(u16 == 0x5aa5); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0xa5); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_9() + { + bs.reset(); + // Load up a bogus string (too short) + u32 = 100; + bs << u32; + bs.append(reinterpret_cast("This is a test"), 14); + string s; + // Should throw underflow + bs >> s; + } + + void bs_10() + { + bs.reset(); + bs1.reset(); + u32 = 0xdeadbeef; + bs << u32; + CPPUNIT_ASSERT(bs.length() == 4); + CPPUNIT_ASSERT(bs1.length() == 0); + bs.swap(bs1); + CPPUNIT_ASSERT(bs1.length() == 4); + CPPUNIT_ASSERT(bs.length() == 0); + bs1 >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + + bs.reset(); + bs1.reset(); + u32 = 0xdeadbeef; + bs << u32; + bs1 << u32; + bs += bs1; + CPPUNIT_ASSERT(bs.length() == 8); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + + bs.reset(); + bs1.reset(); + ByteStream bs2; + u32 = 0xdeadbeef; + bs1 << u32; + bs2 << u32; + bs = bs1 + bs2; + CPPUNIT_ASSERT(bs.length() == 8); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + } + + void bs_11() + { + bs.reset(); + bs1.reset(); + u32 = 0xdeadbeef; + bs << u32; + bs1 << u32; + + // save bs1 state + ByteStream::byte* bs1_fBuf = bs1.fBuf; + ByteStream::byte* bs1_fCurInPtr = bs1.fCurInPtr; + ByteStream::byte* bs1_fCurOutPtr = bs1.fCurOutPtr; + size_t bs1_fMaxLen = bs1.fMaxLen; + + // introduce an error + bs.fCurOutPtr += 1024000; + + // save bs state + ByteStream::byte* bs_fBuf = bs.fBuf; + ByteStream::byte* bs_fCurInPtr = bs.fCurInPtr; + ByteStream::byte* bs_fCurOutPtr = bs.fCurOutPtr; + size_t bs_fMaxLen = bs.fMaxLen; + + try + { + bs1 = bs; + } + catch (out_of_range& ex) { - int i; - - bs.reset(); - srand(time(0)); - - for (i = 0; i < 10240; i++) - { - bs << (uint32_t)rand(); - } - - bs1 = bs; - - uint32_t q1; - - for (i = 0; i < 10240; i++) - { - bs >> u32; - bs1 >> q1; - CPPUNIT_ASSERT(u32 == q1); - } - - bs.reset(); - bs1.reset(); } - void bs_3() + // at this point bs1 should be just as before the assignment + CPPUNIT_ASSERT(bs1.fBuf == bs1_fBuf); + CPPUNIT_ASSERT(bs1.fCurInPtr == bs1_fCurInPtr); + CPPUNIT_ASSERT(bs1.fCurOutPtr == bs1_fCurOutPtr); + CPPUNIT_ASSERT(bs1.fMaxLen == bs1_fMaxLen); + + // same goes for bs + CPPUNIT_ASSERT(bs.fBuf == bs_fBuf); + CPPUNIT_ASSERT(bs.fCurInPtr == bs_fCurInPtr); + CPPUNIT_ASSERT(bs.fCurOutPtr == bs_fCurOutPtr); + CPPUNIT_ASSERT(bs.fMaxLen == bs_fMaxLen); + } + + void bs_12() + { + bs.reset(); + + u64 = 0xdeadbeefbadc0ffeLL; + bs << u64; + CPPUNIT_ASSERT(bs.length() == 8); + u64 = 0; + bs.peek(u64); + CPPUNIT_ASSERT(u64 == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 8); + u64 = 0; + bs >> u64; + CPPUNIT_ASSERT(u64 == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 0); + + u16 = 0xf00f; + bs << u16; + CPPUNIT_ASSERT(bs.length() == 2); + u16 = 0; + bs.peek(u16); + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 2); + u16 = 0; + bs >> u16; + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 0); + + u8 = 0x0f; + bs << u8; + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs.peek(u8); + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + + u32 = 0xdeadbeef; + bs << u32; + CPPUNIT_ASSERT(bs.length() == 4); + u32 = 0; + + u16 = 0xf00f; + bs << u16; + CPPUNIT_ASSERT(bs.length() == 6); + u16 = 0; + + u8 = 0x0f; + bs << u8; + CPPUNIT_ASSERT(bs.length() == 7); + u8 = 0; + + bs.peek(u32); + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 7); + u32 = 0; + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 3); + u16 = 0; + bs.peek(u16); + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 3); + u16 = 0; + bs >> u16; + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs.peek(u8); + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + + string s; + s = "This is a test"; + bs << s; + string s1; + bs.peek(s1); + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == s1.size() + 4); + s1.empty(); + bs >> s1; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_13() + { + string s; + ifstream ifs; + ifs.open("./tdriver.cpp"); + int ifs_len; + ifs.seekg(0, ios::end); + ifs_len = ifs.tellg(); + ifs.seekg(0, ios::beg); + boost::scoped_array buf(new char[ifs_len + 1]); + ifs.read(buf.get(), ifs_len); + buf[ifs_len] = 0; + ifs.close(); + bs.reset(); + s = buf.get(); + bs << s; + ofstream of("bs_13.dat"); + of << bs; + of.close(); + ifs.open("./bs_13.dat"); + ifs.seekg(0, ios::end); + int ifs_len1; + ifs_len1 = ifs.tellg(); + // will be longer than orig file because string length is encoded into stream + CPPUNIT_ASSERT((ifs_len + (int)sizeof(ByteStream::quadbyte)) == ifs_len1); + ifs.seekg(0, ios::beg); + boost::scoped_array buf1(new char[ifs_len1]); + bs1.reset(); + ifs >> bs1; + ifs.close(); + CPPUNIT_ASSERT(bs.length() == bs1.length()); + string s1; + bs1 >> s1; + CPPUNIT_ASSERT(s == s1); + } + + void bs_14() + { + ByteStream bs1(0); + ByteStream bs2(bs1); + CPPUNIT_ASSERT(bs2.fBuf == 0); + ByteStream bs3(0); + bs3 = bs1; + CPPUNIT_ASSERT(bs3.fBuf == 0); + } + + void bs_15() + { + ByteStream b1, b2, empty; + uint8_t u8; + + CPPUNIT_ASSERT(b1 == b2); + CPPUNIT_ASSERT(b2 == b1); + CPPUNIT_ASSERT(b2 == empty); + CPPUNIT_ASSERT(b1 == empty); + + CPPUNIT_ASSERT(!(b1 != b2)); + CPPUNIT_ASSERT(!(b2 != b1)); + CPPUNIT_ASSERT(!(b2 != empty)); + CPPUNIT_ASSERT(!(b1 != empty)); + + b1 << "Woo hoo"; + + CPPUNIT_ASSERT(b1 != b2); + CPPUNIT_ASSERT(b2 != b1); + CPPUNIT_ASSERT(b1 != empty); + + CPPUNIT_ASSERT(!(b1 == b2)); + CPPUNIT_ASSERT(!(b2 == b1)); + CPPUNIT_ASSERT(!(b1 == empty)); + + b2 << "Woo hoo"; + + CPPUNIT_ASSERT(b1 == b2); + CPPUNIT_ASSERT(b2 == b1); + CPPUNIT_ASSERT(!(b1 != b2)); + CPPUNIT_ASSERT(!(b2 != b1)); + + b1 >> u8; + + CPPUNIT_ASSERT(b1 != b2); + CPPUNIT_ASSERT(b2 != b1); + CPPUNIT_ASSERT(!(b1 == b2)); + CPPUNIT_ASSERT(!(b2 == b1)); + + b1 << u8; + + CPPUNIT_ASSERT(b1 != b2); + CPPUNIT_ASSERT(b2 != b1); + CPPUNIT_ASSERT(!(b1 == b2)); + CPPUNIT_ASSERT(!(b2 == b1)); + + b2 >> u8; + b2 << u8; + + CPPUNIT_ASSERT(b1 == b2); + CPPUNIT_ASSERT(b2 == b1); + CPPUNIT_ASSERT(!(b1 != b2)); + CPPUNIT_ASSERT(!(b2 != b1)); + } + + void bs_16() + { + int i; + uint32_t len; + + bs.reset(); + srand(time(0)); + + for (i = 0; i < 10240; i++) { - - uint8_t ba[1024] = { 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, }; - - bs.load(ba, 8); - CPPUNIT_ASSERT(bs.length() == 8); - bs >> u8; - CPPUNIT_ASSERT(b == 0x12); - bs >> u16; - CPPUNIT_ASSERT(d == 0x5634); - bs >> u32; - CPPUNIT_ASSERT(q == 0xdebc9a78); - - CPPUNIT_ASSERT(bs.length() == 1); - - bs.reset(); - CPPUNIT_ASSERT(bs.length() == 0); - - bs.load(ba, 8); - len = bs.length(); - CPPUNIT_ASSERT(len == 8); - bap = new ByteStream::byte[len]; - //bs >> bap; - memcpy(bap, bs.buf(), len); - CPPUNIT_ASSERT(memcmp(ba, bap, len) == 0); - delete [] bap; - bap = 0; - - bs.reset(); - - for (u32 = 0; u32 < 20480; u32++) - { - bs << u32; - } - - len = bs.length(); - CPPUNIT_ASSERT(len == (20480 * sizeof(u32))); - bap = new ByteStream::byte[len]; - //bs >> bap; - memcpy(bap, bs.buf(), len); - - bs.reset(); - - for (u32 = 0; u32 < 20480; u32++) - { - bs << u32; - } - - len = bs.length(); - CPPUNIT_ASSERT(len == (20480 * sizeof(q))); - bap1 = new ByteStream::byte[len]; - //bs >> bap1; - memcpy(bap1, bs.buf(), len); - - CPPUNIT_ASSERT(memcmp(bap1, bap, len) == 0); - - delete [] bap; - bap = 0; - delete [] bap1; - bap1 = 0; - bs.reset(); - } - void bs_4() - { - - for (i32 = 0; i32 < 20480; i32++) - { - bs << i32; - } - - ByteStream bs2(bs); - len = bs2.length(); - CPPUNIT_ASSERT(len == (20480 * sizeof(i32))); - bap = new ByteStream::byte[len]; - //bs2 >> bap; - memcpy(bap, bs2.buf(), len); - - bs1 = bs2; - len = bs1.length(); - CPPUNIT_ASSERT(len == (20480 * sizeof(i32))); - bap1 = new ByteStream::byte[len]; - //bs1 >> bap1; - memcpy(bap1, bs1.buf(), len); - - CPPUNIT_ASSERT(memcmp(bap1, bap, len) == 0); - delete [] bap; - bap = 0; - delete [] bap1; - bap1 = 0; - bs.reset(); - bs1.reset(); - bs2.reset(); - } - - void bs_5_1() - { - bs.reset(); - - u8 = 0x0f; - bs << u8; - - for (;;) bs >> u32; - } - - void bs_5_2() - { - bs.reset(); - - u8 = 0x0f; - bs << u8; - - for (;;) bs >> u16; - } - - void bs_5_3() - { - bs.reset(); - - u8 = 0x0f; - bs << u8; - - for (;;) bs >> u8; - } - - void bs_5_4() - { - bs.reset(); - - i8 = 0x0f; - bs << i8; - - for (;;) bs >> i32; - } - - void bs_5_5() - { - bs.reset(); - - i8 = 0x0f; - bs << i8; - - for (;;) bs >> i16; - } - - void bs_5_6() - { - bs.reset(); - - i8 = 0x0f; - bs << i8; - - for (;;) bs >> i8; - } - - void bs_6() - { - u8 = 0x1a; - bs << u8; - u8 = 0x2b; - bs << u8; - u8 = 0x3c; - bs << u8; - - bs >> u8; - CPPUNIT_ASSERT(u8 == 0x1a); - bs >> u8; - CPPUNIT_ASSERT(u8 == 0x2b); - bs >> u8; - CPPUNIT_ASSERT(u8 == 0x3c); - - bs.reset(); - - u8 = 12; - bs << u8; - u8 = 3; - bs << u8; - u8 = 0; - bs << u8; - u8 = 2; - bs << u8; - - ByteStream bs3(bs); - - bs3 >> u8; - CPPUNIT_ASSERT(u8 == 12); - bs3 >> u8; - CPPUNIT_ASSERT(u8 == 3); - bs3 >> u8; - CPPUNIT_ASSERT(u8 == 0); - bs3 >> u8; - CPPUNIT_ASSERT(u8 == 2); - - } - - void bs_7() - { - size_t i; - - bs.reset(); - bap = new ByteStream::byte[ByteStream::BlockSize * 2]; - ByteStream::byte* bapp; - - for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) *bapp = 0xa5; - - bs.append(bap, ByteStream::BlockSize); - CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 1)); - - for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) *bapp = 0x5a; - - bs.append(bap, ByteStream::BlockSize); - CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 2)); - - for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize * 2; bapp++, i++) *bapp = 0x55; - - bs.append(bap, ByteStream::BlockSize * 2); - CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 4)); - delete [] bap; - bap = new ByteStream::byte[bs.length()]; - //bs >> bap; - memcpy(bap, bs.buf(), bs.length()); - bap1 = new ByteStream::byte[bs.length()]; - - for (bapp = &bap1[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) *bapp = 0xa5; - - for (i = 0; i < ByteStream::BlockSize; bapp++, i++) *bapp = 0x5a; - - for (i = 0; i < ByteStream::BlockSize * 2; bapp++, i++) *bapp = 0x55; - - CPPUNIT_ASSERT(memcmp(bap, bap1, bs.length()) == 0); - delete [] bap; - bap = 0; - delete [] bap1; - bap1 = 0; - - } - - void bs_8() - { - bs.reset(); - string s; - s = "This is a test"; - bs << s; - string s1; - bs >> s1; - CPPUNIT_ASSERT(s == s1); - CPPUNIT_ASSERT(bs.length() == 0); - - ifstream ifs; - ifs.open("./tdriver.cpp"); - int ifs_len; - ifs.seekg(0, ios::end); - ifs_len = ifs.tellg(); - ifs.seekg(0, ios::beg); - boost::scoped_array buf(new char[ifs_len + 1]); - ifs.read(buf.get(), ifs_len); - buf[ifs_len] = 0; - ifs.close(); - bs.reset(); - s = buf.get(); - bs << s; - bs >> s1; - CPPUNIT_ASSERT(s == s1); - CPPUNIT_ASSERT(bs.length() == 0); - - u8 = 0xa5; - bs << u8; - u16 = 0x5aa5; - bs << u16; - u32 = 0xdeadbeef; - bs << u32; - bs << s; - s += s1; - bs << s; - s += s1; - bs << s; - bs << u32; - bs << u16; - bs << u8; - - bs >> u8; - CPPUNIT_ASSERT(u8 == 0xa5); - bs >> u16; - CPPUNIT_ASSERT(u16 == 0x5aa5); - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - bs >> s; - CPPUNIT_ASSERT(s == s1); - CPPUNIT_ASSERT(s.length() == (s1.length() * 1)); - bs >> s; - CPPUNIT_ASSERT(s.length() == (s1.length() * 2)); - bs >> s; - CPPUNIT_ASSERT(s.length() == (s1.length() * 3)); - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - bs >> u16; - CPPUNIT_ASSERT(u16 == 0x5aa5); - bs >> u8; - CPPUNIT_ASSERT(u8 == 0xa5); - CPPUNIT_ASSERT(bs.length() == 0); - - } - - void bs_9() - { - bs.reset(); - // Load up a bogus string (too short) - u32 = 100; - bs << u32; - bs.append(reinterpret_cast("This is a test"), 14); - string s; - // Should throw underflow - bs >> s; - } - - void bs_10() - { - bs.reset(); - bs1.reset(); - u32 = 0xdeadbeef; - bs << u32; - CPPUNIT_ASSERT(bs.length() == 4); - CPPUNIT_ASSERT(bs1.length() == 0); - bs.swap(bs1); - CPPUNIT_ASSERT(bs1.length() == 4); - CPPUNIT_ASSERT(bs.length() == 0); - bs1 >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - - bs.reset(); - bs1.reset(); - u32 = 0xdeadbeef; - bs << u32; - bs1 << u32; - bs += bs1; - CPPUNIT_ASSERT(bs.length() == 8); - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - - bs.reset(); - bs1.reset(); - ByteStream bs2; - u32 = 0xdeadbeef; - bs1 << u32; - bs2 << u32; - bs = bs1 + bs2; - CPPUNIT_ASSERT(bs.length() == 8); - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - - } - - void bs_11() - { - bs.reset(); - bs1.reset(); - u32 = 0xdeadbeef; - bs << u32; - bs1 << u32; - - // save bs1 state - ByteStream::byte* bs1_fBuf = bs1.fBuf; - ByteStream::byte* bs1_fCurInPtr = bs1.fCurInPtr; - ByteStream::byte* bs1_fCurOutPtr = bs1.fCurOutPtr; - size_t bs1_fMaxLen = bs1.fMaxLen; - - //introduce an error - bs.fCurOutPtr += 1024000; - - // save bs state - ByteStream::byte* bs_fBuf = bs.fBuf; - ByteStream::byte* bs_fCurInPtr = bs.fCurInPtr; - ByteStream::byte* bs_fCurOutPtr = bs.fCurOutPtr; - size_t bs_fMaxLen = bs.fMaxLen; - - try - { - bs1 = bs; - } - catch (out_of_range& ex) - { - } - - //at this point bs1 should be just as before the assignment - CPPUNIT_ASSERT(bs1.fBuf == bs1_fBuf); - CPPUNIT_ASSERT(bs1.fCurInPtr == bs1_fCurInPtr); - CPPUNIT_ASSERT(bs1.fCurOutPtr == bs1_fCurOutPtr); - CPPUNIT_ASSERT(bs1.fMaxLen == bs1_fMaxLen); - - //same goes for bs - CPPUNIT_ASSERT(bs.fBuf == bs_fBuf); - CPPUNIT_ASSERT(bs.fCurInPtr == bs_fCurInPtr); - CPPUNIT_ASSERT(bs.fCurOutPtr == bs_fCurOutPtr); - CPPUNIT_ASSERT(bs.fMaxLen == bs_fMaxLen); - - } - - void bs_12() - { - - bs.reset(); - - u64 = 0xdeadbeefbadc0ffeLL; - bs << u64; - CPPUNIT_ASSERT(bs.length() == 8); - u64 = 0; - bs.peek(u64); - CPPUNIT_ASSERT(u64 == 0xdeadbeefbadc0ffeLL); - CPPUNIT_ASSERT(bs.length() == 8); - u64 = 0; - bs >> u64; - CPPUNIT_ASSERT(u64 == 0xdeadbeefbadc0ffeLL); - CPPUNIT_ASSERT(bs.length() == 0); - - u16 = 0xf00f; - bs << u16; - CPPUNIT_ASSERT(bs.length() == 2); - u16 = 0; - bs.peek(u16); - CPPUNIT_ASSERT(u16 == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 2); - u16 = 0; - bs >> u16; - CPPUNIT_ASSERT(u16 == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 0); - - u8 = 0x0f; - bs << u8; - CPPUNIT_ASSERT(bs.length() == 1); - u8 = 0; - bs.peek(u8); - CPPUNIT_ASSERT(u8 == 0x0f); - CPPUNIT_ASSERT(bs.length() == 1); - u8 = 0; - bs >> u8; - CPPUNIT_ASSERT(u8 == 0x0f); - CPPUNIT_ASSERT(bs.length() == 0); - - u32 = 0xdeadbeef; - bs << u32; - CPPUNIT_ASSERT(bs.length() == 4); - u32 = 0; - - u16 = 0xf00f; - bs << u16; - CPPUNIT_ASSERT(bs.length() == 6); - u16 = 0; - - u8 = 0x0f; - bs << u8; - CPPUNIT_ASSERT(bs.length() == 7); - u8 = 0; - - bs.peek(u32); - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - CPPUNIT_ASSERT(bs.length() == 7); - u32 = 0; - bs >> u32; - CPPUNIT_ASSERT(u32 == 0xdeadbeef); - CPPUNIT_ASSERT(bs.length() == 3); - u16 = 0; - bs.peek(u16); - CPPUNIT_ASSERT(u16 == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 3); - u16 = 0; - bs >> u16; - CPPUNIT_ASSERT(u16 == 0xf00f); - CPPUNIT_ASSERT(bs.length() == 1); - u8 = 0; - bs.peek(u8); - CPPUNIT_ASSERT(u8 == 0x0f); - CPPUNIT_ASSERT(bs.length() == 1); - u8 = 0; - bs >> u8; - CPPUNIT_ASSERT(u8 == 0x0f); - CPPUNIT_ASSERT(bs.length() == 0); - - string s; - s = "This is a test"; - bs << s; - string s1; - bs.peek(s1); - CPPUNIT_ASSERT(s == s1); - CPPUNIT_ASSERT(bs.length() == s1.size() + 4); - s1.empty(); - bs >> s1; - CPPUNIT_ASSERT(s == s1); - CPPUNIT_ASSERT(bs.length() == 0); - } - - void bs_13() - { - string s; - ifstream ifs; - ifs.open("./tdriver.cpp"); - int ifs_len; - ifs.seekg(0, ios::end); - ifs_len = ifs.tellg(); - ifs.seekg(0, ios::beg); - boost::scoped_array buf(new char[ifs_len + 1]); - ifs.read(buf.get(), ifs_len); - buf[ifs_len] = 0; - ifs.close(); - bs.reset(); - s = buf.get(); - bs << s; - ofstream of("bs_13.dat"); - of << bs; - of.close(); - ifs.open("./bs_13.dat"); - ifs.seekg(0, ios::end); - int ifs_len1; - ifs_len1 = ifs.tellg(); - // will be longer than orig file because string length is encoded into stream - CPPUNIT_ASSERT((ifs_len + (int)sizeof(ByteStream::quadbyte)) == ifs_len1); - ifs.seekg(0, ios::beg); - boost::scoped_array buf1(new char[ifs_len1]); - bs1.reset(); - ifs >> bs1; - ifs.close(); - CPPUNIT_ASSERT(bs.length() == bs1.length()); - string s1; - bs1 >> s1; - CPPUNIT_ASSERT(s == s1); - } - - void bs_14() - { - ByteStream bs1(0); - ByteStream bs2(bs1); - CPPUNIT_ASSERT(bs2.fBuf == 0); - ByteStream bs3(0); - bs3 = bs1; - CPPUNIT_ASSERT(bs3.fBuf == 0); - } - - void bs_15() - { - ByteStream b1, b2, empty; - uint8_t u8; - - CPPUNIT_ASSERT(b1 == b2); - CPPUNIT_ASSERT(b2 == b1); - CPPUNIT_ASSERT(b2 == empty); - CPPUNIT_ASSERT(b1 == empty); - - CPPUNIT_ASSERT(!(b1 != b2)); - CPPUNIT_ASSERT(!(b2 != b1)); - CPPUNIT_ASSERT(!(b2 != empty)); - CPPUNIT_ASSERT(!(b1 != empty)); - - b1 << "Woo hoo"; - - CPPUNIT_ASSERT(b1 != b2); - CPPUNIT_ASSERT(b2 != b1); - CPPUNIT_ASSERT(b1 != empty); - - CPPUNIT_ASSERT(!(b1 == b2)); - CPPUNIT_ASSERT(!(b2 == b1)); - CPPUNIT_ASSERT(!(b1 == empty)); - - b2 << "Woo hoo"; - - CPPUNIT_ASSERT(b1 == b2); - CPPUNIT_ASSERT(b2 == b1); - CPPUNIT_ASSERT(!(b1 != b2)); - CPPUNIT_ASSERT(!(b2 != b1)); - - b1 >> u8; - - CPPUNIT_ASSERT(b1 != b2); - CPPUNIT_ASSERT(b2 != b1); - CPPUNIT_ASSERT(!(b1 == b2)); - CPPUNIT_ASSERT(!(b2 == b1)); - - b1 << u8; - - CPPUNIT_ASSERT(b1 != b2); - CPPUNIT_ASSERT(b2 != b1); - CPPUNIT_ASSERT(!(b1 == b2)); - CPPUNIT_ASSERT(!(b2 == b1)); - - b2 >> u8; - b2 << u8; - - CPPUNIT_ASSERT(b1 == b2); - CPPUNIT_ASSERT(b2 == b1); - CPPUNIT_ASSERT(!(b1 != b2)); - CPPUNIT_ASSERT(!(b2 != b1)); - - } - - void bs_16() - { - int i; - uint32_t len; - - bs.reset(); - srand(time(0)); - - for (i = 0; i < 10240; i++) - { - bs << (ByteStream::quadbyte)rand(); - } - - boost::scoped_array bp(new ByteStream::byte[bs.length()]); - ByteStream::byte* bpp = bp.get(); - boost::scoped_array bp1(new ByteStream::byte[bs.length()]); - ByteStream::byte* bpp1 = bp1.get(); - - len = bs.length(); - CPPUNIT_ASSERT(len == 10240 * 4); - bs.peek(bpp); - CPPUNIT_ASSERT(bs.length() == len); - CPPUNIT_ASSERT(memcmp(bpp, bs.buf(), len) == 0); - - bs >> bpp1; - CPPUNIT_ASSERT(bs.length() == 0); - CPPUNIT_ASSERT(memcmp(bpp, bpp1, len) == 0); - - bs.reset(); + bs << (ByteStream::quadbyte)rand(); } + boost::scoped_array bp(new ByteStream::byte[bs.length()]); + ByteStream::byte* bpp = bp.get(); + boost::scoped_array bp1(new ByteStream::byte[bs.length()]); + ByteStream::byte* bpp1 = bp1.get(); + + len = bs.length(); + CPPUNIT_ASSERT(len == 10240 * 4); + bs.peek(bpp); + CPPUNIT_ASSERT(bs.length() == len); + CPPUNIT_ASSERT(memcmp(bpp, bs.buf(), len) == 0); + + bs >> bpp1; + CPPUNIT_ASSERT(bs.length() == 0); + CPPUNIT_ASSERT(memcmp(bpp, bpp1, len) == 0); + + bs.reset(); + } }; static string normServ; @@ -934,454 +934,455 @@ volatile static bool isRunning; volatile static bool leakCheck; #define TS_NS(x) (x) -#define TS_US(x) ((x) * 1000) -#define TS_MS(x) ((x) * 1000000) +#define TS_US(x) ((x)*1000) +#define TS_MS(x) ((x)*1000000) static void startServer() { - MessageQueueServer* inMq; - bool retry; + MessageQueueServer* inMq; + bool retry; - do + do + { + try { - try - { - retry = false; - inMq = new MessageQueueServer(normServ, "./Columnstore.xml"); - } - catch (exception& ex) - { - //cout << endl << "MessageQueueServer ctor threw!: " << ex.what() << endl; - ::usleep(5000000); - retry = true; - } - catch (...) - { - //cout << endl << "MessageQueueServer ctor threw!" << endl; - ::usleep(5000000); - retry = true; - } + retry = false; + inMq = new MessageQueueServer(normServ, "./Columnstore.xml"); } - while (retry); - - ByteStream inBs; - //cout << endl << "startServer is starting" << endl; - - // We need to loop here because the big write in mq_1() may (will) not come in one - // read. The other servers will fail if if too much is writen. - - ByteStream bs2; - struct timespec ts = { 0, TS_MS(500) }; - isRunning = true; - IOSocket sock = inMq->accept(); - - for (;;) + catch (exception& ex) { - inBs.reset(); + // cout << endl << "MessageQueueServer ctor threw!: " << ex.what() << endl; + ::usleep(5000000); + retry = true; + } + catch (...) + { + // cout << endl << "MessageQueueServer ctor threw!" << endl; + ::usleep(5000000); + retry = true; + } + } while (retry); - try - { - bs2 = sock.read(&ts); - } - catch (SocketClosed& e) - { - break; - } + ByteStream inBs; + // cout << endl << "startServer is starting" << endl; - inBs += bs2; + // We need to loop here because the big write in mq_1() may (will) not come in one + // read. The other servers will fail if if too much is writen. - while (bs2.length() > 0) - { - //cerr << endl << "startServer: going back for more..." << endl; - try - { - bs2 = sock.read(&ts); - } - catch (SocketClosed& e) - { - break; - } + ByteStream bs2; + struct timespec ts = {0, TS_MS(500)}; + isRunning = true; + IOSocket sock = inMq->accept(); - inBs += bs2; - } + for (;;) + { + inBs.reset(); - //cerr << endl << "startServer: read " << inBs.length() << " bytes" << endl; - if (!keepRunning) break; - - if (inBs.length() > 0) - { - sock.write(inBs); - - if (!keepRunning) break; - } + try + { + bs2 = sock.read(&ts); + } + catch (SocketClosed& e) + { + break; } - delete inMq; - //cerr << endl << "startServer is done" << endl; + inBs += bs2; + + while (bs2.length() > 0) + { + // cerr << endl << "startServer: going back for more..." << endl; + try + { + bs2 = sock.read(&ts); + } + catch (SocketClosed& e) + { + break; + } + + inBs += bs2; + } + + // cerr << endl << "startServer: read " << inBs.length() << " bytes" << endl; + if (!keepRunning) + break; + + if (inBs.length() > 0) + { + sock.write(inBs); + + if (!keepRunning) + break; + } + } + + delete inMq; + // cerr << endl << "startServer is done" << endl; } static void startServer_16() { - MessageQueueServer server("server3", "./Columnstore.xml"); - ByteStream msg; - struct timespec ts = {1, 0}; // 1 second -// const ByteStream::byte *bMsg; -// int i; + MessageQueueServer server("server3", "./Columnstore.xml"); + ByteStream msg; + struct timespec ts = {1, 0}; // 1 second + // const ByteStream::byte *bMsg; + // int i; - isRunning = true; - IOSocket sock = server.accept(); + isRunning = true; + IOSocket sock = server.accept(); - while (keepRunning) + while (keepRunning) + { + // cout << " ... reading" << endl; + try { - -// cout << " ... reading" << endl; - try - { - msg.reset(); - msg = sock.read(&ts); - } - catch (SocketClosed& e) - { - break; - } - - /* - if (msg.length() > 0) { - bMsg = msg.buf(); - cout << "got a message: "; - for (i = 0; i < msg.length(); i++) - cout << bMsg[i] << " "; - cout << endl; - } - */ - if (msg.length() > 0 && keepRunning) - sock.write(msg); + msg.reset(); + msg = sock.read(&ts); + } + catch (SocketClosed& e) + { + break; } - isRunning = false; -// cout << "startserver exiting" << endl; + /* + if (msg.length() > 0) { + bMsg = msg.buf(); + cout << "got a message: "; + for (i = 0; i < msg.length(); i++) + cout << bMsg[i] << " "; + cout << endl; + } + */ + if (msg.length() > 0 && keepRunning) + sock.write(msg); + } + + isRunning = false; + // cout << "startserver exiting" << endl; } static void startServer_17() { - MessageQueueServer server("server3", "./Columnstore.xml"); - server.syncProto(false); - ByteStream msg; - struct timespec ts = {1, 0}; // 1 second -// const ByteStream::byte *bMsg; -// int i; + MessageQueueServer server("server3", "./Columnstore.xml"); + server.syncProto(false); + ByteStream msg; + struct timespec ts = {1, 0}; // 1 second + // const ByteStream::byte *bMsg; + // int i; - isRunning = true; - IOSocket sock = server.accept(); + isRunning = true; + IOSocket sock = server.accept(); - while (keepRunning) + while (keepRunning) + { + // cout << " ... reading" << endl; + try { - -// cout << " ... reading" << endl; - try - { - msg.reset(); - msg = sock.read(&ts); - } - catch (SocketClosed& e) - { - break; - } - - /* - if (msg.length() > 0) { - bMsg = msg.buf(); - cout << "got a message: "; - for (i = 0; i < msg.length(); i++) - cout << bMsg[i] << " "; - cout << endl; - } - */ - if (msg.length() > 0 && keepRunning) - sock.write(msg); + msg.reset(); + msg = sock.read(&ts); + } + catch (SocketClosed& e) + { + break; } - isRunning = false; -// cout << "startserver exiting" << endl; + /* + if (msg.length() > 0) { + bMsg = msg.buf(); + cout << "got a message: "; + for (i = 0; i < msg.length(); i++) + cout << bMsg[i] << " "; + cout << endl; + } + */ + if (msg.length() > 0 && keepRunning) + sock.write(msg); + } + + isRunning = false; + // cout << "startserver exiting" << endl; } struct Serv18thd { - void operator()() + void operator()() + { + ByteStream msg; + struct timespec ts = {1, 0}; + + while (*fKeepRunning) { - ByteStream msg; - struct timespec ts = {1, 0}; + try + { + msg.reset(); + msg = fSock.read(&ts); + } + catch (SocketClosed& e) + { + } - while (*fKeepRunning) - { - try - { - msg.reset(); - msg = fSock.read(&ts); - } - catch (SocketClosed& e) - { - } - - if (msg.length() > 0 && *fKeepRunning) - fSock.write(msg); - } + if (msg.length() > 0 && *fKeepRunning) + fSock.write(msg); } - Serv18thd(const IOSocket& s, volatile bool* kr) : fSock(s), fKeepRunning(kr) {} - ~Serv18thd() {} - IOSocket fSock; - volatile bool* fKeepRunning; + } + Serv18thd(const IOSocket& s, volatile bool* kr) : fSock(s), fKeepRunning(kr) + { + } + ~Serv18thd() + { + } + IOSocket fSock; + volatile bool* fKeepRunning; }; static void startServer_18() { - boost::thread_group tg; - MessageQueueServer server("server3", "./Columnstore.xml"); - struct timespec ts = {1, 0}; - IOSocket sock; - isRunning = true; + boost::thread_group tg; + MessageQueueServer server("server3", "./Columnstore.xml"); + struct timespec ts = {1, 0}; + IOSocket sock; + isRunning = true; - while (keepRunning) - { - sock = server.accept(&ts); + while (keepRunning) + { + sock = server.accept(&ts); - if (sock.socketParms().sd() > -1) - tg.create_thread(Serv18thd(sock, &keepRunning)); - } + if (sock.socketParms().sd() > -1) + tg.create_thread(Serv18thd(sock, &keepRunning)); + } - tg.join_all(); - isRunning = false; + tg.join_all(); + isRunning = false; } static void startBrokenServer() { - MessageQueueServer* inMq; - bool retry; + MessageQueueServer* inMq; + bool retry; - do + do + { + try { - try - { - retry = false; - inMq = new MessageQueueServer(brokeServ, "./Columnstore.xml"); - } - catch (...) - { - //cout << endl << "MessageQueueServer ctor threw!" << endl; - ::usleep(5000000); - retry = true; - } + retry = false; + inMq = new MessageQueueServer(brokeServ, "./Columnstore.xml"); } - while (retry); - - ByteStream inBs; - struct timespec ts = { 0, TS_MS(20) }; - //cout << endl << "startServer is starting" << endl; - - isRunning = true; - IOSocket sock = inMq->accept(); - - for (;;) + catch (...) { - try - { - inBs = sock.read(&ts); - } - catch (SocketClosed& e) - { - break; - } + // cout << endl << "MessageQueueServer ctor threw!" << endl; + ::usleep(5000000); + retry = true; + } + } while (retry); - if (!keepRunning) break; + ByteStream inBs; + struct timespec ts = {0, TS_MS(20)}; + // cout << endl << "startServer is starting" << endl; + + isRunning = true; + IOSocket sock = inMq->accept(); + + for (;;) + { + try + { + inBs = sock.read(&ts); + } + catch (SocketClosed& e) + { + break; } - delete inMq; - //cout << endl << "startServer is done" << endl; + if (!keepRunning) + break; + } + + delete inMq; + // cout << endl << "startServer is done" << endl; } static void startWriteServer() { - MessageQueueServer* inMq; - bool retry; + MessageQueueServer* inMq; + bool retry; - do + do + { + try { - try - { - retry = false; - inMq = new MessageQueueServer(writeServ, "./Columnstore.xml"); - } - catch (...) - { - //cout << endl << "MessageQueueServer ctor threw!" << endl; - ::usleep(5000000); - retry = true; - } + retry = false; + inMq = new MessageQueueServer(writeServ, "./Columnstore.xml"); } - while (retry); + catch (...) + { + // cout << endl << "MessageQueueServer ctor threw!" << endl; + ::usleep(5000000); + retry = true; + } + } while (retry); - isRunning = true; + isRunning = true; - string msg = "This is a test"; - ByteStream outBs; - outBs.load(reinterpret_cast(msg.c_str()), msg.length()); + string msg = "This is a test"; + ByteStream outBs; + outBs.load(reinterpret_cast(msg.c_str()), msg.length()); - IOSocket sock = inMq->accept(); - sock.write(outBs); + IOSocket sock = inMq->accept(); + sock.write(outBs); - while (keepRunning) - ::usleep(10000000); + while (keepRunning) + ::usleep(10000000); - delete inMq; - //cout << endl << "writeServer is done" << endl; + delete inMq; + // cout << endl << "writeServer is done" << endl; } - class MessageQTestSuite : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(MessageQTestSuite); - CPPUNIT_TEST_SUITE( MessageQTestSuite ); + CPPUNIT_TEST(mq_1); + CPPUNIT_TEST(mq_2); + CPPUNIT_TEST(mq_8); + CPPUNIT_TEST_EXCEPTION(mq_3, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(mq_4, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(mq_5, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(mq_6, std::runtime_error); + CPPUNIT_TEST_EXCEPTION(mq_7, std::runtime_error); + CPPUNIT_TEST(mq_9); + CPPUNIT_TEST(mq_10); + CPPUNIT_TEST(mq_12); + CPPUNIT_TEST_EXCEPTION(mq_13a, std::logic_error); + CPPUNIT_TEST_EXCEPTION(mq_14, std::runtime_error); + CPPUNIT_TEST(mq_15); + CPPUNIT_TEST(mq_16); // test the fix for bug #224 + CPPUNIT_TEST(mq_17); + CPPUNIT_TEST(mq_18); + CPPUNIT_TEST(mq_19); - CPPUNIT_TEST( mq_1 ); - CPPUNIT_TEST( mq_2 ); - CPPUNIT_TEST( mq_8 ); - CPPUNIT_TEST_EXCEPTION( mq_3, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( mq_4, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( mq_5, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( mq_6, std::runtime_error ); - CPPUNIT_TEST_EXCEPTION( mq_7, std::runtime_error ); - CPPUNIT_TEST( mq_9 ); - CPPUNIT_TEST( mq_10 ); - CPPUNIT_TEST( mq_12 ); - CPPUNIT_TEST_EXCEPTION( mq_13a, std::logic_error ); - CPPUNIT_TEST_EXCEPTION( mq_14, std::runtime_error ); - CPPUNIT_TEST( mq_15 ); - CPPUNIT_TEST( mq_16 ); // test the fix for bug #224 - CPPUNIT_TEST( mq_17 ); - CPPUNIT_TEST( mq_18 ); - CPPUNIT_TEST( mq_19 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + ByteStream bs; + ByteStream bs1; + boost::thread* srvThread; -private: - ByteStream bs; - ByteStream bs1; - boost::thread* srvThread; + public: + void setUp() + { + bs.reset(); + bs1.reset(); + srvThread = 0; + // setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); + } -public: - void setUp() + void tearDown() + { + bs.reset(); + bs1.reset(); + delete srvThread; + srvThread = 0; + } + + void mq_1() + { + keepRunning = true; + isRunning = false; + normServ = "server1"; + srvThread = new boost::thread(startServer); + + while (!isRunning) { - bs.reset(); - bs1.reset(); - srvThread = 0; - //setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); + // cout << endl << "waiting for startServer" << endl; + ::usleep(2500000); } - void tearDown() - { - bs.reset(); - bs1.reset(); - delete srvThread; - srvThread = 0; - } + Config* cf = Config::makeConfig("./Columnstore.xml"); + MessageQueueClient outMq(normServ, cf); + string msg = "This is a test"; + ByteStream outBs; + outBs.load(reinterpret_cast(msg.c_str()), msg.length()); + // cerr << endl << "mq_1: write " << outBs.length() << " bytes" << endl; + outMq.write(outBs); - void mq_1() - { - keepRunning = true; - isRunning = false; - normServ = "server1"; - srvThread = new boost::thread(startServer); + ByteStream inBs; + inBs = outMq.read(); - while (!isRunning) - { - //cout << endl << "waiting for startServer" << endl; - ::usleep(2500000); - } + // cerr << endl << "mq_1: read " << inBs.length() << " bytes" << endl; + CPPUNIT_ASSERT(outBs.length() == inBs.length()); + CPPUNIT_ASSERT(memcmp(outBs.buf(), inBs.buf(), outBs.length()) == 0); - Config* cf = Config::makeConfig("./Columnstore.xml"); - MessageQueueClient outMq(normServ, cf); - string msg = "This is a test"; - ByteStream outBs; - outBs.load(reinterpret_cast(msg.c_str()), msg.length()); - //cerr << endl << "mq_1: write " << outBs.length() << " bytes" << endl; - outMq.write(outBs); + int i; + ByteStream::byte u8; - ByteStream inBs; - inBs = outMq.read(); + u8 = 0xa5; - //cerr << endl << "mq_1: read " << inBs.length() << " bytes" << endl; - CPPUNIT_ASSERT(outBs.length() == inBs.length()); - CPPUNIT_ASSERT(memcmp(outBs.buf(), inBs.buf(), outBs.length()) == 0); + for (i = 0; i < 2048; i++) + bs << u8; - int i; - ByteStream::byte u8; - - u8 = 0xa5; - - for (i = 0; i < 2048; i++) - bs << u8; - - //cerr << endl << "mq_1: write " << bs.length() << " bytes" << endl; - outMq.write(bs); - bs1 = outMq.read(); + // cerr << endl << "mq_1: write " << bs.length() << " bytes" << endl; + outMq.write(bs); + bs1 = outMq.read(); #if 0 if (bs.length() != bs1.length()) cerr << endl << "bs.length() = " << bs.length() << ", bs1.length() = " << bs1.length() << endl; #endif - CPPUNIT_ASSERT(bs.length() == bs1.length()); - CPPUNIT_ASSERT(memcmp(bs.buf(), bs1.buf(), bs.length()) == 0); + CPPUNIT_ASSERT(bs.length() == bs1.length()); + CPPUNIT_ASSERT(memcmp(bs.buf(), bs1.buf(), bs.length()) == 0); - bs.reset(); - u8 = 0x5a; + bs.reset(); + u8 = 0x5a; - for (i = 0; i < 2048; i++) - bs << u8; + for (i = 0; i < 2048; i++) + bs << u8; - bs << u8; - bs << u8; - bs << u8; + bs << u8; + bs << u8; + bs << u8; - //cerr << endl << "mq_1: write " << bs.length() << " bytes" << endl; - outMq.write(bs); - bs1 = outMq.read(); + // cerr << endl << "mq_1: write " << bs.length() << " bytes" << endl; + outMq.write(bs); + bs1 = outMq.read(); #if 0 if (bs.length() != bs1.length()) cerr << endl << "bs.length() = " << bs.length() << ", bs1.length() = " << bs1.length() << endl; #endif - CPPUNIT_ASSERT(bs.length() == bs1.length()); - CPPUNIT_ASSERT(memcmp(bs.buf(), bs1.buf(), bs.length()) == 0); + CPPUNIT_ASSERT(bs.length() == bs1.length()); + CPPUNIT_ASSERT(memcmp(bs.buf(), bs1.buf(), bs.length()) == 0); - // Now write a really big message and see what happens... + // Now write a really big message and see what happens... - ByteStream::quadbyte u32; + ByteStream::quadbyte u32; - bs.reset(); - u32 = 0xdeadbeef; + bs.reset(); + u32 = 0xdeadbeef; - for (i = 0; i < (1048576 / 4); i++) - bs << u32; + for (i = 0; i < (1048576 / 4); i++) + bs << u32; - //cerr << endl << "mq_1: write " << bs.length() << " bytes" << endl; - outMq.write(bs); - ByteStream bs2; - struct timespec ts = { 5, TS_MS(0) }; + // cerr << endl << "mq_1: write " << bs.length() << " bytes" << endl; + outMq.write(bs); + ByteStream bs2; + struct timespec ts = {5, TS_MS(0)}; - if (leakCheck) ts.tv_sec *= 20; + if (leakCheck) + ts.tv_sec *= 20; - bs1.reset(); - bs2 = outMq.read(&ts); - bs1 += bs2; + bs1.reset(); + bs2 = outMq.read(&ts); + bs1 += bs2; - while (bs2.length() > 0 && bs1.length() < bs.length()) - { - //cerr << endl << "going back for more..." << endl; - bs2 = outMq.read(&ts); - bs1 += bs2; - } + while (bs2.length() > 0 && bs1.length() < bs.length()) + { + // cerr << endl << "going back for more..." << endl; + bs2 = outMq.read(&ts); + bs1 += bs2; + } #if 0 @@ -1389,414 +1390,408 @@ public: cerr << endl << "bs.length() = " << bs.length() << ", bs1.length() = " << bs1.length() << endl; #endif - CPPUNIT_ASSERT(bs.length() == bs1.length()); - CPPUNIT_ASSERT(memcmp(bs.buf(), bs1.buf(), bs.length()) == 0); + CPPUNIT_ASSERT(bs.length() == bs1.length()); + CPPUNIT_ASSERT(memcmp(bs.buf(), bs1.buf(), bs.length()) == 0); - keepRunning = false; - outMq.shutdown(); + keepRunning = false; + outMq.shutdown(); - srvThread->join(); - delete srvThread; - srvThread = 0; - Config::deleteInstanceMap(); - } + srvThread->join(); + delete srvThread; + srvThread = 0; + Config::deleteInstanceMap(); + } - void mq_2() + void mq_2() + { + keepRunning = true; + isRunning = false; + brokeServ = "server2"; + srvThread = new boost::thread(startBrokenServer); + + while (!isRunning) { - keepRunning = true; - isRunning = false; - brokeServ = "server2"; - srvThread = new boost::thread(startBrokenServer); - - while (!isRunning) - { - //cout << endl << "waiting for startBrokenServer" << endl; - ::usleep(2500000); - } - - struct timespec ts = { 0, TS_MS(20) }; - - MessageQueueClient outMq(brokeServ, "./Columnstore.xml"); - - bs1 = outMq.read(&ts); - - keepRunning = false; - - outMq.shutdown(); - - srvThread->join(); - - delete srvThread; - - srvThread = 0; - - Config::deleteInstanceMap(); + // cout << endl << "waiting for startBrokenServer" << endl; + ::usleep(2500000); } - void mq_3() + struct timespec ts = {0, TS_MS(20)}; + + MessageQueueClient outMq(brokeServ, "./Columnstore.xml"); + + bs1 = outMq.read(&ts); + + keepRunning = false; + + outMq.shutdown(); + + srvThread->join(); + + delete srvThread; + + srvThread = 0; + + Config::deleteInstanceMap(); + } + + void mq_3() + { + // Should throw runtime_exception for missing info + // setenv("CALPONT_CONFIG_FILE", "./bogus.xml", 1); + MessageQueueServer* outMq = new MessageQueueServer("ExeMgr", "./bogus.xml"); + + CPPUNIT_ASSERT(0); + IOSocket sock = outMq->accept(); + bs1 = sock.read(); + delete outMq; + Config::deleteInstanceMap(); + } + + void mq_4() + { + boost::scoped_ptr outMq(new MessageQueueClient("server4", "./Columnstore.xml")); + // Should throw runtime_exception for connect failed + bs1 = outMq->read(); + + CPPUNIT_ASSERT(0); + Config::deleteInstanceMap(); + } + + void mq_5() + { + boost::scoped_ptr outMq(new MessageQueueClient("server4", "./Columnstore.xml")); + string msg = "This is a test"; + bs1.load(reinterpret_cast(msg.c_str()), msg.length()); + // Should throw runtime_exception for connect failed + outMq->write(bs1); + + CPPUNIT_ASSERT(0); + Config::deleteInstanceMap(); + } + + void mq_6() + { + Config* cf = Config::makeConfig("./Columnstore.xml"); + boost::scoped_ptr outMq(new MessageQueueServer("server4", cf)); + // Should throw runtime_exception for addr in use + MessageQueueServer* outMq1 = new MessageQueueServer("server4", cf); + + CPPUNIT_ASSERT(0); + delete outMq1; + Config::deleteInstanceMap(); + } + + void mq_7() + { + // Should throw runtime_exception for missing info + setenv("CALPONT_CONFIG_FILE", "./bogus.xml", 1); + MessageQueueClient* outMq = new MessageQueueClient("ExeMgr", "./bogus.xml"); + + CPPUNIT_ASSERT(0); + bs1 = outMq->read(); + delete outMq; + Config::deleteInstanceMap(); + } + + void mq_8() + { + keepRunning = true; + isRunning = false; + writeServ = "server3"; + srvThread = new boost::thread(startWriteServer); + + while (!isRunning) { - // Should throw runtime_exception for missing info - //setenv("CALPONT_CONFIG_FILE", "./bogus.xml", 1); - MessageQueueServer* outMq = new MessageQueueServer("ExeMgr", "./bogus.xml"); - - CPPUNIT_ASSERT(0); - IOSocket sock = outMq->accept(); - bs1 = sock.read(); - delete outMq; - Config::deleteInstanceMap(); + // cout << endl << "waiting for startWriteServer" << endl; + ::usleep(2500000); } - void mq_4() + MessageQueueClient outMq(writeServ, "./Columnstore.xml"); + bs1 = outMq.read(); + CPPUNIT_ASSERT(memcmp(bs1.buf(), "This is a test", bs1.length()) == 0); + + outMq.shutdown(); + keepRunning = false; + + srvThread->join(); + delete srvThread; + srvThread = 0; + Config::deleteInstanceMap(); + } + + // Bug 1735: I don't know how this function "used" to work, or it's intent, + // but it now encounters 2 exceptions, that were not accounted for. + // I added code to log the exceptions and keep going. + // Somebody can investigate further at some point if they like. + void mq_9() + { + InetStreamSocket* iss; + boost::scoped_ptr mq1(new MessageQueueServer("server5", "./Columnstore.xml")); + struct timespec ts = {0, TS_MS(200)}; + // this should block in accept() for ts + IOSocket sock = mq1->accept(&ts); + // Set a bogus fd + int fd; + fd = open("/dev/null", O_RDONLY); + close(fd); + // mq1->fClientSock.fSocket->fSd = fd; + iss = dynamic_cast(mq1->fClientSock.fSocket); + iss->fSocketParms.fSd = fd; + + // mqs::read() will catch a runtime_error and return a zero bs + try { - boost::scoped_ptr outMq(new MessageQueueClient("server4", "./Columnstore.xml")); - // Should throw runtime_exception for connect failed - bs1 = outMq->read(); - - CPPUNIT_ASSERT(0); - Config::deleteInstanceMap(); + sock.read(); } - - void mq_5() + catch (runtime_error& ex) // Bug 1735 { - boost::scoped_ptr outMq(new MessageQueueClient("server4", "./Columnstore.xml")); - string msg = "This is a test"; - bs1.load(reinterpret_cast(msg.c_str()), msg.length()); - // Should throw runtime_exception for connect failed - outMq->write(bs1); - - CPPUNIT_ASSERT(0); - Config::deleteInstanceMap(); + cerr << "Runtime error (OK)..." << ex.what() << endl; } - void mq_6() + boost::scoped_ptr mq2(new MessageQueueClient("server5", "./Columnstore.xml")); + bs.reset(); + bs << "This is a test"; + // close(mq2->fClientSock.fSocketParms.fSd); + iss = dynamic_cast(mq2->fClientSock.fSocket); + close(iss->fSocketParms.fSd); + + try { - Config* cf = Config::makeConfig("./Columnstore.xml"); - boost::scoped_ptr outMq(new MessageQueueServer("server4", cf)); - // Should throw runtime_exception for addr in use - MessageQueueServer* outMq1 = new MessageQueueServer("server4", cf); - - CPPUNIT_ASSERT(0); - delete outMq1; - Config::deleteInstanceMap(); + mq2->write(bs); } - - void mq_7() + catch (runtime_error& ex) // Bug 1735 { - // Should throw runtime_exception for missing info - setenv("CALPONT_CONFIG_FILE", "./bogus.xml", 1); - MessageQueueClient* outMq = new MessageQueueClient("ExeMgr", "./bogus.xml"); - - CPPUNIT_ASSERT(0); - bs1 = outMq->read(); - delete outMq; - Config::deleteInstanceMap(); + cerr << "Connection error (OK)..." << ex.what() << endl; } - void mq_8() - { - keepRunning = true; - isRunning = false; - writeServ = "server3"; - srvThread = new boost::thread(startWriteServer); + bs.reset(); + mq2->shutdown(); + // Should not throw runtime_error + sock.read(&ts); + Config::deleteInstanceMap(); + } - while (!isRunning) - { - //cout << endl << "waiting for startWriteServer" << endl; - ::usleep(2500000); - } + void mq_10() + { + SocketParms s; - MessageQueueClient outMq(writeServ, "./Columnstore.xml"); - bs1 = outMq.read(); - CPPUNIT_ASSERT(memcmp(bs1.buf(), "This is a test", bs1.length()) == 0); + s.sd(0); + CPPUNIT_ASSERT(s.sd() == 0); + s.sd(-1); + CPPUNIT_ASSERT(s.sd() == -1); + s.domain(1); + CPPUNIT_ASSERT(s.domain() == 1); + s.type(2); + CPPUNIT_ASSERT(s.type() == 2); + s.protocol(3); + CPPUNIT_ASSERT(s.protocol() == 3); - outMq.shutdown(); - keepRunning = false; + SocketParms s1(s); + CPPUNIT_ASSERT(s1.sd() == -1); + CPPUNIT_ASSERT(s1.domain() == 1); + CPPUNIT_ASSERT(s1.type() == 2); + CPPUNIT_ASSERT(s1.protocol() == 3); - srvThread->join(); - delete srvThread; - srvThread = 0; - Config::deleteInstanceMap(); - } + SocketParms s2; + s2 = s; + CPPUNIT_ASSERT(s2.sd() == -1); + CPPUNIT_ASSERT(s2.domain() == 1); + CPPUNIT_ASSERT(s2.type() == 2); + CPPUNIT_ASSERT(s2.protocol() == 3); + Config::deleteInstanceMap(); + } -// Bug 1735: I don't know how this function "used" to work, or it's intent, -// but it now encounters 2 exceptions, that were not accounted for. -// I added code to log the exceptions and keep going. -// Somebody can investigate further at some point if they like. - void mq_9() - { - InetStreamSocket* iss; - boost::scoped_ptr mq1(new MessageQueueServer("server5", "./Columnstore.xml")); - struct timespec ts = { 0, TS_MS(200) }; - // this should block in accept() for ts - IOSocket sock = mq1->accept(&ts); - // Set a bogus fd - int fd; - fd = open("/dev/null", O_RDONLY); - close(fd); - //mq1->fClientSock.fSocket->fSd = fd; - iss = dynamic_cast(mq1->fClientSock.fSocket); - iss->fSocketParms.fSd = fd; + void mq_12() + { + InetStreamSocket iss; + iss.fSocketParms.sd(12345); + InetStreamSocket iss1; + iss1 = iss; + CPPUNIT_ASSERT(iss1.socketParms().sd() == iss.socketParms().sd()); + InetStreamSocket iss2(iss1); + CPPUNIT_ASSERT(iss1.socketParms().sd() == iss2.socketParms().sd()); + Config::deleteInstanceMap(); + } - // mqs::read() will catch a runtime_error and return a zero bs - try - { - sock.read(); - } - catch (runtime_error& ex) // Bug 1735 - { - cerr << "Runtime error (OK)..." << ex.what() << endl; - } + void mq_13a() + { + InetStreamSocket iss; + iss.fSocketParms.sd(0); + iss.open(); + } - boost::scoped_ptr mq2(new MessageQueueClient("server5", "./Columnstore.xml")); - bs.reset(); - bs << "This is a test"; - //close(mq2->fClientSock.fSocketParms.fSd); - iss = dynamic_cast(mq2->fClientSock.fSocket); - close(iss->fSocketParms.fSd); + void mq_14() + { + InetStreamSocket* iss; + boost::scoped_ptr mq1(new MessageQueueServer("server6", "./Columnstore.xml")); + // Set a bogus fd + int fd; + fd = open("/dev/null", O_RDONLY); + close(fd); + // mq1->fListenSock.fSocketParms.fSd = fd; + iss = dynamic_cast(mq1->fListenSock.fSocket); + iss->fSocketParms.fSd = fd; + // should throw in accept(); + IOSocket sock = mq1->accept(); + sock.read(); + } - try - { - mq2->write(bs); - } - catch (runtime_error& ex) // Bug 1735 - { - cerr << "Connection error (OK)..." << ex.what() << endl; - } + void mq_15() + { + IOSocket ios(new InetStreamSocket()); + string oss; + oss = ios.toString(); + // CPPUNIT_ASSERT(oss == "IOSocket: sd: -1 domain: 2 type: 1 protocol: 0 inet: 0.0.0.0"); + CPPUNIT_ASSERT(oss.length() > 0); + } - bs.reset(); - mq2->shutdown(); - // Should not throw runtime_error - sock.read(&ts); - Config::deleteInstanceMap(); - } + void mq_16() + { + const char msg1[] = "Message 1"; + const char msg2[] = "message 2"; + string sTmp; + MessageQueueClient client("server3", "./Columnstore.xml"); + ByteStream bs, bs2; + char buf[1000]; + int len, err, socketfd; + struct timespec ts = {2, 0}; // 2 seconds for client, 1 for server - void mq_10() - { - SocketParms s; + isRunning = false; + keepRunning = true; + srvThread = new boost::thread(startServer_16); - s.sd(0); - CPPUNIT_ASSERT(s.sd() == 0); - s.sd(-1); - CPPUNIT_ASSERT(s.sd() == -1); - s.domain(1); - CPPUNIT_ASSERT(s.domain() == 1); - s.type(2); - CPPUNIT_ASSERT(s.type() == 2); - s.protocol(3); - CPPUNIT_ASSERT(s.protocol() == 3); + while (!isRunning) + usleep(250000); - SocketParms s1(s); - CPPUNIT_ASSERT(s1.sd() == -1); - CPPUNIT_ASSERT(s1.domain() == 1); - CPPUNIT_ASSERT(s1.type() == 2); - CPPUNIT_ASSERT(s1.protocol() == 3); + // connect + bs << (uint8_t)1; + client.write(bs); + bs = client.read(); - SocketParms s2; - s2 = s; - CPPUNIT_ASSERT(s2.sd() == -1); - CPPUNIT_ASSERT(s2.domain() == 1); - CPPUNIT_ASSERT(s2.type() == 2); - CPPUNIT_ASSERT(s2.protocol() == 3); - Config::deleteInstanceMap(); + // grab the server's FD for the client, fake a partial ByteStream being written + socketfd = client.fClientSock.socketParms().sd(); + len = strlen(msg1) + 1; + memcpy(buf, &BYTESTREAM_MAGIC, 4); + memcpy(&buf[4], &len, 4); + memcpy(&buf[8], msg1, len); + err = write(socketfd, buf, 12); // only write the first 4 bytes of msg1 + CPPUNIT_ASSERT(err >= 0); - } + // verify the partial message is dropped + bs = client.read(&ts); + CPPUNIT_ASSERT(bs.length() == 0); - void mq_12() - { - InetStreamSocket iss; - iss.fSocketParms.sd(12345); - InetStreamSocket iss1; - iss1 = iss; - CPPUNIT_ASSERT(iss1.socketParms().sd() == iss.socketParms().sd()); - InetStreamSocket iss2(iss1); - CPPUNIT_ASSERT(iss1.socketParms().sd() == iss2.socketParms().sd()); - Config::deleteInstanceMap(); - } + // write the rest of the message + err = write(socketfd, &buf[12], len - 4); - void mq_13a() - { - InetStreamSocket iss; - iss.fSocketParms.sd(0); - iss.open(); - } + // write a full ByteStream, verify that only it is received + bs << msg2; + client.write(bs); + bs2 = client.read(&ts); + CPPUNIT_ASSERT(bs == bs2); + keepRunning = false; + srvThread->join(); + delete srvThread; + srvThread = NULL; + Config::deleteInstanceMap(); + } - void mq_14() - { - InetStreamSocket* iss; - boost::scoped_ptr mq1(new MessageQueueServer("server6", "./Columnstore.xml")); - // Set a bogus fd - int fd; - fd = open("/dev/null", O_RDONLY); - close(fd); - //mq1->fListenSock.fSocketParms.fSd = fd; - iss = dynamic_cast(mq1->fListenSock.fSocket); - iss->fSocketParms.fSd = fd; - // should throw in accept(); - IOSocket sock = mq1->accept(); - sock.read(); + void mq_17() + { + const char msg1[] = "Message 1"; + const char msg2[] = "message 2"; + string sTmp; + MessageQueueClient client("server3", "./Columnstore.xml"); + client.syncProto(false); + ByteStream bs, bs2; + char buf[1000]; + int len, err, socketfd; + struct timespec ts = {2, 0}; // 2 seconds for client, 1 for server - } + isRunning = false; + keepRunning = true; + srvThread = new boost::thread(startServer_17); - void mq_15() - { - IOSocket ios(new InetStreamSocket()); - string oss; - oss = ios.toString(); - //CPPUNIT_ASSERT(oss == "IOSocket: sd: -1 domain: 2 type: 1 protocol: 0 inet: 0.0.0.0"); - CPPUNIT_ASSERT(oss.length() > 0); - } + while (!isRunning) + usleep(250000); - void mq_16() - { + // connect + bs << (uint8_t)1; + client.write(bs); + bs = client.read(); - const char msg1[] = "Message 1"; - const char msg2[] = "message 2"; - string sTmp; - MessageQueueClient client("server3", "./Columnstore.xml"); - ByteStream bs, bs2; - char buf[1000]; - int len, err, socketfd; - struct timespec ts = {2, 0}; // 2 seconds for client, 1 for server + // grab the server's FD for the client, fake a partial ByteStream being written + socketfd = client.fClientSock.socketParms().sd(); + len = strlen(msg1) + 1; + memcpy(buf, &BYTESTREAM_MAGIC, 4); + memcpy(&buf[4], &len, 4); + memcpy(&buf[8], msg1, len); + err = write(socketfd, buf, 12); // only write the first 4 bytes of msg1 + CPPUNIT_ASSERT(err >= 0); - isRunning = false; - keepRunning = true; - srvThread = new boost::thread(startServer_16); + // verify the partial message is dropped + bs = client.read(&ts); + CPPUNIT_ASSERT(bs.length() == 0); - while (!isRunning) - usleep(250000); + // write the rest of the message + err = write(socketfd, &buf[12], len - 4); - //connect - bs << (uint8_t) 1; - client.write(bs); - bs = client.read(); + // write a full ByteStream, verify that only it is received + bs << msg2; + client.write(bs); + bs2 = client.read(&ts); + CPPUNIT_ASSERT(bs == bs2); + keepRunning = false; + srvThread->join(); + delete srvThread; + srvThread = NULL; + Config::deleteInstanceMap(); + } - // grab the server's FD for the client, fake a partial ByteStream being written - socketfd = client.fClientSock.socketParms().sd(); - len = strlen(msg1) + 1; - memcpy(buf, &BYTESTREAM_MAGIC, 4); - memcpy(&buf[4], &len, 4); - memcpy(&buf[8], msg1, len); - err = write(socketfd, buf, 12); // only write the first 4 bytes of msg1 - CPPUNIT_ASSERT(err >= 0); + void mq_18() + { + MessageQueueClient client1("server3", "./Columnstore.xml"); + MessageQueueClient client2("server3", "./Columnstore.xml"); - // verify the partial message is dropped - bs = client.read(&ts); - CPPUNIT_ASSERT(bs.length() == 0); + isRunning = false; + keepRunning = true; + srvThread = new boost::thread(startServer_18); - // write the rest of the message - err = write(socketfd, &buf[12], len - 4); + while (!isRunning) + usleep(250000); - // write a full ByteStream, verify that only it is received - bs << msg2; - client.write(bs); - bs2 = client.read(&ts); - CPPUNIT_ASSERT(bs == bs2); - keepRunning = false; - srvThread->join(); - delete srvThread; - srvThread = NULL; - Config::deleteInstanceMap(); - } + // connect + bs << (uint8_t)1; + client1.write(bs); + bs = client1.read(); + bs << (uint8_t)1; + client2.write(bs); + bs = client2.read(); - void mq_17() - { + // + CPPUNIT_ASSERT(client1.isSameAddr(client2)); - const char msg1[] = "Message 1"; - const char msg2[] = "message 2"; - string sTmp; - MessageQueueClient client("server3", "./Columnstore.xml"); - client.syncProto(false); - ByteStream bs, bs2; - char buf[1000]; - int len, err, socketfd; - struct timespec ts = {2, 0}; // 2 seconds for client, 1 for server + CPPUNIT_ASSERT(client1.addr2String() == "127.0.0.1"); - isRunning = false; - keepRunning = true; - srvThread = new boost::thread(startServer_17); - - while (!isRunning) - usleep(250000); - - //connect - bs << (uint8_t) 1; - client.write(bs); - bs = client.read(); - - // grab the server's FD for the client, fake a partial ByteStream being written - socketfd = client.fClientSock.socketParms().sd(); - len = strlen(msg1) + 1; - memcpy(buf, &BYTESTREAM_MAGIC, 4); - memcpy(&buf[4], &len, 4); - memcpy(&buf[8], msg1, len); - err = write(socketfd, buf, 12); // only write the first 4 bytes of msg1 - CPPUNIT_ASSERT(err >= 0); - - // verify the partial message is dropped - bs = client.read(&ts); - CPPUNIT_ASSERT(bs.length() == 0); - - // write the rest of the message - err = write(socketfd, &buf[12], len - 4); - - // write a full ByteStream, verify that only it is received - bs << msg2; - client.write(bs); - bs2 = client.read(&ts); - CPPUNIT_ASSERT(bs == bs2); - keepRunning = false; - srvThread->join(); - delete srvThread; - srvThread = NULL; - Config::deleteInstanceMap(); - } - - void mq_18() - { - - MessageQueueClient client1("server3", "./Columnstore.xml"); - MessageQueueClient client2("server3", "./Columnstore.xml"); - - isRunning = false; - keepRunning = true; - srvThread = new boost::thread(startServer_18); - - while (!isRunning) - usleep(250000); - - //connect - bs << (uint8_t) 1; - client1.write(bs); - bs = client1.read(); - bs << (uint8_t) 1; - client2.write(bs); - bs = client2.read(); - - // - CPPUNIT_ASSERT(client1.isSameAddr(client2)); - - CPPUNIT_ASSERT(client1.addr2String() == "127.0.0.1"); - - keepRunning = false; - srvThread->join(); - delete srvThread; - srvThread = NULL; - Config::deleteInstanceMap(); - } - - void mq_19() - { - CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.1", 0) == 0); - CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.254", 0) == -1); - struct timespec ts = {20, 0}; - CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.1", &ts) == 0); - CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.254", &ts) == -1); - } + keepRunning = false; + srvThread->join(); + delete srvThread; + srvThread = NULL; + Config::deleteInstanceMap(); + } + void mq_19() + { + CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.1", 0) == 0); + CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.254", 0) == -1); + struct timespec ts = {20, 0}; + CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.1", &ts) == 0); + CPPUNIT_ASSERT(InetStreamSocket::ping("10.100.4.254", &ts) == -1); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( ByteStreamTestSuite ); -CPPUNIT_TEST_SUITE_REGISTRATION( MessageQTestSuite ); +CPPUNIT_TEST_SUITE_REGISTRATION(ByteStreamTestSuite); +CPPUNIT_TEST_SUITE_REGISTRATION(MessageQTestSuite); #include #include @@ -1805,26 +1800,26 @@ CPPUNIT_TEST_SUITE_REGISTRATION( MessageQTestSuite ); void setupSignalHandlers() { - struct sigaction ign; + struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); + sigaction(SIGPIPE, &ign, 0); } -int main( int argc, char** argv) +int main(int argc, char** argv) { - setupSignalHandlers(); + setupSignalHandlers(); - leakCheck = false; + leakCheck = false; - if (argc > 1 && strcmp(argv[1], "--leakcheck") == 0) leakCheck = true; + if (argc > 1 && strcmp(argv[1], "--leakcheck") == 0) + leakCheck = true; - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - diff --git a/utils/multicast/impl.cpp b/utils/multicast/impl.cpp index eb50a344c..09ca6f854 100644 --- a/utils/multicast/impl.cpp +++ b/utils/multicast/impl.cpp @@ -41,220 +41,220 @@ using namespace messageqcpp; #include "impl.h" -struct participantsDb +struct participantsDb { - int nrParticipants; + int nrParticipants; - struct clientDesc - { - struct sockaddr_in addr; - int used; - int capabilities; - unsigned int rcvbuf; - } clientTable[MAX_CLIENTS]; + struct clientDesc + { + struct sockaddr_in addr; + int used; + int capabilities; + unsigned int rcvbuf; + } clientTable[MAX_CLIENTS]; }; struct produconsum { - unsigned int size; - volatile unsigned int produced; - unsigned int consumed; - volatile int atEnd; - pthread_mutex_t mutex; - volatile int consumerIsWaiting; - pthread_cond_t cond; - const char* name; + unsigned int size; + volatile unsigned int produced; + unsigned int consumed; + volatile int atEnd; + pthread_mutex_t mutex; + volatile int consumerIsWaiting; + pthread_cond_t cond; + const char* name; }; struct stats { - int fd; - struct timeval lastPrinted; - long statPeriod; - int printUncompressedPos; + int fd; + struct timeval lastPrinted; + long statPeriod; + int printUncompressedPos; }; struct sender_stats { - FILE* log; - unsigned long long totalBytes; - unsigned long long retransmissions; - int clNo; - unsigned long periodBytes; - struct timeval periodStart; - long bwPeriod; - struct stats s; + FILE* log; + unsigned long long totalBytes; + unsigned long long retransmissions; + int clNo; + unsigned long periodBytes; + struct timeval periodStart; + long bwPeriod; + struct stats s; }; struct receiver_stats { - struct timeval tv_start; - int bytesOrig; - long long totalBytes; - int timerStarted; - struct stats s; + struct timeval tv_start; + int bytesOrig; + long long totalBytes; + int timerStarted; + struct stats s; }; #define SLICEMAGIC 0x41424344 typedef struct slice { - int base; /* base address of slice in buffer */ - int sliceNo; - int bytes; /* bytes in slice */ - int nextBlock; /* index of next buffer to be transmitted */ - enum slice_state - { - SLICE_FREE, /* free slice, and in the queue of free slices */ - SLICE_NEW, /* newly allocated. FEC calculation and first - * transmission */ - SLICE_XMITTED, /* transmitted */ - SLICE_ACKED, /* acknowledged (if applicable) */ - SLICE_PRE_FREE, /* no longer used, but not returned to queue */ - SLICE_RECEIVING, - SLICE_DONE, - }; - volatile enum slice_state state; - char rxmitMap[MAX_SLICE_SIZE / BITS_PER_CHAR]; - /* blocks to be retransmitted */ + int base; /* base address of slice in buffer */ + int sliceNo; + int bytes; /* bytes in slice */ + int nextBlock; /* index of next buffer to be transmitted */ + enum slice_state + { + SLICE_FREE, /* free slice, and in the queue of free slices */ + SLICE_NEW, /* newly allocated. FEC calculation and first + * transmission */ + SLICE_XMITTED, /* transmitted */ + SLICE_ACKED, /* acknowledged (if applicable) */ + SLICE_PRE_FREE, /* no longer used, but not returned to queue */ + SLICE_RECEIVING, + SLICE_DONE, + }; + volatile enum slice_state state; + char rxmitMap[MAX_SLICE_SIZE / BITS_PER_CHAR]; + /* blocks to be retransmitted */ - char isXmittedMap[MAX_SLICE_SIZE / BITS_PER_CHAR]; - /* blocks which have already been retransmitted during this round*/ + char isXmittedMap[MAX_SLICE_SIZE / BITS_PER_CHAR]; + /* blocks which have already been retransmitted during this round*/ - int rxmitId; /* used to distinguish among several retransmission - * requests, so that we can easily discard answers to "old" - * requests */ + int rxmitId; /* used to distinguish among several retransmission + * requests, so that we can easily discard answers to "old" + * requests */ - /* This structure is used to keep track of clients who answered, and - * to make the reqack message - */ - struct reqackBm - { - struct reqack ra; - char readySet[MAX_CLIENTS / BITS_PER_CHAR]; /* who is already ok? */ - } sl_reqack; + /* This structure is used to keep track of clients who answered, and + * to make the reqack message + */ + struct reqackBm + { + struct reqack ra; + char readySet[MAX_CLIENTS / BITS_PER_CHAR]; /* who is already ok? */ + } sl_reqack; - char answeredSet[MAX_CLIENTS / BITS_PER_CHAR]; /* who answered at all? */ + char answeredSet[MAX_CLIENTS / BITS_PER_CHAR]; /* who answered at all? */ - int nrReady; /* number of participants who are ready */ - int nrAnswered; /* number of participants who answered; */ - int needRxmit; /* does this need retransmission? */ - int lastGoodBlock; /* last good block of slice (i.e. last block having not - * needed retransmission */ + int nrReady; /* number of participants who are ready */ + int nrAnswered; /* number of participants who answered; */ + int needRxmit; /* does this need retransmission? */ + int lastGoodBlock; /* last good block of slice (i.e. last block having not + * needed retransmission */ - int lastReqack; /* last req ack sent (debug) */ + int lastReqack; /* last req ack sent (debug) */ #ifdef BB_FEATURE_UDPCAST_FEC - unsigned char* fec_data; + unsigned char* fec_data; #endif - int magic; - int blocksTransferred; /* blocks transferred during this slice */ - int dataBlocksTransferred; /* data blocks transferred during this slice */ - struct retransmit retransmit; - int freePos; /* where the next data part will be stored to */ - int bytesKnown; /* is number of bytes known yet? */ - short missing_data_blocks[MAX_FEC_INTERLEAVE]; -}* slice_t; + int magic; + int blocksTransferred; /* blocks transferred during this slice */ + int dataBlocksTransferred; /* data blocks transferred during this slice */ + struct retransmit retransmit; + int freePos; /* where the next data part will be stored to */ + int bytesKnown; /* is number of bytes known yet? */ + short missing_data_blocks[MAX_FEC_INTERLEAVE]; +} * slice_t; #define QUEUE_SIZE 256 struct returnChannel { - pthread_t thread; /* message receiving thread */ - int rcvSock; /* socket on which we receive the messages */ - produconsum_t incoming; /* where to enqueue incoming messages */ - produconsum_t freeSpace; /* free space */ - struct - { - int clNo; /* client number */ - union message msg; /* its message */ - } q[QUEUE_SIZE]; - struct net_config* config; - participantsDb_t participantsDb; + pthread_t thread; /* message receiving thread */ + int rcvSock; /* socket on which we receive the messages */ + produconsum_t incoming; /* where to enqueue incoming messages */ + produconsum_t freeSpace; /* free space */ + struct + { + int clNo; /* client number */ + union message msg; /* its message */ + } q[QUEUE_SIZE]; + struct net_config* config; + participantsDb_t participantsDb; }; #define NR_SLICES 2 typedef struct senderState { - struct returnChannel rc; - struct fifo* fifo; + struct returnChannel rc; + struct fifo* fifo; - struct net_config* config; - sender_stats_t stats; - int socket; + struct net_config* config; + sender_stats_t stats; + int socket; - struct slice slices[NR_SLICES]; + struct slice slices[NR_SLICES]; - produconsum_t free_slices_pc; + produconsum_t free_slices_pc; - unsigned char* fec_data; - pthread_t fec_thread; - produconsum_t fec_data_pc; -}* sender_state_t; + unsigned char* fec_data; + pthread_t fec_thread; + produconsum_t fec_data_pc; +} * sender_state_t; struct clientState { - struct fifo* fifo; - struct client_config* client_config; - struct net_config* net_config; - union serverDataMsg Msg; + struct fifo* fifo; + struct client_config* client_config; + struct net_config* net_config; + union serverDataMsg Msg; - struct msghdr data_hdr; + struct msghdr data_hdr; - /* pre-prepared messages */ - struct iovec data_iov[2]; + /* pre-prepared messages */ + struct iovec data_iov[2]; - struct slice* currentSlice; - int currentSliceNo; - receiver_stats_t stats; + struct slice* currentSlice; + int currentSliceNo; + receiver_stats_t stats; - produconsum_t free_slices_pc; - struct slice slices[NR_SLICES]; + produconsum_t free_slices_pc; + struct slice slices[NR_SLICES]; - /* Completely received slices */ - int receivedPtr; - int receivedSliceNo; + /* Completely received slices */ + int receivedPtr; + int receivedSliceNo; #ifdef BB_FEATURE_UDPCAST_FEC - int use_fec; /* do we use forward error correction ? */ + int use_fec; /* do we use forward error correction ? */ #endif - produconsum_t fec_data_pc; - struct slice* fec_slices[NR_SLICES]; - pthread_t fec_thread; + produconsum_t fec_data_pc; + struct slice* fec_slices[NR_SLICES]; + pthread_t fec_thread; - /* A reservoir of free blocks for FEC */ - produconsum_t freeBlocks_pc; - unsigned char** blockAddresses; /* adresses of blocks in local queue */ + /* A reservoir of free blocks for FEC */ + produconsum_t freeBlocks_pc; + unsigned char** blockAddresses; /* adresses of blocks in local queue */ - unsigned char** localBlockAddresses; - /* local blocks: freed FEC blocks after we - * have received the corresponding data */ - int localPos; + unsigned char** localBlockAddresses; + /* local blocks: freed FEC blocks after we + * have received the corresponding data */ + int localPos; - unsigned char* blockData; - unsigned char* nextBlock; + unsigned char* blockData; + unsigned char* nextBlock; - int endReached; /* end of transmission reached: - 0: transmission in progress - 2: network transmission _and_ FEC - processing finished - */ + int endReached; /* end of transmission reached: + 0: transmission in progress + 2: network transmission _and_ FEC + processing finished + */ - int netEndReached; /* In case of a FEC transmission; network - * transmission finished. This is needed to avoid - * a race condition, where the receiver thread would - * already prepare to wait for more data, at the same - * time that the FEC would set endReached. To avoid - * this, we do a select without timeout before - * receiving the last few packets, so that if the - * race condition strikes, we have a way to protect - * against - */ + int netEndReached; /* In case of a FEC transmission; network + * transmission finished. This is needed to avoid + * a race condition, where the receiver thread would + * already prepare to wait for more data, at the same + * time that the FEC would set endReached. To avoid + * this, we do a select without timeout before + * receiving the last few packets, so that if the + * race condition strikes, we have a way to protect + * against + */ - int selectedFd; + int selectedFd; - int promptPrinted; /* Has "Press any key..." prompt already been printed */ + int promptPrinted; /* Has "Press any key..." prompt already been printed */ #ifdef BB_FEATURE_UDPCAST_FEC - fec_code_t fec_code; + fec_code_t fec_code; #endif }; @@ -271,7 +271,6 @@ struct clientState */ #define FLAG_PASSIVE 0x0010 - /** * Do not write file synchronously */ @@ -291,950 +290,909 @@ namespace { int udpc_isFullDuplex(int s, const char* ifname) { - #ifdef ETHTOOL_GLINK - struct ifreq ifr; - struct ethtool_cmd ecmd; + struct ifreq ifr; + struct ethtool_cmd ecmd; - ecmd.cmd = ETHTOOL_GSET; + ecmd.cmd = ETHTOOL_GSET; - strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name) - 1); - ifr.ifr_data = (char*) &ecmd; + strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name) - 1); + ifr.ifr_data = (char*)&ecmd; - if (ioctl(s, SIOCETHTOOL, &ifr) == -1) - { - /* Operation not supported */ - return -1; - } - else - { - return ecmd.duplex; - } + if (ioctl(s, SIOCETHTOOL, &ifr) == -1) + { + /* Operation not supported */ + return -1; + } + else + { + return ecmd.duplex; + } #else - return -1; + return -1; #endif } -#define getSinAddr(addr) (((struct sockaddr_in *) addr)->sin_addr) +#define getSinAddr(addr) (((struct sockaddr_in*)addr)->sin_addr) int udpc_ipIsZero(struct sockaddr_in* ip) { - return getSinAddr(ip).s_addr == 0; + return getSinAddr(ip).s_addr == 0; } int hasLink(int s, const char* ifname) { - #ifdef ETHTOOL_GLINK - struct ifreq ifr; - struct ethtool_value edata; + struct ifreq ifr; + struct ethtool_value edata; - edata.cmd = ETHTOOL_GLINK; + edata.cmd = ETHTOOL_GLINK; - strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name) - 1); - ifr.ifr_data = (char*) &edata; + strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name) - 1); + ifr.ifr_data = (char*)&edata; - if (ioctl(s, SIOCETHTOOL, &ifr) == -1) - { - /* Operation not supported */ - return -1; - } - else - { - return edata.data; - } + if (ioctl(s, SIOCETHTOOL, &ifr) == -1) + { + /* Operation not supported */ + return -1; + } + else + { + return edata.data; + } #else - return -1; + return -1; #endif } -#define INET_ATON(a,i) inet_aton(a,i) +#define INET_ATON(a, i) inet_aton(a, i) int udpc_doSend(int s, void* message, size_t len, struct sockaddr_in* to) { - /* flprintf("sent: %08x %d\n", *(int*) message, len);*/ + /* flprintf("sent: %08x %d\n", *(int*) message, len);*/ #ifdef LOSSTEST - loseSendPacket(); + loseSendPacket(); #endif - return sendto(s, message, len, 0, (struct sockaddr*) to, sizeof(*to)); + return sendto(s, message, len, 0, (struct sockaddr*)to, sizeof(*to)); } void udpc_copyToMessage(unsigned char* dst, struct sockaddr_in* src) { - memcpy(dst, (char*) & ((struct sockaddr_in*)src)->sin_addr, - sizeof(struct in_addr)); + memcpy(dst, (char*)&((struct sockaddr_in*)src)->sin_addr, sizeof(struct in_addr)); } -void udpc_sendHello(struct net_config* net_config, int sock, - int streaming) +void udpc_sendHello(struct net_config* net_config, int sock, int streaming) { -//cerr << "sending hello..." << endl; - struct hello hello; + // cerr << "sending hello..." << endl; + struct hello hello; - /* send hello message */ - if (streaming) - hello.opCode = htons(CMD_HELLO_STREAMING); - else - hello.opCode = htons(CMD_HELLO); + /* send hello message */ + if (streaming) + hello.opCode = htons(CMD_HELLO_STREAMING); + else + hello.opCode = htons(CMD_HELLO); - hello.reserved = 0; - hello.capabilities = htonl(net_config->capabilities); - udpc_copyToMessage(hello.mcastAddr, &net_config->dataMcastAddr); - hello.blockSize = htons(net_config->blockSize); - //TODO: FIXME - //rgWaitAll(net_config, sock, net_config->controlMcastAddr.sin_addr.s_addr, sizeof(hello)); - BCAST_CONTROL(sock, hello); + hello.reserved = 0; + hello.capabilities = htonl(net_config->capabilities); + udpc_copyToMessage(hello.mcastAddr, &net_config->dataMcastAddr); + hello.blockSize = htons(net_config->blockSize); + // TODO: FIXME + // rgWaitAll(net_config, sock, net_config->controlMcastAddr.sin_addr.s_addr, sizeof(hello)); + BCAST_CONTROL(sock, hello); } char* udpc_getIpString(struct sockaddr_in* addr, char* buffer) { - long iaddr = htonl(getSinAddr(addr).s_addr); - sprintf(buffer, "%ld.%ld.%ld.%ld", - (iaddr >> 24) & 0xff, - (iaddr >> 16) & 0xff, - (iaddr >> 8) & 0xff, - iaddr & 0xff); - return buffer; + long iaddr = htonl(getSinAddr(addr).s_addr); + sprintf(buffer, "%ld.%ld.%ld.%ld", (iaddr >> 24) & 0xff, (iaddr >> 16) & 0xff, (iaddr >> 8) & 0xff, + iaddr & 0xff); + return buffer; } net_if_t* udpc_getNetIf(const char* wanted) { #ifndef __MINGW32__ - struct ifreq ibuf[100]; - struct ifreq* ifrp, *ifend, *chosen; - struct ifconf ifc; - int s; -#else /* __MINGW32__ */ - int i; + struct ifreq ibuf[100]; + struct ifreq *ifrp, *ifend, *chosen; + struct ifconf ifc; + int s; +#else /* __MINGW32__ */ + int i; - int etherNo = -1; - int wantedEtherNo = -2; /* Wanted ethernet interface */ + int etherNo = -1; + int wantedEtherNo = -2; /* Wanted ethernet interface */ - MIB_IPADDRTABLE* iptab = NULL; - MIB_IFTABLE* iftab = NULL; + MIB_IPADDRTABLE* iptab = NULL; + MIB_IFTABLE* iftab = NULL; - MIB_IPADDRROW* iprow, *chosen = NULL; - MIB_IFROW* chosenIf = NULL; - WORD wVersionRequested; /* Version of Winsock to load */ - WSADATA wsaData; /* Winsock implementation details */ - ULONG a; + MIB_IPADDRROW *iprow, *chosen = NULL; + MIB_IFROW* chosenIf = NULL; + WORD wVersionRequested; /* Version of Winsock to load */ + WSADATA wsaData; /* Winsock implementation details */ + ULONG a; - int r; + int r; #endif /* __MINGW32__ */ - int lastGoodness = 0; - struct in_addr wantedAddress; - int isAddress = 0; - int wantedLen = 0; - net_if_t* net_if; + int lastGoodness = 0; + struct in_addr wantedAddress; + int isAddress = 0; + int wantedLen = 0; + net_if_t* net_if; - if (wanted == NULL) - { - wanted = getenv("IFNAME"); - } + if (wanted == NULL) + { + wanted = getenv("IFNAME"); + } - if (wanted && INET_ATON(wanted, &wantedAddress)) - isAddress = 1; - else - wantedAddress.s_addr = 0; + if (wanted && INET_ATON(wanted, &wantedAddress)) + isAddress = 1; + else + wantedAddress.s_addr = 0; - if (wanted) - wantedLen = strlen(wanted); + if (wanted) + wantedLen = strlen(wanted); - net_if = MALLOC(net_if_t); - //TODO: FIXME - //if(net_if == NULL) - // udpc_fatal(1, "Out of memory error"); + net_if = MALLOC(net_if_t); + // TODO: FIXME + // if(net_if == NULL) + // udpc_fatal(1, "Out of memory error"); #ifndef __MINGW32__ - s = socket(PF_INET, SOCK_DGRAM, 0); + s = socket(PF_INET, SOCK_DGRAM, 0); - if (s < 0) - { - perror("make socket"); - exit(1); - } + if (s < 0) + { + perror("make socket"); + exit(1); + } - ifc.ifc_len = sizeof(ibuf); - ifc.ifc_buf = (caddr_t) ibuf; + ifc.ifc_len = sizeof(ibuf); + ifc.ifc_buf = (caddr_t)ibuf; - if (ioctl(s, SIOCGIFCONF, (char*)&ifc) < 0 || - ifc.ifc_len < (signed int)sizeof(struct ifreq)) - { - perror("udpcast: SIOCGIFCONF: "); - exit(1); - } + if (ioctl(s, SIOCGIFCONF, (char*)&ifc) < 0 || ifc.ifc_len < (signed int)sizeof(struct ifreq)) + { + perror("udpcast: SIOCGIFCONF: "); + exit(1); + } - ifend = (struct ifreq*)((char*)ibuf + ifc.ifc_len); - chosen = NULL; + ifend = (struct ifreq*)((char*)ibuf + ifc.ifc_len); + chosen = NULL; - for (ifrp = ibuf ; ifrp < ifend; + for (ifrp = ibuf; ifrp < ifend; #ifdef IFREQ_SIZE - ifrp = IFREQ_SIZE(*ifrp) + (char*)ifrp + ifrp = IFREQ_SIZE(*ifrp) + (char*)ifrp #else - ifrp++ + ifrp++ #endif - ) + ) + { + unsigned long iaddr = getSinAddr(&ifrp->ifr_addr).s_addr; + int goodness; + + if (ifrp->ifr_addr.sa_family != PF_INET) + continue; + + if (wanted) { - unsigned long iaddr = getSinAddr(&ifrp->ifr_addr).s_addr; - int goodness; - - if (ifrp->ifr_addr.sa_family != PF_INET) - continue; - - if (wanted) - { - if (isAddress && iaddr == wantedAddress.s_addr) - { - goodness = 8; - } - else if (strcmp(wanted, ifrp->ifr_name) == 0) - { - /* perfect match on interface name */ - goodness = 12; - } - else if (wanted != NULL && - strncmp(wanted, ifrp->ifr_name, wantedLen) == 0) - { - /* prefix match on interface name */ - goodness = 7; - } - else - { - /* no match, try next */ - continue; - } - } - else - { - if (iaddr == 0) - { - /* disregard interfaces whose address is zero */ - goodness = 1; - } - else if (iaddr == htonl(0x7f000001)) - { - /* disregard localhost type devices */ - goodness = 2; - } - else if (strcmp("eth0", ifrp->ifr_name) == 0 || - strcmp("en0", ifrp->ifr_name) == 0) - { - /* prefer first ethernet interface */ - goodness = 6; - } - else if (strncmp("eth0:", ifrp->ifr_name, 5) == 0) - { - /* second choice: any secondary addresses of first ethernet */ - goodness = 5; - } - else if (strncmp("eth", ifrp->ifr_name, 3) == 0 || - strncmp("en", ifrp->ifr_name, 2) == 0) - { - /* and, if not available, any other ethernet device */ - goodness = 4; - } - else - { - goodness = 3; - } - } - - if (hasLink(s, ifrp->ifr_name)) - /* Good or unknown link status privileged over known - * disconnected */ - goodness += 3; - - /* If all else is the same, prefer interfaces that - * have broadcast */ - goodness = goodness * 2; - - if (goodness >= lastGoodness) - { - /* Privilege broadcast-enabled interfaces */ - if (ioctl(s, SIOCGIFBRDADDR, ifrp) < 0) - { - //TODO: FIXME - //udpc_fatal(-1, "Error getting broadcast address for %s: %s", ifrp->ifr_name, strerror(errno)); - } - - if (getSinAddr(&ifrp->ifr_ifru.ifru_broadaddr).s_addr) - goodness++; - } - - if (goodness > lastGoodness) - { - chosen = ifrp; - lastGoodness = goodness; - net_if->addr.s_addr = iaddr; - } - } - - - if (!chosen) - { - fprintf(stderr, "No suitable network interface found\n"); - fprintf(stderr, "The following interfaces are available:\n"); - - for (ifrp = ibuf ; ifrp < ifend; -#ifdef IFREQ_SIZE - ifrp = IFREQ_SIZE(*ifrp) + (char*)ifrp -#else - ifrp++ -#endif - ) - { - char buffer[16]; - - if (ifrp->ifr_addr.sa_family != PF_INET) - continue; - - fprintf(stderr, "\t%s\t%s\n", - ifrp->ifr_name, - udpc_getIpString((struct sockaddr_in*)&ifrp->ifr_addr, buffer)); - } - - exit(1); - } - - net_if->name = strdup(chosen->ifr_name); - -#ifdef HAVE_STRUCT_IP_MREQN_IMR_IFINDEX - - /* Index for multicast subscriptions */ - if (ioctl(s, SIOCGIFINDEX, chosen) < 0) - { - //TODO: FIXME - //udpc_fatal(-1, "Error getting index for %s: %s", net_if->name, strerror(errno)); - } - - net_if->index = chosen->ifr_ifindex; -#endif - - /* Broadcast */ - if (ioctl(s, SIOCGIFBRDADDR, chosen) < 0) - { - //TODO: FIXME - //udpc_fatal(-1, "Error getting broadcast address for %s: %s", net_if->name, strerror(errno)); - } - - net_if->bcast = getSinAddr(&chosen->ifr_ifru.ifru_broadaddr); - - close(s); - -#else /* __MINGW32__ */ - - /* WINSOCK initialization */ - wVersionRequested = MAKEWORD(2, 0); /* Request Winsock v2.0 */ - - if (WSAStartup(wVersionRequested, &wsaData) != 0) /* Load Winsock DLL */ - { - fprintf(stderr, "WSAStartup() failed"); - exit(1); - } - - /* End WINSOCK initialization */ - - - a = 0; - r = GetIpAddrTable(iptab, &a, TRUE); - iptab = malloc(a); - r = GetIpAddrTable(iptab, &a, TRUE); - - a = 0; - r = GetIfTable(iftab, &a, TRUE); - iftab = malloc(a); - r = GetIfTable(iftab, &a, TRUE); - - if (wanted && !strncmp(wanted, "eth", 3) && wanted[3]) - { - char* ptr; - int n = strtoul(wanted + 3, &ptr, 10); - - if (!*ptr) - wantedEtherNo = n; - } - - for (i = 0; i < iptab->dwNumEntries; i++) - { - int goodness = -1; - unsigned long iaddr; - int isEther = 0; - MIB_IFROW* ifrow; - - iprow = &iptab->table[i]; - iaddr = iprow->dwAddr; - - ifrow = getIfRow(iftab, iprow->dwIndex); - - if (ifrow && ifrow->dwPhysAddrLen == 6 && iprow->dwBCastAddr) - { - isEther = 1; - etherNo++; - } - - if (wanted) - { - if (isAddress && iaddr == wantedAddress.s_addr) - { - goodness = 8; - } - else if (isEther && wantedEtherNo == etherNo) - { - goodness = 9; - } - else if (ifrow->dwPhysAddrLen) - { - int j; - const char* ptr = wanted; - - for (j = 0; *ptr && j < ifrow->dwPhysAddrLen; j++) - { - int digit = strtoul(ptr, (char**)&ptr, 16); - - if (digit != ifrow->bPhysAddr[j]) - break; /* Digit mismatch */ - - if (*ptr == '-' || *ptr == ':') - { - ptr++; - } - } - - if (!*ptr && j == ifrow->dwPhysAddrLen) - { - goodness = 9; - } - } - } - else - { - if (iaddr == 0) - { - /* disregard interfaces whose address is zero */ - goodness = 1; - } - else if (iaddr == htonl(0x7f000001)) - { - /* disregard localhost type devices */ - goodness = 2; - } - else if (isEther) - { - /* prefer ethernet */ - goodness = 6; - } - else if (ifrow->dwPhysAddrLen) - { - /* then prefer interfaces which have a physical address */ - goodness = 4; - } - else - { - goodness = 3; - } - } - - goodness = goodness * 2; - - /* If all else is the same, prefer interfaces that - * have broadcast */ - if (goodness >= lastGoodness) - { - /* Privilege broadcast-enabled interfaces */ - if (iprow->dwBCastAddr) - goodness++; - } - - if (goodness > lastGoodness) - { - chosen = iprow; - chosenIf = ifrow; - lastGoodness = goodness; - } - } - - if (!chosen) - { - fprintf(stderr, "No suitable network interface found%s%s\n", - wanted ? " for " : "", wanted ? wanted : ""); - fprintf(stderr, "The following interfaces are available:\n"); - - for (i = 0; i < iptab->dwNumEntries; i++) - { - char buffer[16]; - struct sockaddr_in addr; - MIB_IFROW* ifrow; - char* name = NULL; - iprow = &iptab->table[i]; - addr.sin_addr.s_addr = iprow->dwAddr; - ifrow = getIfRow(iftab, iprow->dwIndex); - name = fmtName(ifrow); - fprintf(stderr, " %15s %s\n", - udpc_getIpString(&addr, buffer), - name ? name : ""); - - if (name) - free(name); - } - - exit(1); - } - - net_if->bcast.s_addr = net_if->addr.s_addr = chosen->dwAddr; - - if (chosen->dwBCastAddr) - net_if->bcast.s_addr |= ~chosen->dwMask; - - if (chosenIf) - { - net_if->name = fmtName(chosenIf); + if (isAddress && iaddr == wantedAddress.s_addr) + { + goodness = 8; + } + else if (strcmp(wanted, ifrp->ifr_name) == 0) + { + /* perfect match on interface name */ + goodness = 12; + } + else if (wanted != NULL && strncmp(wanted, ifrp->ifr_name, wantedLen) == 0) + { + /* prefix match on interface name */ + goodness = 7; + } + else + { + /* no match, try next */ + continue; + } } else { - net_if->name = "*"; + if (iaddr == 0) + { + /* disregard interfaces whose address is zero */ + goodness = 1; + } + else if (iaddr == htonl(0x7f000001)) + { + /* disregard localhost type devices */ + goodness = 2; + } + else if (strcmp("eth0", ifrp->ifr_name) == 0 || strcmp("en0", ifrp->ifr_name) == 0) + { + /* prefer first ethernet interface */ + goodness = 6; + } + else if (strncmp("eth0:", ifrp->ifr_name, 5) == 0) + { + /* second choice: any secondary addresses of first ethernet */ + goodness = 5; + } + else if (strncmp("eth", ifrp->ifr_name, 3) == 0 || strncmp("en", ifrp->ifr_name, 2) == 0) + { + /* and, if not available, any other ethernet device */ + goodness = 4; + } + else + { + goodness = 3; + } } - free(iftab); - free(iptab); + if (hasLink(s, ifrp->ifr_name)) + /* Good or unknown link status privileged over known + * disconnected */ + goodness += 3; + + /* If all else is the same, prefer interfaces that + * have broadcast */ + goodness = goodness * 2; + + if (goodness >= lastGoodness) + { + /* Privilege broadcast-enabled interfaces */ + if (ioctl(s, SIOCGIFBRDADDR, ifrp) < 0) + { + // TODO: FIXME + // udpc_fatal(-1, "Error getting broadcast address for %s: %s", ifrp->ifr_name, strerror(errno)); + } + + if (getSinAddr(&ifrp->ifr_ifru.ifru_broadaddr).s_addr) + goodness++; + } + + if (goodness > lastGoodness) + { + chosen = ifrp; + lastGoodness = goodness; + net_if->addr.s_addr = iaddr; + } + } + + if (!chosen) + { + fprintf(stderr, "No suitable network interface found\n"); + fprintf(stderr, "The following interfaces are available:\n"); + + for (ifrp = ibuf; ifrp < ifend; +#ifdef IFREQ_SIZE + ifrp = IFREQ_SIZE(*ifrp) + (char*)ifrp +#else + ifrp++ +#endif + ) + { + char buffer[16]; + + if (ifrp->ifr_addr.sa_family != PF_INET) + continue; + + fprintf(stderr, "\t%s\t%s\n", ifrp->ifr_name, + udpc_getIpString((struct sockaddr_in*)&ifrp->ifr_addr, buffer)); + } + + exit(1); + } + + net_if->name = strdup(chosen->ifr_name); + +#ifdef HAVE_STRUCT_IP_MREQN_IMR_IFINDEX + + /* Index for multicast subscriptions */ + if (ioctl(s, SIOCGIFINDEX, chosen) < 0) + { + // TODO: FIXME + // udpc_fatal(-1, "Error getting index for %s: %s", net_if->name, strerror(errno)); + } + + net_if->index = chosen->ifr_ifindex; +#endif + + /* Broadcast */ + if (ioctl(s, SIOCGIFBRDADDR, chosen) < 0) + { + // TODO: FIXME + // udpc_fatal(-1, "Error getting broadcast address for %s: %s", net_if->name, strerror(errno)); + } + + net_if->bcast = getSinAddr(&chosen->ifr_ifru.ifru_broadaddr); + + close(s); + +#else /* __MINGW32__ */ + + /* WINSOCK initialization */ + wVersionRequested = MAKEWORD(2, 0); /* Request Winsock v2.0 */ + + if (WSAStartup(wVersionRequested, &wsaData) != 0) /* Load Winsock DLL */ + { + fprintf(stderr, "WSAStartup() failed"); + exit(1); + } + + /* End WINSOCK initialization */ + + a = 0; + r = GetIpAddrTable(iptab, &a, TRUE); + iptab = malloc(a); + r = GetIpAddrTable(iptab, &a, TRUE); + + a = 0; + r = GetIfTable(iftab, &a, TRUE); + iftab = malloc(a); + r = GetIfTable(iftab, &a, TRUE); + + if (wanted && !strncmp(wanted, "eth", 3) && wanted[3]) + { + char* ptr; + int n = strtoul(wanted + 3, &ptr, 10); + + if (!*ptr) + wantedEtherNo = n; + } + + for (i = 0; i < iptab->dwNumEntries; i++) + { + int goodness = -1; + unsigned long iaddr; + int isEther = 0; + MIB_IFROW* ifrow; + + iprow = &iptab->table[i]; + iaddr = iprow->dwAddr; + + ifrow = getIfRow(iftab, iprow->dwIndex); + + if (ifrow && ifrow->dwPhysAddrLen == 6 && iprow->dwBCastAddr) + { + isEther = 1; + etherNo++; + } + + if (wanted) + { + if (isAddress && iaddr == wantedAddress.s_addr) + { + goodness = 8; + } + else if (isEther && wantedEtherNo == etherNo) + { + goodness = 9; + } + else if (ifrow->dwPhysAddrLen) + { + int j; + const char* ptr = wanted; + + for (j = 0; *ptr && j < ifrow->dwPhysAddrLen; j++) + { + int digit = strtoul(ptr, (char**)&ptr, 16); + + if (digit != ifrow->bPhysAddr[j]) + break; /* Digit mismatch */ + + if (*ptr == '-' || *ptr == ':') + { + ptr++; + } + } + + if (!*ptr && j == ifrow->dwPhysAddrLen) + { + goodness = 9; + } + } + } + else + { + if (iaddr == 0) + { + /* disregard interfaces whose address is zero */ + goodness = 1; + } + else if (iaddr == htonl(0x7f000001)) + { + /* disregard localhost type devices */ + goodness = 2; + } + else if (isEther) + { + /* prefer ethernet */ + goodness = 6; + } + else if (ifrow->dwPhysAddrLen) + { + /* then prefer interfaces which have a physical address */ + goodness = 4; + } + else + { + goodness = 3; + } + } + + goodness = goodness * 2; + + /* If all else is the same, prefer interfaces that + * have broadcast */ + if (goodness >= lastGoodness) + { + /* Privilege broadcast-enabled interfaces */ + if (iprow->dwBCastAddr) + goodness++; + } + + if (goodness > lastGoodness) + { + chosen = iprow; + chosenIf = ifrow; + lastGoodness = goodness; + } + } + + if (!chosen) + { + fprintf(stderr, "No suitable network interface found%s%s\n", wanted ? " for " : "", wanted ? wanted : ""); + fprintf(stderr, "The following interfaces are available:\n"); + + for (i = 0; i < iptab->dwNumEntries; i++) + { + char buffer[16]; + struct sockaddr_in addr; + MIB_IFROW* ifrow; + char* name = NULL; + iprow = &iptab->table[i]; + addr.sin_addr.s_addr = iprow->dwAddr; + ifrow = getIfRow(iftab, iprow->dwIndex); + name = fmtName(ifrow); + fprintf(stderr, " %15s %s\n", udpc_getIpString(&addr, buffer), name ? name : ""); + + if (name) + free(name); + } + + exit(1); + } + + net_if->bcast.s_addr = net_if->addr.s_addr = chosen->dwAddr; + + if (chosen->dwBCastAddr) + net_if->bcast.s_addr |= ~chosen->dwMask; + + if (chosenIf) + { + net_if->name = fmtName(chosenIf); + } + else + { + net_if->name = "*"; + } + + free(iftab); + free(iptab); #endif /* __MINGW32__ */ - return net_if; + return net_if; } #define IP_MREQN ip_mreqn -int fillMreq(net_if_t* net_if, struct in_addr addr, - struct IP_MREQN* mreq) +int fillMreq(net_if_t* net_if, struct in_addr addr, struct IP_MREQN* mreq) { #ifdef HAVE_STRUCT_IP_MREQN_IMR_IFINDEX - mreq->imr_ifindex = net_if->index; - mreq->imr_address.s_addr = 0; + mreq->imr_ifindex = net_if->index; + mreq->imr_address.s_addr = 0; #else - mreq->imr_interface = net_if->addr; + mreq->imr_interface = net_if->addr; #endif - mreq->imr_multiaddr = addr; + mreq->imr_multiaddr = addr; - return 0; + return 0; } -int mcastOp(int sock, net_if_t* net_if, struct in_addr addr, - int code, const char* message) +int mcastOp(int sock, net_if_t* net_if, struct in_addr addr, int code, const char* message) { - struct IP_MREQN mreq; - int r; + struct IP_MREQN mreq; + int r; - fillMreq(net_if, addr, &mreq); - r = setsockopt(sock, SOL_IP, code, (char*)&mreq, sizeof(mreq)); + fillMreq(net_if, addr, &mreq); + r = setsockopt(sock, SOL_IP, code, (char*)&mreq, sizeof(mreq)); - if (r < 0) - { - perror(message); - exit(1); - } + if (r < 0) + { + perror(message); + exit(1); + } - return 0; + return 0; } int udpc_setMcastDestination(int sock, net_if_t* net_if, struct sockaddr_in* addr) { #ifdef WINDOWS - int r; - struct sockaddr_in interface_addr; - struct in_addr if_addr; - getMyAddress(net_if, &interface_addr); - if_addr = getSinAddr(&interface_addr); - r = setsockopt (sock, IPPROTO_IP, IP_MULTICAST_IF, - (char*) &if_addr, sizeof(if_addr)); + int r; + struct sockaddr_in interface_addr; + struct in_addr if_addr; + getMyAddress(net_if, &interface_addr); + if_addr = getSinAddr(&interface_addr); + r = setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF, (char*)&if_addr, sizeof(if_addr)); - if (r < 0) - fatal(1, "Set multicast send interface"); + if (r < 0) + fatal(1, "Set multicast send interface"); - return 0; + return 0; #else - /* IP_MULTICAST_IF not correctly supported on Cygwin */ - return mcastOp(sock, net_if, getSinAddr(addr), IP_MULTICAST_IF, - "Set multicast send interface"); + /* IP_MULTICAST_IF not correctly supported on Cygwin */ + return mcastOp(sock, net_if, getSinAddr(addr), IP_MULTICAST_IF, "Set multicast send interface"); #endif } -int initSockAddress(addr_type_t addr_type, - net_if_t* net_if, - in_addr_t ip, - unsigned short port, +int initSockAddress(addr_type_t addr_type, net_if_t* net_if, in_addr_t ip, unsigned short port, struct sockaddr_in* addr) { - memset ((char*) addr, 0, sizeof(struct sockaddr_in)); - addr->sin_family = AF_INET; - addr->sin_port = htons(port); + memset((char*)addr, 0, sizeof(struct sockaddr_in)); + addr->sin_family = AF_INET; + addr->sin_port = htons(port); - //TODO: FIXME - //if(!net_if && addr_type != ADDR_TYPE_MCAST) - //udpc_fatal(1, "initSockAddr without ifname\n"); + // TODO: FIXME + // if(!net_if && addr_type != ADDR_TYPE_MCAST) + // udpc_fatal(1, "initSockAddr without ifname\n"); - switch (addr_type) - { - case ADDR_TYPE_UCAST: - addr->sin_addr = net_if->addr; - break; + switch (addr_type) + { + case ADDR_TYPE_UCAST: addr->sin_addr = net_if->addr; break; - case ADDR_TYPE_BCAST: - addr->sin_addr = net_if->bcast; - break; + case ADDR_TYPE_BCAST: addr->sin_addr = net_if->bcast; break; - case ADDR_TYPE_MCAST: - addr->sin_addr.s_addr = ip; - break; - } + case ADDR_TYPE_MCAST: addr->sin_addr.s_addr = ip; break; + } - return 0; + return 0; } int mcastListen(int sock, net_if_t* net_if, struct sockaddr_in* addr) { - return mcastOp(sock, net_if, getSinAddr(addr), IP_ADD_MEMBERSHIP, - "Subscribe to multicast group"); + return mcastOp(sock, net_if, getSinAddr(addr), IP_ADD_MEMBERSHIP, "Subscribe to multicast group"); } -int udpc_makeSocket(addr_type_t addr_type, - net_if_t* net_if, - struct sockaddr_in* tmpl, - int port) +int udpc_makeSocket(addr_type_t addr_type, net_if_t* net_if, struct sockaddr_in* tmpl, int port) { - int ret, s; - struct sockaddr_in myaddr; - in_addr_t ip = 0; + int ret, s; + struct sockaddr_in myaddr; + in_addr_t ip = 0; #ifdef WINDOWS - static int lastSocket = -1; - /* Very ugly hack, but hey!, this is for Windows */ + static int lastSocket = -1; + /* Very ugly hack, but hey!, this is for Windows */ - if (addr_type == ADDR_TYPE_MCAST) - { - mcastListen(lastSocket, net_if, tmpl); - return -1; - } - else if (addr_type != ADDR_TYPE_UCAST) - return -1; + if (addr_type == ADDR_TYPE_MCAST) + { + mcastListen(lastSocket, net_if, tmpl); + return -1; + } + else if (addr_type != ADDR_TYPE_UCAST) + return -1; #endif - s = socket(PF_INET, SOCK_DGRAM, 0); + s = socket(PF_INET, SOCK_DGRAM, 0); - if (s < 0) - { - perror("make socket"); - exit(1); - } + if (s < 0) + { + perror("make socket"); + exit(1); + } - if (addr_type == ADDR_TYPE_MCAST && tmpl != NULL) - { - ip = tmpl->sin_addr.s_addr; - } + if (addr_type == ADDR_TYPE_MCAST && tmpl != NULL) + { + ip = tmpl->sin_addr.s_addr; + } - ret = initSockAddress(addr_type, net_if, ip, port, &myaddr); + ret = initSockAddress(addr_type, net_if, ip, port, &myaddr); - //TODO: FIXME - //if(ret < 0) -// udpc_fatal(1, "Could not get socket address fot %d/%s", -// addr_type, net_if->name); - if (addr_type == ADDR_TYPE_BCAST) - { -//cerr << "for addr_type == ADDR_TYPE_BCAST, myaddr.sin_addr.s_addr = 0x" << hex << myaddr.sin_addr.s_addr << dec << endl; - } + // TODO: FIXME + // if(ret < 0) + // udpc_fatal(1, "Could not get socket address fot %d/%s", + // addr_type, net_if->name); + if (addr_type == ADDR_TYPE_BCAST) + { + // cerr << "for addr_type == ADDR_TYPE_BCAST, myaddr.sin_addr.s_addr = 0x" << hex << + // myaddr.sin_addr.s_addr << dec << endl; + } - if (addr_type == ADDR_TYPE_BCAST && myaddr.sin_addr.s_addr == 0) - { - /* Attempting to bind to broadcast address on not-broadcast media ... */ - closesocket(s); - return -1; - } + if (addr_type == ADDR_TYPE_BCAST && myaddr.sin_addr.s_addr == 0) + { + /* Attempting to bind to broadcast address on not-broadcast media ... */ + closesocket(s); + return -1; + } - ret = bind(s, (struct sockaddr*) &myaddr, sizeof(myaddr)); - //TODO: FIXME -// if (ret < 0) { -// char buffer[16]; -// udpc_fatal(1, "bind socket to %s:%d (%s)\n", -// udpc_getIpString(&myaddr, buffer), -// udpc_getPort(&myaddr), -// strerror(errno)); -// } + ret = bind(s, (struct sockaddr*)&myaddr, sizeof(myaddr)); + // TODO: FIXME + // if (ret < 0) { + // char buffer[16]; + // udpc_fatal(1, "bind socket to %s:%d (%s)\n", + // udpc_getIpString(&myaddr, buffer), + // udpc_getPort(&myaddr), + // strerror(errno)); + // } - if (addr_type == ADDR_TYPE_MCAST) - mcastListen(s, net_if, &myaddr); + if (addr_type == ADDR_TYPE_MCAST) + mcastListen(s, net_if, &myaddr); #ifdef WINDOWS - lastSocket = s; + lastSocket = s; #endif - return s; + return s; } int udpc_setSocketToBroadcast(int sock) { - /* set the socket to broadcast */ - int p = 1; - return setsockopt(sock, SOL_SOCKET, SO_BROADCAST, (char*)&p, sizeof(int)); + /* set the socket to broadcast */ + int p = 1; + return setsockopt(sock, SOL_SOCKET, SO_BROADCAST, (char*)&p, sizeof(int)); } -int udpc_getBroadCastAddress(net_if_t* net_if, struct sockaddr_in* addr, - short port) +int udpc_getBroadCastAddress(net_if_t* net_if, struct sockaddr_in* addr, short port) { - int r = initSockAddress(ADDR_TYPE_BCAST, net_if, INADDR_ANY, port, addr); + int r = initSockAddress(ADDR_TYPE_BCAST, net_if, INADDR_ANY, port, addr); - if (addr->sin_addr.s_addr == 0) - { - /* Quick hack to make it work for loopback */ - struct sockaddr_in ucast; - initSockAddress(ADDR_TYPE_UCAST, net_if, INADDR_ANY, port, &ucast); + if (addr->sin_addr.s_addr == 0) + { + /* Quick hack to make it work for loopback */ + struct sockaddr_in ucast; + initSockAddress(ADDR_TYPE_UCAST, net_if, INADDR_ANY, port, &ucast); - if ((ntohl(ucast.sin_addr.s_addr) & 0xff000000) == 0x7f000000) - addr->sin_addr.s_addr = ucast.sin_addr.s_addr; - } + if ((ntohl(ucast.sin_addr.s_addr) & 0xff000000) == 0x7f000000) + addr->sin_addr.s_addr = ucast.sin_addr.s_addr; + } - return r; + return r; } int safe_inet_aton(const char* address, struct in_addr* ip) { - if (!INET_ATON(address, ip)) - { - //TODO: FIXME - //udpc_fatal(-1, "Bad address %s", address); - } + if (!INET_ATON(address, ip)) + { + // TODO: FIXME + // udpc_fatal(-1, "Bad address %s", address); + } - return 0; + return 0; } -int udpc_getMcastAllAddress(struct sockaddr_in* addr, const char* address, - short port) +int udpc_getMcastAllAddress(struct sockaddr_in* addr, const char* address, short port) { - struct in_addr ip; - int ret; + struct in_addr ip; + int ret; - if (address == NULL || address[0] == '\0') - safe_inet_aton("224.0.0.1", &ip); - else - { - if ((ret = safe_inet_aton(address, &ip)) < 0) - return ret; - } + if (address == NULL || address[0] == '\0') + safe_inet_aton("224.0.0.1", &ip); + else + { + if ((ret = safe_inet_aton(address, &ip)) < 0) + return ret; + } - return initSockAddress(ADDR_TYPE_MCAST, NULL, ip.s_addr, port, addr); + return initSockAddress(ADDR_TYPE_MCAST, NULL, ip.s_addr, port, addr); } void setPort(struct sockaddr_in* addr, unsigned short port) { - ((struct sockaddr_in*) addr)->sin_port = htons(port); + ((struct sockaddr_in*)addr)->sin_port = htons(port); } int isMcastAddress(struct sockaddr_in* addr) { - int ip = ntohl(addr->sin_addr.s_addr) >> 24; - return ip >= 0xe0 && ip < 0xf0; + int ip = ntohl(addr->sin_addr.s_addr) >> 24; + return ip >= 0xe0 && ip < 0xf0; } void udpc_clearIp(struct sockaddr_in* addr) { - addr->sin_addr.s_addr = 0; - addr->sin_family = AF_INET; + addr->sin_addr.s_addr = 0; + addr->sin_family = AF_INET; } void udpc_setSendBuf(int sock, unsigned int bufsize) { - if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char*)&bufsize, sizeof(bufsize)) < 0) - perror("Set send buffer"); + if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char*)&bufsize, sizeof(bufsize)) < 0) + perror("Set send buffer"); } int udpc_setTtl(int sock, int ttl) { - /* set the socket to broadcast */ - return setsockopt(sock, SOL_IP, IP_MULTICAST_TTL, (char*)&ttl, sizeof(int)); + /* set the socket to broadcast */ + return setsockopt(sock, SOL_IP, IP_MULTICAST_TTL, (char*)&ttl, sizeof(int)); } int udpc_getMyAddress(net_if_t* net_if, struct sockaddr_in* addr) { - return initSockAddress(ADDR_TYPE_UCAST, net_if, INADDR_ANY, 0, addr); + return initSockAddress(ADDR_TYPE_UCAST, net_if, INADDR_ANY, 0, addr); } void udpc_getDefaultMcastAddress(net_if_t* net_if, struct sockaddr_in* mcast) { - udpc_getMyAddress(net_if, mcast); - mcast->sin_addr.s_addr &= htonl(0x07ffffff); - mcast->sin_addr.s_addr |= htonl(0xe8000000); + udpc_getMyAddress(net_if, mcast); + mcast->sin_addr.s_addr &= htonl(0x07ffffff); + mcast->sin_addr.s_addr |= htonl(0xe8000000); } void udpc_copyIpFrom(struct sockaddr_in* dst, struct sockaddr_in* src) { - dst->sin_addr = src->sin_addr; - dst->sin_family = src->sin_family; + dst->sin_addr = src->sin_addr; + dst->sin_family = src->sin_family; } int udpc_getSelectedSock(int* socks, int nr, fd_set* read_set) { - int i; - int maxFd; - maxFd = -1; + int i; + int maxFd; + maxFd = -1; - for (i = 0; i < nr; i++) - { - if (socks[i] == -1) - continue; + for (i = 0; i < nr; i++) + { + if (socks[i] == -1) + continue; - if (FD_ISSET(socks[i], read_set)) - return socks[i]; - } + if (FD_ISSET(socks[i], read_set)) + return socks[i]; + } - return -1; + return -1; } int udpc_ipIsEqual(struct sockaddr_in* left, struct sockaddr_in* right) { - return getSinAddr(left).s_addr == getSinAddr(right).s_addr; + return getSinAddr(left).s_addr == getSinAddr(right).s_addr; } void udpc_closeSock(int* socks, int nr, int target) { - int i; - int sock = socks[target]; + int i; + int sock = socks[target]; - socks[target] = -1; + socks[target] = -1; - for (i = 0; i < nr; i++) - if (socks[i] == sock) - return; + for (i = 0; i < nr; i++) + if (socks[i] == sock) + return; - closesocket(sock); + closesocket(sock); } int prepareForSelect(int* socks, int nr, fd_set* read_set) { - int i; - int maxFd; - FD_ZERO(read_set); - maxFd = -1; + int i; + int maxFd; + FD_ZERO(read_set); + maxFd = -1; - for (i = 0; i < nr; i++) - { - if (socks[i] == -1) - continue; + for (i = 0; i < nr; i++) + { + if (socks[i] == -1) + continue; - FD_SET(socks[i], read_set); + FD_SET(socks[i], read_set); - if (socks[i] > maxFd) - maxFd = socks[i]; - } + if (socks[i] > maxFd) + maxFd = socks[i]; + } - return maxFd; + return maxFd; } -int doReceive(int s, void* message, size_t len, - struct sockaddr_in* from, int portBase) +int doReceive(int s, void* message, size_t len, struct sockaddr_in* from, int portBase) { - socklen_t slen; - int r; - unsigned short port; + socklen_t slen; + int r; + unsigned short port; - slen = sizeof(*from); + slen = sizeof(*from); #ifdef LOSSTEST - loseRecvPacket(s); + loseRecvPacket(s); #endif - r = recvfrom(s, message, len, 0, (struct sockaddr*)from, &slen); + r = recvfrom(s, message, len, 0, (struct sockaddr*)from, &slen); - if (r < 0) - return r; - - port = ntohs(from->sin_port); - - if (port != RECEIVER_PORT(portBase) && port != SENDER_PORT(portBase)) - { - return -1; - } - - /* flprintf("recv: %08x %d\n", *(int*) message, r);*/ + if (r < 0) return r; + + port = ntohs(from->sin_port); + + if (port != RECEIVER_PORT(portBase) && port != SENDER_PORT(portBase)) + { + return -1; + } + + /* flprintf("recv: %08x %d\n", *(int*) message, r);*/ + return r; } participantsDb_t udpc_makeParticipantsDb(void) { - return MALLOC(struct participantsDb); + return MALLOC(struct participantsDb); } int udpc_removeParticipant(struct participantsDb* db, int i) { - if (db->clientTable[i].used) - { - db->clientTable[i].used = 0; - db->nrParticipants--; - } + if (db->clientTable[i].used) + { + db->clientTable[i].used = 0; + db->nrParticipants--; + } - return 0; + return 0; } int udpc_lookupParticipant(struct participantsDb* db, struct sockaddr_in* addr) { - int i; + int i; - for (i = 0; i < MAX_CLIENTS; i++) + for (i = 0; i < MAX_CLIENTS; i++) + { + if (db->clientTable[i].used && udpc_ipIsEqual(&db->clientTable[i].addr, addr)) { - if (db->clientTable[i].used && - udpc_ipIsEqual(&db->clientTable[i].addr, addr)) - { - return i; - } + return i; } + } - return -1; + return -1; } int udpc_nrParticipants(participantsDb_t db) { - return db->nrParticipants; + return db->nrParticipants; } -int udpc_addParticipant(participantsDb_t db, - struct sockaddr_in* addr, - int capabilities, - unsigned int rcvbuf, +int udpc_addParticipant(participantsDb_t db, struct sockaddr_in* addr, int capabilities, unsigned int rcvbuf, int pointopoint) { -//cerr << "adding a participant..." << endl; - int i; + // cerr << "adding a participant..." << endl; + int i; - if ((i = udpc_lookupParticipant(db, addr)) >= 0) - return i; + if ((i = udpc_lookupParticipant(db, addr)) >= 0) + return i; - for (i = 0; i < MAX_CLIENTS; i++) + for (i = 0; i < MAX_CLIENTS; i++) + { + if (!db->clientTable[i].used) { - if (!db->clientTable[i].used) - { - db->clientTable[i].addr = *addr; - db->clientTable[i].used = 1; - db->clientTable[i].capabilities = capabilities; - db->clientTable[i].rcvbuf = rcvbuf; - db->nrParticipants++; + db->clientTable[i].addr = *addr; + db->clientTable[i].used = 1; + db->clientTable[i].capabilities = capabilities; + db->clientTable[i].rcvbuf = rcvbuf; + db->nrParticipants++; - return i; - } - else if (pointopoint) - return -1; + return i; } + else if (pointopoint) + return -1; + } - return -1; /* no space left in participant's table */ + return -1; /* no space left in participant's table */ } -int selectWithoutConsole(int maxFd, - fd_set* read_set, struct timeval* tv) +int selectWithoutConsole(int maxFd, fd_set* read_set, struct timeval* tv) { - int ret; + int ret; - ret = select(maxFd, read_set, NULL, NULL, tv); + ret = select(maxFd, read_set, NULL, NULL, tv); - if (ret < 0) - return -1; + if (ret < 0) + return -1; - return ret; + return ret; } #if 0 @@ -1258,1694 +1216,1591 @@ void sendHello(struct net_config* net_config, int sock, } #endif -int checkClientWait(participantsDb_t db, - struct net_config* net_config, - time_t* firstConnected) +int checkClientWait(participantsDb_t db, struct net_config* net_config, time_t* firstConnected) { - time_t now; + time_t now; - if (!udpc_nrParticipants(db) || !firstConnected || !*firstConnected) - return 0; /* do not start: no receivers */ + if (!udpc_nrParticipants(db) || !firstConnected || !*firstConnected) + return 0; /* do not start: no receivers */ - now = time(0); + now = time(0); - /* - * If we have a max_client_wait, start the transfer after first client - * connected + maxSendWait - */ - if (net_config->max_receivers_wait && - (now >= *firstConnected + net_config->max_receivers_wait)) - { + /* + * If we have a max_client_wait, start the transfer after first client + * connected + maxSendWait + */ + if (net_config->max_receivers_wait && (now >= *firstConnected + net_config->max_receivers_wait)) + { #ifdef USE_SYSLOG - syslog(LOG_INFO, "max wait[%d] passed: starting", - net_config->max_receivers_wait ); + syslog(LOG_INFO, "max wait[%d] passed: starting", net_config->max_receivers_wait); #endif - return 1; /* send-wait passed: start */ - } + return 1; /* send-wait passed: start */ + } - /* - * Otherwise check to see if the minimum of clients - * have checked in. - */ - else if (udpc_nrParticipants(db) >= net_config->min_receivers && - /* - * If there are enough clients and there's a min wait time, we'll - * wait around anyway until then. - * Otherwise, we always transfer - */ - (!net_config->min_receivers_wait || - now >= *firstConnected + net_config->min_receivers_wait)) - { + /* + * Otherwise check to see if the minimum of clients + * have checked in. + */ + else if (udpc_nrParticipants(db) >= net_config->min_receivers && + /* + * If there are enough clients and there's a min wait time, we'll + * wait around anyway until then. + * Otherwise, we always transfer + */ + (!net_config->min_receivers_wait || now >= *firstConnected + net_config->min_receivers_wait)) + { #ifdef USE_SYSLOG - syslog(LOG_INFO, "min receivers[%d] reached: starting", - net_config->min_receivers ); + syslog(LOG_INFO, "min receivers[%d] reached: starting", net_config->min_receivers); #endif - return 1; - } - else - return 0; -} - -int sendConnectionReply(participantsDb_t db, - int sock, - struct net_config* config, - struct sockaddr_in* client, - int capabilities, - unsigned int rcvbuf) -{ - struct connectReply reply; - - if (rcvbuf == 0) - rcvbuf = 65536; - - if (capabilities & CAP_BIG_ENDIAN) - { - reply.opCode = htons(CMD_CONNECT_REPLY); - reply.clNr = - htonl(udpc_addParticipant(db, - client, - capabilities, - rcvbuf, - config->flags & FLAG_POINTOPOINT)); - reply.blockSize = htonl(config->blockSize); - } - else - { - //TODO: FIXME - //udpc_fatal(1, "Little endian protocol no longer supported"); - } - - reply.reserved = 0; - - if (config->flags & FLAG_POINTOPOINT) - { - udpc_copyIpFrom(&config->dataMcastAddr, client); - } - - /* new parameters: always big endian */ - reply.capabilities = ntohl(config->capabilities); - udpc_copyToMessage(reply.mcastAddr, &config->dataMcastAddr); - - /*reply.mcastAddress = mcastAddress;*/ - //rgWaitAll(config, sock, client->sin_addr.s_addr, sizeof(reply)); - if (SEND(sock, reply, *client) < 0) - { - perror("reply add new client"); - return -1; - } - + return 1; + } + else return 0; } -int mainDispatcher(int* fd, int nr, - participantsDb_t db, - struct net_config* net_config, - int* tries, +int sendConnectionReply(participantsDb_t db, int sock, struct net_config* config, struct sockaddr_in* client, + int capabilities, unsigned int rcvbuf) +{ + struct connectReply reply; + + if (rcvbuf == 0) + rcvbuf = 65536; + + if (capabilities & CAP_BIG_ENDIAN) + { + reply.opCode = htons(CMD_CONNECT_REPLY); + reply.clNr = + htonl(udpc_addParticipant(db, client, capabilities, rcvbuf, config->flags & FLAG_POINTOPOINT)); + reply.blockSize = htonl(config->blockSize); + } + else + { + // TODO: FIXME + // udpc_fatal(1, "Little endian protocol no longer supported"); + } + + reply.reserved = 0; + + if (config->flags & FLAG_POINTOPOINT) + { + udpc_copyIpFrom(&config->dataMcastAddr, client); + } + + /* new parameters: always big endian */ + reply.capabilities = ntohl(config->capabilities); + udpc_copyToMessage(reply.mcastAddr, &config->dataMcastAddr); + + /*reply.mcastAddress = mcastAddress;*/ + // rgWaitAll(config, sock, client->sin_addr.s_addr, sizeof(reply)); + if (SEND(sock, reply, *client) < 0) + { + perror("reply add new client"); + return -1; + } + + return 0; +} + +int mainDispatcher(int* fd, int nr, participantsDb_t db, struct net_config* net_config, int* tries, time_t* firstConnected) { - struct sockaddr_in client; - union message fromClient; - fd_set read_set; - int ret; - int msgLength; - int startNow = 0; - int selected; + struct sockaddr_in client; + union message fromClient; + fd_set read_set; + int ret; + int msgLength; + int startNow = 0; + int selected; - if (firstConnected && !*firstConnected && udpc_nrParticipants(db)) - { - *firstConnected = time(0); + if (firstConnected && !*firstConnected && udpc_nrParticipants(db)) + { + *firstConnected = time(0); #ifdef USE_SYSLOG - syslog(LOG_INFO, - "first connection: min wait[%d] secs - max wait[%d] - min clients[%d]", - net_config->min_receivers_wait, net_config->max_receivers_wait, - net_config->min_receivers ); + syslog(LOG_INFO, "first connection: min wait[%d] secs - max wait[%d] - min clients[%d]", + net_config->min_receivers_wait, net_config->max_receivers_wait, net_config->min_receivers); #endif - } + } - while (!startNow) + while (!startNow) + { + struct timeval tv; + struct timeval* tvp; + int nr_desc; + + int maxFd = prepareForSelect(fd, nr, &read_set); + + if (net_config->rexmit_hello_interval) { - struct timeval tv; - struct timeval* tvp; - int nr_desc; - - int maxFd = prepareForSelect(fd, nr, &read_set); - - if (net_config->rexmit_hello_interval) - { - tv.tv_usec = (net_config->rexmit_hello_interval % 1000) * 1000; - tv.tv_sec = net_config->rexmit_hello_interval / 1000; - tvp = &tv; - } - else if (firstConnected && udpc_nrParticipants(db)) - { - tv.tv_usec = 0; - tv.tv_sec = 2; - tvp = &tv; - } - else - tvp = 0; - - nr_desc = selectWithoutConsole(maxFd + 1, &read_set, tvp); - - if (nr_desc < 0) - { - perror("select"); - return -1; - } - - if (nr_desc > 0) - /* key pressed, or receiver activity */ - break; - - if (net_config->rexmit_hello_interval) - { - /* retransmit hello message */ - udpc_sendHello(net_config, fd[0], 0); - (*tries)++; - - if (net_config->autostart != 0 && *tries > net_config->autostart) - startNow = 1; - } - - if (firstConnected) - startNow = - startNow || checkClientWait(db, net_config, firstConnected); + tv.tv_usec = (net_config->rexmit_hello_interval % 1000) * 1000; + tv.tv_sec = net_config->rexmit_hello_interval / 1000; + tvp = &tv; } - - selected = udpc_getSelectedSock(fd, nr, &read_set); - - if (selected == -1) - return startNow; - - BZERO(fromClient); /* Zero it out in order to cope with short messages - * from older versions */ - - msgLength = RECV(selected, fromClient, client, net_config->portBase); - - if (msgLength < 0) + else if (firstConnected && udpc_nrParticipants(db)) { - perror("problem getting data from client"); - return 0; /* don't panic if we get weird messages */ + tv.tv_usec = 0; + tv.tv_sec = 2; + tvp = &tv; } + else + tvp = 0; - if (net_config->flags & FLAG_ASYNC) - return 0; + nr_desc = selectWithoutConsole(maxFd + 1, &read_set, tvp); - switch (ntohs(fromClient.opCode)) + if (nr_desc < 0) { - case CMD_CONNECT_REQ: - sendConnectionReply(db, fd[0], - net_config, - &client, - CAP_BIG_ENDIAN | - ntohl(fromClient.connectReq.capabilities), - ntohl(fromClient.connectReq.rcvbuf)); - return startNow; - - case CMD_GO: - return 1; - - case CMD_DISCONNECT: - ret = udpc_lookupParticipant(db, &client); - - if (ret >= 0) - udpc_removeParticipant(db, ret); - - return startNow; - - default: - break; + perror("select"); + return -1; } + if (nr_desc > 0) + /* key pressed, or receiver activity */ + break; + + if (net_config->rexmit_hello_interval) + { + /* retransmit hello message */ + udpc_sendHello(net_config, fd[0], 0); + (*tries)++; + + if (net_config->autostart != 0 && *tries > net_config->autostart) + startNow = 1; + } + + if (firstConnected) + startNow = startNow || checkClientWait(db, net_config, firstConnected); + } + + selected = udpc_getSelectedSock(fd, nr, &read_set); + + if (selected == -1) return startNow; + + BZERO(fromClient); /* Zero it out in order to cope with short messages + * from older versions */ + + msgLength = RECV(selected, fromClient, client, net_config->portBase); + + if (msgLength < 0) + { + perror("problem getting data from client"); + return 0; /* don't panic if we get weird messages */ + } + + if (net_config->flags & FLAG_ASYNC) + return 0; + + switch (ntohs(fromClient.opCode)) + { + case CMD_CONNECT_REQ: + sendConnectionReply(db, fd[0], net_config, &client, + CAP_BIG_ENDIAN | ntohl(fromClient.connectReq.capabilities), + ntohl(fromClient.connectReq.rcvbuf)); + return startNow; + + case CMD_GO: return 1; + + case CMD_DISCONNECT: + ret = udpc_lookupParticipant(db, &client); + + if (ret >= 0) + udpc_removeParticipant(db, ret); + + return startNow; + + default: break; + } + + return startNow; } static int isPointToPoint(participantsDb_t db, int flags) { - if (flags & FLAG_POINTOPOINT) - return 1; + if (flags & FLAG_POINTOPOINT) + return 1; - if (flags & (FLAG_NOPOINTOPOINT | FLAG_ASYNC)) - return 0; + if (flags & (FLAG_NOPOINTOPOINT | FLAG_ASYNC)) + return 0; - return udpc_nrParticipants(db) == 1; + return udpc_nrParticipants(db) == 1; } int getProducedAmount(produconsum_t pc) { - unsigned int produced = pc->produced; - unsigned int consumed = pc->consumed; + unsigned int produced = pc->produced; + unsigned int consumed = pc->consumed; - if (produced < consumed) - return produced + 2 * pc->size - consumed; - else - return produced - consumed; + if (produced < consumed) + return produced + 2 * pc->size - consumed; + else + return produced - consumed; } -int _consumeAny(produconsum_t pc, unsigned int minAmount, - struct timespec* ts) +int _consumeAny(produconsum_t pc, unsigned int minAmount, struct timespec* ts) { - unsigned int amount; + unsigned int amount; #if DEBUG - flprintf("%s: Waiting for %d bytes (%d:%d)\n", - pc->name, minAmount, pc->consumed, pc->produced); + flprintf("%s: Waiting for %d bytes (%d:%d)\n", pc->name, minAmount, pc->consumed, pc->produced); #endif - pc->consumerIsWaiting = 1; - amount = getProducedAmount(pc); + pc->consumerIsWaiting = 1; + amount = getProducedAmount(pc); - if (amount >= minAmount || pc->atEnd) - { - pc->consumerIsWaiting = 0; -#if DEBUG - flprintf("%s: got %d bytes\n", pc->name, amount); -#endif - return amount; - } - - pthread_mutex_lock(&pc->mutex); - - while ((amount = getProducedAmount(pc)) < minAmount && !pc->atEnd) - { -#if DEBUG - flprintf("%s: ..Waiting for %d bytes (%d:%d)\n", - pc->name, minAmount, pc->consumed, pc->produced); -#endif - - if (ts == 0) - pthread_cond_wait(&pc->cond, &pc->mutex); - else - { - int r; -#if DEBUG - flprintf("Before timed wait\n"); -#endif - r = pthread_cond_timedwait(&pc->cond, &pc->mutex, ts); -#if DEBUG - flprintf("After timed wait %d\n", r); -#endif - - if (r == ETIMEDOUT) - { - amount = getProducedAmount(pc); - break; - } - } - } - - pthread_mutex_unlock(&pc->mutex); -#if DEBUG - flprintf("%s: Got them %d (for %d) %d\n", pc->name, - amount, minAmount, pc->atEnd); -#endif + if (amount >= minAmount || pc->atEnd) + { pc->consumerIsWaiting = 0; +#if DEBUG + flprintf("%s: got %d bytes\n", pc->name, amount); +#endif return amount; + } + + pthread_mutex_lock(&pc->mutex); + + while ((amount = getProducedAmount(pc)) < minAmount && !pc->atEnd) + { +#if DEBUG + flprintf("%s: ..Waiting for %d bytes (%d:%d)\n", pc->name, minAmount, pc->consumed, pc->produced); +#endif + + if (ts == 0) + pthread_cond_wait(&pc->cond, &pc->mutex); + else + { + int r; +#if DEBUG + flprintf("Before timed wait\n"); +#endif + r = pthread_cond_timedwait(&pc->cond, &pc->mutex, ts); +#if DEBUG + flprintf("After timed wait %d\n", r); +#endif + + if (r == ETIMEDOUT) + { + amount = getProducedAmount(pc); + break; + } + } + } + + pthread_mutex_unlock(&pc->mutex); +#if DEBUG + flprintf("%s: Got them %d (for %d) %d\n", pc->name, amount, minAmount, pc->atEnd); +#endif + pc->consumerIsWaiting = 0; + return amount; } produconsum_t pc_makeProduconsum(int size, const char* name) { - produconsum_t pc = MALLOC(struct produconsum); - pc->size = size; - pc->produced = 0; - pc->consumed = 0; - pc->atEnd = 0; - pthread_mutex_init(&pc->mutex, NULL); - pc->consumerIsWaiting = 0; - pthread_cond_init(&pc->cond, NULL); - pc->name = name; - return pc; + produconsum_t pc = MALLOC(struct produconsum); + pc->size = size; + pc->produced = 0; + pc->consumed = 0; + pc->atEnd = 0; + pthread_mutex_init(&pc->mutex, NULL); + pc->consumerIsWaiting = 0; + pthread_cond_init(&pc->cond, NULL); + pc->name = name; + return pc; } int pc_consumeAnyWithTimeout(produconsum_t pc, struct timespec* ts) { - return _consumeAny(pc, 1, ts); + return _consumeAny(pc, 1, ts); } unsigned int pc_getProducerPosition(produconsum_t pc) { - return pc->produced % pc->size; + return pc->produced % pc->size; } unsigned int pc_getWaiting(produconsum_t pc) { - return getProducedAmount(pc); + return getProducedAmount(pc); } int pc_consumeAny(produconsum_t pc) { - return _consumeAny(pc, 1, 0); + return _consumeAny(pc, 1, 0); } int pc_consume(produconsum_t pc, int amount) { - return _consumeAny(pc, amount, 0); + return _consumeAny(pc, amount, 0); } void wakeConsumer(produconsum_t pc) { - if (pc->consumerIsWaiting) - { - pthread_mutex_lock(&pc->mutex); - pthread_cond_signal(&pc->cond); - pthread_mutex_unlock(&pc->mutex); - } + if (pc->consumerIsWaiting) + { + pthread_mutex_lock(&pc->mutex); + pthread_cond_signal(&pc->cond); + pthread_mutex_unlock(&pc->mutex); + } } void pc_produceEnd(produconsum_t pc) { - pc->atEnd = 1; - wakeConsumer(pc); + pc->atEnd = 1; + wakeConsumer(pc); } int pc_consumed(produconsum_t pc, int amount) { - unsigned int consumed = pc->consumed; + unsigned int consumed = pc->consumed; - if (consumed >= 2 * pc->size - amount) - { - consumed += amount - 2 * pc->size; - } - else - { - consumed += amount; - } + if (consumed >= 2 * pc->size - amount) + { + consumed += amount - 2 * pc->size; + } + else + { + consumed += amount; + } - pc->consumed = consumed; - return amount; + pc->consumed = consumed; + return amount; } unsigned int pc_getConsumerPosition(produconsum_t pc) { - return pc->consumed % pc->size; + return pc->consumed % pc->size; } void pc_produce(produconsum_t pc, unsigned int amount) { - unsigned int produced = pc->produced; - unsigned int consumed = pc->consumed; + unsigned int produced = pc->produced; + unsigned int consumed = pc->consumed; - /* sanity checks: - * 1. should not produce more than size - * 2. do not pass consumed+size - */ - if (amount > pc->size) - { - //TODO: FIXME - //udpc_fatal(1, "Buffer overflow in produce %s: %d > %d \n", pc->name, amount, pc->size); - } + /* sanity checks: + * 1. should not produce more than size + * 2. do not pass consumed+size + */ + if (amount > pc->size) + { + // TODO: FIXME + // udpc_fatal(1, "Buffer overflow in produce %s: %d > %d \n", pc->name, amount, pc->size); + } - produced += amount; + produced += amount; - if (produced >= 2 * pc->size) - produced -= 2 * pc->size; + if (produced >= 2 * pc->size) + produced -= 2 * pc->size; - if (produced > consumed + pc->size || - (produced < consumed && produced > consumed - pc->size)) - { - //TODO: FIXME - //udpc_fatal(1, "Buffer overflow in produce %s: %d > %d [%d] \n", pc->name, produced, consumed, pc->size); - } + if (produced > consumed + pc->size || (produced < consumed && produced > consumed - pc->size)) + { + // TODO: FIXME + // udpc_fatal(1, "Buffer overflow in produce %s: %d > %d [%d] \n", pc->name, produced, consumed, + // pc->size); + } - pc->produced = produced; - wakeConsumer(pc); + pc->produced = produced; + wakeConsumer(pc); } void udpc_initFifo(struct fifo* fifo, int blockSize) { - fifo->dataBufSize = blockSize * 4096; - fifo->dataBuffer = (unsigned char*)malloc(fifo->dataBufSize + 4096); - fifo->dataBuffer += 4096 - (((unsigned long)fifo->dataBuffer) % 4096); + fifo->dataBufSize = blockSize * 4096; + fifo->dataBuffer = (unsigned char*)malloc(fifo->dataBufSize + 4096); + fifo->dataBuffer += 4096 - (((unsigned long)fifo->dataBuffer) % 4096); - /* Free memory queue is initially full */ - fifo->freeMemQueue = pc_makeProduconsum(fifo->dataBufSize, "free mem"); - pc_produce(fifo->freeMemQueue, fifo->dataBufSize); + /* Free memory queue is initially full */ + fifo->freeMemQueue = pc_makeProduconsum(fifo->dataBufSize, "free mem"); + pc_produce(fifo->freeMemQueue, fifo->dataBufSize); - fifo->data = pc_makeProduconsum(fifo->dataBufSize, "receive"); + fifo->data = pc_makeProduconsum(fifo->dataBufSize, "receive"); } THREAD_RETURN returnChannelMain(void* args) { - struct returnChannel* returnChannel = (struct returnChannel*) args; + struct returnChannel* returnChannel = (struct returnChannel*)args; - while (1) + while (1) + { + struct sockaddr_in from; + int clNo; + int pos = pc_getConsumerPosition(returnChannel->freeSpace); + pc_consumeAny(returnChannel->freeSpace); + + RECV(returnChannel->rcvSock, returnChannel->q[pos].msg, from, returnChannel->config->portBase); + clNo = udpc_lookupParticipant(returnChannel->participantsDb, &from); + + if (clNo < 0) { - struct sockaddr_in from; - int clNo; - int pos = pc_getConsumerPosition(returnChannel->freeSpace); - pc_consumeAny(returnChannel->freeSpace); - - RECV(returnChannel->rcvSock, - returnChannel->q[pos].msg, from, - returnChannel->config->portBase); - clNo = udpc_lookupParticipant(returnChannel->participantsDb, &from); - - if (clNo < 0) - { - /* packet from unknown provenance */ - continue; - } - - returnChannel->q[pos].clNo = clNo; - pc_consumed(returnChannel->freeSpace, 1); - pc_produce(returnChannel->incoming, 1); + /* packet from unknown provenance */ + continue; } - return 0; + returnChannel->q[pos].clNo = clNo; + pc_consumed(returnChannel->freeSpace, 1); + pc_produce(returnChannel->incoming, 1); + } + + return 0; } -void initReturnChannel(struct returnChannel* returnChannel, - struct net_config* config, - int sock) +void initReturnChannel(struct returnChannel* returnChannel, struct net_config* config, int sock) { - returnChannel->config = config; - returnChannel->rcvSock = sock; - returnChannel->freeSpace = pc_makeProduconsum(QUEUE_SIZE, "msg:free-queue"); - pc_produce(returnChannel->freeSpace, QUEUE_SIZE); - returnChannel->incoming = pc_makeProduconsum(QUEUE_SIZE, "msg:incoming"); - - pthread_create(&returnChannel->thread, NULL, - returnChannelMain, returnChannel); + returnChannel->config = config; + returnChannel->rcvSock = sock; + returnChannel->freeSpace = pc_makeProduconsum(QUEUE_SIZE, "msg:free-queue"); + pc_produce(returnChannel->freeSpace, QUEUE_SIZE); + returnChannel->incoming = pc_makeProduconsum(QUEUE_SIZE, "msg:incoming"); + pthread_create(&returnChannel->thread, NULL, returnChannelMain, returnChannel); } void senderStatsAddBytes(sender_stats_t ss, long bytes) { - if (ss != NULL) + if (ss != NULL) + { + ss->totalBytes += bytes; + + if (ss->bwPeriod) { - ss->totalBytes += bytes; + double tdiff, bw; + struct timeval tv; + gettimeofday(&tv, 0); + ss->periodBytes += bytes; - if (ss->bwPeriod) - { - double tdiff, bw; - struct timeval tv; - gettimeofday(&tv, 0); - ss->periodBytes += bytes; + if (tv.tv_sec - ss->periodStart.tv_sec < ss->bwPeriod - 1) + return; - if (tv.tv_sec - ss->periodStart.tv_sec < ss->bwPeriod - 1) - return; + tdiff = (tv.tv_sec - ss->periodStart.tv_sec) * 1000000.0 + tv.tv_usec - ss->periodStart.tv_usec; - tdiff = (tv.tv_sec - ss->periodStart.tv_sec) * 1000000.0 + - tv.tv_usec - ss->periodStart.tv_usec; + if (tdiff < ss->bwPeriod * 1000000.0) + return; - if (tdiff < ss->bwPeriod * 1000000.0) - return; - - bw = ss->periodBytes * 8.0 / tdiff; - ss->periodBytes = 0; - ss->periodStart = tv; - } + bw = ss->periodBytes * 8.0 / tdiff; + ss->periodBytes = 0; + ss->periodStart = tv; } + } } -int ackSlice(struct slice* slice, struct net_config* net_config, - struct fifo* fifo, sender_stats_t stats) +int ackSlice(struct slice* slice, struct net_config* net_config, struct fifo* fifo, sender_stats_t stats) { - if (slice->state == slice::SLICE_ACKED) - /* already acked */ - return 0; - - if (!(net_config->flags & FLAG_SN)) - { - if (net_config->discovery == net_config::DSC_DOUBLING) - { - net_config->sliceSize += net_config->sliceSize / 4; - - if (net_config->sliceSize >= net_config->max_slice_size) - { - net_config->sliceSize = net_config->max_slice_size; - net_config->discovery = net_config::DSC_REDUCING; - } - } - } - - slice->state = slice::SLICE_ACKED; - pc_produce(fifo->freeMemQueue, slice->bytes); - - /* Statistics */ - senderStatsAddBytes(stats, slice->bytes); - - /* End Statistics */ - + if (slice->state == slice::SLICE_ACKED) + /* already acked */ return 0; + + if (!(net_config->flags & FLAG_SN)) + { + if (net_config->discovery == net_config::DSC_DOUBLING) + { + net_config->sliceSize += net_config->sliceSize / 4; + + if (net_config->sliceSize >= net_config->max_slice_size) + { + net_config->sliceSize = net_config->max_slice_size; + net_config->discovery = net_config::DSC_REDUCING; + } + } + } + + slice->state = slice::SLICE_ACKED; + pc_produce(fifo->freeMemQueue, slice->bytes); + + /* Statistics */ + senderStatsAddBytes(stats, slice->bytes); + + /* End Statistics */ + + return 0; } int isSliceAcked(struct slice* slice) { - if (slice->state == slice::SLICE_ACKED) - { - return 1; - } - else - { - return 0; - } + if (slice->state == slice::SLICE_ACKED) + { + return 1; + } + else + { + return 0; + } } int freeSlice(sender_state_t sendst, struct slice* slice) { - int i; - i = slice - sendst->slices; - slice->state = slice::SLICE_PRE_FREE; + int i; + i = slice - sendst->slices; + slice->state = slice::SLICE_PRE_FREE; - while (1) - { - int pos = pc_getProducerPosition(sendst->free_slices_pc); + while (1) + { + int pos = pc_getProducerPosition(sendst->free_slices_pc); - if (sendst->slices[pos].state == slice::SLICE_PRE_FREE) - sendst->slices[pos].state = slice::SLICE_FREE; - else - break; + if (sendst->slices[pos].state == slice::SLICE_PRE_FREE) + sendst->slices[pos].state = slice::SLICE_FREE; + else + break; - pc_produce(sendst->free_slices_pc, 1); - } + pc_produce(sendst->free_slices_pc, 1); + } - return 0; + return 0; } int isSliceXmitted(struct slice* slice) { - if (slice->state == slice::SLICE_XMITTED) - { - return 1; - } - else - { - return 0; - } + if (slice->state == slice::SLICE_XMITTED) + { + return 1; + } + else + { + return 0; + } } int getSliceBlocks(struct slice* slice, struct net_config* net_config) { - return (slice->bytes + net_config->blockSize - 1) / net_config->blockSize; + return (slice->bytes + net_config->blockSize - 1) / net_config->blockSize; } -int sendReqack(struct slice* slice, struct net_config* net_config, - struct fifo* fifo, sender_stats_t stats, +int sendReqack(struct slice* slice, struct net_config* net_config, struct fifo* fifo, sender_stats_t stats, int sock) { - /* in async mode, just confirm slice... */ - if ((net_config->flags & FLAG_ASYNC) && slice->bytes != 0) - { - ackSlice(slice, net_config, fifo, stats); - return 0; - } - - if ((net_config->flags & FLAG_ASYNC) -#ifdef BB_FEATURE_UDPCAST_FEC - && - (net_config->flags & FLAG_FEC) -#endif - ) - { - return 0; - } - - if (!(net_config->flags & FLAG_SN) && slice->rxmitId != 0) - { - int nrBlocks; - nrBlocks = getSliceBlocks(slice, net_config); - - if (slice->lastGoodBlock != 0 && slice->lastGoodBlock < nrBlocks) - { - net_config->discovery = net_config::DSC_REDUCING; - - if (slice->lastGoodBlock < net_config->sliceSize / 2) - { - net_config->sliceSize = net_config->sliceSize / 2; - } - else - { - net_config->sliceSize = slice->lastGoodBlock; - } - - if (net_config->sliceSize < 32) - { - /* a minimum of 32 */ - net_config->sliceSize = 32; - } - } - } - - slice->lastGoodBlock = 0; - slice->sl_reqack.ra.opCode = htons(CMD_REQACK); - slice->sl_reqack.ra.sliceNo = htonl(slice->sliceNo); - slice->sl_reqack.ra.bytes = htonl(slice->bytes); - - slice->sl_reqack.ra.reserved = 0; - memcpy((void*)&slice->answeredSet, (void*)&slice->sl_reqack.readySet, - sizeof(slice->answeredSet)); - slice->nrAnswered = slice->nrReady; - - /* not everybody is ready yet */ - slice->needRxmit = 0; - memset(slice->rxmitMap, 0, sizeof(slice->rxmitMap)); - memset(slice->isXmittedMap, 0, sizeof(slice->isXmittedMap)); - slice->sl_reqack.ra.rxmit = htonl(slice->rxmitId); - - //rgWaitAll(net_config, sock, net_config->dataMcastAddr.sin_addr.s_addr, sizeof(slice->sl_reqack)); - BCAST_DATA(sock, slice->sl_reqack); + /* in async mode, just confirm slice... */ + if ((net_config->flags & FLAG_ASYNC) && slice->bytes != 0) + { + ackSlice(slice, net_config, fifo, stats); return 0; + } + + if ((net_config->flags & FLAG_ASYNC) +#ifdef BB_FEATURE_UDPCAST_FEC + && (net_config->flags & FLAG_FEC) +#endif + ) + { + return 0; + } + + if (!(net_config->flags & FLAG_SN) && slice->rxmitId != 0) + { + int nrBlocks; + nrBlocks = getSliceBlocks(slice, net_config); + + if (slice->lastGoodBlock != 0 && slice->lastGoodBlock < nrBlocks) + { + net_config->discovery = net_config::DSC_REDUCING; + + if (slice->lastGoodBlock < net_config->sliceSize / 2) + { + net_config->sliceSize = net_config->sliceSize / 2; + } + else + { + net_config->sliceSize = slice->lastGoodBlock; + } + + if (net_config->sliceSize < 32) + { + /* a minimum of 32 */ + net_config->sliceSize = 32; + } + } + } + + slice->lastGoodBlock = 0; + slice->sl_reqack.ra.opCode = htons(CMD_REQACK); + slice->sl_reqack.ra.sliceNo = htonl(slice->sliceNo); + slice->sl_reqack.ra.bytes = htonl(slice->bytes); + + slice->sl_reqack.ra.reserved = 0; + memcpy((void*)&slice->answeredSet, (void*)&slice->sl_reqack.readySet, sizeof(slice->answeredSet)); + slice->nrAnswered = slice->nrReady; + + /* not everybody is ready yet */ + slice->needRxmit = 0; + memset(slice->rxmitMap, 0, sizeof(slice->rxmitMap)); + memset(slice->isXmittedMap, 0, sizeof(slice->isXmittedMap)); + slice->sl_reqack.ra.rxmit = htonl(slice->rxmitId); + + // rgWaitAll(net_config, sock, net_config->dataMcastAddr.sin_addr.s_addr, sizeof(slice->sl_reqack)); + BCAST_DATA(sock, slice->sl_reqack); + return 0; } -struct slice* findSlice(struct slice* slice1, - struct slice* slice2, - int sliceNo) +struct slice* findSlice(struct slice* slice1, struct slice* slice2, int sliceNo) { - if (slice1 != NULL && slice1->sliceNo == sliceNo) - return slice1; + if (slice1 != NULL && slice1->sliceNo == sliceNo) + return slice1; - if (slice2 != NULL && slice2->sliceNo == sliceNo) - return slice2; + if (slice2 != NULL && slice2->sliceNo == sliceNo) + return slice2; - return NULL; + return NULL; } void markParticipantAnswered(slice_t slice, int clNo) { - if (BIT_ISSET(clNo, slice->answeredSet)) - /* client already has answered */ - return; + if (BIT_ISSET(clNo, slice->answeredSet)) + /* client already has answered */ + return; - slice->nrAnswered++; - SET_BIT(clNo, slice->answeredSet); + slice->nrAnswered++; + SET_BIT(clNo, slice->answeredSet); } int udpc_isParticipantValid(struct participantsDb* db, int i) { - return db->clientTable[i].used; + return db->clientTable[i].used; } void senderSetAnswered(sender_stats_t ss, int clNo) { - if (ss != NULL) - ss->clNo = clNo; + if (ss != NULL) + ss->clNo = clNo; } -int handleOk(sender_state_t sendst, - struct slice* slice, - int clNo) +int handleOk(sender_state_t sendst, struct slice* slice, int clNo) { - if (slice == NULL) - return 0; - - if (!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) - { - //udpc_flprintf("Invalid participant %d\n", clNo); - return 0; - } - - if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) - { - /* client is already marked ready */ - } - else - { - SET_BIT(clNo, slice->sl_reqack.readySet); - slice->nrReady++; - senderSetAnswered(sendst->stats, clNo); - markParticipantAnswered(slice, clNo); - } - + if (slice == NULL) return 0; + + if (!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) + { + // udpc_flprintf("Invalid participant %d\n", clNo); + return 0; + } + + if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) + { + /* client is already marked ready */ + } + else + { + SET_BIT(clNo, slice->sl_reqack.readySet); + slice->nrReady++; + senderSetAnswered(sendst->stats, clNo); + markParticipantAnswered(slice, clNo); + } + + return 0; } int handleDisconnect1(struct slice* slice, int clNo) { - if (slice != NULL) + if (slice != NULL) + { + if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) { - if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) - { - /* avoid counting client both as left and ready */ - CLR_BIT(clNo, slice->sl_reqack.readySet); - slice->nrReady--; - } - - if (BIT_ISSET(clNo, slice->answeredSet)) - { - slice->nrAnswered--; - CLR_BIT(clNo, slice->answeredSet); - } + /* avoid counting client both as left and ready */ + CLR_BIT(clNo, slice->sl_reqack.readySet); + slice->nrReady--; } - return 0; + if (BIT_ISSET(clNo, slice->answeredSet)) + { + slice->nrAnswered--; + CLR_BIT(clNo, slice->answeredSet); + } + } + + return 0; } -int handleDisconnect(participantsDb_t db, - struct slice* slice1, - struct slice* slice2, - int clNo) +int handleDisconnect(participantsDb_t db, struct slice* slice1, struct slice* slice2, int clNo) { - handleDisconnect1(slice1, clNo); - handleDisconnect1(slice2, clNo); - udpc_removeParticipant(db, clNo); - return 0; + handleDisconnect1(slice1, clNo); + handleDisconnect1(slice2, clNo); + udpc_removeParticipant(db, clNo); + return 0; } -int handleRetransmit(sender_state_t sendst, - struct slice* slice, - int clNo, unsigned char* map, int rxmit) +int handleRetransmit(sender_state_t sendst, struct slice* slice, int clNo, unsigned char* map, int rxmit) { - unsigned int i; + unsigned int i; - if (!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) - { - //udpc_flprintf("Invalid participant %d\n", clNo); - return 0; - } - - if (slice == NULL) - return 0; - - if (rxmit < slice->rxmitId) - { - /* late answer to previous Req Ack */ - return 0; - } - - for (i = 0; i < sizeof(slice->rxmitMap) / sizeof(char); i++) - { - slice->rxmitMap[i] |= ~map[i]; - } - - slice->needRxmit = 1; - markParticipantAnswered(slice, clNo); + if (!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) + { + // udpc_flprintf("Invalid participant %d\n", clNo); return 0; + } + + if (slice == NULL) + return 0; + + if (rxmit < slice->rxmitId) + { + /* late answer to previous Req Ack */ + return 0; + } + + for (i = 0; i < sizeof(slice->rxmitMap) / sizeof(char); i++) + { + slice->rxmitMap[i] |= ~map[i]; + } + + slice->needRxmit = 1; + markParticipantAnswered(slice, clNo); + return 0; } -int handleNextMessage(sender_state_t sendst, - struct slice* xmitSlice, - struct slice* rexmitSlice) +int handleNextMessage(sender_state_t sendst, struct slice* xmitSlice, struct slice* rexmitSlice) { - int pos = pc_getConsumerPosition(sendst->rc.incoming); - union message* msg = &sendst->rc.q[pos].msg; - int clNo = sendst->rc.q[pos].clNo; + int pos = pc_getConsumerPosition(sendst->rc.incoming); + union message* msg = &sendst->rc.q[pos].msg; + int clNo = sendst->rc.q[pos].clNo; - pc_consumeAny(sendst->rc.incoming); + pc_consumeAny(sendst->rc.incoming); - switch (ntohs(msg->opCode)) - { - case CMD_OK: - handleOk(sendst, - findSlice(xmitSlice, rexmitSlice, ntohl(msg->ok.sliceNo)), clNo); - break; + switch (ntohs(msg->opCode)) + { + case CMD_OK: handleOk(sendst, findSlice(xmitSlice, rexmitSlice, ntohl(msg->ok.sliceNo)), clNo); break; - case CMD_DISCONNECT: - handleDisconnect(sendst->rc.participantsDb, - xmitSlice, rexmitSlice, clNo); - break; + case CMD_DISCONNECT: handleDisconnect(sendst->rc.participantsDb, xmitSlice, rexmitSlice, clNo); break; - case CMD_RETRANSMIT: - handleRetransmit(sendst, - findSlice(xmitSlice, rexmitSlice, - ntohl(msg->retransmit.sliceNo)), - clNo, - msg->retransmit.map, - msg->retransmit.rxmit); - break; + case CMD_RETRANSMIT: + handleRetransmit(sendst, findSlice(xmitSlice, rexmitSlice, ntohl(msg->retransmit.sliceNo)), clNo, + msg->retransmit.map, msg->retransmit.rxmit); + break; - default: - //TODO: FIXME - //udpc_flprintf("Bad command %04x\n", (unsigned short) msg->opCode); - break; - } + default: + // TODO: FIXME + // udpc_flprintf("Bad command %04x\n", (unsigned short) msg->opCode); + break; + } - pc_consumed(sendst->rc.incoming, 1); - pc_produce(sendst->rc.freeSpace, 1); - return 0; + pc_consumed(sendst->rc.incoming, 1); + pc_produce(sendst->rc.freeSpace, 1); + return 0; } -int sendRawData(int sock, - struct net_config* config, - char* header, int headerSize, - unsigned char* data, int dataSize) +int sendRawData(int sock, struct net_config* config, char* header, int headerSize, unsigned char* data, + int dataSize) { - struct iovec iov[2]; - struct msghdr hdr; - int packetSize; - int ret; + struct iovec iov[2]; + struct msghdr hdr; + int packetSize; + int ret; - iov[0].iov_base = header; - iov[0].iov_len = headerSize; + iov[0].iov_base = header; + iov[0].iov_len = headerSize; - iov[1].iov_base = data; - iov[1].iov_len = dataSize; + iov[1].iov_base = data; + iov[1].iov_len = dataSize; - hdr.msg_name = &config->dataMcastAddr; - hdr.msg_namelen = sizeof(struct sockaddr_in); - hdr.msg_iov = iov; - hdr.msg_iovlen = 2; - initMsgHdr(&hdr); + hdr.msg_name = &config->dataMcastAddr; + hdr.msg_namelen = sizeof(struct sockaddr_in); + hdr.msg_iov = iov; + hdr.msg_iovlen = 2; + initMsgHdr(&hdr); - packetSize = dataSize + headerSize; - //rgWaitAll(config, sock, config->dataMcastAddr.sin_addr.s_addr, packetSize); - ret = sendmsg(sock, &hdr, 0); + packetSize = dataSize + headerSize; + // rgWaitAll(config, sock, config->dataMcastAddr.sin_addr.s_addr, packetSize); + ret = sendmsg(sock, &hdr, 0); - if (ret < 0) - { - //TODO: FIXME - } + if (ret < 0) + { + // TODO: FIXME + } - return 0; + return 0; } int transmitDataBlock(sender_state_t sendst, struct slice* slice, int i) { - struct fifo* fifo = sendst->fifo; - struct net_config* config = sendst->config; - struct dataBlock msg; + struct fifo* fifo = sendst->fifo; + struct net_config* config = sendst->config; + struct dataBlock msg; - idbassert(i < MAX_SLICE_SIZE); + idbassert(i < MAX_SLICE_SIZE); - msg.opCode = htons(CMD_DATA); - msg.sliceNo = htonl(slice->sliceNo); - msg.blockNo = htons(i); + msg.opCode = htons(CMD_DATA); + msg.sliceNo = htonl(slice->sliceNo); + msg.blockNo = htons(i); - msg.reserved = 0; - msg.reserved2 = 0; - msg.bytes = htonl(slice->bytes); + msg.reserved = 0; + msg.reserved2 = 0; + msg.bytes = htonl(slice->bytes); - sendRawData(sendst->socket, config, - (char*) &msg, sizeof(msg), - fifo->dataBuffer + - (slice->base + i * config->blockSize) % fifo->dataBufSize, - config->blockSize); - return 0; + sendRawData(sendst->socket, config, (char*)&msg, sizeof(msg), + fifo->dataBuffer + (slice->base + i * config->blockSize) % fifo->dataBufSize, + config->blockSize); + return 0; } void senderStatsAddRetransmissions(sender_stats_t ss, int retransmissions) { - if (ss != NULL) - { - ss->retransmissions += retransmissions; - } + if (ss != NULL) + { + ss->retransmissions += retransmissions; + } } -int sendSlice(sender_state_t sendst, struct slice* slice, - int retransmitting) +int sendSlice(sender_state_t sendst, struct slice* slice, int retransmitting) { - struct net_config* config = sendst->config; + struct net_config* config = sendst->config; - int nrBlocks, i, rehello; + int nrBlocks, i, rehello; #ifdef BB_FEATURE_UDPCAST_FEC - int fecBlocks; + int fecBlocks; #endif - int retransmissions = 0; + int retransmissions = 0; + if (retransmitting) + { + slice->nextBlock = 0; + + if (slice->state != slice::SLICE_XMITTED) + return 0; + } + else + { + if (slice->state != slice::SLICE_NEW) + return 0; + } + + nrBlocks = getSliceBlocks(slice, config); +#ifdef BB_FEATURE_UDPCAST_FEC + + if ((config->flags & FLAG_FEC) && !retransmitting) + { + fecBlocks = config->fec_redundancy * config->fec_stripes; + } + else + { + fecBlocks = 0; + } + +#endif + + if ((sendst->config->flags & FLAG_STREAMING)) + { + rehello = nrBlocks - sendst->config->rehelloOffset; + + if (rehello < 0) + rehello = 0; + } + else + { + rehello = -1; + } + + /* transmit the data */ + for (i = slice->nextBlock; i < nrBlocks +#ifdef BB_FEATURE_UDPCAST_FEC + + fecBlocks +#endif + ; + i++) + { if (retransmitting) { - slice->nextBlock = 0; + if (!BIT_ISSET(i, slice->rxmitMap) || BIT_ISSET(i, slice->isXmittedMap)) + { + /* if slice is not in retransmit list, or has _already_ + * been retransmitted, skip it */ + if (i > slice->lastGoodBlock) + slice->lastGoodBlock = i; - if (slice->state != slice::SLICE_XMITTED) - return 0; + continue; + } + + SET_BIT(i, slice->isXmittedMap); + retransmissions++; } + + if (i == rehello) + { + udpc_sendHello(sendst->config, sendst->socket, 1); + } + + if (i < nrBlocks) + transmitDataBlock(sendst, slice, i); + +#ifdef BB_FEATURE_UDPCAST_FEC else - { - if (slice->state != slice::SLICE_NEW) - return 0; - } - - nrBlocks = getSliceBlocks(slice, config); -#ifdef BB_FEATURE_UDPCAST_FEC - - if ((config->flags & FLAG_FEC) && !retransmitting) - { - fecBlocks = config->fec_redundancy * config->fec_stripes; - } - else - { - fecBlocks = 0; - } + transmitFecBlock(sendst, slice, i - nrBlocks); #endif - if ((sendst->config->flags & FLAG_STREAMING)) + if (!retransmitting && pc_getWaiting(sendst->rc.incoming)) { - rehello = nrBlocks - sendst->config->rehelloOffset; - - if (rehello < 0) - rehello = 0; - } - else - { - rehello = -1; + i++; + break; } + } - /* transmit the data */ - for (i = slice->nextBlock; i < nrBlocks + if (retransmissions) + senderStatsAddRetransmissions(sendst->stats, retransmissions); + + slice->nextBlock = i; + + if (i == nrBlocks #ifdef BB_FEATURE_UDPCAST_FEC - + fecBlocks + + fecBlocks #endif - ; i++) - { - if (retransmitting) - { - if (!BIT_ISSET(i, slice->rxmitMap) || - BIT_ISSET(i, slice->isXmittedMap)) - { - /* if slice is not in retransmit list, or has _already_ - * been retransmitted, skip it */ - if (i > slice->lastGoodBlock) - slice->lastGoodBlock = i; + ) + { + slice->needRxmit = 0; - continue; - } + if (!retransmitting) + slice->state = slice::SLICE_XMITTED; - SET_BIT(i, slice->isXmittedMap); - retransmissions++; - } + return 2; + } - if (i == rehello) - { - udpc_sendHello(sendst->config, sendst->socket, 1); - } - - if (i < nrBlocks) - transmitDataBlock(sendst, slice, i); - -#ifdef BB_FEATURE_UDPCAST_FEC - else - transmitFecBlock(sendst, slice, i - nrBlocks); - -#endif - - if (!retransmitting && pc_getWaiting(sendst->rc.incoming)) - { - i++; - break; - } - } - - if (retransmissions) - senderStatsAddRetransmissions(sendst->stats, retransmissions); - - slice->nextBlock = i; - - if (i == nrBlocks -#ifdef BB_FEATURE_UDPCAST_FEC - + fecBlocks -#endif - ) - { - slice->needRxmit = 0; - - if (!retransmitting) - slice->state = slice::SLICE_XMITTED; - - return 2; - } - - return 1; + return 1; } -int doRetransmissions(sender_state_t sendst, - struct slice* slice) +int doRetransmissions(sender_state_t sendst, struct slice* slice) { - if (slice->state == slice::SLICE_ACKED) - return 0; /* nothing to do */ + if (slice->state == slice::SLICE_ACKED) + return 0; /* nothing to do */ - /* FIXME: reduce slice size if needed */ - if (slice->needRxmit) - { - /* do some retransmissions */ - sendSlice(sendst, slice, 1); - } + /* FIXME: reduce slice size if needed */ + if (slice->needRxmit) + { + /* do some retransmissions */ + sendSlice(sendst, slice, 1); + } - return 0; + return 0; } struct slice* makeSlice(sender_state_t sendst, int sliceNo) { - struct net_config* config = sendst->config; - struct fifo* fifo = sendst->fifo; - int i; - struct slice* slice = NULL; + struct net_config* config = sendst->config; + struct fifo* fifo = sendst->fifo; + int i; + struct slice* slice = NULL; - pc_consume(sendst->free_slices_pc, 1); - i = pc_getConsumerPosition(sendst->free_slices_pc); - slice = &sendst->slices[i]; - idbassert(slice->state == slice::SLICE_FREE); - BZERO(*slice); - pc_consumed(sendst->free_slices_pc, 1); + pc_consume(sendst->free_slices_pc, 1); + i = pc_getConsumerPosition(sendst->free_slices_pc); + slice = &sendst->slices[i]; + idbassert(slice->state == slice::SLICE_FREE); + BZERO(*slice); + pc_consumed(sendst->free_slices_pc, 1); - slice->base = pc_getConsumerPosition(sendst->fifo->data); - slice->sliceNo = sliceNo; - slice->bytes = pc_consume(fifo->data, 10 * config->blockSize); + slice->base = pc_getConsumerPosition(sendst->fifo->data); + slice->sliceNo = sliceNo; + slice->bytes = pc_consume(fifo->data, 10 * config->blockSize); - /* fixme: use current slice size here */ - if (slice->bytes > config->blockSize * config->sliceSize) - slice->bytes = config->blockSize * config->sliceSize; + /* fixme: use current slice size here */ + if (slice->bytes > config->blockSize * config->sliceSize) + slice->bytes = config->blockSize * config->sliceSize; - pc_consumed(fifo->data, slice->bytes); - slice->nextBlock = 0; - slice->state = slice::SLICE_NEW; - BZERO(slice->sl_reqack.readySet); - slice->nrReady = 0; + pc_consumed(fifo->data, slice->bytes); + slice->nextBlock = 0; + slice->state = slice::SLICE_NEW; + BZERO(slice->sl_reqack.readySet); + slice->nrReady = 0; #ifdef BB_FEATURE_UDPCAST_FEC - slice->fec_data = sendst->fec_data + (i * config->fec_stripes * - config->fec_redundancy * - config->blockSize); + slice->fec_data = sendst->fec_data + (i * config->fec_stripes * config->fec_redundancy * config->blockSize); #endif - return slice; + return slice; } void cancelReturnChannel(struct returnChannel* returnChannel) { - /* No need to worry about the pthread_cond_wait in produconsum, because - * at the point where we enter here (to cancel the thread), we are sure - * that nobody else uses that produconsum any more - */ - pthread_cancel(returnChannel->thread); - pthread_join(returnChannel->thread, NULL); + /* No need to worry about the pthread_cond_wait in produconsum, because + * at the point where we enter here (to cancel the thread), we are sure + * that nobody else uses that produconsum any more + */ + pthread_cancel(returnChannel->thread); + pthread_join(returnChannel->thread, NULL); } -THREAD_RETURN netSenderMain(void* args0) +THREAD_RETURN netSenderMain(void* args0) { - sender_state_t sendst = (sender_state_t) args0; - struct net_config* config = sendst->config; - struct timeval tv; - struct timespec ts; - int atEnd = 0; - int nrWaited = 0; - unsigned long waitAverage = 10000; /* Exponential average of last wait times */ + sender_state_t sendst = (sender_state_t)args0; + struct net_config* config = sendst->config; + struct timeval tv; + struct timespec ts; + int atEnd = 0; + int nrWaited = 0; + unsigned long waitAverage = 10000; /* Exponential average of last wait times */ - struct slice* xmitSlice = NULL; /* slice being transmitted a first time */ - struct slice* rexmitSlice = NULL; /* slice being re-transmitted */ - int sliceNo = 0; + struct slice* xmitSlice = NULL; /* slice being transmitted a first time */ + struct slice* rexmitSlice = NULL; /* slice being re-transmitted */ + int sliceNo = 0; - /* transmit the data */ - if (config->default_slice_size == 0) - { + /* transmit the data */ + if (config->default_slice_size == 0) + { #ifdef BB_FEATURE_UDPCAST_FEC - if (config->flags & FLAG_FEC) + if (config->flags & FLAG_FEC) + { + config->sliceSize = config->fec_stripesize * config->fec_stripes; + } + else +#endif + if (config->flags & FLAG_ASYNC) + config->sliceSize = 1024; + else if (sendst->config->flags & FLAG_SN) + { + sendst->config->sliceSize = 112; + } + else + sendst->config->sliceSize = 130; + + sendst->config->discovery = net_config::DSC_DOUBLING; + } + else + { + config->sliceSize = config->default_slice_size; +#ifdef BB_FEATURE_UDPCAST_FEC + + if ((config->flags & FLAG_FEC) && (config->sliceSize > 128 * config->fec_stripes)) + config->sliceSize = 128 * config->fec_stripes; + +#endif + } + +#ifdef BB_FEATURE_UDPCAST_FEC + + if ((sendst->config->flags & FLAG_FEC) && config->max_slice_size > config->fec_stripes * 128) + config->max_slice_size = config->fec_stripes * 128; + +#endif + + if (config->sliceSize > config->max_slice_size) + config->sliceSize = config->max_slice_size; + + idbassert(config->sliceSize <= MAX_SLICE_SIZE); + + do + { + /* first, cleanup rexmit Slice if needed */ + + if (rexmitSlice != NULL) + { + if (rexmitSlice->nrReady == udpc_nrParticipants(sendst->rc.participantsDb)) + { +#if DEBUG + flprintf("slice is ready\n"); +#endif + ackSlice(rexmitSlice, sendst->config, sendst->fifo, sendst->stats); + } + + if (isSliceAcked(rexmitSlice)) + { + freeSlice(sendst, rexmitSlice); + rexmitSlice = NULL; + } + } + + /* then shift xmit slice to rexmit slot, if possible */ + if (rexmitSlice == NULL && xmitSlice != NULL && isSliceXmitted(xmitSlice)) + { + rexmitSlice = xmitSlice; + xmitSlice = NULL; + sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, sendst->socket); + } + + /* handle any messages */ + if (pc_getWaiting(sendst->rc.incoming)) + { +#if DEBUG + flprintf("Before message %d\n", pc_getWaiting(sendst->rc.incoming)); +#endif + handleNextMessage(sendst, xmitSlice, rexmitSlice); + + /* restart at beginning of loop: we may have acked the rxmit + * slice, makeing it possible to shift the pipe */ + continue; + } + + /* do any needed retransmissions */ + if (rexmitSlice != NULL && rexmitSlice->needRxmit) + { + doRetransmissions(sendst, rexmitSlice); + /* restart at beginning: new messages may have arrived during + * retransmission */ + continue; + } + + /* if all participants answered, send req ack */ + if (rexmitSlice != NULL && rexmitSlice->nrAnswered == udpc_nrParticipants(sendst->rc.participantsDb)) + { + rexmitSlice->rxmitId++; + sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, sendst->socket); + } + + if (xmitSlice == NULL && !atEnd) + { +#if DEBUG + flprintf("SN=%d\n", sendst->config->flags & FLAG_SN); +#endif + + if ((sendst->config->flags & FLAG_SN) || rexmitSlice == NULL) + { +#ifdef BB_FEATURE_UDPCAST_FEC + + if (sendst->config->flags & FLAG_FEC) { - config->sliceSize = - config->fec_stripesize * config->fec_stripes; + int i; + pc_consume(sendst->fec_data_pc, 1); + i = pc_getConsumerPosition(sendst->fec_data_pc); + xmitSlice = &sendst->slices[i]; + pc_consumed(sendst->fec_data_pc, 1); } else #endif - if (config->flags & FLAG_ASYNC) - config->sliceSize = 1024; - else if (sendst->config->flags & FLAG_SN) - { - sendst->config->sliceSize = 112; - } - else - sendst->config->sliceSize = 130; + { + xmitSlice = makeSlice(sendst, sliceNo++); + } - sendst->config->discovery = net_config::DSC_DOUBLING; - } - else - { - config->sliceSize = config->default_slice_size; -#ifdef BB_FEATURE_UDPCAST_FEC - - if ((config->flags & FLAG_FEC) && - (config->sliceSize > 128 * config->fec_stripes)) - config->sliceSize = 128 * config->fec_stripes; - -#endif + if (xmitSlice->bytes == 0) + atEnd = 1; + } } -#ifdef BB_FEATURE_UDPCAST_FEC - - if ( (sendst->config->flags & FLAG_FEC) && - config->max_slice_size > config->fec_stripes * 128) - config->max_slice_size = config->fec_stripes * 128; - -#endif - - if (config->sliceSize > config->max_slice_size) - config->sliceSize = config->max_slice_size; - - idbassert(config->sliceSize <= MAX_SLICE_SIZE); - - do + if (xmitSlice != NULL && xmitSlice->state == slice::SLICE_NEW) { - /* first, cleanup rexmit Slice if needed */ - - if (rexmitSlice != NULL) - { - if (rexmitSlice->nrReady == - udpc_nrParticipants(sendst->rc.participantsDb)) - { + sendSlice(sendst, xmitSlice, 0); #if DEBUG - flprintf("slice is ready\n"); + flprintf("%d Interrupted at %d/%d\n", xmitSlice->sliceNo, xmitSlice->nextBlock, + getSliceBlocks(xmitSlice, sendst->config)); #endif - ackSlice(rexmitSlice, sendst->config, sendst->fifo, - sendst->stats); - } + continue; + } - if (isSliceAcked(rexmitSlice)) - { - freeSlice(sendst, rexmitSlice); - rexmitSlice = NULL; - } - } + if (atEnd && rexmitSlice == NULL && xmitSlice == NULL) + break; - /* then shift xmit slice to rexmit slot, if possible */ - if (rexmitSlice == NULL && xmitSlice != NULL && - isSliceXmitted(xmitSlice)) - { - rexmitSlice = xmitSlice; - xmitSlice = NULL; - sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, - sendst->socket); - } - - /* handle any messages */ - if (pc_getWaiting(sendst->rc.incoming)) - { -#if DEBUG - flprintf("Before message %d\n", - pc_getWaiting(sendst->rc.incoming)); -#endif - handleNextMessage(sendst, xmitSlice, rexmitSlice); - - /* restart at beginning of loop: we may have acked the rxmit - * slice, makeing it possible to shift the pipe */ - continue; - } - - /* do any needed retransmissions */ - if (rexmitSlice != NULL && rexmitSlice->needRxmit) - { - doRetransmissions(sendst, rexmitSlice); - /* restart at beginning: new messages may have arrived during - * retransmission */ - continue; - } - - /* if all participants answered, send req ack */ - if (rexmitSlice != NULL && - rexmitSlice->nrAnswered == - udpc_nrParticipants(sendst->rc.participantsDb)) - { - rexmitSlice->rxmitId++; - sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, - sendst->socket); - } - - if (xmitSlice == NULL && !atEnd) - { -#if DEBUG - flprintf("SN=%d\n", sendst->config->flags & FLAG_SN); -#endif - - if ((sendst->config->flags & FLAG_SN) || - rexmitSlice == NULL) - { -#ifdef BB_FEATURE_UDPCAST_FEC - - if (sendst->config->flags & FLAG_FEC) - { - int i; - pc_consume(sendst->fec_data_pc, 1); - i = pc_getConsumerPosition(sendst->fec_data_pc); - xmitSlice = &sendst->slices[i]; - pc_consumed(sendst->fec_data_pc, 1); - } - else -#endif - { - xmitSlice = makeSlice(sendst, sliceNo++); - } - - if (xmitSlice->bytes == 0) - atEnd = 1; - } - } - - if (xmitSlice != NULL && xmitSlice->state == slice::SLICE_NEW) - { - sendSlice(sendst, xmitSlice, 0); -#if DEBUG - flprintf("%d Interrupted at %d/%d\n", xmitSlice->sliceNo, - xmitSlice->nextBlock, - getSliceBlocks(xmitSlice, sendst->config)); -#endif - continue; - } - - if (atEnd && rexmitSlice == NULL && xmitSlice == NULL) - break; - - if (sendst->config->flags & FLAG_ASYNC) - break; + if (sendst->config->flags & FLAG_ASYNC) + break; #if DEBUG - flprintf("Waiting for timeout...\n"); + flprintf("Waiting for timeout...\n"); #endif - gettimeofday(&tv, 0); - ts.tv_sec = tv.tv_sec; - ts.tv_nsec = (long int)((tv.tv_usec + 1.1 * waitAverage) * 1000); + gettimeofday(&tv, 0); + ts.tv_sec = tv.tv_sec; + ts.tv_nsec = (long int)((tv.tv_usec + 1.1 * waitAverage) * 1000); #ifdef WINDOWS - /* Windows has a granularity of 1 millisecond in its timer. Take this - * into account here */ + /* Windows has a granularity of 1 millisecond in its timer. Take this + * into account here */ #define GRANULARITY 1000000 - ts.tv_nsec += 3 * GRANULARITY / 2; - ts.tv_nsec -= ts.tv_nsec % GRANULARITY; + ts.tv_nsec += 3 * GRANULARITY / 2; + ts.tv_nsec -= ts.tv_nsec % GRANULARITY; #endif #define BILLION 1000000000 - while (ts.tv_nsec >= BILLION) - { - ts.tv_nsec -= BILLION; - ts.tv_sec++; - } - - if (rexmitSlice->rxmitId > 10) - /* after tenth retransmission, wait minimum one second */ - ts.tv_sec++; - - if (pc_consumeAnyWithTimeout(sendst->rc.incoming, &ts) != 0) - { -#if DEBUG - flprintf("Have data\n"); -#endif - { - struct timeval tv2; - unsigned long timeout; - gettimeofday(&tv2, 0); - timeout = - (tv2.tv_sec - tv.tv_sec) * 1000000 + - tv2.tv_usec - tv.tv_usec; - - if (nrWaited) - timeout += waitAverage; - - waitAverage += 9; /* compensate against rounding errors */ - waitAverage = (long unsigned int)((0.9 * waitAverage + 0.1 * timeout)); - } - nrWaited = 0; - continue; - } - - if (rexmitSlice == NULL) - { - //TODO: FIXME - //udpc_flprintf("Weird. Timeout and no rxmit slice"); - break; - } - - if (nrWaited > 5) - { -#ifndef WINDOWS - /* on Cygwin, we would get too many of those messages... */ - nrWaited = 0; -#endif - } - - nrWaited++; - - if (rexmitSlice->rxmitId > config->retriesUntilDrop) - { - int i; - - for (i = 0; i < MAX_CLIENTS; i++) - { - if (udpc_isParticipantValid(sendst->rc.participantsDb, i) && - !BIT_ISSET(i, rexmitSlice->sl_reqack.readySet)) - { - udpc_removeParticipant(sendst->rc.participantsDb, i); - - if (udpc_nrParticipants(sendst->rc.participantsDb) == 0) - goto exit_main_loop; - } - } - - continue; - } - - rexmitSlice->rxmitId++; - sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, - sendst->socket); + while (ts.tv_nsec >= BILLION) + { + ts.tv_nsec -= BILLION; + ts.tv_sec++; } - while (udpc_nrParticipants(sendst->rc.participantsDb) || - (config->flags & FLAG_ASYNC)); + + if (rexmitSlice->rxmitId > 10) + /* after tenth retransmission, wait minimum one second */ + ts.tv_sec++; + + if (pc_consumeAnyWithTimeout(sendst->rc.incoming, &ts) != 0) + { +#if DEBUG + flprintf("Have data\n"); +#endif + { + struct timeval tv2; + unsigned long timeout; + gettimeofday(&tv2, 0); + timeout = (tv2.tv_sec - tv.tv_sec) * 1000000 + tv2.tv_usec - tv.tv_usec; + + if (nrWaited) + timeout += waitAverage; + + waitAverage += 9; /* compensate against rounding errors */ + waitAverage = (long unsigned int)((0.9 * waitAverage + 0.1 * timeout)); + } + nrWaited = 0; + continue; + } + + if (rexmitSlice == NULL) + { + // TODO: FIXME + // udpc_flprintf("Weird. Timeout and no rxmit slice"); + break; + } + + if (nrWaited > 5) + { +#ifndef WINDOWS + /* on Cygwin, we would get too many of those messages... */ + nrWaited = 0; +#endif + } + + nrWaited++; + + if (rexmitSlice->rxmitId > config->retriesUntilDrop) + { + int i; + + for (i = 0; i < MAX_CLIENTS; i++) + { + if (udpc_isParticipantValid(sendst->rc.participantsDb, i) && + !BIT_ISSET(i, rexmitSlice->sl_reqack.readySet)) + { + udpc_removeParticipant(sendst->rc.participantsDb, i); + + if (udpc_nrParticipants(sendst->rc.participantsDb) == 0) + goto exit_main_loop; + } + } + + continue; + } + + rexmitSlice->rxmitId++; + sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, sendst->socket); + } while (udpc_nrParticipants(sendst->rc.participantsDb) || (config->flags & FLAG_ASYNC)); exit_main_loop: - cancelReturnChannel(&sendst->rc); - pc_produceEnd(sendst->fifo->freeMemQueue); - return 0; + cancelReturnChannel(&sendst->rc); + pc_produceEnd(sendst->fifo->freeMemQueue); + return 0; } -int spawnNetSender(struct fifo* fifo, - int sock, - struct net_config* config, - participantsDb_t db) +int spawnNetSender(struct fifo* fifo, int sock, struct net_config* config, participantsDb_t db) { - int i; + int i; - sender_state_t sendst = MALLOC(struct senderState); - sendst->fifo = fifo; - sendst->socket = sock; - sendst->config = config; - //sendst->stats = stats; + sender_state_t sendst = MALLOC(struct senderState); + sendst->fifo = fifo; + sendst->socket = sock; + sendst->config = config; + // sendst->stats = stats; #ifdef BB_FEATURE_UDPCAST_FEC - if (sendst->config->flags & FLAG_FEC) - sendst->fec_data = xmalloc(NR_SLICES * - config->fec_stripes * - config->fec_redundancy * - config->blockSize); + if (sendst->config->flags & FLAG_FEC) + sendst->fec_data = xmalloc(NR_SLICES * config->fec_stripes * config->fec_redundancy * config->blockSize); #endif - sendst->rc.participantsDb = db; - initReturnChannel(&sendst->rc, sendst->config, sendst->socket); + sendst->rc.participantsDb = db; + initReturnChannel(&sendst->rc, sendst->config, sendst->socket); - sendst->free_slices_pc = pc_makeProduconsum(NR_SLICES, "free slices"); - pc_produce(sendst->free_slices_pc, NR_SLICES); + sendst->free_slices_pc = pc_makeProduconsum(NR_SLICES, "free slices"); + pc_produce(sendst->free_slices_pc, NR_SLICES); - for (i = 0; i < NR_SLICES; i++) - sendst->slices[i].state = slice::SLICE_FREE; + for (i = 0; i < NR_SLICES; i++) + sendst->slices[i].state = slice::SLICE_FREE; #ifdef BB_FEATURE_UDPCAST_FEC - if (sendst->config->flags & FLAG_FEC) - { - /* Free memory queue is initially full */ - fec_init(); - sendst->fec_data_pc = pc_makeProduconsum(NR_SLICES, "fec data"); + if (sendst->config->flags & FLAG_FEC) + { + /* Free memory queue is initially full */ + fec_init(); + sendst->fec_data_pc = pc_makeProduconsum(NR_SLICES, "fec data"); - pthread_create(&sendst->fec_thread, NULL, fecMain, sendst); - } + pthread_create(&sendst->fec_thread, NULL, fecMain, sendst); + } #endif - pthread_create(&fifo->thread, NULL, netSenderMain, sendst); - return 0; + pthread_create(&fifo->thread, NULL, netSenderMain, sendst); + return 0; } struct sockaddr_in* udpc_getParticipantIp(participantsDb_t db, int i) { - return &db->clientTable[i].addr; + return &db->clientTable[i].addr; } int udpc_getParticipantCapabilities(participantsDb_t db, int i) { - return db->clientTable[i].capabilities; + return db->clientTable[i].capabilities; } unsigned int udpc_getParticipantRcvBuf(participantsDb_t db, int i) { - return db->clientTable[i].rcvbuf; + return db->clientTable[i].rcvbuf; } int pc_consumeContiguousMinAmount(produconsum_t pc, int amount) { - int n = _consumeAny(pc, amount, 0); - int l = pc->size - (pc->consumed % pc->size); + int n = _consumeAny(pc, amount, 0); + int l = pc->size - (pc->consumed % pc->size); - if (n > l) - n = l; - - return n; + if (n > l) + n = l; + return n; } #define BLOCKSIZE 4096 void localReader(struct fifo* fifo, const uint8_t* buf, uint32_t len) { -//cerr << "starting to send " << len << " bytes" << endl; - uint32_t offset = 0; + // cerr << "starting to send " << len << " bytes" << endl; + uint32_t offset = 0; - while (1) + while (1) + { + int pos = pc_getConsumerPosition(fifo->freeMemQueue); + int bytes = pc_consumeContiguousMinAmount(fifo->freeMemQueue, BLOCKSIZE); + + if (bytes > (pos + bytes) % BLOCKSIZE) + bytes -= (pos + bytes) % BLOCKSIZE; + + if (offset + bytes > len) + bytes = len - offset; + + // cerr << "sending " << bytes << " bytes from bs..." << endl; + // bytes = read(in, fifo->dataBuffer + pos, bytes); + memcpy(fifo->dataBuffer + pos, buf + offset, bytes); + offset += bytes; + + if (bytes == 0) { - int pos = pc_getConsumerPosition(fifo->freeMemQueue); - int bytes = - pc_consumeContiguousMinAmount(fifo->freeMemQueue, BLOCKSIZE); - - if (bytes > (pos + bytes) % BLOCKSIZE) - bytes -= (pos + bytes) % BLOCKSIZE; - - if (offset + bytes > len) bytes = len - offset; - -//cerr << "sending " << bytes << " bytes from bs..." << endl; - //bytes = read(in, fifo->dataBuffer + pos, bytes); - memcpy(fifo->dataBuffer + pos, buf + offset, bytes); - offset += bytes; - - if (bytes == 0) - { - /* the end */ - pc_produceEnd(fifo->data); - break; - } - else - { - pc_consumed(fifo->freeMemQueue, bytes); - pc_produce(fifo->data, bytes); - } + /* the end */ + pc_produceEnd(fifo->data); + break; } + else + { + pc_consumed(fifo->freeMemQueue, bytes); + pc_produce(fifo->data, bytes); + } + } -//cerr << "done sending" << endl; + // cerr << "done sending" << endl; } unsigned int udpc_getRcvBuf(int sock) { - unsigned int bufsize; - socklen_t len = sizeof(int); + unsigned int bufsize; + socklen_t len = sizeof(int); - if (getsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char*)&bufsize, &len) < 0) - return -1; + if (getsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char*)&bufsize, &len) < 0) + return -1; - return bufsize; + return bufsize; } -int sendConnectReq(struct client_config* client_config, - struct net_config* net_config, - int haveServerAddress) +int sendConnectReq(struct client_config* client_config, struct net_config* net_config, int haveServerAddress) { -//cerr << "sending a connect request" << endl; - struct connectReq connectReq; + // cerr << "sending a connect request" << endl; + struct connectReq connectReq; - if (net_config->flags & FLAG_PASSIVE) - return 0; + if (net_config->flags & FLAG_PASSIVE) + return 0; - connectReq.opCode = htons(CMD_CONNECT_REQ); - connectReq.reserved = 0; - connectReq.capabilities = htonl(RECEIVER_CAPABILITIES); - connectReq.rcvbuf = htonl(udpc_getRcvBuf(client_config->S_UCAST)); + connectReq.opCode = htons(CMD_CONNECT_REQ); + connectReq.reserved = 0; + connectReq.capabilities = htonl(RECEIVER_CAPABILITIES); + connectReq.rcvbuf = htonl(udpc_getRcvBuf(client_config->S_UCAST)); - if (haveServerAddress) - return SSEND(connectReq); - else - return BCAST_CONTROL(client_config->S_UCAST, connectReq); + if (haveServerAddress) + return SSEND(connectReq); + else + return BCAST_CONTROL(client_config->S_UCAST, connectReq); } void udpc_setRcvBuf(int sock, unsigned int bufsize) { - if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, - (char*) &bufsize, sizeof(bufsize)) < 0) - { - //TODO: FIXME - //perror("Set receiver buffer"); - } + if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char*)&bufsize, sizeof(bufsize)) < 0) + { + // TODO: FIXME + // perror("Set receiver buffer"); + } } void udpc_copyFromMessage(struct sockaddr_in* dst, unsigned char* src) { - memcpy((char*) &dst->sin_addr, src, sizeof(struct in_addr)); + memcpy((char*)&dst->sin_addr, src, sizeof(struct in_addr)); } unsigned short udpc_getPort(struct sockaddr_in* addr) { - return ntohs(((struct sockaddr_in*) addr)->sin_port); + return ntohs(((struct sockaddr_in*)addr)->sin_port); } int udpc_selectSock(int* socks, int nr, int startTimeout) { - fd_set read_set; - int r; - int maxFd; - struct timeval tv, *tvp; + fd_set read_set; + int r; + int maxFd; + struct timeval tv, *tvp; - if (startTimeout) - { - tv.tv_sec = startTimeout; - tv.tv_usec = 0; - tvp = &tv; - } - else - { - tvp = NULL; - } + if (startTimeout) + { + tv.tv_sec = startTimeout; + tv.tv_usec = 0; + tvp = &tv; + } + else + { + tvp = NULL; + } - maxFd = prepareForSelect(socks, nr, &read_set); - r = select(maxFd + 1, &read_set, NULL, NULL, tvp); + maxFd = prepareForSelect(socks, nr, &read_set); + r = select(maxFd + 1, &read_set, NULL, NULL, tvp); - if (r < 0) - return r; + if (r < 0) + return r; - return udpc_getSelectedSock(socks, nr, &read_set); + return udpc_getSelectedSock(socks, nr, &read_set); } void udpc_zeroSockArray(int* socks, int nr) { - int i; + int i; - for (i = 0; i < nr; i++) - socks[i] = -1; + for (i = 0; i < nr; i++) + socks[i] = -1; } unsigned char* getBlockSpace(struct clientState* clst) { - int pos; + int pos; - if (clst->localPos) - { - clst->localPos--; - return clst->localBlockAddresses[clst->localPos]; - } + if (clst->localPos) + { + clst->localPos--; + return clst->localBlockAddresses[clst->localPos]; + } - pc_consume(clst->freeBlocks_pc, 1); - pos = pc_getConsumerPosition(clst->freeBlocks_pc); - pc_consumed(clst->freeBlocks_pc, 1); - return clst->blockAddresses[pos]; + pc_consume(clst->freeBlocks_pc, 1); + pos = pc_getConsumerPosition(clst->freeBlocks_pc); + pc_consumed(clst->freeBlocks_pc, 1); + return clst->blockAddresses[pos]; } void setNextBlock(struct clientState* clst) { - clst->nextBlock = getBlockSpace(clst); + clst->nextBlock = getBlockSpace(clst); } int setupMessages(struct clientState* clst) { - /* the messages received from the server */ - clst->data_iov[0].iov_base = (void*)&clst->Msg; - clst->data_iov[0].iov_len = sizeof(clst->Msg); + /* the messages received from the server */ + clst->data_iov[0].iov_base = (void*)&clst->Msg; + clst->data_iov[0].iov_len = sizeof(clst->Msg); - /* namelen set just before reception */ - clst->data_hdr.msg_iov = clst->data_iov; - /* iovlen set just before reception */ + /* namelen set just before reception */ + clst->data_hdr.msg_iov = clst->data_iov; + /* iovlen set just before reception */ - initMsgHdr(&clst->data_hdr); - return 0; + initMsgHdr(&clst->data_hdr); + return 0; } -struct slice* initSlice(struct clientState* clst, - struct slice* slice, - int sliceNo) +struct slice* initSlice(struct clientState* clst, struct slice* slice, int sliceNo) { - idbassert(slice->state == slice::SLICE_FREE || slice->state == slice::SLICE_RECEIVING); + idbassert(slice->state == slice::SLICE_FREE || slice->state == slice::SLICE_RECEIVING); - slice->magic = SLICEMAGIC; - slice->state = slice::SLICE_RECEIVING; - slice->blocksTransferred = 0; - slice->dataBlocksTransferred = 0; - BZERO(slice->retransmit); - slice->freePos = 0; - slice->bytes = 0; + slice->magic = SLICEMAGIC; + slice->state = slice::SLICE_RECEIVING; + slice->blocksTransferred = 0; + slice->dataBlocksTransferred = 0; + BZERO(slice->retransmit); + slice->freePos = 0; + slice->bytes = 0; - if (clst->currentSlice != NULL && !clst->currentSlice->bytesKnown) - { - //udpc_fatal(1, "Previous slice size not known\n"); - } + if (clst->currentSlice != NULL && !clst->currentSlice->bytesKnown) + { + // udpc_fatal(1, "Previous slice size not known\n"); + } - if (clst->currentSliceNo != sliceNo - 1) - { - //udpc_fatal(1, "Slice no mismatch %d <-> %d\n", sliceNo, clst->currentSliceNo); - } + if (clst->currentSliceNo != sliceNo - 1) + { + // udpc_fatal(1, "Slice no mismatch %d <-> %d\n", sliceNo, clst->currentSliceNo); + } - slice->bytesKnown = 0; - slice->sliceNo = sliceNo; + slice->bytesKnown = 0; + slice->sliceNo = sliceNo; - BZERO(slice->missing_data_blocks); + BZERO(slice->missing_data_blocks); #ifdef BB_FEATURE_UDPCAST_FEC - BZERO(slice->fec_stripes); - BZERO(slice->fec_blocks); - BZERO(slice->fec_descs); + BZERO(slice->fec_stripes); + BZERO(slice->fec_blocks); + BZERO(slice->fec_descs); #endif - clst->currentSlice = slice; - clst->currentSliceNo = sliceNo; - return slice; + clst->currentSlice = slice; + clst->currentSliceNo = sliceNo; + return slice; } struct slice* newSlice(struct clientState* clst, int sliceNo) { - struct slice* slice = NULL; - int i; + struct slice* slice = NULL; + int i; - pc_consume(clst->free_slices_pc, 1); - i = pc_getConsumerPosition(clst->free_slices_pc); - pc_consumed(clst->free_slices_pc, 1); - slice = &clst->slices[i]; - idbassert(slice->state == slice::SLICE_FREE); + pc_consume(clst->free_slices_pc, 1); + i = pc_getConsumerPosition(clst->free_slices_pc); + pc_consumed(clst->free_slices_pc, 1); + slice = &clst->slices[i]; + idbassert(slice->state == slice::SLICE_FREE); - /* wait for free data memory */ - slice->base = pc_getConsumerPosition(clst->fifo->freeMemQueue); - pc_consume(clst->fifo->freeMemQueue, - clst->net_config->blockSize * MAX_SLICE_SIZE); - initSlice(clst, slice, sliceNo); - return slice; + /* wait for free data memory */ + slice->base = pc_getConsumerPosition(clst->fifo->freeMemQueue); + pc_consume(clst->fifo->freeMemQueue, clst->net_config->blockSize * MAX_SLICE_SIZE); + initSlice(clst, slice, sliceNo); + return slice; } -#define ADR(x, bs) (fifo->dataBuffer + \ - (slice->base+(x)*bs) % fifo->dataBufSize) +#define ADR(x, bs) (fifo->dataBuffer + (slice->base + (x)*bs) % fifo->dataBufSize) void closeAllExcept(struct clientState* clst, int fd) { - int i; - int* socks = clst->client_config->socks; + int i; + int* socks = clst->client_config->socks; - if (clst->selectedFd >= 0) - return; + if (clst->selectedFd >= 0) + return; - clst->selectedFd = fd; + clst->selectedFd = fd; - for (i = 1; i < NR_CLIENT_SOCKS; i++) - if (socks[i] != -1 && socks[i] != fd) - udpc_closeSock(socks, NR_CLIENT_SOCKS, i); + for (i = 1; i < NR_CLIENT_SOCKS; i++) + if (socks[i] != -1 && socks[i] != fd) + udpc_closeSock(socks, NR_CLIENT_SOCKS, i); } struct slice* rcvFindSlice(struct clientState* clst, int sliceNo) { - if (! clst->currentSlice) + if (!clst->currentSlice) + { + /* Streaming mode? */ + clst->currentSliceNo = sliceNo - 1; + return newSlice(clst, sliceNo); + } + + if (sliceNo <= clst->currentSliceNo) + { + struct slice* slice = clst->currentSlice; + int pos = slice - clst->slices; + idbassert(slice == NULL || slice->magic == SLICEMAGIC); + + while (slice->sliceNo != sliceNo) { - /* Streaming mode? */ - clst->currentSliceNo = sliceNo - 1; - return newSlice(clst, sliceNo); + if (slice->state == slice::SLICE_FREE) + return NULL; + + idbassert(slice->magic == SLICEMAGIC); + pos--; + + if (pos < 0) + pos += NR_SLICES; + + slice = &clst->slices[pos]; } - if (sliceNo <= clst->currentSliceNo) - { - struct slice* slice = clst->currentSlice; - int pos = slice - clst->slices; - idbassert(slice == NULL || slice->magic == SLICEMAGIC); + return slice; + } - while (slice->sliceNo != sliceNo) - { - if (slice->state == slice::SLICE_FREE) - return NULL; + if (sliceNo != clst->currentSliceNo + 1) + { + // TODO: FIXME + // udpc_flprintf("Slice %d skipped\n", sliceNo-1); + // exit(1); + } - idbassert(slice->magic == SLICEMAGIC); - pos--; + if ((clst->net_config->flags & FLAG_STREAMING) && sliceNo != clst->currentSliceNo) + { + return initSlice(clst, clst->currentSlice, sliceNo); + } - if (pos < 0) - pos += NR_SLICES; - - slice = &clst->slices[pos]; - } - - return slice; - } - - if (sliceNo != clst->currentSliceNo + 1) - { - //TODO: FIXME - //udpc_flprintf("Slice %d skipped\n", sliceNo-1); - //exit(1); - } - - if ((clst->net_config->flags & FLAG_STREAMING) && - sliceNo != clst->currentSliceNo) - { - return initSlice(clst, clst->currentSlice, sliceNo); - } - - if (sliceNo > clst->receivedSliceNo + 2) - { - - //TODO: FIXME + if (sliceNo > clst->receivedSliceNo + 2) + { + // TODO: FIXME #if 0 slice_t slice = rcvFindSlice(clst, clst->receivedSliceNo + 1); udpc_flprintf("Dropped by server now=%d last=%d\n", sliceNo, clst->receivedSliceNo); @@ -2955,1086 +2810,1011 @@ struct slice* rcvFindSlice(struct clientState* clst, int sliceNo) exit(1); #endif - } + } - return newSlice(clst, sliceNo); + return newSlice(clst, sliceNo); } -void setSliceBytes(struct slice* slice, - struct clientState* clst, - int bytes) +void setSliceBytes(struct slice* slice, struct clientState* clst, int bytes) { - idbassert(slice->magic == SLICEMAGIC); + idbassert(slice->magic == SLICEMAGIC); - if (slice->bytesKnown) + if (slice->bytesKnown) + { + if (slice->bytes != bytes) { - if (slice->bytes != bytes) - { - //TODO: FIXME - //udpc_fatal(1, "Byte number mismatch %d <-> %d\n", bytes, slice->bytes); - } + // TODO: FIXME + // udpc_fatal(1, "Byte number mismatch %d <-> %d\n", bytes, slice->bytes); } - else - { - slice->bytesKnown = 1; - slice->bytes = bytes; + } + else + { + slice->bytesKnown = 1; + slice->bytes = bytes; - if (bytes == 0) - clst->netEndReached = 1; - } + if (bytes == 0) + clst->netEndReached = 1; + } } void advanceReceivedPointer(struct clientState* clst) { - int pos = clst->receivedPtr; + int pos = clst->receivedPtr; - while (1) - { - slice_t slice = &clst->slices[pos]; + while (1) + { + slice_t slice = &clst->slices[pos]; - if ( + if ( #ifdef BB_FEATURE_UDPCAST_FEC - slice->state != SLICE_FEC && - slice->state != SLICE_FEC_DONE && + slice->state != SLICE_FEC && slice->state != SLICE_FEC_DONE && #endif - slice->state != slice::SLICE_DONE) - break; + slice->state != slice::SLICE_DONE) + break; - pos++; - clst->receivedSliceNo = slice->sliceNo; + pos++; + clst->receivedSliceNo = slice->sliceNo; - if (pos >= NR_SLICES) - pos -= NR_SLICES; - } + if (pos >= NR_SLICES) + pos -= NR_SLICES; + } - clst->receivedPtr = pos; + clst->receivedPtr = pos; } void receiverStatsAddBytes(receiver_stats_t rs, long bytes) { - if (rs != NULL) - rs->totalBytes += bytes; + if (rs != NULL) + rs->totalBytes += bytes; } void cleanupSlices(struct clientState* clst, unsigned int doneState) { - while (1) + while (1) + { + int pos = pc_getProducerPosition(clst->free_slices_pc); + int bytes; + slice_t slice = &clst->slices[pos]; + + if (slice->state != (signed)doneState) + break; + + receiverStatsAddBytes(clst->stats, slice->bytes); + bytes = slice->bytes; + + /* signal data received */ + if (bytes == 0) { - int pos = pc_getProducerPosition(clst->free_slices_pc); - int bytes; - slice_t slice = &clst->slices[pos]; - - if (slice->state != (signed)doneState) - break; - - receiverStatsAddBytes(clst->stats, slice->bytes); - bytes = slice->bytes; - - /* signal data received */ - if (bytes == 0) - { - pc_produceEnd(clst->fifo->data); - } - else - pc_produce(clst->fifo->data, slice->bytes); - - /* free up slice structure */ - clst->slices[pos].state = slice::SLICE_FREE; - pc_produce(clst->free_slices_pc, 1); - - /* if at end, exit this thread */ - if (!bytes) - { - clst->endReached = 2; - } + pc_produceEnd(clst->fifo->data); } + else + pc_produce(clst->fifo->data, slice->bytes); + + /* free up slice structure */ + clst->slices[pos].state = slice::SLICE_FREE; + pc_produce(clst->free_slices_pc, 1); + + /* if at end, exit this thread */ + if (!bytes) + { + clst->endReached = 2; + } + } } -void checkSliceComplete(struct clientState* clst, - struct slice* slice) +void checkSliceComplete(struct clientState* clst, struct slice* slice) { - int blocksInSlice; + int blocksInSlice; - idbassert(slice->magic == SLICEMAGIC); + idbassert(slice->magic == SLICEMAGIC); - if (slice->state != slice::SLICE_RECEIVING) - /* bad starting state */ - return; + if (slice->state != slice::SLICE_RECEIVING) + /* bad starting state */ + return; - /* is this slice ready ? */ - idbassert(clst->net_config->blockSize != 0); - blocksInSlice = (slice->bytes + clst->net_config->blockSize - 1) / - clst->net_config->blockSize; + /* is this slice ready ? */ + idbassert(clst->net_config->blockSize != 0); + blocksInSlice = (slice->bytes + clst->net_config->blockSize - 1) / clst->net_config->blockSize; - if (blocksInSlice == slice->blocksTransferred) + if (blocksInSlice == slice->blocksTransferred) + { + pc_consumed(clst->fifo->freeMemQueue, slice->bytes); + clst->net_config->flags &= ~FLAG_STREAMING; + + if (blocksInSlice == slice->dataBlocksTransferred) + slice->state = slice::SLICE_DONE; + else { - pc_consumed(clst->fifo->freeMemQueue, slice->bytes); - clst->net_config->flags &= ~FLAG_STREAMING; - - if (blocksInSlice == slice->dataBlocksTransferred) - slice->state = slice::SLICE_DONE; - else - { #ifdef BB_FEATURE_UDPCAST_FEC - idbassert(clst->use_fec == 1); - slice->state = SLICE_FEC; + idbassert(clst->use_fec == 1); + slice->state = SLICE_FEC; #else - idbassert(0); + idbassert(0); #endif - } + } - advanceReceivedPointer(clst); + advanceReceivedPointer(clst); #ifdef BB_FEATURE_UDPCAST_FEC - if (clst->use_fec) - { - int n = pc_getProducerPosition(clst->fec_data_pc); - idbassert(slice->state == SLICE_DONE || slice->state == SLICE_FEC); - clst->fec_slices[n] = slice; - pc_produce(clst->fec_data_pc, 1); - } - else -#endif - cleanupSlices(clst, slice::SLICE_DONE); + if (clst->use_fec) + { + int n = pc_getProducerPosition(clst->fec_data_pc); + idbassert(slice->state == SLICE_DONE || slice->state == SLICE_FEC); + clst->fec_slices[n] = slice; + pc_produce(clst->fec_data_pc, 1); } + else +#endif + cleanupSlices(clst, slice::SLICE_DONE); + } } -int processDataBlock(struct clientState* clst, - int sliceNo, - int blockNo, - int bytes) +int processDataBlock(struct clientState* clst, int sliceNo, int blockNo, int bytes) { -//cerr << "processDataBlock(): " << sliceNo << ", " << blockNo << ", " << bytes << endl; - struct fifo* fifo = clst->fifo; - struct slice* slice = rcvFindSlice(clst, sliceNo); - unsigned char* shouldAddress, *isAddress; + // cerr << "processDataBlock(): " << sliceNo << ", " << blockNo << ", " << bytes << endl; + struct fifo* fifo = clst->fifo; + struct slice* slice = rcvFindSlice(clst, sliceNo); + unsigned char *shouldAddress, *isAddress; - idbassert(slice == NULL || slice->magic == SLICEMAGIC); + idbassert(slice == NULL || slice->magic == SLICEMAGIC); - if (slice == NULL || - slice->state == slice::SLICE_FREE || - slice->state == slice::SLICE_DONE + if (slice == NULL || slice->state == slice::SLICE_FREE || slice->state == slice::SLICE_DONE #ifdef BB_FEATURE_UDPCAST_FEC - || - slice->state == SLICE_FEC || - slice->state == SLICE_FEC_DONE + || slice->state == SLICE_FEC || slice->state == SLICE_FEC_DONE #endif - ) - { - /* an old slice. Ignore */ -//cerr << "ignore" << endl; - return 0; - } + ) + { + /* an old slice. Ignore */ + // cerr << "ignore" << endl; + return 0; + } - if (sliceNo > clst->currentSliceNo + 2) - { - //TODO: FIXME - //udpc_fatal(1, "We have been dropped by sender\n"); - } + if (sliceNo > clst->currentSliceNo + 2) + { + // TODO: FIXME + // udpc_fatal(1, "We have been dropped by sender\n"); + } - if (BIT_ISSET(blockNo, slice->retransmit.map)) - { - /* we already have this packet, ignore */ + if (BIT_ISSET(blockNo, slice->retransmit.map)) + { + /* we already have this packet, ignore */ #if 0 flprintf("Packet %d:%d not for us\n", sliceNo, blockNo); #endif -//cerr << "dup" << endl; - return 0; - } + // cerr << "dup" << endl; + return 0; + } - if (slice->base % clst->net_config->blockSize) - { - //TODO: FIXME - //udpc_fatal(1, "Bad base %d, not multiple of block size %d\n", slice->base, clst->net_config->blockSize); -//cerr << "bad base" << endl; - } + if (slice->base % clst->net_config->blockSize) + { + // TODO: FIXME + // udpc_fatal(1, "Bad base %d, not multiple of block size %d\n", slice->base, + // clst->net_config->blockSize); + // cerr << "bad base" << endl; + } -//cerr << "good slice" << endl; + // cerr << "good slice" << endl; - shouldAddress = ADR(blockNo, clst->net_config->blockSize); - isAddress = (unsigned char*)clst->data_hdr.msg_iov[1].iov_base; + shouldAddress = ADR(blockNo, clst->net_config->blockSize); + isAddress = (unsigned char*)clst->data_hdr.msg_iov[1].iov_base; - if (shouldAddress != isAddress) - { - /* copy message to the correct place */ - memcpy(shouldAddress, isAddress, clst->net_config->blockSize); - } + if (shouldAddress != isAddress) + { + /* copy message to the correct place */ + memcpy(shouldAddress, isAddress, clst->net_config->blockSize); + } - if (clst->client_config->sender_is_newgen && bytes != 0) - setSliceBytes(slice, clst, bytes); + if (clst->client_config->sender_is_newgen && bytes != 0) + setSliceBytes(slice, clst, bytes); - if (clst->client_config->sender_is_newgen && bytes == 0) - clst->netEndReached = 0; + if (clst->client_config->sender_is_newgen && bytes == 0) + clst->netEndReached = 0; - SET_BIT(blockNo, slice->retransmit.map); + SET_BIT(blockNo, slice->retransmit.map); #ifdef BB_FEATURE_UDPCAST_FEC - if (slice->fec_stripes) - { - int stripe = blockNo % slice->fec_stripes; - slice->missing_data_blocks[stripe]--; - idbassert(slice->missing_data_blocks[stripe] >= 0); + if (slice->fec_stripes) + { + int stripe = blockNo % slice->fec_stripes; + slice->missing_data_blocks[stripe]--; + idbassert(slice->missing_data_blocks[stripe] >= 0); - if (slice->missing_data_blocks[stripe] < - slice->fec_blocks[stripe]) - { - int blockIdx; - /* FIXME: FEC block should be enqueued in local queue here...*/ - slice->fec_blocks[stripe]--; - blockIdx = stripe + slice->fec_blocks[stripe] * slice->fec_stripes; - idbassert(slice->fec_descs[blockIdx].adr != 0); - clst->localBlockAddresses[clst->localPos++] = - slice->fec_descs[blockIdx].adr; - slice->fec_descs[blockIdx].adr = 0; - slice->blocksTransferred--; - } + if (slice->missing_data_blocks[stripe] < slice->fec_blocks[stripe]) + { + int blockIdx; + /* FIXME: FEC block should be enqueued in local queue here...*/ + slice->fec_blocks[stripe]--; + blockIdx = stripe + slice->fec_blocks[stripe] * slice->fec_stripes; + idbassert(slice->fec_descs[blockIdx].adr != 0); + clst->localBlockAddresses[clst->localPos++] = slice->fec_descs[blockIdx].adr; + slice->fec_descs[blockIdx].adr = 0; + slice->blocksTransferred--; } + } #endif - slice->dataBlocksTransferred++; - slice->blocksTransferred++; + slice->dataBlocksTransferred++; + slice->blocksTransferred++; - while (slice->freePos < MAX_SLICE_SIZE && - BIT_ISSET(slice->freePos, slice->retransmit.map)) - slice->freePos++; + while (slice->freePos < MAX_SLICE_SIZE && BIT_ISSET(slice->freePos, slice->retransmit.map)) + slice->freePos++; - checkSliceComplete(clst, slice); - return 0; + checkSliceComplete(clst, slice); + return 0; } int sendOk(struct client_config* client_config, unsigned int sliceNo) { - struct ok ok; - ok.opCode = htons(CMD_OK); - ok.reserved = 0; - ok.sliceNo = htonl(sliceNo); - return SSEND(ok); + struct ok ok; + ok.opCode = htons(CMD_OK); + ok.reserved = 0; + ok.sliceNo = htonl(sliceNo); + return SSEND(ok); } -int sendRetransmit(struct clientState* clst, - struct slice* slice, - int rxmit) +int sendRetransmit(struct clientState* clst, struct slice* slice, int rxmit) { - struct client_config* client_config = clst->client_config; + struct client_config* client_config = clst->client_config; - idbassert(slice->magic == SLICEMAGIC); - slice->retransmit.opCode = htons(CMD_RETRANSMIT); - slice->retransmit.reserved = 0; - slice->retransmit.sliceNo = htonl(slice->sliceNo); - slice->retransmit.rxmit = htonl(rxmit); - return SSEND(slice->retransmit); + idbassert(slice->magic == SLICEMAGIC); + slice->retransmit.opCode = htons(CMD_RETRANSMIT); + slice->retransmit.reserved = 0; + slice->retransmit.sliceNo = htonl(slice->sliceNo); + slice->retransmit.rxmit = htonl(rxmit); + return SSEND(slice->retransmit); } -int processReqAck(struct clientState* clst, - int sliceNo, int bytes, int rxmit) +int processReqAck(struct clientState* clst, int sliceNo, int bytes, int rxmit) { - struct slice* slice = rcvFindSlice(clst, sliceNo); - int blocksInSlice; - char* readySet = (char*) clst->data_hdr.msg_iov[1].iov_base; + struct slice* slice = rcvFindSlice(clst, sliceNo); + int blocksInSlice; + char* readySet = (char*)clst->data_hdr.msg_iov[1].iov_base; - idbassert(slice == NULL || slice->magic == SLICEMAGIC); + idbassert(slice == NULL || slice->magic == SLICEMAGIC); - { - struct timeval tv; - gettimeofday(&tv, 0); - /* usleep(1); DEBUG: FIXME */ - } + { + struct timeval tv; + gettimeofday(&tv, 0); + /* usleep(1); DEBUG: FIXME */ + } - if (BIT_ISSET(clst->client_config->clientNumber, readySet)) - { - /* not for us */ - return 0; - } + if (BIT_ISSET(clst->client_config->clientNumber, readySet)) + { + /* not for us */ + return 0; + } - if (slice == NULL) - { - /* an old slice => send ok */ - return sendOk(clst->client_config, sliceNo); - } + if (slice == NULL) + { + /* an old slice => send ok */ + return sendOk(clst->client_config, sliceNo); + } - setSliceBytes(slice, clst, bytes); - idbassert(clst->net_config->blockSize != 0); - blocksInSlice = (slice->bytes + clst->net_config->blockSize - 1) / - clst->net_config->blockSize; + setSliceBytes(slice, clst, bytes); + idbassert(clst->net_config->blockSize != 0); + blocksInSlice = (slice->bytes + clst->net_config->blockSize - 1) / clst->net_config->blockSize; - if (blocksInSlice == slice->blocksTransferred) - { - /* send ok */ - sendOk(clst->client_config, slice->sliceNo); - } - else - { - sendRetransmit(clst, slice, rxmit); - } + if (blocksInSlice == slice->blocksTransferred) + { + /* send ok */ + sendOk(clst->client_config, slice->sliceNo); + } + else + { + sendRetransmit(clst, slice, rxmit); + } - checkSliceComplete(clst, slice); /* needed for the final 0 sized slice */ - advanceReceivedPointer(clst); + checkSliceComplete(clst, slice); /* needed for the final 0 sized slice */ + advanceReceivedPointer(clst); #ifdef BB_FEATURE_UDPCAST_FEC - if (!clst->use_fec) - cleanupSlices(clst, SLICE_DONE); + if (!clst->use_fec) + cleanupSlices(clst, SLICE_DONE); #endif - return 0; + return 0; } int udpc_isAddressEqual(struct sockaddr_in* a, struct sockaddr_in* b) { - return !memcmp((char*) a, (char*)b, 8); + return !memcmp((char*)a, (char*)b, 8); } int dispatchMessage(struct clientState* clst) { - int ret; - struct sockaddr_in lserver; - struct fifo* fifo = clst->fifo; - int fd = -1; - struct client_config* client_config = clst->client_config; + int ret; + struct sockaddr_in lserver; + struct fifo* fifo = clst->fifo; + int fd = -1; + struct client_config* client_config = clst->client_config; - /* set up message header */ - if (clst->currentSlice != NULL && - clst->currentSlice->freePos < MAX_SLICE_SIZE) + /* set up message header */ + if (clst->currentSlice != NULL && clst->currentSlice->freePos < MAX_SLICE_SIZE) + { + struct slice* slice = clst->currentSlice; + idbassert(slice == NULL || slice->magic == SLICEMAGIC); + clst->data_iov[1].iov_base = ADR(slice->freePos, clst->net_config->blockSize); + } + else + { + clst->data_iov[1].iov_base = clst->nextBlock; + } + + clst->data_iov[1].iov_len = clst->net_config->blockSize; + clst->data_hdr.msg_iovlen = 2; + + clst->data_hdr.msg_name = &lserver; + clst->data_hdr.msg_namelen = sizeof(struct sockaddr_in); + + while (clst->endReached || clst->netEndReached) + { + int oldEndReached = clst->endReached; + int nr_desc; + struct timeval tv; + fd_set read_set; + + int maxFd = prepareForSelect(client_config->socks, NR_CLIENT_SOCKS, &read_set); + + tv.tv_sec = clst->net_config->exitWait / 1000; + tv.tv_usec = (clst->net_config->exitWait % 1000) * 1000; + nr_desc = select(maxFd, &read_set, 0, 0, &tv); + + if (nr_desc < 0) { - struct slice* slice = clst->currentSlice; - idbassert(slice == NULL || slice->magic == SLICEMAGIC); - clst->data_iov[1].iov_base = - ADR(slice->freePos, clst->net_config->blockSize); + break; + } + + fd = udpc_getSelectedSock(client_config->socks, NR_CLIENT_SOCKS, &read_set); + + if (fd >= 0) + break; + + /* Timeout expired */ + if (oldEndReached >= 2) + { + clst->endReached = 3; + return 0; + } + } + + if (fd < 0) + fd = clst->selectedFd; + + if (fd < 0) + { + struct timeval tv, *tvp; + fd_set read_set; + int maxFd = prepareForSelect(client_config->socks, NR_CLIENT_SOCKS, &read_set); + clst->promptPrinted = 1; + + if (clst->net_config->startTimeout == 0) + { + tvp = NULL; } else { - clst->data_iov[1].iov_base = clst->nextBlock; + tv.tv_sec = clst->net_config->startTimeout; + tv.tv_usec = 0; + tvp = &tv; } - clst->data_iov[1].iov_len = clst->net_config->blockSize; - clst->data_hdr.msg_iovlen = 2; - - clst->data_hdr.msg_name = &lserver; - clst->data_hdr.msg_namelen = sizeof(struct sockaddr_in); - - while (clst->endReached || clst->netEndReached) - { - int oldEndReached = clst->endReached; - int nr_desc; - struct timeval tv; - fd_set read_set; - - int maxFd = prepareForSelect(client_config->socks, - NR_CLIENT_SOCKS, &read_set); - - tv.tv_sec = clst->net_config->exitWait / 1000; - tv.tv_usec = (clst->net_config->exitWait % 1000) * 1000; - nr_desc = select(maxFd, &read_set, 0, 0, &tv); - - if (nr_desc < 0) - { - break; - } - - fd = udpc_getSelectedSock(client_config->socks, NR_CLIENT_SOCKS, &read_set); - - if (fd >= 0) - break; - - /* Timeout expired */ - if (oldEndReached >= 2) - { - clst->endReached = 3; - return 0; - } - } - - if (fd < 0) - fd = clst->selectedFd; - - if (fd < 0) - { - struct timeval tv, *tvp; - fd_set read_set; - int maxFd = prepareForSelect(client_config->socks, - NR_CLIENT_SOCKS, &read_set); - clst->promptPrinted = 1; - - if (clst->net_config->startTimeout == 0) - { - tvp = NULL; - } - else - { - tv.tv_sec = clst->net_config->startTimeout; - tv.tv_usec = 0; - tvp = &tv; - } - -//cerr << "waiting for data..." << endl; - ret = selectWithoutConsole(maxFd + 1, &read_set, tvp); - - if (ret < 0) - { - perror("Select"); - return 0; - } - - if (ret == 0) - { - clst->endReached = 3; - clst->netEndReached = 3; - pc_produceEnd(clst->fifo->data); - return 1; - } - - fd = udpc_getSelectedSock(clst->client_config->socks, - NR_CLIENT_SOCKS, &read_set); - } - -#ifdef LOSSTEST - loseRecvPacket(fd); - ret = RecvMsg(fd, &clst->data_hdr, 0); -#else - ret = recvmsg(fd, &clst->data_hdr, 0); -//cerr << "got some data on fd " << fd << endl; -#endif + // cerr << "waiting for data..." << endl; + ret = selectWithoutConsole(maxFd + 1, &read_set, tvp); if (ret < 0) { -#if DEBUG - flprintf("data recvfrom %d: %s\n", fd, strerror(errno)); -#endif - return -1; + perror("Select"); + return 0; } + if (ret == 0) + { + clst->endReached = 3; + clst->netEndReached = 3; + pc_produceEnd(clst->fifo->data); + return 1; + } + + fd = udpc_getSelectedSock(clst->client_config->socks, NR_CLIENT_SOCKS, &read_set); + } + +#ifdef LOSSTEST + loseRecvPacket(fd); + ret = RecvMsg(fd, &clst->data_hdr, 0); +#else + ret = recvmsg(fd, &clst->data_hdr, 0); +// cerr << "got some data on fd " << fd << endl; +#endif + + if (ret < 0) + { +#if DEBUG + flprintf("data recvfrom %d: %s\n", fd, strerror(errno)); +#endif + return -1; + } + #if 0 fprintf(stderr, "received packet for slice %d, block %d\n", ntohl(Msg.sliceNo), ntohs(db.blockNo)); #endif - if (!udpc_isAddressEqual(&lserver, - &clst->client_config->serverAddr)) - { - return -1; - } + if (!udpc_isAddressEqual(&lserver, &clst->client_config->serverAddr)) + { + return -1; + } - switch (ntohs(clst->Msg.opCode)) - { - case CMD_DATA: -//cerr << "got CMD_DATA" << endl; - closeAllExcept(clst, fd); - //udpc_receiverStatsStartTimer(clst->stats); - clst->client_config->isStarted = 1; - return processDataBlock(clst, - ntohl(clst->Msg.dataBlock.sliceNo), - ntohs(clst->Msg.dataBlock.blockNo), - ntohl(clst->Msg.dataBlock.bytes)); + switch (ntohs(clst->Msg.opCode)) + { + case CMD_DATA: + // cerr << "got CMD_DATA" << endl; + closeAllExcept(clst, fd); + // udpc_receiverStatsStartTimer(clst->stats); + clst->client_config->isStarted = 1; + return processDataBlock(clst, ntohl(clst->Msg.dataBlock.sliceNo), ntohs(clst->Msg.dataBlock.blockNo), + ntohl(clst->Msg.dataBlock.bytes)); #ifdef BB_FEATURE_UDPCAST_FEC - case CMD_FEC: -//cerr << "got CMD_FEC" << endl; - closeAllExcept(clst, fd); - //receiverStatsStartTimer(clst->stats); - clst->client_config->isStarted = 1; - return processFecBlock(clst, - ntohs(clst->Msg.fecBlock.stripes), - ntohl(clst->Msg.fecBlock.sliceNo), - ntohs(clst->Msg.fecBlock.blockNo), - ntohl(clst->Msg.fecBlock.bytes)); + case CMD_FEC: + // cerr << "got CMD_FEC" << endl; + closeAllExcept(clst, fd); + // receiverStatsStartTimer(clst->stats); + clst->client_config->isStarted = 1; + return processFecBlock(clst, ntohs(clst->Msg.fecBlock.stripes), ntohl(clst->Msg.fecBlock.sliceNo), + ntohs(clst->Msg.fecBlock.blockNo), ntohl(clst->Msg.fecBlock.bytes)); #endif - case CMD_REQACK: -//cerr << "got CMD_REQACK" << endl; - closeAllExcept(clst, fd); - //receiverStatsStartTimer(clst->stats); - clst->client_config->isStarted = 1; - return processReqAck(clst, - ntohl(clst->Msg.reqack.sliceNo), - ntohl(clst->Msg.reqack.bytes), - ntohl(clst->Msg.reqack.rxmit)); + case CMD_REQACK: + // cerr << "got CMD_REQACK" << endl; + closeAllExcept(clst, fd); + // receiverStatsStartTimer(clst->stats); + clst->client_config->isStarted = 1; + return processReqAck(clst, ntohl(clst->Msg.reqack.sliceNo), ntohl(clst->Msg.reqack.bytes), + ntohl(clst->Msg.reqack.rxmit)); - case CMD_HELLO_STREAMING: - case CMD_HELLO_NEW: - case CMD_HELLO: -//cerr << "got CMD_HELLO" << endl; - /* retransmission of hello to find other participants ==> ignore */ - return 0; + case CMD_HELLO_STREAMING: + case CMD_HELLO_NEW: + case CMD_HELLO: + // cerr << "got CMD_HELLO" << endl; + /* retransmission of hello to find other participants ==> ignore */ + return 0; - default: - break; - } + default: break; + } - return -1; + return -1; } THREAD_RETURN netReceiverMain(void* args0) { - struct clientState* clst = (struct clientState*) args0; + struct clientState* clst = (struct clientState*)args0; - clst->currentSliceNo = 0; - setupMessages(clst); + clst->currentSliceNo = 0; + setupMessages(clst); - clst->currentSliceNo = -1; + clst->currentSliceNo = -1; + clst->currentSlice = NULL; + clst->promptPrinted = 0; + + if (!(clst->net_config->flags & FLAG_STREAMING)) + newSlice(clst, 0); + else + { clst->currentSlice = NULL; - clst->promptPrinted = 0; + clst->currentSliceNo = 0; + } - if (! (clst->net_config->flags & FLAG_STREAMING)) - newSlice(clst, 0); - else - { - clst->currentSlice = NULL; - clst->currentSliceNo = 0; - } - - while (clst->endReached < 3) - { - dispatchMessage(clst); - } + while (clst->endReached < 3) + { + dispatchMessage(clst); + } #ifdef BB_FEATURE_UDPCAST_FEC - if (clst->use_fec) - pthread_join(clst->fec_thread, NULL); + if (clst->use_fec) + pthread_join(clst->fec_thread, NULL); #endif - return 0; + return 0; } -int spawnNetReceiver(struct fifo* fifo, - struct client_config* client_config, - struct net_config* net_config) +int spawnNetReceiver(struct fifo* fifo, struct client_config* client_config, struct net_config* net_config) { - int i; - struct clientState* clst = MALLOC(struct clientState); - clst->fifo = fifo; - clst->client_config = client_config; - clst->net_config = net_config; - //clst->stats = stats; - clst->endReached = 0; - clst->netEndReached = 0; - clst->selectedFd = -1; + int i; + struct clientState* clst = MALLOC(struct clientState); + clst->fifo = fifo; + clst->client_config = client_config; + clst->net_config = net_config; + // clst->stats = stats; + clst->endReached = 0; + clst->netEndReached = 0; + clst->selectedFd = -1; - clst->free_slices_pc = pc_makeProduconsum(NR_SLICES, "free slices"); - pc_produce(clst->free_slices_pc, NR_SLICES); + clst->free_slices_pc = pc_makeProduconsum(NR_SLICES, "free slices"); + pc_produce(clst->free_slices_pc, NR_SLICES); - for (i = 0; i < NR_SLICES; i++) - clst->slices[i].state = slice::SLICE_FREE; + for (i = 0; i < NR_SLICES; i++) + clst->slices[i].state = slice::SLICE_FREE; - clst->receivedPtr = 0; - clst->receivedSliceNo = 0; + clst->receivedPtr = 0; + clst->receivedSliceNo = 0; #ifdef BB_FEATURE_UDPCAST_FEC - fec_init(); /* fec new involves memory - * allocation. Better do it here */ - clst->use_fec = 0; - clst->fec_data_pc = pc_makeProduconsum(NR_SLICES, "fec data"); + fec_init(); /* fec new involves memory + * allocation. Better do it here */ + clst->use_fec = 0; + clst->fec_data_pc = pc_makeProduconsum(NR_SLICES, "fec data"); #endif #define NR_BLOCKS 4096 - clst->freeBlocks_pc = pc_makeProduconsum(NR_BLOCKS, "free blocks"); - pc_produce(clst->freeBlocks_pc, NR_BLOCKS); - clst->blockAddresses = (unsigned char**)calloc(NR_BLOCKS, sizeof(char*)); - clst->localBlockAddresses = (unsigned char**)calloc(NR_BLOCKS, sizeof(char*)); - clst->blockData = (unsigned char*)malloc(NR_BLOCKS * net_config->blockSize); + clst->freeBlocks_pc = pc_makeProduconsum(NR_BLOCKS, "free blocks"); + pc_produce(clst->freeBlocks_pc, NR_BLOCKS); + clst->blockAddresses = (unsigned char**)calloc(NR_BLOCKS, sizeof(char*)); + clst->localBlockAddresses = (unsigned char**)calloc(NR_BLOCKS, sizeof(char*)); + clst->blockData = (unsigned char*)malloc(NR_BLOCKS * net_config->blockSize); - for (i = 0; i < NR_BLOCKS; i++) - clst->blockAddresses[i] = clst->blockData + i * net_config->blockSize; + for (i = 0; i < NR_BLOCKS; i++) + clst->blockAddresses[i] = clst->blockData + i * net_config->blockSize; - clst->localPos = 0; + clst->localPos = 0; - setNextBlock(clst); - return pthread_create(&client_config->thread, NULL, netReceiverMain, clst); + setNextBlock(clst); + return pthread_create(&client_config->thread, NULL, netReceiverMain, clst); } unsigned int pc_getSize(produconsum_t pc) { - return pc->size; + return pc->size; } int writer(struct fifo* fifo, SBS outbs) { -//cerr << "start appending to outbs" << endl; - outbs->restart(); - int fifoSize = pc_getSize(fifo->data); + // cerr << "start appending to outbs" << endl; + outbs->restart(); + int fifoSize = pc_getSize(fifo->data); - while (1) + while (1) + { + int pos = pc_getConsumerPosition(fifo->data); + int bytes = pc_consumeContiguousMinAmount(fifo->data, BLOCKSIZE); + + if (bytes == 0) { - int pos = pc_getConsumerPosition(fifo->data); - int bytes = pc_consumeContiguousMinAmount(fifo->data, BLOCKSIZE); - - if (bytes == 0) - { -//cerr << "done appending to outbs: " << outbs->length() << endl; - return 0; - } - - /* - * If we have more than blocksize, round down to nearest blocksize - * multiple - */ - if (pos + bytes != fifoSize && - bytes > (pos + bytes) % BLOCKSIZE) - bytes -= (pos + bytes) % BLOCKSIZE; - - /* make sure we don't write to big a chunk... Better to - * liberate small chunks one by one rather than attempt to - * write out a bigger chunk and block reception for too - * long */ - if (bytes > 128 * 1024) - bytes = 64 * 1024; - - //bytes = write(outFile, fifo->dataBuffer + pos, bytes); -//cerr << "appending " << bytes << " bytes to outbs..." << endl; - outbs->append(fifo->dataBuffer + pos, bytes); - pc_consumed(fifo->data, bytes); - pc_produce(fifo->freeMemQueue, bytes); + // cerr << "done appending to outbs: " << outbs->length() << endl; + return 0; } + + /* + * If we have more than blocksize, round down to nearest blocksize + * multiple + */ + if (pos + bytes != fifoSize && bytes > (pos + bytes) % BLOCKSIZE) + bytes -= (pos + bytes) % BLOCKSIZE; + + /* make sure we don't write to big a chunk... Better to + * liberate small chunks one by one rather than attempt to + * write out a bigger chunk and block reception for too + * long */ + if (bytes > 128 * 1024) + bytes = 64 * 1024; + + // bytes = write(outFile, fifo->dataBuffer + pos, bytes); + // cerr << "appending " << bytes << " bytes to outbs..." << endl; + outbs->append(fifo->dataBuffer + pos, bytes); + pc_consumed(fifo->data, bytes); + pc_produce(fifo->freeMemQueue, bytes); + } } -} +} // namespace namespace multicast { - -MulticastImpl::MulticastImpl(int min_receivers, const string& ifName, int portBase, int bufSize) : - fIfName(ifName), - fDb(0) +MulticastImpl::MulticastImpl(int min_receivers, const string& ifName, int portBase, int bufSize) + : fIfName(ifName), fDb(0) { - udpc_clearIp(&fNet_config.dataMcastAddr); - fNet_config.mcastRdv = 0; - fNet_config.blockSize = 1024; //1456; - fNet_config.sliceSize = 16; - fNet_config.portBase = portBase; - fNet_config.nrGovernors = 0; - fNet_config.flags = FLAG_NOKBD; - fNet_config.capabilities = 0; - fNet_config.min_slice_size = 16; - fNet_config.max_slice_size = 1024; - fNet_config.default_slice_size = 0; - fNet_config.ttl = 1; - fNet_config.rexmit_hello_interval = 2000; - fNet_config.autostart = 0; - fNet_config.requestedBufSize = bufSize; + udpc_clearIp(&fNet_config.dataMcastAddr); + fNet_config.mcastRdv = 0; + fNet_config.blockSize = 1024; // 1456; + fNet_config.sliceSize = 16; + fNet_config.portBase = portBase; + fNet_config.nrGovernors = 0; + fNet_config.flags = FLAG_NOKBD; + fNet_config.capabilities = 0; + fNet_config.min_slice_size = 16; + fNet_config.max_slice_size = 1024; + fNet_config.default_slice_size = 0; + fNet_config.ttl = 1; + fNet_config.rexmit_hello_interval = 2000; + fNet_config.autostart = 0; + fNet_config.requestedBufSize = bufSize; - fNet_config.min_receivers = min_receivers; - fNet_config.max_receivers_wait = 0; - fNet_config.min_receivers_wait = 0; + fNet_config.min_receivers = min_receivers; + fNet_config.max_receivers_wait = 0; + fNet_config.min_receivers_wait = 0; - fNet_config.retriesUntilDrop = 200; + fNet_config.retriesUntilDrop = 200; - fNet_config.rehelloOffset = 50; + fNet_config.rehelloOffset = 50; - fStat_config.log = 0; - fStat_config.bwPeriod = 0; - fStat_config.printUncompressedPos = -1; - fStat_config.statPeriod = DEFLT_STAT_PERIOD; + fStat_config.log = 0; + fStat_config.bwPeriod = 0; + fStat_config.printUncompressedPos = -1; + fStat_config.statPeriod = DEFLT_STAT_PERIOD; - fNet_config.net_if = 0; + fNet_config.net_if = 0; - //full-duplex - fNet_config.flags |= FLAG_SN; + // full-duplex + fNet_config.flags |= FLAG_SN; - if (fIfName.empty()) - fIfName = "eth0"; + if (fIfName.empty()) + fIfName = "eth0"; - fSock[0] = -1; - fSock[1] = -1; - fSock[2] = -1; + fSock[0] = -1; + fSock[1] = -1; + fSock[2] = -1; } MulticastImpl::~MulticastImpl() { - delete fDb; + delete fDb; - for (int i = 0; i < 3; i++) + for (int i = 0; i < 3; i++) + { + if (fSock[i] >= 0) { - if (fSock[i] >= 0) - { - shutdown(fSock[i], SHUT_RDWR); - close(fSock[i]); - } + shutdown(fSock[i], SHUT_RDWR); + close(fSock[i]); } + } } void MulticastImpl::startSender() { - int tries; - time_t firstConnected = 0; - time_t* firstConnectedP; + int tries; + time_t firstConnected = 0; + time_t* firstConnectedP; - //participantsDb_t db; + // participantsDb_t db; - /* make the socket and print banner */ - //int fSock[3]; - int nr = 0; - int fd; - int r; - int j; + /* make the socket and print banner */ + // int fSock[3]; + int nr = 0; + int fd; + int r; + int j; - fNet_config.net_if = udpc_getNetIf(fIfName.c_str()); + fNet_config.net_if = udpc_getNetIf(fIfName.c_str()); - fSock[nr++] = udpc_makeSocket(ADDR_TYPE_UCAST, - fNet_config.net_if, - NULL, - SENDER_PORT(fNet_config.portBase)); -//cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; + fSock[nr++] = udpc_makeSocket(ADDR_TYPE_UCAST, fNet_config.net_if, NULL, SENDER_PORT(fNet_config.portBase)); + // cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; - if (! (fNet_config.flags & (FLAG_SN | FLAG_NOTSN)) ) + if (!(fNet_config.flags & (FLAG_SN | FLAG_NOTSN))) + { + if (udpc_isFullDuplex(fSock[0], fNet_config.net_if->name) == 1) { - if (udpc_isFullDuplex(fSock[0], fNet_config.net_if->name) == 1) - { - fNet_config.flags |= FLAG_SN; - } + fNet_config.flags |= FLAG_SN; } + } - fd = udpc_makeSocket(ADDR_TYPE_BCAST, - fNet_config.net_if, - NULL, - SENDER_PORT(fNet_config.portBase)); + fd = udpc_makeSocket(ADDR_TYPE_BCAST, fNet_config.net_if, NULL, SENDER_PORT(fNet_config.portBase)); - if (fd >= 0) - fSock[nr++] = fd; + if (fd >= 0) + fSock[nr++] = fd; -//cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; + // cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; - if (fNet_config.requestedBufSize) - udpc_setSendBuf(fSock[0], fNet_config.requestedBufSize); + if (fNet_config.requestedBufSize) + udpc_setSendBuf(fSock[0], fNet_config.requestedBufSize); - fNet_config.controlMcastAddr.sin_addr.s_addr = 0; + fNet_config.controlMcastAddr.sin_addr.s_addr = 0; - if (fNet_config.ttl == 1 && fNet_config.mcastRdv == NULL) + if (fNet_config.ttl == 1 && fNet_config.mcastRdv == NULL) + { + udpc_getBroadCastAddress(fNet_config.net_if, &fNet_config.controlMcastAddr, + RECEIVER_PORT(fNet_config.portBase)); + udpc_setSocketToBroadcast(fSock[0]); + } + + if (fNet_config.controlMcastAddr.sin_addr.s_addr == 0) + { + udpc_getMcastAllAddress(&fNet_config.controlMcastAddr, fNet_config.mcastRdv, + RECEIVER_PORT(fNet_config.portBase)); + + /* Only do the following if controlMcastAddr is indeed an + mcast address ... */ + if (isMcastAddress(&fNet_config.controlMcastAddr)) { - udpc_getBroadCastAddress(fNet_config.net_if, - &fNet_config.controlMcastAddr, - RECEIVER_PORT(fNet_config.portBase)); - udpc_setSocketToBroadcast(fSock[0]); + udpc_setMcastDestination(fSock[0], fNet_config.net_if, &fNet_config.controlMcastAddr); + udpc_setTtl(fSock[0], fNet_config.ttl); + fSock[nr++] = udpc_makeSocket(ADDR_TYPE_MCAST, fNet_config.net_if, &fNet_config.controlMcastAddr, + SENDER_PORT(fNet_config.portBase)); + // cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; } + } - if (fNet_config.controlMcastAddr.sin_addr.s_addr == 0) - { - udpc_getMcastAllAddress(&fNet_config.controlMcastAddr, - fNet_config.mcastRdv, - RECEIVER_PORT(fNet_config.portBase)); + if (!(fNet_config.flags & FLAG_POINTOPOINT) && udpc_ipIsZero(&fNet_config.dataMcastAddr)) + { + udpc_getDefaultMcastAddress(fNet_config.net_if, &fNet_config.dataMcastAddr); + } - /* Only do the following if controlMcastAddr is indeed an - mcast address ... */ - if (isMcastAddress(&fNet_config.controlMcastAddr)) - { - udpc_setMcastDestination(fSock[0], fNet_config.net_if, - &fNet_config.controlMcastAddr); - udpc_setTtl(fSock[0], fNet_config.ttl); - fSock[nr++] = udpc_makeSocket(ADDR_TYPE_MCAST, - fNet_config.net_if, - &fNet_config.controlMcastAddr, - SENDER_PORT(fNet_config.portBase)); -//cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; - } - } + if (fNet_config.flags & FLAG_POINTOPOINT) + { + udpc_clearIp(&fNet_config.dataMcastAddr); + } - if (!(fNet_config.flags & FLAG_POINTOPOINT) && - udpc_ipIsZero(&fNet_config.dataMcastAddr)) - { - udpc_getDefaultMcastAddress(fNet_config.net_if, - &fNet_config.dataMcastAddr); - } + setPort(&fNet_config.dataMcastAddr, RECEIVER_PORT(fNet_config.portBase)); - if (fNet_config.flags & FLAG_POINTOPOINT) - { - udpc_clearIp(&fNet_config.dataMcastAddr); - } + fNet_config.capabilities = SENDER_CAPABILITIES; - setPort(&fNet_config.dataMcastAddr, RECEIVER_PORT(fNet_config.portBase)); + if (fNet_config.flags & FLAG_ASYNC) + fNet_config.capabilities |= CAP_ASYNC; - fNet_config.capabilities = SENDER_CAPABILITIES; + udpc_sendHello(&fNet_config, fSock[0], 0); - if (fNet_config.flags & FLAG_ASYNC) - fNet_config.capabilities |= CAP_ASYNC; + fDb = udpc_makeParticipantsDb(); + tries = 0; - udpc_sendHello(&fNet_config, fSock[0], 0); + if (fNet_config.min_receivers || fNet_config.min_receivers_wait || fNet_config.max_receivers_wait) + firstConnectedP = &firstConnected; + else + firstConnectedP = NULL; - fDb = udpc_makeParticipantsDb(); - tries = 0; + while (!(r = mainDispatcher(fSock, nr, fDb, &fNet_config, &tries, firstConnectedP))) + ; - if (fNet_config.min_receivers || fNet_config.min_receivers_wait || - fNet_config.max_receivers_wait) - firstConnectedP = &firstConnected; - else - firstConnectedP = NULL; + for (j = 1; j < nr; j++) + if (fSock[j] != fSock[0]) + closesocket(fSock[j]); - while (!(r = mainDispatcher(fSock, nr, fDb, &fNet_config, &tries, firstConnectedP))) - ; + if (r == 1) + { + int i; - for (j = 1; j < nr; j++) - if (fSock[j] != fSock[0]) - closesocket(fSock[j]); + for (i = 1; i < nr; i++) + udpc_closeSock(fSock, nr, i); + } - if (r == 1) - { - int i; - - for (i = 1; i < nr; i++) - udpc_closeSock(fSock, nr, i); - } - - //doTransfer(fSock[0], fDb, &fNet_config, &fStat_config); + // doTransfer(fSock[0], fDb, &fNet_config, &fStat_config); } void MulticastImpl::doTransfer(const uint8_t* buf, uint32_t len) { - int i; - int ret; - struct fifo fifo; - int isPtP = isPointToPoint(fDb, fNet_config.flags); + int i; + int ret; + struct fifo fifo; + int isPtP = isPointToPoint(fDb, fNet_config.flags); - fNet_config.rcvbuf = 0; + fNet_config.rcvbuf = 0; - for (i = 0; i < MAX_CLIENTS; i++) - if (udpc_isParticipantValid(fDb, i)) - { - unsigned int pRcvBuf = udpc_getParticipantRcvBuf(fDb, i); - - if (isPtP) - udpc_copyIpFrom(&fNet_config.dataMcastAddr, - udpc_getParticipantIp(fDb, i)); - - fNet_config.capabilities &= - udpc_getParticipantCapabilities(fDb, i); - - if (pRcvBuf != 0 && - (fNet_config.rcvbuf == 0 || fNet_config.rcvbuf > pRcvBuf)) - fNet_config.rcvbuf = pRcvBuf; - } - - if (isMcastAddress(&fNet_config.dataMcastAddr)) - udpc_setMcastDestination(fSock[0], fNet_config.net_if, - &fNet_config.dataMcastAddr); - - if (! (fNet_config.capabilities & CAP_BIG_ENDIAN)) + for (i = 0; i < MAX_CLIENTS; i++) + if (udpc_isParticipantValid(fDb, i)) { - //TODO: FIXME - //udpc_fatal(1, "Peer with incompatible endianness"); + unsigned int pRcvBuf = udpc_getParticipantRcvBuf(fDb, i); + + if (isPtP) + udpc_copyIpFrom(&fNet_config.dataMcastAddr, udpc_getParticipantIp(fDb, i)); + + fNet_config.capabilities &= udpc_getParticipantCapabilities(fDb, i); + + if (pRcvBuf != 0 && (fNet_config.rcvbuf == 0 || fNet_config.rcvbuf > pRcvBuf)) + fNet_config.rcvbuf = pRcvBuf; } - if (! (fNet_config.capabilities & CAP_NEW_GEN)) - { - fNet_config.dataMcastAddr = fNet_config.controlMcastAddr; - fNet_config.flags &= ~(FLAG_SN | FLAG_ASYNC); - } + if (isMcastAddress(&fNet_config.dataMcastAddr)) + udpc_setMcastDestination(fSock[0], fNet_config.net_if, &fNet_config.dataMcastAddr); - if (fNet_config.flags & FLAG_BCAST) - fNet_config.dataMcastAddr = fNet_config.controlMcastAddr; + if (!(fNet_config.capabilities & CAP_BIG_ENDIAN)) + { + // TODO: FIXME + // udpc_fatal(1, "Peer with incompatible endianness"); + } - udpc_initFifo(&fifo, fNet_config.blockSize); - ret = spawnNetSender(&fifo, fSock[0], &fNet_config, fDb); - localReader(&fifo, buf, len); + if (!(fNet_config.capabilities & CAP_NEW_GEN)) + { + fNet_config.dataMcastAddr = fNet_config.controlMcastAddr; + fNet_config.flags &= ~(FLAG_SN | FLAG_ASYNC); + } - pthread_join(fifo.thread, NULL); + if (fNet_config.flags & FLAG_BCAST) + fNet_config.dataMcastAddr = fNet_config.controlMcastAddr; + udpc_initFifo(&fifo, fNet_config.blockSize); + ret = spawnNetSender(&fifo, fSock[0], &fNet_config, fDb); + localReader(&fifo, buf, len); + + pthread_join(fifo.thread, NULL); } void MulticastImpl::startReceiver() { - union serverControlMsg Msg; - int connectReqSent = 0; - struct sockaddr_in myIp; - int haveServerAddress; + union serverControlMsg Msg; + int connectReqSent = 0; + struct sockaddr_in myIp; + int haveServerAddress; - fClient_config.sender_is_newgen = 0; + fClient_config.sender_is_newgen = 0; - fNet_config.net_if = udpc_getNetIf(fIfName.c_str()); + fNet_config.net_if = udpc_getNetIf(fIfName.c_str()); + { + fprintf(stderr, "net_if:\n\taddr = 0x%x\n\tbcast = 0x%x\n\tname = %s\n\tindex = %d\n", + fNet_config.net_if->addr.s_addr, fNet_config.net_if->bcast.s_addr, fNet_config.net_if->name, + fNet_config.net_if->index); + } + + udpc_zeroSockArray(fClient_config.socks, NR_CLIENT_SOCKS); + + fClient_config.S_UCAST = + udpc_makeSocket(ADDR_TYPE_UCAST, fNet_config.net_if, 0, RECEIVER_PORT(fNet_config.portBase)); + // cerr << "S_UCAST = " << fClient_config.S_UCAST << endl; + fClient_config.S_BCAST = + udpc_makeSocket(ADDR_TYPE_BCAST, fNet_config.net_if, 0, RECEIVER_PORT(fNet_config.portBase)); + // cerr << "S_BCAST = " << fClient_config.S_BCAST << endl; + + if (fNet_config.ttl == 1 && fNet_config.mcastRdv == NULL) + { + udpc_getBroadCastAddress(fNet_config.net_if, &fNet_config.controlMcastAddr, + SENDER_PORT(fNet_config.portBase)); + udpc_setSocketToBroadcast(fClient_config.S_UCAST); + } + else + { + udpc_getMcastAllAddress(&fNet_config.controlMcastAddr, fNet_config.mcastRdv, + SENDER_PORT(fNet_config.portBase)); + + if (isMcastAddress(&fNet_config.controlMcastAddr)) { - fprintf(stderr, "net_if:\n\taddr = 0x%x\n\tbcast = 0x%x\n\tname = %s\n\tindex = %d\n", fNet_config.net_if->addr.s_addr, fNet_config.net_if->bcast.s_addr, fNet_config.net_if->name, fNet_config.net_if->index); + udpc_setMcastDestination(fClient_config.S_UCAST, fNet_config.net_if, &fNet_config.controlMcastAddr); + udpc_setTtl(fClient_config.S_UCAST, fNet_config.ttl); + + fClient_config.S_MCAST_CTRL = + udpc_makeSocket(ADDR_TYPE_MCAST, fNet_config.net_if, &fNet_config.controlMcastAddr, + RECEIVER_PORT(fNet_config.portBase)); + // cerr << "S_MCAST_CTRL = " << fClient_config.S_MCAST_CTRL << endl; + // TODO: subscribe address as receiver to! + } + } + + udpc_clearIp(&fNet_config.dataMcastAddr); + + connectReqSent = 0; + haveServerAddress = 0; + + fClient_config.clientNumber = 0; /*default number for asynchronous transfer*/ + + while (1) + { + // int len; + int msglen; + int sock; + + if (!connectReqSent) + { + if (sendConnectReq(&fClient_config, &fNet_config, haveServerAddress) < 0) + { + // TODO: FIXME + // perror("sendto to locate server"); + } + + connectReqSent = 1; } - udpc_zeroSockArray(fClient_config.socks, NR_CLIENT_SOCKS); - - fClient_config.S_UCAST = udpc_makeSocket(ADDR_TYPE_UCAST, - fNet_config.net_if, - 0, RECEIVER_PORT(fNet_config.portBase)); -//cerr << "S_UCAST = " << fClient_config.S_UCAST << endl; - fClient_config.S_BCAST = udpc_makeSocket(ADDR_TYPE_BCAST, - fNet_config.net_if, - 0, RECEIVER_PORT(fNet_config.portBase)); -//cerr << "S_BCAST = " << fClient_config.S_BCAST << endl; - - if (fNet_config.ttl == 1 && fNet_config.mcastRdv == NULL) - { - udpc_getBroadCastAddress(fNet_config.net_if, - &fNet_config.controlMcastAddr, - SENDER_PORT(fNet_config.portBase)); - udpc_setSocketToBroadcast(fClient_config.S_UCAST); - } - else - { - udpc_getMcastAllAddress(&fNet_config.controlMcastAddr, - fNet_config.mcastRdv, - SENDER_PORT(fNet_config.portBase)); - - if (isMcastAddress(&fNet_config.controlMcastAddr)) - { - udpc_setMcastDestination(fClient_config.S_UCAST, fNet_config.net_if, - &fNet_config.controlMcastAddr); - udpc_setTtl(fClient_config.S_UCAST, fNet_config.ttl); - - fClient_config.S_MCAST_CTRL = - udpc_makeSocket(ADDR_TYPE_MCAST, - fNet_config.net_if, - &fNet_config.controlMcastAddr, - RECEIVER_PORT(fNet_config.portBase)); -//cerr << "S_MCAST_CTRL = " << fClient_config.S_MCAST_CTRL << endl; - // TODO: subscribe address as receiver to! - } - } - - udpc_clearIp(&fNet_config.dataMcastAddr); - - connectReqSent = 0; haveServerAddress = 0; - fClient_config.clientNumber = 0; /*default number for asynchronous transfer*/ + // cerr << "waiting for msg..." << flush << endl; + sock = udpc_selectSock(fClient_config.socks, NR_CLIENT_SOCKS, fNet_config.startTimeout); - while (1) + // cerr << "got something" << endl; + if (sock < 0) { - // int len; - int msglen; - int sock; - - if (!connectReqSent) - { - if (sendConnectReq(&fClient_config, &fNet_config, - haveServerAddress) < 0) - { - //TODO: FIXME - //perror("sendto to locate server"); - } - - connectReqSent = 1; - } - - haveServerAddress = 0; - -//cerr << "waiting for msg..." << flush << endl; - sock = udpc_selectSock(fClient_config.socks, NR_CLIENT_SOCKS, - fNet_config.startTimeout); - -//cerr << "got something" << endl; - if (sock < 0) - { - //TODO: FIXME - } - - // len = sizeof(server); - msglen = RECV(sock, - Msg, fClient_config.serverAddr, fNet_config.portBase); - - if (msglen < 0) - { - //TODO: FIXME - //perror("recvfrom to locate server"); - //exit(1); - } - - if (udpc_getPort(&fClient_config.serverAddr) != - SENDER_PORT(fNet_config.portBase)) - /* not from the right port */ - continue; - - switch (ntohs(Msg.opCode)) - { - case CMD_CONNECT_REPLY: -//cerr << "got conrep" << endl; - fClient_config.clientNumber = ntohl(Msg.connectReply.clNr); - fNet_config.blockSize = ntohl(Msg.connectReply.blockSize); - - if (ntohl(Msg.connectReply.capabilities) & CAP_NEW_GEN) - { - fClient_config.sender_is_newgen = 1; - udpc_copyFromMessage(&fNet_config.dataMcastAddr, - Msg.connectReply.mcastAddr); - } - - if (fClient_config.clientNumber == -1) - { - //TODO: FIXME - //udpc_fatal(1, "Too many clients already connected\n"); - } - - goto break_loop; - - case CMD_HELLO_STREAMING: - case CMD_HELLO_NEW: - case CMD_HELLO: -//cerr << "got hello" << endl; - connectReqSent = 0; - - if (ntohs(Msg.opCode) == CMD_HELLO_STREAMING) - fNet_config.flags |= FLAG_STREAMING; - - if (ntohl(Msg.hello.capabilities) & CAP_NEW_GEN) - { - fClient_config.sender_is_newgen = 1; - udpc_copyFromMessage(&fNet_config.dataMcastAddr, - Msg.hello.mcastAddr); - fNet_config.blockSize = ntohs(Msg.hello.blockSize); - - if (ntohl(Msg.hello.capabilities) & CAP_ASYNC) - fNet_config.flags |= FLAG_PASSIVE; - - if (fNet_config.flags & FLAG_PASSIVE) - goto break_loop; - } - - haveServerAddress = 1; - continue; - - case CMD_CONNECT_REQ: - case CMD_DATA: - case CMD_FEC: - continue; - - default: - break; - } - - //TODO: FIXME - //udpc_fatal(1, "Bad server reply %04x. Other transfer in progress?\n", (unsigned short) ntohs(Msg.opCode)); + // TODO: FIXME } + // len = sizeof(server); + msglen = RECV(sock, Msg, fClient_config.serverAddr, fNet_config.portBase); + + if (msglen < 0) + { + // TODO: FIXME + // perror("recvfrom to locate server"); + // exit(1); + } + + if (udpc_getPort(&fClient_config.serverAddr) != SENDER_PORT(fNet_config.portBase)) + /* not from the right port */ + continue; + + switch (ntohs(Msg.opCode)) + { + case CMD_CONNECT_REPLY: + // cerr << "got conrep" << endl; + fClient_config.clientNumber = ntohl(Msg.connectReply.clNr); + fNet_config.blockSize = ntohl(Msg.connectReply.blockSize); + + if (ntohl(Msg.connectReply.capabilities) & CAP_NEW_GEN) + { + fClient_config.sender_is_newgen = 1; + udpc_copyFromMessage(&fNet_config.dataMcastAddr, Msg.connectReply.mcastAddr); + } + + if (fClient_config.clientNumber == -1) + { + // TODO: FIXME + // udpc_fatal(1, "Too many clients already connected\n"); + } + + goto break_loop; + + case CMD_HELLO_STREAMING: + case CMD_HELLO_NEW: + case CMD_HELLO: + // cerr << "got hello" << endl; + connectReqSent = 0; + + if (ntohs(Msg.opCode) == CMD_HELLO_STREAMING) + fNet_config.flags |= FLAG_STREAMING; + + if (ntohl(Msg.hello.capabilities) & CAP_NEW_GEN) + { + fClient_config.sender_is_newgen = 1; + udpc_copyFromMessage(&fNet_config.dataMcastAddr, Msg.hello.mcastAddr); + fNet_config.blockSize = ntohs(Msg.hello.blockSize); + + if (ntohl(Msg.hello.capabilities) & CAP_ASYNC) + fNet_config.flags |= FLAG_PASSIVE; + + if (fNet_config.flags & FLAG_PASSIVE) + goto break_loop; + } + + haveServerAddress = 1; + continue; + + case CMD_CONNECT_REQ: + case CMD_DATA: + case CMD_FEC: continue; + + default: break; + } + + // TODO: FIXME + // udpc_fatal(1, "Bad server reply %04x. Other transfer in progress?\n", (unsigned short) + // ntohs(Msg.opCode)); + } + break_loop: - udpc_getMyAddress(fNet_config.net_if, &myIp); + udpc_getMyAddress(fNet_config.net_if, &myIp); - if (!udpc_ipIsZero(&fNet_config.dataMcastAddr) && - !udpc_ipIsEqual(&fNet_config.dataMcastAddr, &myIp) && - (udpc_ipIsZero(&fNet_config.controlMcastAddr) || - !udpc_ipIsEqual(&fNet_config.dataMcastAddr, &fNet_config.controlMcastAddr) - )) - { - fClient_config.S_MCAST_DATA = - udpc_makeSocket(ADDR_TYPE_MCAST, fNet_config.net_if, - &fNet_config.dataMcastAddr, - RECEIVER_PORT(fNet_config.portBase)); -//cerr << "S_MCAST_DATA = " << fClient_config.S_MCAST_DATA << endl; - } + if (!udpc_ipIsZero(&fNet_config.dataMcastAddr) && !udpc_ipIsEqual(&fNet_config.dataMcastAddr, &myIp) && + (udpc_ipIsZero(&fNet_config.controlMcastAddr) || + !udpc_ipIsEqual(&fNet_config.dataMcastAddr, &fNet_config.controlMcastAddr))) + { + fClient_config.S_MCAST_DATA = udpc_makeSocket( + ADDR_TYPE_MCAST, fNet_config.net_if, &fNet_config.dataMcastAddr, RECEIVER_PORT(fNet_config.portBase)); + // cerr << "S_MCAST_DATA = " << fClient_config.S_MCAST_DATA << endl; + } + if (fNet_config.requestedBufSize) + { + int i; - if (fNet_config.requestedBufSize) - { - int i; - - for (i = 0; i < NR_CLIENT_SOCKS; i++) - if (fClient_config.socks[i] != -1) - udpc_setRcvBuf(fClient_config.socks[i], fNet_config.requestedBufSize); - } - + for (i = 0; i < NR_CLIENT_SOCKS; i++) + if (fClient_config.socks[i] != -1) + udpc_setRcvBuf(fClient_config.socks[i], fNet_config.requestedBufSize); + } } void MulticastImpl::receive(SBS outbs) { - struct fifo fifo; + struct fifo fifo; - udpc_initFifo(&fifo, fNet_config.blockSize); - fifo.data = pc_makeProduconsum(fifo.dataBufSize, "receive"); - fClient_config.isStarted = 0; + udpc_initFifo(&fifo, fNet_config.blockSize); + fifo.data = pc_makeProduconsum(fifo.dataBufSize, "receive"); + fClient_config.isStarted = 0; - spawnNetReceiver(&fifo, &fClient_config, &fNet_config); - writer(&fifo, outbs); - - pthread_join(fClient_config.thread, NULL); -} + spawnNetReceiver(&fifo, &fClient_config, &fNet_config); + writer(&fifo, outbs); + pthread_join(fClient_config.thread, NULL); } +} // namespace multicast diff --git a/utils/multicast/impl.h b/utils/multicast/impl.h index 035d6c587..1c649ac52 100644 --- a/utils/multicast/impl.h +++ b/utils/multicast/impl.h @@ -40,28 +40,27 @@ namespace multicast { - class MulticastImpl { -public: - MulticastImpl(int min_receivers, const std::string& ifName, int portBase = 9000, int bufSize = 8 * 1024 * 1024); - ~MulticastImpl(); + public: + MulticastImpl(int min_receivers, const std::string& ifName, int portBase = 9000, + int bufSize = 8 * 1024 * 1024); + ~MulticastImpl(); - void startSender(); - void doTransfer(const uint8_t* buf, uint32_t len); + void startSender(); + void doTransfer(const uint8_t* buf, uint32_t len); - void startReceiver(); - void receive(messageqcpp::SBS obs); + void startReceiver(); + void receive(messageqcpp::SBS obs); - struct net_config fNet_config; - struct stat_config fStat_config; - struct client_config fClient_config; - std::string fIfName; - int fSock[3]; - participantsDb_t fDb; + struct net_config fNet_config; + struct stat_config fStat_config; + struct client_config fClient_config; + std::string fIfName; + int fSock[3]; + participantsDb_t fDb; }; -} +} // namespace multicast #endif - diff --git a/utils/multicast/mc_fifo.h b/utils/multicast/mc_fifo.h index c006ef60d..a9eccca9f 100644 --- a/utils/multicast/mc_fifo.h +++ b/utils/multicast/mc_fifo.h @@ -23,14 +23,14 @@ typedef struct fifo { - unsigned char* dataBuffer; - unsigned int dataBufSize; + unsigned char* dataBuffer; + unsigned int dataBufSize; - produconsum_t freeMemQueue; /* queue for free memory */ - produconsum_t data; /* queue for received data or data received - * from disk */ + produconsum_t freeMemQueue; /* queue for free memory */ + produconsum_t data; /* queue for received data or data received + * from disk */ - pthread_t thread; -}* fifo_t; + pthread_t thread; +} * fifo_t; #endif diff --git a/utils/multicast/multicast.cpp b/utils/multicast/multicast.cpp index 90bec6444..116f0382c 100644 --- a/utils/multicast/multicast.cpp +++ b/utils/multicast/multicast.cpp @@ -36,37 +36,35 @@ using namespace config; namespace multicast { - -Multicast::Multicast() : - fPMCount(1), - fIFName("eth0"), - fPortBase(9000), - fBufSize(8 * 1024 * 1024) +Multicast::Multicast() : fPMCount(1), fIFName("eth0"), fPortBase(9000), fBufSize(8 * 1024 * 1024) { - int tmp; - string stmp; + int tmp; + string stmp; - Config* cf = Config::makeConfig(); + Config* cf = Config::makeConfig(); - tmp = Config::fromText(cf->getConfig("PrimitiveServers", "Count")); + tmp = Config::fromText(cf->getConfig("PrimitiveServers", "Count")); - if (tmp > 0) fPMCount = tmp; + if (tmp > 0) + fPMCount = tmp; - stmp = cf->getConfig("Multicast", "Interface"); + stmp = cf->getConfig("Multicast", "Interface"); - if (!stmp.empty()) fIFName = stmp; + if (!stmp.empty()) + fIFName = stmp; - tmp = Config::fromText(cf->getConfig("Multicast", "PortBase")); + tmp = Config::fromText(cf->getConfig("Multicast", "PortBase")); - if (tmp > 0) fPortBase = tmp; + if (tmp > 0) + fPortBase = tmp; - tmp = Config::fromText(cf->getConfig("Multicast", "BufSize")); + tmp = Config::fromText(cf->getConfig("Multicast", "BufSize")); - if (tmp > 0) fBufSize = tmp; + if (tmp > 0) + fBufSize = tmp; } -MulticastReceiver::MulticastReceiver() : - fPimpl(0) +MulticastReceiver::MulticastReceiver() : fPimpl(0) { } @@ -76,12 +74,11 @@ MulticastReceiver::~MulticastReceiver() SBS MulticastReceiver::receive() { - throw runtime_error("Multicast is not available"); - return fByteStream; + throw runtime_error("Multicast is not available"); + return fByteStream; } -MulticastSender::MulticastSender() : - fPimpl(0) +MulticastSender::MulticastSender() : fPimpl(0) { } @@ -91,9 +88,9 @@ MulticastSender::~MulticastSender() void MulticastSender::send(const ByteStream& msg) { - throw runtime_error("Multicast is not available"); + throw runtime_error("Multicast is not available"); } -} //namespace multicast +} // namespace multicast -//vim:ts=4 sw=4: +// vim:ts=4 sw=4: diff --git a/utils/multicast/multicast.h b/utils/multicast/multicast.h index d468c1cde..18148e375 100644 --- a/utils/multicast/multicast.h +++ b/utils/multicast/multicast.h @@ -31,104 +31,101 @@ namespace multicast { - /** @brief MulticastReceive - * Wrapper for multicast proto - */ + * Wrapper for multicast proto + */ class MulticastImpl; class Multicast { -public: - /** @brief ctor - * Base class - */ - Multicast(); + public: + /** @brief ctor + * Base class + */ + Multicast(); - /** @brief dtor - */ - virtual ~Multicast() - { - destroy(); - } + /** @brief dtor + */ + virtual ~Multicast() + { + destroy(); + } - virtual void destroy() { } + virtual void destroy() + { + } - int PMCount() const - { - return fPMCount; - } - std::string iFName() const - { - return fIFName; - } - int portBase() const - { - return fPortBase; - } - int bufSize() const - { - return fBufSize; - } - -private: - int fPMCount; - std::string fIFName; - int fPortBase; - int fBufSize; + int PMCount() const + { + return fPMCount; + } + std::string iFName() const + { + return fIFName; + } + int portBase() const + { + return fPortBase; + } + int bufSize() const + { + return fBufSize; + } + private: + int fPMCount; + std::string fIFName; + int fPortBase; + int fBufSize; }; -class MulticastReceiver: public Multicast +class MulticastReceiver : public Multicast { -public: - /** @brief ctor - * - */ - MulticastReceiver(); + public: + /** @brief ctor + * + */ + MulticastReceiver(); - ~MulticastReceiver(); + ~MulticastReceiver(); - messageqcpp::SBS receive(); + messageqcpp::SBS receive(); -private: - // not copyable - MulticastReceiver(const MulticastReceiver& rhs); - MulticastReceiver& operator=(const MulticastReceiver& rhs); + private: + // not copyable + MulticastReceiver(const MulticastReceiver& rhs); + MulticastReceiver& operator=(const MulticastReceiver& rhs); - messageqcpp::SBS fByteStream; + messageqcpp::SBS fByteStream; - MulticastImpl* fPimpl; + MulticastImpl* fPimpl; }; - class MulticastSender : public Multicast { + public: + /** @brief ctor + * + */ + MulticastSender(); -public: - /** @brief ctor - * - */ - MulticastSender(); + ~MulticastSender(); - ~MulticastSender(); + /** @brief receive + * + * @param bytestream to send + */ + void send(const messageqcpp::ByteStream& bs); - /** @brief receive - * - * @param bytestream to send - */ - void send(const messageqcpp::ByteStream& bs); + private: + // Not copyable + MulticastSender(const MulticastSender& rhs); + MulticastSender& operator=(const MulticastSender& rhs); -private: - //Not copyable - MulticastSender(const MulticastSender& rhs); - MulticastSender& operator=(const MulticastSender& rhs); - - MulticastImpl* fPimpl; + MulticastImpl* fPimpl; }; -} //namespace - -#endif //MULTICAST_H +} // namespace multicast +#endif // MULTICAST_H diff --git a/utils/multicast/socklib.h b/utils/multicast/socklib.h index 28d2b8f4d..b16994209 100644 --- a/utils/multicast/socklib.h +++ b/utils/multicast/socklib.h @@ -19,8 +19,8 @@ #define SOCKLIB_H #ifndef UDPCAST_CONFIG_H -# define UDPCAST_CONFIG_H -# include "mcsconfig.h" +#define UDPCAST_CONFIG_H +#include "mcsconfig.h" #endif #include @@ -65,7 +65,7 @@ #endif #define RECEIVER_PORT(x) (x) -#define SENDER_PORT(x) ((x)+1) +#define SENDER_PORT(x) ((x) + 1) #define loseSendPacket udpc_loseSendPacket #define loseRecvPacket udpc_loseRecvPacket @@ -93,20 +93,20 @@ int RecvMsg(int s, struct msghdr* msg, int flags); struct net_if { - struct in_addr addr; - struct in_addr bcast; - const char* name; + struct in_addr addr; + struct in_addr bcast; + const char* name; #ifdef SIOCGIFINDEX - int index; + int index; #endif }; typedef struct net_if net_if_t; typedef enum addr_type_t { - ADDR_TYPE_UCAST, - ADDR_TYPE_MCAST, - ADDR_TYPE_BCAST + ADDR_TYPE_UCAST, + ADDR_TYPE_MCAST, + ADDR_TYPE_BCAST } addr_type_t; void doAutoRateLimit(int sock, int dir, int qsize, int size); @@ -117,14 +117,11 @@ void printMyIp(net_if_t* net_if); int getSendBuf(int sock); -#define SEND(s, msg, to) \ - doSend(s, &msg, sizeof(msg), &to) +#define SEND(s, msg, to) doSend(s, &msg, sizeof(msg), &to) -#define RECV(s, msg, from, portBase ) \ - doReceive((s), &msg, sizeof(msg), &from, (portBase) ) +#define RECV(s, msg, from, portBase) doReceive((s), &msg, sizeof(msg), &from, (portBase)) -#define BCAST_CONTROL(s, msg) \ - doSend(s, &msg, sizeof(msg), &net_config->controlMcastAddr) +#define BCAST_CONTROL(s, msg) doSend(s, &msg, sizeof(msg), &net_config->controlMcastAddr) void setIpFromString(struct sockaddr_in* addr, char* ip); @@ -136,31 +133,30 @@ int udpc_socklibFatal(int code); struct iovec { - void* iov_base; - int iov_len; + void* iov_base; + int iov_len; }; struct msghdr { - void* msg_name; - int msg_namelen; - struct iovec* msg_iov; - int msg_iovlen; - + void* msg_name; + int msg_namelen; + struct iovec* msg_iov; + int msg_iovlen; }; ssize_t sendmsg(int s, const struct msghdr* msg, int flags); -ssize_t recvmsg (int fd, struct msghdr* msg, int flags); +ssize_t recvmsg(int fd, struct msghdr* msg, int flags); -#define usleep(x) Sleep((x)/1000) -#define sleep(x) Sleep(1000L*(x)) +#define usleep(x) Sleep((x) / 1000) +#define sleep(x) Sleep(1000L * (x)) #endif /* __MINGW32__ */ static inline void initMsgHdr(struct msghdr* hdr) { #ifndef WINDOWS - hdr->msg_control = 0; - hdr->msg_controllen = 0; - hdr->msg_flags = 0; + hdr->msg_control = 0; + hdr->msg_controllen = 0; + hdr->msg_flags = 0; #endif } diff --git a/utils/multicast/statistics.h b/utils/multicast/statistics.h index 76d34d777..e84069f11 100644 --- a/utils/multicast/statistics.h +++ b/utils/multicast/statistics.h @@ -18,7 +18,6 @@ #ifndef STATISTICS_H #define STATISTICS_H - typedef struct receiver_stats* receiver_stats_t; typedef struct sender_stats* sender_stats_t; @@ -26,17 +25,15 @@ typedef struct sender_stats* sender_stats_t; #define receiverStatsStartTimer udpc_receiverStatsStartTimer #define displayReceiverStats udpc_displayReceiverStats -receiver_stats_t udpc_allocReadStats(int fd, long statPeriod, - int printUncompressedPos); +receiver_stats_t udpc_allocReadStats(int fd, long statPeriod, int printUncompressedPos); void udpc_receiverStatsStartTimer(receiver_stats_t); void udpc_displayReceiverStats(receiver_stats_t, int isFinal); #define allocSenderStats udpc_allocSenderStats #define displaySenderStats udpc_displaySenderStats -sender_stats_t udpc_allocSenderStats(int fd, FILE* logfile, long bwPeriod, - long statPeriod, int printUncompressedPos); -void udpc_displaySenderStats(sender_stats_t, int blockSize, int sliceSize, - int isFinal); +sender_stats_t udpc_allocSenderStats(int fd, FILE* logfile, long bwPeriod, long statPeriod, + int printUncompressedPos); +void udpc_displaySenderStats(sender_stats_t, int blockSize, int sliceSize, int isFinal); #endif diff --git a/utils/multicast/threads.h b/utils/multicast/threads.h index e41e4b94b..c0e06db38 100644 --- a/utils/multicast/threads.h +++ b/utils/multicast/threads.h @@ -30,129 +30,115 @@ typedef HANDLE pthread_cond_t; struct timespec { - unsigned long tv_sec; - unsigned long tv_nsec; + unsigned long tv_sec; + unsigned long tv_nsec; }; -static inline int pthread_create(pthread_t* thread, void* dummy1, - LPTHREAD_START_ROUTINE start_routine, +static inline int pthread_create(pthread_t* thread, void* dummy1, LPTHREAD_START_ROUTINE start_routine, void* arg) { - /* Start thread ... - * see http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dllproc/base/createthread.asp - */ - *thread = CreateThread(NULL, /* lpThreadAttributes */ - 0, /* dwStackSize */ - start_routine, - arg, /* lpParameter */ - 0, /* dwCreationFlags */ - NULL /* lpThreadId */); - return *thread != NULL ? 0 : -1; + /* Start thread ... + * see http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dllproc/base/createthread.asp + */ + *thread = CreateThread(NULL, /* lpThreadAttributes */ + 0, /* dwStackSize */ + start_routine, arg, /* lpParameter */ + 0, /* dwCreationFlags */ + NULL /* lpThreadId */); + return *thread != NULL ? 0 : -1; } static inline int pthread_join(pthread_t th, void** thread_return) { - return WaitForSingleObject(th, INFINITE) == WAIT_OBJECT_0 ? 0 : -1; + return WaitForSingleObject(th, INFINITE) == WAIT_OBJECT_0 ? 0 : -1; } -static inline int pthread_mutex_init(pthread_mutex_t* mutex, void* dummy) +static inline int pthread_mutex_init(pthread_mutex_t* mutex, void* dummy) { - InitializeCriticalSection(mutex); + InitializeCriticalSection(mutex); + return 0; +} + +static inline int pthread_mutex_lock(pthread_mutex_t* mutex) +{ + EnterCriticalSection(mutex); + return 0; +} + +static inline int pthread_mutex_unlock(pthread_mutex_t* mutex) +{ + LeaveCriticalSection(mutex); + return 0; +} + +static inline int pthread_cond_init(pthread_cond_t* cond, void* dummy) +{ + *cond = CreateEvent(NULL, TRUE, TRUE, NULL); + + if (*cond == NULL) + return -1; + else return 0; } -static inline int pthread_mutex_lock(pthread_mutex_t* mutex) +static inline int pthread_cond_signal(pthread_cond_t* cond) { - EnterCriticalSection(mutex); - return 0; + return SetEvent(*cond) ? 0 : -1; } -static inline int pthread_mutex_unlock(pthread_mutex_t* mutex) +static inline int pthread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex) { - LeaveCriticalSection(mutex); - return 0; -} - - -static inline int pthread_cond_init(pthread_cond_t* cond, void* dummy) -{ - *cond = CreateEvent(NULL, TRUE, TRUE, NULL); - - if (*cond == NULL) - return -1; - else - return 0; -} - -static inline int pthread_cond_signal(pthread_cond_t* cond) -{ - return SetEvent(*cond) ? 0 : -1; -} - -static inline int pthread_cond_wait(pthread_cond_t* cond, - pthread_mutex_t* mutex) -{ - int r; - ResetEvent(*cond); - LeaveCriticalSection(mutex); - r = WaitForSingleObject(*cond, INFINITE) == WAIT_OBJECT_0 ? 0 : -1; - EnterCriticalSection(mutex); - return r; + int r; + ResetEvent(*cond); + LeaveCriticalSection(mutex); + r = WaitForSingleObject(*cond, INFINITE) == WAIT_OBJECT_0 ? 0 : -1; + EnterCriticalSection(mutex); + return r; } static inline void pthread_cancel(pthread_t* thread) { - TerminateThread(thread, 0); + TerminateThread(thread, 0); } #define ETIMEDOUT -2 -#define MILLION 1000000 +#define MILLION 1000000 #define BILLION 1000000000 -static inline int pthread_cond_timedwait(pthread_cond_t* cond, - pthread_mutex_t* mutex, - struct timespec* ts) +static inline int pthread_cond_timedwait(pthread_cond_t* cond, pthread_mutex_t* mutex, struct timespec* ts) { - int r; - struct timeval tv; - long delta; + int r; + struct timeval tv; + long delta; - gettimeofday(&tv, NULL); + gettimeofday(&tv, NULL); - delta = (ts->tv_sec - tv.tv_sec) * 1000 + - (ts->tv_nsec / BILLION - tv.tv_usec / MILLION); + delta = (ts->tv_sec - tv.tv_sec) * 1000 + (ts->tv_nsec / BILLION - tv.tv_usec / MILLION); - if (delta < 0) - delta = 0; + if (delta < 0) + delta = 0; - ResetEvent(*cond); - LeaveCriticalSection(mutex); + ResetEvent(*cond); + LeaveCriticalSection(mutex); - switch (WaitForSingleObject(*cond, delta )) - { - case WAIT_OBJECT_0: - r = 0; - break; + switch (WaitForSingleObject(*cond, delta)) + { + case WAIT_OBJECT_0: r = 0; break; - case WAIT_TIMEOUT: - r = ETIMEDOUT; - break; + case WAIT_TIMEOUT: r = ETIMEDOUT; break; - default: - r = -1; - break; - } + default: r = -1; break; + } - EnterCriticalSection(mutex); - return r; + EnterCriticalSection(mutex); + return r; } #define THREAD_RETURN DWORD WINAPI - #else /* __MINGW32__ */ #include -#define THREAD_RETURN void * +#define THREAD_RETURN void* #endif /* __MINGW32__ */ #endif diff --git a/utils/multicast/udp-sender.h b/utils/multicast/udp-sender.h index 526f556b4..1539a25da 100644 --- a/utils/multicast/udp-sender.h +++ b/utils/multicast/udp-sender.h @@ -37,20 +37,18 @@ int openFile(struct disk_config* config); int openPipe(struct disk_config* config, int in, int* pid); int localReader(struct fifo* fifo, int in); -#define BCAST_DATA(s, msg) \ - doSend(s, &msg, sizeof(msg), &net_config->dataMcastAddr) - +#define BCAST_DATA(s, msg) doSend(s, &msg, sizeof(msg), &net_config->dataMcastAddr) /** * "switched network" mode: server already starts sending next slice before * first one is acknowledged. Do not use on old coax networks */ -#define FLAG_SN 0x0001 +#define FLAG_SN 0x0001 /** * "not switched network" mode: network is known not to be switched */ -#define FLAG_NOTSN 0x0002 +#define FLAG_NOTSN 0x0002 /** * Asynchronous mode: do not any confirmation at all from clients. @@ -58,14 +56,12 @@ int localReader(struct fifo* fifo, int in); */ #define FLAG_ASYNC 0x0004 - /** * Point-to-point transmission mode: use unicast in the (frequent) * special case where there is only one receiver. */ #define FLAG_POINTOPOINT 0x0008 - /** * Do automatic rate limitation by monitoring socket's send buffer * size. Not very useful, as this still doesn't protect against the @@ -94,7 +90,6 @@ int localReader(struct fifo* fifo, int in); */ #define FLAG_NOPOINTOPOINT 0x0040 - /* * Don't ask for keyboard input on sender end. */ diff --git a/utils/multicast/udpc-protoc.h b/utils/multicast/udpc-protoc.h index 6e298c70c..24e647b14 100644 --- a/utils/multicast/udpc-protoc.h +++ b/utils/multicast/udpc-protoc.h @@ -28,28 +28,28 @@ */ enum opCode { - /* Receiver to sender */ + /* Receiver to sender */ - CMD_OK, /* all is ok, no need to retransmit anything */ - CMD_RETRANSMIT, /* receiver asks for some data to be retransmitted */ - CMD_GO, /* receiver tells server to start */ - CMD_CONNECT_REQ, /* receiver tries to find out server's address */ - CMD_DISCONNECT, /* receiver wants to disconnect itself */ + CMD_OK, /* all is ok, no need to retransmit anything */ + CMD_RETRANSMIT, /* receiver asks for some data to be retransmitted */ + CMD_GO, /* receiver tells server to start */ + CMD_CONNECT_REQ, /* receiver tries to find out server's address */ + CMD_DISCONNECT, /* receiver wants to disconnect itself */ - CMD_UNUSED, /* obsolete version of CMD_HELLO, dating back to the - * time when we had little endianness (PC). This - * opcode contained a long unnoticed bug with parsing of - * blocksize */ + CMD_UNUSED, /* obsolete version of CMD_HELLO, dating back to the + * time when we had little endianness (PC). This + * opcode contained a long unnoticed bug with parsing of + * blocksize */ - /* Sender to receiver */ - CMD_REQACK, /* server request acknowledgments from receiver */ - CMD_CONNECT_REPLY, /* receiver tries to find out server's address */ + /* Sender to receiver */ + CMD_REQACK, /* server request acknowledgments from receiver */ + CMD_CONNECT_REPLY, /* receiver tries to find out server's address */ - CMD_DATA, /* a block of data */ - CMD_FEC, /* a forward-error-correction block */ + CMD_DATA, /* a block of data */ + CMD_FEC, /* a forward-error-correction block */ - CMD_HELLO_NEW, /* sender says he's up */ - CMD_HELLO_STREAMING, /* retransmitted hello during streaming mode */ + CMD_HELLO_NEW, /* sender says he's up */ + CMD_HELLO_STREAMING, /* retransmitted hello during streaming mode */ }; /* Sender says he's up. This is not in the enum with the others, @@ -63,114 +63,110 @@ enum opCode struct connectReq { - unsigned short opCode; - short reserved; - int capabilities; - unsigned int rcvbuf; + unsigned short opCode; + short reserved; + int capabilities; + unsigned int rcvbuf; }; struct retransmit { - unsigned short opCode; - short reserved; - int sliceNo; - int rxmit; - unsigned char map[MAX_SLICE_SIZE / BITS_PER_CHAR]; + unsigned short opCode; + short reserved; + int sliceNo; + int rxmit; + unsigned char map[MAX_SLICE_SIZE / BITS_PER_CHAR]; }; struct ok { - unsigned short opCode; - short reserved; - int sliceNo; + unsigned short opCode; + short reserved; + int sliceNo; } ok; union message { + unsigned short opCode; + struct ok ok; + + struct retransmit retransmit; + + struct connectReq connectReq; + + struct go + { unsigned short opCode; - struct ok ok; + short reserved; + } go; - struct retransmit retransmit; - - struct connectReq connectReq; - - struct go - { - unsigned short opCode; - short reserved; - } go; - - struct disconnect - { - unsigned short opCode; - short reserved; - } disconnect; + struct disconnect + { + unsigned short opCode; + short reserved; + } disconnect; }; - - struct connectReply { - unsigned short opCode; - short reserved; - int clNr; - int blockSize; - int capabilities; - unsigned char mcastAddr[16]; /* provide enough place for IPV6 */ + unsigned short opCode; + short reserved; + int clNr; + int blockSize; + int capabilities; + unsigned char mcastAddr[16]; /* provide enough place for IPV6 */ }; struct hello { - unsigned short opCode; - short reserved; - int capabilities; - unsigned char mcastAddr[16]; /* provide enough place for IPV6 */ - short blockSize; + unsigned short opCode; + short reserved; + int capabilities; + unsigned char mcastAddr[16]; /* provide enough place for IPV6 */ + short blockSize; }; union serverControlMsg { - unsigned short opCode; - short reserved; - struct hello hello; - struct connectReply connectReply; - + unsigned short opCode; + short reserved; + struct hello hello; + struct connectReply connectReply; }; - struct dataBlock { - unsigned short opCode; - short reserved; - int sliceNo; - unsigned short blockNo; - unsigned short reserved2; - int bytes; + unsigned short opCode; + short reserved; + int sliceNo; + unsigned short blockNo; + unsigned short reserved2; + int bytes; }; struct fecBlock { - unsigned short opCode; - short stripes; - int sliceNo; - unsigned short blockNo; - unsigned short reserved2; - int bytes; + unsigned short opCode; + short stripes; + int sliceNo; + unsigned short blockNo; + unsigned short reserved2; + int bytes; }; struct reqack { - unsigned short opCode; - short reserved; - int sliceNo; - int bytes; - int rxmit; + unsigned short opCode; + short reserved; + int sliceNo; + int bytes; + int rxmit; }; union serverDataMsg { - unsigned short opCode; - struct reqack reqack; - struct dataBlock dataBlock; - struct fecBlock fecBlock; + unsigned short opCode; + struct reqack reqack; + struct dataBlock dataBlock; + struct fecBlock fecBlock; }; /* ============================================ @@ -204,14 +200,8 @@ union serverDataMsg #define CAP_ASYNC 0x0020 /* Sender currently supports CAPABILITIES and MULTICAST */ -#define SENDER_CAPABILITIES ( \ - CAP_NEW_GEN | \ - CAP_BIG_ENDIAN) - - -#define RECEIVER_CAPABILITIES ( \ - CAP_NEW_GEN | \ - CAP_BIG_ENDIAN) +#define SENDER_CAPABILITIES (CAP_NEW_GEN | CAP_BIG_ENDIAN) +#define RECEIVER_CAPABILITIES (CAP_NEW_GEN | CAP_BIG_ENDIAN) #endif diff --git a/utils/multicast/udpcast.h b/utils/multicast/udpcast.h index ac31abbc2..f6e920496 100644 --- a/utils/multicast/udpcast.h +++ b/utils/multicast/udpcast.h @@ -26,12 +26,10 @@ #define BITS_PER_INT (sizeof(int) * 8) #define BITS_PER_CHAR 8 +#define MAP_ZERO(l, map) (memset(map, 0, ((l) + BITS_PER_INT - 1) / BIT_PER_INT)) +#define BZERO(data) (memset((void*)&data, 0, sizeof(data))) -#define MAP_ZERO(l, map) (memset(map, 0, ((l) + BITS_PER_INT - 1)/ BIT_PER_INT)) -#define BZERO(data) (memset((void *)&data, 0, sizeof(data))) - - -#define RDATABUFSIZE (2*(MAX_SLICE_SIZE + 1)* MAX_BLOCK_SIZE) +#define RDATABUFSIZE (2 * (MAX_SLICE_SIZE + 1) * MAX_BLOCK_SIZE) #define DATABUFSIZE (RDATABUFSIZE + 4096 - RDATABUFSIZE % 4096) @@ -46,99 +44,99 @@ int udpc_waitForProcess(int pid, const char* message); struct disk_config { - int origOutFile; - const char* fileName; - char* pipeName; - int flags; + int origOutFile; + const char* fileName; + char* pipeName; + int flags; - struct timeval stats_last_printed; + struct timeval stats_last_printed; }; #define MAX_GOVERNORS 10 struct net_config { - net_if_t* net_if; /* Network interface (eth0, isdn0, etc.) on which to - * multicast */ - int portBase; /* Port base */ - int blockSize; - int sliceSize; - struct sockaddr_in controlMcastAddr; - struct sockaddr_in dataMcastAddr; - const char* mcastRdv; - int ttl; - int nrGovernors; - struct rateGovernor_t* rateGovernor[MAX_GOVERNORS]; - void* rateGovernorData[MAX_GOVERNORS]; - /*int async;*/ - /*int pointopoint;*/ - struct timeval ref_tv; + net_if_t* net_if; /* Network interface (eth0, isdn0, etc.) on which to + * multicast */ + int portBase; /* Port base */ + int blockSize; + int sliceSize; + struct sockaddr_in controlMcastAddr; + struct sockaddr_in dataMcastAddr; + const char* mcastRdv; + int ttl; + int nrGovernors; + struct rateGovernor_t* rateGovernor[MAX_GOVERNORS]; + void* rateGovernorData[MAX_GOVERNORS]; + /*int async;*/ + /*int pointopoint;*/ + struct timeval ref_tv; - enum discovery - { - DSC_DOUBLING, - DSC_REDUCING - } discovery; + enum discovery + { + DSC_DOUBLING, + DSC_REDUCING + } discovery; - /* int autoRate; do queue watching using TIOCOUTQ, to avoid overruns */ + /* int autoRate; do queue watching using TIOCOUTQ, to avoid overruns */ - int flags; /* non-capability command line flags */ - int capabilities; + int flags; /* non-capability command line flags */ + int capabilities; - int min_slice_size; - int default_slice_size; - int max_slice_size; - unsigned int rcvbuf; + int min_slice_size; + int default_slice_size; + int max_slice_size; + unsigned int rcvbuf; - int rexmit_hello_interval; /* retransmission interval between hello's. - * If 0, hello message won't be retransmitted - */ - int autostart; /* autostart after that many retransmits */ + int rexmit_hello_interval; /* retransmission interval between hello's. + * If 0, hello message won't be retransmitted + */ + int autostart; /* autostart after that many retransmits */ - int requestedBufSize; /* requested receiver buffer */ + int requestedBufSize; /* requested receiver buffer */ - /* sender-specific parameters */ - int min_receivers; - int max_receivers_wait; - int min_receivers_wait; + /* sender-specific parameters */ + int min_receivers; + int max_receivers_wait; + int min_receivers_wait; - int retriesUntilDrop; + int retriesUntilDrop; - /* receiver-specif parameters */ - int exitWait; /* How many milliseconds to wait on program exit */ + /* receiver-specif parameters */ + int exitWait; /* How many milliseconds to wait on program exit */ - int startTimeout; /* Timeout at start */ + int startTimeout; /* Timeout at start */ - /* FEC config */ + /* FEC config */ #ifdef BB_FEATURE_UDPCAST_FEC - int fec_redundancy; /* how much fec blocks are added per group */ - int fec_stripesize; /* size of FEC group */ - int fec_stripes; /* number of FEC stripes per slice */ + int fec_redundancy; /* how much fec blocks are added per group */ + int fec_stripesize; /* size of FEC group */ + int fec_stripes; /* number of FEC stripes per slice */ #endif - int rehelloOffset; /* how far before end will rehello packet will - be retransmitted */ + int rehelloOffset; /* how far before end will rehello packet will + be retransmitted */ }; struct stat_config { - FILE* log; /* Log file for statistics */ - long bwPeriod; /* How often are bandwidth estimations logged? */ + FILE* log; /* Log file for statistics */ + long bwPeriod; /* How often are bandwidth estimations logged? */ - int statPeriod; - int printUncompressedPos; + int statPeriod; + int printUncompressedPos; }; #define NR_CLIENT_SOCKS 4 struct client_config { - int socks[NR_CLIENT_SOCKS]; - struct sockaddr_in serverAddr; - int clientNumber; - int isStarted; - pthread_t thread; - int sender_is_newgen; + int socks[NR_CLIENT_SOCKS]; + struct sockaddr_in serverAddr; + int clientNumber; + int isStarted; + pthread_t thread; + int sender_is_newgen; }; void* rgInitGovernor(struct net_config* cfg, struct rateGovernor_t* gov); @@ -156,7 +154,7 @@ int udpc_shouldPrintUncompressedPos(int deflt, int fd, int pipe); #define DEFLT_STAT_PERIOD 500000 #ifndef DEBUG -# define DEBUG 0 +#define DEBUG 0 #endif #endif diff --git a/utils/multicast/util.h b/utils/multicast/util.h index c6c7864e8..8f6988eaf 100644 --- a/utils/multicast/util.h +++ b/utils/multicast/util.h @@ -23,12 +23,11 @@ #define MALLOC(type) ((type*)calloc(1, sizeof(type))) /* bitmap manipulation */ -#define BITS_PER_ITEM(map) (sizeof(map[0])*8) -#define MASK(pos,map) (1 << ((pos) % (BITS_PER_ITEM(map)))) -#define POS(pos,map) ((pos) / BITS_PER_ITEM(map)) -#define SET_BIT(x, map) (map[POS(x,map)] |= MASK(x,map)) -#define CLR_BIT(x, map) (map[POS(x,map)] &= ~MASK(x,map)) -#define BIT_ISSET(x, map) (map[POS(x,map)] & MASK(x,map)) - +#define BITS_PER_ITEM(map) (sizeof(map[0]) * 8) +#define MASK(pos, map) (1 << ((pos) % (BITS_PER_ITEM(map)))) +#define POS(pos, map) ((pos) / BITS_PER_ITEM(map)) +#define SET_BIT(x, map) (map[POS(x, map)] |= MASK(x, map)) +#define CLR_BIT(x, map) (map[POS(x, map)] &= ~MASK(x, map)) +#define BIT_ISSET(x, map) (map[POS(x, map)] & MASK(x, map)) #endif diff --git a/utils/querystats/querystats.cpp b/utils/querystats/querystats.cpp index 4810c1650..360e774ce 100644 --- a/utils/querystats/querystats.cpp +++ b/utils/querystats/querystats.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: querystats.cpp 4028 2013-08-02 18:49:00Z zzhu $ -* -* -***********************************************************************/ + * $Id: querystats.cpp 4028 2013-08-02 18:49:00Z zzhu $ + * + * + ***********************************************************************/ #define PREFER_MY_CONFIG_H #include @@ -45,112 +45,111 @@ using namespace logging; namespace querystats { - const string SCHEMA = "infinidb_querystats"; QueryStats::QueryStats() { - reset(); + reset(); } void QueryStats::reset() { - fMaxMemPct = 0; - fNumFiles = 0; - fFileBytes = 0; - fPhyIO = 0; - fCacheIO = 0; - fMsgRcvCnt = 0; - fCPBlocksSkipped = 0; - fMsgBytesIn = 0; - fMsgBytesOut = 0; - fRows = 0; - fStartTime = 0; - fEndTime = 0; - fErrorNo = 0; - fBlocksChanged = 0; - fSessionID = (uint64_t) - 1; - fStartTimeStr.clear(); - fEndTimeStr.clear(); - fQueryType.clear(); - fQuery.clear(); - fHost.clear(); - fUser.clear(); - fPriority.clear(); + fMaxMemPct = 0; + fNumFiles = 0; + fFileBytes = 0; + fPhyIO = 0; + fCacheIO = 0; + fMsgRcvCnt = 0; + fCPBlocksSkipped = 0; + fMsgBytesIn = 0; + fMsgBytesOut = 0; + fRows = 0; + fStartTime = 0; + fEndTime = 0; + fErrorNo = 0; + fBlocksChanged = 0; + fSessionID = (uint64_t)-1; + fStartTimeStr.clear(); + fEndTimeStr.clear(); + fQueryType.clear(); + fQuery.clear(); + fHost.clear(); + fUser.clear(); + fPriority.clear(); } void QueryStats::serialize(ByteStream& b) { - b << (uint64_t)fSessionID; - b << (uint64_t)fMaxMemPct; - b << (uint64_t)fNumFiles; - b << (uint64_t)fFileBytes; - b << (uint64_t)fPhyIO; - b << (uint64_t)fCacheIO; - b << (uint64_t)fMsgRcvCnt; - b << (uint64_t)fCPBlocksSkipped; - b << (uint64_t)fMsgBytesIn; - b << (uint64_t)fMsgBytesOut; - b << (uint64_t)fRows; - b << fStartTimeStr; - b << fEndTimeStr; - b << (uint64_t)fErrorNo; - b << (uint64_t)fBlocksChanged; - b << fQuery; - b << fQueryType; - b << fHost; - b << fUser; - b << fPriority; + b << (uint64_t)fSessionID; + b << (uint64_t)fMaxMemPct; + b << (uint64_t)fNumFiles; + b << (uint64_t)fFileBytes; + b << (uint64_t)fPhyIO; + b << (uint64_t)fCacheIO; + b << (uint64_t)fMsgRcvCnt; + b << (uint64_t)fCPBlocksSkipped; + b << (uint64_t)fMsgBytesIn; + b << (uint64_t)fMsgBytesOut; + b << (uint64_t)fRows; + b << fStartTimeStr; + b << fEndTimeStr; + b << (uint64_t)fErrorNo; + b << (uint64_t)fBlocksChanged; + b << fQuery; + b << fQueryType; + b << fHost; + b << fUser; + b << fPriority; } // unserialize new stats and keep the stats set in this. void QueryStats::unserialize(ByteStream& b) { - uint64_t temp = 0; - string str; - b >> (uint64_t&)temp; - fSessionID = (fSessionID == (uint64_t) - 1 ? temp : fSessionID); - b >> (uint64_t&)temp; - fMaxMemPct = (fMaxMemPct == 0 ? temp : fMaxMemPct); - b >> (uint64_t&)temp; - fNumFiles = (fNumFiles == 0 ? temp : fNumFiles); - b >> (uint64_t&)temp; - fFileBytes = (fFileBytes == 0 ? temp : fFileBytes); - b >> (uint64_t&)temp; - fPhyIO = (fPhyIO == 0 ? temp : fPhyIO); - b >> (uint64_t&)temp; - fCacheIO = (fCacheIO == 0 ? temp : fCacheIO); - b >> (uint64_t&)temp; - fMsgRcvCnt = (fMsgRcvCnt == 0 ? temp : fMsgRcvCnt); - b >> (uint64_t&)temp; - fCPBlocksSkipped = (fCPBlocksSkipped == 0 ? temp : fCPBlocksSkipped); - b >> (uint64_t&)temp; - fMsgBytesIn = (fMsgBytesIn == 0 ? temp : fMsgBytesIn); - b >> (uint64_t&)temp; - fMsgBytesOut = (fMsgBytesOut == 0 ? temp : fMsgBytesOut); - b >> (uint64_t&)temp; - fRows = (fRows == 0 ? temp : fRows); + uint64_t temp = 0; + string str; + b >> (uint64_t&)temp; + fSessionID = (fSessionID == (uint64_t)-1 ? temp : fSessionID); + b >> (uint64_t&)temp; + fMaxMemPct = (fMaxMemPct == 0 ? temp : fMaxMemPct); + b >> (uint64_t&)temp; + fNumFiles = (fNumFiles == 0 ? temp : fNumFiles); + b >> (uint64_t&)temp; + fFileBytes = (fFileBytes == 0 ? temp : fFileBytes); + b >> (uint64_t&)temp; + fPhyIO = (fPhyIO == 0 ? temp : fPhyIO); + b >> (uint64_t&)temp; + fCacheIO = (fCacheIO == 0 ? temp : fCacheIO); + b >> (uint64_t&)temp; + fMsgRcvCnt = (fMsgRcvCnt == 0 ? temp : fMsgRcvCnt); + b >> (uint64_t&)temp; + fCPBlocksSkipped = (fCPBlocksSkipped == 0 ? temp : fCPBlocksSkipped); + b >> (uint64_t&)temp; + fMsgBytesIn = (fMsgBytesIn == 0 ? temp : fMsgBytesIn); + b >> (uint64_t&)temp; + fMsgBytesOut = (fMsgBytesOut == 0 ? temp : fMsgBytesOut); + b >> (uint64_t&)temp; + fRows = (fRows == 0 ? temp : fRows); - b >> str; - fStartTimeStr = (fStartTimeStr.empty() ? str : fStartTimeStr); - b >> str; - fEndTimeStr = (fEndTimeStr.empty() ? str : fEndTimeStr); + b >> str; + fStartTimeStr = (fStartTimeStr.empty() ? str : fStartTimeStr); + b >> str; + fEndTimeStr = (fEndTimeStr.empty() ? str : fEndTimeStr); - b >> (uint64_t&)temp; - fErrorNo = (fErrorNo == 0 ? temp : fErrorNo); - b >> (uint64_t&)temp; - fBlocksChanged = (fBlocksChanged == 0 ? temp : fBlocksChanged); + b >> (uint64_t&)temp; + fErrorNo = (fErrorNo == 0 ? temp : fErrorNo); + b >> (uint64_t&)temp; + fBlocksChanged = (fBlocksChanged == 0 ? temp : fBlocksChanged); - b >> str; - fQuery = (fQuery.empty() ? str : fQuery); - b >> str; - fQueryType = (fQueryType.empty() ? str : fQueryType); - b >> str; - fHost = (fHost.empty() ? str : fHost); - b >> str; - fUser = (fUser.empty() ? str : fUser); - b >> str; - fPriority = (fPriority.empty() ? str : fPriority); + b >> str; + fQuery = (fQuery.empty() ? str : fQuery); + b >> str; + fQueryType = (fQueryType.empty() ? str : fQueryType); + b >> str; + fHost = (fHost.empty() ? str : fHost); + b >> str; + fUser = (fUser.empty() ? str : fUser); + b >> str; + fPriority = (fPriority.empty() ? str : fPriority); } /** @@ -179,131 +178,124 @@ void QueryStats::unserialize(ByteStream& b) */ void QueryStats::insert() { - ResourceManager* rm = ResourceManager::instance(); + ResourceManager* rm = ResourceManager::instance(); - // check if query stats is enabled in Columnstore.xml - if (!rm->queryStatsEnabled()) - return; + // check if query stats is enabled in Columnstore.xml + if (!rm->queryStatsEnabled()) + return; - // get configure for every query to allow only changing of connect info - int ret; - string host, user, pwd; - uint32_t port; + // get configure for every query to allow only changing of connect info + int ret; + string host, user, pwd; + uint32_t port; - if (rm->getMysqldInfo(host, user, pwd, port) == false) - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_CROSS_ENGINE_CONFIG), - ERR_CROSS_ENGINE_CONFIG); + if (rm->getMysqldInfo(host, user, pwd, port) == false) + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_CROSS_ENGINE_CONFIG), ERR_CROSS_ENGINE_CONFIG); - // insert stats to querystats table - utils::LibMySQL mysql; + // insert stats to querystats table + utils::LibMySQL mysql; - ret = mysql.init(host.c_str(), port, user.c_str(), pwd.c_str(), - SCHEMA.c_str()); + ret = mysql.init(host.c_str(), port, user.c_str(), pwd.c_str(), SCHEMA.c_str()); - if (ret != 0) - mysql.handleMySqlError(mysql.getError().c_str(), ret); + if (ret != 0) + mysql.handleMySqlError(mysql.getError().c_str(), ret); - // escape quote characters - boost::scoped_array query(new char[fQuery.length() * 2 + 1]); - mysql_real_escape_string(mysql.getMySqlCon(), query.get(), fQuery.c_str(), fQuery.length()); + // escape quote characters + boost::scoped_array query(new char[fQuery.length() * 2 + 1]); + mysql_real_escape_string(mysql.getMySqlCon(), query.get(), fQuery.c_str(), fQuery.length()); - ostringstream insert; - insert << "insert into querystats values (0, "; - insert << fSessionID << ", "; - insert << "'" << fHost << "', "; - insert << "'" << fUser << "', "; - insert << "'" << fPriority << "', "; - insert << "'" << fQueryType << "', "; - insert << "'" << query.get() << "', "; - insert << "'" << fStartTimeStr << "', "; - insert << "'" << fEndTimeStr << "', "; - insert << fRows << ", "; - insert << fErrorNo << ", "; - insert << fPhyIO << ", "; - insert << fCacheIO << ", "; - insert << fMsgRcvCnt << ", "; - insert << fCPBlocksSkipped << ", "; - insert << fMsgBytesIn << ", "; - insert << fMsgBytesOut << ", "; - insert << fMaxMemPct << ", "; - insert << fBlocksChanged << ", "; - insert << fNumFiles << ", "; - insert << fFileBytes << ")"; // the last 2 fields are not populated yet + ostringstream insert; + insert << "insert into querystats values (0, "; + insert << fSessionID << ", "; + insert << "'" << fHost << "', "; + insert << "'" << fUser << "', "; + insert << "'" << fPriority << "', "; + insert << "'" << fQueryType << "', "; + insert << "'" << query.get() << "', "; + insert << "'" << fStartTimeStr << "', "; + insert << "'" << fEndTimeStr << "', "; + insert << fRows << ", "; + insert << fErrorNo << ", "; + insert << fPhyIO << ", "; + insert << fCacheIO << ", "; + insert << fMsgRcvCnt << ", "; + insert << fCPBlocksSkipped << ", "; + insert << fMsgBytesIn << ", "; + insert << fMsgBytesOut << ", "; + insert << fMaxMemPct << ", "; + insert << fBlocksChanged << ", "; + insert << fNumFiles << ", "; + insert << fFileBytes << ")"; // the last 2 fields are not populated yet - ret = mysql.run(insert.str().c_str(), false); + ret = mysql.run(insert.str().c_str(), false); - if (ret != 0) - mysql.handleMySqlError(mysql.getError().c_str(), ret); + if (ret != 0) + mysql.handleMySqlError(mysql.getError().c_str(), ret); } uint32_t QueryStats::userPriority(string _host, const string _user) { - // priority has been set already - if (!fPriority.empty()) - return fPriorityLevel; + // priority has been set already + if (!fPriority.empty()) + return fPriorityLevel; - ResourceManager rm; - fPriorityLevel = DEFAULT_USER_PRIORITY_LEVEL; + ResourceManager rm; + fPriorityLevel = DEFAULT_USER_PRIORITY_LEVEL; + fPriority = DEFAULT_USER_PRIORITY; + + // check if query stats is enabled in Columnstore.xml + if (!rm.userPriorityEnabled()) + { fPriority = DEFAULT_USER_PRIORITY; + fPriorityLevel = DEFAULT_USER_PRIORITY_LEVEL; + return fPriorityLevel; + } - // check if query stats is enabled in Columnstore.xml - if (!rm.userPriorityEnabled()) - { - fPriority = DEFAULT_USER_PRIORITY; - fPriorityLevel = DEFAULT_USER_PRIORITY_LEVEL; - return fPriorityLevel; - } + string host, user, pwd; + uint32_t port; - string host, user, pwd; - uint32_t port; + // get configure for every query to allow only changing of connect info + if (rm.getMysqldInfo(host, user, pwd, port) == false) + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_CROSS_ENGINE_CONFIG), ERR_CROSS_ENGINE_CONFIG); - // get configure for every query to allow only changing of connect info - if (rm.getMysqldInfo(host, user, pwd, port) == false) - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_CROSS_ENGINE_CONFIG), - ERR_CROSS_ENGINE_CONFIG); + // get user priority + utils::LibMySQL mysql; - // get user priority - utils::LibMySQL mysql; + int ret; + ret = mysql.init(host.c_str(), port, user.c_str(), pwd.c_str(), SCHEMA.c_str()); - int ret; - ret = mysql.init(host.c_str(), port, user.c_str(), pwd.c_str(), - SCHEMA.c_str()); + if (ret != 0) + mysql.handleMySqlError(mysql.getError().c_str(), ret); - if (ret != 0) - mysql.handleMySqlError(mysql.getError().c_str(), ret); + // get the part of host string befor ':' if there is. + size_t pos = _host.find(':', 0); - // get the part of host string befor ':' if there is. - size_t pos = _host.find(':', 0); + if (pos != string::npos) + _host = _host.substr(0, pos); - if (pos != string::npos) - _host = _host.substr(0, pos); - - ostringstream query; - query << "select a.priority, priority_level from user_priority a, priority b where \ + ostringstream query; + query << "select a.priority, priority_level from user_priority a, priority b where \ upper(case when INSTR(host, ':') = 0 \ then host \ else SUBSTR(host, 1, INSTR(host, ':')-1 ) \ end)=upper('" - << _host - << "') and upper(user)=upper('" - << _user - << "') and upper(a.priority) = upper(b.priority)"; + << _host << "') and upper(user)=upper('" << _user << "') and upper(a.priority) = upper(b.priority)"; - ret = mysql.run(query.str().c_str()); + ret = mysql.run(query.str().c_str()); - if (ret != 0) - mysql.handleMySqlError(mysql.getError().c_str(), ret); + if (ret != 0) + mysql.handleMySqlError(mysql.getError().c_str(), ret); - char** rowIn; - rowIn = mysql.nextRow(); + char** rowIn; + rowIn = mysql.nextRow(); - if (rowIn) - { - fPriority = rowIn[0]; - fPriorityLevel = atoi(rowIn[1]); - } + if (rowIn) + { + fPriority = rowIn[0]; + fPriorityLevel = atoi(rowIn[1]); + } - return fPriorityLevel; + return fPriorityLevel; } -} +} // namespace querystats diff --git a/utils/querystats/querystats.h b/utils/querystats/querystats.h index 42d54ef77..60f547a3b 100644 --- a/utils/querystats/querystats.h +++ b/utils/querystats/querystats.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: querystats.h 4028 2013-08-02 18:49:00Z zzhu $ -* -* -***********************************************************************/ + * $Id: querystats.h 4028 2013-08-02 18:49:00Z zzhu $ + * + * + ***********************************************************************/ #ifndef QUERYSTATS_H_ #define QUERYSTATS_H_ @@ -35,108 +35,109 @@ namespace querystats { -const uint32_t DEFAULT_USER_PRIORITY_LEVEL = 33; //Low +const uint32_t DEFAULT_USER_PRIORITY_LEVEL = 33; // Low const std::string DEFAULT_USER_PRIORITY = "LOW"; struct QueryStats { - uint64_t fMaxMemPct; // peak memory percentage used during a query - uint64_t fNumFiles; // number of temp files used for a query - uint64_t fFileBytes; // number of bytes in temp files - uint64_t fPhyIO; // physical block count for a query - uint64_t fCacheIO; // cache block count for a query - uint64_t fMsgRcvCnt; // msg (block) receive count for a query - uint64_t fCPBlocksSkipped; // Casual Partition blks skipped for a query - uint64_t fMsgBytesIn; // number of input msg bytes for a query - uint64_t fMsgBytesOut; // number of output msg bytes for a query - uint64_t fRows; // number of rows return/affected - time_t fStartTime; // query start time - time_t fEndTime; // query end time - std::string fStartTimeStr; // query start time in YYYY-MM-DD HH:MM:SS format - std::string fEndTimeStr; // query end time in YYYY-MM-DD HH:MM:SS format - uint64_t fErrorNo; // query error number. 0 if succeed - uint64_t fBlocksChanged; // blocks changed for DML queries - uint64_t fSessionID; // session id of this query - std::string fQueryType; // query type as "select", "update", "delete" ... - std::string fQuery; // query text - std::string fUser; // user - std::string fHost; // host - std::string fPriority; // priority - uint32_t fPriorityLevel; // priority level + uint64_t fMaxMemPct; // peak memory percentage used during a query + uint64_t fNumFiles; // number of temp files used for a query + uint64_t fFileBytes; // number of bytes in temp files + uint64_t fPhyIO; // physical block count for a query + uint64_t fCacheIO; // cache block count for a query + uint64_t fMsgRcvCnt; // msg (block) receive count for a query + uint64_t fCPBlocksSkipped; // Casual Partition blks skipped for a query + uint64_t fMsgBytesIn; // number of input msg bytes for a query + uint64_t fMsgBytesOut; // number of output msg bytes for a query + uint64_t fRows; // number of rows return/affected + time_t fStartTime; // query start time + time_t fEndTime; // query end time + std::string fStartTimeStr; // query start time in YYYY-MM-DD HH:MM:SS format + std::string fEndTimeStr; // query end time in YYYY-MM-DD HH:MM:SS format + uint64_t fErrorNo; // query error number. 0 if succeed + uint64_t fBlocksChanged; // blocks changed for DML queries + uint64_t fSessionID; // session id of this query + std::string fQueryType; // query type as "select", "update", "delete" ... + std::string fQuery; // query text + std::string fUser; // user + std::string fHost; // host + std::string fPriority; // priority + uint32_t fPriorityLevel; // priority level + QueryStats(); + ~QueryStats() + { + } - QueryStats(); - ~QueryStats() {} + /** + reset the stats fields. + */ + void reset(); - /** - reset the stats fields. - */ - void reset(); + // only += for fields that make sense for subquery + QueryStats operator+=(const QueryStats& rhs) + { + fNumFiles += rhs.fNumFiles; + fFileBytes += rhs.fFileBytes; + fPhyIO += rhs.fPhyIO; + fCacheIO += rhs.fCacheIO; + fMsgRcvCnt += rhs.fMsgRcvCnt; + fCPBlocksSkipped += rhs.fCPBlocksSkipped; + fMsgBytesIn += rhs.fMsgBytesIn; + fMsgBytesOut += rhs.fMsgBytesOut; + fBlocksChanged += rhs.fBlocksChanged; - // only += for fields that make sense for subquery - QueryStats operator+=(const QueryStats& rhs) - { - fNumFiles += rhs.fNumFiles; - fFileBytes += rhs.fFileBytes; - fPhyIO += rhs.fPhyIO; - fCacheIO += rhs.fCacheIO; - fMsgRcvCnt += rhs.fMsgRcvCnt; - fCPBlocksSkipped += rhs.fCPBlocksSkipped; - fMsgBytesIn += rhs.fMsgBytesIn; - fMsgBytesOut += rhs.fMsgBytesOut; - fBlocksChanged += rhs.fBlocksChanged; + return *this; + } - return *this; - } + void setStartTime() + { + time(&fStartTime); + char buffer[80]; + struct tm timeinfo; + localtime_r(&fStartTime, &timeinfo); + strftime(buffer, 80, "%Y-%m-%d %H:%M:%S", &timeinfo); + fStartTimeStr = buffer; + } - void setStartTime() - { - time(&fStartTime); - char buffer [80]; - struct tm timeinfo; - localtime_r(&fStartTime, &timeinfo); - strftime(buffer, 80, "%Y-%m-%d %H:%M:%S", &timeinfo); - fStartTimeStr = buffer; - } + void setEndTime() + { + time(&fEndTime); + char buffer[80]; + struct tm timeinfo; + localtime_r(&fEndTime, &timeinfo); + strftime(buffer, 80, "%Y-%m-%d %H:%M:%S", &timeinfo); + fEndTimeStr = buffer; + } - void setEndTime() - { - time(&fEndTime); - char buffer [80]; - struct tm timeinfo; - localtime_r(&fEndTime, &timeinfo); - strftime(buffer, 80, "%Y-%m-%d %H:%M:%S", &timeinfo); - fEndTimeStr = buffer; - } + // joblist::ResourceManager* rm() { return fRm; } + // void rm(joblist::ResourceManager* rm) + //{ + // delete fRm; + // fRm = rm; + // fIsOwnRm = false; + //} - //joblist::ResourceManager* rm() { return fRm; } - //void rm(joblist::ResourceManager* rm) - //{ - // delete fRm; - // fRm = rm; - // fIsOwnRm = false; - //} + void serialize(messageqcpp::ByteStream& bs); + // unserialize, and merge new stats to this when handling stats comming from different processes. + void unserialize(messageqcpp::ByteStream& bs); - void serialize(messageqcpp::ByteStream& bs); - // unserialize, and merge new stats to this when handling stats comming from different processes. - void unserialize(messageqcpp::ByteStream& bs); + /** + insert to query stats table. The table is pre-created and the number/type of + columns are fixed. If changing the table defintion, this function needs to be + modified accordingly. + */ + void insert(); - /** - insert to query stats table. The table is pre-created and the number/type of - columns are fixed. If changing the table defintion, this function needs to be - modified accordingly. - */ - void insert(); + /* User mysql API to query priority table and get this user's assigned priority */ + uint32_t userPriority(std::string host, const std::string user); - /* User mysql API to query priority table and get this user's assigned priority */ - uint32_t userPriority(std::string host, const std::string user); - -private: -// default okay -// QueryStats(const QueryStats& stats); -// QueryStats& operator=(const QueryStats&); + private: + // default okay + // QueryStats(const QueryStats& stats); + // QueryStats& operator=(const QueryStats&); }; -} +} // namespace querystats #endif diff --git a/utils/querytele/QueryTeleService.cpp b/utils/querytele/QueryTeleService.cpp index 5e29c764b..d1ef258d8 100644 --- a/utils/querytele/QueryTeleService.cpp +++ b/utils/querytele/QueryTeleService.cpp @@ -8,867 +8,848 @@ namespace querytele { - uint32_t QueryTeleService_postQuery_args::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) - { - xfer += this->query.read(iprot); - this->__isset.query = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - default: - xfer += iprot->skip(ftype); - break; - } - - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) + { + xfer += this->query.read(iprot); + this->__isset.query = true; + } + else + { + xfer += iprot->skip(ftype); + } - return xfer; + break; + + default: xfer += iprot->skip(ftype); break; + } + + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postQuery_args::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTeleService_postQuery_args"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postQuery_args"); - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->query.write(oprot); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->query.write(oprot); + xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postQuery_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTeleService_postQuery_pargs"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postQuery_pargs"); - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += (*(this->query)).write(oprot); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->query)).write(oprot); + xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postQuery_result::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - xfer += iprot->skip(ftype); - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } - return xfer; + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postQuery_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; - uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postQuery_result"); - xfer += oprot->writeStructBegin("QueryTeleService_postQuery_result"); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postQuery_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - xfer += iprot->skip(ftype); - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } - return xfer; + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postStep_args::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) - { - xfer += this->query.read(iprot); - this->__isset.query = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - default: - xfer += iprot->skip(ftype); - break; - } - - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) + { + xfer += this->query.read(iprot); + this->__isset.query = true; + } + else + { + xfer += iprot->skip(ftype); + } - return xfer; + break; + + default: xfer += iprot->skip(ftype); break; + } + + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postStep_args::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTeleService_postStep_args"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postStep_args"); - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->query.write(oprot); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->query.write(oprot); + xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postStep_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTeleService_postStep_pargs"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postStep_pargs"); - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += (*(this->query)).write(oprot); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->query)).write(oprot); + xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postStep_result::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - xfer += iprot->skip(ftype); - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } - return xfer; + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postStep_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; - uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postStep_result"); - xfer += oprot->writeStructBegin("QueryTeleService_postStep_result"); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postStep_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - xfer += iprot->skip(ftype); - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } - return xfer; + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postImport_args::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRUCT) - { - xfer += this->query.read(iprot); - this->__isset.query = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - default: - xfer += iprot->skip(ftype); - break; - } - - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) + { + xfer += this->query.read(iprot); + this->__isset.query = true; + } + else + { + xfer += iprot->skip(ftype); + } - return xfer; + break; + + default: xfer += iprot->skip(ftype); break; + } + + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postImport_args::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTeleService_postImport_args"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postImport_args"); - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += this->query.write(oprot); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->query.write(oprot); + xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postImport_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTeleService_postImport_pargs"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postImport_pargs"); - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); - xfer += (*(this->query)).write(oprot); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->query)).write(oprot); + xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postImport_result::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - xfer += iprot->skip(ftype); - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } - return xfer; + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTeleService_postImport_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; - uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTeleService_postImport_result"); - xfer += oprot->writeStructBegin("QueryTeleService_postImport_result"); - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } uint32_t QueryTeleService_postImport_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - xfer += iprot->skip(ftype); - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } - return xfer; + xfer += iprot->readStructEnd(); + + return xfer; } void QueryTeleServiceClient::postQuery(const QueryTele& query) { - send_postQuery(query); - recv_postQuery(); + send_postQuery(query); + recv_postQuery(); } void QueryTeleServiceClient::send_postQuery(const QueryTele& query) { - int32_t cseqid = 0; - oprot_->writeMessageBegin("postQuery", ::apache::thrift::protocol::T_CALL, cseqid); + int32_t cseqid = 0; + oprot_->writeMessageBegin("postQuery", ::apache::thrift::protocol::T_CALL, cseqid); - QueryTeleService_postQuery_pargs args; - args.query = &query; - args.write(oprot_); + QueryTeleService_postQuery_pargs args; + args.query = &query; + args.write(oprot_); - oprot_->writeMessageEnd(); - oprot_->getTransport()->writeEnd(); - oprot_->getTransport()->flush(); + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); } void QueryTeleServiceClient::recv_postQuery() { + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; - int32_t rseqid = 0; - std::string fname; - ::apache::thrift::protocol::TMessageType mtype; + iprot_->readMessageBegin(fname, mtype, rseqid); - iprot_->readMessageBegin(fname, mtype, rseqid); - - if (mtype == ::apache::thrift::protocol::T_EXCEPTION) - { - ::apache::thrift::TApplicationException x; - x.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - throw x; - } - - if (mtype != ::apache::thrift::protocol::T_REPLY) - { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - - if (fname.compare("postQuery") != 0) - { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - - QueryTeleService_postQuery_presult result; - result.read(iprot_); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) + { + ::apache::thrift::TApplicationException x; + x.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); + throw x; + } - return; + if (mtype != ::apache::thrift::protocol::T_REPLY) + { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + + if (fname.compare("postQuery") != 0) + { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + + QueryTeleService_postQuery_presult result; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + return; } void QueryTeleServiceClient::postStep(const StepTele& query) { - send_postStep(query); - recv_postStep(); + send_postStep(query); + recv_postStep(); } void QueryTeleServiceClient::send_postStep(const StepTele& query) { - int32_t cseqid = 0; - oprot_->writeMessageBegin("postStep", ::apache::thrift::protocol::T_CALL, cseqid); + int32_t cseqid = 0; + oprot_->writeMessageBegin("postStep", ::apache::thrift::protocol::T_CALL, cseqid); - QueryTeleService_postStep_pargs args; - args.query = &query; - args.write(oprot_); + QueryTeleService_postStep_pargs args; + args.query = &query; + args.write(oprot_); - oprot_->writeMessageEnd(); - oprot_->getTransport()->writeEnd(); - oprot_->getTransport()->flush(); + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); } void QueryTeleServiceClient::recv_postStep() { + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; - int32_t rseqid = 0; - std::string fname; - ::apache::thrift::protocol::TMessageType mtype; + iprot_->readMessageBegin(fname, mtype, rseqid); - iprot_->readMessageBegin(fname, mtype, rseqid); - - if (mtype == ::apache::thrift::protocol::T_EXCEPTION) - { - ::apache::thrift::TApplicationException x; - x.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - throw x; - } - - if (mtype != ::apache::thrift::protocol::T_REPLY) - { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - - if (fname.compare("postStep") != 0) - { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - - QueryTeleService_postStep_presult result; - result.read(iprot_); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) + { + ::apache::thrift::TApplicationException x; + x.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); + throw x; + } - return; + if (mtype != ::apache::thrift::protocol::T_REPLY) + { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + + if (fname.compare("postStep") != 0) + { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + + QueryTeleService_postStep_presult result; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + return; } void QueryTeleServiceClient::postImport(const ImportTele& query) { - send_postImport(query); - recv_postImport(); + send_postImport(query); + recv_postImport(); } void QueryTeleServiceClient::send_postImport(const ImportTele& query) { - int32_t cseqid = 0; - oprot_->writeMessageBegin("postImport", ::apache::thrift::protocol::T_CALL, cseqid); + int32_t cseqid = 0; + oprot_->writeMessageBegin("postImport", ::apache::thrift::protocol::T_CALL, cseqid); - QueryTeleService_postImport_pargs args; - args.query = &query; - args.write(oprot_); + QueryTeleService_postImport_pargs args; + args.query = &query; + args.write(oprot_); - oprot_->writeMessageEnd(); - oprot_->getTransport()->writeEnd(); - oprot_->getTransport()->flush(); + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); } void QueryTeleServiceClient::recv_postImport() { + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; - int32_t rseqid = 0; - std::string fname; - ::apache::thrift::protocol::TMessageType mtype; + iprot_->readMessageBegin(fname, mtype, rseqid); - iprot_->readMessageBegin(fname, mtype, rseqid); - - if (mtype == ::apache::thrift::protocol::T_EXCEPTION) - { - ::apache::thrift::TApplicationException x; - x.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - throw x; - } - - if (mtype != ::apache::thrift::protocol::T_REPLY) - { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - - if (fname.compare("postImport") != 0) - { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - - QueryTeleService_postImport_presult result; - result.read(iprot_); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) + { + ::apache::thrift::TApplicationException x; + x.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); + throw x; + } - return; + if (mtype != ::apache::thrift::protocol::T_REPLY) + { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + + if (fname.compare("postImport") != 0) + { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + + QueryTeleService_postImport_presult result; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + return; } -bool QueryTeleServiceProcessor::dispatchCall(::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, void* callContext) +bool QueryTeleServiceProcessor::dispatchCall(::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, + const std::string& fname, int32_t seqid, void* callContext) { - ProcessMap::iterator pfn; - pfn = processMap_.find(fname); + ProcessMap::iterator pfn; + pfn = processMap_.find(fname); - if (pfn == processMap_.end()) - { - iprot->skip(::apache::thrift::protocol::T_STRUCT); - iprot->readMessageEnd(); - iprot->getTransport()->readEnd(); - ::apache::thrift::TApplicationException x(::apache::thrift::TApplicationException::UNKNOWN_METHOD, "Invalid method name: '" + fname + "'"); - oprot->writeMessageBegin(fname, ::apache::thrift::protocol::T_EXCEPTION, seqid); - x.write(oprot); - oprot->writeMessageEnd(); - oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - return true; - } - - (this->*(pfn->second))(seqid, iprot, oprot, callContext); + if (pfn == processMap_.end()) + { + iprot->skip(::apache::thrift::protocol::T_STRUCT); + iprot->readMessageEnd(); + iprot->getTransport()->readEnd(); + ::apache::thrift::TApplicationException x(::apache::thrift::TApplicationException::UNKNOWN_METHOD, + "Invalid method name: '" + fname + "'"); + oprot->writeMessageBegin(fname, ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); return true; + } + + (this->*(pfn->second))(seqid, iprot, oprot, callContext); + return true; } -void QueryTeleServiceProcessor::process_postQuery(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +void QueryTeleServiceProcessor::process_postQuery(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, + void* callContext) { - void* ctx = NULL; + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) + { + ctx = this->eventHandler_->getContext("QueryTeleService.postQuery", callContext); + } + + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, + "QueryTeleService.postQuery"); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->preRead(ctx, "QueryTeleService.postQuery"); + } + + QueryTeleService_postQuery_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->postRead(ctx, "QueryTeleService.postQuery", bytes); + } + + QueryTeleService_postQuery_result result; + + try + { + iface_->postQuery(args.query); + } + catch (const std::exception& e) + { if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("QueryTeleService.postQuery", callContext); + this->eventHandler_->handlerError(ctx, "QueryTeleService.postQuery"); } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "QueryTeleService.postQuery"); - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->preRead(ctx, "QueryTeleService.postQuery"); - } - - QueryTeleService_postQuery_args args; - args.read(iprot); - iprot->readMessageEnd(); - uint32_t bytes = iprot->getTransport()->readEnd(); - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->postRead(ctx, "QueryTeleService.postQuery", bytes); - } - - QueryTeleService_postQuery_result result; - - try - { - iface_->postQuery(args.query); - } - catch (const std::exception& e) - { - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->handlerError(ctx, "QueryTeleService.postQuery"); - } - - ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("postQuery", ::apache::thrift::protocol::T_EXCEPTION, seqid); - x.write(oprot); - oprot->writeMessageEnd(); - oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - return; - } - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->preWrite(ctx, "QueryTeleService.postQuery"); - } - - oprot->writeMessageBegin("postQuery", ::apache::thrift::protocol::T_REPLY, seqid); - result.write(oprot); + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("postQuery", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); oprot->writeMessageEnd(); - bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->writeEnd(); oprot->getTransport()->flush(); + return; + } - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->postWrite(ctx, "QueryTeleService.postQuery", bytes); - } + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->preWrite(ctx, "QueryTeleService.postQuery"); + } + + oprot->writeMessageBegin("postQuery", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->postWrite(ctx, "QueryTeleService.postQuery", bytes); + } } -void QueryTeleServiceProcessor::process_postStep(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +void QueryTeleServiceProcessor::process_postStep(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, + void* callContext) { - void* ctx = NULL; + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) + { + ctx = this->eventHandler_->getContext("QueryTeleService.postStep", callContext); + } + + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "QueryTeleService.postStep"); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->preRead(ctx, "QueryTeleService.postStep"); + } + + QueryTeleService_postStep_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->postRead(ctx, "QueryTeleService.postStep", bytes); + } + + QueryTeleService_postStep_result result; + + try + { + iface_->postStep(args.query); + } + catch (const std::exception& e) + { if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("QueryTeleService.postStep", callContext); + this->eventHandler_->handlerError(ctx, "QueryTeleService.postStep"); } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "QueryTeleService.postStep"); - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->preRead(ctx, "QueryTeleService.postStep"); - } - - QueryTeleService_postStep_args args; - args.read(iprot); - iprot->readMessageEnd(); - uint32_t bytes = iprot->getTransport()->readEnd(); - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->postRead(ctx, "QueryTeleService.postStep", bytes); - } - - QueryTeleService_postStep_result result; - - try - { - iface_->postStep(args.query); - } - catch (const std::exception& e) - { - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->handlerError(ctx, "QueryTeleService.postStep"); - } - - ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("postStep", ::apache::thrift::protocol::T_EXCEPTION, seqid); - x.write(oprot); - oprot->writeMessageEnd(); - oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - return; - } - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->preWrite(ctx, "QueryTeleService.postStep"); - } - - oprot->writeMessageBegin("postStep", ::apache::thrift::protocol::T_REPLY, seqid); - result.write(oprot); + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("postStep", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); oprot->writeMessageEnd(); - bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->writeEnd(); oprot->getTransport()->flush(); + return; + } - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->postWrite(ctx, "QueryTeleService.postStep", bytes); - } + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->preWrite(ctx, "QueryTeleService.postStep"); + } + + oprot->writeMessageBegin("postStep", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->postWrite(ctx, "QueryTeleService.postStep", bytes); + } } -void QueryTeleServiceProcessor::process_postImport(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +void QueryTeleServiceProcessor::process_postImport(int32_t seqid, + ::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, + void* callContext) { - void* ctx = NULL; + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) + { + ctx = this->eventHandler_->getContext("QueryTeleService.postImport", callContext); + } + + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, + "QueryTeleService.postImport"); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->preRead(ctx, "QueryTeleService.postImport"); + } + + QueryTeleService_postImport_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->postRead(ctx, "QueryTeleService.postImport", bytes); + } + + QueryTeleService_postImport_result result; + + try + { + iface_->postImport(args.query); + } + catch (const std::exception& e) + { if (this->eventHandler_.get() != NULL) { - ctx = this->eventHandler_->getContext("QueryTeleService.postImport", callContext); + this->eventHandler_->handlerError(ctx, "QueryTeleService.postImport"); } - ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "QueryTeleService.postImport"); - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->preRead(ctx, "QueryTeleService.postImport"); - } - - QueryTeleService_postImport_args args; - args.read(iprot); - iprot->readMessageEnd(); - uint32_t bytes = iprot->getTransport()->readEnd(); - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->postRead(ctx, "QueryTeleService.postImport", bytes); - } - - QueryTeleService_postImport_result result; - - try - { - iface_->postImport(args.query); - } - catch (const std::exception& e) - { - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->handlerError(ctx, "QueryTeleService.postImport"); - } - - ::apache::thrift::TApplicationException x(e.what()); - oprot->writeMessageBegin("postImport", ::apache::thrift::protocol::T_EXCEPTION, seqid); - x.write(oprot); - oprot->writeMessageEnd(); - oprot->getTransport()->writeEnd(); - oprot->getTransport()->flush(); - return; - } - - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->preWrite(ctx, "QueryTeleService.postImport"); - } - - oprot->writeMessageBegin("postImport", ::apache::thrift::protocol::T_REPLY, seqid); - result.write(oprot); + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("postImport", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); oprot->writeMessageEnd(); - bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->writeEnd(); oprot->getTransport()->flush(); + return; + } - if (this->eventHandler_.get() != NULL) - { - this->eventHandler_->postWrite(ctx, "QueryTeleService.postImport", bytes); - } + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->preWrite(ctx, "QueryTeleService.postImport"); + } + + oprot->writeMessageBegin("postImport", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) + { + this->eventHandler_->postWrite(ctx, "QueryTeleService.postImport", bytes); + } } -::boost::shared_ptr< ::apache::thrift::TProcessor > QueryTeleServiceProcessorFactory::getProcessor(const ::apache::thrift::TConnectionInfo& connInfo) +::boost::shared_ptr< ::apache::thrift::TProcessor> QueryTeleServiceProcessorFactory::getProcessor( + const ::apache::thrift::TConnectionInfo& connInfo) { - ::apache::thrift::ReleaseHandler< QueryTeleServiceIfFactory > cleanup(handlerFactory_); - ::boost::shared_ptr< QueryTeleServiceIf > handler(handlerFactory_->getHandler(connInfo), cleanup); - ::boost::shared_ptr< ::apache::thrift::TProcessor > processor(new QueryTeleServiceProcessor(handler)); - return processor; + ::apache::thrift::ReleaseHandler cleanup(handlerFactory_); + ::boost::shared_ptr handler(handlerFactory_->getHandler(connInfo), cleanup); + ::boost::shared_ptr< ::apache::thrift::TProcessor> processor(new QueryTeleServiceProcessor(handler)); + return processor; } -} // namespace - +} // namespace querytele diff --git a/utils/querytele/QueryTeleService.h b/utils/querytele/QueryTeleService.h index 9222d20e8..f3e79cdeb 100644 --- a/utils/querytele/QueryTeleService.h +++ b/utils/querytele/QueryTeleService.h @@ -12,491 +12,504 @@ namespace querytele { - class QueryTeleServiceIf { -public: - virtual ~QueryTeleServiceIf() {} - virtual void postQuery(const QueryTele& query) = 0; - virtual void postStep(const StepTele& query) = 0; - virtual void postImport(const ImportTele& query) = 0; + public: + virtual ~QueryTeleServiceIf() + { + } + virtual void postQuery(const QueryTele& query) = 0; + virtual void postStep(const StepTele& query) = 0; + virtual void postImport(const ImportTele& query) = 0; }; class QueryTeleServiceIfFactory { -public: - typedef QueryTeleServiceIf Handler; + public: + typedef QueryTeleServiceIf Handler; - virtual ~QueryTeleServiceIfFactory() {} + virtual ~QueryTeleServiceIfFactory() + { + } - virtual QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) = 0; - virtual void releaseHandler(QueryTeleServiceIf* /* handler */) = 0; + virtual QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) = 0; + virtual void releaseHandler(QueryTeleServiceIf* /* handler */) = 0; }; class QueryTeleServiceIfSingletonFactory : virtual public QueryTeleServiceIfFactory { -public: - QueryTeleServiceIfSingletonFactory(const boost::shared_ptr& iface) : iface_(iface) {} - virtual ~QueryTeleServiceIfSingletonFactory() {} + public: + QueryTeleServiceIfSingletonFactory(const boost::shared_ptr& iface) : iface_(iface) + { + } + virtual ~QueryTeleServiceIfSingletonFactory() + { + } - virtual QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo&) - { - return iface_.get(); - } - virtual void releaseHandler(QueryTeleServiceIf* /* handler */) {} + virtual QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo&) + { + return iface_.get(); + } + virtual void releaseHandler(QueryTeleServiceIf* /* handler */) + { + } -protected: - boost::shared_ptr iface_; + protected: + boost::shared_ptr iface_; }; class QueryTeleServiceNull : virtual public QueryTeleServiceIf { -public: - virtual ~QueryTeleServiceNull() {} - void postQuery(const QueryTele& /* query */) - { - return; - } - void postStep(const StepTele& /* query */) - { - return; - } - void postImport(const ImportTele& /* query */) - { - return; - } + public: + virtual ~QueryTeleServiceNull() + { + } + void postQuery(const QueryTele& /* query */) + { + return; + } + void postStep(const StepTele& /* query */) + { + return; + } + void postImport(const ImportTele& /* query */) + { + return; + } }; typedef struct _QueryTeleService_postQuery_args__isset { - _QueryTeleService_postQuery_args__isset() : query(false) {} - bool query; + _QueryTeleService_postQuery_args__isset() : query(false) + { + } + bool query; } _QueryTeleService_postQuery_args__isset; class QueryTeleService_postQuery_args { -public: + public: + QueryTeleService_postQuery_args() + { + } - QueryTeleService_postQuery_args() - { - } + virtual ~QueryTeleService_postQuery_args() throw() + { + } - virtual ~QueryTeleService_postQuery_args() throw() {} + QueryTele query; - QueryTele query; + _QueryTeleService_postQuery_args__isset __isset; - _QueryTeleService_postQuery_args__isset __isset; + void __set_query(const QueryTele& val) + { + query = val; + } - void __set_query(const QueryTele& val) - { - query = val; - } + bool operator==(const QueryTeleService_postQuery_args& rhs) const + { + if (!(query == rhs.query)) + return false; - bool operator == (const QueryTeleService_postQuery_args& rhs) const - { - if (!(query == rhs.query)) - return false; + return true; + } + bool operator!=(const QueryTeleService_postQuery_args& rhs) const + { + return !(*this == rhs); + } - return true; - } - bool operator != (const QueryTeleService_postQuery_args& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTeleService_postQuery_args& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + bool operator<(const QueryTeleService_postQuery_args&) const; + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postQuery_pargs { -public: + public: + virtual ~QueryTeleService_postQuery_pargs() throw() + { + } + const QueryTele* query; - virtual ~QueryTeleService_postQuery_pargs() throw() {} - - const QueryTele* query; - - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postQuery_result { -public: + public: + QueryTeleService_postQuery_result() + { + } - QueryTeleService_postQuery_result() - { - } + virtual ~QueryTeleService_postQuery_result() throw() + { + } - virtual ~QueryTeleService_postQuery_result() throw() {} + bool operator==(const QueryTeleService_postQuery_result& /* rhs */) const + { + return true; + } + bool operator!=(const QueryTeleService_postQuery_result& rhs) const + { + return !(*this == rhs); + } + bool operator<(const QueryTeleService_postQuery_result&) const; - bool operator == (const QueryTeleService_postQuery_result& /* rhs */) const - { - return true; - } - bool operator != (const QueryTeleService_postQuery_result& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTeleService_postQuery_result& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postQuery_presult { -public: - - - virtual ~QueryTeleService_postQuery_presult() throw() {} - - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + public: + virtual ~QueryTeleService_postQuery_presult() throw() + { + } + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); }; typedef struct _QueryTeleService_postStep_args__isset { - _QueryTeleService_postStep_args__isset() : query(false) {} - bool query; + _QueryTeleService_postStep_args__isset() : query(false) + { + } + bool query; } _QueryTeleService_postStep_args__isset; class QueryTeleService_postStep_args { -public: + public: + QueryTeleService_postStep_args() + { + } - QueryTeleService_postStep_args() - { - } + virtual ~QueryTeleService_postStep_args() throw() + { + } - virtual ~QueryTeleService_postStep_args() throw() {} + StepTele query; - StepTele query; + _QueryTeleService_postStep_args__isset __isset; - _QueryTeleService_postStep_args__isset __isset; + void __set_query(const StepTele& val) + { + query = val; + } - void __set_query(const StepTele& val) - { - query = val; - } + bool operator==(const QueryTeleService_postStep_args& rhs) const + { + if (!(query == rhs.query)) + return false; - bool operator == (const QueryTeleService_postStep_args& rhs) const - { - if (!(query == rhs.query)) - return false; + return true; + } + bool operator!=(const QueryTeleService_postStep_args& rhs) const + { + return !(*this == rhs); + } - return true; - } - bool operator != (const QueryTeleService_postStep_args& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTeleService_postStep_args& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + bool operator<(const QueryTeleService_postStep_args&) const; + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postStep_pargs { -public: + public: + virtual ~QueryTeleService_postStep_pargs() throw() + { + } + const StepTele* query; - virtual ~QueryTeleService_postStep_pargs() throw() {} - - const StepTele* query; - - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postStep_result { -public: + public: + QueryTeleService_postStep_result() + { + } - QueryTeleService_postStep_result() - { - } + virtual ~QueryTeleService_postStep_result() throw() + { + } - virtual ~QueryTeleService_postStep_result() throw() {} + bool operator==(const QueryTeleService_postStep_result& /* rhs */) const + { + return true; + } + bool operator!=(const QueryTeleService_postStep_result& rhs) const + { + return !(*this == rhs); + } + bool operator<(const QueryTeleService_postStep_result&) const; - bool operator == (const QueryTeleService_postStep_result& /* rhs */) const - { - return true; - } - bool operator != (const QueryTeleService_postStep_result& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTeleService_postStep_result& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postStep_presult { -public: - - - virtual ~QueryTeleService_postStep_presult() throw() {} - - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + public: + virtual ~QueryTeleService_postStep_presult() throw() + { + } + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); }; typedef struct _QueryTeleService_postImport_args__isset { - _QueryTeleService_postImport_args__isset() : query(false) {} - bool query; + _QueryTeleService_postImport_args__isset() : query(false) + { + } + bool query; } _QueryTeleService_postImport_args__isset; class QueryTeleService_postImport_args { -public: + public: + QueryTeleService_postImport_args() + { + } - QueryTeleService_postImport_args() - { - } + virtual ~QueryTeleService_postImport_args() throw() + { + } - virtual ~QueryTeleService_postImport_args() throw() {} + ImportTele query; - ImportTele query; + _QueryTeleService_postImport_args__isset __isset; - _QueryTeleService_postImport_args__isset __isset; + void __set_query(const ImportTele& val) + { + query = val; + } - void __set_query(const ImportTele& val) - { - query = val; - } + bool operator==(const QueryTeleService_postImport_args& rhs) const + { + if (!(query == rhs.query)) + return false; - bool operator == (const QueryTeleService_postImport_args& rhs) const - { - if (!(query == rhs.query)) - return false; + return true; + } + bool operator!=(const QueryTeleService_postImport_args& rhs) const + { + return !(*this == rhs); + } - return true; - } - bool operator != (const QueryTeleService_postImport_args& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTeleService_postImport_args& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + bool operator<(const QueryTeleService_postImport_args&) const; + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postImport_pargs { -public: + public: + virtual ~QueryTeleService_postImport_pargs() throw() + { + } + const ImportTele* query; - virtual ~QueryTeleService_postImport_pargs() throw() {} - - const ImportTele* query; - - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postImport_result { -public: + public: + QueryTeleService_postImport_result() + { + } - QueryTeleService_postImport_result() - { - } + virtual ~QueryTeleService_postImport_result() throw() + { + } - virtual ~QueryTeleService_postImport_result() throw() {} + bool operator==(const QueryTeleService_postImport_result& /* rhs */) const + { + return true; + } + bool operator!=(const QueryTeleService_postImport_result& rhs) const + { + return !(*this == rhs); + } + bool operator<(const QueryTeleService_postImport_result&) const; - bool operator == (const QueryTeleService_postImport_result& /* rhs */) const - { - return true; - } - bool operator != (const QueryTeleService_postImport_result& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTeleService_postImport_result& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; - class QueryTeleService_postImport_presult { -public: - - - virtual ~QueryTeleService_postImport_presult() throw() {} - - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + public: + virtual ~QueryTeleService_postImport_presult() throw() + { + } + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); }; class QueryTeleServiceClient : virtual public QueryTeleServiceIf { -public: - QueryTeleServiceClient(boost::shared_ptr< ::apache::thrift::protocol::TProtocol> prot) : - piprot_(prot), - poprot_(prot) - { - iprot_ = prot.get(); - oprot_ = prot.get(); - } - QueryTeleServiceClient(boost::shared_ptr< ::apache::thrift::protocol::TProtocol> iprot, boost::shared_ptr< ::apache::thrift::protocol::TProtocol> oprot) : - piprot_(iprot), - poprot_(oprot) - { - iprot_ = iprot.get(); - oprot_ = oprot.get(); - } - boost::shared_ptr< ::apache::thrift::protocol::TProtocol> getInputProtocol() - { - return piprot_; - } - boost::shared_ptr< ::apache::thrift::protocol::TProtocol> getOutputProtocol() - { - return poprot_; - } - void postQuery(const QueryTele& query); - void send_postQuery(const QueryTele& query); - void recv_postQuery(); - void postStep(const StepTele& query); - void send_postStep(const StepTele& query); - void recv_postStep(); - void postImport(const ImportTele& query); - void send_postImport(const ImportTele& query); - void recv_postImport(); -protected: - boost::shared_ptr< ::apache::thrift::protocol::TProtocol> piprot_; - boost::shared_ptr< ::apache::thrift::protocol::TProtocol> poprot_; - ::apache::thrift::protocol::TProtocol* iprot_; - ::apache::thrift::protocol::TProtocol* oprot_; + public: + QueryTeleServiceClient(boost::shared_ptr< ::apache::thrift::protocol::TProtocol> prot) + : piprot_(prot), poprot_(prot) + { + iprot_ = prot.get(); + oprot_ = prot.get(); + } + QueryTeleServiceClient(boost::shared_ptr< ::apache::thrift::protocol::TProtocol> iprot, + boost::shared_ptr< ::apache::thrift::protocol::TProtocol> oprot) + : piprot_(iprot), poprot_(oprot) + { + iprot_ = iprot.get(); + oprot_ = oprot.get(); + } + boost::shared_ptr< ::apache::thrift::protocol::TProtocol> getInputProtocol() + { + return piprot_; + } + boost::shared_ptr< ::apache::thrift::protocol::TProtocol> getOutputProtocol() + { + return poprot_; + } + void postQuery(const QueryTele& query); + void send_postQuery(const QueryTele& query); + void recv_postQuery(); + void postStep(const StepTele& query); + void send_postStep(const StepTele& query); + void recv_postStep(); + void postImport(const ImportTele& query); + void send_postImport(const ImportTele& query); + void recv_postImport(); + + protected: + boost::shared_ptr< ::apache::thrift::protocol::TProtocol> piprot_; + boost::shared_ptr< ::apache::thrift::protocol::TProtocol> poprot_; + ::apache::thrift::protocol::TProtocol* iprot_; + ::apache::thrift::protocol::TProtocol* oprot_; }; class QueryTeleServiceProcessor : public ::apache::thrift::TDispatchProcessor { -protected: - boost::shared_ptr iface_; - virtual bool dispatchCall(::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, void* callContext); -private: - typedef void (QueryTeleServiceProcessor::*ProcessFunction)(int32_t, ::apache::thrift::protocol::TProtocol*, ::apache::thrift::protocol::TProtocol*, void*); - typedef std::map ProcessMap; - ProcessMap processMap_; - void process_postQuery(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); - void process_postStep(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); - void process_postImport(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); -public: - QueryTeleServiceProcessor(boost::shared_ptr iface) : - iface_(iface) - { - processMap_["postQuery"] = &QueryTeleServiceProcessor::process_postQuery; - processMap_["postStep"] = &QueryTeleServiceProcessor::process_postStep; - processMap_["postImport"] = &QueryTeleServiceProcessor::process_postImport; - } + protected: + boost::shared_ptr iface_; + virtual bool dispatchCall(::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, + int32_t seqid, void* callContext); - virtual ~QueryTeleServiceProcessor() {} + private: + typedef void (QueryTeleServiceProcessor::*ProcessFunction)(int32_t, ::apache::thrift::protocol::TProtocol*, + ::apache::thrift::protocol::TProtocol*, void*); + typedef std::map ProcessMap; + ProcessMap processMap_; + void process_postQuery(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_postStep(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_postImport(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + + public: + QueryTeleServiceProcessor(boost::shared_ptr iface) : iface_(iface) + { + processMap_["postQuery"] = &QueryTeleServiceProcessor::process_postQuery; + processMap_["postStep"] = &QueryTeleServiceProcessor::process_postStep; + processMap_["postImport"] = &QueryTeleServiceProcessor::process_postImport; + } + + virtual ~QueryTeleServiceProcessor() + { + } }; class QueryTeleServiceProcessorFactory : public ::apache::thrift::TProcessorFactory { -public: - QueryTeleServiceProcessorFactory(const ::boost::shared_ptr< QueryTeleServiceIfFactory >& handlerFactory) : - handlerFactory_(handlerFactory) {} + public: + QueryTeleServiceProcessorFactory(const ::boost::shared_ptr& handlerFactory) + : handlerFactory_(handlerFactory) + { + } - ::boost::shared_ptr< ::apache::thrift::TProcessor > getProcessor(const ::apache::thrift::TConnectionInfo& connInfo); + ::boost::shared_ptr< ::apache::thrift::TProcessor> getProcessor( + const ::apache::thrift::TConnectionInfo& connInfo); -protected: - ::boost::shared_ptr< QueryTeleServiceIfFactory > handlerFactory_; + protected: + ::boost::shared_ptr handlerFactory_; }; class QueryTeleServiceMultiface : virtual public QueryTeleServiceIf { -public: - QueryTeleServiceMultiface(std::vector >& ifaces) : ifaces_(ifaces) - { - } - virtual ~QueryTeleServiceMultiface() {} -protected: - std::vector > ifaces_; - QueryTeleServiceMultiface() {} - void add(boost::shared_ptr iface) - { - ifaces_.push_back(iface); - } -public: - void postQuery(const QueryTele& query) - { - size_t sz = ifaces_.size(); - size_t i = 0; + public: + QueryTeleServiceMultiface(std::vector >& ifaces) : ifaces_(ifaces) + { + } + virtual ~QueryTeleServiceMultiface() + { + } - for (; i < (sz - 1); ++i) - { - ifaces_[i]->postQuery(query); - } + protected: + std::vector > ifaces_; + QueryTeleServiceMultiface() + { + } + void add(boost::shared_ptr iface) + { + ifaces_.push_back(iface); + } - ifaces_[i]->postQuery(query); + public: + void postQuery(const QueryTele& query) + { + size_t sz = ifaces_.size(); + size_t i = 0; + + for (; i < (sz - 1); ++i) + { + ifaces_[i]->postQuery(query); } - void postStep(const StepTele& query) + ifaces_[i]->postQuery(query); + } + + void postStep(const StepTele& query) + { + size_t sz = ifaces_.size(); + size_t i = 0; + + for (; i < (sz - 1); ++i) { - size_t sz = ifaces_.size(); - size_t i = 0; - - for (; i < (sz - 1); ++i) - { - ifaces_[i]->postStep(query); - } - - ifaces_[i]->postStep(query); + ifaces_[i]->postStep(query); } - void postImport(const ImportTele& query) + ifaces_[i]->postStep(query); + } + + void postImport(const ImportTele& query) + { + size_t sz = ifaces_.size(); + size_t i = 0; + + for (; i < (sz - 1); ++i) { - size_t sz = ifaces_.size(); - size_t i = 0; - - for (; i < (sz - 1); ++i) - { - ifaces_[i]->postImport(query); - } - - ifaces_[i]->postImport(query); + ifaces_[i]->postImport(query); } + ifaces_[i]->postImport(query); + } }; -} // namespace +} // namespace querytele #endif diff --git a/utils/querytele/querystepparms.h b/utils/querytele/querystepparms.h index 20cb861d9..12b56f5c2 100644 --- a/utils/querytele/querystepparms.h +++ b/utils/querytele/querystepparms.h @@ -22,22 +22,22 @@ namespace querytele { - class QueryStepParms { -public: - explicit QueryStepParms(StepTeleStats::StepType st = StepTeleStats::T_INVALID) : stepType(st) { } - ~QueryStepParms() { } + public: + explicit QueryStepParms(StepTeleStats::StepType st = StepTeleStats::T_INVALID) : stepType(st) + { + } + ~QueryStepParms() + { + } - StepTeleStats::StepType stepType; - -protected: - -private: + StepTeleStats::StepType stepType; + protected: + private: }; -} +} // namespace querytele #endif - diff --git a/utils/querytele/querytele.cpp b/utils/querytele/querytele.cpp index 777aee71c..6edaf0b49 100644 --- a/utils/querytele/querytele.cpp +++ b/utils/querytele/querytele.cpp @@ -28,32 +28,30 @@ using namespace boost; namespace { -//It's not clear that random_generator is thread-safe, so we'll just mutex it... +// It's not clear that random_generator is thread-safe, so we'll just mutex it... uuids::random_generator uuidgen; mutex uuidgenMtx; -} +} // namespace namespace querytele { - /*static*/ uuids::uuid QueryTeleClient::genUUID() { - mutex::scoped_lock lk(uuidgenMtx); - return uuidgen(); + mutex::scoped_lock lk(uuidgenMtx); + return uuidgen(); } /*static*/ int64_t QueryTeleClient::timeNowms() { - int64_t nowms = -1; - struct timeval tv; + int64_t nowms = -1; + struct timeval tv; - if (gettimeofday(&tv, 0) == 0) - nowms = (tv.tv_sec * 1000) + (tv.tv_usec / 1000); - - return nowms; -} + if (gettimeofday(&tv, 0) == 0) + nowms = (tv.tv_sec * 1000) + (tv.tv_usec / 1000); + return nowms; } +} // namespace querytele diff --git a/utils/querytele/querytele.h b/utils/querytele/querytele.h index 44b35b205..43cf811f9 100644 --- a/utils/querytele/querytele.h +++ b/utils/querytele/querytele.h @@ -28,4 +28,3 @@ namespace querytele } #endif - diff --git a/utils/querytele/querytele_constants.cpp b/utils/querytele/querytele_constants.cpp index f83bf18dc..df2ef185f 100644 --- a/utils/querytele/querytele_constants.cpp +++ b/utils/querytele/querytele_constants.cpp @@ -8,12 +8,10 @@ namespace querytele { - const queryteleConstants g_querytele_constants; queryteleConstants::queryteleConstants() { } -} // namespace - +} // namespace querytele diff --git a/utils/querytele/querytele_constants.h b/utils/querytele/querytele_constants.h index d90cf4537..8d8170fd5 100644 --- a/utils/querytele/querytele_constants.h +++ b/utils/querytele/querytele_constants.h @@ -11,16 +11,14 @@ namespace querytele { - class queryteleConstants { -public: - queryteleConstants(); - + public: + queryteleConstants(); }; extern const queryteleConstants g_querytele_constants; -} // namespace +} // namespace querytele #endif diff --git a/utils/querytele/querytele_types.cpp b/utils/querytele/querytele_types.cpp index e61e3dece..966767e41 100644 --- a/utils/querytele/querytele_types.cpp +++ b/utils/querytele/querytele_types.cpp @@ -10,1357 +10,1295 @@ namespace querytele { +int _kQTTypeValues[] = {QTType::QT_INVALID, QTType::QT_SUMMARY, QTType::QT_PROGRESS, QTType::QT_START}; +const char* _kQTTypeNames[] = {"QT_INVALID", "QT_SUMMARY", "QT_PROGRESS", "QT_START"}; +const std::map _QTType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kQTTypeValues, + _kQTTypeNames), + ::apache::thrift::TEnumIterator(-1, NULL, NULL)); -int _kQTTypeValues[] = -{ - QTType::QT_INVALID, - QTType::QT_SUMMARY, - QTType::QT_PROGRESS, - QTType::QT_START -}; -const char* _kQTTypeNames[] = -{ - "QT_INVALID", - "QT_SUMMARY", - "QT_PROGRESS", - "QT_START" -}; -const std::map _QTType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kQTTypeValues, _kQTTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +int _kSTTypeValues[] = {STType::ST_INVALID, STType::ST_SUMMARY, STType::ST_PROGRESS, STType::ST_START}; +const char* _kSTTypeNames[] = {"ST_INVALID", "ST_SUMMARY", "ST_PROGRESS", "ST_START"}; +const std::map _STType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kSTTypeValues, + _kSTTypeNames), + ::apache::thrift::TEnumIterator(-1, NULL, NULL)); -int _kSTTypeValues[] = -{ - STType::ST_INVALID, - STType::ST_SUMMARY, - STType::ST_PROGRESS, - STType::ST_START -}; -const char* _kSTTypeNames[] = -{ - "ST_INVALID", - "ST_SUMMARY", - "ST_PROGRESS", - "ST_START" -}; -const std::map _STType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kSTTypeValues, _kSTTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +int _kStepTypeValues[] = {StepType::T_INVALID, StepType::T_HJS, StepType::T_DSS, StepType::T_CES, + StepType::T_SQS, StepType::T_TAS, StepType::T_TNS, StepType::T_BPS, + StepType::T_TCS, StepType::T_HVS, StepType::T_WFS, StepType::T_SAS, + StepType::T_TUN}; +const char* _kStepTypeNames[] = {"T_INVALID", "T_HJS", "T_DSS", "T_CES", "T_SQS", "T_TAS", "T_TNS", + "T_BPS", "T_TCS", "T_HVS", "T_WFS", "T_SAS", "T_TUN"}; +const std::map _StepType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(13, + _kStepTypeValues, + _kStepTypeNames), + ::apache::thrift::TEnumIterator(-1, NULL, NULL)); -int _kStepTypeValues[] = -{ - StepType::T_INVALID, - StepType::T_HJS, - StepType::T_DSS, - StepType::T_CES, - StepType::T_SQS, - StepType::T_TAS, - StepType::T_TNS, - StepType::T_BPS, - StepType::T_TCS, - StepType::T_HVS, - StepType::T_WFS, - StepType::T_SAS, - StepType::T_TUN -}; -const char* _kStepTypeNames[] = -{ - "T_INVALID", - "T_HJS", - "T_DSS", - "T_CES", - "T_SQS", - "T_TAS", - "T_TNS", - "T_BPS", - "T_TCS", - "T_HVS", - "T_WFS", - "T_SAS", - "T_TUN" -}; -const std::map _StepType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(13, _kStepTypeValues, _kStepTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); - -int _kITTypeValues[] = -{ - ITType::IT_INVALID, - ITType::IT_SUMMARY, - ITType::IT_PROGRESS, - ITType::IT_START, - ITType::IT_TERM -}; -const char* _kITTypeNames[] = -{ - "IT_INVALID", - "IT_SUMMARY", - "IT_PROGRESS", - "IT_START", - "IT_TERM" -}; -const std::map _ITType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(5, _kITTypeValues, _kITTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +int _kITTypeValues[] = {ITType::IT_INVALID, ITType::IT_SUMMARY, ITType::IT_PROGRESS, ITType::IT_START, + ITType::IT_TERM}; +const char* _kITTypeNames[] = {"IT_INVALID", "IT_SUMMARY", "IT_PROGRESS", "IT_START", "IT_TERM"}; +const std::map _ITType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(5, _kITTypeValues, + _kITTypeNames), + ::apache::thrift::TEnumIterator(-1, NULL, NULL)); const char* QueryTele::ascii_fingerprint = "E179941E2BE9C920811D86261E5AAE67"; -const uint8_t QueryTele::binary_fingerprint[16] = {0xE1, 0x79, 0x94, 0x1E, 0x2B, 0xE9, 0xC9, 0x20, 0x81, 0x1D, 0x86, 0x26, 0x1E, 0x5A, 0xAE, 0x67}; +const uint8_t QueryTele::binary_fingerprint[16] = {0xE1, 0x79, 0x94, 0x1E, 0x2B, 0xE9, 0xC9, 0x20, + 0x81, 0x1D, 0x86, 0x26, 0x1E, 0x5A, 0xAE, 0x67}; uint32_t QueryTele::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->query_uuid); - this->__isset.query_uuid = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 2: - if (ftype == ::apache::thrift::protocol::T_I32) - { - int32_t ecast0; - xfer += iprot->readI32(ecast0); - this->msg_type = (QTType::type)ecast0; - this->__isset.msg_type = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 3: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->max_mem_pct); - this->__isset.max_mem_pct = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 4: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->num_files); - this->__isset.num_files = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 5: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->phy_io); - this->__isset.phy_io = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 6: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->cache_io); - this->__isset.cache_io = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 7: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->msg_rcv_cnt); - this->__isset.msg_rcv_cnt = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 8: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->cp_blocks_skipped); - this->__isset.cp_blocks_skipped = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 9: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->msg_bytes_in); - this->__isset.msg_bytes_in = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 10: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->msg_bytes_out); - this->__isset.msg_bytes_out = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 11: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->rows); - this->__isset.rows = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 12: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->start_time); - this->__isset.start_time = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 13: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->end_time); - this->__isset.end_time = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 14: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->error_no); - this->__isset.error_no = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 15: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->blocks_changed); - this->__isset.blocks_changed = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 16: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->session_id); - this->__isset.session_id = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 17: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->query_type); - this->__isset.query_type = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 18: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->query); - this->__isset.query = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 19: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->user); - this->__isset.user = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 20: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->host); - this->__isset.host = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 21: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->priority); - this->__isset.priority = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 22: - if (ftype == ::apache::thrift::protocol::T_I32) - { - xfer += iprot->readI32(this->priority_level); - this->__isset.priority_level = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 23: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->system_name); - this->__isset.system_name = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 24: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->module_name); - this->__isset.module_name = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 25: - if (ftype == ::apache::thrift::protocol::T_I32) - { - xfer += iprot->readI32(this->local_query); - this->__isset.local_query = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 26: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->schema_name); - this->__isset.schema_name = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - default: - xfer += iprot->skip(ftype); - break; - } - - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->query_uuid); + this->__isset.query_uuid = true; + } + else + { + xfer += iprot->skip(ftype); + } - return xfer; + break; + + case 2: + if (ftype == ::apache::thrift::protocol::T_I32) + { + int32_t ecast0; + xfer += iprot->readI32(ecast0); + this->msg_type = (QTType::type)ecast0; + this->__isset.msg_type = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 3: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->max_mem_pct); + this->__isset.max_mem_pct = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 4: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->num_files); + this->__isset.num_files = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->phy_io); + this->__isset.phy_io = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 6: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->cache_io); + this->__isset.cache_io = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 7: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->msg_rcv_cnt); + this->__isset.msg_rcv_cnt = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 8: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->cp_blocks_skipped); + this->__isset.cp_blocks_skipped = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 9: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->msg_bytes_in); + this->__isset.msg_bytes_in = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 10: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->msg_bytes_out); + this->__isset.msg_bytes_out = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 11: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->rows); + this->__isset.rows = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 12: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->start_time); + this->__isset.start_time = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 13: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->end_time); + this->__isset.end_time = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 14: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->error_no); + this->__isset.error_no = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 15: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->blocks_changed); + this->__isset.blocks_changed = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 16: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->session_id); + this->__isset.session_id = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 17: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->query_type); + this->__isset.query_type = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 18: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->query); + this->__isset.query = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 19: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->user); + this->__isset.user = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 20: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->host); + this->__isset.host = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 21: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->priority); + this->__isset.priority = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 22: + if (ftype == ::apache::thrift::protocol::T_I32) + { + xfer += iprot->readI32(this->priority_level); + this->__isset.priority_level = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 23: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->system_name); + this->__isset.system_name = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 24: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->module_name); + this->__isset.module_name = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 25: + if (ftype == ::apache::thrift::protocol::T_I32) + { + xfer += iprot->readI32(this->local_query); + this->__isset.local_query = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 26: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->schema_name); + this->__isset.schema_name = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + default: xfer += iprot->skip(ftype); break; + } + + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t QueryTele::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("QueryTele"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("QueryTele"); - xfer += oprot->writeFieldBegin("query_uuid", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeString(this->query_uuid); + xfer += oprot->writeFieldBegin("query_uuid", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->query_uuid); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("msg_type", ::apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32((int32_t)this->msg_type); + xfer += oprot->writeFieldEnd(); + + if (this->__isset.max_mem_pct) + { + xfer += oprot->writeFieldBegin("max_mem_pct", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64(this->max_mem_pct); xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeFieldBegin("msg_type", ::apache::thrift::protocol::T_I32, 2); - xfer += oprot->writeI32((int32_t)this->msg_type); + if (this->__isset.num_files) + { + xfer += oprot->writeFieldBegin("num_files", ::apache::thrift::protocol::T_I64, 4); + xfer += oprot->writeI64(this->num_files); xfer += oprot->writeFieldEnd(); + } - if (this->__isset.max_mem_pct) - { - xfer += oprot->writeFieldBegin("max_mem_pct", ::apache::thrift::protocol::T_I64, 3); - xfer += oprot->writeI64(this->max_mem_pct); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.phy_io) + { + xfer += oprot->writeFieldBegin("phy_io", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->phy_io); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.num_files) - { - xfer += oprot->writeFieldBegin("num_files", ::apache::thrift::protocol::T_I64, 4); - xfer += oprot->writeI64(this->num_files); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.cache_io) + { + xfer += oprot->writeFieldBegin("cache_io", ::apache::thrift::protocol::T_I64, 6); + xfer += oprot->writeI64(this->cache_io); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.phy_io) - { - xfer += oprot->writeFieldBegin("phy_io", ::apache::thrift::protocol::T_I64, 5); - xfer += oprot->writeI64(this->phy_io); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.msg_rcv_cnt) + { + xfer += oprot->writeFieldBegin("msg_rcv_cnt", ::apache::thrift::protocol::T_I64, 7); + xfer += oprot->writeI64(this->msg_rcv_cnt); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.cache_io) - { - xfer += oprot->writeFieldBegin("cache_io", ::apache::thrift::protocol::T_I64, 6); - xfer += oprot->writeI64(this->cache_io); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.cp_blocks_skipped) + { + xfer += oprot->writeFieldBegin("cp_blocks_skipped", ::apache::thrift::protocol::T_I64, 8); + xfer += oprot->writeI64(this->cp_blocks_skipped); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.msg_rcv_cnt) - { - xfer += oprot->writeFieldBegin("msg_rcv_cnt", ::apache::thrift::protocol::T_I64, 7); - xfer += oprot->writeI64(this->msg_rcv_cnt); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.msg_bytes_in) + { + xfer += oprot->writeFieldBegin("msg_bytes_in", ::apache::thrift::protocol::T_I64, 9); + xfer += oprot->writeI64(this->msg_bytes_in); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.cp_blocks_skipped) - { - xfer += oprot->writeFieldBegin("cp_blocks_skipped", ::apache::thrift::protocol::T_I64, 8); - xfer += oprot->writeI64(this->cp_blocks_skipped); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.msg_bytes_out) + { + xfer += oprot->writeFieldBegin("msg_bytes_out", ::apache::thrift::protocol::T_I64, 10); + xfer += oprot->writeI64(this->msg_bytes_out); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.msg_bytes_in) - { - xfer += oprot->writeFieldBegin("msg_bytes_in", ::apache::thrift::protocol::T_I64, 9); - xfer += oprot->writeI64(this->msg_bytes_in); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.rows) + { + xfer += oprot->writeFieldBegin("rows", ::apache::thrift::protocol::T_I64, 11); + xfer += oprot->writeI64(this->rows); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.msg_bytes_out) - { - xfer += oprot->writeFieldBegin("msg_bytes_out", ::apache::thrift::protocol::T_I64, 10); - xfer += oprot->writeI64(this->msg_bytes_out); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.start_time) + { + xfer += oprot->writeFieldBegin("start_time", ::apache::thrift::protocol::T_I64, 12); + xfer += oprot->writeI64(this->start_time); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.rows) - { - xfer += oprot->writeFieldBegin("rows", ::apache::thrift::protocol::T_I64, 11); - xfer += oprot->writeI64(this->rows); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.end_time) + { + xfer += oprot->writeFieldBegin("end_time", ::apache::thrift::protocol::T_I64, 13); + xfer += oprot->writeI64(this->end_time); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.start_time) - { - xfer += oprot->writeFieldBegin("start_time", ::apache::thrift::protocol::T_I64, 12); - xfer += oprot->writeI64(this->start_time); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.error_no) + { + xfer += oprot->writeFieldBegin("error_no", ::apache::thrift::protocol::T_I64, 14); + xfer += oprot->writeI64(this->error_no); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.end_time) - { - xfer += oprot->writeFieldBegin("end_time", ::apache::thrift::protocol::T_I64, 13); - xfer += oprot->writeI64(this->end_time); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.blocks_changed) + { + xfer += oprot->writeFieldBegin("blocks_changed", ::apache::thrift::protocol::T_I64, 15); + xfer += oprot->writeI64(this->blocks_changed); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.error_no) - { - xfer += oprot->writeFieldBegin("error_no", ::apache::thrift::protocol::T_I64, 14); - xfer += oprot->writeI64(this->error_no); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.session_id) + { + xfer += oprot->writeFieldBegin("session_id", ::apache::thrift::protocol::T_I64, 16); + xfer += oprot->writeI64(this->session_id); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.blocks_changed) - { - xfer += oprot->writeFieldBegin("blocks_changed", ::apache::thrift::protocol::T_I64, 15); - xfer += oprot->writeI64(this->blocks_changed); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.query_type) + { + xfer += oprot->writeFieldBegin("query_type", ::apache::thrift::protocol::T_STRING, 17); + xfer += oprot->writeString(this->query_type); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.session_id) - { - xfer += oprot->writeFieldBegin("session_id", ::apache::thrift::protocol::T_I64, 16); - xfer += oprot->writeI64(this->session_id); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.query) + { + xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRING, 18); + xfer += oprot->writeString(this->query); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.query_type) - { - xfer += oprot->writeFieldBegin("query_type", ::apache::thrift::protocol::T_STRING, 17); - xfer += oprot->writeString(this->query_type); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.user) + { + xfer += oprot->writeFieldBegin("user", ::apache::thrift::protocol::T_STRING, 19); + xfer += oprot->writeString(this->user); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.query) - { - xfer += oprot->writeFieldBegin("query", ::apache::thrift::protocol::T_STRING, 18); - xfer += oprot->writeString(this->query); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.host) + { + xfer += oprot->writeFieldBegin("host", ::apache::thrift::protocol::T_STRING, 20); + xfer += oprot->writeString(this->host); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.user) - { - xfer += oprot->writeFieldBegin("user", ::apache::thrift::protocol::T_STRING, 19); - xfer += oprot->writeString(this->user); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.priority) + { + xfer += oprot->writeFieldBegin("priority", ::apache::thrift::protocol::T_STRING, 21); + xfer += oprot->writeString(this->priority); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.host) - { - xfer += oprot->writeFieldBegin("host", ::apache::thrift::protocol::T_STRING, 20); - xfer += oprot->writeString(this->host); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.priority_level) + { + xfer += oprot->writeFieldBegin("priority_level", ::apache::thrift::protocol::T_I32, 22); + xfer += oprot->writeI32(this->priority_level); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.priority) - { - xfer += oprot->writeFieldBegin("priority", ::apache::thrift::protocol::T_STRING, 21); - xfer += oprot->writeString(this->priority); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.system_name) + { + xfer += oprot->writeFieldBegin("system_name", ::apache::thrift::protocol::T_STRING, 23); + xfer += oprot->writeString(this->system_name); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.priority_level) - { - xfer += oprot->writeFieldBegin("priority_level", ::apache::thrift::protocol::T_I32, 22); - xfer += oprot->writeI32(this->priority_level); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.module_name) + { + xfer += oprot->writeFieldBegin("module_name", ::apache::thrift::protocol::T_STRING, 24); + xfer += oprot->writeString(this->module_name); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.system_name) - { - xfer += oprot->writeFieldBegin("system_name", ::apache::thrift::protocol::T_STRING, 23); - xfer += oprot->writeString(this->system_name); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.local_query) + { + xfer += oprot->writeFieldBegin("local_query", ::apache::thrift::protocol::T_I32, 25); + xfer += oprot->writeI32(this->local_query); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.module_name) - { - xfer += oprot->writeFieldBegin("module_name", ::apache::thrift::protocol::T_STRING, 24); - xfer += oprot->writeString(this->module_name); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.schema_name) + { + xfer += oprot->writeFieldBegin("schema_name", ::apache::thrift::protocol::T_STRING, 26); + xfer += oprot->writeString(this->schema_name); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.local_query) - { - xfer += oprot->writeFieldBegin("local_query", ::apache::thrift::protocol::T_I32, 25); - xfer += oprot->writeI32(this->local_query); - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.schema_name) - { - xfer += oprot->writeFieldBegin("schema_name", ::apache::thrift::protocol::T_STRING, 26); - xfer += oprot->writeString(this->schema_name); - xfer += oprot->writeFieldEnd(); - } - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } void swap(QueryTele& a, QueryTele& b) { - using ::std::swap; - swap(a.query_uuid, b.query_uuid); - swap(a.msg_type, b.msg_type); - swap(a.max_mem_pct, b.max_mem_pct); - swap(a.num_files, b.num_files); - swap(a.phy_io, b.phy_io); - swap(a.cache_io, b.cache_io); - swap(a.msg_rcv_cnt, b.msg_rcv_cnt); - swap(a.cp_blocks_skipped, b.cp_blocks_skipped); - swap(a.msg_bytes_in, b.msg_bytes_in); - swap(a.msg_bytes_out, b.msg_bytes_out); - swap(a.rows, b.rows); - swap(a.start_time, b.start_time); - swap(a.end_time, b.end_time); - swap(a.error_no, b.error_no); - swap(a.blocks_changed, b.blocks_changed); - swap(a.session_id, b.session_id); - swap(a.query_type, b.query_type); - swap(a.query, b.query); - swap(a.user, b.user); - swap(a.host, b.host); - swap(a.priority, b.priority); - swap(a.priority_level, b.priority_level); - swap(a.system_name, b.system_name); - swap(a.module_name, b.module_name); - swap(a.local_query, b.local_query); - swap(a.schema_name, b.schema_name); - swap(a.__isset, b.__isset); + using ::std::swap; + swap(a.query_uuid, b.query_uuid); + swap(a.msg_type, b.msg_type); + swap(a.max_mem_pct, b.max_mem_pct); + swap(a.num_files, b.num_files); + swap(a.phy_io, b.phy_io); + swap(a.cache_io, b.cache_io); + swap(a.msg_rcv_cnt, b.msg_rcv_cnt); + swap(a.cp_blocks_skipped, b.cp_blocks_skipped); + swap(a.msg_bytes_in, b.msg_bytes_in); + swap(a.msg_bytes_out, b.msg_bytes_out); + swap(a.rows, b.rows); + swap(a.start_time, b.start_time); + swap(a.end_time, b.end_time); + swap(a.error_no, b.error_no); + swap(a.blocks_changed, b.blocks_changed); + swap(a.session_id, b.session_id); + swap(a.query_type, b.query_type); + swap(a.query, b.query); + swap(a.user, b.user); + swap(a.host, b.host); + swap(a.priority, b.priority); + swap(a.priority_level, b.priority_level); + swap(a.system_name, b.system_name); + swap(a.module_name, b.module_name); + swap(a.local_query, b.local_query); + swap(a.schema_name, b.schema_name); + swap(a.__isset, b.__isset); } const char* StepTele::ascii_fingerprint = "4E40C17AE92DFF833A8CC23318F4FE68"; -const uint8_t StepTele::binary_fingerprint[16] = {0x4E, 0x40, 0xC1, 0x7A, 0xE9, 0x2D, 0xFF, 0x83, 0x3A, 0x8C, 0xC2, 0x33, 0x18, 0xF4, 0xFE, 0x68}; +const uint8_t StepTele::binary_fingerprint[16] = {0x4E, 0x40, 0xC1, 0x7A, 0xE9, 0x2D, 0xFF, 0x83, + 0x3A, 0x8C, 0xC2, 0x33, 0x18, 0xF4, 0xFE, 0x68}; uint32_t StepTele::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->query_uuid); - this->__isset.query_uuid = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 2: - if (ftype == ::apache::thrift::protocol::T_I32) - { - int32_t ecast1; - xfer += iprot->readI32(ecast1); - this->msg_type = (STType::type)ecast1; - this->__isset.msg_type = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 3: - if (ftype == ::apache::thrift::protocol::T_I32) - { - int32_t ecast2; - xfer += iprot->readI32(ecast2); - this->step_type = (StepType::type)ecast2; - this->__isset.step_type = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 4: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->step_uuid); - this->__isset.step_uuid = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 5: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->phy_io); - this->__isset.phy_io = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 6: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->cache_io); - this->__isset.cache_io = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 7: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->msg_rcv_cnt); - this->__isset.msg_rcv_cnt = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 8: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->cp_blocks_skipped); - this->__isset.cp_blocks_skipped = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 9: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->msg_bytes_in); - this->__isset.msg_bytes_in = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 10: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->msg_bytes_out); - this->__isset.msg_bytes_out = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 11: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->rows); - this->__isset.rows = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 12: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->start_time); - this->__isset.start_time = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 13: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->end_time); - this->__isset.end_time = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 14: - if (ftype == ::apache::thrift::protocol::T_I32) - { - xfer += iprot->readI32(this->total_units_of_work); - this->__isset.total_units_of_work = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 15: - if (ftype == ::apache::thrift::protocol::T_I32) - { - xfer += iprot->readI32(this->units_of_work_completed); - this->__isset.units_of_work_completed = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - default: - xfer += iprot->skip(ftype); - break; - } - - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->query_uuid); + this->__isset.query_uuid = true; + } + else + { + xfer += iprot->skip(ftype); + } - return xfer; + break; + + case 2: + if (ftype == ::apache::thrift::protocol::T_I32) + { + int32_t ecast1; + xfer += iprot->readI32(ecast1); + this->msg_type = (STType::type)ecast1; + this->__isset.msg_type = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 3: + if (ftype == ::apache::thrift::protocol::T_I32) + { + int32_t ecast2; + xfer += iprot->readI32(ecast2); + this->step_type = (StepType::type)ecast2; + this->__isset.step_type = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 4: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->step_uuid); + this->__isset.step_uuid = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->phy_io); + this->__isset.phy_io = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 6: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->cache_io); + this->__isset.cache_io = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 7: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->msg_rcv_cnt); + this->__isset.msg_rcv_cnt = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 8: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->cp_blocks_skipped); + this->__isset.cp_blocks_skipped = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 9: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->msg_bytes_in); + this->__isset.msg_bytes_in = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 10: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->msg_bytes_out); + this->__isset.msg_bytes_out = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 11: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->rows); + this->__isset.rows = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 12: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->start_time); + this->__isset.start_time = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 13: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->end_time); + this->__isset.end_time = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 14: + if (ftype == ::apache::thrift::protocol::T_I32) + { + xfer += iprot->readI32(this->total_units_of_work); + this->__isset.total_units_of_work = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 15: + if (ftype == ::apache::thrift::protocol::T_I32) + { + xfer += iprot->readI32(this->units_of_work_completed); + this->__isset.units_of_work_completed = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + default: xfer += iprot->skip(ftype); break; + } + + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t StepTele::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("StepTele"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("StepTele"); - xfer += oprot->writeFieldBegin("query_uuid", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeString(this->query_uuid); + xfer += oprot->writeFieldBegin("query_uuid", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->query_uuid); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("msg_type", ::apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32((int32_t)this->msg_type); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("step_type", ::apache::thrift::protocol::T_I32, 3); + xfer += oprot->writeI32((int32_t)this->step_type); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("step_uuid", ::apache::thrift::protocol::T_STRING, 4); + xfer += oprot->writeString(this->step_uuid); + xfer += oprot->writeFieldEnd(); + + if (this->__isset.phy_io) + { + xfer += oprot->writeFieldBegin("phy_io", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->phy_io); xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeFieldBegin("msg_type", ::apache::thrift::protocol::T_I32, 2); - xfer += oprot->writeI32((int32_t)this->msg_type); + if (this->__isset.cache_io) + { + xfer += oprot->writeFieldBegin("cache_io", ::apache::thrift::protocol::T_I64, 6); + xfer += oprot->writeI64(this->cache_io); xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeFieldBegin("step_type", ::apache::thrift::protocol::T_I32, 3); - xfer += oprot->writeI32((int32_t)this->step_type); + if (this->__isset.msg_rcv_cnt) + { + xfer += oprot->writeFieldBegin("msg_rcv_cnt", ::apache::thrift::protocol::T_I64, 7); + xfer += oprot->writeI64(this->msg_rcv_cnt); xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeFieldBegin("step_uuid", ::apache::thrift::protocol::T_STRING, 4); - xfer += oprot->writeString(this->step_uuid); + if (this->__isset.cp_blocks_skipped) + { + xfer += oprot->writeFieldBegin("cp_blocks_skipped", ::apache::thrift::protocol::T_I64, 8); + xfer += oprot->writeI64(this->cp_blocks_skipped); xfer += oprot->writeFieldEnd(); + } - if (this->__isset.phy_io) - { - xfer += oprot->writeFieldBegin("phy_io", ::apache::thrift::protocol::T_I64, 5); - xfer += oprot->writeI64(this->phy_io); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.msg_bytes_in) + { + xfer += oprot->writeFieldBegin("msg_bytes_in", ::apache::thrift::protocol::T_I64, 9); + xfer += oprot->writeI64(this->msg_bytes_in); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.cache_io) - { - xfer += oprot->writeFieldBegin("cache_io", ::apache::thrift::protocol::T_I64, 6); - xfer += oprot->writeI64(this->cache_io); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.msg_bytes_out) + { + xfer += oprot->writeFieldBegin("msg_bytes_out", ::apache::thrift::protocol::T_I64, 10); + xfer += oprot->writeI64(this->msg_bytes_out); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.msg_rcv_cnt) - { - xfer += oprot->writeFieldBegin("msg_rcv_cnt", ::apache::thrift::protocol::T_I64, 7); - xfer += oprot->writeI64(this->msg_rcv_cnt); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.rows) + { + xfer += oprot->writeFieldBegin("rows", ::apache::thrift::protocol::T_I64, 11); + xfer += oprot->writeI64(this->rows); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.cp_blocks_skipped) - { - xfer += oprot->writeFieldBegin("cp_blocks_skipped", ::apache::thrift::protocol::T_I64, 8); - xfer += oprot->writeI64(this->cp_blocks_skipped); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.start_time) + { + xfer += oprot->writeFieldBegin("start_time", ::apache::thrift::protocol::T_I64, 12); + xfer += oprot->writeI64(this->start_time); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.msg_bytes_in) - { - xfer += oprot->writeFieldBegin("msg_bytes_in", ::apache::thrift::protocol::T_I64, 9); - xfer += oprot->writeI64(this->msg_bytes_in); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.end_time) + { + xfer += oprot->writeFieldBegin("end_time", ::apache::thrift::protocol::T_I64, 13); + xfer += oprot->writeI64(this->end_time); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.msg_bytes_out) - { - xfer += oprot->writeFieldBegin("msg_bytes_out", ::apache::thrift::protocol::T_I64, 10); - xfer += oprot->writeI64(this->msg_bytes_out); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.total_units_of_work) + { + xfer += oprot->writeFieldBegin("total_units_of_work", ::apache::thrift::protocol::T_I32, 14); + xfer += oprot->writeI32(this->total_units_of_work); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.rows) - { - xfer += oprot->writeFieldBegin("rows", ::apache::thrift::protocol::T_I64, 11); - xfer += oprot->writeI64(this->rows); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.units_of_work_completed) + { + xfer += oprot->writeFieldBegin("units_of_work_completed", ::apache::thrift::protocol::T_I32, 15); + xfer += oprot->writeI32(this->units_of_work_completed); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.start_time) - { - xfer += oprot->writeFieldBegin("start_time", ::apache::thrift::protocol::T_I64, 12); - xfer += oprot->writeI64(this->start_time); - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.end_time) - { - xfer += oprot->writeFieldBegin("end_time", ::apache::thrift::protocol::T_I64, 13); - xfer += oprot->writeI64(this->end_time); - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.total_units_of_work) - { - xfer += oprot->writeFieldBegin("total_units_of_work", ::apache::thrift::protocol::T_I32, 14); - xfer += oprot->writeI32(this->total_units_of_work); - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.units_of_work_completed) - { - xfer += oprot->writeFieldBegin("units_of_work_completed", ::apache::thrift::protocol::T_I32, 15); - xfer += oprot->writeI32(this->units_of_work_completed); - xfer += oprot->writeFieldEnd(); - } - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } void swap(StepTele& a, StepTele& b) { - using ::std::swap; - swap(a.query_uuid, b.query_uuid); - swap(a.msg_type, b.msg_type); - swap(a.step_type, b.step_type); - swap(a.step_uuid, b.step_uuid); - swap(a.phy_io, b.phy_io); - swap(a.cache_io, b.cache_io); - swap(a.msg_rcv_cnt, b.msg_rcv_cnt); - swap(a.cp_blocks_skipped, b.cp_blocks_skipped); - swap(a.msg_bytes_in, b.msg_bytes_in); - swap(a.msg_bytes_out, b.msg_bytes_out); - swap(a.rows, b.rows); - swap(a.start_time, b.start_time); - swap(a.end_time, b.end_time); - swap(a.total_units_of_work, b.total_units_of_work); - swap(a.units_of_work_completed, b.units_of_work_completed); - swap(a.__isset, b.__isset); + using ::std::swap; + swap(a.query_uuid, b.query_uuid); + swap(a.msg_type, b.msg_type); + swap(a.step_type, b.step_type); + swap(a.step_uuid, b.step_uuid); + swap(a.phy_io, b.phy_io); + swap(a.cache_io, b.cache_io); + swap(a.msg_rcv_cnt, b.msg_rcv_cnt); + swap(a.cp_blocks_skipped, b.cp_blocks_skipped); + swap(a.msg_bytes_in, b.msg_bytes_in); + swap(a.msg_bytes_out, b.msg_bytes_out); + swap(a.rows, b.rows); + swap(a.start_time, b.start_time); + swap(a.end_time, b.end_time); + swap(a.total_units_of_work, b.total_units_of_work); + swap(a.units_of_work_completed, b.units_of_work_completed); + swap(a.__isset, b.__isset); } const char* ImportTele::ascii_fingerprint = "9C47642F47F2AFBEA98238934EB1A211"; -const uint8_t ImportTele::binary_fingerprint[16] = {0x9C, 0x47, 0x64, 0x2F, 0x47, 0xF2, 0xAF, 0xBE, 0xA9, 0x82, 0x38, 0x93, 0x4E, 0xB1, 0xA2, 0x11}; +const uint8_t ImportTele::binary_fingerprint[16] = {0x9C, 0x47, 0x64, 0x2F, 0x47, 0xF2, 0xAF, 0xBE, + 0xA9, 0x82, 0x38, 0x93, 0x4E, 0xB1, 0xA2, 0x11}; uint32_t ImportTele::read(::apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; + xfer += iprot->readStructBegin(fname); - xfer += iprot->readStructBegin(fname); + using ::apache::thrift::protocol::TProtocolException; - using ::apache::thrift::protocol::TProtocolException; + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); - - while (true) + if (ftype == ::apache::thrift::protocol::T_STOP) { - xfer += iprot->readFieldBegin(fname, ftype, fid); - - if (ftype == ::apache::thrift::protocol::T_STOP) - { - break; - } - - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->job_uuid); - this->__isset.job_uuid = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 2: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->import_uuid); - this->__isset.import_uuid = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 3: - if (ftype == ::apache::thrift::protocol::T_I32) - { - int32_t ecast3; - xfer += iprot->readI32(ecast3); - this->msg_type = (ITType::type)ecast3; - this->__isset.msg_type = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 4: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->start_time); - this->__isset.start_time = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 5: - if (ftype == ::apache::thrift::protocol::T_I64) - { - xfer += iprot->readI64(this->end_time); - this->__isset.end_time = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 6: - if (ftype == ::apache::thrift::protocol::T_LIST) - { - { - this->table_list.clear(); - uint32_t _size4; - ::apache::thrift::protocol::TType _etype7; - xfer += iprot->readListBegin(_etype7, _size4); - this->table_list.resize(_size4); - uint32_t _i8; - - for (_i8 = 0; _i8 < _size4; ++_i8) - { - xfer += iprot->readString(this->table_list[_i8]); - } - - xfer += iprot->readListEnd(); - } - this->__isset.table_list = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 7: - if (ftype == ::apache::thrift::protocol::T_LIST) - { - { - this->rows_so_far.clear(); - uint32_t _size9; - ::apache::thrift::protocol::TType _etype12; - xfer += iprot->readListBegin(_etype12, _size9); - this->rows_so_far.resize(_size9); - uint32_t _i13; - - for (_i13 = 0; _i13 < _size9; ++_i13) - { - xfer += iprot->readI64(this->rows_so_far[_i13]); - } - - xfer += iprot->readListEnd(); - } - this->__isset.rows_so_far = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 8: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->system_name); - this->__isset.system_name = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 9: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->module_name); - this->__isset.module_name = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - case 10: - if (ftype == ::apache::thrift::protocol::T_STRING) - { - xfer += iprot->readString(this->schema_name); - this->__isset.schema_name = true; - } - else - { - xfer += iprot->skip(ftype); - } - - break; - - default: - xfer += iprot->skip(ftype); - break; - } - - xfer += iprot->readFieldEnd(); + break; } - xfer += iprot->readStructEnd(); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->job_uuid); + this->__isset.job_uuid = true; + } + else + { + xfer += iprot->skip(ftype); + } - return xfer; + break; + + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->import_uuid); + this->__isset.import_uuid = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 3: + if (ftype == ::apache::thrift::protocol::T_I32) + { + int32_t ecast3; + xfer += iprot->readI32(ecast3); + this->msg_type = (ITType::type)ecast3; + this->__isset.msg_type = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 4: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->start_time); + this->__isset.start_time = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) + { + xfer += iprot->readI64(this->end_time); + this->__isset.end_time = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 6: + if (ftype == ::apache::thrift::protocol::T_LIST) + { + { + this->table_list.clear(); + uint32_t _size4; + ::apache::thrift::protocol::TType _etype7; + xfer += iprot->readListBegin(_etype7, _size4); + this->table_list.resize(_size4); + uint32_t _i8; + + for (_i8 = 0; _i8 < _size4; ++_i8) + { + xfer += iprot->readString(this->table_list[_i8]); + } + + xfer += iprot->readListEnd(); + } + this->__isset.table_list = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 7: + if (ftype == ::apache::thrift::protocol::T_LIST) + { + { + this->rows_so_far.clear(); + uint32_t _size9; + ::apache::thrift::protocol::TType _etype12; + xfer += iprot->readListBegin(_etype12, _size9); + this->rows_so_far.resize(_size9); + uint32_t _i13; + + for (_i13 = 0; _i13 < _size9; ++_i13) + { + xfer += iprot->readI64(this->rows_so_far[_i13]); + } + + xfer += iprot->readListEnd(); + } + this->__isset.rows_so_far = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 8: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->system_name); + this->__isset.system_name = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 9: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->module_name); + this->__isset.module_name = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + case 10: + if (ftype == ::apache::thrift::protocol::T_STRING) + { + xfer += iprot->readString(this->schema_name); + this->__isset.schema_name = true; + } + else + { + xfer += iprot->skip(ftype); + } + + break; + + default: xfer += iprot->skip(ftype); break; + } + + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; } uint32_t ImportTele::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - xfer += oprot->writeStructBegin("ImportTele"); + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ImportTele"); - xfer += oprot->writeFieldBegin("job_uuid", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeString(this->job_uuid); + xfer += oprot->writeFieldBegin("job_uuid", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->job_uuid); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("import_uuid", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->import_uuid); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("msg_type", ::apache::thrift::protocol::T_I32, 3); + xfer += oprot->writeI32((int32_t)this->msg_type); + xfer += oprot->writeFieldEnd(); + + if (this->__isset.start_time) + { + xfer += oprot->writeFieldBegin("start_time", ::apache::thrift::protocol::T_I64, 4); + xfer += oprot->writeI64(this->start_time); xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeFieldBegin("import_uuid", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeString(this->import_uuid); + if (this->__isset.end_time) + { + xfer += oprot->writeFieldBegin("end_time", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->end_time); xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeFieldBegin("msg_type", ::apache::thrift::protocol::T_I32, 3); - xfer += oprot->writeI32((int32_t)this->msg_type); + if (this->__isset.table_list) + { + xfer += oprot->writeFieldBegin("table_list", ::apache::thrift::protocol::T_LIST, 6); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, + static_cast(this->table_list.size())); + std::vector::const_iterator _iter14; + + for (_iter14 = this->table_list.begin(); _iter14 != this->table_list.end(); ++_iter14) + { + xfer += oprot->writeString((*_iter14)); + } + + xfer += oprot->writeListEnd(); + } xfer += oprot->writeFieldEnd(); + } - if (this->__isset.start_time) + if (this->__isset.rows_so_far) + { + xfer += oprot->writeFieldBegin("rows_so_far", ::apache::thrift::protocol::T_LIST, 7); { - xfer += oprot->writeFieldBegin("start_time", ::apache::thrift::protocol::T_I64, 4); - xfer += oprot->writeI64(this->start_time); - xfer += oprot->writeFieldEnd(); + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, + static_cast(this->rows_so_far.size())); + std::vector::const_iterator _iter15; + + for (_iter15 = this->rows_so_far.begin(); _iter15 != this->rows_so_far.end(); ++_iter15) + { + xfer += oprot->writeI64((*_iter15)); + } + + xfer += oprot->writeListEnd(); } + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.end_time) - { - xfer += oprot->writeFieldBegin("end_time", ::apache::thrift::protocol::T_I64, 5); - xfer += oprot->writeI64(this->end_time); - xfer += oprot->writeFieldEnd(); - } + if (this->__isset.system_name) + { + xfer += oprot->writeFieldBegin("system_name", ::apache::thrift::protocol::T_STRING, 8); + xfer += oprot->writeString(this->system_name); + xfer += oprot->writeFieldEnd(); + } - if (this->__isset.table_list) - { - xfer += oprot->writeFieldBegin("table_list", ::apache::thrift::protocol::T_LIST, 6); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->table_list.size())); - std::vector ::const_iterator _iter14; + if (this->__isset.module_name) + { + xfer += oprot->writeFieldBegin("module_name", ::apache::thrift::protocol::T_STRING, 9); + xfer += oprot->writeString(this->module_name); + xfer += oprot->writeFieldEnd(); + } - for (_iter14 = this->table_list.begin(); _iter14 != this->table_list.end(); ++_iter14) - { - xfer += oprot->writeString((*_iter14)); - } + if (this->__isset.schema_name) + { + xfer += oprot->writeFieldBegin("schema_name", ::apache::thrift::protocol::T_STRING, 10); + xfer += oprot->writeString(this->schema_name); + xfer += oprot->writeFieldEnd(); + } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.rows_so_far) - { - xfer += oprot->writeFieldBegin("rows_so_far", ::apache::thrift::protocol::T_LIST, 7); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->rows_so_far.size())); - std::vector ::const_iterator _iter15; - - for (_iter15 = this->rows_so_far.begin(); _iter15 != this->rows_so_far.end(); ++_iter15) - { - xfer += oprot->writeI64((*_iter15)); - } - - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.system_name) - { - xfer += oprot->writeFieldBegin("system_name", ::apache::thrift::protocol::T_STRING, 8); - xfer += oprot->writeString(this->system_name); - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.module_name) - { - xfer += oprot->writeFieldBegin("module_name", ::apache::thrift::protocol::T_STRING, 9); - xfer += oprot->writeString(this->module_name); - xfer += oprot->writeFieldEnd(); - } - - if (this->__isset.schema_name) - { - xfer += oprot->writeFieldBegin("schema_name", ::apache::thrift::protocol::T_STRING, 10); - xfer += oprot->writeString(this->schema_name); - xfer += oprot->writeFieldEnd(); - } - - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; } void swap(ImportTele& a, ImportTele& b) { - using ::std::swap; - swap(a.job_uuid, b.job_uuid); - swap(a.import_uuid, b.import_uuid); - swap(a.msg_type, b.msg_type); - swap(a.start_time, b.start_time); - swap(a.end_time, b.end_time); - swap(a.table_list, b.table_list); - swap(a.rows_so_far, b.rows_so_far); - swap(a.system_name, b.system_name); - swap(a.module_name, b.module_name); - swap(a.schema_name, b.schema_name); - swap(a.__isset, b.__isset); + using ::std::swap; + swap(a.job_uuid, b.job_uuid); + swap(a.import_uuid, b.import_uuid); + swap(a.msg_type, b.msg_type); + swap(a.start_time, b.start_time); + swap(a.end_time, b.end_time); + swap(a.table_list, b.table_list); + swap(a.rows_so_far, b.rows_so_far); + swap(a.system_name, b.system_name); + swap(a.module_name, b.module_name); + swap(a.schema_name, b.schema_name); + swap(a.__isset, b.__isset); } -} // namespace +} // namespace querytele diff --git a/utils/querytele/querytele_types.h b/utils/querytele/querytele_types.h index 9d8a530e3..44a9bb678 100644 --- a/utils/querytele/querytele_types.h +++ b/utils/querytele/querytele_types.h @@ -14,829 +14,936 @@ #include - namespace querytele { - struct QTType { - enum type - { - QT_INVALID = 0, - QT_SUMMARY = 1, - QT_PROGRESS = 2, - QT_START = 3 - }; + enum type + { + QT_INVALID = 0, + QT_SUMMARY = 1, + QT_PROGRESS = 2, + QT_START = 3 + }; }; extern const std::map _QTType_VALUES_TO_NAMES; struct STType { - enum type - { - ST_INVALID = 0, - ST_SUMMARY = 1, - ST_PROGRESS = 2, - ST_START = 3 - }; + enum type + { + ST_INVALID = 0, + ST_SUMMARY = 1, + ST_PROGRESS = 2, + ST_START = 3 + }; }; extern const std::map _STType_VALUES_TO_NAMES; struct StepType { - enum type - { - T_INVALID = 0, - T_HJS = 1, - T_DSS = 2, - T_CES = 3, - T_SQS = 4, - T_TAS = 5, - T_TNS = 6, - T_BPS = 7, - T_TCS = 8, - T_HVS = 9, - T_WFS = 10, - T_SAS = 11, - T_TUN = 12 - }; + enum type + { + T_INVALID = 0, + T_HJS = 1, + T_DSS = 2, + T_CES = 3, + T_SQS = 4, + T_TAS = 5, + T_TNS = 6, + T_BPS = 7, + T_TCS = 8, + T_HVS = 9, + T_WFS = 10, + T_SAS = 11, + T_TUN = 12 + }; }; extern const std::map _StepType_VALUES_TO_NAMES; struct ITType { - enum type - { - IT_INVALID = 0, - IT_SUMMARY = 1, - IT_PROGRESS = 2, - IT_START = 3, - IT_TERM = 4 - }; + enum type + { + IT_INVALID = 0, + IT_SUMMARY = 1, + IT_PROGRESS = 2, + IT_START = 3, + IT_TERM = 4 + }; }; extern const std::map _ITType_VALUES_TO_NAMES; -typedef std::vector StringList; +typedef std::vector StringList; -typedef std::vector I64List; +typedef std::vector I64List; typedef struct _QueryTele__isset { - _QueryTele__isset() : query_uuid(false), msg_type(false), max_mem_pct(false), num_files(false), phy_io(false), cache_io(false), msg_rcv_cnt(false), cp_blocks_skipped(false), msg_bytes_in(false), msg_bytes_out(false), rows(false), start_time(false), end_time(false), error_no(false), blocks_changed(false), session_id(false), query_type(false), query(false), user(false), host(false), priority(false), priority_level(false), system_name(false), module_name(false), local_query(false), schema_name(false) {} - bool query_uuid; - bool msg_type; - bool max_mem_pct; - bool num_files; - bool phy_io; - bool cache_io; - bool msg_rcv_cnt; - bool cp_blocks_skipped; - bool msg_bytes_in; - bool msg_bytes_out; - bool rows; - bool start_time; - bool end_time; - bool error_no; - bool blocks_changed; - bool session_id; - bool query_type; - bool query; - bool user; - bool host; - bool priority; - bool priority_level; - bool system_name; - bool module_name; - bool local_query; - bool schema_name; + _QueryTele__isset() + : query_uuid(false) + , msg_type(false) + , max_mem_pct(false) + , num_files(false) + , phy_io(false) + , cache_io(false) + , msg_rcv_cnt(false) + , cp_blocks_skipped(false) + , msg_bytes_in(false) + , msg_bytes_out(false) + , rows(false) + , start_time(false) + , end_time(false) + , error_no(false) + , blocks_changed(false) + , session_id(false) + , query_type(false) + , query(false) + , user(false) + , host(false) + , priority(false) + , priority_level(false) + , system_name(false) + , module_name(false) + , local_query(false) + , schema_name(false) + { + } + bool query_uuid; + bool msg_type; + bool max_mem_pct; + bool num_files; + bool phy_io; + bool cache_io; + bool msg_rcv_cnt; + bool cp_blocks_skipped; + bool msg_bytes_in; + bool msg_bytes_out; + bool rows; + bool start_time; + bool end_time; + bool error_no; + bool blocks_changed; + bool session_id; + bool query_type; + bool query; + bool user; + bool host; + bool priority; + bool priority_level; + bool system_name; + bool module_name; + bool local_query; + bool schema_name; } _QueryTele__isset; class QueryTele { -public: + public: + static const char* ascii_fingerprint; // = "E179941E2BE9C920811D86261E5AAE67"; + static const uint8_t binary_fingerprint + [16]; // = {0xE1,0x79,0x94,0x1E,0x2B,0xE9,0xC9,0x20,0x81,0x1D,0x86,0x26,0x1E,0x5A,0xAE,0x67}; - static const char* ascii_fingerprint; // = "E179941E2BE9C920811D86261E5AAE67"; - static const uint8_t binary_fingerprint[16]; // = {0xE1,0x79,0x94,0x1E,0x2B,0xE9,0xC9,0x20,0x81,0x1D,0x86,0x26,0x1E,0x5A,0xAE,0x67}; + QueryTele() + : query_uuid() + , msg_type((QTType::type)0) + , max_mem_pct(0) + , num_files(0) + , phy_io(0) + , cache_io(0) + , msg_rcv_cnt(0) + , cp_blocks_skipped(0) + , msg_bytes_in(0) + , msg_bytes_out(0) + , rows(0) + , start_time(0) + , end_time(0) + , error_no(0) + , blocks_changed(0) + , session_id(0) + , query_type() + , query() + , user() + , host() + , priority() + , priority_level(0) + , system_name() + , module_name() + , local_query(0) + , schema_name() + { + } - QueryTele() : query_uuid(), msg_type((QTType::type)0), max_mem_pct(0), num_files(0), phy_io(0), cache_io(0), msg_rcv_cnt(0), cp_blocks_skipped(0), msg_bytes_in(0), msg_bytes_out(0), rows(0), start_time(0), end_time(0), error_no(0), blocks_changed(0), session_id(0), query_type(), query(), user(), host(), priority(), priority_level(0), system_name(), module_name(), local_query(0), schema_name() - { - } + virtual ~QueryTele() throw() + { + } - virtual ~QueryTele() throw() {} + std::string query_uuid; + QTType::type msg_type; + int64_t max_mem_pct; + int64_t num_files; + int64_t phy_io; + int64_t cache_io; + int64_t msg_rcv_cnt; + int64_t cp_blocks_skipped; + int64_t msg_bytes_in; + int64_t msg_bytes_out; + int64_t rows; + int64_t start_time; + int64_t end_time; + int64_t error_no; + int64_t blocks_changed; + int64_t session_id; + std::string query_type; + std::string query; + std::string user; + std::string host; + std::string priority; + int32_t priority_level; + std::string system_name; + std::string module_name; + int32_t local_query; + std::string schema_name; - std::string query_uuid; - QTType::type msg_type; - int64_t max_mem_pct; - int64_t num_files; - int64_t phy_io; - int64_t cache_io; - int64_t msg_rcv_cnt; - int64_t cp_blocks_skipped; - int64_t msg_bytes_in; - int64_t msg_bytes_out; - int64_t rows; - int64_t start_time; - int64_t end_time; - int64_t error_no; - int64_t blocks_changed; - int64_t session_id; - std::string query_type; - std::string query; - std::string user; - std::string host; - std::string priority; - int32_t priority_level; - std::string system_name; - std::string module_name; - int32_t local_query; - std::string schema_name; + _QueryTele__isset __isset; - _QueryTele__isset __isset; + void __set_query_uuid(const std::string& val) + { + query_uuid = val; + } - void __set_query_uuid(const std::string& val) - { - query_uuid = val; - } + void __set_msg_type(const QTType::type val) + { + msg_type = val; + } - void __set_msg_type(const QTType::type val) - { - msg_type = val; - } + void __set_max_mem_pct(const int64_t val) + { + max_mem_pct = val; + __isset.max_mem_pct = true; + } - void __set_max_mem_pct(const int64_t val) - { - max_mem_pct = val; - __isset.max_mem_pct = true; - } + void __set_num_files(const int64_t val) + { + num_files = val; + __isset.num_files = true; + } - void __set_num_files(const int64_t val) - { - num_files = val; - __isset.num_files = true; - } + void __set_phy_io(const int64_t val) + { + phy_io = val; + __isset.phy_io = true; + } - void __set_phy_io(const int64_t val) - { - phy_io = val; - __isset.phy_io = true; - } + void __set_cache_io(const int64_t val) + { + cache_io = val; + __isset.cache_io = true; + } - void __set_cache_io(const int64_t val) - { - cache_io = val; - __isset.cache_io = true; - } + void __set_msg_rcv_cnt(const int64_t val) + { + msg_rcv_cnt = val; + __isset.msg_rcv_cnt = true; + } - void __set_msg_rcv_cnt(const int64_t val) - { - msg_rcv_cnt = val; - __isset.msg_rcv_cnt = true; - } + void __set_cp_blocks_skipped(const int64_t val) + { + cp_blocks_skipped = val; + __isset.cp_blocks_skipped = true; + } - void __set_cp_blocks_skipped(const int64_t val) - { - cp_blocks_skipped = val; - __isset.cp_blocks_skipped = true; - } + void __set_msg_bytes_in(const int64_t val) + { + msg_bytes_in = val; + __isset.msg_bytes_in = true; + } - void __set_msg_bytes_in(const int64_t val) - { - msg_bytes_in = val; - __isset.msg_bytes_in = true; - } + void __set_msg_bytes_out(const int64_t val) + { + msg_bytes_out = val; + __isset.msg_bytes_out = true; + } - void __set_msg_bytes_out(const int64_t val) - { - msg_bytes_out = val; - __isset.msg_bytes_out = true; - } + void __set_rows(const int64_t val) + { + rows = val; + __isset.rows = true; + } - void __set_rows(const int64_t val) - { - rows = val; - __isset.rows = true; - } + void __set_start_time(const int64_t val) + { + start_time = val; + __isset.start_time = true; + } - void __set_start_time(const int64_t val) - { - start_time = val; - __isset.start_time = true; - } + void __set_end_time(const int64_t val) + { + end_time = val; + __isset.end_time = true; + } - void __set_end_time(const int64_t val) - { - end_time = val; - __isset.end_time = true; - } + void __set_error_no(const int64_t val) + { + error_no = val; + __isset.error_no = true; + } - void __set_error_no(const int64_t val) - { - error_no = val; - __isset.error_no = true; - } + void __set_blocks_changed(const int64_t val) + { + blocks_changed = val; + __isset.blocks_changed = true; + } - void __set_blocks_changed(const int64_t val) - { - blocks_changed = val; - __isset.blocks_changed = true; - } + void __set_session_id(const int64_t val) + { + session_id = val; + __isset.session_id = true; + } - void __set_session_id(const int64_t val) - { - session_id = val; - __isset.session_id = true; - } + void __set_query_type(const std::string& val) + { + query_type = val; + __isset.query_type = true; + } - void __set_query_type(const std::string& val) - { - query_type = val; - __isset.query_type = true; - } + void __set_query(const std::string& val) + { + query = val; + __isset.query = true; + } - void __set_query(const std::string& val) - { - query = val; - __isset.query = true; - } + void __set_user(const std::string& val) + { + user = val; + __isset.user = true; + } - void __set_user(const std::string& val) - { - user = val; - __isset.user = true; - } + void __set_host(const std::string& val) + { + host = val; + __isset.host = true; + } - void __set_host(const std::string& val) - { - host = val; - __isset.host = true; - } + void __set_priority(const std::string& val) + { + priority = val; + __isset.priority = true; + } - void __set_priority(const std::string& val) - { - priority = val; - __isset.priority = true; - } + void __set_priority_level(const int32_t val) + { + priority_level = val; + __isset.priority_level = true; + } - void __set_priority_level(const int32_t val) - { - priority_level = val; - __isset.priority_level = true; - } + void __set_system_name(const std::string& val) + { + system_name = val; + __isset.system_name = true; + } - void __set_system_name(const std::string& val) - { - system_name = val; - __isset.system_name = true; - } + void __set_module_name(const std::string& val) + { + module_name = val; + __isset.module_name = true; + } - void __set_module_name(const std::string& val) - { - module_name = val; - __isset.module_name = true; - } + void __set_local_query(const int32_t val) + { + local_query = val; + __isset.local_query = true; + } - void __set_local_query(const int32_t val) - { - local_query = val; - __isset.local_query = true; - } + void __set_schema_name(const std::string& val) + { + schema_name = val; + __isset.schema_name = true; + } - void __set_schema_name(const std::string& val) - { - schema_name = val; - __isset.schema_name = true; - } + bool operator==(const QueryTele& rhs) const + { + if (!(query_uuid == rhs.query_uuid)) + return false; - bool operator == (const QueryTele& rhs) const - { - if (!(query_uuid == rhs.query_uuid)) - return false; + if (!(msg_type == rhs.msg_type)) + return false; - if (!(msg_type == rhs.msg_type)) - return false; + if (__isset.max_mem_pct != rhs.__isset.max_mem_pct) + return false; + else if (__isset.max_mem_pct && !(max_mem_pct == rhs.max_mem_pct)) + return false; - if (__isset.max_mem_pct != rhs.__isset.max_mem_pct) - return false; - else if (__isset.max_mem_pct && !(max_mem_pct == rhs.max_mem_pct)) - return false; + if (__isset.num_files != rhs.__isset.num_files) + return false; + else if (__isset.num_files && !(num_files == rhs.num_files)) + return false; - if (__isset.num_files != rhs.__isset.num_files) - return false; - else if (__isset.num_files && !(num_files == rhs.num_files)) - return false; + if (__isset.phy_io != rhs.__isset.phy_io) + return false; + else if (__isset.phy_io && !(phy_io == rhs.phy_io)) + return false; - if (__isset.phy_io != rhs.__isset.phy_io) - return false; - else if (__isset.phy_io && !(phy_io == rhs.phy_io)) - return false; + if (__isset.cache_io != rhs.__isset.cache_io) + return false; + else if (__isset.cache_io && !(cache_io == rhs.cache_io)) + return false; - if (__isset.cache_io != rhs.__isset.cache_io) - return false; - else if (__isset.cache_io && !(cache_io == rhs.cache_io)) - return false; + if (__isset.msg_rcv_cnt != rhs.__isset.msg_rcv_cnt) + return false; + else if (__isset.msg_rcv_cnt && !(msg_rcv_cnt == rhs.msg_rcv_cnt)) + return false; - if (__isset.msg_rcv_cnt != rhs.__isset.msg_rcv_cnt) - return false; - else if (__isset.msg_rcv_cnt && !(msg_rcv_cnt == rhs.msg_rcv_cnt)) - return false; + if (__isset.cp_blocks_skipped != rhs.__isset.cp_blocks_skipped) + return false; + else if (__isset.cp_blocks_skipped && !(cp_blocks_skipped == rhs.cp_blocks_skipped)) + return false; - if (__isset.cp_blocks_skipped != rhs.__isset.cp_blocks_skipped) - return false; - else if (__isset.cp_blocks_skipped && !(cp_blocks_skipped == rhs.cp_blocks_skipped)) - return false; + if (__isset.msg_bytes_in != rhs.__isset.msg_bytes_in) + return false; + else if (__isset.msg_bytes_in && !(msg_bytes_in == rhs.msg_bytes_in)) + return false; - if (__isset.msg_bytes_in != rhs.__isset.msg_bytes_in) - return false; - else if (__isset.msg_bytes_in && !(msg_bytes_in == rhs.msg_bytes_in)) - return false; + if (__isset.msg_bytes_out != rhs.__isset.msg_bytes_out) + return false; + else if (__isset.msg_bytes_out && !(msg_bytes_out == rhs.msg_bytes_out)) + return false; - if (__isset.msg_bytes_out != rhs.__isset.msg_bytes_out) - return false; - else if (__isset.msg_bytes_out && !(msg_bytes_out == rhs.msg_bytes_out)) - return false; + if (__isset.rows != rhs.__isset.rows) + return false; + else if (__isset.rows && !(rows == rhs.rows)) + return false; - if (__isset.rows != rhs.__isset.rows) - return false; - else if (__isset.rows && !(rows == rhs.rows)) - return false; + if (__isset.start_time != rhs.__isset.start_time) + return false; + else if (__isset.start_time && !(start_time == rhs.start_time)) + return false; - if (__isset.start_time != rhs.__isset.start_time) - return false; - else if (__isset.start_time && !(start_time == rhs.start_time)) - return false; + if (__isset.end_time != rhs.__isset.end_time) + return false; + else if (__isset.end_time && !(end_time == rhs.end_time)) + return false; - if (__isset.end_time != rhs.__isset.end_time) - return false; - else if (__isset.end_time && !(end_time == rhs.end_time)) - return false; + if (__isset.error_no != rhs.__isset.error_no) + return false; + else if (__isset.error_no && !(error_no == rhs.error_no)) + return false; - if (__isset.error_no != rhs.__isset.error_no) - return false; - else if (__isset.error_no && !(error_no == rhs.error_no)) - return false; + if (__isset.blocks_changed != rhs.__isset.blocks_changed) + return false; + else if (__isset.blocks_changed && !(blocks_changed == rhs.blocks_changed)) + return false; - if (__isset.blocks_changed != rhs.__isset.blocks_changed) - return false; - else if (__isset.blocks_changed && !(blocks_changed == rhs.blocks_changed)) - return false; + if (__isset.session_id != rhs.__isset.session_id) + return false; + else if (__isset.session_id && !(session_id == rhs.session_id)) + return false; - if (__isset.session_id != rhs.__isset.session_id) - return false; - else if (__isset.session_id && !(session_id == rhs.session_id)) - return false; + if (__isset.query_type != rhs.__isset.query_type) + return false; + else if (__isset.query_type && !(query_type == rhs.query_type)) + return false; - if (__isset.query_type != rhs.__isset.query_type) - return false; - else if (__isset.query_type && !(query_type == rhs.query_type)) - return false; + if (__isset.query != rhs.__isset.query) + return false; + else if (__isset.query && !(query == rhs.query)) + return false; - if (__isset.query != rhs.__isset.query) - return false; - else if (__isset.query && !(query == rhs.query)) - return false; + if (__isset.user != rhs.__isset.user) + return false; + else if (__isset.user && !(user == rhs.user)) + return false; - if (__isset.user != rhs.__isset.user) - return false; - else if (__isset.user && !(user == rhs.user)) - return false; + if (__isset.host != rhs.__isset.host) + return false; + else if (__isset.host && !(host == rhs.host)) + return false; - if (__isset.host != rhs.__isset.host) - return false; - else if (__isset.host && !(host == rhs.host)) - return false; + if (__isset.priority != rhs.__isset.priority) + return false; + else if (__isset.priority && !(priority == rhs.priority)) + return false; - if (__isset.priority != rhs.__isset.priority) - return false; - else if (__isset.priority && !(priority == rhs.priority)) - return false; + if (__isset.priority_level != rhs.__isset.priority_level) + return false; + else if (__isset.priority_level && !(priority_level == rhs.priority_level)) + return false; - if (__isset.priority_level != rhs.__isset.priority_level) - return false; - else if (__isset.priority_level && !(priority_level == rhs.priority_level)) - return false; + if (__isset.system_name != rhs.__isset.system_name) + return false; + else if (__isset.system_name && !(system_name == rhs.system_name)) + return false; - if (__isset.system_name != rhs.__isset.system_name) - return false; - else if (__isset.system_name && !(system_name == rhs.system_name)) - return false; + if (__isset.module_name != rhs.__isset.module_name) + return false; + else if (__isset.module_name && !(module_name == rhs.module_name)) + return false; - if (__isset.module_name != rhs.__isset.module_name) - return false; - else if (__isset.module_name && !(module_name == rhs.module_name)) - return false; + if (__isset.local_query != rhs.__isset.local_query) + return false; + else if (__isset.local_query && !(local_query == rhs.local_query)) + return false; - if (__isset.local_query != rhs.__isset.local_query) - return false; - else if (__isset.local_query && !(local_query == rhs.local_query)) - return false; + if (__isset.schema_name != rhs.__isset.schema_name) + return false; + else if (__isset.schema_name && !(schema_name == rhs.schema_name)) + return false; - if (__isset.schema_name != rhs.__isset.schema_name) - return false; - else if (__isset.schema_name && !(schema_name == rhs.schema_name)) - return false; + return true; + } + bool operator!=(const QueryTele& rhs) const + { + return !(*this == rhs); + } - return true; - } - bool operator != (const QueryTele& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const QueryTele& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + bool operator<(const QueryTele&) const; + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; void swap(QueryTele& a, QueryTele& b); typedef struct _StepTele__isset { - _StepTele__isset() : query_uuid(false), msg_type(false), step_type(false), step_uuid(false), phy_io(false), cache_io(false), msg_rcv_cnt(false), cp_blocks_skipped(false), msg_bytes_in(false), msg_bytes_out(false), rows(false), start_time(false), end_time(false), total_units_of_work(false), units_of_work_completed(false) {} - bool query_uuid; - bool msg_type; - bool step_type; - bool step_uuid; - bool phy_io; - bool cache_io; - bool msg_rcv_cnt; - bool cp_blocks_skipped; - bool msg_bytes_in; - bool msg_bytes_out; - bool rows; - bool start_time; - bool end_time; - bool total_units_of_work; - bool units_of_work_completed; + _StepTele__isset() + : query_uuid(false) + , msg_type(false) + , step_type(false) + , step_uuid(false) + , phy_io(false) + , cache_io(false) + , msg_rcv_cnt(false) + , cp_blocks_skipped(false) + , msg_bytes_in(false) + , msg_bytes_out(false) + , rows(false) + , start_time(false) + , end_time(false) + , total_units_of_work(false) + , units_of_work_completed(false) + { + } + bool query_uuid; + bool msg_type; + bool step_type; + bool step_uuid; + bool phy_io; + bool cache_io; + bool msg_rcv_cnt; + bool cp_blocks_skipped; + bool msg_bytes_in; + bool msg_bytes_out; + bool rows; + bool start_time; + bool end_time; + bool total_units_of_work; + bool units_of_work_completed; } _StepTele__isset; class StepTele { -public: + public: + static const char* ascii_fingerprint; // = "4E40C17AE92DFF833A8CC23318F4FE68"; + static const uint8_t binary_fingerprint + [16]; // = {0x4E,0x40,0xC1,0x7A,0xE9,0x2D,0xFF,0x83,0x3A,0x8C,0xC2,0x33,0x18,0xF4,0xFE,0x68}; - static const char* ascii_fingerprint; // = "4E40C17AE92DFF833A8CC23318F4FE68"; - static const uint8_t binary_fingerprint[16]; // = {0x4E,0x40,0xC1,0x7A,0xE9,0x2D,0xFF,0x83,0x3A,0x8C,0xC2,0x33,0x18,0xF4,0xFE,0x68}; + StepTele() + : query_uuid() + , msg_type((STType::type)0) + , step_type((StepType::type)0) + , step_uuid() + , phy_io(0) + , cache_io(0) + , msg_rcv_cnt(0) + , cp_blocks_skipped(0) + , msg_bytes_in(0) + , msg_bytes_out(0) + , rows(0) + , start_time(0) + , end_time(0) + , total_units_of_work(0) + , units_of_work_completed(0) + { + } - StepTele() : query_uuid(), msg_type((STType::type)0), step_type((StepType::type)0), step_uuid(), phy_io(0), cache_io(0), msg_rcv_cnt(0), cp_blocks_skipped(0), msg_bytes_in(0), msg_bytes_out(0), rows(0), start_time(0), end_time(0), total_units_of_work(0), units_of_work_completed(0) - { - } + virtual ~StepTele() throw() + { + } - virtual ~StepTele() throw() {} + std::string query_uuid; + STType::type msg_type; + StepType::type step_type; + std::string step_uuid; + int64_t phy_io; + int64_t cache_io; + int64_t msg_rcv_cnt; + int64_t cp_blocks_skipped; + int64_t msg_bytes_in; + int64_t msg_bytes_out; + int64_t rows; + int64_t start_time; + int64_t end_time; + int32_t total_units_of_work; + int32_t units_of_work_completed; - std::string query_uuid; - STType::type msg_type; - StepType::type step_type; - std::string step_uuid; - int64_t phy_io; - int64_t cache_io; - int64_t msg_rcv_cnt; - int64_t cp_blocks_skipped; - int64_t msg_bytes_in; - int64_t msg_bytes_out; - int64_t rows; - int64_t start_time; - int64_t end_time; - int32_t total_units_of_work; - int32_t units_of_work_completed; + _StepTele__isset __isset; - _StepTele__isset __isset; + void __set_query_uuid(const std::string& val) + { + query_uuid = val; + } - void __set_query_uuid(const std::string& val) - { - query_uuid = val; - } + void __set_msg_type(const STType::type val) + { + msg_type = val; + } - void __set_msg_type(const STType::type val) - { - msg_type = val; - } + void __set_step_type(const StepType::type val) + { + step_type = val; + } - void __set_step_type(const StepType::type val) - { - step_type = val; - } + void __set_step_uuid(const std::string& val) + { + step_uuid = val; + } - void __set_step_uuid(const std::string& val) - { - step_uuid = val; - } + void __set_phy_io(const int64_t val) + { + phy_io = val; + __isset.phy_io = true; + } - void __set_phy_io(const int64_t val) - { - phy_io = val; - __isset.phy_io = true; - } + void __set_cache_io(const int64_t val) + { + cache_io = val; + __isset.cache_io = true; + } - void __set_cache_io(const int64_t val) - { - cache_io = val; - __isset.cache_io = true; - } + void __set_msg_rcv_cnt(const int64_t val) + { + msg_rcv_cnt = val; + __isset.msg_rcv_cnt = true; + } - void __set_msg_rcv_cnt(const int64_t val) - { - msg_rcv_cnt = val; - __isset.msg_rcv_cnt = true; - } + void __set_cp_blocks_skipped(const int64_t val) + { + cp_blocks_skipped = val; + __isset.cp_blocks_skipped = true; + } - void __set_cp_blocks_skipped(const int64_t val) - { - cp_blocks_skipped = val; - __isset.cp_blocks_skipped = true; - } + void __set_msg_bytes_in(const int64_t val) + { + msg_bytes_in = val; + __isset.msg_bytes_in = true; + } - void __set_msg_bytes_in(const int64_t val) - { - msg_bytes_in = val; - __isset.msg_bytes_in = true; - } + void __set_msg_bytes_out(const int64_t val) + { + msg_bytes_out = val; + __isset.msg_bytes_out = true; + } - void __set_msg_bytes_out(const int64_t val) - { - msg_bytes_out = val; - __isset.msg_bytes_out = true; - } + void __set_rows(const int64_t val) + { + rows = val; + __isset.rows = true; + } - void __set_rows(const int64_t val) - { - rows = val; - __isset.rows = true; - } + void __set_start_time(const int64_t val) + { + start_time = val; + __isset.start_time = true; + } - void __set_start_time(const int64_t val) - { - start_time = val; - __isset.start_time = true; - } + void __set_end_time(const int64_t val) + { + end_time = val; + __isset.end_time = true; + } - void __set_end_time(const int64_t val) - { - end_time = val; - __isset.end_time = true; - } + void __set_total_units_of_work(const int32_t val) + { + total_units_of_work = val; + __isset.total_units_of_work = true; + } - void __set_total_units_of_work(const int32_t val) - { - total_units_of_work = val; - __isset.total_units_of_work = true; - } + void __set_units_of_work_completed(const int32_t val) + { + units_of_work_completed = val; + __isset.units_of_work_completed = true; + } - void __set_units_of_work_completed(const int32_t val) - { - units_of_work_completed = val; - __isset.units_of_work_completed = true; - } + bool operator==(const StepTele& rhs) const + { + if (!(query_uuid == rhs.query_uuid)) + return false; - bool operator == (const StepTele& rhs) const - { - if (!(query_uuid == rhs.query_uuid)) - return false; + if (!(msg_type == rhs.msg_type)) + return false; - if (!(msg_type == rhs.msg_type)) - return false; + if (!(step_type == rhs.step_type)) + return false; - if (!(step_type == rhs.step_type)) - return false; + if (!(step_uuid == rhs.step_uuid)) + return false; - if (!(step_uuid == rhs.step_uuid)) - return false; + if (__isset.phy_io != rhs.__isset.phy_io) + return false; + else if (__isset.phy_io && !(phy_io == rhs.phy_io)) + return false; - if (__isset.phy_io != rhs.__isset.phy_io) - return false; - else if (__isset.phy_io && !(phy_io == rhs.phy_io)) - return false; + if (__isset.cache_io != rhs.__isset.cache_io) + return false; + else if (__isset.cache_io && !(cache_io == rhs.cache_io)) + return false; - if (__isset.cache_io != rhs.__isset.cache_io) - return false; - else if (__isset.cache_io && !(cache_io == rhs.cache_io)) - return false; + if (__isset.msg_rcv_cnt != rhs.__isset.msg_rcv_cnt) + return false; + else if (__isset.msg_rcv_cnt && !(msg_rcv_cnt == rhs.msg_rcv_cnt)) + return false; - if (__isset.msg_rcv_cnt != rhs.__isset.msg_rcv_cnt) - return false; - else if (__isset.msg_rcv_cnt && !(msg_rcv_cnt == rhs.msg_rcv_cnt)) - return false; + if (__isset.cp_blocks_skipped != rhs.__isset.cp_blocks_skipped) + return false; + else if (__isset.cp_blocks_skipped && !(cp_blocks_skipped == rhs.cp_blocks_skipped)) + return false; - if (__isset.cp_blocks_skipped != rhs.__isset.cp_blocks_skipped) - return false; - else if (__isset.cp_blocks_skipped && !(cp_blocks_skipped == rhs.cp_blocks_skipped)) - return false; + if (__isset.msg_bytes_in != rhs.__isset.msg_bytes_in) + return false; + else if (__isset.msg_bytes_in && !(msg_bytes_in == rhs.msg_bytes_in)) + return false; - if (__isset.msg_bytes_in != rhs.__isset.msg_bytes_in) - return false; - else if (__isset.msg_bytes_in && !(msg_bytes_in == rhs.msg_bytes_in)) - return false; + if (__isset.msg_bytes_out != rhs.__isset.msg_bytes_out) + return false; + else if (__isset.msg_bytes_out && !(msg_bytes_out == rhs.msg_bytes_out)) + return false; - if (__isset.msg_bytes_out != rhs.__isset.msg_bytes_out) - return false; - else if (__isset.msg_bytes_out && !(msg_bytes_out == rhs.msg_bytes_out)) - return false; + if (__isset.rows != rhs.__isset.rows) + return false; + else if (__isset.rows && !(rows == rhs.rows)) + return false; - if (__isset.rows != rhs.__isset.rows) - return false; - else if (__isset.rows && !(rows == rhs.rows)) - return false; + if (__isset.start_time != rhs.__isset.start_time) + return false; + else if (__isset.start_time && !(start_time == rhs.start_time)) + return false; - if (__isset.start_time != rhs.__isset.start_time) - return false; - else if (__isset.start_time && !(start_time == rhs.start_time)) - return false; + if (__isset.end_time != rhs.__isset.end_time) + return false; + else if (__isset.end_time && !(end_time == rhs.end_time)) + return false; - if (__isset.end_time != rhs.__isset.end_time) - return false; - else if (__isset.end_time && !(end_time == rhs.end_time)) - return false; + if (__isset.total_units_of_work != rhs.__isset.total_units_of_work) + return false; + else if (__isset.total_units_of_work && !(total_units_of_work == rhs.total_units_of_work)) + return false; - if (__isset.total_units_of_work != rhs.__isset.total_units_of_work) - return false; - else if (__isset.total_units_of_work && !(total_units_of_work == rhs.total_units_of_work)) - return false; + if (__isset.units_of_work_completed != rhs.__isset.units_of_work_completed) + return false; + else if (__isset.units_of_work_completed && !(units_of_work_completed == rhs.units_of_work_completed)) + return false; - if (__isset.units_of_work_completed != rhs.__isset.units_of_work_completed) - return false; - else if (__isset.units_of_work_completed && !(units_of_work_completed == rhs.units_of_work_completed)) - return false; + return true; + } + bool operator!=(const StepTele& rhs) const + { + return !(*this == rhs); + } - return true; - } - bool operator != (const StepTele& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const StepTele& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + bool operator<(const StepTele&) const; + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; void swap(StepTele& a, StepTele& b); typedef struct _ImportTele__isset { - _ImportTele__isset() : job_uuid(false), import_uuid(false), msg_type(false), start_time(false), end_time(false), table_list(false), rows_so_far(false), system_name(false), module_name(false), schema_name(false) {} - bool job_uuid; - bool import_uuid; - bool msg_type; - bool start_time; - bool end_time; - bool table_list; - bool rows_so_far; - bool system_name; - bool module_name; - bool schema_name; + _ImportTele__isset() + : job_uuid(false) + , import_uuid(false) + , msg_type(false) + , start_time(false) + , end_time(false) + , table_list(false) + , rows_so_far(false) + , system_name(false) + , module_name(false) + , schema_name(false) + { + } + bool job_uuid; + bool import_uuid; + bool msg_type; + bool start_time; + bool end_time; + bool table_list; + bool rows_so_far; + bool system_name; + bool module_name; + bool schema_name; } _ImportTele__isset; class ImportTele { -public: + public: + static const char* ascii_fingerprint; // = "9C47642F47F2AFBEA98238934EB1A211"; + static const uint8_t binary_fingerprint + [16]; // = {0x9C,0x47,0x64,0x2F,0x47,0xF2,0xAF,0xBE,0xA9,0x82,0x38,0x93,0x4E,0xB1,0xA2,0x11}; - static const char* ascii_fingerprint; // = "9C47642F47F2AFBEA98238934EB1A211"; - static const uint8_t binary_fingerprint[16]; // = {0x9C,0x47,0x64,0x2F,0x47,0xF2,0xAF,0xBE,0xA9,0x82,0x38,0x93,0x4E,0xB1,0xA2,0x11}; + ImportTele() + : job_uuid() + , import_uuid() + , msg_type((ITType::type)0) + , start_time(0) + , end_time(0) + , system_name() + , module_name() + , schema_name() + { + } - ImportTele() : job_uuid(), import_uuid(), msg_type((ITType::type)0), start_time(0), end_time(0), system_name(), module_name(), schema_name() - { - } + virtual ~ImportTele() throw() + { + } - virtual ~ImportTele() throw() {} + std::string job_uuid; + std::string import_uuid; + ITType::type msg_type; + int64_t start_time; + int64_t end_time; + StringList table_list; + I64List rows_so_far; + std::string system_name; + std::string module_name; + std::string schema_name; - std::string job_uuid; - std::string import_uuid; - ITType::type msg_type; - int64_t start_time; - int64_t end_time; - StringList table_list; - I64List rows_so_far; - std::string system_name; - std::string module_name; - std::string schema_name; + _ImportTele__isset __isset; - _ImportTele__isset __isset; + void __set_job_uuid(const std::string& val) + { + job_uuid = val; + } - void __set_job_uuid(const std::string& val) - { - job_uuid = val; - } + void __set_import_uuid(const std::string& val) + { + import_uuid = val; + } - void __set_import_uuid(const std::string& val) - { - import_uuid = val; - } + void __set_msg_type(const ITType::type val) + { + msg_type = val; + } - void __set_msg_type(const ITType::type val) - { - msg_type = val; - } + void __set_start_time(const int64_t val) + { + start_time = val; + __isset.start_time = true; + } - void __set_start_time(const int64_t val) - { - start_time = val; - __isset.start_time = true; - } + void __set_end_time(const int64_t val) + { + end_time = val; + __isset.end_time = true; + } - void __set_end_time(const int64_t val) - { - end_time = val; - __isset.end_time = true; - } + void __set_table_list(const StringList& val) + { + table_list = val; + __isset.table_list = true; + } - void __set_table_list(const StringList& val) - { - table_list = val; - __isset.table_list = true; - } + void __set_rows_so_far(const I64List& val) + { + rows_so_far = val; + __isset.rows_so_far = true; + } - void __set_rows_so_far(const I64List& val) - { - rows_so_far = val; - __isset.rows_so_far = true; - } + void __set_system_name(const std::string& val) + { + system_name = val; + __isset.system_name = true; + } - void __set_system_name(const std::string& val) - { - system_name = val; - __isset.system_name = true; - } + void __set_module_name(const std::string& val) + { + module_name = val; + __isset.module_name = true; + } - void __set_module_name(const std::string& val) - { - module_name = val; - __isset.module_name = true; - } + void __set_schema_name(const std::string& val) + { + schema_name = val; + __isset.schema_name = true; + } - void __set_schema_name(const std::string& val) - { - schema_name = val; - __isset.schema_name = true; - } + bool operator==(const ImportTele& rhs) const + { + if (!(job_uuid == rhs.job_uuid)) + return false; - bool operator == (const ImportTele& rhs) const - { - if (!(job_uuid == rhs.job_uuid)) - return false; + if (!(import_uuid == rhs.import_uuid)) + return false; - if (!(import_uuid == rhs.import_uuid)) - return false; + if (!(msg_type == rhs.msg_type)) + return false; - if (!(msg_type == rhs.msg_type)) - return false; + if (__isset.start_time != rhs.__isset.start_time) + return false; + else if (__isset.start_time && !(start_time == rhs.start_time)) + return false; - if (__isset.start_time != rhs.__isset.start_time) - return false; - else if (__isset.start_time && !(start_time == rhs.start_time)) - return false; + if (__isset.end_time != rhs.__isset.end_time) + return false; + else if (__isset.end_time && !(end_time == rhs.end_time)) + return false; - if (__isset.end_time != rhs.__isset.end_time) - return false; - else if (__isset.end_time && !(end_time == rhs.end_time)) - return false; + if (__isset.table_list != rhs.__isset.table_list) + return false; + else if (__isset.table_list && !(table_list == rhs.table_list)) + return false; - if (__isset.table_list != rhs.__isset.table_list) - return false; - else if (__isset.table_list && !(table_list == rhs.table_list)) - return false; + if (__isset.rows_so_far != rhs.__isset.rows_so_far) + return false; + else if (__isset.rows_so_far && !(rows_so_far == rhs.rows_so_far)) + return false; - if (__isset.rows_so_far != rhs.__isset.rows_so_far) - return false; - else if (__isset.rows_so_far && !(rows_so_far == rhs.rows_so_far)) - return false; + if (__isset.system_name != rhs.__isset.system_name) + return false; + else if (__isset.system_name && !(system_name == rhs.system_name)) + return false; - if (__isset.system_name != rhs.__isset.system_name) - return false; - else if (__isset.system_name && !(system_name == rhs.system_name)) - return false; + if (__isset.module_name != rhs.__isset.module_name) + return false; + else if (__isset.module_name && !(module_name == rhs.module_name)) + return false; - if (__isset.module_name != rhs.__isset.module_name) - return false; - else if (__isset.module_name && !(module_name == rhs.module_name)) - return false; + if (__isset.schema_name != rhs.__isset.schema_name) + return false; + else if (__isset.schema_name && !(schema_name == rhs.schema_name)) + return false; - if (__isset.schema_name != rhs.__isset.schema_name) - return false; - else if (__isset.schema_name && !(schema_name == rhs.schema_name)) - return false; + return true; + } + bool operator!=(const ImportTele& rhs) const + { + return !(*this == rhs); + } - return true; - } - bool operator != (const ImportTele& rhs) const - { - return !(*this == rhs); - } - - bool operator < (const ImportTele& ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + bool operator<(const ImportTele&) const; + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; }; void swap(ImportTele& a, ImportTele& b); -} // namespace +} // namespace querytele #endif diff --git a/utils/querytele/queryteleclient.cpp b/utils/querytele/queryteleclient.cpp index 8fa07661e..79d9e7313 100644 --- a/utils/querytele/queryteleclient.cpp +++ b/utils/querytele/queryteleclient.cpp @@ -36,256 +36,233 @@ namespace { using namespace querytele; -#define QT_ASSIGN_(x) out. __set_ ## x (qts. x) +#define QT_ASSIGN_(x) out.__set_##x(qts.x) QueryTele qts2qt(const QueryTeleStats& qts) { - QueryTele out; + QueryTele out; - out.query_uuid = bu::to_string(qts.query_uuid); + out.query_uuid = bu::to_string(qts.query_uuid); - switch (qts.msg_type) - { - case QueryTeleStats::QT_SUMMARY: - out.msg_type = QTType::QT_SUMMARY; - break; + switch (qts.msg_type) + { + case QueryTeleStats::QT_SUMMARY: out.msg_type = QTType::QT_SUMMARY; break; - case QueryTeleStats::QT_PROGRESS: - out.msg_type = QTType::QT_PROGRESS; - break; + case QueryTeleStats::QT_PROGRESS: out.msg_type = QTType::QT_PROGRESS; break; - case QueryTeleStats::QT_START: - out.msg_type = QTType::QT_START; - break; + case QueryTeleStats::QT_START: out.msg_type = QTType::QT_START; break; - default: - out.msg_type = QTType::QT_INVALID; - break; - } + default: out.msg_type = QTType::QT_INVALID; break; + } - QT_ASSIGN_(max_mem_pct); - QT_ASSIGN_(num_files); - QT_ASSIGN_(phy_io); - QT_ASSIGN_(cache_io); - QT_ASSIGN_(msg_rcv_cnt); - QT_ASSIGN_(cp_blocks_skipped); - QT_ASSIGN_(msg_bytes_in); - QT_ASSIGN_(msg_bytes_out); - QT_ASSIGN_(rows); - QT_ASSIGN_(start_time); - QT_ASSIGN_(end_time); - QT_ASSIGN_(error_no); - QT_ASSIGN_(blocks_changed); - QT_ASSIGN_(session_id); - QT_ASSIGN_(query_type); - QT_ASSIGN_(query); - QT_ASSIGN_(user); - QT_ASSIGN_(host); - QT_ASSIGN_(priority); - QT_ASSIGN_(priority_level); - QT_ASSIGN_(system_name); - QT_ASSIGN_(module_name); - QT_ASSIGN_(local_query); - QT_ASSIGN_(schema_name); + QT_ASSIGN_(max_mem_pct); + QT_ASSIGN_(num_files); + QT_ASSIGN_(phy_io); + QT_ASSIGN_(cache_io); + QT_ASSIGN_(msg_rcv_cnt); + QT_ASSIGN_(cp_blocks_skipped); + QT_ASSIGN_(msg_bytes_in); + QT_ASSIGN_(msg_bytes_out); + QT_ASSIGN_(rows); + QT_ASSIGN_(start_time); + QT_ASSIGN_(end_time); + QT_ASSIGN_(error_no); + QT_ASSIGN_(blocks_changed); + QT_ASSIGN_(session_id); + QT_ASSIGN_(query_type); + QT_ASSIGN_(query); + QT_ASSIGN_(user); + QT_ASSIGN_(host); + QT_ASSIGN_(priority); + QT_ASSIGN_(priority_level); + QT_ASSIGN_(system_name); + QT_ASSIGN_(module_name); + QT_ASSIGN_(local_query); + QT_ASSIGN_(schema_name); - return out; + return out; } #undef QT_ASSIGN_ -#define QT_ASSIGN_(x) out. __set_ ## x (sts. x) +#define QT_ASSIGN_(x) out.__set_##x(sts.x) StepTele sts2st(const StepTeleStats& sts) { - StepTele out; + StepTele out; - out.query_uuid = bu::to_string(sts.query_uuid); + out.query_uuid = bu::to_string(sts.query_uuid); - switch (sts.msg_type) - { - case StepTeleStats::ST_SUMMARY: - out.msg_type = STType::ST_SUMMARY; - break; + switch (sts.msg_type) + { + case StepTeleStats::ST_SUMMARY: out.msg_type = STType::ST_SUMMARY; break; - case StepTeleStats::ST_PROGRESS: - out.msg_type = STType::ST_PROGRESS; - break; + case StepTeleStats::ST_PROGRESS: out.msg_type = STType::ST_PROGRESS; break; - case StepTeleStats::ST_START: - out.msg_type = STType::ST_START; - break; + case StepTeleStats::ST_START: out.msg_type = STType::ST_START; break; - default: - out.msg_type = STType::ST_INVALID; - break; - } + default: out.msg_type = STType::ST_INVALID; break; + } - out.step_uuid = bu::to_string(sts.step_uuid); - QT_ASSIGN_(phy_io); - QT_ASSIGN_(cache_io); - QT_ASSIGN_(msg_rcv_cnt); - QT_ASSIGN_(cp_blocks_skipped); - QT_ASSIGN_(msg_bytes_in); - QT_ASSIGN_(msg_bytes_out); - QT_ASSIGN_(rows); - QT_ASSIGN_(start_time); - QT_ASSIGN_(end_time); - QT_ASSIGN_(total_units_of_work); - QT_ASSIGN_(units_of_work_completed); + out.step_uuid = bu::to_string(sts.step_uuid); + QT_ASSIGN_(phy_io); + QT_ASSIGN_(cache_io); + QT_ASSIGN_(msg_rcv_cnt); + QT_ASSIGN_(cp_blocks_skipped); + QT_ASSIGN_(msg_bytes_in); + QT_ASSIGN_(msg_bytes_out); + QT_ASSIGN_(rows); + QT_ASSIGN_(start_time); + QT_ASSIGN_(end_time); + QT_ASSIGN_(total_units_of_work); + QT_ASSIGN_(units_of_work_completed); - return out; + return out; } #undef QT_ASSIGN_ -#define QT_ASSIGN_(x) out. __set_ ## x (its. x) +#define QT_ASSIGN_(x) out.__set_##x(its.x) ImportTele its2it(const ImportTeleStats& its) { - ImportTele out; + ImportTele out; - out.job_uuid = bu::to_string(its.job_uuid); - out.import_uuid = bu::to_string(its.import_uuid); + out.job_uuid = bu::to_string(its.job_uuid); + out.import_uuid = bu::to_string(its.import_uuid); - switch (its.msg_type) - { - case ImportTeleStats::IT_SUMMARY: - out.msg_type = ITType::IT_SUMMARY; - break; + switch (its.msg_type) + { + case ImportTeleStats::IT_SUMMARY: out.msg_type = ITType::IT_SUMMARY; break; - case ImportTeleStats::IT_PROGRESS: - out.msg_type = ITType::IT_PROGRESS; - break; + case ImportTeleStats::IT_PROGRESS: out.msg_type = ITType::IT_PROGRESS; break; - case ImportTeleStats::IT_START: - out.msg_type = ITType::IT_START; - break; + case ImportTeleStats::IT_START: out.msg_type = ITType::IT_START; break; - case ImportTeleStats::IT_TERM: - out.msg_type = ITType::IT_TERM; - break; + case ImportTeleStats::IT_TERM: out.msg_type = ITType::IT_TERM; break; - default: - out.msg_type = ITType::IT_INVALID; - break; - } + default: out.msg_type = ITType::IT_INVALID; break; + } - QT_ASSIGN_(start_time); - QT_ASSIGN_(end_time); - QT_ASSIGN_(table_list); - QT_ASSIGN_(rows_so_far); - QT_ASSIGN_(system_name); - QT_ASSIGN_(module_name); - QT_ASSIGN_(schema_name); + QT_ASSIGN_(start_time); + QT_ASSIGN_(end_time); + QT_ASSIGN_(table_list); + QT_ASSIGN_(rows_so_far); + QT_ASSIGN_(system_name); + QT_ASSIGN_(module_name); + QT_ASSIGN_(schema_name); - return out; + return out; } #undef QT_ASSIGN_ -} +} // namespace namespace querytele { - -QueryTeleClient::QueryTeleClient(const QueryTeleServerParms& sp) : - fProtoImpl(0), - fServerParms(sp) +QueryTeleClient::QueryTeleClient(const QueryTeleServerParms& sp) : fProtoImpl(0), fServerParms(sp) { - if (fServerParms.host.empty() || fServerParms.port == 0) return; + if (fServerParms.host.empty() || fServerParms.port == 0) + return; - fProtoImpl = new QueryTeleProtoImpl(fServerParms); + fProtoImpl = new QueryTeleProtoImpl(fServerParms); } QueryTeleClient::~QueryTeleClient() { - delete fProtoImpl; + delete fProtoImpl; } -QueryTeleClient::QueryTeleClient(const QueryTeleClient& rhs) : - fProtoImpl(0) +QueryTeleClient::QueryTeleClient(const QueryTeleClient& rhs) : fProtoImpl(0) { - fServerParms = rhs.fServerParms; + fServerParms = rhs.fServerParms; - if (rhs.fProtoImpl) - { - fProtoImpl = new QueryTeleProtoImpl(*rhs.fProtoImpl); - } + if (rhs.fProtoImpl) + { + fProtoImpl = new QueryTeleProtoImpl(*rhs.fProtoImpl); + } } QueryTeleClient& QueryTeleClient::operator=(const QueryTeleClient& rhs) { - if (&rhs != this) + if (&rhs != this) + { + fProtoImpl = 0; + fServerParms = rhs.fServerParms; + + if (rhs.fProtoImpl) { - fProtoImpl = 0; - fServerParms = rhs.fServerParms; - - if (rhs.fProtoImpl) - { - fProtoImpl = new QueryTeleProtoImpl(*rhs.fProtoImpl); - } + fProtoImpl = new QueryTeleProtoImpl(*rhs.fProtoImpl); } + } - return *this; + return *this; } void QueryTeleClient::serverParms(const QueryTeleServerParms& sp) { - fServerParms = sp; - delete fProtoImpl; - fProtoImpl = 0; + fServerParms = sp; + delete fProtoImpl; + fProtoImpl = 0; - if (fServerParms.host.empty() || fServerParms.port == 0) return; + if (fServerParms.host.empty() || fServerParms.port == 0) + return; - fProtoImpl = new QueryTeleProtoImpl(fServerParms); + fProtoImpl = new QueryTeleProtoImpl(fServerParms); } void QueryTeleClient::postQueryTele(const QueryTeleStats& qts) { - if (!fProtoImpl) return; + if (!fProtoImpl) + return; - QueryTele qtdata = qts2qt(qts); - fProtoImpl->enqQueryTele(qtdata); + QueryTele qtdata = qts2qt(qts); + fProtoImpl->enqQueryTele(qtdata); } -#define QT_STYPE_CASE_(x) case StepTeleStats:: x: {stdata.step_type = StepType:: x; break;} +#define QT_STYPE_CASE_(x) \ + case StepTeleStats::x: \ + { \ + stdata.step_type = StepType::x; \ + break; \ + } void QueryTeleClient::postStepTele(const StepTeleStats& sts) { - if (!fProtoImpl) return; + if (!fProtoImpl) + return; - StepTele stdata = sts2st(sts); + StepTele stdata = sts2st(sts); - switch (fStepParms.stepType) - { - QT_STYPE_CASE_(T_HJS); - QT_STYPE_CASE_(T_DSS); - QT_STYPE_CASE_(T_CES); - QT_STYPE_CASE_(T_SQS); - QT_STYPE_CASE_(T_TAS); - QT_STYPE_CASE_(T_TNS); - QT_STYPE_CASE_(T_BPS); - QT_STYPE_CASE_(T_TCS); - QT_STYPE_CASE_(T_HVS); - QT_STYPE_CASE_(T_WFS); - QT_STYPE_CASE_(T_SAS); - QT_STYPE_CASE_(T_TUN); + switch (fStepParms.stepType) + { + QT_STYPE_CASE_(T_HJS); + QT_STYPE_CASE_(T_DSS); + QT_STYPE_CASE_(T_CES); + QT_STYPE_CASE_(T_SQS); + QT_STYPE_CASE_(T_TAS); + QT_STYPE_CASE_(T_TNS); + QT_STYPE_CASE_(T_BPS); + QT_STYPE_CASE_(T_TCS); + QT_STYPE_CASE_(T_HVS); + QT_STYPE_CASE_(T_WFS); + QT_STYPE_CASE_(T_SAS); + QT_STYPE_CASE_(T_TUN); - default: - stdata.step_type = StepType::T_INVALID; - break; - } + default: stdata.step_type = StepType::T_INVALID; break; + } - fProtoImpl->enqStepTele(stdata); + fProtoImpl->enqStepTele(stdata); } #undef QT_STYPE_CASE_ void QueryTeleClient::postImportTele(const ImportTeleStats& its) { - if (!fProtoImpl) return; + if (!fProtoImpl) + return; - ImportTele itdata = its2it(its); - fProtoImpl->enqImportTele(itdata); + ImportTele itdata = its2it(its); + fProtoImpl->enqImportTele(itdata); } void QueryTeleClient::waitForQueues() { - if (fProtoImpl) - fProtoImpl->waitForQueues(); -} - + if (fProtoImpl) + fProtoImpl->waitForQueues(); } +} // namespace querytele diff --git a/utils/querytele/queryteleclient.h b/utils/querytele/queryteleclient.h index efeebeefa..1f8de4ec3 100644 --- a/utils/querytele/queryteleclient.h +++ b/utils/querytele/queryteleclient.h @@ -33,62 +33,59 @@ #define EXPORT #endif - namespace querytele { - class QueryTeleProtoImpl; class QueryTeleClient { -public: - QueryTeleClient() : fProtoImpl(0) { } - EXPORT explicit QueryTeleClient(const QueryTeleServerParms&); - EXPORT ~QueryTeleClient(); + public: + QueryTeleClient() : fProtoImpl(0) + { + } + EXPORT explicit QueryTeleClient(const QueryTeleServerParms&); + EXPORT ~QueryTeleClient(); - EXPORT QueryTeleClient(const QueryTeleClient& rhs); - EXPORT QueryTeleClient& operator=(const QueryTeleClient& rhs); + EXPORT QueryTeleClient(const QueryTeleClient& rhs); + EXPORT QueryTeleClient& operator=(const QueryTeleClient& rhs); - EXPORT void postQueryTele(const QueryTeleStats&); - EXPORT void postStepTele(const StepTeleStats&); - EXPORT void postImportTele(const ImportTeleStats&); + EXPORT void postQueryTele(const QueryTeleStats&); + EXPORT void postStepTele(const StepTeleStats&); + EXPORT void postImportTele(const ImportTeleStats&); - EXPORT void serverParms(const QueryTeleServerParms&); - inline const QueryTeleServerParms& serverParms() const - { - return fServerParms; - } + EXPORT void serverParms(const QueryTeleServerParms&); + inline const QueryTeleServerParms& serverParms() const + { + return fServerParms; + } - inline void stepParms(const QueryStepParms& sp) - { - fStepParms = sp; - } - inline const QueryStepParms& stepParms() const - { - return fStepParms; - } - inline QueryStepParms& stepParms() - { - return fStepParms; - } + inline void stepParms(const QueryStepParms& sp) + { + fStepParms = sp; + } + inline const QueryStepParms& stepParms() const + { + return fStepParms; + } + inline QueryStepParms& stepParms() + { + return fStepParms; + } - EXPORT void waitForQueues(); + EXPORT void waitForQueues(); - EXPORT static boost::uuids::uuid genUUID(); - EXPORT static int64_t timeNowms(); - -protected: - -private: - QueryTeleProtoImpl* fProtoImpl; - QueryTeleServerParms fServerParms; - QueryStepParms fStepParms; + EXPORT static boost::uuids::uuid genUUID(); + EXPORT static int64_t timeNowms(); + protected: + private: + QueryTeleProtoImpl* fProtoImpl; + QueryTeleServerParms fServerParms; + QueryStepParms fStepParms; }; -} +} // namespace querytele #undef EXPORT #endif - diff --git a/utils/querytele/queryteleprotoimpl.cpp b/utils/querytele/queryteleprotoimpl.cpp index fa54a0adf..a6239f8f2 100644 --- a/utils/querytele/queryteleprotoimpl.cpp +++ b/utils/querytele/queryteleprotoimpl.cpp @@ -46,10 +46,10 @@ const size_t MaxQueueElems = 1000; template struct TsTeleQueue { - typedef std::queue TeleQueue; + typedef std::queue TeleQueue; - TeleQueue queue; - boost::mutex queueMtx; + TeleQueue queue; + boost::mutex queueMtx; }; TsTeleQueue stQueue; @@ -67,15 +67,14 @@ querytele::StepTele gLastStep; struct QStats { - int qtqueuedrops; - int stqueuedrops; - int stqueuedups; - int itqueuedrops; - QStats() : - qtqueuedrops(0), - stqueuedrops(0), - stqueuedups(0), - itqueuedrops(0) { ; } + int qtqueuedrops; + int stqueuedrops; + int stqueuedups; + int itqueuedrops; + QStats() : qtqueuedrops(0), stqueuedrops(0), stqueuedups(0), itqueuedrops(0) + { + ; + } }; QStats fQStats; @@ -83,395 +82,379 @@ QStats fQStats; #ifdef QUERY_TELE_DEBUG string get_trace_file() { - ostringstream oss; - pid_t pid = getpid(); + ostringstream oss; + pid_t pid = getpid(); #ifdef _MSC_VER - DWORD threadid = GetCurrentThreadId(); + DWORD threadid = GetCurrentThreadId(); #else - pthread_t threadid = pthread_self(); + pthread_t threadid = pthread_self(); #endif - oss << "/tmp/qt-consumer-" << pid << "-" << threadid; + oss << "/tmp/qt-consumer-" << pid << "-" << threadid; - return oss.str(); + return oss.str(); } void log_query(const querytele::QueryTele& qtdata) { - ofstream trace(get_trace_file().c_str(), ios::out | ios::app); - trace << "Query," - << qtdata.query_uuid << "," - << ","; // skip step uuid + ofstream trace(get_trace_file().c_str(), ios::out | ios::app); + trace << "Query," << qtdata.query_uuid << "," + << ","; // skip step uuid - if (qtdata.msg_type == querytele::QTType::QT_SUMMARY) - trace << "SUMMARY,"; - else if (qtdata.msg_type == querytele::QTType::QT_START) - trace << "START,"; - else - trace << "PROGRESS,"; + if (qtdata.msg_type == querytele::QTType::QT_SUMMARY) + trace << "SUMMARY,"; + else if (qtdata.msg_type == querytele::QTType::QT_START) + trace << "START,"; + else + trace << "PROGRESS,"; - trace << ","; // sktp step type + trace << ","; // sktp step type - trace << qtdata.start_time << ","; - trace << qtdata.end_time << ","; + trace << qtdata.start_time << ","; + trace << qtdata.end_time << ","; - trace << qtdata.cache_io << ","; - trace << qtdata.msg_rcv_cnt << ","; - trace << qtdata.rows << ","; - trace << qtdata.max_mem_pct << ","; + trace << qtdata.cache_io << ","; + trace << qtdata.msg_rcv_cnt << ","; + trace << qtdata.rows << ","; + trace << qtdata.max_mem_pct << ","; - trace << qtdata.query_type << ","; - trace << qtdata.schema_name << ","; - trace << qtdata.query << ","; - trace << qtdata.system_name; - trace << endl; - trace.close(); + trace << qtdata.query_type << ","; + trace << qtdata.schema_name << ","; + trace << qtdata.query << ","; + trace << qtdata.system_name; + trace << endl; + trace.close(); } #endif #ifdef QUERY_TELE_DEBUG const string st2str(enum querytele::StepType::type t) { - switch (t) - { - case querytele::StepType::T_HJS: - return "HJS"; + switch (t) + { + case querytele::StepType::T_HJS: return "HJS"; - case querytele::StepType::T_DSS: - return "DSS"; + case querytele::StepType::T_DSS: return "DSS"; - case querytele::StepType::T_CES: - return "CES"; + case querytele::StepType::T_CES: return "CES"; - case querytele::StepType::T_SQS: - return "SQS"; + case querytele::StepType::T_SQS: return "SQS"; - case querytele::StepType::T_TAS: - return "TAS"; + case querytele::StepType::T_TAS: return "TAS"; - case querytele::StepType::T_TNS: - return "TNS"; + case querytele::StepType::T_TNS: return "TNS"; - case querytele::StepType::T_BPS: - return "BPS"; + case querytele::StepType::T_BPS: return "BPS"; - case querytele::StepType::T_TCS: - return "TCS"; + case querytele::StepType::T_TCS: return "TCS"; - case querytele::StepType::T_HVS: - return "HVS"; + case querytele::StepType::T_HVS: return "HVS"; - case querytele::StepType::T_WFS: - return "WFS"; + case querytele::StepType::T_WFS: return "WFS"; - case querytele::StepType::T_SAS: - return "SAS"; + case querytele::StepType::T_SAS: return "SAS"; - case querytele::StepType::T_TUN: - return "TUN"; + case querytele::StepType::T_TUN: return "TUN"; - default: - return "INV"; - } + default: return "INV"; + } - return "INV"; + return "INV"; } #endif #ifdef QUERY_TELE_DEBUG void log_step(const querytele::StepTele& stdata) { - ofstream trace(get_trace_file().c_str(), ios::out | ios::app); + ofstream trace(get_trace_file().c_str(), ios::out | ios::app); - trace << "Step," - << stdata.query_uuid << "," - << stdata.step_uuid << ","; + trace << "Step," << stdata.query_uuid << "," << stdata.step_uuid << ","; - if (stdata.msg_type == querytele::STType::ST_SUMMARY) - trace << "SUMMARY,"; - else if (stdata.msg_type == querytele::STType::ST_START) - trace << "START,"; - else - trace << "PROGRESS,"; + if (stdata.msg_type == querytele::STType::ST_SUMMARY) + trace << "SUMMARY,"; + else if (stdata.msg_type == querytele::STType::ST_START) + trace << "START,"; + else + trace << "PROGRESS,"; - trace << st2str(stdata.step_type) << ","; + trace << st2str(stdata.step_type) << ","; - trace << stdata.start_time << ","; - trace << stdata.end_time << ","; + trace << stdata.start_time << ","; + trace << stdata.end_time << ","; - trace << stdata.cache_io << ","; - trace << stdata.msg_rcv_cnt << ","; - trace << stdata.rows << ","; + trace << stdata.cache_io << ","; + trace << stdata.msg_rcv_cnt << ","; + trace << stdata.rows << ","; - if (stdata.total_units_of_work > 0) - trace << stdata.units_of_work_completed * 100 / stdata.total_units_of_work << ","; - else - trace << "-1,"; + if (stdata.total_units_of_work > 0) + trace << stdata.units_of_work_completed * 100 / stdata.total_units_of_work << ","; + else + trace << "-1,"; - trace << ",,,"; // skip qtype, schemo, etc. - trace << fQStats.stqueuedrops << "," << fQStats.stqueuedups << "," << stQueue.queue.size(); - trace << endl; - trace.close(); + trace << ",,,"; // skip qtype, schemo, etc. + trace << fQStats.stqueuedrops << "," << fQStats.stqueuedups << "," << stQueue.queue.size(); + trace << endl; + trace.close(); } #endif void TeleConsumer() { - bool didSomeWork = false; - boost::mutex::scoped_lock itlk(itQueue.queueMtx, boost::defer_lock); - boost::mutex::scoped_lock qtlk(qtQueue.queueMtx, boost::defer_lock); - boost::mutex::scoped_lock stlk(stQueue.queueMtx, boost::defer_lock); - querytele::QueryTeleServiceClient client(fProtocol); + bool didSomeWork = false; + boost::mutex::scoped_lock itlk(itQueue.queueMtx, boost::defer_lock); + boost::mutex::scoped_lock qtlk(qtQueue.queueMtx, boost::defer_lock); + boost::mutex::scoped_lock stlk(stQueue.queueMtx, boost::defer_lock); + querytele::QueryTeleServiceClient client(fProtocol); - try + try + { + for (;;) { - for (;;) + didSomeWork = false; + + itlk.lock(); + + // Empty the import queue first... + while (!itQueue.queue.empty()) + { + querytele::ImportTele itdata = itQueue.queue.front(); + itQueue.queue.pop(); + itlk.unlock(); + + try { - didSomeWork = false; - - itlk.lock(); - - // Empty the import queue first... - while (!itQueue.queue.empty()) - { - querytele::ImportTele itdata = itQueue.queue.front(); - itQueue.queue.pop(); - itlk.unlock(); - - try - { - fTransport->open(); - client.postImport(itdata); - fTransport->close(); - } - catch (...) - { - try - { - fTransport->close(); - } - catch (...) - { - } - } - - didSomeWork = true; - itlk.lock(); - } - - itlk.unlock(); - - qtlk.lock(); - - // Now empty the query queue... - while (!qtQueue.queue.empty()) - { - querytele::QueryTele qtdata = qtQueue.queue.front(); - qtQueue.queue.pop(); - qtlk.unlock(); - - try - { - fTransport->open(); -#ifdef QUERY_TELE_DEBUG - log_query(qtdata); -#endif - client.postQuery(qtdata); - fTransport->close(); - } - catch (...) - { - try - { - fTransport->close(); - } - catch (...) - { - } - } - - didSomeWork = true; - qtlk.lock(); - } - - qtlk.unlock(); - - stlk.lock(); - - // Finally empty the step queue... - // @bug6088 - Added check for query queue and import queue in while statment below to - // keep the step logs from starving the query and import logs. - while (!stQueue.queue.empty() && qtQueue.queue.empty() && itQueue.queue.empty()) - { - querytele::StepTele stdata = stQueue.queue.front(); - stQueue.queue.pop(); - stlk.unlock(); - - try - { - fTransport->open(); -#ifdef QUERY_TELE_DEBUG - log_step(stdata); -#endif - client.postStep(stdata); - fTransport->close(); - } - catch (...) - { - try - { - fTransport->close(); - } - catch (...) - { - } - } - - didSomeWork = true; - stlk.lock(); - } - - stlk.unlock(); - - if (!didSomeWork) - { - usleep(50000); - } + fTransport->open(); + client.postImport(itdata); + fTransport->close(); } + catch (...) + { + try + { + fTransport->close(); + } + catch (...) + { + } + } + + didSomeWork = true; + itlk.lock(); + } + + itlk.unlock(); + + qtlk.lock(); + + // Now empty the query queue... + while (!qtQueue.queue.empty()) + { + querytele::QueryTele qtdata = qtQueue.queue.front(); + qtQueue.queue.pop(); + qtlk.unlock(); + + try + { + fTransport->open(); +#ifdef QUERY_TELE_DEBUG + log_query(qtdata); +#endif + client.postQuery(qtdata); + fTransport->close(); + } + catch (...) + { + try + { + fTransport->close(); + } + catch (...) + { + } + } + + didSomeWork = true; + qtlk.lock(); + } + + qtlk.unlock(); + + stlk.lock(); + + // Finally empty the step queue... + // @bug6088 - Added check for query queue and import queue in while statment below to + // keep the step logs from starving the query and import logs. + while (!stQueue.queue.empty() && qtQueue.queue.empty() && itQueue.queue.empty()) + { + querytele::StepTele stdata = stQueue.queue.front(); + stQueue.queue.pop(); + stlk.unlock(); + + try + { + fTransport->open(); +#ifdef QUERY_TELE_DEBUG + log_step(stdata); +#endif + client.postStep(stdata); + fTransport->close(); + } + catch (...) + { + try + { + fTransport->close(); + } + catch (...) + { + } + } + + didSomeWork = true; + stlk.lock(); + } + + stlk.unlock(); + + if (!didSomeWork) + { + usleep(50000); + } } - catch (...) - { - //we're probably shutting down, just let this thread die quietly... - } + } + catch (...) + { + // we're probably shutting down, just let this thread die quietly... + } } boost::thread* consThd; -} +} // namespace namespace querytele { - -QueryTeleProtoImpl::QueryTeleProtoImpl(const QueryTeleServerParms& sp) : - fServerParms(sp) +QueryTeleProtoImpl::QueryTeleProtoImpl(const QueryTeleServerParms& sp) : fServerParms(sp) { - if (fServerParms.host.empty() || fServerParms.port == 0) return; + if (fServerParms.host.empty() || fServerParms.port == 0) + return; - boost::mutex::scoped_lock lk(initMux); + boost::mutex::scoped_lock lk(initMux); - atomicops::atomicMb(); + atomicops::atomicMb(); - if (isInited) return; + if (isInited) + return; - fSocket.reset(new att::TSocket(fServerParms.host, fServerParms.port)); - fTransport.reset(new att::TBufferedTransport(fSocket)); - fProtocol.reset(new atp::TBinaryProtocol(fTransport)); + fSocket.reset(new att::TSocket(fServerParms.host, fServerParms.port)); + fTransport.reset(new att::TBufferedTransport(fSocket)); + fProtocol.reset(new atp::TBinaryProtocol(fTransport)); - consThd = new boost::thread(&TeleConsumer); + consThd = new boost::thread(&TeleConsumer); - atomicops::atomicMb(); - isInited = true; + atomicops::atomicMb(); + isInited = true; } int QueryTeleProtoImpl::enqStepTele(const StepTele& stdata) { - try - { - boost::mutex::scoped_lock lk(stQueue.queueMtx); + try + { + boost::mutex::scoped_lock lk(stQueue.queueMtx); - // @bug6088 - Added conditions below to always log progress SUMMARY and START messages to avoid completed queries showing up with progress 0 - // and no steps. - if (stQueue.queue.size() >= MaxQueueElems && stdata.msg_type != querytele::STType::ST_SUMMARY && stdata.msg_type != querytele::STType::ST_START) - { - fQStats.stqueuedrops++; - return -1; - } - - if ( stdata.step_uuid != gLastStep.step_uuid || - stdata.msg_type != gLastStep.msg_type || - stdata.step_type != gLastStep.step_type || - stdata.total_units_of_work != gLastStep.total_units_of_work || - stdata.units_of_work_completed != gLastStep.units_of_work_completed ) - { - stQueue.queue.push(stdata); - gLastStep = stdata; - } - else - { - fQStats.stqueuedups++; - } - } - catch (...) + // @bug6088 - Added conditions below to always log progress SUMMARY and START messages to avoid completed + // queries showing up with progress 0 + // and no steps. + if (stQueue.queue.size() >= MaxQueueElems && stdata.msg_type != querytele::STType::ST_SUMMARY && + stdata.msg_type != querytele::STType::ST_START) { - return -2; + fQStats.stqueuedrops++; + return -1; } - return 0; + if (stdata.step_uuid != gLastStep.step_uuid || stdata.msg_type != gLastStep.msg_type || + stdata.step_type != gLastStep.step_type || + stdata.total_units_of_work != gLastStep.total_units_of_work || + stdata.units_of_work_completed != gLastStep.units_of_work_completed) + { + stQueue.queue.push(stdata); + gLastStep = stdata; + } + else + { + fQStats.stqueuedups++; + } + } + catch (...) + { + return -2; + } + + return 0; } int QueryTeleProtoImpl::enqQueryTele(const QueryTele& qtdata) { - try - { - boost::mutex::scoped_lock lk(qtQueue.queueMtx); + try + { + boost::mutex::scoped_lock lk(qtQueue.queueMtx); - if (qtQueue.queue.size() >= MaxQueueElems) - { - fQStats.qtqueuedrops++; - return -1; - } - - qtQueue.queue.push(qtdata); - } - catch (...) + if (qtQueue.queue.size() >= MaxQueueElems) { - return -2; + fQStats.qtqueuedrops++; + return -1; } - return 0; + qtQueue.queue.push(qtdata); + } + catch (...) + { + return -2; + } + + return 0; } int QueryTeleProtoImpl::enqImportTele(const ImportTele& itdata) { - try - { - boost::mutex::scoped_lock lk(itQueue.queueMtx); + try + { + boost::mutex::scoped_lock lk(itQueue.queueMtx); - if (itQueue.queue.size() >= MaxQueueElems) - { - fQStats.itqueuedrops++; - return -1; - } - - itQueue.queue.push(itdata); - } - catch (...) + if (itQueue.queue.size() >= MaxQueueElems) { - return -2; + fQStats.itqueuedrops++; + return -1; } - return 0; + itQueue.queue.push(itdata); + } + catch (...) + { + return -2; + } + + return 0; } int QueryTeleProtoImpl::waitForQueues() { - try - { - boost::mutex::scoped_lock lk(itQueue.queueMtx); + try + { + boost::mutex::scoped_lock lk(itQueue.queueMtx); - while (!itQueue.queue.empty()) - { - lk.unlock(); - usleep(100000); - lk.lock(); - } - } - catch (...) + while (!itQueue.queue.empty()) { - return -1; + lk.unlock(); + usleep(100000); + lk.lock(); } + } + catch (...) + { + return -1; + } - return 0; + return 0; } -} //namespace querytele - +} // namespace querytele diff --git a/utils/querytele/queryteleprotoimpl.h b/utils/querytele/queryteleprotoimpl.h index e5a9215ca..25021a70b 100644 --- a/utils/querytele/queryteleprotoimpl.h +++ b/utils/querytele/queryteleprotoimpl.h @@ -22,30 +22,28 @@ namespace querytele { - class QueryTeleProtoImpl { -public: - explicit QueryTeleProtoImpl(const QueryTeleServerParms&); - ~QueryTeleProtoImpl() { } + public: + explicit QueryTeleProtoImpl(const QueryTeleServerParms&); + ~QueryTeleProtoImpl() + { + } - int enqStepTele(const StepTele&); - int enqImportTele(const ImportTele&); - int enqQueryTele(const QueryTele&); + int enqStepTele(const StepTele&); + int enqImportTele(const ImportTele&); + int enqQueryTele(const QueryTele&); - /** - * Wait for the consumer thread to post all messages - **/ - int waitForQueues(); - -protected: - -private: - QueryTeleServerParms fServerParms; + /** + * Wait for the consumer thread to post all messages + **/ + int waitForQueues(); + protected: + private: + QueryTeleServerParms fServerParms; }; -} +} // namespace querytele #endif - diff --git a/utils/querytele/queryteleserverparms.h b/utils/querytele/queryteleserverparms.h index cc101883b..c039e28f6 100644 --- a/utils/querytele/queryteleserverparms.h +++ b/utils/querytele/queryteleserverparms.h @@ -22,24 +22,26 @@ namespace querytele { - class QueryTeleServerParms { -public: - QueryTeleServerParms() : port(0) { } - QueryTeleServerParms(const std::string h, int p) : host(h), port(p) { } - ~QueryTeleServerParms() { } + public: + QueryTeleServerParms() : port(0) + { + } + QueryTeleServerParms(const std::string h, int p) : host(h), port(p) + { + } + ~QueryTeleServerParms() + { + } - std::string host; - int port; - -protected: - -private: + std::string host; + int port; + protected: + private: }; -} +} // namespace querytele #endif - diff --git a/utils/querytele/serverdriver.cpp b/utils/querytele/serverdriver.cpp index c52547a1a..bbbbd49c4 100644 --- a/utils/querytele/serverdriver.cpp +++ b/utils/querytele/serverdriver.cpp @@ -41,193 +41,173 @@ using namespace querytele; namespace { - class QueryTeleServiceHandler : public QueryTeleServiceIf { -public: - void postQuery(const QueryTele&); - void postStep(const StepTele&); - void postImport(const ImportTele&); - -protected: - -private: + public: + void postQuery(const QueryTele&); + void postStep(const StepTele&); + void postImport(const ImportTele&); + protected: + private: }; const string st2str(enum StepType::type t) { - switch (t) - { - case StepType::T_HJS: - return "HJS"; + switch (t) + { + case StepType::T_HJS: return "HJS"; - case StepType::T_DSS: - return "DSS"; + case StepType::T_DSS: return "DSS"; - case StepType::T_CES: - return "CES"; + case StepType::T_CES: return "CES"; - case StepType::T_SQS: - return "SQS"; + case StepType::T_SQS: return "SQS"; - case StepType::T_TAS: - return "TAS"; + case StepType::T_TAS: return "TAS"; - case StepType::T_TNS: - return "TNS"; + case StepType::T_TNS: return "TNS"; - case StepType::T_BPS: - return "BPS"; + case StepType::T_BPS: return "BPS"; - case StepType::T_TCS: - return "TCS"; + case StepType::T_TCS: return "TCS"; - case StepType::T_HVS: - return "HVS"; + case StepType::T_HVS: return "HVS"; - case StepType::T_WFS: - return "WFS"; + case StepType::T_WFS: return "WFS"; - case StepType::T_SAS: - return "SAS"; + case StepType::T_SAS: return "SAS"; - case StepType::T_TUN: - return "TUN"; + case StepType::T_TUN: return "TUN"; - default: - return "INV"; - } + default: return "INV"; + } - return "INV"; + return "INV"; } void QueryTeleServiceHandler::postQuery(const QueryTele& qt) { - cout << "postQuery: " << endl; - cout << " uuid: " << qt.query_uuid << endl; + cout << "postQuery: " << endl; + cout << " uuid: " << qt.query_uuid << endl; - if (qt.msg_type == QTType::QT_SUMMARY) - cout << " mt: SUMMARY" << endl; - else if (qt.msg_type == QTType::QT_START) - cout << " mt: START" << endl; - else - cout << " mt: PROGRESS" << endl; + if (qt.msg_type == QTType::QT_SUMMARY) + cout << " mt: SUMMARY" << endl; + else if (qt.msg_type == QTType::QT_START) + cout << " mt: START" << endl; + else + cout << " mt: PROGRESS" << endl; - cout << " qry: " << qt.query << endl; - cout << " mmpct: " << qt.max_mem_pct << endl; - cout << " cache: " << qt.cache_io << endl; - cout << " nmsgs: " << qt.msg_rcv_cnt << endl; - cout << " rows: " << qt.rows << endl; - cout << " qt: " << qt.query_type << endl; - int64_t tt = qt.start_time; - cout << " st: (" << tt << ") "; - tt /= 1000; - cout << ctime(&tt); - tt = qt.end_time; - cout << " et: (" << tt << ") "; - tt /= 1000; - cout << ctime(&tt); - cout << " sn: " << qt.system_name << endl; - cout << " mn: " << qt.module_name << endl; - cout << " lq: " << qt.local_query << endl; - cout << " dn: " << qt.schema_name << endl; - cout << endl; + cout << " qry: " << qt.query << endl; + cout << " mmpct: " << qt.max_mem_pct << endl; + cout << " cache: " << qt.cache_io << endl; + cout << " nmsgs: " << qt.msg_rcv_cnt << endl; + cout << " rows: " << qt.rows << endl; + cout << " qt: " << qt.query_type << endl; + int64_t tt = qt.start_time; + cout << " st: (" << tt << ") "; + tt /= 1000; + cout << ctime(&tt); + tt = qt.end_time; + cout << " et: (" << tt << ") "; + tt /= 1000; + cout << ctime(&tt); + cout << " sn: " << qt.system_name << endl; + cout << " mn: " << qt.module_name << endl; + cout << " lq: " << qt.local_query << endl; + cout << " dn: " << qt.schema_name << endl; + cout << endl; } void QueryTeleServiceHandler::postStep(const StepTele& qt) { - cout << "postStep: " << endl; - cout << " quuid: " << qt.query_uuid << endl; - cout << " uuid: " << qt.step_uuid << endl; + cout << "postStep: " << endl; + cout << " quuid: " << qt.query_uuid << endl; + cout << " uuid: " << qt.step_uuid << endl; - if (qt.msg_type == STType::ST_SUMMARY) - cout << " mt: SUMMARY" << endl; - else if (qt.msg_type == STType::ST_START) - cout << " mt: START" << endl; - else - cout << " mt: PROGRESS" << endl; + if (qt.msg_type == STType::ST_SUMMARY) + cout << " mt: SUMMARY" << endl; + else if (qt.msg_type == STType::ST_START) + cout << " mt: START" << endl; + else + cout << " mt: PROGRESS" << endl; - cout << " st: " << st2str(qt.step_type) << endl; - cout << " cache: " << qt.cache_io << endl; - cout << " nmsgs: " << qt.msg_rcv_cnt << endl; - cout << " rows: " << qt.rows << endl; + cout << " st: " << st2str(qt.step_type) << endl; + cout << " cache: " << qt.cache_io << endl; + cout << " nmsgs: " << qt.msg_rcv_cnt << endl; + cout << " rows: " << qt.rows << endl; - if (qt.total_units_of_work > 0) - cout << " pct: " << qt.units_of_work_completed * 100 / qt.total_units_of_work << endl; - else - cout << " pct: n/a" << endl; + if (qt.total_units_of_work > 0) + cout << " pct: " << qt.units_of_work_completed * 100 / qt.total_units_of_work << endl; + else + cout << " pct: n/a" << endl; - int64_t tt = qt.start_time; - cout << " st: (" << tt << ") "; - tt /= 1000; - cout << ctime(&tt); - tt = qt.end_time; - cout << " et: (" << tt << ") "; - tt /= 1000; - cout << ctime(&tt); - cout << endl; + int64_t tt = qt.start_time; + cout << " st: (" << tt << ") "; + tt /= 1000; + cout << ctime(&tt); + tt = qt.end_time; + cout << " et: (" << tt << ") "; + tt /= 1000; + cout << ctime(&tt); + cout << endl; } void QueryTeleServiceHandler::postImport(const ImportTele& qt) { - cout << "importStep: " << endl; - cout << " juuid: " << qt.job_uuid << endl; - cout << " iuuid: " << qt.import_uuid << endl; + cout << "importStep: " << endl; + cout << " juuid: " << qt.job_uuid << endl; + cout << " iuuid: " << qt.import_uuid << endl; - if (qt.msg_type == ITType::IT_SUMMARY) - cout << " mt: SUMMARY" << endl; - else if (qt.msg_type == ITType::IT_START) - cout << " mt: START" << endl; - else if (qt.msg_type == ITType::IT_TERM) - cout << " mt: TERM" << endl; - else - cout << " mt: PROGRESS" << endl; + if (qt.msg_type == ITType::IT_SUMMARY) + cout << " mt: SUMMARY" << endl; + else if (qt.msg_type == ITType::IT_START) + cout << " mt: START" << endl; + else if (qt.msg_type == ITType::IT_TERM) + cout << " mt: TERM" << endl; + else + cout << " mt: PROGRESS" << endl; - if (qt.table_list.empty()) - cout << " tn: " << "(empty)" << endl; - else - cout << " tn: " << qt.table_list[0] << endl; + if (qt.table_list.empty()) + cout << " tn: " + << "(empty)" << endl; + else + cout << " tn: " << qt.table_list[0] << endl; - if (qt.rows_so_far.empty()) - cout << " rows: " << "(empty)" << endl; - else - cout << " rows: " << qt.rows_so_far[0] << endl; + if (qt.rows_so_far.empty()) + cout << " rows: " + << "(empty)" << endl; + else + cout << " rows: " << qt.rows_so_far[0] << endl; - int64_t tt = qt.start_time; - cout << " st: (" << tt << ") "; - tt /= 1000; - cout << ctime(&tt); - tt = qt.end_time; - cout << " et: (" << tt << ") "; - tt /= 1000; - cout << ctime(&tt); - cout << " sn: " << qt.system_name << endl; - cout << " mn: " << qt.module_name << endl; - cout << " dn: " << qt.schema_name << endl; - cout << endl; + int64_t tt = qt.start_time; + cout << " st: (" << tt << ") "; + tt /= 1000; + cout << ctime(&tt); + tt = qt.end_time; + cout << " et: (" << tt << ") "; + tt /= 1000; + cout << ctime(&tt); + cout << " sn: " << qt.system_name << endl; + cout << " mn: " << qt.module_name << endl; + cout << " dn: " << qt.schema_name << endl; + cout << endl; } -} +} // namespace int main(int argc, char** argv) { + shared_ptr protocolFactory(new atp::TBinaryProtocolFactory()); + shared_ptr handler(new QueryTeleServiceHandler()); + shared_ptr processor(new QueryTeleServiceProcessor(handler)); + shared_ptr serverTransport(new att::TServerSocket(9990)); + shared_ptr transportFactory(new att::TBufferedTransportFactory()); - shared_ptr protocolFactory(new atp::TBinaryProtocolFactory()); - shared_ptr handler(new QueryTeleServiceHandler()); - shared_ptr processor(new QueryTeleServiceProcessor(handler)); - shared_ptr serverTransport(new att::TServerSocket(9990)); - shared_ptr transportFactory(new att::TBufferedTransportFactory()); + ats::TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); - ats::TSimpleServer server(processor, - serverTransport, - transportFactory, - protocolFactory); - - - cout << "Starting the server..." << endl; - server.serve(); - cout << "done." << endl; - return 0; + cout << "Starting the server..." << endl; + server.serve(); + cout << "done." << endl; + return 0; } - diff --git a/utils/querytele/tdriver.cpp b/utils/querytele/tdriver.cpp index 5181e0c02..7316bc0d8 100644 --- a/utils/querytele/tdriver.cpp +++ b/utils/querytele/tdriver.cpp @@ -27,45 +27,44 @@ using namespace querytele; int main(int argc, char** argv) { - bu::random_generator rg; - QueryTeleServerParms qtsp; - qtsp.host = "localhost"; - qtsp.port = 9090; - QueryTeleClient qtc(qtsp); - QueryTeleClient qtc1(qtc); - QueryTeleClient qtc2; - qtc2 = qtc; - QueryTeleStats qts; - qts.query_uuid = rg(); - qts.msg_type = QueryTeleStats::QT_START; - qts.query = "SELECT * FROM NATION;"; - qtc.postQueryTele(qts); + bu::random_generator rg; + QueryTeleServerParms qtsp; + qtsp.host = "localhost"; + qtsp.port = 9090; + QueryTeleClient qtc(qtsp); + QueryTeleClient qtc1(qtc); + QueryTeleClient qtc2; + qtc2 = qtc; + QueryTeleStats qts; + qts.query_uuid = rg(); + qts.msg_type = QueryTeleStats::QT_START; + qts.query = "SELECT * FROM NATION;"; + qtc.postQueryTele(qts); - sleep(1); + sleep(1); - StepTeleStats sts; - sts.query_uuid = qts.query_uuid; - sts.step_uuid = rg(); - sts.msg_type = StepTeleStats::ST_START; - qtc.postStepTele(sts); + StepTeleStats sts; + sts.query_uuid = qts.query_uuid; + sts.step_uuid = rg(); + sts.msg_type = StepTeleStats::ST_START; + qtc.postStepTele(sts); - sleep(1); + sleep(1); - sts.msg_type = StepTeleStats::ST_PROGRESS; - qtc.postStepTele(sts); + sts.msg_type = StepTeleStats::ST_PROGRESS; + qtc.postStepTele(sts); - sleep(1); + sleep(1); - sts.msg_type = StepTeleStats::ST_SUMMARY; - qtc.postStepTele(sts); + sts.msg_type = StepTeleStats::ST_SUMMARY; + qtc.postStepTele(sts); - sleep(1); + sleep(1); - qts.msg_type = QueryTeleStats::QT_SUMMARY; - qtc.postQueryTele(qts); + qts.msg_type = QueryTeleStats::QT_SUMMARY; + qtc.postQueryTele(qts); - sleep(20); + sleep(20); - return 0; + return 0; } - diff --git a/utils/querytele/telestats.h b/utils/querytele/telestats.h index dcb39d167..2a045947c 100644 --- a/utils/querytele/telestats.h +++ b/utils/querytele/telestats.h @@ -29,170 +29,169 @@ namespace querytele { - struct QueryTeleStats { - enum QTType - { - QT_INVALID, - QT_SUMMARY, - QT_PROGRESS, - QT_START, - }; + enum QTType + { + QT_INVALID, + QT_SUMMARY, + QT_PROGRESS, + QT_START, + }; - QueryTeleStats() : - msg_type(QT_INVALID), - max_mem_pct(0), - num_files(0), - phy_io(0), - cache_io(0), - msg_rcv_cnt(0), - cp_blocks_skipped(0), - msg_bytes_in(0), - msg_bytes_out(0), - rows(0), - start_time(-1), - end_time(-1), - error_no(0), - blocks_changed(0), - session_id(0), - priority_level(0), - local_query(0) - { - query_uuid = boost::uuids::nil_generator()(); - } + QueryTeleStats() + : msg_type(QT_INVALID) + , max_mem_pct(0) + , num_files(0) + , phy_io(0) + , cache_io(0) + , msg_rcv_cnt(0) + , cp_blocks_skipped(0) + , msg_bytes_in(0) + , msg_bytes_out(0) + , rows(0) + , start_time(-1) + , end_time(-1) + , error_no(0) + , blocks_changed(0) + , session_id(0) + , priority_level(0) + , local_query(0) + { + query_uuid = boost::uuids::nil_generator()(); + } - ~QueryTeleStats() { } + ~QueryTeleStats() + { + } - boost::uuids::uuid query_uuid; - QTType msg_type; - int64_t max_mem_pct; - int64_t num_files; - int64_t phy_io; - int64_t cache_io; - int64_t msg_rcv_cnt; - int64_t cp_blocks_skipped; - int64_t msg_bytes_in; - int64_t msg_bytes_out; - int64_t rows; - int64_t start_time; - int64_t end_time; - int64_t error_no; - int64_t blocks_changed; - int64_t session_id; - std::string query_type; - std::string query; - std::string user; - std::string host; - std::string priority; - int32_t priority_level; - std::string system_name; - std::string module_name; - int32_t local_query; - std::string schema_name; + boost::uuids::uuid query_uuid; + QTType msg_type; + int64_t max_mem_pct; + int64_t num_files; + int64_t phy_io; + int64_t cache_io; + int64_t msg_rcv_cnt; + int64_t cp_blocks_skipped; + int64_t msg_bytes_in; + int64_t msg_bytes_out; + int64_t rows; + int64_t start_time; + int64_t end_time; + int64_t error_no; + int64_t blocks_changed; + int64_t session_id; + std::string query_type; + std::string query; + std::string user; + std::string host; + std::string priority; + int32_t priority_level; + std::string system_name; + std::string module_name; + int32_t local_query; + std::string schema_name; }; struct StepTeleStats { - enum STType - { - ST_INVALID, - ST_SUMMARY, - ST_PROGRESS, - ST_START, - }; + enum STType + { + ST_INVALID, + ST_SUMMARY, + ST_PROGRESS, + ST_START, + }; - enum StepType - { - T_INVALID, - T_HJS, //TupleHashJoinStep - T_DSS, //DictionaryScanStep - T_CES, //CrossEngineStep - T_SQS, //SubQueryStep - T_TAS, //TupleAggregateStep - T_TNS, //TupleAnnexStep - T_BPS, //TupleBPS - T_TCS, //TupleConstantStep - T_HVS, //TupleHavingStep - T_WFS, //WindowFunctionStep - T_SAS, //SubAdapterStep - T_TUN, //TupleUnion - }; + enum StepType + { + T_INVALID, + T_HJS, // TupleHashJoinStep + T_DSS, // DictionaryScanStep + T_CES, // CrossEngineStep + T_SQS, // SubQueryStep + T_TAS, // TupleAggregateStep + T_TNS, // TupleAnnexStep + T_BPS, // TupleBPS + T_TCS, // TupleConstantStep + T_HVS, // TupleHavingStep + T_WFS, // WindowFunctionStep + T_SAS, // SubAdapterStep + T_TUN, // TupleUnion + }; - StepTeleStats() : - msg_type(ST_INVALID), - step_type(T_INVALID), - phy_io(0), - cache_io(0), - msg_rcv_cnt(0), - cp_blocks_skipped(0), - msg_bytes_in(0), - msg_bytes_out(0), - rows(0), - start_time(-1), - end_time(-1), - total_units_of_work(0), - units_of_work_completed(0) - { - query_uuid = boost::uuids::nil_generator()(); - step_uuid = boost::uuids::nil_generator()(); - } + StepTeleStats() + : msg_type(ST_INVALID) + , step_type(T_INVALID) + , phy_io(0) + , cache_io(0) + , msg_rcv_cnt(0) + , cp_blocks_skipped(0) + , msg_bytes_in(0) + , msg_bytes_out(0) + , rows(0) + , start_time(-1) + , end_time(-1) + , total_units_of_work(0) + , units_of_work_completed(0) + { + query_uuid = boost::uuids::nil_generator()(); + step_uuid = boost::uuids::nil_generator()(); + } - ~StepTeleStats() { } + ~StepTeleStats() + { + } - boost::uuids::uuid query_uuid; - STType msg_type; - StepType step_type; - boost::uuids::uuid step_uuid; - int64_t phy_io; - int64_t cache_io; - int64_t msg_rcv_cnt; - int64_t cp_blocks_skipped; - int64_t msg_bytes_in; - int64_t msg_bytes_out; - int64_t rows; - int64_t start_time; - int64_t end_time; - int32_t total_units_of_work; - int32_t units_of_work_completed; + boost::uuids::uuid query_uuid; + STType msg_type; + StepType step_type; + boost::uuids::uuid step_uuid; + int64_t phy_io; + int64_t cache_io; + int64_t msg_rcv_cnt; + int64_t cp_blocks_skipped; + int64_t msg_bytes_in; + int64_t msg_bytes_out; + int64_t rows; + int64_t start_time; + int64_t end_time; + int32_t total_units_of_work; + int32_t units_of_work_completed; }; struct ImportTeleStats { - typedef std::vector StringList; - typedef std::vector I64List; + typedef std::vector StringList; + typedef std::vector I64List; - enum ITType - { - IT_INVALID, - IT_SUMMARY, - IT_PROGRESS, - IT_START, - IT_TERM, - }; + enum ITType + { + IT_INVALID, + IT_SUMMARY, + IT_PROGRESS, + IT_START, + IT_TERM, + }; - ImportTeleStats() : - msg_type(IT_INVALID), - start_time(-1), - end_time(-1) - { - job_uuid = boost::uuids::nil_generator()(); - import_uuid = boost::uuids::nil_generator()(); - } + ImportTeleStats() : msg_type(IT_INVALID), start_time(-1), end_time(-1) + { + job_uuid = boost::uuids::nil_generator()(); + import_uuid = boost::uuids::nil_generator()(); + } - boost::uuids::uuid job_uuid; - boost::uuids::uuid import_uuid; - ITType msg_type; - int64_t start_time; - int64_t end_time; - StringList table_list; - I64List rows_so_far; - std::string system_name; - std::string module_name; - std::string schema_name; + boost::uuids::uuid job_uuid; + boost::uuids::uuid import_uuid; + ITType msg_type; + int64_t start_time; + int64_t end_time; + StringList table_list; + I64List rows_so_far; + std::string system_name; + std::string module_name; + std::string schema_name; }; -} +} // namespace querytele #endif - diff --git a/utils/regr/corr.cpp b/utils/regr/corr.cpp index 5740bd8fe..4d6768353 100644 --- a/utils/regr/corr.cpp +++ b/utils/regr/corr.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_corr_ToUDAFMap { -public: - Add_corr_ToUDAFMap() - { - UDAFMap::getMap()["corr"] = new corr(); - } + public: + Add_corr_ToUDAFMap() + { + UDAFMap::getMap()["corr"] = new corr(); + } }; static Add_corr_ToUDAFMap addToMap; @@ -38,142 +38,138 @@ static Add_corr_ToUDAFMap addToMap; // Use the simple data model struct corr_data { - uint64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumy2; // sum of (y squared) - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode corr::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode corr::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("corr() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("corr() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(corr_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("corr() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("corr() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(corr_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode corr::reset(mcsv1Context* context) { - struct corr_data* data = (struct corr_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumy2 = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct corr_data* data = (struct corr_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumy2 = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode corr::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct corr_data* data = (struct corr_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct corr_data* data = (struct corr_data*)context->getUserData()->data; - data->sumy += valy; - data->sumy2 += valy*valy; + data->sumy += valy; + data->sumy2 += valy * valy; - data->sumx += valx; - data->sumx2 += valx*valx; + data->sumx += valx; + data->sumx2 += valx * valx; - data->sumxy += valx*valy; + data->sumxy += valx * valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode corr::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct corr_data* outData = (struct corr_data*)context->getUserData()->data; - struct corr_data* inData = (struct corr_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumx2 += inData->sumx2; - outData->sumy += inData->sumy; - outData->sumy2 += inData->sumy2; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct corr_data* outData = (struct corr_data*)context->getUserData()->data; + struct corr_data* inData = (struct corr_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumx2 += inData->sumx2; + outData->sumy += inData->sumy; + outData->sumy2 += inData->sumy2; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode corr::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct corr_data* data = (struct corr_data*)context->getUserData()->data; - double N = data->cnt; - if (N > 1) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumy2 = data->sumy2; - long double sumxy = data->sumxy; + struct corr_data* data = (struct corr_data*)context->getUserData()->data; + double N = data->cnt; + if (N > 1) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; - long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx <= 0) // Catch -0 - { - // When var_popx is 0, NULL is the result. - return mcsv1_UDAF::SUCCESS; - } - long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy <= 0) // Catch -0 - { - // When var_popy is 0, NULL is the result - return mcsv1_UDAF::SUCCESS; - } - long double std_popx = sqrt(var_popx); - long double std_popy = sqrt(var_popy); - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - long double corr = covar_pop / (std_popy * std_popx); - valOut = static_cast(corr); + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + if (var_popx <= 0) // Catch -0 + { + // When var_popx is 0, NULL is the result. + return mcsv1_UDAF::SUCCESS; } - return mcsv1_UDAF::SUCCESS; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + if (var_popy <= 0) // Catch -0 + { + // When var_popy is 0, NULL is the result + return mcsv1_UDAF::SUCCESS; + } + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode corr::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct corr_data* data = (struct corr_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct corr_data* data = (struct corr_data*)context->getUserData()->data; - data->sumy -= valy; - data->sumy2 -= valy*valy; + data->sumy -= valy; + data->sumy2 -= valy * valy; - data->sumx -= valx; - data->sumx2 -= valx*valx; + data->sumx -= valx; + data->sumx2 -= valx * valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/corr.h b/utils/regr/corr.h index 850690da4..0af80c616 100644 --- a/utils/regr/corr.h +++ b/utils/regr/corr.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* corr.h -***********************************************************************/ + * $Id$ + * + * corr.h + ***********************************************************************/ /** * Columnstore interface for for the corr function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the corr value of the dataset -class corr : public mcsv1_UDAF +class corr : public mcsv1_UDAF { -public: - // Defaults OK - corr() : mcsv1_UDAF() {}; - virtual ~corr() {}; + public: + // Defaults OK + corr() : mcsv1_UDAF(){}; + virtual ~corr(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_corr.h - +#endif // HEADER_corr.h diff --git a/utils/regr/covar_pop.cpp b/utils/regr/covar_pop.cpp index 0a3bb923d..7220dd36e 100644 --- a/utils/regr/covar_pop.cpp +++ b/utils/regr/covar_pop.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_covar_pop_ToUDAFMap { -public: - Add_covar_pop_ToUDAFMap() - { - UDAFMap::getMap()["covar_pop"] = new covar_pop(); - } + public: + Add_covar_pop_ToUDAFMap() + { + UDAFMap::getMap()["covar_pop"] = new covar_pop(); + } }; static Add_covar_pop_ToUDAFMap addToMap; @@ -38,113 +38,109 @@ static Add_covar_pop_ToUDAFMap addToMap; // Use the simple data model struct covar_pop_data { - uint64_t cnt; - long double sumx; - long double sumy; - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode covar_pop::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode covar_pop::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("covar_pop() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("covar_pop() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(covar_pop_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_pop() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_pop() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(covar_pop_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_pop::reset(mcsv1Context* context) { - struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_pop::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; - data->sumy += valy; - data->sumx += valx; + data->sumy += valy; + data->sumx += valx; - data->sumxy += valx*valy; + data->sumxy += valx * valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_pop::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct covar_pop_data* outData = (struct covar_pop_data*)context->getUserData()->data; - struct covar_pop_data* inData = (struct covar_pop_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumy += inData->sumy; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct covar_pop_data* outData = (struct covar_pop_data*)context->getUserData()->data; + struct covar_pop_data* inData = (struct covar_pop_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumy += inData->sumy; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_pop::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; - double N = data->cnt; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumxy = data->sumxy; + struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; + double N = data->cnt; + if (N > 0) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - valOut = static_cast(covar_pop); - } - return mcsv1_UDAF::SUCCESS; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + valOut = static_cast(covar_pop); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_pop::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct covar_pop_data* data = (struct covar_pop_data*)context->getUserData()->data; - data->sumy -= valy; - data->sumx -= valx; + data->sumy -= valy; + data->sumx -= valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/covar_pop.h b/utils/regr/covar_pop.h index dd675c7fa..e26af4f73 100644 --- a/utils/regr/covar_pop.h +++ b/utils/regr/covar_pop.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* covar_pop.h -***********************************************************************/ + * $Id$ + * + * covar_pop.h + ***********************************************************************/ /** * Columnstore interface for for the covar_pop function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the covar_pop value of the dataset -class covar_pop : public mcsv1_UDAF +class covar_pop : public mcsv1_UDAF { -public: - // Defaults OK - covar_pop() : mcsv1_UDAF() {}; - virtual ~covar_pop() {}; + public: + // Defaults OK + covar_pop() : mcsv1_UDAF(){}; + virtual ~covar_pop(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_covar_pop.h - +#endif // HEADER_covar_pop.h diff --git a/utils/regr/covar_samp.cpp b/utils/regr/covar_samp.cpp index 8966f8db9..fddcaab51 100644 --- a/utils/regr/covar_samp.cpp +++ b/utils/regr/covar_samp.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_covar_samp_ToUDAFMap { -public: - Add_covar_samp_ToUDAFMap() - { - UDAFMap::getMap()["covar_samp"] = new covar_samp(); - } + public: + Add_covar_samp_ToUDAFMap() + { + UDAFMap::getMap()["covar_samp"] = new covar_samp(); + } }; static Add_covar_samp_ToUDAFMap addToMap; @@ -38,120 +38,115 @@ static Add_covar_samp_ToUDAFMap addToMap; // Use the simple data model struct covar_samp_data { - uint64_t cnt; - long double sumx; - long double sumy; - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode covar_samp::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode covar_samp::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("covar_samp() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("covar_samp() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(covar_samp_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_samp() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_samp() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(covar_samp_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_samp::reset(mcsv1Context* context) { - struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_samp::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; - data->sumy += valy; + data->sumy += valy; - data->sumx += valx; + data->sumx += valx; - data->sumxy += valx*valy; + data->sumxy += valx * valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_samp::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct covar_samp_data* outData = (struct covar_samp_data*)context->getUserData()->data; - struct covar_samp_data* inData = (struct covar_samp_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumy += inData->sumy; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct covar_samp_data* outData = (struct covar_samp_data*)context->getUserData()->data; + struct covar_samp_data* inData = (struct covar_samp_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumy += inData->sumy; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; - double N = data->cnt; - if (N > 1) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumxy = data->sumxy; + struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; + double N = data->cnt; + if (N > 1) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); - valOut = static_cast(covar_samp); - } - else - if (N == 1) - { - valOut = 0; - } - return mcsv1_UDAF::SUCCESS; + long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); + valOut = static_cast(covar_samp); + } + else if (N == 1) + { + valOut = 0; + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode covar_samp::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; - data->sumy -= valy; + data->sumy -= valy; - data->sumx -= valx; + data->sumx -= valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/covar_samp.h b/utils/regr/covar_samp.h index f1511a364..5faf6c0ca 100644 --- a/utils/regr/covar_samp.h +++ b/utils/regr/covar_samp.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* covar_samp.h -***********************************************************************/ + * $Id$ + * + * covar_samp.h + ***********************************************************************/ /** * Columnstore interface for for the covar_samp function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the covar_samp value of the dataset -class covar_samp : public mcsv1_UDAF +class covar_samp : public mcsv1_UDAF { -public: - // Defaults OK - covar_samp() : mcsv1_UDAF() {}; - virtual ~covar_samp() {}; + public: + // Defaults OK + covar_samp() : mcsv1_UDAF(){}; + virtual ~covar_samp(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_covar_samp.h - +#endif // HEADER_covar_samp.h diff --git a/utils/regr/moda.cpp b/utils/regr/moda.cpp index c0980729f..a6a6476a3 100644 --- a/utils/regr/moda.cpp +++ b/utils/regr/moda.cpp @@ -29,11 +29,11 @@ using namespace mcsv1sdk; // map of UDAF for lookup class Add_moda_ToUDAFMap { -public: - Add_moda_ToUDAFMap() - { - UDAFMap::getMap()["moda"] = new moda(); - } + public: + Add_moda_ToUDAFMap() + { + UDAFMap::getMap()["moda"] = new moda(); + } }; static Add_moda_ToUDAFMap addToMap; @@ -55,8 +55,8 @@ static Add_moda_ToUDAFMap addToMap; // This class implements option 3 by creating templated classes. // There are two moda classes, the main one called moda, which is basically // an adapter (Pattern) to the templated class called Moda_impl_T. -// -// The way the API works, each function class is instantiated exactly once per +// +// The way the API works, each function class is instantiated exactly once per // executable and then accessed via a map. This means that the function classes // could be used by any active query, or more than once by a single query. These // classes have no data fields for this reason. All data for a specific query is @@ -65,518 +65,410 @@ static Add_moda_ToUDAFMap addToMap; // Each possible templated instantation is created ate moda creation during startup. // They are the Moda_impl_T members at the bottom of the moda class definition. // At runtime getImpl() gets the right one for the datatype involved based on context. -// +// // More template magic is done in the ModaData class to create and maintained // a hash of the correct type. // getImpl returns the current modaImpl or gets the correct one based on context. mcsv1_UDAF* moda::getImpl(mcsv1Context* context) { - ModaData* data = static_cast(context->getUserData()); - if (data->modaImpl) - return data->modaImpl; - - switch (context->getResultType()) - { - case execplan::CalpontSystemCatalog::TINYINT: - data->modaImpl = &moda_impl_int8; - break; - case execplan::CalpontSystemCatalog::SMALLINT: - data->modaImpl = &moda_impl_int16; - break; - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - data->modaImpl = &moda_impl_int32; - break; - case execplan::CalpontSystemCatalog::BIGINT: - data->modaImpl = &moda_impl_int64; - break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - switch (context->getColWidth()) - { - case 1: - data->modaImpl = &moda_impl_int8; - break; - case 2: - data->modaImpl = &moda_impl_int16; - break; - case 4: - data->modaImpl = &moda_impl_int32; - break; - case 8: - data->modaImpl = &moda_impl_int64; - break; - case 16: - data->modaImpl = &moda_impl_int128; - break; - } - break; - case execplan::CalpontSystemCatalog::UTINYINT: - data->modaImpl = &moda_impl_uint8; - break; - case execplan::CalpontSystemCatalog::USMALLINT: - data->modaImpl = &moda_impl_uint16; - break; - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - data->modaImpl = &moda_impl_uint32; - break; - case execplan::CalpontSystemCatalog::UBIGINT: - data->modaImpl = &moda_impl_uint64; - break; - case execplan::CalpontSystemCatalog::FLOAT: - data->modaImpl = &moda_impl_float; - break; - case execplan::CalpontSystemCatalog::DOUBLE: - data->modaImpl = &moda_impl_double; - break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - data->modaImpl = &moda_impl_longdouble; - break; - default: - data->modaImpl = NULL; - } + ModaData* data = static_cast(context->getUserData()); + if (data->modaImpl) return data->modaImpl; + + switch (context->getResultType()) + { + case execplan::CalpontSystemCatalog::TINYINT: data->modaImpl = &moda_impl_int8; break; + case execplan::CalpontSystemCatalog::SMALLINT: data->modaImpl = &moda_impl_int16; break; + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: data->modaImpl = &moda_impl_int32; break; + case execplan::CalpontSystemCatalog::BIGINT: data->modaImpl = &moda_impl_int64; break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + switch (context->getColWidth()) + { + case 1: data->modaImpl = &moda_impl_int8; break; + case 2: data->modaImpl = &moda_impl_int16; break; + case 4: data->modaImpl = &moda_impl_int32; break; + case 8: data->modaImpl = &moda_impl_int64; break; + case 16: data->modaImpl = &moda_impl_int128; break; + } + break; + case execplan::CalpontSystemCatalog::UTINYINT: data->modaImpl = &moda_impl_uint8; break; + case execplan::CalpontSystemCatalog::USMALLINT: data->modaImpl = &moda_impl_uint16; break; + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: data->modaImpl = &moda_impl_uint32; break; + case execplan::CalpontSystemCatalog::UBIGINT: data->modaImpl = &moda_impl_uint64; break; + case execplan::CalpontSystemCatalog::FLOAT: data->modaImpl = &moda_impl_float; break; + case execplan::CalpontSystemCatalog::DOUBLE: data->modaImpl = &moda_impl_double; break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: data->modaImpl = &moda_impl_longdouble; break; + default: data->modaImpl = NULL; + } + return data->modaImpl; } - -mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() < 1) + if (context->getParameterCount() < 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("moda() with 0 arguments"); + return mcsv1_UDAF::ERROR; + } + + if (context->getParameterCount() > 1) + { + context->setErrorMessage("moda() with more than 1 argument"); + return mcsv1_UDAF::ERROR; + } + + if (!(datatypes::isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("moda() with non-numeric argument"); + return mcsv1_UDAF::ERROR; + } + + context->setResultType(colTypes[0].dataType); + + if (colTypes[0].dataType == execplan::CalpontSystemCatalog::DECIMAL || + colTypes[0].dataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + if (colTypes[0].precision < 3) { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("moda() with 0 arguments"); - return mcsv1_UDAF::ERROR; + context->setColWidth(1); + } + else if (colTypes[0].precision < 4) + { + context->setColWidth(2); + } + else if (colTypes[0].precision < 9) + { + context->setColWidth(4); + } + else if (colTypes[0].precision < 19) + { + context->setColWidth(8); + } + else if (utils::widthByPrecision(colTypes[0].precision)) + { + context->setColWidth(16); } - if (context->getParameterCount() > 1) - { - context->setErrorMessage("moda() with more than 1 argument"); - return mcsv1_UDAF::ERROR; - } + context->setScale(colTypes[0].scale); + } + context->setPrecision(colTypes[0].precision); - if (!(datatypes::isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("moda() with non-numeric argument"); - return mcsv1_UDAF::ERROR; - } - - context->setResultType(colTypes[0].dataType); + mcsv1_UDAF* impl = getImpl(context); - if (colTypes[0].dataType == execplan::CalpontSystemCatalog::DECIMAL - || colTypes[0].dataType == execplan::CalpontSystemCatalog::UDECIMAL) - { - if (colTypes[0].precision < 3) - { - context->setColWidth(1); - } - else if (colTypes[0].precision < 4) - { - context->setColWidth(2); - } - else if (colTypes[0].precision < 9) - { - context->setColWidth(4); - } - else if (colTypes[0].precision < 19) - { - context->setColWidth(8); - } - else if (utils::widthByPrecision(colTypes[0].precision)) - { - context->setColWidth(16); - } - - context->setScale(colTypes[0].scale); - } - context->setPrecision(colTypes[0].precision); - - mcsv1_UDAF* impl = getImpl(context); - - if (!impl) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("moda() with non-numeric argument"); - return mcsv1_UDAF::ERROR; - } + if (!impl) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("moda() with non-numeric argument"); + return mcsv1_UDAF::ERROR; + } - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return impl->init(context, colTypes); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return impl->init(context, colTypes); } -template -mcsv1_UDAF::ReturnCode Moda_impl_T::init(mcsv1Context* context, - ColumnDatum* colTypes) +template +mcsv1_UDAF::ReturnCode Moda_impl_T::init(mcsv1Context* context, ColumnDatum* colTypes) { - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } -template +template mcsv1_UDAF::ReturnCode Moda_impl_T::reset(mcsv1Context* context) { - ModaData* data = static_cast(context->getUserData()); - data->fReturnType = context->getResultType(); - data->fColWidth = context->getColWidth(); - data->clear(); - return mcsv1_UDAF::SUCCESS; + ModaData* data = static_cast(context->getUserData()); + data->fReturnType = context->getResultType(); + data->fColWidth = context->getColWidth(); + data->clear(); + return mcsv1_UDAF::SUCCESS; } -template +template mcsv1_UDAF::ReturnCode Moda_impl_T::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - static_any::any& valIn = valsIn[0].columnData; - ModaData* data = static_cast(context->getUserData()); - std::unordered_map >* map = data->getMap(); + static_any::any& valIn = valsIn[0].columnData; + ModaData* data = static_cast(context->getUserData()); + std::unordered_map >* map = data->getMap(); - if (valIn.empty()) + if (valIn.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } + + T val = convertAnyTo(valIn); + + if (context->getResultType() == execplan::CalpontSystemCatalog::DOUBLE) + { + // For decimal types, we need to move the decimal point. + uint32_t scale = valsIn[0].scale; + + if (val != 0 && scale > 0) { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + val /= datatypes::scaleDivisor(scale); } + } - T val = convertAnyTo(valIn); + data->fSum += val; + ++data->fCount; + (*map)[val]++; - if (context->getResultType() == execplan::CalpontSystemCatalog::DOUBLE) - { - // For decimal types, we need to move the decimal point. - uint32_t scale = valsIn[0].scale; - - if (val != 0 && scale > 0) - { - val /= datatypes::scaleDivisor(scale); - } - } - - data->fSum += val; - ++data->fCount; - (*map)[val]++; - - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } -template +template mcsv1_UDAF::ReturnCode Moda_impl_T::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - ModaData* outData = static_cast(context->getUserData()); - const ModaData* inData = static_cast(userDataIn); - std::unordered_map >* outMap = outData->getMap(); - std::unordered_map >* inMap = inData->getMap(); - typename std::unordered_map >::const_iterator iter; - - for (iter = inMap->begin(); iter != inMap->end(); ++iter) - { - (*outMap)[iter->first] += iter->second; - } - // AVG - outData->fSum += inData->fSum; - outData->fCount += inData->fCount; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + ModaData* outData = static_cast(context->getUserData()); + const ModaData* inData = static_cast(userDataIn); + std::unordered_map >* outMap = outData->getMap(); + std::unordered_map >* inMap = inData->getMap(); + typename std::unordered_map >::const_iterator iter; + + for (iter = inMap->begin(); iter != inMap->end(); ++iter) + { + (*outMap)[iter->first] += iter->second; + } + // AVG + outData->fSum += inData->fSum; + outData->fCount += inData->fCount; + + return mcsv1_UDAF::SUCCESS; } -template +template mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_any::any& valOut) { - uint64_t maxCnt = 0; - long double avg = 0; - T val = 0; - ModaData* data = static_cast(context->getUserData()); - std::unordered_map >* map = data->getMap(); + uint64_t maxCnt = 0; + long double avg = 0; + T val = 0; + ModaData* data = static_cast(context->getUserData()); + std::unordered_map >* map = data->getMap(); - if (map->size() == 0) - { - valOut = (T)0; - return mcsv1_UDAF::SUCCESS; - } - - avg = data->fCount ? data->fSum / data->fCount : 0; - typename std::unordered_map >::iterator iter; - - for (iter = map->begin(); iter != map->end(); ++iter) - { - if (iter->second > maxCnt) - { - val = iter->first; - maxCnt = iter->second; - } - else if (iter->second == maxCnt) - { - T absval = val >= 0 ? val : -val; - T absfirst = iter->first >= 0 ? iter->first : -iter->first; - // Tie breaker: choose the closest to avg. If still tie, choose smallest - long double dist1 = val > avg ? (long double)val-avg : avg-(long double)val; - long double dist2 = iter->first > avg ? (long double)iter->first-avg : avg-(long double)iter->first; - if ((dist1 > dist2) - || ((dist1 == dist2) && (absval > absfirst))) - { - val = iter->first; - } - } - } - - // If scale is > 0, then the original type was DECIMAL. Set the - // ResultType to DECIMAL so the delivery logic moves the decimal point. - if (context->getScale() > 0) - context->setResultType(execplan::CalpontSystemCatalog::DECIMAL); - - valOut = val; + if (map->size() == 0) + { + valOut = (T)0; return mcsv1_UDAF::SUCCESS; + } + + avg = data->fCount ? data->fSum / data->fCount : 0; + typename std::unordered_map >::iterator iter; + + for (iter = map->begin(); iter != map->end(); ++iter) + { + if (iter->second > maxCnt) + { + val = iter->first; + maxCnt = iter->second; + } + else if (iter->second == maxCnt) + { + T absval = val >= 0 ? val : -val; + T absfirst = iter->first >= 0 ? iter->first : -iter->first; + // Tie breaker: choose the closest to avg. If still tie, choose smallest + long double dist1 = val > avg ? (long double)val - avg : avg - (long double)val; + long double dist2 = iter->first > avg ? (long double)iter->first - avg : avg - (long double)iter->first; + if ((dist1 > dist2) || ((dist1 == dist2) && (absval > absfirst))) + { + val = iter->first; + } + } + } + + // If scale is > 0, then the original type was DECIMAL. Set the + // ResultType to DECIMAL so the delivery logic moves the decimal point. + if (context->getScale() > 0) + context->setResultType(execplan::CalpontSystemCatalog::DECIMAL); + + valOut = val; + return mcsv1_UDAF::SUCCESS; } -template +template mcsv1_UDAF::ReturnCode Moda_impl_T::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - static_any::any& valDropped = valsDropped[0].columnData; - ModaData* data = static_cast(context->getUserData()); - std::unordered_map >* map = data->getMap(); + static_any::any& valDropped = valsDropped[0].columnData; + ModaData* data = static_cast(context->getUserData()); + std::unordered_map >* map = data->getMap(); - if (valDropped.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valDropped.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - T val = convertAnyTo(valDropped); + T val = convertAnyTo(valDropped); - data->fSum -= val; - --data->fCount; - (*map)[val]--; + data->fSum -= val; + --data->fCount; + (*map)[val]--; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } void ModaData::serialize(messageqcpp::ByteStream& bs) const { - bs << fReturnType; - bs << fSum; - bs << fCount; - bs << fColWidth; - - switch ((execplan::CalpontSystemCatalog::ColDataType)fReturnType) - { - case execplan::CalpontSystemCatalog::TINYINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::SMALLINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::BIGINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - switch (fColWidth) - { - case 1: - serializeMap(bs); - break; - case 2: - serializeMap(bs); - break; - case 4: - serializeMap(bs); - break; - case 8: - serializeMap(bs); - break; - case 16: - serializeMap(bs); - break; - } - break; - case execplan::CalpontSystemCatalog::UTINYINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::USMALLINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::UBIGINT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::FLOAT: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::DOUBLE: - serializeMap(bs); - break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - serializeMap(bs); - break; - default: - throw std::runtime_error("ModaData::serialize with bad data type"); - break; - } + bs << fReturnType; + bs << fSum; + bs << fCount; + bs << fColWidth; + + switch ((execplan::CalpontSystemCatalog::ColDataType)fReturnType) + { + case execplan::CalpontSystemCatalog::TINYINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::SMALLINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::BIGINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + switch (fColWidth) + { + case 1: serializeMap(bs); break; + case 2: serializeMap(bs); break; + case 4: serializeMap(bs); break; + case 8: serializeMap(bs); break; + case 16: serializeMap(bs); break; + } + break; + case execplan::CalpontSystemCatalog::UTINYINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::USMALLINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::UBIGINT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::FLOAT: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::DOUBLE: serializeMap(bs); break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: serializeMap(bs); break; + default: throw std::runtime_error("ModaData::serialize with bad data type"); break; + } } void ModaData::unserialize(messageqcpp::ByteStream& bs) { - bs >> fReturnType; - bs >> fSum; - bs >> fCount; - bs >> fColWidth; + bs >> fReturnType; + bs >> fSum; + bs >> fCount; + bs >> fColWidth; - switch ((execplan::CalpontSystemCatalog::ColDataType)fReturnType) - { - case execplan::CalpontSystemCatalog::TINYINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::SMALLINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::BIGINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - switch (fColWidth) - { - case 1: - unserializeMap(bs); - break; - case 2: - unserializeMap(bs); - break; - case 4: - unserializeMap(bs); - break; - case 8: - unserializeMap(bs); - break; - case 16: - unserializeMap(bs); - break; - } - break; - case execplan::CalpontSystemCatalog::UTINYINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::USMALLINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::UBIGINT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::FLOAT: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::DOUBLE: - unserializeMap(bs); - break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - unserializeMap(bs); - break; - default: - throw std::runtime_error("ModaData::unserialize with bad data type"); - break; - } + switch ((execplan::CalpontSystemCatalog::ColDataType)fReturnType) + { + case execplan::CalpontSystemCatalog::TINYINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::SMALLINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::BIGINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + switch (fColWidth) + { + case 1: unserializeMap(bs); break; + case 2: unserializeMap(bs); break; + case 4: unserializeMap(bs); break; + case 8: unserializeMap(bs); break; + case 16: unserializeMap(bs); break; + } + break; + case execplan::CalpontSystemCatalog::UTINYINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::USMALLINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::UBIGINT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::FLOAT: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::DOUBLE: unserializeMap(bs); break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: unserializeMap(bs); break; + default: throw std::runtime_error("ModaData::unserialize with bad data type"); break; + } } void ModaData::cleanup() { - if (!fMap) - return; - switch ((execplan::CalpontSystemCatalog::ColDataType)fReturnType) - { - case execplan::CalpontSystemCatalog::TINYINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::SMALLINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::BIGINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - switch (fColWidth) - { - case 1: - clear(); - deleteMap(); - break; - case 2: - clear(); - deleteMap(); - break; - case 4: - clear(); - deleteMap(); - break; - case 8: - clear(); - deleteMap(); - break; - case 16: - clear(); - deleteMap(); - break; - } - break; - case execplan::CalpontSystemCatalog::UTINYINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::USMALLINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::UBIGINT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::FLOAT: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::DOUBLE: - clear(); - deleteMap(); - break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - clear(); - deleteMap(); - break; - default: - throw std::runtime_error("ModaData::unserialize with bad data type"); - break; - } + if (!fMap) + return; + switch ((execplan::CalpontSystemCatalog::ColDataType)fReturnType) + { + case execplan::CalpontSystemCatalog::TINYINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::SMALLINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::BIGINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + switch (fColWidth) + { + case 1: + clear(); + deleteMap(); + break; + case 2: + clear(); + deleteMap(); + break; + case 4: + clear(); + deleteMap(); + break; + case 8: + clear(); + deleteMap(); + break; + case 16: + clear(); + deleteMap(); + break; + } + break; + case execplan::CalpontSystemCatalog::UTINYINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::USMALLINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::UBIGINT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::FLOAT: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::DOUBLE: + clear(); + deleteMap(); + break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + clear(); + deleteMap(); + break; + default: throw std::runtime_error("ModaData::unserialize with bad data type"); break; + } } - diff --git a/utils/regr/moda.h b/utils/regr/moda.h index ae47f8462..7b59d51fa 100644 --- a/utils/regr/moda.h +++ b/utils/regr/moda.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* moda.h -***********************************************************************/ + * $Id$ + * + * moda.h + ***********************************************************************/ /** * Columnstore interface for the moda User Defined Aggregate @@ -55,232 +55,232 @@ namespace mcsv1sdk { -// A hasher that handles int128_t -template +// A hasher that handles int128_t +template struct hasher { - inline size_t operator()(T val) const - { - return fHasher((char*) &val, sizeof(T)); - } + inline size_t operator()(T val) const + { + return fHasher((char*)&val, sizeof(T)); + } -private: - utils::Hasher fHasher; + private: + utils::Hasher fHasher; }; -template<> +template <> struct hasher { - inline size_t operator()(long double val) const + inline size_t operator()(long double val) const + { + if (sizeof(long double) == 8) // Probably just MSC, but you never know. { - if (sizeof(long double) == 8) // Probably just MSC, but you never know. - { - return fHasher((char*) &val, sizeof(long double)); - } - else - { - // For Linux x86_64, long double is stored in 128 bits, but only 80 are significant - return fHasher((char*) &val, 10); - } + return fHasher((char*)&val, sizeof(long double)); } -private: - utils::Hasher fHasher; + else + { + // For Linux x86_64, long double is stored in 128 bits, but only 80 are significant + return fHasher((char*)&val, 10); + } + } + + private: + utils::Hasher fHasher; }; // Override UserData for data storage struct ModaData : public UserData { - ModaData() : fMap(NULL), - fReturnType((uint32_t)execplan::CalpontSystemCatalog::UNDEFINED), - fColWidth(0), - modaImpl(NULL) - {}; + ModaData() + : fMap(NULL) + , fReturnType((uint32_t)execplan::CalpontSystemCatalog::UNDEFINED) + , fColWidth(0) + , modaImpl(NULL){}; - virtual ~ModaData() {cleanup();} + virtual ~ModaData() + { + cleanup(); + } - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + virtual void serialize(messageqcpp::ByteStream& bs) const; + virtual void unserialize(messageqcpp::ByteStream& bs); - template - std::unordered_map >* getMap() + template + std::unordered_map >* getMap() + { + if (!fMap) { - if (!fMap) - { - // Just in time creation - fMap = new std::unordered_map >; - } - return (std::unordered_map >*) fMap; - } - - // The const version is only called by serialize() - // It shouldn't (and can't) create a new map. - template - std::unordered_map >* getMap() const - { - return (std::unordered_map >*) fMap; - } - - template - void deleteMap() - { - if (fMap) - { - delete (std::unordered_map >*) fMap; - fMap = NULL; - } - } - - template - void clear() - { - fSum = 0.0; - fCount = 0; - if (fMap) - getMap()->clear(); + // Just in time creation + fMap = new std::unordered_map >; } + return (std::unordered_map >*)fMap; + } - long double fSum; - uint64_t fCount; - void* fMap; // Will be of type unordered_map<> - uint32_t fReturnType; - uint32_t fColWidth; - mcsv1_UDAF* modaImpl; // A pointer to one of the Moda_impl_T concrete classes - -private: - // For now, copy construction is unwanted - ModaData(UserData&); - - void cleanup(); - - // Templated map streamers - template - void serializeMap(messageqcpp::ByteStream& bs) const + // The const version is only called by serialize() + // It shouldn't (and can't) create a new map. + template + std::unordered_map >* getMap() const + { + return (std::unordered_map >*)fMap; + } + + template + void deleteMap() + { + if (fMap) { - std::unordered_map >* map = getMap(); - if (map) - { - typename std::unordered_map >::const_iterator iter; - bs << (uint64_t)map->size(); - for (iter = map->begin(); iter != map->end(); ++iter) - { - bs << iter->first; - bs << iter->second; - } - } - else - { - bs << (uint64_t)0; - } + delete (std::unordered_map >*)fMap; + fMap = NULL; } - - template - void unserializeMap(messageqcpp::ByteStream& bs) + } + + template + void clear() + { + fSum = 0.0; + fCount = 0; + if (fMap) + getMap()->clear(); + } + + long double fSum; + uint64_t fCount; + void* fMap; // Will be of type unordered_map<> + uint32_t fReturnType; + uint32_t fColWidth; + mcsv1_UDAF* modaImpl; // A pointer to one of the Moda_impl_T concrete classes + + private: + // For now, copy construction is unwanted + ModaData(UserData&); + + void cleanup(); + + // Templated map streamers + template + void serializeMap(messageqcpp::ByteStream& bs) const + { + std::unordered_map >* map = getMap(); + if (map) { - uint32_t cnt; - T num; - uint64_t sz; - bs >> sz; - std::unordered_map >* map = getMap(); - map->clear(); - for (uint64_t i = 0; i < sz; ++i) - { - bs >> num; - bs >> cnt; - (*map)[num] = cnt; - } + typename std::unordered_map >::const_iterator iter; + bs << (uint64_t)map->size(); + for (iter = map->begin(); iter != map->end(); ++iter) + { + bs << iter->first; + bs << iter->second; + } } + else + { + bs << (uint64_t)0; + } + } + + template + void unserializeMap(messageqcpp::ByteStream& bs) + { + uint32_t cnt; + T num; + uint64_t sz; + bs >> sz; + std::unordered_map >* map = getMap(); + map->clear(); + for (uint64_t i = 0; i < sz; ++i) + { + bs >> num; + bs >> cnt; + (*map)[num] = cnt; + } + } }; -template +template class Moda_impl_T : public mcsv1_UDAF { -public: - // Defaults OK - Moda_impl_T() {}; - virtual ~Moda_impl_T() {}; + public: + // Defaults OK + Moda_impl_T(){}; + virtual ~Moda_impl_T(){}; - virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual mcsv1_UDAF::ReturnCode reset(mcsv1Context* context); - virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual mcsv1_UDAF::ReturnCode reset(mcsv1Context* context); + virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); - // Dummy: not used - virtual mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) - { - return mcsv1_UDAF::SUCCESS; - } + // Dummy: not used + virtual mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) + { + return mcsv1_UDAF::SUCCESS; + } }; // moda returns the modal value of the dataset. If more than one value // have the same maximum number of occurances, then the one closest to // AVG wins. If two are the same distance from AVG, then the smaller wins. -class moda : public mcsv1_UDAF +class moda : public mcsv1_UDAF { -public: - // Defaults OK - moda() : mcsv1_UDAF() {}; - virtual ~moda() {}; + public: + // Defaults OK + moda() : mcsv1_UDAF(){}; + virtual ~moda(){}; - virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context) - { - return getImpl(context)->reset(context); - } + virtual ReturnCode reset(mcsv1Context* context) + { + return getImpl(context)->reset(context); + } - virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) - { - return getImpl(context)->nextValue(context, valsIn); - } + virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) + { + return getImpl(context)->nextValue(context, valsIn); + } - virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) - { - return getImpl(context)->subEvaluate(context, valIn); - } - - virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) - { - return getImpl(context)->evaluate(context, valOut); - } - - virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) - { - return getImpl(context)->dropValue(context, valsDropped); - } + virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) + { + return getImpl(context)->subEvaluate(context, valIn); + } - mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) - { - userData = new ModaData; - length = sizeof(ModaData); - return mcsv1_UDAF::SUCCESS; - } + virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) + { + return getImpl(context)->evaluate(context, valOut); + } - mcsv1_UDAF* getImpl(mcsv1Context* context); + virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) + { + return getImpl(context)->dropValue(context, valsDropped); + } -protected: - Moda_impl_T moda_impl_int8; - Moda_impl_T moda_impl_int16; - Moda_impl_T moda_impl_int32; - Moda_impl_T moda_impl_int64; - Moda_impl_T moda_impl_int128; - Moda_impl_T moda_impl_uint8; - Moda_impl_T moda_impl_uint16; - Moda_impl_T moda_impl_uint32; - Moda_impl_T moda_impl_uint64; - Moda_impl_T moda_impl_float; - Moda_impl_T moda_impl_double; - Moda_impl_T moda_impl_longdouble; + mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) + { + userData = new ModaData; + length = sizeof(ModaData); + return mcsv1_UDAF::SUCCESS; + } + + mcsv1_UDAF* getImpl(mcsv1Context* context); + + protected: + Moda_impl_T moda_impl_int8; + Moda_impl_T moda_impl_int16; + Moda_impl_T moda_impl_int32; + Moda_impl_T moda_impl_int64; + Moda_impl_T moda_impl_int128; + Moda_impl_T moda_impl_uint8; + Moda_impl_T moda_impl_uint16; + Moda_impl_T moda_impl_uint32; + Moda_impl_T moda_impl_uint64; + Moda_impl_T moda_impl_float; + Moda_impl_T moda_impl_double; + Moda_impl_T moda_impl_longdouble; }; - -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_mode.h - +#endif // HEADER_mode.h diff --git a/utils/regr/modamysql.cpp b/utils/regr/modamysql.cpp index 770b72b7a..ac3584cef 100644 --- a/utils/regr/modamysql.cpp +++ b/utils/regr/modamysql.cpp @@ -12,90 +12,88 @@ namespace { inline bool isNumeric(int type, const char* attr) { - if (type == INT_RESULT || type == REAL_RESULT || type == DECIMAL_RESULT) - { - return true; - } + if (type == INT_RESULT || type == REAL_RESULT || type == DECIMAL_RESULT) + { + return true; + } #if _MSC_VER - if (_strnicmp("NULL", attr, 4) == 0)) + if (_strnicmp("NULL", attr, 4) == 0)) #else - if (strncasecmp("NULL", attr, 4) == 0) + if (strncasecmp("NULL", attr, 4) == 0) #endif { - return true; + return true; } - return false; + return false; } struct moda_data { - long double fSum; - uint64_t fCount; - enum Item_result fReturnType; - std::unordered_map mapINT; - std::unordered_map mapREAL; - std::unordered_map mapDECIMAL; - std::string result; - void clear() - { - fSum = 0.0; - fCount = 0; - mapINT.clear(); - mapREAL.clear(); - mapDECIMAL.clear(); - } + long double fSum; + uint64_t fCount; + enum Item_result fReturnType; + std::unordered_map mapINT; + std::unordered_map mapREAL; + std::unordered_map mapDECIMAL; + std::string result; + void clear() + { + fSum = 0.0; + fCount = 0; + mapINT.clear(); + mapREAL.clear(); + mapDECIMAL.clear(); + } }; -} +} // namespace -template -char * moda(CONTAINER & container, struct moda_data* data) +template +char* moda(CONTAINER& container, struct moda_data* data) { - TYPE avg = (TYPE)data->fCount ? data->fSum / data->fCount : 0; - TYPE val = 0.0; - uint32_t maxCnt = 0.0; + TYPE avg = (TYPE)data->fCount ? data->fSum / data->fCount : 0; + TYPE val = 0.0; + uint32_t maxCnt = 0.0; - for (auto iter = container.begin(); iter != container.end(); ++iter) + for (auto iter = container.begin(); iter != container.end(); ++iter) + { + if (iter->second > maxCnt) { - if (iter->second > maxCnt) - { - val = iter->first; - maxCnt = iter->second; - } - else if (iter->second == maxCnt) - { - // Tie breaker: choose the closest to avg. If still tie, choose smallest - if ((abs(val-avg) > abs(iter->first-avg)) - || ((abs(val-avg) == abs(iter->first-avg)) && (abs(val) > abs(iter->first)))) - { - val = iter->first; - } - } + val = iter->first; + maxCnt = iter->second; } + else if (iter->second == maxCnt) + { + // Tie breaker: choose the closest to avg. If still tie, choose smallest + if ((abs(val - avg) > abs(iter->first - avg)) || + ((abs(val - avg) == abs(iter->first - avg)) && (abs(val) > abs(iter->first)))) + { + val = iter->first; + } + } + } - data->result = std::to_string(val); + data->result = std::to_string(val); - return const_cast(data->result.c_str()); + return const_cast(data->result.c_str()); } - extern "C" { - #ifdef _MSC_VER -__declspec(dllexport) + __declspec(dllexport) #endif -my_bool moda_init(UDF_INIT* initid, UDF_ARGS* args, char* message) -{ + my_bool moda_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { struct moda_data* data; if (args->arg_count != 1) { - strcpy(message,"moda() requires one argument"); - return 1; + strcpy(message, "moda() requires one argument"); + return 1; } if (!isNumeric(args->arg_type[0], args->attributes[0])) { - strcpy(message,"moda() with a non-numeric argument"); - return 1; + strcpy(message, "moda() with a non-numeric argument"); + return 1; } data = new moda_data; @@ -104,40 +102,37 @@ my_bool moda_init(UDF_INIT* initid, UDF_ARGS* args, char* message) data->fSum = 0.0; initid->ptr = (char*)data; return 0; -} + } #ifdef _MSC_VER -__declspec(dllexport) + __declspec(dllexport) #endif -void moda_deinit(UDF_INIT* initid) -{ + void moda_deinit(UDF_INIT* initid) + { struct moda_data* data = (struct moda_data*)initid->ptr; data->clear(); delete data; -} + } #ifdef _MSC_VER -__declspec(dllexport) + __declspec(dllexport) #endif -void moda_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) -{ + void moda_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { struct moda_data* data = (struct moda_data*)initid->ptr; data->clear(); -} + } #ifdef _MSC_VER -__declspec(dllexport) + __declspec(dllexport) #endif -void moda_add(UDF_INIT* initid, - UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) -{ + void moda_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { // Test for NULL if (args->args[0] == 0) { - return; + return; } struct moda_data* data = (struct moda_data*)initid->ptr; @@ -145,96 +140,88 @@ void moda_add(UDF_INIT* initid, switch (args->arg_type[0]) { - case INT_RESULT: - { - int64_t val = *((int64_t*)args->args[0]); - data->fSum += (long double)val; - data->mapINT[val]++; - break; - } - case REAL_RESULT: - { - double val = *((double*)args->args[0]); - data->fSum += val; - data->mapREAL[val]++; - break; - } - case DECIMAL_RESULT: - case STRING_RESULT: - { - long double val = strtold(args->args[0], 0); - data->fSum += val; - data->mapDECIMAL[val]++; - break; - } - default: - break; + case INT_RESULT: + { + int64_t val = *((int64_t*)args->args[0]); + data->fSum += (long double)val; + data->mapINT[val]++; + break; + } + case REAL_RESULT: + { + double val = *((double*)args->args[0]); + data->fSum += val; + data->mapREAL[val]++; + break; + } + case DECIMAL_RESULT: + case STRING_RESULT: + { + long double val = strtold(args->args[0], 0); + data->fSum += val; + data->mapDECIMAL[val]++; + break; + } + default: break; } -} + } #ifdef _MSC_VER -__declspec(dllexport) + __declspec(dllexport) #endif -void moda_remove(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) -{ + void moda_remove(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { // Test for NULL if (args->args[0] == 0) { - return; + return; } struct moda_data* data = (struct moda_data*)initid->ptr; data->fCount--; switch (args->arg_type[0]) { - case INT_RESULT: - { - int64_t val = *((int64_t*)args->args[0]); - data->fSum -= (long double)val; - data->mapINT[val]--; - break; - } - case REAL_RESULT: - { - double val = *((double*)args->args[0]); - data->fSum -= val; - data->mapREAL[val]--; - break; - } - case DECIMAL_RESULT: - case STRING_RESULT: - { - long double val = strtold(args->args[0], 0); - data->fSum -= val; - data->mapDECIMAL[val]--; - break; - } - default: - break; + case INT_RESULT: + { + int64_t val = *((int64_t*)args->args[0]); + data->fSum -= (long double)val; + data->mapINT[val]--; + break; + } + case REAL_RESULT: + { + double val = *((double*)args->args[0]); + data->fSum -= val; + data->mapREAL[val]--; + break; + } + case DECIMAL_RESULT: + case STRING_RESULT: + { + long double val = strtold(args->args[0], 0); + data->fSum -= val; + data->mapDECIMAL[val]--; + break; + } + default: break; } -} + } #ifdef _MSC_VER -__declspec(dllexport) + __declspec(dllexport) #endif -char* moda(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error __attribute__((unused))) -{ + char* moda(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error __attribute__((unused))) + { struct moda_data* data = (struct moda_data*)initid->ptr; switch (args->arg_type[0]) { - case INT_RESULT: - return moda(data->mapINT, data); - case REAL_RESULT: - return moda(data->mapREAL, data); - case DECIMAL_RESULT: - case STRING_RESULT: - return moda(data->mapDECIMAL, data); - default: - return NULL; + case INT_RESULT: return moda(data->mapINT, data); + case REAL_RESULT: return moda(data->mapREAL, data); + case DECIMAL_RESULT: + case STRING_RESULT: return moda(data->mapDECIMAL, data); + default: return NULL; } return NULL; -} -} // Extern "C" + } +} // Extern "C" diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index 39448efbe..0f134b5b3 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_avgx_ToUDAFMap { -public: - Add_regr_avgx_ToUDAFMap() - { - UDAFMap::getMap()["regr_avgx"] = new regr_avgx(); - } + public: + Add_regr_avgx_ToUDAFMap() + { + UDAFMap::getMap()["regr_avgx"] = new regr_avgx(); + } }; static Add_regr_avgx_ToUDAFMap addToMap; @@ -40,97 +40,93 @@ static Add_regr_avgx_ToUDAFMap addToMap; // Use the simple data model struct regr_avgx_data { - long double sum; - uint64_t cnt; + long double sum; + uint64_t cnt; }; - -mcsv1_UDAF::ReturnCode regr_avgx::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_avgx::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgx() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgx() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } - if (!(isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgx() with a non-numeric x argument"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_avgx_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(colTypes[1].scale + 4); - context->setPrecision(19); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (!(isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgx() with a non-numeric x argument"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_avgx_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(colTypes[1].scale + 4); + context->setPrecision(19); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgx::reset(mcsv1Context* context) { - struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; - data->sum = 0; - data->cnt = 0; - return mcsv1_UDAF::SUCCESS; + struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; + data->sum = 0; + data->cnt = 0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgx::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - DATATYPE val = toDouble(valsIn[1]); - struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; + DATATYPE val = toDouble(valsIn[1]); + struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; - data->sum += val; - ++data->cnt; + data->sum += val; + ++data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgx::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_avgx_data* outData = (struct regr_avgx_data*)context->getUserData()->data; - - struct regr_avgx_data* inData = (struct regr_avgx_data*)userDataIn->data; - - outData->sum += inData->sum; - - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_avgx_data* outData = (struct regr_avgx_data*)context->getUserData()->data; + + struct regr_avgx_data* inData = (struct regr_avgx_data*)userDataIn->data; + + outData->sum += inData->sum; + + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgx::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; + struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; - if (data->cnt > 0) - { - valOut = static_cast(data->sum / (long double)data->cnt); - } + if (data->cnt > 0) + { + valOut = static_cast(data->sum / (long double)data->cnt); + } - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgx::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double val = toDouble(valsDropped[1]); - struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; + double val = toDouble(valsDropped[1]); + struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; - data->sum -= val; - --data->cnt; + data->sum -= val; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_avgx.h b/utils/regr/regr_avgx.h index 58138eff3..c7fc63775 100644 --- a/utils/regr/regr_avgx.h +++ b/utils/regr/regr_avgx.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_avgx.h -***********************************************************************/ + * $Id$ + * + * regr_avgx.h + ***********************************************************************/ /** * Columnstore interface for for the regr_avgx function @@ -52,7 +52,6 @@ namespace mcsv1sdk { - // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or // User Defined Analytic Function (UDAnF). // These will be singleton classes, so don't put any instance @@ -65,32 +64,30 @@ namespace mcsv1sdk // Return the regr_avgx value of the dataset -class regr_avgx : public mcsv1_UDAF +class regr_avgx : public mcsv1_UDAF { -public: - // Defaults OK - regr_avgx() : mcsv1_UDAF() {}; - virtual ~regr_avgx() {}; + public: + // Defaults OK + regr_avgx() : mcsv1_UDAF(){}; + virtual ~regr_avgx(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_avgx.h - +#endif // HEADER_regr_avgx.h diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index ddfb1c914..f8a614a07 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_avgy_ToUDAFMap { -public: - Add_regr_avgy_ToUDAFMap() - { - UDAFMap::getMap()["regr_avgy"] = new regr_avgy(); - } + public: + Add_regr_avgy_ToUDAFMap() + { + UDAFMap::getMap()["regr_avgy"] = new regr_avgy(); + } }; static Add_regr_avgy_ToUDAFMap addToMap; @@ -40,94 +40,90 @@ static Add_regr_avgy_ToUDAFMap addToMap; // Use the simple data model struct regr_avgy_data { - long double sum; - uint64_t cnt; + long double sum; + uint64_t cnt; }; - -mcsv1_UDAF::ReturnCode regr_avgy::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_avgy::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgy() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgy() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgy() with a non-numeric y argument"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_avgy_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(colTypes[0].scale + 4); - context->setPrecision(19); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgy() with a non-numeric y argument"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_avgy_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(colTypes[0].scale + 4); + context->setPrecision(19); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgy::reset(mcsv1Context* context) { - struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; - data->sum = 0; - data->cnt = 0; - return mcsv1_UDAF::SUCCESS; + struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; + data->sum = 0; + data->cnt = 0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgy::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double val = toDouble(valsIn[0]); - struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; + double val = toDouble(valsIn[0]); + struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; - data->sum += val; - ++data->cnt; + data->sum += val; + ++data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgy::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_avgy_data* outData = (struct regr_avgy_data*)context->getUserData()->data; - struct regr_avgy_data* inData = (struct regr_avgy_data*)userDataIn->data; - - outData->sum += inData->sum; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_avgy_data* outData = (struct regr_avgy_data*)context->getUserData()->data; + struct regr_avgy_data* inData = (struct regr_avgy_data*)userDataIn->data; + + outData->sum += inData->sum; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgy::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; + struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; - if (data->cnt > 0) - { - valOut = static_cast(data->sum / (long double)data->cnt); - } - return mcsv1_UDAF::SUCCESS; + if (data->cnt > 0) + { + valOut = static_cast(data->sum / (long double)data->cnt); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_avgy::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double val = toDouble(valsDropped[0]); - struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; + double val = toDouble(valsDropped[0]); + struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; - data->sum -= val; - --data->cnt; + data->sum -= val; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_avgy.h b/utils/regr/regr_avgy.h index 957becdd7..3f061f3b4 100644 --- a/utils/regr/regr_avgy.h +++ b/utils/regr/regr_avgy.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_avgy.h -***********************************************************************/ + * $Id$ + * + * regr_avgy.h + ***********************************************************************/ /** * Columnstore interface for for the regr_avgy function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_avgy value of the dataset -class regr_avgy : public mcsv1_UDAF +class regr_avgy : public mcsv1_UDAF { -public: - // Defaults OK - regr_avgy() : mcsv1_UDAF() {}; - virtual ~regr_avgy() {}; + public: + // Defaults OK + regr_avgy() : mcsv1_UDAF(){}; + virtual ~regr_avgy(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_avgy.h - +#endif // HEADER_regr_avgy.h diff --git a/utils/regr/regr_count.cpp b/utils/regr/regr_count.cpp index eeb5e09a2..689914175 100644 --- a/utils/regr/regr_count.cpp +++ b/utils/regr/regr_count.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_count_ToUDAFMap { -public: - Add_regr_count_ToUDAFMap() - { - UDAFMap::getMap()["regr_count"] = new regr_count(); - } + public: + Add_regr_count_ToUDAFMap() + { + UDAFMap::getMap()["regr_count"] = new regr_count(); + } }; static Add_regr_count_ToUDAFMap addToMap; @@ -38,94 +38,90 @@ static Add_regr_count_ToUDAFMap addToMap; // Use the simple data model struct regr_count_data { - long long cnt; + long long cnt; }; - -mcsv1_UDAF::ReturnCode regr_count::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_count::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_count() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_count_data)); - context->setResultType(execplan::CalpontSystemCatalog::BIGINT); - context->setColWidth(8); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_count() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_count_data)); + context->setResultType(execplan::CalpontSystemCatalog::BIGINT); + context->setColWidth(8); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_count::reset(mcsv1Context* context) { - struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; - data->cnt = 0; - return mcsv1_UDAF::SUCCESS; + struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; + data->cnt = 0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_count::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - static_any::any& valIn_y = valsIn[0].columnData; - static_any::any& valIn_x = valsIn[1].columnData; - struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; + static_any::any& valIn_y = valsIn[0].columnData; + static_any::any& valIn_x = valsIn[1].columnData; + struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; - if (context->isParamNull(0) || context->isParamNull(1)) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } - if (valIn_x.empty() || valIn_y.empty()) // Usually empty if NULL. Probably redundant - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } - ++data->cnt; + if (context->isParamNull(0) || context->isParamNull(1)) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } + if (valIn_x.empty() || valIn_y.empty()) // Usually empty if NULL. Probably redundant + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } + ++data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_count::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_count_data* outData = (struct regr_count_data*)context->getUserData()->data; - struct regr_count_data* inData = (struct regr_count_data*)userDataIn->data; - - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_count_data* outData = (struct regr_count_data*)context->getUserData()->data; + struct regr_count_data* inData = (struct regr_count_data*)userDataIn->data; + + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_count::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; + struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; - valOut = data->cnt; - return mcsv1_UDAF::SUCCESS; + valOut = data->cnt; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_count::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - static_any::any& valIn_y = valsDropped[0].columnData; - static_any::any& valIn_x = valsDropped[1].columnData; - struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; + static_any::any& valIn_y = valsDropped[0].columnData; + static_any::any& valIn_x = valsDropped[1].columnData; + struct regr_count_data* data = (struct regr_count_data*)context->getUserData()->data; - if (context->isParamNull(0) || context->isParamNull(1)) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } - if (valIn_x.empty() || valIn_y.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } - --data->cnt; + if (context->isParamNull(0) || context->isParamNull(1)) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } + if (valIn_x.empty() || valIn_y.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_count.h b/utils/regr/regr_count.h index b63a2d582..cf1ec6a8d 100644 --- a/utils/regr/regr_count.h +++ b/utils/regr/regr_count.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_count.h -***********************************************************************/ + * $Id$ + * + * regr_count.h + ***********************************************************************/ /** * Columnstore interface for for the regr_count function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_count value of the dataset -class regr_count : public mcsv1_UDAF +class regr_count : public mcsv1_UDAF { -public: - // Defaults OK - regr_count() : mcsv1_UDAF() {}; - virtual ~regr_count() {}; + public: + // Defaults OK + regr_count() : mcsv1_UDAF(){}; + virtual ~regr_count(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_count.h - +#endif // HEADER_regr_count.h diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index f27522d63..8dda7c08a 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_intercept_ToUDAFMap { -public: - Add_regr_intercept_ToUDAFMap() - { - UDAFMap::getMap()["regr_intercept"] = new regr_intercept(); - } + public: + Add_regr_intercept_ToUDAFMap() + { + UDAFMap::getMap()["regr_intercept"] = new regr_intercept(); + } }; static Add_regr_intercept_ToUDAFMap addToMap; @@ -38,123 +38,120 @@ static Add_regr_intercept_ToUDAFMap addToMap; // Use the simple data model struct regr_intercept_data { - uint64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode regr_intercept::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_intercept::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_intercept() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_intercept() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_intercept_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_intercept() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_intercept() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_intercept_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_intercept::reset(mcsv1Context* context) { - struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_intercept::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; - data->sumy += valy; - data->sumx += valx; - data->sumx2 += valx*valx; + data->sumy += valy; + data->sumx += valx; + data->sumx2 += valx * valx; - data->sumxy += valx*valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + data->sumxy += valx * valy; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_intercept::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_intercept_data* outData = (struct regr_intercept_data*)context->getUserData()->data; - struct regr_intercept_data* inData = (struct regr_intercept_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumx2 += inData->sumx2; - outData->sumy += inData->sumy; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_intercept_data* outData = (struct regr_intercept_data*)context->getUserData()->data; + struct regr_intercept_data* inData = (struct regr_intercept_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumx2 += inData->sumx2; + outData->sumy += inData->sumy; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; - long double N = data->cnt; - if (N > 1) + struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; + long double N = data->cnt; + if (N > 1) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + // regr_intercept is AVG(y) - slope(y,x)*avg(x) + // We do some algebra and we can get a slightly smaller calculation + long double numerator = sumy * sumx2 - sumx * sumxy; + long double var_pop = + (N * sumx2) - (sumx * sumx); // Not realy var_pop, but sort of after some reductions + if (var_pop > 0) { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumxy = data->sumxy; - // regr_intercept is AVG(y) - slope(y,x)*avg(x) - // We do some algebra and we can get a slightly smaller calculation - long double numerator = sumy * sumx2 - sumx * sumxy; - long double var_pop = (N * sumx2) - (sumx * sumx); // Not realy var_pop, but sort of after some reductions - if (var_pop > 0) - { - valOut = static_cast(numerator / var_pop); - } + valOut = static_cast(numerator / var_pop); } - return mcsv1_UDAF::SUCCESS; + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_intercept::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; - data->sumy -= valy; - data->sumx -= valx; - data->sumx2 -= valx*valx; + data->sumy -= valy; + data->sumx -= valx; + data->sumx2 -= valx * valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_intercept.h b/utils/regr/regr_intercept.h index 2ccd272f5..7bd835819 100644 --- a/utils/regr/regr_intercept.h +++ b/utils/regr/regr_intercept.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_intercept.h -***********************************************************************/ + * $Id$ + * + * regr_intercept.h + ***********************************************************************/ /** * Columnstore interface for for the regr_intercept function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_intercept value of the dataset -class regr_intercept : public mcsv1_UDAF +class regr_intercept : public mcsv1_UDAF { -public: - // Defaults OK - regr_intercept() : mcsv1_UDAF() {}; - virtual ~regr_intercept() {}; + public: + // Defaults OK + regr_intercept() : mcsv1_UDAF(){}; + virtual ~regr_intercept(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_intercept.h - +#endif // HEADER_regr_intercept.h diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index 31b1ba2fb..99a204af8 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_r2_ToUDAFMap { -public: - Add_regr_r2_ToUDAFMap() - { - UDAFMap::getMap()["regr_r2"] = new regr_r2(); - } + public: + Add_regr_r2_ToUDAFMap() + { + UDAFMap::getMap()["regr_r2"] = new regr_r2(); + } }; static Add_regr_r2_ToUDAFMap addToMap; @@ -38,143 +38,139 @@ static Add_regr_r2_ToUDAFMap addToMap; // Use the simple data model struct regr_r2_data { - uint64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumy2; // sum of (y squared) - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode regr_r2::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_r2::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_r2() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_r2() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_r2_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_r2() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_r2() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_r2_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_r2::reset(mcsv1Context* context) { - struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumy2 = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumy2 = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_r2::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; - data->sumy += valy; - data->sumy2 += valy*valy; + data->sumy += valy; + data->sumy2 += valy * valy; - data->sumx += valx; - data->sumx2 += valx*valx; + data->sumx += valx; + data->sumx2 += valx * valx; - data->sumxy += valx*valy; + data->sumxy += valx * valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_r2::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_r2_data* outData = (struct regr_r2_data*)context->getUserData()->data; - struct regr_r2_data* inData = (struct regr_r2_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumx2 += inData->sumx2; - outData->sumy += inData->sumy; - outData->sumy2 += inData->sumy2; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_r2_data* outData = (struct regr_r2_data*)context->getUserData()->data; + struct regr_r2_data* inData = (struct regr_r2_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumx2 += inData->sumx2; + outData->sumy += inData->sumy; + outData->sumy2 += inData->sumy2; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; - double N = data->cnt; - if (N > 1) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumy2 = data->sumy2; - long double sumxy = data->sumxy; + struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; + double N = data->cnt; + if (N > 1) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; - long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx <= 0) // Catch -0 - { - // When var_popx is 0, NULL is the result. - return mcsv1_UDAF::SUCCESS; - } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy <= 0) // Catch -0 - { - // When var_popy is 0, 1 is the result - valOut = 1.0; - return mcsv1_UDAF::SUCCESS; - } - long double std_popx = sqrt(var_popx); - long double std_popy = sqrt(var_popy); - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - long double corr = covar_pop / (std_popy * std_popx); - valOut = static_cast(corr * corr); + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + if (var_popx <= 0) // Catch -0 + { + // When var_popx is 0, NULL is the result. + return mcsv1_UDAF::SUCCESS; } - return mcsv1_UDAF::SUCCESS; + double var_popy = (sumy2 - (sumy * sumy / N)) / N; + if (var_popy <= 0) // Catch -0 + { + // When var_popy is 0, 1 is the result + valOut = 1.0; + return mcsv1_UDAF::SUCCESS; + } + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr * corr); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_r2::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; - data->sumy -= valy; - data->sumy2 -= valy*valy; + data->sumy -= valy; + data->sumy2 -= valy * valy; - data->sumx -= valx; - data->sumx2 -= valx*valx; + data->sumx -= valx; + data->sumx2 -= valx * valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_r2.h b/utils/regr/regr_r2.h index 26082d979..077acaa68 100644 --- a/utils/regr/regr_r2.h +++ b/utils/regr/regr_r2.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_r2.h -***********************************************************************/ + * $Id$ + * + * regr_r2.h + ***********************************************************************/ /** * Columnstore interface for for the regr_r2 function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_r2 value of the dataset -class regr_r2 : public mcsv1_UDAF +class regr_r2 : public mcsv1_UDAF { -public: - // Defaults OK - regr_r2() : mcsv1_UDAF() {}; - virtual ~regr_r2() {}; + public: + // Defaults OK + regr_r2() : mcsv1_UDAF(){}; + virtual ~regr_r2(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_r2.h - +#endif // HEADER_regr_r2.h diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index df85e484d..de182e148 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_slope_ToUDAFMap { -public: - Add_regr_slope_ToUDAFMap() - { - UDAFMap::getMap()["regr_slope"] = new regr_slope(); - } + public: + Add_regr_slope_ToUDAFMap() + { + UDAFMap::getMap()["regr_slope"] = new regr_slope(); + } }; static Add_regr_slope_ToUDAFMap addToMap; @@ -38,122 +38,118 @@ static Add_regr_slope_ToUDAFMap addToMap; // Use the simple data model struct regr_slope_data { - uint64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode regr_slope::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_slope::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_slope() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_slope() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - context->setUserDataSize(sizeof(regr_slope_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; - + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_slope() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_slope() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_slope_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_slope::reset(mcsv1Context* context) { - struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_slope::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; - data->sumy += valy; - data->sumx += valx; - data->sumx2 += valx*valx; - data->sumxy += valx*valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + data->sumy += valy; + data->sumx += valx; + data->sumx2 += valx * valx; + data->sumxy += valx * valy; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_slope::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_slope_data* outData = (struct regr_slope_data*)context->getUserData()->data; - struct regr_slope_data* inData = (struct regr_slope_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumx2 += inData->sumx2; - outData->sumy += inData->sumy; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_slope_data* outData = (struct regr_slope_data*)context->getUserData()->data; + struct regr_slope_data* inData = (struct regr_slope_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumx2 += inData->sumx2; + outData->sumy += inData->sumy; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; - double N = data->cnt; - if (N > 1) + struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; + double N = data->cnt; + if (N > 1) + { + // COVAR_POP(y, x) / VAR_POP(x) + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + // These aren't really covar_pop and var_pop. For the purposes of this calculation + // we multiplied everything by N to reduce calc time and variance. + // It all comes out after the final divide + long double covar_pop = N * sumxy - sumx * sumy; + long double var_pop = N * sumx2 - sumx * sumx; + if (var_pop > 0) { - // COVAR_POP(y, x) / VAR_POP(x) - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumxy = data->sumxy; - // These aren't really covar_pop and var_pop. For the purposes of this calculation - // we multiplied everything by N to reduce calc time and variance. - // It all comes out after the final divide - long double covar_pop = N * sumxy - sumx * sumy; - long double var_pop = N * sumx2 - sumx * sumx; - if (var_pop > 0) - { - valOut = static_cast(covar_pop / var_pop); - } + valOut = static_cast(covar_pop / var_pop); } - return mcsv1_UDAF::SUCCESS; + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_slope::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; - data->sumy -= valy; - data->sumx -= valx; - data->sumx2 -= valx*valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumy -= valy; + data->sumx -= valx; + data->sumx2 -= valx * valx; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_slope.h b/utils/regr/regr_slope.h index d14639cf1..4297413aa 100644 --- a/utils/regr/regr_slope.h +++ b/utils/regr/regr_slope.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_slope.h -***********************************************************************/ + * $Id$ + * + * regr_slope.h + ***********************************************************************/ /** * Columnstore interface for for the regr_slope function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_slope value of the dataset -class regr_slope : public mcsv1_UDAF +class regr_slope : public mcsv1_UDAF { -public: - // Defaults OK - regr_slope() : mcsv1_UDAF() {}; - virtual ~regr_slope() {}; + public: + // Defaults OK + regr_slope() : mcsv1_UDAF(){}; + virtual ~regr_slope(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_slope.h - +#endif // HEADER_regr_slope.h diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index 6141c87d7..f62595a6b 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_sxx_ToUDAFMap { -public: - Add_regr_sxx_ToUDAFMap() - { - UDAFMap::getMap()["regr_sxx"] = new regr_sxx(); - } + public: + Add_regr_sxx_ToUDAFMap() + { + UDAFMap::getMap()["regr_sxx"] = new regr_sxx(); + } }; static Add_regr_sxx_ToUDAFMap addToMap; @@ -38,103 +38,99 @@ static Add_regr_sxx_ToUDAFMap addToMap; // Use the simple data model struct regr_sxx_data { - uint64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) + uint64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) }; - -mcsv1_UDAF::ReturnCode regr_sxx::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_sxx::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_sxx() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_sxx() with a non-numeric independant (second) argument"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_sxx_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxx() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxx() with a non-numeric independant (second) argument"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_sxx_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxx::reset(mcsv1Context* context) { - struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - return mcsv1_UDAF::SUCCESS; + struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxx::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valx = toDouble(valsIn[1]); - struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; + double valx = toDouble(valsIn[1]); + struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; - data->sumx += valx; - data->sumx2 += valx*valx; + data->sumx += valx; + data->sumx2 += valx * valx; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxx::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_sxx_data* outData = (struct regr_sxx_data*)context->getUserData()->data; - struct regr_sxx_data* inData = (struct regr_sxx_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumx2 += inData->sumx2; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_sxx_data* outData = (struct regr_sxx_data*)context->getUserData()->data; + struct regr_sxx_data* inData = (struct regr_sxx_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumx2 += inData->sumx2; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; - long double N = data->cnt; - if (N > 0) - { - long double regr_sxx = (data->sumx2 - (data->sumx * data->sumx / N)); - if (regr_sxx < 0) // catch -0 - regr_sxx = 0; - valOut = static_cast(regr_sxx); - } - return mcsv1_UDAF::SUCCESS; + struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; + long double N = data->cnt; + if (N > 0) + { + long double regr_sxx = (data->sumx2 - (data->sumx * data->sumx / N)); + if (regr_sxx < 0) // catch -0 + regr_sxx = 0; + valOut = static_cast(regr_sxx); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxx::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valx = toDouble(valsDropped[1]); - struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; + double valx = toDouble(valsDropped[1]); + struct regr_sxx_data* data = (struct regr_sxx_data*)context->getUserData()->data; - data->sumx -= valx; - data->sumx2 -= valx*valx; + data->sumx -= valx; + data->sumx2 -= valx * valx; - --data->cnt; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_sxx.h b/utils/regr/regr_sxx.h index 2f180483c..bd9acd309 100644 --- a/utils/regr/regr_sxx.h +++ b/utils/regr/regr_sxx.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_sxx.h -***********************************************************************/ + * $Id$ + * + * regr_sxx.h + ***********************************************************************/ /** * Columnstore interface for for the regr_sxx function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_sxx value of the dataset -class regr_sxx : public mcsv1_UDAF +class regr_sxx : public mcsv1_UDAF { -public: - // Defaults OK - regr_sxx() : mcsv1_UDAF() {}; - virtual ~regr_sxx() {}; + public: + // Defaults OK + regr_sxx() : mcsv1_UDAF(){}; + virtual ~regr_sxx(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_sxx.h - +#endif // HEADER_regr_sxx.h diff --git a/utils/regr/regr_sxy.cpp b/utils/regr/regr_sxy.cpp index b78e98f08..6101f71c1 100644 --- a/utils/regr/regr_sxy.cpp +++ b/utils/regr/regr_sxy.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_sxy_ToUDAFMap { -public: - Add_regr_sxy_ToUDAFMap() - { - UDAFMap::getMap()["regr_sxy"] = new regr_sxy(); - } + public: + Add_regr_sxy_ToUDAFMap() + { + UDAFMap::getMap()["regr_sxy"] = new regr_sxy(); + } }; static Add_regr_sxy_ToUDAFMap addToMap; @@ -38,111 +38,107 @@ static Add_regr_sxy_ToUDAFMap addToMap; // Use the simple data model struct regr_sxy_data { - uint64_t cnt; - long double sumx; - long double sumy; - long double sumxy; // sum of x * y + uint64_t cnt; + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; - -mcsv1_UDAF::ReturnCode regr_sxy::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_sxy::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_sxy() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_sxy() with non-numeric arguments"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_sxy_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxy() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxy() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_sxy_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxy::reset(mcsv1Context* context) { - struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - return mcsv1_UDAF::SUCCESS; + struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxy::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - double valx = toDouble(valsIn[1]); - struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + double valx = toDouble(valsIn[1]); + struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; - data->sumy += valy; + data->sumy += valy; - data->sumx += valx; + data->sumx += valx; - data->sumxy += valx*valy; + data->sumxy += valx * valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxy::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_sxy_data* outData = (struct regr_sxy_data*)context->getUserData()->data; - struct regr_sxy_data* inData = (struct regr_sxy_data*)userDataIn->data; - - outData->sumx += inData->sumx; - outData->sumy += inData->sumy; - outData->sumxy += inData->sumxy; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_sxy_data* outData = (struct regr_sxy_data*)context->getUserData()->data; + struct regr_sxy_data* inData = (struct regr_sxy_data*)userDataIn->data; + + outData->sumx += inData->sumx; + outData->sumy += inData->sumy; + outData->sumxy += inData->sumxy; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxy::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; - long double N = data->cnt; - if (N > 0) - { - long double regr_sxy = (data->sumxy - ((data->sumx * data->sumy) / N)); - valOut = static_cast(regr_sxy); - } - return mcsv1_UDAF::SUCCESS; + struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; + long double N = data->cnt; + if (N > 0) + { + long double regr_sxy = (data->sumxy - ((data->sumx * data->sumy) / N)); + valOut = static_cast(regr_sxy); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_sxy::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - double valx = toDouble(valsDropped[1]); - struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + double valx = toDouble(valsDropped[1]); + struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; - data->sumy -= valy; + data->sumy -= valy; - data->sumx -= valx; + data->sumx -= valx; - data->sumxy -= valx*valy; - --data->cnt; + data->sumxy -= valx * valy; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_sxy.h b/utils/regr/regr_sxy.h index 83517e113..565ce1ea6 100644 --- a/utils/regr/regr_sxy.h +++ b/utils/regr/regr_sxy.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_sxy.h -***********************************************************************/ + * $Id$ + * + * regr_sxy.h + ***********************************************************************/ /** * Columnstore interface for for the regr_sxy function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_sxy value of the dataset -class regr_sxy : public mcsv1_UDAF +class regr_sxy : public mcsv1_UDAF { -public: - // Defaults OK - regr_sxy() : mcsv1_UDAF() {}; - virtual ~regr_sxy() {}; + public: + // Defaults OK + regr_sxy() : mcsv1_UDAF(){}; + virtual ~regr_sxy(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_sxy.h - +#endif // HEADER_regr_sxy.h diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index ab94b866e..2b39fea5d 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -26,11 +26,11 @@ using namespace mcsv1sdk; class Add_regr_syy_ToUDAFMap { -public: - Add_regr_syy_ToUDAFMap() - { - UDAFMap::getMap()["regr_syy"] = new regr_syy(); - } + public: + Add_regr_syy_ToUDAFMap() + { + UDAFMap::getMap()["regr_syy"] = new regr_syy(); + } }; static Add_regr_syy_ToUDAFMap addToMap; @@ -38,103 +38,99 @@ static Add_regr_syy_ToUDAFMap addToMap; // Use the simple data model struct regr_syy_data { - uint64_t cnt; - long double sumy; - long double sumy2; // sum of (y squared) + uint64_t cnt; + long double sumy; + long double sumy2; // sum of (y squared) }; - -mcsv1_UDAF::ReturnCode regr_syy::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode regr_syy::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 2) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_syy() with other than 2 arguments"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_syy() with a non-numeric dependant (first) argument"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(regr_syy_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(DECIMAL_NOT_SPECIFIED); - context->setPrecision(0); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (context->getParameterCount() != 2) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_syy() with other than 2 arguments"); + return mcsv1_UDAF::ERROR; + } + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_syy() with a non-numeric dependant (first) argument"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(regr_syy_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_syy::reset(mcsv1Context* context) { - struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; - data->cnt = 0; - data->sumy = 0.0; - data->sumy2 = 0.0; - return mcsv1_UDAF::SUCCESS; + struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; + data->cnt = 0; + data->sumy = 0.0; + data->sumy2 = 0.0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_syy::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - double valy = toDouble(valsIn[0]); - struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; + double valy = toDouble(valsIn[0]); + struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; - data->sumy += valy; - data->sumy2 += valy*valy; + data->sumy += valy; + data->sumy2 += valy * valy; - ++data->cnt; - - return mcsv1_UDAF::SUCCESS; + ++data->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_syy::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct regr_syy_data* outData = (struct regr_syy_data*)context->getUserData()->data; - struct regr_syy_data* inData = (struct regr_syy_data*)userDataIn->data; - - outData->sumy += inData->sumy; - outData->sumy2 += inData->sumy2; - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct regr_syy_data* outData = (struct regr_syy_data*)context->getUserData()->data; + struct regr_syy_data* inData = (struct regr_syy_data*)userDataIn->data; + + outData->sumy += inData->sumy; + outData->sumy2 += inData->sumy2; + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; - long double N = data->cnt; - if (N > 0) - { - long double var_popy = (data->sumy2 - (data->sumy * data->sumy / N)); - if (var_popy < 0) // might be -0 - var_popy = 0; - valOut = static_cast(var_popy); - } - return mcsv1_UDAF::SUCCESS; + struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; + long double N = data->cnt; + if (N > 0) + { + long double var_popy = (data->sumy2 - (data->sumy * data->sumy / N)); + if (var_popy < 0) // might be -0 + var_popy = 0; + valOut = static_cast(var_popy); + } + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode regr_syy::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - double valy = toDouble(valsDropped[0]); - struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; + double valy = toDouble(valsDropped[0]); + struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; - data->sumy -= valy; - data->sumy2 -= valy*valy; + data->sumy -= valy; + data->sumy2 -= valy * valy; - --data->cnt; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/regr/regr_syy.h b/utils/regr/regr_syy.h index 05d7e5f8f..7729df52b 100644 --- a/utils/regr/regr_syy.h +++ b/utils/regr/regr_syy.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* regr_syy.h -***********************************************************************/ + * $Id$ + * + * regr_syy.h + ***********************************************************************/ /** * Columnstore interface for for the regr_syy function @@ -52,35 +52,32 @@ namespace mcsv1sdk { - // Return the regr_syy value of the dataset -class regr_syy : public mcsv1_UDAF +class regr_syy : public mcsv1_UDAF { -public: - // Defaults OK - regr_syy() : mcsv1_UDAF() {}; - virtual ~regr_syy() {}; + public: + // Defaults OK + regr_syy() : mcsv1_UDAF(){}; + virtual ~regr_syy(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_regr_syy.h - +#endif // HEADER_regr_syy.h diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 1cd2ae6e0..1b480aa84 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -11,47 +11,40 @@ namespace { inline bool isNumeric(int type, const char* attr) { - if (type == INT_RESULT || type == REAL_RESULT || type == DECIMAL_RESULT) - { - return true; - } + if (type == INT_RESULT || type == REAL_RESULT || type == DECIMAL_RESULT) + { + return true; + } #if _MSC_VER - if (_strnicmp("NULL", attr, 4) == 0)) + if (_strnicmp("NULL", attr, 4) == 0)) #else - if (strncasecmp("NULL", attr, 4) == 0) + if (strncasecmp("NULL", attr, 4) == 0) #endif { - return true; + return true; } - return false; + return false; } inline double cvtArgToDouble(int t, const char* v) { - double d = 0.0; + double d = 0.0; - switch (t) - { - case INT_RESULT: - d = (double)(*((long long*)v)); - break; + switch (t) + { + case INT_RESULT: d = (double)(*((long long*)v)); break; - case REAL_RESULT: - d = *((double*)v); - break; + case REAL_RESULT: d = *((double*)v); break; - case DECIMAL_RESULT: - case STRING_RESULT: - d = strtod(v, 0); - break; + case DECIMAL_RESULT: + case STRING_RESULT: d = strtod(v, 0); break; - case ROW_RESULT: - break; - } + case ROW_RESULT: break; + } - return d; -} + return d; } +} // namespace /**************************************************************************** * UDF function interface for MariaDB connector to recognize is defined in @@ -77,1372 +70,1339 @@ inline double cvtArgToDouble(int t, const char* v) */ extern "C" { + //======================================================================= -//======================================================================= + /** + * regr_avgx + */ + struct regr_avgx_data + { + long double sumx; + int64_t cnt; + }; - /** - * regr_avgx - */ - struct regr_avgx_data +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_avgx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_avgx_data* data; + if (args->arg_count != 2) { - long double sumx; - int64_t cnt; - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_avgx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + strcpy(message, "regr_avgx() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[1], args->attributes[1]))) { - struct regr_avgx_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_avgx() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); - return 1; - } - if (initid->decimals != DECIMAL_NOT_SPECIFIED) - { - initid->decimals += 4; - } - - if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumx = 0; - data->cnt = 0; - - initid->ptr = (char*)data; - return 0; + strcpy(message, "regr_avgx() with a non-numeric independant (second) argument"); + return 1; + } + if (initid->decimals != DECIMAL_NOT_SPECIFIED) + { + initid->decimals += 4; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_avgx_deinit(UDF_INIT* initid) + if (!(data = (struct regr_avgx_data*)malloc(sizeof(struct regr_avgx_data)))) { - free(initid->ptr); + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->sumx = 0; + data->cnt = 0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_avgx_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_avgx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; + data->sumx = 0; + data->cnt = 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_avgx_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + ++data->cnt; + data->sumx += xval; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_avgx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; + double valOut = 0; + if (data->cnt > 0) + { + valOut = static_cast(data->sumx / data->cnt); + } + else + { + *is_null = 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_avgx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + return valOut; + } + + //======================================================================= + + /** + * regr_avgy + */ + struct regr_avgy_data + { + long double sumy; + int64_t cnt; + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_avgy_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_avgy_data* data; + if (args->arg_count != 2) { - struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - data->sumx = 0; - data->cnt = 0; + strcpy(message, "regr_avgy() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]))) + { + strcpy(message, "regr_avgy() with a non-numeric dependant (first) argument"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_avgx_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) + if (initid->decimals != DECIMAL_NOT_SPECIFIED) { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - ++data->cnt; - data->sumx += xval; + initid->decimals += 4; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_avgx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) + if (!(data = (struct regr_avgy_data*)malloc(sizeof(struct regr_avgy_data)))) { - struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - double valOut = 0; - if (data->cnt > 0) - { - valOut = static_cast(data->sumx / data->cnt); - } - else - { - *is_null = 1; - } + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->sumy = 0; + data->cnt = 0; - return valOut; + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_avgy_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_avgy_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; + data->sumy = 0; + data->cnt = 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_avgy_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + ++data->cnt; + data->sumy += yval; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_avgy(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; + double valOut = 0; + if (data->cnt > 0) + { + valOut = static_cast(data->sumy / data->cnt); + } + else + { + *is_null = 1; + } + return valOut; + } + + //======================================================================= + + /** + * regr_count + */ + struct regr_count_data + { + int64_t cnt; + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_count_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_count_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_count() requires two arguments"); + return 1; } -//======================================================================= - - /** - * regr_avgy - */ - struct regr_avgy_data + if (!(data = (struct regr_count_data*)malloc(sizeof(struct regr_count_data)))) { - long double sumy; - int64_t cnt; - }; + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_avgy_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_count_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_count_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_count_data* data = (struct regr_count_data*)initid->ptr; + data->cnt = 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_count_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) { - struct regr_avgy_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_avgy() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]))) - { - strcpy(message,"regr_avgy() with a non-numeric dependant (first) argument"); - return 1; - } + return; + } + struct regr_count_data* data = (struct regr_count_data*)initid->ptr; + ++data->cnt; + } - if (initid->decimals != DECIMAL_NOT_SPECIFIED) - { - initid->decimals += 4; - } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long regr_count(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_count_data* data = (struct regr_count_data*)initid->ptr; + return data->cnt; + } - if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumy = 0; - data->cnt = 0; + //======================================================================= - initid->ptr = (char*)data; - return 0; + /** + * regr_slope + */ + struct regr_slope_data + { + int64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_slope_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_slope_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_slope() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "regr_slope() with non-numeric arguments"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_avgy_deinit(UDF_INIT* initid) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_slope_data*)malloc(sizeof(struct regr_slope_data)))) { - free(initid->ptr); + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_slope_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_slope_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_slope_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumx2 += xval * xval; + data->sumxy += xval * yval; + ++data->cnt; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_slope(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + *is_null = 1; + if (N > 0) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double covar_pop = N * sumxy - sumx * sumy; + long double var_pop = N * sumx2 - sumx * sumx; + if (var_pop > 0) + { + valOut = static_cast(covar_pop / var_pop); + *is_null = 0; + } + } + return valOut; + } + + //======================================================================= + + /** + * regr_intercept + */ + struct regr_intercept_data + { + int64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_intercept_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_intercept_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_intercept() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "regr_intercept() with non-numeric arguments"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_avgy_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + if (!(data = (struct regr_intercept_data*)malloc(sizeof(struct regr_intercept_data)))) { - struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; - data->sumy = 0; - data->cnt = 0; + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_intercept_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_intercept_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_intercept_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumx2 += xval * xval; + data->sumxy += xval * yval; + ++data->cnt; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_intercept(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + *is_null = 1; + if (N > 0) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double numerator = sumy * sumx2 - sumx * sumxy; + long double var_pop = (N * sumx2) - (sumx * sumx); + if (var_pop > 0) + { + valOut = static_cast(numerator / var_pop); + *is_null = 0; + } + } + return valOut; + } + + //======================================================================= + + /** + * regr_r2 + */ + struct regr_r2_data + { + int64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_r2_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_r2_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_r2() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "regr_r2() with non-numeric arguments"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_avgy_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_r2_data*)malloc(sizeof(struct regr_r2_data)))) { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - ++data->cnt; - data->sumy += yval; + strmov(message, "Couldn't allocate memory"); + return 1; } + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumy2 = 0.0; + data->sumxy = 0.0; - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_avgy(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_r2_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_r2_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumy2 = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_r2_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) { - struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; - double valOut = 0; - if (data->cnt > 0) - { - valOut = static_cast(data->sumy / data->cnt); - } - else - { - *is_null = 1; - } - return valOut; + return; } + struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumx2 += xval * xval; + data->sumy2 += yval * yval; + data->sumxy += xval * yval; + ++data->cnt; + } -//======================================================================= - - /** - * regr_count - */ - struct regr_count_data +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_r2(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) { - int64_t cnt; - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_count_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct regr_count_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_count() requires two arguments"); - return 1; - } - - if (!(data = (struct regr_count_data*) malloc(sizeof(struct regr_count_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_count_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_count_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct regr_count_data* data = (struct regr_count_data*)initid->ptr; - data->cnt = 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_count_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_count_data* data = (struct regr_count_data*)initid->ptr; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - long long regr_count(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct regr_count_data* data = (struct regr_count_data*)initid->ptr; - return data->cnt; - } - -//======================================================================= - - /** - * regr_slope - */ - struct regr_slope_data - { - int64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumxy; // sum of (x*y) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_slope_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct regr_slope_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_slope() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"regr_slope() with non-numeric arguments"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct regr_slope_data*) malloc(sizeof(struct regr_slope_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_slope_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_slope_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_slope_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumx2 += xval*xval; - data->sumxy += xval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_slope(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + if (var_popx <= 0) // Catch -0 + { + // When var_popx is 0, NULL is the result. *is_null = 1; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumxy = data->sumxy; - long double covar_pop = N * sumxy - sumx * sumy; - long double var_pop = N * sumx2 - sumx * sumx; - if (var_pop > 0) - { - valOut = static_cast(covar_pop / var_pop); - *is_null = 0; - } - } - return valOut; + return 0; + } + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + if (var_popy <= 0) // Catch -0 + { + // When var_popy is 0, 1 is the result + return 1; + } + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr * corr); + } + else + { + *is_null = 1; + } + return valOut; + } + + //======================================================================= + + /** + * corr + */ + struct corr_data + { + int64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool corr_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct corr_data* data; + if (args->arg_count != 2) + { + strcpy(message, "corr() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "corr() with non-numeric arguments"); + return 1; } -//======================================================================= + initid->decimals = DECIMAL_NOT_SPECIFIED; - /** - * regr_intercept - */ - struct regr_intercept_data + if (!(data = (struct corr_data*)malloc(sizeof(struct corr_data)))) { - int64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumxy; // sum of (x*y) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_intercept_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct regr_intercept_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_intercept() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"regr_intercept() with non-numeric arguments"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - - initid->ptr = (char*)data; - return 0; + strmov(message, "Couldn't allocate memory"); + return 1; } + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumy2 = 0.0; + data->sumxy = 0.0; - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_intercept_deinit(UDF_INIT* initid) + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void corr_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void corr_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct corr_data* data = (struct corr_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + data->sumy = 0.0; + data->sumy2 = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void corr_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) { - free(initid->ptr); + return; } + struct corr_data* data = (struct corr_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumx2 += xval * xval; + data->sumy2 += yval * yval; + data->sumxy += xval * yval; + ++data->cnt; + } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_intercept_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double corr(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct corr_data* data = (struct corr_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) { - struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_intercept_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumx2 += xval*xval; - data->sumxy += xval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_intercept(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + if (var_popx <= 0) // Catch -0 + { + // When var_popx is 0, NULL is the result. *is_null = 1; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumxy = data->sumxy; - long double numerator = sumy * sumx2 - sumx * sumxy; - long double var_pop = (N * sumx2) - (sumx * sumx); - if (var_pop > 0) - { - valOut = static_cast(numerator / var_pop); - *is_null = 0; - } - } - return valOut; + return 0; + } + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + if (var_popy <= 0) // Catch -0 + { + // When var_popy is 0, 1 is the result + return 1; + } + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + return static_cast(corr); + } + else + { + *is_null = 1; + } + return valOut; + } + + //======================================================================= + + /** + * regr_sxx + */ + struct regr_sxx_data + { + int64_t cnt; + long double sumx; + long double sumx2; // sum of (x squared) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_sxx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_sxx_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_sxx() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "regr_avgx() with a non-numeric independant (second) argument"); + return 1; } -//======================================================================= + initid->decimals = DECIMAL_NOT_SPECIFIED; - /** - * regr_r2 - */ - struct regr_r2_data + if (!(data = (struct regr_sxx_data*)malloc(sizeof(struct regr_sxx_data)))) { - int64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumy2; // sum of (y squared) - long double sumxy; // sum of (x*y) - }; + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_r2_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_sxx_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_sxx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumx2 = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_sxx_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) { - struct regr_r2_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_r2() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"regr_r2() with non-numeric arguments"); - return 1; - } + return; + } + struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumx += xval; + data->sumx2 += xval * xval; + ++data->cnt; + } - initid->decimals = DECIMAL_NOT_SPECIFIED; +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_sxx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) + { + long double sumx = data->sumx; + long double sumx2 = data->sumx2; + long double sxx = (sumx2 - (sumx * sumx / N)); + if (sxx < 0) // catch -0 + sxx = 0; + valOut = static_cast(sxx); + } + else + { + *is_null = 1; + } + return valOut; + } + //======================================================================= - if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumy2 = 0.0; - data->sumxy = 0.0; + /** + * regr_syy + */ + struct regr_syy_data + { + int64_t cnt; + long double sumy; + long double sumy2; // sum of (y squared) + }; - initid->ptr = (char*)data; - return 0; +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_syy_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_syy_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_syy() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]))) + { + strcpy(message, "regr_syy() with a non-numeric dependant (first) argument"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_r2_deinit(UDF_INIT* initid) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_syy_data*)malloc(sizeof(struct regr_syy_data)))) { - free(initid->ptr); + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; + data->sumy = 0.0; + data->sumy2 = 0.0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_syy_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_syy_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; + data->cnt = 0; + data->sumy = 0.0; + data->sumy2 = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_syy_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + data->sumy += yval; + data->sumy2 += yval * yval; + ++data->cnt; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_syy(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) + { + long double sumy = data->sumy; + long double sumy2 = data->sumy2; + long double syy = (sumy2 - (sumy * sumy / N)); + if (syy < 0) // might be -0 + syy = 0; + valOut = static_cast(syy); + } + else + { + *is_null = 1; + } + return valOut; + } + + //======================================================================= + + /** + * regr_sxy + */ + struct regr_sxy_data + { + int64_t cnt; + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool regr_sxy_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct regr_sxy_data* data; + if (args->arg_count != 2) + { + strcpy(message, "regr_sxy() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "regr_sxy() with non-numeric arguments"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_r2_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_sxy_data*)malloc(sizeof(struct regr_sxy_data)))) { - struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumy2 = 0.0; - data->sumxy = 0.0; + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_sxy_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_sxy_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void regr_sxy_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumxy += xval * yval; + ++data->cnt; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double regr_sxy(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double regr_sxy = (sumxy - ((sumx * sumy) / N)); + valOut = static_cast(regr_sxy); + } + else + { + *is_null = 1; + } + return valOut; + } + + //======================================================================= + + /** + * covar_pop + */ + struct covar_pop_data + { + int64_t cnt; + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool covar_pop_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct covar_pop_data* data; + if (args->arg_count != 2) + { + strcpy(message, "covar_pop() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "covar_pop() with non-numeric arguments"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_r2_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct covar_pop_data*)malloc(sizeof(struct covar_pop_data)))) { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumx2 += xval*xval; - data->sumy2 += yval*yval; - data->sumxy += xval*yval; - ++data->cnt; + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void covar_pop_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void covar_pop_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void covar_pop_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) + { + return; + } + struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumxy += xval * yval; + ++data->cnt; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double covar_pop(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) + { + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + valOut = static_cast(covar_pop); + } + else + { + *is_null = 1; + } + return valOut; + } + //======================================================================= + + /** + * covar_samp + */ + struct covar_samp_data + { + int64_t cnt; + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool covar_samp_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct covar_samp_data* data; + if (args->arg_count != 2) + { + strcpy(message, "covar_samp() requires two arguments"); + return 1; + } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && + isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message, "covar_samp() with non-numeric arguments"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_r2(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct covar_samp_data*)malloc(sizeof(struct covar_samp_data)))) { - struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumy2 = data->sumy2; - long double sumxy = data->sumxy; - long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx <= 0) // Catch -0 - { - // When var_popx is 0, NULL is the result. - *is_null = 1; - return 0; - } - long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy <= 0) // Catch -0 - { - // When var_popy is 0, 1 is the result - return 1; - } - long double std_popx = sqrt(var_popx); - long double std_popy = sqrt(var_popy); - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - long double corr = covar_pop / (std_popy * std_popx); - valOut = static_cast(corr * corr); - } - else - { - *is_null = 1; - } - return valOut; + strmov(message, "Couldn't allocate memory"); + return 1; } + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; -//======================================================================= + initid->ptr = (char*)data; + return 0; + } - /** - * corr - */ - struct corr_data +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void covar_samp_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void covar_samp_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; + data->cnt = 0; + data->sumx = 0.0; + data->sumy = 0.0; + data->sumxy = 0.0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void covar_samp_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // Test for NULL in x and y + if (args->args[0] == 0 || args->args[1] == 0) { - int64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - long double sumy; - long double sumy2; // sum of (y squared) - long double sumxy; // sum of (x*y) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool corr_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct corr_data* data; - if (args->arg_count != 2) - { - strcpy(message,"corr() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"corr() with non-numeric arguments"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumy2 = 0.0; - data->sumxy = 0.0; - - initid->ptr = (char*)data; - return 0; + return; } + struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; + double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); + double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); + data->sumy += yval; + data->sumx += xval; + data->sumxy += xval * yval; + ++data->cnt; + } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void corr_deinit(UDF_INIT* initid) +#ifdef _MSC_VER + __declspec(dllexport) +#endif + double covar_samp(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; + double N = data->cnt; + double valOut = 0; + if (N > 0) { - free(initid->ptr); + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); + valOut = static_cast(covar_samp); } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - corr_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + else { - struct corr_data* data = (struct corr_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - data->sumy = 0.0; - data->sumy2 = 0.0; - data->sumxy = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - corr_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct corr_data* data = (struct corr_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumx2 += xval*xval; - data->sumy2 += yval*yval; - data->sumxy += xval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double corr(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct corr_data* data = (struct corr_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumx2 = data->sumx2; - long double sumy2 = data->sumy2; - long double sumxy = data->sumxy; - long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx <= 0) // Catch -0 - { - // When var_popx is 0, NULL is the result. - *is_null = 1; - return 0; - } - long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy <= 0) // Catch -0 - { - // When var_popy is 0, 1 is the result - return 1; - } - long double std_popx = sqrt(var_popx); - long double std_popy = sqrt(var_popy); - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - long double corr = covar_pop / (std_popy * std_popx); - return static_cast(corr); - } - else - { - *is_null = 1; - } - return valOut; - } - -//======================================================================= - - /** - * regr_sxx - */ - struct regr_sxx_data - { - int64_t cnt; - long double sumx; - long double sumx2; // sum of (x squared) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_sxx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct regr_sxx_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_sxx() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_sxx_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_sxx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumx2 = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_sxx_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumx += xval; - data->sumx2 += xval*xval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_sxx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumx = data->sumx; - long double sumx2 = data->sumx2; - long double sxx = (sumx2 - (sumx * sumx / N)); - if (sxx < 0) // catch -0 - sxx = 0; - valOut = static_cast(sxx); - } - else - { - *is_null = 1; - } - return valOut; - } -//======================================================================= - - /** - * regr_syy - */ - struct regr_syy_data - { - int64_t cnt; - long double sumy; - long double sumy2; // sum of (y squared) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_syy_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct regr_syy_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_syy() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]))) - { - strcpy(message,"regr_syy() with a non-numeric dependant (first) argument"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumy = 0.0; - data->sumy2 = 0.0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_syy_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_syy_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; - data->cnt = 0; - data->sumy = 0.0; - data->sumy2 = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_syy_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - data->sumy += yval; - data->sumy2 += yval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_syy(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumy = data->sumy; - long double sumy2 = data->sumy2; - long double syy = (sumy2 - (sumy * sumy / N)); - if (syy < 0) // might be -0 - syy = 0; - valOut = static_cast(syy); - } - else - { - *is_null = 1; - } - return valOut; - } - -//======================================================================= - - /** - * regr_sxy - */ - struct regr_sxy_data - { - int64_t cnt; - long double sumx; - long double sumy; - long double sumxy; // sum of (x*y) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool regr_sxy_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct regr_sxy_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_sxy() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"regr_sxy() with non-numeric arguments"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void regr_sxy_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_sxy_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - regr_sxy_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumxy += xval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double regr_sxy(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumxy = data->sumxy; - long double regr_sxy = (sumxy - ((sumx * sumy) / N)); - valOut = static_cast(regr_sxy); - } - else - { - *is_null = 1; - } - return valOut; - } - -//======================================================================= - - /** - * covar_pop - */ - struct covar_pop_data - { - int64_t cnt; - long double sumx; - long double sumy; - long double sumxy; // sum of (x*y) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool covar_pop_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct covar_pop_data* data; - if (args->arg_count != 2) - { - strcpy(message,"covar_pop() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"covar_pop() with non-numeric arguments"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void covar_pop_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - covar_pop_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - covar_pop_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumxy += xval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double covar_pop(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumxy = data->sumxy; - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - valOut = static_cast(covar_pop); - } - else - { - *is_null = 1; - } - return valOut; - } -//======================================================================= - - /** - * covar_samp - */ - struct covar_samp_data - { - int64_t cnt; - long double sumx; - long double sumy; - long double sumxy; // sum of (x*y) - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool covar_samp_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct covar_samp_data* data; - if (args->arg_count != 2) - { - strcpy(message,"covar_samp() requires two arguments"); - return 1; - } - if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) - { - strcpy(message,"covar_samp() with non-numeric arguments"); - return 1; - } - - initid->decimals = DECIMAL_NOT_SPECIFIED; - - if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void covar_samp_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - covar_samp_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; - data->cnt = 0; - data->sumx = 0.0; - data->sumy = 0.0; - data->sumxy = 0.0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - covar_samp_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // Test for NULL in x and y - if (args->args[0] == 0 || args->args[1] == 0) - { - return; - } - struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; - double yval = cvtArgToDouble(args->arg_type[0], args->args[0]); - double xval = cvtArgToDouble(args->arg_type[1], args->args[1]); - data->sumy += yval; - data->sumx += xval; - data->sumxy += xval*yval; - ++data->cnt; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - double covar_samp(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; - double N = data->cnt; - double valOut = 0; - if (N > 0) - { - long double sumx = data->sumx; - long double sumy = data->sumy; - long double sumxy = data->sumxy; - long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); - valOut = static_cast(covar_samp); - } - else - { - *is_null = 1; - } - return valOut; + *is_null = 1; } + return valOut; + } } // vim:ts=4 sw=4: - diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp old mode 100755 new mode 100644 index 1ed114c69..0e4e79622 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -51,7 +51,6 @@ #include "calpontsystemcatalog.h" #include "vlarray.h" - #include "threadnaming.h" #include "rowstorage.h" @@ -66,166 +65,165 @@ using namespace dataconvert; // inlines of RowAggregation that used only in this file namespace { - template inline bool minMax(T d1, T d2, int type) { - if (type == rowgroup::ROWAGG_MIN) return d1 < d2; - else return d1 > d2; + if (type == rowgroup::ROWAGG_MIN) + return d1 < d2; + else + return d1 > d2; } inline bool minMax(int128_t* d1, int128_t* d2, int type) { - return (type == rowgroup::ROWAGG_MIN) ? *d1 < *d2 : *d1 > *d2; + return (type == rowgroup::ROWAGG_MIN) ? *d1 < *d2 : *d1 > *d2; } inline int64_t getIntNullValue(int colType) { - switch (colType) - { - case execplan::CalpontSystemCatalog::TINYINT: - return joblist::TINYINTNULL; + switch (colType) + { + case execplan::CalpontSystemCatalog::TINYINT: return joblist::TINYINTNULL; - case execplan::CalpontSystemCatalog::SMALLINT: - return joblist::SMALLINTNULL; + case execplan::CalpontSystemCatalog::SMALLINT: return joblist::SMALLINTNULL; - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - return joblist::INTNULL; + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: return joblist::INTNULL; - case execplan::CalpontSystemCatalog::BIGINT: - default: - return joblist::BIGINTNULL; - } + case execplan::CalpontSystemCatalog::BIGINT: + default: return joblist::BIGINTNULL; + } } - inline uint64_t getUintNullValue(int colType, int colWidth = 0) { - switch (colType) + switch (colType) + { + case execplan::CalpontSystemCatalog::CHAR: { - case execplan::CalpontSystemCatalog::CHAR: - { - if (colWidth == 1) return joblist::CHAR1NULL; - else if (colWidth == 2) return joblist::CHAR2NULL; - else if (colWidth < 5) return joblist::CHAR4NULL; + if (colWidth == 1) + return joblist::CHAR1NULL; + else if (colWidth == 2) + return joblist::CHAR2NULL; + else if (colWidth < 5) + return joblist::CHAR4NULL; - break; + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + if (colWidth < 3) + return joblist::CHAR2NULL; + else if (colWidth < 5) + return joblist::CHAR4NULL; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return joblist::DATENULL; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return joblist::DATETIMENULL; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + return joblist::TIMESTAMPNULL; + } + + case execplan::CalpontSystemCatalog::TIME: + { + return joblist::TIMENULL; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + switch (colWidth) + { + case 1: + { + return joblist::TINYINTNULL; } - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: + case 2: { - if (colWidth < 3) return joblist::CHAR2NULL; - else if (colWidth < 5) return joblist::CHAR4NULL; - - break; + return joblist::SMALLINTNULL; } - case execplan::CalpontSystemCatalog::DATE: + case 4: { - return joblist::DATENULL; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - return joblist::DATETIMENULL; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - return joblist::TIMESTAMPNULL; - } - - case execplan::CalpontSystemCatalog::TIME: - { - return joblist::TIMENULL; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - switch (colWidth) - { - case 1: - { - return joblist::TINYINTNULL; - } - - case 2: - { - return joblist::SMALLINTNULL; - } - - case 4: - { - return joblist::INTNULL; - } - - default: - { - return joblist::BIGINTNULL; - } - } - } - - case execplan::CalpontSystemCatalog::UTINYINT: - { - return joblist::UTINYINTNULL; - } - - case execplan::CalpontSystemCatalog::USMALLINT: - { - return joblist::USMALLINTNULL; - } - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - return joblist::UINTNULL; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - { - return joblist::UBIGINTNULL; + return joblist::INTNULL; } default: { - break; + return joblist::BIGINTNULL; } + } } - return joblist::CHAR8NULL; -} + case execplan::CalpontSystemCatalog::UTINYINT: + { + return joblist::UTINYINTNULL; + } + case execplan::CalpontSystemCatalog::USMALLINT: + { + return joblist::USMALLINTNULL; + } + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + { + return joblist::UINTNULL; + } + + case execplan::CalpontSystemCatalog::UBIGINT: + { + return joblist::UBIGINTNULL; + } + + default: + { + break; + } + } + + return joblist::CHAR8NULL; +} inline double getDoubleNullValue() { - uint64_t x = joblist::DOUBLENULL; - auto* y = (double*)&x; - return *y; + uint64_t x = joblist::DOUBLENULL; + auto* y = (double*)&x; + return *y; } - inline float getFloatNullValue() { - uint32_t x = joblist::FLOATNULL; - auto* y = (float*)&x; - return *y; + uint32_t x = joblist::FLOATNULL; + auto* y = (float*)&x; + return *y; } inline long double getLongDoubleNullValue() { - return joblist::LONGDOUBLENULL; + return joblist::LONGDOUBLENULL; } inline string getStringNullValue() { - return joblist::CPNULLSTRMARK; + return joblist::CPNULLSTRMARK; } -} +} // namespace namespace rowgroup { @@ -251,82 +249,75 @@ static const string overflowMsg("Aggregation overflow."); inline void RowAggregation::updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func) { - int32_t colOutOffset = fRow.getOffset(col); - if (isNull(fRowGroupOut, fRow, col)) - fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); - else if (minMax(val1, val2, func)) - fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); + int32_t colOutOffset = fRow.getOffset(col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); + else if (minMax(val1, val2, func)) + fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); } - inline void RowAggregation::updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - fRow.setIntField(val1, col); - else if (minMax(val1, val2, func)) - fRow.setIntField(val1, col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setIntField(val1, col); + else if (minMax(val1, val2, func)) + fRow.setIntField(val1, col); } - inline void RowAggregation::updateUintMinMax(uint64_t val1, uint64_t val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - fRow.setUintField(val1, col); - else if (minMax(val1, val2, func)) - fRow.setUintField(val1, col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setUintField(val1, col); + else if (minMax(val1, val2, func)) + fRow.setUintField(val1, col); } - inline void RowAggregation::updateCharMinMax(uint64_t val1, uint64_t val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - fRow.setUintField(val1, col); - else if (minMax(uint64ToStr(val1), uint64ToStr(val2), func)) - fRow.setUintField(val1, col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setUintField(val1, col); + else if (minMax(uint64ToStr(val1), uint64ToStr(val2), func)) + fRow.setUintField(val1, col); } - inline void RowAggregation::updateDoubleMinMax(double val1, double val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - fRow.setDoubleField(val1, col); - else if (minMax(val1, val2, func)) - fRow.setDoubleField(val1, col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setDoubleField(val1, col); + else if (minMax(val1, val2, func)) + fRow.setDoubleField(val1, col); } - inline void RowAggregation::updateLongDoubleMinMax(long double val1, long double val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - fRow.setLongDoubleField(val1, col); - else if (minMax(val1, val2, func)) - fRow.setLongDoubleField(val1, col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setLongDoubleField(val1, col); + else if (minMax(val1, val2, func)) + fRow.setLongDoubleField(val1, col); } - inline void RowAggregation::updateFloatMinMax(float val1, float val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - fRow.setFloatField(val1, col); - else if (minMax(val1, val2, func)) - fRow.setFloatField(val1, col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setFloatField(val1, col); + else if (minMax(val1, val2, func)) + fRow.setFloatField(val1, col); } void RowAggregation::updateStringMinMax(string val1, string val2, int64_t col, int func) { - if (isNull(fRowGroupOut, fRow, col)) - { - fRow.setStringField(val1, col); - return; - } - CHARSET_INFO* cs = fRow.getCharset(col); - int tmp = cs->strnncoll(val1.c_str(), val1.length(), val2.c_str(), val2.length()); + if (isNull(fRowGroupOut, fRow, col)) + { + fRow.setStringField(val1, col); + return; + } + CHARSET_INFO* cs = fRow.getCharset(col); + int tmp = cs->strnncoll(val1.c_str(), val1.length(), val2.c_str(), val2.length()); - if ((tmp < 0 && func == rowgroup::ROWAGG_MIN) || - (tmp > 0 && func == rowgroup::ROWAGG_MAX)) - { - fRow.setStringField(val1, col); - } + if ((tmp < 0 && func == rowgroup::ROWAGG_MIN) || (tmp > 0 && func == rowgroup::ROWAGG_MAX)) + { + fRow.setStringField(val1, col); + } } //------------------------------------------------------------------------------ @@ -337,215 +328,209 @@ void RowAggregation::updateStringMinMax(string val1, string val2, int64_t col, i //------------------------------------------------------------------------------ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, int64_t col) { - /* TODO: Can we replace all of this with a call to row.isNullValue(col)? */ - bool ret = false; + /* TODO: Can we replace all of this with a call to row.isNullValue(col)? */ + bool ret = false; - int colDataType = (pRowGroup->getColTypes())[col]; + int colDataType = (pRowGroup->getColTypes())[col]; - switch (colDataType) + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: { - case execplan::CalpontSystemCatalog::TINYINT: - { - ret = ((uint8_t)row.getIntField(col) == joblist::TINYINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - { - ret = ((uint8_t)row.getIntField(col) == joblist::UTINYINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - int colWidth = pRowGroup->getColumnWidth(col); - - // bug 1853, use token to check null - // scale here is used to indicate token, not real string. - if ((pRowGroup->getScale())[col] > 0) - { - if (row.getIntField(col) & joblist::BIGINTNULL) - ret = true; - - // break the case block - break; - } - - // real string to check null - if (colWidth <= 8) - { - if (colWidth == 1) - ret = ((uint8_t)row.getUintField(col) == joblist::CHAR1NULL); - else if (colWidth == 2) - ret = ((uint16_t)row.getUintField(col) == joblist::CHAR2NULL); - else if (colWidth < 5) - ret = ((uint32_t)row.getUintField(col) == joblist::CHAR4NULL); - else - ret = ((uint64_t)row.getUintField(col) == joblist::CHAR8NULL); - } - else - { - //@bug 1821 - auto const str = row.getConstString(col); - ret = str.length() == 0 || str.eq(utils::ConstString(joblist::CPNULLSTRMARK)); - } - - break; - } - - case execplan::CalpontSystemCatalog::SMALLINT: - { - ret = ((uint16_t)row.getIntField(col) == joblist::SMALLINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::USMALLINT: - { - ret = ((uint16_t)row.getIntField(col) == joblist::USMALLINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - ret = ((uint64_t)row.getUintField(col) == joblist::DOUBLENULL); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - ret = (row.getLongDoubleField(col) == joblist::LONGDOUBLENULL); - break; - } - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - { - ret = ((uint32_t)row.getIntField(col) == joblist::INTNULL); - break; - } - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - ret = ((uint32_t)row.getIntField(col) == joblist::UINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - ret = ((uint32_t)row.getUintField(col) == joblist::FLOATNULL); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - ret = ((uint32_t)row.getUintField(col) == joblist::DATENULL); - break; - } - - case execplan::CalpontSystemCatalog::BIGINT: - { - ret = ((uint64_t)row.getIntField(col) == joblist::BIGINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - { - ret = ((uint64_t)row.getIntField(col) == joblist::UBIGINTNULL); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - ret = row.isNullValue(col); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - ret = ((uint64_t)row.getUintField(col) == joblist::DATETIMENULL); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - ret = ((uint64_t)row.getUintField(col) == joblist::TIMESTAMPNULL); - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); - break; - } - - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::BLOB: - { - auto const str = row.getConstString(col); - ret = str.length() == 0 || str.eq(utils::ConstString(joblist::CPNULLSTRMARK)); - break; - } - - default: - break; + ret = ((uint8_t)row.getIntField(col) == joblist::TINYINTNULL); + break; } - return ret; -} + case execplan::CalpontSystemCatalog::UTINYINT: + { + ret = ((uint8_t)row.getIntField(col) == joblist::UTINYINTNULL); + break; + } + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + int colWidth = pRowGroup->getColumnWidth(col); + + // bug 1853, use token to check null + // scale here is used to indicate token, not real string. + if ((pRowGroup->getScale())[col] > 0) + { + if (row.getIntField(col) & joblist::BIGINTNULL) + ret = true; + + // break the case block + break; + } + + // real string to check null + if (colWidth <= 8) + { + if (colWidth == 1) + ret = ((uint8_t)row.getUintField(col) == joblist::CHAR1NULL); + else if (colWidth == 2) + ret = ((uint16_t)row.getUintField(col) == joblist::CHAR2NULL); + else if (colWidth < 5) + ret = ((uint32_t)row.getUintField(col) == joblist::CHAR4NULL); + else + ret = ((uint64_t)row.getUintField(col) == joblist::CHAR8NULL); + } + else + { + //@bug 1821 + auto const str = row.getConstString(col); + ret = str.length() == 0 || str.eq(utils::ConstString(joblist::CPNULLSTRMARK)); + } + + break; + } + + case execplan::CalpontSystemCatalog::SMALLINT: + { + ret = ((uint16_t)row.getIntField(col) == joblist::SMALLINTNULL); + break; + } + + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret = ((uint16_t)row.getIntField(col) == joblist::USMALLINTNULL); + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + ret = ((uint64_t)row.getUintField(col) == joblist::DOUBLENULL); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret = (row.getLongDoubleField(col) == joblist::LONGDOUBLENULL); + break; + } + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + { + ret = ((uint32_t)row.getIntField(col) == joblist::INTNULL); + break; + } + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + { + ret = ((uint32_t)row.getIntField(col) == joblist::UINTNULL); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret = ((uint32_t)row.getUintField(col) == joblist::FLOATNULL); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + ret = ((uint32_t)row.getUintField(col) == joblist::DATENULL); + break; + } + + case execplan::CalpontSystemCatalog::BIGINT: + { + ret = ((uint64_t)row.getIntField(col) == joblist::BIGINTNULL); + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: + { + ret = ((uint64_t)row.getIntField(col) == joblist::UBIGINTNULL); + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = row.isNullValue(col); + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + ret = ((uint64_t)row.getUintField(col) == joblist::DATETIMENULL); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMESTAMPNULL); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); + break; + } + + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: + { + auto const str = row.getConstString(col); + ret = str.length() == 0 || str.eq(utils::ConstString(joblist::CPNULLSTRMARK)); + break; + } + + default: break; + } + + return ret; +} //------------------------------------------------------------------------------ // Row Aggregation default constructor //------------------------------------------------------------------------------ RowAggregation::RowAggregation() - : fRowGroupOut(nullptr) - , fSmallSideRGs(nullptr) - , fLargeSideRG(nullptr) - , fSmallSideCount(0) - , fOrigFunctionCols(nullptr) + : fRowGroupOut(nullptr) + , fSmallSideRGs(nullptr) + , fLargeSideRG(nullptr) + , fSmallSideCount(0) + , fOrigFunctionCols(nullptr) { } - RowAggregation::RowAggregation(const vector& rowAggGroupByCols, - const vector& rowAggFunctionCols, - joblist::ResourceManager* rm, - boost::shared_ptr sl) - : fRowGroupOut(nullptr) - , fSmallSideRGs(nullptr) - , fLargeSideRG(nullptr) - , fSmallSideCount(0) - , fOrigFunctionCols(nullptr) - , fRm(rm) - , fSessionMemLimit(std::move(sl)) + const vector& rowAggFunctionCols, + joblist::ResourceManager* rm, boost::shared_ptr sl) + : fRowGroupOut(nullptr) + , fSmallSideRGs(nullptr) + , fLargeSideRG(nullptr) + , fSmallSideCount(0) + , fOrigFunctionCols(nullptr) + , fRm(rm) + , fSessionMemLimit(std::move(sl)) { - fGroupByCols.assign(rowAggGroupByCols.begin(), rowAggGroupByCols.end()); - fFunctionCols.assign(rowAggFunctionCols.begin(), rowAggFunctionCols.end()); + fGroupByCols.assign(rowAggGroupByCols.begin(), rowAggGroupByCols.end()); + fFunctionCols.assign(rowAggFunctionCols.begin(), rowAggFunctionCols.end()); } - RowAggregation::RowAggregation(const RowAggregation& rhs) - : fRowGroupOut(nullptr) - , fSmallSideRGs(nullptr) - , fLargeSideRG(nullptr) - , fSmallSideCount(0) - , fKeyOnHeap(rhs.fKeyOnHeap) - , fRGContext(rhs.fRGContext) - , fOrigFunctionCols(nullptr) - , fRm(rhs.fRm) - , fSessionMemLimit(rhs.fSessionMemLimit) + : fRowGroupOut(nullptr) + , fSmallSideRGs(nullptr) + , fLargeSideRG(nullptr) + , fSmallSideCount(0) + , fKeyOnHeap(rhs.fKeyOnHeap) + , fRGContext(rhs.fRGContext) + , fOrigFunctionCols(nullptr) + , fRm(rhs.fRm) + , fSessionMemLimit(rhs.fSessionMemLimit) { - fGroupByCols.assign(rhs.fGroupByCols.begin(), rhs.fGroupByCols.end()); - fFunctionCols.assign(rhs.fFunctionCols.begin(), rhs.fFunctionCols.end()); + fGroupByCols.assign(rhs.fGroupByCols.begin(), rhs.fGroupByCols.end()); + fFunctionCols.assign(rhs.fFunctionCols.begin(), rhs.fFunctionCols.end()); } - //------------------------------------------------------------------------------ // Row Aggregation destructor. //------------------------------------------------------------------------------ @@ -553,7 +538,6 @@ RowAggregation::~RowAggregation() { } - //------------------------------------------------------------------------------ // Aggregate the rows in pRows. User should make Multiple calls to // addRowGroup() to aggregate multiple RowGroups. When all RowGroups have @@ -565,69 +549,67 @@ RowAggregation::~RowAggregation() //------------------------------------------------------------------------------ void RowAggregation::addRowGroup(const RowGroup* pRows) { - // no group by == no map, everything done in fRow - if (fGroupByCols.empty()) + // no group by == no map, everything done in fRow + if (fGroupByCols.empty()) + { + fRowGroupOut->setRowCount(1); + + // special, but very common case -- count(*) without groupby columns + if (fFunctionCols.size() == 1 && fFunctionCols[0]->fAggFunction == ROWAGG_COUNT_ASTERISK) { - fRowGroupOut->setRowCount(1); - - // special, but very common case -- count(*) without groupby columns - if (fFunctionCols.size() == 1 && fFunctionCols[0]->fAggFunction == ROWAGG_COUNT_ASTERISK) - { - if (countSpecial(pRows)) - return; - } + if (countSpecial(pRows)) + return; } + } - fRowGroupOut->setDBRoot(pRows->getDBRoot()); + fRowGroupOut->setDBRoot(pRows->getDBRoot()); - Row rowIn; - pRows->initRow(&rowIn); - pRows->getRow(0, &rowIn); + Row rowIn; + pRows->initRow(&rowIn); + pRows->getRow(0, &rowIn); - for (uint64_t i = 0; i < pRows->getRowCount(); ++i) - { - aggregateRow(rowIn); - rowIn.nextRow(); - } - fRowAggStorage->dump(); + for (uint64_t i = 0; i < pRows->getRowCount(); ++i) + { + aggregateRow(rowIn); + rowIn.nextRow(); + } + fRowAggStorage->dump(); } - void RowAggregation::addRowGroup(const RowGroup* pRows, vector>& inRows) { - // this function is for threaded aggregation, which is for group by and distinct. - // if (countSpecial(pRows)) - Row rowIn; - pRows->initRow(&rowIn); + // this function is for threaded aggregation, which is for group by and distinct. + // if (countSpecial(pRows)) + Row rowIn; + pRows->initRow(&rowIn); - for (const auto& inRow : inRows) - { - rowIn.setData(inRow.first); - aggregateRow(rowIn, &inRow.second); - } - fRowAggStorage->dump(); + for (const auto& inRow : inRows) + { + rowIn.setData(inRow.first); + aggregateRow(rowIn, &inRow.second); + } + fRowAggStorage->dump(); } - //------------------------------------------------------------------------------ // Set join rowgroups and mappings //------------------------------------------------------------------------------ void RowAggregation::setJoinRowGroups(vector* pSmallSideRG, RowGroup* pLargeSideRG) { - fSmallSideRGs = pSmallSideRG; - fLargeSideRG = pLargeSideRG; - fSmallSideCount = fSmallSideRGs->size(); - fSmallMappings.reset(new shared_array[fSmallSideCount]); + fSmallSideRGs = pSmallSideRG; + fLargeSideRG = pLargeSideRG; + fSmallSideCount = fSmallSideRGs->size(); + fSmallMappings.reset(new shared_array[fSmallSideCount]); - for (uint32_t i = 0; i < fSmallSideCount; i++) - fSmallMappings[i] = makeMapping((*fSmallSideRGs)[i], fRowGroupIn); + for (uint32_t i = 0; i < fSmallSideCount; i++) + fSmallMappings[i] = makeMapping((*fSmallSideRGs)[i], fRowGroupIn); - fLargeMapping = makeMapping(*fLargeSideRG, fRowGroupIn); + fLargeMapping = makeMapping(*fLargeSideRG, fRowGroupIn); - rowSmalls.reset(new Row[fSmallSideCount]); + rowSmalls.reset(new Row[fSmallSideCount]); - for (uint32_t i = 0; i < fSmallSideCount; i++) - (*fSmallSideRGs)[i].initRow(&rowSmalls[i]); + for (uint32_t i = 0; i < fSmallSideCount; i++) + (*fSmallSideRGs)[i].initRow(&rowSmalls[i]); } //------------------------------------------------------------------------------ @@ -639,30 +621,27 @@ void RowAggregation::setJoinRowGroups(vector* pSmallSideRG, RowGroup* //------------------------------------------------------------------------------ void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColsIdx) { - // RowAggregation and it's functions need to be re-entrant which means - // each instance (thread) needs its own copy of the context object. - // Note: operator=() doesn't copy userData. - fRGContextColl[funcColsIdx] = rowUDAF->fUDAFContext; + // RowAggregation and it's functions need to be re-entrant which means + // each instance (thread) needs its own copy of the context object. + // Note: operator=() doesn't copy userData. + fRGContextColl[funcColsIdx] = rowUDAF->fUDAFContext; - // Call the user reset for the group userData. Since, at this point, - // context's userData will be NULL, reset will generate a new one. - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContextColl[funcColsIdx].getFunction()->reset(&fRGContextColl[funcColsIdx]); + // Call the user reset for the group userData. Since, at this point, + // context's userData will be NULL, reset will generate a new one. + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + rc = fRGContextColl[funcColsIdx].getFunction()->reset(&fRGContextColl[funcColsIdx]); - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContextColl[funcColsIdx].getErrorMessage(), - logging::aggregateFuncErr); - } + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + rowUDAF->bInterrupted = true; + throw logging::QueryDataExcept(fRGContextColl[funcColsIdx].getErrorMessage(), logging::aggregateFuncErr); + } - fRow.setUserDataStore(fRowGroupOut->getRGData()->getUserDataStore()); - fRow.setUserData(fRGContextColl[funcColsIdx], - fRGContextColl[funcColsIdx].getUserDataSP(), - fRGContextColl[funcColsIdx].getUserDataSize(), - rowUDAF->fAuxColumnIndex); - // Prevents calling deleteUserData on the mcsv1Context. - fRGContextColl[funcColsIdx].setUserData(NULL); + fRow.setUserDataStore(fRowGroupOut->getRGData()->getUserDataStore()); + fRow.setUserData(fRGContextColl[funcColsIdx], fRGContextColl[funcColsIdx].getUserDataSP(), + fRGContextColl[funcColsIdx].getUserDataSize(), rowUDAF->fAuxColumnIndex); + // Prevents calling deleteUserData on the mcsv1Context. + fRGContextColl[funcColsIdx].setUserData(NULL); } //------------------------------------------------------------------------------ @@ -671,90 +650,77 @@ void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColsIdx //------------------------------------------------------------------------------ void RowAggregation::initialize() { - // Calculate the length of the hashmap key. - fAggMapKeyCount = fGroupByCols.size(); - bool disk_agg = fRm ? fRm->getAllowDiskAggregation() : false; - bool allow_gen = true; - for (auto& fun : fFunctionCols) + // Calculate the length of the hashmap key. + fAggMapKeyCount = fGroupByCols.size(); + bool disk_agg = fRm ? fRm->getAllowDiskAggregation() : false; + bool allow_gen = true; + for (auto& fun : fFunctionCols) + { + if (fun->fAggFunction == ROWAGG_UDAF || fun->fAggFunction == ROWAGG_GROUP_CONCAT) { - if (fun->fAggFunction == ROWAGG_UDAF || fun->fAggFunction == ROWAGG_GROUP_CONCAT) + allow_gen = false; + break; + } + } + + config::Config* config = config::Config::makeConfig(); + string tmpDir = config->getTempFileDir(config::Config::TempDirPurpose::Aggregates); + string compStr = config->getConfig("RowAggregation", "Compression"); + auto* compressor = compress::getCompressInterfaceByName(compStr); + + if (fKeyOnHeap) + { + fRowAggStorage.reset(new RowAggStorage(tmpDir, fRowGroupOut, &fKeyRG, fAggMapKeyCount, fRm, + fSessionMemLimit, disk_agg, allow_gen, compressor)); + } + else + { + fRowAggStorage.reset(new RowAggStorage(tmpDir, fRowGroupOut, fAggMapKeyCount, fRm, fSessionMemLimit, + disk_agg, allow_gen, compressor)); + } + + // Initialize the work row. + fRowGroupOut->initRow(&fRow); + fRowGroupOut->getRow(0, &fRow); + makeAggFieldsNull(fRow); + + // Keep a copy of the null row to initialize new map entries. + fRowGroupOut->initRow(&fNullRow, true); + fNullRowData.reset(new uint8_t[fNullRow.getSize()]); + fNullRow.setData(fNullRowData.get()); + copyRow(fRow, &fNullRow); + + // Lazy approach w/o a mapping b/w fFunctionCols idx and fRGContextColl idx + fRGContextColl.resize(fFunctionCols.size()); + + // Need map only if groupby list is not empty. + if (fGroupByCols.empty()) + { + fRowGroupOut->setRowCount(1); + attachGroupConcatAg(); + // For UDAF, reset the data + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - allow_gen = false; - break; + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } + } - config::Config* config = config::Config::makeConfig(); - string tmpDir = config->getTempFileDir(config::Config::TempDirPurpose::Aggregates); - string compStr = config->getConfig("RowAggregation", "Compression"); - auto* compressor = compress::getCompressInterfaceByName(compStr); + // for 8k poc: an empty output row group to match message count + fEmptyRowGroup = *fRowGroupOut; + fEmptyRowData.reinit(*fRowGroupOut, 1); + fEmptyRowGroup.setData(&fEmptyRowData); + fEmptyRowGroup.resetRowGroup(0); + fEmptyRowGroup.initRow(&fEmptyRow); + fEmptyRowGroup.getRow(0, &fEmptyRow); - if (fKeyOnHeap) - { - fRowAggStorage.reset(new RowAggStorage(tmpDir, - fRowGroupOut, - &fKeyRG, - fAggMapKeyCount, - fRm, - fSessionMemLimit, - disk_agg, - allow_gen, - compressor)); - } - else - { - fRowAggStorage.reset(new RowAggStorage(tmpDir, - fRowGroupOut, - fAggMapKeyCount, - fRm, - fSessionMemLimit, - disk_agg, - allow_gen, - compressor)); - } + copyRow(fNullRow, &fEmptyRow); - // Initialize the work row. - fRowGroupOut->initRow(&fRow); - fRowGroupOut->getRow(0, &fRow); - makeAggFieldsNull(fRow); - - // Keep a copy of the null row to initialize new map entries. - fRowGroupOut->initRow(&fNullRow, true); - fNullRowData.reset(new uint8_t[fNullRow.getSize()]); - fNullRow.setData(fNullRowData.get()); - copyRow(fRow, &fNullRow); - - // Lazy approach w/o a mapping b/w fFunctionCols idx and fRGContextColl idx - fRGContextColl.resize(fFunctionCols.size()); - - // Need map only if groupby list is not empty. - if (fGroupByCols.empty()) - { - fRowGroupOut->setRowCount(1); - attachGroupConcatAg(); - // For UDAF, reset the data - for (uint64_t i = 0; i < fFunctionCols.size(); i++) - { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) - { - auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); - resetUDAF(rowUDAFColumnPtr, i); - } - } - } - - // for 8k poc: an empty output row group to match message count - fEmptyRowGroup = *fRowGroupOut; - fEmptyRowData.reinit(*fRowGroupOut, 1); - fEmptyRowGroup.setData(&fEmptyRowData); - fEmptyRowGroup.resetRowGroup(0); - fEmptyRowGroup.initRow(&fEmptyRow); - fEmptyRowGroup.getRow(0, &fEmptyRow); - - copyRow(fNullRow, &fEmptyRow); - - if (fGroupByCols.empty()) // no groupby - fEmptyRowGroup.setRowCount(1); + if (fGroupByCols.empty()) // no groupby + fEmptyRowGroup.setRowCount(1); } //------------------------------------------------------------------------------ @@ -762,246 +728,225 @@ void RowAggregation::initialize() //------------------------------------------------------------------------------ void RowAggregation::aggReset() { - bool disk_agg = fRm ? fRm->getAllowDiskAggregation() : false; - bool allow_gen = true; - for (auto& fun : fFunctionCols) + bool disk_agg = fRm ? fRm->getAllowDiskAggregation() : false; + bool allow_gen = true; + for (auto& fun : fFunctionCols) + { + if (fun->fAggFunction == ROWAGG_UDAF || fun->fAggFunction == ROWAGG_GROUP_CONCAT) { - if (fun->fAggFunction == ROWAGG_UDAF || fun->fAggFunction == ROWAGG_GROUP_CONCAT) - { - allow_gen = false; - break; - } + allow_gen = false; + break; } + } - config::Config* config = config::Config::makeConfig(); - string tmpDir = config->getTempFileDir(config::Config::TempDirPurpose::Aggregates); - string compStr = config->getConfig("RowAggregation", "Compression"); - auto* compressor = compress::getCompressInterfaceByName(compStr); + config::Config* config = config::Config::makeConfig(); + string tmpDir = config->getTempFileDir(config::Config::TempDirPurpose::Aggregates); + string compStr = config->getConfig("RowAggregation", "Compression"); + auto* compressor = compress::getCompressInterfaceByName(compStr); - if (fKeyOnHeap) - { - fRowAggStorage.reset(new RowAggStorage(tmpDir, - fRowGroupOut, - &fKeyRG, - fAggMapKeyCount, - fRm, - fSessionMemLimit, - disk_agg, - allow_gen, - compressor)); - } - else - { - fRowAggStorage.reset(new RowAggStorage(tmpDir, - fRowGroupOut, - fAggMapKeyCount, - fRm, - fSessionMemLimit, - disk_agg, - allow_gen, - compressor)); - } - fRowGroupOut->getRow(0, &fRow); - copyNullRow(fRow); - attachGroupConcatAg(); + if (fKeyOnHeap) + { + fRowAggStorage.reset(new RowAggStorage(tmpDir, fRowGroupOut, &fKeyRG, fAggMapKeyCount, fRm, + fSessionMemLimit, disk_agg, allow_gen, compressor)); + } + else + { + fRowAggStorage.reset(new RowAggStorage(tmpDir, fRowGroupOut, fAggMapKeyCount, fRm, fSessionMemLimit, + disk_agg, allow_gen, compressor)); + } + fRowGroupOut->getRow(0, &fRow); + copyNullRow(fRow); + attachGroupConcatAg(); - // For UDAF, reset the data - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + // For UDAF, reset the data + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) - { - auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); - resetUDAF(rowUDAFColumnPtr, i); - } + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } + } } void RowAggregation::append(RowAggregation* other) { - fRowAggStorage->append(*other->fRowAggStorage); + fRowAggStorage->append(*other->fRowAggStorage); } - void RowAggregationUM::aggReset() { - if (fKeyOnHeap) - { - fKeyRG = fRowGroupIn.truncate(fGroupByCols.size()); - } - RowAggregation::aggReset(); + if (fKeyOnHeap) + { + fKeyRG = fRowGroupIn.truncate(fGroupByCols.size()); + } + RowAggregation::aggReset(); } -void RowAggregation::aggregateRow(Row& row, - const uint64_t* hash, +void RowAggregation::aggregateRow(Row& row, const uint64_t* hash, std::vector* rgContextColl) { - // groupby column list is not empty, find the entry. - if (!fGroupByCols.empty()) + // groupby column list is not empty, find the entry. + if (!fGroupByCols.empty()) + { + bool is_new_row; + if (hash != nullptr) + is_new_row = fRowAggStorage->getTargetRow(row, *hash, fRow); + else + is_new_row = fRowAggStorage->getTargetRow(row, fRow); + + if (is_new_row) { - bool is_new_row; - if (hash != nullptr) - is_new_row = fRowAggStorage->getTargetRow(row, *hash, fRow); - else - is_new_row = fRowAggStorage->getTargetRow(row, fRow); + initMapData(row); + attachGroupConcatAg(); - if (is_new_row) + // If there's UDAF involved, reset the user data. + if (fOrigFunctionCols) + { + // This is a multi-distinct query and fFunctionCols may not + // contain all the UDAF we need to reset + for (uint64_t i = 0; i < fOrigFunctionCols->size(); i++) { - initMapData(row); - attachGroupConcatAg(); - - // If there's UDAF involved, reset the user data. - if (fOrigFunctionCols) + if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) { - // This is a multi-distinct query and fFunctionCols may not - // contain all the UDAF we need to reset - for (uint64_t i = 0; i < fOrigFunctionCols->size(); i++) - { - if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) - { - auto rowUDAFColumnPtr = dynamic_cast((*fOrigFunctionCols)[i].get()); - resetUDAF(rowUDAFColumnPtr, i); - } - } - } - else - { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) - { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) - { - auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); - resetUDAF(rowUDAFColumnPtr, i); - } - } + auto rowUDAFColumnPtr = dynamic_cast((*fOrigFunctionCols)[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } + } + else + { + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + { + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); + } + } + } } + } - updateEntry(row, rgContextColl); + updateEntry(row, rgContextColl); } - //------------------------------------------------------------------------------ // Initialize the working row, all aggregation fields to all null values or 0. //------------------------------------------------------------------------------ void RowAggregation::initMapData(const Row& rowIn) { - // First, copy the null row. - copyNullRow(fRow); + // First, copy the null row. + copyNullRow(fRow); - // Then, populate the groupby cols. - for (auto& fGroupByCol : fGroupByCols) + // Then, populate the groupby cols. + for (auto& fGroupByCol : fGroupByCols) + { + int64_t colOut = fGroupByCol->fOutputColumnIndex; + + if (colOut == numeric_limits::max()) + continue; + + int64_t colIn = fGroupByCol->fInputColumnIndex; + int colDataType = ((fRowGroupIn.getColTypes())[colIn]); + + switch (colDataType) { - int64_t colOut = fGroupByCol->fOutputColumnIndex; + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + { + fRow.setIntField(rowIn.getIntField(colIn), colOut); + break; + } - if (colOut == numeric_limits::max()) - continue; - - int64_t colIn = fGroupByCol->fInputColumnIndex; - int colDataType = ((fRowGroupIn.getColTypes())[colIn]); - - switch (colDataType) + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - fRow.setIntField(rowIn.getIntField(colIn), colOut); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) - { - uint32_t colOutOffset = fRow.getOffset(colOut); - fRow.setBinaryField_offset( - rowIn.getBinaryField(colIn), - sizeof(int128_t), - colOutOffset); - } - else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) - { - fRow.setIntField(rowIn.getIntField(colIn), colOut); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregation::initMapData(): DECIMAL bad length."); - } - - break; - } - - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - fRow.setUintField(rowIn.getUintField(colIn), colOut); - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - int colWidth = fRowGroupIn.getColumnWidth(colIn); - - if (colWidth <= 8) - { - fRow.setUintField(rowIn.getUintField(colIn), colOut); - } - else - { - fRow.setStringField(rowIn.getConstString(colIn), colOut); - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - fRow.setDoubleField(rowIn.getDoubleField(colIn), colOut); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - fRow.setLongDoubleField(rowIn.getLongDoubleField(colIn), colOut); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - fRow.setFloatField(rowIn.getFloatField(colIn), colOut); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::TIME: - { - fRow.setUintField(rowIn.getUintField(colIn), colOut); - break; - } - - default: - { - break; - } + uint32_t colOutOffset = fRow.getOffset(colOut); + fRow.setBinaryField_offset(rowIn.getBinaryField(colIn), sizeof(int128_t), colOutOffset); + } + else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + fRow.setIntField(rowIn.getIntField(colIn), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::initMapData(): DECIMAL bad length."); } - } -} + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + fRow.setUintField(rowIn.getUintField(colIn), colOut); + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + int colWidth = fRowGroupIn.getColumnWidth(colIn); + + if (colWidth <= 8) + { + fRow.setUintField(rowIn.getUintField(colIn), colOut); + } + else + { + fRow.setStringField(rowIn.getConstString(colIn), colOut); + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + fRow.setDoubleField(rowIn.getDoubleField(colIn), colOut); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + fRow.setLongDoubleField(rowIn.getLongDoubleField(colIn), colOut); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + fRow.setFloatField(rowIn.getFloatField(colIn), colOut); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setUintField(rowIn.getUintField(colIn), colOut); + break; + } + + default: + { + break; + } + } + } +} //------------------------------------------------------------------------------ // Add group_concat to the initialized working row @@ -1010,150 +955,143 @@ void RowAggregation::attachGroupConcatAg() { } - //------------------------------------------------------------------------------ // Make all aggregation fields to null. //------------------------------------------------------------------------------ void RowAggregation::makeAggFieldsNull(Row& row) { - // initialize all bytes to 0 - memset(row.getData(), 0, row.getSize()); - //row.initToNull(); + // initialize all bytes to 0 + memset(row.getData(), 0, row.getSize()); + // row.initToNull(); - for (auto& fFunctionCol : fFunctionCols) + for (auto& fFunctionCol : fFunctionCols) + { + // Initial count fields to 0. + int64_t colOut = fFunctionCol->fOutputColumnIndex; + + if (fFunctionCol->fAggFunction == ROWAGG_COUNT_ASTERISK || + fFunctionCol->fAggFunction == ROWAGG_COUNT_COL_NAME || + fFunctionCol->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME || + fFunctionCol->fAggFunction == ROWAGG_COUNT_NO_OP || + fFunctionCol->fAggFunction == ROWAGG_GROUP_CONCAT || fFunctionCol->fAggFunction == ROWAGG_STATS) { - // Initial count fields to 0. - int64_t colOut = fFunctionCol->fOutputColumnIndex; - - if (fFunctionCol->fAggFunction == ROWAGG_COUNT_ASTERISK || - fFunctionCol->fAggFunction == ROWAGG_COUNT_COL_NAME || - fFunctionCol->fAggFunction == ROWAGG_COUNT_DISTINCT_COL_NAME || - fFunctionCol->fAggFunction == ROWAGG_COUNT_NO_OP || - fFunctionCol->fAggFunction == ROWAGG_GROUP_CONCAT || - fFunctionCol->fAggFunction == ROWAGG_STATS) - { - continue; - } - - // ROWAGG_BIT_AND : 0xFFFFFFFFFFFFFFFFULL; - // ROWAGG_BIT_OR/ROWAGG_BIT_XOR : 0 (already set). - if (fFunctionCol->fAggFunction == ROWAGG_BIT_OR || - fFunctionCol->fAggFunction == ROWAGG_BIT_XOR) - { - continue; - } - else if (fFunctionCol->fAggFunction == ROWAGG_BIT_AND) - { - row.setUintField(0xFFFFFFFFFFFFFFFFULL, colOut); - continue; - } - - // Initial other aggregation fields to null. - int colDataType = (fRowGroupOut->getColTypes())[colOut]; - - switch (colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - row.setIntField(getIntNullValue(colDataType), colOut); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - row.setUintField(getUintNullValue(colDataType), colOut); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - uint32_t colWidth = fRowGroupOut->getColumnWidth(colOut); - if (LIKELY(colWidth == datatypes::MAXDECIMALWIDTH)) - { - uint32_t offset = row.getOffset(colOut); - row.setBinaryField_offset( - const_cast(&datatypes::Decimal128Null), - colWidth, - offset); - } - else if (colWidth <= datatypes::MAXLEGACYWIDTH) - { - row.setIntField(getUintNullValue(colDataType, colWidth), colOut); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregation::makeAggFieldsNull(): DECIMAL bad length."); - } - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::BLOB: - { - uint32_t colWidth = fRowGroupOut->getColumnWidth(colOut); - - if (colWidth <= datatypes::MAXLEGACYWIDTH) - { - row.setUintField(getUintNullValue(colDataType, colWidth), colOut); - } - else - { - row.setStringField(getStringNullValue(), colOut); - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - row.setDoubleField(getDoubleNullValue(), colOut); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - row.setLongDoubleField(getLongDoubleNullValue(), colOut); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - row.setFloatField(getFloatNullValue(), colOut); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::TIME: - { - row.setUintField(getUintNullValue(colDataType), colOut); - break; - } - - default: - { - break; - } - } + continue; } -} + // ROWAGG_BIT_AND : 0xFFFFFFFFFFFFFFFFULL; + // ROWAGG_BIT_OR/ROWAGG_BIT_XOR : 0 (already set). + if (fFunctionCol->fAggFunction == ROWAGG_BIT_OR || fFunctionCol->fAggFunction == ROWAGG_BIT_XOR) + { + continue; + } + else if (fFunctionCol->fAggFunction == ROWAGG_BIT_AND) + { + row.setUintField(0xFFFFFFFFFFFFFFFFULL, colOut); + continue; + } + + // Initial other aggregation fields to null. + int colDataType = (fRowGroupOut->getColTypes())[colOut]; + + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + { + row.setIntField(getIntNullValue(colDataType), colOut); + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + row.setUintField(getUintNullValue(colDataType), colOut); + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + uint32_t colWidth = fRowGroupOut->getColumnWidth(colOut); + if (LIKELY(colWidth == datatypes::MAXDECIMALWIDTH)) + { + uint32_t offset = row.getOffset(colOut); + row.setBinaryField_offset(const_cast(&datatypes::Decimal128Null), colWidth, offset); + } + else if (colWidth <= datatypes::MAXLEGACYWIDTH) + { + row.setIntField(getUintNullValue(colDataType, colWidth), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::makeAggFieldsNull(): DECIMAL bad length."); + } + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: + { + uint32_t colWidth = fRowGroupOut->getColumnWidth(colOut); + + if (colWidth <= datatypes::MAXLEGACYWIDTH) + { + row.setUintField(getUintNullValue(colDataType, colWidth), colOut); + } + else + { + row.setStringField(getStringNullValue(), colOut); + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + row.setDoubleField(getDoubleNullValue(), colOut); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + row.setLongDoubleField(getLongDoubleNullValue(), colOut); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + row.setFloatField(getFloatNullValue(), colOut); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + { + row.setUintField(getUintNullValue(colDataType), colOut); + break; + } + + default: + { + break; + } + } + } +} //------------------------------------------------------------------------------ // Update the min/max fields if input is not null. @@ -1166,113 +1104,112 @@ void RowAggregation::makeAggFieldsNull(Row& row) //------------------------------------------------------------------------------ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, int funcType) { - int colDataType = (fRowGroupIn.getColTypes())[colIn]; + int colDataType = (fRowGroupIn.getColTypes())[colIn]; - if (isNull(&fRowGroupIn, rowIn, colIn) == true) - return; + if (isNull(&fRowGroupIn, rowIn, colIn) == true) + return; - switch (colDataType) + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - int64_t valIn = rowIn.getIntField(colIn); - int64_t valOut = fRow.getIntField(colOut); - updateIntMinMax(valIn, valOut, colOut, funcType); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) - { - updateIntMinMax(rowIn.getBinaryField(colIn), - fRow.getBinaryField(colOut), - colOut, funcType); - } - else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) - { - int64_t valIn = rowIn.getIntField(colIn); - int64_t valOut = fRow.getIntField(colOut); - updateIntMinMax(valIn, valOut, colOut, funcType); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregation::doMinMax(): DECIMAL bad length."); - } - - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - uint64_t valIn = rowIn.getUintField(colIn); - uint64_t valOut = fRow.getUintField(colOut); - updateUintMinMax(valIn, valOut, colOut, funcType); - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - string valIn = rowIn.getStringField(colIn); - string valOut = fRow.getStringField(colOut); - updateStringMinMax(valIn, valOut, colOut, funcType); - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double valIn = rowIn.getDoubleField(colIn); - double valOut = fRow.getDoubleField(colOut); - updateDoubleMinMax(valIn, valOut, colOut, funcType); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float valIn = rowIn.getFloatField(colIn); - float valOut = fRow.getFloatField(colOut); - updateFloatMinMax(valIn, valOut, colOut, funcType); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::TIME: - { - uint64_t valIn = rowIn.getUintField(colIn); - uint64_t valOut = fRow.getUintField(colOut); - updateUintMinMax(valIn, valOut, colOut, funcType); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double valIn = rowIn.getLongDoubleField(colIn); - long double valOut = fRow.getLongDoubleField(colOut); - updateLongDoubleMinMax(valIn, valOut, colOut, funcType); - break; - } - - default: - { - break; - } + int64_t valIn = rowIn.getIntField(colIn); + int64_t valOut = fRow.getIntField(colOut); + updateIntMinMax(valIn, valOut, colOut, funcType); + break; } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + updateIntMinMax(rowIn.getBinaryField(colIn), fRow.getBinaryField(colOut), colOut, + funcType); + } + else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + int64_t valIn = rowIn.getIntField(colIn); + int64_t valOut = fRow.getIntField(colOut); + updateIntMinMax(valIn, valOut, colOut, funcType); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doMinMax(): DECIMAL bad length."); + } + + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + uint64_t valIn = rowIn.getUintField(colIn); + uint64_t valOut = fRow.getUintField(colOut); + updateUintMinMax(valIn, valOut, colOut, funcType); + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + string valIn = rowIn.getStringField(colIn); + string valOut = fRow.getStringField(colOut); + updateStringMinMax(valIn, valOut, colOut, funcType); + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double valIn = rowIn.getDoubleField(colIn); + double valOut = fRow.getDoubleField(colOut); + updateDoubleMinMax(valIn, valOut, colOut, funcType); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float valIn = rowIn.getFloatField(colIn); + float valOut = fRow.getFloatField(colOut); + updateFloatMinMax(valIn, valOut, colOut, funcType); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + { + uint64_t valIn = rowIn.getUintField(colIn); + uint64_t valOut = fRow.getUintField(colOut); + updateUintMinMax(valIn, valOut, colOut, funcType); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double valIn = rowIn.getLongDoubleField(colIn); + long double valOut = fRow.getLongDoubleField(colOut); + updateLongDoubleMinMax(valIn, valOut, colOut, funcType); + break; + } + + default: + { + break; + } + } } //------------------------------------------------------------------------------ @@ -1286,148 +1223,148 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i //------------------------------------------------------------------------------ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int funcType) { - datatypes::SystemCatalog::ColDataType colDataType = rowIn.getColType(colIn); - long double valIn = 0; - bool isWideDataType = false; - void *wideValInPtr = nullptr; + datatypes::SystemCatalog::ColDataType colDataType = rowIn.getColType(colIn); + long double valIn = 0; + bool isWideDataType = false; + void* wideValInPtr = nullptr; - if (rowIn.isNullValue(colIn)) - return; + if (rowIn.isNullValue(colIn)) + return; - switch (colDataType) + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - valIn = rowIn.getIntField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - valIn = rowIn.getUintField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - uint32_t width = rowIn.getColumnWidth(colIn); - isWideDataType = width == datatypes::MAXDECIMALWIDTH; - if(LIKELY(isWideDataType)) - { - int128_t *dec = rowIn.getBinaryField(colIn); - wideValInPtr = reinterpret_cast(dec); - } - else if (width <= datatypes::MAXLEGACYWIDTH) - { - uint32_t scale = rowIn.getScale(colIn); - valIn = rowIn.getScaledSInt64FieldAsXFloat(colIn, scale); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregation::doSum(): DECIMAL bad length."); - } - - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - std::ostringstream errmsg; - errmsg << "RowAggregation: sum(CHAR[]) is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - valIn = rowIn.getDoubleField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - valIn = rowIn.getFloatField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: - { - std::ostringstream errmsg; - errmsg << "RowAggregation: sum(date|date time) is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - valIn = rowIn.getLongDoubleField(colIn); - break; - } - - default: - { - break; - } + valIn = rowIn.getIntField(colIn); + break; } - if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(colDataType) && !isWideDataType) + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: { - if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) - { - int128_t *valOutPtr = fRow.getBinaryField(colOut); - int128_t sum = *valOutPtr + valIn; - fRow.setBinaryField(&sum, colOut); - } - else - { - int128_t sum = valIn; - fRow.setBinaryField(&sum, colOut); - } + valIn = rowIn.getUintField(colIn); + break; } - else if (isWideDataType) + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - int128_t* dec = reinterpret_cast(wideValInPtr); - if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) - { - int128_t *valOutPtr = fRow.getBinaryField(colOut); - int128_t sum = *valOutPtr + *dec; - fRow.setBinaryField(&sum, colOut); - } - else - { - fRow.setBinaryField(dec, colOut); - } + uint32_t width = rowIn.getColumnWidth(colIn); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if (LIKELY(isWideDataType)) + { + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + uint32_t scale = rowIn.getScale(colIn); + valIn = rowIn.getScaledSInt64FieldAsXFloat(colIn, scale); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doSum(): DECIMAL bad length."); + } + + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + std::ostringstream errmsg; + errmsg << "RowAggregation: sum(CHAR[]) is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + valIn = rowIn.getDoubleField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + valIn = rowIn.getFloatField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: + { + std::ostringstream errmsg; + errmsg << "RowAggregation: sum(date|date time) is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + valIn = rowIn.getLongDoubleField(colIn); + break; + } + + default: + { + break; + } + } + + if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(colDataType) && !isWideDataType) + { + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) + { + int128_t* valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + valIn; + fRow.setBinaryField(&sum, colOut); } else { - if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) - { - long double valOut = fRow.getLongDoubleField(colOut); - fRow.setLongDoubleField(valIn+valOut, colOut); - } - else - { - fRow.setLongDoubleField(valIn, colOut); - } + int128_t sum = valIn; + fRow.setBinaryField(&sum, colOut); } + } + else if (isWideDataType) + { + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) + { + int128_t* valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField(&sum, colOut); + } + else + { + fRow.setBinaryField(dec, colOut); + } + } + else + { + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) + { + long double valOut = fRow.getLongDoubleField(colOut); + fRow.setLongDoubleField(valIn + valOut, colOut); + } + else + { + fRow.setLongDoubleField(valIn, colOut); + } + } } //------------------------------------------------------------------------------ @@ -1441,168 +1378,166 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int //------------------------------------------------------------------------------ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, int funcType) { - int colDataType = (fRowGroupIn.getColTypes())[colIn]; + int colDataType = (fRowGroupIn.getColTypes())[colIn]; - if (isNull(&fRowGroupIn, rowIn, colIn) == true) - return; + if (isNull(&fRowGroupIn, rowIn, colIn) == true) + return; - int64_t valIn = 0; - uint64_t uvalIn = 0; + int64_t valIn = 0; + uint64_t uvalIn = 0; - switch (colDataType) + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - valIn = rowIn.getIntField(colIn); + valIn = rowIn.getIntField(colIn); - if ((fRowGroupIn.getScale())[colIn] != 0) - { - valIn = rowIn.getIntField(colIn); - valIn /= IDB_pow[fRowGroupIn.getScale()[colIn] - 1]; + if ((fRowGroupIn.getScale())[colIn] != 0) + { + valIn = rowIn.getIntField(colIn); + valIn /= IDB_pow[fRowGroupIn.getScale()[colIn] - 1]; - if (valIn > 0) - valIn += 5; - else if (valIn < 0) - valIn -= 5; + if (valIn > 0) + valIn += 5; + else if (valIn < 0) + valIn -= 5; - valIn /= 10; - } + valIn /= 10; + } - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - uvalIn = rowIn.getUintField(colIn); - uint64_t uvalOut = fRow.getUintField(colOut); - - if (funcType == ROWAGG_BIT_AND) - fRow.setUintField(uvalIn & uvalOut, colOut); - else if (funcType == ROWAGG_BIT_OR) - fRow.setUintField(uvalIn | uvalOut, colOut); - else - fRow.setUintField(uvalIn ^ uvalOut, colOut); - - return; - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - string str = rowIn.getStringField(colIn); - valIn = strtoll(str.c_str(), nullptr, 10); - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - double dbl = 0.0; - - if (colDataType == execplan::CalpontSystemCatalog::DOUBLE || - colDataType == execplan::CalpontSystemCatalog::UDOUBLE) - dbl = rowIn.getDoubleField(colIn); - else if (colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE) - dbl = (double)rowIn.getLongDoubleField(colIn); - else - dbl = rowIn.getFloatField(colIn); - - int64_t maxint = 0x7FFFFFFFFFFFFFFFLL; - int64_t minint = 0x8000000000000000LL; - - if (dbl > maxint) - { - valIn = maxint; - } - else if (dbl < minint) - { - valIn = minint; - } - else - { - dbl += (dbl >= 0) ? 0.5 : -0.5; - valIn = (int64_t) dbl; - } - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - uint64_t dt = rowIn.getUintField(colIn); - dt = dt & 0xFFFFFFC0; // no need to set spare bits to 3E, will shift out - valIn = ((dt >> 16) * 10000) + (((dt >> 12) & 0xF) * 100) + ((dt >> 6) & 077); - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - uint64_t dtm = rowIn.getUintField(colIn); - valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) + - (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) + - (((dtm >> 26) & 077) * 100) + ((dtm >> 20) & 077); - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - uint64_t timestamp = rowIn.getUintField(colIn); - string str = DataConvert::timestampToString1(timestamp, fTimeZone); - // strip off micro seconds - str = str.substr(0, 14); - valIn = strtoll(str.c_str(), nullptr, 10); - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - int64_t dtm = rowIn.getUintField(colIn); - // Handle negative correctly - int hour = 0; - - if ((dtm >> 40) & 0x800) - { - hour = 0xfffff000; - } - - hour |= ((dtm >> 40) & 0xfff); - valIn = (hour * 10000) + - (((dtm >> 32) & 0xff) * 100) + ((dtm >> 24) & 0xff); - break; - } - - default: - { - break; - } + break; } - int64_t valOut = fRow.getIntField(colOut); + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + uvalIn = rowIn.getUintField(colIn); + uint64_t uvalOut = fRow.getUintField(colOut); - if (funcType == ROWAGG_BIT_AND) - fRow.setIntField(valIn & valOut, colOut); - else if (funcType == ROWAGG_BIT_OR) - fRow.setIntField(valIn | valOut, colOut); - else - fRow.setIntField(valIn ^ valOut, colOut); + if (funcType == ROWAGG_BIT_AND) + fRow.setUintField(uvalIn & uvalOut, colOut); + else if (funcType == ROWAGG_BIT_OR) + fRow.setUintField(uvalIn | uvalOut, colOut); + else + fRow.setUintField(uvalIn ^ uvalOut, colOut); + + return; + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + string str = rowIn.getStringField(colIn); + valIn = strtoll(str.c_str(), nullptr, 10); + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + double dbl = 0.0; + + if (colDataType == execplan::CalpontSystemCatalog::DOUBLE || + colDataType == execplan::CalpontSystemCatalog::UDOUBLE) + dbl = rowIn.getDoubleField(colIn); + else if (colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE) + dbl = (double)rowIn.getLongDoubleField(colIn); + else + dbl = rowIn.getFloatField(colIn); + + int64_t maxint = 0x7FFFFFFFFFFFFFFFLL; + int64_t minint = 0x8000000000000000LL; + + if (dbl > maxint) + { + valIn = maxint; + } + else if (dbl < minint) + { + valIn = minint; + } + else + { + dbl += (dbl >= 0) ? 0.5 : -0.5; + valIn = (int64_t)dbl; + } + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + uint64_t dt = rowIn.getUintField(colIn); + dt = dt & 0xFFFFFFC0; // no need to set spare bits to 3E, will shift out + valIn = ((dt >> 16) * 10000) + (((dt >> 12) & 0xF) * 100) + ((dt >> 6) & 077); + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + uint64_t dtm = rowIn.getUintField(colIn); + valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) + + (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) + (((dtm >> 26) & 077) * 100) + + ((dtm >> 20) & 077); + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + uint64_t timestamp = rowIn.getUintField(colIn); + string str = DataConvert::timestampToString1(timestamp, fTimeZone); + // strip off micro seconds + str = str.substr(0, 14); + valIn = strtoll(str.c_str(), nullptr, 10); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t dtm = rowIn.getUintField(colIn); + // Handle negative correctly + int hour = 0; + + if ((dtm >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((dtm >> 40) & 0xfff); + valIn = (hour * 10000) + (((dtm >> 32) & 0xff) * 100) + ((dtm >> 24) & 0xff); + break; + } + + default: + { + break; + } + } + + int64_t valOut = fRow.getIntField(colOut); + + if (funcType == ROWAGG_BIT_AND) + fRow.setIntField(valIn & valOut, colOut); + else if (funcType == ROWAGG_BIT_OR) + fRow.setIntField(valIn | valOut, colOut); + else + fRow.setIntField(valIn ^ valOut, colOut); } - //------------------------------------------------------------------------------ // Marks the end of input into aggregation when aggregating multiple RowGroups. //------------------------------------------------------------------------------ @@ -1610,7 +1545,6 @@ void RowAggregation::endOfInput() { } - //------------------------------------------------------------------------------ // Serialize this RowAggregation object into the specified ByteStream. // Primary information to be serialized is the RowAggGroupByCol and @@ -1619,24 +1553,23 @@ void RowAggregation::endOfInput() //------------------------------------------------------------------------------ void RowAggregation::serialize(messageqcpp::ByteStream& bs) const { - // groupby - uint64_t groupbyCount = fGroupByCols.size(); - bs << groupbyCount; + // groupby + uint64_t groupbyCount = fGroupByCols.size(); + bs << groupbyCount; - for (uint64_t i = 0; i < groupbyCount; i++) - bs << *(fGroupByCols[i].get()); + for (uint64_t i = 0; i < groupbyCount; i++) + bs << *(fGroupByCols[i].get()); - // aggregate function - uint64_t functionCount = fFunctionCols.size(); - bs << functionCount; + // aggregate function + uint64_t functionCount = fFunctionCols.size(); + bs << functionCount; - for (uint64_t i = 0; i < functionCount; i++) - fFunctionCols[i]->serialize(bs); + for (uint64_t i = 0; i < functionCount; i++) + fFunctionCols[i]->serialize(bs); - bs << fTimeZone; + bs << fTimeZone; } - //------------------------------------------------------------------------------ // Unserialze the specified ByteStream into this RowAggregation object. // Primary information to be deserialized is the RowAggGroupByCol and @@ -1645,44 +1578,43 @@ void RowAggregation::serialize(messageqcpp::ByteStream& bs) const //------------------------------------------------------------------------------ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) { - // groupby - uint64_t groupbyCount = 0; - bs >> groupbyCount; + // groupby + uint64_t groupbyCount = 0; + bs >> groupbyCount; - for (uint64_t i = 0; i < groupbyCount; i++) + for (uint64_t i = 0; i < groupbyCount; i++) + { + SP_ROWAGG_GRPBY_t grpby(new RowAggGroupByCol(0, 0)); + bs >> *(grpby.get()); + fGroupByCols.push_back(grpby); + } + + // aggregate function + uint64_t functionCount = 0; + bs >> functionCount; + + for (uint64_t i = 0; i < functionCount; i++) + { + uint8_t funcType; + bs.peek(funcType); + SP_ROWAGG_FUNC_t funct; + + if (funcType == ROWAGG_UDAF) { - SP_ROWAGG_GRPBY_t grpby(new RowAggGroupByCol(0, 0)); - bs >> *(grpby.get()); - fGroupByCols.push_back(grpby); + funct.reset(new RowUDAFFunctionCol(0, 0)); + } + else + { + funct.reset(new RowAggFunctionCol(ROWAGG_FUNCT_UNDEFINE, ROWAGG_FUNCT_UNDEFINE, 0, 0)); } - // aggregate function - uint64_t functionCount = 0; - bs >> functionCount; + funct->deserialize(bs); + fFunctionCols.push_back(funct); + } - for (uint64_t i = 0; i < functionCount; i++) - { - uint8_t funcType; - bs.peek(funcType); - SP_ROWAGG_FUNC_t funct; - - if (funcType == ROWAGG_UDAF) - { - funct.reset(new RowUDAFFunctionCol(0, 0)); - } - else - { - funct.reset(new RowAggFunctionCol(ROWAGG_FUNCT_UNDEFINE, ROWAGG_FUNCT_UNDEFINE, 0, 0)); - } - - funct->deserialize(bs); - fFunctionCols.push_back(funct); - } - - bs >> fTimeZone; + bs >> fTimeZone; } - //------------------------------------------------------------------------------ // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for @@ -1690,79 +1622,70 @@ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) // rowIn(in) - Row to be included in aggregation. // rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregation::updateEntry(const Row& rowIn, - std::vector* rgContextColl) +void RowAggregation::updateEntry(const Row& rowIn, std::vector* rgContextColl) { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + int64_t colIn = fFunctionCols[i]->fInputColumnIndex; + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + + switch (fFunctionCols[i]->fAggFunction) { - int64_t colIn = fFunctionCols[i]->fInputColumnIndex; - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + case ROWAGG_COUNT_COL_NAME: - switch (fFunctionCols[i]->fAggFunction) - { - case ROWAGG_COUNT_COL_NAME: + // if NOT null, let execution fall through. + if (isNull(&fRowGroupIn, rowIn, colIn) == true) + break; + /* fall through */ - // if NOT null, let execution fall through. - if (isNull(&fRowGroupIn, rowIn, colIn) == true) break; - /* fall through */ + case ROWAGG_COUNT_ASTERISK: fRow.setUintField<8>(fRow.getUintField<8>(colOut) + 1, colOut); break; - case ROWAGG_COUNT_ASTERISK: - fRow.setUintField<8>(fRow.getUintField<8>(colOut) + 1, colOut); - break; + case ROWAGG_MIN: + case ROWAGG_MAX: doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_MIN: - case ROWAGG_MAX: - doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_SUM: doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_SUM: - doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_AVG: + // count(column) for average is inserted after the sum, + // colOut+1 is the position of the aux count column. + doAvg(rowIn, colIn, colOut, colOut + 1); + break; - case ROWAGG_AVG: - // count(column) for average is inserted after the sum, - // colOut+1 is the position of the aux count column. - doAvg(rowIn, colIn, colOut, colOut + 1); - break; + case ROWAGG_STATS: doStatistics(rowIn, colIn, colOut, colOut + 1); break; - case ROWAGG_STATS: - doStatistics(rowIn, colIn, colOut, colOut + 1); - break; + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); + break; + } - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; - } + case ROWAGG_COUNT_NO_OP: + case ROWAGG_DUP_FUNCT: + case ROWAGG_DUP_AVG: + case ROWAGG_DUP_STATS: + case ROWAGG_DUP_UDAF: + case ROWAGG_CONSTANT: + case ROWAGG_GROUP_CONCAT: break; - case ROWAGG_COUNT_NO_OP: - case ROWAGG_DUP_FUNCT: - case ROWAGG_DUP_AVG: - case ROWAGG_DUP_STATS: - case ROWAGG_DUP_UDAF: - case ROWAGG_CONSTANT: - case ROWAGG_GROUP_CONCAT: - break; + case ROWAGG_UDAF: + { + doUDAF(rowIn, colIn, colOut, colOut + 1, i, rgContextColl); + break; + } - case ROWAGG_UDAF: - { - doUDAF(rowIn, colIn, colOut, colOut + 1, i, rgContextColl); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregation: function (id = " << - (uint64_t) fFunctionCols[i]->fAggFunction << ") is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - } + default: + { + std::ostringstream errmsg; + errmsg << "RowAggregation: function (id = " << (uint64_t)fFunctionCols[i]->fAggFunction + << ") is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } } + } } //------------------------------------------------------------------------------ @@ -1779,55 +1702,44 @@ void RowAggregation::mergeEntries(const Row& rowIn) switch (fFunctionCols[i]->fAggFunction) { - case ROWAGG_COUNT_COL_NAME: - case ROWAGG_COUNT_ASTERISK: - fRow.setUintField<8>(fRow.getUintField<8>(colOut) + rowIn.getUintField<8>(colOut), colOut); - break; + case ROWAGG_COUNT_COL_NAME: + case ROWAGG_COUNT_ASTERISK: + fRow.setUintField<8>(fRow.getUintField<8>(colOut) + rowIn.getUintField<8>(colOut), colOut); + break; - case ROWAGG_MIN: - case ROWAGG_MAX: - doMinMax(rowIn, colOut, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_MIN: + case ROWAGG_MAX: doMinMax(rowIn, colOut, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_SUM: - doSum(rowIn, colOut, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_SUM: doSum(rowIn, colOut, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_AVG: - // count(column) for average is inserted after the sum, - doAvg(rowIn, colOut, colOut, fFunctionCols[i]->fAuxColumnIndex, true); - break; + case ROWAGG_AVG: + // count(column) for average is inserted after the sum, + doAvg(rowIn, colOut, colOut, fFunctionCols[i]->fAuxColumnIndex, true); + break; - case ROWAGG_STATS: - mergeStatistics(rowIn, colOut, fFunctionCols[i]->fAuxColumnIndex); - break; + case ROWAGG_STATS: mergeStatistics(rowIn, colOut, fFunctionCols[i]->fAuxColumnIndex); break; - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - doBitOp(rowIn, colOut, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: doBitOp(rowIn, colOut, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_COUNT_NO_OP: - case ROWAGG_DUP_FUNCT: - case ROWAGG_DUP_AVG: - case ROWAGG_DUP_STATS: - case ROWAGG_DUP_UDAF: - case ROWAGG_CONSTANT: - case ROWAGG_GROUP_CONCAT: - break; + case ROWAGG_COUNT_NO_OP: + case ROWAGG_DUP_FUNCT: + case ROWAGG_DUP_AVG: + case ROWAGG_DUP_STATS: + case ROWAGG_DUP_UDAF: + case ROWAGG_CONSTANT: + case ROWAGG_GROUP_CONCAT: break; - case ROWAGG_UDAF: - doUDAF(rowIn, colOut, colOut, colOut + 1, i); - break; + case ROWAGG_UDAF: doUDAF(rowIn, colOut, colOut, colOut + 1, i); break; - default: - std::ostringstream errmsg; - errmsg << "RowAggregation: function (id = " << - (uint64_t) fFunctionCols[i]->fAggFunction << ") is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; + default: + std::ostringstream errmsg; + errmsg << "RowAggregation: function (id = " << (uint64_t)fFunctionCols[i]->fAggFunction + << ") is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; } } } @@ -1841,147 +1753,145 @@ void RowAggregation::mergeEntries(const Row& rowIn) //------------------------------------------------------------------------------ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, bool merge) { - if (rowIn.isNullValue(colIn)) - return; + if (rowIn.isNullValue(colIn)) + return; - datatypes::SystemCatalog::ColDataType colDataType = rowIn.getColType(colIn); - long double valIn = 0; - bool isWideDataType = false; - void *wideValInPtr = nullptr; + datatypes::SystemCatalog::ColDataType colDataType = rowIn.getColType(colIn); + long double valIn = 0; + bool isWideDataType = false; + void* wideValInPtr = nullptr; - switch (colDataType) + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - valIn = rowIn.getIntField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - valIn = rowIn.getUintField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - uint32_t width = fRowGroupIn.getColumnWidth(colIn); - isWideDataType = width == datatypes::MAXDECIMALWIDTH; - if(LIKELY(isWideDataType)) - { - int128_t* dec = rowIn.getBinaryField(colIn); - wideValInPtr = reinterpret_cast(dec); - } - else if (width <= datatypes::MAXLEGACYWIDTH) - { - uint32_t scale = fRowGroupIn.getScale()[colIn]; - valIn = rowIn.getScaledSInt64FieldAsXFloat(colIn, scale); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregation::doAvg(): DECIMAL bad length."); - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - valIn = rowIn.getDoubleField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - valIn = rowIn.getFloatField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - valIn = rowIn.getLongDoubleField(colIn); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregation: no average for data type: " << colDataType; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } + valIn = rowIn.getIntField(colIn); + break; } - // min(count) = 0 - uint64_t count = fRow.getUintField(colAux); - bool notFirstValue = count > 0; - - // Set count column - if (merge) + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: { - fRow.setUintField<8>(count + rowIn.getUintField<8>(colAux), colAux); + valIn = rowIn.getUintField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + uint32_t width = fRowGroupIn.getColumnWidth(colIn); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if (LIKELY(isWideDataType)) + { + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + uint32_t scale = fRowGroupIn.getScale()[colIn]; + valIn = rowIn.getScaledSInt64FieldAsXFloat(colIn, scale); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doAvg(): DECIMAL bad length."); + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + valIn = rowIn.getDoubleField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + valIn = rowIn.getFloatField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + valIn = rowIn.getLongDoubleField(colIn); + break; + } + + default: + { + std::ostringstream errmsg; + errmsg << "RowAggregation: no average for data type: " << colDataType; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } + } + + // min(count) = 0 + uint64_t count = fRow.getUintField(colAux); + bool notFirstValue = count > 0; + + // Set count column + if (merge) + { + fRow.setUintField<8>(count + rowIn.getUintField<8>(colAux), colAux); + } + else + { + fRow.setUintField<8>(count + 1, colAux); + } + + // Set sum column + if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(colDataType) && !isWideDataType) + { + if (LIKELY(notFirstValue)) + { + int128_t* valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + valIn; + fRow.setBinaryField(&sum, colOut); } else { - fRow.setUintField<8>(count + 1, colAux); + int128_t sum = valIn; + fRow.setBinaryField(&sum, colOut); } - - // Set sum column - if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(colDataType) && !isWideDataType) + } + else if (isWideDataType) + { + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(notFirstValue)) { - if (LIKELY(notFirstValue)) - { - int128_t *valOutPtr = fRow.getBinaryField(colOut); - int128_t sum = *valOutPtr + valIn; - fRow.setBinaryField(&sum, colOut); - } - else - { - int128_t sum = valIn; - fRow.setBinaryField(&sum, colOut); - } - - } - else if (isWideDataType) - { - int128_t* dec = reinterpret_cast(wideValInPtr); - if (LIKELY(notFirstValue)) - { - int128_t *valOutPtr = fRow.getBinaryField(colOut); - int128_t sum = *valOutPtr + *dec; - fRow.setBinaryField(&sum, colOut); - } - else - { - fRow.setBinaryField(dec, colOut); - } + int128_t* valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField(&sum, colOut); } else { - if (LIKELY(notFirstValue)) - { - long double valOut = fRow.getLongDoubleField(colOut); - fRow.setLongDoubleField(valIn + valOut, colOut); - } - else // This is the first value - fRow.setLongDoubleField(valIn, colOut); + fRow.setBinaryField(dec, colOut); } + } + else + { + if (LIKELY(notFirstValue)) + { + long double valOut = fRow.getLongDoubleField(colOut); + fRow.setLongDoubleField(valIn + valOut, colOut); + } + else // This is the first value + fRow.setLongDoubleField(valIn, colOut); + } } - //------------------------------------------------------------------------------ // Update the sum and count fields for average if input is not null. // rowIn(in) - Row to be included in aggregation. @@ -1992,409 +1902,394 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 //------------------------------------------------------------------------------ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux) { - int colDataType = (fRowGroupIn.getColTypes())[colIn]; + int colDataType = (fRowGroupIn.getColTypes())[colIn]; - if (isNull(&fRowGroupIn, rowIn, colIn) == true) - return; + if (isNull(&fRowGroupIn, rowIn, colIn) == true) + return; - long double valIn = 0.0; + long double valIn = 0.0; - switch (colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - valIn = (long double) rowIn.getIntField(colIn); - break; + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: valIn = (long double)rowIn.getIntField(colIn); break; - case execplan::CalpontSystemCatalog::DECIMAL: // handle scale later - case execplan::CalpontSystemCatalog::UDECIMAL: // handle scale later - if (LIKELY(fRowGroupIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) - { - // To save from unaligned memory - datatypes::TSInt128 val128In(rowIn.getBinaryField(colIn)); - valIn = static_cast(val128In.toTFloat128()); - } - else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) - { - valIn = (long double) rowIn.getIntField(colIn); - } - else - { - idbassert(false); - } - break; + case execplan::CalpontSystemCatalog::DECIMAL: // handle scale later + case execplan::CalpontSystemCatalog::UDECIMAL: // handle scale later + if (LIKELY(fRowGroupIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + // To save from unaligned memory + datatypes::TSInt128 val128In(rowIn.getBinaryField(colIn)); + valIn = static_cast(val128In.toTFloat128()); + } + else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + valIn = (long double)rowIn.getIntField(colIn); + } + else + { + idbassert(false); + } + break; - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - valIn = (long double) rowIn.getUintField(colIn); - break; + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: valIn = (long double)rowIn.getUintField(colIn); break; - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - valIn = (long double) rowIn.getDoubleField(colIn); - break; + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: valIn = (long double)rowIn.getDoubleField(colIn); break; - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - valIn = (long double) rowIn.getFloatField(colIn); - break; + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: valIn = (long double)rowIn.getFloatField(colIn); break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - valIn = rowIn.getLongDoubleField(colIn); - break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: valIn = rowIn.getLongDoubleField(colIn); break; - default: - std::ostringstream errmsg; - errmsg << "RowAggregation: no average for data type: " << colDataType; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } + default: + std::ostringstream errmsg; + errmsg << "RowAggregation: no average for data type: " << colDataType; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } - fRow.setDoubleField(fRow.getDoubleField(colOut) + 1.0, colOut); - fRow.setLongDoubleField(fRow.getLongDoubleField(colAux) + valIn, colAux); - fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + valIn * valIn, colAux + 1); + fRow.setDoubleField(fRow.getDoubleField(colOut) + 1.0, colOut); + fRow.setLongDoubleField(fRow.getLongDoubleField(colAux) + valIn, colAux); + fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + valIn * valIn, colAux + 1); } void RowAggregation::mergeStatistics(const Row& rowIn, uint64_t colOut, uint64_t colAux) { fRow.setDoubleField(fRow.getDoubleField(colOut) + rowIn.getDoubleField(colOut), colOut); fRow.setLongDoubleField(fRow.getLongDoubleField(colAux) + rowIn.getLongDoubleField(colAux), colAux); - fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + rowIn.getLongDoubleField(colAux + 1), colAux + 1); + fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + rowIn.getLongDoubleField(colAux + 1), + colAux + 1); } -void RowAggregation::doUDAF(const Row& rowIn, - int64_t colIn, - int64_t colOut, - int64_t colAux, - uint64_t& funcColsIdx, - std::vector* rgContextColl) +void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, + uint64_t& funcColsIdx, std::vector* rgContextColl) { - std::vector* udafContextsCollPtr = &fRGContextColl; - if (UNLIKELY(rgContextColl != nullptr)) + std::vector* udafContextsCollPtr = &fRGContextColl; + if (UNLIKELY(rgContextColl != nullptr)) + { + udafContextsCollPtr = rgContextColl; + } + + std::vector& udafContextsColl = *udafContextsCollPtr; + uint32_t paramCount = udafContextsColl[funcColsIdx].getParameterCount(); + // doUDAF changes funcColsIdx to skip UDAF arguments so the real UDAF + // column idx is the initial value of the funcColsIdx + uint64_t origFuncColsIdx = funcColsIdx; + // The vector of parameters to be sent to the UDAF + utils::VLArray valsIn(paramCount); + utils::VLArray dataFlags(paramCount); + execplan::ConstantColumn* cc; + bool bIsNull = false; + execplan::CalpontSystemCatalog::ColDataType colDataType; + + for (uint32_t i = 0; i < paramCount; ++i) + { + mcsv1sdk::ColumnDatum& datum = valsIn[i]; + // Turn on NULL flags based on the data + dataFlags[i] = 0; + + // If this particular parameter is a constant, then we need + // to access the constant value rather than a row value. + cc = nullptr; + + if (fFunctionCols[funcColsIdx]->fpConstCol) { - udafContextsCollPtr = rgContextColl; + cc = dynamic_cast(fFunctionCols[funcColsIdx]->fpConstCol.get()); } - std::vector& udafContextsColl = *udafContextsCollPtr; - uint32_t paramCount = udafContextsColl[funcColsIdx].getParameterCount(); - // doUDAF changes funcColsIdx to skip UDAF arguments so the real UDAF - // column idx is the initial value of the funcColsIdx - uint64_t origFuncColsIdx = funcColsIdx; - // The vector of parameters to be sent to the UDAF - utils::VLArray valsIn(paramCount); - utils::VLArray dataFlags(paramCount); - execplan::ConstantColumn* cc; - bool bIsNull = false; - execplan::CalpontSystemCatalog::ColDataType colDataType; - - for (uint32_t i = 0; i < paramCount; ++i) + if ((cc && cc->type() == execplan::ConstantColumn::NULLDATA) || + (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) { - mcsv1sdk::ColumnDatum& datum = valsIn[i]; - // Turn on NULL flags based on the data - dataFlags[i] = 0; + if (udafContextsColl[origFuncColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + { + // When Ignore nulls, if there are multiple parameters and any + // one of them is NULL, we ignore the entry. We need to increment + // funcColsIdx the number of extra parameters. + funcColsIdx += paramCount - i - 1; + return; + } - // If this particular parameter is a constant, then we need - // to access the constant value rather than a row value. - cc = nullptr; + dataFlags[i] |= mcsv1sdk::PARAM_IS_NULL; + } - if (fFunctionCols[funcColsIdx]->fpConstCol) + if (cc) + { + colDataType = cc->resultType().colDataType; + } + else + { + colDataType = fRowGroupIn.getColTypes()[colIn]; + } + + if (!(dataFlags[i] & mcsv1sdk::PARAM_IS_NULL)) + { + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - cc = dynamic_cast(fFunctionCols[funcColsIdx]->fpConstCol.get()); + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + + if (cc) + { + datum.columnData = cc->getIntVal(const_cast(rowIn), bIsNull); + datum.scale = cc->resultType().scale; + datum.precision = cc->resultType().precision; + } + else + { + datum.columnData = rowIn.getIntField(colIn); + datum.scale = fRowGroupIn.getScale()[colIn]; + datum.precision = fRowGroupIn.getPrecision()[colIn]; + } + break; } - if ((cc && cc->type() == execplan::ConstantColumn::NULLDATA) - || (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (udafContextsColl[origFuncColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + datum.dataType = colDataType; + + if (cc) + { + datum.columnData = cc->getDecimalVal(const_cast(rowIn), bIsNull).value; + datum.scale = cc->resultType().scale; + datum.precision = cc->resultType().precision; + } + else + { + if (LIKELY(fRowGroupIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) { - // When Ignore nulls, if there are multiple parameters and any - // one of them is NULL, we ignore the entry. We need to increment - // funcColsIdx the number of extra parameters. - funcColsIdx += paramCount - i - 1; - return; + // We can't control boost::any asignment + // so let's get an aligned memory + datatypes::TSInt128 val = rowIn.getTSInt128Field(colIn); + datum.columnData = val.s128Value; } - - dataFlags[i] |= mcsv1sdk::PARAM_IS_NULL; - } - - if (cc) - { - colDataType = cc->resultType().colDataType; - } - else - { - colDataType = fRowGroupIn.getColTypes()[colIn]; - } - - if (!(dataFlags[i] & mcsv1sdk::PARAM_IS_NULL)) - { - switch (colDataType) + else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - datum.dataType = execplan::CalpontSystemCatalog::BIGINT; - - if (cc) - { - datum.columnData = cc->getIntVal(const_cast(rowIn), bIsNull); - datum.scale = cc->resultType().scale; - datum.precision = cc->resultType().precision; - } - else - { - datum.columnData = rowIn.getIntField(colIn); - datum.scale = fRowGroupIn.getScale()[colIn]; - datum.precision = fRowGroupIn.getPrecision()[colIn]; - } - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - datum.dataType = colDataType; - - if (cc) - { - datum.columnData = cc->getDecimalVal(const_cast(rowIn), bIsNull).value; - datum.scale = cc->resultType().scale; - datum.precision = cc->resultType().precision; - } - else - { - if (LIKELY(fRowGroupIn.getColumnWidth(colIn) - == datatypes::MAXDECIMALWIDTH)) - { - // We can't control boost::any asignment - // so let's get an aligned memory - datatypes::TSInt128 val = rowIn.getTSInt128Field(colIn); - datum.columnData = val.s128Value; - } - else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) - { - datum.columnData = rowIn.getIntField(colIn); - } - else - { - idbassert(false); - } - datum.scale = fRowGroupIn.getScale()[colIn]; - datum.precision = fRowGroupIn.getPrecision()[colIn]; - } - - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; - - if (cc) - { - datum.columnData = cc->getUintVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getUintField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - datum.dataType = execplan::CalpontSystemCatalog::DOUBLE; - - if (cc) - { - datum.columnData = cc->getDoubleVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getDoubleField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - datum.dataType = execplan::CalpontSystemCatalog::LONGDOUBLE; - - if (cc) - { - datum.columnData = cc->getLongDoubleVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getLongDoubleField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - datum.dataType = execplan::CalpontSystemCatalog::FLOAT; - - if (cc) - { - datum.columnData = cc->getFloatVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getFloatField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; - - if (cc) - { - datum.columnData = cc->getDateIntVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getUintField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; - - if (cc) - { - datum.columnData = cc->getDatetimeIntVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getUintField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; - - if (cc) - { - datum.columnData = cc->getTimestampIntVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getUintField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - datum.dataType = execplan::CalpontSystemCatalog::BIGINT; - - if (cc) - { - datum.columnData = cc->getTimeIntVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getIntField(colIn); - } - - break; - } - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::CLOB: - case execplan::CalpontSystemCatalog::BLOB: - { - datum.dataType = colDataType; - - if (cc) - { - datum.columnData = cc->getStrVal(const_cast(rowIn), bIsNull); - } - else - { - datum.columnData = rowIn.getStringField(colIn); - } - - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregation " << udafContextsColl[origFuncColsIdx].getName() << - ": No logic for data type: " << colDataType; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } + datum.columnData = rowIn.getIntField(colIn); } + else + { + idbassert(false); + } + datum.scale = fRowGroupIn.getScale()[colIn]; + datum.precision = fRowGroupIn.getPrecision()[colIn]; + } + + break; } - // MCOL-1201: If there are multiple parameters, the next fFunctionCols - // will have the column used. By incrementing the funcColsIdx (passed by - // ref, we also increment the caller's index. - if (fFunctionCols.size() > funcColsIdx + 1 - && fFunctionCols[funcColsIdx + 1]->fAggFunction == ROWAGG_MULTI_PARM) + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: { - ++funcColsIdx; - colIn = fFunctionCols[funcColsIdx]->fInputColumnIndex; - colOut = fFunctionCols[funcColsIdx]->fOutputColumnIndex; - (void)colOut; + datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + + if (cc) + { + datum.columnData = cc->getUintVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getUintField(colIn); + } + + break; } - else + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: { - break; + datum.dataType = execplan::CalpontSystemCatalog::DOUBLE; + + if (cc) + { + datum.columnData = cc->getDoubleVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getDoubleField(colIn); + } + + break; } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + datum.dataType = execplan::CalpontSystemCatalog::LONGDOUBLE; + + if (cc) + { + datum.columnData = cc->getLongDoubleVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getLongDoubleField(colIn); + } + + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + datum.dataType = execplan::CalpontSystemCatalog::FLOAT; + + if (cc) + { + datum.columnData = cc->getFloatVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getFloatField(colIn); + } + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + + if (cc) + { + datum.columnData = cc->getDateIntVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getUintField(colIn); + } + + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + + if (cc) + { + datum.columnData = cc->getDatetimeIntVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getUintField(colIn); + } + + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + + if (cc) + { + datum.columnData = cc->getTimestampIntVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getUintField(colIn); + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + + if (cc) + { + datum.columnData = cc->getTimeIntVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getIntField(colIn); + } + + break; + } + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::CLOB: + case execplan::CalpontSystemCatalog::BLOB: + { + datum.dataType = colDataType; + + if (cc) + { + datum.columnData = cc->getStrVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getStringField(colIn); + } + + break; + } + + default: + { + std::ostringstream errmsg; + errmsg << "RowAggregation " << udafContextsColl[origFuncColsIdx].getName() + << ": No logic for data type: " << colDataType; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } + } } - // The intermediate values are stored in userData referenced by colAux. - udafContextsColl[origFuncColsIdx].setDataFlags(dataFlags); - udafContextsColl[origFuncColsIdx].setUserData(fRow.getUserData(colAux)); - - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = udafContextsColl[origFuncColsIdx].getFunction()->nextValue(&udafContextsColl[origFuncColsIdx], - valsIn); - udafContextsColl[origFuncColsIdx].setUserData(NULL); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + // MCOL-1201: If there are multiple parameters, the next fFunctionCols + // will have the column used. By incrementing the funcColsIdx (passed by + // ref, we also increment the caller's index. + if (fFunctionCols.size() > funcColsIdx + 1 && + fFunctionCols[funcColsIdx + 1]->fAggFunction == ROWAGG_MULTI_PARM) { - RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[origFuncColsIdx].get()); - rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(udafContextsColl[origFuncColsIdx].getErrorMessage(), - logging::aggregateFuncErr); + ++funcColsIdx; + colIn = fFunctionCols[funcColsIdx]->fInputColumnIndex; + colOut = fFunctionCols[funcColsIdx]->fOutputColumnIndex; + (void)colOut; } + else + { + break; + } + } + + // The intermediate values are stored in userData referenced by colAux. + udafContextsColl[origFuncColsIdx].setDataFlags(dataFlags); + udafContextsColl[origFuncColsIdx].setUserData(fRow.getUserData(colAux)); + + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + rc = udafContextsColl[origFuncColsIdx].getFunction()->nextValue(&udafContextsColl[origFuncColsIdx], valsIn); + udafContextsColl[origFuncColsIdx].setUserData(NULL); + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[origFuncColsIdx].get()); + rowUDAF->bInterrupted = true; + throw logging::QueryDataExcept(udafContextsColl[origFuncColsIdx].getErrorMessage(), + logging::aggregateFuncErr); + } } //------------------------------------------------------------------------------ @@ -2427,11 +2322,10 @@ void RowAggregation::loadResult(messageqcpp::ByteStream& bs) bs.append(rgdbs.buf(), rgdbs.length()); } - void RowAggregation::loadEmptySet(messageqcpp::ByteStream& bs) { - bs << (uint32_t) 1; - fEmptyRowGroup.serializeRGData(bs); + bs << (uint32_t)1; + fEmptyRowGroup.serializeRGData(bs); } //------------------------------------------------------------------------------ @@ -2439,60 +2333,56 @@ void RowAggregation::loadEmptySet(messageqcpp::ByteStream& bs) // For one-phase case, from projected RG to final aggregated RG //------------------------------------------------------------------------------ RowAggregationUM::RowAggregationUM(const vector& rowAggGroupByCols, - const vector& rowAggFunctionCols, + const vector& rowAggFunctionCols, joblist::ResourceManager* r, boost::shared_ptr sessionLimit) - : RowAggregation(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) - , fHasAvg(false) - , fHasStatsFunc(false) - , fHasUDAF(false) - , fTotalMemUsage(0) - , fLastMemUsage(0) + : RowAggregation(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) + , fHasAvg(false) + , fHasStatsFunc(false) + , fHasUDAF(false) + , fTotalMemUsage(0) + , fLastMemUsage(0) { - // Check if there are any avg, stats or UDAF functions. - // These flags are used in finalize. - for (uint64_t i = 0; i < fFunctionCols.size(); i++) - { - if (fFunctionCols[i]->fAggFunction == ROWAGG_AVG || - fFunctionCols[i]->fAggFunction == ROWAGG_DISTINCT_AVG) - fHasAvg = true; - else if (fFunctionCols[i]->fAggFunction == ROWAGG_STATS) - fHasStatsFunc = true; - else if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) - fHasUDAF = true; - } + // Check if there are any avg, stats or UDAF functions. + // These flags are used in finalize. + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_AVG || fFunctionCols[i]->fAggFunction == ROWAGG_DISTINCT_AVG) + fHasAvg = true; + else if (fFunctionCols[i]->fAggFunction == ROWAGG_STATS) + fHasStatsFunc = true; + else if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + fHasUDAF = true; + } - // Check if all groupby column selected - for (uint64_t i = 0; i < fGroupByCols.size(); i++) + // Check if all groupby column selected + for (uint64_t i = 0; i < fGroupByCols.size(); i++) + { + if (fGroupByCols[i]->fInputColumnIndex != fGroupByCols[i]->fOutputColumnIndex) { - if (fGroupByCols[i]->fInputColumnIndex != fGroupByCols[i]->fOutputColumnIndex) - { - fKeyOnHeap = true; - break; - } + fKeyOnHeap = true; + break; } + } } - -RowAggregationUM::RowAggregationUM(const RowAggregationUM& rhs) : - RowAggregation(rhs), - fHasAvg(rhs.fHasAvg), - fHasStatsFunc(rhs.fHasStatsFunc), - fHasUDAF(rhs.fHasUDAF), - fExpression(rhs.fExpression), - fTotalMemUsage(rhs.fTotalMemUsage), - fConstantAggregate(rhs.fConstantAggregate), - fGroupConcat(rhs.fGroupConcat), - fLastMemUsage(rhs.fLastMemUsage) +RowAggregationUM::RowAggregationUM(const RowAggregationUM& rhs) + : RowAggregation(rhs) + , fHasAvg(rhs.fHasAvg) + , fHasStatsFunc(rhs.fHasStatsFunc) + , fHasUDAF(rhs.fHasUDAF) + , fExpression(rhs.fExpression) + , fTotalMemUsage(rhs.fTotalMemUsage) + , fConstantAggregate(rhs.fConstantAggregate) + , fGroupConcat(rhs.fGroupConcat) + , fLastMemUsage(rhs.fLastMemUsage) { } - RowAggregationUM::~RowAggregationUM() { - fRm->returnMemory(fTotalMemUsage, fSessionMemLimit); + fRm->returnMemory(fTotalMemUsage, fSessionMemLimit); } - //------------------------------------------------------------------------------ // Marks the end of RowGroup input into aggregation. // @@ -2502,24 +2392,22 @@ void RowAggregationUM::endOfInput() { } - //------------------------------------------------------------------------------ // Initilalize the Group Concat data //------------------------------------------------------------------------------ void RowAggregationUM::initialize() { - if (fGroupConcat.size() > 0) - fFunctionColGc = fFunctionCols; + if (fGroupConcat.size() > 0) + fFunctionColGc = fFunctionCols; - if (fKeyOnHeap) - { - fKeyRG = fRowGroupIn.truncate(fGroupByCols.size()); - } + if (fKeyOnHeap) + { + fKeyRG = fRowGroupIn.truncate(fGroupByCols.size()); + } - RowAggregation::initialize(); + RowAggregation::initialize(); } - //------------------------------------------------------------------------------ // Aggregation finalization can be performed here. For example, this is // where fixing the duplicates and dividing the SUM by COUNT to get the AVG. @@ -2528,687 +2416,642 @@ void RowAggregationUM::initialize() //------------------------------------------------------------------------------ void RowAggregationUM::finalize() { - // copy the duplicates functions, except AVG - fixDuplicates(ROWAGG_DUP_FUNCT); + // copy the duplicates functions, except AVG + fixDuplicates(ROWAGG_DUP_FUNCT); - // UM: it is time to divide SUM by COUNT for any AVG cols. - if (fHasAvg) - { - calculateAvgColumns(); + // UM: it is time to divide SUM by COUNT for any AVG cols. + if (fHasAvg) + { + calculateAvgColumns(); - // copy the duplicate AVGs, if any - fixDuplicates(ROWAGG_DUP_AVG); - } + // copy the duplicate AVGs, if any + fixDuplicates(ROWAGG_DUP_AVG); + } - // UM: it is time to calculate statistics functions - if (fHasStatsFunc) - { - // covers duplicats, too. - calculateStatisticsFunctions(); - } + // UM: it is time to calculate statistics functions + if (fHasStatsFunc) + { + // covers duplicats, too. + calculateStatisticsFunctions(); + } - if (fHasUDAF) - { - calculateUDAFColumns(); - // copy the duplicate UDAF, if any - fixDuplicates(ROWAGG_DUP_UDAF); - } + if (fHasUDAF) + { + calculateUDAFColumns(); + // copy the duplicate UDAF, if any + fixDuplicates(ROWAGG_DUP_UDAF); + } - if (fGroupConcat.size() > 0) - setGroupConcatString(); + if (fGroupConcat.size() > 0) + setGroupConcatString(); - if (fConstantAggregate.size() > 0) - fixConstantAggregate(); + if (fConstantAggregate.size() > 0) + fixConstantAggregate(); - if (fExpression.size() > 0) - evaluateExpression(); + if (fExpression.size() > 0) + evaluateExpression(); } - //------------------------------------------------------------------------------ // Add group_concat to the initialized working row //------------------------------------------------------------------------------ void RowAggregationUM::attachGroupConcatAg() { - if (fGroupConcat.size() > 0) + if (fGroupConcat.size() > 0) + { + uint8_t* data = fRow.getData(); + uint64_t i = 0, j = 0; + + for (; i < fFunctionColGc.size(); i++) { - uint8_t* data = fRow.getData(); - uint64_t i = 0, j = 0; + int64_t colOut = fFunctionColGc[i]->fOutputColumnIndex; - for (; i < fFunctionColGc.size(); i++) - { - int64_t colOut = fFunctionColGc[i]->fOutputColumnIndex; - - if (fFunctionColGc[i]->fAggFunction == ROWAGG_GROUP_CONCAT) - { - // save the object's address in the result row - SP_GroupConcatAg gcc(new joblist::GroupConcatAgUM(fGroupConcat[j++])); - fGroupConcatAg.push_back(gcc); - *((GroupConcatAg**)(data + fRow.getOffset(colOut))) = gcc.get(); - } - } + if (fFunctionColGc[i]->fAggFunction == ROWAGG_GROUP_CONCAT) + { + // save the object's address in the result row + SP_GroupConcatAg gcc(new joblist::GroupConcatAgUM(fGroupConcat[j++])); + fGroupConcatAg.push_back(gcc); + *((GroupConcatAg**)(data + fRow.getOffset(colOut))) = gcc.get(); + } } + } } - //------------------------------------------------------------------------------ // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for count // rowIn(in) - Row to be included in aggregation. // rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationUM::updateEntry(const Row& rowIn, - std::vector* rgContextColl) +void RowAggregationUM::updateEntry(const Row& rowIn, std::vector* rgContextColl) { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + int64_t colIn = fFunctionCols[i]->fInputColumnIndex; + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + + switch (fFunctionCols[i]->fAggFunction) { - int64_t colIn = fFunctionCols[i]->fInputColumnIndex; - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + case ROWAGG_COUNT_COL_NAME: - switch (fFunctionCols[i]->fAggFunction) - { - case ROWAGG_COUNT_COL_NAME: + // if NOT null, let execution fall through. + if (isNull(&fRowGroupIn, rowIn, colIn) == true) + break; + /* fall through */ - // if NOT null, let execution fall through. - if (isNull(&fRowGroupIn, rowIn, colIn) == true) break; - /* fall through */ + case ROWAGG_COUNT_ASTERISK: fRow.setUintField<8>(fRow.getUintField<8>(colOut) + 1, colOut); break; - case ROWAGG_COUNT_ASTERISK: - fRow.setUintField<8>(fRow.getUintField<8>(colOut) + 1, colOut); - break; + case ROWAGG_MIN: + case ROWAGG_MAX: doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_MIN: - case ROWAGG_MAX: - doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_SUM: doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_SUM: - doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_AVG: + { + // The sum and count on UM may not be put next to each other: + // use colOut to store the sum; + // use colAux to store the count. + doAvg(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_AVG: - { - // The sum and count on UM may not be put next to each other: - // use colOut to store the sum; - // use colAux to store the count. - doAvg(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_STATS: + { + doStatistics(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_STATS: - { - doStatistics(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); + break; + } - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; - } + case ROWAGG_GROUP_CONCAT: + { + doGroupConcat(rowIn, colIn, colOut); + break; + } - case ROWAGG_GROUP_CONCAT: - { - doGroupConcat(rowIn, colIn, colOut); - break; - } + case ROWAGG_COUNT_NO_OP: + case ROWAGG_DUP_FUNCT: + case ROWAGG_DUP_AVG: + case ROWAGG_DUP_STATS: + case ROWAGG_DUP_UDAF: + case ROWAGG_CONSTANT: break; - case ROWAGG_COUNT_NO_OP: - case ROWAGG_DUP_FUNCT: - case ROWAGG_DUP_AVG: - case ROWAGG_DUP_STATS: - case ROWAGG_DUP_UDAF: - case ROWAGG_CONSTANT: - break; + case ROWAGG_UDAF: + { + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); + break; + } - case ROWAGG_UDAF: - { - doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); - break; - } - - default: - { - // need a exception to show the value - std::ostringstream errmsg; - errmsg << "RowAggregationUM: function (id = " << - (uint64_t) fFunctionCols[i]->fAggFunction << ") is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - } + default: + { + // need a exception to show the value + std::ostringstream errmsg; + errmsg << "RowAggregationUM: function (id = " << (uint64_t)fFunctionCols[i]->fAggFunction + << ") is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } } + } } - //------------------------------------------------------------------------------ // Concat columns. // rowIn(in) - Row that contains the columns to be concatenated. //------------------------------------------------------------------------------ void RowAggregationUM::doGroupConcat(const Row& rowIn, int64_t, int64_t o) { - uint8_t* data = fRow.getData(); - joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + fRow.getOffset(o))); - gccAg->processRow(rowIn); + uint8_t* data = fRow.getData(); + joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + fRow.getOffset(o))); + gccAg->processRow(rowIn); } - //------------------------------------------------------------------------------ // After all PM rowgroups received, calculate the average value. //------------------------------------------------------------------------------ void RowAggregationUM::calculateAvgColumns() { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_AVG || fFunctionCols[i]->fAggFunction == ROWAGG_DISTINCT_AVG) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_AVG || - fFunctionCols[i]->fAggFunction == ROWAGG_DISTINCT_AVG) + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + + for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++) + { + fRowGroupOut->getRow(j, &fRow); + uint64_t cnt = fRow.getIntField(colAux); + + if (cnt == 0) // empty set, value is initialized to null. + continue; + + uint32_t precision = fRow.getPrecision(colOut); + bool isWideDecimal = datatypes::Decimal::isWideDecimalTypeByPrecision(precision); + + if (!isWideDecimal) { - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; - - for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++) - { - fRowGroupOut->getRow(j, &fRow); - uint64_t cnt = fRow.getIntField(colAux); - - if (cnt == 0) // empty set, value is initialized to null. - continue; - - uint32_t precision = fRow.getPrecision(colOut); - bool isWideDecimal = - datatypes::Decimal::isWideDecimalTypeByPrecision(precision); - - if (!isWideDecimal) - { - long double sum = 0.0; - long double avg = 0.0; - sum = fRow.getLongDoubleField(colOut); - avg = sum / cnt; - fRow.setLongDoubleField(avg, colOut); - } - else - { - uint32_t offset = fRow.getOffset(colOut); - uint32_t scale = fRow.getScale(colOut); - // Get multiplied to deliver AVG with the scale closest - // to the expected original scale + 4. - // There is a counterpart in buildAggregateColumn. - datatypes::Decimal::setScalePrecision4Avg(precision, scale); - int128_t* sumPnt = fRow.getBinaryField_offset(offset); - uint32_t scaleDiff = scale - fRow.getScale(colOut); - // multiplication overflow check - datatypes::MultiplicationOverflowCheck multOp; - int128_t sum = 0; - if (scaleDiff > 0) - multOp(*sumPnt, datatypes::mcs_pow_10[scaleDiff], sum); - else - sum = *sumPnt; - datatypes::lldiv_t_128 avgAndRem = datatypes::lldiv128(sum, cnt); - // Round the last digit - if (datatypes::abs(avgAndRem.rem) * 2 >= (int128_t)cnt) - { - if (utils::is_negative(avgAndRem.rem)) - { - avgAndRem.quot--; - } - else - { - avgAndRem.quot++; - } - } - fRow.setBinaryField_offset(&avgAndRem.quot, - sizeof(avgAndRem.quot), - offset); - } - } + long double sum = 0.0; + long double avg = 0.0; + sum = fRow.getLongDoubleField(colOut); + avg = sum / cnt; + fRow.setLongDoubleField(avg, colOut); } + else + { + uint32_t offset = fRow.getOffset(colOut); + uint32_t scale = fRow.getScale(colOut); + // Get multiplied to deliver AVG with the scale closest + // to the expected original scale + 4. + // There is a counterpart in buildAggregateColumn. + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + int128_t* sumPnt = fRow.getBinaryField_offset(offset); + uint32_t scaleDiff = scale - fRow.getScale(colOut); + // multiplication overflow check + datatypes::MultiplicationOverflowCheck multOp; + int128_t sum = 0; + if (scaleDiff > 0) + multOp(*sumPnt, datatypes::mcs_pow_10[scaleDiff], sum); + else + sum = *sumPnt; + datatypes::lldiv_t_128 avgAndRem = datatypes::lldiv128(sum, cnt); + // Round the last digit + if (datatypes::abs(avgAndRem.rem) * 2 >= (int128_t)cnt) + { + if (utils::is_negative(avgAndRem.rem)) + { + avgAndRem.quot--; + } + else + { + avgAndRem.quot++; + } + } + fRow.setBinaryField_offset(&avgAndRem.quot, sizeof(avgAndRem.quot), offset); + } + } } + } } // Sets the value from valOut into column colOut, performing any conversions. void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) { - execplan::CalpontSystemCatalog::ColDataType colDataType = fRowGroupOut->getColTypes()[colOut]; + execplan::CalpontSystemCatalog::ColDataType colDataType = fRowGroupOut->getColTypes()[colOut]; - if (valOut.empty()) + if (valOut.empty()) + { + // Fields are initialized to NULL, which is what we want for empty; + return; + } + + int64_t intOut; + uint64_t uintOut; + float floatOut; + double doubleOut; + long double longdoubleOut; + ostringstream oss; + std::string strOut; + + bool bSetSuccess = false; + + switch (colDataType) + { + case execplan::CalpontSystemCatalog::BIT: + case execplan::CalpontSystemCatalog::TINYINT: + if (valOut.compatible(charTypeId)) + { + intOut = valOut.cast(); + bSetSuccess = true; + } + else if (valOut.compatible(scharTypeId)) + { + intOut = valOut.cast(); + bSetSuccess = true; + } + + if (bSetSuccess) + { + fRow.setIntField<1>(intOut, colOut); + } + + break; + + case execplan::CalpontSystemCatalog::SMALLINT: + if (valOut.compatible(shortTypeId)) + { + intOut = valOut.cast(); + fRow.setIntField<2>(intOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + if (valOut.compatible(intTypeId)) + { + intOut = valOut.cast(); + bSetSuccess = true; + } + else if (valOut.compatible(longTypeId)) + { + intOut = valOut.cast(); + bSetSuccess = true; + } + + if (bSetSuccess) + { + fRow.setIntField<4>(intOut, colOut); + } + + break; + + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + if (valOut.compatible(llTypeId)) + { + intOut = valOut.cast(); + fRow.setIntField<8>(intOut, colOut); + bSetSuccess = true; + } + else if (valOut.compatible(int128TypeId)) + { + int128_t int128Out = valOut.cast(); + fRow.setInt128Field(int128Out, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + if (valOut.compatible(ucharTypeId)) + { + uintOut = valOut.cast(); + fRow.setUintField<1>(uintOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::USMALLINT: + if (valOut.compatible(ushortTypeId)) + { + uintOut = valOut.cast(); + fRow.setUintField<2>(uintOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + if (valOut.compatible(uintTypeId)) + { + uintOut = valOut.cast(); + fRow.setUintField<4>(uintOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + if (valOut.compatible(ulongTypeId)) + { + uintOut = valOut.cast(); + fRow.setUintField<8>(uintOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + if (valOut.compatible(ulongTypeId)) + { + uintOut = valOut.cast(); + fRow.setUintField<8>(uintOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + if (valOut.compatible(floatTypeId)) + { + floatOut = valOut.cast(); + fRow.setFloatField(floatOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + if (valOut.compatible(doubleTypeId)) + { + doubleOut = valOut.cast(); + fRow.setDoubleField(doubleOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + if (valOut.compatible(strTypeId)) + { + strOut = valOut.cast(); + fRow.setStringField(strOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::CLOB: + case execplan::CalpontSystemCatalog::BLOB: + if (valOut.compatible(strTypeId)) + { + strOut = valOut.cast(); + fRow.setVarBinaryField(strOut, colOut); + bSetSuccess = true; + } + + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + if (valOut.compatible(doubleTypeId)) + { + longdoubleOut = valOut.cast(); + fRow.setLongDoubleField(longdoubleOut, colOut); + bSetSuccess = true; + } + + break; + + default: { - // Fields are initialized to NULL, which is what we want for empty; - return; + std::ostringstream errmsg; + errmsg << "RowAggregation: No logic for data type: " << colDataType; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; } + } - int64_t intOut; - uint64_t uintOut; - float floatOut; - double doubleOut; - long double longdoubleOut; - ostringstream oss; - std::string strOut; - - bool bSetSuccess = false; - - switch (colDataType) - { - case execplan::CalpontSystemCatalog::BIT: - case execplan::CalpontSystemCatalog::TINYINT: - if (valOut.compatible(charTypeId)) - { - intOut = valOut.cast(); - bSetSuccess = true; - } - else if (valOut.compatible(scharTypeId)) - { - intOut = valOut.cast(); - bSetSuccess = true; - } - - if (bSetSuccess) - { - fRow.setIntField<1>(intOut, colOut); - } - - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - if (valOut.compatible(shortTypeId)) - { - intOut = valOut.cast(); - fRow.setIntField<2>(intOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - if (valOut.compatible(intTypeId)) - { - intOut = valOut.cast(); - bSetSuccess = true; - } - else if (valOut.compatible(longTypeId)) - { - intOut = valOut.cast(); - bSetSuccess = true; - } - - if (bSetSuccess) - { - fRow.setIntField<4>(intOut, colOut); - } - - break; - - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - if (valOut.compatible(llTypeId)) - { - intOut = valOut.cast(); - fRow.setIntField<8>(intOut, colOut); - bSetSuccess = true; - } - else if (valOut.compatible(int128TypeId)) - { - int128_t int128Out = valOut.cast(); - fRow.setInt128Field(int128Out, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - if (valOut.compatible(ucharTypeId)) - { - uintOut = valOut.cast(); - fRow.setUintField<1>(uintOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - if (valOut.compatible(ushortTypeId)) - { - uintOut = valOut.cast(); - fRow.setUintField<2>(uintOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - if (valOut.compatible(uintTypeId)) - { - uintOut = valOut.cast(); - fRow.setUintField<4>(uintOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - if (valOut.compatible(ulongTypeId)) - { - uintOut = valOut.cast(); - fRow.setUintField<8>(uintOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - if (valOut.compatible(ulongTypeId)) - { - uintOut = valOut.cast(); - fRow.setUintField<8>(uintOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - if (valOut.compatible(floatTypeId)) - { - floatOut = valOut.cast(); - fRow.setFloatField(floatOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - if (valOut.compatible(doubleTypeId)) - { - doubleOut = valOut.cast(); - fRow.setDoubleField(doubleOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - if (valOut.compatible(strTypeId)) - { - strOut = valOut.cast(); - fRow.setStringField(strOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::CLOB: - case execplan::CalpontSystemCatalog::BLOB: - if (valOut.compatible(strTypeId)) - { - strOut = valOut.cast(); - fRow.setVarBinaryField(strOut, colOut); - bSetSuccess = true; - } - - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - if (valOut.compatible(doubleTypeId)) - { - longdoubleOut = valOut.cast(); - fRow.setLongDoubleField(longdoubleOut, colOut); - bSetSuccess = true; - } - - break; - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregation: No logic for data type: " << colDataType; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - } - - if (!bSetSuccess) - { - // This means the return from the UDAF doesn't match the field - // This handles the mismatch - SetUDAFAnyValue(valOut, colOut); - } - // reset valOut to be ready for the next value - valOut.reset(); + if (!bSetSuccess) + { + // This means the return from the UDAF doesn't match the field + // This handles the mismatch + SetUDAFAnyValue(valOut, colOut); + } + // reset valOut to be ready for the next value + valOut.reset(); } void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) { - execplan::CalpontSystemCatalog::ColDataType colDataType = fRowGroupOut->getColTypes()[colOut]; + execplan::CalpontSystemCatalog::ColDataType colDataType = fRowGroupOut->getColTypes()[colOut]; - // This may seem a bit convoluted. Users shouldn't return a type - // that they didn't set in mcsv1_UDAF::init(), but this - // handles whatever return type is given and casts - // it to whatever they said to return. - // TODO: Save cpu cycles here. For one, we don't need to initialize these + // This may seem a bit convoluted. Users shouldn't return a type + // that they didn't set in mcsv1_UDAF::init(), but this + // handles whatever return type is given and casts + // it to whatever they said to return. + // TODO: Save cpu cycles here. For one, we don't need to initialize these - int64_t intOut = 0; - uint64_t uintOut = 0; - double doubleOut = 0.0; - long double longdoubleOut = 0.0; - int128_t int128Out = 0; - ostringstream oss; - std::string strOut; + int64_t intOut = 0; + uint64_t uintOut = 0; + double doubleOut = 0.0; + long double longdoubleOut = 0.0; + int128_t int128Out = 0; + ostringstream oss; + std::string strOut; - if (valOut.compatible(charTypeId)) + if (valOut.compatible(charTypeId)) + { + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; + oss << intOut; + } + else if (valOut.compatible(scharTypeId)) + { + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; + oss << intOut; + } + else if (valOut.compatible(shortTypeId)) + { + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; + oss << intOut; + } + else if (valOut.compatible(intTypeId)) + { + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; + oss << intOut; + } + else if (valOut.compatible(longTypeId)) + { + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; + oss << intOut; + } + else if (valOut.compatible(llTypeId)) + { + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; + oss << intOut; + } + else if (valOut.compatible(ucharTypeId)) + { + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(ushortTypeId)) + { + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(uintTypeId)) + { + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(ulongTypeId)) + { + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(ullTypeId)) + { + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(int128TypeId)) + { + intOut = uintOut = int128Out = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(floatTypeId) || valOut.compatible(doubleTypeId)) + { + // Should look at scale for decimal and adjust + doubleOut = valOut.cast(); + int128Out = doubleOut; + intOut = uintOut = doubleOut; + oss << doubleOut; + } + else if (valOut.compatible(longdoubleTypeId)) + { + // Should look at scale for decimal and adjust + longdoubleOut = valOut.cast(); + int128Out = longdoubleOut; + doubleOut = (double)longdoubleOut; + uintOut = (uint64_t)doubleOut; + intOut = (int64_t)doubleOut; + oss << doubleOut; + } + + if (valOut.compatible(strTypeId)) + { + strOut = valOut.cast(); + // Convert the string to numeric type, just in case. + intOut = atol(strOut.c_str()); + uintOut = strtoul(strOut.c_str(), nullptr, 10); + doubleOut = strtod(strOut.c_str(), nullptr); + longdoubleOut = strtold(strOut.c_str(), nullptr); + int128Out = longdoubleOut; + } + else + { + strOut = oss.str(); + } + + switch (colDataType) + { + case execplan::CalpontSystemCatalog::BIT: + case execplan::CalpontSystemCatalog::TINYINT: fRow.setIntField<1>(intOut, colOut); break; + + case execplan::CalpontSystemCatalog::SMALLINT: fRow.setIntField<2>(intOut, colOut); break; + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: fRow.setIntField<4>(intOut, colOut); break; + + case execplan::CalpontSystemCatalog::BIGINT: fRow.setIntField<8>(intOut, colOut); break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - int128Out = uintOut = intOut = valOut.cast(); - doubleOut = intOut; - oss << intOut; - } - else if (valOut.compatible(scharTypeId)) - { - int128Out = uintOut = intOut = valOut.cast(); - doubleOut = intOut; - oss << intOut; - } - else if (valOut.compatible(shortTypeId)) - { - int128Out = uintOut = intOut = valOut.cast(); - doubleOut = intOut; - oss << intOut; - } - else if (valOut.compatible(intTypeId)) - { - int128Out = uintOut = intOut = valOut.cast(); - doubleOut = intOut; - oss << intOut; - } - else if (valOut.compatible(longTypeId)) - { - int128Out = uintOut = intOut = valOut.cast(); - doubleOut = intOut; - oss << intOut; - } - else if (valOut.compatible(llTypeId)) - { - int128Out = uintOut = intOut = valOut.cast(); - doubleOut = intOut; - oss << intOut; - } - else if (valOut.compatible(ucharTypeId)) - { - int128Out = intOut = uintOut = valOut.cast(); - doubleOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(ushortTypeId)) - { - int128Out = intOut = uintOut = valOut.cast(); - doubleOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(uintTypeId)) - { - int128Out = intOut = uintOut = valOut.cast(); - doubleOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(ulongTypeId)) - { - int128Out = intOut = uintOut = valOut.cast(); - doubleOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(ullTypeId)) - { - int128Out = intOut = uintOut = valOut.cast(); - doubleOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(int128TypeId)) - { - intOut = uintOut = int128Out = valOut.cast(); - doubleOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(floatTypeId) || valOut.compatible(doubleTypeId)) - { - // Should look at scale for decimal and adjust - doubleOut = valOut.cast(); - int128Out = doubleOut; - intOut = uintOut = doubleOut; - oss << doubleOut; - } - else if (valOut.compatible(longdoubleTypeId)) - { - // Should look at scale for decimal and adjust - longdoubleOut = valOut.cast(); - int128Out = longdoubleOut; - doubleOut = (double)longdoubleOut; - uintOut = (uint64_t)doubleOut; - intOut = (int64_t)doubleOut; - oss << doubleOut; + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + fRow.setInt128Field(int128Out, colOut); + else + fRow.setIntField<8>(intOut, colOut); + break; } - if (valOut.compatible(strTypeId)) + case execplan::CalpontSystemCatalog::UTINYINT: fRow.setUintField<1>(uintOut, colOut); break; + + case execplan::CalpontSystemCatalog::USMALLINT: fRow.setUintField<2>(uintOut, colOut); break; + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: fRow.setUintField<4>(uintOut, colOut); break; + + case execplan::CalpontSystemCatalog::UBIGINT: fRow.setUintField<8>(uintOut, colOut); break; + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: fRow.setUintField<8>(uintOut, colOut); break; + + case execplan::CalpontSystemCatalog::TIME: fRow.setIntField<8>(intOut, colOut); break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: { - strOut = valOut.cast(); - // Convert the string to numeric type, just in case. - intOut = atol(strOut.c_str()); - uintOut = strtoul(strOut.c_str(), nullptr, 10); - doubleOut = strtod(strOut.c_str(), nullptr); - longdoubleOut = strtold(strOut.c_str(), nullptr); - int128Out = longdoubleOut; - } - else - { - strOut = oss.str(); + float floatOut = (float)doubleOut; + fRow.setFloatField(floatOut, colOut); + break; } - switch (colDataType) + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: fRow.setDoubleField(doubleOut, colOut); break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: fRow.setStringField(strOut, colOut); break; + + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::CLOB: + case execplan::CalpontSystemCatalog::BLOB: fRow.setVarBinaryField(strOut, colOut); break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: fRow.setLongDoubleField(longdoubleOut, colOut); break; + + default: { - case execplan::CalpontSystemCatalog::BIT: - case execplan::CalpontSystemCatalog::TINYINT: - fRow.setIntField<1>(intOut, colOut); - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - fRow.setIntField<2>(intOut, colOut); - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - fRow.setIntField<4>(intOut, colOut); - break; - - case execplan::CalpontSystemCatalog::BIGINT: - fRow.setIntField<8>(intOut, colOut); - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - uint32_t width = fRowGroupOut->getColumnWidth(colOut); - if (width == datatypes::MAXDECIMALWIDTH) - fRow.setInt128Field(int128Out, colOut); - else - fRow.setIntField<8>(intOut, colOut); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - fRow.setUintField<1>(uintOut, colOut); - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - fRow.setUintField<2>(uintOut, colOut); - break; - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - fRow.setUintField<4>(uintOut, colOut); - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - fRow.setUintField<8>(uintOut, colOut); - break; - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - - fRow.setUintField<8>(uintOut, colOut); - break; - - case execplan::CalpontSystemCatalog::TIME: - - fRow.setIntField<8>(intOut, colOut); - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float floatOut = (float)doubleOut; - fRow.setFloatField(floatOut, colOut); - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - fRow.setDoubleField(doubleOut, colOut); - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - fRow.setStringField(strOut, colOut); - break; - - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::CLOB: - case execplan::CalpontSystemCatalog::BLOB: - fRow.setVarBinaryField(strOut, colOut); - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - fRow.setLongDoubleField(longdoubleOut, colOut); - break; - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregation: No logic for data type: " << colDataType; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } + std::ostringstream errmsg; + errmsg << "RowAggregation: No logic for data type: " << colDataType; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; } + } } //------------------------------------------------------------------------------ @@ -3217,48 +3060,48 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) //------------------------------------------------------------------------------ void RowAggregationUM::calculateUDAFColumns() { - RowUDAFFunctionCol* rowUDAF = nullptr; - static_any::any valOut; + RowUDAFFunctionCol* rowUDAF = nullptr; + static_any::any valOut; - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction != ROWAGG_UDAF) + continue; + + rowUDAF = dynamic_cast(fFunctionCols[i].get()); + fRGContext = rowUDAF->fUDAFContext; + + int64_t colOut = rowUDAF->fOutputColumnIndex; + int64_t colAux = rowUDAF->fAuxColumnIndex; + + // At this point, each row is an aggregated GROUP BY. + for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++) { - if (fFunctionCols[i]->fAggFunction != ROWAGG_UDAF) - continue; + // Get the user data from the row and evaluate. + fRowGroupOut->getRow(j, &fRow); - rowUDAF = dynamic_cast(fFunctionCols[i].get()); - fRGContext = rowUDAF->fUDAFContext; + // Turn the NULL flag off. We can't know NULL at this point + fRGContext.setDataFlags(nullptr); - int64_t colOut = rowUDAF->fOutputColumnIndex; - int64_t colAux = rowUDAF->fAuxColumnIndex; + // The intermediate values are stored in colAux. + fRGContext.setUserData(fRow.getUserData(colAux)); + // Call the UDAF evaluate function + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + rc = fRGContext.getFunction()->evaluate(&fRGContext, valOut); + fRGContext.setUserData(nullptr); - // At this point, each row is an aggregated GROUP BY. - for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++) - { - // Get the user data from the row and evaluate. - fRowGroupOut->getRow(j, &fRow); + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + rowUDAF->bInterrupted = true; + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } - // Turn the NULL flag off. We can't know NULL at this point - fRGContext.setDataFlags(nullptr); - - // The intermediate values are stored in colAux. - fRGContext.setUserData(fRow.getUserData(colAux)); - // Call the UDAF evaluate function - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->evaluate(&fRGContext, valOut); - fRGContext.setUserData(nullptr); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } - - // Set the returned value into the output row - SetUDAFValue(valOut, colOut); - } - - fRGContext.setUserData(nullptr); + // Set the returned value into the output row + SetUDAFValue(valOut, colOut); } + + fRGContext.setUserData(nullptr); + } } //------------------------------------------------------------------------------ @@ -3266,80 +3109,79 @@ void RowAggregationUM::calculateUDAFColumns() //------------------------------------------------------------------------------ void RowAggregationUM::calculateStatisticsFunctions() { - // ROWAGG_DUP_STATS may be not strictly duplicates, covers for statistics functions. - // They are calculated based on the same set of data: sum(x), sum(x**2) and count. - // array of for duplicates - boost::scoped_array > - auxCount(new pair[fRow.getColumnCount()]); + // ROWAGG_DUP_STATS may be not strictly duplicates, covers for statistics functions. + // They are calculated based on the same set of data: sum(x), sum(x**2) and count. + // array of for duplicates + boost::scoped_array> auxCount(new pair[fRow.getColumnCount()]); - fRowGroupOut->getRow(0, &fRow); + fRowGroupOut->getRow(0, &fRow); - for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++, fRow.nextRow()) + for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++, fRow.nextRow()) + { + for (uint64_t i = 0; i < fFunctionCols.size(); i++) { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + if (fFunctionCols[i]->fAggFunction == ROWAGG_STATS || + fFunctionCols[i]->fAggFunction == ROWAGG_DUP_STATS) + { + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + + double cnt = fRow.getDoubleField(colOut); + + if (fFunctionCols[i]->fAggFunction == ROWAGG_STATS) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_STATS || - fFunctionCols[i]->fAggFunction == ROWAGG_DUP_STATS) - { - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; - - double cnt = fRow.getDoubleField(colOut); - - if (fFunctionCols[i]->fAggFunction == ROWAGG_STATS) - { - auxCount[colOut].first = cnt; - auxCount[colOut].second = colAux; - } - else // ROWAGG_DUP_STATS - { - cnt = auxCount[colAux].first; - colAux = auxCount[colAux].second; - } - - if (cnt == 0.0) // empty set, set null. - { - fRow.setUintField(joblist::DOUBLENULL, colOut); - } - else if (cnt == 1.0) - { - if (fFunctionCols[i]->fStatsFunction == ROWAGG_STDDEV_SAMP || - fFunctionCols[i]->fStatsFunction == ROWAGG_VAR_SAMP) - fRow.setUintField(joblist::DOUBLENULL, colOut); - else - fRow.setDoubleField(0.0, colOut); - } - else // count > 1 - { - long double sum1 = fRow.getLongDoubleField(colAux); - long double sum2 = fRow.getLongDoubleField(colAux + 1); - - uint32_t scale = fRow.getScale(colOut); - auto factor = datatypes::scaleDivisor(scale); - - if (scale != 0) // adjust the scale if necessary - { - sum1 /= factor; - sum2 /= factor * factor; - } - - long double stat = sum1 * sum1 / cnt; - stat = sum2 - stat; - - if (fFunctionCols[i]->fStatsFunction == ROWAGG_STDDEV_POP) - stat = sqrt(stat / cnt); - else if (fFunctionCols[i]->fStatsFunction == ROWAGG_STDDEV_SAMP) - stat = sqrt(stat / (cnt - 1)); - else if (fFunctionCols[i]->fStatsFunction == ROWAGG_VAR_POP) - stat = stat / cnt; - else if (fFunctionCols[i]->fStatsFunction == ROWAGG_VAR_SAMP) - stat = stat / (cnt - 1); - - fRow.setDoubleField(stat, colOut); - } - } + auxCount[colOut].first = cnt; + auxCount[colOut].second = colAux; } + else // ROWAGG_DUP_STATS + { + cnt = auxCount[colAux].first; + colAux = auxCount[colAux].second; + } + + if (cnt == 0.0) // empty set, set null. + { + fRow.setUintField(joblist::DOUBLENULL, colOut); + } + else if (cnt == 1.0) + { + if (fFunctionCols[i]->fStatsFunction == ROWAGG_STDDEV_SAMP || + fFunctionCols[i]->fStatsFunction == ROWAGG_VAR_SAMP) + fRow.setUintField(joblist::DOUBLENULL, colOut); + else + fRow.setDoubleField(0.0, colOut); + } + else // count > 1 + { + long double sum1 = fRow.getLongDoubleField(colAux); + long double sum2 = fRow.getLongDoubleField(colAux + 1); + + uint32_t scale = fRow.getScale(colOut); + auto factor = datatypes::scaleDivisor(scale); + + if (scale != 0) // adjust the scale if necessary + { + sum1 /= factor; + sum2 /= factor * factor; + } + + long double stat = sum1 * sum1 / cnt; + stat = sum2 - stat; + + if (fFunctionCols[i]->fStatsFunction == ROWAGG_STDDEV_POP) + stat = sqrt(stat / cnt); + else if (fFunctionCols[i]->fStatsFunction == ROWAGG_STDDEV_SAMP) + stat = sqrt(stat / (cnt - 1)); + else if (fFunctionCols[i]->fStatsFunction == ROWAGG_VAR_POP) + stat = stat / cnt; + else if (fFunctionCols[i]->fStatsFunction == ROWAGG_VAR_SAMP) + stat = stat / (cnt - 1); + + fRow.setDoubleField(stat, colOut); + } + } } + } } //------------------------------------------------------------------------------ @@ -3347,26 +3189,26 @@ void RowAggregationUM::calculateStatisticsFunctions() //------------------------------------------------------------------------------ void RowAggregationUM::fixDuplicates(RowAggFunctionType funct) { - // find out if any column matches funct - vector dup; + // find out if any column matches funct + vector dup; - for (uint64_t i = 0; i < fFunctionCols.size(); i++) - { - if (fFunctionCols[i]->fAggFunction == funct) - dup.push_back(fFunctionCols[i]); - } + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == funct) + dup.push_back(fFunctionCols[i]); + } - if (0 == dup.size()) - return; + if (0 == dup.size()) + return; - // fix each row in the row group - fRowGroupOut->getRow(0, &fRow); + // fix each row in the row group + fRowGroupOut->getRow(0, &fRow); - for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) - { - for (uint64_t j = 0; j < dup.size(); j++) - fRow.copyField(dup[j]->fOutputColumnIndex, dup[j]->fAuxColumnIndex); - } + for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) + { + for (uint64_t j = 0; j < dup.size(); j++) + fRow.copyField(dup[j]->fOutputColumnIndex, dup[j]->fAuxColumnIndex); + } } //------------------------------------------------------------------------------ @@ -3374,13 +3216,13 @@ void RowAggregationUM::fixDuplicates(RowAggFunctionType funct) //------------------------------------------------------------------------------ void RowAggregationUM::evaluateExpression() { - funcexp::FuncExp* fe = funcexp::FuncExp::instance(); - fRowGroupOut->getRow(0, &fRow); + funcexp::FuncExp* fe = funcexp::FuncExp::instance(); + fRowGroupOut->getRow(0, &fRow); - for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) - { - fe->evaluate(fRow, fExpression); - } + for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) + { + fe->evaluate(fRow, fExpression); + } } //------------------------------------------------------------------------------ @@ -3388,186 +3230,184 @@ void RowAggregationUM::evaluateExpression() //------------------------------------------------------------------------------ void RowAggregationUM::fixConstantAggregate() { - // find the field has the count(*). - int64_t cntIdx = 0; + // find the field has the count(*). + int64_t cntIdx = 0; + + for (uint64_t k = 0; k < fFunctionCols.size(); k++) + { + if (fFunctionCols[k]->fAggFunction == ROWAGG_CONSTANT) + { + cntIdx = fFunctionCols[k]->fAuxColumnIndex; + break; + } + } + + fRowGroupOut->getRow(0, &fRow); + + for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) + { + int64_t rowCnt = fRow.getIntField(cntIdx); + vector::iterator j = fConstantAggregate.begin(); for (uint64_t k = 0; k < fFunctionCols.size(); k++) { - if (fFunctionCols[k]->fAggFunction == ROWAGG_CONSTANT) - { - cntIdx = fFunctionCols[k]->fAuxColumnIndex; - break; - } - } - - fRowGroupOut->getRow(0, &fRow); - - for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) - { - int64_t rowCnt = fRow.getIntField(cntIdx); - vector::iterator j = fConstantAggregate.begin(); - - for (uint64_t k = 0; k < fFunctionCols.size(); k++) - { - if (fFunctionCols[k]->fAggFunction == ROWAGG_CONSTANT) - { - if (j->fIsNull || rowCnt == 0) - doNullConstantAggregate(*j, k); - else - doNotNullConstantAggregate(*j, k); - - j++; - } - } + if (fFunctionCols[k]->fAggFunction == ROWAGG_CONSTANT) + { + if (j->fIsNull || rowCnt == 0) + doNullConstantAggregate(*j, k); + else + doNotNullConstantAggregate(*j, k); + + j++; + } } + } } - //------------------------------------------------------------------------------ // Calculate the aggregate(null) columns //------------------------------------------------------------------------------ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, uint64_t i) { - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int colDataType = (fRowGroupOut->getColTypes())[colOut]; + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + int colDataType = (fRowGroupOut->getColTypes())[colOut]; - switch (aggData.fOp) + switch (aggData.fOp) + { + case ROWAGG_MIN: + case ROWAGG_MAX: + case ROWAGG_AVG: + case ROWAGG_SUM: + case ROWAGG_DISTINCT_AVG: + case ROWAGG_DISTINCT_SUM: + case ROWAGG_STATS: { - case ROWAGG_MIN: - case ROWAGG_MAX: - case ROWAGG_AVG: - case ROWAGG_SUM: - case ROWAGG_DISTINCT_AVG: - case ROWAGG_DISTINCT_SUM: - case ROWAGG_STATS: + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - switch (colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - fRow.setIntField(getIntNullValue(colDataType), colOut); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - auto width = fRow.getColumnWidth(colOut); - if (fRow.getColumnWidth(colOut) == datatypes::MAXDECIMALWIDTH) - { - fRow.setInt128Field(datatypes::Decimal128Null, colOut); - } - else if (width <= datatypes::MAXLEGACYWIDTH) - { - fRow.setIntField(getIntNullValue(colDataType), colOut); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregationUM::doNullConstantAggregate(): DECIMAL bad length."); - } - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - fRow.setUintField(getUintNullValue(colDataType), colOut); - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - fRow.setDoubleField(getDoubleNullValue(), colOut); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - fRow.setFloatField(getFloatNullValue(), colOut); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - fRow.setUintField(getUintNullValue(colDataType), colOut); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - fRow.setIntField(getIntNullValue(colDataType), colOut); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - default: - { - fRow.setStringField("", colOut); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - fRow.setLongDoubleField(getLongDoubleNullValue(), colOut); - } - break; - - } + fRow.setIntField(getIntNullValue(colDataType), colOut); } break; - case ROWAGG_COUNT_COL_NAME: - case ROWAGG_COUNT_DISTINCT_COL_NAME: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - fRow.setUintField(0, colOut); + auto width = fRow.getColumnWidth(colOut); + if (fRow.getColumnWidth(colOut) == datatypes::MAXDECIMALWIDTH) + { + fRow.setInt128Field(datatypes::Decimal128Null, colOut); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNullConstantAggregate(): DECIMAL bad length."); + } } break; - case ROWAGG_BIT_AND: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: { - fRow.setUintField(0xFFFFFFFFFFFFFFFFULL, colOut); + fRow.setUintField(getUintNullValue(colDataType), colOut); } break; - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: { - fRow.setUintField(0, colOut); + fRow.setDoubleField(getDoubleNullValue(), colOut); } break; - case ROWAGG_UDAF: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: { - // For a NULL constant, call nextValue with NULL and then evaluate. - bool bInterrupted = false; - fRGContext.setInterrupted(bInterrupted); - fRGContext.createUserData(); - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - mcsv1sdk::ColumnDatum valsIn[1]; + fRow.setFloatField(getFloatNullValue(), colOut); + } + break; - // Call a reset, then nextValue, then execute. This will evaluate - // the UDAF for the constant. - rc = fRGContext.getFunction()->reset(&fRGContext); + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + fRow.setUintField(getUintNullValue(colDataType), colOut); + } + break; - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - fRGContext.setInterrupted(true); - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + default: + { + fRow.setStringField("", colOut); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + fRow.setLongDoubleField(getLongDoubleNullValue(), colOut); + } + break; + } + } + break; + + case ROWAGG_COUNT_COL_NAME: + case ROWAGG_COUNT_DISTINCT_COL_NAME: + { + fRow.setUintField(0, colOut); + } + break; + + case ROWAGG_BIT_AND: + { + fRow.setUintField(0xFFFFFFFFFFFFFFFFULL, colOut); + } + break; + + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + fRow.setUintField(0, colOut); + } + break; + + case ROWAGG_UDAF: + { + // For a NULL constant, call nextValue with NULL and then evaluate. + bool bInterrupted = false; + fRGContext.setInterrupted(bInterrupted); + fRGContext.createUserData(); + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + mcsv1sdk::ColumnDatum valsIn[1]; + + // Call a reset, then nextValue, then execute. This will evaluate + // the UDAF for the constant. + rc = fRGContext.getFunction()->reset(&fRGContext); + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + fRGContext.setInterrupted(true); + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } #if 0 uint32_t dataFlags[fRGContext.getParameterCount()]; @@ -3580,551 +3420,550 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u } #endif - // Turn the NULL and CONSTANT flags on. - uint32_t flags[1]; - flags[0] = mcsv1sdk::PARAM_IS_NULL | mcsv1sdk::PARAM_IS_CONSTANT; - fRGContext.setDataFlags(flags); + // Turn the NULL and CONSTANT flags on. + uint32_t flags[1]; + flags[0] = mcsv1sdk::PARAM_IS_NULL | mcsv1sdk::PARAM_IS_CONSTANT; + fRGContext.setDataFlags(flags); - // Create a dummy datum - mcsv1sdk::ColumnDatum& datum = valsIn[0]; - datum.dataType = execplan::CalpontSystemCatalog::BIGINT; - datum.columnData = 0; + // Create a dummy datum + mcsv1sdk::ColumnDatum& datum = valsIn[0]; + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + datum.columnData = 0; - rc = fRGContext.getFunction()->nextValue(&fRGContext, valsIn); + rc = fRGContext.getFunction()->nextValue(&fRGContext, valsIn); - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - fRGContext.setInterrupted(true); - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + fRGContext.setInterrupted(true); + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } - static_any::any valOut; - rc = fRGContext.getFunction()->evaluate(&fRGContext, valOut); - fRGContext.setUserData(nullptr); + static_any::any valOut; + rc = fRGContext.getFunction()->evaluate(&fRGContext, valOut); + fRGContext.setUserData(nullptr); - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - fRGContext.setInterrupted(true); - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + fRGContext.setInterrupted(true); + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } - // Set the returned value into the output row - SetUDAFValue(valOut, colOut); - fRGContext.setDataFlags(nullptr); - } - break; - - default: - { - fRow.setStringField("", colOut); - } - break; + // Set the returned value into the output row + SetUDAFValue(valOut, colOut); + fRGContext.setDataFlags(nullptr); } -} + break; + default: + { + fRow.setStringField("", colOut); + } + break; + } +} //------------------------------------------------------------------------------ // Calculate the aggregate(const) columns //------------------------------------------------------------------------------ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData, uint64_t i) { - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - auto colDataType = (fRowGroupOut->getColTypes())[colOut]; - int64_t rowCnt = fRow.getIntField(fFunctionCols[i]->fAuxColumnIndex); + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + auto colDataType = (fRowGroupOut->getColTypes())[colOut]; + int64_t rowCnt = fRow.getIntField(fFunctionCols[i]->fAuxColumnIndex); - switch (aggData.fOp) + switch (aggData.fOp) + { + case ROWAGG_MIN: + case ROWAGG_MAX: + case ROWAGG_AVG: + case ROWAGG_DISTINCT_AVG: + case ROWAGG_DISTINCT_SUM: { - case ROWAGG_MIN: - case ROWAGG_MAX: - case ROWAGG_AVG: - case ROWAGG_DISTINCT_AVG: - case ROWAGG_DISTINCT_SUM: + switch (colDataType) + { + // AVG should not be int result type. + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - switch (colDataType) - { - // AVG should not be int result type. - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - fRow.setIntField(strtol(aggData.fConstValue.c_str(), nullptr, 10), colOut); - } - break; + fRow.setIntField(strtol(aggData.fConstValue.c_str(), nullptr, 10), colOut); + } + break; - // AVG should not be uint32_t result type. - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - fRow.setUintField(strtoul(aggData.fConstValue.c_str(), nullptr, 10), colOut); - } - break; + // AVG should not be uint32_t result type. + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + fRow.setUintField(strtoul(aggData.fConstValue.c_str(), nullptr, 10), colOut); + } + break; - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - auto width = fRow.getColumnWidth(colOut); - if (width == datatypes::MAXDECIMALWIDTH) - { - execplan::CalpontSystemCatalog::TypeHolderStd colType; - colType.colWidth = width; - colType.precision = fRow.getPrecision(i); - colType.scale = fRow.getScale(i); - colType.colDataType = colDataType; - fRow.setInt128Field(colType.decimal128FromString(aggData.fConstValue), colOut); - } - else if (width <= datatypes::MAXLEGACYWIDTH) - { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - auto scale = datatypes::scaleDivisor(fRowGroupOut->getScale()[i]); - // TODO: isn't overflow possible below: - fRow.setIntField((int64_t)(scale * dbl), colOut); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): DECIMAL bad length."); - } - } - break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + auto width = fRow.getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + { + execplan::CalpontSystemCatalog::TypeHolderStd colType; + colType.colWidth = width; + colType.precision = fRow.getPrecision(i); + colType.scale = fRow.getScale(i); + colType.colDataType = colDataType; + fRow.setInt128Field(colType.decimal128FromString(aggData.fConstValue), colOut); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + auto scale = datatypes::scaleDivisor(fRowGroupOut->getScale()[i]); + // TODO: isn't overflow possible below: + fRow.setIntField((int64_t)(scale * dbl), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): DECIMAL bad length."); + } + } + break; - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - fRow.setDoubleField(strtod(aggData.fConstValue.c_str(), nullptr), colOut); - } - break; + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + fRow.setDoubleField(strtod(aggData.fConstValue.c_str(), nullptr), colOut); + } + break; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - fRow.setLongDoubleField(strtold(aggData.fConstValue.c_str(), nullptr), colOut); - } - break; + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + fRow.setLongDoubleField(strtold(aggData.fConstValue.c_str(), nullptr), colOut); + } + break; - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { #ifdef _MSC_VER - fRow.setFloatField(strtod(aggData.fConstValue.c_str(), 0), colOut); + fRow.setFloatField(strtod(aggData.fConstValue.c_str(), 0), colOut); #else - fRow.setFloatField(strtof(aggData.fConstValue.c_str(), nullptr), colOut); + fRow.setFloatField(strtof(aggData.fConstValue.c_str(), nullptr), colOut); #endif - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - fRow.setUintField(DataConvert::stringToDate(aggData.fConstValue), colOut); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - fRow.setUintField(DataConvert::stringToDatetime(aggData.fConstValue), colOut); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - fRow.setUintField(DataConvert::stringToTimestamp(aggData.fConstValue, fTimeZone), colOut); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - default: - { - fRow.setStringField(aggData.fConstValue, colOut); - } - break; - } } break; - case ROWAGG_SUM: + case execplan::CalpontSystemCatalog::DATE: { - switch (colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - int64_t constVal = strtol(aggData.fConstValue.c_str(), nullptr, 10); - - if (constVal != 0) - { - int64_t tmp = numeric_limits::max() / constVal; - - if (constVal < 0) - tmp = numeric_limits::min() / constVal; - - if (rowCnt > tmp) - throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); - } - - fRow.setIntField(constVal * rowCnt, colOut); - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - uint64_t constVal = strtoul(aggData.fConstValue.c_str(), nullptr, 10); - fRow.setUintField(constVal * rowCnt, colOut); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - auto width = fRow.getColumnWidth(colOut); - if (width == datatypes::MAXDECIMALWIDTH) - { - execplan::CalpontSystemCatalog::TypeHolderStd colType; - colType.colWidth = width; - colType.precision = fRow.getPrecision(i); - colType.scale = fRow.getScale(i); - colType.colDataType = colDataType; - int128_t constValue = colType.decimal128FromString(aggData.fConstValue); - int128_t sum; - - datatypes::MultiplicationOverflowCheck multOp; - multOp(constValue, rowCnt, sum); - fRow.setInt128Field(sum, colOut); - } - else if (width == datatypes::MAXLEGACYWIDTH) - { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - // TODO: isn't precision loss possible below? - dbl *= datatypes::scaleDivisor(fRowGroupOut->getScale()[i]); - dbl *= rowCnt; - - if ((dbl > 0 && dbl > (double) numeric_limits::max()) || - (dbl < 0 && dbl < (double) numeric_limits::min())) - throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); - fRow.setIntField((int64_t) dbl, colOut); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): sum() DECIMAL bad length."); - } - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double dbl = strtod(aggData.fConstValue.c_str(), nullptr) * rowCnt; - fRow.setDoubleField(dbl, colOut); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double dbl = strtold(aggData.fConstValue.c_str(), nullptr) * rowCnt; - fRow.setLongDoubleField(dbl, colOut); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - double flt; -#ifdef _MSC_VER - flt = strtod(aggData.fConstValue.c_str(), 0) * rowCnt; -#else - flt = strtof(aggData.fConstValue.c_str(), nullptr) * rowCnt; -#endif - fRow.setFloatField(flt, colOut); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - default: - { - // will not be here, checked in tupleaggregatestep.cpp. - fRow.setStringField("", colOut); - } - break; - } + fRow.setUintField(DataConvert::stringToDate(aggData.fConstValue), colOut); } break; - case ROWAGG_STATS: + case execplan::CalpontSystemCatalog::DATETIME: { - switch (colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - fRow.setIntField(0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - fRow.setUintField(0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - fRow.setDoubleField(0.0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - fRow.setLongDoubleField(0.0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - fRow.setFloatField(0.0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - fRow.setUintField(0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::TIME: - { - fRow.setUintField(0, colOut); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - default: - { - fRow.setStringField(nullptr, colOut); - } - break; - } + fRow.setUintField(DataConvert::stringToDatetime(aggData.fConstValue), colOut); } break; - case ROWAGG_COUNT_COL_NAME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { - fRow.setUintField(rowCnt, colOut); + fRow.setUintField(DataConvert::stringToTimestamp(aggData.fConstValue, fTimeZone), colOut); } break; - case ROWAGG_COUNT_DISTINCT_COL_NAME: + case execplan::CalpontSystemCatalog::TIME: { - fRow.setUintField(1, colOut); - } - break; - - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - { - double dbl = strtod(aggData.fConstValue.c_str(), nullptr); - dbl += (dbl > 0) ? 0.5 : -0.5; - int64_t intVal = (int64_t) dbl; - fRow.setUintField(intVal, colOut); - } - break; - - case ROWAGG_BIT_XOR: - { - fRow.setUintField(0, colOut); - } - break; - - case ROWAGG_UDAF: - { - bool bInterrupted = false; - fRGContext.setInterrupted(bInterrupted); - fRGContext.createUserData(); - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - mcsv1sdk::ColumnDatum valsIn[1]; - - // Call a reset, then nextValue, then execute. This will evaluate - // the UDAF for the constant. - rc = fRGContext.getFunction()->reset(&fRGContext); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - fRGContext.setInterrupted(true); - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } - - // Turn the CONSTANT flags on. - uint32_t flags[1]; - flags[0] = mcsv1sdk::PARAM_IS_CONSTANT; - fRGContext.setDataFlags(flags); - - // Create a datum item for sending to UDAF - mcsv1sdk::ColumnDatum& datum = valsIn[0]; - datum.dataType = (execplan::CalpontSystemCatalog::ColDataType)colDataType; - - switch (colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - datum.columnData = strtol(aggData.fConstValue.c_str(), nullptr, 10); - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - datum.columnData = strtoul(aggData.fConstValue.c_str(), nullptr, 10); - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - // TODO: isn't overflow possible below? - datum.columnData = (int64_t) (dbl * datatypes::scaleDivisor(fRowGroupOut->getScale()[i])); - datum.scale = fRowGroupOut->getScale()[i]; - datum.precision = fRowGroupOut->getPrecision()[i]; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - datum.columnData = strtod(aggData.fConstValue.c_str(), nullptr); - } - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - datum.columnData = strtold(aggData.fConstValue.c_str(), nullptr); - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { -#ifdef _MSC_VER - datum.columnData = strtod(aggData.fConstValue.c_str(), 0); -#else - datum.columnData = strtof(aggData.fConstValue.c_str(), nullptr); -#endif - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - datum.columnData = DataConvert::stringToDate(aggData.fConstValue); - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - datum.columnData = DataConvert::stringToDatetime(aggData.fConstValue); - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - datum.columnData = DataConvert::stringToTimestamp(aggData.fConstValue, fTimeZone); - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - datum.columnData = DataConvert::stringToTime(aggData.fConstValue); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::BLOB: - default: - { - datum.columnData = aggData.fConstValue; - } - break; - } - - rc = fRGContext.getFunction()->nextValue(&fRGContext, valsIn); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - fRGContext.setInterrupted(true); - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } - - static_any::any valOut; - rc = fRGContext.getFunction()->evaluate(&fRGContext, valOut); - fRGContext.setUserData(nullptr); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - fRGContext.setInterrupted(true); - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); - } - - // Set the returned value into the output row - SetUDAFValue(valOut, colOut); - fRGContext.setDataFlags(nullptr); + fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); } break; + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: default: { - fRow.setStringField(aggData.fConstValue, colOut); + fRow.setStringField(aggData.fConstValue, colOut); } break; + } } -} + break; + case ROWAGG_SUM: + { + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + { + int64_t constVal = strtol(aggData.fConstValue.c_str(), nullptr, 10); + + if (constVal != 0) + { + int64_t tmp = numeric_limits::max() / constVal; + + if (constVal < 0) + tmp = numeric_limits::min() / constVal; + + if (rowCnt > tmp) + throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); + } + + fRow.setIntField(constVal * rowCnt, colOut); + } + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + uint64_t constVal = strtoul(aggData.fConstValue.c_str(), nullptr, 10); + fRow.setUintField(constVal * rowCnt, colOut); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + auto width = fRow.getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + { + execplan::CalpontSystemCatalog::TypeHolderStd colType; + colType.colWidth = width; + colType.precision = fRow.getPrecision(i); + colType.scale = fRow.getScale(i); + colType.colDataType = colDataType; + int128_t constValue = colType.decimal128FromString(aggData.fConstValue); + int128_t sum; + + datatypes::MultiplicationOverflowCheck multOp; + multOp(constValue, rowCnt, sum); + fRow.setInt128Field(sum, colOut); + } + else if (width == datatypes::MAXLEGACYWIDTH) + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + // TODO: isn't precision loss possible below? + dbl *= datatypes::scaleDivisor(fRowGroupOut->getScale()[i]); + dbl *= rowCnt; + + if ((dbl > 0 && dbl > (double)numeric_limits::max()) || + (dbl < 0 && dbl < (double)numeric_limits::min())) + throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); + fRow.setIntField((int64_t)dbl, colOut); + } + else + { + idbassert(0); + throw std::logic_error( + "RowAggregationUM::doNotNullConstantAggregate(): sum() DECIMAL bad length."); + } + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double dbl = strtod(aggData.fConstValue.c_str(), nullptr) * rowCnt; + fRow.setDoubleField(dbl, colOut); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double dbl = strtold(aggData.fConstValue.c_str(), nullptr) * rowCnt; + fRow.setLongDoubleField(dbl, colOut); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + double flt; +#ifdef _MSC_VER + flt = strtod(aggData.fConstValue.c_str(), 0) * rowCnt; +#else + flt = strtof(aggData.fConstValue.c_str(), nullptr) * rowCnt; +#endif + fRow.setFloatField(flt, colOut); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + default: + { + // will not be here, checked in tupleaggregatestep.cpp. + fRow.setStringField("", colOut); + } + break; + } + } + break; + + case ROWAGG_STATS: + { + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + fRow.setIntField(0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + fRow.setUintField(0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + fRow.setDoubleField(0.0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + fRow.setLongDoubleField(0.0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + fRow.setFloatField(0.0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + fRow.setUintField(0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setUintField(0, colOut); + } + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + default: + { + fRow.setStringField(nullptr, colOut); + } + break; + } + } + break; + + case ROWAGG_COUNT_COL_NAME: + { + fRow.setUintField(rowCnt, colOut); + } + break; + + case ROWAGG_COUNT_DISTINCT_COL_NAME: + { + fRow.setUintField(1, colOut); + } + break; + + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + { + double dbl = strtod(aggData.fConstValue.c_str(), nullptr); + dbl += (dbl > 0) ? 0.5 : -0.5; + int64_t intVal = (int64_t)dbl; + fRow.setUintField(intVal, colOut); + } + break; + + case ROWAGG_BIT_XOR: + { + fRow.setUintField(0, colOut); + } + break; + + case ROWAGG_UDAF: + { + bool bInterrupted = false; + fRGContext.setInterrupted(bInterrupted); + fRGContext.createUserData(); + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + mcsv1sdk::ColumnDatum valsIn[1]; + + // Call a reset, then nextValue, then execute. This will evaluate + // the UDAF for the constant. + rc = fRGContext.getFunction()->reset(&fRGContext); + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + fRGContext.setInterrupted(true); + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } + + // Turn the CONSTANT flags on. + uint32_t flags[1]; + flags[0] = mcsv1sdk::PARAM_IS_CONSTANT; + fRGContext.setDataFlags(flags); + + // Create a datum item for sending to UDAF + mcsv1sdk::ColumnDatum& datum = valsIn[0]; + datum.dataType = (execplan::CalpontSystemCatalog::ColDataType)colDataType; + + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + { + datum.columnData = strtol(aggData.fConstValue.c_str(), nullptr, 10); + } + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + datum.columnData = strtoul(aggData.fConstValue.c_str(), nullptr, 10); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + // TODO: isn't overflow possible below? + datum.columnData = (int64_t)(dbl * datatypes::scaleDivisor(fRowGroupOut->getScale()[i])); + datum.scale = fRowGroupOut->getScale()[i]; + datum.precision = fRowGroupOut->getPrecision()[i]; + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + datum.columnData = strtod(aggData.fConstValue.c_str(), nullptr); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + datum.columnData = strtold(aggData.fConstValue.c_str(), nullptr); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { +#ifdef _MSC_VER + datum.columnData = strtod(aggData.fConstValue.c_str(), 0); +#else + datum.columnData = strtof(aggData.fConstValue.c_str(), nullptr); +#endif + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + datum.columnData = DataConvert::stringToDate(aggData.fConstValue); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + datum.columnData = DataConvert::stringToDatetime(aggData.fConstValue); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + datum.columnData = DataConvert::stringToTimestamp(aggData.fConstValue, fTimeZone); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + datum.columnData = DataConvert::stringToTime(aggData.fConstValue); + } + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: + default: + { + datum.columnData = aggData.fConstValue; + } + break; + } + + rc = fRGContext.getFunction()->nextValue(&fRGContext, valsIn); + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + fRGContext.setInterrupted(true); + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } + + static_any::any valOut; + rc = fRGContext.getFunction()->evaluate(&fRGContext, valOut); + fRGContext.setUserData(nullptr); + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + fRGContext.setInterrupted(true); + throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + } + + // Set the returned value into the output row + SetUDAFValue(valOut, colOut); + fRGContext.setDataFlags(nullptr); + } + break; + + default: + { + fRow.setStringField(aggData.fConstValue, colOut); + } + break; + } +} //------------------------------------------------------------------------------ // Allocate a new data array for the output RowGroup @@ -4132,38 +3971,37 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData //------------------------------------------------------------------------------ void RowAggregationUM::setGroupConcatString() { - fRowGroupOut->getRow(0, &fRow); + fRowGroupOut->getRow(0, &fRow); - for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) + for (uint64_t i = 0; i < fRowGroupOut->getRowCount(); i++, fRow.nextRow()) + { + for (uint64_t j = 0; j < fFunctionCols.size(); j++) { - for (uint64_t j = 0; j < fFunctionCols.size(); j++) - { - uint8_t* data = fRow.getData(); + uint8_t* data = fRow.getData(); - if (fFunctionCols[j]->fAggFunction == ROWAGG_GROUP_CONCAT) - { - uint8_t* buff = data + fRow.getOffset(fFunctionCols[j]->fOutputColumnIndex); - uint8_t* gcString; - joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)buff); - gcString = gccAg->getResult(); - fRow.setStringField((char*) gcString, fFunctionCols[j]->fOutputColumnIndex); - //gccAg->getResult(buff); - } - } + if (fFunctionCols[j]->fAggFunction == ROWAGG_GROUP_CONCAT) + { + uint8_t* buff = data + fRow.getOffset(fFunctionCols[j]->fOutputColumnIndex); + uint8_t* gcString; + joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)buff); + gcString = gccAg->getResult(); + fRow.setStringField((char*)gcString, fFunctionCols[j]->fOutputColumnIndex); + // gccAg->getResult(buff); + } } + } } void RowAggregationUM::setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) { - RowAggregation::setInputOutput(pRowGroupIn, pRowGroupOut); + RowAggregation::setInputOutput(pRowGroupIn, pRowGroupOut); - if (fKeyOnHeap) - { - fKeyRG = fRowGroupIn.truncate(fGroupByCols.size()); - } + if (fKeyOnHeap) + { + fKeyRG = fRowGroupIn.truncate(fGroupByCols.size()); + } } - //------------------------------------------------------------------------------ // Returns the next group of aggregated rows. // We do not yet cache large aggregations (more than 1 RowGroup result set) @@ -4189,122 +4027,109 @@ bool RowAggregationUM::nextRowGroup() return more; } - //------------------------------------------------------------------------------ // Row Aggregation constructor used on UM // For 2nd phase of two-phase case, from partial RG to final aggregated RG //------------------------------------------------------------------------------ RowAggregationUMP2::RowAggregationUMP2(const vector& rowAggGroupByCols, - const vector& rowAggFunctionCols, - joblist::ResourceManager* r, - boost::shared_ptr sessionLimit) : - RowAggregationUM(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) + const vector& rowAggFunctionCols, + joblist::ResourceManager* r, boost::shared_ptr sessionLimit) + : RowAggregationUM(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) { } - -RowAggregationUMP2::RowAggregationUMP2(const RowAggregationUMP2& rhs) : - RowAggregationUM(rhs) +RowAggregationUMP2::RowAggregationUMP2(const RowAggregationUMP2& rhs) : RowAggregationUM(rhs) { } - RowAggregationUMP2::~RowAggregationUMP2() { } - //------------------------------------------------------------------------------ // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for count // rowIn(in) - Row to be included in aggregation. // rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationUMP2::updateEntry(const Row& rowIn, - std::vector* rgContextColl) +void RowAggregationUMP2::updateEntry(const Row& rowIn, std::vector* rgContextColl) { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + int64_t colIn = fFunctionCols[i]->fInputColumnIndex; + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + + switch (fFunctionCols[i]->fAggFunction) { - int64_t colIn = fFunctionCols[i]->fInputColumnIndex; - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + case ROWAGG_COUNT_ASTERISK: + case ROWAGG_COUNT_COL_NAME: + { + uint64_t count = fRow.getUintField<8>(colOut) + rowIn.getUintField<8>(colIn); + fRow.setUintField<8>(count, colOut); + break; + } - switch (fFunctionCols[i]->fAggFunction) - { - case ROWAGG_COUNT_ASTERISK: - case ROWAGG_COUNT_COL_NAME: - { - uint64_t count = fRow.getUintField<8>(colOut) + rowIn.getUintField<8>(colIn); - fRow.setUintField<8>(count, colOut); - break; - } + case ROWAGG_MIN: + case ROWAGG_MAX: doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_MIN: - case ROWAGG_MAX: - doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_SUM: doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_SUM: - doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_AVG: + { + // The sum and count on UM may not be put next to each other: + // use colOut to store the sum; + // use colAux to store the count. + doAvg(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_AVG: - { - // The sum and count on UM may not be put next to each other: - // use colOut to store the sum; - // use colAux to store the count. - doAvg(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_STATS: + { + doStatistics(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_STATS: - { - doStatistics(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); + break; + } - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; - } + case ROWAGG_GROUP_CONCAT: + { + doGroupConcat(rowIn, colIn, colOut); + break; + } - case ROWAGG_GROUP_CONCAT: - { - doGroupConcat(rowIn, colIn, colOut); - break; - } + case ROWAGG_COUNT_NO_OP: + case ROWAGG_DUP_FUNCT: + case ROWAGG_DUP_AVG: + case ROWAGG_DUP_STATS: + case ROWAGG_DUP_UDAF: + case ROWAGG_CONSTANT: break; - case ROWAGG_COUNT_NO_OP: - case ROWAGG_DUP_FUNCT: - case ROWAGG_DUP_AVG: - case ROWAGG_DUP_STATS: - case ROWAGG_DUP_UDAF: - case ROWAGG_CONSTANT: - break; + case ROWAGG_UDAF: + { + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); + break; + } - case ROWAGG_UDAF: - { - doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregationUMP2: function (id = " << - (uint64_t) fFunctionCols[i]->fAggFunction << ") is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - } + default: + { + std::ostringstream errmsg; + errmsg << "RowAggregationUMP2: function (id = " << (uint64_t)fFunctionCols[i]->fAggFunction + << ") is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } } + } } - //------------------------------------------------------------------------------ // Update the sum and count fields for average if input is not null. // rowIn(in) - Row to be included in aggregation. @@ -4314,139 +4139,139 @@ void RowAggregationUMP2::updateEntry(const Row& rowIn, //------------------------------------------------------------------------------ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, bool merge) { - if (rowIn.isNullValue(colIn)) - return; + if (rowIn.isNullValue(colIn)) + return; - datatypes::SystemCatalog::ColDataType colDataType = rowIn.getColType(colIn); - long double valIn = 0; - bool isWideDataType = false; - void *wideValInPtr = nullptr; + datatypes::SystemCatalog::ColDataType colDataType = rowIn.getColType(colIn); + long double valIn = 0; + bool isWideDataType = false; + void* wideValInPtr = nullptr; - switch (colDataType) + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - valIn = rowIn.getIntField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - valIn = rowIn.getUintField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - uint32_t width = rowIn.getColumnWidth(colIn); - isWideDataType = width == datatypes::MAXDECIMALWIDTH; - if(LIKELY(isWideDataType)) - { - int128_t* dec = rowIn.getBinaryField(colIn); - wideValInPtr = reinterpret_cast(dec); - } - else if (width <= datatypes::MAXLEGACYWIDTH) - { - uint32_t scale = rowIn.getScale(colIn); - valIn = rowIn.getScaledSInt64FieldAsXFloat(colIn, scale); - } - else - { - idbassert(0); - throw std::logic_error("RowAggregationUMP2::doAvg(): DECIMAL bad length."); - } - - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - valIn = rowIn.getDoubleField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - valIn = rowIn.getFloatField(colIn); - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - valIn = rowIn.getLongDoubleField(colIn); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregationUMP2: no average for data type: " << colDataType; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } + valIn = rowIn.getIntField(colIn); + break; } - uint64_t cnt = fRow.getUintField(colAux); - auto colAuxIn = merge ? colAux : (colIn + 1); - - if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(colDataType) && !isWideDataType) + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: { - if (LIKELY(cnt > 0)) - { - int128_t *valOutPtr = fRow.getBinaryField(colOut); - int128_t sum = valIn + *valOutPtr; - fRow.setBinaryField(&sum, colOut); - fRow.setUintField(rowIn.getUintField(colAuxIn) + cnt, colAux); - } - else - { - int128_t sum = valIn; - fRow.setBinaryField(&sum, colOut); - fRow.setUintField(rowIn.getUintField(colAuxIn), colAux); - } + valIn = rowIn.getUintField(colIn); + break; } - else if (isWideDataType) + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - int128_t* dec = reinterpret_cast(wideValInPtr); - if (LIKELY(cnt > 0)) - { - int128_t *valOutPtr = fRow.getBinaryField(colOut); - int128_t sum = *valOutPtr + *dec; - fRow.setBinaryField(&sum, colOut); - fRow.setUintField(rowIn.getUintField(colAuxIn) + cnt, colAux); - } - else - { - fRow.setBinaryField(dec, colOut); - fRow.setUintField(rowIn.getUintField(colAuxIn), colAux); - } + uint32_t width = rowIn.getColumnWidth(colIn); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if (LIKELY(isWideDataType)) + { + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + uint32_t scale = rowIn.getScale(colIn); + valIn = rowIn.getScaledSInt64FieldAsXFloat(colIn, scale); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUMP2::doAvg(): DECIMAL bad length."); + } + + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + valIn = rowIn.getDoubleField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + valIn = rowIn.getFloatField(colIn); + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + valIn = rowIn.getLongDoubleField(colIn); + break; + } + + default: + { + std::ostringstream errmsg; + errmsg << "RowAggregationUMP2: no average for data type: " << colDataType; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } + } + + uint64_t cnt = fRow.getUintField(colAux); + auto colAuxIn = merge ? colAux : (colIn + 1); + + if (datatypes::hasUnderlyingWideDecimalForSumAndAvg(colDataType) && !isWideDataType) + { + if (LIKELY(cnt > 0)) + { + int128_t* valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = valIn + *valOutPtr; + fRow.setBinaryField(&sum, colOut); + fRow.setUintField(rowIn.getUintField(colAuxIn) + cnt, colAux); } else { - if (LIKELY(cnt > 0)) - { - long double valOut = fRow.getLongDoubleField(colOut); - fRow.setLongDoubleField(valIn + valOut, colOut); - fRow.setUintField(rowIn.getUintField(colAuxIn) + cnt, colAux); - } - else - { - fRow.setLongDoubleField(valIn, colOut); - fRow.setUintField(rowIn.getUintField(colAuxIn), colAux); - } + int128_t sum = valIn; + fRow.setBinaryField(&sum, colOut); + fRow.setUintField(rowIn.getUintField(colAuxIn), colAux); } + } + else if (isWideDataType) + { + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(cnt > 0)) + { + int128_t* valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField(&sum, colOut); + fRow.setUintField(rowIn.getUintField(colAuxIn) + cnt, colAux); + } + else + { + fRow.setBinaryField(dec, colOut); + fRow.setUintField(rowIn.getUintField(colAuxIn), colAux); + } + } + else + { + if (LIKELY(cnt > 0)) + { + long double valOut = fRow.getLongDoubleField(colOut); + fRow.setLongDoubleField(valIn + valOut, colOut); + fRow.setUintField(rowIn.getUintField(colAuxIn) + cnt, colAux); + } + else + { + fRow.setLongDoubleField(valIn, colOut); + fRow.setUintField(rowIn.getUintField(colAuxIn), colAux); + } + } } //------------------------------------------------------------------------------ @@ -4459,29 +4284,25 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, // colAux(in) - column in the output row group stores the sum(x) // colAux + 1 - column in the output row group stores the sum(x**2) //------------------------------------------------------------------------------ -void RowAggregationUMP2::doStatistics( - const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux) +void RowAggregationUMP2::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux) { - fRow.setDoubleField(fRow.getDoubleField(colOut) + rowIn.getDoubleField(colIn), colOut); - fRow.setLongDoubleField( - fRow.getLongDoubleField(colAux) + rowIn.getLongDoubleField(colIn + 1), colAux); - fRow.setLongDoubleField( - fRow.getLongDoubleField(colAux + 1) + rowIn.getLongDoubleField(colIn + 2), colAux + 1); + fRow.setDoubleField(fRow.getDoubleField(colOut) + rowIn.getDoubleField(colIn), colOut); + fRow.setLongDoubleField(fRow.getLongDoubleField(colAux) + rowIn.getLongDoubleField(colIn + 1), colAux); + fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + rowIn.getLongDoubleField(colIn + 2), + colAux + 1); } - //------------------------------------------------------------------------------ // Concat columns. // rowIn(in) - Row that contains the columns to be concatenated. //------------------------------------------------------------------------------ void RowAggregationUMP2::doGroupConcat(const Row& rowIn, int64_t i, int64_t o) { - uint8_t* data = fRow.getData(); - joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + fRow.getOffset(o))); - gccAg->merge(rowIn, i); + uint8_t* data = fRow.getData(); + joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + fRow.getOffset(o))); + gccAg->merge(rowIn, i); } - //------------------------------------------------------------------------------ // Update the and/or/xor fields if input is not null. // rowIn(in) - Row to be included in aggregation. @@ -4493,15 +4314,15 @@ void RowAggregationUMP2::doGroupConcat(const Row& rowIn, int64_t i, int64_t o) //------------------------------------------------------------------------------ void RowAggregationUMP2::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, int funcType) { - uint64_t valIn = rowIn.getUintField(colIn); - uint64_t valOut = fRow.getUintField(colOut); + uint64_t valIn = rowIn.getUintField(colIn); + uint64_t valOut = fRow.getUintField(colOut); - if (funcType == ROWAGG_BIT_AND) - fRow.setUintField(valIn & valOut, colOut); - else if (funcType == ROWAGG_BIT_OR) - fRow.setUintField(valIn | valOut, colOut); - else - fRow.setUintField(valIn ^ valOut, colOut); + if (funcType == ROWAGG_BIT_AND) + fRow.setUintField(valIn & valOut, colOut); + else if (funcType == ROWAGG_BIT_OR) + fRow.setUintField(valIn | valOut, colOut); + else + fRow.setUintField(valIn ^ valOut, colOut); } //------------------------------------------------------------------------------ @@ -4514,314 +4335,285 @@ void RowAggregationUMP2::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut // rowUDAF(in) - pointer to the RowUDAFFunctionCol for this UDAF instance // rgContextColl(in) - ptr to a vector that brings UDAF contextx in //------------------------------------------------------------------------------ -void RowAggregationUMP2::doUDAF(const Row& rowIn, - int64_t colIn, - int64_t colOut, - int64_t colAux, - uint64_t& funcColsIdx, - std::vector* rgContextColl) +void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, + uint64_t& funcColsIdx, std::vector* rgContextColl) { - static_any::any valOut; - std::vector* udafContextsCollPtr = &fRGContextColl; - if (UNLIKELY(rgContextColl != nullptr)) + static_any::any valOut; + std::vector* udafContextsCollPtr = &fRGContextColl; + if (UNLIKELY(rgContextColl != nullptr)) + { + udafContextsCollPtr = rgContextColl; + } + + std::vector& udafContextsColl = *udafContextsCollPtr; + + // Get the user data + boost::shared_ptr userDataIn = rowIn.getUserData(colIn + 1); + + // Unlike other aggregates, the data isn't in colIn, so testing it for NULL + // there won't help. In case of NULL, userData will be NULL. + uint32_t flags[1]; + + flags[0] = 0; + + if (!userDataIn) + { + if (udafContextsColl[funcColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { - udafContextsCollPtr = rgContextColl; + return; } - std::vector& udafContextsColl = *udafContextsCollPtr; + // Turn on NULL flags + flags[0] |= mcsv1sdk::PARAM_IS_NULL; + } - // Get the user data - boost::shared_ptr userDataIn = rowIn.getUserData(colIn + 1); + udafContextsColl[funcColsIdx].setDataFlags(flags); - // Unlike other aggregates, the data isn't in colIn, so testing it for NULL - // there won't help. In case of NULL, userData will be NULL. - uint32_t flags[1]; + // The intermediate values are stored in colAux. + udafContextsColl[funcColsIdx].setUserData(fRow.getUserData(colAux)); - flags[0] = 0; + // Call the UDAF subEvaluate method + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + rc = udafContextsColl[funcColsIdx].getFunction()->subEvaluate(&udafContextsColl[funcColsIdx], + userDataIn.get()); + udafContextsColl[funcColsIdx].setUserData(NULL); - if (!userDataIn) - { - if (udafContextsColl[funcColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) - { - return; - } - - // Turn on NULL flags - flags[0] |= mcsv1sdk::PARAM_IS_NULL; - } - - udafContextsColl[funcColsIdx].setDataFlags(flags); - - // The intermediate values are stored in colAux. - udafContextsColl[funcColsIdx].setUserData(fRow.getUserData(colAux)); - - // Call the UDAF subEvaluate method - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = udafContextsColl[funcColsIdx].getFunction()->subEvaluate(&udafContextsColl[funcColsIdx], userDataIn.get()); - udafContextsColl[funcColsIdx].setUserData(NULL); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - auto* rowUDAF = dynamic_cast(fFunctionCols[funcColsIdx].get()); - rowUDAF->bInterrupted = true; - throw logging::IDBExcept(udafContextsColl[funcColsIdx].getErrorMessage(), - logging::aggregateFuncErr); - } + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + auto* rowUDAF = dynamic_cast(fFunctionCols[funcColsIdx].get()); + rowUDAF->bInterrupted = true; + throw logging::IDBExcept(udafContextsColl[funcColsIdx].getErrorMessage(), logging::aggregateFuncErr); + } } - //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ RowAggregationDistinct::RowAggregationDistinct(const vector& rowAggGroupByCols, - const vector& rowAggFunctionCols, - joblist::ResourceManager* r, - boost::shared_ptr sessionLimit) : - RowAggregationUMP2(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) + const vector& rowAggFunctionCols, + joblist::ResourceManager* r, + boost::shared_ptr sessionLimit) + : RowAggregationUMP2(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) { - } - -RowAggregationDistinct::RowAggregationDistinct(const RowAggregationDistinct& rhs): - RowAggregationUMP2(rhs), - fRowGroupDist(rhs.fRowGroupDist) +RowAggregationDistinct::RowAggregationDistinct(const RowAggregationDistinct& rhs) + : RowAggregationUMP2(rhs), fRowGroupDist(rhs.fRowGroupDist) { - fAggregator.reset(rhs.fAggregator->clone()); + fAggregator.reset(rhs.fAggregator->clone()); } - RowAggregationDistinct::~RowAggregationDistinct() { } - //------------------------------------------------------------------------------ // Aggregation // //------------------------------------------------------------------------------ void RowAggregationDistinct::setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) { - fRowGroupIn = fRowGroupDist; - fRowGroupOut = pRowGroupOut; - initialize(); - fDataForDist.reinit(fRowGroupDist, - RowAggStorage::getMaxRows(fRm ? fRm->getAllowDiskAggregation() : false)); - fRowGroupDist.setData(&fDataForDist); - fAggregator->setInputOutput(pRowGroupIn, &fRowGroupDist); + fRowGroupIn = fRowGroupDist; + fRowGroupOut = pRowGroupOut; + initialize(); + fDataForDist.reinit(fRowGroupDist, RowAggStorage::getMaxRows(fRm ? fRm->getAllowDiskAggregation() : false)); + fRowGroupDist.setData(&fDataForDist); + fAggregator->setInputOutput(pRowGroupIn, &fRowGroupDist); } - //------------------------------------------------------------------------------ // Aggregation DISTINCT columns // //------------------------------------------------------------------------------ -void RowAggregationDistinct::addAggregator(const boost::shared_ptr& agg, - const RowGroup& rg) +void RowAggregationDistinct::addAggregator(const boost::shared_ptr& agg, const RowGroup& rg) { - fRowGroupDist = rg; - fAggregator = agg; + fRowGroupDist = rg; + fAggregator = agg; } - //------------------------------------------------------------------------------ // Aggregation DISTINCT columns // //------------------------------------------------------------------------------ void RowAggregationDistinct::addRowGroup(const RowGroup* pRows) { - fAggregator->addRowGroup(pRows); + fAggregator->addRowGroup(pRows); } - -void RowAggregationDistinct::addRowGroup(const RowGroup* pRows, vector>& inRows) +void RowAggregationDistinct::addRowGroup(const RowGroup* pRows, + vector>& inRows) { - fAggregator->addRowGroup(pRows, inRows); + fAggregator->addRowGroup(pRows, inRows); } - //------------------------------------------------------------------------------ // Aggregation DISTINCT columns // //------------------------------------------------------------------------------ void RowAggregationDistinct::doDistinctAggregation() { - while (dynamic_cast(fAggregator.get())->nextRowGroup()) + while (dynamic_cast(fAggregator.get())->nextRowGroup()) + { + fRowGroupIn.setData(fAggregator->getOutputRowGroup()->getRGData()); + + Row rowIn; + fRowGroupIn.initRow(&rowIn); + fRowGroupIn.getRow(0, &rowIn); + + for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i, rowIn.nextRow()) { - fRowGroupIn.setData(fAggregator->getOutputRowGroup()->getRGData()); - - Row rowIn; - fRowGroupIn.initRow(&rowIn); - fRowGroupIn.getRow(0, &rowIn); - - for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i, rowIn.nextRow()) - { - aggregateRow(rowIn); - } + aggregateRow(rowIn); } + } } - void RowAggregationDistinct::doDistinctAggregation_rowVec(vector>& inRows) { - Row rowIn; - fRowGroupIn.initRow(&rowIn); + Row rowIn; + fRowGroupIn.initRow(&rowIn); - for (uint64_t i = 0; i < inRows.size(); ++i) - { - rowIn.setData(inRows[i].first); - aggregateRow(rowIn, &inRows[i].second); - } + for (uint64_t i = 0; i < inRows.size(); ++i) + { + rowIn.setData(inRows[i].first); + aggregateRow(rowIn, &inRows[i].second); + } } - //------------------------------------------------------------------------------ // Update the aggregation totals in the internal hashmap for the specified row. // for non-DISTINCT columns works partially aggregated results // rowIn(in) - Row to be included in aggregation. // rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationDistinct::updateEntry(const Row& rowIn, - std::vector* rgContextColl) +void RowAggregationDistinct::updateEntry(const Row& rowIn, std::vector* rgContextColl) { - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + int64_t colIn = fFunctionCols[i]->fInputColumnIndex; + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; + int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + + switch (fFunctionCols[i]->fAggFunction) { - int64_t colIn = fFunctionCols[i]->fInputColumnIndex; - int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; + case ROWAGG_COUNT_ASTERISK: + case ROWAGG_COUNT_COL_NAME: + { + uint64_t count = fRow.getUintField<8>(colOut) + rowIn.getUintField<8>(colIn); + fRow.setUintField<8>(count, colOut); + break; + } - switch (fFunctionCols[i]->fAggFunction) - { - case ROWAGG_COUNT_ASTERISK: - case ROWAGG_COUNT_COL_NAME: - { - uint64_t count = fRow.getUintField<8>(colOut) + rowIn.getUintField<8>(colIn); - fRow.setUintField<8>(count, colOut); - break; - } + case ROWAGG_COUNT_DISTINCT_COL_NAME: + if (isNull(&fRowGroupIn, rowIn, colIn) != true) + fRow.setUintField<8>(fRow.getUintField<8>(colOut) + 1, colOut); - case ROWAGG_COUNT_DISTINCT_COL_NAME: - if (isNull(&fRowGroupIn, rowIn, colIn) != true) - fRow.setUintField<8>(fRow.getUintField<8>(colOut) + 1, colOut); + break; - break; + case ROWAGG_MIN: + case ROWAGG_MAX: doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_MIN: - case ROWAGG_MAX: - doMinMax(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_SUM: + case ROWAGG_DISTINCT_SUM: doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; - case ROWAGG_SUM: - case ROWAGG_DISTINCT_SUM: - doSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; + case ROWAGG_AVG: + { + // The sum and count on UM may not be put next to each other: + // use colOut to store the sum; + // use colAux to store the count. + doAvg(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_AVG: - { - // The sum and count on UM may not be put next to each other: - // use colOut to store the sum; - // use colAux to store the count. - doAvg(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_DISTINCT_AVG: + { + // The sum and count on UM may not be put next to each other: + // use colOut to store the sum; + // use colAux to store the count. + RowAggregation::doAvg(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_DISTINCT_AVG: - { - // The sum and count on UM may not be put next to each other: - // use colOut to store the sum; - // use colAux to store the count. - RowAggregation::doAvg(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_STATS: + { + doStatistics(rowIn, colIn, colOut, colAux); + break; + } - case ROWAGG_STATS: - { - doStatistics(rowIn, colIn, colOut, colAux); - break; - } + case ROWAGG_BIT_AND: + case ROWAGG_BIT_OR: + case ROWAGG_BIT_XOR: + { + doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); + break; + } - case ROWAGG_BIT_AND: - case ROWAGG_BIT_OR: - case ROWAGG_BIT_XOR: - { - doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); - break; - } + case ROWAGG_GROUP_CONCAT: + { + doGroupConcat(rowIn, colIn, colOut); + break; + } - case ROWAGG_GROUP_CONCAT: - { - doGroupConcat(rowIn, colIn, colOut); - break; - } + case ROWAGG_COUNT_NO_OP: + case ROWAGG_DUP_FUNCT: + case ROWAGG_DUP_AVG: + case ROWAGG_DUP_STATS: + case ROWAGG_DUP_UDAF: + case ROWAGG_CONSTANT: break; - case ROWAGG_COUNT_NO_OP: - case ROWAGG_DUP_FUNCT: - case ROWAGG_DUP_AVG: - case ROWAGG_DUP_STATS: - case ROWAGG_DUP_UDAF: - case ROWAGG_CONSTANT: - break; + case ROWAGG_UDAF: + { + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); + break; + } - case ROWAGG_UDAF: - { - doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); - break; - } - - default: - { - std::ostringstream errmsg; - errmsg << "RowAggregationDistinct: function (id = " << - (uint64_t) fFunctionCols[i]->fAggFunction << ") is not supported."; - cerr << errmsg.str() << endl; - throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); - break; - } - } + default: + { + std::ostringstream errmsg; + errmsg << "RowAggregationDistinct: function (id = " << (uint64_t)fFunctionCols[i]->fAggFunction + << ") is not supported."; + cerr << errmsg.str() << endl; + throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); + break; + } } + } } - //------------------------------------------------------------------------------ // Constructor / destructor //------------------------------------------------------------------------------ -RowAggregationSubDistinct::RowAggregationSubDistinct( - const vector& rowAggGroupByCols, - const vector& rowAggFunctionCols, - joblist::ResourceManager* r, - boost::shared_ptr sessionLimit) : - RowAggregationUM(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) +RowAggregationSubDistinct::RowAggregationSubDistinct(const vector& rowAggGroupByCols, + const vector& rowAggFunctionCols, + joblist::ResourceManager* r, + boost::shared_ptr sessionLimit) + : RowAggregationUM(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) { - fKeyOnHeap = false; + fKeyOnHeap = false; } - -RowAggregationSubDistinct::RowAggregationSubDistinct(const RowAggregationSubDistinct& rhs): - RowAggregationUM(rhs) +RowAggregationSubDistinct::RowAggregationSubDistinct(const RowAggregationSubDistinct& rhs) + : RowAggregationUM(rhs) { } - RowAggregationSubDistinct::~RowAggregationSubDistinct() { } - //------------------------------------------------------------------------------ // Setup the rowgroups and data associations // //------------------------------------------------------------------------------ void RowAggregationSubDistinct::setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) { - // set up input/output association - RowAggregation::setInputOutput(pRowGroupIn, pRowGroupOut); + // set up input/output association + RowAggregation::setInputOutput(pRowGroupIn, pRowGroupOut); - // initialize the aggregate row - fRowGroupOut->initRow(&fDistRow, true); - fDistRowData.reset(new uint8_t[fDistRow.getSize()]); - fDistRow.setData(fDistRowData.get()); + // initialize the aggregate row + fRowGroupOut->initRow(&fDistRow, true); + fDistRowData.reset(new uint8_t[fDistRow.getSize()]); + fDistRow.setData(fDistRowData.get()); } - //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ // Add rowgroup @@ -4829,140 +4621,132 @@ void RowAggregationSubDistinct::setInputOutput(const RowGroup& pRowGroupIn, RowG //------------------------------------------------------------------------------ void RowAggregationSubDistinct::addRowGroup(const RowGroup* pRows) { - Row rowIn; - uint32_t i, j; + Row rowIn; + uint32_t i, j; - pRows->initRow(&rowIn); - pRows->getRow(0, &rowIn); + pRows->initRow(&rowIn); + pRows->getRow(0, &rowIn); - for (i = 0; i < pRows->getRowCount(); ++i, rowIn.nextRow()) + for (i = 0; i < pRows->getRowCount(); ++i, rowIn.nextRow()) + { + /* TODO: We can make the functors a little smarter and avoid doing this copy before the + * tentative insert */ + for (j = 0; j < fGroupByCols.size(); j++) { - /* TODO: We can make the functors a little smarter and avoid doing this copy before the - * tentative insert */ - for (j = 0; j < fGroupByCols.size(); j++) - { - rowIn.copyField(fDistRow, j, fGroupByCols[j]->fInputColumnIndex); - } - - tmpRow = &fDistRow; - if (fRowAggStorage->getTargetRow(fDistRow, fRow)) - { - copyRow(fDistRow, &fRow); - } + rowIn.copyField(fDistRow, j, fGroupByCols[j]->fInputColumnIndex); } + + tmpRow = &fDistRow; + if (fRowAggStorage->getTargetRow(fDistRow, fRow)) + { + copyRow(fDistRow, &fRow); + } + } } -void RowAggregationSubDistinct::addRowGroup(const RowGroup* pRows, std::vector>& inRows) +void RowAggregationSubDistinct::addRowGroup(const RowGroup* pRows, + std::vector>& inRows) { - Row rowIn; - uint32_t i, j; + Row rowIn; + uint32_t i, j; - pRows->initRow(&rowIn); + pRows->initRow(&rowIn); - for (i = 0; i < inRows.size(); ++i, rowIn.nextRow()) + for (i = 0; i < inRows.size(); ++i, rowIn.nextRow()) + { + rowIn.setData(inRows[i].first); + + /* TODO: We can make the functors a little smarter and avoid doing this copy before the + * tentative insert */ + for (j = 0; j < fGroupByCols.size(); j++) + rowIn.copyField(fDistRow, j, fGroupByCols[j]->fInputColumnIndex); + + tmpRow = &fDistRow; + if (fRowAggStorage->getTargetRow(fDistRow, fRow)) { - rowIn.setData(inRows[i].first); - - /* TODO: We can make the functors a little smarter and avoid doing this copy before the - * tentative insert */ - for (j = 0; j < fGroupByCols.size(); j++) - rowIn.copyField(fDistRow, j, fGroupByCols[j]->fInputColumnIndex); - - tmpRow = &fDistRow; - if (fRowAggStorage->getTargetRow(fDistRow, fRow)) - { - copyRow(fDistRow, &fRow); - } + copyRow(fDistRow, &fRow); } + } } - //------------------------------------------------------------------------------ // Concat columns. // rowIn(in) - Row that contains the columns to be concatenated. //------------------------------------------------------------------------------ void RowAggregationSubDistinct::doGroupConcat(const Row& rowIn, int64_t i, int64_t o) { - uint8_t* data = fRow.getData(); - joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + fRow.getOffset(o))); - gccAg->merge(rowIn, i); + uint8_t* data = fRow.getData(); + joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + fRow.getOffset(o))); + gccAg->merge(rowIn, i); } - //------------------------------------------------------------------------------ // Constructor / destructor //------------------------------------------------------------------------------ -RowAggregationMultiDistinct::RowAggregationMultiDistinct( - const vector& rowAggGroupByCols, - const vector& rowAggFunctionCols, - joblist::ResourceManager* r, - boost::shared_ptr sessionLimit) : - RowAggregationDistinct(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) +RowAggregationMultiDistinct::RowAggregationMultiDistinct(const vector& rowAggGroupByCols, + const vector& rowAggFunctionCols, + joblist::ResourceManager* r, + boost::shared_ptr sessionLimit) + : RowAggregationDistinct(rowAggGroupByCols, rowAggFunctionCols, r, sessionLimit) { } - -RowAggregationMultiDistinct::RowAggregationMultiDistinct(const RowAggregationMultiDistinct& rhs): - RowAggregationDistinct(rhs), - fSubRowGroups(rhs.fSubRowGroups), - fSubFunctions(rhs.fSubFunctions) +RowAggregationMultiDistinct::RowAggregationMultiDistinct(const RowAggregationMultiDistinct& rhs) + : RowAggregationDistinct(rhs), fSubRowGroups(rhs.fSubRowGroups), fSubFunctions(rhs.fSubFunctions) { - fAggregator.reset(rhs.fAggregator->clone()); + fAggregator.reset(rhs.fAggregator->clone()); - boost::shared_ptr data; - fSubAggregators.clear(); - fSubRowData.clear(); + boost::shared_ptr data; + fSubAggregators.clear(); + fSubRowData.clear(); - boost::shared_ptr agg; + boost::shared_ptr agg; - for (uint32_t i = 0; i < rhs.fSubAggregators.size(); i++) - { + for (uint32_t i = 0; i < rhs.fSubAggregators.size(); i++) + { #if 0 if (!fRm->getMemory(fSubRowGroups[i].getDataSize(AGG_ROWGROUP_SIZE, fSessionMemLimit))) throw logging::IDBExcept(logging::IDBErrorInfo::instance()-> errorMsg(logging::ERR_AGGREGATION_TOO_BIG), logging::ERR_AGGREGATION_TOO_BIG); fTotalMemUsage += fSubRowGroups[i].getDataSize(AGG_ROWGROUP_SIZE); - -#endif - data.reset(new RGData(fSubRowGroups[i], RowAggStorage::getMaxRows(fRm ? fRm->getAllowDiskAggregation() : false))); - fSubRowData.push_back(data); - fSubRowGroups[i].setData(data.get()); - agg.reset(rhs.fSubAggregators[i]->clone()); - fSubAggregators.push_back(agg); - } -} +#endif + data.reset(new RGData(fSubRowGroups[i], + RowAggStorage::getMaxRows(fRm ? fRm->getAllowDiskAggregation() : false))); + fSubRowData.push_back(data); + fSubRowGroups[i].setData(data.get()); + agg.reset(rhs.fSubAggregators[i]->clone()); + fSubAggregators.push_back(agg); + } +} RowAggregationMultiDistinct::~RowAggregationMultiDistinct() { } - //------------------------------------------------------------------------------ // Setup the rowgroups and data associations // //------------------------------------------------------------------------------ void RowAggregationMultiDistinct::setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) { - // set up base class aggregators - RowAggregationDistinct::setInputOutput(pRowGroupIn, pRowGroupOut); + // set up base class aggregators + RowAggregationDistinct::setInputOutput(pRowGroupIn, pRowGroupOut); - // set up sub aggregators - for (uint64_t i = 0; i < fSubAggregators.size(); ++i) - fSubAggregators[i]->setInputOutput(pRowGroupIn, &fSubRowGroups[i]); + // set up sub aggregators + for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + fSubAggregators[i]->setInputOutput(pRowGroupIn, &fSubRowGroups[i]); } - //------------------------------------------------------------------------------ // Add sub aggregator for each distinct column with aggregate functions // //------------------------------------------------------------------------------ void RowAggregationMultiDistinct::addSubAggregator(const boost::shared_ptr& agg, - const RowGroup& rg, - const vector& funct) + const RowGroup& rg, const vector& funct) { - boost::shared_ptr data; + boost::shared_ptr data; #if 0 if (!fRm->getMemory(rg.getDataSize(AGG_ROWGROUP_SIZE), fSessionMemLimit)) throw logging::IDBExcept(logging::IDBErrorInfo::instance()-> @@ -4970,118 +4754,113 @@ void RowAggregationMultiDistinct::addSubAggregator(const boost::shared_ptrgetAllowDiskAggregation() : false))); - fSubRowData.push_back(data); + data.reset(new RGData(rg, RowAggStorage::getMaxRows(fRm ? fRm->getAllowDiskAggregation() : false))); + fSubRowData.push_back(data); - //assert (agg->aggMapKeyLength() > 0); + // assert (agg->aggMapKeyLength() > 0); - fSubAggregators.push_back(agg); - fSubRowGroups.push_back(rg); - fSubRowGroups.back().setData(data.get()); - fSubFunctions.push_back(funct); + fSubAggregators.push_back(agg); + fSubRowGroups.push_back(rg); + fSubRowGroups.back().setData(data.get()); + fSubFunctions.push_back(funct); } - void RowAggregationMultiDistinct::addRowGroup(const RowGroup* pRows) { - // aggregate to sub-subAggregators - for (uint64_t i = 0; i < fSubAggregators.size(); ++i) - fSubAggregators[i]->addRowGroup(pRows); + // aggregate to sub-subAggregators + for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + fSubAggregators[i]->addRowGroup(pRows); } - //------------------------------------------------------------------------------ // Aggregation DISTINCT columns // //------------------------------------------------------------------------------ void RowAggregationMultiDistinct::addRowGroup(const RowGroup* pRowGroupIn, - vector> >& inRows) + vector>>& inRows) { - for (uint64_t i = 0; i < fSubAggregators.size(); ++i) - { - fSubAggregators[i]->addRowGroup(pRowGroupIn, inRows[i]); - inRows[i].clear(); - } + for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + { + fSubAggregators[i]->addRowGroup(pRowGroupIn, inRows[i]); + inRows[i].clear(); + } } - //------------------------------------------------------------------------------ // Aggregation DISTINCT columns // //------------------------------------------------------------------------------ void RowAggregationMultiDistinct::doDistinctAggregation() { - // backup the function column vector for finalize(). - vector origFunctionCols = fFunctionCols; - fOrigFunctionCols = &origFunctionCols; - // aggregate data from each sub-aggregator to distinct aggregator - for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + // backup the function column vector for finalize(). + vector origFunctionCols = fFunctionCols; + fOrigFunctionCols = &origFunctionCols; + // aggregate data from each sub-aggregator to distinct aggregator + for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + { + fFunctionCols = fSubFunctions[i]; + fRowGroupIn = fSubRowGroups[i]; + auto* rgContextColl = fSubAggregators[i]->rgContextColl(); + Row rowIn; + fRowGroupIn.initRow(&rowIn); + + while (dynamic_cast(fSubAggregators[i].get())->nextRowGroup()) { - fFunctionCols = fSubFunctions[i]; - fRowGroupIn = fSubRowGroups[i]; - auto* rgContextColl = fSubAggregators[i]->rgContextColl(); - Row rowIn; - fRowGroupIn.initRow(&rowIn); + fRowGroupIn.setData(fSubAggregators[i]->getOutputRowGroup()->getRGData()); - while (dynamic_cast(fSubAggregators[i].get())->nextRowGroup()) - { - fRowGroupIn.setData(fSubAggregators[i]->getOutputRowGroup()->getRGData()); + // no group by == no map, everything done in fRow + if (fGroupByCols.empty()) + fRowGroupOut->setRowCount(1); - // no group by == no map, everything done in fRow - if (fGroupByCols.empty()) - fRowGroupOut->setRowCount(1); + fRowGroupIn.getRow(0, &rowIn); - fRowGroupIn.getRow(0, &rowIn); - - for (uint64_t j = 0; j < fRowGroupIn.getRowCount(); ++j, rowIn.nextRow()) - { - aggregateRow(rowIn, nullptr, rgContextColl); - } - } + for (uint64_t j = 0; j < fRowGroupIn.getRowCount(); ++j, rowIn.nextRow()) + { + aggregateRow(rowIn, nullptr, rgContextColl); + } } + } - // restore the function column vector - fFunctionCols = origFunctionCols; - fOrigFunctionCols = nullptr; + // restore the function column vector + fFunctionCols = origFunctionCols; + fOrigFunctionCols = nullptr; } - -void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vector> >& inRows) +void RowAggregationMultiDistinct::doDistinctAggregation_rowVec( + vector>>& inRows) { - // backup the function column vector for finalize(). - vector origFunctionCols = fFunctionCols; - fOrigFunctionCols = &origFunctionCols; + // backup the function column vector for finalize(). + vector origFunctionCols = fFunctionCols; + fOrigFunctionCols = &origFunctionCols; - // aggregate data from each sub-aggregator to distinct aggregator - for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + // aggregate data from each sub-aggregator to distinct aggregator + for (uint64_t i = 0; i < fSubAggregators.size(); ++i) + { + fFunctionCols = fSubFunctions[i]; + fRowGroupIn = fSubRowGroups[i]; + auto* rgContextColl = fSubAggregators[i]->rgContextColl(); + Row rowIn; + fRowGroupIn.initRow(&rowIn); + + for (uint64_t j = 0; j < inRows[i].size(); ++j) { - fFunctionCols = fSubFunctions[i]; - fRowGroupIn = fSubRowGroups[i]; - auto* rgContextColl = fSubAggregators[i]->rgContextColl(); - Row rowIn; - fRowGroupIn.initRow(&rowIn); - - for (uint64_t j = 0; j < inRows[i].size(); ++j) - { - rowIn.setData(inRows[i][j].first); - aggregateRow(rowIn, &inRows[i][j].second, rgContextColl); - } - - inRows[i].clear(); + rowIn.setData(inRows[i][j].first); + aggregateRow(rowIn, &inRows[i][j].second, rgContextColl); } - // restore the function column vector - fFunctionCols = origFunctionCols; - fOrigFunctionCols = nullptr; -} + inRows[i].clear(); + } + // restore the function column vector + fFunctionCols = origFunctionCols; + fOrigFunctionCols = nullptr; +} GroupConcatAg::GroupConcatAg(SP_GroupConcat& gcc) : fGroupConcat(gcc) { } - GroupConcatAg::~GroupConcatAg() { } -} // end of rowgroup namespace +} // namespace rowgroup diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index be785fcff..ae084e1f5 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -54,7 +54,6 @@ #include "mcsv1_udaf.h" #include "constantcolumn.h" - #include "resourcemanager.h" #include "rowstorage.h" @@ -66,62 +65,60 @@ class ResourceManager; namespace rowgroup { - /** @brief Enumerates aggregate functions supported by RowAggregation */ enum RowAggFunctionType { - ROWAGG_FUNCT_UNDEFINE, // default - ROWAGG_COUNT_ASTERISK, // COUNT(*) counts all rows including nulls - ROWAGG_COUNT_COL_NAME, // COUNT(column_name) only counts non-null rows - ROWAGG_SUM, - ROWAGG_AVG, - ROWAGG_MIN, - ROWAGG_MAX, + ROWAGG_FUNCT_UNDEFINE, // default + ROWAGG_COUNT_ASTERISK, // COUNT(*) counts all rows including nulls + ROWAGG_COUNT_COL_NAME, // COUNT(column_name) only counts non-null rows + ROWAGG_SUM, + ROWAGG_AVG, + ROWAGG_MIN, + ROWAGG_MAX, - // Statistics Function, ROWAGG_STATS is the generic name. - ROWAGG_STATS, - ROWAGG_STDDEV_POP, - ROWAGG_STDDEV_SAMP, - ROWAGG_VAR_POP, - ROWAGG_VAR_SAMP, + // Statistics Function, ROWAGG_STATS is the generic name. + ROWAGG_STATS, + ROWAGG_STDDEV_POP, + ROWAGG_STDDEV_SAMP, + ROWAGG_VAR_POP, + ROWAGG_VAR_SAMP, - // BIT Function, ROWAGG_BIT_OP is the generic name. - ROWAGG_BIT_OP, - ROWAGG_BIT_AND, - ROWAGG_BIT_OR, - ROWAGG_BIT_XOR, + // BIT Function, ROWAGG_BIT_OP is the generic name. + ROWAGG_BIT_OP, + ROWAGG_BIT_AND, + ROWAGG_BIT_OR, + ROWAGG_BIT_XOR, - // GROUP_CONCAT - ROWAGG_GROUP_CONCAT, + // GROUP_CONCAT + ROWAGG_GROUP_CONCAT, - // DISTINCT: performed on UM only - ROWAGG_COUNT_DISTINCT_COL_NAME, // COUNT(distinct column_name) only counts non-null rows - ROWAGG_DISTINCT_SUM, - ROWAGG_DISTINCT_AVG, + // DISTINCT: performed on UM only + ROWAGG_COUNT_DISTINCT_COL_NAME, // COUNT(distinct column_name) only counts non-null rows + ROWAGG_DISTINCT_SUM, + ROWAGG_DISTINCT_AVG, - // Constant - ROWAGG_CONSTANT, + // Constant + ROWAGG_CONSTANT, - // User Defined Aggregate Function - ROWAGG_UDAF, + // User Defined Aggregate Function + ROWAGG_UDAF, - // If an Aggregate has more than one parameter, this will be used for parameters after the first - ROWAGG_MULTI_PARM, + // If an Aggregate has more than one parameter, this will be used for parameters after the first + ROWAGG_MULTI_PARM, - // internal function type to avoid duplicate the work - // handling ROWAGG_COUNT_NO_OP, ROWAGG_DUP_FUNCT and ROWAGG_DUP_AVG is a little different - // ROWAGG_COUNT_NO_OP : count done by AVG, no need to copy - // ROWAGG_DUP_FUNCT : copy data before AVG calculation, because SUM may share by AVG - // ROWAGG_DUP_AVG : copy data after AVG calculation - ROWAGG_COUNT_NO_OP, // COUNT(column_name), but leave count() to AVG - ROWAGG_DUP_FUNCT, // duplicate aggregate Function(), except AVG and UDAF, in select - ROWAGG_DUP_AVG, // duplicate AVG(column_name) in select - ROWAGG_DUP_STATS, // duplicate statistics functions in select - ROWAGG_DUP_UDAF // duplicate UDAF function in select + // internal function type to avoid duplicate the work + // handling ROWAGG_COUNT_NO_OP, ROWAGG_DUP_FUNCT and ROWAGG_DUP_AVG is a little different + // ROWAGG_COUNT_NO_OP : count done by AVG, no need to copy + // ROWAGG_DUP_FUNCT : copy data before AVG calculation, because SUM may share by AVG + // ROWAGG_DUP_AVG : copy data after AVG calculation + ROWAGG_COUNT_NO_OP, // COUNT(column_name), but leave count() to AVG + ROWAGG_DUP_FUNCT, // duplicate aggregate Function(), except AVG and UDAF, in select + ROWAGG_DUP_AVG, // duplicate AVG(column_name) in select + ROWAGG_DUP_STATS, // duplicate statistics functions in select + ROWAGG_DUP_UDAF // duplicate UDAF function in select }; - //------------------------------------------------------------------------------ /** @brief Specifies a column in a RowGroup that is part of the aggregation * "GROUP BY" clause. @@ -129,28 +126,30 @@ enum RowAggFunctionType //------------------------------------------------------------------------------ struct RowAggGroupByCol { - /** @brief RowAggGroupByCol constructor - * - * @param inputColIndex(in) column index into input row - * @param outputColIndex(in) column index into output row - * outputColIndex argument should be omitted if this GroupBy - * column is not to be included in the output. - */ - explicit RowAggGroupByCol(int32_t inputColIndex, int32_t outputColIndex = -1) : - fInputColumnIndex(inputColIndex), fOutputColumnIndex(outputColIndex) {} - ~RowAggGroupByCol() = default; + /** @brief RowAggGroupByCol constructor + * + * @param inputColIndex(in) column index into input row + * @param outputColIndex(in) column index into output row + * outputColIndex argument should be omitted if this GroupBy + * column is not to be included in the output. + */ + explicit RowAggGroupByCol(int32_t inputColIndex, int32_t outputColIndex = -1) + : fInputColumnIndex(inputColIndex), fOutputColumnIndex(outputColIndex) + { + } + ~RowAggGroupByCol() = default; - uint32_t fInputColumnIndex; - uint32_t fOutputColumnIndex; + uint32_t fInputColumnIndex; + uint32_t fOutputColumnIndex; }; inline messageqcpp::ByteStream& operator<<(messageqcpp::ByteStream& b, RowAggGroupByCol& o) { - return (b << o.fInputColumnIndex << o.fOutputColumnIndex); + return (b << o.fInputColumnIndex << o.fOutputColumnIndex); } inline messageqcpp::ByteStream& operator>>(messageqcpp::ByteStream& b, RowAggGroupByCol& o) { - return (b >> o.fInputColumnIndex >> o.fOutputColumnIndex); + return (b >> o.fInputColumnIndex >> o.fOutputColumnIndex); } //------------------------------------------------------------------------------ @@ -165,208 +164,211 @@ inline messageqcpp::ByteStream& operator>>(messageqcpp::ByteStream& b, RowAggGro //------------------------------------------------------------------------------ struct RowAggFunctionCol { - /** @brief RowAggFunctionCol constructor - * - * @param aggFunction(in) aggregation function to be performed - * @param inputColIndex(in) column index into input row - * @param outputColIndex(in) column index into output row - * @param auxColIndex(in) auxiliary index into output row for avg/count - * @param stats(in) real statistics function where generic name in aggFunction - */ - RowAggFunctionCol(RowAggFunctionType aggFunction, RowAggFunctionType stats, - int32_t inputColIndex, int32_t outputColIndex, int32_t auxColIndex = -1) : - fAggFunction(aggFunction), fStatsFunction(stats), fInputColumnIndex(inputColIndex), - fOutputColumnIndex(outputColIndex), fAuxColumnIndex(auxColIndex), hasMultiParm(false) {} - virtual ~RowAggFunctionCol() = default; + /** @brief RowAggFunctionCol constructor + * + * @param aggFunction(in) aggregation function to be performed + * @param inputColIndex(in) column index into input row + * @param outputColIndex(in) column index into output row + * @param auxColIndex(in) auxiliary index into output row for avg/count + * @param stats(in) real statistics function where generic name in aggFunction + */ + RowAggFunctionCol(RowAggFunctionType aggFunction, RowAggFunctionType stats, int32_t inputColIndex, + int32_t outputColIndex, int32_t auxColIndex = -1) + : fAggFunction(aggFunction) + , fStatsFunction(stats) + , fInputColumnIndex(inputColIndex) + , fOutputColumnIndex(outputColIndex) + , fAuxColumnIndex(auxColIndex) + , hasMultiParm(false) + { + } + virtual ~RowAggFunctionCol() = default; - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void deserialize(messageqcpp::ByteStream& bs); + virtual void serialize(messageqcpp::ByteStream& bs) const; + virtual void deserialize(messageqcpp::ByteStream& bs); - RowAggFunctionType fAggFunction; // aggregate function - // statistics function stores ROWAGG_STATS in fAggFunction and real function in fStatsFunction - RowAggFunctionType fStatsFunction; + RowAggFunctionType fAggFunction; // aggregate function + // statistics function stores ROWAGG_STATS in fAggFunction and real function in fStatsFunction + RowAggFunctionType fStatsFunction; - uint32_t fInputColumnIndex; - uint32_t fOutputColumnIndex; + uint32_t fInputColumnIndex; + uint32_t fOutputColumnIndex; - // fAuxColumnIndex is used in 4 cases: - // 1. for AVG - point to the count column, the fInputColumnIndex is for sum - // 2. for statistics function - point to sum(x), +1 is sum(x**2) - // 3. for UDAF - contain the context user data as binary - // 4. for duplicate - point to the real aggretate column to be copied from - // Set only on UM, the fAuxColumnIndex is defaulted to fOutputColumnIndex+1 on PM. - uint32_t fAuxColumnIndex; + // fAuxColumnIndex is used in 4 cases: + // 1. for AVG - point to the count column, the fInputColumnIndex is for sum + // 2. for statistics function - point to sum(x), +1 is sum(x**2) + // 3. for UDAF - contain the context user data as binary + // 4. for duplicate - point to the real aggretate column to be copied from + // Set only on UM, the fAuxColumnIndex is defaulted to fOutputColumnIndex+1 on PM. + uint32_t fAuxColumnIndex; - // For UDAF that have more than one parameter and some parameters are constant. - // There will be a series of RowAggFunctionCol created, one for each parameter. - // The first will be a RowUDAFFunctionCol. Subsequent ones will be RowAggFunctionCol - // with fAggFunction == ROWAGG_MULTI_PARM. Order is important. - // If this parameter is constant, that value is here. - execplan::SRCP fpConstCol; - - bool hasMultiParm; + // For UDAF that have more than one parameter and some parameters are constant. + // There will be a series of RowAggFunctionCol created, one for each parameter. + // The first will be a RowUDAFFunctionCol. Subsequent ones will be RowAggFunctionCol + // with fAggFunction == ROWAGG_MULTI_PARM. Order is important. + // If this parameter is constant, that value is here. + execplan::SRCP fpConstCol; + + bool hasMultiParm; }; - struct RowUDAFFunctionCol : public RowAggFunctionCol { - RowUDAFFunctionCol(mcsv1sdk::mcsv1Context& context, int32_t inputColIndex, - int32_t outputColIndex, int32_t auxColIndex = -1) : - RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, - inputColIndex, outputColIndex, auxColIndex), - fUDAFContext(context), bInterrupted(false) - { - fUDAFContext.setInterrupted(&bInterrupted); - } + RowUDAFFunctionCol(mcsv1sdk::mcsv1Context& context, int32_t inputColIndex, int32_t outputColIndex, + int32_t auxColIndex = -1) + : RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, inputColIndex, outputColIndex, auxColIndex) + , fUDAFContext(context) + , bInterrupted(false) + { + fUDAFContext.setInterrupted(&bInterrupted); + } - RowUDAFFunctionCol(int32_t inputColIndex, - int32_t outputColIndex, int32_t auxColIndex = -1) : - RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, - inputColIndex, outputColIndex, auxColIndex), - bInterrupted(false) - {} - RowUDAFFunctionCol(const RowUDAFFunctionCol& rhs) : - RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, rhs.fInputColumnIndex, - rhs.fOutputColumnIndex, rhs.fAuxColumnIndex), - fUDAFContext(rhs.fUDAFContext), - bInterrupted(false) - {} + RowUDAFFunctionCol(int32_t inputColIndex, int32_t outputColIndex, int32_t auxColIndex = -1) + : RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, inputColIndex, outputColIndex, auxColIndex) + , bInterrupted(false) + { + } + RowUDAFFunctionCol(const RowUDAFFunctionCol& rhs) + : RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, rhs.fInputColumnIndex, rhs.fOutputColumnIndex, + rhs.fAuxColumnIndex) + , fUDAFContext(rhs.fUDAFContext) + , bInterrupted(false) + { + } - ~RowUDAFFunctionCol() override = default; + ~RowUDAFFunctionCol() override = default; - void serialize(messageqcpp::ByteStream& bs) const override; - void deserialize(messageqcpp::ByteStream& bs) override; + void serialize(messageqcpp::ByteStream& bs) const override; + void deserialize(messageqcpp::ByteStream& bs) override; - mcsv1sdk::mcsv1Context fUDAFContext; // The UDAF context - bool bInterrupted; // Shared by all the threads - }; + mcsv1sdk::mcsv1Context fUDAFContext; // The UDAF context + bool bInterrupted; // Shared by all the threads +}; inline void RowAggFunctionCol::serialize(messageqcpp::ByteStream& bs) const { - bs << (uint8_t)fAggFunction; - bs << fInputColumnIndex; - bs << fOutputColumnIndex; - - if (fpConstCol) - { - bs << (uint8_t)1; - fpConstCol.get()->serialize(bs); - } - else - { - bs << (uint8_t)0; - } + bs << (uint8_t)fAggFunction; + bs << fInputColumnIndex; + bs << fOutputColumnIndex; + if (fpConstCol) + { + bs << (uint8_t)1; + fpConstCol.get()->serialize(bs); + } + else + { + bs << (uint8_t)0; + } } inline void RowAggFunctionCol::deserialize(messageqcpp::ByteStream& bs) { - bs >> (uint8_t&)fAggFunction; - bs >> fInputColumnIndex; - bs >> fOutputColumnIndex; - uint8_t t; - bs >> t; + bs >> (uint8_t&)fAggFunction; + bs >> fInputColumnIndex; + bs >> fOutputColumnIndex; + uint8_t t; + bs >> t; - if (t) - { - fpConstCol.reset(new execplan::ConstantColumn); - fpConstCol.get()->unserialize(bs); - } + if (t) + { + fpConstCol.reset(new execplan::ConstantColumn); + fpConstCol.get()->unserialize(bs); + } } inline void RowUDAFFunctionCol::serialize(messageqcpp::ByteStream& bs) const { - RowAggFunctionCol::serialize(bs); - fUDAFContext.serialize(bs); + RowAggFunctionCol::serialize(bs); + fUDAFContext.serialize(bs); } inline void RowUDAFFunctionCol::deserialize(messageqcpp::ByteStream& bs) { - // This deserialize is called when the function gets to PrimProc. - // reset is called because we're starting a new sub-evaluate cycle. - RowAggFunctionCol::deserialize(bs); - fUDAFContext.unserialize(bs); - fUDAFContext.setInterrupted(&bInterrupted); - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fUDAFContext.getFunction()->reset(&fUDAFContext); + // This deserialize is called when the function gets to PrimProc. + // reset is called because we're starting a new sub-evaluate cycle. + RowAggFunctionCol::deserialize(bs); + fUDAFContext.unserialize(bs); + fUDAFContext.setInterrupted(&bInterrupted); + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + rc = fUDAFContext.getFunction()->reset(&fUDAFContext); - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - bInterrupted = true; - throw logging::QueryDataExcept(fUDAFContext.getErrorMessage(), logging::aggregateFuncErr); - } + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + bInterrupted = true; + throw logging::QueryDataExcept(fUDAFContext.getErrorMessage(), logging::aggregateFuncErr); + } } struct ConstantAggData { - std::string fConstValue; - std::string fUDAFName; // If a UDAF is called with constant. - RowAggFunctionType fOp; - bool fIsNull; + std::string fConstValue; + std::string fUDAFName; // If a UDAF is called with constant. + RowAggFunctionType fOp; + bool fIsNull; - ConstantAggData() : fOp(ROWAGG_FUNCT_UNDEFINE), fIsNull(false) - {} + ConstantAggData() : fOp(ROWAGG_FUNCT_UNDEFINE), fIsNull(false) + { + } - ConstantAggData(std::string v, RowAggFunctionType f, bool n) : - fConstValue(std::move(v)), fOp(f), fIsNull(n) - {} + ConstantAggData(std::string v, RowAggFunctionType f, bool n) : fConstValue(std::move(v)), fOp(f), fIsNull(n) + { + } - ConstantAggData(std::string v, std::string u, RowAggFunctionType f, bool n) : - fConstValue(std::move(v)), fUDAFName(std::move(u)), fOp(f), fIsNull(n) - {} + ConstantAggData(std::string v, std::string u, RowAggFunctionType f, bool n) + : fConstValue(std::move(v)), fUDAFName(std::move(u)), fOp(f), fIsNull(n) + { + } }; -typedef boost::shared_ptr SP_ROWAGG_GRPBY_t; +typedef boost::shared_ptr SP_ROWAGG_GRPBY_t; typedef boost::shared_ptr SP_ROWAGG_FUNC_t; struct GroupConcat { - // GROUP_CONCAT(DISTINCT col1, 'const', col2 ORDER BY col3 desc SEPARATOR 'sep') - std::vector > fGroupCols; // columns to concatenate, and position - std::vector > fOrderCols; // columns to order by [asc/desc] - std::string fSeparator; - std::vector > fConstCols; // constant columns in group - bool fDistinct; - uint64_t fSize; + // GROUP_CONCAT(DISTINCT col1, 'const', col2 ORDER BY col3 desc SEPARATOR 'sep') + std::vector> fGroupCols; // columns to concatenate, and position + std::vector> fOrderCols; // columns to order by [asc/desc] + std::string fSeparator; + std::vector> fConstCols; // constant columns in group + bool fDistinct; + uint64_t fSize; - RowGroup fRowGroup; - boost::shared_array fMapping; - std::vector > fOrderCond; // position to order by [asc/desc] - joblist::ResourceManager* fRm; // resource manager - boost::shared_ptr fSessionMemLimit; - std::string fTimeZone; + RowGroup fRowGroup; + boost::shared_array fMapping; + std::vector> fOrderCond; // position to order by [asc/desc] + joblist::ResourceManager* fRm; // resource manager + boost::shared_ptr fSessionMemLimit; + std::string fTimeZone; - GroupConcat() : fRm(nullptr) {} + GroupConcat() : fRm(nullptr) + { + } }; -typedef boost::shared_ptr SP_GroupConcat; - +typedef boost::shared_ptr SP_GroupConcat; class GroupConcatAg { -public: - explicit GroupConcatAg(SP_GroupConcat&); - virtual ~GroupConcatAg(); + public: + explicit GroupConcatAg(SP_GroupConcat&); + virtual ~GroupConcatAg(); - virtual void initialize() {}; - virtual void processRow(const rowgroup::Row&) {}; - virtual void merge(const rowgroup::Row&, uint64_t) {}; + virtual void initialize(){}; + virtual void processRow(const rowgroup::Row&){}; + virtual void merge(const rowgroup::Row&, uint64_t){}; - void getResult(uint8_t*) {}; - uint8_t* getResult() - { - return nullptr; - } + void getResult(uint8_t*){}; + uint8_t* getResult() + { + return nullptr; + } -protected: - rowgroup::SP_GroupConcat fGroupConcat; + protected: + rowgroup::SP_GroupConcat fGroupConcat; }; -typedef boost::shared_ptr SP_GroupConcatAg; - - - - +typedef boost::shared_ptr SP_GroupConcatAg; //------------------------------------------------------------------------------ /** @brief Class that aggregates RowGroups. @@ -374,266 +376,259 @@ typedef boost::shared_ptr SP_GroupConcatAg; //------------------------------------------------------------------------------ class RowAggregation : public messageqcpp::Serializeable { -public: - /** @brief RowAggregation default constructor - * - * @param rowAggGroupByCols(in) specify GroupBy columns and their - * mapping from input to output. If vector is empty, then all the - * rows will be aggregated into a single implied group. Order is - * important here. The primary GroupBy column should be first, the - * secondary GroupBy column should be second, etc. - * @param rowAggFunctionCols(in) specify function columns and their - * mapping from input to output. - */ - RowAggregation(); - RowAggregation(const std::vector& rowAggGroupByCols, - const std::vector& rowAggFunctionCols, - joblist::ResourceManager* rm = nullptr, - boost::shared_ptr sessMemLimit = {}); - RowAggregation(const RowAggregation& rhs); + public: + /** @brief RowAggregation default constructor + * + * @param rowAggGroupByCols(in) specify GroupBy columns and their + * mapping from input to output. If vector is empty, then all the + * rows will be aggregated into a single implied group. Order is + * important here. The primary GroupBy column should be first, the + * secondary GroupBy column should be second, etc. + * @param rowAggFunctionCols(in) specify function columns and their + * mapping from input to output. + */ + RowAggregation(); + RowAggregation(const std::vector& rowAggGroupByCols, + const std::vector& rowAggFunctionCols, + joblist::ResourceManager* rm = nullptr, boost::shared_ptr sessMemLimit = {}); + RowAggregation(const RowAggregation& rhs); - /** @brief RowAggregation default destructor - */ - ~RowAggregation() override; + /** @brief RowAggregation default destructor + */ + ~RowAggregation() override; - /** @brief clone this object for multi-thread use - */ - inline virtual RowAggregation* clone() const - { - return new RowAggregation (*this); - } + /** @brief clone this object for multi-thread use + */ + inline virtual RowAggregation* clone() const + { + return new RowAggregation(*this); + } - /** @brief Denotes end of data insertion following multiple calls to addRowGroup(). - */ - virtual void endOfInput(); + /** @brief Denotes end of data insertion following multiple calls to addRowGroup(). + */ + virtual void endOfInput(); - /** @brief reset RowAggregation outputRowGroup and hashMap - */ - virtual void aggReset(); + /** @brief reset RowAggregation outputRowGroup and hashMap + */ + virtual void aggReset(); - /** @brief Define content of data to be aggregated and its aggregated output. - * - * @param pRowGroupIn(in) contains definition of the input data. - * @param pRowGroupOut(out) contains definition of the output data. - */ - virtual void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) - { - fRowGroupIn = pRowGroupIn; - fRowGroupOut = pRowGroupOut; - initialize(); - } + /** @brief Define content of data to be aggregated and its aggregated output. + * + * @param pRowGroupIn(in) contains definition of the input data. + * @param pRowGroupOut(out) contains definition of the output data. + */ + virtual void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) + { + fRowGroupIn = pRowGroupIn; + fRowGroupOut = pRowGroupOut; + initialize(); + } + /** @brief Define content of data to be joined + * + * This method must be call after setInputOutput() for PM hashjoin case. + * + * @param pSmallSideRG(in) contains definition of the small side data. + * @param pLargeSideRG(in) contains definition of the large side data. + */ + void setJoinRowGroups(std::vector* pSmallSideRG, RowGroup* pLargeSideRG); - /** @brief Define content of data to be joined - * - * This method must be call after setInputOutput() for PM hashjoin case. - * - * @param pSmallSideRG(in) contains definition of the small side data. - * @param pLargeSideRG(in) contains definition of the large side data. - */ - void setJoinRowGroups(std::vector* pSmallSideRG, RowGroup* pLargeSideRG); + /** @brief Returns group by column vector + * + * This function is used to duplicate the RowAggregation object + * + * @returns a reference of the group by vector + */ + std::vector& getGroupByCols() + { + return fGroupByCols; + } - /** @brief Returns group by column vector - * - * This function is used to duplicate the RowAggregation object - * - * @returns a reference of the group by vector - */ - std::vector& getGroupByCols() - { - return fGroupByCols; - } + /** @brief Returns aggregate function vector + * + * This function is used to duplicate the RowAggregation object + * + * @returns a reference of the aggregation function vector + */ + std::vector& getAggFunctions() + { + return fFunctionCols; + } - /** @brief Returns aggregate function vector - * - * This function is used to duplicate the RowAggregation object - * - * @returns a reference of the aggregation function vector - */ - std::vector& getAggFunctions() - { - return fFunctionCols; - } + /** @brief Add a group of rows to be aggregated. + * + * This function can be called to iteratively add RowGroups for aggregation. + * + * @parm pRowGroupIn(in) RowGroup to be added to aggregation. + */ + virtual void addRowGroup(const RowGroup* pRowGroupIn); + virtual void addRowGroup(const RowGroup* pRowGroupIn, + std::vector>& inRows); - /** @brief Add a group of rows to be aggregated. - * - * This function can be called to iteratively add RowGroups for aggregation. - * - * @parm pRowGroupIn(in) RowGroup to be added to aggregation. - */ - virtual void addRowGroup(const RowGroup* pRowGroupIn); - virtual void addRowGroup(const RowGroup* pRowGroupIn, std::vector>& inRows); + /** @brief Serialize RowAggregation object into a ByteStream. + * + * @parm bs(out) BytesStream that is to be written to. + */ + void serialize(messageqcpp::ByteStream& bs) const override; - /** @brief Serialize RowAggregation object into a ByteStream. - * - * @parm bs(out) BytesStream that is to be written to. - */ - void serialize(messageqcpp::ByteStream& bs) const override; + /** @brief Unserialize RowAggregation object from a ByteStream. + * + * @parm bs(in) BytesStream that is to be read from. + */ + void deserialize(messageqcpp::ByteStream& bs) override; - /** @brief Unserialize RowAggregation object from a ByteStream. - * - * @parm bs(in) BytesStream that is to be read from. - */ - void deserialize(messageqcpp::ByteStream& bs) override; + /** @brief load result set into byte stream + * + * @parm bs(out) BytesStream that is to be written to. + */ + void loadResult(messageqcpp::ByteStream& bs); + void loadEmptySet(messageqcpp::ByteStream& bs); - /** @brief load result set into byte stream - * - * @parm bs(out) BytesStream that is to be written to. - */ - void loadResult(messageqcpp::ByteStream& bs); - void loadEmptySet(messageqcpp::ByteStream& bs); + /** @brief get output rowgroup + * + * @returns a const pointer of the output rowgroup + */ + const RowGroup* getOutputRowGroup() const + { + return fRowGroupOut; + } + RowGroup* getOutputRowGroup() + { + return fRowGroupOut; + } - /** @brief get output rowgroup - * - * @returns a const pointer of the output rowgroup - */ - const RowGroup* getOutputRowGroup() const - { - return fRowGroupOut; - } - RowGroup* getOutputRowGroup() - { - return fRowGroupOut; - } + void append(RowAggregation* other); - void append(RowAggregation* other); + virtual void aggregateRow(Row& row, const uint64_t* hash = nullptr, + std::vector* rgContextColl = nullptr); + inline uint32_t aggMapKeyLength() const + { + return fAggMapKeyCount; + } - virtual void aggregateRow(Row& row, - const uint64_t* hash = nullptr, - std::vector* rgContextColl = nullptr); - inline uint32_t aggMapKeyLength() const - { - return fAggMapKeyCount; - } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + inline const std::string& timeZone() const + { + return fTimeZone; + } + inline std::vector* rgContextColl() + { + return &fRGContextColl; + } - inline void timeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } - inline const std::string& timeZone() const - { - return fTimeZone; - } - inline std::vector* rgContextColl() - { - return &fRGContextColl; - } + void finalAggregation() + { + return fRowAggStorage->finalize([this](Row& row) { mergeEntries(row); }, fRow); + } - void finalAggregation() - { - return fRowAggStorage->finalize([this](Row& row) { mergeEntries(row);}, fRow); - } + std::unique_ptr moveCurrentRGData() + { + return std::move(fCurRGData); + } - std::unique_ptr moveCurrentRGData() - { - return std::move(fCurRGData); - } + protected: + virtual void initialize(); + virtual void initMapData(const Row& row); + virtual void attachGroupConcatAg(); -protected: - virtual void initialize(); - virtual void initMapData(const Row& row); - virtual void attachGroupConcatAg(); + virtual void updateEntry(const Row& row, std::vector* rgContextColl = nullptr); + void mergeEntries(const Row& row); + virtual void doMinMax(const Row&, int64_t, int64_t, int); + virtual void doSum(const Row&, int64_t, int64_t, int); + virtual void doAvg(const Row&, int64_t, int64_t, int64_t, bool merge = false); + virtual void doStatistics(const Row&, int64_t, int64_t, int64_t); + void mergeStatistics(const Row&, uint64_t colOut, uint64_t colAux); + virtual void doBitOp(const Row&, int64_t, int64_t, int); + virtual void doUDAF(const Row&, int64_t, int64_t, int64_t, uint64_t& funcColsIdx, + std::vector* rgContextColl = nullptr); + virtual bool countSpecial(const RowGroup* pRG) + { + fRow.setUintField<8>(fRow.getUintField<8>(0) + pRG->getRowCount(), 0); + return true; + } - virtual void updateEntry(const Row& row, - std::vector* rgContextColl = nullptr); - void mergeEntries(const Row& row); - virtual void doMinMax(const Row&, int64_t, int64_t, int); - virtual void doSum(const Row&, int64_t, int64_t, int); - virtual void doAvg(const Row&, int64_t, int64_t, int64_t, bool merge = false); - virtual void doStatistics(const Row&, int64_t, int64_t, int64_t); - void mergeStatistics(const Row&, uint64_t colOut, uint64_t colAux); - virtual void doBitOp(const Row&, int64_t, int64_t, int); - virtual void doUDAF(const Row&, - int64_t, - int64_t, - int64_t, - uint64_t& funcColsIdx, - std::vector* rgContextColl = nullptr); - virtual bool countSpecial(const RowGroup* pRG) - { - fRow.setUintField<8>(fRow.getUintField<8>(0) + pRG->getRowCount(), 0); - return true; - } + void resetUDAF(RowUDAFFunctionCol* rowUDAF); + void resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColIdx); - void resetUDAF(RowUDAFFunctionCol* rowUDAF); - void resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColIdx); + inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); + inline void makeAggFieldsNull(Row& row); + inline void copyNullRow(Row& row) + { + copyRow(fNullRow, &row); + } - inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); - inline void makeAggFieldsNull(Row& row); - inline void copyNullRow(Row& row) - { - copyRow(fNullRow, &row); - } + inline void updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func); + inline void updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func); + inline void updateUintMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); + inline void updateCharMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); + inline void updateDoubleMinMax(double val1, double val2, int64_t col, int func); + inline void updateLongDoubleMinMax(long double val1, long double val2, int64_t col, int func); + inline void updateFloatMinMax(float val1, float val2, int64_t col, int func); + inline void updateStringMinMax(std::string val1, std::string val2, int64_t col, int func); + std::vector fGroupByCols; + std::vector fFunctionCols; + uint32_t fAggMapKeyCount; // the number of columns that make up the key + RowGroup fRowGroupIn; + RowGroup* fRowGroupOut; - inline void updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func); - inline void updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func); - inline void updateUintMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); - inline void updateCharMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); - inline void updateDoubleMinMax(double val1, double val2, int64_t col, int func); - inline void updateLongDoubleMinMax(long double val1, long double val2, int64_t col, int func); - inline void updateFloatMinMax(float val1, float val2, int64_t col, int func); - inline void updateStringMinMax(std::string val1, std::string val2, int64_t col, int func); - std::vector fGroupByCols; - std::vector fFunctionCols; - uint32_t fAggMapKeyCount; // the number of columns that make up the key - RowGroup fRowGroupIn; - RowGroup* fRowGroupOut; + // for when the group by & distinct keys are not stored in the output rows + rowgroup::RowGroup fKeyRG; - // for when the group by & distinct keys are not stored in the output rows - rowgroup::RowGroup fKeyRG; + Row fRow; + Row fNullRow; + Row* tmpRow; // used by the hashers & eq functors + boost::scoped_array fNullRowData; - Row fRow; - Row fNullRow; - Row* tmpRow; // used by the hashers & eq functors - boost::scoped_array fNullRowData; + std::unique_ptr fRowAggStorage; - std::unique_ptr fRowAggStorage; + // for support PM aggregation after PM hashjoin + std::vector* fSmallSideRGs; + RowGroup* fLargeSideRG; + boost::shared_array> fSmallMappings; + boost::shared_array fLargeMapping; + uint32_t fSmallSideCount; + boost::scoped_array rowSmalls; - // for support PM aggregation after PM hashjoin - std::vector* fSmallSideRGs; - RowGroup* fLargeSideRG; - boost::shared_array > fSmallMappings; - boost::shared_array fLargeMapping; - uint32_t fSmallSideCount; - boost::scoped_array rowSmalls; + // for 8k poc + RowGroup fEmptyRowGroup; + RGData fEmptyRowData; + Row fEmptyRow; - // for 8k poc - RowGroup fEmptyRowGroup; - RGData fEmptyRowData; - Row fEmptyRow; + bool fKeyOnHeap = false; - bool fKeyOnHeap = false; + std::string fTimeZone; - std::string fTimeZone; + // We need a separate copy for each thread. + mcsv1sdk::mcsv1Context fRGContext; + std::vector fRGContextColl; - // We need a separate copy for each thread. - mcsv1sdk::mcsv1Context fRGContext; - std::vector fRGContextColl; + // These are handy for testing the actual type of static_any for UDAF + static const static_any::any& charTypeId; + static const static_any::any& scharTypeId; + static const static_any::any& shortTypeId; + static const static_any::any& intTypeId; + static const static_any::any& longTypeId; + static const static_any::any& llTypeId; + static const static_any::any& int128TypeId; + static const static_any::any& ucharTypeId; + static const static_any::any& ushortTypeId; + static const static_any::any& uintTypeId; + static const static_any::any& ulongTypeId; + static const static_any::any& ullTypeId; + static const static_any::any& floatTypeId; + static const static_any::any& doubleTypeId; + static const static_any::any& longdoubleTypeId; + static const static_any::any& strTypeId; - // These are handy for testing the actual type of static_any for UDAF - static const static_any::any& charTypeId; - static const static_any::any& scharTypeId; - static const static_any::any& shortTypeId; - static const static_any::any& intTypeId; - static const static_any::any& longTypeId; - static const static_any::any& llTypeId; - static const static_any::any& int128TypeId; - static const static_any::any& ucharTypeId; - static const static_any::any& ushortTypeId; - static const static_any::any& uintTypeId; - static const static_any::any& ulongTypeId; - static const static_any::any& ullTypeId; - static const static_any::any& floatTypeId; - static const static_any::any& doubleTypeId; - static const static_any::any& longdoubleTypeId; - static const static_any::any& strTypeId; + // For UDAF along with with multiple distinct columns + std::vector* fOrigFunctionCols; - // For UDAF along with with multiple distinct columns - std::vector* fOrigFunctionCols; - - joblist::ResourceManager* fRm = nullptr; - boost::shared_ptr fSessionMemLimit; - std::unique_ptr fCurRGData; + joblist::ResourceManager* fRm = nullptr; + boost::shared_ptr fSessionMemLimit; + std::unique_ptr fCurRGData; }; //------------------------------------------------------------------------------ @@ -643,163 +638,159 @@ protected: //------------------------------------------------------------------------------ class RowAggregationUM : public RowAggregation { -public: - /** @brief RowAggregationUM constructor - */ - RowAggregationUM() {} - RowAggregationUM( - const std::vector& rowAggGroupByCols, - const std::vector& rowAggFunctionCols, - joblist::ResourceManager*, - boost::shared_ptr sessionMemLimit); - RowAggregationUM(const RowAggregationUM& rhs); + public: + /** @brief RowAggregationUM constructor + */ + RowAggregationUM() + { + } + RowAggregationUM(const std::vector& rowAggGroupByCols, + const std::vector& rowAggFunctionCols, joblist::ResourceManager*, + boost::shared_ptr sessionMemLimit); + RowAggregationUM(const RowAggregationUM& rhs); - /** @brief RowAggregationUM default destructor - */ - ~RowAggregationUM() override; + /** @brief RowAggregationUM default destructor + */ + ~RowAggregationUM() override; - /** @brief Denotes end of data insertion following multiple calls to addRowGroup(). - */ - void endOfInput() override; + /** @brief Denotes end of data insertion following multiple calls to addRowGroup(). + */ + void endOfInput() override; - /** @brief Finializes the result set before sending back to the front end. - */ - void finalize(); + /** @brief Finializes the result set before sending back to the front end. + */ + void finalize(); - /** @brief Returns aggregated rows in a RowGroup. - * - * This function should be called repeatedly until false is returned (meaning end of data). - * - * @returns true if more data, else false if no more data. - */ - bool nextRowGroup(); + /** @brief Returns aggregated rows in a RowGroup. + * + * This function should be called repeatedly until false is returned (meaning end of data). + * + * @returns true if more data, else false if no more data. + */ + bool nextRowGroup(); - /** @brief Add an aggregator for DISTINCT aggregation - */ - void distinctAggregator(const boost::shared_ptr& da) - { - fDistinctAggregator = da; - } + /** @brief Add an aggregator for DISTINCT aggregation + */ + void distinctAggregator(const boost::shared_ptr& da) + { + fDistinctAggregator = da; + } - /** @brief expressions to be evaluated after aggregation - */ - void expression(const std::vector& exp) - { - fExpression = exp; - } - const std::vector& expression() - { - return fExpression; - } + /** @brief expressions to be evaluated after aggregation + */ + void expression(const std::vector& exp) + { + fExpression = exp; + } + const std::vector& expression() + { + return fExpression; + } - // for multi threaded - joblist::ResourceManager* getRm() - { - return fRm; - } - inline RowAggregationUM* clone() const override - { - return new RowAggregationUM (*this); - } + // for multi threaded + joblist::ResourceManager* getRm() + { + return fRm; + } + inline RowAggregationUM* clone() const override + { + return new RowAggregationUM(*this); + } - /** @brief access the aggregate(constant) columns - */ - void constantAggregate(const std::vector& v) - { - fConstantAggregate = v; - } - const std::vector& constantAggregate() const - { - return fConstantAggregate; - } + /** @brief access the aggregate(constant) columns + */ + void constantAggregate(const std::vector& v) + { + fConstantAggregate = v; + } + const std::vector& constantAggregate() const + { + return fConstantAggregate; + } - /** @brief access the group_concat - */ - void groupConcat(const std::vector& v) - { - fGroupConcat = v; - } - const std::vector& groupConcat() const - { - return fGroupConcat; - } + /** @brief access the group_concat + */ + void groupConcat(const std::vector& v) + { + fGroupConcat = v; + } + const std::vector& groupConcat() const + { + return fGroupConcat; + } - void aggReset() override; + void aggReset() override; - void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; + void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; -protected: - // virtual methods from base - void initialize() override; + protected: + // virtual methods from base + void initialize() override; - void attachGroupConcatAg() override; - void updateEntry(const Row& row, - std::vector* rgContextColl = nullptr) override; - bool countSpecial(const RowGroup* pRG) override - { - fRow.setIntField<8>( - fRow.getIntField<8>( - fFunctionCols[0]->fOutputColumnIndex) + pRG->getRowCount(), - fFunctionCols[0]->fOutputColumnIndex); - return true; - } + void attachGroupConcatAg() override; + void updateEntry(const Row& row, std::vector* rgContextColl = nullptr) override; + bool countSpecial(const RowGroup* pRG) override + { + fRow.setIntField<8>(fRow.getIntField<8>(fFunctionCols[0]->fOutputColumnIndex) + pRG->getRowCount(), + fFunctionCols[0]->fOutputColumnIndex); + return true; + } - // calculate the average after all rows received. UM only function. - void calculateAvgColumns(); + // calculate the average after all rows received. UM only function. + void calculateAvgColumns(); - // calculate the statistics function all rows received. UM only function. - void calculateStatisticsFunctions(); + // calculate the statistics function all rows received. UM only function. + void calculateStatisticsFunctions(); - // Sets the value from valOut into column colOut, performing any conversions. - void SetUDAFValue(static_any::any& valOut, int64_t colOut); + // Sets the value from valOut into column colOut, performing any conversions. + void SetUDAFValue(static_any::any& valOut, int64_t colOut); - // If the datatype returned by evaluate isn't what we expect, convert. - void SetUDAFAnyValue(static_any::any& valOut, int64_t colOut); + // If the datatype returned by evaluate isn't what we expect, convert. + void SetUDAFAnyValue(static_any::any& valOut, int64_t colOut); - // calculate the UDAF function all rows received. UM only function. - void calculateUDAFColumns(); + // calculate the UDAF function all rows received. UM only function. + void calculateUDAFColumns(); - // fix duplicates. UM only function. - void fixDuplicates(RowAggFunctionType funct); + // fix duplicates. UM only function. + void fixDuplicates(RowAggFunctionType funct); - // evaluate expressions - virtual void evaluateExpression(); + // evaluate expressions + virtual void evaluateExpression(); - // fix the aggregate(constant) - virtual void fixConstantAggregate(); - virtual void doNullConstantAggregate(const ConstantAggData&, uint64_t); - virtual void doNotNullConstantAggregate(const ConstantAggData&, uint64_t); + // fix the aggregate(constant) + virtual void fixConstantAggregate(); + virtual void doNullConstantAggregate(const ConstantAggData&, uint64_t); + virtual void doNotNullConstantAggregate(const ConstantAggData&, uint64_t); - // @bug3362, group_concat - virtual void doGroupConcat(const Row&, int64_t, int64_t); - virtual void setGroupConcatString(); + // @bug3362, group_concat + virtual void doGroupConcat(const Row&, int64_t, int64_t); + virtual void setGroupConcatString(); - bool fHasAvg; - bool fHasStatsFunc; - bool fHasUDAF; + bool fHasAvg; + bool fHasStatsFunc; + bool fHasUDAF; - boost::shared_ptr fDistinctAggregator; + boost::shared_ptr fDistinctAggregator; - // for function on aggregation - std::vector fExpression; + // for function on aggregation + std::vector fExpression; - /* Derived classes that use a lot of memory need to update totalMemUsage and request - * the memory from rm in that order. */ - uint64_t fTotalMemUsage; + /* Derived classes that use a lot of memory need to update totalMemUsage and request + * the memory from rm in that order. */ + uint64_t fTotalMemUsage; - // @bug3475, aggregate(constant), sum(0), count(null), etc - std::vector fConstantAggregate; + // @bug3475, aggregate(constant), sum(0), count(null), etc + std::vector fConstantAggregate; - // @bug3362, group_concat - std::vector fGroupConcat; - std::vector fGroupConcatAg; - std::vector fFunctionColGc; + // @bug3362, group_concat + std::vector fGroupConcat; + std::vector fGroupConcatAg; + std::vector fFunctionColGc; -private: - uint64_t fLastMemUsage; + private: + uint64_t fLastMemUsage; }; - //------------------------------------------------------------------------------ /** @brief derived Class that aggregates PM partially aggregated RowGroups on UM * Two-phase case: @@ -811,47 +802,40 @@ private: //------------------------------------------------------------------------------ class RowAggregationUMP2 : public RowAggregationUM { -public: - /** @brief RowAggregationUM constructor - */ - RowAggregationUMP2() {} - RowAggregationUMP2( - const std::vector& rowAggGroupByCols, - const std::vector& rowAggFunctionCols, - joblist::ResourceManager*, - boost::shared_ptr sessionMemLimit); - RowAggregationUMP2(const RowAggregationUMP2& rhs); + public: + /** @brief RowAggregationUM constructor + */ + RowAggregationUMP2() + { + } + RowAggregationUMP2(const std::vector& rowAggGroupByCols, + const std::vector& rowAggFunctionCols, joblist::ResourceManager*, + boost::shared_ptr sessionMemLimit); + RowAggregationUMP2(const RowAggregationUMP2& rhs); - /** @brief RowAggregationUMP2 default destructor - */ - ~RowAggregationUMP2() override; - inline RowAggregationUMP2* clone() const override - { - return new RowAggregationUMP2 (*this); - } + /** @brief RowAggregationUMP2 default destructor + */ + ~RowAggregationUMP2() override; + inline RowAggregationUMP2* clone() const override + { + return new RowAggregationUMP2(*this); + } -protected: - // virtual methods from base - void updateEntry(const Row& row, - std::vector* rgContextColl = nullptr) override; - void doAvg(const Row&, int64_t, int64_t, int64_t, bool merge = false) override; - void doStatistics(const Row&, int64_t, int64_t, int64_t) override; - void doGroupConcat(const Row&, int64_t, int64_t) override; - void doBitOp(const Row&, int64_t, int64_t, int) override; - void doUDAF(const Row&, - int64_t, - int64_t, - int64_t, - uint64_t& funcColsIdx, - std::vector* rgContextColl = nullptr) override; - bool countSpecial(const RowGroup* pRG) override - { - return false; - } + protected: + // virtual methods from base + void updateEntry(const Row& row, std::vector* rgContextColl = nullptr) override; + void doAvg(const Row&, int64_t, int64_t, int64_t, bool merge = false) override; + void doStatistics(const Row&, int64_t, int64_t, int64_t) override; + void doGroupConcat(const Row&, int64_t, int64_t) override; + void doBitOp(const Row&, int64_t, int64_t, int) override; + void doUDAF(const Row&, int64_t, int64_t, int64_t, uint64_t& funcColsIdx, + std::vector* rgContextColl = nullptr) override; + bool countSpecial(const RowGroup* pRG) override + { + return false; + } }; - - //------------------------------------------------------------------------------ /** @brief derived Class that aggregates on distinct columns on UM * The internal aggregator will handle one or two phases aggregation @@ -859,67 +843,66 @@ protected: //------------------------------------------------------------------------------ class RowAggregationDistinct : public RowAggregationUMP2 { -public: - /** @brief RowAggregationDistinct constructor - */ - RowAggregationDistinct() {} - RowAggregationDistinct( - const std::vector& rowAggGroupByCols, - const std::vector& rowAggFunctionCols, - joblist::ResourceManager*, - boost::shared_ptr sessionMemLimit); + public: + /** @brief RowAggregationDistinct constructor + */ + RowAggregationDistinct() + { + } + RowAggregationDistinct(const std::vector& rowAggGroupByCols, + const std::vector& rowAggFunctionCols, joblist::ResourceManager*, + boost::shared_ptr sessionMemLimit); - /** @brief Copy Constructor for multi-threaded aggregation - */ - RowAggregationDistinct(const RowAggregationDistinct& rhs); + /** @brief Copy Constructor for multi-threaded aggregation + */ + RowAggregationDistinct(const RowAggregationDistinct& rhs); - /** @brief RowAggregationDistinct default destructor - */ - ~RowAggregationDistinct() override; + /** @brief RowAggregationDistinct default destructor + */ + ~RowAggregationDistinct() override; - /** @brief Add an aggregator for pre-DISTINCT aggregation - */ - void addAggregator(const boost::shared_ptr& agg, const RowGroup& rg); + /** @brief Add an aggregator for pre-DISTINCT aggregation + */ + void addAggregator(const boost::shared_ptr& agg, const RowGroup& rg); - void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; + void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; - virtual void doDistinctAggregation(); - virtual void doDistinctAggregation_rowVec(std::vector>& inRows); - void addRowGroup(const RowGroup* pRowGroupIn) override; - void addRowGroup(const RowGroup* pRowGroupIn, std::vector>& inRows) override; + virtual void doDistinctAggregation(); + virtual void doDistinctAggregation_rowVec(std::vector>& inRows); + void addRowGroup(const RowGroup* pRowGroupIn) override; + void addRowGroup(const RowGroup* pRowGroupIn, + std::vector>& inRows) override; - // multi-threade debug - boost::shared_ptr& aggregator() - { - return fAggregator; - } - void aggregator(boost::shared_ptr aggregator) - { - fAggregator = std::move(aggregator); - } - RowGroup& rowGroupDist() - { - return fRowGroupDist; - } - void rowGroupDist(RowGroup& rowGroupDist) - { - fRowGroupDist = rowGroupDist; - } - inline RowAggregationDistinct* clone() const override - { - return new RowAggregationDistinct (*this); - } + // multi-threade debug + boost::shared_ptr& aggregator() + { + return fAggregator; + } + void aggregator(boost::shared_ptr aggregator) + { + fAggregator = std::move(aggregator); + } + RowGroup& rowGroupDist() + { + return fRowGroupDist; + } + void rowGroupDist(RowGroup& rowGroupDist) + { + fRowGroupDist = rowGroupDist; + } + inline RowAggregationDistinct* clone() const override + { + return new RowAggregationDistinct(*this); + } -protected: - void updateEntry(const Row& row, - std::vector* rgContextColl = nullptr) override; + protected: + void updateEntry(const Row& row, std::vector* rgContextColl = nullptr) override; - boost::shared_ptr fAggregator; - RowGroup fRowGroupDist; - RGData fDataForDist; + boost::shared_ptr fAggregator; + RowGroup fRowGroupDist; + RGData fDataForDist; }; - //------------------------------------------------------------------------------ /** @brief derived Class for aggregates multiple columns with distinct key word * Get distinct values of the column per group by entry @@ -927,40 +910,40 @@ protected: //------------------------------------------------------------------------------ class RowAggregationSubDistinct : public RowAggregationUM { -public: - /** @brief RowAggregationSubDistinct constructor - */ - RowAggregationSubDistinct() {} - RowAggregationSubDistinct( - const std::vector& rowAggGroupByCols, - const std::vector& rowAggFunctionCols, - joblist::ResourceManager*, - boost::shared_ptr sessionMemLimit); - RowAggregationSubDistinct(const RowAggregationSubDistinct& rhs); + public: + /** @brief RowAggregationSubDistinct constructor + */ + RowAggregationSubDistinct() + { + } + RowAggregationSubDistinct(const std::vector& rowAggGroupByCols, + const std::vector& rowAggFunctionCols, + joblist::ResourceManager*, boost::shared_ptr sessionMemLimit); + RowAggregationSubDistinct(const RowAggregationSubDistinct& rhs); - /** @brief RowAggregationSubDistinct default destructor - */ - ~RowAggregationSubDistinct() override; + /** @brief RowAggregationSubDistinct default destructor + */ + ~RowAggregationSubDistinct() override; - void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; - void addRowGroup(const RowGroup* pRowGroupIn) override; - inline RowAggregationSubDistinct* clone() const override - { - return new RowAggregationSubDistinct (*this); - } + void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; + void addRowGroup(const RowGroup* pRowGroupIn) override; + inline RowAggregationSubDistinct* clone() const override + { + return new RowAggregationSubDistinct(*this); + } - void addRowGroup(const RowGroup* pRowGroupIn, std::vector>& inRow) override; + void addRowGroup(const RowGroup* pRowGroupIn, + std::vector>& inRow) override; -protected: - // virtual methods from RowAggregationUM - void doGroupConcat(const Row&, int64_t, int64_t) override; + protected: + // virtual methods from RowAggregationUM + void doGroupConcat(const Row&, int64_t, int64_t) override; - // for groupby columns and the aggregated distinct column - Row fDistRow; - boost::scoped_array fDistRowData; + // for groupby columns and the aggregated distinct column + Row fDistRow; + boost::scoped_array fDistRowData; }; - //------------------------------------------------------------------------------ /** @brief derived Class that aggregates multiple columns with distinct key word * Each distinct column will have its own aggregator @@ -968,71 +951,67 @@ protected: //------------------------------------------------------------------------------ class RowAggregationMultiDistinct : public RowAggregationDistinct { -public: - /** @brief RowAggregationMultiDistinct constructor - */ - RowAggregationMultiDistinct() {} - RowAggregationMultiDistinct( - const std::vector& rowAggGroupByCols, - const std::vector& rowAggFunctionCols, - joblist::ResourceManager*, - boost::shared_ptr sessionMemLimit); - RowAggregationMultiDistinct(const RowAggregationMultiDistinct& rhs); + public: + /** @brief RowAggregationMultiDistinct constructor + */ + RowAggregationMultiDistinct() + { + } + RowAggregationMultiDistinct(const std::vector& rowAggGroupByCols, + const std::vector& rowAggFunctionCols, + joblist::ResourceManager*, boost::shared_ptr sessionMemLimit); + RowAggregationMultiDistinct(const RowAggregationMultiDistinct& rhs); - /** @brief RowAggregationMultiDistinct default destructor - */ - ~RowAggregationMultiDistinct() override; + /** @brief RowAggregationMultiDistinct default destructor + */ + ~RowAggregationMultiDistinct() override; - /** @brief Add sub aggregators - */ - void addSubAggregator(const boost::shared_ptr& agg, - const RowGroup& rg, - const std::vector& funct); + /** @brief Add sub aggregators + */ + void addSubAggregator(const boost::shared_ptr& agg, const RowGroup& rg, + const std::vector& funct); - void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; - using RowAggregationDistinct::addRowGroup; - void addRowGroup(const RowGroup* pRowGroupIn) override; + void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut) override; + using RowAggregationDistinct::addRowGroup; + void addRowGroup(const RowGroup* pRowGroupIn) override; - using RowAggregationDistinct::doDistinctAggregation; - void doDistinctAggregation() override; - using RowAggregationDistinct::doDistinctAggregation_rowVec; - virtual void doDistinctAggregation_rowVec(std::vector> >& inRows); + using RowAggregationDistinct::doDistinctAggregation; + void doDistinctAggregation() override; + using RowAggregationDistinct::doDistinctAggregation_rowVec; + virtual void doDistinctAggregation_rowVec( + std::vector>>& inRows); - inline RowAggregationMultiDistinct* clone() const override - { - return new RowAggregationMultiDistinct (*this); - } + inline RowAggregationMultiDistinct* clone() const override + { + return new RowAggregationMultiDistinct(*this); + } - void addRowGroup(const RowGroup* pRowGroupIn, std::vector>>& inRows); + void addRowGroup(const RowGroup* pRowGroupIn, + std::vector>>& inRows); - std::vector >& subAggregators() - { - return fSubAggregators; - } + std::vector>& subAggregators() + { + return fSubAggregators; + } - void subAggregators(std::vector >& subAggregators) - { - fSubAggregators = subAggregators; - } + void subAggregators(std::vector>& subAggregators) + { + fSubAggregators = subAggregators; + } -protected: - // virtual methods from base - std::vector > fSubAggregators; - std::vector fSubRowGroups; - std::vector > fSubRowData; - std::vector > fSubFunctions; + protected: + // virtual methods from base + std::vector> fSubAggregators; + std::vector fSubRowGroups; + std::vector> fSubRowData; + std::vector> fSubFunctions; }; +typedef boost::shared_ptr SP_ROWAGG_t; +typedef boost::shared_ptr SP_ROWAGG_PM_t; +typedef boost::shared_ptr SP_ROWAGG_UM_t; +typedef boost::shared_ptr SP_ROWAGG_DIST; - -typedef boost::shared_ptr SP_ROWAGG_t; -typedef boost::shared_ptr SP_ROWAGG_PM_t; -typedef boost::shared_ptr SP_ROWAGG_UM_t; -typedef boost::shared_ptr SP_ROWAGG_DIST; - - - -} // end of rowgroup namespace +} // namespace rowgroup #endif // ROWAGGREGATION_H - diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index fb613bc22..4e767e1d8 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -46,22 +46,22 @@ using namespace execplan; #include "dataconvert.h" #include "columnwidth.h" - namespace rowgroup { - using cscType = execplan::CalpontSystemCatalog::ColDataType; -StringStore::StringStore() : empty(true), fUseStoreStringMutex(false) { } +StringStore::StringStore() : empty(true), fUseStoreStringMutex(false) +{ +} StringStore::StringStore(const StringStore&) { - throw logic_error("Don't call StringStore copy ctor"); + throw logic_error("Don't call StringStore copy ctor"); } StringStore& StringStore::operator=(const StringStore&) { - throw logic_error("Don't call StringStore operator="); + throw logic_error("Don't call StringStore operator="); } StringStore::~StringStore() @@ -86,135 +86,133 @@ StringStore::~StringStore() uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) { - MemChunk* lastMC = NULL; - uint64_t ret = 0; + MemChunk* lastMC = NULL; + uint64_t ret = 0; - empty = false; // At least a NULL is being stored. + empty = false; // At least a NULL is being stored. - // Sometimes the caller actually wants "" to be returned....... argggghhhh...... - //if (len == 0) - // return numeric_limits::max(); + // Sometimes the caller actually wants "" to be returned....... argggghhhh...... + // if (len == 0) + // return numeric_limits::max(); - if ((len == 8 || len == 9) && - *((uint64_t*) data) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())) - return numeric_limits::max(); + if ((len == 8 || len == 9) && *((uint64_t*)data) == *((uint64_t*)joblist::CPNULLSTRMARK.c_str())) + return numeric_limits::max(); - //@bug6065, make StringStore::storeString() thread safe - boost::mutex::scoped_lock lk(fMutex, boost::defer_lock); + //@bug6065, make StringStore::storeString() thread safe + boost::mutex::scoped_lock lk(fMutex, boost::defer_lock); - if (fUseStoreStringMutex) - lk.lock(); + if (fUseStoreStringMutex) + lk.lock(); - if (mem.size() > 0) - lastMC = (MemChunk*) mem.back().get(); + if (mem.size() > 0) + lastMC = (MemChunk*)mem.back().get(); - if ((len + 4) >= CHUNK_SIZE) + if ((len + 4) >= CHUNK_SIZE) + { + shared_array newOne(new uint8_t[len + sizeof(MemChunk) + 4]); + longStrings.push_back(newOne); + lastMC = (MemChunk*)longStrings.back().get(); + lastMC->capacity = lastMC->currentSize = len + 4; + memcpy(lastMC->data, &len, 4); + memcpy(lastMC->data + 4, data, len); + // High bit to mark a long string + ret = 0x8000000000000000; + ret += longStrings.size() - 1; + } + else + { + if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4))) { - shared_array newOne(new uint8_t[len + sizeof(MemChunk) + 4]); - longStrings.push_back(newOne); - lastMC = (MemChunk*) longStrings.back().get(); - lastMC->capacity = lastMC->currentSize = len + 4; - memcpy(lastMC->data, &len, 4); - memcpy(lastMC->data + 4, data, len); - // High bit to mark a long string - ret = 0x8000000000000000; - ret += longStrings.size() - 1; - } - else - { - if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4))) - { - // mem usage debugging - //if (lastMC) - //cout << "Memchunk efficiency = " << lastMC->currentSize << "/" << lastMC->capacity << endl; - shared_array newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); - mem.push_back(newOne); - lastMC = (MemChunk*) mem.back().get(); - lastMC->currentSize = 0; - lastMC->capacity = CHUNK_SIZE; - memset(lastMC->data, 0, CHUNK_SIZE); - } - - - ret = ((mem.size() - 1) * CHUNK_SIZE) + lastMC->currentSize; - - // If this ever happens then we have big problems - if (ret & 0x8000000000000000) - throw logic_error("StringStore memory exceeded."); - - memcpy(&(lastMC->data[lastMC->currentSize]), &len, 4); - memcpy(&(lastMC->data[lastMC->currentSize]) + 4, data, len); - /* - cout << "stored: '" << hex; - for (uint32_t i = 0; i < len ; i++) { - cout << (char) lastMC->data[lastMC->currentSize + i]; - } - cout << "' at position " << lastMC->currentSize << " len " << len << dec << endl; - */ - lastMC->currentSize += len + 4; + // mem usage debugging + // if (lastMC) + // cout << "Memchunk efficiency = " << lastMC->currentSize << "/" << lastMC->capacity << endl; + shared_array newOne(new uint8_t[CHUNK_SIZE + sizeof(MemChunk)]); + mem.push_back(newOne); + lastMC = (MemChunk*)mem.back().get(); + lastMC->currentSize = 0; + lastMC->capacity = CHUNK_SIZE; + memset(lastMC->data, 0, CHUNK_SIZE); } - return ret; + ret = ((mem.size() - 1) * CHUNK_SIZE) + lastMC->currentSize; + + // If this ever happens then we have big problems + if (ret & 0x8000000000000000) + throw logic_error("StringStore memory exceeded."); + + memcpy(&(lastMC->data[lastMC->currentSize]), &len, 4); + memcpy(&(lastMC->data[lastMC->currentSize]) + 4, data, len); + /* + cout << "stored: '" << hex; + for (uint32_t i = 0; i < len ; i++) { + cout << (char) lastMC->data[lastMC->currentSize + i]; + } + cout << "' at position " << lastMC->currentSize << " len " << len << dec << endl; + */ + lastMC->currentSize += len + 4; + } + + return ret; } void StringStore::serialize(ByteStream& bs) const { - uint64_t i; - MemChunk* mc; + uint64_t i; + MemChunk* mc; - bs << (uint64_t) mem.size(); - bs << (uint8_t) empty; + bs << (uint64_t)mem.size(); + bs << (uint8_t)empty; - for (i = 0; i < mem.size(); i++) - { - mc = (MemChunk*) mem[i].get(); - bs << (uint64_t) mc->currentSize; - //cout << "serialized " << mc->currentSize << " bytes\n"; - bs.append(mc->data, mc->currentSize); - } + for (i = 0; i < mem.size(); i++) + { + mc = (MemChunk*)mem[i].get(); + bs << (uint64_t)mc->currentSize; + // cout << "serialized " << mc->currentSize << " bytes\n"; + bs.append(mc->data, mc->currentSize); + } - bs.setLongStrings(longStrings); + bs.setLongStrings(longStrings); } void StringStore::deserialize(ByteStream& bs) { - uint64_t i; - uint64_t count; - uint64_t size; - uint8_t* buf; - MemChunk* mc; - uint8_t tmp8; + uint64_t i; + uint64_t count; + uint64_t size; + uint8_t* buf; + MemChunk* mc; + uint8_t tmp8; - //mem.clear(); - bs >> count; - mem.resize(count); - bs >> tmp8; - empty = (bool) tmp8; + // mem.clear(); + bs >> count; + mem.resize(count); + bs >> tmp8; + empty = (bool)tmp8; - for (i = 0; i < count; i++) - { - bs >> size; - //cout << "deserializing " << size << " bytes\n"; - buf = bs.buf(); - mem[i].reset(new uint8_t[size + sizeof(MemChunk)]); - mc = (MemChunk*) mem[i].get(); - mc->currentSize = size; - mc->capacity = size; - memcpy(mc->data, buf, size); - bs.advance(size); - } + for (i = 0; i < count; i++) + { + bs >> size; + // cout << "deserializing " << size << " bytes\n"; + buf = bs.buf(); + mem[i].reset(new uint8_t[size + sizeof(MemChunk)]); + mc = (MemChunk*)mem[i].get(); + mc->currentSize = size; + mc->capacity = size; + memcpy(mc->data, buf, size); + bs.advance(size); + } - longStrings = bs.getLongStrings(); - return; + longStrings = bs.getLongStrings(); + return; } void StringStore::clear() { - vector > emptyv; - vector > emptyv2; - mem.swap(emptyv); - longStrings.swap(emptyv2); - empty = true; + vector > emptyv; + vector > emptyv2; + mem.swap(emptyv); + longStrings.swap(emptyv2); + empty = true; } UserDataStore::UserDataStore() : fUseUserDataMutex(false) @@ -226,825 +224,754 @@ UserDataStore::~UserDataStore() } uint32_t UserDataStore::storeUserData(mcsv1sdk::mcsv1Context& context, - boost::shared_ptr data, - uint32_t len) + boost::shared_ptr data, uint32_t len) { - uint32_t ret = 0; + uint32_t ret = 0; - if (len == 0 || data == NULL) - { - return numeric_limits::max(); - } + if (len == 0 || data == NULL) + { + return numeric_limits::max(); + } - boost::mutex::scoped_lock lk(fMutex, boost::defer_lock); + boost::mutex::scoped_lock lk(fMutex, boost::defer_lock); - if (fUseUserDataMutex) - lk.lock(); + if (fUseUserDataMutex) + lk.lock(); - StoreData storeData; - storeData.length = len; - storeData.functionName = context.getName(); - storeData.userData = data; - vStoreData.push_back(storeData); + StoreData storeData; + storeData.length = len; + storeData.functionName = context.getName(); + storeData.userData = data; + vStoreData.push_back(storeData); - ret = vStoreData.size(); + ret = vStoreData.size(); - return ret; + return ret; } boost::shared_ptr UserDataStore::getUserData(uint32_t off) const { - if (off == std::numeric_limits::max()) - return boost::shared_ptr(); + if (off == std::numeric_limits::max()) + return boost::shared_ptr(); - if ((vStoreData.size() < off) || off == 0) - return boost::shared_ptr(); + if ((vStoreData.size() < off) || off == 0) + return boost::shared_ptr(); - return vStoreData[off - 1].userData; + return vStoreData[off - 1].userData; } - void UserDataStore::serialize(ByteStream& bs) const { - size_t i; + size_t i; - bs << (uint32_t) vStoreData.size(); + bs << (uint32_t)vStoreData.size(); - for (i = 0; i < vStoreData.size(); ++i) - { - const StoreData& storeData = vStoreData[i]; - bs << storeData.length; - bs << storeData.functionName; - storeData.userData->serialize(bs); - } + for (i = 0; i < vStoreData.size(); ++i) + { + const StoreData& storeData = vStoreData[i]; + bs << storeData.length; + bs << storeData.functionName; + storeData.userData->serialize(bs); + } } void UserDataStore::deserialize(ByteStream& bs) { - size_t i; - uint32_t cnt; - bs >> cnt; + size_t i; + uint32_t cnt; + bs >> cnt; -// vStoreData.clear(); - vStoreData.resize(cnt); + // vStoreData.clear(); + vStoreData.resize(cnt); - for (i = 0; i < cnt; i++) + for (i = 0; i < cnt; i++) + { + bs >> vStoreData[i].length; + bs >> vStoreData[i].functionName; + + // We don't have easy access to the context here, so we do our own lookup + if (vStoreData[i].functionName.length() == 0) { - bs >> vStoreData[i].length; - bs >> vStoreData[i].functionName; - - // We don't have easy access to the context here, so we do our own lookup - if (vStoreData[i].functionName.length() == 0) - { - throw std::logic_error("UserDataStore::deserialize: has empty name"); - } - - mcsv1sdk::UDAF_MAP::iterator funcIter = mcsv1sdk::UDAFMap::getMap().find(vStoreData[i].functionName); - - if (funcIter == mcsv1sdk::UDAFMap::getMap().end()) - { - std::ostringstream errmsg; - errmsg << "UserDataStore::deserialize: " << vStoreData[i].functionName << " is undefined"; - throw std::logic_error(errmsg.str()); - } - - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - mcsv1sdk::UserData* userData = NULL; - rc = funcIter->second->createUserData(userData, vStoreData[i].length); - - if (rc != mcsv1sdk::mcsv1_UDAF::SUCCESS) - { - std::ostringstream errmsg; - errmsg << "UserDataStore::deserialize: " << vStoreData[i].functionName << " createUserData failed(" << rc << ")"; - throw std::logic_error(errmsg.str()); - } - - userData->unserialize(bs); - vStoreData[i].userData = boost::shared_ptr(userData); + throw std::logic_error("UserDataStore::deserialize: has empty name"); } - return; -} + mcsv1sdk::UDAF_MAP::iterator funcIter = mcsv1sdk::UDAFMap::getMap().find(vStoreData[i].functionName); + if (funcIter == mcsv1sdk::UDAFMap::getMap().end()) + { + std::ostringstream errmsg; + errmsg << "UserDataStore::deserialize: " << vStoreData[i].functionName << " is undefined"; + throw std::logic_error(errmsg.str()); + } + + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + mcsv1sdk::UserData* userData = NULL; + rc = funcIter->second->createUserData(userData, vStoreData[i].length); + + if (rc != mcsv1sdk::mcsv1_UDAF::SUCCESS) + { + std::ostringstream errmsg; + errmsg << "UserDataStore::deserialize: " << vStoreData[i].functionName << " createUserData failed(" + << rc << ")"; + throw std::logic_error(errmsg.str()); + } + + userData->unserialize(bs); + vStoreData[i].userData = boost::shared_ptr(userData); + } + + return; +} RGData::RGData() { - //cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; + // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; } RGData::RGData(const RowGroup& rg, uint32_t rowCount) { - //cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; - rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); + // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; + rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); - if (rg.usesStringTable() && rowCount > 0) - strings.reset(new StringStore()); + if (rg.usesStringTable() && rowCount > 0) + strings.reset(new StringStore()); #ifdef VALGRIND - /* In a PM-join, we can serialize entire tables; not every value has been - * filled in yet. Need to look into that. Valgrind complains that - * those bytes are uninitialized, this suppresses that error. - */ - memset(rowData.get(), 0, rg.getDataSize(rowCount)); // XXXPAT: make valgrind happy temporarily + /* In a PM-join, we can serialize entire tables; not every value has been + * filled in yet. Need to look into that. Valgrind complains that + * those bytes are uninitialized, this suppresses that error. + */ + memset(rowData.get(), 0, rg.getDataSize(rowCount)); // XXXPAT: make valgrind happy temporarily #endif - memset(rowData.get(), 0, rg.getDataSize(rowCount)); // XXXPAT: make valgrind happy temporarily + memset(rowData.get(), 0, rg.getDataSize(rowCount)); // XXXPAT: make valgrind happy temporarily } RGData::RGData(const RowGroup& rg) { - //cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; - rowData.reset(new uint8_t[rg.getMaxDataSize()]); + // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; + rowData.reset(new uint8_t[rg.getMaxDataSize()]); - if (rg.usesStringTable()) - strings.reset(new StringStore()); + if (rg.usesStringTable()) + strings.reset(new StringStore()); #ifdef VALGRIND - /* In a PM-join, we can serialize entire tables; not every value has been - * filled in yet. Need to look into that. Valgrind complains that - * those bytes are uninitialized, this suppresses that error. - */ - memset(rowData.get(), 0, rg.getMaxDataSize()); + /* In a PM-join, we can serialize entire tables; not every value has been + * filled in yet. Need to look into that. Valgrind complains that + * those bytes are uninitialized, this suppresses that error. + */ + memset(rowData.get(), 0, rg.getMaxDataSize()); #endif } void RGData::reinit(const RowGroup& rg, uint32_t rowCount) { - rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); + rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); - if (rg.usesStringTable()) - strings.reset(new StringStore()); - else - strings.reset(); + if (rg.usesStringTable()) + strings.reset(new StringStore()); + else + strings.reset(); #ifdef VALGRIND - /* In a PM-join, we can serialize entire tables; not every value has been - * filled in yet. Need to look into that. Valgrind complains that - * those bytes are uninitialized, this suppresses that error. - */ - memset(rowData.get(), 0, rg.getDataSize(rowCount)); + /* In a PM-join, we can serialize entire tables; not every value has been + * filled in yet. Need to look into that. Valgrind complains that + * those bytes are uninitialized, this suppresses that error. + */ + memset(rowData.get(), 0, rg.getDataSize(rowCount)); #endif } void RGData::reinit(const RowGroup& rg) { - reinit(rg, 8192); + reinit(rg, 8192); } RGData::RGData(const RGData& r) : rowData(r.rowData), strings(r.strings), userDataStore(r.userDataStore) { - //cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; + // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; } RGData::~RGData() { - //cout << "rgdata-- = " << __sync_sub_and_fetch(&rgDataCount, 1) << endl; + // cout << "rgdata-- = " << __sync_sub_and_fetch(&rgDataCount, 1) << endl; } void RGData::serialize(ByteStream& bs, uint32_t amount) const { - //cout << "serializing!\n"; - bs << (uint32_t) RGDATA_SIG; - bs << (uint32_t) amount; - bs.append(rowData.get(), amount); + // cout << "serializing!\n"; + bs << (uint32_t)RGDATA_SIG; + bs << (uint32_t)amount; + bs.append(rowData.get(), amount); - if (strings) - { - bs << (uint8_t) 1; - strings->serialize(bs); - } - else - bs << (uint8_t) 0; + if (strings) + { + bs << (uint8_t)1; + strings->serialize(bs); + } + else + bs << (uint8_t)0; - if (userDataStore) - { - bs << (uint8_t) 1; - userDataStore->serialize(bs); - } - else - bs << (uint8_t) 0; + if (userDataStore) + { + bs << (uint8_t)1; + userDataStore->serialize(bs); + } + else + bs << (uint8_t)0; } void RGData::deserialize(ByteStream& bs, uint32_t defAmount) { - uint32_t amount, sig; - uint8_t* buf; - uint8_t tmp8; + uint32_t amount, sig; + uint8_t* buf; + uint8_t tmp8; - bs.peek(sig); + bs.peek(sig); - if (sig == RGDATA_SIG) + if (sig == RGDATA_SIG) + { + bs >> sig; + bs >> amount; + rowData.reset(new uint8_t[std::max(amount, defAmount)]); + buf = bs.buf(); + memcpy(rowData.get(), buf, amount); + bs.advance(amount); + bs >> tmp8; + + if (tmp8) { - bs >> sig; - bs >> amount; - rowData.reset(new uint8_t[std::max(amount, defAmount)]); - buf = bs.buf(); - memcpy(rowData.get(), buf, amount); - bs.advance(amount); - bs >> tmp8; - - if (tmp8) - { - strings.reset(new StringStore()); - strings->deserialize(bs); - } - else - strings.reset(); - - // UDAF user data - bs >> tmp8; - - if (tmp8) - { - userDataStore.reset(new UserDataStore()); - userDataStore->deserialize(bs); - } - else - userDataStore.reset(); + strings.reset(new StringStore()); + strings->deserialize(bs); } + else + strings.reset(); - return; + // UDAF user data + bs >> tmp8; + + if (tmp8) + { + userDataStore.reset(new UserDataStore()); + userDataStore->deserialize(bs); + } + else + userDataStore.reset(); + } + + return; } void RGData::clear() { - rowData.reset(); - strings.reset(); + rowData.reset(); + strings.reset(); } // UserDataStore is only used for UDAF. // Just in time construction because most of the time we don't need one. UserDataStore* RGData::getUserDataStore() { - if (!userDataStore) - { - userDataStore.reset(new UserDataStore); - } + if (!userDataStore) + { + userDataStore.reset(new UserDataStore); + } - return userDataStore.get(); + return userDataStore.get(); } -Row::Row() : data(NULL), strings(NULL), userDataStore(NULL) { } +Row::Row() : data(NULL), strings(NULL), userDataStore(NULL) +{ +} -Row::Row(const Row& r) : columnCount(r.columnCount), baseRid(r.baseRid), - oldOffsets(r.oldOffsets), stOffsets(r.stOffsets), - offsets(r.offsets), colWidths(r.colWidths), types(r.types), - charsetNumbers(r.charsetNumbers), charsets(r.charsets), - data(r.data), scale(r.scale), precision(r.precision), strings(r.strings), - useStringTable(r.useStringTable), hasCollation(r.hasCollation), - hasLongStringField(r.hasLongStringField), sTableThreshold(r.sTableThreshold), - forceInline(r.forceInline), userDataStore(NULL) -{ } +Row::Row(const Row& r) + : columnCount(r.columnCount) + , baseRid(r.baseRid) + , oldOffsets(r.oldOffsets) + , stOffsets(r.stOffsets) + , offsets(r.offsets) + , colWidths(r.colWidths) + , types(r.types) + , charsetNumbers(r.charsetNumbers) + , charsets(r.charsets) + , data(r.data) + , scale(r.scale) + , precision(r.precision) + , strings(r.strings) + , useStringTable(r.useStringTable) + , hasCollation(r.hasCollation) + , hasLongStringField(r.hasLongStringField) + , sTableThreshold(r.sTableThreshold) + , forceInline(r.forceInline) + , userDataStore(NULL) +{ +} -Row::~Row() { } +Row::~Row() +{ +} Row& Row::operator=(const Row& r) { - columnCount = r.columnCount; - baseRid = r.baseRid; - oldOffsets = r.oldOffsets; - stOffsets = r.stOffsets; - offsets = r.offsets; - colWidths = r.colWidths; - types = r.types; - charsetNumbers = r.charsetNumbers; - charsets = r.charsets; - data = r.data; - scale = r.scale; - precision = r.precision; - strings = r.strings; - useStringTable = r.useStringTable; - hasCollation = r.hasCollation; - hasLongStringField = r.hasLongStringField; - sTableThreshold = r.sTableThreshold; - forceInline = r.forceInline; - return *this; + columnCount = r.columnCount; + baseRid = r.baseRid; + oldOffsets = r.oldOffsets; + stOffsets = r.stOffsets; + offsets = r.offsets; + colWidths = r.colWidths; + types = r.types; + charsetNumbers = r.charsetNumbers; + charsets = r.charsets; + data = r.data; + scale = r.scale; + precision = r.precision; + strings = r.strings; + useStringTable = r.useStringTable; + hasCollation = r.hasCollation; + hasLongStringField = r.hasLongStringField; + sTableThreshold = r.sTableThreshold; + forceInline = r.forceInline; + return *this; } string Row::toString(uint32_t rownum) const { - ostringstream os; - uint32_t i; - - //os << getRid() << ": "; - os << "[" << std::setw(5) << rownum << std::setw(0) << "]: "; - os << (int) useStringTable << ": "; - - for (i = 0; i < columnCount; i++) - { - if (isNullValue(i)) - os << "NULL "; - else - switch (types[i]) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - const utils::ConstString tmp = getConstString(i); - os << "(" << tmp.length() << ") '"; - os.write(tmp.str(), tmp.length()); - os << "' "; - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - os << getFloatField(i) << " "; - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - os << getDoubleField(i) << " "; - break; - - case CalpontSystemCatalog::LONGDOUBLE: - os << getLongDoubleField(i) << " "; - break; - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - uint32_t len = getVarBinaryLength(i); - const uint8_t* val = getVarBinaryField(i); - os << "0x" << hex; - - while (len-- > 0) - { - os << (uint32_t)(*val >> 4); - os << (uint32_t)(*val++ & 0x0F); - } - - os << " " << dec; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (colWidths[i] == datatypes::MAXDECIMALWIDTH) - { - datatypes::Decimal dec(0, - scale[i], - precision[i], - getBinaryField(i)); - os << dec << " "; - break; - } - //fallthrough - default: - os << getIntField(i) << " "; - break; - } - } - - return os.str(); -} - -string Row::toCSV() const -{ - ostringstream os; - - for (uint32_t i = 0; i < columnCount; i++) - { - if (i > 0) - { - os << ","; - } - - if (isNullValue(i)) - os << "NULL"; - else - switch (types[i]) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - os << getStringField(i).c_str(); - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - os << getFloatField(i); - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - os << getDoubleField(i); - break; - - case CalpontSystemCatalog::LONGDOUBLE: - os << getLongDoubleField(i); - break; - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - uint32_t len = getVarBinaryLength(i); - const uint8_t* val = getVarBinaryField(i); - os << "0x" << hex; - - while (len-- > 0) - { - os << (uint32_t)(*val >> 4); - os << (uint32_t)(*val++ & 0x0F); - } - - os << dec; - break; - } - - default: - os << getIntField(i); - break; - } - } - - return os.str(); -} - -void Row::initToNull() -{ - uint32_t i; - - for (i = 0; i < columnCount; i++) - { - switch (types[i]) - { - case CalpontSystemCatalog::TINYINT: - data[offsets[i]] = joblist::TINYINTNULL; - break; - - case CalpontSystemCatalog::SMALLINT: - *((int16_t*) &data[offsets[i]]) = static_cast(joblist::SMALLINTNULL); - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - *((int32_t*) &data[offsets[i]]) = static_cast(joblist::INTNULL); - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - *((int32_t*) &data[offsets[i]]) = static_cast(joblist::FLOATNULL); - break; - - case CalpontSystemCatalog::DATE: - *((int32_t*) &data[offsets[i]]) = static_cast(joblist::DATENULL); - break; - - case CalpontSystemCatalog::BIGINT: - if (precision[i] != 9999) - *((uint64_t*) &data[offsets[i]]) = joblist::BIGINTNULL; - else // work around for count() in outer join result. - *((uint64_t*) &data[offsets[i]]) = 0; - - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - *((uint64_t*) &data[offsets[i]]) = joblist::DOUBLENULL; - break; - - case CalpontSystemCatalog::LONGDOUBLE: - *((long double*) &data[offsets[i]]) = joblist::LONGDOUBLENULL; - break; - - case CalpontSystemCatalog::DATETIME: - *((uint64_t*) &data[offsets[i]]) = joblist::DATETIMENULL; - break; - - case CalpontSystemCatalog::TIMESTAMP: - *((uint64_t*) &data[offsets[i]]) = joblist::TIMESTAMPNULL; - break; - - case CalpontSystemCatalog::TIME: - *((uint64_t*) &data[offsets[i]]) = joblist::TIMENULL; - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::STRINT: - { - if (inStringTable(i)) - { - setStringField(joblist::CPNULLSTRMARK, i); - break; - } - - uint32_t len = getColumnWidth(i); - - switch (len) - { - case 1: - data[offsets[i]] = joblist::CHAR1NULL; - break; - - case 2: - *((uint16_t*) &data[offsets[i]]) = joblist::CHAR2NULL; - break; - - case 3: - case 4: - *((uint32_t*) &data[offsets[i]]) = joblist::CHAR4NULL; - break; - - case 5: - case 6: - case 7: - case 8: - *((uint64_t*) &data[offsets[i]]) = joblist::CHAR8NULL; - break; - - default: - *((uint64_t*) &data[offsets[i]]) = *((uint64_t*) joblist::CPNULLSTRMARK.c_str()); - memset(&data[offsets[i] + 8], 0, len - 8); - break; - } - - break; - } - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - *((uint16_t*) &data[offsets[i]]) = 0; - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - uint32_t len = getColumnWidth(i); - - switch (len) - { - case 1 : - data[offsets[i]] = joblist::TINYINTNULL; - break; - - case 2 : - *((int16_t*) &data[offsets[i]]) = static_cast(joblist::SMALLINTNULL); - break; - - case 4 : - *((int32_t*) &data[offsets[i]]) = static_cast(joblist::INTNULL); - break; - - case 16 : - { - int128_t* s128ValuePtr = (int128_t*)(&data[offsets[i]]); - datatypes::TSInt128::storeUnaligned(s128ValuePtr, - datatypes::Decimal128Null); - } - break; - default: - *((int64_t*) &data[offsets[i]]) = static_cast(joblist::BIGINTNULL); - break; - } - - break; - } - - case CalpontSystemCatalog::UTINYINT: - data[offsets[i]] = joblist::UTINYINTNULL; - break; - - case CalpontSystemCatalog::USMALLINT: - *((uint16_t*) &data[offsets[i]]) = joblist::USMALLINTNULL; - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - *((uint32_t*) &data[offsets[i]]) = joblist::UINTNULL; - break; - - case CalpontSystemCatalog::UBIGINT: - *((uint64_t*) &data[offsets[i]]) = joblist::UBIGINTNULL; - break; - - default: - ostringstream os; - os << "Row::initToNull(): got bad column type (" << types[i] << - "). Width=" << getColumnWidth(i) << endl; - os << toString(); - throw logic_error(os.str()); - } - } -} - -template -inline bool Row::isNullValue_offset(uint32_t offset) const -{ - ostringstream os; - os << "Row::isNullValue(): got bad column type at offset("; - os << offset; - os << "). Width="; - os << width << endl; - throw logic_error(os.str()); -} - -template<> -inline bool -Row::isNullValue_offset( - uint32_t offset) const -{ - const int128_t *intPtr = reinterpret_cast(&data[offset]); - return datatypes::Decimal::isWideDecimalNullValue (*intPtr); -} - -template<> -inline bool -Row::isNullValue_offset( - uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) - == static_cast(joblist::BIGINTNULL)); -} - -template<> -inline bool -Row::isNullValue_offset( - uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) - == static_cast(joblist::INTNULL)); -} - -template<> -inline bool -Row::isNullValue_offset( - uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) - == static_cast(joblist::SMALLINTNULL)); -} - -template<> -inline bool -Row::isNullValue_offset( - uint32_t offset) const -{ - return (data[offset] == joblist::TINYINTNULL); -} - -bool Row::isNullValue(uint32_t colIndex) const -{ - switch (types[colIndex]) - { - case CalpontSystemCatalog::TINYINT: - return (data[offsets[colIndex]] == joblist::TINYINTNULL); - - case CalpontSystemCatalog::SMALLINT: - return (*((int16_t*) &data[offsets[colIndex]]) == static_cast(joblist::SMALLINTNULL)); - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - return (*((int32_t*) &data[offsets[colIndex]]) == static_cast(joblist::INTNULL)); - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - return (*((int32_t*) &data[offsets[colIndex]]) == static_cast(joblist::FLOATNULL)); - - case CalpontSystemCatalog::DATE: - return (*((int32_t*) &data[offsets[colIndex]]) == static_cast(joblist::DATENULL)); - - case CalpontSystemCatalog::BIGINT: - return (*((int64_t*) &data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DOUBLENULL); - - case CalpontSystemCatalog::DATETIME: - return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DATETIMENULL); - - case CalpontSystemCatalog::TIMESTAMP: - return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMESTAMPNULL); - - case CalpontSystemCatalog::TIME: - return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMENULL); - + ostringstream os; + uint32_t i; + + // os << getRid() << ": "; + os << "[" << std::setw(5) << rownum << std::setw(0) << "]: "; + os << (int)useStringTable << ": "; + + for (i = 0; i < columnCount; i++) + { + if (isNullValue(i)) + os << "NULL "; + else + switch (types[i]) + { case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::STRINT: { - uint32_t len = getColumnWidth(colIndex); + const utils::ConstString tmp = getConstString(i); + os << "(" << tmp.length() << ") '"; + os.write(tmp.str(), tmp.length()); + os << "' "; + break; + } - if (inStringTable(colIndex)) - { - uint64_t offset; - offset = *((uint64_t*) &data[offsets[colIndex]]); - return strings->isNullValue(offset); - } + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: os << getFloatField(i) << " "; break; - if (data[offsets[colIndex]] == 0) // empty string - return true; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: os << getDoubleField(i) << " "; break; - switch (len) - { - case 1: - return (data[offsets[colIndex]] == joblist::CHAR1NULL); + case CalpontSystemCatalog::LONGDOUBLE: os << getLongDoubleField(i) << " "; break; - case 2: - return (*((uint16_t*) &data[offsets[colIndex]]) == joblist::CHAR2NULL); + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + uint32_t len = getVarBinaryLength(i); + const uint8_t* val = getVarBinaryField(i); + os << "0x" << hex; - case 3: - case 4: - return (*((uint32_t*) &data[offsets[colIndex]]) == joblist::CHAR4NULL); + while (len-- > 0) + { + os << (uint32_t)(*val >> 4); + os << (uint32_t)(*val++ & 0x0F); + } - case 5: - case 6: - case 7: - case 8: - return - (*((uint64_t*) &data[offsets[colIndex]]) == joblist::CHAR8NULL); - default: - return (*((uint64_t*) &data[offsets[colIndex]]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())); - } - - break; + os << " " << dec; + break; } case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - { - // TODO MCOL-641 Allmighty hack. - switch (getColumnWidth(colIndex)) - { - // MCOL-641 - case 16: - return isNullValue_offset - (offsets[colIndex]); - case 1 : - return (data[offsets[colIndex]] == joblist::TINYINTNULL); - - case 2 : - return (*((int16_t*) &data[offsets[colIndex]]) == static_cast(joblist::SMALLINTNULL)); - - case 4 : - return (*((int32_t*) &data[offsets[colIndex]]) == static_cast(joblist::INTNULL)); - - default: - return (*((int64_t*) &data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); - } - + if (colWidths[i] == datatypes::MAXDECIMALWIDTH) + { + datatypes::Decimal dec(0, scale[i], precision[i], getBinaryField(i)); + os << dec << " "; break; - } + } + // fallthrough + default: os << getIntField(i) << " "; break; + } + } - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::VARBINARY: - { - uint32_t pos = offsets[colIndex]; + return os.str(); +} - if (inStringTable(colIndex)) - { - uint64_t offset; - offset = *((uint64_t*) &data[pos]); - return strings->isNullValue(offset); - } +string Row::toCSV() const +{ + ostringstream os; - if (*((uint16_t*) &data[pos]) == 0) - return true; - else if ((strncmp((char*) &data[pos + 2], joblist::CPNULLSTRMARK.c_str(), 8) == 0) && - *((uint16_t*) &data[pos]) == joblist::CPNULLSTRMARK.length()) - return true; - - break; - } - - case CalpontSystemCatalog::UTINYINT: - return (data[offsets[colIndex]] == joblist::UTINYINTNULL); - - case CalpontSystemCatalog::USMALLINT: - return (*((uint16_t*) &data[offsets[colIndex]]) == joblist::USMALLINTNULL); - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - return (*((uint32_t*) &data[offsets[colIndex]]) == joblist::UINTNULL); - - case CalpontSystemCatalog::UBIGINT: - return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::UBIGINTNULL); - - case CalpontSystemCatalog::LONGDOUBLE: - return (*((long double*) &data[offsets[colIndex]]) == joblist::LONGDOUBLENULL); - break; - - default: - { - ostringstream os; - os << "Row::isNullValue(): got bad column type ("; - os << types[colIndex]; - os << "). Width="; - os << getColumnWidth(colIndex) << endl; - throw logic_error(os.str()); - } + for (uint32_t i = 0; i < columnCount; i++) + { + if (i > 0) + { + os << ","; } - return false; + if (isNullValue(i)) + os << "NULL"; + else + switch (types[i]) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: os << getStringField(i).c_str(); break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: os << getFloatField(i); break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: os << getDoubleField(i); break; + + case CalpontSystemCatalog::LONGDOUBLE: os << getLongDoubleField(i); break; + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + uint32_t len = getVarBinaryLength(i); + const uint8_t* val = getVarBinaryField(i); + os << "0x" << hex; + + while (len-- > 0) + { + os << (uint32_t)(*val >> 4); + os << (uint32_t)(*val++ & 0x0F); + } + + os << dec; + break; + } + + default: os << getIntField(i); break; + } + } + + return os.str(); +} + +void Row::initToNull() +{ + uint32_t i; + + for (i = 0; i < columnCount; i++) + { + switch (types[i]) + { + case CalpontSystemCatalog::TINYINT: data[offsets[i]] = joblist::TINYINTNULL; break; + + case CalpontSystemCatalog::SMALLINT: + *((int16_t*)&data[offsets[i]]) = static_cast(joblist::SMALLINTNULL); + break; + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + *((int32_t*)&data[offsets[i]]) = static_cast(joblist::INTNULL); + break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + *((int32_t*)&data[offsets[i]]) = static_cast(joblist::FLOATNULL); + break; + + case CalpontSystemCatalog::DATE: + *((int32_t*)&data[offsets[i]]) = static_cast(joblist::DATENULL); + break; + + case CalpontSystemCatalog::BIGINT: + if (precision[i] != 9999) + *((uint64_t*)&data[offsets[i]]) = joblist::BIGINTNULL; + else // work around for count() in outer join result. + *((uint64_t*)&data[offsets[i]]) = 0; + + break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: *((uint64_t*)&data[offsets[i]]) = joblist::DOUBLENULL; break; + + case CalpontSystemCatalog::LONGDOUBLE: + *((long double*)&data[offsets[i]]) = joblist::LONGDOUBLENULL; + break; + + case CalpontSystemCatalog::DATETIME: *((uint64_t*)&data[offsets[i]]) = joblist::DATETIMENULL; break; + + case CalpontSystemCatalog::TIMESTAMP: *((uint64_t*)&data[offsets[i]]) = joblist::TIMESTAMPNULL; break; + + case CalpontSystemCatalog::TIME: *((uint64_t*)&data[offsets[i]]) = joblist::TIMENULL; break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::STRINT: + { + if (inStringTable(i)) + { + setStringField(joblist::CPNULLSTRMARK, i); + break; + } + + uint32_t len = getColumnWidth(i); + + switch (len) + { + case 1: data[offsets[i]] = joblist::CHAR1NULL; break; + + case 2: *((uint16_t*)&data[offsets[i]]) = joblist::CHAR2NULL; break; + + case 3: + case 4: *((uint32_t*)&data[offsets[i]]) = joblist::CHAR4NULL; break; + + case 5: + case 6: + case 7: + case 8: *((uint64_t*)&data[offsets[i]]) = joblist::CHAR8NULL; break; + + default: + *((uint64_t*)&data[offsets[i]]) = *((uint64_t*)joblist::CPNULLSTRMARK.c_str()); + memset(&data[offsets[i] + 8], 0, len - 8); + break; + } + + break; + } + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: *((uint16_t*)&data[offsets[i]]) = 0; break; + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + uint32_t len = getColumnWidth(i); + + switch (len) + { + case 1: data[offsets[i]] = joblist::TINYINTNULL; break; + + case 2: *((int16_t*)&data[offsets[i]]) = static_cast(joblist::SMALLINTNULL); break; + + case 4: *((int32_t*)&data[offsets[i]]) = static_cast(joblist::INTNULL); break; + + case 16: + { + int128_t* s128ValuePtr = (int128_t*)(&data[offsets[i]]); + datatypes::TSInt128::storeUnaligned(s128ValuePtr, datatypes::Decimal128Null); + } + break; + default: *((int64_t*)&data[offsets[i]]) = static_cast(joblist::BIGINTNULL); break; + } + + break; + } + + case CalpontSystemCatalog::UTINYINT: data[offsets[i]] = joblist::UTINYINTNULL; break; + + case CalpontSystemCatalog::USMALLINT: *((uint16_t*)&data[offsets[i]]) = joblist::USMALLINTNULL; break; + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: *((uint32_t*)&data[offsets[i]]) = joblist::UINTNULL; break; + + case CalpontSystemCatalog::UBIGINT: *((uint64_t*)&data[offsets[i]]) = joblist::UBIGINTNULL; break; + + default: + ostringstream os; + os << "Row::initToNull(): got bad column type (" << types[i] << "). Width=" << getColumnWidth(i) + << endl; + os << toString(); + throw logic_error(os.str()); + } + } +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + ostringstream os; + os << "Row::isNullValue(): got bad column type at offset("; + os << offset; + os << "). Width="; + os << width << endl; + throw logic_error(os.str()); +} + +template <> +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + const int128_t* intPtr = reinterpret_cast(&data[offset]); + return datatypes::Decimal::isWideDecimalNullValue(*intPtr); +} + +template <> +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template <> +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); +} + +template <> +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); +} + +template <> +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (data[offset] == joblist::TINYINTNULL); +} + +bool Row::isNullValue(uint32_t colIndex) const +{ + switch (types[colIndex]) + { + case CalpontSystemCatalog::TINYINT: return (data[offsets[colIndex]] == joblist::TINYINTNULL); + + case CalpontSystemCatalog::SMALLINT: + return (*((int16_t*)&data[offsets[colIndex]]) == static_cast(joblist::SMALLINTNULL)); + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + return (*((int32_t*)&data[offsets[colIndex]]) == static_cast(joblist::INTNULL)); + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + return (*((int32_t*)&data[offsets[colIndex]]) == static_cast(joblist::FLOATNULL)); + + case CalpontSystemCatalog::DATE: + return (*((int32_t*)&data[offsets[colIndex]]) == static_cast(joblist::DATENULL)); + + case CalpontSystemCatalog::BIGINT: + return (*((int64_t*)&data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + return (*((uint64_t*)&data[offsets[colIndex]]) == joblist::DOUBLENULL); + + case CalpontSystemCatalog::DATETIME: + return (*((uint64_t*)&data[offsets[colIndex]]) == joblist::DATETIMENULL); + + case CalpontSystemCatalog::TIMESTAMP: + return (*((uint64_t*)&data[offsets[colIndex]]) == joblist::TIMESTAMPNULL); + + case CalpontSystemCatalog::TIME: return (*((uint64_t*)&data[offsets[colIndex]]) == joblist::TIMENULL); + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::STRINT: + { + uint32_t len = getColumnWidth(colIndex); + + if (inStringTable(colIndex)) + { + uint64_t offset; + offset = *((uint64_t*)&data[offsets[colIndex]]); + return strings->isNullValue(offset); + } + + if (data[offsets[colIndex]] == 0) // empty string + return true; + + switch (len) + { + case 1: return (data[offsets[colIndex]] == joblist::CHAR1NULL); + + case 2: return (*((uint16_t*)&data[offsets[colIndex]]) == joblist::CHAR2NULL); + + case 3: + case 4: return (*((uint32_t*)&data[offsets[colIndex]]) == joblist::CHAR4NULL); + + case 5: + case 6: + case 7: + case 8: return (*((uint64_t*)&data[offsets[colIndex]]) == joblist::CHAR8NULL); + default: + return (*((uint64_t*)&data[offsets[colIndex]]) == *((uint64_t*)joblist::CPNULLSTRMARK.c_str())); + } + + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + // TODO MCOL-641 Allmighty hack. + switch (getColumnWidth(colIndex)) + { + // MCOL-641 + case 16: return isNullValue_offset(offsets[colIndex]); + case 1: return (data[offsets[colIndex]] == joblist::TINYINTNULL); + + case 2: return (*((int16_t*)&data[offsets[colIndex]]) == static_cast(joblist::SMALLINTNULL)); + + case 4: return (*((int32_t*)&data[offsets[colIndex]]) == static_cast(joblist::INTNULL)); + + default: return (*((int64_t*)&data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); + } + + break; + } + + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARBINARY: + { + uint32_t pos = offsets[colIndex]; + + if (inStringTable(colIndex)) + { + uint64_t offset; + offset = *((uint64_t*)&data[pos]); + return strings->isNullValue(offset); + } + + if (*((uint16_t*)&data[pos]) == 0) + return true; + else if ((strncmp((char*)&data[pos + 2], joblist::CPNULLSTRMARK.c_str(), 8) == 0) && + *((uint16_t*)&data[pos]) == joblist::CPNULLSTRMARK.length()) + return true; + + break; + } + + case CalpontSystemCatalog::UTINYINT: return (data[offsets[colIndex]] == joblist::UTINYINTNULL); + + case CalpontSystemCatalog::USMALLINT: + return (*((uint16_t*)&data[offsets[colIndex]]) == joblist::USMALLINTNULL); + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: return (*((uint32_t*)&data[offsets[colIndex]]) == joblist::UINTNULL); + + case CalpontSystemCatalog::UBIGINT: + return (*((uint64_t*)&data[offsets[colIndex]]) == joblist::UBIGINTNULL); + + case CalpontSystemCatalog::LONGDOUBLE: + return (*((long double*)&data[offsets[colIndex]]) == joblist::LONGDOUBLENULL); + break; + + default: + { + ostringstream os; + os << "Row::isNullValue(): got bad column type ("; + os << types[colIndex]; + os << "). Width="; + os << getColumnWidth(colIndex) << endl; + throw logic_error(os.str()); + } + } + + return false; } uint64_t Row::getNullValue(uint32_t colIndex) const { - return utils::getNullValue(types[colIndex], getColumnWidth(colIndex)); + return utils::getNullValue(types[colIndex], getColumnWidth(colIndex)); } /* This fcn might produce overflow warnings from the compiler, but that's OK. @@ -1052,198 +979,225 @@ uint64_t Row::getNullValue(uint32_t colIndex) const */ int64_t Row::getSignedNullValue(uint32_t colIndex) const { - return utils::getSignedNullValue(types[colIndex], getColumnWidth(colIndex)); + return utils::getSignedNullValue(types[colIndex], getColumnWidth(colIndex)); } - bool Row::equals(const Row& r2, uint32_t lastCol) const { - // This check fires with empty r2 only. - if (lastCol >= columnCount) - return true; - - // If there are no strings in the row, then we can just memcmp the whole row. - // hasCollation is true if there is any column of type CHAR, VARCHAR or TEXT - // useStringTable is true if any field declared > max inline field size, including BLOB - // For memcmp to be correct, both must be false. - if (!hasCollation && !useStringTable && !r2.hasCollation && !r2.useStringTable) - return !(memcmp(&data[offsets[0]], &r2.data[offsets[0]], offsets[lastCol + 1] - offsets[0])); - - // There are strings involved, so we need to check each column - // because binary equality is not equality for many charsets/collations - for (uint32_t col = 0; col <= lastCol; col++) - { - cscDataType columnType = getColType(col); - if (UNLIKELY(typeHasCollation(columnType))) - { - datatypes::Charset cs(getCharset(col)); - if (cs.strnncollsp(getConstString(col), r2.getConstString(col))) - { - return false; - } - } - else if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::BLOB)) - { - if (!getConstString(col).eq(r2.getConstString(col))) - return false; - } - else - { - if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) - { - if (getLongDoubleField(col) != r2.getLongDoubleField(col)) - return false; - } - else if (UNLIKELY(datatypes::isWideDecimalType(columnType, colWidths[col]))) - { - if (*getBinaryField(col) != *r2.getBinaryField(col)) - return false; - } - else if (getUintField(col) != r2.getUintField(col)) - { - return false; - } - } - } + // This check fires with empty r2 only. + if (lastCol >= columnCount) return true; + + // If there are no strings in the row, then we can just memcmp the whole row. + // hasCollation is true if there is any column of type CHAR, VARCHAR or TEXT + // useStringTable is true if any field declared > max inline field size, including BLOB + // For memcmp to be correct, both must be false. + if (!hasCollation && !useStringTable && !r2.hasCollation && !r2.useStringTable) + return !(memcmp(&data[offsets[0]], &r2.data[offsets[0]], offsets[lastCol + 1] - offsets[0])); + + // There are strings involved, so we need to check each column + // because binary equality is not equality for many charsets/collations + for (uint32_t col = 0; col <= lastCol; col++) + { + cscDataType columnType = getColType(col); + if (UNLIKELY(typeHasCollation(columnType))) + { + datatypes::Charset cs(getCharset(col)); + if (cs.strnncollsp(getConstString(col), r2.getConstString(col))) + { + return false; + } + } + else if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::BLOB)) + { + if (!getConstString(col).eq(r2.getConstString(col))) + return false; + } + else + { + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { + if (getLongDoubleField(col) != r2.getLongDoubleField(col)) + return false; + } + else if (UNLIKELY(datatypes::isWideDecimalType(columnType, colWidths[col]))) + { + if (*getBinaryField(col) != *r2.getBinaryField(col)) + return false; + } + else if (getUintField(col) != r2.getUintField(col)) + { + return false; + } + } + } + return true; } const CHARSET_INFO* Row::getCharset(uint32_t col) const { - if (charsets[col] == NULL) - { - const_cast(charsets)[col] = & datatypes::Charset(charsetNumbers[col]).getCharset(); - } - return charsets[col]; + if (charsets[col] == NULL) + { + const_cast(charsets)[col] = &datatypes::Charset(charsetNumbers[col]).getCharset(); + } + return charsets[col]; } -RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL), - useStringTable(true), hasCollation(false), hasLongStringField(false), sTableThreshold(20) +RowGroup::RowGroup() + : columnCount(0) + , data(NULL) + , rgData(NULL) + , strings(NULL) + , useStringTable(true) + , hasCollation(false) + , hasLongStringField(false) + , sTableThreshold(20) { - // 1024 is too generous to waste. - oldOffsets.reserve(10); - oids.reserve(10); - keys.reserve(10); - types.reserve(10); - charsetNumbers.reserve(10); - charsets.reserve(10); - scale.reserve(10); - precision.reserve(10); + // 1024 is too generous to waste. + oldOffsets.reserve(10); + oids.reserve(10); + keys.reserve(10); + types.reserve(10); + charsetNumbers.reserve(10); + charsets.reserve(10); + scale.reserve(10); + precision.reserve(10); } -RowGroup::RowGroup(uint32_t colCount, - const vector& positions, - const vector& roids, - const vector& tkeys, - const vector& colTypes, - const vector& csNumbers, - const vector& cscale, - const vector& cprecision, - uint32_t stringTableThreshold, - bool stringTable, - const vector& forceInlineData - ) : - columnCount(colCount), data(NULL), oldOffsets(positions), oids(roids), keys(tkeys), - types(colTypes), charsetNumbers(csNumbers), scale(cscale), precision(cprecision), rgData(NULL), strings(NULL), - sTableThreshold(stringTableThreshold) +RowGroup::RowGroup(uint32_t colCount, const vector& positions, const vector& roids, + const vector& tkeys, const vector& colTypes, + const vector& csNumbers, const vector& cscale, + const vector& cprecision, uint32_t stringTableThreshold, bool stringTable, + const vector& forceInlineData) + : columnCount(colCount) + , data(NULL) + , oldOffsets(positions) + , oids(roids) + , keys(tkeys) + , types(colTypes) + , charsetNumbers(csNumbers) + , scale(cscale) + , precision(cprecision) + , rgData(NULL) + , strings(NULL) + , sTableThreshold(stringTableThreshold) { - uint32_t i; + uint32_t i; - forceInline.reset(new bool[columnCount]); - - if (forceInlineData.empty()) - for (i = 0; i < columnCount; i++) - forceInline[i] = false; - else - for (i = 0; i < columnCount; i++) - forceInline[i] = forceInlineData[i]; - - colWidths.resize(columnCount); - stOffsets.resize(columnCount + 1); - stOffsets[0] = 2; // 2-byte rid - hasLongStringField = false; - hasCollation = false; + forceInline.reset(new bool[columnCount]); + if (forceInlineData.empty()) for (i = 0; i < columnCount; i++) + forceInline[i] = false; + else + for (i = 0; i < columnCount; i++) + forceInline[i] = forceInlineData[i]; + + colWidths.resize(columnCount); + stOffsets.resize(columnCount + 1); + stOffsets[0] = 2; // 2-byte rid + hasLongStringField = false; + hasCollation = false; + + for (i = 0; i < columnCount; i++) + { + colWidths[i] = positions[i + 1] - positions[i]; + + if (colWidths[i] >= sTableThreshold && !forceInline[i]) { - colWidths[i] = positions[i + 1] - positions[i]; - - if (colWidths[i] >= sTableThreshold && !forceInline[i]) - { - hasLongStringField = true; - stOffsets[i + 1] = stOffsets[i] + 8; - } - else - stOffsets[i + 1] = stOffsets[i] + colWidths[i]; - - if (colHasCollation(i)) - { - hasCollation = true; - } + hasLongStringField = true; + stOffsets[i + 1] = stOffsets[i] + 8; } + else + stOffsets[i + 1] = stOffsets[i] + colWidths[i]; - useStringTable = (stringTable && hasLongStringField); - offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - - // Set all the charsets to NULL for jit initialization. - charsets.insert(charsets.begin(), charsetNumbers.size(), NULL); + if (colHasCollation(i)) + { + hasCollation = true; + } + } + + useStringTable = (stringTable && hasLongStringField); + offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + + // Set all the charsets to NULL for jit initialization. + charsets.insert(charsets.begin(), charsetNumbers.size(), NULL); } -RowGroup::RowGroup(const RowGroup& r) : - columnCount(r.columnCount), data(r.data), oldOffsets(r.oldOffsets), - stOffsets(r.stOffsets), colWidths(r.colWidths), - oids(r.oids), keys(r.keys), types(r.types), charsetNumbers(r.charsetNumbers), - charsets(r.charsets), scale(r.scale), precision(r.precision), - rgData(r.rgData), strings(r.strings), useStringTable(r.useStringTable), - hasCollation(r.hasCollation), hasLongStringField(r.hasLongStringField), - sTableThreshold(r.sTableThreshold), forceInline(r.forceInline) +RowGroup::RowGroup(const RowGroup& r) + : columnCount(r.columnCount) + , data(r.data) + , oldOffsets(r.oldOffsets) + , stOffsets(r.stOffsets) + , colWidths(r.colWidths) + , oids(r.oids) + , keys(r.keys) + , types(r.types) + , charsetNumbers(r.charsetNumbers) + , charsets(r.charsets) + , scale(r.scale) + , precision(r.precision) + , rgData(r.rgData) + , strings(r.strings) + , useStringTable(r.useStringTable) + , hasCollation(r.hasCollation) + , hasLongStringField(r.hasLongStringField) + , sTableThreshold(r.sTableThreshold) + , forceInline(r.forceInline) { - //stOffsets and oldOffsets are sometimes empty... - //offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - offsets = 0; + // stOffsets and oldOffsets are sometimes empty... + // offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + offsets = 0; - if (useStringTable && !stOffsets.empty()) - offsets = &stOffsets[0]; - else if (!useStringTable && !oldOffsets.empty()) - offsets = &oldOffsets[0]; + if (useStringTable && !stOffsets.empty()) + offsets = &stOffsets[0]; + else if (!useStringTable && !oldOffsets.empty()) + offsets = &oldOffsets[0]; } RowGroup& RowGroup::operator=(const RowGroup& r) { - columnCount = r.columnCount; - oldOffsets = r.oldOffsets; - stOffsets = r.stOffsets; - colWidths = r.colWidths; - oids = r.oids; - keys = r.keys; - types = r.types; - charsetNumbers = r.charsetNumbers; - charsets = r.charsets; - data = r.data; - scale = r.scale; - precision = r.precision; - rgData = r.rgData; - strings = r.strings; - useStringTable = r.useStringTable; - hasCollation = r.hasCollation; - hasLongStringField = r.hasLongStringField; - sTableThreshold = r.sTableThreshold; - forceInline = r.forceInline; - //offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - offsets = 0; + columnCount = r.columnCount; + oldOffsets = r.oldOffsets; + stOffsets = r.stOffsets; + colWidths = r.colWidths; + oids = r.oids; + keys = r.keys; + types = r.types; + charsetNumbers = r.charsetNumbers; + charsets = r.charsets; + data = r.data; + scale = r.scale; + precision = r.precision; + rgData = r.rgData; + strings = r.strings; + useStringTable = r.useStringTable; + hasCollation = r.hasCollation; + hasLongStringField = r.hasLongStringField; + sTableThreshold = r.sTableThreshold; + forceInline = r.forceInline; + // offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + offsets = 0; - if (useStringTable && !stOffsets.empty()) - offsets = &stOffsets[0]; - else if (!useStringTable && !oldOffsets.empty()) - offsets = &oldOffsets[0]; + if (useStringTable && !stOffsets.empty()) + offsets = &stOffsets[0]; + else if (!useStringTable && !oldOffsets.empty()) + offsets = &oldOffsets[0]; - return *this; + return *this; } -RowGroup::RowGroup(ByteStream& bs): columnCount(0), data(nullptr), rgData(nullptr), strings(nullptr), - useStringTable(true), hasCollation(false), hasLongStringField(false), sTableThreshold(20) +RowGroup::RowGroup(ByteStream& bs) + : columnCount(0) + , data(nullptr) + , rgData(nullptr) + , strings(nullptr) + , useStringTable(true) + , hasCollation(false) + , hasLongStringField(false) + , sTableThreshold(20) { - this->deserialize(bs); + this->deserialize(bs); } RowGroup::~RowGroup() @@ -1252,570 +1206,549 @@ RowGroup::~RowGroup() void RowGroup::resetRowGroup(uint64_t rid) { - *((uint32_t*) &data[rowCountOffset]) = 0; - *((uint64_t*) &data[baseRidOffset]) = rid; - *((uint16_t*) &data[statusOffset]) = 0; - *((uint32_t*) &data[dbRootOffset]) = 0; + *((uint32_t*)&data[rowCountOffset]) = 0; + *((uint64_t*)&data[baseRidOffset]) = rid; + *((uint16_t*)&data[statusOffset]) = 0; + *((uint32_t*)&data[dbRootOffset]) = 0; - if (strings) - strings->clear(); + if (strings) + strings->clear(); } void RowGroup::serialize(ByteStream& bs) const { - bs << columnCount; - serializeInlineVector(bs, oldOffsets); - serializeInlineVector(bs, stOffsets); - serializeInlineVector(bs, colWidths); - serializeInlineVector(bs, oids); - serializeInlineVector(bs, keys); - serializeInlineVector(bs, types); - serializeInlineVector(bs, charsetNumbers); - serializeInlineVector(bs, scale); - serializeInlineVector(bs, precision); - bs << (uint8_t) useStringTable; - bs << (uint8_t) hasCollation; - bs << (uint8_t) hasLongStringField; - bs << sTableThreshold; - bs.append((uint8_t*) &forceInline[0], sizeof(bool) * columnCount); + bs << columnCount; + serializeInlineVector(bs, oldOffsets); + serializeInlineVector(bs, stOffsets); + serializeInlineVector(bs, colWidths); + serializeInlineVector(bs, oids); + serializeInlineVector(bs, keys); + serializeInlineVector(bs, types); + serializeInlineVector(bs, charsetNumbers); + serializeInlineVector(bs, scale); + serializeInlineVector(bs, precision); + bs << (uint8_t)useStringTable; + bs << (uint8_t)hasCollation; + bs << (uint8_t)hasLongStringField; + bs << sTableThreshold; + bs.append((uint8_t*)&forceInline[0], sizeof(bool) * columnCount); } void RowGroup::deserialize(ByteStream& bs) { - uint8_t tmp8; + uint8_t tmp8; - bs >> columnCount; - deserializeInlineVector(bs, oldOffsets); - deserializeInlineVector(bs, stOffsets); - deserializeInlineVector(bs, colWidths); - deserializeInlineVector(bs, oids); - deserializeInlineVector(bs, keys); - deserializeInlineVector(bs, types); - deserializeInlineVector(bs, charsetNumbers); - deserializeInlineVector(bs, scale); - deserializeInlineVector(bs, precision); - bs >> tmp8; - useStringTable = (bool) tmp8; - bs >> tmp8; - hasCollation = (bool) tmp8; - bs >> tmp8; - hasLongStringField = (bool) tmp8; - bs >> sTableThreshold; - forceInline.reset(new bool[columnCount]); - memcpy(&forceInline[0], bs.buf(), sizeof(bool) * columnCount); - bs.advance(sizeof(bool) * columnCount); - //offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - offsets = 0; + bs >> columnCount; + deserializeInlineVector(bs, oldOffsets); + deserializeInlineVector(bs, stOffsets); + deserializeInlineVector(bs, colWidths); + deserializeInlineVector(bs, oids); + deserializeInlineVector(bs, keys); + deserializeInlineVector(bs, types); + deserializeInlineVector(bs, charsetNumbers); + deserializeInlineVector(bs, scale); + deserializeInlineVector(bs, precision); + bs >> tmp8; + useStringTable = (bool)tmp8; + bs >> tmp8; + hasCollation = (bool)tmp8; + bs >> tmp8; + hasLongStringField = (bool)tmp8; + bs >> sTableThreshold; + forceInline.reset(new bool[columnCount]); + memcpy(&forceInline[0], bs.buf(), sizeof(bool) * columnCount); + bs.advance(sizeof(bool) * columnCount); + // offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + offsets = 0; - if (useStringTable && !stOffsets.empty()) - offsets = &stOffsets[0]; - else if (!useStringTable && !oldOffsets.empty()) - offsets = &oldOffsets[0]; + if (useStringTable && !stOffsets.empty()) + offsets = &stOffsets[0]; + else if (!useStringTable && !oldOffsets.empty()) + offsets = &oldOffsets[0]; - // Set all the charsets to NULL for jit initialization. - charsets.insert(charsets.begin(), charsetNumbers.size(), NULL); - + // Set all the charsets to NULL for jit initialization. + charsets.insert(charsets.begin(), charsetNumbers.size(), NULL); } void RowGroup::serializeRGData(ByteStream& bs) const { - //cout << "****** serializing\n" << toString() << en -// if (useStringTable || !hasLongStringField) - rgData->serialize(bs, getDataSize()); -// else { -// uint64_t size; -// RGData *compressed = convertToStringTable(&size); -// compressed->serialize(bs, size); -// if (compressed != rgData) -// delete compressed; -// } + // cout << "****** serializing\n" << toString() << en + // if (useStringTable || !hasLongStringField) + rgData->serialize(bs, getDataSize()); + // else { + // uint64_t size; + // RGData *compressed = convertToStringTable(&size); + // compressed->serialize(bs, size); + // if (compressed != rgData) + // delete compressed; + // } } uint32_t RowGroup::getDataSize() const { - return headerSize + (getRowCount() * offsets[columnCount]); + return headerSize + (getRowCount() * offsets[columnCount]); } uint32_t RowGroup::getDataSize(uint64_t n) const { - return headerSize + (n * offsets[columnCount]); + return headerSize + (n * offsets[columnCount]); } uint32_t RowGroup::getMaxDataSize() const { - return headerSize + (8192 * offsets[columnCount]); + return headerSize + (8192 * offsets[columnCount]); } uint32_t RowGroup::getMaxDataSizeWithStrings() const { - return headerSize + (8192 * oldOffsets[columnCount]); + return headerSize + (8192 * oldOffsets[columnCount]); } uint32_t RowGroup::getEmptySize() const { - return headerSize; + return headerSize; } uint32_t RowGroup::getStatus() const { - return *((uint16_t*) &data[statusOffset]); + return *((uint16_t*)&data[statusOffset]); } void RowGroup::setStatus(uint16_t err) { - *((uint16_t*) &data[statusOffset]) = err; + *((uint16_t*)&data[statusOffset]) = err; } uint32_t RowGroup::getColumnWidth(uint32_t col) const { - return colWidths[col]; + return colWidths[col]; } uint32_t RowGroup::getColumnCount() const { - return columnCount; + return columnCount; } string RowGroup::toString(const std::vector& used) const { - ostringstream os; - ostream_iterator oIter1(os, "\t"); + ostringstream os; + ostream_iterator oIter1(os, "\t"); - os << "columncount = " << columnCount << endl; - os << "oids:\t\t"; - copy(oids.begin(), oids.end(), oIter1); - os << endl; - os << "keys:\t\t"; - copy(keys.begin(), keys.end(), oIter1); - os << endl; - os << "offsets:\t"; - copy(&offsets[0], &offsets[columnCount + 1], oIter1); - os << endl; - os << "colWidths:\t"; - copy(colWidths.begin(), colWidths.end(), oIter1); - os << endl; - os << "types:\t\t"; - copy(types.begin(), types.end(), oIter1); - os << endl; - os << "scales:\t\t"; - copy(scale.begin(), scale.end(), oIter1); - os << endl; - os << "precisions:\t"; - copy(precision.begin(), precision.end(), oIter1); - os << endl; + os << "columncount = " << columnCount << endl; + os << "oids:\t\t"; + copy(oids.begin(), oids.end(), oIter1); + os << endl; + os << "keys:\t\t"; + copy(keys.begin(), keys.end(), oIter1); + os << endl; + os << "offsets:\t"; + copy(&offsets[0], &offsets[columnCount + 1], oIter1); + os << endl; + os << "colWidths:\t"; + copy(colWidths.begin(), colWidths.end(), oIter1); + os << endl; + os << "types:\t\t"; + copy(types.begin(), types.end(), oIter1); + os << endl; + os << "scales:\t\t"; + copy(scale.begin(), scale.end(), oIter1); + os << endl; + os << "precisions:\t"; + copy(precision.begin(), precision.end(), oIter1); + os << endl; - if (useStringTable) - os << "uses a string table\n"; - else - os << "doesn't use a string table\n"; + if (useStringTable) + os << "uses a string table\n"; + else + os << "doesn't use a string table\n"; + if (!used.empty()) + os << "sparse\n"; + + // os << "strings = " << hex << (int64_t) strings << "\n"; + // os << "data = " << (int64_t) data << "\n" << dec; + if (data != NULL) + { + Row r; + initRow(&r); + getRow(0, &r); + os << "rowcount = " << getRowCount() << endl; if (!used.empty()) - os << "sparse\n"; - - //os << "strings = " << hex << (int64_t) strings << "\n"; - //os << "data = " << (int64_t) data << "\n" << dec; - if (data != NULL) { - Row r; - initRow(&r); - getRow(0, &r); - os << "rowcount = " << getRowCount() << endl; - if (!used.empty()) - { - uint64_t cnt = std::accumulate(used.begin(), used.end(), 0ULL, - [](uint64_t a, uint64_t bits) { - return a + __builtin_popcountll(bits); - }); - os << "sparse row count = " << cnt << endl; - } - os << "base rid = " << getBaseRid() << endl; - os << "status = " << getStatus() << endl; - os << "dbroot = " << getDBRoot() << endl; - os << "row data...\n"; - - uint32_t max_cnt = used.empty() ? getRowCount() : (used.size() * 64); - for (uint32_t i = 0; i < max_cnt; i++) - { - if (!used.empty() && !(used[i/64] & (1ULL << (i%64)))) - continue; - os << r.toString(i) << endl; - r.nextRow(); - } + uint64_t cnt = + std::accumulate(used.begin(), used.end(), 0ULL, + [](uint64_t a, uint64_t bits) { return a + __builtin_popcountll(bits); }); + os << "sparse row count = " << cnt << endl; } + os << "base rid = " << getBaseRid() << endl; + os << "status = " << getStatus() << endl; + os << "dbroot = " << getDBRoot() << endl; + os << "row data...\n"; - return os.str(); + uint32_t max_cnt = used.empty() ? getRowCount() : (used.size() * 64); + for (uint32_t i = 0; i < max_cnt; i++) + { + if (!used.empty() && !(used[i / 64] & (1ULL << (i % 64)))) + continue; + os << r.toString(i) << endl; + r.nextRow(); + } + } + + return os.str(); } boost::shared_array makeMapping(const RowGroup& r1, const RowGroup& r2) { - shared_array ret(new int[r1.getColumnCount()]); - //bool reserved[r2.getColumnCount()]; - bool* reserved = (bool*)alloca(r2.getColumnCount() * sizeof(bool)); - uint32_t i, j; + shared_array ret(new int[r1.getColumnCount()]); + // bool reserved[r2.getColumnCount()]; + bool* reserved = (bool*)alloca(r2.getColumnCount() * sizeof(bool)); + uint32_t i, j; - for (i = 0; i < r2.getColumnCount(); i++) - reserved[i] = false; + for (i = 0; i < r2.getColumnCount(); i++) + reserved[i] = false; - for (i = 0; i < r1.getColumnCount(); i++) - { - for (j = 0; j < r2.getColumnCount(); j++) - if ((r1.getKeys()[i] == r2.getKeys()[j]) && !reserved[j]) - { - ret[i] = j; - reserved[j] = true; - break; - } + for (i = 0; i < r1.getColumnCount(); i++) + { + for (j = 0; j < r2.getColumnCount(); j++) + if ((r1.getKeys()[i] == r2.getKeys()[j]) && !reserved[j]) + { + ret[i] = j; + reserved[j] = true; + break; + } - if (j == r2.getColumnCount()) - ret[i] = -1; - } + if (j == r2.getColumnCount()) + ret[i] = -1; + } - return ret; + return ret; } void applyMapping(const boost::shared_array& mapping, const Row& in, Row* out) { - applyMapping(mapping.get(), in, out); + applyMapping(mapping.get(), in, out); } void applyMapping(const std::vector& mapping, const Row& in, Row* out) { - applyMapping((int*) &mapping[0], in, out); + applyMapping((int*)&mapping[0], in, out); } void applyMapping(const int* mapping, const Row& in, Row* out) { - uint32_t i; + uint32_t i; - for (i = 0; i < in.getColumnCount(); i++) - if (mapping[i] != -1) - { - if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || - in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || - in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT)) - out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]); - else if (UNLIKELY(in.isLongString(i))) - out->setStringField(in.getConstString(i), mapping[i]); - else if (UNLIKELY(in.isShortString(i))) - out->setUintField(in.getUintField(i), mapping[i]); - else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) - out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]); - // WIP this doesn't look right b/c we can pushdown colType - // Migrate to offset based methods here - // code precision 2 width convertor - else if (UNLIKELY(datatypes::isWideDecimalType(in.getColTypes()[i], - in.getColumnWidth(i)))) - out->setBinaryField_offset(in.getBinaryField(i), 16, - out->getOffset(mapping[i])); - else if (in.isUnsigned(i)) - out->setUintField(in.getUintField(i), mapping[i]); - else - out->setIntField(in.getIntField(i), mapping[i]); - } + for (i = 0; i < in.getColumnCount(); i++) + if (mapping[i] != -1) + { + if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || + in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || + in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT)) + out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]); + else if (UNLIKELY(in.isLongString(i))) + out->setStringField(in.getConstString(i), mapping[i]); + else if (UNLIKELY(in.isShortString(i))) + out->setUintField(in.getUintField(i), mapping[i]); + else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]); + // WIP this doesn't look right b/c we can pushdown colType + // Migrate to offset based methods here + // code precision 2 width convertor + else if (UNLIKELY(datatypes::isWideDecimalType(in.getColTypes()[i], in.getColumnWidth(i)))) + out->setBinaryField_offset(in.getBinaryField(i), 16, out->getOffset(mapping[i])); + else if (in.isUnsigned(i)) + out->setUintField(in.getUintField(i), mapping[i]); + else + out->setIntField(in.getIntField(i), mapping[i]); + } } RowGroup& RowGroup::operator+=(const RowGroup& rhs) { - boost::shared_array tmp; - uint32_t i, j; - //not appendable if data is set - assert(!data); + boost::shared_array tmp; + uint32_t i, j; + // not appendable if data is set + assert(!data); - tmp.reset(new bool[columnCount + rhs.columnCount]); + tmp.reset(new bool[columnCount + rhs.columnCount]); - for (i = 0; i < columnCount; i++) - tmp[i] = forceInline[i]; + for (i = 0; i < columnCount; i++) + tmp[i] = forceInline[i]; - for (j = 0; j < rhs.columnCount; i++, j++) - tmp[i] = rhs.forceInline[j]; + for (j = 0; j < rhs.columnCount; i++, j++) + tmp[i] = rhs.forceInline[j]; - forceInline.swap(tmp); + forceInline.swap(tmp); - columnCount += rhs.columnCount; - oids.insert(oids.end(), rhs.oids.begin(), rhs.oids.end()); - keys.insert(keys.end(), rhs.keys.begin(), rhs.keys.end()); - types.insert(types.end(), rhs.types.begin(), rhs.types.end()); - charsetNumbers.insert(charsetNumbers.end(), rhs.charsetNumbers.begin(), rhs.charsetNumbers.end()); - charsets.insert(charsets.end(), rhs.charsets.begin(), rhs.charsets.end()); - scale.insert(scale.end(), rhs.scale.begin(), rhs.scale.end()); - precision.insert(precision.end(), rhs.precision.begin(), rhs.precision.end()); - colWidths.insert(colWidths.end(), rhs.colWidths.begin(), rhs.colWidths.end()); + columnCount += rhs.columnCount; + oids.insert(oids.end(), rhs.oids.begin(), rhs.oids.end()); + keys.insert(keys.end(), rhs.keys.begin(), rhs.keys.end()); + types.insert(types.end(), rhs.types.begin(), rhs.types.end()); + charsetNumbers.insert(charsetNumbers.end(), rhs.charsetNumbers.begin(), rhs.charsetNumbers.end()); + charsets.insert(charsets.end(), rhs.charsets.begin(), rhs.charsets.end()); + scale.insert(scale.end(), rhs.scale.begin(), rhs.scale.end()); + precision.insert(precision.end(), rhs.precision.begin(), rhs.precision.end()); + colWidths.insert(colWidths.end(), rhs.colWidths.begin(), rhs.colWidths.end()); - // +4 +4 +8 +2 +4 +8 - // (2, 6, 10, 18) + (2, 4, 8, 16) = (2, 6, 10, 18, 20, 24, 32) - for (i = 1; i < rhs.stOffsets.size(); i++) - { - stOffsets.push_back(stOffsets.back() + rhs.stOffsets[i] - rhs.stOffsets[i - 1]); - oldOffsets.push_back(oldOffsets.back() + rhs.oldOffsets[i] - rhs.oldOffsets[i - 1]); - } + // +4 +4 +8 +2 +4 +8 + // (2, 6, 10, 18) + (2, 4, 8, 16) = (2, 6, 10, 18, 20, 24, 32) + for (i = 1; i < rhs.stOffsets.size(); i++) + { + stOffsets.push_back(stOffsets.back() + rhs.stOffsets[i] - rhs.stOffsets[i - 1]); + oldOffsets.push_back(oldOffsets.back() + rhs.oldOffsets[i] - rhs.oldOffsets[i - 1]); + } - hasLongStringField = rhs.hasLongStringField || hasLongStringField; - useStringTable = rhs.useStringTable || useStringTable; - hasCollation = rhs.hasCollation || hasCollation; - offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + hasLongStringField = rhs.hasLongStringField || hasLongStringField; + useStringTable = rhs.useStringTable || useStringTable; + hasCollation = rhs.hasCollation || hasCollation; + offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - return *this; + return *this; } RowGroup operator+(const RowGroup& lhs, const RowGroup& rhs) { - RowGroup temp(lhs); - return temp += rhs; + RowGroup temp(lhs); + return temp += rhs; } uint32_t RowGroup::getDBRoot() const { - return *((uint32_t*) &data[dbRootOffset]); + return *((uint32_t*)&data[dbRootOffset]); } void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList) { - execplan::ColumnResult* cr; + execplan::ColumnResult* cr; - rowgroup::Row row; - initRow(&row); - uint32_t rowCount = getRowCount(); - uint32_t columnCount = getColumnCount(); + rowgroup::Row row; + initRow(&row); + uint32_t rowCount = getRowCount(); + uint32_t columnCount = getColumnCount(); - for (uint32_t i = 0; i < rowCount; i++) + for (uint32_t i = 0; i < rowCount; i++) + { + getRow(i, &row); + + for (uint32_t j = 0; j < columnCount; j++) { - getRow(i, &row); + int idx = sysDataList.findColumn(getOIDs()[j]); - for (uint32_t j = 0; j < columnCount; j++) + if (idx >= 0) + { + cr = sysDataList.sysDataVec[idx]; + } + else + { + cr = new execplan::ColumnResult(); + cr->SetColumnOID(getOIDs()[j]); + sysDataList.push_back(cr); + } + + // @todo more data type checking. for now only check string, midint and bigint + switch ((getColTypes()[j])) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: { - int idx = sysDataList.findColumn(getOIDs()[j]); + switch (getColumnWidth(j)) + { + case 1: cr->PutData(row.getUintField<1>(j)); break; - if (idx >= 0) + case 2: cr->PutData(row.getUintField<2>(j)); break; + + case 4: cr->PutData(row.getUintField<4>(j)); break; + + case 8: cr->PutData(row.getUintField<8>(j)); break; + case 16: + + default: { - cr = sysDataList.sysDataVec[idx]; - } - else - { - cr = new execplan::ColumnResult(); - cr->SetColumnOID(getOIDs()[j]); - sysDataList.push_back(cr); + string s = row.getStringField(j); + cr->PutStringData(string(s.c_str(), strlen(s.c_str()))); } + } - // @todo more data type checking. for now only check string, midint and bigint - switch ((getColTypes()[j])) - { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - switch (getColumnWidth(j)) - { - case 1: - cr->PutData(row.getUintField<1>(j)); - break; - - case 2: - cr->PutData(row.getUintField<2>(j)); - break; - - case 4: - cr->PutData(row.getUintField<4>(j)); - break; - - case 8: - cr->PutData(row.getUintField<8>(j)); - break; - case 16: - - default: - { - string s = row.getStringField(j); - cr->PutStringData(string(s.c_str(), strlen(s.c_str()))); - } - } - - break; - } - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::UINT: - cr->PutData(row.getIntField<4>(j)); - break; - - case CalpontSystemCatalog::DATE: - cr->PutData(row.getUintField<4>(j)); - break; - - default: - cr->PutData(row.getIntField<8>(j)); - } - - cr->PutRid(row.getFileRelativeRid()); + break; } + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::UINT: cr->PutData(row.getIntField<4>(j)); break; + + case CalpontSystemCatalog::DATE: cr->PutData(row.getUintField<4>(j)); break; + + default: cr->PutData(row.getIntField<8>(j)); + } + + cr->PutRid(row.getFileRelativeRid()); } + } } const CHARSET_INFO* RowGroup::getCharset(uint32_t col) { - if (charsets[col] == NULL) - { - charsets[col] = & datatypes::Charset(charsetNumbers[col]).getCharset(); - } - return charsets[col]; + if (charsets[col] == NULL) + { + charsets[col] = &datatypes::Charset(charsetNumbers[col]).getCharset(); + } + return charsets[col]; } void RowGroup::setDBRoot(uint32_t dbroot) { - *((uint32_t*) &data[dbRootOffset]) = dbroot; + *((uint32_t*)&data[dbRootOffset]) = dbroot; } RGData RowGroup::duplicate() { - RGData ret(*this, getRowCount()); + RGData ret(*this, getRowCount()); - if (useStringTable) + if (useStringTable) + { + // this isn't a straight memcpy of everything b/c it might be remapping strings. + // think about a big memcpy + a remap operation; might be faster. + Row r1, r2; + RowGroup rg(*this); + rg.setData(&ret); + rg.resetRowGroup(getBaseRid()); + rg.setStatus(getStatus()); + rg.setRowCount(getRowCount()); + rg.setDBRoot(getDBRoot()); + initRow(&r1); + initRow(&r2); + getRow(0, &r1); + rg.getRow(0, &r2); + + for (uint32_t i = 0; i < getRowCount(); i++) { - // this isn't a straight memcpy of everything b/c it might be remapping strings. - // think about a big memcpy + a remap operation; might be faster. - Row r1, r2; - RowGroup rg(*this); - rg.setData(&ret); - rg.resetRowGroup(getBaseRid()); - rg.setStatus(getStatus()); - rg.setRowCount(getRowCount()); - rg.setDBRoot(getDBRoot()); - initRow(&r1); - initRow(&r2); - getRow(0, &r1); - rg.getRow(0, &r2); - - for (uint32_t i = 0; i < getRowCount(); i++) - { - copyRow(r1, &r2); - r1.nextRow(); - r2.nextRow(); - } + copyRow(r1, &r2); + r1.nextRow(); + r2.nextRow(); } - else - memcpy(ret.rowData.get(), data, getDataSize()); + } + else + memcpy(ret.rowData.get(), data, getDataSize()); - return ret; + return ret; } - void Row::setStringField(const std::string& val, uint32_t colIndex) { - uint64_t offset; - uint64_t length; + uint64_t offset; + uint64_t length; - //length = strlen(val.c_str()) + 1; - length = val.length(); + // length = strlen(val.c_str()) + 1; + length = val.length(); - if (length > getColumnWidth(colIndex)) - length = getColumnWidth(colIndex); + if (length > getColumnWidth(colIndex)) + length = getColumnWidth(colIndex); - if (inStringTable(colIndex)) - { - offset = strings->storeString((const uint8_t*) val.data(), length); - *((uint64_t*) &data[offsets[colIndex]]) = offset; -// cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) -// << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) -// << endl; - } - else - { - memcpy(&data[offsets[colIndex]], val.data(), length); - memset(&data[offsets[colIndex] + length], 0, - offsets[colIndex + 1] - (offsets[colIndex] + length)); - } + if (inStringTable(colIndex)) + { + offset = strings->storeString((const uint8_t*)val.data(), length); + *((uint64_t*)&data[offsets[colIndex]]) = offset; + // cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) + // << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) + // << endl; + } + else + { + memcpy(&data[offsets[colIndex]], val.data(), length); + memset(&data[offsets[colIndex] + length], 0, offsets[colIndex + 1] - (offsets[colIndex] + length)); + } } void RowGroup::append(RGData& rgd) { - RowGroup tmp(*this); - Row src, dest; + RowGroup tmp(*this); + Row src, dest; - tmp.setData(&rgd); - initRow(&src); - initRow(&dest); - tmp.getRow(0, &src); - getRow(getRowCount(), &dest); + tmp.setData(&rgd); + initRow(&src); + initRow(&dest); + tmp.getRow(0, &src); + getRow(getRowCount(), &dest); - for (uint32_t i = 0; i < tmp.getRowCount(); i++, src.nextRow(), dest.nextRow()) - { - //cerr << "appending row: " << src.toString() << endl; - copyRow(src, &dest); - } + for (uint32_t i = 0; i < tmp.getRowCount(); i++, src.nextRow(), dest.nextRow()) + { + // cerr << "appending row: " << src.toString() << endl; + copyRow(src, &dest); + } - setRowCount(getRowCount() + tmp.getRowCount()); + setRowCount(getRowCount() + tmp.getRowCount()); } void RowGroup::append(RowGroup& rg) { - append(*rg.getRGData()); + append(*rg.getRGData()); } void RowGroup::append(RGData& rgd, uint32_t startPos) { - RowGroup tmp(*this); - Row src, dest; + RowGroup tmp(*this); + Row src, dest; - tmp.setData(&rgd); - initRow(&src); - initRow(&dest); - tmp.getRow(0, &src); - getRow(startPos, &dest); + tmp.setData(&rgd); + initRow(&src); + initRow(&dest); + tmp.getRow(0, &src); + getRow(startPos, &dest); - for (uint32_t i = 0; i < tmp.getRowCount(); i++, src.nextRow(), dest.nextRow()) - { - //cerr << "appending row: " << src.toString() << endl; - copyRow(src, &dest); - } + for (uint32_t i = 0; i < tmp.getRowCount(); i++, src.nextRow(), dest.nextRow()) + { + // cerr << "appending row: " << src.toString() << endl; + copyRow(src, &dest); + } - setRowCount(getRowCount() + tmp.getRowCount()); + setRowCount(getRowCount() + tmp.getRowCount()); } void RowGroup::append(RowGroup& rg, uint32_t startPos) { - append(*rg.getRGData(), startPos); + append(*rg.getRGData(), startPos); } RowGroup RowGroup::truncate(uint32_t cols) { - idbassert(cols <= columnCount); + idbassert(cols <= columnCount); - RowGroup ret(*this); - ret.columnCount = cols; - ret.oldOffsets.resize(cols + 1); - ret.stOffsets.resize(cols + 1); - ret.colWidths.resize(cols); - ret.oids.resize(cols); - ret.keys.resize(cols); - ret.types.resize(cols); - ret.charsetNumbers.resize(cols); - ret.charsets.resize(cols); - ret.scale.resize(cols); - ret.precision.resize(cols); - ret.forceInline.reset(new bool[cols]); - memcpy(ret.forceInline.get(), forceInline.get(), cols * sizeof(bool)); + RowGroup ret(*this); + ret.columnCount = cols; + ret.oldOffsets.resize(cols + 1); + ret.stOffsets.resize(cols + 1); + ret.colWidths.resize(cols); + ret.oids.resize(cols); + ret.keys.resize(cols); + ret.types.resize(cols); + ret.charsetNumbers.resize(cols); + ret.charsets.resize(cols); + ret.scale.resize(cols); + ret.precision.resize(cols); + ret.forceInline.reset(new bool[cols]); + memcpy(ret.forceInline.get(), forceInline.get(), cols * sizeof(bool)); - ret.hasLongStringField = false; - ret.hasCollation = false; + ret.hasLongStringField = false; + ret.hasCollation = false; - for (uint32_t i = 0; i < columnCount && - (!ret.hasLongStringField || !ret.hasCollation); i++) + for (uint32_t i = 0; i < columnCount && (!ret.hasLongStringField || !ret.hasCollation); i++) + { + if (colWidths[i] >= sTableThreshold && !forceInline[i]) { - if (colWidths[i] >= sTableThreshold && !forceInline[i]) - { - ret.hasLongStringField = true; - } - - if (colHasCollation(i)) - { - ret.hasCollation = true; - } + ret.hasLongStringField = true; } - ret.useStringTable = (ret.useStringTable && ret.hasLongStringField); - ret.offsets = (ret.useStringTable ? &ret.stOffsets[0] : &ret.oldOffsets[0]); - return ret; + if (colHasCollation(i)) + { + ret.hasCollation = true; + } + } + + ret.useStringTable = (ret.useStringTable && ret.hasLongStringField); + ret.offsets = (ret.useStringTable ? &ret.stOffsets[0] : &ret.oldOffsets[0]); + return ret; } -} +} // namespace rowgroup // vim:ts=4 sw=4: - diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 44a7fd532..a17449d00 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -66,7 +66,6 @@ namespace rowgroup { - const int16_t rgCommonSize = 8192; /* @@ -127,136 +126,134 @@ const int16_t rgCommonSize = 8192; // VS'08 carps that struct MemChunk is not default copyable because of the zero-length array. // This may be so, and we'll get link errors if someone trys, but so far no one has. #ifdef _MSC_VER -#pragma warning (push) -#pragma warning (disable : 4200) +#pragma warning(push) +#pragma warning(disable : 4200) #endif // Helper to get a value from nested vector pointers. -template -inline T derefFromTwoVectorPtrs(const std::vector* outer, - const std::vector* inner, - const T innerIdx) +template +inline T derefFromTwoVectorPtrs(const std::vector* outer, const std::vector* inner, const T innerIdx) { - auto outerIdx = inner->operator[](innerIdx); - return outer->operator[](outerIdx); + auto outerIdx = inner->operator[](innerIdx); + return outer->operator[](outerIdx); } class StringStore { -public: - StringStore(); - virtual ~StringStore(); + public: + StringStore(); + virtual ~StringStore(); - inline std::string getString(uint64_t offset) const; - uint64_t storeString(const uint8_t* data, uint32_t length); //returns the offset - inline const uint8_t* getPointer(uint64_t offset) const; - inline uint32_t getStringLength(uint64_t offset) const; - inline utils::ConstString getConstString(uint64_t offset) const - { - return utils::ConstString((const char *) getPointer(offset), - getStringLength(offset)); - } - inline bool isEmpty() const; - inline uint64_t getSize() const; - inline bool isNullValue(uint64_t offset) const; + inline std::string getString(uint64_t offset) const; + uint64_t storeString(const uint8_t* data, uint32_t length); // returns the offset + inline const uint8_t* getPointer(uint64_t offset) const; + inline uint32_t getStringLength(uint64_t offset) const; + inline utils::ConstString getConstString(uint64_t offset) const + { + return utils::ConstString((const char*)getPointer(offset), getStringLength(offset)); + } + inline bool isEmpty() const; + inline uint64_t getSize() const; + inline bool isNullValue(uint64_t offset) const; - void clear(); + void clear(); - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&); + void serialize(messageqcpp::ByteStream&) const; + void deserialize(messageqcpp::ByteStream&); - //@bug6065, make StringStore::storeString() thread safe - void useStoreStringMutex(bool b) - { - fUseStoreStringMutex = b; - } - bool useStoreStringMutex() const - { - return fUseStoreStringMutex; - } + //@bug6065, make StringStore::storeString() thread safe + void useStoreStringMutex(bool b) + { + fUseStoreStringMutex = b; + } + bool useStoreStringMutex() const + { + return fUseStoreStringMutex; + } - // This is an overlay b/c the underlying data needs to be any size, - // and alloc'd in one chunk. data can't be a separate dynamic chunk. - // NOTE: Change here, requires a change in 'bytestream.h'. - struct MemChunk - { - uint32_t currentSize; - uint32_t capacity; - uint8_t data[]; - }; + // This is an overlay b/c the underlying data needs to be any size, + // and alloc'd in one chunk. data can't be a separate dynamic chunk. + // NOTE: Change here, requires a change in 'bytestream.h'. + struct MemChunk + { + uint32_t currentSize; + uint32_t capacity; + uint8_t data[]; + }; -private: - std::string empty_str; + private: + std::string empty_str; - StringStore(const StringStore&); - StringStore& operator=(const StringStore&); - static constexpr const uint32_t CHUNK_SIZE = 64 * 1024; // allocators like powers of 2 + StringStore(const StringStore&); + StringStore& operator=(const StringStore&); + static constexpr const uint32_t CHUNK_SIZE = 64 * 1024; // allocators like powers of 2 - std::vector> mem; + std::vector> mem; - // To store strings > 64KB (BLOB/TEXT) - std::vector > longStrings; - bool empty; - bool fUseStoreStringMutex; //@bug6065, make StringStore::storeString() thread safe - boost::mutex fMutex; + // To store strings > 64KB (BLOB/TEXT) + std::vector> longStrings; + bool empty; + bool fUseStoreStringMutex; //@bug6065, make StringStore::storeString() thread safe + boost::mutex fMutex; }; // Where we store user data for UDA(n)F class UserDataStore { - // length represents the fixed portion length of userData. - // There may be variable length data in containers or other - // user created structures. - struct StoreData + // length represents the fixed portion length of userData. + // There may be variable length data in containers or other + // user created structures. + struct StoreData + { + int32_t length; + std::string functionName; + boost::shared_ptr userData; + StoreData() : length(0) { - int32_t length; - std::string functionName; - boost::shared_ptr userData; - StoreData() : length(0) { } - StoreData(const StoreData& rhs) - { - length = rhs.length; - functionName = rhs.functionName; - userData = rhs.userData; - } - }; - -public: - UserDataStore(); - virtual ~UserDataStore(); - - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&); - - //Set to make UserDataStore thread safe - void useUserDataMutex(bool b) - { - fUseUserDataMutex = b; } - bool useUserDataMutex() const + StoreData(const StoreData& rhs) { - return fUseUserDataMutex; + length = rhs.length; + functionName = rhs.functionName; + userData = rhs.userData; } + }; - // Returns the offset - uint32_t storeUserData(mcsv1sdk::mcsv1Context& context, - boost::shared_ptr data, - uint32_t length); + public: + UserDataStore(); + virtual ~UserDataStore(); - boost::shared_ptr getUserData(uint32_t offset) const; + void serialize(messageqcpp::ByteStream&) const; + void deserialize(messageqcpp::ByteStream&); -private: - UserDataStore(const UserDataStore&); - UserDataStore& operator=(const UserDataStore&); + // Set to make UserDataStore thread safe + void useUserDataMutex(bool b) + { + fUseUserDataMutex = b; + } + bool useUserDataMutex() const + { + return fUseUserDataMutex; + } - std::vector vStoreData; + // Returns the offset + uint32_t storeUserData(mcsv1sdk::mcsv1Context& context, boost::shared_ptr data, + uint32_t length); - bool fUseUserDataMutex; - boost::mutex fMutex; + boost::shared_ptr getUserData(uint32_t offset) const; + + private: + UserDataStore(const UserDataStore&); + UserDataStore& operator=(const UserDataStore&); + + std::vector vStoreData; + + bool fUseUserDataMutex; + boost::mutex fMutex; }; #ifdef _MSC_VER -#pragma warning (pop) +#pragma warning(pop) #endif class RowGroup; @@ -265,1444 +262,1353 @@ class Row; /* TODO: OO the rowgroup data to the extent there's no measurable performance hit. */ class RGData { -public: - RGData(); // useless unless followed by an = or a deserialize operation - RGData(const RowGroup& rg, uint32_t rowCount); // allocates memory for rowData - explicit RGData(const RowGroup& rg); - RGData(const RGData&); - virtual ~RGData(); + public: + RGData(); // useless unless followed by an = or a deserialize operation + RGData(const RowGroup& rg, uint32_t rowCount); // allocates memory for rowData + explicit RGData(const RowGroup& rg); + RGData(const RGData&); + virtual ~RGData(); - inline RGData& operator=(const RGData&); + inline RGData& operator=(const RGData&); - // amount should be the # returned by RowGroup::getDataSize() - void serialize(messageqcpp::ByteStream&, uint32_t amount) const; + // amount should be the # returned by RowGroup::getDataSize() + void serialize(messageqcpp::ByteStream&, uint32_t amount) const; - // the 'hasLengthField' is there b/c PM aggregation (and possibly others) currently sends - // inline data with a length field. Once that's converted to string table format, that - // option can go away. - void deserialize(messageqcpp::ByteStream&, uint32_t amount = 0); // returns the # of bytes read + // the 'hasLengthField' is there b/c PM aggregation (and possibly others) currently sends + // inline data with a length field. Once that's converted to string table format, that + // option can go away. + void deserialize(messageqcpp::ByteStream&, uint32_t amount = 0); // returns the # of bytes read - inline uint64_t getStringTableMemUsage(); - void clear(); - void reinit(const RowGroup& rg); - void reinit(const RowGroup& rg, uint32_t rowCount); - inline void setStringStore(boost::shared_ptr& ss) - { - strings = ss; - } + inline uint64_t getStringTableMemUsage(); + void clear(); + void reinit(const RowGroup& rg); + void reinit(const RowGroup& rg, uint32_t rowCount); + inline void setStringStore(boost::shared_ptr& ss) + { + strings = ss; + } - // this will use the pre-configured Row to figure out where row # num is, then set the Row - // to point to it. It's a shortcut around using a RowGroup to do the same thing for cases - // where it's inconvenient to instantiate one. - inline void getRow(uint32_t num, Row* row); + // this will use the pre-configured Row to figure out where row # num is, then set the Row + // to point to it. It's a shortcut around using a RowGroup to do the same thing for cases + // where it's inconvenient to instantiate one. + inline void getRow(uint32_t num, Row* row); - //@bug6065, make StringStore::storeString() thread safe - void useStoreStringMutex(bool b) - { - if (strings) strings->useStoreStringMutex(b); - } - bool useStoreStringMutex() const - { - return (strings ? (strings->useStoreStringMutex()) : false); - } + //@bug6065, make StringStore::storeString() thread safe + void useStoreStringMutex(bool b) + { + if (strings) + strings->useStoreStringMutex(b); + } + bool useStoreStringMutex() const + { + return (strings ? (strings->useStoreStringMutex()) : false); + } - UserDataStore* getUserDataStore(); - // make UserDataStore::storeData() thread safe - void useUserDataMutex(bool b) - { - if (userDataStore) userDataStore->useUserDataMutex(b); - } - bool useUserDataMutex() const - { - return (userDataStore ? (userDataStore->useUserDataMutex()) : false); - } + UserDataStore* getUserDataStore(); + // make UserDataStore::storeData() thread safe + void useUserDataMutex(bool b) + { + if (userDataStore) + userDataStore->useUserDataMutex(b); + } + bool useUserDataMutex() const + { + return (userDataStore ? (userDataStore->useUserDataMutex()) : false); + } - boost::shared_array rowData; - boost::shared_ptr strings; - boost::shared_ptr userDataStore; -private: - //boost::shared_array rowData; - //boost::shared_ptr strings; + boost::shared_array rowData; + boost::shared_ptr strings; + boost::shared_ptr userDataStore; - // Need sig to support backward compat. RGData can deserialize both forms. - static const uint32_t RGDATA_SIG = 0xffffffff; //won't happen for 'old' Rowgroup data + private: + // boost::shared_array rowData; + // boost::shared_ptr strings; - friend class RowGroup; + // Need sig to support backward compat. RGData can deserialize both forms. + static const uint32_t RGDATA_SIG = 0xffffffff; // won't happen for 'old' Rowgroup data + + friend class RowGroup; }; - class Row { -public: - struct Pointer + public: + struct Pointer + { + inline Pointer() : data(NULL), strings(NULL), userDataStore(NULL) { - inline Pointer() : data(NULL), strings(NULL), userDataStore(NULL) { } - - // Pointer(uint8_t*) implicitly makes old code compatible with the string table impl; - inline Pointer(uint8_t* d) : data(d), strings(NULL), userDataStore(NULL) { } - inline Pointer(uint8_t* d, StringStore* s) : data(d), strings(s), userDataStore(NULL) { } - inline Pointer(uint8_t* d, StringStore* s, UserDataStore* u) : - data(d), strings(s), userDataStore(u) { } - uint8_t* data; - StringStore* strings; - UserDataStore* userDataStore; - }; - - Row(); - Row(const Row&); - ~Row(); - - Row& operator=(const Row&); - bool operator==(const Row&) const; - - //void setData(uint8_t *rowData, StringStore *ss); - inline void setData(const Pointer&); // convenience fcn, can go away - inline uint8_t* getData() const; - - inline void setPointer(const Pointer&); - inline Pointer getPointer() const; - - inline void nextRow(); - inline uint32_t getColumnWidth(uint32_t colIndex) const; - inline uint32_t getColumnCount() const; - inline uint32_t getSize() const; // this is only accurate if there is no string table - // if a string table is being used, getRealSize() takes into account variable-length strings - inline uint32_t getRealSize() const; - inline uint32_t getOffset(uint32_t colIndex) const; - inline uint32_t getScale(uint32_t colIndex) const; - inline uint32_t getPrecision(uint32_t colIndex) const; - inline execplan::CalpontSystemCatalog::ColDataType getColType(uint32_t colIndex) const; - inline execplan::CalpontSystemCatalog::ColDataType* getColTypes(); - inline const execplan::CalpontSystemCatalog::ColDataType* getColTypes() const; - inline uint32_t getCharsetNumber(uint32_t colIndex) const; - - // this returns true if the type is not CHAR or VARCHAR - inline bool isCharType(uint32_t colIndex) const; - inline bool isUnsigned(uint32_t colIndex) const; - inline bool isShortString(uint32_t colIndex) const; - inline bool isLongString(uint32_t colIndex) const; - - bool colHasCollation(uint32_t colIndex) const - { - return datatypes::typeHasCollation(getColType(colIndex)); } - template inline uint64_t getUintField(uint32_t colIndex) const; - inline uint64_t getUintField(uint32_t colIndex) const; - template inline int64_t getIntField(uint32_t colIndex) const; - inline int64_t getIntField(uint32_t colIndex) const; - // Get a signed 64-bit integer column value, convert to the given - // floating point data type T (e.g. float, double, long double) - // and divide it according to the scale. - template - inline T getScaledSInt64FieldAsXFloat(uint32_t colIndex, uint32_t scale) const + // Pointer(uint8_t*) implicitly makes old code compatible with the string table impl; + inline Pointer(uint8_t* d) : data(d), strings(NULL), userDataStore(NULL) { - const T d = getIntField(colIndex); - if (!scale) - return d; - return d / datatypes::scaleDivisor(scale); } - template - inline T getScaledSInt64FieldAsXFloat(uint32_t colIndex) const + inline Pointer(uint8_t* d, StringStore* s) : data(d), strings(s), userDataStore(NULL) { - return getScaledSInt64FieldAsXFloat(colIndex, getScale(colIndex)); } - // Get an unsigned 64-bit integer column value, convert to the given - // floating point data type T (e.g. float, double, long double) - // and divide it according to the scale. - template - inline T getScaledUInt64FieldAsXFloat(uint32_t colIndex, uint32_t scale) const + inline Pointer(uint8_t* d, StringStore* s, UserDataStore* u) : data(d), strings(s), userDataStore(u) { - const T d = getUintField(colIndex); - if (!scale) - return d; - return d / datatypes::scaleDivisor(scale); } - template - inline T getScaledUInt64FieldAsXFloat(uint32_t colIndex) const - { - return getScaledUInt64FieldAsXFloat(colIndex, getScale(colIndex)); - } - template - inline bool equals(T* value, uint32_t colIndex) const; - template inline bool equals(uint64_t val, uint32_t colIndex) const; - inline bool equals(long double val, uint32_t colIndex) const; - inline bool equals(const int128_t& val, uint32_t colIndex) const; - - inline double getDoubleField(uint32_t colIndex) const; - inline float getFloatField(uint32_t colIndex) const; - inline datatypes::Decimal getDecimalField(uint32_t colIndex) const - { - if (LIKELY(getColumnWidth(colIndex) == datatypes::MAXDECIMALWIDTH)) - return datatypes::Decimal(0, - (int) getScale(colIndex), - getPrecision(colIndex), - getBinaryField(colIndex)); - return datatypes::Decimal(datatypes::TSInt64(getIntField(colIndex)), - (int) getScale(colIndex), - getPrecision(colIndex)); - } - inline long double getLongDoubleField(uint32_t colIndex) const; - inline void storeInt128FieldIntoPtr(uint32_t colIndex, uint8_t* x) const; - inline void getInt128Field(uint32_t colIndex, int128_t& x) const; - inline datatypes::TSInt128 getTSInt128Field(uint32_t colIndex) const; - - inline uint64_t getBaseRid() const; - inline uint64_t getRid() const; - inline uint16_t getRelRid() const; // returns a rid relative to this logical block - inline uint64_t getExtentRelativeRid() const; // returns a rid relative to the extent it's in - inline uint64_t getFileRelativeRid() const; // returns a file-relative rid - inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, - uint16_t* blockNum, uint16_t* rowNum); - - template void setUintField(uint64_t val, uint32_t colIndex); - - /* Note: these 2 fcns avoid 1 array lookup per call. Using them only - in projection on the PM resulted in a 2.8% performance gain on - the queries listed in bug 2223. - TODO: apply them everywhere else possible, and write equivalents - for the other types as well as the getters. - */ - template void setUintField_offset(uint64_t val, uint32_t offset); - templatevoid setIntField_offset(const T val, const uint32_t offset); - inline void nextRow(uint32_t size); - inline void prevRow(uint32_t size, uint64_t number); - - inline void setUintField(uint64_t val, uint32_t colIndex); - template void setIntField(int64_t, uint32_t colIndex); - inline void setIntField(int64_t, uint32_t colIndex); - - inline void setDoubleField(double val, uint32_t colIndex); - inline void setFloatField(float val, uint32_t colIndex); - inline void setDecimalField(double val, uint32_t colIndex) { }; // TODO: Do something here - inline void setLongDoubleField(const long double& val, uint32_t colIndex); - inline void setInt128Field(const int128_t& val, uint32_t colIndex); - - inline void setRid(uint64_t rid); - - // TODO: remove this (string is not efficient for this), use getConstString() instead - inline std::string getStringField(uint32_t colIndex) const - { - return getConstString(colIndex).toString(); - } - - inline utils::ConstString getConstString(uint32_t colIndex) const; - inline utils::ConstString getShortConstString(uint32_t colIndex) const; - void setStringField(const std::string& val, uint32_t colIndex); - inline void setStringField(const utils::ConstString &str, uint32_t colIndex); - template - inline void setBinaryField(const T* value, uint32_t width, uint32_t colIndex); - template - inline void setBinaryField(const T* value, uint32_t colIndex); - template - inline void setBinaryField_offset(const T* value, uint32_t width, uint32_t colIndex); - // support VARBINARY - // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are - // treated the same, could use one of the length bit to distinguish these two cases. - inline std::string getVarBinaryStringField(uint32_t colIndex) const; - inline void setVarBinaryField(const std::string& val, uint32_t colIndex); - // No string construction is necessary for better performance. - inline uint32_t getVarBinaryLength(uint32_t colIndex) const; - inline const uint8_t* getVarBinaryField(uint32_t colIndex) const; - inline const uint8_t* getVarBinaryField(uint32_t& len, uint32_t colIndex) const; - inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); - - //inline std::string getBinaryField(uint32_t colIndex) const; - template - inline T* getBinaryField(uint32_t colIndex) const; - // To simplify parameter type deduction. - template - inline T* getBinaryField(T* argtype, uint32_t colIndex) const; - template - inline T* getBinaryField_offset(uint32_t offset) const; - - inline boost::shared_ptr getUserData(uint32_t colIndex) const; - inline void setUserData(mcsv1sdk::mcsv1Context& context, - boost::shared_ptr userData, - uint32_t len, uint32_t colIndex); - - uint64_t getNullValue(uint32_t colIndex) const; - bool isNullValue(uint32_t colIndex) const; - template - inline bool isNullValue_offset(uint32_t offset) const; - - // when NULLs are pulled out via getIntField(), they come out with these values. - // Ex: the 1-byte int null value is 0x80. When it gets cast to an int64_t - // it becomes 0xffffffffffffff80, which won't match anything returned by getNullValue(). - int64_t getSignedNullValue(uint32_t colIndex) const; - - // copy data in srcIndex field to destIndex, all data type - inline void copyField(uint32_t destIndex, uint32_t srcIndex) const; - - // copy data in srcIndex field to destAddr, all data type - //inline void copyField(uint8_t* destAddr, uint32_t srcIndex) const; - - // an adapter for code that uses the copyField call above; - // that's not string-table safe, this one is - inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; - - template - inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; - - std::string toString(uint32_t rownum = 0) const; - std::string toCSV() const; - - /* These fcns are used only in joins. The RID doesn't matter on the side that - gets hashed. We steal that field here to "mark" a row. */ - inline void markRow(); - inline void zeroRid(); - inline bool isMarked(); - void initToNull(); - - inline void usesStringTable(bool b) - { - useStringTable = b; - } - inline bool usesStringTable() const - { - return useStringTable; - } - inline bool hasLongString() const - { - return hasLongStringField; - } - - // these are for cases when you already know the type definitions are the same. - // a fcn to check the type defs seperately doesn't exist yet. No normalization. - inline uint64_t hash(uint32_t lastCol) const; // generates a hash for cols [0-lastCol] - inline uint64_t hash() const; // generates a hash for all cols - inline void colUpdateHasher(datatypes::MariaDBHasher& hM, - const utils::Hasher_r& h, - const uint32_t col, - uint32_t& intermediateHash) const; - inline void colUpdateHasherTypeless(datatypes::MariaDBHasher &hasher, uint32_t keyColsIdx, - const std::vector& keyCols, - const std::vector* smallSideKeyColumnsIds, - const std::vector* smallSideColumnsWidths) const; - inline uint64_t hashTypeless(const std::vector& keyCols, - const std::vector* smallSideKeyColumnsIds, - const std::vector* smallSideColumnsWidths) const - { - datatypes::MariaDBHasher h; - for (uint32_t i = 0; i < keyCols.size(); i++) - colUpdateHasherTypeless(h, i, keyCols, smallSideKeyColumnsIds, smallSideColumnsWidths); - return h.finalize(); - } - - bool equals(const Row&, uint32_t lastCol) const; - inline bool equals(const Row&) const; - - inline void setUserDataStore(UserDataStore* u) - { - userDataStore = u; - } - - const CHARSET_INFO* getCharset(uint32_t col) const; - -private: - uint32_t columnCount; - uint64_t baseRid; - - // Note, the mem behind these pointer fields is owned by RowGroup not Row - uint32_t* oldOffsets; - uint32_t* stOffsets; - uint32_t* offsets; - uint32_t* colWidths; - execplan::CalpontSystemCatalog::ColDataType* types; - uint32_t* charsetNumbers; - CHARSET_INFO** charsets; uint8_t* data; - uint32_t* scale; - uint32_t* precision; - StringStore* strings; - bool useStringTable; - bool hasCollation; - bool hasLongStringField; - uint32_t sTableThreshold; - boost::shared_array forceInline; - inline bool inStringTable(uint32_t col) const; + UserDataStore* userDataStore; + }; - UserDataStore* userDataStore; // For UDAF + Row(); + Row(const Row&); + ~Row(); - friend class RowGroup; + Row& operator=(const Row&); + bool operator==(const Row&) const; + + // void setData(uint8_t *rowData, StringStore *ss); + inline void setData(const Pointer&); // convenience fcn, can go away + inline uint8_t* getData() const; + + inline void setPointer(const Pointer&); + inline Pointer getPointer() const; + + inline void nextRow(); + inline uint32_t getColumnWidth(uint32_t colIndex) const; + inline uint32_t getColumnCount() const; + inline uint32_t getSize() const; // this is only accurate if there is no string table + // if a string table is being used, getRealSize() takes into account variable-length strings + inline uint32_t getRealSize() const; + inline uint32_t getOffset(uint32_t colIndex) const; + inline uint32_t getScale(uint32_t colIndex) const; + inline uint32_t getPrecision(uint32_t colIndex) const; + inline execplan::CalpontSystemCatalog::ColDataType getColType(uint32_t colIndex) const; + inline execplan::CalpontSystemCatalog::ColDataType* getColTypes(); + inline const execplan::CalpontSystemCatalog::ColDataType* getColTypes() const; + inline uint32_t getCharsetNumber(uint32_t colIndex) const; + + // this returns true if the type is not CHAR or VARCHAR + inline bool isCharType(uint32_t colIndex) const; + inline bool isUnsigned(uint32_t colIndex) const; + inline bool isShortString(uint32_t colIndex) const; + inline bool isLongString(uint32_t colIndex) const; + + bool colHasCollation(uint32_t colIndex) const + { + return datatypes::typeHasCollation(getColType(colIndex)); + } + + template + inline uint64_t getUintField(uint32_t colIndex) const; + inline uint64_t getUintField(uint32_t colIndex) const; + template + inline int64_t getIntField(uint32_t colIndex) const; + inline int64_t getIntField(uint32_t colIndex) const; + // Get a signed 64-bit integer column value, convert to the given + // floating point data type T (e.g. float, double, long double) + // and divide it according to the scale. + template + inline T getScaledSInt64FieldAsXFloat(uint32_t colIndex, uint32_t scale) const + { + const T d = getIntField(colIndex); + if (!scale) + return d; + return d / datatypes::scaleDivisor(scale); + } + template + inline T getScaledSInt64FieldAsXFloat(uint32_t colIndex) const + { + return getScaledSInt64FieldAsXFloat(colIndex, getScale(colIndex)); + } + // Get an unsigned 64-bit integer column value, convert to the given + // floating point data type T (e.g. float, double, long double) + // and divide it according to the scale. + template + inline T getScaledUInt64FieldAsXFloat(uint32_t colIndex, uint32_t scale) const + { + const T d = getUintField(colIndex); + if (!scale) + return d; + return d / datatypes::scaleDivisor(scale); + } + template + inline T getScaledUInt64FieldAsXFloat(uint32_t colIndex) const + { + return getScaledUInt64FieldAsXFloat(colIndex, getScale(colIndex)); + } + template + inline bool equals(T* value, uint32_t colIndex) const; + template + inline bool equals(uint64_t val, uint32_t colIndex) const; + inline bool equals(long double val, uint32_t colIndex) const; + inline bool equals(const int128_t& val, uint32_t colIndex) const; + + inline double getDoubleField(uint32_t colIndex) const; + inline float getFloatField(uint32_t colIndex) const; + inline datatypes::Decimal getDecimalField(uint32_t colIndex) const + { + if (LIKELY(getColumnWidth(colIndex) == datatypes::MAXDECIMALWIDTH)) + return datatypes::Decimal(0, (int)getScale(colIndex), getPrecision(colIndex), + getBinaryField(colIndex)); + return datatypes::Decimal(datatypes::TSInt64(getIntField(colIndex)), (int)getScale(colIndex), + getPrecision(colIndex)); + } + inline long double getLongDoubleField(uint32_t colIndex) const; + inline void storeInt128FieldIntoPtr(uint32_t colIndex, uint8_t* x) const; + inline void getInt128Field(uint32_t colIndex, int128_t& x) const; + inline datatypes::TSInt128 getTSInt128Field(uint32_t colIndex) const; + + inline uint64_t getBaseRid() const; + inline uint64_t getRid() const; + inline uint16_t getRelRid() const; // returns a rid relative to this logical block + inline uint64_t getExtentRelativeRid() const; // returns a rid relative to the extent it's in + inline uint64_t getFileRelativeRid() const; // returns a file-relative rid + inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum, + uint16_t* rowNum); + + template + void setUintField(uint64_t val, uint32_t colIndex); + + /* Note: these 2 fcns avoid 1 array lookup per call. Using them only + in projection on the PM resulted in a 2.8% performance gain on + the queries listed in bug 2223. + TODO: apply them everywhere else possible, and write equivalents + for the other types as well as the getters. + */ + template + void setUintField_offset(uint64_t val, uint32_t offset); + template + void setIntField_offset(const T val, const uint32_t offset); + inline void nextRow(uint32_t size); + inline void prevRow(uint32_t size, uint64_t number); + + inline void setUintField(uint64_t val, uint32_t colIndex); + template + void setIntField(int64_t, uint32_t colIndex); + inline void setIntField(int64_t, uint32_t colIndex); + + inline void setDoubleField(double val, uint32_t colIndex); + inline void setFloatField(float val, uint32_t colIndex); + inline void setDecimalField(double val, uint32_t colIndex){}; // TODO: Do something here + inline void setLongDoubleField(const long double& val, uint32_t colIndex); + inline void setInt128Field(const int128_t& val, uint32_t colIndex); + + inline void setRid(uint64_t rid); + + // TODO: remove this (string is not efficient for this), use getConstString() instead + inline std::string getStringField(uint32_t colIndex) const + { + return getConstString(colIndex).toString(); + } + + inline utils::ConstString getConstString(uint32_t colIndex) const; + inline utils::ConstString getShortConstString(uint32_t colIndex) const; + void setStringField(const std::string& val, uint32_t colIndex); + inline void setStringField(const utils::ConstString& str, uint32_t colIndex); + template + inline void setBinaryField(const T* value, uint32_t width, uint32_t colIndex); + template + inline void setBinaryField(const T* value, uint32_t colIndex); + template + inline void setBinaryField_offset(const T* value, uint32_t width, uint32_t colIndex); + // support VARBINARY + // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are + // treated the same, could use one of the length bit to distinguish these two cases. + inline std::string getVarBinaryStringField(uint32_t colIndex) const; + inline void setVarBinaryField(const std::string& val, uint32_t colIndex); + // No string construction is necessary for better performance. + inline uint32_t getVarBinaryLength(uint32_t colIndex) const; + inline const uint8_t* getVarBinaryField(uint32_t colIndex) const; + inline const uint8_t* getVarBinaryField(uint32_t& len, uint32_t colIndex) const; + inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); + + // inline std::string getBinaryField(uint32_t colIndex) const; + template + inline T* getBinaryField(uint32_t colIndex) const; + // To simplify parameter type deduction. + template + inline T* getBinaryField(T* argtype, uint32_t colIndex) const; + template + inline T* getBinaryField_offset(uint32_t offset) const; + + inline boost::shared_ptr getUserData(uint32_t colIndex) const; + inline void setUserData(mcsv1sdk::mcsv1Context& context, boost::shared_ptr userData, + uint32_t len, uint32_t colIndex); + + uint64_t getNullValue(uint32_t colIndex) const; + bool isNullValue(uint32_t colIndex) const; + template + inline bool isNullValue_offset(uint32_t offset) const; + + // when NULLs are pulled out via getIntField(), they come out with these values. + // Ex: the 1-byte int null value is 0x80. When it gets cast to an int64_t + // it becomes 0xffffffffffffff80, which won't match anything returned by getNullValue(). + int64_t getSignedNullValue(uint32_t colIndex) const; + + // copy data in srcIndex field to destIndex, all data type + inline void copyField(uint32_t destIndex, uint32_t srcIndex) const; + + // copy data in srcIndex field to destAddr, all data type + // inline void copyField(uint8_t* destAddr, uint32_t srcIndex) const; + + // an adapter for code that uses the copyField call above; + // that's not string-table safe, this one is + inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; + + template + inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; + + std::string toString(uint32_t rownum = 0) const; + std::string toCSV() const; + + /* These fcns are used only in joins. The RID doesn't matter on the side that + gets hashed. We steal that field here to "mark" a row. */ + inline void markRow(); + inline void zeroRid(); + inline bool isMarked(); + void initToNull(); + + inline void usesStringTable(bool b) + { + useStringTable = b; + } + inline bool usesStringTable() const + { + return useStringTable; + } + inline bool hasLongString() const + { + return hasLongStringField; + } + + // these are for cases when you already know the type definitions are the same. + // a fcn to check the type defs seperately doesn't exist yet. No normalization. + inline uint64_t hash(uint32_t lastCol) const; // generates a hash for cols [0-lastCol] + inline uint64_t hash() const; // generates a hash for all cols + inline void colUpdateHasher(datatypes::MariaDBHasher& hM, const utils::Hasher_r& h, const uint32_t col, + uint32_t& intermediateHash) const; + inline void colUpdateHasherTypeless(datatypes::MariaDBHasher& hasher, uint32_t keyColsIdx, + const std::vector& keyCols, + const std::vector* smallSideKeyColumnsIds, + const std::vector* smallSideColumnsWidths) const; + inline uint64_t hashTypeless(const std::vector& keyCols, + const std::vector* smallSideKeyColumnsIds, + const std::vector* smallSideColumnsWidths) const + { + datatypes::MariaDBHasher h; + for (uint32_t i = 0; i < keyCols.size(); i++) + colUpdateHasherTypeless(h, i, keyCols, smallSideKeyColumnsIds, smallSideColumnsWidths); + return h.finalize(); + } + + bool equals(const Row&, uint32_t lastCol) const; + inline bool equals(const Row&) const; + + inline void setUserDataStore(UserDataStore* u) + { + userDataStore = u; + } + + const CHARSET_INFO* getCharset(uint32_t col) const; + + private: + uint32_t columnCount; + uint64_t baseRid; + + // Note, the mem behind these pointer fields is owned by RowGroup not Row + uint32_t* oldOffsets; + uint32_t* stOffsets; + uint32_t* offsets; + uint32_t* colWidths; + execplan::CalpontSystemCatalog::ColDataType* types; + uint32_t* charsetNumbers; + CHARSET_INFO** charsets; + uint8_t* data; + uint32_t* scale; + uint32_t* precision; + + StringStore* strings; + bool useStringTable; + bool hasCollation; + bool hasLongStringField; + uint32_t sTableThreshold; + boost::shared_array forceInline; + inline bool inStringTable(uint32_t col) const; + + UserDataStore* userDataStore; // For UDAF + + friend class RowGroup; }; inline Row::Pointer Row::getPointer() const { - return Pointer(data, strings, userDataStore); + return Pointer(data, strings, userDataStore); } inline uint8_t* Row::getData() const { - return data; + return data; } inline void Row::setPointer(const Pointer& p) { - data = p.data; - strings = p.strings; - bool hasStrings = (strings != 0); + data = p.data; + strings = p.strings; + bool hasStrings = (strings != 0); - if (useStringTable != hasStrings) - { - useStringTable = hasStrings; - offsets = (useStringTable ? stOffsets : oldOffsets); - } + if (useStringTable != hasStrings) + { + useStringTable = hasStrings; + offsets = (useStringTable ? stOffsets : oldOffsets); + } - userDataStore = p.userDataStore; + userDataStore = p.userDataStore; } inline void Row::setData(const Pointer& p) { - setPointer(p); + setPointer(p); } inline void Row::nextRow() { - data += offsets[columnCount]; + data += offsets[columnCount]; } inline uint32_t Row::getColumnCount() const { - return columnCount; + return columnCount; } inline uint32_t Row::getColumnWidth(uint32_t col) const { - return colWidths[col]; + return colWidths[col]; } inline uint32_t Row::getSize() const { - return offsets[columnCount]; + return offsets[columnCount]; } inline uint32_t Row::getRealSize() const { - if (!useStringTable) - return getSize(); + if (!useStringTable) + return getSize(); - uint32_t ret = 2; + uint32_t ret = 2; - for (uint32_t i = 0; i < columnCount; i++) - { - if (!inStringTable(i)) - ret += getColumnWidth(i); - else - ret += getConstString(i).length(); - } + for (uint32_t i = 0; i < columnCount; i++) + { + if (!inStringTable(i)) + ret += getColumnWidth(i); + else + ret += getConstString(i).length(); + } - return ret; + return ret; } inline uint32_t Row::getScale(uint32_t col) const { - return scale[col]; + return scale[col]; } inline uint32_t Row::getPrecision(uint32_t col) const { - return precision[col]; + return precision[col]; } inline execplan::CalpontSystemCatalog::ColDataType Row::getColType(uint32_t colIndex) const { - return types[colIndex]; + return types[colIndex]; } inline execplan::CalpontSystemCatalog::ColDataType* Row::getColTypes() { - return types; + return types; } inline const execplan::CalpontSystemCatalog::ColDataType* Row::getColTypes() const { - return types; + return types; } inline uint32_t Row::getCharsetNumber(uint32_t col) const { - return charsetNumbers[col]; + return charsetNumbers[col]; } inline bool Row::isCharType(uint32_t colIndex) const { - return datatypes::isCharType(types[colIndex]); + return datatypes::isCharType(types[colIndex]); } inline bool Row::isUnsigned(uint32_t colIndex) const { - return datatypes::isUnsigned(types[colIndex]); + return datatypes::isUnsigned(types[colIndex]); } inline bool Row::isShortString(uint32_t colIndex) const { - return (getColumnWidth(colIndex) <= 8 && isCharType(colIndex)); + return (getColumnWidth(colIndex) <= 8 && isCharType(colIndex)); } inline bool Row::isLongString(uint32_t colIndex) const { - return (getColumnWidth(colIndex) > 8 && isCharType(colIndex)); + return (getColumnWidth(colIndex) > 8 && isCharType(colIndex)); } inline bool Row::inStringTable(uint32_t col) const { - return strings && getColumnWidth(col) >= sTableThreshold && !forceInline[col]; + return strings && getColumnWidth(col) >= sTableThreshold && !forceInline[col]; } -template +template inline bool Row::equals(T* value, uint32_t colIndex) const { - return *reinterpret_cast(&data[offsets[colIndex]]) == *value; + return *reinterpret_cast(&data[offsets[colIndex]]) == *value; } -template +template inline bool Row::equals(uint64_t val, uint32_t colIndex) const { - /* I think the compiler will optimize away the switch stmt */ - switch (len) - { - case 1: - return data[offsets[colIndex]] == val; + /* I think the compiler will optimize away the switch stmt */ + switch (len) + { + case 1: return data[offsets[colIndex]] == val; - case 2: - return *((uint16_t*) &data[offsets[colIndex]]) == val; + case 2: return *((uint16_t*)&data[offsets[colIndex]]) == val; - case 4: - return *((uint32_t*) &data[offsets[colIndex]]) == val; + case 4: return *((uint32_t*)&data[offsets[colIndex]]) == val; - case 8: - return *((uint64_t*) &data[offsets[colIndex]]) == val; - default: - idbassert(0); - throw std::logic_error("Row::equals(): bad length."); - } + case 8: return *((uint64_t*)&data[offsets[colIndex]]) == val; + default: idbassert(0); throw std::logic_error("Row::equals(): bad length."); + } } inline bool Row::equals(long double val, uint32_t colIndex) const { - return *((long double*) &data[offsets[colIndex]]) == val; + return *((long double*)&data[offsets[colIndex]]) == val; } inline bool Row::equals(const int128_t& val, uint32_t colIndex) const { - return *((int128_t*) &data[offsets[colIndex]]) == val; + return *((int128_t*)&data[offsets[colIndex]]) == val; } -template +template inline uint64_t Row::getUintField(uint32_t colIndex) const { - /* I think the compiler will optimize away the switch stmt */ - switch (len) - { - case 1: - return data[offsets[colIndex]]; + /* I think the compiler will optimize away the switch stmt */ + switch (len) + { + case 1: return data[offsets[colIndex]]; - case 2: - return *((uint16_t*) &data[offsets[colIndex]]); + case 2: return *((uint16_t*)&data[offsets[colIndex]]); - case 4: - return *((uint32_t*) &data[offsets[colIndex]]); + case 4: return *((uint32_t*)&data[offsets[colIndex]]); - case 8: - return *((uint64_t*) &data[offsets[colIndex]]); - default: - idbassert(0); - throw std::logic_error("Row::getUintField(): bad length."); - } + case 8: return *((uint64_t*)&data[offsets[colIndex]]); + default: idbassert(0); throw std::logic_error("Row::getUintField(): bad length."); + } } inline uint64_t Row::getUintField(uint32_t colIndex) const { - switch (getColumnWidth(colIndex)) - { - case 1: - return data[offsets[colIndex]]; + switch (getColumnWidth(colIndex)) + { + case 1: return data[offsets[colIndex]]; - case 2: - return *((uint16_t*) &data[offsets[colIndex]]); + case 2: return *((uint16_t*)&data[offsets[colIndex]]); - case 4: - return *((uint32_t*) &data[offsets[colIndex]]); - case 8: - return *((uint64_t*) &data[offsets[colIndex]]); + case 4: return *((uint32_t*)&data[offsets[colIndex]]); + case 8: return *((uint64_t*)&data[offsets[colIndex]]); - default: - idbassert(0); - throw std::logic_error("Row::getUintField(): bad length."); - } + default: idbassert(0); throw std::logic_error("Row::getUintField(): bad length."); + } } -template +template inline int64_t Row::getIntField(uint32_t colIndex) const { - /* I think the compiler will optimize away the switch stmt */ - switch (len) - { - case 1: - return (int8_t) data[offsets[colIndex]]; + /* I think the compiler will optimize away the switch stmt */ + switch (len) + { + case 1: return (int8_t)data[offsets[colIndex]]; - case 2: - return *((int16_t*) &data[offsets[colIndex]]); + case 2: return *((int16_t*)&data[offsets[colIndex]]); - case 4: - return *((int32_t*) &data[offsets[colIndex]]); + case 4: return *((int32_t*)&data[offsets[colIndex]]); - case 8: - return *((int64_t*) &data[offsets[colIndex]]); + case 8: return *((int64_t*)&data[offsets[colIndex]]); - default: - std::cout << "Row::getIntField getColumnWidth(colIndex) " << getColumnWidth(colIndex) << std::endl; - idbassert(0); - throw std::logic_error("Row::getIntField(): bad length."); - } + default: + std::cout << "Row::getIntField getColumnWidth(colIndex) " << getColumnWidth(colIndex) << std::endl; + idbassert(0); + throw std::logic_error("Row::getIntField(): bad length."); + } } inline int64_t Row::getIntField(uint32_t colIndex) const { - /* I think the compiler will optimize away the switch stmt */ - switch (getColumnWidth(colIndex)) - { - case 1: - return (int8_t) data[offsets[colIndex]]; + /* I think the compiler will optimize away the switch stmt */ + switch (getColumnWidth(colIndex)) + { + case 1: return (int8_t)data[offsets[colIndex]]; - case 2: - return *((int16_t*) &data[offsets[colIndex]]); + case 2: return *((int16_t*)&data[offsets[colIndex]]); - case 4: - return *((int32_t*) &data[offsets[colIndex]]); + case 4: return *((int32_t*)&data[offsets[colIndex]]); - case 8: - return *((int64_t*) &data[offsets[colIndex]]); + case 8: return *((int64_t*)&data[offsets[colIndex]]); - default: - idbassert(0); - throw std::logic_error("Row::getIntField(): bad length."); - } + default: idbassert(0); throw std::logic_error("Row::getIntField(): bad length."); + } } - -template +template inline void Row::setBinaryField(const T* value, uint32_t width, uint32_t colIndex) { - memcpy(&data[offsets[colIndex]], value, width); + memcpy(&data[offsets[colIndex]], value, width); } -template +template inline void Row::setBinaryField(const T* value, uint32_t colIndex) { - *reinterpret_cast(&data[offsets[colIndex]]) = *value; + *reinterpret_cast(&data[offsets[colIndex]]) = *value; } -template<> +template <> inline void Row::setBinaryField(const int128_t* value, uint32_t colIndex) { - datatypes::TSInt128::assignPtrPtr(&data[offsets[colIndex]], value); + datatypes::TSInt128::assignPtrPtr(&data[offsets[colIndex]], value); } - // This method !cannot! be applied to uint8_t* buffers. -template +template inline void Row::setBinaryField_offset(const T* value, uint32_t width, uint32_t offset) { - *reinterpret_cast(&data[offset]) = *value; + *reinterpret_cast(&data[offset]) = *value; } -template<> +template <> inline void Row::setBinaryField_offset(const uint8_t* value, uint32_t width, uint32_t offset) { - memcpy(&data[offset], value, width); + memcpy(&data[offset], value, width); } -template<> +template <> inline void Row::setBinaryField_offset(const int128_t* value, uint32_t width, uint32_t offset) { - datatypes::TSInt128::assignPtrPtr(&data[offset], value); + datatypes::TSInt128::assignPtrPtr(&data[offset], value); } - inline utils::ConstString Row::getShortConstString(uint32_t colIndex) const { - const char *src= (const char *) &data[offsets[colIndex]]; - return utils::ConstString(src, strnlen(src, getColumnWidth(colIndex))); + const char* src = (const char*)&data[offsets[colIndex]]; + return utils::ConstString(src, strnlen(src, getColumnWidth(colIndex))); } - inline utils::ConstString Row::getConstString(uint32_t colIndex) const { - return inStringTable(colIndex) ? - strings->getConstString(*((uint64_t*) &data[offsets[colIndex]])) : - getShortConstString(colIndex); + return inStringTable(colIndex) ? strings->getConstString(*((uint64_t*)&data[offsets[colIndex]])) + : getShortConstString(colIndex); } - -inline void Row::colUpdateHasher(datatypes::MariaDBHasher& hM, - const utils::Hasher_r& h, - const uint32_t col, +inline void Row::colUpdateHasher(datatypes::MariaDBHasher& hM, const utils::Hasher_r& h, const uint32_t col, uint32_t& intermediateHash) const { - switch (getColType(col)) + switch (getColType(col)) + { + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::BLOB: + case execplan::CalpontSystemCatalog::TEXT: { - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::BLOB: - case execplan::CalpontSystemCatalog::TEXT: - { - CHARSET_INFO *cs = getCharset(col); - hM.add(cs, getConstString(col)); - break; - } - default: - { - intermediateHash = h((const char*) &data[offsets[col]], colWidths[col], intermediateHash); - break; - } + CHARSET_INFO* cs = getCharset(col); + hM.add(cs, getConstString(col)); + break; } + default: + { + intermediateHash = h((const char*)&data[offsets[col]], colWidths[col], intermediateHash); + break; + } + } } - -inline void Row::colUpdateHasherTypeless(datatypes::MariaDBHasher &h, uint32_t keyColsIdx, - const std::vector& keyCols, - const std::vector* smallSideKeyColumnsIds, - const std::vector* smallSideColumnsWidths) const +inline void Row::colUpdateHasherTypeless(datatypes::MariaDBHasher& h, uint32_t keyColsIdx, + const std::vector& keyCols, + const std::vector* smallSideKeyColumnsIds, + const std::vector* smallSideColumnsWidths) const { - auto rowKeyColIdx = keyCols[keyColsIdx]; - auto largeSideColType = getColType(rowKeyColIdx); - switch (largeSideColType) + auto rowKeyColIdx = keyCols[keyColsIdx]; + auto largeSideColType = getColType(rowKeyColIdx); + switch (largeSideColType) + { + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + case datatypes::SystemCatalog::BLOB: + case datatypes::SystemCatalog::TEXT: { - case datatypes::SystemCatalog::CHAR: - case datatypes::SystemCatalog::VARCHAR: - case datatypes::SystemCatalog::BLOB: - case datatypes::SystemCatalog::TEXT: - { - CHARSET_INFO *cs = getCharset(rowKeyColIdx); - h.add(cs, getConstString(rowKeyColIdx)); - break; - } - case datatypes::SystemCatalog::DECIMAL: - { - auto width = getColumnWidth(rowKeyColIdx); - if (datatypes::isWideDecimalType(largeSideColType, - width)) - { - bool joinHasSkewedKeyColumn = (smallSideColumnsWidths); - datatypes::TSInt128 val = getTSInt128Field(rowKeyColIdx); - if (joinHasSkewedKeyColumn && - width != derefFromTwoVectorPtrs(smallSideColumnsWidths, smallSideKeyColumnsIds, keyColsIdx)) - { - if (val.getValue() >= std::numeric_limits::min() && - val.getValue() <= std::numeric_limits::max()) - { - h.add(&my_charset_bin, (const char*)&val.getValue(), datatypes::MAXLEGACYWIDTH); - } - else - h.add(&my_charset_bin, (const char*)&val.getValue(), datatypes::MAXDECIMALWIDTH); - } - else - h.add(&my_charset_bin, (const char*)&val.getValue(), datatypes::MAXDECIMALWIDTH); - } - else - { - int64_t val = getIntField(rowKeyColIdx); - h.add(&my_charset_bin, (const char*) &val, datatypes::MAXLEGACYWIDTH); - } - - break; - } - default: - { - if (isUnsigned(rowKeyColIdx)) - { - uint64_t val = getUintField(rowKeyColIdx); - h.add(&my_charset_bin, (const char*) &val, datatypes::MAXLEGACYWIDTH); - } - else - { - int64_t val = getIntField(rowKeyColIdx); - h.add(&my_charset_bin, (const char*) &val, datatypes::MAXLEGACYWIDTH); - } - - break; - } + CHARSET_INFO* cs = getCharset(rowKeyColIdx); + h.add(cs, getConstString(rowKeyColIdx)); + break; } + case datatypes::SystemCatalog::DECIMAL: + { + auto width = getColumnWidth(rowKeyColIdx); + if (datatypes::isWideDecimalType(largeSideColType, width)) + { + bool joinHasSkewedKeyColumn = (smallSideColumnsWidths); + datatypes::TSInt128 val = getTSInt128Field(rowKeyColIdx); + if (joinHasSkewedKeyColumn && + width != derefFromTwoVectorPtrs(smallSideColumnsWidths, smallSideKeyColumnsIds, keyColsIdx)) + { + if (val.getValue() >= std::numeric_limits::min() && + val.getValue() <= std::numeric_limits::max()) + { + h.add(&my_charset_bin, (const char*)&val.getValue(), datatypes::MAXLEGACYWIDTH); + } + else + h.add(&my_charset_bin, (const char*)&val.getValue(), datatypes::MAXDECIMALWIDTH); + } + else + h.add(&my_charset_bin, (const char*)&val.getValue(), datatypes::MAXDECIMALWIDTH); + } + else + { + int64_t val = getIntField(rowKeyColIdx); + h.add(&my_charset_bin, (const char*)&val, datatypes::MAXLEGACYWIDTH); + } + + break; + } + default: + { + if (isUnsigned(rowKeyColIdx)) + { + uint64_t val = getUintField(rowKeyColIdx); + h.add(&my_charset_bin, (const char*)&val, datatypes::MAXLEGACYWIDTH); + } + else + { + int64_t val = getIntField(rowKeyColIdx); + h.add(&my_charset_bin, (const char*)&val, datatypes::MAXLEGACYWIDTH); + } + + break; + } + } } -inline void Row::setStringField(const utils::ConstString &str, uint32_t colIndex) +inline void Row::setStringField(const utils::ConstString& str, uint32_t colIndex) { - uint64_t offset; + uint64_t offset; - // TODO: add multi-byte safe truncation here - uint32_t length = str.length(); - if (length > getColumnWidth(colIndex)) - length = getColumnWidth(colIndex); + // TODO: add multi-byte safe truncation here + uint32_t length = str.length(); + if (length > getColumnWidth(colIndex)) + length = getColumnWidth(colIndex); - if (inStringTable(colIndex)) - { - offset = strings->storeString((const uint8_t*) str.str(), length); - *((uint64_t*) &data[offsets[colIndex]]) = offset; -// cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) -// << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) -// << endl; - } - else - { - memcpy(&data[offsets[colIndex]], str.str(), length); - memset(&data[offsets[colIndex] + length], 0, - offsets[colIndex + 1] - (offsets[colIndex] + length)); - } + if (inStringTable(colIndex)) + { + offset = strings->storeString((const uint8_t*)str.str(), length); + *((uint64_t*)&data[offsets[colIndex]]) = offset; + // cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) + // << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) + // << endl; + } + else + { + memcpy(&data[offsets[colIndex]], str.str(), length); + memset(&data[offsets[colIndex] + length], 0, offsets[colIndex + 1] - (offsets[colIndex] + length)); + } } template inline T* Row::getBinaryField(uint32_t colIndex) const { - return getBinaryField_offset(offsets[colIndex]); + return getBinaryField_offset(offsets[colIndex]); } template inline T* Row::getBinaryField(T* argtype, uint32_t colIndex) const { - return getBinaryField_offset(offsets[colIndex]); + return getBinaryField_offset(offsets[colIndex]); } template inline T* Row::getBinaryField_offset(uint32_t offset) const { - return reinterpret_cast(&data[offset]); + return reinterpret_cast(&data[offset]); } inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const { - if (inStringTable(colIndex)) - return getConstString(colIndex).toString(); + if (inStringTable(colIndex)) + return getConstString(colIndex).toString(); - return std::string((char*) &data[offsets[colIndex] + 2], *((uint16_t*) &data[offsets[colIndex]])); + return std::string((char*)&data[offsets[colIndex] + 2], *((uint16_t*)&data[offsets[colIndex]])); } inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const { - if (inStringTable(colIndex)) - return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));; + if (inStringTable(colIndex)) + return strings->getStringLength(*((uint64_t*)&data[offsets[colIndex]])); + ; - return *((uint16_t*) &data[offsets[colIndex]]); + return *((uint16_t*)&data[offsets[colIndex]]); } inline const uint8_t* Row::getVarBinaryField(uint32_t colIndex) const { - if (inStringTable(colIndex)) - return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]])); + if (inStringTable(colIndex)) + return strings->getPointer(*((uint64_t*)&data[offsets[colIndex]])); - return &data[offsets[colIndex] + 2]; + return &data[offsets[colIndex] + 2]; } inline const uint8_t* Row::getVarBinaryField(uint32_t& len, uint32_t colIndex) const { - if (inStringTable(colIndex)) - { - len = strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]])); - return getVarBinaryField(colIndex); - } - else - { - len = *((uint16_t*) &data[offsets[colIndex]]); - return &data[offsets[colIndex] + 2]; - } + if (inStringTable(colIndex)) + { + len = strings->getStringLength(*((uint64_t*)&data[offsets[colIndex]])); + return getVarBinaryField(colIndex); + } + else + { + len = *((uint16_t*)&data[offsets[colIndex]]); + return &data[offsets[colIndex] + 2]; + } } inline boost::shared_ptr Row::getUserData(uint32_t colIndex) const { - if (!userDataStore) - { - return boost::shared_ptr(); - } + if (!userDataStore) + { + return boost::shared_ptr(); + } - return userDataStore->getUserData(*((uint32_t*) &data[offsets[colIndex]])); + return userDataStore->getUserData(*((uint32_t*)&data[offsets[colIndex]])); } inline double Row::getDoubleField(uint32_t colIndex) const { - return *((double*) &data[offsets[colIndex]]); + return *((double*)&data[offsets[colIndex]]); } inline float Row::getFloatField(uint32_t colIndex) const { - return *((float*) &data[offsets[colIndex]]); + return *((float*)&data[offsets[colIndex]]); } inline long double Row::getLongDoubleField(uint32_t colIndex) const { - return *((long double*) &data[offsets[colIndex]]); + return *((long double*)&data[offsets[colIndex]]); } inline void Row::storeInt128FieldIntoPtr(uint32_t colIndex, uint8_t* x) const { - datatypes::TSInt128::assignPtrPtr(x, &data[offsets[colIndex]]); + datatypes::TSInt128::assignPtrPtr(x, &data[offsets[colIndex]]); } inline void Row::getInt128Field(uint32_t colIndex, int128_t& x) const { - datatypes::TSInt128::assignPtrPtr(&x, &data[offsets[colIndex]]); + datatypes::TSInt128::assignPtrPtr(&x, &data[offsets[colIndex]]); } inline datatypes::TSInt128 Row::getTSInt128Field(uint32_t colIndex) const { - const int128_t* ptr = getBinaryField(colIndex); - return datatypes::TSInt128(ptr); + const int128_t* ptr = getBinaryField(colIndex); + return datatypes::TSInt128(ptr); } inline uint64_t Row::getRid() const { - return baseRid + *((uint16_t*) data); + return baseRid + *((uint16_t*)data); } inline uint16_t Row::getRelRid() const { - return *((uint16_t*) data); + return *((uint16_t*)data); } inline uint64_t Row::getBaseRid() const { - return baseRid; + return baseRid; } inline void Row::markRow() { - *((uint16_t*) data) = 0xffff; + *((uint16_t*)data) = 0xffff; } inline void Row::zeroRid() { - *((uint16_t*) data) = 0; + *((uint16_t*)data) = 0; } inline bool Row::isMarked() { - return *((uint16_t*) data) == 0xffff; + return *((uint16_t*)data) == 0xffff; } /* Begin speculative code! */ inline uint32_t Row::getOffset(uint32_t colIndex) const { - return offsets[colIndex]; + return offsets[colIndex]; } -template +template inline void Row::setUintField_offset(uint64_t val, uint32_t offset) { - switch (len) - { - case 1: - data[offset] = val; - break; + switch (len) + { + case 1: data[offset] = val; break; - case 2: - *((uint16_t*) &data[offset]) = val; - break; + case 2: *((uint16_t*)&data[offset]) = val; break; - case 4: - *((uint32_t*) &data[offset]) = val; - break; + case 4: *((uint32_t*)&data[offset]) = val; break; - case 8: - *((uint64_t*) &data[offset]) = val; - break; + case 8: *((uint64_t*)&data[offset]) = val; break; - default: - idbassert(0); - throw std::logic_error("Row::setUintField called on a non-uint32_t field"); - } + default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); + } } -template +template inline void Row::setIntField_offset(const T val, const uint32_t offset) { - *((T*) &data[offset]) = val; + *((T*)&data[offset]) = val; } inline void Row::nextRow(uint32_t size) { - data += size; + data += size; } - inline void Row::prevRow(uint32_t size, uint64_t number = 1) { - data -= size * number; + data -= size * number; } -template +template inline void Row::setUintField(uint64_t val, uint32_t colIndex) { - switch (len) - { - case 1: - data[offsets[colIndex]] = val; - break; + switch (len) + { + case 1: data[offsets[colIndex]] = val; break; - case 2: - *((uint16_t*) &data[offsets[colIndex]]) = val; - break; + case 2: *((uint16_t*)&data[offsets[colIndex]]) = val; break; - case 4: - *((uint32_t*) &data[offsets[colIndex]]) = val; - break; + case 4: *((uint32_t*)&data[offsets[colIndex]]) = val; break; - case 8: - *((uint64_t*) &data[offsets[colIndex]]) = val; - break; + case 8: *((uint64_t*)&data[offsets[colIndex]]) = val; break; - default: - idbassert(0); - throw std::logic_error("Row::setUintField called on a non-uint32_t field"); - } + default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); + } } inline void Row::setUintField(uint64_t val, uint32_t colIndex) { - switch (getColumnWidth(colIndex)) - { - case 1: - data[offsets[colIndex]] = val; - break; + switch (getColumnWidth(colIndex)) + { + case 1: data[offsets[colIndex]] = val; break; - case 2: - *((uint16_t*) &data[offsets[colIndex]]) = val; - break; + case 2: *((uint16_t*)&data[offsets[colIndex]]) = val; break; - case 4: - *((uint32_t*) &data[offsets[colIndex]]) = val; - break; + case 4: *((uint32_t*)&data[offsets[colIndex]]) = val; break; - case 8: - *((uint64_t*) &data[offsets[colIndex]]) = val; - break; + case 8: *((uint64_t*)&data[offsets[colIndex]]) = val; break; - default: - idbassert(0); - throw std::logic_error("Row::setUintField: bad length"); - } + default: idbassert(0); throw std::logic_error("Row::setUintField: bad length"); + } } -template +template inline void Row::setIntField(int64_t val, uint32_t colIndex) { - switch (len) - { - case 1: - *((int8_t*) &data[offsets[colIndex]]) = val; - break; + switch (len) + { + case 1: *((int8_t*)&data[offsets[colIndex]]) = val; break; - case 2: - *((int16_t*) &data[offsets[colIndex]]) = val; - break; + case 2: *((int16_t*)&data[offsets[colIndex]]) = val; break; - case 4: - *((int32_t*) &data[offsets[colIndex]]) = val; - break; + case 4: *((int32_t*)&data[offsets[colIndex]]) = val; break; - case 8: - *((int64_t*) &data[offsets[colIndex]]) = val; - break; + case 8: *((int64_t*)&data[offsets[colIndex]]) = val; break; - default: - idbassert(0); - throw std::logic_error("Row::setIntField: bad length"); - } + default: idbassert(0); throw std::logic_error("Row::setIntField: bad length"); + } } inline void Row::setIntField(int64_t val, uint32_t colIndex) { - switch (getColumnWidth(colIndex)) - { - case 1: - *((int8_t*) &data[offsets[colIndex]]) = val; - break; + switch (getColumnWidth(colIndex)) + { + case 1: *((int8_t*)&data[offsets[colIndex]]) = val; break; - case 2: - *((int16_t*) &data[offsets[colIndex]]) = val; - break; + case 2: *((int16_t*)&data[offsets[colIndex]]) = val; break; - case 4: - *((int32_t*) &data[offsets[colIndex]]) = val; - break; + case 4: *((int32_t*)&data[offsets[colIndex]]) = val; break; - case 8: - *((int64_t*) &data[offsets[colIndex]]) = val; - break; + case 8: *((int64_t*)&data[offsets[colIndex]]) = val; break; - default: - idbassert(0); - throw std::logic_error("Row::setIntField: bad length"); - } + default: idbassert(0); throw std::logic_error("Row::setIntField: bad length"); + } } inline void Row::setDoubleField(double val, uint32_t colIndex) { - *((double*) &data[offsets[colIndex]]) = val; + *((double*)&data[offsets[colIndex]]) = val; } inline void Row::setFloatField(float val, uint32_t colIndex) { - //N.B. There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, - // but not necessarily the same bits that you put in. This only seems to be for float (double seems - // to work). - if (std::isnan(val)) - setUintField<4>(joblist::FLOATNULL, colIndex); - else - *((float*) &data[offsets[colIndex]]) = val; + // N.B. There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, + // but not necessarily the same bits that you put in. This only seems to be for float (double seems + // to work). + if (std::isnan(val)) + setUintField<4>(joblist::FLOATNULL, colIndex); + else + *((float*)&data[offsets[colIndex]]) = val; } inline void Row::setLongDoubleField(const long double& val, uint32_t colIndex) { - uint8_t* p = &data[offsets[colIndex]]; - *reinterpret_cast(p) = val; + uint8_t* p = &data[offsets[colIndex]]; + *reinterpret_cast(p) = val; #ifdef MASK_LONGDOUBLE - memset(p+10, 0, 6); -#endif + memset(p + 10, 0, 6); +#endif } inline void Row::setInt128Field(const int128_t& val, uint32_t colIndex) { - setBinaryField(&val, colIndex); + setBinaryField(&val, colIndex); } inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) { - if (inStringTable(colIndex)) - setStringField(val, colIndex); - else - { - *((uint16_t*) &data[offsets[colIndex]]) = static_cast(val.length()); - memcpy(&data[offsets[colIndex] + 2], val.data(), val.length()); - } + if (inStringTable(colIndex)) + setStringField(val, colIndex); + else + { + *((uint16_t*)&data[offsets[colIndex]]) = static_cast(val.length()); + memcpy(&data[offsets[colIndex] + 2], val.data(), val.length()); + } } inline void Row::setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex) { - if (len > getColumnWidth(colIndex)) - len = getColumnWidth(colIndex); + if (len > getColumnWidth(colIndex)) + len = getColumnWidth(colIndex); - if (inStringTable(colIndex)) - { - uint64_t offset = strings->storeString(val, len); - *((uint64_t*) &data[offsets[colIndex]]) = offset; - } - else - { - *((uint16_t*) &data[offsets[colIndex]]) = len; - memcpy(&data[offsets[colIndex] + 2], val, len); - } + if (inStringTable(colIndex)) + { + uint64_t offset = strings->storeString(val, len); + *((uint64_t*)&data[offsets[colIndex]]) = offset; + } + else + { + *((uint16_t*)&data[offsets[colIndex]]) = len; + memcpy(&data[offsets[colIndex] + 2], val, len); + } } -inline void Row::setUserData(mcsv1sdk::mcsv1Context& context, - boost::shared_ptr userData, +inline void Row::setUserData(mcsv1sdk::mcsv1Context& context, boost::shared_ptr userData, uint32_t len, uint32_t colIndex) { - if (!userDataStore) - { - return; - } + if (!userDataStore) + { + return; + } - uint32_t offset = userDataStore->storeUserData(context, userData, len); - *((uint32_t*) &data[offsets[colIndex]]) = offset; - *((uint32_t*) &data[offsets[colIndex] + 4]) = len; + uint32_t offset = userDataStore->storeUserData(context, userData, len); + *((uint32_t*)&data[offsets[colIndex]]) = offset; + *((uint32_t*)&data[offsets[colIndex] + 4]) = len; } inline void Row::copyField(uint32_t destIndex, uint32_t srcIndex) const { - uint32_t n = offsets[destIndex + 1] - offsets[destIndex]; - memmove(&data[offsets[destIndex]], &data[offsets[srcIndex]], n); + uint32_t n = offsets[destIndex + 1] - offsets[destIndex]; + memmove(&data[offsets[destIndex]], &data[offsets[srcIndex]], n); } inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) const { - if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY || - types[srcIndex] == execplan::CalpontSystemCatalog::BLOB || - types[srcIndex] == execplan::CalpontSystemCatalog::TEXT)) - { - out.setVarBinaryField(getVarBinaryStringField(srcIndex), destIndex); - } - else if (UNLIKELY(isLongString(srcIndex))) - { - out.setStringField(getConstString(srcIndex), destIndex); - } - else if (UNLIKELY(isShortString(srcIndex))) - { - out.setUintField(getUintField(srcIndex), destIndex); - } - else if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::LONGDOUBLE)) - { - out.setLongDoubleField(getLongDoubleField(srcIndex), destIndex); - } - else if (UNLIKELY(datatypes::isWideDecimalType( - types[srcIndex], colWidths[srcIndex]))) - { - copyBinaryField(out, destIndex, srcIndex); - } - else - { - out.setIntField(getIntField(srcIndex), destIndex); - } + if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY || + types[srcIndex] == execplan::CalpontSystemCatalog::BLOB || + types[srcIndex] == execplan::CalpontSystemCatalog::TEXT)) + { + out.setVarBinaryField(getVarBinaryStringField(srcIndex), destIndex); + } + else if (UNLIKELY(isLongString(srcIndex))) + { + out.setStringField(getConstString(srcIndex), destIndex); + } + else if (UNLIKELY(isShortString(srcIndex))) + { + out.setUintField(getUintField(srcIndex), destIndex); + } + else if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { + out.setLongDoubleField(getLongDoubleField(srcIndex), destIndex); + } + else if (UNLIKELY(datatypes::isWideDecimalType(types[srcIndex], colWidths[srcIndex]))) + { + copyBinaryField(out, destIndex, srcIndex); + } + else + { + out.setIntField(getIntField(srcIndex), destIndex); + } } -template +template inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const { - out.setBinaryField(getBinaryField(srcIndex), destIndex); + out.setBinaryField(getBinaryField(srcIndex), destIndex); } inline void Row::setRid(uint64_t rid) { - *((uint16_t*) data) = rid & 0xffff; + *((uint16_t*)data) = rid & 0xffff; } inline uint64_t Row::hash() const { - return hash(columnCount - 1); + return hash(columnCount - 1); } - inline uint64_t Row::hash(uint32_t lastCol) const { - // Use two hash classes. MariaDBHasher for text-based - // collation-aware data types and Hasher_r for all other data types. - // We deliver a hash that is a combination of both hashers' results. - utils::Hasher_r h; - datatypes::MariaDBHasher hM; - uint32_t intermediateHash = 0; + // Use two hash classes. MariaDBHasher for text-based + // collation-aware data types and Hasher_r for all other data types. + // We deliver a hash that is a combination of both hashers' results. + utils::Hasher_r h; + datatypes::MariaDBHasher hM; + uint32_t intermediateHash = 0; - // Sometimes we ask this to hash 0 bytes, and it comes through looking like - // lastCol = -1. Return 0. - if (lastCol >= columnCount) - return 0; + // Sometimes we ask this to hash 0 bytes, and it comes through looking like + // lastCol = -1. Return 0. + if (lastCol >= columnCount) + return 0; - for (uint32_t i = 0; i <= lastCol; i++) - colUpdateHasher(hM, h, i, intermediateHash); + for (uint32_t i = 0; i <= lastCol; i++) + colUpdateHasher(hM, h, i, intermediateHash); - return utils::HashFamily(h, intermediateHash, lastCol << 2, hM).finalize(); + return utils::HashFamily(h, intermediateHash, lastCol << 2, hM).finalize(); } inline bool Row::equals(const Row& r2) const { - return equals(r2, columnCount - 1); + return equals(r2, columnCount - 1); } - /** @brief RowGroup is a lightweight interface for processing packed row data - A RowGroup is an interface for parsing and/or modifying row data as described at the top - of this file. Its lifecycle can be tied to a producer or consumer's lifecycle. - Only one instance is required to process any number of blocks with a - given column configuration. The column configuration is specified in the - constructor, and the block data to process is specified through the - setData() function. It will not copy or take ownership of the data it processes; - the caller should do that. + A RowGroup is an interface for parsing and/or modifying row data as described at the top + of this file. Its lifecycle can be tied to a producer or consumer's lifecycle. + Only one instance is required to process any number of blocks with a + given column configuration. The column configuration is specified in the + constructor, and the block data to process is specified through the + setData() function. It will not copy or take ownership of the data it processes; + the caller should do that. - Row and RowGroup share some bits. RowGroup owns the memory they share. + Row and RowGroup share some bits. RowGroup owns the memory they share. */ class RowGroup : public messageqcpp::Serializeable { -public: - /** @brief The default ctor. It does nothing. Need to init by assignment or deserialization */ - RowGroup(); + public: + /** @brief The default ctor. It does nothing. Need to init by assignment or deserialization */ + RowGroup(); - /** @brief The RowGroup ctor, which specifies the column config to process + /** @brief The RowGroup ctor, which specifies the column config to process - @param colCount The number of columns - @param positions An array specifying the offsets within the packed data - of a row where each column begins. It should have colCount + 1 - entries. The first offset is 2, because a row begins with a 2-byte - RID. The last entry should be the offset of the last column + - its length, which is also the size of the entire row including the rid. - @param coids An array of oids for each column. - @param tkeys An array of unique id for each column. - @param colTypes An array of COLTYPEs for each column. - @param charsetNumbers an Array of the lookup numbers for the charset/collation object. - @param scale An array specifying the scale of DECIMAL types (0 for non-decimal) - @param precision An array specifying the precision of DECIMAL types (0 for non-decimal) - */ + @param colCount The number of columns + @param positions An array specifying the offsets within the packed data + of a row where each column begins. It should have colCount + 1 + entries. The first offset is 2, because a row begins with a 2-byte + RID. The last entry should be the offset of the last column + + its length, which is also the size of the entire row including the rid. + @param coids An array of oids for each column. + @param tkeys An array of unique id for each column. + @param colTypes An array of COLTYPEs for each column. + @param charsetNumbers an Array of the lookup numbers for the charset/collation object. + @param scale An array specifying the scale of DECIMAL types (0 for non-decimal) + @param precision An array specifying the precision of DECIMAL types (0 for non-decimal) + */ - RowGroup(uint32_t colCount, - const std::vector& positions, - const std::vector& cOids, - const std::vector& tkeys, - const std::vector& colTypes, - const std::vector& charsetNumbers, - const std::vector& scale, - const std::vector& precision, - uint32_t stringTableThreshold, - bool useStringTable = true, - const std::vector& forceInlineData = std::vector() - ); + RowGroup(uint32_t colCount, const std::vector& positions, const std::vector& cOids, + const std::vector& tkeys, + const std::vector& colTypes, + const std::vector& charsetNumbers, const std::vector& scale, + const std::vector& precision, uint32_t stringTableThreshold, bool useStringTable = true, + const std::vector& forceInlineData = std::vector()); - /** @brief The copiers. It copies metadata, not the row data */ - RowGroup(const RowGroup&); + /** @brief The copiers. It copies metadata, not the row data */ + RowGroup(const RowGroup&); - /** @brief Assignment operator. It copies metadata, not the row data */ - RowGroup& operator=(const RowGroup&); + /** @brief Assignment operator. It copies metadata, not the row data */ + RowGroup& operator=(const RowGroup&); - explicit RowGroup(messageqcpp::ByteStream& bs); + explicit RowGroup(messageqcpp::ByteStream& bs); - ~RowGroup(); + ~RowGroup(); - inline void initRow(Row*, bool forceInlineData = false) const; - inline uint32_t getRowCount() const; - inline void incRowCount(); - inline void setRowCount(uint32_t num); - inline void getRow(uint32_t rowNum, Row*) const; - inline uint32_t getRowSize() const; - inline uint32_t getRowSizeWithStrings() const; - inline uint64_t getBaseRid() const; - void setData(RGData* rgd); - inline void setData(uint8_t* d); - inline uint8_t* getData() const; - inline RGData* getRGData() const; + inline void initRow(Row*, bool forceInlineData = false) const; + inline uint32_t getRowCount() const; + inline void incRowCount(); + inline void setRowCount(uint32_t num); + inline void getRow(uint32_t rowNum, Row*) const; + inline uint32_t getRowSize() const; + inline uint32_t getRowSizeWithStrings() const; + inline uint64_t getBaseRid() const; + void setData(RGData* rgd); + inline void setData(uint8_t* d); + inline uint8_t* getData() const; + inline RGData* getRGData() const; - uint32_t getStatus() const; - void setStatus(uint16_t); + uint32_t getStatus() const; + void setStatus(uint16_t); - uint32_t getDBRoot() const; - void setDBRoot(uint32_t); + uint32_t getDBRoot() const; + void setDBRoot(uint32_t); - uint32_t getDataSize() const; - uint32_t getDataSize(uint64_t n) const; - uint32_t getMaxDataSize() const; - uint32_t getMaxDataSizeWithStrings() const; - uint32_t getEmptySize() const; + uint32_t getDataSize() const; + uint32_t getDataSize(uint64_t n) const; + uint32_t getMaxDataSize() const; + uint32_t getMaxDataSizeWithStrings() const; + uint32_t getEmptySize() const; - // this returns the size of the row data with the string table - inline uint64_t getSizeWithStrings() const; - inline uint64_t getSizeWithStrings(uint64_t n) const; + // this returns the size of the row data with the string table + inline uint64_t getSizeWithStrings() const; + inline uint64_t getSizeWithStrings(uint64_t n) const; - // sets the row count to 0 and the baseRid to something - // effectively initializing whatever chunk of memory - // data points to - void resetRowGroup(uint64_t baseRid); + // sets the row count to 0 and the baseRid to something + // effectively initializing whatever chunk of memory + // data points to + void resetRowGroup(uint64_t baseRid); - /* The Serializeable interface */ - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&); + /* The Serializeable interface */ + void serialize(messageqcpp::ByteStream&) const; + void deserialize(messageqcpp::ByteStream&); - uint32_t getColumnWidth(uint32_t col) const; - uint32_t getColumnCount() const; - inline const std::vector& getOffsets() const; - inline const std::vector& getOIDs() const; - inline const std::vector& getKeys() const; - inline const std::vector& getColWidths() const; - inline execplan::CalpontSystemCatalog::ColDataType getColType(uint32_t colIndex) const; - inline const std::vector& getColTypes() const; - inline std::vector& getColTypes(); - inline const std::vector& getCharsetNumbers() const; - inline uint32_t getCharsetNumber(uint32_t colIndex) const; - inline boost::shared_array& getForceInline(); - static inline uint32_t getHeaderSize() - { - return headerSize; - } + uint32_t getColumnWidth(uint32_t col) const; + uint32_t getColumnCount() const; + inline const std::vector& getOffsets() const; + inline const std::vector& getOIDs() const; + inline const std::vector& getKeys() const; + inline const std::vector& getColWidths() const; + inline execplan::CalpontSystemCatalog::ColDataType getColType(uint32_t colIndex) const; + inline const std::vector& getColTypes() const; + inline std::vector& getColTypes(); + inline const std::vector& getCharsetNumbers() const; + inline uint32_t getCharsetNumber(uint32_t colIndex) const; + inline boost::shared_array& getForceInline(); + static inline uint32_t getHeaderSize() + { + return headerSize; + } - // this returns true if the type is CHAR or VARCHAR - inline bool isCharType(uint32_t colIndex) const; - inline bool isUnsigned(uint32_t colIndex) const; - inline bool isShortString(uint32_t colIndex) const; - inline bool isLongString(uint32_t colIndex) const; + // this returns true if the type is CHAR or VARCHAR + inline bool isCharType(uint32_t colIndex) const; + inline bool isUnsigned(uint32_t colIndex) const; + inline bool isShortString(uint32_t colIndex) const; + inline bool isLongString(uint32_t colIndex) const; - bool colHasCollation(uint32_t colIndex) const - { - return datatypes::typeHasCollation(getColType(colIndex)); - } + bool colHasCollation(uint32_t colIndex) const + { + return datatypes::typeHasCollation(getColType(colIndex)); + } - inline const std::vector& getScale() const; - inline const std::vector& getPrecision() const; + inline const std::vector& getScale() const; + inline const std::vector& getPrecision() const; - inline bool usesStringTable() const; - inline void setUseStringTable(bool); + inline bool usesStringTable() const; + inline void setUseStringTable(bool); -// RGData *convertToInlineData(uint64_t *size = NULL) const; // caller manages the memory returned by this -// void convertToInlineDataInPlace(); -// RGData *convertToStringTable(uint64_t *size = NULL) const; -// void convertToStringTableInPlace(); - void serializeRGData(messageqcpp::ByteStream&) const; - inline uint32_t getStringTableThreshold() const; + // RGData *convertToInlineData(uint64_t *size = NULL) const; // caller manages the memory returned by + //this void convertToInlineDataInPlace(); RGData *convertToStringTable(uint64_t *size = NULL) const; void + //convertToStringTableInPlace(); + void serializeRGData(messageqcpp::ByteStream&) const; + inline uint32_t getStringTableThreshold() const; - void append(RGData&); - void append(RowGroup&); - void append(RGData&, uint pos); // insert starting at position 'pos' - void append(RowGroup&, uint pos); + void append(RGData&); + void append(RowGroup&); + void append(RGData&, uint pos); // insert starting at position 'pos' + void append(RowGroup&, uint pos); - RGData duplicate(); // returns a copy of the attached RGData + RGData duplicate(); // returns a copy of the attached RGData - std::string toString(const std::vector& used = {}) const; + std::string toString(const std::vector& used = {}) const; - /** operator+= - * - * append the metadata of another RowGroup to this RowGroup - */ - RowGroup& operator+=(const RowGroup& rhs); + /** operator+= + * + * append the metadata of another RowGroup to this RowGroup + */ + RowGroup& operator+=(const RowGroup& rhs); - // returns a RowGroup with only the first cols columns. Useful for generating a - // RowGroup where the first cols make up a key of some kind, and the rest is irrelevant. - RowGroup truncate(uint32_t cols); + // returns a RowGroup with only the first cols columns. Useful for generating a + // RowGroup where the first cols make up a key of some kind, and the rest is irrelevant. + RowGroup truncate(uint32_t cols); - /** operator< - * - * Orders RG's based on baseRid - */ - inline bool operator<(const RowGroup& rhs) const; + /** operator< + * + * Orders RG's based on baseRid + */ + inline bool operator<(const RowGroup& rhs) const; - void addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList); + void addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& sysDataList); - /* Base RIDs are now a combination of partition#, segment#, extent#, and block#. */ - inline void setBaseRid(const uint32_t& partNum, const uint16_t& segNum, - const uint8_t& extentNum, const uint16_t& blockNum); - inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, - uint16_t* blockNum); + /* Base RIDs are now a combination of partition#, segment#, extent#, and block#. */ + inline void setBaseRid(const uint32_t& partNum, const uint16_t& segNum, const uint8_t& extentNum, + const uint16_t& blockNum); + inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum); - inline void setStringStore(boost::shared_ptr); + inline void setStringStore(boost::shared_ptr); - const CHARSET_INFO* getCharset(uint32_t col); + const CHARSET_INFO* getCharset(uint32_t col); -private: - uint32_t columnCount; - uint8_t* data; + private: + uint32_t columnCount; + uint8_t* data; - std::vector oldOffsets; //inline data offsets - std::vector stOffsets; //string table offsets - uint32_t* offsets; //offsets either points to oldOffsets or stOffsets - std::vector colWidths; - // oids: the real oid of the column, may have duplicates with alias. - // This oid is necessary for front-end to decide the real column width. - std::vector oids; - // keys: the unique id for pair(oid, alias). bug 1632. - // Used to map the projected column and rowgroup index - std::vector keys; - std::vector types; - // For string collation - std::vector charsetNumbers; - std::vector charsets; + std::vector oldOffsets; // inline data offsets + std::vector stOffsets; // string table offsets + uint32_t* offsets; // offsets either points to oldOffsets or stOffsets + std::vector colWidths; + // oids: the real oid of the column, may have duplicates with alias. + // This oid is necessary for front-end to decide the real column width. + std::vector oids; + // keys: the unique id for pair(oid, alias). bug 1632. + // Used to map the projected column and rowgroup index + std::vector keys; + std::vector types; + // For string collation + std::vector charsetNumbers; + std::vector charsets; - // DECIMAL support. For non-decimal fields, the values are 0. - std::vector scale; - std::vector precision; + // DECIMAL support. For non-decimal fields, the values are 0. + std::vector scale; + std::vector precision; - // string table impl - RGData* rgData; - StringStore* strings; // note, strings and data belong to rgData - bool useStringTable; - bool hasCollation; - bool hasLongStringField; - uint32_t sTableThreshold; - boost::shared_array forceInline; + // string table impl + RGData* rgData; + StringStore* strings; // note, strings and data belong to rgData + bool useStringTable; + bool hasCollation; + bool hasLongStringField; + uint32_t sTableThreshold; + boost::shared_array forceInline; - static const uint32_t headerSize = 18; - static const uint32_t rowCountOffset = 0; - static const uint32_t baseRidOffset = 4; - static const uint32_t statusOffset = 12; - static const uint32_t dbRootOffset = 14; + static const uint32_t headerSize = 18; + static const uint32_t rowCountOffset = 0; + static const uint32_t baseRidOffset = 4; + static const uint32_t statusOffset = 12; + static const uint32_t dbRootOffset = 14; }; -inline uint64_t convertToRid(const uint32_t& partNum, const uint16_t& segNum, - const uint8_t& extentNum, const uint16_t& blockNum); -inline void getLocationFromRid(uint64_t rid, uint32_t* partNum, - uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum); +inline uint64_t convertToRid(const uint32_t& partNum, const uint16_t& segNum, const uint8_t& extentNum, + const uint16_t& blockNum); +inline void getLocationFromRid(uint64_t rid, uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, + uint16_t* blockNum); // returns the first rid of the logical block specified by baseRid inline uint64_t getExtentRelativeRid(uint64_t baseRid); @@ -1711,9 +1617,9 @@ inline uint64_t getExtentRelativeRid(uint64_t baseRid); inline uint64_t getFileRelativeRid(uint64_t baseRid); /** operator+ -* -* add the metadata of 2 RowGroups together and return a new RowGroup -*/ + * + * add the metadata of 2 RowGroups together and return a new RowGroup + */ RowGroup operator+(const RowGroup& lhs, const RowGroup& rhs); boost::shared_array makeMapping(const RowGroup& r1, const RowGroup& r2); @@ -1725,556 +1631,558 @@ void applyMapping(const int* mapping, const Row& in, Row* out); every row, they're a measurable performance penalty */ inline uint32_t RowGroup::getRowCount() const { -// idbassert(data); -// if (!data) throw std::logic_error("RowGroup::getRowCount(): data is NULL!"); - return *((uint32_t*) &data[rowCountOffset]); + // idbassert(data); + // if (!data) throw std::logic_error("RowGroup::getRowCount(): data is NULL!"); + return *((uint32_t*)&data[rowCountOffset]); } inline void RowGroup::incRowCount() { -// idbassert(data); - ++(*((uint32_t*) &data[rowCountOffset])); + // idbassert(data); + ++(*((uint32_t*)&data[rowCountOffset])); } inline void RowGroup::setRowCount(uint32_t num) { -// idbassert(data); - *((uint32_t*) &data[rowCountOffset]) = num; + // idbassert(data); + *((uint32_t*)&data[rowCountOffset]) = num; } inline void RowGroup::getRow(uint32_t rowNum, Row* r) const { -// idbassert(data); - if (useStringTable != r->usesStringTable()) - initRow(r); + // idbassert(data); + if (useStringTable != r->usesStringTable()) + initRow(r); - r->baseRid = getBaseRid(); - r->data = &(data[headerSize + (rowNum * offsets[columnCount])]); - r->strings = strings; - r->userDataStore = rgData->userDataStore.get(); + r->baseRid = getBaseRid(); + r->data = &(data[headerSize + (rowNum * offsets[columnCount])]); + r->strings = strings; + r->userDataStore = rgData->userDataStore.get(); } inline void RowGroup::setData(uint8_t* d) { - data = d; - strings = NULL; - rgData = NULL; - setUseStringTable(false); + data = d; + strings = NULL; + rgData = NULL; + setUseStringTable(false); } inline void RowGroup::setData(RGData* rgd) { - data = rgd->rowData.get(); - strings = rgd->strings.get(); - rgData = rgd; + data = rgd->rowData.get(); + strings = rgd->strings.get(); + rgData = rgd; } inline uint8_t* RowGroup::getData() const { - //assert(!useStringTable); - return data; + // assert(!useStringTable); + return data; } inline RGData* RowGroup::getRGData() const { - return rgData; + return rgData; } inline void RowGroup::setUseStringTable(bool b) { - useStringTable = (b && hasLongStringField); - //offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - offsets = 0; + useStringTable = (b && hasLongStringField); + // offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + offsets = 0; - if (useStringTable && !stOffsets.empty()) - offsets = &stOffsets[0]; - else if (!useStringTable && !oldOffsets.empty()) - offsets = &oldOffsets[0]; + if (useStringTable && !stOffsets.empty()) + offsets = &stOffsets[0]; + else if (!useStringTable && !oldOffsets.empty()) + offsets = &oldOffsets[0]; - if (!useStringTable) - strings = NULL; + if (!useStringTable) + strings = NULL; } inline uint64_t RowGroup::getBaseRid() const { - return *((uint64_t*) &data[baseRidOffset]); + return *((uint64_t*)&data[baseRidOffset]); } inline bool RowGroup::operator<(const RowGroup& rhs) const { - return (getBaseRid() < rhs.getBaseRid()); + return (getBaseRid() < rhs.getBaseRid()); } void RowGroup::initRow(Row* r, bool forceInlineData) const { - r->columnCount = columnCount; + r->columnCount = columnCount; - if (LIKELY(!types.empty())) - { - r->colWidths = (uint32_t*) &colWidths[0]; - r->types = (execplan::CalpontSystemCatalog::ColDataType*) & (types[0]); - r->charsetNumbers = (uint32_t*) & (charsetNumbers[0]); - r->charsets = (CHARSET_INFO**) & (charsets[0]); - r->scale = (uint32_t*) & (scale[0]); - r->precision = (uint32_t*) & (precision[0]); - } + if (LIKELY(!types.empty())) + { + r->colWidths = (uint32_t*)&colWidths[0]; + r->types = (execplan::CalpontSystemCatalog::ColDataType*)&(types[0]); + r->charsetNumbers = (uint32_t*)&(charsetNumbers[0]); + r->charsets = (CHARSET_INFO**)&(charsets[0]); + r->scale = (uint32_t*)&(scale[0]); + r->precision = (uint32_t*)&(precision[0]); + } - if (forceInlineData) - { - r->useStringTable = false; - r->oldOffsets = (uint32_t*) & (oldOffsets[0]); - r->stOffsets = (uint32_t*) & (stOffsets[0]); - r->offsets = (uint32_t*) & (oldOffsets[0]); - } - else - { - r->useStringTable = useStringTable; - r->oldOffsets = (uint32_t*) & (oldOffsets[0]); - r->stOffsets = (uint32_t*) & (stOffsets[0]); - r->offsets = offsets; - } + if (forceInlineData) + { + r->useStringTable = false; + r->oldOffsets = (uint32_t*)&(oldOffsets[0]); + r->stOffsets = (uint32_t*)&(stOffsets[0]); + r->offsets = (uint32_t*)&(oldOffsets[0]); + } + else + { + r->useStringTable = useStringTable; + r->oldOffsets = (uint32_t*)&(oldOffsets[0]); + r->stOffsets = (uint32_t*)&(stOffsets[0]); + r->offsets = offsets; + } - r->hasLongStringField = hasLongStringField; - r->sTableThreshold = sTableThreshold; - r->forceInline = forceInline; - r->hasCollation = hasCollation; + r->hasLongStringField = hasLongStringField; + r->sTableThreshold = sTableThreshold; + r->forceInline = forceInline; + r->hasCollation = hasCollation; } inline uint32_t RowGroup::getRowSize() const { - return offsets[columnCount]; + return offsets[columnCount]; } inline uint32_t RowGroup::getRowSizeWithStrings() const { - return oldOffsets[columnCount]; + return oldOffsets[columnCount]; } inline uint64_t RowGroup::getSizeWithStrings(uint64_t n) const { - if (strings == NULL) - return getDataSize(n); - else - return getDataSize(n) + strings->getSize(); + if (strings == NULL) + return getDataSize(n); + else + return getDataSize(n) + strings->getSize(); } inline uint64_t RowGroup::getSizeWithStrings() const { - return getSizeWithStrings(getRowCount()); + return getSizeWithStrings(getRowCount()); } inline bool RowGroup::isCharType(uint32_t colIndex) const { - return datatypes::isCharType(types[colIndex]); + return datatypes::isCharType(types[colIndex]); } inline bool RowGroup::isUnsigned(uint32_t colIndex) const { - return datatypes::isUnsigned(types[colIndex]); + return datatypes::isUnsigned(types[colIndex]); } inline bool RowGroup::isShortString(uint32_t colIndex) const { - return ((getColumnWidth(colIndex) <= 7 && types[colIndex] == execplan::CalpontSystemCatalog::VARCHAR) || - (getColumnWidth(colIndex) <= 8 && types[colIndex] == execplan::CalpontSystemCatalog::CHAR)); + return ((getColumnWidth(colIndex) <= 7 && types[colIndex] == execplan::CalpontSystemCatalog::VARCHAR) || + (getColumnWidth(colIndex) <= 8 && types[colIndex] == execplan::CalpontSystemCatalog::CHAR)); } inline bool RowGroup::isLongString(uint32_t colIndex) const { - return ((getColumnWidth(colIndex) > 7 && types[colIndex] == execplan::CalpontSystemCatalog::VARCHAR) || - (getColumnWidth(colIndex) > 8 && types[colIndex] == execplan::CalpontSystemCatalog::CHAR) || - types[colIndex] == execplan::CalpontSystemCatalog::VARBINARY || - types[colIndex] == execplan::CalpontSystemCatalog::BLOB || - types[colIndex] == execplan::CalpontSystemCatalog::TEXT); + return ((getColumnWidth(colIndex) > 7 && types[colIndex] == execplan::CalpontSystemCatalog::VARCHAR) || + (getColumnWidth(colIndex) > 8 && types[colIndex] == execplan::CalpontSystemCatalog::CHAR) || + types[colIndex] == execplan::CalpontSystemCatalog::VARBINARY || + types[colIndex] == execplan::CalpontSystemCatalog::BLOB || + types[colIndex] == execplan::CalpontSystemCatalog::TEXT); } inline bool RowGroup::usesStringTable() const { - return useStringTable; + return useStringTable; } inline const std::vector& RowGroup::getOffsets() const { - return oldOffsets; + return oldOffsets; } inline const std::vector& RowGroup::getOIDs() const { - return oids; + return oids; } inline const std::vector& RowGroup::getKeys() const { - return keys; + return keys; } inline execplan::CalpontSystemCatalog::ColDataType RowGroup::getColType(uint32_t colIndex) const { - return types[colIndex]; + return types[colIndex]; } inline const std::vector& RowGroup::getColTypes() const { - return types; + return types; } inline std::vector& RowGroup::getColTypes() { - return types; + return types; } inline const std::vector& RowGroup::getCharsetNumbers() const { - return charsetNumbers; + return charsetNumbers; } inline uint32_t RowGroup::getCharsetNumber(uint32_t colIndex) const { - return charsetNumbers[colIndex]; + return charsetNumbers[colIndex]; } inline const std::vector& RowGroup::getScale() const { - return scale; + return scale; } inline const std::vector& RowGroup::getPrecision() const { - return precision; + return precision; } inline const std::vector& RowGroup::getColWidths() const { - return colWidths; + return colWidths; } inline boost::shared_array& RowGroup::getForceInline() { - return forceInline; + return forceInline; } -inline uint64_t convertToRid(const uint32_t& partitionNum, - const uint16_t& segmentNum, const uint8_t& exNum, const uint16_t& blNum) +inline uint64_t convertToRid(const uint32_t& partitionNum, const uint16_t& segmentNum, const uint8_t& exNum, + const uint16_t& blNum) { - uint64_t partNum = partitionNum, segNum = segmentNum, extentNum = exNum, - blockNum = blNum; + uint64_t partNum = partitionNum, segNum = segmentNum, extentNum = exNum, blockNum = blNum; - // extentNum gets trunc'd to 6 bits, blockNums to 10 bits - extentNum &= 0x3f; - blockNum &= 0x3ff; + // extentNum gets trunc'd to 6 bits, blockNums to 10 bits + extentNum &= 0x3f; + blockNum &= 0x3ff; - return (partNum << 32) | (segNum << 16) | (extentNum << 10) | blockNum; + return (partNum << 32) | (segNum << 16) | (extentNum << 10) | blockNum; } -inline void RowGroup::setBaseRid(const uint32_t& partNum, const uint16_t& segNum, - const uint8_t& extentNum, const uint16_t& blockNum) +inline void RowGroup::setBaseRid(const uint32_t& partNum, const uint16_t& segNum, const uint8_t& extentNum, + const uint16_t& blockNum) { - *((uint64_t*) &data[baseRidOffset]) = convertToRid(partNum, segNum, - extentNum, blockNum); + *((uint64_t*)&data[baseRidOffset]) = convertToRid(partNum, segNum, extentNum, blockNum); } inline uint32_t RowGroup::getStringTableThreshold() const { - return sTableThreshold; + return sTableThreshold; } inline void RowGroup::setStringStore(boost::shared_ptr ss) { - if (useStringTable) - { - rgData->setStringStore(ss); - strings = rgData->strings.get(); - } + if (useStringTable) + { + rgData->setStringStore(ss); + strings = rgData->strings.get(); + } } -inline void getLocationFromRid(uint64_t rid, uint32_t* partNum, - uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum) +inline void getLocationFromRid(uint64_t rid, uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, + uint16_t* blockNum) { - if (partNum) *partNum = rid >> 32; + if (partNum) + *partNum = rid >> 32; - if (segNum) *segNum = rid >> 16; + if (segNum) + *segNum = rid >> 16; - if (extentNum) *extentNum = (rid >> 10) & 0x3f; + if (extentNum) + *extentNum = (rid >> 10) & 0x3f; - if (blockNum) *blockNum = rid & 0x3ff; + if (blockNum) + *blockNum = rid & 0x3ff; } -inline void RowGroup::getLocation(uint32_t* partNum, uint16_t* segNum, - uint8_t* extentNum, uint16_t* blockNum) +inline void RowGroup::getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum) { - getLocationFromRid(getBaseRid(), partNum, segNum, extentNum, blockNum); + getLocationFromRid(getBaseRid(), partNum, segNum, extentNum, blockNum); } // returns the first RID of the logical block identified by baseRid inline uint64_t getExtentRelativeRid(uint64_t baseRid) { - uint64_t blockNum = baseRid & 0x3ff; - return (blockNum << 13); + uint64_t blockNum = baseRid & 0x3ff; + return (blockNum << 13); } inline uint64_t Row::getExtentRelativeRid() const { - return rowgroup::getExtentRelativeRid(baseRid) | (getRelRid() & 0x1fff); + return rowgroup::getExtentRelativeRid(baseRid) | (getRelRid() & 0x1fff); } // returns the first RID of the logical block identified by baseRid inline uint64_t getFileRelativeRid(uint64_t baseRid) { - uint64_t extentNum = (baseRid >> 10) & 0x3f; - uint64_t blockNum = baseRid & 0x3ff; - return (extentNum << 23) | (blockNum << 13); + uint64_t extentNum = (baseRid >> 10) & 0x3f; + uint64_t blockNum = baseRid & 0x3ff; + return (extentNum << 23) | (blockNum << 13); } inline uint64_t Row::getFileRelativeRid() const { - return rowgroup::getFileRelativeRid(baseRid) | (getRelRid() & 0x1fff); + return rowgroup::getFileRelativeRid(baseRid) | (getRelRid() & 0x1fff); } -inline void Row::getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, - uint16_t* blockNum, uint16_t* rowNum) +inline void Row::getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum, + uint16_t* rowNum) { - getLocationFromRid(baseRid, partNum, segNum, extentNum, blockNum); + getLocationFromRid(baseRid, partNum, segNum, extentNum, blockNum); - if (rowNum) *rowNum = getRelRid(); + if (rowNum) + *rowNum = getRelRid(); } inline void copyRow(const Row& in, Row* out, uint32_t colCount) { - if (&in == out) - return; + if (&in == out) + return; - out->setRid(in.getRelRid()); + out->setRid(in.getRelRid()); - if (!in.usesStringTable() && !out->usesStringTable()) + if (!in.usesStringTable() && !out->usesStringTable()) + { + memcpy(out->getData(), in.getData(), std::min(in.getOffset(colCount), out->getOffset(colCount))); + return; + } + + for (uint32_t i = 0; i < colCount; i++) + { + if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || + in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || + in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT || + in.getColTypes()[i] == execplan::CalpontSystemCatalog::CLOB)) { - memcpy(out->getData(), in.getData(), std::min(in.getOffset(colCount), out->getOffset(colCount))); - return; + out->setVarBinaryField(in.getVarBinaryStringField(i), i); } - - for (uint32_t i = 0; i < colCount; i++) + else if (UNLIKELY(in.isLongString(i))) { - if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || - in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || - in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT || - in.getColTypes()[i] == execplan::CalpontSystemCatalog::CLOB)) - { - out->setVarBinaryField(in.getVarBinaryStringField(i), i); - } - else if (UNLIKELY(in.isLongString(i))) - { - out->setStringField(in.getConstString(i), i); - } - else if (UNLIKELY(in.isShortString(i))) - { - out->setUintField(in.getUintField(i), i); - } - else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) - { - out->setLongDoubleField(in.getLongDoubleField(i), i); - } - else if (UNLIKELY(datatypes::isWideDecimalType( - in.getColType(i), in.getColumnWidth(i)))) - { - in.copyBinaryField(*out, i, i); - } - else - { - out->setIntField(in.getIntField(i), i); - } + out->setStringField(in.getConstString(i), i); } + else if (UNLIKELY(in.isShortString(i))) + { + out->setUintField(in.getUintField(i), i); + } + else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { + out->setLongDoubleField(in.getLongDoubleField(i), i); + } + else if (UNLIKELY(datatypes::isWideDecimalType(in.getColType(i), in.getColumnWidth(i)))) + { + in.copyBinaryField(*out, i, i); + } + else + { + out->setIntField(in.getIntField(i), i); + } + } } inline void copyRow(const Row& in, Row* out) { - copyRow(in, out, std::min(in.getColumnCount(), out->getColumnCount())); + copyRow(in, out, std::min(in.getColumnCount(), out->getColumnCount())); } inline std::string StringStore::getString(uint64_t off) const { - uint32_t length; + uint32_t length; - if (off == std::numeric_limits::max()) - return joblist::CPNULLSTRMARK; + if (off == std::numeric_limits::max()) + return joblist::CPNULLSTRMARK; - MemChunk* mc; + MemChunk* mc; - if (off & 0x8000000000000000) - { - //off = off - 0x8000000000000000; - off &= ~0x8000000000000000; + if (off & 0x8000000000000000) + { + // off = off - 0x8000000000000000; + off &= ~0x8000000000000000; - if (longStrings.size() <= off) - return joblist::CPNULLSTRMARK; + if (longStrings.size() <= off) + return joblist::CPNULLSTRMARK; - mc = (MemChunk*) longStrings[off].get(); - memcpy(&length, mc->data, 4); - return std::string((char*) mc->data + 4, length); - } + mc = (MemChunk*)longStrings[off].get(); + memcpy(&length, mc->data, 4); + return std::string((char*)mc->data + 4, length); + } - uint64_t chunk = off / CHUNK_SIZE; - uint64_t offset = off % CHUNK_SIZE; + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; - // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter - // what gets returned, it just can't go out of bounds. - if (mem.size() <= chunk) - return joblist::CPNULLSTRMARK; + // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter + // what gets returned, it just can't go out of bounds. + if (mem.size() <= chunk) + return joblist::CPNULLSTRMARK; - mc = (MemChunk*) mem[chunk].get(); + mc = (MemChunk*)mem[chunk].get(); - memcpy(&length, &mc->data[offset], 4); + memcpy(&length, &mc->data[offset], 4); - if ((offset + length) > mc->currentSize) - return joblist::CPNULLSTRMARK; + if ((offset + length) > mc->currentSize) + return joblist::CPNULLSTRMARK; - return std::string((char*) & (mc->data[offset]) + 4, length); + return std::string((char*)&(mc->data[offset]) + 4, length); } inline const uint8_t* StringStore::getPointer(uint64_t off) const { - if (off == std::numeric_limits::max()) - return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); + if (off == std::numeric_limits::max()) + return (const uint8_t*)joblist::CPNULLSTRMARK.c_str(); - uint64_t chunk = off / CHUNK_SIZE; - uint64_t offset = off % CHUNK_SIZE; - MemChunk* mc; + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; + MemChunk* mc; - if (off & 0x8000000000000000) - { - //off = off - 0x8000000000000000; - off &= ~0x8000000000000000; + if (off & 0x8000000000000000) + { + // off = off - 0x8000000000000000; + off &= ~0x8000000000000000; - if (longStrings.size() <= off) - return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); + if (longStrings.size() <= off) + return (const uint8_t*)joblist::CPNULLSTRMARK.c_str(); - mc = (MemChunk*) longStrings[off].get(); - return mc->data + 4; - } + mc = (MemChunk*)longStrings[off].get(); + return mc->data + 4; + } - // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter - // what gets returned, it just can't go out of bounds. - if (UNLIKELY(mem.size() <= chunk)) - return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); + // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter + // what gets returned, it just can't go out of bounds. + if (UNLIKELY(mem.size() <= chunk)) + return (const uint8_t*)joblist::CPNULLSTRMARK.c_str(); - mc = (MemChunk*) mem[chunk].get(); + mc = (MemChunk*)mem[chunk].get(); - if (offset > mc->currentSize) - return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); + if (offset > mc->currentSize) + return (const uint8_t*)joblist::CPNULLSTRMARK.c_str(); - return &(mc->data[offset]) + 4; + return &(mc->data[offset]) + 4; } inline bool StringStore::isNullValue(uint64_t off) const { - uint32_t length; + uint32_t length; - if (off == std::numeric_limits::max()) - return true; + if (off == std::numeric_limits::max()) + return true; - // Long strings won't be NULL - if (off & 0x8000000000000000) - return false; + // Long strings won't be NULL + if (off & 0x8000000000000000) + return false; - uint32_t chunk = off / CHUNK_SIZE; - uint32_t offset = off % CHUNK_SIZE; - MemChunk* mc; + uint32_t chunk = off / CHUNK_SIZE; + uint32_t offset = off % CHUNK_SIZE; + MemChunk* mc; - if (mem.size() <= chunk) - return true; + if (mem.size() <= chunk) + return true; - mc = (MemChunk*) mem[chunk].get(); - memcpy(&length, &mc->data[offset], 4); + mc = (MemChunk*)mem[chunk].get(); + memcpy(&length, &mc->data[offset], 4); - if (length == 0) - return true; + if (length == 0) + return true; - if (length < 8) - return false; + if (length < 8) + return false; - if ((offset + length) > mc->currentSize) - return true; + if ((offset + length) > mc->currentSize) + return true; - if (mc->data[offset + 4] == 0) // "" = NULL string for some reason... - return true; - return (memcmp(&mc->data[offset+4], joblist::CPNULLSTRMARK.c_str(), 8) == 0); + if (mc->data[offset + 4] == 0) // "" = NULL string for some reason... + return true; + return (memcmp(&mc->data[offset + 4], joblist::CPNULLSTRMARK.c_str(), 8) == 0); } inline uint32_t StringStore::getStringLength(uint64_t off) const { - uint32_t length; - MemChunk* mc; + uint32_t length; + MemChunk* mc; - if (off == std::numeric_limits::max()) - return 0; + if (off == std::numeric_limits::max()) + return 0; - if (off & 0x8000000000000000) - { - //off = off - 0x8000000000000000; - off &= ~0x8000000000000000; + if (off & 0x8000000000000000) + { + // off = off - 0x8000000000000000; + off &= ~0x8000000000000000; - if (longStrings.size() <= off) - return 0; + if (longStrings.size() <= off) + return 0; - mc = (MemChunk*) longStrings[off].get(); - memcpy(&length, mc->data, 4); - } - else - { - uint64_t chunk = off / CHUNK_SIZE; - uint64_t offset = off % CHUNK_SIZE; + mc = (MemChunk*)longStrings[off].get(); + memcpy(&length, mc->data, 4); + } + else + { + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; - if (mem.size() <= chunk) - return 0; + if (mem.size() <= chunk) + return 0; - mc = (MemChunk*) mem[chunk].get(); - memcpy(&length, &mc->data[offset], 4); - } + mc = (MemChunk*)mem[chunk].get(); + memcpy(&length, &mc->data[offset], 4); + } - return length; + return length; } inline bool StringStore::isEmpty() const { - return empty; + return empty; } inline uint64_t StringStore::getSize() const { - uint32_t i; - uint64_t ret = 0; - MemChunk* mc; + uint32_t i; + uint64_t ret = 0; + MemChunk* mc; - ret += sizeof(MemChunk) * mem.size(); - for (i = 0; i < mem.size(); i++) - { - mc = (MemChunk*) mem[i].get(); - ret += mc->capacity; - } + ret += sizeof(MemChunk) * mem.size(); + for (i = 0; i < mem.size(); i++) + { + mc = (MemChunk*)mem[i].get(); + ret += mc->capacity; + } - ret += sizeof(MemChunk) * longStrings.size(); - for (i = 0; i < longStrings.size(); i++) - { - mc = (MemChunk*) longStrings[i].get(); - ret += mc->capacity; - } + ret += sizeof(MemChunk) * longStrings.size(); + for (i = 0; i < longStrings.size(); i++) + { + mc = (MemChunk*)longStrings[i].get(); + ret += mc->capacity; + } - return ret; + return ret; } inline RGData& RGData::operator=(const RGData& r) { - rowData = r.rowData; - strings = r.strings; - userDataStore = r.userDataStore; - return *this; + rowData = r.rowData; + strings = r.strings; + userDataStore = r.userDataStore; + return *this; } inline void RGData::getRow(uint32_t num, Row* row) { - uint32_t size = row->getSize(); - row->setData(Row::Pointer(&rowData[RowGroup::getHeaderSize() + (num * size)], strings.get(), userDataStore.get())); + uint32_t size = row->getSize(); + row->setData( + Row::Pointer(&rowData[RowGroup::getHeaderSize() + (num * size)], strings.get(), userDataStore.get())); } -} +} // namespace rowgroup #endif // vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowstorage.cpp b/utils/rowgroup/rowstorage.cpp index 88f0b104f..7e42fcee6 100644 --- a/utils/rowgroup/rowstorage.cpp +++ b/utils/rowgroup/rowstorage.cpp @@ -26,7 +26,6 @@ namespace { - int writeData(int fd, const char* buf, size_t sz) { if (sz == 0) @@ -99,7 +98,8 @@ inline uint64_t hashData(const void* ptr, uint32_t len, uint64_t x = 0ULL) h ^= x; h *= m; } - for (std::size_t i = 0; i < n_blocks; ++i) { + for (std::size_t i = 0; i < n_blocks; ++i) + { uint64_t k; memcpy(&k, data64 + i, sizeof(k)); @@ -112,31 +112,31 @@ inline uint64_t hashData(const void* ptr, uint32_t len, uint64_t x = 0ULL) } auto const* const data8 = reinterpret_cast(data64 + n_blocks); - switch (len & 7U) { - case 7: - h ^= static_cast(data8[6]) << 48U; - // FALLTHROUGH - case 6: - h ^= static_cast(data8[5]) << 40U; - // FALLTHROUGH - case 5: - h ^= static_cast(data8[4]) << 32U; - // FALLTHROUGH - case 4: - h ^= static_cast(data8[3]) << 24U; - // FALLTHROUGH - case 3: - h ^= static_cast(data8[2]) << 16U; - // FALLTHROUGH - case 2: - h ^= static_cast(data8[1]) << 8U; - // FALLTHROUGH - case 1: - h ^= static_cast(data8[0]); - h *= m; - // FALLTHROUGH - default: - break; + switch (len & 7U) + { + case 7: + h ^= static_cast(data8[6]) << 48U; + // FALLTHROUGH + case 6: + h ^= static_cast(data8[5]) << 40U; + // FALLTHROUGH + case 5: + h ^= static_cast(data8[4]) << 32U; + // FALLTHROUGH + case 4: + h ^= static_cast(data8[3]) << 24U; + // FALLTHROUGH + case 3: + h ^= static_cast(data8[2]) << 16U; + // FALLTHROUGH + case 2: + h ^= static_cast(data8[1]) << 8U; + // FALLTHROUGH + case 1: + h ^= static_cast(data8[0]); + h *= m; + // FALLTHROUGH + default: break; } h ^= h >> r; @@ -146,11 +146,10 @@ inline uint64_t hashData(const void* ptr, uint32_t len, uint64_t x = 0ULL) return h; } -} // anonymous namespace +} // anonymous namespace namespace rowgroup { - uint64_t hashRow(const rowgroup::Row& r, std::size_t lastCol) { uint64_t ret = 0; @@ -163,17 +162,15 @@ uint64_t hashRow(const rowgroup::Row& r, std::size_t lastCol) { switch (r.getColType(i)) { - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::BLOB: - case execplan::CalpontSystemCatalog::TEXT: - h.add(r.getCharset(i), r.getConstString(i)); - strHashUsed = true; - break; + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::BLOB: + case execplan::CalpontSystemCatalog::TEXT: + h.add(r.getCharset(i), r.getConstString(i)); + strHashUsed = true; + break; - default: - ret = hashData(r.getData() + r.getOffset(i), r.getColumnWidth(i), ret); - break; + default: ret = hashData(r.getData() + r.getOffset(i), r.getColumnWidth(i), ret); break; } } @@ -194,59 +191,113 @@ struct LRUIface virtual ~LRUIface() = default; /** @brief Put an ID to cache or set it as last used */ - virtual void add(uint64_t) {} + virtual void add(uint64_t) + { + } /** @brief Remove an ID from cache */ - virtual void remove(uint64_t) {} + virtual void remove(uint64_t) + { + } /** @brief Get iterator of the most recently used ID */ - virtual List::const_reverse_iterator begin() const { return List::const_reverse_iterator(); } + virtual List::const_reverse_iterator begin() const + { + return List::const_reverse_iterator(); + } /** @brief Get iterator after the latest ID */ - virtual List::const_reverse_iterator end() const { return List::const_reverse_iterator(); } + virtual List::const_reverse_iterator end() const + { + return List::const_reverse_iterator(); + } /** @brief Get iterator of the latest ID */ - virtual List::const_iterator rbegin() const { return {}; } + virtual List::const_iterator rbegin() const + { + return {}; + } /** @brief Get iterator after the most recently used ID */ - virtual List::const_iterator rend() const { return {}; } + virtual List::const_iterator rend() const + { + return {}; + } - virtual void clear() {} - virtual std::size_t size() const { return 0; } - virtual bool empty() const { return true; } - virtual LRUIface* clone() const { return new LRUIface(); } + virtual void clear() + { + } + virtual std::size_t size() const + { + return 0; + } + virtual bool empty() const + { + return true; + } + virtual LRUIface* clone() const + { + return new LRUIface(); + } }; struct LRU : public LRUIface { - ~LRU() override { + ~LRU() override + { fMap.clear(); fList.clear(); } - inline void add(uint64_t rgid) final { + inline void add(uint64_t rgid) final + { auto it = fMap.find(rgid); - if (it != fMap.end()) { + if (it != fMap.end()) + { fList.erase(it->second); } fMap[rgid] = fList.insert(fList.end(), rgid); } - inline void remove(uint64_t rgid) final { + inline void remove(uint64_t rgid) final + { auto it = fMap.find(rgid); - if (UNLIKELY(it != fMap.end())) { + if (UNLIKELY(it != fMap.end())) + { fList.erase(it->second); fMap.erase(it); } } - inline List::const_reverse_iterator begin() const final { return fList.crbegin(); } - inline List::const_reverse_iterator end() const final { return fList.crend(); } - inline List::const_iterator rbegin() const final { return fList.cbegin(); } - inline List::const_iterator rend() const final { return fList.cend(); } - inline void clear() final { + inline List::const_reverse_iterator begin() const final + { + return fList.crbegin(); + } + inline List::const_reverse_iterator end() const final + { + return fList.crend(); + } + inline List::const_iterator rbegin() const final + { + return fList.cbegin(); + } + inline List::const_iterator rend() const final + { + return fList.cend(); + } + inline void clear() final + { fMap.clear(); fList.clear(); } - size_t size() const final { return fMap.size(); } - bool empty() const final { return fList.empty(); } + size_t size() const final + { + return fMap.size(); + } + bool empty() const final + { + return fList.empty(); + } - LRUIface* clone() const final { return new LRU(); } + LRUIface* clone() const final + { + return new LRU(); + } robin_hood::unordered_flat_map fMap; List fList; @@ -255,9 +306,12 @@ struct LRU : public LRUIface /** @brief Some service wrapping around ResourceManager (or NoOP) */ class MemManager { -public: - MemManager() {} - virtual ~MemManager() { + public: + MemManager() + { + } + virtual ~MemManager() + { release(fMemUsed); } @@ -275,31 +329,55 @@ public: releaseImpl(amount); } - ssize_t getUsed() const { return fMemUsed; } + ssize_t getUsed() const + { + return fMemUsed; + } virtual int64_t getFree() const { return std::numeric_limits::max(); } - virtual bool isStrict() const { return false; } + virtual bool isStrict() const + { + return false; + } - virtual MemManager* clone() const { return new MemManager(); } + virtual MemManager* clone() const + { + return new MemManager(); + } - virtual joblist::ResourceManager* getResourceManaged() { return nullptr; } - virtual boost::shared_ptr getSessionLimit() { return {}; } + virtual joblist::ResourceManager* getResourceManaged() + { + return nullptr; + } + virtual boost::shared_ptr getSessionLimit() + { + return {}; + } -protected: - virtual bool acquireImpl(std::size_t amount) { fMemUsed += amount; return true; } - virtual void releaseImpl(std::size_t amount) { fMemUsed -= amount; } + protected: + virtual bool acquireImpl(std::size_t amount) + { + fMemUsed += amount; + return true; + } + virtual void releaseImpl(std::size_t amount) + { + fMemUsed -= amount; + } ssize_t fMemUsed = 0; }; class RMMemManager : public MemManager { -public: - RMMemManager(joblist::ResourceManager* rm, boost::shared_ptr sl, bool wait = true, bool strict = true) - : fRm(rm), fSessLimit(std::move(sl)), fWait(wait), fStrict(strict) - {} + public: + RMMemManager(joblist::ResourceManager* rm, boost::shared_ptr sl, bool wait = true, + bool strict = true) + : fRm(rm), fSessLimit(std::move(sl)), fWait(wait), fStrict(strict) + { + } ~RMMemManager() override { @@ -312,17 +390,26 @@ public: return std::min(fRm->availableMemory(), *fSessLimit); } - bool isStrict() const final { return fStrict; } + bool isStrict() const final + { + return fStrict; + } MemManager* clone() const final { return new RMMemManager(fRm, fSessLimit, fWait, fStrict); } - joblist::ResourceManager* getResourceManaged() override { return fRm; } - boost::shared_ptr getSessionLimit() override { return fSessLimit; } + joblist::ResourceManager* getResourceManaged() override + { + return fRm; + } + boost::shared_ptr getSessionLimit() override + { + return fSessLimit; + } -protected: + protected: bool acquireImpl(size_t amount) final { if (amount) @@ -332,11 +419,11 @@ protected: return false; } MemManager::acquireImpl(amount); - } + } return true; } - void releaseImpl(size_t amount) override + void releaseImpl(size_t amount) override { if (amount) { @@ -345,21 +432,21 @@ protected: } } -private: + private: joblist::ResourceManager* fRm = nullptr; boost::shared_ptr fSessLimit; const bool fWait; const bool fStrict; }; -class Dumper { -public: - Dumper(const compress::CompressInterface* comp, MemManager* mm) - : fCompressor(comp) - , fMM(mm->clone()) - {} +class Dumper +{ + public: + Dumper(const compress::CompressInterface* comp, MemManager* mm) : fCompressor(comp), fMM(mm->clone()) + { + } - int write(const std::string &fname, const char *buf, size_t sz) + int write(const std::string& fname, const char* buf, size_t sz) { if (sz == 0) return 0; @@ -368,24 +455,27 @@ public: if (UNLIKELY(fd < 0)) return errno; - const char *tmpbuf; - if (fCompressor) { + const char* tmpbuf; + if (fCompressor) + { auto len = fCompressor->maxCompressedSize(sz); checkBuffer(len); fCompressor->compress(buf, sz, fTmpBuf.data(), &len); tmpbuf = fTmpBuf.data(); sz = len; - } - else + } + else { tmpbuf = buf; } auto to_write = sz; int ret = 0; - while (to_write > 0) { + while (to_write > 0) + { auto r = ::write(fd, tmpbuf + sz - to_write, to_write); - if (UNLIKELY(r < 0)) { + if (UNLIKELY(r < 0)) + { if (errno == EAGAIN) continue; @@ -401,28 +491,36 @@ public: return ret; } - int read(const std::string &fname, std::vector &buf) { + int read(const std::string& fname, std::vector& buf) + { int fd = open(fname.c_str(), O_RDONLY); if (UNLIKELY(fd < 0)) return errno; - struct stat st{}; + struct stat st + { + }; fstat(fd, &st); size_t sz = st.st_size; std::vector* tmpbuf; - if (fCompressor) { + if (fCompressor) + { tmpbuf = &fTmpBuf; checkBuffer(sz); - } else { + } + else + { tmpbuf = &buf; buf.resize(sz); } auto to_read = sz; int ret = 0; - while (to_read > 0) { + while (to_read > 0) + { auto r = ::read(fd, tmpbuf->data() + sz - to_read, to_read); - if (UNLIKELY(r < 0)) { + if (UNLIKELY(r < 0)) + { if (errno == EAGAIN) continue; @@ -435,9 +533,11 @@ public: to_read -= r; } - if (fCompressor) { + if (fCompressor) + { size_t len; - if (!fCompressor->getUncompressedSize(tmpbuf->data(), sz, &len)) { + if (!fCompressor->getUncompressedSize(tmpbuf->data(), sz, &len)) + { ret = EPROTO; close(fd); return ret; @@ -451,13 +551,16 @@ public: return ret; } - size_t size() const { + size_t size() const + { return fTmpBuf.size(); } -private: - void checkBuffer(size_t len) { - if (fTmpBuf.size() < len) { + private: + void checkBuffer(size_t len) + { + if (fTmpBuf.size() < len) + { size_t newtmpsz = (len + 8191) / 8192 * 8192; std::vector tmpvec(newtmpsz); fMM->acquire(newtmpsz - fTmpBuf.size()); @@ -465,7 +568,7 @@ private: } } -private: + private: const compress::CompressInterface* fCompressor; std::unique_ptr fMM; std::vector fTmpBuf; @@ -475,10 +578,10 @@ private: */ class RowGroupStorage { -public: + public: using RGDataStorage = std::vector>; -public: + public: /** @brief Default constructor * * @param tmpDir(in) directory for tmp data @@ -493,20 +596,15 @@ public: * false -> deal with it later * @param compressor pointer to CompressInterface impl or nullptr */ - RowGroupStorage(const std::string& tmpDir, - RowGroup* rowGroupOut, - size_t maxRows, - joblist::ResourceManager* rm = nullptr, - boost::shared_ptr sessLimit = {}, - bool wait = false, - bool strict = false, - compress::CompressInterface* compressor = nullptr) - : fRowGroupOut(rowGroupOut) - , fMaxRows(maxRows) - , fRGDatas() - , fUniqId(this) - , fTmpDir(tmpDir) - , fCompressor(compressor) + RowGroupStorage(const std::string& tmpDir, RowGroup* rowGroupOut, size_t maxRows, + joblist::ResourceManager* rm = nullptr, boost::shared_ptr sessLimit = {}, + bool wait = false, bool strict = false, compress::CompressInterface* compressor = nullptr) + : fRowGroupOut(rowGroupOut) + , fMaxRows(maxRows) + , fRGDatas() + , fUniqId(this) + , fTmpDir(tmpDir) + , fCompressor(compressor) { if (rm) { @@ -549,7 +647,10 @@ public: * * @param o RowGroupStorage to take from */ - void append(std::unique_ptr o) { return append(o.get()); } + void append(std::unique_ptr o) + { + return append(o.get()); + } void append(RowGroupStorage* o) { std::unique_ptr rgd; @@ -564,9 +665,9 @@ public: int64_t memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); if (!fMM->acquire(memSz)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), - logging::ERR_AGGREGATION_TOO_BIG); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), + logging::ERR_AGGREGATION_TOO_BIG); } fLRU->add(rgid); @@ -681,9 +782,9 @@ public: auto memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); if (!fMM->acquire(memSz)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), - logging::ERR_AGGREGATION_TOO_BIG); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), + logging::ERR_AGGREGATION_TOO_BIG); } auto* curRG = new RGData(*fRowGroupOut, fMaxRows); fRowGroupOut->setData(curRG); @@ -716,9 +817,9 @@ public: int64_t memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); if (!fMM->acquire(memSz)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), - logging::ERR_AGGREGATION_TOO_BIG); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), + logging::ERR_AGGREGATION_TOO_BIG); } auto* curRG = new RGData(*fRowGroupOut, fMaxRows); fRowGroupOut->setData(curRG); @@ -801,8 +902,7 @@ public: auto memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); if (!fMM->acquire(memSz)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), logging::ERR_AGGREGATION_TOO_BIG); } fCurRgid = 0; @@ -813,13 +913,12 @@ public: void dumpFinalizedInfo() const { auto fname = makeFinalizedFilename(); - int fd = open(fname.c_str(), - O_WRONLY | O_CREAT | O_TRUNC, 0644); + int fd = open(fname.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); if (UNLIKELY(fd < 0)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), + logging::ERR_DISKAGG_FILEIO_ERROR); } uint64_t sz = fRGDatas.size(); uint64_t finsz = fFinalizedRows.size(); @@ -831,9 +930,9 @@ public: { close(fd); unlink(fname.c_str()); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } close(fd); } @@ -845,9 +944,9 @@ public: int fd = open(fname.c_str(), O_RDONLY); if (fd < 0) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), + logging::ERR_DISKAGG_FILEIO_ERROR); } uint64_t sz; uint64_t finsz; @@ -857,9 +956,9 @@ public: { close(fd); unlink(fname.c_str()); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } fRGDatas.resize(sz); fFinalizedRows.resize(finsz); @@ -867,9 +966,9 @@ public: { close(fd); unlink(fname.c_str()); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } close(fd); } @@ -943,8 +1042,7 @@ public: prefixes.emplace_back(buf); } -private: - + private: /** @brief Get next available RGData and fill filename of the dump if it's * not in the memory. * @@ -1075,9 +1173,7 @@ private: const size_t rowsz = fRowGroupOut->getRowSize(); const size_t hdrsz = RowGroup::getHeaderSize(); uint8_t* data = rgdata->rowData.get() + hdrsz; - memmove(data + to * rowsz, - data + from * rowsz, - numRows * rowsz); + memmove(data + to * rowsz, data + from * rowsz, numRows * rowsz); } /** @brief Load RGData from disk dump. @@ -1104,9 +1200,9 @@ private: if ((errNo = fDumper->read(fname, data)) != 0) { unlink(fname.c_str()); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } messageqcpp::ByteStream bs(reinterpret_cast(data.data()), data.size()); @@ -1121,8 +1217,7 @@ private: if (!fMM->acquire(memSz)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), logging::ERR_AGGREGATION_TOO_BIG); } } @@ -1158,9 +1253,9 @@ private: int errNo; if ((errNo = fDumper->write(makeRGFilename(rgid), (char*)bs.buf(), bs.length())) != 0) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } } @@ -1186,20 +1281,19 @@ private: std::string makeRGFilename(uint64_t rgid) const { char buf[PATH_MAX]; - snprintf(buf, sizeof(buf), "%s/Agg-p%u-t%p-rg%lu-g%u", - fTmpDir.c_str(), getpid(), fUniqId, rgid, fGeneration); + snprintf(buf, sizeof(buf), "%s/Agg-p%u-t%p-rg%lu-g%u", fTmpDir.c_str(), getpid(), fUniqId, rgid, + fGeneration); return buf; } std::string makeFinalizedFilename() const { char fname[PATH_MAX]; - snprintf(fname, sizeof(fname), "%s/AggFin-p%u-t%p-g%u", - fTmpDir.c_str(), getpid(), fUniqId, fGeneration); + snprintf(fname, sizeof(fname), "%s/AggFin-p%u-t%p-g%u", fTmpDir.c_str(), getpid(), fUniqId, fGeneration); return fname; } -private: + private: friend class RowAggStorage; RowGroup* fRowGroupOut{nullptr}; const size_t fMaxRows; @@ -1219,8 +1313,8 @@ private: /** @brief Internal data for the hashmap */ struct RowPosHash { - uint64_t hash; ///< Row hash - uint64_t idx; ///< index in the RowGroupStorage + uint64_t hash; ///< Row hash + uint64_t idx; ///< index in the RowGroupStorage }; /*** @@ -1229,7 +1323,7 @@ struct RowPosHash */ class RowPosHashStorage { -public: + public: /*** * @brief Default constructor * @@ -1241,15 +1335,10 @@ public: * @param sessLimit session memory limit * @param enableDiskAgg is disk aggregation enabled? */ - RowPosHashStorage(const std::string& tmpDir, - size_t size, - joblist::ResourceManager* rm, - boost::shared_ptr sessLimit, - bool enableDiskAgg, + RowPosHashStorage(const std::string& tmpDir, size_t size, joblist::ResourceManager* rm, + boost::shared_ptr sessLimit, bool enableDiskAgg, compress::CompressInterface* compressor) - : fUniqId(this) - , fTmpDir(tmpDir) - , fCompressor(compressor) + : fUniqId(this), fTmpDir(tmpDir), fCompressor(compressor) { if (rm) fMM.reset(new RMMemManager(rm, sessLimit, !enableDiskAgg, !enableDiskAgg)); @@ -1303,8 +1392,7 @@ public: */ void shiftUp(uint64_t startIdx, uint64_t insIdx) { - memmove(&fPosHashes[insIdx + 1], - &fPosHashes[insIdx], + memmove(&fPosHashes[insIdx + 1], &fPosHashes[insIdx], (startIdx - insIdx) * sizeof(decltype(fPosHashes)::value_type)); } @@ -1354,13 +1442,12 @@ public: if ((errNo = fDumper->write(makeDumpName(), (char*)fPosHashes.data(), sz)) != 0) { throw logging::IDBExcept( - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, - errorString(errNo)), + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), logging::ERR_DISKAGG_FILEIO_ERROR); } } -private: + private: RowPosHashStorage() = default; void init(size_t size) @@ -1368,8 +1455,7 @@ private: auto bkts = size + 0xFFUL; if (!fMM->acquire(bkts * sizeof(decltype(fPosHashes)::value_type))) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), logging::ERR_AGGREGATION_TOO_BIG); } fPosHashes.resize(bkts); @@ -1378,8 +1464,8 @@ private: std::string makeDumpName() const { char fname[PATH_MAX]; - snprintf(fname, sizeof(fname), "%s/Agg-PosHash-p%u-t%p-g%u", - fTmpDir.c_str(), getpid(), fUniqId, fGeneration); + snprintf(fname, sizeof(fname), "%s/Agg-PosHash-p%u-t%p-g%u", fTmpDir.c_str(), getpid(), fUniqId, + fGeneration); return fname; } @@ -1390,8 +1476,7 @@ private: if ((errNo = fDumper->read(makeDumpName(), data)) != 0) { throw logging::IDBExcept( - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, - errorString(errNo)), + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), logging::ERR_DISKAGG_FILEIO_ERROR); } @@ -1399,11 +1484,11 @@ private: memcpy(fPosHashes.data(), data.data(), data.size()); } -private: + private: std::unique_ptr fMM; std::vector fPosHashes; - uint16_t fGeneration{0}; ///< current aggregation generation - void* fUniqId; ///< uniq ID to make an uniq dump filename + uint16_t fGeneration{0}; ///< current aggregation generation + void* fUniqId; ///< uniq ID to make an uniq dump filename std::string fTmpDir; compress::CompressInterface* fCompressor; std::unique_ptr fDumper; @@ -1418,24 +1503,20 @@ private: * information (index and hash) in the vector of portions of the same size in * elements, that can be dumped on disk either. ----------------------------------------------------------------------------*/ -RowAggStorage::RowAggStorage(const std::string& tmpDir, - RowGroup *rowGroupOut, RowGroup *keysRowGroup, - uint32_t keyCount, - joblist::ResourceManager *rm, - boost::shared_ptr sessLimit, - bool enabledDiskAgg, - bool allowGenerations, +RowAggStorage::RowAggStorage(const std::string& tmpDir, RowGroup* rowGroupOut, RowGroup* keysRowGroup, + uint32_t keyCount, joblist::ResourceManager* rm, + boost::shared_ptr sessLimit, bool enabledDiskAgg, bool allowGenerations, compress::CompressInterface* compressor) - : fMaxRows(getMaxRows(enabledDiskAgg)) - , fExtKeys(rowGroupOut != keysRowGroup) - , fLastKeyCol(keyCount - 1) - , fUniqId(this) - , fAllowGenerations(allowGenerations) - , fEnabledDiskAggregation(enabledDiskAgg) - , fCompressor(compressor) - , fTmpDir(tmpDir) - , fRowGroupOut(rowGroupOut) - , fKeysRowGroup(keysRowGroup) + : fMaxRows(getMaxRows(enabledDiskAgg)) + , fExtKeys(rowGroupOut != keysRowGroup) + , fLastKeyCol(keyCount - 1) + , fUniqId(this) + , fAllowGenerations(allowGenerations) + , fEnabledDiskAggregation(enabledDiskAgg) + , fCompressor(compressor) + , fTmpDir(tmpDir) + , fRowGroupOut(rowGroupOut) + , fKeysRowGroup(keysRowGroup) { char suffix[PATH_MAX]; snprintf(suffix, sizeof(suffix), "/p%u-t%p/", getpid(), this); @@ -1453,12 +1534,12 @@ RowAggStorage::RowAggStorage(const std::string& tmpDir, fMM.reset(new MemManager()); fNumOfInputRGPerThread = 1; } - fStorage.reset(new RowGroupStorage(fTmpDir, rowGroupOut, 1, rm, sessLimit, - !enabledDiskAgg, !enabledDiskAgg, fCompressor.get())); + fStorage.reset(new RowGroupStorage(fTmpDir, rowGroupOut, 1, rm, sessLimit, !enabledDiskAgg, !enabledDiskAgg, + fCompressor.get())); if (fExtKeys) { - fRealKeysStorage.reset(new RowGroupStorage(fTmpDir, keysRowGroup, 1, rm, sessLimit, - !enabledDiskAgg, !enabledDiskAgg, fCompressor.get())); + fRealKeysStorage.reset(new RowGroupStorage(fTmpDir, keysRowGroup, 1, rm, sessLimit, !enabledDiskAgg, + !enabledDiskAgg, fCompressor.get())); fKeysStorage = fRealKeysStorage.get(); } else @@ -1468,7 +1549,8 @@ RowAggStorage::RowAggStorage(const std::string& tmpDir, fKeysStorage->initRow(fKeyRow); fGens.emplace_back(new Data); fCurData = fGens.back().get(); - fCurData->fHashes.reset(new RowPosHashStorage(fTmpDir, 0, rm, sessLimit, fEnabledDiskAggregation, fCompressor.get())); + fCurData->fHashes.reset( + new RowPosHashStorage(fTmpDir, 0, rm, sessLimit, fEnabledDiskAggregation, fCompressor.get())); } RowAggStorage::~RowAggStorage() @@ -1476,35 +1558,25 @@ RowAggStorage::~RowAggStorage() cleanupAll(); } -bool RowAggStorage::getTargetRow(const Row &row, Row &rowOut) +bool RowAggStorage::getTargetRow(const Row& row, Row& rowOut) { uint64_t hash = hashRow(row, fLastKeyCol); return getTargetRow(row, hash, rowOut); } -bool RowAggStorage::getTargetRow(const Row &row, uint64_t hash, Row &rowOut) +bool RowAggStorage::getTargetRow(const Row& row, uint64_t hash, Row& rowOut) { if (UNLIKELY(!fInitialized)) { fInitialized = true; - fStorage.reset(new RowGroupStorage(fTmpDir, - fRowGroupOut, - fMaxRows, - fMM->getResourceManaged(), - fMM->getSessionLimit(), - !fEnabledDiskAggregation, - !fEnabledDiskAggregation, - fCompressor.get())); + fStorage.reset(new RowGroupStorage(fTmpDir, fRowGroupOut, fMaxRows, fMM->getResourceManaged(), + fMM->getSessionLimit(), !fEnabledDiskAggregation, + !fEnabledDiskAggregation, fCompressor.get())); if (fExtKeys) { - fKeysStorage = new RowGroupStorage(fTmpDir, - fKeysRowGroup, - fMaxRows, - fMM->getResourceManaged(), - fMM->getSessionLimit(), - !fEnabledDiskAggregation, - !fEnabledDiskAggregation, - fCompressor.get()); + fKeysStorage = new RowGroupStorage(fTmpDir, fKeysRowGroup, fMaxRows, fMM->getResourceManaged(), + fMM->getSessionLimit(), !fEnabledDiskAggregation, + !fEnabledDiskAggregation, fCompressor.get()); } else { @@ -1549,7 +1621,7 @@ bool RowAggStorage::getTargetRow(const Row &row, uint64_t hash, Row &rowOut) uint16_t gen = fGeneration - 1; do { - auto *genData = fGens[gen].get(); + auto* genData = fGens[gen].get(); size_t gidx{}; uint32_t ginfo{}; rowHashToIdx(hash, ginfo, gidx, genData); @@ -1632,17 +1704,15 @@ void RowAggStorage::dump() // If the generations are allowed and there are less than half of // rowgroups in memory, then we start a new generation - if (fAllowGenerations && - fStorage->fLRU->size() < fStorage->fRGDatas.size() / 2 && - fStorage->fRGDatas.size() > 10) + if (fAllowGenerations && fStorage->fLRU->size() < fStorage->fRGDatas.size() / 2 && + fStorage->fRGDatas.size() > 10) { startNewGeneration(); } else if (!fAllowGenerations && freeMem < 0 && freeAttempts == 1) { // safety guard so aggregation couldn't eat all available memory - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_TOO_BIG), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_TOO_BIG), logging::ERR_DISKAGG_TOO_BIG); } } @@ -1709,9 +1779,11 @@ void RowAggStorage::freeData() void RowAggStorage::shiftUp(size_t startIdx, size_t insIdx) { auto idx = startIdx; - while (idx != insIdx) { + while (idx != insIdx) + { fCurData->fInfo[idx] = static_cast(fCurData->fInfo[idx - 1] + fCurData->fInfoInc); - if (UNLIKELY(fCurData->fInfo[idx] + fCurData->fInfoInc > 0xFF)) { + if (UNLIKELY(fCurData->fInfo[idx] + fCurData->fInfoInc > 0xFF)) + { fCurData->fMaxSize = 0; } --idx; @@ -1719,20 +1791,14 @@ void RowAggStorage::shiftUp(size_t startIdx, size_t insIdx) fCurData->fHashes->shiftUp(startIdx, insIdx); } -void RowAggStorage::rowToIdx(const Row &row, - uint32_t& info, - size_t& idx, - uint64_t& hash, +void RowAggStorage::rowToIdx(const Row& row, uint32_t& info, size_t& idx, uint64_t& hash, const Data* curData) const { hash = hashRow(row, fLastKeyCol); return rowHashToIdx(hash, info, idx, curData); } -void RowAggStorage::rowToIdx(const Row &row, - uint32_t& info, - size_t& idx, - uint64_t& hash) const +void RowAggStorage::rowToIdx(const Row& row, uint32_t& info, size_t& idx, uint64_t& hash) const { return rowToIdx(row, info, idx, hash, fCurData); } @@ -1751,8 +1817,7 @@ void RowAggStorage::increaseSize() if (fCurData->fSize * 2 < calcMaxSize(fCurData->fMask + 1)) { // something strange happens... - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_ERROR), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_ERROR), logging::ERR_DISKAGG_ERROR); } @@ -1768,8 +1833,7 @@ void RowAggStorage::increaseSize() } else { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), logging::ERR_AGGREGATION_TOO_BIG); } } @@ -1821,8 +1885,7 @@ void RowAggStorage::insertSwap(size_t oldIdx, RowPosHashStorage* oldHashes) { if (fCurData->fMaxSize == 0 && !tryIncreaseInfo()) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_ERROR), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_ERROR), logging::ERR_DISKAGG_ERROR); } @@ -1867,8 +1930,7 @@ void RowAggStorage::initData(size_t elems, const RowPosHashStorage* oldHashes) if (!fMM->acquire(bytes)) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_AGGREGATION_TOO_BIG), + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_AGGREGATION_TOO_BIG), logging::ERR_AGGREGATION_TOO_BIG); } fCurData->fHashes = oldHashes->clone(elems, fGeneration); @@ -1882,18 +1944,20 @@ void RowAggStorage::reserve(size_t c) { auto const minElementsAllowed = (std::max)(c, fCurData->fSize); auto newSize = INIT_SIZE; - while (calcMaxSize(newSize) < minElementsAllowed && newSize != 0) { + while (calcMaxSize(newSize) < minElementsAllowed && newSize != 0) + { newSize *= 2; } - if (UNLIKELY(newSize == 0)) { - throw logging::IDBExcept( - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_ERROR), - logging::ERR_DISKAGG_ERROR); + if (UNLIKELY(newSize == 0)) + { + throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_ERROR), + logging::ERR_DISKAGG_ERROR); } // only actually do anything when the new size is bigger than the old one. This prevents to // continuously allocate for each reserve() call. - if (newSize > fCurData->fMask + 1) { + if (newSize > fCurData->fMask + 1) + { rehashPowerOfTwo(newSize); } } @@ -1937,8 +2001,7 @@ std::string RowAggStorage::makeDumpFilename(int32_t gen) const { char fname[PATH_MAX]; uint16_t rgen = gen < 0 ? fGeneration : gen; - snprintf(fname, sizeof(fname), "%s/AggMap-p%u-t%p-g%u", - fTmpDir.c_str(), getpid(), fUniqId, rgen); + snprintf(fname, sizeof(fname), "%s/AggMap-p%u-t%p-g%u", fTmpDir.c_str(), getpid(), fUniqId, rgen); return fname; } @@ -1957,18 +2020,18 @@ void RowAggStorage::dumpInternalData() const int fd = open(makeDumpFilename().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); if (fd < 0) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), + logging::ERR_DISKAGG_FILEIO_ERROR); } int errNo; if ((errNo = writeData(fd, (const char*)bs.buf(), bs.length())) != 0) { close(fd); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } close(fd); } @@ -2024,7 +2087,7 @@ void RowAggStorage::finalize(std::function mergeFunc, Row& rowOut) std::unique_ptr prevRowStorage; std::unique_ptr prevRealKeyRowStorage; - RowGroupStorage *prevKeyRowStorage{nullptr}; + RowGroupStorage* prevKeyRowStorage{nullptr}; auto elems = calcSizeWithBuffer(fCurData->fMask + 1); @@ -2039,8 +2102,7 @@ void RowAggStorage::finalize(std::function mergeFunc, Row& rowOut) else prevKeyRowStorage = prevRowStorage.get(); - loadGeneration(prevGen, prevSize, prevMask, prevMaxSize, prevInfoInc, - prevInfoHashShift, prevInfo); + loadGeneration(prevGen, prevSize, prevMask, prevMaxSize, prevInfoInc, prevInfoHashShift, prevInfo); prevHashes = fCurData->fHashes->clone(prevMask + 1, prevGen, true); // iterate over current generation rows @@ -2065,8 +2127,7 @@ void RowAggStorage::finalize(std::function mergeFunc, Row& rowOut) } // now try to find row in the previous generation - uint32_t pinfo = - prevInfoInc + static_cast((pos.hash & INFO_MASK) >> prevInfoHashShift); + uint32_t pinfo = prevInfoInc + static_cast((pos.hash & INFO_MASK) >> prevInfoHashShift); uint64_t pidx = (pos.hash >> INIT_INFO_BITS) & prevMask; while (pinfo < prevInfo[pidx]) { @@ -2094,7 +2155,7 @@ void RowAggStorage::finalize(std::function mergeFunc, Row& rowOut) } bool found = false; - auto &keyRow = fExtKeys ? fKeyRow : rowOut; + auto& keyRow = fExtKeys ? fKeyRow : rowOut; fKeysStorage->getRow(pos.idx, keyRow); while (!found && prevInfo[pidx] == pinfo) { @@ -2178,7 +2239,8 @@ void RowAggStorage::finalize(std::function mergeFunc, Row& rowOut) fCurData->fInfo = std::move(prevInfo); fCurData->fHashes = std::move(prevHashes); fStorage = std::move(prevRowStorage); - if (fExtKeys) { + if (fExtKeys) + { fRealKeysStorage = std::move(prevRealKeyRowStorage); fKeysStorage = fRealKeysStorage.get(); } @@ -2205,26 +2267,25 @@ void RowAggStorage::finalize(std::function mergeFunc, Row& rowOut) void RowAggStorage::loadGeneration(uint16_t gen) { - loadGeneration(gen, fCurData->fSize, fCurData->fMask, fCurData->fMaxSize, fCurData->fInfoInc, fCurData->fInfoHashShift, fCurData->fInfo); + loadGeneration(gen, fCurData->fSize, fCurData->fMask, fCurData->fMaxSize, fCurData->fInfoInc, + fCurData->fInfoHashShift, fCurData->fInfo); } -void RowAggStorage::loadGeneration(uint16_t gen, - size_t& size, - size_t& mask, - size_t& maxSize, - uint32_t& infoInc, - uint32_t& infoHashShift, +void RowAggStorage::loadGeneration(uint16_t gen, size_t& size, size_t& mask, size_t& maxSize, + uint32_t& infoInc, uint32_t& infoHashShift, std::unique_ptr& info) { messageqcpp::ByteStream bs; int fd = open(makeDumpFilename(gen).c_str(), O_RDONLY); if (fd < 0) { - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errno)), + logging::ERR_DISKAGG_FILEIO_ERROR); } - struct stat st{}; + struct stat st + { + }; fstat(fd, &st); bs.needAtLeast(st.st_size); bs.restart(); @@ -2232,9 +2293,9 @@ void RowAggStorage::loadGeneration(uint16_t gen, if ((errNo = readData(fd, (char*)bs.getInputPtr(), st.st_size)) != 0) { close(fd); - throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), - logging::ERR_DISKAGG_FILEIO_ERROR); + throw logging::IDBExcept( + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_DISKAGG_FILEIO_ERROR, errorString(errNo)), + logging::ERR_DISKAGG_FILEIO_ERROR); } close(fd); bs.advanceInputPtr(st.st_size); @@ -2278,12 +2339,8 @@ size_t RowAggStorage::getBucketSize() return 1 /* info byte */ + sizeof(RowPosHash); } -uint32_t calcNumberOfBuckets(ssize_t availMem, - uint32_t numOfThreads, - uint32_t numOfBuckets, - uint32_t groupsPerThread, - uint32_t inRowSize, - uint32_t outRowSize, +uint32_t calcNumberOfBuckets(ssize_t availMem, uint32_t numOfThreads, uint32_t numOfBuckets, + uint32_t groupsPerThread, uint32_t inRowSize, uint32_t outRowSize, bool enabledDiskAggr) { if (availMem < 0) @@ -2295,15 +2352,15 @@ uint32_t calcNumberOfBuckets(ssize_t availMem, ssize_t minNeededMemPerThread = groupsPerThread * inRowSize * RowAggStorage::getMaxRows(false); auto rowGroupSize = RowAggStorage::getMaxRows(enabledDiskAggr); - ssize_t minNeededMemPerBucket = outRowSize * rowGroupSize * 2 + - RowAggStorage::calcSizeWithBuffer(rowGroupSize, rowGroupSize) * RowAggStorage::getBucketSize(); + ssize_t minNeededMemPerBucket = + outRowSize * rowGroupSize * 2 + + RowAggStorage::calcSizeWithBuffer(rowGroupSize, rowGroupSize) * RowAggStorage::getBucketSize(); if ((availMem - minNeededMemPerThread * numOfThreads) / numOfBuckets < minNeededMemPerBucket) { ret = 0; if (availMem > minNeededMemPerThread * numOfThreads) { - ret = (availMem - minNeededMemPerThread * numOfThreads) / - minNeededMemPerBucket; + ret = (availMem - minNeededMemPerThread * numOfThreads) / minNeededMemPerBucket; } if (ret < numOfThreads) @@ -2315,4 +2372,4 @@ uint32_t calcNumberOfBuckets(ssize_t availMem, return ret == 0 ? 1 : ret; } -} // namespace rowgroup +} // namespace rowgroup diff --git a/utils/rowgroup/rowstorage.h b/utils/rowgroup/rowstorage.h index 6e5235bcf..df3ed8de7 100644 --- a/utils/rowgroup/rowstorage.h +++ b/utils/rowgroup/rowstorage.h @@ -26,13 +26,8 @@ namespace rowgroup { - -uint32_t calcNumberOfBuckets(ssize_t availMem, - uint32_t numOfThreads, - uint32_t numOfBuckets, - uint32_t groupsPerThread, - uint32_t inRowSize, - uint32_t outRowSize, +uint32_t calcNumberOfBuckets(ssize_t availMem, uint32_t numOfThreads, uint32_t numOfBuckets, + uint32_t groupsPerThread, uint32_t inRowSize, uint32_t outRowSize, bool enabledDiskAggr); class MemManager; @@ -44,29 +39,20 @@ uint64_t hashRow(const rowgroup::Row& r, std::size_t lastCol); class RowAggStorage { -public: - RowAggStorage(const std::string& tmpDir, - RowGroup* rowGroupOut, - RowGroup* keysRowGroup, - uint32_t keyCount, - joblist::ResourceManager* rm = nullptr, - boost::shared_ptr sessLimit = {}, - bool enabledDiskAgg = false, - bool allowGenerations = false, + public: + RowAggStorage(const std::string& tmpDir, RowGroup* rowGroupOut, RowGroup* keysRowGroup, uint32_t keyCount, + joblist::ResourceManager* rm = nullptr, boost::shared_ptr sessLimit = {}, + bool enabledDiskAgg = false, bool allowGenerations = false, compress::CompressInterface* compressor = nullptr); - RowAggStorage(const std::string& tmpDir, - RowGroup* rowGroupOut, - uint32_t keyCount, - joblist::ResourceManager* rm = nullptr, - boost::shared_ptr sessLimit = {}, - bool enabledDiskAgg = false, - bool allowGenerations = false, + RowAggStorage(const std::string& tmpDir, RowGroup* rowGroupOut, uint32_t keyCount, + joblist::ResourceManager* rm = nullptr, boost::shared_ptr sessLimit = {}, + bool enabledDiskAgg = false, bool allowGenerations = false, compress::CompressInterface* compressor = nullptr) - : RowAggStorage(tmpDir, rowGroupOut, rowGroupOut, keyCount, - rm, std::move(sessLimit), - enabledDiskAgg, allowGenerations, compressor) - {} + : RowAggStorage(tmpDir, rowGroupOut, rowGroupOut, keyCount, rm, std::move(sessLimit), enabledDiskAgg, + allowGenerations, compressor) + { + } ~RowAggStorage(); @@ -113,7 +99,7 @@ public: * @param mergeFunc * @param rowOut */ - void finalize(std::function mergeFunc, Row &rowOut); + void finalize(std::function mergeFunc, Row& rowOut); /** @brief Calculate maximum size of hash assuming 80% fullness. * @@ -138,7 +124,7 @@ public: return calcSizeWithBuffer(elems, calcMaxSize(elems)); } -private: + private: struct Data; /** @brief Create new RowAggStorage with the same params and load dumped data * @@ -311,13 +297,8 @@ private: */ void loadGeneration(uint16_t gen); /** @brief Load previously dumped data into the tmp storage */ - void loadGeneration(uint16_t gen, - size_t& size, - size_t& mask, - size_t& maxSize, - uint32_t& infoInc, - uint32_t& infoHashShift, - std::unique_ptr& info); + void loadGeneration(uint16_t gen, size_t& size, size_t& mask, size_t& maxSize, uint32_t& infoInc, + uint32_t& infoHashShift, std::unique_ptr& info); /** @brief Remove temporary data files */ void cleanup(); @@ -328,12 +309,12 @@ private: std::string makeDumpFilename(int32_t gen = -1) const; -private: - static constexpr size_t INIT_SIZE{sizeof(uint64_t)}; + private: + static constexpr size_t INIT_SIZE{sizeof(uint64_t)}; static constexpr uint32_t INIT_INFO_BITS{5}; - static constexpr uint8_t INIT_INFO_INC{1U << INIT_INFO_BITS}; - static constexpr size_t INFO_MASK{INIT_INFO_INC - 1U}; - static constexpr uint8_t INIT_INFO_HASH_SHIFT{0}; + static constexpr uint8_t INIT_INFO_INC{1U << INIT_INFO_BITS}; + static constexpr size_t INFO_MASK{INIT_INFO_INC - 1U}; + static constexpr uint8_t INIT_INFO_HASH_SHIFT{0}; static constexpr uint16_t MAX_INMEMORY_GENS{4}; struct Data @@ -373,6 +354,6 @@ private: rowgroup::RowGroup* fKeysRowGroup; }; -} // namespace rowgroup +} // namespace rowgroup -#endif // MYSQL_ROWSTORAGE_H +#endif // MYSQL_ROWSTORAGE_H diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index 64d413c52..73a9c5db2 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -60,53 +60,54 @@ namespace { using namespace rwlock; -//This mutex needs to be fully instantiated by the runtime static object +// This mutex needs to be fully instantiated by the runtime static object // init mechanism or the lock in makeRWLockShmImpl() will fail boost::mutex instanceMapMutex; typedef std::tr1::unordered_map LockMap_t; -//Windows doesn't init static objects the same as Linux, so make this a ptr +// Windows doesn't init static objects the same as Linux, so make this a ptr LockMap_t* lockMapPtr = 0; -} +} // namespace namespace rwlock { - #if defined(DEBUG) && !defined(_MSC_VER) #define RWLOCK_DEBUG #endif - #ifdef RWLOCK_DEBUG -#define PRINTSTATE() \ - cerr << " reading = " << fPImpl->fState->reading << endl \ - << " writing = " << fPImpl->fState->writing << endl \ - << " readerswaiting = " << fPImpl->fState->readerswaiting << endl \ - << " writerswaiting = " << fPImpl->fState->writerswaiting << endl +#define PRINTSTATE() \ + cerr << " reading = " << fPImpl->fState->reading << endl \ + << " writing = " << fPImpl->fState->writing << endl \ + << " readerswaiting = " << fPImpl->fState->readerswaiting << endl \ + << " writerswaiting = " << fPImpl->fState->writerswaiting << endl -#define CHECKSAFETY() \ - do { if (!\ - (\ - (fPImpl->fState->reading == 0 && \ - (fPImpl->fState->writing == 0 || fPImpl->fState->writing == 1)) || \ - (fPImpl->fState->reading > 0 && fPImpl->fState->writing == 0) \ - )) { \ - cerr << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": safety invariant violation" << endl; \ - PRINTSTATE(); \ - throw std::logic_error("RWLock: safety invariant violation"); \ - } } while (0) +#define CHECKSAFETY() \ + do \ + { \ + if (!((fPImpl->fState->reading == 0 && \ + (fPImpl->fState->writing == 0 || fPImpl->fState->writing == 1)) || \ + (fPImpl->fState->reading > 0 && fPImpl->fState->writing == 0))) \ + { \ + cerr << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": safety invariant violation" << endl; \ + PRINTSTATE(); \ + throw std::logic_error("RWLock: safety invariant violation"); \ + } \ + } while (0) -#define CHECKLIVENESS() \ - do { if (!( \ - (!(fPImpl->fState->readerswaiting > 0 || fPImpl->fState->writerswaiting > 0) || \ - (fPImpl->fState->reading > 0 || fPImpl->fState->writing > 0)) || \ - (!(fPImpl->fState->reading == 0 && fPImpl->fState->writing == 0) || \ - (fPImpl->fState->readerswaiting == 0 && fPImpl->fState->writerswaiting == 0)) \ - )) { \ - cerr << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": liveness invariant violation" << endl; \ - PRINTSTATE(); \ - throw std::logic_error("RWLock: liveness invariant violation"); \ - } } while (0) +#define CHECKLIVENESS() \ + do \ + { \ + if (!((!(fPImpl->fState->readerswaiting > 0 || fPImpl->fState->writerswaiting > 0) || \ + (fPImpl->fState->reading > 0 || fPImpl->fState->writing > 0)) || \ + (!(fPImpl->fState->reading == 0 && fPImpl->fState->writing == 0) || \ + (fPImpl->fState->readerswaiting == 0 && fPImpl->fState->writerswaiting == 0)))) \ + { \ + cerr << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": liveness invariant violation" << endl; \ + PRINTSTATE(); \ + throw std::logic_error("RWLock: liveness invariant violation"); \ + } \ + } while (0) #else #define PRINTSTATE() (void)0 #define CHECKSAFETY() (void)0 @@ -116,92 +117,95 @@ namespace rwlock /*static*/ RWLockShmImpl* RWLockShmImpl::makeRWLockShmImpl(int key, bool* excl) { - boost::mutex::scoped_lock lk(instanceMapMutex); - LockMap_t::iterator iter; + boost::mutex::scoped_lock lk(instanceMapMutex); + LockMap_t::iterator iter; - if (!lockMapPtr) - lockMapPtr = new LockMap_t(); + if (!lockMapPtr) + lockMapPtr = new LockMap_t(); - iter = lockMapPtr->find(key); + iter = lockMapPtr->find(key); - if (iter == lockMapPtr->end()) - { - RWLockShmImpl* ptr = 0; - bool bExcl = excl ? *excl : false; - ptr = new RWLockShmImpl(key, bExcl); - lockMapPtr->insert(make_pair(key, ptr)); - return ptr; - } - else if (excl) - { - *excl = false; // This isn't the first time for this lock. - } + if (iter == lockMapPtr->end()) + { + RWLockShmImpl* ptr = 0; + bool bExcl = excl ? *excl : false; + ptr = new RWLockShmImpl(key, bExcl); + lockMapPtr->insert(make_pair(key, ptr)); + return ptr; + } + else if (excl) + { + *excl = false; // This isn't the first time for this lock. + } - return iter->second; + return iter->second; } RWLockShmImpl::RWLockShmImpl(int key, bool excl) { - string keyName = BRM::ShmKeys::keyToName(key); - fKeyString = keyName; + string keyName = BRM::ShmKeys::keyToName(key); + fKeyString = keyName; + + try + { + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); + shm.truncate(sizeof(struct State)); + fStateShm.swap(shm); + bi::mapped_region region(fStateShm, bi::read_write); + fRegion.swap(region); + fState = static_cast(fRegion.get_address()); + fState->writerswaiting = 0; + fState->readerswaiting = 0; + fState->reading = 0; + + if (excl) + fState->writing = 1; + else + fState->writing = 0; + + new (&fState->sems[RWLock::MUTEX]) bi::interprocess_semaphore(1); + new (&fState->sems[RWLock::READERS]) bi::interprocess_semaphore(0); + new (&fState->sems[RWLock::WRITERS]) bi::interprocess_semaphore(0); + } + catch (bi::interprocess_exception& e) + { + if (e.get_error_code() == bi::security_error) + { + cerr << "RWLock: Failed to create the lock. Check perms on /dev/shm; should be 1777" << endl; + throw; + } + if (e.get_error_code() == bi::already_exists_error && excl) + throw not_excl(); + if (e.get_error_code() != bi::already_exists_error) + throw; try { - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); - shm.truncate(sizeof(struct State)); - fStateShm.swap(shm); - bi::mapped_region region(fStateShm, bi::read_write); - fRegion.swap(region); - fState = static_cast(fRegion.get_address()); - fState->writerswaiting = 0; - fState->readerswaiting = 0; - fState->reading = 0; - - if (excl) - fState->writing = 1; - else - fState->writing = 0; - - new (&fState->sems[RWLock::MUTEX]) bi::interprocess_semaphore(1); - new (&fState->sems[RWLock::READERS]) bi::interprocess_semaphore(0); - new (&fState->sems[RWLock::WRITERS]) bi::interprocess_semaphore(0); + bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); + fStateShm.swap(shm); } - catch (bi::interprocess_exception &e) + catch (exception& e) { - if (e.get_error_code() == bi::security_error) { - cerr << "RWLock: Failed to create the lock. Check perms on /dev/shm; should be 1777" << endl; - throw; - } - if (e.get_error_code() == bi::already_exists_error && excl) - throw not_excl(); - if (e.get_error_code() != bi::already_exists_error) - throw; - - try { - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fStateShm.swap(shm); - } - catch (exception &e) { - cerr << "RWLock failed to attach to the " << keyName << " shared mem segment, got " << e.what() << endl; - throw; - } - bi::mapped_region region(fStateShm, bi::read_write); - fRegion.swap(region); - fState = static_cast(fRegion.get_address()); - } - catch (...) - { - runtime_error rex("RWLockShmImpl::RWLockShmImpl(): caught unknown exception"); - cerr << rex.what() << endl; - throw rex; + cerr << "RWLock failed to attach to the " << keyName << " shared mem segment, got " << e.what() << endl; + throw; } + bi::mapped_region region(fStateShm, bi::read_write); + fRegion.swap(region); + fState = static_cast(fRegion.get_address()); + } + catch (...) + { + runtime_error rex("RWLockShmImpl::RWLockShmImpl(): caught unknown exception"); + cerr << rex.what() << endl; + throw rex; + } } RWLock::RWLock(int key, bool* excl) { - fPImpl = RWLockShmImpl::makeRWLockShmImpl(key, excl); + fPImpl = RWLockShmImpl::makeRWLockShmImpl(key, excl); } RWLock::~RWLock() @@ -210,540 +214,536 @@ RWLock::~RWLock() void RWLock::down(int num, bool block) { - again: - try + try + { + if (block) + fPImpl->fState->sems[num].wait(); + else { - if (block) - fPImpl->fState->sems[num].wait(); - else - { - bool gotIt = fPImpl->fState->sems[num].try_wait(); + bool gotIt = fPImpl->fState->sems[num].try_wait(); - if (!gotIt) - throw wouldblock(); - } + if (!gotIt) + throw wouldblock(); } - catch (bi::interprocess_exception& bipe) - { - ostringstream os; - os << "RWLock::down(): caught boost ipe: " << bipe.what() << " key = " << fPImpl->keyString() << - " error code = " << bipe.get_error_code(); + } + catch (bi::interprocess_exception& bipe) + { + ostringstream os; + os << "RWLock::down(): caught boost ipe: " << bipe.what() << " key = " << fPImpl->keyString() + << " error code = " << bipe.get_error_code(); - if (bipe.get_error_code() == 1) // it passes through EINTR apparently - goto again; + if (bipe.get_error_code() == 1) // it passes through EINTR apparently + goto again; - runtime_error rex(os.str()); - cerr << bipe.what() << endl; - throw rex; - } - catch (std::exception& ex) - { - cerr << + runtime_error rex(os.str()); + cerr << bipe.what() << endl; + throw rex; + } + catch (std::exception& ex) + { + cerr << #ifdef _MSC_VER - __FUNCTION__ << + __FUNCTION__ << #else - __PRETTY_FUNCTION__ << + __PRETTY_FUNCTION__ << #endif - ":" << __LINE__ << ": caught an exception: " << ex.what() << endl; - throw; - } - catch (...) - { - cerr << + ":" << __LINE__ << ": caught an exception: " << ex.what() << endl; + throw; + } + catch (...) + { + cerr << #ifdef _MSC_VER - __FUNCTION__ << + __FUNCTION__ << #else - __PRETTY_FUNCTION__ << + __PRETTY_FUNCTION__ << #endif - ":" << __LINE__ << ": caught an exception" << endl; - throw runtime_error("RWLock::down(): caught an exception"); - } + ":" << __LINE__ << ": caught an exception" << endl; + throw runtime_error("RWLock::down(): caught an exception"); + } } bool RWLock::timed_down(int num, const ptime& delay) { - bool gotTheLock = false; + bool gotTheLock = false; -// cout << "timed_down: current time = " << to_simple_string(microsec_clock::local_time()) << -// " wake time = " << to_simple_string(delay) << endl; + // cout << "timed_down: current time = " << to_simple_string(microsec_clock::local_time()) << + // " wake time = " << to_simple_string(delay) << endl; again: - try + try + { + // I don't think I've seen timed_wait() ever wait, need to do this 'manually' + // gotTheLock = fPImpl->fState->sems[num].timed_wait(delay); + do { - // I don't think I've seen timed_wait() ever wait, need to do this 'manually' - //gotTheLock = fPImpl->fState->sems[num].timed_wait(delay); - do - { - gotTheLock = fPImpl->fState->sems[num].try_wait(); + gotTheLock = fPImpl->fState->sems[num].try_wait(); - if (!gotTheLock) - usleep(100000); - } - while (!gotTheLock && microsec_clock::local_time() < delay); - } - catch (boost::thread_interrupted&) - { - // no need to do anything here - } - catch (bi::interprocess_exception& bipe) - { - ostringstream os; - os << "RWLock::timed_down(): caught boost ipe: " << bipe.what() << " key = " << fPImpl->keyString() << - " error code = " << bipe.get_error_code(); + if (!gotTheLock) + usleep(100000); + } while (!gotTheLock && microsec_clock::local_time() < delay); + } + catch (boost::thread_interrupted&) + { + // no need to do anything here + } + catch (bi::interprocess_exception& bipe) + { + ostringstream os; + os << "RWLock::timed_down(): caught boost ipe: " << bipe.what() << " key = " << fPImpl->keyString() + << " error code = " << bipe.get_error_code(); - if (bipe.get_error_code() == 1) // it passes through EINTR apparently - goto again; + if (bipe.get_error_code() == 1) // it passes through EINTR apparently + goto again; - runtime_error rex(os.str()); - cerr << bipe.what() << endl; - throw rex; - } - catch (std::exception& ex) - { - cerr << + runtime_error rex(os.str()); + cerr << bipe.what() << endl; + throw rex; + } + catch (std::exception& ex) + { + cerr << #ifdef _MSC_VER - __FUNCTION__ << + __FUNCTION__ << #else - __PRETTY_FUNCTION__ << + __PRETTY_FUNCTION__ << #endif - ":" << __LINE__ << ": caught an exception: " << ex.what() << endl; - throw; - } - catch (...) - { - cerr << + ":" << __LINE__ << ": caught an exception: " << ex.what() << endl; + throw; + } + catch (...) + { + cerr << #ifdef _MSC_VER - __FUNCTION__ << + __FUNCTION__ << #else - __PRETTY_FUNCTION__ << + __PRETTY_FUNCTION__ << #endif - ":" << __LINE__ << ": caught an exception" << endl; - throw runtime_error("RWLock::timed_down(): caught an exception"); - } + ":" << __LINE__ << ": caught an exception" << endl; + throw runtime_error("RWLock::timed_down(): caught an exception"); + } - return gotTheLock; + return gotTheLock; } void RWLock::up(int num) { - try - { - fPImpl->fState->sems[num].post(); - } - catch (bi::interprocess_exception& bipe) - { - ostringstream os; - os << "RWLock::up(): caught boost ipe: " << bipe.what() << " key = " << fPImpl->keyString(); - runtime_error rex(os.str()); - cerr << bipe.what() << endl; - throw rex; - } - catch (...) - { - throw runtime_error("RWLock::up(): caught an exception"); - } + try + { + fPImpl->fState->sems[num].post(); + } + catch (bi::interprocess_exception& bipe) + { + ostringstream os; + os << "RWLock::up(): caught boost ipe: " << bipe.what() << " key = " << fPImpl->keyString(); + runtime_error rex(os.str()); + cerr << bipe.what() << endl; + throw rex; + } + catch (...) + { + throw runtime_error("RWLock::up(): caught an exception"); + } } void RWLock::read_lock(bool block) { + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); + + if (fPImpl->fState->writerswaiting > 0 || fPImpl->fState->writing > 0) + { + if (!block) + { + up(MUTEX); + throw wouldblock(); + } + + fPImpl->fState->readerswaiting++; + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + down(READERS); // unblocked by write_unlock(); +#ifdef RWLOCK_DEBUG down(MUTEX, true); CHECKSAFETY(); CHECKLIVENESS(); - - if (fPImpl->fState->writerswaiting > 0 || fPImpl->fState->writing > 0) - { - if (!block) - { - up(MUTEX); - throw wouldblock(); - } - - fPImpl->fState->readerswaiting++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - down(READERS); //unblocked by write_unlock(); -#ifdef RWLOCK_DEBUG - down(MUTEX, true); - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); + up(MUTEX); #endif - } - else - { - fPImpl->fState->reading++; + } + else + { + fPImpl->fState->reading++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - } + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + } } void RWLock::read_lock_priority(bool block) { + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); + + if (fPImpl->fState->writing > 0) + { + if (!block) + { + up(MUTEX); + throw wouldblock(); + } + + fPImpl->fState->readerswaiting++; + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + down(READERS); // unblocked by write_unlock(); +#ifdef RWLOCK_DEBUG down(MUTEX, true); CHECKSAFETY(); CHECKLIVENESS(); - - if (fPImpl->fState->writing > 0) - { - if (!block) - { - up(MUTEX); - throw wouldblock(); - } - - fPImpl->fState->readerswaiting++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - down(READERS); //unblocked by write_unlock(); -#ifdef RWLOCK_DEBUG - down(MUTEX, true); - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); + up(MUTEX); #endif - } - else - { - fPImpl->fState->reading++; + } + else + { + fPImpl->fState->reading++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - } + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + } } void RWLock::read_unlock() { - down(MUTEX, true); - CHECKSAFETY(); - CHECKLIVENESS(); + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); - /* Made this a bit more tolerant of errors b/c the lock recovery code can technically - * be wrong. In the practically zero chance that happens, better to take the chance - * on a read race than a sure assertion failure - */ - if (fPImpl->fState->reading > 0) + /* Made this a bit more tolerant of errors b/c the lock recovery code can technically + * be wrong. In the practically zero chance that happens, better to take the chance + * on a read race than a sure assertion failure + */ + if (fPImpl->fState->reading > 0) + { + --fPImpl->fState->reading; + + if (fPImpl->fState->writerswaiting > 0 && fPImpl->fState->reading == 0) { - --fPImpl->fState->reading; - - if (fPImpl->fState->writerswaiting > 0 && fPImpl->fState->reading == 0) - { - --fPImpl->fState->writerswaiting; - fPImpl->fState->writing++; - up(WRITERS); - } + --fPImpl->fState->writerswaiting; + fPImpl->fState->writing++; + up(WRITERS); } + } - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); } void RWLock::write_lock(bool block) { - down(MUTEX, true); + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); + + if (fPImpl->fState->writing > 0 || fPImpl->fState->reading > 0) + { + if (!block) + { + up(MUTEX); + throw wouldblock(); + } + + fPImpl->fState->writerswaiting++; CHECKSAFETY(); CHECKLIVENESS(); - - if (fPImpl->fState->writing > 0 || fPImpl->fState->reading > 0) - { - if (!block) - { - up(MUTEX); - throw wouldblock(); - } - - fPImpl->fState->writerswaiting++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - down(WRITERS); //unblocked by write_unlock() or read_unlock() + up(MUTEX); + down(WRITERS); // unblocked by write_unlock() or read_unlock() #ifdef RWLOCK_DEBUG - down(MUTEX, true); - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); -#endif - } - else - { - fPImpl->fState->writing++; - - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - } -} - -// this exists only for the sake of code cleanup -#define RETURN_STATE(mutex_state, state) \ - if (state) { \ - state->mutexLocked = mutex_state; \ - state->readerswaiting = fPImpl->fState->readerswaiting; \ - state->reading = fPImpl->fState->reading; \ - state->writerswaiting = fPImpl->fState->writerswaiting; \ - state->writing = fPImpl->fState->writing; \ - } - -bool RWLock::timed_write_lock(const struct timespec& ts, struct LockState* state) -{ - bool gotIt, gotIt2; - ptime delay; - - delay = microsec_clock::local_time() + seconds(ts.tv_sec) + microsec(ts.tv_nsec / 1000); - - gotIt = timed_down(MUTEX, delay); - - if (!gotIt) - { - RETURN_STATE(true, state) - return false; - } - - CHECKSAFETY(); - CHECKLIVENESS(); - - if (fPImpl->fState->writing > 0 || fPImpl->fState->reading > 0) - { - fPImpl->fState->writerswaiting++; - - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - gotIt = timed_down(WRITERS, delay); - - if (!gotIt) - { - // need to grab the mutex again to revoke the lock request - // need to adjust the timeout value to make sure the attempt is made. - delay = microsec_clock::local_time() + seconds(10); - gotIt2 = timed_down(MUTEX, delay); - - if (!gotIt2) - { - RETURN_STATE(true, state); - return false; - } - - /* This thread could have been granted the write lock during that second - lock grab attempt. Observation: the thread that granted it up'd the writers sem, - but didn't wake this thread. If there were other writers waiting, one of those - woke. At this point, if writerswaiting > 0, this thread should consider itself - one that's still waiting and should back out of the request. - If writerswaiting == 0, this thread was the one granted the write lock, and - needs to take possession. - */ - if (fPImpl->fState->writerswaiting == 0) - { - try - { - down(WRITERS, false); - } - catch (const wouldblock&) - { - // Somehow another writer was able to jump in front. This is "impossible". - RETURN_STATE(false, state); - up(MUTEX); - return false; - } - - up(MUTEX); - return true; - } - - fPImpl->fState->writerswaiting--; - - // need to unblock whatever was blocked by this lock attempt - if (fPImpl->fState->writing == 0 && fPImpl->fState->writerswaiting == 0) - { - fPImpl->fState->reading += fPImpl->fState->readerswaiting; - - while (fPImpl->fState->readerswaiting > 0) - { - --fPImpl->fState->readerswaiting; - up(READERS); - } - } - - // else if there's an active writer, do nothing - - CHECKSAFETY(); - CHECKLIVENESS(); - RETURN_STATE(false, state); - up(MUTEX); - return false; - } - else - return true; - } - else - { - fPImpl->fState->writing++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - return true; - } -} - -void RWLock::write_unlock() -{ down(MUTEX, true); CHECKSAFETY(); CHECKLIVENESS(); - - assert(fPImpl->fState->writing > 0); - --fPImpl->fState->writing; - - if (fPImpl->fState->writerswaiting > 0) - { - --fPImpl->fState->writerswaiting; - fPImpl->fState->writing++; - up(WRITERS); - } - else if (fPImpl->fState->readerswaiting > 0) - { - //let up to state->readerswaiting readers go - - fPImpl->fState->reading = fPImpl->fState->readerswaiting; - - while (fPImpl->fState->readerswaiting > 0) - { - --fPImpl->fState->readerswaiting; - up(READERS); - } - } + up(MUTEX); +#endif + } + else + { + fPImpl->fState->writing++; CHECKSAFETY(); CHECKLIVENESS(); up(MUTEX); + } +} + +// this exists only for the sake of code cleanup +#define RETURN_STATE(mutex_state, state) \ + if (state) \ + { \ + state->mutexLocked = mutex_state; \ + state->readerswaiting = fPImpl->fState->readerswaiting; \ + state->reading = fPImpl->fState->reading; \ + state->writerswaiting = fPImpl->fState->writerswaiting; \ + state->writing = fPImpl->fState->writing; \ + } + +bool RWLock::timed_write_lock(const struct timespec& ts, struct LockState* state) +{ + bool gotIt, gotIt2; + ptime delay; + + delay = microsec_clock::local_time() + seconds(ts.tv_sec) + microsec(ts.tv_nsec / 1000); + + gotIt = timed_down(MUTEX, delay); + + if (!gotIt) + { + RETURN_STATE(true, state) + return false; + } + + CHECKSAFETY(); + CHECKLIVENESS(); + + if (fPImpl->fState->writing > 0 || fPImpl->fState->reading > 0) + { + fPImpl->fState->writerswaiting++; + + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + gotIt = timed_down(WRITERS, delay); + + if (!gotIt) + { + // need to grab the mutex again to revoke the lock request + // need to adjust the timeout value to make sure the attempt is made. + delay = microsec_clock::local_time() + seconds(10); + gotIt2 = timed_down(MUTEX, delay); + + if (!gotIt2) + { + RETURN_STATE(true, state); + return false; + } + + /* This thread could have been granted the write lock during that second + lock grab attempt. Observation: the thread that granted it up'd the writers sem, + but didn't wake this thread. If there were other writers waiting, one of those + woke. At this point, if writerswaiting > 0, this thread should consider itself + one that's still waiting and should back out of the request. + If writerswaiting == 0, this thread was the one granted the write lock, and + needs to take possession. + */ + if (fPImpl->fState->writerswaiting == 0) + { + try + { + down(WRITERS, false); + } + catch (const wouldblock&) + { + // Somehow another writer was able to jump in front. This is "impossible". + RETURN_STATE(false, state); + up(MUTEX); + return false; + } + + up(MUTEX); + return true; + } + + fPImpl->fState->writerswaiting--; + + // need to unblock whatever was blocked by this lock attempt + if (fPImpl->fState->writing == 0 && fPImpl->fState->writerswaiting == 0) + { + fPImpl->fState->reading += fPImpl->fState->readerswaiting; + + while (fPImpl->fState->readerswaiting > 0) + { + --fPImpl->fState->readerswaiting; + up(READERS); + } + } + + // else if there's an active writer, do nothing + + CHECKSAFETY(); + CHECKLIVENESS(); + RETURN_STATE(false, state); + up(MUTEX); + return false; + } + else + return true; + } + else + { + fPImpl->fState->writing++; + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + return true; + } +} + +void RWLock::write_unlock() +{ + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); + + assert(fPImpl->fState->writing > 0); + --fPImpl->fState->writing; + + if (fPImpl->fState->writerswaiting > 0) + { + --fPImpl->fState->writerswaiting; + fPImpl->fState->writing++; + up(WRITERS); + } + else if (fPImpl->fState->readerswaiting > 0) + { + // let up to state->readerswaiting readers go + + fPImpl->fState->reading = fPImpl->fState->readerswaiting; + + while (fPImpl->fState->readerswaiting > 0) + { + --fPImpl->fState->readerswaiting; + up(READERS); + } + } + + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); } void RWLock::upgrade_to_write() { + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); + /* Made this a bit more tolerant of errors b/c the lock recovery code can technically + * be wrong. In the practically zero chance that happens, better to take the chance + * on a read race than a sure assertion failure + */ + if (fPImpl->fState->reading > 0) + { + --fPImpl->fState->reading; + + // try to cut in line + // On entry we hold a read lock, so reading > 0, and at this point, reading >= 0 + if (fPImpl->fState->reading == 0) + { + fPImpl->fState->writing++; + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + return; + } + } + + // cut & paste from write_lock() + // The invariants hold that, on entry, writing == 0 and, by now, reading > 0, so this branch is always taken + if (fPImpl->fState->writing > 0 || fPImpl->fState->reading > 0) + { + fPImpl->fState->writerswaiting++; + + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + down(WRITERS, true); // unblocked by write_unlock() or read_unlock() +#ifdef RWLOCK_DEBUG down(MUTEX, true); CHECKSAFETY(); CHECKLIVENESS(); - - /* Made this a bit more tolerant of errors b/c the lock recovery code can technically - * be wrong. In the practically zero chance that happens, better to take the chance - * on a read race than a sure assertion failure - */ - if (fPImpl->fState->reading > 0) - { - --fPImpl->fState->reading; - - // try to cut in line - // On entry we hold a read lock, so reading > 0, and at this point, reading >= 0 - if (fPImpl->fState->reading == 0) - { - fPImpl->fState->writing++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - return; - } - } - - // cut & paste from write_lock() - // The invariants hold that, on entry, writing == 0 and, by now, reading > 0, so this branch is always taken - if (fPImpl->fState->writing > 0 || fPImpl->fState->reading > 0) - { - - fPImpl->fState->writerswaiting++; - - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - down(WRITERS, true); //unblocked by write_unlock() or read_unlock() -#ifdef RWLOCK_DEBUG - down(MUTEX, true); - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); + up(MUTEX); #endif - } - else // don't think we can get here: reading > 0 && writing == 0 && reading == 0 - { - fPImpl->fState->writing++; + } + else // don't think we can get here: reading > 0 && writing == 0 && reading == 0 + { + fPImpl->fState->writing++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); - } + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); + } } /* It's safe (and necessary) to simply convert this writer to a reader without blocking */ void RWLock::downgrade_to_read() { - down(MUTEX, true); - CHECKSAFETY(); - CHECKLIVENESS(); - assert(fPImpl->fState->writing > 0); - --fPImpl->fState->writing; + down(MUTEX, true); + CHECKSAFETY(); + CHECKLIVENESS(); + assert(fPImpl->fState->writing > 0); + --fPImpl->fState->writing; - if (fPImpl->fState->readerswaiting > 0) + if (fPImpl->fState->readerswaiting > 0) + { + // let up to state->readerswaiting readers go + fPImpl->fState->reading = fPImpl->fState->readerswaiting; + + while (fPImpl->fState->readerswaiting > 0) { - //let up to state->readerswaiting readers go - fPImpl->fState->reading = fPImpl->fState->readerswaiting; - - while (fPImpl->fState->readerswaiting > 0) - { - --fPImpl->fState->readerswaiting; - up(READERS); - } + --fPImpl->fState->readerswaiting; + up(READERS); } + } - fPImpl->fState->reading++; - CHECKSAFETY(); - CHECKLIVENESS(); - up(MUTEX); + fPImpl->fState->reading++; + CHECKSAFETY(); + CHECKLIVENESS(); + up(MUTEX); } void RWLock::reset() { - fPImpl->fState->writerswaiting = 0; - fPImpl->fState->readerswaiting = 0; - fPImpl->fState->reading = 0; - fPImpl->fState->writing = 0; + fPImpl->fState->writerswaiting = 0; + fPImpl->fState->readerswaiting = 0; + fPImpl->fState->reading = 0; + fPImpl->fState->writing = 0; - for (int i = 0; i < 3; i++) + for (int i = 0; i < 3; i++) + { + while (fPImpl->fState->sems[i].try_wait()) { - while (fPImpl->fState->sems[i].try_wait()) - { - } } + } - fPImpl->fState->sems[MUTEX].post(); + fPImpl->fState->sems[MUTEX].post(); } LockState RWLock::getLockState() { - bool gotIt; - LockState ret; + bool gotIt; + LockState ret; - gotIt = fPImpl->fState->sems[MUTEX].try_wait(); - ret.reading = fPImpl->fState->reading; - ret.writing = fPImpl->fState->writing; - ret.readerswaiting = fPImpl->fState->readerswaiting; - ret.writerswaiting = fPImpl->fState->writerswaiting; - ret.mutexLocked = !gotIt; + gotIt = fPImpl->fState->sems[MUTEX].try_wait(); + ret.reading = fPImpl->fState->reading; + ret.writing = fPImpl->fState->writing; + ret.readerswaiting = fPImpl->fState->readerswaiting; + ret.writerswaiting = fPImpl->fState->writerswaiting; + ret.mutexLocked = !gotIt; - if (gotIt) - fPImpl->fState->sems[MUTEX].post(); + if (gotIt) + fPImpl->fState->sems[MUTEX].post(); - return ret; + return ret; } -} //namespace rwlock +} // namespace rwlock // vim:ts=4 sw=4: - diff --git a/utils/rwlock/rwlock.h b/utils/rwlock/rwlock.h index 08b7f230b..e40305701 100644 --- a/utils/rwlock/rwlock.h +++ b/utils/rwlock/rwlock.h @@ -43,22 +43,21 @@ namespace rwlock { - /// the layout of the shmseg struct State { #ifdef _MSC_VER - volatile LONG writerswaiting; - volatile LONG writing; - volatile LONG readerswaiting; - volatile LONG reading; + volatile LONG writerswaiting; + volatile LONG writing; + volatile LONG readerswaiting; + volatile LONG reading; #else - volatile int writerswaiting; - volatile int writing; - volatile int readerswaiting; - volatile int reading; + volatile int writerswaiting; + volatile int writing; + volatile int readerswaiting; + volatile int reading; #endif - boost::interprocess::interprocess_semaphore sems[3]; + boost::interprocess::interprocess_semaphore sems[3]; }; /* the lock state without the semaphores, passed out by timed_write_lock() for @@ -67,57 +66,58 @@ class RWLockMonitor struct LockState { #ifdef _MSC_VER - LONG writerswaiting; - LONG writing; - LONG readerswaiting; - LONG reading; - bool mutexLocked; + LONG writerswaiting; + LONG writing; + LONG readerswaiting; + LONG reading; + bool mutexLocked; #else - int writerswaiting; - int writing; - int readerswaiting; - int reading; - bool mutexLocked; + int writerswaiting; + int writing; + int readerswaiting; + int reading; + bool mutexLocked; #endif }; class RWLockShmImpl { -public: - static RWLockShmImpl* makeRWLockShmImpl(int key, bool* excl = 0); + public: + static RWLockShmImpl* makeRWLockShmImpl(int key, bool* excl = 0); - boost::interprocess::shared_memory_object fStateShm; - boost::interprocess::mapped_region fRegion; - State* fState; + boost::interprocess::shared_memory_object fStateShm; + boost::interprocess::mapped_region fRegion; + State* fState; - std::string keyString() - { - return fKeyString; - } -private: - explicit RWLockShmImpl(int key, bool excl = false); - ~RWLockShmImpl(); - RWLockShmImpl(const RWLockShmImpl& rhs); - RWLockShmImpl& operator=(const RWLockShmImpl& rhs); - std::string fKeyString; + std::string keyString() + { + return fKeyString; + } + + private: + explicit RWLockShmImpl(int key, bool excl = false); + ~RWLockShmImpl(); + RWLockShmImpl(const RWLockShmImpl& rhs); + RWLockShmImpl& operator=(const RWLockShmImpl& rhs); + std::string fKeyString; }; class not_excl : public std::exception { -public: - virtual const char* what() const throw() - { - return "not_excl"; - } + public: + virtual const char* what() const throw() + { + return "not_excl"; + } }; class wouldblock : public std::exception { -public: - virtual const char* what() const throw() - { - return "wouldblock"; - } + public: + virtual const char* what() const throw() + { + return "wouldblock"; + } }; /** @brief Implements RW locks for use across threads & processes @@ -139,142 +139,140 @@ public: */ class RWLock { -public: + public: + // semaphore numbers + static const int MUTEX = 0; + static const int READERS = 1; + static const int WRITERS = 2; - // semaphore numbers - static const int MUTEX = 0; - static const int READERS = 1; - static const int WRITERS = 2; + /** @brief Keyed constructor. + * + * Instantiate an RWLock with the given key. All instances that + * share a key share the same lock. + * + * @param key The key + * @param excl If true and this is the first instance with the + * supplied key, it will return holding the write lock. If true and + * this is not the first instance, it will throw not_excl. The intent + * is similar to the IPC_EXCL flag in the sem/shm implementations. + */ + EXPORT explicit RWLock(int key, bool* excl = 0); - /** @brief Keyed constructor. - * - * Instantiate an RWLock with the given key. All instances that - * share a key share the same lock. - * - * @param key The key - * @param excl If true and this is the first instance with the - * supplied key, it will return holding the write lock. If true and - * this is not the first instance, it will throw not_excl. The intent - * is similar to the IPC_EXCL flag in the sem/shm implementations. - */ - EXPORT explicit RWLock(int key, bool* excl = 0); + EXPORT ~RWLock(); - EXPORT ~RWLock(); + /** @brief Grab a read lock + * + * Grab a read lock. This will block iff writers are waiting or + * a writer is active. The version with priority ignores any + * waiting threads and grabs the lock. + * + * @param block (For testing only) If false, will throw + * wouldblock instead of blocking + */ + EXPORT void read_lock(bool block = true); - /** @brief Grab a read lock - * - * Grab a read lock. This will block iff writers are waiting or - * a writer is active. The version with priority ignores any - * waiting threads and grabs the lock. - * - * @param block (For testing only) If false, will throw - * wouldblock instead of blocking - */ - EXPORT void read_lock(bool block = true); + EXPORT void read_lock_priority(bool block = true); - EXPORT void read_lock_priority(bool block = true); + /** @brief Release a read lock. + * + * Release a read lock. + */ + EXPORT void read_unlock(); - /** @brief Release a read lock. - * - * Release a read lock. - */ - EXPORT void read_unlock(); + /** @brief Grab a write lock + * + * Grab a write lock. This will block while another writer or reader is + * active and will have exclusive access on waking. + * + * @param block (For testing only) If false, will throw + * wouldblock instead of blocking + */ + EXPORT void write_lock(bool block = true); - /** @brief Grab a write lock - * - * Grab a write lock. This will block while another writer or reader is - * active and will have exclusive access on waking. - * - * @param block (For testing only) If false, will throw - * wouldblock instead of blocking - */ - EXPORT void write_lock(bool block = true); + /** @brief A timed write lock. + * + * Queues up for the write lock for a specified amount of time. Returns + * true if it got the lock, return false if it timed out first. + * If the timeout happens, it will also return the lock state if passed + * a non-NULL LockState struct. This is a specialization for supporting + * the RWLockMonitor class. + */ + EXPORT bool timed_write_lock(const struct timespec& ts, struct LockState* state = 0); - /** @brief A timed write lock. - * - * Queues up for the write lock for a specified amount of time. Returns - * true if it got the lock, return false if it timed out first. - * If the timeout happens, it will also return the lock state if passed - * a non-NULL LockState struct. This is a specialization for supporting - * the RWLockMonitor class. - */ - EXPORT bool timed_write_lock(const struct timespec& ts, - struct LockState* state = 0); + /** @brief Release a write lock. + * + * Release a write lock. + */ + EXPORT void write_unlock(); - /** @brief Release a write lock. - * - * Release a write lock. - */ - EXPORT void write_unlock(); + /* note: these haven't been proven yet */ - /* note: these haven't been proven yet */ + /** @brief Upgrade a read lock to a write lock + * + * Upgrade a read lock to a write lock. It may have to block + * if there are other readers currently reading. No guarantees of atomicity. + */ + EXPORT void upgrade_to_write(); - /** @brief Upgrade a read lock to a write lock - * - * Upgrade a read lock to a write lock. It may have to block - * if there are other readers currently reading. No guarantees of atomicity. - */ - EXPORT void upgrade_to_write(); + /** @brief Downgrade a write lock to a read lock + * + * Downgrade a write lock to a read lock. The conversion happens + * atomically. + */ + EXPORT void downgrade_to_read(); - /** @brief Downgrade a write lock to a read lock - * - * Downgrade a write lock to a read lock. The conversion happens - * atomically. - */ - EXPORT void downgrade_to_read(); + /** @brief Reset the lock's state (Use with caution!) + * + * If the lock gets into a bad state in testing or something, + * this will reset the state. + * @warning This is safe only if there are no other threads using this + * lock. + */ + EXPORT void reset(); - /** @brief Reset the lock's state (Use with caution!) - * - * If the lock gets into a bad state in testing or something, - * this will reset the state. - * @warning This is safe only if there are no other threads using this - * lock. - */ - EXPORT void reset(); + /* These are for white box testing only */ + inline void lock() + { + down(MUTEX, true); + } + inline void unlock() + { + up(MUTEX); + } + inline int getWriting() const + { + return fPImpl->fState->writing; + } + inline int getReading() const + { + return fPImpl->fState->reading; + } + inline int getWritersWaiting() const + { + return fPImpl->fState->writerswaiting; + } + inline int getReadersWaiting() const + { + return fPImpl->fState->readerswaiting; + } + LockState getLockState(); - /* These are for white box testing only */ - inline void lock() - { - down(MUTEX, true); - } - inline void unlock() - { - up(MUTEX); - } - inline int getWriting() const - { - return fPImpl->fState->writing; - } - inline int getReading() const - { - return fPImpl->fState->reading; - } - inline int getWritersWaiting() const - { - return fPImpl->fState->writerswaiting; - } - inline int getReadersWaiting() const - { - return fPImpl->fState->readerswaiting; - } - LockState getLockState(); + private: + RWLock(const RWLock& rwl); + RWLock& operator=(const RWLock& rwl); -private: - RWLock(const RWLock& rwl); - RWLock& operator=(const RWLock& rwl); + inline int getSemval(int) const + { + return 0; + } + void down(int num, bool block = true); + bool timed_down(int num, const boost::posix_time::ptime& ts); // to support timed_write_lock() + void up(int num); - inline int getSemval(int) const - { - return 0; - } - void down(int num, bool block = true); - bool timed_down(int num, const boost::posix_time::ptime& ts); // to support timed_write_lock() - void up(int num); - - RWLockShmImpl* fPImpl; + RWLockShmImpl* fPImpl; }; -} //namespace rwlock +} // namespace rwlock #undef EXPORT diff --git a/utils/rwlock/rwlock_local.cpp b/utils/rwlock/rwlock_local.cpp index a2752a351..43c88fb06 100644 --- a/utils/rwlock/rwlock_local.cpp +++ b/utils/rwlock/rwlock_local.cpp @@ -44,31 +44,31 @@ using namespace boost; #ifdef DEBUG using namespace std; -#define PRINTSTATE() \ - cerr << " reading = " << state.reading << endl \ - << " writing = " << state.writing << endl \ - << " readerswaiting = " << state.readerswaiting << endl \ - << " writerswaiting = " << state.writerswaiting << endl; +#define PRINTSTATE() \ + cerr << " reading = " << state.reading << endl \ + << " writing = " << state.writing << endl \ + << " readerswaiting = " << state.readerswaiting << endl \ + << " writerswaiting = " << state.writerswaiting << endl; -#define CHECKSAFETY() \ - if (!((state.reading == 0 && (state.writing == 0 || state.writing == 1)) || \ - (state.reading > 0 && state.writing == 0))) { \ - cerr << "RWLock_local::" << __func__ << ": safety invariant violation" << endl; \ - PRINTSTATE(); \ - throw std::logic_error("RWLock_local: safety invariant violation"); \ - } +#define CHECKSAFETY() \ + if (!((state.reading == 0 && (state.writing == 0 || state.writing == 1)) || \ + (state.reading > 0 && state.writing == 0))) \ + { \ + cerr << "RWLock_local::" << __func__ << ": safety invariant violation" << endl; \ + PRINTSTATE(); \ + throw std::logic_error("RWLock_local: safety invariant violation"); \ + } -#define CHECKLIVENESS() \ - if (!( \ - (!(state.readerswaiting > 0 || state.writerswaiting > 0) || \ - (state.reading > 0 || state.writing > 0)) || \ - (!(state.reading == 0 && state.writing == 0) || \ - (state.readerswaiting == 0 && state.writerswaiting == 0)) \ - )) { \ - cerr << "RWLock_local::" << __func__ << ": liveness invariant violation" << endl; \ - PRINTSTATE(); \ - throw std::logic_error("RWLock_local: liveness invariant violation"); \ - } +#define CHECKLIVENESS() \ + if (!((!(state.readerswaiting > 0 || state.writerswaiting > 0) || \ + (state.reading > 0 || state.writing > 0)) || \ + (!(state.reading == 0 && state.writing == 0) || \ + (state.readerswaiting == 0 && state.writerswaiting == 0)))) \ + { \ + cerr << "RWLock_local::" << __func__ << ": liveness invariant violation" << endl; \ + PRINTSTATE(); \ + throw std::logic_error("RWLock_local: liveness invariant violation"); \ + } #undef CHECKLIVENESS #define CHECKLIVENESS() @@ -77,13 +77,12 @@ using namespace std; namespace rwlock { - RWLock_local::RWLock_local() { - state.reading = 0; - state.readerswaiting = 0; - state.writing = 0; - state.writerswaiting = 0; + state.reading = 0; + state.readerswaiting = 0; + state.writing = 0; + state.writerswaiting = 0; } RWLock_local::~RWLock_local() @@ -92,147 +91,145 @@ RWLock_local::~RWLock_local() void RWLock_local::read_lock() { - mutex.lock(); + mutex.lock(); +#ifdef DEBUG + CHECKSAFETY(); + CHECKLIVENESS(); +#endif + + if (state.writerswaiting > 0 || state.writing > 0) + { + state.readerswaiting++; #ifdef DEBUG CHECKSAFETY(); CHECKLIVENESS(); #endif - if (state.writerswaiting > 0 || state.writing > 0) - { - state.readerswaiting++; -#ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); -#endif + while (state.writerswaiting > 0 || state.writing > 0) + okToRead.wait(mutex); - while (state.writerswaiting > 0 || state.writing > 0) - okToRead.wait(mutex); + state.readerswaiting--; + } - state.readerswaiting--; - } - - state.reading++; + state.reading++; #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - mutex.unlock(); + mutex.unlock(); } void RWLock_local::read_unlock() { - mutex.lock(); + mutex.lock(); #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - state.reading--; + state.reading--; - if (state.writerswaiting > 0 && state.reading == 0) - okToWrite.notify_one(); + if (state.writerswaiting > 0 && state.reading == 0) + okToWrite.notify_one(); #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - mutex.unlock(); + mutex.unlock(); } void RWLock_local::write_lock() { - mutex.lock(); + mutex.lock(); #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - if (state.writing > 0 || state.reading > 0) - { - state.writerswaiting++; - -#ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); -#endif - - while (state.writing > 0 || state.reading > 0) - okToWrite.wait(mutex); - - state.writerswaiting--; - } - - state.writing++; + if (state.writing > 0 || state.reading > 0) + { + state.writerswaiting++; #ifdef DEBUG CHECKSAFETY(); CHECKLIVENESS(); #endif + while (state.writing > 0 || state.reading > 0) + okToWrite.wait(mutex); + + state.writerswaiting--; + } + + state.writing++; + +#ifdef DEBUG + CHECKSAFETY(); + CHECKLIVENESS(); +#endif } void RWLock_local::write_unlock() { - #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - state.writing--; + state.writing--; - if (state.writerswaiting > 0) - okToWrite.notify_one(); - else if (state.readerswaiting > 0) - okToRead.notify_all(); + if (state.writerswaiting > 0) + okToWrite.notify_one(); + else if (state.readerswaiting > 0) + okToRead.notify_all(); #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - mutex.unlock(); + mutex.unlock(); } void RWLock_local::upgrade_to_write() { - mutex.lock(); + mutex.lock(); +#ifdef DEBUG + CHECKSAFETY(); + CHECKLIVENESS(); +#endif + state.reading--; + + // try to cut in line + if (state.reading == 0) + { + state.writing++; #ifdef DEBUG CHECKSAFETY(); CHECKLIVENESS(); #endif - state.reading--; + return; + } - // try to cut in line - if (state.reading == 0) - { - state.writing++; -#ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); -#endif - return; - } - - // cut & paste from write_lock() - if (state.writing > 0 || state.reading > 0) - { - state.writerswaiting++; + // cut & paste from write_lock() + if (state.writing > 0 || state.reading > 0) + { + state.writerswaiting++; #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - while (state.writing > 0 || state.reading > 0) - okToWrite.wait(mutex); + while (state.writing > 0 || state.reading > 0) + okToWrite.wait(mutex); - state.writerswaiting--; - } + state.writerswaiting--; + } - state.writing++; + state.writing++; } /* It's safe (and necessary) to simply convert this writer to a reader without @@ -240,88 +237,87 @@ void RWLock_local::upgrade_to_write() void RWLock_local::downgrade_to_read() { #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - state.writing--; + state.writing--; - if (state.readerswaiting > 0) - okToRead.notify_all(); + if (state.readerswaiting > 0) + okToRead.notify_all(); - state.reading++; + state.reading++; #ifdef DEBUG - CHECKSAFETY(); - CHECKLIVENESS(); + CHECKSAFETY(); + CHECKLIVENESS(); #endif - mutex.unlock(); + mutex.unlock(); } void RWLock_local::lock() { - mutex.lock(); + mutex.lock(); } void RWLock_local::unlock() { - mutex.unlock(); + mutex.unlock(); } int RWLock_local::getWriting() { - return state.writing; + return state.writing; } int RWLock_local::getReading() { - return state.reading; + return state.reading; } int RWLock_local::getWritersWaiting() { - return state.writerswaiting; + return state.writerswaiting; } int RWLock_local::getReadersWaiting() { - return state.readerswaiting; + return state.readerswaiting; } ScopedRWLock_local::ScopedRWLock_local(RWLock_local* l, rwlock_mode m) { - thelock = l; - mode = m; - assert(m == R || m == W); - locked = false; - lock(); + thelock = l; + mode = m; + assert(m == R || m == W); + locked = false; + lock(); } ScopedRWLock_local::~ScopedRWLock_local() { - if (locked) - unlock(); + if (locked) + unlock(); } void ScopedRWLock_local::lock() { - if (mode == R) - thelock->read_lock(); - else - thelock->write_lock(); + if (mode == R) + thelock->read_lock(); + else + thelock->write_lock(); - locked = true; + locked = true; } void ScopedRWLock_local::unlock() { - if (mode == R) - thelock->read_unlock(); - else - thelock->write_unlock(); - - locked = false; -} + if (mode == R) + thelock->read_unlock(); + else + thelock->write_unlock(); + locked = false; } +} // namespace rwlock diff --git a/utils/rwlock/rwlock_local.h b/utils/rwlock/rwlock_local.h index 833f01da5..751a6dc61 100644 --- a/utils/rwlock/rwlock_local.h +++ b/utils/rwlock/rwlock_local.h @@ -38,7 +38,6 @@ namespace rwlock { - /** @brief Implements RW locks for use across threads & processes * * Implements RW locks for use across threads & processes. Every @@ -57,145 +56,144 @@ namespace rwlock * eventually deadlock the set of processes that share the same key obviously. */ - class RWLock_local { -public: - - class not_excl : public std::exception + public: + class not_excl : public std::exception + { + public: + virtual const char* what() const throw() { - public: - virtual const char* what() const throw() - { - return "not_excl"; - } - }; + return "not_excl"; + } + }; - class wouldblock : public std::exception + class wouldblock : public std::exception + { + public: + virtual const char* what() const throw() { - public: - virtual const char* what() const throw() - { - return "wouldblock"; - } - }; + return "wouldblock"; + } + }; - /** @brief Keyed constructor. - * - * Instantiate an RWLock_local with the given key. All instances that - * share a key share the same lock. - * - * @param key The key - * @param excl If true and this is the first instance with the - * supplied key, it will return holding the write lock. If true and - * this is not the first instance, it will throw not_excl. The intent - * is similar to the IPC_EXCL flag in the sem/shm implementations. - */ - EXPORT RWLock_local(); + /** @brief Keyed constructor. + * + * Instantiate an RWLock_local with the given key. All instances that + * share a key share the same lock. + * + * @param key The key + * @param excl If true and this is the first instance with the + * supplied key, it will return holding the write lock. If true and + * this is not the first instance, it will throw not_excl. The intent + * is similar to the IPC_EXCL flag in the sem/shm implementations. + */ + EXPORT RWLock_local(); - EXPORT ~RWLock_local(); + EXPORT ~RWLock_local(); - /** @brief Grab a read lock - * - * Grab a read lock. This will block iff writers are waiting or - * a writer is active. - * - * @param block (For testing only) If false, will throw - * wouldblock instead of blocking - */ - EXPORT void read_lock(); + /** @brief Grab a read lock + * + * Grab a read lock. This will block iff writers are waiting or + * a writer is active. + * + * @param block (For testing only) If false, will throw + * wouldblock instead of blocking + */ + EXPORT void read_lock(); - /** @brief Release a read lock. - * - * Release a read lock. - */ - EXPORT void read_unlock(); + /** @brief Release a read lock. + * + * Release a read lock. + */ + EXPORT void read_unlock(); - /** @brief Grab a write lock - * - * Grab a write lock. This will block while another writer or reader is - * active and will have exclusive access on waking. - * - * @param block (For testing only) If false, will throw - * wouldblock instead of blocking - */ - EXPORT void write_lock(); + /** @brief Grab a write lock + * + * Grab a write lock. This will block while another writer or reader is + * active and will have exclusive access on waking. + * + * @param block (For testing only) If false, will throw + * wouldblock instead of blocking + */ + EXPORT void write_lock(); - /** @brief Release a write lock. - * - * Release a write lock. - */ - EXPORT void write_unlock(); + /** @brief Release a write lock. + * + * Release a write lock. + */ + EXPORT void write_unlock(); - /** @brief Upgrade a read lock to a write lock - * - * Upgrade a read lock to a write lock. It may have to block - * if there are other readers currently reading. No guarantees of atomicity. - */ - EXPORT void upgrade_to_write(); + /** @brief Upgrade a read lock to a write lock + * + * Upgrade a read lock to a write lock. It may have to block + * if there are other readers currently reading. No guarantees of atomicity. + */ + EXPORT void upgrade_to_write(); - /** @brief Downgrade a write lock to a read lock - * - * Downgrade a write lock to a read lock. The conversion happens - * atomically. - */ - EXPORT void downgrade_to_read(); + /** @brief Downgrade a write lock to a read lock + * + * Downgrade a write lock to a read lock. The conversion happens + * atomically. + */ + EXPORT void downgrade_to_read(); - /* These are for white box testing only */ - EXPORT void lock(); - EXPORT void unlock(); - EXPORT int getWriting(); - EXPORT int getReading(); - EXPORT int getWritersWaiting(); - EXPORT int getReadersWaiting(); - -private: - // Not copyable - RWLock_local(const RWLock_local& rwl); - RWLock_local& operator=(const RWLock_local& rwl); - - /// the layout of the shmseg - struct State - { - int writerswaiting, writing, readerswaiting, reading; - } state; - boost::mutex mutex; - boost::condition okToRead; - boost::condition okToWrite; + /* These are for white box testing only */ + EXPORT void lock(); + EXPORT void unlock(); + EXPORT int getWriting(); + EXPORT int getReading(); + EXPORT int getWritersWaiting(); + EXPORT int getReadersWaiting(); + private: + // Not copyable + RWLock_local(const RWLock_local& rwl); + RWLock_local& operator=(const RWLock_local& rwl); + /// the layout of the shmseg + struct State + { + int writerswaiting, writing, readerswaiting, reading; + } state; + boost::mutex mutex; + boost::condition okToRead; + boost::condition okToWrite; }; enum rwlock_mode { - R, - W + R, + W }; class ScopedRWLock_local { -public: - ScopedRWLock_local(RWLock_local*, rwlock_mode); - ~ScopedRWLock_local(); + public: + ScopedRWLock_local(RWLock_local*, rwlock_mode); + ~ScopedRWLock_local(); - void lock(); - void unlock(); + void lock(); + void unlock(); -private: - explicit ScopedRWLock_local() {} - explicit ScopedRWLock_local(const ScopedRWLock_local&) {} - ScopedRWLock_local& operator=(const ScopedRWLock_local&) - { - return *this; - } - RWLock_local* thelock; - rwlock_mode mode; - bool locked; + private: + explicit ScopedRWLock_local() + { + } + explicit ScopedRWLock_local(const ScopedRWLock_local&) + { + } + ScopedRWLock_local& operator=(const ScopedRWLock_local&) + { + return *this; + } + RWLock_local* thelock; + rwlock_mode mode; + bool locked; }; - #undef EXPORT -} +} // namespace rwlock #endif diff --git a/utils/rwlock/tdriver-rw.cpp b/utils/rwlock/tdriver-rw.cpp index 2e8430342..87fea40b5 100644 --- a/utils/rwlock/tdriver-rw.cpp +++ b/utils/rwlock/tdriver-rw.cpp @@ -46,255 +46,251 @@ int threadStop; static void* RWRunner(void* arg) { - struct timeval tv; - int op, op2, interval; - RWLock* rwlock; + struct timeval tv; + int op, op2, interval; + RWLock* rwlock; - gettimeofday(&tv, NULL); - rwlock = new RWLock(reinterpret_cast(arg)); + gettimeofday(&tv, NULL); + rwlock = new RWLock(reinterpret_cast(arg)); - while (!threadStop) + while (!threadStop) + { + op = rand_r(reinterpret_cast(&tv.tv_usec)) % 10; + + if (op < 8) // read { - op = rand_r(reinterpret_cast(&tv.tv_usec)) % 10; + interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; + rwlock->read_lock(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() > 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + rwlock->unlock(); + usleep(interval); - if (op < 8) // read - { - interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; - rwlock->read_lock(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() > 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - rwlock->unlock(); - usleep(interval); + op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; - op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; - - if (op2) - { - rwlock->upgrade_to_write(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); - rwlock->unlock(); - usleep(interval); - rwlock->write_unlock(); - } - else - { - /* For testing the lock recovery code in the BRM workernodes */ - /* - int crash = rand_r((uint32_t *) &tv.tv_usec) % 100; - if (crash > 0) // 1% chance of crashing - rwlock->read_unlock(); - */ - } - } - else if (op < 9) // write - { - interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; - rwlock->write_lock(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); - rwlock->unlock(); - usleep(interval); - - op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; - - if (op2) - { - rwlock->downgrade_to_read(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() > 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - rwlock->unlock(); - usleep(interval); + if (op2) + { + rwlock->upgrade_to_write(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + rwlock->unlock(); + usleep(interval); + rwlock->write_unlock(); + } + else + { + /* For testing the lock recovery code in the BRM workernodes */ + /* + int crash = rand_r((uint32_t *) &tv.tv_usec) % 100; + if (crash > 0) // 1% chance of crashing rwlock->read_unlock(); - } - else - - rwlock->write_unlock(); - } - else if (op == 9) // delete - { - delete rwlock; - rwlock = new RWLock(reinterpret_cast(arg)); - } + */ + } } + else if (op < 9) // write + { + interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; + rwlock->write_lock(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + rwlock->unlock(); + usleep(interval); - delete rwlock; - pthread_exit(0); + op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; + + if (op2) + { + rwlock->downgrade_to_read(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() > 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + rwlock->unlock(); + usleep(interval); + rwlock->read_unlock(); + } + else + + rwlock->write_unlock(); + } + else if (op == 9) // delete + { + delete rwlock; + rwlock = new RWLock(reinterpret_cast(arg)); + } + } + + delete rwlock; + pthread_exit(0); } static void* RWRunner_local(void* arg) { - struct timeval tv; - int op, op2, interval; - RWLock_local* rwlock = reinterpret_cast(arg); + struct timeval tv; + int op, op2, interval; + RWLock_local* rwlock = reinterpret_cast(arg); - gettimeofday(&tv, NULL); + gettimeofday(&tv, NULL); - while (!threadStop) + while (!threadStop) + { + op = rand_r(reinterpret_cast(&tv.tv_usec)) % 10; + + // cout << "doing op " << op << endl; + switch (op) { - op = rand_r(reinterpret_cast(&tv.tv_usec)) % 10; + case 0: // read + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + { + interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; + rwlock->read_lock(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() > 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + rwlock->unlock(); + usleep(interval); + op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; -// cout << "doing op " << op << endl; - switch (op) + if (op2) { - case 0: //read - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - case 8: - { - interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; - rwlock->read_lock(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() > 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - rwlock->unlock(); - usleep(interval); - op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; - - if (op2) - { - rwlock->upgrade_to_write(); -// rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); -// rwlock->unlock(); - usleep(interval); - rwlock->write_unlock(); - } - - break; - } - - case 9: //write - { - interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; - rwlock->write_lock(); -// rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); -// rwlock->unlock(); - usleep(interval); - op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; - - if (op2) - { - rwlock->downgrade_to_read(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() > 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - rwlock->unlock(); - usleep(interval); - rwlock->read_unlock(); - } - else - rwlock->write_unlock(); - - break; - } - - default: - break; + rwlock->upgrade_to_write(); + // rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + // rwlock->unlock(); + usleep(interval); + rwlock->write_unlock(); } + + break; + } + + case 9: // write + { + interval = rand_r(reinterpret_cast(&tv.tv_usec)) % 100000; + rwlock->write_lock(); + // rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + // rwlock->unlock(); + usleep(interval); + op2 = rand_r(reinterpret_cast(&tv.tv_usec)) % 2; + + if (op2) + { + rwlock->downgrade_to_read(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() > 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + rwlock->unlock(); + usleep(interval); + rwlock->read_unlock(); + } + else + rwlock->write_unlock(); + + break; + } + + default: break; } + } - pthread_exit(0); + pthread_exit(0); } - class RWLockTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(RWLockTest); - CPPUNIT_TEST_SUITE(RWLockTest); + CPPUNIT_TEST(LongRWTest_1); + // CPPUNIT_TEST(LongRWLocalTest_1); - CPPUNIT_TEST(LongRWTest_1); -//CPPUNIT_TEST(LongRWLocalTest_1); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void LongRWTest_1() + { + int key = 0x20000; // the extentmap key -private: -public: - void LongRWTest_1() + const int threadCount = 30; + int i; + pthread_t threads[threadCount]; + + cerr << endl + << "Multithreaded RWLock test. " + "This runs for 60 minutes." + << endl; + + threadStop = 0; + + for (i = 0; i < threadCount; i++) { - int key = 0x20000; // the extentmap key - - const int threadCount = 30; - int i; - pthread_t threads[threadCount]; - - cerr << endl << "Multithreaded RWLock test. " - "This runs for 60 minutes." << endl; - - threadStop = 0; - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, RWRunner, - reinterpret_cast(key)) < 0) - throw logic_error("Error creating threads for the ipc test"); - } - - sleep(3600); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + if (pthread_create(&threads[i], NULL, RWRunner, reinterpret_cast(key)) < 0) + throw logic_error("Error creating threads for the ipc test"); } - void LongRWLocalTest_1() + sleep(3600); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 40; - int i; - pthread_t threads[threadCount]; - RWLock_local rwlock; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } - cerr << endl << "Multithreaded RWLock_local test. " - "This runs for 30-60 seconds." << endl; + void LongRWLocalTest_1() + { + const int threadCount = 40; + int i; + pthread_t threads[threadCount]; + RWLock_local rwlock; - threadStop = 0; + cerr << endl + << "Multithreaded RWLock_local test. " + "This runs for 30-60 seconds." + << endl; - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, RWRunner_local, - reinterpret_cast(&rwlock)) < 0) - throw logic_error("Error creating threads for the local test"); - } + threadStop = 0; - sleep(30); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, RWRunner_local, reinterpret_cast(&rwlock)) < 0) + throw logic_error("Error creating threads for the local test"); } + sleep(30); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( RWLockTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(RWLockTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/utils/rwlock/tdriver.cpp b/utils/rwlock/tdriver.cpp index dee5ac886..ec93b59fa 100644 --- a/utils/rwlock/tdriver.cpp +++ b/utils/rwlock/tdriver.cpp @@ -40,143 +40,138 @@ using namespace std; class RWLockTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(RWLockTest); - CPPUNIT_TEST_SUITE(RWLockTest); + CPPUNIT_TEST(RWTest_1); - CPPUNIT_TEST(RWTest_1); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void RWTest_1() + { + RWLock* rwlock; + int caughtException = 0; + int key; -private: -public: - void RWTest_1() + srand(time(NULL)); + key = rand(); + + rwlock = new RWLock(key); + rwlock->read_lock(false); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 1); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + rwlock->read_lock(false); + rwlock->read_lock(false); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 3); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + + try { - RWLock* rwlock; - int caughtException = 0; - int key; - - srand(time(NULL)); - key = rand(); - - rwlock = new RWLock(key); - rwlock->read_lock(false); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 1); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - rwlock->read_lock(false); - rwlock->read_lock(false); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 3); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - - try - { - rwlock->write_lock(false); - } - catch (RWLock::wouldblock& e) - { - caughtException++; -// cerr << endl << "Caught expected exception: " << e.what() << endl; - } - - CPPUNIT_ASSERT(caughtException == 1); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 3); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - rwlock->read_unlock(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 2); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - rwlock->read_unlock(); - rwlock->read_unlock(); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 0); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - rwlock->write_lock(false); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - - try - { - rwlock->write_lock(false); - } - catch (RWLock::wouldblock& e) - { - caughtException++; -// cerr << endl << "Caught expected exception: " << e.what() << endl; - } - - CPPUNIT_ASSERT(caughtException == 2); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - - try - { - rwlock->read_lock(false); - } - catch (RWLock::wouldblock& e) - { - caughtException++; -// cerr << endl << "Caught expected exception: " << e.what() << endl; - } - - CPPUNIT_ASSERT(caughtException == 3); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - delete rwlock; - rwlock = new RWLock(key); - rwlock->lock(); - CPPUNIT_ASSERT(rwlock->getReading() == 0); - CPPUNIT_ASSERT(rwlock->getWriting() == 1); - CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); - CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); - rwlock->unlock(); - rwlock->write_unlock(); - delete rwlock; + rwlock->write_lock(false); + } + catch (RWLock::wouldblock& e) + { + caughtException++; + // cerr << endl << "Caught expected exception: " << e.what() << endl; } + CPPUNIT_ASSERT(caughtException == 1); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 3); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + rwlock->read_unlock(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 2); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + rwlock->read_unlock(); + rwlock->read_unlock(); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 0); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + rwlock->write_lock(false); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + try + { + rwlock->write_lock(false); + } + catch (RWLock::wouldblock& e) + { + caughtException++; + // cerr << endl << "Caught expected exception: " << e.what() << endl; + } + + CPPUNIT_ASSERT(caughtException == 2); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + + try + { + rwlock->read_lock(false); + } + catch (RWLock::wouldblock& e) + { + caughtException++; + // cerr << endl << "Caught expected exception: " << e.what() << endl; + } + + CPPUNIT_ASSERT(caughtException == 3); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + delete rwlock; + rwlock = new RWLock(key); + rwlock->lock(); + CPPUNIT_ASSERT(rwlock->getReading() == 0); + CPPUNIT_ASSERT(rwlock->getWriting() == 1); + CPPUNIT_ASSERT(rwlock->getReadersWaiting() == 0); + CPPUNIT_ASSERT(rwlock->getWritersWaiting() == 0); + rwlock->unlock(); + rwlock->write_unlock(); + delete rwlock; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( RWLockTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(RWLockTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 1c1c0e453..65437a9c1 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -17,9 +17,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ #include #include using namespace std; @@ -35,13 +35,12 @@ using namespace boost; using namespace config; -#include - +#include + using namespace std; namespace startup { - /* static */ boost::mutex StartUp::fTmpDirLock; /* static */ @@ -50,69 +49,67 @@ string* StartUp::fTmpDirp = 0; /* static */ const string StartUp::tmpDir() { - boost::mutex::scoped_lock lk(fTmpDirLock); + boost::mutex::scoped_lock lk(fTmpDirLock); - if (fTmpDirp) - return *fTmpDirp; + if (fTmpDirp) + return *fTmpDirp; #ifdef _MSC_VER - fTmpDirp = new string("C:\\Calpont\Tmp"); - string cfStr = IDBreadRegistry(""); + fTmpDirp = new string("C:\\Calpont\Tmp"); + string cfStr = IDBreadRegistry(""); - if (!cfStr.empty()) - *fTmpDirp = cfStr; - - return *fTmpDirp; + if (!cfStr.empty()) + *fTmpDirp = cfStr; + + return *fTmpDirp; #else - //check for non-root user -/* const char* p = getenv("HOME"); - string homedir = p; + // check for non-root user + /* const char* p = getenv("HOME"); + string homedir = p; - if (homedir == "/root") - { - Config* sysConfig = Config::makeConfig(); + if (homedir == "/root") + { + Config* sysConfig = Config::makeConfig(); - string TempFileDir; + string TempFileDir; - try - { - TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir"); - } - catch (...) - {} + try + { + TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir"); + } + catch (...) + {} - fTmpDirp = new string("/tmp"); - - *fTmpDirp = *fTmpDirp + TempFileDir; - } - else - { - // non-root user - fTmpDirp = new string(homedir); + fTmpDirp = new string("/tmp"); - *fTmpDirp = *fTmpDirp + "/.tmp"; - } -*/ - Config* sysConfig = Config::makeConfig(); + *fTmpDirp = *fTmpDirp + TempFileDir; + } + else + { + // non-root user + fTmpDirp = new string(homedir); - string TempFileDir; + *fTmpDirp = *fTmpDirp + "/.tmp"; + } + */ + Config* sysConfig = Config::makeConfig(); + + string TempFileDir; + + try + { + TempFileDir = sysConfig->getConfig("SystemConfig", "SystemTempFileDir"); + } + catch (...) + { + } + + return TempFileDir; - try - { - TempFileDir = sysConfig->getConfig("SystemConfig", "SystemTempFileDir"); - } - catch (...) - {} - - return TempFileDir; - #endif - - } -} +} // namespace startup // vim:ts=4 sw=4: - diff --git a/utils/startup/installdir.h b/utils/startup/installdir.h index f952ba3b2..4d9b89894 100644 --- a/utils/startup/installdir.h +++ b/utils/startup/installdir.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id$ -* -******************************************************************************************/ + * $Id$ + * + ******************************************************************************************/ /** * @file */ @@ -30,26 +30,28 @@ namespace startup { - /** class */ class StartUp { -public: - StartUp() {} - ~StartUp() {} + public: + StartUp() + { + } + ~StartUp() + { + } - static const std::string tmpDir(); + static const std::string tmpDir(); -private: - StartUp(const StartUp& rhs); - StartUp& operator=(const StartUp& rhs); + private: + StartUp(const StartUp& rhs); + StartUp& operator=(const StartUp& rhs); - static boost::mutex fTmpDirLock; - static std::string* fTmpDirp; + static boost::mutex fTmpDirLock; + static std::string* fTmpDirp; }; -} +} // namespace startup #endif // vim:ts=4 sw=4: - diff --git a/utils/testbc/blockcacheclient.cpp b/utils/testbc/blockcacheclient.cpp index 191f531b5..019aad4c1 100644 --- a/utils/testbc/blockcacheclient.cpp +++ b/utils/testbc/blockcacheclient.cpp @@ -28,72 +28,70 @@ namespace dbbc { - blockCacheClient::blockCacheClient() { - fBCCBrp = NULL; + fBCCBrp = NULL; } blockCacheClient::blockCacheClient(BlockRequestProcessor& brp) { - fBCCBrp = &brp; + fBCCBrp = &brp; } blockCacheClient::~blockCacheClient() { } - void blockCacheClient::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache) { - fBCCBrp->check(lbid, ver, flg, wasBlockInCache); + fBCCBrp->check(lbid, ver, flg, wasBlockInCache); } - void blockCacheClient::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount) { - fBCCBrp->check(range, ver, rCount); + fBCCBrp->check(range, ver, rCount); } - FileBuffer* blockCacheClient::getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver) { - FileBuffer* fb = fBCCBrp->getBlockPtr(lbid, ver); - return fb; + FileBuffer* fb = fBCCBrp->getBlockPtr(lbid, ver); + return fb; } const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) { - int ret = fBCCBrp->read(lbid, ver, fb); - return ret; + int ret = fBCCBrp->read(lbid, ver, fb); + return ret; } const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) { - int ret = fBCCBrp->read(lbid, ver, bufferPtr); - return ret; + int ret = fBCCBrp->read(lbid, ver, bufferPtr); + return ret; } -const int blockCacheClient::read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver) +const int blockCacheClient::read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, + const BRM::VER_t ver) { - int ret = fBCCBrp->read(range, fbList, ver); - return ret; + int ret = fBCCBrp->read(range, fbList, ver); + return ret; } -const int blockCacheClient::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, bool& wasCached) +const int blockCacheClient::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, + bool flg, bool& wasCached) { - int ret = fBCCBrp->getBlock(lbid, ver, bufferPtr, flg, wasCached); - return ret; + int ret = fBCCBrp->getBlock(lbid, ver, bufferPtr, flg, wasCached); + return ret; } bool blockCacheClient::exists(BRM::LBID_t lbid, BRM::VER_t ver) { - return fBCCBrp->exists(lbid, ver); + return fBCCBrp->exists(lbid, ver); } void blockCacheClient::flushCache() { - fBCCBrp->flushCache(); + fBCCBrp->flushCache(); } -} +} // namespace dbbc diff --git a/utils/testbc/blockcacheclient.h b/utils/testbc/blockcacheclient.h index 13b345ab9..83d387fe5 100644 --- a/utils/testbc/blockcacheclient.h +++ b/utils/testbc/blockcacheclient.h @@ -29,7 +29,7 @@ #include "brmtypes.h" /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** @@ -42,70 +42,66 @@ namespace dbbc { class blockCacheClient { + public: + /** + * @brief ctor requires reference to BlockRequestProcessor object + **/ + blockCacheClient(BlockRequestProcessor& brp); -public: + /** + * @brief dtor + **/ + virtual ~blockCacheClient(); - /** - * @brief ctor requires reference to BlockRequestProcessor object - **/ - blockCacheClient(BlockRequestProcessor& brp); + /** + * @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache. + **/ + void check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache); - /** - * @brief dtor - **/ - virtual ~blockCacheClient(); + /** + * @brief verify all Disk Blocks for the LBID range are loaded into the Cache + **/ + void check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount); - /** - * @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache. - **/ - void check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache); + /** + * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache + **/ + const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb); - /** - * @brief verify all Disk Blocks for the LBID range are loaded into the Cache - **/ - void check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount); + FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver); - /** - * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache - **/ - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb); + /** + * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache + **/ + const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr); - FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver); + /** + * @brief retrieve all disk Blocks in the LBIDRange range and insert them into fbList + **/ + const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver); - /** - * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache - **/ - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr); + const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, + bool& wasCached); - /** - * @brief retrieve all disk Blocks in the LBIDRange range and insert them into fbList - **/ - const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver); + bool exists(BRM::LBID_t lbid, BRM::VER_t ver); - const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, - bool flg, bool& wasCached); + /** + * @brief flush the cache + **/ + void flushCache(); - bool exists(BRM::LBID_t lbid, BRM::VER_t ver); - - /** - * @brief flush the cache - **/ - void flushCache(); - -private: - - /** - * @brief pointer to the BlockRequestProcessor object on which the API will operate - **/ - BlockRequestProcessor* fBCCBrp; - - //do not implement - blockCacheClient(); - blockCacheClient(const blockCacheClient& bc); - const blockCacheClient& operator=(const blockCacheClient& blk); + private: + /** + * @brief pointer to the BlockRequestProcessor object on which the API will operate + **/ + BlockRequestProcessor* fBCCBrp; + // do not implement + blockCacheClient(); + blockCacheClient(const blockCacheClient& bc); + const blockCacheClient& operator=(const blockCacheClient& blk); }; -} +} // namespace dbbc #endif diff --git a/utils/testbc/blockrequestprocessor.cpp b/utils/testbc/blockrequestprocessor.cpp index 95564877f..118b63035 100644 --- a/utils/testbc/blockrequestprocessor.cpp +++ b/utils/testbc/blockrequestprocessor.cpp @@ -36,218 +36,192 @@ using namespace std; namespace dbbc { - -BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks, - int thrCount, - int blocksPerRead, - uint32_t deleteBlocks, - uint32_t blckSz) : - fbMgr(numBlcks, blckSz, deleteBlocks), - fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead) +BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, + uint32_t deleteBlocks, uint32_t blckSz) + : fbMgr(numBlcks, blckSz, deleteBlocks), fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead) { } - BlockRequestProcessor::~BlockRequestProcessor() { } void BlockRequestProcessor::stop() { - fBRPRequestQueue.stop(); - fIOMgr.stop(); + fBRPRequestQueue.stop(); + fIOMgr.stop(); } - int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount) { - uint64_t maxLbid = range.start; // highest existent lbid - uint64_t rangeLen = range.size; - uint64_t idx; - uint64_t adjSz; - struct timespec start_tm; + uint64_t maxLbid = range.start; // highest existent lbid + uint64_t rangeLen = range.size; + uint64_t idx; + uint64_t adjSz; + struct timespec start_tm; - if (fTrace) clock_gettime(CLOCK_MONOTONIC, &start_tm); + if (fTrace) + clock_gettime(CLOCK_MONOTONIC, &start_tm); - for (idx = 0; fbMgr.exists(maxLbid, ver) == true && idx < rangeLen; maxLbid++, idx++); + for (idx = 0; fbMgr.exists(maxLbid, ver) == true && idx < rangeLen; maxLbid++, idx++) + ; - if (idx == rangeLen) // range is already loaded - { - uint32_t fbo; - BRM::OID_t oid; - fdbrm.lookup(maxLbid, ver, false, oid, fbo); - fLogFile - << oid << " " - << maxLbid << " " - << fbo << " " - << rangeLen << " " - << 0 << " " - << 0 << " " - << 0 << " " - << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) - << endl; - return 0; - } + if (idx == rangeLen) // range is already loaded + { + uint32_t fbo; + BRM::OID_t oid; + fdbrm.lookup(maxLbid, ver, false, oid, fbo); + fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << 0 << " " << 0 << " " << 0 + << " " << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl; + return 0; + } - adjSz = rangeLen - idx; - BRM::InlineLBIDRange adjRange; - adjRange.start = maxLbid; - adjRange.size = adjSz; - fileRequest rqstBlk(adjRange, ver); - check(rqstBlk); - lbidCount = rqstBlk.BlocksRead(); + adjSz = rangeLen - idx; + BRM::InlineLBIDRange adjRange; + adjRange.start = maxLbid; + adjRange.size = adjSz; + fileRequest rqstBlk(adjRange, ver); + check(rqstBlk); + lbidCount = rqstBlk.BlocksRead(); - if (fTrace) - { - uint32_t fbo; - BRM::OID_t oid; - fdbrm.lookup(maxLbid, ver, false, oid, fbo); - fLogFile - << oid << " " - << maxLbid << " " - << fbo << " " - << rangeLen << " " - << adjSz << " " - << rqstBlk.BlocksRead() << " " - << rqstBlk.BlocksLoaded() << " " - << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) - << endl; - } - - return rqstBlk.BlocksLoaded(); -} // check + if (fTrace) + { + uint32_t fbo; + BRM::OID_t oid; + fdbrm.lookup(maxLbid, ver, false, oid, fbo); + fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << adjSz << " " + << rqstBlk.BlocksRead() << " " << rqstBlk.BlocksLoaded() << " " << right << fixed + << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl; + } + return rqstBlk.BlocksLoaded(); +} // check int BlockRequestProcessor::check(fileRequest& rqstBlk) { - pthread_mutex_lock(&rqstBlk.frMutex()); - rqstBlk.SetPredicate(fileRequest::SENDING); - sendRequest(rqstBlk); // start file read request + pthread_mutex_lock(&rqstBlk.frMutex()); + rqstBlk.SetPredicate(fileRequest::SENDING); + sendRequest(rqstBlk); // start file read request - while (rqstBlk.frPredicate() < fileRequest::COMPLETE) - pthread_cond_wait(&rqstBlk.frCond(), &rqstBlk.frMutex()); + while (rqstBlk.frPredicate() < fileRequest::COMPLETE) + pthread_cond_wait(&rqstBlk.frCond(), &rqstBlk.frMutex()); - pthread_mutex_unlock(&rqstBlk.frMutex()); + pthread_mutex_unlock(&rqstBlk.frMutex()); - return 0; + return 0; } - int BlockRequestProcessor::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache) { - if (fbMgr.exists(lbid, ver) == true) - { - wasBlockInCache = true; - return 0; - } - else - { - wasBlockInCache = false; - fileRequest rqstBlk(lbid, ver, flg); - int ret = check(rqstBlk); - return ret; - } + if (fbMgr.exists(lbid, ver) == true) + { + wasBlockInCache = true; + return 0; + } + else + { + wasBlockInCache = false; + fileRequest rqstBlk(lbid, ver, flg); + int ret = check(rqstBlk); + return ret; + } } - int BlockRequestProcessor::sendRequest(fileRequest& blk) { - int ret = fBRPRequestQueue.push(blk); - return ret; + int ret = fBRPRequestQueue.push(blk); + return ret; } - -const int BlockRequestProcessor::read(const BRM::InlineLBIDRange& range, FileBufferList_t& readList, const BRM::VER_t ver) +const int BlockRequestProcessor::read(const BRM::InlineLBIDRange& range, FileBufferList_t& readList, + const BRM::VER_t ver) { - int blksLoaded = 0; - HashObject_t fb = {0, 0, 0}; + int blksLoaded = 0; + HashObject_t fb = {0, 0, 0}; - for (int idx = 0; (uint64_t)idx < range.size; idx++) + for (int idx = 0; (uint64_t)idx < range.size; idx++) + { + fb.lbid = range.start + idx; + fb.ver = ver; + fb.poolIdx = 0; + FileBuffer fbRet(-1, -1); + bool ret = false; // fbMgr.find(fb, fbRet); + + if (ret) { - fb.lbid = range.start + idx; - fb.ver = ver; - fb.poolIdx = 0; - FileBuffer fbRet(-1, -1); - bool ret = false; //fbMgr.find(fb, fbRet); - - if (ret) - { - blksLoaded++; - readList.push_back(fbRet); - } + blksLoaded++; + readList.push_back(fbRet); } + } - return blksLoaded; + return blksLoaded; } - -FileBuffer* BlockRequestProcessor::getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver ) +FileBuffer* BlockRequestProcessor::getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver) { - - HashObject_t hashObj = {lbid, ver, 0}; - FileBuffer* fb = fbMgr.findPtr(hashObj); - return fb; + HashObject_t hashObj = {lbid, ver, 0}; + FileBuffer* fb = fbMgr.findPtr(hashObj); + return fb; } - const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) { + HashObject_t hashObj = {lbid, ver, 0}; + bool ret = fbMgr.find(hashObj, fb); - HashObject_t hashObj = {lbid, ver, 0}; - bool ret = fbMgr.find(hashObj, fb); - - if (ret == true) - return 1; - else - return 0; + if (ret == true) + return 1; + else + return 0; } const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) { - HashObject_t hashObj = {lbid, ver, 0}; - bool ret = fbMgr.find(hashObj, bufferPtr); + HashObject_t hashObj = {lbid, ver, 0}; + bool ret = fbMgr.find(hashObj, bufferPtr); - if (ret == true) - return 1; - else - return 0; + if (ret == true) + return 1; + else + return 0; } -const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, bool& wasCached) +const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, + bool flg, bool& wasCached) { - HashObject_t hashObj = {lbid, ver, 0}; - wasCached = fbMgr.find(hashObj, bufferPtr); + HashObject_t hashObj = {lbid, ver, 0}; + wasCached = fbMgr.find(hashObj, bufferPtr); - if (wasCached) - return 1; - - wasCached = false; - fileRequest rqstBlk(lbid, ver, flg, (uint8_t*) bufferPtr); - check(rqstBlk); + if (wasCached) return 1; + + wasCached = false; + fileRequest rqstBlk(lbid, ver, flg, (uint8_t*)bufferPtr); + check(rqstBlk); + return 1; } bool BlockRequestProcessor::exists(BRM::LBID_t lbid, BRM::VER_t ver) { - HashObject_t ho = {lbid, ver, 0}; + HashObject_t ho = {lbid, ver, 0}; - return fbMgr.exists(ho); + return fbMgr.exists(ho); } void BlockRequestProcessor::flushCache() { - fbMgr.flushCache(); + fbMgr.flushCache(); } /** const uint32_t BlockRequestProcessor::resize(const uint32_t s) { - int rc = fbMgr.resize(s); - return rc; + int rc = fbMgr.resize(s); + return rc; } **/ ostream& BlockRequestProcessor::formatLRUList(ostream& os) const { - return fbMgr.formatLRUList(os); + return fbMgr.formatLRUList(os); } -} // namespace dbbc +} // namespace dbbc diff --git a/utils/testbc/blockrequestprocessor.h b/utils/testbc/blockrequestprocessor.h index c6b061dca..78fc95891 100644 --- a/utils/testbc/blockrequestprocessor.h +++ b/utils/testbc/blockrequestprocessor.h @@ -26,19 +26,17 @@ * * ***************************************************************************/ - #include "blocksize.h" #include "fileblockrequestqueue.h" #include "filebuffermgr.h" #include "iomanager.h" /** - @author Jason Rodriguez + @author Jason Rodriguez */ namespace dbbc { - typedef std::list FileBufferList_t; /** @@ -47,93 +45,89 @@ typedef std::list FileBufferList_t; class BlockRequestProcessor { + public: + /** + * @brief default ctor + **/ + BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0, + uint32_t blckSz = BLOCK_SIZE); -public: + /** + * @brief default dtor + **/ + virtual ~BlockRequestProcessor(); - /** - * @brief default ctor - **/ - BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0, - uint32_t blckSz = BLOCK_SIZE); + /** + * @brief send a request for disk blocks to the IO manager + **/ + int sendRequest(fileRequest& blk); - /** - * @brief default dtor - **/ - virtual ~BlockRequestProcessor(); + /** + * @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not + **/ + int check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache); - /** - * @brief send a request for disk blocks to the IO manager - **/ - int sendRequest(fileRequest& blk); + /** + * @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not + **/ + int check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount); - /** - * @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not - **/ - int check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache); + /** + * @brief retrieve the lbid@ver disk block from the block cache + **/ + FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver); - /** - * @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not - **/ - int check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount); + const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb); - /** - * @brief retrieve the lbid@ver disk block from the block cache - **/ - FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver); + /** + * @brief retrieve the lbid@ver disk block from the block cache + **/ + const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr); - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb); + /** + * @brief retrieve the LBIDRange of disk blocks from the block cache + **/ + const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver); - /** - * @brief retrieve the lbid@ver disk block from the block cache - **/ - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr); + const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, + bool& wasCached); - /** - * @brief retrieve the LBIDRange of disk blocks from the block cache - **/ - const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver); + bool exists(BRM::LBID_t lbid, BRM::VER_t ver); - const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, - bool flg, bool& wasCached); + /** + * @brief + **/ + void flushCache(); - bool exists(BRM::LBID_t lbid, BRM::VER_t ver); + // const uint32_t resize(const uint32_t s); - /** - * @brief - **/ - void flushCache(); + std::ostream& formatLRUList(std::ostream& os) const; - //const uint32_t resize(const uint32_t s); + private: + FileBufferMgr fbMgr; + fileBlockRequestQueue fBRPRequestQueue; + ioManager fIOMgr; + pthread_mutex_t check_mutex; - std::ostream& formatLRUList(std::ostream& os) const; + /** + * helper function for public check functions + **/ + int check(fileRequest& rqstBlk); -private: + /** + * send stop requests for IOmanager and request Q + **/ + void stop(); - FileBufferMgr fbMgr; - fileBlockRequestQueue fBRPRequestQueue; - ioManager fIOMgr; - pthread_mutex_t check_mutex; + std::ofstream fLogFile; + bool fTrace; - /** - * helper function for public check functions - **/ - int check(fileRequest& rqstBlk); - - /** - * send stop requests for IOmanager and request Q - **/ - void stop(); - - std::ofstream fLogFile; - bool fTrace; - - BRM::DBRM fdbrm; - - // do not implement - BlockRequestProcessor(const BlockRequestProcessor& brp); - BlockRequestProcessor& operator=(const BlockRequestProcessor& brp); + BRM::DBRM fdbrm; + // do not implement + BlockRequestProcessor(const BlockRequestProcessor& brp); + BlockRequestProcessor& operator=(const BlockRequestProcessor& brp); }; -} +} // namespace dbbc #endif diff --git a/utils/testbc/fileblockrequestqueue.cpp b/utils/testbc/fileblockrequestqueue.cpp index e135b0217..8fa4c0208 100644 --- a/utils/testbc/fileblockrequestqueue.cpp +++ b/utils/testbc/fileblockrequestqueue.cpp @@ -23,74 +23,70 @@ * * ***************************************************************************/ - #include "fileblockrequestqueue.h" using namespace std; namespace dbbc { - - fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0) { - pthread_mutex_init(&mutex, NULL); - pthread_cond_init(¬Empty, NULL); + pthread_mutex_init(&mutex, NULL); + pthread_cond_init(¬Empty, NULL); } fileBlockRequestQueue::~fileBlockRequestQueue() { - pthread_cond_destroy(¬Empty); - pthread_mutex_destroy(&mutex); + pthread_cond_destroy(¬Empty); + pthread_mutex_destroy(&mutex); } bool fileBlockRequestQueue::empty() const { - return (queueSize == 0); + return (queueSize == 0); } fileRequest* fileBlockRequestQueue::top() const { - return fbQueue.front(); + return fbQueue.front(); } int fileBlockRequestQueue::push(fileRequest& blk) { - pthread_mutex_lock(&mutex); - fbQueue.push_back(&blk); + pthread_mutex_lock(&mutex); + fbQueue.push_back(&blk); - // @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which - // caused only one i/o thread to be working at a time. - if (++queueSize >= 1 && readersWaiting > 0) - pthread_cond_signal(¬Empty); + // @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size + // was 1 which caused only one i/o thread to be working at a time. + if (++queueSize >= 1 && readersWaiting > 0) + pthread_cond_signal(¬Empty); - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&mutex); - return 0; + return 0; } void fileBlockRequestQueue::stop() { - pthread_cond_broadcast(¬Empty); + pthread_cond_broadcast(¬Empty); } - fileRequest* fileBlockRequestQueue::pop(void) { - pthread_mutex_lock(&mutex); + pthread_mutex_lock(&mutex); - while (queueSize == 0) - { - readersWaiting++; - pthread_cond_wait(¬Empty, &mutex); - readersWaiting--; - } + while (queueSize == 0) + { + readersWaiting++; + pthread_cond_wait(¬Empty, &mutex); + readersWaiting--; + } - fileRequest* blk = fbQueue.front(); - fbQueue.pop_front(); - --queueSize; - pthread_mutex_unlock(&mutex); - return blk; + fileRequest* blk = fbQueue.front(); + fbQueue.pop_front(); + --queueSize; + pthread_mutex_unlock(&mutex); + return blk; } -} +} // namespace dbbc diff --git a/utils/testbc/fileblockrequestqueue.h b/utils/testbc/fileblockrequestqueue.h index 569750d75..8adcd147c 100644 --- a/utils/testbc/fileblockrequestqueue.h +++ b/utils/testbc/fileblockrequestqueue.h @@ -24,17 +24,15 @@ * * jrodriguez@calpont.com * * * -***************************************************************************/ - + ***************************************************************************/ #include #include #include #include "filerequest.h" - /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** @@ -42,75 +40,71 @@ **/ namespace dbbc { - typedef std::deque fileBlockRequestQueue_t; /** * @brief class to hold requests for disk blocks in a queue. sorted by the size of a request **/ - class fileBlockRequestQueue { + public: + /** + * @brief default ctor + **/ + fileBlockRequestQueue(); -public: + /** + * @brief dtor + **/ + virtual ~fileBlockRequestQueue(); - /** - * @brief default ctor - **/ - fileBlockRequestQueue(); + /** + * @brief add a request to the queue + **/ + int push(fileRequest& blk); - /** - * @brief dtor - **/ - virtual ~fileBlockRequestQueue(); + /** + * @brief get the next request from the queue and delete it from the queue + **/ + fileRequest* pop(void); - /** - * @brief add a request to the queue - **/ - int push(fileRequest& blk); + /** + * @brief true if no reuquests are in the queue. false if there are requests in the queue + **/ + bool empty() const; - /** - * @brief get the next request from the queue and delete it from the queue - **/ - fileRequest* pop(void); + /** + * @brief number of requests in the queue + **/ + uint32_t size() const + { + return queueSize; + } - /** - * @brief true if no reuquests are in the queue. false if there are requests in the queue - **/ - bool empty() const; + /** + * @brief queue will stop accecpting requests in preparation for the dtor + **/ + void stop(); - /** - * @brief number of requests in the queue - **/ - uint32_t size() const - { - return queueSize; - } - - /** - * @brief queue will stop accecpting requests in preparation for the dtor - **/ - void stop(); - -protected: - pthread_mutex_t mutex; - pthread_cond_t notEmpty; - fileBlockRequestQueue_t fbQueue; - uint32_t queueSize; - uint32_t readersWaiting; - -private: - // do not implement - fileBlockRequestQueue(const fileBlockRequestQueue& Q) {} - const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q); - - /** - * @brief pointer to the next request to be popped from the queue - **/ - fileRequest* top() const; + protected: + pthread_mutex_t mutex; + pthread_cond_t notEmpty; + fileBlockRequestQueue_t fbQueue; + uint32_t queueSize; + uint32_t readersWaiting; + private: + // do not implement + fileBlockRequestQueue(const fileBlockRequestQueue& Q) + { + } + const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q); + /** + * @brief pointer to the next request to be popped from the queue + **/ + fileRequest* top() const; }; -} +} // namespace dbbc #endif diff --git a/utils/testbc/filebuffer.cpp b/utils/testbc/filebuffer.cpp index 1efa64818..ba62983a0 100644 --- a/utils/testbc/filebuffer.cpp +++ b/utils/testbc/filebuffer.cpp @@ -32,60 +32,54 @@ using namespace std; namespace dbbc { - FileBuffer::FileBuffer(const FileBuffer& rhs) { + if (this == NULL || this == &rhs) + return; - if (this == NULL || this == &rhs) - return; - - fLbid = rhs.fLbid; - fVerid = rhs.fVerid; - setData(rhs.fByteData, rhs.fDataLen); - fListLoc = rhs.listLoc(); - fDataLen = rhs.fDataLen; + fLbid = rhs.fLbid; + fVerid = rhs.fVerid; + setData(rhs.fByteData, rhs.fDataLen); + fListLoc = rhs.listLoc(); + fDataLen = rhs.fDataLen; } - FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len) { - fLbid = lbid; - fVerid = ver; - fDataLen = len; - setData(data, fDataLen); + fLbid = lbid; + fVerid = ver; + fDataLen = len; + setData(data, fDataLen); } - FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver) { - fLbid = lbid; - fVerid = ver; - fDataLen = 0; + fLbid = lbid; + fVerid = ver; + fDataLen = 0; } - -FileBuffer& FileBuffer::operator= (const FileBuffer& rhs) +FileBuffer& FileBuffer::operator=(const FileBuffer& rhs) { - fLbid = rhs.fLbid; - fVerid = rhs.fVerid; - fDataLen = rhs.fDataLen; - setData(rhs.fByteData, fDataLen); - fListLoc = rhs.listLoc(); - return *this; + fLbid = rhs.fLbid; + fVerid = rhs.fVerid; + fDataLen = rhs.fDataLen; + setData(rhs.fByteData, fDataLen); + fListLoc = rhs.listLoc(); + return *this; } void FileBuffer::setData(const uint8_t* d, const int len) { - if (d == NULL || len <= 0) - return; + if (d == NULL || len <= 0) + return; - fDataLen = len; - memcpy(fByteData, d, len); + fDataLen = len; + memcpy(fByteData, d, len); } - FileBuffer::~FileBuffer() { } -} +} // namespace dbbc diff --git a/utils/testbc/filebuffer.h b/utils/testbc/filebuffer.h index a14bb1fae..87a4bdafc 100644 --- a/utils/testbc/filebuffer.h +++ b/utils/testbc/filebuffer.h @@ -21,7 +21,7 @@ * * jrodriguez@calpont.com * * * -***************************************************************************/ + ***************************************************************************/ #ifndef FILEBUFFER_H #define FILEBUFFER_H @@ -38,22 +38,20 @@ #include /** - @author Jason Rodriguez + @author Jason Rodriguez */ - /** * @brief represents a disk blockrequest **/ namespace dbbc { - -//Set block cache alogorithm to last recently used by defining LRU. -//Otherwise it will be FIFO. +// Set block cache alogorithm to last recently used by defining LRU. +// Otherwise it will be FIFO. typedef struct FBData { - BRM::LBID_t lbid; - BRM::VER_t ver; + BRM::LBID_t lbid; + BRM::VER_t ver; } FBData_t; //@bug 669 Change to list for last recently used cache @@ -62,128 +60,125 @@ typedef std::list::iterator filebuffer_list_iter_t; class FileBuffer { + public: + /** + * @brief copy ctor + **/ + FileBuffer(const FileBuffer& fb); -public: + /** + * @brief the disk block from lbid@ver, and a data block len bytes long + **/ + FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len); - /** - * @brief copy ctor - **/ - FileBuffer(const FileBuffer& fb); + /** + * @brief disk block lbid@ver and empty data + **/ + FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver); - /** - * @brief the disk block from lbid@ver, and a data block len bytes long - **/ - FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len); + /** + * @brief class dtor + **/ + ~FileBuffer(); - /** - * @brief disk block lbid@ver and empty data - **/ - FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver); + /** + * @brief set the data value of this block to d have len bytestream + **/ + void setData(const uint8_t* d, const int len = 8192); - /** - * @brief class dtor - **/ - ~FileBuffer(); + /** + * @brief retrieve the data in byte* format from this data block + **/ + const uint8_t* getData() const + { + return fByteData; + } + uint8_t* getData() + { + return fByteData; + } - /** - * @brief set the data value of this block to d have len bytestream - **/ - void setData(const uint8_t* d, const int len = 8192); + const uint32_t datLen() const + { + return fDataLen; + } - /** - * @brief retrieve the data in byte* format from this data block - **/ - const uint8_t* getData() const - { - return fByteData; - } - uint8_t* getData() - { - return fByteData; - } + /** + * @brief assignment operator + **/ + FileBuffer& operator=(const FileBuffer& rhs); - const uint32_t datLen() const - { - return fDataLen; - } + /** + * @brief equality operator is based on lbid@ver + **/ + bool operator==(const FileBuffer& rhs) const + { + return (fLbid == rhs.fLbid && fVerid == rhs.fVerid); + } - /** - * @brief assignment operator - **/ - FileBuffer& operator= (const FileBuffer& rhs); + /** + * @brief inequality operator + **/ + bool operator!=(const FileBuffer& rhs) const + { + return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid)); + } - /** - * @brief equality operator is based on lbid@ver - **/ - bool operator==(const FileBuffer& rhs) const - { - return (fLbid == rhs.fLbid && fVerid == rhs.fVerid); - } + FileBuffer* thisPtr() + { + return this; + } + /** + * @brief return the lbid value of disk bloc + **/ + const BRM::LBID_t Lbid() const + { + return fLbid; + } + void Lbid(const BRM::LBID_t l) + { + fLbid = l; + } - /** - * @brief inequality operator - **/ - bool operator!=(const FileBuffer& rhs) const - { - return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid)); - } + /** + * @brief return the version of this disk block. ignored for range retrievals + **/ + const BRM::VER_t Verid() const + { + return fVerid; + } + void Verid(BRM::VER_t v) + { + fVerid = v; + } - FileBuffer* thisPtr() - { - return this; - } - /** - * @brief return the lbid value of disk bloc - **/ - const BRM::LBID_t Lbid() const - { - return fLbid; - } - void Lbid(const BRM::LBID_t l) - { - fLbid = l; - } + /** + * @brief return the number of bytes in this disk blockrequest + **/ - /** - * @brief return the version of this disk block. ignored for range retrievals - **/ - const BRM::VER_t Verid() const - { - return fVerid; - } - void Verid(BRM::VER_t v) - { - fVerid = v; - } + void listLoc(const filebuffer_list_iter_t& loc) + { + fListLoc = loc; + } - /** - * @brief return the number of bytes in this disk blockrequest - **/ + const filebuffer_list_iter_t& listLoc() const + { + return fListLoc; + } - void listLoc(const filebuffer_list_iter_t& loc) - { - fListLoc = loc; - } + private: + uint8_t fByteData[WriteEngine::BYTE_PER_BLOCK]; + uint32_t fDataLen; - const filebuffer_list_iter_t& listLoc() const - { - return fListLoc; - } + BRM::LBID_t fLbid; + BRM::VER_t fVerid; + filebuffer_list_iter_t fListLoc; -private: - - uint8_t fByteData[WriteEngine::BYTE_PER_BLOCK]; - uint32_t fDataLen; - - BRM::LBID_t fLbid; - BRM::VER_t fVerid; - filebuffer_list_iter_t fListLoc; - - // do not implement - FileBuffer() {}; + // do not implement + FileBuffer(){}; }; typedef std::vector FileBufferPool_t; -} +} // namespace dbbc #endif diff --git a/utils/testbc/filebuffermgr.cpp b/utils/testbc/filebuffermgr.cpp index 2aea7a852..e9b882347 100644 --- a/utils/testbc/filebuffermgr.cpp +++ b/utils/testbc/filebuffermgr.cpp @@ -24,11 +24,11 @@ ***************************************************************************/ /** * InitialDBBCSize - the starting number of elements the unordered set used to store disk blocks. - This does not instantiate InitialDBBCSize disk blocks but only the initial size of the unordered_set + This does not instantiate InitialDBBCSize disk blocks but only the initial size of the unordered_set **/ -#define NDEBUG //Turn off assert macro +#define NDEBUG // Turn off assert macro #include #include #include @@ -43,129 +43,123 @@ extern uint32_t gSession; namespace dbbc { - FileBufferMgr::FileBufferMgr(const uint32_t numBlcks, const uint32_t blkSz, const uint32_t deleteBlocks) - : fMaxNumBlocks(numBlcks), - fBlockSz(blkSz), - fWLock(), - fbSet(), - fbList(), - fCacheSize(0), - fFBPool(), - aging(false), - fDeleteBlocks(deleteBlocks), - fEmptyPoolSlots() + : fMaxNumBlocks(numBlcks) + , fBlockSz(blkSz) + , fWLock() + , fbSet() + , fbList() + , fCacheSize(0) + , fFBPool() + , aging(false) + , fDeleteBlocks(deleteBlocks) + , fEmptyPoolSlots() { - fFBPool.reserve(numBlcks); - fEmptyPoolSlots.reserve(deleteBlocks); + fFBPool.reserve(numBlcks); + fEmptyPoolSlots.reserve(deleteBlocks); } FileBufferMgr::~FileBufferMgr() { - flushCache(); + flushCache(); } void FileBufferMgr::flushCache() { - mutex::scoped_lock lk(fWLock); - fbList.clear(); - fbSet.clear(); - fFBPool.clear(); - fEmptyPoolSlots.clear(); - //TODO:: re-init blocks in pool and HWM + mutex::scoped_lock lk(fWLock); + fbList.clear(); + fbSet.clear(); + fFBPool.clear(); + fEmptyPoolSlots.clear(); + // TODO:: re-init blocks in pool and HWM } - bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const { - const HashObject_t fb = {lbid, ver, 0}; - const bool b = exists(fb); - return b; -} // bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const - + const HashObject_t fb = {lbid, ver, 0}; + const bool b = exists(fb); + return b; +} // bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const FileBuffer* FileBufferMgr::findPtr(const HashObject_t& keyFb) { - mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(keyFb); + mutex::scoped_lock lk(fWLock); + filebuffer_uset_iter_t it = fbSet.find(keyFb); - if (fbSet.end() != it) - { - FileBuffer* fb = &(fFBPool[it->poolIdx]); - fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() ); - return fb; - } + if (fbSet.end() != it) + { + FileBuffer* fb = &(fFBPool[it->poolIdx]); + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + return fb; + } - return NULL; - -} // end findPtr(const HashObject_t& keyFB) + return NULL; +} // end findPtr(const HashObject_t& keyFB) bool FileBufferMgr::find(const HashObject_t& keyFb, FileBuffer& fb) { - bool ret = false; + bool ret = false; - mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(keyFb); + mutex::scoped_lock lk(fWLock); + filebuffer_uset_iter_t it = fbSet.find(keyFb); - if (fbSet.end() != it) - { - fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() ); - lk.unlock(); - fb = fFBPool[it->poolIdx]; - ret = true; - } + if (fbSet.end() != it) + { + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + lk.unlock(); + fb = fFBPool[it->poolIdx]; + ret = true; + } - return ret; - -} // end find(const HashObject_t& keyFB, HashObject_t& fb) + return ret; +} // end find(const HashObject_t& keyFB, HashObject_t& fb) bool FileBufferMgr::find(const HashObject_t& keyFb, void* bufferPtr) { - bool ret = false; + bool ret = false; + + if (gPMProfOn && gPMStatsPtr) + gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L'); + + mutex::scoped_lock lk(fWLock); + + if (gPMProfOn && gPMStatsPtr) + gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M'); + + filebuffer_uset_iter_t it = fbSet.find(keyFb); + + if (fbSet.end() != it) + { + //@bug 669 LRU cache, move block to front of list as last recently used. + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + lk.unlock(); + memcpy(bufferPtr, (fFBPool[it->poolIdx]).getData(), 8); if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L'); + gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U'); - mutex::scoped_lock lk(fWLock); + ret = true; + } - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M'); - - filebuffer_uset_iter_t it = fbSet.find(keyFb); - - if (fbSet.end() != it) - { - //@bug 669 LRU cache, move block to front of list as last recently used. - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - lk.unlock(); - memcpy(bufferPtr, (fFBPool[it->poolIdx]).getData(), 8); - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U'); - - ret = true; - } - - return ret; - -} // end find(const FileBuffer& keyFB, void* bufferPtr) + return ret; +} // end find(const FileBuffer& keyFB, void* bufferPtr) bool FileBufferMgr::exists(const HashObject_t& fb) const { - bool find_bool = false; - mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(fb); + bool find_bool = false; + mutex::scoped_lock lk(fWLock); + filebuffer_uset_iter_t it = fbSet.find(fb); - if (it != fbSet.end()) - { - find_bool = true; - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - } + if (it != fbSet.end()) + { + find_bool = true; + fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); + } - return find_bool; + return find_bool; } // default insert operation. @@ -176,131 +170,130 @@ bool FileBufferMgr::exists(const HashObject_t& fb) const const int FileBufferMgr::insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data) { - int ret = 0; + int ret = 0; + if (gPMProfOn && gPMStatsPtr) + gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I'); + + mutex::scoped_lock lk(fWLock); + HashObject_t fbIndex = {lbid, ver, 0}; + filebuffer_pair_t pr = fbSet.insert(fbIndex); + + if (pr.second) + { + // It was inserted (it wasn't there before) + // Right now we have an invalid cache: we have inserted an entry with a -1 index. + // We need to fix this quickly... + fCacheSize++; + FBData_t fbdata = {lbid, ver}; + fbList.push_front(fbdata); + } + else + { + // if it's a duplicate there's nothing to do if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I'); + gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D'); - mutex::scoped_lock lk(fWLock); - HashObject_t fbIndex = { lbid, ver, 0}; - filebuffer_pair_t pr = fbSet.insert(fbIndex); - - if (pr.second) - { - // It was inserted (it wasn't there before) - // Right now we have an invalid cache: we have inserted an entry with a -1 index. - // We need to fix this quickly... - fCacheSize++; - FBData_t fbdata = {lbid, ver}; - fbList.push_front(fbdata); - } - else - { - // if it's a duplicate there's nothing to do - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D'); - - return ret; - } - - uint32_t pi = INT_MAX; - - if (fCacheSize > maxCacheSize()) - { - // If the insert above caused the cache to exceed its max size, find the lru block in - // the cache and use its pool index to store the block data. - FBData_t& fbdata = fbList.back(); //the lru block - HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0}; - filebuffer_uset_iter_t iter = fbSet.find( lastFB ); //should be there - - idbassert(iter != fbSet.end()); - pi = iter->poolIdx; - idbassert(pi < maxCacheSize()); - idbassert(pi < fFBPool.size()); - - /* Why does this iterator return a const HashObject_t? */ - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - - //replace the lru block with this block - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool[pi] = fb; - fFBPool[pi].setData(data, BLOCK_SIZE); - fbSet.erase(iter); - fbList.pop_back(); - - fCacheSize--; - depleteCache(); - ret = 1; - } - else - { - if ( ! fEmptyPoolSlots.empty() ) - { - pi = fEmptyPoolSlots.back(); - fEmptyPoolSlots.pop_back(); - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool[pi] = fb; - fFBPool[pi].setData(data, 8); - } - else - { - pi = fFBPool.size(); - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool.push_back(fb); - fFBPool[pi].setData(data, 8); - } - - /* Why does this iterator return a const? */ - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - ret = 1; - } - - idbassert(pi < fFBPool.size()); - fFBPool[pi].listLoc(fbList.begin()); - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J'); - - idbassert(fCacheSize <= maxCacheSize()); -// idbassert(fCacheSize == fbSet.size()); -// idbassert(fCacheSize == fbList.size()); return ret; -} + } + uint32_t pi = INT_MAX; + + if (fCacheSize > maxCacheSize()) + { + // If the insert above caused the cache to exceed its max size, find the lru block in + // the cache and use its pool index to store the block data. + FBData_t& fbdata = fbList.back(); // the lru block + HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0}; + filebuffer_uset_iter_t iter = fbSet.find(lastFB); // should be there + + idbassert(iter != fbSet.end()); + pi = iter->poolIdx; + idbassert(pi < maxCacheSize()); + idbassert(pi < fFBPool.size()); + + /* Why does this iterator return a const HashObject_t? */ + HashObject_t& ref = const_cast(*pr.first); + ref.poolIdx = pi; + + // replace the lru block with this block + FileBuffer fb(lbid, ver, NULL, 0); + fFBPool[pi] = fb; + fFBPool[pi].setData(data, BLOCK_SIZE); + fbSet.erase(iter); + fbList.pop_back(); + + fCacheSize--; + depleteCache(); + ret = 1; + } + else + { + if (!fEmptyPoolSlots.empty()) + { + pi = fEmptyPoolSlots.back(); + fEmptyPoolSlots.pop_back(); + FileBuffer fb(lbid, ver, NULL, 0); + fFBPool[pi] = fb; + fFBPool[pi].setData(data, 8); + } + else + { + pi = fFBPool.size(); + FileBuffer fb(lbid, ver, NULL, 0); + fFBPool.push_back(fb); + fFBPool[pi].setData(data, 8); + } + + /* Why does this iterator return a const? */ + HashObject_t& ref = const_cast(*pr.first); + ref.poolIdx = pi; + ret = 1; + } + + idbassert(pi < fFBPool.size()); + fFBPool[pi].listLoc(fbList.begin()); + + if (gPMProfOn && gPMStatsPtr) + gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J'); + + idbassert(fCacheSize <= maxCacheSize()); + // idbassert(fCacheSize == fbSet.size()); + // idbassert(fCacheSize == fbList.size()); + return ret; +} void FileBufferMgr::depleteCache() { - for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i) - { - FBData_t fbdata(fbList.back()); //the lru block - HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0}; - filebuffer_uset_iter_t iter = fbSet.find( lastFB ); + for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i) + { + FBData_t fbdata(fbList.back()); // the lru block + HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0}; + filebuffer_uset_iter_t iter = fbSet.find(lastFB); - idbassert(iter != fbSet.end()); - uint32_t idx = iter->poolIdx; - idbassert(idx < fFBPool.size()); - //Save position in FileBuffer pool for reuse. - fEmptyPoolSlots.push_back(idx); - fbSet.erase(iter); - fbList.pop_back(); - fCacheSize--; - } + idbassert(iter != fbSet.end()); + uint32_t idx = iter->poolIdx; + idbassert(idx < fFBPool.size()); + // Save position in FileBuffer pool for reuse. + fEmptyPoolSlots.push_back(idx); + fbSet.erase(iter); + fbList.pop_back(); + fCacheSize--; + } } ostream& FileBufferMgr::formatLRUList(ostream& os) const { - filebuffer_list_t::const_iterator iter = fbList.begin(); - filebuffer_list_t::const_iterator end = fbList.end(); + filebuffer_list_t::const_iterator iter = fbList.begin(); + filebuffer_list_t::const_iterator end = fbList.end(); - while (iter != end) - { - os << iter->lbid << '\t' << iter->ver << endl; - ++iter; - } + while (iter != end) + { + os << iter->lbid << '\t' << iter->ver << endl; + ++iter; + } - return os; + return os; } -} +} // namespace dbbc diff --git a/utils/testbc/filebuffermgr.h b/utils/testbc/filebuffermgr.h index 76ddda3d9..4a756c280 100644 --- a/utils/testbc/filebuffermgr.h +++ b/utils/testbc/filebuffermgr.h @@ -32,170 +32,168 @@ #include /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** - * @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and list. + * @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and + *list. * **/ namespace dbbc { - /** * @brief used as the hasher algorithm for the unordered_set used to store the disk blocks **/ typedef struct { - BRM::LBID_t lbid; - BRM::VER_t ver; - uint32_t poolIdx; + BRM::LBID_t lbid; + BRM::VER_t ver; + uint32_t poolIdx; } FileBufferIndex_t; typedef FileBufferIndex_t HashObject_t; class bcHasher { -public: - size_t operator()(const HashObject_t& rhs) const - { - return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL)); - } + public: + size_t operator()(const HashObject_t& rhs) const + { + return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL)); + } }; class bcEqual { -public: - size_t operator()(const HashObject_t& f1, const HashObject_t& f2) const - { - return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver)); - } + public: + size_t operator()(const HashObject_t& f1, const HashObject_t& f2) const + { + return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver)); + } }; inline bool operator<(const HashObject_t& f1, const HashObject_t& f2) { - //return ((f1.lbid < f2.lbid) || (f1.ver < f2.ver)); + // return ((f1.lbid < f2.lbid) || (f1.ver < f2.ver)); #if 1 - if (f1.lbid < f2.lbid) - return true; - else if (f1.lbid == f2.lbid) - return (f1.ver < f2.ver); + if (f1.lbid < f2.lbid) + return true; + else if (f1.lbid == f2.lbid) + return (f1.ver < f2.ver); - return false; + return false; #else - bcHasher bh1, bh2; - return (bh1(f1) < bh2(f2)); + bcHasher bh1, bh2; + return (bh1(f1) < bh2(f2)); #endif } - class FileBufferMgr { + public: + typedef std::tr1::unordered_set filebuffer_uset_t; + typedef std::tr1::unordered_set::const_iterator filebuffer_uset_iter_t; + typedef std::pair filebuffer_pair_t; // return type for insert -public: + typedef std::vector intvec_t; - typedef std::tr1::unordered_set filebuffer_uset_t; - typedef std::tr1::unordered_set::const_iterator filebuffer_uset_iter_t; - typedef std::pair filebuffer_pair_t; // return type for insert + /** + * @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz + **/ - typedef std::vector intvec_t; + FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0); - /** - * @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz - **/ + /** + * @brief default dtor + **/ + virtual ~FileBufferMgr(); - FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0); + /** + * @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return + *FALSE. + **/ + bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const; - /** - * @brief default dtor - **/ - virtual ~FileBufferMgr(); + /** + * @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache + *otherwise return FALSE. + **/ + bool exists(const HashObject_t& fb) const; - /** - * @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE. - **/ - bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const; + /** + * @brief add the Disk Block reference by fb into the Disk Block Buffer Cache + **/ + const int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data); - /** - * @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE. - **/ - bool exists(const HashObject_t& fb) const; + /** + * @brief returns the total number of Disk Blocks in the Cache + **/ + uint32_t size() const + { + return fbSet.size(); + } - /** - * @brief add the Disk Block reference by fb into the Disk Block Buffer Cache - **/ - const int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data); + /** + * @brief + **/ + void flushCache(); - /** - * @brief returns the total number of Disk Blocks in the Cache - **/ - uint32_t size() const - { - return fbSet.size(); - } + /** + * @brief return the disk Block referenced by fb + **/ - /** - * @brief - **/ - void flushCache(); + FileBuffer* findPtr(const HashObject_t& keyFb); - /** - * @brief return the disk Block referenced by fb - **/ + bool find(const HashObject_t& keyFb, FileBuffer& fb); - FileBuffer* findPtr(const HashObject_t& keyFb); + /** + * @brief return the disk Block referenced by bufferPtr + **/ - bool find(const HashObject_t& keyFb, FileBuffer& fb); + bool find(const HashObject_t& keyFb, void* bufferPtr); - /** - * @brief return the disk Block referenced by bufferPtr - **/ + uint32_t maxCacheSize() const + { + return fMaxNumBlocks; + } - bool find(const HashObject_t& keyFb, void* bufferPtr); + uint32_t listSize() const + { + return fbList.size(); + } - uint32_t maxCacheSize() const - { - return fMaxNumBlocks; - } + const filebuffer_uset_iter_t end() const + { + return fbSet.end(); + } - uint32_t listSize() const - { - return fbList.size(); - } + void displayCounts() const; - const filebuffer_uset_iter_t end() const - { - return fbSet.end(); - } + std::ostream& formatLRUList(std::ostream& os) const; - void displayCounts() const; + private: + uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list + uint32_t fBlockSz; // size in bytes size of a data block - probably 8 - std::ostream& formatLRUList(std::ostream& os) const; + mutable boost::mutex fWLock; + mutable filebuffer_uset_t fbSet; -private: + mutable filebuffer_list_t fbList; // rename this + uint32_t fCacheSize; - uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list - uint32_t fBlockSz; // size in bytes size of a data block - probably 8 + FileBufferPool_t fFBPool; // vector - mutable boost::mutex fWLock; - mutable filebuffer_uset_t fbSet; + // do not implement + FileBufferMgr(const FileBufferMgr& fbm); + const FileBufferMgr& operator=(const FileBufferMgr& fbm); + bool aging; - mutable filebuffer_list_t fbList; // rename this - uint32_t fCacheSize; + uint32_t fDeleteBlocks; + intvec_t fEmptyPoolSlots; // keep track of FBPool slots that can be reused - FileBufferPool_t fFBPool; // vector - - // do not implement - FileBufferMgr(const FileBufferMgr& fbm); - const FileBufferMgr& operator =(const FileBufferMgr& fbm); - bool aging; - - uint32_t fDeleteBlocks; - intvec_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused - - void depleteCache(); + void depleteCache(); }; -} +} // namespace dbbc #endif diff --git a/utils/testbc/filerequest.cpp b/utils/testbc/filerequest.cpp index 6fc0e231d..9c9799c7b 100644 --- a/utils/testbc/filerequest.cpp +++ b/utils/testbc/filerequest.cpp @@ -23,84 +23,76 @@ * * ***************************************************************************/ - #include "filerequest.h" #include using namespace std; - namespace dbbc { fileRequest::fileRequest() { - fLBID = -1; - fVer = -1; - data = NULL; - init(); - fRqstType = LBIDREQUEST; + fLBID = -1; + fVer = -1; + data = NULL; + init(); + fRqstType = LBIDREQUEST; } - -fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg) : - data(NULL), fLBID(lbid), fVer(ver), fFlg(flg) +fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg) + : data(NULL), fLBID(lbid), fVer(ver), fFlg(flg) { - init(); - fLength = 1; - fRqstType = LBIDREQUEST; + init(); + fLength = 1; + fRqstType = LBIDREQUEST; } - -fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr) : - fLBID(lbid), fVer(ver), fFlg(flg) +fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr) + : fLBID(lbid), fVer(ver), fFlg(flg) { - init(); - fLength = 1; - fRqstType = LBIDREQUEST; - data = ptr; + init(); + fLength = 1; + fRqstType = LBIDREQUEST; + data = ptr; } - -fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver) : - data(NULL), fLBID(range.start), fVer(ver), fFlg(false), fLength(range.size), - fRqstType(RANGEREQUEST) +fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver) + : data(NULL), fLBID(range.start), fVer(ver), fFlg(false), fLength(range.size), fRqstType(RANGEREQUEST) { - init(); - fLength = range.size; + init(); + fLength = range.size; } - fileRequest::fileRequest(const fileRequest& blk) { - fLBID = blk.fLBID; - fVer = blk.fVer; - fLength = blk.fLength; - fblksRead = blk.fblksRead; - fRqstType = blk.fRqstType; - fRqstStatus = blk.fRqstStatus; - data = blk.data; - init(); + fLBID = blk.fLBID; + fVer = blk.fVer; + fLength = blk.fLength; + fblksRead = blk.fblksRead; + fRqstType = blk.fRqstType; + fRqstStatus = blk.fRqstStatus; + data = blk.data; + init(); } - void fileRequest::init() { - if (pthread_mutex_init(&fFRMutex, NULL) != 0) - throw runtime_error("mutex initialization failure"); + if (pthread_mutex_init(&fFRMutex, NULL) != 0) + throw runtime_error("mutex initialization failure"); - if (pthread_cond_init(&fFRCond, NULL) != 0) - throw runtime_error("cond var initialization failure"); + if (pthread_cond_init(&fFRCond, NULL) != 0) + throw runtime_error("cond var initialization failure"); - fFRPredicate = INIT; - fLength = 0; - fblksRead = 0; - fRqstStatus = 0; + fFRPredicate = INIT; + fLength = 0; + fblksRead = 0; + fRqstStatus = 0; } fileRequest::~fileRequest() { - pthread_mutex_destroy(&fFRMutex); - pthread_cond_destroy(&fFRCond); + pthread_mutex_destroy(&fFRMutex); + pthread_cond_destroy(&fFRCond); } -} +} // namespace dbbc diff --git a/utils/testbc/filerequest.h b/utils/testbc/filerequest.h index 5aa4ba4f7..56f34d50a 100644 --- a/utils/testbc/filerequest.h +++ b/utils/testbc/filerequest.h @@ -28,9 +28,8 @@ #include "brmtypes.h" - /** - @author Jason Rodriguez + @author Jason Rodriguez */ /** @@ -38,222 +37,218 @@ **/ namespace dbbc { - class fileRequest { + public: + /** + * @brief default ctor + **/ + fileRequest(); -public: + /** + * @brief copy constructor + **/ + fileRequest(const fileRequest& blk); - /** - * @brief default ctor - **/ - fileRequest(); + /** + * @brief request for the disk block lbid@ver + **/ + fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg); - /** - * @brief copy constructor - **/ - fileRequest(const fileRequest& blk); + fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr); + /** + * @brief request a range of disk blocks + **/ + fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver); - /** - * @brief request for the disk block lbid@ver - **/ - fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg); + /** + * @brief class dtor + **/ + virtual ~fileRequest(); - fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr); - /** - * @brief request a range of disk blocks - **/ - fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver); + /** + * @brief less-than operator + **/ + bool operator<(const fileRequest& rhs) const + { + return fLength < rhs.fLength; + } - /** - * @brief class dtor - **/ - virtual ~fileRequest(); + /** + * @brief greater-than operator + **/ + bool operator>(const fileRequest& rhs) const + { + return fLength > rhs.fLength; + } - /** - * @brief less-than operator - **/ - bool operator< (const fileRequest& rhs) const - { - return fLength < rhs.fLength; - } + /** + * @brief equality operator + **/ + bool operator==(const fileRequest& rhs) const + { + return fLength == rhs.fLength; + } - /** - * @brief greater-than operator - **/ - bool operator> (const fileRequest& rhs) const - { - return fLength > rhs.fLength; - } + enum request_status_enum + { + SUCCESSFUL, + FAILED + }; - /** - * @brief equality operator - **/ - bool operator== (const fileRequest& rhs) const - { - return fLength == rhs.fLength; - } + enum request_type_enum + { + LBIDREQUEST, + RANGEREQUEST + }; - enum request_status_enum - { - SUCCESSFUL, - FAILED - }; + /** + * @brief used to manage request processing synchronzation + **/ + enum predicate_status_enum + { + INIT, + SENDING, + READING, + COMPLETE, + STOP + }; - enum request_type_enum - { - LBIDREQUEST, - RANGEREQUEST - }; + /** + * @brief lbid requested + **/ + const BRM::LBID_t Lbid() const + { + return fLBID; + } - /** - * @brief used to manage request processing synchronzation - **/ - enum predicate_status_enum - { - INIT, - SENDING, - READING, - COMPLETE, - STOP - }; + /** + * @brief version of the lbid requested + **/ + const BRM::VER_t Ver() const + { + return fVer; + } - /** - * @brief lbid requested - **/ - const BRM::LBID_t Lbid() const - { - return fLBID; - } + /** + * @brief VBBM flag of the LBID/Ver + **/ + const bool Flg() const + { + return fFlg; + } - /** - * @brief version of the lbid requested - **/ - const BRM::VER_t Ver() const - { - return fVer; - } + /** + * @brief number of blocks requested + **/ + const uint32_t BlocksRequested() const + { + return fLength; + } - /** - * @brief VBBM flag of the LBID/Ver - **/ - const bool Flg() const - { - return fFlg; - } + /** + * @brief setter for blocks requested + **/ + void BlocksRequested(const int l) + { + fLength = l; + } - /** - * @brief number of blocks requested - **/ - const uint32_t BlocksRequested() const - { - return fLength; - } + /** + * @brief number of blocks read from disk + **/ + const uint32_t BlocksRead() const + { + return fblksRead; + } + const uint32_t BlocksLoaded() const + { + return fblksLoaded; + } - /** - * @brief setter for blocks requested - **/ - void BlocksRequested(const int l) - { - fLength = l; - } + /** + * @brief setter for blocks read from disk + **/ + void BlocksRead(const int l) + { + fblksRead = l; + } + void BlocksLoaded(const int l) + { + fblksLoaded = l; + } - /** - * @brief number of blocks read from disk - **/ - const uint32_t BlocksRead() const - { - return fblksRead; - } - const uint32_t BlocksLoaded() const - { - return fblksLoaded; - } + /** + * @brief did the request succeed for fail + **/ + int RequestStatus() const + { + return fRqstStatus; + } - /** - * @brief setter for blocks read from disk - **/ - void BlocksRead(const int l) - { - fblksRead = l; - } - void BlocksLoaded(const int l) - { - fblksLoaded = l; - } + /** + * @brief setter for the request status + **/ + void RequestStatus(int s) + { + fRqstStatus = s; + } - /** - * @brief did the request succeed for fail - **/ - int RequestStatus() const - { - return fRqstStatus; - } + /** + * @brief return BLOCK or RANGE requested + **/ + int RequestType() const + { + return fRqstType; + } - /** - * @brief setter for the request status - **/ - void RequestStatus(int s) - { - fRqstStatus = s; - } + /** + * @brief mutex to control synchronzation of request processing + **/ + pthread_mutex_t& frMutex() const + { + return fFRMutex; + } - /** - * @brief return BLOCK or RANGE requested - **/ - int RequestType() const - { - return fRqstType; - } + /** + * @brief condition variable. signal when request is complete + **/ + pthread_cond_t& frCond() const + { + return fFRCond; + } - /** - * @brief mutex to control synchronzation of request processing - **/ - pthread_mutex_t& frMutex() const - { - return fFRMutex; - } + /** + * @brief + **/ + const enum predicate_status_enum& frPredicate() const + { + return fFRPredicate; + } - /** - * @brief condition variable. signal when request is complete - **/ - pthread_cond_t& frCond() const - { - return fFRCond; - } + /** + * @brief setter for the predicate + **/ + void SetPredicate(const enum predicate_status_enum& p) + { + fFRPredicate = p; + } - /** - * @brief - **/ - const enum predicate_status_enum& frPredicate() const - { - return fFRPredicate; - } + uint8_t* data; - /** - * @brief setter for the predicate - **/ - void SetPredicate(const enum predicate_status_enum& p) - { - fFRPredicate = p; - } - - uint8_t* data; - -private: - - void init(); - BRM::LBID_t fLBID; - BRM::VER_t fVer; - bool fFlg; - mutable pthread_mutex_t fFRMutex; - mutable pthread_cond_t fFRCond; - predicate_status_enum fFRPredicate; - uint32_t fLength; // lbids requested - uint32_t fblksRead; // lbids read - uint32_t fblksLoaded; // lbids loaded into cache - int fRqstStatus; - enum request_type_enum fRqstType; + private: + void init(); + BRM::LBID_t fLBID; + BRM::VER_t fVer; + bool fFlg; + mutable pthread_mutex_t fFRMutex; + mutable pthread_cond_t fFRCond; + predicate_status_enum fFRPredicate; + uint32_t fLength; // lbids requested + uint32_t fblksRead; // lbids read + uint32_t fblksLoaded; // lbids loaded into cache + int fRqstStatus; + enum request_type_enum fRqstType; }; -} +} // namespace dbbc #endif diff --git a/utils/testbc/iomanager.cpp b/utils/testbc/iomanager.cpp index 47d7614b4..7c3cc4d6c 100644 --- a/utils/testbc/iomanager.cpp +++ b/utils/testbc/iomanager.cpp @@ -53,297 +53,276 @@ using namespace config; #include "iomanager.h" namespace { - using namespace dbbc; using namespace std; -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } struct IOMThreadArg { - ioManager* iom; - int32_t thdId; + ioManager* iom; + int32_t thdId; }; typedef IOMThreadArg IOMThreadArg_t; void* thr_popper(void* arg) { - ioManager* iom = ((IOMThreadArg*)arg)->iom; - int32_t iomThdId = ((IOMThreadArg*)arg)->thdId; - FileBufferMgr* fbm; - int totalRqst = 0; - fileRequest* fr = 0; - BRM::LBID_t lbid = 0; - BRM::OID_t oid = 0; - BRM::VER_t ver = 0; - int blocksLoaded = 0; - int blocksRead = 0; - const unsigned pageSize = 4096; - fbm = &iom->fileBufferManager(); - char fileName[WriteEngine::FILE_NAME_SIZE]; - const uint64_t fileBlockSize = BLOCK_SIZE; - uint32_t offset = 0; - bool flg = false; - char* fileNamePtr = fileName; - uint64_t longSeekOffset = 0; - int err; - uint32_t dlen = 0, acc, readSize, blocksThisRead, j; - uint32_t blocksRequested = 0; - ssize_t i; - uint32_t sz = 0; - char* alignedbuff = 0; - boost::scoped_array realbuff; - pthread_t threadId = 0; - ostringstream iomLogFileName; - ofstream lFile; + ioManager* iom = ((IOMThreadArg*)arg)->iom; + int32_t iomThdId = ((IOMThreadArg*)arg)->thdId; + FileBufferMgr* fbm; + int totalRqst = 0; + fileRequest* fr = 0; + BRM::LBID_t lbid = 0; + BRM::OID_t oid = 0; + BRM::VER_t ver = 0; + int blocksLoaded = 0; + int blocksRead = 0; + const unsigned pageSize = 4096; + fbm = &iom->fileBufferManager(); + char fileName[WriteEngine::FILE_NAME_SIZE]; + const uint64_t fileBlockSize = BLOCK_SIZE; + uint32_t offset = 0; + bool flg = false; + char* fileNamePtr = fileName; + uint64_t longSeekOffset = 0; + int err; + uint32_t dlen = 0, acc, readSize, blocksThisRead, j; + uint32_t blocksRequested = 0; + ssize_t i; + uint32_t sz = 0; + char* alignedbuff = 0; + boost::scoped_array realbuff; + pthread_t threadId = 0; + ostringstream iomLogFileName; + ofstream lFile; - threadId = pthread_self(); + threadId = pthread_self(); - uint32_t readBufferSz = iom->blocksPerRead * BLOCK_SIZE + pageSize; + uint32_t readBufferSz = iom->blocksPerRead * BLOCK_SIZE + pageSize; - realbuff.reset(new char[readBufferSz]); + realbuff.reset(new char[readBufferSz]); - if (realbuff.get() == 0) - { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return 0; - } + if (realbuff.get() == 0) + { + cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; + return 0; + } #if __WORDSIZE > 32 - //alignedbuff=(char*)((((ptrdiff_t)&realbuff[0] + pageSize - 1) / pageSize) * pageSize); - // pagesize == (1 << 12) - alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize); + // alignedbuff=(char*)((((ptrdiff_t)&realbuff[0] + pageSize - 1) / pageSize) * pageSize); + // pagesize == (1 << 12) + alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize); #else - //alignedbuff=(char*)(((((ptrdiff_t)&realbuff[0] & 0xffffffff) + pageSize - 1) / pageSize) * pageSize); - alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize); + // alignedbuff=(char*)(((((ptrdiff_t)&realbuff[0] & 0xffffffff) + pageSize - 1) / pageSize) * pageSize); + alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize); #endif - idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize); - idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0); + idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize); + idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0); - for ( ; ; ) + for (;;) + { + fr = iom->getNextRequest(); + lbid = fr->Lbid(); + ver = fr->Ver(); + flg = fr->Flg(); + blocksLoaded = 0; + blocksRead = 0; + dlen = fr->BlocksRequested(); + blocksRequested = fr->BlocksRequested(); + + err = iom->lbidLookup(lbid, ver, flg, oid, offset); + + if (err < 0) { + cerr << "lbid=" << lbid << " ver=" << ver << " flg=" << (flg ? 1 : 0) << endl; + throw runtime_error("thr_popper: BRM lookup failure"); + } - fr = iom->getNextRequest(); - lbid = fr->Lbid(); - ver = fr->Ver(); - flg = fr->Flg(); - blocksLoaded = 0; - blocksRead = 0; - dlen = fr->BlocksRequested(); - blocksRequested = fr->BlocksRequested(); + longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize; + totalRqst++; + sz = 0; - err = iom->lbidLookup(lbid, ver, flg, oid, offset); + uint32_t readCount = 0; + uint32_t bytesRead = 0; + uint32_t jend = blocksRequested / iom->blocksPerRead; - if (err < 0) + for (j = 0; j <= jend; j++) + { + blocksThisRead = std::min(dlen, iom->blocksPerRead); + readSize = blocksThisRead * BLOCK_SIZE; + + acc = 0; + + while (acc < readSize) + { + i = readSize; // pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset); + + /* XXXPAT: Need to decide how to handle errors here */ + if (i < 0 && errno == EINTR) { - cerr << "lbid=" << lbid << " ver=" << ver << " flg=" << (flg ? 1 : 0) << endl; - throw runtime_error("thr_popper: BRM lookup failure"); + continue; + } + else if (i < 0) + { + perror("thr_popper::read"); + return 0; // shuts down this thread, + // probably not the right thing to do + } + else if (i == 0) + { + try + { + } + catch (exception& exc) + { + cerr << "FileName Err:" << exc.what() << endl; + } + + cerr << "thr_popper: Early EOF in file " << fileNamePtr << endl; + return 0; } - longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize; - totalRqst++; - sz = 0; + acc += i; + longSeekOffset += (uint64_t)i; + readCount++; + bytesRead += i; + } // while(acc... - uint32_t readCount = 0; - uint32_t bytesRead = 0; - uint32_t jend = blocksRequested / iom->blocksPerRead; + blocksRead += blocksThisRead; - for (j = 0; j <= jend; j++) - { + for (i = 0; (unsigned)i < blocksThisRead; ++i) + { + if (fbm->insert((lbid + i) + (j * iom->blocksPerRead), ver, (uint8_t*)&alignedbuff[i * BLOCK_SIZE])) + ++blocksLoaded; + } - blocksThisRead = std::min(dlen, iom->blocksPerRead); - readSize = blocksThisRead * BLOCK_SIZE; + dlen -= blocksThisRead; + } // for (j... - acc = 0; + fr->BlocksRead(blocksRead); + fr->BlocksLoaded(blocksLoaded); - while (acc < readSize) - { - i = readSize; //pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset); + if (fr->data != 0 && blocksRequested == 1) + memcpy(fr->data, alignedbuff, BLOCK_SIZE); - /* XXXPAT: Need to decide how to handle errors here */ - if (i < 0 && errno == EINTR) - { - continue; - } - else if (i < 0) - { - perror("thr_popper::read"); - return 0; // shuts down this thread, - // probably not the right thing to do - } - else if (i == 0) - { - try - { + pthread_mutex_lock(&fr->frMutex()); + fr->SetPredicate(fileRequest::COMPLETE); + pthread_cond_signal(&fr->frCond()); + pthread_mutex_unlock(&fr->frMutex()); - } - catch (exception& exc) - { - cerr << "FileName Err:" << exc.what() << endl; - } + } // for(;;) - cerr << "thr_popper: Early EOF in file " << fileNamePtr << endl; - return 0; - } + lFile.close(); - acc += i; - longSeekOffset += (uint64_t)i; - readCount++; - bytesRead += i; - } // while(acc... + return 0; +} // end thr_popper - blocksRead += blocksThisRead; - - for (i = 0; (unsigned)i < blocksThisRead; ++i) - { - if (fbm->insert( (lbid + i) + (j * iom->blocksPerRead), ver, (uint8_t*)&alignedbuff[i * BLOCK_SIZE])) - ++blocksLoaded; - } - - dlen -= blocksThisRead; - } // for (j... - - fr->BlocksRead(blocksRead); - fr->BlocksLoaded(blocksLoaded); - - if (fr->data != 0 && blocksRequested == 1) - memcpy(fr->data, alignedbuff, BLOCK_SIZE); - - pthread_mutex_lock(&fr->frMutex()); - fr->SetPredicate(fileRequest::COMPLETE); - pthread_cond_signal(&fr->frCond()); - pthread_mutex_unlock(&fr->frMutex()); - - } // for(;;) - - lFile.close(); - - return 0; -} // end thr_popper - -} // anonymous namespace +} // anonymous namespace namespace dbbc { - -ioManager::ioManager(FileBufferMgr& fbm, - fileBlockRequestQueue& fbrq, - int thrCount, - int bsPerRead): - blocksPerRead(bsPerRead), - fIOMfbMgr(fbm), - fIOMRequestQueue(fbrq) +ioManager::ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead) + : blocksPerRead(bsPerRead), fIOMfbMgr(fbm), fIOMRequestQueue(fbrq) { + if (thrCount <= 0) + thrCount = 1; - if (thrCount <= 0) - thrCount = 1; + if (thrCount > 256) + thrCount = 256; - if (thrCount > 256) - thrCount = 256; + fConfig = Config::makeConfig(); + string val = fConfig->getConfig("DBBC", "IOMTracing"); + int temp = 0; - fConfig = Config::makeConfig(); - string val = fConfig->getConfig("DBBC", "IOMTracing"); - int temp = 0; + if (val.length() > 0) + temp = static_cast(Config::fromText(val)); - if (val.length() > 0) temp = static_cast(Config::fromText(val)); + if (temp > 0) + fIOTrace = true; + else + fIOTrace = false; - if (temp > 0) - fIOTrace = true; - else - fIOTrace = false; + fThreadCount = thrCount; + go(); - fThreadCount = thrCount; - go(); - -} // ioManager +} // ioManager void ioManager::buildOidFileName(const BRM::OID_t oid, char* file_name) { - - if (fFileOp.getFileName(oid, file_name) != WriteEngine::NO_ERROR) - { - file_name[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } + if (fFileOp.getFileName(oid, file_name) != WriteEngine::NO_ERROR) + { + file_name[0] = 0; + throw std::runtime_error("fileOp.getFileName failed"); + } } - -BRM::LBID_t ioManager::lbidLookup(BRM::LBID_t lbid, - BRM::VER_t verid, - bool vbFlag, - BRM::OID_t& oid, +BRM::LBID_t ioManager::lbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, uint32_t& offset) { - int rc = fdbrm.lookup(lbid, verid, vbFlag, oid, offset); - return rc; + int rc = fdbrm.lookup(lbid, verid, vbFlag, oid, offset); + return rc; } int ioManager::createReaders() { - int realCnt = 0; - IOMThreadArg_t fThdArgArr[256]; + int realCnt = 0; + IOMThreadArg_t fThdArgArr[256]; - for (int idx = 0; idx < fThreadCount; idx++) - { - fThdArgArr[realCnt].iom = this; - fThdArgArr[realCnt].thdId = realCnt; - int ret = pthread_create(&fThreadArr[realCnt], 0, thr_popper, &fThdArgArr[realCnt]); + for (int idx = 0; idx < fThreadCount; idx++) + { + fThdArgArr[realCnt].iom = this; + fThdArgArr[realCnt].thdId = realCnt; + int ret = pthread_create(&fThreadArr[realCnt], 0, thr_popper, &fThdArgArr[realCnt]); - if (ret != 0) - perror("createReaders::pthread_create"); - else - realCnt++; - } + if (ret != 0) + perror("createReaders::pthread_create"); + else + realCnt++; + } - fThreadCount = realCnt; - return fThreadCount; + fThreadCount = realCnt; + return fThreadCount; } - ioManager::~ioManager() { - stop(); + stop(); } void ioManager::go(void) { - createReaders(); + createReaders(); } - void ioManager::stop() { - for (int idx = 0; idx < fThreadCount; idx++) - { - pthread_detach(fThreadArr[idx]); - } + for (int idx = 0; idx < fThreadCount; idx++) + { + pthread_detach(fThreadArr[idx]); + } } - fileRequest* ioManager::getNextRequest() { - fileRequest* blk = 0; - - try - { - blk = fIOMRequestQueue.pop(); - return blk; - } - catch (exception& e) - { - cerr << "ioManager::getNextRequest() ERROR " << endl; - } + fileRequest* blk = 0; + try + { + blk = fIOMRequestQueue.pop(); return blk; + } + catch (exception& e) + { + cerr << "ioManager::getNextRequest() ERROR " << endl; + } + return blk; } -} +} // namespace dbbc // vim:ts=4 sw=4: diff --git a/utils/testbc/iomanager.h b/utils/testbc/iomanager.h index b47efaf59..20704c58d 100644 --- a/utils/testbc/iomanager.h +++ b/utils/testbc/iomanager.h @@ -30,7 +30,7 @@ // /** - @author Jason Rodriguez + @author Jason Rodriguez */ #include @@ -44,69 +44,60 @@ namespace dbbc { - class ioManager { + public: + ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead); + // ioManager(FileBufferMgr& fbm, int thrCount); + ~ioManager(); + int readerCount() const + { + return fThreadCount; + } + fileRequest* getNextRequest(); + void go(void); + void stop(); + FileBufferMgr& fileBufferManager() + { + return fIOMfbMgr; + } + config::Config* configPtr() + { + return fConfig; + } + BRM::LBID_t lbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, uint32_t& offset); + void buildOidFileName(const BRM::OID_t oid, char* file_name); -public: + uint32_t getExtentSize() + { + return fdbrm.getExtentSize(); + } - ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, - int bsPerRead); - //ioManager(FileBufferMgr& fbm, int thrCount); - ~ioManager(); - int readerCount() const - { - return fThreadCount; - } - fileRequest* getNextRequest(); - void go(void); - void stop(); - FileBufferMgr& fileBufferManager() - { - return fIOMfbMgr; - } - config::Config* configPtr() - { - return fConfig; - } - BRM::LBID_t lbidLookup(BRM::LBID_t lbid, - BRM::VER_t verid, - bool vbFlag, - BRM::OID_t& oid, - uint32_t& offset); - void buildOidFileName(const BRM::OID_t oid, char* file_name); + uint32_t blocksPerRead; - uint32_t getExtentSize() - { - return fdbrm.getExtentSize(); - } + bool IOTrace() const + { + return fIOTrace; + } - uint32_t blocksPerRead; + private: + FileBufferMgr& fIOMfbMgr; + fileBlockRequestQueue& fIOMRequestQueue; + int fThreadCount; + pthread_t fPoppertid; + pthread_t fThreadArr[256]; + int createReaders(); + config::Config* fConfig; + BRM::DBRM fdbrm; + WriteEngine::FileOp fFileOp; - bool IOTrace() const - { - return fIOTrace; - } - -private: - - FileBufferMgr& fIOMfbMgr; - fileBlockRequestQueue& fIOMRequestQueue; - int fThreadCount; - pthread_t fPoppertid; - pthread_t fThreadArr[256]; - int createReaders(); - config::Config* fConfig; - BRM::DBRM fdbrm; - WriteEngine::FileOp fFileOp; - - // do not implement - ioManager(); - ioManager(const ioManager& iom); - const ioManager& operator=(const ioManager& iom); - bool fIOTrace; + // do not implement + ioManager(); + ioManager(const ioManager& iom); + const ioManager& operator=(const ioManager& iom); + bool fIOTrace; }; -} +} // namespace dbbc #endif // vim:ts=4 sw=4: diff --git a/utils/testbc/logger.cpp b/utils/testbc/logger.cpp index 42147fdf8..a9912fb77 100644 --- a/utils/testbc/logger.cpp +++ b/utils/testbc/logger.cpp @@ -31,38 +31,33 @@ using namespace logging; namespace primitiveprocessor { - -Logger::Logger() : - fMl1(LoggingID(28)) +Logger::Logger() : fMl1(LoggingID(28)) { - fMsgMap[logging::M0000] = Message(logging::M0000); - fMsgMap[logging::M0016] = Message(logging::M0016); - fMsgMap[logging::M0045] = Message(logging::M0045); - fMsgMap[logging::M0053] = Message(logging::M0053); + fMsgMap[logging::M0000] = Message(logging::M0000); + fMsgMap[logging::M0016] = Message(logging::M0016); + fMsgMap[logging::M0045] = Message(logging::M0045); + fMsgMap[logging::M0053] = Message(logging::M0053); } -void Logger::logMessage(const Message::MessageID mid, - const Message::Args& args, - bool critical) +void Logger::logMessage(const Message::MessageID mid, const Message::Args& args, bool critical) { - mutex::scoped_lock lk(fLogLock); - MsgMap::iterator msgIter = fMsgMap.find(mid); + mutex::scoped_lock lk(fLogLock); + MsgMap::iterator msgIter = fMsgMap.find(mid); - if (msgIter == fMsgMap.end()) - msgIter = fMsgMap.find(logging::M0000); + if (msgIter == fMsgMap.end()) + msgIter = fMsgMap.find(logging::M0000); - msgIter->second.reset(); - msgIter->second.format(args); - - if (critical) - { - fMl1.logCriticalMessage(msgIter->second); - } - else - { - fMl1.logWarningMessage(msgIter->second); - } -} + msgIter->second.reset(); + msgIter->second.format(args); + if (critical) + { + fMl1.logCriticalMessage(msgIter->second); + } + else + { + fMl1.logWarningMessage(msgIter->second); + } } +} // namespace primitiveprocessor diff --git a/utils/testbc/logger.h b/utils/testbc/logger.h index 8fd85e697..6e8a03836 100644 --- a/utils/testbc/logger.h +++ b/utils/testbc/logger.h @@ -32,38 +32,34 @@ namespace primitiveprocessor { - /** @brief message log wrapper class */ class Logger { -public: - Logger(); + public: + Logger(); - void logMessage(const logging::Message::MessageID mid, - const logging::Message::Args& args, - bool critical = false); + void logMessage(const logging::Message::MessageID mid, const logging::Message::Args& args, + bool critical = false); - void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 ) - { - logging::Message::Args args; - args.add(msg); - logMessage(mid, args, true); - } + void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0) + { + logging::Message::Args args; + args.add(msg); + logMessage(mid, args, true); + } -private: - // defaults okay - //Logger(const Logger& rhs); - //Logger& operator=(const Logger& rhs); + private: + // defaults okay + // Logger(const Logger& rhs); + // Logger& operator=(const Logger& rhs); - typedef std::map MsgMap; + typedef std::map MsgMap; - MsgMap fMsgMap; - boost::mutex fLogLock; - logging::MessageLog fMl1; + MsgMap fMsgMap; + boost::mutex fLogLock; + logging::MessageLog fMl1; }; - -} +} // namespace primitiveprocessor #endif - diff --git a/utils/testbc/stats.cpp b/utils/testbc/stats.cpp index 524275eb5..bc5d1e7ff 100644 --- a/utils/testbc/stats.cpp +++ b/utils/testbc/stats.cpp @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /* -* $Id: stats.cpp 699 2008-09-09 19:44:18Z rdempsey $ -*/ + * $Id: stats.cpp 699 2008-09-09 19:44:18Z rdempsey $ + */ #include #include @@ -41,237 +41,233 @@ using namespace BRM; namespace { - void pause_(unsigned delay) { - struct timespec req; - struct timespec rem; + struct timespec req; + struct timespec rem; - req.tv_sec = delay; - req.tv_nsec = 0; + req.tv_sec = delay; + req.tv_nsec = 0; - rem.tv_sec = 0; - rem.tv_nsec = 0; + rem.tv_sec = 0; + rem.tv_nsec = 0; again: - if (nanosleep(&req, &rem) != 0) - if (rem.tv_sec > 0 || rem.tv_nsec > 0) - { - req = rem; - goto again; - } + if (nanosleep(&req, &rem) != 0) + if (rem.tv_sec > 0 || rem.tv_nsec > 0) + { + req = rem; + goto again; + } } const string timestr() { - // Get a timestamp for output. - struct tm tm; - struct timeval tv; + // Get a timestamp for output. + struct tm tm; + struct timeval tv; - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); - ostringstream oss; - oss << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec << '.' - << setw(4) << tv.tv_usec / 100; + ostringstream oss; + oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec + << '.' << setw(4) << tv.tv_usec / 100; - return oss.str(); + return oss.str(); } class TraceFile { -public: - TraceFile(uint32_t sessionID, const char* name) + public: + TraceFile(uint32_t sessionID, const char* name) + { + if (sessionID > 0) { - if (sessionID > 0 ) - { - const char* outName; + const char* outName; - if (name == 0) - outName = "lbids"; - else - outName = name; + if (name == 0) + outName = "lbids"; + else + outName = name; - ostringstream oss; - oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID; - oFile.reset(new ofstream()); - oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); - } + ostringstream oss; + oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID; + oFile.reset(new ofstream()); + oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); } + } - ~TraceFile() + ~TraceFile() + { + } + + void close() + { + if (oFile) { + oFile->close(); } + } - void close() - { - if (oFile) - { - oFile->close(); - } - } + void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') + { + *oFile << oid << ' ' << timestr() << ' ' << lbid << ' ' << thdid; - void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') - { - *oFile << oid << ' ' << timestr() << ' ' << lbid - << ' ' << thdid; + if (event != '\0') + *oFile << ' ' << event; - if (event != '\0') - *oFile << ' ' << event; - - *oFile << endl; - oFile->flush(); - } - -private: - //Compiler defaults okay - //TraceFile(const TraceFile& rhs); - //TraceFile operator=(const TraceFile& rhs); - shared_ptr oFile; + *oFile << endl; + oFile->flush(); + } + private: + // Compiler defaults okay + // TraceFile(const TraceFile& rhs); + // TraceFile operator=(const TraceFile& rhs); + shared_ptr oFile; }; struct TraceFileInfo { - TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) { } - ~TraceFileInfo() { } + TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) + { + } + ~TraceFileInfo() + { + } - void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') - { - traceFile.log(oid, lbid, thdid, event); - lastTouched = time(0); - } + void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') + { + traceFile.log(oid, lbid, thdid, event); + lastTouched = time(0); + } - void close() - { - traceFile.close(); - } + void close() + { + traceFile.close(); + } - TraceFile traceFile; - time_t lastTouched; + TraceFile traceFile; + time_t lastTouched; -private: - //Compiler defaults okay - //TraceFileInfo(const TraceFileInfo& rhs); - //TraceFileInfo operator=(const TraceFileInfo& rhs); + private: + // Compiler defaults okay + // TraceFileInfo(const TraceFileInfo& rhs); + // TraceFileInfo operator=(const TraceFileInfo& rhs); }; -//map a session id to a trace file +// map a session id to a trace file typedef map TraceFileMap_t; TraceFileMap_t traceFileMap; -//map mutex +// map mutex mutex traceFileMapMutex; class StatMon { -public: - StatMon() - { - sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGPIPE); - sigaddset(&sigset, SIGUSR1); - sigaddset(&sigset, SIGUSR2); - pthread_sigmask(SIG_BLOCK, &sigset, 0); - } - void operator()() const - { - //struct timespec ts = { 60 * 1, 0 }; - mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter; - TraceFileMap_t::iterator end; + public: + StatMon() + { + sigset_t sigset; + sigemptyset(&sigset); + sigaddset(&sigset, SIGPIPE); + sigaddset(&sigset, SIGUSR1); + sigaddset(&sigset, SIGUSR2); + pthread_sigmask(SIG_BLOCK, &sigset, 0); + } + void operator()() const + { + // struct timespec ts = { 60 * 1, 0 }; + mutex::scoped_lock lk(traceFileMapMutex); + TraceFileMap_t::iterator iter; + TraceFileMap_t::iterator end; - for (;;) + for (;;) + { + lk.unlock(); + time_t beforeSleep = time(0); + // nanosleep(&ts, 0); + pause_(60); + lk.lock(); + iter = traceFileMap.begin(); + end = traceFileMap.end(); + + while (iter != end) + { + if (iter->second.lastTouched < beforeSleep) { - lk.unlock(); - time_t beforeSleep = time(0); - //nanosleep(&ts, 0); - pause_(60); - lk.lock(); - iter = traceFileMap.begin(); - end = traceFileMap.end(); - - while (iter != end) - { - if (iter->second.lastTouched < beforeSleep) - { - //remove this session trace file - iter->second.close(); - traceFileMap.erase(iter++); - } - else - ++iter; - } + // remove this session trace file + iter->second.close(); + traceFileMap.erase(iter++); } + else + ++iter; + } } -private: - //Compiler defaults okay - //StatMon(const StatMon& rhs); - //StatMon operator=(const StatMon& rhs); + } + + private: + // Compiler defaults okay + // StatMon(const StatMon& rhs); + // StatMon operator=(const StatMon& rhs); }; -} +} // namespace namespace dbbc { - -Stats::Stats() : - fMonitorp(0) +Stats::Stats() : fMonitorp(0) { - - fMonitorp = new thread(StatMon()); + fMonitorp = new thread(StatMon()); } -Stats::Stats(const char* name) : - fMonitorp(0), fName(name) +Stats::Stats(const char* name) : fMonitorp(0), fName(name) { - fMonitorp = new thread(StatMon()); - //fName << name; + fMonitorp = new thread(StatMon()); + // fName << name; } Stats::~Stats() { - delete fMonitorp; + delete fMonitorp; } void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session) { - if (lbid < 0 || session == 0) return; + if (lbid < 0 || session == 0) + return; - mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter = traceFileMap.find(session); + mutex::scoped_lock lk(traceFileMapMutex); + TraceFileMap_t::iterator iter = traceFileMap.find(session); - if (iter == traceFileMap.end()) - { - traceFileMap[session] = TraceFileInfo(session); - iter = traceFileMap.find(session); - idbassert(iter != traceFileMap.end()); - } + if (iter == traceFileMap.end()) + { + traceFileMap[session] = TraceFileInfo(session); + iter = traceFileMap.find(session); + idbassert(iter != traceFileMap.end()); + } - iter->second.log(lbid2oid(lbid), lbid, thdid); + iter->second.log(lbid2oid(lbid), lbid, thdid); } void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event) { - if (lbid < 0 || session == 0) return; + if (lbid < 0 || session == 0) + return; - mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter = traceFileMap.find(session); + mutex::scoped_lock lk(traceFileMapMutex); + TraceFileMap_t::iterator iter = traceFileMap.find(session); - if (iter == traceFileMap.end()) - { - traceFileMap[session] = TraceFileInfo(session, fName); - iter = traceFileMap.find(session); - idbassert(iter != traceFileMap.end()); - } - - iter->second.log(lbid2oid(lbid), lbid, thdid, event); -} + if (iter == traceFileMap.end()) + { + traceFileMap[session] = TraceFileInfo(session, fName); + iter = traceFileMap.find(session); + idbassert(iter != traceFileMap.end()); + } + iter->second.log(lbid2oid(lbid), lbid, thdid, event); } +} // namespace dbbc diff --git a/utils/testbc/stats.h b/utils/testbc/stats.h index 4a1b33b7a..e0cec2501 100644 --- a/utils/testbc/stats.h +++ b/utils/testbc/stats.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: stats.h 609 2008-06-11 12:40:07Z rdempsey $ -*/ + * $Id: stats.h 609 2008-06-11 12:40:07Z rdempsey $ + */ /** @file */ @@ -33,36 +33,34 @@ namespace dbbc { - class Stats { -public: - Stats(); - Stats(const char* name); - virtual ~Stats(); + public: + Stats(); + Stats(const char* name); + virtual ~Stats(); - void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0); - void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event); + void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0); + void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event); - inline BRM::OID_t lbid2oid(uint64_t lbid) - { - BRM::OID_t oid; - uint32_t fbo; - brm.lookup(lbid, 0, false, oid, fbo); - return oid; - } + inline BRM::OID_t lbid2oid(uint64_t lbid) + { + BRM::OID_t oid; + uint32_t fbo; + brm.lookup(lbid, 0, false, oid, fbo); + return oid; + } -private: - Stats(const Stats& rhs); - Stats& operator=(const Stats& rhs); - - boost::thread* fMonitorp; - BRM::DBRM brm; - //ostringstream fName; - const char* fName; + private: + Stats(const Stats& rhs); + Stats& operator=(const Stats& rhs); + boost::thread* fMonitorp; + BRM::DBRM brm; + // ostringstream fName; + const char* fName; }; -} +} // namespace dbbc -#endif //PRIMPROC_STATS_H +#endif // PRIMPROC_STATS_H diff --git a/utils/testbc/testbc.cpp b/utils/testbc/testbc.cpp index d3908047c..5be43c7d2 100644 --- a/utils/testbc/testbc.cpp +++ b/utils/testbc/testbc.cpp @@ -50,11 +50,9 @@ bool gPMProfOn = false; uint32_t gSession = 0; uint32_t lastRangeListIdx = 0; -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } namespace primitiveprocessor @@ -62,317 +60,302 @@ namespace primitiveprocessor Logger ml; } - class BCTest { - -public: - - struct OidRanges + public: + struct OidRanges + { + OID_t oid; + HWM_t hwm; + LBIDRange_v ranges; + OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) { - OID_t oid; - HWM_t hwm; - LBIDRange_v ranges; - OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) - { - oid = o; - hwm = h; - ranges = r; - } - }; //struct OidRanges + oid = o; + hwm = h; + ranges = r; + } + }; // struct OidRanges - BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); + BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); - typedef OidRanges OidRanges_t; - typedef vectorOidRangesList_t; - OidRangesList_t OidRangesList; + typedef OidRanges OidRanges_t; + typedef vector OidRangesList_t; + OidRangesList_t OidRangesList; - DBRM dbrm; - uint32_t extentSize; - BRM::OID_t maxOid; + DBRM dbrm; + uint32_t extentSize; + BRM::OID_t maxOid; - int fCacheSz; - int fReadThr; - int fReadAhead; - uint32_t maxBlocksAvailable; - uint32_t fExtentSize; + int fCacheSz; + int fReadThr; + int fReadAhead; + uint32_t maxBlocksAvailable; + uint32_t fExtentSize; - void setUp(); - int LoadOid(const OidRanges_t& o, uint32_t& loadCount); - void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); - int ReadOidRanges(const OidRanges_t& v); - void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); + void setUp(); + int LoadOid(const OidRanges_t& o, uint32_t& loadCount); + void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); + int ReadOidRanges(const OidRanges_t& v); + void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); - BlockRequestProcessor BRP; + BlockRequestProcessor BRP; -}; // class BCTest +}; // class BCTest - -BCTest::BCTest(int cacheSz, int readThr, int readAhead) : - fCacheSz(cacheSz), - fReadThr(readThr), - fReadAhead(readAhead), - BRP(fCacheSz, fReadThr, fReadAhead) +BCTest::BCTest(int cacheSz, int readThr, int readAhead) + : fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead) { - setUp(); -} // ctor + setUp(); +} // ctor // void BCTest::setUp() { - LBIDRange_v r; - HWM_t hwm; - OID_t oid = 1000; - extentSize = dbrm.getExtentSize(); - maxBlocksAvailable = 0; - int i = 0; - fExtentSize = dbrm.getExtentSize(); + LBIDRange_v r; + HWM_t hwm; + OID_t oid = 1000; + extentSize = dbrm.getExtentSize(); + maxBlocksAvailable = 0; + int i = 0; + fExtentSize = dbrm.getExtentSize(); - while ( oid < 5000 ) + while (oid < 5000) + { + int ret = 0; + ret = dbrm.lookup(oid, r); + + if (ret == 0 && r.size() > 0) { - int ret = 0; - ret = dbrm.lookup(oid, r); - - if (ret == 0 && r.size() > 0) - { - dbrm.getHWM(oid, hwm); - maxBlocksAvailable += (r.size() * extentSize); - OidRanges_t oid_range(oid, hwm, r); - OidRangesList.push_back(oid_range); - //cout << "Setup i: " << i++ << " o: " << oid - // << " r: " << ret << " s: " << r.size() - // << " m: " << maxBlocksAvailable - // << endl; - hwm = 0; - r.clear(); - } - - oid++; + dbrm.getHWM(oid, hwm); + maxBlocksAvailable += (r.size() * extentSize); + OidRanges_t oid_range(oid, hwm, r); + OidRangesList.push_back(oid_range); + // cout << "Setup i: " << i++ << " o: " << oid + // << " r: " << ret << " s: " << r.size() + // << " m: " << maxBlocksAvailable + // << endl; + hwm = 0; + r.clear(); } - //cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; - i = 0; -} // setUp() + oid++; + } + + // cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; + i = 0; +} // setUp() int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount) { - blockCacheClient bc(BRP); - uint32_t rCount = 0; + blockCacheClient bc(BRP); + uint32_t rCount = 0; - for (uint32_t i = 0; i < o.ranges.size() ; i++) + for (uint32_t i = 0; i < o.ranges.size(); i++) + { + const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; + + if (r.size > 0) { - const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; + bc.check(r, 0, rCount); + // cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; + loadCount += rCount; + } - if (r.size > 0) - { - bc.check(r, 0, rCount); - //cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; - loadCount += rCount; - } + rCount = 0; + } // for - rCount = 0; - } // for + // cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; - //cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; + return loadCount; - return loadCount; - -} // LoadOid +} // LoadOid int BCTest::ReadOidRanges(const OidRanges_t& v) { + blockCacheClient bc(BRP); + int32_t readBlocks = 0; + int32_t missBlocks = 0; - blockCacheClient bc(BRP); - int32_t readBlocks = 0; - int32_t missBlocks = 0; + // int ret; + for (uint32_t i = 0; i < v.ranges.size(); i++) + { + FileBuffer fb(-1, -1); + const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; - //int ret; - for (uint32_t i = 0; i < v.ranges.size(); i++) + for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++) { - FileBuffer fb(-1, -1); - const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; + // ret=0; + // ret=bc.read(j, 0, fb); + FileBuffer* ptr = bc.getBlockPtr(j, 0); - for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++) - { - //ret=0; - //ret=bc.read(j, 0, fb); - FileBuffer* ptr = bc.getBlockPtr(j, 0); - - if (ptr) - readBlocks++; - else - missBlocks++; - } - - //cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl; + if (ptr) + readBlocks++; + else + missBlocks++; } - return readBlocks; + // cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " + // << v.hwm << endl; + } -} // ReadRange + return readBlocks; + +} // ReadRange // add one block to block cache // void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); -} // LoadLbid + blockCacheClient bc(BRP); + bool b; + bc.check(lbid, ver, false, b); +} // LoadLbid // get one block out of block cache // void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver) { - uint8_t d[8192] = {'\0'}; - blockCacheClient bc(BRP); - bc.read(lbid, ver, d); -} // ReadLbid - + uint8_t d[8192] = {'\0'}; + blockCacheClient bc(BRP); + bc.read(lbid, ver, d); +} // ReadLbid struct loadThr { - loadThr::loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {} + loadThr::loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) + { + } - void operator()() - { - uint32_t loadedBlocks = 0; - uint32_t oidBlocks; - uint32_t i = 0; - uint32_t rc = 0; + void operator()() + { + uint32_t loadedBlocks = 0; + uint32_t oidBlocks; + uint32_t i = 0; + uint32_t rc = 0; - clock_gettime(CLOCK_REALTIME, &tm1); + clock_gettime(CLOCK_REALTIME, &tm1); - for (uint32_t j = 0; j < fReps; j++) - for (i = 0; /*loadedBlocks " << endl; + cout << "testbc " << endl; } // int main(int argc, char* argv[]) { + int cacheSz = 128; // K number of blocks + int thr = 1; + int ra = 1024; + int clients = 1; + int reps = 1; - int cacheSz = 128; // K number of blocks - int thr = 1; - int ra = 1024; - int clients = 1; - int reps = 1; + if (argc > 1 && atoi(argv[1]) > 0) + cacheSz = atoi(argv[1]) * 1024; - if (argc > 1 && atoi(argv[1]) > 0) - cacheSz = atoi(argv[1]) * 1024; + if (argc > 2 && atoi(argv[2]) > 0) + thr = atoi(argv[2]); - if (argc > 2 && atoi(argv[2]) > 0) - thr = atoi(argv[2]); + if (argc > 3 && atoi(argv[3]) > 0) + ra = atoi(argv[3]); - if (argc > 3 && atoi(argv[3]) > 0) - ra = atoi(argv[3]); + if (argc > 4 && atoi(argv[4]) > 0) + clients = atoi(argv[4]); - if (argc > 4 && atoi(argv[4]) > 0) - clients = atoi(argv[4]); + if (argc > 5 && atoi(argv[5]) > 0) + reps = atoi(argv[5]); - if (argc > 5 && atoi(argv[5]) > 0) - reps = atoi(argv[5]); + BCTest bc(cacheSz, thr, ra); - BCTest bc(cacheSz, thr, ra); + cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra + << " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable + << endl; - cout << - "Cache Size: " << cacheSz << - " read Threads: " << thr << - " read Ahead: " << ra << - " clients: " << clients << - " repetitions: " << reps << - " max Blocks: " << bc.maxBlocksAvailable << - endl; + // loader test + struct loadThr loader1(bc, reps); + vector v; -// loader test - struct loadThr loader1(bc, reps); - vector v; + for (int i = 0; i < clients; i++) + { + boost::thread* th1 = new boost::thread(loader1); + v.push_back(th1); + } - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = new boost::thread(loader1); - v.push_back(th1); - } + for (int i = 0; i < clients; i++) + { + boost::thread* th1 = v[i]; + th1->join(); + delete th1; + } - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = v[i]; - th1->join(); - delete th1; - } + v.clear(); - v.clear(); + // reader test -// reader test + return 0; - - return 0; - -} // end main +} // end main diff --git a/utils/testbc/testbc2.cpp b/utils/testbc/testbc2.cpp index a818be073..94407d6db 100644 --- a/utils/testbc/testbc2.cpp +++ b/utils/testbc/testbc2.cpp @@ -51,11 +51,9 @@ uint32_t gSession = 0; uint32_t lastRangeListIdx = 0; const uint32_t maxLoadBlocks(1024 * 1024); -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } namespace primitiveprocessor @@ -63,309 +61,294 @@ namespace primitiveprocessor Logger ml; } - class BCTest { - -public: - - struct OidRanges + public: + struct OidRanges + { + OID_t oid; + HWM_t hwm; + LBIDRange_v ranges; + OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) { - OID_t oid; - HWM_t hwm; - LBIDRange_v ranges; - OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) - { - oid = o; - hwm = h; - ranges = r; - } - }; //struct OidRanges + oid = o; + hwm = h; + ranges = r; + } + }; // struct OidRanges - BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); + BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); - typedef OidRanges OidRanges_t; - typedef vectorOidRangesList_t; - OidRangesList_t OidRangesList; + typedef OidRanges OidRanges_t; + typedef vector OidRangesList_t; + OidRangesList_t OidRangesList; - DBRM dbrm; - uint32_t extentSize; - BRM::OID_t maxOid; + DBRM dbrm; + uint32_t extentSize; + BRM::OID_t maxOid; - int fCacheSz; - int fReadThr; - int fReadAhead; - uint32_t maxBlocksAvailable; - uint32_t fExtentSize; + int fCacheSz; + int fReadThr; + int fReadAhead; + uint32_t maxBlocksAvailable; + uint32_t fExtentSize; - void setUp(); - int LoadOid(const OidRanges_t& o, uint32_t& loadCount); - void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); - int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss); - void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); + void setUp(); + int LoadOid(const OidRanges_t& o, uint32_t& loadCount); + void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); + int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss); + void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); - BlockRequestProcessor BRP; + BlockRequestProcessor BRP; -}; // class BCTest +}; // class BCTest - -BCTest::BCTest(int cacheSz, int readThr, int readAhead) : - fCacheSz(cacheSz), - fReadThr(readThr), - fReadAhead(readAhead), - BRP(fCacheSz, fReadThr, fReadAhead) +BCTest::BCTest(int cacheSz, int readThr, int readAhead) + : fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead) { - setUp(); -} // ctor + setUp(); +} // ctor // void BCTest::setUp() { - LBIDRange_v r; - HWM_t hwm; - OID_t oid = 1000; - extentSize = dbrm.getExtentSize(); - maxBlocksAvailable = 0; - int i = 0; - fExtentSize = dbrm.getExtentSize(); + LBIDRange_v r; + HWM_t hwm; + OID_t oid = 1000; + extentSize = dbrm.getExtentSize(); + maxBlocksAvailable = 0; + int i = 0; + fExtentSize = dbrm.getExtentSize(); - while ( oid < 5000 ) + while (oid < 5000) + { + int ret = 0; + ret = dbrm.lookup(oid, r); + + if (ret == 0 && r.size() > 0) { - int ret = 0; - ret = dbrm.lookup(oid, r); - - if (ret == 0 && r.size() > 0) - { - dbrm.getHWM(oid, hwm); - maxBlocksAvailable += (r.size() * extentSize); - OidRanges_t oid_range(oid, hwm, r); - OidRangesList.push_back(oid_range); - //cout << "Setup i: " << i++ << " o: " << oid - // << " r: " << ret << " s: " << r.size() - // << " m: " << maxBlocksAvailable - // << endl; - hwm = 0; - r.clear(); - } - - oid++; + dbrm.getHWM(oid, hwm); + maxBlocksAvailable += (r.size() * extentSize); + OidRanges_t oid_range(oid, hwm, r); + OidRangesList.push_back(oid_range); + // cout << "Setup i: " << i++ << " o: " << oid + // << " r: " << ret << " s: " << r.size() + // << " m: " << maxBlocksAvailable + // << endl; + hwm = 0; + r.clear(); } - //cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; - i = 0; -} // setUp() + oid++; + } + + // cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; + i = 0; +} // setUp() int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount) { - blockCacheClient bc(BRP); - uint32_t rCount = 0; + blockCacheClient bc(BRP); + uint32_t rCount = 0; - for (uint32_t i = 0; i < o.ranges.size() ; i++) + for (uint32_t i = 0; i < o.ranges.size(); i++) + { + const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; + + if (r.size > 0) { - const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; + bc.check(r, 0, rCount); + // cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; + loadCount += rCount; + } - if (r.size > 0) - { - bc.check(r, 0, rCount); - //cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; - loadCount += rCount; - } + rCount = 0; + } // for - rCount = 0; - } // for + // cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; - //cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; + return loadCount; - return loadCount; - -} // LoadOid +} // LoadOid int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss) { + blockCacheClient bc(BRP); + uint8_t inBuff[8192]; + int32_t readBlocks = 0; + int32_t missBlocks = 0; - blockCacheClient bc(BRP); - uint8_t inBuff[8192]; - int32_t readBlocks = 0; - int32_t missBlocks = 0; + for (uint32_t i = 0; i < v.ranges.size(); i++) + { + FileBuffer fb(-1, -1); + const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; - for (uint32_t i = 0; i < v.ranges.size(); i++) + for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++) { - FileBuffer fb(-1, -1); - const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; + FileBuffer* ptr = bc.getBlockPtr(j, 0); - for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++) - { - FileBuffer* ptr = bc.getBlockPtr(j, 0); - - if (ptr) - { - readBlocks++; - memcpy(inBuff, ptr->getData(), 8192); - } - else - missBlocks++; - } - - *hits += readBlocks; - *miss += missBlocks; - missBlocks = 0; - readBlocks = 0; - //cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl; + if (ptr) + { + readBlocks++; + memcpy(inBuff, ptr->getData(), 8192); + } + else + missBlocks++; } - return *hits; + *hits += readBlocks; + *miss += missBlocks; + missBlocks = 0; + readBlocks = 0; + // cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " + // << v.hwm << endl; + } -} // ReadRange + return *hits; + +} // ReadRange // add one block to block cache // void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); -} // LoadLbid + blockCacheClient bc(BRP); + bool b; + bc.check(lbid, ver, false, b); +} // LoadLbid // get one block out of block cache // void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver) { - uint8_t d[8192] = {'\0'}; - blockCacheClient bc(BRP); - bc.read(lbid, ver, d); -} // ReadLbid - + uint8_t d[8192] = {'\0'}; + blockCacheClient bc(BRP); + bc.read(lbid, ver, d); +} // ReadLbid struct loadThr { - loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {} + loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) + { + } - void operator()() - { - uint32_t loadedBlocks = 0; - uint32_t readBlocks = 0; - uint32_t missBlocks = 0; - uint32_t oidBlocks; - uint32_t i = 0; - uint32_t rc = 0; + void operator()() + { + uint32_t loadedBlocks = 0; + uint32_t readBlocks = 0; + uint32_t missBlocks = 0; + uint32_t oidBlocks; + uint32_t i = 0; + uint32_t rc = 0; - clock_gettime(CLOCK_REALTIME, &tm1); + clock_gettime(CLOCK_REALTIME, &tm1); - for (uint32_t j = 0; j < fReps; j++) - for (i = 0; loadedBlocks < maxLoadBlocks && i < fBC.OidRangesList.size(); i++) - { - oidBlocks = 0; - rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks); - /** - cout << "." - << "-- " << i << " " << fBC.OidRangesList[i].oid - << " h: " << fBC.OidRangesList[i].hwm - << "/" << oidBlocks - << endl; - **/ - loadedBlocks += oidBlocks; - readBlocks += fBC.ReadOidRanges(fBC.OidRangesList[i], &readBlocks, &missBlocks); + for (uint32_t j = 0; j < fReps; j++) + for (i = 0; loadedBlocks < maxLoadBlocks && i < fBC.OidRangesList.size(); i++) + { + oidBlocks = 0; + rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks); + /** + cout << "." + << "-- " << i << " " << fBC.OidRangesList[i].oid + << " h: " << fBC.OidRangesList[i].hwm + << "/" << oidBlocks + << endl; + **/ + loadedBlocks += oidBlocks; + readBlocks += fBC.ReadOidRanges(fBC.OidRangesList[i], &readBlocks, &missBlocks); + } - } + clock_gettime(CLOCK_REALTIME, &tm2); + double tm3; + timespec_sub(tm1, tm2, tm3); + lastRangeListIdx = i; - clock_gettime(CLOCK_REALTIME, &tm2); - double tm3; - timespec_sub(tm1, tm2, tm3); - lastRangeListIdx = i; + cout << "loadtest ld: " << loadedBlocks << " rd: " << readBlocks << "/" << missBlocks << " sz: " + << fBC.fCacheSz + //<< " last: " << lastRangeListIdx + << " tm: " << right << setw(10) << fixed << tm3 << endl; - cout << "loadtest ld: " << loadedBlocks - << " rd: " << readBlocks << "/" << missBlocks - << " sz: " << fBC.fCacheSz - //<< " last: " << lastRangeListIdx - << " tm: " << right << setw(10) << fixed << tm3 - << endl; - - } // operator() - - BCTest& fBC; - uint32_t fReps; - struct timespec tm1; - struct timespec tm2; + } // operator() + BCTest& fBC; + uint32_t fReps; + struct timespec tm1; + struct timespec tm2; }; struct readThr { + readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) + { + } - readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {} - - void operator()() + void operator()() + { + for (uint32_t k = 0; k < fReps; k++) { - for (uint32_t k = 0; k < fReps; k++) - { - } + } - } // operator() + } // operator() - BCTest& fBC; - uint32_t fReps; + BCTest& fBC; + uint32_t fReps; }; void usage() { - cout << "testbc " << endl; + cout << "testbc " << endl; } // int main(int argc, char* argv[]) { + int cacheSz = 128; // K number of blocks + int thr = 1; + int ra = 1024; + int clients = 1; + int reps = 1; - int cacheSz = 128; // K number of blocks - int thr = 1; - int ra = 1024; - int clients = 1; - int reps = 1; + if (argc > 1 && atoi(argv[1]) > 0) + cacheSz = atoi(argv[1]) * 1024; - if (argc > 1 && atoi(argv[1]) > 0) - cacheSz = atoi(argv[1]) * 1024; + if (argc > 2 && atoi(argv[2]) > 0) + thr = atoi(argv[2]); - if (argc > 2 && atoi(argv[2]) > 0) - thr = atoi(argv[2]); + if (argc > 3 && atoi(argv[3]) > 0) + ra = atoi(argv[3]); - if (argc > 3 && atoi(argv[3]) > 0) - ra = atoi(argv[3]); + if (argc > 4 && atoi(argv[4]) > 0) + clients = atoi(argv[4]); - if (argc > 4 && atoi(argv[4]) > 0) - clients = atoi(argv[4]); + if (argc > 5 && atoi(argv[5]) > 0) + reps = atoi(argv[5]); - if (argc > 5 && atoi(argv[5]) > 0) - reps = atoi(argv[5]); + BCTest bc(cacheSz, thr, ra); - BCTest bc(cacheSz, thr, ra); + cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra + << " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable + << endl; - cout << - "Cache Size: " << cacheSz << - " read Threads: " << thr << - " read Ahead: " << ra << - " clients: " << clients << - " repetitions: " << reps << - " max Blocks: " << bc.maxBlocksAvailable << - endl; + // loader test + struct loadThr loader1(bc, reps); + vector v; -// loader test - struct loadThr loader1(bc, reps); - vector v; + for (int i = 0; i < clients; i++) + { + boost::thread* th1 = new boost::thread(loader1); + v.push_back(th1); + } - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = new boost::thread(loader1); - v.push_back(th1); - } + for (int i = 0; i < clients; i++) + { + boost::thread* th1 = v[i]; + th1->join(); + delete th1; + } - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = v[i]; - th1->join(); - delete th1; - } + v.clear(); - v.clear(); + return 0; - return 0; - -} // end main +} // end main diff --git a/utils/testbc/testbc3.cpp b/utils/testbc/testbc3.cpp index 28dd09b84..ae5478450 100644 --- a/utils/testbc/testbc3.cpp +++ b/utils/testbc/testbc3.cpp @@ -51,11 +51,9 @@ uint32_t gSession = 0; uint32_t lastRangeListIdx = 0; const uint32_t maxLoadBlocks(1024 * 1024); -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } namespace primitiveprocessor @@ -63,352 +61,334 @@ namespace primitiveprocessor Logger ml; } - class BCTest { - -public: - - struct OidRanges + public: + struct OidRanges + { + OID_t oid; + HWM_t hwm; + LBIDRange_v ranges; + OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) { - OID_t oid; - HWM_t hwm; - LBIDRange_v ranges; - OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) - { - oid = o; - hwm = h; - ranges = r; - } - }; //struct OidRanges + oid = o; + hwm = h; + ranges = r; + } + }; // struct OidRanges - BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); + BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); - typedef OidRanges OidRanges_t; - typedef vectorOidRangesList_t; - OidRangesList_t OidRangesList; + typedef OidRanges OidRanges_t; + typedef vector OidRangesList_t; + OidRangesList_t OidRangesList; - DBRM dbrm; - uint32_t extentSize; - BRM::OID_t maxOid; + DBRM dbrm; + uint32_t extentSize; + BRM::OID_t maxOid; - int fCacheSz; - int fReadThr; - int fReadAhead; - uint32_t maxBlocksAvailable; - uint32_t fExtentSize; + int fCacheSz; + int fReadThr; + int fReadAhead; + uint32_t maxBlocksAvailable; + uint32_t fExtentSize; - void setUp(); - int LoadOid(const OidRanges_t& o, uint32_t& loadCount); - void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); - int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss); - void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); + void setUp(); + int LoadOid(const OidRanges_t& o, uint32_t& loadCount); + void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); + int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss); + void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); - BlockRequestProcessor BRP; + BlockRequestProcessor BRP; -}; // class BCTest +}; // class BCTest - -BCTest::BCTest(int cacheSz, int readThr, int readAhead) : - fCacheSz(cacheSz), - fReadThr(readThr), - fReadAhead(readAhead), - BRP(fCacheSz, fReadThr, fReadAhead) +BCTest::BCTest(int cacheSz, int readThr, int readAhead) + : fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead) { - setUp(); -} // ctor + setUp(); +} // ctor // void BCTest::setUp() { - LBIDRange_v r; - HWM_t hwm; - OID_t oid = 1000; - extentSize = dbrm.getExtentSize(); - maxBlocksAvailable = 0; - int i = 0; - fExtentSize = dbrm.getExtentSize(); + LBIDRange_v r; + HWM_t hwm; + OID_t oid = 1000; + extentSize = dbrm.getExtentSize(); + maxBlocksAvailable = 0; + int i = 0; + fExtentSize = dbrm.getExtentSize(); - while ( oid < 5000 ) + while (oid < 5000) + { + int ret = 0; + ret = dbrm.lookup(oid, r); + + if (ret == 0 && r.size() > 0) { - int ret = 0; - ret = dbrm.lookup(oid, r); - - if (ret == 0 && r.size() > 0) - { - dbrm.getHWM(oid, hwm); - maxBlocksAvailable += (r.size() * extentSize); - OidRanges_t oid_range(oid, hwm, r); - OidRangesList.push_back(oid_range); - //cout << "Setup i: " << i++ << " o: " << oid - // << " r: " << ret << " s: " << r.size() - // << " m: " << maxBlocksAvailable - // << endl; - hwm = 0; - r.clear(); - } - - oid++; + dbrm.getHWM(oid, hwm); + maxBlocksAvailable += (r.size() * extentSize); + OidRanges_t oid_range(oid, hwm, r); + OidRangesList.push_back(oid_range); + // cout << "Setup i: " << i++ << " o: " << oid + // << " r: " << ret << " s: " << r.size() + // << " m: " << maxBlocksAvailable + // << endl; + hwm = 0; + r.clear(); } - //cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; - i = 0; -} // setUp() + oid++; + } + + // cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; + i = 0; +} // setUp() int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount) { - blockCacheClient bc(BRP); - uint32_t rCount = 0; + blockCacheClient bc(BRP); + uint32_t rCount = 0; - for (uint32_t i = 0; i < o.ranges.size() ; i++) + for (uint32_t i = 0; i < o.ranges.size(); i++) + { + const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; + + if (r.size > 0) { - const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; + bc.check(r, 0, rCount); + // cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; + loadCount += rCount; + } - if (r.size > 0) - { - bc.check(r, 0, rCount); - //cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; - loadCount += rCount; - } + rCount = 0; + } // for - rCount = 0; - } // for + // cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; - //cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; + return loadCount; - return loadCount; - -} // LoadOid +} // LoadOid int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss) { + blockCacheClient bc(BRP); + uint32_t readBlocks = 0; + uint32_t missBlocks = 0; + uint8_t inBuff[8192]; + int ret; - blockCacheClient bc(BRP); - uint32_t readBlocks = 0; - uint32_t missBlocks = 0; - uint8_t inBuff[8192]; - int ret; + for (uint32_t i = 0; i < v.ranges.size() && i < lastRangeListIdx; i++) + { + FileBuffer fb(-1, -1); + const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; - for (uint32_t i = 0; i < v.ranges.size() && i < lastRangeListIdx; i++) + for (int j = r.start; j < r.start + r.size; j++) { - FileBuffer fb(-1, -1); - const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; + ret = bc.read(j, 0, fb); + FileBuffer* ptr = bc.getBlockPtr(j, 0); - for (int j = r.start; j < r.start + r.size; j++) - { - ret = bc.read(j, 0, fb); - FileBuffer* ptr = bc.getBlockPtr(j, 0); - - if (ptr) - { - readBlocks++; - memcpy(inBuff, ptr->getData(), 8192); - } - else - missBlocks++; - } - - *hits += readBlocks; - *miss += missBlocks; - - //cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl; + if (ptr) + { + readBlocks++; + memcpy(inBuff, ptr->getData(), 8192); + } + else + missBlocks++; } - return readBlocks; + *hits += readBlocks; + *miss += missBlocks; -} // ReadRange + // cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " + // << v.hwm << endl; + } + + return readBlocks; + +} // ReadRange // add one block to block cache // void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) { - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); -} // LoadLbid + blockCacheClient bc(BRP); + bool b; + bc.check(lbid, ver, false, b); +} // LoadLbid // get one block out of block cache // void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver) { - uint8_t d[8192] = {'\0'}; - blockCacheClient bc(BRP); - bc.read(lbid, ver, d); -} // ReadLbid - + uint8_t d[8192] = {'\0'}; + blockCacheClient bc(BRP); + bc.read(lbid, ver, d); +} // ReadLbid struct loadThr { - loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) + loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) + { + } + + void operator()() + { + uint32_t loadedBlocks = 0; + uint32_t oidBlocks; + uint32_t i = 0; + uint32_t rc = 0; + struct timespec tm1; + struct timespec tm2; + + tm1.tv_sec = 0; + tm1.tv_nsec = 0; + tm2.tv_sec = 0; + tm2.tv_nsec = 0; + double tm3 = 0; + clock_gettime(CLOCK_REALTIME, &tm1); + + for (i = 0; (loadedBlocks + (fBC.OidRangesList[i].ranges.size() * fBC.extentSize)) < fBC.fCacheSz && + i < fBC.OidRangesList.size(); + i++) { - } + oidBlocks = 0; + rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks); + // cout + // << "-- Load " << i << " " << fBC.OidRangesList[i].oid + // << " h: " << fBC.OidRangesList[i].hwm + // << "/" << oidBlocks << " " << loadedBlocks + // << endl; + loadedBlocks += oidBlocks; + } // for (i... - void operator()() - { - uint32_t loadedBlocks = 0; - uint32_t oidBlocks; - uint32_t i = 0; - uint32_t rc = 0; - struct timespec tm1; - struct timespec tm2; + lastRangeListIdx = i; - tm1.tv_sec = 0; - tm1.tv_nsec = 0; - tm2.tv_sec = 0; - tm2.tv_nsec = 0; - double tm3 = 0; - clock_gettime(CLOCK_REALTIME, &tm1); + clock_gettime(CLOCK_REALTIME, &tm2); + timespec_sub(tm1, tm2, tm3); - for (i = 0; (loadedBlocks + (fBC.OidRangesList[i].ranges.size()*fBC.extentSize)) < fBC.fCacheSz && i < fBC.OidRangesList.size(); i++) - { - oidBlocks = 0; - rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks); - //cout - // << "-- Load " << i << " " << fBC.OidRangesList[i].oid - // << " h: " << fBC.OidRangesList[i].hwm - // << "/" << oidBlocks << " " << loadedBlocks - // << endl; - loadedBlocks += oidBlocks; - } // for (i... + cout << "load ld: " << loadedBlocks << " sz: " << fBC.fCacheSz << " rng: " << lastRangeListIdx + << " tm: " << right << setw(10) << fixed << tm3 << endl; - lastRangeListIdx = i; - - clock_gettime(CLOCK_REALTIME, &tm2); - timespec_sub(tm1, tm2, tm3); - - cout << "load ld: " << loadedBlocks - << " sz: " << fBC.fCacheSz - << " rng: " << lastRangeListIdx - << " tm: " << right << setw(10) << fixed << tm3 - << endl; - - } // operator() - - BCTest& fBC; - uint32_t fReps; + } // operator() + BCTest& fBC; + uint32_t fReps; }; struct readThr { + readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) + { + } - readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {} + void operator()() + { + uint32_t rc = 0; + uint32_t readBlocks = 0; + uint32_t hits = 0; + uint32_t miss = 0; + uint32_t hitsTot = 0; + uint32_t missTot = 0; + struct timespec tm1; + struct timespec tm2; + tm1.tv_sec = 0; + tm1.tv_nsec = 0; + tm2.tv_sec = 0; + tm2.tv_nsec = 0; + double tm3 = 0; + clock_gettime(CLOCK_REALTIME, &tm1); - void operator()() + for (uint32_t k = 0; k < fReps; k++) { - uint32_t rc = 0; - uint32_t readBlocks = 0; - uint32_t hits = 0; - uint32_t miss = 0; - uint32_t hitsTot = 0; - uint32_t missTot = 0; - struct timespec tm1; - struct timespec tm2; - tm1.tv_sec = 0; - tm1.tv_nsec = 0; - tm2.tv_sec = 0; - tm2.tv_nsec = 0; - double tm3 = 0; - clock_gettime(CLOCK_REALTIME, &tm1); + for (uint32_t i = 0; i < lastRangeListIdx && i < fBC.OidRangesList.size(); i++) + { + rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss); + // cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" << + // rc << endl; + readBlocks += rc; + rc = 0; + } - for (uint32_t k = 0; k < fReps; k++) - { - for (uint32_t i = 0; i < lastRangeListIdx && i < fBC.OidRangesList.size() ; i++) - { - rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss); - //cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" << rc << endl; - readBlocks += rc; - rc = 0; - } + hitsTot += hits; + missTot += miss; + hits = 0; + miss = 0; + } - hitsTot += hits; - missTot += miss; - hits = 0; - miss = 0; - } + clock_gettime(CLOCK_REALTIME, &tm2); + timespec_sub(tm1, tm2, tm3); - clock_gettime(CLOCK_REALTIME, &tm2); - timespec_sub(tm1, tm2, tm3); + cout << "readtest rd: " << hitsTot << "/" << missTot << " sz: " << fBC.fCacheSz << " tm: " << right + << setw(10) << fixed << tm3 << endl; - cout << "readtest rd: " << hitsTot << "/" << missTot - << " sz: " << fBC.fCacheSz - << " tm: " << right << setw(10) << fixed << tm3 - << endl; - - } // operator() - - - BCTest& fBC; - uint32_t fReps; + } // operator() + BCTest& fBC; + uint32_t fReps; }; void usage() { - cout << "testbc " << endl; + cout << "testbc " << endl; } // int main(int argc, char* argv[]) { + int cacheSz = 128; // K number of blocks + int thr = 1; + int ra = 1024; + int clients = 1; + int reps = 1; - int cacheSz = 128; // K number of blocks - int thr = 1; - int ra = 1024; - int clients = 1; - int reps = 1; + if (argc > 1 && atoi(argv[1]) > 0) + cacheSz = atoi(argv[1]) * 1024; - if (argc > 1 && atoi(argv[1]) > 0) - cacheSz = atoi(argv[1]) * 1024; + if (argc > 2 && atoi(argv[2]) > 0) + thr = atoi(argv[2]); - if (argc > 2 && atoi(argv[2]) > 0) - thr = atoi(argv[2]); + if (argc > 3 && atoi(argv[3]) > 0) + ra = atoi(argv[3]); - if (argc > 3 && atoi(argv[3]) > 0) - ra = atoi(argv[3]); + if (argc > 4 && atoi(argv[4]) > 0) + clients = atoi(argv[4]); - if (argc > 4 && atoi(argv[4]) > 0) - clients = atoi(argv[4]); + if (argc > 5 && atoi(argv[5]) > 0) + reps = atoi(argv[5]); - if (argc > 5 && atoi(argv[5]) > 0) - reps = atoi(argv[5]); + BCTest bc(cacheSz, thr, ra); - BCTest bc(cacheSz, thr, ra); + cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra + << " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable + << endl; - cout << - "Cache Size: " << cacheSz << - " read Threads: " << thr << - " read Ahead: " << ra << - " clients: " << clients << - " repetitions: " << reps << - " max Blocks: " << bc.maxBlocksAvailable << - endl; + // loader cache + struct loadThr loader1(bc, 1); + struct readThr reader1(bc, reps); + vector v; - // loader cache - struct loadThr loader1(bc, 1); - struct readThr reader1(bc, reps); - vector v; + boost::thread* th1 = new boost::thread(loader1); + th1->join(); - boost::thread* th1 = new boost::thread(loader1); - th1->join(); + for (int i = 0; i < clients; i++) + { + boost::thread* rd1 = new boost::thread(reader1); + v.push_back(rd1); + } - for (int i = 0; i < clients; i++) - { - boost::thread* rd1 = new boost::thread(reader1); - v.push_back(rd1); - } + for (int i = 0; i < clients; i++) + { + boost::thread* rd1 = v[i]; + rd1->join(); + delete rd1; + } - for (int i = 0; i < clients; i++) - { - boost::thread* rd1 = v[i]; - rd1->join(); - delete rd1; - } + v.clear(); - v.clear(); + delete th1; - delete th1; + return 0; - return 0; - -} // end main +} // end main diff --git a/utils/threadpool/prioritythreadpool.cpp b/utils/threadpool/prioritythreadpool.cpp index 3ed4c7d1a..b26144b62 100644 --- a/utils/threadpool/prioritythreadpool.cpp +++ b/utils/threadpool/prioritythreadpool.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: threadpool.cpp 553 2008-02-27 17:51:16Z rdempsey $ -* -* -***********************************************************************/ + * $Id: threadpool.cpp 553 2008-02-27 17:51:16Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -38,295 +38,288 @@ using namespace boost; namespace threadpool { - -PriorityThreadPool::PriorityThreadPool(uint targetWeightPerRun, uint highThreads, - uint midThreads, uint lowThreads, uint ID) : - _stop(false), weightPerRun(targetWeightPerRun), id(ID), - blockedThreads(0), extraThreads(0), stopExtra(true) +PriorityThreadPool::PriorityThreadPool(uint targetWeightPerRun, uint highThreads, uint midThreads, + uint lowThreads, uint ID) + : _stop(false), weightPerRun(targetWeightPerRun), id(ID), blockedThreads(0), extraThreads(0), stopExtra(true) { - boost::thread* newThread; - for (uint32_t i = 0; i < highThreads; i++) - { - newThread = threads.create_thread(ThreadHelper(this, HIGH)); - newThread->detach(); - } - for (uint32_t i = 0; i < midThreads; i++) - { - newThread = threads.create_thread(ThreadHelper(this, MEDIUM)); - newThread->detach(); - } - for (uint32_t i = 0; i < lowThreads; i++) - { - newThread = threads.create_thread(ThreadHelper(this, LOW)); - newThread->detach(); - } - cout << "started " << highThreads << " high, " << midThreads << " med, " << lowThreads - << " low.\n"; - defaultThreadCounts[HIGH] = threadCounts[HIGH] = highThreads; - defaultThreadCounts[MEDIUM] = threadCounts[MEDIUM] = midThreads; - defaultThreadCounts[LOW] = threadCounts[LOW] = lowThreads; + boost::thread* newThread; + for (uint32_t i = 0; i < highThreads; i++) + { + newThread = threads.create_thread(ThreadHelper(this, HIGH)); + newThread->detach(); + } + for (uint32_t i = 0; i < midThreads; i++) + { + newThread = threads.create_thread(ThreadHelper(this, MEDIUM)); + newThread->detach(); + } + for (uint32_t i = 0; i < lowThreads; i++) + { + newThread = threads.create_thread(ThreadHelper(this, LOW)); + newThread->detach(); + } + cout << "started " << highThreads << " high, " << midThreads << " med, " << lowThreads << " low.\n"; + defaultThreadCounts[HIGH] = threadCounts[HIGH] = highThreads; + defaultThreadCounts[MEDIUM] = threadCounts[MEDIUM] = midThreads; + defaultThreadCounts[LOW] = threadCounts[LOW] = lowThreads; } PriorityThreadPool::~PriorityThreadPool() { - stop(); + stop(); } void PriorityThreadPool::addJob(const Job& job, bool useLock) { - boost::thread* newThread; - boost::mutex::scoped_lock lk(mutex, boost::defer_lock_t()); + boost::thread* newThread; + boost::mutex::scoped_lock lk(mutex, boost::defer_lock_t()); - if (useLock) - lk.lock(); + if (useLock) + lk.lock(); - // Create any missing threads - if (defaultThreadCounts[HIGH] != threadCounts[HIGH]) - { - newThread = threads.create_thread(ThreadHelper(this, HIGH)); - newThread->detach(); - threadCounts[HIGH]++; - } + // Create any missing threads + if (defaultThreadCounts[HIGH] != threadCounts[HIGH]) + { + newThread = threads.create_thread(ThreadHelper(this, HIGH)); + newThread->detach(); + threadCounts[HIGH]++; + } - if (defaultThreadCounts[MEDIUM] != threadCounts[MEDIUM]) - { - newThread = threads.create_thread(ThreadHelper(this, MEDIUM)); - newThread->detach(); - threadCounts[MEDIUM]++; - } + if (defaultThreadCounts[MEDIUM] != threadCounts[MEDIUM]) + { + newThread = threads.create_thread(ThreadHelper(this, MEDIUM)); + newThread->detach(); + threadCounts[MEDIUM]++; + } - if (defaultThreadCounts[LOW] != threadCounts[LOW]) - { - newThread = threads.create_thread(ThreadHelper(this, LOW)); - newThread->detach(); - threadCounts[LOW]++; - } - - // If some threads have blocked (because of output queue full) - // Temporarily add some extra worker threads to make up for the blocked threads. - if (blockedThreads > extraThreads) - { - stopExtra = false; - newThread = threads.create_thread(ThreadHelper(this, EXTRA)); - newThread->detach(); - extraThreads++; - } - else if (blockedThreads == 0) - { - // Release the temporary threads -- some threads have become unblocked. - stopExtra = true; - } - - if (job.priority > 66) - jobQueues[HIGH].push_back(job); - else if (job.priority > 33) - jobQueues[MEDIUM].push_back(job); - else - jobQueues[LOW].push_back(job); + if (defaultThreadCounts[LOW] != threadCounts[LOW]) + { + newThread = threads.create_thread(ThreadHelper(this, LOW)); + newThread->detach(); + threadCounts[LOW]++; + } - if (useLock) - newJob.notify_one(); + // If some threads have blocked (because of output queue full) + // Temporarily add some extra worker threads to make up for the blocked threads. + if (blockedThreads > extraThreads) + { + stopExtra = false; + newThread = threads.create_thread(ThreadHelper(this, EXTRA)); + newThread->detach(); + extraThreads++; + } + else if (blockedThreads == 0) + { + // Release the temporary threads -- some threads have become unblocked. + stopExtra = true; + } + + if (job.priority > 66) + jobQueues[HIGH].push_back(job); + else if (job.priority > 33) + jobQueues[MEDIUM].push_back(job); + else + jobQueues[LOW].push_back(job); + + if (useLock) + newJob.notify_one(); } void PriorityThreadPool::removeJobs(uint32_t id) { - list::iterator it; + list::iterator it; - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - for (uint32_t i = 0; i < _COUNT; i++) - for (it = jobQueues[i].begin(); it != jobQueues[i].end();) - if (it->id == id) - it = jobQueues[i].erase(it); - else - ++it; + for (uint32_t i = 0; i < _COUNT; i++) + for (it = jobQueues[i].begin(); it != jobQueues[i].end();) + if (it->id == id) + it = jobQueues[i].erase(it); + else + ++it; } PriorityThreadPool::Priority PriorityThreadPool::pickAQueue(Priority preference) { - if (preference != EXTRA && !jobQueues[preference].empty()) - return preference; - else if (!jobQueues[HIGH].empty()) - return HIGH; - else if (!jobQueues[MEDIUM].empty()) - return MEDIUM; - else - return LOW; + if (preference != EXTRA && !jobQueues[preference].empty()) + return preference; + else if (!jobQueues[HIGH].empty()) + return HIGH; + else if (!jobQueues[MEDIUM].empty()) + return MEDIUM; + else + return LOW; } void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() { - if (preferredQueue == EXTRA) - utils::setThreadName("Extra"); - else - utils::setThreadName("Idle"); - Priority queue = LOW; - uint32_t weight, i = 0; - vector runList; - vector reschedule; - uint32_t rescheduleCount; - uint32_t queueSize; - bool running = false; + if (preferredQueue == EXTRA) + utils::setThreadName("Extra"); + else + utils::setThreadName("Idle"); + Priority queue = LOW; + uint32_t weight, i = 0; + vector runList; + vector reschedule; + uint32_t rescheduleCount; + uint32_t queueSize; + bool running = false; + try + { + while (!_stop) + { + boost::mutex::scoped_lock lk(mutex); + + queue = pickAQueue(preferredQueue); + + if (jobQueues[queue].empty()) + { + // If this is an EXTRA thread due toother threads blocking, and all blockers are unblocked, + // we don't want this one any more. + if (preferredQueue == EXTRA && stopExtra) + { + extraThreads--; + return; + } + + newJob.wait(lk); + continue; + } + + queueSize = jobQueues[queue].size(); + weight = 0; + // 3 conditions stop this thread from grabbing all jobs in the queue + // + // 1: The weight limit has been exceeded + // 2: The queue is empty + // 3: It has grabbed more than half of the jobs available & + // should leave some to the other threads + + while ((weight < weightPerRun) && (!jobQueues[queue].empty()) && (runList.size() <= queueSize / 2)) + { + runList.push_back(jobQueues[queue].front()); + jobQueues[queue].pop_front(); + weight += runList.back().weight; + } + + lk.unlock(); + + reschedule.resize(runList.size()); + rescheduleCount = 0; + + for (i = 0; i < runList.size() && !_stop; i++) + { + reschedule[i] = false; + running = true; + reschedule[i] = (*(runList[i].functor))(); + running = false; + + if (reschedule[i]) + rescheduleCount++; + } + if (preferredQueue == EXTRA) + utils::setThreadName("Extra (used)"); + else + utils::setThreadName("Idle"); + + // no real work was done, prevent intensive busy waiting + if (rescheduleCount == runList.size()) + usleep(1000); + + if (rescheduleCount > 0) + { + lk.lock(); + + for (i = 0; i < runList.size(); i++) + if (reschedule[i]) + addJob(runList[i], false); + + if (rescheduleCount > 1) + newJob.notify_all(); + else + newJob.notify_one(); + + lk.unlock(); + } + + runList.clear(); + } + } + catch (std::exception& ex) + { + // Log the exception and exit this thread try { - while (!_stop) - { - - boost::mutex::scoped_lock lk(mutex); - - queue = pickAQueue(preferredQueue); - - if (jobQueues[queue].empty()) - { - // If this is an EXTRA thread due toother threads blocking, and all blockers are unblocked, - // we don't want this one any more. - if (preferredQueue == EXTRA && stopExtra) - { - extraThreads--; - return; - } - - newJob.wait(lk); - continue; - } - - queueSize = jobQueues[queue].size(); - weight = 0; - // 3 conditions stop this thread from grabbing all jobs in the queue - // - // 1: The weight limit has been exceeded - // 2: The queue is empty - // 3: It has grabbed more than half of the jobs available & - // should leave some to the other threads - - while ((weight < weightPerRun) && (!jobQueues[queue].empty()) - && (runList.size() <= queueSize / 2)) - { - runList.push_back(jobQueues[queue].front()); - jobQueues[queue].pop_front(); - weight += runList.back().weight; - } - - lk.unlock(); - - reschedule.resize(runList.size()); - rescheduleCount = 0; - - for (i = 0; i < runList.size() && !_stop; i++) - { - reschedule[i] = false; - running = true; - reschedule[i] = (*(runList[i].functor))(); - running = false; - - if (reschedule[i]) - rescheduleCount++; - } - if (preferredQueue == EXTRA) - utils::setThreadName("Extra (used)"); - else - utils::setThreadName("Idle"); - - // no real work was done, prevent intensive busy waiting - if (rescheduleCount == runList.size()) - usleep(1000); - - if (rescheduleCount > 0) - { - lk.lock(); - - for (i = 0; i < runList.size(); i++) - if (reschedule[i]) - addJob(runList[i], false); - - if (rescheduleCount > 1) - newJob.notify_all(); - else - newJob.notify_one(); - - lk.unlock(); - } - - runList.clear(); - - } - } - catch (std::exception& ex) - { - // Log the exception and exit this thread - try - { - threadCounts[queue]--; + threadCounts[queue]--; #ifndef NOLOGGING - logging::Message::Args args; - logging::Message message(5); - args.add("threadFcn: Caught exception: "); - args.add(ex.what()); + logging::Message::Args args; + logging::Message message(5); + args.add("threadFcn: Caught exception: "); + args.add(ex.what()); - message.format( args ); + message.format(args); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); - ml.logErrorMessage( message ); + ml.logErrorMessage(message); #endif - if (running) - sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock); - } - catch (...) - { - } + if (running) + sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock); } catch (...) { - - // Log the exception and exit this thread - try - { - threadCounts[queue]--; + } + } + catch (...) + { + // Log the exception and exit this thread + try + { + threadCounts[queue]--; #ifndef NOLOGGING - logging::Message::Args args; - logging::Message message(6); - args.add("threadFcn: Caught unknown exception!"); + logging::Message::Args args; + logging::Message message(6); + args.add("threadFcn: Caught unknown exception!"); - message.format( args ); + message.format(args); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); - ml.logErrorMessage( message ); + ml.logErrorMessage(message); #endif - if (running) - sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock); - } - catch (...) - { - } + if (running) + sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock); } + catch (...) + { + } + } } void PriorityThreadPool::sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock) { - ISMPacketHeader ism; - PrimitiveHeader ph = {0,0,0,0,0,0}; + ISMPacketHeader ism; + PrimitiveHeader ph = {0, 0, 0, 0, 0, 0}; - ism.Status = logging::primitiveServerErr; - ph.UniqueID = id; - ph.StepID = step; - messageqcpp::ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - msg.append((uint8_t*) &ism, sizeof(ism)); - msg.append((uint8_t*) &ph, sizeof(ph)); + ism.Status = logging::primitiveServerErr; + ph.UniqueID = id; + ph.StepID = step; + messageqcpp::ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + msg.append((uint8_t*)&ism, sizeof(ism)); + msg.append((uint8_t*)&ph, sizeof(ph)); - sock->write(msg); + sock->write(msg); } void PriorityThreadPool::stop() { - _stop = true; + _stop = true; } -} // namespace threadpool +} // namespace threadpool // vim:ts=4 sw=4: diff --git a/utils/threadpool/prioritythreadpool.h b/utils/threadpool/prioritythreadpool.h index a97ca621b..ac5b74821 100644 --- a/utils/threadpool/prioritythreadpool.h +++ b/utils/threadpool/prioritythreadpool.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: $ -* -* -***********************************************************************/ + * $Id: $ + * + * + ***********************************************************************/ /** @file */ #ifndef PRIORITYTHREADPOOL_H @@ -35,121 +35,123 @@ #include #include #include -#include +#include #include "../winport/winport.h" #include "primitives/primproc/umsocketselector.h" #include "atomicops.h" namespace threadpool { - class PriorityThreadPool { -public: + public: + class Functor + { + public: + virtual ~Functor(){}; + // as of 12/3/13, all implementors return 0 and -1. -1 will cause + // this thread pool to reschedule the job, 0 will throw it away on return. + virtual int operator()() = 0; + }; - class Functor + // typedef boost::function0 Functor; + + struct Job + { + Job() : weight(1), priority(0), id(0) { - public: - virtual ~Functor() { }; - // as of 12/3/13, all implementors return 0 and -1. -1 will cause - // this thread pool to reschedule the job, 0 will throw it away on return. - virtual int operator()() = 0; - }; - - //typedef boost::function0 Functor; - - struct Job - { - Job() : weight(1), priority(0), id(0) { } - boost::shared_ptr functor; - uint32_t weight; - uint32_t priority; - uint32_t id; - uint32_t uniqueID; - uint32_t stepID; - primitiveprocessor::SP_UM_IOSOCK sock; - }; - - enum Priority - { - LOW, - MEDIUM, - HIGH, - _COUNT, - EXTRA // After _COUNT because _COUNT is for jobQueue size and EXTRA isn't a jobQueue. But we need EXTRA in places where Priority is used. - }; - - /********************************************* - * ctor/dtor - * - *********************************************/ - - /** @brief ctor - */ - - PriorityThreadPool(uint targetWeightPerRun, uint highThreads, uint midThreads, - uint lowThreads, uint id = 0); - virtual ~PriorityThreadPool(); - - void removeJobs(uint32_t id); - void addJob(const Job& job, bool useLock = true); - void stop(); - - /** @brief for use in debugging - */ - void dump(); - - // If a job is blocked, we want to temporarily increase the number of threads managed by the pool - // A problem can occur if all threads are running long or blocked for a single query. Other - // queries won't get serviced, even though there are cpu cycles available. - // These calls are currently protected by respondLock in sendThread(). If you call from other - // places, you need to consider atomicity. - void incBlockedThreads() - { - blockedThreads++; } - void decBlockedThreads() + boost::shared_ptr functor; + uint32_t weight; + uint32_t priority; + uint32_t id; + uint32_t uniqueID; + uint32_t stepID; + primitiveprocessor::SP_UM_IOSOCK sock; + }; + + enum Priority + { + LOW, + MEDIUM, + HIGH, + _COUNT, + EXTRA // After _COUNT because _COUNT is for jobQueue size and EXTRA isn't a jobQueue. But we need EXTRA + // in places where Priority is used. + }; + + /********************************************* + * ctor/dtor + * + *********************************************/ + + /** @brief ctor + */ + + PriorityThreadPool(uint targetWeightPerRun, uint highThreads, uint midThreads, uint lowThreads, + uint id = 0); + virtual ~PriorityThreadPool(); + + void removeJobs(uint32_t id); + void addJob(const Job& job, bool useLock = true); + void stop(); + + /** @brief for use in debugging + */ + void dump(); + + // If a job is blocked, we want to temporarily increase the number of threads managed by the pool + // A problem can occur if all threads are running long or blocked for a single query. Other + // queries won't get serviced, even though there are cpu cycles available. + // These calls are currently protected by respondLock in sendThread(). If you call from other + // places, you need to consider atomicity. + void incBlockedThreads() + { + blockedThreads++; + } + void decBlockedThreads() + { + blockedThreads--; + } + + protected: + private: + struct ThreadHelper + { + ThreadHelper(PriorityThreadPool* impl, Priority queue) : ptp(impl), preferredQueue(queue) { - blockedThreads--; } - -protected: - -private: - struct ThreadHelper + void operator()() { - ThreadHelper(PriorityThreadPool* impl, Priority queue) : ptp(impl), preferredQueue(queue) { } - void operator()() - { - ptp->threadFcn(preferredQueue); - } - PriorityThreadPool* ptp; - Priority preferredQueue; - }; + ptp->threadFcn(preferredQueue); + } + PriorityThreadPool* ptp; + Priority preferredQueue; + }; - explicit PriorityThreadPool(); - explicit PriorityThreadPool(const PriorityThreadPool&); - PriorityThreadPool& operator=(const PriorityThreadPool&); + explicit PriorityThreadPool(); + explicit PriorityThreadPool(const PriorityThreadPool&); + PriorityThreadPool& operator=(const PriorityThreadPool&); - Priority pickAQueue(Priority preference); - void threadFcn(const Priority preferredQueue) throw(); - void sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock); + Priority pickAQueue(Priority preference); + void threadFcn(const Priority preferredQueue) throw(); + void sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock); - std::list jobQueues[3]; // higher indexes = higher priority - uint32_t threadCounts[3]; - uint32_t defaultThreadCounts[3]; - boost::mutex mutex; - boost::condition newJob; - boost::thread_group threads; - bool _stop; - uint32_t weightPerRun; - volatile uint id; // prevent it from being optimized out - - std::atomic blockedThreads; - std::atomic extraThreads; - bool stopExtra; + std::list jobQueues[3]; // higher indexes = higher priority + uint32_t threadCounts[3]; + uint32_t defaultThreadCounts[3]; + boost::mutex mutex; + boost::condition newJob; + boost::thread_group threads; + bool _stop; + uint32_t weightPerRun; + volatile uint id; // prevent it from being optimized out + + std::atomic blockedThreads; + std::atomic extraThreads; + bool stopExtra; }; -} // namespace threadpool +} // namespace threadpool -#endif //PRIORITYTHREADPOOL_H +#endif // PRIORITYTHREADPOOL_H diff --git a/utils/threadpool/tdriver.cpp b/utils/threadpool/tdriver.cpp index bd76f1648..4e371b15f 100644 --- a/utils/threadpool/tdriver.cpp +++ b/utils/threadpool/tdriver.cpp @@ -36,92 +36,81 @@ using namespace std; int thecount = 0; boost::mutex mutex; - - class ThreadPoolTestSuite : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ThreadPoolTestSuite); - CPPUNIT_TEST_SUITE( ThreadPoolTestSuite ); + CPPUNIT_TEST(test_1); - CPPUNIT_TEST( test_1 ); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); - -private: - - // Functor class - struct foo + private: + // Functor class + struct foo + { + void operator()() { - void operator ()() - { - for (int i = 0; i < 500; i++) - { - // simulate some work - fData++; - } - - boost::mutex::scoped_lock lock(mutex); - - std::cout << "count = " << ++thecount << ' ' << fData << std::endl; - } - - foo(int i): - fData(i) - {} - - foo(const foo& copy) - : fData(copy.fData) - {} - - int fData; - - }; - -public: - void setUp() - {} - - void tearDown() - {} - - void test_1() - { - - - threadpool::ThreadPool pool( 5, 10 ); - - for (int y = 0; y < 10; y++) - { - foo bar(y); - - for (int i = 0; i < 10; ++i) - { - pool.invoke(bar); - } - - // Wait until all of the queued up and in-progress work has finished - pool.wait(); - pool.dump(); - } + for (int i = 0; i < 500; i++) + { + // simulate some work + fData++; + } + boost::mutex::scoped_lock lock(mutex); + std::cout << "count = " << ++thecount << ' ' << fData << std::endl; } + foo(int i) : fData(i) + { + } + + foo(const foo& copy) : fData(copy.fData) + { + } + + int fData; + }; + + public: + void setUp() + { + } + + void tearDown() + { + } + + void test_1() + { + threadpool::ThreadPool pool(5, 10); + + for (int y = 0; y < 10; y++) + { + foo bar(y); + + for (int i = 0; i < 10; ++i) + { + pool.invoke(bar); + } + + // Wait until all of the queued up and in-progress work has finished + pool.wait(); + pool.dump(); + } + } }; - -CPPUNIT_TEST_SUITE_REGISTRATION( ThreadPoolTestSuite ); +CPPUNIT_TEST_SUITE_REGISTRATION(ThreadPoolTestSuite); #include #include - -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 1256fcbeb..fb9af113f 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: threadpool.cpp 3495 2013-01-21 14:09:51Z rdempsey $ -* -* -***********************************************************************/ + * $Id: threadpool.cpp 3495 2013-01-21 14:09:51Z rdempsey $ + * + * + ***********************************************************************/ #include #include using namespace std; @@ -37,539 +37,508 @@ using namespace logging; namespace threadpool { - -ThreadPool::ThreadPool() - : fMaxThreads( 0 ), fQueueSize( 0 ) +ThreadPool::ThreadPool() : fMaxThreads(0), fQueueSize(0) { - init(); + init(); } -ThreadPool::ThreadPool( size_t maxThreads, size_t queueSize ) - :fMaxThreads( maxThreads ), fQueueSize( queueSize ), - fPruneThread( NULL ) +ThreadPool::ThreadPool(size_t maxThreads, size_t queueSize) + : fMaxThreads(maxThreads), fQueueSize(queueSize), fPruneThread(NULL) { - init(); + init(); } - - ThreadPool::~ThreadPool() throw() { - try - { - boost::mutex::scoped_lock initLock(fInitMutex); - stop(); - } - catch (...) - { - } + try + { + boost::mutex::scoped_lock initLock(fInitMutex); + stop(); + } + catch (...) + { + } } void ThreadPool::init() { - boost::mutex::scoped_lock initLock(fInitMutex); - fThreadCount = 0; - fGeneralErrors = 0; - fFunctorErrors = 0; - waitingFunctorsSize = 0; - fIssued = 0; - fDebug = false; - fStop = false; - fNextFunctor = fWaitingFunctors.end(); - fNextHandle = 1; - fPruneThread = new boost::thread(boost::bind(&ThreadPool::pruneThread, this)); + boost::mutex::scoped_lock initLock(fInitMutex); + fThreadCount = 0; + fGeneralErrors = 0; + fFunctorErrors = 0; + waitingFunctorsSize = 0; + fIssued = 0; + fDebug = false; + fStop = false; + fNextFunctor = fWaitingFunctors.end(); + fNextHandle = 1; + fPruneThread = new boost::thread(boost::bind(&ThreadPool::pruneThread, this)); } void ThreadPool::setQueueSize(size_t queueSize) { - boost::mutex::scoped_lock lock1(fMutex); - fQueueSize = queueSize; + boost::mutex::scoped_lock lock1(fMutex); + fQueueSize = queueSize; } void ThreadPool::pruneThread() { - utils::setThreadName("pruneThread"); - boost::unique_lock lock2(fPruneMutex); + utils::setThreadName("pruneThread"); + boost::unique_lock lock2(fPruneMutex); - while(true) + while (true) + { + if (fStop) + return; + if (fPruneThreadEnd.wait_for(lock2, boost::chrono::minutes{1}) == boost::cv_status::timeout) { - - if (fStop) - return; - if (fPruneThreadEnd.wait_for(lock2, boost::chrono::minutes{1}) == - boost::cv_status::timeout) + while (!fPruneThreads.empty()) + { + if (fDebug) { - while(!fPruneThreads.empty()) - { - if (fDebug) - { - ostringstream oss; - oss << "pruning thread " << fPruneThreads.top(); - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } - fThreads.join_one(fPruneThreads.top()); - fPruneThreads.pop(); - } - } - else - { - break; + ostringstream oss; + oss << "pruning thread " << fPruneThreads.top(); + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format(args); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage(message); } + fThreads.join_one(fPruneThreads.top()); + fPruneThreads.pop(); + } } + else + { + break; + } + } } void ThreadPool::setMaxThreads(size_t maxThreads) { - boost::mutex::scoped_lock lock1(fMutex); - fMaxThreads = maxThreads; + boost::mutex::scoped_lock lock1(fMutex); + fMaxThreads = maxThreads; } void ThreadPool::stop() { - boost::mutex::scoped_lock lock1(fMutex); - if (fStop) - return; // Was stopped earlier - fStop = true; - lock1.unlock(); + boost::mutex::scoped_lock lock1(fMutex); + if (fStop) + return; // Was stopped earlier + fStop = true; + lock1.unlock(); - fPruneThreadEnd.notify_all(); - fPruneThread->join(); - delete fPruneThread; - fNeedThread.notify_all(); - fThreads.join_all(); + fPruneThreadEnd.notify_all(); + fPruneThread->join(); + delete fPruneThread; + fNeedThread.notify_all(); + fThreads.join_all(); } - void ThreadPool::wait() { - boost::mutex::scoped_lock lock1(fMutex); + boost::mutex::scoped_lock lock1(fMutex); - while (waitingFunctorsSize > 0) - { - fThreadAvailable.wait(lock1); - //cerr << "woke!" << endl; - } + while (waitingFunctorsSize > 0) + { + fThreadAvailable.wait(lock1); + // cerr << "woke!" << endl; + } } void ThreadPool::join(uint64_t thrHandle) { - boost::mutex::scoped_lock lock1(fMutex); + boost::mutex::scoped_lock lock1(fMutex); - while (waitingFunctorsSize > 0) + while (waitingFunctorsSize > 0) + { + Container_T::iterator iter; + Container_T::iterator end = fWaitingFunctors.end(); + bool foundit = false; + + for (iter = fWaitingFunctors.begin(); iter != end; ++iter) { - Container_T::iterator iter; - Container_T::iterator end = fWaitingFunctors.end(); - bool foundit = false; + foundit = false; - for (iter = fWaitingFunctors.begin(); iter != end; ++iter) - { - foundit = false; - - if (iter->hndl == thrHandle) - { - foundit = true; - break; - } - } - - if (!foundit) - { - break; - } - - fThreadAvailable.wait(lock1); + if (iter->hndl == thrHandle) + { + foundit = true; + break; + } } + + if (!foundit) + { + break; + } + + fThreadAvailable.wait(lock1); + } } void ThreadPool::join(std::vector& thrHandle) { - boost::mutex::scoped_lock lock1(fMutex); + boost::mutex::scoped_lock lock1(fMutex); - while (waitingFunctorsSize > 0) + while (waitingFunctorsSize > 0) + { + Container_T::iterator iter; + Container_T::iterator end = fWaitingFunctors.end(); + bool foundit = false; + + for (iter = fWaitingFunctors.begin(); iter != end; ++iter) { - Container_T::iterator iter; - Container_T::iterator end = fWaitingFunctors.end(); - bool foundit = false; + foundit = false; + std::vector::iterator thrIter; + std::vector::iterator thrEnd = thrHandle.end(); - for (iter = fWaitingFunctors.begin(); iter != end; ++iter) + for (thrIter = thrHandle.begin(); thrIter != thrEnd; ++thrIter) + { + if (iter->hndl == *thrIter) { - foundit = false; - std::vector::iterator thrIter; - std::vector::iterator thrEnd = thrHandle.end(); - - for (thrIter = thrHandle.begin(); thrIter != thrEnd; ++thrIter) - { - if (iter->hndl == *thrIter) - { - foundit = true; - break; - } - } - - if (foundit == true) - { - break; - } + foundit = true; + break; } + } - // If we didn't find any of the handles, then all are complete - if (!foundit) - { - break; - } - - fThreadAvailable.wait(lock1); + if (foundit == true) + { + break; + } } + + // If we didn't find any of the handles, then all are complete + if (!foundit) + { + break; + } + + fThreadAvailable.wait(lock1); + } } uint64_t ThreadPool::invoke(const Functor_T& threadfunc) { - boost::mutex::scoped_lock lock1(fMutex); - uint64_t thrHandle = 0; + boost::mutex::scoped_lock lock1(fMutex); + uint64_t thrHandle = 0; - for (;;) + for (;;) + { + try { - try - { - if (waitingFunctorsSize < fThreadCount) - { - // Don't create a thread unless it's needed. There - // is a thread available to service this request. - thrHandle = addFunctor(threadfunc); - lock1.unlock(); - break; - } + if (waitingFunctorsSize < fThreadCount) + { + // Don't create a thread unless it's needed. There + // is a thread available to service this request. + thrHandle = addFunctor(threadfunc); + lock1.unlock(); + break; + } - bool bAdded = false; + bool bAdded = false; - if (waitingFunctorsSize < fQueueSize || fQueueSize == 0) - { - // Don't create a thread unless you have to - thrHandle = addFunctor(threadfunc); - bAdded = true; - } + if (waitingFunctorsSize < fQueueSize || fQueueSize == 0) + { + // Don't create a thread unless you have to + thrHandle = addFunctor(threadfunc); + bAdded = true; + } - if (fDebug) - { - ostringstream oss; - oss << "invoke thread " << " on " << fName - << " max " << fMaxThreads - << " queue " << fQueueSize - << " threads " << fThreadCount - << " running " << fIssued - << " waiting " << (waitingFunctorsSize - fIssued) - << " total " << waitingFunctorsSize; - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } + if (fDebug) + { + ostringstream oss; + oss << "invoke thread " + << " on " << fName << " max " << fMaxThreads << " queue " << fQueueSize << " threads " + << fThreadCount << " running " << fIssued << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format(args); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage(message); + } - // fQueueSize = 0 disables the queue and is an indicator to allow any number of threads to actually run. - if (fThreadCount < fMaxThreads || fQueueSize == 0) - { - ++fThreadCount; + // fQueueSize = 0 disables the queue and is an indicator to allow any number of threads to actually run. + if (fThreadCount < fMaxThreads || fQueueSize == 0) + { + ++fThreadCount; - lock1.unlock(); - fThreads.create_thread(beginThreadFunc(*this)); + lock1.unlock(); + fThreads.create_thread(beginThreadFunc(*this)); - if (bAdded) - break; + if (bAdded) + break; - // If the mutex is unlocked before creating the thread - // this allows fThreadAvailable to be triggered - // before the wait below runs. So run the loop again. - lock1.lock(); - continue; - } + // If the mutex is unlocked before creating the thread + // this allows fThreadAvailable to be triggered + // before the wait below runs. So run the loop again. + lock1.lock(); + continue; + } - if (bAdded) - { - lock1.unlock(); - break; - } + if (bAdded) + { + lock1.unlock(); + break; + } - if (fDebug) - { - logging::Message::Args args; - logging::Message message(5); - args.add("invoke: Blocked waiting for thread. Count "); - args.add(static_cast(fThreadCount)); - args.add("max "); - args.add(static_cast(fMaxThreads)); - message.format(args); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } + if (fDebug) + { + logging::Message::Args args; + logging::Message message(5); + args.add("invoke: Blocked waiting for thread. Count "); + args.add(static_cast(fThreadCount)); + args.add("max "); + args.add(static_cast(fMaxThreads)); + message.format(args); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage(message); + } - fThreadAvailable.wait(lock1); - } - catch (...) - { - ++fGeneralErrors; - throw; - } + fThreadAvailable.wait(lock1); } + catch (...) + { + ++fGeneralErrors; + throw; + } + } - fNeedThread.notify_one(); - return thrHandle; + fNeedThread.notify_one(); + return thrHandle; } void ThreadPool::beginThread() throw() { - utils::setThreadName("Idle"); - try + utils::setThreadName("Idle"); + try + { + boost::unique_lock lock1(fMutex); + + for (;;) { - boost::unique_lock lock1(fMutex); + if (fStop) + break; - for (;;) + if (fNextFunctor == fWaitingFunctors.end()) + { + // Wait until someone needs a thread + // Add the timed wait for queueSize == 0 so we can idle away threads + // over fMaxThreads + if (fQueueSize > 0) { - if (fStop) - break; - - if (fNextFunctor == fWaitingFunctors.end()) - { - // Wait until someone needs a thread - // Add the timed wait for queueSize == 0 so we can idle away threads - // over fMaxThreads - if (fQueueSize > 0) - { - fNeedThread.wait(lock1); - } - else - { - // Wait no more than 10 minutes - if (fNeedThread.wait_for(lock1, boost::chrono::minutes{10}) == - boost::cv_status::timeout) - { - if (fThreadCount > fMaxThreads) - { - boost::mutex::scoped_lock lock2(fPruneMutex); - fPruneThreads.push(boost::this_thread::get_id()); - --fThreadCount; - return; - } - } - } - } - else - { - // If there's anything waiting, run it - if (waitingFunctorsSize - fIssued > 0) - { - Container_T::iterator todo = fNextFunctor++; - ++fIssued; - - if (fDebug) - { - ostringstream oss; - oss << "starting thread " << " on " << fName - << " max " << fMaxThreads - << " queue " << fQueueSize - << " threads " << fThreadCount - << " running " << fIssued - << " waiting " << (waitingFunctorsSize - fIssued) - << " total " << waitingFunctorsSize; - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } - - lock1.unlock(); - - utils::setThreadName("Unspecified"); - try - { - todo->functor(); - } - catch (exception& e) - { - ++fFunctorErrors; -#ifndef NOLOGGING - logging::Message::Args args; - logging::Message message(5); - args.add("ThreadPool: Caught exception during execution: "); - args.add(e.what()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logErrorMessage( message ); -#endif - } - - utils::setThreadName("Idle"); - lock1.lock(); - --fIssued; - --waitingFunctorsSize; - fWaitingFunctors.erase(todo); - if (fDebug) - { - ostringstream oss; - oss << "Ending thread " << " on " << fName - << " max " << fMaxThreads - << " queue " << fQueueSize - << " threads " << fThreadCount - << " running " << fIssued - << " waiting " << (waitingFunctorsSize - fIssued) - << " total " << waitingFunctorsSize; - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } - } - fThreadAvailable.notify_all(); - } + fNeedThread.wait(lock1); } - } - catch (exception& ex) - { - - ++fGeneralErrors; - - // Log the exception and exit this thread - try + else { + // Wait no more than 10 minutes + if (fNeedThread.wait_for(lock1, boost::chrono::minutes{10}) == boost::cv_status::timeout) + { + if (fThreadCount > fMaxThreads) + { + boost::mutex::scoped_lock lock2(fPruneMutex); + fPruneThreads.push(boost::this_thread::get_id()); + --fThreadCount; + return; + } + } + } + } + else + { + // If there's anything waiting, run it + if (waitingFunctorsSize - fIssued > 0) + { + Container_T::iterator todo = fNextFunctor++; + ++fIssued; + + if (fDebug) + { + ostringstream oss; + oss << "starting thread " + << " on " << fName << " max " << fMaxThreads << " queue " << fQueueSize << " threads " + << fThreadCount << " running " << fIssued << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format(args); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage(message); + } + + lock1.unlock(); + + utils::setThreadName("Unspecified"); + try + { + todo->functor(); + } + catch (exception& e) + { + ++fFunctorErrors; #ifndef NOLOGGING logging::Message::Args args; logging::Message message(5); - args.add("beginThread: Caught exception: "); - args.add(ex.what()); - - message.format( args ); - + args.add("ThreadPool: Caught exception during execution: "); + args.add(e.what()); + message.format(args); logging::LoggingID lid(22); logging::MessageLog ml(lid); - - ml.logErrorMessage( message ); + ml.logErrorMessage(message); #endif - } - catch (...) - { - } + } + utils::setThreadName("Idle"); + lock1.lock(); + --fIssued; + --waitingFunctorsSize; + fWaitingFunctors.erase(todo); + if (fDebug) + { + ostringstream oss; + oss << "Ending thread " + << " on " << fName << " max " << fMaxThreads << " queue " << fQueueSize << " threads " + << fThreadCount << " running " << fIssued << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format(args); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage(message); + } + } + fThreadAvailable.notify_all(); + } + } + } + catch (exception& ex) + { + ++fGeneralErrors; + + // Log the exception and exit this thread + try + { +#ifndef NOLOGGING + logging::Message::Args args; + logging::Message message(5); + args.add("beginThread: Caught exception: "); + args.add(ex.what()); + + message.format(args); + + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + + ml.logErrorMessage(message); +#endif } catch (...) { - - ++fGeneralErrors; - - // Log the exception and exit this thread - try - { -#ifndef NOLOGGING - logging::Message::Args args; - logging::Message message(6); - args.add("beginThread: Caught unknown exception!"); - - message.format( args ); - - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - - ml.logErrorMessage( message ); -#endif - } - catch (...) - { - } } + } + catch (...) + { + ++fGeneralErrors; + + // Log the exception and exit this thread + try + { +#ifndef NOLOGGING + logging::Message::Args args; + logging::Message message(6); + args.add("beginThread: Caught unknown exception!"); + + message.format(args); + + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + + ml.logErrorMessage(message); +#endif + } + catch (...) + { + } + } } uint64_t ThreadPool::addFunctor(const Functor_T& func) { - bool bAtEnd = false; + bool bAtEnd = false; - if (fNextFunctor == fWaitingFunctors.end()) - bAtEnd = true; + if (fNextFunctor == fWaitingFunctors.end()) + bAtEnd = true; - PoolFunction_T poolFunction; - poolFunction.hndl = fNextHandle; - poolFunction.functor = func; - fWaitingFunctors.push_back(poolFunction); - waitingFunctorsSize++; + PoolFunction_T poolFunction; + poolFunction.hndl = fNextHandle; + poolFunction.functor = func; + fWaitingFunctors.push_back(poolFunction); + waitingFunctorsSize++; - if (bAtEnd) - { - --fNextFunctor; - } + if (bAtEnd) + { + --fNextFunctor; + } - return fNextHandle++; + return fNextHandle++; } void ThreadPool::dump() { - std::cout << "General Errors: " << fGeneralErrors << std::endl; - std::cout << "Functor Errors: " << fFunctorErrors << std::endl; - std::cout << "Waiting functors: " << fWaitingFunctors.size() << std::endl; + std::cout << "General Errors: " << fGeneralErrors << std::endl; + std::cout << "Functor Errors: " << fFunctorErrors << std::endl; + std::cout << "Waiting functors: " << fWaitingFunctors.size() << std::endl; } - void ThreadPoolMonitor::operator()() { - ostringstream filename; - filename << MCSLOGDIR << "/trace/ThreadPool_" << fPool->name() << ".log"; - fLog = new ofstream(filename.str().c_str()); + ostringstream filename; + filename << MCSLOGDIR << "/trace/ThreadPool_" << fPool->name() << ".log"; + fLog = new ofstream(filename.str().c_str()); - for (;;) + for (;;) + { + if (!fLog || !fLog->is_open()) { - if (!fLog || !fLog->is_open()) - { - ostringstream oss; - oss << "ThreadPoolMonitor " << fPool->name() << " has no file "; - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - return; - } - - // Get a timestamp for output. - struct tm tm; - struct timeval tv; - - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); - - (*fLog) << setfill('0') - << setw(2) << tm.tm_hour << ':' - << setw(2) << tm.tm_min << ':' - << setw(2) << tm.tm_sec - << '.' - << setw(4) << tv.tv_usec / 100 - << " Name " << fPool->fName - << " Active " << fPool->waitingFunctorsSize - << " running " << fPool->fIssued - << " waiting " << (fPool->waitingFunctorsSize - fPool->fIssued) - << " ThdCnt " << fPool->fThreadCount - << " Max " << fPool->fMaxThreads - << " Q " << fPool->fQueueSize - << endl; - -// struct timespec req = { 0, 1000 * 100 }; //100 usec -// nanosleep(&req, 0); - sleep(2); + ostringstream oss; + oss << "ThreadPoolMonitor " << fPool->name() << " has no file "; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format(args); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage(message); + return; } + + // Get a timestamp for output. + struct tm tm; + struct timeval tv; + + gettimeofday(&tv, 0); + localtime_r(&tv.tv_sec, &tm); + + (*fLog) << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) + << tm.tm_sec << '.' << setw(4) << tv.tv_usec / 100 << " Name " << fPool->fName << " Active " + << fPool->waitingFunctorsSize << " running " << fPool->fIssued << " waiting " + << (fPool->waitingFunctorsSize - fPool->fIssued) << " ThdCnt " << fPool->fThreadCount << " Max " + << fPool->fMaxThreads << " Q " << fPool->fQueueSize << endl; + + // struct timespec req = { 0, 1000 * 100 }; //100 usec + // nanosleep(&req, 0); + sleep(2); + } } -} // namespace threadpool +} // namespace threadpool diff --git a/utils/threadpool/threadpool.h b/utils/threadpool/threadpool.h index 5d2c9b9b1..a5c22b3b9 100644 --- a/utils/threadpool/threadpool.h +++ b/utils/threadpool/threadpool.h @@ -16,14 +16,14 @@ MA 02110-1301, USA. */ /*********************************************************************** -* -* Work dervied from Devguy.com's Open Source C++ thread pool implementation -* released under public domain: -* http://web.archive.org/liveweb/http://dgpctk.cvs.sourceforge.net/viewvc/dgpctk/dgc%2B%2B/include/dg/thread/threadpool.h?revision=1.22&content-type=text%2Fplain -* -* http://web.archive.org/web/20100104101109/http://devguy.com/bb/viewtopic.php?t=460 -* -***********************************************************************/ + * + * Work dervied from Devguy.com's Open Source C++ thread pool implementation + * released under public domain: + * http://web.archive.org/liveweb/http://dgpctk.cvs.sourceforge.net/viewvc/dgpctk/dgc%2B%2B/include/dg/thread/threadpool.h?revision=1.22&content-type=text%2Fplain + * + * http://web.archive.org/web/20100104101109/http://devguy.com/bb/viewtopic.php?t=460 + * + ***********************************************************************/ /** @file */ @@ -59,351 +59,346 @@ namespace threadpool // Taken from boost::thread_group and adapted class ThreadPoolGroup { -private: - ThreadPoolGroup(ThreadPoolGroup const&); - ThreadPoolGroup& operator=(ThreadPoolGroup const&); -public: - ThreadPoolGroup() {} - ~ThreadPoolGroup() - { - for(std::list::iterator it=threads.begin(),end=threads.end(); - it!=end; - ++it) - { - delete *it; - } - } + private: + ThreadPoolGroup(ThreadPoolGroup const&); + ThreadPoolGroup& operator=(ThreadPoolGroup const&); - template - boost::thread* create_thread(F threadfunc) + public: + ThreadPoolGroup() + { + } + ~ThreadPoolGroup() + { + for (std::list::iterator it = threads.begin(), end = threads.end(); it != end; ++it) { - boost::lock_guard guard(m); + delete *it; + } + } + + template + boost::thread* create_thread(F threadfunc) + { + boost::lock_guard guard(m); #if __cplusplus >= 201103L - std::unique_ptr new_thread(new boost::thread(threadfunc)); + std::unique_ptr new_thread(new boost::thread(threadfunc)); #else - std::auto_ptr new_thread(new boost::thread(threadfunc)); + std::auto_ptr new_thread(new boost::thread(threadfunc)); #endif - threads.push_back(new_thread.get()); - return new_thread.release(); - } + threads.push_back(new_thread.get()); + return new_thread.release(); + } - void add_thread(boost::thread* thrd) + void add_thread(boost::thread* thrd) + { + if (thrd) { - if(thrd) - { - boost::lock_guard guard(m); - threads.push_back(thrd); - } + boost::lock_guard guard(m); + threads.push_back(thrd); } + } - void remove_thread(boost::thread* thrd) + void remove_thread(boost::thread* thrd) + { + boost::lock_guard guard(m); + std::list::iterator const it = std::find(threads.begin(), threads.end(), thrd); + if (it != threads.end()) { - boost::lock_guard guard(m); - std::list::iterator const it=std::find(threads.begin(),threads.end(),thrd); - if(it!=threads.end()) - { - threads.erase(it); - } + threads.erase(it); } + } - void join_all() + void join_all() + { + boost::shared_lock guard(m); + + for (std::list::iterator it = threads.begin(), end = threads.end(); it != end; ++it) { - boost::shared_lock guard(m); - - for(std::list::iterator it=threads.begin(),end=threads.end(); - it!=end; - ++it) - { - (*it)->join(); - } + (*it)->join(); } + } - void interrupt_all() + void interrupt_all() + { + boost::shared_lock guard(m); + + for (std::list::iterator it = threads.begin(), end = threads.end(); it != end; ++it) { - boost::shared_lock guard(m); - - for(std::list::iterator it=threads.begin(),end=threads.end(); - it!=end; - ++it) - { - (*it)->interrupt(); - } + (*it)->interrupt(); } + } - size_t size() const + size_t size() const + { + boost::shared_lock guard(m); + return threads.size(); + } + + void join_one(boost::thread::id id) + { + boost::shared_lock guard(m); + for (std::list::iterator it = threads.begin(), end = threads.end(); it != end; ++it) { - boost::shared_lock guard(m); - return threads.size(); + if ((*it)->get_id() == id) + { + (*it)->join(); + threads.erase(it); + return; + } } + } - void join_one(boost::thread::id id) - { - boost::shared_lock guard(m); - for(std::list::iterator it=threads.begin(),end=threads.end(); - it!=end; - ++it) - { - if ((*it)->get_id() == id) - { - (*it)->join(); - threads.erase(it); - return; - } - } - - } - -private: - std::list threads; - mutable boost::shared_mutex m; + private: + std::list threads; + mutable boost::shared_mutex m; }; - /** @brief ThreadPool is a component for working with pools of threads and asynchronously - * executing tasks. It is responsible for creating threads and tracking which threads are "busy" - * and which are idle. Idle threads are utilized as "work" is added to the system. - */ + * executing tasks. It is responsible for creating threads and tracking which threads are "busy" + * and which are idle. Idle threads are utilized as "work" is added to the system. + */ class ThreadPool { -public: - typedef boost::function0 Functor_T; + public: + typedef boost::function0 Functor_T; - /********************************************* - * ctor/dtor - * - *********************************************/ + /********************************************* + * ctor/dtor + * + *********************************************/ - /** @brief ctor - */ - EXPORT ThreadPool(); + /** @brief ctor + */ + EXPORT ThreadPool(); - /** @brief ctor - * - * @param maxThreads the maximum number of threads in this pool. This is the maximum number - * of simultaneuous operations that can go on. - * @param queueSize the maximum number of work tasks in the queue. This is the maximum - * number of jobs that can queue up in the work list before invoke() blocks. - * If 0, then threads never block and total threads may - * exceed maxThreads. Nothing waits. Thread count will - * idle down to maxThreads when less work is required. - */ - EXPORT explicit ThreadPool( size_t maxThreads, size_t queueSize ); + /** @brief ctor + * + * @param maxThreads the maximum number of threads in this pool. This is the maximum number + * of simultaneuous operations that can go on. + * @param queueSize the maximum number of work tasks in the queue. This is the maximum + * number of jobs that can queue up in the work list before invoke() blocks. + * If 0, then threads never block and total threads may + * exceed maxThreads. Nothing waits. Thread count will + * idle down to maxThreads when less work is required. + */ + EXPORT explicit ThreadPool(size_t maxThreads, size_t queueSize); - /** @brief dtor - */ - EXPORT ~ThreadPool() throw(); + /** @brief dtor + */ + EXPORT ~ThreadPool() throw(); + /********************************************* + * accessors/mutators + * + *********************************************/ + /** @brief set the work queue size + * + * @param queueSize the size of the work queue + */ + EXPORT void setQueueSize(size_t queueSize); - /********************************************* - * accessors/mutators - * - *********************************************/ - /** @brief set the work queue size - * - * @param queueSize the size of the work queue - */ - EXPORT void setQueueSize( size_t queueSize ); + /** @brief fet the work queue size + */ + inline size_t getQueueSize() const + { + return fQueueSize; + } - /** @brief fet the work queue size - */ - inline size_t getQueueSize() const + /** @brief set the maximum number of threads to be used to process + * the work queue + * + * @param maxThreads the maximum number of threads + */ + EXPORT void setMaxThreads(size_t maxThreads); + + /** @brief get the maximum number of threads + */ + inline size_t getMaxThreads() const + { + return fMaxThreads; + } + + /** @brief get the issued number of threads + */ + inline size_t getIssuedThreads() + { + return fIssued; + } + + /** @brief queue size accessor + * + */ + inline uint32_t getWaiting() const + { + return waitingFunctorsSize; + } + + /********************************************* + * operations + * + *********************************************/ + + /** @brief invoke a functor in a separate thread managed by the pool + * + * If all maxThreads are busy, threadfunc will be added to a work list and + * will run when a thread comes free. If all threads are busy and there are + * queueSize tasks already waiting, invoke() will block until a slot in the + * queue comes free. + */ + EXPORT uint64_t invoke(const Functor_T& threadfunc); + + /** @brief stop the threads + */ + EXPORT void stop(); + + /** @brief wait on all the threads to complete + */ + EXPORT void wait(); + + /** @brief Wait for a specific thread + */ + EXPORT void join(uint64_t thrHandle); + + /** @brief Wait for a specific thread + */ + EXPORT void join(std::vector& thrHandle); + + /** @brief for use in debugging + */ + EXPORT void dump(); + + EXPORT std::string& name() + { + return fName; + } + + EXPORT void setName(std::string name) + { + fName = name; + } + EXPORT void setName(const char* name) + { + fName = name; + } + + EXPORT bool debug() + { + return fDebug; + } + + EXPORT void setDebug(bool d) + { + fDebug = d; + } + + friend class ThreadPoolMonitor; + + protected: + private: + // Used internally to keep a handle associated with each functor for join() + struct PoolFunction_T + { + uint64_t hndl; + Functor_T functor; + }; + + /** @brief initialize data memebers + */ + void init(); + + /** @brief add a functor to the list + */ + uint64_t addFunctor(const Functor_T& func); + + /** @brief thread entry point + */ + void beginThread() throw(); + + void pruneThread(); + + ThreadPool(const ThreadPool&); + ThreadPool& operator=(const ThreadPool&); + + friend struct beginThreadFunc; + + struct beginThreadFunc + { + beginThreadFunc(ThreadPool& impl) : fImpl(impl) { - return fQueueSize; } - /** @brief set the maximum number of threads to be used to process - * the work queue - * - * @param maxThreads the maximum number of threads - */ - EXPORT void setMaxThreads( size_t maxThreads ); - - /** @brief get the maximum number of threads - */ - inline size_t getMaxThreads() const + void operator()() { - return fMaxThreads; + fImpl.beginThread(); } - /** @brief get the issued number of threads - */ - inline size_t getIssuedThreads() { return fIssued; } + ThreadPool& fImpl; + }; - /** @brief queue size accessor - * - */ - inline uint32_t getWaiting() const + struct NoOp + { + void operator()() const { - return waitingFunctorsSize; } + }; + size_t fThreadCount; + size_t fMaxThreads; + size_t fQueueSize; - /********************************************* - * operations - * - *********************************************/ + typedef std::list Container_T; + Container_T fWaitingFunctors; + Container_T::iterator fNextFunctor; - /** @brief invoke a functor in a separate thread managed by the pool - * - * If all maxThreads are busy, threadfunc will be added to a work list and - * will run when a thread comes free. If all threads are busy and there are - * queueSize tasks already waiting, invoke() will block until a slot in the - * queue comes free. - */ - EXPORT uint64_t invoke(const Functor_T& threadfunc); + uint32_t fIssued; + boost::mutex fMutex; + boost::condition_variable fThreadAvailable; // triggered when a thread is available + boost::condition_variable fNeedThread; // triggered when a thread is needed + ThreadPoolGroup fThreads; - /** @brief stop the threads - */ - EXPORT void stop(); + bool fStop; + long fGeneralErrors; + long fFunctorErrors; + uint32_t waitingFunctorsSize; + uint64_t fNextHandle; - /** @brief wait on all the threads to complete - */ - EXPORT void wait(); - - /** @brief Wait for a specific thread - */ - EXPORT void join(uint64_t thrHandle); - - /** @brief Wait for a specific thread - */ - EXPORT void join(std::vector& thrHandle); - - /** @brief for use in debugging - */ - EXPORT void dump(); - - EXPORT std::string& name() - { - return fName; - } - - EXPORT void setName(std::string name) - { - fName = name; - } - EXPORT void setName(const char* name) - { - fName = name; - } - - EXPORT bool debug() - { - return fDebug; - } - - EXPORT void setDebug(bool d) - { - fDebug = d; - } - - - friend class ThreadPoolMonitor; -protected: - -private: - // Used internally to keep a handle associated with each functor for join() - struct PoolFunction_T - { - uint64_t hndl; - Functor_T functor; - }; - - /** @brief initialize data memebers - */ - void init(); - - /** @brief add a functor to the list - */ - uint64_t addFunctor(const Functor_T& func); - - /** @brief thread entry point - */ - void beginThread() throw(); - - void pruneThread(); - - ThreadPool(const ThreadPool&); - ThreadPool& operator = (const ThreadPool&); - - friend struct beginThreadFunc; - - struct beginThreadFunc - { - beginThreadFunc(ThreadPool& impl) - : fImpl(impl) - {} - - void operator() () - { - fImpl.beginThread(); - } - - ThreadPool& fImpl; - }; - - struct NoOp - { - void operator () () const - {} - }; - - size_t fThreadCount; - size_t fMaxThreads; - size_t fQueueSize; - - typedef std::list Container_T; - Container_T fWaitingFunctors; - Container_T::iterator fNextFunctor; - - uint32_t fIssued; - boost::mutex fMutex; - boost::condition_variable fThreadAvailable; // triggered when a thread is available - boost::condition_variable fNeedThread; // triggered when a thread is needed - ThreadPoolGroup fThreads; - - bool fStop; - long fGeneralErrors; - long fFunctorErrors; - uint32_t waitingFunctorsSize; - uint64_t fNextHandle; - - std::string fName; // Optional to add a name to the pool for debugging. - bool fDebug; - boost::mutex fInitMutex; - boost::mutex fPruneMutex; - boost::condition_variable fPruneThreadEnd; - boost::thread* fPruneThread; - std::stack fPruneThreads; // A list of stale thread IDs to be joined + std::string fName; // Optional to add a name to the pool for debugging. + bool fDebug; + boost::mutex fInitMutex; + boost::mutex fPruneMutex; + boost::condition_variable fPruneThreadEnd; + boost::thread* fPruneThread; + std::stack fPruneThreads; // A list of stale thread IDs to be joined }; // This class, if instantiated, will continuously log details about the indicated threadpool // The log will end up in /var/log/mariadb/columnstore/trace/threadpool_.log class ThreadPoolMonitor { -public: - ThreadPoolMonitor(ThreadPool* pool) : fPool(pool), fLog(NULL) - { - } + public: + ThreadPoolMonitor(ThreadPool* pool) : fPool(pool), fLog(NULL) + { + } - ~ThreadPoolMonitor() + ~ThreadPoolMonitor() + { + if (fLog) { - if (fLog) - { - delete fLog; - } + delete fLog; } + } - void operator()(); -private: - //defaults okay - //ThreadPoolMonitor(const ThreadPoolMonitor& rhs); - //ThreadPoolMonitor& operator=(const ThreadPoolMonitor& rhs); - ThreadPool* fPool; - std::ofstream* fLog; + void operator()(); + + private: + // defaults okay + // ThreadPoolMonitor(const ThreadPoolMonitor& rhs); + // ThreadPoolMonitor& operator=(const ThreadPoolMonitor& rhs); + ThreadPool* fPool; + std::ofstream* fLog; }; -} // namespace threadpool +} // namespace threadpool #undef EXPORT -#endif //THREADPOOL_H +#endif // THREADPOOL_H diff --git a/utils/threadpool/tp.cpp b/utils/threadpool/tp.cpp index fc3b17610..e626504a9 100644 --- a/utils/threadpool/tp.cpp +++ b/utils/threadpool/tp.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include #include #include @@ -35,114 +34,114 @@ boost::mutex mutex; const string timeNow() { - time_t outputTime = time(0); - struct tm ltm; - char buf[32]; //ctime(3) says at least 26 - size_t len = 0; + time_t outputTime = time(0); + struct tm ltm; + char buf[32]; // ctime(3) says at least 26 + size_t len = 0; #ifdef _MSC_VER - asctime_s(buf, 32, localtime_r(&outputTime, <m)); + asctime_s(buf, 32, localtime_r(&outputTime, <m)); #else - asctime_r(localtime_r(&outputTime, <m), buf); + asctime_r(localtime_r(&outputTime, <m), buf); #endif - len = strlen(buf); + len = strlen(buf); - if (len > 0) --len; + if (len > 0) + --len; - if (buf[len] == '\n') buf[len] = 0; + if (buf[len] == '\n') + buf[len] = 0; - return buf; + return buf; } // Functor class struct foo { - int64_t fData; - int64_t fThd; - string start; - bool running; + int64_t fData; + int64_t fThd; + string start; + bool running; - void operator ()() - { - start = timeNow(); + void operator()() + { + start = timeNow(); - std::cout << "foo thd = " << fThd << " start " << start << std::endl; + std::cout << "foo thd = " << fThd << " start " << start << std::endl; - for (int64_t i = 0; i < 1024 * 1024 * (fThd + 0) * 128; i++) - // simulate some work - fData++; + for (int64_t i = 0; i < 1024 * 1024 * (fThd + 0) * 128; i++) + // simulate some work + fData++; - boost::mutex::scoped_lock lock(mutex); - std::cout << "foo thd = " << fThd << " start " << start << " fin " << timeNow() << std::endl; - } + boost::mutex::scoped_lock lock(mutex); + std::cout << "foo thd = " << fThd << " start " << start << " fin " << timeNow() << std::endl; + } - foo(int64_t i) : fThd(i), fData(i), running(true) - { - start = timeNow(); - } + foo(int64_t i) : fThd(i), fData(i), running(true) + { + start = timeNow(); + } - foo(const foo& copy) : fData(copy.fData), fThd(copy.fThd), start(copy.start), running(copy.running) - { - std::cout << "new foo " << fThd << endl; - } + foo(const foo& copy) : fData(copy.fData), fThd(copy.fThd), start(copy.start), running(copy.running) + { + std::cout << "new foo " << fThd << endl; + } - ~foo() - { - running = false; - } + ~foo() + { + running = false; + } }; - - -int main( int argc, char** argv) +int main(int argc, char** argv) { - threadpool::ThreadPool pool( 20, 10 ); - std::vector hndl; - hndl.reserve(10); - int t1 = hndl.capacity(); - uint64_t testHndl; - uint64_t thdhndl = 999; - int64_t thd = 1; - boost::function0 foofunc; - boost::function0 foofunc2; + threadpool::ThreadPool pool(20, 10); + std::vector hndl; + hndl.reserve(10); + int t1 = hndl.capacity(); + uint64_t testHndl; + uint64_t thdhndl = 999; + int64_t thd = 1; + boost::function0 foofunc; + boost::function0 foofunc2; - for (int64_t y = 0; y < 1; y++) + for (int64_t y = 0; y < 1; y++) + { + foo bar(y); + // foofunc = bar; + // foofunc2 = foofunc; + std::cout << "Done with assign" << std::endl; + + for (int64_t i = 0; i < 1; ++i) { - foo bar(y); -// foofunc = bar; -// foofunc2 = foofunc; - std::cout << "Done with assign" << std::endl; + bar.fThd = thd++; + thdhndl = pool.invoke(bar); - for (int64_t i = 0; i < 1; ++i) - { - bar.fThd = thd++; - thdhndl = pool.invoke(bar); + if (y < 10) + { + hndl.push_back(thdhndl); + } - if (y < 10) - { - hndl.push_back(thdhndl); - } - - if (y == 0) - { - testHndl = thdhndl; - } - } - - boost::mutex::scoped_lock lock(mutex); + if (y == 0) + { + testHndl = thdhndl; + } } - // Wait until all of the queued up and in-progress work has finished - std::cout << "Threads for join " << hndl.size() << std::endl; - pool.dump(); - std::cout << "*** JOIN 1 ***" << std::endl; - pool.join(testHndl); - pool.dump(); - std::cout << "*** JOIN 10 ***" << std::endl; - pool.join(hndl); - pool.dump(); - std::cout << "*** WAIT ***" << std::endl; - pool.wait(); - pool.dump(); - sleep(2); - return 0; + boost::mutex::scoped_lock lock(mutex); + } + + // Wait until all of the queued up and in-progress work has finished + std::cout << "Threads for join " << hndl.size() << std::endl; + pool.dump(); + std::cout << "*** JOIN 1 ***" << std::endl; + pool.join(testHndl); + pool.dump(); + std::cout << "*** JOIN 10 ***" << std::endl; + pool.join(hndl); + pool.dump(); + std::cout << "*** WAIT ***" << std::endl; + pool.wait(); + pool.dump(); + sleep(2); + return 0; } diff --git a/utils/threadpool/weightedthreadpool.cpp b/utils/threadpool/weightedthreadpool.cpp index 9ab05643d..45cee2a5e 100644 --- a/utils/threadpool/weightedthreadpool.cpp +++ b/utils/threadpool/weightedthreadpool.cpp @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id: threadpool.cpp 553 2008-02-27 17:51:16Z rdempsey $ -* -* -***********************************************************************/ + * $Id: threadpool.cpp 553 2008-02-27 17:51:16Z rdempsey $ + * + * + ***********************************************************************/ #include #include @@ -33,378 +33,362 @@ using namespace logging; namespace threadpool { - -WeightedThreadPool::WeightedThreadPool() - : fMaxThreadWeight(0), fMaxThreads( 0 ), fQueueSize( 0 ) +WeightedThreadPool::WeightedThreadPool() : fMaxThreadWeight(0), fMaxThreads(0), fQueueSize(0) { - init(); + init(); } -WeightedThreadPool::WeightedThreadPool( size_t maxThreadWeight, size_t maxThreads, size_t queueSize ) - : fMaxThreadWeight(maxThreadWeight), fMaxThreads( maxThreads ), fQueueSize( queueSize ) +WeightedThreadPool::WeightedThreadPool(size_t maxThreadWeight, size_t maxThreads, size_t queueSize) + : fMaxThreadWeight(maxThreadWeight), fMaxThreads(maxThreads), fQueueSize(queueSize) { - init(); + init(); - if (fQueueSize == 0) - fQueueSize = fMaxThreads * 2; + if (fQueueSize == 0) + fQueueSize = fMaxThreads * 2; } - WeightedThreadPool::~WeightedThreadPool() throw() { -// delete fThreadCreated; - try - { - stop(); - } - catch (...) - {} + // delete fThreadCreated; + try + { + stop(); + } + catch (...) + { + } } - void WeightedThreadPool::init() { - fThreadCount = 0; - fGeneralErrors = 0; - fFunctorErrors = 0; - fWaitingFunctorsSize = 0; - fWaitingFunctorsWeight = 0; - issued = 0; - fStop = false; -// fThreadCreated = new NoOp(); - fNextFunctor = fWaitingFunctors.end(); + fThreadCount = 0; + fGeneralErrors = 0; + fFunctorErrors = 0; + fWaitingFunctorsSize = 0; + fWaitingFunctorsWeight = 0; + issued = 0; + fStop = false; + // fThreadCreated = new NoOp(); + fNextFunctor = fWaitingFunctors.end(); } - void WeightedThreadPool::setQueueSize(size_t queueSize) { - boost::mutex::scoped_lock lock1(fMutex); - fQueueSize = queueSize; + boost::mutex::scoped_lock lock1(fMutex); + fQueueSize = queueSize; } - void WeightedThreadPool::setMaxThreads(size_t maxThreads) { - boost::mutex::scoped_lock lock1(fMutex); - fMaxThreads = maxThreads; + boost::mutex::scoped_lock lock1(fMutex); + fMaxThreads = maxThreads; } - void WeightedThreadPool::setMaxThreadWeight(size_t maxWeight) { - boost::mutex::scoped_lock lock1(fMutex); - fMaxThreadWeight = maxWeight; + boost::mutex::scoped_lock lock1(fMutex); + fMaxThreadWeight = maxWeight; } - void WeightedThreadPool::setThreadCreatedListener(const Functor_T& f) { -// fThreadCreated = f; + // fThreadCreated = f; } - void WeightedThreadPool::stop() { - boost::mutex::scoped_lock lock1(fMutex); - fStop = true; - lock1.unlock(); + boost::mutex::scoped_lock lock1(fMutex); + fStop = true; + lock1.unlock(); - fNeedThread.notify_all(); - fThreads.join_all(); + fNeedThread.notify_all(); + fThreads.join_all(); } - void WeightedThreadPool::wait() { - boost::mutex::scoped_lock lock1(fMutex); + boost::mutex::scoped_lock lock1(fMutex); - while (fWaitingFunctorsSize > 0) - { - //cout << "waiting ..." << endl; - fThreadAvailable.wait(lock1); - //cerr << "woke!" << endl; - } + while (fWaitingFunctorsSize > 0) + { + // cout << "waiting ..." << endl; + fThreadAvailable.wait(lock1); + // cerr << "woke!" << endl; + } } void WeightedThreadPool::removeJobs(uint32_t id) { - boost::mutex::scoped_lock lock1(fMutex); - Container_T::iterator it; + boost::mutex::scoped_lock lock1(fMutex); + Container_T::iterator it; - it = fNextFunctor; + it = fNextFunctor; - while (it != fWaitingFunctors.end()) + while (it != fWaitingFunctors.end()) + { + if (it->id == id) { - if (it->id == id) - { - fWaitingFunctorsWeight -= it->functorWeight; - fWaitingFunctorsSize--; + fWaitingFunctorsWeight -= it->functorWeight; + fWaitingFunctorsSize--; - if (it == fNextFunctor) - { - fWaitingFunctors.erase(fNextFunctor++); - it = fNextFunctor; - } - else - fWaitingFunctors.erase(it++); - } - else - ++it; + if (it == fNextFunctor) + { + fWaitingFunctors.erase(fNextFunctor++); + it = fNextFunctor; + } + else + fWaitingFunctors.erase(it++); } + else + ++it; + } } -void WeightedThreadPool::invoke(const Functor_T& threadfunc, uint32_t functor_weight, - uint32_t id) +void WeightedThreadPool::invoke(const Functor_T& threadfunc, uint32_t functor_weight, uint32_t id) { - boost::mutex::scoped_lock lock1(fMutex); + boost::mutex::scoped_lock lock1(fMutex); - for (;;) + for (;;) + { + try { - try - { - if ( fWaitingFunctorsSize < fThreadCount) - { - // Don't create a thread unless it's needed. There - // is a thread available to service this request. - addFunctor(threadfunc, functor_weight, id); - lock1.unlock(); - break; - } + if (fWaitingFunctorsSize < fThreadCount) + { + // Don't create a thread unless it's needed. There + // is a thread available to service this request. + addFunctor(threadfunc, functor_weight, id); + lock1.unlock(); + break; + } - bool bAdded = false; + bool bAdded = false; - if ( fWaitingFunctorsSize < fQueueSize) - { - // Don't create a thread unless you have to - addFunctor(threadfunc, functor_weight, id); - bAdded = true; - } + if (fWaitingFunctorsSize < fQueueSize) + { + // Don't create a thread unless you have to + addFunctor(threadfunc, functor_weight, id); + bAdded = true; + } - // add a thread is necessary - if ( fThreadCount < fMaxThreads) - { - ++fThreadCount; - //cout << "\t++invoke() tcnt=" << fThreadCount << endl; - lock1.unlock(); - fThreads.create_thread(beginThreadFunc(*this)); + // add a thread is necessary + if (fThreadCount < fMaxThreads) + { + ++fThreadCount; + // cout << "\t++invoke() tcnt=" << fThreadCount << endl; + lock1.unlock(); + fThreads.create_thread(beginThreadFunc(*this)); - if (bAdded) - break; + if (bAdded) + break; - // If the mutex is unlocked before creating the thread - // this allows fThreadAvailable to be triggered - // before the wait below runs. So run the loop again. - lock1.lock(); - continue; - } + // If the mutex is unlocked before creating the thread + // this allows fThreadAvailable to be triggered + // before the wait below runs. So run the loop again. + lock1.lock(); + continue; + } - //else - // cout << "invoke() no thread created c=" << fThreadCount << " m=" << fMaxThreads << endl; + // else + // cout << "invoke() no thread created c=" << fThreadCount << " m=" << fMaxThreads << endl; - if (bAdded) - { - lock1.unlock(); - break; - } + if (bAdded) + { + lock1.unlock(); + break; + } - fThreadAvailable.wait(lock1); - } - catch (...) - { - ++fGeneralErrors; - throw; - } + fThreadAvailable.wait(lock1); } + catch (...) + { + ++fGeneralErrors; + throw; + } + } - fNeedThread.notify_one(); + fNeedThread.notify_one(); } void WeightedThreadPool::beginThread() throw() { - vector reschedule; + vector reschedule; + try + { + // fThreadCreated(); + boost::mutex::scoped_lock lock1(fMutex); + + for (;;) + { + if (fStop) + break; + + if (fNextFunctor == fWaitingFunctors.end()) + { + // Wait until someone needs a thread + fNeedThread.wait(lock1); + } + else + { + vector todoList; + int i, num = (fWaitingFunctorsSize - issued); + Container_T::const_iterator iter; + uint32_t weight = 0; + + for (i = 0; i < num && weight < fMaxThreadWeight; i++) + { + weight += (*fNextFunctor).functorWeight; + todoList.push_back(fNextFunctor++); + } + + issued += i; + num = i; + lock1.unlock(); + + // cerr << "beginThread() " << num + // << " jobs - fWaitingFunctorsSize=" << fWaitingFunctorsSize + // << " fWaitingFunctorsWeight=" << fWaitingFunctorsWeight + // << " weight=" << weight + // << " issued=" << issued << " todo=" << todoList.size() + // << " fThreadCount=" << fThreadCount << endl; + + i = 0; + reschedule.resize(num); + bool allWereRescheduled = true, someWereRescheduled = false; + + while (i < num) + { + try + { + for (; i < num; i++) + { + reschedule[i] = false; // in case of exception in the next line + reschedule[i] = ((*todoList[i]).functor)(); + allWereRescheduled &= reschedule[i]; + someWereRescheduled |= reschedule[i]; + } + } + catch (exception& e) + { + i++; + ++fFunctorErrors; + cerr << e.what() << endl; + } + } + + // no real work was done, prevent intensive busy waiting + if (allWereRescheduled) + usleep(1000); + + // cout << "running " << i << "/" << num << " functor" < 1) + fNeedThread.notify_all(); + else + fNeedThread.notify_one(); + } + + issued -= num; + + for (i = 0; i < num; i++) + { + fWaitingFunctorsWeight -= (*todoList[i]).functorWeight; + fWaitingFunctors.erase(todoList[i]); + } + + fWaitingFunctorsSize -= num; + + // if (fWaitingFunctorsSize != fWaitingFunctors.size()) ; + // cerr << "num=" << num << " cleaned=" << i << " size=" + // << fWaitingFunctorsSize << " list size=" + // << fWaitingFunctors.size() + // << " w="< todoList; - int i, num = (fWaitingFunctorsSize - issued); - Container_T::const_iterator iter; - uint32_t weight = 0; - - for (i = 0; i < num && weight < fMaxThreadWeight; i++) - { - weight += (*fNextFunctor).functorWeight; - todoList.push_back(fNextFunctor++); - } - - issued += i; - num = i; - lock1.unlock(); - - //cerr << "beginThread() " << num - // << " jobs - fWaitingFunctorsSize=" << fWaitingFunctorsSize - // << " fWaitingFunctorsWeight=" << fWaitingFunctorsWeight - // << " weight=" << weight - // << " issued=" << issued << " todo=" << todoList.size() - // << " fThreadCount=" << fThreadCount << endl; - - i = 0; - reschedule.resize(num); - bool allWereRescheduled = true, someWereRescheduled = false; - - while (i < num) - { - try - { - for (; i < num; i++) - { - reschedule[i] = false; // in case of exception in the next line - reschedule[i] = ((*todoList[i]).functor)(); - allWereRescheduled &= reschedule[i]; - someWereRescheduled |= reschedule[i]; - } - } - catch (exception& e) - { - i++; - ++fFunctorErrors; - cerr << e.what() << endl; - } - } - - // no real work was done, prevent intensive busy waiting - if (allWereRescheduled) - usleep(1000); - - //cout << "running " << i << "/" << num << " functor" < 1) - fNeedThread.notify_all(); - else - fNeedThread.notify_one(); - } - - issued -= num; - - for (i = 0; i < num; i++) - { - fWaitingFunctorsWeight -= (*todoList[i]).functorWeight; - fWaitingFunctors.erase(todoList[i]); - } - - fWaitingFunctorsSize -= num; - - //if (fWaitingFunctorsSize != fWaitingFunctors.size()) ; - // cerr << "num=" << num << " cleaned=" << i << " size=" - // << fWaitingFunctorsSize << " list size=" - // << fWaitingFunctors.size() - // << " w="< Functor_T; + public: + typedef boost::function0 Functor_T; - /********************************************* - * ctor/dtor - * - *********************************************/ + /********************************************* + * ctor/dtor + * + *********************************************/ - /** @brief ctor - */ - WeightedThreadPool(); + /** @brief ctor + */ + WeightedThreadPool(); - /** @brief ctor - * - * @param maxThreads the maximum number of threads in this pool. This is the maximum number - * of simultaneuous operations that can go on. - * @param queueSize the maximum number of work tasks in the queue. This is the maximum - * number of jobs that can queue up in the work list before invoke() blocks. - */ - explicit WeightedThreadPool( size_t maxThreadWeight, size_t maxThreads, size_t queueSize ); + /** @brief ctor + * + * @param maxThreads the maximum number of threads in this pool. This is the maximum number + * of simultaneuous operations that can go on. + * @param queueSize the maximum number of work tasks in the queue. This is the maximum + * number of jobs that can queue up in the work list before invoke() blocks. + */ + explicit WeightedThreadPool(size_t maxThreadWeight, size_t maxThreads, size_t queueSize); - /** @brief dtor - */ - ~WeightedThreadPool() throw(); + /** @brief dtor + */ + ~WeightedThreadPool() throw(); + /********************************************* + * accessors/mutators + * + *********************************************/ + /** @brief set the work queue size + * + * @param queueSize the size of the work queue + */ + void setQueueSize(size_t queueSize); - /********************************************* - * accessors/mutators - * - *********************************************/ - /** @brief set the work queue size - * - * @param queueSize the size of the work queue - */ - void setQueueSize( size_t queueSize ); + /** @brief fet the work queue size + */ + inline size_t getQueueSize() const + { + return fQueueSize; + } - /** @brief fet the work queue size - */ - inline size_t getQueueSize() const + /** @brief set the maximum number of threads to be used to process + * the work queue + * + * @param maxThreads the maximum number of threads + */ + void setMaxThreads(size_t maxThreads); + + /** @brief get the maximum number of threads + */ + inline size_t getMaxThreads() const + { + return fMaxThreads; + } + + /** @brief set the maximum processing weight of a thread to be + * submitted for execution from the existing jobs + * scheduled in the work queue + * + * @param maxWeight for execution + */ + void setMaxThreadWeight(size_t maxWeight); + + /** @brief get the maximum number of threads + */ + inline uint32_t getMaxThreadWeight() const + { + return fMaxThreadWeight; + } + + /** @brief register a functor to be called when a new thread + * is created + */ + void setThreadCreatedListener(const Functor_T& f); + + /** @brief queue size accessor + * + */ + inline uint32_t getWaiting() const + { + return fWaitingFunctorsSize; + } + + inline uint32_t getWeight() const + { + return fWaitingFunctorsWeight; + } + + void removeJobs(uint32_t id); + + /********************************************* + * operations + * + *********************************************/ + + /** @brief invoke a functor in a separate thread managed by the pool + * + * If all maxThreads are busy, threadfunc will be added to a work list and + * will run when a thread comes free. If all threads are busy and there are + * queueSize tasks already waiting, invoke() will block until a slot in the + * queue comes free. + */ + void invoke(const Functor_T& threadfunc, uint32_t functor_weight, uint32_t id); + + /** @brief stop the threads + */ + void stop(); + + /** @brief wait on all the threads to complete + */ + void wait(); + + /** @brief for use in debugging + */ + void dump(); + + protected: + private: + /** @brief initialize data memebers + */ + void init(); + + /** @brief add a functor to the list + */ + void addFunctor(const Functor_T& func, uint32_t functor_weight, uint32_t id); + + /** @brief thread entry point + */ + void beginThread() throw(); + + WeightedThreadPool(const WeightedThreadPool&); + WeightedThreadPool& operator=(const WeightedThreadPool&); + + friend struct beginThreadFunc; + + struct beginThreadFunc + { + beginThreadFunc(WeightedThreadPool& impl) : fImpl(impl) { - return fQueueSize; } - /** @brief set the maximum number of threads to be used to process - * the work queue - * - * @param maxThreads the maximum number of threads - */ - void setMaxThreads( size_t maxThreads ); - - /** @brief get the maximum number of threads - */ - inline size_t getMaxThreads() const + void operator()() { - return fMaxThreads; + fImpl.beginThread(); } - /** @brief set the maximum processing weight of a thread to be - * submitted for execution from the existing jobs - * scheduled in the work queue - * - * @param maxWeight for execution - */ - void setMaxThreadWeight( size_t maxWeight ); + WeightedThreadPool& fImpl; + }; - /** @brief get the maximum number of threads - */ - inline uint32_t getMaxThreadWeight() const + struct NoOp + { + void operator()() const { - return fMaxThreadWeight; } + }; - /** @brief register a functor to be called when a new thread - * is created - */ - void setThreadCreatedListener(const Functor_T& f) ; + size_t fThreadCount; + size_t fMaxThreadWeight; + size_t fMaxThreads; + size_t fQueueSize; - /** @brief queue size accessor - * - */ - inline uint32_t getWaiting() const - { - return fWaitingFunctorsSize; - } + // typedef std::list Container_T; + struct FunctorListItemStruct + { + Functor_T functor; + uint32_t functorWeight; + uint32_t id; + }; - inline uint32_t getWeight() const - { - return fWaitingFunctorsWeight; - } + typedef FunctorListItemStruct FunctorListItem; + typedef std::list Container_T; + Container_T fWaitingFunctors; + Container_T::iterator fNextFunctor; - void removeJobs(uint32_t id); - - /********************************************* - * operations - * - *********************************************/ - - /** @brief invoke a functor in a separate thread managed by the pool - * - * If all maxThreads are busy, threadfunc will be added to a work list and - * will run when a thread comes free. If all threads are busy and there are - * queueSize tasks already waiting, invoke() will block until a slot in the - * queue comes free. - */ - void invoke(const Functor_T& threadfunc, uint32_t functor_weight, uint32_t id); - - /** @brief stop the threads - */ - void stop(); - - /** @brief wait on all the threads to complete - */ - void wait(); - - /** @brief for use in debugging - */ - void dump(); - -protected: - -private: - /** @brief initialize data memebers - */ - void init(); - - /** @brief add a functor to the list - */ - void addFunctor(const Functor_T& func, uint32_t functor_weight, uint32_t id); - - /** @brief thread entry point - */ - void beginThread() throw(); - - - WeightedThreadPool(const WeightedThreadPool&); - WeightedThreadPool& operator = (const WeightedThreadPool&); - - friend struct beginThreadFunc; - - struct beginThreadFunc - { - beginThreadFunc(WeightedThreadPool& impl) - : fImpl(impl) - {} - - void operator() () - { - fImpl.beginThread(); - } - - WeightedThreadPool& fImpl; - }; - - struct NoOp - { - void operator () () const - {} - }; - - size_t fThreadCount; - size_t fMaxThreadWeight; - size_t fMaxThreads; - size_t fQueueSize; - - //typedef std::list Container_T; - struct FunctorListItemStruct - { - Functor_T functor; - uint32_t functorWeight; - uint32_t id; - }; - - typedef FunctorListItemStruct FunctorListItem; - typedef std::list Container_T; - Container_T fWaitingFunctors; - Container_T::iterator fNextFunctor; - - uint32_t issued; - boost::mutex fMutex; - boost::condition fThreadAvailable; // triggered when a thread is available - boost::condition fNeedThread; // triggered when a thread is needed - boost::thread_group fThreads; - - bool fStop; - long fGeneralErrors; - long fFunctorErrors; - uint16_t fWaitingFunctorsSize; - uint16_t fWaitingFunctorsWeight; + uint32_t issued; + boost::mutex fMutex; + boost::condition fThreadAvailable; // triggered when a thread is available + boost::condition fNeedThread; // triggered when a thread is needed + boost::thread_group fThreads; + bool fStop; + long fGeneralErrors; + long fFunctorErrors; + uint16_t fWaitingFunctorsSize; + uint16_t fWaitingFunctorsWeight; }; -} // namespace threadpool +} // namespace threadpool -#endif //WEIGHTEDTHREADPOOL_H +#endif // WEIGHTEDTHREADPOOL_H diff --git a/utils/threadpool/wtp.cpp b/utils/threadpool/wtp.cpp index 28561f64c..9b31e0f9a 100644 --- a/utils/threadpool/wtp.cpp +++ b/utils/threadpool/wtp.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - #include #include #include @@ -37,49 +36,45 @@ boost::mutex mutex; // Functor class struct foo { - void operator ()() - { - for (int i = 0; i < 1024 * 1024 * 10; i++) - // simulate some work - fData++; + void operator()() + { + for (int i = 0; i < 1024 * 1024 * 10; i++) + // simulate some work + fData++; - //boost::mutex::scoped_lock lock(mutex); - //std::cout << "foo count = " << ++thecount << " " << fData << std::endl; - } + // boost::mutex::scoped_lock lock(mutex); + // std::cout << "foo count = " << ++thecount << " " << fData << std::endl; + } - foo(int i): - fData(i) - {} + foo(int i) : fData(i) + { + } - foo(const foo& copy) - : fData(copy.fData) - {} - - int fData; + foo(const foo& copy) : fData(copy.fData) + { + } + int fData; }; - - -int main( int argc, char** argv) +int main(int argc, char** argv) { - threadpool::WeightedThreadPool pool( 100, 10, 5 ); + threadpool::WeightedThreadPool pool(100, 10, 5); - for (int y = 0; y < 10; y++) + for (int y = 0; y < 10; y++) + { + foo bar(y); + + for (int i = 0; i < 10; ++i) { - foo bar(y); - - for (int i = 0; i < 10; ++i) - { - pool.invoke(bar, 25); - } - - boost::mutex::scoped_lock lock(mutex); - std::cout << "count = " << ++thecount << std::endl; - - // Wait until all of the queued up and in-progress work has finished - pool.wait(); - pool.dump(); + pool.invoke(bar, 25); } + boost::mutex::scoped_lock lock(mutex); + std::cout << "count = " << ++thecount << std::endl; + + // Wait until all of the queued up and in-progress work has finished + pool.wait(); + pool.dump(); + } } diff --git a/utils/udfsdk/allnull.cpp b/utils/udfsdk/allnull.cpp index 79f978124..1de4124d4 100644 --- a/utils/udfsdk/allnull.cpp +++ b/utils/udfsdk/allnull.cpp @@ -21,70 +21,67 @@ using namespace mcsv1sdk; struct allnull_data { - uint64_t totalQuantity; - uint64_t totalNulls; + uint64_t totalQuantity; + uint64_t totalNulls; }; #define OUT_TYPE int64_t -mcsv1_UDAF::ReturnCode allnull::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode allnull::init(mcsv1Context* context, ColumnDatum* colTypes) { - context->setUserDataSize(sizeof(allnull_data)); + context->setUserDataSize(sizeof(allnull_data)); - if (context->getParameterCount() < 1) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("allnull() with 0 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() < 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("allnull() with 0 arguments"); + return mcsv1_UDAF::ERROR; + } - context->setResultType(execplan::CalpontSystemCatalog::TINYINT); + context->setResultType(execplan::CalpontSystemCatalog::TINYINT); - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode allnull::reset(mcsv1Context* context) { - struct allnull_data* data = (struct allnull_data*)context->getUserData()->data; - data->totalQuantity = 0; - data->totalNulls = 0; - return mcsv1_UDAF::SUCCESS; + struct allnull_data* data = (struct allnull_data*)context->getUserData()->data; + data->totalQuantity = 0; + data->totalNulls = 0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode allnull::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - struct allnull_data* data = (struct allnull_data*)context->getUserData()->data; + struct allnull_data* data = (struct allnull_data*)context->getUserData()->data; - for (size_t i = 0; i < context->getParameterCount(); i++) + for (size_t i = 0; i < context->getParameterCount(); i++) + { + data->totalQuantity++; + + if (context->isParamNull(0)) { - data->totalQuantity++; - - if (context->isParamNull(0)) - { - data->totalNulls++; - } + data->totalNulls++; } + } - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode allnull::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - struct allnull_data* outData = (struct allnull_data*)context->getUserData()->data; - struct allnull_data* inData = (struct allnull_data*)userDataIn->data; - outData->totalQuantity += inData->totalQuantity; - outData->totalNulls += inData->totalNulls; - return mcsv1_UDAF::SUCCESS; + struct allnull_data* outData = (struct allnull_data*)context->getUserData()->data; + struct allnull_data* inData = (struct allnull_data*)userDataIn->data; + outData->totalQuantity += inData->totalQuantity; + outData->totalNulls += inData->totalNulls; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode allnull::evaluate(mcsv1Context* context, static_any::any& valOut) { - OUT_TYPE allNull; - struct allnull_data* data = (struct allnull_data*)context->getUserData()->data; - allNull = data->totalQuantity > 0 && data->totalNulls == data->totalQuantity; - valOut = allNull; - return mcsv1_UDAF::SUCCESS; + OUT_TYPE allNull; + struct allnull_data* data = (struct allnull_data*)context->getUserData()->data; + allNull = data->totalQuantity > 0 && data->totalNulls == data->totalQuantity; + valOut = allNull; + return mcsv1_UDAF::SUCCESS; } - - diff --git a/utils/udfsdk/allnull.h b/utils/udfsdk/allnull.h index 203bfbbaf..7f78de52f 100644 --- a/utils/udfsdk/allnull.h +++ b/utils/udfsdk/allnull.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* mcsv1_UDAF.h -***********************************************************************/ + * $Id$ + * + * mcsv1_UDAF.h + ***********************************************************************/ /** * Columnstore interface for writing a User Defined Aggregate @@ -66,7 +66,6 @@ namespace mcsv1sdk { - // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or // User Defined Analytic Function (UDAnF). // These will be singleton classes, so don't put any instance @@ -76,123 +75,121 @@ namespace mcsv1sdk // Each API function returns a ReturnCode. If ERROR is returned at any time, // the query is aborted, getInterrupted() will begin to return true and the // message set in config->setErrorMessage() is returned to MariaDB. -class allnull : public mcsv1_UDAF +class allnull : public mcsv1_UDAF { -public: - // Defaults OK - allnull() : mcsv1_UDAF() {}; - virtual ~allnull() {}; + public: + // Defaults OK + allnull() : mcsv1_UDAF(){}; + virtual ~allnull(){}; - /** - * init() - * - * Mandatory. Implement this to initialize flags and instance - * data. Called once per SQL statement. You can do any sanity - * checks here. - * - * colTypes (in) - A vector of ColDataType defining the - * parameters of the UDA(n)F call. These can be used to decide - * to override the default return type. If desired, the new - * return type can be set by context->setReturnType() and - * decimal precision can be set in context-> - * setResultDecimalCharacteristics. - * - * Return mcsv1_UDAF::ERROR on any error, such as non-compatible - * colTypes or wrong number of arguments. Else return - * mcsv1_UDAF::SUCCESS. - */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + /** + * init() + * + * Mandatory. Implement this to initialize flags and instance + * data. Called once per SQL statement. You can do any sanity + * checks here. + * + * colTypes (in) - A vector of ColDataType defining the + * parameters of the UDA(n)F call. These can be used to decide + * to override the default return type. If desired, the new + * return type can be set by context->setReturnType() and + * decimal precision can be set in context-> + * setResultDecimalCharacteristics. + * + * Return mcsv1_UDAF::ERROR on any error, such as non-compatible + * colTypes or wrong number of arguments. Else return + * mcsv1_UDAF::SUCCESS. + */ + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - /** - * reset() - * - * Mandatory. Reset the UDA(n)F for a new group, partition or, - * in some cases, new Window Frame. Do not free any memory - * allocated by context->setUserDataSize(). The SDK Framework owns - * that memory and will handle that. Use this opportunity to - * reset any variables in context->getUserData() needed for the - * next aggregation. May be called multiple times if running in - * a ditributed fashion. - * - * Use this opportunity to initialize the userData. - */ - virtual ReturnCode reset(mcsv1Context* context); + /** + * reset() + * + * Mandatory. Reset the UDA(n)F for a new group, partition or, + * in some cases, new Window Frame. Do not free any memory + * allocated by context->setUserDataSize(). The SDK Framework owns + * that memory and will handle that. Use this opportunity to + * reset any variables in context->getUserData() needed for the + * next aggregation. May be called multiple times if running in + * a ditributed fashion. + * + * Use this opportunity to initialize the userData. + */ + virtual ReturnCode reset(mcsv1Context* context); - /** - * nextValue() - * - * Mandatory. Handle a single row. - * - * colsIn - A vector of data structure describing the input - * data. - * - * This function is called once for every row in the filtered - * result set (before aggregation). It is very important that - * this function is efficient. - * - * If the UDAF is running in a distributed fashion, nextValue - * cannot depend on order, as it will only be called for each - * row found on the specific PM. - * - * valsIn (in) - a vector of the parameters from the row. - */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + /** + * nextValue() + * + * Mandatory. Handle a single row. + * + * colsIn - A vector of data structure describing the input + * data. + * + * This function is called once for every row in the filtered + * result set (before aggregation). It is very important that + * this function is efficient. + * + * If the UDAF is running in a distributed fashion, nextValue + * cannot depend on order, as it will only be called for each + * row found on the specific PM. + * + * valsIn (in) - a vector of the parameters from the row. + */ + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - /** - * subEvaluate() - * - * Mandatory -- Called if the UDAF is running in a distributed - * fashion. Columnstore tries to run all aggregate functions - * distributed, depending on context. - * - * Perform an aggregation on rows partially aggregated by - * nextValue. Columnstore calls nextValue for each row on a - * given PM for a group (GROUP BY). subEvaluate is called on the - * UM to consolodate those values into a single instance of - * userData. Keep your aggregated totals in context's userData. - * The first time this is called for a group, reset() would have - * been called with this version of userData. - * - * Called for every partial data set in each group in GROUP BY. - * - * When subEvaluate has been called for all subAggregated data - * sets, Evaluate will be called with the same context as here. - * - * valIn (In) - This is a pointer to a memory block of the size - * set in setUserDataSize. It will contain the value of userData - * as seen in the last call to NextValue for a given PM. - * - */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); + /** + * subEvaluate() + * + * Mandatory -- Called if the UDAF is running in a distributed + * fashion. Columnstore tries to run all aggregate functions + * distributed, depending on context. + * + * Perform an aggregation on rows partially aggregated by + * nextValue. Columnstore calls nextValue for each row on a + * given PM for a group (GROUP BY). subEvaluate is called on the + * UM to consolodate those values into a single instance of + * userData. Keep your aggregated totals in context's userData. + * The first time this is called for a group, reset() would have + * been called with this version of userData. + * + * Called for every partial data set in each group in GROUP BY. + * + * When subEvaluate has been called for all subAggregated data + * sets, Evaluate will be called with the same context as here. + * + * valIn (In) - This is a pointer to a memory block of the size + * set in setUserDataSize. It will contain the value of userData + * as seen in the last call to NextValue for a given PM. + * + */ + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); - /** - * evaluate() - * - * Mandatory. Get the aggregated value. - * - * Called for every new group if UDAF GROUP BY, UDAnF partition - * or, in some cases, new Window Frame. - * - * Set the aggregated value into valOut. The datatype is assumed - * to be the same as that set in the init() function; - * - * If the UDAF is running in a distributed fashion, evaluate is - * called after a series of subEvaluate calls. - * - * valOut (out) - Set the aggregated value here. The datatype is - * assumed to be the same as that set in the init() function; - * - * To return a NULL value, don't assign to valOut. - */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - -protected: + /** + * evaluate() + * + * Mandatory. Get the aggregated value. + * + * Called for every new group if UDAF GROUP BY, UDAnF partition + * or, in some cases, new Window Frame. + * + * Set the aggregated value into valOut. The datatype is assumed + * to be the same as that set in the init() function; + * + * If the UDAF is running in a distributed fashion, evaluate is + * called after a series of subEvaluate calls. + * + * valOut (out) - Set the aggregated value here. The datatype is + * assumed to be the same as that set in the init() function; + * + * To return a NULL value, don't assign to valOut. + */ + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_allnull.h - +#endif // HEADER_allnull.h diff --git a/utils/udfsdk/avg_mode.cpp b/utils/udfsdk/avg_mode.cpp index 5c0ebfbc7..686e4d50c 100644 --- a/utils/udfsdk/avg_mode.cpp +++ b/utils/udfsdk/avg_mode.cpp @@ -24,159 +24,156 @@ using namespace mcsv1sdk; -mcsv1_UDAF::ReturnCode avg_mode::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode avg_mode::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() < 1) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("avg_mode() with 0 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() < 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("avg_mode() with 0 arguments"); + return mcsv1_UDAF::ERROR; + } - if (context->getParameterCount() > 1) - { - context->setErrorMessage("avg_mode() with more than 1 argument"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() > 1) + { + context->setErrorMessage("avg_mode() with more than 1 argument"); + return mcsv1_UDAF::ERROR; + } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("avg_mode() with non-numeric argument"); - return mcsv1_UDAF::ERROR; - } - - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(context->getScale() * 2); - context->setPrecision(19); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("avg_mode() with non-numeric argument"); + return mcsv1_UDAF::ERROR; + } + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(context->getScale() * 2); + context->setPrecision(19); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avg_mode::reset(mcsv1Context* context) { - ModeData* data = static_cast(context->getUserData()); - data->mData.clear(); - return mcsv1_UDAF::SUCCESS; + ModeData* data = static_cast(context->getUserData()); + data->mData.clear(); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avg_mode::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - MODE_DATA& data = static_cast(context->getUserData())->mData; + MODE_DATA& data = static_cast(context->getUserData())->mData; - if (valsIn[0].columnData.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valsIn[0].columnData.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = toDouble(valsIn[0]); + DATATYPE val = toDouble(valsIn[0]); - data[val]++; + data[val]++; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avg_mode::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - MODE_DATA& outData = static_cast(context->getUserData())->mData; - const MODE_DATA& inData = static_cast(userDataIn)->mData; - MODE_DATA::const_iterator iter = inData.begin(); - - for (; iter != inData.end(); ++iter) - { - outData[iter->first] += iter->second; - } - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + MODE_DATA& outData = static_cast(context->getUserData())->mData; + const MODE_DATA& inData = static_cast(userDataIn)->mData; + MODE_DATA::const_iterator iter = inData.begin(); + + for (; iter != inData.end(); ++iter) + { + outData[iter->first] += iter->second; + } + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avg_mode::evaluate(mcsv1Context* context, static_any::any& valOut) { - uint64_t maxCnt = 0; - MODE_DATA& data = static_cast(context->getUserData())->mData; - - if (data.size() == 0) - { - valOut = (DATATYPE)0; - return mcsv1_UDAF::SUCCESS; - } - - MODE_DATA::iterator iter(data.begin()); - - for (; iter != data.end(); ++iter) - { - if (iter->second > maxCnt) - { - valOut = iter->first; - maxCnt = iter->second; - } - } + uint64_t maxCnt = 0; + MODE_DATA& data = static_cast(context->getUserData())->mData; + if (data.size() == 0) + { + valOut = (DATATYPE)0; return mcsv1_UDAF::SUCCESS; + } + + MODE_DATA::iterator iter(data.begin()); + + for (; iter != data.end(); ++iter) + { + if (iter->second > maxCnt) + { + valOut = iter->first; + maxCnt = iter->second; + } + } + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avg_mode::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - MODE_DATA& data = static_cast(context->getUserData())->mData; + MODE_DATA& data = static_cast(context->getUserData())->mData; - if (valsDropped[0].columnData.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valsDropped[0].columnData.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = toDouble(valsDropped[0]); + DATATYPE val = toDouble(valsDropped[0]); - data[val]--; + data[val]--; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avg_mode::createUserData(UserData*& userData, int32_t& length) { - userData = new ModeData; - length = sizeof(ModeData); - return mcsv1_UDAF::SUCCESS; + userData = new ModeData; + length = sizeof(ModeData); + return mcsv1_UDAF::SUCCESS; } void ModeData::serialize(messageqcpp::ByteStream& bs) const { - MODE_DATA::const_iterator iter = mData.begin(); - DATATYPE num; - uint32_t cnt; - bs << (int32_t)mData.size(); + MODE_DATA::const_iterator iter = mData.begin(); + DATATYPE num; + uint32_t cnt; + bs << (int32_t)mData.size(); - for (; iter != mData.end(); ++iter) - { - num = iter->first; - bs << num; - cnt = iter->second; - bs << cnt; - } + for (; iter != mData.end(); ++iter) + { + num = iter->first; + bs << num; + cnt = iter->second; + bs << cnt; + } } void ModeData::unserialize(messageqcpp::ByteStream& bs) { - mData.clear(); - int32_t sz; - DATATYPE num; - uint32_t cnt; - bs >> sz; + mData.clear(); + int32_t sz; + DATATYPE num; + uint32_t cnt; + bs >> sz; - for (int i = 0; i < sz; ++i) - { - bs >> num; - bs >> cnt; - mData[num] = cnt; - } + for (int i = 0; i < sz; ++i) + { + bs >> num; + bs >> cnt; + mData[num] = cnt; + } } - diff --git a/utils/udfsdk/avg_mode.h b/utils/udfsdk/avg_mode.h index da0962c7f..a9e1f8703 100644 --- a/utils/udfsdk/avg_mode.h +++ b/utils/udfsdk/avg_mode.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* avg_mode.h -***********************************************************************/ + * $Id$ + * + * avg_mode.h + ***********************************************************************/ /** * Columnstore interface for writing a User Defined Aggregate @@ -74,24 +74,26 @@ namespace mcsv1sdk { - #define DATATYPE double typedef std::tr1::unordered_map MODE_DATA; // Override UserData for data storage struct ModeData : public UserData { - ModeData() {}; + ModeData(){}; - virtual ~ModeData() {} + virtual ~ModeData() + { + } - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + virtual void serialize(messageqcpp::ByteStream& bs) const; + virtual void unserialize(messageqcpp::ByteStream& bs); - MODE_DATA mData; -private: - // For now, copy construction is unwanted - ModeData(UserData&); + MODE_DATA mData; + + private: + // For now, copy construction is unwanted + ModeData(UserData&); }; // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or @@ -106,175 +108,174 @@ private: // Return the avg_mode value of the dataset -class avg_mode : public mcsv1_UDAF +class avg_mode : public mcsv1_UDAF { -public: - // Defaults OK - avg_mode() : mcsv1_UDAF() {}; - virtual ~avg_mode() {}; + public: + // Defaults OK + avg_mode() : mcsv1_UDAF(){}; + virtual ~avg_mode(){}; - /** - * init() - * - * Mandatory. Implement this to initialize flags and instance - * data. Called once per SQL statement. You can do any sanity - * checks here. - * - * colTypes (in) - A vector of ColDataType defining the - * parameters of the UDA(n)F call. These can be used to decide - * to override the default return type. If desired, the new - * return type can be set by context->setReturnType() and - * decimal scale and precision can be set by context->setScale - * and context->setPrecision respectively. - * - * Return mcsv1_UDAF::ERROR on any error, such as non-compatible - * colTypes or wrong number of arguments. Else return - * mcsv1_UDAF::SUCCESS. - */ - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + /** + * init() + * + * Mandatory. Implement this to initialize flags and instance + * data. Called once per SQL statement. You can do any sanity + * checks here. + * + * colTypes (in) - A vector of ColDataType defining the + * parameters of the UDA(n)F call. These can be used to decide + * to override the default return type. If desired, the new + * return type can be set by context->setReturnType() and + * decimal scale and precision can be set by context->setScale + * and context->setPrecision respectively. + * + * Return mcsv1_UDAF::ERROR on any error, such as non-compatible + * colTypes or wrong number of arguments. Else return + * mcsv1_UDAF::SUCCESS. + */ + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - /** - * reset() - * - * Mandatory. Reset the UDA(n)F for a new group, partition or, - * in some cases, new Window Frame. Do not free any memory - * allocated by context->setUserDataSize(). The SDK Framework owns - * that memory and will handle that. Use this opportunity to - * reset any variables in context->getUserData() needed for the - * next aggregation. May be called multiple times if running in - * a ditributed fashion. - * - * Use this opportunity to initialize the userData. - */ - virtual ReturnCode reset(mcsv1Context* context); + /** + * reset() + * + * Mandatory. Reset the UDA(n)F for a new group, partition or, + * in some cases, new Window Frame. Do not free any memory + * allocated by context->setUserDataSize(). The SDK Framework owns + * that memory and will handle that. Use this opportunity to + * reset any variables in context->getUserData() needed for the + * next aggregation. May be called multiple times if running in + * a ditributed fashion. + * + * Use this opportunity to initialize the userData. + */ + virtual ReturnCode reset(mcsv1Context* context); - /** - * nextValue() - * - * Mandatory. Handle a single row. - * - * colsIn - A vector of data structure describing the input - * data. - * - * This function is called once for every row in the filtered - * result set (before aggregation). It is very important that - * this function is efficient. - * - * If the UDAF is running in a distributed fashion, nextValue - * cannot depend on order, as it will only be called for each - * row found on the specific PM. - * - * valsIn (in) - a vector of the parameters from the row. - */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + /** + * nextValue() + * + * Mandatory. Handle a single row. + * + * colsIn - A vector of data structure describing the input + * data. + * + * This function is called once for every row in the filtered + * result set (before aggregation). It is very important that + * this function is efficient. + * + * If the UDAF is running in a distributed fashion, nextValue + * cannot depend on order, as it will only be called for each + * row found on the specific PM. + * + * valsIn (in) - a vector of the parameters from the row. + */ + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - /** - * subEvaluate() - * - * Mandatory -- Called if the UDAF is running in a distributed - * fashion. Columnstore tries to run all aggregate functions - * distributed, depending on context. - * - * Perform an aggregation on rows partially aggregated by - * nextValue. Columnstore calls nextValue for each row on a - * given PM for a group (GROUP BY). subEvaluate is called on the - * UM to consolodate those values into a single instance of - * userData. Keep your aggregated totals in context's userData. - * The first time this is called for a group, reset() would have - * been called with this version of userData. - * - * Called for every partial data set in each group in GROUP BY. - * - * When subEvaluate has been called for all subAggregated data - * sets, Evaluate will be called with the same context as here. - * - * valIn (In) - This is a pointer to a memory block of the size - * set in setUserDataSize. It will contain the value of userData - * as seen in the last call to NextValue for a given PM. - * - */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + /** + * subEvaluate() + * + * Mandatory -- Called if the UDAF is running in a distributed + * fashion. Columnstore tries to run all aggregate functions + * distributed, depending on context. + * + * Perform an aggregation on rows partially aggregated by + * nextValue. Columnstore calls nextValue for each row on a + * given PM for a group (GROUP BY). subEvaluate is called on the + * UM to consolodate those values into a single instance of + * userData. Keep your aggregated totals in context's userData. + * The first time this is called for a group, reset() would have + * been called with this version of userData. + * + * Called for every partial data set in each group in GROUP BY. + * + * When subEvaluate has been called for all subAggregated data + * sets, Evaluate will be called with the same context as here. + * + * valIn (In) - This is a pointer to a memory block of the size + * set in setUserDataSize. It will contain the value of userData + * as seen in the last call to NextValue for a given PM. + * + */ + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - /** - * evaluate() - * - * Mandatory. Get the aggregated value. - * - * Called for every new group if UDAF GROUP BY, UDAnF partition - * or, in some cases, new Window Frame. - * - * Set the aggregated value into valOut. The datatype is assumed - * to be the same as that set in the init() function; - * - * If the UDAF is running in a distributed fashion, evaluate is - * called after a series of subEvaluate calls. - * - * valOut (out) - Set the aggregated value here. The datatype is - * assumed to be the same as that set in the init() function; - * - * To return a NULL value, don't assign to valOut. - */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + /** + * evaluate() + * + * Mandatory. Get the aggregated value. + * + * Called for every new group if UDAF GROUP BY, UDAnF partition + * or, in some cases, new Window Frame. + * + * Set the aggregated value into valOut. The datatype is assumed + * to be the same as that set in the init() function; + * + * If the UDAF is running in a distributed fashion, evaluate is + * called after a series of subEvaluate calls. + * + * valOut (out) - Set the aggregated value here. The datatype is + * assumed to be the same as that set in the init() function; + * + * To return a NULL value, don't assign to valOut. + */ + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - /** - * dropValue() - * - * Optional -- If defined, the server will call this instead of - * reset for UDAnF. - * - * Don't implement if a UDAnF has one or more of the following: - * The UDAnF can't be used with a Window Frame - * The UDAnF is not reversable in some way - * The UDAnF is not interested in optimal performance - * - * If not implemented, reset() followed by a series of - * nextValue() will be called for each movement of the Window - * Frame. - * - * If implemented, then each movement of the Window Frame will - * result in dropValue() being called for each row falling out - * of the Frame and nextValue() being called for each new row - * coming into the Frame. - * - * valsDropped (in) - a vector of the parameters from the row - * leaving the Frame - * - * dropValue() will not be called for unbounded/current row type - * frames, as those are already optimized. - */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + /** + * dropValue() + * + * Optional -- If defined, the server will call this instead of + * reset for UDAnF. + * + * Don't implement if a UDAnF has one or more of the following: + * The UDAnF can't be used with a Window Frame + * The UDAnF is not reversable in some way + * The UDAnF is not interested in optimal performance + * + * If not implemented, reset() followed by a series of + * nextValue() will be called for each movement of the Window + * Frame. + * + * If implemented, then each movement of the Window Frame will + * result in dropValue() being called for each row falling out + * of the Frame and nextValue() being called for each new row + * coming into the Frame. + * + * valsDropped (in) - a vector of the parameters from the row + * leaving the Frame + * + * dropValue() will not be called for unbounded/current row type + * frames, as those are already optimized. + */ + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); - /** - * createUserData() - * - * Optional -- If defined, the server will call this instead of - * createUserData on context. - * - * Create your variable length data structure via - * data = new - * - * The data structure may contain references to containers or - * pointers to other objects. Remember that for distributed - * processing, this may be called multiple times for variaous - * computing blocks. At the least, it will be called once per PM - * that processes the data, and once more for the UM. For UDAnF, - * it may only be called once. - * - * Set length to the length of the data structure you create. - * - * For each call to createUserData(), there will be a - * corresponding deleteUserData() where you must clean up. Any - * memory leaks are your fault. - * - */ - virtual ReturnCode createUserData(UserData*& data, int32_t& length); -protected: + /** + * createUserData() + * + * Optional -- If defined, the server will call this instead of + * createUserData on context. + * + * Create your variable length data structure via + * data = new + * + * The data structure may contain references to containers or + * pointers to other objects. Remember that for distributed + * processing, this may be called multiple times for variaous + * computing blocks. At the least, it will be called once per PM + * that processes the data, and once more for the UM. For UDAnF, + * it may only be called once. + * + * Set length to the length of the data structure you create. + * + * For each call to createUserData(), there will be a + * corresponding deleteUserData() where you must clean up. Any + * memory leaks are your fault. + * + */ + virtual ReturnCode createUserData(UserData*& data, int32_t& length); + + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_mode.h - +#endif // HEADER_mode.h diff --git a/utils/udfsdk/avgx.cpp b/utils/udfsdk/avgx.cpp index f57b98691..c2b95062a 100644 --- a/utils/udfsdk/avgx.cpp +++ b/utils/udfsdk/avgx.cpp @@ -29,105 +29,101 @@ using namespace mcsv1sdk; // Use the simple data model struct avgx_data { - double sum; - uint64_t cnt; + double sum; + uint64_t cnt; }; - -mcsv1_UDAF::ReturnCode avgx::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode avgx::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() != 1) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("avgx() with other than 1 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() != 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("avgx() with other than 1 arguments"); + return mcsv1_UDAF::ERROR; + } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("avgx() with a non-numeric x argument"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(avgx_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(colTypes[0].scale + 4); - context->setPrecision(19); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("avgx() with a non-numeric x argument"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(avgx_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(colTypes[0].scale + 4); + context->setPrecision(19); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avgx::reset(mcsv1Context* context) { - struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; - data->sum = 0; - data->cnt = 0; - return mcsv1_UDAF::SUCCESS; + struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; + data->sum = 0; + data->cnt = 0; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avgx::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; + struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; - if (valsIn[0].columnData.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valsIn[0].columnData.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = toDouble(valsIn[0]); + DATATYPE val = toDouble(valsIn[0]); - data->sum += val; - ++data->cnt; + data->sum += val; + ++data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avgx::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - struct avgx_data* outData = (struct avgx_data*)context->getUserData()->data; - - struct avgx_data* inData = (struct avgx_data*)userDataIn->data; - - outData->sum += inData->sum; - - outData->cnt += inData->cnt; - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + struct avgx_data* outData = (struct avgx_data*)context->getUserData()->data; + + struct avgx_data* inData = (struct avgx_data*)userDataIn->data; + + outData->sum += inData->sum; + + outData->cnt += inData->cnt; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avgx::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; + struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; - valOut = data->sum / (double)data->cnt; - return mcsv1_UDAF::SUCCESS; + valOut = data->sum / (double)data->cnt; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode avgx::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; + struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; - if (valsDropped[0].columnData.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valsDropped[0].columnData.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = toDouble(valsDropped[0]); + DATATYPE val = toDouble(valsDropped[0]); - data->sum -= val; - --data->cnt; + data->sum -= val; + --data->cnt; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/udfsdk/avgx.h b/utils/udfsdk/avgx.h index 61d3d02b6..2deead586 100644 --- a/utils/udfsdk/avgx.h +++ b/utils/udfsdk/avgx.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* avgx.h -***********************************************************************/ + * $Id$ + * + * avgx.h + ***********************************************************************/ /** * Columnstore interface for for the avgx function @@ -53,7 +53,6 @@ namespace mcsv1sdk { - // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or // User Defined Analytic Function (UDAnF). // These will be singleton classes, so don't put any instance @@ -66,32 +65,30 @@ namespace mcsv1sdk // Return the avgx value of the dataset -class avgx : public mcsv1_UDAF +class avgx : public mcsv1_UDAF { -public: - // Defaults OK - avgx() : mcsv1_UDAF() {}; - virtual ~avgx() {}; + public: + // Defaults OK + avgx() : mcsv1_UDAF(){}; + virtual ~avgx(){}; - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - virtual ReturnCode reset(mcsv1Context* context); + virtual ReturnCode reset(mcsv1Context* context); - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_.h - +#endif // HEADER_.h diff --git a/utils/udfsdk/distinct_count.cpp b/utils/udfsdk/distinct_count.cpp index b0a7c15bc..0cf57c0cf 100644 --- a/utils/udfsdk/distinct_count.cpp +++ b/utils/udfsdk/distinct_count.cpp @@ -22,79 +22,75 @@ using namespace mcsv1sdk; struct distinct_count_data { - long long cnt; + long long cnt; }; #define OUT_TYPE int64_t -mcsv1_UDAF::ReturnCode distinct_count::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode distinct_count::init(mcsv1Context* context, ColumnDatum* colTypes) { - context->setUserDataSize(sizeof(distinct_count_data)); - if (context->getParameterCount() != 1) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("distinct_count() with other than 1 arguments"); - return mcsv1_UDAF::ERROR; - } - context->setResultType(execplan::CalpontSystemCatalog::BIGINT); - context->setColWidth(8); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - context->setRunFlag(mcsv1sdk::UDAF_DISTINCT); - context->setRunFlag(mcsv1sdk::UDAF_OVER_REQUIRED); + context->setUserDataSize(sizeof(distinct_count_data)); + if (context->getParameterCount() != 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("distinct_count() with other than 1 arguments"); + return mcsv1_UDAF::ERROR; + } + context->setResultType(execplan::CalpontSystemCatalog::BIGINT); + context->setColWidth(8); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + context->setRunFlag(mcsv1sdk::UDAF_DISTINCT); + context->setRunFlag(mcsv1sdk::UDAF_OVER_REQUIRED); - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode distinct_count::reset(mcsv1Context* context) { - struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; - data->cnt = 0; - return mcsv1_UDAF::SUCCESS; + struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; + data->cnt = 0; + return mcsv1_UDAF::SUCCESS; } -mcsv1_UDAF::ReturnCode distinct_count::nextValue(mcsv1Context* context, - ColumnDatum* valsIn) +mcsv1_UDAF::ReturnCode distinct_count::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - static_any::any& valIn = valsIn[0].columnData; - struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; + static_any::any& valIn = valsIn[0].columnData; + struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; - if (valIn.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } - data->cnt++; - return mcsv1_UDAF::SUCCESS; + if (valIn.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } + data->cnt++; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode distinct_count::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - struct distinct_count_data* outData = (struct distinct_count_data*)context->getUserData()->data; - struct distinct_count_data* inData = (struct distinct_count_data*)userDataIn->data; - outData->cnt += inData->cnt; - return mcsv1_UDAF::SUCCESS; + struct distinct_count_data* outData = (struct distinct_count_data*)context->getUserData()->data; + struct distinct_count_data* inData = (struct distinct_count_data*)userDataIn->data; + outData->cnt += inData->cnt; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode distinct_count::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; - valOut = data->cnt; - return mcsv1_UDAF::SUCCESS; + struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; + valOut = data->cnt; + return mcsv1_UDAF::SUCCESS; } -mcsv1_UDAF::ReturnCode distinct_count::dropValue(mcsv1Context* context, - ColumnDatum* valsDropped) +mcsv1_UDAF::ReturnCode distinct_count::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - static_any::any& valIn = valsDropped[0].columnData; - struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; + static_any::any& valIn = valsDropped[0].columnData; + struct distinct_count_data* data = (struct distinct_count_data*)context->getUserData()->data; - if (valIn.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valIn.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - data->cnt--; + data->cnt--; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/udfsdk/distinct_count.h b/utils/udfsdk/distinct_count.h index 5e0fa84ac..fd27262cd 100644 --- a/utils/udfsdk/distinct_count.h +++ b/utils/udfsdk/distinct_count.h @@ -16,31 +16,31 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* mcsv1_UDAF.h -***********************************************************************/ + * $Id$ + * + * mcsv1_UDAF.h + ***********************************************************************/ -/** - * Columnstore interface for writing a User Defined Aggregate - * Functions (UDAF) and User Defined Analytic Functions (UDAnF) - * or a function that can act as either - UDA(n)F - * +/** + * Columnstore interface for writing a User Defined Aggregate + * Functions (UDAF) and User Defined Analytic Functions (UDAnF) + * or a function that can act as either - UDA(n)F + * * The basic steps are: * - * 1. Create a the UDA(n)F function interface in some .h file. - * 2. Create the UDF function implementation in some .cpp file - * 3. Create the connector stub (MariaDB UDAF definition) for - * this UDF function. - * 4. build the dynamic library using all of the source. - * 5 Put the library in $COLUMNSTORE_INSTALL/lib of - * all modules - * 6. restart the Columnstore system. + * 1. Create a the UDA(n)F function interface in some .h file. + * 2. Create the UDF function implementation in some .cpp file + * 3. Create the connector stub (MariaDB UDAF definition) for + * this UDF function. + * 4. build the dynamic library using all of the source. + * 5 Put the library in $COLUMNSTORE_INSTALL/lib of + * all modules + * 6. restart the Columnstore system. * 7. notify mysqld about the new functions with commands like: - * + * * CREATE AGGREGATE FUNCTION distinct_count returns INT * soname 'libudf_mysql.so'; - * + * */ #ifndef HEADER_distinct_count #define HEADER_distinct_count @@ -61,161 +61,158 @@ namespace mcsv1sdk { - -// Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or +// Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or // User Defined Analytic Function (UDAnF). // These will be singleton classes, so don't put any instance // specific data in here. All instance data is stored in mcsv1Context // passed to each user function and retrieved by the getUserData() method. -// -// Each API function returns a ReturnCode. If ERROR is returned at any time, -// the query is aborted, getInterrupted() will begin to return true and the -// message set in config->setErrorMessage() is returned to MariaDB. -class distinct_count : public mcsv1_UDAF +// +// Each API function returns a ReturnCode. If ERROR is returned at any time, +// the query is aborted, getInterrupted() will begin to return true and the +// message set in config->setErrorMessage() is returned to MariaDB. +class distinct_count : public mcsv1_UDAF { -public: - // Defaults OK - distinct_count() : mcsv1_UDAF(){}; - virtual ~distinct_count(){}; + public: + // Defaults OK + distinct_count() : mcsv1_UDAF(){}; + virtual ~distinct_count(){}; - /** - * init() - * - * Mandatory. Implement this to initialize flags and instance - * data. Called once per SQL statement. You can do any sanity - * checks here. - * - * colTypes (in) - A vector of ColDataType defining the - * parameters of the UDA(n)F call. These can be used to decide - * to override the default return type. If desired, the new - * return type can be set by context->setReturnType() and - * decimal precision can be set in context-> - * setResultDecimalCharacteristics. - * - * Return mcsv1_UDAF::ERROR on any error, such as non-compatible - * colTypes or wrong number of arguments. Else return - * mcsv1_UDAF::SUCCESS. - */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + /** + * init() + * + * Mandatory. Implement this to initialize flags and instance + * data. Called once per SQL statement. You can do any sanity + * checks here. + * + * colTypes (in) - A vector of ColDataType defining the + * parameters of the UDA(n)F call. These can be used to decide + * to override the default return type. If desired, the new + * return type can be set by context->setReturnType() and + * decimal precision can be set in context-> + * setResultDecimalCharacteristics. + * + * Return mcsv1_UDAF::ERROR on any error, such as non-compatible + * colTypes or wrong number of arguments. Else return + * mcsv1_UDAF::SUCCESS. + */ + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - /** - * reset() - * - * Mandatory. Reset the UDA(n)F for a new group, partition or, - * in some cases, new Window Frame. Do not free any memory - * allocated by context->setUserDataSize(). The SDK Framework owns - * that memory and will handle that. Use this opportunity to - * reset any variables in context->getUserData() needed for the - * next aggregation. May be called multiple times if running in - * a ditributed fashion. - * - * Use this opportunity to initialize the userData. - */ - virtual ReturnCode reset(mcsv1Context* context); + /** + * reset() + * + * Mandatory. Reset the UDA(n)F for a new group, partition or, + * in some cases, new Window Frame. Do not free any memory + * allocated by context->setUserDataSize(). The SDK Framework owns + * that memory and will handle that. Use this opportunity to + * reset any variables in context->getUserData() needed for the + * next aggregation. May be called multiple times if running in + * a ditributed fashion. + * + * Use this opportunity to initialize the userData. + */ + virtual ReturnCode reset(mcsv1Context* context); - /** - * nextValue() - * - * Mandatory. Handle a single row. - * - * colsIn - A vector of data structure describing the input - * data. - * - * This function is called once for every row in the filtered - * result set (before aggregation). It is very important that - * this function is efficient. - * - * If the UDAF is running in a distributed fashion, nextValue - * cannot depend on order, as it will only be called for each - * row found on the specific PM. - * - * valsIn (in) - a vector of the parameters from the row. - */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + /** + * nextValue() + * + * Mandatory. Handle a single row. + * + * colsIn - A vector of data structure describing the input + * data. + * + * This function is called once for every row in the filtered + * result set (before aggregation). It is very important that + * this function is efficient. + * + * If the UDAF is running in a distributed fashion, nextValue + * cannot depend on order, as it will only be called for each + * row found on the specific PM. + * + * valsIn (in) - a vector of the parameters from the row. + */ + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - /** - * subEvaluate() - * - * Mandatory -- Called if the UDAF is running in a distributed - * fashion. Columnstore tries to run all aggregate functions - * distributed, depending on context. - * - * Perform an aggregation on rows partially aggregated by - * nextValue. Columnstore calls nextValue for each row on a - * given PM for a group (GROUP BY). subEvaluate is called on the - * UM to consolodate those values into a single instance of - * userData. Keep your aggregated totals in context's userData. - * The first time this is called for a group, reset() would have - * been called with this version of userData. - * - * Called for every partial data set in each group in GROUP BY. - * - * When subEvaluate has been called for all subAggregated data - * sets, Evaluate will be called with the same context as here. - * - * valIn (In) - This is a pointer to a memory block of the size - * set in setUserDataSize. It will contain the value of userData - * as seen in the last call to NextValue for a given PM. - * - */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); + /** + * subEvaluate() + * + * Mandatory -- Called if the UDAF is running in a distributed + * fashion. Columnstore tries to run all aggregate functions + * distributed, depending on context. + * + * Perform an aggregation on rows partially aggregated by + * nextValue. Columnstore calls nextValue for each row on a + * given PM for a group (GROUP BY). subEvaluate is called on the + * UM to consolodate those values into a single instance of + * userData. Keep your aggregated totals in context's userData. + * The first time this is called for a group, reset() would have + * been called with this version of userData. + * + * Called for every partial data set in each group in GROUP BY. + * + * When subEvaluate has been called for all subAggregated data + * sets, Evaluate will be called with the same context as here. + * + * valIn (In) - This is a pointer to a memory block of the size + * set in setUserDataSize. It will contain the value of userData + * as seen in the last call to NextValue for a given PM. + * + */ + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); - /** - * evaluate() - * - * Mandatory. Get the aggregated value. - * - * Called for every new group if UDAF GROUP BY, UDAnF partition - * or, in some cases, new Window Frame. - * - * Set the aggregated value into valOut. The datatype is assumed - * to be the same as that set in the init() function; - * - * If the UDAF is running in a distributed fashion, evaluate is - * called after a series of subEvaluate calls. - * - * valOut (out) - Set the aggregated value here. The datatype is - * assumed to be the same as that set in the init() function; - * - * To return a NULL value, don't assign to valOut. - */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + /** + * evaluate() + * + * Mandatory. Get the aggregated value. + * + * Called for every new group if UDAF GROUP BY, UDAnF partition + * or, in some cases, new Window Frame. + * + * Set the aggregated value into valOut. The datatype is assumed + * to be the same as that set in the init() function; + * + * If the UDAF is running in a distributed fashion, evaluate is + * called after a series of subEvaluate calls. + * + * valOut (out) - Set the aggregated value here. The datatype is + * assumed to be the same as that set in the init() function; + * + * To return a NULL value, don't assign to valOut. + */ + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - /** - * dropValue() - * - * Optional -- If defined, the server will call this instead of - * reset for UDAnF. - * - * Don't implement if a UDAnF has one or more of the following: - * The UDAnF can't be used with a Window Frame - * The UDAnF is not reversable in some way - * The UDAnF is not interested in optimal performance - * - * If not implemented, reset() followed by a series of - * nextValue() will be called for each movement of the Window - * Frame. - * - * If implemented, then each movement of the Window Frame will - * result in dropValue() being called for each row falling out - * of the Frame and nextValue() being called for each new row - * coming into the Frame. - * - * valsDropped (in) - a vector of the parameters from the row - * leaving the Frame - * - * dropValue() will not be called for unbounded/current row type - * frames, as those are already optimized. - */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); - -protected: + /** + * dropValue() + * + * Optional -- If defined, the server will call this instead of + * reset for UDAnF. + * + * Don't implement if a UDAnF has one or more of the following: + * The UDAnF can't be used with a Window Frame + * The UDAnF is not reversable in some way + * The UDAnF is not interested in optimal performance + * + * If not implemented, reset() followed by a series of + * nextValue() will be called for each movement of the Window + * Frame. + * + * If implemented, then each movement of the Window Frame will + * result in dropValue() being called for each row falling out + * of the Frame and nextValue() being called for each new row + * coming into the Frame. + * + * valsDropped (in) - a vector of the parameters from the row + * leaving the Frame + * + * dropValue() will not be called for unbounded/current row type + * frames, as those are already optimized. + */ + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_distinct_count.h - +#endif // HEADER_distinct_count.h diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp old mode 100755 new mode 100644 index b9d7cdac1..4b705865c --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -40,236 +40,218 @@ using namespace mcsv1sdk; UDAF_MAP& UDAFMap::fm() { - static UDAF_MAP* m = new UDAF_MAP; - return *m; + static UDAF_MAP* m = new UDAF_MAP; + return *m; } UDAF_MAP& UDAFMap::getMap() { - UDAF_MAP& fm = UDAFMap::fm(); - if (fm.size() > 0) - { - return fm; - } - - // first: function name - // second: Function pointer - // please use lower case for the function name. Because the names might be - // case-insensitive in MySQL depending on the setting. In such case, - // the function names passed to the interface is always in lower case. - fm["allnull"] = new allnull(); - fm["ssq"] = new ssq(); -// fm["median"] = new median(); - fm["avg_mode"] = new avg_mode(); - fm["avgx"] = new avgx(); - + UDAF_MAP& fm = UDAFMap::fm(); + if (fm.size() > 0) + { return fm; + } + + // first: function name + // second: Function pointer + // please use lower case for the function name. Because the names might be + // case-insensitive in MySQL depending on the setting. In such case, + // the function names passed to the interface is always in lower case. + fm["allnull"] = new allnull(); + fm["ssq"] = new ssq(); + // fm["median"] = new median(); + fm["avg_mode"] = new avg_mode(); + fm["avgx"] = new avgx(); + + return fm; } int32_t mcsv1Context::getColWidth() { - if (fColWidth > 0) - { - return fColWidth; - } - - // JIT initialization for types that have a defined size. - switch (fResultType) - { - case execplan::CalpontSystemCatalog::BIT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::CHAR: - fColWidth = 1; - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::USMALLINT: - fColWidth = 2; - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DATE: - fColWidth = 4; - break; - - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::STRINT: - fColWidth = 8; - break; - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - fColWidth = sizeof(long double); - break; - - default: - break; - } - + if (fColWidth > 0) + { return fColWidth; + } + + // JIT initialization for types that have a defined size. + switch (fResultType) + { + case execplan::CalpontSystemCatalog::BIT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::CHAR: fColWidth = 1; break; + + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::USMALLINT: fColWidth = 2; break; + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DATE: fColWidth = 4; break; + + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::STRINT: fColWidth = 8; break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: fColWidth = sizeof(long double); break; + + default: break; + } + + return fColWidth; } bool mcsv1Context::operator==(const mcsv1Context& c) const { - // We don't test the per row data fields. They don't determine - // if it's the same Context. - if (getName() != c.getName() - || fRunFlags != c.fRunFlags - || fContextFlags != c.fContextFlags - || fUserDataSize != c.fUserDataSize - || fResultType != c.fResultType - || fResultscale != c.fResultscale - || fResultPrecision != c.fResultPrecision - || fStartFrame != c.fStartFrame - || fEndFrame != c.fEndFrame - || fStartConstant != c.fStartConstant - || fEndConstant != c.fEndConstant - || fParamCount != c.fParamCount) - return false; + // We don't test the per row data fields. They don't determine + // if it's the same Context. + if (getName() != c.getName() || fRunFlags != c.fRunFlags || fContextFlags != c.fContextFlags || + fUserDataSize != c.fUserDataSize || fResultType != c.fResultType || fResultscale != c.fResultscale || + fResultPrecision != c.fResultPrecision || fStartFrame != c.fStartFrame || fEndFrame != c.fEndFrame || + fStartConstant != c.fStartConstant || fEndConstant != c.fEndConstant || fParamCount != c.fParamCount) + return false; - return true; + return true; } bool mcsv1Context::operator!=(const mcsv1Context& c) const { - return (!(*this == c)); + return (!(*this == c)); } const std::string mcsv1Context::toString() const { - std::ostringstream output; - output << "mcsv1Context: " << getName() << std::endl; - output << " RunFlags=" << fRunFlags << " ContextFlags=" << fContextFlags << std::endl; - output << " UserDataSize=" << fUserDataSize << " ResultType=" << execplan::colDataTypeToString(fResultType) << std::endl; - output << " Resultscale=" << fResultscale << " ResultPrecision=" << fResultPrecision << std::endl; - output << " ErrorMsg=" << errorMsg << std::endl; - output << " bInterrupted=" << bInterrupted << std::endl; - output << " StartFrame=" << fStartFrame << " EndFrame=" << fEndFrame << std::endl; - output << " StartConstant=" << fStartConstant << " EndConstant=" << fEndConstant << std::endl; - return output.str(); + std::ostringstream output; + output << "mcsv1Context: " << getName() << std::endl; + output << " RunFlags=" << fRunFlags << " ContextFlags=" << fContextFlags << std::endl; + output << " UserDataSize=" << fUserDataSize << " ResultType=" << execplan::colDataTypeToString(fResultType) + << std::endl; + output << " Resultscale=" << fResultscale << " ResultPrecision=" << fResultPrecision << std::endl; + output << " ErrorMsg=" << errorMsg << std::endl; + output << " bInterrupted=" << bInterrupted << std::endl; + output << " StartFrame=" << fStartFrame << " EndFrame=" << fEndFrame << std::endl; + output << " StartConstant=" << fStartConstant << " EndConstant=" << fEndConstant << std::endl; + return output.str(); } mcsv1sdk::mcsv1_UDAF* mcsv1Context::getFunction() { - if (func) - { - return func; - } - - // Just in time initialization - if (functionName.length() == 0) - { - std::ostringstream errmsg; - errmsg << "mcsv1Context::getFunction: " << functionName << " is empty"; - throw std::logic_error(errmsg.str()); - } - - mcsv1sdk::UDAF_MAP::iterator funcIter = mcsv1sdk::UDAFMap::getMap().find(functionName); - - if (funcIter == mcsv1sdk::UDAFMap::getMap().end()) - { - std::ostringstream errmsg; - errmsg << "mcsv1Context::getFunction: " << functionName << " is undefined"; - throw std::logic_error(errmsg.str()); - } - - func = funcIter->second; + if (func) + { return func; + } + + // Just in time initialization + if (functionName.length() == 0) + { + std::ostringstream errmsg; + errmsg << "mcsv1Context::getFunction: " << functionName << " is empty"; + throw std::logic_error(errmsg.str()); + } + + mcsv1sdk::UDAF_MAP::iterator funcIter = mcsv1sdk::UDAFMap::getMap().find(functionName); + + if (funcIter == mcsv1sdk::UDAFMap::getMap().end()) + { + std::ostringstream errmsg; + errmsg << "mcsv1Context::getFunction: " << functionName << " is undefined"; + throw std::logic_error(errmsg.str()); + } + + func = funcIter->second; + return func; } mcsv1sdk::mcsv1_UDAF* mcsv1Context::getFunction() const { - return const_cast(this)->getFunction(); + return const_cast(this)->getFunction(); } void mcsv1Context::createUserData() { - // Try the function. If not implemented, create a byte array. - UserData* userData = NULL; - mcsv1_UDAF::ReturnCode rc = getFunction()->createUserData(userData, fUserDataSize); + // Try the function. If not implemented, create a byte array. + UserData* userData = NULL; + mcsv1_UDAF::ReturnCode rc = getFunction()->createUserData(userData, fUserDataSize); - if (rc == mcsv1_UDAF::ERROR) - { - std::ostringstream errmsg; - errmsg << "mcsv1Context::createUserData: " << functionName << errorMsg.c_str(); - throw std::logic_error(errmsg.str()); - } + if (rc == mcsv1_UDAF::ERROR) + { + std::ostringstream errmsg; + errmsg << "mcsv1Context::createUserData: " << functionName << errorMsg.c_str(); + throw std::logic_error(errmsg.str()); + } - setUserData(userData); + setUserData(userData); } void mcsv1Context::serialize(messageqcpp::ByteStream& b) const { - b.needAtLeast(sizeof(mcsv1Context)); - b << (execplan::ObjectReader::id_t) execplan::ObjectReader::MCSV1_CONTEXT; - b << functionName; - b << fRunFlags; - // Dont send context flags, These are set for each call - b << fUserDataSize; - b << (uint32_t)fResultType; - b << fColWidth; - b << fResultscale; - b << fResultPrecision; - b << errorMsg; - // Don't send dataflags. These are set for each call - // bInterrupted is set internally. - b << (uint32_t)fStartFrame; - b << (uint32_t)fEndFrame; - b << fStartConstant; - b << fEndConstant; - b << fParamCount; - b << (uint32_t)mariadbReturnType; + b.needAtLeast(sizeof(mcsv1Context)); + b << (execplan::ObjectReader::id_t)execplan::ObjectReader::MCSV1_CONTEXT; + b << functionName; + b << fRunFlags; + // Dont send context flags, These are set for each call + b << fUserDataSize; + b << (uint32_t)fResultType; + b << fColWidth; + b << fResultscale; + b << fResultPrecision; + b << errorMsg; + // Don't send dataflags. These are set for each call + // bInterrupted is set internally. + b << (uint32_t)fStartFrame; + b << (uint32_t)fEndFrame; + b << fStartConstant; + b << fEndConstant; + b << fParamCount; + b << (uint32_t)mariadbReturnType; } void mcsv1Context::unserialize(messageqcpp::ByteStream& b) { - execplan::ObjectReader::checkType(b, execplan::ObjectReader::MCSV1_CONTEXT); - b >> functionName; - b >> fRunFlags; - b >> fUserDataSize; - uint32_t iResultType; - b >> iResultType; - fResultType = (execplan::CalpontSystemCatalog::ColDataType)iResultType; - b >> fColWidth; - b >> fResultscale; - b >> fResultPrecision; - b >> errorMsg; - uint32_t frame; - b >> frame; - fStartFrame = (execplan::WF_FRAME)frame; - b >> frame; - fEndFrame = (execplan::WF_FRAME)frame; - b >> fStartConstant; - b >> fEndConstant; - b >> fParamCount; - uint32_t mrt; - b >> mrt; - mariadbReturnType = (enum_mariadb_return_type)mrt; + execplan::ObjectReader::checkType(b, execplan::ObjectReader::MCSV1_CONTEXT); + b >> functionName; + b >> fRunFlags; + b >> fUserDataSize; + uint32_t iResultType; + b >> iResultType; + fResultType = (execplan::CalpontSystemCatalog::ColDataType)iResultType; + b >> fColWidth; + b >> fResultscale; + b >> fResultPrecision; + b >> errorMsg; + uint32_t frame; + b >> frame; + fStartFrame = (execplan::WF_FRAME)frame; + b >> frame; + fEndFrame = (execplan::WF_FRAME)frame; + b >> fStartConstant; + b >> fEndConstant; + b >> fParamCount; + uint32_t mrt; + b >> mrt; + mariadbReturnType = (enum_mariadb_return_type)mrt; } void UserData::serialize(messageqcpp::ByteStream& bs) const { - bs << size; - bs.append(data, size); + bs << size; + bs.append(data, size); } void UserData::unserialize(messageqcpp::ByteStream& bs) { - bs >> size; - memcpy(data, bs.buf(), size); - bs.advance(size); + bs >> size; + memcpy(data, bs.buf(), size); + bs.advance(size); } const std::string typeStr(""); @@ -288,5 +270,3 @@ const static_any::any& mcsv1_UDAF::ullTypeId((unsigned long long)1); const static_any::any& mcsv1_UDAF::floatTypeId((float)1); const static_any::any& mcsv1_UDAF::doubleTypeId((double)1); const static_any::any& mcsv1_UDAF::strTypeId(typeStr); - - diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h old mode 100755 new mode 100644 index caf7393d4..2e67560cf --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* mcsv1_UDAF.h -***********************************************************************/ + * $Id$ + * + * mcsv1_UDAF.h + ***********************************************************************/ /** * Columnstore interface for writing a User Defined Aggregate @@ -87,11 +87,12 @@ namespace mcsv1sdk { // The return type of the CREATE AGGREGATE statement -enum enum_mariadb_return_type { - MYSQL_TYPE_DOUBLE = 5, - MYSQL_TYPE_LONGLONG = 8, - MYSQL_TYPE_VARCHAR=15, - MYSQL_TYPE_NEWDECIMAL = 246 +enum enum_mariadb_return_type +{ + MYSQL_TYPE_DOUBLE = 5, + MYSQL_TYPE_LONGLONG = 8, + MYSQL_TYPE_VARCHAR = 15, + MYSQL_TYPE_NEWDECIMAL = 246 }; /** @@ -107,14 +108,15 @@ typedef std::tr1::unordered_map UDAF_MAP; class UDAFMap { -public: - EXPORT UDAFMap() {}; + public: + EXPORT UDAFMap(){}; - EXPORT ~UDAFMap() {}; + EXPORT ~UDAFMap(){}; - static EXPORT UDAF_MAP& getMap(); -private: - static UDAF_MAP& fm(); + static EXPORT UDAF_MAP& getMap(); + + private: + static UDAF_MAP& fm(); }; /** @@ -136,79 +138,87 @@ class mcsv1Context; struct UserData { - UserData() : size(0), data(NULL) {}; - UserData(size_t sz) - { - size = sz; - data = new uint8_t[sz]; - } + UserData() : size(0), data(NULL){}; + UserData(size_t sz) + { + size = sz; + data = new uint8_t[sz]; + } - virtual ~UserData() - { - if (data) delete [] data; - } + virtual ~UserData() + { + if (data) + delete[] data; + } - /** - * serialize() - * - * User data is passed between processes. In order to do so, it - * must be serialized. Since user data can have sub objects, - * containers and the like, it is up to the UDAF to provide the - * serialize function. The streaming functionality of - * messageqcpp::ByteStream must be used. - * - * The default streams the size and data buffer to the - * ByteStream - */ - virtual void serialize(messageqcpp::ByteStream& bs) const; + /** + * serialize() + * + * User data is passed between processes. In order to do so, it + * must be serialized. Since user data can have sub objects, + * containers and the like, it is up to the UDAF to provide the + * serialize function. The streaming functionality of + * messageqcpp::ByteStream must be used. + * + * The default streams the size and data buffer to the + * ByteStream + */ + virtual void serialize(messageqcpp::ByteStream& bs) const; - /** - * unserialize() - * - * User data is passed between processes. In order to do so, it - * must be unserialized. Since user data can have sub objects, - * containers and the like, it is up to the UDAF to provide the - * unserialize function. The streaming functionality of - * messageqcpp::ByteStream must be used. - * - * data is the datablock returned by createUserData. - * - * The default creates the data array and streams into data. - */ - virtual void unserialize(messageqcpp::ByteStream& bs); + /** + * unserialize() + * + * User data is passed between processes. In order to do so, it + * must be unserialized. Since user data can have sub objects, + * containers and the like, it is up to the UDAF to provide the + * unserialize function. The streaming functionality of + * messageqcpp::ByteStream must be used. + * + * data is the datablock returned by createUserData. + * + * The default creates the data array and streams into data. + */ + virtual void unserialize(messageqcpp::ByteStream& bs); - // The default data store. You may or may not wish to use these fields. - uint32_t size; - uint8_t* data; -private: - // For now, copy construction is unwanted - UserData(UserData&); + // The default data store. You may or may not wish to use these fields. + uint32_t size; + uint8_t* data; + + private: + // For now, copy construction is unwanted + UserData(UserData&); }; // Flags to define the type and limitations of a UDA(n)F // Used in context->fRunFlags -static uint64_t UDAF_OVER_REQUIRED __attribute__ ((unused)) = 1; // May only be used as UDAnF -static uint64_t UDAF_OVER_ALLOWED __attribute__ ((unused)) = 1 << 1; // May be used as UDAF or UDAnF -static uint64_t UDAF_ORDER_REQUIRED __attribute__ ((unused)) = 1 << 2; // If used as UDAnF, ORDER BY is required -static uint64_t UDAF_ORDER_ALLOWED __attribute__ ((unused)) = 1 << 3; // If used as UDAnF, ORDER BY is optional -static uint64_t UDAF_WINDOWFRAME_REQUIRED __attribute__ ((unused)) = 1 << 4; // If used as UDAnF, a WINDOW FRAME is required -static uint64_t UDAF_WINDOWFRAME_ALLOWED __attribute__ ((unused)) = 1 << 5; // If used as UDAnF, a WINDOW FRAME is optional -static uint64_t UDAF_MAYBE_NULL __attribute__ ((unused)) = 1 << 6; // If UDA(n)F might return NULL. -static uint64_t UDAF_IGNORE_NULLS __attribute__ ((unused)) = 1 << 7; // If UDA(n)F wants NULL rows suppressed. -static uint64_t UDAF_DISTINCT __attribute__ ((unused)) = 1 << 8; // Force UDA(n)F to be distinct on first param. +static uint64_t UDAF_OVER_REQUIRED __attribute__((unused)) = 1; // May only be used as UDAnF +static uint64_t UDAF_OVER_ALLOWED __attribute__((unused)) = 1 << 1; // May be used as UDAF or UDAnF +static uint64_t UDAF_ORDER_REQUIRED __attribute__((unused)) = 1 + << 2; // If used as UDAnF, ORDER BY is required +static uint64_t UDAF_ORDER_ALLOWED __attribute__((unused)) = 1 + << 3; // If used as UDAnF, ORDER BY is optional +static uint64_t UDAF_WINDOWFRAME_REQUIRED __attribute__((unused)) = + 1 << 4; // If used as UDAnF, a WINDOW FRAME is required +static uint64_t UDAF_WINDOWFRAME_ALLOWED __attribute__((unused)) = + 1 << 5; // If used as UDAnF, a WINDOW FRAME is optional +static uint64_t UDAF_MAYBE_NULL __attribute__((unused)) = 1 << 6; // If UDA(n)F might return NULL. +static uint64_t UDAF_IGNORE_NULLS __attribute__((unused)) = 1 << 7; // If UDA(n)F wants NULL rows suppressed. +static uint64_t UDAF_DISTINCT __attribute__((unused)) = 1 + << 8; // Force UDA(n)F to be distinct on first param. // Flags set by the framework to define the context of the call. // User code shouldn't use these directly // used in context->fContextFlags -static uint64_t CONTEXT_IS_ANALYTIC __attribute__ ((unused)) = 1; // If called using OVER -static uint64_t CONTEXT_HAS_CURRENT_ROW __attribute__ ((unused)) = 1 << 1; // The current window contains the current row. -static uint64_t CONTEXT_IS_PM __attribute__ ((unused)) = 1 << 2; // The call was made by the PM +static uint64_t CONTEXT_IS_ANALYTIC __attribute__((unused)) = 1; // If called using OVER +static uint64_t CONTEXT_HAS_CURRENT_ROW __attribute__((unused)) = + 1 << 1; // The current window contains the current row. +static uint64_t CONTEXT_IS_PM __attribute__((unused)) = 1 << 2; // The call was made by the PM // Flags that describe the contents of a specific input parameter // These will be set in context->dataFlags for each method call by the framework. // User code shouldn't use these directly -static uint32_t PARAM_IS_NULL __attribute__ ((unused)) = 1; -static uint32_t PARAM_IS_CONSTANT __attribute__ ((unused)) = 1 << 1; +static uint32_t PARAM_IS_NULL __attribute__((unused)) = 1; +static uint32_t PARAM_IS_CONSTANT __attribute__((unused)) = 1 << 1; // This is the context class that is passed to all API callbacks // The framework potentially sets data here for each invocation of @@ -222,207 +232,205 @@ static uint32_t PARAM_IS_CONSTANT __attribute__ ((unused)) = 1 << 1; // the context. UDA(n)F may need to be re-compiled in this case. class mcsv1Context { -public: - EXPORT mcsv1Context(); - EXPORT mcsv1Context(const mcsv1Context& rhs); - // The destructor is virtual only in case a version 2 is made derived from v1 - // to promote backward compatibility. - // mcsv1Context should never be subclassed by UDA(n)F developers - EXPORT virtual ~mcsv1Context(); + public: + EXPORT mcsv1Context(); + EXPORT mcsv1Context(const mcsv1Context& rhs); + // The destructor is virtual only in case a version 2 is made derived from v1 + // to promote backward compatibility. + // mcsv1Context should never be subclassed by UDA(n)F developers + EXPORT virtual ~mcsv1Context(); - // Set an error message if something goes wrong - EXPORT void setErrorMessage(std::string errmsg); + // Set an error message if something goes wrong + EXPORT void setErrorMessage(std::string errmsg); - // Get the previously set error message - EXPORT const std::string& getErrorMessage() const; + // Get the previously set error message + EXPORT const std::string& getErrorMessage() const; - // Set the flags as a set. Return the previous flags. - EXPORT uint64_t setRunFlags(uint64_t flags); - // return the flags - EXPORT uint64_t getRunFlags() const; + // Set the flags as a set. Return the previous flags. + EXPORT uint64_t setRunFlags(uint64_t flags); + // return the flags + EXPORT uint64_t getRunFlags() const; - // The following set, get, clear and toggle methods can be used to manipulate - // multiple flags by ORing them together in the call sequence. - // Ex setRunFlag(UDAF_OVER_REQUIRED | UDAF_ORDER_REQUIRED); - // sets both flags and returns true if BOTH flags are already set. - // - // Set a specific flag and return its previous setting - EXPORT bool setRunFlag(uint64_t flag); - // Get a specific flag - EXPORT bool getRunFlag(uint64_t flag); - // clear a specific flag and return its previous setting - EXPORT bool clearRunFlag(uint64_t flag); - // toggle a specific flag and return its previous setting - EXPORT bool toggleRunFlag(uint64_t flag); + // The following set, get, clear and toggle methods can be used to manipulate + // multiple flags by ORing them together in the call sequence. + // Ex setRunFlag(UDAF_OVER_REQUIRED | UDAF_ORDER_REQUIRED); + // sets both flags and returns true if BOTH flags are already set. + // + // Set a specific flag and return its previous setting + EXPORT bool setRunFlag(uint64_t flag); + // Get a specific flag + EXPORT bool getRunFlag(uint64_t flag); + // clear a specific flag and return its previous setting + EXPORT bool clearRunFlag(uint64_t flag); + // toggle a specific flag and return its previous setting + EXPORT bool toggleRunFlag(uint64_t flag); - // Use these to determine the way your UDA(n)F was called - // Valid in all method calls - EXPORT bool isAnalytic(); - EXPORT bool isWindowHasCurrentRow(); + // Use these to determine the way your UDA(n)F was called + // Valid in all method calls + EXPORT bool isAnalytic(); + EXPORT bool isWindowHasCurrentRow(); - // Determine if the call is made by the UM - // This could be because the UDA(n)F is not being distributed - // Or it could be during setup or during consolodation of PM values. - // valid in all calls - EXPORT bool isUM(); + // Determine if the call is made by the UM + // This could be because the UDA(n)F is not being distributed + // Or it could be during setup or during consolodation of PM values. + // valid in all calls + EXPORT bool isUM(); - // Determine if the call is made by the PM - // This will be during partial aggregation performed on the PM - // valid in all calls - EXPORT bool isPM(); + // Determine if the call is made by the PM + // This will be during partial aggregation performed on the PM + // valid in all calls + EXPORT bool isPM(); - // Parameter refinement description accessors + // Parameter refinement description accessors - // How many actual parameters were entered. - // valid in all calls - size_t getParameterCount() const; + // How many actual parameters were entered. + // valid in all calls + size_t getParameterCount() const; - // Determine if an input parameter is NULL - // valid in nextValue and dropValue - EXPORT bool isParamNull(int paramIdx); + // Determine if an input parameter is NULL + // valid in nextValue and dropValue + EXPORT bool isParamNull(int paramIdx); - // If a parameter is a constant, the UDA(n)F could presumably optimize its workings - // during the first call to nextValue(). - // Is there a better way to determine this? - // valid in nextValue - EXPORT bool isParamConstant(int paramIdx); + // If a parameter is a constant, the UDA(n)F could presumably optimize its workings + // during the first call to nextValue(). + // Is there a better way to determine this? + // valid in nextValue + EXPORT bool isParamConstant(int paramIdx); - // For getting the result type. - EXPORT execplan::CalpontSystemCatalog::ColDataType getResultType() const; + // For getting the result type. + EXPORT execplan::CalpontSystemCatalog::ColDataType getResultType() const; - // For getting the decimal characteristics for the return type. - // These will be set to the default before init(). - EXPORT int32_t getScale() const; - EXPORT int32_t getPrecision() const; + // For getting the decimal characteristics for the return type. + // These will be set to the default before init(). + EXPORT int32_t getScale() const; + EXPORT int32_t getPrecision() const; - // If you want to change the result type - // valid in init() - EXPORT bool setResultType(execplan::CalpontSystemCatalog::ColDataType resultType); + // If you want to change the result type + // valid in init() + EXPORT bool setResultType(execplan::CalpontSystemCatalog::ColDataType resultType); - // For setting the decimal characteristics for the return value. - // This only makes sense if the return type is decimal, but should be set - // to (0, -1) for other types if the inout is decimal. - // valid in init() - // Set the scale to DECIMAL_NOT_SPECIFIED if you want a floating decimal. - EXPORT bool setScale(int32_t scale); - EXPORT bool setPrecision(int32_t precision); + // For setting the decimal characteristics for the return value. + // This only makes sense if the return type is decimal, but should be set + // to (0, -1) for other types if the inout is decimal. + // valid in init() + // Set the scale to DECIMAL_NOT_SPECIFIED if you want a floating decimal. + EXPORT bool setScale(int32_t scale); + EXPORT bool setPrecision(int32_t precision); - // For all types, get the return column width in bytes. Ex. INT will return 4. - EXPORT int32_t getColWidth(); + // For all types, get the return column width in bytes. Ex. INT will return 4. + EXPORT int32_t getColWidth(); - // For non-numric return types, set the return column width. This defaults - // to a length determined by the input datatype. - // valid in init() - EXPORT bool setColWidth(int32_t colWidth); + // For non-numric return types, set the return column width. This defaults + // to a length determined by the input datatype. + // valid in init() + EXPORT bool setColWidth(int32_t colWidth); - // If a method is known to take a while, call this periodically to see if something - // interupted the processing. If getInterrupted() returns true, then the executing - // method should clean up and exit. - EXPORT bool getInterrupted() const; + // If a method is known to take a while, call this periodically to see if something + // interupted the processing. If getInterrupted() returns true, then the executing + // method should clean up and exit. + EXPORT bool getInterrupted() const; - // Allocate instance specific memory. This should be type cast to a structure overlay - // defined by the function. The actual allocatoin occurs in the various modules that - // do the aggregation. If the UDAF is being calculated in a distributed fashion, then - // multiple instances of this data may be allocated. Calls to the subaggregate functions - // do not share a context. - // You do not need to worry about freeing this memory. The framework handles all management. - // Call this during init() - EXPORT void setUserDataSize(int bytes); + // Allocate instance specific memory. This should be type cast to a structure overlay + // defined by the function. The actual allocatoin occurs in the various modules that + // do the aggregation. If the UDAF is being calculated in a distributed fashion, then + // multiple instances of this data may be allocated. Calls to the subaggregate functions + // do not share a context. + // You do not need to worry about freeing this memory. The framework handles all management. + // Call this during init() + EXPORT void setUserDataSize(int bytes); - // Call this everywhere except init() - EXPORT UserData* getUserData(); + // Call this everywhere except init() + EXPORT UserData* getUserData(); - // Many UDAnF need a default Window Frame. If none is set here, the default is - // UNBOUNDED PRECEDING to CURRENT ROW. - // It's possible to not allow the the WINDOW FRAME phrase in the UDAnF by setting - // the UDAF_WINDOWFRAME_REQUIRED and UDAF_WINDOWFRAME_ALLOWED both to false. Columnstore - // requires a Window Frame in order to process UDAnF. In this case, the default will - // be used for all calls. - // Possible values for start frame are - // WF_UNBOUNDED_PRECEDING, WF_CURRENT_ROW, WF_PRECEDING or WF_FOLLOWING - // possible values for end frame are - // WF_CURRENT_ROW, WF_UNBOUNDED_FOLLOWING, WF_PRECEDING or WF_FOLLOWING - // If WF_PRECEEdING and/or WF_FOLLOWING, a start or end constant should - // be included to say how many preceeding or following is the default - // Set this during init() - EXPORT bool setDefaultWindowFrame(execplan::WF_FRAME defaultStartFrame, - execplan::WF_FRAME defaultEndFrame, - int32_t startConstant = 0, // For WF_PRECEEDING or WF_FOLLOWING - int32_t endConstant = 0); // For WF_PRECEEDING or WF_FOLLOWING + // Many UDAnF need a default Window Frame. If none is set here, the default is + // UNBOUNDED PRECEDING to CURRENT ROW. + // It's possible to not allow the the WINDOW FRAME phrase in the UDAnF by setting + // the UDAF_WINDOWFRAME_REQUIRED and UDAF_WINDOWFRAME_ALLOWED both to false. Columnstore + // requires a Window Frame in order to process UDAnF. In this case, the default will + // be used for all calls. + // Possible values for start frame are + // WF_UNBOUNDED_PRECEDING, WF_CURRENT_ROW, WF_PRECEDING or WF_FOLLOWING + // possible values for end frame are + // WF_CURRENT_ROW, WF_UNBOUNDED_FOLLOWING, WF_PRECEDING or WF_FOLLOWING + // If WF_PRECEEdING and/or WF_FOLLOWING, a start or end constant should + // be included to say how many preceeding or following is the default + // Set this during init() + EXPORT bool setDefaultWindowFrame(execplan::WF_FRAME defaultStartFrame, execplan::WF_FRAME defaultEndFrame, + int32_t startConstant = 0, // For WF_PRECEEDING or WF_FOLLOWING + int32_t endConstant = 0); // For WF_PRECEEDING or WF_FOLLOWING - // There may be times you want to know the actual frame set by the caller - EXPORT void getStartFrame(execplan::WF_FRAME& startFrame, int32_t& startConstant) const; - EXPORT void getEndFrame(execplan::WF_FRAME& endFrame, int32_t& endConstant) const; + // There may be times you want to know the actual frame set by the caller + EXPORT void getStartFrame(execplan::WF_FRAME& startFrame, int32_t& startConstant) const; + EXPORT void getEndFrame(execplan::WF_FRAME& endFrame, int32_t& endConstant) const; - // Deep Equivalence - bool operator==(const mcsv1Context& c) const; - bool operator!=(const mcsv1Context& c) const; + // Deep Equivalence + bool operator==(const mcsv1Context& c) const; + bool operator!=(const mcsv1Context& c) const; - // stream operator for debugging - EXPORT const std::string toString() const; + // stream operator for debugging + EXPORT const std::string toString() const; - // Get the name of the function - EXPORT const std::string& getName() const; + // Get the name of the function + EXPORT const std::string& getName() const; - // Get the return type as set by CREATE AGGREGATE FUNCTION - EXPORT enum_mariadb_return_type getMariaDBReturnType() const; + // Get the return type as set by CREATE AGGREGATE FUNCTION + EXPORT enum_mariadb_return_type getMariaDBReturnType() const; - EXPORT mcsv1Context& operator=(const mcsv1Context& rhs); - EXPORT mcsv1Context& copy(const mcsv1Context& rhs); + EXPORT mcsv1Context& operator=(const mcsv1Context& rhs); + EXPORT mcsv1Context& copy(const mcsv1Context& rhs); - // Character collation support - EXPORT void setCharsetNumber(uint32_t csNum); - EXPORT uint32_t getCharsetNumber(); // Returns the unique ID for the language/collation - EXPORT CHARSET_INFO* getCharset(); + // Character collation support + EXPORT void setCharsetNumber(uint32_t csNum); + EXPORT uint32_t getCharsetNumber(); // Returns the unique ID for the language/collation + EXPORT CHARSET_INFO* getCharset(); -private: - - uint64_t fRunFlags; // Set by the user to define the type of UDA(n)F - uint64_t fContextFlags; // Set by the framework to define this specific call. - int32_t fUserDataSize; - boost::shared_ptr fUserData; - execplan::CalpontSystemCatalog::ColDataType fResultType; - int32_t fColWidth; // The length in bytes of the return type - int32_t fResultscale; // For scale, the number of digits to the right of the decimal - int32_t fResultPrecision; // The max number of digits allowed in the decimal value - std::string errorMsg; - uint32_t* dataFlags; // an integer array wirh one entry for each parameter - bool* bInterrupted; // Gets set to true by the Framework if something happens - execplan::WF_FRAME fStartFrame; // Is set to default to start, then modified by the actual frame in the call - execplan::WF_FRAME fEndFrame; // Is set to default to start, then modified by the actual frame in the call - int32_t fStartConstant; // for start frame WF_PRECEEDIMG or WF_FOLLOWING - int32_t fEndConstant; // for end frame WF_PRECEEDIMG or WF_FOLLOWING - std::string functionName; - mcsv1sdk::mcsv1_UDAF* func; - int32_t fParamCount; - std::vector paramKeys; - enum_mariadb_return_type mariadbReturnType; - uint32_t fCharsetNumber; - -public: - // For use by the framework - EXPORT void serialize(messageqcpp::ByteStream& b) const; - EXPORT void unserialize(messageqcpp::ByteStream& b); - EXPORT void createUserData(); - EXPORT void setUserData(boost::shared_ptr userData); - EXPORT void setUserData(UserData* userData); - EXPORT void setName(std::string name); - EXPORT void setContextFlags(uint64_t flags); - EXPORT void setContextFlag(uint64_t flag); - EXPORT void clearContextFlag(uint64_t flag); - EXPORT uint64_t getContextFlags() const; - EXPORT uint32_t getUserDataSize() const; - EXPORT uint32_t* getDataFlags(); - EXPORT void setDataFlags(uint32_t* flags); - EXPORT void setInterrupted(bool interrupted); - EXPORT void setInterrupted(bool* interrupted); - EXPORT mcsv1sdk::mcsv1_UDAF* getFunction(); - EXPORT mcsv1sdk::mcsv1_UDAF* getFunction() const; - EXPORT boost::shared_ptr getUserDataSP(); - EXPORT void setParamCount(int32_t paramCount); - std::vector* getParamKeys(); - EXPORT void setMariaDBReturnType(enum_mariadb_return_type rt); + private: + uint64_t fRunFlags; // Set by the user to define the type of UDA(n)F + uint64_t fContextFlags; // Set by the framework to define this specific call. + int32_t fUserDataSize; + boost::shared_ptr fUserData; + execplan::CalpontSystemCatalog::ColDataType fResultType; + int32_t fColWidth; // The length in bytes of the return type + int32_t fResultscale; // For scale, the number of digits to the right of the decimal + int32_t fResultPrecision; // The max number of digits allowed in the decimal value + std::string errorMsg; + uint32_t* dataFlags; // an integer array wirh one entry for each parameter + bool* bInterrupted; // Gets set to true by the Framework if something happens + execplan::WF_FRAME + fStartFrame; // Is set to default to start, then modified by the actual frame in the call + execplan::WF_FRAME fEndFrame; // Is set to default to start, then modified by the actual frame in the call + int32_t fStartConstant; // for start frame WF_PRECEEDIMG or WF_FOLLOWING + int32_t fEndConstant; // for end frame WF_PRECEEDIMG or WF_FOLLOWING + std::string functionName; + mcsv1sdk::mcsv1_UDAF* func; + int32_t fParamCount; + std::vector paramKeys; + enum_mariadb_return_type mariadbReturnType; + uint32_t fCharsetNumber; + public: + // For use by the framework + EXPORT void serialize(messageqcpp::ByteStream& b) const; + EXPORT void unserialize(messageqcpp::ByteStream& b); + EXPORT void createUserData(); + EXPORT void setUserData(boost::shared_ptr userData); + EXPORT void setUserData(UserData* userData); + EXPORT void setName(std::string name); + EXPORT void setContextFlags(uint64_t flags); + EXPORT void setContextFlag(uint64_t flag); + EXPORT void clearContextFlag(uint64_t flag); + EXPORT uint64_t getContextFlags() const; + EXPORT uint32_t getUserDataSize() const; + EXPORT uint32_t* getDataFlags(); + EXPORT void setDataFlags(uint32_t* flags); + EXPORT void setInterrupted(bool interrupted); + EXPORT void setInterrupted(bool* interrupted); + EXPORT mcsv1sdk::mcsv1_UDAF* getFunction(); + EXPORT mcsv1sdk::mcsv1_UDAF* getFunction() const; + EXPORT boost::shared_ptr getUserDataSP(); + EXPORT void setParamCount(int32_t paramCount); + std::vector* getParamKeys(); + EXPORT void setMariaDBReturnType(enum_mariadb_return_type rt); }; // Since aggregate functions can operate on any data type, we use the following structure @@ -440,14 +448,14 @@ public: // For char, varchar, text, varbinary and blob types, columnData will be std::string. struct ColumnDatum { - execplan::CalpontSystemCatalog::ColDataType dataType; // defined in calpontsystemcatalog.h - static_any::any columnData; // Not valid in init() - uint32_t scale; // If dataType is a DECIMAL type - uint32_t precision; // If dataType is a DECIMAL type - std::string alias; // Only filled in for init() - uint32_t charsetNumber; // For string collations - ColumnDatum() : dataType(execplan::CalpontSystemCatalog::UNDEFINED), - scale(0), precision(-1), charsetNumber(8) {}; + execplan::CalpontSystemCatalog::ColDataType dataType; // defined in calpontsystemcatalog.h + static_any::any columnData; // Not valid in init() + uint32_t scale; // If dataType is a DECIMAL type + uint32_t precision; // If dataType is a DECIMAL type + std::string alias; // Only filled in for init() + uint32_t charsetNumber; // For string collations + ColumnDatum() + : dataType(execplan::CalpontSystemCatalog::UNDEFINED), scale(0), precision(-1), charsetNumber(8){}; }; // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or @@ -461,250 +469,247 @@ struct ColumnDatum // message set in config->setErrorMessage() is returned to MariaDB. class mcsv1_UDAF { -public: - enum ReturnCode - { - ERROR = 0, - SUCCESS = 1, - NOT_IMPLEMENTED = 2 // User UDA(n)F shouldn't return this - }; - // Defaults OK - mcsv1_UDAF() {}; - virtual ~mcsv1_UDAF() {}; + public: + enum ReturnCode + { + ERROR = 0, + SUCCESS = 1, + NOT_IMPLEMENTED = 2 // User UDA(n)F shouldn't return this + }; + // Defaults OK + mcsv1_UDAF(){}; + virtual ~mcsv1_UDAF(){}; - /** - * init() - * - * Mandatory. Implement this to initialize flags and instance - * data. Called once per SQL statement. You can do any sanity - * checks here. - * - * colTypes (in) - A vector of ColDataType defining the - * parameters of the UDA(n)F call. These can be used to decide - * to override the default return type. If desired, the new - * return type can be set by context->setReturnType() and - * decimal scale and precision can be set by context->setScale - * and context->setPrecision respectively. - * - * Return mcsv1_UDAF::ERROR on any error, such as non-compatible - * colTypes or wrong number of arguments. Else return - * mcsv1_UDAF::SUCCESS. - */ - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes) = 0; + /** + * init() + * + * Mandatory. Implement this to initialize flags and instance + * data. Called once per SQL statement. You can do any sanity + * checks here. + * + * colTypes (in) - A vector of ColDataType defining the + * parameters of the UDA(n)F call. These can be used to decide + * to override the default return type. If desired, the new + * return type can be set by context->setReturnType() and + * decimal scale and precision can be set by context->setScale + * and context->setPrecision respectively. + * + * Return mcsv1_UDAF::ERROR on any error, such as non-compatible + * colTypes or wrong number of arguments. Else return + * mcsv1_UDAF::SUCCESS. + */ + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) = 0; - /** - * reset() - * - * Mandatory. Reset the UDA(n)F for a new group, partition or, - * in some cases, new Window Frame. Do not free any memory - * allocated by createUserData(). The SDK Framework owns - * that memory and will handle that. Use this opportunity to - * reset any variables in context->getUserData() needed for the - * next aggregation. May be called multiple times if running in - * a ditributed fashion. - * - * Use this opportunity to initialize the userData. - */ - virtual ReturnCode reset(mcsv1Context* context) = 0; + /** + * reset() + * + * Mandatory. Reset the UDA(n)F for a new group, partition or, + * in some cases, new Window Frame. Do not free any memory + * allocated by createUserData(). The SDK Framework owns + * that memory and will handle that. Use this opportunity to + * reset any variables in context->getUserData() needed for the + * next aggregation. May be called multiple times if running in + * a ditributed fashion. + * + * Use this opportunity to initialize the userData. + */ + virtual ReturnCode reset(mcsv1Context* context) = 0; - /** - * nextValue() - * - * Mandatory. Handle a single row. - * - * colsIn - A vector of data structure describing the input - * data. - * - * This function is called once for every row in the filtered - * result set (before aggregation). It is very important that - * this function is efficient. - * - * If the UDAF is running in a distributed fashion, nextValue - * cannot depend on order, as it will only be called for each - * row found on the specific PM. - * - * valsIn (in) - a vector of the parameters from the row. - */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) = 0; + /** + * nextValue() + * + * Mandatory. Handle a single row. + * + * colsIn - A vector of data structure describing the input + * data. + * + * This function is called once for every row in the filtered + * result set (before aggregation). It is very important that + * this function is efficient. + * + * If the UDAF is running in a distributed fashion, nextValue + * cannot depend on order, as it will only be called for each + * row found on the specific PM. + * + * valsIn (in) - a vector of the parameters from the row. + */ + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) = 0; - /** - * subEvaluate() - * - * Mandatory -- Called if the UDAF is running in a distributed - * fashion. Columnstore tries to run all aggregate functions - * distributed, depending on context. - * - * Perform an aggregation on rows partially aggregated by - * nextValue. Columnstore calls nextValue for each row on a - * given PM for a group (GROUP BY). subEvaluate is called on the - * UM to consolodate those values into a single instance of - * userData. Keep your aggregated totals in context's userData. - * The first time this is called for a group, reset() would have - * been called with this version of userData. - * - * Called for every partial data set in each group in GROUP BY. - * - * When subEvaluate has been called for all subAggregated data - * sets, Evaluate will be called with the same context as here. - * - * valIn (In) - This is a pointer to a UserData class with the - * partially aggregated values. It will contain the value of - * userData as seen in the last call to NextValue for a given - * PM. - * - */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn) = 0; + /** + * subEvaluate() + * + * Mandatory -- Called if the UDAF is running in a distributed + * fashion. Columnstore tries to run all aggregate functions + * distributed, depending on context. + * + * Perform an aggregation on rows partially aggregated by + * nextValue. Columnstore calls nextValue for each row on a + * given PM for a group (GROUP BY). subEvaluate is called on the + * UM to consolodate those values into a single instance of + * userData. Keep your aggregated totals in context's userData. + * The first time this is called for a group, reset() would have + * been called with this version of userData. + * + * Called for every partial data set in each group in GROUP BY. + * + * When subEvaluate has been called for all subAggregated data + * sets, Evaluate will be called with the same context as here. + * + * valIn (In) - This is a pointer to a UserData class with the + * partially aggregated values. It will contain the value of + * userData as seen in the last call to NextValue for a given + * PM. + * + */ + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn) = 0; - /** - * evaluate() - * - * Mandatory. Get the aggregated value. - * - * Called for every new group if UDAF GROUP BY, UDAnF partition - * or, in some cases, new Window Frame. - * - * Set the aggregated value into valOut. The datatype is assumed - * to be the same as that set in the init() function; - * - * If the UDAF is running in a distributed fashion, evaluate is - * called after a series of subEvaluate calls. - * - * valOut (out) - Set the aggregated value here. The datatype is - * assumed to be the same as that set in the init() function; - * - * To return a NULL value, don't assign to valOut. - */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) = 0; + /** + * evaluate() + * + * Mandatory. Get the aggregated value. + * + * Called for every new group if UDAF GROUP BY, UDAnF partition + * or, in some cases, new Window Frame. + * + * Set the aggregated value into valOut. The datatype is assumed + * to be the same as that set in the init() function; + * + * If the UDAF is running in a distributed fashion, evaluate is + * called after a series of subEvaluate calls. + * + * valOut (out) - Set the aggregated value here. The datatype is + * assumed to be the same as that set in the init() function; + * + * To return a NULL value, don't assign to valOut. + */ + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) = 0; - /** - * dropValue() - * - * Optional -- If defined, the server will call this instead of - * reset for UDAnF. - * - * Don't implement if a UDAnF has one or more of the following: - * The UDAnF can't be used with a Window Frame - * The UDAnF is not reversable in some way - * The UDAnF is not interested in optimal performance - * - * If not implemented, reset() followed by a series of - * nextValue() will be called for each movement of the Window - * Frame. - * - * If implemented, then each movement of the Window Frame will - * result in dropValue() being called for each row falling out - * of the Frame and nextValue() being called for each new row - * coming into the Frame. - * - * valsDropped (in) - a vector of the parameters from the row - * leaving the Frame - * - * dropValue() will not be called for unbounded/current row type - * frames, as those are already optimized. - */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + /** + * dropValue() + * + * Optional -- If defined, the server will call this instead of + * reset for UDAnF. + * + * Don't implement if a UDAnF has one or more of the following: + * The UDAnF can't be used with a Window Frame + * The UDAnF is not reversable in some way + * The UDAnF is not interested in optimal performance + * + * If not implemented, reset() followed by a series of + * nextValue() will be called for each movement of the Window + * Frame. + * + * If implemented, then each movement of the Window Frame will + * result in dropValue() being called for each row falling out + * of the Frame and nextValue() being called for each new row + * coming into the Frame. + * + * valsDropped (in) - a vector of the parameters from the row + * leaving the Frame + * + * dropValue() will not be called for unbounded/current row type + * frames, as those are already optimized. + */ + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); - /** - * createUserData() - * - * Optional -- The default is to create a data byte array of - * size as set in context->setUserDataSize() - * - * Create your variable length data structure via - * userData = new - * - * The data structure may contain references to containers or - * pointers to other objects. Remember that for distributed - * processing, this may be called multiple times for variaous - * computing blocks. At the least, it will be called once per PM - * that processes the data, and once more for the UM. For UDAnF, - * it may only be called once. - * - * Set length to the base length of the data structure you - * create. - * - */ - virtual ReturnCode createUserData(UserData*& userdata, int32_t& length); + /** + * createUserData() + * + * Optional -- The default is to create a data byte array of + * size as set in context->setUserDataSize() + * + * Create your variable length data structure via + * userData = new + * + * The data structure may contain references to containers or + * pointers to other objects. Remember that for distributed + * processing, this may be called multiple times for variaous + * computing blocks. At the least, it will be called once per PM + * that processes the data, and once more for the UM. For UDAnF, + * it may only be called once. + * + * Set length to the base length of the data structure you + * create. + * + */ + virtual ReturnCode createUserData(UserData*& userdata, int32_t& length); -protected: + protected: + double toDouble(ColumnDatum& datum) const + { + double val = convertAnyTo(datum.columnData); + if (val != 0 && datum.scale > 0) + val /= datatypes::scaleDivisor(datum.scale); + return val; + } - double toDouble(ColumnDatum &datum) const - { - double val = convertAnyTo(datum.columnData); - if (val != 0 && datum.scale > 0) - val /= datatypes::scaleDivisor(datum.scale); - return val; - } - - // some handy conversion routines - template - T convertAnyTo(static_any::any&) const; - // These are handy for testing the actual type of static_any - static const static_any::any& charTypeId; - static const static_any::any& scharTypeId; - static const static_any::any& shortTypeId; - static const static_any::any& intTypeId; - static const static_any::any& longTypeId; - static const static_any::any& llTypeId; - static const static_any::any& int128TypeId; - static const static_any::any& ucharTypeId; - static const static_any::any& ushortTypeId; - static const static_any::any& uintTypeId; - static const static_any::any& ulongTypeId; - static const static_any::any& ullTypeId; - static const static_any::any& floatTypeId; - static const static_any::any& doubleTypeId; - static const static_any::any& strTypeId; + // some handy conversion routines + template + T convertAnyTo(static_any::any&) const; + // These are handy for testing the actual type of static_any + static const static_any::any& charTypeId; + static const static_any::any& scharTypeId; + static const static_any::any& shortTypeId; + static const static_any::any& intTypeId; + static const static_any::any& longTypeId; + static const static_any::any& llTypeId; + static const static_any::any& int128TypeId; + static const static_any::any& ucharTypeId; + static const static_any::any& ushortTypeId; + static const static_any::any& uintTypeId; + static const static_any::any& ulongTypeId; + static const static_any::any& ullTypeId; + static const static_any::any& floatTypeId; + static const static_any::any& doubleTypeId; + static const static_any::any& strTypeId; }; /*********************************************************************** * There is no user modifiable code past this point ***********************************************************************/ // Function definitions for mcsv1Context -inline mcsv1Context::mcsv1Context() : - fRunFlags(UDAF_OVER_ALLOWED | UDAF_ORDER_ALLOWED | UDAF_WINDOWFRAME_ALLOWED), - fContextFlags(0), - fUserDataSize(0), - fResultType(execplan::CalpontSystemCatalog::UNDEFINED), - fColWidth(0), - fResultscale(0), - fResultPrecision(18), - dataFlags(NULL), - bInterrupted(NULL), - fStartFrame(execplan::WF_UNBOUNDED_PRECEDING), - fEndFrame(execplan::WF_CURRENT_ROW), - fStartConstant(0), - fEndConstant(0), - func(NULL), - fParamCount(0), - fCharsetNumber(8) // Latin1 +inline mcsv1Context::mcsv1Context() + : fRunFlags(UDAF_OVER_ALLOWED | UDAF_ORDER_ALLOWED | UDAF_WINDOWFRAME_ALLOWED) + , fContextFlags(0) + , fUserDataSize(0) + , fResultType(execplan::CalpontSystemCatalog::UNDEFINED) + , fColWidth(0) + , fResultscale(0) + , fResultPrecision(18) + , dataFlags(NULL) + , bInterrupted(NULL) + , fStartFrame(execplan::WF_UNBOUNDED_PRECEDING) + , fEndFrame(execplan::WF_CURRENT_ROW) + , fStartConstant(0) + , fEndConstant(0) + , func(NULL) + , fParamCount(0) + , fCharsetNumber(8) // Latin1 { } -inline mcsv1Context::mcsv1Context(const mcsv1Context& rhs) : - dataFlags(NULL) +inline mcsv1Context::mcsv1Context(const mcsv1Context& rhs) : dataFlags(NULL) { - copy(rhs); + copy(rhs); } inline mcsv1Context& mcsv1Context::copy(const mcsv1Context& rhs) { - fRunFlags = rhs.fRunFlags; - fContextFlags = rhs.fContextFlags; - fResultType = rhs.fResultType; - fUserDataSize = rhs.fUserDataSize; - fColWidth = rhs.fColWidth; - fResultscale = rhs.fResultscale; - fResultPrecision = rhs.fResultPrecision; - rhs.getStartFrame(fStartFrame, fStartConstant); - rhs.getEndFrame(fEndFrame, fEndConstant); - functionName = rhs.functionName; - bInterrupted = rhs.bInterrupted; // Multiple threads will use the same reference - func = rhs.func; - fParamCount = rhs.fParamCount; - fCharsetNumber = rhs.fCharsetNumber; - return *this; + fRunFlags = rhs.fRunFlags; + fContextFlags = rhs.fContextFlags; + fResultType = rhs.fResultType; + fUserDataSize = rhs.fUserDataSize; + fColWidth = rhs.fColWidth; + fResultscale = rhs.fResultscale; + fResultPrecision = rhs.fResultPrecision; + rhs.getStartFrame(fStartFrame, fStartConstant); + rhs.getEndFrame(fEndFrame, fEndConstant); + functionName = rhs.functionName; + bInterrupted = rhs.bInterrupted; // Multiple threads will use the same reference + func = rhs.func; + fParamCount = rhs.fParamCount; + fCharsetNumber = rhs.fCharsetNumber; + return *this; } inline mcsv1Context::~mcsv1Context() @@ -713,381 +718,379 @@ inline mcsv1Context::~mcsv1Context() inline mcsv1Context& mcsv1Context::operator=(const mcsv1Context& rhs) { - dataFlags = NULL; - return copy(rhs); + dataFlags = NULL; + return copy(rhs); } inline void mcsv1Context::setErrorMessage(std::string errmsg) { - errorMsg = errmsg; + errorMsg = errmsg; } inline const std::string& mcsv1Context::getErrorMessage() const { - return errorMsg; + return errorMsg; } inline uint64_t mcsv1Context::setRunFlags(uint64_t flags) { - uint64_t f = fRunFlags; - fRunFlags = flags; - return f; + uint64_t f = fRunFlags; + fRunFlags = flags; + return f; } inline uint64_t mcsv1Context::getRunFlags() const { - return fRunFlags; + return fRunFlags; } inline bool mcsv1Context::setRunFlag(uint64_t flag) { - bool b = fRunFlags & flag; - fRunFlags |= flag; - return b; + bool b = fRunFlags & flag; + fRunFlags |= flag; + return b; } inline bool mcsv1Context::getRunFlag(uint64_t flag) { - return fRunFlags & flag; + return fRunFlags & flag; } inline bool mcsv1Context::clearRunFlag(uint64_t flag) { - bool b = fRunFlags & flag; - fRunFlags &= ~flag; - return b; + bool b = fRunFlags & flag; + fRunFlags &= ~flag; + return b; } inline bool mcsv1Context::toggleRunFlag(uint64_t flag) { - bool b = fRunFlags & flag; - fRunFlags ^= flag; - return b; + bool b = fRunFlags & flag; + fRunFlags ^= flag; + return b; } inline bool mcsv1Context::isAnalytic() { - return fContextFlags & CONTEXT_IS_ANALYTIC; + return fContextFlags & CONTEXT_IS_ANALYTIC; } inline bool mcsv1Context::isWindowHasCurrentRow() { - return fContextFlags & CONTEXT_HAS_CURRENT_ROW; + return fContextFlags & CONTEXT_HAS_CURRENT_ROW; } inline bool mcsv1Context::isUM() { - return !(fContextFlags & CONTEXT_IS_PM); + return !(fContextFlags & CONTEXT_IS_PM); } inline bool mcsv1Context::isPM() { - return fContextFlags & CONTEXT_IS_PM; + return fContextFlags & CONTEXT_IS_PM; } inline size_t mcsv1Context::getParameterCount() const { - return fParamCount; + return fParamCount; } inline bool mcsv1Context::isParamNull(int paramIdx) { - if (dataFlags) - return dataFlags[paramIdx] & PARAM_IS_NULL; + if (dataFlags) + return dataFlags[paramIdx] & PARAM_IS_NULL; - return false; + return false; } inline bool mcsv1Context::isParamConstant(int paramIdx) { - if (dataFlags) - return dataFlags[paramIdx] & PARAM_IS_CONSTANT; + if (dataFlags) + return dataFlags[paramIdx] & PARAM_IS_CONSTANT; - return false; + return false; } inline execplan::CalpontSystemCatalog::ColDataType mcsv1Context::getResultType() const { - return fResultType; + return fResultType; } inline bool mcsv1Context::setResultType(execplan::CalpontSystemCatalog::ColDataType resultType) { - fResultType = resultType; - return true; // We may want to sanity check here. + fResultType = resultType; + return true; // We may want to sanity check here. } inline int32_t mcsv1Context::getScale() const { - return fResultscale; + return fResultscale; } inline int32_t mcsv1Context::getPrecision() const { - return fResultPrecision; + return fResultPrecision; } inline bool mcsv1Context::setScale(int32_t scale) { - fResultscale = scale; - return true; + fResultscale = scale; + return true; } inline bool mcsv1Context::setPrecision(int32_t precision) { - fResultPrecision = precision; - return true; + fResultPrecision = precision; + return true; } inline bool mcsv1Context::setColWidth(int32_t colWidth) { - fColWidth = colWidth; - return true; + fColWidth = colWidth; + return true; } inline void mcsv1Context::setInterrupted(bool interrupted) { - if (bInterrupted) - { - *bInterrupted = interrupted; - } + if (bInterrupted) + { + *bInterrupted = interrupted; + } } inline void mcsv1Context::setInterrupted(bool* interrupted) { - bInterrupted = interrupted; + bInterrupted = interrupted; } inline bool mcsv1Context::getInterrupted() const { - return bInterrupted && *bInterrupted; + return bInterrupted && *bInterrupted; } inline void mcsv1Context::setUserDataSize(int bytes) { - fUserDataSize = bytes; + fUserDataSize = bytes; } inline UserData* mcsv1Context::getUserData() { - if (!fUserData) - { - createUserData(); - } + if (!fUserData) + { + createUserData(); + } - return fUserData.get(); + return fUserData.get(); } inline boost::shared_ptr mcsv1Context::getUserDataSP() { - if (!fUserData) - { - createUserData(); - } + if (!fUserData) + { + createUserData(); + } - return fUserData; + return fUserData; } inline void mcsv1Context::setUserData(boost::shared_ptr userData) { - fUserData = userData; + fUserData = userData; } inline void mcsv1Context::setUserData(UserData* userData) { - if (userData) - { - fUserData.reset(userData); - } - else - { - fUserData.reset(); - } + if (userData) + { + fUserData.reset(userData); + } + else + { + fUserData.reset(); + } } inline bool mcsv1Context::setDefaultWindowFrame(execplan::WF_FRAME defaultStartFrame, - execplan::WF_FRAME defaultEndFrame, - int32_t startConstant, - int32_t endConstant) + execplan::WF_FRAME defaultEndFrame, int32_t startConstant, + int32_t endConstant) { - // TODO: Add sanity checks - fStartFrame = defaultStartFrame; - fEndFrame = defaultEndFrame; - fStartConstant = startConstant; - fEndConstant = endConstant; - return true; + // TODO: Add sanity checks + fStartFrame = defaultStartFrame; + fEndFrame = defaultEndFrame; + fStartConstant = startConstant; + fEndConstant = endConstant; + return true; } inline void mcsv1Context::getStartFrame(execplan::WF_FRAME& startFrame, int32_t& startConstant) const { - startFrame = fStartFrame; - startConstant = fStartConstant; + startFrame = fStartFrame; + startConstant = fStartConstant; } inline void mcsv1Context::getEndFrame(execplan::WF_FRAME& endFrame, int32_t& endConstant) const { - endFrame = fEndFrame; - endConstant = fEndConstant; + endFrame = fEndFrame; + endConstant = fEndConstant; } inline const std::string& mcsv1Context::getName() const { - return functionName; + return functionName; } inline void mcsv1Context::setName(std::string name) { - functionName = name; + functionName = name; } inline uint64_t mcsv1Context::getContextFlags() const { - return fContextFlags; + return fContextFlags; } inline void mcsv1Context::setContextFlags(uint64_t flags) { - fContextFlags = flags; + fContextFlags = flags; } inline void mcsv1Context::setContextFlag(uint64_t flag) { - fContextFlags |= flag; + fContextFlags |= flag; } inline void mcsv1Context::clearContextFlag(uint64_t flag) { - fContextFlags &= ~flag; + fContextFlags &= ~flag; } inline uint32_t mcsv1Context::getUserDataSize() const { - return fUserDataSize; + return fUserDataSize; } inline uint32_t* mcsv1Context::getDataFlags() { - return dataFlags; + return dataFlags; } inline void mcsv1Context::setDataFlags(uint32_t* flags) { - dataFlags = flags; + dataFlags = flags; } inline void mcsv1Context::setParamCount(int32_t paramCount) { - fParamCount = paramCount; + fParamCount = paramCount; } inline std::vector* mcsv1Context::getParamKeys() { - return ¶mKeys; + return ¶mKeys; } inline enum_mariadb_return_type mcsv1Context::getMariaDBReturnType() const { - return mariadbReturnType; + return mariadbReturnType; } inline void mcsv1Context::setMariaDBReturnType(enum_mariadb_return_type rt) { - mariadbReturnType = rt; + mariadbReturnType = rt; } inline void mcsv1Context::setCharsetNumber(uint32_t csNum) { - fCharsetNumber=csNum; + fCharsetNumber = csNum; } inline uint32_t mcsv1Context::getCharsetNumber() { - return fCharsetNumber; + return fCharsetNumber; } inline mcsv1_UDAF::ReturnCode mcsv1_UDAF::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - return NOT_IMPLEMENTED; + return NOT_IMPLEMENTED; } inline mcsv1_UDAF::ReturnCode mcsv1_UDAF::createUserData(UserData*& userData, int32_t& length) { - userData = new UserData(length); - userData->size = length; - return SUCCESS; + userData = new UserData(length); + userData->size = length; + return SUCCESS; } // Handy helper functions -template +template inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn) const { - T val = 0; - if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(int128TypeId)) - { - val = valIn.cast(); - } - else - { - throw std::runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type"); - } - return val; + T val = 0; + if (valIn.compatible(longTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(charTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(scharTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(shortTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(intTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(llTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(ucharTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(ushortTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(uintTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(ulongTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(ullTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(floatTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(doubleTypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(int128TypeId)) + { + val = valIn.cast(); + } + else + { + throw std::runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type"); + } + return val; } -}; // namespace mcssdk +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_mcsv1_udaf.h - +#endif // HEADER_mcsv1_udaf.h diff --git a/utils/udfsdk/median.cpp b/utils/udfsdk/median.cpp index 3ef7a0187..11eb8e678 100644 --- a/utils/udfsdk/median.cpp +++ b/utils/udfsdk/median.cpp @@ -24,202 +24,199 @@ using namespace mcsv1sdk; -mcsv1_UDAF::ReturnCode median::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode median::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() < 1) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("median() with 0 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() < 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("median() with 0 arguments"); + return mcsv1_UDAF::ERROR; + } - if (context->getParameterCount() > 1) - { - context->setErrorMessage("median() with more than 1 argument"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() > 1) + { + context->setErrorMessage("median() with more than 1 argument"); + return mcsv1_UDAF::ERROR; + } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("median() with non-numeric argument"); - return mcsv1_UDAF::ERROR; - } - - context->setResultType(CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(context->getScale() * 2); - context->setPrecision(19); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("median() with non-numeric argument"); + return mcsv1_UDAF::ERROR; + } + context->setResultType(CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(context->getScale() * 2); + context->setPrecision(19); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode median::reset(mcsv1Context* context) { - MedianData* data = static_cast(context->getUserData()); - data->mData.clear(); - return mcsv1_UDAF::SUCCESS; + MedianData* data = static_cast(context->getUserData()); + data->mData.clear(); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode median::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - static_any::any& valIn = valsIn[0].columnData; - MEDIAN_DATA& data = static_cast(context->getUserData())->mData; + static_any::any& valIn = valsIn[0].columnData; + MEDIAN_DATA& data = static_cast(context->getUserData())->mData; - if (valIn.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valIn.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = convertAnyTo(valIn); + DATATYPE val = convertAnyTo(valIn); - // For decimal types, we need to move the decimal point. - uint32_t scale = valsIn[0].scale; + // For decimal types, we need to move the decimal point. + uint32_t scale = valsIn[0].scale; - if (val != 0 && scale > 0) - { - val /= pow(10.0, (double)scale); - } + if (val != 0 && scale > 0) + { + val /= pow(10.0, (double)scale); + } - data[val]++; + data[val]++; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode median::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - if (!userDataIn) - { - return mcsv1_UDAF::SUCCESS; - } - - MEDIAN_DATA& outData = static_cast(context->getUserData())->mData; - const MEDIAN_DATA& inData = static_cast(userDataIn)->mData; - MEDIAN_DATA::const_iterator iter = inData.begin(); - - for (; iter != inData.end(); ++iter) - { - outData[iter->first] += iter->second; - } - + if (!userDataIn) + { return mcsv1_UDAF::SUCCESS; + } + + MEDIAN_DATA& outData = static_cast(context->getUserData())->mData; + const MEDIAN_DATA& inData = static_cast(userDataIn)->mData; + MEDIAN_DATA::const_iterator iter = inData.begin(); + + for (; iter != inData.end(); ++iter) + { + outData[iter->first] += iter->second; + } + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode median::evaluate(mcsv1Context* context, static_any::any& valOut) { - uint64_t cnt1 = 0, cnt2 = 0; - MEDIAN_DATA& data = static_cast(context->getUserData())->mData; + uint64_t cnt1 = 0, cnt2 = 0; + MEDIAN_DATA& data = static_cast(context->getUserData())->mData; - if (data.size() == 0) - { - valOut = (DATATYPE)0; - return mcsv1_UDAF::SUCCESS; - } - - MEDIAN_DATA::iterator iter(data.begin()); - MEDIAN_DATA::iterator revfrom(data.end()); - MEDIAN_DATA::reverse_iterator riter(revfrom); - cnt1 += iter->second; - cnt2 += riter->second; - - while (iter->first < riter->first) - { - while (cnt1 < cnt2 && iter->first < riter->first) - { - ++iter; - cnt1 += iter->second; - } - - while (cnt2 < cnt1 && iter->first < riter->first) - { - ++riter; - cnt2 += riter->second; - } - - while (cnt1 == cnt2 && iter->first < riter->first) - { - ++iter; - cnt1 += iter->second; - - if (iter->first > riter->first) - { - break; - } - - ++riter; - cnt2 += riter->second; - } - } - - valOut = (iter->first + riter->first) / 2; + if (data.size() == 0) + { + valOut = (DATATYPE)0; return mcsv1_UDAF::SUCCESS; + } + + MEDIAN_DATA::iterator iter(data.begin()); + MEDIAN_DATA::iterator revfrom(data.end()); + MEDIAN_DATA::reverse_iterator riter(revfrom); + cnt1 += iter->second; + cnt2 += riter->second; + + while (iter->first < riter->first) + { + while (cnt1 < cnt2 && iter->first < riter->first) + { + ++iter; + cnt1 += iter->second; + } + + while (cnt2 < cnt1 && iter->first < riter->first) + { + ++riter; + cnt2 += riter->second; + } + + while (cnt1 == cnt2 && iter->first < riter->first) + { + ++iter; + cnt1 += iter->second; + + if (iter->first > riter->first) + { + break; + } + + ++riter; + cnt2 += riter->second; + } + } + + valOut = (iter->first + riter->first) / 2; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode median::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - static_any::any& valIn = valsDropped[0].columnData; - MEDIAN_DATA& data = static_cast(context->getUserData())->mData; + static_any::any& valIn = valsDropped[0].columnData; + MEDIAN_DATA& data = static_cast(context->getUserData())->mData; - if (valIn.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valIn.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = convertAnyTo(valIn); + DATATYPE val = convertAnyTo(valIn); - // For decimal types, we need to move the decimal point. - uint32_t scale = valsDropped[0].scale; + // For decimal types, we need to move the decimal point. + uint32_t scale = valsDropped[0].scale; - if (val != 0 && scale > 0) - { - val /= pow(10.0, (double)scale); - } + if (val != 0 && scale > 0) + { + val /= pow(10.0, (double)scale); + } - data[val]--; + data[val]--; - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode median::createUserData(UserData*& userData, int32_t& length) { - userData = new MedianData; - length = sizeof(MedianData); - return mcsv1_UDAF::SUCCESS; + userData = new MedianData; + length = sizeof(MedianData); + return mcsv1_UDAF::SUCCESS; } void MedianData::serialize(messageqcpp::ByteStream& bs) const { - MEDIAN_DATA::const_iterator iter = mData.begin(); - DATATYPE num; - uint32_t cnt; - bs << (int32_t)mData.size(); + MEDIAN_DATA::const_iterator iter = mData.begin(); + DATATYPE num; + uint32_t cnt; + bs << (int32_t)mData.size(); - for (; iter != mData.end(); ++iter) - { - num = iter->first; - bs << num; - cnt = iter->second; - bs << cnt; - } + for (; iter != mData.end(); ++iter) + { + num = iter->first; + bs << num; + cnt = iter->second; + bs << cnt; + } } void MedianData::unserialize(messageqcpp::ByteStream& bs) { - mData.clear(); - int32_t sz; - DATATYPE num; - uint32_t cnt; - bs >> sz; + mData.clear(); + int32_t sz; + DATATYPE num; + uint32_t cnt; + bs >> sz; - for (int i = 0; i < sz; ++i) - { - bs >> num; - bs >> cnt; - mData[num] = cnt; - } + for (int i = 0; i < sz; ++i) + { + bs >> num; + bs >> cnt; + mData[num] = cnt; + } } - diff --git a/utils/udfsdk/median.h b/utils/udfsdk/median.h index 5ae3025dd..43ae4fd93 100644 --- a/utils/udfsdk/median.h +++ b/utils/udfsdk/median.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* mcsv1_UDAF.h -***********************************************************************/ + * $Id$ + * + * mcsv1_UDAF.h + ***********************************************************************/ /** * Columnstore interface for writing a User Defined Aggregate @@ -74,24 +74,26 @@ namespace mcsv1sdk { - #define DATATYPE double typedef std::map MEDIAN_DATA; // Override UserData for data storage struct MedianData : public UserData { - MedianData() {}; + MedianData(){}; - virtual ~MedianData() {} + virtual ~MedianData() + { + } - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + virtual void serialize(messageqcpp::ByteStream& bs) const; + virtual void unserialize(messageqcpp::ByteStream& bs); - MEDIAN_DATA mData; -private: - // For now, copy construction is unwanted - MedianData(UserData&); + MEDIAN_DATA mData; + + private: + // For now, copy construction is unwanted + MedianData(UserData&); }; // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or @@ -106,175 +108,174 @@ private: // Return the median value of the dataset -class median : public mcsv1_UDAF +class median : public mcsv1_UDAF { -public: - // Defaults OK - median() : mcsv1_UDAF() {}; - virtual ~median() {}; + public: + // Defaults OK + median() : mcsv1_UDAF(){}; + virtual ~median(){}; - /** - * init() - * - * Mandatory. Implement this to initialize flags and instance - * data. Called once per SQL statement. You can do any sanity - * checks here. - * - * colTypes (in) - A vector of ColDataType defining the - * parameters of the UDA(n)F call. These can be used to decide - * to override the default return type. If desired, the new - * return type can be set by context->setReturnType() and - * decimal scale and precision can be set by context->setScale - * and context->setPrecision respectively. - * - * Return mcsv1_UDAF::ERROR on any error, such as non-compatible - * colTypes or wrong number of arguments. Else return - * mcsv1_UDAF::SUCCESS. - */ - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + /** + * init() + * + * Mandatory. Implement this to initialize flags and instance + * data. Called once per SQL statement. You can do any sanity + * checks here. + * + * colTypes (in) - A vector of ColDataType defining the + * parameters of the UDA(n)F call. These can be used to decide + * to override the default return type. If desired, the new + * return type can be set by context->setReturnType() and + * decimal scale and precision can be set by context->setScale + * and context->setPrecision respectively. + * + * Return mcsv1_UDAF::ERROR on any error, such as non-compatible + * colTypes or wrong number of arguments. Else return + * mcsv1_UDAF::SUCCESS. + */ + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - /** - * reset() - * - * Mandatory. Reset the UDA(n)F for a new group, partition or, - * in some cases, new Window Frame. Do not free any memory - * allocated by context->setUserDataSize(). The SDK Framework owns - * that memory and will handle that. Use this opportunity to - * reset any variables in context->getUserData() needed for the - * next aggregation. May be called multiple times if running in - * a ditributed fashion. - * - * Use this opportunity to initialize the userData. - */ - virtual ReturnCode reset(mcsv1Context* context); + /** + * reset() + * + * Mandatory. Reset the UDA(n)F for a new group, partition or, + * in some cases, new Window Frame. Do not free any memory + * allocated by context->setUserDataSize(). The SDK Framework owns + * that memory and will handle that. Use this opportunity to + * reset any variables in context->getUserData() needed for the + * next aggregation. May be called multiple times if running in + * a ditributed fashion. + * + * Use this opportunity to initialize the userData. + */ + virtual ReturnCode reset(mcsv1Context* context); - /** - * nextValue() - * - * Mandatory. Handle a single row. - * - * colsIn - A vector of data structure describing the input - * data. - * - * This function is called once for every row in the filtered - * result set (before aggregation). It is very important that - * this function is efficient. - * - * If the UDAF is running in a distributed fashion, nextValue - * cannot depend on order, as it will only be called for each - * row found on the specific PM. - * - * valsIn (in) - a vector of the parameters from the row. - */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + /** + * nextValue() + * + * Mandatory. Handle a single row. + * + * colsIn - A vector of data structure describing the input + * data. + * + * This function is called once for every row in the filtered + * result set (before aggregation). It is very important that + * this function is efficient. + * + * If the UDAF is running in a distributed fashion, nextValue + * cannot depend on order, as it will only be called for each + * row found on the specific PM. + * + * valsIn (in) - a vector of the parameters from the row. + */ + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - /** - * subEvaluate() - * - * Mandatory -- Called if the UDAF is running in a distributed - * fashion. Columnstore tries to run all aggregate functions - * distributed, depending on context. - * - * Perform an aggregation on rows partially aggregated by - * nextValue. Columnstore calls nextValue for each row on a - * given PM for a group (GROUP BY). subEvaluate is called on the - * UM to consolodate those values into a single instance of - * userData. Keep your aggregated totals in context's userData. - * The first time this is called for a group, reset() would have - * been called with this version of userData. - * - * Called for every partial data set in each group in GROUP BY. - * - * When subEvaluate has been called for all subAggregated data - * sets, Evaluate will be called with the same context as here. - * - * valIn (In) - This is a pointer to a memory block of the size - * set in setUserDataSize. It will contain the value of userData - * as seen in the last call to NextValue for a given PM. - * - */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + /** + * subEvaluate() + * + * Mandatory -- Called if the UDAF is running in a distributed + * fashion. Columnstore tries to run all aggregate functions + * distributed, depending on context. + * + * Perform an aggregation on rows partially aggregated by + * nextValue. Columnstore calls nextValue for each row on a + * given PM for a group (GROUP BY). subEvaluate is called on the + * UM to consolodate those values into a single instance of + * userData. Keep your aggregated totals in context's userData. + * The first time this is called for a group, reset() would have + * been called with this version of userData. + * + * Called for every partial data set in each group in GROUP BY. + * + * When subEvaluate has been called for all subAggregated data + * sets, Evaluate will be called with the same context as here. + * + * valIn (In) - This is a pointer to a memory block of the size + * set in setUserDataSize. It will contain the value of userData + * as seen in the last call to NextValue for a given PM. + * + */ + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - /** - * evaluate() - * - * Mandatory. Get the aggregated value. - * - * Called for every new group if UDAF GROUP BY, UDAnF partition - * or, in some cases, new Window Frame. - * - * Set the aggregated value into valOut. The datatype is assumed - * to be the same as that set in the init() function; - * - * If the UDAF is running in a distributed fashion, evaluate is - * called after a series of subEvaluate calls. - * - * valOut (out) - Set the aggregated value here. The datatype is - * assumed to be the same as that set in the init() function; - * - * To return a NULL value, don't assign to valOut. - */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + /** + * evaluate() + * + * Mandatory. Get the aggregated value. + * + * Called for every new group if UDAF GROUP BY, UDAnF partition + * or, in some cases, new Window Frame. + * + * Set the aggregated value into valOut. The datatype is assumed + * to be the same as that set in the init() function; + * + * If the UDAF is running in a distributed fashion, evaluate is + * called after a series of subEvaluate calls. + * + * valOut (out) - Set the aggregated value here. The datatype is + * assumed to be the same as that set in the init() function; + * + * To return a NULL value, don't assign to valOut. + */ + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - /** - * dropValue() - * - * Optional -- If defined, the server will call this instead of - * reset for UDAnF. - * - * Don't implement if a UDAnF has one or more of the following: - * The UDAnF can't be used with a Window Frame - * The UDAnF is not reversable in some way - * The UDAnF is not interested in optimal performance - * - * If not implemented, reset() followed by a series of - * nextValue() will be called for each movement of the Window - * Frame. - * - * If implemented, then each movement of the Window Frame will - * result in dropValue() being called for each row falling out - * of the Frame and nextValue() being called for each new row - * coming into the Frame. - * - * valsDropped (in) - a vector of the parameters from the row - * leaving the Frame - * - * dropValue() will not be called for unbounded/current row type - * frames, as those are already optimized. - */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + /** + * dropValue() + * + * Optional -- If defined, the server will call this instead of + * reset for UDAnF. + * + * Don't implement if a UDAnF has one or more of the following: + * The UDAnF can't be used with a Window Frame + * The UDAnF is not reversable in some way + * The UDAnF is not interested in optimal performance + * + * If not implemented, reset() followed by a series of + * nextValue() will be called for each movement of the Window + * Frame. + * + * If implemented, then each movement of the Window Frame will + * result in dropValue() being called for each row falling out + * of the Frame and nextValue() being called for each new row + * coming into the Frame. + * + * valsDropped (in) - a vector of the parameters from the row + * leaving the Frame + * + * dropValue() will not be called for unbounded/current row type + * frames, as those are already optimized. + */ + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); - /** - * createUserData() - * - * Optional -- If defined, the server will call this instead of - * createUserData on context. - * - * Create your variable length data structure via - * data = new - * - * The data structure may contain references to containers or - * pointers to other objects. Remember that for distributed - * processing, this may be called multiple times for variaous - * computing blocks. At the least, it will be called once per PM - * that processes the data, and once more for the UM. For UDAnF, - * it may only be called once. - * - * Set length to the length of the data structure you create. - * - * For each call to createUserData(), there will be a - * corresponding deleteUserData() where you must clean up. Any - * memory leaks are your fault. - * - */ - virtual ReturnCode createUserData(UserData*& data, int32_t& length); -protected: + /** + * createUserData() + * + * Optional -- If defined, the server will call this instead of + * createUserData on context. + * + * Create your variable length data structure via + * data = new + * + * The data structure may contain references to containers or + * pointers to other objects. Remember that for distributed + * processing, this may be called multiple times for variaous + * computing blocks. At the least, it will be called once per PM + * that processes the data, and once more for the UM. For UDAnF, + * it may only be called once. + * + * Set length to the length of the data structure you create. + * + * For each call to createUserData(), there will be a + * corresponding deleteUserData() where you must clean up. Any + * memory leaks are your fault. + * + */ + virtual ReturnCode createUserData(UserData*& data, int32_t& length); + + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_median.h - +#endif // HEADER_median.h diff --git a/utils/udfsdk/resource.h b/utils/udfsdk/resource.h index ac902ed1b..73a47fdb3 100644 --- a/utils/udfsdk/resource.h +++ b/utils/udfsdk/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/utils/udfsdk/ssq.cpp b/utils/udfsdk/ssq.cpp index ecd96ff78..d82ce2421 100644 --- a/utils/udfsdk/ssq.cpp +++ b/utils/udfsdk/ssq.cpp @@ -27,112 +27,111 @@ using namespace mcsv1sdk; struct ssq_data { - uint64_t scale; - DATATYPE sumsq; - ssq_data() : scale(0) {} + uint64_t scale; + DATATYPE sumsq; + ssq_data() : scale(0) + { + } }; #define OUT_TYPE int64_t -mcsv1_UDAF::ReturnCode ssq::init(mcsv1Context* context, - ColumnDatum* colTypes) +mcsv1_UDAF::ReturnCode ssq::init(mcsv1Context* context, ColumnDatum* colTypes) { - if (context->getParameterCount() < 1) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("ssq() with 0 arguments"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() < 1) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("ssq() with 0 arguments"); + return mcsv1_UDAF::ERROR; + } - if (context->getParameterCount() > 1) - { - context->setErrorMessage("ssq() with more than 1 argument"); - return mcsv1_UDAF::ERROR; - } + if (context->getParameterCount() > 1) + { + context->setErrorMessage("ssq() with more than 1 argument"); + return mcsv1_UDAF::ERROR; + } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("ssq() with non-numeric argument"); - return mcsv1_UDAF::ERROR; - } - - context->setUserDataSize(sizeof(ssq_data)); - context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); - context->setColWidth(8); - context->setScale(context->getScale() * 2); - context->setPrecision(19); - context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); - return mcsv1_UDAF::SUCCESS; + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("ssq() with non-numeric argument"); + return mcsv1_UDAF::ERROR; + } + context->setUserDataSize(sizeof(ssq_data)); + context->setResultType(execplan::CalpontSystemCatalog::DOUBLE); + context->setColWidth(8); + context->setScale(context->getScale() * 2); + context->setPrecision(19); + context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode ssq::reset(mcsv1Context* context) { - struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; + struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; - if (data) - { - data->scale = 0; - data->sumsq = 0; - } + if (data) + { + data->scale = 0; + data->sumsq = 0; + } - return mcsv1_UDAF::SUCCESS; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode ssq::nextValue(mcsv1Context* context, ColumnDatum* valsIn) { - struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; + struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; - if (context->isParamNull(0) || valsIn[0].columnData.empty()) - { - return mcsv1_UDAF::SUCCESS; - } - - DATATYPE val = toDouble(valsIn[0]); - - data->sumsq += val * val; + if (context->isParamNull(0) || valsIn[0].columnData.empty()) + { return mcsv1_UDAF::SUCCESS; + } + + DATATYPE val = toDouble(valsIn[0]); + + data->sumsq += val * val; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode ssq::subEvaluate(mcsv1Context* context, const UserData* userDataIn) { - // If we turn off UDAF_IGNORE_NULLS in init(), then NULLS may be sent here in cases of Joins. - // When a NULL value is sent here, userDataIn will be NULL, so check for NULLS. - if (context->isParamNull(0)) - { - return mcsv1_UDAF::SUCCESS; - } - - struct ssq_data* outData = (struct ssq_data*)context->getUserData()->data; - - struct ssq_data* inData = (struct ssq_data*)userDataIn->data; - - outData->sumsq += inData->sumsq; - + // If we turn off UDAF_IGNORE_NULLS in init(), then NULLS may be sent here in cases of Joins. + // When a NULL value is sent here, userDataIn will be NULL, so check for NULLS. + if (context->isParamNull(0)) + { return mcsv1_UDAF::SUCCESS; + } + + struct ssq_data* outData = (struct ssq_data*)context->getUserData()->data; + + struct ssq_data* inData = (struct ssq_data*)userDataIn->data; + + outData->sumsq += inData->sumsq; + + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode ssq::evaluate(mcsv1Context* context, static_any::any& valOut) { - struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; - valOut = data->sumsq; - return mcsv1_UDAF::SUCCESS; + struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; + valOut = data->sumsq; + return mcsv1_UDAF::SUCCESS; } mcsv1_UDAF::ReturnCode ssq::dropValue(mcsv1Context* context, ColumnDatum* valsDropped) { - struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; + struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; - if (valsDropped[0].columnData.empty()) - { - return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. - } + if (valsDropped[0].columnData.empty()) + { + return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. + } - DATATYPE val = toDouble(valsDropped[0]); + DATATYPE val = toDouble(valsDropped[0]); - data->sumsq -= val * val; - return mcsv1_UDAF::SUCCESS; + data->sumsq -= val * val; + return mcsv1_UDAF::SUCCESS; } - diff --git a/utils/udfsdk/ssq.h b/utils/udfsdk/ssq.h index c291d7fd1..5cd1137b0 100644 --- a/utils/udfsdk/ssq.h +++ b/utils/udfsdk/ssq.h @@ -16,10 +16,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* mcsv1_UDAF.h -***********************************************************************/ + * $Id$ + * + * mcsv1_UDAF.h + ***********************************************************************/ /** * Columnstore interface for writing a User Defined Aggregate @@ -74,7 +74,6 @@ namespace mcsv1sdk { - // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or // User Defined Analytic Function (UDAnF). // These will be singleton classes, so don't put any instance @@ -86,149 +85,147 @@ namespace mcsv1sdk // message set in config->setErrorMessage() is returned to MariaDB. // A simple aggregate to return the sum of squares -class ssq : public mcsv1_UDAF +class ssq : public mcsv1_UDAF { -public: - // Defaults OK - ssq() : mcsv1_UDAF() {}; - virtual ~ssq() {}; + public: + // Defaults OK + ssq() : mcsv1_UDAF(){}; + virtual ~ssq(){}; - /** - * init() - * - * Mandatory. Implement this to initialize flags and instance - * data. Called once per SQL statement. You can do any sanity - * checks here. - * - * colTypes (in) - A vector of ColDataType defining the - * parameters of the UDA(n)F call. These can be used to decide - * to override the default return type. If desired, the new - * return type can be set by context->setReturnType() and - * decimal scale and precision can be set by context->setScale - * and context->setPrecision respectively. - * - * Return mcsv1_UDAF::ERROR on any error, such as non-compatible - * colTypes or wrong number of arguments. Else return - * mcsv1_UDAF::SUCCESS. - */ - virtual ReturnCode init(mcsv1Context* context, - ColumnDatum* colTypes); + /** + * init() + * + * Mandatory. Implement this to initialize flags and instance + * data. Called once per SQL statement. You can do any sanity + * checks here. + * + * colTypes (in) - A vector of ColDataType defining the + * parameters of the UDA(n)F call. These can be used to decide + * to override the default return type. If desired, the new + * return type can be set by context->setReturnType() and + * decimal scale and precision can be set by context->setScale + * and context->setPrecision respectively. + * + * Return mcsv1_UDAF::ERROR on any error, such as non-compatible + * colTypes or wrong number of arguments. Else return + * mcsv1_UDAF::SUCCESS. + */ + virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); - /** - * reset() - * - * Mandatory. Reset the UDA(n)F for a new group, partition or, - * in some cases, new Window Frame. Do not free any memory - * allocated by context->createUserData(). The SDK Framework - * owns that memory and will handle that. Use this opportunity - * to reset any variables in context->getUserData() needed for - * the next aggregation. May be called multiple times on - * different modules. - */ - virtual ReturnCode reset(mcsv1Context* context); + /** + * reset() + * + * Mandatory. Reset the UDA(n)F for a new group, partition or, + * in some cases, new Window Frame. Do not free any memory + * allocated by context->createUserData(). The SDK Framework + * owns that memory and will handle that. Use this opportunity + * to reset any variables in context->getUserData() needed for + * the next aggregation. May be called multiple times on + * different modules. + */ + virtual ReturnCode reset(mcsv1Context* context); - /** - * nextValue() - * - * Mandatory. Handle a single row. - * - * colsIn - A vector of data structure describing the input - * data. - * - * This function is called once for every row in the filtered - * result set (before aggregation). It is very important that - * this function is efficient. - * - * If the UDAF is running in a distributed fashion, nextValue - * cannot depend on order, as it will only be called for each - * row found on the specific PM. - * - * valsIn (in) - a vector of the parameters from the row. - */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + /** + * nextValue() + * + * Mandatory. Handle a single row. + * + * colsIn - A vector of data structure describing the input + * data. + * + * This function is called once for every row in the filtered + * result set (before aggregation). It is very important that + * this function is efficient. + * + * If the UDAF is running in a distributed fashion, nextValue + * cannot depend on order, as it will only be called for each + * row found on the specific PM. + * + * valsIn (in) - a vector of the parameters from the row. + */ + virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - /** - * subEvaluate() - * - * Mandatory -- Called if the UDAF is running in a distributed - * fashion. Columnstore tries to run all aggregate functions - * distributed, depending on context. - * - * Perform an aggregation on rows partially aggregated by - * nextValue. Columnstore calls nextValue for each row on a - * given PM for a group (GROUP BY). subEvaluate is called on the - * UM to consolodate those values into a single instance of - * userData. Keep your aggregated totals in context's userData. - * The first time this is called for a group, reset() would have - * been called with this version of userData. - * - * Called for every partial data set in each group in GROUP BY. - * - * When subEvaluate has been called for all subAggregated data - * sets, Evaluate will be called. - * - * valIn (In) - This is a pointer to a memory block of the size - * set in setUserDataSize. It will contain the value of userData - * as seen in the last call to NextValue for a given PM. - * - */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); + /** + * subEvaluate() + * + * Mandatory -- Called if the UDAF is running in a distributed + * fashion. Columnstore tries to run all aggregate functions + * distributed, depending on context. + * + * Perform an aggregation on rows partially aggregated by + * nextValue. Columnstore calls nextValue for each row on a + * given PM for a group (GROUP BY). subEvaluate is called on the + * UM to consolodate those values into a single instance of + * userData. Keep your aggregated totals in context's userData. + * The first time this is called for a group, reset() would have + * been called with this version of userData. + * + * Called for every partial data set in each group in GROUP BY. + * + * When subEvaluate has been called for all subAggregated data + * sets, Evaluate will be called. + * + * valIn (In) - This is a pointer to a memory block of the size + * set in setUserDataSize. It will contain the value of userData + * as seen in the last call to NextValue for a given PM. + * + */ + virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); - /** - * evaluate() - * - * Mandatory. Get the aggregated value. - * - * Called for every new group if UDAF GROUP BY, UDAnF partition - * or, in some cases, new Window Frame. - * - * Set the aggregated value into valOut. The datatype is assumed - * to be the same as that set in the init() function; - * - * If the UDAF is running in a distributed fashion, evaluate is - * called after a series of subEvaluate calls. - * - * valOut (out) - Set the aggregated value here. The datatype is - * assumed to be the same as that set in the init() function; - * - * To return a NULL value, don't assign to valOut. - */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + /** + * evaluate() + * + * Mandatory. Get the aggregated value. + * + * Called for every new group if UDAF GROUP BY, UDAnF partition + * or, in some cases, new Window Frame. + * + * Set the aggregated value into valOut. The datatype is assumed + * to be the same as that set in the init() function; + * + * If the UDAF is running in a distributed fashion, evaluate is + * called after a series of subEvaluate calls. + * + * valOut (out) - Set the aggregated value here. The datatype is + * assumed to be the same as that set in the init() function; + * + * To return a NULL value, don't assign to valOut. + */ + virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - /** - * dropValue() - * - * Optional -- If defined, the server will call this instead of - * reset for UDAnF. - * - * Don't implement if a UDAnF has one or more of the following: - * The UDAnF can't be used with a Window Frame - * The UDAnF is not reversable in some way - * The UDAnF is not interested in optimal performance - * - * If not implemented, reset() followed by a series of - * nextValue() will be called for each movement of the Window - * Frame. - * - * If implemented, then each movement of the Window Frame will - * result in dropValue() being called for each row falling out - * of the Frame and nextValue() being called for each new row - * coming into the Frame. - * - * valsDropped (in) - a vector of the parameters from the row - * leaving the Frame - * - * dropValue() will not be called for unbounded/current row type - * frames, as those are already optimized. - */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + /** + * dropValue() + * + * Optional -- If defined, the server will call this instead of + * reset for UDAnF. + * + * Don't implement if a UDAnF has one or more of the following: + * The UDAnF can't be used with a Window Frame + * The UDAnF is not reversable in some way + * The UDAnF is not interested in optimal performance + * + * If not implemented, reset() followed by a series of + * nextValue() will be called for each movement of the Window + * Frame. + * + * If implemented, then each movement of the Window Frame will + * result in dropValue() being called for each row falling out + * of the Frame and nextValue() being called for each new row + * coming into the Frame. + * + * valsDropped (in) - a vector of the parameters from the row + * leaving the Frame + * + * dropValue() will not be called for unbounded/current row type + * frames, as those are already optimized. + */ + virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); -protected: + protected: }; -}; // namespace +}; // namespace mcsv1sdk #undef EXPORT -#endif // HEADER_ssq.h - +#endif // HEADER_ssq.h diff --git a/utils/udfsdk/udfmysql.cpp b/utils/udfsdk/udfmysql.cpp index c26663991..552729421 100644 --- a/utils/udfsdk/udfmysql.cpp +++ b/utils/udfsdk/udfmysql.cpp @@ -10,30 +10,23 @@ namespace { inline double cvtArgToDouble(int t, const char* v) { - double d = 0.0; + double d = 0.0; - switch (t) - { - case INT_RESULT: - d = (double)(*((long long*)v)); - break; + switch (t) + { + case INT_RESULT: d = (double)(*((long long*)v)); break; - case REAL_RESULT: - d = *((double*)v); - break; + case REAL_RESULT: d = *((double*)v); break; - case DECIMAL_RESULT: - case STRING_RESULT: - d = strtod(v, 0); - break; + case DECIMAL_RESULT: + case STRING_RESULT: d = strtod(v, 0); break; - case ROW_RESULT: - break; - } + case ROW_RESULT: break; + } - return d; -} + return d; } +} // namespace /**************************************************************************** * UDF function interface for MariaDB connector to recognize is defined in @@ -59,510 +52,487 @@ inline double cvtArgToDouble(int t, const char* v) */ extern "C" { - /** - * MCS_ADD connector stub - */ + /** + * MCS_ADD connector stub + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool mcs_add_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcs_add_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 2) { - if (args->arg_count != 2) - { - strcpy(message, "mcs_add() requires two argument"); - return 1; - } - - return 0; + strcpy(message, "mcs_add() requires two argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void mcs_add_deinit(UDF_INIT* initid) - { - } + void mcs_add_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - double mcs_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - double op1, op2; + double mcs_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + double op1, op2; - op1 = cvtArgToDouble(args->arg_type[0], args->args[0]); - op2 = cvtArgToDouble(args->arg_type[1], args->args[1]); + op1 = cvtArgToDouble(args->arg_type[0], args->args[0]); + op2 = cvtArgToDouble(args->arg_type[1], args->args[1]); - return op1 + op2; - } + return op1 + op2; + } - /** - * MCS_ISNULL connector stub - */ + /** + * MCS_ISNULL connector stub + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool mcs_isnull_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcs_isnull_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { - if (args->arg_count != 1) - { - strcpy(message, "mcs_isnull() requires one argument"); - return 1; - } - - return 0; + strcpy(message, "mcs_isnull() requires one argument"); + return 1; } + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void mcs_isnull_deinit(UDF_INIT* initid) - { - } + void mcs_isnull_deinit(UDF_INIT* initid) + { + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long mcs_isnull(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) - { - return 0; - } + long long mcs_isnull(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return 0; + } - /** - * ALLNULL connector stub - */ - struct allnull_data - { - ulonglong totalQuantity; - ulonglong totalNulls; - }; + /** + * ALLNULL connector stub + */ + struct allnull_data + { + ulonglong totalQuantity; + ulonglong totalNulls; + }; #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool allnull_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool allnull_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct allnull_data* data; + // if (args->arg_count != 1) + // { + // strcpy(message,"allnull() requires one argument"); + // return 1; + // } + + if (!(data = (struct allnull_data*)malloc(sizeof(struct allnull_data)))) { - struct allnull_data* data; -// if (args->arg_count != 1) -// { -// strcpy(message,"allnull() requires one argument"); -// return 1; -// } - - if (!(data = (struct allnull_data*) malloc(sizeof(struct allnull_data)))) - { - strmov(message, "Couldn't allocate memory"); - return 1; - } - - data->totalQuantity = 0; - data->totalNulls = 0; - - initid->ptr = (char*)data; - - return 0; + strmov(message, "Couldn't allocate memory"); + return 1; } + data->totalQuantity = 0; + data->totalNulls = 0; + + initid->ptr = (char*)data; + + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void allnull_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } + void allnull_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long allnull(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct allnull_data* data = (struct allnull_data*)initid->ptr; - return data->totalQuantity > 0 && data->totalNulls == data->totalQuantity; - } + long long allnull(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct allnull_data* data = (struct allnull_data*)initid->ptr; + return data->totalQuantity > 0 && data->totalNulls == data->totalQuantity; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void - allnull_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct allnull_data* data = (struct allnull_data*)initid->ptr; - data->totalQuantity = 0; - data->totalNulls = 0; - } + void allnull_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct allnull_data* data = (struct allnull_data*)initid->ptr; + data->totalQuantity = 0; + data->totalNulls = 0; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void - allnull_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - struct allnull_data* data = (struct allnull_data*)initid->ptr; - const char* word = args->args[0]; - data->totalQuantity++; + void allnull_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { + struct allnull_data* data = (struct allnull_data*)initid->ptr; + const char* word = args->args[0]; + data->totalQuantity++; - if (!word) - { - data->totalNulls++; - } + if (!word) + { + data->totalNulls++; } + } - /** - * SSQ connector stub - */ - struct ssq_data - { - double sumsq; - }; + /** + * SSQ connector stub + */ + struct ssq_data + { + double sumsq; + }; #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool ssq_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool ssq_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct ssq_data* data; + + if (args->arg_count != 1) { - struct ssq_data* data; - - if (args->arg_count != 1) - { - strcpy(message, "ssq() requires one argument"); - return 1; - } - - if (!(data = (struct ssq_data*) malloc(sizeof(struct ssq_data)))) - { - strmov(message, "Couldn't allocate memory"); - return 1; - } - - data->sumsq = 0; - - initid->ptr = (char*)data; - return 0; + strcpy(message, "ssq() requires one argument"); + return 1; } + if (!(data = (struct ssq_data*)malloc(sizeof(struct ssq_data)))) + { + strmov(message, "Couldn't allocate memory"); + return 1; + } + + data->sumsq = 0; + + initid->ptr = (char*)data; + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void ssq_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } + void ssq_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void - ssq_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct ssq_data* data = (struct ssq_data*)initid->ptr; - data->sumsq = 0; - } + void ssq_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct ssq_data* data = (struct ssq_data*)initid->ptr; + data->sumsq = 0; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void - ssq_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - struct ssq_data* data = (struct ssq_data*)initid->ptr; - double val = cvtArgToDouble(args->arg_type[0], args->args[0]); - data->sumsq = val * val; - } + void ssq_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { + struct ssq_data* data = (struct ssq_data*)initid->ptr; + double val = cvtArgToDouble(args->arg_type[0], args->args[0]); + data->sumsq = val * val; + } #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - long long ssq(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct ssq_data* data = (struct ssq_data*)initid->ptr; - return data->sumsq; - } + long long ssq(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct ssq_data* data = (struct ssq_data*)initid->ptr; + return data->sumsq; + } -//======================================================================= + //======================================================================= - /** - * MEDIAN connector stub - */ + /** + * MEDIAN connector stub + */ #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - my_bool median_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool median_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) { - if (args->arg_count != 1) - { - strcpy(message, "median() requires one argument"); - return 1; - } - - /* - if (!(data = (struct ssq_data*) malloc(sizeof(struct ssq_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumsq = 0; - - initid->ptr = (char*)data; - */ - return 0; + strcpy(message, "median() requires one argument"); + return 1; } + /* + if (!(data = (struct ssq_data*) malloc(sizeof(struct ssq_data)))) + { + strmov(message,"Couldn't allocate memory"); + return 1; + } + data->sumsq = 0; + + initid->ptr = (char*)data; + */ + return 0; + } + #ifdef _MSC_VER - __declspec(dllexport) + __declspec(dllexport) #endif - void median_deinit(UDF_INIT* initid) - { -// free(initid->ptr); - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void - median_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { -// struct ssq_data* data = (struct ssq_data*)initid->ptr; -// data->sumsq = 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void - median_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { -// struct ssq_data* data = (struct ssq_data*)initid->ptr; -// double val = cvtArgToDouble(args->arg_type[0], args->args[0]); -// data->sumsq = val*val; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - long long median(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { -// struct ssq_data* data = (struct ssq_data*)initid->ptr; -// return data->sumsq; - return 0; - } - - /** - * avg_mode connector stub - */ -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool avg_mode_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message, "avg_mode() requires one argument"); - return 1; - } - - /* - if (!(data = (struct ssq_data*) malloc(sizeof(struct ssq_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumsq = 0; - - initid->ptr = (char*)data; - */ - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void avg_mode_deinit(UDF_INIT* initid) - { -// free(initid->ptr); - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void - avg_mode_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { -// struct ssq_data* data = (struct ssq_data*)initid->ptr; -// data->sumsq = 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void - avg_mode_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { -// struct ssq_data* data = (struct ssq_data*)initid->ptr; -// double val = cvtArgToDouble(args->arg_type[0], args->args[0]); -// data->sumsq = val*val; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - long long avg_mode(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { -// struct ssq_data* data = (struct ssq_data*)initid->ptr; -// return data->sumsq; - return 0; - } - -//======================================================================= - - /** - * avgx connector stub. Exactly the same functionality as the - * built in avg() function. Use to test the performance of the - * API - */ - struct avgx_data - { - double sumx; - int64_t cnt; - }; - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool avgx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - struct avgx_data* data; - if (args->arg_count != 1) - { - strcpy(message,"avgx() requires one argument"); - return 1; - } - - if (!(data = (struct avgx_data*) malloc(sizeof(struct avgx_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumx = 0; - data->cnt = 0; - - initid->ptr = (char*)data; - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void avgx_deinit(UDF_INIT* initid) - { - free(initid->ptr); - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - avgx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) - { - struct avgx_data* data = (struct avgx_data*)initid->ptr; - data->sumx = 0; - data->cnt = 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - avgx_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) - { - // TODO test for NULL in x and y - struct avgx_data* data = (struct avgx_data*)initid->ptr; - double xval = cvtArgToDouble(args->arg_type[1], args->args[0]); - ++data->cnt; - data->sumx += xval; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - long long avgx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - struct avgx_data* data = (struct avgx_data*)initid->ptr; - return data->sumx / data->cnt; - } - - /** - * distinct_count connector stub - */ - #ifdef _MSC_VER - __declspec(dllexport) - #endif - my_bool distinct_count_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - if (args->arg_count != 1) - { - strcpy(message,"distinct_count() requires one argument"); - return 1; - } - - return 0; - } - - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void distinct_count_deinit(UDF_INIT* initid) - { + void median_deinit(UDF_INIT* initid) + { // free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void median_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + // struct ssq_data* data = (struct ssq_data*)initid->ptr; + // data->sumsq = 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void median_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { + // struct ssq_data* data = (struct ssq_data*)initid->ptr; + // double val = cvtArgToDouble(args->arg_type[0], args->args[0]); + // data->sumsq = val*val; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long median(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + // struct ssq_data* data = (struct ssq_data*)initid->ptr; + // return data->sumsq; + return 0; + } + + /** + * avg_mode connector stub + */ +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool avg_mode_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) + { + strcpy(message, "avg_mode() requires one argument"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - distinct_count_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + /* + if (!(data = (struct ssq_data*) malloc(sizeof(struct ssq_data)))) + { + strmov(message,"Couldn't allocate memory"); + return 1; + } + data->sumsq = 0; + + initid->ptr = (char*)data; + */ + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void avg_mode_deinit(UDF_INIT* initid) + { + // free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void avg_mode_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + // struct ssq_data* data = (struct ssq_data*)initid->ptr; + // data->sumsq = 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void avg_mode_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + // struct ssq_data* data = (struct ssq_data*)initid->ptr; + // double val = cvtArgToDouble(args->arg_type[0], args->args[0]); + // data->sumsq = val*val; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long avg_mode(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + // struct ssq_data* data = (struct ssq_data*)initid->ptr; + // return data->sumsq; + return 0; + } + + //======================================================================= + + /** + * avgx connector stub. Exactly the same functionality as the + * built in avg() function. Use to test the performance of the + * API + */ + struct avgx_data + { + double sumx; + int64_t cnt; + }; + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool avgx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + struct avgx_data* data; + if (args->arg_count != 1) { + strcpy(message, "avgx() requires one argument"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - void - distinct_count_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) + if (!(data = (struct avgx_data*)malloc(sizeof(struct avgx_data)))) { + strmov(message, "Couldn't allocate memory"); + return 1; + } + data->sumx = 0; + data->cnt = 0; + + initid->ptr = (char*)data; + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void avgx_deinit(UDF_INIT* initid) + { + free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void avgx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + struct avgx_data* data = (struct avgx_data*)initid->ptr; + data->sumx = 0; + data->cnt = 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void avgx_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* message __attribute__((unused))) + { + // TODO test for NULL in x and y + struct avgx_data* data = (struct avgx_data*)initid->ptr; + double xval = cvtArgToDouble(args->arg_type[1], args->args[0]); + ++data->cnt; + data->sumx += xval; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long avgx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + struct avgx_data* data = (struct avgx_data*)initid->ptr; + return data->sumx / data->cnt; + } + +/** + * distinct_count connector stub + */ +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool distinct_count_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1) + { + strcpy(message, "distinct_count() requires one argument"); + return 1; } - #ifdef _MSC_VER - __declspec(dllexport) - #endif - long long distinct_count(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) - { - return 0; - } + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void distinct_count_deinit(UDF_INIT* initid) + { + // free(initid->ptr); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void distinct_count_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), + char* message __attribute__((unused))) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void distinct_count_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, + char* message __attribute__((unused))) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long distinct_count(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), char* is_null, + char* error __attribute__((unused))) + { + return 0; + } } // vim:ts=4 sw=4: - diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index 97ef87d2e..0cf806181 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* -***********************************************************************/ + * $Id$ + * + * + ***********************************************************************/ //#include #include @@ -59,17 +59,17 @@ UDFSDK::~UDFSDK() */ FuncMap UDFSDK::UDFMap() const { - FuncMap fm; + FuncMap fm; - // first: function name - // second: Function pointer - // please use lower case for the function name. Because the names might be - // case-insensitive in MariaDB depending on the setting. In such case, - // the function names passed to the interface is always in lower case. - fm["mcs_add"] = new MCS_add(); - fm["mcs_isnull"] = new MCS_isnull(); + // first: function name + // second: Function pointer + // please use lower case for the function name. Because the names might be + // case-insensitive in MariaDB depending on the setting. In such case, + // the function names passed to the interface is always in lower case. + fm["mcs_add"] = new MCS_add(); + fm["mcs_isnull"] = new MCS_isnull(); - return fm; + return fm; } /*************************************************************************** @@ -77,61 +77,61 @@ FuncMap UDFSDK::UDFMap() const * * OperationType() definition */ -CalpontSystemCatalog::ColType MCS_add::operationType (FunctionParm& fp, - CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType MCS_add::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type of MCS_add is determined by the argument types - assert (fp.size() == 2); - CalpontSystemCatalog::ColType rt; + // operation type of MCS_add is determined by the argument types + assert(fp.size() == 2); + CalpontSystemCatalog::ColType rt; - if (fp[0]->data()->resultType() == fp[1]->data()->resultType()) + if (fp[0]->data()->resultType() == fp[1]->data()->resultType()) + { + rt = fp[0]->data()->resultType(); + } + else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DOUBLE || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DOUBLE) + { + rt.colDataType = CalpontSystemCatalog::DOUBLE; + rt.colWidth = 8; + } + else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIME || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TIME) + { + rt.colDataType = CalpontSystemCatalog::BIGINT; + rt.colWidth = 8; + } + else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DECIMAL || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::UDECIMAL || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DECIMAL || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) + { + rt.colDataType = CalpontSystemCatalog::DECIMAL; + rt.colWidth = 8; + } + else + { + if (isUnsigned(fp[0]->data()->resultType().colDataType) || + isUnsigned(fp[1]->data()->resultType().colDataType)) { - rt = fp[0]->data()->resultType(); - } - else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::CHAR || - fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::VARCHAR || - fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DOUBLE || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DOUBLE) - { - rt.colDataType = CalpontSystemCatalog::DOUBLE; - rt.colWidth = 8; - } - else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || - fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIME || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TIME) - { - rt.colDataType = CalpontSystemCatalog::BIGINT; - rt.colWidth = 8; - } - else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DECIMAL || - fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::UDECIMAL || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DECIMAL || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) - { - rt.colDataType = CalpontSystemCatalog::DECIMAL; - rt.colWidth = 8; + rt.colDataType = CalpontSystemCatalog::UBIGINT; + rt.colWidth = 8; } else { - if (isUnsigned(fp[0]->data()->resultType().colDataType) || - isUnsigned(fp[1]->data()->resultType().colDataType)) - { - rt.colDataType = CalpontSystemCatalog::UBIGINT; - rt.colWidth = 8; - } - else - { - rt.colDataType = CalpontSystemCatalog::BIGINT; - rt.colWidth = 8; - } + rt.colDataType = CalpontSystemCatalog::BIGINT; + rt.colWidth = 8; } + } - return rt; + return rt; } /** @@ -139,82 +139,69 @@ CalpontSystemCatalog::ColType MCS_add::operationType (FunctionParm& fp, * * This API is called when an double value is needed to return from the UDF function */ -double MCS_add::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +double MCS_add::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - switch (op_ct.colDataType) + switch (op_ct.colDataType) + { + // The APIs for the evaluation of the function parameters are the same getXXXval() + // functions. However, only two arguments are passed in, the current + // row reference and the NULL indicator isNull. + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::TINYINT: + return (parm[0]->data()->getIntVal(row, isNull) + parm[1]->data()->getIntVal(row, isNull)); + + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UTINYINT: + return (parm[0]->data()->getUintVal(row, isNull) + parm[1]->data()->getUintVal(row, isNull)); + + case CalpontSystemCatalog::DOUBLE: + return (parm[0]->data()->getDoubleVal(row, isNull) + parm[1]->data()->getDoubleVal(row, isNull)); + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - // The APIs for the evaluation of the function parameters are the same getXXXval() - // functions. However, only two arguments are passed in, the current - // row reference and the NULL indicator isNull. - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::TINYINT: - return ( parm[0]->data()->getIntVal(row, isNull) + - parm[1]->data()->getIntVal(row, isNull)); + IDB_Decimal dec; + IDB_Decimal op1 = parm[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal op2 = parm[1]->data()->getDecimalVal(row, isNull); - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UTINYINT: - return ( parm[0]->data()->getUintVal(row, isNull) + - parm[1]->data()->getUintVal(row, isNull)); + if (op1.scale == op2.scale) + { + dec.scale = op1.scale; + } + else if (op1.scale >= op2.scale) + { + dec.scale = op2.scale; + op1.value *= datatypes::scaleDivisor((uint32_t)(op1.scale - op2.scale)); + } + else + { + dec.scale = op1.scale; + op2.value *= datatypes::scaleDivisor((uint32_t)(op2.scale - op1.scale)); + } - case CalpontSystemCatalog::DOUBLE: - return ( parm[0]->data()->getDoubleVal(row, isNull) + - parm[1]->data()->getDoubleVal(row, isNull)); - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - IDB_Decimal dec; - IDB_Decimal op1 = parm[0]->data()->getDecimalVal(row, isNull); - IDB_Decimal op2 = parm[1]->data()->getDecimalVal(row, isNull); - - if (op1.scale == op2.scale) - { - dec.scale = op1.scale; - } - else if (op1.scale >= op2.scale) - { - dec.scale = op2.scale; - op1.value *= datatypes::scaleDivisor((uint32_t) (op1.scale - op2.scale)); - } - else - { - dec.scale = op1.scale; - op2.value *= datatypes::scaleDivisor((uint32_t) (op2.scale - op1.scale)); - } - - dec.value = op1.value + op2.value; - return dec.decimal64ToXFloat(); - } - - default: - return ( parm[0]->data()->getDoubleVal(row, isNull) + - parm[1]->data()->getDoubleVal(row, isNull)); + dec.value = op1.value + op2.value; + return dec.decimal64ToXFloat(); } - return 0; + default: return (parm[0]->data()->getDoubleVal(row, isNull) + parm[1]->data()->getDoubleVal(row, isNull)); + } + + return 0; } -long double MCS_add::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +long double MCS_add::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - return getDoubleVal(row, parm, isNull, op_ct); + return getDoubleVal(row, parm, isNull, op_ct); } -float MCS_add::getFloatVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +float MCS_add::getFloatVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (float)getDoubleVal(row, parm, isNull, op_ct); + return (float)getDoubleVal(row, parm, isNull, op_ct); } /** @@ -226,35 +213,27 @@ float MCS_add::getFloatVal(Row& row, * getDoubleVal and apply the conversion. This method may not fit for all the UDF * implementation. */ -int64_t MCS_add::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +int64_t MCS_add::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (int64_t)getDoubleVal(row, parm, isNull, op_ct); + return (int64_t)getDoubleVal(row, parm, isNull, op_ct); } -string MCS_add::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +string MCS_add::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - // One will need a more efficient implementation if this API is frequently - // called for this UDF function. This code is for demonstration purpose. - ostringstream oss; - oss << getDoubleVal(row, parm, isNull, op_ct); - return oss.str(); + // One will need a more efficient implementation if this API is frequently + // called for this UDF function. This code is for demonstration purpose. + ostringstream oss; + oss << getDoubleVal(row, parm, isNull, op_ct); + return oss.str(); } -IDB_Decimal MCS_add::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +IDB_Decimal MCS_add::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal dec; - dec.value = getIntVal(row, parm, isNull, op_ct); - dec.scale = 0; - return dec; + IDB_Decimal dec; + dec.value = getIntVal(row, parm, isNull, op_ct); + dec.scale = 0; + return dec; } /** @@ -263,12 +242,10 @@ IDB_Decimal MCS_add::getDecimalVal(Row& row, * either not implement this API and an MCS5001 error will be thrown, * or throw a customized exception here. */ -int32_t MCS_add::getDateIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int32_t MCS_add::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - throw logic_error("Invalid API called for MCS_ADD"); + throw logic_error("Invalid API called for MCS_ADD"); } /** @@ -277,20 +254,15 @@ int32_t MCS_add::getDateIntVal(Row& row, * either not implement this API and an MCS-5001 error will be thrown, * or throw a customized exception here. */ -int64_t MCS_add::getDatetimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t MCS_add::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (int64_t)getDoubleVal(row, parm, isNull, op_ct); + return (int64_t)getDoubleVal(row, parm, isNull, op_ct); } -bool MCS_add::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +bool MCS_add::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return false; + return false; } /*************************************************************************** @@ -298,12 +270,12 @@ bool MCS_add::getBoolVal(Row& row, * * OperationType() definition */ -CalpontSystemCatalog::ColType MCS_isnull::operationType (FunctionParm& fp, - CalpontSystemCatalog::ColType& resultType) +CalpontSystemCatalog::ColType MCS_isnull::operationType(FunctionParm& fp, + CalpontSystemCatalog::ColType& resultType) { - // operation type of MCS_isnull should be the same as the argument type - assert (fp.size() == 1); - return fp[0]->data()->resultType(); + // operation type of MCS_isnull should be the same as the argument type + assert(fp.size() == 1); + return fp[0]->data()->resultType(); } /** @@ -311,37 +283,29 @@ CalpontSystemCatalog::ColType MCS_isnull::operationType (FunctionParm& fp, * * This would be the most commonly called API for MCS_isnull function */ -bool MCS_isnull::getBoolVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +bool MCS_isnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - switch (op_ct.colDataType) - { - // For the purpose of this function, one does not need to get the value of - // the argument. One only need to know if the argument is NULL. The passed - // in parameter isNull will be set if the parameter is evaluated NULL. - // Please note that before this function returns, isNull should be set to - // false, otherwise the result of the function would be considered NULL, - // which is not possible for MCS_isnull(). - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - parm[0]->data()->getDecimalVal(row, isNull); - break; + switch (op_ct.colDataType) + { + // For the purpose of this function, one does not need to get the value of + // the argument. One only need to know if the argument is NULL. The passed + // in parameter isNull will be set if the parameter is evaluated NULL. + // Please note that before this function returns, isNull should be set to + // false, otherwise the result of the function would be considered NULL, + // which is not possible for MCS_isnull(). + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: parm[0]->data()->getDecimalVal(row, isNull); break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - parm[0]->data()->getStrVal(row, isNull); - break; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: parm[0]->data()->getStrVal(row, isNull); break; - default: - parm[0]->data()->getIntVal(row, isNull); - } + default: parm[0]->data()->getIntVal(row, isNull); + } - bool ret = isNull; - // It's important to reset isNull indicator. - isNull = false; - return ret; + bool ret = isNull; + // It's important to reset isNull indicator. + isNull = false; + return ret; } /** @@ -349,28 +313,22 @@ bool MCS_isnull::getBoolVal(Row& row, * * This API is called when a double value is needed to return from the UDF function */ -double MCS_isnull::getDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, +double MCS_isnull::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); } -long double MCS_isnull::getLongDoubleVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +long double MCS_isnull::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { - return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); } -float MCS_isnull::getFloatVal(Row& row, - FunctionParm& parm, - bool& isNull, +float MCS_isnull::getFloatVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); } /** @@ -382,52 +340,39 @@ float MCS_isnull::getFloatVal(Row& row, * getDoubleVal and apply the conversion. This method may not fit for all the UDF * implementations. */ -int64_t MCS_isnull::getIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t MCS_isnull::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); } -string MCS_isnull::getStrVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +string MCS_isnull::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - // This needs to be more efficient if this API will be frequently - // called for this UDF function. - return (getBoolVal(row, parm, isNull, op_ct) ? "1" : "0"); - + // This needs to be more efficient if this API will be frequently + // called for this UDF function. + return (getBoolVal(row, parm, isNull, op_ct) ? "1" : "0"); } -IDB_Decimal MCS_isnull::getDecimalVal(Row& row, - FunctionParm& parm, - bool& isNull, +IDB_Decimal MCS_isnull::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - IDB_Decimal dec; - dec.value = (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); - dec.scale = 0; - return dec; + IDB_Decimal dec; + dec.value = (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + dec.scale = 0; + return dec; } -int32_t MCS_isnull::getDateIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int32_t MCS_isnull::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); } -int64_t MCS_isnull::getDatetimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, +int64_t MCS_isnull::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); + return (getBoolVal(row, parm, isNull, op_ct) ? 1 : 0); } -} +} // namespace udfsdk // vim:ts=4 sw=4: - diff --git a/utils/udfsdk/udfsdk.h b/utils/udfsdk/udfsdk.h index dafb89bd7..9e53601b3 100644 --- a/utils/udfsdk/udfsdk.h +++ b/utils/udfsdk/udfsdk.h @@ -17,10 +17,10 @@ MA 02110-1301, USA. */ /*********************************************************************** -* $Id$ -* -* -***********************************************************************/ + * $Id$ + * + * + ***********************************************************************/ /** * MariaDB ColumnStore interface for writing a user defined function (UDF). @@ -55,26 +55,23 @@ namespace udfsdk { - /** * UDFSDK interface. Do not make modification here. */ class UDFSDK { -public: - EXPORT UDFSDK(); + public: + EXPORT UDFSDK(); - EXPORT ~UDFSDK(); + EXPORT ~UDFSDK(); - EXPORT funcexp::FuncMap UDFMap() const; - -protected: - -private: - //defaults okay - //UDFSDK(const UDFSDK& rhs); - //UDFSDK& operator=(const UDFSDK& rhs); + EXPORT funcexp::FuncMap UDFMap() const; + protected: + private: + // defaults okay + // UDFSDK(const UDFSDK& rhs); + // UDFSDK& operator=(const UDFSDK& rhs); }; /** @@ -104,135 +101,122 @@ private: */ class MCS_add : public funcexp::Func { -public: - /* - * Constructor. Pass the function name to the base constructor. - */ - MCS_add() : Func("mcs_add") {} + public: + /* + * Constructor. Pass the function name to the base constructor. + */ + MCS_add() : Func("mcs_add") + { + } - /* - * Destructor. MCS_add does not need to do anything here to clean up. - */ - virtual ~MCS_add() {} + /* + * Destructor. MCS_add does not need to do anything here to clean up. + */ + virtual ~MCS_add() + { + } - /** - * Decide on the function's operation type - * - * Operation type decides which API needs to be called for each function - * parameter. Sometimes it is obvious. e.g. for function substr (c1, c2, c3), - * one knows that getStrVal(), getIntVal() and getIntVal() should be called for - * the three parameters in sequence. In that case, a dummy type can be returned - * because it won't be used in the function implementation. Sometimes the - * operation type is decided by the data type of the function parameters. - * e.g., isnull(c1) function, one should call the corresponding getXXXval() - * function that in compatible with the result type of c1. - * - * @parm fp vector of function parameters - * Each element is a boost::shared_ptr of execplan::ParseTree. class - * ParseTree is defined in ~/dbcon/execplan/parsetree.h - * @parm resultType result type of this function - * Sometimes it may affect the operation type, but most of the time it - * can be ignored. Struct ColType is defined in ~/dbcon/execplan/calpontsystemcatalog.h - * @return operation type for this function - * - * This function is called only one from the connector. Once it's determined, it - * will be passed to the getXXXval() APIs during function evaluation. - */ - execplan::CalpontSystemCatalog::ColType operationType(funcexp::FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + /** + * Decide on the function's operation type + * + * Operation type decides which API needs to be called for each function + * parameter. Sometimes it is obvious. e.g. for function substr (c1, c2, c3), + * one knows that getStrVal(), getIntVal() and getIntVal() should be called for + * the three parameters in sequence. In that case, a dummy type can be returned + * because it won't be used in the function implementation. Sometimes the + * operation type is decided by the data type of the function parameters. + * e.g., isnull(c1) function, one should call the corresponding getXXXval() + * function that in compatible with the result type of c1. + * + * @parm fp vector of function parameters + * Each element is a boost::shared_ptr of execplan::ParseTree. class + * ParseTree is defined in ~/dbcon/execplan/parsetree.h + * @parm resultType result type of this function + * Sometimes it may affect the operation type, but most of the time it + * can be ignored. Struct ColType is defined in ~/dbcon/execplan/calpontsystemcatalog.h + * @return operation type for this function + * + * This function is called only one from the connector. Once it's determined, it + * will be passed to the getXXXval() APIs during function evaluation. + */ + execplan::CalpontSystemCatalog::ColType operationType(funcexp::FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - /** - * Returns an integer result of this function. - * All the getXXXvalue APIs take the same arguments. They will be called - * for every row in the result set when the function is being evaluated. - * So these functions needs to be efficient. - * - * @parm row reference of the current row - * @parm fp function parameters - * @parm isNull NULL indicator throughout this function evaluation. - * the same reference is passed to all the function argument - * evaluations. One always need to know if any argument is NULL - * to decide the result of the function. It's explained in detail - * in MCS_isnull() function example. - * @parm op_ct the operation type that is determined in operationType(). - * - */ - virtual int64_t getIntVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a double result of this function. - */ - - virtual double getDoubleVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a long double result of this function. - */ - - virtual long double getLongDoubleVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - /** - * Returns a float result of this function. - */ - virtual float getFloatVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a string result of this function. - */ - virtual std::string getStrVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a bool result of this function. - */ - virtual bool getBoolVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, + /** + * Returns an integer result of this function. + * All the getXXXvalue APIs take the same arguments. They will be called + * for every row in the result set when the function is being evaluated. + * So these functions needs to be efficient. + * + * @parm row reference of the current row + * @parm fp function parameters + * @parm isNull NULL indicator throughout this function evaluation. + * the same reference is passed to all the function argument + * evaluations. One always need to know if any argument is NULL + * to decide the result of the function. It's explained in detail + * in MCS_isnull() function example. + * @parm op_ct the operation type that is determined in operationType(). + * + */ + virtual int64_t getIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - /** - * Returns a decimal result of this function. - * - * IDB_Decimal is defined in ~/execplan/treenode.h - */ - virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + /** + * Returns a double result of this function. + */ - /** - * Returns an integer representation of a date result of the function. - * - * Check the date/time functions in ~/utils/funcexp for implementation - * example of this API. - */ - virtual int32_t getDateIntVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + virtual double getDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - /** - * Returns an integer representation of a datetime result of the function. - * - * Check the date/time functions in ~/utils/funcexp for implementation - * example of this API. - */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + /** + * Returns a long double result of this function. + */ + + virtual long double getLongDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + /** + * Returns a float result of this function. + */ + virtual float getFloatVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns a string result of this function. + */ + virtual std::string getStrVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns a bool result of this function. + */ + virtual bool getBoolVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns a decimal result of this function. + * + * IDB_Decimal is defined in ~/execplan/treenode.h + */ + virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns an integer representation of a date result of the function. + * + * Check the date/time functions in ~/utils/funcexp for implementation + * example of this API. + */ + virtual int32_t getDateIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns an integer representation of a datetime result of the function. + * + * Check the date/time functions in ~/utils/funcexp for implementation + * example of this API. + */ + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @@ -242,106 +226,92 @@ public: */ class MCS_isnull : public funcexp::Func { -public: - /* - * Constructor. Pass the function name to the base constructor. - */ - MCS_isnull() : Func("mcs_isnull") {} + public: + /* + * Constructor. Pass the function name to the base constructor. + */ + MCS_isnull() : Func("mcs_isnull") + { + } - /* - * Destructor. MCS_add does not need to do anything here to clean up. - */ - virtual ~MCS_isnull() {} + /* + * Destructor. MCS_add does not need to do anything here to clean up. + */ + virtual ~MCS_isnull() + { + } - /** - * Decide on the function's operation type - */ - execplan::CalpontSystemCatalog::ColType operationType(funcexp::FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + /** + * Decide on the function's operation type + */ + execplan::CalpontSystemCatalog::ColType operationType(funcexp::FunctionParm& fp, + execplan::CalpontSystemCatalog::ColType& resultType); - /** - * Returns an integer result of this function. - */ - virtual int64_t getIntVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a double result of this function. - */ - virtual double getDoubleVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a double result of this function. - */ - virtual long double getLongDoubleVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a float result of this function. - */ - virtual float getFloatVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a string result of this function. - */ - virtual std::string getStrVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - - /** - * Returns a bool result of this function. - */ - virtual bool getBoolVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, + /** + * Returns an integer result of this function. + */ + virtual int64_t getIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - /** - * Returns a decimal result of this function. - * - * IDB_Decimal is defined in ~/execplan/treenode.h - */ - virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + /** + * Returns a double result of this function. + */ + virtual double getDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - /** - * Returns an integer representation of a date result of the function. - * - * Check the date/time functions in ~/utils/funcexp for implementation - * example of this API. - */ - virtual int32_t getDateIntVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + /** + * Returns a double result of this function. + */ + virtual long double getLongDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); - /** - * Returns an integer representation of a datetime result of the function. - * - * Check the date/time functions in ~/utils/funcexp for implementation - * example of this API. - */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, - funcexp::FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + /** + * Returns a float result of this function. + */ + virtual float getFloatVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns a string result of this function. + */ + virtual std::string getStrVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns a bool result of this function. + */ + virtual bool getBoolVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns a decimal result of this function. + * + * IDB_Decimal is defined in ~/execplan/treenode.h + */ + virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns an integer representation of a date result of the function. + * + * Check the date/time functions in ~/utils/funcexp for implementation + * example of this API. + */ + virtual int32_t getDateIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + /** + * Returns an integer representation of a datetime result of the function. + * + * Check the date/time functions in ~/utils/funcexp for implementation + * example of this API. + */ + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; -} +} // namespace udfsdk #undef EXPORT // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/framebound.cpp b/utils/windowfunction/framebound.cpp index e47b91c08..043620f53 100644 --- a/utils/windowfunction/framebound.cpp +++ b/utils/windowfunction/framebound.cpp @@ -17,7 +17,6 @@ // $Id: framebound.cpp 3828 2013-05-22 17:58:14Z xlou $ - //#define NDEBUG #include #include @@ -35,62 +34,42 @@ using namespace ordering; #include "framebound.h" - namespace windowfunction { - - int64_t FrameBound::getBound(int64_t b, int64_t e, int64_t c) { - if (fBoundType == WF__UNBOUNDED_PRECEDING) - return b; + if (fBoundType == WF__UNBOUNDED_PRECEDING) + return b; - return e; + return e; } - const string FrameBound::toString() const { - ostringstream oss; + ostringstream oss; - switch (fBoundType) - { - case WF__UNBOUNDED_PRECEDING: - oss << "unbound preceding"; - break; + switch (fBoundType) + { + case WF__UNBOUNDED_PRECEDING: oss << "unbound preceding"; break; - case WF__UNBOUNDED_FOLLOWING: - oss << "unbound following"; - break; + case WF__UNBOUNDED_FOLLOWING: oss << "unbound following"; break; - case WF__CONSTANT_PRECEDING: - oss << "constant preceding"; - break; + case WF__CONSTANT_PRECEDING: oss << "constant preceding"; break; - case WF__CONSTANT_FOLLOWING: - oss << "constant following"; - break; + case WF__CONSTANT_FOLLOWING: oss << "constant following"; break; - case WF__EXPRESSION_PRECEDING: - oss << "expression preceding"; - break; + case WF__EXPRESSION_PRECEDING: oss << "expression preceding"; break; - case WF__EXPRESSION_FOLLOWING: - oss << "expression following"; - break; + case WF__EXPRESSION_FOLLOWING: oss << "expression following"; break; - case WF__CURRENT_ROW: - default: - oss << "current row"; - break; - } + case WF__CURRENT_ROW: + default: oss << "current row"; break; + } - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/framebound.h b/utils/windowfunction/framebound.h index e6fbea4c2..27038ae11 100644 --- a/utils/windowfunction/framebound.h +++ b/utils/windowfunction/framebound.h @@ -17,7 +17,6 @@ // $Id: framebound.h 3932 2013-06-25 16:08:10Z xlou $ - #ifndef UTILS_FRAMEBOUND_H #define UTILS_FRAMEBOUND_H @@ -26,132 +25,126 @@ #include "rowgroup.h" #include "windowfunctionstep.h" - namespace ordering { // forward reference class EqualCompData; -}; +}; // namespace ordering namespace windowfunction { - - -const int WF__UNBOUNDED_PRECEDING = 0; -const int WF__CONSTANT_PRECEDING = 1; +const int WF__UNBOUNDED_PRECEDING = 0; +const int WF__CONSTANT_PRECEDING = 1; const int WF__EXPRESSION_PRECEDING = 2; -const int WF__CURRENT_ROW = 3; -const int WF__UNBOUNDED_FOLLOWING = 4; -const int WF__CONSTANT_FOLLOWING = 5; +const int WF__CURRENT_ROW = 3; +const int WF__UNBOUNDED_FOLLOWING = 4; +const int WF__CONSTANT_FOLLOWING = 5; const int WF__EXPRESSION_FOLLOWING = 6; - -const int WF__BOUND_ALL = -1; // unbounded - unbounded -const int WF__BOUND_ROLLING = -2; // unbounded - current row +const int WF__BOUND_ALL = -1; // unbounded - unbounded +const int WF__BOUND_ROLLING = -2; // unbounded - current row /** @brief class FrameBound * */ class FrameBound { -public: - /** @brief FrameBound constructor - * @param t, frame type - */ - FrameBound(int t = 0) : fBoundType(t), fStart(true) {}; + public: + /** @brief FrameBound constructor + * @param t, frame type + */ + FrameBound(int t = 0) : fBoundType(t), fStart(true){}; - /** @brief FrameBound destructor - */ - virtual ~FrameBound() {}; + /** @brief FrameBound destructor + */ + virtual ~FrameBound(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return new FrameBound(*this); - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return new FrameBound(*this); + } - /** @brief virtual void getBound - * @param b: partition start position - * @param e: partition end position - * @param c: current position - * @return : frame position - */ - virtual int64_t getBound(int64_t b, int64_t e, int64_t c); + /** @brief virtual void getBound + * @param b: partition start position + * @param e: partition end position + * @param c: current position + * @return : frame position + */ + virtual int64_t getBound(int64_t b, int64_t e, int64_t c); - virtual const std::string toString() const; + virtual const std::string toString() const; - void setRowData(const boost::shared_ptr >& d) - { - fRowData = d; - } - void setRowMetaData(const rowgroup::RowGroup& g, const rowgroup::Row& r) - { - fRowGroup = g; - fRow = r; - } + void setRowData(const boost::shared_ptr >& d) + { + fRowData = d; + } + void setRowMetaData(const rowgroup::RowGroup& g, const rowgroup::Row& r) + { + fRowGroup = g; + fRow = r; + } - int64_t boundType() const - { - return fBoundType; - } - void boundType(int64_t t) - { - fBoundType = t; - } + int64_t boundType() const + { + return fBoundType; + } + void boundType(int64_t t) + { + fBoundType = t; + } - bool start() const - { - return fStart; - } - void start(bool s) - { - fStart = s; - } + bool start() const + { + return fStart; + } + void start(bool s) + { + fStart = s; + } - const boost::shared_ptr& peer() const - { - return fPeer; - } - void peer(const boost::shared_ptr& p) - { - fPeer = p; - } + const boost::shared_ptr& peer() const + { + return fPeer; + } + void peer(const boost::shared_ptr& p) + { + fPeer = p; + } - // for string table - void setCallback(joblist::WindowFunctionStep* step) - { - fStep = step; - } - rowgroup::Row::Pointer getPointer(joblist::RowPosition& r) - { - return fStep->getPointer(r, fRowGroup, fRow); - } + // for string table + void setCallback(joblist::WindowFunctionStep* step) + { + fStep = step; + } + rowgroup::Row::Pointer getPointer(joblist::RowPosition& r) + { + return fStep->getPointer(r, fRowGroup, fRow); + } -protected: - // boundary type - int64_t fBoundType; - bool fStart; + protected: + // boundary type + int64_t fBoundType; + bool fStart; - // data - boost::shared_ptr > fRowData; + // data + boost::shared_ptr > fRowData; - // row meta data - rowgroup::RowGroup fRowGroup; - rowgroup::Row fRow; + // row meta data + rowgroup::RowGroup fRowGroup; + rowgroup::Row fRow; - // functor for peer checking - boost::shared_ptr fPeer; + // functor for peer checking + boost::shared_ptr fPeer; - // pointer back to step - joblist::WindowFunctionStep* fStep; + // pointer back to step + joblist::WindowFunctionStep* fStep; }; - extern std::map colType2String; - -} // namespace +} // namespace windowfunction #endif // UTILS_FRAMEBOUND_H diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index e2ae172e1..0b356bce7 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -18,7 +18,6 @@ // $Id: frameboundrange.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -36,398 +35,380 @@ using namespace ordering; #include "frameboundrange.h" - namespace windowfunction { - - int64_t FrameBoundRange::getBound(int64_t b, int64_t e, int64_t c) { - if (fStart) + if (fStart) + { + while (c > b) { - while (c > b) - { - if (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c - 1)))) - break; + if (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c - 1)))) + break; - c--; - } + c--; } - else + } + else + { + while (c < e) { - while (c < e) - { - if (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c + 1)))) - break; + if (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c + 1)))) + break; - c++; - } + c++; } + } - return c; + return c; } - const string FrameBoundRange::toString() const { - return FrameBound::toString(); + return FrameBound::toString(); } - -template +template int64_t FrameBoundConstantRange::getBound(int64_t b, int64_t e, int64_t c) { - // set row data - fRow.setData(getPointer(fRowData->at(c))); - getValue(fValue, fIndex[2]); + // set row data + fRow.setData(getPointer(fRowData->at(c))); + getValue(fValue, fIndex[2]); - // make sure the expr is not negative - validate(); + // make sure the expr is not negative + validate(); - // calculate the offset, and move - if (fIsZero) - c = FrameBoundRange::getBound(b, e, c); - else if (fBoundType < WF__CURRENT_ROW) - c -= getPrecedingOffset(c, b); - else - c += getFollowingOffset(c, e); + // calculate the offset, and move + if (fIsZero) + c = FrameBoundRange::getBound(b, e, c); + else if (fBoundType < WF__CURRENT_ROW) + c -= getPrecedingOffset(c, b); + else + c += getFollowingOffset(c, e); - return c; + return c; } - -template +template int64_t FrameBoundConstantRange::getPrecedingOffset(int64_t c, int64_t b) { - // test each row to find the bound - bool next = true; - int64_t i = c; - int64_t j = 0; + // test each row to find the bound + bool next = true; + int64_t i = c; + int64_t j = 0; - for (i--, j++; i >= b && next; i--, j++) + for (i--, j++; i >= b && next; i--, j++) + { + // set row data, get order by column value + fRow.setData(getPointer(fRowData->at(i))); + ValueType v; + getValue(v, fIndex[0]); + + if (v.fIsNull) { - // set row data, get order by column value - fRow.setData(getPointer(fRowData->at(i))); - ValueType v; - getValue(v, fIndex[0]); - - if (v.fIsNull) - { - next = fValue.fIsNull; // let null = null - } - else if (fValue.fIsNull) - { - next = false; - } - else if (fAsc && v.fValue < fValue.fValue) - { - next = false; - } - else if (!fAsc && v.fValue > fValue.fValue) - { - next = false; - } - else if (!fStart && v.fValue == fValue.fValue) - { - next = false; - } + next = fValue.fIsNull; // let null = null } - - if (!next) + else if (fValue.fIsNull) { - if (fStart) - j -= 2; - else - j -= 1; + next = false; } + else if (fAsc && v.fValue < fValue.fValue) + { + next = false; + } + else if (!fAsc && v.fValue > fValue.fValue) + { + next = false; + } + else if (!fStart && v.fValue == fValue.fValue) + { + next = false; + } + } - return j; + if (!next) + { + if (fStart) + j -= 2; + else + j -= 1; + } + + return j; } - -template +template int64_t FrameBoundConstantRange::getFollowingOffset(int64_t c, int64_t e) { - // test each row to find the bound - bool next = true; - int64_t i = c; - int64_t j = 0; + // test each row to find the bound + bool next = true; + int64_t i = c; + int64_t j = 0; - for (i++, j++; i <= e && next; i++, j++) + for (i++, j++; i <= e && next; i++, j++) + { + // set row data, get order by column value + fRow.setData(getPointer(fRowData->at(i))); + ValueType v; + getValue(v, fIndex[0]); + + if (v.fIsNull) { - // set row data, get order by column value - fRow.setData(getPointer(fRowData->at(i))); - ValueType v; - getValue(v, fIndex[0]); - - if (v.fIsNull) - { - next = fValue.fIsNull; // let null = null - } - else if (fValue.fIsNull) - { - next = false; - } - else if (fAsc && v.fValue > fValue.fValue) - { - next = false; - } - else if (!fAsc && v.fValue < fValue.fValue) - { - next = false; - } - else if (fStart && v.fValue == fValue.fValue) - { - next = false; - } + next = fValue.fIsNull; // let null = null } - - if (!next) + else if (fValue.fIsNull) { - if (fStart) - j -= 1; - else - j -= 2; + next = false; } + else if (fAsc && v.fValue > fValue.fValue) + { + next = false; + } + else if (!fAsc && v.fValue < fValue.fValue) + { + next = false; + } + else if (fStart && v.fValue == fValue.fValue) + { + next = false; + } + } - return j; + if (!next) + { + if (fStart) + j -= 1; + else + j -= 2; + } + + return j; } - -template +template void FrameBoundConstantRange::getValue(ValueType& v, int64_t i) { - v.fIsNull = fRow.isNullValue(i); + v.fIsNull = fRow.isNullValue(i); - if (!v.fIsNull) - v.fValue = fRow.getIntField(i); + if (!v.fIsNull) + v.fValue = fRow.getIntField(i); } - -template +template T FrameBoundConstantRange::getValueByType(int64_t i) { - T t; - return t; + T t; + return t; } - -template<> int64_t FrameBoundConstantRange::getValueByType(int64_t i) +template <> +int64_t FrameBoundConstantRange::getValueByType(int64_t i) { - return fRow.getIntField(i); + return fRow.getIntField(i); } - -template<> uint64_t FrameBoundConstantRange::getValueByType(int64_t i) +template <> +uint64_t FrameBoundConstantRange::getValueByType(int64_t i) { - uint64_t v = fRow.getUintField(i); + uint64_t v = fRow.getUintField(i); - // convert date to datetime, [refer to treenode.h] - if (fRow.getColType(fIndex[0]) == execplan::CalpontSystemCatalog::DATE && i == 0) - v = v << 32; + // convert date to datetime, [refer to treenode.h] + if (fRow.getColType(fIndex[0]) == execplan::CalpontSystemCatalog::DATE && i == 0) + v = v << 32; - return v; + return v; } - -template<> double FrameBoundConstantRange::getValueByType(int64_t i) +template <> +double FrameBoundConstantRange::getValueByType(int64_t i) { - return fRow.getDoubleField(i); + return fRow.getDoubleField(i); } - -template<> float FrameBoundConstantRange::getValueByType(int64_t i) +template <> +float FrameBoundConstantRange::getValueByType(int64_t i) { - return fRow.getFloatField(i); + return fRow.getFloatField(i); } - -template +template const string FrameBoundConstantRange::toString() const { - ostringstream oss; - oss << fValue.fValue << " " << FrameBound::toString(); - return oss.str(); + ostringstream oss; + oss << fValue.fValue << " " << FrameBound::toString(); + return oss.str(); } - -template +template int64_t FrameBoundExpressionRange::getPrecedingOffset(int64_t c, int64_t b) { - return FrameBoundConstantRange::getPrecedingOffset(c, b); + return FrameBoundConstantRange::getPrecedingOffset(c, b); } - -template +template int64_t FrameBoundExpressionRange::getFollowingOffset(int64_t c, int64_t e) { - return FrameBoundConstantRange::getFollowingOffset(c, e); + return FrameBoundConstantRange::getFollowingOffset(c, e); } - -template +template void FrameBoundExpressionRange::validate() { - bool invalid = false; - ostringstream oss; + bool invalid = false; + ostringstream oss; - if (this->fRow.isNullValue(this->fIndex[1])) + if (this->fRow.isNullValue(this->fIndex[1])) + { + invalid = true; + oss << "NULL"; + } + else + { + switch (this->fRow.getColType(this->fIndex[1])) { - invalid = true; - oss << "NULL"; - } - else - { - switch (this->fRow.getColType(this->fIndex[1])) + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + { + int64_t tmp = this->fRow.getIntField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - int64_t tmp = this->fRow.getIntField(this->fIndex[1]); - this->fIsZero = (tmp == 0); - - if (tmp < 0) - { - invalid = true; - oss << tmp; - } - - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - { - if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) - < datatypes::MAXDECIMALWIDTH)) - { - int64_t tmp = this->fRow.getIntField(this->fIndex[1]); - this->fIsZero = (tmp == 0); - - if (tmp < 0) - { - invalid = true; - oss << ""; - } - } - else - { - datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); - this->fIsZero = (tmp == 0); - - if (tmp < 0) - { - invalid = true; - oss << ""; - } - } - break; - } - - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) - < datatypes::MAXDECIMALWIDTH)) - { - uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); - this->fIsZero = (tmp == 0); - } - else - { - datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); - this->fIsZero = (tmp == 0); - } - break; - } - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - double tmp = this->fRow.getDoubleField(this->fIndex[1]); - this->fIsZero = (tmp == 0.0); - - if (tmp < 0) - { - invalid = true; - oss << tmp; - } - - break; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - long double tmp = this->fRow.getLongDoubleField(this->fIndex[1]); - this->fIsZero = (tmp == 0.0); - - if (tmp < 0) - { - invalid = true; - oss << tmp; - } - - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - float tmp = this->fRow.getFloatField(this->fIndex[1]); - this->fIsZero = (tmp == 0.0); - - if (tmp < 0) - { - invalid = true; - oss << tmp; - } - - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - default: - { - uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); - this->fIsZero = (tmp == 0); - break; - } + invalid = true; + oss << tmp; } - } - if (invalid) - { - oss << " (expr)"; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, oss.str()), - ERR_WF_BOUND_OUT_OF_RANGE); + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + { + if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) < datatypes::MAXDECIMALWIDTH)) + { + int64_t tmp = this->fRow.getIntField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) + { + invalid = true; + oss << ""; + } + } + else + { + datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) + { + invalid = true; + oss << ""; + } + } + break; + } + + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) < datatypes::MAXDECIMALWIDTH)) + { + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + } + else + { + datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + } + break; + } + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + double tmp = this->fRow.getDoubleField(this->fIndex[1]); + this->fIsZero = (tmp == 0.0); + + if (tmp < 0) + { + invalid = true; + oss << tmp; + } + + break; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + long double tmp = this->fRow.getLongDoubleField(this->fIndex[1]); + this->fIsZero = (tmp == 0.0); + + if (tmp < 0) + { + invalid = true; + oss << tmp; + } + + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + float tmp = this->fRow.getFloatField(this->fIndex[1]); + this->fIsZero = (tmp == 0.0); + + if (tmp < 0) + { + invalid = true; + oss << tmp; + } + + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + default: + { + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + break; + } } + } + + if (invalid) + { + oss << " (expr)"; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, oss.str()), + ERR_WF_BOUND_OUT_OF_RANGE); + } } - -template +template const string FrameBoundExpressionRange::toString() const { - ostringstream oss; - oss << " value_expr " << FrameBound::toString(); - return oss.str(); + ostringstream oss; + oss << " value_expr " << FrameBound::toString(); + return oss.str(); } +template class FrameBoundConstantRange; +template class FrameBoundConstantRange; +template class FrameBoundConstantRange; +template class FrameBoundConstantRange; -template class FrameBoundConstantRange; -template class FrameBoundConstantRange; -template class FrameBoundConstantRange; -template class FrameBoundConstantRange; +template class FrameBoundExpressionRange; +template class FrameBoundExpressionRange; +template class FrameBoundExpressionRange; +template class FrameBoundExpressionRange; -template class FrameBoundExpressionRange; -template class FrameBoundExpressionRange; -template class FrameBoundExpressionRange; -template class FrameBoundExpressionRange; - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/frameboundrange.h b/utils/windowfunction/frameboundrange.h index 3bfe04762..0a36a5434 100644 --- a/utils/windowfunction/frameboundrange.h +++ b/utils/windowfunction/frameboundrange.h @@ -17,217 +17,202 @@ // $Id: frameboundrange.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_FRAMEBOUNDRANGE_H #define UTILS_FRAMEBOUNDRANGE_H - #include "framebound.h" - namespace windowfunction { - - /** @brief class FrameBoundRange * */ class FrameBoundRange : public FrameBound { -public: - /** @brief FrameBoundRange constructor - * @param t, frame type - * @param a, order by sort spec: asc | desc - * @param n, order by sort spec: null first | null last - * @param v, order by column data type - */ - FrameBoundRange(int t = 0, bool a = true, bool n = true) - : FrameBound(t), fAsc(a), fNullFirst(n), fIsZero(false) {}; + public: + /** @brief FrameBoundRange constructor + * @param t, frame type + * @param a, order by sort spec: asc | desc + * @param n, order by sort spec: null first | null last + * @param v, order by column data type + */ + FrameBoundRange(int t = 0, bool a = true, bool n = true) + : FrameBound(t), fAsc(a), fNullFirst(n), fIsZero(false){}; - /** @brief FrameBoundRange destructor - */ - virtual ~FrameBoundRange() {}; + /** @brief FrameBoundRange destructor + */ + virtual ~FrameBoundRange(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return NULL; // abstract class - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return NULL; // abstract class + } - /** @brief virtual void getBound - */ - int64_t getBound(int64_t, int64_t, int64_t); + /** @brief virtual void getBound + */ + int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; + const std::string toString() const; - void setTupleId(std::vector ids) - { - fTupleId = ids; - } - std::vector getTupleId() const - { - return fTupleId; - } + void setTupleId(std::vector ids) + { + fTupleId = ids; + } + std::vector getTupleId() const + { + return fTupleId; + } - void setIndex(vector idx) - { - fIndex = idx; - } - const std::vector& getIndex() const - { - return fIndex; - } + void setIndex(vector idx) + { + fIndex = idx; + } + const std::vector& getIndex() const + { + return fIndex; + } - void isZero(bool z) - { - fIsZero = z; - } - bool isZero() const - { - return fIsZero; - } + void isZero(bool z) + { + fIsZero = z; + } + bool isZero() const + { + return fIsZero; + } -protected: + protected: + // for range calculation + // data, row meta data, order by column index, ascending | descending + // [0]: order by; [1]: interval; [2]: [0] +/- [1] + std::vector fTupleId; + std::vector fIndex; - // for range calculation - // data, row meta data, order by column index, ascending | descending - // [0]: order by; [1]: interval; [2]: [0] +/- [1] - std::vector fTupleId; - std::vector fIndex; - - // order by sort specification - bool fAsc; - bool fNullFirst; - - // in case expr evaluates to 0 - bool fIsZero; + // order by sort specification + bool fAsc; + bool fNullFirst; + // in case expr evaluates to 0 + bool fIsZero; }; - /** @brief struct ValueType * */ -template +template struct ValueType { - T fValue; - bool fIsNull; + T fValue; + bool fIsNull; - // constructor - ValueType() : fValue(0), fIsNull(false) {} + // constructor + ValueType() : fValue(0), fIsNull(false) + { + } }; - /** @brief class FrameBoundConstantRange * */ -template +template class FrameBoundConstantRange : public FrameBoundRange { -public: - /** @brief FrameBoundConstant constructor - * @param t, frame type - * @param a, order by sort spec: asc | desc - * @param n, order by sort spec: null first | null last - * @param c, constant value. !! caller need to check NULL or negative value !! - */ - FrameBoundConstantRange(int t = 0, bool a = true, bool n = true, void* c = NULL) - : FrameBoundRange(t, a, n) - { - fValue.fIsNull = (c == NULL); + public: + /** @brief FrameBoundConstant constructor + * @param t, frame type + * @param a, order by sort spec: asc | desc + * @param n, order by sort spec: null first | null last + * @param c, constant value. !! caller need to check NULL or negative value !! + */ + FrameBoundConstantRange(int t = 0, bool a = true, bool n = true, void* c = NULL) : FrameBoundRange(t, a, n) + { + fValue.fIsNull = (c == NULL); - if (!fValue.fIsNull) - fValue.fValue = *((T*) c); - }; + if (!fValue.fIsNull) + fValue.fValue = *((T*)c); + }; - /** @brief FrameBoundConstantRange destructor - */ - virtual ~FrameBoundConstantRange() {}; + /** @brief FrameBoundConstantRange destructor + */ + virtual ~FrameBoundConstantRange(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return new FrameBoundConstantRange(*this); - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return new FrameBoundConstantRange(*this); + } - /** @brief virtual void getBound - */ - int64_t getBound(int64_t, int64_t, int64_t); + /** @brief virtual void getBound + */ + int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; + const std::string toString() const; -protected: + protected: + // find the range offset + // i: partition upper bound + // j: current row + // k: partition lower bound + virtual int64_t getPrecedingOffset(int64_t j, int64_t i); + virtual int64_t getFollowingOffset(int64_t j, int64_t k); - // find the range offset - // i: partition upper bound - // j: current row - // k: partition lower bound - virtual int64_t getPrecedingOffset(int64_t j, int64_t i); - virtual int64_t getFollowingOffset(int64_t j, int64_t k); + // validate value is not negative + virtual void validate() + { + } - // validate value is not negative - virtual void validate() {} - - // get value at fIndex[x] - void getValue(ValueType&, int64_t); - T getValueByType(int64_t); - - // order by column value type - ValueType fValue; + // get value at fIndex[x] + void getValue(ValueType&, int64_t); + T getValueByType(int64_t); + // order by column value type + ValueType fValue; }; - /** @brief class FrameBoundExpressionRange * */ -template +template class FrameBoundExpressionRange : public FrameBoundConstantRange { -public: - /** @brief FrameBoundExpressionRange constructor - * @param t, frame type - * @param a, order by sort spec: asc | desc - * @param n, order by sort spec: null first | null last - */ - FrameBoundExpressionRange(int t = 0, bool a = true, bool n = true) : - FrameBoundConstantRange(t, a, n) {}; + public: + /** @brief FrameBoundExpressionRange constructor + * @param t, frame type + * @param a, order by sort spec: asc | desc + * @param n, order by sort spec: null first | null last + */ + FrameBoundExpressionRange(int t = 0, bool a = true, bool n = true) : FrameBoundConstantRange(t, a, n){}; - /** @brief FrameBoundExpressionRange destructor - */ - virtual ~FrameBoundExpressionRange() {}; + /** @brief FrameBoundExpressionRange destructor + */ + virtual ~FrameBoundExpressionRange(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return new FrameBoundExpressionRange(*this); - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return new FrameBoundExpressionRange(*this); + } - /** @brief virtual void getBound - */ - // int64_t getBound(int64_t, int64_t, int64_t); + /** @brief virtual void getBound + */ + // int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; + const std::string toString() const; + protected: + // virtual in FrameBoundRange + int64_t getPrecedingOffset(int64_t j, int64_t i); + int64_t getFollowingOffset(int64_t j, int64_t k); -protected: - - - // virtual in FrameBoundRange - int64_t getPrecedingOffset(int64_t j, int64_t i); - int64_t getFollowingOffset(int64_t j, int64_t k); - - // validate the expression is not negative - void validate(); - + // validate the expression is not negative + void validate(); }; - -} // namespace +} // namespace windowfunction #endif // UTILS_FRAMEBOUNDRANGE_H diff --git a/utils/windowfunction/frameboundrow.cpp b/utils/windowfunction/frameboundrow.cpp index 5fd976626..616b1e0eb 100644 --- a/utils/windowfunction/frameboundrow.cpp +++ b/utils/windowfunction/frameboundrow.cpp @@ -17,7 +17,6 @@ // $Id: frameboundrow.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -36,119 +35,108 @@ using namespace ordering; #include "treenode.h" #include "frameboundrow.h" - namespace windowfunction { - - int64_t FrameBoundRow::getBound(int64_t b, int64_t e, int64_t c) { - return c; + return c; } - const string FrameBoundRow::toString() const { - return FrameBound::toString(); + return FrameBound::toString(); } - int64_t FrameBoundConstantRow::getBound(int64_t b, int64_t e, int64_t c) { - if (fBoundType < WF__CURRENT_ROW) - { - if (fOffset <= (c - b)) - c -= fOffset; - else - c = b - (!fStart ? 1 : 0); - } + if (fBoundType < WF__CURRENT_ROW) + { + if (fOffset <= (c - b)) + c -= fOffset; else - { - if (fOffset <= (e - c)) - c += fOffset; - else - c = e + (fStart ? 1 : 0); - } + c = b - (!fStart ? 1 : 0); + } + else + { + if (fOffset <= (e - c)) + c += fOffset; + else + c = e + (fStart ? 1 : 0); + } - return c; + return c; } - const string FrameBoundConstantRow::toString() const { - ostringstream oss; - oss << fOffset << " " << FrameBound::toString(); - return oss.str(); + ostringstream oss; + oss << fOffset << " " << FrameBound::toString(); + return oss.str(); } - -template +template int64_t FrameBoundExpressionRow::getBound(int64_t b, int64_t e, int64_t c) { - // set row data - // get expression int value - fRow.setData(getPointer(fRowData->at(c))); + // set row data + // get expression int value + fRow.setData(getPointer(fRowData->at(c))); - if (fRow.isNullValue(fExprIdx)) - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, "NULL"), - ERR_WF_BOUND_OUT_OF_RANGE); + if (fRow.isNullValue(fExprIdx)) + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, "NULL"), + ERR_WF_BOUND_OUT_OF_RANGE); - getOffset(); + getOffset(); - if (fOffset < 0) - { - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, fOffset), - ERR_WF_BOUND_OUT_OF_RANGE); - } + if (fOffset < 0) + { + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_BOUND_OUT_OF_RANGE, fOffset), + ERR_WF_BOUND_OUT_OF_RANGE); + } - return FrameBoundConstantRow::getBound(b, e, c); + return FrameBoundConstantRow::getBound(b, e, c); } - -template +template const string FrameBoundExpressionRow::toString() const { - ostringstream oss; - oss << "value_expr " << FrameBound::toString(); - return oss.str(); + ostringstream oss; + oss << "value_expr " << FrameBound::toString(); + return oss.str(); } - -template void FrameBoundExpressionRow::getOffset() +template +void FrameBoundExpressionRow::getOffset() { } - -template<> void FrameBoundExpressionRow::getOffset() +template <> +void FrameBoundExpressionRow::getOffset() { - fOffset = fRow.getIntField(fExprIdx); + fOffset = fRow.getIntField(fExprIdx); } - -template<> void FrameBoundExpressionRow::getOffset() +template <> +void FrameBoundExpressionRow::getOffset() { - fOffset = fRow.getUintField(fExprIdx); + fOffset = fRow.getUintField(fExprIdx); } - -template<> void FrameBoundExpressionRow::getOffset() +template <> +void FrameBoundExpressionRow::getOffset() { - fOffset = (int64_t) fRow.getDoubleField(fExprIdx); + fOffset = (int64_t)fRow.getDoubleField(fExprIdx); } - -template<> void FrameBoundExpressionRow::getOffset() +template <> +void FrameBoundExpressionRow::getOffset() { - fOffset = (int64_t) fRow.getFloatField(fExprIdx); + fOffset = (int64_t)fRow.getFloatField(fExprIdx); } - template class FrameBoundExpressionRow; template class FrameBoundExpressionRow; template class FrameBoundExpressionRow; template class FrameBoundExpressionRow; - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/frameboundrow.h b/utils/windowfunction/frameboundrow.h index 30aebf947..ac0b6c0a3 100644 --- a/utils/windowfunction/frameboundrow.h +++ b/utils/windowfunction/frameboundrow.h @@ -17,153 +17,135 @@ // $Id: frameboundrow.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_FRAMEBOUNDROW_H #define UTILS_FRAMEBOUNDROW_H - #include "framebound.h" - namespace windowfunction { - - /** @brief class FrameBoundRow * */ class FrameBoundRow : public FrameBound { -public: - /** @brief FrameBoundRow constructor - * @param t, frame type - */ - FrameBoundRow(int t = 0) : FrameBound(t) {}; + public: + /** @brief FrameBoundRow constructor + * @param t, frame type + */ + FrameBoundRow(int t = 0) : FrameBound(t){}; - /** @brief FrameBoundRow destructor - */ - virtual ~FrameBoundRow() {}; + /** @brief FrameBoundRow destructor + */ + virtual ~FrameBoundRow(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return new FrameBoundRow(*this); - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return new FrameBoundRow(*this); + } - /** @brief virtual void getBound - */ - int64_t getBound(int64_t, int64_t, int64_t); + /** @brief virtual void getBound + */ + int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; - - -protected: + const std::string toString() const; + protected: }; - /** @brief class FrameBoundConstantRow * */ class FrameBoundConstantRow : public FrameBoundRow { -public: - /** @brief FrameBoundConstant constructor - * @param t, frame type - * @param c, constant value. !! caller need to check NULL or negative value !! - */ - FrameBoundConstantRow(int t = 0, int c = 0) : FrameBoundRow(t), fOffset(c) {}; + public: + /** @brief FrameBoundConstant constructor + * @param t, frame type + * @param c, constant value. !! caller need to check NULL or negative value !! + */ + FrameBoundConstantRow(int t = 0, int c = 0) : FrameBoundRow(t), fOffset(c){}; - /** @brief FrameBoundConstantRow destructor - */ - virtual ~FrameBoundConstantRow() {}; + /** @brief FrameBoundConstantRow destructor + */ + virtual ~FrameBoundConstantRow(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return new FrameBoundConstantRow(*this); - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return new FrameBoundConstantRow(*this); + } - /** @brief virtual void getBound - */ - int64_t getBound(int64_t, int64_t, int64_t); + /** @brief virtual void getBound + */ + int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; - - -protected: - - // constant offset - int64_t fOffset; + const std::string toString() const; + protected: + // constant offset + int64_t fOffset; }; - /** @brief class FrameBoundExpressionRow * */ -template +template class FrameBoundExpressionRow : public FrameBoundConstantRow { -public: - /** @brief FrameBoundExpressionRow constructor - * @param t, frame type - */ - FrameBoundExpressionRow(int t, uint64_t id = -1, int idx = -1) : - FrameBoundConstantRow(t), fExprTupleId(id), fExprIdx(idx) {}; + public: + /** @brief FrameBoundExpressionRow constructor + * @param t, frame type + */ + FrameBoundExpressionRow(int t, uint64_t id = -1, int idx = -1) + : FrameBoundConstantRow(t), fExprTupleId(id), fExprIdx(idx){}; - /** @brief FrameBoundExpressionRow destructor - */ - virtual ~FrameBoundExpressionRow() {}; + /** @brief FrameBoundExpressionRow destructor + */ + virtual ~FrameBoundExpressionRow(){}; - /** @brief clone - */ - virtual FrameBound* clone() - { - return new FrameBoundExpressionRow(*this); - } + /** @brief clone + */ + virtual FrameBound* clone() + { + return new FrameBoundExpressionRow(*this); + } - /** @brief virtual void getBound - */ - int64_t getBound(int64_t, int64_t, int64_t); + /** @brief virtual void getBound + */ + int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; + const std::string toString() const; - void setExprTupleId(int id) - { - fExprTupleId = id; - } - uint64_t getExprTupleId() const - { - return fExprTupleId; - } + void setExprTupleId(int id) + { + fExprTupleId = id; + } + uint64_t getExprTupleId() const + { + return fExprTupleId; + } - void setExprIndex(int i) - { - fExprIdx = i; - } - uint64_t getExprIndex() const - { - return fExprIdx; - } + void setExprIndex(int i) + { + fExprIdx = i; + } + uint64_t getExprIndex() const + { + return fExprIdx; + } + protected: + void getOffset(); -protected: - - void getOffset(); - - // id and index in row - uint64_t fExprTupleId; - int fExprIdx; - + // id and index in row + uint64_t fExprTupleId; + int fExprIdx; }; - - - -} // namespace +} // namespace windowfunction #endif // UTILS_FRAMEBOUNDROW_H diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index e5557ae77..30d442e74 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -49,877 +49,853 @@ using namespace rowgroup; #include "joblisttypes.h" #include "mcs_decimal.h" -// See agg_arg_charsets in sql_type.h to see conversion rules for +// See agg_arg_charsets in sql_type.h to see conversion rules for // items that have different char sets namespace ordering { int TinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - int8_t v1 = l->row1().getIntField(fSpec.fIndex); - int8_t v2 = l->row2().getIntField(fSpec.fIndex); - int8_t nullValue = static_cast(joblist::TINYINTNULL); + int ret = 0; + int8_t v1 = l->row1().getIntField(fSpec.fIndex); + int8_t v2 = l->row2().getIntField(fSpec.fIndex); + int8_t nullValue = static_cast(joblist::TINYINTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int SmallIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - int16_t v1 = l->row1().getIntField(fSpec.fIndex); - int16_t v2 = l->row2().getIntField(fSpec.fIndex); - int16_t nullValue = static_cast(joblist::SMALLINTNULL); + int ret = 0; + int16_t v1 = l->row1().getIntField(fSpec.fIndex); + int16_t v2 = l->row2().getIntField(fSpec.fIndex); + int16_t nullValue = static_cast(joblist::SMALLINTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int IntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - int32_t v1 = l->row1().getIntField(fSpec.fIndex); - int32_t v2 = l->row2().getIntField(fSpec.fIndex); - int32_t nullValue = static_cast(joblist::INTNULL); + int ret = 0; + int32_t v1 = l->row1().getIntField(fSpec.fIndex); + int32_t v2 = l->row2().getIntField(fSpec.fIndex); + int32_t nullValue = static_cast(joblist::INTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int BigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - int64_t v1 = l->row1().getIntField(fSpec.fIndex); - int64_t v2 = l->row2().getIntField(fSpec.fIndex); - int64_t nullValue = static_cast(joblist::BIGINTNULL); + int ret = 0; + int64_t v1 = l->row1().getIntField(fSpec.fIndex); + int64_t v2 = l->row2().getIntField(fSpec.fIndex); + int64_t nullValue = static_cast(joblist::BIGINTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int WideDecimalCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; + int ret = 0; - int128_t v1, v2; - l->row1().getInt128Field(fSpec.fIndex, v1); - l->row2().getInt128Field(fSpec.fIndex, v2); + int128_t v1, v2; + l->row1().getInt128Field(fSpec.fIndex, v1); + l->row2().getInt128Field(fSpec.fIndex, v2); - bool v1IsNull = v1 == datatypes::Decimal128Null; - bool v2IsNull = v2 == datatypes::Decimal128Null; + bool v1IsNull = v1 == datatypes::Decimal128Null; + bool v2IsNull = v2 == datatypes::Decimal128Null; - if (v1IsNull || v2IsNull) - { - if (!v1IsNull && v2IsNull) - ret = fSpec.fNf; - else if (v1IsNull && !v2IsNull) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1IsNull || v2IsNull) + { + if (!v1IsNull && v2IsNull) + ret = fSpec.fNf; + else if (v1IsNull && !v2IsNull) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int UTinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - uint8_t v1 = l->row1().getUintField(fSpec.fIndex); - uint8_t v2 = l->row2().getUintField(fSpec.fIndex); - uint8_t nullValue = static_cast(joblist::UTINYINTNULL); + int ret = 0; + uint8_t v1 = l->row1().getUintField(fSpec.fIndex); + uint8_t v2 = l->row2().getUintField(fSpec.fIndex); + uint8_t nullValue = static_cast(joblist::UTINYINTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int USmallIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - uint16_t v1 = l->row1().getUintField(fSpec.fIndex); - uint16_t v2 = l->row2().getUintField(fSpec.fIndex); - uint16_t nullValue = static_cast(joblist::USMALLINTNULL); + int ret = 0; + uint16_t v1 = l->row1().getUintField(fSpec.fIndex); + uint16_t v2 = l->row2().getUintField(fSpec.fIndex); + uint16_t nullValue = static_cast(joblist::USMALLINTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int UIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - uint32_t v1 = l->row1().getUintField(fSpec.fIndex); - uint32_t v2 = l->row2().getUintField(fSpec.fIndex); - uint32_t nullValue = static_cast(joblist::UINTNULL); + int ret = 0; + uint32_t v1 = l->row1().getUintField(fSpec.fIndex); + uint32_t v2 = l->row2().getUintField(fSpec.fIndex); + uint32_t nullValue = static_cast(joblist::UINTNULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int UBigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - uint64_t v1 = l->row1().getUintField(fSpec.fIndex); - uint64_t v2 = l->row2().getUintField(fSpec.fIndex); + int ret = 0; + uint64_t v1 = l->row1().getUintField(fSpec.fIndex); + uint64_t v2 = l->row2().getUintField(fSpec.fIndex); - if (v1 == joblist::UBIGINTNULL || v2 == joblist::UBIGINTNULL) - { - if (v1 != joblist::UBIGINTNULL && v2 == joblist::UBIGINTNULL) - ret = fSpec.fNf; - else if (v1 == joblist::UBIGINTNULL && v2 != joblist::UBIGINTNULL) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == joblist::UBIGINTNULL || v2 == joblist::UBIGINTNULL) + { + if (v1 != joblist::UBIGINTNULL && v2 == joblist::UBIGINTNULL) + ret = fSpec.fNf; + else if (v1 == joblist::UBIGINTNULL && v2 != joblist::UBIGINTNULL) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int StringCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - bool b1 = l->row1().isNullValue(fSpec.fIndex); - bool b2 = l->row2().isNullValue(fSpec.fIndex); + bool b1 = l->row1().isNullValue(fSpec.fIndex); + bool b2 = l->row2().isNullValue(fSpec.fIndex); - int ret = 0; + int ret = 0; - if (b1 == true || b2 == true) - { - if (b1 == false && b2 == true) - ret = fSpec.fNf; - else if (b1 == true && b2 == false) - ret = -fSpec.fNf; - } - else - { - auto const str1 = l->row1().getConstString(fSpec.fIndex); - auto const str2 = l->row2().getConstString(fSpec.fIndex); + if (b1 == true || b2 == true) + { + if (b1 == false && b2 == true) + ret = fSpec.fNf; + else if (b1 == true && b2 == false) + ret = -fSpec.fNf; + } + else + { + auto const str1 = l->row1().getConstString(fSpec.fIndex); + auto const str2 = l->row2().getConstString(fSpec.fIndex); - if (!cs) - cs = l->rowGroup()->getCharset(fSpec.fIndex); - - ret = fSpec.fAsc * cs->strnncollsp(str1.str(), str1.length(), - str2.str(), str2.length()); - } + if (!cs) + cs = l->rowGroup()->getCharset(fSpec.fIndex); - return ret; + ret = fSpec.fAsc * cs->strnncollsp(str1.str(), str1.length(), str2.str(), str2.length()); + } + + return ret; } int DoubleCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - uint64_t uiv1 = l->row1().getUintField(fSpec.fIndex); - uint64_t uiv2 = l->row2().getUintField(fSpec.fIndex); - int ret = 0; + uint64_t uiv1 = l->row1().getUintField(fSpec.fIndex); + uint64_t uiv2 = l->row2().getUintField(fSpec.fIndex); + int ret = 0; - if (uiv1 == joblist::DOUBLENULL || uiv2 == joblist::DOUBLENULL) - { - if (uiv1 != joblist::DOUBLENULL && uiv2 == joblist::DOUBLENULL) - ret = fSpec.fNf; - else if (uiv1 == joblist::DOUBLENULL && uiv2 != joblist::DOUBLENULL) - ret = -fSpec.fNf; - } - else - { - double v1 = l->row1().getDoubleField(fSpec.fIndex); - double v2 = l->row2().getDoubleField(fSpec.fIndex); - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (uiv1 == joblist::DOUBLENULL || uiv2 == joblist::DOUBLENULL) + { + if (uiv1 != joblist::DOUBLENULL && uiv2 == joblist::DOUBLENULL) + ret = fSpec.fNf; + else if (uiv1 == joblist::DOUBLENULL && uiv2 != joblist::DOUBLENULL) + ret = -fSpec.fNf; + } + else + { + double v1 = l->row1().getDoubleField(fSpec.fIndex); + double v2 = l->row2().getDoubleField(fSpec.fIndex); + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int FloatCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int32_t iv1 = l->row1().getIntField(fSpec.fIndex); - int32_t iv2 = l->row2().getIntField(fSpec.fIndex); - int32_t nullValue = static_cast(joblist::FLOATNULL); - int ret = 0; + int32_t iv1 = l->row1().getIntField(fSpec.fIndex); + int32_t iv2 = l->row2().getIntField(fSpec.fIndex); + int32_t nullValue = static_cast(joblist::FLOATNULL); + int ret = 0; - if (iv1 == nullValue || iv2 == nullValue) - { - if (iv1 != nullValue && iv2 == nullValue) - ret = fSpec.fNf; - else if (iv1 == nullValue && iv2 != nullValue) - ret = -fSpec.fNf; - } - else - { - float v1 = l->row1().getFloatField(fSpec.fIndex); - float v2 = l->row2().getFloatField(fSpec.fIndex); - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (iv1 == nullValue || iv2 == nullValue) + { + if (iv1 != nullValue && iv2 == nullValue) + ret = fSpec.fNf; + else if (iv1 == nullValue && iv2 != nullValue) + ret = -fSpec.fNf; + } + else + { + float v1 = l->row1().getFloatField(fSpec.fIndex); + float v2 = l->row2().getFloatField(fSpec.fIndex); + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int LongDoubleCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - long double v1 = l->row1().getLongDoubleField(fSpec.fIndex); - long double v2 = l->row2().getLongDoubleField(fSpec.fIndex); + long double v1 = l->row1().getLongDoubleField(fSpec.fIndex); + long double v2 = l->row2().getLongDoubleField(fSpec.fIndex); - int ret = 0; + int ret = 0; - if (v1 == joblist::LONGDOUBLENULL || v2 == joblist::LONGDOUBLENULL) - { - if (v1 != joblist::LONGDOUBLENULL && v2 == joblist::LONGDOUBLENULL) - ret = fSpec.fNf; - else if (v1 == joblist::LONGDOUBLENULL && v2 != joblist::LONGDOUBLENULL) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == joblist::LONGDOUBLENULL || v2 == joblist::LONGDOUBLENULL) + { + if (v1 != joblist::LONGDOUBLENULL && v2 == joblist::LONGDOUBLENULL) + ret = fSpec.fNf; + else if (v1 == joblist::LONGDOUBLENULL && v2 != joblist::LONGDOUBLENULL) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int DateCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - uint32_t v1 = l->row1().getUintField(fSpec.fIndex); - uint32_t v2 = l->row2().getUintField(fSpec.fIndex); - uint32_t nullValue = static_cast(joblist::DATENULL); + int ret = 0; + uint32_t v1 = l->row1().getUintField(fSpec.fIndex); + uint32_t v2 = l->row2().getUintField(fSpec.fIndex); + uint32_t nullValue = static_cast(joblist::DATENULL); - if (v1 == nullValue || v2 == nullValue) - { - if (v1 != nullValue && v2 == nullValue) - ret = fSpec.fNf; - else if (v1 == nullValue && v2 != nullValue) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == nullValue || v2 == nullValue) + { + if (v1 != nullValue && v2 == nullValue) + ret = fSpec.fNf; + else if (v1 == nullValue && v2 != nullValue) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int DatetimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int ret = 0; - uint64_t v1 = l->row1().getUintField(fSpec.fIndex); - uint64_t v2 = l->row2().getUintField(fSpec.fIndex); + int ret = 0; + uint64_t v1 = l->row1().getUintField(fSpec.fIndex); + uint64_t v2 = l->row2().getUintField(fSpec.fIndex); - if (v1 == joblist::DATETIMENULL || v2 == joblist::DATETIMENULL) - { - if (v1 != joblist::DATETIMENULL && v2 == joblist::DATETIMENULL) - ret = fSpec.fNf; - else if (v1 == joblist::DATETIMENULL && v2 != joblist::DATETIMENULL) - ret = -fSpec.fNf; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 == joblist::DATETIMENULL || v2 == joblist::DATETIMENULL) + { + if (v1 != joblist::DATETIMENULL && v2 == joblist::DATETIMENULL) + ret = fSpec.fNf; + else if (v1 == joblist::DATETIMENULL && v2 != joblist::DATETIMENULL) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } - return ret; + return ret; } int TimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { - l->row1().setData(r1); - l->row2().setData(r2); + l->row1().setData(r1); + l->row2().setData(r2); - int64_t v1 = l->row1().getIntField(fSpec.fIndex); - int64_t v2 = l->row2().getIntField(fSpec.fIndex); + int64_t v1 = l->row1().getIntField(fSpec.fIndex); + int64_t v2 = l->row2().getIntField(fSpec.fIndex); - bool b1 = (joblist::TIMENULL == (uint64_t) v1); - bool b2 = (joblist::TIMENULL == (uint64_t) v2); + bool b1 = (joblist::TIMENULL == (uint64_t)v1); + bool b2 = (joblist::TIMENULL == (uint64_t)v2); - int ret = 0; + int ret = 0; - if (b1 == true || b2 == true) + if (b1 == true || b2 == true) + { + if (b1 == false && b2 == true) + ret = fSpec.fNf; + else if (b1 == true && b2 == false) + ret = -fSpec.fNf; + } + else + { + // ((int64_t) -00:00:26) > ((int64_t) -00:00:25) + // i.e. For 2 negative TIME values, we invert the order of + // comparison operations to force "-00:00:26" to appear before + // "-00:00:25" in ascending order. + if (v1 < 0 && v2 < 0) { - if (b1 == false && b2 == true) - ret = fSpec.fNf; - else if (b1 == true && b2 == false) - ret = -fSpec.fNf; + // Unset the MSB. + v1 &= ~(1ULL << 63); + v2 &= ~(1ULL << 63); + if (v1 < v2) + ret = fSpec.fAsc; + else if (v1 > v2) + ret = -fSpec.fAsc; } else { - // ((int64_t) -00:00:26) > ((int64_t) -00:00:25) - // i.e. For 2 negative TIME values, we invert the order of - // comparison operations to force "-00:00:26" to appear before - // "-00:00:25" in ascending order. - if (v1 < 0 && v2 < 0) - { - // Unset the MSB. - v1 &= ~(1ULL << 63); - v2 &= ~(1ULL << 63); - if (v1 < v2) - ret = fSpec.fAsc; - else if (v1 > v2) - ret = -fSpec.fAsc; - } - else - { - if (v1 > v2) - ret = fSpec.fAsc; - else if (v1 < v2) - ret = -fSpec.fAsc; - } + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; } + } - return ret; + return ret; } - bool CompareRule::less(Row::Pointer r1, Row::Pointer r2) { - for (vector::iterator i = fCompares.begin(); i != fCompares.end(); i++) - { - int c = ((*(*i))(fIdbCompare, r1, r2)); + for (vector::iterator i = fCompares.begin(); i != fCompares.end(); i++) + { + int c = ((*(*i))(fIdbCompare, r1, r2)); - if (c < 0) - return true; - else if (c > 0) - return false; - } + if (c < 0) + return true; + else if (c > 0) + return false; + } - return false; + return false; } void CompareRule::revertRules() { - std::vector::iterator fCompareIter = fCompares.begin(); - for(; fCompareIter!=fCompares.end(); fCompareIter++) - { - (*fCompareIter)->revertSortSpec(); - } + std::vector::iterator fCompareIter = fCompares.begin(); + for (; fCompareIter != fCompares.end(); fCompareIter++) + { + (*fCompareIter)->revertSortSpec(); + } } - - void CompareRule::compileRules(const std::vector& spec, const rowgroup::RowGroup& rg) { - const vector& types = rg.getColTypes(); - const auto& offsets = rg.getOffsets(); + const vector& types = rg.getColTypes(); + const auto& offsets = rg.getOffsets(); - for (vector::const_iterator i = spec.begin(); i != spec.end(); i++) + for (vector::const_iterator i = spec.begin(); i != spec.end(); i++) + { + switch (types[i->fIndex]) { - switch (types[i->fIndex]) + case CalpontSystemCatalog::TINYINT: + { + Compare* c = new TinyIntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::SMALLINT: + { + Compare* c = new SmallIntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + { + Compare* c = new IntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::BIGINT: + { + Compare* c = new BigIntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + Compare* c; + uint32_t len = rg.getColumnWidth(i->fIndex); + switch (len) { - case CalpontSystemCatalog::TINYINT: - { - Compare* c = new TinyIntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::SMALLINT: - { - Compare* c = new SmallIntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - { - Compare* c = new IntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::BIGINT: - { - Compare* c = new BigIntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - Compare* c; - uint32_t len = rg.getColumnWidth(i->fIndex); - switch (len) - { - case datatypes::MAXDECIMALWIDTH: - c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; - case datatypes::MAXLEGACYWIDTH: - c = new BigIntCompare(*i); break; - case 1 : - c = new TinyIntCompare(*i); break; - case 2 : - c = new SmallIntCompare(*i); break; - case 4 : - c = new IntCompare(*i); break; - } - - fCompares.push_back(c); - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - Compare* c = new UTinyIntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::USMALLINT: - { - Compare* c = new USmallIntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - { - Compare* c = new UIntCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::UBIGINT: - { - Compare* c = new UBigIntCompare(*i); - fCompares.push_back(c); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - Compare* c = new StringCompare(*i); - fCompares.push_back(c); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - Compare* c = new DoubleCompare(*i); - fCompares.push_back(c); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - Compare* c = new FloatCompare(*i); - fCompares.push_back(c); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - Compare* c = new LongDoubleCompare(*i); - fCompares.push_back(c); - break; - } - - case CalpontSystemCatalog::DATE: - { - Compare* c = new DateCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - { - Compare* c = new DatetimeCompare(*i); - fCompares.push_back(c); - break; - } - case CalpontSystemCatalog::TIME: - { - Compare* c = new TimeCompare(*i); - fCompares.push_back(c); - break; - } - - default: - { - break; - } + case datatypes::MAXDECIMALWIDTH: c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; + case datatypes::MAXLEGACYWIDTH: c = new BigIntCompare(*i); break; + case 1: c = new TinyIntCompare(*i); break; + case 2: c = new SmallIntCompare(*i); break; + case 4: c = new IntCompare(*i); break; } - } -} + fCompares.push_back(c); + break; + } + + case CalpontSystemCatalog::UTINYINT: + { + Compare* c = new UTinyIntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::USMALLINT: + { + Compare* c = new USmallIntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + { + Compare* c = new UIntCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::UBIGINT: + { + Compare* c = new UBigIntCompare(*i); + fCompares.push_back(c); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::TEXT: + { + Compare* c = new StringCompare(*i); + fCompares.push_back(c); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + Compare* c = new DoubleCompare(*i); + fCompares.push_back(c); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + Compare* c = new FloatCompare(*i); + fCompares.push_back(c); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + Compare* c = new LongDoubleCompare(*i); + fCompares.push_back(c); + break; + } + + case CalpontSystemCatalog::DATE: + { + Compare* c = new DateCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + { + Compare* c = new DatetimeCompare(*i); + fCompares.push_back(c); + break; + } + case CalpontSystemCatalog::TIME: + { + Compare* c = new TimeCompare(*i); + fCompares.push_back(c); + break; + } + + default: + { + break; + } + } + } +} void IdbCompare::initialize(const RowGroup& rg) { - fRowGroup = rg; - fRowGroup.initRow(&fRow1); - fRowGroup.initRow(&fRow2); + fRowGroup = rg; + fRowGroup.initRow(&fRow1); + fRowGroup.initRow(&fRow2); } - void IdbCompare::setStringTable(bool b) { - fRowGroup.setUseStringTable(b); - fRowGroup.initRow(&fRow1); - fRowGroup.initRow(&fRow2); + fRowGroup.setUseStringTable(b); + fRowGroup.initRow(&fRow1); + fRowGroup.initRow(&fRow2); } - // OrderByData class implementation OrderByData::OrderByData(const std::vector& spec, const rowgroup::RowGroup& rg) { - IdbCompare::initialize(rg); - fRule.compileRules(spec, rg); - fRule.fIdbCompare = this; + IdbCompare::initialize(rg); + fRule.compileRules(spec, rg); + fRule.fIdbCompare = this; } - // OrderByData class dtor OrderByData::~OrderByData() { - // delete compare objects - vector::iterator i = fRule.fCompares.begin(); + // delete compare objects + vector::iterator i = fRule.fCompares.begin(); - while (i != fRule.fCompares.end()) + while (i != fRule.fCompares.end()) + { + if (*i) { - if (*i) - { - delete *i; - *i = NULL; - } - i++; + delete *i; + *i = NULL; } + i++; + } } - // IdbOrderBy class implementation -IdbOrderBy::IdbOrderBy() : - fDistinct(false), fMemSize(0), fRowsPerRG(rowgroup::rgCommonSize), - fErrorCode(0), - fRm(NULL) -{ } - +IdbOrderBy::IdbOrderBy() + : fDistinct(false), fMemSize(0), fRowsPerRG(rowgroup::rgCommonSize), fErrorCode(0), fRm(NULL) +{ +} IdbOrderBy::~IdbOrderBy() { - if (fRm) - fRm->returnMemory(fMemSize, fSessionMemLimit); + if (fRm) + fRm->returnMemory(fMemSize, fSessionMemLimit); - // delete compare objects - vector::iterator i = fRule.fCompares.begin(); + // delete compare objects + vector::iterator i = fRule.fCompares.begin(); - while (i != fRule.fCompares.end()) - delete *i++; + while (i != fRule.fCompares.end()) + delete *i++; } - void IdbOrderBy::initialize(const RowGroup& rg) { - // initialize rows - IdbCompare::initialize(rg); + // initialize rows + IdbCompare::initialize(rg); - uint64_t newSize = rg.getSizeWithStrings(fRowsPerRG); - if (fRm && !fRm->getMemory(newSize, fSessionMemLimit)) - { - cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(fErrorCode); - } - fMemSize += newSize; - fData.reinit(fRowGroup, fRowsPerRG); - fRowGroup.setData(&fData); - fRowGroup.resetRowGroup(0); - fRowGroup.initRow(&fRow0); - fRowGroup.getRow(0, &fRow0); + uint64_t newSize = rg.getSizeWithStrings(fRowsPerRG); + if (fRm && !fRm->getMemory(newSize, fSessionMemLimit)) + { + cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(fErrorCode); + } + fMemSize += newSize; + fData.reinit(fRowGroup, fRowsPerRG); + fRowGroup.setData(&fData); + fRowGroup.resetRowGroup(0); + fRowGroup.initRow(&fRow0); + fRowGroup.getRow(0, &fRow0); - // set compare functors - fRule.compileRules(fOrderByCond, fRowGroup); + // set compare functors + fRule.compileRules(fOrderByCond, fRowGroup); - fRowGroup.initRow(&row1); - fRowGroup.initRow(&row2); + fRowGroup.initRow(&row1); + fRowGroup.initRow(&row2); - if (fDistinct) - { - fDistinctMap.reset(new DistinctMap_t(10, Hasher(this, getKeyLength()), Eq(this, getKeyLength()))); - } + if (fDistinct) + { + fDistinctMap.reset(new DistinctMap_t(10, Hasher(this, getKeyLength()), Eq(this, getKeyLength()))); + } } - bool IdbOrderBy::getData(RGData& data) { - if (fDataQueue.empty()) - return false; + if (fDataQueue.empty()) + return false; - data = fDataQueue.front(); - fDataQueue.pop(); + data = fDataQueue.front(); + fDataQueue.pop(); - return true; + return true; } - bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) { - bool eq = true; - fRow1.setData(a); - fRow2.setData(b); + bool eq = true; + fRow1.setData(a); + fRow2.setData(b); - for (vector::const_iterator i = fIndex.begin(); i != fIndex.end() && eq; i++) + for (vector::const_iterator i = fIndex.begin(); i != fIndex.end() && eq; i++) + { + CalpontSystemCatalog::ColDataType type = fRow1.getColType(*i); + + switch (type) { - CalpontSystemCatalog::ColDataType type = fRow1.getColType(*i); + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + { + // equal compare. ignore sign and null + eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); + break; + } - switch (type) + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + // equal compare. ignore sign and null + if (UNLIKELY(fRow1.getColumnWidth(*i) < datatypes::MAXDECIMALWIDTH)) { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - { - // equal compare. ignore sign and null - eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - // equal compare. ignore sign and null - if (UNLIKELY(fRow1.getColumnWidth(*i) < datatypes::MAXDECIMALWIDTH)) - { - eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); - } - else if (fRow1.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH) - { - eq = (*fRow1.getBinaryField(*i) == - *fRow2.getBinaryField(*i)); - } - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - eq = (fRow1.getStringField(*i) == fRow2.getStringField(*i)); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - eq = (fRow1.getDoubleField(*i) == fRow2.getDoubleField(*i)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - eq = (fRow1.getFloatField(*i) == fRow2.getFloatField(*i)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - eq = (fRow1.getLongDoubleField(*i) == fRow2.getLongDoubleField(*i)); - break; - } - - default: - { - eq = false; - uint64_t ec = ERR_WF_UNKNOWN_COL_TYPE; - cerr << IDBErrorInfo::instance()->errorMsg(ec, type) - << " @" << __FILE__ << ":" << __LINE__; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ec, type), ec); - break; - } + eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); } - } + else if (fRow1.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH) + { + eq = (*fRow1.getBinaryField(*i) == *fRow2.getBinaryField(*i)); + } + break; + } - return eq; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + { + eq = (fRow1.getStringField(*i) == fRow2.getStringField(*i)); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + eq = (fRow1.getDoubleField(*i) == fRow2.getDoubleField(*i)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + eq = (fRow1.getFloatField(*i) == fRow2.getFloatField(*i)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + eq = (fRow1.getLongDoubleField(*i) == fRow2.getLongDoubleField(*i)); + break; + } + + default: + { + eq = false; + uint64_t ec = ERR_WF_UNKNOWN_COL_TYPE; + cerr << IDBErrorInfo::instance()->errorMsg(ec, type) << " @" << __FILE__ << ":" << __LINE__; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ec, type), ec); + break; + } + } + } + + return eq; } uint64_t IdbOrderBy::Hasher::operator()(const Row::Pointer& p) const { - Row& row = ts->row1; - row.setPointer(p); - // MCOL-1829 Row::hash uses colcount - 1 as an array idx down a callstack. - uint64_t ret = row.hash(colCount - 1); - return ret; + Row& row = ts->row1; + row.setPointer(p); + // MCOL-1829 Row::hash uses colcount - 1 as an array idx down a callstack. + uint64_t ret = row.hash(colCount - 1); + return ret; } bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) const { - Row& r1 = ts->row1, &r2 = ts->row2; - r1.setPointer(d1); - r2.setPointer(d2); - // MCOL-1829 Row::equals uses 2nd argument as key columns container size boundary - bool ret = r1.equals(r2, colCount - 1); + Row &r1 = ts->row1, &r2 = ts->row2; + r1.setPointer(d1); + r2.setPointer(d2); + // MCOL-1829 Row::equals uses 2nd argument as key columns container size boundary + bool ret = r1.equals(r2, colCount - 1); - return ret; + return ret; } - -} +} // namespace ordering // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/idborderby.h b/utils/windowfunction/idborderby.h index 30f49c1de..92ed9bbeb 100644 --- a/utils/windowfunction/idborderby.h +++ b/utils/windowfunction/idborderby.h @@ -18,7 +18,6 @@ // $Id: idborderby.h 4012 2013-07-24 21:04:45Z pleblanc $ - /** @file */ #ifndef IDB_ORDER_BY_H @@ -44,31 +43,38 @@ // forward reference namespace joblist { -class ResourceManager; +class ResourceManager; } - namespace ordering { - -template, - typename _Compare = std::less > -class reservablePQ: private std::priority_queue<_Tp, _Sequence, _Compare> +template , + typename _Compare = std::less > +class reservablePQ : private std::priority_queue<_Tp, _Sequence, _Compare> { -public: - typedef typename std::priority_queue<_Tp, _Sequence, _Compare>::size_type size_type; - reservablePQ(size_type capacity = 0) { reserve(capacity); }; - void reserve(size_type capacity) { this->c.reserve(capacity); } - size_type capacity() const { return this->c.capacity(); } - using std::priority_queue<_Tp, _Sequence, _Compare>::size; - using std::priority_queue<_Tp, _Sequence, _Compare>::top; - using std::priority_queue<_Tp, _Sequence, _Compare>::pop; - using std::priority_queue<_Tp, _Sequence, _Compare>::push; - using std::priority_queue<_Tp, _Sequence, _Compare>::empty; + public: + typedef typename std::priority_queue<_Tp, _Sequence, _Compare>::size_type size_type; + reservablePQ(size_type capacity = 0) + { + reserve(capacity); + }; + void reserve(size_type capacity) + { + this->c.reserve(capacity); + } + size_type capacity() const + { + return this->c.capacity(); + } + using std::priority_queue<_Tp, _Sequence, _Compare>::size; + using std::priority_queue<_Tp, _Sequence, _Compare>::top; + using std::priority_queue<_Tp, _Sequence, _Compare>::pop; + using std::priority_queue<_Tp, _Sequence, _Compare>::push; + using std::priority_queue<_Tp, _Sequence, _Compare>::empty; }; // forward reference -class IdbCompare; +class IdbCompare; class OrderByRow; typedef reservablePQ SortingPQ; @@ -76,80 +82,97 @@ typedef reservablePQ SortingPQ; // order by specification struct IdbSortSpec { - int fIndex; - // TODO There are three ordering specs since 10.2 - int fAsc; // ::= ASC | DESC - int fNf; // ::= NULLS FIRST | NULLS LAST + int fIndex; + // TODO There are three ordering specs since 10.2 + int fAsc; // ::= ASC | DESC + int fNf; // ::= NULLS FIRST | NULLS LAST - IdbSortSpec() : fIndex(-1), fAsc(1), fNf(1) {} - IdbSortSpec(int i, bool b) : fIndex(i), fAsc(b ? 1 : -1), fNf(fAsc) {} - IdbSortSpec(int i, bool b, bool n) : fIndex(i), fAsc(b ? 1 : -1), fNf(n ? 1 : -1) {} + IdbSortSpec() : fIndex(-1), fAsc(1), fNf(1) + { + } + IdbSortSpec(int i, bool b) : fIndex(i), fAsc(b ? 1 : -1), fNf(fAsc) + { + } + IdbSortSpec(int i, bool b, bool n) : fIndex(i), fAsc(b ? 1 : -1), fNf(n ? 1 : -1) + { + } }; - // compare functor for different datatypes // cannot use template because Row's getXxxField method. class Compare { -public: - Compare(const IdbSortSpec& spec) : fSpec(spec) {} - virtual ~Compare() {} + public: + Compare(const IdbSortSpec& spec) : fSpec(spec) + { + } + virtual ~Compare() + { + } - virtual int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) = 0; - void revertSortSpec() - { - fSpec.fAsc = -fSpec.fAsc; - fSpec.fNf = -fSpec.fNf; - } + virtual int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) = 0; + void revertSortSpec() + { + fSpec.fAsc = -fSpec.fAsc; + fSpec.fNf = -fSpec.fNf; + } -protected: - IdbSortSpec fSpec; + protected: + IdbSortSpec fSpec; }; // Comparators for signed types class TinyIntCompare : public Compare { -public: - TinyIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + TinyIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class SmallIntCompare : public Compare { -public: - SmallIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + SmallIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class IntCompare : public Compare { -public: - IntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + IntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class BigIntCompare : public Compare { -public: - BigIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + BigIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; class WideDecimalCompare : public Compare { - int keyColumnOffset; -public: - WideDecimalCompare(const IdbSortSpec& spec, int offset) : Compare(spec), keyColumnOffset(offset) { } + int keyColumnOffset; - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + public: + WideDecimalCompare(const IdbSortSpec& spec, int offset) : Compare(spec), keyColumnOffset(offset) + { + } + + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; // End of comparators for signed types @@ -157,37 +180,42 @@ public: class UTinyIntCompare : public Compare { -public: - UTinyIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + UTinyIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class USmallIntCompare : public Compare { -public: - USmallIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + USmallIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class UIntCompare : public Compare { -public: - UIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + UIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class UBigIntCompare : public Compare { -public: - UBigIntCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + UBigIntCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; // end of comparators for unsigned types @@ -196,28 +224,32 @@ public: class DoubleCompare : public Compare { -public: - DoubleCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + DoubleCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class LongDoubleCompare : public Compare { -public: - LongDoubleCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + LongDoubleCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class FloatCompare : public Compare { -public: - FloatCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + FloatCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; // End of comparators for float types @@ -225,28 +257,32 @@ public: class DateCompare : public Compare { -public: - DateCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + DateCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class DatetimeCompare : public Compare { -public: - DatetimeCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + DatetimeCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; - class TimeCompare : public Compare { -public: - TimeCompare(const IdbSortSpec& spec) : Compare(spec) {} + public: + TimeCompare(const IdbSortSpec& spec) : Compare(spec) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; // End of comparators for temporal types @@ -255,185 +291,191 @@ public: class StringCompare : public Compare { -public: - StringCompare(const IdbSortSpec& spec) : Compare(spec), cs(NULL) {} + public: + StringCompare(const IdbSortSpec& spec) : Compare(spec), cs(NULL) + { + } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); - - CHARSET_INFO* cs; + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + + CHARSET_INFO* cs; }; // End of comparators for variable sized types class CompareRule { -public: - CompareRule(IdbCompare* c = NULL) : fIdbCompare(c) {} + public: + CompareRule(IdbCompare* c = NULL) : fIdbCompare(c) + { + } + bool less(rowgroup::Row::Pointer r1, rowgroup::Row::Pointer r2); - bool less(rowgroup::Row::Pointer r1, rowgroup::Row::Pointer r2); + void compileRules(const std::vector&, const rowgroup::RowGroup&); + void revertRules(); - void compileRules(const std::vector&, const rowgroup::RowGroup&); - void revertRules(); - - std::vector fCompares; - IdbCompare* fIdbCompare; + std::vector fCompares; + IdbCompare* fIdbCompare; }; - class IdbCompare { -public: - IdbCompare() {}; - virtual ~IdbCompare() {}; + public: + IdbCompare(){}; + virtual ~IdbCompare(){}; - virtual void initialize(const rowgroup::RowGroup&); - void setStringTable(bool b); + virtual void initialize(const rowgroup::RowGroup&); + void setStringTable(bool b); - rowgroup::Row& row1() - { - return fRow1; - } - rowgroup::Row& row2() - { - return fRow2; - } + rowgroup::Row& row1() + { + return fRow1; + } + rowgroup::Row& row2() + { + return fRow2; + } - rowgroup::RowGroup* rowGroup() - { - return &fRowGroup; - } -protected: - rowgroup::RowGroup fRowGroup; - rowgroup::Row fRow1; - rowgroup::Row fRow2; + rowgroup::RowGroup* rowGroup() + { + return &fRowGroup; + } + + protected: + rowgroup::RowGroup fRowGroup; + rowgroup::Row fRow1; + rowgroup::Row fRow2; }; - class OrderByRow { -public: - OrderByRow(const rowgroup::Row& r, CompareRule& c) : fData(r.getPointer()), fRule(&c) {} + public: + OrderByRow(const rowgroup::Row& r, CompareRule& c) : fData(r.getPointer()), fRule(&c) + { + } - bool operator < (const OrderByRow& rhs) const - { - return fRule->less(fData, rhs.fData); - } + bool operator<(const OrderByRow& rhs) const + { + return fRule->less(fData, rhs.fData); + } - rowgroup::Row::Pointer fData; - CompareRule* fRule; + rowgroup::Row::Pointer fData; + CompareRule* fRule; }; - class EqualCompData : public IdbCompare { -public: - EqualCompData(std::vector& v) : fIndex(v) {} - EqualCompData(std::vector& v, const rowgroup::RowGroup& rg) : - fIndex(v) - { - initialize(rg); - } + public: + EqualCompData(std::vector& v) : fIndex(v) + { + } + EqualCompData(std::vector& v, const rowgroup::RowGroup& rg) : fIndex(v) + { + initialize(rg); + } - ~EqualCompData() {}; + ~EqualCompData(){}; - bool operator()(rowgroup::Row::Pointer, rowgroup::Row::Pointer); + bool operator()(rowgroup::Row::Pointer, rowgroup::Row::Pointer); - std::vector fIndex; + std::vector fIndex; }; - class OrderByData : public IdbCompare { -public: - OrderByData(const std::vector&, const rowgroup::RowGroup&); - virtual ~OrderByData(); + public: + OrderByData(const std::vector&, const rowgroup::RowGroup&); + virtual ~OrderByData(); - bool operator() (rowgroup::Row::Pointer p1, rowgroup::Row::Pointer p2) - { - return fRule.less(p1, p2); - } - const CompareRule& rule() const - { - return fRule; - } + bool operator()(rowgroup::Row::Pointer p1, rowgroup::Row::Pointer p2) + { + return fRule.less(p1, p2); + } + const CompareRule& rule() const + { + return fRule; + } -protected: - CompareRule fRule; + protected: + CompareRule fRule; }; - // base classs for order by clause used in IDB class IdbOrderBy : public IdbCompare { -public: - IdbOrderBy(); - virtual ~IdbOrderBy(); + public: + IdbOrderBy(); + virtual ~IdbOrderBy(); - virtual void initialize(const rowgroup::RowGroup&); - virtual void processRow(const rowgroup::Row&) = 0; - virtual uint64_t getKeyLength() const = 0; - virtual const std::string toString() const = 0; + virtual void initialize(const rowgroup::RowGroup&); + virtual void processRow(const rowgroup::Row&) = 0; + virtual uint64_t getKeyLength() const = 0; + virtual const std::string toString() const = 0; - bool getData(rowgroup::RGData& data); + bool getData(rowgroup::RGData& data); - void distinct(bool b) + void distinct(bool b) + { + fDistinct = b; + } + bool distinct() const + { + return fDistinct; + } + SortingPQ& getQueue() + { + return fOrderByQueue; + } + CompareRule& getRule() + { + return fRule; + } + + SortingPQ fOrderByQueue; + + protected: + std::vector fOrderByCond; + rowgroup::Row fRow0; + CompareRule fRule; + + rowgroup::RGData fData; + std::queue fDataQueue; + + struct Hasher + { + IdbOrderBy* ts; + utils::Hasher_r h; + uint32_t colCount; + Hasher(IdbOrderBy* t, uint32_t c) : ts(t), colCount(c) { - fDistinct = b; } - bool distinct() const + uint64_t operator()(const rowgroup::Row::Pointer&) const; + }; + struct Eq + { + IdbOrderBy* ts; + uint32_t colCount; + Eq(IdbOrderBy* t, uint32_t c) : ts(t), colCount(c) { - return fDistinct; - } - SortingPQ& getQueue() - { - return fOrderByQueue; - } - CompareRule &getRule() - { - return fRule; } + bool operator()(const rowgroup::Row::Pointer&, const rowgroup::Row::Pointer&) const; + }; - SortingPQ fOrderByQueue; -protected: - std::vector fOrderByCond; - rowgroup::Row fRow0; - CompareRule fRule; + typedef std::tr1::unordered_set > + DistinctMap_t; + boost::scoped_ptr fDistinctMap; + rowgroup::Row row1, row2; // scratch space for Hasher & Eq - rowgroup::RGData fData; - std::queue fDataQueue; - - struct Hasher - { - IdbOrderBy* ts; - utils::Hasher_r h; - uint32_t colCount; - Hasher(IdbOrderBy* t, uint32_t c) : ts(t), colCount(c) { } - uint64_t operator()(const rowgroup::Row::Pointer&) const; - }; - struct Eq - { - IdbOrderBy* ts; - uint32_t colCount; - Eq(IdbOrderBy* t, uint32_t c) : ts(t), colCount(c) { } - bool operator()(const rowgroup::Row::Pointer&, const rowgroup::Row::Pointer&) const; - }; - - typedef std::tr1::unordered_set > DistinctMap_t; - boost::scoped_ptr fDistinctMap; - rowgroup::Row row1, row2; // scratch space for Hasher & Eq - - bool fDistinct; - uint64_t fMemSize; - uint64_t fRowsPerRG; - uint64_t fErrorCode; - joblist::ResourceManager* fRm; - boost::shared_ptr fSessionMemLimit; + bool fDistinct; + uint64_t fMemSize; + uint64_t fRowsPerRG; + uint64_t fErrorCode; + joblist::ResourceManager* fRm; + boost::shared_ptr fSessionMemLimit; }; - -} +} // namespace ordering #endif // IDB_ORDER_BY_H - diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index 48fe3da25..c45cd2978 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -17,7 +17,6 @@ // $Id: wf_count.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -49,150 +48,140 @@ using namespace joblist; #include "wf_count.h" - namespace windowfunction { - - -template -boost::shared_ptr WF_count::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_count::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - switch (ct) + switch (ct) + { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: { - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - { - func.reset(new WF_count(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->functionParms()[0]->resultType().colWidth; - if (width < datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_count(id, name)); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_count(id, name)); - } - break; - } - default: - { - func.reset(new WF_count(id, name)); - break; - } + func.reset(new WF_count(id, name)); + break; } - return func; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_count(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_count(id, name)); + } + break; + } + default: + { + func.reset(new WF_count(id, name)); + break; + } + } + + return func; } - -template +template WindowFunctionType* WF_count::clone() const { - return new WF_count(*this); + return new WF_count(*this); } - -template +template void WF_count::resetData() { - fCount = 0; - fSet.clear(); + fCount = 0; + fSet.clear(); - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_count::operator()(int64_t b, int64_t e, int64_t c) { - if ((fFrameUnit == WF__FRAME_ROWS) || - (fPrev == -1) || - (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || + (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + { + // for unbounded - current row special handling + if (fPrev >= b && fPrev < c) + b = c; + else if (fPrev <= e && fPrev > c) + e = c; + + // for count(*), the column is optimized out, index[1] does not exist. + int64_t colIn = (fFunctionId == WF__COUNT_ASTERISK) ? 0 : fFieldIndex[1]; + + // constant param will have fFieldIndex[1] of -1. Get value of constant param + if (colIn == -1) { - // for unbounded - current row special handling - if (fPrev >= b && fPrev < c) - b = c; - else if (fPrev <= e && fPrev > c) - e = c; + ConstantColumn* cc = static_cast(fConstantParms[0].get()); - // for count(*), the column is optimized out, index[1] does not exist. - int64_t colIn = (fFunctionId == WF__COUNT_ASTERISK) ? 0 : fFieldIndex[1]; + if (cc) + { + bool isNull = false; + cc->getIntVal(fRow, isNull); - // constant param will have fFieldIndex[1] of -1. Get value of constant param - if (colIn == -1) + if (!isNull) { - ConstantColumn* cc = static_cast(fConstantParms[0].get()); - - if (cc) - { - bool isNull = false; - cc->getIntVal(fRow, isNull); - - if (!isNull) - { - // constant, set fCount to row count - fCount += e - b + 1; - } - } + // constant, set fCount to row count + fCount += e - b + 1; } - else if (fFunctionId == WF__COUNT_ASTERISK) + } + } + else if (fFunctionId == WF__COUNT_ASTERISK) + { + // count(*), set fCount to row count + fCount += e - b + 1; + } + else + { + for (int64_t i = b; i <= e; i++) + { + if (i % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer(fRowData->at(i))); + + if (fRow.isNullValue(colIn) == true) + continue; + + if (fFunctionId != WF__COUNT_DISTINCT) { - // count(*), set fCount to row count - fCount += e - b + 1; + fCount++; } else { - for (int64_t i = b; i <= e; i++) - { - if (i % 1000 == 0 && fStep->cancelled()) - break; + T valIn; + getValue(colIn, valIn); - fRow.setData(getPointer(fRowData->at(i))); + if (fSet.find(valIn) == fSet.end()) + { + fCount++; - if (fRow.isNullValue(colIn) == true) - continue; - - if (fFunctionId != WF__COUNT_DISTINCT) - { - fCount++; - } - else - { - T valIn; - getValue(colIn, valIn); - - if (fSet.find(valIn) == fSet.end()) - { - fCount++; - - if (fFunctionId == WF__COUNT_DISTINCT) - fSet.insert(valIn); - } - } - } + if (fFunctionId == WF__COUNT_DISTINCT) + fSet.insert(valIn); + } } + } } + } - setValue(CalpontSystemCatalog::BIGINT, b, e, c, &fCount); + setValue(CalpontSystemCatalog::BIGINT, b, e, c, &fCount); - fPrev = c; + fPrev = c; } +template boost::shared_ptr WF_count::makeFunction(int, const string&, int, + WindowFunctionColumn*); -template -boost::shared_ptr WF_count::makeFunction(int, const string&, int, WindowFunctionColumn*); - - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_count.h b/utils/windowfunction/wf_count.h index 934489815..2dcbd963b 100644 --- a/utils/windowfunction/wf_count.h +++ b/utils/windowfunction/wf_count.h @@ -17,44 +17,37 @@ // $Id: wf_count.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_COUNT_H #define UTILS_WF_COUNT_H #include #include "windowfunctiontype.h" - namespace windowfunction { - - -template +template class WF_count : public WindowFunctionType { -public: - WF_count(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_count(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - - uint64_t fCount; - std::set fSet; + protected: + uint64_t fCount; + std::set fSet; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_COUNT_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index ff363151b..4e7de53ff 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -18,7 +18,6 @@ // $Id: wf_lead_lag.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -47,266 +46,257 @@ using namespace joblist; #include "wf_lead_lag.h" - namespace windowfunction { - - -template -boost::shared_ptr WF_lead_lag::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_lead_lag::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - switch (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - func.reset(new WF_lead_lag(id, name)); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - { - func.reset(new WF_lead_lag(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->functionParms()[0]->resultType().colWidth; - if (width < datatypes::MAXDECIMALWIDTH) - { - if (ct == CalpontSystemCatalog::UDECIMAL) - func.reset(new WF_lead_lag(id, name)); - else - func.reset(new WF_lead_lag(id, name)); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_lead_lag(id, name)); - } - break; - } - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - func.reset(new WF_lead_lag(id, name)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_lead_lag(id, name)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_lead_lag(id, name)); - break; - } - - default: - { - func.reset(new WF_lead_lag(id, name)); - break; - } + func.reset(new WF_lead_lag(id, name)); + break; } - return func; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + { + func.reset(new WF_lead_lag(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_lead_lag(id, name)); + else + func.reset(new WF_lead_lag(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_lead_lag(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + func.reset(new WF_lead_lag(id, name)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_lead_lag(id, name)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_lead_lag(id, name)); + break; + } + + default: + { + func.reset(new WF_lead_lag(id, name)); + break; + } + } + + return func; } - -template +template WindowFunctionType* WF_lead_lag::clone() const { - return new WF_lead_lag(*this); + return new WF_lead_lag(*this); } - -template +template void WF_lead_lag::resetData() { - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_lead_lag::parseParms(const std::vector& parms) { - // lead | lag - fLead = 1; - fRespectNulls = true; - fDefNull = false; -// fDefault = (T)0; // Won't work for std::string. Default should always be set below. + // lead | lag + fLead = 1; + fRespectNulls = true; + fDefNull = false; + // fDefault = (T)0; // Won't work for std::string. Default should always be set below. + fOffsetNull = false; + fOffset = 0; + + if (fFunctionId == WF__LAG) + fLead = -1; + + // parms[0]: value-expr + // skip + + // parms[1]: offset + ConstantColumn* cc = dynamic_cast(parms[1].get()); + + if (cc != NULL) + { fOffsetNull = false; - fOffset = 0; + fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. + } - if (fFunctionId == WF__LAG) - fLead = -1; + // parms[2]: default value + cc = dynamic_cast(parms[2].get()); - // parms[0]: value-expr - // skip + if (cc != NULL) + { + fDefNull = false; + getConstValue(cc, fDefault, fDefNull); + } - // parms[1]: offset - ConstantColumn* cc = dynamic_cast(parms[1].get()); - - if (cc != NULL) - { - fOffsetNull = false; - fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. - } - - // parms[2]: default value - cc = dynamic_cast(parms[2].get()); - - if (cc != NULL) - { - fDefNull = false; - getConstValue(cc, fDefault, fDefNull); - } - - // parms[3]: respect null | ignore null - cc = dynamic_cast(parms[3].get()); - if (cc != NULL) - { - bool isNull = false; // dummy. Return not used - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); - } + // parms[3]: respect null | ignore null + cc = dynamic_cast(parms[3].get()); + if (cc != NULL) + { + bool isNull = false; // dummy. Return not used + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + } } - - -template +template void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) { - uint64_t colIn = fFieldIndex[1]; - bool isNull = true; + uint64_t colIn = fFieldIndex[1]; + bool isNull = true; - for (int64_t c = b; c <= e; c++) + for (int64_t c = b; c <= e; c++) + { + if (c % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer(fRowData->at(c))); + // get offset if not constant + int64_t idx = fFieldIndex[2]; + + if (idx != -1) { - if (c % 1000 == 0 && fStep->cancelled()) - break; + double tmp = 0.0; // use double to cover all column types + fOffsetNull = fRow.isNullValue(idx); - fRow.setData(getPointer(fRowData->at(c))); - // get offset if not constant - int64_t idx = fFieldIndex[2]; - - if (idx != -1) - { - double tmp = 0.0; // use double to cover all column types - fOffsetNull = fRow.isNullValue(idx); - - if (!fOffsetNull) - { - implicit2T(idx, tmp, 0); - fOffset = round(tmp); - fOffset *= fLead; - } - } - - // get default if not constant - idx = fFieldIndex[3]; - - if (idx != -1) - { - fDefNull = fRow.isNullValue(idx); - - if (!fDefNull) - implicit2T(idx, fDefault, (int) fRow.getScale(idx)); - } - - int64_t o = c + fOffset; - - if (o < b || o > e || fOffsetNull) // out of bound - { - T* v = (fDefNull) ? NULL : &fDefault; - setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); - continue; - } - - if (fRespectNulls == false && fRow.isNullValue(colIn) == true) - { - if (fOffset > 0) - { - while (++o < e) - { - fRow.setData(getPointer(fRowData->at(o))); - - if (fRow.isNullValue(colIn) == false) - break; - } - - if (o <= e) - { - fRow.setData(getPointer(fRowData->at(o))); - getValue(colIn, fValue); - isNull = fRow.isNullValue(colIn); - } - } - else if (fOffset < 0) - { - while (--o > b) - { - fRow.setData(getPointer(fRowData->at(o))); - - if (fRow.isNullValue(colIn) == false) - break; - } - - if (o >= b) - { - fRow.setData(getPointer(fRowData->at(o))); - getValue(colIn, fValue); - isNull = fRow.isNullValue(colIn); - } - } - - T* v = NULL; - - if (!isNull) - v = &fValue; - else if (!fDefNull) - v = &fDefault; - - setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); - } - else - { - fRow.setData(getPointer(fRowData->at(o))); - getValue(colIn, fValue); - isNull = fRow.isNullValue(colIn); - - T* v = NULL; - - if (!isNull) - v = &fValue; - else if (!fDefNull) - v = &fDefault; - - setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); - } + if (!fOffsetNull) + { + implicit2T(idx, tmp, 0); + fOffset = round(tmp); + fOffset *= fLead; + } } + + // get default if not constant + idx = fFieldIndex[3]; + + if (idx != -1) + { + fDefNull = fRow.isNullValue(idx); + + if (!fDefNull) + implicit2T(idx, fDefault, (int)fRow.getScale(idx)); + } + + int64_t o = c + fOffset; + + if (o < b || o > e || fOffsetNull) // out of bound + { + T* v = (fDefNull) ? NULL : &fDefault; + setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + continue; + } + + if (fRespectNulls == false && fRow.isNullValue(colIn) == true) + { + if (fOffset > 0) + { + while (++o < e) + { + fRow.setData(getPointer(fRowData->at(o))); + + if (fRow.isNullValue(colIn) == false) + break; + } + + if (o <= e) + { + fRow.setData(getPointer(fRowData->at(o))); + getValue(colIn, fValue); + isNull = fRow.isNullValue(colIn); + } + } + else if (fOffset < 0) + { + while (--o > b) + { + fRow.setData(getPointer(fRowData->at(o))); + + if (fRow.isNullValue(colIn) == false) + break; + } + + if (o >= b) + { + fRow.setData(getPointer(fRowData->at(o))); + getValue(colIn, fValue); + isNull = fRow.isNullValue(colIn); + } + } + + T* v = NULL; + + if (!isNull) + v = &fValue; + else if (!fDefNull) + v = &fDefault; + + setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + } + else + { + fRow.setData(getPointer(fRowData->at(o))); + getValue(colIn, fValue); + isNull = fRow.isNullValue(colIn); + + T* v = NULL; + + if (!isNull) + v = &fValue; + else if (!fDefNull) + v = &fDefault; + + setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + } + } } - -template -boost::shared_ptr WF_lead_lag::makeFunction(int, const string&, int, WindowFunctionColumn*); +template boost::shared_ptr WF_lead_lag::makeFunction(int, const string&, int, + WindowFunctionColumn*); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); @@ -315,6 +305,5 @@ template void WF_lead_lag::parseParms(const std::vector&) template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_lead_lag.h b/utils/windowfunction/wf_lead_lag.h index 81b707ae2..9aaea0c86 100644 --- a/utils/windowfunction/wf_lead_lag.h +++ b/utils/windowfunction/wf_lead_lag.h @@ -17,49 +17,42 @@ // $Id: wf_lead_lag.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_LEAD_LAG_H #define UTILS_WF_LEAD_LAG_H #include "windowfunctiontype.h" - namespace windowfunction { - - -template +template class WF_lead_lag : public WindowFunctionType { -public: - WF_lead_lag(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_lead_lag(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); + void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); - -protected: - T fValue; - T fDefault; - int64_t fOffset; - int64_t fLead; - bool fOffsetNull; - bool fDefNull; - bool fRespectNulls; // respect null | ignore null + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + protected: + T fValue; + T fDefault; + int64_t fOffset; + int64_t fLead; + bool fOffsetNull; + bool fDefNull; + bool fRespectNulls; // respect null | ignore null }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_LEAD_LAG_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 036a393fc..4120fe32a 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -18,7 +18,6 @@ // $Id: wf_min_max.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -46,154 +45,144 @@ using namespace joblist; #include "wf_min_max.h" - namespace windowfunction { - - -template -boost::shared_ptr WF_min_max::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_min_max::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - switch (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - func.reset(new WF_min_max(id, name)); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - { - func.reset(new WF_min_max(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->functionParms()[0]->resultType().colWidth; - - if (width < datatypes::MAXDECIMALWIDTH) - { - if (ct == CalpontSystemCatalog::UDECIMAL) - func.reset(new WF_min_max(id, name)); - else - func.reset(new WF_min_max(id, name)); - } - else - { - func.reset(new WF_min_max(id, name)); - } - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - func.reset(new WF_min_max(id, name)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_min_max(id, name)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_min_max(id, name)); - break; - } - - default: - { - func.reset(new WF_min_max(id, name)); - break; - } + func.reset(new WF_min_max(id, name)); + break; } - return func; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + { + func.reset(new WF_min_max(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->functionParms()[0]->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_min_max(id, name)); + else + func.reset(new WF_min_max(id, name)); + } + else + { + func.reset(new WF_min_max(id, name)); + } + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + func.reset(new WF_min_max(id, name)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_min_max(id, name)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_min_max(id, name)); + break; + } + + default: + { + func.reset(new WF_min_max(id, name)); + break; + } + } + + return func; } - -template +template WindowFunctionType* WF_min_max::clone() const { - return new WF_min_max(*this); + return new WF_min_max(*this); } - -template +template void WF_min_max::resetData() { - fCount = 0; + fCount = 0; - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_min_max::operator()(int64_t b, int64_t e, int64_t c) { - // for unbounded - current row special handling - if (fPrev >= b && fPrev < c) - b = c; - else if (fPrev <= e && fPrev > c) - e = c; + // for unbounded - current row special handling + if (fPrev >= b && fPrev < c) + b = c; + else if (fPrev <= e && fPrev > c) + e = c; - uint64_t colIn = fFieldIndex[1]; + uint64_t colIn = fFieldIndex[1]; - for (int64_t i = b; i <= e; i++) + for (int64_t i = b; i <= e; i++) + { + if (i % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer(fRowData->at(i))); + + if (fRow.isNullValue(colIn) == true) + continue; + + T valIn; + getValue(colIn, valIn); + + if ((fCount == 0) || (valIn < fValue && fFunctionId == WF__MIN) || + (valIn > fValue && fFunctionId == WF__MAX)) { - if (i % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer(fRowData->at(i))); - - if (fRow.isNullValue(colIn) == true) - continue; - - T valIn; - getValue(colIn, valIn); - - if ((fCount == 0) || - (valIn < fValue && fFunctionId == WF__MIN) || - (valIn > fValue && fFunctionId == WF__MAX)) - { - fValue = valIn; - } - - fCount++; + fValue = valIn; } - T* v = ((fCount > 0) ? &fValue : NULL); - setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + fCount++; + } - fPrev = c; + T* v = ((fCount > 0) ? &fValue : NULL); + setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + + fPrev = c; } +template boost::shared_ptr WF_min_max::makeFunction(int, const string&, int, + WindowFunctionColumn*); -template -boost::shared_ptr WF_min_max::makeFunction(int, const string&, int, WindowFunctionColumn*); - - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_min_max.h b/utils/windowfunction/wf_min_max.h index ebe12df65..bd8091ed8 100644 --- a/utils/windowfunction/wf_min_max.h +++ b/utils/windowfunction/wf_min_max.h @@ -17,42 +17,36 @@ // $Id: wf_min_max.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_MIN_MAX_H #define UTILS_WF_MIN_MAX_H #include "windowfunctiontype.h" - namespace windowfunction { - - -template +template class WF_min_max : public WindowFunctionType { -public: - WF_min_max(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_min_max(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - T fValue; - uint64_t fCount; + protected: + T fValue; + uint64_t fCount; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_MIN_MAX_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index d838c8ec3..338984a04 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -18,7 +18,6 @@ // $Id: wf_nth_value.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -47,239 +46,229 @@ using namespace joblist; #include "wf_nth_value.h" - namespace windowfunction { - - -template -boost::shared_ptr WF_nth_value::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_nth_value::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - switch (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - func.reset(new WF_nth_value(id, name)); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - { - func.reset(new WF_nth_value(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->functionParms()[0]->resultType().colWidth; - if (width < datatypes::MAXDECIMALWIDTH) - { - if (ct == CalpontSystemCatalog::UDECIMAL) - func.reset(new WF_nth_value(id, name)); - else - func.reset(new WF_nth_value(id, name)); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_nth_value(id, name)); - } - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - func.reset(new WF_nth_value(id, name)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_nth_value(id, name)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_nth_value(id, name)); - break; - } - - default: - { - func.reset(new WF_nth_value(id, name)); - break; - } + func.reset(new WF_nth_value(id, name)); + break; } - return func; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + { + func.reset(new WF_nth_value(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_nth_value(id, name)); + else + func.reset(new WF_nth_value(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_nth_value(id, name)); + } + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + func.reset(new WF_nth_value(id, name)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_nth_value(id, name)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_nth_value(id, name)); + break; + } + + default: + { + func.reset(new WF_nth_value(id, name)); + break; + } + } + + return func; } - -template +template WindowFunctionType* WF_nth_value::clone() const { - return new WF_nth_value(*this); + return new WF_nth_value(*this); } - -template +template void WF_nth_value::resetData() { - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_nth_value::parseParms(const std::vector& parms) { - // parms[0]: value-expr - // skip + // parms[0]: value-expr + // skip - // parms[1]: nth value - ConstantColumn* cc = dynamic_cast(parms[1].get()); + // parms[1]: nth value + ConstantColumn* cc = dynamic_cast(parms[1].get()); - if (cc != NULL) + if (cc != NULL) + { + fNthNull = false; + fNth = cc->getIntVal(fRow, fNthNull); // row not used, no need to setData. + + if (fNth <= 0) { - fNthNull = false; - fNth = cc->getIntVal(fRow, fNthNull); // row not used, no need to setData. - - if (fNth <= 0) - { - ostringstream oss; - oss << fNth; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } + ostringstream oss; + oss << fNth; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, oss.str()), + ERR_WF_ARG_OUT_OF_RANGE); } + } - // parms[2]: from first | from last - bool isNull = false; - cc = dynamic_cast(parms[2].get()); - idbassert(cc != NULL); - fFromFirst = (cc->getIntVal(fRow, isNull) > 0); + // parms[2]: from first | from last + bool isNull = false; + cc = dynamic_cast(parms[2].get()); + idbassert(cc != NULL); + fFromFirst = (cc->getIntVal(fRow, isNull) > 0); - // parms[3]: respect null | ignore null - cc = dynamic_cast(parms[3].get()); - idbassert(cc != NULL); - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + // parms[3]: respect null | ignore null + cc = dynamic_cast(parms[3].get()); + idbassert(cc != NULL); + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); } - -template +template void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) { - int64_t s = b; - int64_t t = e; + int64_t s = b; + int64_t t = e; - if (c != WF__BOUND_ALL) - s = t = c; + if (c != WF__BOUND_ALL) + s = t = c; - for (int64_t c = s; c <= t; c++) + for (int64_t c = s; c <= t; c++) + { + if (c % 1000 == 0 && fStep->cancelled()) + break; + + int64_t idx = fFieldIndex[2]; + fRow.setData(getPointer(fRowData->at(c))); + + if (idx != -1) { - if (c % 1000 == 0 && fStep->cancelled()) - break; + double tmp = 1.0; + fNthNull = fRow.isNullValue(idx); - int64_t idx = fFieldIndex[2]; - fRow.setData(getPointer(fRowData->at(c))); - - if (idx != -1) - { - double tmp = 1.0; - fNthNull = fRow.isNullValue(idx); - - if (!fNthNull) - { - implicit2T(idx, tmp, 0); - fNth = round(tmp); - } - } - - bool isNull = true; - - if ((!fNthNull) && ((b + fNth - 1) <= e)) - { - uint64_t colIn = fFieldIndex[1]; - - if (fFromFirst) - { - int64_t k = b; - fRow.setData(getPointer(fRowData->at(k))); - - if (fRespectNulls == false && fRow.isNullValue(colIn) == true) - { - while (++k < e) - { - fRow.setData(getPointer(fRowData->at(k))); - - if (fRow.isNullValue(colIn) == false) - break; - } - } - - int64_t n = k + fNth - 1; - - if (n <= e && n >= 0) - { - fRow.setData(getPointer(fRowData->at(n))); - getValue(colIn, fValue); - isNull = fRow.isNullValue(colIn); - } - } - else // from last - { - int64_t k = e; - fRow.setData(getPointer(fRowData->at(k))); - - if (fRespectNulls == false && fRow.isNullValue(colIn) == true) - { - while (--k > b) - { - fRow.setData(getPointer(fRowData->at(k))); - - if (fRow.isNullValue(colIn) == false) - break; - } - } - - int64_t n = k - fNth + 1; - - if (n >= b) - { - fRow.setData(getPointer(fRowData->at(n))); - getValue(colIn, fValue); - isNull = fRow.isNullValue(colIn); - } - } - } - - T* v = (isNull) ? NULL : &fValue; - setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + if (!fNthNull) + { + implicit2T(idx, tmp, 0); + fNth = round(tmp); + } } + + bool isNull = true; + + if ((!fNthNull) && ((b + fNth - 1) <= e)) + { + uint64_t colIn = fFieldIndex[1]; + + if (fFromFirst) + { + int64_t k = b; + fRow.setData(getPointer(fRowData->at(k))); + + if (fRespectNulls == false && fRow.isNullValue(colIn) == true) + { + while (++k < e) + { + fRow.setData(getPointer(fRowData->at(k))); + + if (fRow.isNullValue(colIn) == false) + break; + } + } + + int64_t n = k + fNth - 1; + + if (n <= e && n >= 0) + { + fRow.setData(getPointer(fRowData->at(n))); + getValue(colIn, fValue); + isNull = fRow.isNullValue(colIn); + } + } + else // from last + { + int64_t k = e; + fRow.setData(getPointer(fRowData->at(k))); + + if (fRespectNulls == false && fRow.isNullValue(colIn) == true) + { + while (--k > b) + { + fRow.setData(getPointer(fRowData->at(k))); + + if (fRow.isNullValue(colIn) == false) + break; + } + } + + int64_t n = k - fNth + 1; + + if (n >= b) + { + fRow.setData(getPointer(fRowData->at(n))); + getValue(colIn, fValue); + isNull = fRow.isNullValue(colIn); + } + } + } + + T* v = (isNull) ? NULL : &fValue; + setValue(fRow.getColType(fFieldIndex[0]), b, e, c, v); + } } +template boost::shared_ptr WF_nth_value::makeFunction(int, const string&, int, + WindowFunctionColumn*); -template -boost::shared_ptr WF_nth_value::makeFunction(int, const string&, int, WindowFunctionColumn*); - - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_nth_value.h b/utils/windowfunction/wf_nth_value.h index 8ea294b02..d5d98b4b9 100644 --- a/utils/windowfunction/wf_nth_value.h +++ b/utils/windowfunction/wf_nth_value.h @@ -17,47 +17,40 @@ // $Id: wf_nth_value.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_NTH_VALUE_H #define UTILS_WF_NTH_VALUE_H #include "windowfunctiontype.h" - namespace windowfunction { - - -template +template class WF_nth_value : public WindowFunctionType { -public: - WF_nth_value(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_nth_value(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); + void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); - -protected: - T fValue; - int64_t fNth; - bool fNthNull; - bool fFromFirst; // from first | from last - bool fRespectNulls; // respect null | ignore null + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + protected: + T fValue; + int64_t fNth; + bool fNthNull; + bool fFromFirst; // from first | from last + bool fRespectNulls; // respect null | ignore null }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_NTH_VALUE_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 716b10a2c..375fd81d2 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -17,7 +17,6 @@ // $Id: wf_ntile.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -49,118 +48,110 @@ using namespace joblist; #include "wf_ntile.h" - namespace windowfunction { - - -boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func(new WF_ntile(id, name)); - return func; + boost::shared_ptr func(new WF_ntile(id, name)); + return func; } - WindowFunctionType* WF_ntile::clone() const { - return new WF_ntile(*this); + return new WF_ntile(*this); } - void WF_ntile::resetData() { - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - void WF_ntile::parseParms(const std::vector& parms) { - // parms[0]: nt - ConstantColumn* cc = dynamic_cast(parms[0].get()); + // parms[0]: nt + ConstantColumn* cc = dynamic_cast(parms[0].get()); - if (cc != NULL) + if (cc != NULL) + { + fNtileNull = false; + fNtile = cc->getIntVal(fRow, fNtileNull); // row not used, no need to setData. + + if (!fNtileNull && fNtile <= 0) { - fNtileNull = false; - fNtile = cc->getIntVal(fRow, fNtileNull); // row not used, no need to setData. - - if (!fNtileNull && fNtile <= 0) - { - ostringstream oss; - oss << fNtile; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } + ostringstream oss; + oss << fNtile; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, oss.str()), + ERR_WF_ARG_OUT_OF_RANGE); } + } } - void WF_ntile::operator()(int64_t b, int64_t e, int64_t c) { - int64_t idx = fFieldIndex[1]; + int64_t idx = fFieldIndex[1]; + + if (idx != -1) + { + fRow.setData(getPointer(fRowData->at(b))); if (idx != -1) { - fRow.setData(getPointer(fRowData->at(b))); + double tmp = 1.0; + fNtileNull = fRow.isNullValue(idx); - if (idx != -1) - { - double tmp = 1.0; - fNtileNull = fRow.isNullValue(idx); + if (!fNtileNull) + implicit2T(idx, tmp, 0); - if (!fNtileNull) - implicit2T(idx, tmp, 0); + if (!fNtileNull && tmp <= 0) + { + ostringstream oss; + oss << tmp; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, oss.str()), + ERR_WF_ARG_OUT_OF_RANGE); + } - if (!fNtileNull && tmp <= 0) - { - ostringstream oss; - oss << tmp; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } - - fNtile = round(tmp); - } + fNtile = round(tmp); } + } - c = b; + c = b; - if (!fNtileNull) + if (!fNtileNull) + { + int64_t rowPerBucket = (e - b + 1) / fNtile; + int64_t n = rowPerBucket * fNtile; + int64_t x = (e - b + 1) - n; // extra + int64_t y = 0; + int64_t z = 0; + + while (c <= e) { - int64_t rowPerBucket = (e - b + 1) / fNtile; - int64_t n = rowPerBucket * fNtile; - int64_t x = (e - b + 1) - n; // extra - int64_t y = 0; - int64_t z = 0; + if (c % 1000 == 0 && fStep->cancelled()) + break; - while (c <= e) - { - if (c % 1000 == 0 && fStep->cancelled()) - break; + y = rowPerBucket + ((x-- > 0) ? 1 : 0); + z++; - y = rowPerBucket + ((x-- > 0) ? 1 : 0); - z++; - - for (int64_t i = 0; i < y && c <= e; i++) - { - fRow.setData(getPointer(fRowData->at(c++))); - setIntValue(fFieldIndex[0], z); - } - } + for (int64_t i = 0; i < y && c <= e; i++) + { + fRow.setData(getPointer(fRowData->at(c++))); + setIntValue(fFieldIndex[0], z); + } } - else + } + else + { + while (c <= e) { - while (c <= e) - { - if (c % 1000 == 0 && fStep->cancelled()) - break; + if (c % 1000 == 0 && fStep->cancelled()) + break; - fRow.setData(getPointer(fRowData->at(c++))); - setIntValue(fFieldIndex[0], joblist::BIGINTNULL); - } + fRow.setData(getPointer(fRowData->at(c++))); + setIntValue(fFieldIndex[0], joblist::BIGINTNULL); } + } } - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_ntile.h b/utils/windowfunction/wf_ntile.h index 45d5ba64a..154f725b1 100644 --- a/utils/windowfunction/wf_ntile.h +++ b/utils/windowfunction/wf_ntile.h @@ -17,44 +17,37 @@ // $Id: wf_ntile.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_NTILE_H #define UTILS_WF_NTILE_H #include #include "windowfunctiontype.h" - namespace windowfunction { - - class WF_ntile : public WindowFunctionType { -public: - WF_ntile(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_ntile(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); + void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - - uint64_t fNtile; - bool fNtileNull; + protected: + uint64_t fNtile; + bool fNtileNull; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_NTILE_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index d23f9e712..4d0dfce07 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -18,7 +18,6 @@ // $Id: wf_percentile.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -51,347 +50,336 @@ using namespace joblist; #include "wf_percentile.h" - namespace windowfunction { - -template -boost::shared_ptr WF_percentile::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_percentile::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - if (id == WF__PERCENTILE_DISC) + if (id == WF__PERCENTILE_DISC) + { + switch (ct) { - switch (ct) + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + { + func.reset(new WF_percentile(id, name)); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: + { + func.reset(new WF_percentile(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - func.reset(new WF_percentile(id, name)); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - { - func.reset(new WF_percentile(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->resultType().colWidth; - - if (width < datatypes::MAXDECIMALWIDTH) - { - if (ct == CalpontSystemCatalog::UDECIMAL) - func.reset(new WF_percentile(id, name)); - else - func.reset(new WF_percentile(id, name)); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_percentile(id, name)); - } - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - func.reset(new WF_percentile(id, name)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_percentile(id, name)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_percentile(id, name)); - break; - } - - default: - { - if (id == WF__PERCENTILE_DISC) - { - func.reset(new WF_percentile(id, name)); - } - else - { - string errStr = name + "(" + colType2String[ct] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); - } - - break; - } + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_percentile(id, name)); + else + func.reset(new WF_percentile(id, name)); } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_percentile(id, name)); + } + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + func.reset(new WF_percentile(id, name)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_percentile(id, name)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_percentile(id, name)); + break; + } + + default: + { + if (id == WF__PERCENTILE_DISC) + { + func.reset(new WF_percentile(id, name)); + } + else + { + string errStr = name + "(" + colType2String[ct] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + } + + break; + } } - else + } + else + { + switch (ct) { - switch (ct) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_percentile(id, name)); - break; - } + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UDECIMAL: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_percentile(id, name)); + break; + } - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_percentile(id, name)); - break; - } + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_percentile(id, name)); + break; + } - default: - { - string errStr = name + "(" + colType2String[ct] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); - break; - } - } + default: + { + string errStr = name + "(" + colType2String[ct] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + break; + } } + } - return func; + return func; } - -template +template WindowFunctionType* WF_percentile::clone() const { - return new WF_percentile(*this); + return new WF_percentile(*this); } - -template +template void WF_percentile::resetData() { - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_percentile::parseParms(const std::vector& parms) { - // parms[0]: nve - ConstantColumn* cc = dynamic_cast(parms[0].get()); + // parms[0]: nve + ConstantColumn* cc = dynamic_cast(parms[0].get()); - if (cc != NULL) + if (cc != NULL) + { + fNveNull = false; + fNve = cc->getDoubleVal(fRow, fNveNull); // row not used, no need to setData. + + if (!fNveNull && (fNve < 0 || fNve > 1)) { - fNveNull = false; - fNve = cc->getDoubleVal(fRow, fNveNull); // row not used, no need to setData. - - if (!fNveNull && (fNve < 0 || fNve > 1)) - { - ostringstream oss; - oss << fNve; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } + ostringstream oss; + oss << fNve; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, oss.str()), + ERR_WF_ARG_OUT_OF_RANGE); } + } - // workaround for the within group order by column index - idbassert(fPeer->fIndex.size() > 0); - fFieldIndex.push_back(fPeer->fIndex[0]); + // workaround for the within group order by column index + idbassert(fPeer->fIndex.size() > 0); + fFieldIndex.push_back(fPeer->fIndex[0]); } - -template +template void WF_percentile::operator()(int64_t b, int64_t e, int64_t c) { - int64_t idx = fFieldIndex[1]; - fRow.setData(getPointer(fRowData->at(b))); + int64_t idx = fFieldIndex[1]; + fRow.setData(getPointer(fRowData->at(b))); + if (idx != -1) + { if (idx != -1) { - if (idx != -1) - { - fNveNull = fRow.isNullValue(idx); - implicit2T(idx, fNve, 0); + fNveNull = fRow.isNullValue(idx); + implicit2T(idx, fNve, 0); - if (!fNveNull && (fNve < 0 || fNve > 1)) - { - ostringstream oss; - oss << fNve; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } - } + if (!fNveNull && (fNve < 0 || fNve > 1)) + { + ostringstream oss; + oss << fNve; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, oss.str()), + ERR_WF_ARG_OUT_OF_RANGE); + } } + } - if (fNveNull) - { - for (c = b; c <= e; c++) - { - if (c % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer(fRowData->at(c))); - setValue(fRow.getColType(fFieldIndex[0]), b, e, c, (T*) NULL); - } - - return; - } - - idx = fFieldIndex[2]; - int64_t rank = 0; - int64_t dups = 0; - int64_t b1 = -1; - int64_t e1 = -1; - scoped_array rk(new int64_t[e - b + 1]); - + if (fNveNull) + { for (c = b; c <= e; c++) { - if (c % 1000 == 0 && fStep->cancelled()) + if (c % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer(fRowData->at(c))); + setValue(fRow.getColType(fFieldIndex[0]), b, e, c, (T*)NULL); + } + + return; + } + + idx = fFieldIndex[2]; + int64_t rank = 0; + int64_t dups = 0; + int64_t b1 = -1; + int64_t e1 = -1; + scoped_array rk(new int64_t[e - b + 1]); + + for (c = b; c <= e; c++) + { + if (c % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer(fRowData->at(c))); + + if (fRow.isNullValue(idx)) + continue; + + // ignore nulls + if (b1 == -1) + b1 = c; + + e1 = c; + + if (fFunctionId == WF__PERCENTILE_DISC) + { + // need cume_rank + if (c != b && fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c - 1)))) + { + dups++; + } + else + { + rank++; + rank += dups; + dups = 0; + } + + rk[c - b] = rank; + } + } + + T* p = NULL; + T v; + int ct = (fFunctionId == WF__PERCENTILE_CONT) ? CalpontSystemCatalog::DOUBLE : fRow.getColType(idx); + + if (b1 != -1) + { + double cnt = (e1 - b1 + 1); + + if (fFunctionId == WF__PERCENTILE_CONT) + { + // @bug5820, this "rn" is the normalized row number, not the real row number. + // Using real row number here will introduce a small calculation error in double result. + double rn = fNve * (cnt - 1); + double crn = ceil(rn); + double frn = floor(rn); + double vd = 0; + + if (crn == rn && rn == frn) + { + fRow.setData(getPointer(fRowData->at((size_t)rn + (size_t)b1))); + implicit2T(idx, vd, 0); + } + else + { + double cv = 0.0, fv = 0.0; + fRow.setData(getPointer(fRowData->at((size_t)frn + (size_t)b1))); + implicit2T(idx, fv, 0); + fRow.setData(getPointer(fRowData->at((size_t)crn + (size_t)b1))); + implicit2T(idx, cv, 0); + vd = (crn - rn) * fv + (rn - frn) * cv; + } + + v = *(reinterpret_cast(&vd)); + p = &v; + } + else // (fFunctionId == WF__PERCENTILE_DISC) + { + int prevRank = ++rank + dups; + double cumeDist = 1; + fRow.setData(getPointer(fRowData->at(e1))); + + for (c = e1; c >= b1; c--) + { + int currRank = rk[c - b]; + + if (currRank != prevRank) + { + cumeDist = ((double)(prevRank - 1)) / cnt; + + if (cumeDist < fNve) break; - fRow.setData(getPointer(fRowData->at(c))); - - if (fRow.isNullValue(idx)) - continue; - - // ignore nulls - if (b1 == -1) - b1 = c; - - e1 = c; - - if (fFunctionId == WF__PERCENTILE_DISC) - { - // need cume_rank - if (c != b && - fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c - 1)))) - { - dups++; - } - else - { - rank++; - rank += dups; - dups = 0; - } - - rk[c - b] = rank; + prevRank = currRank; } + } + + c++; + + fRow.setData(getPointer(fRowData->at(c))); + getValue(idx, v); + + p = &v; } + } - T* p = NULL; - T v; - int ct = (fFunctionId == WF__PERCENTILE_CONT) ? - CalpontSystemCatalog::DOUBLE : fRow.getColType(idx); + for (c = b; c <= e; c++) + { + if (c % 1000 == 0 && fStep->cancelled()) + break; - if (b1 != -1) - { - double cnt = (e1 - b1 + 1); - - if (fFunctionId == WF__PERCENTILE_CONT) - { - // @bug5820, this "rn" is the normalized row number, not the real row number. - // Using real row number here will introduce a small calculation error in double result. - double rn = fNve * (cnt - 1); - double crn = ceil(rn); - double frn = floor(rn); - double vd = 0; - - if (crn == rn && rn == frn) - { - fRow.setData(getPointer(fRowData->at((size_t) rn + (size_t) b1))); - implicit2T(idx, vd, 0); - } - else - { - double cv = 0.0, fv = 0.0; - fRow.setData(getPointer(fRowData->at((size_t) frn + (size_t) b1))); - implicit2T(idx, fv, 0); - fRow.setData(getPointer(fRowData->at((size_t) crn + (size_t) b1))); - implicit2T(idx, cv, 0); - vd = (crn - rn) * fv + (rn - frn) * cv; - } - - v = *(reinterpret_cast(&vd)); - p = &v; - } - else // (fFunctionId == WF__PERCENTILE_DISC) - { - int prevRank = ++rank + dups; - double cumeDist = 1; - fRow.setData(getPointer(fRowData->at(e1))); - - for (c = e1; c >= b1; c--) - { - int currRank = rk[c - b]; - - if (currRank != prevRank) - { - cumeDist = ((double) (prevRank - 1)) / cnt; - - if (cumeDist < fNve) - break; - - prevRank = currRank; - } - } - - c++; - - fRow.setData(getPointer(fRowData->at(c))); - getValue(idx, v); - - p = &v; - } - } - - for (c = b; c <= e; c++) - { - if (c % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer(fRowData->at(c))); - setValue(ct, b, e, c, p); - } + fRow.setData(getPointer(fRowData->at(c))); + setValue(ct, b, e, c, p); + } } +template boost::shared_ptr WF_percentile::makeFunction(int, const string&, int, + WindowFunctionColumn*); -template -boost::shared_ptr WF_percentile::makeFunction(int, const string&, int, WindowFunctionColumn*); - - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_percentile.h b/utils/windowfunction/wf_percentile.h index ae2c8db50..952f29afc 100644 --- a/utils/windowfunction/wf_percentile.h +++ b/utils/windowfunction/wf_percentile.h @@ -17,45 +17,38 @@ // $Id: wf_percentile.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_PERCENTILE_H #define UTILS_WF_PERCENTILE_H #include #include "windowfunctiontype.h" - namespace windowfunction { - - -template +template class WF_percentile : public WindowFunctionType { -public: - WF_percentile(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_percentile(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); + void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - - double fNve; - bool fNveNull; + protected: + double fNve; + bool fNveNull; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_PERCENTILE_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_ranking.cpp b/utils/windowfunction/wf_ranking.cpp index 818820ee5..f0459b7fe 100644 --- a/utils/windowfunction/wf_ranking.cpp +++ b/utils/windowfunction/wf_ranking.cpp @@ -17,7 +17,6 @@ // $Id: wf_ranking.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -48,110 +47,102 @@ using namespace joblist; #include "wf_ranking.h" - namespace windowfunction { - - -boost::shared_ptr WF_ranking::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +boost::shared_ptr WF_ranking::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func(new WF_ranking(id, name)); - return func; + boost::shared_ptr func(new WF_ranking(id, name)); + return func; } - WindowFunctionType* WF_ranking::clone() const { - return new WF_ranking(*this); + return new WF_ranking(*this); } - void WF_ranking::resetData() { - fRank = 0; - fDups = 0; + fRank = 0; + fDups = 0; - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - void WF_ranking::operator()(int64_t b, int64_t e, int64_t c) { - // one row handling - if (fPartition.first == fPartition.second) + // one row handling + if (fPartition.first == fPartition.second) + { + fRow.setData(getPointer(fRowData->at(fPartition.first))); + int64_t r = (fFunctionId == WF__PERCENT_RANK) ? 0 : 1; + + if (fFunctionId == WF__RANK || fFunctionId == WF__DENSE_RANK) + setIntValue(fFieldIndex[0], r); + else + setDoubleValue(fFieldIndex[0], r); + + return; + } + + // more than one row, e > b + b = fPartition.first; + e = fPartition.second; + double n1 = e - b; // count(*) - 1, n will not be 0. + + for (c = b; c <= e; c++) + { + if (c % 1000 == 0 && fStep->cancelled()) + break; + + if (c != b && fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c - 1)))) { - fRow.setData(getPointer(fRowData->at(fPartition.first))); - int64_t r = (fFunctionId == WF__PERCENT_RANK) ? 0 : 1; + fDups++; + } + else + { + fRank++; - if (fFunctionId == WF__RANK || fFunctionId == WF__DENSE_RANK) - setIntValue(fFieldIndex[0], r); - else - setDoubleValue(fFieldIndex[0], r); + if (fFunctionId != WF__DENSE_RANK) + fRank += fDups; - return; + fDups = 0; } - // more than one row, e > b - b = fPartition.first; - e = fPartition.second; - double n1 = e - b; // count(*) - 1, n will not be 0. + fRow.setData(getPointer(fRowData->at(c))); - for (c = b; c <= e; c++) + if (fFunctionId != WF__PERCENT_RANK) + setIntValue(fFieldIndex[0], fRank); + else + setDoubleValue(fFieldIndex[0], (fRank - 1) / n1); + } + + // Two-pass, need to find peers. + if (fFunctionId == WF__CUME_DIST) + { + int prevRank = ++fRank + fDups; // hypothetical row at (e+1) + double n0 = (e - b + 1); // count(*) + double cumeDist = 1; + fRow.setData(getPointer(fRowData->at(e))); + + for (c = e; c >= b; c--) { - if (c % 1000 == 0 && fStep->cancelled()) - break; + if (c % 1000 == 0 && fStep->cancelled()) + break; - if (c != b && - fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(c - 1)))) - { - fDups++; - } - else - { - fRank++; + fRow.setData(getPointer(fRowData->at(c))); + int currRank = getIntValue(fFieldIndex[0]); - if (fFunctionId != WF__DENSE_RANK) - fRank += fDups; + if (currRank != prevRank) + { + cumeDist = ((double)(prevRank - 1)) / n0; + prevRank = currRank; + } - fDups = 0; - } - - fRow.setData(getPointer(fRowData->at(c))); - - if (fFunctionId != WF__PERCENT_RANK) - setIntValue(fFieldIndex[0], fRank); - else - setDoubleValue(fFieldIndex[0], (fRank - 1) / n1); - } - - // Two-pass, need to find peers. - if (fFunctionId == WF__CUME_DIST) - { - int prevRank = ++fRank + fDups; // hypothetical row at (e+1) - double n0 = (e - b + 1); // count(*) - double cumeDist = 1; - fRow.setData(getPointer(fRowData->at(e))); - - for (c = e; c >= b; c--) - { - if (c % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer(fRowData->at(c))); - int currRank = getIntValue(fFieldIndex[0]); - - if (currRank != prevRank) - { - cumeDist = ((double) (prevRank - 1)) / n0; - prevRank = currRank; - } - - setDoubleValue(fFieldIndex[0], cumeDist); - } + setDoubleValue(fFieldIndex[0], cumeDist); } + } } - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_ranking.h b/utils/windowfunction/wf_ranking.h index ce95bd122..df2e74711 100644 --- a/utils/windowfunction/wf_ranking.h +++ b/utils/windowfunction/wf_ranking.h @@ -17,43 +17,36 @@ // $Id: wf_ranking.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_RANKING_H #define UTILS_WF_RANKING_H #include #include "windowfunctiontype.h" - namespace windowfunction { - - class WF_ranking : public WindowFunctionType { -public: - WF_ranking(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_ranking(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - - uint64_t fRank; - uint64_t fDups; + protected: + uint64_t fRank; + uint64_t fDups; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_RANKING_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_row_number.cpp b/utils/windowfunction/wf_row_number.cpp index 048422614..6453c0a08 100644 --- a/utils/windowfunction/wf_row_number.cpp +++ b/utils/windowfunction/wf_row_number.cpp @@ -17,7 +17,6 @@ // $Id: wf_row_number.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -48,50 +47,43 @@ using namespace joblist; #include "wf_row_number.h" - namespace windowfunction { - - -boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func(new WF_row_number(id, name)); - return func; + boost::shared_ptr func(new WF_row_number(id, name)); + return func; } - WindowFunctionType* WF_row_number::clone() const { - return new WF_row_number(*this); + return new WF_row_number(*this); } - void WF_row_number::resetData() { - fRowNumber = 0; + fRowNumber = 0; - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - void WF_row_number::operator()(int64_t b, int64_t e, int64_t c) { - b = fPartition.first; - e = fPartition.second; + b = fPartition.first; + e = fPartition.second; - for (c = b; c <= e; c++) - { - if (c % 1000 == 0 && fStep->cancelled()) - break; + for (c = b; c <= e; c++) + { + if (c % 1000 == 0 && fStep->cancelled()) + break; - fRow.setData(getPointer(fRowData->at(c))); - fRowNumber++; + fRow.setData(getPointer(fRowData->at(c))); + fRowNumber++; - setIntValue(fFieldIndex[0], fRowNumber); - } + setIntValue(fFieldIndex[0], fRowNumber); + } } - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_row_number.h b/utils/windowfunction/wf_row_number.h index 4b6a61c08..d0976c25f 100644 --- a/utils/windowfunction/wf_row_number.h +++ b/utils/windowfunction/wf_row_number.h @@ -17,42 +17,35 @@ // $Id: wf_row_number.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_ROW_NUMBER_H #define UTILS_WF_ROW_NUMBER_H #include #include "windowfunctiontype.h" - namespace windowfunction { - - class WF_row_number : public WindowFunctionType { -public: - WF_row_number(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_row_number(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - - uint64_t fRowNumber; + protected: + uint64_t fRowNumber; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_ROW_NUMBER_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index 0c871a508..b7a01de79 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -18,7 +18,6 @@ // $Id: wf_stats.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -49,206 +48,195 @@ using namespace joblist; #include "wf_stats.h" - namespace windowfunction { - - -template -boost::shared_ptr WF_stats::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_stats::makeFunction(int id, const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - switch (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - func.reset(new WF_stats(id, name)); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - { - func.reset(new WF_stats(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->functionParms()[0]->resultType().colWidth; - if (width < datatypes::MAXDECIMALWIDTH) - { - if (ct == CalpontSystemCatalog::UDECIMAL) - func.reset(new WF_stats(id, name)); - else - func.reset(new WF_stats(id, name)); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_stats(id, name)); - } - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - func.reset(new WF_stats(id, name)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_stats(id, name)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_stats(id, name)); - break; - } - - default: - { - string errStr = name + "(" + colType2String[ct] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); - - break; - } + func.reset(new WF_stats(id, name)); + break; } - return func; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + func.reset(new WF_stats(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_stats(id, name)); + else + func.reset(new WF_stats(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_stats(id, name)); + } + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + func.reset(new WF_stats(id, name)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_stats(id, name)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_stats(id, name)); + break; + } + + default: + { + string errStr = name + "(" + colType2String[ct] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + + break; + } + } + + return func; } - -template +template WindowFunctionType* WF_stats::clone() const { - return new WF_stats(*this); + return new WF_stats(*this); } - -template +template void WF_stats::resetData() { - fSum1 = 0; - fSum2 = 0; - fCount = 0; - fStats = 0.0; + fSum1 = 0; + fSum2 = 0; + fCount = 0; + fStats = 0.0; - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_stats::operator()(int64_t b, int64_t e, int64_t c) { - CDT cdt; - if ((fFrameUnit == WF__FRAME_ROWS) || - (fPrev == -1) || - (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + CDT cdt; + if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || + (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + { + // for unbounded - current row special handling + if (fPrev >= b && fPrev < c) + b = c; + else if (fPrev <= e && fPrev > c) + e = c; + + uint64_t colIn = fFieldIndex[1]; + + for (int64_t i = b; i <= e; i++) { - // for unbounded - current row special handling - if (fPrev >= b && fPrev < c) - b = c; - else if (fPrev <= e && fPrev > c) - e = c; + if (i % 1000 == 0 && fStep->cancelled()) + break; - uint64_t colIn = fFieldIndex[1]; + fRow.setData(getPointer(fRowData->at(i))); - for (int64_t i = b; i <= e; i++) - { - if (i % 1000 == 0 && fStep->cancelled()) - break; + if (fRow.isNullValue(colIn) == true) + continue; - fRow.setData(getPointer(fRowData->at(i))); + T valIn; + getValue(colIn, valIn, &cdt); + long double val = (long double)valIn; - if (fRow.isNullValue(colIn) == true) - continue; - - T valIn; - getValue(colIn, valIn, &cdt); - long double val = (long double) valIn; - - fSum1 += val; - fSum2 += val * val; - fCount++; - } - - if (fCount > 1) - { - uint32_t scale = fRow.getScale(colIn); - auto factor = datatypes::scaleDivisor(scale); - long double ldSum1 = fSum1; - long double ldSum2 = fSum2; - - // adjust the scale if necessary - if (scale != 0 && - cdt != CalpontSystemCatalog::LONGDOUBLE) - { - ldSum1 /= factor; - ldSum2 /= factor * factor; - } - - long double stat = ldSum1 * ldSum1 / fCount; - stat = ldSum2 - stat; - - if (fFunctionId == WF__STDDEV_POP) - stat = sqrt(stat / fCount); - else if (fFunctionId == WF__STDDEV_SAMP) - stat = sqrt(stat / (fCount - 1)); - else if (fFunctionId == WF__VAR_POP) - stat = stat / fCount; - else if (fFunctionId == WF__VAR_SAMP) - stat = stat / (fCount - 1); - - fStats = (double) stat; - } + fSum1 += val; + fSum2 += val * val; + fCount++; } - if (fCount == 0) + if (fCount > 1) { - setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*) NULL); - } - else if (fCount == 1) - { - if (fFunctionId == WF__STDDEV_SAMP || fFunctionId == WF__VAR_SAMP) - { - setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*) NULL); - } - else // fFunctionId == WF__STDDEV_POP || fFunctionId == WF__VAR_POP - { - double temp = 0.0; - setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*) &temp); - } - } - else - { - setValue(CalpontSystemCatalog::DOUBLE, b, e, c, &fStats); - } + uint32_t scale = fRow.getScale(colIn); + auto factor = datatypes::scaleDivisor(scale); + long double ldSum1 = fSum1; + long double ldSum2 = fSum2; - fPrev = c; + // adjust the scale if necessary + if (scale != 0 && cdt != CalpontSystemCatalog::LONGDOUBLE) + { + ldSum1 /= factor; + ldSum2 /= factor * factor; + } + + long double stat = ldSum1 * ldSum1 / fCount; + stat = ldSum2 - stat; + + if (fFunctionId == WF__STDDEV_POP) + stat = sqrt(stat / fCount); + else if (fFunctionId == WF__STDDEV_SAMP) + stat = sqrt(stat / (fCount - 1)); + else if (fFunctionId == WF__VAR_POP) + stat = stat / fCount; + else if (fFunctionId == WF__VAR_SAMP) + stat = stat / (fCount - 1); + + fStats = (double)stat; + } + } + + if (fCount == 0) + { + setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)NULL); + } + else if (fCount == 1) + { + if (fFunctionId == WF__STDDEV_SAMP || fFunctionId == WF__VAR_SAMP) + { + setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)NULL); + } + else // fFunctionId == WF__STDDEV_POP || fFunctionId == WF__VAR_POP + { + double temp = 0.0; + setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)&temp); + } + } + else + { + setValue(CalpontSystemCatalog::DOUBLE, b, e, c, &fStats); + } + + fPrev = c; } +template boost::shared_ptr WF_stats::makeFunction(int, const string&, int, + WindowFunctionColumn*); -template -boost::shared_ptr WF_stats::makeFunction(int, const string&, int, WindowFunctionColumn*); - - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_stats.h b/utils/windowfunction/wf_stats.h index 6bff926a6..7bfc74b86 100644 --- a/utils/windowfunction/wf_stats.h +++ b/utils/windowfunction/wf_stats.h @@ -17,44 +17,38 @@ // $Id: wf_stats.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_STATS_H #define UTILS_WF_STATS_H #include "windowfunctiontype.h" - namespace windowfunction { - - -template +template class WF_stats : public WindowFunctionType { -public: - WF_stats(int id, const std::string& name) : WindowFunctionType(id, name) - { - resetData(); - } + public: + WF_stats(int id, const std::string& name) : WindowFunctionType(id, name) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - long double fSum1; - long double fSum2; - uint64_t fCount; - double fStats; + protected: + long double fSum1; + long double fSum2; + uint64_t fCount; + double fStats; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_STATS_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index 5a0fbbfc0..d988f0a14 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -18,7 +18,6 @@ // $Id: wf_sum_avg.cpp 3932 2013-06-25 16:08:10Z xlou $ - //#define NDEBUG #include #include @@ -49,275 +48,262 @@ using namespace joblist; #include "wf_sum_avg.h" - namespace windowfunction { - -template -inline void WF_sum_avg::checkSumLimit(const T_IN& val, - const T_OUT& sum) -{ } - -template<> -inline void WF_sum_avg::checkSumLimit(const int128_t& val, - const int128_t& sum) +template +inline void WF_sum_avg::checkSumLimit(const T_IN& val, const T_OUT& sum) { - datatypes::AdditionOverflowCheck ofCheckOp; - ofCheckOp(sum, val); } -template<> -inline void WF_sum_avg::checkSumLimit(const long double& val, - const long double& sum) -{ } +template <> +inline void WF_sum_avg::checkSumLimit(const int128_t& val, const int128_t& sum) +{ + datatypes::AdditionOverflowCheck ofCheckOp; + ofCheckOp(sum, val); +} -template<> -inline void WF_sum_avg::checkSumLimit(const float&, - const long double&) -{ } +template <> +inline void WF_sum_avg::checkSumLimit(const long double& val, + const long double& sum) +{ +} -template<> -inline void WF_sum_avg::checkSumLimit(const long&, - const long double&) -{ } +template <> +inline void WF_sum_avg::checkSumLimit(const float&, const long double&) +{ +} -template<> -inline void WF_sum_avg::checkSumLimit(const unsigned long&, - const long double&) -{ } -template<> -inline void WF_sum_avg::checkSumLimit(const double&, - const long double&) -{ } +template <> +inline void WF_sum_avg::checkSumLimit(const long&, const long double&) +{ +} -template<> -void WF_sum_avg::checkSumLimit(const int128_t&, const int128_t&); -template<> -void WF_sum_avg::checkSumLimit(const long double& val, const long double&); -template<> +template <> +inline void WF_sum_avg::checkSumLimit(const unsigned long&, const long double&) +{ +} +template <> +inline void WF_sum_avg::checkSumLimit(const double&, const long double&) +{ +} + +template <> +void WF_sum_avg::checkSumLimit(const int128_t&, const int128_t&); +template <> +void WF_sum_avg::checkSumLimit(const long double& val, const long double&); +template <> void WF_sum_avg::checkSumLimit(const float&, const long double&); -template<> +template <> void WF_sum_avg::checkSumLimit(const long&, const long double&); -template<> -void WF_sum_avg::checkSumLimit(const unsigned long&, - const long double&); -template<> +template <> +void WF_sum_avg::checkSumLimit(const unsigned long&, const long double&); +template <> void WF_sum_avg::checkSumLimit(const double&, const long double&); -template -int128_t WF_sum_avg::calculateAvg(const int128_t& sum, - const uint64_t count, - const int scale) +template +int128_t WF_sum_avg::calculateAvg(const int128_t& sum, const uint64_t count, const int scale) { - int128_t avg = 0; - int128_t factor; - datatypes::getScaleDivisor(factor, scale); - if (scale > 0) + int128_t avg = 0; + int128_t factor; + datatypes::getScaleDivisor(factor, scale); + if (scale > 0) + { + if ((sum * factor) / factor == sum) { - if ((sum * factor) / factor == sum) - { - avg = sum * factor; - avg /= count; - } - else - { - // scale won't fit before divide, we're gonna lose precision. - avg = sum / count; - if ((avg * factor) / factor != avg) // Still won't fit - { - string errStr = string("AVG(int)"); - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } - avg *= factor; - } + avg = sum * factor; + avg /= count; } else { - avg = sum / count; + // scale won't fit before divide, we're gonna lose precision. + avg = sum / count; + if ((avg * factor) / factor != avg) // Still won't fit + { + string errStr = string("AVG(int)"); + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_OVERFLOW); + } + avg *= factor; } - - avg += (avg < 0) ? (-0.5) : (0.5); - return avg; + } + else + { + avg = sum / count; + } + + avg += (avg < 0) ? (-0.5) : (0.5); + return avg; } -template -inline long double WF_sum_avg::calculateAvg(const long double& sum, - const uint64_t count, - const int scale) +template +inline long double WF_sum_avg::calculateAvg(const long double& sum, const uint64_t count, + const int scale) { - return sum / count; + return sum / count; } // For the static function makeFunction, the template parameters are ignored -template -boost::shared_ptr WF_sum_avg::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) +template +boost::shared_ptr WF_sum_avg::makeFunction(int id, const string& name, + int ct, WindowFunctionColumn* wc) { - boost::shared_ptr func; - switch (ct) + boost::shared_ptr func; + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - // Look into using int128_t instead of long double - func.reset(new WF_sum_avg(id, name)); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - { - func.reset(new WF_sum_avg(id, name)); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - decltype(datatypes::MAXDECIMALWIDTH) width = - wc->functionParms()[0]->resultType().colWidth; - - if (width < datatypes::MAXDECIMALWIDTH) - { - if (ct == CalpontSystemCatalog::UDECIMAL) - func.reset(new WF_sum_avg(id, name)); - else - func.reset(new WF_sum_avg(id, name)); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - func.reset(new WF_sum_avg(id, name)); - } - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - func.reset(new WF_sum_avg(id, name)); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - func.reset(new WF_sum_avg(id, name)); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - func.reset(new WF_sum_avg(id, name)); - break; - } - default: - { - string errStr = name + "(" + colType2String[ct] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); - - break; - } + // Look into using int128_t instead of long double + func.reset(new WF_sum_avg(id, name)); + break; } - return func; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + func.reset(new WF_sum_avg(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = wc->functionParms()[0]->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_sum_avg(id, name)); + else + func.reset(new WF_sum_avg(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_sum_avg(id, name)); + } + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + func.reset(new WF_sum_avg(id, name)); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + func.reset(new WF_sum_avg(id, name)); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + func.reset(new WF_sum_avg(id, name)); + break; + } + default: + { + string errStr = name + "(" + colType2String[ct] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + + break; + } + } + + return func; } - -template +template WindowFunctionType* WF_sum_avg::clone() const { - return new WF_sum_avg(*this); + return new WF_sum_avg(*this); } - -template +template void WF_sum_avg::resetData() { - fAvg = 0; - fSum = 0; - fCount = 0; - fSet.clear(); + fAvg = 0; + fSum = 0; + fCount = 0; + fSet.clear(); - WindowFunctionType::resetData(); + WindowFunctionType::resetData(); } - -template +template void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) { - uint64_t colOut = fFieldIndex[0]; + uint64_t colOut = fFieldIndex[0]; - if ((fFrameUnit == WF__FRAME_ROWS) || - (fPrev == -1) || - (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || + (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + { + // for unbounded - current row special handling + if (fPrev >= b && fPrev < c) + b = c; + else if (fPrev <= e && fPrev > c) + e = c; + + uint64_t colIn = fFieldIndex[1]; + int scale = fRow.getScale(colOut) - fRow.getScale(colIn); + for (int64_t i = b; i <= e; i++) { - // for unbounded - current row special handling - if (fPrev >= b && fPrev < c) - b = c; - else if (fPrev <= e && fPrev > c) - e = c; + if (i % 1000 == 0 && fStep->cancelled()) + break; - uint64_t colIn = fFieldIndex[1]; - int scale = fRow.getScale(colOut) - fRow.getScale(colIn); - for (int64_t i = b; i <= e; i++) - { - if (i % 1000 == 0 && fStep->cancelled()) - break; + fRow.setData(getPointer(fRowData->at(i))); - fRow.setData(getPointer(fRowData->at(i))); + if (fRow.isNullValue(colIn) == true) + continue; - if (fRow.isNullValue(colIn) == true) - continue; + CDT cdt; + getValue(colIn, fVal, &cdt); - CDT cdt; - getValue(colIn, fVal, &cdt); + if ((!fDistinct) || (fSet.find(fVal) == fSet.end())) + { + checkSumLimit(fVal, fSum); + fSum += (T_OUT)fVal; + fCount++; - if ((!fDistinct) || (fSet.find(fVal) == fSet.end())) - { - checkSumLimit(fVal, fSum); - fSum += (T_OUT)fVal; - fCount++; - - if (fDistinct) - fSet.insert(fVal); - } - } - - if ((fCount > 0) && (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT)) - { - fAvg = calculateAvg(fSum, fCount, scale); - } + if (fDistinct) + fSet.insert(fVal); + } } - T_OUT* v = NULL; - - if (fCount > 0) + if ((fCount > 0) && (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT)) { - if (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT) - v = &fAvg; - else - v = &fSum; + fAvg = calculateAvg(fSum, fCount, scale); } + } - setValue(fRow.getColType(colOut), b, e, c, v); + T_OUT* v = NULL; - fPrev = c; + if (fCount > 0) + { + if (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT) + v = &fAvg; + else + v = &fSum; + } + + setValue(fRow.getColType(colOut), b, e, c, v); + + fPrev = c; } +template boost::shared_ptr WF_sum_avg::makeFunction( + int, const string&, int, WindowFunctionColumn*); -template -boost::shared_ptr WF_sum_avg::makeFunction(int, const string&, int, WindowFunctionColumn*); - - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index 0e2a4ffe2..14cbd3560 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -18,53 +18,49 @@ // $Id: wf_sum_avg.h 3868 2013-06-06 22:13:05Z xlou $ - #ifndef UTILS_WF_SUM_AVG_H #define UTILS_WF_SUM_AVG_H #include #include "windowfunctiontype.h" - namespace windowfunction { - // T_IN is the data type of the input values. -// T_OUT is the data type we are using for output and internal values -template +// T_OUT is the data type we are using for output and internal values +template class WF_sum_avg : public WindowFunctionType { -public: - WF_sum_avg(int id, const std::string& name) : - WindowFunctionType(id, name), fDistinct(id != WF__SUM && id != WF__AVG) - { - resetData(); - } + public: + WF_sum_avg(int id, const std::string& name) + : WindowFunctionType(id, name), fDistinct(id != WF__SUM && id != WF__AVG) + { + resetData(); + } - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); -protected: - T_IN fVal; - T_OUT fAvg; - T_OUT fSum; - uint64_t fCount; - bool fDistinct; - std::set fSet; + protected: + T_IN fVal; + T_OUT fAvg; + T_OUT fSum; + uint64_t fCount; + bool fDistinct; + std::set fSet; - void checkSumLimit(const T_IN& val, const T_OUT& sum); + void checkSumLimit(const T_IN& val, const T_OUT& sum); - int128_t calculateAvg(const int128_t& sum, const uint64_t count, const int scale); - long double calculateAvg(const long double& sum, const uint64_t count, const int scale); + int128_t calculateAvg(const int128_t& sum, const uint64_t count, const int scale); + long double calculateAvg(const long double& sum, const uint64_t count, const int scale); }; -} // namespace +} // namespace windowfunction #endif // UTILS_WF_SUM_AVG_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index f9a28f1b5..274c97018 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -17,7 +17,6 @@ 51 Franklin St., Fifth Floor, Boston, MA 02110, USA *************************************************************************************/ - //#define NDEBUG #include #include @@ -52,69 +51,745 @@ using namespace joblist; #include "vlarray.h" - namespace windowfunction { - -boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn* wc) +boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, + mcsv1sdk::mcsv1Context& context, + WindowFunctionColumn* wc) { - boost::shared_ptr func; + boost::shared_ptr func; - func.reset(new WF_udaf(id, name, context)); + func.reset(new WF_udaf(id, name, context)); - // Get the UDAnF function object - WF_udaf* wfUDAF = (WF_udaf*)func.get(); - mcsv1sdk::mcsv1Context& udafContext = wfUDAF->getContext(); - udafContext.setInterrupted(wfUDAF->getInterruptedPtr()); - wfUDAF->resetData(); - return func; + // Get the UDAnF function object + WF_udaf* wfUDAF = (WF_udaf*)func.get(); + mcsv1sdk::mcsv1Context& udafContext = wfUDAF->getContext(); + udafContext.setInterrupted(wfUDAF->getInterruptedPtr()); + wfUDAF->resetData(); + return func; } -WF_udaf::WF_udaf(WF_udaf& rhs) : - WindowFunctionType(rhs.functionId(), rhs.functionName()), - fUDAFContext(rhs.getContext()), - bInterrupted(rhs.getInterrupted()), - fDistinct(rhs.getDistinct()) +WF_udaf::WF_udaf(WF_udaf& rhs) + : WindowFunctionType(rhs.functionId(), rhs.functionName()) + , fUDAFContext(rhs.getContext()) + , bInterrupted(rhs.getInterrupted()) + , fDistinct(rhs.getDistinct()) { - getContext().setInterrupted(getInterruptedPtr()); + getContext().setInterrupted(getInterruptedPtr()); } WindowFunctionType* WF_udaf::clone() const { - return new WF_udaf(*const_cast(this)); + return new WF_udaf(*const_cast(this)); } void WF_udaf::resetData() { - getContext().getFunction()->reset(&getContext()); - fDistinctMap.clear(); - WindowFunctionType::resetData(); + getContext().getFunction()->reset(&getContext()); + fDistinctMap.clear(); + WindowFunctionType::resetData(); } void WF_udaf::parseParms(const std::vector& parms) { - bRespectNulls = true; - // The last parms: respect null | ignore null - ConstantColumn* cc = dynamic_cast(parms[parms.size() - 1].get()); - idbassert(cc != NULL); - bool isNull = false; // dummy, harded coded - bRespectNulls = (cc->getIntVal(fRow, isNull) > 0); - if (getContext().getRunFlag(mcsv1sdk::UDAF_DISTINCT)) - { - setDistinct(); - } + bRespectNulls = true; + // The last parms: respect null | ignore null + ConstantColumn* cc = dynamic_cast(parms[parms.size() - 1].get()); + idbassert(cc != NULL); + bool isNull = false; // dummy, harded coded + bRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + if (getContext().getRunFlag(mcsv1sdk::UDAF_DISTINCT)) + { + setDistinct(); + } } bool WF_udaf::dropValues(int64_t b, int64_t e) { - if (!bHasDropValue) + if (!bHasDropValue) + { + // Save work if we discovered dropValue is not implemented in the UDAnF + return false; + } + + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + bool isNull = false; + + // Turn on the Analytic flag so the function is aware it is being called + // as a Window Function. + getContext().setContextFlag(mcsv1sdk::CONTEXT_IS_ANALYTIC); + + // Put the parameter metadata (type, scale, precision) into valsIn + utils::VLArray valsIn(getContext().getParameterCount()); + ConstantColumn* cc = NULL; + + for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) + { + mcsv1sdk::ColumnDatum& datum = valsIn[i]; + cc = static_cast(fConstantParms[i].get()); + + if (cc) { - // Save work if we discovered dropValue is not implemented in the UDAnF - return false; + datum.dataType = cc->resultType().colDataType; + datum.scale = cc->resultType().scale; + datum.precision = cc->resultType().precision; + } + else + { + uint64_t colIn = fFieldIndex[i + 1]; + datum.dataType = fRow.getColType(colIn); + datum.scale = fRow.getScale(colIn); + datum.precision = fRow.getPrecision(colIn); + } + } + + utils::VLArray flags(getContext().getParameterCount()); + for (int64_t i = b; i < e; i++) + { + if (i % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer(fRowData->at(i))); + + // NULL flags + bool bSkipIt = false; + + for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) + { + cc = static_cast(fConstantParms[k].get()); + uint64_t colIn = fFieldIndex[k + 1]; + mcsv1sdk::ColumnDatum& datum = valsIn[k]; + + // Turn on Null flags or skip based on respect nulls + flags[k] = 0; + if ((!cc && fRow.isNullValue(colIn) == true) || (cc && cc->type() == ConstantColumn::NULLDATA)) + { + if (!bRespectNulls) + { + bSkipIt = true; + break; + } + + flags[k] |= mcsv1sdk::PARAM_IS_NULL; + } + + if (!bSkipIt && !(flags[k] & mcsv1sdk::PARAM_IS_NULL)) + { + switch (datum.dataType) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + { + int64_t valIn; + + if (cc) + { + valIn = cc->getIntVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + int64_t valIn; + + if (cc) + { + valIn = cc->getDecimalVal(fRow, isNull).value; + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + { + uint64_t valIn; + + if (cc) + { + valIn = cc->getUintVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double valIn; + + if (cc) + { + valIn = cc->getDoubleVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float valIn; + + if (cc) + { + valIn = cc->getFloatVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + double valIn; + + if (cc) + { + valIn = cc->getLongDoubleVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::BLOB: + { + string valIn; + + if (cc) + { + valIn = cc->getStrVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0) + { + if (fDistinct) + { + DistinctMap::iterator distinct; + distinct = fDistinctMap.find(valIn); + if (distinct != fDistinctMap.end()) + { + // This is a duplicate: decrement the count + --(*distinct).second; + if ((*distinct).second > 0) // still more of these + { + bSkipIt = true; + continue; + } + else + { + fDistinctMap.erase(distinct); + } + } + } + } + + datum.columnData = valIn; + break; + } + + default: + { + string errStr = "(" + colType2String[(int)datum.dataType] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + + break; + } + } + } } - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - bool isNull = false; + if (bSkipIt) + { + continue; + } + + getContext().setDataFlags(flags); + + rc = getContext().getFunction()->dropValue(&getContext(), valsIn); + + if (rc == mcsv1sdk::mcsv1_UDAF::NOT_IMPLEMENTED) + { + bHasDropValue = false; + return false; + } + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + bInterrupted = true; + string errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_UDANF_ERROR, getContext().getErrorMessage()); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_UDANF_ERROR); + } + } + + WindowFunctionType::resetData(); + return true; +} + +// Sets the value from valOut into column colOut, performing any conversions. +void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t b, int64_t e, int64_t c) +{ + static const static_any::any& charTypeId = (char)1; + static const static_any::any& scharTypeId = (signed char)1; + static const static_any::any& shortTypeId = (short)1; + static const static_any::any& intTypeId = (int)1; + static const static_any::any& longTypeId = (long)1; + static const static_any::any& llTypeId = (long long)1; + static const static_any::any& int128TypeId = (int128_t)1; + static const static_any::any& ucharTypeId = (unsigned char)1; + static const static_any::any& ushortTypeId = (unsigned short)1; + static const static_any::any& uintTypeId = (unsigned int)1; + static const static_any::any& ulongTypeId = (unsigned long)1; + static const static_any::any& ullTypeId = (unsigned long long)1; + static const static_any::any& floatTypeId = (float)1; + static const static_any::any& doubleTypeId = (double)1; + static const std::string typeStr(""); + static const static_any::any& strTypeId = typeStr; + + CDT colDataType = fRow.getColType(colOut); + + // This may seem a bit convoluted. Users shouldn't return a type + // that they didn't set in mcsv1_UDAF::init(), but this + // handles whatever return type is given and casts + // it to whatever they said to return. + int64_t intOut = 0; + uint64_t uintOut = 0; + int128_t int128Out = 0; + float floatOut = 0.0; + double doubleOut = 0.0; + long double longdoubleOut = 0.0; + ostringstream oss; + std::string strOut; + + if (valOut.compatible(charTypeId)) + { + uintOut = intOut = valOut.cast(); + floatOut = intOut; + oss << intOut; + } + else if (valOut.compatible(scharTypeId)) + { + uintOut = intOut = valOut.cast(); + floatOut = intOut; + oss << intOut; + } + else if (valOut.compatible(shortTypeId)) + { + uintOut = intOut = valOut.cast(); + floatOut = intOut; + oss << intOut; + } + else if (valOut.compatible(intTypeId)) + { + uintOut = intOut = valOut.cast(); + floatOut = intOut; + oss << intOut; + } + else if (valOut.compatible(longTypeId)) + { + uintOut = intOut = valOut.cast(); + floatOut = intOut; + oss << intOut; + } + else if (valOut.compatible(llTypeId)) + { + uintOut = intOut = valOut.cast(); + floatOut = intOut; + oss << intOut; + } + else if (valOut.compatible(ucharTypeId)) + { + intOut = uintOut = valOut.cast(); + floatOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(ushortTypeId)) + { + intOut = uintOut = valOut.cast(); + floatOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(uintTypeId)) + { + intOut = uintOut = valOut.cast(); + floatOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(ulongTypeId)) + { + intOut = uintOut = valOut.cast(); + floatOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(ullTypeId)) + { + intOut = uintOut = valOut.cast(); + floatOut = uintOut; + oss << uintOut; + } + else if (valOut.compatible(floatTypeId)) + { + floatOut = valOut.cast(); + doubleOut = floatOut; + longdoubleOut = floatOut; + intOut = uintOut = floatOut; + oss << floatOut; + } + else if (valOut.compatible(doubleTypeId)) + { + doubleOut = valOut.cast(); + longdoubleOut = doubleOut; + floatOut = (float)doubleOut; + uintOut = (uint64_t)doubleOut; + intOut = (int64_t)doubleOut; + oss << doubleOut; + } + else if (valOut.compatible(int128TypeId)) + { + int128Out = valOut.cast(); + uintOut = intOut = int128Out; // may truncate + floatOut = int128Out; + doubleOut = int128Out; + longdoubleOut = int128Out; + oss << longdoubleOut; + } + + if (valOut.compatible(strTypeId)) + { + std::string strOut = valOut.cast(); + // Convert the string to numeric type, just in case. + intOut = atol(strOut.c_str()); + uintOut = strtoul(strOut.c_str(), NULL, 10); + doubleOut = strtod(strOut.c_str(), NULL); + longdoubleOut = doubleOut; + floatOut = (float)doubleOut; + } + else + { + strOut = oss.str(); + } + + switch (colDataType) + { + case execplan::CalpontSystemCatalog::BIT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + if (valOut.empty()) + { + setValue(colDataType, b, e, c, (int64_t*)NULL); + } + else + { + setValue(colDataType, b, e, c, &intOut); + } + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + if (valOut.empty()) + { + setValue(colDataType, b, e, c, (uint64_t*)NULL); + } + else + { + setValue(colDataType, b, e, c, &uintOut); + } + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + if (valOut.empty()) + { + setValue(colDataType, b, e, c, (float*)NULL); + } + else + { + setValue(colDataType, b, e, c, &floatOut); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + if (valOut.empty()) + { + setValue(colDataType, b, e, c, (double*)NULL); + } + else + { + setValue(colDataType, b, e, c, &doubleOut); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + if (valOut.empty()) + { + setValue(colDataType, b, e, c, (long double*)NULL); + } + else + { + setValue(colDataType, b, e, c, &longdoubleOut); + } + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::CLOB: + case execplan::CalpontSystemCatalog::BLOB: + if (valOut.empty()) + { + setValue(colDataType, b, e, c, (string*)NULL); + } + else + { + setValue(colDataType, b, e, c, &strOut); + } + break; + + default: + { + std::ostringstream errmsg; + errmsg << "WF_udaf: No logic for data type: " << colDataType; + cerr << errmsg.str() << endl; + throw runtime_error(errmsg.str().c_str()); + break; + } + } +} + +void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) +{ + mcsv1sdk::mcsv1_UDAF::ReturnCode rc; + uint64_t colOut = fFieldIndex[0]; + bool isNull = false; + + if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || + (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) + { + fValOut.reset(); + // for unbounded - current row special handling + if (fPrev >= b && fPrev < c) + b = c; + else if (fPrev <= e && fPrev > c) + e = c; // Turn on the Analytic flag so the function is aware it is being called // as a Window Function. @@ -126,1074 +801,394 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) { - mcsv1sdk::ColumnDatum& datum = valsIn[i]; - cc = static_cast(fConstantParms[i].get()); + mcsv1sdk::ColumnDatum& datum = valsIn[i]; + cc = static_cast(fConstantParms[i].get()); - if (cc) - { - datum.dataType = cc->resultType().colDataType; - datum.scale = cc->resultType().scale; - datum.precision = cc->resultType().precision; - } - else - { - uint64_t colIn = fFieldIndex[i + 1]; - datum.dataType = fRow.getColType(colIn); - datum.scale = fRow.getScale(colIn); - datum.precision = fRow.getPrecision(colIn); - } + if (cc) + { + datum.dataType = cc->resultType().colDataType; + datum.scale = cc->resultType().scale; + datum.precision = cc->resultType().precision; + } + else + { + uint64_t colIn = fFieldIndex[i + 1]; + datum.dataType = fRow.getColType(colIn); + datum.scale = fRow.getScale(colIn); + datum.precision = fRow.getPrecision(colIn); + } } - utils::VLArray flags(getContext().getParameterCount()); - for (int64_t i = b; i < e; i++) - { - if (i % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer(fRowData->at(i))); - - // NULL flags - bool bSkipIt = false; - - for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) - { - cc = static_cast(fConstantParms[k].get()); - uint64_t colIn = fFieldIndex[k + 1]; - mcsv1sdk::ColumnDatum& datum = valsIn[k]; - - // Turn on Null flags or skip based on respect nulls - flags[k] = 0; - if ((!cc && fRow.isNullValue(colIn) == true) - || (cc && cc->type() == ConstantColumn::NULLDATA)) - { - if (!bRespectNulls) - { - bSkipIt = true; - break; - } - - flags[k] |= mcsv1sdk::PARAM_IS_NULL; - } - - if (!bSkipIt && !(flags[k] & mcsv1sdk::PARAM_IS_NULL)) - { - switch (datum.dataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - int64_t valIn; - - if (cc) - { - valIn = cc->getIntVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - int64_t valIn; - - if (cc) - { - valIn = cc->getDecimalVal(fRow, isNull).value; - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - { - uint64_t valIn; - - if (cc) - { - valIn = cc->getUintVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double valIn; - - if (cc) - { - valIn = cc->getDoubleVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float valIn; - - if (cc) - { - valIn = cc->getFloatVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - double valIn; - - if (cc) - { - valIn = cc->getLongDoubleVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::BLOB: - { - string valIn; - - if (cc) - { - valIn = cc->getStrVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0) - { - if (fDistinct) - { - DistinctMap::iterator distinct; - distinct = fDistinctMap.find(valIn); - if (distinct != fDistinctMap.end()) - { - // This is a duplicate: decrement the count - --(*distinct).second; - if ((*distinct).second > 0) // still more of these - { - bSkipIt = true; - continue; - } - else - { - fDistinctMap.erase(distinct); - } - } - } - } - - datum.columnData = valIn; - break; - } - - default: - { - string errStr = "(" + colType2String[(int)datum.dataType] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); - - break; - } - } - } - } - - if (bSkipIt) - { - continue; - } - - getContext().setDataFlags(flags); - - rc = getContext().getFunction()->dropValue(&getContext(), valsIn); - - if (rc == mcsv1sdk::mcsv1_UDAF::NOT_IMPLEMENTED) - { - bHasDropValue = false; - return false; - } - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - bInterrupted = true; - string errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_UDANF_ERROR, getContext().getErrorMessage()); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_UDANF_ERROR); - } - } - - WindowFunctionType::resetData(); - return true; -} - -// Sets the value from valOut into column colOut, performing any conversions. -void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, - int64_t b, int64_t e, int64_t c) -{ - static const static_any::any& charTypeId = (char)1; - static const static_any::any& scharTypeId = (signed char)1; - static const static_any::any& shortTypeId = (short)1; - static const static_any::any& intTypeId = (int)1; - static const static_any::any& longTypeId = (long)1; - static const static_any::any& llTypeId = (long long)1; - static const static_any::any& int128TypeId = (int128_t)1; - static const static_any::any& ucharTypeId = (unsigned char)1; - static const static_any::any& ushortTypeId = (unsigned short)1; - static const static_any::any& uintTypeId = (unsigned int)1; - static const static_any::any& ulongTypeId = (unsigned long)1; - static const static_any::any& ullTypeId = (unsigned long long)1; - static const static_any::any& floatTypeId = (float)1; - static const static_any::any& doubleTypeId = (double)1; - static const std::string typeStr(""); - static const static_any::any& strTypeId = typeStr; - - CDT colDataType = fRow.getColType(colOut); - - // This may seem a bit convoluted. Users shouldn't return a type - // that they didn't set in mcsv1_UDAF::init(), but this - // handles whatever return type is given and casts - // it to whatever they said to return. - int64_t intOut = 0; - uint64_t uintOut = 0; - int128_t int128Out = 0; - float floatOut = 0.0; - double doubleOut = 0.0; - long double longdoubleOut = 0.0; - ostringstream oss; - std::string strOut; - - if (valOut.compatible(charTypeId)) - { - uintOut = intOut = valOut.cast(); - floatOut = intOut; - oss << intOut; - } - else if (valOut.compatible(scharTypeId)) - { - uintOut = intOut = valOut.cast(); - floatOut = intOut; - oss << intOut; - } - else if (valOut.compatible(shortTypeId)) - { - uintOut = intOut = valOut.cast(); - floatOut = intOut; - oss << intOut; - } - else if (valOut.compatible(intTypeId)) - { - uintOut = intOut = valOut.cast(); - floatOut = intOut; - oss << intOut; - } - else if (valOut.compatible(longTypeId)) - { - uintOut = intOut = valOut.cast(); - floatOut = intOut; - oss << intOut; - } - else if (valOut.compatible(llTypeId)) - { - uintOut = intOut = valOut.cast(); - floatOut = intOut; - oss << intOut; - } - else if (valOut.compatible(ucharTypeId)) - { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(ushortTypeId)) - { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(uintTypeId)) - { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(ulongTypeId)) - { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(ullTypeId)) - { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; - oss << uintOut; - } - else if (valOut.compatible(floatTypeId)) - { - floatOut = valOut.cast(); - doubleOut = floatOut; - longdoubleOut = floatOut; - intOut = uintOut = floatOut; - oss << floatOut; - } - else if (valOut.compatible(doubleTypeId)) - { - doubleOut = valOut.cast(); - longdoubleOut = doubleOut; - floatOut = (float)doubleOut; - uintOut = (uint64_t)doubleOut; - intOut = (int64_t)doubleOut; - oss << doubleOut; - } - else if (valOut.compatible(int128TypeId)) - { - int128Out = valOut.cast(); - uintOut = intOut = int128Out; // may truncate - floatOut = int128Out; - doubleOut = int128Out; - longdoubleOut = int128Out; - oss << longdoubleOut; - } - - if (valOut.compatible(strTypeId)) - { - std::string strOut = valOut.cast(); - // Convert the string to numeric type, just in case. - intOut = atol(strOut.c_str()); - uintOut = strtoul(strOut.c_str(), NULL, 10); - doubleOut = strtod(strOut.c_str(), NULL); - longdoubleOut = doubleOut; - floatOut = (float)doubleOut; - } + if (b <= c && c <= e) + getContext().setContextFlag(mcsv1sdk::CONTEXT_HAS_CURRENT_ROW); else + getContext().clearContextFlag(mcsv1sdk::CONTEXT_HAS_CURRENT_ROW); + + bool bSkipIt = false; + utils::VLArray flags(getContext().getParameterCount()); + for (int64_t i = b; i <= e; i++) { - strOut = oss.str(); - } + if (i % 1000 == 0 && fStep->cancelled()) + break; - switch (colDataType) - { - case execplan::CalpontSystemCatalog::BIT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - if (valOut.empty()) - { - setValue(colDataType, b, e, c, (int64_t*)NULL); - } - else - { - setValue(colDataType, b, e, c, &intOut); - } - break; + fRow.setData(getPointer(fRowData->at(i))); - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::TIME: - if (valOut.empty()) - { - setValue(colDataType, b, e, c, (uint64_t*)NULL); - } - else - { - setValue(colDataType, b, e, c, &uintOut); - } - break; + // NULL flags + bSkipIt = false; - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - if (valOut.empty()) - { - setValue(colDataType, b, e, c, (float*)NULL); - } - else - { - setValue(colDataType, b, e, c, &floatOut); - } - break; + for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) + { + cc = static_cast(fConstantParms[k].get()); + uint64_t colIn = fFieldIndex[k + 1]; + mcsv1sdk::ColumnDatum& datum = valsIn[k]; - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - if (valOut.empty()) - { - setValue(colDataType, b, e, c, (double*)NULL); - } - else - { - setValue(colDataType, b, e, c, &doubleOut); - } - break; + // Turn on Null flags or skip based on respect nulls + flags[k] = 0; - case execplan::CalpontSystemCatalog::LONGDOUBLE: - if (valOut.empty()) - { - setValue(colDataType, b, e, c, (long double*)NULL); - } - else - { - setValue(colDataType, b, e, c, &longdoubleOut); - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::CLOB: - case execplan::CalpontSystemCatalog::BLOB: - if (valOut.empty()) - { - setValue(colDataType, b, e, c, (string*)NULL); - } - else - { - setValue(colDataType, b, e, c, &strOut); - } - break; - - default: + if ((!cc && fRow.isNullValue(colIn) == true) || (cc && cc->type() == ConstantColumn::NULLDATA)) { - std::ostringstream errmsg; - errmsg << "WF_udaf: No logic for data type: " << colDataType; - cerr << errmsg.str() << endl; - throw runtime_error(errmsg.str().c_str()); + if (!bRespectNulls) + { + bSkipIt = true; break; - } - } -} + } -void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) -{ - mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - uint64_t colOut = fFieldIndex[0]; - bool isNull = false; - - if ((fFrameUnit == WF__FRAME_ROWS) || - (fPrev == -1) || - (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) - { - fValOut.reset(); - // for unbounded - current row special handling - if (fPrev >= b && fPrev < c) - b = c; - else if (fPrev <= e && fPrev > c) - e = c; - - // Turn on the Analytic flag so the function is aware it is being called - // as a Window Function. - getContext().setContextFlag(mcsv1sdk::CONTEXT_IS_ANALYTIC); - - // Put the parameter metadata (type, scale, precision) into valsIn - utils::VLArray valsIn(getContext().getParameterCount()); - ConstantColumn* cc = NULL; - - for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) - { - mcsv1sdk::ColumnDatum& datum = valsIn[i]; - cc = static_cast(fConstantParms[i].get()); - - if (cc) - { - datum.dataType = cc->resultType().colDataType; - datum.scale = cc->resultType().scale; - datum.precision = cc->resultType().precision; - } - else - { - uint64_t colIn = fFieldIndex[i + 1]; - datum.dataType = fRow.getColType(colIn); - datum.scale = fRow.getScale(colIn); - datum.precision = fRow.getPrecision(colIn); - } + flags[k] |= mcsv1sdk::PARAM_IS_NULL; } - if (b <= c && c <= e) - getContext().setContextFlag(mcsv1sdk::CONTEXT_HAS_CURRENT_ROW); - else - getContext().clearContextFlag(mcsv1sdk::CONTEXT_HAS_CURRENT_ROW); - - bool bSkipIt = false; - utils::VLArray flags(getContext().getParameterCount()); - for (int64_t i = b; i <= e; i++) + if (!bSkipIt && !(flags[k] & mcsv1sdk::PARAM_IS_NULL)) { - if (i % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer(fRowData->at(i))); - - // NULL flags - bSkipIt = false; - - for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) + switch (datum.dataType) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: { - cc = static_cast(fConstantParms[k].get()); - uint64_t colIn = fFieldIndex[k + 1]; - mcsv1sdk::ColumnDatum& datum = valsIn[k]; + int64_t valIn; - // Turn on Null flags or skip based on respect nulls - flags[k] = 0; + if (cc) + { + valIn = cc->getIntVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } - if ((!cc && fRow.isNullValue(colIn) == true) - || (cc && cc->type() == ConstantColumn::NULLDATA)) + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + // MCOL-1698 + std::pair val = make_pair(valIn, 1); + // Unordered_map will not insert a duplicate key (valIn). + // If it doesn't insert, the original pair will be returned + // in distinct.first and distinct.second will be a bool -- + // true if newly inserted, false if a duplicate. + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) { - if (!bRespectNulls) - { - bSkipIt = true; - break; - } + // This is a duplicate: increment the count + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } - flags[k] |= mcsv1sdk::PARAM_IS_NULL; + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (fRow.getColumnWidth(colIn) < 16) + { + int64_t valIn; + + if (cc) + { + valIn = cc->getDecimalVal(fRow, isNull).value; + } + else + { + getValue(colIn, valIn); } - if (!bSkipIt && !(flags[k] & mcsv1sdk::PARAM_IS_NULL)) + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) { - switch (datum.dataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - int64_t valIn; - - if (cc) - { - valIn = cc->getIntVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - // MCOL-1698 - std::pair val = make_pair(valIn, 1); - // Unordered_map will not insert a duplicate key (valIn). - // If it doesn't insert, the original pair will be returned - // in distinct.first and distinct.second will be a bool -- - // true if newly inserted, false if a duplicate. - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - // This is a duplicate: increment the count - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fRow.getColumnWidth(colIn) < 16) - { - int64_t valIn; - - if (cc) - { - valIn = cc->getDecimalVal(fRow, isNull).value; - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - } - else - { - int128_t valIn; - - if (cc) - { - valIn = cc->getDecimalVal(fRow, isNull).s128Value; - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - } - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - { - uint64_t valIn; - - if (cc) - { - valIn = cc->getUintVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double valIn; - - if (cc) - { - valIn = cc->getDoubleVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float valIn; - - if (cc) - { - valIn = cc->getFloatVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double valIn; - - if (cc) - { - valIn = cc->getLongDoubleVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::BLOB: - { - string valIn; - - if (cc) - { - valIn = cc->getStrVal(fRow, isNull); - } - else - { - getValue(colIn, valIn); - } - - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) - { - ++(*distinct.first).second; - bSkipIt = true; - continue; - } - } - - datum.columnData = valIn; - break; - } - - default: - { - string errStr = "(" + colType2String[(int)datum.dataType] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); - - break; - } - } + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } } + + datum.columnData = valIn; + } + else + { + int128_t valIn; + + if (cc) + { + valIn = cc->getDecimalVal(fRow, isNull).s128Value; + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + } + break; } - // Skip if any value is NULL and respect nulls is off. - if (bSkipIt) + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: { - continue; + uint64_t valIn; + + if (cc) + { + valIn = cc->getUintVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + break; } - getContext().setDataFlags(flags); - - rc = getContext().getFunction()->nextValue(&getContext(), valsIn); - - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: { - bInterrupted = true; - string errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_UDANF_ERROR, getContext().getErrorMessage()); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_UDANF_ERROR); + double valIn; + + if (cc) + { + valIn = cc->getDoubleVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + break; } - } - rc = getContext().getFunction()->evaluate(&getContext(), fValOut); + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float valIn; - if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) - { - bInterrupted = true; - string errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_UDANF_ERROR, getContext().getErrorMessage()); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_UDANF_ERROR); + if (cc) + { + valIn = cc->getFloatVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double valIn; + + if (cc) + { + valIn = cc->getLongDoubleVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::BLOB: + { + string valIn; + + if (cc) + { + valIn = cc->getStrVal(fRow, isNull); + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + break; + } + + default: + { + string errStr = "(" + colType2String[(int)datum.dataType] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + + break; + } + } } + } + + // Skip if any value is NULL and respect nulls is off. + if (bSkipIt) + { + continue; + } + + getContext().setDataFlags(flags); + + rc = getContext().getFunction()->nextValue(&getContext(), valsIn); + + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + bInterrupted = true; + string errStr = + IDBErrorInfo::instance()->errorMsg(ERR_WF_UDANF_ERROR, getContext().getErrorMessage()); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_UDANF_ERROR); + } } - SetUDAFValue(fValOut, colOut, b, e, c); + rc = getContext().getFunction()->evaluate(&getContext(), fValOut); - fPrev = c; + if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) + { + bInterrupted = true; + string errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_UDANF_ERROR, getContext().getErrorMessage()); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_UDANF_ERROR); + } + } + + SetUDAFValue(fValOut, colOut, b, e, c); + + fPrev = c; } -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index eca42588f..ab777761e 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -17,7 +17,6 @@ 51 Franklin St., Fifth Floor, Boston, MA 02110, USA *************************************************************************************/ - #ifndef UTILS_WF_UDAF_H #define UTILS_WF_UDAF_H @@ -29,91 +28,91 @@ #include "windowfunctiontype.h" #include "mcsv1_udaf.h" - namespace windowfunction { // Hash classes for the distinct hashmap class DistinctHasher { -public: - inline size_t operator()(const static_any::any& a) const - { - return a.getHash(); - } + public: + inline size_t operator()(const static_any::any& a) const + { + return a.getHash(); + } }; class DistinctEqual { -public: - inline bool operator()(const static_any::any lhs, static_any::any rhs) const - { - return lhs == rhs; - } + public: + inline bool operator()(const static_any::any lhs, static_any::any rhs) const + { + return lhs == rhs; + } }; // A class to control the execution of User Define Analytic Functions (UDAnF) // as defined by a specialization of mcsv1sdk::mcsv1_UDAF class WF_udaf : public WindowFunctionType { -public: - WF_udaf(int id, const std::string& name, mcsv1sdk::mcsv1Context& context) : - WindowFunctionType(id, name), fUDAFContext(context), fDistinct(false), bHasDropValue(true) {} - WF_udaf(WF_udaf& rhs); - // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); - virtual bool dropValues(int64_t, int64_t); + public: + WF_udaf(int id, const std::string& name, mcsv1sdk::mcsv1Context& context) + : WindowFunctionType(id, name), fUDAFContext(context), fDistinct(false), bHasDropValue(true) + { + } + WF_udaf(WF_udaf& rhs); + // pure virtual in base + void operator()(int64_t b, int64_t e, int64_t c); + WindowFunctionType* clone() const; + void resetData(); + void parseParms(const std::vector&); + virtual bool dropValues(int64_t, int64_t); - mcsv1sdk::mcsv1Context& getContext() - { - return fUDAFContext; - } + mcsv1sdk::mcsv1Context& getContext() + { + return fUDAFContext; + } - bool getInterrupted() - { - return bInterrupted; - } + bool getInterrupted() + { + return bInterrupted; + } - bool * getInterruptedPtr() - { - return &bInterrupted; - } + bool* getInterruptedPtr() + { + return &bInterrupted; + } - bool getDistinct() - { - return fDistinct; - } + bool getDistinct() + { + return fDistinct; + } - void setDistinct(bool d = true) - { - fDistinct = d; - } + void setDistinct(bool d = true) + { + fDistinct = d; + } -protected: - void SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t b, int64_t e, int64_t c); + protected: + void SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t b, int64_t e, int64_t c); - mcsv1sdk::mcsv1Context fUDAFContext; // The UDAF context - bool bInterrupted; // Shared by all the threads - bool fDistinct; - bool bRespectNulls; // respect null | ignore null - bool bHasDropValue; // Set to false when we discover the UDAnF doesn't implement dropValue. - // To hold distinct values and their counts - typedef std::tr1::unordered_map DistinctMap; - DistinctMap fDistinctMap; + mcsv1sdk::mcsv1Context fUDAFContext; // The UDAF context + bool bInterrupted; // Shared by all the threads + bool fDistinct; + bool bRespectNulls; // respect null | ignore null + bool bHasDropValue; // Set to false when we discover the UDAnF doesn't implement dropValue. + // To hold distinct values and their counts + typedef std::tr1::unordered_map DistinctMap; + DistinctMap fDistinctMap; - static_any::any fValOut; // The return value + static_any::any fValOut; // The return value -public: - static boost::shared_ptr makeFunction(int id, const string& name, - int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn* wc); + public: + static boost::shared_ptr makeFunction(int id, const string& name, int ct, + mcsv1sdk::mcsv1Context& context, + WindowFunctionColumn* wc); }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WF_UDAF_H // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/windowframe.cpp b/utils/windowfunction/windowframe.cpp index e8213df99..cbc9352c6 100644 --- a/utils/windowfunction/windowframe.cpp +++ b/utils/windowfunction/windowframe.cpp @@ -18,7 +18,6 @@ MA 02110-1301, USA. */ - //#define NDEBUG #include #include @@ -40,52 +39,46 @@ using namespace rowgroup; #include "windowframe.h" - namespace { string UnitStr[] = {"ROWS", "RANGE"}; } - namespace windowfunction { - - pair WindowFrame::getWindow(int64_t b, int64_t e, int64_t c) { - int64_t upper = fUpper->getBound(b, e, c); - int64_t lower = fLower->getBound(b, e, c); + int64_t upper = fUpper->getBound(b, e, c); + int64_t lower = fLower->getBound(b, e, c); - // case 1 || case 2 || case 3 - if ((upper > lower) || (upper < b && lower < b) || (upper > e && lower > e)) - { - // construct an empty window - upper = b + 1; - lower = b; - } + // case 1 || case 2 || case 3 + if ((upper > lower) || (upper < b && lower < b) || (upper > e && lower > e)) + { + // construct an empty window + upper = b + 1; + lower = b; + } - if (upper < b) // case 2, lower >= b - { - upper = b; - } + if (upper < b) // case 2, lower >= b + { + upper = b; + } - if (lower > e) // case 3, upper <= e - { - lower = e; - } + if (lower > e) // case 3, upper <= e + { + lower = e; + } - return make_pair(upper, lower); + return make_pair(upper, lower); } - const string WindowFrame::toString() const { - string ret(UnitStr[fUnit]); - ret = ret + " between " + fUpper->toString() + " and " + fLower->toString(); + string ret(UnitStr[fUnit]); + ret = ret + " between " + fUpper->toString() + " and " + fLower->toString(); - return ret; + return ret; } -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/windowframe.h b/utils/windowfunction/windowframe.h index 55996a13f..159ee84d2 100644 --- a/utils/windowfunction/windowframe.h +++ b/utils/windowfunction/windowframe.h @@ -17,113 +17,108 @@ // $Id: windowframe.h 3932 2013-06-25 16:08:10Z xlou $ - #ifndef UTILS_WINDOWFRAME_H #define UTILS_WINDOWFRAME_H -#include // std::pair, std::make_pair +#include // std::pair, std::make_pair #include #include "framebound.h" namespace windowfunction { - -const int64_t WF__FRAME_ROWS = 0; +const int64_t WF__FRAME_ROWS = 0; const int64_t WF__FRAME_RANGE = 1; - /** @brief class WindowFrame * */ class WindowFrame { -public: - /** @brief WindowFrame constructor - */ - WindowFrame(int64_t t, boost::shared_ptr& u, boost::shared_ptr& l) : - fUnit(t), fUpper(u), fLower(l) - {} + public: + /** @brief WindowFrame constructor + */ + WindowFrame(int64_t t, boost::shared_ptr& u, boost::shared_ptr& l) + : fUnit(t), fUpper(u), fLower(l) + { + } - /** @brief WindowFrame copy constructor - */ - WindowFrame(const WindowFrame& rhs) : - fUnit(rhs.fUnit), - fUpper(rhs.fUpper->clone()), - fLower(rhs.fLower->clone()) - {} + /** @brief WindowFrame copy constructor + */ + WindowFrame(const WindowFrame& rhs) + : fUnit(rhs.fUnit), fUpper(rhs.fUpper->clone()), fLower(rhs.fLower->clone()) + { + } - /** @brief WindowFrame destructor - */ - virtual ~WindowFrame() {}; + /** @brief WindowFrame destructor + */ + virtual ~WindowFrame(){}; - /** @brief clone - */ - virtual WindowFrame* clone() - { - return new WindowFrame(*this); - }; + /** @brief clone + */ + virtual WindowFrame* clone() + { + return new WindowFrame(*this); + }; - /** @brief virtual void getWindow - */ - std::pair getWindow(int64_t, int64_t, int64_t); + /** @brief virtual void getWindow + */ + std::pair getWindow(int64_t, int64_t, int64_t); - const std::string toString() const; + const std::string toString() const; - /** @brief set methods - */ - void setRowMetaData(const rowgroup::RowGroup& g, const rowgroup::Row& r) - { - fUpper->setRowMetaData(g, r); - fLower->setRowMetaData(g, r); - } - void setRowData(boost::shared_ptr >& d) - { - fUpper->setRowData(d); - fLower->setRowData(d); - } - void setCallback(joblist::WindowFunctionStep* s) - { - fUpper->setCallback(s); - fLower->setCallback(s); - } + /** @brief set methods + */ + void setRowMetaData(const rowgroup::RowGroup& g, const rowgroup::Row& r) + { + fUpper->setRowMetaData(g, r); + fLower->setRowMetaData(g, r); + } + void setRowData(boost::shared_ptr >& d) + { + fUpper->setRowData(d); + fLower->setRowData(d); + } + void setCallback(joblist::WindowFunctionStep* s) + { + fUpper->setCallback(s); + fLower->setCallback(s); + } - int64_t unit() const - { - return fUnit; - } - void unit(int64_t t) - { - fUnit = t; - } - const boost::shared_ptr& upper() const - { - return fUpper; - } - void upper(const boost::shared_ptr& u) - { - fUpper = u; - } - const boost::shared_ptr& lower() const - { - return fLower; - } - void lower(const boost::shared_ptr& l) - { - fLower = l; - } + int64_t unit() const + { + return fUnit; + } + void unit(int64_t t) + { + fUnit = t; + } + const boost::shared_ptr& upper() const + { + return fUpper; + } + void upper(const boost::shared_ptr& u) + { + fUpper = u; + } + const boost::shared_ptr& lower() const + { + return fLower; + } + void lower(const boost::shared_ptr& l) + { + fLower = l; + } -protected: + protected: + // type + int64_t fUnit; - // type - int64_t fUnit; - - // data - boost::shared_ptr fUpper; - boost::shared_ptr fLower; + // data + boost::shared_ptr fUpper; + boost::shared_ptr fLower; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WINDOWFRAME_H diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index 80a6c2ba5..a0e844933 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -45,232 +45,218 @@ using namespace joblist; #include "windowframe.h" #include "windowfunction.h" - namespace windowfunction { - WindowFunction::WindowFunction(boost::shared_ptr& f, boost::shared_ptr& p, - boost::shared_ptr& o, - boost::shared_ptr& w, - const RowGroup& g, - const Row& r) : - fFunctionType(f), fPartitionBy(p), fOrderBy(o), fFrame(w), fRowGroup(g), fRow(r) + boost::shared_ptr& o, boost::shared_ptr& w, + const RowGroup& g, const Row& r) + : fFunctionType(f), fPartitionBy(p), fOrderBy(o), fFrame(w), fRowGroup(g), fRow(r) { } - WindowFunction::~WindowFunction() { } - void WindowFunction::operator()() { - try + try + { + fRowData.reset(new vector(fStep->getRowData())); + + if (fOrderBy->rule().fCompares.size() > 0) + sort(fRowData->begin(), fRowData->size()); + + // get partitions + if (fPartitionBy.get() != NULL && !fStep->cancelled()) { - fRowData.reset(new vector(fStep->getRowData())); + int64_t i = 0; + int64_t j = 1; + int64_t rowCnt = fRowData->size(); - if (fOrderBy->rule().fCompares.size() > 0) - sort(fRowData->begin(), fRowData->size()); + for (j = 1; j < rowCnt; j++) + { + if ((*(fPartitionBy.get()))(getPointer((*fRowData)[j - 1]), getPointer((*fRowData)[j]))) + continue; - // get partitions - if (fPartitionBy.get() != NULL && !fStep->cancelled()) + fPartition.push_back(make_pair(i, j - 1)); + i = j; + } + + fPartition.push_back(make_pair(i, j - 1)); + } + else + { + fPartition.push_back(make_pair(0, fRowData->size())); + } + + // compute partition by partition + int64_t uft = fFrame->upper()->boundType(); + int64_t lft = fFrame->lower()->boundType(); + bool upperUbnd = (uft == WF__UNBOUNDED_PRECEDING || uft == WF__UNBOUNDED_FOLLOWING); + bool lowerUbnd = (lft == WF__UNBOUNDED_PRECEDING || lft == WF__UNBOUNDED_FOLLOWING); + bool upperCnrw = (uft == WF__CURRENT_ROW); + bool lowerCnrw = (lft == WF__CURRENT_ROW); + fFunctionType->setRowData(fRowData); + fFunctionType->setRowMetaData(fRowGroup, fRow); + fFrame->setRowData(fRowData); + fFrame->setRowMetaData(fRowGroup, fRow); + + for (uint64_t k = 0; k < fPartition.size() && !fStep->cancelled(); k++) + { + fFunctionType->resetData(); + fFunctionType->partition(fPartition[k]); + + int64_t begin = fPartition[k].first; + int64_t end = fPartition[k].second; + + if (upperUbnd && lowerUbnd) + { + fFunctionType->operator()(begin, end, WF__BOUND_ALL); + } + else if (upperUbnd && lowerCnrw) + { + if (fFrame->unit() == WF__FRAME_ROWS) { - int64_t i = 0; - int64_t j = 1; - int64_t rowCnt = fRowData->size(); - - for (j = 1; j < rowCnt; j++) - { - if ((*(fPartitionBy.get())) - (getPointer((*fRowData)[j - 1]), getPointer((*fRowData)[j]))) - continue; - - fPartition.push_back(make_pair(i, j - 1)); - i = j; - } - - fPartition.push_back(make_pair(i, j - 1)); + for (int64_t i = begin; i <= end && !fStep->cancelled(); i++) + { + fFunctionType->operator()(begin, i, i); + } } else { - fPartition.push_back(make_pair(0, fRowData->size())); + for (int64_t i = begin; i <= end && !fStep->cancelled(); i++) + { + pair w = fFrame->getWindow(begin, end, i); + int64_t j = i; + + if (w.second > i) + j = w.second; + + fFunctionType->operator()(begin, j, i); + } } - - // compute partition by partition - int64_t uft = fFrame->upper()->boundType(); - int64_t lft = fFrame->lower()->boundType(); - bool upperUbnd = (uft == WF__UNBOUNDED_PRECEDING || uft == WF__UNBOUNDED_FOLLOWING); - bool lowerUbnd = (lft == WF__UNBOUNDED_PRECEDING || lft == WF__UNBOUNDED_FOLLOWING); - bool upperCnrw = (uft == WF__CURRENT_ROW); - bool lowerCnrw = (lft == WF__CURRENT_ROW); - fFunctionType->setRowData(fRowData); - fFunctionType->setRowMetaData(fRowGroup, fRow); - fFrame->setRowData(fRowData); - fFrame->setRowMetaData(fRowGroup, fRow); - - for (uint64_t k = 0; k < fPartition.size() && !fStep->cancelled(); k++) + } + else if (upperCnrw && lowerUbnd) + { + if (fFrame->unit() == WF__FRAME_ROWS) { - fFunctionType->resetData(); - fFunctionType->partition(fPartition[k]); - - int64_t begin = fPartition[k].first; - int64_t end = fPartition[k].second; - - if (upperUbnd && lowerUbnd) - { - fFunctionType->operator()(begin, end, WF__BOUND_ALL); - } - else if (upperUbnd && lowerCnrw) - { - if (fFrame->unit() == WF__FRAME_ROWS) - { - for (int64_t i = begin; i <= end && !fStep->cancelled(); i++) - { - fFunctionType->operator()(begin, i, i); - } - } - else - { - for (int64_t i = begin; i <= end && !fStep->cancelled(); i++) - { - pair w = fFrame->getWindow(begin, end, i); - int64_t j = i; - - if (w.second > i) - j = w.second; - - fFunctionType->operator()(begin, j, i); - } - } - } - else if (upperCnrw && lowerUbnd) - { - if (fFrame->unit() == WF__FRAME_ROWS) - { - for (int64_t i = end; i >= begin && !fStep->cancelled(); i--) - { - fFunctionType->operator()(i, end, i); - } - } - else - { - for (int64_t i = end; i >= begin && !fStep->cancelled(); i--) - { - pair w = fFrame->getWindow(begin, end, i); - int64_t j = i; - - if (w.first < i) - j = w.first; - - fFunctionType->operator()(j, end, i); - } - } - } - else - { - pair w; - pair prevFrame; - int64_t b, e; - bool firstTime = true; - - for (int64_t i = begin; i <= end && !fStep->cancelled(); i++) - { - w = fFrame->getWindow(begin, end, i); - b = w.first; - e = w.second; - - if (firstTime) - { - prevFrame = w; - } - - // UDAnF functions may have a dropValue function implemented. - // If they do, we can optimize by calling dropValue() for those - // values leaving the window and nextValue for those entering, rather - // than a resetData() and then iterating over the entire window. - // Built-in functions may have this functionality added in the future. - // If b > e then the frame is entirely outside of the partition - // and there's no values to drop - if (!firstTime && (b <= e) && fFunctionType->dropValues(prevFrame.first, w.first)) - { - // Adjust the beginning of the frame for nextValue - // to start where the previous frame left off. - b = prevFrame.second + 1; - } - else - { - // If dropValues failed or doesn't exist, - // calculate the entire frame. - fFunctionType->resetData(); - } - fFunctionType->operator()(b, e, i); // UDAnF: Calls nextValue and evaluate - prevFrame = w; - firstTime = false; - } - } + for (int64_t i = end; i >= begin && !fStep->cancelled(); i--) + { + fFunctionType->operator()(i, end, i); + } } - } - catch (...) - { - fStep->handleException(std::current_exception(), - logging::ERR_EXECUTE_WINDOW_FUNCTION, - logging::ERR_WF_DATA_SET_TOO_BIG, - "WindowFunction::operator()"); - } -} + else + { + for (int64_t i = end; i >= begin && !fStep->cancelled(); i--) + { + pair w = fFrame->getWindow(begin, end, i); + int64_t j = i; + if (w.first < i) + j = w.first; + + fFunctionType->operator()(j, end, i); + } + } + } + else + { + pair w; + pair prevFrame; + int64_t b, e; + bool firstTime = true; + + for (int64_t i = begin; i <= end && !fStep->cancelled(); i++) + { + w = fFrame->getWindow(begin, end, i); + b = w.first; + e = w.second; + + if (firstTime) + { + prevFrame = w; + } + + // UDAnF functions may have a dropValue function implemented. + // If they do, we can optimize by calling dropValue() for those + // values leaving the window and nextValue for those entering, rather + // than a resetData() and then iterating over the entire window. + // Built-in functions may have this functionality added in the future. + // If b > e then the frame is entirely outside of the partition + // and there's no values to drop + if (!firstTime && (b <= e) && fFunctionType->dropValues(prevFrame.first, w.first)) + { + // Adjust the beginning of the frame for nextValue + // to start where the previous frame left off. + b = prevFrame.second + 1; + } + else + { + // If dropValues failed or doesn't exist, + // calculate the entire frame. + fFunctionType->resetData(); + } + fFunctionType->operator()(b, e, i); // UDAnF: Calls nextValue and evaluate + prevFrame = w; + firstTime = false; + } + } + } + } + catch (...) + { + fStep->handleException(std::current_exception(), logging::ERR_EXECUTE_WINDOW_FUNCTION, + logging::ERR_WF_DATA_SET_TOO_BIG, "WindowFunction::operator()"); + } +} void WindowFunction::setCallback(joblist::WindowFunctionStep* step, int id) { - fStep = step; - fId = id; - fFunctionType->setCallback(step); - fFrame->setCallback(step); + fStep = step; + fId = id; + fFunctionType->setCallback(step); + fFrame->setCallback(step); } - const Row& WindowFunction::getRow() const { - return fRow; + return fRow; } - void WindowFunction::sort(std::vector::iterator v, uint64_t n) { - // recursive function termination condition. - if (n < 2 || fStep->cancelled()) - return; + // recursive function termination condition. + if (n < 2 || fStep->cancelled()) + return; - RowPosition p = *(v + n / 2); // pivot value - vector::iterator l = v; // low address - vector::iterator h = v + (n - 1); // high address + RowPosition p = *(v + n / 2); // pivot value + vector::iterator l = v; // low address + vector::iterator h = v + (n - 1); // high address - while (l <= h && !(fStep->cancelled())) + while (l <= h && !(fStep->cancelled())) + { + // Can use while here, but need check boundary and cancel status. + if (fOrderBy->operator()(getPointer(*l), getPointer(p))) { - // Can use while here, but need check boundary and cancel status. - if (fOrderBy->operator()(getPointer(*l), getPointer(p))) - { - l++; - } - else if (fOrderBy->operator()(getPointer(p), getPointer(*h))) - { - h--; - } - else - { - RowPosition t = *l; // temp value for swap - *l++ = *h; - *h-- = t; - } + l++; } + else if (fOrderBy->operator()(getPointer(p), getPointer(*h))) + { + h--; + } + else + { + RowPosition t = *l; // temp value for swap + *l++ = *h; + *h-- = t; + } + } - sort(v, std::distance(v, h) + 1); - sort(l, std::distance(l, v) + n); + sort(v, std::distance(v, h) + 1); + sort(l, std::distance(l, v) + n); } - -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/windowfunction.h b/utils/windowfunction/windowfunction.h index 684501b19..53f5b9a41 100644 --- a/utils/windowfunction/windowfunction.h +++ b/utils/windowfunction/windowfunction.h @@ -17,7 +17,6 @@ // $Id: windowfunction.h 3932 2013-06-25 16:08:10Z xlou $ - #ifndef UTILS_WINDOWFUNCTION_H #define UTILS_WINDOWFUNCTION_H @@ -34,93 +33,84 @@ namespace ordering // forward reference class EqualCompData; class OrderByData; -}; - +}; // namespace ordering namespace windowfunction { - // forward reference class WindowFunctionType; class WindowFrame; - /** @brief class WindowFunction * */ class WindowFunction { -public: - /** @brief WindowFunction constructor - * @param f: shared pointer to a WindowFuncitonType - * @param p: shared pointer to equal compare functor - * @param o: shared pointer to order by functor - * @param w: shared pointer to window specification - * @param r: row meta data - */ - WindowFunction(boost::shared_ptr& f, - boost::shared_ptr& p, - boost::shared_ptr& o, - boost::shared_ptr& w, - const rowgroup::RowGroup& g, - const rowgroup::Row& r); + public: + /** @brief WindowFunction constructor + * @param f: shared pointer to a WindowFuncitonType + * @param p: shared pointer to equal compare functor + * @param o: shared pointer to order by functor + * @param w: shared pointer to window specification + * @param r: row meta data + */ + WindowFunction(boost::shared_ptr& f, boost::shared_ptr& p, + boost::shared_ptr& o, boost::shared_ptr& w, + const rowgroup::RowGroup& g, const rowgroup::Row& r); - /** @brief WindowFunction destructor - */ - virtual ~WindowFunction(); + /** @brief WindowFunction destructor + */ + virtual ~WindowFunction(); - /** @brief virtual void Run method - */ - void operator()(); + /** @brief virtual void Run method + */ + void operator()(); - const std::string toString() const; + const std::string toString() const; - void setCallback(joblist::WindowFunctionStep*, int); - const rowgroup::Row& getRow() const; + void setCallback(joblist::WindowFunctionStep*, int); + const rowgroup::Row& getRow() const; + protected: + // cancellable sort function + void sort(std::vector::iterator, uint64_t); -protected: + // special window frames + void processUnboundedWindowFrame1(); + void processUnboundedWindowFrame2(); + void processUnboundedWindowFrame3(); + void processExprWindowFrame(); - // cancellable sort function - void sort(std::vector::iterator, uint64_t); + // for string table + rowgroup::Row::Pointer getPointer(joblist::RowPosition& r) + { + return fStep->getPointer(r, fRowGroup, fRow); + } - // special window frames - void processUnboundedWindowFrame1(); - void processUnboundedWindowFrame2(); - void processUnboundedWindowFrame3(); - void processExprWindowFrame(); + // function type + boost::shared_ptr fFunctionType; - // for string table - rowgroup::Row::Pointer getPointer(joblist::RowPosition& r) - { - return fStep->getPointer(r, fRowGroup, fRow); - } + // window clause + boost::shared_ptr fPartitionBy; + boost::shared_ptr fOrderBy; + boost::shared_ptr fFrame; + std::vector > fPartition; - // function type - boost::shared_ptr fFunctionType; + // data + boost::shared_ptr > fRowData; - // window clause - boost::shared_ptr fPartitionBy; - boost::shared_ptr fOrderBy; - boost::shared_ptr fFrame; - std::vector > fPartition; + // row meta data + rowgroup::RowGroup fRowGroup; + rowgroup::Row fRow; - // data - boost::shared_ptr > fRowData; + // pointer back to step + joblist::WindowFunctionStep* fStep; + int fId; - // row meta data - rowgroup::RowGroup fRowGroup; - rowgroup::Row fRow; - - // pointer back to step - joblist::WindowFunctionStep* fStep; - int fId; - - friend class joblist::WindowFunctionStep; + friend class joblist::WindowFunctionStep; }; - -} // namespace +} // namespace windowfunction #endif // UTILS_WINDOWFUNCTION_H diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index d66c6c485..ccf74609d 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -27,8 +27,8 @@ using namespace std; #include #include -#include // for map_list_of() -#include // for to_upper_copy +#include // for map_list_of() +#include // for to_upper_copy using namespace boost; #include "loggingid.h" @@ -61,565 +61,569 @@ using namespace joblist; namespace windowfunction { - -map colType2String = assign::map_list_of - (CalpontSystemCatalog::BIT, "BIT") - (CalpontSystemCatalog::TINYINT, "TINYINT") - (CalpontSystemCatalog::CHAR, "CHAR") - (CalpontSystemCatalog::SMALLINT, "SMALLINT") - (CalpontSystemCatalog::DECIMAL, "DECIMAL") - (CalpontSystemCatalog::MEDINT, "MEDINT") - (CalpontSystemCatalog::INT, "INT") - (CalpontSystemCatalog::FLOAT, "FLOAT") - (CalpontSystemCatalog::DATE, "DATE") - (CalpontSystemCatalog::BIGINT, "BIGINT") - (CalpontSystemCatalog::DOUBLE, "DOUBLE") - (CalpontSystemCatalog::DATETIME, "DATETIME") - (CalpontSystemCatalog::VARCHAR, "VARCHAR") - (CalpontSystemCatalog::VARBINARY, "VARBINARY") - (CalpontSystemCatalog::CLOB, "CLOB") - (CalpontSystemCatalog::BLOB, "BLOB") - (CalpontSystemCatalog::UTINYINT, "UNSIGNED TINYINT") - (CalpontSystemCatalog::USMALLINT, "UNSIGNED SMALLINT") - (CalpontSystemCatalog::UDECIMAL, "UNSIGNED DECIMAL") - (CalpontSystemCatalog::UMEDINT, "UNSIGNED MEDINT") - (CalpontSystemCatalog::UINT, "UNSIGNED INT") - (CalpontSystemCatalog::UFLOAT, "UNSIGNED FLOAT") - (CalpontSystemCatalog::UBIGINT, "UNSIGNED BIGINT") - (CalpontSystemCatalog::UDOUBLE, "UNSIGNED DOUBLE") - (CalpontSystemCatalog::LONGDOUBLE, "INTERNAL LONG DOUBLE") - (CalpontSystemCatalog::STRINT, "INTERNAL SHORT STRING") - (CalpontSystemCatalog::TEXT, "TEXT") - (CalpontSystemCatalog::TIME, "TIME") - (CalpontSystemCatalog::TIMESTAMP, "TIMESTAMP") - ; - +map colType2String = + assign::map_list_of(CalpontSystemCatalog::BIT, "BIT")(CalpontSystemCatalog::TINYINT, "TINYINT")( + CalpontSystemCatalog::CHAR, + "CHAR")(CalpontSystemCatalog::SMALLINT, "SMALLINT")(CalpontSystemCatalog::DECIMAL, + "DECIMAL")(CalpontSystemCatalog::MEDINT, + "MEDINT")(CalpontSystemCatalog::INT, + "INT")(CalpontSystemCatalog:: + FLOAT, + "FLOA" + "T")(CalpontSystemCatalog:: + DATE, + "DAT" + "E")(CalpontSystemCatalog:: + BIGINT, + "BIGIN" + "T")(CalpontSystemCatalog:: + DOUBLE, + "DOUBL" + "E")(CalpontSystemCatalog:: + DATETIME, + "DATETIME")(CalpontSystemCatalog:: + VARCHAR, + "VARCHAR")(CalpontSystemCatalog:: + VARBINARY, + "VARBINARY")(CalpontSystemCatalog:: + CLOB, + "CLOB")(CalpontSystemCatalog:: + BLOB, + "BLOB")(CalpontSystemCatalog:: + UTINYINT, + "UNSIGNED TINYINT")(CalpontSystemCatalog:: + USMALLINT, + "UNSIGNED SMALLINT")(CalpontSystemCatalog:: + UDECIMAL, + "UNSIGNED DECIMAL")(CalpontSystemCatalog:: + UMEDINT, + "UNSIGNED MEDINT")(CalpontSystemCatalog:: + UINT, + "UNSIGNED INT")(CalpontSystemCatalog:: + UFLOAT, + "UNSIGNED FLOAT")(CalpontSystemCatalog:: + UBIGINT, + "UNSIGNED BIGINT")(CalpontSystemCatalog:: + UDOUBLE, + "UNSIGNED DOUBLE")(CalpontSystemCatalog:: + LONGDOUBLE, + "INTERNAL LONG DOUBLE")(CalpontSystemCatalog:: + STRINT, + "INTERNAL SHORT STRING")(CalpontSystemCatalog:: + TEXT, + "TEXT")(CalpontSystemCatalog:: + TIME, + "TIME")(CalpontSystemCatalog:: + TIMESTAMP, + "TIMESTAMP"); // function name to function id convert -map WindowFunctionType::windowFunctionId = assign::map_list_of - (string("COUNT(*)"), WF__COUNT_ASTERISK) - (string("COUNT"), WF__COUNT) - (string("SUM"), WF__SUM) - (string("AVG"), WF__AVG) - (string("MIN"), WF__MIN) - (string("MAX"), WF__MAX) - (string("COUNT_DISTINCT"), WF__COUNT_DISTINCT) - (string("SUM_DISTINCT"), WF__SUM_DISTINCT) - (string("AVG_DISTINCT"), WF__AVG_DISTINCT) - (string("MIN_DISTINCT"), WF__MIN) - (string("MAX_DISTINCT"), WF__MAX) - (string("STD"), WF__STDDEV_POP) - (string("STDDEV"), WF__STDDEV_POP) - (string("STDDEV_POP"), WF__STDDEV_POP) - (string("STDDEV_SAMP"), WF__STDDEV_SAMP) - (string("VARIANCE"), WF__VAR_POP) - (string("VAR_POP"), WF__VAR_POP) - (string("VAR_SAMP"), WF__VAR_SAMP) - (string("ROW_NUMBER"), WF__ROW_NUMBER) - (string("RANK"), WF__RANK) - (string("PERCENT_RANK"), WF__PERCENT_RANK) - (string("DENSE_RANK"), WF__DENSE_RANK) - (string("CUME_DIST"), WF__CUME_DIST) - (string("FIRST_VALUE"), WF__FIRST_VALUE) - (string("LAST_VALUE"), WF__LAST_VALUE) - (string("NTH_VALUE"), WF__NTH_VALUE) - (string("LAG"), WF__LAG) - (string("LEAD"), WF__LEAD) - (string("NTILE"), WF__NTILE) - (string("MEDIAN"), WF__PERCENTILE_CONT) - (string("PERCENTILE"), WF__PERCENTILE_CONT) - (string("PERCENTILE_CONT"), WF__PERCENTILE_CONT) - (string("PERCENTILE_DISC"), WF__PERCENTILE_DISC) - (string("REGR_SLOPE"), WF__REGR_SLOPE) - (string("REGR_INTERCEPT"), WF__REGR_INTERCEPT) - (string("REGR_COUNT"), WF__REGR_COUNT) - (string("REGR_R2"), WF__REGR_R2) - (string("REGR_AVGX"), WF__REGR_AVGX) - (string("REGR_AVGY"), WF__REGR_AVGY) - (string("REGR_SXX"), WF__REGR_SXX) - (string("REGR_SXY"), WF__REGR_SXY) - (string("REGR_SYY"), WF__REGR_SYY) - (string("UDAF_FUNC"), WF__UDAF) - ; +map + WindowFunctionType::windowFunctionId = + assign::map_list_of( + string("COUNT(*)"), WF__COUNT_ASTERISK)(string("COUNT"), + WF__COUNT)(string("SUM"), WF__SUM)(string("AVG"), WF__AVG)(string("MIN"), WF__MIN)(string("MAX"), WF__MAX)(string("COUNT_DISTINCT"), WF__COUNT_DISTINCT)(string("SUM_DISTINCT"), WF__SUM_DISTINCT)(string("AVG_DISTINCT"), WF__AVG_DISTINCT)(string("MIN_DISTINCT"), WF__MIN)(string("MAX_DISTINCT"), WF__MAX)(string("STD"), WF__STDDEV_POP)(string("STDDEV"), WF__STDDEV_POP)(string("STDDEV_POP"), WF__STDDEV_POP)(string( + "STDDEV_SAMP"), + WF__STDDEV_SAMP)(string("VARIANCE"), WF__VAR_POP)(string("VAR_POP"), WF__VAR_POP)(string("VAR_SAMP"), WF__VAR_SAMP)(string("ROW_NUMBER"), WF__ROW_NUMBER)(string("RANK"), WF__RANK)(string( + "PERCENT_RANK"), + WF__PERCENT_RANK)(string("DENSE_RANK"), WF__DENSE_RANK)(string( + "CUME_DIST"), + WF__CUME_DIST)(string("FIRST_VALUE"), WF__FIRST_VALUE)(string("LAST_VALUE"), WF__LAST_VALUE)(string("NTH_VALUE"), WF__NTH_VALUE)(string( + "LAG"), + WF__LAG)(string( + "LEAD"), + WF__LEAD)(string("NTILE"), WF__NTILE)(string( + "MEDIAN"), + WF__PERCENTILE_CONT)(string("PERCENTILE"), WF__PERCENTILE_CONT)(string( + "PERCENTILE_CONT"), + WF__PERCENTILE_CONT)(string("PERCENTILE_DISC"), WF__PERCENTILE_DISC)(string( + "REGR_SLOPE"), + WF__REGR_SLOPE)(string("REGR_INTERCEPT"), WF__REGR_INTERCEPT)(string( + "REGR_COUNT"), + WF__REGR_COUNT)(string( + "REGR_R2"), + WF__REGR_R2)(string( + "REGR_AVGX"), + WF__REGR_AVGX)(string("REGR_AVGY"), WF__REGR_AVGY)(string( + "REGR_SXX"), + WF__REGR_SXX)(string("REGR_SXY"), WF__REGR_SXY)(string( + "REGR_SYY"), + WF__REGR_SYY)(string( + "UDAF_FUNC"), + WF__UDAF); -boost::shared_ptr -WindowFunctionType::makeWindowFunction(const string& name, int ct, WindowFunctionColumn* wc) +boost::shared_ptr WindowFunctionType::makeWindowFunction(const string& name, int ct, + WindowFunctionColumn* wc) { - boost::shared_ptr af; - int functionId = windowFunctionId[algorithm::to_upper_copy(name)]; + boost::shared_ptr af; + int functionId = windowFunctionId[algorithm::to_upper_copy(name)]; - // The template parameters here are dummies to execute the static makeFunction - // which sets the real types based on ct. - switch (functionId) - { - case WF__COUNT_ASTERISK: - case WF__COUNT: - case WF__COUNT_DISTINCT: - af = WF_count::makeFunction(functionId, name, ct, wc); - break; + // The template parameters here are dummies to execute the static makeFunction + // which sets the real types based on ct. + switch (functionId) + { + case WF__COUNT_ASTERISK: + case WF__COUNT: + case WF__COUNT_DISTINCT: af = WF_count::makeFunction(functionId, name, ct, wc); break; - case WF__MIN: - case WF__MAX: - af = WF_min_max::makeFunction(functionId, name, ct, wc); - break; + case WF__MIN: + case WF__MAX: af = WF_min_max::makeFunction(functionId, name, ct, wc); break; - case WF__SUM: - case WF__AVG: - case WF__SUM_DISTINCT: - case WF__AVG_DISTINCT: - af = WF_sum_avg::makeFunction(functionId, name, ct, wc); - break; + case WF__SUM: + case WF__AVG: + case WF__SUM_DISTINCT: + case WF__AVG_DISTINCT: + af = WF_sum_avg::makeFunction(functionId, name, ct, wc); + break; - case WF__STDDEV_POP: - case WF__STDDEV_SAMP: - case WF__VAR_POP: - case WF__VAR_SAMP: - af = WF_stats::makeFunction(functionId, name, ct, wc); - break; + case WF__STDDEV_POP: + case WF__STDDEV_SAMP: + case WF__VAR_POP: + case WF__VAR_SAMP: af = WF_stats::makeFunction(functionId, name, ct, wc); break; - case WF__ROW_NUMBER: - af = WF_row_number::makeFunction(functionId, name, ct, wc); - break; + case WF__ROW_NUMBER: af = WF_row_number::makeFunction(functionId, name, ct, wc); break; - case WF__RANK: - case WF__DENSE_RANK: - case WF__PERCENT_RANK: - case WF__CUME_DIST: - af = WF_ranking::makeFunction(functionId, name, ct, wc); - break; + case WF__RANK: + case WF__DENSE_RANK: + case WF__PERCENT_RANK: + case WF__CUME_DIST: af = WF_ranking::makeFunction(functionId, name, ct, wc); break; - case WF__FIRST_VALUE: - case WF__LAST_VALUE: - case WF__NTH_VALUE: - af = WF_nth_value::makeFunction(functionId, name, ct, wc); - break; + case WF__FIRST_VALUE: + case WF__LAST_VALUE: + case WF__NTH_VALUE: af = WF_nth_value::makeFunction(functionId, name, ct, wc); break; - case WF__LEAD: - case WF__LAG: - af = WF_lead_lag::makeFunction(functionId, name, ct, wc); - break; + case WF__LEAD: + case WF__LAG: af = WF_lead_lag::makeFunction(functionId, name, ct, wc); break; - case WF__NTILE: - af = WF_ntile::makeFunction(functionId, name, ct, wc); - break; + case WF__NTILE: af = WF_ntile::makeFunction(functionId, name, ct, wc); break; - case WF__PERCENTILE_CONT: - case WF__PERCENTILE_DISC: - af = WF_percentile::makeFunction(functionId, name, ct, wc); - break; + case WF__PERCENTILE_CONT: + case WF__PERCENTILE_DISC: af = WF_percentile::makeFunction(functionId, name, ct, wc); break; - case WF__UDAF: - af = WF_udaf::makeFunction(functionId, name, ct, wc->getUDAFContext(), wc); - break; + case WF__UDAF: af = WF_udaf::makeFunction(functionId, name, ct, wc->getUDAFContext(), wc); break; - case WF__REGR_SLOPE: - case WF__REGR_INTERCEPT: - case WF__REGR_COUNT: - case WF__REGR_R2: - case WF__REGR_AVGX: - case WF__REGR_AVGY: - case WF__REGR_SXX: - case WF__REGR_SXY: - case WF__REGR_SYY: - case WF__UNDEFINED: - default: - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_SUPPORT, name), - ERR_WF_NOT_SUPPORT); - break; - } + case WF__REGR_SLOPE: + case WF__REGR_INTERCEPT: + case WF__REGR_COUNT: + case WF__REGR_R2: + case WF__REGR_AVGX: + case WF__REGR_AVGY: + case WF__REGR_SXX: + case WF__REGR_SXY: + case WF__REGR_SYY: + case WF__UNDEFINED: + default: + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_SUPPORT, name), ERR_WF_NOT_SUPPORT); + break; + } - // Copy only the constant parameter pointers - af->constParms(wc->functionParms()); + // Copy only the constant parameter pointers + af->constParms(wc->functionParms()); - return af; + return af; } const string WindowFunctionType::toString() const { - ostringstream oss; - oss << "Window Function Id: " << fFunctionId << ", field indices: "; + ostringstream oss; + oss << "Window Function Id: " << fFunctionId << ", field indices: "; - for (uint64_t i = 0; i < fFieldIndex.size(); i++) - oss << fFieldIndex[i] << " "; + for (uint64_t i = 0; i < fFieldIndex.size(); i++) + oss << fFieldIndex[i] << " "; - oss << endl; + oss << endl; - return oss.str(); + return oss.str(); } -template void WindowFunctionType::getValue(uint64_t i, T& t, CDT* cdt) +template +void WindowFunctionType::getValue(uint64_t i, T& t, CDT* cdt) { } -template<> void WindowFunctionType::getValue(uint64_t i, int64_t& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, int64_t& t, CDT* cdt) { - t = fRow.getIntField(i); + t = fRow.getIntField(i); - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::BIGINT; - } + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::BIGINT; + } } -template<> void WindowFunctionType::getValue(uint64_t i, uint64_t& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, uint64_t& t, CDT* cdt) { - t = fRow.getUintField(i); + t = fRow.getUintField(i); - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::UBIGINT; - } + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::UBIGINT; + } } -template<> void WindowFunctionType::getValue(uint64_t i, double& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, double& t, CDT* cdt) { - t = fRow.getDoubleField(i); + t = fRow.getDoubleField(i); - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::DOUBLE; - } + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::DOUBLE; + } } -template<> void WindowFunctionType::getValue(uint64_t i, float& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, float& t, CDT* cdt) { - t = fRow.getFloatField(i); + t = fRow.getFloatField(i); - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::FLOAT; - } + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::FLOAT; + } } -template<> void WindowFunctionType::getValue(uint64_t i, long double& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, long double& t, CDT* cdt) { - t = fRow.getLongDoubleField(i); + t = fRow.getLongDoubleField(i); - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::LONGDOUBLE; - } + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::LONGDOUBLE; + } } -template<> void WindowFunctionType::getValue(uint64_t i, string& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, string& t, CDT* cdt) { - t = fRow.getStringField(i); - // By not setting cdt, we let it default to the column's type + t = fRow.getStringField(i); + // By not setting cdt, we let it default to the column's type } -template<> void WindowFunctionType::getValue(uint64_t i, int128_t& t, CDT* cdt) +template <> +void WindowFunctionType::getValue(uint64_t i, int128_t& t, CDT* cdt) { - fRow.getInt128Field(i, t); + fRow.getInt128Field(i, t); - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::DECIMAL; - } + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::DECIMAL; + } } -template void WindowFunctionType::setValue(uint64_t i, T& t) +template +void WindowFunctionType::setValue(uint64_t i, T& t) { } -template<> void WindowFunctionType::setValue(uint64_t i, int64_t& t) +template <> +void WindowFunctionType::setValue(uint64_t i, int64_t& t) { - fRow.setIntField(t, i); + fRow.setIntField(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, uint64_t& t) +template <> +void WindowFunctionType::setValue(uint64_t i, uint64_t& t) { - fRow.setUintField(t, i); + fRow.setUintField(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, double& t) +template <> +void WindowFunctionType::setValue(uint64_t i, double& t) { - fRow.setDoubleField(t, i); + fRow.setDoubleField(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, float& t) +template <> +void WindowFunctionType::setValue(uint64_t i, float& t) { - fRow.setFloatField(t, i); + fRow.setFloatField(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, long double& t) +template <> +void WindowFunctionType::setValue(uint64_t i, long double& t) { - fRow.setLongDoubleField(t, i); + fRow.setLongDoubleField(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, int128_t& t) +template <> +void WindowFunctionType::setValue(uint64_t i, int128_t& t) { - fRow.setInt128Field(t, i); + fRow.setInt128Field(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, string& t) +template <> +void WindowFunctionType::setValue(uint64_t i, string& t) { - fRow.setStringField(t, i); + fRow.setStringField(t, i); } // MCOL-1676 Need a separate specialization for string now. -template<> +template <> void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, string* v) { - if (c != WF__BOUND_ALL) - b = e = c; + if (c != WF__BOUND_ALL) + b = e = c; - uint64_t i = fFieldIndex[0]; + uint64_t i = fFieldIndex[0]; - if (v == NULL) - v = (string*) getNullValueByType(ct, i); + if (v == NULL) + v = (string*)getNullValueByType(ct, i); - for (int64_t j = b; j <= e; j++) - { - if (j % 1000 == 0 && fStep->cancelled()) - break; + for (int64_t j = b; j <= e; j++) + { + if (j % 1000 == 0 && fStep->cancelled()) + break; - fRow.setData(getPointer((*fRowData)[j])); - setValue(i, *v); - } + fRow.setData(getPointer((*fRowData)[j])); + setValue(i, *v); + } } -template +template void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) { - if (c != WF__BOUND_ALL) - b = e = c; + if (c != WF__BOUND_ALL) + b = e = c; - uint64_t i = fFieldIndex[0]; + uint64_t i = fFieldIndex[0]; - if (v == NULL) - v = (T*) getNullValueByType(ct, i); + if (v == NULL) + v = (T*)getNullValueByType(ct, i); - for (int64_t j = b; j <= e; j++) - { - if (j % 1000 == 0 && fStep->cancelled()) - break; - - fRow.setData(getPointer((*fRowData)[j])); - // MCOL-1676 Set the data based on out column type (ct) - switch (ct) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - { - int64_t iv = *v; - setValue(i, iv); - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - { - uint64_t uv = *v; - setValue(i, uv); - break; - } - - case CalpontSystemCatalog::DECIMAL: - { - if (sizeof(T) == 8) - { - int64_t iv = *v; - setValue(i, iv); - } - else - { - int128_t iv = *v; - setValue(i, iv); - } - break; - } - - case CalpontSystemCatalog::UDECIMAL: - { - if (sizeof(T) == 8) - { - uint64_t iv = *v; - setValue(i, iv); - } - else - { - int128_t iv = *v; - setValue(i, iv); - } - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dv = *v; - setValue(i, dv); - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float fv = *v; - setValue(i, fv); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dv = *v; - setValue(i, dv); - break; - } - - default: - { - setValue(i, *v); - } - } - } -} - -template -void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) -{ - int ct = fRow.getColType(i); + for (int64_t j = b; j <= e; j++) + { + if (j % 1000 == 0 && fStep->cancelled()) + break; + fRow.setData(getPointer((*fRowData)[j])); + // MCOL-1676 Set the data based on out column type (ct) switch (ct) { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + { + int64_t iv = *v; + setValue(i, iv); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + uint64_t uv = *v; + setValue(i, uv); + break; + } + + case CalpontSystemCatalog::DECIMAL: + { + if (sizeof(T) == 8) { - t = (T) fRow.getIntField(i); - break; + int64_t iv = *v; + setValue(i, iv); } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: + else { - t = (T) fRow.getUintField(i); - break; + int128_t iv = *v; + setValue(i, iv); } + break; + } - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (sizeof(T) == 8) { - decltype(datatypes::MAXDECIMALWIDTH) width = - fRow.getColumnWidth(i);; - - if (width < datatypes::MAXDECIMALWIDTH) - { - t = (ct == execplan::CalpontSystemCatalog::DECIMAL) ? - (T) fRow.getIntField(i) : - (T) fRow.getUintField(i); - } - else if (width == datatypes::MAXDECIMALWIDTH) - { - datatypes::TSInt128::assignPtrPtr(&t, fRow.getBinaryField(i)); - } - break; + uint64_t iv = *v; + setValue(i, iv); } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: + else { - t = (T) fRow.getDoubleField(i); - break; + int128_t iv = *v; + setValue(i, iv); } + break; + } - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - t = (T) fRow.getFloatField(i); - break; - } + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dv = *v; + setValue(i, dv); + break; + } - case CalpontSystemCatalog::LONGDOUBLE: - { - t = (T) fRow.getLongDoubleField(i); - break; - } + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float fv = *v; + setValue(i, fv); + break; + } - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - default: - { - string errStr = fFunctionName + "(" + colType2String[ct] + ")"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + case CalpontSystemCatalog::LONGDOUBLE: + { + long double dv = *v; + setValue(i, dv); + break; + } - break; - } + default: + { + setValue(i, *v); + } + } + } +} + +template +void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) +{ + int ct = fRow.getColType(i); + + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + { + t = (T)fRow.getIntField(i); + break; } - T divisor = 1; - s -= fRow.getScale(i); // we scale only the difference of scales - datatypes::getScaleDivisor(divisor, abs(s)); - if (s > 0) - t *= divisor; - else if (s < 0) - t /= divisor; + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + { + t = (T)fRow.getUintField(i); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = fRow.getColumnWidth(i); + ; + + if (width < datatypes::MAXDECIMALWIDTH) + { + t = (ct == execplan::CalpontSystemCatalog::DECIMAL) ? (T)fRow.getIntField(i) + : (T)fRow.getUintField(i); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + datatypes::TSInt128::assignPtrPtr(&t, fRow.getBinaryField(i)); + } + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + t = (T)fRow.getDoubleField(i); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + t = (T)fRow.getFloatField(i); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + t = (T)fRow.getLongDoubleField(i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + default: + { + string errStr = fFunctionName + "(" + colType2String[ct] + ")"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_INVALID_PARM_TYPE, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_INVALID_PARM_TYPE); + + break; + } + } + + T divisor = 1; + s -= fRow.getScale(i); // we scale only the difference of scales + datatypes::getScaleDivisor(divisor, abs(s)); + if (s > 0) + t *= divisor; + else if (s < 0) + t /= divisor; } -template<> +template <> void WindowFunctionType::implicit2T(uint64_t i, string& t, int) { - t = fRow.getStringField(i); + t = fRow.getStringField(i); } -template +template void WindowFunctionType::getConstValue(ConstantColumn* cc, T& t, bool& b) { } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, int64_t& t, bool& b) { - t = cc->getIntVal(fRow, b); + t = cc->getIntVal(fRow, b); } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, uint64_t& t, bool& b) { - t = cc->getUintVal(fRow, b); + t = cc->getUintVal(fRow, b); } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, int128_t& t, bool& b) { - t = cc->getDecimalVal(fRow, b).s128Value; + t = cc->getDecimalVal(fRow, b).s128Value; } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, double& t, bool& b) { - t = cc->getDoubleVal(fRow, b); + t = cc->getDoubleVal(fRow, b); } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, long double& t, bool& b) { - t = cc->getLongDoubleVal(fRow, b); + t = cc->getLongDoubleVal(fRow, b); } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, float& t, bool& b) { - t = cc->getFloatVal(fRow, b); + t = cc->getFloatVal(fRow, b); } -template<> +template <> void WindowFunctionType::getConstValue(ConstantColumn* cc, string& t, bool& b) { - t = cc->getStrVal(fRow, b); + t = cc->getStrVal(fRow, b); } template void WindowFunctionType::implicit2T(uint64_t, int64_t&, int); @@ -637,78 +641,58 @@ template void WindowFunctionType::setValue(int, int64_t, int64_t, int6 void* WindowFunctionType::getNullValueByType(int ct, int pos) { - static uint64_t bigIntNull = joblist::BIGINTNULL; - static uint64_t intNull = joblist::INTNULL; - static uint64_t smallIntNull = joblist::SMALLINTNULL; - static uint64_t tinyIntNull = joblist::TINYINTNULL; - static uint64_t ubigIntNull = joblist::UBIGINTNULL; - static uint64_t uintNull = joblist::UINTNULL; - static uint64_t usmallIntNull = joblist::USMALLINTNULL; - static uint64_t utinyIntNull = joblist::UTINYINTNULL; - static uint64_t floatNull = joblist::FLOATNULL; - static uint64_t doubleNull = joblist::DOUBLENULL; - static long double longDoubleNull= joblist::LONGDOUBLENULL; - static uint64_t dateNull = joblist::DATENULL; - static uint64_t datetimeNull = joblist::DATETIMENULL; - static uint64_t timeNull = joblist::TIMENULL; - static uint64_t timestampNull = joblist::TIMESTAMPNULL; -// static uint64_t char1Null = joblist::CHAR1NULL; -// static uint64_t char2Null = joblist::CHAR2NULL; -// static uint64_t char4Null = joblist::CHAR4NULL; -// static uint64_t char8Null = joblist::CHAR8NULL; - static string stringNull(""); - static int128_t int128Null; // Set at runtime; - - void* v = NULL; + static uint64_t bigIntNull = joblist::BIGINTNULL; + static uint64_t intNull = joblist::INTNULL; + static uint64_t smallIntNull = joblist::SMALLINTNULL; + static uint64_t tinyIntNull = joblist::TINYINTNULL; + static uint64_t ubigIntNull = joblist::UBIGINTNULL; + static uint64_t uintNull = joblist::UINTNULL; + static uint64_t usmallIntNull = joblist::USMALLINTNULL; + static uint64_t utinyIntNull = joblist::UTINYINTNULL; + static uint64_t floatNull = joblist::FLOATNULL; + static uint64_t doubleNull = joblist::DOUBLENULL; + static long double longDoubleNull = joblist::LONGDOUBLENULL; + static uint64_t dateNull = joblist::DATENULL; + static uint64_t datetimeNull = joblist::DATETIMENULL; + static uint64_t timeNull = joblist::TIMENULL; + static uint64_t timestampNull = joblist::TIMESTAMPNULL; + // static uint64_t char1Null = joblist::CHAR1NULL; + // static uint64_t char2Null = joblist::CHAR2NULL; + // static uint64_t char4Null = joblist::CHAR4NULL; + // static uint64_t char8Null = joblist::CHAR8NULL; + static string stringNull(""); + static int128_t int128Null; // Set at runtime; - switch (ct) + void* v = NULL; + + switch (ct) + { + case CalpontSystemCatalog::TINYINT: v = &tinyIntNull; break; + + case CalpontSystemCatalog::SMALLINT: v = &smallIntNull; break; + + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: v = &intNull; break; + + case CalpontSystemCatalog::BIGINT: v = &bigIntNull; break; + + case CalpontSystemCatalog::DATE: v = &dateNull; break; + + case CalpontSystemCatalog::DATETIME: v = &datetimeNull; break; + + case CalpontSystemCatalog::TIMESTAMP: v = ×tampNull; break; + + case CalpontSystemCatalog::TIME: v = &timeNull; break; + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: v = &floatNull; break; + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: v = &doubleNull; break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: { - case CalpontSystemCatalog::TINYINT: - v = &tinyIntNull; - break; - - case CalpontSystemCatalog::SMALLINT: - v = &smallIntNull; - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - v = &intNull; - break; - - case CalpontSystemCatalog::BIGINT: - v = &bigIntNull; - break; - - case CalpontSystemCatalog::DATE: - v = &dateNull; - break; - - case CalpontSystemCatalog::DATETIME: - v = &datetimeNull; - break; - - case CalpontSystemCatalog::TIMESTAMP: - v = ×tampNull; - break; - - case CalpontSystemCatalog::TIME: - v = &timeNull; - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - v = &floatNull; - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - v = &doubleNull; - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { // uint64_t len = fRow.getColumnWidth(pos); #if 0 switch (len) @@ -739,99 +723,78 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) } #endif - v = &stringNull; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - uint64_t len = fRow.getColumnWidth(pos); - - switch (len) - { - case 1: - v = &tinyIntNull; - break; - - case 2: - v = &smallIntNull; - break; - - case 4: - v = &intNull; - break; - - case 8: - v = &bigIntNull; - break; - - case 16: - datatypes::Decimal::setWideDecimalNullValue(int128Null); - v = &int128Null; - break; - - default: - break; - - } - - break; - } - - case CalpontSystemCatalog::UTINYINT: - v = &utinyIntNull; - break; - - case CalpontSystemCatalog::USMALLINT: - v = &usmallIntNull; - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - v = &uintNull; - break; - - case CalpontSystemCatalog::UBIGINT: - v = &ubigIntNull; - break; - - case CalpontSystemCatalog::LONGDOUBLE: - v = &longDoubleNull; - break; - - case CalpontSystemCatalog::VARBINARY: - default: - std::ostringstream oss; - oss << "not supported data type: " << colType2String[ct]; - throw logic_error(oss.str()); - break; + v = &stringNull; + break; } - return v; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + uint64_t len = fRow.getColumnWidth(pos); + + switch (len) + { + case 1: v = &tinyIntNull; break; + + case 2: v = &smallIntNull; break; + + case 4: v = &intNull; break; + + case 8: v = &bigIntNull; break; + + case 16: + datatypes::Decimal::setWideDecimalNullValue(int128Null); + v = &int128Null; + break; + + default: break; + } + + break; + } + + case CalpontSystemCatalog::UTINYINT: v = &utinyIntNull; break; + + case CalpontSystemCatalog::USMALLINT: v = &usmallIntNull; break; + + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: v = &uintNull; break; + + case CalpontSystemCatalog::UBIGINT: v = &ubigIntNull; break; + + case CalpontSystemCatalog::LONGDOUBLE: v = &longDoubleNull; break; + + case CalpontSystemCatalog::VARBINARY: + default: + std::ostringstream oss; + oss << "not supported data type: " << colType2String[ct]; + throw logic_error(oss.str()); + break; + } + + return v; } void WindowFunctionType::constParms(const std::vector& functionParms) { - // fConstantParms will end up with a copy of functionParms, but only - // the constant types will be copied. Other types will take up space but - // be NULL. This allows us to acces the constants without the overhead - // of dynamic_cast for every row. - for (size_t i = 0; i < functionParms.size(); ++i) - { - ConstantColumn* cc = dynamic_cast(functionParms[i].get()); + // fConstantParms will end up with a copy of functionParms, but only + // the constant types will be copied. Other types will take up space but + // be NULL. This allows us to acces the constants without the overhead + // of dynamic_cast for every row. + for (size_t i = 0; i < functionParms.size(); ++i) + { + ConstantColumn* cc = dynamic_cast(functionParms[i].get()); - if (cc) - { - fConstantParms.push_back(functionParms[i]); - } - else - { - fConstantParms.push_back(SRCP(cc)); - } + if (cc) + { + fConstantParms.push_back(functionParms[i]); } + else + { + fConstantParms.push_back(SRCP(cc)); + } + } } -} //namespace +} // namespace windowfunction // vim:ts=4 sw=4: - diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index dfda6e053..c1af9bc78 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -18,7 +18,6 @@ // $Id: windowfunctiontype.h 3932 2013-06-25 16:08:10Z xlou $ - #ifndef UTILS_WINDOWFUNCTIONTYPE_H #define UTILS_WINDOWFUNCTIONTYPE_H @@ -39,68 +38,67 @@ namespace ordering { // forward reference class EqualCompData; -}; +}; // namespace ordering namespace execplan { // forward reference class ConstantColumn; -}; +}; // namespace execplan namespace joblist { // forward reference class WindowFunctionStep; -}; +}; // namespace joblist namespace windowfunction { - // forward reference class WindowFunctionType; class IdbOrderBy; class WindowFrame; -const int WF__UNDEFINED = 0; -const int WF__COUNT_ASTERISK = 1; -const int WF__COUNT = 2; -const int WF__SUM = 3; -const int WF__AVG = 4; -const int WF__MIN = 5; -const int WF__MAX = 6; -const int WF__COUNT_DISTINCT = 7; -const int WF__SUM_DISTINCT = 8; -const int WF__AVG_DISTINCT = 9; -const int WF__STDDEV_POP = 10; -const int WF__STDDEV_SAMP = 11; -const int WF__VAR_POP = 12; -const int WF__VAR_SAMP = 13; +const int WF__UNDEFINED = 0; +const int WF__COUNT_ASTERISK = 1; +const int WF__COUNT = 2; +const int WF__SUM = 3; +const int WF__AVG = 4; +const int WF__MIN = 5; +const int WF__MAX = 6; +const int WF__COUNT_DISTINCT = 7; +const int WF__SUM_DISTINCT = 8; +const int WF__AVG_DISTINCT = 9; +const int WF__STDDEV_POP = 10; +const int WF__STDDEV_SAMP = 11; +const int WF__VAR_POP = 12; +const int WF__VAR_SAMP = 13; -const int WF__ROW_NUMBER = 14; -const int WF__RANK = 15; -const int WF__PERCENT_RANK = 16; -const int WF__DENSE_RANK = 17; -const int WF__CUME_DIST = 18; +const int WF__ROW_NUMBER = 14; +const int WF__RANK = 15; +const int WF__PERCENT_RANK = 16; +const int WF__DENSE_RANK = 17; +const int WF__CUME_DIST = 18; -const int WF__FIRST_VALUE = 19; -const int WF__LAST_VALUE = 20; -const int WF__NTH_VALUE = 21; -const int WF__LAG = 22; -const int WF__LEAD = 23; -const int WF__NTILE = 24; +const int WF__FIRST_VALUE = 19; +const int WF__LAST_VALUE = 20; +const int WF__NTH_VALUE = 21; +const int WF__LAG = 22; +const int WF__LEAD = 23; +const int WF__NTILE = 24; const int WF__PERCENTILE_CONT = 25; const int WF__PERCENTILE_DISC = 26; -const int WF__REGR_SLOPE = 27; -const int WF__REGR_INTERCEPT = 28; -const int WF__REGR_COUNT = 29; -const int WF__REGR_R2 = 30; -const int WF__REGR_AVGX = 31; -const int WF__REGR_AVGY = 32; -const int WF__REGR_SXX = 33; -const int WF__REGR_SXY = 34; -const int WF__REGR_SYY = 35; -const int WF__UDAF = 36; +const int WF__REGR_SLOPE = 27; +const int WF__REGR_INTERCEPT = 28; +const int WF__REGR_COUNT = 29; +const int WF__REGR_R2 = 30; +const int WF__REGR_AVGX = 31; +const int WF__REGR_AVGY = 32; +const int WF__REGR_SXX = 33; +const int WF__REGR_SXY = 34; +const int WF__REGR_SYY = 35; +const int WF__UDAF = 36; typedef execplan::CalpontSystemCatalog::ColDataType CDT; @@ -109,200 +107,203 @@ typedef execplan::CalpontSystemCatalog::ColDataType CDT; */ class WindowFunctionType { -public: - // @brief WindowFunctionType constructor - WindowFunctionType(int id = 0, const std::string& name = "") : - fFunctionId(id), fFunctionName(name), fFrameUnit(0) {}; + public: + // @brief WindowFunctionType constructor + WindowFunctionType(int id = 0, const std::string& name = "") + : fFunctionId(id), fFunctionName(name), fFrameUnit(0){}; - // use default copy construct - //WindowFunctionType(const WindowFunctionType&); + // use default copy construct + // WindowFunctionType(const WindowFunctionType&); - // @brief WindowFunctionType destructor - virtual ~WindowFunctionType() {}; + // @brief WindowFunctionType destructor + virtual ~WindowFunctionType(){}; - // @brief virtual operator(begin, end, current, data, row) - virtual void operator()(int64_t, int64_t, int64_t) = 0; + // @brief virtual operator(begin, end, current, data, row) + virtual void operator()(int64_t, int64_t, int64_t) = 0; - // @brief virtual clone() - virtual WindowFunctionType* clone() const = 0; + // @brief virtual clone() + virtual WindowFunctionType* clone() const = 0; - // @brief virtual resetData() - virtual void resetData() - { - fPrev = -1; - } + // @brief virtual resetData() + virtual void resetData() + { + fPrev = -1; + } - // @brief virtual parseParms() - virtual void parseParms(const std::vector&) {} + // @brief virtual parseParms() + virtual void parseParms(const std::vector&) + { + } - // @brief virtual dropValues() For UDAnF functions - // return false if there's no dropValue() implemented in the function. - virtual bool dropValues(int64_t, int64_t) - { - return false; - } + // @brief virtual dropValues() For UDAnF functions + // return false if there's no dropValue() implemented in the function. + virtual bool dropValues(int64_t, int64_t) + { + return false; + } - // @brief virtual display method - virtual const std::string toString() const; + // @brief virtual display method + virtual const std::string toString() const; - // @brief access methods - int64_t functionId() const - { - return fFunctionId; - } + // @brief access methods + int64_t functionId() const + { + return fFunctionId; + } - const std::string functionName() const - { - return fFunctionName; - } + const std::string functionName() const + { + return fFunctionName; + } - void functionId(int id) - { - fFunctionId = id; - } - const std::vector& fieldIndex() const - { - return fFieldIndex; - } - void fieldIndex(const std::vector& v) - { - fFieldIndex = v; - } - void setRowMetaData(const rowgroup::RowGroup& g, const rowgroup::Row& r) - { - fRowGroup = g; - fRow = r; - } - void setRowData(const boost::shared_ptr >& d) - { - fRowData = d; - } - int64_t frameUnit() const - { - return fFrameUnit; - } - void frameUnit(int u) - { - fFrameUnit = u; - } - std::pair partition() const - { - return fPartition; - } - void partition(std::pair& p) - { - fPartition = p; - } - const boost::shared_ptr& peer() const - { - return fPeer; - } - void peer(const boost::shared_ptr& p) - { - fPeer = p; - } - void setCallback(joblist::WindowFunctionStep* step) - { - fStep = step; - } + void functionId(int id) + { + fFunctionId = id; + } + const std::vector& fieldIndex() const + { + return fFieldIndex; + } + void fieldIndex(const std::vector& v) + { + fFieldIndex = v; + } + void setRowMetaData(const rowgroup::RowGroup& g, const rowgroup::Row& r) + { + fRowGroup = g; + fRow = r; + } + void setRowData(const boost::shared_ptr >& d) + { + fRowData = d; + } + int64_t frameUnit() const + { + return fFrameUnit; + } + void frameUnit(int u) + { + fFrameUnit = u; + } + std::pair partition() const + { + return fPartition; + } + void partition(std::pair& p) + { + fPartition = p; + } + const boost::shared_ptr& peer() const + { + return fPeer; + } + void peer(const boost::shared_ptr& p) + { + fPeer = p; + } + void setCallback(joblist::WindowFunctionStep* step) + { + fStep = step; + } - void constParms(const std::vector& functionParms); + void constParms(const std::vector& functionParms); - static boost::shared_ptr makeWindowFunction(const std::string&, int ct, execplan::WindowFunctionColumn* wc); + static boost::shared_ptr makeWindowFunction(const std::string&, int ct, + execplan::WindowFunctionColumn* wc); -protected: + protected: + static std::map windowFunctionId; - static std::map windowFunctionId; + // utility methods + template + void getValue(uint64_t, T&, CDT* cdt = NULL); + template + void setValue(int, int64_t, int64_t, int64_t, T* = NULL); + template + void setValue(uint64_t, T&); + template + void implicit2T(uint64_t, T&, int); + template + void getConstValue(execplan::ConstantColumn*, T&, bool&); - // utility methods - template void getValue(uint64_t, T&, CDT* cdt = NULL); - template void setValue(int, int64_t, int64_t, int64_t, T* = NULL); - template void setValue(uint64_t, T&); - template void implicit2T(uint64_t, T&, int); - template void getConstValue(execplan::ConstantColumn*, T&, bool&); + virtual void* getNullValueByType(int, int); - virtual void* getNullValueByType(int, int); + // There are two types of getters for integral types and for + // DTs wider then 8 bytes. + void getInt128Value(uint64_t i, int128_t& x) + { + return fRow.getInt128Field(i, x); + } + int64_t getIntValue(uint64_t i) + { + return fRow.getIntField(i); + } + double getDoubleValue(uint64_t i) + { + return fRow.getDoubleField(i); + } + long double getLongDoubleValue(uint64_t i) + { + return fRow.getLongDoubleField(i); + } + void setIntValue(int64_t i, int64_t v) + { + fRow.setIntField(v, i); + } + void setInt128Value(int64_t i, const int128_t& v) + { + fRow.setInt128Field(v, i); + } + void setDoubleValue(int64_t i, double v) + { + fRow.setDoubleField(v, i); + } + void setLongDoubleValue(int64_t i, const long double& v) + { + fRow.setLongDoubleField(v, i); + } - // There are two types of getters for integral types and for - // DTs wider then 8 bytes. - void getInt128Value(uint64_t i, int128_t& x) - { - return fRow.getInt128Field(i, x); - } - int64_t getIntValue(uint64_t i) - { - return fRow.getIntField(i); - } - double getDoubleValue(uint64_t i) - { - return fRow.getDoubleField(i); - } - long double getLongDoubleValue(uint64_t i) - { - return fRow.getLongDoubleField(i); - } - void setIntValue(int64_t i, int64_t v) - { - fRow.setIntField(v, i); - } - void setInt128Value(int64_t i, const int128_t& v) - { - fRow.setInt128Field(v, i); - } - void setDoubleValue(int64_t i, double v) - { - fRow.setDoubleField(v, i); - } - void setLongDoubleValue(int64_t i, const long double& v) - { - fRow.setLongDoubleField(v, i); - } + // for string table + rowgroup::Row::Pointer getPointer(joblist::RowPosition& r) + { + return fStep->getPointer(r, fRowGroup, fRow); + } - // for string table - rowgroup::Row::Pointer getPointer(joblist::RowPosition& r) - { - return fStep->getPointer(r, fRowGroup, fRow); - } + // function type + int64_t fFunctionId; + std::string fFunctionName; - // function type - int64_t fFunctionId; - std::string fFunctionName; + // output and input field indices: [0] - output + std::vector fFieldIndex; - // output and input field indices: [0] - output - std::vector fFieldIndex; + // constant function parameters -- needed for udaf with constant + std::vector fConstantParms; - // constant function parameters -- needed for udaf with constant - std::vector fConstantParms; + // row meta data + rowgroup::RowGroup fRowGroup; + rowgroup::Row fRow; - // row meta data - rowgroup::RowGroup fRowGroup; - rowgroup::Row fRow; + // data set + boost::shared_ptr > fRowData; - // data set - boost::shared_ptr > fRowData; + // frame unit ( ROWS | RANGE ) + int64_t fFrameUnit; - // frame unit ( ROWS | RANGE ) - int64_t fFrameUnit; + // partition + std::pair fPartition; - // partition - std::pair fPartition; - - // functor for peer checking - boost::shared_ptr fPeer; - int64_t fPrev; - - // for checking if query is cancelled - joblist::WindowFunctionStep* fStep; + // functor for peer checking + boost::shared_ptr fPeer; + int64_t fPrev; + // for checking if query is cancelled + joblist::WindowFunctionStep* fStep; }; - extern std::map colType2String; - -} // namespace +} // namespace windowfunction #endif // UTILS_WINDOWFUNCTIONTYPE_H // vim:ts=4 sw=4: - diff --git a/utils/winport/WinSyslog.cpp b/utils/winport/WinSyslog.cpp index cf03d454b..00e2db62f 100644 --- a/utils/winport/WinSyslog.cpp +++ b/utils/winport/WinSyslog.cpp @@ -42,118 +42,118 @@ WinSyslog* WinSyslog::fpSysLog = WinSyslog::instance(); // mutex name fhMutex. If creation succeeds, we set fbGoodIPMutex so the code knows // it's good. If we fail, fMutex (Only this processes threads) will be used - it's // better than nothing. -boost::mutex WinSyslog::fMutex; // Failsafe Intra process mutex -HANDLE WinSyslog::fhMutex; // Interprocess named mutex +boost::mutex WinSyslog::fMutex; // Failsafe Intra process mutex +HANDLE WinSyslog::fhMutex; // Interprocess named mutex bool WinSyslog::fbGoodIPMutex = false; -#define WINSYSLOG_MUTEX_NAME "WinSysLogMutex" // For the Windows Named Mutex +#define WINSYSLOG_MUTEX_NAME "WinSysLogMutex" // For the Windows Named Mutex // Constructor, wherein we initialize everything -WinSyslog::WinSyslog() : - fLastArchiveTime(0), - fLastArchiveDay(0), - fLogLineheaderSize(0) +WinSyslog::WinSyslog() : fLastArchiveTime(0), fLastArchiveDay(0), fLogLineheaderSize(0) { - fLogDirName = IDBreadRegistry("") + "\\log"; - fLogFileName = IDBreadRegistry("") + "\\log\\InfiniDBLog.txt"; - fTimeFileName = IDBreadRegistry("") + "\\log\\InfiniDBLastArchive.dat"; + fLogDirName = IDBreadRegistry("") + "\\log"; + fLogFileName = IDBreadRegistry("") + "\\log\\InfiniDBLog.txt"; + fTimeFileName = IDBreadRegistry("") + "\\log\\InfiniDBLastArchive.dat"; - // Set up a buffer with all the line header except the timestamp. - // Leave room in the front for the timestamp. - // This header will approximate the linux standard syslog log header. - memset(fLogLineHeader, 0, LOG_BUF_SIZE); - char* pEnd = fLogLineHeader + LOG_BUF_SIZE; - char* pHeader = fLogLineHeader + TIMESTAMP_SIZE; - char exePath[LOG_BUF_SIZE]; - DWORD nameSize = (DWORD)(pEnd - pHeader); - GetComputerName(pHeader, &nameSize); - pHeader += nameSize; - *pHeader++ = ' '; - GetModuleFileName(0, exePath, LOG_BUF_SIZE); - _splitpath_s(exePath, NULL, 0, NULL, 0, pHeader, pEnd - pHeader, NULL, 0); - pHeader += strlen(pHeader);; - *pHeader++ = '['; - int pid = GetCurrentProcessId(); - snprintf(pHeader, pEnd - pHeader, "%d", pid); - pHeader += strlen(pHeader);; - *pHeader++ = ']'; - *pHeader++ = ':'; - fLogLineheaderSize = (int)(pHeader - fLogLineHeader); - // Set up the date we compare against for archiving. - // This will either be today (if fTimeFileName doesn't exist) - // or the the date stored in fTimeFileName. - time_t now = time(0); - FILE* lf; - lf = fopen(fTimeFileName.c_str(), "r+b"); + // Set up a buffer with all the line header except the timestamp. + // Leave room in the front for the timestamp. + // This header will approximate the linux standard syslog log header. + memset(fLogLineHeader, 0, LOG_BUF_SIZE); + char* pEnd = fLogLineHeader + LOG_BUF_SIZE; + char* pHeader = fLogLineHeader + TIMESTAMP_SIZE; + char exePath[LOG_BUF_SIZE]; + DWORD nameSize = (DWORD)(pEnd - pHeader); + GetComputerName(pHeader, &nameSize); + pHeader += nameSize; + *pHeader++ = ' '; + GetModuleFileName(0, exePath, LOG_BUF_SIZE); + _splitpath_s(exePath, NULL, 0, NULL, 0, pHeader, pEnd - pHeader, NULL, 0); + pHeader += strlen(pHeader); + ; + *pHeader++ = '['; + int pid = GetCurrentProcessId(); + snprintf(pHeader, pEnd - pHeader, "%d", pid); + pHeader += strlen(pHeader); + ; + *pHeader++ = ']'; + *pHeader++ = ':'; + fLogLineheaderSize = (int)(pHeader - fLogLineHeader); + // Set up the date we compare against for archiving. + // This will either be today (if fTimeFileName doesn't exist) + // or the the date stored in fTimeFileName. + time_t now = time(0); + FILE* lf; + lf = fopen(fTimeFileName.c_str(), "r+b"); - if (lf == NULL) // First time archiving. + if (lf == NULL) // First time archiving. + { + fLastArchiveTime = now; + // Persist the timestamp of last archive. + lf = fopen(fTimeFileName.c_str(), "w+b"); + + if (lf != 0) { - fLastArchiveTime = now; - // Persist the timestamp of last archive. - lf = fopen(fTimeFileName.c_str(), "w+b"); - - if (lf != 0) - { - fwrite(&fLastArchiveTime, sizeof(time_t), 1, lf); - fclose(lf); - } - } - else - { - fread(&fLastArchiveTime, sizeof(time_t), 1, lf); - fclose(lf); + fwrite(&fLastArchiveTime, sizeof(time_t), 1, lf); + fclose(lf); } + } + else + { + fread(&fLastArchiveTime, sizeof(time_t), 1, lf); + fclose(lf); + } - struct tm lasttm; + struct tm lasttm; - localtime_s(&lasttm, &fLastArchiveTime); + localtime_s(&lasttm, &fLastArchiveTime); - fLastArchiveDay = lasttm.tm_yday; + fLastArchiveDay = lasttm.tm_yday; - // Create an interprocess mutex to coordinate the archiving function - fhMutex = CreateMutex(NULL, FALSE, WINSYSLOG_MUTEX_NAME); + // Create an interprocess mutex to coordinate the archiving function + fhMutex = CreateMutex(NULL, FALSE, WINSYSLOG_MUTEX_NAME); - if (fhMutex == NULL) - { - syslog(LOG_ERR, "WinSyslog CreateMutex error: %d\n", GetLastError()); - } - else - { - fbGoodIPMutex = true; - } + if (fhMutex == NULL) + { + syslog(LOG_ERR, "WinSyslog CreateMutex error: %d\n", GetLastError()); + } + else + { + fbGoodIPMutex = true; + } } WinSyslog::~WinSyslog() { - CloseHandle(fhMutex); + CloseHandle(fhMutex); } // Log() is called by ::syslog() to actually write stuff to the file int WinSyslog::Log(int priority, const char* format, va_list& args) { - struct tm nowtm; - FILE* f; - time_t now = time(0); + struct tm nowtm; + FILE* f; + time_t now = time(0); - localtime_s(&nowtm, &now); + localtime_s(&nowtm, &now); - // If now isn't the same day as the last archive date, archve. - if (nowtm.tm_yday != fLastArchiveDay) - { - Archive(nowtm); - } + // If now isn't the same day as the last archive date, archve. + if (nowtm.tm_yday != fLastArchiveDay) + { + Archive(nowtm); + } - // Log the line. - strftime(fLogLineHeader, TIMESTAMP_SIZE, "%b %d %H:%M:%S", &nowtm); - fLogLineHeader[TIMESTAMP_SIZE - 1] = ' '; - f = fopen(fLogFileName.c_str(), "a+"); + // Log the line. + strftime(fLogLineHeader, TIMESTAMP_SIZE, "%b %d %H:%M:%S", &nowtm); + fLogLineHeader[TIMESTAMP_SIZE - 1] = ' '; + f = fopen(fLogFileName.c_str(), "a+"); - if (f == 0) return -1; + if (f == 0) + return -1; - fwrite(fLogLineHeader, 1, fLogLineheaderSize, f); - vfprintf(f, format, args); - fwrite("\n", 1, 1, f); - fclose(f); - return 0; + fwrite(fLogLineHeader, 1, fLogLineheaderSize, f); + vfprintf(f, format, args); + fwrite("\n", 1, 1, f); + fclose(f); + return 0; } // Archive() is called by Log() when the date changes @@ -161,165 +161,163 @@ int WinSyslog::Log(int priority, const char* format, va_list& args) // any old files. void WinSyslog::Archive(const tm& nowtm) { - struct tm yesterdaytm; - bool bArchive = true; - time_t writeTime; - char ctimebuf[TIME_BUF_SIZE] = {0}; - DWORD dwWaitResult; - char* szError = ""; + struct tm yesterdaytm; + bool bArchive = true; + time_t writeTime; + char ctimebuf[TIME_BUF_SIZE] = {0}; + DWORD dwWaitResult; + char* szError = ""; - // Lock it so we don't have two threads archiving. - if (fbGoodIPMutex) + // Lock it so we don't have two threads archiving. + if (fbGoodIPMutex) + { + // We have a good interprocess mutex. Try for one second to + // lock it. Abandon trying after a second. Something's stuck. + dwWaitResult = WaitForSingleObject(fhMutex, 1000); + + switch (dwWaitResult) { - // We have a good interprocess mutex. Try for one second to - // lock it. Abandon trying after a second. Something's stuck. - dwWaitResult = WaitForSingleObject(fhMutex, 1000); + // The thread got ownership of the mutex. Continue on. + case WAIT_OBJECT_0: break; - switch (dwWaitResult) - { - // The thread got ownership of the mutex. Continue on. - case WAIT_OBJECT_0: - break; + // The thread got ownership of an abandoned mutex, most likely because + // some process crashed in the middle of archiving. Continue on. + case WAIT_ABANDONED: break; - // The thread got ownership of an abandoned mutex, most likely because - // some process crashed in the middle of archiving. Continue on. - case WAIT_ABANDONED: - break; + // We timed out. Something's not right. Don't archive. + case WAIT_TIMEOUT: + szError = "WinSyslog::Archive WAIT_TIMEOUT"; + bArchive = false; + break; - // We timed out. Something's not right. Don't archive. - case WAIT_TIMEOUT: - szError = "WinSyslog::Archive WAIT_TIMEOUT"; - bArchive = false; - break; + // Horrible failure. Don't archive + case WAIT_FAILED: + szError = "WinSyslog::Archive WAIT_FAILED"; + bArchive = false; + break; + } + } + else + { + boost::mutex::scoped_lock lock(fMutex); + } - // Horrible failure. Don't archive - case WAIT_FAILED: - szError = "WinSyslog::Archive WAIT_FAILED"; - bArchive = false; - break; - } + try + { + // Check to see if any other thread or process already archived. + // If the value in fTimeFileName is different than our saved value + // Then somebody else beat us to it. Don't do it again. + time_t storedLastArchiveTime; + FILE* lf; + lf = fopen(fTimeFileName.c_str(), "r+b"); + + if (lf != NULL) + { + fread(&storedLastArchiveTime, sizeof(time_t), 1, lf); + fclose(lf); + + if (storedLastArchiveTime != fLastArchiveTime) + { + bArchive = false; + } } else { - boost::mutex::scoped_lock lock(fMutex); + // If we had something go wrong earlier, and then failed to read the Time file, + // this should prevent retrying to archive every log line. + storedLastArchiveTime = fLastArchiveTime = time(NULL); } - try + if (bArchive == false) { - // Check to see if any other thread or process already archived. - // If the value in fTimeFileName is different than our saved value - // Then somebody else beat us to it. Don't do it again. - time_t storedLastArchiveTime; - FILE* lf; - lf = fopen(fTimeFileName.c_str(), "r+b"); + // Something's not right. Try to set the internals so we don't attempt + // archiving again. + fLastArchiveTime = storedLastArchiveTime; + fLastArchiveDay = nowtm.tm_yday; - if (lf != NULL) - { - fread(&storedLastArchiveTime, sizeof(time_t), 1, lf); - fclose(lf); + if (szError) + { + syslog(LOG_ERR, szError); + } - if (storedLastArchiveTime != fLastArchiveTime) - { - bArchive = false; - } - } - else - { - // If we had something go wrong earlier, and then failed to read the Time file, - // this should prevent retrying to archive every log line. - storedLastArchiveTime = fLastArchiveTime = time(NULL); - } - - if (bArchive == false) - { - // Something's not right. Try to set the internals so we don't attempt - // archiving again. - fLastArchiveTime = storedLastArchiveTime; - fLastArchiveDay = nowtm.tm_yday; - - if (szError) - { - syslog(LOG_ERR, szError); - } - - return; - } - - // Get last archive date (usually yesterday) - yesterdaytm = nowtm; - yesterdaytm.tm_mday -= 1; // May be 0. mktime() adjusts accordingly. - time_t yesterday = mktime(&yesterdaytm); - localtime_s(&yesterdaytm, &yesterday); - string archiveFileName = fLogFileName; - size_t tail = archiveFileName.find(".txt"); - archiveFileName.erase(tail, 4); - strftime(ctimebuf, TIME_BUF_SIZE, "-%Y-%m-%d.txt", &yesterdaytm); - archiveFileName += ctimebuf; - rename(fLogFileName.c_str(), archiveFileName.c_str()); - - // Persist the timestamp of last archive. In case of reboot, we - // can still archive properly - lf = fopen(fTimeFileName.c_str(), "w+b"); - - if (lf != 0) - { - fLastArchiveTime = time(0); - fwrite(&fLastArchiveTime, sizeof(time_t), 1, lf); - fclose(lf); - } - - fLastArchiveDay = nowtm.tm_yday; - - // Get the dates of all archive files. - fs::path sourceDir(fLogDirName); - fs::directory_iterator iter(sourceDir); - fs::directory_iterator end_iter; - std::multimap fileSet; // Stays sorted - - while (iter != end_iter) - { - fs::path archiveFile = *iter; - - if (fs::is_regular_file(archiveFile) ) - { - if (archiveFile.extension() == ".txt") - { - writeTime = fs::last_write_time(archiveFile); - fileSet.insert(make_pair(writeTime, archiveFile)); - } - } - - ++iter; - } - - // Delete anything past the first seven newest files - typedef std::multimap::iterator PATH_ITER; - PATH_ITER fileIter = fileSet.begin(); - PATH_ITER fileIterEnd = fileSet.end(); - std::reverse_iterator revEnd(fileIter); - std::reverse_iterator revIter(fileIterEnd); - int cnt = 0; - - for (; revIter != revEnd; ++revIter) - { - if (cnt++ < 7) - continue; - - _wunlink(revIter->second.c_str()); - } + return; } - catch (exception& e) + + // Get last archive date (usually yesterday) + yesterdaytm = nowtm; + yesterdaytm.tm_mday -= 1; // May be 0. mktime() adjusts accordingly. + time_t yesterday = mktime(&yesterdaytm); + localtime_s(&yesterdaytm, &yesterday); + string archiveFileName = fLogFileName; + size_t tail = archiveFileName.find(".txt"); + archiveFileName.erase(tail, 4); + strftime(ctimebuf, TIME_BUF_SIZE, "-%Y-%m-%d.txt", &yesterdaytm); + archiveFileName += ctimebuf; + rename(fLogFileName.c_str(), archiveFileName.c_str()); + + // Persist the timestamp of last archive. In case of reboot, we + // can still archive properly + lf = fopen(fTimeFileName.c_str(), "w+b"); + + if (lf != 0) { - syslog(LOG_ERR, "WinSyslog::Archive Exception %s", e.what()); - } - catch (...) - { - syslog(LOG_ERR, "WinSyslog::Archive Exception ..."); + fLastArchiveTime = time(0); + fwrite(&fLastArchiveTime, sizeof(time_t), 1, lf); + fclose(lf); } - if (fbGoodIPMutex) + fLastArchiveDay = nowtm.tm_yday; + + // Get the dates of all archive files. + fs::path sourceDir(fLogDirName); + fs::directory_iterator iter(sourceDir); + fs::directory_iterator end_iter; + std::multimap fileSet; // Stays sorted + + while (iter != end_iter) { - if (!ReleaseMutex(fhMutex)) - syslog(LOG_ERR, "WinSyslog::Archive ReleaseMutex failed %d", GetLastError()); + fs::path archiveFile = *iter; + + if (fs::is_regular_file(archiveFile)) + { + if (archiveFile.extension() == ".txt") + { + writeTime = fs::last_write_time(archiveFile); + fileSet.insert(make_pair(writeTime, archiveFile)); + } + } + + ++iter; } + + // Delete anything past the first seven newest files + typedef std::multimap::iterator PATH_ITER; + PATH_ITER fileIter = fileSet.begin(); + PATH_ITER fileIterEnd = fileSet.end(); + std::reverse_iterator revEnd(fileIter); + std::reverse_iterator revIter(fileIterEnd); + int cnt = 0; + + for (; revIter != revEnd; ++revIter) + { + if (cnt++ < 7) + continue; + + _wunlink(revIter->second.c_str()); + } + } + catch (exception& e) + { + syslog(LOG_ERR, "WinSyslog::Archive Exception %s", e.what()); + } + catch (...) + { + syslog(LOG_ERR, "WinSyslog::Archive Exception ..."); + } + + if (fbGoodIPMutex) + { + if (!ReleaseMutex(fhMutex)) + syslog(LOG_ERR, "WinSyslog::Archive ReleaseMutex failed %d", GetLastError()); + } } diff --git a/utils/winport/WinSyslog.h b/utils/winport/WinSyslog.h index 04b3f40e1..f8d87b95b 100644 --- a/utils/winport/WinSyslog.h +++ b/utils/winport/WinSyslog.h @@ -23,64 +23,64 @@ class WinSyslog { -public: - /** - * @brief dtor - * Might work as private, but not sure. - */ - ~WinSyslog(); - /** - * @brief Singleton accessor. - * Doesn't need locking because we use early construction. - */ - static WinSyslog* instance() - { - if (!fpSysLog) - fpSysLog = new WinSyslog(); + public: + /** + * @brief dtor + * Might work as private, but not sure. + */ + ~WinSyslog(); + /** + * @brief Singleton accessor. + * Doesn't need locking because we use early construction. + */ + static WinSyslog* instance() + { + if (!fpSysLog) + fpSysLog = new WinSyslog(); - return fpSysLog; - }; + return fpSysLog; + }; - int OpenLog() - { - return 0; - }; - int CloseLog() - { - return 0; - }; + int OpenLog() + { + return 0; + }; + int CloseLog() + { + return 0; + }; - /** - * @brief Logging function. - * Called by ::syslog to do the logging - */ - int Log(int priority, const char* format, va_list& args); + /** + * @brief Logging function. + * Called by ::syslog to do the logging + */ + int Log(int priority, const char* format, va_list& args); -private: - // Disable public construction, destruction and assignment - WinSyslog(); - WinSyslog(WinSyslog&); - void operator=(WinSyslog const&); + private: + // Disable public construction, destruction and assignment + WinSyslog(); + WinSyslog(WinSyslog&); + void operator=(WinSyslog const&); - /** - * @brief Archiving function. - * Called by Log() at midnight to archive the log - * file and create a new one - */ - void Archive(const tm& nowtm); + /** + * @brief Archiving function. + * Called by Log() at midnight to archive the log + * file and create a new one + */ + void Archive(const tm& nowtm); - time_t fLastArchiveTime; - int fLastArchiveDay; - string fLogDirName; - string fLogFileName; - string fTimeFileName; - char fLogLineHeader[LOG_BUF_SIZE]; - int fLogLineheaderSize; + time_t fLastArchiveTime; + int fLastArchiveDay; + string fLogDirName; + string fLogFileName; + string fTimeFileName; + char fLogLineHeader[LOG_BUF_SIZE]; + int fLogLineheaderSize; - // Singleton pointer - static WinSyslog* fpSysLog; - // Synchronize objects for Archiving. - static boost::mutex fMutex; - static HANDLE fhMutex; - static bool fbGoodIPMutex; + // Singleton pointer + static WinSyslog* fpSysLog; + // Synchronize objects for Archiving. + static boost::mutex fMutex; + static HANDLE fhMutex; + static bool fbGoodIPMutex; }; diff --git a/utils/winport/bootstrap.cpp b/utils/winport/bootstrap.cpp index dd8d44c55..c4664e48d 100644 --- a/utils/winport/bootstrap.cpp +++ b/utils/winport/bootstrap.cpp @@ -44,328 +44,337 @@ const size_t cmdLineLen = 2 * 1024; int fixPath() { - int rc; - string newDir = installDir + "\\bin"; - rc = _chdir(newDir.c_str()); - return rc; + int rc; + string newDir = installDir + "\\bin"; + rc = _chdir(newDir.c_str()); + return rc; } int runIt(const string& pName) { - char* cmdLine = (char*)_malloca(cmdLineLen); - strncpy_s(cmdLine, cmdLineLen, pName.c_str(), pName.size()); - PROCESS_INFORMATION pInfo; - ZeroMemory(&pInfo, sizeof(pInfo)); - STARTUPINFO sInfo; - ZeroMemory(&sInfo, sizeof(sInfo)); + char* cmdLine = (char*)_malloca(cmdLineLen); + strncpy_s(cmdLine, cmdLineLen, pName.c_str(), pName.size()); + PROCESS_INFORMATION pInfo; + ZeroMemory(&pInfo, sizeof(pInfo)); + STARTUPINFO sInfo; + ZeroMemory(&sInfo, sizeof(sInfo)); - if (CreateProcess(0, cmdLine, 0, 0, false, 0, 0, 0, &sInfo, &pInfo) == 0) - { - _freea(cmdLine); - return -1; - } - - if (WaitForSingleObject(pInfo.hProcess, INFINITE) != WAIT_OBJECT_0) - { - _freea(cmdLine); - return -1; - } - - DWORD exitCode; - GetExitCodeProcess(pInfo.hProcess, &exitCode); - - if (exitCode != 0) - { - _freea(cmdLine); - return -1; - } - - CloseHandle(pInfo.hProcess); + if (CreateProcess(0, cmdLine, 0, 0, false, 0, 0, 0, &sInfo, &pInfo) == 0) + { _freea(cmdLine); - return 0; + return -1; + } + + if (WaitForSingleObject(pInfo.hProcess, INFINITE) != WAIT_OBJECT_0) + { + _freea(cmdLine); + return -1; + } + + DWORD exitCode; + GetExitCodeProcess(pInfo.hProcess, &exitCode); + + if (exitCode != 0) + { + _freea(cmdLine); + return -1; + } + + CloseHandle(pInfo.hProcess); + _freea(cmdLine); + return 0; } int installMySQL() { - int rc; - char* cmdLine = (char*)_malloca(cmdLineLen); - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysqld.exe --defaults-file=%s\\my.ini --bootstrap --loose-skip-innodb < %s\\etc\\win_setup_mysql_part1.sql", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - _freea(cmdLine); - return rc; + int rc; + char* cmdLine = (char*)_malloca(cmdLineLen); + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysqld.exe --defaults-file=%s\\my.ini --bootstrap --loose-skip-innodb < " + "%s\\etc\\win_setup_mysql_part1.sql", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + _freea(cmdLine); + return rc; } int installIDBConn() { - int rc; - char* cmdLine = (char*)_malloca(cmdLineLen); - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\etc\\win_setup_mysql_part2.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - _freea(cmdLine); - return rc; + int rc; + char* cmdLine = (char*)_malloca(cmdLineLen); + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\etc\\win_setup_mysql_part2.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + _freea(cmdLine); + return rc; } int installIDBConnStep2() { - int rc = -1; - char* cmdLine = (char*)_malloca(cmdLineLen); - FILE* p; - bool needAlterTable1 = false; - bool needAlterTable2 = false; + int rc = -1; + char* cmdLine = (char*)_malloca(cmdLineLen); + FILE* p; + bool needAlterTable1 = false; + bool needAlterTable2 = false; - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --execute=\"describe syscolumn;\" calpontsys 2>&1", - installDir.c_str(), installDir.c_str()); + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --execute=\"describe syscolumn;\" " + "calpontsys 2>&1", + installDir.c_str(), installDir.c_str()); - p = _popen(cmdLine, "rt"); + p = _popen(cmdLine, "rt"); - if (p == NULL) goto out; + if (p == NULL) + goto out; - if (!grepit(syncstream::isyncstream(p), "^compressiontype[ \t].*")) - needAlterTable1 = true; + if (!grepit(syncstream::isyncstream(p), "^compressiontype[ \t].*")) + needAlterTable1 = true; - _pclose(p); + _pclose(p); - p = _popen(cmdLine, "rt"); + p = _popen(cmdLine, "rt"); - if (p == NULL) goto out; + if (p == NULL) + goto out; - if (!grepit(syncstream::isyncstream(p), "^nextvalue[ \t].*")) - needAlterTable2 = true; + if (!grepit(syncstream::isyncstream(p), "^nextvalue[ \t].*")) + needAlterTable2 = true; - _pclose(p); + _pclose(p); - if (needAlterTable1 || needAlterTable2) + if (needAlterTable1 || needAlterTable2) + { + cout << "Checking InfiniDB components and system catalog for necessary upgrades..." << endl; + + // This script needs to be force-fed to mysqld and any errors ignored + if (needAlterTable1) { - cout << "Checking InfiniDB components and system catalog for necessary upgrades..." << endl; + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\etc\\win_setup_mysql_part3.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); - //This script needs to be force-fed to mysqld and any errors ignored - if (needAlterTable1) - { - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\etc\\win_setup_mysql_part3.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - - rc = system(cmdLine); - } - - if (needAlterTable2) - { - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\etc\\win_setup_mysql_part3.1.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - - rc = system(cmdLine); - } + rc = system(cmdLine); } + if (needAlterTable2) + { + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\etc\\win_setup_mysql_part3.1.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + + rc = system(cmdLine); + } + } + #ifndef SKIP_MYSQL_SETUP4 - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\etc\\win_setup_mysql_part4.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - rc = 0; + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\etc\\win_setup_mysql_part4.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + rc = 0; #endif - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\etc\\win_setup_mysql_part5.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - rc = 0; + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\etc\\win_setup_mysql_part5.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + rc = 0; - // Install the user priority stored procedures - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\sql\\calsetuserpriority.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - rc = 0; + // Install the user priority stored procedures + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\sql\\calsetuserpriority.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + rc = 0; - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\sql\\calremoveuserpriority.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - rc = 0; + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\sql\\calremoveuserpriority.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + rc = 0; - sprintf_s(cmdLine, cmdLineLen, - "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < %s\\sql\\calshowprocesslist.sql > nul 2>&1", - installDir.c_str(), installDir.c_str(), installDir.c_str()); - rc = system(cmdLine); - rc = 0; + sprintf_s(cmdLine, cmdLineLen, + "%s\\bin\\mysql.exe --defaults-file=%s\\my.ini --user=root --force < " + "%s\\sql\\calshowprocesslist.sql > nul 2>&1", + installDir.c_str(), installDir.c_str(), installDir.c_str()); + rc = system(cmdLine); + rc = 0; out: - _freea(cmdLine); - return rc; + _freea(cmdLine); + return rc; } int startupIDB() { - int rc; - string cmd; - cmd = installDir + "\\bin\\winfinidb.exe start"; - rc = runIt(cmd); - return rc; + int rc; + string cmd; + cmd = installDir + "\\bin\\winfinidb.exe start"; + rc = runIt(cmd); + return rc; } int installIDBDB() { - int rc; + int rc; - for (int retry = 0; retry < 5; retry++) - { - rc = installIDBConn(); + for (int retry = 0; retry < 5; retry++) + { + rc = installIDBConn(); - if (rc == 0) - break; + if (rc == 0) + break; - Sleep(5 * (retry + 1) * 1000); - } + Sleep(5 * (retry + 1) * 1000); + } - if (rc) - return rc; - - rc = installIDBConnStep2(); - - if (rc) - return rc; - - string cmd; - cmd = installDir + "\\bin\\dbbuilder.exe 7"; - rc = runIt(cmd); + if (rc) return rc; + + rc = installIDBConnStep2(); + + if (rc) + return rc; + + string cmd; + cmd = installDir + "\\bin\\dbbuilder.exe 7"; + rc = runIt(cmd); + return rc; } int upgradeInfiniDB() { - int rc = 0; + int rc = 0; - //We'll just have to blast these changes in... - installIDBConn(); + // We'll just have to blast these changes in... + installIDBConn(); - //Add new enterprise functions & new syscolumn columns - rc = installIDBConnStep2(); - - if (rc) - return rc; + // Add new enterprise functions & new syscolumn columns + rc = installIDBConnStep2(); + if (rc) return rc; + + return rc; } -} +} // namespace int main(int argc, char** argv) { - opterr = 0; - vFlg = false; - int c; + opterr = 0; + vFlg = false; + int c; - while ((c = getopt(argc, argv, "v")) != EOF) - switch (c) - { - case 'v': - vFlg = true; - break; - - case '?': - default: - break; - } - - string mysqlPort("3306"); - - if (argc - optind >= 1) - mysqlPort = argv[optind++]; - - installDir = IDBreadRegistry("", true); - - if (fixPath()) + while ((c = getopt(argc, argv, "v")) != EOF) + switch (c) { - cerr << "Something went wrong trying to change to the install directory " << installDir << endl; - return 1; + case 'v': vFlg = true; break; + + case '?': + default: break; } - cout << "Running InfiniDB bootstrap installer..." << endl; + string mysqlPort("3306"); - cout << "Tuning configuration..." << endl; + if (argc - optind >= 1) + mysqlPort = argv[optind++]; - string moduleFile = installDir + "\\local\\module"; + installDir = IDBreadRegistry("", true); - if (_access(moduleFile.c_str(), F_OK) != 0) + if (fixPath()) + { + cerr << "Something went wrong trying to change to the install directory " << installDir << endl; + return 1; + } + + cout << "Running InfiniDB bootstrap installer..." << endl; + + cout << "Tuning configuration..." << endl; + + string moduleFile = installDir + "\\local\\module"; + + if (_access(moduleFile.c_str(), F_OK) != 0) + { + ofstream mfs(moduleFile.c_str()); + + if (!mfs.good()) { - ofstream mfs(moduleFile.c_str()); - - if (!mfs.good()) - { - cerr << "Something went wrong creating the module file" << endl; - return 1; - } - - mfs << "pm1" << endl; - mfs.close(); + cerr << "Something went wrong creating the module file" << endl; + return 1; } - if (fixupConfig(installDir, mysqlPort)) + mfs << "pm1" << endl; + mfs.close(); + } + + if (fixupConfig(installDir, mysqlPort)) + { + cerr << "Something went wrong fixing up a config file" << endl; + return 1; + } + + string mysqldb = installDir + "\\mysqldb\\mysql\\user.frm"; + struct _stat statbuf; + + if (_stat(mysqldb.c_str(), &statbuf) == 0) + { + cout << "Using existing MySQL database." << endl; + } + else + { + cout << "Installing empty MySQL database..." << endl; + + if (installMySQL()) { - cerr << "Something went wrong fixing up a config file" << endl; - return 1; + cerr << "Something went wrong trying to setup MySQL" << endl; + return 1; } + } - string mysqldb = installDir + "\\mysqldb\\mysql\\user.frm"; - struct _stat statbuf; + cout << "Starting InfiniDB..." << endl; - if (_stat(mysqldb.c_str(), &statbuf) == 0) + if (startupIDB()) + { + cerr << "Something went wrong trying to start InfiniDB" << endl; + return 1; + } + + Sleep(5 * 1000); + + string cnffile = installDir + "\\data1\\000.dir"; + + if (_stat(cnffile.c_str(), &statbuf) == 0) + { + cout << "Using existing InfiniDB database." << endl; + } + else + { + cout << "Installing InfiniDB components and system catalog..." << endl; + + if (installIDBDB()) { - cout << "Using existing MySQL database." << endl; + cerr << "Something went wrong trying to install InfiniDB components" << endl; + return 1; } - else - { - cout << "Installing empty MySQL database..." << endl; + } - if (installMySQL()) - { - cerr << "Something went wrong trying to setup MySQL" << endl; - return 1; - } - } + if (upgradeInfiniDB()) + { + cerr << "Something went wrong trying to upgrade InfiniDB components" << endl; + return 1; + } - cout << "Starting InfiniDB..." << endl; + cout << "Done!" << endl; - if (startupIDB()) - { - cerr << "Something went wrong trying to start InfiniDB" << endl; - return 1; - } + Sleep(3 * 1000); - Sleep(5 * 1000); - - string cnffile = installDir + "\\data1\\000.dir"; - - if (_stat(cnffile.c_str(), &statbuf) == 0) - { - cout << "Using existing InfiniDB database." << endl; - } - else - { - cout << "Installing InfiniDB components and system catalog..." << endl; - - if (installIDBDB()) - { - cerr << "Something went wrong trying to install InfiniDB components" << endl; - return 1; - } - } - - if (upgradeInfiniDB()) - { - cerr << "Something went wrong trying to upgrade InfiniDB components" << endl; - return 1; - } - - cout << "Done!" << endl; - - Sleep(3 * 1000); - - return 0; + return 0; } diff --git a/utils/winport/fixup.cpp b/utils/winport/fixup.cpp index 4e926d147..dfba53a50 100644 --- a/utils/winport/fixup.cpp +++ b/utils/winport/fixup.cpp @@ -36,538 +36,536 @@ using namespace winport; namespace { - int fixupCalpontXML() { - int rc = -1; //assume the worst + int rc = -1; // assume the worst - DWORDLONG totalMemSize = 1024ULL * 1024ULL * 1024ULL * 2ULL; + DWORDLONG totalMemSize = 1024ULL * 1024ULL * 1024ULL * 2ULL; #ifdef _WIN64 - totalMemSize *= 4ULL; + totalMemSize *= 4ULL; #endif - MEMORYSTATUSEX memStat; - memStat.dwLength = sizeof(memStat); + MEMORYSTATUSEX memStat; + memStat.dwLength = sizeof(memStat); - if (GlobalMemoryStatusEx(&memStat) == 0) - //FIXME: Assume 2GB? - (void)0; + if (GlobalMemoryStatusEx(&memStat) == 0) + // FIXME: Assume 2GB? + (void)0; + else + { +#ifndef _WIN64 + memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); +#endif + // We now have the total phys mem in bytes + // FIXME: should we use avail phys mem instead? + totalMemSize = memStat.ullTotalPhys; + } + + try + { + Config* cf = Config::makeConfig(); + string section; + string parm; + string val; + + // Fixup ConnectionsPerPrimProc + section = "PrimitiveServers"; + parm = "ConnectionsPerPrimProc"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "2"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } else { -#ifndef _WIN64 - memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys); -#endif - //We now have the total phys mem in bytes - //FIXME: should we use avail phys mem instead? - totalMemSize = memStat.ullTotalPhys; + cf->delConfig(section, parm); + cf->setConfig(section, parm, val); } - try + // Fixup PrefetchThreshold + parm = "PrefetchThreshold"; + val = cf->getConfig(section, parm); + + if (val == "30") { - Config* cf = Config::makeConfig(); - string section; - string parm; - string val; - - //Fixup ConnectionsPerPrimProc - section = "PrimitiveServers"; - parm = "ConnectionsPerPrimProc"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "2"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - else - { - cf->delConfig(section, parm); - cf->setConfig(section, parm, val); - } - - //Fixup PrefetchThreshold - parm = "PrefetchThreshold"; - val = cf->getConfig(section, parm); - - if (val == "30") - { - val = "5"; - cf->setConfig(section, parm, val); - cout << "Fixing " << section << "." << parm << " = " << val << endl; - } - - //Fixup Count - parm = "Count"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup PMS2 - section = "PMS2"; - parm = "IPAddr"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "127.0.0.1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "Port"; - val = "8620"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup UmMaxMemorySmallSide - section = "HashJoin"; - parm = "UmMaxMemorySmallSide"; - val = cf->getConfig(section, parm); -#ifdef _WIN64 - - if (val == "4G") -#else - if (val == "256M") -#endif - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup TotalUmMaxMemorySmallSide - parm = "TotalUmMaxMemorySmallSide"; - val = cf->getConfig(section, parm); -#ifdef _WIN64 - - if (val == "8G") -#else - if (val == "512M") -#endif - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup TotalUmMemory - parm = "TotalUmMemory"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { -#ifdef _WIN64 - totalMemSize /= (1024ULL * 1024ULL * 1024ULL); - - if (totalMemSize >= 31) - val = "16G"; - else if (totalMemSize >= 15) - val = "8G"; - else if (totalMemSize >= 7) - val = "4G"; - else - val = "2G"; - -#else - val = "2G"; -#endif - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup MaxMemoryPerUnion - section = "Union"; - parm = "MaxMemoryPerUnion"; - val = cf->getConfig(section, parm); -#ifdef _WIN64 - - if (val == "4G") -#else - if (val == "256M") -#endif - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup TotalUnionMemory - parm = "TotalUnionMemory"; - val = cf->getConfig(section, parm); -#ifdef _WIN64 - - if (val == "8G") -#else - if (val == "512M") -#endif - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup RowAggregation.MaxMemory - section = "RowAggregation"; - parm = "MaxMemory"; - val = cf->getConfig(section, parm); -#ifdef _WIN64 - - if (val == "1G") -#else - if (val == "128M") -#endif - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup OrderByLimit.MaxMemory - section = "OrderByLimit"; - parm = "MaxMemory"; - val = cf->getConfig(section, parm); -#ifdef _WIN64 - - if (val == "1G") -#else - if (val == "128M") -#endif - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup Installation.UMStorageType - section = "Installation"; - parm = "UMStorageType"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "internal"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup DBRootStorageType - parm = "DBRootStorageType"; - val = cf->getConfig(section, parm); - - if (val == "local") - { - val = "internal"; - cf->setConfig(section, parm, val); - cout << "Fixing " << section << "." << parm << " = " << val << endl; - } - - //Fixup WES - section = "pm1_WriteEngineServer"; - parm = "IPAddr"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "127.0.0.1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "Port"; - val = "8630"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup TableLockSaveFile - section = "SystemConfig"; - parm = "TableLockSaveFile"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = IDBreadRegistry("") + "/dbrm/tablelocks"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup SystemModuleConfig - section = "SystemModuleConfig"; - parm = "ModuleDBRootCount1-1"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "0"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "ModuleDBRootCount1-2"; - val = "0"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "ModuleDBRootCount1-3"; - val = "1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "ModuleDBRootID1-1-1"; - val = "0"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "ModuleDBRootID1-1-2"; - val = "0"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "ModuleDBRootID1-1-3"; - val = "1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup NVBF - section = "VersionBuffer"; - parm = "NumVersionBufferFiles"; - val = cf->getConfig(section, parm); - - if (!val.empty()) - { - cf->delConfig(section, parm); - cout << "Deleting " << section << "." << parm << endl; - } - - //Fixup AllowDiskBasedJoin - section = "HashJoin"; - parm = "AllowDiskBasedJoin"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "N"; // We default to No - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup TempFilePath - section = "HashJoin"; - parm = "TempFilePath"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "$INSTALLDIR/tmp"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - //Fixup TempFileCompression - section = "HashJoin"; - parm = "TempFileCompression"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "Y"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - - cf->write(); - rc = 0; + val = "5"; + cf->setConfig(section, parm, val); + cout << "Fixing " << section << "." << parm << " = " << val << endl; } - catch (exception& e) + + // Fixup Count + parm = "Count"; + val = cf->getConfig(section, parm); + + if (val.empty()) { - cout << "fixupCalpontXML caught exception: " << e.what() << endl; - } - catch (...) - { - cout << "fixupCalpontXML caught exception (...): " << endl; + val = "1"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; } - return rc; + // Fixup PMS2 + section = "PMS2"; + parm = "IPAddr"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "127.0.0.1"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "Port"; + val = "8620"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup UmMaxMemorySmallSide + section = "HashJoin"; + parm = "UmMaxMemorySmallSide"; + val = cf->getConfig(section, parm); +#ifdef _WIN64 + + if (val == "4G") +#else + if (val == "256M") +#endif + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup TotalUmMaxMemorySmallSide + parm = "TotalUmMaxMemorySmallSide"; + val = cf->getConfig(section, parm); +#ifdef _WIN64 + + if (val == "8G") +#else + if (val == "512M") +#endif + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup TotalUmMemory + parm = "TotalUmMemory"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { +#ifdef _WIN64 + totalMemSize /= (1024ULL * 1024ULL * 1024ULL); + + if (totalMemSize >= 31) + val = "16G"; + else if (totalMemSize >= 15) + val = "8G"; + else if (totalMemSize >= 7) + val = "4G"; + else + val = "2G"; + +#else + val = "2G"; +#endif + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup MaxMemoryPerUnion + section = "Union"; + parm = "MaxMemoryPerUnion"; + val = cf->getConfig(section, parm); +#ifdef _WIN64 + + if (val == "4G") +#else + if (val == "256M") +#endif + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup TotalUnionMemory + parm = "TotalUnionMemory"; + val = cf->getConfig(section, parm); +#ifdef _WIN64 + + if (val == "8G") +#else + if (val == "512M") +#endif + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup RowAggregation.MaxMemory + section = "RowAggregation"; + parm = "MaxMemory"; + val = cf->getConfig(section, parm); +#ifdef _WIN64 + + if (val == "1G") +#else + if (val == "128M") +#endif + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup OrderByLimit.MaxMemory + section = "OrderByLimit"; + parm = "MaxMemory"; + val = cf->getConfig(section, parm); +#ifdef _WIN64 + + if (val == "1G") +#else + if (val == "128M") +#endif + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup Installation.UMStorageType + section = "Installation"; + parm = "UMStorageType"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "internal"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup DBRootStorageType + parm = "DBRootStorageType"; + val = cf->getConfig(section, parm); + + if (val == "local") + { + val = "internal"; + cf->setConfig(section, parm, val); + cout << "Fixing " << section << "." << parm << " = " << val << endl; + } + + // Fixup WES + section = "pm1_WriteEngineServer"; + parm = "IPAddr"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "127.0.0.1"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "Port"; + val = "8630"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup TableLockSaveFile + section = "SystemConfig"; + parm = "TableLockSaveFile"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = IDBreadRegistry("") + "/dbrm/tablelocks"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup SystemModuleConfig + section = "SystemModuleConfig"; + parm = "ModuleDBRootCount1-1"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "0"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "ModuleDBRootCount1-2"; + val = "0"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "ModuleDBRootCount1-3"; + val = "1"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "ModuleDBRootID1-1-1"; + val = "0"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "ModuleDBRootID1-1-2"; + val = "0"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + parm = "ModuleDBRootID1-1-3"; + val = "1"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup NVBF + section = "VersionBuffer"; + parm = "NumVersionBufferFiles"; + val = cf->getConfig(section, parm); + + if (!val.empty()) + { + cf->delConfig(section, parm); + cout << "Deleting " << section << "." << parm << endl; + } + + // Fixup AllowDiskBasedJoin + section = "HashJoin"; + parm = "AllowDiskBasedJoin"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "N"; // We default to No + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup TempFilePath + section = "HashJoin"; + parm = "TempFilePath"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "$INSTALLDIR/tmp"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + // Fixup TempFileCompression + section = "HashJoin"; + parm = "TempFileCompression"; + val = cf->getConfig(section, parm); + + if (val.empty()) + { + val = "Y"; + cf->setConfig(section, parm, val); + cout << "Adding " << section << "." << parm << " = " << val << endl; + } + + cf->write(); + rc = 0; + } + catch (exception& e) + { + cout << "fixupCalpontXML caught exception: " << e.what() << endl; + } + catch (...) + { + cout << "fixupCalpontXML caught exception (...): " << endl; + } + + return rc; } int fixupMyIni() { - return 0; + return 0; } -} +} // namespace namespace bootstrap { - int fixupConfig(const string& installDir, const string& mysqlPort) { - int rc = -1; + int rc = -1; - string id = installDir; - string::size_type p; + string id = installDir; + string::size_type p; + p = id.find('\\'); + + while (p != string::npos) + { + id[p] = '/'; p = id.find('\\'); + } - while (p != string::npos) - { - id[p] = '/'; - p = id.find('\\'); - } + fs::path cFilePath; + fs::path tmpPath; + ifstream ifs; + ofstream ofs; + string strLine; + bool okayToRename = false; - fs::path cFilePath; - fs::path tmpPath; - ifstream ifs; - ofstream ofs; - string strLine; - bool okayToRename = false; + cFilePath = installDir; + cFilePath /= "etc"; + cFilePath /= "Columnstore.xml"; + tmpPath = cFilePath; + tmpPath.replace_extension(".tmp"); + fs::remove(tmpPath); + ifs.open(cFilePath.string().c_str()); + ofs.open(tmpPath.string().c_str()); + getline(ifs, strLine); - cFilePath = installDir; - cFilePath /= "etc"; - cFilePath /= "Columnstore.xml"; - tmpPath = cFilePath; - tmpPath.replace_extension(".tmp"); - fs::remove(tmpPath); - ifs.open(cFilePath.string().c_str()); - ofs.open(tmpPath.string().c_str()); + while (ifs.good()) + { + sedit(strLine, "##INSTDIR##", id); + ofs << strLine << endl; getline(ifs, strLine); + } - while (ifs.good()) + if (!ifs.bad() && !ofs.bad()) + okayToRename = true; + + ofs.close(); + ifs.close(); + + if (okayToRename) + { + try { - sedit(strLine, "##INSTDIR##", id); - ofs << strLine << endl; - getline(ifs, strLine); + fs::remove(cFilePath); + fs::rename(tmpPath, cFilePath); } - - if (!ifs.bad() && !ofs.bad()) - okayToRename = true; - - ofs.close(); - ifs.close(); - - if (okayToRename) + catch (exception& e) { - try - { - fs::remove(cFilePath); - fs::rename(tmpPath, cFilePath); - } - catch (exception& e) - { - cout << "Failed to remove " << cFilePath.c_str() << e.what() << endl; - } + cout << "Failed to remove " << cFilePath.c_str() << e.what() << endl; } - else - return -1; + } + else + return -1; - ifs.clear(); - ofs.clear(); + ifs.clear(); + ofs.clear(); - okayToRename = false; - cFilePath = installDir; - cFilePath /= "my.ini"; - tmpPath = cFilePath; - tmpPath.replace_extension(".tmp"); - fs::remove(tmpPath); - ifs.open(cFilePath.string().c_str()); - ofs.open(tmpPath.string().c_str()); - getline(ifs, strLine); + okayToRename = false; + cFilePath = installDir; + cFilePath /= "my.ini"; + tmpPath = cFilePath; + tmpPath.replace_extension(".tmp"); + fs::remove(tmpPath); + ifs.open(cFilePath.string().c_str()); + ofs.open(tmpPath.string().c_str()); + getline(ifs, strLine); - while (ifs.good()) + while (ifs.good()) + { + if (strLine[0] != '#') { - if (strLine[0] != '#') - { - sedit(strLine, "##INSTDIR##", id); - sedit(strLine, "##PORT##", mysqlPort); - } + sedit(strLine, "##INSTDIR##", id); + sedit(strLine, "##PORT##", mysqlPort); + } #ifndef SKIP_MYSQL_SETUP4 - sedit(strLine, "#infinidb_compression_type=0", "infinidb_compression_type=2"); - sedit(strLine, "infinidb_compression_type=1", "infinidb_compression_type=2"); + sedit(strLine, "#infinidb_compression_type=0", "infinidb_compression_type=2"); + sedit(strLine, "infinidb_compression_type=1", "infinidb_compression_type=2"); #endif - ofs << strLine << endl; - getline(ifs, strLine); - } - - if (!ifs.bad() && !ofs.bad()) - okayToRename = true; - - ofs.close(); - ifs.close(); - - if (okayToRename) - { - try - { - fs::remove(cFilePath); - fs::rename(tmpPath, cFilePath); - } - catch (boost::exception const& ex) - { - std::exception const* se = dynamic_cast(&ex); - - if (se) - { - // will enter here only for my application exception and not for pure boost exception - cerr << se->what() << "my.ini not fixedup" << endl; - } - else - { - cerr << " Boost exception while renaming my.ini: Failed to decode the exception" << endl; - } - } - catch (std::exception const& ex) - { - cerr << ex.what() << "my.ini not fixedup" << endl; - } - catch (...) - { - cerr << "An unknown exception (...) was caught while renaming my.ini" << endl; - } - } - else - return -1; - - ifs.clear(); - ofs.clear(); - - // Add install timestamp to CalpontVersion.txt - char szTime[24]; - time_t now = time(NULL); - struct tm nowtm; - localtime_s(&nowtm, &now); - strftime(szTime, 24, "%Y-%m-%d %H:%M:%S", &nowtm); - okayToRename = false; - cFilePath = installDir; - cFilePath /= "etc/CalpontVersion.txt"; - tmpPath = cFilePath; - tmpPath.replace_extension(".tmp"); - fs::remove(tmpPath); - ifs.open(cFilePath.string().c_str()); - ofs.open(tmpPath.string().c_str()); + ofs << strLine << endl; getline(ifs, strLine); + } - while (ifs.good()) + if (!ifs.bad() && !ofs.bad()) + okayToRename = true; + + ofs.close(); + ifs.close(); + + if (okayToRename) + { + try { - sedit(strLine, "@@INSTALLDATE@@", szTime); - ofs << strLine << endl; - getline(ifs, strLine); + fs::remove(cFilePath); + fs::rename(tmpPath, cFilePath); } - - if (!ifs.bad() && !ofs.bad()) - okayToRename = true; - - ofs.close(); - ifs.close(); - - if (okayToRename) + catch (boost::exception const& ex) { - fs::remove(cFilePath); - fs::rename(tmpPath, cFilePath); + std::exception const* se = dynamic_cast(&ex); + + if (se) + { + // will enter here only for my application exception and not for pure boost exception + cerr << se->what() << "my.ini not fixedup" << endl; + } + else + { + cerr << " Boost exception while renaming my.ini: Failed to decode the exception" << endl; + } } - else - return -1; + catch (std::exception const& ex) + { + cerr << ex.what() << "my.ini not fixedup" << endl; + } + catch (...) + { + cerr << "An unknown exception (...) was caught while renaming my.ini" << endl; + } + } + else + return -1; - ifs.clear(); - ofs.clear(); + ifs.clear(); + ofs.clear(); - //Now do any final fixups... - rc = fixupMyIni(); + // Add install timestamp to CalpontVersion.txt + char szTime[24]; + time_t now = time(NULL); + struct tm nowtm; + localtime_s(&nowtm, &now); + strftime(szTime, 24, "%Y-%m-%d %H:%M:%S", &nowtm); + okayToRename = false; + cFilePath = installDir; + cFilePath /= "etc/CalpontVersion.txt"; + tmpPath = cFilePath; + tmpPath.replace_extension(".tmp"); + fs::remove(tmpPath); + ifs.open(cFilePath.string().c_str()); + ofs.open(tmpPath.string().c_str()); + getline(ifs, strLine); - if (rc == 0) - rc = fixupCalpontXML(); + while (ifs.good()) + { + sedit(strLine, "@@INSTALLDATE@@", szTime); + ofs << strLine << endl; + getline(ifs, strLine); + } - return rc; + if (!ifs.bad() && !ofs.bad()) + okayToRename = true; + + ofs.close(); + ifs.close(); + + if (okayToRename) + { + fs::remove(cFilePath); + fs::rename(tmpPath, cFilePath); + } + else + return -1; + + ifs.clear(); + ofs.clear(); + + // Now do any final fixups... + rc = fixupMyIni(); + + if (rc == 0) + rc = fixupCalpontXML(); + + return rc; } -} +} // namespace bootstrap diff --git a/utils/winport/glob.h b/utils/winport/glob.h index 1dc3b5253..6957b7b81 100644 --- a/utils/winport/glob.h +++ b/utils/winport/glob.h @@ -20,42 +20,41 @@ #include -#define GLOB_ERR 0x0001 -#define GLOB_MARK 0x0002 -#define GLOB_NOSORT 0x0004 -#define GLOB_DOOFFS 0x0008 -#define GLOB_NOCHECK 0x0010 -#define GLOB_APPEND 0x0020 -#define GLOB_NOESCAPE 0x0040 -#define GLOB_PERIOD 0x0080 -#define GLOB_ALTDIRFUNC 0x0100 -#define GLOB_BRACE 0x0200 -#define GLOB_NOMAGIC 0x0400 -#define GLOB_TILDE 0x0800 -#define GLOB_ONLYDIR 0x1000 -#define GLOB_MAGCHAR 0x2000 +#define GLOB_ERR 0x0001 +#define GLOB_MARK 0x0002 +#define GLOB_NOSORT 0x0004 +#define GLOB_DOOFFS 0x0008 +#define GLOB_NOCHECK 0x0010 +#define GLOB_APPEND 0x0020 +#define GLOB_NOESCAPE 0x0040 +#define GLOB_PERIOD 0x0080 +#define GLOB_ALTDIRFUNC 0x0100 +#define GLOB_BRACE 0x0200 +#define GLOB_NOMAGIC 0x0400 +#define GLOB_TILDE 0x0800 +#define GLOB_ONLYDIR 0x1000 +#define GLOB_MAGCHAR 0x2000 -#define GLOB_NOSPACE 0x01 -#define GLOB_ABORTED 0x02 -#define GLOB_NOMATCH 0x03 +#define GLOB_NOSPACE 0x01 +#define GLOB_ABORTED 0x02 +#define GLOB_NOMATCH 0x03 typedef struct { - size_t gl_pathc; /* Count of paths matched so far */ - char** gl_pathv; /* List of matched pathnames. */ - size_t gl_offs; /* Slots to reserve in gl_pathv. */ - size_t gl_alloccnt; + size_t gl_pathc; /* Count of paths matched so far */ + char** gl_pathv; /* List of matched pathnames. */ + size_t gl_offs; /* Slots to reserve in gl_pathv. */ + size_t gl_alloccnt; } glob_t; #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif -extern int glob(const char* pattern, int flags, - int errfunc(const char* epath, int eerrno), - glob_t* pglob); + extern int glob(const char* pattern, int flags, int errfunc(const char* epath, int eerrno), glob_t* pglob); -extern void globfree(glob_t* pglob); + extern void globfree(glob_t* pglob); #ifdef __cplusplus } diff --git a/utils/winport/grepit.cpp b/utils/winport/grepit.cpp index 5b285ba88..f20182efa 100644 --- a/utils/winport/grepit.cpp +++ b/utils/winport/grepit.cpp @@ -32,18 +32,18 @@ namespace winport { bool grepit(istream& is, const string& pattern) { - boost::regex pat(pattern); - string cInput; + boost::regex pat(pattern); + string cInput; + getline(is, cInput); + + while (is.good()) + { + if (boost::regex_match(cInput, pat)) + return true; + getline(is, cInput); + } - while (is.good()) - { - if (boost::regex_match(cInput, pat)) - return true; - - getline(is, cInput); - } - - return false; -} + return false; } +} // namespace winport diff --git a/utils/winport/idb_getopt.cpp b/utils/winport/idb_getopt.cpp index 1d0e9aebb..a73a8def9 100644 --- a/utils/winport/idb_getopt.cpp +++ b/utils/winport/idb_getopt.cpp @@ -25,44 +25,46 @@ using namespace std; struct GetOptData { - bool initialized; - set opSet; - set opWithArgSet; - char* nextchar; - GetOptData() : initialized(false), nextchar(0) {} + bool initialized; + set opSet; + set opWithArgSet; + char* nextchar; + GetOptData() : initialized(false), nextchar(0) + { + } }; /* Same as GNU interface. For communication from `getopt' to the caller. - When `getopt' finds an option that takes an argument, - the argument value is returned here. - Also, when `ordering' is RETURN_IN_ORDER, - each non-option ARGV-element is returned here. */ + When `getopt' finds an option that takes an argument, + the argument value is returned here. + Also, when `ordering' is RETURN_IN_ORDER, + each non-option ARGV-element is returned here. */ char* optarg; /* Same as GNU interface. Index in ARGV of the next element to be scanned. - This is used for communication to and from the caller - and for communication between successive calls to `getopt'. + This is used for communication to and from the caller + and for communication between successive calls to `getopt'. - When `getopt' returns -1, this is the index of the first of the - non-option elements that the caller should itself scan. + When `getopt' returns -1, this is the index of the first of the + non-option elements that the caller should itself scan. - Otherwise, `optind' communicates from one call to the next - how much of ARGV has been scanned so far.*/ + Otherwise, `optind' communicates from one call to the next + how much of ARGV has been scanned so far.*/ /* 1003.2 says this must be 1 before any call.*/ -int optind = 1; +int optind = 1; /* Same as GNU interface. Callers store zero here to inhibit the error message - for unrecognized options. */ + for unrecognized options. */ -int opterr = 1; +int opterr = 1; /* Same as GNU interface. Set to an option character which was unrecognized. - This must be initialized on some systems to avoid linking in the - system's own getopt implementation. */ + This must be initialized on some systems to avoid linking in the + system's own getopt implementation. */ -int optopt = '?'; +int optopt = '?'; namespace { @@ -73,134 +75,134 @@ GetOptData getopt_data; // will be ignored void initialize_opt(const char* optstring) { - for (unsigned i = 0; i < strlen(optstring); i++) + for (unsigned i = 0; i < strlen(optstring); i++) + { + if (isalnum(optstring[i])) { - if (isalnum(optstring[i])) - { - if ((i < strlen(optstring) - 1) && (optstring[i + 1] == ':')) - { - getopt_data.opWithArgSet.insert(optstring[i]); - i++; - } - else - { - getopt_data.opSet.insert(optstring[i]); - } - } + if ((i < strlen(optstring) - 1) && (optstring[i + 1] == ':')) + { + getopt_data.opWithArgSet.insert(optstring[i]); + i++; + } + else + { + getopt_data.opSet.insert(optstring[i]); + } } + } - getopt_data.initialized = true; + getopt_data.initialized = true; } -} //namespace +} // namespace int getopt(int argc, char* const* argv, const char* optstring) { - if (argc < 1) - return -1; + if (argc < 1) + return -1; - if (!getopt_data.initialized) - initialize_opt(optstring); + if (!getopt_data.initialized) + initialize_opt(optstring); - int retchar = 0; - char* arg = 0; - bool newArg = false; - int curind = optind; + int retchar = 0; + char* arg = 0; + bool newArg = false; + int curind = optind; - if (optind < argc) + if (optind < argc) + { + if (getopt_data.nextchar == 0) { - if (getopt_data.nextchar == 0) - { - arg = argv[optind]; - newArg = true; - } - else - { - arg = getopt_data.nextchar; - } + arg = argv[optind]; + newArg = true; + } + else + { + arg = getopt_data.nextchar; + } - // all valid op should be started with '-', with or without arg following. - // stop process when hitting the first non '-' arg that does not belong to a - // previous option. - if (newArg && (strlen(arg) <= 1 || *arg != '-')) - return -1; + // all valid op should be started with '-', with or without arg following. + // stop process when hitting the first non '-' arg that does not belong to a + // previous option. + if (newArg && (strlen(arg) <= 1 || *arg != '-')) + return -1; - if (newArg && (strcmp(arg, "--") == 0)) + if (newArg && (strcmp(arg, "--") == 0)) + { + optind++; + return -1; + } + + if (newArg && *arg == '-') + arg++; + + if (arg != 0) + { + // option without arg + if (getopt_data.opSet.find(*arg) != getopt_data.opSet.end()) + { + retchar = *arg; + + if (*(arg + 1) == 0) // "-a" { + optind++; + getopt_data.nextchar = 0; + } + else // "-abc" + { + getopt_data.nextchar = arg + 1; + } + } + // option with arg + else if (getopt_data.opWithArgSet.find(*arg) != getopt_data.opWithArgSet.end()) + { + getopt_data.nextchar = 0; + + if (*(arg + 1) == 0) // "-c foo" next arg is arg of c + { + if (optind < argc - 1) + { + retchar = *arg; optind++; - return -1; + optarg = argv[optind++]; + } + else // error. no arg provided for argop. this option is invalid + { + optopt = *arg; + optind++; + retchar = '?'; + } } - - if (newArg && *arg == '-') - arg++; - - if (arg != 0) + else // "-cfoo" arg is in the string right after option letter { - // option without arg - if (getopt_data.opSet.find(*arg) != getopt_data.opSet.end()) - { - retchar = *arg; - - if (*(arg + 1) == 0) // "-a" - { - optind++; - getopt_data.nextchar = 0; - } - else // "-abc" - { - getopt_data.nextchar = arg + 1; - } - } - // option with arg - else if (getopt_data.opWithArgSet.find(*arg) != getopt_data.opWithArgSet.end()) - { - getopt_data.nextchar = 0; - - if (*(arg + 1) == 0) // "-c foo" next arg is arg of c - { - if (optind < argc - 1) - { - retchar = *arg; - optind++; - optarg = argv[optind++]; - } - else // error. no arg provided for argop. this option is invalid - { - optopt = *arg; - optind++; - retchar = '?'; - } - } - else // "-cfoo" arg is in the string right after option letter - { - retchar = *arg; - optarg = arg + 1; - optind++; - } - } - else // invalid option - { - optopt = *arg; - retchar = '?'; - - if (*(arg + 1) == 0) // "-x" - { - optind++; - getopt_data.nextchar = 0; - } - else // "-xbc" - { - getopt_data.nextchar = arg + 1; - } - } + retchar = *arg; + optarg = arg + 1; + optind++; } - } + } + else // invalid option + { + optopt = *arg; + retchar = '?'; - if (optind == argc && retchar == 0) - { - optind = curind; - return -1; + if (*(arg + 1) == 0) // "-x" + { + optind++; + getopt_data.nextchar = 0; + } + else // "-xbc" + { + getopt_data.nextchar = arg + 1; + } + } } + } - return retchar; + if (optind == argc && retchar == 0) + { + optind = curind; + return -1; + } + + return retchar; } diff --git a/utils/winport/idbregistry.cpp b/utils/winport/idbregistry.cpp index 30290b8e7..feca6c820 100644 --- a/utils/winport/idbregistry.cpp +++ b/utils/winport/idbregistry.cpp @@ -25,30 +25,30 @@ using namespace std; const string IDBreadRegistry(const string& name, bool returnShortName) { - HKEY hkResult; - LONG lResult; - lResult = RegOpenKeyEx(HKEY_LOCAL_MACHINE, "Software\\Calpont\\InfiniDB", 0, KEY_READ, &hkResult); + HKEY hkResult; + LONG lResult; + lResult = RegOpenKeyEx(HKEY_LOCAL_MACHINE, "Software\\Calpont\\InfiniDB", 0, KEY_READ, &hkResult); - if (lResult != ERROR_SUCCESS) - return string(); + if (lResult != ERROR_SUCCESS) + return string(); - DWORD cbData = 1024; - TCHAR CfnameBuf[1024]; - lResult = RegQueryValueEx(hkResult, name.c_str(), 0, 0, (LPBYTE)CfnameBuf, &cbData); - CloseHandle(hkResult); + DWORD cbData = 1024; + TCHAR CfnameBuf[1024]; + lResult = RegQueryValueEx(hkResult, name.c_str(), 0, 0, (LPBYTE)CfnameBuf, &cbData); + CloseHandle(hkResult); - if (lResult != ERROR_SUCCESS) - return string(); + if (lResult != ERROR_SUCCESS) + return string(); - if (!returnShortName) - return string(CfnameBuf); + if (!returnShortName) + return string(CfnameBuf); - cbData = 1024; - TCHAR snbuffer[1024]; - lResult = GetShortPathName(CfnameBuf, snbuffer, cbData); + cbData = 1024; + TCHAR snbuffer[1024]; + lResult = GetShortPathName(CfnameBuf, snbuffer, cbData); - if (lResult == 0 || lResult > 1024) - return string(); + if (lResult == 0 || lResult > 1024) + return string(); - return string(snbuffer); + return string(snbuffer); } diff --git a/utils/winport/main.cpp b/utils/winport/main.cpp index e72ec0c2c..79b5ae746 100644 --- a/utils/winport/main.cpp +++ b/utils/winport/main.cpp @@ -16,90 +16,81 @@ using namespace std; extern int optind, opterr, optopt; */ - int main(int argc, char** argv) { - opterr = 0; - int c; - bool invalid = false; - bool aflg = false; - bool bflg = false; - bool cflg = false; - bool dflg = false; - string dval; - bool eflg = false; - string eval; - char invchar = 0; + opterr = 0; + int c; + bool invalid = false; + bool aflg = false; + bool bflg = false; + bool cflg = false; + bool dflg = false; + string dval; + bool eflg = false; + string eval; + char invchar = 0; #if defined(TEST1) - char* opts = ""; - char* av[] = { 0 }; + char* opts = ""; + char* av[] = {0}; #elif defined(TEST2) - char* opts = "-a+bcd:e:"; + char* opts = "-a+bcd:e:"; // char* av[] = { "prg", "-ab", "-dfoo", "-xe", "bar", "file1", "file2", 0 }; // char* av[] = {"prg", "file1", "file2"}; #else #error Need a test to run! #endif -// int ac = sizeof(av) / sizeof(av[0]) - 1; + // int ac = sizeof(av) / sizeof(av[0]) - 1; - while ((c = getopt(argc, argv, opts)) != -1) - switch (c) - { - case 'a': - aflg = true; - break; + while ((c = getopt(argc, argv, opts)) != -1) + switch (c) + { + case 'a': aflg = true; break; - case 'b': - bflg = true; - break; + case 'b': bflg = true; break; - case 'c': - cflg = true; - break; + case 'c': cflg = true; break; - case 'd': - dflg = true; - dval = optarg; - break; + case 'd': + dflg = true; + dval = optarg; + break; - case 'e': - eflg = true; - eval = optarg; - break; + case 'e': + eflg = true; + eval = optarg; + break; - case '?': - invalid = true; - invchar = optopt; - break; + case '?': + invalid = true; + invchar = optopt; + break; - default: - invalid = true; - invchar = 0; - break; - } + default: + invalid = true; + invchar = 0; + break; + } #if defined(TEST1) - assert(!invalid && !aflg && !bflg && !cflg); - assert(optind == 1); + assert(!invalid && !aflg && !bflg && !cflg); + assert(optind == 1); #elif defined(TEST2) - /* cout << "optind=" << optind << endl; - assert(invalid && aflg && bflg && !cflg); - assert((ac - optind) == 2); - assert(string(av[optind+0]) == "file1"); - assert(string(av[optind+1]) == "file2"); - assert(dflg); - assert(dval == "foo"); - assert(eflg); - assert(eval == "bar"); - assert(invchar == 'x');*/ - cout << "aflag=" << aflg << " bflag=" << bflg << " cflg=" << cflg - << " dflag=" << dflg << " eflag=" << eflg << endl; - cout << " eval=" << eval << " dval=" << dval << endl; - cout << " invchar=" << invchar << endl; - cout << " final optind=" << optind << endl; + /* cout << "optind=" << optind << endl; + assert(invalid && aflg && bflg && !cflg); + assert((ac - optind) == 2); + assert(string(av[optind+0]) == "file1"); + assert(string(av[optind+1]) == "file2"); + assert(dflg); + assert(dval == "foo"); + assert(eflg); + assert(eval == "bar"); + assert(invchar == 'x');*/ + cout << "aflag=" << aflg << " bflag=" << bflg << " cflg=" << cflg << " dflag=" << dflg << " eflag=" << eflg + << endl; + cout << " eval=" << eval << " dval=" << dval << endl; + cout << " invchar=" << invchar << endl; + cout << " final optind=" << optind << endl; #endif - return 0; + return 0; } - - diff --git a/utils/winport/poll.h b/utils/winport/poll.h index 58b1993c9..a43196844 100644 --- a/utils/winport/poll.h +++ b/utils/winport/poll.h @@ -3,84 +3,84 @@ NAME - poll - select(2)-based poll() emulation function for BSD systems. + poll - select(2)-based poll() emulation function for BSD systems. SYNOPSIS - #include "poll.h" + #include "poll.h" - struct pollfd - { - int fd; - short events; - short revents; - } + struct pollfd + { + int fd; + short events; + short revents; + } - int poll (struct pollfd *pArray, unsigned long n_fds, int timeout) + int poll (struct pollfd *pArray, unsigned long n_fds, int timeout) DESCRIPTION - This file, and the accompanying "poll.c", implement the System V - poll(2) system call for BSD systems (which typically do not provide - poll()). Poll() provides a method for multiplexing input and output - on multiple open file descriptors; in traditional BSD systems, that - capability is provided by select(). While the semantics of select() - differ from those of poll(), poll() can be readily emulated in terms - of select() -- which is how this function is implemented. + This file, and the accompanying "poll.c", implement the System V + poll(2) system call for BSD systems (which typically do not provide + poll()). Poll() provides a method for multiplexing input and output + on multiple open file descriptors; in traditional BSD systems, that + capability is provided by select(). While the semantics of select() + differ from those of poll(), poll() can be readily emulated in terms + of select() -- which is how this function is implemented. REFERENCES - Stevens, W. Richard. Unix Network Programming. Prentice-Hall, 1990. + Stevens, W. Richard. Unix Network Programming. Prentice-Hall, 1990. NOTES - 1. This software requires an ANSI C compiler. + 1. This software requires an ANSI C compiler. LICENSE - This software is released under the following license: + This software is released under the following license: - Copyright (c) 1995-2002 Brian M. Clapper - All rights reserved. + Copyright (c) 1995-2002 Brian M. Clapper + All rights reserved. - Redistribution and use in source and binary forms are - permitted provided that: (1) source distributions retain - this entire copyright notice and comment; (2) modifications - made to the software are prominently mentioned, and a copy - of the original software (or a pointer to its location) are - included; and (3) distributions including binaries display - the following acknowledgement: "This product includes - software developed by Brian M. Clapper " - in the documentation or other materials provided with the - distribution. The name of the author may not be used to - endorse or promote products derived from this software - without specific prior written permission. + Redistribution and use in source and binary forms are + permitted provided that: (1) source distributions retain + this entire copyright notice and comment; (2) modifications + made to the software are prominently mentioned, and a copy + of the original software (or a pointer to its location) are + included; and (3) distributions including binaries display + the following acknowledgement: "This product includes + software developed by Brian M. Clapper " + in the documentation or other materials provided with the + distribution. The name of the author may not be used to + endorse or promote products derived from this software + without specific prior written permission. - THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS - OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - PARTICULAR PURPOSE. + THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE. - Effectively, this means you can do what you want with the software - except remove this notice or take advantage of the author's name. - If you modify the software and redistribute your modified version, - you must indicate that your version is a modification of the - original, and you must provide either a pointer to or a copy of the - original. + Effectively, this means you can do what you want with the software + except remove this notice or take advantage of the author's name. + If you modify the software and redistribute your modified version, + you must indicate that your version is a modification of the + original, and you must provide either a pointer to or a copy of the + original. \*---------------------------------------------------------------------------*/ #ifndef _POLL_EMUL_H_ #define _POLL_EMUL_H_ -#define POLLIN 0x01 -#define POLLPRI 0x02 -#define POLLOUT 0x04 -#define POLLERR 0x08 -#define POLLHUP 0x10 -#define POLLNVAL 0x20 +#define POLLIN 0x01 +#define POLLPRI 0x02 +#define POLLOUT 0x04 +#define POLLERR 0x08 +#define POLLHUP 0x10 +#define POLLNVAL 0x20 struct pollfd { - int fd; - short events; - short revents; + int fd; + short events; + short revents; }; typedef unsigned long nfds_t; @@ -91,7 +91,7 @@ extern "C" #endif #if (__STDC__ > 0) || defined(__cplusplus) -extern int poll (struct pollfd* pArray, nfds_t n_fds, int timeout); + extern int poll(struct pollfd* pArray, nfds_t n_fds, int timeout); #else extern int poll(); #endif diff --git a/utils/winport/resource.h b/utils/winport/resource.h index 86fde5402..427478ac0 100644 --- a/utils/winport/resource.h +++ b/utils/winport/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/utils/winport/sedit.cpp b/utils/winport/sedit.cpp index 72ef0f967..2f5411bf4 100644 --- a/utils/winport/sedit.cpp +++ b/utils/winport/sedit.cpp @@ -22,28 +22,29 @@ using namespace std; namespace winport { - bool sedit(string& line, const string& find, const string& replace) { - string::size_type idx; - idx = line.find(find); + string::size_type idx; + idx = line.find(find); - if (idx == string::npos) return false; + if (idx == string::npos) + return false; - line.replace(idx, find.length(), replace); - return true; + line.replace(idx, find.length(), replace); + return true; } const string sedit2(const string& line, const string& find, const string& replace) { - string::size_type idx; - idx = line.find(find); + string::size_type idx; + idx = line.find(find); - if (idx == string::npos) return line; + if (idx == string::npos) + return line; - string nline(line); - nline.replace(idx, find.length(), replace); - return nline; + string nline(line); + nline.replace(idx, find.length(), replace); + return nline; } -} +} // namespace winport diff --git a/utils/winport/sedit.h b/utils/winport/sedit.h index dce78df92..9d79f5d12 100644 --- a/utils/winport/sedit.h +++ b/utils/winport/sedit.h @@ -24,6 +24,6 @@ namespace winport { extern bool sedit(std::string& line, const std::string& find, const std::string& replace); extern const string sedit2(const std::string& line, const std::string& find, const std::string& replace); -} +} // namespace winport #endif diff --git a/utils/winport/sys/time.h b/utils/winport/sys/time.h index b7715b1bb..74ce66e5e 100644 --- a/utils/winport/sys/time.h +++ b/utils/winport/sys/time.h @@ -18,4 +18,3 @@ #ifndef _SYS_TIME_H #define _SYS_TIME_H 1 #endif - diff --git a/utils/winport/sysinfo.cpp b/utils/winport/sysinfo.cpp index d3b8b563c..05a6ec603 100644 --- a/utils/winport/sysinfo.cpp +++ b/utils/winport/sysinfo.cpp @@ -28,81 +28,67 @@ // we'll just have to make our own >:( inline int GetProcessList() { - HANDLE hProcessSnap; - PROCESSENTRY32 pe32; - long Processes = 0; - - - // Snapshot the current processes and make sure it's valid - hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); - - - if (hProcessSnap == INVALID_HANDLE_VALUE) - return Processes; - - - // Set the size of the structure before using it. - pe32.dwSize = sizeof(PROCESSENTRY32); - - - // Retrieve information about the first process, - // and exit if unsuccessful - if (!Process32First(hProcessSnap, &pe32)) - { - CloseHandle(hProcessSnap); // clean the snapshot object - return Processes; - } - - - // Count the total number of processes - do - { - Processes++; - } - while (Process32Next(hProcessSnap, &pe32)); - - - CloseHandle(hProcessSnap); + HANDLE hProcessSnap; + PROCESSENTRY32 pe32; + long Processes = 0; + // Snapshot the current processes and make sure it's valid + hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (hProcessSnap == INVALID_HANDLE_VALUE) return Processes; + + // Set the size of the structure before using it. + pe32.dwSize = sizeof(PROCESSENTRY32); + + // Retrieve information about the first process, + // and exit if unsuccessful + if (!Process32First(hProcessSnap, &pe32)) + { + CloseHandle(hProcessSnap); // clean the snapshot object + return Processes; + } + + // Count the total number of processes + do + { + Processes++; + } while (Process32Next(hProcessSnap, &pe32)); + + CloseHandle(hProcessSnap); + + return Processes; } int sysinfo(struct sysinfo* info) { - SYSTEM_INFO si; - MEMORYSTATUSEX statex; + SYSTEM_INFO si; + MEMORYSTATUSEX statex; + ZeroMemory(&si, sizeof(SYSTEM_INFO)); + statex.dwLength = sizeof(statex); + GetSystemInfo(&si); - ZeroMemory(&si, sizeof(SYSTEM_INFO)); - statex.dwLength = sizeof(statex); - GetSystemInfo(&si); + if (!GlobalMemoryStatusEx(&statex)) + return -1; + // System Uptime + info->uptime = GetTickCount64() / 1000 % 60; - if (!GlobalMemoryStatusEx(&statex)) - return -1; + // Load times - windows does not have this so say -1 or 0 or nothing basically + info->loads[0] = -1; + info->loads[1] = -1; + info->loads[2] = -1; + // Ram usages - note that these may not be exact to what linux has + info->freeram = statex.ullAvailPhys; + info->freeswap = statex.ullAvailVirtual; + info->sharedram = 0; + info->totalram = statex.ullTotalPhys; + info->bufferram = statex.ullTotalPageFile; + info->totalswap = statex.ullTotalVirtual; - // System Uptime - info->uptime = GetTickCount64() / 1000 % 60; - - - // Load times - windows does not have this so say -1 or 0 or nothing basically - info->loads[0] = -1; - info->loads[1] = -1; - info->loads[2] = -1; - - - // Ram usages - note that these may not be exact to what linux has - info->freeram = statex.ullAvailPhys; - info->freeswap = statex.ullAvailVirtual; - info->sharedram = 0; - info->totalram = statex.ullTotalPhys; - info->bufferram = statex.ullTotalPageFile; - info->totalswap = statex.ullTotalVirtual; - - - // Processes - info->procs = GetProcessList(); - return 0; + // Processes + info->procs = GetProcessList(); + return 0; } diff --git a/utils/winport/sysinfo.h b/utils/winport/sysinfo.h index 54a2d79fc..299510246 100644 --- a/utils/winport/sysinfo.h +++ b/utils/winport/sysinfo.h @@ -21,20 +21,20 @@ // System info stuff struct sysinfo { - long uptime; /* Seconds since boot */ - unsigned long loads[3]; /* 1, 5, and 15 minute load averages */ - unsigned long totalram; /* Total usable main memory size */ - unsigned long freeram; /* Available memory size */ - unsigned long sharedram; /* Amount of shared memory */ - unsigned long bufferram; /* Memory used by buffers */ - unsigned long totalswap; /* Total swap space size */ - unsigned long freeswap; /* swap space still available */ - unsigned short procs; /* Number of current processes */ - unsigned long totalhigh; /* Total high memory size */ - unsigned long freehigh; /* Available high memory size */ - unsigned int mem_unit; /* Memory unit size in bytes */ - char _f[20 - 2 * sizeof(long) - sizeof(int)]; /* Padding to 64 bytes */ + long uptime; /* Seconds since boot */ + unsigned long loads[3]; /* 1, 5, and 15 minute load averages */ + unsigned long totalram; /* Total usable main memory size */ + unsigned long freeram; /* Available memory size */ + unsigned long sharedram; /* Amount of shared memory */ + unsigned long bufferram; /* Memory used by buffers */ + unsigned long totalswap; /* Total swap space size */ + unsigned long freeswap; /* swap space still available */ + unsigned short procs; /* Number of current processes */ + unsigned long totalhigh; /* Total high memory size */ + unsigned long freehigh; /* Available high memory size */ + unsigned int mem_unit; /* Memory unit size in bytes */ + char _f[20 - 2 * sizeof(long) - sizeof(int)]; /* Padding to 64 bytes */ }; extern int sysinfo(sysinfo*); -#endif // IDB_SYSINFO_H_ +#endif // IDB_SYSINFO_H_ diff --git a/utils/winport/syslog.h b/utils/winport/syslog.h index bc13c2a3f..6753aa646 100644 --- a/utils/winport/syslog.h +++ b/utils/winport/syslog.h @@ -20,43 +20,42 @@ /* From sys/syslog.h */ -#define LOG_EMERG 0 /* system is unusable */ -#define LOG_ALERT 1 /* action must be taken immediately */ -#define LOG_CRIT 2 /* critical conditions */ -#define LOG_ERR 3 /* error conditions */ -#define LOG_WARNING 4 /* warning conditions */ -#define LOG_NOTICE 5 /* normal but significant condition */ -#define LOG_INFO 6 /* informational */ -#define LOG_DEBUG 7 /* debug-level messages */ +#define LOG_EMERG 0 /* system is unusable */ +#define LOG_ALERT 1 /* action must be taken immediately */ +#define LOG_CRIT 2 /* critical conditions */ +#define LOG_ERR 3 /* error conditions */ +#define LOG_WARNING 4 /* warning conditions */ +#define LOG_NOTICE 5 /* normal but significant condition */ +#define LOG_INFO 6 /* informational */ +#define LOG_DEBUG 7 /* debug-level messages */ -#define LOG_PID 0x01 /* log the pid with each message */ -#define LOG_CONS 0x02 /* log on the console if errors in sending */ -#define LOG_ODELAY 0x04 /* delay open until first syslog() (default) */ -#define LOG_NDELAY 0x08 /* don't delay open */ -#define LOG_NOWAIT 0x10 /* don't wait for console forks: DEPRECATED */ -#define LOG_PERROR 0x20 /* log to stderr as well */ +#define LOG_PID 0x01 /* log the pid with each message */ +#define LOG_CONS 0x02 /* log on the console if errors in sending */ +#define LOG_ODELAY 0x04 /* delay open until first syslog() (default) */ +#define LOG_NDELAY 0x08 /* don't delay open */ +#define LOG_NOWAIT 0x10 /* don't wait for console forks: DEPRECATED */ +#define LOG_PERROR 0x20 /* log to stderr as well */ -#define LOG_KERN (0<<3) /* kernel messages */ -#define LOG_USER (1<<3) /* random user-level messages */ -#define LOG_MAIL (2<<3) /* mail system */ -#define LOG_DAEMON (3<<3) /* system daemons */ -#define LOG_AUTH (4<<3) /* security/authorization messages */ -#define LOG_SYSLOG (5<<3) /* messages generated internally by syslogd */ -#define LOG_LPR (6<<3) /* line printer subsystem */ -#define LOG_NEWS (7<<3) /* network news subsystem */ -#define LOG_UUCP (8<<3) /* UUCP subsystem */ -#define LOG_CRON (9<<3) /* clock daemon */ -#define LOG_AUTHPRIV (10<<3) /* security/authorization messages (private) */ -#define LOG_FTP (11<<3) /* ftp daemon */ +#define LOG_KERN (0 << 3) /* kernel messages */ +#define LOG_USER (1 << 3) /* random user-level messages */ +#define LOG_MAIL (2 << 3) /* mail system */ +#define LOG_DAEMON (3 << 3) /* system daemons */ +#define LOG_AUTH (4 << 3) /* security/authorization messages */ +#define LOG_SYSLOG (5 << 3) /* messages generated internally by syslogd */ +#define LOG_LPR (6 << 3) /* line printer subsystem */ +#define LOG_NEWS (7 << 3) /* network news subsystem */ +#define LOG_UUCP (8 << 3) /* UUCP subsystem */ +#define LOG_CRON (9 << 3) /* clock daemon */ +#define LOG_AUTHPRIV (10 << 3) /* security/authorization messages (private) */ +#define LOG_FTP (11 << 3) /* ftp daemon */ -#define LOG_LOCAL0 (16<<3) /* reserved for local use */ -#define LOG_LOCAL1 (17<<3) /* reserved for local use */ -#define LOG_LOCAL2 (18<<3) /* reserved for local use */ -#define LOG_LOCAL3 (19<<3) /* reserved for local use */ -#define LOG_LOCAL4 (20<<3) /* reserved for local use */ -#define LOG_LOCAL5 (21<<3) /* reserved for local use */ -#define LOG_LOCAL6 (22<<3) /* reserved for local use */ -#define LOG_LOCAL7 (23<<3) /* reserved for local use */ +#define LOG_LOCAL0 (16 << 3) /* reserved for local use */ +#define LOG_LOCAL1 (17 << 3) /* reserved for local use */ +#define LOG_LOCAL2 (18 << 3) /* reserved for local use */ +#define LOG_LOCAL3 (19 << 3) /* reserved for local use */ +#define LOG_LOCAL4 (20 << 3) /* reserved for local use */ +#define LOG_LOCAL5 (21 << 3) /* reserved for local use */ +#define LOG_LOCAL6 (22 << 3) /* reserved for local use */ +#define LOG_LOCAL7 (23 << 3) /* reserved for local use */ #endif - diff --git a/utils/winport/unistd.h b/utils/winport/unistd.h index 6d363e23f..830035913 100644 --- a/utils/winport/unistd.h +++ b/utils/winport/unistd.h @@ -30,17 +30,18 @@ #include "inttypes.h" #ifdef __cplusplus #include -extern "C" { +extern "C" +{ #endif #if _MSC_VER < 1800 -extern unsigned long long strtoull(const char*, char**, int); -extern long long atoll(const char*); + extern unsigned long long strtoull(const char*, char**, int); + extern long long atoll(const char*); #if _MSC_VER < 1600 -extern lldiv_t lldiv(const long long, const long long); + extern lldiv_t lldiv(const long long, const long long); #endif #endif -extern unsigned int sleep(unsigned int); + extern unsigned int sleep(unsigned int); #define strerror_r(e, b, l) strerror_s((b), (l), (e)) @@ -62,66 +63,66 @@ extern unsigned int sleep(unsigned int); #define LOCK_UN 1 #define LOCK_EX 2 #endif -struct flock -{ + struct flock + { int l_type; int l_whence; int l_start; int l_len; int l_pid; -}; -extern int flock(int, int); -extern int fcntl(int, int, ...); + }; + extern int flock(int, int); + extern int fcntl(int, int, ...); #ifndef _my_pthread_h -struct timespec -{ + struct timespec + { long tv_sec; long tv_nsec; -}; + }; #endif -int poll(struct pollfd*, unsigned long, int); + int poll(struct pollfd*, unsigned long, int); #ifndef SHUT_RDWR #define SHUT_RDWR SD_BOTH #endif -extern int inet_aton(const char*, struct in_addr*); -struct timezone -{ - int tz_minuteswest; /* minutes W of Greenwich */ - int tz_dsttime; /* type of dst correction */ -}; -extern int gettimeofday(struct timeval*, struct timezone*); + extern int inet_aton(const char*, struct in_addr*); + struct timezone + { + int tz_minuteswest; /* minutes W of Greenwich */ + int tz_dsttime; /* type of dst correction */ + }; + extern int gettimeofday(struct timeval*, struct timezone*); #define ctime_r(tp, b) ctime_s((b), sizeof(b), (tp)) -//These are also in MySQL, so we need to fudge them... +// These are also in MySQL, so we need to fudge them... #ifndef _my_pthread_h #define localtime_r idb_localtime_r -extern struct tm* idb_localtime_r(const time_t*, struct tm*); + extern struct tm* idb_localtime_r(const time_t*, struct tm*); #define strtoll _strtoi64 #define strtoull _strtoui64 #define crc32 idb_crc32 -extern unsigned int idb_crc32(const unsigned int, const unsigned char*, const size_t); + extern unsigned int idb_crc32(const unsigned int, const unsigned char*, const size_t); #endif #define CLOCK_REALTIME 1 #define CLOCK_MONOTONIC 2 -extern long clock_gettime(clockid_t, struct timespec*); + extern long clock_gettime(clockid_t, struct timespec*); -extern int syslog(int, const char*, ...); + extern int syslog(int, const char*, ...); #ifdef __cplusplus -extern int closelog(...); -extern int openlog(...); + extern int closelog(...); + extern int openlog(...); #else extern int closelog(); extern int openlog(); #endif -extern int usleep(unsigned int); -extern int fork(); + extern int usleep(unsigned int); + extern int fork(); -extern int getpagesize(); + extern int getpagesize(); -extern int pipe(int[2]); -extern pid_t getppid(); -extern pid_t waitpid(pid_t, int*, int); + extern int pipe(int[2]); + extern pid_t getppid(); + extern pid_t waitpid(pid_t, int*, int); #define WIFEXITED(x) 0 #define WEXITSTATUS(x) 0 @@ -132,8 +133,8 @@ extern pid_t waitpid(pid_t, int*, int); #define WUNTRACED 0x02 #define WCONTINUED 0x04 -extern int kill(pid_t, int); -extern int setuid(uid_t); + extern int kill(pid_t, int); + extern int setuid(uid_t); #define snprintf _snprintf @@ -151,4 +152,3 @@ extern std::string IDBSysErrorStr(DWORD err); #endif #endif - diff --git a/utils/winport/winfinidb.cpp b/utils/winport/winfinidb.cpp index f075993d4..0f69c486c 100644 --- a/utils/winport/winfinidb.cpp +++ b/utils/winport/winfinidb.cpp @@ -45,7 +45,6 @@ namespace ba = boost::algorithm; namespace { - bool vFlg; bool useSCMAPI; @@ -55,15 +54,14 @@ HANDLE svcStopEvent; struct ProcInfo { - ProcInfo() - { - } - ProcInfo(const string& name) : - pName(name) - { - } - string pName; - string pCmdLine; + ProcInfo() + { + } + ProcInfo(const string& name) : pName(name) + { + } + string pName; + string pCmdLine; }; typedef vector ProcInfoVec; @@ -77,250 +75,251 @@ bool shuttingDown = false; int runIt(const string& pName) { - int rc = 0; - char* cmdLine = (char*)alloca(cmdLineLen); - strncpy_s(cmdLine, cmdLineLen, pName.c_str(), pName.size()); - PROCESS_INFORMATION pInfo; - ZeroMemory(&pInfo, sizeof(pInfo)); - STARTUPINFO sInfo; - ZeroMemory(&sInfo, sizeof(sInfo)); + int rc = 0; + char* cmdLine = (char*)alloca(cmdLineLen); + strncpy_s(cmdLine, cmdLineLen, pName.c_str(), pName.size()); + PROCESS_INFORMATION pInfo; + ZeroMemory(&pInfo, sizeof(pInfo)); + STARTUPINFO sInfo; + ZeroMemory(&sInfo, sizeof(sInfo)); - try - { - if (CreateProcess(0, cmdLine, 0, 0, false, 0, 0, 0, &sInfo, &pInfo) == 0) - return -1; - } - catch (exception& e) - { - cout << e.what() << endl; - } + try + { + if (CreateProcess(0, cmdLine, 0, 0, false, 0, 0, 0, &sInfo, &pInfo) == 0) + return -1; + } + catch (exception& e) + { + cout << e.what() << endl; + } - if (WaitForSingleObject(pInfo.hProcess, INFINITE) != WAIT_OBJECT_0) - { - rc = -1; - goto out; - } + if (WaitForSingleObject(pInfo.hProcess, INFINITE) != WAIT_OBJECT_0) + { + rc = -1; + goto out; + } - DWORD exitCode; + DWORD exitCode; - if (GetExitCodeProcess(pInfo.hProcess, &exitCode) == 0) - { - rc = -1; - goto out; - } + if (GetExitCodeProcess(pInfo.hProcess, &exitCode) == 0) + { + rc = -1; + goto out; + } - if (exitCode != 0) - rc = -1; + if (exitCode != 0) + rc = -1; out: - CloseHandle(pInfo.hProcess); - return rc; + CloseHandle(pInfo.hProcess); + return rc; } int loadBRM() { - // if no save file just return - string saveFile = installDir + "\\dbrm\\BRM_saves_current"; - - if (GetFileAttributes(saveFile.c_str()) == INVALID_FILE_ATTRIBUTES) - return 0; - - // read contents of save file - ifstream ifs(saveFile.c_str()); - - if (!ifs) - return -1; - - string saveFilePfx; - getline(ifs, saveFilePfx); - - if (saveFilePfx.empty()) - return 0; - - // run load_brm and wait for it to finish - string brmCmd = "load_brm \"" + saveFilePfx + "\""; - - if (runIt(brmCmd)) - return -1; + // if no save file just return + string saveFile = installDir + "\\dbrm\\BRM_saves_current"; + if (GetFileAttributes(saveFile.c_str()) == INVALID_FILE_ATTRIBUTES) return 0; + + // read contents of save file + ifstream ifs(saveFile.c_str()); + + if (!ifs) + return -1; + + string saveFilePfx; + getline(ifs, saveFilePfx); + + if (saveFilePfx.empty()) + return 0; + + // run load_brm and wait for it to finish + string brmCmd = "load_brm \"" + saveFilePfx + "\""; + + if (runIt(brmCmd)) + return -1; + + return 0; } void ReportSvcStatus(DWORD currentState, DWORD win32ExitCode, DWORD waitHint) { - static DWORD checkPoint = 1; + static DWORD checkPoint = 1; - svcStatus.dwCurrentState = currentState; - svcStatus.dwWin32ExitCode = win32ExitCode; - svcStatus.dwWaitHint = waitHint; + svcStatus.dwCurrentState = currentState; + svcStatus.dwWin32ExitCode = win32ExitCode; + svcStatus.dwWaitHint = waitHint; - if (currentState == SERVICE_START_PENDING) - svcStatus.dwControlsAccepted = 0; - else - svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN; + if (currentState == SERVICE_START_PENDING) + svcStatus.dwControlsAccepted = 0; + else + svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN; - if (currentState == SERVICE_RUNNING || currentState == SERVICE_STOPPED) - svcStatus.dwCheckPoint = 0; - else - svcStatus.dwCheckPoint = checkPoint++; + if (currentState == SERVICE_RUNNING || currentState == SERVICE_STOPPED) + svcStatus.dwCheckPoint = 0; + else + svcStatus.dwCheckPoint = checkPoint++; - SetServiceStatus(svcStatusHandle, &svcStatus); + SetServiceStatus(svcStatusHandle, &svcStatus); } extern "C" DWORD WINAPI procRunner(LPVOID); int startUp() { - int rc; + int rc; - syslog(LOG_INFO, "System is starting"); + syslog(LOG_INFO, "System is starting"); - if (runIt("clearShm")) - return -1; + if (runIt("clearShm")) + return -1; - rc = loadBRM(); + rc = loadBRM(); - if (rc) - return rc; + if (rc) + return rc; - ProcInfoVec procInfo; - procInfo.push_back(ProcInfo("workernode DBRM_Worker1 fg")); - procInfo.push_back(ProcInfo("controllernode fg")); - procInfo.push_back(ProcInfo("DecomSvr")); - procInfo.push_back(ProcInfo("PrimProc")); - procInfo.push_back(ProcInfo("WriteEngineServer")); - procInfo.push_back(ProcInfo("ExeMgr")); - procInfo.push_back(ProcInfo("DDLProc")); - procInfo.push_back(ProcInfo("DMLProc")); - string mysqldCmd = "mysqld --defaults-file=" + installDir + "\\my.ini"; - procInfo.push_back(ProcInfo(mysqldCmd)); + ProcInfoVec procInfo; + procInfo.push_back(ProcInfo("workernode DBRM_Worker1 fg")); + procInfo.push_back(ProcInfo("controllernode fg")); + procInfo.push_back(ProcInfo("DecomSvr")); + procInfo.push_back(ProcInfo("PrimProc")); + procInfo.push_back(ProcInfo("WriteEngineServer")); + procInfo.push_back(ProcInfo("ExeMgr")); + procInfo.push_back(ProcInfo("DDLProc")); + procInfo.push_back(ProcInfo("DMLProc")); + string mysqldCmd = "mysqld --defaults-file=" + installDir + "\\my.ini"; + procInfo.push_back(ProcInfo(mysqldCmd)); - const ProcInfoVec::size_type numProcs = procInfo.size(); + const ProcInfoVec::size_type numProcs = procInfo.size(); - for (unsigned pidx = 0; pidx < numProcs; pidx++) + for (unsigned pidx = 0; pidx < numProcs; pidx++) + { + cout << "Starting " << procInfo[pidx].pName << "..."; + + HANDLE thd; + DWORD tid; + thd = CreateThread(0, 0, procRunner, &procInfo[pidx], 0, &tid); + + if (thd == NULL) { - cout << "Starting " << procInfo[pidx].pName << "..."; + LPTSTR lpBuffer; + LPTSTR* lppBuffer = &lpBuffer; + DWORD fmRes = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, 0, + GetLastError(), 0, (LPSTR)lppBuffer, 0, 0); - HANDLE thd; - DWORD tid; - thd = CreateThread(0, 0, procRunner, &procInfo[pidx], 0, &tid); + cerr << endl; + ostringstream ostr; + ostr << "Failed to start process runner thread for " << procInfo[pidx].pName << ": "; - if (thd == NULL) - { - LPTSTR lpBuffer; - LPTSTR* lppBuffer = &lpBuffer; - DWORD fmRes = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - 0, GetLastError(), 0, (LPSTR)lppBuffer, 0, 0); + if (fmRes > 0) + ostr << lpBuffer; + else + ostr << "Unknown error"; - cerr << endl; - ostringstream ostr; - ostr << "Failed to start process runner thread for " << procInfo[pidx].pName << ": "; - - if (fmRes > 0) - ostr << lpBuffer; - else - ostr << "Unknown error"; - - cerr << ostr.str() << endl; - syslog(LOG_ERR, ostr.str().c_str()); - return -1; - } - - Sleep(interProcessSleepTime * 1000); - cout << endl; - - if (useSCMAPI) - ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, interProcessSleepTime * (static_cast(numProcs) - pidx - 1) * 1000); + cerr << ostr.str() << endl; + syslog(LOG_ERR, ostr.str().c_str()); + return -1; } - return 0; + Sleep(interProcessSleepTime * 1000); + cout << endl; + + if (useSCMAPI) + ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, + interProcessSleepTime * (static_cast(numProcs) - pidx - 1) * 1000); + } + + return 0; } DWORD WINAPI procRunner(LPVOID parms) { - ProcInfo* pip = reinterpret_cast(parms); - ProcInfo pi = *pip; - char* cmdLine = (char*)alloca(cmdLineLen); - strncpy_s(cmdLine, cmdLineLen, pi.pName.c_str(), pi.pName.size()); - BOOL cpRc; - PROCESS_INFORMATION pInfo; - STARTUPINFO sInfo; + ProcInfo* pip = reinterpret_cast(parms); + ProcInfo pi = *pip; + char* cmdLine = (char*)alloca(cmdLineLen); + strncpy_s(cmdLine, cmdLineLen, pi.pName.c_str(), pi.pName.size()); + BOOL cpRc; + PROCESS_INFORMATION pInfo; + STARTUPINFO sInfo; - for (;;) + for (;;) + { + ZeroMemory(&sInfo, sizeof(sInfo)); + ZeroMemory(&pInfo, sizeof(pInfo)); + cpRc = CreateProcess(0, cmdLine, 0, 0, false, 0, 0, 0, &sInfo, &pInfo); + + if (cpRc == 0) { - ZeroMemory(&sInfo, sizeof(sInfo)); - ZeroMemory(&pInfo, sizeof(pInfo)); - cpRc = CreateProcess(0, cmdLine, 0, 0, false, 0, 0, 0, &sInfo, &pInfo); + LPTSTR lpBuffer; + LPTSTR* lppBuffer = &lpBuffer; + DWORD fmRes = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, 0, + GetLastError(), 0, (LPSTR)lppBuffer, 0, 0); + ostringstream ostr; + cerr << endl; + ostr << "Failed to start process " << pi.pName << ": "; - if (cpRc == 0) - { - LPTSTR lpBuffer; - LPTSTR* lppBuffer = &lpBuffer; - DWORD fmRes = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - 0, GetLastError(), 0, (LPSTR)lppBuffer, 0, 0); - ostringstream ostr; - cerr << endl; - ostr << "Failed to start process " << pi.pName << ": "; + if (fmRes > 0) + ostr << lpBuffer; + else + ostr << "Unknown error"; - if (fmRes > 0) - ostr << lpBuffer; - else - ostr << "Unknown error"; - - cerr << ostr.str() << endl; - syslog(LOG_ERR, ostr.str().c_str()); - return -1; - } - else - { - ostringstream ostr; - ostr << pi.pName << " started "; - cerr << ostr.str() << endl; - syslog(LOG_INFO, ostr.str().c_str()); - } - - WaitForSingleObject(pInfo.hProcess, INFINITE); - ostringstream ostr; - ostr << pi.pName; - - if (shuttingDown) - { - ostr << " shut down"; - syslog(LOG_INFO, ostr.str().c_str()); - } - else - { - ostr << " has died unexpectedly"; - syslog(LOG_ERR, ostr.str().c_str()); - } - - cerr << ostr.str() << endl; - CloseHandle(pInfo.hProcess); - - if (shuttingDown) - return 0; - - Sleep(10 * 1000); + cerr << ostr.str() << endl; + syslog(LOG_ERR, ostr.str().c_str()); + return -1; + } + else + { + ostringstream ostr; + ostr << pi.pName << " started "; + cerr << ostr.str() << endl; + syslog(LOG_INFO, ostr.str().c_str()); } - return -1; + WaitForSingleObject(pInfo.hProcess, INFINITE); + ostringstream ostr; + ostr << pi.pName; + + if (shuttingDown) + { + ostr << " shut down"; + syslog(LOG_INFO, ostr.str().c_str()); + } + else + { + ostr << " has died unexpectedly"; + syslog(LOG_ERR, ostr.str().c_str()); + } + + cerr << ostr.str() << endl; + CloseHandle(pInfo.hProcess); + + if (shuttingDown) + return 0; + + Sleep(10 * 1000); + } + + return -1; } int killProcByPid(DWORD pid) { - int rc = -1; - HANDLE hProc; - hProc = OpenProcess(PROCESS_TERMINATE, false, pid); + int rc = -1; + HANDLE hProc; + hProc = OpenProcess(PROCESS_TERMINATE, false, pid); - if (hProc != NULL) - { - if (TerminateProcess(hProc, 0) != 0) - rc = 0; + if (hProc != NULL) + { + if (TerminateProcess(hProc, 0) != 0) + rc = 0; - CloseHandle(hProc); - } + CloseHandle(hProc); + } - return rc; + return rc; } DWORD maxPids = 64; @@ -328,267 +327,260 @@ DWORD* pids = 0; int killProcByName(const string& pname) { - if (!pids) - pids = (DWORD*)malloc(maxPids * sizeof(DWORD)); + if (!pids) + pids = (DWORD*)malloc(maxPids * sizeof(DWORD)); - DWORD needed = 0; + DWORD needed = 0; + + if (EnumProcesses(pids, maxPids * sizeof(DWORD), &needed) == 0) + return -1; + + while (needed == maxPids * sizeof(DWORD)) + { + maxPids *= 2; + pids = (DWORD*)realloc(pids, maxPids * sizeof(DWORD)); if (EnumProcesses(pids, maxPids * sizeof(DWORD), &needed) == 0) - return -1; + return -1; + } - while (needed == maxPids * sizeof(DWORD)) + int rc = -1; + DWORD numPids = needed / sizeof(DWORD); + DWORD i; + + for (i = 0; i < numPids; i++) + { + bool found = false; + + if (pids[i] != 0) { - maxPids *= 2; - pids = (DWORD*)realloc(pids, maxPids * sizeof(DWORD)); + TCHAR szProcessName[MAX_PATH] = TEXT(""); - if (EnumProcesses(pids, maxPids * sizeof(DWORD), &needed) == 0) - return -1; + // Get a handle to the process. + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pids[i]); + + // Get the process name. + if (NULL != hProcess) + { + HMODULE hMod; + DWORD cbNeeded; + + if (EnumProcessModules(hProcess, &hMod, sizeof(hMod), &cbNeeded)) + { + GetModuleBaseName(hProcess, hMod, szProcessName, sizeof(szProcessName) / sizeof(TCHAR)); + } + } + + if (pname == szProcessName) + found = true; + + CloseHandle(hProcess); } - int rc = -1; - DWORD numPids = needed / sizeof(DWORD); - DWORD i; - - for (i = 0; i < numPids; i++) + if (found) { - bool found = false; - - if (pids[i] != 0) - { - TCHAR szProcessName[MAX_PATH] = TEXT(""); - - // Get a handle to the process. - HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION | - PROCESS_VM_READ, - FALSE, pids[i] ); - - // Get the process name. - if (NULL != hProcess ) - { - HMODULE hMod; - DWORD cbNeeded; - - if ( EnumProcessModules( hProcess, &hMod, sizeof(hMod), - &cbNeeded) ) - { - GetModuleBaseName( hProcess, hMod, szProcessName, - sizeof(szProcessName) / sizeof(TCHAR) ); - } - } - - if (pname == szProcessName) - found = true; - - CloseHandle( hProcess ); - } - - if (found) - { - rc = killProcByPid(pids[i]); - break; - } + rc = killProcByPid(pids[i]); + break; } + } - return rc; + return rc; } int shutDown() { - shuttingDown = true; - syslog(LOG_INFO, "System is shutting down"); + shuttingDown = true; + syslog(LOG_INFO, "System is shutting down"); - vector pList; + vector pList; - pList.push_back("mysqld.exe"); - pList.push_back("DMLProc.exe"); - pList.push_back("DDLProc.exe"); - pList.push_back("ExeMgr.exe"); - pList.push_back("WriteEngineServer.exe"); - pList.push_back("PrimProc.exe"); - pList.push_back("DecomSvr.exe"); - pList.push_back("controllernode.exe"); - pList.push_back("workernode.exe"); + pList.push_back("mysqld.exe"); + pList.push_back("DMLProc.exe"); + pList.push_back("DDLProc.exe"); + pList.push_back("ExeMgr.exe"); + pList.push_back("WriteEngineServer.exe"); + pList.push_back("PrimProc.exe"); + pList.push_back("DecomSvr.exe"); + pList.push_back("controllernode.exe"); + pList.push_back("workernode.exe"); - vector::iterator iter = pList.begin(); - vector::iterator end = pList.end(); - vector::size_type i = pList.size(); + vector::iterator iter = pList.begin(); + vector::iterator end = pList.end(); + vector::size_type i = pList.size(); - while (iter != end) - { - if (killProcByName(*iter) == 0) - Sleep(interProcessSleepTime * 1000); - - if (useSCMAPI) - ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, static_cast(--i) * interProcessSleepTime * 1000); - - ++iter; - } - - if (runIt("save_brm") == 0) - runIt("clearShm"); + while (iter != end) + { + if (killProcByName(*iter) == 0) + Sleep(interProcessSleepTime * 1000); if (useSCMAPI) - ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, static_cast(--i) * interProcessSleepTime * 1000); - return 0; + ++iter; + } + + if (runIt("save_brm") == 0) + runIt("clearShm"); + + if (useSCMAPI) + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); + + return 0; } void svcStop() { - useSCMAPI = true; - ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 20 * 1000); - SetEvent(svcStopEvent); + useSCMAPI = true; + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 20 * 1000); + SetEvent(svcStopEvent); } extern "C" VOID WINAPI svcMain(DWORD, LPSTR*); -//The SCM has asked us to start as a service +// The SCM has asked us to start as a service int svcStart() { - SERVICE_TABLE_ENTRY st[] = - { - { "InfiniDB", svcMain }, - { 0, 0 }, - }; + SERVICE_TABLE_ENTRY st[] = { + {"InfiniDB", svcMain}, + {0, 0}, + }; - //This call returns when the service has stopped - StartServiceCtrlDispatcher(st); + // This call returns when the service has stopped + StartServiceCtrlDispatcher(st); - return 0; + return 0; } extern "C" VOID WINAPI svcCtrlHandler(DWORD); VOID WINAPI svcMain(DWORD dwArgc, LPSTR* lpszArgv) { - useSCMAPI = true; + useSCMAPI = true; - svcStatusHandle = RegisterServiceCtrlHandler("InfiniDB", svcCtrlHandler); + svcStatusHandle = RegisterServiceCtrlHandler("InfiniDB", svcCtrlHandler); - svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - svcStatus.dwServiceSpecificExitCode = 0; + svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + svcStatus.dwServiceSpecificExitCode = 0; - ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, interProcessSleepTime * 7 * 1000); + ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, interProcessSleepTime * 7 * 1000); - svcStopEvent = CreateEvent(0, 1, 0, 0); + svcStopEvent = CreateEvent(0, 1, 0, 0); - if (startUp() != 0) - { - ReportSvcStatus(SERVICE_STOPPED, 1, 0); - return; - } + if (startUp() != 0) + { + ReportSvcStatus(SERVICE_STOPPED, 1, 0); + return; + } - ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); + ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); - WaitForSingleObject(svcStopEvent, INFINITE); + WaitForSingleObject(svcStopEvent, INFINITE); - shutDown(); + shutDown(); - CloseHandle(svcStopEvent); - - ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); + CloseHandle(svcStopEvent); + ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); } VOID WINAPI svcCtrlHandler(DWORD dwCtrl) { - // Handle the requested control code. + // Handle the requested control code. - switch (dwCtrl) - { - case SERVICE_CONTROL_STOP: //Notifies a service that it should stop - case SERVICE_CONTROL_SHUTDOWN: //Notifies a service that the system is shutting down so the service can perform cleanup tasks - svcStop(); - break; + switch (dwCtrl) + { + case SERVICE_CONTROL_STOP: // Notifies a service that it should stop + case SERVICE_CONTROL_SHUTDOWN: // Notifies a service that the system is shutting down so the service can + // perform cleanup tasks + svcStop(); + break; - case SERVICE_CONTROL_INTERROGATE: //Notifies a service that it should report its current status information to the service control manager - break; + case SERVICE_CONTROL_INTERROGATE: // Notifies a service that it should report its current status + // information to the service control manager + break; - case SERVICE_CONTROL_CONTINUE: //Notifies a paused service that it should resume - break; + case SERVICE_CONTROL_CONTINUE: // Notifies a paused service that it should resume + break; - case SERVICE_CONTROL_PARAMCHANGE: //Notifies a service that its startup parameters have changed. The service should reread its startup parameters - break; + case SERVICE_CONTROL_PARAMCHANGE: // Notifies a service that its startup parameters have changed. The + // service should reread its startup parameters + break; - case SERVICE_CONTROL_PAUSE: //Notifies a service that it should pause - break; + case SERVICE_CONTROL_PAUSE: // Notifies a service that it should pause + break; - default: - break; - } + default: break; + } } -} +} // namespace int main(int argc, char** argv) { - opterr = 0; - vFlg = false; - int c; - int rc; - useSCMAPI = false; + opterr = 0; + vFlg = false; + int c; + int rc; + useSCMAPI = false; - while ((c = getopt(argc, argv, "v")) != EOF) - switch (c) - { - case 'v': - vFlg = true; - break; - - case '?': - default: - break; - } - - if (!vFlg) + while ((c = getopt(argc, argv, "v")) != EOF) + switch (c) { - _close(0); - _close(1); - _close(2); - int fd = -1; - _sopen_s(&fd, "NUL", _O_TEXT | _O_RDONLY, _SH_DENYNO, _S_IREAD); - assert(fd == 0); - fd = -1; - _sopen_s(&fd, "NUL", _O_TEXT | _O_APPEND | _O_WRONLY, _SH_DENYNO, _S_IWRITE); - assert(fd == 1); - fd = -1; - _sopen_s(&fd, "NUL", _O_TEXT | _O_APPEND | _O_WRONLY, _SH_DENYNO, _S_IWRITE); - assert(fd == 2); + case 'v': vFlg = true; break; + + case '?': + default: break; } - int (*cmdFp)(); + if (!vFlg) + { + _close(0); + _close(1); + _close(2); + int fd = -1; + _sopen_s(&fd, "NUL", _O_TEXT | _O_RDONLY, _SH_DENYNO, _S_IREAD); + assert(fd == 0); + fd = -1; + _sopen_s(&fd, "NUL", _O_TEXT | _O_APPEND | _O_WRONLY, _SH_DENYNO, _S_IWRITE); + assert(fd == 1); + fd = -1; + _sopen_s(&fd, "NUL", _O_TEXT | _O_APPEND | _O_WRONLY, _SH_DENYNO, _S_IWRITE); + assert(fd == 2); + } - string command; + int (*cmdFp)(); - if (argc - optind < 1) - command = "svcstart"; - else - command = argv[optind]; + string command; - if (ba::istarts_with(command, "sta")) - { - cmdFp = startUp; - } - else if (ba::istarts_with(command, "sto") || ba::istarts_with(command, "sh")) - { - cmdFp = shutDown; - } - else if (ba::istarts_with(command, "sv")) - { - cmdFp = svcStart; - } - else - { - cerr << "Unknown command " << command << " (try 'start' or 'shutdown')" << endl; - return 1; - } + if (argc - optind < 1) + command = "svcstart"; + else + command = argv[optind]; - installDir = IDBreadRegistry("", true); + if (ba::istarts_with(command, "sta")) + { + cmdFp = startUp; + } + else if (ba::istarts_with(command, "sto") || ba::istarts_with(command, "sh")) + { + cmdFp = shutDown; + } + else if (ba::istarts_with(command, "sv")) + { + cmdFp = svcStart; + } + else + { + cerr << "Unknown command " << command << " (try 'start' or 'shutdown')" << endl; + return 1; + } - string newDir = installDir + "\\bin"; - rc = _chdir(newDir.c_str()); + installDir = IDBreadRegistry("", true); - rc = cmdFp(); + string newDir = installDir + "\\bin"; + rc = _chdir(newDir.c_str()); - return (rc ? 1 : 0); + rc = cmdFp(); + + return (rc ? 1 : 0); } diff --git a/utils/winport/winport.cpp b/utils/winport/winport.cpp index e7aa6606f..55bab33bd 100644 --- a/utils/winport/winport.cpp +++ b/utils/winport/winport.cpp @@ -32,196 +32,188 @@ using namespace std; #include "idbregistry.h" #include "WinSyslog.h" #include "io.h" -//This is the number of msecs between 1601 and 1970 (+/-) +// This is the number of msecs between 1601 and 1970 (+/-) //#define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 -//This is the number of 100-nsec intvls btwn 1601 and 1970 (+/-) -const unsigned __int64 EPOCH_DELTA = ((1970Ui64 - 1601Ui64) * 365Ui64 + - ((1970Ui64 - 1601Ui64) / 4Ui64) - - ((1970Ui64 - 1601Ui64) / 100Ui64) + - ((1970Ui64 - 1601Ui64) / 400Ui64)) * +// This is the number of 100-nsec intvls btwn 1601 and 1970 (+/-) +const unsigned __int64 EPOCH_DELTA = ((1970Ui64 - 1601Ui64) * 365Ui64 + ((1970Ui64 - 1601Ui64) / 4Ui64) - + ((1970Ui64 - 1601Ui64) / 100Ui64) + ((1970Ui64 - 1601Ui64) / 400Ui64)) * 86400Ui64 * 1000Ui64 * 1000Ui64 * 10Ui64; // returns the secs+usecs since the epoch int gettimeofday(struct timeval* tvp, struct timezone* tzp) { - FILETIME ft; - unsigned __int64 tmpres = 0; - static int tzflag; + FILETIME ft; + unsigned __int64 tmpres = 0; + static int tzflag; - if (0 != tvp) + if (0 != tvp) + { + // returns the current time as the number of 100-nanosecond intervals since January 1, 1601 (UTC) + GetSystemTimeAsFileTime(&ft); + + tmpres |= ft.dwHighDateTime; + tmpres <<= 32; + tmpres |= ft.dwLowDateTime; + + /*converting file time to unix epoch*/ + tmpres -= EPOCH_DELTA; + tmpres /= 10; /*convert into microseconds*/ + tvp->tv_sec = (long)(tmpres / 1000000UL); + tvp->tv_usec = (long)(tmpres % 1000000UL); + } + + if (0 != tzp) + { + if (!tzflag) { - //returns the current time as the number of 100-nanosecond intervals since January 1, 1601 (UTC) - GetSystemTimeAsFileTime(&ft); - - tmpres |= ft.dwHighDateTime; - tmpres <<= 32; - tmpres |= ft.dwLowDateTime; - - /*converting file time to unix epoch*/ - tmpres -= EPOCH_DELTA; - tmpres /= 10; /*convert into microseconds*/ - tvp->tv_sec = (long)(tmpres / 1000000UL); - tvp->tv_usec = (long)(tmpres % 1000000UL); + _tzset(); + tzflag++; } - if (0 != tzp) - { - if (!tzflag) - { - _tzset(); - tzflag++; - } + tzp->tz_minuteswest = _timezone / 60; + tzp->tz_dsttime = _daylight; + } - tzp->tz_minuteswest = _timezone / 60; - tzp->tz_dsttime = _daylight; - } - - return 0; + return 0; } int closelog(...) { - return 0; + return 0; } int openlog(...) { - return 0; + return 0; } int syslog(int priority, const char* format, ...) { - int rtn; - va_list args; - va_start(args, format); - rtn = WinSyslog::instance()->Log(priority, format, args); - va_end(args); - return rtn; + int rtn; + va_list args; + va_start(args, format); + rtn = WinSyslog::instance()->Log(priority, format, args); + va_end(args); + return rtn; } int fcntl(int i1, int i2, ...) { - return 0; + return 0; } int inet_aton(const char* c, struct in_addr* p) { - p->S_un.S_addr = inet_addr(c); - return 1; + p->S_un.S_addr = inet_addr(c); + return 1; } int flock(int i1, int i2) { - return 0; + return 0; } int usleep(unsigned int usecs) { - unsigned int msecs; - //cvt usecs to msecs - msecs = usecs / 1000; + unsigned int msecs; + // cvt usecs to msecs + msecs = usecs / 1000; - if (msecs == 0) msecs++; + if (msecs == 0) + msecs++; - Sleep(msecs); - return 0; + Sleep(msecs); + return 0; } int fork() { - return -1; + return -1; } int getpagesize() { - return 4096; + return 4096; } struct tm* idb_localtime_r(const time_t* tp, struct tm* tmp) { - time_t t = *tp; - errno_t p = 0; - p = localtime_s(tmp, &t); + time_t t = *tp; + errno_t p = 0; + p = localtime_s(tmp, &t); - if (p != 0) - memset(tmp, 0, sizeof(struct tm)); + if (p != 0) + memset(tmp, 0, sizeof(struct tm)); - return tmp; + return tmp; } -//FIXME: need a better impl! +// FIXME: need a better impl! long clock_gettime(clockid_t, struct timespec* tp) { - SYSTEMTIME st; - GetSystemTime(&st); - tp->tv_sec = st.wHour * 3600 + st.wMinute * 60 + st.wSecond; - tp->tv_nsec = st.wMilliseconds * 1000000; - return 0; + SYSTEMTIME st; + GetSystemTime(&st); + tp->tv_sec = st.wHour * 3600 + st.wMinute * 60 + st.wSecond; + tp->tv_nsec = st.wMilliseconds * 1000000; + return 0; } #if _MSC_VER < 1600 lldiv_t lldiv(const long long numer, const long long denom) { - lldiv_t ret; - ret.quot = numer / denom; - ret.rem = numer % denom; - return ret; + lldiv_t ret; + ret.quot = numer / denom; + ret.rem = numer % denom; + return ret; } #endif unsigned int sleep(unsigned int secs) { - Sleep(secs * 1000); - return 0; + Sleep(secs * 1000); + return 0; } int pipe(int fds[2]) { - return -1; + return -1; } pid_t getppid() { - return -1; + return -1; } pid_t waitpid(pid_t, int*, int) { - return -1; + return -1; } int kill(pid_t, int) { - return -1; + return -1; } int setuid(uid_t) { - return -1; + return -1; } string IDBSysErrorStr(DWORD err) { - // Retrieve the system error message for the last-error code + // Retrieve the system error message for the last-error code - string errstr; + string errstr; - LPVOID lpMsgBuf; + LPVOID lpMsgBuf; - FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR) &lpMsgBuf, - 0, NULL ); + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&lpMsgBuf, 0, NULL); - // Display the error message and exit the process + // Display the error message and exit the process - errstr = (LPCTSTR)lpMsgBuf; + errstr = (LPCTSTR)lpMsgBuf; - LocalFree(lpMsgBuf); + LocalFree(lpMsgBuf); - return errstr; + return errstr; } diff --git a/versioning/BRM/autoincrementmanager.cpp b/versioning/BRM/autoincrementmanager.cpp index 9decda641..f256924b3 100644 --- a/versioning/BRM/autoincrementmanager.cpp +++ b/versioning/BRM/autoincrementmanager.cpp @@ -33,7 +33,6 @@ using namespace boost::posix_time; namespace BRM { - AutoincrementManager::AutoincrementManager() { } @@ -43,118 +42,116 @@ AutoincrementManager::~AutoincrementManager() } void AutoincrementManager::startSequence(uint32_t oid, uint64_t firstNum, uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType) + execplan::CalpontSystemCatalog::ColDataType colDataType) { - boost::mutex::scoped_lock lk(lock); - map::iterator it; - sequence s; + boost::mutex::scoped_lock lk(lock); + map::iterator it; + sequence s; - it = sequences.find(oid); + it = sequences.find(oid); - if (it != sequences.end()) - return; + if (it != sequences.end()) + return; - s.value = firstNum; + s.value = firstNum; - if (isUnsigned(colDataType)) - { - s.overflow = (0xFFFFFFFFFFFFFFFFULL >> (64 - colWidth * 8)) - 1; - } - else - { - s.overflow = (1ULL << (colWidth * 8 - 1)); - } + if (isUnsigned(colDataType)) + { + s.overflow = (0xFFFFFFFFFFFFFFFFULL >> (64 - colWidth * 8)) - 1; + } + else + { + s.overflow = (1ULL << (colWidth * 8 - 1)); + } - sequences[oid] = s; + sequences[oid] = s; } bool AutoincrementManager::getAIRange(uint32_t oid, uint64_t count, uint64_t* firstNum) { - boost::mutex::scoped_lock lk(lock); - map::iterator it; + boost::mutex::scoped_lock lk(lock); + map::iterator it; - it = sequences.find(oid); + it = sequences.find(oid); - if (it == sequences.end()) - throw runtime_error("There is no sequence with that lock"); + if (it == sequences.end()) + throw runtime_error("There is no sequence with that lock"); - if ((count >= it->second.overflow || - count + it->second.value > it->second.overflow || - count + it->second.value <= it->second.value) - && count != 0) - return false; + if ((count >= it->second.overflow || count + it->second.value > it->second.overflow || + count + it->second.value <= it->second.value) && + count != 0) + return false; - *firstNum = it->second.value; - it->second.value += count; - return true; + *firstNum = it->second.value; + it->second.value += count; + return true; } void AutoincrementManager::resetSequence(uint32_t oid, uint64_t value) { - boost::mutex::scoped_lock lk(lock); - map::iterator it; + boost::mutex::scoped_lock lk(lock); + map::iterator it; - it = sequences.find(oid); + it = sequences.find(oid); - if (it == sequences.end()) - return; + if (it == sequences.end()) + return; - it->second.value = value; + it->second.value = value; } const uint32_t AutoincrementManager::lockTime = 30; void AutoincrementManager::getLock(uint32_t oid) { - boost::mutex::scoped_lock lk(lock); - map::iterator it; - ptime stealTime = microsec_clock::local_time() + seconds(lockTime); + boost::mutex::scoped_lock lk(lock); + map::iterator it; + ptime stealTime = microsec_clock::local_time() + seconds(lockTime); - bool gotIt = false; + bool gotIt = false; - it = sequences.find(oid); + it = sequences.find(oid); - if (it == sequences.end()) - throw runtime_error("There is no sequence with that lock"); + if (it == sequences.end()) + throw runtime_error("There is no sequence with that lock"); - lk.unlock(); + lk.unlock(); - while (!gotIt && microsec_clock::local_time() < stealTime) - { - gotIt = it->second.lock.try_lock(); + while (!gotIt && microsec_clock::local_time() < stealTime) + { + gotIt = it->second.lock.try_lock(); - if (!gotIt) - usleep(100000); - } + if (!gotIt) + usleep(100000); + } - // If !gotIt, take possession + // If !gotIt, take possession } void AutoincrementManager::releaseLock(uint32_t oid) { - boost::mutex::scoped_lock lk(lock); - map::iterator it; + boost::mutex::scoped_lock lk(lock); + map::iterator it; - it = sequences.find(oid); + it = sequences.find(oid); - if (it == sequences.end()) - return; // it's unlocked if the lock doesn't exist... + if (it == sequences.end()) + return; // it's unlocked if the lock doesn't exist... - lk.unlock(); + lk.unlock(); - it->second.lock.unlock(); + it->second.lock.unlock(); } void AutoincrementManager::deleteSequence(uint32_t oid) { - boost::mutex::scoped_lock lk(lock); - map::iterator it; + boost::mutex::scoped_lock lk(lock); + map::iterator it; - it = sequences.find(oid); + it = sequences.find(oid); - if (it != sequences.end()) - sequences.erase(it); + if (it != sequences.end()) + sequences.erase(it); } - } /* namespace BRM */ diff --git a/versioning/BRM/autoincrementmanager.h b/versioning/BRM/autoincrementmanager.h index 133d99d51..984384d9e 100644 --- a/versioning/BRM/autoincrementmanager.h +++ b/versioning/BRM/autoincrementmanager.h @@ -38,40 +38,43 @@ namespace BRM { - class AutoincrementManager { -public: - EXPORT AutoincrementManager(); - EXPORT virtual ~AutoincrementManager(); + public: + EXPORT AutoincrementManager(); + EXPORT virtual ~AutoincrementManager(); - EXPORT void startSequence(uint32_t OID, uint64_t firstNum, uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType); - EXPORT bool getAIRange(uint32_t OID, uint64_t count, uint64_t* firstNum); - EXPORT void resetSequence(uint32_t OID, uint64_t value); - EXPORT void getLock(uint32_t OID); - EXPORT void releaseLock(uint32_t OID); - EXPORT void deleteSequence(uint32_t OID); + EXPORT void startSequence(uint32_t OID, uint64_t firstNum, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT bool getAIRange(uint32_t OID, uint64_t count, uint64_t* firstNum); + EXPORT void resetSequence(uint32_t OID, uint64_t value); + EXPORT void getLock(uint32_t OID); + EXPORT void releaseLock(uint32_t OID); + EXPORT void deleteSequence(uint32_t OID); -private: - static const uint32_t lockTime; // 30 seconds - struct sequence + private: + static const uint32_t lockTime; // 30 seconds + struct sequence + { + sequence() : value(0), overflow(0) { - sequence() : value(0), overflow(0) { } - sequence(const sequence& s) : value(s.value), overflow(s.overflow) { } - sequence& operator=(const sequence& s) - { - value = s.value; - overflow = s.overflow; - return *this; - } - uint64_t value; - uint64_t overflow; - boost::mutex lock; - }; - + } + sequence(const sequence& s) : value(s.value), overflow(s.overflow) + { + } + sequence& operator=(const sequence& s) + { + value = s.value; + overflow = s.overflow; + return *this; + } + uint64_t value; + uint64_t overflow; boost::mutex lock; - std::map sequences; + }; + + boost::mutex lock; + std::map sequences; }; } /* namespace BRM */ diff --git a/versioning/BRM/autoinctest.cpp b/versioning/BRM/autoinctest.cpp index 1fca39f80..07736e149 100644 --- a/versioning/BRM/autoinctest.cpp +++ b/versioning/BRM/autoinctest.cpp @@ -3,7 +3,6 @@ #include "dbrm.h" #include "IDBPolicy.h" - using namespace std; using namespace BRM; @@ -20,173 +19,134 @@ DBRM dbrm; void doStartSeq() { + try + { + dbrm.startAISequence(12345, 67890, 1, execplan::CalpontSystemCatalog::INT); + } + catch (exception& e) + { + cout << "got exception: " << e.what() << endl; + exit(1); + } - try - { - dbrm.startAISequence(12345, 67890, 1, execplan::CalpontSystemCatalog::INT); - } - catch (exception& e) - { - cout << "got exception: " << e.what() << endl; - exit(1); - } - - cout << "OK.\n"; + cout << "OK.\n"; } void doGetRange() { - uint64_t firstNum; - bool ret; + uint64_t firstNum; + bool ret; - try - { - ret = dbrm.getAIRange(12345, 110000, &firstNum); - } - catch (exception& e) - { - cout << "got exception: " << e.what() << endl; - exit(1); - } + try + { + ret = dbrm.getAIRange(12345, 110000, &firstNum); + } + catch (exception& e) + { + cout << "got exception: " << e.what() << endl; + exit(1); + } - if (!ret) - cout << "getAIRange failed\n"; - else - cout << "got firstNum " << firstNum << endl; + if (!ret) + cout << "getAIRange failed\n"; + else + cout << "got firstNum " << firstNum << endl; } void doGetValue() { - uint64_t val; - bool ret; + uint64_t val; + bool ret; - try - { - ret = dbrm.getAIValue(12345, &val); - } - catch (exception& e) - { - cout << "got exception: " << e.what() << endl; - exit(1); - } + try + { + ret = dbrm.getAIValue(12345, &val); + } + catch (exception& e) + { + cout << "got exception: " << e.what() << endl; + exit(1); + } - if (!ret) - cout << "getAIValue failed\n"; - else - cout << "got val " << val << endl; + if (!ret) + cout << "getAIValue failed\n"; + else + cout << "got val " << val << endl; } void doReset() { + try + { + dbrm.resetAISequence(12345, 11111); + } + catch (exception& e) + { + cout << "got exception: " << e.what() << endl; + exit(1); + } - try - { - dbrm.resetAISequence(12345, 11111); - } - catch (exception& e) - { - cout << "got exception: " << e.what() << endl; - exit(1); - } - - cout << "OK.\n"; + cout << "OK.\n"; } void doLock() { + try + { + dbrm.getAILock(12345); + } + catch (exception& e) + { + cout << "got exception: " << e.what() << endl; + exit(1); + } - try - { - dbrm.getAILock(12345); - } - catch (exception& e) - { - cout << "got exception: " << e.what() << endl; - exit(1); - } - - cout << "OK.\n"; + cout << "OK.\n"; } void doUnlock() { + try + { + dbrm.releaseAILock(12345); + } + catch (exception& e) + { + cout << "got exception: " << e.what() << endl; + exit(1); + } - try - { - dbrm.releaseAILock(12345); - } - catch (exception& e) - { - cout << "got exception: " << e.what() << endl; - exit(1); - } - - cout << "OK.\n"; + cout << "OK.\n"; } int main(int argc, char** argv) { + if (argc < 2) + { + cout << "Usage: " << argv[0] << " s | r | v | R | l | u. Check the code to see what they do. :P\n"; + exit(1); + } - if (argc < 2) - { - cout << "Usage: " << argv[0] << " s | r | v | R | l | u. Check the code to see what they do. :P\n"; - exit(1); - } + char cmd = argv[1][0]; - char cmd = argv[1][0]; + idbdatafile::IDBPolicy::configIDBPolicy(); - idbdatafile::IDBPolicy::configIDBPolicy(); + switch (cmd) + { + case 's': doStartSeq(); break; - switch (cmd) - { - case 's': - doStartSeq(); - break; + case 'r': doGetRange(); break; - case 'r': - doGetRange(); - break; + case 'v': doGetValue(); break; - case 'v': - doGetValue(); - break; + case 'R': doReset(); break; - case 'R': - doReset(); - break; + case 'l': doLock(); break; - case 'l': - doLock(); - break; + case 'u': doUnlock(); break; - case 'u': - doUnlock(); - break; - - default: - cout << "Usage: " << argv[0] << " s | r | v | R | l | u. Check the code to see what they do. :P\n"; - exit(1); - } + default: + cout << "Usage: " << argv[0] << " s | r | v | R | l | u. Check the code to see what they do. :P\n"; + exit(1); + } } - - - - - - - - - - - - - - - - - - - - - - diff --git a/versioning/BRM/blockresolutionmanager.cpp b/versioning/BRM/blockresolutionmanager.cpp index 6004a3832..b1706cb35 100644 --- a/versioning/BRM/blockresolutionmanager.cpp +++ b/versioning/BRM/blockresolutionmanager.cpp @@ -50,21 +50,20 @@ using namespace std; namespace BRM { - BlockResolutionManager::BlockResolutionManager(bool ronly) throw() { - if (ronly) - { - em.setReadOnly(); - vss.setReadOnly(); - vbbm.setReadOnly(); - copylocks.setReadOnly(); - } + if (ronly) + { + em.setReadOnly(); + vss.setReadOnly(); + vbbm.setReadOnly(); + copylocks.setReadOnly(); + } } BlockResolutionManager::BlockResolutionManager(const BlockResolutionManager& brm) { - throw logic_error("BRM: Don't use the copy constructor."); + throw logic_error("BRM: Don't use the copy constructor."); } BlockResolutionManager::~BlockResolutionManager() throw() @@ -73,124 +72,122 @@ BlockResolutionManager::~BlockResolutionManager() throw() BlockResolutionManager& BlockResolutionManager::operator=(const BlockResolutionManager& brm) { - throw logic_error("BRM: Don't use the = operator."); + throw logic_error("BRM: Don't use the = operator."); } int BlockResolutionManager::loadExtentMap(const string& filename, bool fixFL) { - em.load(filename, fixFL); - return 0; + em.load(filename, fixFL); + return 0; } int BlockResolutionManager::saveExtentMap(const string& filename) { - em.save(filename); - return 0; + em.save(filename); + return 0; } int BlockResolutionManager::saveState(string filename) throw() { - string emFilename = filename + "_em"; - string vssFilename = filename + "_vss"; - string vbbmFilename = filename + "_vbbm"; - string journalFilename = filename + "_journal"; + string emFilename = filename + "_em"; + string vssFilename = filename + "_vss"; + string vbbmFilename = filename + "_vbbm"; + string journalFilename = filename + "_journal"; - bool locked[2] = { false, false }; + bool locked[2] = {false, false}; - try - { - vbbm.lock(VBBM::READ); - locked[0] = true; - vss.lock(VSS::READ); - locked[1] = true; + try + { + vbbm.lock(VBBM::READ); + locked[0] = true; + vss.lock(VSS::READ); + locked[1] = true; - saveExtentMap(emFilename); + saveExtentMap(emFilename); - // truncate teh file if already exists since no truncate in HDFS. - const char* filename_p = journalFilename.c_str(); + // truncate teh file if already exists since no truncate in HDFS. + const char* filename_p = journalFilename.c_str(); - IDBDataFile* journal = IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", 0); - delete journal; + IDBDataFile* journal = + IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", 0); + delete journal; - vbbm.save(vbbmFilename); - vss.save(vssFilename); + vbbm.save(vbbmFilename); + vss.save(vssFilename); - vss.release(VSS::READ); - locked[1] = false; - vbbm.release(VBBM::READ); - locked[0] = false; - } - catch (exception& e) - { - if (locked[1]) - vss.release(VSS::READ); + vss.release(VSS::READ); + locked[1] = false; + vbbm.release(VBBM::READ); + locked[0] = false; + } + catch (exception& e) + { + if (locked[1]) + vss.release(VSS::READ); - if (locked[0]) - vbbm.release(VBBM::READ); + if (locked[0]) + vbbm.release(VBBM::READ); - cout << e.what() << endl; - return -1; - } + cout << e.what() << endl; + return -1; + } - return 0; + return 0; } int BlockResolutionManager::loadState(string filename, bool fixFL) throw() { - string emFilename = filename + "_em"; - string vssFilename = filename + "_vss"; - string vbbmFilename = filename + "_vbbm"; - bool locked[2] = { false, false}; + string emFilename = filename + "_em"; + string vssFilename = filename + "_vss"; + string vbbmFilename = filename + "_vbbm"; + bool locked[2] = {false, false}; - try - { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; - loadExtentMap(emFilename, fixFL); - vbbm.load(vbbmFilename); - vss.load(vssFilename); + loadExtentMap(emFilename, fixFL); + vbbm.load(vbbmFilename); + vss.load(vssFilename); - vss.release(VSS::WRITE); - locked[1] = false; - vbbm.release(VBBM::WRITE); - locked[0] = false; - } - catch (exception& e) - { - if (locked[1]) - vss.release(VSS::WRITE); + vss.release(VSS::WRITE); + locked[1] = false; + vbbm.release(VBBM::WRITE); + locked[0] = false; + } + catch (exception& e) + { + if (locked[1]) + vss.release(VSS::WRITE); - if (locked[0]) - vbbm.release(VBBM::WRITE); + if (locked[0]) + vbbm.release(VBBM::WRITE); - cout << e.what() << endl; - return -1; - } + cout << e.what() << endl; + return -1; + } - return 0; + return 0; } int BlockResolutionManager::replayJournal(string prefix) throw() { - SlaveComm sc; - int err = -1; + SlaveComm sc; + int err = -1; - try - { - err = sc.replayJournal(prefix); - } - catch (exception& e) - { - cout << e.what(); - } + try + { + err = sc.replayJournal(prefix); + } + catch (exception& e) + { + cout << e.what(); + } - return err; + return err; } - -} //namespace - +} // namespace BRM diff --git a/versioning/BRM/blockresolutionmanager.h b/versioning/BRM/blockresolutionmanager.h index b140f1b95..3cc87cfda 100644 --- a/versioning/BRM/blockresolutionmanager.h +++ b/versioning/BRM/blockresolutionmanager.h @@ -46,7 +46,6 @@ namespace BRM { - /** @brief The BlockResolutionManager manages the Logical Block ID space. * * The BlockResolutionManager manages the Logical Block ID space. Its @@ -57,73 +56,72 @@ namespace BRM */ class BlockResolutionManager { -public: - EXPORT explicit BlockResolutionManager(bool ronly = false) throw(); - EXPORT ~BlockResolutionManager() throw(); + public: + EXPORT explicit BlockResolutionManager(bool ronly = false) throw(); + EXPORT ~BlockResolutionManager() throw(); - /** @brief Persistence API. Loads the local Extent Map from a file. - * - * Persistence API. Loads the local Extent Map from a file. - * - * @warning The load must be done on each slave node atomically wrt - * writing operations, otherwise nodes may be out of synch. - * @param filename Relative or absolute path to a file saved with saveExtentMap. - * @return 0, throws if EM throws - */ - EXPORT int loadExtentMap(const std::string& filename, bool fixFL); + /** @brief Persistence API. Loads the local Extent Map from a file. + * + * Persistence API. Loads the local Extent Map from a file. + * + * @warning The load must be done on each slave node atomically wrt + * writing operations, otherwise nodes may be out of synch. + * @param filename Relative or absolute path to a file saved with saveExtentMap. + * @return 0, throws if EM throws + */ + EXPORT int loadExtentMap(const std::string& filename, bool fixFL); - /** @brief Persistence API. Saves the local Extent Map to a file. - * - * Persistence API. Saves the local Extent Map to a file. - * - * @param filename Relative or absolute path to save to. - * @return 0 on success, throws if EM throws - */ - EXPORT int saveExtentMap(const std::string& filename); + /** @brief Persistence API. Saves the local Extent Map to a file. + * + * Persistence API. Saves the local Extent Map to a file. + * + * @param filename Relative or absolute path to save to. + * @return 0 on success, throws if EM throws + */ + EXPORT int saveExtentMap(const std::string& filename); - /** @brief Persistence API. Loads all BRM snapshots. - * - * Loads all local BRM structures from files saved with saveState(). - * - * @warning The load must be done on each slave node atomically wrt - * writing operations, otherwise nodes may be out of synch. - * @param filename The filename prefix to use. Loads 4 files with that prefix. - * @return 0 on success, -1 on error - */ - EXPORT int loadState(std::string filename, bool fixFL = false) throw(); + /** @brief Persistence API. Loads all BRM snapshots. + * + * Loads all local BRM structures from files saved with saveState(). + * + * @warning The load must be done on each slave node atomically wrt + * writing operations, otherwise nodes may be out of synch. + * @param filename The filename prefix to use. Loads 4 files with that prefix. + * @return 0 on success, -1 on error + */ + EXPORT int loadState(std::string filename, bool fixFL = false) throw(); - /** @brief Persistence API. Loads the BRM deltas since the last snapshot. - * - * Loads all local BRM structures from files saved with saveState(). - * - * @warning The load must be done on each slave node atomically wrt - * writing operations, otherwise nodes may be out of synch. - * @param filename The filename prefix to use. Loads 4 files with that prefix. - * @return 0 on success, -1 on error - */ - EXPORT int replayJournal(std::string filename) throw(); + /** @brief Persistence API. Loads the BRM deltas since the last snapshot. + * + * Loads all local BRM structures from files saved with saveState(). + * + * @warning The load must be done on each slave node atomically wrt + * writing operations, otherwise nodes may be out of synch. + * @param filename The filename prefix to use. Loads 4 files with that prefix. + * @return 0 on success, -1 on error + */ + EXPORT int replayJournal(std::string filename) throw(); - /** @brief Persistence API. Saves all BRM structures. - * - * Saves all local BRM structures to files. - * - * @param filename The filename prefix to use. Saves 4 files with that prefix. - * @return 0 on success, -1 on error - */ - EXPORT int saveState(std::string filename) throw(); - -private: - explicit BlockResolutionManager(const BlockResolutionManager& brm); - BlockResolutionManager& operator=(const BlockResolutionManager& brm); - MasterSegmentTable mst; - ExtentMap em; - VBBM vbbm; - VSS vss; - CopyLocks copylocks; + /** @brief Persistence API. Saves all BRM structures. + * + * Saves all local BRM structures to files. + * + * @param filename The filename prefix to use. Saves 4 files with that prefix. + * @return 0 on success, -1 on error + */ + EXPORT int saveState(std::string filename) throw(); + private: + explicit BlockResolutionManager(const BlockResolutionManager& brm); + BlockResolutionManager& operator=(const BlockResolutionManager& brm); + MasterSegmentTable mst; + ExtentMap em; + VBBM vbbm; + VSS vss; + CopyLocks copylocks; }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 3bf834188..2c25caf8b 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -40,199 +40,204 @@ namespace bi = boost::interprocess; namespace BRM { - -BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : - fKey(key), fSize(size), fReadOnly(readOnly) +BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : fKey(key), fSize(size), fReadOnly(readOnly) { - string keyName = ShmKeys::keyToName(fKey); + string keyName = ShmKeys::keyToName(fKey); - if (fSize == 0) - { - unsigned tries = 0; -again: - - try - { - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - off_t curSize = 0; -#ifdef _MSC_VER - bi::offset_t tmp = 0; - shm.get_size(tmp); - curSize = static_cast(tmp); -#else - shm.get_size(curSize); -#endif - - if (curSize == 0) throw - bi::interprocess_exception("shm size is zero"); - } - catch (bi::interprocess_exception&) - { - if (++tries > 10) - { - log("BRMShmImpl::BRMShmImpl(): retrying on size==0"); - throw; - } - - cerr << "BRMShmImpl::BRMShmImpl(): retrying on size==0" << endl; - usleep(500 * 1000); - goto again; - } - } + if (fSize == 0) + { + unsigned tries = 0; + again: try { - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); - idbassert(fSize > 0); - shm.truncate(fSize); - fShmobj.swap(shm); - } - catch (bi::interprocess_exception &b) - { - if (b.get_error_code() != bi::already_exists_error) { - ostringstream o; - o << "BRM caught an exception creating a shared memory segment: " << b.what(); - log(o.str()); - throw; - } - bi::shared_memory_object *shm = NULL; - try { - shm = new bi::shared_memory_object(bi::open_only, keyName.c_str(), bi::read_write); - } - catch (exception &e) { - ostringstream o; - o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << b.what(); - log(o.str()); - throw; - } - off_t curSize = 0; + bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); + off_t curSize = 0; #ifdef _MSC_VER - bi::offset_t tmp = 0; - shm->get_size(tmp); - curSize = static_cast(tmp); + bi::offset_t tmp = 0; + shm.get_size(tmp); + curSize = static_cast(tmp); #else - shm->get_size(curSize); + shm.get_size(curSize); #endif - idbassert(curSize > 0); - idbassert(curSize >= fSize); - fShmobj.swap(*shm); - delete shm; - fSize = curSize; - } - if (fReadOnly) - { - bi::mapped_region ro_region(fShmobj, bi::read_only); - fMapreg.swap(ro_region); + if (curSize == 0) + throw bi::interprocess_exception("shm size is zero"); } - else + catch (bi::interprocess_exception&) { - bi::mapped_region region(fShmobj, bi::read_write); - fMapreg.swap(region); + if (++tries > 10) + { + log("BRMShmImpl::BRMShmImpl(): retrying on size==0"); + throw; + } + + cerr << "BRMShmImpl::BRMShmImpl(): retrying on size==0" << endl; + usleep(500 * 1000); + goto again; } + } + + try + { + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); + idbassert(fSize > 0); + shm.truncate(fSize); + fShmobj.swap(shm); + } + catch (bi::interprocess_exception& b) + { + if (b.get_error_code() != bi::already_exists_error) + { + ostringstream o; + o << "BRM caught an exception creating a shared memory segment: " << b.what(); + log(o.str()); + throw; + } + bi::shared_memory_object* shm = NULL; + try + { + shm = new bi::shared_memory_object(bi::open_only, keyName.c_str(), bi::read_write); + } + catch (exception& e) + { + ostringstream o; + o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << b.what(); + log(o.str()); + throw; + } + off_t curSize = 0; +#ifdef _MSC_VER + bi::offset_t tmp = 0; + shm->get_size(tmp); + curSize = static_cast(tmp); +#else + shm->get_size(curSize); +#endif + idbassert(curSize > 0); + idbassert(curSize >= fSize); + fShmobj.swap(*shm); + delete shm; + fSize = curSize; + } + + if (fReadOnly) + { + bi::mapped_region ro_region(fShmobj, bi::read_only); + fMapreg.swap(ro_region); + } + else + { + bi::mapped_region region(fShmobj, bi::read_write); + fMapreg.swap(region); + } } int BRMShmImpl::grow(unsigned newKey, off_t newSize) { - idbassert(newKey != fKey); - idbassert(newSize >= fSize); + idbassert(newKey != fKey); + idbassert(newSize >= fSize); - string oldName = fShmobj.get_name(); + string oldName = fShmobj.get_name(); - string keyName = ShmKeys::keyToName(newKey); - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); - shm.truncate(newSize); + string keyName = ShmKeys::keyToName(newKey); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); + shm.truncate(newSize); - bi::mapped_region region(shm, bi::read_write); + bi::mapped_region region(shm, bi::read_write); - //Copy old data into new region - memcpy(region.get_address(), fMapreg.get_address(), fSize); - //clear new region - //make some versions of gcc happier... - memset(reinterpret_cast(reinterpret_cast(region.get_address()) + fSize), 0, newSize - fSize); + // Copy old data into new region + memcpy(region.get_address(), fMapreg.get_address(), fSize); + // clear new region + // make some versions of gcc happier... + memset(reinterpret_cast(reinterpret_cast(region.get_address()) + fSize), 0, + newSize - fSize); - fShmobj.swap(shm); - fMapreg.swap(region); + fShmobj.swap(shm); + fMapreg.swap(region); - if (!oldName.empty()) bi::shared_memory_object::remove(oldName.c_str()); + if (!oldName.empty()) + bi::shared_memory_object::remove(oldName.c_str()); - fKey = newKey; - fSize = newSize; + fKey = newKey; + fSize = newSize; - if (fReadOnly) - { - bi::mapped_region ro_region(fShmobj, bi::read_only); - fMapreg.swap(ro_region); - } + if (fReadOnly) + { + bi::mapped_region ro_region(fShmobj, bi::read_only); + fMapreg.swap(ro_region); + } - return 0; + return 0; } int BRMShmImpl::clear(unsigned newKey, off_t newSize) { - idbassert(newKey != fKey); + idbassert(newKey != fKey); - string oldName = fShmobj.get_name(); + string oldName = fShmobj.get_name(); - string keyName = ShmKeys::keyToName(newKey); - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); - shm.truncate(newSize); + string keyName = ShmKeys::keyToName(newKey); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); + shm.truncate(newSize); - bi::mapped_region region(shm, bi::read_write); + bi::mapped_region region(shm, bi::read_write); - //clear new region - memset(region.get_address(), 0, newSize); + // clear new region + memset(region.get_address(), 0, newSize); - fShmobj.swap(shm); - fMapreg.swap(region); + fShmobj.swap(shm); + fMapreg.swap(region); - if (!oldName.empty()) bi::shared_memory_object::remove(oldName.c_str()); + if (!oldName.empty()) + bi::shared_memory_object::remove(oldName.c_str()); - fKey = newKey; - fSize = newSize; + fKey = newKey; + fSize = newSize; - if (fReadOnly) - { - bi::mapped_region ro_region(fShmobj, bi::read_only); - fMapreg.swap(ro_region); - } + if (fReadOnly) + { + bi::mapped_region ro_region(fShmobj, bi::read_only); + fMapreg.swap(ro_region); + } - return 0; + return 0; } void BRMShmImpl::setReadOnly() { - if (fReadOnly) return; + if (fReadOnly) + return; - bi::mapped_region ro_region(fShmobj, bi::read_only); - fMapreg.swap(ro_region); + bi::mapped_region ro_region(fShmobj, bi::read_only); + fMapreg.swap(ro_region); - fReadOnly = true; + fReadOnly = true; } void BRMShmImpl::swap(BRMShmImpl& rhs) { - fShmobj.swap(rhs.fShmobj); - fMapreg.swap(rhs.fMapreg); - std::swap(fKey, rhs.fKey); - std::swap(fSize, rhs.fSize); - std::swap(fReadOnly, rhs.fReadOnly); + fShmobj.swap(rhs.fShmobj); + fMapreg.swap(rhs.fMapreg); + std::swap(fKey, rhs.fKey); + std::swap(fSize, rhs.fSize); + std::swap(fReadOnly, rhs.fReadOnly); } void BRMShmImpl::destroy() { - string oldName = fShmobj.get_name(); + string oldName = fShmobj.get_name(); - if (!oldName.empty()) bi::shared_memory_object::remove(oldName.c_str()); + if (!oldName.empty()) + bi::shared_memory_object::remove(oldName.c_str()); } -} //namespace +} // namespace BRM // vim:ts=4 sw=4: - diff --git a/versioning/BRM/brmshmimpl.h b/versioning/BRM/brmshmimpl.h index 5fd66c116..d2177afec 100644 --- a/versioning/BRM/brmshmimpl.h +++ b/versioning/BRM/brmshmimpl.h @@ -35,45 +35,46 @@ namespace BRM { - class BRMShmImpl { -public: - BRMShmImpl(unsigned key, off_t size, bool readOnly = false); - ~BRMShmImpl() { } + public: + BRMShmImpl(unsigned key, off_t size, bool readOnly = false); + ~BRMShmImpl() + { + } - inline unsigned key() const - { - return fKey; - } - inline off_t size() const - { - return fSize; - } - inline bool isReadOnly() const - { - return fReadOnly; - } + inline unsigned key() const + { + return fKey; + } + inline off_t size() const + { + return fSize; + } + inline bool isReadOnly() const + { + return fReadOnly; + } - void setReadOnly(); - int grow(unsigned newKey, off_t newSize); - int clear(unsigned newKey, off_t newSize); + void setReadOnly(); + int grow(unsigned newKey, off_t newSize); + int clear(unsigned newKey, off_t newSize); - void swap(BRMShmImpl& rhs); - void destroy(); + void swap(BRMShmImpl& rhs); + void destroy(); - boost::interprocess::shared_memory_object fShmobj; - boost::interprocess::mapped_region fMapreg; + boost::interprocess::shared_memory_object fShmobj; + boost::interprocess::mapped_region fMapreg; -private: - BRMShmImpl(const BRMShmImpl& rhs); - BRMShmImpl& operator=(const BRMShmImpl& rhs); + private: + BRMShmImpl(const BRMShmImpl& rhs); + BRMShmImpl& operator=(const BRMShmImpl& rhs); - unsigned fKey; - off_t fSize; - bool fReadOnly; + unsigned fKey; + off_t fSize; + bool fReadOnly; }; -} //namespace +} // namespace BRM #endif diff --git a/versioning/BRM/brmtypes.cpp b/versioning/BRM/brmtypes.cpp index 848cecdb1..3992b5c61 100644 --- a/versioning/BRM/brmtypes.cpp +++ b/versioning/BRM/brmtypes.cpp @@ -53,29 +53,27 @@ using namespace idbdatafile; // local unnamed namespace namespace { -unsigned int subSystemLoggingId = - (unsigned int)BRM::SubSystemLogId_controllerNode; +unsigned int subSystemLoggingId = (unsigned int)BRM::SubSystemLogId_controllerNode; } namespace BRM { - LBIDRange::LBIDRange() { - start = 0; - size = 0; + start = 0; + size = 0; } LBIDRange::LBIDRange(const LBIDRange& l) { - start = l.start; - size = l.size; + start = l.start; + size = l.size; } LBIDRange::LBIDRange(const InlineLBIDRange& l) { - start = l.start; - size = l.size; + start = l.start; + size = l.size; } LBIDRange::~LBIDRange() @@ -84,16 +82,16 @@ LBIDRange::~LBIDRange() LBIDRange& LBIDRange::operator=(const LBIDRange& l) { - start = l.start; - size = l.size; - return *this; + start = l.start; + size = l.size; + return *this; } LBIDRange& LBIDRange::operator=(const InlineLBIDRange& l) { - start = l.start; - size = l.size; - return *this; + start = l.start; + size = l.size; + return *this; } // this value should be impossible in version 1 @@ -105,184 +103,162 @@ LBIDRange& LBIDRange::operator=(const InlineLBIDRange& l) void LBIDRange::deserialize(ByteStream& bs) { - uint64_t tmp; + uint64_t tmp; + bs >> tmp; + + if (tmp == RANGE_V2_MAGIC) + { bs >> tmp; - - if (tmp == RANGE_V2_MAGIC) - { - bs >> tmp; - start = tmp; - bs >> size; - } - else - { - // version 1 - size = (tmp & 0xffffffff00000000ULL) >> 32; - start = (tmp & 0x00000000ffffffffULL); - } + start = tmp; + bs >> size; + } + else + { + // version 1 + size = (tmp & 0xffffffff00000000ULL) >> 32; + start = (tmp & 0x00000000ffffffffULL); + } } void LBIDRange::serialize(ByteStream& bs) const { - bs << (uint64_t) RANGE_V2_MAGIC; - bs << (uint64_t) start; - bs << size; + bs << (uint64_t)RANGE_V2_MAGIC; + bs << (uint64_t)start; + bs << size; } _TxnID::_TxnID() { - id = 0; - valid = false; + id = 0; + valid = false; } _SIDTIDEntry::_SIDTIDEntry() { - init(); + init(); } void _SIDTIDEntry::init() { - sessionid = 0; - txnid.id = 0; - txnid.valid = false; + sessionid = 0; + txnid.id = 0; + txnid.valid = false; } VBRange::VBRange() { - vbOID = 0; - vbFBO = 0; - size = 0; + vbOID = 0; + vbFBO = 0; + size = 0; } VBRange::VBRange(const VBRange& v) { - vbOID = v.vbOID; - vbFBO = v.vbFBO; - size = v.size; + vbOID = v.vbOID; + vbFBO = v.vbFBO; + size = v.size; } VBRange::~VBRange() { } -VBRange& VBRange::operator= (const VBRange& v) +VBRange& VBRange::operator=(const VBRange& v) { - vbOID = v.vbOID; - vbFBO = v.vbFBO; - size = v.size; - return *this; + vbOID = v.vbOID; + vbFBO = v.vbFBO; + size = v.size; + return *this; } void VBRange::deserialize(ByteStream& bs) { - uint32_t tmp; + uint32_t tmp; - bs >> tmp; - vbOID = tmp; - bs >> vbFBO; - bs >> size; + bs >> tmp; + vbOID = tmp; + bs >> vbFBO; + bs >> size; } void VBRange::serialize(ByteStream& bs) const { - bs << (uint32_t) vbOID; - bs << vbFBO; - bs << size; + bs << (uint32_t)vbOID; + bs << vbFBO; + bs << size; } -void logInit ( SubSystemLogId subSystemId ) +void logInit(SubSystemLogId subSystemId) { - subSystemLoggingId = (unsigned int)subSystemId; + subSystemLoggingId = (unsigned int)subSystemId; } void log(const string& msg, logging::LOG_TYPE level) { - logging::MessageLog logger((logging::LoggingID(subSystemLoggingId))); - logging::Message message; - logging::Message::Args args; + logging::MessageLog logger((logging::LoggingID(subSystemLoggingId))); + logging::Message message; + logging::Message::Args args; - args.add(msg); - message.format(args); + args.add(msg); + message.format(args); - switch (level) - { - case logging::LOG_TYPE_DEBUG: - logger.logDebugMessage(message); - break; + switch (level) + { + case logging::LOG_TYPE_DEBUG: logger.logDebugMessage(message); break; - case logging::LOG_TYPE_INFO: - logger.logInfoMessage(message); - break; + case logging::LOG_TYPE_INFO: logger.logInfoMessage(message); break; - case logging::LOG_TYPE_WARNING: - logger.logWarningMessage(message); - break; + case logging::LOG_TYPE_WARNING: logger.logWarningMessage(message); break; - case logging::LOG_TYPE_ERROR: - logger.logErrorMessage(message); - break; + case logging::LOG_TYPE_ERROR: logger.logErrorMessage(message); break; - case logging::LOG_TYPE_CRITICAL: - logger.logCriticalMessage(message); - break; + case logging::LOG_TYPE_CRITICAL: logger.logCriticalMessage(message); break; - default: - logger.logInfoMessage(message); - } + default: logger.logInfoMessage(message); + } } void log_errno(const string& msg, logging::LOG_TYPE level) { - int tmp = errno; - char test[1000]; - logging::MessageLog logger((logging::LoggingID(subSystemLoggingId))); - logging::Message message; - logging::Message::Args args; + int tmp = errno; + char test[1000]; + logging::MessageLog logger((logging::LoggingID(subSystemLoggingId))); + logging::Message message; + logging::Message::Args args; - args.add(msg + ": "); + args.add(msg + ": "); #if STRERROR_R_CHAR_P - const char* cMsg; - cMsg = strerror_r(tmp, &test[0], 1000); + const char* cMsg; + cMsg = strerror_r(tmp, &test[0], 1000); - if (cMsg == NULL) - args.add(string("strerror failed")); - else - args.add(string(cMsg)); + if (cMsg == NULL) + args.add(string("strerror failed")); + else + args.add(string(cMsg)); #else - int cMsg; - cMsg = strerror_r(tmp, &test[0], 1000); - args.add(string(test)); + int cMsg; + cMsg = strerror_r(tmp, &test[0], 1000); + args.add(string(test)); #endif - message.format(args); + message.format(args); - switch (level) - { - case logging::LOG_TYPE_DEBUG: - logger.logDebugMessage(message); - break; + switch (level) + { + case logging::LOG_TYPE_DEBUG: logger.logDebugMessage(message); break; - case logging::LOG_TYPE_INFO: - logger.logInfoMessage(message); - break; + case logging::LOG_TYPE_INFO: logger.logInfoMessage(message); break; - case logging::LOG_TYPE_WARNING: - logger.logWarningMessage(message); - break; + case logging::LOG_TYPE_WARNING: logger.logWarningMessage(message); break; - case logging::LOG_TYPE_ERROR: - logger.logErrorMessage(message); - break; + case logging::LOG_TYPE_ERROR: logger.logErrorMessage(message); break; - case logging::LOG_TYPE_CRITICAL: - logger.logCriticalMessage(message); - break; + case logging::LOG_TYPE_CRITICAL: logger.logCriticalMessage(message); break; - default: - logger.logInfoMessage(message); - } + default: logger.logInfoMessage(message); + } } //------------------------------------------------------------------------------ @@ -290,245 +266,245 @@ void log_errno(const string& msg, logging::LOG_TYPE level) //------------------------------------------------------------------------------ void errString(int rc, string& errMsg) { - switch (rc) + switch (rc) + { + case ERR_OK: { - case ERR_OK: - { - errMsg = "OKAY"; - break; - } - - case ERR_FAILURE: - { - errMsg = "FAILED"; - break; - } - - case ERR_SLAVE_INCONSISTENCY: - { - errMsg = "image inconsistency"; - break; - } - - case ERR_NETWORK: - { - errMsg = "network error"; - break; - } - - case ERR_TIMEOUT: - { - errMsg = "network timeout"; - break; - } - - case ERR_READONLY: - { - errMsg = "DBRM is in READ-ONLY mode"; - break; - } - - case ERR_DEADLOCK: - { - errMsg = "deadlock reserving LBID range"; - break; - } - - case ERR_KILLED: - { - errMsg = "killed reserving LBID range"; - break; - } - - case ERR_VBBM_OVERFLOW: - { - errMsg = "VBBM overflow"; - break; - } - - case ERR_TABLE_LOCKED_ALREADY: - { - errMsg = "table already locked"; - break; - } - - case ERR_INVALID_OP_LAST_PARTITION: - { - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_LAST_PARTITION); - break; - } - - case ERR_PARTITION_DISABLED: - { - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_DISABLED); - break; - } - - case ERR_NOT_EXIST_PARTITION: - { - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST); - break; - } - - case ERR_PARTITION_ENABLED: - { - errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_ENABLED); - break; - } - - case ERR_OLDTXN_OVERWRITING_NEWTXN: - { - errMsg = "A newer transaction has already written to the same block(s)"; - break; - } - - default: - { - ostringstream oss; - oss << "UNKNOWN (" << rc << ")"; - errMsg = oss.str(); - break; - } + errMsg = "OKAY"; + break; } + + case ERR_FAILURE: + { + errMsg = "FAILED"; + break; + } + + case ERR_SLAVE_INCONSISTENCY: + { + errMsg = "image inconsistency"; + break; + } + + case ERR_NETWORK: + { + errMsg = "network error"; + break; + } + + case ERR_TIMEOUT: + { + errMsg = "network timeout"; + break; + } + + case ERR_READONLY: + { + errMsg = "DBRM is in READ-ONLY mode"; + break; + } + + case ERR_DEADLOCK: + { + errMsg = "deadlock reserving LBID range"; + break; + } + + case ERR_KILLED: + { + errMsg = "killed reserving LBID range"; + break; + } + + case ERR_VBBM_OVERFLOW: + { + errMsg = "VBBM overflow"; + break; + } + + case ERR_TABLE_LOCKED_ALREADY: + { + errMsg = "table already locked"; + break; + } + + case ERR_INVALID_OP_LAST_PARTITION: + { + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_LAST_PARTITION); + break; + } + + case ERR_PARTITION_DISABLED: + { + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_DISABLED); + break; + } + + case ERR_NOT_EXIST_PARTITION: + { + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST); + break; + } + + case ERR_PARTITION_ENABLED: + { + errMsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_ENABLED); + break; + } + + case ERR_OLDTXN_OVERWRITING_NEWTXN: + { + errMsg = "A newer transaction has already written to the same block(s)"; + break; + } + + default: + { + ostringstream oss; + oss << "UNKNOWN (" << rc << ")"; + errMsg = oss.str(); + break; + } + } } /* Tablelock impl */ bool TableLockInfo::overlaps(const TableLockInfo& t, const std::set& sDbrootList) const { - if (tableOID != t.tableOID) - return false; - - for (uint32_t i = 0; i < dbrootList.size(); i++) - if (sDbrootList.find(dbrootList[i]) != sDbrootList.end()) - return true; - + if (tableOID != t.tableOID) return false; + + for (uint32_t i = 0; i < dbrootList.size(); i++) + if (sDbrootList.find(dbrootList[i]) != sDbrootList.end()) + return true; + + return false; } void TableLockInfo::serialize(ByteStream& bs) const { - bs << id << tableOID << ownerName << ownerPID << (uint32_t) ownerSessionID << - (uint32_t) ownerTxnID << (uint8_t) state; - bs << (uint64_t) creationTime; - serializeInlineVector(bs, dbrootList); + bs << id << tableOID << ownerName << ownerPID << (uint32_t)ownerSessionID << (uint32_t)ownerTxnID + << (uint8_t)state; + bs << (uint64_t)creationTime; + serializeInlineVector(bs, dbrootList); } void TableLockInfo::deserialize(ByteStream& bs) { - uint8_t tmp8; - uint32_t tmp321, tmp322; - uint64_t tmp64; - bs >> id >> tableOID >> ownerName >> ownerPID >> tmp321 >> tmp322 >> tmp8 >> tmp64; - ownerSessionID = tmp321; - ownerTxnID = tmp322; - state = (LockState) tmp8; - creationTime = tmp64; - deserializeInlineVector(bs, dbrootList); + uint8_t tmp8; + uint32_t tmp321, tmp322; + uint64_t tmp64; + bs >> id >> tableOID >> ownerName >> ownerPID >> tmp321 >> tmp322 >> tmp8 >> tmp64; + ownerSessionID = tmp321; + ownerTxnID = tmp322; + state = (LockState)tmp8; + creationTime = tmp64; + deserializeInlineVector(bs, dbrootList); } void TableLockInfo::serialize(ostream& o) const { - uint16_t nameLen = ownerName.length(); - uint16_t dbrootListSize = dbrootList.size(); + uint16_t nameLen = ownerName.length(); + uint16_t dbrootListSize = dbrootList.size(); - o.write((char*) &id, 8); - o.write((char*) &tableOID, 4); - o.write((char*) &ownerPID, 4); - o.write((char*) &state, 4); - o.write((char*) &ownerSessionID, 4); - o.write((char*) &ownerTxnID, 4); - o.write((char*) &creationTime, sizeof(time_t)); - o.write((char*) &nameLen, 2); - o.write((char*) ownerName.c_str(), nameLen); - o.write((char*) &dbrootListSize, 2); + o.write((char*)&id, 8); + o.write((char*)&tableOID, 4); + o.write((char*)&ownerPID, 4); + o.write((char*)&state, 4); + o.write((char*)&ownerSessionID, 4); + o.write((char*)&ownerTxnID, 4); + o.write((char*)&creationTime, sizeof(time_t)); + o.write((char*)&nameLen, 2); + o.write((char*)ownerName.c_str(), nameLen); + o.write((char*)&dbrootListSize, 2); - for (uint32_t j = 0; j < dbrootListSize; j++) - o.write((char*) &dbrootList[j], 4); + for (uint32_t j = 0; j < dbrootListSize; j++) + o.write((char*)&dbrootList[j], 4); } void TableLockInfo::deserialize(istream& i) { - uint16_t nameLen; - uint16_t dbrootListSize; - boost::scoped_array buf; + uint16_t nameLen; + uint16_t dbrootListSize; + boost::scoped_array buf; - i.read((char*) &id, 8); - i.read((char*) &tableOID, 4); - i.read((char*) &ownerPID, 4); - i.read((char*) &state, 4); - i.read((char*) &ownerSessionID, 4); - i.read((char*) &ownerTxnID, 4); - i.read((char*) &creationTime, sizeof(time_t)); - i.read((char*) &nameLen, 2); - buf.reset(new char[nameLen]); - i.read(buf.get(), nameLen); - ownerName = string(buf.get(), nameLen); - i.read((char*) &dbrootListSize, 2); - dbrootList.resize(dbrootListSize); + i.read((char*)&id, 8); + i.read((char*)&tableOID, 4); + i.read((char*)&ownerPID, 4); + i.read((char*)&state, 4); + i.read((char*)&ownerSessionID, 4); + i.read((char*)&ownerTxnID, 4); + i.read((char*)&creationTime, sizeof(time_t)); + i.read((char*)&nameLen, 2); + buf.reset(new char[nameLen]); + i.read(buf.get(), nameLen); + ownerName = string(buf.get(), nameLen); + i.read((char*)&dbrootListSize, 2); + dbrootList.resize(dbrootListSize); - for (uint32_t j = 0; j < dbrootListSize; j++) - i.read((char*) &dbrootList[j], 4); + for (uint32_t j = 0; j < dbrootListSize; j++) + i.read((char*)&dbrootList[j], 4); } void TableLockInfo::serialize(IDBDataFile* o) const { - uint16_t nameLen = ownerName.length(); - uint16_t dbrootListSize = dbrootList.size(); + uint16_t nameLen = ownerName.length(); + uint16_t dbrootListSize = dbrootList.size(); - o->write((char*) &id, 8); - o->write((char*) &tableOID, 4); - o->write((char*) &ownerPID, 4); - o->write((char*) &state, 4); - o->write((char*) &ownerSessionID, 4); - o->write((char*) &ownerTxnID, 4); - o->write((char*) &creationTime, sizeof(time_t)); - o->write((char*) &nameLen, 2); - o->write((char*) ownerName.c_str(), nameLen); - o->write((char*) &dbrootListSize, 2); + o->write((char*)&id, 8); + o->write((char*)&tableOID, 4); + o->write((char*)&ownerPID, 4); + o->write((char*)&state, 4); + o->write((char*)&ownerSessionID, 4); + o->write((char*)&ownerTxnID, 4); + o->write((char*)&creationTime, sizeof(time_t)); + o->write((char*)&nameLen, 2); + o->write((char*)ownerName.c_str(), nameLen); + o->write((char*)&dbrootListSize, 2); - for (uint32_t j = 0; j < dbrootListSize; j++) - o->write((char*) &dbrootList[j], 4); + for (uint32_t j = 0; j < dbrootListSize; j++) + o->write((char*)&dbrootList[j], 4); } void TableLockInfo::deserialize(IDBDataFile* i) { - uint16_t nameLen; - uint16_t dbrootListSize; - boost::scoped_array buf; + uint16_t nameLen; + uint16_t dbrootListSize; + boost::scoped_array buf; - i->read((char*) &id, 8); - i->read((char*) &tableOID, 4); - i->read((char*) &ownerPID, 4); - i->read((char*) &state, 4); - i->read((char*) &ownerSessionID, 4); - i->read((char*) &ownerTxnID, 4); - i->read((char*) &creationTime, sizeof(time_t)); - i->read((char*) &nameLen, 2); - buf.reset(new char[nameLen]); - i->read(buf.get(), nameLen); - ownerName = string(buf.get(), nameLen); - i->read((char*) &dbrootListSize, 2); - dbrootList.resize(dbrootListSize); + i->read((char*)&id, 8); + i->read((char*)&tableOID, 4); + i->read((char*)&ownerPID, 4); + i->read((char*)&state, 4); + i->read((char*)&ownerSessionID, 4); + i->read((char*)&ownerTxnID, 4); + i->read((char*)&creationTime, sizeof(time_t)); + i->read((char*)&nameLen, 2); + buf.reset(new char[nameLen]); + i->read(buf.get(), nameLen); + ownerName = string(buf.get(), nameLen); + i->read((char*)&dbrootListSize, 2); + dbrootList.resize(dbrootListSize); - for (uint32_t j = 0; j < dbrootListSize; j++) - i->read((char*) &dbrootList[j], 4); + for (uint32_t j = 0; j < dbrootListSize; j++) + i->read((char*)&dbrootList[j], 4); } bool TableLockInfo::operator<(const TableLockInfo& tli) const { - return (id < tli.id); + return (id < tli.id); } ostream& operator<<(ostream& os, const QueryContext& qc) { - os << " SCN: " << qc.currentScn << endl; - os << " Txns: "; + os << " SCN: " << qc.currentScn << endl; + os << " Txns: "; - for (uint32_t i = 0; i < qc.currentTxns->size(); i++) - os << (*qc.currentTxns)[i] << " "; + for (uint32_t i = 0; i < qc.currentTxns->size(); i++) + os << (*qc.currentTxns)[i] << " "; - return os; + return os; } -} //namespace +} // namespace BRM diff --git a/versioning/BRM/brmtypes.h b/versioning/BRM/brmtypes.h index 93abec9b9..dfc888391 100644 --- a/versioning/BRM/brmtypes.h +++ b/versioning/BRM/brmtypes.h @@ -47,25 +47,23 @@ namespace std { namespace tr1 { -template<> -struct hash - : public std::unary_function +template <> +struct hash : public std::unary_function { - std::size_t - operator()(long long int val) const - { - return static_cast(val); - } + std::size_t operator()(long long int val) const + { + return static_cast(val); + } }; -} -} -#endif //if __GNUC__ +} // namespace tr1 +} // namespace std +#endif // if __GNUC__ -#endif //if !__LP64__ -#endif //_UNORDERED_MAP_FIX_ +#endif // if !__LP64__ +#endif //_UNORDERED_MAP_FIX_ #else #include -#endif //_MSC_VER +#endif //_MSC_VER #include "calpontsystemcatalog.h" #include "bytestream.h" @@ -84,12 +82,10 @@ namespace idbdatafile class IDBDataFile; } - namespace BRM { - /* these types should be defined in the system catalog header */ -typedef int64_t LBID_t; /// The LBID space is currently 36 bits. We use 64 here. +typedef int64_t LBID_t; /// The LBID space is currently 36 bits. We use 64 here. typedef uint32_t HWM_t; typedef int32_t VER_t; /// Object ID type. Currently a 32-bit number. 24 for Object number, 8 for partition number. @@ -105,10 +101,10 @@ to inherit Serializeable, hence the all-public definitions */ /// The InlineLBIDRange struct is for use internally by the ExtentMap struct InlineLBIDRange { - LBID_t start; - uint32_t size; + LBID_t start; + uint32_t size; #ifndef __LP64__ - int32_t pad1; + int32_t pad1; #endif }; @@ -119,11 +115,11 @@ typedef uint32_t SID; /** @brief A type describing a single transaction ID */ struct _TxnID { - /// The TransactionID number - execplan::CalpontSystemCatalog::SCN id; - /// True iff the id is valid. - bool valid; - EXPORT _TxnID(); + /// The TransactionID number + execplan::CalpontSystemCatalog::SCN id; + /// True iff the id is valid. + bool valid; + EXPORT _TxnID(); }; /** @brief A type describing a single transaction ID */ typedef struct _TxnID TxnID; @@ -132,63 +128,62 @@ typedef struct _TxnID TxnID; */ struct _SIDTIDEntry { - /// The Transaction ID. txnid.valid determines whether or not this SIDTIDEntry is valid - TxnID txnid; - /// The session doing the transaction - SID sessionid; - EXPORT _SIDTIDEntry(); - EXPORT void init(); + /// The Transaction ID. txnid.valid determines whether or not this SIDTIDEntry is valid + TxnID txnid; + /// The session doing the transaction + SID sessionid; + EXPORT _SIDTIDEntry(); + EXPORT void init(); }; /** @brief A type associating a session with a transaction */ typedef struct _SIDTIDEntry SIDTIDEntry; -// @bug 1970 - Added CPInfo and CPMaxMin structs used by new interface that allows setting the max and min CP data -// for multiple extents. +// @bug 1970 - Added CPInfo and CPMaxMin structs used by new interface that allows setting the max and min CP +// data for multiple extents. // Special seqNum field values. -#define SEQNUM_MARK_INVALID (-1) -#define SEQNUM_MARK_INVALID_SET_RANGE (-2) +#define SEQNUM_MARK_INVALID (-1) +#define SEQNUM_MARK_INVALID_SET_RANGE (-2) #define SEQNUM_MARK_UPDATING_INVALID_SET_RANGE (-3) // Used in vectors. struct CPInfo { - LBID_t firstLbid; - int64_t max; - int64_t min; - int32_t seqNum; - union - { - int128_t bigMax; - int64_t max_; - }; - union - { - int128_t bigMin; - int64_t min_; - }; - bool isBinaryColumn; // XXX: we should remove these two fields and replace it with type handler. + LBID_t firstLbid; + int64_t max; + int64_t min; + int32_t seqNum; + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; + bool isBinaryColumn; // XXX: we should remove these two fields and replace it with type handler. }; typedef std::vector CPInfoList_t; // Used for map where lbid is the key. struct CPMaxMin { - int64_t max; - int64_t min; - int32_t seqNum; - union - { - int128_t bigMax; - int64_t max_; - }; - union - { - int128_t bigMin; - int64_t min_; - }; - bool isBinaryColumn; // XXX: these two fields should be replaced with type handler pointer. - + int64_t max; + int64_t min; + int32_t seqNum; + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; + bool isBinaryColumn; // XXX: these two fields should be replaced with type handler pointer. }; typedef std::tr1::unordered_map CPMaxMinMap_t; @@ -196,23 +191,23 @@ typedef std::tr1::unordered_map CPMaxMinMap_t; struct CPInfoMerge { - LBID_t startLbid; // starting LBID for relevant extent - int64_t max; // max value to be merged with current max value - int64_t min; // min value to be merged with current min value - int32_t seqNum; // sequence number (not currently used) - execplan::CalpontSystemCatalog::ColDataType type; - int32_t colWidth; - bool newExtent; // is this to be treated as a new extent - union - { - int128_t bigMax; - int64_t max_; - }; - union - { - int128_t bigMin; - int64_t min_; - }; + LBID_t startLbid; // starting LBID for relevant extent + int64_t max; // max value to be merged with current max value + int64_t min; // min value to be merged with current min value + int32_t seqNum; // sequence number (not currently used) + execplan::CalpontSystemCatalog::ColDataType type; + int32_t colWidth; + bool newExtent; // is this to be treated as a new extent + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; }; typedef std::vector CPInfoMergeList_t; @@ -220,144 +215,145 @@ typedef std::vector CPInfoMergeList_t; // those in CPInfoMerge. struct CPMaxMinMerge { - int64_t max; - int64_t min; - int32_t seqNum; - execplan::CalpontSystemCatalog::ColDataType type; - int32_t colWidth; - bool newExtent; - union - { - int128_t bigMax; - int64_t max_; - }; - union - { - int128_t bigMin; - int64_t min_; - }; + int64_t max; + int64_t min; + int32_t seqNum; + execplan::CalpontSystemCatalog::ColDataType type; + int32_t colWidth; + bool newExtent; + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; }; typedef std::tr1::unordered_map CPMaxMinMergeMap_t; - // end of bug 2117 -typedef std::tr1::unordered_map OidsMap_t; +typedef std::tr1::unordered_map + OidsMap_t; struct ExtentInfo { - execplan::CalpontSystemCatalog::OID oid; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml - HWM_t hwm; - bool newFile; + execplan::CalpontSystemCatalog::OID oid; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + HWM_t hwm; + bool newFile; }; struct FileInfo { - execplan::CalpontSystemCatalog::OID oid; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml - uint16_t compType; // compression type + execplan::CalpontSystemCatalog::OID oid; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + uint16_t compType; // compression type }; typedef std::tr1::unordered_map ExtentsInfoMap_t; enum LockState { - LOADING, - CLEANUP + LOADING, + CLEANUP }; struct TableLockInfo : public messageqcpp::Serializeable { - uint64_t id; - uint32_t tableOID; - std::string ownerName; - uint32_t ownerPID; - int32_t ownerSessionID; - int32_t ownerTxnID; - LockState state; - time_t creationTime; - std::vector dbrootList; + uint64_t id; + uint32_t tableOID; + std::string ownerName; + uint32_t ownerPID; + int32_t ownerSessionID; + int32_t ownerTxnID; + LockState state; + time_t creationTime; + std::vector dbrootList; - bool overlaps(const TableLockInfo&, const std::set& sPMList) const; - EXPORT void serialize(messageqcpp::ByteStream& bs) const; - EXPORT void serialize(std::ostream&) const; - EXPORT void deserialize(std::istream&); - EXPORT void deserialize(messageqcpp::ByteStream& bs); - EXPORT void serialize(idbdatafile::IDBDataFile*) const; - EXPORT void deserialize(idbdatafile::IDBDataFile*); - bool operator<(const TableLockInfo&) const; + bool overlaps(const TableLockInfo&, const std::set& sPMList) const; + EXPORT void serialize(messageqcpp::ByteStream& bs) const; + EXPORT void serialize(std::ostream&) const; + EXPORT void deserialize(std::istream&); + EXPORT void deserialize(messageqcpp::ByteStream& bs); + EXPORT void serialize(idbdatafile::IDBDataFile*) const; + EXPORT void deserialize(idbdatafile::IDBDataFile*); + bool operator<(const TableLockInfo&) const; }; /// A Serializeable version of InlineLBIDRange class LBIDRange : public messageqcpp::Serializeable { + public: + LBID_t start; + uint32_t size; -public: - LBID_t start; - uint32_t size; + EXPORT LBIDRange(); + EXPORT LBIDRange(const LBIDRange& l); + EXPORT LBIDRange(const InlineLBIDRange& l); + EXPORT LBIDRange& operator=(const LBIDRange& l); + EXPORT LBIDRange& operator=(const InlineLBIDRange& l); + EXPORT virtual ~LBIDRange(); - EXPORT LBIDRange(); - EXPORT LBIDRange(const LBIDRange& l); - EXPORT LBIDRange(const InlineLBIDRange& l); - EXPORT LBIDRange& operator=(const LBIDRange& l); - EXPORT LBIDRange& operator=(const InlineLBIDRange& l); - EXPORT virtual ~LBIDRange(); - - /** The Serializeable interface. Exports the instance to the bytestream */ - EXPORT virtual void serialize(messageqcpp::ByteStream& bs) const; - /** The Serializeable interface. Initializes itself from the bytestrem. */ - EXPORT virtual void deserialize(messageqcpp::ByteStream& bs); + /** The Serializeable interface. Exports the instance to the bytestream */ + EXPORT virtual void serialize(messageqcpp::ByteStream& bs) const; + /** The Serializeable interface. Initializes itself from the bytestrem. */ + EXPORT virtual void deserialize(messageqcpp::ByteStream& bs); }; /* To support bulkVSSLookup() */ struct VSSData { - VER_t verID; - bool vbFlag; - int returnCode; + VER_t verID; + bool vbFlag; + int returnCode; }; /* Arg type for DBRM::bulkSetHWM() */ struct BulkSetHWMArg { - OID_t oid; - uint32_t partNum; - uint16_t segNum; - HWM_t hwm; + OID_t oid; + uint32_t partNum; + uint16_t segNum; + HWM_t hwm; }; /* Arg type for DBRM::bulkUpdateDBRoot() */ struct BulkUpdateDBRootArg { - LBID_t startLBID; // starting LBID for the extent to update - uint16_t dbRoot; // the new dbRoot + LBID_t startLBID; // starting LBID for the extent to update + uint16_t dbRoot; // the new dbRoot - inline bool operator<(const BulkUpdateDBRootArg& b) const - { - return startLBID < b.startLBID; - } - BulkUpdateDBRootArg(LBID_t l = 0, uint16_t d = 0) : startLBID(l), dbRoot(d) {} + inline bool operator<(const BulkUpdateDBRootArg& b) const + { + return startLBID < b.startLBID; + } + BulkUpdateDBRootArg(LBID_t l = 0, uint16_t d = 0) : startLBID(l), dbRoot(d) + { + } }; /* Input Arg type for DBRM::createStripeColumnExtents() */ struct CreateStripeColumnExtentsArgIn { - OID_t oid; // column OID - uint32_t width; // column width in bytes - execplan::CalpontSystemCatalog::ColDataType colDataType; + OID_t oid; // column OID + uint32_t width; // column width in bytes + execplan::CalpontSystemCatalog::ColDataType colDataType; }; /* Output Arg type for DBRM:createStripeColumnExtents() */ struct CreateStripeColumnExtentsArgOut { - LBID_t startLbid; // starting LBID of allocated extent - int allocSize; // number of blocks in allocated extent - uint32_t startBlkOffset; // starting file block offset for allocated extent + LBID_t startLbid; // starting LBID of allocated extent + int allocSize; // number of blocks in allocated extent + uint32_t startBlkOffset; // starting file block offset for allocated extent }; /// A container for LBIDRanges @@ -366,55 +362,54 @@ typedef std::vector LBIDRange_v; /// Describes a contiguous range of blocks in the Version Buffer class VBRange : public messageqcpp::Serializeable { + public: + OID_t vbOID; + uint32_t vbFBO; + uint32_t size; -public: - OID_t vbOID; - uint32_t vbFBO; - uint32_t size; - - EXPORT VBRange(); - EXPORT VBRange(const VBRange& v); - EXPORT VBRange& operator= (const VBRange& v); - EXPORT virtual ~VBRange(); - EXPORT virtual void serialize(messageqcpp::ByteStream& bs) const; - EXPORT virtual void deserialize(messageqcpp::ByteStream& bs); + EXPORT VBRange(); + EXPORT VBRange(const VBRange& v); + EXPORT VBRange& operator=(const VBRange& v); + EXPORT virtual ~VBRange(); + EXPORT virtual void serialize(messageqcpp::ByteStream& bs) const; + EXPORT virtual void deserialize(messageqcpp::ByteStream& bs); }; // Structure used to return HWM information for each DbRoot in a PM struct EmDbRootHWMInfo { - uint32_t partitionNum; // last partition in dbRoot - uint16_t dbRoot; // applicable dbRoot - uint16_t segmentNum; // last segment file in dbRoot - HWM_t localHWM; // local HWM in last file for this dbRoot - uint32_t fbo; // starting block offset to HWM extent - LBID_t startLbid; // starting LBID for HWM extent - uint64_t totalBlocks; // cumulative non-outOfService blks for this dbRoot. - // 0 block count means no extents in this dbRoot, - // unless status is OutOfService; in which case - // the dbRoot has blocks that are all OutOfService - int hwmExtentIndex;//Internal use (idx to HWM extent in extent map) - int16_t status; // Avail, unAvail, outOfService - EmDbRootHWMInfo() - { - init(0); - } - EmDbRootHWMInfo(uint16_t root) - { - init(root); - } - void init (uint16_t root) - { - partitionNum = 0; - dbRoot = root; - segmentNum = 0; - localHWM = 0; - fbo = 0; - startLbid = 0; - hwmExtentIndex = -1; - totalBlocks = 0; - status = 0; - } + uint32_t partitionNum; // last partition in dbRoot + uint16_t dbRoot; // applicable dbRoot + uint16_t segmentNum; // last segment file in dbRoot + HWM_t localHWM; // local HWM in last file for this dbRoot + uint32_t fbo; // starting block offset to HWM extent + LBID_t startLbid; // starting LBID for HWM extent + uint64_t totalBlocks; // cumulative non-outOfService blks for this dbRoot. + // 0 block count means no extents in this dbRoot, + // unless status is OutOfService; in which case + // the dbRoot has blocks that are all OutOfService + int hwmExtentIndex; // Internal use (idx to HWM extent in extent map) + int16_t status; // Avail, unAvail, outOfService + EmDbRootHWMInfo() + { + init(0); + } + EmDbRootHWMInfo(uint16_t root) + { + init(root); + } + void init(uint16_t root) + { + partitionNum = 0; + dbRoot = root; + segmentNum = 0; + localHWM = 0; + fbo = 0; + startLbid = 0; + hwmExtentIndex = -1; + totalBlocks = 0; + status = 0; + } }; typedef std::vector EmDbRootHWMInfo_v; @@ -439,35 +434,35 @@ typedef std::vector VBRange_v; */ struct ImageDelta { - void* start; - int size; - char data[ID_MAXSIZE]; /// Has to be as large as the largest change + void* start; + int size; + char data[ID_MAXSIZE]; /// Has to be as large as the largest change }; // SubSystemLogId enumeration values should be in sync with SubsystemID[] // that is defined in messagelog.cpp enum SubSystemLogId { - SubSystemLogId_controllerNode = 29, - SubSystemLogId_workerNode = 30 + SubSystemLogId_controllerNode = 29, + SubSystemLogId_workerNode = 30 }; -EXPORT void logInit ( SubSystemLogId subSystemId ); +EXPORT void logInit(SubSystemLogId subSystemId); EXPORT void log(const std::string& msg, logging::LOG_TYPE = logging::LOG_TYPE_CRITICAL); EXPORT void log_errno(const std::string& msg, logging::LOG_TYPE = logging::LOG_TYPE_CRITICAL); -EXPORT void errString( int rc, std::string& errMsg ); +EXPORT void errString(int rc, std::string& errMsg); const struct timespec FIVE_MIN_TIMEOUT = {300, 0}; /* Function identifiers used for master-slave communication. - The format of the messages from DBRM to Master and from Master to Slaves - will look like - - The format of the messages from the Slaves to Master and Master to DBRM - will look like - + The format of the messages from DBRM to Master and from Master to Slaves + will look like + + The format of the messages from the Slaves to Master and Master to DBRM + will look like + */ -//FIXME: put these in a enum. The current arrangement is error-prone. +// FIXME: put these in a enum. The current arrangement is error-prone. const uint8_t DELETE_OID = 1; const uint8_t WRITE_VB_ENTRY = 3; const uint8_t BEGIN_VB_COPY = 4; @@ -499,7 +494,7 @@ const uint8_t MERGEMANYEXTENTSMAXMIN = 32; const uint8_t DELETE_PARTITION = 33; const uint8_t MARK_PARTITION_FOR_DELETION = 34; const uint8_t RESTORE_PARTITION = 35; -const uint8_t CREATE_COLUMN_EXTENT_DBROOT = 36; // @bug 4091: To be deprecated +const uint8_t CREATE_COLUMN_EXTENT_DBROOT = 36; // @bug 4091: To be deprecated const uint8_t BULK_SET_HWM = 37; const uint8_t ROLLBACK_COLUMN_EXTENTS_DBROOT = 38; const uint8_t ROLLBACK_DICT_STORE_EXTENTS_DBROOT = 39; @@ -541,7 +536,7 @@ const uint8_t CHANGE_TABLE_LOCK_OWNER = 73; const uint8_t GET_ALL_TABLE_LOCKS = 74; const uint8_t RELEASE_ALL_TABLE_LOCKS = 75; const uint8_t GET_TABLE_LOCK_INFO = 76; -const uint8_t OWNER_CHECK = 77; // the msg from the controller to worker +const uint8_t OWNER_CHECK = 77; // the msg from the controller to worker /* Autoincrement interface (WIP) */ const uint8_t START_AI_SEQUENCE = 80; @@ -560,7 +555,6 @@ const uint8_t BULK_UPDATE_DBROOT = 100; const uint8_t GET_SYSTEM_CATALOG = 101; const uint8_t BULK_WRITE_VB_ENTRY = 102; - /* Error codes returned by the DBRM functions. */ /// The operation was successful const int8_t ERR_OK = 0; @@ -568,22 +562,28 @@ const int8_t ERR_OK = 0; /// There was some unspecific failure and if the operation was a "write", no change was made const int8_t ERR_FAILURE = 1; -/// The operation failed because at least one slave has a different image than the others. No change was made. +/// The operation failed because at least one slave has a different image than the others. No change was +/// made. const int8_t ERR_SLAVE_INCONSISTENCY = 2; /// The operation failed because of a communication problem. const int8_t ERR_NETWORK = 3; -/// The operation failed because one slave did not send a response within 10 seconds. The administrator should look into it. +/// The operation failed because one slave did not send a response within 10 seconds. The administrator +/// should look into it. const int8_t ERR_TIMEOUT = 4; -/// The operation failed because the Master is in read-only mode. Either it detected an serious error (ie ERR_SLAVE_INCONSISTENCY), or the administrator set it using dbrmctl. The system needs the administrator to fix the problem and enable read-write mode. +/// The operation failed because the Master is in read-only mode. Either it detected an serious error (ie +/// ERR_SLAVE_INCONSISTENCY), or the administrator set it using dbrmctl. The system needs the administrator +/// to fix the problem and enable read-write mode. const int8_t ERR_READONLY = 5; -/// beginVBCopy was attempted, but deadlocked and was cancelled. The transaction must be rolled back after receiving this error code. +/// beginVBCopy was attempted, but deadlocked and was cancelled. The transaction must be rolled back after +/// receiving this error code. const int8_t ERR_DEADLOCK = 6; -/// While waiting for beginVBCopy to allocate the requested LBIDs, the transaction was killed by another thread. The caller must roll back after receiving this error code. +/// While waiting for beginVBCopy to allocate the requested LBIDs, the transaction was killed by another +/// thread. The caller must roll back after receiving this error code. const int8_t ERR_KILLED = 7; /// version buffer overflow error @@ -600,62 +600,62 @@ const int8_t ERR_TABLE_NOT_LOCKED = 14; const int8_t ERR_SNAPSHOT_TOO_OLD = 15; const int8_t ERR_NO_PARTITION_PERFORMED = 16; -/// This error code is returned by writeVBEntry when a session with a low txnid attempts to write to a block with a higher verid +/// This error code is returned by writeVBEntry when a session with a low txnid attempts to write to a block +/// with a higher verid const int8_t ERR_OLDTXN_OVERWRITING_NEWTXN = 17; // structure used to hold the information to identify a partition for shared-nothing struct PartitionInfo { - LogicalPartition lp; - OID_t oid; + LogicalPartition lp; + OID_t oid; - void serialize (messageqcpp::ByteStream& b) const - { - lp.serialize(b); - b << (uint32_t)oid; - } + void serialize(messageqcpp::ByteStream& b) const + { + lp.serialize(b); + b << (uint32_t)oid; + } - void unserialize (messageqcpp::ByteStream& b) - { - lp.unserialize(b); - b >> (uint32_t&)oid; - } + void unserialize(messageqcpp::ByteStream& b) + { + lp.unserialize(b); + b >> (uint32_t&)oid; + } }; // Note: Copies share the currentTxns array class QueryContext : public messageqcpp::Serializeable { -public: - explicit QueryContext(VER_t scn = 0) - : currentScn(scn) - { - currentTxns.reset(new std::vector()); - } + public: + explicit QueryContext(VER_t scn = 0) : currentScn(scn) + { + currentTxns.reset(new std::vector()); + } - void serialize(messageqcpp::ByteStream& bs) const - { - bs << currentScn; - serializeInlineVector(bs, *currentTxns); - } + void serialize(messageqcpp::ByteStream& bs) const + { + bs << currentScn; + serializeInlineVector(bs, *currentTxns); + } - void deserialize(messageqcpp::ByteStream& bs) - { - bs >> currentScn; - deserializeInlineVector(bs, *currentTxns); - } + void deserialize(messageqcpp::ByteStream& bs) + { + bs >> currentScn; + deserializeInlineVector(bs, *currentTxns); + } - execplan::CalpontSystemCatalog::SCN currentScn; - boost::shared_ptr > currentTxns; + execplan::CalpontSystemCatalog::SCN currentScn; + boost::shared_ptr > currentTxns; -private: - //defaults okay? - //QueryContext(const QueryContext& rhs); - //QueryContext& operator=(const QueryContext& rhs); + private: + // defaults okay? + // QueryContext(const QueryContext& rhs); + // QueryContext& operator=(const QueryContext& rhs); }; std::ostream& operator<<(std::ostream&, const QueryContext&); -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/copylocks.cpp b/versioning/BRM/copylocks.cpp index a993a6d73..172dac4d7 100644 --- a/versioning/BRM/copylocks.cpp +++ b/versioning/BRM/copylocks.cpp @@ -48,16 +48,16 @@ namespace bi = boost::interprocess; #define CL_MAGIC_V1 0x789ba6c1 #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_DIRECT -# define O_DIRECT 0 +#define O_DIRECT 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif using namespace std; @@ -70,12 +70,11 @@ using namespace idbdatafile; namespace BRM { - CopyLockEntry::CopyLockEntry() { - start = 0; - size = 0; - txnID = 0; + start = 0; + size = 0; + txnID = 0; } /*static*/ @@ -88,37 +87,36 @@ CopyLocksImpl* CopyLocksImpl::fInstance = 0; /*static*/ CopyLocksImpl* CopyLocksImpl::makeCopyLocksImpl(unsigned key, off_t size, bool readOnly) { - boost::mutex::scoped_lock lk(fInstanceMutex); + boost::mutex::scoped_lock lk(fInstanceMutex); - if (fInstance) + if (fInstance) + { + if (key != fInstance->fCopyLocks.key()) { - if (key != fInstance->fCopyLocks.key()) - { - BRMShmImpl newShm(key, size, readOnly); - fInstance->swapout(newShm); - } - - idbassert(key == fInstance->fCopyLocks.key()); - return fInstance; + BRMShmImpl newShm(key, size, readOnly); + fInstance->swapout(newShm); } - fInstance = new CopyLocksImpl(key, size, readOnly); - + idbassert(key == fInstance->fCopyLocks.key()); return fInstance; + } + + fInstance = new CopyLocksImpl(key, size, readOnly); + + return fInstance; } -CopyLocksImpl::CopyLocksImpl(unsigned key, off_t size, bool readOnly) : - fCopyLocks(key, size, readOnly) +CopyLocksImpl::CopyLocksImpl(unsigned key, off_t size, bool readOnly) : fCopyLocks(key, size, readOnly) { } CopyLocks::CopyLocks() { - entries = NULL; - currentShmkey = shmid = 0; - shminfo = NULL; - r_only = false; - fCopyLocksImpl = 0; + entries = NULL; + currentShmkey = shmid = 0; + shminfo = NULL; + r_only = false; + fCopyLocksImpl = 0; } CopyLocks::~CopyLocks() @@ -127,182 +125,183 @@ CopyLocks::~CopyLocks() void CopyLocks::setReadOnly() { - r_only = true; + r_only = true; } /* always returns holding the specified lock type, and with the EM seg mapped */ void CopyLocks::lock(OPS op) { - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - if (op == READ) - shminfo = mst.getTable_read(MasterSegmentTable::CLSegment); - else - shminfo = mst.getTable_write(MasterSegmentTable::CLSegment); + if (op == READ) + shminfo = mst.getTable_read(MasterSegmentTable::CLSegment); + else + shminfo = mst.getTable_write(MasterSegmentTable::CLSegment); - if (currentShmkey != shminfo->tableShmkey) - { - if (entries != NULL) - entries = NULL; + if (currentShmkey != shminfo->tableShmkey) + { + if (entries != NULL) + entries = NULL; + + if (shminfo->allocdSize == 0) + if (op == READ) + { + mst.getTable_upgrade(MasterSegmentTable::CLSegment); if (shminfo->allocdSize == 0) - if (op == READ) - { - mst.getTable_upgrade(MasterSegmentTable::CLSegment); + growCL(); - if (shminfo->allocdSize == 0) - growCL(); + mst.getTable_downgrade(MasterSegmentTable::CLSegment); + } + else + growCL(); + else + { + currentShmkey = shminfo->tableShmkey; + fCopyLocksImpl = CopyLocksImpl::makeCopyLocksImpl(currentShmkey, 0, r_only); + entries = fCopyLocksImpl->get(); - mst.getTable_downgrade(MasterSegmentTable::CLSegment); - } - else - growCL(); - else - { - currentShmkey = shminfo->tableShmkey; - fCopyLocksImpl = CopyLocksImpl::makeCopyLocksImpl(currentShmkey, 0, r_only); - entries = fCopyLocksImpl->get(); - - if (entries == NULL) - { - log_errno(string("CopyLocks::lock(): shmat failed")); - throw std::runtime_error("CopyLocks::lock(): shmat failed. Check the error log."); - } - } + if (entries == NULL) + { + log_errno(string("CopyLocks::lock(): shmat failed")); + throw std::runtime_error("CopyLocks::lock(): shmat failed. Check the error log."); + } } + } } void CopyLocks::release(OPS op) { - if (op == READ) - mst.releaseTable_read(MasterSegmentTable::CLSegment); - else - mst.releaseTable_write(MasterSegmentTable::CLSegment); + if (op == READ) + mst.releaseTable_read(MasterSegmentTable::CLSegment); + else + mst.releaseTable_write(MasterSegmentTable::CLSegment); } key_t CopyLocks::chooseShmkey() { - int fixedKeys = 1; - key_t ret; + int fixedKeys = 1; + key_t ret; - if (shminfo->tableShmkey + 1 == (key_t) (fShmKeys.KEYRANGE_CL_BASE + - fShmKeys.KEYRANGE_SIZE - 1) || (unsigned)shminfo->tableShmkey < fShmKeys.KEYRANGE_CL_BASE) - ret = fShmKeys.KEYRANGE_CL_BASE + fixedKeys; - else - ret = shminfo->tableShmkey + 1; + if (shminfo->tableShmkey + 1 == (key_t)(fShmKeys.KEYRANGE_CL_BASE + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)shminfo->tableShmkey < fShmKeys.KEYRANGE_CL_BASE) + ret = fShmKeys.KEYRANGE_CL_BASE + fixedKeys; + else + ret = shminfo->tableShmkey + 1; - return ret; + return ret; } void CopyLocks::growCL() { - int allocSize; - key_t newshmkey; + int allocSize; + key_t newshmkey; - if (shminfo->allocdSize == 0) - allocSize = CL_INITIAL_SIZE; - else - allocSize = shminfo->allocdSize + CL_INCREMENT; + if (shminfo->allocdSize == 0) + allocSize = CL_INITIAL_SIZE; + else + allocSize = shminfo->allocdSize + CL_INCREMENT; - newshmkey = chooseShmkey(); - idbassert((allocSize == CL_INITIAL_SIZE && !fCopyLocksImpl) || fCopyLocksImpl); + newshmkey = chooseShmkey(); + idbassert((allocSize == CL_INITIAL_SIZE && !fCopyLocksImpl) || fCopyLocksImpl); - if (!fCopyLocksImpl) - fCopyLocksImpl = CopyLocksImpl::makeCopyLocksImpl(newshmkey, allocSize, r_only); - else - fCopyLocksImpl->grow(newshmkey, allocSize); + if (!fCopyLocksImpl) + fCopyLocksImpl = CopyLocksImpl::makeCopyLocksImpl(newshmkey, allocSize, r_only); + else + fCopyLocksImpl->grow(newshmkey, allocSize); - shminfo->tableShmkey = currentShmkey = newshmkey; - shminfo->allocdSize = allocSize; + shminfo->tableShmkey = currentShmkey = newshmkey; + shminfo->allocdSize = allocSize; - if (r_only) - fCopyLocksImpl->makeReadOnly(); + if (r_only) + fCopyLocksImpl->makeReadOnly(); - entries = fCopyLocksImpl->get(); - // Temporary fix. Get rid of the old undo records that now point to nothing. - // Would be nice to be able to carry them forward. - confirmChanges(); + entries = fCopyLocksImpl->get(); + // Temporary fix. Get rid of the old undo records that now point to nothing. + // Would be nice to be able to carry them forward. + confirmChanges(); } // this fcn is dumb; relies on external check on whether it's safe or not // also relies on external write lock grab void CopyLocks::lockRange(const LBIDRange& l, VER_t txnID) { - int i, numEntries; + int i, numEntries; - // grow if necessary - if (shminfo->currentSize == shminfo->allocdSize) - growCL(); + // grow if necessary + if (shminfo->currentSize == shminfo->allocdSize) + growCL(); - /* debugging code, check for an existing lock */ - //assert(!isLocked(l)); + /* debugging code, check for an existing lock */ + // assert(!isLocked(l)); - //ostringstream os; - //os << "Copylocks locking <" << l.start << ", " << l.size << "> txnID = " << txnID; - //log(os.str()); + // ostringstream os; + // os << "Copylocks locking <" << l.start << ", " << l.size << "> txnID = " << txnID; + // log(os.str()); - // scan for an empty entry - numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); + // scan for an empty entry + numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); - for (i = 0; i < numEntries; i++) + for (i = 0; i < numEntries; i++) + { + if (entries[i].size == 0) { - if (entries[i].size == 0) - { - makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); - entries[i].start = l.start; - entries[i].size = l.size; - entries[i].txnID = txnID; - makeUndoRecord(shminfo, sizeof(MSTEntry)); - shminfo->currentSize += sizeof(CopyLockEntry); + makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); + entries[i].start = l.start; + entries[i].size = l.size; + entries[i].txnID = txnID; + makeUndoRecord(shminfo, sizeof(MSTEntry)); + shminfo->currentSize += sizeof(CopyLockEntry); - // make sure isLocked() now sees the lock - //assert(isLocked(l)); - return; - } + // make sure isLocked() now sees the lock + // assert(isLocked(l)); + return; } + } - log(string("CopyLocks::lockRange(): shm metadata problem: could not find an empty copylock entry")); - throw std::logic_error("CopyLocks::lockRange(): shm metadata problem: could not find an empty copylock entry"); + log(string("CopyLocks::lockRange(): shm metadata problem: could not find an empty copylock entry")); + throw std::logic_error( + "CopyLocks::lockRange(): shm metadata problem: could not find an empty copylock entry"); } // this fcn is dumb; relies on external check on whether it's safe or not // also relies on external write lock grab void CopyLocks::releaseRange(const LBIDRange& l) { - int i, numEntries; - LBID_t lastBlock = l.start + l.size - 1; - LBID_t eLastBlock; + int i, numEntries; + LBID_t lastBlock = l.start + l.size - 1; + LBID_t eLastBlock; #ifdef BRM_DEBUG - // debatable whether this should be included or not given the timers - // that automatically release locks - idbassert(isLocked(l)); + // debatable whether this should be included or not given the timers + // that automatically release locks + idbassert(isLocked(l)); #endif - numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); + numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); - for (i = 0; i < numEntries; i++) + for (i = 0; i < numEntries; i++) + { + CopyLockEntry& e = entries[i]; + + if (e.size != 0) { - CopyLockEntry& e = entries[i]; + eLastBlock = e.start + e.size - 1; - if (e.size != 0) - { - eLastBlock = e.start + e.size - 1; - - if (l.start <= eLastBlock && lastBlock >= e.start) - { - makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); - e.size = 0; - makeUndoRecord(shminfo, sizeof(MSTEntry)); - shminfo->currentSize -= sizeof(CopyLockEntry); - } - } + if (l.start <= eLastBlock && lastBlock >= e.start) + { + makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); + e.size = 0; + makeUndoRecord(shminfo, sizeof(MSTEntry)); + shminfo->currentSize -= sizeof(CopyLockEntry); + } } + } #ifdef BRM_DEBUG - idbassert(!isLocked(l)); - //log(string("CopyLocks::releaseRange(): that range isn't locked", LOG_TYPE_WARNING)); - //throw std::invalid_argument("CopyLocks::releaseRange(): that range isn't locked"); + idbassert(!isLocked(l)); + // log(string("CopyLocks::releaseRange(): that range isn't locked", LOG_TYPE_WARNING)); + // throw std::invalid_argument("CopyLocks::releaseRange(): that range isn't locked"); #endif } @@ -310,89 +309,86 @@ void CopyLocks::releaseRange(const LBIDRange& l) * shouldn't use makeUndoRecord() */ void CopyLocks::forceRelease(const LBIDRange& l) { - int i, numEntries; - LBID_t lastBlock = l.start + l.size - 1; - LBID_t eLastBlock; + int i, numEntries; + LBID_t lastBlock = l.start + l.size - 1; + LBID_t eLastBlock; - numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); + numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); - //ostringstream os; - //os << "Copylocks force-releasing <" << l.start << ", " << l.size << ">"; - //log(os.str()); + // ostringstream os; + // os << "Copylocks force-releasing <" << l.start << ", " << l.size << ">"; + // log(os.str()); + /* If a range intersects l, get rid of it. */ + for (i = 0; i < numEntries; i++) + { + CopyLockEntry& e = entries[i]; - /* If a range intersects l, get rid of it. */ - for (i = 0; i < numEntries; i++) + if (e.size != 0) { - CopyLockEntry& e = entries[i]; + eLastBlock = e.start + e.size - 1; - if (e.size != 0) - { - eLastBlock = e.start + e.size - 1; - - if (l.start <= eLastBlock && lastBlock >= e.start) - { - makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); - e.size = 0; - makeUndoRecord(shminfo, sizeof(MSTEntry)); - shminfo->currentSize -= sizeof(CopyLockEntry); - } - } + if (l.start <= eLastBlock && lastBlock >= e.start) + { + makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); + e.size = 0; + makeUndoRecord(shminfo, sizeof(MSTEntry)); + shminfo->currentSize -= sizeof(CopyLockEntry); + } } + } - //assert(!isLocked(l)); + // assert(!isLocked(l)); } -//assumes read lock +// assumes read lock bool CopyLocks::isLocked(const LBIDRange& l) const { - int i, numEntries; - LBID_t lLastBlock, lastBlock; + int i, numEntries; + LBID_t lLastBlock, lastBlock; - numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); - lLastBlock = l.start + l.size - 1; + numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); + lLastBlock = l.start + l.size - 1; - for (i = 0; i < numEntries; i++) + for (i = 0; i < numEntries; i++) + { + if (entries[i].size != 0) { - if (entries[i].size != 0) - { - lastBlock = entries[i].start + entries[i].size - 1; + lastBlock = entries[i].start + entries[i].size - 1; - if (lLastBlock >= entries[i].start && l.start <= lastBlock) - return true; - } + if (lLastBlock >= entries[i].start && l.start <= lastBlock) + return true; } + } - return false; + return false; } void CopyLocks::rollback(VER_t txnID) { - int i, numEntries; + int i, numEntries; - numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); + numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); - for (i = 0; i < numEntries; i++) - if (entries[i].size != 0 && entries[i].txnID == txnID) - { - makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); - entries[i].size = 0; - makeUndoRecord(shminfo, sizeof(MSTEntry)); - shminfo->currentSize -= sizeof(CopyLockEntry); - } + for (i = 0; i < numEntries; i++) + if (entries[i].size != 0 && entries[i].txnID == txnID) + { + makeUndoRecord(&entries[i], sizeof(CopyLockEntry)); + entries[i].size = 0; + makeUndoRecord(shminfo, sizeof(MSTEntry)); + shminfo->currentSize -= sizeof(CopyLockEntry); + } } void CopyLocks::getCurrentTxnIDs(std::set& list) const { - int i, numEntries; + int i, numEntries; - numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); + numEntries = shminfo->allocdSize / sizeof(CopyLockEntry); - for (i = 0; i < numEntries; i++) - if (entries[i].size != 0) - list.insert(entries[i].txnID); + for (i = 0; i < numEntries; i++) + if (entries[i].size != 0) + list.insert(entries[i].txnID); } - -} // namespace - +} // namespace BRM diff --git a/versioning/BRM/copylocks.h b/versioning/BRM/copylocks.h index f8d60b7c5..652b086a1 100644 --- a/versioning/BRM/copylocks.h +++ b/versioning/BRM/copylocks.h @@ -45,8 +45,8 @@ #include "brmshmimpl.h" /* Should load these from a config file */ -#define CL_INITIAL_SIZE (50*sizeof(CopyLockEntry)) -#define CL_INCREMENT (50*sizeof(CopyLockEntry)) +#define CL_INITIAL_SIZE (50 * sizeof(CopyLockEntry)) +#define CL_INCREMENT (50 * sizeof(CopyLockEntry)) #if defined(_MSC_VER) && defined(xxxCOPYLOCKS_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -54,7 +54,6 @@ #define EXPORT #endif - namespace idbdatafile { class IDBDataFile; @@ -62,112 +61,110 @@ class IDBDataFile; namespace BRM { - struct CopyLockEntry { - LBID_t start; - int size; - VER_t txnID; - EXPORT CopyLockEntry(); + LBID_t start; + int size; + VER_t txnID; + EXPORT CopyLockEntry(); }; class CopyLocksImpl { -public: - static CopyLocksImpl* makeCopyLocksImpl(unsigned key, off_t size, bool readOnly = false); + public: + static CopyLocksImpl* makeCopyLocksImpl(unsigned key, off_t size, bool readOnly = false); - inline void grow(unsigned key, off_t size) + inline void grow(unsigned key, off_t size) #ifdef NDEBUG - { - fCopyLocks.grow(key, size); - } + { + fCopyLocks.grow(key, size); + } #else - { - int rc = fCopyLocks.grow(key, size); - idbassert(rc == 0); - } + { + int rc = fCopyLocks.grow(key, size); + idbassert(rc == 0); + } #endif - inline void makeReadOnly() - { - fCopyLocks.setReadOnly(); - } - inline void clear(unsigned key, off_t size) - { - fCopyLocks.clear(key, size); - } - inline void swapout(BRMShmImpl& rhs) - { - fCopyLocks.swap(rhs); - rhs.destroy(); - } - inline unsigned key() const - { - return fCopyLocks.key(); - } + inline void makeReadOnly() + { + fCopyLocks.setReadOnly(); + } + inline void clear(unsigned key, off_t size) + { + fCopyLocks.clear(key, size); + } + inline void swapout(BRMShmImpl& rhs) + { + fCopyLocks.swap(rhs); + rhs.destroy(); + } + inline unsigned key() const + { + return fCopyLocks.key(); + } - inline CopyLockEntry* get() const - { - return reinterpret_cast(fCopyLocks.fMapreg.get_address()); - } + inline CopyLockEntry* get() const + { + return reinterpret_cast(fCopyLocks.fMapreg.get_address()); + } -private: - CopyLocksImpl(unsigned key, off_t size, bool readOnly = false); - ~CopyLocksImpl(); - CopyLocksImpl(const CopyLocksImpl& rhs); - CopyLocksImpl& operator=(const CopyLocksImpl& rhs); + private: + CopyLocksImpl(unsigned key, off_t size, bool readOnly = false); + ~CopyLocksImpl(); + CopyLocksImpl(const CopyLocksImpl& rhs); + CopyLocksImpl& operator=(const CopyLocksImpl& rhs); - BRMShmImpl fCopyLocks; + BRMShmImpl fCopyLocks; - static boost::mutex fInstanceMutex; - static CopyLocksImpl* fInstance; + static boost::mutex fInstanceMutex; + static CopyLocksImpl* fInstance; }; class CopyLocks : public Undoable { -public: + public: + enum OPS + { + NONE, + READ, + WRITE + }; - enum OPS - { - NONE, - READ, - WRITE - }; + EXPORT CopyLocks(); + EXPORT ~CopyLocks(); - EXPORT CopyLocks(); - EXPORT ~CopyLocks(); + EXPORT void lockRange(const LBIDRange& range, VER_t txnID); + EXPORT void releaseRange(const LBIDRange& range); + EXPORT bool isLocked(const LBIDRange& range) const; + EXPORT void rollback(VER_t txnID); - EXPORT void lockRange(const LBIDRange& range, VER_t txnID); - EXPORT void releaseRange(const LBIDRange& range); - EXPORT bool isLocked(const LBIDRange& range) const; - EXPORT void rollback(VER_t txnID); + EXPORT void lock(OPS op); + EXPORT void release(OPS op); + EXPORT void setReadOnly(); + EXPORT void getCurrentTxnIDs(std::set& txnList) const; - EXPORT void lock(OPS op); - EXPORT void release(OPS op); - EXPORT void setReadOnly(); - EXPORT void getCurrentTxnIDs(std::set& txnList) const; + EXPORT void forceRelease(const LBIDRange& range); - EXPORT void forceRelease(const LBIDRange& range); + private: + CopyLocks(const CopyLocks&); + CopyLocks& operator=(const CopyLocks&); -private: - CopyLocks(const CopyLocks&); - CopyLocks& operator=(const CopyLocks&); + key_t chooseShmkey(); + void growCL(); - key_t chooseShmkey(); - void growCL(); - - CopyLockEntry* entries; - key_t currentShmkey; - int shmid; //shmid's necessary? - MSTEntry* shminfo; - MasterSegmentTable mst; - bool r_only; - static boost::mutex mutex; - static const int MAX_IO_RETRIES = 10; - ShmKeys fShmKeys; - CopyLocksImpl* fCopyLocksImpl; + CopyLockEntry* entries; + key_t currentShmkey; + int shmid; // shmid's necessary? + MSTEntry* shminfo; + MasterSegmentTable mst; + bool r_only; + static boost::mutex mutex; + static const int MAX_IO_RETRIES = 10; + ShmKeys fShmKeys; + CopyLocksImpl* fCopyLocksImpl; }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/copylocktest.cpp b/versioning/BRM/copylocktest.cpp index f774bcdc5..bd09beac5 100644 --- a/versioning/BRM/copylocktest.cpp +++ b/versioning/BRM/copylocktest.cpp @@ -10,105 +10,92 @@ DBRM dbrm; void lockRange() { - LBIDRange r; - r.start = 12345; - r.size = 5; - copylocks.lock(CopyLocks::WRITE); + LBIDRange r; + r.start = 12345; + r.size = 5; + copylocks.lock(CopyLocks::WRITE); - if (copylocks.isLocked(r)) - cout << "already locked\n"; - else - { - copylocks.lockRange(r, -1); - cout << "got the 1st lock\n"; - } + if (copylocks.isLocked(r)) + cout << "already locked\n"; + else + { + copylocks.lockRange(r, -1); + cout << "got the 1st lock\n"; + } - r.start = 12349; + r.start = 12349; - if (copylocks.isLocked(r)) - cout << "2nd range is locked\n"; - else - cout << "2nd range is NOT locked\n"; + if (copylocks.isLocked(r)) + cout << "2nd range is locked\n"; + else + cout << "2nd range is NOT locked\n"; + r.start = 12350; - r.start = 12350; + if (copylocks.isLocked(r)) + cout << "3rd range is locked\n"; + else + { + copylocks.lockRange(r, -1); + cout << "got the 3rd lock\n"; + } - if (copylocks.isLocked(r)) - cout << "3rd range is locked\n"; - else - { - copylocks.lockRange(r, -1); - cout << "got the 3rd lock\n"; - } - - copylocks.release(CopyLocks::WRITE); + copylocks.release(CopyLocks::WRITE); } void dbrmLockRange() { - dbrm.lockLBIDRange(12345, 5); - cout << "OK\n"; + dbrm.lockLBIDRange(12345, 5); + cout << "OK\n"; } void dbrmReleaseRange() { - dbrm.releaseLBIDRange(12345, 5); - cout << "OK\n"; + dbrm.releaseLBIDRange(12345, 5); + cout << "OK\n"; } - void releaseRange() { - LBIDRange r; - r.start = 12345; - r.size = 5; - copylocks.lock(CopyLocks::WRITE); - copylocks.releaseRange(r); + LBIDRange r; + r.start = 12345; + r.size = 5; + copylocks.lock(CopyLocks::WRITE); + copylocks.releaseRange(r); - r.start = 12350; - copylocks.releaseRange(r); - copylocks.release(CopyLocks::WRITE); + r.start = 12350; + copylocks.releaseRange(r); + copylocks.release(CopyLocks::WRITE); - - cout << "OK\n"; + cout << "OK\n"; } int main(int argc, char** argv) { - char cmd; + char cmd; - if (argc < 2) - goto usage; + if (argc < 2) + goto usage; - cmd = argv[1][0]; + cmd = argv[1][0]; - idbdatafile::IDBPolicy::configIDBPolicy(); + idbdatafile::IDBPolicy::configIDBPolicy(); - switch (cmd) - { - case 'l': - lockRange(); - break; + switch (cmd) + { + case 'l': lockRange(); break; - case 'r': - releaseRange(); - break; + case 'r': releaseRange(); break; - case 'L': - dbrmLockRange(); - break; + case 'L': dbrmLockRange(); break; - case 'R': - dbrmReleaseRange(); - break; + case 'R': dbrmReleaseRange(); break; - default: - goto usage; - } + default: goto usage; + } - exit(0); + exit(0); usage: - cout << "Usage: " << argv[0] << " l | r | L | R. lock/release. Check the code for specifics. :P" << endl; - exit(1); + cout << "Usage: " << argv[0] << " l | r | L | R. lock/release. Check the code for specifics. :P" << endl; + exit(1); } - diff --git a/versioning/BRM/cvt_em.cpp b/versioning/BRM/cvt_em.cpp index 239e9ba1b..e8ef60e4a 100644 --- a/versioning/BRM/cvt_em.cpp +++ b/versioning/BRM/cvt_em.cpp @@ -37,200 +37,202 @@ namespace { void usage(char* name) { - cout << "Usage: " << name << " " << endl; - exit(1); -} + cout << "Usage: " << name << " " << endl; + exit(1); } +} // namespace namespace BRM { class ExtentMapConverter { -public: - ExtentMapConverter() {}; + public: + ExtentMapConverter(){}; - int doCvt(unsigned oldExtentSize, unsigned newExtentSize, const string& filename); + int doCvt(unsigned oldExtentSize, unsigned newExtentSize, const string& filename); -private: - ExtentMapConverter(const ExtentMapConverter& rhs); - ExtentMapConverter& operator=(const ExtentMapConverter& rhs); + private: + ExtentMapConverter(const ExtentMapConverter& rhs); + ExtentMapConverter& operator=(const ExtentMapConverter& rhs); - ExtentMap em; + ExtentMap em; }; int ExtentMapConverter::doCvt(unsigned oldExtentSize, unsigned newExtentSize, const string& filename) { - int currentSize, loadSize[3]; - ifstream in; + int currentSize, loadSize[3]; + ifstream in; - em.grabEMEntryTable(ExtentMap::WRITE); + em.grabEMEntryTable(ExtentMap::WRITE); - try - { - em.grabFreeList(ExtentMap::WRITE); - } - catch (...) - { - em.releaseEMEntryTable(ExtentMap::WRITE); - throw; - } + try + { + em.grabFreeList(ExtentMap::WRITE); + } + catch (...) + { + em.releaseEMEntryTable(ExtentMap::WRITE); + throw; + } - div_t d = div((int)oldExtentSize, (int)newExtentSize); - idbassert(d.quot > 1); - idbassert(d.rem == 0); + div_t d = div((int)oldExtentSize, (int)newExtentSize); + idbassert(d.quot > 1); + idbassert(d.rem == 0); - const unsigned mult = d.quot; + const unsigned mult = d.quot; - in.open(filename.c_str()); + in.open(filename.c_str()); - if (!in) - { - log_errno("ExtentMap::load(): open"); - em.releaseFreeList(ExtentMap::WRITE); - em.releaseEMEntryTable(ExtentMap::WRITE); - throw std::ios_base::failure("ExtentMap::load(): open failed. Check the error log."); - } + if (!in) + { + log_errno("ExtentMap::load(): open"); + em.releaseFreeList(ExtentMap::WRITE); + em.releaseEMEntryTable(ExtentMap::WRITE); + throw std::ios_base::failure("ExtentMap::load(): open failed. Check the error log."); + } - in.exceptions(ios_base::badbit | ios_base::failbit); + in.exceptions(ios_base::badbit | ios_base::failbit); - try - { - in.read((char*) &loadSize, 3 * sizeof(int)); - } - catch (...) - { - in.close(); - em.releaseFreeList(ExtentMap::WRITE); - em.releaseEMEntryTable(ExtentMap::WRITE); - throw; - } + try + { + in.read((char*)&loadSize, 3 * sizeof(int)); + } + catch (...) + { + in.close(); + em.releaseFreeList(ExtentMap::WRITE); + em.releaseEMEntryTable(ExtentMap::WRITE); + throw; + } - const int emVersion = loadSize[0]; - const int emNumElements = loadSize[1]; - const int flNumElements = loadSize[2]; - /* What's a safe upper limit on the # of EM and FL entries? */ + const int emVersion = loadSize[0]; + const int emNumElements = loadSize[1]; + const int flNumElements = loadSize[2]; + /* What's a safe upper limit on the # of EM and FL entries? */ #define EM_MAGIC_V3 0x76f78b1e - if ( emVersion != EM_MAGIC_V3 || emNumElements < 0 || flNumElements < 0) + if (emVersion != EM_MAGIC_V3 || emNumElements < 0 || flNumElements < 0) + { + in.close(); + em.releaseFreeList(ExtentMap::WRITE); + em.releaseEMEntryTable(ExtentMap::WRITE); + log("ExtentMap::load64(): That file is not a valid 64-bit ExtentMap image"); + throw std::runtime_error("ExtentMap::load64(): That file is not a valid 64-bit ExtentMap image"); + } + + memset(em.fExtentMap, 0, em.fEMShminfo->allocdSize); + memset(em.fFreeList, 0, em.fFLShminfo->allocdSize); + em.fEMShminfo->currentSize = 0; + em.fFLShminfo->currentSize = 0; + + int j = 0; + int maxLoops = (emNumElements * (signed)mult - em.fEMShminfo->allocdSize / sizeof(EMEntry)) / 100 + 1; + int target = (int)(maxLoops * .02); + + if (maxLoops < 50) + target = 1; + + // allocate shared memory for extent data + for (currentSize = em.fEMShminfo->allocdSize / sizeof(EMEntry); + currentSize < (emNumElements * (signed)mult); + currentSize = em.fEMShminfo->allocdSize / sizeof(EMEntry)) + { + em.growEMShmseg(); + + if ((j % target) == 0) + cout << '.' << flush; + + j++; + } + + cout << endl; + + // allocate shared memory for freelist + for (currentSize = em.fFLShminfo->allocdSize / sizeof(InlineLBIDRange); currentSize < flNumElements; + currentSize = em.fFLShminfo->allocdSize / sizeof(InlineLBIDRange)) + { + em.growFLShmseg(); + } + + try + { + typedef map > OIDMap_t; + OIDMap_t OIDMap; + uint8_t buf[emNumElements * sizeof(EMEntry)]; + uint8_t buf2[flNumElements * sizeof(InlineLBIDRange)]; + + in.read((char*)buf, emNumElements * sizeof(EMEntry)); + + // memcpy(fExtentMap, buf, emNumElements * sizeof(EMEntry)); + EMEntry* emSrc = reinterpret_cast(&buf[0]); + j = 0; + + for (int i = 0; i < emNumElements; i++) { - in.close(); - em.releaseFreeList(ExtentMap::WRITE); - em.releaseEMEntryTable(ExtentMap::WRITE); - log("ExtentMap::load64(): That file is not a valid 64-bit ExtentMap image"); - throw std::runtime_error("ExtentMap::load64(): That file is not a valid 64-bit ExtentMap image"); - } - - memset(em.fExtentMap, 0, em.fEMShminfo->allocdSize); - memset(em.fFreeList, 0, em.fFLShminfo->allocdSize); - em.fEMShminfo->currentSize = 0; - em.fFLShminfo->currentSize = 0; - - int j = 0; - int maxLoops = (emNumElements * (signed)mult - em.fEMShminfo->allocdSize / sizeof(EMEntry)) / 100 + 1; - int target = (int)(maxLoops * .02); - - if (maxLoops < 50) target = 1; - - // allocate shared memory for extent data - for (currentSize = em.fEMShminfo->allocdSize / sizeof(EMEntry); - currentSize < (emNumElements * (signed)mult); - currentSize = em.fEMShminfo->allocdSize / sizeof(EMEntry)) - { - em.growEMShmseg(); - - if ((j % target) == 0) cout << '.' << flush; + vector& oidv = OIDMap[emSrc[i].fileID]; + for (unsigned k = 0; k < mult; k++) + { + oidv.push_back(j); + // em.fExtentMap[j].range.start = emSrc[i].range.start; + em.fExtentMap[j].range.start = emSrc[i].range.start + (k * newExtentSize); + // em.fExtentMap[j].range.size = emSrc[i].range.size; + em.fExtentMap[j].range.size = newExtentSize / 1024; + em.fExtentMap[j].fileID = emSrc[i].fileID; + em.fExtentMap[j].blockOffset = emSrc[i].blockOffset + (k * newExtentSize); + em.fExtentMap[j].HWM = emSrc[i].HWM; + em.fExtentMap[j].txnID = emSrc[i].txnID; + em.fExtentMap[j].secondHWM = emSrc[i].secondHWM; + em.fExtentMap[j].nextHeader = emSrc[i].nextHeader; + em.fExtentMap[j].partition.type = emSrc[i].partition.type; + // em.fExtentMap[j].partition.cprange.hi_val = emSrc[i].partition.cprange.hi_val; + em.fExtentMap[j].partition.cprange.hi_val = numeric_limits::min(); + // em.fExtentMap[j].partition.cprange.lo_val = emSrc[i].partition.cprange.lo_val; + em.fExtentMap[j].partition.cprange.lo_val = numeric_limits::max(); + // em.fExtentMap[j].partition.cprange.sequenceNum = emSrc[i].partition.cprange.sequenceNum; + em.fExtentMap[j].partition.cprange.sequenceNum = 0; + // em.fExtentMap[j].partition.cprange.isValid = emSrc[i].partition.cprange.isValid; + em.fExtentMap[j].partition.cprange.isValid = CP_INVALID; j++; + } } - cout << endl; + em.fEMShminfo->currentSize = j * sizeof(EMEntry); - // allocate shared memory for freelist - for (currentSize = em.fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - currentSize < flNumElements; - currentSize = em.fFLShminfo->allocdSize / sizeof(InlineLBIDRange)) + cout << j << " total new em entries from " << emNumElements << " file entries" << endl; + cout << OIDMap.size() << " OIDs added to em" << endl; + + OIDMap_t::const_iterator iter = OIDMap.begin(); + OIDMap_t::const_iterator end = OIDMap.end(); + + int l = 0; + + while (iter != end) { - em.growFLShmseg(); + const vector& oidv = iter->second; + vector::const_reverse_iterator riter = oidv.rbegin(); + vector::const_reverse_iterator rend = oidv.rend(); + HWM_t hwm = em.fExtentMap[*riter].HWM; + + while (riter != rend) + { + if (em.fExtentMap[*riter].blockOffset > hwm) + { + em.fExtentMap[*riter].fileID = numeric_limits::max(); + em.fExtentMap[*riter].blockOffset = 0; + em.fExtentMap[*riter].HWM = 0; + l++; + } + else + break; + + ++riter; + } + + ++iter; } - try - { - typedef map > OIDMap_t; - OIDMap_t OIDMap; - uint8_t buf[emNumElements * sizeof(EMEntry)]; - uint8_t buf2[flNumElements * sizeof(InlineLBIDRange)]; - - in.read((char*) buf, emNumElements * sizeof(EMEntry)); - - //memcpy(fExtentMap, buf, emNumElements * sizeof(EMEntry)); - EMEntry* emSrc = reinterpret_cast(&buf[0]); - j = 0; - - for (int i = 0; i < emNumElements; i++) - { - vector& oidv = OIDMap[emSrc[i].fileID]; - - for (unsigned k = 0; k < mult; k++) - { - oidv.push_back(j); - //em.fExtentMap[j].range.start = emSrc[i].range.start; - em.fExtentMap[j].range.start = emSrc[i].range.start + (k * newExtentSize); - //em.fExtentMap[j].range.size = emSrc[i].range.size; - em.fExtentMap[j].range.size = newExtentSize / 1024; - em.fExtentMap[j].fileID = emSrc[i].fileID; - em.fExtentMap[j].blockOffset = emSrc[i].blockOffset + (k * newExtentSize); - em.fExtentMap[j].HWM = emSrc[i].HWM; - em.fExtentMap[j].txnID = emSrc[i].txnID; - em.fExtentMap[j].secondHWM = emSrc[i].secondHWM; - em.fExtentMap[j].nextHeader = emSrc[i].nextHeader; - em.fExtentMap[j].partition.type = emSrc[i].partition.type; - //em.fExtentMap[j].partition.cprange.hi_val = emSrc[i].partition.cprange.hi_val; - em.fExtentMap[j].partition.cprange.hi_val = numeric_limits::min(); - //em.fExtentMap[j].partition.cprange.lo_val = emSrc[i].partition.cprange.lo_val; - em.fExtentMap[j].partition.cprange.lo_val = numeric_limits::max(); - //em.fExtentMap[j].partition.cprange.sequenceNum = emSrc[i].partition.cprange.sequenceNum; - em.fExtentMap[j].partition.cprange.sequenceNum = 0; - //em.fExtentMap[j].partition.cprange.isValid = emSrc[i].partition.cprange.isValid; - em.fExtentMap[j].partition.cprange.isValid = CP_INVALID; - j++; - } - } - - em.fEMShminfo->currentSize = j * sizeof(EMEntry); - - cout << j << " total new em entries from " << emNumElements << " file entries" << endl; - cout << OIDMap.size() << " OIDs added to em" << endl; - - OIDMap_t::const_iterator iter = OIDMap.begin(); - OIDMap_t::const_iterator end = OIDMap.end(); - - int l = 0; - - while (iter != end) - { - const vector& oidv = iter->second; - vector::const_reverse_iterator riter = oidv.rbegin(); - vector::const_reverse_iterator rend = oidv.rend(); - HWM_t hwm = em.fExtentMap[*riter].HWM; - - while (riter != rend) - { - if (em.fExtentMap[*riter].blockOffset > hwm) - { - em.fExtentMap[*riter].fileID = numeric_limits::max(); - em.fExtentMap[*riter].blockOffset = 0; - em.fExtentMap[*riter].HWM = 0; - l++; - } - else break; - - ++riter; - } - - ++iter; - } - - cout << l << " entries moved to OID " << numeric_limits::max() << endl; + cout << l << " entries moved to OID " << numeric_limits::max() << endl; #if 0 int k = j; @@ -241,53 +243,52 @@ int ExtentMapConverter::doCvt(unsigned oldExtentSize, unsigned newExtentSize, co em.fExtentMap[j].txnID << '\t' << em.fExtentMap[j].secondHWM << '\t' << em.fExtentMap[j].nextHeader << endl; #endif - in.read((char*) buf2, flNumElements * sizeof(InlineLBIDRange)); + in.read((char*)buf2, flNumElements * sizeof(InlineLBIDRange)); - //memcpy(fFreeList, buf2, flNumElements * sizeof(InlineLBIDRange)); - InlineLBIDRange* lrSrc = reinterpret_cast(&buf2[0]); - j = 0; + // memcpy(fFreeList, buf2, flNumElements * sizeof(InlineLBIDRange)); + InlineLBIDRange* lrSrc = reinterpret_cast(&buf2[0]); + j = 0; - for (int i = 0; i < flNumElements; i++) - { - em.fFreeList[j].start = lrSrc[i].start; - em.fFreeList[j].size = lrSrc[i].size; - j++; - } - - em.fFLShminfo->currentSize = j * sizeof(InlineLBIDRange); - - cout << j << " total new fl entries from " << flNumElements << " file entries" << endl; - - } - catch (...) + for (int i = 0; i < flNumElements; i++) { - in.close(); - em.releaseFreeList(ExtentMap::WRITE); - em.releaseEMEntryTable(ExtentMap::WRITE); - throw; + em.fFreeList[j].start = lrSrc[i].start; + em.fFreeList[j].size = lrSrc[i].size; + j++; } + em.fFLShminfo->currentSize = j * sizeof(InlineLBIDRange); + + cout << j << " total new fl entries from " << flNumElements << " file entries" << endl; + } + catch (...) + { in.close(); em.releaseFreeList(ExtentMap::WRITE); em.releaseEMEntryTable(ExtentMap::WRITE); + throw; + } - return 0; + in.close(); + em.releaseFreeList(ExtentMap::WRITE); + em.releaseEMEntryTable(ExtentMap::WRITE); + + return 0; } -} +} // namespace BRM int main(int argc, char** argv) { - ExtentMapConverter emc; + ExtentMapConverter emc; - if (emc.doCvt(8192, 1024, "BRM_saves_em") != 0) - { - cerr << "Conversion failed." << endl; - return 1; - } - else - { - cout << "OK" << endl; - return 0; - } + if (emc.doCvt(8192, 1024, "BRM_saves_em") != 0) + { + cerr << "Conversion failed." << endl; + return 1; + } + else + { + cout << "OK" << endl; + return 0; + } } diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index 494acc8d6..1cc3fb983 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -48,17 +48,17 @@ #ifdef BRM_DEBUG #define CHECK_EMPTY(x) \ - if (x.length() != 0) \ - throw logic_error("DBRM: got a message of the wrong size"); + if (x.length() != 0) \ + throw logic_error("DBRM: got a message of the wrong size"); #else #define CHECK_EMPTY(x) #endif -#define DO_ERR_NETWORK \ - MessageQueueClientPool::releaseInstance(msgClient); \ - msgClient = NULL; \ - mutex.unlock(); \ - return ERR_NETWORK; +#define DO_ERR_NETWORK \ + MessageQueueClientPool::releaseInstance(msgClient); \ + msgClient = NULL; \ + mutex.unlock(); \ + return ERR_NETWORK; using namespace std; using namespace messageqcpp; @@ -70,238 +70,238 @@ using namespace oam; namespace BRM { - DBRM::DBRM(bool noBRMinit) : fDebug(false) { - if (!noBRMinit) - { - mst.reset(new MasterSegmentTable()); - em.reset(new ExtentMap()); - vss.reset(new VSS()); - vbbm.reset(new VBBM()); - copylocks.reset(new CopyLocks()); + if (!noBRMinit) + { + mst.reset(new MasterSegmentTable()); + em.reset(new ExtentMap()); + vss.reset(new VSS()); + vbbm.reset(new VBBM()); + copylocks.reset(new CopyLocks()); - em->setReadOnly(); - vss->setReadOnly(); - vbbm->setReadOnly(); - } + em->setReadOnly(); + vss->setReadOnly(); + vbbm->setReadOnly(); + } - msgClient = NULL; - masterName = "DBRM_Controller"; - config = config::Config::makeConfig(); + msgClient = NULL; + masterName = "DBRM_Controller"; + config = config::Config::makeConfig(); #ifdef BRM_INFO - fDebug = ("Y" == config->getConfig("DBRM", "Debug")); + fDebug = ("Y" == config->getConfig("DBRM", "Debug")); #endif } DBRM::DBRM(const DBRM& brm) { - throw logic_error("DBRM: Don't use the copy constructor."); + throw logic_error("DBRM: Don't use the copy constructor."); } DBRM::~DBRM() throw() { - if (msgClient != NULL) - MessageQueueClientPool::releaseInstance(msgClient); + if (msgClient != NULL) + MessageQueueClientPool::releaseInstance(msgClient); } DBRM& DBRM::operator=(const DBRM& brm) { - throw logic_error("DBRM: Don't use the = operator."); + throw logic_error("DBRM: Don't use the = operator."); } - int DBRM::saveState() throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("saveState()"); + if (fDebug) + TRACER_WRITENOW("saveState()"); #endif - string prefix = config->getConfig("SystemConfig", "DBRMRoot"); + string prefix = config->getConfig("SystemConfig", "DBRMRoot"); - if (prefix.length() == 0) - { - cerr << "Error: Need a valid Calpont configuation file" << endl; - exit(1); - } + if (prefix.length() == 0) + { + cerr << "Error: Need a valid Calpont configuation file" << endl; + exit(1); + } - int rc = saveState(prefix); + int rc = saveState(prefix); - return rc; + return rc; } int DBRM::saveState(string filename) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("saveState(filename)"); - TRACER_ADDSTRINPUT(filename); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("saveState(filename)"); + TRACER_ADDSTRINPUT(filename); + TRACER_WRITE; + } #endif - string emFilename = filename + "_em"; - string vssFilename = filename + "_vss"; - string vbbmFilename = filename + "_vbbm"; - bool locked[3] = { false, false, false }; + string emFilename = filename + "_em"; + string vssFilename = filename + "_vss"; + string vbbmFilename = filename + "_vbbm"; + bool locked[3] = {false, false, false}; - try - { - vbbm->lock(VBBM::READ); - locked[0] = true; - vss->lock(VSS::READ); - locked[1] = true; - copylocks->lock(CopyLocks::READ); - locked[2] = true; + try + { + vbbm->lock(VBBM::READ); + locked[0] = true; + vss->lock(VSS::READ); + locked[1] = true; + copylocks->lock(CopyLocks::READ); + locked[2] = true; - saveExtentMap(emFilename); - vbbm->save(vbbmFilename); - vss->save(vssFilename); + saveExtentMap(emFilename); + vbbm->save(vbbmFilename); + vss->save(vssFilename); - copylocks->release(CopyLocks::READ); - locked[2] = false; - vss->release(VSS::READ); - locked[1] = false; - vbbm->release(VBBM::READ); - locked[0] = false; - } - catch (exception& e) - { - if (locked[2]) - copylocks->release(CopyLocks::READ); + copylocks->release(CopyLocks::READ); + locked[2] = false; + vss->release(VSS::READ); + locked[1] = false; + vbbm->release(VBBM::READ); + locked[0] = false; + } + catch (exception& e) + { + if (locked[2]) + copylocks->release(CopyLocks::READ); - if (locked[1]) - vss->release(VSS::READ); + if (locked[1]) + vss->release(VSS::READ); - if (locked[0]) - vbbm->release(VBBM::READ); + if (locked[0]) + vbbm->release(VBBM::READ); - return -1; - } + return -1; + } - return 0; + return 0; } int DBRM::saveExtentMap(const string& filename) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("saveExtentMap"); - TRACER_ADDSTRINPUT(filename); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("saveExtentMap"); + TRACER_ADDSTRINPUT(filename); + TRACER_WRITE; + } #endif - try - { - em->save(filename); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->save(filename); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } // @bug 1055+. New functions added for multiple files per OID enhancement. -int DBRM::lookupLocal(LBID_t lbid, VER_t verid, bool vbFlag, OID_t& oid, - uint16_t& dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset) throw() +int DBRM::lookupLocal(LBID_t lbid, VER_t verid, bool vbFlag, OID_t& oid, uint16_t& dbRoot, + uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocal(lbid,ver,..)"); - TRACER_ADDINPUT(lbid); - TRACER_ADDINPUT(verid); - TRACER_ADDBOOLINPUT(vbFlag); - TRACER_ADDOUTPUT(oid); - TRACER_ADDSHORTOUTPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDOUTPUT(fileBlockOffset); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocal(lbid,ver,..)"); + TRACER_ADDINPUT(lbid); + TRACER_ADDINPUT(verid); + TRACER_ADDBOOLINPUT(vbFlag); + TRACER_ADDOUTPUT(oid); + TRACER_ADDSHORTOUTPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDOUTPUT(fileBlockOffset); + TRACER_WRITE; + } #endif - bool locked[2] = {false, false}; - int ret; - bool tooOld = false; + bool locked[2] = {false, false}; + int ret; + bool tooOld = false; - try + try + { + if (!vbFlag) + return em->lookupLocal(lbid, (int&)oid, dbRoot, partitionNum, segmentNum, fileBlockOffset); + else { - if (!vbFlag) - return em->lookupLocal(lbid, (int&)oid, dbRoot, partitionNum, segmentNum, fileBlockOffset); - else - { - vbbm->lock(VBBM::READ); - locked[0] = true; - ret = vbbm->lookup(lbid, verid, oid, fileBlockOffset); - vbbm->release(VBBM::READ); - locked[0] = false; + vbbm->lock(VBBM::READ); + locked[0] = true; + ret = vbbm->lookup(lbid, verid, oid, fileBlockOffset); + vbbm->release(VBBM::READ); + locked[0] = false; - if (ret < 0) - { - vss->lock(VSS::READ); - locked[1] = true; - tooOld = vss->isTooOld(lbid, verid); - vss->release(VSS::READ); - locked[1] = false; + if (ret < 0) + { + vss->lock(VSS::READ); + locked[1] = true; + tooOld = vss->isTooOld(lbid, verid); + vss->release(VSS::READ); + locked[1] = false; - if (tooOld) - return ERR_SNAPSHOT_TOO_OLD; - } + if (tooOld) + return ERR_SNAPSHOT_TOO_OLD; + } - return ret; - } + return ret; } - catch (exception& e) - { - if (locked[1]) - vss->release(VSS::READ); + } + catch (exception& e) + { + if (locked[1]) + vss->release(VSS::READ); - if (locked[0]) - vbbm->release(VBBM::READ); + if (locked[0]) + vbbm->release(VBBM::READ); - cerr << e.what() << endl; - return -1; - } + cerr << e.what() << endl; + return -1; + } } -int DBRM::lookupLocal(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, LBID_t& lbid) throw() +int DBRM::lookupLocal(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, + LBID_t& lbid) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocal(oid,fbo,..)"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(fileBlockOffset); - TRACER_ADDOUTPUT(lbid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocal(oid,fbo,..)"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(fileBlockOffset); + TRACER_ADDOUTPUT(lbid); + TRACER_WRITE; + } #endif - try - { - return em->lookupLocal(oid, partitionNum, segmentNum, fileBlockOffset, lbid); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + return em->lookupLocal(oid, partitionNum, segmentNum, fileBlockOffset, lbid); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } int DBRM::lookupLocal_DBroot(OID_t oid, uint32_t dbroot, uint32_t partitionNum, uint16_t segmentNum, @@ -309,28 +309,28 @@ int DBRM::lookupLocal_DBroot(OID_t oid, uint32_t dbroot, uint32_t partitionNum, { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocal(oid,fbo,..)"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(fileBlockOffset); - TRACER_ADDOUTPUT(lbid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocal(oid,fbo,..)"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(fileBlockOffset); + TRACER_ADDOUTPUT(lbid); + TRACER_WRITE; + } #endif - try - { - return em->lookupLocal_DBroot(oid, dbroot, partitionNum, segmentNum, fileBlockOffset, lbid); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + return em->lookupLocal_DBroot(oid, dbroot, partitionNum, segmentNum, fileBlockOffset, lbid); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } // @bug 1055- @@ -339,62 +339,58 @@ int DBRM::lookupLocal_DBroot(OID_t oid, uint32_t dbroot, uint32_t partitionNum, // Lookup/return starting LBID for the specified OID, partition, segment, and // file block offset. //------------------------------------------------------------------------------ -int DBRM::lookupLocalStartLbid(OID_t oid, - uint32_t partitionNum, - uint16_t segmentNum, - uint32_t fileBlockOffset, - LBID_t& lbid) throw() +int DBRM::lookupLocalStartLbid(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, + uint32_t fileBlockOffset, LBID_t& lbid) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocalStartLbid(oid,fbo,..)"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(fileBlockOffset); - TRACER_ADDOUTPUT(lbid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocalStartLbid(oid,fbo,..)"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(fileBlockOffset); + TRACER_ADDOUTPUT(lbid); + TRACER_WRITE; + } #endif - try - { - return em->lookupLocalStartLbid(oid, partitionNum, segmentNum, - fileBlockOffset, lbid); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + return em->lookupLocalStartLbid(oid, partitionNum, segmentNum, fileBlockOffset, lbid); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } int DBRM::lookup(OID_t oid, LBIDRange_v& lbidList) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookup(oid,range)"); - TRACER_ADDINPUT(oid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookup(oid,range)"); + TRACER_ADDINPUT(oid); + TRACER_WRITE; + } #endif - try - { - em->lookup(oid, lbidList); - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->lookup(oid, lbidList); + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } // Casual Partitioning support @@ -403,63 +399,65 @@ int DBRM::markExtentInvalid(const LBID_t lbid, { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("markExtentInvalid"); - TRACER_ADDINPUT(lbid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("markExtentInvalid"); + TRACER_ADDINPUT(lbid); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << MARKEXTENTINVALID << (uint64_t)lbid << (uint32_t)colDataType; - err = send_recv(command, response); + command << MARKEXTENTINVALID << (uint64_t)lbid << (uint32_t)colDataType; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::markExtentsInvalid(const vector& lbids, - const std::vector& colDataTypes) DBRM_THROW + const std::vector& colDataTypes) + DBRM_THROW { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("markExtentsInvalid"); + if (fDebug) + TRACER_WRITENOW("markExtentsInvalid"); #endif - ByteStream command, response; - uint8_t err; - uint32_t size = lbids.size(), i; + ByteStream command, response; + uint8_t err; + uint32_t size = lbids.size(), i; - command << MARKMANYEXTENTSINVALID << size; + command << MARKMANYEXTENTSINVALID << size; - for (i = 0; i < size; i++) - { - command << (uint64_t) lbids[i]; - command << (uint32_t) colDataTypes[i]; - } + for (i = 0; i < size; i++) + { + command << (uint64_t)lbids[i]; + command << (uint32_t)colDataTypes[i]; + } - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } template @@ -467,133 +465,133 @@ int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) th { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtentMaxMin"); - TRACER_ADDINPUT(lbid); - TRACER_ADDOUTPUT(max); - TRACER_ADDOUTPUT(min); - TRACER_ADDOUTPUT(seqNum); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtentMaxMin"); + TRACER_ADDINPUT(lbid); + TRACER_ADDOUTPUT(max); + TRACER_ADDOUTPUT(min); + TRACER_ADDOUTPUT(seqNum); + TRACER_WRITE; + } #endif - try - { - int ret = em->getMaxMin(lbid, max, min, seqNum); - return ret; - } - catch (exception& e) - { - cerr << e.what() << endl; - return false; - } + try + { + int ret = em->getMaxMin(lbid, max, min, seqNum); + return ret; + } + catch (exception& e) + { + cerr << e.what() << endl; + return false; + } } int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw() { - try - { - em->getCPMaxMin(lbid, cpMaxMin); - } - catch (...) - { - return ERR_FAILURE; - } - return ERR_OK; + try + { + em->getCPMaxMin(lbid, cpMaxMin); + } + catch (...) + { + return ERR_FAILURE; + } + return ERR_OK; } - - -int DBRM::setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum) DBRM_THROW +int DBRM::setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, + const int32_t seqNum) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setExtentMaxMin"); - TRACER_ADDINPUT(lbid); - TRACER_ADDINPUT(max); - TRACER_ADDINPUT(min); - TRACER_ADDINPUT(seqNum); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("setExtentMaxMin"); + TRACER_ADDINPUT(lbid); + TRACER_ADDINPUT(max); + TRACER_ADDINPUT(min); + TRACER_ADDINPUT(seqNum); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << SETEXTENTMAXMIN << (uint64_t)lbid << (uint64_t)max << (uint64_t)min << (uint64_t)seqNum; - err = send_recv(command, response); + command << SETEXTENTMAXMIN << (uint64_t)lbid << (uint64_t)max << (uint64_t)min << (uint64_t)seqNum; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } // @bug 1970 - Added function below to set multiple extents casual partition info in one call. int DBRM::setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW { - CPInfoList_t::const_iterator it; + CPInfoList_t::const_iterator it; #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setExtentsMaxMin"); - - for (it = cpInfos.begin(); it != cpInfos.end(); it++) - { - TRACER_ADDINPUT(it->firstLbid); - TRACER_ADDINPUT(it->max); - TRACER_ADDINPUT(it->min); - TRACER_ADDINPUT(it->seqNum); - TRACER_WRITE; - } - } - -#endif - ByteStream command, response; - uint8_t err = 0; - - if (cpInfos.size() == 0) - return err; - - if (cpInfos.empty()) - return ERR_OK; - - command << SETMANYEXTENTSMAXMIN << (uint32_t)cpInfos.size(); + if (fDebug) + { + TRACER_WRITELATER("setExtentsMaxMin"); for (it = cpInfos.begin(); it != cpInfos.end(); it++) { - if (it->isBinaryColumn) - { - command << (uint8_t)1 << (uint64_t)it->firstLbid << (uint128_t)it->bigMax << (uint128_t)it->bigMin << (uint32_t)it->seqNum; - } - else - { - command << (uint8_t)0 << (uint64_t)it->firstLbid << (uint64_t)it->max << (uint64_t)it->min << (uint32_t)it->seqNum; - } + TRACER_ADDINPUT(it->firstLbid); + TRACER_ADDINPUT(it->max); + TRACER_ADDINPUT(it->min); + TRACER_ADDINPUT(it->seqNum); + TRACER_WRITE; } + } - err = send_recv(command, response); +#endif + ByteStream command, response; + uint8_t err = 0; - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (cpInfos.size() == 0) return err; + + if (cpInfos.empty()) + return ERR_OK; + + command << SETMANYEXTENTSMAXMIN << (uint32_t)cpInfos.size(); + + for (it = cpInfos.begin(); it != cpInfos.end(); it++) + { + if (it->isBinaryColumn) + { + command << (uint8_t)1 << (uint64_t)it->firstLbid << (uint128_t)it->bigMax << (uint128_t)it->bigMin + << (uint32_t)it->seqNum; + } + else + { + command << (uint8_t)0 << (uint64_t)it->firstLbid << (uint64_t)it->max << (uint64_t)it->min + << (uint32_t)it->seqNum; + } + } + + err = send_recv(command, response); + + if (err != ERR_OK) + return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ @@ -602,627 +600,604 @@ int DBRM::setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW //------------------------------------------------------------------------------ int DBRM::mergeExtentsMaxMin(const CPInfoMergeList_t& cpInfos) DBRM_THROW { - CPInfoMergeList_t::const_iterator it; + CPInfoMergeList_t::const_iterator it; #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("updateExtentsMaxMin"); - - for (it = cpInfos.begin(); it != cpInfos.end(); it++) - { - TRACER_ADDINPUT(it->startLbid); - TRACER_ADDINPUT(it->max); - TRACER_ADDINPUT(it->min); - TRACER_ADDINPUT(it->seqNum); - TRACER_ADDINPUT(it->type); - TRACER_ADDINPUT(it->newExtent); - TRACER_WRITE; - } - } - -#endif - ByteStream command, response; - uint8_t err; - - command << MERGEMANYEXTENTSMAXMIN << (uint32_t)cpInfos.size(); + if (fDebug) + { + TRACER_WRITELATER("updateExtentsMaxMin"); for (it = cpInfos.begin(); it != cpInfos.end(); it++) { - command << (uint64_t)it->startLbid << - (uint64_t)it->max << - (uint64_t)it->min << - (uint32_t)it->seqNum << - (uint32_t)it->type << - (uint32_t)it->newExtent; + TRACER_ADDINPUT(it->startLbid); + TRACER_ADDINPUT(it->max); + TRACER_ADDINPUT(it->min); + TRACER_ADDINPUT(it->seqNum); + TRACER_ADDINPUT(it->type); + TRACER_ADDINPUT(it->newExtent); + TRACER_WRITE; } + } - err = send_recv(command, response); +#endif + ByteStream command, response; + uint8_t err; - if (err != ERR_OK) - return err; + command << MERGEMANYEXTENTSMAXMIN << (uint32_t)cpInfos.size(); - if (response.length() == 0) - return ERR_NETWORK; + for (it = cpInfos.begin(); it != cpInfos.end(); it++) + { + command << (uint64_t)it->startLbid << (uint64_t)it->max << (uint64_t)it->min << (uint32_t)it->seqNum + << (uint32_t)it->type << (uint32_t)it->newExtent; + } - response >> err; - CHECK_EMPTY(response); + err = send_recv(command, response); + + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } -int DBRM::vssLookup(LBID_t lbid, const QueryContext& verInfo, VER_t txnID, VER_t* outVer, - bool* vbFlag, bool vbOnly) throw() +int DBRM::vssLookup(LBID_t lbid, const QueryContext& verInfo, VER_t txnID, VER_t* outVer, bool* vbFlag, + bool vbOnly) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("vssLookup"); - TRACER_ADDINPUT(lbid); - TRACER_ADDINPUT(verInfo); - TRACER_ADDINPUT(txnID); - TRACER_ADDBOOLINPUT(vbOnly); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("vssLookup"); + TRACER_ADDINPUT(lbid); + TRACER_ADDINPUT(verInfo); + TRACER_ADDINPUT(txnID); + TRACER_ADDBOOLINPUT(vbOnly); + TRACER_WRITE; + } #endif - if (!vbOnly && vss->isEmpty()) - { - *outVer = 0; - *vbFlag = false; - return -1; - } + if (!vbOnly && vss->isEmpty()) + { + *outVer = 0; + *vbFlag = false; + return -1; + } - bool locked = false; + bool locked = false; - try - { - int rc = 0; - vss->lock(VSS::READ); - locked = true; - rc = vss->lookup(lbid, verInfo, txnID, outVer, vbFlag, vbOnly); - vss->release(VSS::READ); - return rc; - } - catch (exception& e) - { - if (locked) - vss->release(VSS::READ); + try + { + int rc = 0; + vss->lock(VSS::READ); + locked = true; + rc = vss->lookup(lbid, verInfo, txnID, outVer, vbFlag, vbOnly); + vss->release(VSS::READ); + return rc; + } + catch (exception& e) + { + if (locked) + vss->release(VSS::READ); - cerr << e.what() << endl; - return -1; - } + cerr << e.what() << endl; + return -1; + } } -int DBRM::bulkVSSLookup(const std::vector& lbids, const QueryContext_vss& verInfo, - VER_t txnID, std::vector* out) +int DBRM::bulkVSSLookup(const std::vector& lbids, const QueryContext_vss& verInfo, VER_t txnID, + std::vector* out) { - uint32_t i; - bool locked = false; + uint32_t i; + bool locked = false; - try + try + { + out->resize(lbids.size()); + vss->lock(VSS::READ); + locked = true; + + if (vss->isEmpty(false)) { - out->resize(lbids.size()); - vss->lock(VSS::READ); - locked = true; - - if (vss->isEmpty(false)) - { - for (i = 0; i < lbids.size(); i++) - { - VSSData& vd = (*out)[i]; - vd.verID = 0; - vd.vbFlag = false; - vd.returnCode = -1; - } - } - else - { - for (i = 0; i < lbids.size(); i++) - { - VSSData& vd = (*out)[i]; - vd.returnCode = vss->lookup(lbids[i], verInfo, txnID, &vd.verID, &vd.vbFlag, false); - } - } - - vss->release(VSS::READ); - return 0; + for (i = 0; i < lbids.size(); i++) + { + VSSData& vd = (*out)[i]; + vd.verID = 0; + vd.vbFlag = false; + vd.returnCode = -1; + } } - catch (exception& e) + else { - cerr << e.what() << endl; - } - catch (...) - { - cerr << "bulkVSSLookup: caught an exception" << endl; + for (i = 0; i < lbids.size(); i++) + { + VSSData& vd = (*out)[i]; + vd.returnCode = vss->lookup(lbids[i], verInfo, txnID, &vd.verID, &vd.vbFlag, false); + } } - if (locked) - vss->release(VSS::READ); + vss->release(VSS::READ); + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + } + catch (...) + { + cerr << "bulkVSSLookup: caught an exception" << endl; + } - out->clear(); - return -1; + if (locked) + vss->release(VSS::READ); + + out->clear(); + return -1; } VER_t DBRM::getCurrentVersion(LBID_t lbid, bool* isLocked) const { - bool locked = false; - VER_t ret = 0; + bool locked = false; + VER_t ret = 0; - try - { - vss->lock(VSS::READ); - locked = true; - ret = vss->getCurrentVersion(lbid, isLocked); - vss->release(VSS::READ); - locked = false; - } - catch (exception& e) - { - cerr << e.what() << endl; + try + { + vss->lock(VSS::READ); + locked = true; + ret = vss->getCurrentVersion(lbid, isLocked); + vss->release(VSS::READ); + locked = false; + } + catch (exception& e) + { + cerr << e.what() << endl; - if (locked) - vss->release(VSS::READ); + if (locked) + vss->release(VSS::READ); - throw; - } + throw; + } - return ret; + return ret; } int DBRM::bulkGetCurrentVersion(const vector& lbids, vector* versions, vector* isLocked) const { - bool locked = false; + bool locked = false; - versions->resize(lbids.size()); + versions->resize(lbids.size()); + + if (isLocked != NULL) + isLocked->resize(lbids.size()); + + try + { + vss->lock(VSS::READ); + locked = true; if (isLocked != NULL) - isLocked->resize(lbids.size()); - - try { - vss->lock(VSS::READ); - locked = true; + bool tmp = false; - if (isLocked != NULL) - { - bool tmp = false; - - for (uint32_t i = 0; i < lbids.size(); i++) - { - (*versions)[i] = vss->getCurrentVersion(lbids[i], &tmp); - (*isLocked)[i] = tmp; - } - } - else - for (uint32_t i = 0; i < lbids.size(); i++) - (*versions)[i] = vss->getCurrentVersion(lbids[i], NULL); - - vss->release(VSS::READ); - locked = false; - return 0; + for (uint32_t i = 0; i < lbids.size(); i++) + { + (*versions)[i] = vss->getCurrentVersion(lbids[i], &tmp); + (*isLocked)[i] = tmp; + } } - catch (exception& e) - { - versions->clear(); - cerr << e.what() << endl; + else + for (uint32_t i = 0; i < lbids.size(); i++) + (*versions)[i] = vss->getCurrentVersion(lbids[i], NULL); - if (locked) - vss->release(VSS::READ); + vss->release(VSS::READ); + locked = false; + return 0; + } + catch (exception& e) + { + versions->clear(); + cerr << e.what() << endl; - return -1; - } + if (locked) + vss->release(VSS::READ); + + return -1; + } } - VER_t DBRM::getHighestVerInVB(LBID_t lbid, VER_t max) const { - bool locked = false; - VER_t ret = -1; + bool locked = false; + VER_t ret = -1; - try - { - vss->lock(VSS::READ); - locked = true; - ret = vss->getHighestVerInVB(lbid, max); - vss->release(VSS::READ); - locked = false; - } - catch (exception& e) - { - cerr << e.what() << endl; + try + { + vss->lock(VSS::READ); + locked = true; + ret = vss->getHighestVerInVB(lbid, max); + vss->release(VSS::READ); + locked = false; + } + catch (exception& e) + { + cerr << e.what() << endl; - if (locked) - vss->release(VSS::READ); + if (locked) + vss->release(VSS::READ); - throw; - } + throw; + } - return ret; + return ret; } bool DBRM::isVersioned(LBID_t lbid, VER_t ver) const { - bool ret = false; - bool locked = false; + bool ret = false; + bool locked = false; - try - { - vss->lock(VSS::READ); - locked = true; - ret = vss->isVersioned(lbid, ver); - vss->release(VSS::READ); - locked = false; - } - catch (exception& e) - { - cerr << e.what() << endl; + try + { + vss->lock(VSS::READ); + locked = true; + ret = vss->isVersioned(lbid, ver); + vss->release(VSS::READ); + locked = false; + } + catch (exception& e) + { + cerr << e.what() << endl; - if (locked) - vss->release(VSS::READ); + if (locked) + vss->release(VSS::READ); - throw; - } + throw; + } - return ret; + return ret; } int8_t DBRM::send_recv(const ByteStream& in, ByteStream& out) throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("send_recv"); + if (fDebug) + TRACER_WRITENOW("send_recv"); #endif - bool firstAttempt = true; - bool secondAttempt = true; + bool firstAttempt = true; + bool secondAttempt = true; - mutex.lock(); + mutex.lock(); reconnect: - if (msgClient == NULL) - try - { - msgClient = MessageQueueClientPool::getInstance(masterName); - } - catch (exception& e) - { - cerr << "class DBRM failed to create a MessageQueueClient: " << - e.what() << endl; - msgClient = NULL; - mutex.unlock(); - return ERR_NETWORK; - } - + if (msgClient == NULL) try { - msgClient->write(in); - out = msgClient->read(); + msgClient = MessageQueueClientPool::getInstance(masterName); } - /* If we add a timeout to the read() call, uncomment this clause - catch (SocketClosed &e) { - cerr << "DBRM::send_recv: controller node closed the connection" << endl; - DO_ERR_NETWORK; - } - */ catch (exception& e) { - cerr << "DBRM::send_recv caught: " << e.what() << endl; - - if (firstAttempt) - { - firstAttempt = false; - MessageQueueClientPool::releaseInstance(msgClient); - msgClient = NULL; - goto reconnect; - } - - DO_ERR_NETWORK; + cerr << "class DBRM failed to create a MessageQueueClient: " << e.what() << endl; + msgClient = NULL; + mutex.unlock(); + return ERR_NETWORK; } - if (out.length() == 0) + try + { + msgClient->write(in); + out = msgClient->read(); + } + /* If we add a timeout to the read() call, uncomment this clause + catch (SocketClosed &e) { + cerr << "DBRM::send_recv: controller node closed the connection" << endl; + DO_ERR_NETWORK; + } + */ + catch (exception& e) + { + cerr << "DBRM::send_recv caught: " << e.what() << endl; + + if (firstAttempt) { - cerr << "DBRM::send_recv: controller node closed the connection" << endl; - - if (secondAttempt) - { - MessageQueueClientPool::releaseInstance(msgClient); - msgClient = NULL; - if (!firstAttempt) - { - secondAttempt = false; - sleep(3); - } - firstAttempt = false; - goto reconnect; - } - - DO_ERR_NETWORK; + firstAttempt = false; + MessageQueueClientPool::releaseInstance(msgClient); + msgClient = NULL; + goto reconnect; } - mutex.unlock(); - return ERR_OK; + DO_ERR_NETWORK; + } + + if (out.length() == 0) + { + cerr << "DBRM::send_recv: controller node closed the connection" << endl; + + if (secondAttempt) + { + MessageQueueClientPool::releaseInstance(msgClient); + msgClient = NULL; + if (!firstAttempt) + { + secondAttempt = false; + sleep(3); + } + firstAttempt = false; + goto reconnect; + } + + DO_ERR_NETWORK; + } + + mutex.unlock(); + return ERR_OK; } //------------------------------------------------------------------------------ // Send a request to create a "stripe" of column extents for the specified // column OIDs and DBRoot. //------------------------------------------------------------------------------ -int DBRM::createStripeColumnExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - std::vector& extents) DBRM_THROW +int DBRM::createStripeColumnExtents(const std::vector& cols, uint16_t dbRoot, + uint32_t& partitionNum, uint16_t& segmentNum, + std::vector& extents) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createStripeColumnExtents"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createStripeColumnExtents"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint16_t tmp16; - uint32_t tmp32; + ByteStream command, response; + uint8_t err; + uint16_t tmp16; + uint32_t tmp32; - command << CREATE_STRIPE_COLUMN_EXTENTS; - serializeInlineVector(command, cols); - command << dbRoot << partitionNum; + command << CREATE_STRIPE_COLUMN_EXTENTS; + serializeInlineVector(command, cols); + command << dbRoot << partitionNum; - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() == 0) - return ERR_NETWORK; + if (response.length() == 0) + return ERR_NETWORK; - try - { - response >> err; + try + { + response >> err; - if (err != 0) - return (int) err; + if (err != 0) + return (int)err; - response >> tmp32; - partitionNum = tmp32; - response >> tmp16; - segmentNum = tmp16; - deserializeInlineVector(response, extents); - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + response >> tmp32; + partitionNum = tmp32; + response >> tmp16; + segmentNum = tmp16; + deserializeInlineVector(response, extents); + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - CHECK_EMPTY(response); - return 0; + CHECK_EMPTY(response); + return 0; } //------------------------------------------------------------------------------ // Send a request to create a column extent for the specified OID and DBRoot. //------------------------------------------------------------------------------ -int DBRM::createColumnExtent_DBroot(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t& partitionNum, +int DBRM::createColumnExtent_DBroot(OID_t oid, uint32_t colWidth, uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) DBRM_THROW + execplan::CalpontSystemCatalog::ColDataType colDataType, LBID_t& lbid, + int& allocdSize, uint32_t& startBlockOffset) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createColumnExtent_DBroot"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(colWidth); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDINT64OUTPUT(lbid); - TRACER_ADDOUTPUT(allocdSize); - TRACER_ADDOUTPUT(startBlockOffset); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createColumnExtent_DBroot"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(colWidth); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDINT64OUTPUT(lbid); + TRACER_ADDOUTPUT(allocdSize); + TRACER_ADDOUTPUT(startBlockOffset); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint32_t tmp8 = (uint8_t)colDataType; - uint16_t tmp16; - uint32_t tmp32; - uint64_t tmp64; + ByteStream command, response; + uint8_t err; + uint32_t tmp8 = (uint8_t)colDataType; + uint16_t tmp16; + uint32_t tmp32; + uint64_t tmp64; - command << CREATE_COLUMN_EXTENT_DBROOT << (ByteStream::quadbyte) oid << - colWidth << dbRoot << partitionNum << segmentNum << tmp8; - err = send_recv(command, response); + command << CREATE_COLUMN_EXTENT_DBROOT << (ByteStream::quadbyte)oid << colWidth << dbRoot << partitionNum + << segmentNum << tmp8; + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() == 0) - return ERR_NETWORK; + if (response.length() == 0) + return ERR_NETWORK; - try - { - response >> err; + try + { + response >> err; - if (err != 0) - return (int) err; + if (err != 0) + return (int)err; - response >> tmp32; - partitionNum = tmp32; - response >> tmp16; - segmentNum = tmp16; - response >> tmp64; - lbid = (int64_t)tmp64; - response >> tmp32; - allocdSize = (int32_t)tmp32; - response >> tmp32; - startBlockOffset = (int32_t)tmp32; - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + response >> tmp32; + partitionNum = tmp32; + response >> tmp16; + segmentNum = tmp16; + response >> tmp64; + lbid = (int64_t)tmp64; + response >> tmp32; + allocdSize = (int32_t)tmp32; + response >> tmp32; + startBlockOffset = (int32_t)tmp32; + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - CHECK_EMPTY(response); - return 0; + CHECK_EMPTY(response); + return 0; } //------------------------------------------------------------------------------ // Send a request to create a column extent for the exact segment file // specified by the requested OID, DBRoot, partition, and segment. //------------------------------------------------------------------------------ -int DBRM::createColumnExtentExactFile(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, +int DBRM::createColumnExtentExactFile(OID_t oid, uint32_t colWidth, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) DBRM_THROW + execplan::CalpontSystemCatalog::ColDataType colDataType, LBID_t& lbid, + int& allocdSize, uint32_t& startBlockOffset) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createColumnExtentExactFile"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(colWidth); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDINT64OUTPUT(lbid); - TRACER_ADDOUTPUT(allocdSize); - TRACER_ADDOUTPUT(startBlockOffset); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createColumnExtentExactFile"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(colWidth); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDINT64OUTPUT(lbid); + TRACER_ADDOUTPUT(allocdSize); + TRACER_ADDOUTPUT(startBlockOffset); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint8_t tmp8; - uint16_t tmp16; - uint32_t tmp32; - uint64_t tmp64; + ByteStream command, response; + uint8_t err; + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + uint64_t tmp64; - tmp8 = (uint8_t)colDataType; - command << CREATE_COLUMN_EXTENT_EXACT_FILE << (ByteStream::quadbyte) oid << - colWidth << dbRoot << partitionNum << segmentNum << tmp8; - err = send_recv(command, response); + tmp8 = (uint8_t)colDataType; + command << CREATE_COLUMN_EXTENT_EXACT_FILE << (ByteStream::quadbyte)oid << colWidth << dbRoot + << partitionNum << segmentNum << tmp8; + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() == 0) - return ERR_NETWORK; + if (response.length() == 0) + return ERR_NETWORK; - try - { - response >> err; + try + { + response >> err; - if (err != 0) - return (int) err; + if (err != 0) + return (int)err; - response >> tmp32; - partitionNum = tmp32; - response >> tmp16; - segmentNum = tmp16; - response >> tmp64; - lbid = (int64_t)tmp64; - response >> tmp32; - allocdSize = (int32_t)tmp32; - response >> tmp32; - startBlockOffset = (int32_t)tmp32; - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + response >> tmp32; + partitionNum = tmp32; + response >> tmp16; + segmentNum = tmp16; + response >> tmp64; + lbid = (int64_t)tmp64; + response >> tmp32; + allocdSize = (int32_t)tmp32; + response >> tmp32; + startBlockOffset = (int32_t)tmp32; + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - CHECK_EMPTY(response); - return 0; + CHECK_EMPTY(response); + return 0; } //------------------------------------------------------------------------------ // Send a request to create a dictionary store extent. //------------------------------------------------------------------------------ -int DBRM::createDictStoreExtent(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - LBID_t& lbid, - int& allocdSize) DBRM_THROW +int DBRM::createDictStoreExtent(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum, + LBID_t& lbid, int& allocdSize) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createDictStoreExtent"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINT64OUTPUT(lbid); - TRACER_ADDOUTPUT(allocdSize); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createDictStoreExtent"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINT64OUTPUT(lbid); + TRACER_ADDOUTPUT(allocdSize); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint32_t tmp32; - uint64_t tmp64; + ByteStream command, response; + uint8_t err; + uint32_t tmp32; + uint64_t tmp64; - command << CREATE_DICT_STORE_EXTENT << (ByteStream::quadbyte) oid << dbRoot << - partitionNum << segmentNum; - err = send_recv(command, response); + command << CREATE_DICT_STORE_EXTENT << (ByteStream::quadbyte)oid << dbRoot << partitionNum << segmentNum; + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() == 0) - return ERR_NETWORK; + if (response.length() == 0) + return ERR_NETWORK; - try - { - response >> err; + try + { + response >> err; - if (err != 0) - return (int) err; + if (err != 0) + return (int)err; - response >> tmp64; - lbid = (int64_t)tmp64; - response >> tmp32; - allocdSize = (int32_t)tmp32; - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + response >> tmp64; + lbid = (int64_t)tmp64; + response >> tmp32; + allocdSize = (int32_t)tmp32; + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - CHECK_EMPTY(response); - return 0; + CHECK_EMPTY(response); + return 0; } //------------------------------------------------------------------------------ @@ -1230,46 +1205,41 @@ int DBRM::createDictStoreExtent(OID_t oid, // DBRoot, and to return the extents to the free list. HWMs for the last // stripe of extents in the specified DBRoot are updated accordingly. //------------------------------------------------------------------------------ -int DBRM::rollbackColumnExtents_DBroot(OID_t oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm) DBRM_THROW +int DBRM::rollbackColumnExtents_DBroot(OID_t oid, bool bDeleteAll, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, HWM_t hwm) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("rollbackColumnExtents"); - TRACER_ADDINPUT(oid); - TRACER_ADDBOOLINPUT(bDeleteAll); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(hwm); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("rollbackColumnExtents"); + TRACER_ADDINPUT(oid); + TRACER_ADDBOOLINPUT(bDeleteAll); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(hwm); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << ROLLBACK_COLUMN_EXTENTS_DBROOT << (ByteStream::quadbyte) oid << - (uint8_t)bDeleteAll << dbRoot << partitionNum << - segmentNum << hwm; - err = send_recv(command, response); + command << ROLLBACK_COLUMN_EXTENTS_DBROOT << (ByteStream::quadbyte)oid << (uint8_t)bDeleteAll << dbRoot + << partitionNum << segmentNum << hwm; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ @@ -1277,206 +1247,206 @@ int DBRM::rollbackColumnExtents_DBroot(OID_t oid, // OID and DBRoot, and to return the extents to the free list. HWMs for the // last stripe of extents are updated accordingly. //------------------------------------------------------------------------------ -int DBRM::rollbackDictStoreExtents_DBroot(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - const vector& segNums, - const vector& hwms) DBRM_THROW +int DBRM::rollbackDictStoreExtents_DBroot(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, + const vector& segNums, + const vector& hwms) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("rollbackDictStoreExtents"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDINPUT(partitionNum); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("rollbackDictStoreExtents"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDINPUT(partitionNum); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << ROLLBACK_DICT_STORE_EXTENTS_DBROOT << - (ByteStream::quadbyte) oid << - dbRoot << partitionNum; - serializeVector(command, segNums); - serializeVector(command, hwms); - err = send_recv(command, response); + command << ROLLBACK_DICT_STORE_EXTENTS_DBROOT << (ByteStream::quadbyte)oid << dbRoot << partitionNum; + serializeVector(command, segNums); + serializeVector(command, hwms); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::deleteEmptyColExtents(const std::vector& extentsInfo) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteEmptyColExtents"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteEmptyColExtents"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint32_t size = extentsInfo.size(); - command << DELETE_EMPTY_COL_EXTENTS; - command << size; + ByteStream command, response; + uint8_t err; + uint32_t size = extentsInfo.size(); + command << DELETE_EMPTY_COL_EXTENTS; + command << size; - for ( unsigned i = 0; i < extentsInfo.size(); i++) - { - command << (ByteStream::quadbyte) extentsInfo[i].oid; - command << extentsInfo[i].partitionNum; - command << extentsInfo[i].segmentNum; - command << extentsInfo[i].dbRoot; - command << extentsInfo[i].hwm; - } + for (unsigned i = 0; i < extentsInfo.size(); i++) + { + command << (ByteStream::quadbyte)extentsInfo[i].oid; + command << extentsInfo[i].partitionNum; + command << extentsInfo[i].segmentNum; + command << extentsInfo[i].dbRoot; + command << extentsInfo[i].hwm; + } - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::deleteEmptyDictStoreExtents(const std::vector& extentsInfo) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteEmptyDictStoreExtents"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteEmptyDictStoreExtents"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint32_t size = extentsInfo.size(); - command << DELETE_EMPTY_DICT_STORE_EXTENTS; - command << size; + ByteStream command, response; + uint8_t err; + uint32_t size = extentsInfo.size(); + command << DELETE_EMPTY_DICT_STORE_EXTENTS; + command << size; - for ( unsigned i = 0; i < extentsInfo.size(); i++) - { - command << (ByteStream::quadbyte) extentsInfo[i].oid; - command << extentsInfo[i].partitionNum; - command << extentsInfo[i].segmentNum; - command << extentsInfo[i].dbRoot; - command << extentsInfo[i].hwm; - command << (uint8_t)extentsInfo[i].newFile; - } + for (unsigned i = 0; i < extentsInfo.size(); i++) + { + command << (ByteStream::quadbyte)extentsInfo[i].oid; + command << extentsInfo[i].partitionNum; + command << extentsInfo[i].segmentNum; + command << extentsInfo[i].dbRoot; + command << extentsInfo[i].hwm; + command << (uint8_t)extentsInfo[i].newFile; + } - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::deleteOID(OID_t oid) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteOID"); - TRACER_ADDINPUT(oid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteOID"); + TRACER_ADDINPUT(oid); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << DELETE_OID << (ByteStream::quadbyte) oid; - err = send_recv(command, response); - - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); - - try - { - deleteAISequence(oid); - } - catch (...) { } // an error here means a network problem, will be caught elsewhere + command << DELETE_OID << (ByteStream::quadbyte)oid; + err = send_recv(command, response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + + try + { + deleteAISequence(oid); + } + catch (...) + { + } // an error here means a network problem, will be caught elsewhere + + return err; } int DBRM::deleteOIDs(const std::vector& oids) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteOIDs"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteOIDs"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; - uint32_t size = oids.size(); - command << DELETE_OIDS; - command << size; + ByteStream command, response; + uint8_t err; + uint32_t size = oids.size(); + command << DELETE_OIDS; + command << size; - for ( unsigned i = 0; i < oids.size(); i++) - { - command << (ByteStream::quadbyte) oids[i]; - } + for (unsigned i = 0; i < oids.size(); i++) + { + command << (ByteStream::quadbyte)oids[i]; + } - err = send_recv(command, response); - - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); - - try - { - for (uint32_t i = 0; i < oids.size(); i++) - deleteAISequence(oids[i]); - } - catch (...) { } // an error here means a network problem, will be caught elsewhere + err = send_recv(command, response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + + try + { + for (uint32_t i = 0; i < oids.size(); i++) + deleteAISequence(oids[i]); + } + catch (...) + { + } // an error here means a network problem, will be caught elsewhere + + return err; } //------------------------------------------------------------------------------ @@ -1487,37 +1457,35 @@ int DBRM::deleteOIDs(const std::vector& oids) DBRM_THROW // If no available or outOfService extent is found, then bFound is returned // as false. //------------------------------------------------------------------------------ -int DBRM::getLastHWM_DBroot(int oid, uint16_t dbRoot, uint32_t& partitionNum, - uint16_t& segmentNum, HWM_t& hwm, - int& status, bool& bFound) throw() +int DBRM::getLastHWM_DBroot(int oid, uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + HWM_t& hwm, int& status, bool& bFound) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getLastHWM_DBroot"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTOUTPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDOUTPUT(hwm); - TRACER_ADDOUTPUT(status); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getLastHWM_DBroot"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTOUTPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDOUTPUT(hwm); + TRACER_ADDOUTPUT(status); + TRACER_WRITE; + } #endif - try - { - hwm = em->getLastHWM_DBroot(oid, dbRoot, partitionNum, segmentNum, - status, bFound); - } - catch (exception& e) - { - return ERR_FAILURE; - } + try + { + hwm = em->getLastHWM_DBroot(oid, dbRoot, partitionNum, segmentNum, status, bFound); + } + catch (exception& e) + { + return ERR_FAILURE; + } - return ERR_OK; + return ERR_OK; } //------------------------------------------------------------------------------ @@ -1525,104 +1493,102 @@ int DBRM::getLastHWM_DBroot(int oid, uint16_t dbRoot, uint32_t& partitionNum, // This is used to get the HWM for a particular dictionary segment store file, // or a specific column segment file. //------------------------------------------------------------------------------ -int DBRM::getLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, - HWM_t& hwm, int& status) throw() +int DBRM::getLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t& hwm, int& status) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getLocalHWM"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDOUTPUT(hwm); - TRACER_ADDOUTPUT(status); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getLocalHWM"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDOUTPUT(hwm); + TRACER_ADDOUTPUT(status); + TRACER_WRITE; + } #endif - try - { - hwm = em->getLocalHWM(oid, partitionNum, segmentNum, status); - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + try + { + hwm = em->getLocalHWM(oid, partitionNum, segmentNum, status); + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - return ERR_OK; + return ERR_OK; } //------------------------------------------------------------------------------ // Set the local HWM for the file referenced by the specified OID, partition // number, and segment number. //------------------------------------------------------------------------------ -int DBRM::setLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, - HWM_t hwm) DBRM_THROW +int DBRM::setLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setLocalHWM"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(hwm); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("setLocalHWM"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(hwm); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << SET_LOCAL_HWM << (ByteStream::quadbyte) oid << partitionNum << segmentNum << hwm; - err = send_recv(command, response); + command << SET_LOCAL_HWM << (ByteStream::quadbyte)oid << partitionNum << segmentNum << hwm; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::bulkSetHWM(const vector& v, VER_t transID) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("bulkSetHWM"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("bulkSetHWM"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << BULK_SET_HWM; - serializeInlineVector(command, v); - command << (uint32_t) transID; - err = send_recv(command, response); + command << BULK_SET_HWM; + serializeInlineVector(command, v); + command << (uint32_t)transID; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::bulkSetHWMAndCP(const vector& v, const vector& setCPDataArgs, @@ -1630,132 +1596,129 @@ int DBRM::bulkSetHWMAndCP(const vector& v, const vector& { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("bulkSetHWMAndCP"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("bulkSetHWMAndCP"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << BULK_SET_HWM_AND_CP; - serializeInlineVector(command, v); - serializeInlineVector(command, setCPDataArgs); - serializeInlineVector(command, mergeCPDataArgs); - command << (uint32_t) transID; - err = send_recv(command, response); + command << BULK_SET_HWM_AND_CP; + serializeInlineVector(command, v); + serializeInlineVector(command, setCPDataArgs); + serializeInlineVector(command, mergeCPDataArgs); + command << (uint32_t)transID; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::bulkUpdateDBRoot(const vector& args) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("bulkUpdateDBRoot"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("bulkUpdateDBRoot"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << BULK_UPDATE_DBROOT; - serializeInlineVector(command, args); - err = send_recv(command, response); + command << BULK_UPDATE_DBROOT; + serializeInlineVector(command, args); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; -} + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; +} //------------------------------------------------------------------------------ // For the specified OID and PM number, this function will return a vector // of objects carrying HWM info (for the last segment file) and block count // information about each DBRoot assigned to the specified PM. //------------------------------------------------------------------------------ -int DBRM::getDbRootHWMInfo(OID_t oid, uint16_t pmNumber, - EmDbRootHWMInfo_v& emDBRootHwmInfos) throw() +int DBRM::getDbRootHWMInfo(OID_t oid, uint16_t pmNumber, EmDbRootHWMInfo_v& emDBRootHwmInfos) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getDbRootHWMInfo"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTINPUT(pmNumber); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getDbRootHWMInfo"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTINPUT(pmNumber); + TRACER_WRITE; + } #endif - try - { - em->getDbRootHWMInfo(oid, pmNumber, emDBRootHwmInfos); - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + try + { + em->getDbRootHWMInfo(oid, pmNumber, emDBRootHwmInfos); + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - return ERR_OK; + return ERR_OK; } //------------------------------------------------------------------------------ // Return the status or state of the extents in the segment file specified // by the arguments: oid, partitionNum, and segment Num. //------------------------------------------------------------------------------ -int DBRM::getExtentState(OID_t oid, uint32_t partitionNum, - uint16_t segmentNum, bool& bFound, int& status) throw() +int DBRM::getExtentState(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, bool& bFound, + int& status) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtentState"); - TRACER_ADDINPUT(oid); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDOUTPUT(status); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtentState"); + TRACER_ADDINPUT(oid); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDOUTPUT(status); + TRACER_WRITE; + } #endif - try - { - em->getExtentState(oid, partitionNum, - segmentNum, bFound, status); - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_FAILURE; - } + try + { + em->getExtentState(oid, partitionNum, segmentNum, bFound, status); + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_FAILURE; + } - return ERR_OK; + return ERR_OK; } // dmc-should eventually deprecate @@ -1763,105 +1726,105 @@ int DBRM::getExtentSize() throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("getExtentSize"); + if (fDebug) + TRACER_WRITENOW("getExtentSize"); #endif - return em->getExtentSize(); + return em->getExtentSize(); } unsigned DBRM::getExtentRows() throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("getExtentRows"); + if (fDebug) + TRACER_WRITENOW("getExtentRows"); #endif - return em->getExtentRows(); + return em->getExtentRows(); } -int DBRM::getExtents(int OID, std::vector& entries, - bool sorted, bool notFoundErr, bool incOutOfService) +int DBRM::getExtents(int OID, std::vector& entries, bool sorted, bool notFoundErr, + bool incOutOfService) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtents"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtents"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif - try - { - em->getExtents(OID, entries, sorted, notFoundErr, incOutOfService); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->getExtents(OID, entries, sorted, notFoundErr, incOutOfService); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } -int DBRM::getExtents_dbroot(int OID, std::vector& entries, - const uint16_t dbroot) throw() +int DBRM::getExtents_dbroot(int OID, std::vector& entries, const uint16_t dbroot) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtents_dbroot"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtents_dbroot"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif - try - { - em->getExtents_dbroot(OID, entries, dbroot); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->getExtents_dbroot(OID, entries, dbroot); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Return the number of extents for the specified OID and DBRoot. // Any out-of-service extents can optionally be included or excluded. //------------------------------------------------------------------------------ -int DBRM::getExtentCount_dbroot(int OID, uint16_t dbroot, - bool incOutOfService, uint64_t& numExtents) throw() +int DBRM::getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfService, uint64_t& numExtents) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtentCount_dbroot"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtentCount_dbroot"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif - try - { - em->getExtentCount_dbroot(OID, dbroot, incOutOfService, numExtents); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->getExtentCount_dbroot(OID, dbroot, incOutOfService, numExtents); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -1874,85 +1837,85 @@ int DBRM::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getSysCatDBRoot"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTOUTPUT(dbRoot); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getSysCatDBRoot"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTOUTPUT(dbRoot); + TRACER_WRITE; + } #endif - try - { - em->getSysCatDBRoot(oid, dbRoot); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->getSysCatDBRoot(oid, dbRoot); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Delete all extents for the specified OID(s) and partition number. //------------------------------------------------------------------------------ -int DBRM::deletePartition(const std::vector& oids, - const std::set& partitionNums, string& emsg) DBRM_THROW +int DBRM::deletePartition(const std::vector& oids, const std::set& partitionNums, + string& emsg) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("deletePartition"); + std::ostringstream oss; + oss << "partitionNum: "; + std::set::const_iterator partIt; + + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + oss << (*partIt) << " "; + oss << "; OIDS: "; + + std::vector::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("deletePartition"); - std::ostringstream oss; - oss << "partitionNum: "; - std::set::const_iterator partIt; - - for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*partIt) << " "; - oss << "; OIDS: "; - - std::vector::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - ByteStream command, response; - uint8_t err; - command << DELETE_PARTITION; - serializeSet(command, partitionNums); - uint32_t oidSize = oids.size(); - command << oidSize; + ByteStream command, response; + uint8_t err; + command << DELETE_PARTITION; + serializeSet(command, partitionNums); + uint32_t oidSize = oids.size(); + command << oidSize; - for ( unsigned i = 0; i < oidSize; i++) - command << (ByteStream::quadbyte) oids[i]; + for (unsigned i = 0; i < oidSize; i++) + command << (ByteStream::quadbyte)oids[i]; - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - - if (err != 0) - response >> emsg; - - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + + if (err != 0) + response >> emsg; + + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ @@ -1964,54 +1927,54 @@ int DBRM::markPartitionForDeletion(const std::vector& oids, { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("markPartitionForDeletion"); + std::ostringstream oss; + oss << "partitionNum: "; + std::set::const_iterator partIt; + + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + oss << (*partIt) << " "; + oss << "; OIDS: "; + + std::vector::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("markPartitionForDeletion"); - std::ostringstream oss; - oss << "partitionNum: "; - std::set::const_iterator partIt; - - for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*partIt) << " "; - oss << "; OIDS: "; - - std::vector::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - ByteStream command, response; - uint8_t err; - command << MARK_PARTITION_FOR_DELETION; - serializeSet(command, partitionNums); - uint32_t oidSize = oids.size(); - command << oidSize; + ByteStream command, response; + uint8_t err; + command << MARK_PARTITION_FOR_DELETION; + serializeSet(command, partitionNums); + uint32_t oidSize = oids.size(); + command << oidSize; - for ( unsigned i = 0; i < oidSize; i++) - command << (ByteStream::quadbyte) oids[i]; + for (unsigned i = 0; i < oidSize; i++) + command << (ByteStream::quadbyte)oids[i]; - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - - if (err) - response >> emsg; - - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + + if (err) + response >> emsg; + + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ @@ -2021,134 +1984,133 @@ int DBRM::markAllPartitionForDeletion(const std::vector& oids) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("markAllPartitionForDeletion"); + std::ostringstream oss; + oss << "OIDS: "; + std::vector::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("markAllPartitionForDeletion"); - std::ostringstream oss; - oss << "OIDS: "; - std::vector::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - ByteStream command, response; - uint8_t err; - uint32_t size = oids.size(); + ByteStream command, response; + uint8_t err; + uint32_t size = oids.size(); - command << MARK_ALL_PARTITION_FOR_DELETION << size; + command << MARK_ALL_PARTITION_FOR_DELETION << size; - for ( unsigned i = 0; i < size; i++) - { - command << (ByteStream::quadbyte) oids[i]; - } + for (unsigned i = 0; i < size; i++) + { + command << (ByteStream::quadbyte)oids[i]; + } - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ // Restore all extents for the specified OID(s) and partition number. //------------------------------------------------------------------------------ -int DBRM::restorePartition(const std::vector& oids, - const std::set& partitionNums, string& emsg) DBRM_THROW +int DBRM::restorePartition(const std::vector& oids, const std::set& partitionNums, + string& emsg) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("restorePartition"); + std::ostringstream oss; + oss << "partitionNum: "; + std::set::const_iterator partIt; + + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + oss << (*partIt) << " "; + oss << "; OIDS: "; + + std::vector::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("restorePartition"); - std::ostringstream oss; - oss << "partitionNum: "; - std::set::const_iterator partIt; - - for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*partIt) << " "; - oss << "; OIDS: "; - - std::vector::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RESTORE_PARTITION; - serializeSet(command, partitionNums); - uint32_t oidSize = oids.size(); - command << oidSize; + command << RESTORE_PARTITION; + serializeSet(command, partitionNums); + uint32_t oidSize = oids.size(); + command << oidSize; - for ( unsigned i = 0; i < oidSize; i++) - command << (ByteStream::quadbyte) oids[i]; + for (unsigned i = 0; i < oidSize; i++) + command << (ByteStream::quadbyte)oids[i]; - err = send_recv(command, response); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - - if (err) - response >> emsg; - - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + + if (err) + response >> emsg; + + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ // Return all the out-of-service partitions for the specified OID. //------------------------------------------------------------------------------ -int DBRM::getOutOfServicePartitions(OID_t oid, - std::set& partitionNums) throw() +int DBRM::getOutOfServicePartitions(OID_t oid, std::set& partitionNums) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getOutOfServicePartitions"); - TRACER_ADDINPUT(oid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getOutOfServicePartitions"); + TRACER_ADDINPUT(oid); + TRACER_WRITE; + } #endif - try - { - em->getOutOfServicePartitions(oid, partitionNums); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->getOutOfServicePartitions(oid, partitionNums); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return ERR_OK; + return ERR_OK; } //------------------------------------------------------------------------------ @@ -2158,1597 +2120,1585 @@ int DBRM::deleteDBRoot(uint16_t dbroot) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITENOW("deleteDBRoot"); - std::ostringstream oss; - oss << "DBRoot: " << dbroot; - TRACER_WRITEDIRECT(oss.str()); - } + if (fDebug) + { + TRACER_WRITENOW("deleteDBRoot"); + std::ostringstream oss; + oss << "DBRoot: " << dbroot; + TRACER_WRITEDIRECT(oss.str()); + } #endif - ByteStream command, response; - uint8_t err; - command << DELETE_DBROOT; - uint32_t q = static_cast(dbroot); - command << q; - err = send_recv(command, response); + ByteStream command, response; + uint8_t err; + command << DELETE_DBROOT; + uint32_t q = static_cast(dbroot); + command << q; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() == 0) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() == 0) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } //------------------------------------------------------------------------------ // Does the specified DBRoot have any extents. // Returns an error if extentmap shared memory is not loaded. //------------------------------------------------------------------------------ -int DBRM::isDBRootEmpty(uint16_t dbroot, - bool& isEmpty, std::string& errMsg) throw() +int DBRM::isDBRootEmpty(uint16_t dbroot, bool& isEmpty, std::string& errMsg) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("isDBRootEmpty"); - TRACER_ADDINPUT(dbroot); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("isDBRootEmpty"); + TRACER_ADDINPUT(dbroot); + TRACER_WRITE; + } #endif - errMsg.clear(); + errMsg.clear(); - try - { - isEmpty = em->isDBRootEmpty(dbroot); - } - catch (exception& e) - { - cerr << e.what() << endl; - errMsg = e.what(); - return ERR_FAILURE; - } + try + { + isEmpty = em->isDBRootEmpty(dbroot); + } + catch (exception& e) + { + cerr << e.what() << endl; + errMsg = e.what(); + return ERR_FAILURE; + } - return ERR_OK; + return ERR_OK; } -int DBRM::writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, - uint32_t vbFBO) DBRM_THROW +int DBRM::writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, uint32_t vbFBO) DBRM_THROW { - #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("writeVBEntry"); - TRACER_ADDINPUT(transID); - TRACER_ADDINPUT(lbid); - TRACER_ADDINPUT(vbOID); - TRACER_ADDINPUT(vbFBO); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("writeVBEntry"); + TRACER_ADDINPUT(transID); + TRACER_ADDINPUT(lbid); + TRACER_ADDINPUT(vbOID); + TRACER_ADDINPUT(vbFBO); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << WRITE_VB_ENTRY << (uint32_t) transID << (uint64_t) lbid << (uint32_t) vbOID << vbFBO; - err = send_recv(command, response); + command << WRITE_VB_ENTRY << (uint32_t)transID << (uint64_t)lbid << (uint32_t)vbOID << vbFBO; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } -int DBRM::bulkWriteVBEntry(VER_t transID, - const std::vector& lbids, - OID_t vbOID, +int DBRM::bulkWriteVBEntry(VER_t transID, const std::vector& lbids, OID_t vbOID, const std::vector& vbFBOs) DBRM_THROW { - #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("bulkWriteVBEntry"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("bulkWriteVBEntry"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << BULK_WRITE_VB_ENTRY << (uint32_t) transID; - serializeInlineVector(command, lbids); - command << (uint32_t) vbOID; - serializeInlineVector(command, vbFBOs); - err = send_recv(command, response); + command << BULK_WRITE_VB_ENTRY << (uint32_t)transID; + serializeInlineVector(command, lbids); + command << (uint32_t)vbOID; + serializeInlineVector(command, vbFBOs); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } struct _entry { - _entry(LBID_t l) : lbid(l) { }; - LBID_t lbid; - inline bool operator<(const _entry& e) const - { - return ((e.lbid >> 10) < (lbid >> 10)); - } + _entry(LBID_t l) : lbid(l){}; + LBID_t lbid; + inline bool operator<(const _entry& e) const + { + return ((e.lbid >> 10) < (lbid >> 10)); + } }; int DBRM::getDBRootsForRollback(VER_t transID, vector* dbroots) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getDBRootsForRollback"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getDBRootsForRollback"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - bool locked[2] = {false, false}; - set vbOIDs; - set::iterator vbIt; - vector lbidList; - uint32_t i, size; - uint32_t tmp32; - OID_t vbOID; - int err; + bool locked[2] = {false, false}; + set vbOIDs; + set::iterator vbIt; + vector lbidList; + uint32_t i, size; + uint32_t tmp32; + OID_t vbOID; + int err; - set<_entry> lbidPruner; - set<_entry>::iterator it; + set<_entry> lbidPruner; + set<_entry>::iterator it; - try + try + { + vbbm->lock(VBBM::READ); + locked[0] = true; + vss->lock(VSS::READ); + locked[1] = true; + + vss->getUncommittedLBIDs(transID, lbidList); + + // prune the list; will leave at most 1 entry per 1024-lbid range + for (i = 0, size = lbidList.size(); i < size; i++) + lbidPruner.insert(_entry(lbidList[i])); + + // get the VB oids + for (it = lbidPruner.begin(); it != lbidPruner.end(); ++it) { - vbbm->lock(VBBM::READ); - locked[0] = true; - vss->lock(VSS::READ); - locked[1] = true; + err = vbbm->lookup(it->lbid, transID, vbOID, tmp32); - vss->getUncommittedLBIDs(transID, lbidList); + if (err) // this error will be caught by DML; more appropriate to handle it there + continue; - // prune the list; will leave at most 1 entry per 1024-lbid range - for (i = 0, size = lbidList.size(); i < size; i++) - lbidPruner.insert(_entry(lbidList[i])); - - // get the VB oids - for (it = lbidPruner.begin(); it != lbidPruner.end(); ++it) - { - err = vbbm->lookup(it->lbid, transID, vbOID, tmp32); - - if (err) // this error will be caught by DML; more appropriate to handle it there - continue; - - vbOIDs.insert(vbOID); - } - - // get the dbroots - for (vbIt = vbOIDs.begin(); vbIt != vbOIDs.end(); ++vbIt) - { - err = getDBRootOfVBOID(*vbIt); - - if (err) - { - ostringstream os; - os << "DBRM::getDBRootOfVBOID() returned an error looking for vbOID " << *vbIt; - log(os.str()); - return ERR_FAILURE; - } - - dbroots->push_back((uint16_t) err); - } - - vss->release(VSS::READ); - locked[1] = false; - vbbm->release(VBBM::READ); - locked[0] = false; - - return ERR_OK; - } - catch (exception& e) - { - if (locked[0]) - vbbm->release(VBBM::READ); - - if (locked[1]) - vss->release(VSS::READ); - - return -1; + vbOIDs.insert(vbOID); } + // get the dbroots + for (vbIt = vbOIDs.begin(); vbIt != vbOIDs.end(); ++vbIt) + { + err = getDBRootOfVBOID(*vbIt); + + if (err) + { + ostringstream os; + os << "DBRM::getDBRootOfVBOID() returned an error looking for vbOID " << *vbIt; + log(os.str()); + return ERR_FAILURE; + } + + dbroots->push_back((uint16_t)err); + } + + vss->release(VSS::READ); + locked[1] = false; + vbbm->release(VBBM::READ); + locked[0] = false; + + return ERR_OK; + } + catch (exception& e) + { + if (locked[0]) + vbbm->release(VBBM::READ); + + if (locked[1]) + vss->release(VSS::READ); + + return -1; + } } int DBRM::getUncommittedLBIDs(VER_t transID, vector& lbidList) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getUncommittedLBIDs"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getUncommittedLBIDs"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - bool locked = false; + bool locked = false; - try - { - vss->lock(VSS::READ); - locked = true; + try + { + vss->lock(VSS::READ); + locked = true; - vss->getUncommittedLBIDs(transID, lbidList); + vss->getUncommittedLBIDs(transID, lbidList); - vss->release(VSS::READ); - locked = false; - return 0; - } - catch (exception& e) - { - if (locked) - vss->release(VSS::READ); + vss->release(VSS::READ); + locked = false; + return 0; + } + catch (exception& e) + { + if (locked) + vss->release(VSS::READ); - return -1; - } + return -1; + } } -// @bug 1509. New function that returns one LBID per extent touched as part of the transaction. Used to get a list -// of blocks to use for updating casual partitioning when the transaction is committed. +// @bug 1509. New function that returns one LBID per extent touched as part of the transaction. Used to get +// a list of blocks to use for updating casual partitioning when the transaction is committed. int DBRM::getUncommittedExtentLBIDs(VER_t transID, vector& lbidList) throw() { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getUncommittedExtentLBIDs"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getUncommittedExtentLBIDs"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - bool locked = false; - vector::iterator lbidIt; - typedef pair range_t; - range_t range; - vector ranges; - vector::iterator rangeIt; + bool locked = false; + vector::iterator lbidIt; + typedef pair range_t; + range_t range; + vector ranges; + vector::iterator rangeIt; - try + try + { + vss->lock(VSS::READ); + locked = true; + + // Get a full list of uncommitted LBIDs related to this transactin. + vss->getUncommittedLBIDs(transID, lbidList); + + vss->release(VSS::READ); + locked = false; + + if (lbidList.size() > 0) { - vss->lock(VSS::READ); - locked = true; + // Sort the vector. + std::sort::iterator>(lbidList.begin(), lbidList.end()); - // Get a full list of uncommitted LBIDs related to this transactin. - vss->getUncommittedLBIDs(transID, lbidList); + // Get the LBID range for the first block in the list. + lbidIt = lbidList.begin(); - vss->release(VSS::READ); - locked = false; + if (em->lookup(*lbidIt, range.first, range.second) < 0) + { + return -1; + } - if (lbidList.size() > 0) + ranges.push_back(range); + + // Loop through the LBIDs and add the new ranges. + ++lbidIt; + + while (lbidIt != lbidList.end()) + { + if (*lbidIt > range.second) { + if (em->lookup(*lbidIt, range.first, range.second) < 0) + { + return -1; + } - // Sort the vector. - std::sort::iterator>(lbidList.begin(), lbidList.end()); - - // Get the LBID range for the first block in the list. - lbidIt = lbidList.begin(); - - if (em->lookup(*lbidIt, range.first, range.second) < 0) - { - return -1; - } - - ranges.push_back(range); - - // Loop through the LBIDs and add the new ranges. - ++lbidIt; - - while (lbidIt != lbidList.end()) - { - if (*lbidIt > range.second) - { - if (em->lookup(*lbidIt, range.first, range.second) < 0) - { - return -1; - } - - ranges.push_back(range); - } - - ++lbidIt; - } - - // Reset the lbidList and return only the first LBID in each extent that was changed - // in the transaction. - lbidList.clear(); - - for (rangeIt = ranges.begin(); rangeIt != ranges.end(); rangeIt++) - { - lbidList.push_back(rangeIt->first); - } + ranges.push_back(range); } - return 0; - } - catch (exception& e) - { - if (locked) - vss->release(VSS::READ); + ++lbidIt; + } - return -1; + // Reset the lbidList and return only the first LBID in each extent that was changed + // in the transaction. + lbidList.clear(); + + for (rangeIt = ranges.begin(); rangeIt != ranges.end(); rangeIt++) + { + lbidList.push_back(rangeIt->first); + } } + + return 0; + } + catch (exception& e) + { + if (locked) + vss->release(VSS::READ); + + return -1; + } } - int DBRM::beginVBCopy(VER_t transID, uint16_t dbRoot, const LBIDRange_v& ranges, VBRange_v& freeList) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("beginVBCopy"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("beginVBCopy"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << BEGIN_VB_COPY << (ByteStream::quadbyte) transID << dbRoot; - serializeVector(command, ranges); - err = send_recv(command, response); + command << BEGIN_VB_COPY << (ByteStream::quadbyte)transID << dbRoot; + serializeVector(command, ranges); + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() == 0) - return ERR_NETWORK; + if (response.length() == 0) + return ERR_NETWORK; - try - { - response >> err; + try + { + response >> err; - if (err != 0) - return err; + if (err != 0) + return err; - deserializeVector(response, freeList); - } - catch (exception& e) - { - cerr << e.what() << endl; - return ERR_NETWORK; - } + deserializeVector(response, freeList); + } + catch (exception& e) + { + cerr << e.what() << endl; + return ERR_NETWORK; + } - CHECK_EMPTY(response); - return 0; + CHECK_EMPTY(response); + return 0; } -int DBRM::endVBCopy(VER_t transID, const LBIDRange_v& ranges) -DBRM_THROW +int DBRM::endVBCopy(VER_t transID, const LBIDRange_v& ranges) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("endVBCopy"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("endVBCopy"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << END_VB_COPY << (ByteStream::quadbyte) transID; - serializeVector(command, ranges); - err = send_recv(command, response); + command << END_VB_COPY << (ByteStream::quadbyte)transID; + serializeVector(command, ranges); + err = send_recv(command, response); - if (response.length() != 1) - return ERR_NETWORK; + if (response.length() != 1) + return ERR_NETWORK; - response >> err; - CHECK_EMPTY(response); - return err; + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::vbCommit(VER_t transID) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("vbCommit"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("vbCommit"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << VB_COMMIT << (ByteStream::quadbyte) transID; - err = send_recv(command, response); + command << VB_COMMIT << (ByteStream::quadbyte)transID; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::vbRollback(VER_t transID, const LBIDRange_v& lbidList) DBRM_THROW { - #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("vbRollback "); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("vbRollback "); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << VB_ROLLBACK1 << (ByteStream::quadbyte) transID; - serializeVector(command, lbidList); - err = send_recv(command, response); + command << VB_ROLLBACK1 << (ByteStream::quadbyte)transID; + serializeVector(command, lbidList); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::vbRollback(VER_t transID, const vector& lbidList) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("vbRollback"); - TRACER_ADDINPUT(transID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("vbRollback"); + TRACER_ADDINPUT(transID); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << VB_ROLLBACK2 << (ByteStream::quadbyte) transID; - serializeVector(command, lbidList); - err = send_recv(command, response); + command << VB_ROLLBACK2 << (ByteStream::quadbyte)transID; + serializeVector(command, lbidList); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::halt() DBRM_THROW { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("halt"); + if (fDebug) + TRACER_WRITENOW("halt"); #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << HALT; - err = send_recv(command, response); + command << HALT; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::resume() DBRM_THROW { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("resume"); + if (fDebug) + TRACER_WRITENOW("resume"); #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RESUME; - err = send_recv(command, response); + command << RESUME; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::forceReload() DBRM_THROW { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("forceReload"); + if (fDebug) + TRACER_WRITENOW("forceReload"); #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RELOAD; - err = send_recv(command, response); + command << RELOAD; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::setReadOnly(bool b) DBRM_THROW { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setReadOnly"); - TRACER_ADDBOOLINPUT(b); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("setReadOnly"); + TRACER_ADDBOOLINPUT(b); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << (b ? SETREADONLY : SETREADWRITE); - err = send_recv(command, response); + command << (b ? SETREADONLY : SETREADWRITE); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::isReadWrite() throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("isReadWrite"); + if (fDebug) + TRACER_WRITENOW("isReadWrite"); #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << GETREADONLY; - err = send_recv(command, response); + command << GETREADONLY; + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() != 1) - return ERR_NETWORK; + if (response.length() != 1) + return ERR_NETWORK; - response >> err; - //CHECK_EMPTY(response); - return (err == 0 ? ERR_OK : ERR_READONLY); + response >> err; + // CHECK_EMPTY(response); + return (err == 0 ? ERR_OK : ERR_READONLY); } int DBRM::dmlLockLBIDRanges(const vector& ranges, int txnID) { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("clear"); + if (fDebug) + TRACER_WRITENOW("clear"); #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << LOCK_LBID_RANGES; - serializeVector(command, ranges); - command << (uint32_t) txnID; - err = send_recv(command, response); + command << LOCK_LBID_RANGES; + serializeVector(command, ranges); + command << (uint32_t)txnID; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::dmlReleaseLBIDRanges(const vector& ranges) { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("clear"); + if (fDebug) + TRACER_WRITENOW("clear"); #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RELEASE_LBID_RANGES; - serializeVector(command, ranges); - err = send_recv(command, response); + command << RELEASE_LBID_RANGES; + serializeVector(command, ranges); + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::clear() DBRM_THROW { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << BRM_CLEAR; - err = send_recv(command, response); + command << BRM_CLEAR; + err = send_recv(command, response); - if (err != ERR_OK) - return err; - - if (response.length() != 1) - return ERR_NETWORK; - - response >> err; - CHECK_EMPTY(response); + if (err != ERR_OK) return err; + + if (response.length() != 1) + return ERR_NETWORK; + + response >> err; + CHECK_EMPTY(response); + return err; } int DBRM::checkConsistency() throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("checkConsistency"); + if (fDebug) + TRACER_WRITENOW("checkConsistency"); #endif - bool locked[2] = {false, false}; + bool locked[2] = {false, false}; - try - { - em->checkConsistency(); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em->checkConsistency(); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - try - { - vbbm->lock(VBBM::READ); - locked[0] = true; - vss->lock(VSS::READ); - locked[1] = true; - vss->checkConsistency(*vbbm, *em); - vss->release(VSS::READ); - locked[1] = false; - vbbm->release(VBBM::READ); - locked[0] = false; - } - catch (exception& e) - { - cerr << e.what() << endl; + try + { + vbbm->lock(VBBM::READ); + locked[0] = true; + vss->lock(VSS::READ); + locked[1] = true; + vss->checkConsistency(*vbbm, *em); + vss->release(VSS::READ); + locked[1] = false; + vbbm->release(VBBM::READ); + locked[0] = false; + } + catch (exception& e) + { + cerr << e.what() << endl; - if (locked[1]) - vss->release(VSS::READ); + if (locked[1]) + vss->release(VSS::READ); - if (locked[0]) - vbbm->release(VBBM::READ); + if (locked[0]) + vbbm->release(VBBM::READ); - return -1; - } + return -1; + } - try - { - vbbm->lock(VBBM::READ); - vbbm->checkConsistency(); - vbbm->release(VBBM::READ); - } - catch (exception& e) - { - cerr << e.what() << endl; - vbbm->release(VBBM::READ); - return -1; - } + try + { + vbbm->lock(VBBM::READ); + vbbm->checkConsistency(); + vbbm->release(VBBM::READ); + } + catch (exception& e) + { + cerr << e.what() << endl; + vbbm->release(VBBM::READ); + return -1; + } - return 0; + return 0; } int DBRM::getCurrentTxnIDs(set& txnList) throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("getCurrentTxnIDs"); + if (fDebug) + TRACER_WRITENOW("getCurrentTxnIDs"); #endif - bool locked[2] = { false, false }; + bool locked[2] = {false, false}; - try - { - txnList.clear(); - vss->lock(VSS::READ); - locked[0] = true; - copylocks->lock(CopyLocks::READ); - locked[1] = true; - copylocks->getCurrentTxnIDs(txnList); - vss->getCurrentTxnIDs(txnList); - copylocks->release(CopyLocks::READ); - locked[1] = false; - vss->release(VSS::READ); - locked[0] = false; - } - catch (exception& e) - { - if (locked[1]) - copylocks->release(CopyLocks::READ); + try + { + txnList.clear(); + vss->lock(VSS::READ); + locked[0] = true; + copylocks->lock(CopyLocks::READ); + locked[1] = true; + copylocks->getCurrentTxnIDs(txnList); + vss->getCurrentTxnIDs(txnList); + copylocks->release(CopyLocks::READ); + locked[1] = false; + vss->release(VSS::READ); + locked[0] = false; + } + catch (exception& e) + { + if (locked[1]) + copylocks->release(CopyLocks::READ); - if (locked[0]) - vss->release(VSS::READ); + if (locked[0]) + vss->release(VSS::READ); - cerr << e.what() << endl; - return -1; - } + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } const QueryContext DBRM::verID() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("verID"); + if (fDebug) + TRACER_WRITENOW("verID"); #endif - ByteStream command, response; - uint8_t err; - QueryContext ret; + ByteStream command, response; + uint8_t err; + QueryContext ret; - command << VER_ID; - err = send_recv(command, response); - - if (err != ERR_OK) - { - cerr << "DBRM: SessionManager::verID(): network error" << endl; - ret.currentScn = -1; - return ret; - } - - try - { - response >> err; - response >> ret; - CHECK_EMPTY(response); - } - catch (exception& e) - { - cerr << "DBRM: SessionManager::verID(): bad response" << endl; - log("DBRM: SessionManager::verID(): bad response", logging::LOG_TYPE_WARNING); - ret.currentScn = -1; - } + command << VER_ID; + err = send_recv(command, response); + if (err != ERR_OK) + { + cerr << "DBRM: SessionManager::verID(): network error" << endl; + ret.currentScn = -1; return ret; + } + + try + { + response >> err; + response >> ret; + CHECK_EMPTY(response); + } + catch (exception& e) + { + cerr << "DBRM: SessionManager::verID(): bad response" << endl; + log("DBRM: SessionManager::verID(): bad response", logging::LOG_TYPE_WARNING); + ret.currentScn = -1; + } + + return ret; } const QueryContext DBRM::sysCatVerID() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("sysCatVerID"); + if (fDebug) + TRACER_WRITENOW("sysCatVerID"); #endif - ByteStream command, response; - uint8_t err; - QueryContext ret; + ByteStream command, response; + uint8_t err; + QueryContext ret; - command << SYSCAT_VER_ID; - err = send_recv(command, response); - - if (err != ERR_OK) - { - cerr << "DBRM: SessionManager::sysCatVerID(): network error" << endl; - ret.currentScn = -1; - return ret; - } - - try - { - response >> err; - response >> ret; - CHECK_EMPTY(response); - } - catch (exception& e) - { - cerr << "DBRM: SessionManager::sysCatVerID(): bad response" << endl; - log("DBRM: SessionManager::sysCatVerID(): bad response", logging::LOG_TYPE_WARNING); - ret.currentScn = -1; - } + command << SYSCAT_VER_ID; + err = send_recv(command, response); + if (err != ERR_OK) + { + cerr << "DBRM: SessionManager::sysCatVerID(): network error" << endl; + ret.currentScn = -1; return ret; + } + + try + { + response >> err; + response >> ret; + CHECK_EMPTY(response); + } + catch (exception& e) + { + cerr << "DBRM: SessionManager::sysCatVerID(): bad response" << endl; + log("DBRM: SessionManager::sysCatVerID(): bad response", logging::LOG_TYPE_WARNING); + ret.currentScn = -1; + } + + return ret; } - -const TxnID DBRM::newTxnID(const SessionManagerServer::SID session, bool block, - bool isDDL) +const TxnID DBRM::newTxnID(const SessionManagerServer::SID session, bool block, bool isDDL) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("newTxnID"); - TRACER_ADDINPUT(session); - TRACER_ADDBOOLINPUT(block); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("newTxnID"); + TRACER_ADDINPUT(session); + TRACER_ADDBOOLINPUT(block); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err, tmp; - uint32_t tmp32; - TxnID ret; + ByteStream command, response; + uint8_t err, tmp; + uint32_t tmp32; + TxnID ret; - command << NEW_TXN_ID << session << (uint8_t) block << (uint8_t)isDDL; - err = send_recv(command, response); + command << NEW_TXN_ID << session << (uint8_t)block << (uint8_t)isDDL; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: SessionManager::newTxnID(): network error"); - ret.valid = false; - return ret; - } - - if (response.length() != 6) - { - log("DBRM: SessionManager::newTxnID(): bad response"); - ret.valid = false; - return ret; - } - - response >> err; - response >> tmp32; - ret.id = tmp32; - response >> tmp; - ret.valid = (tmp == 0 ? false : true); - CHECK_EMPTY(response); + if (err != ERR_OK) + { + log("DBRM: SessionManager::newTxnID(): network error"); + ret.valid = false; return ret; + } + + if (response.length() != 6) + { + log("DBRM: SessionManager::newTxnID(): bad response"); + ret.valid = false; + return ret; + } + + response >> err; + response >> tmp32; + ret.id = tmp32; + response >> tmp; + ret.valid = (tmp == 0 ? false : true); + CHECK_EMPTY(response); + return ret; } void DBRM::committed(TxnID& txnid) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("committed"); - TRACER_ADDINPUT(txnid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("committed"); + TRACER_ADDINPUT(txnid); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << COMMITTED << (uint32_t) txnid.id << (uint8_t) txnid.valid; - err = send_recv(command, response); - txnid.valid = false; + command << COMMITTED << (uint32_t)txnid.id << (uint8_t)txnid.valid; + err = send_recv(command, response); + txnid.valid = false; - if (err != ERR_OK) - log("DBRM: error: SessionManager::committed() failed"); - else if (response.length() != 1) - log("DBRM: error: SessionManager::committed() failed (bad response)", - logging::LOG_TYPE_ERROR); + if (err != ERR_OK) + log("DBRM: error: SessionManager::committed() failed"); + else if (response.length() != 1) + log("DBRM: error: SessionManager::committed() failed (bad response)", logging::LOG_TYPE_ERROR); - response >> err; - - if (err != ERR_OK) - log("DBRM: error: SessionManager::committed() failed (valid error code)", - logging::LOG_TYPE_ERROR); + response >> err; + if (err != ERR_OK) + log("DBRM: error: SessionManager::committed() failed (valid error code)", logging::LOG_TYPE_ERROR); } - void DBRM::rolledback(TxnID& txnid) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("rolledback"); - TRACER_ADDINPUT(txnid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("rolledback"); + TRACER_ADDINPUT(txnid); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err, tmp; + ByteStream command, response; + uint8_t err, tmp; - command << ROLLED_BACK << (uint32_t) txnid.id << (uint8_t) txnid.valid; - err = send_recv(command, response); - txnid.valid = false; + command << ROLLED_BACK << (uint32_t)txnid.id << (uint8_t)txnid.valid; + err = send_recv(command, response); + txnid.valid = false; - if (err != ERR_OK) - log("DBRM: error: SessionManager::rolledback() failed (network)"); - else if (response.length() != 1) - log("DBRM: error: SessionManager::rolledback() failed (bad response)", - logging::LOG_TYPE_ERROR); + if (err != ERR_OK) + log("DBRM: error: SessionManager::rolledback() failed (network)"); + else if (response.length() != 1) + log("DBRM: error: SessionManager::rolledback() failed (bad response)", logging::LOG_TYPE_ERROR); - response >> tmp; + response >> tmp; - if (tmp != ERR_OK) - { - if (getSystemReady() != 0 ) - log("DBRM: error: SessionManager::rolledback() failed (valid error code)", - logging::LOG_TYPE_ERROR); - } + if (tmp != ERR_OK) + { + if (getSystemReady() != 0) + log("DBRM: error: SessionManager::rolledback() failed (valid error code)", logging::LOG_TYPE_ERROR); + } } int DBRM::getUnlockedLBIDs(BlockList_t* list) DBRM_THROW { - bool locked = false; + bool locked = false; - list->clear(); + list->clear(); - try - { - vss->lock(VSS::READ); - locked = true; - vss->getUnlockedLBIDs(*list); - vss->release(VSS::READ); - locked = false; - return 0; - } - catch (exception& e) - { - if (locked) - vss->release(VSS::READ); + try + { + vss->lock(VSS::READ); + locked = true; + vss->getUnlockedLBIDs(*list); + vss->release(VSS::READ); + locked = false; + return 0; + } + catch (exception& e) + { + if (locked) + vss->release(VSS::READ); - cerr << e.what() << endl; - return -1; - } + cerr << e.what() << endl; + return -1; + } } - -const TxnID DBRM::getTxnID -(const SessionManagerServer::SID session) +const TxnID DBRM::getTxnID(const SessionManagerServer::SID session) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getTxnID"); - TRACER_ADDINPUT(session); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getTxnID"); + TRACER_ADDINPUT(session); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err, tmp8; - uint32_t tmp32; - TxnID ret; + ByteStream command, response; + uint8_t err, tmp8; + uint32_t tmp32; + TxnID ret; - command << GET_TXN_ID << (uint32_t) session; - err = send_recv(command, response); + command << GET_TXN_ID << (uint32_t)session; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: error: SessionManager::getTxnID() failed (network)", logging::LOG_TYPE_ERROR); - ret.valid = false; - return ret; - } - - response >> err; - - if (err != ERR_OK) - { - log("DBRM: error: SessionManager::getTxnID() failed (got an error)", - logging::LOG_TYPE_ERROR); - ret.valid = false; - return ret; - } - - response >> tmp32 >> tmp8; - ret.id = tmp32; - ret.valid = tmp8; + if (err != ERR_OK) + { + log("DBRM: error: SessionManager::getTxnID() failed (network)", logging::LOG_TYPE_ERROR); + ret.valid = false; return ret; + } + + response >> err; + + if (err != ERR_OK) + { + log("DBRM: error: SessionManager::getTxnID() failed (got an error)", logging::LOG_TYPE_ERROR); + ret.valid = false; + return ret; + } + + response >> tmp32 >> tmp8; + ret.id = tmp32; + ret.valid = tmp8; + return ret; } boost::shared_array DBRM::SIDTIDMap(int& len) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("SIDTIDMap"); - TRACER_ADDOUTPUT(len); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("SIDTIDMap"); + TRACER_ADDOUTPUT(len); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err, tmp8; - uint32_t tmp32; - int i; - boost::shared_array ret; + ByteStream command, response; + uint8_t err, tmp8; + uint32_t tmp32; + int i; + boost::shared_array ret; - command << SID_TID_MAP; - err = send_recv(command, response); + command << SID_TID_MAP; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: error: SessionManager::SIDTIDEntry() failed (network)"); - return ret; - } - - response >> err; - - if (err != ERR_OK) - { - log("DBRM: error: SessionManager::SIDTIDEntry() failed (valid error code)", - logging::LOG_TYPE_ERROR); - return ret; - } - - response >> tmp32; - len = (int) tmp32; - ret.reset(new SIDTIDEntry[len]); - - for (i = 0; i < len; i++) - { - response >> tmp32 >> tmp8; - ret[i].txnid.id = tmp32; - ret[i].txnid.valid = (tmp8 == 0 ? false : true); - response >> tmp32; - ret[i].sessionid = tmp32; - } - - CHECK_EMPTY(response); + if (err != ERR_OK) + { + log("DBRM: error: SessionManager::SIDTIDEntry() failed (network)"); return ret; + } + + response >> err; + + if (err != ERR_OK) + { + log("DBRM: error: SessionManager::SIDTIDEntry() failed (valid error code)", logging::LOG_TYPE_ERROR); + return ret; + } + + response >> tmp32; + len = (int)tmp32; + ret.reset(new SIDTIDEntry[len]); + + for (i = 0; i < len; i++) + { + response >> tmp32 >> tmp8; + ret[i].txnid.id = tmp32; + ret[i].txnid.valid = (tmp8 == 0 ? false : true); + response >> tmp32; + ret[i].sessionid = tmp32; + } + + CHECK_EMPTY(response); + return ret; } uint32_t DBRM::getUnique32() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("getUnique32"); + if (fDebug) + TRACER_WRITENOW("getUnique32"); #endif - ByteStream command, response; - uint8_t err; - uint32_t ret; + ByteStream command, response; + uint8_t err; + uint32_t ret; - command << GET_UNIQUE_UINT32; - err = send_recv(command, response); + command << GET_UNIQUE_UINT32; + err = send_recv(command, response); - if (err != ERR_OK) - { - cerr << "DBRM: getUnique32() failed (network)\n"; - log("DBRM: getUnique32() failed (network)", logging::LOG_TYPE_ERROR); - throw runtime_error("DBRM: getUnique32() failed check the controllernode"); - return 0; - } + if (err != ERR_OK) + { + cerr << "DBRM: getUnique32() failed (network)\n"; + log("DBRM: getUnique32() failed (network)", logging::LOG_TYPE_ERROR); + throw runtime_error("DBRM: getUnique32() failed check the controllernode"); + return 0; + } - /* Some jobsteps don't need the connection after this so close it to free up - resources on the controller node */ - /* Comment the following 4 lines out. The DBRM instance is a singleton so no need to - remove the client. Plus, it may cause weird network issue when the socket is being - released and re-established very quickly*/ - //pthread_mutex_lock(&mutex); - //delete msgClient; - //msgClient = NULL; - //pthread_mutex_unlock(&mutex); + /* Some jobsteps don't need the connection after this so close it to free up + resources on the controller node */ + /* Comment the following 4 lines out. The DBRM instance is a singleton so no need to + remove the client. Plus, it may cause weird network issue when the socket is being + released and re-established very quickly*/ + // pthread_mutex_lock(&mutex); + // delete msgClient; + // msgClient = NULL; + // pthread_mutex_unlock(&mutex); - response >> err; + response >> err; - if (err != ERR_OK) - { - cerr << "DBRM: getUnique32() failed (got an error)\n"; - log("DBRM: getUnique32() failed (got an error)", - logging::LOG_TYPE_ERROR); - throw runtime_error("DBRM: getUnique32() failed check the controllernode"); - return 0; - } + if (err != ERR_OK) + { + cerr << "DBRM: getUnique32() failed (got an error)\n"; + log("DBRM: getUnique32() failed (got an error)", logging::LOG_TYPE_ERROR); + throw runtime_error("DBRM: getUnique32() failed check the controllernode"); + return 0; + } - response >> ret; -// cerr << "DBRM returning " << ret << endl; - return ret; + response >> ret; + // cerr << "DBRM returning " << ret << endl; + return ret; } uint64_t DBRM::getUnique64() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("getUnique64"); + if (fDebug) + TRACER_WRITENOW("getUnique64"); #endif - ByteStream command, response; - uint8_t err; - uint64_t ret; + ByteStream command, response; + uint8_t err; + uint64_t ret; - command << GET_UNIQUE_UINT64; - err = send_recv(command, response); + command << GET_UNIQUE_UINT64; + err = send_recv(command, response); - if (err != ERR_OK) - { - cerr << "DBRM: getUnique64() failed (network)\n"; - log("DBRM: getUnique64() failed (network)", logging::LOG_TYPE_ERROR); - throw runtime_error("DBRM: getUnique64() failed check the controllernode"); - return 0; - } + if (err != ERR_OK) + { + cerr << "DBRM: getUnique64() failed (network)\n"; + log("DBRM: getUnique64() failed (network)", logging::LOG_TYPE_ERROR); + throw runtime_error("DBRM: getUnique64() failed check the controllernode"); + return 0; + } - /* Some jobsteps don't need the connection after this so close it to free up - resources on the controller node */ - /* Comment the following 4 lines out. The DBRM instance is a singleton so no need to - remove the client. Plus, it may cause weird network issue when the socket is being - released and re-established very quickly*/ - //pthread_mutex_lock(&mutex); - //delete msgClient; - //msgClient = NULL; - //pthread_mutex_unlock(&mutex); + /* Some jobsteps don't need the connection after this so close it to free up + resources on the controller node */ + /* Comment the following 4 lines out. The DBRM instance is a singleton so no need to + remove the client. Plus, it may cause weird network issue when the socket is being + released and re-established very quickly*/ + // pthread_mutex_lock(&mutex); + // delete msgClient; + // msgClient = NULL; + // pthread_mutex_unlock(&mutex); - response >> err; + response >> err; - if (err != ERR_OK) - { - cerr << "DBRM: getUnique64() failed (got an error)\n"; - log("DBRM: getUnique64() failed (got an error)", - logging::LOG_TYPE_ERROR); - throw runtime_error("DBRM: getUnique64() failed check the controllernode"); - return 0; - } + if (err != ERR_OK) + { + cerr << "DBRM: getUnique64() failed (got an error)\n"; + log("DBRM: getUnique64() failed (got an error)", logging::LOG_TYPE_ERROR); + throw runtime_error("DBRM: getUnique64() failed check the controllernode"); + return 0; + } - response >> ret; -// cerr << "DBRM returning " << ret << endl; - return ret; + response >> ret; + // cerr << "DBRM returning " << ret << endl; + return ret; } void DBRM::sessionmanager_reset() { - ByteStream command, response; - command << SM_RESET; - send_recv(command, response); + ByteStream command, response; + command << SM_RESET; + send_recv(command, response); } bool DBRM::isEMEmpty() throw() { - bool res = false; + bool res = false; - try - { - res = em->empty(); - } - catch (...) - { - res = false; - } + try + { + res = em->empty(); + } + catch (...) + { + res = false; + } - return res; + return res; } vector DBRM::getEMFreeListEntries() throw() { - vector res; + vector res; - try - { - res = em->getFreeListEntries(); - } - catch (...) - { - res.clear(); - } + try + { + res = em->getFreeListEntries(); + } + catch (...) + { + res.clear(); + } - return res; + return res; } -int DBRM::takeSnapshot() throw () +int DBRM::takeSnapshot() throw() { - return 0; // don't know why, but we're calling this all the time. Need to take most/all of those calls out, it's very wasteful. + return 0; // don't know why, but we're calling this all the time. Need to take most/all of those calls + // out, it's very wasteful. - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << TAKE_SNAPSHOT; - err = send_recv(command, response); + command << TAKE_SNAPSHOT; + err = send_recv(command, response); - if (err != ERR_OK) - return err; + if (err != ERR_OK) + return err; - if (response.length() == 0) - return ERR_NETWORK; + if (response.length() == 0) + return ERR_NETWORK; - return 0; + return 0; } int DBRM::getSystemReady() throw() { - uint32_t stateFlags; + uint32_t stateFlags; - if (getSystemState(stateFlags) < 0) - { - return -1; - } + if (getSystemState(stateFlags) < 0) + { + return -1; + } - return (stateFlags & SessionManagerServer::SS_READY); + return (stateFlags & SessionManagerServer::SS_READY); } int DBRM::getSystemQueryReady() throw() { - uint32_t stateFlags; + uint32_t stateFlags; - if (getSystemState(stateFlags) < 0) - { - return -1; - } + if (getSystemState(stateFlags) < 0) + { + return -1; + } - return (stateFlags & SessionManagerServer::SS_QUERY_READY); + return (stateFlags & SessionManagerServer::SS_QUERY_READY); } int DBRM::getSystemSuspended() throw() { - uint32_t stateFlags; + uint32_t stateFlags; - if (getSystemState(stateFlags) < 0) - { - return -1; - } + if (getSystemState(stateFlags) < 0) + { + return -1; + } - return (stateFlags & SessionManagerServer::SS_SUSPENDED); + return (stateFlags & SessionManagerServer::SS_SUSPENDED); } int DBRM::getSystemSuspendPending(bool& bRollback) throw() { - uint32_t stateFlags; + uint32_t stateFlags; - if (getSystemState(stateFlags) < 0) - { - return -1; - } + if (getSystemState(stateFlags) < 0) + { + return -1; + } - bRollback = stateFlags & SessionManagerServer::SS_ROLLBACK; + bRollback = stateFlags & SessionManagerServer::SS_ROLLBACK; - return (stateFlags & SessionManagerServer::SS_SUSPEND_PENDING); + return (stateFlags & SessionManagerServer::SS_SUSPEND_PENDING); } int DBRM::getSystemShutdownPending(bool& bRollback, bool& bForce) throw() { - uint32_t stateFlags; + uint32_t stateFlags; - if (getSystemState(stateFlags) < 0) - { - return -1; - } + if (getSystemState(stateFlags) < 0) + { + return -1; + } - bRollback = stateFlags & SessionManagerServer::SS_ROLLBACK; - bForce = stateFlags & SessionManagerServer::SS_FORCE; + bRollback = stateFlags & SessionManagerServer::SS_ROLLBACK; + bForce = stateFlags & SessionManagerServer::SS_FORCE; - return (stateFlags & SessionManagerServer::SS_SHUTDOWN_PENDING); + return (stateFlags & SessionManagerServer::SS_SHUTDOWN_PENDING); } int DBRM::setSystemReady(bool bReady) throw() { - if (bReady) - { - return setSystemState(SessionManagerServer::SS_READY); - } - else - { - return clearSystemState(SessionManagerServer::SS_READY); - } + if (bReady) + { + return setSystemState(SessionManagerServer::SS_READY); + } + else + { + return clearSystemState(SessionManagerServer::SS_READY); + } } int DBRM::setSystemQueryReady(bool bReady) throw() { - if (bReady) - { - return setSystemState(SessionManagerServer::SS_QUERY_READY); - } - else - { - return clearSystemState(SessionManagerServer::SS_QUERY_READY); - } + if (bReady) + { + return setSystemState(SessionManagerServer::SS_QUERY_READY); + } + else + { + return clearSystemState(SessionManagerServer::SS_QUERY_READY); + } } int DBRM::setSystemSuspended(bool bSuspended) throw() { - uint32_t stateFlags = 0; + uint32_t stateFlags = 0; - if (bSuspended) + if (bSuspended) + { + if (setSystemState(SessionManagerServer::SS_SUSPENDED) < 0) { - if (setSystemState(SessionManagerServer::SS_SUSPENDED) < 0) - { - return -1; - } - } - else - { - stateFlags = SessionManagerServer::SS_SUSPENDED; + return -1; } + } + else + { + stateFlags = SessionManagerServer::SS_SUSPENDED; + } - // In either case, we need to clear the pending and rollback flags - stateFlags |= SessionManagerServer::SS_SUSPEND_PENDING; - stateFlags |= SessionManagerServer::SS_ROLLBACK; - return clearSystemState(stateFlags); + // In either case, we need to clear the pending and rollback flags + stateFlags |= SessionManagerServer::SS_SUSPEND_PENDING; + stateFlags |= SessionManagerServer::SS_ROLLBACK; + return clearSystemState(stateFlags); } int DBRM::setSystemSuspendPending(bool bPending, bool bRollback) throw() { - uint32_t stateFlags = SessionManagerServer::SS_SUSPEND_PENDING; + uint32_t stateFlags = SessionManagerServer::SS_SUSPEND_PENDING; - if (bPending) + if (bPending) + { + if (bRollback) { - if (bRollback) - { - stateFlags |= SessionManagerServer::SS_ROLLBACK; - } + stateFlags |= SessionManagerServer::SS_ROLLBACK; + } - return setSystemState(stateFlags); - } - else - { - stateFlags |= SessionManagerServer::SS_ROLLBACK; - return clearSystemState(stateFlags); - } + return setSystemState(stateFlags); + } + else + { + stateFlags |= SessionManagerServer::SS_ROLLBACK; + return clearSystemState(stateFlags); + } } int DBRM::setSystemShutdownPending(bool bPending, bool bRollback, bool bForce) throw() { - int rtn = 0; - uint32_t stateFlags = SessionManagerServer::SS_SHUTDOWN_PENDING; + int rtn = 0; + uint32_t stateFlags = SessionManagerServer::SS_SHUTDOWN_PENDING; - if (bPending) + if (bPending) + { + if (bForce) { - if (bForce) - { - stateFlags |= SessionManagerServer::SS_FORCE; - } - else if (bRollback) - { - stateFlags |= SessionManagerServer::SS_ROLLBACK; - } - - rtn = setSystemState(stateFlags); + stateFlags |= SessionManagerServer::SS_FORCE; } - else + else if (bRollback) { - stateFlags |= SessionManagerServer::SS_ROLLBACK; - stateFlags |= SessionManagerServer::SS_FORCE; - rtn = clearSystemState(stateFlags); // Clears the flags that are turned on in stateFlags + stateFlags |= SessionManagerServer::SS_ROLLBACK; } - return rtn; + rtn = setSystemState(stateFlags); + } + else + { + stateFlags |= SessionManagerServer::SS_ROLLBACK; + stateFlags |= SessionManagerServer::SS_FORCE; + rtn = clearSystemState(stateFlags); // Clears the flags that are turned on in stateFlags + } + + return rtn; } /* Return the shm stateflags */ int DBRM::getSystemState(uint32_t& stateFlags) throw() { - try - { + try + { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getSystemState"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getSystemState"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << GET_SYSTEM_STATE; - err = send_recv(command, response); + command << GET_SYSTEM_STATE; + err = send_recv(command, response); - if (err != ERR_OK) - { - std::ostringstream oss; - oss << "DBRM: error: SessionManager::getSystemState() failed (network)"; - log(oss.str(), logging::LOG_TYPE_ERROR); - return -1; - } - - response >> err; - - if (err != ERR_OK) - { - std::ostringstream oss; - oss << "DBRM: error: SessionManager::getSystemState() failed (error " << err << ")"; - log(oss.str(), logging::LOG_TYPE_ERROR); - return -1; - } - - response >> stateFlags; - return 1; - } - catch (...) + if (err != ERR_OK) { + std::ostringstream oss; + oss << "DBRM: error: SessionManager::getSystemState() failed (network)"; + log(oss.str(), logging::LOG_TYPE_ERROR); + return -1; } - return -1; + response >> err; + + if (err != ERR_OK) + { + std::ostringstream oss; + oss << "DBRM: error: SessionManager::getSystemState() failed (error " << err << ")"; + log(oss.str(), logging::LOG_TYPE_ERROR); + return -1; + } + + response >> stateFlags; + return 1; + } + catch (...) + { + } + + return -1; } /* Set the shm stateflags that are set in the parameter */ int DBRM::setSystemState(uint32_t stateFlags) throw() { - try - { + try + { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setSystemState"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("setSystemState"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << SET_SYSTEM_STATE << static_cast(stateFlags); - err = send_recv(command, response); + command << SET_SYSTEM_STATE << static_cast(stateFlags); + err = send_recv(command, response); - if (err != ERR_OK) - { - std::ostringstream oss; - oss << "DBRM: error: SessionManager::setSystemState() failed (network)"; - log(oss.str(), logging::LOG_TYPE_ERROR); - stateFlags = 0; - return -1; - } - - response >> err; - - if (err != ERR_OK) - { - std::ostringstream oss; - oss << "DBRM: error: SessionManager::setSystemState() failed (got an error)"; - log(oss.str(), logging::LOG_TYPE_ERROR); - stateFlags = 0; - return -1; - } - - return 1; - } - catch (...) + if (err != ERR_OK) { + std::ostringstream oss; + oss << "DBRM: error: SessionManager::setSystemState() failed (network)"; + log(oss.str(), logging::LOG_TYPE_ERROR); + stateFlags = 0; + return -1; } - stateFlags = 0; - return -1; + response >> err; + + if (err != ERR_OK) + { + std::ostringstream oss; + oss << "DBRM: error: SessionManager::setSystemState() failed (got an error)"; + log(oss.str(), logging::LOG_TYPE_ERROR); + stateFlags = 0; + return -1; + } + + return 1; + } + catch (...) + { + } + + stateFlags = 0; + return -1; } /* Clear the shm stateflags that are set in the parameter */ int DBRM::clearSystemState(uint32_t stateFlags) throw() { - try - { + try + { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("clearSystemState"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("clearSystemState"); + TRACER_WRITE; + } #endif - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << CLEAR_SYSTEM_STATE << static_cast(stateFlags); - err = send_recv(command, response); + command << CLEAR_SYSTEM_STATE << static_cast(stateFlags); + err = send_recv(command, response); - if (err != ERR_OK) - { - std::ostringstream oss; - oss << "DBRM: error: SessionManager::clearSystemState() failed (network)"; - log(oss.str(), logging::LOG_TYPE_ERROR); - return -1; - } - - response >> err; - - if (err != ERR_OK) - { - std::ostringstream oss; - oss << "DBRM: error: SessionManager::clearSystemState() failed (got an error)"; - log(oss.str(), logging::LOG_TYPE_ERROR); - return -1; - } - - return 1; - } - catch (...) + if (err != ERR_OK) { + std::ostringstream oss; + oss << "DBRM: error: SessionManager::clearSystemState() failed (network)"; + log(oss.str(), logging::LOG_TYPE_ERROR); + return -1; } - return -1; + response >> err; + + if (err != ERR_OK) + { + std::ostringstream oss; + oss << "DBRM: error: SessionManager::clearSystemState() failed (got an error)"; + log(oss.str(), logging::LOG_TYPE_ERROR); + return -1; + } + + return 1; + } + catch (...) + { + } + + return -1; } /* Ping the controller node. Don't print anything. @@ -3757,41 +3707,42 @@ bool DBRM::isDBRMReady() throw() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("isDBRMReady"); + if (fDebug) + TRACER_WRITENOW("isDBRMReady"); #endif - boost::mutex::scoped_lock scoped(mutex); + boost::mutex::scoped_lock scoped(mutex); - try + try + { + for (int attempt = 0; attempt < 2; ++attempt) { - for (int attempt = 0; attempt < 2; ++attempt) + try + { + if (msgClient == NULL) { - try - { - if (msgClient == NULL) - { - msgClient = MessageQueueClientPool::getInstance(masterName); - } - - if (msgClient->connect()) - { - return true; - } - } - catch (...) - { - } - - MessageQueueClientPool::releaseInstance(msgClient); - msgClient = NULL; - sleep(1); + msgClient = MessageQueueClientPool::getInstance(masterName); } - } - catch (...) - { - } - return false; + if (msgClient->connect()) + { + return true; + } + } + catch (...) + { + } + + MessageQueueClientPool::releaseInstance(msgClient); + msgClient = NULL; + sleep(1); + } + } + catch (...) + { + } + + return false; } /* This waits for the lock up to 30 sec. After 30 sec, the assumption is something @@ -3800,80 +3751,80 @@ bool DBRM::isDBRMReady() throw() */ void DBRM::lockLBIDRange(LBID_t start, uint32_t count) { - bool locked = false, lockedRange = false; - LBIDRange range; - const uint32_t waitInterval = 50000; // in usec - const uint32_t maxRetries = 30000000 / waitInterval; // 30 secs - uint32_t retries = 0; + bool locked = false, lockedRange = false; + LBIDRange range; + const uint32_t waitInterval = 50000; // in usec + const uint32_t maxRetries = 30000000 / waitInterval; // 30 secs + uint32_t retries = 0; - range.start = start; - range.size = count; + range.start = start; + range.size = count; - try + try + { + copylocks->lock(CopyLocks::WRITE); + locked = true; + + while (copylocks->isLocked(range) && retries < maxRetries) { - copylocks->lock(CopyLocks::WRITE); - locked = true; - - while (copylocks->isLocked(range) && retries < maxRetries) - { - copylocks->release(CopyLocks::WRITE); - locked = false; - usleep(waitInterval); - retries++; - copylocks->lock(CopyLocks::WRITE); - locked = true; - } - - if (retries >= maxRetries) - copylocks->forceRelease(range); - - copylocks->lockRange(range, -1); - lockedRange = true; - copylocks->confirmChanges(); - copylocks->release(CopyLocks::WRITE); - locked = false; + copylocks->release(CopyLocks::WRITE); + locked = false; + usleep(waitInterval); + retries++; + copylocks->lock(CopyLocks::WRITE); + locked = true; } - catch (...) + + if (retries >= maxRetries) + copylocks->forceRelease(range); + + copylocks->lockRange(range, -1); + lockedRange = true; + copylocks->confirmChanges(); + copylocks->release(CopyLocks::WRITE); + locked = false; + } + catch (...) + { + if (lockedRange) + copylocks->releaseRange(range); + + if (locked) { - if (lockedRange) - copylocks->releaseRange(range); - - if (locked) - { - copylocks->confirmChanges(); - copylocks->release(CopyLocks::WRITE); - } - - throw; + copylocks->confirmChanges(); + copylocks->release(CopyLocks::WRITE); } + + throw; + } } void DBRM::releaseLBIDRange(LBID_t start, uint32_t count) { - bool locked = false; - LBIDRange range; - range.start = start; - range.size = count; + bool locked = false; + LBIDRange range; + range.start = start; + range.size = count; - try + try + { + copylocks->lock(CopyLocks::WRITE); + locked = true; + copylocks->releaseRange(range); + copylocks->confirmChanges(); + copylocks->release(CopyLocks::WRITE); + locked = false; + } + catch (...) + { + if (locked) { - copylocks->lock(CopyLocks::WRITE); - locked = true; - copylocks->releaseRange(range); - copylocks->confirmChanges(); - copylocks->release(CopyLocks::WRITE); - locked = false; + copylocks->confirmChanges(); + copylocks->release(CopyLocks::WRITE); } - catch (...) - { - if (locked) - { - copylocks->confirmChanges(); - copylocks->release(CopyLocks::WRITE); - } - throw; - } + throw; + } } /* OID Manager section */ @@ -3882,730 +3833,732 @@ int DBRM::allocOIDs(int num) { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("allocOID"); + if (fDebug) + TRACER_WRITENOW("allocOID"); #endif - ByteStream command, response; - uint8_t err; - uint32_t ret; + ByteStream command, response; + uint8_t err; + uint32_t ret; - command << ALLOC_OIDS; - command << (uint32_t) num; - err = send_recv(command, response); + command << ALLOC_OIDS; + command << (uint32_t)num; + err = send_recv(command, response); + + if (err != ERR_OK) + { + cerr << "DBRM: OIDManager::allocOIDs(): network error" << endl; + log("DBRM: OIDManager::allocOIDs(): network error", logging::LOG_TYPE_CRITICAL); + return -1; + } + + try + { + response >> err; if (err != ERR_OK) - { - cerr << "DBRM: OIDManager::allocOIDs(): network error" << endl; - log("DBRM: OIDManager::allocOIDs(): network error", logging::LOG_TYPE_CRITICAL); - return -1; - } + return -1; - try - { - response >> err; - - if (err != ERR_OK) - return -1; - - response >> ret; - CHECK_EMPTY(response); - return (int) ret; - } - catch (...) - { - log("DBRM: OIDManager::allocOIDs(): bad response", logging::LOG_TYPE_CRITICAL); - return -1; - } + response >> ret; + CHECK_EMPTY(response); + return (int)ret; + } + catch (...) + { + log("DBRM: OIDManager::allocOIDs(): bad response", logging::LOG_TYPE_CRITICAL); + return -1; + } } void DBRM::returnOIDs(int start, int end) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RETURN_OIDS; - command << (uint32_t) start; - command << (uint32_t) end; - err = send_recv(command, response); + command << RETURN_OIDS; + command << (uint32_t)start; + command << (uint32_t)end; + err = send_recv(command, response); - if (err == ERR_NETWORK) - { - cerr << "DBRM: OIDManager::returnOIDs(): network error" << endl; - log("DBRM: OIDManager::returnOIDs(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::returnOIDs(): network error"); - } + if (err == ERR_NETWORK) + { + cerr << "DBRM: OIDManager::returnOIDs(): network error" << endl; + log("DBRM: OIDManager::returnOIDs(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::returnOIDs(): network error"); + } - try - { - response >> err; - CHECK_EMPTY(response); - } - catch (...) - { - err = ERR_FAILURE; - } + try + { + response >> err; + CHECK_EMPTY(response); + } + catch (...) + { + err = ERR_FAILURE; + } - if (err != ERR_OK) - { - log("DBRM: OIDManager::returnOIDs() failed", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::returnOIDs() failed"); - } + if (err != ERR_OK) + { + log("DBRM: OIDManager::returnOIDs() failed", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::returnOIDs() failed"); + } } int DBRM::oidm_size() { - ByteStream command, response; - uint8_t err; - uint32_t ret; + ByteStream command, response; + uint8_t err; + uint32_t ret; - command << OIDM_SIZE; - err = send_recv(command, response); + command << OIDM_SIZE; + err = send_recv(command, response); - if (err != ERR_OK) + if (err != ERR_OK) + { + cerr << "DBRM: OIDManager::size(): network error" << endl; + log("DBRM: OIDManager::size(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::size(): network error"); + } + + try + { + response >> err; + + if (err == ERR_OK) { - cerr << "DBRM: OIDManager::size(): network error" << endl; - log("DBRM: OIDManager::size(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::size(): network error"); + response >> ret; + CHECK_EMPTY(response); + return ret; } - try - { - response >> err; - - if (err == ERR_OK) - { - response >> ret; - CHECK_EMPTY(response); - return ret; - } - - CHECK_EMPTY(response); - return -1; - } - catch (...) - { - log("DBRM: OIDManager::size(): bad response", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::size(): bad response"); - } + CHECK_EMPTY(response); + return -1; + } + catch (...) + { + log("DBRM: OIDManager::size(): bad response", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::size(): bad response"); + } } int DBRM::allocVBOID(uint32_t dbroot) { - ByteStream command, response; - uint8_t err; - uint32_t ret; + ByteStream command, response; + uint8_t err; + uint32_t ret; - command << ALLOC_VBOID << (uint32_t) dbroot; - err = send_recv(command, response); + command << ALLOC_VBOID << (uint32_t)dbroot; + err = send_recv(command, response); - if (err != ERR_OK) + if (err != ERR_OK) + { + cerr << "DBRM: OIDManager::allocVBOID(): network error" << endl; + log("DBRM: OIDManager::allocVBOID(): network error", logging::LOG_TYPE_CRITICAL); + return -1; + } + + try + { + response >> err; + + if (err == ERR_OK) { - cerr << "DBRM: OIDManager::allocVBOID(): network error" << endl; - log("DBRM: OIDManager::allocVBOID(): network error", logging::LOG_TYPE_CRITICAL); - return -1; + response >> ret; + CHECK_EMPTY(response); + return ret; } - try - { - response >> err; - - if (err == ERR_OK) - { - response >> ret; - CHECK_EMPTY(response); - return ret; - } - - CHECK_EMPTY(response); - return -1; - } - catch (...) - { - log("DBRM: OIDManager::allocVBOID(): bad response", logging::LOG_TYPE_CRITICAL); - return -1; - } + CHECK_EMPTY(response); + return -1; + } + catch (...) + { + log("DBRM: OIDManager::allocVBOID(): bad response", logging::LOG_TYPE_CRITICAL); + return -1; + } } int DBRM::getDBRootOfVBOID(uint32_t vbOID) { - ByteStream command, response; - uint8_t err; - uint32_t ret; + ByteStream command, response; + uint8_t err; + uint32_t ret; - command << GETDBROOTOFVBOID << (uint32_t) vbOID; - err = send_recv(command, response); + command << GETDBROOTOFVBOID << (uint32_t)vbOID; + err = send_recv(command, response); - if (err != ERR_OK) + if (err != ERR_OK) + { + cerr << "DBRM: OIDManager::getDBRootOfVBOID(): network error" << endl; + log("DBRM: OIDManager::getDBRootOfVBOID(): network error", logging::LOG_TYPE_CRITICAL); + return -1; + } + + try + { + response >> err; + + if (err == ERR_OK) { - cerr << "DBRM: OIDManager::getDBRootOfVBOID(): network error" << endl; - log("DBRM: OIDManager::getDBRootOfVBOID(): network error", logging::LOG_TYPE_CRITICAL); - return -1; + response >> ret; + CHECK_EMPTY(response); + return (int)ret; } - try - { - response >> err; - - if (err == ERR_OK) - { - response >> ret; - CHECK_EMPTY(response); - return (int) ret; - } - - CHECK_EMPTY(response); - return -1; - } - catch (...) - { - log("DBRM: OIDManager::getDBRootOfVBOID(): bad response", logging::LOG_TYPE_CRITICAL); - return -1; - } + CHECK_EMPTY(response); + return -1; + } + catch (...) + { + log("DBRM: OIDManager::getDBRootOfVBOID(): bad response", logging::LOG_TYPE_CRITICAL); + return -1; + } } vector DBRM::getVBOIDToDBRootMap() { - ByteStream command, response; - uint8_t err; - vector ret; + ByteStream command, response; + uint8_t err; + vector ret; - command << GETVBOIDTODBROOTMAP; - err = send_recv(command, response); + command << GETVBOIDTODBROOTMAP; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: OIDManager::getVBOIDToDBRootMap(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::getVBOIDToDBRootMap(): network error"); - } - - try - { - response >> err; - - if (err != ERR_OK) - { - log("DBRM: OIDManager::getVBOIDToDBRootMap(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::getVBOIDToDBRootMap(): processing error"); - } - - deserializeInlineVector(response, ret); - CHECK_EMPTY(response); - return ret; - } - catch (...) - { - log("DBRM: OIDManager::getVBOIDToDBRootMap(): bad response", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: OIDManager::getVBOIDToDBRootMap(): bad response"); - } -} - -uint64_t DBRM::getTableLock(const vector& pmList, uint32_t tableOID, - string* ownerName, uint32_t* ownerPID, int32_t* ownerSessionID, int32_t* ownerTxnID, LockState state) -{ - ByteStream command, response; - uint8_t err; - uint64_t ret; - TableLockInfo tli; - uint32_t tmp32; - vector dbRootsList; - OamCache* oamcache = OamCache::makeOamCache(); - OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); - int moduleId = 0; - - for (uint32_t i = 0; i < pmList.size(); i++) - { - moduleId = pmList[i]; - vector dbroots = (*pmDbroots)[moduleId]; - - for (uint32_t j = 0; j < dbroots.size(); j++) - dbRootsList.push_back((uint32_t)dbroots[j]); - } - - tli.id = 0; - tli.ownerName = *ownerName; - tli.ownerPID = *ownerPID; - tli.ownerSessionID = *ownerSessionID; - tli.ownerTxnID = *ownerTxnID; - tli.dbrootList = dbRootsList; - tli.state = state; - tli.tableOID = tableOID; - tli.creationTime = time(NULL); - - command << GET_TABLE_LOCK << tli; - err = send_recv(command, response); - - if (err != ERR_OK) - { - log("DBRM: getTableLock(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getTableLock(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: OIDManager::getVBOIDToDBRootMap(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::getVBOIDToDBRootMap(): network error"); + } + try + { response >> err; - /* TODO: this means a save failure, need a specific exception type */ if (err != ERR_OK) - throw runtime_error("Table lock save file failure"); - - response >> ret; - - if (ret == 0) { - response >> *ownerPID; - response >> *ownerName; - response >> tmp32; - *ownerSessionID = tmp32; - response >> tmp32; - *ownerTxnID = tmp32; + log("DBRM: OIDManager::getVBOIDToDBRootMap(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::getVBOIDToDBRootMap(): processing error"); } - idbassert(response.length() == 0); + deserializeInlineVector(response, ret); + CHECK_EMPTY(response); return ret; + } + catch (...) + { + log("DBRM: OIDManager::getVBOIDToDBRootMap(): bad response", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: OIDManager::getVBOIDToDBRootMap(): bad response"); + } +} + +uint64_t DBRM::getTableLock(const vector& pmList, uint32_t tableOID, string* ownerName, + uint32_t* ownerPID, int32_t* ownerSessionID, int32_t* ownerTxnID, LockState state) +{ + ByteStream command, response; + uint8_t err; + uint64_t ret; + TableLockInfo tli; + uint32_t tmp32; + vector dbRootsList; + OamCache* oamcache = OamCache::makeOamCache(); + OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); + int moduleId = 0; + + for (uint32_t i = 0; i < pmList.size(); i++) + { + moduleId = pmList[i]; + vector dbroots = (*pmDbroots)[moduleId]; + + for (uint32_t j = 0; j < dbroots.size(); j++) + dbRootsList.push_back((uint32_t)dbroots[j]); + } + + tli.id = 0; + tli.ownerName = *ownerName; + tli.ownerPID = *ownerPID; + tli.ownerSessionID = *ownerSessionID; + tli.ownerTxnID = *ownerTxnID; + tli.dbrootList = dbRootsList; + tli.state = state; + tli.tableOID = tableOID; + tli.creationTime = time(NULL); + + command << GET_TABLE_LOCK << tli; + err = send_recv(command, response); + + if (err != ERR_OK) + { + log("DBRM: getTableLock(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getTableLock(): network error"); + } + + response >> err; + + /* TODO: this means a save failure, need a specific exception type */ + if (err != ERR_OK) + throw runtime_error("Table lock save file failure"); + + response >> ret; + + if (ret == 0) + { + response >> *ownerPID; + response >> *ownerName; + response >> tmp32; + *ownerSessionID = tmp32; + response >> tmp32; + *ownerTxnID = tmp32; + } + + idbassert(response.length() == 0); + return ret; } bool DBRM::releaseTableLock(uint64_t id) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RELEASE_TABLE_LOCK << id; - err = send_recv(command, response); + command << RELEASE_TABLE_LOCK << id; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: releaseTableLock(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: releaseTableLock(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: releaseTableLock(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: releaseTableLock(): network error"); + } - response >> err; + response >> err; - /* TODO: this means a save failure, need a specific exception type */ - if (err != ERR_OK) - throw runtime_error("Table lock save file failure"); + /* TODO: this means a save failure, need a specific exception type */ + if (err != ERR_OK) + throw runtime_error("Table lock save file failure"); - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - return (bool) err; + return (bool)err; } bool DBRM::changeState(uint64_t id, LockState state) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << CHANGE_TABLE_LOCK_STATE << id << (uint32_t) state; - err = send_recv(command, response); + command << CHANGE_TABLE_LOCK_STATE << id << (uint32_t)state; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: changeState(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: changeState(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: changeState(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: changeState(): network error"); + } - response >> err; + response >> err; - /* TODO: this means a save failure, need a specific exception type */ - if (err != ERR_OK) - throw runtime_error("Table lock save file failure"); + /* TODO: this means a save failure, need a specific exception type */ + if (err != ERR_OK) + throw runtime_error("Table lock save file failure"); - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - return (bool) err; + return (bool)err; } bool DBRM::changeOwner(uint64_t id, const string& ownerName, uint32_t ownerPID, int32_t ownerSessionID, int32_t ownerTxnID) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << CHANGE_TABLE_LOCK_OWNER << id << ownerName << ownerPID << - (uint32_t) ownerSessionID << (uint32_t) ownerTxnID; - err = send_recv(command, response); + command << CHANGE_TABLE_LOCK_OWNER << id << ownerName << ownerPID << (uint32_t)ownerSessionID + << (uint32_t)ownerTxnID; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: changeOwner(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: changeOwner(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: changeOwner(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: changeOwner(): network error"); + } - response >> err; + response >> err; - /* TODO: this means a save failure, need a specific exception type */ - if (err != ERR_OK) - throw runtime_error("Table lock save file failure"); + /* TODO: this means a save failure, need a specific exception type */ + if (err != ERR_OK) + throw runtime_error("Table lock save file failure"); - response >> err; - idbassert(response.length() == 0); - return (bool) err; + response >> err; + idbassert(response.length() == 0); + return (bool)err; } bool DBRM::checkOwner(uint64_t id) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << OWNER_CHECK << id; - err = send_recv(command, response); + command << OWNER_CHECK << id; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: ownerCheck(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: ownerCheck(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: ownerCheck(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: ownerCheck(): network error"); + } - response >> err; + response >> err; - /* TODO: this means a save failure, need a specific exception type */ - if (err != ERR_OK) - throw runtime_error("Table lock save file failure"); + /* TODO: this means a save failure, need a specific exception type */ + if (err != ERR_OK) + throw runtime_error("Table lock save file failure"); - response >> err; - idbassert(response.length() == 0); - return (bool) err; // Return true means the owner is valid + response >> err; + idbassert(response.length() == 0); + return (bool)err; // Return true means the owner is valid } vector DBRM::getAllTableLocks() { - ByteStream command, response; - uint8_t err; - vector ret; + ByteStream command, response; + uint8_t err; + vector ret; - command << GET_ALL_TABLE_LOCKS; - err = send_recv(command, response); + command << GET_ALL_TABLE_LOCKS; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: getAllTableLocks(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getAllTableLocks(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: getAllTableLocks(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getAllTableLocks(): network error"); + } - response >> err; + response >> err; - if (err != ERR_OK) - { - log("DBRM: getAllTableLocks(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getAllTableLocks(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: getAllTableLocks(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getAllTableLocks(): processing error"); + } - deserializeVector(response, ret); - idbassert(response.length() == 0); - return ret; + deserializeVector(response, ret); + idbassert(response.length() == 0); + return ret; } void DBRM::releaseAllTableLocks() { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RELEASE_ALL_TABLE_LOCKS; - err = send_recv(command, response); + command << RELEASE_ALL_TABLE_LOCKS; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: releaseAllTableLocks(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: releaseAllTableLocks(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: releaseAllTableLocks(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: releaseAllTableLocks(): network error"); + } - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - if (err != ERR_OK) - throw runtime_error("DBRM: releaseAllTableLocks(): processing error"); + if (err != ERR_OK) + throw runtime_error("DBRM: releaseAllTableLocks(): processing error"); } bool DBRM::getTableLockInfo(uint64_t id, TableLockInfo* tli) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << GET_TABLE_LOCK_INFO << id; - err = send_recv(command, response); + command << GET_TABLE_LOCK_INFO << id; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: getTableLockInfo(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getTableLockInfo(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: getTableLockInfo(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getTableLockInfo(): network error"); + } - response >> err; + response >> err; - if (err != ERR_OK) - throw runtime_error("DBRM: getTableLockInfo() processing error"); + if (err != ERR_OK) + throw runtime_error("DBRM: getTableLockInfo() processing error"); - response >> err; + response >> err; - if (err) - response >> *tli; + if (err) + response >> *tli; - return (bool) err; + return (bool)err; } void DBRM::startAISequence(uint32_t OID, uint64_t firstNum, uint32_t colWidth, execplan::CalpontSystemCatalog::ColDataType colDataType) { - ByteStream command, response; - uint8_t err; - uint8_t tmp8 = colDataType; + ByteStream command, response; + uint8_t err; + uint8_t tmp8 = colDataType; - command << START_AI_SEQUENCE << OID << firstNum << colWidth << tmp8; - err = send_recv(command, response); + command << START_AI_SEQUENCE << OID << firstNum << colWidth << tmp8; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: startAISequence(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: startAISequence(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: startAISequence(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: startAISequence(): network error"); + } - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - if (err != ERR_OK) - { - log("DBRM: startAISequence(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: startAISequence(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: startAISequence(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: startAISequence(): processing error"); + } } bool DBRM::getAIRange(uint32_t OID, uint32_t count, uint64_t* firstNum) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << GET_AI_RANGE << OID << count; - err = send_recv(command, response); + command << GET_AI_RANGE << OID << count; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: getAIRange(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getAIRange(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: getAIRange(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getAIRange(): network error"); + } - response >> err; + response >> err; - if (err != ERR_OK) - { - log("DBRM: getAIRange(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getAIRange(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: getAIRange(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getAIRange(): processing error"); + } - response >> err; + response >> err; - if (err == 0) - return false; + if (err == 0) + return false; - response >> *firstNum; - idbassert(response.length() == 0); - return true; + response >> *firstNum; + idbassert(response.length() == 0); + return true; } bool DBRM::getAIValue(uint32_t OID, uint64_t* value) { - return getAIRange(OID, 0, value); + return getAIRange(OID, 0, value); } void DBRM::resetAISequence(uint32_t OID, uint64_t value) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RESET_AI_SEQUENCE << OID << value; - err = send_recv(command, response); + command << RESET_AI_SEQUENCE << OID << value; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: resetAISequence(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: resetAISequence(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: resetAISequence(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: resetAISequence(): network error"); + } - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - if (err != ERR_OK) - { - log("DBRM: resetAISequence(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: resetAISequence(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: resetAISequence(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: resetAISequence(): processing error"); + } } void DBRM::getAILock(uint32_t OID) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << GET_AI_LOCK << OID; - err = send_recv(command, response); + command << GET_AI_LOCK << OID; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: getAILock(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getAILock(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: getAILock(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getAILock(): network error"); + } - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - if (err != ERR_OK) - { - log("DBRM: getAILock(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: getAILock(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: getAILock(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: getAILock(): processing error"); + } } void DBRM::releaseAILock(uint32_t OID) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << RELEASE_AI_LOCK << OID; - err = send_recv(command, response); + command << RELEASE_AI_LOCK << OID; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM: releaseAILock(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: releaseAILock(): network error"); - } + if (err != ERR_OK) + { + log("DBRM: releaseAILock(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: releaseAILock(): network error"); + } - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - if (err != ERR_OK) - { - log("DBRM: releaseAILock(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: releaseAILock(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: releaseAILock(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: releaseAILock(): processing error"); + } } void DBRM::deleteAISequence(uint32_t OID) { - ByteStream command, response; - uint8_t err; + ByteStream command, response; + uint8_t err; - command << DELETE_AI_SEQUENCE << OID; - err = send_recv(command, response); + command << DELETE_AI_SEQUENCE << OID; + err = send_recv(command, response); - if (err != ERR_OK) - { - log("DBRM:deleteAILock(): network error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: deleteAILock(): network error"); - } + if (err != ERR_OK) + { + log("DBRM:deleteAILock(): network error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: deleteAILock(): network error"); + } - response >> err; - idbassert(response.length() == 0); + response >> err; + idbassert(response.length() == 0); - if (err != ERR_OK) - { - log("DBRM: deleteAILock(): processing error", logging::LOG_TYPE_CRITICAL); - throw runtime_error("DBRM: deleteAILock(): processing error"); - } + if (err != ERR_OK) + { + log("DBRM: deleteAILock(): processing error", logging::LOG_TYPE_CRITICAL); + throw runtime_error("DBRM: deleteAILock(): processing error"); + } } -void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents, vector* plbidList) +void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents, + vector* plbidList) { - // Here we want to minimize the number of calls to dbrm - // Given that, and the fact that we need to know the column type - // in order to set the invalid min and max correctly in the extents, - // We do the following: - // 1) Maintain a vector of all extents we've looked at. - // 2) Get the list of uncommitted lbids for the transaction. - // 3) Look in that list to see if we've already looked at this extent. - // 4) If not, - // a) lookup the min and max lbid for the extent it belongs to - // b) lookup the column oid for that lbid - // c) add to the vector of extents - // 5) Create a list of CPInfo structures with the first lbid and col type of each extent - // 6) Lookup the column type for each retrieved oid. - // 7) mark each extent invalid, just like we would during update. This sets the proper - // min and max (and set the state to CP_UPDATING. - // 6) Call setExtentsMaxMin to set the state to CP_INVALID. + // Here we want to minimize the number of calls to dbrm + // Given that, and the fact that we need to know the column type + // in order to set the invalid min and max correctly in the extents, + // We do the following: + // 1) Maintain a vector of all extents we've looked at. + // 2) Get the list of uncommitted lbids for the transaction. + // 3) Look in that list to see if we've already looked at this extent. + // 4) If not, + // a) lookup the min and max lbid for the extent it belongs to + // b) lookup the column oid for that lbid + // c) add to the vector of extents + // 5) Create a list of CPInfo structures with the first lbid and col type of each extent + // 6) Lookup the column type for each retrieved oid. + // 7) mark each extent invalid, just like we would during update. This sets the proper + // min and max (and set the state to CP_UPDATING. + // 6) Call setExtentsMaxMin to set the state to CP_INVALID. - vector localLBIDList; + vector localLBIDList; - boost::shared_ptr csc; - CPInfoList_t cpInfos; - CPInfo aInfo; - int oid; - uint16_t dbRoot; - uint32_t partitionNum; - uint16_t segmentNum; - uint32_t fileBlockOffset; + boost::shared_ptr csc; + CPInfoList_t cpInfos; + CPInfo aInfo; + int oid; + uint16_t dbRoot; + uint32_t partitionNum; + uint16_t segmentNum; + uint32_t fileBlockOffset; - // 2) Get the list of uncommitted lbids for the transaction, if we weren't given one. - if (plbidList == NULL) + // 2) Get the list of uncommitted lbids for the transaction, if we weren't given one. + if (plbidList == NULL) + { + getUncommittedExtentLBIDs(static_cast(txnid), localLBIDList); + plbidList = &localLBIDList; + } + + if (plbidList->size() == 0) + { + return; // Nothing to do. + } + + vector::const_iterator iter = plbidList->begin(); + vector::const_iterator end = plbidList->end(); + csc = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(); + + for (; iter != end; ++iter) + { + LBID_t lbid = *iter; + aInfo.firstLbid = lbid; + + // lookup the column oid for that lbid (all we care about is oid here) + if (em->lookupLocal(lbid, oid, dbRoot, partitionNum, segmentNum, fileBlockOffset) == 0) { - getUncommittedExtentLBIDs(static_cast(txnid), localLBIDList); - plbidList = &localLBIDList; - } - - if (plbidList->size() == 0) - { - return; // Nothing to do. - } - - vector::const_iterator iter = plbidList->begin(); - vector::const_iterator end = plbidList->end(); - csc = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(); - - for (; iter != end; ++iter) - { - LBID_t lbid = *iter; - aInfo.firstLbid = lbid; - - // lookup the column oid for that lbid (all we care about is oid here) - if (em->lookupLocal(lbid, oid, dbRoot, partitionNum, segmentNum, fileBlockOffset) == 0) + execplan::CalpontSystemCatalog::ColType colType = csc->colType(oid); + bool isBinaryColumn = colType.colWidth > 8; + aInfo.isBinaryColumn = isBinaryColumn; + if (!isBinaryColumn) + { + if (datatypes::isUnsigned(colType.colDataType)) { - execplan::CalpontSystemCatalog::ColType colType = csc->colType(oid); - bool isBinaryColumn = colType.colWidth > 8; - aInfo.isBinaryColumn = isBinaryColumn; - if (!isBinaryColumn) - { - if (datatypes::isUnsigned(colType.colDataType)) - { - aInfo.max = 0; - aInfo.min = numeric_limits::max(); - } - else - { - aInfo.max = numeric_limits::min(); - aInfo.min = numeric_limits::max(); - } - } - else - { - if (datatypes::isUnsigned(colType.colDataType)) - { - aInfo.bigMax = 0; - aInfo.bigMin = -1; - } - else - { - utils::int128Min(aInfo.bigMax); - utils::int128Max(aInfo.bigMin); - } - } + aInfo.max = 0; + aInfo.min = numeric_limits::max(); } else { - // We have a problem, but we need to put something in. This should never happen. - aInfo.max = numeric_limits::min(); - aInfo.min = numeric_limits::max(); - // MCOL-641 is this correct? - aInfo.isBinaryColumn = false; + aInfo.max = numeric_limits::min(); + aInfo.min = numeric_limits::max(); } - - aInfo.seqNum = allExtents ? SEQNUM_MARK_INVALID_SET_RANGE : SEQNUM_MARK_UPDATING_INVALID_SET_RANGE; - cpInfos.push_back(aInfo); + } + else + { + if (datatypes::isUnsigned(colType.colDataType)) + { + aInfo.bigMax = 0; + aInfo.bigMin = -1; + } + else + { + utils::int128Min(aInfo.bigMax); + utils::int128Max(aInfo.bigMin); + } + } + } + else + { + // We have a problem, but we need to put something in. This should never happen. + aInfo.max = numeric_limits::min(); + aInfo.min = numeric_limits::max(); + // MCOL-641 is this correct? + aInfo.isBinaryColumn = false; } - // Call setExtentsMaxMin to invalidate and set the proper max/min in each extent - setExtentsMaxMin(cpInfos); + aInfo.seqNum = allExtents ? SEQNUM_MARK_INVALID_SET_RANGE : SEQNUM_MARK_UPDATING_INVALID_SET_RANGE; + cpInfos.push_back(aInfo); + } + + // Call setExtentsMaxMin to invalidate and set the proper max/min in each extent + setExtentsMaxMin(cpInfos); } -template -int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, int32_t& seqNum) throw(); +template int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, + int32_t& seqNum) throw(); -template -int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw(); +template int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, + int32_t& seqNum) throw(); -} //namespace +} // namespace BRM diff --git a/versioning/BRM/dbrm.h b/versioning/BRM/dbrm.h index e7ec67dce..e4ddcff27 100644 --- a/versioning/BRM/dbrm.h +++ b/versioning/BRM/dbrm.h @@ -62,7 +62,6 @@ namespace BRM { - /** @brief The interface to the Distributed BRM system. * * There are 3 components of the Distributed BRM (DBRM). @@ -101,953 +100,911 @@ namespace BRM class DBRM { -public: - // The param noBRMFcns suppresses init of the ExtentMap, VSS, VBBM, and CopyLocks. - // It can speed up init if the caller only needs the other structures. - EXPORT DBRM(bool noBRMFcns = false); - EXPORT ~DBRM() throw(); + public: + // The param noBRMFcns suppresses init of the ExtentMap, VSS, VBBM, and CopyLocks. + // It can speed up init if the caller only needs the other structures. + EXPORT DBRM(bool noBRMFcns = false); + EXPORT ~DBRM() throw(); - EXPORT static void refreshShm() - { - MasterSegmentTableImpl::refreshShm(); - ExtentMapImpl::refreshShm(); - FreeListImpl::refreshShm(); - } + EXPORT static void refreshShm() + { + MasterSegmentTableImpl::refreshShm(); + ExtentMapImpl::refreshShm(); + FreeListImpl::refreshShm(); + } - // @bug 1055+ - Added functions below for multiple files per OID enhancement. + // @bug 1055+ - Added functions below for multiple files per OID enhancement. - /** @brief Get the OID, offset, db root, partition, and segment of a logical block ID. - * - * Get the OID, offset, db root, and partition of a logical block ID. - * @param lbid (in) The LBID to look up - * @param verid (in) The version of the LBID to look up - * @param vbFlags (in) If true, look for the block in the version buffer - * if false, the extent map - * @param oid (out) The OID of the file the LBID is allocated to - * @param fileBlockOffset (out) The file block offset of the LBID - * @param dbRoot (out) The DBRoot number that contains the file. - * @param partitionNum (out) The partition number for the file. - * @param segmentNum (out) The segment number for the file. - * - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int lookupLocal(LBID_t lbid, VER_t verid, bool vbFlag, OID_t& oid, - uint16_t& dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset) throw(); + /** @brief Get the OID, offset, db root, partition, and segment of a logical block ID. + * + * Get the OID, offset, db root, and partition of a logical block ID. + * @param lbid (in) The LBID to look up + * @param verid (in) The version of the LBID to look up + * @param vbFlags (in) If true, look for the block in the version buffer + * if false, the extent map + * @param oid (out) The OID of the file the LBID is allocated to + * @param fileBlockOffset (out) The file block offset of the LBID + * @param dbRoot (out) The DBRoot number that contains the file. + * @param partitionNum (out) The partition number for the file. + * @param segmentNum (out) The segment number for the file. + * + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int lookupLocal(LBID_t lbid, VER_t verid, bool vbFlag, OID_t& oid, uint16_t& dbRoot, + uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset) throw(); - /** @brief Get the LBID assigned to the given OID, block offset, partion, and segment. - * - * Get the LBID assigned to the given OID, block offset, partition, and segment. - * @param oid (in) The OID - * @param partitionNum (in) The partition number - * @parm segmentNum (in) The segment number - * @param fileBlockOffset (in) The offset in the OID to return the LBID of - * @param lbid (out) The LBID of the offset of the OID. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int lookupLocal(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, LBID_t& lbid) throw(); + /** @brief Get the LBID assigned to the given OID, block offset, partion, and segment. + * + * Get the LBID assigned to the given OID, block offset, partition, and segment. + * @param oid (in) The OID + * @param partitionNum (in) The partition number + * @parm segmentNum (in) The segment number + * @param fileBlockOffset (in) The offset in the OID to return the LBID of + * @param lbid (out) The LBID of the offset of the OID. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int lookupLocal(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, + LBID_t& lbid) throw(); - /** @brief A dbroot-specific version of lookupLocal() */ - EXPORT int lookupLocal_DBroot(OID_t oid, uint32_t dbroot, - uint32_t partitionNum, uint16_t segmentNum, + /** @brief A dbroot-specific version of lookupLocal() */ + EXPORT int lookupLocal_DBroot(OID_t oid, uint32_t dbroot, uint32_t partitionNum, uint16_t segmentNum, + uint32_t fileBlockOffset, LBID_t& lbid) throw(); + // @bug 1055- + + /** @brief Get the starting LBID assigned to the extent containing + * the specified OID, block offset, partition, and segment. + * + * Get the LBID assigned to the given OID, partition, segment, + * and block offset. + * @param oid (in) The OID + * @param partitionNum (in) The partition number + * @parm segmentNum (in) The segment number + * @param fileBlockOffset (in) The requested offset in the specified OID + * @param lbid (out) The starting LBID of extent with specified offset + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int lookupLocalStartLbid(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, LBID_t& lbid) throw(); - // @bug 1055- - /** @brief Get the starting LBID assigned to the extent containing - * the specified OID, block offset, partition, and segment. - * - * Get the LBID assigned to the given OID, partition, segment, - * and block offset. - * @param oid (in) The OID - * @param partitionNum (in) The partition number - * @parm segmentNum (in) The segment number - * @param fileBlockOffset (in) The requested offset in the specified OID - * @param lbid (out) The starting LBID of extent with specified offset - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int lookupLocalStartLbid(OID_t oid, - uint32_t partitionNum, - uint16_t segmentNum, - uint32_t fileBlockOffset, - LBID_t& lbid) throw(); + /**@brief Do a VSS lookup. + * + * Do a VSS lookup. Gets the version ID of the block the caller should use + * and determines whether it is in the version buffer or the main database. + * @param lbid (in) The block number + * @param qc (in) The SCN & txn list provided by SessionManager::verID(). + * @param txnID (in) If the caller has a transaction ID, put it here. + * Otherwise use 0. + * @param outVer (out) The version the caller should use. + * @param vbFlag (out) If true, the block is in the version buffer; + * false, the block is in the main database. + * @param vbOnly (in) If true, it will only consider entries in the Version Buffer + * (vbFlag will be true on return). If false, it will also consider the main DB + * entries. This defaults to false. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int vssLookup(LBID_t lbid, const QueryContext& qc, VER_t txnID, VER_t* outVer, bool* vbFlag, + bool vbOnly = false) throw(); - /**@brief Do a VSS lookup. - * - * Do a VSS lookup. Gets the version ID of the block the caller should use - * and determines whether it is in the version buffer or the main database. - * @param lbid (in) The block number - * @param qc (in) The SCN & txn list provided by SessionManager::verID(). - * @param txnID (in) If the caller has a transaction ID, put it here. - * Otherwise use 0. - * @param outVer (out) The version the caller should use. - * @param vbFlag (out) If true, the block is in the version buffer; - * false, the block is in the main database. - * @param vbOnly (in) If true, it will only consider entries in the Version Buffer - * (vbFlag will be true on return). If false, it will also consider the main DB - * entries. This defaults to false. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int vssLookup(LBID_t lbid, const QueryContext& qc, VER_t txnID, VER_t* outVer, - bool* vbFlag, bool vbOnly = false) throw(); + /** @brief Do many VSS lookups under one lock + * + * Do many VSS lookups under one lock. + * @param lbids (in) The LBIDs to look up + * @param qc (in) The input version info, equivalent to the verID param in vssLookup() + * @param txnID (in) The input transaction number, equivalent to the txnID param in vssLookup() + * @param out (out) The values equivalent to the out parameters in vssLookup() including the individual + * return codes, ordered as the lbid list. + * @return 0 on success, -1 on a fatal error. + */ + EXPORT int bulkVSSLookup(const std::vector& lbids, const QueryContext_vss& qc, VER_t txnID, + std::vector* out); - /** @brief Do many VSS lookups under one lock - * - * Do many VSS lookups under one lock. - * @param lbids (in) The LBIDs to look up - * @param qc (in) The input version info, equivalent to the verID param in vssLookup() - * @param txnID (in) The input transaction number, equivalent to the txnID param in vssLookup() - * @param out (out) The values equivalent to the out parameters in vssLookup() including the individual return codes, ordered as the lbid list. - * @return 0 on success, -1 on a fatal error. - */ - EXPORT int bulkVSSLookup(const std::vector& lbids, const QueryContext_vss& qc, VER_t txnID, - std::vector* out); + /// returns the version in the main DB files or 0 if none exist + EXPORT VER_t getCurrentVersion(LBID_t lbid, bool* isLocked = NULL) const; - /// returns the version in the main DB files or 0 if none exist - EXPORT VER_t getCurrentVersion(LBID_t lbid, bool* isLocked = NULL) const; + /// returns the highest version # in the version buffer, or -1 if none exist + EXPORT VER_t getHighestVerInVB(LBID_t lbid, + VER_t max = std::numeric_limits::max()) const; // returns - /// returns the highest version # in the version buffer, or -1 if none exist - EXPORT VER_t getHighestVerInVB(LBID_t lbid, VER_t max = std::numeric_limits::max()) const; // returns + /// returns true if that block is in the version buffer, false otherwise including on error + EXPORT bool isVersioned(LBID_t lbid, VER_t version) const; - /// returns true if that block is in the version buffer, false otherwise including on error - EXPORT bool isVersioned(LBID_t lbid, VER_t version) const; + /// Do many getCurrentVersion lookups under one lock grab + EXPORT int bulkGetCurrentVersion(const std::vector& lbids, std::vector* versions, + std::vector* isLocked = NULL) const; - /// Do many getCurrentVersion lookups under one lock grab - EXPORT int bulkGetCurrentVersion(const std::vector& lbids, std::vector* versions, - std::vector* isLocked = NULL) const; + /** @brief Get a complete list of LBIDs assigned to an OID + * + * Get a complete list of LBIDs assigned to an OID. + */ + EXPORT int lookup(OID_t oid, LBIDRange_v& lbidList) throw(); - /** @brief Get a complete list of LBIDs assigned to an OID - * - * Get a complete list of LBIDs assigned to an OID. - */ - EXPORT int lookup(OID_t oid, LBIDRange_v& lbidList) throw(); + /** @brief Allocate a "stripe" of extents for columns in a table. + * + * Allocate a "stripe" of extents for the specified columns and DBRoot + * @param cols (in) List of column OIDs and column widths + * @param dbRoot (in) DBRoot for requested extents. + * @param partitionNum (in/out) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (out) Segment number selected for new extents. + * @param extents (out) list of lbids, numBlks, and fbo for new extents + * @return 0 on success, -1 on error + */ + EXPORT int createStripeColumnExtents(const std::vector& cols, + uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + std::vector& extents) DBRM_THROW; - /** @brief Allocate a "stripe" of extents for columns in a table. - * - * Allocate a "stripe" of extents for the specified columns and DBRoot - * @param cols (in) List of column OIDs and column widths - * @param dbRoot (in) DBRoot for requested extents. - * @param partitionNum (in/out) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (out) Segment number selected for new extents. - * @param extents (out) list of lbids, numBlks, and fbo for new extents - * @return 0 on success, -1 on error - */ - EXPORT int createStripeColumnExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - std::vector& extents) DBRM_THROW; + /** @brief Allocate an extent for a column file + * + * Allocate a column extent for the specified OID and DBRoot. + * @param OID (in) The OID requesting the extent. + * @param colWidth (in) Column width of the OID. + * @param dbRoot (in) DBRoot where extent is to be added. + * @param partitionNum (in/out) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (out) Segment number in file path. + * @param colDataType (in) the column type + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @param startBlockOffset (out) The first block of the extent created. + * @return 0 on success, -1 on error + */ + // @bug 4091: To be deprecated. Replaced by createStripeColumnExtents(). + EXPORT int createColumnExtent_DBroot(OID_t oid, uint32_t colWidth, uint16_t dbRoot, uint32_t& partitionNum, + uint16_t& segmentNum, + execplan::CalpontSystemCatalog::ColDataType colDataType, LBID_t& lbid, + int& allocdSize, uint32_t& startBlockOffset) DBRM_THROW; - /** @brief Allocate an extent for a column file - * - * Allocate a column extent for the specified OID and DBRoot. - * @param OID (in) The OID requesting the extent. - * @param colWidth (in) Column width of the OID. - * @param dbRoot (in) DBRoot where extent is to be added. - * @param partitionNum (in/out) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (out) Segment number in file path. - * @param colDataType (in) the column type - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @param startBlockOffset (out) The first block of the extent created. - * @return 0 on success, -1 on error - */ - // @bug 4091: To be deprecated. Replaced by createStripeColumnExtents(). - EXPORT int createColumnExtent_DBroot(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, + /** @brief Allocate extent for specified segment file + * + * Allocate column extent for the exact segment file + * specified by OID, DBRoot, partition, and segment. + * @param OID (in) The OID requesting the extent. + * @param colWidth (in) Column width of the OID. + * @param dbRoot (in) DBRoot where extent is to be added. + * @param partitionNum (in) Partition number in file path. + * @param segmentNum (in) Segment number in file path. + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @param startBlockOffset (out) The first block of the extent created. + * @return 0 on success, -1 on error + */ + EXPORT int createColumnExtentExactFile(OID_t oid, uint32_t colWidth, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdSize, + LBID_t& lbid, int& allocdSize, uint32_t& startBlockOffset) DBRM_THROW; - /** @brief Allocate extent for specified segment file - * - * Allocate column extent for the exact segment file - * specified by OID, DBRoot, partition, and segment. - * @param OID (in) The OID requesting the extent. - * @param colWidth (in) Column width of the OID. - * @param dbRoot (in) DBRoot where extent is to be added. - * @param partitionNum (in) Partition number in file path. - * @param segmentNum (in) Segment number in file path. - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @param startBlockOffset (out) The first block of the extent created. - * @return 0 on success, -1 on error - */ - EXPORT int createColumnExtentExactFile(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) DBRM_THROW; - - /** @brief Allocate an extent for a dictionary store file - * - * Allocate a dictionary store extent for the specified OID, dbRoot, - * partition number, and segment number. - * @param OID (in) The OID requesting the extent. - * @param dbRoot (in) DBRoot to assign to the extent. - * @param partitionNum (in) Partition number to assign to the extent. - * @param segmentNum (in) Segment number to assign to the extent. - * @param colDataType (in) the column type - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @return 0 on success, -1 on error - */ - EXPORT int createDictStoreExtent(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - LBID_t& lbid, - int& allocdSize) DBRM_THROW; - - /** @brief Rollback (delete) set of column extents for specific OID & DBRoot - * - * Deletes all the extents that logically follow the specified - * column extent; and sets the HWM for the specified extent. - * @param oid OID of the extents to be deleted. - * @param bDeleteAll Indicates if all extents in the oid and dbroot are to - * be deleted; else part#, seg#, and hwm are used. - * @param dbRoot DBRoot of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segmentNum Last segment in partitionNum to be kept. - * @param hwm HWM to be assigned to the last extent that is kept. - * @return 0 on success - */ - EXPORT int rollbackColumnExtents_DBroot(OID_t oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm) DBRM_THROW; - - /** @brief Rollback (delete) a set of dict store extents for an OID & DBRoot - * - * Arguments specify the last stripe. Any extents after this are - * deleted. The hwm's of the extents in the last stripe are updated - * based on the contents of the hwm vector. If hwms is a partial list, - * (as in the first stripe of a partition), then any extents in sub- - * sequent segment files for that partition are deleted. if hwms is empty - * then all the extents in dbRoot are deleted. - * @param oid OID of the extents to be deleted or updated. - * @param dbRoot DBRoot of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segNums Vector of segment numbers for last partition to be kept. - * @param hwms Vector of hwms for the last partition to be kept. - * @return 0 on success - */ - EXPORT int rollbackDictStoreExtents_DBroot(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - const std::vector& segNums, - const std::vector& hwms) DBRM_THROW; - - /** @brief delete of column extents for the specified extents. - * - * Deletes the extents from extent map - * @param extentInfo the information for extents - */ - EXPORT int deleteEmptyColExtents(const std::vector& extentsInfo) DBRM_THROW; - - /** @brief delete of dictionary extents for the specified extents. - * - * Deletes the extents from extent map - * @param extentInfo the information for extents - */ - EXPORT int deleteEmptyDictStoreExtents(const std::vector& extentsInfo) DBRM_THROW; - - /** @brief Delete the extents of an OID and invalidate VSS references to them - * - * Delete the extents assigned to an OID and deletes entries in the VSS - * that refer to the LBIDs used by it. - * @note The old version of this function deliberately did not delete the entries - * in the version buffer. - * @note This function is ridiculously slow right now. - * @param OID The OID of the object being deleted - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int deleteOID(OID_t oid) DBRM_THROW; - - - /** @brief Delete the extents of an OID and invalidate VSS references to them - * - * Delete the extents assigned to an OID and deletes entries in the VSS - * that refer to the LBIDs used by it. - * @note The old version of this function deliberately did not delete the entries - * in the version buffer. - * @note This function is ridiculously slow right now. - * @param OID The OID of the object being deleted - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int deleteOIDs( const std::vector& oids) DBRM_THROW; - - /** @brief Gets the last local high water mark for an OID and dbRoot - * - * Get last local high water mark of an OID for a given dbRoot, relative to - * a segment file. The partition and segment numbers for the pertinent - * segment are also returned. - * @param OID (in) The OID - * @param dbRoot (in) The relevant DBRoot - * @param partitionNum (out) The relevant partition number - * @param segmentNum (out) The relevant segment number - * @param hwm (out) Last local high water mark for specified input - * @param status (out) State of the extent (Available, OutOfService, etc) - * @param bFound (out) Indicates whether an extent was found on the dbRoot - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getLastHWM_DBroot(int OID, uint16_t dbRoot, - uint32_t& partitionNum, uint16_t& segmentNum, HWM_t& hwm, - int& status, bool& bFound) throw(); - - /** @brief Get the "high water mark" of an OID, partition, and segment - * - * Get local high water mark (aka, the highest numbered written block - * offset) for an OID,partition,segment relative to the segment file. - * This applies to either column or dictionary store OIDs. - * @param oid (in) The OID - * @param partitionNum (in) Relevant partition number. - * @param segmentNum (in) Relevant segment number. - * @param hwm (out) The high water mark of oid - * @param status (out) State of the extent (Available, OutOfService, etc) - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getLocalHWM(OID_t oid, uint32_t partitionNum, - uint16_t segmentNum, HWM_t& hwm, int& status) throw(); - - /** @brief Set the "high water mark" of an OID, partition, and segment - * - * Set the local high water mark (aka, the highest numbered written - * block offset) for the segment file referenced by the specified OID, - * partition, and segment number. - * This applies to either column or dictionary store OIDs. - * @param oid (in) The OID - * @param partitionNum (in) Relevant partition number. - * @param segmentNum (in) Relevant segment number. - * @param hwm (in) The high water mark of oid - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int setLocalHWM(OID_t oid, uint32_t partitionNum, - uint16_t segmentNum, HWM_t hwm) DBRM_THROW; - - EXPORT int bulkSetHWM(const std::vector&, VER_t transID = 0) DBRM_THROW; - - /** @brief Does setLocalHWM, casual partitioning changes, & commit. The HWM & - * CP changes are atomic with this fcn. All functionality is optional. Passing - * in an empty vector for any of these parms makes it do nothing for the - * corresponding operation. - * - * It returns 0 if all operations succeed, -1 if any operation fails. - * The controllernode will undo all changes made on error. - */ - EXPORT int bulkSetHWMAndCP(const std::vector& hwmArgs, - const std::vector& setCPDataArgs, - const std::vector& mergeCPDataArgs, - VER_t transID) DBRM_THROW; - - EXPORT int bulkUpdateDBRoot(const std::vector&); - - /** @brief Get HWM information about last segment file for each DBRoot - * assigned to a specific PM. - * - * Vector will contain an entry for each DBRoot. If no "available" extents - * are found for a DBRoot, then totalBlocks will be 0 (and hwmExtentIndex - * will be -1) for that DBRoot. - * @param oid The oid of interest. - * @param pmNumber The PM number of interest. - * @param emDbRootHwmInfos The vector of DbRoot/HWM related objects. - */ - EXPORT int getDbRootHWMInfo(OID_t oid, uint16_t pmNumber, - EmDbRootHWMInfo_v& emDbRootHwmInfos) throw(); - - /** @brief Get the status (AVAILABLE, OUTOFSERVICE, etc) for the - * segment file represented by the specified OID, part# and seg#. - * - * Unlike many of the other DBRM functions, this function does - * not return a bad return code if no extent is found; the "found" - * flag indicates whether an extent was found or not. - * - * @param oid (in) The OID of interest - * @param partitionNum (in) The partition number of interest - * @param segmentNum (in) The segment number of interest - * @param bFound (out) Indicates if extent was found or not - * @param status (out) The state of the extents in the specified - * segment file. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getExtentState(OID_t oid, uint32_t partitionNum, - uint16_t segmentNum, bool& bFound, int& status) throw(); - - /** @brief Gets the extents of a given OID - * - * Gets the extents of a given OID. - * @param OID (in) The OID to get the extents for. - * @param entries (out) A snapshot of the OID's Extent Map entries - * sorted by starting LBID; note that The Real Entries can change at - * any time. Also, internally the size of an extent is measured in - * multiples of 1024 LBIDs. So, if the size field is 10, it represents - * 10240 LBIDs. - * @param sorted (in) indicates if output is to be sorted - * @param notFoundErr (in) indicates if no extents is considered an err - * @param incOutOfService (in) include/exclude out of service extents - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getExtents(int OID, std::vector& entries, - bool sorted = true, bool notFoundErr = true, - bool incOutOfService = false); - - /** @brief Gets the extents of a given OID under specified dbroot - * - * Gets the extents of a given OID under specified dbroot. - * @param OID (in) The OID to get the extents for. - * @param entries (out) A snapshot of the OID's Extent Map entries - * @param dbroot (in) dbroot - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getExtents_dbroot(int OID, std::vector& entries, - const uint16_t dbroot) throw(); - - /** @brief Gets the number of extents for the specified OID and DBRoot - * - * @param OID (in) The OID of interest - * @param dbroot (in) The DBRoot of interest - * @param incOutOfService (in) include/exclude out of service extents - * @param numExtents (out) number of extents found for OID and dbroot - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getExtentCount_dbroot(int OID, uint16_t dbroot, - bool incOutOfService, uint64_t& numExtents) throw(); - - /** @brief Gets the number of rows in an extent - * - * Gets the number of rows in an extent. - * @return The extent size - */ -// dmc-getExtentSize() to be deprecated, replaced by getExtentRows() - EXPORT int getExtentSize() throw(); - EXPORT unsigned getExtentRows() throw(); - - /** @brief Gets the DBRoot for the specified system catalog OID - * - * Function should only be called for System Catalog OIDs, as it assumes - * the OID is fully contained on a single DBRoot, returning the first - * DBRoot found. This only makes since for a System Catalog OID, because - * all other column OIDs can span multiple DBRoots. - * - * @param oid The system catalog OID - * @param dbRoot (out) the DBRoot holding the system catalog OID - */ - EXPORT int getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) throw(); - - /** @brief Delete a Partition for the specified OID(s). - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be deleted. - */ - EXPORT int deletePartition(const std::vector& oids, - const std::set& partitionNums, std::string& emsg) DBRM_THROW; - - /** @brief Mark a Partition for the specified OID(s) as out of service. - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be marked out of service. - */ - EXPORT int markAllPartitionForDeletion(const std::vector& oids) DBRM_THROW; - - /** @brief Mark a Partition for the specified OID(s) as out of service. - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be marked out of service. - */ - EXPORT int markPartitionForDeletion(const std::vector& oids, - const std::set& partitionNums, std::string& emsg) DBRM_THROW; - - /** @brief Restore a Partition for the specified OID(s). - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be restored. - */ - EXPORT int restorePartition(const std::vector& oids, - const std::set& partitionNums, std::string& emsg) DBRM_THROW; - - /** @brief Get the list of out-of-service partitions for a given OID - * - * @param OID (in) the OID of interest. - * @param partitionNums (out) the out-of-service partitions for the oid. - * partitionNums will be in sorted order. - */ - EXPORT int getOutOfServicePartitions(OID_t oid, - std::set& partitionNums) throw(); - - /** @brief Delete all rows in the extent map that reside on the given DBRoot - * - * @param dbroot (in) the dbroot to be deleted - */ - EXPORT int deleteDBRoot(uint16_t dbroot) DBRM_THROW; - - /** @brief Is the specified DBRoot empty with no extents. - * Returns error if extentmap shared memory is not loaded. - * - * @param dbroot DBRoot of interest - * @param isEmpty (output) Flag indiicating whether DBRoot is empty - * @param errMsg (output) Error message corresponding to bad return code - * @return ERR_OK on success - */ - EXPORT int isDBRootEmpty(uint16_t dbroot, - bool& isEmpty, std::string& errMsg) throw(); - - /** @brief Registers a version buffer entry. - * - * Registers a version buffer entry at with - * values of . - * @note The version buffer locations must hold the 'copy' lock - * first. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, - uint32_t vbFBO) DBRM_THROW; - - /** @brief Bulk registers a version buffer entry. - * - * Similar to writeVBEntry, but registers the version buffer - * entries in bulk for a list of lbids and vbFBOs, for a given - * transID and vbOID. - * @note The version buffer locations must hold the 'copy' lock - * first. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int bulkWriteVBEntry(VER_t transID, - const std::vector& lbids, - OID_t vbOID, - const std::vector& vbFBOs) DBRM_THROW; - - /** @brief Retrieves a list of uncommitted LBIDs. - * - * Retrieves a list of uncommitted LBIDs for the given transaction ID. - * @param lbidList (out) On success this contains the ranges of LBIDs - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getUncommittedLBIDs(VER_t transID, - std::vector& lbidList) - throw(); - - /** @brief Does what you think it does. - * - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getDBRootsForRollback(VER_t transID, std::vector* dbRootList) - throw(); - - // @bug 1509. Added getUncommittedExtentLBIDs function. - /** @brief Retrieves a list of uncommitted extent LBIDs. - * - * Retrieves a list of uncommitted LBIDs for the given transaction ID. - * This function differs from getUncommittedLBIDs in that only one LBID per - * extent is returned. It is used to return a list that can be used to update - * casual partitioning information which is tracked at the extent level rather - * than the block level. - * @param lbidList (out) On success this contains the ranges of LBIDs - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getUncommittedExtentLBIDs(VER_t transID, std::vector& lbidList) throw(); - - /** @brief Atomically prepare to copy data to the version buffer - * - * Atomically sets the copy flag on the specified LBID ranges - * and allocate blocks in the version buffer to copy them to. - * If any LBID in the range cannot be locked, none will be - * and this will return -1. - * @param transID The transaction ID doing the operation - * @param ranges (in) A list of LBID ranges that will be copied - * @param freeList (out) On success, a list of ranges of the version - * buffer blocks to copy the LBID range to. - * @note The caller must do a rollback or commit immediately after getting ERR_DEADLOCK (6). - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int beginVBCopy(VER_t transID, uint16_t dbRoot, const LBIDRange_v& ranges, - VBRange_v& freeList) DBRM_THROW; - - /** @brief Atomically unset the copy lock & update the VSS. Beware! Read the warning! - * - * Atomically unset the copy lock for the specified LBID ranges - * and add a new locked VSS entry for each LBID in the range. - * @note The elements of the ranges parameter MUST be the - * same elements passed to beginVBCopy(). The number and order of the - * elements can be different, but every element in ranges must also - * have been an element in beginVBCopy's ranges. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int endVBCopy(VER_t transID, const LBIDRange_v& ranges) - DBRM_THROW; - - /** @brief Commit the changes made for the given transaction. - * - * This unlocks the VSS entries with VerID = transID. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int vbCommit(VER_t transID) DBRM_THROW; - - /** @brief Reverse the changes made during the given transaction. - * - * Record that the given LBID was reverted to version verID. - * @warning This removes the copy locks held on all ranges by transID. - * @param transID The transaction ID - * @param lbidList The list of ranges to rollback. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int vbRollback(VER_t transID, const LBIDRange_v& lbidList) - DBRM_THROW; - - /** @brief Reverse the changes made during the given transaction. - * - * Record that the given LBID was reverted to version verID. - * @warning This removes the copy locks held on all ranges by transID. - * @param transID The transaction ID - * @param lbidList The list of singular LBIDs to rollback. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int vbRollback(VER_t transID, const std::vector& lbidList) - DBRM_THROW; - - - EXPORT int getUnlockedLBIDs(BlockList_t* list) DBRM_THROW; - - /** @brief Reinitialize the versioning data structures. - * - * This entry point empties the VSS and VBBM structures on the - * slave nodes. At system startup and after recovery, the data left in - * the VSS and VBBM are unnecessary. The primary purpose of this function - * is to free up memory. - * @return 0 on success, non-0 on error (see brmtypes.h). - */ - EXPORT int clear() DBRM_THROW; - - /** @brief Check the consistency of each data structure - * - * Check the consistency of each data structure - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int checkConsistency() throw(); - - /** @brief Get a list of the transactions currently in progress. - * - * Get a list of the transactions currently in progress. This scans - * the copy locks & VSS for LBIDs that are locked by some transaction - * and stores the transaction ID. - * - * @param txnList Caller-supplied set to store the results in. - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT int getCurrentTxnIDs(std::set& txnList) throw(); - - /** @brief Persistence API. Saves the local Extent Map to a file. - * - * Persistence API. Saves the local Extent Map to a file. - * - * @param filename Relative or absolute path to save to. - * @return 0 on success, -1 on error - */ - EXPORT int saveExtentMap(const std::string& filename) throw(); - - /** @brief Persistence API. Saves all BRM structures. - * - * Saves all local BRM structures to files. - * - * @param filename The filename prefix to use. Saves 4 files with that prefix. - * @return 0 on success, -1 on error - */ - EXPORT int saveState(std::string filename) throw(); - - /** @brief Persistence API. Saves all BRM structures using the filenames from Columnstore.xml. - * - * Saves all local BRM structures to files. - * - * @param filename The filename prefix to use. Saves 4 files with that prefix. - * @return 0 on success, -1 on error - */ - EXPORT int saveState() throw(); - - /** @brief A function that forces the BRM to write a snapshot of its data structures to disk - * - * A function that forces the BRM to write a snapshot of its data structures to disk. - * This happens automatically after commits and rollbacks. Bulk imports should call - * this fcn after an import. - * - * @return 0 on success, non-0 on error (see brmtypes.h). - */ - EXPORT int takeSnapshot() throw(); - - /* SessionManager interface */ - EXPORT const QueryContext verID(); - EXPORT const QueryContext sysCatVerID(); - EXPORT const TxnID newTxnID(const SessionManagerServer::SID session, bool block, - bool isDDL = false); - EXPORT void committed(BRM::TxnID& txnid); - EXPORT void rolledback(BRM::TxnID& txnid); - EXPORT const BRM::TxnID getTxnID(const SessionManagerServer::SID session); - EXPORT boost::shared_array SIDTIDMap(int& len); - EXPORT void sessionmanager_reset(); - - /* Note, these pull #s from two separate sequences. That is, they both - return 0, then 1, 2, 3, etc. */ - EXPORT uint32_t getUnique32(); - EXPORT uint64_t getUnique64(); - - /* New table lock interface */ - /* returns a unique ID (> 0) for the lock on success, 0 on failure. - * Also, on failure, the ownerName, pid, and session ID parameters will be set - * to the owner of one of the overlapping locks. */ - EXPORT uint64_t getTableLock(const std::vector& pmList, uint32_t tableOID, - std::string* ownerName, uint32_t* ownerPID, int32_t* ownerSessionID, - int32_t* ownerTxnID, LockState state); - EXPORT bool releaseTableLock(uint64_t id); - EXPORT bool changeState(uint64_t id, LockState state); - EXPORT bool changeOwner(uint64_t id, const std::string& ownerName, uint32_t ownerPID, - int32_t ownerSessionID, int32_t ownerTxnID); - EXPORT bool checkOwner(uint64_t id); - EXPORT std::vector getAllTableLocks(); - EXPORT void releaseAllTableLocks(); - EXPORT bool getTableLockInfo(uint64_t id, TableLockInfo* out); - - /** Casual partitioning support **/ - EXPORT int markExtentInvalid(const LBID_t lbid, - execplan::CalpontSystemCatalog::ColDataType colDataType) DBRM_THROW; - EXPORT int markExtentsInvalid(const std::vector& lbids, - const std::vector& colDataTypes) DBRM_THROW; - template - EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw(); - - EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum) DBRM_THROW; - - EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw(); - - /** @brief Updates the max and min casual partitioning info for the passed extents. - * - * @bug 1970. - * - * @param cpInfos vector of CPInfo objects. The firstLbid must be the first LBID in the extent. - * @return 0 on success, -1 on error - */ - EXPORT int setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW; - - /** @brief Merges max/min casual partitioning info for the specified - * extents, with the current CP info. - * - * @param cpInfos vector of CPInfo objects. The Lbids must be the - * starting LBID in the relevant extent. - * @return 0 on success, -1 on error - */ - EXPORT int mergeExtentsMaxMin(const CPInfoMergeList_t& cpInfos) DBRM_THROW; - - /* read-side interface for locking LBID ranges (used by PrimProc) */ - EXPORT void lockLBIDRange(LBID_t start, uint32_t count); - EXPORT void releaseLBIDRange(LBID_t start, uint32_t count); - - /* write-side interface for locking LBID ranges (used by DML) */ - EXPORT int dmlLockLBIDRanges(const std::vector& ranges, int txnID); - EXPORT int dmlReleaseLBIDRanges(const std::vector& ranges); - - /* OAM Interface */ - EXPORT int halt() DBRM_THROW; - EXPORT int resume() DBRM_THROW; - EXPORT int forceReload() DBRM_THROW; - EXPORT int setReadOnly(bool b) DBRM_THROW; - EXPORT int isReadWrite() throw(); - - EXPORT bool isEMEmpty() throw(); - - EXPORT std::vector getEMFreeListEntries() throw(); - - /** @brief Check if the system is ready for updates - * - * Check is the system has completed rollback processing and is - * ready for updates - */ - EXPORT int getSystemReady() throw(); - - /** @brief Check if the system is ready for queries - * - * Check is the system has completed rollback processing and is - * ready for updates - */ - EXPORT int getSystemQueryReady() throw(); - - /** @brief Check if the system is suspended - * - * Suspended is caused by user at calpont-console - * @return 0 - system is not suspended, > 0 - system is - * suspended, < 0 on error - */ - EXPORT int getSystemSuspended() throw(); - - /** @brief Check if system suspension is pending - * - * If the user at calpont-console asks for suspension, but ddl, - * dml or cpimport are running, then we can't suspend. If user - * says suspend when all work completed, then this flag is set - * to prevent new work from starting. - * @param bRollback (out) if true, system is in a rollback mode - * before suspension. - * @return 0 - system is not in a suspend pending state, > 0 - - * system is in a suspend pending state, < 0 on error - */ - EXPORT int getSystemSuspendPending(bool& bRollback) throw(); - - /** @brief Check if system shutdown is pending - * - * If the user at calpont-console asks for shutdown, but ddl, - * dml or cpimport are running, then we can't shutdown. If user - * says shutdown when all work completed, then this flag is set - * to prevent new work from starting. - * @param bRollback (out) if true, system is in rollback mode - * before shutdown. - * @param bForce (out) if true, system is in force shutdown - * mode. No work of any kind should be done. - * @return 0 - system is not pending a shutdown, > 0 - system is - * pending a shutdown, < 0 on error - */ - EXPORT int getSystemShutdownPending(bool& bRollback, bool& bForce) throw(); - - /** @brief Mark the system as ready for updates - * - * The system has completed rollback processing and is therefore ready for updates - * @param bReady the state the ready flag should be set to. - * @return < 0 on error - */ - EXPORT int setSystemReady(bool bReady) throw(); - - /** @brief Mark the system as ready for queries - * - * The system has completed bring up all modules and is ready for queries - * Queries are allowed during DMLRollback, so SS_READY won't fork for queries. - * @param bReady the state the ready flag should be set to. - * @return < 0 on error - */ - EXPORT int setSystemQueryReady(bool bReady) throw(); - - /** @brief Mark the system as suspended (or not) - * - * @param bSuspended the suspend state the system should be set - * to. - * @return < 0 on error - */ - EXPORT int setSystemSuspended(bool bSuspended) throw(); - - /** @brief Mark the system as suspension pending - * - * @param bPending the suspend pending state the system should - * be set to. - * @param bRollback if true, rollback all active transactions - * before full suspension of writes (only used - * if bPending is true). - * @return < 0 on error - */ - EXPORT int setSystemSuspendPending(bool bPending, bool bRollback = false) throw(); - - /** @brief Mark the system as shutdown pending - * - * @param bPending the suspend pending state the system should - * be set to. - * @param bRollback if true, rollback all active transactions - * before shutting down. (only used if bPending - * is true). - * @param bForce if true, we're shutting down now. No further - * work, including rollback, should be done. - * (only used if bPending is true). - * @return < 0 on error - */ - EXPORT int setSystemShutdownPending(bool bPending, bool bRollback = false, bool bForce = false) throw(); - - /** @brief get the flags in the system state bit map. - * @param stateFlags (out) - * @return < 0 on error - */ - int getSystemState(uint32_t& stateFlags) throw(); - - /** @brief set the flags in the system state bit map. - * - * sets the flags that are set in stateFlags leaving other flags undisturbed - * @param stateFlags (out) - * @return < 0 on error - */ - int setSystemState(uint32_t stateFlags) throw(); - - /** @brief set the flags in the system state bit map. - * - * Clears the flags that are set in stateFlags leaving other flags undisturbed - * @param stateFlags (out) - * @return < 0 on error - */ - int clearSystemState(uint32_t stateFlags) throw(); - - /** @brief Check to see if Controller Node is functioning - */ - bool isDBRMReady() throw(); - - /* OID Manager interface. See oidserver.h for the API documentation. */ - EXPORT int allocOIDs(int num); - EXPORT void returnOIDs(int start, int end); - EXPORT int oidm_size(); - EXPORT int allocVBOID(uint32_t dbroot); - EXPORT int getDBRootOfVBOID(uint32_t vbOID); - EXPORT std::vector getVBOIDToDBRootMap(); - - /* Autoincrement interface */ - EXPORT void startAISequence(uint32_t OID, uint64_t firstNum, uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType); - EXPORT bool getAIRange(uint32_t OID, uint32_t count, uint64_t* firstNum); - EXPORT bool getAIValue(uint32_t OID, uint64_t* value); - EXPORT void resetAISequence(uint32_t OID, uint64_t value); - EXPORT void getAILock(uint32_t OID); - EXPORT void releaseAILock(uint32_t OID); - - /* Added to support unsigned */ - /** @brief Invalidate the casual partitioning for all uncommited - * extents. - * - * Either txnid or plbidList can be passed . Only one will be - * used. - * @param txnid (in) - The transaction for which to get the lbid - * list. - * @param plbidList (in) - a list of lbids whose extents are to - * be invalidated. Only one lbid per extent should - * be in this list, such as returned in - * getUncommittedExtentLBIDs(). - * @return nothing. - */ - EXPORT void invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, - bool allExtents, - std::vector* plbidList = NULL); - -private: - DBRM(const DBRM& brm); - DBRM& operator=(const DBRM& brm); - int8_t send_recv(const messageqcpp::ByteStream& in, - messageqcpp::ByteStream& out) throw(); - - void deleteAISequence(uint32_t OID); // called as part of deleteOID & deleteOIDs - - boost::scoped_ptr mst; - boost::scoped_ptr em; - boost::scoped_ptr vbbm; - boost::scoped_ptr vss; - boost::scoped_ptr copylocks; - messageqcpp::MessageQueueClient* msgClient; - std::string masterName; - boost::mutex mutex; - config::Config* config; - bool fDebug; + /** @brief Allocate an extent for a dictionary store file + * + * Allocate a dictionary store extent for the specified OID, dbRoot, + * partition number, and segment number. + * @param OID (in) The OID requesting the extent. + * @param dbRoot (in) DBRoot to assign to the extent. + * @param partitionNum (in) Partition number to assign to the extent. + * @param segmentNum (in) Segment number to assign to the extent. + * @param colDataType (in) the column type + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @return 0 on success, -1 on error + */ + EXPORT int createDictStoreExtent(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum, + LBID_t& lbid, int& allocdSize) DBRM_THROW; + + /** @brief Rollback (delete) set of column extents for specific OID & DBRoot + * + * Deletes all the extents that logically follow the specified + * column extent; and sets the HWM for the specified extent. + * @param oid OID of the extents to be deleted. + * @param bDeleteAll Indicates if all extents in the oid and dbroot are to + * be deleted; else part#, seg#, and hwm are used. + * @param dbRoot DBRoot of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segmentNum Last segment in partitionNum to be kept. + * @param hwm HWM to be assigned to the last extent that is kept. + * @return 0 on success + */ + EXPORT int rollbackColumnExtents_DBroot(OID_t oid, bool bDeleteAll, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, HWM_t hwm) DBRM_THROW; + + /** @brief Rollback (delete) a set of dict store extents for an OID & DBRoot + * + * Arguments specify the last stripe. Any extents after this are + * deleted. The hwm's of the extents in the last stripe are updated + * based on the contents of the hwm vector. If hwms is a partial list, + * (as in the first stripe of a partition), then any extents in sub- + * sequent segment files for that partition are deleted. if hwms is empty + * then all the extents in dbRoot are deleted. + * @param oid OID of the extents to be deleted or updated. + * @param dbRoot DBRoot of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segNums Vector of segment numbers for last partition to be kept. + * @param hwms Vector of hwms for the last partition to be kept. + * @return 0 on success + */ + EXPORT int rollbackDictStoreExtents_DBroot(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, + const std::vector& segNums, + const std::vector& hwms) DBRM_THROW; + + /** @brief delete of column extents for the specified extents. + * + * Deletes the extents from extent map + * @param extentInfo the information for extents + */ + EXPORT int deleteEmptyColExtents(const std::vector& extentsInfo) DBRM_THROW; + + /** @brief delete of dictionary extents for the specified extents. + * + * Deletes the extents from extent map + * @param extentInfo the information for extents + */ + EXPORT int deleteEmptyDictStoreExtents(const std::vector& extentsInfo) DBRM_THROW; + + /** @brief Delete the extents of an OID and invalidate VSS references to them + * + * Delete the extents assigned to an OID and deletes entries in the VSS + * that refer to the LBIDs used by it. + * @note The old version of this function deliberately did not delete the entries + * in the version buffer. + * @note This function is ridiculously slow right now. + * @param OID The OID of the object being deleted + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int deleteOID(OID_t oid) DBRM_THROW; + + /** @brief Delete the extents of an OID and invalidate VSS references to them + * + * Delete the extents assigned to an OID and deletes entries in the VSS + * that refer to the LBIDs used by it. + * @note The old version of this function deliberately did not delete the entries + * in the version buffer. + * @note This function is ridiculously slow right now. + * @param OID The OID of the object being deleted + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int deleteOIDs(const std::vector& oids) DBRM_THROW; + + /** @brief Gets the last local high water mark for an OID and dbRoot + * + * Get last local high water mark of an OID for a given dbRoot, relative to + * a segment file. The partition and segment numbers for the pertinent + * segment are also returned. + * @param OID (in) The OID + * @param dbRoot (in) The relevant DBRoot + * @param partitionNum (out) The relevant partition number + * @param segmentNum (out) The relevant segment number + * @param hwm (out) Last local high water mark for specified input + * @param status (out) State of the extent (Available, OutOfService, etc) + * @param bFound (out) Indicates whether an extent was found on the dbRoot + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + HWM_t& hwm, int& status, bool& bFound) throw(); + + /** @brief Get the "high water mark" of an OID, partition, and segment + * + * Get local high water mark (aka, the highest numbered written block + * offset) for an OID,partition,segment relative to the segment file. + * This applies to either column or dictionary store OIDs. + * @param oid (in) The OID + * @param partitionNum (in) Relevant partition number. + * @param segmentNum (in) Relevant segment number. + * @param hwm (out) The high water mark of oid + * @param status (out) State of the extent (Available, OutOfService, etc) + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t& hwm, + int& status) throw(); + + /** @brief Set the "high water mark" of an OID, partition, and segment + * + * Set the local high water mark (aka, the highest numbered written + * block offset) for the segment file referenced by the specified OID, + * partition, and segment number. + * This applies to either column or dictionary store OIDs. + * @param oid (in) The OID + * @param partitionNum (in) Relevant partition number. + * @param segmentNum (in) Relevant segment number. + * @param hwm (in) The high water mark of oid + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int setLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm) DBRM_THROW; + + EXPORT int bulkSetHWM(const std::vector&, VER_t transID = 0) DBRM_THROW; + + /** @brief Does setLocalHWM, casual partitioning changes, & commit. The HWM & + * CP changes are atomic with this fcn. All functionality is optional. Passing + * in an empty vector for any of these parms makes it do nothing for the + * corresponding operation. + * + * It returns 0 if all operations succeed, -1 if any operation fails. + * The controllernode will undo all changes made on error. + */ + EXPORT int bulkSetHWMAndCP(const std::vector& hwmArgs, + const std::vector& setCPDataArgs, + const std::vector& mergeCPDataArgs, VER_t transID) DBRM_THROW; + + EXPORT int bulkUpdateDBRoot(const std::vector&); + + /** @brief Get HWM information about last segment file for each DBRoot + * assigned to a specific PM. + * + * Vector will contain an entry for each DBRoot. If no "available" extents + * are found for a DBRoot, then totalBlocks will be 0 (and hwmExtentIndex + * will be -1) for that DBRoot. + * @param oid The oid of interest. + * @param pmNumber The PM number of interest. + * @param emDbRootHwmInfos The vector of DbRoot/HWM related objects. + */ + EXPORT int getDbRootHWMInfo(OID_t oid, uint16_t pmNumber, EmDbRootHWMInfo_v& emDbRootHwmInfos) throw(); + + /** @brief Get the status (AVAILABLE, OUTOFSERVICE, etc) for the + * segment file represented by the specified OID, part# and seg#. + * + * Unlike many of the other DBRM functions, this function does + * not return a bad return code if no extent is found; the "found" + * flag indicates whether an extent was found or not. + * + * @param oid (in) The OID of interest + * @param partitionNum (in) The partition number of interest + * @param segmentNum (in) The segment number of interest + * @param bFound (out) Indicates if extent was found or not + * @param status (out) The state of the extents in the specified + * segment file. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getExtentState(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, bool& bFound, + int& status) throw(); + + /** @brief Gets the extents of a given OID + * + * Gets the extents of a given OID. + * @param OID (in) The OID to get the extents for. + * @param entries (out) A snapshot of the OID's Extent Map entries + * sorted by starting LBID; note that The Real Entries can change at + * any time. Also, internally the size of an extent is measured in + * multiples of 1024 LBIDs. So, if the size field is 10, it represents + * 10240 LBIDs. + * @param sorted (in) indicates if output is to be sorted + * @param notFoundErr (in) indicates if no extents is considered an err + * @param incOutOfService (in) include/exclude out of service extents + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getExtents(int OID, std::vector& entries, bool sorted = true, + bool notFoundErr = true, bool incOutOfService = false); + + /** @brief Gets the extents of a given OID under specified dbroot + * + * Gets the extents of a given OID under specified dbroot. + * @param OID (in) The OID to get the extents for. + * @param entries (out) A snapshot of the OID's Extent Map entries + * @param dbroot (in) dbroot + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getExtents_dbroot(int OID, std::vector& entries, const uint16_t dbroot) throw(); + + /** @brief Gets the number of extents for the specified OID and DBRoot + * + * @param OID (in) The OID of interest + * @param dbroot (in) The DBRoot of interest + * @param incOutOfService (in) include/exclude out of service extents + * @param numExtents (out) number of extents found for OID and dbroot + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfService, + uint64_t& numExtents) throw(); + + /** @brief Gets the number of rows in an extent + * + * Gets the number of rows in an extent. + * @return The extent size + */ + // dmc-getExtentSize() to be deprecated, replaced by getExtentRows() + EXPORT int getExtentSize() throw(); + EXPORT unsigned getExtentRows() throw(); + + /** @brief Gets the DBRoot for the specified system catalog OID + * + * Function should only be called for System Catalog OIDs, as it assumes + * the OID is fully contained on a single DBRoot, returning the first + * DBRoot found. This only makes since for a System Catalog OID, because + * all other column OIDs can span multiple DBRoots. + * + * @param oid The system catalog OID + * @param dbRoot (out) the DBRoot holding the system catalog OID + */ + EXPORT int getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) throw(); + + /** @brief Delete a Partition for the specified OID(s). + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be deleted. + */ + EXPORT int deletePartition(const std::vector& oids, const std::set& partitionNums, + std::string& emsg) DBRM_THROW; + + /** @brief Mark a Partition for the specified OID(s) as out of service. + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be marked out of service. + */ + EXPORT int markAllPartitionForDeletion(const std::vector& oids) DBRM_THROW; + + /** @brief Mark a Partition for the specified OID(s) as out of service. + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be marked out of service. + */ + EXPORT int markPartitionForDeletion(const std::vector& oids, + const std::set& partitionNums, + std::string& emsg) DBRM_THROW; + + /** @brief Restore a Partition for the specified OID(s). + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be restored. + */ + EXPORT int restorePartition(const std::vector& oids, const std::set& partitionNums, + std::string& emsg) DBRM_THROW; + + /** @brief Get the list of out-of-service partitions for a given OID + * + * @param OID (in) the OID of interest. + * @param partitionNums (out) the out-of-service partitions for the oid. + * partitionNums will be in sorted order. + */ + EXPORT int getOutOfServicePartitions(OID_t oid, std::set& partitionNums) throw(); + + /** @brief Delete all rows in the extent map that reside on the given DBRoot + * + * @param dbroot (in) the dbroot to be deleted + */ + EXPORT int deleteDBRoot(uint16_t dbroot) DBRM_THROW; + + /** @brief Is the specified DBRoot empty with no extents. + * Returns error if extentmap shared memory is not loaded. + * + * @param dbroot DBRoot of interest + * @param isEmpty (output) Flag indiicating whether DBRoot is empty + * @param errMsg (output) Error message corresponding to bad return code + * @return ERR_OK on success + */ + EXPORT int isDBRootEmpty(uint16_t dbroot, bool& isEmpty, std::string& errMsg) throw(); + + /** @brief Registers a version buffer entry. + * + * Registers a version buffer entry at with + * values of . + * @note The version buffer locations must hold the 'copy' lock + * first. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, uint32_t vbFBO) DBRM_THROW; + + /** @brief Bulk registers a version buffer entry. + * + * Similar to writeVBEntry, but registers the version buffer + * entries in bulk for a list of lbids and vbFBOs, for a given + * transID and vbOID. + * @note The version buffer locations must hold the 'copy' lock + * first. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int bulkWriteVBEntry(VER_t transID, const std::vector& lbids, OID_t vbOID, + const std::vector& vbFBOs) DBRM_THROW; + + /** @brief Retrieves a list of uncommitted LBIDs. + * + * Retrieves a list of uncommitted LBIDs for the given transaction ID. + * @param lbidList (out) On success this contains the ranges of LBIDs + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getUncommittedLBIDs(VER_t transID, std::vector& lbidList) throw(); + + /** @brief Does what you think it does. + * + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getDBRootsForRollback(VER_t transID, std::vector* dbRootList) throw(); + + // @bug 1509. Added getUncommittedExtentLBIDs function. + /** @brief Retrieves a list of uncommitted extent LBIDs. + * + * Retrieves a list of uncommitted LBIDs for the given transaction ID. + * This function differs from getUncommittedLBIDs in that only one LBID per + * extent is returned. It is used to return a list that can be used to update + * casual partitioning information which is tracked at the extent level rather + * than the block level. + * @param lbidList (out) On success this contains the ranges of LBIDs + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getUncommittedExtentLBIDs(VER_t transID, std::vector& lbidList) throw(); + + /** @brief Atomically prepare to copy data to the version buffer + * + * Atomically sets the copy flag on the specified LBID ranges + * and allocate blocks in the version buffer to copy them to. + * If any LBID in the range cannot be locked, none will be + * and this will return -1. + * @param transID The transaction ID doing the operation + * @param ranges (in) A list of LBID ranges that will be copied + * @param freeList (out) On success, a list of ranges of the version + * buffer blocks to copy the LBID range to. + * @note The caller must do a rollback or commit immediately after getting ERR_DEADLOCK (6). + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int beginVBCopy(VER_t transID, uint16_t dbRoot, const LBIDRange_v& ranges, + VBRange_v& freeList) DBRM_THROW; + + /** @brief Atomically unset the copy lock & update the VSS. Beware! Read the warning! + * + * Atomically unset the copy lock for the specified LBID ranges + * and add a new locked VSS entry for each LBID in the range. + * @note The elements of the ranges parameter MUST be the + * same elements passed to beginVBCopy(). The number and order of the + * elements can be different, but every element in ranges must also + * have been an element in beginVBCopy's ranges. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int endVBCopy(VER_t transID, const LBIDRange_v& ranges) DBRM_THROW; + + /** @brief Commit the changes made for the given transaction. + * + * This unlocks the VSS entries with VerID = transID. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int vbCommit(VER_t transID) DBRM_THROW; + + /** @brief Reverse the changes made during the given transaction. + * + * Record that the given LBID was reverted to version verID. + * @warning This removes the copy locks held on all ranges by transID. + * @param transID The transaction ID + * @param lbidList The list of ranges to rollback. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int vbRollback(VER_t transID, const LBIDRange_v& lbidList) DBRM_THROW; + + /** @brief Reverse the changes made during the given transaction. + * + * Record that the given LBID was reverted to version verID. + * @warning This removes the copy locks held on all ranges by transID. + * @param transID The transaction ID + * @param lbidList The list of singular LBIDs to rollback. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int vbRollback(VER_t transID, const std::vector& lbidList) DBRM_THROW; + + EXPORT int getUnlockedLBIDs(BlockList_t* list) DBRM_THROW; + + /** @brief Reinitialize the versioning data structures. + * + * This entry point empties the VSS and VBBM structures on the + * slave nodes. At system startup and after recovery, the data left in + * the VSS and VBBM are unnecessary. The primary purpose of this function + * is to free up memory. + * @return 0 on success, non-0 on error (see brmtypes.h). + */ + EXPORT int clear() DBRM_THROW; + + /** @brief Check the consistency of each data structure + * + * Check the consistency of each data structure + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int checkConsistency() throw(); + + /** @brief Get a list of the transactions currently in progress. + * + * Get a list of the transactions currently in progress. This scans + * the copy locks & VSS for LBIDs that are locked by some transaction + * and stores the transaction ID. + * + * @param txnList Caller-supplied set to store the results in. + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT int getCurrentTxnIDs(std::set& txnList) throw(); + + /** @brief Persistence API. Saves the local Extent Map to a file. + * + * Persistence API. Saves the local Extent Map to a file. + * + * @param filename Relative or absolute path to save to. + * @return 0 on success, -1 on error + */ + EXPORT int saveExtentMap(const std::string& filename) throw(); + + /** @brief Persistence API. Saves all BRM structures. + * + * Saves all local BRM structures to files. + * + * @param filename The filename prefix to use. Saves 4 files with that prefix. + * @return 0 on success, -1 on error + */ + EXPORT int saveState(std::string filename) throw(); + + /** @brief Persistence API. Saves all BRM structures using the filenames from Columnstore.xml. + * + * Saves all local BRM structures to files. + * + * @param filename The filename prefix to use. Saves 4 files with that prefix. + * @return 0 on success, -1 on error + */ + EXPORT int saveState() throw(); + + /** @brief A function that forces the BRM to write a snapshot of its data structures to disk + * + * A function that forces the BRM to write a snapshot of its data structures to disk. + * This happens automatically after commits and rollbacks. Bulk imports should call + * this fcn after an import. + * + * @return 0 on success, non-0 on error (see brmtypes.h). + */ + EXPORT int takeSnapshot() throw(); + + /* SessionManager interface */ + EXPORT const QueryContext verID(); + EXPORT const QueryContext sysCatVerID(); + EXPORT const TxnID newTxnID(const SessionManagerServer::SID session, bool block, bool isDDL = false); + EXPORT void committed(BRM::TxnID& txnid); + EXPORT void rolledback(BRM::TxnID& txnid); + EXPORT const BRM::TxnID getTxnID(const SessionManagerServer::SID session); + EXPORT boost::shared_array SIDTIDMap(int& len); + EXPORT void sessionmanager_reset(); + + /* Note, these pull #s from two separate sequences. That is, they both + return 0, then 1, 2, 3, etc. */ + EXPORT uint32_t getUnique32(); + EXPORT uint64_t getUnique64(); + + /* New table lock interface */ + /* returns a unique ID (> 0) for the lock on success, 0 on failure. + * Also, on failure, the ownerName, pid, and session ID parameters will be set + * to the owner of one of the overlapping locks. */ + EXPORT uint64_t getTableLock(const std::vector& pmList, uint32_t tableOID, std::string* ownerName, + uint32_t* ownerPID, int32_t* ownerSessionID, int32_t* ownerTxnID, + LockState state); + EXPORT bool releaseTableLock(uint64_t id); + EXPORT bool changeState(uint64_t id, LockState state); + EXPORT bool changeOwner(uint64_t id, const std::string& ownerName, uint32_t ownerPID, + int32_t ownerSessionID, int32_t ownerTxnID); + EXPORT bool checkOwner(uint64_t id); + EXPORT std::vector getAllTableLocks(); + EXPORT void releaseAllTableLocks(); + EXPORT bool getTableLockInfo(uint64_t id, TableLockInfo* out); + + /** Casual partitioning support **/ + EXPORT int markExtentInvalid(const LBID_t lbid, + execplan::CalpontSystemCatalog::ColDataType colDataType) DBRM_THROW; + EXPORT int markExtentsInvalid(const std::vector& lbids, + const std::vector& colDataTypes) + DBRM_THROW; + template + EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw(); + + EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, + const int32_t seqNum) DBRM_THROW; + + EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw(); + + /** @brief Updates the max and min casual partitioning info for the passed extents. + * + * @bug 1970. + * + * @param cpInfos vector of CPInfo objects. The firstLbid must be the first LBID in the extent. + * @return 0 on success, -1 on error + */ + EXPORT int setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW; + + /** @brief Merges max/min casual partitioning info for the specified + * extents, with the current CP info. + * + * @param cpInfos vector of CPInfo objects. The Lbids must be the + * starting LBID in the relevant extent. + * @return 0 on success, -1 on error + */ + EXPORT int mergeExtentsMaxMin(const CPInfoMergeList_t& cpInfos) DBRM_THROW; + + /* read-side interface for locking LBID ranges (used by PrimProc) */ + EXPORT void lockLBIDRange(LBID_t start, uint32_t count); + EXPORT void releaseLBIDRange(LBID_t start, uint32_t count); + + /* write-side interface for locking LBID ranges (used by DML) */ + EXPORT int dmlLockLBIDRanges(const std::vector& ranges, int txnID); + EXPORT int dmlReleaseLBIDRanges(const std::vector& ranges); + + /* OAM Interface */ + EXPORT int halt() DBRM_THROW; + EXPORT int resume() DBRM_THROW; + EXPORT int forceReload() DBRM_THROW; + EXPORT int setReadOnly(bool b) DBRM_THROW; + EXPORT int isReadWrite() throw(); + + EXPORT bool isEMEmpty() throw(); + + EXPORT std::vector getEMFreeListEntries() throw(); + + /** @brief Check if the system is ready for updates + * + * Check is the system has completed rollback processing and is + * ready for updates + */ + EXPORT int getSystemReady() throw(); + + /** @brief Check if the system is ready for queries + * + * Check is the system has completed rollback processing and is + * ready for updates + */ + EXPORT int getSystemQueryReady() throw(); + + /** @brief Check if the system is suspended + * + * Suspended is caused by user at calpont-console + * @return 0 - system is not suspended, > 0 - system is + * suspended, < 0 on error + */ + EXPORT int getSystemSuspended() throw(); + + /** @brief Check if system suspension is pending + * + * If the user at calpont-console asks for suspension, but ddl, + * dml or cpimport are running, then we can't suspend. If user + * says suspend when all work completed, then this flag is set + * to prevent new work from starting. + * @param bRollback (out) if true, system is in a rollback mode + * before suspension. + * @return 0 - system is not in a suspend pending state, > 0 - + * system is in a suspend pending state, < 0 on error + */ + EXPORT int getSystemSuspendPending(bool& bRollback) throw(); + + /** @brief Check if system shutdown is pending + * + * If the user at calpont-console asks for shutdown, but ddl, + * dml or cpimport are running, then we can't shutdown. If user + * says shutdown when all work completed, then this flag is set + * to prevent new work from starting. + * @param bRollback (out) if true, system is in rollback mode + * before shutdown. + * @param bForce (out) if true, system is in force shutdown + * mode. No work of any kind should be done. + * @return 0 - system is not pending a shutdown, > 0 - system is + * pending a shutdown, < 0 on error + */ + EXPORT int getSystemShutdownPending(bool& bRollback, bool& bForce) throw(); + + /** @brief Mark the system as ready for updates + * + * The system has completed rollback processing and is therefore ready for updates + * @param bReady the state the ready flag should be set to. + * @return < 0 on error + */ + EXPORT int setSystemReady(bool bReady) throw(); + + /** @brief Mark the system as ready for queries + * + * The system has completed bring up all modules and is ready for queries + * Queries are allowed during DMLRollback, so SS_READY won't fork for queries. + * @param bReady the state the ready flag should be set to. + * @return < 0 on error + */ + EXPORT int setSystemQueryReady(bool bReady) throw(); + + /** @brief Mark the system as suspended (or not) + * + * @param bSuspended the suspend state the system should be set + * to. + * @return < 0 on error + */ + EXPORT int setSystemSuspended(bool bSuspended) throw(); + + /** @brief Mark the system as suspension pending + * + * @param bPending the suspend pending state the system should + * be set to. + * @param bRollback if true, rollback all active transactions + * before full suspension of writes (only used + * if bPending is true). + * @return < 0 on error + */ + EXPORT int setSystemSuspendPending(bool bPending, bool bRollback = false) throw(); + + /** @brief Mark the system as shutdown pending + * + * @param bPending the suspend pending state the system should + * be set to. + * @param bRollback if true, rollback all active transactions + * before shutting down. (only used if bPending + * is true). + * @param bForce if true, we're shutting down now. No further + * work, including rollback, should be done. + * (only used if bPending is true). + * @return < 0 on error + */ + EXPORT int setSystemShutdownPending(bool bPending, bool bRollback = false, bool bForce = false) throw(); + + /** @brief get the flags in the system state bit map. + * @param stateFlags (out) + * @return < 0 on error + */ + int getSystemState(uint32_t& stateFlags) throw(); + + /** @brief set the flags in the system state bit map. + * + * sets the flags that are set in stateFlags leaving other flags undisturbed + * @param stateFlags (out) + * @return < 0 on error + */ + int setSystemState(uint32_t stateFlags) throw(); + + /** @brief set the flags in the system state bit map. + * + * Clears the flags that are set in stateFlags leaving other flags undisturbed + * @param stateFlags (out) + * @return < 0 on error + */ + int clearSystemState(uint32_t stateFlags) throw(); + + /** @brief Check to see if Controller Node is functioning + */ + bool isDBRMReady() throw(); + + /* OID Manager interface. See oidserver.h for the API documentation. */ + EXPORT int allocOIDs(int num); + EXPORT void returnOIDs(int start, int end); + EXPORT int oidm_size(); + EXPORT int allocVBOID(uint32_t dbroot); + EXPORT int getDBRootOfVBOID(uint32_t vbOID); + EXPORT std::vector getVBOIDToDBRootMap(); + + /* Autoincrement interface */ + EXPORT void startAISequence(uint32_t OID, uint64_t firstNum, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT bool getAIRange(uint32_t OID, uint32_t count, uint64_t* firstNum); + EXPORT bool getAIValue(uint32_t OID, uint64_t* value); + EXPORT void resetAISequence(uint32_t OID, uint64_t value); + EXPORT void getAILock(uint32_t OID); + EXPORT void releaseAILock(uint32_t OID); + + /* Added to support unsigned */ + /** @brief Invalidate the casual partitioning for all uncommited + * extents. + * + * Either txnid or plbidList can be passed . Only one will be + * used. + * @param txnid (in) - The transaction for which to get the lbid + * list. + * @param plbidList (in) - a list of lbids whose extents are to + * be invalidated. Only one lbid per extent should + * be in this list, such as returned in + * getUncommittedExtentLBIDs(). + * @return nothing. + */ + EXPORT void invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents, + std::vector* plbidList = NULL); + + private: + DBRM(const DBRM& brm); + DBRM& operator=(const DBRM& brm); + int8_t send_recv(const messageqcpp::ByteStream& in, messageqcpp::ByteStream& out) throw(); + + void deleteAISequence(uint32_t OID); // called as part of deleteOID & deleteOIDs + + boost::scoped_ptr mst; + boost::scoped_ptr em; + boost::scoped_ptr vbbm; + boost::scoped_ptr vss; + boost::scoped_ptr copylocks; + messageqcpp::MessageQueueClient* msgClient; + std::string masterName; + boost::mutex mutex; + config::Config* config; + bool fDebug; }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/dbrmctl.cpp b/versioning/BRM/dbrmctl.cpp index f3942bc49..68274c140 100644 --- a/versioning/BRM/dbrmctl.cpp +++ b/versioning/BRM/dbrmctl.cpp @@ -20,7 +20,6 @@ * ****************************************************************************/ - #include #include using namespace std; @@ -40,157 +39,143 @@ bool vflg; void usage(char* c) { - cerr << "Usage: " << c << " [-vh] status | halt | resume | readonly | readwrite | reload" << endl; - exit(1); + cerr << "Usage: " << c << " [-vh] status | halt | resume | readonly | readwrite | reload" << endl; + exit(1); } void errMsg(int err) { - switch (err) - { - case ERR_OK: - cout << "OK."; + switch (err) + { + case ERR_OK: + cout << "OK."; - if (vflg) - { - if (dbrm.getSystemReady() > 0) - cout << " (and the system is ready)"; - else - cout << " (but the system is not ready)"; - } + if (vflg) + { + if (dbrm.getSystemReady() > 0) + cout << " (and the system is ready)"; + else + cout << " (but the system is not ready)"; + } - cout << endl; - break; + cout << endl; + break; - case ERR_NETWORK: - cout << "Failure: an unspecific communication error." << endl; - break; + case ERR_NETWORK: cout << "Failure: an unspecific communication error." << endl; break; - case ERR_TIMEOUT: - cout << "Failure: controller node timed out." << endl; - break; + case ERR_TIMEOUT: cout << "Failure: controller node timed out." << endl; break; - case ERR_READONLY: - cout << "DBRM is currently Read Only!" << endl; - break; + case ERR_READONLY: cout << "DBRM is currently Read Only!" << endl; break; - case 20: - cout << "System is ready" << endl; - break; + case 20: cout << "System is ready" << endl; break; - case 21: - cout << "System is not ready" << endl; - break; + case 21: cout << "System is not ready" << endl; break; - default: - cout << "Failure: an unexpected error (" << err << ")" << endl; - break; - } + default: cout << "Failure: an unexpected error (" << err << ")" << endl; break; + } } void do_halt() { - int err; + int err; - err = dbrm.halt(); - errMsg(err); + err = dbrm.halt(); + errMsg(err); } void do_resume() { - int err; + int err; - err = dbrm.resume(); - errMsg(err); + err = dbrm.resume(); + errMsg(err); } void do_reload() { - int err; + int err; - err = dbrm.forceReload(); - errMsg(err); + err = dbrm.forceReload(); + errMsg(err); } void set_readonly(bool b) { - int err; + int err; - err = dbrm.setReadOnly(b); - errMsg(err); + err = dbrm.setReadOnly(b); + errMsg(err); } void do_status() { - int err; + int err; - err = dbrm.isReadWrite(); - errMsg(err); + err = dbrm.isReadWrite(); + errMsg(err); } void do_sysstatus() { - int err; + int err; - bool ready = dbrm.getSystemReady() > 0 ? true : false; + bool ready = dbrm.getSystemReady() > 0 ? true : false; - if (ready) - err = 20; - else - err = 21; + if (ready) + err = 20; + else + err = 21; - errMsg(err); + errMsg(err); } -} +} // namespace int main(int argc, char** argv) { - int c; - vflg = false; + int c; + vflg = false; - opterr = 0; + opterr = 0; - while ((c = getopt(argc, argv, "vh")) != EOF) - switch (c) - { - case 'v': - vflg = true; - break; + while ((c = getopt(argc, argv, "vh")) != EOF) + switch (c) + { + case 'v': vflg = true; break; - case 'h': - case '?': - default: - usage(argv[0]); - return (c == 'h' ? 0 : 1); - break; - } - - string cmd; - - if ((argc - optind) < 1) + case 'h': + case '?': + default: usage(argv[0]); + return (c == 'h' ? 0 : 1); + break; + } - idbdatafile::IDBPolicy::configIDBPolicy(); + string cmd; - cmd = argv[optind++]; + if ((argc - optind) < 1) + usage(argv[0]); - if (cmd == "status") - do_status(); - else if (cmd == "halt") - do_halt(); - else if (cmd == "resume") - do_resume(); - else if (cmd == "readonly") - set_readonly(true); - else if (cmd == "readwrite") - set_readonly(false); - else if (cmd == "reload") - do_reload(); - else if (cmd == "sysstatus") - do_sysstatus(); - else - usage(argv[0]); + idbdatafile::IDBPolicy::configIDBPolicy(); - return 0; + cmd = argv[optind++]; + + if (cmd == "status") + do_status(); + else if (cmd == "halt") + do_halt(); + else if (cmd == "resume") + do_resume(); + else if (cmd == "readonly") + set_readonly(true); + else if (cmd == "readwrite") + set_readonly(false); + else if (cmd == "reload") + do_reload(); + else if (cmd == "sysstatus") + do_sysstatus(); + else + usage(argv[0]); + + return 0; } diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 136373bbc..6235f3017 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -69,7 +69,7 @@ namespace bi = boost::interprocess; #include "extentmap.h" #undef EXTENTMAP_DLLEXPORT -#define EM_MAX_SEQNUM 2000000000 +#define EM_MAX_SEQNUM 2000000000 #define MAX_IO_RETRIES 10 #define EM_MAGIC_V1 0x76f78b1c #define EM_MAGIC_V2 0x76f78b1d @@ -78,11 +78,12 @@ namespace bi = boost::interprocess; #define EM_MAGIC_V5 0x76f78b20 #ifndef NDEBUG -#define ASSERT(x) \ - if (!(x)) { \ - cerr << "assertion at file " << __FILE__ << " line " << __LINE__ << " failed" << endl; \ - throw logic_error("assertion failed"); \ - } +#define ASSERT(x) \ + if (!(x)) \ + { \ + cerr << "assertion at file " << __FILE__ << " line " << __LINE__ << " failed" << endl; \ + throw logic_error("assertion failed"); \ + } #else #define ASSERT(x) #endif @@ -94,68 +95,68 @@ using namespace idbdatafile; namespace { -unsigned ExtentSize = 0; // dmc-need to deprecate -unsigned ExtentRows = 0; +unsigned ExtentSize = 0; // dmc-need to deprecate +unsigned ExtentRows = 0; unsigned filesPerColumnPartition = 0; unsigned extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE; // Increment CP sequence (version) number, and wrap-around when applicable inline void incSeqNum(int32_t& seqNum) { - seqNum++; + seqNum++; - if (seqNum > EM_MAX_SEQNUM) - seqNum = 0; + if (seqNum > EM_MAX_SEQNUM) + seqNum = 0; } -} +} // namespace namespace BRM { - //------------------------------------------------------------------------------ // EMCasualPartition_struct methods //------------------------------------------------------------------------------ EMCasualPartition_struct::EMCasualPartition_struct() { - utils::int128Max(bigLoVal); - utils::int128Min(bigHiVal); - sequenceNum = 0; - isValid = CP_INVALID; + utils::int128Max(bigLoVal); + utils::int128Min(bigHiVal); + sequenceNum = 0; + isValid = CP_INVALID; } EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum) { - loVal = lo; - hiVal = hi; - sequenceNum = seqNum; - isValid = CP_INVALID; + loVal = lo; + hiVal = hi; + sequenceNum = seqNum; + isValid = CP_INVALID; } -EMCasualPartition_struct::EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum) +EMCasualPartition_struct::EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, + const int32_t seqNum) { - bigLoVal = bigLo; - bigHiVal = bigHi; - sequenceNum = seqNum; - isValid = CP_INVALID; + bigLoVal = bigLo; + bigHiVal = bigHi; + sequenceNum = seqNum; + isValid = CP_INVALID; } EMCasualPartition_struct::EMCasualPartition_struct(const EMCasualPartition_struct& em) { - bigLoVal = em.bigLoVal; - bigHiVal = em.bigHiVal; - sequenceNum = em.sequenceNum; - isValid = em.isValid; + bigLoVal = em.bigLoVal; + bigHiVal = em.bigHiVal; + sequenceNum = em.sequenceNum; + isValid = em.isValid; } -EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPartition_struct& em) +EMCasualPartition_struct& EMCasualPartition_struct::operator=(const EMCasualPartition_struct& em) { - bigLoVal = em.bigLoVal; - bigHiVal = em.bigHiVal; - sequenceNum = em.sequenceNum; - isValid = em.isValid; - return *this; + bigLoVal = em.bigLoVal; + bigHiVal = em.bigHiVal; + sequenceNum = em.sequenceNum; + isValid = em.isValid; + return *this; } //------------------------------------------------------------------------------ @@ -164,53 +165,53 @@ EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPar EMEntry::EMEntry() { - fileID = 0; - blockOffset = 0; - HWM = 0; - partitionNum = 0; - segmentNum = 0; - dbRoot = 0; - colWid = 0; - status = 0; + fileID = 0; + blockOffset = 0; + HWM = 0; + partitionNum = 0; + segmentNum = 0; + dbRoot = 0; + colWid = 0; + status = 0; } EMEntry::EMEntry(const EMEntry& e) { - range.start = e.range.start; - range.size = e.range.size; - fileID = e.fileID; - blockOffset = e.blockOffset; - HWM = e.HWM; - partition = e.partition; - partitionNum = e.partitionNum; - segmentNum = e.segmentNum; - dbRoot = e.dbRoot; - colWid = e.colWid; - status = e.status; + range.start = e.range.start; + range.size = e.range.size; + fileID = e.fileID; + blockOffset = e.blockOffset; + HWM = e.HWM; + partition = e.partition; + partitionNum = e.partitionNum; + segmentNum = e.segmentNum; + dbRoot = e.dbRoot; + colWid = e.colWid; + status = e.status; } -EMEntry& EMEntry::operator= (const EMEntry& e) +EMEntry& EMEntry::operator=(const EMEntry& e) { - range.start = e.range.start; - range.size = e.range.size; - fileID = e.fileID; - blockOffset = e.blockOffset; - HWM = e.HWM; - partition = e.partition; - partitionNum = e.partitionNum; - segmentNum = e.segmentNum; - colWid = e.colWid; - dbRoot = e.dbRoot; - status = e.status; - return *this; + range.start = e.range.start; + range.size = e.range.size; + fileID = e.fileID; + blockOffset = e.blockOffset; + HWM = e.HWM; + partition = e.partition; + partitionNum = e.partitionNum; + segmentNum = e.segmentNum; + colWid = e.colWid; + dbRoot = e.dbRoot; + status = e.status; + return *this; } -bool EMEntry::operator< (const EMEntry& e) const +bool EMEntry::operator<(const EMEntry& e) const { - if (range.start < e.range.start) - return true; + if (range.start < e.range.start) + return true; - return false; + return false; } /*static*/ @@ -223,27 +224,26 @@ ExtentMapImpl* ExtentMapImpl::fInstance = 0; /*static*/ ExtentMapImpl* ExtentMapImpl::makeExtentMapImpl(unsigned key, off_t size, bool readOnly) { - boost::mutex::scoped_lock lk(fInstanceMutex); + boost::mutex::scoped_lock lk(fInstanceMutex); - if (fInstance) + if (fInstance) + { + if (key != fInstance->fExtMap.key()) { - if (key != fInstance->fExtMap.key()) - { - BRMShmImpl newShm(key, 0); - fInstance->swapout(newShm); - } - - ASSERT(key == fInstance->fExtMap.key()); - return fInstance; + BRMShmImpl newShm(key, 0); + fInstance->swapout(newShm); } - fInstance = new ExtentMapImpl(key, size, readOnly); - + ASSERT(key == fInstance->fExtMap.key()); return fInstance; + } + + fInstance = new ExtentMapImpl(key, size, readOnly); + + return fInstance; } -ExtentMapImpl::ExtentMapImpl(unsigned key, off_t size, bool readOnly) : - fExtMap(key, size, readOnly) +ExtentMapImpl::ExtentMapImpl(unsigned key, off_t size, bool readOnly) : fExtMap(key, size, readOnly) { } @@ -256,224 +256,220 @@ FreeListImpl* FreeListImpl::fInstance = 0; /*static*/ FreeListImpl* FreeListImpl::makeFreeListImpl(unsigned key, off_t size, bool readOnly) { - boost::mutex::scoped_lock lk(fInstanceMutex); + boost::mutex::scoped_lock lk(fInstanceMutex); - if (fInstance) + if (fInstance) + { + if (key != fInstance->fFreeList.key()) { - if (key != fInstance->fFreeList.key()) - { - BRMShmImpl newShm(key, 0); - fInstance->swapout(newShm); - } - - ASSERT(key == fInstance->fFreeList.key()); - return fInstance; + BRMShmImpl newShm(key, 0); + fInstance->swapout(newShm); } - fInstance = new FreeListImpl(key, size, readOnly); - + ASSERT(key == fInstance->fFreeList.key()); return fInstance; + } + + fInstance = new FreeListImpl(key, size, readOnly); + + return fInstance; } -FreeListImpl::FreeListImpl(unsigned key, off_t size, bool readOnly) : - fFreeList(key, size, readOnly) +FreeListImpl::FreeListImpl(unsigned key, off_t size, bool readOnly) : fFreeList(key, size, readOnly) { } ExtentMap::ExtentMap() { - fExtentMap = NULL; - fFreeList = NULL; - fCurrentEMShmkey = -1; - fCurrentFLShmkey = -1; - fEMShminfo = NULL; - fFLShminfo = NULL; - r_only = false; - flLocked = false; - emLocked = false; - fPExtMapImpl = 0; - fPFreeListImpl = 0; + fExtentMap = NULL; + fFreeList = NULL; + fCurrentEMShmkey = -1; + fCurrentFLShmkey = -1; + fEMShminfo = NULL; + fFLShminfo = NULL; + r_only = false; + flLocked = false; + emLocked = false; + fPExtMapImpl = 0; + fPFreeListImpl = 0; #ifdef BRM_INFO - fDebug = ("Y" == config::Config::makeConfig()->getConfig("DBRM", "Debug")); + fDebug = ("Y" == config::Config::makeConfig()->getConfig("DBRM", "Debug")); #endif } ExtentMap::~ExtentMap() { - PmDbRootMap_t::iterator iter = fPmDbRootMap.begin(); - PmDbRootMap_t::iterator end = fPmDbRootMap.end(); + PmDbRootMap_t::iterator iter = fPmDbRootMap.begin(); + PmDbRootMap_t::iterator end = fPmDbRootMap.end(); - while (iter != end) - { - delete iter->second; - iter->second = 0; - ++iter; - } + while (iter != end) + { + delete iter->second; + iter->second = 0; + ++iter; + } - fPmDbRootMap.clear(); + fPmDbRootMap.clear(); } // Casual Partioning support // /** -* @brief mark the max/min values of an extent as invalid -* -* mark the extent containing the lbid as invalid and -* increment the sequenceNum value. If the lbid is found -* in the extent map a 0 is returned otherwise a 1. -* -**/ + * @brief mark the max/min values of an extent as invalid + * + * mark the extent containing the lbid as invalid and + * increment the sequenceNum value. If the lbid is found + * in the extent map a 0 is returned otherwise a 1. + * + **/ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType) { - int entries; - int i; - LBID_t lastBlock; + int entries; + int i; + LBID_t lastBlock; - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (i = 0; i < entries; i++) + for (i = 0; i < entries; i++) + { + lastBlock = fExtentMap[i].range.start + (static_cast(fExtentMap[i].range.size) * 1024) - 1; + + if (fExtentMap[i].range.size != 0) { - lastBlock = fExtentMap[i].range.start + - (static_cast(fExtentMap[i].range.size) * 1024) - 1; + if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + fExtentMap[i].partition.cprange.isValid = CP_UPDATING; - if (fExtentMap[i].range.size != 0) + if (isUnsigned(colDataType)) { - if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.isValid = CP_UPDATING; - - if (isUnsigned(colDataType)) - { - if (fExtentMap[i].colWid != datatypes::MAXDECIMALWIDTH) - { - fExtentMap[i].partition.cprange.loVal = numeric_limits::max(); - fExtentMap[i].partition.cprange.hiVal = numeric_limits::min(); - } - else - { - fExtentMap[i].partition.cprange.bigLoVal = -1; // XXX: unsigned wide decimals do not exceed rang of signed wide decimals. - fExtentMap[i].partition.cprange.bigHiVal = 0; - } - } - else - { - if (fExtentMap[i].colWid != datatypes::MAXDECIMALWIDTH) - { - fExtentMap[i].partition.cprange.loVal = numeric_limits::max(); - fExtentMap[i].partition.cprange.hiVal = numeric_limits::min(); - } - else - { - utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); - utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); - } - } - - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); -#ifdef BRM_DEBUG - ostringstream os; - os << "ExtentMap::_markInvalid(): casual partitioning update: firstLBID=" << - fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + - fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " min=" << fExtentMap[i].partition.cprange.loVal << - " max=" << fExtentMap[i].partition.cprange.hiVal << - "seq=" << fExtentMap[i].partition.cprange.sequenceNum; - log(os.str(), logging::LOG_TYPE_DEBUG); -#endif - return 0; - } + if (fExtentMap[i].colWid != datatypes::MAXDECIMALWIDTH) + { + fExtentMap[i].partition.cprange.loVal = numeric_limits::max(); + fExtentMap[i].partition.cprange.hiVal = numeric_limits::min(); + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = + -1; // XXX: unsigned wide decimals do not exceed rang of signed wide decimals. + fExtentMap[i].partition.cprange.bigHiVal = 0; + } + } + else + { + if (fExtentMap[i].colWid != datatypes::MAXDECIMALWIDTH) + { + fExtentMap[i].partition.cprange.loVal = numeric_limits::max(); + fExtentMap[i].partition.cprange.hiVal = numeric_limits::min(); + } + else + { + utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + } } - } - throw logic_error("ExtentMap::markInvalid(): lbid isn't allocated"); + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); +#ifdef BRM_DEBUG + ostringstream os; + os << "ExtentMap::_markInvalid(): casual partitioning update: firstLBID=" << fExtentMap[i].range.start + << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 + << " OID=" << fExtentMap[i].fileID << " min=" << fExtentMap[i].partition.cprange.loVal + << " max=" << fExtentMap[i].partition.cprange.hiVal + << "seq=" << fExtentMap[i].partition.cprange.sequenceNum; + log(os.str(), logging::LOG_TYPE_DEBUG); +#endif + return 0; + } + } + } + + throw logic_error("ExtentMap::markInvalid(): lbid isn't allocated"); } -int ExtentMap::markInvalid(const LBID_t lbid, - const execplan::CalpontSystemCatalog::ColDataType colDataType) +int ExtentMap::markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType) { #ifdef BRM_DEBUG - if (lbid < 0) - throw invalid_argument("ExtentMap::markInvalid(): lbid must be >= 0"); + if (lbid < 0) + throw invalid_argument("ExtentMap::markInvalid(): lbid must be >= 0"); #endif #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("_markInvalid"); - TRACER_ADDINPUT(lbid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("_markInvalid"); + TRACER_ADDINPUT(lbid); + TRACER_WRITE; + } #endif #ifdef BRM_DEBUG - ostringstream os; - os << "ExtentMap::markInvalid(" << lbid << "," << colDataType << ")"; - log(os.str(), logging::LOG_TYPE_DEBUG); + ostringstream os; + os << "ExtentMap::markInvalid(" << lbid << "," << colDataType << ")"; + log(os.str(), logging::LOG_TYPE_DEBUG); #endif - grabEMEntryTable(WRITE); - return _markInvalid(lbid, colDataType); + grabEMEntryTable(WRITE); + return _markInvalid(lbid, colDataType); } /** -* @brief calls markInvalid(LBID_t lbid) for each extent containing any lbid in vector& lbids -* -**/ + * @brief calls markInvalid(LBID_t lbid) for each extent containing any lbid in vector& lbids + * + **/ int ExtentMap::markInvalid(const vector& lbids, const vector& colDataTypes) { - uint32_t i, size = lbids.size(); + uint32_t i, size = lbids.size(); #ifdef BRM_DEBUG - for (i = 0; i < size; ++i) - if (lbids[i] < 0) - throw invalid_argument("ExtentMap::markInvalid(vector): all lbids must be >= 0"); + for (i = 0; i < size; ++i) + if (lbids[i] < 0) + throw invalid_argument("ExtentMap::markInvalid(vector): all lbids must be >= 0"); #endif #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("_markInvalid"); - TRACER_ADDINPUT(size); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("_markInvalid"); + TRACER_ADDINPUT(size); + TRACER_WRITE; + } #endif - grabEMEntryTable(WRITE); + grabEMEntryTable(WRITE); - // XXXPAT: what's the proper return code when one and only one fails? - for (i = 0; i < size; ++i) - { + // XXXPAT: what's the proper return code when one and only one fails? + for (i = 0; i < size; ++i) + { #ifdef BRM_DEBUG - ostringstream os; - os << "ExtentMap::markInvalid() lbids[" << i << "]=" << lbids[i] << - " colDataTypes[" << i << "]=" << colDataTypes[i]; - log(os.str(), logging::LOG_TYPE_DEBUG); + ostringstream os; + os << "ExtentMap::markInvalid() lbids[" << i << "]=" << lbids[i] << " colDataTypes[" << i + << "]=" << colDataTypes[i]; + log(os.str(), logging::LOG_TYPE_DEBUG); #endif - try - { - _markInvalid(lbids[i], colDataTypes[i]); - } - catch (std::exception& e) - { - cerr << "ExtentMap::markInvalid(vector): warning! lbid " << lbids[i] << - " caused " << e.what() << endl; - } + try + { + _markInvalid(lbids[i], colDataTypes[i]); } + catch (std::exception& e) + { + cerr << "ExtentMap::markInvalid(vector): warning! lbid " << lbids[i] << " caused " << e.what() << endl; + } + } - return 0; + return 0; } /** @@ -489,283 +485,274 @@ int ExtentMap::markInvalid(const vector& lbids, // TODO MCOL-641 Not adding support here since this function appears to be unused anywhere. -int ExtentMap::setMaxMin(const LBID_t lbid, - const int64_t max, - const int64_t min, - const int32_t seqNum, +int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum, bool firstNode) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("updateMaxMin"); - TRACER_ADDINPUT(lbid); - TRACER_ADDINPUT(max); - TRACER_ADDINPUT(min); - TRACER_ADDINPUT(seqNum); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("updateMaxMin"); + TRACER_ADDINPUT(lbid); + TRACER_ADDINPUT(max); + TRACER_ADDINPUT(min); + TRACER_ADDINPUT(seqNum); + TRACER_WRITE; + } #endif - int entries; - int i; - LBID_t lastBlock; - int32_t curSequence; + int entries; + int i; + LBID_t lastBlock; + int32_t curSequence; #ifdef BRM_DEBUG - if (lbid < 0) - throw invalid_argument("ExtentMap::setMaxMin(): lbid must be >= 0"); + if (lbid < 0) + throw invalid_argument("ExtentMap::setMaxMin(): lbid must be >= 0"); #endif - grabEMEntryTable(WRITE); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(WRITE); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (i = 0; i < entries; i++) + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) + lastBlock = fExtentMap[i].range.start + (static_cast(fExtentMap[i].range.size) * 1024) - 1; + curSequence = fExtentMap[i].partition.cprange.sequenceNum; + + if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) + { +#ifdef BRM_DEBUG + + if (firstNode) { - lastBlock = fExtentMap[i].range.start + - (static_cast(fExtentMap[i].range.size) * 1024) - 1; - curSequence = fExtentMap[i].partition.cprange.sequenceNum; - - if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) - { -#ifdef BRM_DEBUG - - if (firstNode) - { - ostringstream os; - os << "ExtentMap::setMaxMin(): casual partitioning update: firstLBID=" << - fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + - fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " min=" << min << " max=" << max << "seq=" << seqNum; - log(os.str(), logging::LOG_TYPE_DEBUG); - } + ostringstream os; + os << "ExtentMap::setMaxMin(): casual partitioning update: firstLBID=" << fExtentMap[i].range.start + << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 + << " OID=" << fExtentMap[i].fileID << " min=" << min << " max=" << max << "seq=" << seqNum; + log(os.str(), logging::LOG_TYPE_DEBUG); + } #endif - if (curSequence == seqNum) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hiVal = max; - fExtentMap[i].partition.cprange.loVal = min; - fExtentMap[i].partition.cprange.isValid = CP_VALID; - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - return 0; - } - //special val to indicate a reset--used by editem -c. - //Also used by COMMIT and ROLLBACK to invalidate CP. - else if (seqNum == SEQNUM_MARK_INVALID) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - // We set hiVal and loVal to correct values for signed or unsigned - // during the markinvalid step, which sets the invalid variable to CP_UPDATING. - // During this step (seqNum == SEQNUM_MARK_INVALID), the min and max passed in are not reliable - // and should not be used. - fExtentMap[i].partition.cprange.isValid = CP_INVALID; - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - return 0; - } - else - { - return 0; - } - } + if (curSequence == seqNum) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + fExtentMap[i].partition.cprange.hiVal = max; + fExtentMap[i].partition.cprange.loVal = min; + fExtentMap[i].partition.cprange.isValid = CP_VALID; + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + return 0; } + // special val to indicate a reset--used by editem -c. + // Also used by COMMIT and ROLLBACK to invalidate CP. + else if (seqNum == SEQNUM_MARK_INVALID) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + // We set hiVal and loVal to correct values for signed or unsigned + // during the markinvalid step, which sets the invalid variable to CP_UPDATING. + // During this step (seqNum == SEQNUM_MARK_INVALID), the min and max passed in are not reliable + // and should not be used. + fExtentMap[i].partition.cprange.isValid = CP_INVALID; + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + return 0; + } + else + { + return 0; + } + } } + } - if (emLocked) - releaseEMEntryTable(WRITE); + if (emLocked) + releaseEMEntryTable(WRITE); - throw logic_error("ExtentMap::setMaxMin(): lbid isn't allocated"); -// return -1; + throw logic_error("ExtentMap::setMaxMin(): lbid isn't allocated"); + // return -1; } // @bug 1970. Added updateExtentsMaxMin function. // @note - The key passed in the map must the the first LBID in the extent. void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, bool useLock) { - CPMaxMinMap_t::const_iterator it; + CPMaxMinMap_t::const_iterator it; #ifdef BRM_DEBUG - log("ExtentMap::setExtentsMaxMin()", logging::LOG_TYPE_DEBUG); + log("ExtentMap::setExtentsMaxMin()", logging::LOG_TYPE_DEBUG); - for (it = cpMap.begin(); it != cpMap.end(); ++it) - { - ostringstream os; - os << "FirstLBID=" << it->first << - " min=" << it->second.min << - " max=" << it->second.max << - " seq=" << it->second.seqNum; - log(os.str(), logging::LOG_TYPE_DEBUG); - } + for (it = cpMap.begin(); it != cpMap.end(); ++it) + { + ostringstream os; + os << "FirstLBID=" << it->first << " min=" << it->second.min << " max=" << it->second.max + << " seq=" << it->second.seqNum; + log(os.str(), logging::LOG_TYPE_DEBUG); + } #endif #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setExtentsMaxMin"); + if (fDebug) + { + TRACER_WRITELATER("setExtentsMaxMin"); - for (it = cpMap.begin(); it != cpMap.end(); ++it) - { - TRACER_ADDINPUT((*it).first); - TRACER_ADDINPUT((*it).second.max); - TRACER_ADDINPUT((*it).second.min); - TRACER_ADDINPUT((*it).second.seqNum); - TRACER_WRITE; - } + for (it = cpMap.begin(); it != cpMap.end(); ++it) + { + TRACER_ADDINPUT((*it).first); + TRACER_ADDINPUT((*it).second.max); + TRACER_ADDINPUT((*it).second.min); + TRACER_ADDINPUT((*it).second.seqNum); + TRACER_WRITE; } + } #endif - int entries; - int i; - int32_t curSequence; - const int32_t extentsToUpdate = cpMap.size(); - int32_t extentsUpdated = 0; + int entries; + int i; + int32_t curSequence; + const int32_t extentsToUpdate = cpMap.size(); + int32_t extentsUpdated = 0; #ifdef BRM_DEBUG - if (extentsToUpdate <= 0) - throw invalid_argument("ExtentMap::setExtentsMaxMin(): cpMap must be populated"); + if (extentsToUpdate <= 0) + throw invalid_argument("ExtentMap::setExtentsMaxMin(): cpMap must be populated"); #endif - if (useLock) - grabEMEntryTable(WRITE); + if (useLock) + grabEMEntryTable(WRITE); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0) + { + it = cpMap.find(fExtentMap[i].range.start); + + if (it != cpMap.end()) + { + curSequence = fExtentMap[i].partition.cprange.sequenceNum; + + if (curSequence == it->second.seqNum && fExtentMap[i].partition.cprange.isValid == CP_INVALID) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; + } + fExtentMap[i].partition.cprange.isValid = CP_VALID; + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + extentsUpdated++; +#ifdef BRM_DEBUG + + if (firstNode) + { + ostringstream os; + os << "ExtentMap::setExtentsMaxMin(): casual partitioning update: firstLBID=" + << fExtentMap[i].range.start + << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 + << " OID=" << fExtentMap[i].fileID << " min=" << it->second.min << " max=" << it->second.max + << " seq=" << it->second.seqNum; + log(os.str(), logging::LOG_TYPE_DEBUG); + } + +#endif + } + // special val to indicate a reset -- ignore the min/max + else if (it->second.seqNum == SEQNUM_MARK_INVALID) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + // We set hiVal and loVal to correct values for signed or unsigned + // during the markinvalid step, which sets the invalid variable to CP_UPDATING. + // During this step (seqNum == SEQNUM_MARK_INVALID), the min and max passed in are not reliable + // and should not be used. + fExtentMap[i].partition.cprange.isValid = CP_INVALID; + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + extentsUpdated++; + } + // special val to indicate a reset -- assign the min/max + else if (it->second.seqNum == SEQNUM_MARK_INVALID_SET_RANGE) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; + } + fExtentMap[i].partition.cprange.isValid = CP_INVALID; + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + extentsUpdated++; + } + else if (it->second.seqNum == SEQNUM_MARK_UPDATING_INVALID_SET_RANGE) + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + if (fExtentMap[i].partition.cprange.isValid == CP_UPDATING) + { + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; + } + fExtentMap[i].partition.cprange.isValid = CP_INVALID; + } + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + extentsUpdated++; + } + // else sequence has changed since start of the query. Don't update the EM entry. + else + { + extentsUpdated++; + } + + if (extentsUpdated == extentsToUpdate) + { + return; + } + } + } + } + + ostringstream oss; + oss << "ExtentMap::setExtentsMaxMin(): LBIDs not allocated:"; + for (it = cpMap.begin(); it != cpMap.end(); it++) + { for (i = 0; i < entries; i++) { - if (fExtentMap[i].range.size != 0) - { - it = cpMap.find(fExtentMap[i].range.start); - - if (it != cpMap.end()) - { - curSequence = fExtentMap[i].partition.cprange.sequenceNum; - - if (curSequence == it->second.seqNum && - fExtentMap[i].partition.cprange.isValid == CP_INVALID) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - if (it->second.isBinaryColumn) - { - fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; - fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; - } - else - { - fExtentMap[i].partition.cprange.hiVal = it->second.max; - fExtentMap[i].partition.cprange.loVal = it->second.min; - } - fExtentMap[i].partition.cprange.isValid = CP_VALID; - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - extentsUpdated++; -#ifdef BRM_DEBUG - - if (firstNode) - { - ostringstream os; - os << "ExtentMap::setExtentsMaxMin(): casual partitioning update: firstLBID=" << - fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + - fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " min=" << it->second.min << " max=" << - it->second.max << " seq=" << - it->second.seqNum; - log(os.str(), logging::LOG_TYPE_DEBUG); - } - -#endif - } - //special val to indicate a reset -- ignore the min/max - else if (it->second.seqNum == SEQNUM_MARK_INVALID) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - // We set hiVal and loVal to correct values for signed or unsigned - // during the markinvalid step, which sets the invalid variable to CP_UPDATING. - // During this step (seqNum == SEQNUM_MARK_INVALID), the min and max passed in are not reliable - // and should not be used. - fExtentMap[i].partition.cprange.isValid = CP_INVALID; - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - extentsUpdated++; - } - //special val to indicate a reset -- assign the min/max - else if (it->second.seqNum == SEQNUM_MARK_INVALID_SET_RANGE) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - if (it->second.isBinaryColumn) - { - fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; - fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; - } - else - { - fExtentMap[i].partition.cprange.hiVal = it->second.max; - fExtentMap[i].partition.cprange.loVal = it->second.min; - } - fExtentMap[i].partition.cprange.isValid = CP_INVALID; - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - extentsUpdated++; - } - else if (it->second.seqNum == SEQNUM_MARK_UPDATING_INVALID_SET_RANGE) - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - if (fExtentMap[i].partition.cprange.isValid == CP_UPDATING) - { - if (it->second.isBinaryColumn) - { - fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; - fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; - } - else - { - fExtentMap[i].partition.cprange.hiVal = it->second.max; - fExtentMap[i].partition.cprange.loVal = it->second.min; - } - fExtentMap[i].partition.cprange.isValid = CP_INVALID; - } - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - extentsUpdated++; - } - // else sequence has changed since start of the query. Don't update the EM entry. - else - { - extentsUpdated++; - } - - if (extentsUpdated == extentsToUpdate) - { - return; - } - } - } + if (fExtentMap[i].range.start == it->first) + { + break; + } } - - ostringstream oss; - oss << "ExtentMap::setExtentsMaxMin(): LBIDs not allocated:"; - for (it = cpMap.begin(); it != cpMap.end(); it ++) + if (i < entries) { - for (i = 0; i < entries; i++) - { - if (fExtentMap[i].range.start == it->first) - { - break; - } - } - if (i < entries) - { - continue; - } - oss << " " << it->first; + continue; } + oss << " " << it->first; + } - throw logic_error(oss.str()); + throw logic_error(oss.str()); } //------------------------------------------------------------------------------ @@ -787,286 +774,252 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo //------------------------------------------------------------------------------ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) { - CPMaxMinMergeMap_t::const_iterator it; + CPMaxMinMergeMap_t::const_iterator it; - // TODO MCOL-641 Add support in the debugging outputs here. + // TODO MCOL-641 Add support in the debugging outputs here. #ifdef BRM_DEBUG - log("ExtentMap::mergeExtentsMaxMin()", logging::LOG_TYPE_DEBUG); + log("ExtentMap::mergeExtentsMaxMin()", logging::LOG_TYPE_DEBUG); - for (it = cpMap.begin(); it != cpMap.end(); ++it) - { - ostringstream os; - os << "FirstLBID=" << it->first << - " min=" << it->second.min << - " max=" << it->second.max << - " seq=" << it->second.seqNum << - " typ: " << (*it).second.type << - " new: " << (*it).second.newExtent; - log(os.str(), logging::LOG_TYPE_DEBUG); - } + for (it = cpMap.begin(); it != cpMap.end(); ++it) + { + ostringstream os; + os << "FirstLBID=" << it->first << " min=" << it->second.min << " max=" << it->second.max + << " seq=" << it->second.seqNum << " typ: " << (*it).second.type << " new: " << (*it).second.newExtent; + log(os.str(), logging::LOG_TYPE_DEBUG); + } #endif #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITENOW("mergeExtentsMaxMin"); - unsigned int count = 1; + if (fDebug) + { + TRACER_WRITENOW("mergeExtentsMaxMin"); + unsigned int count = 1; - for (it = cpMap.begin(); it != cpMap.end(); ++it) - { - ostringstream oss; - oss << " " << count << - ". LBID: " << (*it).first << - "; max: " << (*it).second.max << - "; min: " << (*it).second.min << - "; seq: " << (*it).second.seqNum << - "; typ: " << (*it).second.type << - "; new: " << (*it).second.newExtent; - TRACER_WRITEDIRECT(oss.str()); - count++; - } + for (it = cpMap.begin(); it != cpMap.end(); ++it) + { + ostringstream oss; + oss << " " << count << ". LBID: " << (*it).first << "; max: " << (*it).second.max + << "; min: " << (*it).second.min << "; seq: " << (*it).second.seqNum + << "; typ: " << (*it).second.type << "; new: " << (*it).second.newExtent; + TRACER_WRITEDIRECT(oss.str()); + count++; } + } #endif - const int32_t extentsToMerge = cpMap.size(); - int32_t extentsMerged = 0; + const int32_t extentsToMerge = cpMap.size(); + int32_t extentsMerged = 0; #ifdef BRM_DEBUG - if (extentsToMerge <= 0) - throw invalid_argument("ExtentMap::mergeExtentsMaxMin(): " - "cpMap must be populated"); + if (extentsToMerge <= 0) + throw invalid_argument( + "ExtentMap::mergeExtentsMaxMin(): " + "cpMap must be populated"); #endif - if (useLock) - grabEMEntryTable(WRITE); + if (useLock) + grabEMEntryTable(WRITE); - int entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + int entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0; i < entries; i++) // loop through all extents + for (int i = 0; i < entries; i++) // loop through all extents + { + if (fExtentMap[i].range.size != 0) // find eligible extents { - if (fExtentMap[i].range.size != 0) // find eligible extents - { - it = cpMap.find(fExtentMap[i].range.start); + it = cpMap.find(fExtentMap[i].range.start); - if (it != cpMap.end()) - { + if (it != cpMap.end()) + { #ifdef BRM_DEBUG - ostringstream os; - os << "ExtentMap::mergeExtentsMaxMin(): casual partitioning update: firstLBID=" << - fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + - fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " hiVal=" << fExtentMap[i].partition.cprange.hiVal << - " loVal=" << fExtentMap[i].partition.cprange.loVal << - " min=" << it->second.min << " max=" << it->second.max << - " seq=" << it->second.seqNum; - log(os.str(), logging::LOG_TYPE_DEBUG); + ostringstream os; + os << "ExtentMap::mergeExtentsMaxMin(): casual partitioning update: firstLBID=" + << fExtentMap[i].range.start + << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 + << " OID=" << fExtentMap[i].fileID << " hiVal=" << fExtentMap[i].partition.cprange.hiVal + << " loVal=" << fExtentMap[i].partition.cprange.loVal << " min=" << it->second.min + << " max=" << it->second.max << " seq=" << it->second.seqNum; + log(os.str(), logging::LOG_TYPE_DEBUG); #endif - bool isBinaryColumn = it->second.colWidth > 8; + bool isBinaryColumn = it->second.colWidth > 8; - switch (fExtentMap[i].partition.cprange.isValid) + switch (fExtentMap[i].partition.cprange.isValid) + { + // Merge input min/max with current min/max + case CP_VALID: + { + if ((!isBinaryColumn && !isValidCPRange(it->second.max, it->second.min, it->second.type)) || + (isBinaryColumn && !isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) + { + break; + } + + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + + // We check the validity of the current min/max, + // because isValid could be CP_VALID for an extent + // having all NULL values, in which case the current + // min/max needs to be set instead of merged. + + if ((!isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.hiVal, + fExtentMap[i].partition.cprange.loVal, it->second.type)) || + (isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.bigHiVal, + fExtentMap[i].partition.cprange.bigLoVal, it->second.type))) + { + // Swap byte order to do binary string comparison + if (isCharType(it->second.type)) + { + uint64_t newMinVal = + static_cast(uint64ToStr(static_cast(it->second.min))); + uint64_t newMaxVal = + static_cast(uint64ToStr(static_cast(it->second.max))); + uint64_t oldMinVal = static_cast( + uint64ToStr(static_cast(fExtentMap[i].partition.cprange.loVal))); + uint64_t oldMaxVal = static_cast( + uint64ToStr(static_cast(fExtentMap[i].partition.cprange.hiVal))); + + if (newMinVal < oldMinVal) + fExtentMap[i].partition.cprange.loVal = it->second.min; + + if (newMaxVal > oldMaxVal) + fExtentMap[i].partition.cprange.hiVal = it->second.max; + } + else if (isUnsigned(it->second.type)) + { + if (!isBinaryColumn) { - // Merge input min/max with current min/max - case CP_VALID: - { - if ((!isBinaryColumn && !isValidCPRange(it->second.max, it->second.min, it->second.type)) || - (isBinaryColumn && !isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) - { - break; - } + if (static_cast(it->second.min) < + static_cast(fExtentMap[i].partition.cprange.loVal)) + { + fExtentMap[i].partition.cprange.loVal = it->second.min; + } - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - - // We check the validity of the current min/max, - // because isValid could be CP_VALID for an extent - // having all NULL values, in which case the current - // min/max needs to be set instead of merged. - - if ((!isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.hiVal, fExtentMap[i].partition.cprange.loVal, it->second.type)) || - (isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.bigHiVal, fExtentMap[i].partition.cprange.bigLoVal, it->second.type))) - { - // Swap byte order to do binary string comparison - if (isCharType(it->second.type)) - { - uint64_t newMinVal = - static_cast( uint64ToStr( - static_cast(it->second.min))); - uint64_t newMaxVal = - static_cast( uint64ToStr( - static_cast(it->second.max))); - uint64_t oldMinVal = - static_cast( uint64ToStr( - static_cast( - fExtentMap[i].partition.cprange.loVal)) ); - uint64_t oldMaxVal = - static_cast( uint64ToStr( - static_cast( - fExtentMap[i].partition.cprange.hiVal)) ); - - if (newMinVal < oldMinVal) - fExtentMap[i].partition.cprange.loVal = - it->second.min; - - if (newMaxVal > oldMaxVal) - fExtentMap[i].partition.cprange.hiVal = - it->second.max; - } - else if (isUnsigned(it->second.type)) - { - if (!isBinaryColumn) - { - if (static_cast(it->second.min) < - static_cast(fExtentMap[i].partition.cprange.loVal)) - { - fExtentMap[i].partition.cprange.loVal = - it->second.min; - } - - if (static_cast(it->second.max) > - static_cast(fExtentMap[i].partition.cprange.hiVal)) - { - fExtentMap[i].partition.cprange.hiVal = - it->second.max; - } - } - else - { - if (static_cast(it->second.bigMin) < - static_cast(fExtentMap[i].partition.cprange.bigLoVal)) - { - fExtentMap[i].partition.cprange.bigLoVal = - it->second.bigMin; - } - - if (static_cast(it->second.bigMax) > - static_cast(fExtentMap[i].partition.cprange.bigHiVal)) - { - fExtentMap[i].partition.cprange.bigHiVal = - it->second.bigMax; - } - } - } - else - { - if (!isBinaryColumn) - { - if (it->second.min < - fExtentMap[i].partition.cprange.loVal) - fExtentMap[i].partition.cprange.loVal = - it->second.min; - - if (it->second.max > - fExtentMap[i].partition.cprange.hiVal) - fExtentMap[i].partition.cprange.hiVal = - it->second.max; - } - else - { - if (it->second.bigMin < - fExtentMap[i].partition.cprange.bigLoVal) - fExtentMap[i].partition.cprange.bigLoVal = - it->second.bigMin; - - if (it->second.bigMax > - fExtentMap[i].partition.cprange.bigHiVal) - fExtentMap[i].partition.cprange.bigHiVal = - it->second.bigMax; - } - } - } - else - { - if (!isBinaryColumn) - { - fExtentMap[i].partition.cprange.loVal = - it->second.min; - fExtentMap[i].partition.cprange.hiVal = - it->second.max; - } - else - { - fExtentMap[i].partition.cprange.bigLoVal = - it->second.bigMin; - fExtentMap[i].partition.cprange.bigHiVal = - it->second.bigMax; - } - } - - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - - break; - } - - // DML is updating; just increment seqnum. - // This case is here for completeness. Table lock should - // prevent this state from occurring (see notes at top of - // this function) - case CP_UPDATING: - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - - break; - } - - // Reset min/max to new min/max only "if" we can treat this - // as a new extent, else leave the extent marked as INVALID - case CP_INVALID: - default: - { - makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - - if (it->second.newExtent) - { - if ((!isBinaryColumn && isValidCPRange(it->second.max, it->second.min, it->second.type)) || - (isBinaryColumn && isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) - { - if (!isBinaryColumn) - { - fExtentMap[i].partition.cprange.loVal = - it->second.min; - fExtentMap[i].partition.cprange.hiVal = - it->second.max; - } - else - { - fExtentMap[i].partition.cprange.bigLoVal = - it->second.bigMin; - fExtentMap[i].partition.cprange.bigHiVal = - it->second.bigMax; - } - } - - // Even if invalid range; we set state to CP_VALID, - // because the extent is valid, it is just empty. - fExtentMap[i].partition.cprange.isValid = CP_VALID; - } - - incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); - break; - } - } // switch on isValid state - - extentsMerged++; - - if (extentsMerged == extentsToMerge) - { - return; // Leave when all extents in map are matched + if (static_cast(it->second.max) > + static_cast(fExtentMap[i].partition.cprange.hiVal)) + { + fExtentMap[i].partition.cprange.hiVal = it->second.max; + } } + else + { + if (static_cast(it->second.bigMin) < + static_cast(fExtentMap[i].partition.cprange.bigLoVal)) + { + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } - // Deleting objects from map, may speed up successive searches - cpMap.erase( it ); + if (static_cast(it->second.bigMax) > + static_cast(fExtentMap[i].partition.cprange.bigHiVal)) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + } + } + } + else + { + if (!isBinaryColumn) + { + if (it->second.min < fExtentMap[i].partition.cprange.loVal) + fExtentMap[i].partition.cprange.loVal = it->second.min; - } // found a matching extent in the Map - } // extent map range size != 0 - } // end of loop through extent map + if (it->second.max > fExtentMap[i].partition.cprange.hiVal) + fExtentMap[i].partition.cprange.hiVal = it->second.max; + } + else + { + if (it->second.bigMin < fExtentMap[i].partition.cprange.bigLoVal) + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; - throw logic_error("ExtentMap::mergeExtentsMaxMin(): lbid not found"); + if (it->second.bigMax > fExtentMap[i].partition.cprange.bigHiVal) + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + } + } + } + else + { + if (!isBinaryColumn) + { + fExtentMap[i].partition.cprange.loVal = it->second.min; + fExtentMap[i].partition.cprange.hiVal = it->second.max; + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + } + } + + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + + break; + } + + // DML is updating; just increment seqnum. + // This case is here for completeness. Table lock should + // prevent this state from occurring (see notes at top of + // this function) + case CP_UPDATING: + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + + break; + } + + // Reset min/max to new min/max only "if" we can treat this + // as a new extent, else leave the extent marked as INVALID + case CP_INVALID: + default: + { + makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); + + if (it->second.newExtent) + { + if ((!isBinaryColumn && isValidCPRange(it->second.max, it->second.min, it->second.type)) || + (isBinaryColumn && isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) + { + if (!isBinaryColumn) + { + fExtentMap[i].partition.cprange.loVal = it->second.min; + fExtentMap[i].partition.cprange.hiVal = it->second.max; + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + } + } + + // Even if invalid range; we set state to CP_VALID, + // because the extent is valid, it is just empty. + fExtentMap[i].partition.cprange.isValid = CP_VALID; + } + + incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); + break; + } + } // switch on isValid state + + extentsMerged++; + + if (extentsMerged == extentsToMerge) + { + return; // Leave when all extents in map are matched + } + + // Deleting objects from map, may speed up successive searches + cpMap.erase(it); + + } // found a matching extent in the Map + } // extent map range size != 0 + } // end of loop through extent map + + throw logic_error("ExtentMap::mergeExtentsMaxMin(): lbid not found"); } //------------------------------------------------------------------------------ @@ -1075,431 +1028,414 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // (min()+1). For unsigned types NULL is max() and EMPTY is max()-1. //------------------------------------------------------------------------------ template -bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const +bool ExtentMap::isValidCPRange(const T& max, const T& min, + execplan::CalpontSystemCatalog::ColDataType type) const { - if (isUnsigned(type)) + if (isUnsigned(type)) + { + if (typeid(T) != typeid(int128_t)) { - if (typeid(T) != typeid(int128_t)) - { - if ( (static_cast(min) >= (numeric_limits::max() - 1)) || - (static_cast(max) >= (numeric_limits::max() - 1)) ) - { - return false; - } - } - else - { - uint128_t temp; - utils::uint128Max(temp); - - if ( (static_cast(min) >= (temp - 1)) || - (static_cast(max) >= (temp - 1)) ) - { - return false; - } - } + if ((static_cast(min) >= (numeric_limits::max() - 1)) || + (static_cast(max) >= (numeric_limits::max() - 1))) + { + return false; + } } else { - if (typeid(T) != typeid(int128_t)) - { - if ( (min <= (numeric_limits::min() + 1)) || - (max <= (numeric_limits::min() + 1)) ) - { - return false; - } - } - else - { - int128_t temp; - utils::int128Min(temp); + uint128_t temp; + utils::uint128Max(temp); - if ( (min <= (temp + 1)) || - (max <= (temp + 1)) ) - { - return false; - } - } + if ((static_cast(min) >= (temp - 1)) || (static_cast(max) >= (temp - 1))) + { + return false; + } } + } + else + { + if (typeid(T) != typeid(int128_t)) + { + if ((min <= (numeric_limits::min() + 1)) || (max <= (numeric_limits::min() + 1))) + { + return false; + } + } + else + { + int128_t temp; + utils::int128Min(temp); - return true; + if ((min <= (temp + 1)) || (max <= (temp + 1))) + { + return false; + } + } + } + + return true; } /** -* @brief retrieve the hiVal and loVal or sequenceNum of the extent containing the LBID lbid. -* -* For the extent containing the LBID lbid, return the max/min values if the extent range values -* are valid and a -1 in the seqNum parameter. If the range values are flaged as invalid -* return the sequenceNum of the extent and the max/min values as -1. -**/ + * @brief retrieve the hiVal and loVal or sequenceNum of the extent containing the LBID lbid. + * + * For the extent containing the LBID lbid, return the max/min values if the extent range values + * are valid and a -1 in the seqNum parameter. If the range values are flaged as invalid + * return the sequenceNum of the extent and the max/min values as -1. + **/ template -int ExtentMap::getMaxMin(const LBID_t lbid, - T& max, T& min, - int32_t& seqNum) +int ExtentMap::getMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getMaxMin"); - TRACER_ADDINPUT(lbid); - TRACER_ADDOUTPUT(max); - TRACER_ADDOUTPUT(min); - TRACER_ADDOUTPUT(seqNum); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getMaxMin"); + TRACER_ADDINPUT(lbid); + TRACER_ADDOUTPUT(max); + TRACER_ADDOUTPUT(min); + TRACER_ADDOUTPUT(seqNum); + TRACER_WRITE; + } #endif - if (typeid(T) == typeid(int128_t)) - { - int128_t tmpMax, tmpMin; - utils::int128Min(tmpMax); - utils::int128Max(tmpMin); - max = tmpMax; - min = tmpMin; - } - else - { - max = numeric_limits::min(); - min = numeric_limits::max(); - } - seqNum *= (-1); - int entries; - int i; - LBID_t lastBlock; - int isValid = CP_INVALID; + if (typeid(T) == typeid(int128_t)) + { + int128_t tmpMax, tmpMin; + utils::int128Min(tmpMax); + utils::int128Max(tmpMin); + max = tmpMax; + min = tmpMin; + } + else + { + max = numeric_limits::min(); + min = numeric_limits::max(); + } + seqNum *= (-1); + int entries; + int i; + LBID_t lastBlock; + int isValid = CP_INVALID; #ifdef BRM_DEBUG - if (lbid < 0) - throw invalid_argument("ExtentMap::getMaxMin(): lbid must be >= 0"); + if (lbid < 0) + throw invalid_argument("ExtentMap::getMaxMin(): lbid must be >= 0"); #endif - grabEMEntryTable(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(READ); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (i = 0; i < entries; i++) + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) + lastBlock = fExtentMap[i].range.start + (static_cast(fExtentMap[i].range.size) * 1024) - 1; + + if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) + { + if (typeid(T) == typeid(int128_t)) { - lastBlock = fExtentMap[i].range.start + - (static_cast(fExtentMap[i].range.size) * 1024) - 1; - - if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) - { - if (typeid(T) == typeid(int128_t)) - { - max = fExtentMap[i].partition.cprange.bigHiVal; - min = fExtentMap[i].partition.cprange.bigLoVal; - } - else - { - max = fExtentMap[i].partition.cprange.hiVal; - min = fExtentMap[i].partition.cprange.loVal; - } - seqNum = fExtentMap[i].partition.cprange.sequenceNum; - isValid = fExtentMap[i].partition.cprange.isValid; - releaseEMEntryTable(READ); - return isValid; - } + max = fExtentMap[i].partition.cprange.bigHiVal; + min = fExtentMap[i].partition.cprange.bigLoVal; } + else + { + max = fExtentMap[i].partition.cprange.hiVal; + min = fExtentMap[i].partition.cprange.loVal; + } + seqNum = fExtentMap[i].partition.cprange.sequenceNum; + isValid = fExtentMap[i].partition.cprange.isValid; + releaseEMEntryTable(READ); + return isValid; + } } + } - releaseEMEntryTable(READ); - throw logic_error("ExtentMap::getMaxMin(): that lbid isn't allocated"); -// return -1; + releaseEMEntryTable(READ); + throw logic_error("ExtentMap::getMaxMin(): that lbid isn't allocated"); + // return -1; } void ExtentMap::getCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMin) { - int entries; - int i; - LBID_t lastBlock; + int entries; + int i; + LBID_t lastBlock; #ifdef BRM_DEBUG - if (lbid < 0) - throw invalid_argument("ExtentMap::getMaxMin(): lbid must be >= 0"); + if (lbid < 0) + throw invalid_argument("ExtentMap::getMaxMin(): lbid must be >= 0"); #endif - grabEMEntryTable(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(READ); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (i = 0; i < entries; i++) + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) - { - lastBlock = fExtentMap[i].range.start + - (static_cast(fExtentMap[i].range.size) * 1024) - 1; + lastBlock = fExtentMap[i].range.start + (static_cast(fExtentMap[i].range.size) * 1024) - 1; - if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) - { - cpMaxMin.bigMax = fExtentMap[i].partition.cprange.bigHiVal; - cpMaxMin.bigMin = fExtentMap[i].partition.cprange.bigLoVal; - cpMaxMin.max = fExtentMap[i].partition.cprange.hiVal; - cpMaxMin.min = fExtentMap[i].partition.cprange.loVal; - cpMaxMin.seqNum = fExtentMap[i].partition.cprange.sequenceNum; + if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) + { + cpMaxMin.bigMax = fExtentMap[i].partition.cprange.bigHiVal; + cpMaxMin.bigMin = fExtentMap[i].partition.cprange.bigLoVal; + cpMaxMin.max = fExtentMap[i].partition.cprange.hiVal; + cpMaxMin.min = fExtentMap[i].partition.cprange.loVal; + cpMaxMin.seqNum = fExtentMap[i].partition.cprange.sequenceNum; - releaseEMEntryTable(READ); - return ; - } - } + releaseEMEntryTable(READ); + return; + } } + } - releaseEMEntryTable(READ); - throw logic_error("ExtentMap::getMaxMin(): that lbid isn't allocated"); + releaseEMEntryTable(READ); + throw logic_error("ExtentMap::getMaxMin(): that lbid isn't allocated"); } /* Removes a range from the freelist. Used by load() */ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size) { - int i; - int flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - LBID_t lastLBID = start + (size * 1024) - 1; - int32_t freeIndex = -1; + int i; + int flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + LBID_t lastLBID = start + (size * 1024) - 1; + int32_t freeIndex = -1; - /* Find a range the request intersects. There should be one and only one. */ - for (i = 0; i < flEntries; i++) + /* Find a range the request intersects. There should be one and only one. */ + for (i = 0; i < flEntries; i++) + { + LBID_t eLastLBID; + + // while scanning, grab the first free slot + if (fFreeList[i].size == 0) { - LBID_t eLastLBID; + if (freeIndex == -1) + freeIndex = i; - // while scanning, grab the first free slot - if (fFreeList[i].size == 0) + continue; + } + + eLastLBID = fFreeList[i].start + (((int64_t)fFreeList[i].size) * 1024) - 1; + + /* if it's at the front... */ + if (start == fFreeList[i].start) + { + /* if the request is larger than the freelist entry -> implies an extent + * overlap. This is debugging code. */ + // idbassert(size > fFreeList[i].size); + makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); + fFreeList[i].start += size * 1024; + fFreeList[i].size -= size; + + if (fFreeList[i].size == 0) + { + makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); + fFLShminfo->currentSize -= sizeof(InlineLBIDRange); + } + + break; + } + /* if it's at the back... */ + else if (eLastLBID == lastLBID) + { + makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); + fFreeList[i].size -= size; + + if (fFreeList[i].size == 0) + { + makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); + fFLShminfo->currentSize -= sizeof(InlineLBIDRange); + } + + break; + /* This entry won't be the same size as the request or the first + * clause would have run instead. + */ + } + /* if it's in the middle... */ + /* break it into two elements */ + else if (fFreeList[i].start < start && eLastLBID > lastLBID) + { + if (freeIndex == -1) + { + if (fFLShminfo->currentSize == fFLShminfo->allocdSize) { - if (freeIndex == -1) - freeIndex = i; - - continue; + growFLShmseg(); + freeIndex = flEntries; } - - eLastLBID = fFreeList[i].start + (((int64_t) fFreeList[i].size) * 1024) - 1; - - /* if it's at the front... */ - if (start == fFreeList[i].start) - { - /* if the request is larger than the freelist entry -> implies an extent - * overlap. This is debugging code. */ - //idbassert(size > fFreeList[i].size); - makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); - fFreeList[i].start += size * 1024; - fFreeList[i].size -= size; - - if (fFreeList[i].size == 0) - { - makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); - fFLShminfo->currentSize -= sizeof(InlineLBIDRange); - } - - break; - } - /* if it's at the back... */ - else if (eLastLBID == lastLBID) - { - makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); - fFreeList[i].size -= size; - - if (fFreeList[i].size == 0) - { - makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); - fFLShminfo->currentSize -= sizeof(InlineLBIDRange); - } - - break; - /* This entry won't be the same size as the request or the first - * clause would have run instead. - */ - } - /* if it's in the middle... */ - /* break it into two elements */ - else if (fFreeList[i].start < start && eLastLBID > lastLBID) - { - if (freeIndex == -1) - { - if (fFLShminfo->currentSize == fFLShminfo->allocdSize) - { - growFLShmseg(); - freeIndex = flEntries; - } - else - for (freeIndex = i + 1; freeIndex < flEntries; freeIndex++) - if (fFreeList[freeIndex].size == 0) - break; + else + for (freeIndex = i + 1; freeIndex < flEntries; freeIndex++) + if (fFreeList[freeIndex].size == 0) + break; #ifdef BRM_DEBUG - idbassert(nextIndex < flEntries); + idbassert(nextIndex < flEntries); #endif - } + } - makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); - makeUndoRecord(&fFreeList[freeIndex], sizeof(InlineLBIDRange)); - makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); - fFreeList[i].size = (start - fFreeList[i].start) / 1024; - fFreeList[freeIndex].start = start + (size * 1024); - fFreeList[freeIndex].size = (eLastLBID - lastLBID) / 1024; - fFLShminfo->currentSize += sizeof(InlineLBIDRange); - break; - } + makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); + makeUndoRecord(&fFreeList[freeIndex], sizeof(InlineLBIDRange)); + makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); + fFreeList[i].size = (start - fFreeList[i].start) / 1024; + fFreeList[freeIndex].start = start + (size * 1024); + fFreeList[freeIndex].size = (eLastLBID - lastLBID) / 1024; + fFLShminfo->currentSize += sizeof(InlineLBIDRange); + break; } + } } /* - The file layout looks like this: + The file layout looks like this: - EM Magic (32-bits) - number of EM entries (32-bits) - number of FL entries (32-bits) - EMEntry - ... (* numEM) - struct InlineLBIDRange - ... (* numFL) + EM Magic (32-bits) + number of EM entries (32-bits) + number of FL entries (32-bits) + EMEntry + ... (* numEM) + struct InlineLBIDRange + ... (* numFL) */ - template void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) { - int emNumElements = 0, flNumElements = 0; + int emNumElements = 0, flNumElements = 0; - int nbytes = 0; - nbytes += in->read((char*) &emNumElements, sizeof(int)); - nbytes += in->read((char*) &flNumElements, sizeof(int)); - idbassert(emNumElements > 0); + int nbytes = 0; + nbytes += in->read((char*)&emNumElements, sizeof(int)); + nbytes += in->read((char*)&flNumElements, sizeof(int)); + idbassert(emNumElements > 0); - if ((size_t) nbytes != sizeof(int) + sizeof(int)) + if ((size_t)nbytes != sizeof(int) + sizeof(int)) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); + } + + void* fExtentMapPtr = static_cast(fExtentMap); + memset(fExtentMapPtr, 0, fEMShminfo->allocdSize); + fEMShminfo->currentSize = 0; + + // init the free list + memset(fFreeList, 0, fFLShminfo->allocdSize); + fFreeList[0].size = (1 << 26); // 2^36 LBIDs + fFLShminfo->currentSize = sizeof(InlineLBIDRange); + + // @Bug 3498 + // Calculate how big an extent map we're going to need and allocate it in one call + if ((fEMShminfo->allocdSize / sizeof(EMEntry)) < (unsigned)emNumElements) + { + size_t nrows = emNumElements; + + // Round up to the nearest EM_INCREMENT_ROWS + if ((nrows % EM_INCREMENT_ROWS) != 0) { + nrows /= EM_INCREMENT_ROWS; + nrows++; + nrows *= EM_INCREMENT_ROWS; + } + + growEMShmseg(nrows); + } + + int err; + char* writePos; + size_t progress, writeSize; + + if (!upgradeV4ToV5) + { + progress = 0; + writeSize = emNumElements * sizeof(EMEntry); + writePos = (char*)fExtentMap; + + while (progress < writeSize) + { + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { log_errno("ExtentMap::loadVersion4or5(): read "); throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); + } + progress += (uint)err; } - - void *fExtentMapPtr = static_cast(fExtentMap); - memset(fExtentMapPtr, 0, fEMShminfo->allocdSize); - fEMShminfo->currentSize = 0; - - // init the free list - memset(fFreeList, 0, fFLShminfo->allocdSize); - fFreeList[0].size = (1 << 26); // 2^36 LBIDs - fFLShminfo->currentSize = sizeof(InlineLBIDRange); - - // @Bug 3498 - // Calculate how big an extent map we're going to need and allocate it in one call - if ((fEMShminfo->allocdSize / sizeof(EMEntry)) < (unsigned)emNumElements) - { - size_t nrows = emNumElements; - - //Round up to the nearest EM_INCREMENT_ROWS - if ((nrows % EM_INCREMENT_ROWS) != 0) - { - nrows /= EM_INCREMENT_ROWS; - nrows++; - nrows *= EM_INCREMENT_ROWS; - } - - growEMShmseg(nrows); - } - - int err; - char* writePos; - size_t progress, writeSize; - - if (!upgradeV4ToV5) - { - progress = 0; - writeSize = emNumElements * sizeof(EMEntry); - writePos = (char *) fExtentMap; - - while (progress < writeSize) - { - err = in->read(writePos + progress, writeSize - progress); - if (err <= 0) - { - log_errno("ExtentMap::loadVersion4or5(): read "); - throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); - } - progress += (uint) err; - } - } - else - { - // We are upgrading extent map from v4 to v5. - for (int i = 0; i < emNumElements; i++) - { - EMEntry_v4 emEntryV4; - progress = 0; - writeSize = sizeof(EMEntry_v4); - writePos = (char *) &(emEntryV4); - while (progress < writeSize) - { - err = in->read(writePos + progress, writeSize - progress); - if (err <= 0) - { - log_errno("ExtentMap::loadVersion4or5(): read "); - throw runtime_error("ExtentMap::loadVersion4or5(): read failed during upgrade. Check the error log."); - } - progress += (uint) err; - } - - fExtentMap[i].range.start = emEntryV4.range.start; - fExtentMap[i].range.size = emEntryV4.range.size; - fExtentMap[i].fileID = emEntryV4.fileID; - fExtentMap[i].blockOffset = emEntryV4.blockOffset; - fExtentMap[i].HWM = emEntryV4.HWM; - fExtentMap[i].partitionNum = emEntryV4.partitionNum; - fExtentMap[i].segmentNum = emEntryV4.segmentNum; - fExtentMap[i].dbRoot = emEntryV4.dbRoot; - fExtentMap[i].colWid = emEntryV4.colWid; - fExtentMap[i].status = emEntryV4.status; - fExtentMap[i].partition.cprange.hiVal = emEntryV4.partition.cprange.hi_val; - fExtentMap[i].partition.cprange.loVal = emEntryV4.partition.cprange.lo_val; - fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum; - fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid; - } - - std::cout<read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error( + "ExtentMap::loadVersion4or5(): read failed during upgrade. Check the error log."); + } + progress += (uint)err; + } - //@bug 1911 - verify status value is valid - if (fExtentMap[i].status < EXTENTSTATUSMIN || - fExtentMap[i].status > EXTENTSTATUSMAX) - fExtentMap[i].status = EXTENTAVAILABLE; + fExtentMap[i].range.start = emEntryV4.range.start; + fExtentMap[i].range.size = emEntryV4.range.size; + fExtentMap[i].fileID = emEntryV4.fileID; + fExtentMap[i].blockOffset = emEntryV4.blockOffset; + fExtentMap[i].HWM = emEntryV4.HWM; + fExtentMap[i].partitionNum = emEntryV4.partitionNum; + fExtentMap[i].segmentNum = emEntryV4.segmentNum; + fExtentMap[i].dbRoot = emEntryV4.dbRoot; + fExtentMap[i].colWid = emEntryV4.colWid; + fExtentMap[i].status = emEntryV4.status; + fExtentMap[i].partition.cprange.hiVal = emEntryV4.partition.cprange.hi_val; + fExtentMap[i].partition.cprange.loVal = emEntryV4.partition.cprange.lo_val; + fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum; + fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid; } - fEMShminfo->currentSize = emNumElements * sizeof(EMEntry); + std::cout << emNumElements << " extents successfully upgraded" << std::endl; + } + + for (int i = 0; i < emNumElements; i++) + { + reserveLBIDRange(fExtentMap[i].range.start, fExtentMap[i].range.size); + + //@bug 1911 - verify status value is valid + if (fExtentMap[i].status < EXTENTSTATUSMIN || fExtentMap[i].status > EXTENTSTATUSMAX) + fExtentMap[i].status = EXTENTAVAILABLE; + } + + fEMShminfo->currentSize = emNumElements * sizeof(EMEntry); #ifdef DUMP_EXTENT_MAP - EMEntry* emSrc = fExtentMap; - cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl; + EMEntry* emSrc = fExtentMap; + cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl; - for (int i = 0; i < emNumElements; i++) - { - cout << - emSrc[i].start - << '\t' << emSrc[i].size - << '\t' << emSrc[i].fileID - << '\t' << emSrc[i].blockOffset - << '\t' << emSrc[i].HWM - << '\t' << emSrc[i].partitionNum - << '\t' << emSrc[i].segmentNum - << '\t' << emSrc[i].dbRoot - << '\t' << emSrc[i].status - << '\t' << emSrc[i].partition.cprange.hiVal - << '\t' << emSrc[i].partition.cprange.loVal - << '\t' << emSrc[i].partition.cprange.sequenceNum - << '\t' << (int)(emSrc[i].partition.cprange.isValid) - << endl; - } + for (int i = 0; i < emNumElements; i++) + { + cout << emSrc[i].start << '\t' << emSrc[i].size << '\t' << emSrc[i].fileID << '\t' << emSrc[i].blockOffset + << '\t' << emSrc[i].HWM << '\t' << emSrc[i].partitionNum << '\t' << emSrc[i].segmentNum << '\t' + << emSrc[i].dbRoot << '\t' << emSrc[i].status << '\t' << emSrc[i].partition.cprange.hiVal << '\t' + << emSrc[i].partition.cprange.loVal << '\t' << emSrc[i].partition.cprange.sequenceNum << '\t' + << (int)(emSrc[i].partition.cprange.isValid) << endl; + } - cout << "Free list entries:" << endl; - cout << "start\tsize" << endl; + cout << "Free list entries:" << endl; + cout << "start\tsize" << endl; - for (int i = 0; i < flNumElements; i++) - cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl; + for (int i = 0; i < flNumElements; i++) + cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl; #endif } @@ -1508,55 +1444,54 @@ void ExtentMap::load(const string& filename, bool fixFL) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("load"); - TRACER_ADDSTRINPUT(filename); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("load"); + TRACER_ADDSTRINPUT(filename); + TRACER_WRITE; + } #endif - grabEMEntryTable(WRITE); + grabEMEntryTable(WRITE); - try - { - grabFreeList(WRITE); - } - catch (...) - { - releaseEMEntryTable(WRITE); - throw; - } + try + { + grabFreeList(WRITE); + } + catch (...) + { + releaseEMEntryTable(WRITE); + throw; + } - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "r", 0)); - - if (!in) - { - log_errno("ExtentMap::load(): open"); - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); - } - - try - { - load(in.get()); - } - - catch (...) - { - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw; - } + const char* filename_p = filename.c_str(); + scoped_ptr in( + IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "r", 0)); + if (!in) + { + log_errno("ExtentMap::load(): open"); releaseFreeList(WRITE); releaseEMEntryTable(WRITE); -// checkConsistency(); + throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); + } + + try + { + load(in.get()); + } + + catch (...) + { + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); + throw; + } + + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); + // checkConsistency(); } // This is a quick workaround, to be able to initialize initial system tables @@ -1564,650 +1499,647 @@ void ExtentMap::load(const string& filename, bool fixFL) // This should be updated, probably we need inherit from `IDBDataFile`. struct EMBinaryReader { - EMBinaryReader(const char* data) : src(data) {} + EMBinaryReader(const char* data) : src(data) + { + } - ssize_t read(char* dst, size_t size) - { - memcpy(dst, src, size); - src += size; - return size; - } + ssize_t read(char* dst, size_t size) + { + memcpy(dst, src, size); + src += size; + return size; + } - const char* src; + const char* src; }; void ExtentMap::loadFromBinaryBlob(const char* blob) { - grabEMEntryTable(WRITE); + grabEMEntryTable(WRITE); - try - { - grabFreeList(WRITE); - } - catch (...) - { - releaseEMEntryTable(WRITE); - throw; - } - - try - { - EMBinaryReader emBinReader(blob); - load(&emBinReader); - } - catch (...) - { - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw; - } + try + { + grabFreeList(WRITE); + } + catch (...) + { + releaseEMEntryTable(WRITE); + throw; + } + try + { + EMBinaryReader emBinReader(blob); + load(&emBinReader); + } + catch (...) + { releaseFreeList(WRITE); releaseEMEntryTable(WRITE); + throw; + } + + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); } -template void ExtentMap::load(T* in) +template +void ExtentMap::load(T* in) { - if (!in) - return; + if (!in) + return; - try - { - int emVersion = 0; - int bytes = in->read((char*) &emVersion, sizeof(int)); + try + { + int emVersion = 0; + int bytes = in->read((char*)&emVersion, sizeof(int)); - if (bytes == (int) sizeof(int) && - (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) - { - loadVersion4or5(in, emVersion == EM_MAGIC_V4); - } - else - { - log("ExtentMap::load(): That file is not a valid ExtentMap image"); - throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); - } - } - catch (...) + if (bytes == (int)sizeof(int) && (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) { - throw; + loadVersion4or5(in, emVersion == EM_MAGIC_V4); } + else + { + log("ExtentMap::load(): That file is not a valid ExtentMap image"); + throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); + } + } + catch (...) + { + throw; + } } void ExtentMap::save(const string& filename) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("save"); - TRACER_ADDSTRINPUT(filename); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("save"); + TRACER_ADDSTRINPUT(filename); + TRACER_WRITE; + } #endif - int allocdSize, loadSize[3], i; + int allocdSize, loadSize[3], i; - grabEMEntryTable(READ); + grabEMEntryTable(READ); - try - { - grabFreeList(READ); - } - catch (...) - { - releaseEMEntryTable(READ); - throw; - } - - if (fEMShminfo->currentSize == 0) - { - log("ExtentMap::save(): got request to save an empty BRM"); - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw runtime_error("ExtentMap::save(): got request to save an empty BRM"); - } - - const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", IDBDataFile::USE_VBUF)); - - if (!out) - { - log_errno("ExtentMap::save(): open"); - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); - } - - loadSize[0] = EM_MAGIC_V5; - loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); - loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries - - int bytes = 0; - - try - { - const int wsize = 3 * sizeof(int); - bytes = out->write((char*)loadSize, wsize); - - if (bytes != wsize) - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); - } - catch (...) - { - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw; - } - - allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry); - //const int emEntrySize = sizeof(EMEntry); - - int first = -1, last = -1, err; - size_t progress, writeSize; - for (i = 0; i < allocdSize; i++) - { - if (fExtentMap[i].range.size > 0 && first == -1) - first = i; - else if (fExtentMap[i].range.size <= 0 && first != -1) - { - last = i; - writeSize = (last - first) * sizeof(EMEntry); - progress = 0; - char *writePos = (char *) &fExtentMap[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); - } - progress += err; - } - first = -1; - } - - } - if (first != -1) - { - writeSize = (allocdSize - first) * sizeof(EMEntry); - progress = 0; - char *writePos = (char *) &fExtentMap[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); - } - progress += err; - } - } - - //allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - //const int inlineLbidRangeSize = sizeof(InlineLBIDRange); - - progress = 0; - writeSize = fFLShminfo->allocdSize; - char *writePos = (char *) fFreeList; - while (progress < writeSize) + try + { + grabFreeList(READ); + } + catch (...) + { + releaseEMEntryTable(READ); + throw; + } + + if (fEMShminfo->currentSize == 0) + { + log("ExtentMap::save(): got request to save an empty BRM"); + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw runtime_error("ExtentMap::save(): got request to save an empty BRM"); + } + + const char* filename_p = filename.c_str(); + scoped_ptr out(IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", IDBDataFile::USE_VBUF)); + + if (!out) + { + log_errno("ExtentMap::save(): open"); + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); + } + + loadSize[0] = EM_MAGIC_V5; + loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); + loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries + + int bytes = 0; + + try + { + const int wsize = 3 * sizeof(int); + bytes = out->write((char*)loadSize, wsize); + + if (bytes != wsize) + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); + } + catch (...) + { + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw; + } + + allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry); + // const int emEntrySize = sizeof(EMEntry); + + int first = -1, last = -1, err; + size_t progress, writeSize; + for (i = 0; i < allocdSize; i++) + { + if (fExtentMap[i].range.size > 0 && first == -1) + first = i; + else if (fExtentMap[i].range.size <= 0 && first != -1) { + last = i; + writeSize = (last - first) * sizeof(EMEntry); + progress = 0; + char* writePos = (char*)&fExtentMap[first]; + while (progress < writeSize) + { err = out->write(writePos + progress, writeSize - progress); if (err < 0) { - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } - progress += err; + } + first = -1; + } + } + if (first != -1) + { + writeSize = (allocdSize - first) * sizeof(EMEntry); + progress = 0; + char* writePos = (char*)&fExtentMap[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); + } + progress += err; + } + } + + // allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + // const int inlineLbidRangeSize = sizeof(InlineLBIDRange); + + progress = 0; + writeSize = fFLShminfo->allocdSize; + char* writePos = (char*)fFreeList; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } - releaseFreeList(READ); - releaseEMEntryTable(READ); + progress += err; + } + + releaseFreeList(READ); + releaseEMEntryTable(READ); } /* always returns holding the EM lock, and with the EM seg mapped */ void ExtentMap::grabEMEntryTable(OPS op) { - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - if (op == READ) - fEMShminfo = fMST.getTable_read(MasterSegmentTable::EMTable); - else + if (op == READ) + fEMShminfo = fMST.getTable_read(MasterSegmentTable::EMTable); + else + { + fEMShminfo = fMST.getTable_write(MasterSegmentTable::EMTable); + emLocked = true; + } + + if (!fPExtMapImpl || fPExtMapImpl->key() != (unsigned)fEMShminfo->tableShmkey) + { + if (fExtentMap != NULL) { - fEMShminfo = fMST.getTable_write(MasterSegmentTable::EMTable); - emLocked = true; + fExtentMap = NULL; } - if (!fPExtMapImpl || fPExtMapImpl->key() != (unsigned)fEMShminfo->tableShmkey) + if (fEMShminfo->allocdSize == 0) { - if (fExtentMap != NULL) - { - fExtentMap = NULL; - } + if (op == READ) + { + fMST.getTable_upgrade(MasterSegmentTable::EMTable); + emLocked = true; if (fEMShminfo->allocdSize == 0) - { - if (op == READ) - { - fMST.getTable_upgrade(MasterSegmentTable::EMTable); - emLocked = true; + growEMShmseg(); - if (fEMShminfo->allocdSize == 0) - growEMShmseg(); - - emLocked = false; // has to be done holding the write lock - fMST.getTable_downgrade(MasterSegmentTable::EMTable); - } - else - growEMShmseg(); - } - else - { - fPExtMapImpl = ExtentMapImpl::makeExtentMapImpl(fEMShminfo->tableShmkey, 0); - ASSERT(fPExtMapImpl); - - if (r_only) - fPExtMapImpl->makeReadOnly(); - - fExtentMap = fPExtMapImpl->get(); - - if (fExtentMap == NULL) - { - log_errno("ExtentMap::grabEMEntryTable(): shmat"); - throw runtime_error("ExtentMap::grabEMEntryTable(): shmat failed. Check the error log."); - } - } + emLocked = false; // has to be done holding the write lock + fMST.getTable_downgrade(MasterSegmentTable::EMTable); + } + else + growEMShmseg(); } else - fExtentMap = fPExtMapImpl->get(); + { + fPExtMapImpl = ExtentMapImpl::makeExtentMapImpl(fEMShminfo->tableShmkey, 0); + ASSERT(fPExtMapImpl); + + if (r_only) + fPExtMapImpl->makeReadOnly(); + + fExtentMap = fPExtMapImpl->get(); + + if (fExtentMap == NULL) + { + log_errno("ExtentMap::grabEMEntryTable(): shmat"); + throw runtime_error("ExtentMap::grabEMEntryTable(): shmat failed. Check the error log."); + } + } + } + else + fExtentMap = fPExtMapImpl->get(); } /* always returns holding the FL lock */ void ExtentMap::grabFreeList(OPS op) { - boost::mutex::scoped_lock lk(mutex, boost::defer_lock); + boost::mutex::scoped_lock lk(mutex, boost::defer_lock); - if (op == READ) + if (op == READ) + { + fFLShminfo = fMST.getTable_read(MasterSegmentTable::EMFreeList); + lk.lock(); + } + else + { + fFLShminfo = fMST.getTable_write(MasterSegmentTable::EMFreeList); + flLocked = true; + } + + if (!fPFreeListImpl || fPFreeListImpl->key() != (unsigned)fFLShminfo->tableShmkey) + { + if (fFreeList != NULL) { - fFLShminfo = fMST.getTable_read(MasterSegmentTable::EMFreeList); - lk.lock(); + fFreeList = NULL; } - else + + if (fFLShminfo->allocdSize == 0) { - fFLShminfo = fMST.getTable_write(MasterSegmentTable::EMFreeList); + if (op == READ) + { + lk.unlock(); + fMST.getTable_upgrade(MasterSegmentTable::EMFreeList); flLocked = true; - } - - if (!fPFreeListImpl || fPFreeListImpl->key() != (unsigned)fFLShminfo->tableShmkey) - { - if (fFreeList != NULL) - { - fFreeList = NULL; - } if (fFLShminfo->allocdSize == 0) - { - if (op == READ) - { - lk.unlock(); - fMST.getTable_upgrade(MasterSegmentTable::EMFreeList); - flLocked = true; + growFLShmseg(); - if (fFLShminfo->allocdSize == 0) - growFLShmseg(); - - flLocked = false; // has to be done holding the write lock - fMST.getTable_downgrade(MasterSegmentTable::EMFreeList); - } - else - growFLShmseg(); - } - else - { - fPFreeListImpl = FreeListImpl::makeFreeListImpl(fFLShminfo->tableShmkey, 0); - ASSERT(fPFreeListImpl); - - if (r_only) - fPFreeListImpl->makeReadOnly(); - - fFreeList = fPFreeListImpl->get(); - - if (fFreeList == NULL) - { - log_errno("ExtentMap::grabFreeList(): shmat"); - throw runtime_error("ExtentMap::grabFreeList(): shmat failed. Check the error log."); - } - - if (op == READ) - lk.unlock(); - } + flLocked = false; // has to be done holding the write lock + fMST.getTable_downgrade(MasterSegmentTable::EMFreeList); + } + else + growFLShmseg(); } else { - fFreeList = fPFreeListImpl->get(); + fPFreeListImpl = FreeListImpl::makeFreeListImpl(fFLShminfo->tableShmkey, 0); + ASSERT(fPFreeListImpl); - if (op == READ) - lk.unlock(); + if (r_only) + fPFreeListImpl->makeReadOnly(); + + fFreeList = fPFreeListImpl->get(); + + if (fFreeList == NULL) + { + log_errno("ExtentMap::grabFreeList(): shmat"); + throw runtime_error("ExtentMap::grabFreeList(): shmat failed. Check the error log."); + } + + if (op == READ) + lk.unlock(); } + } + else + { + fFreeList = fPFreeListImpl->get(); + + if (op == READ) + lk.unlock(); + } } void ExtentMap::releaseEMEntryTable(OPS op) { - if (op == READ) - fMST.releaseTable_read(MasterSegmentTable::EMTable); - else - { - /* - Note: Technically we should mark it unlocked after it's unlocked, - however, that's a race condition. The only reason the up operation - here will fail is if the underlying semaphore doesn't exist anymore - or there is a locking logic error somewhere else. Either way, - declaring the EM unlocked here is OK. Same with all similar assignments. - */ - emLocked = false; - fMST.releaseTable_write(MasterSegmentTable::EMTable); - } + if (op == READ) + fMST.releaseTable_read(MasterSegmentTable::EMTable); + else + { + /* + Note: Technically we should mark it unlocked after it's unlocked, + however, that's a race condition. The only reason the up operation + here will fail is if the underlying semaphore doesn't exist anymore + or there is a locking logic error somewhere else. Either way, + declaring the EM unlocked here is OK. Same with all similar assignments. + */ + emLocked = false; + fMST.releaseTable_write(MasterSegmentTable::EMTable); + } } void ExtentMap::releaseFreeList(OPS op) { - if (op == READ) - fMST.releaseTable_read(MasterSegmentTable::EMFreeList); - else - { - flLocked = false; - fMST.releaseTable_write(MasterSegmentTable::EMFreeList); - } + if (op == READ) + fMST.releaseTable_read(MasterSegmentTable::EMFreeList); + else + { + flLocked = false; + fMST.releaseTable_write(MasterSegmentTable::EMFreeList); + } } key_t ExtentMap::chooseEMShmkey() { - int fixedKeys = 1; - key_t ret; + int fixedKeys = 1; + key_t ret; - if (fEMShminfo->tableShmkey + 1 == (key_t) (fShmKeys.KEYRANGE_EXTENTMAP_BASE + - fShmKeys.KEYRANGE_SIZE - 1) || (unsigned)fEMShminfo->tableShmkey < fShmKeys.KEYRANGE_EXTENTMAP_BASE) - ret = fShmKeys.KEYRANGE_EXTENTMAP_BASE + fixedKeys; - else - ret = fEMShminfo->tableShmkey + 1; + if (fEMShminfo->tableShmkey + 1 == (key_t)(fShmKeys.KEYRANGE_EXTENTMAP_BASE + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)fEMShminfo->tableShmkey < fShmKeys.KEYRANGE_EXTENTMAP_BASE) + ret = fShmKeys.KEYRANGE_EXTENTMAP_BASE + fixedKeys; + else + ret = fEMShminfo->tableShmkey + 1; - return ret; + return ret; } key_t ExtentMap::chooseFLShmkey() { - int fixedKeys = 1, ret; + int fixedKeys = 1, ret; - if (fFLShminfo->tableShmkey + 1 == (key_t) (fShmKeys.KEYRANGE_EMFREELIST_BASE + - fShmKeys.KEYRANGE_SIZE - 1) || (unsigned)fFLShminfo->tableShmkey < fShmKeys.KEYRANGE_EMFREELIST_BASE) - ret = fShmKeys.KEYRANGE_EMFREELIST_BASE + fixedKeys; - else - ret = fFLShminfo->tableShmkey + 1; + if (fFLShminfo->tableShmkey + 1 == + (key_t)(fShmKeys.KEYRANGE_EMFREELIST_BASE + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)fFLShminfo->tableShmkey < fShmKeys.KEYRANGE_EMFREELIST_BASE) + ret = fShmKeys.KEYRANGE_EMFREELIST_BASE + fixedKeys; + else + ret = fFLShminfo->tableShmkey + 1; - return ret; + return ret; } /* Must be called holding the EM write lock Returns with the new shmseg mapped */ void ExtentMap::growEMShmseg(size_t nrows) { - size_t allocSize; - key_t newshmkey; + size_t allocSize; + key_t newshmkey; - if (fEMShminfo->allocdSize == 0) - allocSize = EM_INITIAL_SIZE; - else - allocSize = fEMShminfo->allocdSize + EM_INCREMENT; + if (fEMShminfo->allocdSize == 0) + allocSize = EM_INITIAL_SIZE; + else + allocSize = fEMShminfo->allocdSize + EM_INCREMENT; - newshmkey = chooseEMShmkey(); - ASSERT((allocSize == EM_INITIAL_SIZE && !fPExtMapImpl) || fPExtMapImpl); + newshmkey = chooseEMShmkey(); + ASSERT((allocSize == EM_INITIAL_SIZE && !fPExtMapImpl) || fPExtMapImpl); - //Use the larger of the calculated value or the specified value - allocSize = max(allocSize, nrows * sizeof(EMEntry)); + // Use the larger of the calculated value or the specified value + allocSize = max(allocSize, nrows * sizeof(EMEntry)); - if (!fPExtMapImpl) - { - fPExtMapImpl = ExtentMapImpl::makeExtentMapImpl(newshmkey, allocSize, r_only); - } - else - { - fPExtMapImpl->grow(newshmkey, allocSize); - } + if (!fPExtMapImpl) + { + fPExtMapImpl = ExtentMapImpl::makeExtentMapImpl(newshmkey, allocSize, r_only); + } + else + { + fPExtMapImpl->grow(newshmkey, allocSize); + } - fEMShminfo->tableShmkey = newshmkey; - fEMShminfo->allocdSize = allocSize; + fEMShminfo->tableShmkey = newshmkey; + fEMShminfo->allocdSize = allocSize; - if (r_only) - fPExtMapImpl->makeReadOnly(); + if (r_only) + fPExtMapImpl->makeReadOnly(); - fExtentMap = fPExtMapImpl->get(); + fExtentMap = fPExtMapImpl->get(); } /* Must be called holding the FL lock Returns with the new shmseg mapped */ void ExtentMap::growFLShmseg() { - size_t allocSize; - key_t newshmkey; + size_t allocSize; + key_t newshmkey; - if (fFLShminfo->allocdSize == 0) - allocSize = EM_FREELIST_INITIAL_SIZE; - else - allocSize = fFLShminfo->allocdSize + EM_FREELIST_INCREMENT; + if (fFLShminfo->allocdSize == 0) + allocSize = EM_FREELIST_INITIAL_SIZE; + else + allocSize = fFLShminfo->allocdSize + EM_FREELIST_INCREMENT; - newshmkey = chooseFLShmkey(); - ASSERT((allocSize == EM_FREELIST_INITIAL_SIZE && !fPFreeListImpl) || fPFreeListImpl); + newshmkey = chooseFLShmkey(); + ASSERT((allocSize == EM_FREELIST_INITIAL_SIZE && !fPFreeListImpl) || fPFreeListImpl); - if (!fPFreeListImpl) - fPFreeListImpl = FreeListImpl::makeFreeListImpl(newshmkey, allocSize, false); - else - fPFreeListImpl->grow(newshmkey, allocSize); + if (!fPFreeListImpl) + fPFreeListImpl = FreeListImpl::makeFreeListImpl(newshmkey, allocSize, false); + else + fPFreeListImpl->grow(newshmkey, allocSize); - fFLShminfo->tableShmkey = newshmkey; - fFreeList = fPFreeListImpl->get(); + fFLShminfo->tableShmkey = newshmkey; + fFreeList = fPFreeListImpl->get(); - // init freelist entry - if (fFLShminfo->allocdSize == 0) - { - fFreeList->size = (1ULL << 36) / 1024; - fFLShminfo->currentSize = sizeof(InlineLBIDRange); - } + // init freelist entry + if (fFLShminfo->allocdSize == 0) + { + fFreeList->size = (1ULL << 36) / 1024; + fFLShminfo->currentSize = sizeof(InlineLBIDRange); + } - fFLShminfo->allocdSize = allocSize; + fFLShminfo->allocdSize = allocSize; - if (r_only) - fPFreeListImpl->makeReadOnly(); + if (r_only) + fPFreeListImpl->makeReadOnly(); - fFreeList = fPFreeListImpl->get(); + fFreeList = fPFreeListImpl->get(); } -// @bug 1509. Added new version of lookup that returns the first and last lbid for the extent that contains the -// given lbid. +// @bug 1509. Added new version of lookup that returns the first and last lbid for the extent that contains +// the given lbid. int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookup"); - TRACER_ADDINPUT(lbid); - TRACER_ADDOUTPUT(firstLbid); - TRACER_ADDOUTPUT(lastLbid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookup"); + TRACER_ADDINPUT(lbid); + TRACER_ADDOUTPUT(firstLbid); + TRACER_ADDOUTPUT(lastLbid); + TRACER_WRITE; + } #endif - int entries, i; - LBID_t lastBlock; + int entries, i; + LBID_t lastBlock; #ifdef BRM_DEBUG -//printEM(); - if (lbid < 0) - { - log("ExtentMap::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - cout << "ExtentMap::lookup(): lbid must be >= 0. Lbid passed was " << lbid << endl; - throw invalid_argument("ExtentMap::lookup(): lbid must be >= 0"); - } + // printEM(); + if (lbid < 0) + { + log("ExtentMap::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + cout << "ExtentMap::lookup(): lbid must be >= 0. Lbid passed was " << lbid << endl; + throw invalid_argument("ExtentMap::lookup(): lbid must be >= 0"); + } #endif - grabEMEntryTable(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(READ); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (i = 0; i < entries; i++) + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) - { - lastBlock = fExtentMap[i].range.start + - (static_cast(fExtentMap[i].range.size) * 1024) - 1; + lastBlock = fExtentMap[i].range.start + (static_cast(fExtentMap[i].range.size) * 1024) - 1; - if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) - { - firstLbid = fExtentMap[i].range.start; - lastLbid = lastBlock; - releaseEMEntryTable(READ); - return 0; - } - } + if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) + { + firstLbid = fExtentMap[i].range.start; + lastLbid = lastBlock; + releaseEMEntryTable(READ); + return 0; + } } + } - releaseEMEntryTable(READ); - return -1; + releaseEMEntryTable(READ); + return -1; } // @bug 1055+. New functions added for multiple files per OID enhancement. -int ExtentMap::lookupLocal(LBID_t lbid, int& OID, uint16_t& dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset) +int ExtentMap::lookupLocal(LBID_t lbid, int& OID, uint16_t& dbRoot, uint32_t& partitionNum, + uint16_t& segmentNum, uint32_t& fileBlockOffset) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocal"); - TRACER_ADDINPUT(lbid); - TRACER_ADDOUTPUT(OID); - TRACER_ADDSHORTOUTPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDOUTPUT(fileBlockOffset); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocal"); + TRACER_ADDINPUT(lbid); + TRACER_ADDOUTPUT(OID); + TRACER_ADDSHORTOUTPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDOUTPUT(fileBlockOffset); + TRACER_WRITE; + } #endif #ifdef EM_AS_A_TABLE_POC__ - if (lbid >= (1LL << 54)) - { - OID = 1084; - dbRoot = 1; - partitionNum = 0; - segmentNum = 0; - fileBlockOffset = 0; - return 0; - } + if (lbid >= (1LL << 54)) + { + OID = 1084; + dbRoot = 1; + partitionNum = 0; + segmentNum = 0; + fileBlockOffset = 0; + return 0; + } #endif - int entries, i, offset; - LBID_t lastBlock; + int entries, i, offset; + LBID_t lastBlock; - if (lbid < 0) + if (lbid < 0) + { + ostringstream oss; + oss << "ExtentMap::lookupLocal(): invalid lbid requested: " << lbid; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0) { - ostringstream oss; - oss << "ExtentMap::lookupLocal(): invalid lbid requested: " << lbid; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + lastBlock = fExtentMap[i].range.start + (static_cast(fExtentMap[i].range.size) * 1024) - 1; + + if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) + { + OID = fExtentMap[i].fileID; + dbRoot = fExtentMap[i].dbRoot; + segmentNum = fExtentMap[i].segmentNum; + partitionNum = fExtentMap[i].partitionNum; + + // TODO: Offset logic. + offset = lbid - fExtentMap[i].range.start; + fileBlockOffset = fExtentMap[i].blockOffset + offset; + + releaseEMEntryTable(READ); + return 0; + } } + } - grabEMEntryTable(READ); - - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < entries; i++) - { - if (fExtentMap[i].range.size != 0) - { - lastBlock = fExtentMap[i].range.start + - (static_cast(fExtentMap[i].range.size) * 1024) - 1; - - if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) - { - OID = fExtentMap[i].fileID; - dbRoot = fExtentMap[i].dbRoot; - segmentNum = fExtentMap[i].segmentNum; - partitionNum = fExtentMap[i].partitionNum; - - // TODO: Offset logic. - offset = lbid - fExtentMap[i].range.start; - fileBlockOffset = fExtentMap[i].blockOffset + offset; - - releaseEMEntryTable(READ); - return 0; - } - } - } - - releaseEMEntryTable(READ); - return -1; + releaseEMEntryTable(READ); + return -1; } -int ExtentMap::lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, LBID_t& LBID) +int ExtentMap::lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, + LBID_t& LBID) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocal"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(fileBlockOffset); - TRACER_ADDOUTPUT(LBID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocal"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(fileBlockOffset); + TRACER_ADDOUTPUT(LBID); + TRACER_WRITE; + } #endif - int entries, i, offset; + int entries, i, offset; - if (OID < 0) + if (OID < 0) + { + log("ExtentMap::lookup(): OID and FBO must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::lookup(): OID and FBO must be >= 0"); + } + + grabEMEntryTable(READ); + + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < entries; i++) + { + // TODO: Blockoffset logic. + if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && + fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) { - log("ExtentMap::lookup(): OID and FBO must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("ExtentMap::lookup(): OID and FBO must be >= 0"); + offset = fileBlockOffset - fExtentMap[i].blockOffset; + LBID = fExtentMap[i].range.start + offset; + releaseEMEntryTable(READ); + return 0; } + } - grabEMEntryTable(READ); - - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < entries; i++) - { - - // TODO: Blockoffset logic. - if (fExtentMap[i].range.size != 0 && - fExtentMap[i].fileID == OID && - fExtentMap[i].partitionNum == partitionNum && - fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= (fExtentMap[i].blockOffset + - (static_cast(fExtentMap[i].range.size) * 1024) - 1)) - { - - offset = fileBlockOffset - fExtentMap[i].blockOffset; - LBID = fExtentMap[i].range.start + offset; - releaseEMEntryTable(READ); - return 0; - } - } - - releaseEMEntryTable(READ); - return -1; + releaseEMEntryTable(READ); + return -1; } int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNum, uint16_t segmentNum, @@ -2215,53 +2147,48 @@ int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNu { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocal"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(fileBlockOffset); - TRACER_ADDOUTPUT(LBID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocal"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(fileBlockOffset); + TRACER_ADDOUTPUT(LBID); + TRACER_WRITE; + } #endif - int entries, i, offset; + int entries, i, offset; - if (OID < 0) + if (OID < 0) + { + log("ExtentMap::lookup(): OID and FBO must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::lookup(): OID and FBO must be >= 0"); + } + + grabEMEntryTable(READ); + + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < entries; i++) + { + // TODO: Blockoffset logic. + if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && fExtentMap[i].dbRoot == dbroot && + fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) { - log("ExtentMap::lookup(): OID and FBO must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("ExtentMap::lookup(): OID and FBO must be >= 0"); + offset = fileBlockOffset - fExtentMap[i].blockOffset; + LBID = fExtentMap[i].range.start + offset; + releaseEMEntryTable(READ); + return 0; } + } - grabEMEntryTable(READ); - - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < entries; i++) - { - - // TODO: Blockoffset logic. - if (fExtentMap[i].range.size != 0 && - fExtentMap[i].fileID == OID && - fExtentMap[i].dbRoot == dbroot && - fExtentMap[i].partitionNum == partitionNum && - fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= (fExtentMap[i].blockOffset + - (static_cast(fExtentMap[i].range.size) * 1024) - 1)) - { - - offset = fileBlockOffset - fExtentMap[i].blockOffset; - LBID = fExtentMap[i].range.start + offset; - releaseEMEntryTable(READ); - return 0; - } - } - - releaseEMEntryTable(READ); - return -1; + releaseEMEntryTable(READ); + return -1; } // @bug 1055-. @@ -2270,58 +2197,53 @@ int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNu // Lookup/return starting LBID for the specified OID, partition, segment, and // file block offset. //------------------------------------------------------------------------------ -int ExtentMap::lookupLocalStartLbid(int OID, - uint32_t partitionNum, - uint16_t segmentNum, - uint32_t fileBlockOffset, - LBID_t& LBID) +int ExtentMap::lookupLocalStartLbid(int OID, uint32_t partitionNum, uint16_t segmentNum, + uint32_t fileBlockOffset, LBID_t& LBID) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookupLocalStartLbid"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(fileBlockOffset); - TRACER_ADDOUTPUT(LBID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookupLocalStartLbid"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(fileBlockOffset); + TRACER_ADDOUTPUT(LBID); + TRACER_WRITE; + } #endif - int entries, i; + int entries, i; - if (OID < 0) + if (OID < 0) + { + log("ExtentMap::lookupLocalStartLbid(): OID and FBO must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument( + "ExtentMap::lookupLocalStartLbid(): " + "OID and FBO must be >= 0"); + } + + grabEMEntryTable(READ); + entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < entries; i++) + { + if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && + fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) { - log("ExtentMap::lookupLocalStartLbid(): OID and FBO must be >= 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument("ExtentMap::lookupLocalStartLbid(): " - "OID and FBO must be >= 0"); + LBID = fExtentMap[i].range.start; + releaseEMEntryTable(READ); + return 0; } + } - grabEMEntryTable(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + releaseEMEntryTable(READ); - for (i = 0; i < entries; i++) - { - if (fExtentMap[i].range.size != 0 && - fExtentMap[i].fileID == OID && - fExtentMap[i].partitionNum == partitionNum && - fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= (fExtentMap[i].blockOffset + - (static_cast(fExtentMap[i].range.size) * 1024) - 1)) - { - LBID = fExtentMap[i].range.start; - releaseEMEntryTable(READ); - return 0; - } - } - - releaseEMEntryTable(READ); - - return -1; + return -1; } //------------------------------------------------------------------------------ @@ -2339,70 +2261,53 @@ int ExtentMap::lookupLocalStartLbid(int OID, // segmentNum - Segment number for new exents // extents - starting Lbid, numBlocks, and FBO for new extents //------------------------------------------------------------------------------ -void ExtentMap::createStripeColumnExtents( - const vector& cols, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - vector& extents) +void ExtentMap::createStripeColumnExtents(const vector& cols, uint16_t dbRoot, + uint32_t& partitionNum, uint16_t& segmentNum, + vector& extents) { - LBID_t startLbid; - int allocSize; - uint32_t startBlkOffset; + LBID_t startLbid; + int allocSize; + uint32_t startBlkOffset; - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - OID_t baselineOID = -1; - uint16_t baselineSegmentNum = -1; - uint32_t baselinePartNum = -1; + OID_t baselineOID = -1; + uint16_t baselineSegmentNum = -1; + uint32_t baselinePartNum = -1; - for (uint32_t i = 0; i < cols.size(); i++) + for (uint32_t i = 0; i < cols.size(); i++) + { + createColumnExtent_DBroot(cols[i].oid, cols[i].width, dbRoot, cols[i].colDataType, partitionNum, + segmentNum, startLbid, allocSize, startBlkOffset, false); + + if (i == 0) { - createColumnExtent_DBroot( - cols[i].oid, - cols[i].width, - dbRoot, - cols[i].colDataType, - partitionNum, - segmentNum, - startLbid, - allocSize, - startBlkOffset, - false); - - if (i == 0) - { - baselineOID = cols[i].oid; - baselineSegmentNum = segmentNum; - baselinePartNum = partitionNum; - } - else - { - if ((segmentNum != baselineSegmentNum) || - (partitionNum != baselinePartNum)) - { - ostringstream oss; - oss << "ExtentMap::createStripeColumnExtents(): " - "Inconsistent segment extent creation: " << - "DBRoot: " << dbRoot << - "OID1: " << baselineOID << - "; Part#: " << baselinePartNum << - "; Seg#: " << baselineSegmentNum << - " OID2: " << cols[i].oid << - "; Part#: " << partitionNum << - "; Seg#: " << segmentNum; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } - } - - CreateStripeColumnExtentsArgOut extentInfo; - extentInfo.startLbid = startLbid; - extentInfo.allocSize = allocSize; - extentInfo.startBlkOffset = startBlkOffset; - extents.push_back( extentInfo ); + baselineOID = cols[i].oid; + baselineSegmentNum = segmentNum; + baselinePartNum = partitionNum; } + else + { + if ((segmentNum != baselineSegmentNum) || (partitionNum != baselinePartNum)) + { + ostringstream oss; + oss << "ExtentMap::createStripeColumnExtents(): " + "Inconsistent segment extent creation: " + << "DBRoot: " << dbRoot << "OID1: " << baselineOID << "; Part#: " << baselinePartNum + << "; Seg#: " << baselineSegmentNum << " OID2: " << cols[i].oid + << "; Part#: " << partitionNum << "; Seg#: " << segmentNum; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + } + + CreateStripeColumnExtentsArgOut extentInfo; + extentInfo.startLbid = startLbid; + extentInfo.allocSize = allocSize; + extentInfo.startBlkOffset = startBlkOffset; + extents.push_back(extentInfo); + } } //------------------------------------------------------------------------------ @@ -2425,70 +2330,63 @@ void ExtentMap::createStripeColumnExtents( // allocdsize - number of LBIDs allocated // startBlockOffset-starting block of the created extent //------------------------------------------------------------------------------ -void ExtentMap::createColumnExtent_DBroot(int OID, - uint32_t colWidth, - uint16_t dbRoot, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& partitionNum, - uint16_t& segmentNum, - LBID_t& lbid, - int& allocdsize, - uint32_t& startBlockOffset, - bool useLock) // defaults to true +void ExtentMap::createColumnExtent_DBroot(int OID, uint32_t colWidth, uint16_t dbRoot, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& partitionNum, uint16_t& segmentNum, LBID_t& lbid, + int& allocdsize, uint32_t& startBlockOffset, + bool useLock) // defaults to true { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createColumnExtent_DBroot"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(colWidth); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDINT64OUTPUT(lbid); - TRACER_ADDOUTPUT(allocdsize); - TRACER_ADDOUTPUT(startBlockOffset); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createColumnExtent_DBroot"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(colWidth); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDINT64OUTPUT(lbid); + TRACER_ADDOUTPUT(allocdsize); + TRACER_ADDOUTPUT(startBlockOffset); + TRACER_WRITE; + } #endif #ifdef BRM_DEBUG - if (OID <= 0) - { - log("ExtentMap::createColumnExtent_DBroot(): OID must be > 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument( - "ExtentMap::createColumnExtent_DBroot(): OID must be > 0"); - } + if (OID <= 0) + { + log("ExtentMap::createColumnExtent_DBroot(): OID must be > 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::createColumnExtent_DBroot(): OID must be > 0"); + } #endif - // Convert extent size in rows to extent size in 8192-byte blocks. - // extentRows should be multiple of blocksize (8192). - const unsigned EXTENT_SIZE = (getExtentRows() * colWidth) / BLOCK_SIZE; + // Convert extent size in rows to extent size in 8192-byte blocks. + // extentRows should be multiple of blocksize (8192). + const unsigned EXTENT_SIZE = (getExtentRows() * colWidth) / BLOCK_SIZE; - if (useLock) - { - grabEMEntryTable(WRITE); - grabFreeList(WRITE); - } + if (useLock) + { + grabEMEntryTable(WRITE); + grabFreeList(WRITE); + } - if (fEMShminfo->currentSize == fEMShminfo->allocdSize) - growEMShmseg(); + if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + growEMShmseg(); -// size is the number of multiples of 1024 blocks. -// ex: size=1 --> 1024 blocks -// size=2 --> 2048 blocks -// size=3 --> 3072 blocks, etc. - uint32_t size = EXTENT_SIZE / 1024; + // size is the number of multiples of 1024 blocks. + // ex: size=1 --> 1024 blocks + // size=2 --> 2048 blocks + // size=3 --> 3072 blocks, etc. + uint32_t size = EXTENT_SIZE / 1024; - lbid = _createColumnExtent_DBroot(size, OID, colWidth, - dbRoot, colDataType, partitionNum, segmentNum, startBlockOffset); + lbid = _createColumnExtent_DBroot(size, OID, colWidth, dbRoot, colDataType, partitionNum, segmentNum, + startBlockOffset); - allocdsize = EXTENT_SIZE; + allocdsize = EXTENT_SIZE; } //------------------------------------------------------------------------------ @@ -2511,459 +2409,436 @@ void ExtentMap::createColumnExtent_DBroot(int OID, // startBlockOffset-starting block of the created extent // returns starting LBID of the created extent. //------------------------------------------------------------------------------ -LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, - uint32_t colWidth, - uint16_t dbRoot, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& partitionNum, - uint16_t& segmentNum, - uint32_t& startBlockOffset) +LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& partitionNum, uint16_t& segmentNum, + uint32_t& startBlockOffset) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - uint32_t highestPartNum = 0; - uint16_t highestSegNum = 0; - const unsigned FILES_PER_COL_PART = getFilesPerColumnPartition(); - const unsigned EXTENT_ROWS = getExtentRows(); - const unsigned EXTENTS_PER_SEGFILE = getExtentsPerSegmentFile(); - const unsigned DBROOT_COUNT = getDbRootCount(); + int emptyEMEntry = -1; + int lastExtentIndex = -1; + uint32_t highestOffset = 0; + uint32_t highestPartNum = 0; + uint16_t highestSegNum = 0; + const unsigned FILES_PER_COL_PART = getFilesPerColumnPartition(); + const unsigned EXTENT_ROWS = getExtentRows(); + const unsigned EXTENTS_PER_SEGFILE = getExtentsPerSegmentFile(); + const unsigned DBROOT_COUNT = getDbRootCount(); - // Variables that track list of segfiles in target (HWM) DBRoot & partition. - // Map segment number to the highest fbo extent in each file - typedef tr1::unordered_map TargetDbRootSegsMap; - typedef TargetDbRootSegsMap::iterator TargetDbRootSegsMapIter; - typedef TargetDbRootSegsMap::const_iterator TargetDbRootSegsMapConstIter; - TargetDbRootSegsMap targetDbRootSegs; + // Variables that track list of segfiles in target (HWM) DBRoot & partition. + // Map segment number to the highest fbo extent in each file + typedef tr1::unordered_map TargetDbRootSegsMap; + typedef TargetDbRootSegsMap::iterator TargetDbRootSegsMapIter; + typedef TargetDbRootSegsMap::const_iterator TargetDbRootSegsMapConstIter; + TargetDbRootSegsMap targetDbRootSegs; - uint32_t highEmptySegNum = 0; // high seg num for user specified partition; - // only comes into play for empty DBRoot. - bool bHighEmptySegNumSet = false; + uint32_t highEmptySegNum = 0; // high seg num for user specified partition; + // only comes into play for empty DBRoot. + bool bHighEmptySegNumSet = false; - //-------------------------------------------------------------------------- - // First Step: Scan ExtentMap - // 1. find HWM extent in relevant DBRoot - // 2. if DBRoot is empty, track highest seg num in user specified partition - // 3. Find first unused extent map entry - //-------------------------------------------------------------------------- - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + //-------------------------------------------------------------------------- + // First Step: Scan ExtentMap + // 1. find HWM extent in relevant DBRoot + // 2. if DBRoot is empty, track highest seg num in user specified partition + // 3. Find first unused extent map entry + //-------------------------------------------------------------------------- + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList( size ); + LBID_t startLBID = getLBIDsFromFreeList(size); + + // Find the first empty Entry; and find last extent for this OID and dbRoot + for (int i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) + { + if (fExtentMap[i].fileID == OID) + { + // 1. Find HWM extent in relevant DBRoot + if (fExtentMap[i].dbRoot == dbRoot) + { + if ((fExtentMap[i].partitionNum > highestPartNum) || + ((fExtentMap[i].partitionNum == highestPartNum) && + (fExtentMap[i].blockOffset > highestOffset)) || + ((fExtentMap[i].partitionNum == highestPartNum) && + (fExtentMap[i].blockOffset == highestOffset) && (fExtentMap[i].segmentNum >= highestSegNum))) + { + lastExtentIndex = i; + highestPartNum = fExtentMap[i].partitionNum; + highestSegNum = fExtentMap[i].segmentNum; + highestOffset = fExtentMap[i].blockOffset; + } + } + + // 2. for empty DBRoot track hi seg# in user specified part# + if ((lastExtentIndex == -1) && (fExtentMap[i].partitionNum == partitionNum)) + { + if ((fExtentMap[i].segmentNum > highEmptySegNum) || (!bHighEmptySegNumSet)) + { + highEmptySegNum = fExtentMap[i].segmentNum; + bHighEmptySegNumSet = true; + } + } + } // found extentmap entry for specified OID + } // found valid extentmap entry + + // 3. Find first available extent map entry that can be reused + else if (emptyEMEntry < 0) + emptyEMEntry = i; + } // Loop through extent map entries + + if (emptyEMEntry == -1) + { + ostringstream oss; + oss << "ExtentMap::_createColumnExtent_DBroot(): " + "could not find an empty EMEntry for OID " + << OID << "; Extent Map is full", + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw logic_error(oss.str()); + } + + //-------------------------------------------------------------------------- + // If DBRoot is not empty, then... + // Second Step: Scan ExtentMap again after I know the last partition + // 4. track highest seg num for HWM+1 partition + // 5. track highest seg num for HWM partition + // 6. save list of segment numbers and fbos in target DBRoot and partition + // + // Scanning the extentmap a second time is not a good thing to be doing. + // But the alternative isn't good either. There is certain information + // I need to capture about the last partition and DBRoot, and for the next + // partition as well (which may contain segment files on other DBRoots), + // but until I scan the extentmap, I don't know what my last partition is. + // If I try to do this in a single scan, then I am forced to spend time + // capturing information about partitions that turn out to be inconse- + // quential because the "known" last partition will keep changing as I + // scan the extentmap. + //-------------------------------------------------------------------------- + bool bSegsOutOfService = false; + int partHighSeg = -1; // hi seg num for last partition + int partHighSegNext = -1; // hi seg num for next partition + + if (lastExtentIndex >= 0) + { + uint32_t targetDbRootPart = fExtentMap[lastExtentIndex].partitionNum; + uint32_t targetDbRootPartNext = targetDbRootPart + 1; + partHighSeg = fExtentMap[lastExtentIndex].segmentNum; + targetDbRootSegs.insert(TargetDbRootSegsMap::value_type(fExtentMap[lastExtentIndex].segmentNum, + fExtentMap[lastExtentIndex].blockOffset)); - // Find the first empty Entry; and find last extent for this OID and dbRoot for (int i = 0; i < emEntries; i++) { - if (fExtentMap[i].range.size != 0) + if (fExtentMap[i].range.size != 0) + { + if (fExtentMap[i].fileID == OID) { - if (fExtentMap[i].fileID == OID) + // 4. Track hi seg for hwm+1 partition + if (fExtentMap[i].partitionNum == targetDbRootPartNext) + { + if (fExtentMap[i].segmentNum > partHighSegNext) { + partHighSegNext = fExtentMap[i].segmentNum; + } + } - // 1. Find HWM extent in relevant DBRoot - if (fExtentMap[i].dbRoot == dbRoot) + // 5. Track hi seg for hwm partition + else if (fExtentMap[i].partitionNum == targetDbRootPart) + { + if (fExtentMap[i].segmentNum > partHighSeg) + { + partHighSeg = fExtentMap[i].segmentNum; + } + + // 6. Save list of seg files in target DBRoot/Partition, + // along with the highest fbo for each seg file + if (fExtentMap[i].dbRoot == dbRoot) + { + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + bSegsOutOfService = true; + + TargetDbRootSegsMapIter iter = targetDbRootSegs.find(fExtentMap[i].segmentNum); + + if (iter == targetDbRootSegs.end()) + { + targetDbRootSegs.insert( + TargetDbRootSegsMap::value_type(fExtentMap[i].segmentNum, fExtentMap[i].blockOffset)); + } + else + { + if (fExtentMap[i].blockOffset > iter->second) { - if ( (fExtentMap[i].partitionNum > highestPartNum) || - ((fExtentMap[i].partitionNum == highestPartNum) && - (fExtentMap[i].blockOffset > highestOffset)) || - ((fExtentMap[i].partitionNum == highestPartNum) && - (fExtentMap[i].blockOffset == highestOffset) && - (fExtentMap[i].segmentNum >= highestSegNum)) ) - { - - lastExtentIndex = i; - highestPartNum = fExtentMap[i].partitionNum; - highestSegNum = fExtentMap[i].segmentNum; - highestOffset = fExtentMap[i].blockOffset; - } + iter->second = fExtentMap[i].blockOffset; } + } + } + } + } // found extentmap entry for specified OID + } // found valid extentmap entry + } // loop through extent map entries + } // (lastExtentIndex >= 0) - // 2. for empty DBRoot track hi seg# in user specified part# - if ((lastExtentIndex == -1) && - (fExtentMap[i].partitionNum == partitionNum)) - { - if ((fExtentMap[i].segmentNum > highEmptySegNum) || - (!bHighEmptySegNumSet)) - { - highEmptySegNum = fExtentMap[i].segmentNum; - bHighEmptySegNumSet = true; - } - } - } // found extentmap entry for specified OID - } // found valid extentmap entry + //-------------------------------------------------------------------------- + // Third Step: Select partition and segment number for new extent + // 1. Loop through targetDbRootSegs to find segment file for next extent + // 2. Check for exceptions that warrant going to next physical partition + // a. See if any extents are marked outOfService + // b. See if extents are not evenly layered as expected + // 3. Perform additional new partition/segment logic as applicable + // a. No action taken if 2a or 2b already detected need for new partition + // b. If HWM extent is in last file of DBRoot/Partition, see if next + // extent goes in new partition, or if wrap-around within current + // partition. + // c. If extent needs to go in next partition, figure out the next + // partition and the next available segment in that partition. + // 4. Set blockOffset of new extent based on where extent is being added + //-------------------------------------------------------------------------- + uint16_t newDbRoot = dbRoot; + uint32_t newPartitionNum = partitionNum; + uint16_t newSegmentNum = 0; + uint32_t newBlockOffset = 0; - // 3. Find first available extent map entry that can be reused - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries + // If this is not the first extent for this OID and DBRoot then + // extrapolate part# and seg# from last extent; wrap around segment and + // partition number as needed. + // else + // use part# that the user specifies + if (lastExtentIndex >= 0) + { + bool startNewPartition = false; + bool startNewStripeInSegFile = false; + const unsigned int filesPerDBRootPerPartition = FILES_PER_COL_PART / DBROOT_COUNT; - if (emptyEMEntry == -1) + int& lastExtIdx = lastExtentIndex; + + // Find first, last, next seg files in target partition and DBRoot + uint16_t firstTargetSeg = fExtentMap[lastExtIdx].segmentNum; + uint16_t lastTargetSeg = fExtentMap[lastExtIdx].segmentNum; + uint16_t nextTargetSeg = fExtentMap[lastExtIdx].segmentNum; + + // 1. Loop thru targetDbRootSegs[] to find next segment after + // lastExtIdx in target list. + // We save low and high segment to use in wrap-around case. + if (targetDbRootSegs.size() > 1) { - ostringstream oss; - oss << "ExtentMap::_createColumnExtent_DBroot(): " - "could not find an empty EMEntry for OID " << OID << - "; Extent Map is full", - log(oss.str(), - logging::LOG_TYPE_CRITICAL); - throw logic_error( oss.str() ); + bool bNextSegSet = false; + + for (TargetDbRootSegsMapConstIter iter = targetDbRootSegs.begin(); iter != targetDbRootSegs.end(); + ++iter) + { + uint16_t targetSeg = iter->first; + + if (targetSeg < firstTargetSeg) + firstTargetSeg = targetSeg; + else if (targetSeg > lastTargetSeg) + lastTargetSeg = targetSeg; + + if (targetSeg > fExtentMap[lastExtIdx].segmentNum) + { + if ((targetSeg < nextTargetSeg) || (!bNextSegSet)) + { + nextTargetSeg = targetSeg; + bNextSegSet = true; + } + } + } } - //-------------------------------------------------------------------------- - // If DBRoot is not empty, then... - // Second Step: Scan ExtentMap again after I know the last partition - // 4. track highest seg num for HWM+1 partition - // 5. track highest seg num for HWM partition - // 6. save list of segment numbers and fbos in target DBRoot and partition - // - // Scanning the extentmap a second time is not a good thing to be doing. - // But the alternative isn't good either. There is certain information - // I need to capture about the last partition and DBRoot, and for the next - // partition as well (which may contain segment files on other DBRoots), - // but until I scan the extentmap, I don't know what my last partition is. - // If I try to do this in a single scan, then I am forced to spend time - // capturing information about partitions that turn out to be inconse- - // quential because the "known" last partition will keep changing as I - // scan the extentmap. - //-------------------------------------------------------------------------- - bool bSegsOutOfService = false; - int partHighSeg = -1; // hi seg num for last partition - int partHighSegNext = -1; // hi seg num for next partition + newPartitionNum = fExtentMap[lastExtIdx].partitionNum; - if (lastExtentIndex >= 0) + // 2a. Skip to next physical partition if any extents in HWM partition/ + // DBRoot are marked as outOfService + if (bSegsOutOfService) { - uint32_t targetDbRootPart = fExtentMap[lastExtentIndex].partitionNum; - uint32_t targetDbRootPartNext = targetDbRootPart + 1; - partHighSeg = fExtentMap[lastExtentIndex].segmentNum; - targetDbRootSegs.insert( TargetDbRootSegsMap::value_type( - fExtentMap[lastExtentIndex].segmentNum, - fExtentMap[lastExtentIndex].blockOffset) ); + // cout << "Skipping to next partition (outOfService segs)" << + // ": oid-" << fExtentMap[lastExtentIndex].fileID << + // "; root-" << fExtentMap[lastExtentIndex].dbRoot << + // "; part-" << fExtentMap[lastExtentIndex].partitionNum << endl; - for (int i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - if (fExtentMap[i].fileID == OID) - { - - // 4. Track hi seg for hwm+1 partition - if (fExtentMap[i].partitionNum == targetDbRootPartNext) - { - if (fExtentMap[i].segmentNum > partHighSegNext) - { - partHighSegNext = fExtentMap[i].segmentNum; - } - } - - // 5. Track hi seg for hwm partition - else if (fExtentMap[i].partitionNum == targetDbRootPart) - { - if (fExtentMap[i].segmentNum > partHighSeg) - { - partHighSeg = fExtentMap[i].segmentNum; - } - - // 6. Save list of seg files in target DBRoot/Partition, - // along with the highest fbo for each seg file - if (fExtentMap[i].dbRoot == dbRoot) - { - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - bSegsOutOfService = true; - - TargetDbRootSegsMapIter iter = - targetDbRootSegs.find(fExtentMap[i].segmentNum); - - if (iter == targetDbRootSegs.end()) - { - targetDbRootSegs.insert( - TargetDbRootSegsMap::value_type( - fExtentMap[i].segmentNum, - fExtentMap[i].blockOffset) ); - } - else - { - if (fExtentMap[i].blockOffset > iter->second) - { - iter->second = fExtentMap[i].blockOffset; - } - } - } - } - } // found extentmap entry for specified OID - } // found valid extentmap entry - } // loop through extent map entries - } // (lastExtentIndex >= 0) - - //-------------------------------------------------------------------------- - // Third Step: Select partition and segment number for new extent - // 1. Loop through targetDbRootSegs to find segment file for next extent - // 2. Check for exceptions that warrant going to next physical partition - // a. See if any extents are marked outOfService - // b. See if extents are not evenly layered as expected - // 3. Perform additional new partition/segment logic as applicable - // a. No action taken if 2a or 2b already detected need for new partition - // b. If HWM extent is in last file of DBRoot/Partition, see if next - // extent goes in new partition, or if wrap-around within current - // partition. - // c. If extent needs to go in next partition, figure out the next - // partition and the next available segment in that partition. - // 4. Set blockOffset of new extent based on where extent is being added - //-------------------------------------------------------------------------- - uint16_t newDbRoot = dbRoot; - uint32_t newPartitionNum = partitionNum; - uint16_t newSegmentNum = 0; - uint32_t newBlockOffset = 0; - - // If this is not the first extent for this OID and DBRoot then - // extrapolate part# and seg# from last extent; wrap around segment and - // partition number as needed. - // else - // use part# that the user specifies - if (lastExtentIndex >= 0) - { - bool startNewPartition = false; - bool startNewStripeInSegFile = false; - const unsigned int filesPerDBRootPerPartition = - FILES_PER_COL_PART / DBROOT_COUNT; - - int& lastExtIdx = lastExtentIndex; - - // Find first, last, next seg files in target partition and DBRoot - uint16_t firstTargetSeg = fExtentMap[lastExtIdx].segmentNum; - uint16_t lastTargetSeg = fExtentMap[lastExtIdx].segmentNum; - uint16_t nextTargetSeg = fExtentMap[lastExtIdx].segmentNum; - - // 1. Loop thru targetDbRootSegs[] to find next segment after - // lastExtIdx in target list. - // We save low and high segment to use in wrap-around case. - if (targetDbRootSegs.size() > 1) - { - bool bNextSegSet = false; - - for (TargetDbRootSegsMapConstIter iter = targetDbRootSegs.begin(); - iter != targetDbRootSegs.end(); - ++iter) - { - uint16_t targetSeg = iter->first; - - if (targetSeg < firstTargetSeg) - firstTargetSeg = targetSeg; - else if (targetSeg > lastTargetSeg) - lastTargetSeg = targetSeg; - - if (targetSeg > fExtentMap[lastExtIdx].segmentNum) - { - if ((targetSeg < nextTargetSeg) || (!bNextSegSet)) - { - nextTargetSeg = targetSeg; - bNextSegSet = true; - } - } - } - } - - newPartitionNum = fExtentMap[lastExtIdx].partitionNum; - - // 2a. Skip to next physical partition if any extents in HWM partition/ - // DBRoot are marked as outOfService - if (bSegsOutOfService) - { - -// cout << "Skipping to next partition (outOfService segs)" << -// ": oid-" << fExtentMap[lastExtentIndex].fileID << -// "; root-" << fExtentMap[lastExtentIndex].dbRoot << -// "; part-" << fExtentMap[lastExtentIndex].partitionNum << endl; - - startNewPartition = true; - } - - // @bug 4765 - // 2b. Skip to next physical partition if we have a set of - // segment files that are not "layered" as expected, meaning we - // have > 1 layer of extents with an incomplete lower layer (could - // be caused by the dropping of logical partitions). - else if (targetDbRootSegs.size() < filesPerDBRootPerPartition) - { - for (TargetDbRootSegsMapConstIter iter = targetDbRootSegs.begin(); - iter != targetDbRootSegs.end(); - ++iter) - { - if (iter->second > 0) - { - -// cout << "Skipping to next partition (unbalanced)" << -// ": oid-" << fExtentMap[lastExtentIndex].fileID << -// "; root-" << fExtentMap[lastExtentIndex].dbRoot << -// "; part-" << fExtentMap[lastExtentIndex].partitionNum << -// "; seg-" << iter->first << -// "; hifbo-"<< iter->second << endl; - - startNewPartition = true; - break; - } - } - } - - // 3a.If we already detected need for new partition, then take no action - if (startNewPartition) - { - // no action taken here; we take additional action later. - } - - // 3b.If HWM extent is in last seg file for this partition and DBRoot, - // find out if we need to add a new partition for next extent. - else if (targetDbRootSegs.size() >= filesPerDBRootPerPartition) - { - if (fExtentMap[lastExtIdx].segmentNum == lastTargetSeg) - { - // Use blockOffset of lastExtIdx to see if we need to add - // the next extent to a new partition. - if (fExtentMap[lastExtIdx].blockOffset == - ((EXTENTS_PER_SEGFILE - 1) * - (EXTENT_ROWS * colWidth / BLOCK_SIZE)) ) - { - startNewPartition = true; - } - else // Wrap-around; add extent to low seg in this partition - { - startNewStripeInSegFile = true; - newSegmentNum = firstTargetSeg; - } - } - else - { - newSegmentNum = nextTargetSeg; - } - } - else // Select next segment file in current HWM partition - { - newSegmentNum = partHighSeg + 1; - } - - // 3c. Find new partition and segment if we can't create - // an extent for this DBRoot in the current HWM partition. - if (startNewPartition) - { - newPartitionNum++; - - if (partHighSegNext == -1) - newSegmentNum = 0; - else - newSegmentNum = partHighSegNext + 1; - } - - // 4. Set blockOffset (fbo) for new extent relative to it's seg file - // case1: Init fbo to 0 if first extent in partition/DbRoot - // case2: Init fbo to 0 if first extent in segment file (other than - // first segment in this partition/DbRoot, which case1 handled) - // case3: Init fbo based on previous extent - - // case1: leave newBlockOffset set to 0 - if (startNewPartition) - { - //...no action necessary - } - - // case2: leave newBlockOffset set to 0 - else if ((fExtentMap[lastExtIdx].blockOffset == 0) && - (newSegmentNum > firstTargetSeg)) - { - //...no action necessary - } - - // case3: Init blockOffset based on previous extent. If we are adding - // extent to 1st seg file, then need to bump up the offset; else - // adding extent to same stripe and can repeat the same offset. - else - { - if (startNewStripeInSegFile) // start next stripe - { - newBlockOffset = static_cast - (fExtentMap[lastExtIdx].range.size) * 1024 + - fExtentMap[lastExtIdx].blockOffset; - } - else // next extent, same stripe - { - newBlockOffset = fExtentMap[lastExtIdx].blockOffset; - } - } - } // lastExtentIndex >= 0 - else // Empty DBRoot; use part# that the user specifies - { - if (bHighEmptySegNumSet) - newSegmentNum = highEmptySegNum + 1; - else - newSegmentNum = 0; + startNewPartition = true; } - //-------------------------------------------------------------------------- - // Fourth Step: Construct the new extentmap entry - //-------------------------------------------------------------------------- - - makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); - EMEntry* e = &fExtentMap[emptyEMEntry]; - - e->range.start = startLBID; - e->range.size = size; - e->fileID = OID; - - if (isUnsigned(colDataType)) + // @bug 4765 + // 2b. Skip to next physical partition if we have a set of + // segment files that are not "layered" as expected, meaning we + // have > 1 layer of extents with an incomplete lower layer (could + // be caused by the dropping of logical partitions). + else if (targetDbRootSegs.size() < filesPerDBRootPerPartition) { - if (colWidth != datatypes::MAXDECIMALWIDTH) + for (TargetDbRootSegsMapConstIter iter = targetDbRootSegs.begin(); iter != targetDbRootSegs.end(); + ++iter) + { + if (iter->second > 0) { - e->partition.cprange.loVal = numeric_limits::max(); - e->partition.cprange.hiVal = 0; + // cout << "Skipping to next partition (unbalanced)" << + // ": oid-" << fExtentMap[lastExtentIndex].fileID << + // "; root-" << fExtentMap[lastExtentIndex].dbRoot << + // "; part-" << fExtentMap[lastExtentIndex].partitionNum + //<< + // "; seg-" << iter->first << + // "; hifbo-"<< iter->second << endl; + + startNewPartition = true; + break; } - else + } + } + + // 3a.If we already detected need for new partition, then take no action + if (startNewPartition) + { + // no action taken here; we take additional action later. + } + + // 3b.If HWM extent is in last seg file for this partition and DBRoot, + // find out if we need to add a new partition for next extent. + else if (targetDbRootSegs.size() >= filesPerDBRootPerPartition) + { + if (fExtentMap[lastExtIdx].segmentNum == lastTargetSeg) + { + // Use blockOffset of lastExtIdx to see if we need to add + // the next extent to a new partition. + if (fExtentMap[lastExtIdx].blockOffset == + ((EXTENTS_PER_SEGFILE - 1) * (EXTENT_ROWS * colWidth / BLOCK_SIZE))) { - e->partition.cprange.bigLoVal = -1; - e->partition.cprange.bigHiVal = 0; + startNewPartition = true; } + else // Wrap-around; add extent to low seg in this partition + { + startNewStripeInSegFile = true; + newSegmentNum = firstTargetSeg; + } + } + else + { + newSegmentNum = nextTargetSeg; + } + } + else // Select next segment file in current HWM partition + { + newSegmentNum = partHighSeg + 1; + } + + // 3c. Find new partition and segment if we can't create + // an extent for this DBRoot in the current HWM partition. + if (startNewPartition) + { + newPartitionNum++; + + if (partHighSegNext == -1) + newSegmentNum = 0; + else + newSegmentNum = partHighSegNext + 1; + } + + // 4. Set blockOffset (fbo) for new extent relative to it's seg file + // case1: Init fbo to 0 if first extent in partition/DbRoot + // case2: Init fbo to 0 if first extent in segment file (other than + // first segment in this partition/DbRoot, which case1 handled) + // case3: Init fbo based on previous extent + + // case1: leave newBlockOffset set to 0 + if (startNewPartition) + { + //...no action necessary + } + + // case2: leave newBlockOffset set to 0 + else if ((fExtentMap[lastExtIdx].blockOffset == 0) && (newSegmentNum > firstTargetSeg)) + { + //...no action necessary + } + + // case3: Init blockOffset based on previous extent. If we are adding + // extent to 1st seg file, then need to bump up the offset; else + // adding extent to same stripe and can repeat the same offset. + else + { + if (startNewStripeInSegFile) // start next stripe + { + newBlockOffset = static_cast(fExtentMap[lastExtIdx].range.size) * 1024 + + fExtentMap[lastExtIdx].blockOffset; + } + else // next extent, same stripe + { + newBlockOffset = fExtentMap[lastExtIdx].blockOffset; + } + } + } // lastExtentIndex >= 0 + else // Empty DBRoot; use part# that the user specifies + { + if (bHighEmptySegNumSet) + newSegmentNum = highEmptySegNum + 1; + else + newSegmentNum = 0; + } + + //-------------------------------------------------------------------------- + // Fourth Step: Construct the new extentmap entry + //-------------------------------------------------------------------------- + + makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); + EMEntry* e = &fExtentMap[emptyEMEntry]; + + e->range.start = startLBID; + e->range.size = size; + e->fileID = OID; + + if (isUnsigned(colDataType)) + { + if (colWidth != datatypes::MAXDECIMALWIDTH) + { + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = 0; } else { - if (colWidth != datatypes::MAXDECIMALWIDTH) - { - e->partition.cprange.loVal = numeric_limits::max(); - e->partition.cprange.hiVal = numeric_limits::min(); - } - else - { - utils::int128Max(e->partition.cprange.bigLoVal); - utils::int128Min(e->partition.cprange.bigHiVal); - } + e->partition.cprange.bigLoVal = -1; + e->partition.cprange.bigHiVal = 0; + } + } + else + { + if (colWidth != datatypes::MAXDECIMALWIDTH) + { + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = numeric_limits::min(); } - - e->partition.cprange.sequenceNum = 0; - - e->colWid = colWidth; - - e->dbRoot = newDbRoot; - e->partitionNum = newPartitionNum; - e->segmentNum = newSegmentNum; - - e->blockOffset = newBlockOffset; - e->HWM = 0; - e->status = EXTENTUNAVAILABLE; // mark extent as in process - - // Partition, segment, and blockOffset 0 represents new table or column. - // When DDL creates a table, we can mark the first extent as VALID, since - // the table has no data. Marking as VALID enables cpimport to update - // the CP min/max for the first import. - // If DDL is adding a column to an existing table, setting to VALID won't - // hurt, because DDL resets to INVALID after the extent is created. - if ((e->partitionNum == 0) && - (e->segmentNum == 0) && - (e->blockOffset == 0)) - e->partition.cprange.isValid = CP_VALID; else - e->partition.cprange.isValid = CP_INVALID; + { + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); + } + } - partitionNum = e->partitionNum; - segmentNum = e->segmentNum; - startBlockOffset = e->blockOffset; + e->partition.cprange.sequenceNum = 0; - makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); - fEMShminfo->currentSize += sizeof(struct EMEntry); + e->colWid = colWidth; - return startLBID; + e->dbRoot = newDbRoot; + e->partitionNum = newPartitionNum; + e->segmentNum = newSegmentNum; + + e->blockOffset = newBlockOffset; + e->HWM = 0; + e->status = EXTENTUNAVAILABLE; // mark extent as in process + + // Partition, segment, and blockOffset 0 represents new table or column. + // When DDL creates a table, we can mark the first extent as VALID, since + // the table has no data. Marking as VALID enables cpimport to update + // the CP min/max for the first import. + // If DDL is adding a column to an existing table, setting to VALID won't + // hurt, because DDL resets to INVALID after the extent is created. + if ((e->partitionNum == 0) && (e->segmentNum == 0) && (e->blockOffset == 0)) + e->partition.cprange.isValid = CP_VALID; + else + e->partition.cprange.isValid = CP_INVALID; + + partitionNum = e->partitionNum; + segmentNum = e->segmentNum; + startBlockOffset = e->blockOffset; + + makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); + fEMShminfo->currentSize += sizeof(struct EMEntry); + + return startLBID; } //------------------------------------------------------------------------------ @@ -2981,65 +2856,58 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, // allocdsize - number of LBIDs allocated // startBlockOffset-starting block of the created extent //------------------------------------------------------------------------------ -void ExtentMap::createColumnExtentExactFile(int OID, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdsize, - uint32_t& startBlockOffset) +void ExtentMap::createColumnExtentExactFile(int OID, uint32_t colWidth, uint16_t dbRoot, + uint32_t partitionNum, uint16_t segmentNum, + execplan::CalpontSystemCatalog::ColDataType colDataType, + LBID_t& lbid, int& allocdsize, uint32_t& startBlockOffset) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createColumnExtentExactFile"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(colWidth); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDINT64OUTPUT(lbid); - TRACER_ADDOUTPUT(allocdsize); - TRACER_ADDOUTPUT(startBlockOffset); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createColumnExtentExactFile"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(colWidth); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDINT64OUTPUT(lbid); + TRACER_ADDOUTPUT(allocdsize); + TRACER_ADDOUTPUT(startBlockOffset); + TRACER_WRITE; + } #endif #ifdef BRM_DEBUG - if (OID <= 0) - { - log("ExtentMap::createColumnExtentExactFile(): OID must be > 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument( - "ExtentMap::createColumnExtentExactFile(): OID must be > 0"); - } + if (OID <= 0) + { + log("ExtentMap::createColumnExtentExactFile(): OID must be > 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::createColumnExtentExactFile(): OID must be > 0"); + } #endif - // Convert extent size in rows to extent size in 8192-byte blocks. - // extentRows should be multiple of blocksize (8192). - const unsigned EXTENT_SIZE = (getExtentRows() * colWidth) / BLOCK_SIZE; - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + // Convert extent size in rows to extent size in 8192-byte blocks. + // extentRows should be multiple of blocksize (8192). + const unsigned EXTENT_SIZE = (getExtentRows() * colWidth) / BLOCK_SIZE; + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - if (fEMShminfo->currentSize == fEMShminfo->allocdSize) - growEMShmseg(); + if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + growEMShmseg(); -// size is the number of multiples of 1024 blocks. -// ex: size=1 --> 1024 blocks -// size=2 --> 2048 blocks -// size=3 --> 3072 blocks, etc. - uint32_t size = EXTENT_SIZE / 1024; + // size is the number of multiples of 1024 blocks. + // ex: size=1 --> 1024 blocks + // size=2 --> 2048 blocks + // size=3 --> 3072 blocks, etc. + uint32_t size = EXTENT_SIZE / 1024; - lbid = _createColumnExtentExactFile(size, OID, colWidth, - dbRoot, partitionNum, segmentNum, colDataType, startBlockOffset); + lbid = _createColumnExtentExactFile(size, OID, colWidth, dbRoot, partitionNum, segmentNum, colDataType, + startBlockOffset); - allocdsize = EXTENT_SIZE; + allocdsize = EXTENT_SIZE; } //------------------------------------------------------------------------------ @@ -3060,133 +2928,124 @@ void ExtentMap::createColumnExtentExactFile(int OID, // startBlockOffset-starting block of the created extent // returns starting LBID of the created extent. //------------------------------------------------------------------------------ -LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& startBlockOffset) +LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot, + uint32_t partitionNum, uint16_t segmentNum, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& startBlockOffset) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; + int emptyEMEntry = -1; + int lastExtentIndex = -1; + uint32_t highestOffset = 0; - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList( size ); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + LBID_t startLBID = getLBIDsFromFreeList(size); - // Find the first empty Entry; and find the last extent for this - // combination of OID, partition, and segment. - for (int i = 0; i < emEntries; i++) + // Find the first empty Entry; and find the last extent for this + // combination of OID, partition, and segment. + for (int i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) + if (fExtentMap[i].fileID == OID) + { + if ((fExtentMap[i].dbRoot == dbRoot) && (fExtentMap[i].partitionNum == partitionNum) && + (fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) { - if (fExtentMap[i].fileID == OID) - { - if ((fExtentMap[i].dbRoot == dbRoot) && - (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum) && - (fExtentMap[i].blockOffset >= highestOffset)) - { - lastExtentIndex = i; - highestOffset = fExtentMap[i].blockOffset; - } - } + lastExtentIndex = i; + highestOffset = fExtentMap[i].blockOffset; } - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries - - if (emptyEMEntry == -1) - { - ostringstream oss; - oss << "ExtentMap::_createColumnExtentExactFile(): " - "could not find an empty EMEntry for OID " << OID << - "; Extent Map is full", - log(oss.str(), - logging::LOG_TYPE_CRITICAL); - throw logic_error( oss.str() ); + } } + else if (emptyEMEntry < 0) + emptyEMEntry = i; + } // Loop through extent map entries - makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); - EMEntry* e = &fExtentMap[emptyEMEntry]; + if (emptyEMEntry == -1) + { + ostringstream oss; + oss << "ExtentMap::_createColumnExtentExactFile(): " + "could not find an empty EMEntry for OID " + << OID << "; Extent Map is full", + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw logic_error(oss.str()); + } - e->range.start = startLBID; - e->range.size = size; - e->fileID = OID; + makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); + EMEntry* e = &fExtentMap[emptyEMEntry]; - if (isUnsigned(colDataType)) + e->range.start = startLBID; + e->range.size = size; + e->fileID = OID; + + if (isUnsigned(colDataType)) + { + if (colWidth != datatypes::MAXDECIMALWIDTH) { - if (colWidth != datatypes::MAXDECIMALWIDTH) - { - e->partition.cprange.loVal = numeric_limits::max(); - e->partition.cprange.hiVal = 0; - } - else - { - e->partition.cprange.bigLoVal = -1; - e->partition.cprange.bigHiVal = 0; - } + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = 0; } else { - if (colWidth != datatypes::MAXDECIMALWIDTH) - { - e->partition.cprange.loVal = numeric_limits::max(); - e->partition.cprange.hiVal = numeric_limits::min(); - } - else - { - utils::int128Max(e->partition.cprange.bigLoVal); - utils::int128Min(e->partition.cprange.bigHiVal); - } + e->partition.cprange.bigLoVal = -1; + e->partition.cprange.bigHiVal = 0; } - - e->partition.cprange.sequenceNum = 0; - - e->colWid = colWidth; - - e->dbRoot = dbRoot; - e->partitionNum = partitionNum; - e->segmentNum = segmentNum; - e->status = EXTENTUNAVAILABLE; // mark extent as in process - - // If first extent for this OID, partition, dbroot, and segment then - // blockOffset is set to 0 - // else - // blockOffset is extrapolated from the last extent - if (lastExtentIndex == -1) + } + else + { + if (colWidth != datatypes::MAXDECIMALWIDTH) { - e->blockOffset = 0; - e->HWM = 0; + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = numeric_limits::min(); } else { - e->blockOffset = static_cast - (fExtentMap[lastExtentIndex].range.size) * 1024 + - fExtentMap[lastExtentIndex].blockOffset; - e->HWM = 0; + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); } + } - // Partition, segment, and blockOffset 0 represents new table or column. - // When DDL creates a table, we can mark the first extent as VALID, since - // the table has no data. Marking as VALID enables cpimport to update - // the CP min/max for the first import. - // If DDL is adding a column to an existing table, setting to VALID won't - // hurt, because DDL resets to INVALID after the extent is created. - if ((e->partitionNum == 0) && - (e->segmentNum == 0) && - (e->blockOffset == 0)) - e->partition.cprange.isValid = CP_VALID; - else - e->partition.cprange.isValid = CP_INVALID; + e->partition.cprange.sequenceNum = 0; - startBlockOffset = e->blockOffset; + e->colWid = colWidth; - makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); - fEMShminfo->currentSize += sizeof(struct EMEntry); + e->dbRoot = dbRoot; + e->partitionNum = partitionNum; + e->segmentNum = segmentNum; + e->status = EXTENTUNAVAILABLE; // mark extent as in process - return startLBID; + // If first extent for this OID, partition, dbroot, and segment then + // blockOffset is set to 0 + // else + // blockOffset is extrapolated from the last extent + if (lastExtentIndex == -1) + { + e->blockOffset = 0; + e->HWM = 0; + } + else + { + e->blockOffset = static_cast(fExtentMap[lastExtentIndex].range.size) * 1024 + + fExtentMap[lastExtentIndex].blockOffset; + e->HWM = 0; + } + + // Partition, segment, and blockOffset 0 represents new table or column. + // When DDL creates a table, we can mark the first extent as VALID, since + // the table has no data. Marking as VALID enables cpimport to update + // the CP min/max for the first import. + // If DDL is adding a column to an existing table, setting to VALID won't + // hurt, because DDL resets to INVALID after the extent is created. + if ((e->partitionNum == 0) && (e->segmentNum == 0) && (e->blockOffset == 0)) + e->partition.cprange.isValid = CP_VALID; + else + e->partition.cprange.isValid = CP_INVALID; + + startBlockOffset = e->blockOffset; + + makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); + fEMShminfo->currentSize += sizeof(struct EMEntry); + + return startLBID; } //------------------------------------------------------------------------------ @@ -3201,61 +3060,54 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, // lbid - starting LBID of the created extent // allocdsize - number LBIDs of allocated //------------------------------------------------------------------------------ -void ExtentMap::createDictStoreExtent(int OID, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - LBID_t& lbid, - int& allocdsize) +void ExtentMap::createDictStoreExtent(int OID, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum, + LBID_t& lbid, int& allocdsize) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("createDictStoreExtent"); - TRACER_ADDINPUT(OID); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINT64OUTPUT(lbid); - TRACER_ADDOUTPUT(allocdsize); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("createDictStoreExtent"); + TRACER_ADDINPUT(OID); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINT64OUTPUT(lbid); + TRACER_ADDOUTPUT(allocdsize); + TRACER_WRITE; + } #endif #ifdef BRM_DEBUG - if (OID <= 0) - { - log("ExtentMap::createDictStoreExtent(): OID must be > 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument( - "ExtentMap::createDictStoreExtent(): OID must be > 0"); - } + if (OID <= 0) + { + log("ExtentMap::createDictStoreExtent(): OID must be > 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::createDictStoreExtent(): OID must be > 0"); + } #endif - // Convert extent size in rows to extent size in 8192-byte blocks. - // extentRows should be multiple of blocksize (8192). - const unsigned EXTENT_SIZE = (getExtentRows() * DICT_COL_WIDTH) / BLOCK_SIZE; + // Convert extent size in rows to extent size in 8192-byte blocks. + // extentRows should be multiple of blocksize (8192). + const unsigned EXTENT_SIZE = (getExtentRows() * DICT_COL_WIDTH) / BLOCK_SIZE; - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - if (fEMShminfo->currentSize == fEMShminfo->allocdSize) - growEMShmseg(); + if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + growEMShmseg(); -// size is the number of multiples of 1024 blocks. -// ex: size=1 --> 1024 blocks -// size=2 --> 2048 blocks -// size=3 --> 3072 blocks, etc. - uint32_t size = EXTENT_SIZE / 1024; + // size is the number of multiples of 1024 blocks. + // ex: size=1 --> 1024 blocks + // size=2 --> 2048 blocks + // size=3 --> 3072 blocks, etc. + uint32_t size = EXTENT_SIZE / 1024; - lbid = _createDictStoreExtent(size, OID, - dbRoot, partitionNum, segmentNum); + lbid = _createDictStoreExtent(size, OID, dbRoot, partitionNum, segmentNum); - allocdsize = EXTENT_SIZE; + allocdsize = EXTENT_SIZE; } //------------------------------------------------------------------------------ @@ -3272,91 +3124,85 @@ void ExtentMap::createDictStoreExtent(int OID, // segmentNum - segment number to be assigned to the new extent // returns starting LBID of the created extent. //------------------------------------------------------------------------------ -LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum) +LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; + int emptyEMEntry = -1; + int lastExtentIndex = -1; + uint32_t highestOffset = 0; - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList( size ); + LBID_t startLBID = getLBIDsFromFreeList(size); - // Find the first empty Entry; and find the last extent for this - // combination of OID, partition, and segment. - for (int i = 0; i < emEntries; i++) + // Find the first empty Entry; and find the last extent for this + // combination of OID, partition, and segment. + for (int i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) - { - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum) && - (fExtentMap[i].blockOffset >= highestOffset)) - { - lastExtentIndex = i; - highestOffset = fExtentMap[i].blockOffset; - } - } - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries - - if (emptyEMEntry == -1) - { - ostringstream oss; - oss << "ExtentMap::_createDictStoreExtent(): " - "could not find an empty EMEntry for OID " << OID << - "; Extent Map is full", - log(oss.str(), - logging::LOG_TYPE_CRITICAL); - throw logic_error( oss.str() ); + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].partitionNum == partitionNum) && + (fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) + { + lastExtentIndex = i; + highestOffset = fExtentMap[i].blockOffset; + } } + else if (emptyEMEntry < 0) + emptyEMEntry = i; + } // Loop through extent map entries - makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); - EMEntry* e = &fExtentMap[emptyEMEntry]; + if (emptyEMEntry == -1) + { + ostringstream oss; + oss << "ExtentMap::_createDictStoreExtent(): " + "could not find an empty EMEntry for OID " + << OID << "; Extent Map is full", + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw logic_error(oss.str()); + } - e->range.start = startLBID; - e->range.size = size; - e->fileID = OID; - e->status = EXTENTUNAVAILABLE;// @bug 1911 mark extent as in process - utils::int128Max(e->partition.cprange.bigLoVal); - utils::int128Min(e->partition.cprange.bigHiVal); - e->partition.cprange.sequenceNum = 0; - e->partition.cprange.isValid = CP_INVALID; + makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); + EMEntry* e = &fExtentMap[emptyEMEntry]; - // If this is first extent for this OID, partition, segment then - // everything is set to 0 or taken from user input - // else - // everything is extrapolated from the last extent - if (lastExtentIndex == -1) - { - e->blockOffset = 0; - e->HWM = 0; - e->segmentNum = segmentNum; - e->partitionNum = partitionNum; - e->dbRoot = dbRoot; - e->colWid = 0; // we don't store col width for dictionaries; - // this helps to flag this as a dictionary extent - } - else - { - e->blockOffset = static_cast - (fExtentMap[lastExtentIndex].range.size) * 1024 + - fExtentMap[lastExtentIndex].blockOffset; - e->HWM = 0; - e->segmentNum = fExtentMap[lastExtentIndex].segmentNum; - e->partitionNum = fExtentMap[lastExtentIndex].partitionNum; - e->dbRoot = fExtentMap[lastExtentIndex].dbRoot; - e->colWid = fExtentMap[lastExtentIndex].colWid; - } + e->range.start = startLBID; + e->range.size = size; + e->fileID = OID; + e->status = EXTENTUNAVAILABLE; // @bug 1911 mark extent as in process + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); + e->partition.cprange.sequenceNum = 0; + e->partition.cprange.isValid = CP_INVALID; - makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); - fEMShminfo->currentSize += sizeof(struct EMEntry); + // If this is first extent for this OID, partition, segment then + // everything is set to 0 or taken from user input + // else + // everything is extrapolated from the last extent + if (lastExtentIndex == -1) + { + e->blockOffset = 0; + e->HWM = 0; + e->segmentNum = segmentNum; + e->partitionNum = partitionNum; + e->dbRoot = dbRoot; + e->colWid = 0; // we don't store col width for dictionaries; + // this helps to flag this as a dictionary extent + } + else + { + e->blockOffset = static_cast(fExtentMap[lastExtentIndex].range.size) * 1024 + + fExtentMap[lastExtentIndex].blockOffset; + e->HWM = 0; + e->segmentNum = fExtentMap[lastExtentIndex].segmentNum; + e->partitionNum = fExtentMap[lastExtentIndex].partitionNum; + e->dbRoot = fExtentMap[lastExtentIndex].dbRoot; + e->colWid = fExtentMap[lastExtentIndex].colWid; + } - return startLBID; + makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); + fEMShminfo->currentSize += sizeof(struct EMEntry); + + return startLBID; } //------------------------------------------------------------------------------ @@ -3369,116 +3215,104 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, // size=3 --> 3072 blocks, etc. // returns selected starting LBID. //------------------------------------------------------------------------------ -LBID_t ExtentMap::getLBIDsFromFreeList ( uint32_t size ) +LBID_t ExtentMap::getLBIDsFromFreeList(uint32_t size) { - LBID_t ret = -1; - int i; - int flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + LBID_t ret = -1; + int i; + int flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - for (i = 0; i < flEntries; i++) + for (i = 0; i < flEntries; i++) + { + if (size <= fFreeList[i].size) { - if (size <= fFreeList[i].size) - { - makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); - ret = fFreeList[i].start; - fFreeList[i].start += size * 1024; - fFreeList[i].size -= size; + makeUndoRecord(&fFreeList[i], sizeof(InlineLBIDRange)); + ret = fFreeList[i].start; + fFreeList[i].start += size * 1024; + fFreeList[i].size -= size; - if (fFreeList[i].size == 0) - { - makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); - fFLShminfo->currentSize -= sizeof(InlineLBIDRange); - } + if (fFreeList[i].size == 0) + { + makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); + fFLShminfo->currentSize -= sizeof(InlineLBIDRange); + } - break; - } + break; } + } - if (i == flEntries) - { - log("ExtentMap::getLBIDsFromFreeList(): out of LBID space"); - throw runtime_error( - "ExtentMap::getLBIDsFromFreeList(): out of LBID space"); - } + if (i == flEntries) + { + log("ExtentMap::getLBIDsFromFreeList(): out of LBID space"); + throw runtime_error("ExtentMap::getLBIDsFromFreeList(): out of LBID space"); + } - return ret; + return ret; } #ifdef BRM_DEBUG void ExtentMap::printEM(const EMEntry& em) const { - cout << " Start " - << em.range.start << " Size " - << (long) em.range.size << " OID " - << (long) em.fileID << " offset " - << (long) em.blockOffset - << " LV " << em.partition.cprange.loVal - << " HV " << em.partition.cprange.hiVal; - cout << endl; + cout << " Start " << em.range.start << " Size " << (long)em.range.size << " OID " << (long)em.fileID + << " offset " << (long)em.blockOffset << " LV " << em.partition.cprange.loVal << " HV " + << em.partition.cprange.hiVal; + cout << endl; } - void ExtentMap::printEM(const OID_t& oid) const { - int emEntries = 0; + int emEntries = 0; - if (fEMShminfo) - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + if (fEMShminfo) + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - cout << "Extent Map (OID=" << oid << ")" << endl; + cout << "Extent Map (OID=" << oid << ")" << endl; - for (int idx = 0; idx < emEntries ; idx++) - { - struct EMEntry& em = fExtentMap[idx]; + for (int idx = 0; idx < emEntries; idx++) + { + struct EMEntry& em = fExtentMap[idx]; - if (em.fileID == oid && em.range.size != 0) - printEM(em); - } + if (em.fileID == oid && em.range.size != 0) + printEM(em); + } - cout << endl; + cout << endl; } void ExtentMap::printEM() const { + int emEntries = 0; - int emEntries = 0; + if (fEMShminfo) + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - if (fEMShminfo) - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + cout << "Extent Map (" << emEntries << ")" << endl; - cout << "Extent Map (" << emEntries << ")" << endl; + for (int idx = 0; idx < emEntries; idx++) + { + struct EMEntry& em = fExtentMap[idx]; - for (int idx = 0; idx < emEntries ; idx++) - { - struct EMEntry& em = fExtentMap[idx]; + if (em.range.size != 0) + printEM(em); + } - if (em.range.size != 0) - printEM(em); - } - - cout << endl; + cout << endl; } void ExtentMap::printFL() const { + int flEntries = 0; - int flEntries = 0; + if (fFLShminfo) + flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - if (fFLShminfo) - flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + cout << "Free List" << endl; - cout << "Free List" << endl; + for (int idx = 0; idx < flEntries; idx++) + { + cout << idx << " " << fFreeList[idx].start << " " << fFreeList[idx].size << endl; + } - for (int idx = 0; idx < flEntries; idx++) - { - - cout << idx << " " - << fFreeList[idx].start << " " - << fFreeList[idx].size - << endl; - } - - cout << endl; + cout << endl; } #endif @@ -3495,184 +3329,175 @@ void ExtentMap::printFL() const // segmentNum - segment number of the last logical extent to be retained // hwm - HWM to be assigned to the last logical extent retained //------------------------------------------------------------------------------ -void ExtentMap::rollbackColumnExtents_DBroot ( int oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm) +void ExtentMap::rollbackColumnExtents_DBroot(int oid, bool bDeleteAll, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, HWM_t hwm) { - //bool oidExists = false; + // bool oidExists = false; #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("rollbackColumnExtents"); - TRACER_ADDINPUT(oid); - TRACER_ADDBOOLINPUT(bDeleteAll); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(hwm); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("rollbackColumnExtents"); + TRACER_ADDINPUT(oid); + TRACER_ADDBOOLINPUT(bDeleteAll); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(hwm); + TRACER_WRITE; + } #endif #ifdef BRM_DEBUG - if (oid < 0) - { - log("ExtentMap::rollbackColumnExtents_DBroot(): OID must be >= 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument( - "ExtentMap::rollbackColumnExtents_DBroot(): OID must be >= 0"); - } + if (oid < 0) + { + log("ExtentMap::rollbackColumnExtents_DBroot(): OID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::rollbackColumnExtents_DBroot(): OID must be >= 0"); + } #endif - uint32_t fboLo = 0; - uint32_t fboHi = 0; - uint32_t fboLoPreviousStripe = 0; + uint32_t fboLo = 0; + uint32_t fboHi = 0; + uint32_t fboLoPreviousStripe = 0; - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == oid) && (fExtentMap[i].dbRoot == dbRoot)) { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == oid) && - (fExtentMap[i].dbRoot == dbRoot)) + // oidExists = true; + + // Don't rollback extents that are out of service + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + continue; + + // If bDeleteAll is true, then we delete extent w/o regards to + // partition number, segment number, or HWM + if (bDeleteAll) + { + deleteExtent(i); // case 0 + continue; + } + + // Calculate fbo range for the stripe containing the given hwm + if (fboHi == 0) + { + uint32_t range = fExtentMap[i].range.size * 1024; + fboLo = hwm - (hwm % range); + fboHi = fboLo + range - 1; + + if (fboLo > 0) + fboLoPreviousStripe = fboLo - range; + } + + // Delete, update, or ignore this extent: + // Later partition: + // case 1: extent in later partition than last extent, so delete + // Same partition: + // case 2: extent is in later stripe than last extent, so delete + // case 3: extent is in earlier stripe in the same partition. + // No action necessary for case3B and case3C. + // case 3A: extent is in trailing segment in previous stripe. + // This extent is now the last extent in that segment + // file, so reset the local HWM if it was altered. + // case 3B: extent in previous stripe but not a trailing segment + // case 3C: extent is in stripe that precedes previous stripe + // case 4: extent is in the same partition and stripe as the + // last logical extent we are to keep. + // case 4A: extent is in later segment so can be deleted + // case 4B: extent is in earlier segment, reset HWM if changed + // case 4C: this is last logical extent, reset HWM if changed + // Earlier partition: + // case 5: extent is in earlier parition, no action necessary + + if (fExtentMap[i].partitionNum > partitionNum) + { + deleteExtent(i); // case 1 + } + else if (fExtentMap[i].partitionNum == partitionNum) + { + if (fExtentMap[i].blockOffset > fboHi) { - - //oidExists = true; - - // Don't rollback extents that are out of service - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - continue; - - // If bDeleteAll is true, then we delete extent w/o regards to - // partition number, segment number, or HWM - if (bDeleteAll) + deleteExtent(i); // case 2 + } + else if (fExtentMap[i].blockOffset < fboLo) + { + if (fExtentMap[i].blockOffset >= fboLoPreviousStripe) + { + if (fExtentMap[i].segmentNum > segmentNum) { - deleteExtent( i ); // case 0 - continue; - } - - // Calculate fbo range for the stripe containing the given hwm - if (fboHi == 0) - { - uint32_t range = fExtentMap[i].range.size * 1024; - fboLo = hwm - (hwm % range); - fboHi = fboLo + range - 1; - - if (fboLo > 0) - fboLoPreviousStripe = fboLo - range; - } - - // Delete, update, or ignore this extent: - // Later partition: - // case 1: extent in later partition than last extent, so delete - // Same partition: - // case 2: extent is in later stripe than last extent, so delete - // case 3: extent is in earlier stripe in the same partition. - // No action necessary for case3B and case3C. - // case 3A: extent is in trailing segment in previous stripe. - // This extent is now the last extent in that segment - // file, so reset the local HWM if it was altered. - // case 3B: extent in previous stripe but not a trailing segment - // case 3C: extent is in stripe that precedes previous stripe - // case 4: extent is in the same partition and stripe as the - // last logical extent we are to keep. - // case 4A: extent is in later segment so can be deleted - // case 4B: extent is in earlier segment, reset HWM if changed - // case 4C: this is last logical extent, reset HWM if changed - // Earlier partition: - // case 5: extent is in earlier parition, no action necessary - - if (fExtentMap[i].partitionNum > partitionNum) - { - deleteExtent( i ); // case 1 - } - else if (fExtentMap[i].partitionNum == partitionNum) - { - if (fExtentMap[i].blockOffset > fboHi) - { - deleteExtent( i ); // case 2 - } - else if (fExtentMap[i].blockOffset < fboLo) - { - if (fExtentMap[i].blockOffset >= fboLoPreviousStripe) - { - if (fExtentMap[i].segmentNum > segmentNum) - { - if (fExtentMap[i].HWM != (fboLo - 1)) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = fboLo - 1; //case 3A - fExtentMap[i].status = EXTENTAVAILABLE; - } - } - else - { - // not a trailing segment in prev stripe case 3B - } - } - else - { - // extent precedes previous stripe case 3C - } - } - else // extent is in same stripe - { - if (fExtentMap[i].segmentNum > segmentNum) - { - deleteExtent( i ); // case 4A - } - else if (fExtentMap[i].segmentNum < segmentNum) - { - if (fExtentMap[i].HWM != fboHi) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = fboHi; // case 4B - fExtentMap[i].status = EXTENTAVAILABLE; - } - } - else // fExtentMap[i].segmentNum == segmentNum - { - if (fExtentMap[i].HWM != hwm) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = hwm; // case 4C - fExtentMap[i].status = EXTENTAVAILABLE; - } - } - } + if (fExtentMap[i].HWM != (fboLo - 1)) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = fboLo - 1; // case 3A + fExtentMap[i].status = EXTENTAVAILABLE; + } } else { - // extent in earlier partition; no action necessary case 5 + // not a trailing segment in prev stripe case 3B } - } // extent map entry with matching oid - } // loop through the extent map + } + else + { + // extent precedes previous stripe case 3C + } + } + else // extent is in same stripe + { + if (fExtentMap[i].segmentNum > segmentNum) + { + deleteExtent(i); // case 4A + } + else if (fExtentMap[i].segmentNum < segmentNum) + { + if (fExtentMap[i].HWM != fboHi) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = fboHi; // case 4B + fExtentMap[i].status = EXTENTAVAILABLE; + } + } + else // fExtentMap[i].segmentNum == segmentNum + { + if (fExtentMap[i].HWM != hwm) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = hwm; // case 4C + fExtentMap[i].status = EXTENTAVAILABLE; + } + } + } + } + else + { + // extent in earlier partition; no action necessary case 5 + } + } // extent map entry with matching oid + } // loop through the extent map - // If this function is called, we are already in error recovery mode; so - // don't worry about reporting an error if the OID is not found, because - // we don't want/need the extents for that OID anyway. - //if (!oidExists) - //{ - // ostringstream oss; - // oss << "ExtentMap::rollbackColumnExtents_DBroot(): " - // "Rollback failed: no extents exist for: OID-" << oid << - // "; dbRoot-" << dbRoot << - // "; partition-" << partitionNum << - // "; segment-" << segmentNum << - // "; hwm-" << hwm; - // log(oss.str(), logging::LOG_TYPE_CRITICAL); - // throw invalid_argument(oss.str()); - //} + // If this function is called, we are already in error recovery mode; so + // don't worry about reporting an error if the OID is not found, because + // we don't want/need the extents for that OID anyway. + // if (!oidExists) + //{ + // ostringstream oss; + // oss << "ExtentMap::rollbackColumnExtents_DBroot(): " + // "Rollback failed: no extents exist for: OID-" << oid << + // "; dbRoot-" << dbRoot << + // "; partition-" << partitionNum << + // "; segment-" << segmentNum << + // "; hwm-" << hwm; + // log(oss.str(), logging::LOG_TYPE_CRITICAL); + // throw invalid_argument(oss.str()); + //} } //------------------------------------------------------------------------------ @@ -3696,162 +3521,155 @@ void ExtentMap::rollbackColumnExtents_DBroot ( int oid, // hwms[0] applies to segment store file segNums[0]; // hwms[1] applies to segment store file segNums[1]; etc. //------------------------------------------------------------------------------ -void ExtentMap::rollbackDictStoreExtents_DBroot ( int oid, - uint16_t dbRoot, - uint32_t partitionNum, - const vector& segNums, - const vector& hwms) +void ExtentMap::rollbackDictStoreExtents_DBroot(int oid, uint16_t dbRoot, uint32_t partitionNum, + const vector& segNums, const vector& hwms) { - //bool oidExists = false; + // bool oidExists = false; #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + ostringstream oss; + + for (unsigned int k = 0; k < hwms.size(); k++) + oss << "; hwms[" << k << "]-" << hwms[k]; + + const string& hwmString(oss.str()); + + // put TRACE inside separate scope {} to insure that temporary + // hwmString still exists when tracer destructor tries to print it. { - ostringstream oss; - - for (unsigned int k = 0; k < hwms.size(); k++) - oss << "; hwms[" << k << "]-" << hwms[k]; - - const string& hwmString(oss.str()); - - // put TRACE inside separate scope {} to insure that temporary - // hwmString still exists when tracer destructor tries to print it. - { - TRACER_WRITELATER("rollbackDictStoreExtents_DBroot"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSTRINPUT(hwmString); - TRACER_WRITE; - } + TRACER_WRITELATER("rollbackDictStoreExtents_DBroot"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSTRINPUT(hwmString); + TRACER_WRITE; } + } #endif - // Delete all extents for the specified OID and DBRoot, - // if we are not given any hwms and segment files. - bool bDeleteAll = false; + // Delete all extents for the specified OID and DBRoot, + // if we are not given any hwms and segment files. + bool bDeleteAll = false; - if (hwms.size() == 0) - bDeleteAll = true; + if (hwms.size() == 0) + bDeleteAll = true; - // segToHwmMap maps segment file number to corresponding pair - tr1::unordered_map > segToHwmMap; - tr1::unordered_map >::const_iterator - segToHwmMapIter; + // segToHwmMap maps segment file number to corresponding pair + tr1::unordered_map > segToHwmMap; + tr1::unordered_map >::const_iterator segToHwmMapIter; - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == oid) && (fExtentMap[i].dbRoot == dbRoot)) { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == oid) && - (fExtentMap[i].dbRoot == dbRoot)) + // oidExists = true; + + // Don't rollback extents that are out of service + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + continue; + + // If bDeleteAll is true, then we delete extent w/o regards to + // partition number, segment number, or HWM + if (bDeleteAll) + { + deleteExtent(i); // case 0 + continue; + } + + // Calculate fbo's for the list of hwms we are given; and store + // the fbo and hwm in a map, using the segment file number as a key. + if (segToHwmMap.size() == 0) + { + uint32_t range = fExtentMap[i].range.size * 1024; + pair segToHwmMapEntry; + + for (unsigned int k = 0; k < hwms.size(); k++) { + uint32_t fboLo = hwms[k] - (hwms[k] % range); + segToHwmMapEntry.first = hwms[k]; + segToHwmMapEntry.second = fboLo; + segToHwmMap[segNums[k]] = segToHwmMapEntry; + } + } - //oidExists = true; + // Delete, update, or ignore this extent: + // Later partition: + // case 1: extent is in later partition, so delete the extent + // Same partition: + // case 2: extent is in trailing seg file we don't need; so delete + // case 3: extent is in partition and segment file of interest + // case 3A: earlier extent in segment file; no action necessary + // case 3B: specified HWM falls in this extent, so reset HWM + // case 3C: later extent in segment file; so delete the extent + // Earlier partition: + // case 4: extent is in earlier parition, no action necessary - // Don't rollback extents that are out of service - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - continue; + if (fExtentMap[i].partitionNum > partitionNum) + { + deleteExtent(i); // case 1 + } + else if (fExtentMap[i].partitionNum == partitionNum) + { + unsigned segNum = fExtentMap[i].segmentNum; + segToHwmMapIter = segToHwmMap.find(segNum); - // If bDeleteAll is true, then we delete extent w/o regards to - // partition number, segment number, or HWM - if (bDeleteAll) + if (segToHwmMapIter == segToHwmMap.end()) + { + deleteExtent(i); // case 2 + } + else // segment number in the map of files to keep + { + uint32_t fboLo = segToHwmMapIter->second.second; + + if (fExtentMap[i].blockOffset < fboLo) + { + // no action necessary case 3A + } + else if (fExtentMap[i].blockOffset == fboLo) + { + uint32_t hwm = segToHwmMapIter->second.first; + + if (fExtentMap[i].HWM != hwm) { - deleteExtent( i ); // case 0 - continue; + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = hwm; + fExtentMap[i].status = EXTENTAVAILABLE; // case 3B } + } + else + { + deleteExtent(i); // case 3C + } + } + } + else + { + // extent in earlier partition; no action necessary case 4 + } + } // extent map entry with matching oid + } // loop through the extent map - // Calculate fbo's for the list of hwms we are given; and store - // the fbo and hwm in a map, using the segment file number as a key. - if (segToHwmMap.size() == 0) - { - uint32_t range = fExtentMap[i].range.size * 1024; - pair segToHwmMapEntry; - - for (unsigned int k = 0; k < hwms.size(); k++) - { - uint32_t fboLo = hwms[k] - (hwms[k] % range); - segToHwmMapEntry.first = hwms[k]; - segToHwmMapEntry.second = fboLo; - segToHwmMap[ segNums[k] ] = segToHwmMapEntry; - } - } - - // Delete, update, or ignore this extent: - // Later partition: - // case 1: extent is in later partition, so delete the extent - // Same partition: - // case 2: extent is in trailing seg file we don't need; so delete - // case 3: extent is in partition and segment file of interest - // case 3A: earlier extent in segment file; no action necessary - // case 3B: specified HWM falls in this extent, so reset HWM - // case 3C: later extent in segment file; so delete the extent - // Earlier partition: - // case 4: extent is in earlier parition, no action necessary - - if (fExtentMap[i].partitionNum > partitionNum) - { - deleteExtent( i ); // case 1 - } - else if (fExtentMap[i].partitionNum == partitionNum) - { - unsigned segNum = fExtentMap[i].segmentNum; - segToHwmMapIter = segToHwmMap.find( segNum ); - - if (segToHwmMapIter == segToHwmMap.end()) - { - deleteExtent( i ); // case 2 - } - else // segment number in the map of files to keep - { - uint32_t fboLo = segToHwmMapIter->second.second; - - if (fExtentMap[i].blockOffset < fboLo) - { - // no action necessary case 3A - } - else if (fExtentMap[i].blockOffset == fboLo) - { - uint32_t hwm = segToHwmMapIter->second.first; - - if (fExtentMap[i].HWM != hwm) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = hwm; - fExtentMap[i].status = EXTENTAVAILABLE; // case 3B - } - } - else - { - deleteExtent( i ); // case 3C - } - } - } - else - { - // extent in earlier partition; no action necessary case 4 - } - } // extent map entry with matching oid - } // loop through the extent map - - // If this function is called, we are already in error recovery mode; so - // don't worry about reporting an error if the OID is not found, because - // we don't want/need the extents for that OID anyway. - //if (!oidExists) - //{ - // ostringstream oss; - // oss << "ExtentMap::rollbackDictStoreExtents_DBroot(): " - // "Rollback failed: no extents exist for: OID-" << oid << - // "; dbRoot-" << dbRoot << - // "; partition-" << partitionNum; - // log(oss.str(), logging::LOG_TYPE_CRITICAL); - // throw invalid_argument(oss.str()); - //} + // If this function is called, we are already in error recovery mode; so + // don't worry about reporting an error if the OID is not found, because + // we don't want/need the extents for that OID anyway. + // if (!oidExists) + //{ + // ostringstream oss; + // oss << "ExtentMap::rollbackDictStoreExtents_DBroot(): " + // "Rollback failed: no extents exist for: OID-" << oid << + // "; dbRoot-" << dbRoot << + // "; partition-" << partitionNum; + // log(oss.str(), logging::LOG_TYPE_CRITICAL); + // throw invalid_argument(oss.str()); + //} } //------------------------------------------------------------------------------ @@ -3861,251 +3679,250 @@ void ExtentMap::deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteEmptyColExtents"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteEmptyColExtents"); + TRACER_WRITE; + } #endif - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - uint32_t fboLo = 0; - uint32_t fboHi = 0; - uint32_t fboLoPreviousStripe = 0; + uint32_t fboLo = 0; + uint32_t fboHi = 0; + uint32_t fboLoPreviousStripe = 0; - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - ExtentsInfoMap_t::const_iterator it; + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + ExtentsInfoMap_t::const_iterator it; - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) + it = extentsInfo.find(fExtentMap[i].fileID); + + if (it != extentsInfo.end()) + { + // Don't rollback extents that are out of service + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + continue; + + // Calculate fbo range for the stripe containing the given hwm + if (fboHi == 0) { - it = extentsInfo.find ( fExtentMap[i].fileID ); + uint32_t range = fExtentMap[i].range.size * 1024; + fboLo = it->second.hwm - (it->second.hwm % range); + fboHi = fboLo + range - 1; - if ( it != extentsInfo.end() ) - { - // Don't rollback extents that are out of service - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - continue; - - // Calculate fbo range for the stripe containing the given hwm - if (fboHi == 0) - { - uint32_t range = fExtentMap[i].range.size * 1024; - fboLo = it->second.hwm - (it->second.hwm % range); - fboHi = fboLo + range - 1; - - if (fboLo > 0) - fboLoPreviousStripe = fboLo - range; - } - - // Delete, update, or ignore this extent: - // Later partition: - // case 1: extent in later partition than last extent, so delete - // Same partition: - // case 2: extent is in later stripe than last extent, so delete - // case 3: extent is in earlier stripe in the same partition. - // No action necessary for case3B and case3C. - // case 3A: extent is in trailing segment in previous stripe. - // This extent is now the last extent in that segment - // file, so reset the local HWM if it was altered. - // case 3B: extent in previous stripe but not a trailing segment - // case 3C: extent is in stripe that precedes previous stripe - // case 4: extent is in the same partition and stripe as the - // last logical extent we are to keep. - // case 4A: extent is in later segment so can be deleted - // case 4B: extent is in earlier segment, reset HWM if changed - // case 4C: this is last logical extent, reset HWM if changed - // Earlier partition: - // case 5: extent is in earlier parition, no action necessary - - if (fExtentMap[i].partitionNum > it->second.partitionNum) - { - deleteExtent( i ); // case 1 - } - else if (fExtentMap[i].partitionNum == it->second.partitionNum) - { - if (fExtentMap[i].blockOffset > fboHi) - { - deleteExtent( i ); // case 2 - } - else if (fExtentMap[i].blockOffset < fboLo) - { - if (fExtentMap[i].blockOffset >= fboLoPreviousStripe) - { - if (fExtentMap[i].segmentNum > it->second.segmentNum) - { - if (fExtentMap[i].HWM != (fboLo - 1)) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = fboLo - 1; //case 3A - fExtentMap[i].status = EXTENTAVAILABLE; - } - } - else - { - // not a trailing segment in prev stripe case 3B - } - } - else - { - // extent precedes previous stripe case 3C - } - } - else - { - // extent is in same stripe - if (fExtentMap[i].segmentNum > it->second.segmentNum) - { - deleteExtent( i ); // case 4A - } - else if (fExtentMap[i].segmentNum < it->second.segmentNum) - { - if (fExtentMap[i].HWM != fboHi) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = fboHi; // case 4B - fExtentMap[i].status = EXTENTAVAILABLE; - } - } - else - { - // fExtentMap[i].segmentNum == segmentNum - if (fExtentMap[i].HWM != it->second.hwm) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = it->second.hwm;// case 4C - fExtentMap[i].status = EXTENTAVAILABLE; - } - } - } - } - else - { - // extent in earlier partition; no action necessary case 5 - } - } // extent map entry with matching oid + if (fboLo > 0) + fboLoPreviousStripe = fboLo - range; } - } // loop through the extent map + + // Delete, update, or ignore this extent: + // Later partition: + // case 1: extent in later partition than last extent, so delete + // Same partition: + // case 2: extent is in later stripe than last extent, so delete + // case 3: extent is in earlier stripe in the same partition. + // No action necessary for case3B and case3C. + // case 3A: extent is in trailing segment in previous stripe. + // This extent is now the last extent in that segment + // file, so reset the local HWM if it was altered. + // case 3B: extent in previous stripe but not a trailing segment + // case 3C: extent is in stripe that precedes previous stripe + // case 4: extent is in the same partition and stripe as the + // last logical extent we are to keep. + // case 4A: extent is in later segment so can be deleted + // case 4B: extent is in earlier segment, reset HWM if changed + // case 4C: this is last logical extent, reset HWM if changed + // Earlier partition: + // case 5: extent is in earlier parition, no action necessary + + if (fExtentMap[i].partitionNum > it->second.partitionNum) + { + deleteExtent(i); // case 1 + } + else if (fExtentMap[i].partitionNum == it->second.partitionNum) + { + if (fExtentMap[i].blockOffset > fboHi) + { + deleteExtent(i); // case 2 + } + else if (fExtentMap[i].blockOffset < fboLo) + { + if (fExtentMap[i].blockOffset >= fboLoPreviousStripe) + { + if (fExtentMap[i].segmentNum > it->second.segmentNum) + { + if (fExtentMap[i].HWM != (fboLo - 1)) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = fboLo - 1; // case 3A + fExtentMap[i].status = EXTENTAVAILABLE; + } + } + else + { + // not a trailing segment in prev stripe case 3B + } + } + else + { + // extent precedes previous stripe case 3C + } + } + else + { + // extent is in same stripe + if (fExtentMap[i].segmentNum > it->second.segmentNum) + { + deleteExtent(i); // case 4A + } + else if (fExtentMap[i].segmentNum < it->second.segmentNum) + { + if (fExtentMap[i].HWM != fboHi) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = fboHi; // case 4B + fExtentMap[i].status = EXTENTAVAILABLE; + } + } + else + { + // fExtentMap[i].segmentNum == segmentNum + if (fExtentMap[i].HWM != it->second.hwm) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = it->second.hwm; // case 4C + fExtentMap[i].status = EXTENTAVAILABLE; + } + } + } + } + else + { + // extent in earlier partition; no action necessary case 5 + } + } // extent map entry with matching oid + } + } // loop through the extent map } void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteEmptyDictStoreExtents"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteEmptyDictStoreExtents"); + TRACER_WRITE; + } #endif - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - ExtentsInfoMap_t::const_iterator it; + ExtentsInfoMap_t::const_iterator it; - uint32_t fboLo = 0; - uint32_t fboHi = 0; + uint32_t fboLo = 0; + uint32_t fboHi = 0; - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - it = extentsInfo.begin(); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + it = extentsInfo.begin(); - if ( it->second.newFile ) //The extent is the new extent + if (it->second.newFile) // The extent is the new extent + { + for (int i = 0; i < emEntries; i++) { - for (int i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - it = extentsInfo.find ( fExtentMap[i].fileID ); + if (fExtentMap[i].range.size != 0) + { + it = extentsInfo.find(fExtentMap[i].fileID); - if ( it != extentsInfo.end() ) - { - if ((fExtentMap[i].partitionNum == it->second.partitionNum) - && (fExtentMap[i].segmentNum == it->second.segmentNum) - && (fExtentMap[i].dbRoot == it->second.dbRoot) ) - deleteExtent( i ); - } - } + if (it != extentsInfo.end()) + { + if ((fExtentMap[i].partitionNum == it->second.partitionNum) && + (fExtentMap[i].segmentNum == it->second.segmentNum) && + (fExtentMap[i].dbRoot == it->second.dbRoot)) + deleteExtent(i); } + } } - else //The extent is the old one + } + else // The extent is the old one + { + for (int i = 0; i < emEntries; i++) { + if (fExtentMap[i].range.size != 0) + { + it = extentsInfo.find(fExtentMap[i].fileID); - for (int i = 0; i < emEntries; i++) + if (it != extentsInfo.end()) { - if (fExtentMap[i].range.size != 0) + // Don't rollback extents that are out of service + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + continue; + + // Calculate fbo + if (fboHi == 0) + { + uint32_t range = fExtentMap[i].range.size * 1024; + fboLo = it->second.hwm - (it->second.hwm % range); + fboHi = fboLo + range - 1; + } + + // Delete, update, or ignore this extent: + // Later partition: + // case 1: extent is in later partition, so delete the extent + // Same partition: + // case 2: extent is in partition and segment file of interest + // case 2A: earlier extent in segment file; no action necessary + // case 2B: specified HWM falls in this extent, so reset HWM + // case 2C: later extent in segment file; so delete the extent + // Earlier partition: + // case 3: extent is in earlier parition, no action necessary + + if (fExtentMap[i].partitionNum > it->second.partitionNum) + { + deleteExtent(i); // case 1 + } + else if (fExtentMap[i].partitionNum == it->second.partitionNum) + { + if (fExtentMap[i].segmentNum == it->second.segmentNum) { - it = extentsInfo.find ( fExtentMap[i].fileID ); - - if ( it != extentsInfo.end() ) + if (fExtentMap[i].blockOffset < fboLo) + { + // no action necessary case 2A + } + else if (fExtentMap[i].blockOffset == fboLo) + { + if (fExtentMap[i].HWM != it->second.hwm) { - // Don't rollback extents that are out of service - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - continue; - - // Calculate fbo - if (fboHi == 0) - { - uint32_t range = fExtentMap[i].range.size * 1024; - fboLo = it->second.hwm - (it->second.hwm % range); - fboHi = fboLo + range - 1; - } - - // Delete, update, or ignore this extent: - // Later partition: - // case 1: extent is in later partition, so delete the extent - // Same partition: - // case 2: extent is in partition and segment file of interest - // case 2A: earlier extent in segment file; no action necessary - // case 2B: specified HWM falls in this extent, so reset HWM - // case 2C: later extent in segment file; so delete the extent - // Earlier partition: - // case 3: extent is in earlier parition, no action necessary - - if (fExtentMap[i].partitionNum > it->second.partitionNum) - { - deleteExtent( i ); // case 1 - } - else if (fExtentMap[i].partitionNum == it->second.partitionNum) - { - if ( fExtentMap[i].segmentNum == it->second.segmentNum) - { - if (fExtentMap[i].blockOffset < fboLo) - { - // no action necessary case 2A - } - else if (fExtentMap[i].blockOffset == fboLo) - { - if (fExtentMap[i].HWM != it->second.hwm) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].HWM = it->second.hwm; - fExtentMap[i].status = EXTENTAVAILABLE;//case 2B - } - } - else - { - deleteExtent( i ); // case 3C - } - } - else - { - // no action necessary - } - } - else - { - // extent in earlier partition; no action necessary case 4 - } - } // extent map entry with matching oid + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].HWM = it->second.hwm; + fExtentMap[i].status = EXTENTAVAILABLE; // case 2B + } + } + else + { + deleteExtent(i); // case 3C + } } - } // loop through the extent map - } + else + { + // no action necessary + } + } + else + { + // extent in earlier partition; no action necessary case 4 + } + } // extent map entry with matching oid + } + } // loop through the extent map + } } //------------------------------------------------------------------------------ // Delete all the extents for the specified OID @@ -4114,55 +3931,51 @@ void ExtentMap::deleteOID(int OID) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteOID"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteOID"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif - bool OIDExists = false; + bool OIDExists = false; #ifdef BRM_DEBUG - if (OID < 0) - { - log("ExtentMap::deleteOID(): OID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("ExtentMap::deleteOID(): OID must be >= 0"); - } + if (OID < 0) + { + log("ExtentMap::deleteOID(): OID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::deleteOID(): OID must be >= 0"); + } #endif - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int emIndex = 0; emIndex < emEntries; emIndex++) + for (int emIndex = 0; emIndex < emEntries; emIndex++) + { + if (fExtentMap[emIndex].range.size > 0 && fExtentMap[emIndex].fileID == OID) { + OIDExists = true; - if (fExtentMap[emIndex].range.size > 0 && - fExtentMap[emIndex].fileID == OID) - { - OIDExists = true; - - deleteExtent( emIndex ); - } + deleteExtent(emIndex); } + } - if (!OIDExists) - { - ostringstream oss; - oss << "ExtentMap::deleteOID(): There are no extent entries for OID " << OID << endl; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } + if (!OIDExists) + { + ostringstream oss; + oss << "ExtentMap::deleteOID(): There are no extent entries for OID " << OID << endl; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } } - - //------------------------------------------------------------------------------ // Delete all the extents for the specified OIDs //------------------------------------------------------------------------------ @@ -4170,162 +3983,159 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("deleteOIDs"); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("deleteOIDs"); + TRACER_WRITE; + } #endif - grabEMEntryTable(WRITE); - grabFreeList(WRITE); - OidsMap_t::const_iterator it; - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); + OidsMap_t::const_iterator it; + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int emIndex = 0; emIndex < emEntries; emIndex++) + for (int emIndex = 0; emIndex < emEntries; emIndex++) + { + if (fExtentMap[emIndex].range.size > 0) { - if (fExtentMap[emIndex].range.size > 0 ) - { - it = OIDs.find ( fExtentMap[emIndex].fileID ); + it = OIDs.find(fExtentMap[emIndex].fileID); - if ( it != OIDs.end() ) - deleteExtent( emIndex ); - } + if (it != OIDs.end()) + deleteExtent(emIndex); } + } } - //------------------------------------------------------------------------------ // Delete the specified extent from the extentmap and return to the free list. // emIndex - the index (from the extent map) of the extent to be deleted //------------------------------------------------------------------------------ void ExtentMap::deleteExtent(int emIndex) { - int flIndex, freeFLIndex, flEntries, preceedingExtent, succeedingExtent; - LBID_t flBlockEnd, emBlockEnd; + int flIndex, freeFLIndex, flEntries, preceedingExtent, succeedingExtent; + LBID_t flBlockEnd, emBlockEnd; - flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - emBlockEnd = fExtentMap[emIndex].range.start + - (static_cast(fExtentMap[emIndex].range.size) * 1024); + emBlockEnd = fExtentMap[emIndex].range.start + (static_cast(fExtentMap[emIndex].range.size) * 1024); - //scan the freelist to see where this entry fits in - for (flIndex = 0, preceedingExtent = -1, succeedingExtent = -1, freeFLIndex = -1; - flIndex < flEntries; flIndex++) - { - if (fFreeList[flIndex].size == 0) - freeFLIndex = flIndex; - else - { - flBlockEnd = fFreeList[flIndex].start + - (static_cast(fFreeList[flIndex].size) * 1024); - - if (emBlockEnd == fFreeList[flIndex].start) - succeedingExtent = flIndex; - else if (flBlockEnd == fExtentMap[emIndex].range.start) - preceedingExtent = flIndex; - } - } - - //update the freelist - - //this space is in between 2 blocks in the FL - if (preceedingExtent != -1 && succeedingExtent != -1) - { - makeUndoRecord(&fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); - - // migrate the entry upward if there's a space - if (freeFLIndex < preceedingExtent && freeFLIndex != -1) - { - makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); - memcpy(&fFreeList[freeFLIndex], &fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); - fFreeList[preceedingExtent].size = 0; - preceedingExtent = freeFLIndex; - } - - fFreeList[preceedingExtent].size += fFreeList[succeedingExtent].size + - fExtentMap[emIndex].range.size; - makeUndoRecord(&fFreeList[succeedingExtent], sizeof(InlineLBIDRange)); - fFreeList[succeedingExtent].size = 0; - makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); - fFLShminfo->currentSize -= sizeof(InlineLBIDRange); - } - - //this space has a free block at the end - else if (succeedingExtent != -1) - { - makeUndoRecord(&fFreeList[succeedingExtent], sizeof(InlineLBIDRange)); - - // migrate the entry upward if there's a space - if (freeFLIndex < succeedingExtent && freeFLIndex != -1) - { - makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); - memcpy(&fFreeList[freeFLIndex], &fFreeList[succeedingExtent], sizeof(InlineLBIDRange)); - fFreeList[succeedingExtent].size = 0; - succeedingExtent = freeFLIndex; - } - - fFreeList[succeedingExtent].start = fExtentMap[emIndex].range.start; - fFreeList[succeedingExtent].size += fExtentMap[emIndex].range.size; - } - - //this space has a free block at the beginning - else if (preceedingExtent != -1) - { - makeUndoRecord(&fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); - - // migrate the entry upward if there's a space - if (freeFLIndex < preceedingExtent && freeFLIndex != -1) - { - makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); - memcpy(&fFreeList[freeFLIndex], &fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); - fFreeList[preceedingExtent].size = 0; - preceedingExtent = freeFLIndex; - } - - fFreeList[preceedingExtent].size += fExtentMap[emIndex].range.size; - } - - //the freelist has no adjacent blocks, so make a new entry + // scan the freelist to see where this entry fits in + for (flIndex = 0, preceedingExtent = -1, succeedingExtent = -1, freeFLIndex = -1; flIndex < flEntries; + flIndex++) + { + if (fFreeList[flIndex].size == 0) + freeFLIndex = flIndex; else { - if (fFLShminfo->currentSize == fFLShminfo->allocdSize) - { - growFLShmseg(); -#ifdef BRM_DEBUG + flBlockEnd = fFreeList[flIndex].start + (static_cast(fFreeList[flIndex].size) * 1024); - if (freeFLIndex != -1) - { - log("ExtentMap::deleteOID(): found a free FL entry in a supposedly full shmseg", logging::LOG_TYPE_DEBUG); - throw logic_error("ExtentMap::deleteOID(): found a free FL entry in a supposedly full shmseg"); - } + if (emBlockEnd == fFreeList[flIndex].start) + succeedingExtent = flIndex; + else if (flBlockEnd == fExtentMap[emIndex].range.start) + preceedingExtent = flIndex; + } + } -#endif - freeFLIndex = flEntries; // happens to be the right index - flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - } + // update the freelist -#ifdef BRM_DEBUG + // this space is in between 2 blocks in the FL + if (preceedingExtent != -1 && succeedingExtent != -1) + { + makeUndoRecord(&fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); - if (freeFLIndex == -1) - { - log("ExtentMap::deleteOID(): no available free list entries?", logging::LOG_TYPE_DEBUG); - throw logic_error("ExtentMap::deleteOID(): no available free list entries?"); - } - -#endif - makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); - fFreeList[freeFLIndex].start = fExtentMap[emIndex].range.start; - fFreeList[freeFLIndex].size = fExtentMap[emIndex].range.size; - makeUndoRecord(&fFLShminfo, sizeof(MSTEntry)); - fFLShminfo->currentSize += sizeof(InlineLBIDRange); + // migrate the entry upward if there's a space + if (freeFLIndex < preceedingExtent && freeFLIndex != -1) + { + makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); + memcpy(&fFreeList[freeFLIndex], &fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); + fFreeList[preceedingExtent].size = 0; + preceedingExtent = freeFLIndex; } - //invalidate the entry in the Extent Map - makeUndoRecord(&fExtentMap[emIndex], sizeof(EMEntry)); - fExtentMap[emIndex].range.size = 0; - makeUndoRecord(&fEMShminfo, sizeof(MSTEntry)); - fEMShminfo->currentSize -= sizeof(struct EMEntry); + fFreeList[preceedingExtent].size += fFreeList[succeedingExtent].size + fExtentMap[emIndex].range.size; + makeUndoRecord(&fFreeList[succeedingExtent], sizeof(InlineLBIDRange)); + fFreeList[succeedingExtent].size = 0; + makeUndoRecord(fFLShminfo, sizeof(MSTEntry)); + fFLShminfo->currentSize -= sizeof(InlineLBIDRange); + } + + // this space has a free block at the end + else if (succeedingExtent != -1) + { + makeUndoRecord(&fFreeList[succeedingExtent], sizeof(InlineLBIDRange)); + + // migrate the entry upward if there's a space + if (freeFLIndex < succeedingExtent && freeFLIndex != -1) + { + makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); + memcpy(&fFreeList[freeFLIndex], &fFreeList[succeedingExtent], sizeof(InlineLBIDRange)); + fFreeList[succeedingExtent].size = 0; + succeedingExtent = freeFLIndex; + } + + fFreeList[succeedingExtent].start = fExtentMap[emIndex].range.start; + fFreeList[succeedingExtent].size += fExtentMap[emIndex].range.size; + } + + // this space has a free block at the beginning + else if (preceedingExtent != -1) + { + makeUndoRecord(&fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); + + // migrate the entry upward if there's a space + if (freeFLIndex < preceedingExtent && freeFLIndex != -1) + { + makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); + memcpy(&fFreeList[freeFLIndex], &fFreeList[preceedingExtent], sizeof(InlineLBIDRange)); + fFreeList[preceedingExtent].size = 0; + preceedingExtent = freeFLIndex; + } + + fFreeList[preceedingExtent].size += fExtentMap[emIndex].range.size; + } + + // the freelist has no adjacent blocks, so make a new entry + else + { + if (fFLShminfo->currentSize == fFLShminfo->allocdSize) + { + growFLShmseg(); +#ifdef BRM_DEBUG + + if (freeFLIndex != -1) + { + log("ExtentMap::deleteOID(): found a free FL entry in a supposedly full shmseg", + logging::LOG_TYPE_DEBUG); + throw logic_error("ExtentMap::deleteOID(): found a free FL entry in a supposedly full shmseg"); + } + +#endif + freeFLIndex = flEntries; // happens to be the right index + flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + } + +#ifdef BRM_DEBUG + + if (freeFLIndex == -1) + { + log("ExtentMap::deleteOID(): no available free list entries?", logging::LOG_TYPE_DEBUG); + throw logic_error("ExtentMap::deleteOID(): no available free list entries?"); + } + +#endif + makeUndoRecord(&fFreeList[freeFLIndex], sizeof(InlineLBIDRange)); + fFreeList[freeFLIndex].start = fExtentMap[emIndex].range.start; + fFreeList[freeFLIndex].size = fExtentMap[emIndex].range.size; + makeUndoRecord(&fFLShminfo, sizeof(MSTEntry)); + fFLShminfo->currentSize += sizeof(InlineLBIDRange); + } + + // invalidate the entry in the Extent Map + makeUndoRecord(&fExtentMap[emIndex], sizeof(EMEntry)); + fExtentMap[emIndex].range.size = 0; + makeUndoRecord(&fEMShminfo, sizeof(MSTEntry)); + fEMShminfo->currentSize -= sizeof(struct EMEntry); } //------------------------------------------------------------------------------ @@ -4339,81 +4149,77 @@ void ExtentMap::deleteExtent(int emIndex) // If no available or outOfService extent is found, then bFound is returned // as false. //------------------------------------------------------------------------------ -HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, - uint32_t& partitionNum, uint16_t& segmentNum, int& status, bool& bFound) +HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + int& status, bool& bFound) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getLastHWM_DBroot"); - TRACER_ADDINPUT(OID); - TRACER_ADDSHORTINPUT(dbRoot); - TRACER_ADDOUTPUT(partitionNum); - TRACER_ADDSHORTOUTPUT(segmentNum); - TRACER_ADDOUTPUT(status); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getLastHWM_DBroot"); + TRACER_ADDINPUT(OID); + TRACER_ADDSHORTINPUT(dbRoot); + TRACER_ADDOUTPUT(partitionNum); + TRACER_ADDSHORTOUTPUT(segmentNum); + TRACER_ADDOUTPUT(status); + TRACER_WRITE; + } #endif - uint32_t lastExtent = 0; - int lastExtentIndex = -1; - partitionNum = 0; - segmentNum = 0; - HWM_t hwm = 0; - bFound = false; + uint32_t lastExtent = 0; + int lastExtentIndex = -1; + partitionNum = 0; + segmentNum = 0; + HWM_t hwm = 0; + bFound = false; - if (OID < 0) + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getLastHWM_DBroot(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + + // Searching the array in reverse order should be faster since the last + // extent is usually at the bottom. We still have to search the entire + // array (just in case), but the number of operations per loop iteration + // will be less. + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (int i = emEntries - 1; i >= 0; i--) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && + (fExtentMap[i].dbRoot == dbRoot) && + ((fExtentMap[i].status == EXTENTAVAILABLE) || (fExtentMap[i].status == EXTENTOUTOFSERVICE))) { - ostringstream oss; - oss << "ExtentMap::getLastHWM_DBroot(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + if ((fExtentMap[i].partitionNum > partitionNum) || + ((fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].blockOffset > lastExtent)) || + ((fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].blockOffset == lastExtent) && + (fExtentMap[i].segmentNum >= segmentNum))) + { + lastExtent = fExtentMap[i].blockOffset; + partitionNum = fExtentMap[i].partitionNum; + segmentNum = fExtentMap[i].segmentNum; + lastExtentIndex = i; + } } + } - grabEMEntryTable(READ); + // save additional information before we release the read-lock + if (lastExtentIndex != -1) + { + hwm = fExtentMap[lastExtentIndex].HWM; + status = fExtentMap[lastExtentIndex].status; + bFound = true; + } - // Searching the array in reverse order should be faster since the last - // extent is usually at the bottom. We still have to search the entire - // array (just in case), but the number of operations per loop iteration - // will be less. - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + releaseEMEntryTable(READ); - for (int i = emEntries - 1; i >= 0; i--) - { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == OID) && - (fExtentMap[i].dbRoot == dbRoot) && - ((fExtentMap[i].status == EXTENTAVAILABLE) || - (fExtentMap[i].status == EXTENTOUTOFSERVICE))) - { - if ( (fExtentMap[i].partitionNum > partitionNum) || - ((fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].blockOffset > lastExtent)) || - ((fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].blockOffset == lastExtent) && - (fExtentMap[i].segmentNum >= segmentNum)) ) - { - lastExtent = fExtentMap[i].blockOffset; - partitionNum = fExtentMap[i].partitionNum; - segmentNum = fExtentMap[i].segmentNum; - lastExtentIndex = i; - } - } - } - - // save additional information before we release the read-lock - if (lastExtentIndex != -1) - { - hwm = fExtentMap[lastExtentIndex].HWM; - status = fExtentMap[lastExtentIndex].status; - bFound = true; - } - - releaseEMEntryTable(READ); - - return hwm; + return hwm; } //------------------------------------------------------------------------------ @@ -4421,146 +4227,136 @@ HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, // of objects carrying HWM info (for the last segment file) and block count // information about each DBRoot assigned to the specified PM. //------------------------------------------------------------------------------ -void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, - EmDbRootHWMInfo_v& emDbRootHwmInfos) +void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& emDbRootHwmInfos) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getDbRootHWMInfo"); - TRACER_ADDINPUT(OID); - TRACER_ADDSHORTINPUT(pmNumber); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getDbRootHWMInfo"); + TRACER_ADDINPUT(OID); + TRACER_ADDSHORTINPUT(pmNumber); + TRACER_WRITE; + } #endif - if (OID < 0) - { - ostringstream oss; - oss << "ExtentMap::getDbRootHWMInfo(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getDbRootHWMInfo(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } - // Determine List of DBRoots for specified PM, and construct map of - // EmDbRootHWMInfo objects. - tr1::unordered_map emDbRootMap; - vector dbRootList; - getPmDbRoots( pmNumber, dbRootList ); + // Determine List of DBRoots for specified PM, and construct map of + // EmDbRootHWMInfo objects. + tr1::unordered_map emDbRootMap; + vector dbRootList; + getPmDbRoots(pmNumber, dbRootList); - if ( dbRootList.size() > 0 ) + if (dbRootList.size() > 0) + { + for (unsigned int iroot = 0; iroot < dbRootList.size(); iroot++) { - for (unsigned int iroot = 0; iroot < dbRootList.size(); iroot++) - { - uint16_t rootID = dbRootList[iroot]; - EmDbRootHWMInfo emDbRootInfo(rootID); - emDbRootMap[rootID] = emDbRootInfo; - } + uint16_t rootID = dbRootList[iroot]; + EmDbRootHWMInfo emDbRootInfo(rootID); + emDbRootMap[rootID] = emDbRootInfo; } - else + } + else + { + ostringstream oss; + oss << "ExtentMap::getDbRootHWMInfo(): " + "There are no DBRoots for OID " + << OID << " and PM " << pmNumber << endl; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + tr1::unordered_map::iterator emIter; + + // Searching the array in reverse order should be faster since the last + // extent is usually at the bottom. We still have to search the entire + // array (just in case), but the number of operations per loop iteration + // will be less. + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (int i = emEntries - 1; i >= 0; i--) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID)) { + // Include this extent in the search, only if the extent's + // DBRoot falls in the list of DBRoots for this PM. + emIter = emDbRootMap.find(fExtentMap[i].dbRoot); + + if (emIter == emDbRootMap.end()) + continue; + + EmDbRootHWMInfo& emDbRoot = emIter->second; + + if ((fExtentMap[i].status != EXTENTOUTOFSERVICE) && (fExtentMap[i].HWM != 0)) + emDbRoot.totalBlocks += (fExtentMap[i].HWM + 1); + + if ((fExtentMap[i].partitionNum > emDbRoot.partitionNum) || + ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && + (fExtentMap[i].blockOffset > emDbRoot.fbo)) || + ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && + (fExtentMap[i].blockOffset == emDbRoot.fbo) && (fExtentMap[i].segmentNum >= emDbRoot.segmentNum))) + { + emDbRoot.fbo = fExtentMap[i].blockOffset; + emDbRoot.partitionNum = fExtentMap[i].partitionNum; + emDbRoot.segmentNum = fExtentMap[i].segmentNum; + emDbRoot.localHWM = fExtentMap[i].HWM; + emDbRoot.startLbid = fExtentMap[i].range.start; + emDbRoot.status = fExtentMap[i].status; + emDbRoot.hwmExtentIndex = i; + } + } + } + + releaseEMEntryTable(READ); + + for (tr1::unordered_map::iterator iter = emDbRootMap.begin(); + iter != emDbRootMap.end(); ++iter) + { + EmDbRootHWMInfo& emDbRoot = iter->second; + + if (emDbRoot.hwmExtentIndex != -1) + { + // @bug 5349: make sure HWM extent for each DBRoot is AVAILABLE + if (emDbRoot.status == EXTENTUNAVAILABLE) + { ostringstream oss; oss << "ExtentMap::getDbRootHWMInfo(): " - "There are no DBRoots for OID " << OID << - " and PM " << pmNumber << endl; + << "OID " << OID << " has HWM extent that is UNAVAILABLE for " + << "DBRoot" << emDbRoot.dbRoot << "; part#: " << emDbRoot.partitionNum + << ", seg#: " << emDbRoot.segmentNum << ", fbo: " << emDbRoot.fbo + << ", localHWM: " << emDbRoot.localHWM << ", lbid: " << emDbRoot.startLbid << endl; log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + throw runtime_error(oss.str()); + } + + // In the loop above we ignored "all" the extents with HWM of 0, + // which is okay most of the time, because each segment file's HWM + // is carried in the last extent only. BUT if we have a segment + // file with HWM=0, having a single extent and a single block at + // the "end" of the data, we still need to account for this last + // block. So we increment the block count for this isolated case. + if ((emDbRoot.localHWM == 0) && (emDbRoot.status == EXTENTAVAILABLE)) + { + emDbRoot.totalBlocks++; + } } + } - grabEMEntryTable(READ); - tr1::unordered_map::iterator emIter; - - // Searching the array in reverse order should be faster since the last - // extent is usually at the bottom. We still have to search the entire - // array (just in case), but the number of operations per loop iteration - // will be less. - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (int i = emEntries - 1; i >= 0; i--) - { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == OID)) - { - - // Include this extent in the search, only if the extent's - // DBRoot falls in the list of DBRoots for this PM. - emIter = emDbRootMap.find( fExtentMap[i].dbRoot ); - - if (emIter == emDbRootMap.end()) - continue; - - EmDbRootHWMInfo& emDbRoot = emIter->second; - - if ((fExtentMap[i].status != EXTENTOUTOFSERVICE) && - (fExtentMap[i].HWM != 0)) - emDbRoot.totalBlocks += (fExtentMap[i].HWM + 1); - - if ( (fExtentMap[i].partitionNum > emDbRoot.partitionNum) || - ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && - (fExtentMap[i].blockOffset > emDbRoot.fbo)) || - ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && - (fExtentMap[i].blockOffset == emDbRoot.fbo) && - (fExtentMap[i].segmentNum >= emDbRoot.segmentNum)) ) - { - emDbRoot.fbo = fExtentMap[i].blockOffset; - emDbRoot.partitionNum = fExtentMap[i].partitionNum; - emDbRoot.segmentNum = fExtentMap[i].segmentNum; - emDbRoot.localHWM = fExtentMap[i].HWM; - emDbRoot.startLbid = fExtentMap[i].range.start; - emDbRoot.status = fExtentMap[i].status; - emDbRoot.hwmExtentIndex = i; - } - } - } - - releaseEMEntryTable(READ); - - for (tr1::unordered_map::iterator iter = - emDbRootMap.begin(); iter != emDbRootMap.end(); ++iter) - { - EmDbRootHWMInfo& emDbRoot = iter->second; - - if (emDbRoot.hwmExtentIndex != -1) - { - // @bug 5349: make sure HWM extent for each DBRoot is AVAILABLE - if (emDbRoot.status == EXTENTUNAVAILABLE) - { - ostringstream oss; - oss << "ExtentMap::getDbRootHWMInfo(): " << - "OID " << OID << - " has HWM extent that is UNAVAILABLE for " << - "DBRoot" << emDbRoot.dbRoot << - "; part#: " << emDbRoot.partitionNum << - ", seg#: " << emDbRoot.segmentNum << - ", fbo: " << emDbRoot.fbo << - ", localHWM: " << emDbRoot.localHWM << - ", lbid: " << emDbRoot.startLbid << endl; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw runtime_error(oss.str()); - } - - // In the loop above we ignored "all" the extents with HWM of 0, - // which is okay most of the time, because each segment file's HWM - // is carried in the last extent only. BUT if we have a segment - // file with HWM=0, having a single extent and a single block at - // the "end" of the data, we still need to account for this last - // block. So we increment the block count for this isolated case. - if ((emDbRoot.localHWM == 0) && - (emDbRoot.status == EXTENTAVAILABLE)) - { - emDbRoot.totalBlocks++; - } - } - } - - // Copy internal map to the output vector argument - for (tr1::unordered_map::iterator iter = - emDbRootMap.begin(); iter != emDbRootMap.end(); ++iter) - { - emDbRootHwmInfos.push_back( iter->second ); - } + // Copy internal map to the output vector argument + for (tr1::unordered_map::iterator iter = emDbRootMap.begin(); + iter != emDbRootMap.end(); ++iter) + { + emDbRootHwmInfos.push_back(iter->second); + } } //------------------------------------------------------------------------------ @@ -4571,132 +4367,125 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, // The value returned in the "status" variable is based on the first extent // found, since all the extents in a segment file should have the same state. //------------------------------------------------------------------------------ -void ExtentMap::getExtentState(int OID, uint32_t partitionNum, - uint16_t segmentNum, bool& bFound, int& status) +void ExtentMap::getExtentState(int OID, uint32_t partitionNum, uint16_t segmentNum, bool& bFound, int& status) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtentState"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDOUTPUT(status); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtentState"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDOUTPUT(status); + TRACER_WRITE; + } #endif - int i, emEntries; - bFound = false; - status = EXTENTAVAILABLE; + int i, emEntries; + bFound = false; + status = EXTENTAVAILABLE; - if (OID < 0) + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getExtentState(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && + (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) { - ostringstream oss; - oss << "ExtentMap::getExtentState(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + bFound = true; + status = fExtentMap[i].status; + break; } + } - grabEMEntryTable(READ); - - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum)) - { - bFound = true; - status = fExtentMap[i].status; - break; - } - } - - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); } //------------------------------------------------------------------------------ // Returns the HWM for the specified OID, partition, and segment numbers. // Used to get the HWM for a specific column or dictionary store segment file. //------------------------------------------------------------------------------ -HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, - uint16_t segmentNum, int& status) +HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, int& status) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getLocalHWM"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDOUTPUT(status); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getLocalHWM"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDOUTPUT(status); + TRACER_WRITE; + } #endif #ifdef EM_AS_A_TABLE_POC__ - if (OID == 1084) - { - return 0; - } + if (OID == 1084) + { + return 0; + } #endif - int i, emEntries; - HWM_t ret = 0; - bool OIDPartSegExists = false; + int i, emEntries; + HWM_t ret = 0; + bool OIDPartSegExists = false; - if (OID < 0) + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getLocalHWM(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && + (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) { - ostringstream oss; - oss << "ExtentMap::getLocalHWM(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + OIDPartSegExists = true; + status = fExtentMap[i].status; + + if (fExtentMap[i].HWM != 0) + { + ret = fExtentMap[i].HWM; + releaseEMEntryTable(READ); + return ret; + } } + } - grabEMEntryTable(READ); + releaseEMEntryTable(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum)) - { - OIDPartSegExists = true; - status = fExtentMap[i].status; - - if (fExtentMap[i].HWM != 0) - { - ret = fExtentMap[i].HWM; - releaseEMEntryTable(READ); - return ret; - } - } - } - - releaseEMEntryTable(READ); - - if (OIDPartSegExists) - return 0; - else - { - ostringstream oss; - oss << "ExtentMap::getLocalHWM(): There are no extent entries for OID " << - OID << "; partition " << partitionNum << "; segment " << - segmentNum << endl; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } + if (OIDPartSegExists) + return 0; + else + { + ostringstream oss; + oss << "ExtentMap::getLocalHWM(): There are no extent entries for OID " << OID << "; partition " + << partitionNum << "; segment " << segmentNum << endl; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } } //------------------------------------------------------------------------------ @@ -4706,302 +4495,285 @@ HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, // (per segment file). // Used for dictionary or column OIDs to set the HWM for specific segment file. //------------------------------------------------------------------------------ -void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, - uint16_t segmentNum, HWM_t newHWM, bool firstNode, bool uselock) +void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, HWM_t newHWM, bool firstNode, + bool uselock) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("setLocalHWM"); - TRACER_ADDINPUT(OID); - TRACER_ADDINPUT(partitionNum); - TRACER_ADDSHORTINPUT(segmentNum); - TRACER_ADDINPUT(newHWM); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("setLocalHWM"); + TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(partitionNum); + TRACER_ADDSHORTINPUT(segmentNum); + TRACER_ADDINPUT(newHWM); + TRACER_WRITE; + } - bool addedAnExtent = false; + bool addedAnExtent = false; - if (OID < 0) - { - log("ExtentMap::setLocalHWM(): OID must be >= 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument( - "ExtentMap::setLocalHWM(): OID must be >= 0"); - } + if (OID < 0) + { + log("ExtentMap::setLocalHWM(): OID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("ExtentMap::setLocalHWM(): OID must be >= 0"); + } #endif - int lastExtentIndex = -1; - int oldHWMExtentIndex = -1; - uint32_t highestOffset = 0; + int lastExtentIndex = -1; + int oldHWMExtentIndex = -1; + uint32_t highestOffset = 0; - if (uselock) - grabEMEntryTable(WRITE); + if (uselock) + grabEMEntryTable(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && + (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum)) - { + // Find current HWM extent + if (fExtentMap[i].blockOffset >= highestOffset) + { + highestOffset = fExtentMap[i].blockOffset; + lastExtentIndex = i; + } - // Find current HWM extent - if (fExtentMap[i].blockOffset >= highestOffset) - { - highestOffset = fExtentMap[i].blockOffset; - lastExtentIndex = i; - } - - // Find previous HWM extent - if (fExtentMap[i].HWM != 0) - { - oldHWMExtentIndex = i; - } - } + // Find previous HWM extent + if (fExtentMap[i].HWM != 0) + { + oldHWMExtentIndex = i; + } } + } - if (lastExtentIndex == -1) - { - ostringstream oss; - oss << "ExtentMap::setLocalHWM(): Bad OID/partition/segment argument; " - "no extent entries for OID " << OID << "; partition " << - partitionNum << "; segment " << segmentNum << endl; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } + if (lastExtentIndex == -1) + { + ostringstream oss; + oss << "ExtentMap::setLocalHWM(): Bad OID/partition/segment argument; " + "no extent entries for OID " + << OID << "; partition " << partitionNum << "; segment " << segmentNum << endl; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } - if (newHWM >= (fExtentMap[lastExtentIndex].blockOffset + - fExtentMap[lastExtentIndex].range.size * 1024)) - { - ostringstream oss; - oss << "ExtentMap::setLocalHWM(): " - "new HWM is past the end of the file for OID " << OID << "; partition " << - partitionNum << "; segment " << segmentNum << "; HWM " << newHWM; - log(oss.str(), logging::LOG_TYPE_DEBUG); - throw invalid_argument(oss.str()); - } + if (newHWM >= (fExtentMap[lastExtentIndex].blockOffset + fExtentMap[lastExtentIndex].range.size * 1024)) + { + ostringstream oss; + oss << "ExtentMap::setLocalHWM(): " + "new HWM is past the end of the file for OID " + << OID << "; partition " << partitionNum << "; segment " << segmentNum << "; HWM " << newHWM; + log(oss.str(), logging::LOG_TYPE_DEBUG); + throw invalid_argument(oss.str()); + } - // Save HWM in last extent for this segment file; and mark as AVAILABLE - makeUndoRecord(&fExtentMap[lastExtentIndex], sizeof(EMEntry)); - fExtentMap[lastExtentIndex].HWM = newHWM; - fExtentMap[lastExtentIndex].status = EXTENTAVAILABLE; + // Save HWM in last extent for this segment file; and mark as AVAILABLE + makeUndoRecord(&fExtentMap[lastExtentIndex], sizeof(EMEntry)); + fExtentMap[lastExtentIndex].HWM = newHWM; + fExtentMap[lastExtentIndex].status = EXTENTAVAILABLE; - // Reset HWM in old HWM extent to 0 - if ((oldHWMExtentIndex != -1) && (oldHWMExtentIndex != lastExtentIndex)) - { - makeUndoRecord(&fExtentMap[oldHWMExtentIndex], sizeof(EMEntry)); - fExtentMap[oldHWMExtentIndex].HWM = 0; -#ifdef BRM_INFO - addedAnExtent = true; + // Reset HWM in old HWM extent to 0 + if ((oldHWMExtentIndex != -1) && (oldHWMExtentIndex != lastExtentIndex)) + { + makeUndoRecord(&fExtentMap[oldHWMExtentIndex], sizeof(EMEntry)); + fExtentMap[oldHWMExtentIndex].HWM = 0; +#ifdef BRM_INFO + addedAnExtent = true; #endif - } + } -#ifdef BRM_INFO +#ifdef BRM_INFO - if (firstNode) + if (firstNode) + { + ostringstream os; + os << "ExtentMap::setLocalHWM(): firstLBID=" << fExtentMap[lastExtentIndex].range.start << " lastLBID=" + << fExtentMap[lastExtentIndex].range.start + fExtentMap[lastExtentIndex].range.size * 1024 - 1 + << " newHWM=" << fExtentMap[lastExtentIndex].HWM + << " min=" << fExtentMap[lastExtentIndex].partition.cprange.loVal + << " max=" << fExtentMap[lastExtentIndex].partition.cprange.hiVal + << " seq=" << fExtentMap[lastExtentIndex].partition.cprange.sequenceNum << " status="; + + switch (fExtentMap[lastExtentIndex].partition.cprange.isValid) { - ostringstream os; - os << "ExtentMap::setLocalHWM(): firstLBID=" << fExtentMap[lastExtentIndex].range.start << - " lastLBID=" << fExtentMap[lastExtentIndex].range.start + - fExtentMap[lastExtentIndex].range.size * 1024 - 1 << " newHWM=" << fExtentMap[lastExtentIndex].HWM - << " min=" << fExtentMap[lastExtentIndex].partition.cprange.loVal << " max=" << - fExtentMap[lastExtentIndex].partition.cprange.hiVal << " seq=" << - fExtentMap[lastExtentIndex].partition.cprange.sequenceNum << " status="; + case CP_INVALID: os << "invalid."; break; - switch (fExtentMap[lastExtentIndex].partition.cprange.isValid) - { - case CP_INVALID: - os << "invalid."; - break; + case CP_UPDATING: os << "updating."; break; - case CP_UPDATING: - os << "updating."; - break; + case CP_VALID: os << "valid."; break; - case CP_VALID: - os << "valid."; - break; - - default: - os << "unknown(!!)."; - break; - } - - if (addedAnExtent) - os << " Data extended into a new extent."; - - log(os.str(), logging::LOG_TYPE_DEBUG); + default: os << "unknown(!!)."; break; } + if (addedAnExtent) + os << " Data extended into a new extent."; + + log(os.str(), logging::LOG_TYPE_DEBUG); + } + #endif } void ExtentMap::bulkSetHWM(const vector& v, bool firstNode) { - grabEMEntryTable(WRITE); + grabEMEntryTable(WRITE); - for (uint32_t i = 0; i < v.size(); i++) - setLocalHWM(v[i].oid, v[i].partNum, v[i].segNum, v[i].hwm, firstNode, false); + for (uint32_t i = 0; i < v.size(); i++) + setLocalHWM(v[i].oid, v[i].partNum, v[i].segNum, v[i].hwm, firstNode, false); } class BUHasher { -public: - inline uint64_t operator()(const BulkUpdateDBRootArg& b) const - { - return b.startLBID; - } + public: + inline uint64_t operator()(const BulkUpdateDBRootArg& b) const + { + return b.startLBID; + } }; class BUEqual { -public: - inline bool operator()(const BulkUpdateDBRootArg& b1, const BulkUpdateDBRootArg& b2) const - { - return b1.startLBID == b2.startLBID; - } + public: + inline bool operator()(const BulkUpdateDBRootArg& b1, const BulkUpdateDBRootArg& b2) const + { + return b1.startLBID == b2.startLBID; + } }; void ExtentMap::bulkUpdateDBRoot(const vector& args) { - tr1::unordered_set sArgs; - tr1::unordered_set::iterator sit; - BulkUpdateDBRootArg key; - int emEntries; + tr1::unordered_set sArgs; + tr1::unordered_set::iterator sit; + BulkUpdateDBRootArg key; + int emEntries; - for (uint32_t i = 0; i < args.size(); i++) - sArgs.insert(args[i]); + for (uint32_t i = 0; i < args.size(); i++) + sArgs.insert(args[i]); - grabEMEntryTable(WRITE); + grabEMEntryTable(WRITE); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0; i < emEntries; i++) - { - key.startLBID = fExtentMap[i].range.start; - sit = sArgs.find(key); + for (int i = 0; i < emEntries; i++) + { + key.startLBID = fExtentMap[i].range.start; + sit = sArgs.find(key); - if (sit != sArgs.end()) - fExtentMap[i].dbRoot = sit->dbRoot; - } + if (sit != sArgs.end()) + fExtentMap[i].dbRoot = sit->dbRoot; + } } -void ExtentMap::getExtents(int OID, vector& entries, - bool sorted, bool notFoundErr, bool incOutOfService) +void ExtentMap::getExtents(int OID, vector& entries, bool sorted, bool notFoundErr, + bool incOutOfService) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtents"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtents"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif - int i, emEntries; + int i, emEntries; - entries.clear(); + entries.clear(); - if (OID < 0) - { - ostringstream oss; - oss << "ExtentMap::getExtents(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getExtents(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } - grabEMEntryTable(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - // Pre-expand entries to stop lots of small allocs - entries.reserve(emEntries); + grabEMEntryTable(READ); + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + // Pre-expand entries to stop lots of small allocs + entries.reserve(emEntries); - if (incOutOfService) - { - for (i = 0 ; i < emEntries; i++) - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].range.size != 0)) - entries.push_back(fExtentMap[i]); - } - else - { - for (i = 0 ; i < emEntries; i++) - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].range.size != 0) && - (fExtentMap[i].status != EXTENTOUTOFSERVICE)) - entries.push_back(fExtentMap[i]); - } + if (incOutOfService) + { + for (i = 0; i < emEntries; i++) + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0)) + entries.push_back(fExtentMap[i]); + } + else + { + for (i = 0; i < emEntries; i++) + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && + (fExtentMap[i].status != EXTENTOUTOFSERVICE)) + entries.push_back(fExtentMap[i]); + } - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); - if (sorted) - sort::iterator>(entries.begin(), entries.end()); + if (sorted) + sort::iterator>(entries.begin(), entries.end()); } void ExtentMap::getExtents_dbroot(int OID, vector& entries, const uint16_t dbroot) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtents"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtents"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif #ifdef EM_AS_A_TABLE_POC__ - if (OID == 1084) - { - EMEntry fakeEntry; - fakeEntry.range.start = (1LL << 54); - fakeEntry.range.size = 4; - fakeEntry.fileID = 1084; - fakeEntry.blockOffset = 0; - fakeEntry.HWM = 1; - fakeEntry.partitionNum = 0; - fakeEntry.segmentNum = 0; - fakeEntry.dbRoot = 1; - fakeEntry.colWid = 4; - fakeEntry.status = EXTENTAVAILABLE; - fakeEntry.partition.cprange.hiVal = numeric_limits::min() + 2; - fakeEntry.partition.cprange.loVal = numeric_limits::max(); - fakeEntry.partition.cprange.sequenceNum = 0; - fakeEntry.partition.cprange.isValid = CP_INVALID; - entries.push_back(fakeEntry); - return; - } + if (OID == 1084) + { + EMEntry fakeEntry; + fakeEntry.range.start = (1LL << 54); + fakeEntry.range.size = 4; + fakeEntry.fileID = 1084; + fakeEntry.blockOffset = 0; + fakeEntry.HWM = 1; + fakeEntry.partitionNum = 0; + fakeEntry.segmentNum = 0; + fakeEntry.dbRoot = 1; + fakeEntry.colWid = 4; + fakeEntry.status = EXTENTAVAILABLE; + fakeEntry.partition.cprange.hiVal = numeric_limits::min() + 2; + fakeEntry.partition.cprange.loVal = numeric_limits::max(); + fakeEntry.partition.cprange.sequenceNum = 0; + fakeEntry.partition.cprange.isValid = CP_INVALID; + entries.push_back(fakeEntry); + return; + } #endif - int i, emEntries; + int i, emEntries; - entries.clear(); + entries.clear(); - if (OID < 0) - { - ostringstream oss; - oss << "ExtentMap::getExtents(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); - } + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getExtents(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } - grabEMEntryTable(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(READ); + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (i = 0 ; i < emEntries; i++) - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].range.size != 0) && (fExtentMap[i].dbRoot == dbroot)) - entries.push_back(fExtentMap[i]); + for (i = 0; i < emEntries; i++) + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && (fExtentMap[i].dbRoot == dbroot)) + entries.push_back(fExtentMap[i]); - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); } //------------------------------------------------------------------------------ @@ -5009,48 +4781,43 @@ void ExtentMap::getExtents_dbroot(int OID, vector& entries, cons // OutOfService extents are included/excluded depending on the // value of the incOutOfService flag. //------------------------------------------------------------------------------ -void ExtentMap::getExtentCount_dbroot(int OID, uint16_t dbroot, - bool incOutOfService, uint64_t& numExtents) +void ExtentMap::getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfService, uint64_t& numExtents) { - int i, emEntries; + int i, emEntries; - if (OID < 0) + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::getExtentsCount_dbroot(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + numExtents = 0; + + if (incOutOfService) + { + for (i = 0; i < emEntries; i++) { - ostringstream oss; - oss << "ExtentMap::getExtentsCount_dbroot(): invalid OID requested: " << - OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && + (fExtentMap[i].dbRoot == dbroot)) + numExtents++; } - - grabEMEntryTable(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - numExtents = 0; - - if (incOutOfService) + } + else + { + for (i = 0; i < emEntries; i++) { - for (i = 0 ; i < emEntries; i++) - { - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].range.size != 0) && - (fExtentMap[i].dbRoot == dbroot)) - numExtents++; - } - } - else - { - for (i = 0 ; i < emEntries; i++) - { - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].range.size != 0) && - (fExtentMap[i].dbRoot == dbroot) && - (fExtentMap[i].status != EXTENTOUTOFSERVICE)) - numExtents++; - } + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && + (fExtentMap[i].dbRoot == dbroot) && (fExtentMap[i].status != EXTENTOUTOFSERVICE)) + numExtents++; } + } - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); } //------------------------------------------------------------------------------ @@ -5063,40 +4830,39 @@ void ExtentMap::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getSysCatDBRoot"); - TRACER_ADDINPUT(oid); - TRACER_ADDSHORTOUTPUT(dbRoot); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getSysCatDBRoot"); + TRACER_ADDINPUT(oid); + TRACER_ADDSHORTOUTPUT(dbRoot); + TRACER_WRITE; + } #endif - bool bFound = false; - grabEMEntryTable(READ); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + bool bFound = false; + grabEMEntryTable(READ); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0 ; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == oid)) { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].fileID == oid)) - { - dbRoot = fExtentMap[i].dbRoot; - bFound = true; - break; - } + dbRoot = fExtentMap[i].dbRoot; + bFound = true; + break; } + } - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); - if (!bFound) - { - ostringstream oss; - oss << "ExtentMap::getSysCatDBRoot(): OID not found: " << oid; - log(oss.str(), logging::LOG_TYPE_WARNING); - throw logic_error(oss.str()); - } + if (!bFound) + { + ostringstream oss; + oss << "ExtentMap::getSysCatDBRoot(): OID not found: " << oid; + log(oss.str(), logging::LOG_TYPE_WARNING); + throw logic_error(oss.str()); + } } //------------------------------------------------------------------------------ @@ -5104,102 +4870,99 @@ void ExtentMap::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) // @bug 5237 - Removed restriction that prevented deletion of segment files in // the last partition (for a DBRoot). //------------------------------------------------------------------------------ -void ExtentMap::deletePartition(const set& oids, - const set& partitionNums, string& emsg) +void ExtentMap::deletePartition(const set& oids, const set& partitionNums, + string& emsg) { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("deletePartition"); + ostringstream oss; + set::const_iterator partIt; + oss << "partitionNums: "; + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + oss << (*partIt) << " "; + + oss << endl; + oss << "OIDS: "; + set::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("deletePartition"); - ostringstream oss; - set::const_iterator partIt; - oss << "partitionNums: "; - for (partIt=partitionNums.begin(); partIt!=partitionNums.end(); ++partIt) - oss << (*partIt) << " "; - - oss << endl; - oss << "OIDS: "; - set::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - if (oids.size() == 0) - return; + if (oids.size() == 0) + return; - int rc = 0; + int rc = 0; - grabEMEntryTable(WRITE); - grabFreeList(WRITE); - set foundPartitions; - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - vector extents; + grabEMEntryTable(WRITE); + grabFreeList(WRITE); + set foundPartitions; + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + vector extents; - // First: validate against referencing non-existent logical partitions - std::set::const_iterator it; + // First: validate against referencing non-existent logical partitions + std::set::const_iterator it; - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + LogicalPartition lp(fExtentMap[i].dbRoot, fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + + if ((fExtentMap[i].range.size != 0) && (partitionNums.find(lp) != partitionNums.end())) { - LogicalPartition lp(fExtentMap[i].dbRoot, - fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + it = oids.find(fExtentMap[i].fileID); - if ((fExtentMap[i].range.size != 0) && - (partitionNums.find(lp) != partitionNums.end())) - { - it = oids.find( fExtentMap[i].fileID ); + if (it != oids.end()) + { + foundPartitions.insert(lp); + extents.push_back(i); + } + } + } - if (it != oids.end()) - { - foundPartitions.insert(lp); - extents.push_back(i); - } - } + if (foundPartitions.size() != partitionNums.size()) + { + set::const_iterator partIt; + Message::Args args; + ostringstream oss; + + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + { + if (foundPartitions.find((*partIt)) == foundPartitions.end()) + { + if (!oss.str().empty()) + oss << ", "; + + oss << (*partIt).toString(); + } } - if (foundPartitions.size() != partitionNums.size()) - { - set::const_iterator partIt; - Message::Args args; - ostringstream oss; + args.add(oss.str()); + emsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST, args); + rc = ERR_PARTITION_NOT_EXIST; + } - for (partIt = partitionNums.begin(); - partIt != partitionNums.end(); ++partIt) - { - if (foundPartitions.find((*partIt)) == foundPartitions.end()) - { - if (!oss.str().empty()) - oss << ", "; + // this has to be the last error code to set and can not be over-written + if (foundPartitions.empty()) + rc = WARN_NO_PARTITION_PERFORMED; - oss << (*partIt).toString(); - } - } + // really delete extents + for (uint32_t i = 0; i < extents.size(); i++) + { + deleteExtent(extents[i]); + } - args.add(oss.str()); - emsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST, args); - rc = ERR_PARTITION_NOT_EXIST; - } - - // this has to be the last error code to set and can not be over-written - if (foundPartitions.empty()) - rc = WARN_NO_PARTITION_PERFORMED; - - // really delete extents - for (uint32_t i = 0; i < extents.size(); i++) - { - deleteExtent(extents[i]); - } - - // @bug 4772 throw exception on any error because they are all warnings. - if (rc) - throw IDBExcept(emsg, rc); + // @bug 4772 throw exception on any error because they are all warnings. + if (rc) + throw IDBExcept(emsg, rc); } //------------------------------------------------------------------------------ @@ -5208,120 +4971,115 @@ void ExtentMap::deletePartition(const set& oids, // @bug 5237 - Removed restriction that prevented deletion of segment files in // the last partition (for a DBRoot). //------------------------------------------------------------------------------ -void ExtentMap::markPartitionForDeletion(const set& oids, - const set& partitionNums, string& emsg) +void ExtentMap::markPartitionForDeletion(const set& oids, const set& partitionNums, + string& emsg) { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("markPartitionForDeletion"); + ostringstream oss; + set::const_iterator partIt; + oss << "partitionNums: "; + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + oss << (*partIt) << " "; + + oss << endl; + oss << "OIDS: "; + set::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("markPartitionForDeletion"); - ostringstream oss; - set::const_iterator partIt; - oss << "partitionNums: "; - for (partIt=partitionNums.begin(); partIt!=partitionNums.end(); ++partIt) - oss << (*partIt) << " "; - - oss << endl; - oss << "OIDS: "; - set::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - if (oids.size() == 0) - return; + if (oids.size() == 0) + return; - int rc = 0; + int rc = 0; - grabEMEntryTable(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - set foundPartitions; - vector extents; - bool partitionAlreadyDisabled = false; + grabEMEntryTable(WRITE); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + set foundPartitions; + vector extents; + bool partitionAlreadyDisabled = false; - // Identify not exists partition first. Then mark disable. - std::set::const_iterator it; + // Identify not exists partition first. Then mark disable. + std::set::const_iterator it; - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + LogicalPartition lp(fExtentMap[i].dbRoot, fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + + if ((fExtentMap[i].range.size != 0) && (partitionNums.find(lp) != partitionNums.end())) { - LogicalPartition lp(fExtentMap[i].dbRoot, - fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + it = oids.find(fExtentMap[i].fileID); - if ((fExtentMap[i].range.size != 0) && - (partitionNums.find(lp) != partitionNums.end())) + if (it != oids.end()) + { + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) { - it = oids.find( fExtentMap[i].fileID ); - - if (it != oids.end()) - { - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - { - partitionAlreadyDisabled = true; - } - - foundPartitions.insert(lp); - extents.push_back(i); - } - } - } - - // really disable partitions - for (uint32_t i = 0; i < extents.size(); i++) - { - makeUndoRecord(&fExtentMap[extents[i]], sizeof(EMEntry)); - fExtentMap[extents[i]].status = EXTENTOUTOFSERVICE; - } - - // validate against referencing non-existent logical partitions - if (foundPartitions.size() != partitionNums.size()) - { - set::const_iterator partIt; - Message::Args args; - ostringstream oss; - - for (partIt = partitionNums.begin(); - partIt != partitionNums.end(); ++partIt) - { - if (foundPartitions.find((*partIt)) == foundPartitions.end()) - { - if (!oss.str().empty()) - oss << ", "; - - oss << (*partIt).toString(); - } + partitionAlreadyDisabled = true; } - args.add(oss.str()); - emsg = emsg + string("\n") + IDBErrorInfo::instance()->errorMsg( - ERR_PARTITION_NOT_EXIST, args); - rc = ERR_PARTITION_NOT_EXIST; + foundPartitions.insert(lp); + extents.push_back(i); + } } + } - // check already disabled error now, which could be a non-error - if (partitionAlreadyDisabled) + // really disable partitions + for (uint32_t i = 0; i < extents.size(); i++) + { + makeUndoRecord(&fExtentMap[extents[i]], sizeof(EMEntry)); + fExtentMap[extents[i]].status = EXTENTOUTOFSERVICE; + } + + // validate against referencing non-existent logical partitions + if (foundPartitions.size() != partitionNums.size()) + { + set::const_iterator partIt; + Message::Args args; + ostringstream oss; + + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) { - emsg = emsg + string("\n") + IDBErrorInfo::instance()->errorMsg( - ERR_PARTITION_ALREADY_DISABLED); - rc = ERR_PARTITION_ALREADY_DISABLED; + if (foundPartitions.find((*partIt)) == foundPartitions.end()) + { + if (!oss.str().empty()) + oss << ", "; + + oss << (*partIt).toString(); + } } - // this rc has to be the last one set and can not be over-written by others. - if (foundPartitions.empty()) - { - rc = WARN_NO_PARTITION_PERFORMED; - } + args.add(oss.str()); + emsg = emsg + string("\n") + IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST, args); + rc = ERR_PARTITION_NOT_EXIST; + } - // @bug 4772 throw exception on any error because they are all warnings. - if (rc) - throw IDBExcept(emsg, rc); + // check already disabled error now, which could be a non-error + if (partitionAlreadyDisabled) + { + emsg = emsg + string("\n") + IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_DISABLED); + rc = ERR_PARTITION_ALREADY_DISABLED; + } + + // this rc has to be the last one set and can not be over-written by others. + if (foundPartitions.empty()) + { + rc = WARN_NO_PARTITION_PERFORMED; + } + + // @bug 4772 throw exception on any error because they are all warnings. + if (rc) + throw IDBExcept(emsg, rc); } //------------------------------------------------------------------------------ @@ -5331,192 +5089,188 @@ void ExtentMap::markAllPartitionForDeletion(const set& oids) { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("markPartitionForDeletion"); + ostringstream oss; + oss << "OIDS: "; + set::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("markPartitionForDeletion"); - ostringstream oss; - oss << "OIDS: "; - set::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - if (oids.size() == 0) - return; + if (oids.size() == 0) + return; - set::const_iterator it; + set::const_iterator it; - grabEMEntryTable(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(WRITE); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0 ) - { - it = oids.find( fExtentMap[i].fileID ); + it = oids.find(fExtentMap[i].fileID); - if (it != oids.end()) - { - makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); - fExtentMap[i].status = EXTENTOUTOFSERVICE; - } - } + if (it != oids.end()) + { + makeUndoRecord(&fExtentMap[i], sizeof(EMEntry)); + fExtentMap[i].status = EXTENTOUTOFSERVICE; + } } + } } //------------------------------------------------------------------------------ // Restore all extents for the specified OID(s) and partition number. //------------------------------------------------------------------------------ -void ExtentMap::restorePartition(const set& oids, - const set& partitionNums, string& emsg) +void ExtentMap::restorePartition(const set& oids, const set& partitionNums, + string& emsg) { #ifdef BRM_INFO - if (fDebug) + if (fDebug) + { + TRACER_WRITENOW("restorePartition"); + ostringstream oss; + set::const_iterator partIt; + oss << "partitionNums: "; + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + oss << (*partIt) << " "; + + oss << endl; + oss << "OIDS: "; + set::const_iterator it; + + for (it = oids.begin(); it != oids.end(); ++it) { - TRACER_WRITENOW("restorePartition"); - ostringstream oss; - set::const_iterator partIt; - oss << "partitionNums: "; - for (partIt=partitionNums.begin(); partIt!=partitionNums.end(); ++partIt) - oss << (*partIt) << " "; - - oss << endl; - oss << "OIDS: "; - set::const_iterator it; - - for (it = oids.begin(); it != oids.end(); ++it) - { - oss << (*it) << ", "; - } - - TRACER_WRITEDIRECT(oss.str()); + oss << (*it) << ", "; } + TRACER_WRITEDIRECT(oss.str()); + } + #endif - if (oids.size() == 0) - return; + if (oids.size() == 0) + return; - set::const_iterator it; - grabEMEntryTable(WRITE); + set::const_iterator it; + grabEMEntryTable(WRITE); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - vector extents; - set foundPartitions; - bool partitionAlreadyEnabled = false; + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + vector extents; + set foundPartitions; + bool partitionAlreadyEnabled = false; - for (int i = 0; i < emEntries; i++) + for (int i = 0; i < emEntries; i++) + { + LogicalPartition lp(fExtentMap[i].dbRoot, fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + + if ((fExtentMap[i].range.size != 0) && partitionNums.find(lp) != partitionNums.end()) { - LogicalPartition lp(fExtentMap[i].dbRoot, fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + it = oids.find(fExtentMap[i].fileID); - if ((fExtentMap[i].range.size != 0 ) && partitionNums.find(lp) != partitionNums.end()) + if (it != oids.end()) + { + if (fExtentMap[i].status == EXTENTAVAILABLE) { - it = oids.find( fExtentMap[i].fileID ); - - if (it != oids.end()) - { - if (fExtentMap[i].status == EXTENTAVAILABLE) - { - partitionAlreadyEnabled = true; - } - - extents.push_back(i); - foundPartitions.insert(lp); - } - } - } - - if (foundPartitions.size() != partitionNums.size()) - { - set::const_iterator partIt; - Message::Args args; - ostringstream oss; - - for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - { - if (foundPartitions.empty() || foundPartitions.find((*partIt)) == foundPartitions.end()) - { - if (!oss.str().empty()) - oss << ", "; - - oss << (*partIt).toString(); - } + partitionAlreadyEnabled = true; } - args.add(oss.str()); - emsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST, args); - throw IDBExcept(emsg, ERR_PARTITION_NOT_EXIST); + extents.push_back(i); + foundPartitions.insert(lp); + } + } + } + + if (foundPartitions.size() != partitionNums.size()) + { + set::const_iterator partIt; + Message::Args args; + ostringstream oss; + + for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) + { + if (foundPartitions.empty() || foundPartitions.find((*partIt)) == foundPartitions.end()) + { + if (!oss.str().empty()) + oss << ", "; + + oss << (*partIt).toString(); + } } - // really enable partitions - for (uint32_t i = 0; i < extents.size(); i++) - { - makeUndoRecord(&fExtentMap[extents[i]], sizeof(EMEntry)); - fExtentMap[extents[i]].status = EXTENTAVAILABLE; - } + args.add(oss.str()); + emsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_NOT_EXIST, args); + throw IDBExcept(emsg, ERR_PARTITION_NOT_EXIST); + } - if (partitionAlreadyEnabled) - { - emsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_ENABLED); - throw IDBExcept(emsg, ERR_PARTITION_ALREADY_ENABLED); - } + // really enable partitions + for (uint32_t i = 0; i < extents.size(); i++) + { + makeUndoRecord(&fExtentMap[extents[i]], sizeof(EMEntry)); + fExtentMap[extents[i]].status = EXTENTAVAILABLE; + } + + if (partitionAlreadyEnabled) + { + emsg = IDBErrorInfo::instance()->errorMsg(ERR_PARTITION_ALREADY_ENABLED); + throw IDBExcept(emsg, ERR_PARTITION_ALREADY_ENABLED); + } } //------------------------------------------------------------------------------ // Return all the out-of-service partitions for the specified OID. //------------------------------------------------------------------------------ -void ExtentMap::getOutOfServicePartitions(OID_t oid, - set& partitionNums) +void ExtentMap::getOutOfServicePartitions(OID_t oid, set& partitionNums) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("getExtents"); - TRACER_ADDINPUT(oid); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("getExtents"); + TRACER_ADDINPUT(oid); + TRACER_WRITE; + } #endif - partitionNums.clear(); + partitionNums.clear(); - if (oid < 0) + if (oid < 0) + { + ostringstream oss; + oss << "ExtentMap::getOutOfServicePartitions(): " + "invalid OID requested: " + << oid; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (int i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == oid) && + (fExtentMap[i].status == EXTENTOUTOFSERVICE)) { - ostringstream oss; - oss << "ExtentMap::getOutOfServicePartitions(): " - "invalid OID requested: " << oid; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + // need to be logical partition number + LogicalPartition lp(fExtentMap[i].dbRoot, fExtentMap[i].partitionNum, fExtentMap[i].segmentNum); + partitionNums.insert(lp); } + } - grabEMEntryTable(READ); - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (int i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0 ) && - (fExtentMap[i].fileID == oid) && - (fExtentMap[i].status == EXTENTOUTOFSERVICE)) - { - - // need to be logical partition number - LogicalPartition lp(fExtentMap[i].dbRoot, - fExtentMap[i].partitionNum, - fExtentMap[i].segmentNum); - partitionNums.insert(lp); - } - } - - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); } //------------------------------------------------------------------------------ @@ -5526,22 +5280,22 @@ void ExtentMap::deleteDBRoot(uint16_t dbroot) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITENOW("deleteDBRoot"); - ostringstream oss; - oss << "dbroot: " << dbroot; - TRACER_WRITEDIRECT(oss.str()); - } + if (fDebug) + { + TRACER_WRITENOW("deleteDBRoot"); + ostringstream oss; + oss << "dbroot: " << dbroot; + TRACER_WRITEDIRECT(oss.str()); + } #endif - grabEMEntryTable(WRITE); - grabFreeList(WRITE); + grabEMEntryTable(WRITE); + grabFreeList(WRITE); - for (unsigned i = 0; i < fEMShminfo->allocdSize / sizeof(struct EMEntry); i++) - if (fExtentMap[i].range.size != 0 && fExtentMap[i].dbRoot == dbroot) - deleteExtent(i); + for (unsigned i = 0; i < fEMShminfo->allocdSize / sizeof(struct EMEntry); i++) + if (fExtentMap[i].range.size != 0 && fExtentMap[i].dbRoot == dbroot) + deleteExtent(i); } //------------------------------------------------------------------------------ @@ -5552,61 +5306,59 @@ bool ExtentMap::isDBRootEmpty(uint16_t dbroot) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("isDBRootEmpty"); - TRACER_ADDINPUT(dbroot); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("isDBRootEmpty"); + TRACER_ADDINPUT(dbroot); + TRACER_WRITE; + } #endif - bool bEmpty = true; - int i, emEntries; - grabEMEntryTable(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + bool bEmpty = true; + int i, emEntries; + grabEMEntryTable(READ); + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - if (fEMShminfo->currentSize == 0) + if (fEMShminfo->currentSize == 0) + { + throw runtime_error("ExtentMap::isDBRootEmpty() shared memory not loaded"); + } + + for (i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].dbRoot == dbroot)) { - throw runtime_error( - "ExtentMap::isDBRootEmpty() shared memory not loaded"); + bEmpty = false; + break; } + } - for (i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0) && - (fExtentMap[i].dbRoot == dbroot)) - { - bEmpty = false; - break; - } - } + releaseEMEntryTable(READ); - releaseEMEntryTable(READ); - - return bEmpty; + return bEmpty; } void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITELATER("lookup"); - TRACER_ADDINPUT(OID); - TRACER_WRITE; - } + if (fDebug) + { + TRACER_WRITELATER("lookup"); + TRACER_ADDINPUT(OID); + TRACER_WRITE; + } #endif #ifdef EM_AS_A_TABLE_POC__ - if (OID == 1084) - { - EMEntry fakeEntry; - fakeEntry.range.start = (1LL << 54); - fakeEntry.range.size = 4; + if (OID == 1084) + { + EMEntry fakeEntry; + fakeEntry.range.start = (1LL << 54); + fakeEntry.range.size = 4; #if 0 fakeEntry.fileID = 1084; fakeEntry.blockOffset = 0; @@ -5621,179 +5373,178 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) fakeEntry.partition.cprange.sequenceNum = 0; fakeEntry.partition.cprange.isValid = CP_INVALID; #endif - ranges.push_back(fakeEntry.range); - return; - } + ranges.push_back(fakeEntry.range); + return; + } #endif - int i, emEntries; - LBIDRange tmp; + int i, emEntries; + LBIDRange tmp; - ranges.clear(); + ranges.clear(); - if (OID < 0) + if (OID < 0) + { + ostringstream oss; + oss << "ExtentMap::lookup(): invalid OID requested: " << OID; + log(oss.str(), logging::LOG_TYPE_CRITICAL); + throw invalid_argument(oss.str()); + } + + grabEMEntryTable(READ); + emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + for (i = 0; i < emEntries; i++) + if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && + (fExtentMap[i].status != EXTENTOUTOFSERVICE)) { - ostringstream oss; - oss << "ExtentMap::lookup(): invalid OID requested: " << OID; - log(oss.str(), logging::LOG_TYPE_CRITICAL); - throw invalid_argument(oss.str()); + tmp.start = fExtentMap[i].range.start; + tmp.size = fExtentMap[i].range.size * 1024; + ranges.push_back(tmp); } - grabEMEntryTable(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0 ; i < emEntries; i++) - if ((fExtentMap[i].fileID == OID) && - (fExtentMap[i].range.size != 0) && - (fExtentMap[i].status != EXTENTOUTOFSERVICE)) - { - tmp.start = fExtentMap[i].range.start; - tmp.size = fExtentMap[i].range.size * 1024; - ranges.push_back(tmp); - } - - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); } - int ExtentMap::checkConsistency() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("checkConsistency"); + if (fDebug) + TRACER_WRITENOW("checkConsistency"); #endif - /* - LBID space consistency checks - 1. verify that every LBID is either in the EM xor the freelist - a. for every segment in the EM, make sure there is no overlapping entry in the FL - b. scan both lists to verify that the entire space is represented - 2. verify that there are no adjacent entries in the freelist - OID consistency - 3. make sure there are no gaps in the file offsets - 4. make sure that only the last extent has a non-zero HWM - Struct integrity - 5. verify that the number of entries in each table is consistent with - the recorded current size - */ + /* + LBID space consistency checks + 1. verify that every LBID is either in the EM xor the freelist + a. for every segment in the EM, make sure there is no overlapping entry in the FL + b. scan both lists to verify that the entire space is represented + 2. verify that there are no adjacent entries in the freelist + OID consistency + 3. make sure there are no gaps in the file offsets + 4. make sure that only the last extent has a non-zero HWM + Struct integrity + 5. verify that the number of entries in each table is consistent with + the recorded current size + */ - LBID_t emBegin, emEnd, flBegin, flEnd; - int i, j, flEntries, emEntries; - uint32_t usedEntries; + LBID_t emBegin, emEnd, flBegin, flEnd; + int i, j, flEntries, emEntries; + uint32_t usedEntries; - grabEMEntryTable(READ); + grabEMEntryTable(READ); - try + try + { + grabFreeList(READ); + } + catch (...) + { + releaseEMEntryTable(READ); + throw; + } + + flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + emEntries = fEMShminfo->allocdSize / sizeof(EMEntry); + + // test 1a - make sure every entry in the EM is not overlapped by an entry in the FL + for (i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) { - grabFreeList(READ); + emBegin = fExtentMap[i].range.start; + emEnd = emBegin + (fExtentMap[i].range.size * 1024) - 1; + + for (j = 0; j < flEntries; j++) + { + if (fFreeList[j].size != 0) + { + flBegin = fFreeList[j].start; + flEnd = flBegin + (fFreeList[j].size * 1024) - 1; + + // em entry overlaps the beginning + // em entry is contained within + // em entry overlaps the end + if ((emBegin <= flBegin && emEnd >= flBegin) || (emBegin >= flBegin && emEnd <= flEnd) || + (emBegin <= flEnd && emEnd >= flEnd)) + { + cerr << "EM::checkConsistency(): Improper LBID allocation detected" << endl; + throw logic_error("EM checkConsistency test 1a (data structures are read-locked)"); + } + } + } } - catch (...) + } + + cout << "test 1a passed\n"; + + // test 1b - verify that the entire LBID space is accounted for + + int lbid, oldlbid; + + lbid = 0; + + while (lbid < 67108864) // 2^26 (2^36/1024) + { + oldlbid = lbid; + + for (i = 0; i < flEntries; i++) { - releaseEMEntryTable(READ); - throw; + if (fFreeList[i].start % 1024 != 0) + { + cerr << "EM::checkConsistency(): A freelist entry is not 1024-block aligned" << endl; + throw logic_error("EM checkConsistency test 1b (data structures are read-locked)"); + } + + if (fFreeList[i].start / 1024 == lbid) + lbid += fFreeList[i].size; } - flEntries = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - emEntries = fEMShminfo->allocdSize / sizeof(EMEntry); - - // test 1a - make sure every entry in the EM is not overlapped by an entry in the FL - for (i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - emBegin = fExtentMap[i].range.start; - emEnd = emBegin + (fExtentMap[i].range.size * 1024) - 1; - - for (j = 0; j < flEntries; j++) - { - if (fFreeList[j].size != 0) - { - flBegin = fFreeList[j].start; - flEnd = flBegin + (fFreeList[j].size * 1024) - 1; - - //em entry overlaps the beginning - //em entry is contained within - //em entry overlaps the end - if ((emBegin <= flBegin && emEnd >= flBegin) || - (emBegin >= flBegin && emEnd <= flEnd) || - (emBegin <= flEnd && emEnd >= flEnd)) - { - cerr << "EM::checkConsistency(): Improper LBID allocation detected" << endl; - throw logic_error("EM checkConsistency test 1a (data structures are read-locked)"); - } - } - } - } - } - - cout << "test 1a passed\n"; - - //test 1b - verify that the entire LBID space is accounted for - - int lbid, oldlbid; - - lbid = 0; - - while (lbid < 67108864) // 2^26 (2^36/1024) - { - oldlbid = lbid; - - for (i = 0; i < flEntries; i++) - { - if (fFreeList[i].start % 1024 != 0) - { - cerr << "EM::checkConsistency(): A freelist entry is not 1024-block aligned" << endl; - throw logic_error("EM checkConsistency test 1b (data structures are read-locked)"); - } - - if (fFreeList[i].start / 1024 == lbid) - lbid += fFreeList[i].size; - } - - for (i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.start % 1024 != 0) - { - cerr << "EM::checkConsistency(): An extent map entry is not 1024-block aligned " << i << " " << fExtentMap[i].range.start << endl; - throw logic_error("EM checkConsistency test 1b (data structures are read-locked)"); - } - - if (fExtentMap[i].range.start / 1024 == lbid) - lbid += fExtentMap[i].range.size; - } - - if (oldlbid == lbid) - { - cerr << "EM::checkConsistency(): There is a gap in the LBID space at block #" << - static_cast(lbid * 1024) << endl; - throw logic_error("EM checkConsistency test 1b (data structures are read-locked)"); - } - } - - cout << "test 1b passed\n"; - - // test 1c - verify that no dbroot is < 1 - bool errorOut = false; - for (i = 0; i < emEntries; i++) { - if (fExtentMap[i].range.size != 0) - { - //cout << "EM[" << i << "]: dbRoot=" << fExtentMap[i].dbRoot(listMan) << endl; - if (fExtentMap[i].dbRoot == 0) - { - errorOut = true; - cerr << "EM::checkConsistency(): index " << i << " has a 0 dbroot\n"; - } - } + if (fExtentMap[i].range.start % 1024 != 0) + { + cerr << "EM::checkConsistency(): An extent map entry is not 1024-block aligned " << i << " " + << fExtentMap[i].range.start << endl; + throw logic_error("EM checkConsistency test 1b (data structures are read-locked)"); + } + + if (fExtentMap[i].range.start / 1024 == lbid) + lbid += fExtentMap[i].range.size; } - if (errorOut) - throw logic_error("EM checkConsistency test 1c (data structures are read-locked)"); + if (oldlbid == lbid) + { + cerr << "EM::checkConsistency(): There is a gap in the LBID space at block #" + << static_cast(lbid * 1024) << endl; + throw logic_error("EM checkConsistency test 1b (data structures are read-locked)"); + } + } - cout << "test 1c passed\n"; + cout << "test 1b passed\n"; + + // test 1c - verify that no dbroot is < 1 + bool errorOut = false; + + for (i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) + { + // cout << "EM[" << i << "]: dbRoot=" << fExtentMap[i].dbRoot(listMan) << endl; + if (fExtentMap[i].dbRoot == 0) + { + errorOut = true; + cerr << "EM::checkConsistency(): index " << i << " has a 0 dbroot\n"; + } + } + } + + if (errorOut) + throw logic_error("EM checkConsistency test 1c (data structures are read-locked)"); + + cout << "test 1c passed\n"; #if 0 // a test ported from the tek2 branch, which requires a RID field to be stored; not relevant here // test 1d - verify that each pair is unique @@ -5816,20 +5567,20 @@ int ExtentMap::checkConsistency() cout << "Test 1d passed\n"; #endif - // test 2 - verify that the freelist is consolidated - for (i = 0; i < flEntries; i++) + // test 2 - verify that the freelist is consolidated + for (i = 0; i < flEntries; i++) + { + if (fFreeList[i].size != 0) { - if (fFreeList[i].size != 0) - { - flEnd = fFreeList[i].start + (fFreeList[i].size * 1024); + flEnd = fFreeList[i].start + (fFreeList[i].size * 1024); - for (j = i + 1; j < flEntries; j++) - if (fFreeList[j].size != 0 && fFreeList[j].start == flEnd) - throw logic_error("EM checkConsistency test 2 (data structures are read-locked)"); - } + for (j = i + 1; j < flEntries; j++) + if (fFreeList[j].size != 0 && fFreeList[j].start == flEnd) + throw logic_error("EM checkConsistency test 2 (data structures are read-locked)"); } + } - cout << "test 2 passed\n"; + cout << "test 2 passed\n"; // needs to be updated #if 0 @@ -5877,85 +5628,83 @@ int ExtentMap::checkConsistency() oids.clear(); #endif + // test 5a - scan freelist to make sure the current size is accurate - // test 5a - scan freelist to make sure the current size is accurate + for (i = 0, usedEntries = 0; i < emEntries; i++) + if (fExtentMap[i].range.size != 0) + usedEntries++; - for (i = 0, usedEntries = 0; i < emEntries; i++) - if (fExtentMap[i].range.size != 0) - usedEntries++; + if (usedEntries != fEMShminfo->currentSize / sizeof(EMEntry)) + { + cerr << "checkConsistency: used extent map entries = " << usedEntries << " metadata says " + << fEMShminfo->currentSize / sizeof(EMEntry) << endl; + throw logic_error("EM checkConsistency test 5a (data structures are read-locked)"); + } - if (usedEntries != fEMShminfo->currentSize / sizeof(EMEntry)) - { - cerr << "checkConsistency: used extent map entries = " << usedEntries - << " metadata says " << fEMShminfo->currentSize / sizeof(EMEntry) - << endl; - throw logic_error("EM checkConsistency test 5a (data structures are read-locked)"); - } + for (i = 0, usedEntries = 0; i < flEntries; i++) + if (fFreeList[i].size != 0) + usedEntries++; - for (i = 0, usedEntries = 0; i < flEntries; i++) - if (fFreeList[i].size != 0) - usedEntries++; + if (usedEntries != fFLShminfo->currentSize / sizeof(InlineLBIDRange)) + { + cerr << "checkConsistency: used freelist entries = " << usedEntries << " metadata says " + << fFLShminfo->currentSize / sizeof(InlineLBIDRange) << endl; + throw logic_error("EM checkConsistency test 5a (data structures are read-locked)"); + } - if (usedEntries != fFLShminfo->currentSize / sizeof(InlineLBIDRange)) - { - cerr << "checkConsistency: used freelist entries = " << usedEntries - << " metadata says " << fFLShminfo->currentSize / sizeof(InlineLBIDRange) - << endl; - throw logic_error("EM checkConsistency test 5a (data structures are read-locked)"); - } + cout << "test 5a passed\n"; - cout << "test 5a passed\n"; - - releaseFreeList(READ); - releaseEMEntryTable(READ); - return 0; + releaseFreeList(READ); + releaseEMEntryTable(READ); + return 0; } - void ExtentMap::setReadOnly() { - r_only = true; + r_only = true; } void ExtentMap::undoChanges() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("undoChanges"); + if (fDebug) + TRACER_WRITENOW("undoChanges"); #endif - Undoable::undoChanges(); - finishChanges(); + Undoable::undoChanges(); + finishChanges(); } void ExtentMap::confirmChanges() { #ifdef BRM_INFO - if (fDebug) TRACER_WRITENOW("confirmChanges"); + if (fDebug) + TRACER_WRITENOW("confirmChanges"); #endif - Undoable::confirmChanges(); - finishChanges(); + Undoable::confirmChanges(); + finishChanges(); } void ExtentMap::finishChanges() { - if (flLocked) - releaseFreeList(WRITE); + if (flLocked) + releaseFreeList(WRITE); - if (emLocked) - releaseEMEntryTable(WRITE); + if (emLocked) + releaseEMEntryTable(WRITE); } const bool* ExtentMap::getEMFLLockStatus() { - return &flLocked; + return &flLocked; } const bool* ExtentMap::getEMLockStatus() { - return &emLocked; + return &emLocked; } //------------------------------------------------------------------------------ @@ -5963,79 +5712,79 @@ const bool* ExtentMap::getEMLockStatus() //------------------------------------------------------------------------------ void ExtentMap::checkReloadConfig() { - config::Config* cf = config::Config::makeConfig(); + config::Config* cf = config::Config::makeConfig(); - // Immediately return if Columnstore.xml timestamp has not changed - if (cf->getCurrentMTime() == fCacheTime) - return; + // Immediately return if Columnstore.xml timestamp has not changed + if (cf->getCurrentMTime() == fCacheTime) + return; - //-------------------------------------------------------------------------- - // Initialize outdated attribute still used by primitiveserver. - // Hardcode to 8K for now, since that's all we support. - //-------------------------------------------------------------------------- - ExtentSize = 0x2000; + //-------------------------------------------------------------------------- + // Initialize outdated attribute still used by primitiveserver. + // Hardcode to 8K for now, since that's all we support. + //-------------------------------------------------------------------------- + ExtentSize = 0x2000; -// string es = cf->getConfig("ExtentMap", "ExtentSize"); -// if (es.length() == 0) es = "8K"; -// if (es == "8K" || es == "8k") -// { -// ExtentSize = 0x2000; -// } -// else if (es == "1K" || es == "1k") -// { -// ExtentSize = 0x400; -// } -// else if (es == "64K" || es == "64k") -// { -// ExtentSize = 0x10000; -// } -// else -// { -// throw logic_error("Invalid ExtentSize found in config file!"); -// } + // string es = cf->getConfig("ExtentMap", "ExtentSize"); + // if (es.length() == 0) es = "8K"; + // if (es == "8K" || es == "8k") + // { + // ExtentSize = 0x2000; + // } + // else if (es == "1K" || es == "1k") + // { + // ExtentSize = 0x400; + // } + // else if (es == "64K" || es == "64k") + // { + // ExtentSize = 0x10000; + // } + // else + // { + // throw logic_error("Invalid ExtentSize found in config file!"); + // } - //-------------------------------------------------------------------------- - // Initialize number of rows per extent - // Hardcode to 8M for now, since that's all we support. - //-------------------------------------------------------------------------- - ExtentRows = 0x800000; + //-------------------------------------------------------------------------- + // Initialize number of rows per extent + // Hardcode to 8M for now, since that's all we support. + //-------------------------------------------------------------------------- + ExtentRows = 0x800000; -// string er = cf->getConfig("ExtentMap", "ExtentRows"); -// if (er.length() == 0) er = "8M"; -// if (er == "8M" || er == "8m") -// { -// ExtentRows = 0x800000; -// } -// else if (er == "1M" || er == "1m") -// { -// ExtentRows = 0x100000; -// } -// else if (er == "64M" || er == "64m") -// { -// ExtentRows = 0x4000000; -// } -// else -// { -// throw logic_error("Invalid ExtentRows found in config file!"); -// } + // string er = cf->getConfig("ExtentMap", "ExtentRows"); + // if (er.length() == 0) er = "8M"; + // if (er == "8M" || er == "8m") + // { + // ExtentRows = 0x800000; + // } + // else if (er == "1M" || er == "1m") + // { + // ExtentRows = 0x100000; + // } + // else if (er == "64M" || er == "64m") + // { + // ExtentRows = 0x4000000; + // } + // else + // { + // throw logic_error("Invalid ExtentRows found in config file!"); + // } - //-------------------------------------------------------------------------- - // Initialize segment files per physical partition - //-------------------------------------------------------------------------- - string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); - filesPerColumnPartition = cf->uFromText(fpc); + //-------------------------------------------------------------------------- + // Initialize segment files per physical partition + //-------------------------------------------------------------------------- + string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); + filesPerColumnPartition = cf->uFromText(fpc); - if (filesPerColumnPartition == 0) - filesPerColumnPartition = 4; + if (filesPerColumnPartition == 0) + filesPerColumnPartition = 4; - // Get latest Columnstore.xml timestamp after first access forced a reload - fCacheTime = cf ->getLastMTime(); + // Get latest Columnstore.xml timestamp after first access forced a reload + fCacheTime = cf->getLastMTime(); - //-------------------------------------------------------------------------- - // Initialize extents per segment file - //-------------------------------------------------------------------------- - // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). - extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE; + //-------------------------------------------------------------------------- + // Initialize extents per segment file + //-------------------------------------------------------------------------- + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE; } //------------------------------------------------------------------------------ @@ -6043,13 +5792,13 @@ void ExtentMap::checkReloadConfig() // Mutex lock and call to checkReloadConfig() not currently necessary since, // going with hardcoded value. See checkReloadConfig(). //------------------------------------------------------------------------------ -unsigned ExtentMap::getExtentSize() // dmc-should deprecate +unsigned ExtentMap::getExtentSize() // dmc-should deprecate { -// boost::mutex::scoped_lock lk(fConfigCacheMutex); -// checkReloadConfig( ); + // boost::mutex::scoped_lock lk(fConfigCacheMutex); + // checkReloadConfig( ); - ExtentSize = 0x2000; - return ExtentSize; + ExtentSize = 0x2000; + return ExtentSize; } //------------------------------------------------------------------------------ @@ -6060,11 +5809,11 @@ unsigned ExtentMap::getExtentSize() // dmc-should deprecate //------------------------------------------------------------------------------ unsigned ExtentMap::getExtentRows() { -// boost::mutex::scoped_lock lk(fConfigCacheMutex); -// checkReloadConfig( ); + // boost::mutex::scoped_lock lk(fConfigCacheMutex); + // checkReloadConfig( ); - ExtentRows = 0x800000; - return ExtentRows; + ExtentRows = 0x800000; + return ExtentRows; } //------------------------------------------------------------------------------ @@ -6073,10 +5822,10 @@ unsigned ExtentMap::getExtentRows() //------------------------------------------------------------------------------ unsigned ExtentMap::getFilesPerColumnPartition() { - boost::mutex::scoped_lock lk(fConfigCacheMutex); - checkReloadConfig( ); + boost::mutex::scoped_lock lk(fConfigCacheMutex); + checkReloadConfig(); - return filesPerColumnPartition; + return filesPerColumnPartition; } //------------------------------------------------------------------------------ @@ -6084,10 +5833,10 @@ unsigned ExtentMap::getFilesPerColumnPartition() //------------------------------------------------------------------------------ unsigned ExtentMap::getExtentsPerSegmentFile() { - boost::mutex::scoped_lock lk(fConfigCacheMutex); - checkReloadConfig( ); + boost::mutex::scoped_lock lk(fConfigCacheMutex); + checkReloadConfig(); - return extentsPerSegmentFile; + return extentsPerSegmentFile; } //------------------------------------------------------------------------------ @@ -6095,123 +5844,113 @@ unsigned ExtentMap::getExtentsPerSegmentFile() //------------------------------------------------------------------------------ unsigned ExtentMap::getDbRootCount() { - oam::OamCache* oamcache = oam::OamCache::makeOamCache(); - unsigned int rootCnt = oamcache->getDBRootCount(); + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + unsigned int rootCnt = oamcache->getDBRootCount(); - return rootCnt; + return rootCnt; } //------------------------------------------------------------------------------ // Get list of DBRoots that map to the specified PM. DBRoot list is cached // internally in fPmDbRootMap after getting from Columnstore.xml via OAM. //------------------------------------------------------------------------------ -void ExtentMap::getPmDbRoots( int pm, vector& dbRootList ) +void ExtentMap::getPmDbRoots(int pm, vector& dbRootList) { - oam::OamCache* oamcache = oam::OamCache::makeOamCache(); - oam::OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + oam::OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); - dbRootList.clear(); - dbRootList = (*pmDbroots)[pm]; + dbRootList.clear(); + dbRootList = (*pmDbroots)[pm]; } vector ExtentMap::getFreeListEntries() { - vector v; - grabEMEntryTable(READ); - grabFreeList(READ); + vector v; + grabEMEntryTable(READ); + grabFreeList(READ); - int allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + int allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - for (int i = 0; i < allocdSize; i++) - v.push_back(fFreeList[i]); + for (int i = 0; i < allocdSize; i++) + v.push_back(fFreeList[i]); - releaseFreeList(READ); - releaseEMEntryTable(READ); - return v; + releaseFreeList(READ); + releaseEMEntryTable(READ); + return v; } void ExtentMap::dumpTo(ostream& os) { - grabEMEntryTable(READ); - unsigned emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMEntryTable(READ); + unsigned emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - for (unsigned i = 0; i < emEntries; i++) + for (unsigned i = 0; i < emEntries; i++) + { + if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].range.size != 0) - { - os << fExtentMap[i].range.start << '|' - << fExtentMap[i].range.size << '|' - << fExtentMap[i].fileID << '|' - << fExtentMap[i].blockOffset << '|' - << fExtentMap[i].HWM << '|' - << fExtentMap[i].partitionNum << '|' - << fExtentMap[i].segmentNum << '|' - << fExtentMap[i].dbRoot << '|' - << fExtentMap[i].colWid << '|' - << fExtentMap[i].status << '|' - << fExtentMap[i].partition.cprange.hiVal << '|' - << fExtentMap[i].partition.cprange.loVal << '|' - << fExtentMap[i].partition.cprange.sequenceNum << '|' - << (int)fExtentMap[i].partition.cprange.isValid << '|' - << endl; - } + os << fExtentMap[i].range.start << '|' << fExtentMap[i].range.size << '|' << fExtentMap[i].fileID << '|' + << fExtentMap[i].blockOffset << '|' << fExtentMap[i].HWM << '|' << fExtentMap[i].partitionNum << '|' + << fExtentMap[i].segmentNum << '|' << fExtentMap[i].dbRoot << '|' << fExtentMap[i].colWid << '|' + << fExtentMap[i].status << '|' << fExtentMap[i].partition.cprange.hiVal << '|' + << fExtentMap[i].partition.cprange.loVal << '|' << fExtentMap[i].partition.cprange.sequenceNum << '|' + << (int)fExtentMap[i].partition.cprange.isValid << '|' << endl; } + } - releaseEMEntryTable(READ); + releaseEMEntryTable(READ); } /*int ExtentMap::physicalPartitionNum(const set& oids, - const set& partitionNums, - vector& partitionInfos) + const set& partitionNums, + vector& partitionInfos) { #ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITENOW("physicalPartitionNum"); - ostringstream oss; - set::const_iterator partIt; - oss << "partitionNums: " - for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) - oss << (*it) << " "; - oss << endl; - TRACER_WRITEDIRECT(oss.str()); - } + if (fDebug) + { + TRACER_WRITENOW("physicalPartitionNum"); + ostringstream oss; + set::const_iterator partIt; + oss << "partitionNums: " + for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) + oss << (*it) << " "; + oss << endl; + TRACER_WRITEDIRECT(oss.str()); + } #endif - set::const_iterator it; - grabEMEntryTable(READ); + set::const_iterator it; + grabEMEntryTable(READ); - int emEntries = fEMShminfo->allocdSize/sizeof(struct EMEntry); - PartitionInfo partInfo; - vector extents; - set foundPartitions; - for (int i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0 ) && - partitionNums.find(logicalPartitionNum(fExtentMap[i])) != partitionNums.end()) - { - it = oids.find( fExtentMap[i].fileID ); - if (it != oids.end()) - { - partInfo.oid = fExtentMap[i].fileID; - partInfo.lp.dbroot = fExtentMap[i].dbRoot; - partInfo.lp.pp = fExtentMap[i].partitionNum; - partInfo.lp.seg = fExtentMap[i].segmentNum; - partitionInfos.push_back(partInfo); - } - } - } - releaseEMEntryTable(READ); - return 0; + int emEntries = fEMShminfo->allocdSize/sizeof(struct EMEntry); + PartitionInfo partInfo; + vector extents; + set foundPartitions; + for (int i = 0; i < emEntries; i++) + { + if ((fExtentMap[i].range.size != 0 ) && + partitionNums.find(logicalPartitionNum(fExtentMap[i])) != partitionNums.end()) + { + it = oids.find( fExtentMap[i].fileID ); + if (it != oids.end()) + { + partInfo.oid = fExtentMap[i].fileID; + partInfo.lp.dbroot = fExtentMap[i].dbRoot; + partInfo.lp.pp = fExtentMap[i].partitionNum; + partInfo.lp.seg = fExtentMap[i].segmentNum; + partitionInfos.push_back(partInfo); + } + } + } + releaseEMEntryTable(READ); + return 0; } */ -template -int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& max, int128_t& min, int32_t& seqNum); +template int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& max, int128_t& min, + int32_t& seqNum); -template -int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); +template int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, + int32_t& seqNum); -} //namespace +} // namespace BRM // vim:ts=4 sw=4: - diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 29b97c0be..e70f61492 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -76,22 +76,21 @@ class IDBDataFile; namespace BRM { - // assumed column width when calculating dictionary store extent size #define DICT_COL_WIDTH 8 // valid values for EMEntry.status -const int16_t EXTENTSTATUSMIN(0); // equal to minimum valid status value +const int16_t EXTENTSTATUSMIN(0); // equal to minimum valid status value const int16_t EXTENTAVAILABLE(0); const int16_t EXTENTUNAVAILABLE(1); const int16_t EXTENTOUTOFSERVICE(2); -const int16_t EXTENTSTATUSMAX(2); // equal to maximum valid status value +const int16_t EXTENTSTATUSMAX(2); // equal to maximum valid status value enum partition_type_enum { - PART_DEFAULT = 0, - PART_CASUAL, - PART_RANGE + PART_DEFAULT = 0, + PART_CASUAL, + PART_RANGE }; typedef partition_type_enum EMPartitionType_t; typedef int64_t RangePartitionData_t; @@ -104,218 +103,221 @@ const char CP_VALID = 2; // from v4 to v5; see ExtentMap::loadVersion4or5 for details. struct EMCasualPartition_struct_v4 { - RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. - RangePartitionData_t lo_val; - int32_t sequenceNum; - char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid + RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. + RangePartitionData_t lo_val; + int32_t sequenceNum; + char isValid; // CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. + // CP_VALID- min/max is valid }; struct EMPartition_struct_v4 { - EMCasualPartition_struct_v4 cprange; + EMCasualPartition_struct_v4 cprange; }; struct EMEntry_v4 { - InlineLBIDRange range; - int fileID; - uint32_t blockOffset; - HWM_t HWM; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml - uint16_t colWid; - int16_t status; //extent avail for query or not, or out of service - EMPartition_struct_v4 partition; + InlineLBIDRange range; + int fileID; + uint32_t blockOffset; + HWM_t HWM; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + uint16_t colWid; + int16_t status; // extent avail for query or not, or out of service + EMPartition_struct_v4 partition; }; // MCOL-641: v5 structs of the extent map. This version supports int128_t min // and max values for casual partitioning. struct EMCasualPartition_struct { - int32_t sequenceNum; - char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid - union - { - int128_t bigLoVal; // These need to be reinterpreted as unsigned for uint64_t/uint128_t column types. - int64_t loVal; - }; - union - { - int128_t bigHiVal; - int64_t hiVal; - }; - EXPORT EMCasualPartition_struct(); - EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); - EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); + int32_t sequenceNum; + char isValid; // CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. + // CP_VALID- min/max is valid + union + { + int128_t bigLoVal; // These need to be reinterpreted as unsigned for uint64_t/uint128_t column types. + int64_t loVal; + }; + union + { + int128_t bigHiVal; + int64_t hiVal; + }; + EXPORT EMCasualPartition_struct(); + EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); + EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em); }; typedef EMCasualPartition_struct EMCasualPartition_t; struct EMPartition_struct { - EMCasualPartition_t cprange; + EMCasualPartition_t cprange; }; typedef EMPartition_struct EMPartition_t; struct EMEntry { - InlineLBIDRange range; - int fileID; - uint32_t blockOffset; - HWM_t HWM; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml - uint16_t colWid; - int16_t status; //extent avail for query or not, or out of service - EMPartition_t partition; - EXPORT EMEntry(); - EXPORT EMEntry(const EMEntry&); - EXPORT EMEntry& operator= (const EMEntry&); - EXPORT bool operator< (const EMEntry&) const; + InlineLBIDRange range; + int fileID; + uint32_t blockOffset; + HWM_t HWM; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + uint16_t colWid; + int16_t status; // extent avail for query or not, or out of service + EMPartition_t partition; + EXPORT EMEntry(); + EXPORT EMEntry(const EMEntry&); + EXPORT EMEntry& operator=(const EMEntry&); + EXPORT bool operator<(const EMEntry&) const; }; // Bug 2989, moved from joblist struct ExtentSorter { - bool operator()(const EMEntry& e1, const EMEntry& e2) - { - if (e1.dbRoot < e2.dbRoot) - return true; + bool operator()(const EMEntry& e1, const EMEntry& e2) + { + if (e1.dbRoot < e2.dbRoot) + return true; - if (e1.dbRoot == e2.dbRoot && e1.partitionNum < e2.partitionNum) - return true; + if (e1.dbRoot == e2.dbRoot && e1.partitionNum < e2.partitionNum) + return true; - if (e1.dbRoot == e2.dbRoot && e1.partitionNum == e2.partitionNum && e1.blockOffset < e2.blockOffset) - return true; + if (e1.dbRoot == e2.dbRoot && e1.partitionNum == e2.partitionNum && e1.blockOffset < e2.blockOffset) + return true; - if (e1.dbRoot == e2.dbRoot && e1.partitionNum == e2.partitionNum && e1.blockOffset == e2.blockOffset && e1.segmentNum < e2.segmentNum) - return true; + if (e1.dbRoot == e2.dbRoot && e1.partitionNum == e2.partitionNum && e1.blockOffset == e2.blockOffset && + e1.segmentNum < e2.segmentNum) + return true; - return false; - } + return false; + } }; class ExtentMapImpl { -public: - ~ExtentMapImpl(){}; + public: + ~ExtentMapImpl(){}; - static ExtentMapImpl* makeExtentMapImpl(unsigned key, off_t size, bool readOnly = false); - static void refreshShm() + static ExtentMapImpl* makeExtentMapImpl(unsigned key, off_t size, bool readOnly = false); + static void refreshShm() + { + if (fInstance) { - if (fInstance) - { - delete fInstance; - fInstance = NULL; - } + delete fInstance; + fInstance = NULL; } + } - inline void grow(unsigned key, off_t size) + inline void grow(unsigned key, off_t size) #ifdef NDEBUG - { - fExtMap.grow(key, size); - } + { + fExtMap.grow(key, size); + } #else - { - int rc = fExtMap.grow(key, size); - idbassert(rc == 0); - } + { + int rc = fExtMap.grow(key, size); + idbassert(rc == 0); + } #endif - inline void makeReadOnly() - { - fExtMap.setReadOnly(); - } - inline void clear(unsigned key, off_t size) - { - fExtMap.clear(key, size); - } - inline void swapout(BRMShmImpl& rhs) - { - fExtMap.swap(rhs); - rhs.destroy(); - } - inline unsigned key() const - { - return fExtMap.key(); - } + inline void makeReadOnly() + { + fExtMap.setReadOnly(); + } + inline void clear(unsigned key, off_t size) + { + fExtMap.clear(key, size); + } + inline void swapout(BRMShmImpl& rhs) + { + fExtMap.swap(rhs); + rhs.destroy(); + } + inline unsigned key() const + { + return fExtMap.key(); + } - inline EMEntry* get() const - { - return reinterpret_cast(fExtMap.fMapreg.get_address()); - } + inline EMEntry* get() const + { + return reinterpret_cast(fExtMap.fMapreg.get_address()); + } -private: - ExtentMapImpl(unsigned key, off_t size, bool readOnly = false); - ExtentMapImpl(const ExtentMapImpl& rhs); - ExtentMapImpl& operator=(const ExtentMapImpl& rhs); + private: + ExtentMapImpl(unsigned key, off_t size, bool readOnly = false); + ExtentMapImpl(const ExtentMapImpl& rhs); + ExtentMapImpl& operator=(const ExtentMapImpl& rhs); - BRMShmImpl fExtMap; + BRMShmImpl fExtMap; - static boost::mutex fInstanceMutex; - static ExtentMapImpl* fInstance; + static boost::mutex fInstanceMutex; + static ExtentMapImpl* fInstance; }; class FreeListImpl { -public: - ~FreeListImpl(){}; + public: + ~FreeListImpl(){}; - static FreeListImpl* makeFreeListImpl(unsigned key, off_t size, bool readOnly = false); - static void refreshShm() + static FreeListImpl* makeFreeListImpl(unsigned key, off_t size, bool readOnly = false); + static void refreshShm() + { + if (fInstance) { - if (fInstance) - { - delete fInstance; - fInstance = NULL; - } + delete fInstance; + fInstance = NULL; } + } - inline void grow(unsigned key, off_t size) + inline void grow(unsigned key, off_t size) #ifdef NDEBUG - { - fFreeList.grow(key, size); - } + { + fFreeList.grow(key, size); + } #else - { - int rc = fFreeList.grow(key, size); - idbassert(rc == 0); - } + { + int rc = fFreeList.grow(key, size); + idbassert(rc == 0); + } #endif - inline void makeReadOnly() - { - fFreeList.setReadOnly(); - } - inline void clear(unsigned key, off_t size) - { - fFreeList.clear(key, size); - } - inline void swapout(BRMShmImpl& rhs) - { - fFreeList.swap(rhs); - rhs.destroy(); - } - inline unsigned key() const - { - return fFreeList.key(); - } + inline void makeReadOnly() + { + fFreeList.setReadOnly(); + } + inline void clear(unsigned key, off_t size) + { + fFreeList.clear(key, size); + } + inline void swapout(BRMShmImpl& rhs) + { + fFreeList.swap(rhs); + rhs.destroy(); + } + inline unsigned key() const + { + return fFreeList.key(); + } - inline InlineLBIDRange* get() const - { - return reinterpret_cast(fFreeList.fMapreg.get_address()); - } + inline InlineLBIDRange* get() const + { + return reinterpret_cast(fFreeList.fMapreg.get_address()); + } -private: - FreeListImpl(unsigned key, off_t size, bool readOnly = false); - FreeListImpl(const FreeListImpl& rhs); - FreeListImpl& operator=(const FreeListImpl& rhs); + private: + FreeListImpl(unsigned key, off_t size, bool readOnly = false); + FreeListImpl(const FreeListImpl& rhs); + FreeListImpl& operator=(const FreeListImpl& rhs); - BRMShmImpl fFreeList; + BRMShmImpl fFreeList; - static boost::mutex fInstanceMutex; - static FreeListImpl* fInstance; + static boost::mutex fInstanceMutex; + static FreeListImpl* fInstance; }; /** @brief This class encapsulates the extent map functionality of the system @@ -328,712 +330,665 @@ private: */ class ExtentMap : public Undoable { -public: - EXPORT ExtentMap(); - EXPORT ~ExtentMap(); + public: + EXPORT ExtentMap(); + EXPORT ~ExtentMap(); - /** @brief Loads the ExtentMap entries from a file - * - * Loads the ExtentMap entries from a file. This will - * clear out any existing entries. The intention is that before - * the system starts, an external tool instantiates a single Extent - * Map and loads the stored entries. - * @param filename The file to load from. - * @note Throws an ios_base::failure exception on an IO error, runtime_error - * if the file "looks" bad. - */ - EXPORT void load(const std::string& filename, bool fixFL = false); + /** @brief Loads the ExtentMap entries from a file + * + * Loads the ExtentMap entries from a file. This will + * clear out any existing entries. The intention is that before + * the system starts, an external tool instantiates a single Extent + * Map and loads the stored entries. + * @param filename The file to load from. + * @note Throws an ios_base::failure exception on an IO error, runtime_error + * if the file "looks" bad. + */ + EXPORT void load(const std::string& filename, bool fixFL = false); - /** @brief Loads the ExtentMap entries from a binayr blob. - * - * Loads the ExtentMap entries from a file. This will - * clear out any existing entries. The intention is that before - * the system starts, an external tool instantiates a single Extent - * Map and loads the stored entries. - * @param pointer to a binary blob. - */ - EXPORT void loadFromBinaryBlob(const char* blob); + /** @brief Loads the ExtentMap entries from a binayr blob. + * + * Loads the ExtentMap entries from a file. This will + * clear out any existing entries. The intention is that before + * the system starts, an external tool instantiates a single Extent + * Map and loads the stored entries. + * @param pointer to a binary blob. + */ + EXPORT void loadFromBinaryBlob(const char* blob); - /** @brief Saves the ExtentMap entries to a file - * - * Saves the ExtentMap entries to a file. - * @param filename The file to save to. - */ - EXPORT void save(const std::string& filename); + /** @brief Saves the ExtentMap entries to a file + * + * Saves the ExtentMap entries to a file. + * @param filename The file to save to. + */ + EXPORT void save(const std::string& filename); - // @bug 1509. Added new version of lookup below. - /** @brief Returns the first and last LBID in the range for a given LBID - * - * Get the first and last LBID for the extent that contains the given LBID. - * @param LBID (in) The lbid to search for - * @param firstLBID (out) The first lbid for the extent - * @param lastLBID (out) the last lbid for the extent - * @return 0 on success, -1 on error - */ - EXPORT int lookup(LBID_t LBID, LBID_t& firstLBID, LBID_t& lastLBID); + // @bug 1509. Added new version of lookup below. + /** @brief Returns the first and last LBID in the range for a given LBID + * + * Get the first and last LBID for the extent that contains the given LBID. + * @param LBID (in) The lbid to search for + * @param firstLBID (out) The first lbid for the extent + * @param lastLBID (out) the last lbid for the extent + * @return 0 on success, -1 on error + */ + EXPORT int lookup(LBID_t LBID, LBID_t& firstLBID, LBID_t& lastLBID); - // @bug 1055+. New functions added for multiple files per OID enhancement. + // @bug 1055+. New functions added for multiple files per OID enhancement. - /** @brief Look up the OID and file block offset assiciated with an LBID - * - * Look up the OID and file block offset assiciated with an LBID - * @param LBID (in) The lbid to search for - * @param OID (out) The OID associated with lbid - * @param dbRoot (out) The db root containing the LBID - * @param partitionNum (out) The partition containing the LBID - * @param segmentNum (out) The segment containing the LBID - * @param fileBlockOffset (out) The file block offset associated - * with LBID - * @return 0 on success, -1 on error - */ - EXPORT int lookupLocal(LBID_t LBID, int& OID, uint16_t& dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& fileBlockOffset); + /** @brief Look up the OID and file block offset assiciated with an LBID + * + * Look up the OID and file block offset assiciated with an LBID + * @param LBID (in) The lbid to search for + * @param OID (out) The OID associated with lbid + * @param dbRoot (out) The db root containing the LBID + * @param partitionNum (out) The partition containing the LBID + * @param segmentNum (out) The segment containing the LBID + * @param fileBlockOffset (out) The file block offset associated + * with LBID + * @return 0 on success, -1 on error + */ + EXPORT int lookupLocal(LBID_t LBID, int& OID, uint16_t& dbRoot, uint32_t& partitionNum, + uint16_t& segmentNum, uint32_t& fileBlockOffset); - /** @brief Look up the LBID associated with a given OID, offset, partition, and segment. - * - * Look up the LBID associated with a given OID, offset, partition, and segment. - * @param OID (in) The OID to look up - * @param fileBlockOffset (in) The file block offset - * @param partitionNum (in) The partition containing the lbid - * @param segmentNum (in) The segement containing the lbid - * @param LBID (out) The LBID associated with the given offset of the OID. - * @return 0 on success, -1 on error - */ - EXPORT int lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, LBID_t& LBID); + /** @brief Look up the LBID associated with a given OID, offset, partition, and segment. + * + * Look up the LBID associated with a given OID, offset, partition, and segment. + * @param OID (in) The OID to look up + * @param fileBlockOffset (in) The file block offset + * @param partitionNum (in) The partition containing the lbid + * @param segmentNum (in) The segement containing the lbid + * @param LBID (out) The LBID associated with the given offset of the OID. + * @return 0 on success, -1 on error + */ + EXPORT int lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, + LBID_t& LBID); - /** @brief Look up the LBID associated with a given dbroot, OID, offset, - * partition, and segment. - * - * Look up LBID associated with a given OID, offset, partition, and segment. - * @param OID (in) The OID to look up - * @param fileBlockOffset (in) The file block offset - * @param partitionNum (in) The partition containing the lbid - * @param segmentNum (in) The segement containing the lbid - * @param LBID (out) The LBID associated with the given offset of the OID. - * @return 0 on success, -1 on error - */ - EXPORT int lookupLocal_DBroot(int OID, uint16_t dbroot, - uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, - LBID_t& LBID); + /** @brief Look up the LBID associated with a given dbroot, OID, offset, + * partition, and segment. + * + * Look up LBID associated with a given OID, offset, partition, and segment. + * @param OID (in) The OID to look up + * @param fileBlockOffset (in) The file block offset + * @param partitionNum (in) The partition containing the lbid + * @param segmentNum (in) The segement containing the lbid + * @param LBID (out) The LBID associated with the given offset of the OID. + * @return 0 on success, -1 on error + */ + EXPORT int lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNum, uint16_t segmentNum, + uint32_t fileBlockOffset, LBID_t& LBID); - // @bug 1055-. + // @bug 1055-. - /** @brief Look up the starting LBID associated with a given OID, - * partition, segment, and offset. - * - * @param OID (in) The OID to look up - * @param partitionNum (in) The partition containing the lbid - * @param segmentNum (in) The segement containing the lbid - * @param fileBlockOffset (in) The file block offset - * @param LBID (out) The starting LBID associated with the extent - * containing the given offset - * @return 0 on success, -1 on error - */ - int lookupLocalStartLbid(int OID, - uint32_t partitionNum, - uint16_t segmentNum, - uint32_t fileBlockOffset, - LBID_t& LBID); + /** @brief Look up the starting LBID associated with a given OID, + * partition, segment, and offset. + * + * @param OID (in) The OID to look up + * @param partitionNum (in) The partition containing the lbid + * @param segmentNum (in) The segement containing the lbid + * @param fileBlockOffset (in) The file block offset + * @param LBID (out) The starting LBID associated with the extent + * containing the given offset + * @return 0 on success, -1 on error + */ + int lookupLocalStartLbid(int OID, uint32_t partitionNum, uint16_t segmentNum, uint32_t fileBlockOffset, + LBID_t& LBID); - /** @brief Get a complete list of LBID ranges assigned to an OID - * - * Get a complete list of LBID ranges assigned to an OID. - */ - EXPORT void lookup(OID_t oid, LBIDRange_v& ranges); + /** @brief Get a complete list of LBID ranges assigned to an OID + * + * Get a complete list of LBID ranges assigned to an OID. + */ + EXPORT void lookup(OID_t oid, LBIDRange_v& ranges); - /** @brief Allocate a "stripe" of extents for columns in a table (in DBRoot) - * - * If this is the first extent for the OID/DBRoot, it will start at - * file offset 0. If space for the OID already exists, the new - * extent will "logically" be appended to the end of the already- - * allocated space, although the extent may reside in a different - * physical file as indicated by dbRoot, partition, and segment. - * Partition and segment numbers are 0 based, dbRoot is 1 based. - * - * Allocate a "stripe" of extents for the specified columns and DBRoot - * @param cols (in) List of column OIDs and column widths - * @param dbRoot (in) DBRoot for requested extents. - * @param partitionNum (in/out) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (out) Segment number selected for new extents. - * @param extents (out) list of lbids, numBlks, and fbo for new extents - * @return 0 on success, -1 on error - */ - EXPORT void createStripeColumnExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - std::vector& extents); + /** @brief Allocate a "stripe" of extents for columns in a table (in DBRoot) + * + * If this is the first extent for the OID/DBRoot, it will start at + * file offset 0. If space for the OID already exists, the new + * extent will "logically" be appended to the end of the already- + * allocated space, although the extent may reside in a different + * physical file as indicated by dbRoot, partition, and segment. + * Partition and segment numbers are 0 based, dbRoot is 1 based. + * + * Allocate a "stripe" of extents for the specified columns and DBRoot + * @param cols (in) List of column OIDs and column widths + * @param dbRoot (in) DBRoot for requested extents. + * @param partitionNum (in/out) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (out) Segment number selected for new extents. + * @param extents (out) list of lbids, numBlks, and fbo for new extents + * @return 0 on success, -1 on error + */ + EXPORT void createStripeColumnExtents(const std::vector& cols, + uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + std::vector& extents); - /** @brief Allocates an extent for a column file - * - * Allocates an extent for the specified OID and DBroot. - * If this is the first extent for the OID/DBRoot, it will start at - * file offset 0. If space for the OID already exists, the new - * extent will "logically" be appended to the end of the already- - * allocated space, although the extent may reside in a different - * physical file as indicated by dbRoot, partition, and segment. - * Partition and segment numbers are 0 based, dbRoot is 1 based. - * - * @param OID (in) The OID requesting the extent. - * @param colWidth (in) Column width of the OID. - * @param dbRoot (in) DBRoot where extent is to be added. - * @param colDataType (in) the column type - * @param partitionNum (in/out) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (out) Segment number assigned to the extent. - * @param lbid (out) The first LBID of the extent created. - * @param allocdsize (out) The total number of LBIDs allocated. - * @param startBlockOffset (out) The first block of the extent created. - * @param useLock Grab ExtentMap and FreeList WRITE lock to perform work - */ - // @bug 4091: To be deprecated as public function. Should just be a - // private function used by createStripeColumnExtents(). - EXPORT void createColumnExtent_DBroot(int OID, - uint32_t colWidth, - uint16_t dbRoot, + /** @brief Allocates an extent for a column file + * + * Allocates an extent for the specified OID and DBroot. + * If this is the first extent for the OID/DBRoot, it will start at + * file offset 0. If space for the OID already exists, the new + * extent will "logically" be appended to the end of the already- + * allocated space, although the extent may reside in a different + * physical file as indicated by dbRoot, partition, and segment. + * Partition and segment numbers are 0 based, dbRoot is 1 based. + * + * @param OID (in) The OID requesting the extent. + * @param colWidth (in) Column width of the OID. + * @param dbRoot (in) DBRoot where extent is to be added. + * @param colDataType (in) the column type + * @param partitionNum (in/out) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (out) Segment number assigned to the extent. + * @param lbid (out) The first LBID of the extent created. + * @param allocdsize (out) The total number of LBIDs allocated. + * @param startBlockOffset (out) The first block of the extent created. + * @param useLock Grab ExtentMap and FreeList WRITE lock to perform work + */ + // @bug 4091: To be deprecated as public function. Should just be a + // private function used by createStripeColumnExtents(). + EXPORT void createColumnExtent_DBroot(int OID, uint32_t colWidth, uint16_t dbRoot, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& partitionNum, uint16_t& segmentNum, LBID_t& lbid, + int& allocdsize, uint32_t& startBlockOffset, bool useLock = true); + + /** @brief Allocates extent for exact file that is specified + * + * Allocates an extent for the exact file specified by OID, DBRoot, + * partition, and segment. + * If this is the first extent for the OID/DBRoot, it will start at + * file offset 0. If space for the OID already exists, the new + * extent will "logically" be appended to the end of the already- + * allocated space. + * Partition and segment numbers are 0 based, dbRoot is 1 based. + * + * @param OID (in) The OID requesting the extent. + * @param colWidth (in) Column width of the OID. + * @param dbRoot (in) DBRoot where extent is to be added. + * @param partitionNum (in) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (in) Segment number in file path. + * If allocating OID's first extent for this DBRoot, then + * segmentNum is input, else it is an output arg. + * @param colDataType (in) the column type + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @param startBlockOffset (out) The first block of the extent created. + */ + EXPORT void createColumnExtentExactFile(int OID, uint32_t colWidth, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& partitionNum, - uint16_t& segmentNum, - LBID_t& lbid, - int& allocdsize, - uint32_t& startBlockOffset, - bool useLock = true); + LBID_t& lbid, int& allocdsize, uint32_t& startBlockOffset); - /** @brief Allocates extent for exact file that is specified - * - * Allocates an extent for the exact file specified by OID, DBRoot, - * partition, and segment. - * If this is the first extent for the OID/DBRoot, it will start at - * file offset 0. If space for the OID already exists, the new - * extent will "logically" be appended to the end of the already- - * allocated space. - * Partition and segment numbers are 0 based, dbRoot is 1 based. - * - * @param OID (in) The OID requesting the extent. - * @param colWidth (in) Column width of the OID. - * @param dbRoot (in) DBRoot where extent is to be added. - * @param partitionNum (in) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (in) Segment number in file path. - * If allocating OID's first extent for this DBRoot, then - * segmentNum is input, else it is an output arg. - * @param colDataType (in) the column type - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @param startBlockOffset (out) The first block of the extent created. - */ - EXPORT void createColumnExtentExactFile(int OID, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdsize, - uint32_t& startBlockOffset); + /** @brief Allocates an extent for a dictionary store file + * + * Allocates an extent for the specified dictionary store OID, + * dbRoot, partition number, and segment number. These should + * correlate with those belonging to the corresponding token file. + * The first extent for each store file will start at file offset 0. + * Other extents will be appended to the end of the already- + * allocated space for the same store file. + * Partition and segment numbers are 0 based, dbRoot is 1 based. + * + * @param OID (in) The OID requesting the extent. + * @param dbRoot (in) DBRoot to assign to the extent. + * @param partitionNum (in) Partition number to assign to the extent. + * @param segmentNum (in) Segment number to assign to the extent. + * @param lbid (out) The first LBID of the extent created. + * @param allocdsize (out) The total number of LBIDs allocated. + */ + EXPORT void createDictStoreExtent(int OID, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum, + LBID_t& lbid, int& allocdsize); - /** @brief Allocates an extent for a dictionary store file - * - * Allocates an extent for the specified dictionary store OID, - * dbRoot, partition number, and segment number. These should - * correlate with those belonging to the corresponding token file. - * The first extent for each store file will start at file offset 0. - * Other extents will be appended to the end of the already- - * allocated space for the same store file. - * Partition and segment numbers are 0 based, dbRoot is 1 based. - * - * @param OID (in) The OID requesting the extent. - * @param dbRoot (in) DBRoot to assign to the extent. - * @param partitionNum (in) Partition number to assign to the extent. - * @param segmentNum (in) Segment number to assign to the extent. - * @param lbid (out) The first LBID of the extent created. - * @param allocdsize (out) The total number of LBIDs allocated. - */ - EXPORT void createDictStoreExtent(int OID, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - LBID_t& lbid, - int& allocdsize); + /** @brief Rollback (delete) a set of extents for the specified OID. + * + * Deletes all the extents that logically follow the specified + * column extent; and sets the HWM for the specified extent. + * @param oid OID of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segmentNum Last segment in partitionNum to be kept. + * @param hwm HWM to be assigned to the last extent that is kept. + */ + EXPORT void rollbackColumnExtents(int oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm); - /** @brief Rollback (delete) a set of extents for the specified OID. - * - * Deletes all the extents that logically follow the specified - * column extent; and sets the HWM for the specified extent. - * @param oid OID of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segmentNum Last segment in partitionNum to be kept. - * @param hwm HWM to be assigned to the last extent that is kept. - */ - EXPORT void rollbackColumnExtents(int oid, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm); + /** @brief Rollback (delete) set of extents for specified OID & DBRoot. + * + * Deletes all the extents that logically follow the specified + * column extent; and sets the HWM for the specified extent. + * @param oid OID of the extents to be deleted. + * @param bDeleteAll Flag indicates if all extents for oid and dbroot are + * to be deleted, else part#, seg#, and HWM are used. + * @param dbRoot DBRoot of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segmentNum Last segment in partitionNum to be kept. + * @param hwm HWM to be assigned to the last extent that is kept. + */ + EXPORT void rollbackColumnExtents_DBroot(int oid, bool bDeleteAll, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, HWM_t hwm); - /** @brief Rollback (delete) set of extents for specified OID & DBRoot. - * - * Deletes all the extents that logically follow the specified - * column extent; and sets the HWM for the specified extent. - * @param oid OID of the extents to be deleted. - * @param bDeleteAll Flag indicates if all extents for oid and dbroot are - * to be deleted, else part#, seg#, and HWM are used. - * @param dbRoot DBRoot of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segmentNum Last segment in partitionNum to be kept. - * @param hwm HWM to be assigned to the last extent that is kept. - */ - EXPORT void rollbackColumnExtents_DBroot(int oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm); + /** @brief delete of column extents for the specified extents. + * + * Deletes the extents that logically follow the specified + * column extent in extentsInfo. It use the same algorithm as in + * rollbackColumnExtents. + * @param extentInfo the information for extents + */ + EXPORT void deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo); - /** @brief delete of column extents for the specified extents. - * - * Deletes the extents that logically follow the specified - * column extent in extentsInfo. It use the same algorithm as in - * rollbackColumnExtents. - * @param extentInfo the information for extents - */ - EXPORT void deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo); + /** @brief delete of dictionary extents for the specified extents. + * + * Arguments specify the last stripe for all the oids. Any extents after this are + * deleted. The hwm's of the extents in the last stripe are updated + * based on the hwm in extentsInfo. It use the same algorithm as in + * rollbackDictStoreExtents. + * @param extentInfo the information for extents to be resetted + */ + EXPORT void deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo); - /** @brief delete of dictionary extents for the specified extents. - * - * Arguments specify the last stripe for all the oids. Any extents after this are - * deleted. The hwm's of the extents in the last stripe are updated - * based on the hwm in extentsInfo. It use the same algorithm as in - * rollbackDictStoreExtents. - * @param extentInfo the information for extents to be resetted - */ - EXPORT void deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo); + /** @brief Rollback (delete) a set of dict store extents for an OID. + * + * Arguments specify the last stripe. Any extents after this are + * deleted. The hwm's of the extents in the last stripe are updated + * based on the contents of the hwm vector. If hwms is a partial list, + * (as in the first stripe of a partition), then any extents in sub- + * sequent segment files for that partition are deleted. + * @param oid OID of the extents to be deleted or updated. + * @param partitionNum Last partition to be kept. + * @param hwms Vector of hwms for the last partition to be kept. + */ + EXPORT void rollbackDictStoreExtents(int oid, uint32_t partitionNum, const std::vector& hwms); - /** @brief Rollback (delete) a set of dict store extents for an OID. - * - * Arguments specify the last stripe. Any extents after this are - * deleted. The hwm's of the extents in the last stripe are updated - * based on the contents of the hwm vector. If hwms is a partial list, - * (as in the first stripe of a partition), then any extents in sub- - * sequent segment files for that partition are deleted. - * @param oid OID of the extents to be deleted or updated. - * @param partitionNum Last partition to be kept. - * @param hwms Vector of hwms for the last partition to be kept. - */ - EXPORT void rollbackDictStoreExtents(int oid, - uint32_t partitionNum, - const std::vector& hwms); + /** @brief Rollback (delete) a set of dict store extents for an OID & DBRoot + * + * Arguments specify the last stripe. Any extents after this are + * deleted. The hwm's of the extents in the last stripe are updated + * based on the contents of the hwm vector. If hwms is a partial list, + * (as in the first stripe of a partition), then any extents in sub- + * sequent segment files for that partition are deleted. If hwms is empty + * then all the extents in dbRoot are deleted. + * @param oid OID of the extents to be deleted or updated. + * @param dbRoot DBRoot of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segNums Vector of segment files in last partition to be kept. + * @param hwms Vector of hwms for the last partition to be kept. + */ + EXPORT void rollbackDictStoreExtents_DBroot(int oid, uint16_t dbRoot, uint32_t partitionNum, + const std::vector& segNums, + const std::vector& hwms); - /** @brief Rollback (delete) a set of dict store extents for an OID & DBRoot - * - * Arguments specify the last stripe. Any extents after this are - * deleted. The hwm's of the extents in the last stripe are updated - * based on the contents of the hwm vector. If hwms is a partial list, - * (as in the first stripe of a partition), then any extents in sub- - * sequent segment files for that partition are deleted. If hwms is empty - * then all the extents in dbRoot are deleted. - * @param oid OID of the extents to be deleted or updated. - * @param dbRoot DBRoot of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segNums Vector of segment files in last partition to be kept. - * @param hwms Vector of hwms for the last partition to be kept. - */ - EXPORT void rollbackDictStoreExtents_DBroot(int oid, - uint16_t dbRoot, - uint32_t partitionNum, - const std::vector& segNums, - const std::vector& hwms); + /** @brief Deallocates all extents associated with OID + * + * Deallocates all extents associated with OID + * @param OID The OID to delete + */ + EXPORT void deleteOID(int OID); - /** @brief Deallocates all extents associated with OID - * - * Deallocates all extents associated with OID - * @param OID The OID to delete - */ - EXPORT void deleteOID(int OID); + /** @brief Deallocates all extents associated with each OID + * + * Deallocates all extents associated with each OID + * @param OIDs The OIDs to delete + */ + EXPORT void deleteOIDs(const OidsMap_t& OIDs); - /** @brief Deallocates all extents associated with each OID - * - * Deallocates all extents associated with each OID - * @param OIDs The OIDs to delete - */ - EXPORT void deleteOIDs(const OidsMap_t& OIDs); + /** @brief Check if any of the given partitions is the last one of a DBroot + * + * This is for partitioning operations to use. The last partition of a DBroot + * can not be dropped or disabled. + * + * @param OID (in) The OID + * @param partitionNums (in) The logical partition numbers to check. + * @return true if any of the partitions in the set is the last partition of + * a DBroot. + */ - /** @brief Check if any of the given partitions is the last one of a DBroot - * - * This is for partitioning operations to use. The last partition of a DBroot - * can not be dropped or disabled. - * - * @param OID (in) The OID - * @param partitionNums (in) The logical partition numbers to check. - * @return true if any of the partitions in the set is the last partition of - * a DBroot. - */ + /** @brief Gets the last local high water mark of an OID for a given dbRoot + * + * Get last local high water mark of an OID for a given dbRoot, relative to + * a segment file. The partition and segment numbers for the pertinent + * segment are also returned. - /** @brief Gets the last local high water mark of an OID for a given dbRoot - * - * Get last local high water mark of an OID for a given dbRoot, relative to - * a segment file. The partition and segment numbers for the pertinent - * segment are also returned. + * @param OID (in) The OID + * @param dbRoot (in) The relevant DBRoot + * @param partitionNum (out) The relevant partition number + * @param segmentNum (out) The relevant segment number + * @param status (out) State of the extent (Available, OutOfService, etc) + * @param bFound (out) Indicates whether an extent was found for dbRoot + * @return The last file block number written to in the last + * partition/segment file for the given OID. + */ + EXPORT HWM_t getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + int& status, bool& bFound); - * @param OID (in) The OID - * @param dbRoot (in) The relevant DBRoot - * @param partitionNum (out) The relevant partition number - * @param segmentNum (out) The relevant segment number - * @param status (out) State of the extent (Available, OutOfService, etc) - * @param bFound (out) Indicates whether an extent was found for dbRoot - * @return The last file block number written to in the last - * partition/segment file for the given OID. - */ - EXPORT HWM_t getLastHWM_DBroot(int OID, uint16_t dbRoot, - uint32_t& partitionNum, uint16_t& segmentNum, - int& status, bool& bFound); + /** @brief Gets the current high water mark of an OID,partition,segment + * + * Get current local high water mark of an OID, partition, segment; + * where HWM is relative to the specific segment file. + * @param OID (in) The OID + * @param partitionNum (in) The relevant partition number + * @param segmentNum (in) The relevant segment number + * @param status (out) State of the extent (Available, OutOfService, etc) + * @return The last file block number written to in the specified + * partition/segment file for the given OID. + */ + EXPORT HWM_t getLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, int& status); - /** @brief Gets the current high water mark of an OID,partition,segment - * - * Get current local high water mark of an OID, partition, segment; - * where HWM is relative to the specific segment file. - * @param OID (in) The OID - * @param partitionNum (in) The relevant partition number - * @param segmentNum (in) The relevant segment number - * @param status (out) State of the extent (Available, OutOfService, etc) - * @return The last file block number written to in the specified - * partition/segment file for the given OID. - */ - EXPORT HWM_t getLocalHWM(int OID, uint32_t partitionNum, - uint16_t segmentNum, int& status); + /** @brief Sets the current high water mark of an OID,partition,segment + * + * Sets the current local high water mark of an OID, partition, segment; + * where HWM is relative to the specific segment file. + * @param OID The OID + * @param partitionNum (in) The relevant partition number + * @param segmentNum (in) The relevant segment number + * @param HWM The high water mark to record + */ + EXPORT void setLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, HWM_t HWM, bool firstNode, + bool uselock = true); - /** @brief Sets the current high water mark of an OID,partition,segment - * - * Sets the current local high water mark of an OID, partition, segment; - * where HWM is relative to the specific segment file. - * @param OID The OID - * @param partitionNum (in) The relevant partition number - * @param segmentNum (in) The relevant segment number - * @param HWM The high water mark to record - */ - EXPORT void setLocalHWM(int OID, uint32_t partitionNum, - uint16_t segmentNum, HWM_t HWM, bool firstNode, - bool uselock = true); + EXPORT void bulkSetHWM(const std::vector&, bool firstNode); - EXPORT void bulkSetHWM(const std::vector&, bool firstNode); + EXPORT void bulkUpdateDBRoot(const std::vector&); - EXPORT void bulkUpdateDBRoot(const std::vector&); + /** @brief Get HWM information about last segment file for each DBRoot + * assigned to a specific PM. + * + * Vector will contain an entry for each DBRoot. If no "available" extents + * are found for a DBRoot, then totalBlocks will be 0 (and hwmExtentIndex + * will be -1) for that DBRoot. + * @param OID The oid of interest. + * @param pmNumber The PM number of interest. + * @param emDbRootHwmInfos The vector of DbRoot/HWM related objects. + */ + EXPORT void getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& emDbRootHwmInfos); - /** @brief Get HWM information about last segment file for each DBRoot - * assigned to a specific PM. - * - * Vector will contain an entry for each DBRoot. If no "available" extents - * are found for a DBRoot, then totalBlocks will be 0 (and hwmExtentIndex - * will be -1) for that DBRoot. - * @param OID The oid of interest. - * @param pmNumber The PM number of interest. - * @param emDbRootHwmInfos The vector of DbRoot/HWM related objects. - */ - EXPORT void getDbRootHWMInfo(int OID, uint16_t pmNumber, - EmDbRootHWMInfo_v& emDbRootHwmInfos); + /** @brief Get the status (AVAILABLE, OUTOFSERVICE, etc) for the + * segment file represented by the specified OID, part# and seg#. + * + * Unlike many of the other DBRM functions, this function does + * not throw an exception if no extent is found; the "found" + * flag indicates whether an extent was found or not. + * + * @param oid (in) The OID of interest + * @param partitionNum (in) The partition number of interest + * @param segmentNum (in) The segment number of interest + * @param bFound (out) Indicates if extent was found or not + * @param status (out) The state of the extents in the specified + * segment file. + */ + EXPORT void getExtentState(int OID, uint32_t partitionNum, uint16_t segmentNum, bool& bFound, int& status); - /** @brief Get the status (AVAILABLE, OUTOFSERVICE, etc) for the - * segment file represented by the specified OID, part# and seg#. - * - * Unlike many of the other DBRM functions, this function does - * not throw an exception if no extent is found; the "found" - * flag indicates whether an extent was found or not. - * - * @param oid (in) The OID of interest - * @param partitionNum (in) The partition number of interest - * @param segmentNum (in) The segment number of interest - * @param bFound (out) Indicates if extent was found or not - * @param status (out) The state of the extents in the specified - * segment file. - */ - EXPORT void getExtentState(int OID, uint32_t partitionNum, - uint16_t segmentNum, bool& bFound, int& status); + /** @brief Gets the extents of a given OID + * + * Gets the extents of a given OID. The returned entries will + * be NULL-terminated and will have to be destroyed individually + * using delete. + * @note Untested + * @param OID (in) The OID to get the extents for. + * @param entries (out) A snapshot of the OID's Extent Map entries + * sorted by starting LBID; note that The Real Entries can change at + * any time. + * @param sorted (in) indicates if output is to be sorted + * @param notFoundErr (in) indicates if no extents is considered an err + * @param incOutOfService (in) include/exclude out of service extents + */ + EXPORT void getExtents(int OID, std::vector& entries, bool sorted = true, + bool notFoundErr = true, bool incOutOfService = false); - /** @brief Gets the extents of a given OID - * - * Gets the extents of a given OID. The returned entries will - * be NULL-terminated and will have to be destroyed individually - * using delete. - * @note Untested - * @param OID (in) The OID to get the extents for. - * @param entries (out) A snapshot of the OID's Extent Map entries - * sorted by starting LBID; note that The Real Entries can change at - * any time. - * @param sorted (in) indicates if output is to be sorted - * @param notFoundErr (in) indicates if no extents is considered an err - * @param incOutOfService (in) include/exclude out of service extents - */ - EXPORT void getExtents(int OID, std::vector& entries, - bool sorted = true, bool notFoundErr = true, - bool incOutOfService = false); + /** @brief Gets the extents of a given OID under specified dbroot + * + * Gets the extents of a given OID under specified dbroot. The returned entries will + * be NULL-terminated and will have to be destroyed individually + * using delete. + * @param OID (in) The OID to get the extents for. + * @param entries (out) A snapshot of the OID's Extent Map entries for the dbroot + * @param dbroot (in) the specified dbroot + */ + EXPORT void getExtents_dbroot(int OID, std::vector& entries, const uint16_t dbroot); - /** @brief Gets the extents of a given OID under specified dbroot - * - * Gets the extents of a given OID under specified dbroot. The returned entries will - * be NULL-terminated and will have to be destroyed individually - * using delete. - * @param OID (in) The OID to get the extents for. - * @param entries (out) A snapshot of the OID's Extent Map entries for the dbroot - * @param dbroot (in) the specified dbroot - */ - EXPORT void getExtents_dbroot(int OID, std::vector& entries, - const uint16_t dbroot); + /** @brief Gets the number of extents for the specified OID and DBRoot + * + * @param OID (in) The OID of interest + * @param dbroot (in) The DBRoot of interest + * @param incOutOfService (in) include/exclude out of service extents + * @param numExtents (out) number of extents found for OID and dbroot + * @return 0 on success, non-0 on error (see brmtypes.h) + */ + EXPORT void getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfService, uint64_t& numExtents); - /** @brief Gets the number of extents for the specified OID and DBRoot - * - * @param OID (in) The OID of interest - * @param dbroot (in) The DBRoot of interest - * @param incOutOfService (in) include/exclude out of service extents - * @param numExtents (out) number of extents found for OID and dbroot - * @return 0 on success, non-0 on error (see brmtypes.h) - */ - EXPORT void getExtentCount_dbroot(int OID, uint16_t dbroot, - bool incOutOfService, uint64_t& numExtents); + /** @brief Gets the size of an extent in rows + * + * Gets the size of an extent in rows. + * @return The number of rows in an extent. + */ + EXPORT unsigned getExtentSize(); // dmc-consider deprecating + EXPORT unsigned getExtentRows(); - /** @brief Gets the size of an extent in rows - * - * Gets the size of an extent in rows. - * @return The number of rows in an extent. - */ - EXPORT unsigned getExtentSize(); //dmc-consider deprecating - EXPORT unsigned getExtentRows(); + /** @brief Gets the DBRoot for the specified system catalog OID + * + * Function should only be called for System Catalog OIDs, as it assumes + * the OID is fully contained on a single DBRoot, returning the first + * DBRoot found. This only makes sence for a System Catalog + * OID, because all other column OIDs can span multiple DBRoots. + * + * @param oid The system catalog OID + * @param dbRoot (out) the DBRoot holding the system catalog OID + */ + EXPORT void getSysCatDBRoot(OID_t oid, uint16_t& dbRoot); - /** @brief Gets the DBRoot for the specified system catalog OID - * - * Function should only be called for System Catalog OIDs, as it assumes - * the OID is fully contained on a single DBRoot, returning the first - * DBRoot found. This only makes sence for a System Catalog - * OID, because all other column OIDs can span multiple DBRoots. - * - * @param oid The system catalog OID - * @param dbRoot (out) the DBRoot holding the system catalog OID - */ - EXPORT void getSysCatDBRoot(OID_t oid, uint16_t& dbRoot); + /** @brief Delete a Partition for the specified OID(s). + * + * @param oids (in) the OIDs of interest. + * @param partitionNums (in) the set of partitions to be deleted. + */ + EXPORT void deletePartition(const std::set& oids, const std::set& partitionNums, + std::string& emsg); - /** @brief Delete a Partition for the specified OID(s). - * - * @param oids (in) the OIDs of interest. - * @param partitionNums (in) the set of partitions to be deleted. - */ - EXPORT void deletePartition(const std::set& oids, - const std::set& partitionNums, std::string& emsg); + /** @brief Mark a Partition for the specified OID(s) as out of service. + * + * @param oids (in) the OIDs of interest. + * @param partitionNums (in) the set of partitions to be marked out of service. + */ + EXPORT void markPartitionForDeletion(const std::set& oids, + const std::set& partitionNums, std::string& emsg); - /** @brief Mark a Partition for the specified OID(s) as out of service. - * - * @param oids (in) the OIDs of interest. - * @param partitionNums (in) the set of partitions to be marked out of service. - */ - EXPORT void markPartitionForDeletion(const std::set& oids, - const std::set& partitionNums, std::string& emsg); + /** @brief Mark all Partition for the specified OID(s) as out of service. + * + * @param oids (in) the OIDs of interest. + */ + EXPORT void markAllPartitionForDeletion(const std::set& oids); - /** @brief Mark all Partition for the specified OID(s) as out of service. - * - * @param oids (in) the OIDs of interest. - */ - EXPORT void markAllPartitionForDeletion(const std::set& oids); + /** @brief Restore a Partition for the specified OID(s). + * + * @param oids (in) the OIDs of interest. + * @param partitionNums (in) the set of partitions to be restored. + */ + EXPORT void restorePartition(const std::set& oids, const std::set& partitionNums, + std::string& emsg); - /** @brief Restore a Partition for the specified OID(s). - * - * @param oids (in) the OIDs of interest. - * @param partitionNums (in) the set of partitions to be restored. - */ - EXPORT void restorePartition(const std::set& oids, - const std::set& partitionNums, std::string& emsg); + /** @brief Get the list of out-of-service partitions for a given OID + * + * @param OID (in) the OID of interest. + * @param partitionNums (out) the out-of-service partitions for the oid. + * partitionNums will be in sorted order. + */ + EXPORT void getOutOfServicePartitions(OID_t oid, std::set& partitionNums); - /** @brief Get the list of out-of-service partitions for a given OID - * - * @param OID (in) the OID of interest. - * @param partitionNums (out) the out-of-service partitions for the oid. - * partitionNums will be in sorted order. - */ - EXPORT void getOutOfServicePartitions(OID_t oid, - std::set& partitionNums); + /** @brief Delete all extent map rows for the specified dbroot + * + * @param dbroot (in) the dbroot + */ + EXPORT void deleteDBRoot(uint16_t dbroot); - /** @brief Delete all extent map rows for the specified dbroot - * - * @param dbroot (in) the dbroot - */ - EXPORT void deleteDBRoot(uint16_t dbroot); + /** @brief Is the specified DBRoot empty with no extents. + * Throws exception if extentmap shared memory is not loaded. + * + * @param dbroot DBRoot of interest + */ + EXPORT bool isDBRootEmpty(uint16_t dbroot); - /** @brief Is the specified DBRoot empty with no extents. - * Throws exception if extentmap shared memory is not loaded. - * - * @param dbroot DBRoot of interest - */ - EXPORT bool isDBRootEmpty(uint16_t dbroot); + /** @brief Performs internal consistency checks (for testing only) + * + * Performs internal consistency checks (for testing only). + * @note It's incomplete + * @return 0 if all tests pass, -1 (or throws logic_error) if not. + */ + EXPORT int checkConsistency(); - /** @brief Performs internal consistency checks (for testing only) - * - * Performs internal consistency checks (for testing only). - * @note It's incomplete - * @return 0 if all tests pass, -1 (or throws logic_error) if not. - */ - EXPORT int checkConsistency(); + EXPORT void setReadOnly(); - EXPORT void setReadOnly(); + EXPORT virtual void undoChanges(); - EXPORT virtual void undoChanges(); + EXPORT virtual void confirmChanges(); - EXPORT virtual void confirmChanges(); + EXPORT int markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT int markInvalid(const std::vector& lbids, + const std::vector& colDataTypes); - EXPORT int markInvalid(const LBID_t lbid, - const execplan::CalpontSystemCatalog::ColDataType colDataType); - EXPORT int markInvalid(const std::vector& lbids, - const std::vector& colDataTypes); + EXPORT int setMaxMin(const LBID_t lbidRange, const int64_t max, const int64_t min, const int32_t seqNum, + bool firstNode); - EXPORT int setMaxMin(const LBID_t lbidRange, const int64_t max, const int64_t min, const int32_t seqNum, - bool firstNode); + // @bug 1970. Added setExtentsMaxMin function below. - // @bug 1970. Added setExtentsMaxMin function below. + /** @brief Updates the extents in the passed map of CPMaxMin objects. + * @param cpMap - The key must be the first LBID in the range. + * The values are a CPMaxMin struct with the + * min, max, and sequence. + * @param firstNode - if true, logs a debugging msg when CP data is updated + * @return 0 if all tests pass, -1 (or throws logic_error) if not. + */ + EXPORT void setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, bool useLock = true); - /** @brief Updates the extents in the passed map of CPMaxMin objects. - * @param cpMap - The key must be the first LBID in the range. - * The values are a CPMaxMin struct with the - * min, max, and sequence. - * @param firstNode - if true, logs a debugging msg when CP data is updated - * @return 0 if all tests pass, -1 (or throws logic_error) if not. - */ - EXPORT void setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, bool useLock = true); + /** @brief Merges the CP info for the extents contained in cpMap. + * @param cpMap - The key must be the starting LBID in the range. + * @return 0 if all tests pass, -1 (or throws logic_error) if not. + */ + void mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock = true); - /** @brief Merges the CP info for the extents contained in cpMap. - * @param cpMap - The key must be the starting LBID in the range. - * @return 0 if all tests pass, -1 (or throws logic_error) if not. - */ - void mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock = true); + template + EXPORT int getMaxMin(const LBID_t lbidRange, T& max, T& min, int32_t& seqNum); - template - EXPORT int getMaxMin(const LBID_t lbidRange, T& max, T& min, int32_t& seqNum); + EXPORT void getCPMaxMin(const LBID_t lbidRange, + CPMaxMin& cpMaxMin); /** @brief Get whole record for untyped use. */ - EXPORT void getCPMaxMin(const LBID_t lbidRange, CPMaxMin& cpMaxMin); /** @brief Get whole record for untyped use. */ - - inline bool empty() + inline bool empty() + { + if (fEMShminfo == 0) { - if (fEMShminfo == 0) - { - grabEMEntryTable(BRM::ExtentMap::READ); - releaseEMEntryTable(BRM::ExtentMap::READ); - } - - return (fEMShminfo->currentSize == 0); + grabEMEntryTable(BRM::ExtentMap::READ); + releaseEMEntryTable(BRM::ExtentMap::READ); } - EXPORT std::vector getFreeListEntries(); + return (fEMShminfo->currentSize == 0); + } - EXPORT void dumpTo(std::ostream& os); - EXPORT const bool* getEMLockStatus(); - EXPORT const bool* getEMFLLockStatus(); + EXPORT std::vector getFreeListEntries(); + + EXPORT void dumpTo(std::ostream& os); + EXPORT const bool* getEMLockStatus(); + EXPORT const bool* getEMFLLockStatus(); #ifdef BRM_DEBUG - EXPORT void printEM() const; - EXPORT void printEM(const OID_t& oid) const; - EXPORT void printEM(const EMEntry& em) const; - EXPORT void printFL() const; + EXPORT void printEM() const; + EXPORT void printEM(const OID_t& oid) const; + EXPORT void printEM(const EMEntry& em) const; + EXPORT void printFL() const; #endif -private: - static const size_t EM_INCREMENT_ROWS = 100; - static const size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); - static const size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); - static const size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); - static const size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); + private: + static const size_t EM_INCREMENT_ROWS = 100; + static const size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); + static const size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); + static const size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); + static const size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); - ExtentMap(const ExtentMap& em); - ExtentMap& operator=(const ExtentMap& em); + ExtentMap(const ExtentMap& em); + ExtentMap& operator=(const ExtentMap& em); - EMEntry* fExtentMap; - InlineLBIDRange* fFreeList; - key_t fCurrentEMShmkey; - key_t fCurrentFLShmkey; - MSTEntry* fEMShminfo; - MSTEntry* fFLShminfo; - const MasterSegmentTable fMST; - bool r_only; - typedef std::tr1::unordered_map PmDbRootMap_t; - PmDbRootMap_t fPmDbRootMap; - time_t fCacheTime; // timestamp associated with config cache + EMEntry* fExtentMap; + InlineLBIDRange* fFreeList; + key_t fCurrentEMShmkey; + key_t fCurrentFLShmkey; + MSTEntry* fEMShminfo; + MSTEntry* fFLShminfo; + const MasterSegmentTable fMST; + bool r_only; + typedef std::tr1::unordered_map PmDbRootMap_t; + PmDbRootMap_t fPmDbRootMap; + time_t fCacheTime; // timestamp associated with config cache - int numUndoRecords; - bool flLocked, emLocked; - static boost::mutex mutex; // @bug5355 - made mutex static - boost::mutex fConfigCacheMutex; // protect access to Config Cache + int numUndoRecords; + bool flLocked, emLocked; + static boost::mutex mutex; // @bug5355 - made mutex static + boost::mutex fConfigCacheMutex; // protect access to Config Cache - enum OPS - { - NONE, - READ, - WRITE - }; + enum OPS + { + NONE, + READ, + WRITE + }; - OPS EMLock, FLLock; + OPS EMLock, FLLock; - LBID_t _createColumnExtent_DBroot(uint32_t size, int OID, - uint32_t colWidth, - uint16_t dbRoot, + LBID_t _createColumnExtent_DBroot(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& startBlockOffset); + LBID_t _createColumnExtentExactFile(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot, + uint32_t partitionNum, uint16_t segmentNum, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& partitionNum, - uint16_t& segmentNum, uint32_t& startBlockOffset); - LBID_t _createColumnExtentExactFile(uint32_t size, int OID, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& startBlockOffset); - LBID_t _createDictStoreExtent(uint32_t size, int OID, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum); - template - bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const; - void deleteExtent(int emIndex); - LBID_t getLBIDsFromFreeList(uint32_t size); - void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs + LBID_t _createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum); + template + bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const; + void deleteExtent(int emIndex); + LBID_t getLBIDsFromFreeList(uint32_t size); + void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs - key_t chooseEMShmkey(); //see the code for how keys are segmented - key_t chooseFLShmkey(); //see the code for how keys are segmented - void grabEMEntryTable(OPS op); - void grabFreeList(OPS op); - void releaseEMEntryTable(OPS op); - void releaseFreeList(OPS op); - void growEMShmseg(size_t nrows = 0); - void growFLShmseg(); - void finishChanges(); + key_t chooseEMShmkey(); // see the code for how keys are segmented + key_t chooseFLShmkey(); // see the code for how keys are segmented + void grabEMEntryTable(OPS op); + void grabFreeList(OPS op); + void releaseEMEntryTable(OPS op); + void releaseFreeList(OPS op); + void growEMShmseg(size_t nrows = 0); + void growFLShmseg(); + void finishChanges(); - EXPORT unsigned getFilesPerColumnPartition(); - unsigned getExtentsPerSegmentFile(); - unsigned getDbRootCount(); - void getPmDbRoots(int pm, std::vector& dbRootList); - void checkReloadConfig(); - ShmKeys fShmKeys; + EXPORT unsigned getFilesPerColumnPartition(); + unsigned getExtentsPerSegmentFile(); + unsigned getDbRootCount(); + void getPmDbRoots(int pm, std::vector& dbRootList); + void checkReloadConfig(); + ShmKeys fShmKeys; - bool fDebug; + bool fDebug; - int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); + int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); - template void load(T* in); - /** @brief Loads the extent map from a file into memory. - * - * @param in (in) the file to load the extent map from. - * @param upgradeV4ToV5 (in) flag indicating whether we are upgrading - * extent map from v4 to v5. - */ - template void loadVersion4or5(T* in, bool upgradeV4ToV5); + template + void load(T* in); + /** @brief Loads the extent map from a file into memory. + * + * @param in (in) the file to load the extent map from. + * @param upgradeV4ToV5 (in) flag indicating whether we are upgrading + * extent map from v4 to v5. + */ + template + void loadVersion4or5(T* in, bool upgradeV4ToV5); - ExtentMapImpl* fPExtMapImpl; - FreeListImpl* fPFreeListImpl; + ExtentMapImpl* fPExtMapImpl; + FreeListImpl* fPFreeListImpl; }; inline std::ostream& operator<<(std::ostream& os, ExtentMap& rhs) { - rhs.dumpTo(os); - return os; + rhs.dumpTo(os); + return os; } -} //namespace +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/lbidresourcegraph.cpp b/versioning/BRM/lbidresourcegraph.cpp index 34e4802a5..f5775d63a 100644 --- a/versioning/BRM/lbidresourcegraph.cpp +++ b/versioning/BRM/lbidresourcegraph.cpp @@ -31,58 +31,56 @@ using namespace std; namespace BRM { - LBIDResourceGraph::LBIDResourceGraph() : color(0) { } LBIDResourceGraph::LBIDResourceGraph(const LBIDResourceGraph& r) { - throw logic_error("Don't do that"); + throw logic_error("Don't do that"); } LBIDResourceGraph::~LBIDResourceGraph() { - std::map::iterator tnit; - RNodes_t::iterator rit; - TransactionNode* txnNode; + std::map::iterator tnit; + RNodes_t::iterator rit; + TransactionNode* txnNode; - for (tnit = txns.begin(); tnit != txns.end(); ) + for (tnit = txns.begin(); tnit != txns.end();) + { + txnNode = (*tnit).second; + + if (txnNode->sleeping()) { - txnNode = (*tnit).second; - - if (txnNode->sleeping()) - { - txnNode->die(); - txnNode->wake(); - ++tnit; - } - else - { - txns.erase(tnit++); - delete txnNode; - } + txnNode->die(); + txnNode->wake(); + ++tnit; } - - for (rit = resources.begin(); rit != resources.end(); ) + else { - delete *rit; - resources.erase(rit++); + txns.erase(tnit++); + delete txnNode; } + } + + for (rit = resources.begin(); rit != resources.end();) + { + delete *rit; + resources.erase(rit++); + } } LBIDResourceGraph& LBIDResourceGraph::operator=(const LBIDResourceGraph& r) { - throw logic_error("Don't do that"); + throw logic_error("Don't do that"); } -void LBIDResourceGraph::connectResources(LBID_t start, LBID_t end, - TransactionNode* txnNode) +void LBIDResourceGraph::connectResources(LBID_t start, LBID_t end, TransactionNode* txnNode) { - vector intersection, reserveList; - RNodes_t::iterator sit; - vector::iterator it; - LBID_t i; + vector intersection, reserveList; + RNodes_t::iterator sit; + vector::iterator it; + LBID_t i; #if 0 /* This version creates ResourceNodes that span a range of LBIDs. @@ -133,239 +131,237 @@ void LBIDResourceGraph::connectResources(LBID_t start, LBID_t end, #endif - /* This version creates one ResourceNode per LBID requested */ + /* This version creates one ResourceNode per LBID requested */ - /* - Search for each LBID in the range - if it exists, put the node in intersections - else, put a new node in the reserve list - continue... - */ + /* + Search for each LBID in the range + if it exists, put the node in intersections + else, put a new node in the reserve list + continue... + */ - for (i = start; i <= end; i++) + for (i = start; i <= end; i++) + { + ResourceNode rn(i); + sit = resources.find(&rn); + + if (sit == resources.end()) { - ResourceNode rn(i); - sit = resources.find(&rn); - - if (sit == resources.end()) - { - ResourceNode* tmp = new ResourceNode(i); - resources.insert(tmp); - reserveList.push_back(tmp); - } - else - intersection.push_back(*sit); - } - - // at this point, reserveList and intersection compose a contiguous range - // including [start, end]. reserveList has newly created ranges, intersection - // has the previously existing ones. - - // of the previously existing resources; if it's not already owned (existing in edge), - // then it's one this transaction has to wait on (new out edge). - // (the set class takes care of duplicates in the out edges) - for (it = intersection.begin(); it != intersection.end(); it++) - { - if (txnNode->in.find(*it) == txnNode->in.end()) - txnNode->addOutEdge(*it); - } - - // reserve the new resources - for (it = reserveList.begin(); it != reserveList.end(); it++) - txnNode->addInEdge(*it); - - // at this point, txnNode is adjacent to a set of ResourcesNodes s.t. - // [start, end] is contained within it. -} - - -/* - 0 = OK - 1 = transaction node was not found on wake - -1 = deadlock detected, transaction node destroyed and resources released - - mutex should be slavelock -*/ -int LBIDResourceGraph::reserveRange(LBID_t start, LBID_t end, VER_t txn, - boost::mutex& mutex) -{ - TransactionNode* txnNode; - map::iterator it; - - /* - look for existing transaction node T - - make one if necessary - connectResources(); - checkDeadlock(); - while (txnNode.out.size() > 0) - block on T's condvar - connectResources(); - checkDeadlock(); - } - */ - - it = txns.find(txn); - - if (it == txns.end()) - { - txnNode = new TransactionNode(txn); - txns[txn] = txnNode; + ResourceNode* tmp = new ResourceNode(i); + resources.insert(tmp); + reserveList.push_back(tmp); } else - txnNode = (*it).second; + intersection.push_back(*sit); + } - connectResources(start, end, txnNode); + // at this point, reserveList and intersection compose a contiguous range + // including [start, end]. reserveList has newly created ranges, intersection + // has the previously existing ones. - // "If txnNode is waiting on at least one LBID range..." - while (txnNode->out.size() > 0) + // of the previously existing resources; if it's not already owned (existing in edge), + // then it's one this transaction has to wait on (new out edge). + // (the set class takes care of duplicates in the out edges) + for (it = intersection.begin(); it != intersection.end(); it++) + { + if (txnNode->in.find(*it) == txnNode->in.end()) + txnNode->addOutEdge(*it); + } + + // reserve the new resources + for (it = reserveList.begin(); it != reserveList.end(); it++) + txnNode->addInEdge(*it); + + // at this point, txnNode is adjacent to a set of ResourcesNodes s.t. + // [start, end] is contained within it. +} + +/* + 0 = OK + 1 = transaction node was not found on wake + -1 = deadlock detected, transaction node destroyed and resources released + + mutex should be slavelock +*/ +int LBIDResourceGraph::reserveRange(LBID_t start, LBID_t end, VER_t txn, boost::mutex& mutex) +{ + TransactionNode* txnNode; + map::iterator it; + + /* + look for existing transaction node T + - make one if necessary + connectResources(); + checkDeadlock(); + while (txnNode.out.size() > 0) + block on T's condvar + connectResources(); + checkDeadlock(); + } + */ + + it = txns.find(txn); + + if (it == txns.end()) + { + txnNode = new TransactionNode(txn); + txns[txn] = txnNode; + } + else + txnNode = (*it).second; + + connectResources(start, end, txnNode); + + // "If txnNode is waiting on at least one LBID range..." + while (txnNode->out.size() > 0) + { + // make sure there's no deadlock before blocking + if (checkDeadlock(*txnNode)) { - // make sure there's no deadlock before blocking - if (checkDeadlock(*txnNode)) - { -// releaseResources(txn); - return ERR_DEADLOCK; - } - -#ifdef BRM_VERBOSE - cerr << " RG: sleeping transaction " << txn << endl; - - set::iterator sit; - cerr << " waiting on: " << endl; - - for (sit = txnNode->out.begin(); sit != txnNode->out.end(); sit++) - { - ResourceNode* rn = dynamic_cast(*sit); - cerr << hex << rn << dec << " " << rn->lbid() << endl; - } - -#endif - - txnNode->sleep(mutex); -#ifdef BRM_VERBOSE - cerr << " RG: txn " << txn << " is awake" << endl; -#endif - - if (txnNode->dead()) - { - txns.erase(txn); - delete txnNode; - return ERR_KILLED; - } - - // attempt to grab remaining resources - connectResources(start, end, txnNode); + // releaseResources(txn); + return ERR_DEADLOCK; } - // txn has all requested LBID ranges - return ERR_OK; +#ifdef BRM_VERBOSE + cerr << " RG: sleeping transaction " << txn << endl; + + set::iterator sit; + cerr << " waiting on: " << endl; + + for (sit = txnNode->out.begin(); sit != txnNode->out.end(); sit++) + { + ResourceNode* rn = dynamic_cast(*sit); + cerr << hex << rn << dec << " " << rn->lbid() << endl; + } + +#endif + + txnNode->sleep(mutex); +#ifdef BRM_VERBOSE + cerr << " RG: txn " << txn << " is awake" << endl; +#endif + + if (txnNode->dead()) + { + txns.erase(txn); + delete txnNode; + return ERR_KILLED; + } + + // attempt to grab remaining resources + connectResources(start, end, txnNode); + } + + // txn has all requested LBID ranges + return ERR_OK; } void LBIDResourceGraph::releaseResources(VER_t txn) { - /* - get transaction node - get all inbound nodes - detach them and wake all txns on the in-edges - delete the resource nodes - get all outbound nodes (this can happen if a rollback comes in while blocked) - detach them - if txnNode isn't sleeping, - delete the transaction node - else - mark it dead and wake it - */ + /* + get transaction node + get all inbound nodes + detach them and wake all txns on the in-edges + delete the resource nodes + get all outbound nodes (this can happen if a rollback comes in while blocked) + detach them + if txnNode isn't sleeping, + delete the transaction node + else + mark it dead and wake it + */ - TransactionNode* txnNode; - ResourceNode* rNode; - map::iterator it; - set::iterator sit; - set::iterator dummy_sit; + TransactionNode* txnNode; + ResourceNode* rNode; + map::iterator it; + set::iterator sit; + set::iterator dummy_sit; - it = txns.find(txn); + it = txns.find(txn); - if (it == txns.end()) - return; + if (it == txns.end()) + return; - txnNode = (*it).second; + txnNode = (*it).second; - for (sit = txnNode->in.begin(); sit != txnNode->in.end(); ) - { - rNode = dynamic_cast(*sit); - dummy_sit = ++sit; - rNode->wakeAndDetach(); - txnNode->removeInEdge(rNode); - resources.erase(rNode); - delete rNode; - sit = dummy_sit; - } + for (sit = txnNode->in.begin(); sit != txnNode->in.end();) + { + rNode = dynamic_cast(*sit); + dummy_sit = ++sit; + rNode->wakeAndDetach(); + txnNode->removeInEdge(rNode); + resources.erase(rNode); + delete rNode; + sit = dummy_sit; + } - for (sit = txnNode->out.begin(); sit != txnNode->out.end(); ) - { - rNode = dynamic_cast(*sit); - dummy_sit = ++sit; - txnNode->removeOutEdge(rNode); - sit = dummy_sit; - } + for (sit = txnNode->out.begin(); sit != txnNode->out.end();) + { + rNode = dynamic_cast(*sit); + dummy_sit = ++sit; + txnNode->removeOutEdge(rNode); + sit = dummy_sit; + } - if (txnNode->sleeping()) - { - txnNode->die(); - txnNode->wake(); - } - else - { - txns.erase(txn); - delete txnNode; - } + if (txnNode->sleeping()) + { + txnNode->die(); + txnNode->wake(); + } + else + { + txns.erase(txn); + delete txnNode; + } } void LBIDResourceGraph::releaseResource(LBID_t lbid) { - RNodes_t::iterator sit; - TransactionNode* txnNode; + RNodes_t::iterator sit; + TransactionNode* txnNode; - for (sit = resources.begin(); sit != resources.end(); sit++) - if (**sit == lbid) - break; + for (sit = resources.begin(); sit != resources.end(); sit++) + if (**sit == lbid) + break; - if (sit != resources.end()) - { - (*sit)->wakeAndDetach(); + if (sit != resources.end()) + { + (*sit)->wakeAndDetach(); - //should only be one out edge of any resource node - txnNode = dynamic_cast(*(*sit)->out.begin()); - (*sit)->removeOutEdge(txnNode); - resources.erase(*sit); - delete *sit; - } + // should only be one out edge of any resource node + txnNode = dynamic_cast(*(*sit)->out.begin()); + (*sit)->removeOutEdge(txnNode); + resources.erase(*sit); + delete *sit; + } } bool LBIDResourceGraph::DFSStep(RGNode* curNode, uint64_t gray, uint64_t black) const { - set::iterator it; + set::iterator it; - if (curNode->color() == gray) + if (curNode->color() == gray) + return true; + + curNode->color(gray); + + for (it = curNode->out.begin(); it != curNode->out.end(); it++) + { + if ((*it)->color() != black) + if (DFSStep(*it, gray, black)) return true; + } - curNode->color(gray); + curNode->color(black); - for (it = curNode->out.begin(); it != curNode->out.end(); it++) - { - if ((*it)->color() != black) - if (DFSStep(*it, gray, black)) - return true; - } - - curNode->color(black); - - return false; + return false; } bool LBIDResourceGraph::checkDeadlock(TransactionNode& start) { - uint64_t gray = ++color, black = ++color; + uint64_t gray = ++color, black = ++color; - return DFSStep(&start, gray, black); + return DFSStep(&start, gray, black); } -} +} // namespace BRM diff --git a/versioning/BRM/lbidresourcegraph.h b/versioning/BRM/lbidresourcegraph.h index f2d92d417..3337b73fc 100644 --- a/versioning/BRM/lbidresourcegraph.h +++ b/versioning/BRM/lbidresourcegraph.h @@ -48,40 +48,37 @@ namespace BRM { - class LBIDResourceGraph { + public: + typedef std::tr1::unordered_set RNodes_t; -public: - typedef std::tr1::unordered_set RNodes_t; + EXPORT LBIDResourceGraph(); + EXPORT ~LBIDResourceGraph(); - EXPORT LBIDResourceGraph(); - EXPORT ~LBIDResourceGraph(); + EXPORT int reserveRange(LBID_t start, LBID_t end, VER_t txn, boost::mutex& mutex); - EXPORT int reserveRange(LBID_t start, LBID_t end, VER_t txn, boost::mutex& mutex); + /// releases all resources held by txn + EXPORT void releaseResources(VER_t txn); - /// releases all resources held by txn - EXPORT void releaseResources(VER_t txn); + /// releases one resource + EXPORT void releaseResource(LBID_t start); - /// releases one resource - EXPORT void releaseResource(LBID_t start); + private: + uint64_t color; -private: - uint64_t color; + LBIDResourceGraph(const LBIDResourceGraph&); + LBIDResourceGraph& operator=(const LBIDResourceGraph&); - LBIDResourceGraph(const LBIDResourceGraph&); - LBIDResourceGraph& operator=(const LBIDResourceGraph&); - - void connectResources(LBID_t start, LBID_t end, TransactionNode* txnNode); - bool checkDeadlock(TransactionNode&); - bool DFSStep(RGNode*, uint64_t, uint64_t) const; - - std::map txns; - RNodes_t resources; + void connectResources(LBID_t start, LBID_t end, TransactionNode* txnNode); + bool checkDeadlock(TransactionNode&); + bool DFSStep(RGNode*, uint64_t, uint64_t) const; + std::map txns; + RNodes_t resources; }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/load_brm.cpp b/versioning/BRM/load_brm.cpp index 6eaebd2af..4378fb766 100644 --- a/versioning/BRM/load_brm.cpp +++ b/versioning/BRM/load_brm.cpp @@ -38,97 +38,93 @@ using namespace BRM; namespace { - void usage() { - cout << "usage: load_brm [-fh] prefix" << endl << endl; - cout << " -h display this help" << endl; - cout << " -f possibly fix a corrupted Free List" << endl; + cout << "usage: load_brm [-fh] prefix" << endl << endl; + cout << " -h display this help" << endl; + cout << " -f possibly fix a corrupted Free List" << endl; } struct CtlShmImage { - bi::interprocess_mutex controlFifoMutex; + bi::interprocess_mutex controlFifoMutex; }; -} +} // namespace int main(int argc, char** argv) { - opterr = 0; - bool fflg = false; + opterr = 0; + bool fflg = false; - int c; + int c; - while ((c = getopt(argc, argv, "fh")) != EOF) - switch (c) - { - case 'f': - fflg = true; - break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if ((argc - optind) != 1) + while ((c = getopt(argc, argv, "fh")) != EOF) + switch (c) { + case 'f': fflg = true; break; + + case 'h': + case '?': + default: usage(); - return 1; + return (c == 'h' ? 0 : 1); + break; } - idbdatafile::IDBPolicy::configIDBPolicy(); + if ((argc - optind) != 1) + { + usage(); + return 1; + } - BlockResolutionManager brm; - int err; - string prefix; + idbdatafile::IDBPolicy::configIDBPolicy(); - prefix = argv[optind]; - err = brm.loadState(prefix, fflg); + BlockResolutionManager brm; + int err; + string prefix; - if (err != 0) - { - cout << "Loading BRM snapshot failed (" << prefix << ")\n"; - return 1; - } + prefix = argv[optind]; + err = brm.loadState(prefix, fflg); - err = brm.replayJournal(prefix); + if (err != 0) + { + cout << "Loading BRM snapshot failed (" << prefix << ")\n"; + return 1; + } - if (err < 0) - { - cout << "Could not load BRM journal file\n"; - return 1; - } + err = brm.replayJournal(prefix); - ShmKeys shmkeys; - string key_name = ShmKeys::keyToName(shmkeys.DECOMSVRMUTEX_SYSVKEY); - bi::shared_memory_object::remove(key_name.c_str()); - bi::permissions perms; - perms.set_unrestricted(); + if (err < 0) + { + cout << "Could not load BRM journal file\n"; + return 1; + } - try - { - bi::shared_memory_object shm(bi::create_only, key_name.c_str(), bi::read_write, perms); - shm.truncate(sizeof(CtlShmImage)); - bi::mapped_region region(shm, bi::read_write); - (void)new (region.get_address()) CtlShmImage; - } - catch (...) - { - //Hmm...we just deleted it above, but the create failed...just bail out - throw runtime_error("couldn't create DecomSvr shm"); - } + ShmKeys shmkeys; + string key_name = ShmKeys::keyToName(shmkeys.DECOMSVRMUTEX_SYSVKEY); + bi::shared_memory_object::remove(key_name.c_str()); + bi::permissions perms; + perms.set_unrestricted(); - /* An OAM friendly success msg */ - cout << "OK.\n"; - cout << "Successfully loaded BRM snapshot\n"; - cout << "Successfully replayed " << err << " BRM transactions\n"; + try + { + bi::shared_memory_object shm(bi::create_only, key_name.c_str(), bi::read_write, perms); + shm.truncate(sizeof(CtlShmImage)); + bi::mapped_region region(shm, bi::read_write); + (void)new (region.get_address()) CtlShmImage; + } + catch (...) + { + // Hmm...we just deleted it above, but the create failed...just bail out + throw runtime_error("couldn't create DecomSvr shm"); + } - return 0; + /* An OAM friendly success msg */ + cout << "OK.\n"; + cout << "Successfully loaded BRM snapshot\n"; + cout << "Successfully replayed " << err << " BRM transactions\n"; + + return 0; } // vim:ts=4 sw=4: - diff --git a/versioning/BRM/load_brm64.cpp b/versioning/BRM/load_brm64.cpp index 0df81fb30..ef31deae3 100644 --- a/versioning/BRM/load_brm64.cpp +++ b/versioning/BRM/load_brm64.cpp @@ -42,40 +42,40 @@ using namespace std; void usage(char* name) { - cout << "Usage: " << name << " " << endl; - exit(1); + cout << "Usage: " << name << " " << endl; + exit(1); } int main(int argc, char** argv) { #if __WORDSIZE <= 32 -//This code is OBE now that the structs are padded correctly - BlockResolutionManager brm; - int err; - string prefix; + // This code is OBE now that the structs are padded correctly + BlockResolutionManager brm; + int err; + string prefix; - if (argc > 2) - usage(argv[0]); - else if (argc == 2) - prefix = argv[1]; - else - prefix = "BRM_state"; + if (argc > 2) + usage(argv[0]); + else if (argc == 2) + prefix = argv[1]; + else + prefix = "BRM_state"; - idbdatafile::IDBPolicy::configIDBPolicy(); + idbdatafile::IDBPolicy::configIDBPolicy(); - err = brm.loadState(prefix); + err = brm.loadState(prefix); - if (err == 0) - cout << "OK." << endl; - else - { - cout << "Load failed" << endl; - return 1; - } - - return 0; -#else - cerr << "This tool does not work on 64-bit arch!" << endl; + if (err == 0) + cout << "OK." << endl; + else + { + cout << "Load failed" << endl; return 1; + } + + return 0; +#else + cerr << "This tool does not work on 64-bit arch!" << endl; + return 1; #endif } diff --git a/versioning/BRM/load_brm_from_file.cpp b/versioning/BRM/load_brm_from_file.cpp index 6b936b04f..bc7c05d9a 100644 --- a/versioning/BRM/load_brm_from_file.cpp +++ b/versioning/BRM/load_brm_from_file.cpp @@ -22,9 +22,10 @@ * this tool to create a binary BRM_saves_em file. * * compile with - * g++ -g -Wall -o load_brm_from_file -I$HOME/genii/export/include -I/usr/include/libxml2 load_brm_from_file.cpp + * g++ -g -Wall -o load_brm_from_file -I$HOME/genii/export/include -I/usr/include/libxml2 + * load_brm_from_file.cpp * -*/ + */ #include #include #include @@ -48,99 +49,99 @@ namespace BRM { EMEntry::EMEntry() { - fileID = 0; - blockOffset = 0; - HWM = 0; - partitionNum = 0; - segmentNum = 0; - dbRoot = 0; - colWid = 0; - status = 0; + fileID = 0; + blockOffset = 0; + HWM = 0; + partitionNum = 0; + segmentNum = 0; + dbRoot = 0; + colWid = 0; + status = 0; } EMCasualPartition_struct::EMCasualPartition_struct() { - lo_val = numeric_limits::min(); - hi_val = numeric_limits::max(); - sequenceNum = 0; - isValid = CP_INVALID; -} + lo_val = numeric_limits::min(); + hi_val = numeric_limits::max(); + sequenceNum = 0; + isValid = CP_INVALID; } +} // namespace BRM int main(int argc, char** argv) { - int e; + int e; - int loadSize[3]; + int loadSize[3]; - if (argc < 2) - { - cerr << "filename arg needed" << endl; - return 1; - } + if (argc < 2) + { + cerr << "filename arg needed" << endl; + return 1; + } - ifstream in(argv[1]); - e = errno; + ifstream in(argv[1]); + e = errno; - if (!in) - { - cerr << "file read error: " << strerror(e) << endl; - return 1; - } + if (!in) + { + cerr << "file read error: " << strerror(e) << endl; + return 1; + } - //Brute force count the number of lines - int numEMEntries = 0; + // Brute force count the number of lines + int numEMEntries = 0; - string line; + string line; + getline(in, line); + + while (!in.eof()) + { + numEMEntries++; getline(in, line); + } - while (!in.eof()) - { - numEMEntries++; - getline(in, line); - } + // start at the beginning again... + in.clear(); + in.seekg(0, ios_base::beg); - //start at the beginning again... - in.clear(); - in.seekg(0, ios_base::beg); + idbassert(in.good()); + idbassert(in.tellg() == static_cast(0)); - idbassert(in.good()); - idbassert(in.tellg() == static_cast(0)); + string outname(argv[1]); + outname += ".out"; - string outname(argv[1]); - outname += ".out"; + ofstream out(outname.c_str()); + e = errno; - ofstream out(outname.c_str()); - e = errno; + if (!out) + { + cerr << "file write error: " << strerror(e) << endl; + return 1; + } - if (!out) - { - cerr << "file write error: " << strerror(e) << endl; - return 1; - } + loadSize[0] = EM_MAGIC_V4; + loadSize[1] = numEMEntries; + loadSize[2] = 1; // one free list entry + out.write((char*)&loadSize, (3 * sizeof(int))); - loadSize[0] = EM_MAGIC_V4; - loadSize[1] = numEMEntries; - loadSize[2] = 1; //one free list entry - out.write((char*)&loadSize, (3 * sizeof(int))); + InlineLBIDRange fl; + fl.start = 0; + // the max lbid is 2^54-1, the size is in units of 1k + fl.size = numeric_limits::max(); - InlineLBIDRange fl; - fl.start = 0; - //the max lbid is 2^54-1, the size is in units of 1k - fl.size = numeric_limits::max(); + InlineLBIDRange maxLBIDinUse; + maxLBIDinUse.start = 0; + maxLBIDinUse.size = 0; - InlineLBIDRange maxLBIDinUse; - maxLBIDinUse.start = 0; - maxLBIDinUse.size = 0; + getline(in, line); - getline(in, line); - - while (!in.eof()) - { - EMEntry em; - int64_t v; - tokenizer<> tok(line); - tokenizer<>::iterator beg = tok.begin(); + while (!in.eof()) + { + EMEntry em; + int64_t v; + tokenizer<> tok(line); + tokenizer<>::iterator beg = tok.begin(); #if 0 emSrc[i].range.start << '\t' << emSrc[i].range.size @@ -157,81 +158,80 @@ int main(int argc, char** argv) << '\t' << emSrc[i].partition.cprange.sequenceNum << '\t' << (int)(emSrc[i].partition.cprange.isValid) #endif - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.range.start = v; + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.range.start = v; - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.range.size = v; + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.range.size = v; - if (em.range.start > maxLBIDinUse.start) - { - maxLBIDinUse.start = em.range.start; - maxLBIDinUse.size = em.range.size; - } - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.fileID = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.blockOffset = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.HWM = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partitionNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.segmentNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.dbRoot = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.colWid = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.status = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.hi_val = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.lo_val = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.sequenceNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.isValid = v; - - out.write((char*)&em, sizeof(em)); - - getline(in, line); + if (em.range.start > maxLBIDinUse.start) + { + maxLBIDinUse.start = em.range.start; + maxLBIDinUse.size = em.range.size; } - fl.start = maxLBIDinUse.start + maxLBIDinUse.size * 1024; - fl.size -= fl.start / 1024; + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.fileID = v; - out.write((char*)&fl, sizeof(fl)); + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.blockOffset = v; - out.close(); - in.close(); + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.HWM = v; - return 0; + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.partitionNum = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.segmentNum = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.dbRoot = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.colWid = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.status = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.partition.cprange.hi_val = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.partition.cprange.lo_val = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.partition.cprange.sequenceNum = v; + + v = strtoll(beg->c_str(), 0, 0); + ++beg; + em.partition.cprange.isValid = v; + + out.write((char*)&em, sizeof(em)); + + getline(in, line); + } + + fl.start = maxLBIDinUse.start + maxLBIDinUse.size * 1024; + fl.size -= fl.start / 1024; + + out.write((char*)&fl, sizeof(fl)); + + out.close(); + in.close(); + + return 0; } - diff --git a/versioning/BRM/load_em.cpp b/versioning/BRM/load_em.cpp index 43ec8eb0f..a15875efa 100644 --- a/versioning/BRM/load_em.cpp +++ b/versioning/BRM/load_em.cpp @@ -39,34 +39,34 @@ using namespace std; void usage(char* name) { - cout << "Usage: " << name << " " << endl; - exit(1); + cout << "Usage: " << name << " " << endl; + exit(1); } int main(int argc, char** argv) { - ExtentMap em; - string prefix; + ExtentMap em; + string prefix; - if (argc > 2) - usage(argv[0]); - else if (argc == 2) - prefix = argv[1]; - else - prefix = "BRM_state"; + if (argc > 2) + usage(argv[0]); + else if (argc == 2) + prefix = argv[1]; + else + prefix = "BRM_state"; - idbdatafile::IDBPolicy::configIDBPolicy(); + idbdatafile::IDBPolicy::configIDBPolicy(); - try - { - em.load(prefix); - cout << "OK." << endl; - } - catch (exception& e) - { - cout << "Load failed." << endl; - return 1; - } + try + { + em.load(prefix); + cout << "OK." << endl; + } + catch (exception& e) + { + cout << "Load failed." << endl; + return 1; + } - return 0; + return 0; } diff --git a/versioning/BRM/lock_grabber.cpp b/versioning/BRM/lock_grabber.cpp index 7293c0dc9..6932b7708 100644 --- a/versioning/BRM/lock_grabber.cpp +++ b/versioning/BRM/lock_grabber.cpp @@ -32,58 +32,58 @@ char* name; void usage() { - cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" << - " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n" << - " which_side_to_use: r|w (read or write)\n" << - " lock_or_unlock: l|u (lock or unlock)\n"; - exit(1); + cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" + << " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n" + << " which_side_to_use: r|w (read or write)\n" + << " lock_or_unlock: l|u (lock or unlock)\n"; + exit(1); } int main(int argc, char** argv) { - uint32_t which_lock; // 1-5 - uint32_t which_side; // 0 or 1 - uint32_t lock_unlock; // 0 or 1 - RWLock* rwlock; + uint32_t which_lock; // 1-5 + uint32_t which_side; // 0 or 1 + uint32_t lock_unlock; // 0 or 1 + RWLock* rwlock; - name = argv[0]; + name = argv[0]; - if (argc != 4) - usage(); + if (argc != 4) + usage(); - if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) - usage(); + if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) + usage(); - which_lock = atoi(argv[1]); + which_lock = atoi(argv[1]); - if (which_lock < 1 || which_lock > 5) - usage(); + if (which_lock < 1 || which_lock > 5) + usage(); - if (argv[2][0] == 'r') - which_side = 0; - else if (argv[2][0] == 'w') - which_side = 1; + if (argv[2][0] == 'r') + which_side = 0; + else if (argv[2][0] == 'w') + which_side = 1; + else + usage(); + + if (argv[3][0] == 'l') + lock_unlock = 0; + else if (argv[3][0] == 'u') + lock_unlock = 1; + else + usage(); + + rwlock = new RWLock(0x10000 * which_lock); + + if (which_side == 0) + if (lock_unlock == 0) + rwlock->read_lock(); else - usage(); + rwlock->read_unlock(); + else if (lock_unlock == 0) + rwlock->write_lock(); + else + rwlock->write_unlock(); - if (argv[3][0] == 'l') - lock_unlock = 0; - else if (argv[3][0] == 'u') - lock_unlock = 1; - else - usage(); - - rwlock = new RWLock(0x10000 * which_lock); - - if (which_side == 0) - if (lock_unlock == 0) - rwlock->read_lock(); - else - rwlock->read_unlock(); - else if (lock_unlock == 0) - rwlock->write_lock(); - else - rwlock->write_unlock(); - - return 0; + return 0; } diff --git a/versioning/BRM/lock_state.cpp b/versioning/BRM/lock_state.cpp index 9890236c9..812b59c08 100644 --- a/versioning/BRM/lock_state.cpp +++ b/versioning/BRM/lock_state.cpp @@ -32,37 +32,37 @@ char* name; void usage() { - cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" << - " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n"; - exit(1); + cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" + << " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n"; + exit(1); } int main(int argc, char** argv) { - uint32_t which_lock; // 1-5 - RWLock* rwlock; - LockState state; + uint32_t which_lock; // 1-5 + RWLock* rwlock; + LockState state; - name = argv[0]; + name = argv[0]; - if (argc != 2) - usage(); + if (argc != 2) + usage(); - if (strlen(argv[1]) != 1) - usage(); + if (strlen(argv[1]) != 1) + usage(); - which_lock = atoi(argv[1]); + which_lock = atoi(argv[1]); - if (which_lock < 1 || which_lock > 5) - usage(); + if (which_lock < 1 || which_lock > 5) + usage(); - rwlock = new RWLock(0x10000 * which_lock); - state = rwlock->getLockState(); - cout << "readers = " << state.reading << endl - << "writers = " << state.writing << endl - << "readers waiting = " << state.readerswaiting << endl - << "writers waiting = " << state.writerswaiting << endl - << "mutex locked = " << (int) state.mutexLocked << endl; + rwlock = new RWLock(0x10000 * which_lock); + state = rwlock->getLockState(); + cout << "readers = " << state.reading << endl + << "writers = " << state.writing << endl + << "readers waiting = " << state.readerswaiting << endl + << "writers waiting = " << state.writerswaiting << endl + << "mutex locked = " << (int)state.mutexLocked << endl; - return 0; + return 0; } diff --git a/versioning/BRM/logicalpartition.cpp b/versioning/BRM/logicalpartition.cpp index 45a5233b3..087ffbf18 100644 --- a/versioning/BRM/logicalpartition.cpp +++ b/versioning/BRM/logicalpartition.cpp @@ -34,9 +34,9 @@ namespace BRM { std::string LogicalPartition::toString() const { - char buf[256] = {0}; - std::sprintf(buf, "%d.%d.%d", pp, seg, dbroot); - return std::string(buf); + char buf[256] = {0}; + std::sprintf(buf, "%d.%d.%d", pp, seg, dbroot); + return std::string(buf); } /** @@ -44,18 +44,18 @@ std::string LogicalPartition::toString() const */ std::ostream& operator<<(std::ostream& output, const LogicalPartition& rhs) { - output << rhs.pp << "." << rhs.seg << "." << rhs.dbroot; - return output; + output << rhs.pp << "." << rhs.seg << "." << rhs.dbroot; + return output; } std::istream& operator>>(std::istream& input, LogicalPartition& rhs) { - input >> rhs.pp; - input.ignore(); - input >> rhs.seg; - input.ignore(); - input >> rhs.dbroot; - return input; + input >> rhs.pp; + input.ignore(); + input >> rhs.seg; + input.ignore(); + input >> rhs.dbroot; + return input; } /** @@ -63,14 +63,14 @@ std::istream& operator>>(std::istream& input, LogicalPartition& rhs) */ messageqcpp::ByteStream& operator<<(messageqcpp::ByteStream& bs, const LogicalPartition& rhs) { - rhs.serialize(bs); - return bs; + rhs.serialize(bs); + return bs; } messageqcpp::ByteStream& operator>>(messageqcpp::ByteStream& bs, LogicalPartition& rhs) { - rhs.unserialize(bs); - return bs; + rhs.unserialize(bs); + return bs; } -} +} // namespace BRM diff --git a/versioning/BRM/logicalpartition.h b/versioning/BRM/logicalpartition.h index e2445f742..868739d58 100644 --- a/versioning/BRM/logicalpartition.h +++ b/versioning/BRM/logicalpartition.h @@ -35,42 +35,39 @@ namespace BRM // Logical partition number descriptor struct LogicalPartition { - uint16_t dbroot; // dbroot # - uint32_t pp; // physical partition # - uint16_t seg; // segment # + uint16_t dbroot; // dbroot # + uint32_t pp; // physical partition # + uint16_t seg; // segment # - LogicalPartition() : dbroot ((uint16_t) - 1), - pp ((uint32_t) - 1), - seg ((uint16_t) - 1) {} + LogicalPartition() : dbroot((uint16_t)-1), pp((uint32_t)-1), seg((uint16_t)-1) + { + } - LogicalPartition(uint16_t d, uint32_t p, uint16_t s) : dbroot(d), - pp(p), - seg(s) - {} + LogicalPartition(uint16_t d, uint32_t p, uint16_t s) : dbroot(d), pp(p), seg(s) + { + } - bool operator<( const LogicalPartition& n ) const - { - return ((pp < n.pp) || - (pp == n.pp && seg < n.seg) || - (pp == n.pp && seg == n.seg && dbroot < n.dbroot)); - } + bool operator<(const LogicalPartition& n) const + { + return ((pp < n.pp) || (pp == n.pp && seg < n.seg) || (pp == n.pp && seg == n.seg && dbroot < n.dbroot)); + } - void serialize(messageqcpp::ByteStream& b) const - { - b << (uint16_t)dbroot; - b << (uint32_t)pp; - b << (uint16_t)seg; - } + void serialize(messageqcpp::ByteStream& b) const + { + b << (uint16_t)dbroot; + b << (uint32_t)pp; + b << (uint16_t)seg; + } - void unserialize(messageqcpp::ByteStream& b) - { - b >> (uint16_t&)dbroot; - b >> (uint32_t&)pp; - b >> (uint16_t&)seg; - } + void unserialize(messageqcpp::ByteStream& b) + { + b >> (uint16_t&)dbroot; + b >> (uint32_t&)pp; + b >> (uint16_t&)seg; + } - /** @bug4816. For output to user purpose */ - std::string toString() const; + /** @bug4816. For output to user purpose */ + std::string toString() const; }; /** @@ -86,5 +83,5 @@ messageqcpp::ByteStream& operator<<(messageqcpp::ByteStream& bs, const LogicalPa messageqcpp::ByteStream& operator>>(messageqcpp::ByteStream& bs, LogicalPartition& rhs); -} +} // namespace BRM #endif diff --git a/versioning/BRM/masterdbrmnode.cpp b/versioning/BRM/masterdbrmnode.cpp index 580b108e3..cc757266a 100644 --- a/versioning/BRM/masterdbrmnode.cpp +++ b/versioning/BRM/masterdbrmnode.cpp @@ -34,1098 +34,1014 @@ // #define BRM_VERBOSE // minor improvement to code clarity... -#define CHECK_ERROR1(x) \ - if (halting) { \ - for (it = responses.begin(); it != responses.end(); it++) \ - delete *it; \ - responses.clear(); \ - undo(); \ - slaveLock.unlock(); \ - usleep(50000); \ - goto retrycmd; \ - } \ - if (x) { \ - undo(); \ - slaveLock.unlock(); \ - sendError(p->sock, x); \ - goto out; \ - } else {} +#define CHECK_ERROR1(x) \ + if (halting) \ + { \ + for (it = responses.begin(); it != responses.end(); it++) \ + delete *it; \ + responses.clear(); \ + undo(); \ + slaveLock.unlock(); \ + usleep(50000); \ + goto retrycmd; \ + } \ + if (x) \ + { \ + undo(); \ + slaveLock.unlock(); \ + sendError(p->sock, x); \ + goto out; \ + } \ + else \ + { \ + } -#define THREAD_EXIT { \ - mutex.lock(); \ - for (vector::iterator _it = activeSessions.begin(); \ - _it != activeSessions.end(); ++_it) \ - if (p->sock == *_it) { \ - activeSessions.erase(_it); \ - break; \ - } \ - mutex.unlock(); \ - p->sock->close(); \ - delete p->sock; \ - delete p->t; \ - delete p; \ - for (it = responses.begin(); it != responses.end(); it++) \ - delete *it; } +#define THREAD_EXIT \ + { \ + mutex.lock(); \ + for (vector::iterator _it = activeSessions.begin(); _it != activeSessions.end(); ++_it) \ + if (p->sock == *_it) \ + { \ + activeSessions.erase(_it); \ + break; \ + } \ + mutex.unlock(); \ + p->sock->close(); \ + delete p->sock; \ + delete p->t; \ + delete p; \ + for (it = responses.begin(); it != responses.end(); it++) \ + delete *it; \ + } using namespace std; using namespace messageqcpp; using namespace logging; namespace BRM { - MasterDBRMNode::MasterDBRMNode() { - config::Config* config; + config::Config* config; - config = config::Config::makeConfig(); + config = config::Config::makeConfig(); - if (config == NULL) - throw invalid_argument("MasterDBRMNode: Configuration error."); + if (config == NULL) + throw invalid_argument("MasterDBRMNode: Configuration error."); - runners = 0; - die = false; - reloadCmd = false; - readOnly = false; - halting = false; + runners = 0; + die = false; + reloadCmd = false; + readOnly = false; + halting = false; - tableLockServer.reset(new TableLockServer(&sm)); - initMsgQueues(config); - rg = new LBIDResourceGraph(); - //@Bug 2325 DBRMTimeOut is default to 60 seconds - std::string retStr = config->getConfig("SystemConfig", "DBRMTimeOut"); - int secondsToWait = config->fromText(retStr); - MSG_TIMEOUT.tv_nsec = 0; + tableLockServer.reset(new TableLockServer(&sm)); + initMsgQueues(config); + rg = new LBIDResourceGraph(); + //@Bug 2325 DBRMTimeOut is default to 60 seconds + std::string retStr = config->getConfig("SystemConfig", "DBRMTimeOut"); + int secondsToWait = config->fromText(retStr); + MSG_TIMEOUT.tv_nsec = 0; - if ( secondsToWait > 0 ) - MSG_TIMEOUT.tv_sec = secondsToWait; - else - MSG_TIMEOUT.tv_sec = 20; + if (secondsToWait > 0) + MSG_TIMEOUT.tv_sec = secondsToWait; + else + MSG_TIMEOUT.tv_sec = 20; } MasterDBRMNode::~MasterDBRMNode() { - die = true; - finalCleanup(); + die = true; + finalCleanup(); } MasterDBRMNode::MasterDBRMNode(const MasterDBRMNode& m) { - throw logic_error("Don't use the MasterDBRMNode copy constructor"); + throw logic_error("Don't use the MasterDBRMNode copy constructor"); } MasterDBRMNode& MasterDBRMNode::operator=(const MasterDBRMNode& m) { - throw logic_error("Don't use the MasterDBRMNode = operator"); + throw logic_error("Don't use the MasterDBRMNode = operator"); } void MasterDBRMNode::initMsgQueues(config::Config* config) { - std::string methodName("MasterDBRMNode::initMsgQueues()"); - size_t connectTimeoutSecs = 0; - getNumWorkersAndTimeout(connectTimeoutSecs, methodName, config); + std::string methodName("MasterDBRMNode::initMsgQueues()"); + size_t connectTimeoutSecs = 0; + getNumWorkersAndTimeout(connectTimeoutSecs, methodName, config); - serverLock.lock(); - - try - { - dbrmServer = new MessageQueueServer("DBRM_Controller", config); - } - catch (...) - { - serverLock.unlock(); - throw; - } + serverLock.lock(); + try + { + dbrmServer = new MessageQueueServer("DBRM_Controller", config); + } + catch (...) + { serverLock.unlock(); + throw; + } - connectToWorkers(connectTimeoutSecs); + serverLock.unlock(); + + connectToWorkers(connectTimeoutSecs); } -void MasterDBRMNode::getNumWorkersAndTimeout(size_t& connectTimeoutSecs, - const std::string& methodName, +void MasterDBRMNode::getNumWorkersAndTimeout(size_t& connectTimeoutSecs, const std::string& methodName, config::Config* config) { - string stmp; - int ltmp; - connectTimeoutSecs = 30; // default + string stmp; + int ltmp; + connectTimeoutSecs = 30; // default - stmp = config->getConfig("DBRM_Controller", "NumWorkers"); + stmp = config->getConfig("DBRM_Controller", "NumWorkers"); - if (stmp.length() == 0) - throw runtime_error(methodName + - ": config file error looking for "); + if (stmp.length() == 0) + throw runtime_error(methodName + ": config file error looking for "); + ltmp = static_cast(config::Config::fromText(stmp)); + + if (ltmp < 1) + throw runtime_error(methodName + ": Bad NumWorkers value"); + + NumWorkers = ltmp; + stmp = config->getConfig("DBRM_Controller", "WorkerConnectionTimeout"); + if (stmp.length() > 0) + { ltmp = static_cast(config::Config::fromText(stmp)); - - if (ltmp < 1) - throw runtime_error(methodName + ": Bad NumWorkers value"); - - NumWorkers = ltmp; - stmp = config->getConfig("DBRM_Controller", "WorkerConnectionTimeout"); - if (stmp.length() > 0) - { - ltmp = static_cast(config::Config::fromText(stmp)); - if (ltmp > 1) - connectTimeoutSecs = ltmp; - } + if (ltmp > 1) + connectTimeoutSecs = ltmp; + } } void MasterDBRMNode::connectToWorkers(const size_t connectTimeoutSecs) { - size_t timeoutMsecs = connectTimeoutSecs * 1000000; - size_t timeSpent = 0; - int workersOnline = 0; - bool initialRun = true; - while (timeoutMsecs > timeSpent && workersOnline < NumWorkers) + size_t timeoutMsecs = connectTimeoutSecs * 1000000; + size_t timeSpent = 0; + int workersOnline = 0; + bool initialRun = true; + while (timeoutMsecs > timeSpent && workersOnline < NumWorkers) + { + char ctmp[50]; + for (int i = 0; i < NumWorkers; i++) { - char ctmp[50]; - for (int i = 0; i < NumWorkers; i++) - { - snprintf(ctmp, sizeof(ctmp), "DBRM_Worker%d", i+1); - std::string module(ctmp); - if (static_cast(slaves.size()) < NumWorkers) - { - slaves.push_back(MessageQueueClientPool::getInstance(module)); - } + snprintf(ctmp, sizeof(ctmp), "DBRM_Worker%d", i + 1); + std::string module(ctmp); + if (static_cast(slaves.size()) < NumWorkers) + { + slaves.push_back(MessageQueueClientPool::getInstance(module)); + } - if (!slaves[i]->isConnected()) - { - if (!slaves[i]->connect()) - { - // first iteration - if (initialRun) - log("DBRM Controller: Warning: could not connect to " + module, - logging::LOG_TYPE_WARNING); - } - else - { - log("DBRM Controller: Connected to " + module, - logging::LOG_TYPE_DEBUG); - workersOnline++; - } - } - } - if (initialRun) - initialRun = false; - if (workersOnline < NumWorkers) + if (!slaves[i]->isConnected()) + { + if (!slaves[i]->connect()) { - usleep(connectTimeoutStep); - timeSpent += connectTimeoutStep; + // first iteration + if (initialRun) + log("DBRM Controller: Warning: could not connect to " + module, logging::LOG_TYPE_WARNING); } + else + { + log("DBRM Controller: Connected to " + module, logging::LOG_TYPE_DEBUG); + workersOnline++; + } + } } + if (initialRun) + initialRun = false; + if (workersOnline < NumWorkers) + { + usleep(connectTimeoutStep); + timeSpent += connectTimeoutStep; + } + } } void MasterDBRMNode::stop() { - die = true; + die = true; } void MasterDBRMNode::lock() { - slaveLock.lock(); + slaveLock.lock(); } void MasterDBRMNode::unlock() { - slaveLock.unlock(); + slaveLock.unlock(); } void MasterDBRMNode::reload() { - config::Config* config; + config::Config* config; #ifdef BRM_VERBOSE - cerr << "DBRM Controller: reloading the config file." << endl; + cerr << "DBRM Controller: reloading the config file." << endl; #endif - reloadCmd = false; - config = config::Config::makeConfig(); + reloadCmd = false; + config = config::Config::makeConfig(); - if (config == NULL) - throw runtime_error("DBRM Controller: Configuration error. Reload aborted."); + if (config == NULL) + throw runtime_error("DBRM Controller: Configuration error. Reload aborted."); - die = true; - finalCleanup(); + die = true; + finalCleanup(); #ifdef BRM_VERBOSE - cerr << "DBRM Controller: reinitializing..." << endl; + cerr << "DBRM Controller: reinitializing..." << endl; #endif - try - { - initMsgQueues(config); - } - catch (exception&) - { - throw runtime_error("DBRM Controller: Configuration error. Reload aborted."); - // masterdbrm.run() will exit after this - } + try + { + initMsgQueues(config); + } + catch (exception&) + { + throw runtime_error("DBRM Controller: Configuration error. Reload aborted."); + // masterdbrm.run() will exit after this + } - die = false; + die = false; - rg = new LBIDResourceGraph(); + rg = new LBIDResourceGraph(); #ifdef BRM_VERBOSE - cerr << "DBRM Controller: reload successful." << endl; + cerr << "DBRM Controller: reload successful." << endl; #endif - readOnly = false; + readOnly = false; } void MasterDBRMNode::setReadOnly(bool ro) { - slaveLock.lock(); - readOnly = ro; - slaveLock.unlock(); + slaveLock.lock(); + readOnly = ro; + slaveLock.unlock(); } void MasterDBRMNode::run() { - ByteStream msg; - IOSocket* s; - boost::thread* reader; + ByteStream msg; + IOSocket* s; + boost::thread* reader; - while (!die) - { - s = new IOSocket(); + while (!die) + { + s = new IOSocket(); #ifdef BRM_VERBOSE - cerr << "DBRM Controller waiting..." << endl; + cerr << "DBRM Controller waiting..." << endl; #endif - serverLock.lock(); - - if (dbrmServer != NULL) - try - { - *s = dbrmServer->accept(&MSG_TIMEOUT); - } - catch (runtime_error& e) - { - cerr << e.what() << " continuing...\n"; - serverLock.unlock(); - delete s; - continue; - } + serverLock.lock(); + if (dbrmServer != NULL) + try + { + *s = dbrmServer->accept(&MSG_TIMEOUT); + } + catch (runtime_error& e) + { + cerr << e.what() << " continuing...\n"; serverLock.unlock(); + delete s; + continue; + } - if (reloadCmd) - { - if (s->isOpen()) - s->close(); + serverLock.unlock(); - delete s; - reload(); - continue; - } + if (reloadCmd) + { + if (s->isOpen()) + s->close(); - if (die || !s->isOpen()) - { - if (s->isOpen()) - s->close(); - - delete s; - continue; - } - -#ifdef BRM_VERBOSE - cerr << "DBRM Controller: got a connection..." << endl; -#endif - mutex.lock(); - activeSessions.push_back(s); - params = new ThreadParams(); -#ifdef BRM_VERBOSE - cerr << "DBRM Controller: starting another thread" << endl; -#endif - mutex2.lock(); - - try - { - reader = new boost::thread(MsgProcessor(this)); - } - catch (boost::thread_resource_error&) - { - log("DBRM Controller: WARNING!! Got thread resource error! Increase system stack size or decrease the # of active sessions."); -#ifdef BRM_VERBOSE - cerr << "DBRM Controller: WARNING!! Got thread resource error! Increase system stack size or decrease the # of active sessions.\n"; -#endif - activeSessions.pop_back(); - sendError(s, ERR_NETWORK); - sleep(1); // don't close right away to avoid broken pipe on the client - s->close(); - delete s; - delete params; - mutex2.unlock(); - mutex.unlock(); - continue; - } - - params->t = reader; - params->sock = s; - mutex2.unlock(); -#ifdef __FreeBSD__ - mutex.unlock(); -#endif + delete s; + reload(); + continue; } - serverLock.lock(); - delete dbrmServer; - dbrmServer = NULL; - serverLock.unlock(); + if (die || !s->isOpen()) + { + if (s->isOpen()) + s->close(); + + delete s; + continue; + } + +#ifdef BRM_VERBOSE + cerr << "DBRM Controller: got a connection..." << endl; +#endif + mutex.lock(); + activeSessions.push_back(s); + params = new ThreadParams(); +#ifdef BRM_VERBOSE + cerr << "DBRM Controller: starting another thread" << endl; +#endif + mutex2.lock(); + + try + { + reader = new boost::thread(MsgProcessor(this)); + } + catch (boost::thread_resource_error&) + { + log("DBRM Controller: WARNING!! Got thread resource error! Increase system stack size or decrease " + "the # of active sessions."); +#ifdef BRM_VERBOSE + cerr << "DBRM Controller: WARNING!! Got thread resource error! Increase system stack size or " + "decrease the # of active sessions.\n"; +#endif + activeSessions.pop_back(); + sendError(s, ERR_NETWORK); + sleep(1); // don't close right away to avoid broken pipe on the client + s->close(); + delete s; + delete params; + mutex2.unlock(); + mutex.unlock(); + continue; + } + + params->t = reader; + params->sock = s; + mutex2.unlock(); +#ifdef __FreeBSD__ + mutex.unlock(); +#endif + } + + serverLock.lock(); + delete dbrmServer; + dbrmServer = NULL; + serverLock.unlock(); } void MasterDBRMNode::msgProcessor() { - struct ThreadParams* p; - ByteStream msg; - vector responses; - vector::iterator it; - int err; - uint8_t cmd; - StopWatch timer; + struct ThreadParams* p; + ByteStream msg; + vector responses; + vector::iterator it; + int err; + uint8_t cmd; + StopWatch timer; #ifdef BRM_VERBOSE - cerr << "DBRM Controller: msgProcessor()" << endl; + cerr << "DBRM Controller: msgProcessor()" << endl; #endif - mutex2.lock(); - p = params; - mutex2.unlock(); + mutex2.lock(); + p = params; + mutex2.unlock(); #ifndef __FreeBSD__ - mutex.unlock(); + mutex.unlock(); #endif - while (!die) + while (!die) + { + try { + msg = p->sock->read(&MSG_TIMEOUT); + } + catch (...) + { + THREAD_EXIT + throw; + } + + if (die) // || msg.length() == 0) + break; + + else if (msg.length() == 0) + continue; + + /* Check for an command for the master */ + msg.peek(cmd); +#ifdef BRM_VERBOSE + cerr << "DBRM Controller: recv'd message " << (int)cmd << " length " << msg.length() << endl; +#endif + + switch (cmd) + { + case HALT: doHalt(p->sock); continue; + + case RESUME: doResume(p->sock); continue; + + case RELOAD: try { - msg = p->sock->read(&MSG_TIMEOUT); + doReload(p->sock); } - catch (...) + catch (exception& e) { - THREAD_EXIT - throw; + cerr << e.what() << endl; } - if (die) // || msg.length() == 0) - break; + continue; - else if (msg.length() == 0) - continue; + case SETREADONLY: doSetReadOnly(p->sock, true); continue; - /* Check for an command for the master */ - msg.peek(cmd); -#ifdef BRM_VERBOSE - cerr << "DBRM Controller: recv'd message " << (int)cmd << " length " << msg.length() << endl; -#endif + case SETREADWRITE: doSetReadOnly(p->sock, false); continue; - switch (cmd) + case GETREADONLY: doGetReadOnly(p->sock); continue; + + case GET_SYSTEM_CATALOG: doGetSystemCatalog(msg, p); continue; + } + + /* Process SessionManager calls */ + switch (cmd) + { + case VER_ID: doVerID(msg, p); continue; + + case SYSCAT_VER_ID: doSysCatVerID(msg, p); continue; + + case NEW_TXN_ID: doNewTxnID(msg, p); continue; + + case COMMITTED: doCommitted(msg, p); continue; + + case ROLLED_BACK: doRolledBack(msg, p); continue; + + case GET_TXN_ID: doGetTxnID(msg, p); continue; + + case SID_TID_MAP: doSIDTIDMap(msg, p); continue; + + case GET_UNIQUE_UINT32: doGetUniqueUint32(msg, p); continue; + + case GET_UNIQUE_UINT64: doGetUniqueUint64(msg, p); continue; + + case GET_SYSTEM_STATE: doGetSystemState(msg, p); continue; + + case SET_SYSTEM_STATE: doSetSystemState(msg, p); continue; + + case CLEAR_SYSTEM_STATE: doClearSystemState(msg, p); continue; + + case SM_RESET: doSessionManagerReset(msg, p); continue; + + case GET_UNCOMMITTED_LBIDS: doGetUncommittedLbids(msg, p); continue; + } + + /* Process TableLock calls */ + switch (cmd) + { + case GET_TABLE_LOCK: doGetTableLock(msg, p); continue; + + case RELEASE_TABLE_LOCK: doReleaseTableLock(msg, p); continue; + + case CHANGE_TABLE_LOCK_STATE: doChangeTableLockState(msg, p); continue; + + case CHANGE_TABLE_LOCK_OWNER: doChangeTableLockOwner(msg, p); continue; + + case GET_ALL_TABLE_LOCKS: doGetAllTableLocks(msg, p); continue; + + case RELEASE_ALL_TABLE_LOCKS: doReleaseAllTableLocks(msg, p); continue; + + case GET_TABLE_LOCK_INFO: doGetTableLockInfo(msg, p); continue; + + case OWNER_CHECK: doOwnerCheck(msg, p); continue; + } + + /* Process OIDManager calls */ + switch (cmd) + { + case ALLOC_OIDS: doAllocOIDs(msg, p); continue; + + case RETURN_OIDS: doReturnOIDs(msg, p); continue; + + case OIDM_SIZE: doOidmSize(msg, p); continue; + + case ALLOC_VBOID: doAllocVBOID(msg, p); continue; + + case GETDBROOTOFVBOID: doGetDBRootOfVBOID(msg, p); continue; + + case GETVBOIDTODBROOTMAP: doGetVBOIDToDBRootMap(msg, p); continue; + } + + /* Process Autoincrement calls */ + switch (cmd) + { + case START_AI_SEQUENCE: doStartAISequence(msg, p); continue; + + case GET_AI_RANGE: doGetAIRange(msg, p); continue; + + case RESET_AI_SEQUENCE: doResetAISequence(msg, p); continue; + + case GET_AI_LOCK: doGetAILock(msg, p); continue; + + case RELEASE_AI_LOCK: doReleaseAILock(msg, p); continue; + + case DELETE_AI_SEQUENCE: doDeleteAISequence(msg, p); continue; + } + + retrycmd: + uint32_t haltloops = 0; + + while (halting && ++haltloops < static_cast(FIVE_MIN_TIMEOUT.tv_sec)) + sleep(1); + + slaveLock.lock(); + + if (haltloops == FIVE_MIN_TIMEOUT.tv_sec) + { + ostringstream os; + os << "A node is unresponsive for cmd = " << (uint32_t)cmd << ", no reconfigure in at least " + << FIVE_MIN_TIMEOUT.tv_sec << " seconds. Setting read-only mode."; + log(os.str()); + readOnly = true; + halting = false; + } + + if (readOnly) + { + slaveLock.unlock(); + sendError(p->sock, ERR_READONLY); + goto out; + } + + /* TODO: Separate these out-of-band items into separate functions */ + + /* Need to get the dbroot, convert to vbOID */ + if (cmd == BEGIN_VB_COPY) + { + try + { + boost::mutex::scoped_lock lk(oidsMutex); + uint8_t* buf = msg.buf(); + + // dbroot is currently after the cmd and transid + uint16_t* dbRoot = (uint16_t*)&buf[1 + 4]; + + // If that dbroot has no vboid, create one + int16_t err; + err = oids.getVBOIDOfDBRoot(*dbRoot); + + // cout << "dbRoot " << *dbRoot << " -> vbOID " << err << endl; + if (err < 0) { - case HALT: - doHalt(p->sock); - continue; - - case RESUME: - doResume(p->sock); - continue; - - case RELOAD: - try - { - doReload(p->sock); - } - catch (exception& e) - { - cerr << e.what() << endl; - } - - continue; - - case SETREADONLY: - doSetReadOnly(p->sock, true); - continue; - - case SETREADWRITE: - doSetReadOnly(p->sock, false); - continue; - - case GETREADONLY: - doGetReadOnly(p->sock); - continue; - - case GET_SYSTEM_CATALOG: - doGetSystemCatalog(msg, p); - continue; + err = oids.allocVBOID(*dbRoot); + // cout << " - allocated oid " << err << endl; } - /* Process SessionManager calls */ - switch (cmd) + *dbRoot = err; + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: Begin VBCopy failure. " << ex.what(); + log(os.str()); + sendError(p->sock, -1); + goto out; + } + } + + /* Check for deadlock on beginVBCopy */ + /* if (cmd == BEGIN_VB_COPY) { + ByteStream params(msg); + VER_t txn; + uint8_t tmp8; + uint32_t tmp32; + LBIDRange_v ranges; + LBIDRange_v::iterator it; + LBID_t end; + + params >> tmp8; //throw away the cmd + params >> tmp32; + txn = tmp32; + deserializeVector(params, ranges); + for (it = ranges.begin(); it != ranges.end(); it++) { + end = (*it).start + (*it).size - 1; + err = rg->reserveRange((*it).start, end, txn, slaveLock); + if (err != ERR_OK) { + pthread_mutex_unlock(&slaveLock); + sendError(p->sock, err); + goto out; + } + } + } + */ + /* Release all blocks of lbids on vbRollback or vbCommit */ + if (cmd == VB_ROLLBACK1 || cmd == VB_ROLLBACK2 || cmd == VB_COMMIT) + { + ByteStream params(msg); + VER_t txn; + uint8_t tmp8; + uint32_t tmp32; + + params >> tmp8; + params >> tmp32; + txn = tmp32; + rg->releaseResources(txn); + } + + /* XXXPAT: If beginVBCopy, vbRollback, or vbCommit fail for some reason, + the resource graph will be out of sync with the copylocks and/or vss. + There are 2 reasons they might fail: + 1) logical inconsistency between the resource graph and the + copylocks & vss + 2) "out of band" failures, like a network problem + */ + + /* Need to atomically do the safety check and the clear. */ + if (cmd == BRM_CLEAR) + { + uint32_t txnCount = sm.getTxnCount(); + + // do nothing if there's an active transaction + if (txnCount != 0) + { + ByteStream* reply = new ByteStream(); + *reply << (uint8_t)ERR_FAILURE; + responses.push_back(reply); + goto no_confirm; + } + } + + for (int retry = 0;; retry++) + { + try + { + distribute(&msg); + } + catch (...) + { + if (!halting) { - case VER_ID: - doVerID(msg, p); - continue; - - case SYSCAT_VER_ID: - doSysCatVerID(msg, p); - continue; - - case NEW_TXN_ID: - doNewTxnID(msg, p); - continue; - - case COMMITTED: - doCommitted(msg, p); - continue; - - case ROLLED_BACK: - doRolledBack(msg, p); - continue; - - case GET_TXN_ID: - doGetTxnID(msg, p); - continue; - - case SID_TID_MAP: - doSIDTIDMap(msg, p); - continue; - - case GET_UNIQUE_UINT32: - doGetUniqueUint32(msg, p); - continue; - - case GET_UNIQUE_UINT64: - doGetUniqueUint64(msg, p); - continue; - - case GET_SYSTEM_STATE: - doGetSystemState(msg, p); - continue; - - case SET_SYSTEM_STATE: - doSetSystemState(msg, p); - continue; - - case CLEAR_SYSTEM_STATE: - doClearSystemState(msg, p); - continue; - - case SM_RESET: - doSessionManagerReset(msg, p); - continue; - - case GET_UNCOMMITTED_LBIDS: - doGetUncommittedLbids(msg, p); - continue; + undo(); + readOnly = true; + slaveLock.unlock(); + ostringstream ostr; + ostr << "DBRM Controller: Caught network error. " + "Sending command " + << (uint32_t)cmd << ", length " << msg.length() << ". Setting read-only mode."; + log(ostr.str()); + sendError(p->sock, ERR_NETWORK); + goto out; } - - /* Process TableLock calls */ - switch (cmd) - { - case GET_TABLE_LOCK: - doGetTableLock(msg, p); - continue; - - case RELEASE_TABLE_LOCK: - doReleaseTableLock(msg, p); - continue; - - case CHANGE_TABLE_LOCK_STATE: - doChangeTableLockState(msg, p); - continue; - - case CHANGE_TABLE_LOCK_OWNER: - doChangeTableLockOwner(msg, p); - continue; - - case GET_ALL_TABLE_LOCKS: - doGetAllTableLocks(msg, p); - continue; - - case RELEASE_ALL_TABLE_LOCKS: - doReleaseAllTableLocks(msg, p); - continue; - - case GET_TABLE_LOCK_INFO: - doGetTableLockInfo(msg, p); - continue; - - case OWNER_CHECK: - doOwnerCheck(msg, p); - continue; - } - - /* Process OIDManager calls */ - switch (cmd) - { - case ALLOC_OIDS: - doAllocOIDs(msg, p); - continue; - - case RETURN_OIDS: - doReturnOIDs(msg, p); - continue; - - case OIDM_SIZE: - doOidmSize(msg, p); - continue; - - case ALLOC_VBOID: - doAllocVBOID(msg, p); - continue; - - case GETDBROOTOFVBOID: - doGetDBRootOfVBOID(msg, p); - continue; - - case GETVBOIDTODBROOTMAP: - doGetVBOIDToDBRootMap(msg, p); - continue; - } - - /* Process Autoincrement calls */ - switch (cmd) - { - case START_AI_SEQUENCE: - doStartAISequence(msg, p); - continue; - - case GET_AI_RANGE: - doGetAIRange(msg, p); - continue; - - case RESET_AI_SEQUENCE: - doResetAISequence(msg, p); - continue; - - case GET_AI_LOCK: - doGetAILock(msg, p); - continue; - - case RELEASE_AI_LOCK: - doReleaseAILock(msg, p); - continue; - - case DELETE_AI_SEQUENCE: - doDeleteAISequence(msg, p); - continue; - } - -retrycmd: - uint32_t haltloops = 0; - - while (halting && ++haltloops < static_cast(FIVE_MIN_TIMEOUT.tv_sec)) - sleep(1); - - slaveLock.lock(); - - if (haltloops == FIVE_MIN_TIMEOUT.tv_sec) - { - ostringstream os; - os << "A node is unresponsive for cmd = " << (uint32_t)cmd << - ", no reconfigure in at least " << - FIVE_MIN_TIMEOUT.tv_sec << " seconds. Setting read-only mode."; - log(os.str()); - readOnly = true; - halting = false; - } - - if (readOnly) - { - slaveLock.unlock(); - sendError(p->sock, ERR_READONLY); - goto out; - } - - /* TODO: Separate these out-of-band items into separate functions */ - - /* Need to get the dbroot, convert to vbOID */ - if (cmd == BEGIN_VB_COPY) - { - try - { - boost::mutex::scoped_lock lk(oidsMutex); - uint8_t* buf = msg.buf(); - - // dbroot is currently after the cmd and transid - uint16_t* dbRoot = (uint16_t*) &buf[1 + 4]; - - // If that dbroot has no vboid, create one - int16_t err; - err = oids.getVBOIDOfDBRoot(*dbRoot); - - //cout << "dbRoot " << *dbRoot << " -> vbOID " << err << endl; - if (err < 0) - { - err = oids.allocVBOID(*dbRoot); - // cout << " - allocated oid " << err << endl; - } - - *dbRoot = err; - } - catch (exception& ex) - { - ostringstream os; - os << "DBRM Controller: Begin VBCopy failure. " << ex.what(); - log(os.str()); - sendError(p->sock, -1); - goto out; - } - } - - /* Check for deadlock on beginVBCopy */ - /* if (cmd == BEGIN_VB_COPY) { - ByteStream params(msg); - VER_t txn; - uint8_t tmp8; - uint32_t tmp32; - LBIDRange_v ranges; - LBIDRange_v::iterator it; - LBID_t end; - - params >> tmp8; //throw away the cmd - params >> tmp32; - txn = tmp32; - deserializeVector(params, ranges); - for (it = ranges.begin(); it != ranges.end(); it++) { - end = (*it).start + (*it).size - 1; - err = rg->reserveRange((*it).start, end, txn, slaveLock); - if (err != ERR_OK) { - pthread_mutex_unlock(&slaveLock); - sendError(p->sock, err); - goto out; - } - } - } - */ - /* Release all blocks of lbids on vbRollback or vbCommit */ - if (cmd == VB_ROLLBACK1 || cmd == VB_ROLLBACK2 || cmd == VB_COMMIT) - { - ByteStream params(msg); - VER_t txn; - uint8_t tmp8; - uint32_t tmp32; - - params >> tmp8; - params >> tmp32; - txn = tmp32; - rg->releaseResources(txn); - } - - /* XXXPAT: If beginVBCopy, vbRollback, or vbCommit fail for some reason, - the resource graph will be out of sync with the copylocks and/or vss. - There are 2 reasons they might fail: - 1) logical inconsistency between the resource graph and the - copylocks & vss - 2) "out of band" failures, like a network problem - */ - - /* Need to atomically do the safety check and the clear. */ - if (cmd == BRM_CLEAR) - { - uint32_t txnCount = sm.getTxnCount(); - - // do nothing if there's an active transaction - if (txnCount != 0) - { - ByteStream* reply = new ByteStream(); - *reply << (uint8_t) ERR_FAILURE; - responses.push_back(reply); - goto no_confirm; - } - } - - for ( int retry = 0 ; ; retry ++ ) - { - try - { - distribute(&msg); - } - catch (...) - { - if (!halting) - { - undo(); - readOnly = true; - slaveLock.unlock(); - ostringstream ostr; - ostr << "DBRM Controller: Caught network error. " - "Sending command " << (uint32_t)cmd << - ", length " << msg.length() << ". Setting read-only mode."; - log(ostr.str()); - sendError(p->sock, ERR_NETWORK); - goto out; - } - } + } #ifdef BRM_VERBOSE - cerr << "DBRM Controller: distributed msg" << endl; + cerr << "DBRM Controller: distributed msg" << endl; #endif - bool readErrFlag; // ignore this flag in this case - err = gatherResponses(cmd, msg.length(), &responses, readErrFlag); + bool readErrFlag; // ignore this flag in this case + err = gatherResponses(cmd, msg.length(), &responses, readErrFlag); #ifdef BRM_VERBOSE - cerr << "DBRM Controller: got responses" << endl; + cerr << "DBRM Controller: got responses" << endl; #endif - CHECK_ERROR1(err) - err = compareResponses(cmd, msg.length(), responses); + CHECK_ERROR1(err) + err = compareResponses(cmd, msg.length(), responses); #ifdef BRM_VERBOSE - cerr << "DBRM Controller: compared responses" << endl; + cerr << "DBRM Controller: compared responses" << endl; #endif - if ( err != ERR_SLAVE_INCONSISTENCY ) - { - break; - } - else - { - if ( retry > 1 ) - { - readOnly = true; - ostringstream ostr; - ostr << "DBRM Controller: image inconsistency detected multi times. Setting read-only mode."; - log(ostr.str()); - break; - } - } + if (err != ERR_SLAVE_INCONSISTENCY) + { + break; + } + else + { + if (retry > 1) + { + readOnly = true; + ostringstream ostr; + ostr << "DBRM Controller: image inconsistency detected multi times. Setting read-only mode."; + log(ostr.str()); + break; } + } + } #ifdef BRM_DEBUG - if ((cmd == BEGIN_VB_COPY || cmd == VB_ROLLBACK1 || cmd == VB_ROLLBACK2 || - cmd == VB_COMMIT) && err == -1) - cerr << "DBRM Controller: inconsistency detected between the resource graph and the VSS or CopyLocks logic." << endl; + if ((cmd == BEGIN_VB_COPY || cmd == VB_ROLLBACK1 || cmd == VB_ROLLBACK2 || cmd == VB_COMMIT) && err == -1) + cerr << "DBRM Controller: inconsistency detected between the resource graph and the VSS or CopyLocks " + "logic." + << endl; #endif - // these command will have error message carried in the response - if (!responses.empty() && (cmd == DELETE_PARTITION || cmd == MARK_PARTITION_FOR_DELETION || cmd == RESTORE_PARTITION) - && err) - { - if (err != ERR_PARTITION_DISABLED && err != ERR_PARTITION_ENABLED && - err != ERR_INVALID_OP_LAST_PARTITION && err != ERR_NOT_EXIST_PARTITION && - err != ERR_NO_PARTITION_PERFORMED) - undo(); + // these command will have error message carried in the response + if (!responses.empty() && + (cmd == DELETE_PARTITION || cmd == MARK_PARTITION_FOR_DELETION || cmd == RESTORE_PARTITION) && err) + { + if (err != ERR_PARTITION_DISABLED && err != ERR_PARTITION_ENABLED && + err != ERR_INVALID_OP_LAST_PARTITION && err != ERR_NOT_EXIST_PARTITION && + err != ERR_NO_PARTITION_PERFORMED) + undo(); - //goto no_confirm; - } - else - { - CHECK_ERROR1(err) - } - - // these cmds don't need the 2-phase commit - if (cmd == FLUSH_INODE_CACHES || cmd == BRM_CLEAR || cmd == TAKE_SNAPSHOT) - goto no_confirm; - -#ifdef BRM_VERBOSE - cerr << "DBRM Controller: sending confirmation" << endl; -#endif - - try - { - confirm(); - } - catch (...) - { - if (!halting) - { - ostringstream ostr; - ostr << "DBRM Controller: Caught network error. " - "Confirming command " << (uint32_t)cmd << - ", length " << msg.length() << ". Setting read-only mode."; - log(ostr.str()); - readOnly = true; - } - } - -no_confirm: - slaveLock.unlock(); - - try - { - p->sock->write(*(responses.front())); - } - catch (...) - { - p->sock->close(); - log("DBRM Controller: Warning: could not send the reply to a command", logging::LOG_TYPE_WARNING); - } - -out: - - for (it = responses.begin(); it != responses.end(); it++) - delete *it; - - responses.clear(); + // goto no_confirm; + } + else + { + CHECK_ERROR1(err) } + // these cmds don't need the 2-phase commit + if (cmd == FLUSH_INODE_CACHES || cmd == BRM_CLEAR || cmd == TAKE_SNAPSHOT) + goto no_confirm; + #ifdef BRM_VERBOSE - cerr << "DBRM Controller: closing connection" << endl; + cerr << "DBRM Controller: sending confirmation" << endl; #endif - THREAD_EXIT - return; + + try + { + confirm(); + } + catch (...) + { + if (!halting) + { + ostringstream ostr; + ostr << "DBRM Controller: Caught network error. " + "Confirming command " + << (uint32_t)cmd << ", length " << msg.length() << ". Setting read-only mode."; + log(ostr.str()); + readOnly = true; + } + } + + no_confirm: + slaveLock.unlock(); + + try + { + p->sock->write(*(responses.front())); + } + catch (...) + { + p->sock->close(); + log("DBRM Controller: Warning: could not send the reply to a command", logging::LOG_TYPE_WARNING); + } + + out: + + for (it = responses.begin(); it != responses.end(); it++) + delete *it; + + responses.clear(); + } + +#ifdef BRM_VERBOSE + cerr << "DBRM Controller: closing connection" << endl; +#endif + THREAD_EXIT + return; } void MasterDBRMNode::distribute(ByteStream* msg) { - uint32_t i; + uint32_t i; - for (i = 0, iSlave = slaves.begin(); iSlave != slaves.end() && !halting; iSlave++, i++) - try - { - (*iSlave)->write(*msg); - } - catch (exception&) - { - if (!halting) - { - ostringstream os; - os << "DBRM Controller: network error distributing command to worker " << - i + 1 << endl; - log(os.str()); - throw; - } - } + for (i = 0, iSlave = slaves.begin(); iSlave != slaves.end() && !halting; iSlave++, i++) + try + { + (*iSlave)->write(*msg); + } + catch (exception&) + { + if (!halting) + { + ostringstream os; + os << "DBRM Controller: network error distributing command to worker " << i + 1 << endl; + log(os.str()); + throw; + } + } } // readErrFlag is a separate return flag used by doChangeTableLockOwner() // (or any subsequent function) which calls gatherResponses() outside the // scope of msgProcessor() which instead uses the halting flag for error // handling. -int MasterDBRMNode::gatherResponses(uint8_t cmd, - uint32_t cmdMsgLength, - vector* responses, +int MasterDBRMNode::gatherResponses(uint8_t cmd, uint32_t cmdMsgLength, vector* responses, bool& readErrFlag) throw() { - int i; - ByteStream* tmp = 0; - readErrFlag = false; + int i; + ByteStream* tmp = 0; + readErrFlag = false; - //Bug 2258 gather all responses - int error = 0; + // Bug 2258 gather all responses + int error = 0; - for (i = 0, iSlave = slaves.begin(); iSlave != slaves.end() && !halting; iSlave++, i++) - { - tmp = new ByteStream(); - - try - { - // can't just block for 5 mins - timespec newtimeout = {10, 0}; - uint32_t ntRetries = FIVE_MIN_TIMEOUT.tv_sec / newtimeout.tv_sec; - uint32_t retries = 0; - - while (++retries < ntRetries && tmp->length() == 0 && !halting) - *tmp = (*iSlave)->read(&newtimeout); - - //*tmp = (*iSlave)->read(); - } - catch (...) - { - /* 2/21/12 - instead of setting readonly here, to support a peaceful failover - we will wait for a configuration change to come, then report the error - after a long timeout. - */ - - ostringstream os; - os << "DBRM Controller: Network error reading from node " << i + 1 << - ". Reading response to command " << (uint32_t)cmd << - ", length " << cmdMsgLength << ". Will see if retry is possible."; - log(os.str()); - - halting = true; - readErrFlag = true; - delete tmp; - return ERR_OK; - - /* - ostringstream os; - if (!halting) { - SEND_ALARM - readOnly = true; - os << "DBRM Controller: Network error reading from node " << i + 1 << - ". Reading response to command " << (uint32_t)cmd << - ", length " << cmdMsgLength << ". Setting read-only mode."; - log(os.str()); - error++; - } - */ - } - - if (tmp->length() == 0 && !halting) - { - /* See the comment above */ - ostringstream os; - os << "DBRM Controller: Network error reading from node " << i + 1 << - ". Reading response to command " << (uint32_t)cmd << - ", length " << cmdMsgLength << - ". 0 length response, possible time-out" - ". Will see if retry is possible."; - log(os.str()); - halting = true; - readErrFlag = true; - delete tmp; - return ERR_OK; - - /* - ostringstream os; - - SEND_ALARM; - - readOnly = true; - os << "DBRM Controller: Network error reading from node " << i + 1<< - ". Reading response to command " << (uint32_t)cmd << - ", length " << cmdMsgLength << - ". 0 length response, possible time-out" - ". Setting read-only mode."; - log(os.str()); - error++; - */ - } - - if ( error == 0 ) - responses->push_back(tmp); - else - delete tmp; - } - - if ( error > 0 ) - return ERR_NETWORK; - else - return ERR_OK; -} - -int MasterDBRMNode::compareResponses(uint8_t cmd, - uint32_t cmdMsgLength, - const vector& responses) const -{ - vector::const_iterator it, it2; - uint8_t errCode; - ByteStream* first; - int i; - - first = responses.front(); + for (i = 0, iSlave = slaves.begin(); iSlave != slaves.end() && !halting; iSlave++, i++) + { + tmp = new ByteStream(); try { - first->peek(errCode); + // can't just block for 5 mins + timespec newtimeout = {10, 0}; + uint32_t ntRetries = FIVE_MIN_TIMEOUT.tv_sec / newtimeout.tv_sec; + uint32_t retries = 0; + + while (++retries < ntRetries && tmp->length() == 0 && !halting) + *tmp = (*iSlave)->read(&newtimeout); + + //*tmp = (*iSlave)->read(); } - catch (exception&) + catch (...) { - readOnly = true; - ostringstream os; - os << "DBRM Controller: Network error on node 1. " - "Verifying response to command " << (uint32_t)cmd << - ", length " << cmdMsgLength << ". Reverting to read-only mode."; - log(os.str()); - return ERR_NETWORK; + /* 2/21/12 - instead of setting readonly here, to support a peaceful failover + we will wait for a configuration change to come, then report the error + after a long timeout. + */ + + ostringstream os; + os << "DBRM Controller: Network error reading from node " << i + 1 << ". Reading response to command " + << (uint32_t)cmd << ", length " << cmdMsgLength << ". Will see if retry is possible."; + log(os.str()); + + halting = true; + readErrFlag = true; + delete tmp; + return ERR_OK; + + /* + ostringstream os; + if (!halting) { + SEND_ALARM + readOnly = true; + os << "DBRM Controller: Network error reading from node " << i + 1 << + ". Reading response to command " << (uint32_t)cmd << + ", length " << cmdMsgLength << ". Setting read-only mode."; + log(os.str()); + error++; + } + */ } - /*if (errCode != ERR_OK) { - #ifdef BRM_VERBOSE - cerr << "DBRM Controller: first response has error code " << errCode << endl; - #endif - return errCode; - }*/ + if (tmp->length() == 0 && !halting) + { + /* See the comment above */ + ostringstream os; + os << "DBRM Controller: Network error reading from node " << i + 1 << ". Reading response to command " + << (uint32_t)cmd << ", length " << cmdMsgLength + << ". 0 length response, possible time-out" + ". Will see if retry is possible."; + log(os.str()); + halting = true; + readErrFlag = true; + delete tmp; + return ERR_OK; - for (it = responses.begin(), it2 = it + 1, i = 2; it2 != responses.end(); it++, it2++, i++) - if (**it != **it2 && !halting) - { - ostringstream ostr; + /* + ostringstream os; + + SEND_ALARM; + + readOnly = true; + os << "DBRM Controller: Network error reading from node " << i + 1<< + ". Reading response to command " << (uint32_t)cmd << + ", length " << cmdMsgLength << + ". 0 length response, possible time-out" + ". Setting read-only mode."; + log(os.str()); + error++; + */ + } + + if (error == 0) + responses->push_back(tmp); + else + delete tmp; + } + + if (error > 0) + return ERR_NETWORK; + else + return ERR_OK; +} + +int MasterDBRMNode::compareResponses(uint8_t cmd, uint32_t cmdMsgLength, + const vector& responses) const +{ + vector::const_iterator it, it2; + uint8_t errCode; + ByteStream* first; + int i; + + first = responses.front(); + + try + { + first->peek(errCode); + } + catch (exception&) + { + readOnly = true; + ostringstream os; + os << "DBRM Controller: Network error on node 1. " + "Verifying response to command " + << (uint32_t)cmd << ", length " << cmdMsgLength << ". Reverting to read-only mode."; + log(os.str()); + return ERR_NETWORK; + } + + /*if (errCode != ERR_OK) { + #ifdef BRM_VERBOSE + cerr << "DBRM Controller: first response has error code " << errCode << endl; + #endif + return errCode; + }*/ + + for (it = responses.begin(), it2 = it + 1, i = 2; it2 != responses.end(); it++, it2++, i++) + if (**it != **it2 && !halting) + { + ostringstream ostr; #ifdef BRM_VERBOSE - cerr << "DBRM Controller: error: response from node " << i << " is different" << endl; + cerr << "DBRM Controller: error: response from node " << i << " is different" << endl; #endif -// readOnly = true; - ostr << "DBRM Controller: image inconsistency detected at node " << i << - ". Verifying response to command " << (uint32_t)cmd << - ", length " << cmdMsgLength; - log(ostr.str()); - return ERR_SLAVE_INCONSISTENCY; - } + // readOnly = true; + ostr << "DBRM Controller: image inconsistency detected at node " << i + << ". Verifying response to command " << (uint32_t)cmd << ", length " << cmdMsgLength; + log(ostr.str()); + return ERR_SLAVE_INCONSISTENCY; + } - //return ERR_OK; - return errCode; + // return ERR_OK; + return errCode; } void MasterDBRMNode::undo() throw() { - vector::iterator it, lastSlave; - ByteStream undoMsg; - int i; + vector::iterator it, lastSlave; + ByteStream undoMsg; + int i; #ifdef BRM_VERBOSE - cerr << "DBRM Controller: sending undo()" << endl; + cerr << "DBRM Controller: sending undo()" << endl; #endif - undoMsg << (uint8_t) BRM_UNDO; + undoMsg << (uint8_t)BRM_UNDO; - if (iSlave != slaves.end()) - lastSlave = iSlave + 1; //@Bug 2258 - else - lastSlave = iSlave; + if (iSlave != slaves.end()) + lastSlave = iSlave + 1; //@Bug 2258 + else + lastSlave = iSlave; - for (it = slaves.begin(), i = 1; it != lastSlave; it++, i++) + for (it = slaves.begin(), i = 1; it != lastSlave; it++, i++) + { + try { - try - { #ifdef BRM_VERBOSE - cerr << "DBRM Controller: sending undo() to worker number " << i << endl; + cerr << "DBRM Controller: sending undo() to worker number " << i << endl; #endif - (*it)->write(undoMsg); - } - catch (...) - { - ostringstream ostr; - - ostr << "DBRM Controller: undo(): warning, could not contact worker number " - << i << endl; - log(ostr.str()); - } + (*it)->write(undoMsg); } + catch (...) + { + ostringstream ostr; + + ostr << "DBRM Controller: undo(): warning, could not contact worker number " << i << endl; + log(ostr.str()); + } + } } void MasterDBRMNode::confirm() { - ByteStream confirmMsg; + ByteStream confirmMsg; - confirmMsg << CONFIRM; - distribute(&confirmMsg); + confirmMsg << CONFIRM; + distribute(&confirmMsg); } void MasterDBRMNode::finalCleanup() { - vector::iterator sIt; - int retry = 0; + vector::iterator sIt; + int retry = 0; - cerr << "DBRM Controller: Waiting for threads to finish..." << endl; + cerr << "DBRM Controller: Waiting for threads to finish..." << endl; - delete rg; //unblocks any waiting transactions - rg = NULL; + delete rg; // unblocks any waiting transactions + rg = NULL; - // XXXPAT: assumption here: join_all() blocks until all threads are joined - // which implies the case where all threads are removed from the group. - // We're relying on that second condition for synchronization here. - // Problem: it looks as if join_all holds a mutex which prevents other calls - // on dbrmSessions, so threads can't be removed from the group. -// dbrmSessions.join_all(); - // blah: busy wait for now, max 15 seconds, then assume everything's dead. - // @bug 1381: change retry from 15 to 5 more than the socket read() timeout. - while (runners > 0 && retry++ < (MSG_TIMEOUT.tv_sec + 5)) - sleep(1); + // XXXPAT: assumption here: join_all() blocks until all threads are joined + // which implies the case where all threads are removed from the group. + // We're relying on that second condition for synchronization here. + // Problem: it looks as if join_all holds a mutex which prevents other calls + // on dbrmSessions, so threads can't be removed from the group. + // dbrmSessions.join_all(); + // blah: busy wait for now, max 15 seconds, then assume everything's dead. + // @bug 1381: change retry from 15 to 5 more than the socket read() timeout. + while (runners > 0 && retry++ < (MSG_TIMEOUT.tv_sec + 5)) + sleep(1); #ifdef BRM_VERBOSE - cerr << "Closing connections" << endl; + cerr << "Closing connections" << endl; #endif - for (sIt = slaves.begin(); sIt != slaves.end(); sIt++) - { - MessageQueueClientPool::releaseInstance(*sIt); - *sIt = NULL; - } + for (sIt = slaves.begin(); sIt != slaves.end(); sIt++) + { + MessageQueueClientPool::releaseInstance(*sIt); + *sIt = NULL; + } - slaves.clear(); + slaves.clear(); - /* Close any connections that lost their thread somehow. This should "never" happen. - */ + /* Close any connections that lost their thread somehow. This should "never" happen. + */ -#if 0 // if we see instances of blockage here flip this switch +#if 0 // if we see instances of blockage here flip this switch int tmp; tmp = pthread_mutex_trylock(&mutex); @@ -1150,1578 +1066,1682 @@ void MasterDBRMNode::finalCleanup() } #else - mutex.lock(); + mutex.lock(); #endif #ifdef BRM_VERBOSE - if (activeSessions.size() > 0) - cerr << "There are still live sessions\n"; + if (activeSessions.size() > 0) + cerr << "There are still live sessions\n"; #endif - for (uint32_t i = 0; i < activeSessions.size(); ++i) - { - activeSessions[i]->close(); - delete activeSessions[i]; - } + for (uint32_t i = 0; i < activeSessions.size(); ++i) + { + activeSessions[i]->close(); + delete activeSessions[i]; + } - activeSessions.clear(); - mutex.unlock(); + activeSessions.clear(); + mutex.unlock(); - serverLock.lock(); - delete dbrmServer; - dbrmServer = NULL; - serverLock.unlock(); + serverLock.lock(); + delete dbrmServer; + dbrmServer = NULL; + serverLock.unlock(); } void MasterDBRMNode::sendError(IOSocket* caller, uint8_t err) const throw() { - ByteStream msg; + ByteStream msg; #ifdef BRM_VERBOSE - cerr << "DBRM Controller: returning " << (int) err << " to the caller" << endl; + cerr << "DBRM Controller: returning " << (int)err << " to the caller" << endl; #endif - msg << err; + msg << err; - try - { - caller->write(msg); - } - catch (exception&) - { - log("DBRM Controller: failed to send return code", logging::LOG_TYPE_WARNING); + try + { + caller->write(msg); + } + catch (exception&) + { + log("DBRM Controller: failed to send return code", logging::LOG_TYPE_WARNING); #ifdef BRM_VERBOSE - cerr << "DBRM Controller: failed to send return code" << endl; + cerr << "DBRM Controller: failed to send return code" << endl; #endif - } + } } void MasterDBRMNode::doHalt(messageqcpp::IOSocket* sock) { - ByteStream reply; + ByteStream reply; - log("Halting...", LOG_TYPE_INFO); - halting = true; - lock(); - log("Halted", LOG_TYPE_INFO); - reply << (uint8_t) ERR_OK; + log("Halting...", LOG_TYPE_INFO); + halting = true; + lock(); + log("Halted", LOG_TYPE_INFO); + reply << (uint8_t)ERR_OK; - try - { - sock->write(reply); - } - catch (exception&) { } + try + { + sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doResume(messageqcpp::IOSocket* sock) { - ByteStream reply; + ByteStream reply; - unlock(); - halting = false; - log("Resuming", LOG_TYPE_INFO); - reply << (uint8_t) ERR_OK; + unlock(); + halting = false; + log("Resuming", LOG_TYPE_INFO); + reply << (uint8_t)ERR_OK; - try - { - sock->write(reply); - } - catch (exception&) { } + try + { + sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doSetReadOnly(messageqcpp::IOSocket* sock, bool b) { - ByteStream reply; + ByteStream reply; - setReadOnly(b); - reply << (uint8_t) ERR_OK; + setReadOnly(b); + reply << (uint8_t)ERR_OK; - try - { - sock->write(reply); - } - catch (exception&) { } + try + { + sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doGetReadOnly(messageqcpp::IOSocket* sock) { - ByteStream reply; + ByteStream reply; - reply << (uint8_t)isReadOnly(); + reply << (uint8_t)isReadOnly(); - try - { - sock->write(reply); - } - catch (exception&) { } + try + { + sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doReload(messageqcpp::IOSocket* sock) { - /* This version relies on the caller to do a 'halt' and 'resume' around - * the 'reload' call, but it is synchronous. When reload() returns - * the new workernode connections have been established. - */ + /* This version relies on the caller to do a 'halt' and 'resume' around + * the 'reload' call, but it is synchronous. When reload() returns + * the new workernode connections have been established. + */ - ByteStream reply; - config::Config* config = config::Config::makeConfig(); + ByteStream reply; + config::Config* config = config::Config::makeConfig(); - log("Reloading", LOG_TYPE_INFO); + log("Reloading", LOG_TYPE_INFO); - std::string methodName("MasterDBRMNode::doReload()"); - size_t connectTimeoutSecs = 0; - try - { - getNumWorkersAndTimeout(connectTimeoutSecs, methodName, config); - } - catch (std::exception&) - { - reply << (uint8_t) ERR_FAILURE; - - try - { - sock->write(reply); - } - catch (exception&) { } - - throw; - } - - for (int i = 0; i < (int) slaves.size(); i++) - { - MessageQueueClientPool::deleteInstance(slaves[i]); - slaves[i] = nullptr; - } - - slaves.clear(); - - connectToWorkers(connectTimeoutSecs); - - iSlave = slaves.end(); - undo(); - - readOnly = false; - - reply << (uint8_t) ERR_OK; + std::string methodName("MasterDBRMNode::doReload()"); + size_t connectTimeoutSecs = 0; + try + { + getNumWorkersAndTimeout(connectTimeoutSecs, methodName, config); + } + catch (std::exception&) + { + reply << (uint8_t)ERR_FAILURE; try { - sock->write(reply); + sock->write(reply); + } + catch (exception&) + { } - catch (exception&) { } + throw; + } + + for (int i = 0; i < (int)slaves.size(); i++) + { + MessageQueueClientPool::deleteInstance(slaves[i]); + slaves[i] = nullptr; + } + + slaves.clear(); + + connectToWorkers(connectTimeoutSecs); + + iSlave = slaves.end(); + undo(); + + readOnly = false; + + reply << (uint8_t)ERR_OK; + + try + { + sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doVerID(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - QueryContext context; + ByteStream reply; + QueryContext context; - context = sm.verID(); + context = sm.verID(); #ifdef BRM_VERBOSE - cerr << "doVerID returning " << ver << endl; + cerr << "doVerID returning " << ver << endl; #endif - reply << (uint8_t) ERR_OK; - reply << context; + reply << (uint8_t)ERR_OK; + reply << context; - try - { - p->sock->write(reply); - } - catch (exception&) { } + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doGetSystemCatalog(ByteStream& msg, ThreadParams* p) { - ByteStream reply; + ByteStream reply; - reply << (uint8_t) ERR_OK; + reply << (uint8_t)ERR_OK; - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(); - const std::vector< std::pair > catalog_tables - = systemCatalogPtr->getTables(); + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(); + const std::vector< + std::pair > + catalog_tables = systemCatalogPtr->getTables(); - reply << (uint32_t) catalog_tables.size(); + reply << (uint32_t)catalog_tables.size(); - for (std::vector >::const_iterator it = catalog_tables.begin(); - it != catalog_tables.end(); ++it) + for (std::vector >::const_iterator it = + catalog_tables.begin(); + it != catalog_tables.end(); ++it) + { + execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second); + reply << (uint32_t)(*it).first; + reply << (*it).second.schema; + reply << (*it).second.table; + reply << (uint32_t)tb_info.numOfCols; + execplan::CalpontSystemCatalog::RIDList column_rid_list = + systemCatalogPtr->columnRIDs((*it).second, true); + + for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) { - execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second); - reply << (uint32_t)(*it).first; - reply << (*it).second.schema; - reply << (*it).second.table; - reply << (uint32_t)tb_info.numOfCols; - execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true); + execplan::CalpontSystemCatalog::TableColName tcn = + systemCatalogPtr->colName(column_rid_list[col_num].objnum); + execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum); + reply << (uint32_t)column_rid_list[col_num].objnum; + reply << tcn.column; - for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) - { - execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum); - execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum); - reply << (uint32_t)column_rid_list[col_num].objnum; - reply << tcn.column; + if (ct.ddn.dictOID == std::numeric_limits::min()) + { + reply << (uint32_t)0; + } + else + { + reply << (uint32_t)ct.ddn.dictOID; + } - if (ct.ddn.dictOID == std::numeric_limits::min()) - { - reply << (uint32_t) 0; - } - else - { - reply << (uint32_t) ct.ddn.dictOID; - } + reply << (uint8_t)ct.colDataType; + reply << (uint32_t)ct.colWidth; + reply << (uint32_t)ct.colPosition; + reply << ct.defaultValue; + reply << (uint8_t)ct.autoincrement; + reply << (uint32_t)ct.precision; + reply << (uint32_t)ct.scale; - reply << (uint8_t) ct.colDataType; - reply << (uint32_t) ct.colWidth; - reply << (uint32_t) ct.colPosition; - reply << ct.defaultValue; - reply << (uint8_t) ct.autoincrement; - reply << (uint32_t) ct.precision; - reply << (uint32_t) ct.scale; + if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + reply << (uint8_t)1; + } + else + { + reply << (uint8_t)0; + } - if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - reply << (uint8_t) 1; - } - else - { - reply << (uint8_t) 0; - } - - reply << (uint8_t) ct.compressionType; - } + reply << (uint8_t)ct.compressionType; } + } - try - { - p->sock->write(reply); - } - catch (exception&) { } + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doSysCatVerID(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - QueryContext context; + ByteStream reply; + QueryContext context; - context = sm.sysCatVerID(); + context = sm.sysCatVerID(); #ifdef BRM_VERBOSE - cerr << "doSysCatVerID returning " << ver << endl; + cerr << "doSysCatVerID returning " << ver << endl; #endif - reply << (uint8_t) ERR_OK; - reply << context; + reply << (uint8_t)ERR_OK; + reply << context; - try - { - p->sock->write(reply); - } - catch (exception&) { } + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doNewTxnID(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - TxnID txnid; - uint32_t sessionID; - uint8_t block, cmd, isDDL; + ByteStream reply; + TxnID txnid; + uint32_t sessionID; + uint8_t block, cmd, isDDL; - try - { - msg >> cmd; - msg >> sessionID; - msg >> block; - msg >> isDDL; - txnid = sm.newTxnID(sessionID, (block != 0), (isDDL != 0)); - reply << (uint8_t) ERR_OK; - reply << (uint32_t) txnid.id; - reply << (uint8_t) txnid.valid; + try + { + msg >> cmd; + msg >> sessionID; + msg >> block; + msg >> isDDL; + txnid = sm.newTxnID(sessionID, (block != 0), (isDDL != 0)); + reply << (uint8_t)ERR_OK; + reply << (uint32_t)txnid.id; + reply << (uint8_t)txnid.valid; #ifdef BRM_VERBOSE - cerr << "newTxnID returning id=" << txnid.id << " valid=" << txnid.valid << endl; + cerr << "newTxnID returning id=" << txnid.id << " valid=" << txnid.valid << endl; #endif - } - catch (exception&) - { - reply.reset(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } + } + catch (exception&) + { + reply.reset(); + reply << (uint8_t)ERR_FAILURE; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) { } + catch (...) + { + } + + return; + } + + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doCommitted(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - TxnID txnid; - uint8_t cmd, tmp; - uint32_t tmp32; + ByteStream reply; + TxnID txnid; + uint8_t cmd, tmp; + uint32_t tmp32; - try - { - msg >> cmd; - msg >> tmp32; - txnid.id = tmp32; - msg >> tmp; - txnid.valid = (tmp != 0 ? true : false); + try + { + msg >> cmd; + msg >> tmp32; + txnid.id = tmp32; + msg >> tmp; + txnid.valid = (tmp != 0 ? true : false); #ifdef BRM_VERBOSE - cerr << "doCommitted" << endl; + cerr << "doCommitted" << endl; #endif - sm.committed(txnid); - } - catch (exception&) - { - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } - - reply << (uint8_t) ERR_OK; + sm.committed(txnid); + } + catch (exception&) + { + reply << (uint8_t)ERR_FAILURE; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (...) { } + catch (...) + { + } + + return; + } + + reply << (uint8_t)ERR_OK; + + try + { + p->sock->write(reply); + } + catch (...) + { + } } void MasterDBRMNode::doRolledBack(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - TxnID txnid; - uint8_t cmd, tmp; - uint32_t tmp32; + ByteStream reply; + TxnID txnid; + uint8_t cmd, tmp; + uint32_t tmp32; - try - { - msg >> cmd; - msg >> tmp32; - msg >> tmp; + try + { + msg >> cmd; + msg >> tmp32; + msg >> tmp; - txnid.id = tmp32; - txnid.valid = (tmp != 0 ? true : false); + txnid.id = tmp32; + txnid.valid = (tmp != 0 ? true : false); #ifdef BRM_VERBOSE - cerr << "doRolledBack" << endl; + cerr << "doRolledBack" << endl; #endif - sm.rolledback(txnid); - } - catch (exception&) - { - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } - - reply << (uint8_t) ERR_OK; + sm.rolledback(txnid); + } + catch (exception&) + { + reply << (uint8_t)ERR_FAILURE; try { - p->sock->write(reply); + p->sock->write(reply); + } + catch (...) + { } - catch (...) { } -} + return; + } + + reply << (uint8_t)ERR_OK; + + try + { + p->sock->write(reply); + } + catch (...) + { + } +} void MasterDBRMNode::doGetTxnID(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - SessionManagerServer::SID sid; - TxnID txnid; - uint8_t cmd; + ByteStream reply; + SessionManagerServer::SID sid; + TxnID txnid; + uint8_t cmd; - try - { - msg >> cmd; - msg >> sid; + try + { + msg >> cmd; + msg >> sid; - txnid = sm.getTxnID(sid); + txnid = sm.getTxnID(sid); #ifdef BRM_VERBOSE - cerr << "doGetTxnID returning id=" << txnid.id << " valid=" << - txnid.valid << endl; + cerr << "doGetTxnID returning id=" << txnid.id << " valid=" << txnid.valid << endl; #endif - } - catch (exception&) - { - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } - - reply << (uint8_t) ERR_OK; - reply << (uint32_t) txnid.id; - reply << (uint8_t) txnid.valid; + } + catch (exception&) + { + reply << (uint8_t)ERR_FAILURE; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (...) { } + catch (...) + { + } + + return; + } + + reply << (uint8_t)ERR_OK; + reply << (uint32_t)txnid.id; + reply << (uint8_t)txnid.valid; + + try + { + p->sock->write(reply); + } + catch (...) + { + } } void MasterDBRMNode::doSIDTIDMap(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - int len, i; - boost::shared_array entries; + ByteStream reply; + int len, i; + boost::shared_array entries; + + try + { + entries = sm.SIDTIDMap(len); + } + catch (exception&) + { + reply << (uint8_t)ERR_FAILURE; try { - entries = sm.SIDTIDMap(len); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; } - reply << (uint8_t) ERR_OK; - reply << (uint32_t) len; + return; + } + + reply << (uint8_t)ERR_OK; + reply << (uint32_t)len; #ifdef BRM_VERBOSE - cerr << "doSIDTIDMap returning " << len << " elements... " << endl; + cerr << "doSIDTIDMap returning " << len << " elements... " << endl; #endif - for (i = 0; i < len; i++) - { + for (i = 0; i < len; i++) + { #ifdef BRM_VERBOSE - cerr << " " << i << ": txnid=" << entries[i].txnid.id << " valid=" - << entries[i].txnid.valid << " sessionid=" << entries[i].sessionid - << endl; + cerr << " " << i << ": txnid=" << entries[i].txnid.id << " valid=" << entries[i].txnid.valid + << " sessionid=" << entries[i].sessionid << endl; #endif - reply << (uint32_t) entries[i].txnid.id << (uint8_t) entries[i].txnid.valid << - (uint32_t) entries[i].sessionid; - } + reply << (uint32_t)entries[i].txnid.id << (uint8_t)entries[i].txnid.valid + << (uint32_t)entries[i].sessionid; + } - try - { - p->sock->write(reply); - } - catch (...) { } + try + { + p->sock->write(reply); + } + catch (...) + { + } } void MasterDBRMNode::doGetUncommittedLbids(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - vector lbidList; - VSS vss; - ExtentMap em; - bool locked = false; - vector::iterator lbidIt; - typedef pair range_t; - range_t range; - vector ranges; - vector::iterator rangeIt; - ByteStream::byte cmd; - ByteStream::quadbyte transID; - msg >> cmd; - msg >> transID; + ByteStream reply; + vector lbidList; + VSS vss; + ExtentMap em; + bool locked = false; + vector::iterator lbidIt; + typedef pair range_t; + range_t range; + vector ranges; + vector::iterator rangeIt; + ByteStream::byte cmd; + ByteStream::quadbyte transID; + msg >> cmd; + msg >> transID; + + try + { + vss.lock(VSS::READ); + locked = true; + + // Get a full list of uncommitted LBIDs related to this transactin. + vss.getUncommittedLBIDs(transID, lbidList); + + vss.release(VSS::READ); + locked = false; + + if (lbidList.size() > 0) + { + // Sort the vector. + std::sort::iterator>(lbidList.begin(), lbidList.end()); + + // Get the LBID range for the first block in the list. + lbidIt = lbidList.begin(); + + if (em.lookup(*lbidIt, range.first, range.second) < 0) + { + reply.reset(); + reply << (uint8_t)ERR_FAILURE; + + try + { + p->sock->write(reply); + } + catch (...) + { + } + + return; + } + + ranges.push_back(range); + + // Loop through the LBIDs and add the new ranges. + ++lbidIt; + + while (lbidIt != lbidList.end()) + { + if (*lbidIt > range.second) + { + if (em.lookup(*lbidIt, range.first, range.second) < 0) + { + reply.reset(); + reply << (uint8_t)ERR_FAILURE; + + try + { + p->sock->write(reply); + } + catch (...) + { + } + + return; + } + + ranges.push_back(range); + } + + ++lbidIt; + } + + // Reset the lbidList and return only the first LBID in each extent that was changed + // in the transaction. + lbidList.clear(); + + for (rangeIt = ranges.begin(); rangeIt != ranges.end(); rangeIt++) + { + lbidList.push_back(rangeIt->first); + } + } + + reply << (uint8_t)ERR_OK; + serializeInlineVector(reply, lbidList); try { - vss.lock(VSS::READ); - locked = true; - - // Get a full list of uncommitted LBIDs related to this transactin. - vss.getUncommittedLBIDs(transID, lbidList); - - vss.release(VSS::READ); - locked = false; - - if (lbidList.size() > 0) - { - - // Sort the vector. - std::sort::iterator>(lbidList.begin(), lbidList.end()); - - // Get the LBID range for the first block in the list. - lbidIt = lbidList.begin(); - - if (em.lookup(*lbidIt, range.first, range.second) < 0) - { - reply.reset(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } - - ranges.push_back(range); - - // Loop through the LBIDs and add the new ranges. - ++lbidIt; - - while (lbidIt != lbidList.end()) - { - if (*lbidIt > range.second) - { - if (em.lookup(*lbidIt, range.first, range.second) < 0) - { - reply.reset(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - - } - - ranges.push_back(range); - } - - ++lbidIt; - } - - // Reset the lbidList and return only the first LBID in each extent that was changed - // in the transaction. - lbidList.clear(); - - for (rangeIt = ranges.begin(); rangeIt != ranges.end(); rangeIt++) - { - lbidList.push_back(rangeIt->first); - } - } - - reply << (uint8_t) ERR_OK; - serializeInlineVector(reply, lbidList); - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; + p->sock->write(reply); } - catch (exception& e) + catch (...) { - if (locked) - vss.release(VSS::READ); - - reply.reset(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; } + + return; + } + catch (exception& e) + { + if (locked) + vss.release(VSS::READ); + + reply.reset(); + reply << (uint8_t)ERR_FAILURE; + + try + { + p->sock->write(reply); + } + catch (...) + { + } + + return; + } } void MasterDBRMNode::doGetUniqueUint32(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint32_t ret; + ByteStream reply; + uint32_t ret; - try - { - ret = sm.getUnique32(); - reply << (uint8_t) ERR_OK; - reply << ret; + try + { + ret = sm.getUnique32(); + reply << (uint8_t)ERR_OK; + reply << ret; #ifdef BRM_VERBOSE - cerr << "getUnique32() returning " << ret << endl; + cerr << "getUnique32() returning " << ret << endl; #endif - } - catch (exception&) - { - reply.reset(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } + } + catch (exception&) + { + reply.reset(); + reply << (uint8_t)ERR_FAILURE; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) { } + catch (...) + { + } + + return; + } + + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doGetUniqueUint64(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint64_t ret; + ByteStream reply; + uint64_t ret; - try - { - ret = sm.getUnique64(); - reply << (uint8_t) ERR_OK; - reply << ret; + try + { + ret = sm.getUnique64(); + reply << (uint8_t)ERR_OK; + reply << ret; #ifdef BRM_VERBOSE - cerr << "getUnique64() returning " << ret << endl; + cerr << "getUnique64() returning " << ret << endl; #endif - } - catch (exception&) - { - reply.reset(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } + } + catch (exception&) + { + reply.reset(); + reply << (uint8_t)ERR_FAILURE; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) { } + catch (...) + { + } + + return; + } + + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doGetSystemState(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint32_t ss = 0; - ByteStream::byte err = ERR_FAILURE; + ByteStream reply; + uint32_t ss = 0; + ByteStream::byte err = ERR_FAILURE; - try - { - sm.getSystemState(ss); - err = ERR_OK; - reply << err; - reply << static_cast(ss); + try + { + sm.getSystemState(ss); + err = ERR_OK; + reply << err; + reply << static_cast(ss); #ifdef BRM_VERBOSE - cerr << "getSystemState() returning " << static_cast(err) << endl; + cerr << "getSystemState() returning " << static_cast(err) << endl; #endif - } - catch (exception&) - { - reply.reset(); - err = ERR_FAILURE; - reply << err; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } + } + catch (exception&) + { + reply.reset(); + err = ERR_FAILURE; + reply << err; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) { } + catch (...) + { + } + + return; + } + + try + { + p->sock->write(reply); + } + catch (exception&) + { + } } void MasterDBRMNode::doSetSystemState(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - ByteStream::byte cmd; - ByteStream::byte err = ERR_FAILURE; - uint32_t ss; + ByteStream reply; + ByteStream::byte cmd; + ByteStream::byte err = ERR_FAILURE; + uint32_t ss; - try - { - msg >> cmd; - msg >> ss; + try + { + msg >> cmd; + msg >> ss; - sm.setSystemState(ss); + sm.setSystemState(ss); #ifdef BRM_VERBOSE - cerr << "doSetSystemState setting " << hex << ss << dec << endl; + cerr << "doSetSystemState setting " << hex << ss << dec << endl; #endif - } - catch (exception&) - { - err = ERR_FAILURE; - reply << err; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } - - err = ERR_OK; + } + catch (exception&) + { + err = ERR_FAILURE; reply << err; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (...) { } + catch (...) + { + } + + return; + } + + err = ERR_OK; + reply << err; + + try + { + p->sock->write(reply); + } + catch (...) + { + } } void MasterDBRMNode::doClearSystemState(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - ByteStream::byte cmd; - ByteStream::byte err = ERR_FAILURE; - uint32_t ss; + ByteStream reply; + ByteStream::byte cmd; + ByteStream::byte err = ERR_FAILURE; + uint32_t ss; - try - { - msg >> cmd; - msg >> ss; + try + { + msg >> cmd; + msg >> ss; - sm.clearSystemState(ss); - } - catch (exception&) - { - err = ERR_FAILURE; - reply << err; - - try - { - p->sock->write(reply); - } - catch (...) { } - - return; - } - - err = ERR_OK; + sm.clearSystemState(ss); + } + catch (exception&) + { + err = ERR_FAILURE; reply << err; try { - p->sock->write(reply); + p->sock->write(reply); } - catch (...) { } + catch (...) + { + } + + return; + } + + err = ERR_OK; + reply << err; + + try + { + p->sock->write(reply); + } + catch (...) + { + } } void MasterDBRMNode::doSessionManagerReset(ByteStream& msg, ThreadParams* p) { - ByteStream reply; + ByteStream reply; - try - { - sm.reset(); - reply << (uint8_t) ERR_OK; - } - catch (...) - { - reply << (uint8_t) ERR_FAILURE; - } + try + { + sm.reset(); + reply << (uint8_t)ERR_OK; + } + catch (...) + { + reply << (uint8_t)ERR_FAILURE; + } - try - { - p->sock->write(reply); - } - catch (...) { } + try + { + p->sock->write(reply); + } + catch (...) + { + } } void MasterDBRMNode::doAllocOIDs(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - int ret; - uint32_t tmp32; - int num; - uint8_t cmd; + ByteStream reply; + int ret; + uint32_t tmp32; + int num; + uint8_t cmd; + + try + { + boost::mutex::scoped_lock lk(oidsMutex); + + msg >> cmd; + msg >> tmp32; + num = (int)tmp32; + ret = oids.allocOIDs(num); + reply << (uint8_t)ERR_OK; + reply << (uint32_t)ret; + p->sock->write(reply); + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: OID allocation failure. " << ex.what(); + log(os.str()); + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - boost::mutex::scoped_lock lk(oidsMutex); - - msg >> cmd; - msg >> tmp32; - num = (int) tmp32; - ret = oids.allocOIDs(num); - reply << (uint8_t) ERR_OK; - reply << (uint32_t) ret; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception& ex) + catch (...) { - ostringstream os; - os << "DBRM Controller: OID allocation failure. " << ex.what(); - log(os.str()); - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doReturnOIDs(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint8_t cmd; - uint32_t tmp32; - int start, end; + ByteStream reply; + uint8_t cmd; + uint32_t tmp32; + int start, end; + + try + { + boost::mutex::scoped_lock lk(oidsMutex); + + msg >> cmd; + msg >> tmp32; + start = (int)tmp32; + msg >> tmp32; + end = (int)tmp32; + oids.returnOIDs(start, end); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: Return OIDs failure. " << ex.what(); + log(os.str()); + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - boost::mutex::scoped_lock lk(oidsMutex); - - msg >> cmd; - msg >> tmp32; - start = (int) tmp32; - msg >> tmp32; - end = (int) tmp32; - oids.returnOIDs(start, end); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception& ex) + catch (...) { - ostringstream os; - os << "DBRM Controller: Return OIDs failure. " << ex.what(); - log(os.str()); - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doOidmSize(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - int ret; + ByteStream reply; + int ret; + + try + { + boost::mutex::scoped_lock lk(oidsMutex); + + ret = oids.size(); + reply << (uint8_t)ERR_OK; + reply << (uint32_t)ret; + p->sock->write(reply); + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: Failure to get OID count. " << ex.what(); + log(os.str()); + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - boost::mutex::scoped_lock lk(oidsMutex); - - ret = oids.size(); - reply << (uint8_t) ERR_OK; - reply << (uint32_t) ret; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception& ex) + catch (...) { - ostringstream os; - os << "DBRM Controller: Failure to get OID count. " << ex.what(); - log(os.str()); - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doAllocVBOID(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint32_t dbroot; - uint32_t ret; - uint8_t cmd; + ByteStream reply; + uint32_t dbroot; + uint32_t ret; + uint8_t cmd; - msg >> cmd; + msg >> cmd; + + try + { + boost::mutex::scoped_lock lk(oidsMutex); + + msg >> dbroot; + ret = oids.allocVBOID(dbroot); + reply << (uint8_t)ERR_OK; + reply << ret; + p->sock->write(reply); + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: VB OID allocation failure. " << ex.what(); + log(os.str()); + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - boost::mutex::scoped_lock lk(oidsMutex); - - msg >> dbroot; - ret = oids.allocVBOID(dbroot); - reply << (uint8_t) ERR_OK; - reply << ret; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception& ex) + catch (...) { - ostringstream os; - os << "DBRM Controller: VB OID allocation failure. " << ex.what(); - log(os.str()); - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doGetDBRootOfVBOID(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint32_t vbOID; - uint32_t ret; - uint8_t cmd; + ByteStream reply; + uint32_t vbOID; + uint32_t ret; + uint8_t cmd; - msg >> cmd; + msg >> cmd; + + try + { + boost::mutex::scoped_lock lk(oidsMutex); + + msg >> vbOID; + ret = oids.getDBRootOfVBOID(vbOID); + reply << (uint8_t)ERR_OK; + reply << ret; + p->sock->write(reply); + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: Get DBRoot of VB OID failure. " << ex.what(); + log(os.str()); + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - boost::mutex::scoped_lock lk(oidsMutex); - - msg >> vbOID; - ret = oids.getDBRootOfVBOID(vbOID); - reply << (uint8_t) ERR_OK; - reply << ret; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception& ex) + catch (...) { - ostringstream os; - os << "DBRM Controller: Get DBRoot of VB OID failure. " << ex.what(); - log(os.str()); - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doGetVBOIDToDBRootMap(ByteStream& msg, ThreadParams* p) { - ByteStream reply; - uint8_t cmd; + ByteStream reply; + uint8_t cmd; - msg >> cmd; + msg >> cmd; + + try + { + boost::mutex::scoped_lock lk(oidsMutex); + + const vector& ret = oids.getVBOIDToDBRootMap(); + reply << (uint8_t)ERR_OK; + serializeInlineVector(reply, ret); + p->sock->write(reply); + } + catch (exception& ex) + { + ostringstream os; + os << "DBRM Controller: Get VB OID DBRoot map failure. " << ex.what(); + log(os.str()); + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - boost::mutex::scoped_lock lk(oidsMutex); - - const vector& ret = oids.getVBOIDToDBRootMap(); - reply << (uint8_t) ERR_OK; - serializeInlineVector(reply, ret); - p->sock->write(reply); + p->sock->write(reply); } - catch (exception& ex) + catch (...) { - ostringstream os; - os << "DBRM Controller: Get VB OID DBRoot map failure. " << ex.what(); - log(os.str()); - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } - void MasterDBRMNode::doGetTableLock(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - TableLockInfo tli; - uint64_t id; - ByteStream reply; + uint8_t cmd; + TableLockInfo tli; + uint64_t id; + ByteStream reply; - msg >> cmd; + msg >> cmd; + + try + { + msg >> tli; + idbassert(msg.length() == 0); + id = tableLockServer->lock(&tli); + reply << (uint8_t)ERR_OK; + reply << id; + + if (id == 0) + { + reply << tli.ownerPID; + reply << tli.ownerName; + reply << (uint32_t)tli.ownerSessionID; + reply << (uint32_t)tli.ownerTxnID; + } + + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> tli; - idbassert(msg.length() == 0); - id = tableLockServer->lock(&tli); - reply << (uint8_t) ERR_OK; - reply << id; - - if (id == 0) - { - reply << tli.ownerPID; - reply << tli.ownerName; - reply << (uint32_t) tli.ownerSessionID; - reply << (uint32_t) tli.ownerTxnID; - } - - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doReleaseTableLock(ByteStream& msg, ThreadParams* p) { - uint64_t id; - uint8_t cmd; - bool ret; - ByteStream reply; + uint64_t id; + uint8_t cmd; + bool ret; + ByteStream reply; - msg >> cmd; + msg >> cmd; + + try + { + msg >> id; + idbassert(msg.length() == 0); + ret = tableLockServer->unlock(id); + reply << (uint8_t)ERR_OK; + reply << (uint8_t)ret; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> id; - idbassert(msg.length() == 0); - ret = tableLockServer->unlock(id); - reply << (uint8_t) ERR_OK; - reply << (uint8_t) ret; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doChangeTableLockState(ByteStream& msg, ThreadParams* p) { - uint64_t id; - uint32_t tmp32; - uint8_t cmd; - LockState state; - bool ret; - ByteStream reply; + uint64_t id; + uint32_t tmp32; + uint8_t cmd; + LockState state; + bool ret; + ByteStream reply; - msg >> cmd; + msg >> cmd; + + try + { + msg >> id; + msg >> tmp32; + idbassert(msg.length() == 0); + state = (LockState)tmp32; + ret = tableLockServer->changeState(id, state); + reply << (uint8_t)ERR_OK; + reply << (uint8_t)ret; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> id; - msg >> tmp32; - idbassert(msg.length() == 0); - state = (LockState) tmp32; - ret = tableLockServer->changeState(id, state); - reply << (uint8_t) ERR_OK; - reply << (uint8_t) ret; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doChangeTableLockOwner(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - uint64_t id; - string name; - uint32_t pid; - ByteStream reply; - ByteStream workerNodeCmd; - uint32_t tmp32; - int32_t sessionID; - int32_t txnID; - bool ret; - vector responses; - int err; + uint8_t cmd; + uint64_t id; + string name; + uint32_t pid; + ByteStream reply; + ByteStream workerNodeCmd; + uint32_t tmp32; + int32_t sessionID; + int32_t txnID; + bool ret; + vector responses; + int err; - // current owner vars - TableLockInfo tli; - string processName; - string::size_type namelen; - bool exists; + // current owner vars + TableLockInfo tli; + string processName; + string::size_type namelen; + bool exists; + + try + { + msg >> cmd >> id >> name >> pid >> tmp32; + sessionID = tmp32; + msg >> tmp32; + txnID = tmp32; + idbassert(msg.length() == 0); + + /* get the current owner info + * send cmd to look for the owner + * if there's an existing owner, reject the request + */ + + ret = tableLockServer->getLockInfo(id, &tli); + + if (!ret) + { + reply << (uint8_t)ERR_OK << (uint8_t)ret; + goto write; + } + + namelen = tli.ownerName.find_first_of(" "); + + if (namelen == string::npos) + processName = tli.ownerName; + else + processName = tli.ownerName.substr(0, namelen); + + workerNodeCmd << (uint8_t)OWNER_CHECK << processName << tli.ownerPID; + bool readErrFlag; + { + boost::mutex::scoped_lock lk(slaveLock); + distribute(&workerNodeCmd); + err = gatherResponses(OWNER_CHECK, workerNodeCmd.length(), &responses, readErrFlag); + } + + if ((err != ERR_OK) || (readErrFlag)) + { + reply << (uint8_t)ERR_FAILURE; + goto write; + } + + exists = false; + + for (uint32_t i = 0; i < responses.size(); i++) + { + /* Parse msg from worker node */ + uint8_t ret; + idbassert(responses[i]->length() == 1); + *(responses[i]) >> ret; + + if (ret == 1) + exists = true; + + delete responses[i]; + } + + if (exists) + { + reply << (uint8_t)ERR_OK << (uint8_t) false; + goto write; + } + + ret = tableLockServer->changeOwner(id, name, pid, sessionID, txnID); + reply << (uint8_t)ERR_OK << (uint8_t)ret; + + write: + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> id >> name >> pid >> tmp32; - sessionID = tmp32; - msg >> tmp32; - txnID = tmp32; - idbassert(msg.length() == 0); - - /* get the current owner info - * send cmd to look for the owner - * if there's an existing owner, reject the request - */ - - ret = tableLockServer->getLockInfo(id, &tli); - - if (!ret) - { - reply << (uint8_t) ERR_OK << (uint8_t) ret; - goto write; - } - - namelen = tli.ownerName.find_first_of(" "); - - if (namelen == string::npos) - processName = tli.ownerName; - else - processName = tli.ownerName.substr(0, namelen); - - workerNodeCmd << (uint8_t) OWNER_CHECK << processName << tli.ownerPID; - bool readErrFlag; - { - boost::mutex::scoped_lock lk(slaveLock); - distribute(&workerNodeCmd); - err = gatherResponses(OWNER_CHECK, workerNodeCmd.length(), &responses, - readErrFlag); - } - - if ((err != ERR_OK) || (readErrFlag)) - { - reply << (uint8_t) ERR_FAILURE; - goto write; - } - - exists = false; - - for (uint32_t i = 0; i < responses.size(); i++) - { - /* Parse msg from worker node */ - uint8_t ret; - idbassert(responses[i]->length() == 1); - *(responses[i]) >> ret; - - if (ret == 1) - exists = true; - - delete responses[i]; - } - - if (exists) - { - reply << (uint8_t) ERR_OK << (uint8_t) false; - goto write; - } - - ret = tableLockServer->changeOwner(id, name, pid, sessionID, txnID); - reply << (uint8_t) ERR_OK << (uint8_t) ret; - -write: - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doGetAllTableLocks(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - vector ret; - ByteStream reply; + uint8_t cmd; + vector ret; + ByteStream reply; + + try + { + msg >> cmd; + idbassert(msg.length() == 0); + ret = tableLockServer->getAllLocks(); + reply << (uint8_t)ERR_OK; + serializeVector(reply, ret); + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd; - idbassert(msg.length() == 0); - ret = tableLockServer->getAllLocks(); - reply << (uint8_t) ERR_OK; - serializeVector(reply, ret); - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doReleaseAllTableLocks(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; + uint8_t cmd; + ByteStream reply; + + try + { + msg >> cmd; + idbassert(msg.length() == 0); + tableLockServer->releaseAllLocks(); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd; - idbassert(msg.length() == 0); - tableLockServer->releaseAllLocks(); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doGetTableLockInfo(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint64_t id; - TableLockInfo tli; - bool ret; + uint8_t cmd; + ByteStream reply; + uint64_t id; + TableLockInfo tli; + bool ret; + + try + { + msg >> cmd >> id; + idbassert(msg.length() == 0); + ret = tableLockServer->getLockInfo(id, &tli); + reply << (uint8_t)ERR_OK << (uint8_t)ret; + + if (ret) + reply << tli; + + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> id; - idbassert(msg.length() == 0); - ret = tableLockServer->getLockInfo(id, &tli); - reply << (uint8_t) ERR_OK << (uint8_t) ret; - - if (ret) - reply << tli; - - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doOwnerCheck(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - uint64_t id; - ByteStream reply; - ByteStream workerNodeCmd; - bool ret; - vector responses; - int err; + uint8_t cmd; + uint64_t id; + ByteStream reply; + ByteStream workerNodeCmd; + bool ret; + vector responses; + int err; - // current owner vars - TableLockInfo tli; - string processName; - string::size_type namelen; - bool exists; + // current owner vars + TableLockInfo tli; + string processName; + string::size_type namelen; + bool exists; + + try + { + msg >> cmd >> id; + idbassert(msg.length() == 0); + + /* get the current owner info + * send cmd to look for the owner + * if there's an existing owner, reject the request + */ + + ret = tableLockServer->getLockInfo(id, &tli); + + if (!ret) + { + reply << (uint8_t)ERR_OK << (uint8_t)ret; + goto write; + } + + namelen = tli.ownerName.find_first_of(" "); + + if (namelen == string::npos) + processName = tli.ownerName; + else + processName = tli.ownerName.substr(0, namelen); + + workerNodeCmd << (uint8_t)OWNER_CHECK << processName << tli.ownerPID; + bool readErrFlag; + { + boost::mutex::scoped_lock lk(slaveLock); + distribute(&workerNodeCmd); + err = gatherResponses(OWNER_CHECK, workerNodeCmd.length(), &responses, readErrFlag); + } + + if ((err != ERR_OK) || (readErrFlag)) + { + reply << (uint8_t)ERR_FAILURE; + goto write; + } + + exists = false; + + for (uint32_t i = 0; i < responses.size(); i++) + { + /* Parse msg from worker node */ + uint8_t ret; + idbassert(responses[i]->length() == 1); + *(responses[i]) >> ret; + + if (ret == 1) + exists = true; + + delete responses[i]; + } + + reply << (uint8_t)ERR_OK << (uint8_t)exists; + + write: + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> id; - idbassert(msg.length() == 0); - - /* get the current owner info - * send cmd to look for the owner - * if there's an existing owner, reject the request - */ - - ret = tableLockServer->getLockInfo(id, &tli); - - if (!ret) - { - reply << (uint8_t) ERR_OK << (uint8_t) ret; - goto write; - } - - namelen = tli.ownerName.find_first_of(" "); - - if (namelen == string::npos) - processName = tli.ownerName; - else - processName = tli.ownerName.substr(0, namelen); - - workerNodeCmd << (uint8_t) OWNER_CHECK << processName << tli.ownerPID; - bool readErrFlag; - { - boost::mutex::scoped_lock lk(slaveLock); - distribute(&workerNodeCmd); - err = gatherResponses(OWNER_CHECK, workerNodeCmd.length(), &responses, - readErrFlag); - } - - if ((err != ERR_OK) || (readErrFlag)) - { - reply << (uint8_t) ERR_FAILURE; - goto write; - } - - exists = false; - - for (uint32_t i = 0; i < responses.size(); i++) - { - /* Parse msg from worker node */ - uint8_t ret; - idbassert(responses[i]->length() == 1); - *(responses[i]) >> ret; - - if (ret == 1) - exists = true; - - delete responses[i]; - } - - reply << (uint8_t) ERR_OK << (uint8_t) exists; - -write: - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doStartAISequence(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint8_t tmp8; - uint32_t oid, colWidth; - uint64_t firstNum; - execplan::CalpontSystemCatalog::ColDataType colDataType; + uint8_t cmd; + ByteStream reply; + uint8_t tmp8; + uint32_t oid, colWidth; + uint64_t firstNum; + execplan::CalpontSystemCatalog::ColDataType colDataType; + + try + { + msg >> cmd >> oid >> firstNum >> colWidth >> tmp8; + colDataType = (execplan::CalpontSystemCatalog::ColDataType)tmp8; + idbassert(msg.length() == 0); + aiManager.startSequence(oid, firstNum, colWidth, colDataType); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> oid >> firstNum >> colWidth >> tmp8; - colDataType = (execplan::CalpontSystemCatalog::ColDataType)tmp8; - idbassert(msg.length() == 0); - aiManager.startSequence(oid, firstNum, colWidth, colDataType); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doGetAIRange(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint32_t oid, count; - uint64_t nextVal; - bool ret; + uint8_t cmd; + ByteStream reply; + uint32_t oid, count; + uint64_t nextVal; + bool ret; + + try + { + msg >> cmd >> oid >> count; + idbassert(msg.length() == 0); + ret = aiManager.getAIRange(oid, count, &nextVal); + reply << (uint8_t)ERR_OK << (uint8_t)ret << nextVal; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> oid >> count; - idbassert(msg.length() == 0); - ret = aiManager.getAIRange(oid, count, &nextVal); - reply << (uint8_t) ERR_OK << (uint8_t) ret << nextVal; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doResetAISequence(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint32_t oid; - uint64_t val; + uint8_t cmd; + ByteStream reply; + uint32_t oid; + uint64_t val; + + try + { + msg >> cmd >> oid >> val; + idbassert(msg.length() == 0); + aiManager.resetSequence(oid, val); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> oid >> val; - idbassert(msg.length() == 0); - aiManager.resetSequence(oid, val); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doGetAILock(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint32_t oid; + uint8_t cmd; + ByteStream reply; + uint32_t oid; + + try + { + msg >> cmd >> oid; + idbassert(msg.length() == 0); + aiManager.getLock(oid); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> oid; - idbassert(msg.length() == 0); - aiManager.getLock(oid); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doReleaseAILock(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint32_t oid; + uint8_t cmd; + ByteStream reply; + uint32_t oid; + + try + { + msg >> cmd >> oid; + idbassert(msg.length() == 0); + aiManager.releaseLock(oid); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> oid; - idbassert(msg.length() == 0); - aiManager.releaseLock(oid); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } void MasterDBRMNode::doDeleteAISequence(ByteStream& msg, ThreadParams* p) { - uint8_t cmd; - ByteStream reply; - uint32_t oid; + uint8_t cmd; + ByteStream reply; + uint32_t oid; + + try + { + msg >> cmd >> oid; + idbassert(msg.length() == 0); + aiManager.deleteSequence(oid); + reply << (uint8_t)ERR_OK; + p->sock->write(reply); + } + catch (exception&) + { + reply.restart(); + reply << (uint8_t)ERR_FAILURE; try { - msg >> cmd >> oid; - idbassert(msg.length() == 0); - aiManager.deleteSequence(oid); - reply << (uint8_t) ERR_OK; - p->sock->write(reply); + p->sock->write(reply); } - catch (exception&) + catch (...) { - reply.restart(); - reply << (uint8_t) ERR_FAILURE; - - try - { - p->sock->write(reply); - } - catch (...) { } } + } } MasterDBRMNode::MsgProcessor::MsgProcessor(MasterDBRMNode* master) : m(master) @@ -2730,40 +2750,40 @@ MasterDBRMNode::MsgProcessor::MsgProcessor(MasterDBRMNode* master) : m(master) void MasterDBRMNode::MsgProcessor::operator()() { - m->runners++; + m->runners++; - try - { - m->msgProcessor(); - } + try + { + m->msgProcessor(); + } #ifdef BRM_VERBOSE - catch (SocketClosed& e) - { - cerr << e.what() << endl; + catch (SocketClosed& e) + { + cerr << e.what() << endl; #else - catch (SocketClosed& ) - { + catch (SocketClosed&) + { #endif - } - catch (exception& e) - { - log(e.what()); + } + catch (exception& e) + { + log(e.what()); #ifdef BRM_VERBOSE - cerr << e.what() << endl; + cerr << e.what() << endl; #endif - } - catch (...) - { - log("caught something that's not an exception", logging::LOG_TYPE_WARNING); - cerr << "DBRM Controller: caught something that's not an exception" << endl; - } + } + catch (...) + { + log("caught something that's not an exception", logging::LOG_TYPE_WARNING); + cerr << "DBRM Controller: caught something that's not an exception" << endl; + } - m->runners--; + m->runners--; } MasterDBRMNode::MsgProcessor::~MsgProcessor() { } -} //namespace +} // namespace BRM diff --git a/versioning/BRM/masterdbrmnode.h b/versioning/BRM/masterdbrmnode.h index f5fbf2200..37a98f2b3 100644 --- a/versioning/BRM/masterdbrmnode.h +++ b/versioning/BRM/masterdbrmnode.h @@ -43,7 +43,6 @@ namespace BRM { - /** @brief The Master node of the DBRM system. * * There are 3 components of the Distributed BRM (DBRM). @@ -85,182 +84,178 @@ constexpr size_t connectTimeoutStep = 50000; class MasterDBRMNode { -public: - MasterDBRMNode(); - ~MasterDBRMNode(); + public: + MasterDBRMNode(); + ~MasterDBRMNode(); - /** @brief The primary function of the class. - * - * The main loop of the master node. It accepts connections from the DBRM - * class, receives commands, and distributes them to each slave. It returns - * only after stop() or the destructor is called by another thread. - */ - void run(); + /** @brief The primary function of the class. + * + * The main loop of the master node. It accepts connections from the DBRM + * class, receives commands, and distributes them to each slave. It returns + * only after stop() or the destructor is called by another thread. + */ + void run(); - /** @brief Tells the Master to shut down cleanly. - * - * Tells the Master to shut down cleanly. - */ - void stop(); + /** @brief Tells the Master to shut down cleanly. + * + * Tells the Master to shut down cleanly. + */ + void stop(); - /** @brief Effectively makes the whole DBRM system stop. - * - * Grabs a lock that effectively halts all further BRM data changes. - * @warning Use with care. It's basically an accessor to a raw pthread_mutex. - */ - void lock(); + /** @brief Effectively makes the whole DBRM system stop. + * + * Grabs a lock that effectively halts all further BRM data changes. + * @warning Use with care. It's basically an accessor to a raw pthread_mutex. + */ + void lock(); - /** @brief Resumes DBRM functionality. - * - * Releases a lock that allows the DBRM to continue processing changes. - * @warning Use with care. It's basically an accessor to a raw pthread_mutex. - */ - void unlock(); + /** @brief Resumes DBRM functionality. + * + * Releases a lock that allows the DBRM to continue processing changes. + * @warning Use with care. It's basically an accessor to a raw pthread_mutex. + */ + void unlock(); - /** @brief Reload the config file and reconnect to all slaves. - * - * Drops all existing connections, reloads the config file and - * reconnects with all slaves. - * @note Doesn't work yet. Redundant anyway. - */ - void reload(); + /** @brief Reload the config file and reconnect to all slaves. + * + * Drops all existing connections, reloads the config file and + * reconnects with all slaves. + * @note Doesn't work yet. Redundant anyway. + */ + void reload(); - /** @brief Sets either read/write or read-only mode - * - * Sets either read/write or read-only mode. When in read-only mode - * all BRM change requests will return ERR_READONLY immediately. - * @param ro true specifies read-only, false specifies read/write - */ - void setReadOnly(bool ro); + /** @brief Sets either read/write or read-only mode + * + * Sets either read/write or read-only mode. When in read-only mode + * all BRM change requests will return ERR_READONLY immediately. + * @param ro true specifies read-only, false specifies read/write + */ + void setReadOnly(bool ro); - /** @brief Returns true if the Master is in read-only mode, false if in RW mode. - * - * @returns true if the Master is in read-only mode, false if in read-write - * mode - */ - bool isReadOnly() const - { - return readOnly; - } - /** @brief Connects to the all workers */ - void connectToWorkers(const size_t connectTimeoutSecs); + /** @brief Returns true if the Master is in read-only mode, false if in RW mode. + * + * @returns true if the Master is in read-only mode, false if in read-write + * mode + */ + bool isReadOnly() const + { + return readOnly; + } + /** @brief Connects to the all workers */ + void connectToWorkers(const size_t connectTimeoutSecs); - /** @brief Extracts number of workers and connection timeout from the config */ - void getNumWorkersAndTimeout(size_t& connectTimeoutSecs, - const std::string& methodName, - config::Config* config); + /** @brief Extracts number of workers and connection timeout from the config */ + void getNumWorkersAndTimeout(size_t& connectTimeoutSecs, const std::string& methodName, + config::Config* config); -private: + private: + class MsgProcessor + { + public: + MsgProcessor(MasterDBRMNode* master); + ~MsgProcessor(); + void operator()(); - class MsgProcessor - { - public: - MsgProcessor(MasterDBRMNode* master); - ~MsgProcessor(); - void operator()(); - private: - MasterDBRMNode* m; - }; + private: + MasterDBRMNode* m; + }; - struct ThreadParams - { - messageqcpp::IOSocket* sock; - boost::thread* t; - }; + struct ThreadParams + { + messageqcpp::IOSocket* sock; + boost::thread* t; + }; + MasterDBRMNode(const MasterDBRMNode& m); + MasterDBRMNode& operator=(const MasterDBRMNode& m); - MasterDBRMNode(const MasterDBRMNode& m); - MasterDBRMNode& operator=(const MasterDBRMNode& m); + void initMsgQueues(config::Config* config); + void msgProcessor(); + void distribute(messageqcpp::ByteStream* msg); + void undo() throw(); + void confirm(); + void sendError(messageqcpp::IOSocket* dest, uint8_t err) const throw(); + int gatherResponses(uint8_t cmd, uint32_t msgCmdLength, std::vector* responses, + bool& readErrFlag) throw(); + int compareResponses(uint8_t cmd, uint32_t msgCmdLength, + const std::vector& responses) const; + void finalCleanup(); - void initMsgQueues(config::Config* config); - void msgProcessor(); - void distribute(messageqcpp::ByteStream* msg); - void undo() throw(); - void confirm(); - void sendError(messageqcpp::IOSocket* dest, uint8_t err) const throw(); - int gatherResponses(uint8_t cmd, uint32_t msgCmdLength, - std::vector* responses, - bool& readErrFlag) throw(); - int compareResponses(uint8_t cmd, uint32_t msgCmdLength, - const std::vector & responses) const; - void finalCleanup(); + /* Commands the master executes */ + void doHalt(messageqcpp::IOSocket* sock); + void doResume(messageqcpp::IOSocket* sock); + void doReload(messageqcpp::IOSocket* sock); + void doSetReadOnly(messageqcpp::IOSocket* sock, bool b); + void doGetReadOnly(messageqcpp::IOSocket* sock); - /* Commands the master executes */ - void doHalt(messageqcpp::IOSocket* sock); - void doResume(messageqcpp::IOSocket* sock); - void doReload(messageqcpp::IOSocket* sock); - void doSetReadOnly(messageqcpp::IOSocket* sock, bool b); - void doGetReadOnly(messageqcpp::IOSocket* sock); + /* SessionManager interface */ + SessionManagerServer sm; + void doVerID(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetSystemCatalog(messageqcpp::ByteStream& msg, ThreadParams* p); + void doSysCatVerID(messageqcpp::ByteStream& msg, ThreadParams* p); + void doNewTxnID(messageqcpp::ByteStream& msg, ThreadParams* p); + void doCommitted(messageqcpp::ByteStream& msg, ThreadParams* p); + void doRolledBack(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetTxnID(messageqcpp::ByteStream& msg, ThreadParams* p); + void doSIDTIDMap(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetShmContents(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetUniqueUint32(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetUniqueUint64(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetSystemState(messageqcpp::ByteStream& msg, ThreadParams* p); + void doSetSystemState(messageqcpp::ByteStream& msg, ThreadParams* p); + void doClearSystemState(messageqcpp::ByteStream& msg, ThreadParams* p); + void doSessionManagerReset(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetUncommittedLbids(messageqcpp::ByteStream& msg, ThreadParams* p); - /* SessionManager interface */ - SessionManagerServer sm; - void doVerID(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetSystemCatalog(messageqcpp::ByteStream& msg, ThreadParams* p); - void doSysCatVerID(messageqcpp::ByteStream& msg, ThreadParams* p); - void doNewTxnID(messageqcpp::ByteStream& msg, ThreadParams* p); - void doCommitted(messageqcpp::ByteStream& msg, ThreadParams* p); - void doRolledBack(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetTxnID(messageqcpp::ByteStream& msg, ThreadParams* p); - void doSIDTIDMap(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetShmContents(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetUniqueUint32(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetUniqueUint64(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetSystemState(messageqcpp::ByteStream& msg, ThreadParams* p); - void doSetSystemState(messageqcpp::ByteStream& msg, ThreadParams* p); - void doClearSystemState(messageqcpp::ByteStream& msg, ThreadParams* p); - void doSessionManagerReset(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetUncommittedLbids(messageqcpp::ByteStream& msg, ThreadParams* p); + /* OID Manager interface */ + OIDServer oids; + boost::mutex oidsMutex; + void doAllocOIDs(messageqcpp::ByteStream& msg, ThreadParams* p); + void doReturnOIDs(messageqcpp::ByteStream& msg, ThreadParams* p); + void doOidmSize(messageqcpp::ByteStream& msg, ThreadParams* p); + void doAllocVBOID(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetDBRootOfVBOID(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetVBOIDToDBRootMap(messageqcpp::ByteStream& msg, ThreadParams* p); + /* Table lock interface */ + boost::scoped_ptr tableLockServer; + void doGetTableLock(messageqcpp::ByteStream& msg, ThreadParams* p); + void doReleaseTableLock(messageqcpp::ByteStream& msg, ThreadParams* p); + void doChangeTableLockState(messageqcpp::ByteStream& msg, ThreadParams* p); + void doChangeTableLockOwner(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetAllTableLocks(messageqcpp::ByteStream& msg, ThreadParams* p); + void doReleaseAllTableLocks(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetTableLockInfo(messageqcpp::ByteStream& msg, ThreadParams* p); + void doOwnerCheck(messageqcpp::ByteStream& msg, ThreadParams* p); - /* OID Manager interface */ - OIDServer oids; - boost::mutex oidsMutex; - void doAllocOIDs(messageqcpp::ByteStream& msg, ThreadParams* p); - void doReturnOIDs(messageqcpp::ByteStream& msg, ThreadParams* p); - void doOidmSize(messageqcpp::ByteStream& msg, ThreadParams* p); - void doAllocVBOID(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetDBRootOfVBOID(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetVBOIDToDBRootMap(messageqcpp::ByteStream& msg, ThreadParams* p); + /* Autoincrement interface */ + AutoincrementManager aiManager; + void doStartAISequence(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetAIRange(messageqcpp::ByteStream& msg, ThreadParams* p); + void doResetAISequence(messageqcpp::ByteStream& msg, ThreadParams* p); + void doGetAILock(messageqcpp::ByteStream& msg, ThreadParams* p); + void doReleaseAILock(messageqcpp::ByteStream& msg, ThreadParams* p); + void doDeleteAISequence(messageqcpp::ByteStream& msg, ThreadParams* p); - /* Table lock interface */ - boost::scoped_ptr tableLockServer; - void doGetTableLock(messageqcpp::ByteStream& msg, ThreadParams* p); - void doReleaseTableLock(messageqcpp::ByteStream& msg, ThreadParams* p); - void doChangeTableLockState(messageqcpp::ByteStream& msg, ThreadParams* p); - void doChangeTableLockOwner(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetAllTableLocks(messageqcpp::ByteStream& msg, ThreadParams* p); - void doReleaseAllTableLocks(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetTableLockInfo(messageqcpp::ByteStream& msg, ThreadParams* p); - void doOwnerCheck(messageqcpp::ByteStream& msg, ThreadParams* p); + messageqcpp::MessageQueueServer* dbrmServer; + std::vector slaves; + std::vector::iterator iSlave; + std::vector activeSessions; - /* Autoincrement interface */ - AutoincrementManager aiManager; - void doStartAISequence(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetAIRange(messageqcpp::ByteStream& msg, ThreadParams* p); - void doResetAISequence(messageqcpp::ByteStream& msg, ThreadParams* p); - void doGetAILock(messageqcpp::ByteStream& msg, ThreadParams* p); - void doReleaseAILock(messageqcpp::ByteStream& msg, ThreadParams* p); - void doDeleteAISequence(messageqcpp::ByteStream& msg, ThreadParams* p); + LBIDResourceGraph* rg; - messageqcpp::MessageQueueServer* dbrmServer; - std::vector slaves; - std::vector::iterator iSlave; - std::vector activeSessions; - - LBIDResourceGraph* rg; - - boost::mutex mutex; - boost::mutex mutex2; // protects params and the hand-off TODO: simplify - boost::mutex slaveLock; // syncs communication with the slaves - boost::mutex serverLock; // kludge to synchronize reloading - int runners, NumWorkers; - ThreadParams* params; - volatile bool die, halting; - bool reloadCmd; - mutable bool readOnly; - struct timespec MSG_TIMEOUT; + boost::mutex mutex; + boost::mutex mutex2; // protects params and the hand-off TODO: simplify + boost::mutex slaveLock; // syncs communication with the slaves + boost::mutex serverLock; // kludge to synchronize reloading + int runners, NumWorkers; + ThreadParams* params; + volatile bool die, halting; + bool reloadCmd; + mutable bool readOnly; + struct timespec MSG_TIMEOUT; }; -} +} // namespace BRM #endif diff --git a/versioning/BRM/masternode.cpp b/versioning/BRM/masternode.cpp index e8c92267a..9a00f1522 100644 --- a/versioning/BRM/masternode.cpp +++ b/versioning/BRM/masternode.cpp @@ -24,7 +24,6 @@ * The executable source that runs a Master DBRM Node. */ - #include "masterdbrmnode.h" #include "liboamcpp.h" #include @@ -42,180 +41,174 @@ #define MAX_RETRIES 10 BRM::MasterDBRMNode* m; -bool die= false; +bool die = false; using namespace std; using namespace BRM; class Opt { -protected: - const char *m_progname; - bool m_fg; -public: - Opt(int argc, char **argv) - :m_progname(argv[0]), - m_fg(argc >= 2 && string(argv[1]) == "fg") - { } + protected: + const char* m_progname; + bool m_fg; + + public: + Opt(int argc, char** argv) : m_progname(argv[0]), m_fg(argc >= 2 && string(argv[1]) == "fg") + { + } }; - -class ServiceControllerNode: public Service, public Opt +class ServiceControllerNode : public Service, public Opt { -protected: - void setupChildSignalHandlers(); + protected: + void setupChildSignalHandlers(); -public: - ServiceControllerNode(const Opt &opt) - :Service("ControllerNode"), Opt(opt) - { } - void LogErrno() override - { - perror(m_progname); - log_errno(std::string(m_progname)); - } - void ParentLogChildMessage(const std::string &str) override - { - log(str, logging::LOG_TYPE_INFO); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServiceControllerNode(const Opt& opt) : Service("ControllerNode"), Opt(opt) + { + } + void LogErrno() override + { + perror(m_progname); + log_errno(std::string(m_progname)); + } + void ParentLogChildMessage(const std::string& str) override + { + log(str, logging::LOG_TYPE_INFO); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - - void stop(int num) { #ifdef BRM_VERBOSE - std::cerr << "stopping..." << std::endl; + std::cerr << "stopping..." << std::endl; #endif - die = true; + die = true; - if (m != NULL) - m->stop(); + if (m != NULL) + m->stop(); } void restart(int num) { #ifdef BRM_VERBOSE - std::cerr << "stopping this instance..." << std::endl; + std::cerr << "stopping this instance..." << std::endl; #endif - if (m != NULL) - m->stop(); + if (m != NULL) + m->stop(); } /* doesn't quite work yet... void reload(int num) { #ifdef BRM_VERBOSE - std::cerr << "reloading the config file" << std::endl; + std::cerr << "reloading the config file" << std::endl; #endif - m->lock(); - try { - m->reload(); - } - catch (std::exception &e) { - m->setReadOnly(true); - std::cerr << "Reload failed. Check the config file. Reverting to read-only mode." - << std::endl; - } - m->unlock(); + m->lock(); + try { + m->reload(); + } + catch (std::exception &e) { + m->setReadOnly(true); + std::cerr << "Reload failed. Check the config file. Reverting to read-only mode." + << std::endl; + } + m->unlock(); } */ - void ServiceControllerNode::setupChildSignalHandlers() { - /* XXXPAT: we might want to install signal handlers for every signal */ + /* XXXPAT: we might want to install signal handlers for every signal */ - signal(SIGINT, stop); - signal(SIGTERM, stop); + signal(SIGINT, stop); + signal(SIGTERM, stop); #ifndef _MSC_VER - signal(SIGHUP, SIG_IGN); - signal(SIGUSR1, restart); - signal(SIGPIPE, SIG_IGN); + signal(SIGHUP, SIG_IGN); + signal(SIGUSR1, restart); + signal(SIGPIPE, SIG_IGN); #endif - struct sigaction ign; + struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); } - int ServiceControllerNode::Child() { - int retries = 0; + int retries = 0; - (void)config::Config::makeConfig(); + (void)config::Config::makeConfig(); - setupChildSignalHandlers(); + setupChildSignalHandlers(); - idbdatafile::IDBPolicy::configIDBPolicy(); + idbdatafile::IDBPolicy::configIDBPolicy(); - m = NULL; + m = NULL; - while (retries < MAX_RETRIES && !die) + while (retries < MAX_RETRIES && !die) + { + try { - try - { - if (m != NULL) - delete m; + if (m != NULL) + delete m; - m = new BRM::MasterDBRMNode(); + m = new BRM::MasterDBRMNode(); - NotifyServiceStarted(); + NotifyServiceStarted(); - m->run(); - retries = 0; - delete m; - m = NULL; - } - catch (std::exception& e) - { - ostringstream os; - os << e.what(); - os << "... attempt #" << retries + 1 << "/" << MAX_RETRIES << " to restart the DBRM controller node"; - cerr << os.str() << endl; - log(os.str()); - sleep(5); - } - - retries++; + m->run(); + retries = 0; + delete m; + m = NULL; + } + catch (std::exception& e) + { + ostringstream os; + os << e.what(); + os << "... attempt #" << retries + 1 << "/" << MAX_RETRIES << " to restart the DBRM controller node"; + cerr << os.str() << endl; + log(os.str()); + sleep(5); } - if (retries == MAX_RETRIES) - { - NotifyServiceInitializationFailed(); - log(string("Exiting after too many errors")); - return 1; - } + retries++; + } - std::cerr << "Exiting..." << std::endl; - return 0; + if (retries == MAX_RETRIES) + { + NotifyServiceInitializationFailed(); + log(string("Exiting after too many errors")); + return 1; + } + + std::cerr << "Exiting..." << std::endl; + return 0; } - int main(int argc, char** argv) { - Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); + Opt opt(argc, argv); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); - BRM::logInit ( BRM::SubSystemLogId_controllerNode ); + BRM::logInit(BRM::SubSystemLogId_controllerNode); - /* - Need to shutdown TheadPool before fork(), - otherwise it would get stuck when trying to join fPruneThread. - */ - joblist::JobStep::jobstepThreadPool.stop(); + /* + Need to shutdown TheadPool before fork(), + otherwise it would get stuck when trying to join fPruneThread. + */ + joblist::JobStep::jobstepThreadPool.stop(); - return ServiceControllerNode(opt).Run(); + return ServiceControllerNode(opt).Run(); } diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index 54921b644..c88dc13ba 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -51,7 +51,6 @@ using namespace BRM; namespace BRM { - /*static*/ boost::mutex MasterSegmentTableImpl::fInstanceMutex; @@ -61,208 +60,209 @@ MasterSegmentTableImpl* MasterSegmentTableImpl::fInstance = 0; /*static*/ MasterSegmentTableImpl* MasterSegmentTableImpl::makeMasterSegmentTableImpl(int key, int size) { - boost::mutex::scoped_lock lk(fInstanceMutex); - - if (fInstance) - return fInstance; - - fInstance = new MasterSegmentTableImpl(key, size); + boost::mutex::scoped_lock lk(fInstanceMutex); + if (fInstance) return fInstance; + + fInstance = new MasterSegmentTableImpl(key, size); + + return fInstance; } MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) { - string keyName = ShmKeys::keyToName(key); + string keyName = ShmKeys::keyToName(key); - try + try + { + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); + shm.truncate(size); + fShmobj.swap(shm); + } + catch (bi::interprocess_exception& biex) + { + if (biex.get_error_code() == bi::already_exists_error) { - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); - shm.truncate(size); + try + { + bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); fShmobj.swap(shm); + } + catch (exception& e) + { + ostringstream o; + o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << e.what(); + log(o.str()); + throw; + } } - catch (bi::interprocess_exception& biex) + else { - if (biex.get_error_code() == bi::already_exists_error) { - try { - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fShmobj.swap(shm); - } - catch (exception &e) { - ostringstream o; - o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << e.what(); - log(o.str()); - throw; - } - } - else { - ostringstream o; - o << "BRM caught an exception creating a shared memory segment (" << keyName << "): " << biex.what(); - log(o.str()); - throw; - } + ostringstream o; + o << "BRM caught an exception creating a shared memory segment (" << keyName << "): " << biex.what(); + log(o.str()); + throw; } - bi::mapped_region region(fShmobj, bi::read_write); - fMapreg.swap(region); + } + bi::mapped_region region(fShmobj, bi::read_write); + fMapreg.swap(region); } -MSTEntry::MSTEntry() : - tableShmkey(-1), - allocdSize(0), - currentSize(0) +MSTEntry::MSTEntry() : tableShmkey(-1), allocdSize(0), currentSize(0) { } MasterSegmentTable::MasterSegmentTable() { #ifdef _MSC_VER - const char* envp = getenv("SystemRoot"); - string SystemRoot; + const char* envp = getenv("SystemRoot"); + string SystemRoot; - if (envp && *envp) - SystemRoot = envp; - else - SystemRoot = "C:\\WINDOWS"; + if (envp && *envp) + SystemRoot = envp; + else + SystemRoot = "C:\\WINDOWS"; - string tmpEnv = "TMP=" + SystemRoot + "\\Temp"; - _putenv(tmpEnv.c_str()); + string tmpEnv = "TMP=" + SystemRoot + "\\Temp"; + _putenv(tmpEnv.c_str()); #endif - int i; - bool initializer = false; + int i; + bool initializer = false; - RWLockKeys[0] = fShmKeys.KEYRANGE_EXTENTMAP_BASE; - RWLockKeys[1] = fShmKeys.KEYRANGE_EMFREELIST_BASE; - RWLockKeys[2] = fShmKeys.KEYRANGE_VBBM_BASE; - RWLockKeys[3] = fShmKeys.KEYRANGE_VSS_BASE; - RWLockKeys[4] = fShmKeys.KEYRANGE_CL_BASE; + RWLockKeys[0] = fShmKeys.KEYRANGE_EXTENTMAP_BASE; + RWLockKeys[1] = fShmKeys.KEYRANGE_EMFREELIST_BASE; + RWLockKeys[2] = fShmKeys.KEYRANGE_VBBM_BASE; + RWLockKeys[3] = fShmKeys.KEYRANGE_VSS_BASE; + RWLockKeys[4] = fShmKeys.KEYRANGE_CL_BASE; - try - { - // if initializer is returned false, then this is not the first time for this key. - rwlock[0].reset(new RWLock(RWLockKeys[0], &initializer)); - } - catch (exception& e) - { - cerr << "ControllerSegmentTable: RWLock() threw: " << e.what() << endl; - throw; - } + try + { + // if initializer is returned false, then this is not the first time for this key. + rwlock[0].reset(new RWLock(RWLockKeys[0], &initializer)); + } + catch (exception& e) + { + cerr << "ControllerSegmentTable: RWLock() threw: " << e.what() << endl; + throw; + } - if (rwlock[0] == NULL) - { - cerr << "ControllerSegmentTable(): RWLock() failed..?" << endl; - throw runtime_error("ControllerSegmentTable(): RWLock() failed..?"); - } + if (rwlock[0] == NULL) + { + cerr << "ControllerSegmentTable(): RWLock() failed..?" << endl; + throw runtime_error("ControllerSegmentTable(): RWLock() failed..?"); + } - for (i = 1; i < nTables; i++) - rwlock[i].reset(new RWLock(RWLockKeys[i])); + for (i = 1; i < nTables; i++) + rwlock[i].reset(new RWLock(RWLockKeys[i])); - makeMSTSegment(); + makeMSTSegment(); - if (initializer) - { - initMSTData(); - rwlock[0]->write_unlock(); - } - else - { - rwlock[0]->read_lock_priority(); // this is to synch with the initializer - rwlock[0]->read_unlock(); - } + if (initializer) + { + initMSTData(); + rwlock[0]->write_unlock(); + } + else + { + rwlock[0]->read_lock_priority(); // this is to synch with the initializer + rwlock[0]->read_unlock(); + } } MasterSegmentTable::~MasterSegmentTable() { -// int i; + // int i; -// for (i = 0; i < nTables; i++) -// delete rwlock[i]; + // for (i = 0; i < nTables; i++) + // delete rwlock[i]; } void MasterSegmentTable::makeMSTSegment() { - fPImpl = MasterSegmentTableImpl::makeMasterSegmentTableImpl(fShmKeys.MST_SYSVKEY, MSTshmsize); - fShmDescriptors = static_cast(fPImpl->fMapreg.get_address()); + fPImpl = MasterSegmentTableImpl::makeMasterSegmentTableImpl(fShmKeys.MST_SYSVKEY, MSTshmsize); + fShmDescriptors = static_cast(fPImpl->fMapreg.get_address()); } void MasterSegmentTable::initMSTData() { - void *dp = static_cast(&fShmDescriptors); - memset(dp, 0, MSTshmsize); + void* dp = static_cast(&fShmDescriptors); + memset(dp, 0, MSTshmsize); } MSTEntry* MasterSegmentTable::getTable_read(int num, bool block) const { - if (num < 0 || num > nTables - 1) - throw std::invalid_argument("ControllerSegmentTable::getTable_read()"); + if (num < 0 || num > nTables - 1) + throw std::invalid_argument("ControllerSegmentTable::getTable_read()"); - if (!block) - try - { - rwlock[num]->read_lock(false); - } - catch (rwlock::wouldblock& e) - { - return NULL; - } - else - rwlock[num]->read_lock(); + if (!block) + try + { + rwlock[num]->read_lock(false); + } + catch (rwlock::wouldblock& e) + { + return NULL; + } + else + rwlock[num]->read_lock(); - return &fShmDescriptors[num]; + return &fShmDescriptors[num]; } MSTEntry* MasterSegmentTable::getTable_write(int num, bool block) const { - if (num < 0 || num > nTables - 1) - throw std::invalid_argument("ControllerSegmentTable::getTable_write()"); + if (num < 0 || num > nTables - 1) + throw std::invalid_argument("ControllerSegmentTable::getTable_write()"); - if (!block) - try - { - rwlock[num]->write_lock(false); - } - catch (rwlock::wouldblock& e) - { - return NULL; - } - else - rwlock[num]->write_lock(); + if (!block) + try + { + rwlock[num]->write_lock(false); + } + catch (rwlock::wouldblock& e) + { + return NULL; + } + else + rwlock[num]->write_lock(); - return &fShmDescriptors[num]; + return &fShmDescriptors[num]; } void MasterSegmentTable::getTable_upgrade(int num) const { - if (num < 0 || num > nTables - 1) - throw std::invalid_argument("ControllerSegmentTable::getTable_upgrade()"); + if (num < 0 || num > nTables - 1) + throw std::invalid_argument("ControllerSegmentTable::getTable_upgrade()"); - rwlock[num]->upgrade_to_write(); + rwlock[num]->upgrade_to_write(); } void MasterSegmentTable::getTable_downgrade(int num) const { - if (num < 0 || num > nTables - 1) - throw std::invalid_argument("ControllerSegmentTable::getTable_downgrade()"); + if (num < 0 || num > nTables - 1) + throw std::invalid_argument("ControllerSegmentTable::getTable_downgrade()"); - rwlock[num]->downgrade_to_read(); + rwlock[num]->downgrade_to_read(); } void MasterSegmentTable::releaseTable_read(int num) const { - if (num < 0 || num >= nTables) - throw std::invalid_argument("ControllerSegmentTable::releaseTable()"); + if (num < 0 || num >= nTables) + throw std::invalid_argument("ControllerSegmentTable::releaseTable()"); - rwlock[num]->read_unlock(); + rwlock[num]->read_unlock(); } void MasterSegmentTable::releaseTable_write(int num) const { - if (num < 0 || num >= nTables) - throw std::invalid_argument("ControllerSegmentTable::releaseTable()"); + if (num < 0 || num >= nTables) + throw std::invalid_argument("ControllerSegmentTable::releaseTable()"); - rwlock[num]->write_unlock(); + rwlock[num]->write_unlock(); } -} //namespace +} // namespace BRM diff --git a/versioning/BRM/mastersegmenttable.h b/versioning/BRM/mastersegmenttable.h index e71a1556d..beab0fa49 100644 --- a/versioning/BRM/mastersegmenttable.h +++ b/versioning/BRM/mastersegmenttable.h @@ -52,40 +52,39 @@ namespace BRM { - struct MSTEntry { - key_t tableShmkey; - int allocdSize; - int currentSize; - EXPORT MSTEntry(); + key_t tableShmkey; + int allocdSize; + int currentSize; + EXPORT MSTEntry(); }; class MasterSegmentTableImpl { -public: - ~MasterSegmentTableImpl(){}; - static MasterSegmentTableImpl* makeMasterSegmentTableImpl(int key, int size); + public: + ~MasterSegmentTableImpl(){}; + static MasterSegmentTableImpl* makeMasterSegmentTableImpl(int key, int size); - static void refreshShm() + static void refreshShm() + { + if (fInstance) { - if (fInstance) - { - delete fInstance; - fInstance = NULL; - } + delete fInstance; + fInstance = NULL; } - - boost::interprocess::shared_memory_object fShmobj; - boost::interprocess::mapped_region fMapreg; - -private: - MasterSegmentTableImpl(int key, int size); - MasterSegmentTableImpl(const MasterSegmentTableImpl& rhs); - MasterSegmentTableImpl& operator=(const MasterSegmentTableImpl& rhs); + } - static boost::mutex fInstanceMutex; - static MasterSegmentTableImpl* fInstance; + boost::interprocess::shared_memory_object fShmobj; + boost::interprocess::mapped_region fMapreg; + + private: + MasterSegmentTableImpl(int key, int size); + MasterSegmentTableImpl(const MasterSegmentTableImpl& rhs); + MasterSegmentTableImpl& operator=(const MasterSegmentTableImpl& rhs); + + static boost::mutex fInstanceMutex; + static MasterSegmentTableImpl* fInstance; }; /** @brief This class regulates access to the BRM tables in shared memory @@ -94,122 +93,122 @@ private: */ class MasterSegmentTable { -public: - /** @brief Constructor. - * @note Throws runtime_error on a semaphore-related error. - */ - EXPORT MasterSegmentTable(); - EXPORT ~MasterSegmentTable(); + public: + /** @brief Constructor. + * @note Throws runtime_error on a semaphore-related error. + */ + EXPORT MasterSegmentTable(); + EXPORT ~MasterSegmentTable(); - /// specifies the Extent Map table - static const int EMTable = 0; - /// specifies the Extent Map's Freelist table - static const int EMFreeList = 1; - /// specifies the Version Buffer Block Map segment - static const int VBBMSegment = 2; - /// specifies the Version Substitution Structure segment - static const int VSSSegment = 3; - /// specifies the copy lock segment - static const int CLSegment = 4; - /// the number of tables currently defined - static const int nTables = 5; + /// specifies the Extent Map table + static const int EMTable = 0; + /// specifies the Extent Map's Freelist table + static const int EMFreeList = 1; + /// specifies the Version Buffer Block Map segment + static const int VBBMSegment = 2; + /// specifies the Version Substitution Structure segment + static const int VSSSegment = 3; + /// specifies the copy lock segment + static const int CLSegment = 4; + /// the number of tables currently defined + static const int nTables = 5; - /** @brief This function gets the specified table. - * - * This function gets the specified table and grabs the - * associated read lock. - * @param num EMTable, EMFreeList, or VBBMTable - * @param block If false, it won't block. - * @note throws invalid_argument if num is outside the valid range - * and runtime_error on a semaphore-related error. - * @return If block == true, it always returns the specified MSTEntry; - * if block == false, it can also return NULL if it could not grab - * the table's lock. - */ - EXPORT MSTEntry* getTable_read(int num, bool block = true) const; + /** @brief This function gets the specified table. + * + * This function gets the specified table and grabs the + * associated read lock. + * @param num EMTable, EMFreeList, or VBBMTable + * @param block If false, it won't block. + * @note throws invalid_argument if num is outside the valid range + * and runtime_error on a semaphore-related error. + * @return If block == true, it always returns the specified MSTEntry; + * if block == false, it can also return NULL if it could not grab + * the table's lock. + */ + EXPORT MSTEntry* getTable_read(int num, bool block = true) const; - /** @brief This function gets the specified table. - * - * This function gets the specified table and grabs the - * associated write lock. - * @param num EMTable, EMFreeList, or VBBMTable - * @param block If false, it won't block. - * @note throws invalid_argument if num is outside the valid range - * and runtime_error on a semaphore-related error. - * @return If block == true, it always returns the specified MSTEntry; - * if block == false, it can also return NULL if it could not grab - * the table's lock. - */ - EXPORT MSTEntry* getTable_write(int num, bool block = true) const; + /** @brief This function gets the specified table. + * + * This function gets the specified table and grabs the + * associated write lock. + * @param num EMTable, EMFreeList, or VBBMTable + * @param block If false, it won't block. + * @note throws invalid_argument if num is outside the valid range + * and runtime_error on a semaphore-related error. + * @return If block == true, it always returns the specified MSTEntry; + * if block == false, it can also return NULL if it could not grab + * the table's lock. + */ + EXPORT MSTEntry* getTable_write(int num, bool block = true) const; - /** @brief Upgrade a read lock to a write lock. - * - * Upgrade a read lock to a write lock. This is not an atomic - * operation. - * @param num The table the caller holds the read lock to. - */ - EXPORT void getTable_upgrade(int num) const; + /** @brief Upgrade a read lock to a write lock. + * + * Upgrade a read lock to a write lock. This is not an atomic + * operation. + * @param num The table the caller holds the read lock to. + */ + EXPORT void getTable_upgrade(int num) const; - /** @brief Downgrade a write lock to a read lock. - * - * Downgrade a write lock to a read lock. This is an atomic - * operation. - * @param num The table the caller holds the write lock to. - */ - EXPORT void getTable_downgrade(int num) const; + /** @brief Downgrade a write lock to a read lock. + * + * Downgrade a write lock to a read lock. This is an atomic + * operation. + * @param num The table the caller holds the write lock to. + */ + EXPORT void getTable_downgrade(int num) const; - /** @brief This function unlocks the specified table. - * - * This function unlocks the specified table. - * - * @param num EMTable, EMFreeList, or VBBMTable - * @note throws invalid_argument if num is outside the valid range - * and runtime_error on a semaphore-related error. - */ - EXPORT void releaseTable_read(int num) const; + /** @brief This function unlocks the specified table. + * + * This function unlocks the specified table. + * + * @param num EMTable, EMFreeList, or VBBMTable + * @note throws invalid_argument if num is outside the valid range + * and runtime_error on a semaphore-related error. + */ + EXPORT void releaseTable_read(int num) const; - /** @brief This function unlocks the specified table. - * - * This function unlocks the specified table. - * - * @param num EMTable, EMFreeList, or VBBMTable - * @note throws invalid_argument if num is outside the valid range - * and runtime_error on a semaphore-related error. - */ - EXPORT void releaseTable_write(int num) const; + /** @brief This function unlocks the specified table. + * + * This function unlocks the specified table. + * + * @param num EMTable, EMFreeList, or VBBMTable + * @note throws invalid_argument if num is outside the valid range + * and runtime_error on a semaphore-related error. + */ + EXPORT void releaseTable_write(int num) const; - /** @brief This function gets the current VSS key out of shared memory without locking - * - * This function gets the current VSS key out of shared memory without any locking. It is used - * (eventually) by DBRM::vssLookup() to try and correctly & efficiently short-circuit a vss - * lookup when there are no locked blocks in the vss. This read should be atomic. Even if it's - * not, we should still be okay. - */ - inline key_t getVSSShmkey() const - { - return fShmDescriptors[VSSSegment].tableShmkey; - } + /** @brief This function gets the current VSS key out of shared memory without locking + * + * This function gets the current VSS key out of shared memory without any locking. It is used + * (eventually) by DBRM::vssLookup() to try and correctly & efficiently short-circuit a vss + * lookup when there are no locked blocks in the vss. This read should be atomic. Even if it's + * not, we should still be okay. + */ + inline key_t getVSSShmkey() const + { + return fShmDescriptors[VSSSegment].tableShmkey; + } -private: - MasterSegmentTable(const MasterSegmentTable& mst); - MasterSegmentTable& operator=(const MasterSegmentTable& mst); + private: + MasterSegmentTable(const MasterSegmentTable& mst); + MasterSegmentTable& operator=(const MasterSegmentTable& mst); - int shmid; - mutable boost::scoped_ptr rwlock[nTables]; + int shmid; + mutable boost::scoped_ptr rwlock[nTables]; - static const int MSTshmsize = nTables * sizeof(MSTEntry); - int RWLockKeys[nTables]; + static const int MSTshmsize = nTables * sizeof(MSTEntry); + int RWLockKeys[nTables]; - /// indexed by EMTable, EMFreeList, and VBBMTable - MSTEntry* fShmDescriptors; + /// indexed by EMTable, EMFreeList, and VBBMTable + MSTEntry* fShmDescriptors; - void makeMSTSegment(); - void initMSTData(); - ShmKeys fShmKeys; - MasterSegmentTableImpl* fPImpl; + void makeMSTSegment(); + void initMSTData(); + ShmKeys fShmKeys; + MasterSegmentTableImpl* fPImpl; }; -} //namespace +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/oidserver.cpp b/versioning/BRM/oidserver.cpp index d8184ed4e..39125416f 100644 --- a/versioning/BRM/oidserver.cpp +++ b/versioning/BRM/oidserver.cpp @@ -92,16 +92,16 @@ using namespace idbdatafile; #include "dbrm.h" #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_DIRECT -# define O_DIRECT 0 +#define O_DIRECT 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif namespace @@ -109,289 +109,289 @@ namespace boost::mutex CtorMutex; class EOFException : public exception -{}; -} +{ +}; +} // namespace namespace BRM { - boost::mutex OIDServer::fMutex; void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const { - int errCount, err, progress; - off_t seekerr = -1; + int errCount, err, progress; + off_t seekerr = -1; - if (size == 0) - return; + if (size == 0) + return; - for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) + for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) + { + seekerr = fFp->seek(offset, SEEK_SET); + + if (seekerr >= 0) + seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. + + if (seekerr < 0) + perror("OIDServer::writeData(): lseek"); + } + + if (errCount == MaxRetries) + throw ios_base::failure( + "OIDServer::writeData(): lseek failed " + "too many times"); + + for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) + { + err = fFp->write(&buf[progress], size - progress); + + if (err < 0) { - seekerr = fFp->seek(offset, SEEK_SET); - - if (seekerr >= 0) - seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. - - if (seekerr < 0) - perror("OIDServer::writeData(): lseek"); + if (errno != EINTR) // EINTR isn't really an error + { + errCount++; + perror("OIDServer::writeData(): write (retrying)"); + } } + else + progress += err; + } - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::writeData(): lseek failed " - "too many times"); + fFp->tell(); - for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) - { - err = fFp->write(&buf[progress], size - progress); - - if (err < 0) - { - if (errno != EINTR) // EINTR isn't really an error - { - errCount++; - perror("OIDServer::writeData(): write (retrying)"); - } - } - else - progress += err; - } - - fFp->tell(); - - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::writeData(): write error"); + if (errCount == MaxRetries) + throw ios_base::failure("OIDServer::writeData(): write error"); } void OIDServer::readData(uint8_t* buf, off_t offset, int size) const { - int errCount, err, progress; - off_t seekerr = -1; + int errCount, err, progress; + off_t seekerr = -1; - if (size == 0) - return; + if (size == 0) + return; - for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) + for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) + { + seekerr = fFp->seek(offset, SEEK_SET); + + if (seekerr >= 0) + seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. + + if (seekerr < 0) + perror("OIDServer::readData(): lseek"); + } + + if (errCount == MaxRetries) + throw ios_base::failure( + "OIDServer::readData(): lseek failed " + "too many times"); + + for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) + { + err = fFp->read(&buf[progress], size - progress); + + if (err < 0) { - seekerr = fFp->seek(offset, SEEK_SET); - - if (seekerr >= 0) - seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. - - if (seekerr < 0) - perror("OIDServer::readData(): lseek"); + if (errno != EINTR) // EINTR isn't really an error + { + errCount++; + perror("OIDServer::readData(): read (retrying)"); + } } + else if (err == 0) + throw EOFException(); + else + progress += err; + } - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::readData(): lseek failed " - "too many times"); - - for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) - { - err = fFp->read(&buf[progress], size - progress); - - if (err < 0) - { - if (errno != EINTR) // EINTR isn't really an error - { - errCount++; - perror("OIDServer::readData(): read (retrying)"); - } - } - else if (err == 0) - throw EOFException(); - else - progress += err; - } - - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::readData(): read error"); + if (errCount == MaxRetries) + throw ios_base::failure("OIDServer::readData(): read error"); } void OIDServer::initializeBitmap() const { - uint8_t buf[HeaderSize]; - int i, bitmapSize = StartOfVBOidSection - HeaderSize; - struct FEntry* h1; - string stmp; - int64_t ltmp; - int firstOID; - config::Config* conf; + uint8_t buf[HeaderSize]; + int i, bitmapSize = StartOfVBOidSection - HeaderSize; + struct FEntry* h1; + string stmp; + int64_t ltmp; + int firstOID; + config::Config* conf; - conf = config::Config::makeConfig(); + conf = config::Config::makeConfig(); - try - { - stmp = conf->getConfig("OIDManager", "FirstOID"); - } - catch (exception&) - { - } + try + { + stmp = conf->getConfig("OIDManager", "FirstOID"); + } + catch (exception&) + { + } - if (stmp.empty()) stmp = "3000"; + if (stmp.empty()) + stmp = "3000"; - ltmp = config::Config::fromText(stmp); + ltmp = config::Config::fromText(stmp); - if (ltmp > numeric_limits::max() || ltmp < 0) - { - ltmp = config::Config::fromText("3000"); - } + if (ltmp > numeric_limits::max() || ltmp < 0) + { + ltmp = config::Config::fromText("3000"); + } - firstOID = static_cast(ltmp); + firstOID = static_cast(ltmp); - boost::mutex::scoped_lock lk(fMutex); - h1 = reinterpret_cast(buf); - //write the initial header - h1[0].begin = firstOID; - h1[0].end = 0x00ffffff; + boost::mutex::scoped_lock lk(fMutex); + h1 = reinterpret_cast(buf); + // write the initial header + h1[0].begin = firstOID; + h1[0].end = 0x00ffffff; - for (i = 1; i < FreeListEntries; i++) - { - h1[i].begin = -1; - h1[i].end = -1; - } + for (i = 1; i < FreeListEntries; i++) + { + h1[i].begin = -1; + h1[i].end = -1; + } - writeData(buf, 0, HeaderSize); - - // write the new bitmap file - uint8_t *bitmapbuf = new uint8_t[bitmapSize]; - memset(bitmapbuf, 0, bitmapSize); - writeData(bitmapbuf, HeaderSize, bitmapSize); - delete[] bitmapbuf; - + writeData(buf, 0, HeaderSize); - flipOIDBlock(0, firstOID, 0); + // write the new bitmap file + uint8_t* bitmapbuf = new uint8_t[bitmapSize]; + memset(bitmapbuf, 0, bitmapSize); + writeData(bitmapbuf, HeaderSize, bitmapSize); + delete[] bitmapbuf; - buf[0] = 0; - buf[1] = 0; - /* append a 16-bit 0 to indicate 0 entries in the vboid->dbroot mapping */ - writeData(buf, StartOfVBOidSection, 2); + flipOIDBlock(0, firstOID, 0); + + buf[0] = 0; + buf[1] = 0; + /* append a 16-bit 0 to indicate 0 entries in the vboid->dbroot mapping */ + writeData(buf, StartOfVBOidSection, 2); } OIDServer::OIDServer() : fFp(NULL), fFd(-1) { - boost::mutex::scoped_lock lk(CtorMutex); + boost::mutex::scoped_lock lk(CtorMutex); - config::Config* conf; - string tmp; - ostringstream os; + config::Config* conf; + string tmp; + ostringstream os; - conf = config::Config::makeConfig(); - fFilename = conf->getConfig("OIDManager", "OIDBitmapFile"); + conf = config::Config::makeConfig(); + fFilename = conf->getConfig("OIDManager", "OIDBitmapFile"); - if (fFilename.empty()) + if (fFilename.empty()) + { + os << "OIDServer: must exist in the config file"; + log(os.str()); + throw runtime_error(os.str()); + } + + if (!IDBPolicy::exists(fFilename.c_str())) // no bitmap file + { + BRM::DBRM em; + + if (!em.isEMEmpty()) { - os << "OIDServer: must exist in the config file"; - log(os.str()); - throw runtime_error(os.str()); + os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only"; + cerr << os.str() << endl; + log(os.str()); + em.setReadOnly(true); + throw runtime_error(os.str()); } - if (!IDBPolicy::exists(fFilename.c_str())) //no bitmap file + fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), fFilename.c_str(), + "w+b", 0, 1); + + if (!fFp) { - BRM::DBRM em; - - if (!em.isEMEmpty()) - { - os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only"; - cerr << os.str() << endl; - log(os.str()); - em.setReadOnly(true); - throw runtime_error(os.str()); - } - - fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), - fFilename.c_str(), "w+b", 0, 1); - - if (!fFp) - { - os << "Couldn't create oid bitmap file " << fFilename << ": " << - strerror(errno); - log(os.str()); - throw ios_base::failure(os.str()); - } + os << "Couldn't create oid bitmap file " << fFilename << ": " << strerror(errno); + log(os.str()); + throw ios_base::failure(os.str()); + } #ifndef _MSC_VER - if (fFp) - chmod(fFilename.c_str(), 0664); // XXXPAT: override umask at least for testing + if (fFp) + chmod(fFilename.c_str(), 0664); // XXXPAT: override umask at least for testing #endif - try - { - initializeBitmap(); - } - catch (...) - { - delete fFp; - fFp = NULL; - throw; - } - } - else + try { - fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), - fFilename.c_str(), "r+b", 0, 1); - - if (!fFp) - { - ostringstream os; - os << "Couldn't open oid bitmap file" << fFilename << ": " << - strerror(errno); - log(os.str()); - throw ios_base::failure(os.str()); - } + initializeBitmap(); } + catch (...) + { + delete fFp; + fFp = NULL; + throw; + } + } + else + { + fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), fFilename.c_str(), + "r+b", 0, 1); - loadVBOIDs(); + if (!fFp) + { + ostringstream os; + os << "Couldn't open oid bitmap file" << fFilename << ": " << strerror(errno); + log(os.str()); + throw ios_base::failure(os.str()); + } + } + + loadVBOIDs(); } OIDServer::~OIDServer() { - if (fFd >= 0) - close(fFd); + if (fFd >= 0) + close(fFd); - delete fFp; - fFp = NULL; + delete fFp; + fFp = NULL; } void OIDServer::loadVBOIDs() { - uint16_t size; + uint16_t size; - try - { - readData((uint8_t*) &size, StartOfVBOidSection, 2); - } - catch (EOFException& e) - { - /* convert old OID bitmap */ - size = 0; - writeData((uint8_t*) &size, StartOfVBOidSection, 2); - } + try + { + readData((uint8_t*)&size, StartOfVBOidSection, 2); + } + catch (EOFException& e) + { + /* convert old OID bitmap */ + size = 0; + writeData((uint8_t*)&size, StartOfVBOidSection, 2); + } - if (size == 0) - return; + if (size == 0) + return; - vbOidDBRootMap.resize(size); - readData((uint8_t*) &vbOidDBRootMap[0], StartOfVBOidSection + 2, size * 2); + vbOidDBRootMap.resize(size); + readData((uint8_t*)&vbOidDBRootMap[0], StartOfVBOidSection + 2, size * 2); - //cout << "loaded " << size << " vboids: "; - //for (uint32_t i = 0; i < size; i++) - // cout << (int) vbOidDBRootMap[i] << " "; - //cout << endl; + // cout << "loaded " << size << " vboids: "; + // for (uint32_t i = 0; i < size; i++) + // cout << (int) vbOidDBRootMap[i] << " "; + // cout << endl; } void OIDServer::useFreeListEntry(struct FEntry& fe, int num) { - int blockSize; + int blockSize; - blockSize = fe.end - fe.begin + 1; + blockSize = fe.end - fe.begin + 1; - if (blockSize == num) - { - fe.begin = -1; - fe.end = -1; - } - else - fe.begin += num; + if (blockSize == num) + { + fe.begin = -1; + fe.end = -1; + } + else + fe.begin += num; } // mode = 0 -> allocate, mode = 1 -> deallocate @@ -401,427 +401,427 @@ void OIDServer::useFreeListEntry(struct FEntry& fe, int num) // matter. void OIDServer::flipOIDBlock(int blockStart, int num, int mode) const { - int offset, i, oidCount, byteSize, blockEnd; - uint8_t* buf; - uint8_t mask; + int offset, i, oidCount, byteSize, blockEnd; + uint8_t* buf; + uint8_t mask; - // safety check - if (blockStart + num - 1 > 0x00ffffff) - throw logic_error("flipOIDBlock: request overruns oid space"); + // safety check + if (blockStart + num - 1 > 0x00ffffff) + throw logic_error("flipOIDBlock: request overruns oid space"); - blockEnd = blockStart + num - 1; - offset = blockStart / 8 + HeaderSize; - byteSize = blockEnd / 8 - blockStart / 8 + 1; - i = 0; + blockEnd = blockStart + num - 1; + offset = blockStart / 8 + HeaderSize; + byteSize = blockEnd / 8 - blockStart / 8 + 1; + i = 0; retry: - try - { - buf = new uint8_t[byteSize]; - } - catch (bad_alloc&) - { - if (++i == MaxRetries) - throw; + try + { + buf = new uint8_t[byteSize]; + } + catch (bad_alloc&) + { + if (++i == MaxRetries) + throw; - cerr << "flipOIDBlock: mem alloc failed (retrying)" << endl; - goto retry; + cerr << "flipOIDBlock: mem alloc failed (retrying)" << endl; + goto retry; + } + + oidCount = 0; + readData(buf, offset, byteSize); + + // verify 1st byte + mask = 0x80 >> (blockStart % 8); + + while (mask != 0 && oidCount < num) + { + if ((buf[0] & mask) != (mode ? mask : 0)) + { + delete[] buf; + throw logic_error("flipOIDBlock: bad allocation or deallocation attempted (1)"); } - oidCount = 0; - readData(buf, offset, byteSize); + if (mode) + buf[0] &= ~mask; + else + buf[0] |= mask; - // verify 1st byte - mask = 0x80 >> (blockStart % 8); + mask >>= 1; + oidCount++; + } - while (mask != 0 && oidCount < num) + if (oidCount == num) + { + writeData(buf, offset, byteSize); + + fFp->flush(); + + delete[] buf; + return; + } + + // verify the middle bytes + for (i = 1; i < byteSize - 1; i++, oidCount += 8) + if (buf[i] != (mode ? 0xff : 0)) { - if ((buf[0] & mask) != (mode ? mask : 0)) - { - delete [] buf; - throw logic_error("flipOIDBlock: bad allocation or deallocation attempted (1)"); - } + delete[] buf; + throw logic_error("flipOIDBlock: bad allocation or deallocation attempted (2)"); + } + else if (mode) + buf[i] = 0; + else + buf[i] = 0xff; - if (mode) - buf[0] &= ~mask; - else - buf[0] |= mask; + // verify the last byte + mask = 0x80; - mask >>= 1; - oidCount++; + while (mask != 0 && oidCount < num) + { + if ((buf[byteSize - 1] & mask) != (mode ? mask : 0)) + { + delete[] buf; + + if (mode) + throw logic_error("flipOIDBlock: bad deallocation attempted"); + else + throw logic_error("flipOIDBlock: bad allocation attempted"); } - if (oidCount == num) - { - writeData(buf, offset, byteSize); + if (mode) + buf[byteSize - 1] &= ~mask; + else + buf[byteSize - 1] |= mask; - fFp->flush(); + mask >>= 1; + oidCount++; + } - delete [] buf; - return; - } + if (oidCount == num) + { + writeData(buf, offset, byteSize); - // verify the middle bytes - for (i = 1; i < byteSize - 1; i++, oidCount += 8) - if (buf[i] != (mode ? 0xff : 0)) - { - delete [] buf; - throw logic_error("flipOIDBlock: bad allocation or deallocation attempted (2)"); - } - else if (mode) - buf[i] = 0; - else - buf[i] = 0xff; + fFp->flush(); - // verify the last byte - mask = 0x80; + delete[] buf; + return; + } - while (mask != 0 && oidCount < num) - { - if ((buf[byteSize - 1] & mask) != (mode ? mask : 0)) - { - delete [] buf; - - if ( mode ) - throw logic_error("flipOIDBlock: bad deallocation attempted"); - else - throw logic_error("flipOIDBlock: bad allocation attempted"); - } - - if (mode) - buf[byteSize - 1] &= ~mask; - else - buf[byteSize - 1] |= mask; - - mask >>= 1; - oidCount++; - } - - if (oidCount == num) - { - writeData(buf, offset, byteSize); - - fFp->flush(); - - delete [] buf; - return; - } - - delete [] buf; - throw logic_error("logic error in flipOIDBlock detected"); + delete[] buf; + throw logic_error("logic error in flipOIDBlock detected"); } int OIDServer::fullScan(int num, struct FEntry* freelist) const { - uint8_t buf[4096]; - int fileOffset, i, blockCount = 0, blockStart = 0, bitOffset; - uint8_t mask; - bool countingZeros = false; + uint8_t buf[4096]; + int fileOffset, i, blockCount = 0, blockStart = 0, bitOffset; + uint8_t mask; + bool countingZeros = false; - fileOffset = HeaderSize; + fileOffset = HeaderSize; - // this assumes the bitmap is a multiple of 4096 - while (fileOffset < StartOfVBOidSection) + // this assumes the bitmap is a multiple of 4096 + while (fileOffset < StartOfVBOidSection) + { + readData(reinterpret_cast(buf), fileOffset, 4096); + + for (i = 0; i < 4096; i++) { - readData(reinterpret_cast(buf), fileOffset, 4096); + if (countingZeros) + { + mask = 0x80; + bitOffset = 0; + goto countZeros; + } + else if (buf[i] != 0xff) + { + mask = 0x80; + bitOffset = 0; + skipOnes: - for (i = 0; i < 4096; i++) + while ((buf[i] & mask) == mask && bitOffset < 8) { - if (countingZeros) - { - mask = 0x80; - bitOffset = 0; - goto countZeros; - } - else if (buf[i] != 0xff) - { - mask = 0x80; - bitOffset = 0; -skipOnes: - - while ((buf[i] & mask) == mask && bitOffset < 8) - { - mask >>= 1; - bitOffset++; - } - - if (bitOffset == 8) - continue; - - countingZeros = true; - blockStart = ((fileOffset - HeaderSize + i) * 8) + bitOffset; - blockCount = 1; - bitOffset++; - mask >>= 1; -countZeros: - - while ((buf[i] & mask) == 0 && bitOffset < 8 && blockCount < num) - { - mask >>= 1; - blockCount++; - bitOffset++; - } - - if (blockCount == num) //found a match - { - patchFreelist(freelist, blockStart, blockCount); - flipOIDBlock(blockStart, blockCount, 0); - return blockStart; - } - - if (bitOffset == 8) - continue; - - countingZeros = false; - goto skipOnes; - } + mask >>= 1; + bitOffset++; } - fileOffset += 4096; + if (bitOffset == 8) + continue; + + countingZeros = true; + blockStart = ((fileOffset - HeaderSize + i) * 8) + bitOffset; + blockCount = 1; + bitOffset++; + mask >>= 1; + countZeros: + + while ((buf[i] & mask) == 0 && bitOffset < 8 && blockCount < num) + { + mask >>= 1; + blockCount++; + bitOffset++; + } + + if (blockCount == num) // found a match + { + patchFreelist(freelist, blockStart, blockCount); + flipOIDBlock(blockStart, blockCount, 0); + return blockStart; + } + + if (bitOffset == 8) + continue; + + countingZeros = false; + goto skipOnes; + } } - return -1; + fileOffset += 4096; + } + + return -1; } void OIDServer::patchFreelist(struct FEntry* freelist, int start, int num) const { - int i, changed = 0, end = start + num - 1; + int i, changed = 0, end = start + num - 1; - for (i = 0; i < FreeListEntries; i++) + for (i = 0; i < FreeListEntries; i++) + { + if (freelist[i].begin == -1) + continue; + + // the allocated block overlaps the beginning of this entry. + // this is the only clause that should execute unless there's an + // error in the implementation somewhere (probably fullscan) + if (start <= freelist[i].begin && end >= freelist[i].begin) { + changed = 1; - if (freelist[i].begin == -1) - continue; - - // the allocated block overlaps the beginning of this entry. - // this is the only clause that should execute unless there's an - // error in the implementation somewhere (probably fullscan) - if (start <= freelist[i].begin && end >= freelist[i].begin) - { - changed = 1; - - // if possible, truncate this entry otherwise invalidate it - if (end < freelist[i].end) - freelist[i].begin = end + 1; - else - { - freelist[i].begin = -1; - freelist[i].end = -1; - } - } - - // the allocated block is contained in the middle of this block. - // (this shouldn't be possible; allocOIDs should have - // picked this entry to allocate from) - else if (start >= freelist[i].begin && end <= freelist[i].end) - throw logic_error("patchFreelist: a block was allocated in the " - "middle of a known-free block"); + // if possible, truncate this entry otherwise invalidate it + if (end < freelist[i].end) + freelist[i].begin = end + 1; + else + { + freelist[i].begin = -1; + freelist[i].end = -1; + } } - if (changed) - { - writeData(reinterpret_cast(freelist), 0, HeaderSize); - fFp->flush(); - } + // the allocated block is contained in the middle of this block. + // (this shouldn't be possible; allocOIDs should have + // picked this entry to allocate from) + else if (start >= freelist[i].begin && end <= freelist[i].end) + throw logic_error( + "patchFreelist: a block was allocated in the " + "middle of a known-free block"); + } + + if (changed) + { + writeData(reinterpret_cast(freelist), 0, HeaderSize); + fFp->flush(); + } } int OIDServer::allocVBOID(uint16_t dbroot) { - int ret; - uint32_t offset; + int ret; + uint32_t offset; - ret = vbOidDBRootMap.size(); + ret = vbOidDBRootMap.size(); - offset = StartOfVBOidSection + 2 + (vbOidDBRootMap.size() * 2); - vbOidDBRootMap.push_back(dbroot); + offset = StartOfVBOidSection + 2 + (vbOidDBRootMap.size() * 2); + vbOidDBRootMap.push_back(dbroot); - try - { - uint16_t size = vbOidDBRootMap.size(); - boost::mutex::scoped_lock lk(fMutex); - writeData((uint8_t*) &size, StartOfVBOidSection, 2); - writeData((uint8_t*) &dbroot, offset, 2); - } - catch (...) - { - vbOidDBRootMap.pop_back(); - throw; - } + try + { + uint16_t size = vbOidDBRootMap.size(); + boost::mutex::scoped_lock lk(fMutex); + writeData((uint8_t*)&size, StartOfVBOidSection, 2); + writeData((uint8_t*)&dbroot, offset, 2); + } + catch (...) + { + vbOidDBRootMap.pop_back(); + throw; + } - fFp->flush(); + fFp->flush(); - return ret; + return ret; } int OIDServer::getDBRootOfVBOID(uint32_t vbOID) { - //cout << "getDBRootOfVBOID, vbOID = " << vbOID << " size = " << vbOidDBRootMap.size() << endl; - if (vbOID >= vbOidDBRootMap.size()) - return -1; + // cout << "getDBRootOfVBOID, vbOID = " << vbOID << " size = " << vbOidDBRootMap.size() << endl; + if (vbOID >= vbOidDBRootMap.size()) + return -1; - return vbOidDBRootMap[vbOID]; + return vbOidDBRootMap[vbOID]; } const vector& OIDServer::getVBOIDToDBRootMap() { - return vbOidDBRootMap; + return vbOidDBRootMap; } int OIDServer::getVBOIDOfDBRoot(uint32_t dbRoot) { - uint32_t i; + uint32_t i; - for (i = 0; i < vbOidDBRootMap.size(); i++) - if (vbOidDBRootMap[i] == dbRoot) - return i; + for (i = 0; i < vbOidDBRootMap.size(); i++) + if (vbOidDBRootMap[i] == dbRoot) + return i; - return -1; + return -1; } int OIDServer::allocOIDs(int num) { - struct FEntry freelist[FreeListEntries]; - int i, size, bestMatchIndex, bestMatchSize, bestMatchBegin = 0; + struct FEntry freelist[FreeListEntries]; + int i, size, bestMatchIndex, bestMatchSize, bestMatchBegin = 0; - boost::mutex::scoped_lock lk(fMutex); - readData(reinterpret_cast(freelist), 0, HeaderSize); + boost::mutex::scoped_lock lk(fMutex); + readData(reinterpret_cast(freelist), 0, HeaderSize); - // scan freelist using best fit strategy (an attempt to maximize hits on - // the freelist) - bestMatchSize = numeric_limits::max(); + // scan freelist using best fit strategy (an attempt to maximize hits on + // the freelist) + bestMatchSize = numeric_limits::max(); - for (i = 0, bestMatchIndex = -1; i < FreeListEntries; i++) + for (i = 0, bestMatchIndex = -1; i < FreeListEntries; i++) + { + if (freelist[i].begin == -1) + continue; + + size = freelist[i].end - freelist[i].begin + 1; + + if (size == num) { - if (freelist[i].begin == -1) - continue; - - size = freelist[i].end - freelist[i].begin + 1; - - if (size == num) - { - bestMatchIndex = i; - bestMatchBegin = freelist[i].begin; - break; - } - - if (size > num && size < bestMatchSize) - { - bestMatchIndex = i; - bestMatchSize = size; - bestMatchBegin = freelist[i].begin; - } + bestMatchIndex = i; + bestMatchBegin = freelist[i].begin; + break; } - if (bestMatchIndex == -1) + if (size > num && size < bestMatchSize) { - return fullScan(num, freelist); + bestMatchIndex = i; + bestMatchSize = size; + bestMatchBegin = freelist[i].begin; } + } - useFreeListEntry(freelist[bestMatchIndex], num); - writeData(reinterpret_cast(freelist), 0, HeaderSize); - flipOIDBlock(bestMatchBegin, num, 0); + if (bestMatchIndex == -1) + { + return fullScan(num, freelist); + } - fFp->flush(); + useFreeListEntry(freelist[bestMatchIndex], num); + writeData(reinterpret_cast(freelist), 0, HeaderSize); + flipOIDBlock(bestMatchBegin, num, 0); - return bestMatchBegin; + fFp->flush(); + + return bestMatchBegin; } void OIDServer::returnOIDs(int start, int end) const { - //@Bug 1412. Do not reuse oids for now. - /* struct FEntry freelist[FreeListEntries]; - int i, emptyEntry = -1, entryBefore = -1, entryAfter = -1; - bool changed = true; + //@Bug 1412. Do not reuse oids for now. + /* struct FEntry freelist[FreeListEntries]; + int i, emptyEntry = -1, entryBefore = -1, entryAfter = -1; + bool changed = true; - lockFile(); + lockFile(); - try { - readData(reinterpret_cast(freelist), 0, HeaderSize); - } - catch (...) { - flock(fFd, LOCK_UN); - throw; - } + try { + readData(reinterpret_cast(freelist), 0, HeaderSize); + } + catch (...) { + flock(fFd, LOCK_UN); + throw; + } - for (i = 0; i < FreeListEntries; i++) { + for (i = 0; i < FreeListEntries; i++) { - //this entry is empty - if (freelist[i].begin == -1) { - if (emptyEntry == -1) - emptyEntry = i; - } - else - //this entry is before the block to return - if (freelist[i].end == start - 1) - entryBefore = i; - //the entry is after - else if (freelist[i].begin == end + 1) - entryAfter = i; - } + //this entry is empty + if (freelist[i].begin == -1) { + if (emptyEntry == -1) + emptyEntry = i; + } + else + //this entry is before the block to return + if (freelist[i].end == start - 1) + entryBefore = i; + //the entry is after + else if (freelist[i].begin == end + 1) + entryAfter = i; + } - if (entryBefore != -1 && entryAfter != -1) { - freelist[entryBefore].end = freelist[entryAfter].end; - freelist[entryAfter].begin = -1; - } - else if (entryBefore != -1) - freelist[entryBefore].end = end; - else if (entryAfter != -1) - freelist[entryAfter].begin = start; - else if (emptyEntry != -1) { - freelist[emptyEntry].begin = start; - freelist[emptyEntry].end = end; - } - else - changed = false; + if (entryBefore != -1 && entryAfter != -1) { + freelist[entryBefore].end = freelist[entryAfter].end; + freelist[entryAfter].begin = -1; + } + else if (entryBefore != -1) + freelist[entryBefore].end = end; + else if (entryAfter != -1) + freelist[entryAfter].begin = start; + else if (emptyEntry != -1) { + freelist[emptyEntry].begin = start; + freelist[emptyEntry].end = end; + } + else + changed = false; - // "Old Reliable" - for (i = 0; i < FreeListEntries; i++) - if (freelist[i].begin == -1) { - freelist[i].begin = start; - freelist[i].end = end; - break; - } + // "Old Reliable" + for (i = 0; i < FreeListEntries; i++) + if (freelist[i].begin == -1) { + freelist[i].begin = start; + freelist[i].end = end; + break; + } - try { - if (i != FreeListEntries) - if (changed) - writeData(reinterpret_cast(freelist), 0, HeaderSize); - flipOIDBlock(start, end - start + 1, 1); - } - catch (...) { - flock(fFd, LOCK_UN); - throw; - } + try { + if (i != FreeListEntries) + if (changed) + writeData(reinterpret_cast(freelist), 0, HeaderSize); + flipOIDBlock(start, end - start + 1, 1); + } + catch (...) { + flock(fFd, LOCK_UN); + throw; + } - flock(fFd, LOCK_UN); - */ + flock(fFd, LOCK_UN); + */ } int OIDServer::size() const { - int ret = 0, offset = 0, bytenum = 0; - uint8_t buf[4096], mask = 0; + int ret = 0, offset = 0, bytenum = 0; + uint8_t buf[4096], mask = 0; - boost::mutex::scoped_lock lk(fMutex); + boost::mutex::scoped_lock lk(fMutex); - for (offset = HeaderSize; offset < StartOfVBOidSection; offset += 4096) - { - readData(buf, offset, 4096); + for (offset = HeaderSize; offset < StartOfVBOidSection; offset += 4096) + { + readData(buf, offset, 4096); - for (bytenum = 0; bytenum < 4096; bytenum++) - for (mask = 0x80; mask != 0; mask >>= 1) - if ((buf[bytenum] & mask) == mask) - ret++; - } + for (bytenum = 0; bytenum < 4096; bytenum++) + for (mask = 0x80; mask != 0; mask >>= 1) + if ((buf[bytenum] & mask) == mask) + ret++; + } - return ret; + return ret; } const string OIDServer::getFilename() const { - return fFilename; + return fFilename; } -} // namespace +} // namespace BRM diff --git a/versioning/BRM/oidserver.h b/versioning/BRM/oidserver.h index d4e0762a5..3313c4bd2 100644 --- a/versioning/BRM/oidserver.h +++ b/versioning/BRM/oidserver.h @@ -45,10 +45,8 @@ namespace idbdatafile class IDBDataFile; } - namespace BRM { - /** @brief The class that manages the Object ID space * * The class that manages the allocation and deallocation of IDs @@ -56,159 +54,158 @@ namespace BRM */ class OIDServer { -private: - struct FEntry - { - int begin, end; - }; + private: + struct FEntry + { + int begin, end; + }; -public: - /** @brief Default constructor - * - * @note Throws ios::failure on a file IO error - */ - EXPORT OIDServer(); - EXPORT virtual ~OIDServer(); + public: + /** @brief Default constructor + * + * @note Throws ios::failure on a file IO error + */ + EXPORT OIDServer(); + EXPORT virtual ~OIDServer(); - /** @brief Allocate a contiguous range of Object IDs - * - * @param num The number of contiguous Object IDs to allocate - * @return The first OID allocated on success, -1 on failure - */ - EXPORT int allocOIDs(int num); + /** @brief Allocate a contiguous range of Object IDs + * + * @param num The number of contiguous Object IDs to allocate + * @return The first OID allocated on success, -1 on failure + */ + EXPORT int allocOIDs(int num); + /** @brief Allocates a single OID for a version buffer file, given its dbroot + * + * A new requirement in the shared nothing implementation is for version buffer + * files to be anchored to a specific dbroot. This association is stored on disk as + * an array of uint16s. Version buffer OIDs use 0-999, so the 0th offset in the + * array is the dbroot of the OID 0, and so on. + */ + EXPORT int allocVBOID(uint16_t dbroot); - /** @brief Allocates a single OID for a version buffer file, given its dbroot - * - * A new requirement in the shared nothing implementation is for version buffer - * files to be anchored to a specific dbroot. This association is stored on disk as - * an array of uint16s. Version buffer OIDs use 0-999, so the 0th offset in the - * array is the dbroot of the OID 0, and so on. - */ - EXPORT int allocVBOID(uint16_t dbroot); + /** @brief Returns the DBRoot of the given version buffer file. + * + */ + EXPORT int getDBRootOfVBOID(uint32_t vbOID); - /** @brief Returns the DBRoot of the given version buffer file. - * - */ - EXPORT int getDBRootOfVBOID(uint32_t vbOID); + /** @brief Returns the VB OID of the given DB root. + * + */ + EXPORT int getVBOIDOfDBRoot(uint32_t vbOID); - /** @brief Returns the VB OID of the given DB root. - * - */ - EXPORT int getVBOIDOfDBRoot(uint32_t vbOID); + /** @brief Returns the whole VB OID -> DBRoot mapping. The returned vector is indexed by VB OID. + * + */ + EXPORT const std::vector& getVBOIDToDBRootMap(); - /** @brief Returns the whole VB OID -> DBRoot mapping. The returned vector is indexed by VB OID. - * - */ - EXPORT const std::vector& getVBOIDToDBRootMap(); + /** @brief Return a list of OIDs to the pool + * + * @param start The first OID to return + * @param end The last OID to return + */ + EXPORT void returnOIDs(int start, int end) const; - /** @brief Return a list of OIDs to the pool - * - * @param start The first OID to return - * @param end The last OID to return - */ - EXPORT void returnOIDs(int start, int end) const; + /** @brief Counts the number of allocated OIDs + * + * @note This currently assumes the bitmap length is a multiple of 4096 + * @return The number of allocated OIDs + */ + EXPORT int size() const; - /** @brief Counts the number of allocated OIDs - * - * @note This currently assumes the bitmap length is a multiple of 4096 - * @return The number of allocated OIDs - */ - EXPORT int size() const; + /** @brief Get the OID bitmap filename + */ + const std::string getFilename() const; - /** @brief Get the OID bitmap filename - */ - const std::string getFilename() const; + private: + std::string fFilename; + static const int FreeListEntries = 256; + static const int HeaderSize = FreeListEntries * sizeof(FEntry); + static const int StartOfVBOidSection = HeaderSize + 2097152; // (2^24/8) + static const int MaxRetries = 10; /// max number of retries on file operations + static boost::mutex fMutex; + idbdatafile::IDBDataFile* fFp; + int fFd; /// file descriptor referencing the bitmap file + std::vector vbOidDBRootMap; -private: - std::string fFilename; - static const int FreeListEntries = 256; - static const int HeaderSize = FreeListEntries * sizeof(FEntry); - static const int StartOfVBOidSection = HeaderSize + 2097152; // (2^24/8) - static const int MaxRetries = 10; /// max number of retries on file operations - static boost::mutex fMutex; - idbdatafile::IDBDataFile* fFp; - int fFd; /// file descriptor referencing the bitmap file - std::vector vbOidDBRootMap; + /** @brief Grab the file lock + * + * Grab the file lock. + * @note Throws ios::failure after MaxRetries hard errors + */ + // void lockFile() const; - /** @brief Grab the file lock - * - * Grab the file lock. - * @note Throws ios::failure after MaxRetries hard errors - */ - //void lockFile() const; + /** @brief Reliably reads data from the bitmap file + * + * Reliably reads data from the bitmap file. + * @note Throws ios::failure after MaxRetries hard errors + * @note Caller is responsible for locking & unlocking + * @param buf the buffer to read into + * @param offset the offset to start reading at + * @param size the number of bytes to read into buf + */ + void readData(uint8_t* buf, off_t offset, int size) const; - /** @brief Reliably reads data from the bitmap file - * - * Reliably reads data from the bitmap file. - * @note Throws ios::failure after MaxRetries hard errors - * @note Caller is responsible for locking & unlocking - * @param buf the buffer to read into - * @param offset the offset to start reading at - * @param size the number of bytes to read into buf - */ - void readData(uint8_t* buf, off_t offset, int size) const; + /** @brief Reliably writes data to the bitmap file + * + * Reliably write data to the bitmap file. + * @note Throws ios::failure after MaxRetries hard errors + * @note Caller is responsible for locking & unlocking + * @param buf the data to write + * @param offset the offset to start writing at + * @param size the number of bytes to write + */ + void writeData(uint8_t* buf, off_t offset, int size) const; - /** @brief Reliably writes data to the bitmap file - * - * Reliably write data to the bitmap file. - * @note Throws ios::failure after MaxRetries hard errors - * @note Caller is responsible for locking & unlocking - * @param buf the data to write - * @param offset the offset to start writing at - * @param size the number of bytes to write - */ - void writeData(uint8_t* buf, off_t offset, int size) const; + /** @brief If there is no bitmap file yet, this is used to make one + * + * If there is no bitmap file yet, this is used to make one. + * @note It assumes no OIDs are allocated, we may want + * to fully rebuild it from actual DB contents instead. + */ + void initializeBitmap() const; - /** @brief If there is no bitmap file yet, this is used to make one - * - * If there is no bitmap file yet, this is used to make one. - * @note It assumes no OIDs are allocated, we may want - * to fully rebuild it from actual DB contents instead. - */ - void initializeBitmap() const; + /** @brief Allocates OIDs using a designated freelist entry + * + * Allocates OIDs using a designated freelist entry + * @param fe The freelist entry to use. + * @param num The number of OIDs to allocate from that block + */ + void useFreeListEntry(struct FEntry& fe, int num); - /** @brief Allocates OIDs using a designated freelist entry - * - * Allocates OIDs using a designated freelist entry - * @param fe The freelist entry to use. - * @param num The number of OIDs to allocate from that block - */ - void useFreeListEntry(struct FEntry& fe, int num); + /** @brief This allocates or deallocates a block of OIDs + * + * This allocates or deallocates a block of OIDs + * @param start The first OID to alloc/dealloc + * @param end The number of OIDs to flip + * @param mode mode = 0 means 'allocate', mode = 1 means 'deallocate' + */ + void flipOIDBlock(int start, int num, int mode) const; - /** @brief This allocates or deallocates a block of OIDs - * - * This allocates or deallocates a block of OIDs - * @param start The first OID to alloc/dealloc - * @param end The number of OIDs to flip - * @param mode mode = 0 means 'allocate', mode = 1 means 'deallocate' - */ - void flipOIDBlock(int start, int num, int mode) const; + /** @brief This scans the whole bitmap for a block of free OIDs + * + * This scans the whole bitmap for a block of free OIDs + * @param num the size of the block to locate + * @param freelist the freelist + * @return The first OID of the block allocated, or -1 if there is no match + */ + int fullScan(int num, struct FEntry* freelist) const; - /** @brief This scans the whole bitmap for a block of free OIDs - * - * This scans the whole bitmap for a block of free OIDs - * @param num the size of the block to locate - * @param freelist the freelist - * @return The first OID of the block allocated, or -1 if there is no match - */ - int fullScan(int num, struct FEntry* freelist) const; + /** @brief This is used by allocOIDs to fix the freelist after a full scan + * + * This is used by fullScan to fix the freelist before an allocation + * @param freelist The freelist + * @param start The first OID of the block allocated by fullScan + * @param num The number of OIDs allocated + * @note At the moment it throws logic_error if it detects a specific error in + * fullscan + */ + void patchFreelist(struct FEntry* freelist, int start, int num) const; - /** @brief This is used by allocOIDs to fix the freelist after a full scan - * - * This is used by fullScan to fix the freelist before an allocation - * @param freelist The freelist - * @param start The first OID of the block allocated by fullScan - * @param num The number of OIDs allocated - * @note At the moment it throws logic_error if it detects a specific error in - * fullscan - */ - void patchFreelist(struct FEntry* freelist, int start, int num) const; - - void loadVBOIDs(); + void loadVBOIDs(); }; -} // namespace +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/reset_locks.cpp b/versioning/BRM/reset_locks.cpp index 293e512b1..82f7acee9 100644 --- a/versioning/BRM/reset_locks.cpp +++ b/versioning/BRM/reset_locks.cpp @@ -18,7 +18,6 @@ // $Id: reset_locks.cpp 1823 2013-01-21 14:13:09Z rdempsey $ // - #include #include using namespace std; @@ -34,15 +33,21 @@ using namespace rwlock; using namespace BRM; -#define RESET(str, i) \ - do { cout << " - checking " << str << " semaphore" << endl; \ - rwlock[i]->reset(); } while (0) +#define RESET(str, i) \ + do \ + { \ + cout << " - checking " << str << " semaphore" << endl; \ + rwlock[i]->reset(); \ + } while (0) -#define PRINT(str, i) \ - do { cout << " - checking " << str << " semaphore" << endl \ - << " - r: " << rwlock[i]->getReading() << " w: " \ - << rwlock[i]->getWriting() << " rwt: " << rwlock[i]->getReadersWaiting() \ - << " wwt: " << rwlock[i]->getWritersWaiting() << endl; } while (0) +#define PRINT(str, i) \ + do \ + { \ + cout << " - checking " << str << " semaphore" << endl \ + << " - r: " << rwlock[i]->getReading() << " w: " << rwlock[i]->getWriting() \ + << " rwt: " << rwlock[i]->getReadersWaiting() << " wwt: " << rwlock[i]->getWritersWaiting() \ + << endl; \ + } while (0) namespace { @@ -52,126 +57,119 @@ bool sFlg; void usage() { - cout << "usage: reset_locks [-vnh]" << endl; - cout << " reset all ColumnStore shared memory locks" << endl; - cout << " -h display this help" << endl; - cout << " -v verbose output" << endl; - cout << " -s skip session mgr resets" << endl; - cout << " -n don't actually reset anything (implies -v)" << endl; -} + cout << "usage: reset_locks [-vnh]" << endl; + cout << " reset all ColumnStore shared memory locks" << endl; + cout << " -h display this help" << endl; + cout << " -v verbose output" << endl; + cout << " -s skip session mgr resets" << endl; + cout << " -n don't actually reset anything (implies -v)" << endl; } +} // namespace int main(int argc, char** argv) { #ifdef _MSC_VER - const char* envp = getenv("SystemRoot"); - string SystemRoot; + const char* envp = getenv("SystemRoot"); + string SystemRoot; - if (envp && *envp) - SystemRoot = envp; - else - SystemRoot = "C:\\WINDOWS"; + if (envp && *envp) + SystemRoot = envp; + else + SystemRoot = "C:\\WINDOWS"; - string tmpEnv = "TMP=" + SystemRoot + "\\Temp"; - _putenv(tmpEnv.c_str()); + string tmpEnv = "TMP=" + SystemRoot + "\\Temp"; + _putenv(tmpEnv.c_str()); #endif - BRM::DBRM dbrm; + BRM::DBRM dbrm; - int c; - opterr = 0; - vFlg = false; - nFlg = false; - sFlg = false; + int c; + opterr = 0; + vFlg = false; + nFlg = false; + sFlg = false; - while ((c = getopt(argc, argv, "vnh")) != EOF) - switch (c) + while ((c = getopt(argc, argv, "vnh")) != EOF) + switch (c) + { + case 'v': vFlg = true; break; + + case 'n': nFlg = true; break; + + case 's': sFlg = true; break; + + case 'h': + default: + usage(); + return (c == 'h' ? 0 : 1); + break; + } + + if (nFlg) + vFlg = true; + + ShmKeys keys; + + cerr << "(Exception msgs are probably OK)" << std::endl; + + RWLock* rwlock[MasterSegmentTable::nTables]; + int RWLockKeys[MasterSegmentTable::nTables]; + int i; + + if (MasterSegmentTable::nTables != 5) + { + cout << "There are more locks than reset_locks knows of. Time for an update." << endl; + exit(1); + } + + RWLockKeys[0] = keys.KEYRANGE_EXTENTMAP_BASE; + RWLockKeys[1] = keys.KEYRANGE_EMFREELIST_BASE; + RWLockKeys[2] = keys.KEYRANGE_VBBM_BASE; + RWLockKeys[3] = keys.KEYRANGE_VSS_BASE; + RWLockKeys[4] = keys.KEYRANGE_CL_BASE; + + for (i = 0; i < MasterSegmentTable::nTables; i++) + rwlock[i] = new RWLock(RWLockKeys[i]); + + if (vFlg) + { + PRINT("Extent Map", 0); + PRINT("Extent Map free list", 1); + PRINT("VBBM", 2); + PRINT("VSS", 3); + PRINT("CL", 4); + } + + if (!nFlg) + { + RESET("Extent Map", 0); + RESET("Extent Map free list", 1); + RESET("VBBM", 2); + RESET("VSS", 3); + RESET("CL", 4); + + if (!sFlg) + { + if (dbrm.isDBRMReady()) + { + cout << " - resetting SessionManager semaphore" << endl; + + try { - case 'v': - vFlg = true; - break; - - case 'n': - nFlg = true; - break; - - case 's': - sFlg = true; - break; - - case 'h': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; + execplan::SessionManager sm; + sm.reset(); } - - if (nFlg) - vFlg = true; - - ShmKeys keys; - - cerr << "(Exception msgs are probably OK)" << std::endl; - - RWLock* rwlock[MasterSegmentTable::nTables]; - int RWLockKeys[MasterSegmentTable::nTables]; - int i; - - if (MasterSegmentTable::nTables != 5) - { - cout << "There are more locks than reset_locks knows of. Time for an update." << endl; - exit(1); - } - - RWLockKeys[0] = keys.KEYRANGE_EXTENTMAP_BASE; - RWLockKeys[1] = keys.KEYRANGE_EMFREELIST_BASE; - RWLockKeys[2] = keys.KEYRANGE_VBBM_BASE; - RWLockKeys[3] = keys.KEYRANGE_VSS_BASE; - RWLockKeys[4] = keys.KEYRANGE_CL_BASE; - - for (i = 0; i < MasterSegmentTable::nTables; i++) - rwlock[i] = new RWLock(RWLockKeys[i]); - - if (vFlg) - { - PRINT("Extent Map", 0); - PRINT("Extent Map free list", 1); - PRINT("VBBM", 2); - PRINT("VSS", 3); - PRINT("CL", 4); - } - - if (!nFlg) - { - RESET("Extent Map", 0); - RESET("Extent Map free list", 1); - RESET("VBBM", 2); - RESET("VSS", 3); - RESET("CL", 4); - - if (!sFlg) + catch (std::exception& e) { - if (dbrm.isDBRMReady()) - { - cout << " - resetting SessionManager semaphore" << endl; - - try - { - execplan::SessionManager sm; - sm.reset(); - } - catch (std::exception& e) - { - std::cout << e.what() << std::endl; - } - } + std::cout << e.what() << std::endl; } + } } + } - for (i = 0; i < MasterSegmentTable::nTables; i++) - delete rwlock[i]; + for (i = 0; i < MasterSegmentTable::nTables; i++) + delete rwlock[i]; - std::cout << "OK." << std::endl; - return 0; + std::cout << "OK." << std::endl; + return 0; } - diff --git a/versioning/BRM/resource.h b/versioning/BRM/resource.h index 487e09b7f..d0afc2fd1 100644 --- a/versioning/BRM/resource.h +++ b/versioning/BRM/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/versioning/BRM/resourcenode.cpp b/versioning/BRM/resourcenode.cpp index 60debf93d..6bab54d2e 100644 --- a/versioning/BRM/resourcenode.cpp +++ b/versioning/BRM/resourcenode.cpp @@ -29,7 +29,6 @@ using namespace std; namespace BRM { - ResourceNode::ResourceNode() : _lbid(0) { } @@ -48,48 +47,47 @@ ResourceNode::~ResourceNode() ResourceNode& ResourceNode::operator=(const ResourceNode& n) { - static_cast(*this) = static_cast(n); - _lbid = n._lbid; - return *this; + static_cast(*this) = static_cast(n); + _lbid = n._lbid; + return *this; } bool ResourceNode::operator==(const ResourceNode& n) const { - return (_lbid == n._lbid); + return (_lbid == n._lbid); } bool ResourceNode::operator==(LBID_t l) const { - return (_lbid == l); + return (_lbid == l); } bool ResourceNode::operator<(const ResourceNode& n) const { - return (_lbid < n._lbid); + return (_lbid < n._lbid); } LBID_t ResourceNode::lbid() const { - return _lbid; + return _lbid; } void ResourceNode::wakeAndDetach() { - TransactionNode* txn; - set::iterator sit; - set::iterator dummy_sit; + TransactionNode* txn; + set::iterator sit; + set::iterator dummy_sit; - for (sit = in.begin(); sit != in.end(); ) - { - txn = dynamic_cast(*sit); - txn->wake(); - dummy_sit = ++sit; - removeInEdge(txn); - sit = dummy_sit; - } - - in.clear(); -} + for (sit = in.begin(); sit != in.end();) + { + txn = dynamic_cast(*sit); + txn->wake(); + dummy_sit = ++sit; + removeInEdge(txn); + sit = dummy_sit; + } + in.clear(); } +} // namespace BRM diff --git a/versioning/BRM/resourcenode.h b/versioning/BRM/resourcenode.h index f8a91d176..e0bb83cde 100644 --- a/versioning/BRM/resourcenode.h +++ b/versioning/BRM/resourcenode.h @@ -37,53 +37,53 @@ namespace BRM { - class ResourceNode : public RGNode { -public: - EXPORT ResourceNode(); - EXPORT ResourceNode(const ResourceNode&); - EXPORT ResourceNode(LBID_t); - EXPORT virtual ~ResourceNode(); + public: + EXPORT ResourceNode(); + EXPORT ResourceNode(const ResourceNode&); + EXPORT ResourceNode(LBID_t); + EXPORT virtual ~ResourceNode(); - EXPORT ResourceNode& operator=(const ResourceNode&); - EXPORT bool operator==(const ResourceNode&) const; - EXPORT bool operator==(LBID_t) const; - EXPORT bool operator<(const ResourceNode&) const; + EXPORT ResourceNode& operator=(const ResourceNode&); + EXPORT bool operator==(const ResourceNode&) const; + EXPORT bool operator==(LBID_t) const; + EXPORT bool operator<(const ResourceNode&) const; - EXPORT void wakeAndDetach(); + EXPORT void wakeAndDetach(); - EXPORT LBID_t lbid() const; -private: - LBID_t _lbid; + EXPORT LBID_t lbid() const; + + private: + LBID_t _lbid; }; -template +template struct RNLess { - bool operator()(const T& x, const T& y) const - { - return *x < *y; - } + bool operator()(const T& x, const T& y) const + { + return *x < *y; + } }; struct RNHasher { - size_t operator()(const ResourceNode* x) const - { - return x->lbid(); - } + size_t operator()(const ResourceNode* x) const + { + return x->lbid(); + } }; struct RNEquals { - bool operator()(const ResourceNode* x, const ResourceNode* y) const - { - return *x == *y; - } + bool operator()(const ResourceNode* x, const ResourceNode* y) const + { + return *x == *y; + } }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/rgnode.cpp b/versioning/BRM/rgnode.cpp index bb44a0869..911a7ac26 100644 --- a/versioning/BRM/rgnode.cpp +++ b/versioning/BRM/rgnode.cpp @@ -28,7 +28,6 @@ using namespace std; namespace BRM { - RGNode::RGNode() : _color(0) { } @@ -39,62 +38,61 @@ RGNode::RGNode(const RGNode& n) : out(n.out), in(n.in), _color(n._color) RGNode::~RGNode() { - set::iterator it; + set::iterator it; - for (it = in.begin(); it != in.end(); ) - { - (*it)->out.erase(this); - in.erase(it++); - } - - for (it = out.begin(); it != out.end(); ) - { - (*it)->in.erase(this); - out.erase(it++); - } + for (it = in.begin(); it != in.end();) + { + (*it)->out.erase(this); + in.erase(it++); + } + for (it = out.begin(); it != out.end();) + { + (*it)->in.erase(this); + out.erase(it++); + } } RGNode& RGNode::operator=(const RGNode& n) { - _color = n._color; - in = n.in; - out = n.out; - return *this; + _color = n._color; + in = n.in; + out = n.out; + return *this; } uint64_t RGNode::color() const { - return _color; + return _color; } void RGNode::color(uint64_t c) { - _color = c; + _color = c; } void RGNode::addOutEdge(RGNode* n) { - out.insert(n); - n->in.insert(this); + out.insert(n); + n->in.insert(this); } void RGNode::addInEdge(RGNode* n) { - in.insert(n); - n->out.insert(this); + in.insert(n); + n->out.insert(this); } void RGNode::removeOutEdge(RGNode* n) { - out.erase(n); - n->in.erase(this); + out.erase(n); + n->in.erase(this); } void RGNode::removeInEdge(RGNode* n) { - in.erase(n); - n->out.erase(this); + in.erase(n); + n->out.erase(this); } -} // namespace +} // namespace BRM diff --git a/versioning/BRM/rgnode.h b/versioning/BRM/rgnode.h index 97ca89230..6e6eb40fe 100644 --- a/versioning/BRM/rgnode.h +++ b/versioning/BRM/rgnode.h @@ -39,38 +39,37 @@ namespace BRM { - class RGNode; class RGNode { -public: - EXPORT RGNode(); - EXPORT RGNode(const RGNode&); - EXPORT virtual ~RGNode(); + public: + EXPORT RGNode(); + EXPORT RGNode(const RGNode&); + EXPORT virtual ~RGNode(); - EXPORT RGNode& operator=(const RGNode&); + EXPORT RGNode& operator=(const RGNode&); - EXPORT uint64_t color() const; - EXPORT void color(uint64_t); - EXPORT void addOutEdge(RGNode*); - EXPORT void addInEdge(RGNode*); - EXPORT void removeOutEdge(RGNode*); - EXPORT void removeInEdge(RGNode*); + EXPORT uint64_t color() const; + EXPORT void color(uint64_t); + EXPORT void addOutEdge(RGNode*); + EXPORT void addInEdge(RGNode*); + EXPORT void removeOutEdge(RGNode*); + EXPORT void removeInEdge(RGNode*); - friend class LBIDResourceGraph; + friend class LBIDResourceGraph; -protected: - // adjacency lists. Technically these should be private, but ResourceNode - // currently uses them to wake connected transactions. TBD... - std::set out; - std::set in; + protected: + // adjacency lists. Technically these should be private, but ResourceNode + // currently uses them to wake connected transactions. TBD... + std::set out; + std::set in; -private: - uint64_t _color; + private: + uint64_t _color; }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/rollback.cpp b/versioning/BRM/rollback.cpp index 2db4720e0..241b321ee 100644 --- a/versioning/BRM/rollback.cpp +++ b/versioning/BRM/rollback.cpp @@ -29,7 +29,6 @@ * rollback -r transID (for each one to roll them back) */ - #include "dbrm.h" #include @@ -42,84 +41,76 @@ using namespace BRM; void help(string name) { - - cout << "Usage: " << endl << name << " -r txnID" << endl; - cout << name << " -p" << endl << endl; - cout << "Options:" << endl; - cout << " -r -- rollback a transaction in the BRM data structures" << endl; - cout << " -p -- print current transactions according to the BRM" << endl; + cout << "Usage: " << endl << name << " -r txnID" << endl; + cout << name << " -p" << endl << endl; + cout << "Options:" << endl; + cout << " -r -- rollback a transaction in the BRM data structures" << endl; + cout << " -p -- print current transactions according to the BRM" << endl; } - - void printTxnIDs() { - DBRM brm; - set txnList; - set::iterator it; - int err; + DBRM brm; + set txnList; + set::iterator it; + int err; - err = brm.getCurrentTxnIDs(txnList); + err = brm.getCurrentTxnIDs(txnList); - if (err != 0) - return; + if (err != 0) + return; - for (it = txnList.begin(); it != txnList.end(); it++) - cout << *it << endl; + for (it = txnList.begin(); it != txnList.end(); it++) + cout << *it << endl; } - void rollbackTxn(VER_t txnID) { - DBRM brm; - vector lbidList; - int err; + DBRM brm; + vector lbidList; + int err; - err = brm.getUncommittedLBIDs(txnID, lbidList); + err = brm.getUncommittedLBIDs(txnID, lbidList); - if (err != 0) - return; + if (err != 0) + return; - err = brm.vbRollback(txnID, lbidList); + err = brm.vbRollback(txnID, lbidList); - if (err != 0) - return; + if (err != 0) + return; - cout << "OK." << endl; + cout << "OK." << endl; } int main(int argc, char** argv) { + int opt; + char options[] = "pr:"; + VER_t txnID = -1; + string progname(argv[0]); - int opt; - char options[] = "pr:"; - VER_t txnID = -1; - string progname(argv[0]); - - while ((opt = getopt (argc, argv, options)) != -1) + while ((opt = getopt(argc, argv, options)) != -1) + { + switch (opt) { - switch (opt) + case 'r': + txnID = atoi(optarg); + + if (txnID < 1) { - case 'r': - txnID = atoi (optarg); - - if (txnID < 1) - { - help(progname); - exit(0); - } - - idbdatafile::IDBPolicy::configIDBPolicy(); - rollbackTxn(txnID); - exit(0); - - case 'p': - printTxnIDs(); - exit(0); + help(progname); + exit(0); } + + idbdatafile::IDBPolicy::configIDBPolicy(); + rollbackTxn(txnID); + exit(0); + + case 'p': printTxnIDs(); exit(0); } + } - help(progname); - exit(0); + help(progname); + exit(0); } - diff --git a/versioning/BRM/rwlockmonitor.cpp b/versioning/BRM/rwlockmonitor.cpp index 3d7bc64d5..e117b40cd 100644 --- a/versioning/BRM/rwlockmonitor.cpp +++ b/versioning/BRM/rwlockmonitor.cpp @@ -35,14 +35,12 @@ using namespace logging; namespace BRM { - -RWLockMonitor::RWLockMonitor(const bool* d, const bool* ls, const uint32_t k) : - die(d), lockStatus(ls), key(k) +RWLockMonitor::RWLockMonitor(const bool* d, const bool* ls, const uint32_t k) : die(d), lockStatus(ls), key(k) { - ts.tv_sec = 210; // 3:30 timer - ts.tv_nsec = 0; - secsBetweenAttempts = 30; - lock.reset(new RWLock(key)); + ts.tv_sec = 210; // 3:30 timer + ts.tv_nsec = 0; + secsBetweenAttempts = 30; + lock.reset(new RWLock(key)); } RWLockMonitor::~RWLockMonitor() @@ -51,106 +49,105 @@ RWLockMonitor::~RWLockMonitor() void RWLockMonitor::operator()() { + /* + * Grab a timed write lock. + * on failure + * if there's an active reader, do read_unlock() + * log everything else. + * *** write lock fixing is being postponed for now. + */ - /* - * Grab a timed write lock. - * on failure - * if there's an active reader, do read_unlock() - * log everything else. - * *** write lock fixing is being postponed for now. - */ + LockState state; + bool gotTheLock; + bool reportedProblem = false; + Logger logger(30); - LockState state; - bool gotTheLock; - bool reportedProblem = false; - Logger logger(30); + while (!(*die)) + { + gotTheLock = lock->timed_write_lock(ts, &state); - while (!(*die)) + if (*die) + break; + + if (!gotTheLock) { - gotTheLock = lock->timed_write_lock(ts, &state); - - if (*die) - break; - - if (!gotTheLock) + if (state.mutexLocked) + { + if (!reportedProblem) { - if (state.mutexLocked) - { - if (!reportedProblem) - { - //Message msg(ERR_BRM_MUTEX); - Message msg(M0092); - logger.logMessage(LOG_TYPE_CRITICAL, msg, LoggingID()); - reportedProblem = true; - } - } - - else if (state.reading > 0) - { - if (!reportedProblem) - { - //Message msg(ERR_RECOVERABLE_LOCK_STATE); - Message msg(M0094); - Message::Args args; - - args.add(state.reading); - args.add(state.readerswaiting); - args.add(state.writing); - args.add(state.writerswaiting); - msg.format(args); - logger.logMessage(LOG_TYPE_WARNING, msg, LoggingID()); - reportedProblem = true; - } - - for (int i = 0; i < state.reading; i++) - lock->read_unlock(); - } - - // the write lock is held but not by this process, not good. - // there's a slight race here between these two vars but it's miniscule, - // and the worst thing that happens is a false positive error msg. - else if (state.writing > 0 && !(*lockStatus)) - { - if (!reportedProblem) - { - //Message msg(ERR_UNRECOVERABLE_LOCK_STATE); - Message msg(M0093); - Message::Args args; - - args.add(state.reading); - args.add(state.readerswaiting); - args.add(state.writing); - args.add(state.writerswaiting); - msg.format(args); - logger.logMessage(LOG_TYPE_CRITICAL, msg, LoggingID()); - reportedProblem = true; - } - - /* put write lock recovery code here */ - } - else - { - // the workernode is legitmately taking a long time - //cout << "holds the lock. " << " r=" << state.reading << " rwt=" << state.readerswaiting << - // " w=" << state.writing << " wwt=" << state.writerswaiting << endl; - } + // Message msg(ERR_BRM_MUTEX); + Message msg(M0092); + logger.logMessage(LOG_TYPE_CRITICAL, msg, LoggingID()); + reportedProblem = true; } - else + } + + else if (state.reading > 0) + { + if (!reportedProblem) { - /* got the write lock. If there was a problem before it's been fixed. */ - lock->write_unlock(); + // Message msg(ERR_RECOVERABLE_LOCK_STATE); + Message msg(M0094); + Message::Args args; - if (reportedProblem) - { - //Message msg(ERR_SUCCESSFUL_RECOVERY); - Message msg(M0095); - logger.logMessage(LOG_TYPE_WARNING, msg, LoggingID()); - reportedProblem = false; - } - - sleep(secsBetweenAttempts); + args.add(state.reading); + args.add(state.readerswaiting); + args.add(state.writing); + args.add(state.writerswaiting); + msg.format(args); + logger.logMessage(LOG_TYPE_WARNING, msg, LoggingID()); + reportedProblem = true; } + + for (int i = 0; i < state.reading; i++) + lock->read_unlock(); + } + + // the write lock is held but not by this process, not good. + // there's a slight race here between these two vars but it's miniscule, + // and the worst thing that happens is a false positive error msg. + else if (state.writing > 0 && !(*lockStatus)) + { + if (!reportedProblem) + { + // Message msg(ERR_UNRECOVERABLE_LOCK_STATE); + Message msg(M0093); + Message::Args args; + + args.add(state.reading); + args.add(state.readerswaiting); + args.add(state.writing); + args.add(state.writerswaiting); + msg.format(args); + logger.logMessage(LOG_TYPE_CRITICAL, msg, LoggingID()); + reportedProblem = true; + } + + /* put write lock recovery code here */ + } + else + { + // the workernode is legitmately taking a long time + // cout << "holds the lock. " << " r=" << state.reading << " rwt=" << state.readerswaiting << + // " w=" << state.writing << " wwt=" << state.writerswaiting << endl; + } } + else + { + /* got the write lock. If there was a problem before it's been fixed. */ + lock->write_unlock(); + + if (reportedProblem) + { + // Message msg(ERR_SUCCESSFUL_RECOVERY); + Message msg(M0095); + logger.logMessage(LOG_TYPE_WARNING, msg, LoggingID()); + reportedProblem = false; + } + + sleep(secsBetweenAttempts); + } + } } } /* namespace BRM */ diff --git a/versioning/BRM/rwlockmonitor.h b/versioning/BRM/rwlockmonitor.h index 93e2a6c8a..8ba3151d6 100644 --- a/versioning/BRM/rwlockmonitor.h +++ b/versioning/BRM/rwlockmonitor.h @@ -38,30 +38,29 @@ namespace BRM { - class RWLockMonitor { -public: - // d = die, ls = lock status, k = key - EXPORT RWLockMonitor(const bool* d, const bool* ls, const uint32_t k); + public: + // d = die, ls = lock status, k = key + EXPORT RWLockMonitor(const bool* d, const bool* ls, const uint32_t k); - EXPORT virtual ~RWLockMonitor(); + EXPORT virtual ~RWLockMonitor(); - EXPORT void operator()(); + EXPORT void operator()(); -private: - //Are these defaults okay? - //RWLockMonitor(const RWLockMonitor&rhs); - //RWLockMonitor& operator=(const RWLockMonitor&rhs); + private: + // Are these defaults okay? + // RWLockMonitor(const RWLockMonitor&rhs); + // RWLockMonitor& operator=(const RWLockMonitor&rhs); - /* Some of these vars are only useful once we implement write_lock checking. */ - const bool* die; - const bool* lockStatus; - uint32_t key; - boost::shared_ptr lock; + /* Some of these vars are only useful once we implement write_lock checking. */ + const bool* die; + const bool* lockStatus; + uint32_t key; + boost::shared_ptr lock; - struct timespec ts; // 3:30 timer - uint32_t secsBetweenAttempts; // :30 + struct timespec ts; // 3:30 timer + uint32_t secsBetweenAttempts; // :30 }; } /* namespace BRM */ @@ -69,4 +68,3 @@ private: #undef EXPORT #endif /* RWLOCKMONITOR_H_ */ - diff --git a/versioning/BRM/save_brm.cpp b/versioning/BRM/save_brm.cpp index 5eef1a9bf..aae9e94fd 100644 --- a/versioning/BRM/save_brm.cpp +++ b/versioning/BRM/save_brm.cpp @@ -26,7 +26,6 @@ * More detailed description */ - #include #include #include @@ -48,79 +47,76 @@ using namespace BRM; #include "configcpp.h" -int main (int argc, char** argv) +int main(int argc, char** argv) { - BlockResolutionManager brm; - config::Config* config = config::Config::makeConfig(); - int err; - string prefix, currentFilename; - IDBDataFile* currentFile = NULL; + BlockResolutionManager brm; + config::Config* config = config::Config::makeConfig(); + int err; + string prefix, currentFilename; + IDBDataFile* currentFile = NULL; - if (argc > 1) - prefix = argv[1]; - else + if (argc > 1) + prefix = argv[1]; + else + { + prefix = config->getConfig("SystemConfig", "DBRMRoot"); + + if (prefix.length() == 0) { - prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - cerr << "Error: Need a valid Calpont configuation file" << endl; - exit(1); - } + cerr << "Error: Need a valid Calpont configuation file" << endl; + exit(1); } + } - idbdatafile::IDBPolicy::configIDBPolicy(); + idbdatafile::IDBPolicy::configIDBPolicy(); - err = brm.saveState(prefix); + err = brm.saveState(prefix); - if (err == 0) - cout << "Saved to " << prefix << endl; - else - { - cout << "Save failed" << endl; - exit(1); - } + if (err == 0) + cout << "Saved to " << prefix << endl; + else + { + cout << "Save failed" << endl; + exit(1); + } - (void)::umask(0); + (void)::umask(0); - currentFilename = prefix + "_current"; - currentFile = IDBDataFile::open(IDBPolicy::getType(currentFilename.c_str(), - IDBPolicy::WRITEENG), - currentFilename.c_str(), - "wb", - 0); + currentFilename = prefix + "_current"; + currentFile = IDBDataFile::open(IDBPolicy::getType(currentFilename.c_str(), IDBPolicy::WRITEENG), + currentFilename.c_str(), "wb", 0); - if (!currentFile) - { - cerr << "Error: could not open " << currentFilename << "for writing" << endl; - exit(1); - } + if (!currentFile) + { + cerr << "Error: could not open " << currentFilename << "for writing" << endl; + exit(1); + } - try - { + try + { #ifndef _MSC_VER - prefix += '\n'; + prefix += '\n'; #endif - // for MCOL-1558. Make the _current file relative to DBRMRoot - string relative = prefix.substr(prefix.find_last_of('/') + 1); - currentFile->write(relative.c_str(), relative.length()); - } - catch (exception& e) - { - cerr << "Error: failed to write to " << currentFilename << ": " << e.what() << endl; - exit(1); - } + // for MCOL-1558. Make the _current file relative to DBRMRoot + string relative = prefix.substr(prefix.find_last_of('/') + 1); + currentFile->write(relative.c_str(), relative.length()); + } + catch (exception& e) + { + cerr << "Error: failed to write to " << currentFilename << ": " << e.what() << endl; + exit(1); + } - try - { - delete currentFile; - currentFile = NULL; - } - catch (exception& e) - { - cerr << "Error: failed to close " << currentFilename << ": " << e.what() << endl; - exit(1); - } + try + { + delete currentFile; + currentFile = NULL; + } + catch (exception& e) + { + cerr << "Error: failed to close " << currentFilename << ": " << e.what() << endl; + exit(1); + } - return 0; + return 0; } diff --git a/versioning/BRM/sessionmanagerserver.cpp b/versioning/BRM/sessionmanagerserver.cpp index 55645615f..1e7cc0d15 100644 --- a/versioning/BRM/sessionmanagerserver.cpp +++ b/versioning/BRM/sessionmanagerserver.cpp @@ -56,82 +56,85 @@ using namespace execplan; #undef SESSIONMANAGERSERVER_DLLEXPORT #ifndef O_BINARY -# define O_BINARY 0 +#define O_BINARY 0 #endif #ifndef O_DIRECT -# define O_DIRECT 0 +#define O_DIRECT 0 #endif #ifndef O_LARGEFILE -# define O_LARGEFILE 0 +#define O_LARGEFILE 0 #endif #ifndef O_NOATIME -# define O_NOATIME 0 +#define O_NOATIME 0 #endif #include "IDBDataFile.h" #include "IDBPolicy.h" using namespace idbdatafile; - namespace BRM { - -const uint32_t SessionManagerServer::SS_READY = 1 << 0; // Set by dmlProc one time when dmlProc is ready -const uint32_t SessionManagerServer::SS_SUSPENDED = 1 << 1; // Set by console when the system has been suspended by user. -const uint32_t SessionManagerServer::SS_SUSPEND_PENDING = 1 << 2; // Set by console when user wants to suspend, but writing is occuring. -const uint32_t SessionManagerServer::SS_SHUTDOWN_PENDING = 1 << 3; // Set by console when user wants to shutdown, but writing is occuring. -const uint32_t SessionManagerServer::SS_ROLLBACK = 1 << 4; // In combination with a PENDING flag, force a rollback as soom as possible. -const uint32_t SessionManagerServer::SS_FORCE = 1 << 5; // In combination with a PENDING flag, force a shutdown without rollback. -const uint32_t SessionManagerServer::SS_QUERY_READY = 1 << 6; // Set by ProcManager when system is ready for queries - +const uint32_t SessionManagerServer::SS_READY = 1 << 0; // Set by dmlProc one time when dmlProc is ready +const uint32_t SessionManagerServer::SS_SUSPENDED = + 1 << 1; // Set by console when the system has been suspended by user. +const uint32_t SessionManagerServer::SS_SUSPEND_PENDING = + 1 << 2; // Set by console when user wants to suspend, but writing is occuring. +const uint32_t SessionManagerServer::SS_SHUTDOWN_PENDING = + 1 << 3; // Set by console when user wants to shutdown, but writing is occuring. +const uint32_t SessionManagerServer::SS_ROLLBACK = + 1 << 4; // In combination with a PENDING flag, force a rollback as soom as possible. +const uint32_t SessionManagerServer::SS_FORCE = + 1 << 5; // In combination with a PENDING flag, force a shutdown without rollback. +const uint32_t SessionManagerServer::SS_QUERY_READY = + 1 << 6; // Set by ProcManager when system is ready for queries SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0) { - config::Config* conf; - string stmp; - const char* ctmp; + config::Config* conf; + string stmp; + const char* ctmp; - conf = config::Config::makeConfig(); + conf = config::Config::makeConfig(); - try - { - stmp = conf->getConfig("SessionManager", "MaxConcurrentTransactions"); - } - catch (const std::exception& e) - { - cout << e.what() << endl; - stmp.clear(); - } + try + { + stmp = conf->getConfig("SessionManager", "MaxConcurrentTransactions"); + } + catch (const std::exception& e) + { + cout << e.what() << endl; + stmp.clear(); + } - if (stmp != "") - { - int64_t tmp; - ctmp = stmp.c_str(); - tmp = config::Config::fromText(ctmp); + if (stmp != "") + { + int64_t tmp; + ctmp = stmp.c_str(); + tmp = config::Config::fromText(ctmp); - if (tmp < 1) - maxTxns = 1; - else - maxTxns = static_cast(tmp); - } + if (tmp < 1) + maxTxns = 1; else - maxTxns = 1; + maxTxns = static_cast(tmp); + } + else + maxTxns = 1; - txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); + txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); - semValue = maxTxns; - _verID = 0; - _sysCatVerID = 0; - systemState = 0; + semValue = maxTxns; + _verID = 0; + _sysCatVerID = 0; + systemState = 0; - try - { - loadState(); - } - catch (...) - { - // first-time run most likely, ignore the error - } + try + { + loadState(); + } + catch (...) + { + // first-time run most likely, ignore the error + } } SessionManagerServer::~SessionManagerServer() @@ -139,295 +142,293 @@ SessionManagerServer::~SessionManagerServer() } void SessionManagerServer::reset() { - mutex.try_lock(); - semValue = maxTxns; - condvar.notify_all(); - activeTxns.clear(); - mutex.unlock(); + mutex.try_lock(); + semValue = maxTxns; + condvar.notify_all(); + activeTxns.clear(); + mutex.unlock(); } void SessionManagerServer::loadState() { - int lastTxnID; - int err; - int lastSysCatVerId; + int lastTxnID; + int err; + int lastSysCatVerId; again: - // There are now 3 pieces of info stored in the txnidfd file: last - // transaction id, last system catalog version id, and the - // system state flags. All these values are stored in shared, an - // instance of struct Overlay. - // If we fail to read a full four bytes for any value, then the - // value isn't in the file, and we start with the default. + // There are now 3 pieces of info stored in the txnidfd file: last + // transaction id, last system catalog version id, and the + // system state flags. All these values are stored in shared, an + // instance of struct Overlay. + // If we fail to read a full four bytes for any value, then the + // value isn't in the file, and we start with the default. - if (IDBPolicy::exists(txnidFilename.c_str())) + if (IDBPolicy::exists(txnidFilename.c_str())) + { + scoped_ptr txnidfp(IDBDataFile::open( + IDBPolicy::getType(txnidFilename.c_str(), IDBPolicy::WRITEENG), txnidFilename.c_str(), "rb", 0)); + + if (!txnidfp) { - scoped_ptr txnidfp(IDBDataFile::open( - IDBPolicy::getType(txnidFilename.c_str(), - IDBPolicy::WRITEENG), - txnidFilename.c_str(), "rb", 0)); - - if (!txnidfp) - { - perror("SessionManagerServer(): open"); - throw runtime_error("SessionManagerServer: Could not open the transaction ID file"); - } - - // Last transaction id - txnidfp->seek(0, SEEK_SET); - err = txnidfp->read(&lastTxnID, 4); - - if (err < 0 && errno != EINTR) - { - perror("Sessionmanager::initSegment(): read"); - throw runtime_error("SessionManagerServer: read failed, aborting"); - } - else if (err < 0) - goto again; - else if (err == sizeof(int)) - _verID = lastTxnID; - - // last system catalog version id - err = txnidfp->read(&lastSysCatVerId, 4); - - if (err < 0 && errno != EINTR) - { - perror("Sessionmanager::initSegment(): read"); - throw runtime_error("SessionManagerServer: read failed, aborting"); - } - else if (err < 0) - goto again; - else if (err == sizeof(int)) - _sysCatVerID = lastSysCatVerId; - - // System state. Contains flags regarding the suspend state of the system. - err = txnidfp->read(&systemState, 4); - - if (err < 0 && errno == EINTR) - { - goto again; - } - else if (err == sizeof(int)) - { - // Turn off the pending and force flags. They make no sense for a clean start. - // Turn off the ready flag. DMLProc will set it back on when - // initialized. - systemState &= - ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_ROLLBACK | SS_FORCE); - } - else - { - // else no problem. System state wasn't saved. Might be an upgraded system. - systemState = 0; - } + perror("SessionManagerServer(): open"); + throw runtime_error("SessionManagerServer: Could not open the transaction ID file"); } + + // Last transaction id + txnidfp->seek(0, SEEK_SET); + err = txnidfp->read(&lastTxnID, 4); + + if (err < 0 && errno != EINTR) + { + perror("Sessionmanager::initSegment(): read"); + throw runtime_error("SessionManagerServer: read failed, aborting"); + } + else if (err < 0) + goto again; + else if (err == sizeof(int)) + _verID = lastTxnID; + + // last system catalog version id + err = txnidfp->read(&lastSysCatVerId, 4); + + if (err < 0 && errno != EINTR) + { + perror("Sessionmanager::initSegment(): read"); + throw runtime_error("SessionManagerServer: read failed, aborting"); + } + else if (err < 0) + goto again; + else if (err == sizeof(int)) + _sysCatVerID = lastSysCatVerId; + + // System state. Contains flags regarding the suspend state of the system. + err = txnidfp->read(&systemState, 4); + + if (err < 0 && errno == EINTR) + { + goto again; + } + else if (err == sizeof(int)) + { + // Turn off the pending and force flags. They make no sense for a clean start. + // Turn off the ready flag. DMLProc will set it back on when + // initialized. + systemState &= + ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_ROLLBACK | SS_FORCE); + } + else + { + // else no problem. System state wasn't saved. Might be an upgraded system. + systemState = 0; + } + } } /* Save the systemState flags of the Overlay * segment. This is saved in the third * word of txnid File -*/ + */ void SessionManagerServer::saveSystemState() { - saveSMTxnIDAndState(); + saveSMTxnIDAndState(); } const QueryContext SessionManagerServer::verID() { - QueryContext ret; + QueryContext ret; - boost::mutex::scoped_lock lk(mutex); - ret.currentScn = _verID; + boost::mutex::scoped_lock lk(mutex); + ret.currentScn = _verID; - for (iterator i = activeTxns.begin(); i != activeTxns.end(); ++i) - ret.currentTxns->push_back(i->second); + for (iterator i = activeTxns.begin(); i != activeTxns.end(); ++i) + ret.currentTxns->push_back(i->second); - return ret; + return ret; } const QueryContext SessionManagerServer::sysCatVerID() { - QueryContext ret; + QueryContext ret; - boost::mutex::scoped_lock lk(mutex); - ret.currentScn = _sysCatVerID; + boost::mutex::scoped_lock lk(mutex); + ret.currentScn = _sysCatVerID; - for (iterator i = activeTxns.begin(); i != activeTxns.end(); ++i) - ret.currentTxns->push_back(i->second); + for (iterator i = activeTxns.begin(); i != activeTxns.end(); ++i) + ret.currentTxns->push_back(i->second); - return ret; + return ret; } const TxnID SessionManagerServer::newTxnID(const SID session, bool block, bool isDDL) { - TxnID ret; //ctor must set valid = false - iterator it; + TxnID ret; // ctor must set valid = false + iterator it; - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - // if it already has a txn... - it = activeTxns.find(session); + // if it already has a txn... + it = activeTxns.find(session); - if (it != activeTxns.end()) - { - ret.id = it->second; - ret.valid = true; - return ret; - } - - if (!block && semValue == 0) - return ret; - else while (semValue == 0) - condvar.wait(lk); - - semValue--; - idbassert(semValue <= (uint32_t)maxTxns); - - ret.id = ++_verID; + if (it != activeTxns.end()) + { + ret.id = it->second; ret.valid = true; - activeTxns[session] = ret.id; - - if (isDDL) - ++_sysCatVerID; - - saveSMTxnIDAndState(); - return ret; + } + + if (!block && semValue == 0) + return ret; + else + while (semValue == 0) + condvar.wait(lk); + + semValue--; + idbassert(semValue <= (uint32_t)maxTxns); + + ret.id = ++_verID; + ret.valid = true; + activeTxns[session] = ret.id; + + if (isDDL) + ++_sysCatVerID; + + saveSMTxnIDAndState(); + + return ret; } void SessionManagerServer::finishTransaction(TxnID& txn) { - iterator it; - boost::mutex::scoped_lock lk(mutex); - bool found = false; + iterator it; + boost::mutex::scoped_lock lk(mutex); + bool found = false; - if (!txn.valid) - throw invalid_argument("SessionManagerServer::finishTransaction(): transaction is invalid"); + if (!txn.valid) + throw invalid_argument("SessionManagerServer::finishTransaction(): transaction is invalid"); - for (it = activeTxns.begin(); it != activeTxns.end(); ) + for (it = activeTxns.begin(); it != activeTxns.end();) + { + if (it->second == txn.id) { - if (it->second == txn.id) - { - activeTxns.erase(it++); - txn.valid = false; - found = true; - //we could probably break at this point, but there won't be that many active txns, and, - // even though it'd be an error to have multiple entries for the same txn, we might - // well just get rid of them... - } - else - ++it; - } - - if (found) - { - semValue++; - idbassert(semValue <= (uint32_t)maxTxns); - condvar.notify_one(); + activeTxns.erase(it++); + txn.valid = false; + found = true; + // we could probably break at this point, but there won't be that many active txns, and, + // even though it'd be an error to have multiple entries for the same txn, we might + // well just get rid of them... } else - throw invalid_argument("SessionManagerServer::finishTransaction(): transaction doesn't exist"); + ++it; + } + + if (found) + { + semValue++; + idbassert(semValue <= (uint32_t)maxTxns); + condvar.notify_one(); + } + else + throw invalid_argument("SessionManagerServer::finishTransaction(): transaction doesn't exist"); } const TxnID SessionManagerServer::getTxnID(const SID session) { - TxnID ret; - iterator it; + TxnID ret; + iterator it; - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - it = activeTxns.find(session); + it = activeTxns.find(session); - if (it != activeTxns.end()) - { - ret.id = it->second; - ret.valid = true; - } + if (it != activeTxns.end()) + { + ret.id = it->second; + ret.valid = true; + } - return ret; + return ret; } shared_array SessionManagerServer::SIDTIDMap(int& len) { - int j; - shared_array ret; - boost::mutex::scoped_lock lk(mutex); - iterator it; + int j; + shared_array ret; + boost::mutex::scoped_lock lk(mutex); + iterator it; - ret.reset(new SIDTIDEntry[activeTxns.size()]); + ret.reset(new SIDTIDEntry[activeTxns.size()]); - len = activeTxns.size(); + len = activeTxns.size(); - for (it = activeTxns.begin(), j = 0; it != activeTxns.end(); ++it, ++j) - { - ret[j].sessionid = it->first; - ret[j].txnid.id = it->second; - ret[j].txnid.valid = true; - } + for (it = activeTxns.begin(), j = 0; it != activeTxns.end(); ++it, ++j) + { + ret[j].sessionid = it->first; + ret[j].txnid.id = it->second; + ret[j].txnid.valid = true; + } - return ret; + return ret; } void SessionManagerServer::setSystemState(uint32_t state) { - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - systemState |= state; - saveSystemState(); + systemState |= state; + saveSystemState(); } void SessionManagerServer::clearSystemState(uint32_t state) { - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); - systemState &= ~state; - saveSystemState(); + systemState &= ~state; + saveSystemState(); } uint32_t SessionManagerServer::getTxnCount() { - boost::mutex::scoped_lock lk(mutex); - return activeTxns.size(); + boost::mutex::scoped_lock lk(mutex); + return activeTxns.size(); } void SessionManagerServer::saveSMTxnIDAndState() { - // caller holds the lock - scoped_ptr txnidfp(IDBDataFile::open( - IDBPolicy::getType(txnidFilename.c_str(), IDBPolicy::WRITEENG), - txnidFilename.c_str(), "wb", 0)); + // caller holds the lock + scoped_ptr txnidfp(IDBDataFile::open( + IDBPolicy::getType(txnidFilename.c_str(), IDBPolicy::WRITEENG), txnidFilename.c_str(), "wb", 0)); - if (!txnidfp) - { - perror("SessionManagerServer(): open"); - throw runtime_error("SessionManagerServer: Could not open the transaction ID file"); - } + if (!txnidfp) + { + perror("SessionManagerServer(): open"); + throw runtime_error("SessionManagerServer: Could not open the transaction ID file"); + } - int filedata[2]; - filedata[0] = _verID; - filedata[1] = _sysCatVerID; + int filedata[2]; + filedata[0] = _verID; + filedata[1] = _sysCatVerID; - int err = txnidfp->write(filedata, 8); + int err = txnidfp->write(filedata, 8); - if (err < 0) - { - perror("SessionManagerServer::newTxnID(): write(verid)"); - throw runtime_error("SessionManagerServer::newTxnID(): write(verid) failed"); - } + if (err < 0) + { + perror("SessionManagerServer::newTxnID(): write(verid)"); + throw runtime_error("SessionManagerServer::newTxnID(): write(verid) failed"); + } - uint32_t lSystemState = systemState; - // We don't save the pending flags, the force flag or the ready flags. - lSystemState &= ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_FORCE); - err = txnidfp->write(&lSystemState, sizeof(int)); + uint32_t lSystemState = systemState; + // We don't save the pending flags, the force flag or the ready flags. + lSystemState &= ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_FORCE); + err = txnidfp->write(&lSystemState, sizeof(int)); - if (err < 0) - { - perror("SessionManagerServer::saveSystemState(): write(systemState)"); - throw runtime_error("SessionManagerServer::saveSystemState(): write(systemState) failed"); - } + if (err < 0) + { + perror("SessionManagerServer::saveSystemState(): write(systemState)"); + throw runtime_error("SessionManagerServer::saveSystemState(): write(systemState) failed"); + } - txnidfp->flush(); + txnidfp->flush(); } -} //namespace +} // namespace BRM diff --git a/versioning/BRM/sessionmanagerserver.h b/versioning/BRM/sessionmanagerserver.h index e1782e3e8..918491104 100644 --- a/versioning/BRM/sessionmanagerserver.h +++ b/versioning/BRM/sessionmanagerserver.h @@ -45,17 +45,16 @@ #endif #ifdef _MSC_VER -#pragma warning (push) -#pragma warning (disable : 4200) +#pragma warning(push) +#pragma warning(disable : 4200) #endif namespace BRM { - /** @brief Issues transaction IDs and keeps track of the current system-wide version ID. * - * This class's primary purpose is to keep track of the current system-wide version ID and issue transaction IDs. - * It can be used simultaneously by multiple threads and processes. + * This class's primary purpose is to keep track of the current system-wide version ID and issue transaction + * IDs. It can be used simultaneously by multiple threads and processes. * * It uses system-wide semaphores and shared memory segments for IPC. * It allocates both if they don't already exist, but it only deallocates @@ -92,203 +91,207 @@ namespace BRM class SessionManagerServer { -public: - /** @brief SID = Session ID */ - typedef uint32_t SID; + public: + /** @brief SID = Session ID */ + typedef uint32_t SID; - // State flags. These bit values are stored in Overlay::state and reflect the current state of the system - static const uint32_t SS_READY; /// bit 0 => Set by dmlProc one time when dmlProc is ready - static const uint32_t SS_SUSPENDED; /// bit 1 => Set by console when the system has been suspended by user. - static const uint32_t SS_SUSPEND_PENDING; /// bit 2 => Set by console when user wants to suspend, but writing is occuring. - static const uint32_t SS_SHUTDOWN_PENDING; /// bit 3 => Set by console when user wants to shutdown, but writing is occuring. - static const uint32_t SS_ROLLBACK; /// bit 4 => In combination with a PENDING flag, force a rollback as soom as possible. - static const uint32_t SS_FORCE; /// bit 5 => In combination with a PENDING flag, force a shutdown without rollback. - static const uint32_t SS_QUERY_READY; /// bit 6 => Set by ProcManager when system is ready for queries + // State flags. These bit values are stored in Overlay::state and reflect the current state of the system + static const uint32_t SS_READY; /// bit 0 => Set by dmlProc one time when dmlProc is ready + static const uint32_t SS_SUSPENDED; /// bit 1 => Set by console when the system has been suspended by user. + static const uint32_t + SS_SUSPEND_PENDING; /// bit 2 => Set by console when user wants to suspend, but writing is occuring. + static const uint32_t + SS_SHUTDOWN_PENDING; /// bit 3 => Set by console when user wants to shutdown, but writing is occuring. + static const uint32_t + SS_ROLLBACK; /// bit 4 => In combination with a PENDING flag, force a rollback as soom as possible. + static const uint32_t + SS_FORCE; /// bit 5 => In combination with a PENDING flag, force a shutdown without rollback. + static const uint32_t SS_QUERY_READY; /// bit 6 => Set by ProcManager when system is ready for queries - /** @brief Constructor - * - * This sets up the shared memory segment & semaphores used by this - * instance. No additional set-up calls are necessary. - * @note throws ios_base::failure on file IO error, runtime_error for - * other types of errors. It might be worthwhile to define additional - * exceptions for all the different types of failures that can happen - * here. - */ - EXPORT SessionManagerServer(); + /** @brief Constructor + * + * This sets up the shared memory segment & semaphores used by this + * instance. No additional set-up calls are necessary. + * @note throws ios_base::failure on file IO error, runtime_error for + * other types of errors. It might be worthwhile to define additional + * exceptions for all the different types of failures that can happen + * here. + */ + EXPORT SessionManagerServer(); - /** @brief Destructor - * - * This detaches the shared memory segment. If DESTROYSHMSEG is defined and this - * is the last reference to it, it will be saved to disk and destroyed. - * It does not destroy the semaphores. Those persist until the system - * is shut down. - */ - virtual ~SessionManagerServer(); + /** @brief Destructor + * + * This detaches the shared memory segment. If DESTROYSHMSEG is defined and this + * is the last reference to it, it will be saved to disk and destroyed. + * It does not destroy the semaphores. Those persist until the system + * is shut down. + */ + virtual ~SessionManagerServer(); - /** @brief Gets the current version ID - * - * Gets the current version ID. - */ - EXPORT const QueryContext verID(); + /** @brief Gets the current version ID + * + * Gets the current version ID. + */ + EXPORT const QueryContext verID(); - /** @brief Gets the current version ID - * - * Gets the current version ID. - */ - EXPORT const QueryContext sysCatVerID(); + /** @brief Gets the current version ID + * + * Gets the current version ID. + */ + EXPORT const QueryContext sysCatVerID(); - /** @brief Gets a new Transaction ID - * - * Makes a new Transaction ID, associates it with the given session - * and increments the system-wide version ID - * @note This blocks until (# active transactions) \< MaxTxns unless block == false - * @note Throws runtime_error on semaphore-related error - * @note This will always return a valid TxnID unless block == false, in which case - * it will return a valid TxnID iff it doesn't have to block. - * @param session The session ID to associate with the transaction ID - * @param block If true, this will block until there is an available slot, otherwise - * it will return a TxnID marked invalid, signifying that it could not start a new transaction - * @return The new transaction ID. - */ - EXPORT const TxnID newTxnID(const SID session, bool block = true, bool isDDL = false); + /** @brief Gets a new Transaction ID + * + * Makes a new Transaction ID, associates it with the given session + * and increments the system-wide version ID + * @note This blocks until (# active transactions) \< MaxTxns unless block == false + * @note Throws runtime_error on semaphore-related error + * @note This will always return a valid TxnID unless block == false, in which case + * it will return a valid TxnID iff it doesn't have to block. + * @param session The session ID to associate with the transaction ID + * @param block If true, this will block until there is an available slot, otherwise + * it will return a TxnID marked invalid, signifying that it could not start a new transaction + * @return The new transaction ID. + */ + EXPORT const TxnID newTxnID(const SID session, bool block = true, bool isDDL = false); - /** @brief Record that a transaction has been committed - * - * Record that a transaction has been committed. - * @note Throws runtime_error on a semaphore-related error, invalid_argument - * when txnid can't be found - * @param txnid The committed transaction ID. This is marked invalid - * on return. - */ - void committed(TxnID& txn) - { - finishTransaction(txn); - } + /** @brief Record that a transaction has been committed + * + * Record that a transaction has been committed. + * @note Throws runtime_error on a semaphore-related error, invalid_argument + * when txnid can't be found + * @param txnid The committed transaction ID. This is marked invalid + * on return. + */ + void committed(TxnID& txn) + { + finishTransaction(txn); + } - /** @brief Record that a transaction has been rolled back - * - * Record that a transaction has been rolled back. - * @note Throws runtime_error on a semaphore-related error, invalid_argument - * when txnid can't be found - * @param txnid The rolled back transaction ID. This is marked invalid - * on return. - */ - void rolledback(TxnID& txn) - { - finishTransaction(txn); - } + /** @brief Record that a transaction has been rolled back + * + * Record that a transaction has been rolled back. + * @note Throws runtime_error on a semaphore-related error, invalid_argument + * when txnid can't be found + * @param txnid The rolled back transaction ID. This is marked invalid + * on return. + */ + void rolledback(TxnID& txn) + { + finishTransaction(txn); + } - /** @brief Gets the transaction ID associated with a given session ID - * - * Gets the transaction ID associated with a given session ID. - * @note Throws runtime_error on a semaphore-related error - * @param session The session ID - * @return A valid transaction ID if there's an association; an invalid - * one if there isn't. - */ - EXPORT const TxnID getTxnID(const SID session); + /** @brief Gets the transaction ID associated with a given session ID + * + * Gets the transaction ID associated with a given session ID. + * @note Throws runtime_error on a semaphore-related error + * @param session The session ID + * @return A valid transaction ID if there's an association; an invalid + * one if there isn't. + */ + EXPORT const TxnID getTxnID(const SID session); - /** @brief Gets an array containing all active SID-TID associations - * - * Gets an array containing the SID-TID associations of all active - * transactions. This is intended to be used by another object - * that can determine which sessions are still live and which - * transactions need to go away. - * @note Throws runtime_error on a semaphore-related error - * @param len (out) the length of the array - * @return A pointer to the array. Note: The caller is responsible for - * deallocating it. Use delete[]. - */ - EXPORT boost::shared_array SIDTIDMap(int& len); + /** @brief Gets an array containing all active SID-TID associations + * + * Gets an array containing the SID-TID associations of all active + * transactions. This is intended to be used by another object + * that can determine which sessions are still live and which + * transactions need to go away. + * @note Throws runtime_error on a semaphore-related error + * @param len (out) the length of the array + * @return A pointer to the array. Note: The caller is responsible for + * deallocating it. Use delete[]. + */ + EXPORT boost::shared_array SIDTIDMap(int& len); - /** - * get a unique 32-bit number - */ - uint32_t getUnique32() - { - return atomicops::atomicInc(&unique32); - } + /** + * get a unique 32-bit number + */ + uint32_t getUnique32() + { + return atomicops::atomicInc(&unique32); + } - /** - * get a unique 64-bit number - */ - uint64_t getUnique64() - { - return atomicops::atomicInc(&unique64); - } + /** + * get a unique 64-bit number + */ + uint64_t getUnique64() + { + return atomicops::atomicInc(&unique64); + } - /** @brief Resets the semaphores to their original state. For testing only. - * - * Resets the semaphores to their original state. For testing only. - */ - EXPORT void reset(); + /** @brief Resets the semaphores to their original state. For testing only. + * + * Resets the semaphores to their original state. For testing only. + */ + EXPORT void reset(); - /** - * get the Txn ID filename - */ - std::string getTxnIDFilename() const - { - return txnidFilename; - } + /** + * get the Txn ID filename + */ + std::string getTxnIDFilename() const + { + return txnidFilename; + } - /** @brief set system state info - * - * Sets the bits on in Overlay::systemState that are found on in - * state. That is Overlay::systemState | state. - */ - EXPORT void setSystemState(uint32_t state); + /** @brief set system state info + * + * Sets the bits on in Overlay::systemState that are found on in + * state. That is Overlay::systemState | state. + */ + EXPORT void setSystemState(uint32_t state); - /** @brief set system state info - * - * Clears the bits on in Overlay::systemState that are found on - * in state. That is Overlay::systemState & ~state. - */ - EXPORT void clearSystemState(uint32_t state); + /** @brief set system state info + * + * Clears the bits on in Overlay::systemState that are found on + * in state. That is Overlay::systemState & ~state. + */ + EXPORT void clearSystemState(uint32_t state); - /** @brief get system state info - * - * Returns the Overlay::systemState flags - */ - void getSystemState(uint32_t& state) - { - state = systemState; - } + /** @brief get system state info + * + * Returns the Overlay::systemState flags + */ + void getSystemState(uint32_t& state) + { + state = systemState; + } - /** - * get the number of active txn's - */ - EXPORT uint32_t getTxnCount(); + /** + * get the number of active txn's + */ + EXPORT uint32_t getTxnCount(); -private: - SessionManagerServer(const SessionManagerServer&); - SessionManagerServer& operator=(const SessionManagerServer&); + private: + SessionManagerServer(const SessionManagerServer&); + SessionManagerServer& operator=(const SessionManagerServer&); - void loadState(); - void saveSystemState(); - void finishTransaction(TxnID& txn); - void saveSMTxnIDAndState(); + void loadState(); + void saveSystemState(); + void finishTransaction(TxnID& txn); + void saveSMTxnIDAndState(); - volatile uint32_t unique32; - volatile uint64_t unique64; + volatile uint32_t unique32; + volatile uint64_t unique64; - int maxTxns; // the maximum number of concurrent transactions - std::string txnidFilename; - execplan::CalpontSystemCatalog::SCN _verID; - execplan::CalpontSystemCatalog::SCN _sysCatVerID; - uint32_t systemState; + int maxTxns; // the maximum number of concurrent transactions + std::string txnidFilename; + execplan::CalpontSystemCatalog::SCN _verID; + execplan::CalpontSystemCatalog::SCN _sysCatVerID; + uint32_t systemState; - std::map activeTxns; - typedef std::map::iterator iterator; + std::map activeTxns; + typedef std::map::iterator iterator; - boost::mutex mutex; - boost::condition_variable condvar; // used to synthesize a semaphore - uint32_t semValue; + boost::mutex mutex; + boost::condition_variable condvar; // used to synthesize a semaphore + uint32_t semValue; }; -} //namespace +} // namespace BRM #ifdef _MSC_VER -#pragma warning (pop) +#pragma warning(pop) #endif #undef EXPORT diff --git a/versioning/BRM/shmkeys.cpp b/versioning/BRM/shmkeys.cpp index 6abe149ac..f7642ff9f 100644 --- a/versioning/BRM/shmkeys.cpp +++ b/versioning/BRM/shmkeys.cpp @@ -32,41 +32,39 @@ using namespace std; namespace BRM { - ShmKeys::ShmKeys() { - uint32_t BRM_UID = 0x0; + uint32_t BRM_UID = 0x0; #ifdef WANT_EM_BRM_UID - config::Config* cf = config::Config::makeConfig(); - string brm_str = cf->getConfig("ExtentMap", "BRM_UID"); + config::Config* cf = config::Config::makeConfig(); + string brm_str = cf->getConfig("ExtentMap", "BRM_UID"); - if (brm_str.length() > 0) - BRM_UID = static_cast(config::Config::uFromText(brm_str)); + if (brm_str.length() > 0) + BRM_UID = static_cast(config::Config::uFromText(brm_str)); #endif - KEYRANGE_VSS_BASE = 0x10000 | (BRM_UID << 20); - KEYRANGE_EXTENTMAP_BASE = 0x20000 | (BRM_UID << 20); - KEYRANGE_EMFREELIST_BASE = 0x30000 | (BRM_UID << 20); - KEYRANGE_VBBM_BASE = 0x40000 | (BRM_UID << 20); - KEYRANGE_CL_BASE = 0x50000 | (BRM_UID << 20); - MST_SYSVKEY = 0xff000000 | BRM_UID; - PROCESSSTATUS_SYSVKEY = 0xfd000000 | BRM_UID; - SYSTEMSTATUS_SYSVKEY = 0xfc000000 | BRM_UID; - SWITCHSTATUS_SYSVKEY = 0xfb000000 | BRM_UID; - NICSTATUS_SYSVKEY = 0xf9000000 | BRM_UID; - DBROOTSTATUS_SYSVKEY = 0xf8000000 | BRM_UID; - DECOMSVRMUTEX_SYSVKEY = 0xf7000000 | BRM_UID; + KEYRANGE_VSS_BASE = 0x10000 | (BRM_UID << 20); + KEYRANGE_EXTENTMAP_BASE = 0x20000 | (BRM_UID << 20); + KEYRANGE_EMFREELIST_BASE = 0x30000 | (BRM_UID << 20); + KEYRANGE_VBBM_BASE = 0x40000 | (BRM_UID << 20); + KEYRANGE_CL_BASE = 0x50000 | (BRM_UID << 20); + MST_SYSVKEY = 0xff000000 | BRM_UID; + PROCESSSTATUS_SYSVKEY = 0xfd000000 | BRM_UID; + SYSTEMSTATUS_SYSVKEY = 0xfc000000 | BRM_UID; + SWITCHSTATUS_SYSVKEY = 0xfb000000 | BRM_UID; + NICSTATUS_SYSVKEY = 0xf9000000 | BRM_UID; + DBROOTSTATUS_SYSVKEY = 0xf8000000 | BRM_UID; + DECOMSVRMUTEX_SYSVKEY = 0xf7000000 | BRM_UID; } string ShmKeys::keyToName(unsigned key) { - ostringstream oss; - oss << "InfiniDB-shm-"; - oss << setw(8) << setfill('0') << hex << key; - return oss.str(); -} - + ostringstream oss; + oss << "InfiniDB-shm-"; + oss << setw(8) << setfill('0') << hex << key; + return oss.str(); } +} // namespace BRM diff --git a/versioning/BRM/shmkeys.h b/versioning/BRM/shmkeys.h index 62dd67866..29df2a2b4 100644 --- a/versioning/BRM/shmkeys.h +++ b/versioning/BRM/shmkeys.h @@ -31,51 +31,51 @@ namespace BRM { - /** A struct to hold shared memory keys * * A struct to hold shared memory keys that can be tuned via the config file */ struct ShmKeys { -public: - EXPORT ShmKeys(); - ~ShmKeys() { } + public: + EXPORT ShmKeys(); + ~ShmKeys() + { + } - /******** The shmseg/sem key range assigments *******/ + /******** The shmseg/sem key range assigments *******/ #if defined(COMMUNITY_KEYRANGE) - const static uint32_t KEYRANGE_SIZE = 0x1000; + const static uint32_t KEYRANGE_SIZE = 0x1000; #elif defined(_MSC_VER) - const static uint32_t KEYRANGE_SIZE = 0x3800; + const static uint32_t KEYRANGE_SIZE = 0x3800; #else - const static uint32_t KEYRANGE_SIZE = 0x10000; + const static uint32_t KEYRANGE_SIZE = 0x10000; #endif - uint32_t KEYRANGE_CL_BASE; - uint32_t KEYRANGE_EXTENTMAP_BASE; - uint32_t KEYRANGE_EMFREELIST_BASE; - uint32_t KEYRANGE_VBBM_BASE; - uint32_t KEYRANGE_VSS_BASE; + uint32_t KEYRANGE_CL_BASE; + uint32_t KEYRANGE_EXTENTMAP_BASE; + uint32_t KEYRANGE_EMFREELIST_BASE; + uint32_t KEYRANGE_VBBM_BASE; + uint32_t KEYRANGE_VSS_BASE; - /****** Fixed location assignments *******/ - uint32_t MST_SYSVKEY; - uint32_t PROCESSSTATUS_SYSVKEY; - uint32_t SYSTEMSTATUS_SYSVKEY; - uint32_t SWITCHSTATUS_SYSVKEY; - uint32_t NICSTATUS_SYSVKEY; - uint32_t DBROOTSTATUS_SYSVKEY; - uint32_t DECOMSVRMUTEX_SYSVKEY; + /****** Fixed location assignments *******/ + uint32_t MST_SYSVKEY; + uint32_t PROCESSSTATUS_SYSVKEY; + uint32_t SYSTEMSTATUS_SYSVKEY; + uint32_t SWITCHSTATUS_SYSVKEY; + uint32_t NICSTATUS_SYSVKEY; + uint32_t DBROOTSTATUS_SYSVKEY; + uint32_t DECOMSVRMUTEX_SYSVKEY; - EXPORT static std::string keyToName(unsigned key); + EXPORT static std::string keyToName(unsigned key); -private: - //defaults okay - //ShmKeys(const ShmKeys& rhs); - //ShmKeys operator=(const ShmKeys& rhs); + private: + // defaults okay + // ShmKeys(const ShmKeys& rhs); + // ShmKeys operator=(const ShmKeys& rhs); }; -} +} // namespace BRM #undef EXPORT #endif - diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index f63ffcadf..7b665a1a6 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -52,432 +52,356 @@ using namespace idbdatafile; namespace { #ifdef USE_VERY_COMPLEX_DROP_CACHES -void timespec_sub(const struct timespec& tv1, - const struct timespec& tv2, - double& tm) +void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) { - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); + tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); } #endif -} +} // namespace namespace BRM { - -SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) : - slave(s), currentSaveFile(NULL), journalh(NULL) +SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) + : slave(s) + , currentSaveFile(NULL) + , journalh(NULL) #ifdef _MSC_VER - , fPids(0), fMaxPids(64) + , fPids(0) + , fMaxPids(64) #endif { - config::Config* config = config::Config::makeConfig(); - string tmp; + config::Config* config = config::Config::makeConfig(); + string tmp; - bool tellUser = true; - - for (;;) - { - try - { - server = new MessageQueueServer(hostname); - break; - } - catch (runtime_error& re) - { - string what = re.what(); - - if (what.find("Address already in use") != string::npos) - { - if (tellUser) - { - cerr << "Address already in use, retrying..." << endl; - tellUser = false; - } - - sleep(5); - } - else - { - throw; - } - } - } - - string tmpDir = startup::StartUp::tmpDir(); - - /* NOTE: this string has to match whatever is designated as the first slave */ - if (hostname == "DBRM_Worker1") - { - try - { - savefile = config->getConfig("SystemConfig", "DBRMRoot"); - } - catch (exception& e) - { - savefile = tmpDir + "/BRM_SaveFiles"; - } - - if (savefile == "") - savefile = tmpDir + "/BRM_SaveFiles"; - - tmp = ""; - - try - { - tmp = config->getConfig("SystemConfig", "DBRMSnapshotInterval"); - } - catch (exception& e) { } - - if (tmp == "") - snapshotInterval = 100000; - else - snapshotInterval = config->fromText(tmp); - - journalCount = 0; - - firstSlave = true; - journalName = savefile + "_journal"; - const char* filename = journalName.c_str(); - - journalh = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0); - if (journalh == NULL) - throw runtime_error("Could not open the BRM journal for writing!"); - } - else - { - savefile = ""; - firstSlave = false; - } - - takeSnapshot = false; - doSaveDelta = false; - saveFileToggle = true; // start with the suffix "A" rather than "B". Arbitrary. - release = false; - die = false; - standalone = false; - printOnly = false; - //@Bug 2258 DBRMTimeOut is default to 20 seconds - //@BUG 3189 set timeout to 1 second, don't use config setting -// std::string retStr = config->getConfig("SystemConfig", "DBRMTimeOut"); -// int secondsToWait = config->fromText(retStr); - MSG_TIMEOUT.tv_nsec = 0; -// if ( secondsToWait > 0 ) -// MSG_TIMEOUT.tv_sec = secondsToWait; -// else - MSG_TIMEOUT.tv_sec = 1; -} - -SlaveComm::SlaveComm() - : currentSaveFile(NULL), journalh(NULL) -#ifdef _MSC_VER - , fPids(0), fMaxPids(64) -#endif -{ - config::Config* config = config::Config::makeConfig(); - - string tmpDir = startup::StartUp::tmpDir(); + bool tellUser = true; + for (;;) + { try { - savefile = config->getConfig("SystemConfig", "DBRMRoot"); + server = new MessageQueueServer(hostname); + break; + } + catch (runtime_error& re) + { + string what = re.what(); + + if (what.find("Address already in use") != string::npos) + { + if (tellUser) + { + cerr << "Address already in use, retrying..." << endl; + tellUser = false; + } + + sleep(5); + } + else + { + throw; + } + } + } + + string tmpDir = startup::StartUp::tmpDir(); + + /* NOTE: this string has to match whatever is designated as the first slave */ + if (hostname == "DBRM_Worker1") + { + try + { + savefile = config->getConfig("SystemConfig", "DBRMRoot"); } catch (exception& e) { - savefile = tmpDir + "/BRM_SaveFiles"; + savefile = tmpDir + "/BRM_SaveFiles"; } if (savefile == "") - savefile = tmpDir + "/BRM_SaveFiles"; + savefile = tmpDir + "/BRM_SaveFiles"; + tmp = ""; + + try + { + tmp = config->getConfig("SystemConfig", "DBRMSnapshotInterval"); + } + catch (exception& e) + { + } + + if (tmp == "") + snapshotInterval = 100000; + else + snapshotInterval = config->fromText(tmp); + + journalCount = 0; + + firstSlave = true; journalName = savefile + "_journal"; + const char* filename = journalName.c_str(); - takeSnapshot = false; - doSaveDelta = false; - saveFileToggle = true; // start with the suffix "A" rather than "B". Arbitrary. - release = false; - die = false; + journalh = IDBDataFile::open(IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0); + if (journalh == NULL) + throw runtime_error("Could not open the BRM journal for writing!"); + } + else + { + savefile = ""; firstSlave = false; - server = NULL; - standalone = true; - printOnly = false; - slave = new SlaveDBRMNode(); + } + + takeSnapshot = false; + doSaveDelta = false; + saveFileToggle = true; // start with the suffix "A" rather than "B". Arbitrary. + release = false; + die = false; + standalone = false; + printOnly = false; + //@Bug 2258 DBRMTimeOut is default to 20 seconds + //@BUG 3189 set timeout to 1 second, don't use config setting + // std::string retStr = config->getConfig("SystemConfig", "DBRMTimeOut"); + // int secondsToWait = config->fromText(retStr); + MSG_TIMEOUT.tv_nsec = 0; + // if ( secondsToWait > 0 ) + // MSG_TIMEOUT.tv_sec = secondsToWait; + // else + MSG_TIMEOUT.tv_sec = 1; +} + +SlaveComm::SlaveComm() + : currentSaveFile(NULL) + , journalh(NULL) +#ifdef _MSC_VER + , fPids(0) + , fMaxPids(64) +#endif +{ + config::Config* config = config::Config::makeConfig(); + + string tmpDir = startup::StartUp::tmpDir(); + + try + { + savefile = config->getConfig("SystemConfig", "DBRMRoot"); + } + catch (exception& e) + { + savefile = tmpDir + "/BRM_SaveFiles"; + } + + if (savefile == "") + savefile = tmpDir + "/BRM_SaveFiles"; + + journalName = savefile + "_journal"; + + takeSnapshot = false; + doSaveDelta = false; + saveFileToggle = true; // start with the suffix "A" rather than "B". Arbitrary. + release = false; + die = false; + firstSlave = false; + server = NULL; + standalone = true; + printOnly = false; + slave = new SlaveDBRMNode(); } SlaveComm::~SlaveComm() { - delete server; - server = NULL; + delete server; + server = NULL; - if (firstSlave) - { - delete currentSaveFile; - currentSaveFile = NULL; - } + if (firstSlave) + { + delete currentSaveFile; + currentSaveFile = NULL; + } - delete journalh; - journalh = NULL; + delete journalh; + journalh = NULL; } void SlaveComm::stop() { - die = true; + die = true; } void SlaveComm::reset() { - release = true; + release = true; } void SlaveComm::run() { - ByteStream msg; + ByteStream msg; - while (!die) - { + while (!die) + { #ifdef BRM_VERBOSE // cerr << "WorkerComm: waiting for a connection" << endl; #endif - master = server->accept(&MSG_TIMEOUT); + master = server->accept(&MSG_TIMEOUT); - while (!die && master.isOpen()) - { - try - { - msg = master.read(&MSG_TIMEOUT); - } - catch (SocketClosed& e) - { + while (!die && master.isOpen()) + { + try + { + msg = master.read(&MSG_TIMEOUT); + } + catch (SocketClosed& e) + { #ifdef BRM_VERBOSE - cerr << "WorkerComm: remote closed" << endl; + cerr << "WorkerComm: remote closed" << endl; #endif - break; - } - catch (...) - { + break; + } + catch (...) + { #ifdef BRM_VERBOSE - cerr << "WorkerComm: read failed, closing connection" << endl; + cerr << "WorkerComm: read failed, closing connection" << endl; #endif - break; - } + break; + } - if (release) - break; + if (release) + break; - if (die) // || msg.length() == 0) - break; + if (die) // || msg.length() == 0) + break; - if (msg.length() == 0) - continue; + if (msg.length() == 0) + continue; #ifdef BRM_VERBOSE - cerr << "WorkerComm: got a command" << endl; + cerr << "WorkerComm: got a command" << endl; #endif - try - { - processCommand(msg); - } - catch (exception& e) - { - /* - * The error is either that msg was too short (really slow sender possibly), - * there was a bigger communication failure, or there was a file IO - * error. Closing the connection for now. - */ - cerr << e.what() << endl; - do_undo(); - master.close(); - } - } - - release = false; + try + { + processCommand(msg); + } + catch (exception& e) + { + /* + * The error is either that msg was too short (really slow sender possibly), + * there was a bigger communication failure, or there was a file IO + * error. Closing the connection for now. + */ + cerr << e.what() << endl; + do_undo(); master.close(); + } } + release = false; + master.close(); + } + #ifdef BRM_VERBOSE - cerr << "WorkerComm: exiting..." << endl; + cerr << "WorkerComm: exiting..." << endl; #endif } void SlaveComm::processCommand(ByteStream& msg) { - uint8_t cmd; + uint8_t cmd; - if (firstSlave) - { - msg.peek(cmd); + if (firstSlave) + { + msg.peek(cmd); - if (cmd != CONFIRM) - delta = msg; - } + if (cmd != CONFIRM) + delta = msg; + } - msg >> cmd; + msg >> cmd; #ifdef BRM_VERBOSE - cerr << "WorkerComm: command " << (int) cmd << endl; + cerr << "WorkerComm: command " << (int)cmd << endl; #endif - switch (cmd) - { - case CREATE_STRIPE_COLUMN_EXTENTS: - do_createStripeColumnExtents(msg); - break; + switch (cmd) + { + case CREATE_STRIPE_COLUMN_EXTENTS: do_createStripeColumnExtents(msg); break; - case CREATE_COLUMN_EXTENT_DBROOT: - do_createColumnExtent_DBroot(msg); - break; + case CREATE_COLUMN_EXTENT_DBROOT: do_createColumnExtent_DBroot(msg); break; - case CREATE_COLUMN_EXTENT_EXACT_FILE: - do_createColumnExtentExactFile(msg); - break; + case CREATE_COLUMN_EXTENT_EXACT_FILE: do_createColumnExtentExactFile(msg); break; - case CREATE_DICT_STORE_EXTENT: - do_createDictStoreExtent(msg); - break; + case CREATE_DICT_STORE_EXTENT: do_createDictStoreExtent(msg); break; - case ROLLBACK_COLUMN_EXTENTS_DBROOT: - do_rollbackColumnExtents_DBroot(msg); - break; + case ROLLBACK_COLUMN_EXTENTS_DBROOT: do_rollbackColumnExtents_DBroot(msg); break; - case ROLLBACK_DICT_STORE_EXTENTS_DBROOT: - do_rollbackDictStoreExtents_DBroot(msg); - break; + case ROLLBACK_DICT_STORE_EXTENTS_DBROOT: do_rollbackDictStoreExtents_DBroot(msg); break; - case DELETE_EMPTY_COL_EXTENTS: - do_deleteEmptyColExtents(msg); - break; + case DELETE_EMPTY_COL_EXTENTS: do_deleteEmptyColExtents(msg); break; - case DELETE_EMPTY_DICT_STORE_EXTENTS: - do_deleteEmptyDictStoreExtents(msg); - break; + case DELETE_EMPTY_DICT_STORE_EXTENTS: do_deleteEmptyDictStoreExtents(msg); break; - case DELETE_OID: - do_deleteOID(msg); - break; + case DELETE_OID: do_deleteOID(msg); break; - case DELETE_OIDS: - do_deleteOIDs(msg); - break; + case DELETE_OIDS: do_deleteOIDs(msg); break; - case SET_LOCAL_HWM: - do_setLocalHWM(msg); - break; + case SET_LOCAL_HWM: do_setLocalHWM(msg); break; - case BULK_SET_HWM: - do_bulkSetHWM(msg); - break; + case BULK_SET_HWM: do_bulkSetHWM(msg); break; - case BULK_SET_HWM_AND_CP: - do_bulkSetHWMAndCP(msg); - break; + case BULK_SET_HWM_AND_CP: do_bulkSetHWMAndCP(msg); break; - case WRITE_VB_ENTRY: - do_writeVBEntry(msg); - break; + case WRITE_VB_ENTRY: do_writeVBEntry(msg); break; - case BULK_WRITE_VB_ENTRY: - do_bulkWriteVBEntry(msg); - break; + case BULK_WRITE_VB_ENTRY: do_bulkWriteVBEntry(msg); break; - case BEGIN_VB_COPY: - do_beginVBCopy(msg); - break; + case BEGIN_VB_COPY: do_beginVBCopy(msg); break; - case END_VB_COPY: - do_endVBCopy(msg); - break; + case END_VB_COPY: do_endVBCopy(msg); break; - case VB_ROLLBACK1: - do_vbRollback1(msg); - break; + case VB_ROLLBACK1: do_vbRollback1(msg); break; - case VB_ROLLBACK2: - do_vbRollback2(msg); - break; + case VB_ROLLBACK2: do_vbRollback2(msg); break; - case VB_COMMIT: - do_vbCommit(msg); - break; + case VB_COMMIT: do_vbCommit(msg); break; - case BRM_UNDO: - do_undo(); - break; + case BRM_UNDO: do_undo(); break; - case CONFIRM: - do_confirm(); - break; + case CONFIRM: do_confirm(); break; - case FLUSH_INODE_CACHES: - do_flushInodeCache(); - break; + case FLUSH_INODE_CACHES: do_flushInodeCache(); break; - case BRM_CLEAR: - do_clear(); - break; + case BRM_CLEAR: do_clear(); break; - case MARKEXTENTINVALID: - do_markInvalid(msg); - break; + case MARKEXTENTINVALID: do_markInvalid(msg); break; - case MARKMANYEXTENTSINVALID: - do_markManyExtentsInvalid(msg); - break; + case MARKMANYEXTENTSINVALID: do_markManyExtentsInvalid(msg); break; - case SETEXTENTMAXMIN: - do_setExtentMaxMin(msg); - break; + case SETEXTENTMAXMIN: do_setExtentMaxMin(msg); break; - case SETMANYEXTENTSMAXMIN: - do_setExtentsMaxMin(msg); - break; + case SETMANYEXTENTSMAXMIN: do_setExtentsMaxMin(msg); break; - case TAKE_SNAPSHOT: - do_takeSnapshot(); - break; + case TAKE_SNAPSHOT: do_takeSnapshot(); break; - case MERGEMANYEXTENTSMAXMIN: - do_mergeExtentsMaxMin(msg); - break; + case MERGEMANYEXTENTSMAXMIN: do_mergeExtentsMaxMin(msg); break; - case DELETE_PARTITION: - do_deletePartition(msg); - break; + case DELETE_PARTITION: do_deletePartition(msg); break; - case MARK_PARTITION_FOR_DELETION: - do_markPartitionForDeletion(msg); - break; + case MARK_PARTITION_FOR_DELETION: do_markPartitionForDeletion(msg); break; - case MARK_ALL_PARTITION_FOR_DELETION: - do_markAllPartitionForDeletion(msg); - break; + case MARK_ALL_PARTITION_FOR_DELETION: do_markAllPartitionForDeletion(msg); break; - case RESTORE_PARTITION: - do_restorePartition(msg); - break; + case RESTORE_PARTITION: do_restorePartition(msg); break; - case OWNER_CHECK: - do_ownerCheck(msg); - break; + case OWNER_CHECK: do_ownerCheck(msg); break; - case LOCK_LBID_RANGES: - do_dmlLockLBIDRanges(msg); - break; + case LOCK_LBID_RANGES: do_dmlLockLBIDRanges(msg); break; - case RELEASE_LBID_RANGES: - do_dmlReleaseLBIDRanges(msg); - break; + case RELEASE_LBID_RANGES: do_dmlReleaseLBIDRanges(msg); break; - case DELETE_DBROOT: - do_deleteDBRoot(msg); - break; + case DELETE_DBROOT: do_deleteDBRoot(msg); break; - case BULK_UPDATE_DBROOT: - do_bulkUpdateDBRoot(msg); - break; + case BULK_UPDATE_DBROOT: do_bulkUpdateDBRoot(msg); break; - default: - cerr << "WorkerComm: unknown command " << (int) cmd << endl; - } + default: cerr << "WorkerComm: unknown command " << (int)cmd << endl; + } } //------------------------------------------------------------------------------ @@ -485,63 +409,59 @@ void SlaveComm::processCommand(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_createStripeColumnExtents(ByteStream& msg) { - int err; - uint16_t tmp16; - uint16_t tmp32; - uint16_t dbRoot; - uint32_t partitionNum; - uint16_t segmentNum; - std::vector cols; - std::vector extents; - ByteStream reply; + int err; + uint16_t tmp16; + uint16_t tmp32; + uint16_t dbRoot; + uint32_t partitionNum; + uint16_t segmentNum; + std::vector cols; + std::vector extents; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createStripeColumnExtents()" << endl; + cerr << "WorkerComm: do_createStripeColumnExtents()" << endl; #endif - deserializeInlineVector(msg, cols); - msg >> tmp16; - dbRoot = tmp16; - msg >> tmp32; - partitionNum = tmp32; + deserializeInlineVector(msg, cols); + msg >> tmp16; + dbRoot = tmp16; + msg >> tmp32; + partitionNum = tmp32; - if (printOnly) - { - cout << "createStripeColumnExtents(). " << - "DBRoot=" << dbRoot << "; Part#=" << partitionNum << endl; + if (printOnly) + { + cout << "createStripeColumnExtents(). " + << "DBRoot=" << dbRoot << "; Part#=" << partitionNum << endl; - for (uint32_t i = 0; i < cols.size(); i++) - cout << "StripeColExt arg " << i + 1 << - ": oid=" << cols[i].oid << - " width=" << cols[i].width << endl; + for (uint32_t i = 0; i < cols.size(); i++) + cout << "StripeColExt arg " << i + 1 << ": oid=" << cols[i].oid << " width=" << cols[i].width << endl; - return; - } + return; + } - err = slave->createStripeColumnExtents(cols, dbRoot, - partitionNum, segmentNum, extents); - reply << (uint8_t) err; + err = slave->createStripeColumnExtents(cols, dbRoot, partitionNum, segmentNum, extents); + reply << (uint8_t)err; - if (err == ERR_OK) - { - reply << partitionNum; - reply << segmentNum; - serializeInlineVector( reply, extents ); - } + if (err == ERR_OK) + { + reply << partitionNum; + reply << segmentNum; + serializeInlineVector(reply, extents); + } #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createStripeColumnExtents() err code is " << - err << endl; + cerr << "WorkerComm: do_createStripeColumnExtents() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - // see bug 3596. Need to make sure a snapshot file exists. - if ((cols.size() > 0) && (cols[0].oid < 3000)) - takeSnapshot = true; - else - doSaveDelta = true; + // see bug 3596. Need to make sure a snapshot file exists. + if ((cols.size() > 0) && (cols[0].oid < 3000)) + takeSnapshot = true; + else + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -549,71 +469,67 @@ void SlaveComm::do_createStripeColumnExtents(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_createColumnExtent_DBroot(ByteStream& msg) { - int allocdSize, err; - uint8_t tmp8; - uint16_t tmp16; - uint32_t tmp32; - OID_t oid; - uint32_t colWidth; - uint16_t dbRoot; - uint32_t partitionNum; - uint16_t segmentNum; - LBID_t lbid; - uint32_t startBlockOffset; - ByteStream reply; - execplan::CalpontSystemCatalog::ColDataType colDataType; + int allocdSize, err; + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + OID_t oid; + uint32_t colWidth; + uint16_t dbRoot; + uint32_t partitionNum; + uint16_t segmentNum; + LBID_t lbid; + uint32_t startBlockOffset; + ByteStream reply; + execplan::CalpontSystemCatalog::ColDataType colDataType; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createColumnExtent_DBroot()" << endl; + cerr << "WorkerComm: do_createColumnExtent_DBroot()" << endl; #endif - msg >> tmp32; - oid = tmp32; - msg >> tmp32; - colWidth = tmp32; - msg >> tmp16; - dbRoot = tmp16; - msg >> tmp32; - partitionNum = tmp32; - msg >> tmp16; - segmentNum = tmp16; - msg >> tmp8; - colDataType = (execplan::CalpontSystemCatalog::ColDataType)tmp8; + msg >> tmp32; + oid = tmp32; + msg >> tmp32; + colWidth = tmp32; + msg >> tmp16; + dbRoot = tmp16; + msg >> tmp32; + partitionNum = tmp32; + msg >> tmp16; + segmentNum = tmp16; + msg >> tmp8; + colDataType = (execplan::CalpontSystemCatalog::ColDataType)tmp8; - if (printOnly) - { - cout << "createColumnExtent_DBroot: oid=" << oid << - " colWidth=" << colWidth << - " dbRoot=" << dbRoot << - " partitionNum=" << partitionNum << - " segmentNum=" << segmentNum << endl; - return; - } + if (printOnly) + { + cout << "createColumnExtent_DBroot: oid=" << oid << " colWidth=" << colWidth << " dbRoot=" << dbRoot + << " partitionNum=" << partitionNum << " segmentNum=" << segmentNum << endl; + return; + } - err = slave->createColumnExtent_DBroot(oid, colWidth, dbRoot, colDataType, - partitionNum, segmentNum, lbid, allocdSize, startBlockOffset); - reply << (uint8_t) err; + err = slave->createColumnExtent_DBroot(oid, colWidth, dbRoot, colDataType, partitionNum, segmentNum, lbid, + allocdSize, startBlockOffset); + reply << (uint8_t)err; - if (err == ERR_OK) - { - reply << partitionNum; - reply << segmentNum; - reply << (uint64_t) lbid; - reply << (uint32_t) allocdSize; - reply << (uint32_t) startBlockOffset; - } + if (err == ERR_OK) + { + reply << partitionNum; + reply << segmentNum; + reply << (uint64_t)lbid; + reply << (uint32_t)allocdSize; + reply << (uint32_t)startBlockOffset; + } #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createColumnExtent_DBroot() err code is " << - err << endl; + cerr << "WorkerComm: do_createColumnExtent_DBroot() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - if (oid < 3000) // see bug 3596. Need to make sure a snapshot file exists. - takeSnapshot = true; - else - doSaveDelta = true; + if (oid < 3000) // see bug 3596. Need to make sure a snapshot file exists. + takeSnapshot = true; + else + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -622,71 +538,67 @@ void SlaveComm::do_createColumnExtent_DBroot(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_createColumnExtentExactFile(ByteStream& msg) { - int allocdSize, err; - uint8_t tmp8; - uint16_t tmp16; - uint32_t tmp32; - OID_t oid; - uint32_t colWidth; - uint16_t dbRoot; - uint32_t partitionNum; - uint16_t segmentNum; - LBID_t lbid; - uint32_t startBlockOffset; - ByteStream reply; - execplan::CalpontSystemCatalog::ColDataType colDataType; + int allocdSize, err; + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + OID_t oid; + uint32_t colWidth; + uint16_t dbRoot; + uint32_t partitionNum; + uint16_t segmentNum; + LBID_t lbid; + uint32_t startBlockOffset; + ByteStream reply; + execplan::CalpontSystemCatalog::ColDataType colDataType; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createColumnExtentExactFile()" << endl; + cerr << "WorkerComm: do_createColumnExtentExactFile()" << endl; #endif - msg >> tmp32; - oid = tmp32; - msg >> tmp32; - colWidth = tmp32; - msg >> tmp16; - dbRoot = tmp16; - msg >> tmp32; - partitionNum = tmp32; - msg >> tmp16; - segmentNum = tmp16; - msg >> tmp8; - colDataType = (execplan::CalpontSystemCatalog::ColDataType)tmp8; + msg >> tmp32; + oid = tmp32; + msg >> tmp32; + colWidth = tmp32; + msg >> tmp16; + dbRoot = tmp16; + msg >> tmp32; + partitionNum = tmp32; + msg >> tmp16; + segmentNum = tmp16; + msg >> tmp8; + colDataType = (execplan::CalpontSystemCatalog::ColDataType)tmp8; - if (printOnly) - { - cout << "createColumnExtentExactFile: oid=" << oid << - " colWidth=" << colWidth << - " dbRoot=" << dbRoot << - " partitionNum=" << partitionNum << - " segmentNum=" << segmentNum << endl; - return; - } + if (printOnly) + { + cout << "createColumnExtentExactFile: oid=" << oid << " colWidth=" << colWidth << " dbRoot=" << dbRoot + << " partitionNum=" << partitionNum << " segmentNum=" << segmentNum << endl; + return; + } - err = slave->createColumnExtentExactFile(oid, colWidth, dbRoot, - partitionNum, segmentNum, colDataType, lbid, allocdSize, startBlockOffset); - reply << (uint8_t) err; + err = slave->createColumnExtentExactFile(oid, colWidth, dbRoot, partitionNum, segmentNum, colDataType, lbid, + allocdSize, startBlockOffset); + reply << (uint8_t)err; - if (err == ERR_OK) - { - reply << partitionNum; - reply << segmentNum; - reply << (uint64_t) lbid; - reply << (uint32_t) allocdSize; - reply << (uint32_t) startBlockOffset; - } + if (err == ERR_OK) + { + reply << partitionNum; + reply << segmentNum; + reply << (uint64_t)lbid; + reply << (uint32_t)allocdSize; + reply << (uint32_t)startBlockOffset; + } #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createColumnExtentExactFile() err code is " << - err << endl; + cerr << "WorkerComm: do_createColumnExtentExactFile() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - if (oid < 3000) // see bug 3596. Need to make sure a snapshot file exists. - takeSnapshot = true; - else - doSaveDelta = true; + if (oid < 3000) // see bug 3596. Need to make sure a snapshot file exists. + takeSnapshot = true; + else + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -694,53 +606,52 @@ void SlaveComm::do_createColumnExtentExactFile(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_createDictStoreExtent(ByteStream& msg) { - int allocdSize, err; - uint16_t tmp16; - uint32_t tmp32; - OID_t oid; - uint16_t dbRoot; - uint32_t partitionNum; - uint16_t segmentNum; - LBID_t lbid; - ByteStream reply; + int allocdSize, err; + uint16_t tmp16; + uint32_t tmp32; + OID_t oid; + uint16_t dbRoot; + uint32_t partitionNum; + uint16_t segmentNum; + LBID_t lbid; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createDictStoreExtent()" << endl; + cerr << "WorkerComm: do_createDictStoreExtent()" << endl; #endif - msg >> tmp32; - oid = tmp32; - msg >> tmp16; - dbRoot = tmp16; - msg >> tmp32; - partitionNum = tmp32; - msg >> tmp16; - segmentNum = tmp16; + msg >> tmp32; + oid = tmp32; + msg >> tmp16; + dbRoot = tmp16; + msg >> tmp32; + partitionNum = tmp32; + msg >> tmp16; + segmentNum = tmp16; - if (printOnly) - { - cout << "createDictStoreExtent: oid=" << oid << " dbRoot=" << dbRoot << - " partitionNum=" << partitionNum << " segmentNum=" << segmentNum << endl; - return; - } + if (printOnly) + { + cout << "createDictStoreExtent: oid=" << oid << " dbRoot=" << dbRoot << " partitionNum=" << partitionNum + << " segmentNum=" << segmentNum << endl; + return; + } - err = slave->createDictStoreExtent(oid, dbRoot, - partitionNum, segmentNum, lbid, allocdSize); - reply << (uint8_t) err; + err = slave->createDictStoreExtent(oid, dbRoot, partitionNum, segmentNum, lbid, allocdSize); + reply << (uint8_t)err; - if (err == ERR_OK) - { - reply << (uint64_t) lbid; - reply << (uint32_t) allocdSize; - } + if (err == ERR_OK) + { + reply << (uint64_t)lbid; + reply << (uint32_t)allocdSize; + } #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_createDictStoreExtent() err code is " << err << endl; + cerr << "WorkerComm: do_createDictStoreExtent() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -749,57 +660,54 @@ void SlaveComm::do_createDictStoreExtent(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_rollbackColumnExtents_DBroot(ByteStream& msg) { - int err; - OID_t oid; - bool bDeleteAll; - uint32_t partitionNum; - uint16_t segmentNum; - uint16_t dbRoot; - HWM_t hwm; - uint8_t tmp8; - uint16_t tmp16; - uint32_t tmp32; - ByteStream reply; + int err; + OID_t oid; + bool bDeleteAll; + uint32_t partitionNum; + uint16_t segmentNum; + uint16_t dbRoot; + HWM_t hwm; + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_rollbackColumnExtents_DBroot()" << endl; + cerr << "WorkerComm: do_rollbackColumnExtents_DBroot()" << endl; #endif - msg >> tmp32; - oid = tmp32; - msg >> tmp8; - bDeleteAll = tmp8; - msg >> tmp16; - dbRoot = tmp16; - msg >> tmp32; - partitionNum = tmp32; - msg >> tmp16; - segmentNum = tmp16; - msg >> tmp32; - hwm = tmp32; + msg >> tmp32; + oid = tmp32; + msg >> tmp8; + bDeleteAll = tmp8; + msg >> tmp16; + dbRoot = tmp16; + msg >> tmp32; + partitionNum = tmp32; + msg >> tmp16; + segmentNum = tmp16; + msg >> tmp32; + hwm = tmp32; - if (printOnly) - { - cout << "rollbackColumnExtents_DBroot: oid=" << oid << - " bDeleteAll=" << bDeleteAll << " dbRoot=" << dbRoot << - " partitionNum=" << partitionNum << - " segmentNum=" << segmentNum << " hwm=" << hwm << endl; - return; - } + if (printOnly) + { + cout << "rollbackColumnExtents_DBroot: oid=" << oid << " bDeleteAll=" << bDeleteAll + << " dbRoot=" << dbRoot << " partitionNum=" << partitionNum << " segmentNum=" << segmentNum + << " hwm=" << hwm << endl; + return; + } - err = slave->rollbackColumnExtents_DBroot( - oid, bDeleteAll, dbRoot, partitionNum, segmentNum, hwm); - reply << (uint8_t) err; + err = slave->rollbackColumnExtents_DBroot(oid, bDeleteAll, dbRoot, partitionNum, segmentNum, hwm); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_rollbackColumnExtents_DBroot() err code is " << - err << endl; + cerr << "WorkerComm: do_rollbackColumnExtents_DBroot() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -808,247 +716,241 @@ void SlaveComm::do_rollbackColumnExtents_DBroot(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_rollbackDictStoreExtents_DBroot(ByteStream& msg) { - int err; - OID_t oid; - uint32_t partitionNum; - uint16_t dbRoot; - uint32_t tmp32; - uint16_t tmp16; - ByteStream reply; - vector segNums; - vector hwms; + int err; + OID_t oid; + uint32_t partitionNum; + uint16_t dbRoot; + uint32_t tmp32; + uint16_t tmp16; + ByteStream reply; + vector segNums; + vector hwms; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_rollbackDictStoreExtents()" << endl; + cerr << "WorkerComm: do_rollbackDictStoreExtents()" << endl; #endif - msg >> tmp32; - oid = tmp32; - msg >> tmp16; - dbRoot = tmp16; - msg >> tmp32; - partitionNum = tmp32; - deserializeVector(msg, segNums); - deserializeVector(msg, hwms); + msg >> tmp32; + oid = tmp32; + msg >> tmp16; + dbRoot = tmp16; + msg >> tmp32; + partitionNum = tmp32; + deserializeVector(msg, segNums); + deserializeVector(msg, hwms); - if (printOnly) - { - cout << "rollbackDictStore: oid=" << oid << - " dbRoot=" << dbRoot << - " partitionNum=" << partitionNum << - " hwms..." << endl; + if (printOnly) + { + cout << "rollbackDictStore: oid=" << oid << " dbRoot=" << dbRoot << " partitionNum=" << partitionNum + << " hwms..." << endl; - for (uint32_t i = 0; i < hwms.size(); i++) - cout << " " << i << ": " << hwms[i] << endl; + for (uint32_t i = 0; i < hwms.size(); i++) + cout << " " << i << ": " << hwms[i] << endl; - return; - } + return; + } - - err = slave->rollbackDictStoreExtents_DBroot( - oid, dbRoot, partitionNum, segNums, hwms); - reply << (uint8_t) err; + err = slave->rollbackDictStoreExtents_DBroot(oid, dbRoot, partitionNum, segNums, hwms); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_rollbackDictStoreExtents() err code is " << - err << endl; + cerr << "WorkerComm: do_rollbackDictStoreExtents() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_deleteEmptyColExtents(messageqcpp::ByteStream& msg) { - OID_t oid; - uint32_t tmp1; - uint16_t tmp2; - int err; - ByteStream reply; - uint32_t size; - ExtentsInfoMap_t extentsInfoMap; + OID_t oid; + uint32_t tmp1; + uint16_t tmp2; + int err; + ByteStream reply; + uint32_t size; + ExtentsInfoMap_t extentsInfoMap; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteEmptyColExtents()" << endl; + cerr << "WorkerComm: do_deleteEmptyColExtents()" << endl; #endif - msg >> size; + msg >> size; + + if (printOnly) + cout << "deleteEmptyColExtents: size=" << size << " extentsInfoMap..." << endl; + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp1; + oid = tmp1; + extentsInfoMap[oid].oid = oid; + msg >> tmp1; + extentsInfoMap[oid].partitionNum = tmp1; + msg >> tmp2; + extentsInfoMap[oid].segmentNum = tmp2; + msg >> tmp2; + extentsInfoMap[oid].dbRoot = tmp2; + msg >> tmp1; + extentsInfoMap[oid].hwm = tmp1; if (printOnly) - cout << "deleteEmptyColExtents: size=" << size << " extentsInfoMap..." << endl; - - for ( unsigned i = 0; i < size; i++ ) { - msg >> tmp1; - oid = tmp1; - extentsInfoMap[oid].oid = oid; - msg >> tmp1; - extentsInfoMap[oid].partitionNum = tmp1; - msg >> tmp2; - extentsInfoMap[oid].segmentNum = tmp2; - msg >> tmp2; - extentsInfoMap[oid].dbRoot = tmp2; - msg >> tmp1; - extentsInfoMap[oid].hwm = tmp1; - - if (printOnly) - { - cout << " oid=" << oid << " partitionNum=" << extentsInfoMap[oid].partitionNum - << " segmentNum=" << extentsInfoMap[oid].segmentNum << " dbRoot=" << - extentsInfoMap[oid].dbRoot << " hwm=" << extentsInfoMap[oid].hwm << endl; - } + cout << " oid=" << oid << " partitionNum=" << extentsInfoMap[oid].partitionNum + << " segmentNum=" << extentsInfoMap[oid].segmentNum << " dbRoot=" << extentsInfoMap[oid].dbRoot + << " hwm=" << extentsInfoMap[oid].hwm << endl; } + } - if (printOnly) - return; + if (printOnly) + return; - err = slave->deleteEmptyColExtents(extentsInfoMap); - reply << (uint8_t) err; + err = slave->deleteEmptyColExtents(extentsInfoMap); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteEmptyColExtents() err code is " << err << endl; + cerr << "WorkerComm: do_deleteEmptyColExtents() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_deleteEmptyDictStoreExtents(messageqcpp::ByteStream& msg) { - OID_t oid; - uint32_t tmp1; - uint16_t tmp2; - uint8_t tmp3; - int err; - ByteStream reply; - uint32_t size; - ExtentsInfoMap_t extentsInfoMap; + OID_t oid; + uint32_t tmp1; + uint16_t tmp2; + uint8_t tmp3; + int err; + ByteStream reply; + uint32_t size; + ExtentsInfoMap_t extentsInfoMap; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteEmptyDictStoreExtents()" << endl; + cerr << "WorkerComm: do_deleteEmptyDictStoreExtents()" << endl; #endif - msg >> size; + msg >> size; + + if (printOnly) + cout << "deleteEmptyDictStoreExtents: size=" << size << " extentsInfoMap..." << endl; + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp1; + oid = tmp1; + extentsInfoMap[oid].oid = oid; + msg >> tmp1; + extentsInfoMap[oid].partitionNum = tmp1; + msg >> tmp2; + extentsInfoMap[oid].segmentNum = tmp2; + msg >> tmp2; + extentsInfoMap[oid].dbRoot = tmp2; + msg >> tmp1; + extentsInfoMap[oid].hwm = tmp1; + msg >> tmp3; + extentsInfoMap[oid].newFile = (bool)tmp3; if (printOnly) - cout << "deleteEmptyDictStoreExtents: size=" << size << " extentsInfoMap..." << endl; - - for ( unsigned i = 0; i < size; i++ ) { - msg >> tmp1; - oid = tmp1; - extentsInfoMap[oid].oid = oid; - msg >> tmp1; - extentsInfoMap[oid].partitionNum = tmp1; - msg >> tmp2; - extentsInfoMap[oid].segmentNum = tmp2; - msg >> tmp2; - extentsInfoMap[oid].dbRoot = tmp2; - msg >> tmp1; - extentsInfoMap[oid].hwm = tmp1; - msg >> tmp3; - extentsInfoMap[oid].newFile = (bool) tmp3; - - if (printOnly) - { - cout << " oid=" << oid << " partitionNum=" << extentsInfoMap[oid].partitionNum << - " segmentNum=" << extentsInfoMap[oid].segmentNum << " dbRoot=" << - extentsInfoMap[oid].dbRoot << " hwm=" << extentsInfoMap[oid].hwm << - " newFile=" << (int) extentsInfoMap[oid].newFile << endl; - } + cout << " oid=" << oid << " partitionNum=" << extentsInfoMap[oid].partitionNum + << " segmentNum=" << extentsInfoMap[oid].segmentNum << " dbRoot=" << extentsInfoMap[oid].dbRoot + << " hwm=" << extentsInfoMap[oid].hwm << " newFile=" << (int)extentsInfoMap[oid].newFile << endl; } + } - if (printOnly) - return; + if (printOnly) + return; - err = slave->deleteEmptyDictStoreExtents(extentsInfoMap); - reply << (uint8_t) err; + err = slave->deleteEmptyDictStoreExtents(extentsInfoMap); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteEmptyDictStoreExtents() err code is " << err << endl; + cerr << "WorkerComm: do_deleteEmptyDictStoreExtents() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_deleteOID(ByteStream& msg) { - OID_t oid; - uint32_t tmp; - int err; - ByteStream reply; + OID_t oid; + uint32_t tmp; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteOID()" << endl; + cerr << "WorkerComm: do_deleteOID()" << endl; #endif - msg >> tmp; - oid = tmp; + msg >> tmp; + oid = tmp; - if (printOnly) - { - cout << "deleteOID: oid=" << oid << endl; - return; - } + if (printOnly) + { + cout << "deleteOID: oid=" << oid << endl; + return; + } - err = slave->deleteOID(oid); - reply << (uint8_t) err; + err = slave->deleteOID(oid); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteOID() err code is " << err << endl; + cerr << "WorkerComm: do_deleteOID() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_deleteOIDs(ByteStream& msg) { - OID_t oid; - uint32_t tmp; - int err; - ByteStream reply; - uint32_t size; - std::vector Oids; - OidsMap_t oidsMap; + OID_t oid; + uint32_t tmp; + int err; + ByteStream reply; + uint32_t size; + std::vector Oids; + OidsMap_t oidsMap; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteOIDs()" << endl; + cerr << "WorkerComm: do_deleteOIDs()" << endl; #endif - msg >> size; + msg >> size; + + if (printOnly) + cout << "deleteOIDs: size=" << size << endl; + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp; + oid = tmp; + oidsMap[oid] = oid; if (printOnly) - cout << "deleteOIDs: size=" << size << endl; + cout << " oid=" << oid << endl; + } - for ( unsigned i = 0; i < size; i++ ) - { - msg >> tmp; - oid = tmp; - oidsMap[oid] = oid; + if (printOnly) + return; - if (printOnly) - cout << " oid=" << oid << endl; - } - - if (printOnly) - return; - - err = slave->deleteOIDs(oidsMap); - reply << (uint8_t) err; + err = slave->deleteOIDs(oidsMap); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteOIDs() err code is " << err << endl; + cerr << "WorkerComm: do_deleteOIDs() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1057,105 +959,105 @@ void SlaveComm::do_deleteOIDs(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_setLocalHWM(ByteStream& msg) { - OID_t oid; - HWM_t hwm; - uint32_t partitionNum; - uint16_t segmentNum; - int err; - uint16_t tmp16; - uint32_t tmp32; - ByteStream reply; + OID_t oid; + HWM_t hwm; + uint32_t partitionNum; + uint16_t segmentNum; + int err; + uint16_t tmp16; + uint32_t tmp32; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setLocalHWM()" << endl; + cerr << "WorkerComm: do_setLocalHWM()" << endl; #endif - msg >> tmp32; - oid = tmp32; - msg >> tmp32; - partitionNum = tmp32; - msg >> tmp16; - segmentNum = tmp16; - msg >> tmp32; - hwm = tmp32; + msg >> tmp32; + oid = tmp32; + msg >> tmp32; + partitionNum = tmp32; + msg >> tmp16; + segmentNum = tmp16; + msg >> tmp32; + hwm = tmp32; - if (printOnly) - { - cout << "setLocalHWM: oid=" << oid << " partitionNum=" << partitionNum << - " segmentNum=" << segmentNum << " hwm=" << hwm << endl; - return; - } + if (printOnly) + { + cout << "setLocalHWM: oid=" << oid << " partitionNum=" << partitionNum << " segmentNum=" << segmentNum + << " hwm=" << hwm << endl; + return; + } - err = slave->setLocalHWM(oid, partitionNum, segmentNum, hwm, firstSlave); - reply << (uint8_t) err; + err = slave->setLocalHWM(oid, partitionNum, segmentNum, hwm, firstSlave); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setLocalHWM() err code is " << err << endl; + cerr << "WorkerComm: do_setLocalHWM() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_bulkSetHWM(ByteStream& msg) { - vector args; - int err; - VER_t transID; - uint32_t tmp32; - ByteStream reply; + vector args; + int err; + VER_t transID; + uint32_t tmp32; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setLocalHWM()" << endl; + cerr << "WorkerComm: do_setLocalHWM()" << endl; #endif - deserializeInlineVector(msg, args); - msg >> tmp32; - transID = tmp32; + deserializeInlineVector(msg, args); + msg >> tmp32; + transID = tmp32; - if (printOnly) - { - cout << "bulkSetHWM(). TransID = " << transID << endl; + if (printOnly) + { + cout << "bulkSetHWM(). TransID = " << transID << endl; - for (uint32_t i = 0; i < args.size(); i++) - cout << "bulkSetHWM arg " << i + 1 << ": oid=" << args[i].oid << " partitionNum=" << args[i].partNum << - " segmentNum=" << args[i].segNum << " hwm=" << args[i].hwm << endl; + for (uint32_t i = 0; i < args.size(); i++) + cout << "bulkSetHWM arg " << i + 1 << ": oid=" << args[i].oid << " partitionNum=" << args[i].partNum + << " segmentNum=" << args[i].segNum << " hwm=" << args[i].hwm << endl; - return; - } + return; + } - err = slave->bulkSetHWM(args, transID, firstSlave); - reply << (uint8_t) err; + err = slave->bulkSetHWM(args, transID, firstSlave); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setLocalHWM() err code is " << err << endl; + cerr << "WorkerComm: do_setLocalHWM() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_bulkSetHWMAndCP(ByteStream& msg) { - vector hwmArgs; - vector setCPDataArgs; - vector mergeCPDataArgs; - int err; - VER_t transID; - uint32_t tmp32; - ByteStream reply; + vector hwmArgs; + vector setCPDataArgs; + vector mergeCPDataArgs; + int err; + VER_t transID; + uint32_t tmp32; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setLocalHWM()" << endl; + cerr << "WorkerComm: do_setLocalHWM()" << endl; #endif - deserializeInlineVector(msg, hwmArgs); - deserializeInlineVector(msg, setCPDataArgs); - deserializeInlineVector(msg, mergeCPDataArgs); - msg >> tmp32; - transID = tmp32; + deserializeInlineVector(msg, hwmArgs); + deserializeInlineVector(msg, setCPDataArgs); + deserializeInlineVector(msg, mergeCPDataArgs); + msg >> tmp32; + transID = tmp32; #if 0 @@ -1172,233 +1074,233 @@ void SlaveComm::do_bulkSetHWMAndCP(ByteStream& msg) #endif - err = slave->bulkSetHWMAndCP(hwmArgs, setCPDataArgs, mergeCPDataArgs, transID, firstSlave); - reply << (uint8_t) err; + err = slave->bulkSetHWMAndCP(hwmArgs, setCPDataArgs, mergeCPDataArgs, transID, firstSlave); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setLocalHWM() err code is " << err << endl; + cerr << "WorkerComm: do_setLocalHWM() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_bulkUpdateDBRoot(ByteStream& msg) { - vector args; - ByteStream reply; - int err; + vector args; + ByteStream reply; + int err; - deserializeInlineVector(msg, args); - err = slave->bulkUpdateDBRoot(args); - reply << (uint8_t) err; + deserializeInlineVector(msg, args); + err = slave->bulkUpdateDBRoot(args); + reply << (uint8_t)err; - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_markInvalid(ByteStream& msg) { - LBID_t lbid; - uint32_t colDataType; - int err; - ByteStream reply; + LBID_t lbid; + uint32_t colDataType; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markInvalid()" << endl; + cerr << "WorkerComm: do_markInvalid()" << endl; #endif - msg >> lbid; - msg >> colDataType; + msg >> lbid; + msg >> colDataType; - if (printOnly) - { - cout << "markExtentInvalid: lbid=" << lbid << "colDataType=" << colDataType << endl; - return; - } + if (printOnly) + { + cout << "markExtentInvalid: lbid=" << lbid << "colDataType=" << colDataType << endl; + return; + } - err = slave->markExtentInvalid(lbid, (execplan::CalpontSystemCatalog::ColDataType)colDataType); - reply << (uint8_t)err; + err = slave->markExtentInvalid(lbid, (execplan::CalpontSystemCatalog::ColDataType)colDataType); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markInvalid() err code is " << err << endl; + cerr << "WorkerComm: do_markInvalid() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_markManyExtentsInvalid(ByteStream& msg) { - uint64_t tmp64; - uint32_t colDataType; - int err; - ByteStream reply; - vector lbids; - vector colDataTypes; - uint32_t size, i; + uint64_t tmp64; + uint32_t colDataType; + int err; + ByteStream reply; + vector lbids; + vector colDataTypes; + uint32_t size, i; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markManyExtentsInvalid()" << endl; + cerr << "WorkerComm: do_markManyExtentsInvalid()" << endl; #endif - msg >> size; + msg >> size; + + if (printOnly) + cout << "markManyExtentsInvalid: size=" << size << " lbids..." << endl; + + for (i = 0; i < size; ++i) + { + msg >> tmp64; + msg >> colDataType; + lbids.push_back(tmp64); + colDataTypes.push_back((execplan::CalpontSystemCatalog::ColDataType)colDataType); if (printOnly) - cout << "markManyExtentsInvalid: size=" << size << " lbids..." << endl; + cout << " " << tmp64 << " " << colDataType << endl; + } - for (i = 0; i < size; ++i) - { - msg >> tmp64; - msg >> colDataType; - lbids.push_back(tmp64); - colDataTypes.push_back((execplan::CalpontSystemCatalog::ColDataType)colDataType); + if (printOnly) + return; - if (printOnly) - cout << " " << tmp64 << " " << colDataType << endl; - } - - if (printOnly) - return; - - err = slave->markExtentsInvalid(lbids, colDataTypes); - reply << (uint8_t)err; + err = slave->markExtentsInvalid(lbids, colDataTypes); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markManyExtentsInvalid() err code is " << err << endl; + cerr << "WorkerComm: do_markManyExtentsInvalid() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_setExtentMaxMin(ByteStream& msg) { - LBID_t lbid; - int64_t max; - int64_t min; - int32_t sequence; - uint64_t tmp64; - uint32_t tmp32; - int err; - ByteStream reply; + LBID_t lbid; + int64_t max; + int64_t min; + int32_t sequence; + uint64_t tmp64; + uint32_t tmp32; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setExtentMaxMin()" << endl; + cerr << "WorkerComm: do_setExtentMaxMin()" << endl; #endif - msg >> tmp64; - lbid = tmp64; + msg >> tmp64; + lbid = tmp64; - msg >> tmp64; - max = tmp64; + msg >> tmp64; + max = tmp64; - msg >> tmp64; - min = tmp64; + msg >> tmp64; + min = tmp64; - msg >> tmp32; - sequence = tmp32; + msg >> tmp32; + sequence = tmp32; - if (printOnly) - { - cout << "setExtentMaxMin: lbid=" << lbid << " max=" << max << " min=" << min << - " sequence=" << sequence << endl; - return; - } + if (printOnly) + { + cout << "setExtentMaxMin: lbid=" << lbid << " max=" << max << " min=" << min << " sequence=" << sequence + << endl; + return; + } - err = slave->setExtentMaxMin(lbid, max, min, sequence, firstSlave); - reply << (uint8_t)err; + err = slave->setExtentMaxMin(lbid, max, min, sequence, firstSlave); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setExtentMaxMin() err code is " << err << endl; + cerr << "WorkerComm: do_setExtentMaxMin() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } // @bug 1970 - added do_setExtentsMaxMin to set multiple extents CP info. void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) { - LBID_t lbid; - uint64_t tmp64; - uint32_t tmp32; - uint8_t tmp8; - uint128_t tmp128; - int err; - ByteStream reply; - int32_t updateCount; - bool isBinaryColumn = false; + LBID_t lbid; + uint64_t tmp64; + uint32_t tmp32; + uint8_t tmp8; + uint128_t tmp128; + int err; + ByteStream reply; + int32_t updateCount; + bool isBinaryColumn = false; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setExtentsMaxMin()" << endl; + cerr << "WorkerComm: do_setExtentsMaxMin()" << endl; #endif - msg >> tmp32; - updateCount = tmp32; - CPMaxMinMap_t cpMap; - CPMaxMin cpMaxMin; + msg >> tmp32; + updateCount = tmp32; + CPMaxMinMap_t cpMap; + CPMaxMin cpMaxMin; - if (printOnly) - cout << "setExtentsMaxMin: size=" << updateCount << " CPdata..." << endl; + if (printOnly) + cout << "setExtentsMaxMin: size=" << updateCount << " CPdata..." << endl; - // Loop through extents and add each one to a map. - for (int64_t i = 0; i < updateCount; i++) + // Loop through extents and add each one to a map. + for (int64_t i = 0; i < updateCount; i++) + { + msg >> tmp8; + isBinaryColumn = (tmp8 != 0); + + msg >> tmp64; + lbid = tmp64; + + cpMaxMin.isBinaryColumn = isBinaryColumn; + + if (isBinaryColumn) { - msg >> tmp8; - isBinaryColumn = (tmp8 != 0); + msg >> tmp128; + cpMaxMin.bigMax = tmp128; - msg >> tmp64; - lbid = tmp64; + msg >> tmp128; + cpMaxMin.bigMin = tmp128; + } + else + { + msg >> tmp64; + cpMaxMin.max = tmp64; - cpMaxMin.isBinaryColumn = isBinaryColumn; - - if (isBinaryColumn) - { - msg >> tmp128; - cpMaxMin.bigMax = tmp128; - - msg >> tmp128; - cpMaxMin.bigMin = tmp128; - } - else - { - msg >> tmp64; - cpMaxMin.max = tmp64; - - msg >> tmp64; - cpMaxMin.min = tmp64; - } - - msg >> tmp32; - cpMaxMin.seqNum = tmp32; - - cpMap[lbid] = cpMaxMin; - - if (printOnly) - cout << " lbid=" << lbid << " max=" << cpMaxMin.max << " min=" << - cpMaxMin.min << " sequenceNum=" << cpMaxMin.seqNum << endl; + msg >> tmp64; + cpMaxMin.min = tmp64; } - if (printOnly) - return; + msg >> tmp32; + cpMaxMin.seqNum = tmp32; - err = slave->setExtentsMaxMin(cpMap, firstSlave); - reply << (uint8_t)err; + cpMap[lbid] = cpMaxMin; + + if (printOnly) + cout << " lbid=" << lbid << " max=" << cpMaxMin.max << " min=" << cpMaxMin.min + << " sequenceNum=" << cpMaxMin.seqNum << endl; + } + + if (printOnly) + return; + + err = slave->setExtentsMaxMin(cpMap, firstSlave); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_setExtentsMaxMin() err code is " << err << endl; + cerr << "WorkerComm: do_setExtentsMaxMin() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1406,67 +1308,67 @@ void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_mergeExtentsMaxMin(ByteStream& msg) { - LBID_t startLbid; - uint64_t tmp64; - uint32_t tmp32; - int err; - ByteStream reply; - int32_t mergeCount; + LBID_t startLbid; + uint64_t tmp64; + uint32_t tmp32; + int err; + ByteStream reply; + int32_t mergeCount; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_mergeExtentsMaxMin()" << endl; + cerr << "WorkerComm: do_mergeExtentsMaxMin()" << endl; #endif + msg >> tmp32; + mergeCount = tmp32; + CPMaxMinMergeMap_t cpMap; + CPMaxMinMerge cpMaxMin; + + if (printOnly) + cout << "mergeExtentsMaxMin: size=" << mergeCount << " CPdata..." << endl; + + // Loop through extents and add each one to a map. + for (int64_t i = 0; i < mergeCount; i++) + { + msg >> tmp64; + startLbid = tmp64; + + msg >> tmp64; + cpMaxMin.max = tmp64; + + msg >> tmp64; + cpMaxMin.min = tmp64; + msg >> tmp32; - mergeCount = tmp32; - CPMaxMinMergeMap_t cpMap; - CPMaxMinMerge cpMaxMin; + cpMaxMin.seqNum = tmp32; + + msg >> tmp32; + cpMaxMin.type = (execplan::CalpontSystemCatalog::ColDataType)tmp32; + + msg >> tmp32; + cpMaxMin.newExtent = tmp32; + + cpMap[startLbid] = cpMaxMin; if (printOnly) - cout << "mergeExtentsMaxMin: size=" << mergeCount << " CPdata..." << endl; + cout << " startLBID=" << startLbid << " max=" << cpMaxMin.max << " min=" << cpMaxMin.min + << " sequenceNum=" << cpMaxMin.seqNum << " type=" << (int)cpMaxMin.type + << " newExtent=" << (int)cpMaxMin.newExtent << endl; + } - // Loop through extents and add each one to a map. - for (int64_t i = 0; i < mergeCount; i++) - { - msg >> tmp64; - startLbid = tmp64; + if (printOnly) + return; - msg >> tmp64; - cpMaxMin.max = tmp64; - - msg >> tmp64; - cpMaxMin.min = tmp64; - - msg >> tmp32; - cpMaxMin.seqNum = tmp32; - - msg >> tmp32; - cpMaxMin.type = (execplan::CalpontSystemCatalog::ColDataType)tmp32; - - msg >> tmp32; - cpMaxMin.newExtent = tmp32; - - cpMap[startLbid] = cpMaxMin; - - if (printOnly) - cout << " startLBID=" << startLbid << " max=" << cpMaxMin.max << " min=" << - cpMaxMin.min << " sequenceNum=" << cpMaxMin.seqNum << " type=" << (int) - cpMaxMin.type << " newExtent=" << (int) cpMaxMin.newExtent << endl; - } - - if (printOnly) - return; - - err = slave->mergeExtentsMaxMin(cpMap); - reply << (uint8_t)err; + err = slave->mergeExtentsMaxMin(cpMap); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_mergeExtentsMaxMin() err code is " << err << endl; + cerr << "WorkerComm: do_mergeExtentsMaxMin() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1474,61 +1376,61 @@ void SlaveComm::do_mergeExtentsMaxMin(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_deletePartition(ByteStream& msg) { - OID_t oid; - uint32_t tmp32; - int err; - ByteStream reply; - uint32_t size; - std::set oids; + OID_t oid; + uint32_t tmp32; + int err; + ByteStream reply; + uint32_t size; + std::set oids; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deletePartition()" << endl; + cerr << "WorkerComm: do_deletePartition()" << endl; #endif - set partitionNums; - deserializeSet(msg, partitionNums); + set partitionNums; + deserializeSet(msg, partitionNums); - msg >> size; + msg >> size; + + if (printOnly) + { + cout << "deletePartition: partitionNum: "; + set::const_iterator it; + + for (it = partitionNums.begin(); it != partitionNums.end(); ++it) + cout << (*it) << " "; + + cout << "\nsize=" << size << " oids..." << endl; + } + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp32; + oid = tmp32; + oids.insert(oid); if (printOnly) - { - cout << "deletePartition: partitionNum: "; - set::const_iterator it; + cout << " " << oid << endl; + } - for (it = partitionNums.begin(); it != partitionNums.end(); ++it) - cout << (*it) << " "; + if (printOnly) + return; - cout << "\nsize=" << size << " oids..." << endl; - } + string emsg; + err = slave->deletePartition(oids, partitionNums, emsg); + reply << (uint8_t)err; - for (unsigned i = 0; i < size; i++) - { - msg >> tmp32; - oid = tmp32; - oids.insert( oid ); - - if (printOnly) - cout << " " << oid << endl; - } - - if (printOnly) - return; - - string emsg; - err = slave->deletePartition(oids, partitionNums, emsg); - reply << (uint8_t) err; - - if (err != 0) - reply << emsg; + if (err != 0) + reply << emsg; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deletePartition() err code is " << err << endl; + cerr << "WorkerComm: do_deletePartition() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1537,61 +1439,60 @@ void SlaveComm::do_deletePartition(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_markPartitionForDeletion(ByteStream& msg) { - OID_t oid; - uint32_t tmp32; - int err; - ByteStream reply; - uint32_t size; - std::set oids; + OID_t oid; + uint32_t tmp32; + int err; + ByteStream reply; + uint32_t size; + std::set oids; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markPartitionForDeletion()" << endl; + cerr << "WorkerComm: do_markPartitionForDeletion()" << endl; #endif - set partitionNums; - deserializeSet(msg, partitionNums); - msg >> size; + set partitionNums; + deserializeSet(msg, partitionNums); + msg >> size; + + if (printOnly) + { + cout << "markPartitionForDeletion: partitionNum: "; + set::const_iterator it; + + for (it = partitionNums.begin(); it != partitionNums.end(); ++it) + cout << (*it) << " "; + + cout << "\nsize=" << size << " oids..." << endl; + } + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp32; + oid = tmp32; + oids.insert(oid); if (printOnly) - { - cout << "markPartitionForDeletion: partitionNum: "; - set::const_iterator it; + cout << " " << oid << endl; + } - for (it = partitionNums.begin(); it != partitionNums.end(); ++it) - cout << (*it) << " "; + if (printOnly) + return; - cout << "\nsize=" << size << " oids..." << endl; - } + string emsg; + err = slave->markPartitionForDeletion(oids, partitionNums, emsg); + reply << (uint8_t)err; - for (unsigned i = 0; i < size; i++) - { - msg >> tmp32; - oid = tmp32; - oids.insert( oid ); - - if (printOnly) - cout << " " << oid << endl; - } - - if (printOnly) - return; - - string emsg; - err = slave->markPartitionForDeletion(oids, partitionNums, emsg); - reply << (uint8_t) err; - - if (err != 0) - reply << emsg; + if (err != 0) + reply << emsg; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markPartitionforDeletion() err code is " << - err << endl; + cerr << "WorkerComm: do_markPartitionforDeletion() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1600,46 +1501,44 @@ void SlaveComm::do_markPartitionForDeletion(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_markAllPartitionForDeletion(ByteStream& msg) { - OID_t oid; - uint32_t tmp32; - int err; - ByteStream reply; - uint32_t size; - std::set oids; + OID_t oid; + uint32_t tmp32; + int err; + ByteStream reply; + uint32_t size; + std::set oids; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markAllPartitionForDeletion()" << endl; + cerr << "WorkerComm: do_markAllPartitionForDeletion()" << endl; #endif - msg >> size; + msg >> size; + + if (printOnly) + cout << "markAllPartitionForDeletion: size=" << size << " oids..." << endl; + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp32; + oid = tmp32; + oids.insert(oid); if (printOnly) - cout << "markAllPartitionForDeletion: size=" - << size << " oids..." << endl; + cout << " " << oid << endl; + } - for (unsigned i = 0; i < size; i++) - { - msg >> tmp32; - oid = tmp32; - oids.insert( oid ); + if (printOnly) + return; - if (printOnly) - cout << " " << oid << endl; - } - - if (printOnly) - return; - - err = slave->markAllPartitionForDeletion(oids); - reply << (uint8_t) err; + err = slave->markAllPartitionForDeletion(oids); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_markAllPartitionforDeletion() err code is " << - err << endl; + cerr << "WorkerComm: do_markAllPartitionforDeletion() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1647,61 +1546,61 @@ void SlaveComm::do_markAllPartitionForDeletion(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_restorePartition(ByteStream& msg) { - OID_t oid; - uint32_t tmp32; - int err; - ByteStream reply; - uint32_t size; - std::set oids; + OID_t oid; + uint32_t tmp32; + int err; + ByteStream reply; + uint32_t size; + std::set oids; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_restorePartition()" << endl; + cerr << "WorkerComm: do_restorePartition()" << endl; #endif - set partitionNums; - deserializeSet(msg, partitionNums); + set partitionNums; + deserializeSet(msg, partitionNums); - msg >> size; + msg >> size; + + if (printOnly) + { + cout << "restorePartition: partitionNum: "; + set::const_iterator it; + + for (it = partitionNums.begin(); it != partitionNums.end(); ++it) + cout << (*it) << " "; + + cout << "\nsize=" << size << " oids..." << endl; + } + + for (unsigned i = 0; i < size; i++) + { + msg >> tmp32; + oid = tmp32; + oids.insert(oid); if (printOnly) - { - cout << "restorePartition: partitionNum: "; - set::const_iterator it; + cout << " " << oid << endl; + } - for (it = partitionNums.begin(); it != partitionNums.end(); ++it) - cout << (*it) << " "; + if (printOnly) + return; - cout << "\nsize=" << size << " oids..." << endl; - } + string emsg; + err = slave->restorePartition(oids, partitionNums, emsg); + reply << (uint8_t)err; - for (unsigned i = 0; i < size; i++) - { - msg >> tmp32; - oid = tmp32; - oids.insert( oid ); - - if (printOnly) - cout << " " << oid << endl; - } - - if (printOnly) - return; - - string emsg; - err = slave->restorePartition(oids, partitionNums, emsg); - reply << (uint8_t) err; - - if (err != 0) - reply << emsg; + if (err != 0) + reply << emsg; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_restorePartition() err code is " << err << endl; + cerr << "WorkerComm: do_restorePartition() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } //------------------------------------------------------------------------------ @@ -1709,767 +1608,756 @@ void SlaveComm::do_restorePartition(ByteStream& msg) //------------------------------------------------------------------------------ void SlaveComm::do_deleteDBRoot(ByteStream& msg) { - int err; - ByteStream reply; - uint32_t q; - uint16_t dbroot; + int err; + ByteStream reply; + uint32_t q; + uint16_t dbroot; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteDBroot()" << endl; + cerr << "WorkerComm: do_deleteDBroot()" << endl; #endif - msg >> q; - dbroot = static_cast(q); + msg >> q; + dbroot = static_cast(q); - if (printOnly) - { - cout << "deleteDBRoot: " << dbroot << endl; - return; - } + if (printOnly) + { + cout << "deleteDBRoot: " << dbroot << endl; + return; + } - err = slave->deleteDBRoot(dbroot); - reply << (uint8_t) err; + err = slave->deleteDBRoot(dbroot); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_deleteDBRoot() err code is " << err << endl; + cerr << "WorkerComm: do_deleteDBRoot() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_writeVBEntry(ByteStream& msg) { - VER_t transID; - LBID_t lbid; - OID_t vbOID; - uint32_t vbFBO, tmp; - uint64_t tmp64; - int err; - ByteStream reply; + VER_t transID; + LBID_t lbid; + OID_t vbOID; + uint32_t vbFBO, tmp; + uint64_t tmp64; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_writeVBEntry()" << endl; + cerr << "WorkerComm: do_writeVBEntry()" << endl; #endif - msg >> tmp; - transID = tmp; - msg >> tmp64; - lbid = tmp64; - msg >> tmp; - vbOID = tmp; - msg >> vbFBO; + msg >> tmp; + transID = tmp; + msg >> tmp64; + lbid = tmp64; + msg >> tmp; + vbOID = tmp; + msg >> vbFBO; - if (printOnly) - { - cout << "writeVBEntry: transID=" << transID << " lbid=" << lbid << " vbOID=" << - vbOID << " vbFBO=" << vbFBO << endl; - return; - } + if (printOnly) + { + cout << "writeVBEntry: transID=" << transID << " lbid=" << lbid << " vbOID=" << vbOID + << " vbFBO=" << vbFBO << endl; + return; + } - err = slave->writeVBEntry(transID, lbid, vbOID, vbFBO); - reply << (uint8_t) err; + err = slave->writeVBEntry(transID, lbid, vbOID, vbFBO); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_writeVBEntry() err code is " << err << endl; + cerr << "WorkerComm: do_writeVBEntry() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_bulkWriteVBEntry(ByteStream& msg) { - VER_t transID; - std::vector lbids; - OID_t vbOID; - std::vector vbFBOs; - uint32_t tmp; - int err; - ByteStream reply; + VER_t transID; + std::vector lbids; + OID_t vbOID; + std::vector vbFBOs; + uint32_t tmp; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_bulkWriteVBEntry()" << endl; + cerr << "WorkerComm: do_bulkWriteVBEntry()" << endl; #endif - msg >> tmp; - transID = tmp; - deserializeInlineVector(msg, lbids); - msg >> tmp; - vbOID = tmp; - deserializeInlineVector(msg, vbFBOs); + msg >> tmp; + transID = tmp; + deserializeInlineVector(msg, lbids); + msg >> tmp; + vbOID = tmp; + deserializeInlineVector(msg, vbFBOs); - if (printOnly) - { - cout << "bulkWriteVBEntry: transID=" << transID << endl; + if (printOnly) + { + cout << "bulkWriteVBEntry: transID=" << transID << endl; - for (size_t i = 0; i < lbids.size(); i++) - cout << "bulkWriteVBEntry arg " << i + 1 << ": lbid=" << lbids[i] << " vbOID=" << - vbOID << " vbFBO=" << vbFBOs[i] << endl; - return; - } + for (size_t i = 0; i < lbids.size(); i++) + cout << "bulkWriteVBEntry arg " << i + 1 << ": lbid=" << lbids[i] << " vbOID=" << vbOID + << " vbFBO=" << vbFBOs[i] << endl; + return; + } - err = slave->bulkWriteVBEntry(transID, lbids, vbOID, vbFBOs); - reply << (uint8_t) err; + err = slave->bulkWriteVBEntry(transID, lbids, vbOID, vbFBOs); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_bulkWriteVBEntry() err code is " << err << endl; + cerr << "WorkerComm: do_bulkWriteVBEntry() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_beginVBCopy(ByteStream& msg) { - VER_t transID; - LBIDRange_v ranges; - VBRange_v freeList; - uint32_t tmp32; - uint16_t dbRoot; - int err; - ByteStream reply; + VER_t transID; + LBIDRange_v ranges; + VBRange_v freeList; + uint32_t tmp32; + uint16_t dbRoot; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_beginVBCopy()" << endl; + cerr << "WorkerComm: do_beginVBCopy()" << endl; #endif - msg >> tmp32; - transID = tmp32; - msg >> dbRoot; - deserializeVector(msg, ranges); + msg >> tmp32; + transID = tmp32; + msg >> dbRoot; + deserializeVector(msg, ranges); - if (printOnly) - { - cout << "beginVBCopy: transID=" << transID << " dbRoot=" << dbRoot << " size=" - << ranges.size() << " ranges..." << endl; + if (printOnly) + { + cout << "beginVBCopy: transID=" << transID << " dbRoot=" << dbRoot << " size=" << ranges.size() + << " ranges..." << endl; - for (uint32_t i = 0; i < ranges.size(); i++) - cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; + for (uint32_t i = 0; i < ranges.size(); i++) + cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; - return; - } + return; + } - err = slave->beginVBCopy(transID, dbRoot, ranges, freeList, firstSlave && !standalone); - reply << (uint8_t) err; + err = slave->beginVBCopy(transID, dbRoot, ranges, freeList, firstSlave && !standalone); + reply << (uint8_t)err; - if (err == ERR_OK) - serializeVector(reply, freeList); + if (err == ERR_OK) + serializeVector(reply, freeList); #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_beginVBCopy() err code is " << err << endl; + cerr << "WorkerComm: do_beginVBCopy() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_endVBCopy(ByteStream& msg) { - VER_t transID; - LBIDRange_v ranges; - uint32_t tmp; - int err; - ByteStream reply; + VER_t transID; + LBIDRange_v ranges; + uint32_t tmp; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_endVBCopy()" << endl; + cerr << "WorkerComm: do_endVBCopy()" << endl; #endif - msg >> tmp; - transID = tmp; - deserializeVector(msg, ranges); + msg >> tmp; + transID = tmp; + deserializeVector(msg, ranges); - if (printOnly) - { - cout << "endVBCopy: transID=" << transID << " size=" << ranges.size() << - " ranges..." << endl; + if (printOnly) + { + cout << "endVBCopy: transID=" << transID << " size=" << ranges.size() << " ranges..." << endl; - for (uint32_t i = 0; i < ranges.size(); i++) - cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; + for (uint32_t i = 0; i < ranges.size(); i++) + cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; - return; - } + return; + } - err = slave->endVBCopy(transID, ranges); - reply << (uint8_t) err; + err = slave->endVBCopy(transID, ranges); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_endVBCopy() err code is " << err << endl; + cerr << "WorkerComm: do_endVBCopy() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_vbRollback1(ByteStream& msg) { - VER_t transID; - LBIDRange_v lbidList; - uint32_t tmp; - int err; - ByteStream reply; + VER_t transID; + LBIDRange_v lbidList; + uint32_t tmp; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_vbRollback1()" << endl; + cerr << "WorkerComm: do_vbRollback1()" << endl; #endif - msg >> tmp; - transID = tmp; - deserializeVector(msg, lbidList); + msg >> tmp; + transID = tmp; + deserializeVector(msg, lbidList); - if (printOnly) - { - cout << "vbRollback1: transID=" << transID << " size=" << lbidList.size() << - " lbids..." << endl; + if (printOnly) + { + cout << "vbRollback1: transID=" << transID << " size=" << lbidList.size() << " lbids..." << endl; - for (uint32_t i = 0; i < lbidList.size(); i++) - cout << " start=" << lbidList[i].start << " size=" << lbidList[i].size << endl; + for (uint32_t i = 0; i < lbidList.size(); i++) + cout << " start=" << lbidList[i].start << " size=" << lbidList[i].size << endl; - return; - } + return; + } - err = slave->vbRollback(transID, lbidList, firstSlave && !standalone); - reply << (uint8_t) err; + err = slave->vbRollback(transID, lbidList, firstSlave && !standalone); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_vbRollback1() err code is " << err << endl; + cerr << "WorkerComm: do_vbRollback1() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_vbRollback2(ByteStream& msg) { - VER_t transID; - vector lbidList; - uint32_t tmp; - int err; - ByteStream reply; + VER_t transID; + vector lbidList; + uint32_t tmp; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_vbRollback2()" << endl; + cerr << "WorkerComm: do_vbRollback2()" << endl; #endif - msg >> tmp; - transID = tmp; - deserializeVector(msg, lbidList); + msg >> tmp; + transID = tmp; + deserializeVector(msg, lbidList); - if (printOnly) - { - cout << "vbRollback2: transID=" << transID << " size=" << lbidList.size() << - " lbids..." << endl; + if (printOnly) + { + cout << "vbRollback2: transID=" << transID << " size=" << lbidList.size() << " lbids..." << endl; - for (uint32_t i = 0; i < lbidList.size(); i++) - cout << " " << lbidList[i] << endl; + for (uint32_t i = 0; i < lbidList.size(); i++) + cout << " " << lbidList[i] << endl; - return; - } + return; + } - err = slave->vbRollback(transID, lbidList, firstSlave && !standalone); - reply << (uint8_t) err; + err = slave->vbRollback(transID, lbidList, firstSlave && !standalone); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_vbRollback2() err code is " << err << endl; + cerr << "WorkerComm: do_vbRollback2() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_vbCommit(ByteStream& msg) { - VER_t transID; - uint32_t tmp; - int err; - ByteStream reply; + VER_t transID; + uint32_t tmp; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_vbCommit()" << endl; + cerr << "WorkerComm: do_vbCommit()" << endl; #endif - msg >> tmp; - transID = tmp; + msg >> tmp; + transID = tmp; - if (printOnly) - { - cout << "vbCommit: transID=" << transID << endl; - return; - } + if (printOnly) + { + cout << "vbCommit: transID=" << transID << endl; + return; + } - err = slave->vbCommit(transID); - reply << (uint8_t) err; + err = slave->vbCommit(transID); + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_vbCommit() err code is " << err << endl; + cerr << "WorkerComm: do_vbCommit() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_undo() { #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_undo()" << endl; + cerr << "WorkerComm: do_undo()" << endl; #endif - if (printOnly) - { - cout << "undoChanges" << endl; - return; - } + if (printOnly) + { + cout << "undoChanges" << endl; + return; + } - slave->undoChanges(); - takeSnapshot = false; - doSaveDelta = false; + slave->undoChanges(); + takeSnapshot = false; + doSaveDelta = false; } void SlaveComm::do_confirm() { #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_confirm()" << endl; + cerr << "WorkerComm: do_confirm()" << endl; #endif - if (printOnly) + if (printOnly) + { + cout << "confirmChanges" << endl; + return; + } + + if (firstSlave && doSaveDelta && (journalCount < snapshotInterval || snapshotInterval < 0)) + { + doSaveDelta = false; + saveDelta(); + } + + slave->confirmChanges(); + + string tmp = savefile + "_current"; + + if (firstSlave && (takeSnapshot || (journalCount >= snapshotInterval && snapshotInterval >= 0))) + { + if (!currentSaveFile) { - cout << "confirmChanges" << endl; - return; + currentSaveFile = + IDBDataFile::open(IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0); } - if (firstSlave && doSaveDelta && (journalCount < snapshotInterval || snapshotInterval < 0)) + if (currentSaveFile == NULL) { - doSaveDelta = false; - saveDelta(); + ostringstream os; + os << "WorkerComm: failed to open the current savefile. errno: " << strerror(errno); + log(os.str()); + throw runtime_error(os.str()); } - slave->confirmChanges(); - - string tmp = savefile + "_current"; - - if (firstSlave && (takeSnapshot || - (journalCount >= snapshotInterval && snapshotInterval >= 0))) - { - if (!currentSaveFile) - { - currentSaveFile = IDBDataFile::open( - IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0); - } - - if (currentSaveFile == NULL) - { - ostringstream os; - os << "WorkerComm: failed to open the current savefile. errno: " - << strerror(errno); - log(os.str()); - throw runtime_error(os.str()); - } - - - tmp = savefile + (saveFileToggle ? 'A' : 'B'); - slave->saveState(tmp); + tmp = savefile + (saveFileToggle ? 'A' : 'B'); + slave->saveState(tmp); #ifndef _MSC_VER - tmp += '\n'; + tmp += '\n'; #endif - int err = 0; + int err = 0; - // MCOL-1558. Make the _current file relative to DBRMRoot. - string relative = tmp.substr(tmp.find_last_of('/') + 1); - err = currentSaveFile->write(relative.c_str(), relative.length()); + // MCOL-1558. Make the _current file relative to DBRMRoot. + string relative = tmp.substr(tmp.find_last_of('/') + 1); + err = currentSaveFile->write(relative.c_str(), relative.length()); - if (err < (int) relative.length()) - { - ostringstream os; - os << "WorkerComm: currentfile write() returned " << err - << " file pointer is " << currentSaveFile; + if (err < (int)relative.length()) + { + ostringstream os; + os << "WorkerComm: currentfile write() returned " << err << " file pointer is " << currentSaveFile; - if (err < 0) - os << " errno: " << strerror(errno); + if (err < 0) + os << " errno: " << strerror(errno); - log(os.str()); - } - - currentSaveFile->flush(); - delete currentSaveFile; - currentSaveFile = NULL; - saveFileToggle = !saveFileToggle; - - delete journalh; - journalh = IDBDataFile::open( - IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), journalName.c_str(), "w+b", 0); - - if (!journalh) - throw runtime_error("Could not open the BRM journal for writing!"); - - takeSnapshot = false; - doSaveDelta = false; - journalCount = 0; + log(os.str()); } + + currentSaveFile->flush(); + delete currentSaveFile; + currentSaveFile = NULL; + saveFileToggle = !saveFileToggle; + + delete journalh; + journalh = IDBDataFile::open(IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), + journalName.c_str(), "w+b", 0); + + if (!journalh) + throw runtime_error("Could not open the BRM journal for writing!"); + + takeSnapshot = false; + doSaveDelta = false; + journalCount = 0; + } } void SlaveComm::do_flushInodeCache() { - ByteStream reply; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_flushInodeCache()" << endl; + cerr << "WorkerComm: do_flushInodeCache()" << endl; #endif - if (printOnly) - { - cout << "flushInodeCache" << endl; - return; - } + if (printOnly) + { + cout << "flushInodeCache" << endl; + return; + } #ifdef __linux__ #ifdef USE_VERY_COMPLEX_DROP_CACHES - double elapsedTime = 0.0; - char msgChString[100]; - struct timespec tm1, tm2; - clock_gettime(CLOCK_REALTIME, &tm1); + double elapsedTime = 0.0; + char msgChString[100]; + struct timespec tm1, tm2; + clock_gettime(CLOCK_REALTIME, &tm1); - int fd; - fd = open("/proc/sys/vm/drop_caches", O_WRONLY); + int fd; + fd = open("/proc/sys/vm/drop_caches", O_WRONLY); - if (fd >= 0) + if (fd >= 0) + { + ssize_t writeCnt = write(fd, "3\n", 2); + clock_gettime(CLOCK_REALTIME, &tm2); + timespec_sub(tm1, tm2, elapsedTime); + + if (writeCnt == 2) { - ssize_t writeCnt = write(fd, "3\n", 2); - clock_gettime(CLOCK_REALTIME, &tm2); - timespec_sub(tm1, tm2, elapsedTime); - - if (writeCnt == 2) - { - snprintf(msgChString, sizeof(msgChString), - "WorkerNode updating drop_caches to flush cache; " - "elapsedTime-%f sec", elapsedTime); - log(string(msgChString), - logging::LOG_TYPE_DEBUG); - } - else - { - snprintf(msgChString, sizeof(msgChString), - "WorkerNode unable to update drop_caches and flush cache; " - "elapsedTime-%f sec", elapsedTime); - log_errno(string(msgChString), - logging::LOG_TYPE_WARNING); - } - - close(fd); + snprintf(msgChString, sizeof(msgChString), + "WorkerNode updating drop_caches to flush cache; " + "elapsedTime-%f sec", + elapsedTime); + log(string(msgChString), logging::LOG_TYPE_DEBUG); } else { - clock_gettime(CLOCK_REALTIME, &tm2); - timespec_sub(tm1, tm2, elapsedTime); - snprintf(msgChString, sizeof(msgChString), - "WorkerNode unable to open drop_caches and flush cache; " - "elapsedTime-%f sec", elapsedTime); - log_errno(string(msgChString), - logging::LOG_TYPE_WARNING); + snprintf(msgChString, sizeof(msgChString), + "WorkerNode unable to update drop_caches and flush cache; " + "elapsedTime-%f sec", + elapsedTime); + log_errno(string(msgChString), logging::LOG_TYPE_WARNING); } + close(fd); + } + else + { + clock_gettime(CLOCK_REALTIME, &tm2); + timespec_sub(tm1, tm2, elapsedTime); + snprintf(msgChString, sizeof(msgChString), + "WorkerNode unable to open drop_caches and flush cache; " + "elapsedTime-%f sec", + elapsedTime); + log_errno(string(msgChString), logging::LOG_TYPE_WARNING); + } + #else - int fd = -1; + int fd = -1; - if ((fd = open("/proc/sys/vm/drop_caches", O_WRONLY)) >= 0) + if ((fd = open("/proc/sys/vm/drop_caches", O_WRONLY)) >= 0) + { + ssize_t written = write(fd, "3\n", 2); + int rc = close(fd); + if (!written || rc) { - ssize_t written = write(fd, "3\n", 2); - int rc = close(fd); - if ( !written || rc ) - { - std::cerr << "Could not write into or close /proc/sys/vm/drop_caches" << std::endl; - } + std::cerr << "Could not write into or close /proc/sys/vm/drop_caches" << std::endl; } + } #endif #endif - reply << (uint8_t) ERR_OK; + reply << (uint8_t)ERR_OK; - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); } void SlaveComm::do_clear() { - int err; - ByteStream reply; + int err; + ByteStream reply; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_clear()" << endl; + cerr << "WorkerComm: do_clear()" << endl; #endif - if (printOnly) - { - cout << "clear" << endl; - return; - } + if (printOnly) + { + cout << "clear" << endl; + return; + } - err = slave->clear(); + err = slave->clear(); - if (err) - throw runtime_error("Clear failed."); + if (err) + throw runtime_error("Clear failed."); - /* This doesn't get confirmed, so we have to save its delta here */ - if (firstSlave) - saveDelta(); + /* This doesn't get confirmed, so we have to save its delta here */ + if (firstSlave) + saveDelta(); - reply << (uint8_t) (err == 0 ? ERR_OK : ERR_FAILURE); + reply << (uint8_t)(err == 0 ? ERR_OK : ERR_FAILURE); - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); } int SlaveComm::replayJournal(string prefix) { - ByteStream cmd; - uint32_t len; - int ret = 0; + ByteStream cmd; + uint32_t len; + int ret = 0; - // @Bug 2667+ - // Fix for issue where load_brm was using the journal file from DBRMRoot instead of the one from the command line - // argument. + // @Bug 2667+ + // Fix for issue where load_brm was using the journal file from DBRMRoot instead of the one from the command + // line argument. - // If A or B files are being loaded, strip off the A or B for the journal file name as there is only one journal file. - // For example, if prefix is "/usr/local/mariadb/columnstore/data1/systemFiles/dbrm/BRM_savesA" the journal file name will be - // "/usr/local/mariadb/columnstore/data1/systemFiles/dbrm/BRM_saves_journal". + // If A or B files are being loaded, strip off the A or B for the journal file name as there is only one + // journal file. For example, if prefix is + // "/usr/local/mariadb/columnstore/data1/systemFiles/dbrm/BRM_savesA" the journal file name will be + // "/usr/local/mariadb/columnstore/data1/systemFiles/dbrm/BRM_saves_journal". - string tmp = prefix.substr(prefix.length() - 1); - string fName; + string tmp = prefix.substr(prefix.length() - 1); + string fName; - if ((tmp.compare("A") == 0) || (tmp.compare("B") == 0)) - { - fName = prefix.substr(0, prefix.length() - 1) + "_journal"; - } + if ((tmp.compare("A") == 0) || (tmp.compare("B") == 0)) + { + fName = prefix.substr(0, prefix.length() - 1) + "_journal"; + } #ifdef _MSC_VER - else if (tmp == "a" || tmp == "b") - fName = prefix.substr(0, prefix.length() - 1) + "_journal"; + else if (tmp == "a" || tmp == "b") + fName = prefix.substr(0, prefix.length() - 1) + "_journal"; #endif - else + else + { + fName = prefix + "_journal"; + } + + const char* filename = fName.c_str(); + + IDBDataFile* journalf = + IDBDataFile::open(IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0); + + if (!journalf) + { + cout << "Error opening journal file " << fName << endl; + return -1; + } + + if (journalf->size() == 0) // empty file, nothing to replay + return 0; + + ssize_t readIn = 0; + + do + { + readIn = journalf->read((char*)&len, sizeof(len)); + + if (readIn > 0) { - fName = prefix + "_journal"; - } + cmd.needAtLeast(len); + readIn = journalf->read((char*)cmd.getInputPtr(), len); + cmd.advanceInputPtr(len); - const char* filename = fName.c_str(); - - IDBDataFile* journalf = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0); - - if (!journalf) - { - cout << "Error opening journal file " << fName << endl; + try + { + processCommand(cmd); + } + catch (exception& e) + { + cout << e.what() << " Journal replay was incomplete." << endl; + slave->undoChanges(); return -1; + } + + slave->confirmChanges(); + cmd.restart(); + ret++; } + } while (readIn > 0); - if (journalf->size() == 0) // empty file, nothing to replay - return 0; - - ssize_t readIn = 0; - - do - { - readIn = journalf->read((char*) &len, sizeof(len)); - - if (readIn > 0) - { - - cmd.needAtLeast(len); - readIn = journalf->read((char*) cmd.getInputPtr(), len); - cmd.advanceInputPtr(len); - - try - { - processCommand(cmd); - } - catch (exception& e) - { - cout << e.what() << " Journal replay was incomplete." << endl; - slave->undoChanges(); - return -1; - } - - slave->confirmChanges(); - cmd.restart(); - ret++; - } - } while (readIn > 0); - - return ret; + return ret; } void SlaveComm::do_takeSnapshot() { - ByteStream reply; + ByteStream reply; - if (printOnly) - { - cout << "takeSnapshot" << endl; - return; - } + if (printOnly) + { + cout << "takeSnapshot" << endl; + return; + } - takeSnapshot = true; - do_confirm(); - reply << (uint8_t) 0; + takeSnapshot = true; + do_confirm(); + reply << (uint8_t)0; - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); } void SlaveComm::saveDelta() { - try - { - uint32_t len = delta.length(); + try + { + uint32_t len = delta.length(); - journalh->seek(0, SEEK_END); - journalh->write((const char*) &len, sizeof(len)); - journalh->write((const char*) delta.buf(), delta.length()); - journalh->flush(); - journalCount++; - } - catch (exception& e) - { - cerr << "Journal write error: " << e.what() << endl; - throw; - } + journalh->seek(0, SEEK_END); + journalh->write((const char*)&len, sizeof(len)); + journalh->write((const char*)delta.buf(), delta.length()); + journalh->flush(); + journalCount++; + } + catch (exception& e) + { + cerr << "Journal write error: " << e.what() << endl; + throw; + } } int SlaveComm::printJournal(string prefix) { - int ret; - printOnly = true; - ret = replayJournal(prefix); - printOnly = false; - return ret; + int ret; + printOnly = true; + ret = replayJournal(prefix); + printOnly = false; + return ret; } void SlaveComm::do_ownerCheck(ByteStream& msg) { - string processName; - uint32_t pid; - ByteStream::byte rb = 0; + string processName; + uint32_t pid; + ByteStream::byte rb = 0; - msg >> processName >> pid; - idbassert(msg.length() == 0); + msg >> processName >> pid; + idbassert(msg.length() == 0); - if (standalone) - return; + if (standalone) + return; - if (processExists(pid, processName)) - rb++; + if (processExists(pid, processName)) + rb++; - ByteStream reply; - reply << rb; - master.write(reply); + ByteStream reply; + reply << rb; + master.write(reply); } -//FIXME: needs to be refactored along with SessionManagerServer::lookupProcessStatus() +// FIXME: needs to be refactored along with SessionManagerServer::lookupProcessStatus() #if defined(__linux__) bool SlaveComm::processExists(const uint32_t pid, const string& pname) { - string stat; - ostringstream procFileName; - ostringstream statProcessField; - ifstream in; - string::size_type pos; - ByteStream reply; - char buf[2048]; + string stat; + ostringstream procFileName; + ostringstream statProcessField; + ifstream in; + string::size_type pos; + ByteStream reply; + char buf[2048]; - procFileName << "/proc/" << pid << "/stat"; - in.open(procFileName.str().c_str()); + procFileName << "/proc/" << pid << "/stat"; + in.open(procFileName.str().c_str()); - if (!in) - { - return false; - } + if (!in) + { + return false; + } - statProcessField << "(" << pname << ")"; + statProcessField << "(" << pname << ")"; - in.getline(buf, 1024); - stat = buf; - pos = stat.find(statProcessField.str()); + in.getline(buf, 1024); + stat = buf; + pos = stat.find(statProcessField.str()); - if (pos == string::npos) - { - return false; - } + if (pos == string::npos) + { + return false; + } - return true; + return true; } - #elif defined(_MSC_VER) -//FIXME +// FIXME bool SlaveComm::processExists(const uint32_t pid, const string& pname) { - boost::mutex::scoped_lock lk(fPidMemLock); + boost::mutex::scoped_lock lk(fPidMemLock); - if (!fPids) - fPids = (DWORD*)malloc(fMaxPids * sizeof(DWORD)); + if (!fPids) + fPids = (DWORD*)malloc(fMaxPids * sizeof(DWORD)); - DWORD needed = 0; + DWORD needed = 0; + + if (EnumProcesses(fPids, fMaxPids * sizeof(DWORD), &needed) == 0) + return false; + + while (needed == fMaxPids * sizeof(DWORD)) + { + fMaxPids *= 2; + fPids = (DWORD*)realloc(fPids, fMaxPids * sizeof(DWORD)); if (EnumProcesses(fPids, fMaxPids * sizeof(DWORD), &needed) == 0) - return false; + return false; + } - while (needed == fMaxPids * sizeof(DWORD)) + DWORD numPids = needed / sizeof(DWORD); + + for (DWORD i = 0; i < numPids; i++) + { + if (fPids[i] == pid) { - fMaxPids *= 2; - fPids = (DWORD*)realloc(fPids, fMaxPids * sizeof(DWORD)); + TCHAR szProcessName[MAX_PATH] = TEXT(""); - if (EnumProcesses(fPids, fMaxPids * sizeof(DWORD), &needed) == 0) - return false; + // Get a handle to the process. + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, fPids[i]); + + // Get the process name. + if (hProcess != NULL) + { + HMODULE hMod; + DWORD cbNeeded; + + if (EnumProcessModules(hProcess, &hMod, sizeof(hMod), &cbNeeded)) + GetModuleBaseName(hProcess, hMod, szProcessName, sizeof(szProcessName) / sizeof(TCHAR)); + + CloseHandle(hProcess); + + if (pname == szProcessName) + return true; + } } + } - DWORD numPids = needed / sizeof(DWORD); - - for (DWORD i = 0; i < numPids; i++) - { - if (fPids[i] == pid) - { - TCHAR szProcessName[MAX_PATH] = TEXT(""); - - // Get a handle to the process. - HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | - PROCESS_VM_READ, - FALSE, fPids[i]); - - // Get the process name. - if (hProcess != NULL) - { - HMODULE hMod; - DWORD cbNeeded; - - if (EnumProcessModules(hProcess, &hMod, sizeof(hMod), &cbNeeded)) - GetModuleBaseName(hProcess, hMod, szProcessName, - sizeof(szProcessName) / sizeof(TCHAR)); - - CloseHandle(hProcess); - - if (pname == szProcessName) - return true; - } - } - } - - return false; + return false; } #elif defined(__FreeBSD__) -//FIXME +// FIXME bool SlaveComm::processExists(const uint32_t pid, const string& pname) { - return false; + return false; } #else #error Need to port processExists() @@ -2477,72 +2365,71 @@ bool SlaveComm::processExists(const uint32_t pid, const string& pname) void SlaveComm::do_dmlLockLBIDRanges(ByteStream& msg) { - ByteStream reply; - vector ranges; - int txnID; - uint32_t tmp32; - int err; + ByteStream reply; + vector ranges; + int txnID; + uint32_t tmp32; + int err; - deserializeVector(msg, ranges); - msg >> tmp32; - assert(msg.length() == 0); - txnID = (int) tmp32; + deserializeVector(msg, ranges); + msg >> tmp32; + assert(msg.length() == 0); + txnID = (int)tmp32; - if (printOnly) - { - cout << "dmlLockLBIDRanges: transID=" << txnID << " size=" - << ranges.size() << " ranges..." << endl; + if (printOnly) + { + cout << "dmlLockLBIDRanges: transID=" << txnID << " size=" << ranges.size() << " ranges..." << endl; - for (uint32_t i = 0; i < ranges.size(); i++) - cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; + for (uint32_t i = 0; i < ranges.size(); i++) + cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; - return; - } + return; + } - err = slave->dmlLockLBIDRanges(ranges, txnID); + err = slave->dmlLockLBIDRanges(ranges, txnID); - reply << (uint8_t) err; + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_dmlLockLBIDRanges() err code is " << err << endl; + cerr << "WorkerComm: do_dmlLockLBIDRanges() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } void SlaveComm::do_dmlReleaseLBIDRanges(ByteStream& msg) { - ByteStream reply; - vector ranges; - int err; + ByteStream reply; + vector ranges; + int err; - deserializeVector(msg, ranges); + deserializeVector(msg, ranges); - if (printOnly) - { - cout << "dmlLockLBIDRanges: size=" << ranges.size() << " ranges..." << endl; + if (printOnly) + { + cout << "dmlLockLBIDRanges: size=" << ranges.size() << " ranges..." << endl; - for (uint32_t i = 0; i < ranges.size(); i++) - cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; + for (uint32_t i = 0; i < ranges.size(); i++) + cout << " start=" << ranges[i].start << " size=" << ranges[i].size << endl; - return; - } + return; + } - err = slave->dmlReleaseLBIDRanges(ranges); + err = slave->dmlReleaseLBIDRanges(ranges); - reply << (uint8_t) err; + reply << (uint8_t)err; #ifdef BRM_VERBOSE - cerr << "WorkerComm: do_dmlReleaseLBIDRanges() err code is " << err << endl; + cerr << "WorkerComm: do_dmlReleaseLBIDRanges() err code is " << err << endl; #endif - if (!standalone) - master.write(reply); + if (!standalone) + master.write(reply); - doSaveDelta = true; + doSaveDelta = true; } -} +} // namespace BRM // vim:ts=4 sw=4: diff --git a/versioning/BRM/slavecomm.h b/versioning/BRM/slavecomm.h index a7b7b44f3..5d671655f 100644 --- a/versioning/BRM/slavecomm.h +++ b/versioning/BRM/slavecomm.h @@ -49,96 +49,93 @@ namespace idbdatafile class IDBDataFile; } - namespace BRM { - class SlaveComm { -public: - /** Use this ctor to instantiate a standalone SlaveComm ie, for replaying - the journal. Should be used only by load_brm. */ - EXPORT SlaveComm(); + public: + /** Use this ctor to instantiate a standalone SlaveComm ie, for replaying + the journal. Should be used only by load_brm. */ + EXPORT SlaveComm(); - /** Use this ctor to have it connected to the rest of the DBRM system */ - EXPORT SlaveComm(std::string hostname, SlaveDBRMNode* s); //hostname = 'DBRM_WorkerN' - EXPORT ~SlaveComm(); + /** Use this ctor to have it connected to the rest of the DBRM system */ + EXPORT SlaveComm(std::string hostname, SlaveDBRMNode* s); // hostname = 'DBRM_WorkerN' + EXPORT ~SlaveComm(); - EXPORT void run(); - EXPORT void stop(); - EXPORT void reset(); + EXPORT void run(); + EXPORT void stop(); + EXPORT void reset(); - EXPORT int replayJournal(std::string prefix); - EXPORT int printJournal(std::string prefix); + EXPORT int replayJournal(std::string prefix); + EXPORT int printJournal(std::string prefix); -private: + private: + SlaveComm& operator=(const SlaveComm& s); - SlaveComm& operator=(const SlaveComm& s); + void processCommand(messageqcpp::ByteStream& msg); - void processCommand(messageqcpp::ByteStream& msg); + void do_createStripeColumnExtents(messageqcpp::ByteStream& msg); + void do_createColumnExtent_DBroot(messageqcpp::ByteStream& msg); + void do_createColumnExtentExactFile(messageqcpp::ByteStream& msg); + void do_createDictStoreExtent(messageqcpp::ByteStream& msg); + void do_rollbackColumnExtents_DBroot(messageqcpp::ByteStream& msg); + void do_deleteEmptyColExtents(messageqcpp::ByteStream& msg); + void do_deleteEmptyDictStoreExtents(messageqcpp::ByteStream& msg); + void do_rollbackDictStoreExtents_DBroot(messageqcpp::ByteStream& msg); + void do_deleteOID(messageqcpp::ByteStream& msg); + void do_deleteOIDs(messageqcpp::ByteStream& msg); + void do_setLocalHWM(messageqcpp::ByteStream& msg); + void do_bulkSetHWM(messageqcpp::ByteStream& msg); + void do_bulkSetHWMAndCP(messageqcpp::ByteStream& msg); + void do_writeVBEntry(messageqcpp::ByteStream& msg); + void do_bulkWriteVBEntry(messageqcpp::ByteStream& msg); + void do_beginVBCopy(messageqcpp::ByteStream& msg); + void do_endVBCopy(messageqcpp::ByteStream& msg); + void do_vbRollback1(messageqcpp::ByteStream& msg); + void do_vbRollback2(messageqcpp::ByteStream& msg); + void do_vbCommit(messageqcpp::ByteStream& msg); + void do_markInvalid(messageqcpp::ByteStream& msg); + void do_markManyExtentsInvalid(messageqcpp::ByteStream& msg); + void do_mergeExtentsMaxMin(messageqcpp::ByteStream& msg); + void do_setExtentMaxMin(messageqcpp::ByteStream& msg); + void do_setExtentsMaxMin(messageqcpp::ByteStream& msg); // bug 1970 + void do_deletePartition(messageqcpp::ByteStream& msg); + void do_markPartitionForDeletion(messageqcpp::ByteStream& msg); + void do_markAllPartitionForDeletion(messageqcpp::ByteStream& msg); + void do_restorePartition(messageqcpp::ByteStream& msg); + void do_dmlLockLBIDRanges(messageqcpp::ByteStream& msg); + void do_dmlReleaseLBIDRanges(messageqcpp::ByteStream& msg); + void do_deleteDBRoot(messageqcpp::ByteStream& msg); + void do_bulkUpdateDBRoot(messageqcpp::ByteStream& msg); - void do_createStripeColumnExtents(messageqcpp::ByteStream& msg); - void do_createColumnExtent_DBroot(messageqcpp::ByteStream& msg); - void do_createColumnExtentExactFile(messageqcpp::ByteStream& msg); - void do_createDictStoreExtent(messageqcpp::ByteStream& msg); - void do_rollbackColumnExtents_DBroot(messageqcpp::ByteStream& msg); - void do_deleteEmptyColExtents(messageqcpp::ByteStream& msg); - void do_deleteEmptyDictStoreExtents(messageqcpp::ByteStream& msg); - void do_rollbackDictStoreExtents_DBroot(messageqcpp::ByteStream& msg); - void do_deleteOID(messageqcpp::ByteStream& msg); - void do_deleteOIDs(messageqcpp::ByteStream& msg); - void do_setLocalHWM(messageqcpp::ByteStream& msg); - void do_bulkSetHWM(messageqcpp::ByteStream& msg); - void do_bulkSetHWMAndCP(messageqcpp::ByteStream& msg); - void do_writeVBEntry(messageqcpp::ByteStream& msg); - void do_bulkWriteVBEntry(messageqcpp::ByteStream& msg); - void do_beginVBCopy(messageqcpp::ByteStream& msg); - void do_endVBCopy(messageqcpp::ByteStream& msg); - void do_vbRollback1(messageqcpp::ByteStream& msg); - void do_vbRollback2(messageqcpp::ByteStream& msg); - void do_vbCommit(messageqcpp::ByteStream& msg); - void do_markInvalid(messageqcpp::ByteStream& msg); - void do_markManyExtentsInvalid(messageqcpp::ByteStream& msg); - void do_mergeExtentsMaxMin(messageqcpp::ByteStream& msg); - void do_setExtentMaxMin(messageqcpp::ByteStream& msg); - void do_setExtentsMaxMin(messageqcpp::ByteStream& msg); // bug 1970 - void do_deletePartition(messageqcpp::ByteStream& msg); - void do_markPartitionForDeletion(messageqcpp::ByteStream& msg); - void do_markAllPartitionForDeletion(messageqcpp::ByteStream& msg); - void do_restorePartition(messageqcpp::ByteStream& msg); - void do_dmlLockLBIDRanges(messageqcpp::ByteStream& msg); - void do_dmlReleaseLBIDRanges(messageqcpp::ByteStream& msg); - void do_deleteDBRoot(messageqcpp::ByteStream& msg); - void do_bulkUpdateDBRoot(messageqcpp::ByteStream& msg); + void do_undo(); + void do_confirm(); + void do_flushInodeCache(); + void do_clear(); + void do_ownerCheck(messageqcpp::ByteStream& msg); + void do_takeSnapshot(); + void saveDelta(); + bool processExists(const uint32_t pid, const std::string& pname); - void do_undo(); - void do_confirm(); - void do_flushInodeCache(); - void do_clear(); - void do_ownerCheck(messageqcpp::ByteStream& msg); - void do_takeSnapshot(); - void saveDelta(); - bool processExists(const uint32_t pid, const std::string& pname); - - messageqcpp::MessageQueueServer* server; - messageqcpp::IOSocket master; - SlaveDBRMNode* slave; - std::string savefile; - bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly; - messageqcpp::ByteStream delta; - idbdatafile::IDBDataFile* currentSaveFile; - std::string journalName; - idbdatafile::IDBDataFile* journalh; - int64_t snapshotInterval, journalCount; - struct timespec MSG_TIMEOUT; + messageqcpp::MessageQueueServer* server; + messageqcpp::IOSocket master; + SlaveDBRMNode* slave; + std::string savefile; + bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly; + messageqcpp::ByteStream delta; + idbdatafile::IDBDataFile* currentSaveFile; + std::string journalName; + idbdatafile::IDBDataFile* journalh; + int64_t snapshotInterval, journalCount; + struct timespec MSG_TIMEOUT; #ifdef _MSC_VER - boost::mutex fPidMemLock; - DWORD* fPids; - DWORD fMaxPids; + boost::mutex fPidMemLock; + DWORD* fPids; + DWORD fMaxPids; #endif }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/slavedbrmnode.cpp b/versioning/BRM/slavedbrmnode.cpp index f163ea31f..ae7bb1d11 100644 --- a/versioning/BRM/slavedbrmnode.cpp +++ b/versioning/BRM/slavedbrmnode.cpp @@ -50,18 +50,16 @@ using namespace logging; namespace BRM { - - SlaveDBRMNode::SlaveDBRMNode() throw() { - locked[0] = false; - locked[1] = false; - locked[2] = false; + locked[0] = false; + locked[1] = false; + locked[2] = false; } SlaveDBRMNode::SlaveDBRMNode(const SlaveDBRMNode& brm) { - throw logic_error("WorkerDBRMNode: Don't use the copy constructor."); + throw logic_error("WorkerDBRMNode: Don't use the copy constructor."); } SlaveDBRMNode::~SlaveDBRMNode() throw() @@ -70,151 +68,127 @@ SlaveDBRMNode::~SlaveDBRMNode() throw() SlaveDBRMNode& SlaveDBRMNode::operator=(const SlaveDBRMNode& brm) { - throw logic_error("WorkerDBRMNode: Don't use the = operator."); + throw logic_error("WorkerDBRMNode: Don't use the = operator."); } int SlaveDBRMNode::lookup(OID_t oid, LBIDRange_v& lbidList) throw() { - - try - { - em.lookup(oid, lbidList); - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.lookup(oid, lbidList); + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } //------------------------------------------------------------------------------ // Create a "stripe" of column extents for the specified column OIDs and DBRoot. //------------------------------------------------------------------------------ -int SlaveDBRMNode::createStripeColumnExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - std::vector& extents) throw() +int SlaveDBRMNode::createStripeColumnExtents(const std::vector& cols, + uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + std::vector& extents) throw() { - try - { - em.createStripeColumnExtents(cols, dbRoot, - partitionNum, segmentNum, extents ); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.createStripeColumnExtents(cols, dbRoot, partitionNum, segmentNum, extents); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Create an extent for the specified OID and DBRoot. //------------------------------------------------------------------------------ -int SlaveDBRMNode::createColumnExtent_DBroot(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& partitionNum, - uint16_t& segmentNum, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) throw() +int SlaveDBRMNode::createColumnExtent_DBroot(OID_t oid, uint32_t colWidth, uint16_t dbRoot, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& partitionNum, uint16_t& segmentNum, LBID_t& lbid, + int& allocdSize, uint32_t& startBlockOffset) throw() { - try - { - em.createColumnExtent_DBroot(oid, colWidth, dbRoot, colDataType, - partitionNum, segmentNum, lbid, allocdSize, startBlockOffset); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.createColumnExtent_DBroot(oid, colWidth, dbRoot, colDataType, partitionNum, segmentNum, lbid, + allocdSize, startBlockOffset); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Create extent for the exact segment file specified by the requested // OID, DBRoot, partition, and segment. //------------------------------------------------------------------------------ -int SlaveDBRMNode::createColumnExtentExactFile(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) throw() +int SlaveDBRMNode::createColumnExtentExactFile(OID_t oid, uint32_t colWidth, uint16_t dbRoot, + uint32_t partitionNum, uint16_t segmentNum, + execplan::CalpontSystemCatalog::ColDataType colDataType, + LBID_t& lbid, int& allocdSize, + uint32_t& startBlockOffset) throw() { - try - { - em.createColumnExtentExactFile(oid, colWidth, dbRoot, partitionNum, - segmentNum, colDataType, lbid, allocdSize, startBlockOffset); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.createColumnExtentExactFile(oid, colWidth, dbRoot, partitionNum, segmentNum, colDataType, lbid, + allocdSize, startBlockOffset); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Create a dictionary store extent for the specified OID, dbRoot, partition // number and segment number. //------------------------------------------------------------------------------ -int SlaveDBRMNode::createDictStoreExtent(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - LBID_t& lbid, - int& allocdSize) throw() +int SlaveDBRMNode::createDictStoreExtent(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, LBID_t& lbid, int& allocdSize) throw() { - try - { - em.createDictStoreExtent(oid, dbRoot, partitionNum, - segmentNum, lbid, allocdSize ); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.createDictStoreExtent(oid, dbRoot, partitionNum, segmentNum, lbid, allocdSize); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Rollback (delete) the extents that logically trail the specified extent for // the given OID and DBRoot. Also sets the HWM for the specified extent. //------------------------------------------------------------------------------ -int SlaveDBRMNode::rollbackColumnExtents_DBroot(OID_t oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm) throw() +int SlaveDBRMNode::rollbackColumnExtents_DBroot(OID_t oid, bool bDeleteAll, uint16_t dbRoot, + uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm) throw() { - try - { - em.rollbackColumnExtents_DBroot( - oid, bDeleteAll, dbRoot, partitionNum, segmentNum, hwm); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.rollbackColumnExtents_DBroot(oid, bDeleteAll, dbRoot, partitionNum, segmentNum, hwm); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -223,923 +197,910 @@ int SlaveDBRMNode::rollbackColumnExtents_DBroot(OID_t oid, // Also sets the HWMs for the last extents to be kept in each segment file in // the specified partition. //------------------------------------------------------------------------------ -int SlaveDBRMNode::rollbackDictStoreExtents_DBroot(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - const vector& segNums, - const vector& hwms) throw() +int SlaveDBRMNode::rollbackDictStoreExtents_DBroot(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, + const vector& segNums, + const vector& hwms) throw() { - try - { - em.rollbackDictStoreExtents_DBroot( - oid, dbRoot, partitionNum, segNums, hwms); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.rollbackDictStoreExtents_DBroot(oid, dbRoot, partitionNum, segNums, hwms); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo) throw() { - try - { - em.deleteEmptyColExtents(extentsInfo); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.deleteEmptyColExtents(extentsInfo); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) throw() { - try - { - em.deleteEmptyDictStoreExtents(extentsInfo); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.deleteEmptyDictStoreExtents(extentsInfo); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::deleteOID(OID_t oid) throw() { - LBIDRange_v lbids; - LBIDRange_v::iterator it; - int err; + LBIDRange_v lbids; + LBIDRange_v::iterator it; + int err; - try - { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; - err = lookup(oid, lbids); + err = lookup(oid, lbids); - if (err == -1 || lbids.empty()) - return -1; + if (err == -1 || lbids.empty()) + return -1; - for (it = lbids.begin(); it != lbids.end(); it++) - vss.removeEntriesFromDB(*it, vbbm); + for (it = lbids.begin(); it != lbids.end(); it++) + vss.removeEntriesFromDB(*it, vbbm); - em.deleteOID(oid); - } - catch (exception& e) - { + em.deleteOID(oid); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - cerr << e.what() << endl; - return -1; - } - - return 0; + return 0; } - - -int SlaveDBRMNode::deleteOIDs( const OidsMap_t& oids) throw() +int SlaveDBRMNode::deleteOIDs(const OidsMap_t& oids) throw() { - LBIDRange_v::iterator it; - int err; + LBIDRange_v::iterator it; + int err; - try + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; + + OidsMap_t::const_iterator mapit; + + for (mapit = oids.begin(); mapit != oids.end(); ++mapit) { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; + LBIDRange_v lbids; + err = lookup(mapit->second, lbids); - OidsMap_t::const_iterator mapit; - - for (mapit = oids.begin(); mapit != oids.end(); ++mapit) - { - LBIDRange_v lbids; - err = lookup(mapit->second, lbids); - - if (err == -1) - return -1; - - for (it = lbids.begin(); it != lbids.end(); it++) - vss.removeEntriesFromDB(*it, vbbm); - } - - em.deleteOIDs(oids); - } - catch (exception& e) - { - - cerr << e.what() << endl; + if (err == -1) return -1; + + for (it = lbids.begin(); it != lbids.end(); it++) + vss.removeEntriesFromDB(*it, vbbm); } - return 0; + em.deleteOIDs(oids); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } //------------------------------------------------------------------------------ // Set the HWM for the specified OID, partition, and segment. Used to set the // HWM for a specific dictionary or column segment file. //------------------------------------------------------------------------------ -int SlaveDBRMNode::setLocalHWM(OID_t oid, uint32_t partitionNum, - uint16_t segmentNum, HWM_t hwm, bool firstNode) throw() +int SlaveDBRMNode::setLocalHWM(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm, + bool firstNode) throw() { + try + { + em.setLocalHWM(oid, partitionNum, segmentNum, hwm, firstNode); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - try - { - em.setLocalHWM(oid, partitionNum, segmentNum, hwm, firstNode); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } - - return 0; + return 0; } -int SlaveDBRMNode::bulkSetHWM(const vector& args, VER_t transID, - bool firstNode) throw() +int SlaveDBRMNode::bulkSetHWM(const vector& args, VER_t transID, bool firstNode) throw() { - try - { - if (transID) - vbCommit(transID); + try + { + if (transID) + vbCommit(transID); - em.bulkSetHWM(args, firstNode); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + em.bulkSetHWM(args, firstNode); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::bulkSetHWMAndCP(const vector& hwmArgs, const std::vector& setCPDataArgs, - const std::vector& mergeCPDataArgs, - VER_t transID, bool firstNode) throw() + const std::vector& mergeCPDataArgs, VER_t transID, + bool firstNode) throw() { - uint32_t i; - bool firstCall = true; - CPMaxMin setCPEntry; - CPMaxMinMap_t bulkSetCPMap; - CPMaxMinMerge mergeCPEntry; - CPMaxMinMergeMap_t bulkMergeCPMap; + uint32_t i; + bool firstCall = true; + CPMaxMin setCPEntry; + CPMaxMinMap_t bulkSetCPMap; + CPMaxMinMerge mergeCPEntry; + CPMaxMinMergeMap_t bulkMergeCPMap; - try + try + { + if (transID) + vbCommit(transID); + + for (i = 0; i < hwmArgs.size(); i++) { - if (transID) - vbCommit(transID); - - for (i = 0; i < hwmArgs.size(); i++) - { - em.setLocalHWM(hwmArgs[i].oid, hwmArgs[i].partNum, hwmArgs[i].segNum, hwmArgs[i].hwm, - firstNode, firstCall); - firstCall = false; - } - - if (setCPDataArgs.size() > 0) - { - for (i = 0; i < setCPDataArgs.size(); i++) - { - setCPEntry.isBinaryColumn = setCPDataArgs[i].isBinaryColumn; - if (!setCPEntry.isBinaryColumn) - { - setCPEntry.max = setCPDataArgs[i].max; - setCPEntry.min = setCPDataArgs[i].min; - } - else - { - setCPEntry.bigMax = setCPDataArgs[i].bigMax; - setCPEntry.bigMin = setCPDataArgs[i].bigMin; - } - setCPEntry.seqNum = setCPDataArgs[i].seqNum; - bulkSetCPMap[setCPDataArgs[i].firstLbid] = setCPEntry; - } - - em.setExtentsMaxMin(bulkSetCPMap, firstNode, firstCall); - firstCall = false; - } - - if (mergeCPDataArgs.size() > 0) - { - for (i = 0; i < mergeCPDataArgs.size(); i++) - { - mergeCPEntry.type = mergeCPDataArgs[i].type; - mergeCPEntry.colWidth = mergeCPDataArgs[i].colWidth; - if (mergeCPDataArgs[i].colWidth <= 8) - { - mergeCPEntry.max = mergeCPDataArgs[i].max; - mergeCPEntry.min = mergeCPDataArgs[i].min; - } - else - { - mergeCPEntry.bigMax = mergeCPDataArgs[i].bigMax; - mergeCPEntry.bigMin = mergeCPDataArgs[i].bigMin; - } - mergeCPEntry.newExtent = mergeCPDataArgs[i].newExtent; - mergeCPEntry.seqNum = mergeCPDataArgs[i].seqNum; - bulkMergeCPMap[mergeCPDataArgs[i].startLbid] = mergeCPEntry; - } - - em.mergeExtentsMaxMin(bulkMergeCPMap, firstCall); - firstCall = false; - } - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; + em.setLocalHWM(hwmArgs[i].oid, hwmArgs[i].partNum, hwmArgs[i].segNum, hwmArgs[i].hwm, firstNode, + firstCall); + firstCall = false; } - return 0; + if (setCPDataArgs.size() > 0) + { + for (i = 0; i < setCPDataArgs.size(); i++) + { + setCPEntry.isBinaryColumn = setCPDataArgs[i].isBinaryColumn; + if (!setCPEntry.isBinaryColumn) + { + setCPEntry.max = setCPDataArgs[i].max; + setCPEntry.min = setCPDataArgs[i].min; + } + else + { + setCPEntry.bigMax = setCPDataArgs[i].bigMax; + setCPEntry.bigMin = setCPDataArgs[i].bigMin; + } + setCPEntry.seqNum = setCPDataArgs[i].seqNum; + bulkSetCPMap[setCPDataArgs[i].firstLbid] = setCPEntry; + } + + em.setExtentsMaxMin(bulkSetCPMap, firstNode, firstCall); + firstCall = false; + } + + if (mergeCPDataArgs.size() > 0) + { + for (i = 0; i < mergeCPDataArgs.size(); i++) + { + mergeCPEntry.type = mergeCPDataArgs[i].type; + mergeCPEntry.colWidth = mergeCPDataArgs[i].colWidth; + if (mergeCPDataArgs[i].colWidth <= 8) + { + mergeCPEntry.max = mergeCPDataArgs[i].max; + mergeCPEntry.min = mergeCPDataArgs[i].min; + } + else + { + mergeCPEntry.bigMax = mergeCPDataArgs[i].bigMax; + mergeCPEntry.bigMin = mergeCPDataArgs[i].bigMin; + } + mergeCPEntry.newExtent = mergeCPDataArgs[i].newExtent; + mergeCPEntry.seqNum = mergeCPDataArgs[i].seqNum; + bulkMergeCPMap[mergeCPDataArgs[i].startLbid] = mergeCPEntry; + } + + em.mergeExtentsMaxMin(bulkMergeCPMap, firstCall); + firstCall = false; + } + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } int SlaveDBRMNode::bulkUpdateDBRoot(const vector& args) throw() { - try - { - em.bulkUpdateDBRoot(args); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.bulkUpdateDBRoot(args); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } - -int SlaveDBRMNode::writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, - uint32_t vbFBO) throw() +int SlaveDBRMNode::writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, uint32_t vbFBO) throw() { - VER_t oldVerID; + VER_t oldVerID; - /* - LBIDRange r; - r.start = lbid; - r.size = 1; - if (!copylocks.isLocked(r)) - cout << "Copylock error: lbid " << lbid << " isn't locked\n"; - */ + /* + LBIDRange r; + r.start = lbid; + r.size = 1; + if (!copylocks.isLocked(r)) + cout << "Copylock error: lbid " << lbid << " isn't locked\n"; + */ - try + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; + + // figure out the current version of the block + // NOTE! This will currently error out to preserve the assumption that + // larger version numbers imply more recent changes. If we ever change that + // assumption, we'll need to revise the vbRollback() fcns as well. + oldVerID = vss.getCurrentVersion(lbid, NULL); + + if (oldVerID == transID) + return 0; + else if (oldVerID > transID) { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; + ostringstream str; - // figure out the current version of the block - // NOTE! This will currently error out to preserve the assumption that - // larger version numbers imply more recent changes. If we ever change that - // assumption, we'll need to revise the vbRollback() fcns as well. - oldVerID = vss.getCurrentVersion(lbid, NULL); - - if (oldVerID == transID) - return 0; - else if (oldVerID > transID) - { - ostringstream str; - - str << "WorkerDBRMNode::writeVBEntry(): Overlapping transactions detected. " - "Transaction " << transID << " cannot overwrite blocks written by " - "transaction " << oldVerID; - log(str.str()); - return ERR_OLDTXN_OVERWRITING_NEWTXN; - } - - vbbm.insert(lbid, oldVerID, vbOID, vbFBO); - - if (oldVerID > 0) - vss.setVBFlag(lbid, oldVerID, true); - else - vss.insert(lbid, oldVerID, true, false); - - // XXXPAT: There's a problem if we use transID as the new version here. - // Need to use at least oldVerID + 1. OldverID can be > TransID - vss.insert(lbid, transID, false, true); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; + str << "WorkerDBRMNode::writeVBEntry(): Overlapping transactions detected. " + "Transaction " + << transID + << " cannot overwrite blocks written by " + "transaction " + << oldVerID; + log(str.str()); + return ERR_OLDTXN_OVERWRITING_NEWTXN; } - return 0; + vbbm.insert(lbid, oldVerID, vbOID, vbFBO); + + if (oldVerID > 0) + vss.setVBFlag(lbid, oldVerID, true); + else + vss.insert(lbid, oldVerID, true, false); + + // XXXPAT: There's a problem if we use transID as the new version here. + // Need to use at least oldVerID + 1. OldverID can be > TransID + vss.insert(lbid, transID, false, true); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } -int SlaveDBRMNode::bulkWriteVBEntry(VER_t transID, - const std::vector& lbids, - OID_t vbOID, +int SlaveDBRMNode::bulkWriteVBEntry(VER_t transID, const std::vector& lbids, OID_t vbOID, const std::vector& vbFBOs) throw() { - VER_t oldVerID; + VER_t oldVerID; - /* - LBIDRange r; - r.start = lbid; - r.size = 1; - if (!copylocks.isLocked(r)) - cout << "Copylock error: lbid " << lbid << " isn't locked\n"; - */ + /* + LBIDRange r; + r.start = lbid; + r.size = 1; + if (!copylocks.isLocked(r)) + cout << "Copylock error: lbid " << lbid << " isn't locked\n"; + */ - try + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; + + for (size_t i = 0; i < lbids.size(); i++) { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; + // figure out the current version of the block + // NOTE! This will currently error out to preserve the assumption that + // larger version numbers imply more recent changes. If we ever change that + // assumption, we'll need to revise the vbRollback() fcns as well. + oldVerID = vss.getCurrentVersion(lbids[i], NULL); - for (size_t i = 0; i < lbids.size(); i++) - { - // figure out the current version of the block - // NOTE! This will currently error out to preserve the assumption that - // larger version numbers imply more recent changes. If we ever change that - // assumption, we'll need to revise the vbRollback() fcns as well. - oldVerID = vss.getCurrentVersion(lbids[i], NULL); + if (oldVerID == transID) + continue; + else if (oldVerID > transID) + { + ostringstream str; - if (oldVerID == transID) - continue; - else if (oldVerID > transID) - { - ostringstream str; + str << "WorkerDBRMNode::bulkWriteVBEntry(): Overlapping transactions detected. " + "Transaction " + << transID + << " cannot overwrite blocks written by " + "transaction " + << oldVerID; + log(str.str()); + return ERR_OLDTXN_OVERWRITING_NEWTXN; + } - str << "WorkerDBRMNode::bulkWriteVBEntry(): Overlapping transactions detected. " - "Transaction " << transID << " cannot overwrite blocks written by " - "transaction " << oldVerID; - log(str.str()); - return ERR_OLDTXN_OVERWRITING_NEWTXN; - } + vbbm.insert(lbids[i], oldVerID, vbOID, vbFBOs[i]); - vbbm.insert(lbids[i], oldVerID, vbOID, vbFBOs[i]); + if (oldVerID > 0) + vss.setVBFlag(lbids[i], oldVerID, true); + else + vss.insert(lbids[i], oldVerID, true, false); - if (oldVerID > 0) - vss.setVBFlag(lbids[i], oldVerID, true); - else - vss.insert(lbids[i], oldVerID, true, false); - - // XXXPAT: There's a problem if we use transID as the new version here. - // Need to use at least oldVerID + 1. OldverID can be > TransID - vss.insert(lbids[i], transID, false, true); - } - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; + // XXXPAT: There's a problem if we use transID as the new version here. + // Need to use at least oldVerID + 1. OldverID can be > TransID + vss.insert(lbids[i], transID, false, true); } + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } -int SlaveDBRMNode::beginVBCopy(VER_t transID, uint16_t vbOID, - const LBIDRange_v& ranges, VBRange_v& freeList, bool flushPMCache) throw() +int SlaveDBRMNode::beginVBCopy(VER_t transID, uint16_t vbOID, const LBIDRange_v& ranges, VBRange_v& freeList, + bool flushPMCache) throw() { - int64_t sum = 0; - uint64_t maxRetries; - uint64_t waitInterval = 50000; // usecs to sleep between retries - uint64_t retries; - bool* lockedRanges = (bool*)alloca(ranges.size() * sizeof(bool)); - bool allLocked; - uint32_t i; - + int64_t sum = 0; + uint64_t maxRetries; + uint64_t waitInterval = 50000; // usecs to sleep between retries + uint64_t retries; + bool* lockedRanges = (bool*)alloca(ranges.size() * sizeof(bool)); + bool allLocked; + uint32_t i; #ifdef BRM_DEBUG - if (transID < 1) - { - cerr << "WorkerDBRMNode::beginVBCopy(): transID must be > 0" << endl; - return -1; - } + if (transID < 1) + { + cerr << "WorkerDBRMNode::beginVBCopy(): transID must be > 0" << endl; + return -1; + } #endif - /* XXXPAT: The controller node will wait up to 5 mins for the response. - * For now, this alg will try for 1 min to grab all of the locks. - * After that, it will release them all then grab them. Releasing - * them by force opens the slight possibility of a bad result, but more - * likely something crashed, and there has to be some kind of recovery. The worst - * case is better than stalling the system or causing the BRM to go read-only. - * It should be extremely rare that it has to be done. + /* XXXPAT: The controller node will wait up to 5 mins for the response. + * For now, this alg will try for 1 min to grab all of the locks. + * After that, it will release them all then grab them. Releasing + * them by force opens the slight possibility of a bad result, but more + * likely something crashed, and there has to be some kind of recovery. The worst + * case is better than stalling the system or causing the BRM to go read-only. + * It should be extremely rare that it has to be done. + */ + maxRetries = (60 * 1000000) / waitInterval; + + for (i = 0; i < ranges.size(); i++) + { + sum += ranges[i].size; + lockedRanges[i] = false; + } + + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; + + /* This check doesn't need to be repeated after the retry loop below. + * For now, there is no other transaction that could lock these + * ranges. When we support multiple transactions at once, the resource + * graph in the controller node should make this redundant anyway. */ - maxRetries = (60 * 1000000) / waitInterval; - for (i = 0; i < ranges.size(); i++) - { - sum += ranges[i].size; - lockedRanges[i] = false; - } + if (vss.isLocked(ranges[i], transID)) + return -1; - try + copylocks.lock(CopyLocks::WRITE); + locked[2] = true; + allLocked = false; + /* This version grabs all unlocked ranges in each pass. + * If there are locked ranges it waits and tries again. + */ + retries = 0; + + while (!allLocked && retries < maxRetries) { + allLocked = true; + + for (i = 0; i < ranges.size(); i++) + { + if (!lockedRanges[i]) + { + if (copylocks.isLocked(ranges[i])) + allLocked = false; + else + { + copylocks.lockRange(ranges[i], transID); + lockedRanges[i] = true; + } + } + } + + /* PrimProc is reading at least 1 range and it could need the locks. + */ + if (!allLocked) + { + copylocks.release(CopyLocks::WRITE); + locked[2] = false; + vss.release(VSS::WRITE); + locked[1] = false; + vbbm.release(VBBM::WRITE); + locked[0] = false; + usleep(waitInterval); + retries++; vbbm.lock(VBBM::WRITE); locked[0] = true; vss.lock(VSS::WRITE); locked[1] = true; - - /* This check doesn't need to be repeated after the retry loop below. - * For now, there is no other transaction that could lock these - * ranges. When we support multiple transactions at once, the resource - * graph in the controller node should make this redundant anyway. - */ - for (i = 0; i < ranges.size(); i++) - if (vss.isLocked(ranges[i], transID)) - return -1; - copylocks.lock(CopyLocks::WRITE); locked[2] = true; - allLocked = false; - /* This version grabs all unlocked ranges in each pass. - * If there are locked ranges it waits and tries again. - */ - retries = 0; - - while (!allLocked && retries < maxRetries) - { - allLocked = true; - - for (i = 0; i < ranges.size(); i++) - { - if (!lockedRanges[i]) - { - if (copylocks.isLocked(ranges[i])) - allLocked = false; - else - { - copylocks.lockRange(ranges[i], transID); - lockedRanges[i] = true; - } - } - } - - /* PrimProc is reading at least 1 range and it could need the locks. - */ - if (!allLocked) - { - copylocks.release(CopyLocks::WRITE); - locked[2] = false; - vss.release(VSS::WRITE); - locked[1] = false; - vbbm.release(VBBM::WRITE); - locked[0] = false; - usleep(waitInterval); - retries++; - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; - copylocks.lock(CopyLocks::WRITE); - locked[2] = true; - } - } - - if (retries >= maxRetries) - { - for (i = 0; i < ranges.size(); i++) - { - if (!lockedRanges[i]) - { - copylocks.forceRelease(ranges[i]); - copylocks.lockRange(ranges[i], transID); - lockedRanges[i] = true; - } - } - } - - vbbm.getBlocks(sum, vbOID, freeList, vss, flushPMCache); - /* - for (i = 0; i < ranges.size(); i++) - assert(copylocks.isLocked(ranges[i])); - */ - return 0; + } } - catch (const logging::VBBMBufferOverFlowExcept& e) + + if (retries >= maxRetries) { - cerr << e.what() << endl; - - for (i = 0; i < ranges.size(); i++) - if (lockedRanges[i]) - copylocks.releaseRange(ranges[i]); - - return e.errorCode(); + for (i = 0; i < ranges.size(); i++) + { + if (!lockedRanges[i]) + { + copylocks.forceRelease(ranges[i]); + copylocks.lockRange(ranges[i], transID); + lockedRanges[i] = true; + } + } } - catch (exception& e) - { - for (i = 0; i < ranges.size(); i++) - if (lockedRanges[i]) - copylocks.releaseRange(ranges[i]); - cerr << e.what() << endl; - return -1; - } + vbbm.getBlocks(sum, vbOID, freeList, vss, flushPMCache); + /* + for (i = 0; i < ranges.size(); i++) + assert(copylocks.isLocked(ranges[i])); + */ + return 0; + } + catch (const logging::VBBMBufferOverFlowExcept& e) + { + cerr << e.what() << endl; + + for (i = 0; i < ranges.size(); i++) + if (lockedRanges[i]) + copylocks.releaseRange(ranges[i]); + + return e.errorCode(); + } + catch (exception& e) + { + for (i = 0; i < ranges.size(); i++) + if (lockedRanges[i]) + copylocks.releaseRange(ranges[i]); + + cerr << e.what() << endl; + return -1; + } } -int SlaveDBRMNode::endVBCopy(VER_t transID, const LBIDRange_v& ranges) -throw() +int SlaveDBRMNode::endVBCopy(VER_t transID, const LBIDRange_v& ranges) throw() { - LBIDRange_v::const_iterator it; + LBIDRange_v::const_iterator it; - try - { - copylocks.lock(CopyLocks::WRITE); - locked[2] = true; + try + { + copylocks.lock(CopyLocks::WRITE); + locked[2] = true; - for (it = ranges.begin(); it != ranges.end(); ++it) - copylocks.releaseRange(*it); + for (it = ranges.begin(); it != ranges.end(); ++it) + copylocks.releaseRange(*it); - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } int SlaveDBRMNode::vbCommit(VER_t transID) throw() { - try - { - vss.lock(VSS::WRITE); - locked[1] = true; - vss.commit(transID); - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + vss.lock(VSS::WRITE); + locked[1] = true; + vss.commit(transID); + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } -int SlaveDBRMNode::vbRollback(VER_t transID, const LBIDRange_v& lbidList, bool flushPMCache) -throw() +int SlaveDBRMNode::vbRollback(VER_t transID, const LBIDRange_v& lbidList, bool flushPMCache) throw() { - LBIDRange_v::const_iterator it; - LBID_t lbid; - VER_t oldVerID; - vector flushList; + LBIDRange_v::const_iterator it; + LBID_t lbid; + VER_t oldVerID; + vector flushList; #ifdef BRM_DEBUG - if (transID < 1) - { - cerr << "SlaveDBRMNode::vbRollback(): transID must be > 0" << endl; - return -1; - } + if (transID < 1) + { + cerr << "SlaveDBRMNode::vbRollback(): transID must be > 0" << endl; + return -1; + } #endif - try + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; + copylocks.lock(CopyLocks::WRITE); + locked[2] = true; + + copylocks.rollback(transID); + + for (it = lbidList.begin(); it != lbidList.end(); it++) { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; - copylocks.lock(CopyLocks::WRITE); - locked[2] = true; + for (lbid = (*it).start; lbid < (*it).start + (*it).size; lbid++) + { + oldVerID = vss.getHighestVerInVB(lbid, transID); - copylocks.rollback(transID); - - for (it = lbidList.begin(); it != lbidList.end(); it++) + if (oldVerID != -1) { - for (lbid = (*it).start; lbid < (*it).start + (*it).size; lbid++) - { - oldVerID = vss.getHighestVerInVB(lbid, transID); - - if (oldVerID != -1) - { - vbbm.removeEntry(lbid, oldVerID); - vss.setVBFlag(lbid, oldVerID, false); - } - - vss.removeEntry(lbid, transID, &flushList); - } + vbbm.removeEntry(lbid, oldVerID); + vss.setVBFlag(lbid, oldVerID, false); } - if (flushPMCache && !flushList.empty()) - cacheutils::flushPrimProcAllverBlocks(flushList); + vss.removeEntry(lbid, transID, &flushList); + } + } - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - ostringstream ostr; - ostr << "SlaveDBRMNode::vbRollback error. " << e.what(); - log(ostr.str()); - return -1; - } + if (flushPMCache && !flushList.empty()) + cacheutils::flushPrimProcAllverBlocks(flushList); + + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + ostringstream ostr; + ostr << "SlaveDBRMNode::vbRollback error. " << e.what(); + log(ostr.str()); + return -1; + } } -int SlaveDBRMNode::vbRollback(VER_t transID, const vector& lbidList, bool flushPMCache) -throw() +int SlaveDBRMNode::vbRollback(VER_t transID, const vector& lbidList, bool flushPMCache) throw() { - vector::const_iterator it; - VER_t oldVerID; - vector flushList; + vector::const_iterator it; + VER_t oldVerID; + vector flushList; #ifdef BRM_DEBUG - if (transID < 1) - { - cerr << "SlaveDBRMNode::vbRollback(): transID must be > 0" << endl; - return -1; - } + if (transID < 1) + { + cerr << "SlaveDBRMNode::vbRollback(): transID must be > 0" << endl; + return -1; + } #endif - try + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; + copylocks.lock(CopyLocks::WRITE); + locked[2] = true; + + copylocks.rollback(transID); + + for (it = lbidList.begin(); it != lbidList.end(); it++) { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; - copylocks.lock(CopyLocks::WRITE); - locked[2] = true; + oldVerID = vss.getHighestVerInVB(*it, transID); - copylocks.rollback(transID); + if (oldVerID != -1) + { + vbbm.removeEntry(*it, oldVerID); + vss.setVBFlag(*it, oldVerID, false); + } - for (it = lbidList.begin(); it != lbidList.end(); it++) - { - oldVerID = vss.getHighestVerInVB(*it, transID); - - if (oldVerID != -1) - { - vbbm.removeEntry(*it, oldVerID); - vss.setVBFlag(*it, oldVerID, false); - } - - vss.removeEntry(*it, transID, &flushList); - } - - if (flushPMCache && !flushList.empty()) - cacheutils::flushPrimProcAllverBlocks(flushList); - - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; + vss.removeEntry(*it, transID, &flushList); } + + if (flushPMCache && !flushList.empty()) + cacheutils::flushPrimProcAllverBlocks(flushList); + + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } void SlaveDBRMNode::confirmChanges() throw() { + try + { + em.confirmChanges(); - try + if (locked[0]) { - em.confirmChanges(); - - if (locked[0]) - { - vbbm.confirmChanges(); - vbbm.release(VBBM::WRITE); - locked[0] = false; - } - - if (locked[1]) - { - vss.confirmChanges(); - vss.release(VSS::WRITE); - locked[1] = false; - } - - if (locked[2]) - { - copylocks.confirmChanges(); - copylocks.release(CopyLocks::WRITE); - locked[2] = false; - } + vbbm.confirmChanges(); + vbbm.release(VBBM::WRITE); + locked[0] = false; } - catch (exception& e) + + if (locked[1]) { - cerr << e.what() << endl; + vss.confirmChanges(); + vss.release(VSS::WRITE); + locked[1] = false; } + + if (locked[2]) + { + copylocks.confirmChanges(); + copylocks.release(CopyLocks::WRITE); + locked[2] = false; + } + } + catch (exception& e) + { + cerr << e.what() << endl; + } } void SlaveDBRMNode::undoChanges() throw() { + try + { + em.undoChanges(); - try + if (locked[0]) { - em.undoChanges(); - - if (locked[0]) - { - vbbm.undoChanges(); - vbbm.release(VBBM::WRITE); - locked[0] = false; - } - - if (locked[1]) - { - vss.undoChanges(); - vss.release(VSS::WRITE); - locked[1] = false; - } - - if (locked[2]) - { - copylocks.undoChanges(); - copylocks.release(CopyLocks::WRITE); - locked[2] = false; - } + vbbm.undoChanges(); + vbbm.release(VBBM::WRITE); + locked[0] = false; } - catch (exception& e) + + if (locked[1]) { - cerr << e.what() << endl; + vss.undoChanges(); + vss.release(VSS::WRITE); + locked[1] = false; } + + if (locked[2]) + { + copylocks.undoChanges(); + copylocks.release(CopyLocks::WRITE); + locked[2] = false; + } + } + catch (exception& e) + { + cerr << e.what() << endl; + } } int SlaveDBRMNode::clear() throw() { - bool llocked[2] = {false, false}; + bool llocked[2] = {false, false}; - try - { - vbbm.lock(VBBM::WRITE); - llocked[0] = true; - vss.lock(VSS::WRITE); - llocked[1] = true; + try + { + vbbm.lock(VBBM::WRITE); + llocked[0] = true; + vss.lock(VSS::WRITE); + llocked[1] = true; - vbbm.clear(); - vss.clear(); + vbbm.clear(); + vss.clear(); - vss.release(VSS::WRITE); - llocked[1] = false; - vbbm.release(VBBM::WRITE); - llocked[0] = false; - } - catch (exception& e) - { - cerr << e.what() << endl; + vss.release(VSS::WRITE); + llocked[1] = false; + vbbm.release(VBBM::WRITE); + llocked[0] = false; + } + catch (exception& e) + { + cerr << e.what() << endl; - if (llocked[1]) - vss.release(VSS::WRITE); + if (llocked[1]) + vss.release(VSS::WRITE); - if (llocked[0]) - vbbm.release(VBBM::WRITE); + if (llocked[0]) + vbbm.release(VBBM::WRITE); - return -1; - } + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::checkConsistency() throw() { - bool llocked[2] = {false, false}; + bool llocked[2] = {false, false}; - try - { - em.checkConsistency(); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.checkConsistency(); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - try - { - vbbm.lock(VBBM::READ); - llocked[0] = true; - vss.lock(VSS::READ); - llocked[1] = true; - vss.checkConsistency(vbbm, em); - vss.release(VSS::READ); - llocked[1] = false; - vbbm.release(VBBM::READ); - llocked[0] = false; - } - catch (exception& e) - { - cerr << e.what() << endl; + try + { + vbbm.lock(VBBM::READ); + llocked[0] = true; + vss.lock(VSS::READ); + llocked[1] = true; + vss.checkConsistency(vbbm, em); + vss.release(VSS::READ); + llocked[1] = false; + vbbm.release(VBBM::READ); + llocked[0] = false; + } + catch (exception& e) + { + cerr << e.what() << endl; - if (llocked[1]) - vss.release(VSS::READ); + if (llocked[1]) + vss.release(VSS::READ); - if (llocked[0]) - vbbm.release(VBBM::READ); + if (llocked[0]) + vbbm.release(VBBM::READ); - return -1; - } + return -1; + } - try - { - vbbm.lock(VBBM::READ); - vbbm.checkConsistency(); - vbbm.release(VBBM::READ); - } - catch (exception& e) - { - cerr << e.what() << endl; - vbbm.release(VBBM::READ); - return -1; - } + try + { + vbbm.lock(VBBM::READ); + vbbm.checkConsistency(); + vbbm.release(VBBM::READ); + } + catch (exception& e) + { + cerr << e.what() << endl; + vbbm.release(VBBM::READ); + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::saveState(string filename) throw() { - string emFilename = filename + "_em"; - string vssFilename = filename + "_vss"; - string vbbmFilename = filename + "_vbbm"; - bool locked[2] = { false, false }; + string emFilename = filename + "_em"; + string vssFilename = filename + "_vss"; + string vbbmFilename = filename + "_vbbm"; + bool locked[2] = {false, false}; - try - { - vbbm.lock(VBBM::READ); - locked[0] = true; - vss.lock(VSS::READ); - locked[1] = true; + try + { + vbbm.lock(VBBM::READ); + locked[0] = true; + vss.lock(VSS::READ); + locked[1] = true; - saveExtentMap(emFilename); - vbbm.save(vbbmFilename); - vss.save(vssFilename); + saveExtentMap(emFilename); + vbbm.save(vbbmFilename); + vss.save(vssFilename); - vss.release(VSS::READ); - locked[1] = false; - vbbm.release(VBBM::READ); - locked[0] = false; - } - catch (exception& e) - { - if (locked[1]) - vss.release(VSS::READ); + vss.release(VSS::READ); + locked[1] = false; + vbbm.release(VBBM::READ); + locked[0] = false; + } + catch (exception& e) + { + if (locked[1]) + vss.release(VSS::READ); - if (locked[0]) - vbbm.release(VBBM::READ); + if (locked[0]) + vbbm.release(VBBM::READ); - return -1; - } + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::loadState(string filename) throw() { - string emFilename = filename + "_em"; - string vssFilename = filename + "_vss"; - string vbbmFilename = filename + "_vbbm"; - bool locked[2] = { false, false }; + string emFilename = filename + "_em"; + string vssFilename = filename + "_vss"; + string vbbmFilename = filename + "_vbbm"; + bool locked[2] = {false, false}; - try - { - vbbm.lock(VBBM::WRITE); - locked[0] = true; - vss.lock(VSS::WRITE); - locked[1] = true; + try + { + vbbm.lock(VBBM::WRITE); + locked[0] = true; + vss.lock(VSS::WRITE); + locked[1] = true; - loadExtentMap(emFilename); - vbbm.load(vbbmFilename); - vss.load(vssFilename); + loadExtentMap(emFilename); + vbbm.load(vbbmFilename); + vss.load(vssFilename); - vss.release(VSS::WRITE); - locked[1] = false; - vbbm.release(VBBM::WRITE); - locked[0] = false; - } - catch (exception& e) - { - if (locked[1]) - vss.release(VSS::WRITE); + vss.release(VSS::WRITE); + locked[1] = false; + vbbm.release(VBBM::WRITE); + locked[0] = false; + } + catch (exception& e) + { + if (locked[1]) + vss.release(VSS::WRITE); - if (locked[0]) - vbbm.release(VBBM::WRITE); + if (locked[0]) + vbbm.release(VBBM::WRITE); - return -1; - } + return -1; + } - return 0; + return 0; } int SlaveDBRMNode::loadExtentMap(const string& filename) { - em.load(filename); - return 0; + em.load(filename); + return 0; } int SlaveDBRMNode::saveExtentMap(const string& filename) { - em.save(filename); - return 0; + em.save(filename); + return 0; } // Casual partitioning support @@ -1147,77 +1108,74 @@ int SlaveDBRMNode::saveExtentMap(const string& filename) int SlaveDBRMNode::markExtentInvalid(const LBID_t lbid, execplan::CalpontSystemCatalog::ColDataType colDataType) { - int err = 0; + int err = 0; - try - { - err = em.markInvalid(lbid, colDataType); - //em.confirmChanges(); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + err = em.markInvalid(lbid, colDataType); + // em.confirmChanges(); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return err; + return err; } -int SlaveDBRMNode::markExtentsInvalid(const vector& lbids, - const std::vector& colDataTypes) +int SlaveDBRMNode::markExtentsInvalid( + const vector& lbids, const std::vector& colDataTypes) { - int err = 0; + int err = 0; - try - { - err = em.markInvalid(lbids, colDataTypes); - //em.confirmChanges(); - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + err = em.markInvalid(lbids, colDataTypes); + // em.confirmChanges(); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return err; + return err; } -int SlaveDBRMNode::setExtentMaxMin(const LBID_t lbid, - const int64_t max, - const int64_t min, - const int32_t seqNum, - bool firstNode) +int SlaveDBRMNode::setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, + const int32_t seqNum, bool firstNode) { - int err = 0; + int err = 0; - try - { - err = em.setMaxMin(lbid, max, min, seqNum, firstNode); - //em.confirmChanges(); - } - catch ( exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + err = em.setMaxMin(lbid, max, min, seqNum, firstNode); + // em.confirmChanges(); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return err; + return err; } // @bug 1970 - Added setExtentsMaxMin below. int SlaveDBRMNode::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode) { - try - { - em.setExtentsMaxMin(cpMap, firstNode); - } - catch ( exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.setExtentsMaxMin(cpMap, firstNode); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ @@ -1225,94 +1183,94 @@ int SlaveDBRMNode::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode) //------------------------------------------------------------------------------ int SlaveDBRMNode::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap) { - try - { - em.mergeExtentsMaxMin(cpMap); - } - catch ( exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.mergeExtentsMaxMin(cpMap); + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } - return 0; + return 0; } //------------------------------------------------------------------------------ // Delete all extents for the specified OID(s) and partition number. //------------------------------------------------------------------------------ -int SlaveDBRMNode::deletePartition(const std::set& oids, - set& partitionNums, string& emsg) throw() +int SlaveDBRMNode::deletePartition(const std::set& oids, set& partitionNums, + string& emsg) throw() { - try - { - em.deletePartition(oids, partitionNums, emsg); - } - catch (IDBExcept& iex) - { - cerr << iex.what() << endl; - - if ( iex.errorCode() == ERR_PARTITION_NOT_EXIST ) - return ERR_NOT_EXIST_PARTITION; - else if ( iex.errorCode() == ERR_INVALID_LAST_PARTITION ) - return ERR_INVALID_OP_LAST_PARTITION; - else if (iex.errorCode() == WARN_NO_PARTITION_PERFORMED) - return ERR_NO_PARTITION_PERFORMED; - else - return -1; - } - catch (DBRMException& e) - { - // exceptions that can be ignored - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.deletePartition(oids, partitionNums, emsg); + } + catch (IDBExcept& iex) + { + cerr << iex.what() << endl; + if (iex.errorCode() == ERR_PARTITION_NOT_EXIST) + return ERR_NOT_EXIST_PARTITION; + else if (iex.errorCode() == ERR_INVALID_LAST_PARTITION) + return ERR_INVALID_OP_LAST_PARTITION; + else if (iex.errorCode() == WARN_NO_PARTITION_PERFORMED) + return ERR_NO_PARTITION_PERFORMED; + else + return -1; + } + catch (DBRMException& e) + { + // exceptions that can be ignored return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } //------------------------------------------------------------------------------ // Mark all extents as out of service, for the specified OID(s) and partition // number. //------------------------------------------------------------------------------ -int SlaveDBRMNode::markPartitionForDeletion(const std::set& oids, - set& partitionNums, string& emsg) throw() +int SlaveDBRMNode::markPartitionForDeletion(const std::set& oids, set& partitionNums, + string& emsg) throw() { - try - { - em.markPartitionForDeletion(oids, partitionNums, emsg); - } - catch (IDBExcept& iex) - { - cerr << iex.what() << endl; - - if ( iex.errorCode() == ERR_PARTITION_ALREADY_DISABLED ) - return ERR_PARTITION_DISABLED; - else if ( iex.errorCode() == ERR_PARTITION_NOT_EXIST ) - return ERR_NOT_EXIST_PARTITION; - else if ( iex.errorCode() == ERR_INVALID_LAST_PARTITION ) - return ERR_INVALID_OP_LAST_PARTITION; - else if (iex.errorCode() == WARN_NO_PARTITION_PERFORMED) - return ERR_NO_PARTITION_PERFORMED; - else - return -1; - } - catch (DBRMException& e) - { - // exceptions that can be ignored - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.markPartitionForDeletion(oids, partitionNums, emsg); + } + catch (IDBExcept& iex) + { + cerr << iex.what() << endl; + if (iex.errorCode() == ERR_PARTITION_ALREADY_DISABLED) + return ERR_PARTITION_DISABLED; + else if (iex.errorCode() == ERR_PARTITION_NOT_EXIST) + return ERR_NOT_EXIST_PARTITION; + else if (iex.errorCode() == ERR_INVALID_LAST_PARTITION) + return ERR_INVALID_OP_LAST_PARTITION; + else if (iex.errorCode() == WARN_NO_PARTITION_PERFORMED) + return ERR_NO_PARTITION_PERFORMED; + else + return -1; + } + catch (DBRMException& e) + { + // exceptions that can be ignored return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } //------------------------------------------------------------------------------ @@ -1320,72 +1278,72 @@ int SlaveDBRMNode::markPartitionForDeletion(const std::set& oids, //------------------------------------------------------------------------------ int SlaveDBRMNode::markAllPartitionForDeletion(const std::set& oids) throw() { - try - { - em.markAllPartitionForDeletion(oids); - } - catch (IDBExcept& iex) - { - cerr << iex.what() << endl; - - if ( iex.errorCode() == ERR_PARTITION_ALREADY_DISABLED ) - return ERR_PARTITION_DISABLED; - else if ( iex.errorCode() == ERR_PARTITION_NOT_EXIST ) - return ERR_NOT_EXIST_PARTITION; - else if ( iex.errorCode() == ERR_INVALID_LAST_PARTITION ) - return ERR_INVALID_OP_LAST_PARTITION; - else - return -1; - } - catch (DBRMException& e) - { - // exceptions that can be ignored - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.markAllPartitionForDeletion(oids); + } + catch (IDBExcept& iex) + { + cerr << iex.what() << endl; + if (iex.errorCode() == ERR_PARTITION_ALREADY_DISABLED) + return ERR_PARTITION_DISABLED; + else if (iex.errorCode() == ERR_PARTITION_NOT_EXIST) + return ERR_NOT_EXIST_PARTITION; + else if (iex.errorCode() == ERR_INVALID_LAST_PARTITION) + return ERR_INVALID_OP_LAST_PARTITION; + else + return -1; + } + catch (DBRMException& e) + { + // exceptions that can be ignored return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } //------------------------------------------------------------------------------ // Restore all extents for the specified OID(s) and partition number. //------------------------------------------------------------------------------ -int SlaveDBRMNode::restorePartition(const std::set& oids, - set& partitionNums, string& emsg) throw() +int SlaveDBRMNode::restorePartition(const std::set& oids, set& partitionNums, + string& emsg) throw() { - try - { - em.restorePartition(oids, partitionNums, emsg); - } - catch (IDBExcept& iex) - { - cerr << iex.what() << endl; - - if ( iex.errorCode() == ERR_PARTITION_NOT_EXIST ) - return ERR_NOT_EXIST_PARTITION; - else if ( iex.errorCode() == ERR_PARTITION_ALREADY_ENABLED ) - return ERR_PARTITION_ENABLED; - else if ( iex.errorCode() == ERR_INVALID_LAST_PARTITION ) - return ERR_INVALID_OP_LAST_PARTITION; - else - return -1; - } - catch (DBRMException& e) - { - // exceptions that can be ignored - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + try + { + em.restorePartition(oids, partitionNums, emsg); + } + catch (IDBExcept& iex) + { + cerr << iex.what() << endl; + if (iex.errorCode() == ERR_PARTITION_NOT_EXIST) + return ERR_NOT_EXIST_PARTITION; + else if (iex.errorCode() == ERR_PARTITION_ALREADY_ENABLED) + return ERR_PARTITION_ENABLED; + else if (iex.errorCode() == ERR_INVALID_LAST_PARTITION) + return ERR_INVALID_OP_LAST_PARTITION; + else + return -1; + } + catch (DBRMException& e) + { + // exceptions that can be ignored return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } //------------------------------------------------------------------------------ @@ -1393,158 +1351,155 @@ int SlaveDBRMNode::restorePartition(const std::set& oids, //------------------------------------------------------------------------------ int SlaveDBRMNode::deleteDBRoot(uint16_t dbroot) throw() { - try - { - em.deleteDBRoot(dbroot); - } - catch (IDBExcept& iex) - { - cerr << iex.what() << endl; - return -1; - } - catch (DBRMException& e) - { - // exceptions that can be ignored - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } - + try + { + em.deleteDBRoot(dbroot); + } + catch (IDBExcept& iex) + { + cerr << iex.what() << endl; + return -1; + } + catch (DBRMException& e) + { + // exceptions that can be ignored return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } + + return 0; } int SlaveDBRMNode::dmlLockLBIDRanges(const vector& ranges, int txnID) { - uint64_t maxRetries; - uint64_t waitInterval = 50000; // usecs to sleep between retries - uint64_t retries; - bool* lockedRanges = (bool*)alloca(ranges.size() * sizeof(bool)); - bool allLocked; - uint32_t i; + uint64_t maxRetries; + uint64_t waitInterval = 50000; // usecs to sleep between retries + uint64_t retries; + bool* lockedRanges = (bool*)alloca(ranges.size() * sizeof(bool)); + bool allLocked; + uint32_t i; - /* XXXPAT: The controller node will wait up to 5 mins for the response. - * For now, this alg will try for 1 min to grab all of the locks. - * After that, it will release them all then grab them. Releasing - * them by force opens the slight possibility of a bad result, but more - * likely something crashed, and there has to be some kind of recovery. The worst - * case is better than stalling the system or causing the BRM to go read-only. - * It should be extremely rare that it has to be done. + /* XXXPAT: The controller node will wait up to 5 mins for the response. + * For now, this alg will try for 1 min to grab all of the locks. + * After that, it will release them all then grab them. Releasing + * them by force opens the slight possibility of a bad result, but more + * likely something crashed, and there has to be some kind of recovery. The worst + * case is better than stalling the system or causing the BRM to go read-only. + * It should be extremely rare that it has to be done. + */ + maxRetries = (60 * 1000000) / waitInterval; + + for (i = 0; i < ranges.size(); i++) + lockedRanges[i] = false; + + try + { + copylocks.lock(CopyLocks::WRITE); + locked[2] = true; + allLocked = false; + /* This version grabs all unlocked ranges in each pass. + * If there are locked ranges it waits and tries again. */ - maxRetries = (60 * 1000000) / waitInterval; + retries = 0; - for (i = 0; i < ranges.size(); i++) - lockedRanges[i] = false; - - try + while (!allLocked && retries < maxRetries) { + allLocked = true; + + for (i = 0; i < ranges.size(); i++) + { + if (!lockedRanges[i]) + { + if (copylocks.isLocked(ranges[i])) + allLocked = false; + else + { + copylocks.lockRange(ranges[i], txnID); + lockedRanges[i] = true; + } + } + } + + /* PrimProc is reading at least 1 range and it could need the locks. + */ + if (!allLocked) + { + copylocks.release(CopyLocks::WRITE); + locked[2] = false; + usleep(waitInterval); + retries++; copylocks.lock(CopyLocks::WRITE); locked[2] = true; - allLocked = false; - /* This version grabs all unlocked ranges in each pass. - * If there are locked ranges it waits and tries again. - */ - retries = 0; - - while (!allLocked && retries < maxRetries) - { - allLocked = true; - - for (i = 0; i < ranges.size(); i++) - { - if (!lockedRanges[i]) - { - if (copylocks.isLocked(ranges[i])) - allLocked = false; - else - { - copylocks.lockRange(ranges[i], txnID); - lockedRanges[i] = true; - } - } - } - - /* PrimProc is reading at least 1 range and it could need the locks. - */ - if (!allLocked) - { - copylocks.release(CopyLocks::WRITE); - locked[2] = false; - usleep(waitInterval); - retries++; - copylocks.lock(CopyLocks::WRITE); - locked[2] = true; - } - } - - if (retries >= maxRetries) - { - for (i = 0; i < ranges.size(); i++) - { - if (!lockedRanges[i]) - { - copylocks.forceRelease(ranges[i]); - copylocks.lockRange(ranges[i], txnID); - lockedRanges[i] = true; - } - } - } - - return 0; + } } - catch (exception& e) + + if (retries >= maxRetries) { - for (i = 0; i < ranges.size(); i++) - if (lockedRanges[i]) - copylocks.releaseRange(ranges[i]); - - cerr << e.what() << endl; - return -1; + for (i = 0; i < ranges.size(); i++) + { + if (!lockedRanges[i]) + { + copylocks.forceRelease(ranges[i]); + copylocks.lockRange(ranges[i], txnID); + lockedRanges[i] = true; + } + } } + + return 0; + } + catch (exception& e) + { + for (i = 0; i < ranges.size(); i++) + if (lockedRanges[i]) + copylocks.releaseRange(ranges[i]); + + cerr << e.what() << endl; + return -1; + } } int SlaveDBRMNode::dmlReleaseLBIDRanges(const vector& ranges) { - try - { - copylocks.lock(CopyLocks::WRITE); - locked[2] = true; + try + { + copylocks.lock(CopyLocks::WRITE); + locked[2] = true; - for (uint32_t i = 0; i < ranges.size(); ++i) - copylocks.releaseRange(ranges[i]); - - return 0; - } - catch (exception& e) - { - cerr << e.what() << endl; - return -1; - } + for (uint32_t i = 0; i < ranges.size(); ++i) + copylocks.releaseRange(ranges[i]); + return 0; + } + catch (exception& e) + { + cerr << e.what() << endl; + return -1; + } } - const bool* SlaveDBRMNode::getEMFLLockStatus() { - return em.getEMFLLockStatus(); + return em.getEMFLLockStatus(); } const bool* SlaveDBRMNode::getEMLockStatus() { - return em.getEMLockStatus(); + return em.getEMLockStatus(); } const bool* SlaveDBRMNode::getVBBMLockStatus() { - return &locked[0]; + return &locked[0]; } const bool* SlaveDBRMNode::getVSSLockStatus() { - return &locked[1]; + return &locked[1]; } -} //namespace - +} // namespace BRM diff --git a/versioning/BRM/slavedbrmnode.h b/versioning/BRM/slavedbrmnode.h index 482df198f..8512c0090 100644 --- a/versioning/BRM/slavedbrmnode.h +++ b/versioning/BRM/slavedbrmnode.h @@ -48,7 +48,6 @@ namespace BRM { - /** @brief The Slave node of the DBRM system * * There are 3 components of the Distributed BRM (DBRM). @@ -90,432 +89,396 @@ namespace BRM class SlaveDBRMNode { -public: - EXPORT SlaveDBRMNode() throw(); - EXPORT ~SlaveDBRMNode() throw(); + public: + EXPORT SlaveDBRMNode() throw(); + EXPORT ~SlaveDBRMNode() throw(); - /** @brief Allocate a "stripe" of extents for columns in a table. - * - * Allocate a "stripe" of extents for the specified columns and DBRoot - * @param cols (in) List of column OIDs and column widths - * @param dbRoot (in) DBRoot for requested extents. - * @param partitionNum (in/out) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (out) Segment number selected for new extents. - * @param extents (out) list of lbids, numBlks, and fbo for new extents - * @return 0 on success, -1 on error - */ - EXPORT int createStripeColumnExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partitionNum, - uint16_t& segmentNum, - std::vector& extents) throw(); + /** @brief Allocate a "stripe" of extents for columns in a table. + * + * Allocate a "stripe" of extents for the specified columns and DBRoot + * @param cols (in) List of column OIDs and column widths + * @param dbRoot (in) DBRoot for requested extents. + * @param partitionNum (in/out) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (out) Segment number selected for new extents. + * @param extents (out) list of lbids, numBlks, and fbo for new extents + * @return 0 on success, -1 on error + */ + EXPORT int createStripeColumnExtents(const std::vector& cols, + uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum, + std::vector& extents) throw(); - /** @brief Allocate extent in the specified segment file - * - * Allocate column extent for the exact segment file specified by the - * requested OID,DBRoot, partition, and segment. - * @param OID (in) The OID requesting the extent. - * @param colWidth (in) Column width of the OID. - * @param dbRoot (in) DBRoot where extent is to be added. - * @param partitionNum (in) Partition number in file path. - * @param segmentNum (in) Segment number in file path. - * @param colDataType (in) the column type - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @param startBlockOffset (out) The first block of the extent created. - * @return 0 on success, -1 on error - */ - EXPORT int createColumnExtentExactFile(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) throw(); - - /** @brief Allocate an extent for a column file - * - * Allocate a column extent for the specified OID and DBRoot. - * @param OID (in) The OID requesting the extent. - * @param colWidth (in) Column width of the OID. - * @param dbRoot (in) DBRoot where extent is to be added. - * @param colDataType (in) the column type - * @param partitionNum (in/out) Partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partitionNum is input, else it is an output arg. - * @param segmentNum (in/out) Segment number in file path. - * If allocating OID's first extent for this DBRoot, then - * segmentNum is input, else it is an output arg. - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @param startBlockOffset (out) The first block of the extent created. - * @return 0 on success, -1 on error - */ - EXPORT int createColumnExtent_DBroot(OID_t oid, - uint32_t colWidth, - uint16_t dbRoot, + /** @brief Allocate extent in the specified segment file + * + * Allocate column extent for the exact segment file specified by the + * requested OID,DBRoot, partition, and segment. + * @param OID (in) The OID requesting the extent. + * @param colWidth (in) Column width of the OID. + * @param dbRoot (in) DBRoot where extent is to be added. + * @param partitionNum (in) Partition number in file path. + * @param segmentNum (in) Segment number in file path. + * @param colDataType (in) the column type + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @param startBlockOffset (out) The first block of the extent created. + * @return 0 on success, -1 on error + */ + EXPORT int createColumnExtentExactFile(OID_t oid, uint32_t colWidth, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t& partitionNum, - uint16_t& segmentNum, - LBID_t& lbid, - int& allocdSize, - uint32_t& startBlockOffset) throw(); + LBID_t& lbid, int& allocdSize, uint32_t& startBlockOffset) throw(); - /** @brief Allocate an extent for a dictionary store file - * - * Allocate a dictionary store extent for the specified OID, dbRoot, - * partition number, and segment number. - * @param OID (in) The OID requesting the extent. - * @param dbRoot (in) DBRoot to assign to the extent. - * @param partitionNum (in) Partition number to assign to the extent. - * @param segmentNum (in) Segment number to assign to the extent. - * @param lbid (out) The first LBID of the extent created. - * @param allocdSize (out) The total number of LBIDs allocated. - * @return 0 on success, -1 on error - */ - EXPORT int createDictStoreExtent(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - LBID_t& lbid, - int& allocdSize) throw(); + /** @brief Allocate an extent for a column file + * + * Allocate a column extent for the specified OID and DBRoot. + * @param OID (in) The OID requesting the extent. + * @param colWidth (in) Column width of the OID. + * @param dbRoot (in) DBRoot where extent is to be added. + * @param colDataType (in) the column type + * @param partitionNum (in/out) Partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partitionNum is input, else it is an output arg. + * @param segmentNum (in/out) Segment number in file path. + * If allocating OID's first extent for this DBRoot, then + * segmentNum is input, else it is an output arg. + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @param startBlockOffset (out) The first block of the extent created. + * @return 0 on success, -1 on error + */ + EXPORT int createColumnExtent_DBroot(OID_t oid, uint32_t colWidth, uint16_t dbRoot, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t& partitionNum, uint16_t& segmentNum, LBID_t& lbid, + int& allocdSize, uint32_t& startBlockOffset) throw(); - /** @brief Rollback (delete) a set of extents for the specified OID. - * - * Deletes all the extents that logically follow the specified - * column extent; and sets the HWM for the specified extent. - * @param oid OID of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segmentNum Last segment in partitionNum to be kept. - * @param hwm HWM to be assigned to the last extent that is kept. - * @return 0 on success, -1 on error - */ - EXPORT int rollbackColumnExtents(OID_t oid, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm) throw(); + /** @brief Allocate an extent for a dictionary store file + * + * Allocate a dictionary store extent for the specified OID, dbRoot, + * partition number, and segment number. + * @param OID (in) The OID requesting the extent. + * @param dbRoot (in) DBRoot to assign to the extent. + * @param partitionNum (in) Partition number to assign to the extent. + * @param segmentNum (in) Segment number to assign to the extent. + * @param lbid (out) The first LBID of the extent created. + * @param allocdSize (out) The total number of LBIDs allocated. + * @return 0 on success, -1 on error + */ + EXPORT int createDictStoreExtent(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum, + LBID_t& lbid, int& allocdSize) throw(); - /** @brief Rollback (delete) set of extents for specified OID & DBRoot. - * - * Deletes all the extents that logically follow the specified - * column extent; and sets the HWM for the specified extent. - * @param oid OID of the extents to be deleted. - * @param bDeleteAll Indicates if all extents in oid and dbroot are to - * be deleted; else part#, seg#, and hwm are used. - * @param dbRoot DBRoot of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param segmentNum Last segment in partitionNum to be kept. - * @param hwm HWM to be assigned to the last extent that is kept. - * @return 0 on success, -1 on error - */ - EXPORT int rollbackColumnExtents_DBroot(OID_t oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partitionNum, - uint16_t segmentNum, - HWM_t hwm) throw(); + /** @brief Rollback (delete) a set of extents for the specified OID. + * + * Deletes all the extents that logically follow the specified + * column extent; and sets the HWM for the specified extent. + * @param oid OID of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segmentNum Last segment in partitionNum to be kept. + * @param hwm HWM to be assigned to the last extent that is kept. + * @return 0 on success, -1 on error + */ + EXPORT int rollbackColumnExtents(OID_t oid, uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm) throw(); - /** @brief Rollback (delete) a set of dict store extents for an OID. - * - * Arguments specify the last stripe. Any extents after this are - * deleted. The hwm's of the extents in the last stripe are updated - * based on the contents of the hwm vector. If hwms is a partial list, - * (as in the first stripe of a partition), then any extents in sub- - * sequent segment files for that partition are deleted. - * @param oid OID of the extents to be deleted or updated. - * @param partitionNum Last partition to be kept. - * @param hwms Vector of hwms for the last partition to be kept. - * @return 0 on success, -1 on error - */ - EXPORT int rollbackDictStoreExtents(OID_t oid, - uint32_t partitionNum, - const std::vector& hwms) throw (); + /** @brief Rollback (delete) set of extents for specified OID & DBRoot. + * + * Deletes all the extents that logically follow the specified + * column extent; and sets the HWM for the specified extent. + * @param oid OID of the extents to be deleted. + * @param bDeleteAll Indicates if all extents in oid and dbroot are to + * be deleted; else part#, seg#, and hwm are used. + * @param dbRoot DBRoot of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param segmentNum Last segment in partitionNum to be kept. + * @param hwm HWM to be assigned to the last extent that is kept. + * @return 0 on success, -1 on error + */ + EXPORT int rollbackColumnExtents_DBroot(OID_t oid, bool bDeleteAll, uint16_t dbRoot, uint32_t partitionNum, + uint16_t segmentNum, HWM_t hwm) throw(); - /** @brief Rollback (delete) a set of dict store extents for an OID & - * DBRoot. - * - * Arguments specify the last stripe. Any extents after this are - * deleted. The hwm's of the extents in the last stripe are updated - * based on the contents of the hwm vector. If hwms is a partial list, - * (as in the first stripe of a partition), then any extents in sub- - * sequent segment files for that partition are deleted. If hwms is - * empty then all the extents in dbRoot are deleted. - * @param oid OID of the extents to be deleted or updated. - * @param dbRoot DBRoot of the extents to be deleted. - * @param partitionNum Last partition to be kept. - * @param hwms Vector of hwms for the last partition to be kept. - * @return 0 on success, -1 on error - */ - EXPORT int rollbackDictStoreExtents_DBroot(OID_t oid, - uint16_t dbRoot, - uint32_t partitionNum, - const std::vector& segNums, - const std::vector& hwms) throw (); + /** @brief Rollback (delete) a set of dict store extents for an OID. + * + * Arguments specify the last stripe. Any extents after this are + * deleted. The hwm's of the extents in the last stripe are updated + * based on the contents of the hwm vector. If hwms is a partial list, + * (as in the first stripe of a partition), then any extents in sub- + * sequent segment files for that partition are deleted. + * @param oid OID of the extents to be deleted or updated. + * @param partitionNum Last partition to be kept. + * @param hwms Vector of hwms for the last partition to be kept. + * @return 0 on success, -1 on error + */ + EXPORT int rollbackDictStoreExtents(OID_t oid, uint32_t partitionNum, + const std::vector& hwms) throw(); - /** @brief delete of column extents for the specified extents. - * - * Deletes the extents from extent map - * @param extentInfo the information for extents - */ - EXPORT int deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo) throw(); + /** @brief Rollback (delete) a set of dict store extents for an OID & + * DBRoot. + * + * Arguments specify the last stripe. Any extents after this are + * deleted. The hwm's of the extents in the last stripe are updated + * based on the contents of the hwm vector. If hwms is a partial list, + * (as in the first stripe of a partition), then any extents in sub- + * sequent segment files for that partition are deleted. If hwms is + * empty then all the extents in dbRoot are deleted. + * @param oid OID of the extents to be deleted or updated. + * @param dbRoot DBRoot of the extents to be deleted. + * @param partitionNum Last partition to be kept. + * @param hwms Vector of hwms for the last partition to be kept. + * @return 0 on success, -1 on error + */ + EXPORT int rollbackDictStoreExtents_DBroot(OID_t oid, uint16_t dbRoot, uint32_t partitionNum, + const std::vector& segNums, + const std::vector& hwms) throw(); - /** @brief delete of dictionary extents for the specified extents. - * - * Deletes the extents from extent map - * @param extentInfo the information for extents - */ - EXPORT int deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) throw(); + /** @brief delete of column extents for the specified extents. + * + * Deletes the extents from extent map + * @param extentInfo the information for extents + */ + EXPORT int deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo) throw(); - /** @brief Delete the extents of an OID and invalidate VSS references to them - * - * Delete the extents assigned to an OID and deletes entries in the VSS - * that refer to the LBIDs used by it. - * @note The old version of this function deliberately did not delete the entries - * in the version buffer. - * @note This function is ridiculously slow right now. - * @param OID The OID of the object being deleted - * @return 0 on success, -1 on error - */ - EXPORT int deleteOID(OID_t oid) throw(); + /** @brief delete of dictionary extents for the specified extents. + * + * Deletes the extents from extent map + * @param extentInfo the information for extents + */ + EXPORT int deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) throw(); - /** @brief Delete the extents of OIDs and invalidate VSS references to them - * - * Delete the extents assigned to OIDs and deletes entries in the VSS - * that refer to the LBIDs used by it. - * @note The old version of this function deliberately did not delete the entries - * in the version buffer. - * @param OIDs The OIDs of the object being deleted - * @return 0 on success, -1 on error - */ - EXPORT int deleteOIDs(const OidsMap_t& oids) throw(); + /** @brief Delete the extents of an OID and invalidate VSS references to them + * + * Delete the extents assigned to an OID and deletes entries in the VSS + * that refer to the LBIDs used by it. + * @note The old version of this function deliberately did not delete the entries + * in the version buffer. + * @note This function is ridiculously slow right now. + * @param OID The OID of the object being deleted + * @return 0 on success, -1 on error + */ + EXPORT int deleteOID(OID_t oid) throw(); - /** @brief Set the "high water mark" of an OID, partition, segment - * - * Set the high water mark (aka, the highest numbered written - * block offset) for a specific OID, partition, segment file. - * @param oid (in) The OID - * @param partitionNum (in) The relevant partition number - * @param segmentNum (in) The relevant segment number - * @param hwm (in) The high water mark of oid - * @return 0 on success, -1 on error - */ - EXPORT int setLocalHWM(OID_t, uint32_t partitionNum, uint16_t segmentNum, - HWM_t hwm, bool firstNode) throw(); + /** @brief Delete the extents of OIDs and invalidate VSS references to them + * + * Delete the extents assigned to OIDs and deletes entries in the VSS + * that refer to the LBIDs used by it. + * @note The old version of this function deliberately did not delete the entries + * in the version buffer. + * @param OIDs The OIDs of the object being deleted + * @return 0 on success, -1 on error + */ + EXPORT int deleteOIDs(const OidsMap_t& oids) throw(); - EXPORT int bulkSetHWM(const std::vector&, VER_t transID, - bool firstNode) throw(); + /** @brief Set the "high water mark" of an OID, partition, segment + * + * Set the high water mark (aka, the highest numbered written + * block offset) for a specific OID, partition, segment file. + * @param oid (in) The OID + * @param partitionNum (in) The relevant partition number + * @param segmentNum (in) The relevant segment number + * @param hwm (in) The high water mark of oid + * @return 0 on success, -1 on error + */ + EXPORT int setLocalHWM(OID_t, uint32_t partitionNum, uint16_t segmentNum, HWM_t hwm, + bool firstNode) throw(); - EXPORT int bulkSetHWMAndCP(const std::vector& hwmArgs, - const std::vector& setCPDataArgs, - const std::vector& mergeCPDataArgs, - VER_t transID, bool firstNode) throw(); + EXPORT int bulkSetHWM(const std::vector&, VER_t transID, bool firstNode) throw(); - EXPORT int bulkUpdateDBRoot(const std::vector&) throw(); + EXPORT int bulkSetHWMAndCP(const std::vector& hwmArgs, + const std::vector& setCPDataArgs, + const std::vector& mergeCPDataArgs, VER_t transID, + bool firstNode) throw(); - /** @brief Delete a Partition for the specified OID(s). - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be deleted. - */ - EXPORT int deletePartition(const std::set& oids, - std::set& partitionNums, std::string& emsg) throw(); + EXPORT int bulkUpdateDBRoot(const std::vector&) throw(); - /** @brief Mark a Partition for the specified OID(s) as out of service. - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be marked out of service. - */ - EXPORT int markPartitionForDeletion(const std::set& oids, - std::set& partitionNums, std::string& emsg) throw(); + /** @brief Delete a Partition for the specified OID(s). + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be deleted. + */ + EXPORT int deletePartition(const std::set& oids, std::set& partitionNums, + std::string& emsg) throw(); - /** @brief Mark all Partitions for the specified OID(s) as out of service. - * - * @param OID (in) the OID of interest. - */ - EXPORT int markAllPartitionForDeletion(const std::set& oids) throw(); + /** @brief Mark a Partition for the specified OID(s) as out of service. + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be marked out of service. + */ + EXPORT int markPartitionForDeletion(const std::set& oids, std::set& partitionNums, + std::string& emsg) throw(); - /** @brief Restore a Partition for the specified OID(s). - * - * @param OID (in) the OID of interest. - * @param partitionNums (in) the set of partitions to be restored. - */ - EXPORT int restorePartition(const std::set& oids, - std::set& partitionNum, std::string& emsg) throw(); + /** @brief Mark all Partitions for the specified OID(s) as out of service. + * + * @param OID (in) the OID of interest. + */ + EXPORT int markAllPartitionForDeletion(const std::set& oids) throw(); - /** @brief Delete all extent map rows for the specified dbroot - * - * @param dbroot (in) the dbroot - */ - EXPORT int deleteDBRoot(uint16_t dbroot) throw(); + /** @brief Restore a Partition for the specified OID(s). + * + * @param OID (in) the OID of interest. + * @param partitionNums (in) the set of partitions to be restored. + */ + EXPORT int restorePartition(const std::set& oids, std::set& partitionNum, + std::string& emsg) throw(); - /** @brief Registers a version buffer entry. - * - * Registers a version buffer entry at with - * values of . - * @note The version buffer locations must hold the 'copy' lock - * first. - * @return 0 on success, -1 on error - */ - EXPORT int writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, - uint32_t vbFBO) throw(); + /** @brief Delete all extent map rows for the specified dbroot + * + * @param dbroot (in) the dbroot + */ + EXPORT int deleteDBRoot(uint16_t dbroot) throw(); - /** @brief Bulk registers a version buffer entry. - * - * Similar to writeVBEntry, but registers the version buffer - * entries in bulk for a list of lbids and vbFBOs, for a given - * transID and vbOID. - * @note The version buffer locations must hold the 'copy' lock - * first. - * @return 0 on success, -1 on error - */ - EXPORT int bulkWriteVBEntry(VER_t transID, - const std::vector& lbids, - OID_t vbOID, - const std::vector& vbFBOs) throw(); + /** @brief Registers a version buffer entry. + * + * Registers a version buffer entry at with + * values of . + * @note The version buffer locations must hold the 'copy' lock + * first. + * @return 0 on success, -1 on error + */ + EXPORT int writeVBEntry(VER_t transID, LBID_t lbid, OID_t vbOID, uint32_t vbFBO) throw(); - /** @brief Atomically prepare to copy data to the version buffer - * - * Atomically sets the copy flag on the specified LBID ranges - * and allocate blocks in the version buffer to copy them to. - * If any LBID in the range cannot be locked, none will be - * and this will return -1. - * @param transID The transaction ID doing the operation - * @param ranges (in) A list of LBID ranges that will be copied - * @param freeList (out) On success, a list of ranges of the version - * buffer blocks to copy the LBID range to. - * @return 0 on success, -1 on error. - */ + /** @brief Bulk registers a version buffer entry. + * + * Similar to writeVBEntry, but registers the version buffer + * entries in bulk for a list of lbids and vbFBOs, for a given + * transID and vbOID. + * @note The version buffer locations must hold the 'copy' lock + * first. + * @return 0 on success, -1 on error + */ + EXPORT int bulkWriteVBEntry(VER_t transID, const std::vector& lbids, OID_t vbOID, + const std::vector& vbFBOs) throw(); - /* Note, the params to the analogous DBRM class fcn are slightly different. - * It takes a DBRoot param instead of a VB OID. The conversion is - * done in the controllernode b/c the OID server is housed there. - */ - EXPORT int beginVBCopy(VER_t transID, uint16_t vbOID, - const LBIDRange_v& ranges, VBRange_v& freeList, bool flushPMCache) throw(); + /** @brief Atomically prepare to copy data to the version buffer + * + * Atomically sets the copy flag on the specified LBID ranges + * and allocate blocks in the version buffer to copy them to. + * If any LBID in the range cannot be locked, none will be + * and this will return -1. + * @param transID The transaction ID doing the operation + * @param ranges (in) A list of LBID ranges that will be copied + * @param freeList (out) On success, a list of ranges of the version + * buffer blocks to copy the LBID range to. + * @return 0 on success, -1 on error. + */ - /** @brief Atomically unset the copy lock & update the VSS. Beware! Read the warning! - * - * Atomically unset the copy lock for the specified LBID ranges - * and add a new locked VSS entry for each LBID in the range. - * @note The elements of the ranges parameter MUST be the - * same elements passed to beginVBCopy(). The number and order of the - * elements can be different, but every element in ranges must also - * have been an element in beginVBCopy's ranges. - * @return 0 on success, -1 on error. - */ - EXPORT int endVBCopy(VER_t transID, const LBIDRange_v& ranges) - throw(); + /* Note, the params to the analogous DBRM class fcn are slightly different. + * It takes a DBRoot param instead of a VB OID. The conversion is + * done in the controllernode b/c the OID server is housed there. + */ + EXPORT int beginVBCopy(VER_t transID, uint16_t vbOID, const LBIDRange_v& ranges, VBRange_v& freeList, + bool flushPMCache) throw(); - /** @brief Commit the changes made for the given transaction. - * - * This unlocks the VSS entries with VerID = transID. - * @return 0 on success, -1 on error. - */ - EXPORT int vbCommit(VER_t transID) throw(); + /** @brief Atomically unset the copy lock & update the VSS. Beware! Read the warning! + * + * Atomically unset the copy lock for the specified LBID ranges + * and add a new locked VSS entry for each LBID in the range. + * @note The elements of the ranges parameter MUST be the + * same elements passed to beginVBCopy(). The number and order of the + * elements can be different, but every element in ranges must also + * have been an element in beginVBCopy's ranges. + * @return 0 on success, -1 on error. + */ + EXPORT int endVBCopy(VER_t transID, const LBIDRange_v& ranges) throw(); - /** @brief Reverse the changes made during the given transaction. - * - * Record that the given LBID was reverted to version verID. - * @warning This removes the copy locks held on all ranges by transID. - * @param transID The transaction ID - * @param lbidList The list of ranges to rollback. - * @param verID The version of the block now in the database. - * @return 0 on success, -1 on error. - */ - EXPORT int vbRollback(VER_t transID, const LBIDRange_v& lbidList, - bool flushPMCache) throw(); + /** @brief Commit the changes made for the given transaction. + * + * This unlocks the VSS entries with VerID = transID. + * @return 0 on success, -1 on error. + */ + EXPORT int vbCommit(VER_t transID) throw(); - /** @brief Reverse the changes made during the given transaction. - * - * Record that the given LBID was reverted to version verID. - * @warning This removes the copy locks held on all ranges by transID. - * @param transID The transaction ID - * @param lbidList The list of singular LBIDs to rollback. - * @param verID The version of the block now in the database. - * @return 0 on success, -1 on error. - */ - EXPORT int vbRollback(VER_t transID, const std::vector& lbidList, - bool flushPMCache) throw(); + /** @brief Reverse the changes made during the given transaction. + * + * Record that the given LBID was reverted to version verID. + * @warning This removes the copy locks held on all ranges by transID. + * @param transID The transaction ID + * @param lbidList The list of ranges to rollback. + * @param verID The version of the block now in the database. + * @return 0 on success, -1 on error. + */ + EXPORT int vbRollback(VER_t transID, const LBIDRange_v& lbidList, bool flushPMCache) throw(); - EXPORT int clear() throw(); + /** @brief Reverse the changes made during the given transaction. + * + * Record that the given LBID was reverted to version verID. + * @warning This removes the copy locks held on all ranges by transID. + * @param transID The transaction ID + * @param lbidList The list of singular LBIDs to rollback. + * @param verID The version of the block now in the database. + * @return 0 on success, -1 on error. + */ + EXPORT int vbRollback(VER_t transID, const std::vector& lbidList, bool flushPMCache) throw(); - /** @brief Check the consistency of each data structure - * - * Check the consistency of each data structure - * @return 0 on success, -1 on error. - */ - EXPORT int checkConsistency() throw(); + EXPORT int clear() throw(); - EXPORT void confirmChanges() throw(); - EXPORT void undoChanges() throw(); + /** @brief Check the consistency of each data structure + * + * Check the consistency of each data structure + * @return 0 on success, -1 on error. + */ + EXPORT int checkConsistency() throw(); - EXPORT int loadExtentMap(const std::string& filename); - EXPORT int saveExtentMap(const std::string& filename); + EXPORT void confirmChanges() throw(); + EXPORT void undoChanges() throw(); - // Casual partitioning support - // - /** @brief mark the extent containing the lbid as not having valid max and min values - * - **/ - EXPORT int markExtentInvalid(const LBID_t lbid, - execplan::CalpontSystemCatalog::ColDataType colDataType); - EXPORT int markExtentsInvalid(const std::vector& lbids, - const std::vector& colDataTypes); + EXPORT int loadExtentMap(const std::string& filename); + EXPORT int saveExtentMap(const std::string& filename); - /** @brief update the extent with the lbidRange with max, min, & seqNum values - * - **/ - EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, - const int32_t seqNum, bool firstNode); + // Casual partitioning support + // + /** @brief mark the extent containing the lbid as not having valid max and min values + * + **/ + EXPORT int markExtentInvalid(const LBID_t lbid, execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT int markExtentsInvalid(const std::vector& lbids, + const std::vector& colDataTypes); - // Casual partitioning support - // - /** @brief Sets min and max values from the information in the passed map. - * - * @param cpMaxMinMap - Map with cp info. The key must be the starting LBID. - * - **/ - // @bug 1970. Added setExtentsMaxMin. - EXPORT int setExtentsMaxMin(const CPMaxMinMap_t& cpMaxMinMap, bool firstNode); + /** @brief update the extent with the lbidRange with max, min, & seqNum values + * + **/ + EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum, + bool firstNode); - /** @brief Merges list of min/max values with current CP min/max info - * - * @param cpMaxMinMap - Map with CP info. The key is the starting LBID - */ - EXPORT int mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMaxMinMap); + // Casual partitioning support + // + /** @brief Sets min and max values from the information in the passed map. + * + * @param cpMaxMinMap - Map with cp info. The key must be the starting LBID. + * + **/ + // @bug 1970. Added setExtentsMaxMin. + EXPORT int setExtentsMaxMin(const CPMaxMinMap_t& cpMaxMinMap, bool firstNode); - /* Write-side copylocks interface */ - EXPORT int dmlLockLBIDRanges(const std::vector& ranges, int txnID); - EXPORT int dmlReleaseLBIDRanges(const std::vector& ranges); + /** @brief Merges list of min/max values with current CP min/max info + * + * @param cpMaxMinMap - Map with CP info. The key is the starting LBID + */ + EXPORT int mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMaxMinMap); - EXPORT int loadState(std::string filename) throw(); - EXPORT int saveState(std::string filename) throw(); + /* Write-side copylocks interface */ + EXPORT int dmlLockLBIDRanges(const std::vector& ranges, int txnID); + EXPORT int dmlReleaseLBIDRanges(const std::vector& ranges); - EXPORT const bool* getEMFLLockStatus(); - EXPORT const bool* getEMLockStatus(); - EXPORT const bool* getVBBMLockStatus(); - EXPORT const bool* getVSSLockStatus(); + EXPORT int loadState(std::string filename) throw(); + EXPORT int saveState(std::string filename) throw(); -private: - explicit SlaveDBRMNode(const SlaveDBRMNode& brm); - SlaveDBRMNode& operator=(const SlaveDBRMNode& brm); - int lookup(OID_t oid, LBIDRange_v& lbidList) throw(); + EXPORT const bool* getEMFLLockStatus(); + EXPORT const bool* getEMLockStatus(); + EXPORT const bool* getVBBMLockStatus(); + EXPORT const bool* getVSSLockStatus(); - MasterSegmentTable mst; - ExtentMap em; - VBBM vbbm; - VSS vss; - CopyLocks copylocks; - bool locked[3]; // 0 = VBBM, 1 = VSS, 2 = CopyLocks + private: + explicit SlaveDBRMNode(const SlaveDBRMNode& brm); + SlaveDBRMNode& operator=(const SlaveDBRMNode& brm); + int lookup(OID_t oid, LBIDRange_v& lbidList) throw(); + MasterSegmentTable mst; + ExtentMap em; + VBBM vbbm; + VSS vss; + CopyLocks copylocks; + bool locked[3]; // 0 = VBBM, 1 = VSS, 2 = CopyLocks }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index 2482791d6..8651dc823 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -20,7 +20,6 @@ * ****************************************************************************/ - #include #include #include @@ -45,161 +44,152 @@ using namespace BRM; using namespace std; SlaveComm* comm; -bool die= false; +bool die = false; boost::thread_group monitorThreads; class Opt { -protected: - const char *m_progname; - const char *m_nodename; - bool m_fg; -public: - Opt(int argc, char **argv) - :m_progname(argv[0]), - m_nodename(argc > 1 ? argv[1] : nullptr), - m_fg(argc > 2 && string(argv[2]) == "fg") - { } + protected: + const char* m_progname; + const char* m_nodename; + bool m_fg; + + public: + Opt(int argc, char** argv) + : m_progname(argv[0]), m_nodename(argc > 1 ? argv[1] : nullptr), m_fg(argc > 2 && string(argv[2]) == "fg") + { + } }; - -class ServiceWorkerNode: public Service, public Opt +class ServiceWorkerNode : public Service, public Opt { -protected: - void setupChildSignalHandlers(); + protected: + void setupChildSignalHandlers(); -public: - ServiceWorkerNode(const Opt &opt) - :Service("WorkerNode"), Opt(opt) - { } - void LogErrno() override - { - perror(m_progname); - log_errno(std::string(m_progname)); - } - void ParentLogChildMessage(const std::string &str) override - { - log(str, logging::LOG_TYPE_INFO); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } + public: + ServiceWorkerNode(const Opt& opt) : Service("WorkerNode"), Opt(opt) + { + } + void LogErrno() override + { + perror(m_progname); + log_errno(std::string(m_progname)); + } + void ParentLogChildMessage(const std::string& str) override + { + log(str, logging::LOG_TYPE_INFO); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } }; - void stop(int sig) { - if (!die) - { - die = true; - comm->stop(); - monitorThreads.interrupt_all(); - } + if (!die) + { + die = true; + comm->stop(); + monitorThreads.interrupt_all(); + } } void reset(int sig) { - comm->reset(); + comm->reset(); } - void ServiceWorkerNode::setupChildSignalHandlers() { #ifdef SIGHUP - signal(SIGHUP, reset); + signal(SIGHUP, reset); #endif - signal(SIGINT, stop); - signal(SIGTERM, stop); + signal(SIGINT, stop); + signal(SIGTERM, stop); #ifdef SIGPIPE - signal(SIGPIPE, SIG_IGN); + signal(SIGPIPE, SIG_IGN); #endif - struct sigaction ign; - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); } - int ServiceWorkerNode::Child() { - setupChildSignalHandlers(); + setupChildSignalHandlers(); - SlaveDBRMNode slave; - ShmKeys keys; + SlaveDBRMNode slave; + ShmKeys keys; - try - { - comm = new SlaveComm(std::string(m_nodename), &slave); - NotifyServiceStarted(); - } - catch (exception& e) - { - ostringstream os; - os << "An error occured: " << e.what(); - cerr << os.str() << endl; - log(os.str()); - NotifyServiceInitializationFailed(); - return 1; - } + try + { + comm = new SlaveComm(std::string(m_nodename), &slave); + NotifyServiceStarted(); + } + catch (exception& e) + { + ostringstream os; + os << "An error occured: " << e.what(); + cerr << os.str() << endl; + log(os.str()); + NotifyServiceInitializationFailed(); + return 1; + } - /* Start 4 threads to monitor write lock state */ - monitorThreads.create_thread(RWLockMonitor - (&die, slave.getEMFLLockStatus(), keys.KEYRANGE_EMFREELIST_BASE)); - monitorThreads.create_thread(RWLockMonitor - (&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE)); - monitorThreads.create_thread(RWLockMonitor - (&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE)); - monitorThreads.create_thread(RWLockMonitor - (&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE)); + /* Start 4 threads to monitor write lock state */ + monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMFLLockStatus(), keys.KEYRANGE_EMFREELIST_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE)); - try - { - comm->run(); - } - catch (exception& e) - { - ostringstream os; - os << "An error occurred: " << e.what(); - cerr << os.str() << endl; - log(os.str()); - return 1; - } - return 0; + try + { + comm->run(); + } + catch (exception& e) + { + ostringstream os; + os << "An error occurred: " << e.what(); + cerr << os.str() << endl; + log(os.str()); + return 1; + } + return 0; } - int main(int argc, char** argv) { - Opt opt(argc, argv); + Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); - BRM::logInit ( BRM::SubSystemLogId_workerNode ); + BRM::logInit(BRM::SubSystemLogId_workerNode); - /* - Need to shutdown TheadPool before fork(), - otherwise it would get stuck when trying to join fPruneThread. - */ - joblist::JobStep::jobstepThreadPool.stop(); + /* + Need to shutdown TheadPool before fork(), + otherwise it would get stuck when trying to join fPruneThread. + */ + joblist::JobStep::jobstepThreadPool.stop(); - if (argc < 2) - { - ostringstream os; - os << "Usage: " << argv[0] << " DBRM_WorkerN"; - cerr << os.str() << endl; - log(os.str()); - exit(1); - } + if (argc < 2) + { + ostringstream os; + os << "Usage: " << argv[0] << " DBRM_WorkerN"; + cerr << os.str() << endl; + log(os.str()); + exit(1); + } - // TODO: this should move to child() probably, like in masternode.cpp - idbdatafile::IDBPolicy::configIDBPolicy(); + // TODO: this should move to child() probably, like in masternode.cpp + idbdatafile::IDBPolicy::configIDBPolicy(); - return ServiceWorkerNode(opt).Run(); + return ServiceWorkerNode(opt).Run(); } diff --git a/versioning/BRM/tablelockserver.cpp b/versioning/BRM/tablelockserver.cpp index 5772e7135..fd13d211e 100644 --- a/versioning/BRM/tablelockserver.cpp +++ b/versioning/BRM/tablelockserver.cpp @@ -37,18 +37,19 @@ using namespace idbdatafile; namespace BRM { - TableLockServer::TableLockServer(SessionManagerServer* sm) : sms(sm) { - boost::mutex::scoped_lock lk(mutex); - config::Config* config = config::Config::makeConfig(); + boost::mutex::scoped_lock lk(mutex); + config::Config* config = config::Config::makeConfig(); - filename = config->getConfig("SystemConfig", "TableLockSaveFile"); + filename = config->getConfig("SystemConfig", "TableLockSaveFile"); - if (filename == "") - throw invalid_argument("TableLockServer: Need to define SystemConfig/TableLockSaveFile in config file"); // todo, replace this + if (filename == "") + throw invalid_argument( + "TableLockServer: Need to define SystemConfig/TableLockSaveFile in config file"); // todo, replace + // this - load(); + load(); } TableLockServer::~TableLockServer() @@ -58,264 +59,259 @@ TableLockServer::~TableLockServer() // call with lock held void TableLockServer::save() { - lit_t it; - uint32_t count = locks.size(); + lit_t it; + uint32_t count = locks.size(); - const char* filename_p = filename.c_str(); + const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", 0)); + scoped_ptr out( + IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", 0)); + if (!out) + throw runtime_error("TableLockServer::save(): could not open save file"); + + out->write((char*)&count, 4); + + for (it = locks.begin(); it != locks.end(); ++it) + { if (!out) - throw runtime_error("TableLockServer::save(): could not open save file"); + throw runtime_error("TableLockServer::save(): could not write save file"); - out->write((char*) &count, 4); - - for (it = locks.begin(); it != locks.end(); ++it) - { - if (!out) - throw runtime_error("TableLockServer::save(): could not write save file"); - - it->second.serialize(out.get()); - } + it->second.serialize(out.get()); + } } // call with lock held void TableLockServer::load() { - uint32_t size; - uint32_t i = 0; - TableLockInfo tli; + uint32_t size; + uint32_t i = 0; + TableLockInfo tli; - /* Need to standardize the file error handling */ - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "rb", 0)); + /* Need to standardize the file error handling */ + const char* filename_p = filename.c_str(); + scoped_ptr in( + IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "rb", 0)); - if (!in) + if (!in) + { + ostringstream os; + os << "TableLockServer::load(): could not open the save file" << filename; + log(os.str(), logging::LOG_TYPE_DEBUG); + return; + } + + try + { + in->read((char*)&size, 4); + + for (i = 0; i < size; i++) { - ostringstream os; - os << "TableLockServer::load(): could not open the save file" - << filename; - log(os.str(), logging::LOG_TYPE_DEBUG); - return; - } - - try - { - in->read((char*) &size, 4); - - for (i = 0; i < size; i++) - { - tli.deserialize(in.get()); - tli.id = sms->getUnique64(); // Need new #s... - - if (tli.id == 0) // 0 is an error code - tli.id = sms->getUnique64(); - - locks[tli.id] = tli; - } - } - catch (std::exception& e) - { - ostringstream os; - os << "TableLockServer::load(): could not load save file " << filename << - " loaded " << i << "/" << size << " entries\n"; - log(os.str(), logging::LOG_TYPE_DEBUG); - throw; + tli.deserialize(in.get()); + tli.id = sms->getUnique64(); // Need new #s... + + if (tli.id == 0) // 0 is an error code + tli.id = sms->getUnique64(); + + locks[tli.id] = tli; } + } + catch (std::exception& e) + { + ostringstream os; + os << "TableLockServer::load(): could not load save file " << filename << " loaded " << i << "/" << size + << " entries\n"; + log(os.str(), logging::LOG_TYPE_DEBUG); + throw; + } } - // throws on a failed save() uint64_t TableLockServer::lock(TableLockInfo* tli) { - set dbroots; - lit_t it; - uint32_t i; - boost::mutex::scoped_lock lk(mutex); + set dbroots; + lit_t it; + uint32_t i; + boost::mutex::scoped_lock lk(mutex); - for (i = 0; i < tli->dbrootList.size(); i++) - dbroots.insert(tli->dbrootList[i]); + for (i = 0; i < tli->dbrootList.size(); i++) + dbroots.insert(tli->dbrootList[i]); - for (it = locks.begin(); it != locks.end(); ++it) + for (it = locks.begin(); it != locks.end(); ++it) + { + if (it->second.overlaps(*tli, dbroots)) { - if (it->second.overlaps(*tli, dbroots)) - { - tli->ownerName = it->second.ownerName; - tli->ownerPID = it->second.ownerPID; - tli->ownerSessionID = it->second.ownerSessionID; - tli->ownerTxnID = it->second.ownerTxnID; - return false; - } + tli->ownerName = it->second.ownerName; + tli->ownerPID = it->second.ownerPID; + tli->ownerSessionID = it->second.ownerSessionID; + tli->ownerTxnID = it->second.ownerTxnID; + return false; } + } + tli->id = sms->getUnique64(); + + if (tli->id == 0) // 0 is an error code tli->id = sms->getUnique64(); - if (tli->id == 0) // 0 is an error code - tli->id = sms->getUnique64(); + locks[tli->id] = *tli; - locks[tli->id] = *tli; + try + { + save(); + } + catch (...) + { + locks.erase(tli->id); + throw; + } - try - { - save(); - } - catch (...) - { - locks.erase(tli->id); - throw; - } - - return tli->id; + return tli->id; } bool TableLockServer::unlock(uint64_t id) { - std::map::iterator it; - TableLockInfo tli; + std::map::iterator it; + TableLockInfo tli; - boost::mutex::scoped_lock lk(mutex); - it = locks.find(id); + boost::mutex::scoped_lock lk(mutex); + it = locks.find(id); - if (it != locks.end()) + if (it != locks.end()) + { + tli = it->second; + locks.erase(it); + + try { - tli = it->second; - locks.erase(it); - - try - { - save(); - } - catch (...) - { - locks[tli.id] = tli; - throw; - } - - return true; + save(); + } + catch (...) + { + locks[tli.id] = tli; + throw; } - return false; + return true; + } + + return false; } bool TableLockServer::changeState(uint64_t id, LockState state) { - lit_t it; - boost::mutex::scoped_lock lk(mutex); - LockState old; + lit_t it; + boost::mutex::scoped_lock lk(mutex); + LockState old; - it = locks.find(id); + it = locks.find(id); - if (it == locks.end()) - return false; + if (it == locks.end()) + return false; - old = it->second.state; - it->second.state = state; + old = it->second.state; + it->second.state = state; - try - { - save(); - } - catch (...) - { - it->second.state = old; - throw; - } + try + { + save(); + } + catch (...) + { + it->second.state = old; + throw; + } - return true; + return true; } bool TableLockServer::changeOwner(uint64_t id, const string& ownerName, uint32_t pid, int32_t session, int32_t txnID) { - lit_t it; - boost::mutex::scoped_lock lk(mutex); - string oldName; - uint32_t oldPID; - int32_t oldSession; - int32_t oldTxnID; + lit_t it; + boost::mutex::scoped_lock lk(mutex); + string oldName; + uint32_t oldPID; + int32_t oldSession; + int32_t oldTxnID; - it = locks.find(id); + it = locks.find(id); - if (it == locks.end()) - return false; + if (it == locks.end()) + return false; - oldName = it->second.ownerName; - oldPID = it->second.ownerPID; - oldSession = it->second.ownerSessionID; - oldTxnID = it->second.ownerTxnID; - it->second.ownerName = ownerName; - it->second.ownerPID = pid; - it->second.ownerSessionID = session; - it->second.ownerTxnID = txnID; + oldName = it->second.ownerName; + oldPID = it->second.ownerPID; + oldSession = it->second.ownerSessionID; + oldTxnID = it->second.ownerTxnID; + it->second.ownerName = ownerName; + it->second.ownerPID = pid; + it->second.ownerSessionID = session; + it->second.ownerTxnID = txnID; - try - { - save(); - } - catch (...) - { - it->second.ownerName = oldName; - it->second.ownerPID = oldPID; - it->second.ownerSessionID = oldSession; - it->second.ownerTxnID = oldTxnID; - throw; - } + try + { + save(); + } + catch (...) + { + it->second.ownerName = oldName; + it->second.ownerPID = oldPID; + it->second.ownerSessionID = oldSession; + it->second.ownerTxnID = oldTxnID; + throw; + } - return true; + return true; } vector TableLockServer::getAllLocks() const { - vector ret; - boost::mutex::scoped_lock lk(mutex); - constlit_t it; + vector ret; + boost::mutex::scoped_lock lk(mutex); + constlit_t it; - for (it = locks.begin(); it != locks.end(); ++it) - ret.push_back(it->second); + for (it = locks.begin(); it != locks.end(); ++it) + ret.push_back(it->second); - return ret; + return ret; } void TableLockServer::releaseAllLocks() { - std::map tmp; + std::map tmp; - boost::mutex::scoped_lock lk(mutex); + boost::mutex::scoped_lock lk(mutex); + tmp.swap(locks); + + try + { + save(); + } + catch (...) + { tmp.swap(locks); - - try - { - save(); - } - catch (...) - { - tmp.swap(locks); - throw; - } + throw; + } } bool TableLockServer::getLockInfo(uint64_t id, TableLockInfo* out) const { - constlit_t it; - boost::mutex::scoped_lock lk(mutex); + constlit_t it; + boost::mutex::scoped_lock lk(mutex); - it = locks.find(id); + it = locks.find(id); - if (out == NULL) - return (it != locks.end()); + if (out == NULL) + return (it != locks.end()); - if (it != locks.end()) - { - *out = it->second; - return true; - } - - return false; -} + if (it != locks.end()) + { + *out = it->second; + return true; + } + return false; } +} // namespace BRM diff --git a/versioning/BRM/tablelockserver.h b/versioning/BRM/tablelockserver.h index dc10771b2..7573be535 100644 --- a/versioning/BRM/tablelockserver.h +++ b/versioning/BRM/tablelockserver.h @@ -36,35 +36,34 @@ namespace BRM { - class TableLockServer { -public: - EXPORT TableLockServer(SessionManagerServer*); - virtual ~TableLockServer(); + public: + EXPORT TableLockServer(SessionManagerServer*); + virtual ~TableLockServer(); - EXPORT uint64_t lock(TableLockInfo*); - EXPORT bool unlock(uint64_t id); - EXPORT bool changeState(uint64_t id, LockState state); - EXPORT bool changeOwner(uint64_t id, const std::string& ownerName, uint32_t pid, int32_t sessionID, - int32_t txnID); - EXPORT std::vector getAllLocks() const; - EXPORT void releaseAllLocks(); - EXPORT bool getLockInfo(uint64_t id, TableLockInfo* out) const; + EXPORT uint64_t lock(TableLockInfo*); + EXPORT bool unlock(uint64_t id); + EXPORT bool changeState(uint64_t id, LockState state); + EXPORT bool changeOwner(uint64_t id, const std::string& ownerName, uint32_t pid, int32_t sessionID, + int32_t txnID); + EXPORT std::vector getAllLocks() const; + EXPORT void releaseAllLocks(); + EXPORT bool getLockInfo(uint64_t id, TableLockInfo* out) const; -private: - void load(); - void save(); + private: + void load(); + void save(); - mutable boost::mutex mutex; - std::map locks; - typedef std::map::iterator lit_t; - typedef std::map::const_iterator constlit_t; - std::string filename; - SessionManagerServer* sms; + mutable boost::mutex mutex; + std::map locks; + typedef std::map::iterator lit_t; + typedef std::map::const_iterator constlit_t; + std::string filename; + SessionManagerServer* sms; }; -} +} // namespace BRM #undef EXPORT diff --git a/versioning/BRM/tablelocktest.cpp b/versioning/BRM/tablelocktest.cpp index 9b914c4e0..2019874b3 100644 --- a/versioning/BRM/tablelocktest.cpp +++ b/versioning/BRM/tablelocktest.cpp @@ -2,236 +2,228 @@ #include "dbrm.h" - using namespace std; using namespace BRM; void grabLock() { - DBRM dbrm; - uint64_t id; - vector pmlist; + DBRM dbrm; + uint64_t id; + vector pmlist; - string name = "init"; - uint32_t pid = 1; - int32_t sid = 5678; + string name = "init"; + uint32_t pid = 1; + int32_t sid = 5678; - pmlist.push_back(1); + pmlist.push_back(1); - try - { - id = dbrm.getTableLock(pmlist, 1234, &name, &pid, &sid, &sid, LOADING); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + id = dbrm.getTableLock(pmlist, 1234, &name, &pid, &sid, &sid, LOADING); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - cout << "lock ID = " << id << endl; - - if (id == 0) - cout << " existing owner name = " << name << " pid = " << pid << " session = " << sid << endl; + cout << "lock ID = " << id << endl; + if (id == 0) + cout << " existing owner name = " << name << " pid = " << pid << " session = " << sid << endl; } void releaseLock(uint64_t id) { - DBRM dbrm; - bool ret; + DBRM dbrm; + bool ret; - cout << "releasing lock " << id << endl; + cout << "releasing lock " << id << endl; - try - { - ret = dbrm.releaseTableLock(id); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + ret = dbrm.releaseTableLock(id); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - if (ret) - cout << "success\n"; - else - cout << "failed\n"; + if (ret) + cout << "success\n"; + else + cout << "failed\n"; } void changeState(uint64_t id) { - DBRM dbrm; - bool ret; + DBRM dbrm; + bool ret; - cout << "changing state of lock " << id << endl; + cout << "changing state of lock " << id << endl; - try - { - ret = dbrm.changeState(id, CLEANUP); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + ret = dbrm.changeState(id, CLEANUP); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - if (ret) - cout << "success\n"; - else - cout << "failed\n"; + if (ret) + cout << "success\n"; + else + cout << "failed\n"; } void changeOwner(uint64_t id) { - DBRM dbrm; - bool ret; + DBRM dbrm; + bool ret; - cout << "changing owner of lock " << id << endl; + cout << "changing owner of lock " << id << endl; - try - { - ret = dbrm.changeOwner(id, "blah blah", 5678, 1234, 9999); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + ret = dbrm.changeOwner(id, "blah blah", 5678, 1234, 9999); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - if (ret) - cout << "success\n"; - else - cout << "failed\n"; + if (ret) + cout << "success\n"; + else + cout << "failed\n"; } void printAllLocks() { - DBRM dbrm; - vector locks; - uint32_t i; + DBRM dbrm; + vector locks; + uint32_t i; - try - { - locks = dbrm.getAllTableLocks(); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + locks = dbrm.getAllTableLocks(); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - cout << "got " << locks.size() << " locks\n"; + cout << "got " << locks.size() << " locks\n"; - for (i = 0; i < locks.size(); i++) - cout << " lock[" << i << "] id=" << locks[i].id << " tableOID=" << locks[i].tableOID << - " owner: " << locks[i].ownerName << ":" << locks[i].ownerPID << ":" << - locks[i].ownerSessionID << " " << ctime(&locks[i].creationTime) << endl; + for (i = 0; i < locks.size(); i++) + cout << " lock[" << i << "] id=" << locks[i].id << " tableOID=" << locks[i].tableOID + << " owner: " << locks[i].ownerName << ":" << locks[i].ownerPID << ":" << locks[i].ownerSessionID + << " " << ctime(&locks[i].creationTime) << endl; } void releaseAllLocks() { - DBRM dbrm; + DBRM dbrm; - try - { - dbrm.releaseAllTableLocks(); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + dbrm.releaseAllTableLocks(); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - cout << "OK\n"; + cout << "OK\n"; } void getLockInfo(uint64_t id) { - DBRM dbrm; - bool ret; - TableLockInfo tli; + DBRM dbrm; + bool ret; + TableLockInfo tli; - try - { - ret = dbrm.getTableLockInfo(id, &tli); - } - catch (exception& e) - { - cout << "caught an exception: " << e.what() << endl; - exit(1); - } + try + { + ret = dbrm.getTableLockInfo(id, &tli); + } + catch (exception& e) + { + cout << "caught an exception: " << e.what() << endl; + exit(1); + } - if (ret) - cout << "id=" << tli.id << " tableOID=" << tli.tableOID << - " owner: " << tli.ownerName << ":" << tli.ownerPID << ":" << - tli.ownerSessionID << endl; - else - cout << "failed\n"; + if (ret) + cout << "id=" << tli.id << " tableOID=" << tli.tableOID << " owner: " << tli.ownerName << ":" + << tli.ownerPID << ":" << tli.ownerSessionID << endl; + else + cout << "failed\n"; } int main(int argc, char** argv) { + if (argc < 2) + { + cout << "Usage: " << argv[0] << " g | r | s | o | p | R | z. Check the code to see what they do. :P\n"; + exit(1); + } - if (argc < 2) + char cmd = argv[1][0]; + + if (cmd == 'g') + grabLock(); + else if (cmd == 'r') + { + if (argc < 3) { - cout << "Usage: " << argv[0] << " g | r | s | o | p | R | z. Check the code to see what they do. :P\n"; - exit(1); + cout << "need an ID\n"; + exit(1); } - char cmd = argv[1][0]; - - if (cmd == 'g') - grabLock(); - else if (cmd == 'r') + releaseLock(atoi(argv[2])); + } + else if (cmd == 's') + { + if (argc < 3) { - if (argc < 3) - { - cout << "need an ID\n"; - exit(1); - } - - releaseLock(atoi(argv[2])); + cout << "need an ID\n"; + exit(1); } - else if (cmd == 's') + + changeState(atoi(argv[2])); + } + else if (cmd == 'o') + { + if (argc < 3) { - if (argc < 3) - { - cout << "need an ID\n"; - exit(1); - } - - changeState(atoi(argv[2])); + cout << "need an ID\n"; + exit(1); } - else if (cmd == 'o') + + changeOwner(atoi(argv[2])); + } + else if (cmd == 'p') + printAllLocks(); + else if (cmd == 'R') + releaseAllLocks(); + else if (cmd == 'z') + { + if (argc < 3) { - if (argc < 3) - { - cout << "need an ID\n"; - exit(1); - } - - changeOwner(atoi(argv[2])); + cout << "need an ID\n"; + exit(1); } - else if (cmd == 'p') - printAllLocks(); - else if (cmd == 'R') - releaseAllLocks(); - else if (cmd == 'z') - { - if (argc < 3) - { - cout << "need an ID\n"; - exit(1); - } - getLockInfo(atoi(argv[2])); - } - else - cout << "bad command, need g | r | s | o | p\n"; + getLockInfo(atoi(argv[2])); + } + else + cout << "bad command, need g | r | s | o | p\n"; - exit(0); + exit(0); } - - - - diff --git a/versioning/BRM/tdriver-brm.cpp b/versioning/BRM/tdriver-brm.cpp index 40a1cec3a..ef9e4cb89 100644 --- a/versioning/BRM/tdriver-brm.cpp +++ b/versioning/BRM/tdriver-brm.cpp @@ -50,2401 +50,2343 @@ pthread_mutex_t mutex; struct Range { - LBID_t start, end, nextBlock; - VER_t txnID; - Range() - { - start = end = nextBlock = 0; - txnID = 0; - } + LBID_t start, end, nextBlock; + VER_t txnID; + Range() + { + start = end = nextBlock = 0; + txnID = 0; + } }; static void* BRMRunner_2(void* arg) { + vector copyList, copiedList, committedList; + vector::iterator rit; + vector writtenList; + vector::iterator lit; - vector copyList, copiedList, committedList; - vector::iterator rit; - vector writtenList; - vector::iterator lit; + pthread_mutex_t listMutex; + int op; + uint32_t randstate; + BlockResolutionManager* brm; + struct timeval tv; - pthread_mutex_t listMutex; - int op; - uint32_t randstate; - BlockResolutionManager* brm; - struct timeval tv; + pthread_mutex_init(&listMutex, NULL); + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new BlockResolutionManager(); - pthread_mutex_init(&listMutex, NULL); - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - brm = new BlockResolutionManager(); + while (!threadStop) + { + op = rand_r(&randstate) % 9; - while (!threadStop) + switch (op) { - op = rand_r(&randstate) % 9; + case 0: // beginVBCopy + { + int blockCount, size, err; + Range newEntry; + VBRange_v vbRanges; + VBRange_v::iterator vit; + LBIDRange_v ranges; + LBIDRange range; + VER_t txnID; - switch (op) - { - case 0: // beginVBCopy - { - int blockCount, size, err; - Range newEntry; - VBRange_v vbRanges; - VBRange_v::iterator vit; - LBIDRange_v ranges; - LBIDRange range; - VER_t txnID; + size = rand_r(&randstate) % 10000; - size = rand_r(&randstate) % 10000; + pthread_mutex_lock(&mutex); + newEntry.start = lbidCounter; + lbidCounter += size; + txnID = nextTxnID++; + pthread_mutex_unlock(&mutex); - pthread_mutex_lock(&mutex); - newEntry.start = lbidCounter; - lbidCounter += size; - txnID = nextTxnID++; - pthread_mutex_unlock(&mutex); + newEntry.nextBlock = newEntry.start; + newEntry.end = newEntry.start + size; + range.start = newEntry.start; + range.size = size; - newEntry.nextBlock = newEntry.start; - newEntry.end = newEntry.start + size; - range.start = newEntry.start; - range.size = size; + err = brm->beginVBCopy(txnID, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); - err = brm->beginVBCopy(txnID, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); + for (blockCount = 0, vit = vbRanges.begin(); vit != vbRanges.end(); vit++) + blockCount += (*vit).size; - for (blockCount = 0, vit = vbRanges.begin(); vit != vbRanges.end(); vit++) - blockCount += (*vit).size; + CPPUNIT_ASSERT(blockCount == size); - CPPUNIT_ASSERT(blockCount == size); + pthread_mutex_lock(&listMutex); + copyList.push_back(newEntry); + pthread_mutex_unlock(&listMutex); - pthread_mutex_lock(&listMutex); - copyList.push_back(newEntry); - pthread_mutex_unlock(&listMutex); + err = brm->beginVBCopy(txnID, ranges, vbRanges); + CPPUNIT_ASSERT(err == -1); + break; + } - err = brm->beginVBCopy(txnID, ranges, vbRanges); - CPPUNIT_ASSERT(err == -1); - break; - } + case 1: // writeVBEntry + { + int randIndex; + VER_t txnID; + Range* entry; - case 1: // writeVBEntry - { - int randIndex; - VER_t txnID; - Range* entry; + pthread_mutex_lock(&listMutex); - pthread_mutex_lock(&listMutex); + if (copyList.size() == 0) + break; - if (copyList.size() == 0) - break; + randIndex = rand_r(&randstate) % copyList.size(); + entry = &(copyList[randIndex]); + entry->nextBlock++; + txnID = entry->txnID; + /********** WORKING HERE **********/ + } - randIndex = rand_r(&randstate) % copyList.size(); - entry = &(copyList[randIndex]); - entry->nextBlock++; - txnID = entry->txnID; - /********** WORKING HERE **********/ - } - - default: - cerr << "not finished yet" << endl; - } + default: cerr << "not finished yet" << endl; } + } - return NULL; + return NULL; } - - static void* BRMRunner_1(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - BlockResolutionManager* brm; - ExtentMap em; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - brm = new BlockResolutionManager(); - - while (!threadStop) - { - op = rand_r(&randstate) % 10; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize, err; - - pthread_mutex_lock(&mutex); - OID = oid++; - opCount++; - pthread_mutex_unlock(&mutex); - - err = brm->createExtent(size, OID, lbids, allocdSize); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm->getExtentSize(); - - if ((size % brm->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = brm->getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * brm->getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize, err; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - err = brm->createExtent(size, oid, lbids, allocdSize); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm->getExtentSize(); - - if ((size % brm->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = brm->getExtentSize(); - else - newEm->size = size % brm->getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * brm->getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid, err; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - err = brm->deleteOID(oid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = brm->lookup(target, 0, false, oid, fbo); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = brm->lookup(oid, offset + tmp->FBO, lbid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); - em.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - err = brm->getHWM(tmp->OID, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid, err; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + brm->getExtentSize()); - err = brm->setHWM(oid, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em.checkConsistency(); - break; - } - - case 7: // renew this EM object - { - delete brm; - brm = new BlockResolutionManager(); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "got a new BRM instance" << endl; -#endif - em.checkConsistency(); - break; - } - - case 8: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = brm->getBulkInsertVars(lbid, hwm, txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 9: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = brm->setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - delete brm; - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + BlockResolutionManager* brm; + ExtentMap em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new BlockResolutionManager(); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + + pthread_mutex_lock(&mutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&mutex); + + err = brm->createExtent(size, OID, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = brm->getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * brm->getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm->createExtent(size, oid, lbids, allocdSize); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = brm->getExtentSize(); + else + newEm->size = size % brm->getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * brm->getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm->deleteOID(oid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm->lookup(target, 0, false, oid, fbo); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm->lookup(oid, offset + tmp->FBO, lbid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + err = brm->getHWM(tmp->OID, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm->getExtentSize()); + err = brm->setHWM(oid, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: // renew this EM object + { + delete brm; + brm = new BlockResolutionManager(); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "got a new BRM instance" << endl; +#endif + em.checkConsistency(); + break; + } + + case 8: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm->getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm->setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + delete brm; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } BlockResolutionManager brm_si; static void* BRMRunner_si(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - ExtentMap em; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - - while (!threadStop) - { - op = rand_r(&randstate) % 10; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize, err; - - pthread_mutex_lock(&mutex); - OID = oid++; - opCount++; - pthread_mutex_unlock(&mutex); - - err = brm_si.createExtent(size, OID, lbids, allocdSize); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm_si.getExtentSize(); - - if ((size % brm_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = brm_si.getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * brm_si.getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize, err; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - err = brm_si.createExtent(size, oid, lbids, allocdSize); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm_si.getExtentSize(); - - if ((size % brm_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = brm_si.getExtentSize(); - else - newEm->size = size % brm_si.getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * brm_si.getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid, err; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - err = brm_si.deleteOID(oid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = brm_si.lookup(target, 0, false, oid, fbo); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = brm_si.lookup(oid, offset + tmp->FBO, lbid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); - em.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - err = brm_si.getHWM(tmp->OID, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid, err; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize()); - err = brm_si.setHWM(oid, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em.checkConsistency(); - break; - } - - case 7: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = brm_si.getBulkInsertVars(lbid, hwm, txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 8: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + ExtentMap em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + + pthread_mutex_lock(&mutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&mutex); + + err = brm_si.createExtent(size, OID, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = brm_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * brm_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm_si.createExtent(size, oid, lbids, allocdSize); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = brm_si.getExtentSize(); + else + newEm->size = size % brm_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * brm_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm_si.deleteOID(oid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm_si.lookup(target, 0, false, oid, fbo); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm_si.lookup(oid, offset + tmp->FBO, lbid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + err = brm_si.getHWM(tmp->OID, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize()); + err = brm_si.setHWM(oid, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm_si.getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } static void* EMRunner(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - ExtentMap* em; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - em = new ExtentMap(); - - while (!threadStop) - { - op = rand_r(&randstate) % 10; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize; - - pthread_mutex_lock(&mutex); - OID = oid++; - pthread_mutex_unlock(&mutex); - - em->createExtent(size, OID, lbids, allocdSize); - em->confirmChanges(); - - entries = size / em->getExtentSize(); - - if ((size % em->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = em->getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * em->getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em->checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - em->createExtent(size, oid, lbids, allocdSize); - em->confirmChanges(); - - entries = size / em->getExtentSize(); - - if ((size % em->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = em->getExtentSize(); - else - newEm->size = size % em->getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * em->getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em->checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - em->deleteOID(oid); - em->confirmChanges(); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em->checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = em->lookup(target, oid, fbo); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em->checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = em->lookup(oid, offset + tmp->FBO, lbid); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); - em->checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - hwm = em->getHWM(tmp->OID); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em->checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + em->getExtentSize()); - em->setHWM(oid, hwm); - em->confirmChanges(); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em->checkConsistency(); - break; - } - - case 7: // renew this EM object - { - delete em; - em = new ExtentMap(); -#ifdef BRM_VERBOSE - cerr << "got a new EM instance" << endl; -#endif - em->checkConsistency(); - break; - } - - case 8: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = em->getBulkInsertVars(lbid, hwm, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 9: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = em->setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - em->confirmChanges(); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - delete em; - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + ExtentMap* em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + em = new ExtentMap(); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&mutex); + OID = oid++; + pthread_mutex_unlock(&mutex); + + em->createExtent(size, OID, lbids, allocdSize); + em->confirmChanges(); + + entries = size / em->getExtentSize(); + + if ((size % em->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = em->getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * em->getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em->checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + em->createExtent(size, oid, lbids, allocdSize); + em->confirmChanges(); + + entries = size / em->getExtentSize(); + + if ((size % em->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = em->getExtentSize(); + else + newEm->size = size % em->getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * em->getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em->checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em->deleteOID(oid); + em->confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em->checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = em->lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em->checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = em->lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + em->checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em->getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em->checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + em->getExtentSize()); + em->setHWM(oid, hwm); + em->confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em->checkConsistency(); + break; + } + + case 7: // renew this EM object + { + delete em; + em = new ExtentMap(); +#ifdef BRM_VERBOSE + cerr << "got a new EM instance" << endl; +#endif + em->checkConsistency(); + break; + } + + case 8: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em->getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em->setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + em->confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + delete em; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } ExtentMap em_si; static void* EMRunner_si(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - - while (!threadStop) - { - op = rand_r(&randstate) % 9; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize; - - pthread_mutex_lock(&mutex); - OID = oid++; - pthread_mutex_unlock(&mutex); - - em_si.createExtent(size, OID, lbids, allocdSize); - em_si.confirmChanges(); - - entries = size / em_si.getExtentSize(); - - if ((size % em_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = em_si.getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * em_si.getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - em_si.createExtent(size, oid, lbids, allocdSize); - em_si.confirmChanges(); - - entries = size / em_si.getExtentSize(); - - if ((size % em_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = em_si.getExtentSize(); - else - newEm->size = size % em_si.getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * em_si.getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - em_si.deleteOID(oid); - em_si.confirmChanges(); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = em_si.lookup(target, oid, fbo); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em_si.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = em_si.lookup(oid, offset + tmp->FBO, lbid); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); - em_si.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - hwm = em_si.getHWM(tmp->OID); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em_si.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + em_si.getExtentSize()); - em_si.setHWM(oid, hwm); - em_si.confirmChanges(); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 7: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = em_si.getBulkInsertVars(lbid, hwm, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 8: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = em_si.setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - em_si.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 9; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&mutex); + OID = oid++; + pthread_mutex_unlock(&mutex); + + em_si.createExtent(size, OID, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = em_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * em_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + em_si.createExtent(size, oid, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = em_si.getExtentSize(); + else + newEm->size = size % em_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * em_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em_si.deleteOID(oid); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = em_si.lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em_si.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = em_si.lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + em_si.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em_si.getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em_si.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + em_si.getExtentSize()); + em_si.setHWM(oid, hwm); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 7: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em_si.getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em_si.setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + em_si.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } #ifdef PARTITIONING ExtentMap emrange_si; static void* EMRangeRunner_si(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + int64_t lo_val; // low value for this partition + int64_t hi_val; // hi value for this partition + uint32_t extCnt; // number of extents in this partition + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - int64_t lo_val; // low value for this partition - int64_t hi_val; // hi value for this partition - uint32_t extCnt; // number of extents in this partition - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; + } + }; + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + const int numOps = 11; + int opRuns[numOps] = {0}; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + const int MaxPart = 16; + static int64_t pLowValue = 0; + + while (!threadStop) + { + op = rand_r(&randstate) % numOps; + opRuns[op]++; + +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int partSize = rand_r(&randstate) % MaxPart + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&mutex); + OID = oid++; + pthread_mutex_unlock(&mutex); + + EMRangePartition_t emr(pLowValue + 1, pLowValue + 100, true, true); + pthread_mutex_lock(&mutex); + pLowValue += 100; + pthread_mutex_unlock(&mutex); + + emrange_si.createRangeExtent(size, OID, lbids, allocdSize, emr); + emrange_si.confirmChanges(); + + entries = size / emrange_si.getExtentSize(); + + if ((size % emrange_si.getExtentSize()) != 0) + entries++; + + int partLen = entries; + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + int j = 0; + + for (i = 0; i < entries && j < partSize; i++) { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; + newEm = new EMEntries(); + newEm->size = emrange_si.getExtentSize(); + newEm->lo_val = emr.lo_val; + newEm->hi_val = emr.hi_val; + newEm->extCnt = partLen; + newEm->OID = OID; + newEm->FBO = i * emrange_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; } - }; #ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - const int numOps = 11; - int opRuns[numOps] = {0}; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; + cerr << "created new space for OID " << newEm->OID << endl; #endif - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - const int MaxPart = 16; - static int64_t pLowValue = 0; + emrange_si.checkConsistency(); + break; + } - while (!threadStop) - { - op = rand_r(&randstate) % numOps; - opRuns[op]++; + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int partSize = rand_r(&randstate) % MaxPart + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; - switch (op) + EMRangePartition_t emr(pLowValue + 1, pLowValue + 100, true, true); + pthread_mutex_lock(&mutex); + pLowValue += 100; + pthread_mutex_unlock(&mutex); + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int partSize = rand_r(&randstate) % MaxPart + 1; - int entries, OID, allocdSize; + if (tmp->OID != oid) + continue; - pthread_mutex_lock(&mutex); - OID = oid++; - pthread_mutex_unlock(&mutex); + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; - EMRangePartition_t emr(pLowValue + 1, pLowValue + 100, true, true); - pthread_mutex_lock(&mutex); - pLowValue += 100; - pthread_mutex_unlock(&mutex); - - emrange_si.createRangeExtent(size, OID, lbids, allocdSize, emr); - emrange_si.confirmChanges(); - - entries = size / emrange_si.getExtentSize(); - - if ((size % emrange_si.getExtentSize()) != 0) - entries++; - - int partLen = entries; - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - int j = 0; - - for (i = 0 ; i < entries && j < partSize; i++) - { - newEm = new EMEntries(); - newEm->size = emrange_si.getExtentSize(); - newEm->lo_val = emr.lo_val; - newEm->hi_val = emr.hi_val; - newEm->extCnt = partLen; - newEm->OID = OID; - newEm->FBO = i * emrange_si.getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - - emrange_si.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int partSize = rand_r(&randstate) % MaxPart + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize; - vector lbids; - - EMRangePartition_t emr(pLowValue + 1, pLowValue + 100, true, true); - pthread_mutex_lock(&mutex); - pLowValue += 100; - pthread_mutex_unlock(&mutex); - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - emrange_si.createRangeExtent(size, oid, lbids, allocdSize, emr); - emrange_si.confirmChanges(); - - entries = size / emrange_si.getExtentSize(); - - if ((size % emrange_si.getExtentSize()) != 0) - entries++; - - int partLen = entries; - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - int j = 0; - - for (i = 0; i < entries && j < partSize; i++) - { - newEm = new EMEntries(); - - if (i != entries) - newEm->size = emrange_si.getExtentSize(); - else - newEm->size = size % emrange_si.getExtentSize(); - - newEm->OID = oid; - newEm->lo_val = emr.lo_val; - newEm->hi_val = emr.hi_val; - newEm->extCnt = partLen; - newEm->FBO = lastExtent + (i * emrange_si.getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - - emrange_si.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - emrange_si.deleteOID(oid); - emrange_si.confirmChanges(); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - emrange_si.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = emrange_si.lookup(target, oid, fbo); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - emrange_si.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = emrange_si.lookup(oid, offset + tmp->FBO, lbid); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT((uint32_t)lbid == tmp->LBIDstart + offset); - emrange_si.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - hwm = emrange_si.getHWM(tmp->OID); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - emrange_si.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + emrange_si.getExtentSize()); - emrange_si.setHWM(oid, hwm); - emrange_si.confirmChanges(); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - emrange_si.checkConsistency(); - break; - } - - case 7: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = emrange_si.getBulkInsertVars(lbid, hwm, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT((uint32_t)txnID == tmp->txnID); - break; - } - - case 8: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = emrange_si.setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - emrange_si.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - break; - } - - case 9: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = emrange_si.setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - emrange_si.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - break; - } - - case 10: - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int target = rand_r(&randstate) % 0x0FFFFFFF; - int i, err, offset; - OID_t oid; - struct EMEntries* tmp; - LBIDRange_v lbids; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - oid = tmp->OID; - target = tmp->lo_val + 1; - err = emrange_si.lookup(oid, target, lbids); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - -#ifdef BRM_VERBOSE - cout << "Lookup OID/Range " << tmp->OID << " target " << target << - " start " << tmp->LBIDstart << - " len " << tmp->size << - " found " << lbids.size() << " ranges. " << - " extLen " << tmp->extCnt << - " partLo " << tmp->lo_val << - " partHi " << tmp->hi_val << endl; -#endif - - CPPUNIT_ASSERT((uint32_t)err == lbids.size() ); - CPPUNIT_ASSERT((uint32_t)tmp->extCnt == lbids.size() ); - CPPUNIT_ASSERT(tmp->lo_val < target && tmp->hi_val >= target); - - emrange_si.checkConsistency(); - lbids.clear(); - break; - } - - default: - break; + if (lastExtent < blockEnd) + lastExtent = blockEnd; } - } - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + emrange_si.createRangeExtent(size, oid, lbids, allocdSize, emr); + emrange_si.confirmChanges(); + + entries = size / emrange_si.getExtentSize(); + + if ((size % emrange_si.getExtentSize()) != 0) + entries++; + + int partLen = entries; + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + int j = 0; + + for (i = 0; i < entries && j < partSize; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = emrange_si.getExtentSize(); + else + newEm->size = size % emrange_si.getExtentSize(); + + newEm->OID = oid; + newEm->lo_val = emr.lo_val; + newEm->hi_val = emr.hi_val; + newEm->extCnt = partLen; + newEm->FBO = lastExtent + (i * emrange_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + + emrange_si.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + emrange_si.deleteOID(oid); + emrange_si.confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + emrange_si.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = emrange_si.lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + emrange_si.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = emrange_si.lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT((uint32_t)lbid == tmp->LBIDstart + offset); + emrange_si.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = emrange_si.getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + emrange_si.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + emrange_si.getExtentSize()); + emrange_si.setHWM(oid, hwm); + emrange_si.confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + emrange_si.checkConsistency(); + break; + } + + case 7: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = emrange_si.getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT((uint32_t)txnID == tmp->txnID); + break; + } + + case 8: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = emrange_si.setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + emrange_si.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + case 9: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = emrange_si.setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + emrange_si.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + case 10: + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int target = rand_r(&randstate) % 0x0FFFFFFF; + int i, err, offset; + OID_t oid; + struct EMEntries* tmp; + LBIDRange_v lbids; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + oid = tmp->OID; + target = tmp->lo_val + 1; + err = emrange_si.lookup(oid, target, lbids); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + +#ifdef BRM_VERBOSE + cout << "Lookup OID/Range " << tmp->OID << " target " << target << " start " << tmp->LBIDstart + << " len " << tmp->size << " found " << lbids.size() << " ranges. " + << " extLen " << tmp->extCnt << " partLo " << tmp->lo_val << " partHi " << tmp->hi_val << endl; +#endif + + CPPUNIT_ASSERT((uint32_t)err == lbids.size()); + CPPUNIT_ASSERT((uint32_t)tmp->extCnt == lbids.size()); + CPPUNIT_ASSERT(tmp->lo_val < target && tmp->hi_val >= target); + + emrange_si.checkConsistency(); + lbids.clear(); + break; + } + + default: break; } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } #ifdef BRM_VERBOSE - for (int i = 0; i < numOps; i++) - { - cout << "Op " << i << " was called " << opRuns[i] << " times." << endl; + for (int i = 0; i < numOps; i++) + { + cout << "Op " << i << " was called " << opRuns[i] << " times." << endl; + } - } - - cerr << "thread " << threadNum << " exiting" << endl; + cerr << "thread " << threadNum << " exiting" << endl; #endif - return NULL; + return NULL; } #endif class LongBRMTests : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(LongBRMTests); - CPPUNIT_TEST_SUITE(LongBRMTests); - - CPPUNIT_TEST(longEMTest_1); - CPPUNIT_TEST(longEMTest_2); + CPPUNIT_TEST(longEMTest_1); + CPPUNIT_TEST(longEMTest_2); #ifdef PARTITIONING - CPPUNIT_TEST(longEMRangeTest_1); + CPPUNIT_TEST(longEMRangeTest_1); #endif - CPPUNIT_TEST(longBRMTest_1); - CPPUNIT_TEST(longBRMTest_2); + CPPUNIT_TEST(longBRMTest_1); + CPPUNIT_TEST(longBRMTest_2); - CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST_SUITE_END(); -private: -public: + private: + public: + void longEMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; - void longEMTest_1() + cerr << endl << "Multithreaded, multiple instance ExtentMap test. " << endl; + + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + if (pthread_create(&threads[i], NULL, EMRunner, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); - cerr << endl << "Multithreaded, multiple instance ExtentMap test. " - << endl; - - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, EMRunner, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the ExtentMap test"); - - usleep(1000); - } - - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + usleep(1000); } - void longEMTest_2() + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } - cerr << endl << "Multithreaded, single instance ExtentMap test. " - << endl; + void longEMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; - threadStop = 0; - pthread_mutex_init(&mutex, NULL); + cerr << endl << "Multithreaded, single instance ExtentMap test. " << endl; - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, EMRunner_si, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the ExtentMap test"); + threadStop = 0; + pthread_mutex_init(&mutex, NULL); - usleep(1000); - } + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, EMRunner_si, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + usleep(1000); } - void longBRMTest_1() + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } - cerr << endl << "Multithreaded, multiple instance BlockResolutionManager test. " - << endl; + void longBRMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - opCount = 0; + cerr << endl << "Multithreaded, multiple instance BlockResolutionManager test. " << endl; - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, BRMRunner_1, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the BlockResolutionManager test"); + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + opCount = 0; - usleep(1000); - } + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_1, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the BlockResolutionManager test"); - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } - - cerr << "opCount = " << opCount << endl; + usleep(1000); } - void longBRMTest_2() + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; - - cerr << endl << "Multithreaded, single instance BlockResolutionManager test. " - << endl; - - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - opCount = 0; - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, BRMRunner_si, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the BlockResolutionManager test"); - - usleep(1000); - } - - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } - - cerr << "opCount = " << opCount << endl; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); } + cerr << "opCount = " << opCount << endl; + } + + void longBRMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, single instance BlockResolutionManager test. " << endl; + + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + opCount = 0; + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_si, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the BlockResolutionManager test"); + + usleep(1000); + } + + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + + cerr << "opCount = " << opCount << endl; + } + #ifdef PARTITIONING - void longEMRangeTest_1() + void longEMRangeTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, single instance ExtentMap(Range) test." << endl; + + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + if (pthread_create(&threads[i], NULL, EMRangeRunner_si, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); - cerr << endl << "Multithreaded, single instance ExtentMap(Range) test." << endl; - - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, EMRangeRunner_si, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the ExtentMap test"); - - usleep(100); - } - - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + usleep(100); } -#endif + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } +#endif }; -CPPUNIT_TEST_SUITE_REGISTRATION( LongBRMTests ); +CPPUNIT_TEST_SUITE_REGISTRATION(LongBRMTests); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - idbdatafile::IDBPolicy::configIDBPolicy(); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + idbdatafile::IDBPolicy::configIDBPolicy(); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - - diff --git a/versioning/BRM/tdriver-dbrm.cpp b/versioning/BRM/tdriver-dbrm.cpp index a6c893bf4..22097a4b2 100644 --- a/versioning/BRM/tdriver-dbrm.cpp +++ b/versioning/BRM/tdriver-dbrm.cpp @@ -47,1126 +47,1106 @@ using namespace execplan; void keepalive(int signum) { - cerr << "Yes, it's still going..." << endl; - alarm(290); + cerr << "Yes, it's still going..." << endl; + alarm(290); } void* DBRM_dummy_1(void* arg) { - DBRM dbrm; - int err; - VER_t txn = 2; - vector ranges; + DBRM dbrm; + int err; + VER_t txn = 2; + vector ranges; + LBIDRange range; + vector freeList; + + range.start = 500; + range.size = 1000; + ranges.push_back(range); + err = dbrm.beginVBCopy(txn, ranges, freeList); + CPPUNIT_ASSERT(err == 0); + err = dbrm.endVBCopy(txn, ranges); + CPPUNIT_ASSERT(err == 0); + err = dbrm.vbCommit(txn); + CPPUNIT_ASSERT(err == 0); + + return NULL; +} + +void* DBRM_deadlock(void* arg) +{ + DBRM dbrm; + int terr; + VER_t txn = 2; + vector ranges; + LBIDRange range; + vector freeList; + + range.start = 2000; + range.size = 1000; + ranges.push_back(range); + terr = dbrm.beginVBCopy(txn, ranges, freeList); + + range.start = 1000; + range.size = 1000; + ranges.clear(); + freeList.clear(); + ranges.push_back(range); + terr = dbrm.beginVBCopy(txn, ranges, freeList); + + // block waiting on the main thread, main thread deadlocks, rolls back, + // thread wakes + + CPPUNIT_ASSERT(terr == 0); + terr = dbrm.endVBCopy(txn, ranges); + + CPPUNIT_ASSERT(terr == 0); + terr = dbrm.vbCommit(txn); + + CPPUNIT_ASSERT(terr == 0); + + return NULL; +} + +class BRMTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(BRMTest); + + CPPUNIT_TEST(sessionmanager_1); + CPPUNIT_TEST(sessionmanager_2); + CPPUNIT_TEST(sessionmanager_3); + CPPUNIT_TEST(sessionmanager_4); + CPPUNIT_TEST(sessionmanager_5); + CPPUNIT_TEST(sessionmanager_6); + CPPUNIT_TEST(sessionmanager_7); + CPPUNIT_TEST(sessionmanager_8); + CPPUNIT_TEST(brm_dumb_1); + CPPUNIT_TEST(brm_extentmap_good_1); + CPPUNIT_TEST(brm_good_2); + CPPUNIT_TEST(brm_good_3); + CPPUNIT_TEST(brm_deleteOID); + CPPUNIT_TEST(brm_HWM); + CPPUNIT_TEST(dbrm_clear); + + CPPUNIT_TEST(DBRM_resource_graph_1); + CPPUNIT_TEST(DBRM_resource_graph_deadlock); + CPPUNIT_TEST(brm_markExtentsInvalid); + + CPPUNIT_TEST_SUITE_END(); + + private: + public: + void brm_dumb_1() + { + DBRM brm; + vector lbids; + int allocdSize, err; + const uint32_t extentSize = brm.getExtentSize(); + + err = brm.createExtent(extentSize, 1, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + err = brm.markExtentInvalid(lbids[0]); + CPPUNIT_ASSERT(err == 0); + int64_t min; + int64_t max; + int32_t seq; + err = brm.getExtentMaxMin(lbids[0], max, min, seq); +#ifdef SAFE_CP + CPPUNIT_ASSERT(err == 0); +#else + CPPUNIT_ASSERT(err == 1); +#endif + + err = brm.setExtentMaxMin(lbids[0], max, min, seq); + CPPUNIT_ASSERT(err == 0); + + err = brm.deleteOID(1); + CPPUNIT_ASSERT(err == 0); + } + + void brm_extentmap_good_1() + { + DBRM brm; + int i, err, oid, allocdSize, + iterations = 100; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + uint32_t fbo, hwm; + vector lbids; + HWM_t hwm2; + VER_t txnID; + const uint32_t extentSize = brm.getExtentSize(); + + cerr << "brm_extentmap_good_1" << endl; + + for (i = 1; i < iterations; i++) + { + err = brm.createExtent(extentSize, i, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + + err = brm.markExtentInvalid(lbids[0]); + CPPUNIT_ASSERT(err == 0); + + int64_t min; + int64_t max; + int32_t seq; + err = brm.getExtentMaxMin(lbids[0], max, min, seq); +#ifdef SAFE_CP + CPPUNIT_ASSERT(err == 0); +#else + CPPUNIT_ASSERT(err == 1); +#endif + + err = brm.setExtentMaxMin(lbids[0], max, min, seq); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(err == 0); + + for (i = 1; i < iterations; i++) + { + err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 0); + + if (i != 1) + { + err = brm.lookup(static_cast((i - 1) * extentSize - 1), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i - 1); + CPPUNIT_ASSERT(fbo == extentSize - 1); + } + + if (i != iterations) + { + err = brm.lookup(static_cast((i - 1) * extentSize + 1), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 1); + } + + err = brm.markExtentInvalid(oid); + CPPUNIT_ASSERT(err == 0); + err = brm.markExtentInvalid(lbids[0]); + CPPUNIT_ASSERT(err == 0); + int64_t min; + int64_t max; + int32_t seq; + err = brm.getExtentMaxMin(lbids[0], max, min, seq); +#ifdef SAFE_CP + CPPUNIT_ASSERT(err == 0); +#else + CPPUNIT_ASSERT(err == 1); +#endif + + err = brm.setExtentMaxMin(lbids[0], max, min, seq); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + for (i = 1; i < iterations; i++) + { + err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == 0); + CPPUNIT_ASSERT(txnID == 0); + err = brm.setBulkInsertVars(static_cast((i - 1) * extentSize), i, i + 1); + CPPUNIT_ASSERT(err == 0); + err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == static_cast(i)); + CPPUNIT_ASSERT(txnID == static_cast(i + 1)); + + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == 0); + err = brm.setHWM(i, ((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); + CPPUNIT_ASSERT(err == 0); + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == static_cast((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + +#ifdef BRM_DEBUG + err = brm.setHWM(i, hwm); + CPPUNIT_ASSERT(err != 0); +#endif + + for (i = 1; i < iterations; i++) + { + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == 0); + } + + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err != 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.saveState(); + CPPUNIT_ASSERT(err == 0); + } + + void brm_good_2() + { + DBRM brm; + VBBM vbbm; + VSS vss; + CopyLocks cl; + int i, err, size; + vector lbids; + vector extents; + LBIDRange_v ranges; + LBIDRange_v::iterator lbidRangeIT; + vector vbRanges, vbRanges2; + vector::iterator vbRangesIT; + EMEntry em; + OID_t oid; + uint32_t fbo; LBIDRange range; + VBRange vbRange; + VER_t verID; + bool vbFlag; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + cerr << "brm_good_2" << endl; + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + err = brm.lookup(0, 0, false, oid, fbo); + CPPUNIT_ASSERT(err != 0); + err = brm.lookup(0, 0, true, oid, fbo); + CPPUNIT_ASSERT(err != 0); + + err = brm.createExtent(8000, 1, lbids, size); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(size == brm.getExtentSize()); + CPPUNIT_ASSERT(lbids.size() == 1); + // CPPUNIT_ASSERT(*(lbids.begin()) == 0); + + err = brm.getExtents(1, extents); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.size() == 1); + + em = *(extents.begin()); + // CPPUNIT_ASSERT(em.range.start == 0); + CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); + CPPUNIT_ASSERT(em.HWM == 0); + CPPUNIT_ASSERT(em.blockOffset == 0); + + for (i = 0; i < 5; i++) + { + range.start = i * 100; + range.size = 100; + ranges.push_back(range); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.beginVBCopy(1, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.beginVBCopy(1, ranges, vbRanges2); + CPPUNIT_ASSERT(err != 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + for (i = 0; i < 500; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err != 0); + } + + vbRange = *(vbRanges.begin()); + + // CPPUNIT_ASSERT(vbRange.vbFBO == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + } + + for (; i < 500; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err != 0); + } + + err = brm.endVBCopy(0, ranges); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + brm.vbCommit(1); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + + verID = 0; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + + err = brm.lookup(i, verID, vbFlag, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == vbRange.vbOID); + CPPUNIT_ASSERT(fbo == static_cast(i + vbRange.vbFBO)); + + vbbm.lock(VBBM::WRITE); + vss.lock(VSS::WRITE); + vbbm.removeEntry(i, verID); + // vss.removeEntry(i, 1); + vss.removeEntry(i, verID); + vbbm.confirmChanges(); + vss.confirmChanges(); + vss.release(VSS::WRITE); + vbbm.release(VBBM::WRITE); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + brm.deleteOID(1); + + vss.lock(VSS::READ); + vbbm.lock(VBBM::READ); + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.saveState(); + CPPUNIT_ASSERT(err == 0); + } + + // cut & pasted from brm_good_2(), but with rollback instead of commit. + void brm_good_3() + { + DBRM brm; + VBBM vbbm; + VSS vss; + CopyLocks cl; + int i, err, size; + vector lbids; + vector::iterator lbid; + vector extents; + LBIDRange_v ranges; + LBIDRange_v::iterator lbidRangeIT; + VBRange_v vbRanges, vbRanges2; + VBRange_v::iterator vbRangesIT; + LBIDRange_v tmp; + EMEntry em; + OID_t oid; + uint32_t fbo; + LBIDRange range; + VBRange vbRange; + VER_t verID; + bool vbFlag; + bool caughtException; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + cerr << "brm_good_3" << endl; + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + err = brm.lookup(0, 0, false, oid, fbo); + CPPUNIT_ASSERT(err != 0); + err = brm.lookup(0, 0, true, oid, fbo); + CPPUNIT_ASSERT(err != 0); + + err = brm.createExtent(8000, 1, lbids, size); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(size == brm.getExtentSize()); + CPPUNIT_ASSERT(lbids.size() == 1); + CPPUNIT_ASSERT(*(lbids.begin()) == 0); + + err = brm.getExtents(1, extents); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.size() == 1); + + em = *(extents.begin()); + CPPUNIT_ASSERT(em.range.start == 0); + CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); + CPPUNIT_ASSERT(em.HWM == 0); + CPPUNIT_ASSERT(em.blockOffset == 0); + + for (i = 0; i < 5; i++) + { + range.start = i * 100; + range.size = 100; + ranges.push_back(range); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.beginVBCopy(1, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.beginVBCopy(1, ranges, vbRanges2); + CPPUNIT_ASSERT(err != 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + for (i = 0; i < 500; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err != 0); + } + + vbRange = *(vbRanges.begin()); + + // CPPUNIT_ASSERT(vbRange.vbFBO == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + } + + for (; i < 500; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err != 0); + } + + err = brm.endVBCopy(0, ranges); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.getUncommittedLBIDs(1, lbids); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbids.size() == vbRange.size); + sort::iterator>(lbids.begin(), lbids.end()); + lbid = lbids.begin(); + + for (i = 0; i < static_cast(lbids.size()); i++, lbid++) + CPPUNIT_ASSERT((*lbid) == static_cast(i)); + + range.start = 0; + range.size = i; + tmp.push_back(range); + err = brm.vbRollback(1, tmp); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == false); + + err = brm.lookup(i, verID, vbFlag, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == 1); + CPPUNIT_ASSERT(fbo == static_cast(i)); + + vbbm.lock(VBBM::WRITE); + vss.lock(VSS::WRITE); + +#ifdef BRM_DEBUG + caughtException = false; + + try + { + vbbm.removeEntry(i, verID); + vbbm.confirmChanges(); + } + catch (logic_error& e) + { + vbbm.undoChanges(); + caughtException = true; + } + + CPPUNIT_ASSERT(caughtException); + caughtException = false; + + try + { + vss.removeEntry(i, 1); + vss.confirmChanges(); + } + catch (logic_error& e) + { + vss.undoChanges(); + caughtException = true; + } + + CPPUNIT_ASSERT(caughtException); +#endif + + vss.removeEntry(i, verID); + vss.confirmChanges(); + vss.release(VSS::WRITE); + vbbm.release(VBBM::WRITE); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + brm.deleteOID(1); + + vbbm.lock(VBBM::READ); + vss.lock(VSS::READ); + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.saveState(); + CPPUNIT_ASSERT(err == 0); + } + + /* This test verifies that deleteOID returns an error for + non-existant OIDs (bug #105) */ + void brm_deleteOID() + { + DBRM brm; + int err; + vector extents; + + cerr << "brm_deleteOID" << endl; + + err = brm.getExtents(1, extents); +#ifdef BRM_DEBUG + + if (err == 0) + cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; + +#endif + CPPUNIT_ASSERT(err != 0); + CPPUNIT_ASSERT(extents.empty()); + err = brm.deleteOID(1); + CPPUNIT_ASSERT(err != 0); + } + + /* This test verifies that setHWM and getHWM return an error for + nonexistant OIDs (bugs #106, 107) */ + + void brm_HWM() + { + DBRM brm; + int err; + HWM_t hwm; + vector extents; + + cerr << "brm_HWM" << endl; + + err = brm.getExtents(1, extents); +#ifdef BRM_DEBUG + + if (err == 0) + cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; + +#endif + CPPUNIT_ASSERT(extents.size() == 0); + err = brm.setHWM(1, 10); + CPPUNIT_ASSERT(err != 0); + err = brm.getHWM(1, hwm); + CPPUNIT_ASSERT(err != 0); + } + + void DBRM_resource_graph_1() + { + DBRM dbrm; + pthread_t t; + VER_t txn = 1; + int err, allocSize; + LBIDRange range; + vector lbids; + vector ranges; vector freeList; - range.start = 500; + err = dbrm.createExtent(8000, 1, lbids, allocSize); + CPPUNIT_ASSERT(err == 0); + + range.start = 1000; range.size = 1000; ranges.push_back(range); err = dbrm.beginVBCopy(txn, ranges, freeList); CPPUNIT_ASSERT(err == 0); + + pthread_create(&t, NULL, DBRM_dummy_1, NULL); + sleep(1); + + // thread tries to grab 500-1500, blocks err = dbrm.endVBCopy(txn, ranges); CPPUNIT_ASSERT(err == 0); err = dbrm.vbCommit(txn); CPPUNIT_ASSERT(err == 0); - return NULL; -} + dbrm.deleteOID(1); -void* DBRM_deadlock(void* arg) -{ + // thread finishes + pthread_join(t, NULL); + } + + void DBRM_resource_graph_deadlock() + { DBRM dbrm; - int terr; - VER_t txn = 2; - vector ranges; + pthread_t t; + VER_t txn = 1; + int err, i; LBIDRange range; + vector lbids; + vector ranges; vector freeList; - range.start = 2000; - range.size = 1000; - ranges.push_back(range); - terr = dbrm.beginVBCopy(txn, ranges, freeList); - range.start = 1000; range.size = 1000; - ranges.clear(); - freeList.clear(); ranges.push_back(range); - terr = dbrm.beginVBCopy(txn, ranges, freeList); + err = dbrm.beginVBCopy(txn, ranges, freeList); + CPPUNIT_ASSERT(err == 0); - // block waiting on the main thread, main thread deadlocks, rolls back, - // thread wakes - - CPPUNIT_ASSERT(terr == 0); - terr = dbrm.endVBCopy(txn, ranges); - - CPPUNIT_ASSERT(terr == 0); - terr = dbrm.vbCommit(txn); - - CPPUNIT_ASSERT(terr == 0); - - return NULL; -} - -class BRMTest : public CppUnit::TestFixture -{ - - CPPUNIT_TEST_SUITE(BRMTest); - - - CPPUNIT_TEST(sessionmanager_1); - CPPUNIT_TEST(sessionmanager_2); - CPPUNIT_TEST(sessionmanager_3); - CPPUNIT_TEST(sessionmanager_4); - CPPUNIT_TEST(sessionmanager_5); - CPPUNIT_TEST(sessionmanager_6); - CPPUNIT_TEST(sessionmanager_7); - CPPUNIT_TEST(sessionmanager_8); - CPPUNIT_TEST(brm_dumb_1); - CPPUNIT_TEST(brm_extentmap_good_1); - CPPUNIT_TEST(brm_good_2); - CPPUNIT_TEST(brm_good_3); - CPPUNIT_TEST(brm_deleteOID); - CPPUNIT_TEST(brm_HWM); - CPPUNIT_TEST(dbrm_clear); - - CPPUNIT_TEST(DBRM_resource_graph_1); - CPPUNIT_TEST(DBRM_resource_graph_deadlock); - CPPUNIT_TEST(brm_markExtentsInvalid); - - CPPUNIT_TEST_SUITE_END(); - -private: -public: - - void brm_dumb_1() + for (i = range.start; i < range.start + range.size; i++) { - DBRM brm; - vector lbids; - int allocdSize, err; - const uint32_t extentSize = brm.getExtentSize(); - - err = brm.createExtent(extentSize, 1, lbids, allocdSize); - CPPUNIT_ASSERT(err == 0); - err = brm.markExtentInvalid(lbids[0]); - CPPUNIT_ASSERT(err == 0); - int64_t min; - int64_t max; - int32_t seq; - err = brm.getExtentMaxMin(lbids[0], max, min, seq); -#ifdef SAFE_CP - CPPUNIT_ASSERT(err == 0); -#else - CPPUNIT_ASSERT(err == 1); -#endif - - err = brm.setExtentMaxMin(lbids[0], max, min, seq); - CPPUNIT_ASSERT(err == 0); - - err = brm.deleteOID(1); - CPPUNIT_ASSERT(err == 0); + err = dbrm.writeVBEntry(txn, i, 1, i); + CPPUNIT_ASSERT(err == 0); } - void brm_extentmap_good_1() + err = dbrm.endVBCopy(txn, ranges); + CPPUNIT_ASSERT(err == 0); + + pthread_create(&t, NULL, DBRM_deadlock, NULL); + + // thread grabs 2000-2999 and 1000-1999 as 2 seperate ranges + sleep(1); + + range.start = 2000; + range.size = 1000; + ranges.clear(); + ranges.push_back(range); + err = dbrm.beginVBCopy(txn, ranges, freeList); + CPPUNIT_ASSERT(err == ERR_DEADLOCK); + + // roll back the blocks we "wrote" + range.start = 1000; + range.size = 1000; + ranges.clear(); + ranges.push_back(range); + err = dbrm.vbRollback(txn, ranges); + CPPUNIT_ASSERT(err == ERR_OK); + + // thread finishes + + txn = 3; + err = dbrm.beginVBCopy(txn, ranges, freeList); + CPPUNIT_ASSERT(err == 0); + + for (i = range.start; i < range.start + range.size; i++) { - - DBRM brm; - int i, err, oid, allocdSize, - iterations = 100; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - uint32_t fbo, hwm; - vector lbids; - HWM_t hwm2; - VER_t txnID; - const uint32_t extentSize = brm.getExtentSize(); - - cerr << "brm_extentmap_good_1" << endl; - - for (i = 1; i < iterations; i++) - { - err = brm.createExtent(extentSize, i, lbids, allocdSize); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - - err = brm.markExtentInvalid(lbids[0]); - CPPUNIT_ASSERT(err == 0); - - int64_t min; - int64_t max; - int32_t seq; - err = brm.getExtentMaxMin(lbids[0], max, min, seq); -#ifdef SAFE_CP - CPPUNIT_ASSERT(err == 0); -#else - CPPUNIT_ASSERT(err == 1); -#endif - - err = brm.setExtentMaxMin(lbids[0], max, min, seq); - CPPUNIT_ASSERT(err == 0); - - } - - CPPUNIT_ASSERT(err == 0); - - for (i = 1; i < iterations; i++) - { - err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 0); - - if (i != 1) - { - err = brm.lookup(static_cast((i - 1) * extentSize - 1), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i - 1); - CPPUNIT_ASSERT(fbo == extentSize - 1); - } - - if (i != iterations) - { - err = brm.lookup(static_cast((i - 1) * extentSize + 1), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 1); - } - - err = brm.markExtentInvalid(oid); - CPPUNIT_ASSERT(err == 0); - err = brm.markExtentInvalid(lbids[0]); - CPPUNIT_ASSERT(err == 0); - int64_t min; - int64_t max; - int32_t seq; - err = brm.getExtentMaxMin(lbids[0], max, min, seq); -#ifdef SAFE_CP - CPPUNIT_ASSERT(err == 0); -#else - CPPUNIT_ASSERT(err == 1); -#endif - - err = brm.setExtentMaxMin(lbids[0], max, min, seq); - CPPUNIT_ASSERT(err == 0); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - - for (i = 1; i < iterations; i++) - { - err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == 0); - CPPUNIT_ASSERT(txnID == 0); - err = brm.setBulkInsertVars(static_cast((i - 1) * extentSize), - i, i + 1); - CPPUNIT_ASSERT(err == 0); - err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == static_cast(i)); - CPPUNIT_ASSERT(txnID == static_cast(i + 1)); - - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == 0); - err = brm.setHWM(i, ((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); - CPPUNIT_ASSERT(err == 0); - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == static_cast((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - -#ifdef BRM_DEBUG - err = brm.setHWM(i, hwm); - CPPUNIT_ASSERT(err != 0); -#endif - - for (i = 1; i < iterations; i++) - { - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err != 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.saveState(); - CPPUNIT_ASSERT(err == 0); - + err = dbrm.writeVBEntry(txn, i, 1, i); + CPPUNIT_ASSERT(err == 0); } - void brm_good_2() + err = dbrm.endVBCopy(txn, ranges); + CPPUNIT_ASSERT(err == 0); + + err = dbrm.vbRollback(txn, ranges); + CPPUNIT_ASSERT(err == 0); + + pthread_join(t, NULL); + } + + void dbrm_clear() + { + DBRM dbrm; + VSS vss; + VBBM vbbm; + int err, vssShmid, vbbmShmid, txnID = 1, i; + struct shmid_ds vssShminfo[3], vbbmShminfo[3]; + LBIDRange_v ranges; + LBIDRange range; + VBRange_v freelist; + + err = dbrm.clear(); + CPPUNIT_ASSERT(err == ERR_OK); + + // grab the size of vss and vbbm shmsegs somehow + vss.lock(VSS::READ); + vbbm.lock(VBBM::READ); + vssShmid = vss.getShmid(); + vbbmShmid = vbbm.getShmid(); + err = shmctl(vssShmid, IPC_STAT, &vssShminfo[0]); + CPPUNIT_ASSERT(err == 0); + err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[0]); + CPPUNIT_ASSERT(err == 0); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + // do begin, write, end vbcopy for 150k blocks + cerr << endl << "Adding 150k block entries. "; + range.start = 1; + range.size = 150000; + ranges.push_back(range); + err = dbrm.beginVBCopy(txnID, ranges, freelist); + CPPUNIT_ASSERT(err == 0); + + for (i = range.start; (uint32_t)i < range.size; i++) { - DBRM brm; - VBBM vbbm; - VSS vss; - CopyLocks cl; - int i, err, size; - vector lbids; - vector extents; - LBIDRange_v ranges; - LBIDRange_v::iterator lbidRangeIT; - vector vbRanges, vbRanges2; - vector::iterator vbRangesIT; - EMEntry em; - OID_t oid; - uint32_t fbo; - LBIDRange range; - VBRange vbRange; - VER_t verID; - bool vbFlag; + if (i % 50000 == 0) + cerr << " ... " << i; - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - - cerr << "brm_good_2" << endl; - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - - err = brm.lookup(0, 0, false, oid, fbo); - CPPUNIT_ASSERT(err != 0); - err = brm.lookup(0, 0, true, oid, fbo); - CPPUNIT_ASSERT(err != 0); - - err = brm.createExtent(8000, 1, lbids, size); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(size == brm.getExtentSize()); - CPPUNIT_ASSERT(lbids.size() == 1); -// CPPUNIT_ASSERT(*(lbids.begin()) == 0); - - err = brm.getExtents(1, extents); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.size() == 1); - - em = *(extents.begin()); -// CPPUNIT_ASSERT(em.range.start == 0); - CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); - CPPUNIT_ASSERT(em.HWM == 0); - CPPUNIT_ASSERT(em.blockOffset == 0); - - for (i = 0; i < 5; i++) - { - range.start = i * 100; - range.size = 100; - ranges.push_back(range); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.beginVBCopy(1, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.beginVBCopy(1, ranges, vbRanges2); - CPPUNIT_ASSERT(err != 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - for (i = 0; i < 500; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err != 0); - } - - vbRange = *(vbRanges.begin()); - -// CPPUNIT_ASSERT(vbRange.vbFBO == 0); - for (i = 0; i < (int)vbRange.size; i++) - { - err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); - CPPUNIT_ASSERT(err == 0); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - } - - for (; i < 500; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err != 0); - } - - err = brm.endVBCopy(0, ranges); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - brm.vbCommit(1); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - - verID = 0; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - - err = brm.lookup(i, verID, vbFlag, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == vbRange.vbOID); - CPPUNIT_ASSERT(fbo == static_cast(i + vbRange.vbFBO)); - - vbbm.lock(VBBM::WRITE); - vss.lock(VSS::WRITE); - vbbm.removeEntry(i, verID); -// vss.removeEntry(i, 1); - vss.removeEntry(i, verID); - vbbm.confirmChanges(); - vss.confirmChanges(); - vss.release(VSS::WRITE); - vbbm.release(VBBM::WRITE); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - brm.deleteOID(1); - - vss.lock(VSS::READ); - vbbm.lock(VBBM::READ); - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.saveState(); - CPPUNIT_ASSERT(err == 0); + err = dbrm.writeVBEntry(txnID, i, 1, i); + CPPUNIT_ASSERT(err == 0); } - // cut & pasted from brm_good_2(), but with rollback instead of commit. - void brm_good_3() - { - DBRM brm; - VBBM vbbm; - VSS vss; - CopyLocks cl; - int i, err, size; - vector lbids; - vector::iterator lbid; - vector extents; - LBIDRange_v ranges; - LBIDRange_v::iterator lbidRangeIT; - VBRange_v vbRanges, vbRanges2; - VBRange_v::iterator vbRangesIT; - LBIDRange_v tmp; - EMEntry em; - OID_t oid; - uint32_t fbo; - LBIDRange range; - VBRange vbRange; - VER_t verID; - bool vbFlag; - bool caughtException; - - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - - cerr << "brm_good_3" << endl; - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - - err = brm.lookup(0, 0, false, oid, fbo); - CPPUNIT_ASSERT(err != 0); - err = brm.lookup(0, 0, true, oid, fbo); - CPPUNIT_ASSERT(err != 0); - - err = brm.createExtent(8000, 1, lbids, size); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(size == brm.getExtentSize()); - CPPUNIT_ASSERT(lbids.size() == 1); - CPPUNIT_ASSERT(*(lbids.begin()) == 0); - - err = brm.getExtents(1, extents); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.size() == 1); - - em = *(extents.begin()); - CPPUNIT_ASSERT(em.range.start == 0); - CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); - CPPUNIT_ASSERT(em.HWM == 0); - CPPUNIT_ASSERT(em.blockOffset == 0); - - for (i = 0; i < 5; i++) - { - range.start = i * 100; - range.size = 100; - ranges.push_back(range); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.beginVBCopy(1, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.beginVBCopy(1, ranges, vbRanges2); - CPPUNIT_ASSERT(err != 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - for (i = 0; i < 500; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err != 0); - } - - vbRange = *(vbRanges.begin()); - -// CPPUNIT_ASSERT(vbRange.vbFBO == 0); - for (i = 0; i < (int)vbRange.size; i++) - { - err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); - CPPUNIT_ASSERT(err == 0); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - } - - for (; i < 500; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err != 0); - } - - err = brm.endVBCopy(0, ranges); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.getUncommittedLBIDs(1, lbids); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbids.size() == vbRange.size); - sort::iterator>(lbids.begin(), lbids.end()); - lbid = lbids.begin(); - - for (i = 0; i < static_cast(lbids.size()); i++, lbid++) - CPPUNIT_ASSERT((*lbid) == static_cast(i)); - - range.start = 0; - range.size = i; - tmp.push_back(range); - err = brm.vbRollback(1, tmp); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == false); - - err = brm.lookup(i, verID, vbFlag, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == 1); - CPPUNIT_ASSERT(fbo == static_cast(i)); - - vbbm.lock(VBBM::WRITE); - vss.lock(VSS::WRITE); - -#ifdef BRM_DEBUG - caughtException = false; - - try - { - vbbm.removeEntry(i, verID); - vbbm.confirmChanges(); - } - catch (logic_error& e) - { - vbbm.undoChanges(); - caughtException = true; - } - - CPPUNIT_ASSERT(caughtException); - caughtException = false; - - try - { - vss.removeEntry(i, 1); - vss.confirmChanges(); - } - catch (logic_error& e) - { - vss.undoChanges(); - caughtException = true; - } - - CPPUNIT_ASSERT(caughtException); -#endif - - vss.removeEntry(i, verID); - vss.confirmChanges(); - vss.release(VSS::WRITE); - vbbm.release(VBBM::WRITE); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - brm.deleteOID(1); - - vbbm.lock(VBBM::READ); - vss.lock(VSS::READ); - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.saveState(); - CPPUNIT_ASSERT(err == 0); - } - - /* This test verifies that deleteOID returns an error for - non-existant OIDs (bug #105) */ - void brm_deleteOID() - { - DBRM brm; - int err; - vector extents; - - cerr << "brm_deleteOID" << endl; - - err = brm.getExtents(1, extents); -#ifdef BRM_DEBUG - - if (err == 0) - cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; - -#endif - CPPUNIT_ASSERT(err != 0); - CPPUNIT_ASSERT(extents.empty()); - err = brm.deleteOID(1); - CPPUNIT_ASSERT(err != 0); - } - - /* This test verifies that setHWM and getHWM return an error for - nonexistant OIDs (bugs #106, 107) */ - - void brm_HWM() - { - DBRM brm; - int err; - HWM_t hwm; - vector extents; - - cerr << "brm_HWM" << endl; - - err = brm.getExtents(1, extents); -#ifdef BRM_DEBUG - - if (err == 0) - cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; - -#endif - CPPUNIT_ASSERT(extents.size() == 0); - err = brm.setHWM(1, 10); - CPPUNIT_ASSERT(err != 0); - err = brm.getHWM(1, hwm); - CPPUNIT_ASSERT(err != 0); - } - - void DBRM_resource_graph_1() - { - DBRM dbrm; - pthread_t t; - VER_t txn = 1; - int err, allocSize; - LBIDRange range; - vector lbids; - vector ranges; - vector freeList; - - err = dbrm.createExtent(8000, 1, lbids, allocSize); - CPPUNIT_ASSERT(err == 0); - - range.start = 1000; - range.size = 1000; - ranges.push_back(range); - err = dbrm.beginVBCopy(txn, ranges, freeList); - CPPUNIT_ASSERT(err == 0); - - pthread_create(&t, NULL, DBRM_dummy_1, NULL); - sleep(1); - - // thread tries to grab 500-1500, blocks - err = dbrm.endVBCopy(txn, ranges); - CPPUNIT_ASSERT(err == 0); - err = dbrm.vbCommit(txn); - CPPUNIT_ASSERT(err == 0); - - dbrm.deleteOID(1); - - //thread finishes - pthread_join(t, NULL); - } - - void DBRM_resource_graph_deadlock() - { - DBRM dbrm; - pthread_t t; - VER_t txn = 1; - int err, i; - LBIDRange range; - vector lbids; - vector ranges; - vector freeList; - - range.start = 1000; - range.size = 1000; - ranges.push_back(range); - err = dbrm.beginVBCopy(txn, ranges, freeList); - CPPUNIT_ASSERT(err == 0); - - for (i = range.start; i < range.start + range.size; i++) - { - err = dbrm.writeVBEntry(txn, i, 1, i); - CPPUNIT_ASSERT(err == 0); - } - - err = dbrm.endVBCopy(txn, ranges); - CPPUNIT_ASSERT(err == 0); - - pthread_create(&t, NULL, DBRM_deadlock, NULL); - - // thread grabs 2000-2999 and 1000-1999 as 2 seperate ranges - sleep(1); - - range.start = 2000; - range.size = 1000; - ranges.clear(); - ranges.push_back(range); - err = dbrm.beginVBCopy(txn, ranges, freeList); - CPPUNIT_ASSERT(err == ERR_DEADLOCK); - - // roll back the blocks we "wrote" - range.start = 1000; - range.size = 1000; - ranges.clear(); - ranges.push_back(range); - err = dbrm.vbRollback(txn, ranges); - CPPUNIT_ASSERT(err == ERR_OK); - - // thread finishes - - txn = 3; - err = dbrm.beginVBCopy(txn, ranges, freeList); - CPPUNIT_ASSERT(err == 0); - - for (i = range.start; i < range.start + range.size; i++) - { - err = dbrm.writeVBEntry(txn, i, 1, i); - CPPUNIT_ASSERT(err == 0); - } - - err = dbrm.endVBCopy(txn, ranges); - CPPUNIT_ASSERT(err == 0); - - err = dbrm.vbRollback(txn, ranges); - CPPUNIT_ASSERT(err == 0); - - pthread_join(t, NULL); - } - - void dbrm_clear() - { - DBRM dbrm; - VSS vss; - VBBM vbbm; - int err, vssShmid, vbbmShmid, txnID = 1, i; - struct shmid_ds vssShminfo[3], vbbmShminfo[3]; - LBIDRange_v ranges; - LBIDRange range; - VBRange_v freelist; - - err = dbrm.clear(); - CPPUNIT_ASSERT(err == ERR_OK); - - // grab the size of vss and vbbm shmsegs somehow - vss.lock(VSS::READ); - vbbm.lock(VBBM::READ); - vssShmid = vss.getShmid(); - vbbmShmid = vbbm.getShmid(); - err = shmctl(vssShmid, IPC_STAT, &vssShminfo[0]); - CPPUNIT_ASSERT(err == 0); - err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[0]); - CPPUNIT_ASSERT(err == 0); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - // do begin, write, end vbcopy for 150k blocks - cerr << endl << "Adding 150k block entries. "; - range.start = 1; - range.size = 150000; - ranges.push_back(range); - err = dbrm.beginVBCopy(txnID, ranges, freelist); - CPPUNIT_ASSERT(err == 0); - - for (i = range.start; (uint32_t) i < range.size; i++) - { - if (i % 50000 == 0) - cerr << " ... " << i; - - err = dbrm.writeVBEntry(txnID, i, 1, i); - CPPUNIT_ASSERT(err == 0); - } - - err = dbrm.endVBCopy(txnID, ranges); - CPPUNIT_ASSERT(err == 0); - cerr << " done." << endl; - - // grab the sizes again - vss.lock(VSS::READ); - vbbm.lock(VBBM::READ); - vssShmid = vss.getShmid(); - vbbmShmid = vbbm.getShmid(); - err = shmctl(vssShmid, IPC_STAT, &vssShminfo[1]); - CPPUNIT_ASSERT(err == 0); - err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[1]); - CPPUNIT_ASSERT(err == 0); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - // make sure they grew - CPPUNIT_ASSERT(vssShminfo[0].shm_segsz < vssShminfo[1].shm_segsz); - CPPUNIT_ASSERT(vbbmShminfo[0].shm_segsz < vbbmShminfo[1].shm_segsz); - - dbrm.clear(); - - // check that the new size is the same as the original - vss.lock(VSS::READ); - vbbm.lock(VBBM::READ); - vssShmid = vss.getShmid(); - vbbmShmid = vbbm.getShmid(); - err = shmctl(vssShmid, IPC_STAT, &vssShminfo[2]); - CPPUNIT_ASSERT(err == 0); - err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[2]); - CPPUNIT_ASSERT(err == 0); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(vssShminfo[0].shm_segsz == vssShminfo[2].shm_segsz); - CPPUNIT_ASSERT(vbbmShminfo[0].shm_segsz == vbbmShminfo[2].shm_segsz); - - } - - void sessionmanager_1() - { - DBRM dbrm; - int tmp = -1; - - tmp = dbrm.verID(); - // there's no "correct" value b/c it's monotonically increasing. - } - - void sessionmanager_2() - { - DBRM dbrm; - SessionManagerServer::TxnID tmp; - int ver1, ver2; - - ver1 = dbrm.verID(); - tmp = dbrm.newTxnID(1000, true); - ver2 = dbrm.verID(); - - CPPUNIT_ASSERT(ver2 == ver1 + 1); - CPPUNIT_ASSERT(ver2 == tmp.id); - CPPUNIT_ASSERT(tmp.valid == true); - - dbrm.committed(tmp); - CPPUNIT_ASSERT(tmp.valid == false); - } - - void sessionmanager_3() - { - DBRM dbrm; - SessionManagerServer::TxnID txn, txn2; - - txn = dbrm.newTxnID(1000, true); - txn2 = dbrm.getTxnID(1000); - - CPPUNIT_ASSERT(txn.id == txn2.id); - CPPUNIT_ASSERT(txn.valid == txn2.valid == true); - - dbrm.rolledback(txn); - CPPUNIT_ASSERT(txn.valid == false); - - txn2 = dbrm.getTxnID(1000); - CPPUNIT_ASSERT(txn2.valid == false); - } - - void sessionmanager_4() - { - DBRM dbrm; - SessionManagerServer::TxnID txn, txn2; - const SessionManagerServer::SIDTIDEntry* stmap; - int len; - - txn = dbrm.newTxnID(1000, true); - txn2 = dbrm.newTxnID(1001, true); - stmap = dbrm.SIDTIDMap(len); - - CPPUNIT_ASSERT(len == 2); + err = dbrm.endVBCopy(txnID, ranges); + CPPUNIT_ASSERT(err == 0); + cerr << " done." << endl; + + // grab the sizes again + vss.lock(VSS::READ); + vbbm.lock(VBBM::READ); + vssShmid = vss.getShmid(); + vbbmShmid = vbbm.getShmid(); + err = shmctl(vssShmid, IPC_STAT, &vssShminfo[1]); + CPPUNIT_ASSERT(err == 0); + err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[1]); + CPPUNIT_ASSERT(err == 0); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + // make sure they grew + CPPUNIT_ASSERT(vssShminfo[0].shm_segsz < vssShminfo[1].shm_segsz); + CPPUNIT_ASSERT(vbbmShminfo[0].shm_segsz < vbbmShminfo[1].shm_segsz); + + dbrm.clear(); + + // check that the new size is the same as the original + vss.lock(VSS::READ); + vbbm.lock(VBBM::READ); + vssShmid = vss.getShmid(); + vbbmShmid = vbbm.getShmid(); + err = shmctl(vssShmid, IPC_STAT, &vssShminfo[2]); + CPPUNIT_ASSERT(err == 0); + err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[2]); + CPPUNIT_ASSERT(err == 0); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + CPPUNIT_ASSERT(vssShminfo[0].shm_segsz == vssShminfo[2].shm_segsz); + CPPUNIT_ASSERT(vbbmShminfo[0].shm_segsz == vbbmShminfo[2].shm_segsz); + } + + void sessionmanager_1() + { + DBRM dbrm; + int tmp = -1; + + tmp = dbrm.verID(); + // there's no "correct" value b/c it's monotonically increasing. + } + + void sessionmanager_2() + { + DBRM dbrm; + SessionManagerServer::TxnID tmp; + int ver1, ver2; + + ver1 = dbrm.verID(); + tmp = dbrm.newTxnID(1000, true); + ver2 = dbrm.verID(); + + CPPUNIT_ASSERT(ver2 == ver1 + 1); + CPPUNIT_ASSERT(ver2 == tmp.id); + CPPUNIT_ASSERT(tmp.valid == true); + + dbrm.committed(tmp); + CPPUNIT_ASSERT(tmp.valid == false); + } + + void sessionmanager_3() + { + DBRM dbrm; + SessionManagerServer::TxnID txn, txn2; + + txn = dbrm.newTxnID(1000, true); + txn2 = dbrm.getTxnID(1000); + + CPPUNIT_ASSERT(txn.id == txn2.id); + CPPUNIT_ASSERT(txn.valid == txn2.valid == true); + + dbrm.rolledback(txn); + CPPUNIT_ASSERT(txn.valid == false); + + txn2 = dbrm.getTxnID(1000); + CPPUNIT_ASSERT(txn2.valid == false); + } + + void sessionmanager_4() + { + DBRM dbrm; + SessionManagerServer::TxnID txn, txn2; + const SessionManagerServer::SIDTIDEntry* stmap; + int len; + + txn = dbrm.newTxnID(1000, true); + txn2 = dbrm.newTxnID(1001, true); + stmap = dbrm.SIDTIDMap(len); + + CPPUNIT_ASSERT(len == 2); #ifdef BRM_VERBOSE - int i; - cerr << "len = " << len << endl; + int i; + cerr << "len = " << len << endl; - for (i = 0; i < len; i++) - { - cerr << " " << i << ": txnid=" << stmap[i].txnid.id << " valid=" << - stmap[i].txnid.valid << " sessionid=" << stmap[i].sessionid << endl; - } + for (i = 0; i < len; i++) + { + cerr << " " << i << ": txnid=" << stmap[i].txnid.id << " valid=" << stmap[i].txnid.valid + << " sessionid=" << stmap[i].sessionid << endl; + } #endif - dbrm.committed(txn); - dbrm.committed(txn2); - delete [] stmap; - stmap = dbrm.SIDTIDMap(len); + dbrm.committed(txn); + dbrm.committed(txn2); + delete[] stmap; + stmap = dbrm.SIDTIDMap(len); - CPPUNIT_ASSERT(len == 0); - delete [] stmap; - } + CPPUNIT_ASSERT(len == 0); + delete[] stmap; + } - void sessionmanager_5() - { - SessionManager sm; - int tmp = -1; + void sessionmanager_5() + { + SessionManager sm; + int tmp = -1; - tmp = sm.verID(); - // there's no "correct" value b/c it's monotonically increasing. - } + tmp = sm.verID(); + // there's no "correct" value b/c it's monotonically increasing. + } - void sessionmanager_6() - { - SessionManager sm; - SessionManager::TxnID tmp; - int ver1, ver2; + void sessionmanager_6() + { + SessionManager sm; + SessionManager::TxnID tmp; + int ver1, ver2; - ver1 = sm.verID(); - tmp = sm.newTxnID(1000); - ver2 = sm.verID(); + ver1 = sm.verID(); + tmp = sm.newTxnID(1000); + ver2 = sm.verID(); - CPPUNIT_ASSERT(ver2 == ver1 + 1); - CPPUNIT_ASSERT(ver2 == tmp.id); - CPPUNIT_ASSERT(tmp.valid == true); + CPPUNIT_ASSERT(ver2 == ver1 + 1); + CPPUNIT_ASSERT(ver2 == tmp.id); + CPPUNIT_ASSERT(tmp.valid == true); - sm.committed(tmp); - CPPUNIT_ASSERT(tmp.valid == false); - } + sm.committed(tmp); + CPPUNIT_ASSERT(tmp.valid == false); + } - void sessionmanager_7() - { - SessionManager sm; - SessionManager::TxnID txn, txn2; + void sessionmanager_7() + { + SessionManager sm; + SessionManager::TxnID txn, txn2; - txn = sm.newTxnID(1000); - txn2 = sm.getTxnID(1000); + txn = sm.newTxnID(1000); + txn2 = sm.getTxnID(1000); - CPPUNIT_ASSERT(txn.id == txn2.id); - CPPUNIT_ASSERT(txn.valid == txn2.valid == true); + CPPUNIT_ASSERT(txn.id == txn2.id); + CPPUNIT_ASSERT(txn.valid == txn2.valid == true); - sm.rolledback(txn); - CPPUNIT_ASSERT(txn.valid == false); + sm.rolledback(txn); + CPPUNIT_ASSERT(txn.valid == false); - txn2 = sm.getTxnID(1000); - CPPUNIT_ASSERT(txn2.valid == false); - } + txn2 = sm.getTxnID(1000); + CPPUNIT_ASSERT(txn2.valid == false); + } - void sessionmanager_8() - { - SessionManager sm; - SessionManager::TxnID txn, txn2; - const SessionManager::SIDTIDEntry* stmap; - int len; + void sessionmanager_8() + { + SessionManager sm; + SessionManager::TxnID txn, txn2; + const SessionManager::SIDTIDEntry* stmap; + int len; - txn = sm.newTxnID(1000, true); - txn2 = sm.newTxnID(1001, true); - stmap = sm.SIDTIDMap(len); + txn = sm.newTxnID(1000, true); + txn2 = sm.newTxnID(1001, true); + stmap = sm.SIDTIDMap(len); - CPPUNIT_ASSERT(len == 2); + CPPUNIT_ASSERT(len == 2); #ifdef BRM_VERBOSE - int i; - cerr << "len = " << len << endl; + int i; + cerr << "len = " << len << endl; - for (i = 0; i < len; i++) - { - cerr << " " << i << ": txnid=" << stmap[i].txnid.id << " valid=" << - stmap[i].txnid.valid << " sessionid=" << stmap[i].sessionid << endl; - } + for (i = 0; i < len; i++) + { + cerr << " " << i << ": txnid=" << stmap[i].txnid.id << " valid=" << stmap[i].txnid.valid + << " sessionid=" << stmap[i].sessionid << endl; + } #endif - sm.committed(txn); - sm.committed(txn2); - delete [] stmap; - stmap = sm.SIDTIDMap(len); + sm.committed(txn); + sm.committed(txn2); + delete[] stmap; + stmap = sm.SIDTIDMap(len); - CPPUNIT_ASSERT(len == 0); - delete [] stmap; - } + CPPUNIT_ASSERT(len == 0); + delete[] stmap; + } - void brm_markExtentsInvalid() + void brm_markExtentsInvalid() + { + DBRM brm; + int i, err, allocdSize, iterations = 100; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + vector lbids; + int64_t min, max; + int32_t seqNum; + const uint32_t extentSize = brm.getExtentSize(); + + cerr << "brm_markExtentsInvalid" << endl; + + err = brm.createExtent(extentSize * iterations, 1, lbids, allocdSize); + CPPUNIT_ASSERT(lbids.size() == iterations); + CPPUNIT_ASSERT(err == 0); + + // mark all extents valid + for (i = 0; i < iterations; i++) { - - DBRM brm; - int i, err, allocdSize, iterations = 100; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - vector lbids; - int64_t min, max; - int32_t seqNum; - const uint32_t extentSize = brm.getExtentSize(); - - cerr << "brm_markExtentsInvalid" << endl; - - err = brm.createExtent(extentSize * iterations, 1, lbids, allocdSize); - CPPUNIT_ASSERT(lbids.size() == iterations); - CPPUNIT_ASSERT(err == 0); - - // mark all extents valid - for (i = 0; i < iterations; i++) - { - err = brm.setExtentMaxMin(lbids[i], 1, 0, 0); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.markExtentsInvalid(lbids); - CPPUNIT_ASSERT(err == 0); - - // check that they are all invalid/updating/whatever. - // != CP_VALID is what we're looking for - for (i = 0; i < iterations; i++) - { - err = brm.getExtentMaxMin(lbids[i], max, min, seqNum); - CPPUNIT_ASSERT(err == CP_UPDATING); - } - - // cleanup - err = brm.deleteOID(1); + err = brm.setExtentMaxMin(lbids[i], 1, 0, 0); + CPPUNIT_ASSERT(err == 0); } + + err = brm.markExtentsInvalid(lbids); + CPPUNIT_ASSERT(err == 0); + + // check that they are all invalid/updating/whatever. + // != CP_VALID is what we're looking for + for (i = 0; i < iterations; i++) + { + err = brm.getExtentMaxMin(lbids[i], max, min, seqNum); + CPPUNIT_ASSERT(err == CP_UPDATING); + } + + // cleanup + err = brm.deleteOID(1); + } }; - -CPPUNIT_TEST_SUITE_REGISTRATION( BRMTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(BRMTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - idbdatafile::IDBPolicy::configIDBPolicy(); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + idbdatafile::IDBPolicy::configIDBPolicy(); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/versioning/BRM/tdriver-dbrm2.cpp b/versioning/BRM/tdriver-dbrm2.cpp index 5abae48f8..0f74f6e5d 100644 --- a/versioning/BRM/tdriver-dbrm2.cpp +++ b/versioning/BRM/tdriver-dbrm2.cpp @@ -50,1879 +50,1843 @@ pthread_mutex_t mutex; struct Range { - LBID_t start, end, nextBlock; - VER_t txnID; - Range() - { - start = end = nextBlock = 0; - txnID = 0; - } + LBID_t start, end, nextBlock; + VER_t txnID; + Range() + { + start = end = nextBlock = 0; + txnID = 0; + } }; static void* BRMRunner_2(void* arg) { + vector copyList, copiedList, committedList; + vector::iterator rit; + vector writtenList; + vector::iterator lit; - vector copyList, copiedList, committedList; - vector::iterator rit; - vector writtenList; - vector::iterator lit; + pthread_mutex_t listMutex; + int op; + uint32_t randstate; + DBRM* brm; + struct timeval tv; - pthread_mutex_t listMutex; - int op; - uint32_t randstate; - DBRM* brm; - struct timeval tv; + pthread_mutex_init(&listMutex, NULL); + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new DBRM(); - pthread_mutex_init(&listMutex, NULL); - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - brm = new DBRM(); + while (!threadStop) + { + op = rand_r(&randstate) % 9; - while (!threadStop) + switch (op) { - op = rand_r(&randstate) % 9; + case 0: // beginVBCopy + { + int blockCount, size, err; + Range newEntry; + VBRange_v vbRanges; + VBRange_v::iterator vit; + LBIDRange_v ranges; + LBIDRange range; + VER_t txnID; - switch (op) - { - case 0: // beginVBCopy - { - int blockCount, size, err; - Range newEntry; - VBRange_v vbRanges; - VBRange_v::iterator vit; - LBIDRange_v ranges; - LBIDRange range; - VER_t txnID; + size = rand_r(&randstate) % 10000; - size = rand_r(&randstate) % 10000; + pthread_mutex_lock(&mutex); + newEntry.start = lbidCounter; + lbidCounter += size; + txnID = nextTxnID++; + pthread_mutex_unlock(&mutex); - pthread_mutex_lock(&mutex); - newEntry.start = lbidCounter; - lbidCounter += size; - txnID = nextTxnID++; - pthread_mutex_unlock(&mutex); + newEntry.nextBlock = newEntry.start; + newEntry.end = newEntry.start + size; + range.start = newEntry.start; + range.size = size; - newEntry.nextBlock = newEntry.start; - newEntry.end = newEntry.start + size; - range.start = newEntry.start; - range.size = size; + err = brm->beginVBCopy(txnID, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); - err = brm->beginVBCopy(txnID, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); + for (blockCount = 0, vit = vbRanges.begin(); vit != vbRanges.end(); vit++) + blockCount += (*vit).size; - for (blockCount = 0, vit = vbRanges.begin(); vit != vbRanges.end(); vit++) - blockCount += (*vit).size; + CPPUNIT_ASSERT(blockCount == size); - CPPUNIT_ASSERT(blockCount == size); + pthread_mutex_lock(&listMutex); + copyList.push_back(newEntry); + pthread_mutex_unlock(&listMutex); - pthread_mutex_lock(&listMutex); - copyList.push_back(newEntry); - pthread_mutex_unlock(&listMutex); + err = brm->beginVBCopy(txnID, ranges, vbRanges); + CPPUNIT_ASSERT(err == -1); + break; + } - err = brm->beginVBCopy(txnID, ranges, vbRanges); - CPPUNIT_ASSERT(err == -1); - break; - } + case 1: // writeVBEntry + { + int randIndex; + VER_t txnID; + Range* entry; - case 1: // writeVBEntry - { - int randIndex; - VER_t txnID; - Range* entry; + pthread_mutex_lock(&listMutex); - pthread_mutex_lock(&listMutex); + if (copyList.size() == 0) + break; - if (copyList.size() == 0) - break; + randIndex = rand_r(&randstate) % copyList.size(); + entry = &(copyList[randIndex]); + entry->nextBlock++; + txnID = entry->txnID; + /********** WORKING HERE **********/ + } - randIndex = rand_r(&randstate) % copyList.size(); - entry = &(copyList[randIndex]); - entry->nextBlock++; - txnID = entry->txnID; - /********** WORKING HERE **********/ - } - - default: - cerr << "not finished yet" << endl; - } + default: cerr << "not finished yet" << endl; } + } - return NULL; + return NULL; } - - static void* BRMRunner_1(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - DBRM* brm; - ExtentMap em; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - brm = new DBRM(); - - while (!threadStop) - { - op = rand_r(&randstate) % 10; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize, err; - - pthread_mutex_lock(&mutex); - OID = oid++; - opCount++; - pthread_mutex_unlock(&mutex); - - err = brm->createExtent(size, OID, lbids, allocdSize); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm->getExtentSize(); - - if ((size % brm->getExtentSize()) != 0) - entries++; - - if ((uint32_t)entries != lbids.size()) - cerr << "entries = " << entries << " lbids.size = " << lbids.size() << endl; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = brm->getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * brm->getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize, err; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - err = brm->createExtent(size, oid, lbids, allocdSize); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm->getExtentSize(); - - if ((size % brm->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = brm->getExtentSize(); - else - newEm->size = size % brm->getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * brm->getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid, err; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - err = brm->deleteOID(oid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = brm->lookup(target, 0, false, oid, fbo); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = brm->lookup(oid, offset + tmp->FBO, lbid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); - em.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - err = brm->getHWM(tmp->OID, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid, err; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + brm->getExtentSize()); - err = brm->setHWM(oid, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em.checkConsistency(); - break; - } - - case 7: // renew this EM object - { - delete brm; - brm = new DBRM(); -#ifdef BRM_VERBOSE - cerr << "got a new BRM instance" << endl; -#endif - em.checkConsistency(); - break; - } - - case 8: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = brm->getBulkInsertVars(lbid, hwm, txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 9: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = brm->setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - delete brm; - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + DBRM* brm; + ExtentMap em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new DBRM(); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + + pthread_mutex_lock(&mutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&mutex); + + err = brm->createExtent(size, OID, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + if ((uint32_t)entries != lbids.size()) + cerr << "entries = " << entries << " lbids.size = " << lbids.size() << endl; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = brm->getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * brm->getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm->createExtent(size, oid, lbids, allocdSize); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = brm->getExtentSize(); + else + newEm->size = size % brm->getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * brm->getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm->deleteOID(oid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm->lookup(target, 0, false, oid, fbo); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm->lookup(oid, offset + tmp->FBO, lbid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + err = brm->getHWM(tmp->OID, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm->getExtentSize()); + err = brm->setHWM(oid, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: // renew this EM object + { + delete brm; + brm = new DBRM(); +#ifdef BRM_VERBOSE + cerr << "got a new BRM instance" << endl; +#endif + em.checkConsistency(); + break; + } + + case 8: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm->getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm->setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + delete brm; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } DBRM brm_si; static void* BRMRunner_si(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - ExtentMap em; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - - while (!threadStop) - { - op = rand_r(&randstate) % 10; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize, err; - - pthread_mutex_lock(&mutex); - OID = oid++; - opCount++; - pthread_mutex_unlock(&mutex); - - err = brm_si.createExtent(size, OID, lbids, allocdSize); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm_si.getExtentSize(); - - if ((size % brm_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = brm_si.getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * brm_si.getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize, err; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - err = brm_si.createExtent(size, oid, lbids, allocdSize); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - entries = size / brm_si.getExtentSize(); - - if ((size % brm_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = brm_si.getExtentSize(); - else - newEm->size = size % brm_si.getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * brm_si.getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid, err; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - err = brm_si.deleteOID(oid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = brm_si.lookup(target, 0, false, oid, fbo); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = brm_si.lookup(oid, offset + tmp->FBO, lbid); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); - em.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - err = brm_si.getHWM(tmp->OID, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid, err; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize()); - err = brm_si.setHWM(oid, hwm); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em.checkConsistency(); - break; - } - - case 7: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = brm_si.getBulkInsertVars(lbid, hwm, txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 8: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - pthread_mutex_lock(&mutex); - opCount++; - pthread_mutex_unlock(&mutex); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + ExtentMap em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + + pthread_mutex_lock(&mutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&mutex); + + err = brm_si.createExtent(size, OID, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = brm_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * brm_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm_si.createExtent(size, oid, lbids, allocdSize); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = brm_si.getExtentSize(); + else + newEm->size = size % brm_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * brm_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm_si.deleteOID(oid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm_si.lookup(target, 0, false, oid, fbo); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm_si.lookup(oid, offset + tmp->FBO, lbid); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + err = brm_si.getHWM(tmp->OID, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize()); + err = brm_si.setHWM(oid, hwm); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm_si.getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&mutex); + opCount++; + pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } static void* EMRunner(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - ExtentMap* em; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - em = new ExtentMap(); - - while (!threadStop) - { - op = rand_r(&randstate) % 10; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize; - - pthread_mutex_lock(&mutex); - OID = oid++; - pthread_mutex_unlock(&mutex); - - em->createExtent(size, OID, lbids, allocdSize); - em->confirmChanges(); - - entries = size / em->getExtentSize(); - - if ((size % em->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = em->getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * em->getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em->checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - em->createExtent(size, oid, lbids, allocdSize); - em->confirmChanges(); - - entries = size / em->getExtentSize(); - - if ((size % em->getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = em->getExtentSize(); - else - newEm->size = size % em->getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * em->getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em->checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - em->deleteOID(oid); - em->confirmChanges(); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em->checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = em->lookup(target, oid, fbo); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em->checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = em->lookup(oid, offset + tmp->FBO, lbid); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); - em->checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - hwm = em->getHWM(tmp->OID); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em->checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + em->getExtentSize()); - em->setHWM(oid, hwm); - em->confirmChanges(); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em->checkConsistency(); - break; - } - - case 7: // renew this EM object - { - delete em; - em = new ExtentMap(); -#ifdef BRM_VERBOSE - cerr << "got a new EM instance" << endl; -#endif - em->checkConsistency(); - break; - } - - case 8: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = em->getBulkInsertVars(lbid, hwm, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 9: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = em->setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - em->confirmChanges(); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - delete em; - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + ExtentMap* em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + em = new ExtentMap(); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&mutex); + OID = oid++; + pthread_mutex_unlock(&mutex); + + em->createExtent(size, OID, lbids, allocdSize); + em->confirmChanges(); + + entries = size / em->getExtentSize(); + + if ((size % em->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = em->getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * em->getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em->checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + em->createExtent(size, oid, lbids, allocdSize); + em->confirmChanges(); + + entries = size / em->getExtentSize(); + + if ((size % em->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = em->getExtentSize(); + else + newEm->size = size % em->getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * em->getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em->checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em->deleteOID(oid); + em->confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em->checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = em->lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em->checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = em->lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + em->checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em->getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em->checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + em->getExtentSize()); + em->setHWM(oid, hwm); + em->confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em->checkConsistency(); + break; + } + + case 7: // renew this EM object + { + delete em; + em = new ExtentMap(); +#ifdef BRM_VERBOSE + cerr << "got a new EM instance" << endl; +#endif + em->checkConsistency(); + break; + } + + case 8: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em->getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em->setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + em->confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + delete em; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } ExtentMap em_si; static void* EMRunner_si(void* arg) { + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. - // keep track of LBID ranges allocated here and - // randomly allocate, lookup, delete, get/set HWM, and - // destroy the EM object. - - struct EMEntries + struct EMEntries + { + u_int64_t LBIDstart; + uint32_t size; + int OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + uint32_t txnID; + struct EMEntries* next; + EMEntries() { - u_int64_t LBIDstart; - uint32_t size; - int OID; - uint32_t FBO; - uint32_t HWM; - uint32_t secondHWM; - uint32_t txnID; - struct EMEntries* next; - EMEntries() - { - next = NULL; - HWM = 0; - secondHWM = 0; - txnID = 0; - } - }; - -#ifdef BRM_VERBOSE - int threadNum = reinterpret_cast(arg); -#endif - int op, listSize = 0, i; - uint32_t randstate; - struct EMEntries* head = NULL, *tmp; - struct timeval tv; - vector lbids; - -#ifdef BRM_VERBOSE - cerr << "thread number " << threadNum << " started." << endl; -#endif - - gettimeofday(&tv, NULL); - randstate = static_cast(tv.tv_usec); - - while (!threadStop) - { - op = rand_r(&randstate) % 9; -#ifdef BRM_VERBOSE - cerr << "next op is " << op << endl; -#endif - - switch (op) - { - case 0: //allocate space for a new file - { - struct EMEntries* newEm; - int size = rand_r(&randstate) % 102399 + 1; - int entries, OID, allocdSize; - - pthread_mutex_lock(&mutex); - OID = oid++; - pthread_mutex_unlock(&mutex); - - em_si.createExtent(size, OID, lbids, allocdSize); - em_si.confirmChanges(); - - entries = size / em_si.getExtentSize(); - - if ((size % em_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0 ; i < entries; i++) - { - - newEm = new EMEntries(); - newEm->size = em_si.getExtentSize(); - newEm->OID = OID; - newEm->FBO = i * em_si.getExtentSize(); - newEm->LBIDstart = lbids[i]; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created new space for OID " << newEm->OID << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 1: //allocate space for an existing file - { - if (listSize == 0) - break; - - struct EMEntries* newEm, *tmp; - int size = rand_r(&randstate) % 102399 + 1; - int fileRand = rand_r(&randstate) % listSize; - int i, lastExtent, blockEnd, oid; - int tmpHWM, entries, allocdSize; - vector lbids; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) - { - if (tmp->OID != oid) - continue; - - tmpHWM = tmp->HWM; - blockEnd = tmp->FBO + tmp->size; - - if (lastExtent < blockEnd) - lastExtent = blockEnd; - } - - em_si.createExtent(size, oid, lbids, allocdSize); - em_si.confirmChanges(); - - entries = size / em_si.getExtentSize(); - - if ((size % em_si.getExtentSize()) != 0) - entries++; - - CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); - - for (i = 0; i < entries; i++) - { - - newEm = new EMEntries(); - - if (i != entries) - newEm->size = em_si.getExtentSize(); - else - newEm->size = size % em_si.getExtentSize(); - - newEm->OID = oid; - newEm->FBO = lastExtent + (i * em_si.getExtentSize()); - newEm->LBIDstart = lbids[i]; - newEm->HWM = tmpHWM; - - newEm->next = head; - head = newEm; - listSize++; - } - -#ifdef BRM_VERBOSE - cerr << "created another extent for OID " << newEm->OID << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 2: //delete an OID - { - if (listSize == 0) - break; - - struct EMEntries* tmp, *prev; - int fileRand = rand_r(&randstate) % listSize; - int i, oid; - - for (i = 0, tmp = head; i < fileRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - - em_si.deleteOID(oid); - em_si.confirmChanges(); - - for (tmp = head; tmp != NULL;) - { - if (tmp->OID == oid) - { - if (tmp == head) - { - head = head->next; - delete tmp; - tmp = head; - } - else - { - prev->next = tmp->next; - delete tmp; - tmp = prev->next; - } - - listSize--; - } - else - { - prev = tmp; - tmp = tmp->next; - } - } - -#ifdef BRM_VERBOSE - cerr << "deleted OID " << oid << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 3: //lookup by LBID - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset, oid; - struct EMEntries* tmp; - LBID_t target; - uint32_t fbo; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - - target = tmp->LBIDstart + offset; - err = em_si.lookup(target, oid, fbo); -#ifdef BRM_VERBOSE - cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; - cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == tmp->OID); - CPPUNIT_ASSERT(fbo == offset + tmp->FBO); - em_si.checkConsistency(); - break; - } - - case 4: //lookup by OID, FBO - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, oid, err, offset; - struct EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - oid = tmp->OID; - - err = em_si.lookup(oid, offset + tmp->FBO, lbid); -#ifdef BRM_VERBOSE - cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << - " got lbid " << lbid << endl; - cerr << " lbid should be " << tmp->LBIDstart + offset << endl; -#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); - em_si.checkConsistency(); - break; - } - - case 5: //getHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i; - struct EMEntries* tmp; - uint32_t hwm; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - hwm = em_si.getHWM(tmp->OID); -#ifdef BRM_VERBOSE - cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM - << " BRM says it's " << hwm << endl; -#endif - CPPUNIT_ASSERT(hwm == tmp->HWM); - em_si.checkConsistency(); - break; - } - - case 6: //setHWM - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, hwm, oid; - struct EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - oid = tmp->OID; - hwm = rand_r(&randstate) % (tmp->FBO + em_si.getExtentSize()); - em_si.setHWM(oid, hwm); - em_si.confirmChanges(); - - for (tmp = head; tmp != NULL; tmp = tmp->next) - if (tmp->OID == oid) - tmp->HWM = hwm; - -#ifdef BRM_VERBOSE - cerr << "setHWM of OID " << oid << " to " << hwm << endl; -#endif - em_si.checkConsistency(); - break; - } - - case 7: //getBulkInsertVars - { - if (listSize == 0) - break; - - HWM_t hwm; - VER_t txnID; - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - LBID_t lbid; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - lbid = tmp->LBIDstart + offset; - err = em_si.getBulkInsertVars(lbid, hwm, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == tmp->secondHWM); - CPPUNIT_ASSERT(txnID == tmp->txnID); - break; - } - - case 8: //setBulkInsertVars - { - if (listSize == 0) - break; - - int entryRand = rand_r(&randstate) % listSize; - int i, err, offset; - EMEntries* tmp; - - for (i = 0, tmp = head; i < entryRand; i++) - tmp = tmp->next; - - offset = rand_r(&randstate) % tmp->size; - tmp->secondHWM = rand_r(&randstate) % MAXINT; - tmp->txnID = rand_r(&randstate) % MAXINT; - err = em_si.setBulkInsertVars(tmp->LBIDstart + offset, - tmp->secondHWM, tmp->txnID); - em_si.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - break; - } - - default: - break; - } - } - - while (head != NULL) - { - tmp = head->next; - delete head; - head = tmp; + next = NULL; + HWM = 0; + secondHWM = 0; + txnID = 0; } + }; #ifdef BRM_VERBOSE - cerr << "thread " << threadNum << " exiting" << endl; + int threadNum = reinterpret_cast(arg); #endif - return NULL; + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries *head = NULL, *tmp; + struct timeval tv; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 9; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: // allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&mutex); + OID = oid++; + pthread_mutex_unlock(&mutex); + + em_si.createExtent(size, OID, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + newEm->size = em_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * em_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 1: // allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries *newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + em_si.createExtent(size, oid, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries(); + + if (i != entries) + newEm->size = em_si.getExtentSize(); + else + newEm->size = size % em_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * em_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 2: // delete an OID + { + if (listSize == 0) + break; + + struct EMEntries *tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em_si.deleteOID(oid); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 3: // lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = em_si.lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em_si.checkConsistency(); + break; + } + + case 4: // lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = em_si.lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + em_si.checkConsistency(); + break; + } + + case 5: // getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em_si.getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em_si.checkConsistency(); + break; + } + + case 6: // setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + em_si.getExtentSize()); + em_si.setHWM(oid, hwm); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 7: // getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em_si.getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: // setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em_si.setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); + em_si.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; } - - class LongBRMTests : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(LongBRMTests); - CPPUNIT_TEST_SUITE(LongBRMTests); + // CPPUNIT_TEST(longEMTest_1); + // CPPUNIT_TEST(longEMTest_2); + CPPUNIT_TEST(longBRMTest_1); + CPPUNIT_TEST(longBRMTest_2); -// CPPUNIT_TEST(longEMTest_1); -// CPPUNIT_TEST(longEMTest_2); - CPPUNIT_TEST(longBRMTest_1); - CPPUNIT_TEST(longBRMTest_2); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void longEMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; -private: -public: + cerr << endl + << "Multithreaded, multiple instance ExtentMap test. " + "This runs for 5 minutes." + << endl; - void longEMTest_1() + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + if (pthread_create(&threads[i], NULL, EMRunner, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); - cerr << endl << "Multithreaded, multiple instance ExtentMap test. " - "This runs for 5 minutes." << endl; - - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, EMRunner, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the ExtentMap test"); - - usleep(1000); - } - - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + usleep(1000); } - void longEMTest_2() + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } - cerr << endl << "Multithreaded, single instance ExtentMap test. " - "This runs for 5 minutes." << endl; + void longEMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; - threadStop = 0; - pthread_mutex_init(&mutex, NULL); + cerr << endl + << "Multithreaded, single instance ExtentMap test. " + "This runs for 5 minutes." + << endl; - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, EMRunner_si, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the ExtentMap test"); + threadStop = 0; + pthread_mutex_init(&mutex, NULL); - usleep(1000); - } + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, EMRunner_si, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); - sleep(60); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } + usleep(1000); } - void longBRMTest_1() + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } - cerr << endl << "Multithreaded, multiple instance DBRM test. " - "This runs for 5 minutes." << endl; + void longBRMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - opCount = 0; + cerr << endl + << "Multithreaded, multiple instance DBRM test. " + "This runs for 5 minutes." + << endl; - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, BRMRunner_1, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the DBRM test"); + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + opCount = 0; - usleep(1000); - } + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_1, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the DBRM test"); - sleep(300); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } - - cerr << "opCount = " << opCount << endl; + usleep(1000); } - void longBRMTest_2() + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) { - const int threadCount = 10; - int i; - pthread_t threads[threadCount]; - - cerr << endl << "Multithreaded, single instance DBRM test. " - "This runs for 5 minutes." << endl; - - threadStop = 0; - pthread_mutex_init(&mutex, NULL); - opCount = 0; - - for (i = 0; i < threadCount; i++) - { - if (pthread_create(&threads[i], NULL, BRMRunner_si, - reinterpret_cast(i + 1)) < 0) - throw logic_error("Error creating threads for the DBRM test"); - - usleep(1000); - } - - sleep(300); - threadStop = 1; - - for (i = 0; i < threadCount; i++) - { - cerr << "Waiting for thread #" << i << endl; - pthread_join(threads[i], NULL); - } - - cerr << "opCount = " << opCount << endl; + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); } + cerr << "opCount = " << opCount << endl; + } + + void longBRMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl + << "Multithreaded, single instance DBRM test. " + "This runs for 5 minutes." + << endl; + + threadStop = 0; + pthread_mutex_init(&mutex, NULL); + opCount = 0; + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_si, reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the DBRM test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + + cerr << "opCount = " << opCount << endl; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( LongBRMTests ); +CPPUNIT_TEST_SUITE_REGISTRATION(LongBRMTests); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - idbdatafile::IDBPolicy::configIDBPolicy(); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + idbdatafile::IDBPolicy::configIDBPolicy(); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - - diff --git a/versioning/BRM/tdriver-load.cpp b/versioning/BRM/tdriver-load.cpp index 13d91ebcb..a1983b1ab 100644 --- a/versioning/BRM/tdriver-load.cpp +++ b/versioning/BRM/tdriver-load.cpp @@ -50,962 +50,938 @@ using namespace std; void keepalive(int signum) { - cerr << "Yes, it's still going..." << endl; - alarm(290); + cerr << "Yes, it's still going..." << endl; + alarm(290); } class BRMTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(BRMTest); - CPPUNIT_TEST_SUITE(BRMTest); + CPPUNIT_TEST(extentMap_good_1); + CPPUNIT_TEST(extentMap_freelist); + // CPPUNIT_TEST(extentMap_overfill); + CPPUNIT_TEST(many_ExtentMap_instances); + CPPUNIT_TEST(brm_extentmap_good_1); + CPPUNIT_TEST(brm_good_2); + CPPUNIT_TEST(brm_good_3); + CPPUNIT_TEST(brm_deleteOID); + CPPUNIT_TEST(brm_HWM); - CPPUNIT_TEST(extentMap_good_1); - CPPUNIT_TEST(extentMap_freelist); -//CPPUNIT_TEST(extentMap_overfill); - CPPUNIT_TEST(many_ExtentMap_instances); - CPPUNIT_TEST(brm_extentmap_good_1); - CPPUNIT_TEST(brm_good_2); - CPPUNIT_TEST(brm_good_3); - CPPUNIT_TEST(brm_deleteOID); - CPPUNIT_TEST(brm_HWM); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE_END(); + private: + public: + void masterSegmentTest_goodbad() + { + MasterSegmentTable mst; + struct MSTEntry* mt; -private: -public: + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt == NULL); - void masterSegmentTest_goodbad() + mst.releaseTable_write(MasterSegmentTable::EMTable); + mst.releaseTable_write(MasterSegmentTable::EMFreeList); + mst.releaseTable_write(MasterSegmentTable::VBBMSegment); + + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt == NULL); + mst.releaseTable_write(MasterSegmentTable::EMTable); + mst.releaseTable_write(MasterSegmentTable::EMFreeList); + mst.releaseTable_write(MasterSegmentTable::VBBMSegment); + + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mst.releaseTable_read(MasterSegmentTable::EMTable); + mst.releaseTable_read(MasterSegmentTable::EMTable); + mst.releaseTable_read(MasterSegmentTable::EMTable); + + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mst.releaseTable_write(MasterSegmentTable::EMTable); + } + + void extentMap_good_1() + { + ExtentMap em; + int i, err, oid, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + int caughtException = 0, allocdSize; + uint32_t fbo, hwm; + BRM::HWM_t hwm2; + BRM::VER_t txnID; + vector lbids; + const uint32_t extentSize = em.getExtentSize(); + + em.load(string("EMImage")); + em.checkConsistency(); + + for (i = 0; i < iterations; i++) { - MasterSegmentTable mst; - struct MSTEntry* mt; + err = em.lookup(static_cast(i * extentSize), oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 0); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt == NULL); + if (i != 0) + { + err = em.lookup(static_cast(i * extentSize - 1), oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i - 1); + CPPUNIT_ASSERT(fbo == extentSize - 1); + } - mst.releaseTable_write(MasterSegmentTable::EMTable); - mst.releaseTable_write(MasterSegmentTable::EMFreeList); - mst.releaseTable_write(MasterSegmentTable::VBBMSegment); - - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt == NULL); - mst.releaseTable_write(MasterSegmentTable::EMTable); - mst.releaseTable_write(MasterSegmentTable::EMFreeList); - mst.releaseTable_write(MasterSegmentTable::VBBMSegment); - - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mst.releaseTable_read(MasterSegmentTable::EMTable); - mst.releaseTable_read(MasterSegmentTable::EMTable); - mst.releaseTable_read(MasterSegmentTable::EMTable); - - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mst.releaseTable_write(MasterSegmentTable::EMTable); + if (i != iterations - 1) + { + err = em.lookup(static_cast(i * extentSize + 1), oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 1); + } } - void extentMap_good_1() + em.checkConsistency(); + + err = em.lookup(static_cast(i * extentSize), oid, fbo); + CPPUNIT_ASSERT(err == -1); + + for (i = 0; i < iterations; i++) { - ExtentMap em; - int i, err, oid, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - int caughtException = 0, allocdSize; - uint32_t fbo, hwm; - BRM::HWM_t hwm2; - BRM::VER_t txnID; - vector lbids; - const uint32_t extentSize = em.getExtentSize(); + err = em.getBulkInsertVars(static_cast(i * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == 0); + CPPUNIT_ASSERT(txnID == 0); + err = em.setBulkInsertVars(static_cast(i * extentSize), i, i + 1); + em.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + err = em.getBulkInsertVars(static_cast(i * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == static_cast(i)); + CPPUNIT_ASSERT(txnID == static_cast(i + 1)); - em.load(string("EMImage")); - em.checkConsistency(); + hwm = em.getHWM(i); + CPPUNIT_ASSERT(hwm == 0); + em.setHWM(i, (i > (extentSize - 1) ? extentSize - 1 : i)); + em.confirmChanges(); + hwm = em.getHWM(i); + CPPUNIT_ASSERT(hwm == static_cast(i > extentSize - 1 ? extentSize - 1 : i)); + } - for (i = 0; i < iterations; i++) - { - err = em.lookup(static_cast(i * extentSize), oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 0); - - if (i != 0) - { - err = em.lookup(static_cast(i * extentSize - 1), oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i - 1); - CPPUNIT_ASSERT(fbo == extentSize - 1); - } - - if (i != iterations - 1) - { - err = em.lookup(static_cast(i * extentSize + 1), oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 1); - } - } - - em.checkConsistency(); - - err = em.lookup(static_cast(i * extentSize), oid, fbo); - CPPUNIT_ASSERT(err == -1); - - for (i = 0; i < iterations; i++) - { - err = em.getBulkInsertVars(static_cast(i * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == 0); - CPPUNIT_ASSERT(txnID == 0); - err = em.setBulkInsertVars(static_cast(i * extentSize), - i, i + 1); - em.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - err = em.getBulkInsertVars(static_cast(i * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == static_cast(i)); - CPPUNIT_ASSERT(txnID == static_cast(i + 1)); - - hwm = em.getHWM(i); - CPPUNIT_ASSERT(hwm == 0); - em.setHWM(i, (i > (extentSize - 1) ? extentSize - 1 : i)); - em.confirmChanges(); - hwm = em.getHWM(i); - CPPUNIT_ASSERT(hwm == static_cast(i > extentSize - 1 ? extentSize - 1 : i)); - } - - em.checkConsistency(); + em.checkConsistency(); #ifdef BRM_DEBUG - caughtException = 0; + caughtException = 0; - try - { - em.setHWM(i, hwm); - } - catch (std::invalid_argument e) - { - caughtException = 1; - } + try + { + em.setHWM(i, hwm); + } + catch (std::invalid_argument e) + { + caughtException = 1; + } - em.undoChanges(); - CPPUNIT_ASSERT(caughtException == 1); + em.undoChanges(); + CPPUNIT_ASSERT(caughtException == 1); #endif - for (i = 0; i < iterations; i++) - { - em.deleteOID(i); - em.confirmChanges(); - } + for (i = 0; i < iterations; i++) + { + em.deleteOID(i); + em.confirmChanges(); + } #ifdef BRM_DEBUG - caughtException = 0; + caughtException = 0; - try - { - em.deleteOID(i); - } - catch (std::invalid_argument& e) - { - caughtException = 1; - } + try + { + em.deleteOID(i); + } + catch (std::invalid_argument& e) + { + caughtException = 1; + } - em.undoChanges(); - CPPUNIT_ASSERT(caughtException == 1); + em.undoChanges(); + CPPUNIT_ASSERT(caughtException == 1); #endif - em.checkConsistency(); + em.checkConsistency(); + } + + void extentMap_freelist() + { + ExtentMap em; + int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) + vector lbids; + const int extentSize = em.getExtentSize(); + + for (i = 0; i < iterations; i++) + { + em.createExtent(extentSize, i, lbids, allocdSize); + em.confirmChanges(); + CPPUNIT_ASSERT(lbids.back() == static_cast(i * extentSize)); } - void extentMap_freelist() + em.checkConsistency(); + + // frag the lbid space to blow up the free list + for (i = 0; i < iterations; i += 2) { - ExtentMap em; - int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) - vector lbids; - const int extentSize = em.getExtentSize(); - - for (i = 0; i < iterations; i++) - { - em.createExtent(extentSize, i, lbids, allocdSize); - em.confirmChanges(); - CPPUNIT_ASSERT(lbids.back() == static_cast(i * extentSize)); - } - - em.checkConsistency(); - - //frag the lbid space to blow up the free list - for (i = 0; i < iterations; i += 2) - { - em.deleteOID(i); - em.confirmChanges(); - } - - em.checkConsistency(); - - //fill in the holes - for (i = 0; i < iterations; i += 2) - { - em.createExtent(extentSize, i, lbids, allocdSize); - em.confirmChanges(); - } - - for (i = 0; i < iterations; i += 2) - { - em.deleteOID(i); - em.confirmChanges(); - } - - for (i = 1; i < iterations; i += 2) - { - em.deleteOID(i); - em.confirmChanges(); - } - - em.checkConsistency(); + em.deleteOID(i); + em.confirmChanges(); } - /* This test verifies that the ExtentMap stops allocating space when it's full. - (Bug #104) It takes a long time to run with the default LBID - space characteristics so by default it's disabled.*/ - void extentMap_overfill() + em.checkConsistency(); + + // fill in the holes + for (i = 0; i < iterations; i += 2) { - int i, err, tmp; - BlockResolutionManager brm; - vector lbids; - - for (i = 0; i < 67108; i++) - { - err = brm.createExtent(1024000, i, lbids, tmp); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.createExtent(1024000, i, lbids, tmp); - CPPUNIT_ASSERT(err == -1); - - for (i = 0; i < 67108; i++) - { - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == -1); + em.createExtent(extentSize, i, lbids, allocdSize); + em.confirmChanges(); } - void copyLocks_good_1() + for (i = 0; i < iterations; i += 2) { - CopyLocks cl; - LBIDRange range; - int i, iterations = 1000; - - cl.lock(CopyLocks::WRITE); - range.start = 5; - range.size = 2; - cl.lockRange(range, 0); - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.start++; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.start++; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - range.start = 0; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - range.start = 3; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - range.start++; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.start = 5; - range.size = 1; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.size = 2; - cl.releaseRange(range); - - //make sure it can grow - for (i = 0; i < iterations * 2; i += 2) - { - range.start = i; - cl.lockRange(range, 0); - } - - range.size = 1; - - for (i = 0; i < iterations * 2; i++) - { - range.start = i; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - } - - range.start = i; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - - for (i = 0; i < iterations * 2; i += 2) - { - range.start = i; - cl.releaseRange(range); - } - - cl.release(CopyLocks::WRITE); + em.deleteOID(i); + em.confirmChanges(); } - void brm_extentmap_good_1() + for (i = 1; i < iterations; i += 2) { + em.deleteOID(i); + em.confirmChanges(); + } - BlockResolutionManager brm; - int i, err, oid, allocdSize, - iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - uint32_t fbo, hwm; - vector lbids; - HWM_t hwm2; - VER_t txnID; - const uint32_t extentSize = brm.getExtentSize(); + em.checkConsistency(); + } - for (i = 0; i < iterations; i++) - { - brm.createExtent(extentSize, i, lbids, allocdSize); - CPPUNIT_ASSERT(lbids.back() == static_cast(i * extentSize)); - } + /* This test verifies that the ExtentMap stops allocating space when it's full. + (Bug #104) It takes a long time to run with the default LBID + space characteristics so by default it's disabled.*/ + void extentMap_overfill() + { + int i, err, tmp; + BlockResolutionManager brm; + vector lbids; - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - brm.saveExtentMap(string("EMImage")); - brm.loadExtentMap(string("EMImage")); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + for (i = 0; i < 67108; i++) + { + err = brm.createExtent(1024000, i, lbids, tmp); + CPPUNIT_ASSERT(err == 0); + } - for (i = 0; i < iterations; i++) - { - err = brm.lookup(static_cast(i * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 0); + err = brm.createExtent(1024000, i, lbids, tmp); + CPPUNIT_ASSERT(err == -1); - if (i != 0) - { - err = brm.lookup(static_cast(i * extentSize - 1), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i - 1); - CPPUNIT_ASSERT(fbo == extentSize - 1); - } + for (i = 0; i < 67108; i++) + { + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == 0); + } - if (i != iterations) - { - err = brm.lookup(static_cast(i * extentSize + 1), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 1); - } - } + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == -1); + } - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + void copyLocks_good_1() + { + CopyLocks cl; + LBIDRange range; + int i, iterations = 1000; - err = brm.lookup(static_cast(i * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); + cl.lock(CopyLocks::WRITE); + range.start = 5; + range.size = 2; + cl.lockRange(range, 0); + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.start++; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.start++; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + range.start = 0; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + range.start = 3; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + range.start++; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.start = 5; + range.size = 1; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.size = 2; + cl.releaseRange(range); - for (i = 0; i < iterations; i++) - { - err = brm.getBulkInsertVars(static_cast(i * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == 0); - CPPUNIT_ASSERT(txnID == 0); - err = brm.setBulkInsertVars(static_cast(i * extentSize), - i, i + 1); - CPPUNIT_ASSERT(err == 0); - err = brm.getBulkInsertVars(static_cast(i * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == static_cast(i)); - CPPUNIT_ASSERT(txnID == static_cast(i + 1)); + // make sure it can grow + for (i = 0; i < iterations * 2; i += 2) + { + range.start = i; + cl.lockRange(range, 0); + } - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == 0); - err = brm.setHWM(i, (i > extentSize - 1 ? extentSize - 1 : i)); - CPPUNIT_ASSERT(err == 0); - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == static_cast(i > extentSize - 1 ? extentSize - 1 : i)); - } + range.size = 1; - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + for (i = 0; i < iterations * 2; i++) + { + range.start = i; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + } + + range.start = i; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + + for (i = 0; i < iterations * 2; i += 2) + { + range.start = i; + cl.releaseRange(range); + } + + cl.release(CopyLocks::WRITE); + } + + void brm_extentmap_good_1() + { + BlockResolutionManager brm; + int i, err, oid, allocdSize, + iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + uint32_t fbo, hwm; + vector lbids; + HWM_t hwm2; + VER_t txnID; + const uint32_t extentSize = brm.getExtentSize(); + + for (i = 0; i < iterations; i++) + { + brm.createExtent(extentSize, i, lbids, allocdSize); + CPPUNIT_ASSERT(lbids.back() == static_cast(i * extentSize)); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + brm.saveExtentMap(string("EMImage")); + brm.loadExtentMap(string("EMImage")); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < iterations; i++) + { + err = brm.lookup(static_cast(i * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 0); + + if (i != 0) + { + err = brm.lookup(static_cast(i * extentSize - 1), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i - 1); + CPPUNIT_ASSERT(fbo == extentSize - 1); + } + + if (i != iterations) + { + err = brm.lookup(static_cast(i * extentSize + 1), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 1); + } + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.lookup(static_cast(i * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + for (i = 0; i < iterations; i++) + { + err = brm.getBulkInsertVars(static_cast(i * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == 0); + CPPUNIT_ASSERT(txnID == 0); + err = brm.setBulkInsertVars(static_cast(i * extentSize), i, i + 1); + CPPUNIT_ASSERT(err == 0); + err = brm.getBulkInsertVars(static_cast(i * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == static_cast(i)); + CPPUNIT_ASSERT(txnID == static_cast(i + 1)); + + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == 0); + err = brm.setHWM(i, (i > extentSize - 1 ? extentSize - 1 : i)); + CPPUNIT_ASSERT(err == 0); + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == static_cast(i > extentSize - 1 ? extentSize - 1 : i)); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); #ifdef BRM_DEBUG - err = brm.setHWM(i, hwm); - CPPUNIT_ASSERT(err == -1); + err = brm.setHWM(i, hwm); + CPPUNIT_ASSERT(err == -1); #endif - for (i = 0; i < iterations; i++) - { - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == -1); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - + for (i = 0; i < iterations; i++) + { + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == 0); } - void brm_good_2() + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == -1); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + } + + void brm_good_2() + { + BlockResolutionManager brm; + VBBM vbbm; + VSS vss; + CopyLocks cl; + int i, err, size; + vector lbids; + vector extents; + LBIDRange_v ranges; + LBIDRange_v::iterator lbidRangeIT; + VBRange_v vbRanges, vbRanges2; + VBRange_v::iterator vbRangesIT; + EMEntry em; + OID_t oid; + uint32_t fbo; + LBIDRange range; + VBRange vbRange; + VER_t verID; + bool vbFlag; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + err = brm.lookup(0, 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + err = brm.lookup(0, 0, true, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + err = brm.createExtent(10000, 0, lbids, size); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(size == brm.getExtentSize()); + CPPUNIT_ASSERT(lbids.size() == 1); + // CPPUNIT_ASSERT(*(lbids.begin()) == 0); + + err = brm.getExtents(0, extents); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.size() == 1); + + em = *(extents.begin()); + // CPPUNIT_ASSERT(em.range.start == 0); + CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); + CPPUNIT_ASSERT(em.HWM == 0); + CPPUNIT_ASSERT(em.blockOffset == 0); + + for (i = 0; i < 50; i++) { - BlockResolutionManager brm; - VBBM vbbm; - VSS vss; - CopyLocks cl; - int i, err, size; - vector lbids; - vector extents; - LBIDRange_v ranges; - LBIDRange_v::iterator lbidRangeIT; - VBRange_v vbRanges, vbRanges2; - VBRange_v::iterator vbRangesIT; - EMEntry em; - OID_t oid; - uint32_t fbo; - LBIDRange range; - VBRange vbRange; - VER_t verID; - bool vbFlag; - - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - - err = brm.lookup(0, 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - err = brm.lookup(0, 0, true, oid, fbo); - CPPUNIT_ASSERT(err == -1); - - err = brm.createExtent(10000, 0, lbids, size); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(size == brm.getExtentSize()); - CPPUNIT_ASSERT(lbids.size() == 1); -// CPPUNIT_ASSERT(*(lbids.begin()) == 0); - - err = brm.getExtents(0, extents); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.size() == 1); - - em = *(extents.begin()); -// CPPUNIT_ASSERT(em.range.start == 0); - CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); - CPPUNIT_ASSERT(em.HWM == 0); - CPPUNIT_ASSERT(em.blockOffset == 0); - - for (i = 0; i < 50; i++) - { - range.start = i * 100; - range.size = 100; - ranges.push_back(range); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.beginVBCopy(1, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.beginVBCopy(1, ranges, vbRanges2); - CPPUNIT_ASSERT(err == -1); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - for (i = 0; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - vbRange = *(vbRanges.begin()); - -// CPPUNIT_ASSERT(vbRange.vbFBO == 0); - for (i = 0; i < (int)vbRange.size; i++) - { - err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); - CPPUNIT_ASSERT(err == 0); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - } - - for (; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - err = brm.endVBCopy(0, ranges); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - brm.vbCommit(1); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - - verID = 0; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - - err = brm.lookup(i, verID, vbFlag, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == vbRange.vbOID); - CPPUNIT_ASSERT(fbo == static_cast(i + vbRange.vbFBO)); - - vbbm.lock(VBBM::WRITE); - vss.lock(VSS::WRITE); - vbbm.removeEntry(i, verID); -// vss.removeEntry(i, 1); - vss.removeEntry(i, verID); - vss.release(VSS::WRITE); - vbbm.release(VBBM::WRITE); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - brm.deleteOID(0); - - vss.lock(VSS::READ); - vbbm.lock(VBBM::READ); - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - + range.start = i * 100; + range.size = 100; + ranges.push_back(range); } - // cut & pasted from brm_good_2(), but with rollback instead of commit. - void brm_good_3() + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.beginVBCopy(1, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.beginVBCopy(1, ranges, vbRanges2); + CPPUNIT_ASSERT(err == -1); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + for (i = 0; i < 5000; i++) { - BlockResolutionManager brm; - VBBM vbbm; - VSS vss; - CopyLocks cl; - int i, err, size; - vector lbids; - vector::iterator lbid; - vector extents; - LBIDRange_v ranges; - LBIDRange_v::iterator lbidRangeIT; - VBRange_v vbRanges, vbRanges2; - VBRange_v::iterator vbRangesIT; - LBIDRange_v tmp; - EMEntry em; - OID_t oid; - uint32_t fbo; - LBIDRange range; - VBRange vbRange; - VER_t verID; - bool vbFlag; - bool caughtException; + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); + vbRange = *(vbRanges.begin()); - oldsig = signal(SIGALRM, keepalive); - alarm(290); + // CPPUNIT_ASSERT(vbRange.vbFBO == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); + CPPUNIT_ASSERT(err == 0); + } - err = brm.lookup(0, 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - err = brm.lookup(0, 0, true, oid, fbo); - CPPUNIT_ASSERT(err == -1); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - err = brm.createExtent(10000, 0, lbids, size); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(size == brm.getExtentSize()); - CPPUNIT_ASSERT(lbids.size() == 1); - CPPUNIT_ASSERT(*(lbids.begin()) == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + } - err = brm.getExtents(0, extents); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.size() == 1); + for (; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } - em = *(extents.begin()); - CPPUNIT_ASSERT(em.range.start == 0); - CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); - CPPUNIT_ASSERT(em.HWM == 0); - CPPUNIT_ASSERT(em.blockOffset == 0); + err = brm.endVBCopy(0, ranges); - for (i = 0; i < 50; i++) - { - range.start = i * 100; - range.size = 100; - ranges.push_back(range); - } + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + cl.lock(CopyLocks::READ); - err = brm.beginVBCopy(1, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + cl.release(CopyLocks::READ); - cl.lock(CopyLocks::READ); + brm.vbCommit(1); - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - cl.release(CopyLocks::READ); + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); - err = brm.beginVBCopy(1, ranges, vbRanges2); - CPPUNIT_ASSERT(err == -1); + verID = 0; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + err = brm.lookup(i, verID, vbFlag, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == vbRange.vbOID); + CPPUNIT_ASSERT(fbo == static_cast(i + vbRange.vbFBO)); - cl.lock(CopyLocks::READ); + vbbm.lock(VBBM::WRITE); + vss.lock(VSS::WRITE); + vbbm.removeEntry(i, verID); + // vss.removeEntry(i, 1); + vss.removeEntry(i, verID); + vss.release(VSS::WRITE); + vbbm.release(VBBM::WRITE); + } - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - cl.release(CopyLocks::READ); + brm.deleteOID(0); - for (i = 0; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } + vss.lock(VSS::READ); + vbbm.lock(VBBM::READ); + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); - vbRange = *(vbRanges.begin()); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + } -// CPPUNIT_ASSERT(vbRange.vbFBO == 0); - for (i = 0; i < (int)vbRange.size; i++) - { - err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); - CPPUNIT_ASSERT(err == 0); - } + // cut & pasted from brm_good_2(), but with rollback instead of commit. + void brm_good_3() + { + BlockResolutionManager brm; + VBBM vbbm; + VSS vss; + CopyLocks cl; + int i, err, size; + vector lbids; + vector::iterator lbid; + vector extents; + LBIDRange_v ranges; + LBIDRange_v::iterator lbidRangeIT; + VBRange_v vbRanges, vbRanges2; + VBRange_v::iterator vbRangesIT; + LBIDRange_v tmp; + EMEntry em; + OID_t oid; + uint32_t fbo; + LBIDRange range; + VBRange vbRange; + VER_t verID; + bool vbFlag; + bool caughtException; - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - } + oldsig = signal(SIGALRM, keepalive); + alarm(290); - for (; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } + err = brm.lookup(0, 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + err = brm.lookup(0, 0, true, oid, fbo); + CPPUNIT_ASSERT(err == -1); - err = brm.endVBCopy(0, ranges); + err = brm.createExtent(10000, 0, lbids, size); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(size == brm.getExtentSize()); + CPPUNIT_ASSERT(lbids.size() == 1); + CPPUNIT_ASSERT(*(lbids.begin()) == 0); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + err = brm.getExtents(0, extents); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.size() == 1); - cl.lock(CopyLocks::READ); + em = *(extents.begin()); + CPPUNIT_ASSERT(em.range.start == 0); + CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); + CPPUNIT_ASSERT(em.HWM == 0); + CPPUNIT_ASSERT(em.blockOffset == 0); - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); + for (i = 0; i < 50; i++) + { + range.start = i * 100; + range.size = 100; + ranges.push_back(range); + } - cl.release(CopyLocks::READ); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - err = brm.getUncommittedLBIDs(1, lbids); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbids.size() == vbRange.size); - sort::iterator>(lbids.begin(), lbids.end()); - lbid = lbids.begin(); + err = brm.beginVBCopy(1, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); - for (i = 0; i < static_cast(lbids.size()); i++, lbid++) - CPPUNIT_ASSERT((*lbid) == static_cast(i)); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - range.start = 0; - range.size = i; - tmp.push_back(range); - err = brm.vbRollback(1, tmp); - CPPUNIT_ASSERT(err == 0); + cl.lock(CopyLocks::READ); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; + cl.release(CopyLocks::READ); - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == false); + err = brm.beginVBCopy(1, ranges, vbRanges2); + CPPUNIT_ASSERT(err == -1); - err = brm.lookup(i, verID, vbFlag, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == 0); - CPPUNIT_ASSERT(fbo == static_cast(i)); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - vbbm.lock(VBBM::WRITE); - vss.lock(VSS::WRITE); + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + for (i = 0; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + + vbRange = *(vbRanges.begin()); + + // CPPUNIT_ASSERT(vbRange.vbFBO == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + } + + for (; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + + err = brm.endVBCopy(0, ranges); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.getUncommittedLBIDs(1, lbids); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbids.size() == vbRange.size); + sort::iterator>(lbids.begin(), lbids.end()); + lbid = lbids.begin(); + + for (i = 0; i < static_cast(lbids.size()); i++, lbid++) + CPPUNIT_ASSERT((*lbid) == static_cast(i)); + + range.start = 0; + range.size = i; + tmp.push_back(range); + err = brm.vbRollback(1, tmp); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == false); + + err = brm.lookup(i, verID, vbFlag, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == 0); + CPPUNIT_ASSERT(fbo == static_cast(i)); + + vbbm.lock(VBBM::WRITE); + vss.lock(VSS::WRITE); #ifdef BRM_DEBUG - caughtException = false; + caughtException = false; - try - { - vbbm.removeEntry(i, verID); - } - catch (logic_error& e) - { - caughtException = true; - } + try + { + vbbm.removeEntry(i, verID); + } + catch (logic_error& e) + { + caughtException = true; + } - CPPUNIT_ASSERT(caughtException); - caughtException = false; + CPPUNIT_ASSERT(caughtException); + caughtException = false; - try - { - vss.removeEntry(i, 1); - } - catch (logic_error& e) - { - caughtException = true; - } + try + { + vss.removeEntry(i, 1); + } + catch (logic_error& e) + { + caughtException = true; + } - CPPUNIT_ASSERT(caughtException); + CPPUNIT_ASSERT(caughtException); #endif - vss.removeEntry(i, verID); - vss.release(VSS::WRITE); - vbbm.release(VBBM::WRITE); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - brm.deleteOID(0); - - vbbm.lock(VBBM::READ); - vss.lock(VSS::READ); - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); + vss.removeEntry(i, verID); + vss.release(VSS::WRITE); + vbbm.release(VBBM::WRITE); } - /* This test verifies that deleteOID returns an error for - non-existant OIDs (bug #105) */ - void brm_deleteOID() - { - BlockResolutionManager brm; - int err; - vector extents; + CPPUNIT_ASSERT(brm.checkConsistency() == 0); - err = brm.getExtents(0, extents); + brm.deleteOID(0); + + vbbm.lock(VBBM::READ); + vss.lock(VSS::READ); + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + } + + /* This test verifies that deleteOID returns an error for + non-existant OIDs (bug #105) */ + void brm_deleteOID() + { + BlockResolutionManager brm; + int err; + vector extents; + + err = brm.getExtents(0, extents); #ifdef BRM_DEBUG - if (err != -1) - cerr << "Make sure OID 0 isn't allocated in the extent map" << endl; + if (err != -1) + cerr << "Make sure OID 0 isn't allocated in the extent map" << endl; - CPPUNIT_ASSERT(err == -1); + CPPUNIT_ASSERT(err == -1); #else - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.empty()); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.empty()); #endif - err = brm.deleteOID(0); - CPPUNIT_ASSERT(err == -1); - } + err = brm.deleteOID(0); + CPPUNIT_ASSERT(err == -1); + } - /* This test verifies that setHWM and getHWM return an error for - nonexistant OIDs (bugs #106, 107) */ + /* This test verifies that setHWM and getHWM return an error for + nonexistant OIDs (bugs #106, 107) */ - void brm_HWM() - { - BlockResolutionManager brm; - int err; - HWM_t hwm; - vector extents; + void brm_HWM() + { + BlockResolutionManager brm; + int err; + HWM_t hwm; + vector extents; - err = brm.getExtents(0, extents); + err = brm.getExtents(0, extents); #ifdef BRM_DEBUG - if (err != -1) - cerr << "Make sure OID 0 isn't allocated in the extent map" << endl; + if (err != -1) + cerr << "Make sure OID 0 isn't allocated in the extent map" << endl; - CPPUNIT_ASSERT(err == -1); + CPPUNIT_ASSERT(err == -1); #else - CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(err == 0); #endif - CPPUNIT_ASSERT(extents.size() == 0); - err = brm.setHWM(0, 10); - CPPUNIT_ASSERT(err == -1); - err = brm.getHWM(0, hwm); - CPPUNIT_ASSERT(err == -1); - } + CPPUNIT_ASSERT(extents.size() == 0); + err = brm.setHWM(0, 10); + CPPUNIT_ASSERT(err == -1); + err = brm.getHWM(0, hwm); + CPPUNIT_ASSERT(err == -1); + } - /* This test was suggested by Jean, who observed some odd behavior at around - 1032 EM instances in her code. This test creates 2000 instances and uses 1400 of them - to do the extentmap_freelist_1 test above. */ - void many_ExtentMap_instances() + /* This test was suggested by Jean, who observed some odd behavior at around + 1032 EM instances in her code. This test creates 2000 instances and uses 1400 of them + to do the extentmap_freelist_1 test above. */ + void many_ExtentMap_instances() + { + ExtentMap** ems; + int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) + vector lbids; + uint32_t extentSize; + + // cerr << endl << "Extent Map instance test" << endl; + + ems = new ExtentMap*[2000]; + + for (i = 0; i < 2000; i++) + ems[i] = new ExtentMap(); + + extentSize = ems[0]->getExtentSize(); + + // cerr << " - Successfully created 2000 instances, starting the freelist test" + // << endl; + for (i = 0; i < iterations; i++) { - ExtentMap** ems; - int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) - vector lbids; - uint32_t extentSize; - -// cerr << endl << "Extent Map instance test" << endl; - - ems = new ExtentMap*[2000]; - - for (i = 0; i < 2000; i++) - ems[i] = new ExtentMap(); - - extentSize = ems[0]->getExtentSize(); - -// cerr << " - Successfully created 2000 instances, starting the freelist test" -// << endl; - for (i = 0; i < iterations; i++) - { - ems[i]->createExtent(extentSize, i, lbids, allocdSize); - ems[i]->confirmChanges(); - CPPUNIT_ASSERT(lbids.back() == static_cast(i * extentSize)); - } - - //frag the lbid space to blow up the free list - for (i = 0; i < iterations; i += 2) - { - ems[i]->deleteOID(i); - ems[i]->confirmChanges(); - } - - //fill in the holes - for (i = 0; i < iterations; i += 2) - { - ems[i]->createExtent(extentSize, i, lbids, allocdSize); - ems[i]->confirmChanges(); - } - - for (i = 0; i < iterations; i += 2) - { - ems[i]->deleteOID(i); - ems[i]->confirmChanges(); - } - - for (i = 1; i < iterations; i += 2) - { - ems[i]->deleteOID(i); - ems[i]->confirmChanges(); - } - -// cerr << "done. sleeping for 30 seconds." << endl; -// sleep(30); - - for (i = 0; i < 2000; i++) - delete ems[i]; - - delete [] ems; - + ems[i]->createExtent(extentSize, i, lbids, allocdSize); + ems[i]->confirmChanges(); + CPPUNIT_ASSERT(lbids.back() == static_cast(i * extentSize)); } + // frag the lbid space to blow up the free list + for (i = 0; i < iterations; i += 2) + { + ems[i]->deleteOID(i); + ems[i]->confirmChanges(); + } + // fill in the holes + for (i = 0; i < iterations; i += 2) + { + ems[i]->createExtent(extentSize, i, lbids, allocdSize); + ems[i]->confirmChanges(); + } + + for (i = 0; i < iterations; i += 2) + { + ems[i]->deleteOID(i); + ems[i]->confirmChanges(); + } + + for (i = 1; i < iterations; i += 2) + { + ems[i]->deleteOID(i); + ems[i]->confirmChanges(); + } + + // cerr << "done. sleeping for 30 seconds." << endl; + // sleep(30); + + for (i = 0; i < 2000; i++) + delete ems[i]; + + delete[] ems; + } }; - - -CPPUNIT_TEST_SUITE_REGISTRATION( BRMTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(BRMTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - idbdatafile::IDBPolicy::configIDBPolicy(); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + idbdatafile::IDBPolicy::configIDBPolicy(); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/versioning/BRM/tdriver.cpp b/versioning/BRM/tdriver.cpp index 8c519e779..280d9a3a4 100644 --- a/versioning/BRM/tdriver.cpp +++ b/versioning/BRM/tdriver.cpp @@ -48,1868 +48,1832 @@ pthread_mutex_t mutex; void keepalive(int signum) { - cerr << "Yes, it's still going..." << endl; - alarm(290); + cerr << "Yes, it's still going..." << endl; + alarm(290); } void* dummy_1(void* arg) { - LBIDResourceGraph* rg = (LBIDResourceGraph*) arg; - int err; - VER_t txn = 2; + LBIDResourceGraph* rg = (LBIDResourceGraph*)arg; + int err; + VER_t txn = 2; - pthread_mutex_lock(&mutex); - err = rg->reserveRange(0, 2000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - rg->releaseResources(txn); - pthread_mutex_unlock(&mutex); - return NULL; + pthread_mutex_lock(&mutex); + err = rg->reserveRange(0, 2000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + rg->releaseResources(txn); + pthread_mutex_unlock(&mutex); + return NULL; } void* dummy_deadlock(void* arg) { - LBIDResourceGraph* rg = (LBIDResourceGraph*) arg; - int err; - VER_t txn = 2; + LBIDResourceGraph* rg = (LBIDResourceGraph*)arg; + int err; + VER_t txn = 2; - pthread_mutex_lock(&mutex); - err = rg->reserveRange(1001, 2000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - err = rg->reserveRange(0, 1000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - rg->releaseResources(txn); - CPPUNIT_ASSERT(err == 0); - pthread_mutex_unlock(&mutex); - return NULL; + pthread_mutex_lock(&mutex); + err = rg->reserveRange(1001, 2000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + err = rg->reserveRange(0, 1000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + rg->releaseResources(txn); + CPPUNIT_ASSERT(err == 0); + pthread_mutex_unlock(&mutex); + return NULL; } void* fragd_range_helper(void* arg) { - LBIDResourceGraph* rg = (LBIDResourceGraph*) arg; - set >* resources; - map* txns; - int err; - VER_t txn = 2; + LBIDResourceGraph* rg = (LBIDResourceGraph*)arg; + set >* resources; + map* txns; + int err; + VER_t txn = 2; - pthread_mutex_lock(&mutex); - resources = rg->getResources(); - txns = rg->getTxns(); - CPPUNIT_ASSERT(resources->size() == 2); - CPPUNIT_ASSERT(txns->size() == 1); - err = rg->reserveRange(0, 5000, txn, mutex); + pthread_mutex_lock(&mutex); + resources = rg->getResources(); + txns = rg->getTxns(); + CPPUNIT_ASSERT(resources->size() == 2); + CPPUNIT_ASSERT(txns->size() == 1); + err = rg->reserveRange(0, 5000, txn, mutex); - // blocks, main thread releases + // blocks, main thread releases - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(resources->size() == 5); - CPPUNIT_ASSERT(txns->size() == 1); - rg->releaseResources(txn); - pthread_mutex_unlock(&mutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(resources->size() == 5); + CPPUNIT_ASSERT(txns->size() == 1); + rg->releaseResources(txn); + pthread_mutex_unlock(&mutex); - return NULL; + return NULL; } void* overlapping_range_helper(void* arg) { - LBIDResourceGraph* rg = (LBIDResourceGraph*) arg; + LBIDResourceGraph* rg = (LBIDResourceGraph*)arg; + set >* resources; + set >::const_iterator it; + map* txns; + int err; + VER_t txn = 2; + + pthread_mutex_lock(&mutex); + resources = rg->getResources(); + txns = rg->getTxns(); + CPPUNIT_ASSERT(resources->size() == 2); + CPPUNIT_ASSERT(txns->size() == 1); + err = rg->reserveRange(1100, 3900, txn, mutex); + + // blocks, main thread releases + + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(resources->size() == 3); + it = resources->begin(); + CPPUNIT_ASSERT((*it)->start() == 1100); + CPPUNIT_ASSERT((*it)->end() == 2000); + it++; + CPPUNIT_ASSERT((*it)->start() == 2001); + CPPUNIT_ASSERT((*it)->end() == 3000); + it++; + CPPUNIT_ASSERT((*it)->start() == 3001); + CPPUNIT_ASSERT((*it)->end() == 3900); + CPPUNIT_ASSERT(txns->size() == 1); + rg->releaseResources(txn); + pthread_mutex_unlock(&mutex); + + return NULL; +} + +class BRMTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(BRMTest); + CPPUNIT_TEST(masterSegmentTest_goodbad); + CPPUNIT_TEST(extentMap_good_1); +#ifdef PARTITIONING + CPPUNIT_TEST(extentMap_range_1); +#endif + CPPUNIT_TEST(extentMap_freelist); + // CPPUNIT_TEST(extentMap_overfill); + // CPPUNIT_TEST(many_ExtentMap_instances); //Jean unsuggested this case since writeengine use singleton + CPPUNIT_TEST(copyLocks_good_1); + CPPUNIT_TEST(vbbm_good_1); + CPPUNIT_TEST(vbbm_good_1); + CPPUNIT_TEST(vss_good_1); + CPPUNIT_TEST(vss_good_1); +// CPPUNIT_TEST(brm_extentmap_good_1);//fix for bug 640, this test case has a bug +#ifdef PARTITIONING + CPPUNIT_TEST(brm_extentmap_range_1); +#endif + CPPUNIT_TEST(brm_good_2); + CPPUNIT_TEST(brm_good_3); + CPPUNIT_TEST(brm_deleteOID); + CPPUNIT_TEST(brm_HWM); + CPPUNIT_TEST(resource_graph_1); + CPPUNIT_TEST(resource_graph_fragd_range); + CPPUNIT_TEST(resource_graph_deadlock); + CPPUNIT_TEST(resource_graph_overlapping_ranges); + + CPPUNIT_TEST_SUITE_END(); + + private: + public: + void masterSegmentTest_goodbad() + { + MasterSegmentTable mst; + struct MSTEntry* mt; + + // cerr << "MST goodbad" << endl; + + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt == NULL); + + mst.releaseTable_write(MasterSegmentTable::EMTable); + mst.releaseTable_write(MasterSegmentTable::EMFreeList); + mst.releaseTable_write(MasterSegmentTable::VBBMSegment); + + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); + CPPUNIT_ASSERT(mt == NULL); + mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); + CPPUNIT_ASSERT(mt == NULL); + mst.releaseTable_write(MasterSegmentTable::EMTable); + mst.releaseTable_write(MasterSegmentTable::EMFreeList); + mst.releaseTable_write(MasterSegmentTable::VBBMSegment); + + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mst.releaseTable_read(MasterSegmentTable::EMTable); + mst.releaseTable_read(MasterSegmentTable::EMTable); + mst.releaseTable_read(MasterSegmentTable::EMTable); + + mt = mst.getTable_write(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt != NULL); + mt = mst.getTable_read(MasterSegmentTable::EMTable, false); + CPPUNIT_ASSERT(mt == NULL); + mst.releaseTable_write(MasterSegmentTable::EMTable); + Config::deleteInstanceMap(); + } + + void extentMap_good_1() + { + ExtentMap em; + int i, err, oid, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + int caughtException = 0, allocdSize; + uint32_t fbo, hwm; + BRM::HWM_t hwm2; + BRM::VER_t txnID; + vector lbids; + const uint32_t extentSize = em.getExtentSize(); + + // cerr << "em_good_1" << endl; + + for (i = 1; i < iterations; i++) + { + em.createExtent(extentSize, i, lbids, allocdSize); + em.confirmChanges(); + CPPUNIT_ASSERT(lbids.size() == 1); + CPPUNIT_ASSERT((uint32_t)allocdSize == extentSize); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + } + + em.checkConsistency(); + em.save(string("EMImage")); + em.load(string("EMImage")); + em.checkConsistency(); + + for (i = 1; i < iterations; i++) + { + err = em.lookup(static_cast((i - 1) * extentSize), oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 0); + + if (i != 1) + { + err = em.lookup(static_cast((i - 1) * extentSize - 1), oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i - 1); + CPPUNIT_ASSERT(fbo == extentSize - 1); + } + + if (i != iterations - 1) + { + err = em.lookup(static_cast((i - 1) * extentSize + 1), oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 1); + } + } + + em.checkConsistency(); + + err = em.lookup(static_cast(i * extentSize), oid, fbo); + CPPUNIT_ASSERT(err == -1); + + for (i = 1; i < iterations; i++) + { + err = em.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == 0); + CPPUNIT_ASSERT(txnID == 0); + err = em.setBulkInsertVars(static_cast((i - 1) * extentSize), i, i + 1); + em.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + err = em.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == static_cast(i)); + CPPUNIT_ASSERT(txnID == static_cast(i + 1)); + + hwm = em.getHWM(i); + CPPUNIT_ASSERT(hwm == 0); + em.setHWM(i, ((uint32_t)i > (extentSize - 1) ? extentSize - 1 : i)); + em.confirmChanges(); + hwm = em.getHWM(i); + CPPUNIT_ASSERT(hwm == static_cast((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); + } + + em.checkConsistency(); + +#ifdef BRM_DEBUG + caughtException = 0; + + try + { + em.setHWM(i, hwm); + } + catch (std::invalid_argument e) + { + caughtException = 1; + } + + em.undoChanges(); + CPPUNIT_ASSERT(caughtException == 1); +#endif + + for (i = 1; i < iterations; i++) + { + em.deleteOID(i); + em.confirmChanges(); + } + +#ifdef BRM_DEBUG + caughtException = 0; + + try + { + em.deleteOID(i); + } + catch (std::invalid_argument& e) + { + caughtException = 1; + } + + em.undoChanges(); + CPPUNIT_ASSERT(caughtException == 1); +#endif + + em.checkConsistency(); + Config::deleteInstanceMap(); + } + +#ifdef PARTITIONING + void extentMap_range_1() + { + ExtentMap em; + int i, err, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + int caughtException = 0, allocdSize; + uint32_t hwm; + BRM::HWM_t hwm2; + BRM::VER_t txnID; + vector lbids; + const uint32_t extentSize = em.getExtentSize(); + + for (i = 1; i < iterations; i++) + { + em.createRangeExtent(extentSize, i, lbids, allocdSize, EMRangePartition_t(0, 10, true, true)); + em.confirmChanges(); + CPPUNIT_ASSERT(lbids.size() == 1); + CPPUNIT_ASSERT((uint32_t)allocdSize == extentSize); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + } + + em.checkConsistency(); + em.save(string("EMImage")); + em.load(string("EMImage")); + em.checkConsistency(); + + LBIDRange_v lbidList; + + for (i = 1; i < iterations; i++) + { + err = em.lookup(i, 1, lbidList); + CPPUNIT_ASSERT(lbidList.size() == 1); + CPPUNIT_ASSERT(err == lbidList.size()); + + err = em.lookup(i, i + 10, lbidList); + CPPUNIT_ASSERT(lbidList.size() == 0); + CPPUNIT_ASSERT(err == lbidList.size()); + + err = em.lookup(i * 10000, 1, lbidList); + CPPUNIT_ASSERT(lbidList.size() == 0); + CPPUNIT_ASSERT(err == lbidList.size()); + } + + em.checkConsistency(); + + err = em.lookup(i, 99, lbidList); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbidList.size() == 0); + +#ifdef BRM_DEBUG + caughtException = 0; + + try + { + em.setHWM(i, hwm); + } + catch (std::invalid_argument e) + { + caughtException = 1; + } + + em.undoChanges(); + CPPUNIT_ASSERT(caughtException == 1); +#endif + + for (i = 1; i < iterations; i++) + { + em.deleteOID(i); + em.confirmChanges(); + } + +#ifdef BRM_DEBUG + caughtException = 0; + + try + { + em.deleteOID(i); + } + catch (std::invalid_argument& e) + { + caughtException = 1; + } + + em.undoChanges(); + CPPUNIT_ASSERT(caughtException == 1); +#endif + + em.checkConsistency(); + Config::deleteInstanceMap(); + } +#endif + + void extentMap_freelist() + { + ExtentMap em; + int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) + vector lbids; + const int extentSize = em.getExtentSize(); + + // cerr << "em_freelist" << endl; + + for (i = 1; i < iterations; i++) + { + em.createExtent(extentSize, i, lbids, allocdSize); + em.confirmChanges(); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + } + + em.checkConsistency(); + + // frag the lbid space to blow up the free list + for (i = 1; i < iterations; i += 2) + { + em.deleteOID(i); + em.confirmChanges(); + } + + em.checkConsistency(); + + // fill in the holes + for (i = 1; i < iterations; i += 2) + { + em.createExtent(extentSize, i, lbids, allocdSize); + em.confirmChanges(); + } + + for (i = 1; i < iterations; i += 2) + { + em.deleteOID(i); + em.confirmChanges(); + } + + for (i = 2; i < iterations; i += 2) + { + em.deleteOID(i); + em.confirmChanges(); + } + + em.checkConsistency(); + Config::deleteInstanceMap(); + } + + /* This test verifies that the ExtentMap stops allocating space when it's full. + (Bug #104) It takes a long time to run with the default LBID + space characteristics so by default it's disabled.*/ + void extentMap_overfill() + { + int i, err, tmp; + BlockResolutionManager brm; + vector lbids; + + for (i = 1; i < 67109; i++) + { + err = brm.createExtent(1024000, i, lbids, tmp); + CPPUNIT_ASSERT(err == 0); + } + + err = brm.createExtent(1024000, i, lbids, tmp); + CPPUNIT_ASSERT(err == -1); + + for (i = 1; i < 67109; i++) + { + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == 0); + } + + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == -1); + Config::deleteInstanceMap(); + } + + void copyLocks_good_1() + { + CopyLocks cl; + LBIDRange range; + int i, iterations = 1000; + + // cerr << "copylocks_good_1" << endl; + + cl.lock(CopyLocks::WRITE); + range.start = 5; + range.size = 2; + cl.lockRange(range, 0); + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.start++; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.start++; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + range.start = 0; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + range.start = 3; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + range.start++; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.start = 5; + range.size = 1; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + range.size = 2; + cl.releaseRange(range); + + // make sure it can grow + for (i = 0; i < iterations * 2; i += 2) + { + range.start = i; + cl.lockRange(range, 0); + } + + range.size = 1; + + cl.save(string("CLImage")); + cl.load(string("CLImage")); + + for (i = 0; i < iterations * 2; i++) + { + range.start = i; + CPPUNIT_ASSERT(cl.isLocked(range) == true); + } + + range.start = i; + CPPUNIT_ASSERT(cl.isLocked(range) == false); + + for (i = 0; i < iterations * 2; i += 2) + { + range.start = i; + cl.releaseRange(range); + } + + cl.release(CopyLocks::WRITE); + Config::deleteInstanceMap(); + } + + void brm_extentmap_good_1() + { + BlockResolutionManager brm; + int i, err, oid, allocdSize, + iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + uint32_t fbo, hwm; + vector lbids; + HWM_t hwm2; + VER_t txnID; + const uint32_t extentSize = brm.getExtentSize(); + + // cerr << "brm_extentmap_good_1" << endl; + + for (i = 1; i < iterations; i++) + { + brm.createExtent(extentSize, i, lbids, allocdSize); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + brm.saveExtentMap(string("EMImage")); + brm.loadExtentMap(string("EMImage")); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + ExtentMap em; + + // int32_t seqNum=0; + // int64_t max=0; + // int64_t min=0; + LBID_t lbid; + /* + for (i = 1; i < iterations; i++) { + lbid=(i-1)*extentSize; + err=em.updateMaxMin(lbid, 1, 0, seqNum); // sequenceNum=0 + em.confirmChanges(); + CPPUNIT_ASSERT(err==0); + + max=-1; + min=-1; + err=em.getMaxMin(lbid, max, min, seqNum); + CPPUNIT_ASSERT(max==1); + CPPUNIT_ASSERT(min==0); + CPPUNIT_ASSERT(err==2); + CPPUNIT_ASSERT(seqNum==0); + + err=em.updateMaxMin(lbid, 4, 2, seqNum); // sequenceNum=0 + em.confirmChanges(); + CPPUNIT_ASSERT(err==0); + + max=-1; + min=-1; + seqNum++; + err=em.getMaxMin(lbid, max, min, seqNum); + CPPUNIT_ASSERT(max==4); + CPPUNIT_ASSERT(min==2); + CPPUNIT_ASSERT(err==2); + CPPUNIT_ASSERT(seqNum==0); + + err=em.markInvalid(lbid); // sequenceNum=1, valid=1 + em.confirmChanges(); + CPPUNIT_ASSERT(err==0); + + seqNum=0; // re-init seqNum + err=em.getMaxMin(lbid, max, min, seqNum); + CPPUNIT_ASSERT(err==1); + CPPUNIT_ASSERT(seqNum==1); + + seqNum=0; + } + */ + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + vector lbidVector; + uint32_t offset; + + for (i = 1; i < iterations; i++) + { + lbid = (i - 1) * extentSize; + + for (int j = 0; (uint32_t)j < extentSize; j++) + lbidVector.push_back(lbid + j); + + err = em.markInvalid(lbidVector); + em.confirmChanges(); + CPPUNIT_ASSERT(err == 1); + lbidVector.clear(); + em.lookup(lbid, oid, offset); + err = em.markInvalid(oid); + em.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + // CPPUNIT_ASSERT(oid==i); + oid = -1; + offset = 0; + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 1; i < iterations; i++) + { + err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 0); + + if (i != 1) + { + err = brm.lookup(static_cast((i - 1) * extentSize - 1), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i - 1); + CPPUNIT_ASSERT(fbo == extentSize - 1); + } + + if (i != iterations) + { + err = brm.lookup(static_cast((i - 1) * extentSize + 1), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i); + CPPUNIT_ASSERT(fbo == 1); + } + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + for (i = 1; i < iterations; i++) + { + err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == 0); + CPPUNIT_ASSERT(txnID == 0); + err = brm.setBulkInsertVars(static_cast((i - 1) * extentSize), i, i + 1); + CPPUNIT_ASSERT(err == 0); + err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == static_cast(i)); + CPPUNIT_ASSERT(txnID == static_cast(i + 1)); + + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == 0); + err = brm.setHWM(i, ((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); + CPPUNIT_ASSERT(err == 0); + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == static_cast((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + +#ifdef BRM_DEBUG + err = brm.setHWM(i, hwm); + CPPUNIT_ASSERT(err == -1); +#endif + + for (i = 1; i < iterations; i++) + { + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == 0); + } + + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == -1); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + Config::deleteInstanceMap(); + } + +#ifdef PARTITIONING + void brm_extentmap_range_1() + { + BlockResolutionManager brm; + int i, err, oid, allocdSize, + iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) + uint32_t fbo, hwm; + vector lbids; + HWM_t hwm2; + VER_t txnID; + const uint32_t extentSize = brm.getExtentSize(); + LBIDRange_v lbidList; + + // cerr << "brm_extentmap_range_1" << endl; + + for (i = 1; i < iterations; i++) + { + brm.createRangeExtent(extentSize, i, lbids, allocdSize, EMRangePartition_t(0, 9, true, true)); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + CPPUNIT_ASSERT(lbids.size() == 1); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + brm.saveExtentMap(string("EMImage")); + brm.loadExtentMap(string("EMImage")); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 1; i < iterations; i++) + { + err = brm.lookup(i, 1, lbidList); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbidList.size() == 1); + + err = brm.lookup(i, 10, lbidList); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbidList.size() == 0); + + err = brm.lookup(i * 10000, 1, lbidList); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbidList.size() == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + for (i = 1; i < iterations; i++) + { + err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == 0); + CPPUNIT_ASSERT(txnID == 0); + err = brm.setBulkInsertVars(static_cast((i - 1) * extentSize), i, i + 1); + CPPUNIT_ASSERT(err == 0); + err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), hwm2, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm2 == static_cast(i)); + CPPUNIT_ASSERT(txnID == static_cast(i + 1)); + + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == 0); + err = brm.setHWM(i, (i > extentSize - 1 ? extentSize - 1 : i)); + CPPUNIT_ASSERT(err == 0); + err = brm.getHWM(i, hwm); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == static_cast(i > extentSize - 1 ? extentSize - 1 : i)); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + +#ifdef BRM_DEBUG + err = brm.setHWM(i, hwm); + CPPUNIT_ASSERT(err == -1); +#endif + + for (i = 1; i < iterations; i++) + { + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == 0); + } + + err = brm.deleteOID(i); + CPPUNIT_ASSERT(err == -1); + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + Config::deleteInstanceMap(); + } +#endif + + void brm_good_2() + { + BlockResolutionManager brm; + VBBM vbbm; + VSS vss; + CopyLocks cl; + int i, err, size; + vector lbids; + vector extents; + LBIDRange_v ranges; + LBIDRange_v::iterator lbidRangeIT; + vector vbRanges, vbRanges2; + vector::iterator vbRangesIT; + EMEntry em; + OID_t oid; + uint32_t fbo; + LBIDRange range; + VBRange vbRange; + VER_t verID; + bool vbFlag; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + // cerr << "brm_good_2" << endl; + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + err = brm.lookup(0, 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + err = brm.lookup(0, 0, true, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + err = brm.createExtent(8000, 1, lbids, size); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(size == brm.getExtentSize()); + CPPUNIT_ASSERT(lbids.size() == 1); + // CPPUNIT_ASSERT(*(lbids.begin()) == 0); + + err = brm.getExtents(1, extents); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.size() == 1); + + em = *(extents.begin()); + // CPPUNIT_ASSERT(em.range.start == 0); + CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); + CPPUNIT_ASSERT(em.HWM == 0); + CPPUNIT_ASSERT(em.blockOffset == 0); + + for (i = 0; i < 50; i++) + { + range.start = i * 100; + range.size = 100; + ranges.push_back(range); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.beginVBCopy(1, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.beginVBCopy(1, ranges, vbRanges2); + CPPUNIT_ASSERT(err == -1); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + for (i = 0; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + + vbRange = *(vbRanges.begin()); + + // CPPUNIT_ASSERT(vbRange.vbFBO == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + } + + for (; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + + err = brm.endVBCopy(0, ranges); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + brm.saveState("TestImage"); + brm.loadState("TestImage"); + + brm.vbCommit(1); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + + verID = 0; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + + err = brm.lookup(i, verID, vbFlag, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == vbRange.vbOID); + CPPUNIT_ASSERT(fbo == static_cast(i + vbRange.vbFBO)); + + vbbm.lock(VBBM::WRITE); + vss.lock(VSS::WRITE); + vbbm.removeEntry(i, verID); + // vss.removeEntry(i, 1); + vss.removeEntry(i, verID); + vbbm.confirmChanges(); + vss.confirmChanges(); + vss.release(VSS::WRITE); + vbbm.release(VBBM::WRITE); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + brm.deleteOID(1); + + vss.lock(VSS::READ); + vbbm.lock(VBBM::READ); + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + Config::deleteInstanceMap(); + } + + // cut & pasted from brm_good_2(), but with rollback instead of commit. + void brm_good_3() + { + BlockResolutionManager brm; + VBBM vbbm; + VSS vss; + CopyLocks cl; + int i, err, size; + vector lbids; + vector::iterator lbid; + vector extents; + LBIDRange_v ranges; + LBIDRange_v::iterator lbidRangeIT; + VBRange_v vbRanges, vbRanges2; + VBRange_v::iterator vbRangesIT; + LBIDRange_v tmp; + EMEntry em; + OID_t oid; + uint32_t fbo; + LBIDRange range; + VBRange vbRange; + VER_t verID; + bool vbFlag; + bool caughtException; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + // cerr << "brm_good_3" << endl; + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + err = brm.lookup(0, 0, false, oid, fbo); + CPPUNIT_ASSERT(err == -1); + err = brm.lookup(0, 0, true, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + err = brm.createExtent(8000, 1, lbids, size); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(size == brm.getExtentSize()); + CPPUNIT_ASSERT(lbids.size() == 1); + CPPUNIT_ASSERT(*(lbids.begin()) == 0); + + err = brm.getExtents(1, extents); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.size() == 1); + + em = *(extents.begin()); + CPPUNIT_ASSERT(em.range.start == 0); + CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); + CPPUNIT_ASSERT(em.HWM == 0); + CPPUNIT_ASSERT(em.blockOffset == 0); + + for (i = 0; i < 50; i++) + { + range.start = i * 100; + range.size = 100; + ranges.push_back(range); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + err = brm.beginVBCopy(1, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.beginVBCopy(1, ranges, vbRanges2); + CPPUNIT_ASSERT(err == -1); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + for (i = 0; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + + vbRange = *(vbRanges.begin()); + + // CPPUNIT_ASSERT(vbRange.vbFBO == 0); + for (i = 0; i < (int)vbRange.size; i++) + { + err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); + CPPUNIT_ASSERT(err == 0); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 1); + CPPUNIT_ASSERT(vbFlag == false); + verID = MAXINT; + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == true); + } + + for (; i < 5000; i++) + { + verID = MAXINT; + err = brm.vssLookup(i, verID, 1, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + + err = brm.endVBCopy(0, ranges); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + cl.lock(CopyLocks::READ); + + for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); lbidRangeIT++) + CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); + + cl.release(CopyLocks::READ); + + err = brm.getUncommittedLBIDs(1, lbids); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbids.size() == vbRange.size); + sort::iterator>(lbids.begin(), lbids.end()); + lbid = lbids.begin(); + + for (i = 0; i < static_cast(lbids.size()); i++, lbid++) + CPPUNIT_ASSERT((*lbid) == static_cast(i)); + + range.start = 0; + range.size = i; + tmp.push_back(range); + err = brm.vbRollback(1, tmp); + CPPUNIT_ASSERT(err == 0); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + for (i = 0; i < (int)vbRange.size; i++) + { + verID = MAXINT; + + err = brm.vssLookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == 0); + CPPUNIT_ASSERT(vbFlag == false); + + err = brm.lookup(i, verID, vbFlag, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == 1); + CPPUNIT_ASSERT(fbo == static_cast(i)); + + vbbm.lock(VBBM::WRITE); + vss.lock(VSS::WRITE); + +#ifdef BRM_DEBUG + caughtException = false; + + try + { + vbbm.removeEntry(i, verID); + vbbm.confirmChanges(); + } + catch (logic_error& e) + { + vbbm.undoChanges(); + caughtException = true; + } + + CPPUNIT_ASSERT(caughtException); + caughtException = false; + + try + { + vss.removeEntry(i, 1); + vss.confirmChanges(); + } + catch (logic_error& e) + { + vss.undoChanges(); + caughtException = true; + } + + CPPUNIT_ASSERT(caughtException); +#endif + + vss.removeEntry(i, verID); + vss.confirmChanges(); + vss.release(VSS::WRITE); + vbbm.release(VBBM::WRITE); + } + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + + brm.deleteOID(1); + + vbbm.lock(VBBM::READ); + vss.lock(VSS::READ); + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::READ); + vbbm.release(VBBM::READ); + + CPPUNIT_ASSERT(brm.checkConsistency() == 0); + Config::deleteInstanceMap(); + } + + /* This test verifies that deleteOID returns an error for + non-existant OIDs (bug #105) */ + void brm_deleteOID() + { + BlockResolutionManager brm; + int err; + vector extents; + + // cerr << "brm_deleteOID" << endl; + + err = brm.getExtents(1, extents); +#ifdef BRM_DEBUG + + if (err != -1) + cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; + + CPPUNIT_ASSERT(err == -1); +#else + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(extents.empty()); +#endif + err = brm.deleteOID(1); + CPPUNIT_ASSERT(err == -1); + Config::deleteInstanceMap(); + } + + /* This test verifies that setHWM and getHWM return an error for + nonexistant OIDs (bugs #106, 107) */ + + void brm_HWM() + { + BlockResolutionManager brm; + int err; + HWM_t hwm; + vector extents; + + // cerr << "brm_HWM" << endl; + + err = brm.getExtents(1, extents); +#ifdef BRM_DEBUG + + if (err != -1) + cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; + + CPPUNIT_ASSERT(err == -1); +#else + CPPUNIT_ASSERT(err == 0); +#endif + CPPUNIT_ASSERT(extents.size() == 0); + err = brm.setHWM(1, 10); + CPPUNIT_ASSERT(err == -1); + err = brm.getHWM(1, hwm); + CPPUNIT_ASSERT(err == -1); + Config::deleteInstanceMap(); + } + + void vbbm_good_1() + { + // add some VBBM entries + // do some lookups + // manipulate them + // do more lookups + + VBBM vbbm; + VSS vss; + vector ranges; + vector::iterator it; + int i, iterations = 100010, err; + OID_t oid; + uint32_t fbo, fbo2; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + // cerr << "vbbm_good_1" << endl; + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + // cerr << "\nVBBM test (this can take awhile)\n"; + vbbm.lock(VBBM::READ); + vbbm.release(VBBM::READ); + vbbm.lock(VBBM::WRITE); + vbbm.release(VBBM::WRITE); + + vbbm.lock(VBBM::WRITE); + + err = vbbm.lookup(0, 0, oid, fbo); + CPPUNIT_ASSERT(err == -1); + + // cerr << "step 1/4: get some VB blocks" << endl; + + vss.lock(VSS::WRITE); + vbbm.getBlocks(iterations, ranges, vss); + vbbm.confirmChanges(); + vss.confirmChanges(); + vss.release(VSS::WRITE); + CPPUNIT_ASSERT(!ranges.empty()); + + // cerr << "step 2/4: inserts & lookups" << endl; + for (it = ranges.begin(), i = 0; it != ranges.end(); it++) + { + for (fbo = (*it).vbFBO; fbo < (*it).vbFBO + (*it).size; fbo++, i++) + { + vbbm.insert(i, i + 1, (*it).vbOID, fbo); + vbbm.confirmChanges(); + err = vbbm.lookup(i, i + 1, oid, fbo2); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == (*it).vbOID); + CPPUNIT_ASSERT(fbo2 == fbo); + } + } + + /* + for (i = 0; i < iterations; i++) { + + vbbm.insert(i, i+1, i+2, i+3); + err = vbbm.lookup(i, i+1, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i + 2); + CPPUNIT_ASSERT(fbo == static_cast(i + 3)); + } + */ + + vbbm.save(string("VBBMImage")); + vbbm.load(string("VBBMImage")); + + // cerr << "step 3/4: lookups" << endl; + for (it = ranges.begin(), i = 0; it != ranges.end(); it++) + { + for (fbo = (*it).vbFBO; fbo < (*it).vbFBO + (*it).size; fbo++, i++) + { + err = vbbm.lookup(i, i + 1, oid, fbo2); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == (*it).vbOID); + CPPUNIT_ASSERT(fbo2 == fbo); + } + } + + /* + for (i = 0; i < iterations; i++) { + err = vbbm.lookup(i, i+1, oid, fbo); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == i + 2); + CPPUNIT_ASSERT(fbo == static_cast(i + 3)); + } */ + + // cerr << "step 4/4: remove all entries" << endl; + for (i = 0; i < iterations; i++) + { + vbbm.removeEntry(i, i + 1); + vbbm.confirmChanges(); + } + + CPPUNIT_ASSERT(vbbm.size() == 0); + CPPUNIT_ASSERT(vbbm.hashEmpty()); + + vbbm.release(VBBM::WRITE); + Config::deleteInstanceMap(); + } + + void vss_good_1() + { + VSS vss; + VBBM vbbm; + ExtentMap em; + int i, err, iterations = 100010; + VER_t verID; + bool vbFlag; + vector lbids; + LBIDRange range; + + // Buildbot times out on the getBlocks() call during leakcheck b/c it takes + // > 5 mins for some reason. Have to ping it before 300 seconds go by. + void (*oldsig)(int); + + // cerr << "vss_good_1" << endl; + + oldsig = signal(SIGALRM, keepalive); + alarm(290); + + cerr << endl << "VSS test (this can take awhile)" << endl; + + vss.lock(VSS::READ); + vss.release(VSS::READ); + + vss.lock(VSS::WRITE); + + cerr << "step 1/5: insert 2 entries for 100010 LBIDs & test lookup logic" << endl; + + range.start = 0; + range.size = 200000; + + CPPUNIT_ASSERT(!vss.isLocked(range)); + + for (i = 0; i < iterations; i++) + { + vss.insert(i, i + 1, (i % 2 ? true : false), (i % 2 ? false : true)); + vss.insert(i, i, true, false); + vss.confirmChanges(); + + if (i == 0) + { + range.start = 0; + range.size = 1; + CPPUNIT_ASSERT(vss.isLocked(range)); + } + else if (i == 1) + { + range.start = 1; + CPPUNIT_ASSERT(!vss.isLocked(range)); + } + + verID = i + 10; + err = vss.lookup(i, verID, i + 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == i + 1); + CPPUNIT_ASSERT((vbFlag && (i % 2)) || (!vbFlag && !(i % 2))); + verID = i + 10; + err = vss.lookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == (i % 2 ? i + 1 : i)); + CPPUNIT_ASSERT(vbFlag == true); + + if (i > 0) + { + verID = i - 1; + err = vss.lookup(i, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == -1); + } + } + + range.start = 0; + range.size = 200000; + CPPUNIT_ASSERT(vss.isLocked(range)); + vss.save(string("VSSImage")); + vss.load(string("VSSImage")); + + // this loop actually breaks the locked -> !vbFlag invariant + // switch it back afterward + cerr << "step 2/5: flip the vbFlag on half of the entries & test lookup logic" << endl; + + for (i = 0; i < iterations; i++) + { + vss.setVBFlag(i, i + 1, (i % 2 ? false : true)); + vss.confirmChanges(); + } + + for (i = 0; i < iterations; i++) + { + verID = i + 10; + err = vss.lookup(i, verID, i + 1, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == i + 1); + CPPUNIT_ASSERT((!vbFlag && (i % 2)) || (vbFlag && !(i % 2))); + } + + cerr << "step 3/5: some clean up" << endl; + + for (i = 0; i < iterations; i++) + { + vss.setVBFlag(i, i + 1, (i % 2 ? true : false)); + vss.confirmChanges(); + } + + // there are 2 entries for every txnid at this point, one + // which is locked, one which isn't. That's technically a violation, + // so we have to get rid of the unlocked ones for the next test. + for (i = 0; i < iterations; i++) + { + vss.removeEntry(i, i); + vss.confirmChanges(); + } + + // speed up the next step! + for (i = iterations / 50; i < iterations; i++) + { + vss.removeEntry(i, i + 1); + vss.confirmChanges(); + } + + cerr << "step 4/5: get \'uncommitted\' LBIDs, commit, and test lookup logic" << endl; + + for (i = 1; i < iterations / 50; i++) + { + bool ex = false; + + if (i % 2 == 0) + { +#ifdef BRM_DEBUG + + try + { + vss.getUncommittedLBIDs(i, lbids); + } + catch (logic_error& e) + { + ex = true; + } + + CPPUNIT_ASSERT(ex == true); +#endif + } + else + { + LBID_t lbid; + + vss.getUncommittedLBIDs(i, lbids); + CPPUNIT_ASSERT(lbids.size() == 1); + lbid = *(lbids.begin()); + CPPUNIT_ASSERT(lbid == i - 1); + verID = i + 10; + err = vss.lookup(i - 1, verID, i, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == i); + CPPUNIT_ASSERT(vbFlag == false); + + err = vss.lookup(i - 1, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == -1); + + vss.commit(i); + vss.confirmChanges(); + verID = i + 10; + err = vss.lookup(i - 1, verID, 0, vbFlag); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(verID == i); + CPPUNIT_ASSERT(vbFlag == false); + } + } + + cerr << "step 5/5: final clean up" << endl; + + CPPUNIT_ASSERT(vss.size() == iterations / 50); + range.start = 0; + range.size = iterations / 50; + vss.removeEntriesFromDB(range, vbbm, false); + vbbm.confirmChanges(); + vss.confirmChanges(); + + // the new logic for removeEntriesFromDB deletes all entries in the range, + // locked or not, in the VB or not + + // CPPUNIT_ASSERT(vss.size() == iterations/100); + + // for (i = 1; i < iterations/50; i+=2) + // vss.removeEntry(i, i+1); + + CPPUNIT_ASSERT(vss.size() == 0); + CPPUNIT_ASSERT(vss.hashEmpty()); + vss.release(VSS::WRITE); + + alarm(0); + signal(SIGALRM, oldsig); + Config::deleteInstanceMap(); + } + + /* This test was suggested by Jean, who observed some odd behavior at around + 1032 EM instances in her code. This test creates 2000 instances and uses 1400 of them + to do the extentmap_freelist_1 test above. */ + void many_ExtentMap_instances() + { + ExtentMap** ems; + int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) + vector lbids; + uint32_t extentSize; + + // cerr << endl << "Extent Map instance test" << endl; + + ems = new ExtentMap*[2000]; + + for (i = 0; i < 2000; i++) + ems[i] = new ExtentMap(); + + extentSize = ems[0]->getExtentSize(); + + // cerr << " - Successfully created 2000 instances, starting the freelist test" + // << endl; + for (i = 1; i < iterations; i++) + { + ems[i]->createExtent(extentSize, i, lbids, allocdSize); + ems[i]->confirmChanges(); + CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1) * extentSize)); + } + + // frag the lbid space to blow up the free list + for (i = 1; i < iterations; i += 2) + { + ems[i]->deleteOID(i); + ems[i]->confirmChanges(); + } + + // fill in the holes + for (i = 1; i < iterations; i += 2) + { + ems[i]->createExtent(extentSize, i, lbids, allocdSize); + ems[i]->confirmChanges(); + } + + for (i = 1; i < iterations; i += 2) + { + ems[i]->deleteOID(i); + ems[i]->confirmChanges(); + } + + for (i = 2; i < iterations; i += 2) + { + ems[i]->deleteOID(i); + ems[i]->confirmChanges(); + } + + // cerr << "done. sleeping for 30 seconds." << endl; + // sleep(30); + + for (i = 0; i < 2000; i++) + delete ems[i]; + + delete[] ems; + Config::deleteInstanceMap(); + } + + void resource_graph_1() + { + LBIDResourceGraph rg; + pthread_t t; + VER_t txn = 1; + int err; + + pthread_mutex_init(&mutex, NULL); + pthread_mutex_lock(&mutex); + + err = rg.reserveRange(1001, 2000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + err = rg.reserveRange(2001, 2000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + err = rg.reserveRange(0, 1000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + err = rg.reserveRange(3001, 2000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + pthread_create(&t, NULL, dummy_1, &rg); + pthread_mutex_unlock(&mutex); + sleep(5); + + // thread should sleep here + + pthread_mutex_lock(&mutex); + rg.releaseResources(txn); + pthread_mutex_unlock(&mutex); + + // thread should be unblocked and finish here + + pthread_join(t, NULL); + pthread_mutex_lock(&mutex); + err = rg.reserveRange(0, 1000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + rg.releaseResources(txn); + pthread_mutex_unlock(&mutex); + pthread_mutex_destroy(&mutex); + Config::deleteInstanceMap(); + } + + void resource_graph_deadlock() + { + LBIDResourceGraph rg; + pthread_t t; + VER_t txn = 1; + int err; + + pthread_mutex_init(&mutex, NULL); + pthread_mutex_lock(&mutex); + err = rg.reserveRange(0, 1000, txn, mutex); + CPPUNIT_ASSERT(err == 0); + pthread_create(&t, NULL, dummy_deadlock, &rg); + pthread_mutex_unlock(&mutex); + sleep(1); + + // thread grabs 1001-2000, then 0-1000 and blocks + + pthread_mutex_lock(&mutex); + err = rg.reserveRange(1001, 2000, txn, mutex); // deadlock + CPPUNIT_ASSERT(err == ERR_DEADLOCK); + rg.releaseResources(txn); + + pthread_mutex_unlock(&mutex); // should wake the thread + pthread_join(t, NULL); // wait for it to finish + + pthread_mutex_lock(&mutex); + err = rg.reserveRange(0, 2000, txn, mutex); // verify we can now grab both ranges + CPPUNIT_ASSERT(err == ERR_OK); + pthread_mutex_unlock(&mutex); + pthread_mutex_destroy(&mutex); + Config::deleteInstanceMap(); + } + + void resource_graph_fragd_range() + { + LBIDResourceGraph rg; + set >* resources; + map* txns; + pthread_t t; + VER_t txn = 1; + int err; + + pthread_mutex_init(&mutex, NULL); + pthread_mutex_lock(&mutex); + + err = rg.reserveRange(1001, 2000, txn, mutex); + CPPUNIT_ASSERT(err == ERR_OK); + err = rg.reserveRange(3001, 4000, txn, mutex); + CPPUNIT_ASSERT(err == ERR_OK); + pthread_create(&t, NULL, fragd_range_helper, &rg); + pthread_mutex_unlock(&mutex); + sleep(1); + + // thread wakes, tries to grab 0-5000 + // check that there are 5 resource nodes + + pthread_mutex_lock(&mutex); + resources = rg.getResources(); + txns = rg.getTxns(); + CPPUNIT_ASSERT(resources->size() == 5); + CPPUNIT_ASSERT(txns->size() == 2); + rg.releaseResources(txn); + CPPUNIT_ASSERT(resources->size() == 3); + CPPUNIT_ASSERT(txns->size() == 1); + pthread_mutex_unlock(&mutex); + + // thread releases, exits + + pthread_join(t, NULL); + CPPUNIT_ASSERT(resources->size() == 0); + CPPUNIT_ASSERT(txns->size() == 0); + + pthread_mutex_destroy(&mutex); + Config::deleteInstanceMap(); + } + + void resource_graph_overlapping_ranges() + { + LBIDResourceGraph rg; set >* resources; set >::const_iterator it; map* txns; + pthread_t t; + VER_t txn = 1; int err; - VER_t txn = 2; + + // make a fragmented range, then request a range that overlaps existing + // ranges on both ends + + pthread_mutex_init(&mutex, NULL); + pthread_mutex_lock(&mutex); + + err = rg.reserveRange(1001, 2000, txn, mutex); + CPPUNIT_ASSERT(err == ERR_OK); + err = rg.reserveRange(3001, 4000, txn, mutex); + CPPUNIT_ASSERT(err == ERR_OK); + pthread_create(&t, NULL, overlapping_range_helper, &rg); + pthread_mutex_unlock(&mutex); + sleep(1); + + // thread executes, tries to grab 1100-3900 pthread_mutex_lock(&mutex); - resources = rg->getResources(); - txns = rg->getTxns(); - CPPUNIT_ASSERT(resources->size() == 2); - CPPUNIT_ASSERT(txns->size() == 1); - err = rg->reserveRange(1100, 3900, txn, mutex); + resources = rg.getResources(); + txns = rg.getTxns(); - // blocks, main thread releases +#ifdef BRM_VERBOSE + cerr << " resources->size = " << resources->size() << endl; + + for (it = resources->begin(); it != resources->end(); it++) + cerr << " " << (*it)->start() << "-" << (*it)->end() << endl; + +#endif - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(resources->size() == 3); it = resources->begin(); - CPPUNIT_ASSERT((*it)->start() == 1100); + CPPUNIT_ASSERT(resources->size() == 3); + CPPUNIT_ASSERT((*it)->start() == 1001); CPPUNIT_ASSERT((*it)->end() == 2000); it++; CPPUNIT_ASSERT((*it)->start() == 2001); CPPUNIT_ASSERT((*it)->end() == 3000); it++; CPPUNIT_ASSERT((*it)->start() == 3001); - CPPUNIT_ASSERT((*it)->end() == 3900); + CPPUNIT_ASSERT((*it)->end() == 4000); + + CPPUNIT_ASSERT(txns->size() == 2); + rg.releaseResources(txn); CPPUNIT_ASSERT(txns->size() == 1); - rg->releaseResources(txn); pthread_mutex_unlock(&mutex); - return NULL; -} - -class BRMTest : public CppUnit::TestFixture -{ - - CPPUNIT_TEST_SUITE(BRMTest); - CPPUNIT_TEST(masterSegmentTest_goodbad); - CPPUNIT_TEST(extentMap_good_1); -#ifdef PARTITIONING - CPPUNIT_TEST(extentMap_range_1); -#endif - CPPUNIT_TEST(extentMap_freelist); -// CPPUNIT_TEST(extentMap_overfill); -//CPPUNIT_TEST(many_ExtentMap_instances); //Jean unsuggested this case since writeengine use singleton - CPPUNIT_TEST(copyLocks_good_1); - CPPUNIT_TEST(vbbm_good_1); - CPPUNIT_TEST(vbbm_good_1); - CPPUNIT_TEST(vss_good_1); - CPPUNIT_TEST(vss_good_1); -//CPPUNIT_TEST(brm_extentmap_good_1);//fix for bug 640, this test case has a bug -#ifdef PARTITIONING - CPPUNIT_TEST(brm_extentmap_range_1); -#endif - CPPUNIT_TEST(brm_good_2); - CPPUNIT_TEST(brm_good_3); - CPPUNIT_TEST(brm_deleteOID); - CPPUNIT_TEST(brm_HWM); - CPPUNIT_TEST(resource_graph_1); - CPPUNIT_TEST(resource_graph_fragd_range); - CPPUNIT_TEST(resource_graph_deadlock); - CPPUNIT_TEST(resource_graph_overlapping_ranges); - - CPPUNIT_TEST_SUITE_END(); - -private: -public: - - void masterSegmentTest_goodbad() - { - MasterSegmentTable mst; - struct MSTEntry* mt; - -// cerr << "MST goodbad" << endl; - - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt == NULL); - - mst.releaseTable_write(MasterSegmentTable::EMTable); - mst.releaseTable_write(MasterSegmentTable::EMFreeList); - mst.releaseTable_write(MasterSegmentTable::VBBMSegment); - - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::EMFreeList, false); - CPPUNIT_ASSERT(mt == NULL); - mt = mst.getTable_write(MasterSegmentTable::VBBMSegment, false); - CPPUNIT_ASSERT(mt == NULL); - mst.releaseTable_write(MasterSegmentTable::EMTable); - mst.releaseTable_write(MasterSegmentTable::EMFreeList); - mst.releaseTable_write(MasterSegmentTable::VBBMSegment); - - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mst.releaseTable_read(MasterSegmentTable::EMTable); - mst.releaseTable_read(MasterSegmentTable::EMTable); - mst.releaseTable_read(MasterSegmentTable::EMTable); - - mt = mst.getTable_write(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt != NULL); - mt = mst.getTable_read(MasterSegmentTable::EMTable, false); - CPPUNIT_ASSERT(mt == NULL); - mst.releaseTable_write(MasterSegmentTable::EMTable); - Config::deleteInstanceMap(); - } - - void extentMap_good_1() - { - ExtentMap em; - int i, err, oid, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - int caughtException = 0, allocdSize; - uint32_t fbo, hwm; - BRM::HWM_t hwm2; - BRM::VER_t txnID; - vector lbids; - const uint32_t extentSize = em.getExtentSize(); - -// cerr << "em_good_1" << endl; - - for (i = 1; i < iterations; i++) - { - em.createExtent(extentSize, i, lbids, allocdSize); - em.confirmChanges(); - CPPUNIT_ASSERT(lbids.size() == 1); - CPPUNIT_ASSERT((uint32_t)allocdSize == extentSize); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - } - - em.checkConsistency(); - em.save(string("EMImage")); - em.load(string("EMImage")); - em.checkConsistency(); - - for (i = 1; i < iterations; i++) - { - err = em.lookup(static_cast((i - 1) * extentSize), oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 0); - - if (i != 1) - { - err = em.lookup(static_cast((i - 1) * extentSize - 1), oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i - 1); - CPPUNIT_ASSERT(fbo == extentSize - 1); - } - - if (i != iterations - 1) - { - err = em.lookup(static_cast((i - 1) * extentSize + 1), oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 1); - } - } - - em.checkConsistency(); - - err = em.lookup(static_cast(i * extentSize), oid, fbo); - CPPUNIT_ASSERT(err == -1); - - for (i = 1; i < iterations; i++) - { - err = em.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == 0); - CPPUNIT_ASSERT(txnID == 0); - err = em.setBulkInsertVars(static_cast((i - 1) * extentSize), - i, i + 1); - em.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - err = em.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == static_cast(i)); - CPPUNIT_ASSERT(txnID == static_cast(i + 1)); - - hwm = em.getHWM(i); - CPPUNIT_ASSERT(hwm == 0); - em.setHWM(i, ((uint32_t)i > (extentSize - 1) ? extentSize - 1 : i)); - em.confirmChanges(); - hwm = em.getHWM(i); - CPPUNIT_ASSERT(hwm == static_cast((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); - } - - em.checkConsistency(); - -#ifdef BRM_DEBUG - caughtException = 0; - - try - { - em.setHWM(i, hwm); - } - catch (std::invalid_argument e) - { - caughtException = 1; - } - - em.undoChanges(); - CPPUNIT_ASSERT(caughtException == 1); -#endif - - for (i = 1; i < iterations; i++) - { - em.deleteOID(i); - em.confirmChanges(); - } - -#ifdef BRM_DEBUG - caughtException = 0; - - try - { - em.deleteOID(i); - } - catch (std::invalid_argument& e) - { - caughtException = 1; - } - - em.undoChanges(); - CPPUNIT_ASSERT(caughtException == 1); -#endif - - em.checkConsistency(); - Config::deleteInstanceMap(); - } - -#ifdef PARTITIONING - void extentMap_range_1() - { - ExtentMap em; - int i, err, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - int caughtException = 0, allocdSize; - uint32_t hwm; - BRM::HWM_t hwm2; - BRM::VER_t txnID; - vector lbids; - const uint32_t extentSize = em.getExtentSize(); - - for (i = 1; i < iterations; i++) - { - em.createRangeExtent(extentSize, i, lbids, allocdSize, EMRangePartition_t(0, 10, true, true)); - em.confirmChanges(); - CPPUNIT_ASSERT(lbids.size() == 1); - CPPUNIT_ASSERT((uint32_t)allocdSize == extentSize); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - } - - em.checkConsistency(); - em.save(string("EMImage")); - em.load(string("EMImage")); - em.checkConsistency(); - - LBIDRange_v lbidList; - - for (i = 1; i < iterations; i++) - { - - err = em.lookup(i, 1, lbidList); - CPPUNIT_ASSERT(lbidList.size() == 1); - CPPUNIT_ASSERT(err == lbidList.size()); - - err = em.lookup(i, i + 10, lbidList); - CPPUNIT_ASSERT(lbidList.size() == 0); - CPPUNIT_ASSERT(err == lbidList.size()); - - err = em.lookup(i * 10000, 1, lbidList); - CPPUNIT_ASSERT(lbidList.size() == 0); - CPPUNIT_ASSERT(err == lbidList.size()); - } - - em.checkConsistency(); - - err = em.lookup(i, 99, lbidList); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbidList.size() == 0); - -#ifdef BRM_DEBUG - caughtException = 0; - - try - { - em.setHWM(i, hwm); - } - catch (std::invalid_argument e) - { - caughtException = 1; - } - - em.undoChanges(); - CPPUNIT_ASSERT(caughtException == 1); -#endif - - for (i = 1; i < iterations; i++) - { - em.deleteOID(i); - em.confirmChanges(); - } - -#ifdef BRM_DEBUG - caughtException = 0; - - try - { - em.deleteOID(i); - } - catch (std::invalid_argument& e) - { - caughtException = 1; - } - - em.undoChanges(); - CPPUNIT_ASSERT(caughtException == 1); -#endif - - em.checkConsistency(); - Config::deleteInstanceMap(); - } -#endif - - void extentMap_freelist() - { - ExtentMap em; - int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) - vector lbids; - const int extentSize = em.getExtentSize(); - -// cerr << "em_freelist" << endl; - - for (i = 1; i < iterations; i++) - { - em.createExtent(extentSize, i, lbids, allocdSize); - em.confirmChanges(); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - } - - em.checkConsistency(); - - //frag the lbid space to blow up the free list - for (i = 1; i < iterations; i += 2) - { - em.deleteOID(i); - em.confirmChanges(); - } - - em.checkConsistency(); - - //fill in the holes - for (i = 1; i < iterations; i += 2) - { - em.createExtent(extentSize, i, lbids, allocdSize); - em.confirmChanges(); - } - - for (i = 1; i < iterations; i += 2) - { - em.deleteOID(i); - em.confirmChanges(); - } - - for (i = 2; i < iterations; i += 2) - { - em.deleteOID(i); - em.confirmChanges(); - } - - em.checkConsistency(); - Config::deleteInstanceMap(); - } - - /* This test verifies that the ExtentMap stops allocating space when it's full. - (Bug #104) It takes a long time to run with the default LBID - space characteristics so by default it's disabled.*/ - void extentMap_overfill() - { - int i, err, tmp; - BlockResolutionManager brm; - vector lbids; - - for (i = 1; i < 67109; i++) - { - err = brm.createExtent(1024000, i, lbids, tmp); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.createExtent(1024000, i, lbids, tmp); - CPPUNIT_ASSERT(err == -1); - - for (i = 1; i < 67109; i++) - { - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == -1); - Config::deleteInstanceMap(); - } - - void copyLocks_good_1() - { - CopyLocks cl; - LBIDRange range; - int i, iterations = 1000; - -// cerr << "copylocks_good_1" << endl; - - cl.lock(CopyLocks::WRITE); - range.start = 5; - range.size = 2; - cl.lockRange(range, 0); - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.start++; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.start++; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - range.start = 0; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - range.start = 3; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - range.start++; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.start = 5; - range.size = 1; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - range.size = 2; - cl.releaseRange(range); - - //make sure it can grow - for (i = 0; i < iterations * 2; i += 2) - { - range.start = i; - cl.lockRange(range, 0); - } - - range.size = 1; - - cl.save(string("CLImage")); - cl.load(string("CLImage")); - - for (i = 0; i < iterations * 2; i++) - { - range.start = i; - CPPUNIT_ASSERT(cl.isLocked(range) == true); - } - - range.start = i; - CPPUNIT_ASSERT(cl.isLocked(range) == false); - - for (i = 0; i < iterations * 2; i += 2) - { - range.start = i; - cl.releaseRange(range); - } - - cl.release(CopyLocks::WRITE); - Config::deleteInstanceMap(); - } - - void brm_extentmap_good_1() - { - - BlockResolutionManager brm; - int i, err, oid, allocdSize, - iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - uint32_t fbo, hwm; - vector lbids; - HWM_t hwm2; - VER_t txnID; - const uint32_t extentSize = brm.getExtentSize(); - -// cerr << "brm_extentmap_good_1" << endl; - - for (i = 1; i < iterations; i++) - { - brm.createExtent(extentSize, i, lbids, allocdSize); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - brm.saveExtentMap(string("EMImage")); - brm.loadExtentMap(string("EMImage")); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - ExtentMap em; - - //int32_t seqNum=0; - //int64_t max=0; - //int64_t min=0; - LBID_t lbid; - /* - for (i = 1; i < iterations; i++) { - lbid=(i-1)*extentSize; - err=em.updateMaxMin(lbid, 1, 0, seqNum); // sequenceNum=0 - em.confirmChanges(); - CPPUNIT_ASSERT(err==0); - - max=-1; - min=-1; - err=em.getMaxMin(lbid, max, min, seqNum); - CPPUNIT_ASSERT(max==1); - CPPUNIT_ASSERT(min==0); - CPPUNIT_ASSERT(err==2); - CPPUNIT_ASSERT(seqNum==0); - - err=em.updateMaxMin(lbid, 4, 2, seqNum); // sequenceNum=0 - em.confirmChanges(); - CPPUNIT_ASSERT(err==0); - - max=-1; - min=-1; - seqNum++; - err=em.getMaxMin(lbid, max, min, seqNum); - CPPUNIT_ASSERT(max==4); - CPPUNIT_ASSERT(min==2); - CPPUNIT_ASSERT(err==2); - CPPUNIT_ASSERT(seqNum==0); - - err=em.markInvalid(lbid); // sequenceNum=1, valid=1 - em.confirmChanges(); - CPPUNIT_ASSERT(err==0); - - seqNum=0; // re-init seqNum - err=em.getMaxMin(lbid, max, min, seqNum); - CPPUNIT_ASSERT(err==1); - CPPUNIT_ASSERT(seqNum==1); - - seqNum=0; - } - */ - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - vector lbidVector; - uint32_t offset; - - for (i = 1; i < iterations; i++) - { - lbid = (i - 1) * extentSize; - - for (int j = 0; (uint32_t)j < extentSize; j++) - lbidVector.push_back(lbid + j); - - err = em.markInvalid(lbidVector); - em.confirmChanges(); - CPPUNIT_ASSERT(err == 1); - lbidVector.clear(); - em.lookup(lbid, oid, offset); - err = em.markInvalid(oid); - em.confirmChanges(); - CPPUNIT_ASSERT(err == 0); - //CPPUNIT_ASSERT(oid==i); - oid = -1; - offset = 0; - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 1; i < iterations; i++) - { - err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 0); - - if (i != 1) - { - err = brm.lookup(static_cast((i - 1) * extentSize - 1), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i - 1); - CPPUNIT_ASSERT(fbo == extentSize - 1); - } - - if (i != iterations) - { - err = brm.lookup(static_cast((i - 1) * extentSize + 1), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i); - CPPUNIT_ASSERT(fbo == 1); - } - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - - for (i = 1; i < iterations; i++) - { - err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == 0); - CPPUNIT_ASSERT(txnID == 0); - err = brm.setBulkInsertVars(static_cast((i - 1) * extentSize), - i, i + 1); - CPPUNIT_ASSERT(err == 0); - err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == static_cast(i)); - CPPUNIT_ASSERT(txnID == static_cast(i + 1)); - - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == 0); - err = brm.setHWM(i, ((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); - CPPUNIT_ASSERT(err == 0); - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == static_cast((uint32_t)i > extentSize - 1 ? extentSize - 1 : i)); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - -#ifdef BRM_DEBUG - err = brm.setHWM(i, hwm); - CPPUNIT_ASSERT(err == -1); -#endif - - for (i = 1; i < iterations; i++) - { - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == -1); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - Config::deleteInstanceMap(); - } - -#ifdef PARTITIONING - void brm_extentmap_range_1() - { - - BlockResolutionManager brm; - int i, - err, - oid, - allocdSize, - iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) - uint32_t fbo, hwm; - vector lbids; - HWM_t hwm2; - VER_t txnID; - const uint32_t extentSize = brm.getExtentSize(); - LBIDRange_v lbidList; - -// cerr << "brm_extentmap_range_1" << endl; - - for (i = 1; i < iterations; i++) - { - brm.createRangeExtent(extentSize, i, lbids, allocdSize, - EMRangePartition_t(0, 9, true, true) ); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - CPPUNIT_ASSERT(lbids.size() == 1); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - brm.saveExtentMap(string("EMImage")); - brm.loadExtentMap(string("EMImage")); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 1; i < iterations; i++) - { - - err = brm.lookup(i, 1, lbidList); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbidList.size() == 1); - - err = brm.lookup(i, 10, lbidList); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbidList.size() == 0); - - err = brm.lookup(i * 10000, 1, lbidList); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbidList.size() == 0); - - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.lookup(static_cast((i - 1) * extentSize), 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - - for (i = 1; i < iterations; i++) - { - err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == 0); - CPPUNIT_ASSERT(txnID == 0); - err = brm.setBulkInsertVars(static_cast((i - 1) * extentSize), - i, i + 1); - CPPUNIT_ASSERT(err == 0); - err = brm.getBulkInsertVars(static_cast((i - 1) * extentSize), - hwm2, txnID); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm2 == static_cast(i)); - CPPUNIT_ASSERT(txnID == static_cast(i + 1)); - - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == 0); - err = brm.setHWM(i, (i > extentSize - 1 ? extentSize - 1 : i)); - CPPUNIT_ASSERT(err == 0); - err = brm.getHWM(i, hwm); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(hwm == static_cast(i > extentSize - 1 ? extentSize - 1 : i)); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - -#ifdef BRM_DEBUG - err = brm.setHWM(i, hwm); - CPPUNIT_ASSERT(err == -1); -#endif - - for (i = 1; i < iterations; i++) - { - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == 0); - } - - err = brm.deleteOID(i); - CPPUNIT_ASSERT(err == -1); - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - Config::deleteInstanceMap(); - } -#endif - - void brm_good_2() - { - BlockResolutionManager brm; - VBBM vbbm; - VSS vss; - CopyLocks cl; - int i, err, size; - vector lbids; - vector extents; - LBIDRange_v ranges; - LBIDRange_v::iterator lbidRangeIT; - vector vbRanges, vbRanges2; - vector::iterator vbRangesIT; - EMEntry em; - OID_t oid; - uint32_t fbo; - LBIDRange range; - VBRange vbRange; - VER_t verID; - bool vbFlag; - - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - -// cerr << "brm_good_2" << endl; - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - - err = brm.lookup(0, 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - err = brm.lookup(0, 0, true, oid, fbo); - CPPUNIT_ASSERT(err == -1); - - err = brm.createExtent(8000, 1, lbids, size); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(size == brm.getExtentSize()); - CPPUNIT_ASSERT(lbids.size() == 1); -// CPPUNIT_ASSERT(*(lbids.begin()) == 0); - - err = brm.getExtents(1, extents); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.size() == 1); - - em = *(extents.begin()); -// CPPUNIT_ASSERT(em.range.start == 0); - CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); - CPPUNIT_ASSERT(em.HWM == 0); - CPPUNIT_ASSERT(em.blockOffset == 0); - - for (i = 0; i < 50; i++) - { - range.start = i * 100; - range.size = 100; - ranges.push_back(range); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.beginVBCopy(1, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.beginVBCopy(1, ranges, vbRanges2); - CPPUNIT_ASSERT(err == -1); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - for (i = 0; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - vbRange = *(vbRanges.begin()); - -// CPPUNIT_ASSERT(vbRange.vbFBO == 0); - for (i = 0; i < (int)vbRange.size; i++) - { - err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); - CPPUNIT_ASSERT(err == 0); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - } - - for (; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - err = brm.endVBCopy(0, ranges); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - brm.saveState("TestImage"); - brm.loadState("TestImage"); - - brm.vbCommit(1); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - - verID = 0; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - - err = brm.lookup(i, verID, vbFlag, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == vbRange.vbOID); - CPPUNIT_ASSERT(fbo == static_cast(i + vbRange.vbFBO)); - - vbbm.lock(VBBM::WRITE); - vss.lock(VSS::WRITE); - vbbm.removeEntry(i, verID); -// vss.removeEntry(i, 1); - vss.removeEntry(i, verID); - vbbm.confirmChanges(); - vss.confirmChanges(); - vss.release(VSS::WRITE); - vbbm.release(VBBM::WRITE); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - brm.deleteOID(1); - - vss.lock(VSS::READ); - vbbm.lock(VBBM::READ); - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - Config::deleteInstanceMap(); - } - - // cut & pasted from brm_good_2(), but with rollback instead of commit. - void brm_good_3() - { - BlockResolutionManager brm; - VBBM vbbm; - VSS vss; - CopyLocks cl; - int i, err, size; - vector lbids; - vector::iterator lbid; - vector extents; - LBIDRange_v ranges; - LBIDRange_v::iterator lbidRangeIT; - VBRange_v vbRanges, vbRanges2; - VBRange_v::iterator vbRangesIT; - LBIDRange_v tmp; - EMEntry em; - OID_t oid; - uint32_t fbo; - LBIDRange range; - VBRange vbRange; - VER_t verID; - bool vbFlag; - bool caughtException; - - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - -// cerr << "brm_good_3" << endl; - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - - err = brm.lookup(0, 0, false, oid, fbo); - CPPUNIT_ASSERT(err == -1); - err = brm.lookup(0, 0, true, oid, fbo); - CPPUNIT_ASSERT(err == -1); - - err = brm.createExtent(8000, 1, lbids, size); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(size == brm.getExtentSize()); - CPPUNIT_ASSERT(lbids.size() == 1); - CPPUNIT_ASSERT(*(lbids.begin()) == 0); - - err = brm.getExtents(1, extents); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.size() == 1); - - em = *(extents.begin()); - CPPUNIT_ASSERT(em.range.start == 0); - CPPUNIT_ASSERT(em.range.size * 1024 == static_cast(brm.getExtentSize())); - CPPUNIT_ASSERT(em.HWM == 0); - CPPUNIT_ASSERT(em.blockOffset == 0); - - for (i = 0; i < 50; i++) - { - range.start = i * 100; - range.size = 100; - ranges.push_back(range); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - err = brm.beginVBCopy(1, ranges, vbRanges); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.beginVBCopy(1, ranges, vbRanges2); - CPPUNIT_ASSERT(err == -1); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - for (i = 0; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - vbRange = *(vbRanges.begin()); - -// CPPUNIT_ASSERT(vbRange.vbFBO == 0); - for (i = 0; i < (int)vbRange.size; i++) - { - err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i); - CPPUNIT_ASSERT(err == 0); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 1); - CPPUNIT_ASSERT(vbFlag == false); - verID = MAXINT; - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == true); - } - - for (; i < 5000; i++) - { - verID = MAXINT; - err = brm.vssLookup(i, verID, 1, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - err = brm.endVBCopy(0, ranges); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - cl.lock(CopyLocks::READ); - - for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end(); - lbidRangeIT++) - CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT)); - - cl.release(CopyLocks::READ); - - err = brm.getUncommittedLBIDs(1, lbids); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(lbids.size() == vbRange.size); - sort::iterator>(lbids.begin(), lbids.end()); - lbid = lbids.begin(); - - for (i = 0; i < static_cast(lbids.size()); i++, lbid++) - CPPUNIT_ASSERT((*lbid) == static_cast(i)); - - range.start = 0; - range.size = i; - tmp.push_back(range); - err = brm.vbRollback(1, tmp); - CPPUNIT_ASSERT(err == 0); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - for (i = 0; i < (int)vbRange.size; i++) - { - verID = MAXINT; - - err = brm.vssLookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == 0); - CPPUNIT_ASSERT(vbFlag == false); - - err = brm.lookup(i, verID, vbFlag, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == 1); - CPPUNIT_ASSERT(fbo == static_cast(i)); - - vbbm.lock(VBBM::WRITE); - vss.lock(VSS::WRITE); - -#ifdef BRM_DEBUG - caughtException = false; - - try - { - vbbm.removeEntry(i, verID); - vbbm.confirmChanges(); - } - catch (logic_error& e) - { - vbbm.undoChanges(); - caughtException = true; - } - - CPPUNIT_ASSERT(caughtException); - caughtException = false; - - try - { - vss.removeEntry(i, 1); - vss.confirmChanges(); - } - catch (logic_error& e) - { - vss.undoChanges(); - caughtException = true; - } - - CPPUNIT_ASSERT(caughtException); -#endif - - vss.removeEntry(i, verID); - vss.confirmChanges(); - vss.release(VSS::WRITE); - vbbm.release(VBBM::WRITE); - } - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - - brm.deleteOID(1); - - vbbm.lock(VBBM::READ); - vss.lock(VSS::READ); - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::READ); - vbbm.release(VBBM::READ); - - CPPUNIT_ASSERT(brm.checkConsistency() == 0); - Config::deleteInstanceMap(); - } - - /* This test verifies that deleteOID returns an error for - non-existant OIDs (bug #105) */ - void brm_deleteOID() - { - BlockResolutionManager brm; - int err; - vector extents; - -// cerr << "brm_deleteOID" << endl; - - err = brm.getExtents(1, extents); -#ifdef BRM_DEBUG - - if (err != -1) - cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; - - CPPUNIT_ASSERT(err == -1); -#else - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(extents.empty()); -#endif - err = brm.deleteOID(1); - CPPUNIT_ASSERT(err == -1); - Config::deleteInstanceMap(); - } - - /* This test verifies that setHWM and getHWM return an error for - nonexistant OIDs (bugs #106, 107) */ - - void brm_HWM() - { - BlockResolutionManager brm; - int err; - HWM_t hwm; - vector extents; - -// cerr << "brm_HWM" << endl; - - err = brm.getExtents(1, extents); -#ifdef BRM_DEBUG - - if (err != -1) - cerr << "Make sure OID 1 isn't allocated in the extent map" << endl; - - CPPUNIT_ASSERT(err == -1); -#else - CPPUNIT_ASSERT(err == 0); -#endif - CPPUNIT_ASSERT(extents.size() == 0); - err = brm.setHWM(1, 10); - CPPUNIT_ASSERT(err == -1); - err = brm.getHWM(1, hwm); - CPPUNIT_ASSERT(err == -1); - Config::deleteInstanceMap(); - } - - void vbbm_good_1() - { - - // add some VBBM entries - // do some lookups - // manipulate them - // do more lookups - - VBBM vbbm; - VSS vss; - vector ranges; - vector::iterator it; - int i, iterations = 100010, err; - OID_t oid; - uint32_t fbo, fbo2; - - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - -// cerr << "vbbm_good_1" << endl; - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - -// cerr << "\nVBBM test (this can take awhile)\n"; - vbbm.lock(VBBM::READ); - vbbm.release(VBBM::READ); - vbbm.lock(VBBM::WRITE); - vbbm.release(VBBM::WRITE); - - vbbm.lock(VBBM::WRITE); - - err = vbbm.lookup(0, 0, oid, fbo); - CPPUNIT_ASSERT(err == -1); - -// cerr << "step 1/4: get some VB blocks" << endl; - - vss.lock(VSS::WRITE); - vbbm.getBlocks(iterations, ranges, vss); - vbbm.confirmChanges(); - vss.confirmChanges(); - vss.release(VSS::WRITE); - CPPUNIT_ASSERT(!ranges.empty()); - -// cerr << "step 2/4: inserts & lookups" << endl; - for (it = ranges.begin(), i = 0; it != ranges.end(); it++) - { - for (fbo = (*it).vbFBO; fbo < (*it).vbFBO + (*it).size; fbo++, i++) - { - vbbm.insert(i, i + 1, (*it).vbOID, fbo); - vbbm.confirmChanges(); - err = vbbm.lookup(i, i + 1, oid, fbo2); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == (*it).vbOID); - CPPUNIT_ASSERT(fbo2 == fbo); - } - } - - /* - for (i = 0; i < iterations; i++) { - - vbbm.insert(i, i+1, i+2, i+3); - err = vbbm.lookup(i, i+1, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i + 2); - CPPUNIT_ASSERT(fbo == static_cast(i + 3)); - } - */ - - vbbm.save(string("VBBMImage")); - vbbm.load(string("VBBMImage")); - -// cerr << "step 3/4: lookups" << endl; - for (it = ranges.begin(), i = 0; it != ranges.end(); it++) - { - for (fbo = (*it).vbFBO; fbo < (*it).vbFBO + (*it).size; fbo++, i++) - { - err = vbbm.lookup(i, i + 1, oid, fbo2); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == (*it).vbOID); - CPPUNIT_ASSERT(fbo2 == fbo); - } - } - - /* - for (i = 0; i < iterations; i++) { - err = vbbm.lookup(i, i+1, oid, fbo); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(oid == i + 2); - CPPUNIT_ASSERT(fbo == static_cast(i + 3)); - } */ - -// cerr << "step 4/4: remove all entries" << endl; - for (i = 0; i < iterations; i++) - { - vbbm.removeEntry(i, i + 1); - vbbm.confirmChanges(); - } - - CPPUNIT_ASSERT(vbbm.size() == 0); - CPPUNIT_ASSERT(vbbm.hashEmpty()); - - vbbm.release(VBBM::WRITE); - Config::deleteInstanceMap(); - } - - void vss_good_1() - { - VSS vss; - VBBM vbbm; - ExtentMap em; - int i, err, iterations = 100010; - VER_t verID; - bool vbFlag; - vector lbids; - LBIDRange range; - - // Buildbot times out on the getBlocks() call during leakcheck b/c it takes - // > 5 mins for some reason. Have to ping it before 300 seconds go by. - void (*oldsig)(int); - -// cerr << "vss_good_1" << endl; - - oldsig = signal(SIGALRM, keepalive); - alarm(290); - - cerr << endl << "VSS test (this can take awhile)" << endl; - - vss.lock(VSS::READ); - vss.release(VSS::READ); - - vss.lock(VSS::WRITE); - - cerr << "step 1/5: insert 2 entries for 100010 LBIDs & test lookup logic" << endl; - - range.start = 0; - range.size = 200000; - - CPPUNIT_ASSERT(!vss.isLocked(range)); - - for (i = 0; i < iterations; i++) - { - vss.insert(i, i + 1, (i % 2 ? true : false), (i % 2 ? false : true)); - vss.insert(i, i, true, false); - vss.confirmChanges(); - - if (i == 0) - { - range.start = 0; - range.size = 1; - CPPUNIT_ASSERT(vss.isLocked(range)); - } - else if (i == 1) - { - range.start = 1; - CPPUNIT_ASSERT(!vss.isLocked(range)); - } - - verID = i + 10; - err = vss.lookup(i, verID, i + 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == i + 1); - CPPUNIT_ASSERT((vbFlag && (i % 2)) || (!vbFlag && !(i % 2))); - verID = i + 10; - err = vss.lookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == (i % 2 ? i + 1 : i)); - CPPUNIT_ASSERT(vbFlag == true); - - if (i > 0) - { - verID = i - 1; - err = vss.lookup(i, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == -1); - } - - } - - range.start = 0; - range.size = 200000; - CPPUNIT_ASSERT(vss.isLocked(range)); - vss.save(string("VSSImage")); - vss.load(string("VSSImage")); - - // this loop actually breaks the locked -> !vbFlag invariant - // switch it back afterward - cerr << "step 2/5: flip the vbFlag on half of the entries & test lookup logic" << endl; - - for (i = 0; i < iterations; i++) - { - vss.setVBFlag(i, i + 1, (i % 2 ? false : true)); - vss.confirmChanges(); - } - - for (i = 0; i < iterations; i++) - { - verID = i + 10; - err = vss.lookup(i, verID, i + 1, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == i + 1); - CPPUNIT_ASSERT((!vbFlag && (i % 2)) || (vbFlag && !(i % 2))); - } - - cerr << "step 3/5: some clean up" << endl; - - for (i = 0; i < iterations; i++) - { - vss.setVBFlag(i, i + 1, (i % 2 ? true : false)); - vss.confirmChanges(); - } - - - // there are 2 entries for every txnid at this point, one - // which is locked, one which isn't. That's technically a violation, - // so we have to get rid of the unlocked ones for the next test. - for (i = 0; i < iterations; i++) - { - vss.removeEntry(i, i); - vss.confirmChanges(); - } - - //speed up the next step! - for (i = iterations / 50; i < iterations; i++) - { - vss.removeEntry(i, i + 1); - vss.confirmChanges(); - } - - cerr << "step 4/5: get \'uncommitted\' LBIDs, commit, and test lookup logic" << endl; - - for (i = 1; i < iterations / 50; i++) - { - bool ex = false; - - if (i % 2 == 0) - { -#ifdef BRM_DEBUG - - try - { - vss.getUncommittedLBIDs(i, lbids); - } - catch (logic_error& e) - { - ex = true; - } - - CPPUNIT_ASSERT(ex == true); -#endif - } - else - { - LBID_t lbid; - - vss.getUncommittedLBIDs(i, lbids); - CPPUNIT_ASSERT(lbids.size() == 1); - lbid = *(lbids.begin()); - CPPUNIT_ASSERT(lbid == i - 1); - verID = i + 10; - err = vss.lookup(i - 1, verID, i, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == i); - CPPUNIT_ASSERT(vbFlag == false); - - err = vss.lookup(i - 1, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == -1); - - vss.commit(i); - vss.confirmChanges(); - verID = i + 10; - err = vss.lookup(i - 1, verID, 0, vbFlag); - CPPUNIT_ASSERT(err == 0); - CPPUNIT_ASSERT(verID == i); - CPPUNIT_ASSERT(vbFlag == false); - - } - } - - cerr << "step 5/5: final clean up" << endl; - - CPPUNIT_ASSERT(vss.size() == iterations / 50); - range.start = 0; - range.size = iterations / 50; - vss.removeEntriesFromDB(range, vbbm, false); - vbbm.confirmChanges(); - vss.confirmChanges(); - - // the new logic for removeEntriesFromDB deletes all entries in the range, - // locked or not, in the VB or not - -// CPPUNIT_ASSERT(vss.size() == iterations/100); - -// for (i = 1; i < iterations/50; i+=2) -// vss.removeEntry(i, i+1); - - CPPUNIT_ASSERT(vss.size() == 0); - CPPUNIT_ASSERT(vss.hashEmpty()); - vss.release(VSS::WRITE); - - alarm(0); - signal(SIGALRM, oldsig); - Config::deleteInstanceMap(); - } - - /* This test was suggested by Jean, who observed some odd behavior at around - 1032 EM instances in her code. This test creates 2000 instances and uses 1400 of them - to do the extentmap_freelist_1 test above. */ - void many_ExtentMap_instances() - { - ExtentMap** ems; - int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) - vector lbids; - uint32_t extentSize; - -// cerr << endl << "Extent Map instance test" << endl; - - ems = new ExtentMap*[2000]; - - for (i = 0; i < 2000; i++) - ems[i] = new ExtentMap(); - - extentSize = ems[0]->getExtentSize(); - -// cerr << " - Successfully created 2000 instances, starting the freelist test" -// << endl; - for (i = 1; i < iterations; i++) - { - ems[i]->createExtent(extentSize, i, lbids, allocdSize); - ems[i]->confirmChanges(); - CPPUNIT_ASSERT(lbids.back() == static_cast((i - 1)*extentSize)); - } - - //frag the lbid space to blow up the free list - for (i = 1; i < iterations; i += 2) - { - ems[i]->deleteOID(i); - ems[i]->confirmChanges(); - } - - //fill in the holes - for (i = 1; i < iterations; i += 2) - { - ems[i]->createExtent(extentSize, i, lbids, allocdSize); - ems[i]->confirmChanges(); - } - - for (i = 1; i < iterations; i += 2) - { - ems[i]->deleteOID(i); - ems[i]->confirmChanges(); - } - - for (i = 2; i < iterations; i += 2) - { - ems[i]->deleteOID(i); - ems[i]->confirmChanges(); - } - -// cerr << "done. sleeping for 30 seconds." << endl; -// sleep(30); - - for (i = 0; i < 2000; i++) - delete ems[i]; - - delete [] ems; - Config::deleteInstanceMap(); - - } - - void resource_graph_1() - { - LBIDResourceGraph rg; - pthread_t t; - VER_t txn = 1; - int err; - - pthread_mutex_init(&mutex, NULL); - pthread_mutex_lock(&mutex); - - err = rg.reserveRange(1001, 2000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - err = rg.reserveRange(2001, 2000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - err = rg.reserveRange(0, 1000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - err = rg.reserveRange(3001, 2000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - pthread_create(&t, NULL, dummy_1, &rg); - pthread_mutex_unlock(&mutex); - sleep(5); - - // thread should sleep here - - pthread_mutex_lock(&mutex); - rg.releaseResources(txn); - pthread_mutex_unlock(&mutex); - - // thread should be unblocked and finish here - - pthread_join(t, NULL); - pthread_mutex_lock(&mutex); - err = rg.reserveRange(0, 1000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - rg.releaseResources(txn); - pthread_mutex_unlock(&mutex); - pthread_mutex_destroy(&mutex); - Config::deleteInstanceMap(); - } - - void resource_graph_deadlock() - { - LBIDResourceGraph rg; - pthread_t t; - VER_t txn = 1; - int err; - - pthread_mutex_init(&mutex, NULL); - pthread_mutex_lock(&mutex); - err = rg.reserveRange(0, 1000, txn, mutex); - CPPUNIT_ASSERT(err == 0); - pthread_create(&t, NULL, dummy_deadlock, &rg); - pthread_mutex_unlock(&mutex); - sleep(1); - - // thread grabs 1001-2000, then 0-1000 and blocks - - pthread_mutex_lock(&mutex); - err = rg.reserveRange(1001, 2000, txn, mutex); // deadlock - CPPUNIT_ASSERT(err == ERR_DEADLOCK); - rg.releaseResources(txn); - - pthread_mutex_unlock(&mutex); // should wake the thread - pthread_join(t, NULL); // wait for it to finish - - pthread_mutex_lock(&mutex); - err = rg.reserveRange(0, 2000, txn, mutex); //verify we can now grab both ranges - CPPUNIT_ASSERT(err == ERR_OK); - pthread_mutex_unlock(&mutex); - pthread_mutex_destroy(&mutex); - Config::deleteInstanceMap(); - } - - void resource_graph_fragd_range() - { - LBIDResourceGraph rg; - set >* resources; - map* txns; - pthread_t t; - VER_t txn = 1; - int err; - - pthread_mutex_init(&mutex, NULL); - pthread_mutex_lock(&mutex); - - err = rg.reserveRange(1001, 2000, txn, mutex); - CPPUNIT_ASSERT(err == ERR_OK); - err = rg.reserveRange(3001, 4000, txn, mutex); - CPPUNIT_ASSERT(err == ERR_OK); - pthread_create(&t, NULL, fragd_range_helper, &rg); - pthread_mutex_unlock(&mutex); - sleep(1); - - // thread wakes, tries to grab 0-5000 - // check that there are 5 resource nodes - - pthread_mutex_lock(&mutex); - resources = rg.getResources(); - txns = rg.getTxns(); - CPPUNIT_ASSERT(resources->size() == 5); - CPPUNIT_ASSERT(txns->size() == 2); - rg.releaseResources(txn); - CPPUNIT_ASSERT(resources->size() == 3); - CPPUNIT_ASSERT(txns->size() == 1); - pthread_mutex_unlock(&mutex); - - // thread releases, exits - - pthread_join(t, NULL); - CPPUNIT_ASSERT(resources->size() == 0); - CPPUNIT_ASSERT(txns->size() == 0); - - pthread_mutex_destroy(&mutex); - Config::deleteInstanceMap(); - } - - void resource_graph_overlapping_ranges() - { - LBIDResourceGraph rg; - set >* resources; - set >::const_iterator it; - map* txns; - pthread_t t; - VER_t txn = 1; - int err; - - // make a fragmented range, then request a range that overlaps existing - // ranges on both ends - - pthread_mutex_init(&mutex, NULL); - pthread_mutex_lock(&mutex); - - err = rg.reserveRange(1001, 2000, txn, mutex); - CPPUNIT_ASSERT(err == ERR_OK); - err = rg.reserveRange(3001, 4000, txn, mutex); - CPPUNIT_ASSERT(err == ERR_OK); - pthread_create(&t, NULL, overlapping_range_helper, &rg); - pthread_mutex_unlock(&mutex); - sleep(1); - - // thread executes, tries to grab 1100-3900 - - pthread_mutex_lock(&mutex); - resources = rg.getResources(); - txns = rg.getTxns(); - -#ifdef BRM_VERBOSE - cerr << " resources->size = " << resources->size() << endl; - - for (it = resources->begin(); it != resources->end(); it++) - cerr << " " << (*it)->start() << "-" << (*it)->end() << endl; - -#endif - - it = resources->begin(); - CPPUNIT_ASSERT(resources->size() == 3); - CPPUNIT_ASSERT((*it)->start() == 1001); - CPPUNIT_ASSERT((*it)->end() == 2000); - it++; - CPPUNIT_ASSERT((*it)->start() == 2001); - CPPUNIT_ASSERT((*it)->end() == 3000); - it++; - CPPUNIT_ASSERT((*it)->start() == 3001); - CPPUNIT_ASSERT((*it)->end() == 4000); - - CPPUNIT_ASSERT(txns->size() == 2); - rg.releaseResources(txn); - CPPUNIT_ASSERT(txns->size() == 1); - pthread_mutex_unlock(&mutex); - - pthread_join(t, NULL); - CPPUNIT_ASSERT(resources->size() == 0); - CPPUNIT_ASSERT(txns->size() == 0); - - pthread_mutex_destroy(&mutex); - Config::deleteInstanceMap(); - } + pthread_join(t, NULL); + CPPUNIT_ASSERT(resources->size() == 0); + CPPUNIT_ASSERT(txns->size() == 0); + pthread_mutex_destroy(&mutex); + Config::deleteInstanceMap(); + } }; - - -CPPUNIT_TEST_SUITE_REGISTRATION( BRMTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(BRMTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/versioning/BRM/tracer.cpp b/versioning/BRM/tracer.cpp index 22c46fbb7..9fa3b0ee1 100644 --- a/versioning/BRM/tracer.cpp +++ b/versioning/BRM/tracer.cpp @@ -45,153 +45,148 @@ namespace BRM #ifdef BRM_INFO std::ofstream brmlog(string(MCSLOGDIR) + "/brm.log", std::ios::app); -Tracer::Tracer(const std::string& file, int line, const std::string& msg, bool debug, bool writeNow): fFileName(file), fLine(line), fMsg(msg), fDebug(debug), fpid(getpid()) +Tracer::Tracer(const std::string& file, int line, const std::string& msg, bool debug, bool writeNow) + : fFileName(file), fLine(line), fMsg(msg), fDebug(debug), fpid(getpid()) { - if (writeNow) - writeBegin(); + if (writeNow) + writeBegin(); } Tracer::~Tracer() { - writeEnd(); + writeEnd(); } - void printIntVec(const pair& p) { - brmlog << p.first << ": (" << *p.second << ") "; + brmlog << p.first << ": (" << *p.second << ") "; } -void printBoolVec(const pair& p) +void printBoolVec(const pair& p) { - brmlog << p.first << ": (" << *p.second << ") "; + brmlog << p.first << ": (" << *p.second << ") "; } void printStrVec(const pair& p) { - brmlog << p.first << ": (" << *p.second << ") "; + brmlog << p.first << ": (" << *p.second << ") "; } void printShortVec(const pair& p) { - brmlog << p.first << ": (" << *p.second << ") "; + brmlog << p.first << ": (" << *p.second << ") "; } void printInt64Vec(const pair& p) { - brmlog << p.first << ": (" << *p.second << ") "; + brmlog << p.first << ": (" << *p.second << ") "; } - void Tracer::writeBegin() { - brmlog << timeStamp() << fpid << ":" << fFileName << "@" << fLine << " " << fMsg << " begin - "; - for_each(fInputs.begin(), fInputs.end(), printIntVec); - for_each(fBoolInputs.begin(), fBoolInputs.end(), printBoolVec); - for_each(fStrInputs.begin(), fStrInputs.end(), printStrVec); - for_each(fShortInputs.begin(), fShortInputs.end(), printShortVec); - for_each(fInt64Inputs.begin(), fInt64Inputs.end(), printInt64Vec); - brmlog << endl << flush; - + brmlog << timeStamp() << fpid << ":" << fFileName << "@" << fLine << " " << fMsg << " begin - "; + for_each(fInputs.begin(), fInputs.end(), printIntVec); + for_each(fBoolInputs.begin(), fBoolInputs.end(), printBoolVec); + for_each(fStrInputs.begin(), fStrInputs.end(), printStrVec); + for_each(fShortInputs.begin(), fShortInputs.end(), printShortVec); + for_each(fInt64Inputs.begin(), fInt64Inputs.end(), printInt64Vec); + brmlog << endl << flush; } void Tracer::writeEnd() { - brmlog << timeStamp() << fpid << ":" << fFileName << " " << fMsg << " end "; + brmlog << timeStamp() << fpid << ":" << fFileName << " " << fMsg << " end "; - if (! fOutputs.empty() || ! fBoolOutputs.empty()) - brmlog << "- "; - - for_each(fOutputs.begin(), fOutputs.end(), printIntVec); - for_each(fBoolOutputs.begin(), fBoolOutputs.end(), printBoolVec); - for_each(fShortOutputs.begin(), fShortOutputs.end(), printShortVec); - for_each(fInt64Outputs.begin(), fInt64Outputs.end(), printInt64Vec); - brmlog << endl << flush; + if (!fOutputs.empty() || !fBoolOutputs.empty()) + brmlog << "- "; + for_each(fOutputs.begin(), fOutputs.end(), printIntVec); + for_each(fBoolOutputs.begin(), fBoolOutputs.end(), printBoolVec); + for_each(fShortOutputs.begin(), fShortOutputs.end(), printShortVec); + for_each(fInt64Outputs.begin(), fInt64Outputs.end(), printInt64Vec); + brmlog << endl << flush; } void Tracer::writeDirect(const std::string& msg) { - brmlog << msg << endl; + brmlog << msg << endl; } string Tracer::timeStamp() { - time_t outputTime; - time(&outputTime); - string datestr(ctime(&outputTime)); + time_t outputTime; + time(&outputTime); + string datestr(ctime(&outputTime)); - try + try + { + // replace newline + return datestr.replace(datestr.length() - 1, 1, string(":")); + } + catch (exception& ex) + { + size_t tries = 3; + + while (!datestr.length() && tries--) + datestr = ctime(&outputTime); + + if (datestr.length()) + return datestr.replace(datestr.length() - 1, 1, string(":")); + else { - //replace newline - return datestr.replace(datestr.length() - 1, 1, string(":")); - } - catch (exception& ex) - { - size_t tries = 3; - - while (!datestr.length() && tries--) - datestr = ctime(&outputTime); - - if (datestr.length()) - return datestr.replace(datestr.length() - 1, 1, string(":")); - else - { - std::cerr << __FILE__ << "@" << __LINE__ << " " << ex.what() << " "; - cerr << datestr << " length " << datestr.length() << " Source: "; - cerr << fFileName << "@" << fLine << " " << fMsg << endl; - return string("timestamp error:"); - } + std::cerr << __FILE__ << "@" << __LINE__ << " " << ex.what() << " "; + cerr << datestr << " length " << datestr.length() << " Source: "; + cerr << fFileName << "@" << fLine << " " << fMsg << endl; + return string("timestamp error:"); } + } } - void Tracer::addInput(const string& name, const int* value) { - fInputs.push_back(make_pair(name, value)); + fInputs.push_back(make_pair(name, value)); } void Tracer::addInput(const string& name, const string* value) { - fStrInputs.push_back(make_pair(name, value)); + fStrInputs.push_back(make_pair(name, value)); } void Tracer::addInput(const string& name, const bool* value) { - fBoolInputs.push_back(make_pair(name, value)); + fBoolInputs.push_back(make_pair(name, value)); } void Tracer::addInput(const string& name, const short* value) { - fShortInputs.push_back(make_pair(name, value)); + fShortInputs.push_back(make_pair(name, value)); } void Tracer::addInput(const string& name, const int64_t* value) { - fInt64Inputs.push_back(make_pair(name, value)); + fInt64Inputs.push_back(make_pair(name, value)); } - void Tracer::addOutput(const string& name, const int* value) { - fOutputs.push_back(make_pair(name, value)); + fOutputs.push_back(make_pair(name, value)); } void Tracer::addOutput(const string& name, const bool* value) { - fBoolOutputs.push_back(make_pair(name, value)); + fBoolOutputs.push_back(make_pair(name, value)); } void Tracer::addOutput(const string& name, const short* value) { - fShortOutputs.push_back(make_pair(name, value)); + fShortOutputs.push_back(make_pair(name, value)); } void Tracer::addOutput(const string& name, const int64_t* value) { - fInt64Outputs.push_back(make_pair(name, value)); + fInt64Outputs.push_back(make_pair(name, value)); } #endif -} //namespace +} // namespace BRM diff --git a/versioning/BRM/tracer.h b/versioning/BRM/tracer.h index 3f40e33b9..a999f358b 100644 --- a/versioning/BRM/tracer.h +++ b/versioning/BRM/tracer.h @@ -40,155 +40,139 @@ namespace BRM { - /** @brief Trace beginning and end of a call. -* -* When turned on, fDebug is true, Tracer writes informational messages to /var/log/mariadb/columnstore/brm.log. -* It keeps track of integral, string and boolean inputs, and integral and boolean outputs. -* If it should print a message immediately from the constructor, call it with the final parameter, -* writeNow, true (default). If input and output parameters need to be added, call it with -* writeNow false. Then writeBegin() must be called explicitely. -* -* Input parameters are added with addInput, output with addOutput. -* -* On end, if debug is set, an end message is printed with the output parameters. -*/ + * + * When turned on, fDebug is true, Tracer writes informational messages to + * /var/log/mariadb/columnstore/brm.log. It keeps track of integral, string and boolean inputs, and integral + * and boolean outputs. If it should print a message immediately from the constructor, call it with the final + * parameter, writeNow, true (default). If input and output parameters need to be added, call it with + * writeNow false. Then writeBegin() must be called explicitely. + * + * Input parameters are added with addInput, output with addOutput. + * + * On end, if debug is set, an end message is printed with the output parameters. + */ class Tracer { -public: - /** @brief Tracer Constructor - * - * @param file (in string) the file name. - * @param line (in int ) the line number. - * @param msg (in string) the message that will be printed at the begin and end. - * @param debug (in bool) if information should be printed - * @param writeNow (in bool) whether to print the begin message from the constructor - * (printed if debug is true.) - */ - EXPORT Tracer(const std::string& file, int line, const std::string& msg, bool debug, bool writeNow = true); + public: + /** @brief Tracer Constructor + * + * @param file (in string) the file name. + * @param line (in int ) the line number. + * @param msg (in string) the message that will be printed at the begin and end. + * @param debug (in bool) if information should be printed + * @param writeNow (in bool) whether to print the begin message from the constructor + * (printed if debug is true.) + */ + EXPORT Tracer(const std::string& file, int line, const std::string& msg, bool debug, bool writeNow = true); - /** @brief Tracer destructor - * - * Prints the output message if debug is true. - */ - EXPORT ~Tracer(); + /** @brief Tracer destructor + * + * Prints the output message if debug is true. + */ + EXPORT ~Tracer(); - /** @brief writeBegin - * - * Prints the begin message in form: - * file@line number begin - followed by the input parameters: - * name: (value) - */ - EXPORT void writeBegin(); + /** @brief writeBegin + * + * Prints the begin message in form: + * file@line number begin - followed by the input parameters: + * name: (value) + */ + EXPORT void writeBegin(); - /** @brief writeEnd - * - * Prints the end message in form: - * file@line number begin - followed by the output parameters: - * name: (value) - */ - EXPORT void writeEnd(); + /** @brief writeEnd + * + * Prints the end message in form: + * file@line number begin - followed by the output parameters: + * name: (value) + */ + EXPORT void writeEnd(); - /** @brief writeDirect - * - * Prints specified string immediately (directly) to the brm log. - * Different than the addInput and addOutput functions that - * accumulate the data for later printing to the brm log. - * @param msg String to be printed to the brm log - */ - void writeDirect(const std::string& msg); + /** @brief writeDirect + * + * Prints specified string immediately (directly) to the brm log. + * Different than the addInput and addOutput functions that + * accumulate the data for later printing to the brm log. + * @param msg String to be printed to the brm log + */ + void writeDirect(const std::string& msg); - /** @brief addInput - * - * @param name (in string) the variable name. - * @param value (in int* or string* or bool* ) the variable - */ - EXPORT void addInput(const std::string& name, const int* value); - EXPORT void addInput(const std::string& name, const std::string* value); - EXPORT void addInput(const std::string& name, const bool* value); - EXPORT void addInput(const std::string& name, const short* value); - EXPORT void addInput(const std::string& name, const int64_t* value); + /** @brief addInput + * + * @param name (in string) the variable name. + * @param value (in int* or string* or bool* ) the variable + */ + EXPORT void addInput(const std::string& name, const int* value); + EXPORT void addInput(const std::string& name, const std::string* value); + EXPORT void addInput(const std::string& name, const bool* value); + EXPORT void addInput(const std::string& name, const short* value); + EXPORT void addInput(const std::string& name, const int64_t* value); - /** @brief addOutput - * - * @param name (in string) the variable name. - * @param value (in int* or bool* ) the variable - */ - EXPORT void addOutput(const std::string& name, const int* value); - EXPORT void addOutput(const std::string& name, const bool* value); - EXPORT void addOutput(const std::string& name, const short* value); - EXPORT void addOutput(const std::string& name, const int64_t* value); + /** @brief addOutput + * + * @param name (in string) the variable name. + * @param value (in int* or bool* ) the variable + */ + EXPORT void addOutput(const std::string& name, const int* value); + EXPORT void addOutput(const std::string& name, const bool* value); + EXPORT void addOutput(const std::string& name, const short* value); + EXPORT void addOutput(const std::string& name, const int64_t* value); -private: - std::string timeStamp(); - typedef std::vector > ValuesVec; - typedef std::vector > ValuesStrVec; - typedef std::vector > ValuesBoolVec; - typedef std::vector > ValuesShortVec; - typedef std::vector > ValuesInt64Vec; - std::string fFileName; - int fLine; - std::string fMsg; - bool fDebug; - ValuesVec fInputs; - ValuesStrVec fStrInputs; - ValuesBoolVec fBoolInputs; - ValuesShortVec fShortInputs; - ValuesInt64Vec fInt64Inputs; - ValuesVec fOutputs; - ValuesBoolVec fBoolOutputs; - ValuesShortVec fShortOutputs; - ValuesInt64Vec fInt64Outputs; - pid_t fpid; + private: + std::string timeStamp(); + typedef std::vector > ValuesVec; + typedef std::vector > ValuesStrVec; + typedef std::vector > ValuesBoolVec; + typedef std::vector > ValuesShortVec; + typedef std::vector > ValuesInt64Vec; + std::string fFileName; + int fLine; + std::string fMsg; + bool fDebug; + ValuesVec fInputs; + ValuesStrVec fStrInputs; + ValuesBoolVec fBoolInputs; + ValuesShortVec fShortInputs; + ValuesInt64Vec fInt64Inputs; + ValuesVec fOutputs; + ValuesBoolVec fBoolOutputs; + ValuesShortVec fShortOutputs; + ValuesInt64Vec fInt64Outputs; + pid_t fpid; }; /** @brief tracer macros -* -* Use the contructors if there is a bool fDebug in scope. -*/ + * + * Use the contructors if there is a bool fDebug in scope. + */ +#define TRACER_WRITENOW(a) Tracer tracer(__FILE__, __LINE__, a, fDebug); -#define TRACER_WRITENOW(a)\ - Tracer tracer(__FILE__, __LINE__, a, fDebug); +#define TRACER_WRITELATER(a) Tracer tracer(__FILE__, __LINE__, a, fDebug, false); -#define TRACER_WRITELATER(a)\ - Tracer tracer(__FILE__, __LINE__, a, fDebug, false); +#define TRACER_ADDINPUT(a) tracer.addInput((std::string) #a, (int*)&a); -#define TRACER_ADDINPUT(a)\ - tracer.addInput((std::string)#a, (int*)&a); +#define TRACER_ADDSTRINPUT(a) tracer.addInput((std::string) #a, &a); -#define TRACER_ADDSTRINPUT(a)\ - tracer.addInput((std::string)#a, &a); +#define TRACER_ADDBOOLINPUT(a) tracer.addInput((std::string) #a, &a); -#define TRACER_ADDBOOLINPUT(a)\ - tracer.addInput((std::string)#a, &a); +#define TRACER_ADDSHORTINPUT(a) tracer.addInput((std::string) #a, (short*)&a); -#define TRACER_ADDSHORTINPUT(a)\ - tracer.addInput((std::string)#a, (short*)&a); +#define TRACER_ADDINT64INPUT(a) tracer.addInput((std::string) #a, (int64_t*)&a); -#define TRACER_ADDINT64INPUT(a)\ - tracer.addInput((std::string)#a, (int64_t*)&a); +#define TRACER_ADDOUTPUT(a) tracer.addOutput((std::string) #a, (int*)&a); -#define TRACER_ADDOUTPUT(a)\ - tracer.addOutput((std::string)#a, (int*)&a); +#define TRACER_ADDBOOLOUTPUT(a) tracer.addOutput((std::string) #a, &a); -#define TRACER_ADDBOOLOUTPUT(a)\ - tracer.addOutput((std::string)#a, &a); +#define TRACER_ADDSHORTOUTPUT(a) tracer.addOutput((std::string) #a, (short*)&a); -#define TRACER_ADDSHORTOUTPUT(a)\ - tracer.addOutput((std::string)#a, (short*)&a); +#define TRACER_ADDINT64OUTPUT(a) tracer.addOutput((std::string) #a, (int64_t*)&a); -#define TRACER_ADDINT64OUTPUT(a)\ - tracer.addOutput((std::string)#a, (int64_t*)&a); +#define TRACER_WRITE tracer.writeBegin(); -#define TRACER_WRITE\ - tracer.writeBegin(); - -#define TRACER_WRITEDIRECT(msg)\ - tracer.writeDirect(msg); -} +#define TRACER_WRITEDIRECT(msg) tracer.writeDirect(msg); +} // namespace BRM #undef EXPORT #endif - diff --git a/versioning/BRM/transactionnode.cpp b/versioning/BRM/transactionnode.cpp index a3826972c..0b1ec0926 100644 --- a/versioning/BRM/transactionnode.cpp +++ b/versioning/BRM/transactionnode.cpp @@ -29,7 +29,6 @@ namespace BRM { - TransactionNode::TransactionNode(int txn) : txnID(txn), _die(false), _sleeping(false) { } @@ -40,39 +39,39 @@ TransactionNode::~TransactionNode() void TransactionNode::setTxnID(VER_t txn) { - txnID = txn; + txnID = txn; } int TransactionNode::getTxnID() const { - return txnID; + return txnID; } void TransactionNode::sleep(boost::mutex& mutex) { - _sleeping = true; - condVar.wait(mutex); + _sleeping = true; + condVar.wait(mutex); } void TransactionNode::wake() { - condVar.notify_one(); - _sleeping = false; + condVar.notify_one(); + _sleeping = false; } void TransactionNode::die() { - _die = true; + _die = true; } bool TransactionNode::dead() { - return _die; + return _die; } bool TransactionNode::sleeping() { - return _sleeping; + return _sleeping; } -} // namespace +} // namespace BRM diff --git a/versioning/BRM/transactionnode.h b/versioning/BRM/transactionnode.h index 8c12caab6..d89dc0a62 100644 --- a/versioning/BRM/transactionnode.h +++ b/versioning/BRM/transactionnode.h @@ -41,33 +41,32 @@ namespace BRM { - class TransactionNode : public RGNode { -public: - EXPORT explicit TransactionNode(int txnid = 0); - EXPORT virtual ~TransactionNode(); + public: + EXPORT explicit TransactionNode(int txnid = 0); + EXPORT virtual ~TransactionNode(); - EXPORT void setTxnID(VER_t); - EXPORT int getTxnID() const; + EXPORT void setTxnID(VER_t); + EXPORT int getTxnID() const; - EXPORT void sleep(boost::mutex& mutex); - EXPORT void wake(); - EXPORT void die(); - EXPORT bool dead(); - EXPORT bool sleeping(); + EXPORT void sleep(boost::mutex& mutex); + EXPORT void wake(); + EXPORT void die(); + EXPORT bool dead(); + EXPORT bool sleeping(); -private: - TransactionNode(const TransactionNode&); - TransactionNode& operator=(const TransactionNode&); + private: + TransactionNode(const TransactionNode&); + TransactionNode& operator=(const TransactionNode&); - boost::condition condVar; - VER_t txnID; - bool _die, _sleeping; + boost::condition condVar; + VER_t txnID; + bool _die, _sleeping; }; #undef EXPORT -} // namespace +} // namespace BRM -#endif // TRANSACTIONNODE_H_ +#endif // TRANSACTIONNODE_H_ diff --git a/versioning/BRM/undoable.cpp b/versioning/BRM/undoable.cpp index b4ea347b9..af5db79d1 100644 --- a/versioning/BRM/undoable.cpp +++ b/versioning/BRM/undoable.cpp @@ -30,7 +30,6 @@ using namespace std; namespace BRM { - Undoable::Undoable() { } @@ -41,30 +40,30 @@ Undoable::~Undoable() inline void Undoable::makeUndoRecord(void* start, int size) { - ImageDelta d; + ImageDelta d; - if (size > ID_MAXSIZE) - throw overflow_error("Undoable::makeUndoRecord(): size > max"); + if (size > ID_MAXSIZE) + throw overflow_error("Undoable::makeUndoRecord(): size > max"); - d.start = start; - d.size = size; - memcpy(d.data, start, size); - undoRecords.push_back(d); + d.start = start; + d.size = size; + memcpy(d.data, start, size); + undoRecords.push_back(d); } void Undoable::undoChanges() { - vector::reverse_iterator it; + vector::reverse_iterator it; - for (it = undoRecords.rbegin(); it != undoRecords.rend(); it++) - memcpy((*it).start, (*it).data, (*it).size); + for (it = undoRecords.rbegin(); it != undoRecords.rend(); it++) + memcpy((*it).start, (*it).data, (*it).size); - undoRecords.clear(); + undoRecords.clear(); } void Undoable::confirmChanges() { - undoRecords.clear(); + undoRecords.clear(); } -} +} // namespace BRM diff --git a/versioning/BRM/undoable.h b/versioning/BRM/undoable.h index 2008aa543..273856e72 100644 --- a/versioning/BRM/undoable.h +++ b/versioning/BRM/undoable.h @@ -40,23 +40,22 @@ namespace BRM { - class Undoable { -public: - EXPORT Undoable(); - EXPORT virtual ~Undoable(); + public: + EXPORT Undoable(); + EXPORT virtual ~Undoable(); - EXPORT virtual void confirmChanges(); - EXPORT virtual void undoChanges(); + EXPORT virtual void confirmChanges(); + EXPORT virtual void undoChanges(); -protected: - virtual void makeUndoRecord(void* start, int size); - std::vector undoRecords; + protected: + virtual void makeUndoRecord(void* start, int size); + std::vector undoRecords; }; #undef EXPORT -} +} // namespace BRM #endif diff --git a/versioning/BRM/vbbm.cpp b/versioning/BRM/vbbm.cpp index 4e97102c0..253bd0512 100644 --- a/versioning/BRM/vbbm.cpp +++ b/versioning/BRM/vbbm.cpp @@ -71,14 +71,13 @@ using namespace idbdatafile; namespace BRM { - VBBMEntry::VBBMEntry() { - lbid = -1; - verID = 0; - vbOID = 0; - vbFBO = 0; - next = -1; + lbid = -1; + verID = 0; + vbOID = 0; + vbFBO = 0; + next = -1; } /*static*/ @@ -91,39 +90,38 @@ VBBMImpl* VBBMImpl::fInstance = 0; /*static*/ VBBMImpl* VBBMImpl::makeVBBMImpl(unsigned key, off_t size, bool readOnly) { - boost::mutex::scoped_lock lk(fInstanceMutex); + boost::mutex::scoped_lock lk(fInstanceMutex); - if (fInstance) + if (fInstance) + { + if (key != fInstance->fVBBM.key()) { - if (key != fInstance->fVBBM.key()) - { - BRMShmImpl newShm(key, size); - fInstance->swapout(newShm); - } - - idbassert(key == fInstance->fVBBM.key()); - return fInstance; + BRMShmImpl newShm(key, size); + fInstance->swapout(newShm); } - fInstance = new VBBMImpl(key, size, readOnly); - + idbassert(key == fInstance->fVBBM.key()); return fInstance; + } + + fInstance = new VBBMImpl(key, size, readOnly); + + return fInstance; } -VBBMImpl::VBBMImpl(unsigned key, off_t size, bool readOnly) : - fVBBM(key, size, readOnly) +VBBMImpl::VBBMImpl(unsigned key, off_t size, bool readOnly) : fVBBM(key, size, readOnly) { } VBBM::VBBM() { - vbbm = NULL; - currentVBBMShmkey = -1; - vbbmShmid = 0; - vbbmShminfo = NULL; - r_only = false; - fPVBBMImpl = 0; - currentFileSize = 0; + vbbm = NULL; + currentVBBMShmkey = -1; + vbbmShmid = 0; + vbbmShminfo = NULL; + r_only = false; + fPVBBMImpl = 0; + currentFileSize = 0; } VBBM::~VBBM() @@ -132,842 +130,819 @@ VBBM::~VBBM() void VBBM::initShmseg(int nFiles) { -// VBFileMetadata *newfiles; - int* newBuckets; - VBBMEntry* newStorage; - int i; - char* shmseg; + // VBFileMetadata *newfiles; + int* newBuckets; + VBBMEntry* newStorage; + int i; + char* shmseg; - vbbm->vbCapacity = VBSTORAGE_INITIAL_SIZE / sizeof(VBBMEntry); - vbbm->vbCurrentSize = 0; - vbbm->vbLWM = 0; - vbbm->numHashBuckets = VBTABLE_INITIAL_SIZE / sizeof(int); - shmseg = reinterpret_cast(vbbm); -// newfiles = reinterpret_cast -// (&shmseg[sizeof(VBShmsegHeader)]); - newBuckets = reinterpret_cast - (&shmseg[sizeof(VBShmsegHeader) + - nFiles * sizeof(VBFileMetadata)]); - newStorage = reinterpret_cast(&shmseg[sizeof(VBShmsegHeader) + - nFiles * sizeof(VBFileMetadata) + - vbbm->numHashBuckets * sizeof(int)]); - setCurrentFileSize(); - vbbm->nFiles = nFiles; + vbbm->vbCapacity = VBSTORAGE_INITIAL_SIZE / sizeof(VBBMEntry); + vbbm->vbCurrentSize = 0; + vbbm->vbLWM = 0; + vbbm->numHashBuckets = VBTABLE_INITIAL_SIZE / sizeof(int); + shmseg = reinterpret_cast(vbbm); + // newfiles = reinterpret_cast + // (&shmseg[sizeof(VBShmsegHeader)]); + newBuckets = reinterpret_cast(&shmseg[sizeof(VBShmsegHeader) + nFiles * sizeof(VBFileMetadata)]); + newStorage = reinterpret_cast( + &shmseg[sizeof(VBShmsegHeader) + nFiles * sizeof(VBFileMetadata) + vbbm->numHashBuckets * sizeof(int)]); + setCurrentFileSize(); + vbbm->nFiles = nFiles; - for (i = 0; i < vbbm->numHashBuckets; i++) - newBuckets[i] = -1; + for (i = 0; i < vbbm->numHashBuckets; i++) + newBuckets[i] = -1; - for (i = 0; i < vbbm->vbCapacity; i++) - newStorage[i].lbid = -1; + for (i = 0; i < vbbm->vbCapacity; i++) + newStorage[i].lbid = -1; } // ported from ExtentMap void VBBM::lock(OPS op) { - char* shmseg; + char* shmseg; - if (op == READ) + if (op == READ) + { + vbbmShminfo = mst.getTable_read(MasterSegmentTable::VBBMSegment); + mutex.lock(); + } + else + vbbmShminfo = mst.getTable_write(MasterSegmentTable::VBBMSegment); + + // this means that either the VBBM isn't attached or that it was resized + if (currentVBBMShmkey != vbbmShminfo->tableShmkey) + { + if (vbbm != NULL) { - vbbmShminfo = mst.getTable_read(MasterSegmentTable::VBBMSegment); - mutex.lock(); + vbbm = NULL; } - else - vbbmShminfo = mst.getTable_write(MasterSegmentTable::VBBMSegment); - // this means that either the VBBM isn't attached or that it was resized - if (currentVBBMShmkey != vbbmShminfo->tableShmkey) + if (vbbmShminfo->allocdSize == 0) { - if (vbbm != NULL) - { - vbbm = NULL; - } + if (op == READ) + { + mutex.unlock(); + mst.getTable_upgrade(MasterSegmentTable::VBBMSegment); if (vbbmShminfo->allocdSize == 0) { - if (op == READ) - { - mutex.unlock(); - mst.getTable_upgrade(MasterSegmentTable::VBBMSegment); - - if (vbbmShminfo->allocdSize == 0) - { - try - { - growVBBM(); - } - catch (...) - { - release(WRITE); - throw; - } - } - - mst.getTable_downgrade(MasterSegmentTable::VBBMSegment); - } - else - { - try - { - growVBBM(); - } - catch (...) - { - release(WRITE); - throw; - } - } + try + { + growVBBM(); + } + catch (...) + { + release(WRITE); + throw; + } } - else + + mst.getTable_downgrade(MasterSegmentTable::VBBMSegment); + } + else + { + try { - currentVBBMShmkey = vbbmShminfo->tableShmkey; - fPVBBMImpl = VBBMImpl::makeVBBMImpl(currentVBBMShmkey, 0); - idbassert(fPVBBMImpl); - - if (r_only) - fPVBBMImpl->makeReadOnly(); - - vbbm = fPVBBMImpl->get(); - shmseg = reinterpret_cast(vbbm); - files = reinterpret_cast - (&shmseg[sizeof(VBShmsegHeader)]); - hashBuckets = reinterpret_cast - (&shmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata)]); - storage = reinterpret_cast(&shmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata) + - vbbm->numHashBuckets * sizeof(int)]); - - if (op == READ) - mutex.unlock(); + growVBBM(); } + catch (...) + { + release(WRITE); + throw; + } + } } - else if (op == READ) + else + { + currentVBBMShmkey = vbbmShminfo->tableShmkey; + fPVBBMImpl = VBBMImpl::makeVBBMImpl(currentVBBMShmkey, 0); + idbassert(fPVBBMImpl); + + if (r_only) + fPVBBMImpl->makeReadOnly(); + + vbbm = fPVBBMImpl->get(); + shmseg = reinterpret_cast(vbbm); + files = reinterpret_cast(&shmseg[sizeof(VBShmsegHeader)]); + hashBuckets = + reinterpret_cast(&shmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata)]); + storage = reinterpret_cast( + &shmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata) + + vbbm->numHashBuckets * sizeof(int)]); + + if (op == READ) mutex.unlock(); + } + } + else if (op == READ) + mutex.unlock(); } // ported from ExtentMap void VBBM::release(OPS op) { - if (op == READ) - mst.releaseTable_read(MasterSegmentTable::VBBMSegment); - else - mst.releaseTable_write(MasterSegmentTable::VBBMSegment); + if (op == READ) + mst.releaseTable_read(MasterSegmentTable::VBBMSegment); + else + mst.releaseTable_write(MasterSegmentTable::VBBMSegment); } -//assumes write lock is held +// assumes write lock is held // Right now, adding a file and growing are mutually exclusive ops. void VBBM::growVBBM(bool addAFile) { - int allocSize; - int nFiles = -1; - key_t newshmkey; - char* newshmseg; + int allocSize; + int nFiles = -1; + key_t newshmkey; + char* newshmseg; - if (vbbmShminfo->allocdSize == 0) - { - if (addAFile) - nFiles = 1; - else - nFiles = 0; + if (vbbmShminfo->allocdSize == 0) + { + if (addAFile) + nFiles = 1; + else + nFiles = 0; - allocSize = (sizeof(VBShmsegHeader) + - (nFiles * sizeof(VBFileMetadata)) + - VBSTORAGE_INITIAL_SIZE + VBTABLE_INITIAL_SIZE); - } + allocSize = (sizeof(VBShmsegHeader) + (nFiles * sizeof(VBFileMetadata)) + VBSTORAGE_INITIAL_SIZE + + VBTABLE_INITIAL_SIZE); + } + else + { + if (!addAFile) + allocSize = vbbmShminfo->allocdSize + VBBM_INCREMENT; else { - if (!addAFile) - allocSize = vbbmShminfo->allocdSize + VBBM_INCREMENT; - else - { - vbbm->nFiles++; - allocSize = vbbmShminfo->allocdSize + sizeof(VBFileMetadata); - } + vbbm->nFiles++; + allocSize = vbbmShminfo->allocdSize + sizeof(VBFileMetadata); } + } - newshmkey = chooseShmkey(); + newshmkey = chooseShmkey(); - if (fPVBBMImpl) + if (fPVBBMImpl) + { + BRMShmImpl newShm(newshmkey, allocSize); + newshmseg = static_cast(newShm.fMapreg.get_address()); + memset(newshmseg, 0, allocSize); + + if (vbbm != NULL) { - BRMShmImpl newShm(newshmkey, allocSize); - newshmseg = static_cast(newShm.fMapreg.get_address()); - memset(newshmseg, 0, allocSize); + VBShmsegHeader* tmp = reinterpret_cast(newshmseg); + tmp->vbCapacity = vbbm->vbCapacity; + tmp->numHashBuckets = vbbm->numHashBuckets; - if (vbbm != NULL) - { - VBShmsegHeader* tmp = reinterpret_cast(newshmseg); - tmp->vbCapacity = vbbm->vbCapacity; - tmp->numHashBuckets = vbbm->numHashBuckets; + if (!addAFile) + { + tmp->vbCapacity += VBSTORAGE_INCREMENT / sizeof(VBBMEntry); + tmp->numHashBuckets += VBTABLE_INCREMENT / sizeof(int); + } - if (!addAFile) - { - tmp->vbCapacity += VBSTORAGE_INCREMENT / sizeof(VBBMEntry); - tmp->numHashBuckets += VBTABLE_INCREMENT / sizeof(int); - } - - tmp->vbLWM = 0; - copyVBBM(tmp); - } - - undoRecords.clear(); - fPVBBMImpl->swapout(newShm); - } - else - { - fPVBBMImpl = VBBMImpl::makeVBBMImpl(newshmkey, allocSize); - newshmseg = reinterpret_cast(fPVBBMImpl->get()); - memset(newshmseg, 0, allocSize); + tmp->vbLWM = 0; + copyVBBM(tmp); } - vbbm = fPVBBMImpl->get(); + undoRecords.clear(); + fPVBBMImpl->swapout(newShm); + } + else + { + fPVBBMImpl = VBBMImpl::makeVBBMImpl(newshmkey, allocSize); + newshmseg = reinterpret_cast(fPVBBMImpl->get()); + memset(newshmseg, 0, allocSize); + } - if (vbbmShminfo->allocdSize == 0) // this means the shmseg was created by this call - initShmseg(nFiles); + vbbm = fPVBBMImpl->get(); - vbbmShminfo->tableShmkey = currentVBBMShmkey = newshmkey; - vbbmShminfo->allocdSize = allocSize; + if (vbbmShminfo->allocdSize == 0) // this means the shmseg was created by this call + initShmseg(nFiles); - if (r_only) - fPVBBMImpl->makeReadOnly(); + vbbmShminfo->tableShmkey = currentVBBMShmkey = newshmkey; + vbbmShminfo->allocdSize = allocSize; - files = reinterpret_cast - (&newshmseg[sizeof(VBShmsegHeader)]); - hashBuckets = reinterpret_cast - (&newshmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata)]); - storage = reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata) + - vbbm->numHashBuckets * sizeof(int)]); + if (r_only) + fPVBBMImpl->makeReadOnly(); + + files = reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader)]); + hashBuckets = + reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata)]); + storage = + reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata) + + vbbm->numHashBuckets * sizeof(int)]); } void VBBM::growForLoad(int count) { - int allocSize; - int nFiles; - key_t newshmkey; - char* newshmseg; - int i; + int allocSize; + int nFiles; + key_t newshmkey; + char* newshmseg; + int i; - if (vbbm) - nFiles = vbbm->nFiles; - else - nFiles = 0; + if (vbbm) + nFiles = vbbm->nFiles; + else + nFiles = 0; - if (count < VBSTORAGE_INITIAL_COUNT) - count = VBSTORAGE_INITIAL_COUNT; + if (count < VBSTORAGE_INITIAL_COUNT) + count = VBSTORAGE_INITIAL_COUNT; - // round up to next normal increment point out of paranoia. - if (count % VBSTORAGE_INCREMENT_COUNT) - count = ((count / VBSTORAGE_INCREMENT_COUNT) + 1) * VBSTORAGE_INCREMENT_COUNT; + // round up to next normal increment point out of paranoia. + if (count % VBSTORAGE_INCREMENT_COUNT) + count = ((count / VBSTORAGE_INCREMENT_COUNT) + 1) * VBSTORAGE_INCREMENT_COUNT; - allocSize = VBBM_SIZE(nFiles, count); + allocSize = VBBM_SIZE(nFiles, count); - newshmkey = chooseShmkey(); + newshmkey = chooseShmkey(); - if (fPVBBMImpl) - { - BRMShmImpl newShm(newshmkey, allocSize); - newshmseg = static_cast(newShm.fMapreg.get_address()); - // copy the file meta to the new segment - memcpy((char*) &newshmseg[sizeof(VBShmsegHeader)], files, sizeof(VBFileMetadata)*nFiles); - fPVBBMImpl->swapout(newShm); - } - else - { - fPVBBMImpl = VBBMImpl::makeVBBMImpl(newshmkey, allocSize); - } + if (fPVBBMImpl) + { + BRMShmImpl newShm(newshmkey, allocSize); + newshmseg = static_cast(newShm.fMapreg.get_address()); + // copy the file meta to the new segment + memcpy((char*)&newshmseg[sizeof(VBShmsegHeader)], files, sizeof(VBFileMetadata) * nFiles); + fPVBBMImpl->swapout(newShm); + } + else + { + fPVBBMImpl = VBBMImpl::makeVBBMImpl(newshmkey, allocSize); + } - vbbm = fPVBBMImpl->get(); - vbbm->nFiles = nFiles; - vbbm->vbCapacity = count; - vbbm->vbLWM = 0; - vbbm->numHashBuckets = count / 4; + vbbm = fPVBBMImpl->get(); + vbbm->nFiles = nFiles; + vbbm->vbCapacity = count; + vbbm->vbLWM = 0; + vbbm->numHashBuckets = count / 4; - vbbmShminfo->tableShmkey = currentVBBMShmkey = newshmkey; - vbbmShminfo->allocdSize = allocSize; - newshmseg = (char*) vbbm; - files = reinterpret_cast - (&newshmseg[sizeof(VBShmsegHeader)]); - hashBuckets = reinterpret_cast - (&newshmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata)]); - storage = reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata) + - vbbm->numHashBuckets * sizeof(int)]); + vbbmShminfo->tableShmkey = currentVBBMShmkey = newshmkey; + vbbmShminfo->allocdSize = allocSize; + newshmseg = (char*)vbbm; + files = reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader)]); + hashBuckets = + reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata)]); + storage = + reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata) + + vbbm->numHashBuckets * sizeof(int)]); - for (i = 0; i < vbbm->numHashBuckets; i++) - hashBuckets[i] = -1; + for (i = 0; i < vbbm->numHashBuckets; i++) + hashBuckets[i] = -1; - for (i = 0; i < vbbm->vbCapacity; i++) - storage[i].lbid = -1; + for (i = 0; i < vbbm->vbCapacity; i++) + storage[i].lbid = -1; - undoRecords.clear(); + undoRecords.clear(); } -//assumes write lock is held and the src is vbbm -//and that dest->{numHashBuckets, vbCapacity, vbLWM} have been set. +// assumes write lock is held and the src is vbbm +// and that dest->{numHashBuckets, vbCapacity, vbLWM} have been set. void VBBM::copyVBBM(VBShmsegHeader* dest) { - int i; - int* newHashtable; - VBBMEntry* newStorage; - VBFileMetadata* newFiles; - char* cDest = reinterpret_cast(dest); + int i; + int* newHashtable; + VBBMEntry* newStorage; + VBFileMetadata* newFiles; + char* cDest = reinterpret_cast(dest); - // copy metadata - dest->nFiles = vbbm->nFiles; - dest->vbCurrentSize = vbbm->vbCurrentSize; + // copy metadata + dest->nFiles = vbbm->nFiles; + dest->vbCurrentSize = vbbm->vbCurrentSize; - newFiles = reinterpret_cast(&cDest[sizeof(VBShmsegHeader)]); - newHashtable = reinterpret_cast(&cDest[sizeof(VBShmsegHeader) + - dest->nFiles * sizeof(VBFileMetadata)]); - newStorage = reinterpret_cast(&cDest[sizeof(VBShmsegHeader) + - dest->nFiles * sizeof(VBFileMetadata) + - dest->numHashBuckets * sizeof(int)]); + newFiles = reinterpret_cast(&cDest[sizeof(VBShmsegHeader)]); + newHashtable = + reinterpret_cast(&cDest[sizeof(VBShmsegHeader) + dest->nFiles * sizeof(VBFileMetadata)]); + newStorage = + reinterpret_cast(&cDest[sizeof(VBShmsegHeader) + dest->nFiles * sizeof(VBFileMetadata) + + dest->numHashBuckets * sizeof(int)]); - memcpy(newFiles, files, sizeof(VBFileMetadata)*vbbm->nFiles); + memcpy(newFiles, files, sizeof(VBFileMetadata) * vbbm->nFiles); - //initialize new storage & hash - for (i = 0; i < dest->numHashBuckets; i++) - newHashtable[i] = -1; + // initialize new storage & hash + for (i = 0; i < dest->numHashBuckets; i++) + newHashtable[i] = -1; - for (i = 0; i < dest->vbCapacity; i++) - newStorage[i].lbid = -1; + for (i = 0; i < dest->vbCapacity; i++) + newStorage[i].lbid = -1; - //walk the storage & re-hash all entries; - for (i = 0; i < vbbm->vbCurrentSize; i++) - if (storage[i].lbid != -1) - { - _insert(storage[i], dest, newHashtable, newStorage, true); - //confirmChanges(); - } + // walk the storage & re-hash all entries; + for (i = 0; i < vbbm->vbCurrentSize; i++) + if (storage[i].lbid != -1) + { + _insert(storage[i], dest, newHashtable, newStorage, true); + // confirmChanges(); + } } key_t VBBM::chooseShmkey() const { - int fixedKeys = 1; - key_t ret; + int fixedKeys = 1; + key_t ret; - if (vbbmShminfo->tableShmkey + 1 == (key_t) (fShmKeys.KEYRANGE_VBBM_BASE + - fShmKeys.KEYRANGE_SIZE - 1) || (unsigned)vbbmShminfo->tableShmkey < - fShmKeys.KEYRANGE_VBBM_BASE) - ret = fShmKeys.KEYRANGE_VBBM_BASE + fixedKeys; - else - ret = vbbmShminfo->tableShmkey + 1; + if (vbbmShminfo->tableShmkey + 1 == (key_t)(fShmKeys.KEYRANGE_VBBM_BASE + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)vbbmShminfo->tableShmkey < fShmKeys.KEYRANGE_VBBM_BASE) + ret = fShmKeys.KEYRANGE_VBBM_BASE + fixedKeys; + else + ret = vbbmShminfo->tableShmkey + 1; - return ret; + return ret; } -//write lock +// write lock void VBBM::insert(LBID_t lbid, VER_t verID, OID_t vbOID, uint32_t vbFBO, bool loading) { - VBBMEntry entry; + VBBMEntry entry; #ifdef BRM_DEBUG - int i; + int i; - if (lbid < 0) - { - log("VBBM::insert(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::insert(): lbid must be >= 0"); - } + if (lbid < 0) + { + log("VBBM::insert(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::insert(): lbid must be >= 0"); + } - if (verID < 0) - { - log("VBBM::insert(): verID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::insert(): verID must be >= 0"); - } + if (verID < 0) + { + log("VBBM::insert(): verID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::insert(): verID must be >= 0"); + } - for (i = 0; i < vbbm->nFiles; i++) - if (vbOID == files[i].OID) - break; + for (i = 0; i < vbbm->nFiles; i++) + if (vbOID == files[i].OID) + break; - if (i == vbbm->nFiles) - { - log("VBBM::insert(): vbOID must be the OID of a Version Buffer file", - logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::insert(): vbOID must be the OID of a Version Buffer file"); - } + if (i == vbbm->nFiles) + { + log("VBBM::insert(): vbOID must be the OID of a Version Buffer file", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::insert(): vbOID must be the OID of a Version Buffer file"); + } - if (vbFBO > (files[i].fileSize / BLOCK_SIZE) || vbFBO < 0) - { - log("VBBM::insert(): vbFBO is out of bounds for that vbOID", - logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::insert(): vbFBO is out of bounds for that vbOID"); - } + if (vbFBO > (files[i].fileSize / BLOCK_SIZE) || vbFBO < 0) + { + log("VBBM::insert(): vbFBO is out of bounds for that vbOID", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::insert(): vbFBO is out of bounds for that vbOID"); + } #endif - entry.lbid = lbid; - entry.verID = verID; - entry.vbOID = vbOID; - entry.vbFBO = vbFBO; + entry.lbid = lbid; + entry.verID = verID; + entry.vbOID = vbOID; + entry.vbFBO = vbFBO; - // check for resize - if (vbbm->vbCurrentSize == vbbm->vbCapacity) - growVBBM(); + // check for resize + if (vbbm->vbCurrentSize == vbbm->vbCapacity) + growVBBM(); - _insert(entry, vbbm, hashBuckets, storage, loading); + _insert(entry, vbbm, hashBuckets, storage, loading); - if (!loading) - makeUndoRecord(&vbbm->vbCurrentSize, sizeof(vbbm->vbCurrentSize)); + if (!loading) + makeUndoRecord(&vbbm->vbCurrentSize, sizeof(vbbm->vbCurrentSize)); - vbbm->vbCurrentSize++; + vbbm->vbCurrentSize++; } -//assumes write lock is held and that it is properly sized already -void VBBM::_insert(VBBMEntry& e, VBShmsegHeader* dest, int* destHash, - VBBMEntry* destStorage, bool loading) +// assumes write lock is held and that it is properly sized already +void VBBM::_insert(VBBMEntry& e, VBShmsegHeader* dest, int* destHash, VBBMEntry* destStorage, bool loading) { - int hashIndex, cHashlen = sizeof(LBID_t) + sizeof(VER_t), insertIndex; - char* cHash = (char*)alloca(cHashlen); - utils::Hasher hasher; + int hashIndex, cHashlen = sizeof(LBID_t) + sizeof(VER_t), insertIndex; + char* cHash = (char*)alloca(cHashlen); + utils::Hasher hasher; - memcpy(cHash, &e.lbid, sizeof(LBID_t)); - memcpy(&cHash[sizeof(LBID_t)], &e.verID, sizeof(VER_t)); - hashIndex = hasher(cHash, cHashlen) % dest->numHashBuckets; + memcpy(cHash, &e.lbid, sizeof(LBID_t)); + memcpy(&cHash[sizeof(LBID_t)], &e.verID, sizeof(VER_t)); + hashIndex = hasher(cHash, cHashlen) % dest->numHashBuckets; - insertIndex = dest->vbLWM; + insertIndex = dest->vbLWM; - while (destStorage[insertIndex].lbid != -1) - { - insertIndex++; + while (destStorage[insertIndex].lbid != -1) + { + insertIndex++; #ifdef BRM_DEBUG - if (insertIndex == dest->vbCapacity) - { - log("VBBM:_insert(): There are no empty entries. Possibly bad resize condition.", - logging::LOG_TYPE_DEBUG); - throw logic_error("VBBM:_insert(): There are no empty entries. Possibly bad resize condition."); - } + if (insertIndex == dest->vbCapacity) + { + log("VBBM:_insert(): There are no empty entries. Possibly bad resize condition.", + logging::LOG_TYPE_DEBUG); + throw logic_error("VBBM:_insert(): There are no empty entries. Possibly bad resize condition."); + } #endif - } + } - if (!loading) - { - makeUndoRecord(dest, sizeof(VBShmsegHeader)); - makeUndoRecord(&destStorage[insertIndex], sizeof(VBBMEntry)); - makeUndoRecord(&destHash[hashIndex], sizeof(int)); - } + if (!loading) + { + makeUndoRecord(dest, sizeof(VBShmsegHeader)); + makeUndoRecord(&destStorage[insertIndex], sizeof(VBBMEntry)); + makeUndoRecord(&destHash[hashIndex], sizeof(int)); + } - dest->vbLWM = insertIndex; + dest->vbLWM = insertIndex; - e.next = destHash[hashIndex]; - destStorage[insertIndex] = e; - destHash[hashIndex] = insertIndex; + e.next = destHash[hashIndex]; + destStorage[insertIndex] = e; + destHash[hashIndex] = insertIndex; } -//assumes read lock is held +// assumes read lock is held int VBBM::lookup(LBID_t lbid, VER_t verID, OID_t& oid, uint32_t& fbo) const { - int index, prev, bucket; + int index, prev, bucket; -//#ifdef BRM_DEBUG - if (lbid < 0) - { - log("VBBM::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::lookup(): lbid must be >= 0"); - } + //#ifdef BRM_DEBUG + if (lbid < 0) + { + log("VBBM::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::lookup(): lbid must be >= 0"); + } - if (verID < 0) - { - log("VBBM::lookup(): verID must be > 1)", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::lookup(): verID must be > 1)"); - } + if (verID < 0) + { + log("VBBM::lookup(): verID must be > 1)", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::lookup(): verID must be > 1)"); + } -//#endif + //#endif - index = getIndex(lbid, verID, prev, bucket); + index = getIndex(lbid, verID, prev, bucket); - if (index == -1) - return -1; + if (index == -1) + return -1; - oid = storage[index].vbOID; - fbo = storage[index].vbFBO; - return 0; + oid = storage[index].vbOID; + fbo = storage[index].vbFBO; + return 0; } -//assumes write lock +// assumes write lock void VBBM::getBlocks(int num, OID_t vbOID, vector& freeRanges, VSS& vss, bool flushPMCache) { - int blocksLeftInFile, blocksGathered = 0, i; - uint32_t fileIndex; - uint32_t firstFBO, lastFBO; - VBRange range; - vector::iterator it; - vector flushList; + int blocksLeftInFile, blocksGathered = 0, i; + uint32_t fileIndex; + uint32_t firstFBO, lastFBO; + VBRange range; + vector::iterator it; + vector flushList; - freeRanges.clear(); + freeRanges.clear(); - fileIndex = addVBFileIfNotExists(vbOID); + fileIndex = addVBFileIfNotExists(vbOID); - /* - for (i = 0; i < vbbm->nFiles; i++) { - cout << "file " << i << " vbOID=" << files[i].OID << " size=" << files[i].fileSize - << endl; - } - */ + /* + for (i = 0; i < vbbm->nFiles; i++) { + cout << "file " << i << " vbOID=" << files[i].OID << " size=" << files[i].fileSize + << endl; + } + */ - if ((uint32_t) num > files[fileIndex].fileSize / BLOCK_SIZE) - { - cout << "num = " << num << " filesize = " << files[fileIndex].fileSize << endl; - log("VBBM::getBlocks(): num is larger than the size of the version buffer", - logging::LOG_TYPE_DEBUG); - throw logging::VBBMBufferOverFlowExcept - ("VBBM::getBlocks(): num is larger than the size of the version buffer"); - } + if ((uint32_t)num > files[fileIndex].fileSize / BLOCK_SIZE) + { + cout << "num = " << num << " filesize = " << files[fileIndex].fileSize << endl; + log("VBBM::getBlocks(): num is larger than the size of the version buffer", logging::LOG_TYPE_DEBUG); + throw logging::VBBMBufferOverFlowExcept( + "VBBM::getBlocks(): num is larger than the size of the version buffer"); + } - while ((vbbm->vbCurrentSize + num) > vbbm->vbCapacity) - { - growVBBM(); - //cout << " requested num = " << num << " and Growing vbbm ... " << endl; - } + while ((vbbm->vbCurrentSize + num) > vbbm->vbCapacity) + { + growVBBM(); + // cout << " requested num = " << num << " and Growing vbbm ... " << endl; + } - while (blocksGathered < num) - { - blocksLeftInFile = (files[fileIndex].fileSize - files[fileIndex].nextOffset) / BLOCK_SIZE; - int blocksLeft = num - blocksGathered; + while (blocksGathered < num) + { + blocksLeftInFile = (files[fileIndex].fileSize - files[fileIndex].nextOffset) / BLOCK_SIZE; + int blocksLeft = num - blocksGathered; - range.vbOID = files[fileIndex].OID; - range.vbFBO = files[fileIndex].nextOffset / BLOCK_SIZE; - range.size = (blocksLeftInFile >= blocksLeft ? blocksLeft : blocksLeftInFile); - makeUndoRecord(&files[fileIndex], sizeof(VBFileMetadata)); + range.vbOID = files[fileIndex].OID; + range.vbFBO = files[fileIndex].nextOffset / BLOCK_SIZE; + range.size = (blocksLeftInFile >= blocksLeft ? blocksLeft : blocksLeftInFile); + makeUndoRecord(&files[fileIndex], sizeof(VBFileMetadata)); - if (range.size == (uint32_t) blocksLeftInFile) - files[fileIndex].nextOffset = 0; - else - files[fileIndex].nextOffset += range.size * BLOCK_SIZE; + if (range.size == (uint32_t)blocksLeftInFile) + files[fileIndex].nextOffset = 0; + else + files[fileIndex].nextOffset += range.size * BLOCK_SIZE; - blocksGathered += range.size; - freeRanges.push_back(range); - } + blocksGathered += range.size; + freeRanges.push_back(range); + } - //age the returned blocks out of the VB - for (it = freeRanges.begin(); it != freeRanges.end(); it++) - { - uint32_t firstChunk, lastChunk; + // age the returned blocks out of the VB + for (it = freeRanges.begin(); it != freeRanges.end(); it++) + { + uint32_t firstChunk, lastChunk; - vbOID = it->vbOID; - firstFBO = it->vbFBO; - lastFBO = it->vbFBO + it->size - 1; + vbOID = it->vbOID; + firstFBO = it->vbFBO; + lastFBO = it->vbFBO + it->size - 1; - /* Age out at least 100 blocks at a time to reduce the # of times we have to do it. - * How to detect when it needs to be done and when it doesn't? - * - * Split VB space into 100-block chunks. When a chunk boundary is crossed, - * clear the whole chunk. - */ + /* Age out at least 100 blocks at a time to reduce the # of times we have to do it. + * How to detect when it needs to be done and when it doesn't? + * + * Split VB space into 100-block chunks. When a chunk boundary is crossed, + * clear the whole chunk. + */ - firstChunk = firstFBO / VBBM_CHUNK_SIZE; - lastChunk = lastFBO / VBBM_CHUNK_SIZE; + firstChunk = firstFBO / VBBM_CHUNK_SIZE; + lastChunk = lastFBO / VBBM_CHUNK_SIZE; - // if the current range falls in the middle of a chunk and doesn't span chunks, - // there's nothing to do b/c the chunk is assumed to have been cleared already - if (((firstFBO % VBBM_CHUNK_SIZE) != 0) && (firstChunk == lastChunk)) - continue; + // if the current range falls in the middle of a chunk and doesn't span chunks, + // there's nothing to do b/c the chunk is assumed to have been cleared already + if (((firstFBO % VBBM_CHUNK_SIZE) != 0) && (firstChunk == lastChunk)) + continue; - // round up to the next chunk boundaries - if ((firstFBO % VBBM_CHUNK_SIZE) != 0) // this implies the range spans chunks - firstFBO = (firstChunk + 1) * VBBM_CHUNK_SIZE; // the first FBO of the next chunk + // round up to the next chunk boundaries + if ((firstFBO % VBBM_CHUNK_SIZE) != 0) // this implies the range spans chunks + firstFBO = (firstChunk + 1) * VBBM_CHUNK_SIZE; // the first FBO of the next chunk - lastFBO = ((lastChunk + 1) * VBBM_CHUNK_SIZE - 1); // the last FBO of the last chunk + lastFBO = ((lastChunk + 1) * VBBM_CHUNK_SIZE - 1); // the last FBO of the last chunk - // don't go past the end of the file - if (lastFBO > files[fileIndex].fileSize / BLOCK_SIZE) - lastFBO = files[fileIndex].fileSize / BLOCK_SIZE; + // don't go past the end of the file + if (lastFBO > files[fileIndex].fileSize / BLOCK_SIZE) + lastFBO = files[fileIndex].fileSize / BLOCK_SIZE; - // at this point [firstFBO, lastFBO] is the range to age out. + // at this point [firstFBO, lastFBO] is the range to age out. - // ugh, walk the whole vbbm looking for matches. - for (i = 0; i < vbbm->vbCapacity; i++) - if (storage[i].lbid != -1 && storage[i].vbOID == vbOID && - storage[i].vbFBO >= firstFBO && storage[i].vbFBO <= lastFBO) - { - if (vss.isEntryLocked(storage[i].lbid, storage[i].verID)) - { - ostringstream msg; - msg << "VBBM::getBlocks(): version buffer overflow. Increase VersionBufferFileSize. Overflow occured in aged blocks. Requested NumBlocks:VbOid:vbFBO:lastFBO = " - << num << ":" << vbOID << ":" << firstFBO << ":" << lastFBO << " lbid locked is " << storage[i].lbid << endl; - log(msg.str(), logging::LOG_TYPE_CRITICAL); - freeRanges.clear(); - throw logging::VBBMBufferOverFlowExcept(msg.str()); - } + // ugh, walk the whole vbbm looking for matches. + for (i = 0; i < vbbm->vbCapacity; i++) + if (storage[i].lbid != -1 && storage[i].vbOID == vbOID && storage[i].vbFBO >= firstFBO && + storage[i].vbFBO <= lastFBO) + { + if (vss.isEntryLocked(storage[i].lbid, storage[i].verID)) + { + ostringstream msg; + msg << "VBBM::getBlocks(): version buffer overflow. Increase VersionBufferFileSize. Overflow " + "occured in aged blocks. Requested NumBlocks:VbOid:vbFBO:lastFBO = " + << num << ":" << vbOID << ":" << firstFBO << ":" << lastFBO << " lbid locked is " + << storage[i].lbid << endl; + log(msg.str(), logging::LOG_TYPE_CRITICAL); + freeRanges.clear(); + throw logging::VBBMBufferOverFlowExcept(msg.str()); + } - vss.removeEntry(storage[i].lbid, storage[i].verID, &flushList); - removeEntry(storage[i].lbid, storage[i].verID); - } - } + vss.removeEntry(storage[i].lbid, storage[i].verID, &flushList); + removeEntry(storage[i].lbid, storage[i].verID); + } + } - if (flushPMCache && !flushList.empty()) - cacheutils::flushPrimProcAllverBlocks(flushList); + if (flushPMCache && !flushList.empty()) + cacheutils::flushPrimProcAllverBlocks(flushList); } -//read lock +// read lock int VBBM::getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const { - int cHashlen = sizeof(LBID_t) + sizeof(VER_t), currentIndex; - char* cHash = (char*)alloca(cHashlen); - VBBMEntry* listEntry; - utils::Hasher hasher; + int cHashlen = sizeof(LBID_t) + sizeof(VER_t), currentIndex; + char* cHash = (char*)alloca(cHashlen); + VBBMEntry* listEntry; + utils::Hasher hasher; - memcpy(cHash, &lbid, sizeof(LBID_t)); - memcpy(&cHash[sizeof(LBID_t)], &verID, sizeof(VER_t)); - bucket = hasher(cHash, cHashlen) % vbbm->numHashBuckets; - prev = -1; - - if (hashBuckets[bucket] == -1) - return -1; - - currentIndex = hashBuckets[bucket]; - - while (currentIndex != -1) - { - listEntry = &storage[currentIndex]; - - if (listEntry->lbid == lbid && listEntry->verID == verID) - return currentIndex; - - prev = currentIndex; - currentIndex = listEntry->next; - } + memcpy(cHash, &lbid, sizeof(LBID_t)); + memcpy(&cHash[sizeof(LBID_t)], &verID, sizeof(VER_t)); + bucket = hasher(cHash, cHashlen) % vbbm->numHashBuckets; + prev = -1; + if (hashBuckets[bucket] == -1) return -1; + + currentIndex = hashBuckets[bucket]; + + while (currentIndex != -1) + { + listEntry = &storage[currentIndex]; + + if (listEntry->lbid == lbid && listEntry->verID == verID) + return currentIndex; + + prev = currentIndex; + currentIndex = listEntry->next; + } + + return -1; } void VBBM::removeEntry(LBID_t lbid, VER_t verID) { - int index, prev, bucket; + int index, prev, bucket; #ifdef BRM_DEBUG - if (lbid < 0) - { - log("VBBM::removeEntry(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::removeEntry(): lbid must be >= 0"); - } + if (lbid < 0) + { + log("VBBM::removeEntry(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::removeEntry(): lbid must be >= 0"); + } - if (verID < 0) - { - log("VBBM::removeEntry(): verID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VBBM::removeEntry(): verID must be >= 0"); - } + if (verID < 0) + { + log("VBBM::removeEntry(): verID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VBBM::removeEntry(): verID must be >= 0"); + } #endif - index = getIndex(lbid, verID, prev, bucket); + index = getIndex(lbid, verID, prev, bucket); - if (index == -1) - { + if (index == -1) + { #ifdef BRM_DEBUG - ostringstream ostr; + ostringstream ostr; - ostr << "VBBM::removeEntry(): that entry doesn't exist lbid = " << lbid << - " verID = " << verID << endl; - log(ostr.str(), logging::LOG_TYPE_DEBUG); - throw logic_error(ostr.str()); + ostr << "VBBM::removeEntry(): that entry doesn't exist lbid = " << lbid << " verID = " << verID << endl; + log(ostr.str(), logging::LOG_TYPE_DEBUG); + throw logic_error(ostr.str()); #else - return; + return; #endif - } + } - makeUndoRecord(&storage[index], sizeof(VBBMEntry)); - storage[index].lbid = -1; + makeUndoRecord(&storage[index], sizeof(VBBMEntry)); + storage[index].lbid = -1; - if (prev != -1) - { - makeUndoRecord(&storage[prev], sizeof(VBBMEntry)); - storage[prev].next = storage[index].next; - } - else - { - makeUndoRecord(&hashBuckets[bucket], sizeof(int)); - hashBuckets[bucket] = storage[index].next; - } + if (prev != -1) + { + makeUndoRecord(&storage[prev], sizeof(VBBMEntry)); + storage[prev].next = storage[index].next; + } + else + { + makeUndoRecord(&hashBuckets[bucket], sizeof(int)); + hashBuckets[bucket] = storage[index].next; + } - makeUndoRecord(vbbm, sizeof(VBShmsegHeader)); - vbbm->vbCurrentSize--; + makeUndoRecord(vbbm, sizeof(VBShmsegHeader)); + vbbm->vbCurrentSize--; - if (vbbm->vbLWM > index) - vbbm->vbLWM = index; + if (vbbm->vbLWM > index) + vbbm->vbLWM = index; } -//read lock +// read lock int VBBM::size() const { #ifdef BRM_DEBUG - int i, ret = 0; + int i, ret = 0; - for (i = 0; i < vbbm->vbCapacity; i++) - if (storage[i].lbid != -1) - ret++; + for (i = 0; i < vbbm->vbCapacity; i++) + if (storage[i].lbid != -1) + ret++; - if (ret != vbbm->vbCurrentSize) - { - ostringstream ostr; + if (ret != vbbm->vbCurrentSize) + { + ostringstream ostr; - ostr << "VBBM::checkConsistency(): actual size is " << ret << - ", recorded size is " << vbbm->vbCurrentSize; - log(ostr.str(), logging::LOG_TYPE_DEBUG); - throw logic_error(ostr.str()); - } + ostr << "VBBM::checkConsistency(): actual size is " << ret << ", recorded size is " + << vbbm->vbCurrentSize; + log(ostr.str(), logging::LOG_TYPE_DEBUG); + throw logic_error(ostr.str()); + } - return ret; + return ret; #else - return vbbm->vbCurrentSize; + return vbbm->vbCurrentSize; #endif } -//read lock +// read lock bool VBBM::hashEmpty() const { - int i; + int i; - for (i = 0; i < vbbm->numHashBuckets; i++) - if (hashBuckets[i] != -1) - return false; + for (i = 0; i < vbbm->numHashBuckets; i++) + if (hashBuckets[i] != -1) + return false; - return true; + return true; } -//write lock +// write lock void VBBM::clear() { - int allocSize; - int newshmkey; - int nFiles = -1; - char* newshmseg; + int allocSize; + int newshmkey; + int nFiles = -1; + char* newshmseg; - // save vars we need after the clear() - boost::scoped_array newFiles(new VBFileMetadata[vbbm->nFiles]); - memcpy(&newFiles[0], files, vbbm->nFiles * sizeof(VBFileMetadata)); + // save vars we need after the clear() + boost::scoped_array newFiles(new VBFileMetadata[vbbm->nFiles]); + memcpy(&newFiles[0], files, vbbm->nFiles * sizeof(VBFileMetadata)); - setCurrentFileSize(); + setCurrentFileSize(); - for (int i = 0; i < vbbm->nFiles; i++) - { - newFiles[i].fileSize = currentFileSize; - newFiles[i].nextOffset = 0; - } + for (int i = 0; i < vbbm->nFiles; i++) + { + newFiles[i].fileSize = currentFileSize; + newFiles[i].nextOffset = 0; + } - nFiles = vbbm->nFiles; + nFiles = vbbm->nFiles; - allocSize = (sizeof(VBShmsegHeader) + - (nFiles * sizeof(VBFileMetadata)) + - VBSTORAGE_INITIAL_SIZE + VBTABLE_INITIAL_SIZE); - //cout << "clear:: allocSize = " << allocSize << endl; - newshmkey = chooseShmkey(); - fPVBBMImpl->clear(newshmkey, allocSize); - vbbm = fPVBBMImpl->get(); - newshmseg = reinterpret_cast(vbbm); - initShmseg(nFiles); - vbbmShminfo->tableShmkey = currentVBBMShmkey = newshmkey; - vbbmShminfo->allocdSize = allocSize; + allocSize = (sizeof(VBShmsegHeader) + (nFiles * sizeof(VBFileMetadata)) + VBSTORAGE_INITIAL_SIZE + + VBTABLE_INITIAL_SIZE); + // cout << "clear:: allocSize = " << allocSize << endl; + newshmkey = chooseShmkey(); + fPVBBMImpl->clear(newshmkey, allocSize); + vbbm = fPVBBMImpl->get(); + newshmseg = reinterpret_cast(vbbm); + initShmseg(nFiles); + vbbmShminfo->tableShmkey = currentVBBMShmkey = newshmkey; + vbbmShminfo->allocdSize = allocSize; - files = reinterpret_cast - (&newshmseg[sizeof(VBShmsegHeader)]); - hashBuckets = reinterpret_cast - (&newshmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata)]); - storage = reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + - vbbm->nFiles * sizeof(VBFileMetadata) + - vbbm->numHashBuckets * sizeof(int)]); - memcpy(files, &newFiles[0], vbbm->nFiles * sizeof(VBFileMetadata)); + files = reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader)]); + hashBuckets = + reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata)]); + storage = + reinterpret_cast(&newshmseg[sizeof(VBShmsegHeader) + vbbm->nFiles * sizeof(VBFileMetadata) + + vbbm->numHashBuckets * sizeof(int)]); + memcpy(files, &newFiles[0], vbbm->nFiles * sizeof(VBFileMetadata)); } -//read lock +// read lock int VBBM::checkConsistency() const { + /* - /* + Struct integrity tests + 1: Verify that the recorded size matches the actual size + 2: Verify there are no empty entries reachable from the hash table + 3: Verify there are no empty entries below the LWM - Struct integrity tests - 1: Verify that the recorded size matches the actual size - 2: Verify there are no empty entries reachable from the hash table - 3: Verify there are no empty entries below the LWM + 4a: Make sure every VBBM entry points to a unique position in the VB + 4b: Make sure every VBBM entry has unique LBID & VERID. + */ - 4a: Make sure every VBBM entry points to a unique position in the VB - 4b: Make sure every VBBM entry has unique LBID & VERID. - */ + int i, j, k; - int i, j, k; + /* Test 1 is already implemented */ + size(); - /* Test 1 is already implemented */ - size(); + /* Test 2 - no empty elements reachable from the hash table */ - /* Test 2 - no empty elements reachable from the hash table */ + int nextElement; - int nextElement; + for (i = 0; i < vbbm->numHashBuckets; i++) + { + if (hashBuckets[i] != -1) + for (nextElement = hashBuckets[i]; nextElement != -1; nextElement = storage[nextElement].next) + if (storage[nextElement].lbid == -1) + throw logic_error( + "VBBM::checkConsistency(): an empty storage entry is reachable from the hash table"); + } - for (i = 0; i < vbbm->numHashBuckets; i++) + /* Test 3 - verify that there are no empty entries below the LWM */ + + for (i = 0; i < vbbm->vbLWM; i++) + { + if (storage[i].lbid == -1) { - if (hashBuckets[i] != -1) - for (nextElement = hashBuckets[i]; nextElement != -1; - nextElement = storage[nextElement].next) - if (storage[nextElement].lbid == -1) - throw logic_error("VBBM::checkConsistency(): an empty storage entry is reachable from the hash table"); + cerr << "VBBM: LWM=" << vbbm->vbLWM << " first empty entry=" << i << endl; + throw logic_error("VBBM::checkConsistency(): LWM accounting error"); } + } - /* Test 3 - verify that there are no empty entries below the LWM */ + /* Test 4b - verify the uniqueness of the entries */ - for (i = 0; i < vbbm->vbLWM; i++) - { - if (storage[i].lbid == -1) - { - cerr << "VBBM: LWM=" << vbbm->vbLWM << " first empty entry=" << i << endl; - throw logic_error("VBBM::checkConsistency(): LWM accounting error"); - } - } + for (i = 0; i < vbbm->numHashBuckets; i++) + if (hashBuckets[i] != -1) + for (j = hashBuckets[i]; j != -1; j = storage[j].next) + for (k = storage[j].next; k != -1; k = storage[k].next) + if (storage[j].lbid == storage[k].lbid && storage[j].verID == storage[k].verID) + { + cerr << "VBBM: lbid=" << storage[j].lbid << " verID=" << storage[j].verID << endl; + throw logic_error("VBBM::checkConsistency(): Duplicate entry found"); + } - /* Test 4b - verify the uniqueness of the entries */ + /* Test 4a - verify the uniqueness of vbOID, vbFBO fields */ + for (i = 0; i < vbbm->vbCapacity; i++) + if (storage[i].lbid != -1) + for (j = i + 1; j < vbbm->vbCapacity; j++) + if (storage[j].lbid != -1) + if (storage[j].vbOID == storage[i].vbOID && storage[j].vbFBO == storage[i].vbFBO) + { + cerr << "VBBM: lbid1=" << storage[i].lbid << " lbid2=" << storage[j].lbid + << " verID1=" << storage[i].verID << " verID2=" << storage[j].verID + << " share vbOID=" << storage[j].vbOID << " vbFBO=" << storage[j].vbFBO << endl; + throw logic_error("VBBM::checkConsistency(): 2 VBBM entries share space in the VB"); + } - for (i = 0; i < vbbm->numHashBuckets; i++) - if (hashBuckets[i] != -1) - for (j = hashBuckets[i]; j != -1; j = storage[j].next) - for (k = storage[j].next; k != -1; k = storage[k].next) - if (storage[j].lbid == storage[k].lbid && - storage[j].verID == storage[k].verID) - { - cerr << "VBBM: lbid=" << storage[j].lbid << " verID=" << - storage[j].verID << endl; - throw logic_error("VBBM::checkConsistency(): Duplicate entry found"); - } - - /* Test 4a - verify the uniqueness of vbOID, vbFBO fields */ - for (i = 0; i < vbbm->vbCapacity; i++) - if (storage[i].lbid != -1) - for (j = i + 1; j < vbbm->vbCapacity; j++) - if (storage[j].lbid != -1) - if (storage[j].vbOID == storage[i].vbOID && - storage[j].vbFBO == storage[i].vbFBO) - { - cerr << "VBBM: lbid1=" << storage[i].lbid << " lbid2=" << - storage[j].lbid << " verID1=" << storage[i].verID << - " verID2=" << storage[j].verID << " share vbOID=" << - storage[j].vbOID << " vbFBO=" << storage[j].vbFBO << - endl; - throw logic_error("VBBM::checkConsistency(): 2 VBBM entries share space in the VB"); - } - - return 0; + return 0; } void VBBM::setReadOnly() { - r_only = true; + r_only = true; } /* File Format (V1) - VBBM V1 magic (32-bits) - # of VBBM entries in capacity (32-bits) - struct VBBMEntry * # + VBBM V1 magic (32-bits) + # of VBBM entries in capacity (32-bits) + struct VBBMEntry * # - These entries are considered optional to support going backward - and forward between versions. - nFiles (32-bits) - currentFileIndex (32-bits) - VBFileMetadata * nFiles + These entries are considered optional to support going backward + and forward between versions. + nFiles (32-bits) + currentFileIndex (32-bits) + VBFileMetadata * nFiles */ /* File Format (V2): @@ -979,188 +954,179 @@ void VBBM::setReadOnly() * struct VBBMEntry * numEntries */ - void VBBM::loadVersion2(IDBDataFile* in) { - int vbbmEntries; - int nFiles; - int i; - VBBMEntry entry; + int vbbmEntries; + int nFiles; + int i; + VBBMEntry entry; - if (in->read((char*) &vbbmEntries, 4) != 4) + if (in->read((char*)&vbbmEntries, 4) != 4) + { + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to read entry number"); + } + + if (in->read((char*)&nFiles, 4) != 4) + { + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to read file number"); + } + + // Need to make clear() truncate the files section + if (vbbm->nFiles > nFiles) + vbbm->nFiles = nFiles; + + clear(); + + while (vbbm->nFiles < nFiles) + growVBBM(true); // this allocates one file, doesn't grow the main storage + + growForLoad(vbbmEntries); + + const int nfileSize = sizeof(VBFileMetadata) * nFiles; + + if (in->read((char*)files, nfileSize) != nfileSize) + { + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to load vb file meta data"); + } + + size_t readSize = vbbmEntries * sizeof(entry); + char* readBuf = new char[readSize]; + size_t progress = 0; + int err; + while (progress < readSize) + { + err = in->read(readBuf + progress, readSize - progress); + if (err < 0) { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to read entry number"); + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to load, check the critical log file"); } - - if (in->read((char*) &nFiles, 4) != 4) + else if (err == 0) { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to read file number"); + log("VBBM::load(): Got early EOF"); + throw runtime_error("VBBM::load(): Got early EOF"); } + progress += err; + } - // Need to make clear() truncate the files section - if (vbbm->nFiles > nFiles) - vbbm->nFiles = nFiles; - - clear(); - - while (vbbm->nFiles < nFiles) - growVBBM(true); // this allocates one file, doesn't grow the main storage - - growForLoad(vbbmEntries); - - const int nfileSize = sizeof(VBFileMetadata) * nFiles; - - if (in->read((char*)files, nfileSize) != nfileSize) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to load vb file meta data"); - } - - size_t readSize = vbbmEntries * sizeof(entry); - char *readBuf = new char[readSize]; - size_t progress = 0; - int err; - while (progress < readSize) - { - err = in->read(readBuf + progress, readSize - progress); - if (err < 0) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to load, check the critical log file"); - } - else if (err == 0) - { - log("VBBM::load(): Got early EOF"); - throw runtime_error("VBBM::load(): Got early EOF"); - } - progress += err; - } - - VBBMEntry *loadedEntries = (VBBMEntry *) readBuf; - for (i = 0; i < vbbmEntries; i++) - insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbOID, - loadedEntries[i].vbFBO, true); - + VBBMEntry* loadedEntries = (VBBMEntry*)readBuf; + for (i = 0; i < vbbmEntries; i++) + insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbOID, loadedEntries[i].vbFBO, + true); } //#include "boost/date_time/posix_time/posix_time.hpp" -//using namespace boost::posix_time; +// using namespace boost::posix_time; void VBBM::load(string filename) { - int magic; - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "rb", 0)); - //ptime time1, time2; + int magic; + const char* filename_p = filename.c_str(); + scoped_ptr in( + IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "rb", 0)); + // ptime time1, time2; - //time1 = microsec_clock::local_time(); - //cout << "loading the VBBM " << time1 << endl; + // time1 = microsec_clock::local_time(); + // cout << "loading the VBBM " << time1 << endl; - if (!in) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to open the file"); - } + if (!in) + { + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to open the file"); + } - int bytes = in->read((char*) &magic, 4); + int bytes = in->read((char*)&magic, 4); - if (bytes != 4) - { - log("VBBM::load(): failed to read magic."); - throw runtime_error("VBBM::load(): failed to read magic."); - } + if (bytes != 4) + { + log("VBBM::load(): failed to read magic."); + throw runtime_error("VBBM::load(): failed to read magic."); + } - switch (magic) - { + switch (magic) + { + case VBBM_MAGIC_V2: loadVersion2(in.get()); break; - case VBBM_MAGIC_V2: - loadVersion2(in.get()); - break; + default: + log("VBBM::load(): Bad magic. Not a VBBM file?"); + throw runtime_error("VBBM::load(): Bad magic. Not a VBBM file?"); + } - default: - log("VBBM::load(): Bad magic. Not a VBBM file?"); - throw runtime_error("VBBM::load(): Bad magic. Not a VBBM file?"); - } - - //time2 = microsec_clock::local_time(); - //cout << "done loading " << time2 << " duration: " << time2-time1 << endl; + // time2 = microsec_clock::local_time(); + // cout << "done loading " << time2 << " duration: " << time2-time1 << endl; } // read lock void VBBM::save(string filename) { - int i; - int var; + int i; + int var; - const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", IDBDataFile::USE_VBUF)); + const char* filename_p = filename.c_str(); + scoped_ptr out(IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", IDBDataFile::USE_VBUF)); - if (!out) + if (!out) + { + log_errno("VBBM::save()"); + throw runtime_error("VBBM::save(): Failed to open the file"); + } + + var = VBBM_MAGIC_V2; + int bytesWritten = 0; + int bytesToWrite = 12; + bytesWritten += out->write((char*)&var, 4); + bytesWritten += out->write((char*)&vbbm->vbCurrentSize, 4); + bytesWritten += out->write((char*)&vbbm->nFiles, 4); + + bytesWritten += out->write((char*)files, sizeof(VBFileMetadata) * vbbm->nFiles); + bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles; + + int first = -1, last = -1, err; + size_t progress, writeSize; + + for (i = 0; i < vbbm->vbCapacity; i++) + { + if (storage[i].lbid != -1 && first == -1) + first = i; + else if (storage[i].lbid == -1 && first != -1) { + last = i; + writeSize = (last - first) * sizeof(VBBMEntry); + progress = 0; + char* writePos = (char*)&storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VBBM::save()"); + throw runtime_error("VBBM::save(): Failed to write the file"); + } + progress += err; + } + first = -1; + } + } + if (first != -1) + { + writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry); + progress = 0; + char* writePos = (char*)&storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to open the file"); + throw runtime_error("VBBM::save(): Failed to write the file"); + } + progress += err; } - - var = VBBM_MAGIC_V2; - int bytesWritten = 0; - int bytesToWrite = 12; - bytesWritten += out->write((char*) &var, 4); - bytesWritten += out->write((char*) &vbbm->vbCurrentSize, 4); - bytesWritten += out->write((char*) &vbbm->nFiles, 4); - - bytesWritten += out->write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles); - bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles; - - int first = -1, last = -1, err; - size_t progress, writeSize; - - for (i = 0; i < vbbm->vbCapacity; i++) - { - if (storage[i].lbid != -1 && first == -1) - first = i; - else if (storage[i].lbid == -1 && first != -1) - { - last = i; - writeSize = (last - first) * sizeof(VBBMEntry); - progress = 0; - char *writePos = (char *) &storage[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to write the file"); - } - progress += err; - } - first = -1; - } - - } - if (first != -1) - { - writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry); - progress = 0; - char *writePos = (char *) &storage[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to write the file"); - } - progress += err; - } - } - + } #if 0 cout << "saving... nfiles=" << vbbm->nFiles << "\n"; @@ -1175,65 +1141,65 @@ void VBBM::save(string filename) uint32_t VBBM::addVBFileIfNotExists(OID_t vbOID) { - int i; + int i; - /* Check if vbOID exists, - * add it if not, init to pos=0 - */ + /* Check if vbOID exists, + * add it if not, init to pos=0 + */ - for (i = 0; i < vbbm->nFiles; i++) - if (files[i].OID == vbOID) - break; + for (i = 0; i < vbbm->nFiles; i++) + if (files[i].OID == vbOID) + break; - if (i == vbbm->nFiles) - { - setCurrentFileSize(); - growVBBM(true); - files[i].OID = vbOID; - files[i].fileSize = currentFileSize; - files[i].nextOffset = 0; - } + if (i == vbbm->nFiles) + { + setCurrentFileSize(); + growVBBM(true); + files[i].OID = vbOID; + files[i].fileSize = currentFileSize; + files[i].nextOffset = 0; + } - return i; + return i; } void VBBM::setCurrentFileSize() { - config::Config* conf = config::Config::makeConfig(); - string stmp; - int64_t ltmp; + config::Config* conf = config::Config::makeConfig(); + string stmp; + int64_t ltmp; - currentFileSize = 2147483648ULL; // 2 GB default + currentFileSize = 2147483648ULL; // 2 GB default - try - { - stmp = conf->getConfig("VersionBuffer", "VersionBufferFileSize"); - } - catch (std::exception& e) - { - log("VBBM: Missing a VersionBuffer/VersionBufferFileSize key in the config file"); - throw invalid_argument("VBBM: Missing a VersionBuffer/VersionBufferFileSize key in the config file"); - } + try + { + stmp = conf->getConfig("VersionBuffer", "VersionBufferFileSize"); + } + catch (std::exception& e) + { + log("VBBM: Missing a VersionBuffer/VersionBufferFileSize key in the config file"); + throw invalid_argument("VBBM: Missing a VersionBuffer/VersionBufferFileSize key in the config file"); + } - ltmp = conf->fromText(stmp.c_str()); + ltmp = conf->fromText(stmp.c_str()); - if (ltmp < 1) - { - log("VBBM: Config error: VersionBuffer/VersionBufferFileSize must be positive"); - throw invalid_argument("VBBM: Config error: VersionBuffer/VersionBufferFileSize must be positive"); - } - else - { - currentFileSize = ltmp; - } + if (ltmp < 1) + { + log("VBBM: Config error: VersionBuffer/VersionBufferFileSize must be positive"); + throw invalid_argument("VBBM: Config error: VersionBuffer/VersionBufferFileSize must be positive"); + } + else + { + currentFileSize = ltmp; + } } #ifdef BRM_DEBUG // read lock int VBBM::getShmid() const { - return vbbmShmid; + return vbbmShmid; } #endif -} //namespace +} // namespace BRM diff --git a/versioning/BRM/vbbm.h b/versioning/BRM/vbbm.h index b2e430d3e..1411398d4 100644 --- a/versioning/BRM/vbbm.h +++ b/versioning/BRM/vbbm.h @@ -41,22 +41,21 @@ // These config parameters need to be loaded -//will get a small hash function performance boost by using powers of 2 +// will get a small hash function performance boost by using powers of 2 #define VBSTORAGE_INITIAL_COUNT 100000 -#define VBSTORAGE_INITIAL_SIZE (VBSTORAGE_INITIAL_COUNT*sizeof(VBBMEntry)) +#define VBSTORAGE_INITIAL_SIZE (VBSTORAGE_INITIAL_COUNT * sizeof(VBBMEntry)) #define VBSTORAGE_INCREMENT_COUNT 10000 -#define VBSTORAGE_INCREMENT (VBSTORAGE_INCREMENT_COUNT*sizeof(VBBMEntry)) +#define VBSTORAGE_INCREMENT (VBSTORAGE_INCREMENT_COUNT * sizeof(VBBMEntry)) // (average list length = 4) -#define VBTABLE_INITIAL_SIZE (25000*sizeof(int)) -#define VBTABLE_INCREMENT (2500*sizeof(int)) +#define VBTABLE_INITIAL_SIZE (25000 * sizeof(int)) +#define VBTABLE_INCREMENT (2500 * sizeof(int)) #define VBBM_INCREMENT (VBTABLE_INCREMENT + VBSTORAGE_INCREMENT) -#define VBBM_SIZE(files, entries) \ - ((entries * sizeof(VBBMEntry)) + (entries/4 * sizeof(int)) \ - + (files * sizeof(VBFileMetadata)) + sizeof(VBShmsegHeader)) - +#define VBBM_SIZE(files, entries) \ + ((entries * sizeof(VBBMEntry)) + (entries / 4 * sizeof(int)) + (files * sizeof(VBFileMetadata)) + \ + sizeof(VBShmsegHeader)) #if defined(_MSC_VER) && defined(xxxVBBM_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -64,7 +63,6 @@ #define EXPORT #endif - namespace idbdatafile { class IDBDataFile; @@ -72,89 +70,88 @@ class IDBDataFile; namespace BRM { - class VSS; struct VBFileMetadata { - OID_t OID; - uint64_t fileSize; - uint64_t nextOffset; + OID_t OID; + uint64_t fileSize; + uint64_t nextOffset; }; struct VBBMEntry { - LBID_t lbid; - VER_t verID; - OID_t vbOID; - uint32_t vbFBO; - int next; - EXPORT VBBMEntry(); + LBID_t lbid; + VER_t verID; + OID_t vbOID; + uint32_t vbFBO; + int next; + EXPORT VBBMEntry(); }; struct VBShmsegHeader { - int nFiles; - int vbCapacity; - int vbCurrentSize; - int vbLWM; - int numHashBuckets; + int nFiles; + int vbCapacity; + int vbCurrentSize; + int vbLWM; + int numHashBuckets; - // the rest of the overlay looks like this -// VBFileMetadata files[nFiles]; -// int hashBuckets[numHashBuckets]; -// VBBMEntry storage[vbCapacity]; + // the rest of the overlay looks like this + // VBFileMetadata files[nFiles]; + // int hashBuckets[numHashBuckets]; + // VBBMEntry storage[vbCapacity]; }; class VBBMImpl { -public: - static VBBMImpl* makeVBBMImpl(unsigned key, off_t size, bool readOnly = false); + public: + static VBBMImpl* makeVBBMImpl(unsigned key, off_t size, bool readOnly = false); - inline void grow(unsigned key, off_t size) + inline void grow(unsigned key, off_t size) #ifdef NDEBUG - { - fVBBM.grow(key, size); - } + { + fVBBM.grow(key, size); + } #else - { - int rc = fVBBM.grow(key, size); - idbassert(rc == 0); - } + { + int rc = fVBBM.grow(key, size); + idbassert(rc == 0); + } #endif - inline void makeReadOnly() - { - fVBBM.setReadOnly(); - } - inline void clear(unsigned key, off_t size) - { - fVBBM.clear(key, size); - } - inline void swapout(BRMShmImpl& rhs) - { - fVBBM.swap(rhs); - rhs.destroy(); - } - inline unsigned key() const - { - return fVBBM.key(); - } + inline void makeReadOnly() + { + fVBBM.setReadOnly(); + } + inline void clear(unsigned key, off_t size) + { + fVBBM.clear(key, size); + } + inline void swapout(BRMShmImpl& rhs) + { + fVBBM.swap(rhs); + rhs.destroy(); + } + inline unsigned key() const + { + return fVBBM.key(); + } - inline VBShmsegHeader* get() const - { - return reinterpret_cast(fVBBM.fMapreg.get_address()); - } + inline VBShmsegHeader* get() const + { + return reinterpret_cast(fVBBM.fMapreg.get_address()); + } -private: - VBBMImpl(unsigned key, off_t size, bool readOnly = false); - ~VBBMImpl(); - VBBMImpl(const VBBMImpl& rhs); - VBBMImpl& operator=(const VBBMImpl& rhs); + private: + VBBMImpl(unsigned key, off_t size, bool readOnly = false); + ~VBBMImpl(); + VBBMImpl(const VBBMImpl& rhs); + VBBMImpl& operator=(const VBBMImpl& rhs); - BRMShmImpl fVBBM; + BRMShmImpl fVBBM; - static boost::mutex fInstanceMutex; - static VBBMImpl* fInstance; + static boost::mutex fInstanceMutex; + static VBBMImpl* fInstance; }; /** @brief The Version Buffer Block Map (VBBM) @@ -187,78 +184,75 @@ private: class VBBM : public Undoable { -public: - enum OPS - { - NONE, - READ, - WRITE - }; + public: + enum OPS + { + NONE, + READ, + WRITE + }; + EXPORT VBBM(); + EXPORT ~VBBM(); - EXPORT VBBM(); - EXPORT ~VBBM(); + EXPORT void lock(OPS op); + EXPORT void release(OPS op); + EXPORT int lookup(LBID_t lbid, VER_t ver, OID_t& oid, uint32_t& fbo) const; + EXPORT void insert(LBID_t lbid, VER_t ver, OID_t oid, uint32_t fbo, bool loading = false); + EXPORT void getBlocks(int num, OID_t vbOID, std::vector& vbRanges, VSS& vss, bool flushPMCache); + EXPORT void removeEntry(LBID_t, VER_t ver); - EXPORT void lock(OPS op); - EXPORT void release(OPS op); - EXPORT int lookup(LBID_t lbid, VER_t ver, OID_t& oid, uint32_t& fbo) const; - EXPORT void insert(LBID_t lbid, VER_t ver, OID_t oid, uint32_t fbo, bool loading = false); - EXPORT void getBlocks(int num, OID_t vbOID, std::vector& vbRanges, VSS& vss, - bool flushPMCache); - EXPORT void removeEntry(LBID_t, VER_t ver); + EXPORT int size() const; + EXPORT bool hashEmpty() const; + EXPORT int checkConsistency() const; + EXPORT void setReadOnly(); - EXPORT int size() const; - EXPORT bool hashEmpty() const; - EXPORT int checkConsistency() const; - EXPORT void setReadOnly(); - - EXPORT void clear(); - EXPORT void load(std::string filename); - EXPORT void loadVersion2(idbdatafile::IDBDataFile* in); - EXPORT void save(std::string filename); + EXPORT void clear(); + EXPORT void load(std::string filename); + EXPORT void loadVersion2(idbdatafile::IDBDataFile* in); + EXPORT void save(std::string filename); #ifdef BRM_DEBUG - EXPORT int getShmid() const; + EXPORT int getShmid() const; #endif -private: - VBBM(const VBBM&); - VBBM& operator=(const VBBM&); + private: + VBBM(const VBBM&); + VBBM& operator=(const VBBM&); - VBShmsegHeader* vbbm; - VBFileMetadata* files; - int* hashBuckets; - VBBMEntry* storage; + VBShmsegHeader* vbbm; + VBFileMetadata* files; + int* hashBuckets; + VBBMEntry* storage; - key_t currentVBBMShmkey; - int vbbmShmid; - bool r_only; - MSTEntry* vbbmShminfo; - MasterSegmentTable mst; - static boost::mutex mutex; // @bug5355 - made mutex static - static const int MAX_IO_RETRIES = 10; + key_t currentVBBMShmkey; + int vbbmShmid; + bool r_only; + MSTEntry* vbbmShminfo; + MasterSegmentTable mst; + static boost::mutex mutex; // @bug5355 - made mutex static + static const int MAX_IO_RETRIES = 10; - key_t chooseShmkey() const; - void growVBBM(bool addAFile = false); - void growForLoad(int count); - void copyVBBM(VBShmsegHeader* dest); - void initShmseg(int nFiles); + key_t chooseShmkey() const; + void growVBBM(bool addAFile = false); + void growForLoad(int count); + void copyVBBM(VBShmsegHeader* dest); + void initShmseg(int nFiles); - void _insert(VBBMEntry& e, VBShmsegHeader* dest, int* destTable, VBBMEntry* - destStorage, bool loading = false); - int getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const; - ShmKeys fShmKeys; - VBBMImpl* fPVBBMImpl; + void _insert(VBBMEntry& e, VBShmsegHeader* dest, int* destTable, VBBMEntry* destStorage, + bool loading = false); + int getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const; + ShmKeys fShmKeys; + VBBMImpl* fPVBBMImpl; - /* Shared nothing mods */ - uint64_t currentFileSize; - void setCurrentFileSize(); - uint32_t addVBFileIfNotExists(OID_t vbOID); + /* Shared nothing mods */ + uint64_t currentFileSize; + void setCurrentFileSize(); + uint32_t addVBFileIfNotExists(OID_t vbOID); }; -} +} // namespace BRM #undef EXPORT -#endif // _VBBM_H_ - +#endif // _VBBM_H_ diff --git a/versioning/BRM/vss.cpp b/versioning/BRM/vss.cpp index c225560a3..3b8cfcc04 100644 --- a/versioning/BRM/vss.cpp +++ b/versioning/BRM/vss.cpp @@ -45,12 +45,11 @@ #include "IDBDataFile.h" #include "IDBPolicy.h" - #define VSS_DLLEXPORT #include "vss.h" #undef VSS_DLLEXPORT -#define VSS_MAGIC_V1 0x7218db12 +#define VSS_MAGIC_V1 0x7218db12 using namespace std; using namespace boost; @@ -58,14 +57,13 @@ using namespace idbdatafile; namespace BRM { - VSSEntry::VSSEntry() { - lbid = -1; - verID = 0; - vbFlag = false; - locked = false; - next = -1; + lbid = -1; + verID = 0; + vbFlag = false; + locked = false; + next = -1; } /*static*/ @@ -78,40 +76,39 @@ VSSImpl* VSSImpl::fInstance = 0; /*static*/ VSSImpl* VSSImpl::makeVSSImpl(unsigned key, off_t size, bool readOnly) { - boost::mutex::scoped_lock lk(fInstanceMutex); + boost::mutex::scoped_lock lk(fInstanceMutex); - if (fInstance) + if (fInstance) + { + if (key != fInstance->fVSS.key()) { - if (key != fInstance->fVSS.key()) - { - BRMShmImpl newShm(key, size); - fInstance->swapout(newShm); - } - - idbassert(key == fInstance->fVSS.key()); - return fInstance; + BRMShmImpl newShm(key, size); + fInstance->swapout(newShm); } - fInstance = new VSSImpl(key, size, readOnly); - + idbassert(key == fInstance->fVSS.key()); return fInstance; + } + + fInstance = new VSSImpl(key, size, readOnly); + + return fInstance; } -VSSImpl::VSSImpl(unsigned key, off_t size, bool readOnly) : - fVSS(key, size, readOnly) +VSSImpl::VSSImpl(unsigned key, off_t size, bool readOnly) : fVSS(key, size, readOnly) { } VSS::VSS() { - vss = 0; - hashBuckets = 0; - storage = 0; - currentVSSShmkey = -1; - vssShmid = 0; - vssShminfo = NULL; - r_only = false; - fPVSSImpl = 0; + vss = 0; + hashBuckets = 0; + storage = 0; + currentVSSShmkey = -1; + vssShmid = 0; + vssShminfo = NULL; + r_only = false; + fPVSSImpl = 0; } VSS::~VSS() @@ -121,1084 +118,1056 @@ VSS::~VSS() // ported from ExtentMap void VSS::lock(OPS op) { - char* shmseg; + char* shmseg; + + if (op == READ) + { + vssShminfo = mst.getTable_read(MasterSegmentTable::VSSSegment); + mutex.lock(); + } + else + vssShminfo = mst.getTable_write(MasterSegmentTable::VSSSegment); + + // this means that either the VSS isn't attached or that it was resized + if (!fPVSSImpl || fPVSSImpl->key() != (unsigned)vssShminfo->tableShmkey) + { + if (vssShminfo->allocdSize == 0) + { + if (op == READ) + { + mutex.unlock(); + mst.getTable_upgrade(MasterSegmentTable::VSSSegment); + + try + { + growVSS(); + } + catch (...) + { + release(WRITE); + throw; + } + + mst.getTable_downgrade(MasterSegmentTable::VSSSegment); + } + else + { + try + { + growVSS(); + } + catch (...) + { + release(WRITE); + throw; + } + } + } + else + { + fPVSSImpl = VSSImpl::makeVSSImpl(vssShminfo->tableShmkey, 0); + idbassert(fPVSSImpl); + + if (r_only) + fPVSSImpl->makeReadOnly(); + + vss = fPVSSImpl->get(); + shmseg = reinterpret_cast(vss); + hashBuckets = reinterpret_cast(&shmseg[sizeof(VSSShmsegHeader)]); + storage = + reinterpret_cast(&shmseg[sizeof(VSSShmsegHeader) + vss->numHashBuckets * sizeof(int)]); + + if (op == READ) + mutex.unlock(); + } + } + else + { + vss = fPVSSImpl->get(); + shmseg = reinterpret_cast(vss); + hashBuckets = reinterpret_cast(&shmseg[sizeof(VSSShmsegHeader)]); + storage = + reinterpret_cast(&shmseg[sizeof(VSSShmsegHeader) + vss->numHashBuckets * sizeof(int)]); if (op == READ) - { - vssShminfo = mst.getTable_read(MasterSegmentTable::VSSSegment); - mutex.lock(); - } - else - vssShminfo = mst.getTable_write(MasterSegmentTable::VSSSegment); - - // this means that either the VSS isn't attached or that it was resized - if (!fPVSSImpl || fPVSSImpl->key() != (unsigned)vssShminfo->tableShmkey) - { - if (vssShminfo->allocdSize == 0) - { - if (op == READ) - { - mutex.unlock(); - mst.getTable_upgrade(MasterSegmentTable::VSSSegment); - - try - { - growVSS(); - } - catch (...) - { - release(WRITE); - throw; - } - - mst.getTable_downgrade(MasterSegmentTable::VSSSegment); - } - else - { - try - { - growVSS(); - } - catch (...) - { - release(WRITE); - throw; - } - } - } - else - { - fPVSSImpl = VSSImpl::makeVSSImpl(vssShminfo->tableShmkey, 0); - idbassert(fPVSSImpl); - - if (r_only) - fPVSSImpl->makeReadOnly(); - - vss = fPVSSImpl->get(); - shmseg = reinterpret_cast(vss); - hashBuckets = reinterpret_cast - (&shmseg[sizeof(VSSShmsegHeader)]); - storage = reinterpret_cast(&shmseg[sizeof(VSSShmsegHeader) + - vss->numHashBuckets * sizeof(int)]); - - if (op == READ) - mutex.unlock(); - } - } - else - { - vss = fPVSSImpl->get(); - shmseg = reinterpret_cast(vss); - hashBuckets = reinterpret_cast - (&shmseg[sizeof(VSSShmsegHeader)]); - storage = reinterpret_cast(&shmseg[sizeof(VSSShmsegHeader) + - vss->numHashBuckets * sizeof(int)]); - - if (op == READ) - mutex.unlock(); - } + mutex.unlock(); + } } // ported from ExtentMap void VSS::release(OPS op) { - if (op == READ) - mst.releaseTable_read(MasterSegmentTable::VSSSegment); - else - mst.releaseTable_write(MasterSegmentTable::VSSSegment); + if (op == READ) + mst.releaseTable_read(MasterSegmentTable::VSSSegment); + else + mst.releaseTable_write(MasterSegmentTable::VSSSegment); } void VSS::initShmseg() { - int i; - char* newshmseg; - int* buckets; - VSSEntry* stor; + int i; + char* newshmseg; + int* buckets; + VSSEntry* stor; - vss->capacity = VSSSTORAGE_INITIAL_SIZE / sizeof(VSSEntry); - vss->currentSize = 0; - vss->lockedEntryCount = 0; - vss->LWM = 0; - vss->numHashBuckets = VSSTABLE_INITIAL_SIZE / sizeof(int); - newshmseg = reinterpret_cast(vss); + vss->capacity = VSSSTORAGE_INITIAL_SIZE / sizeof(VSSEntry); + vss->currentSize = 0; + vss->lockedEntryCount = 0; + vss->LWM = 0; + vss->numHashBuckets = VSSTABLE_INITIAL_SIZE / sizeof(int); + newshmseg = reinterpret_cast(vss); - buckets = reinterpret_cast - (&newshmseg[sizeof(VSSShmsegHeader)]); - stor = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + - vss->numHashBuckets * sizeof(int)]); + buckets = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader)]); + stor = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + vss->numHashBuckets * sizeof(int)]); - for (i = 0; i < vss->numHashBuckets; i++) - buckets[i] = -1; + for (i = 0; i < vss->numHashBuckets; i++) + buckets[i] = -1; - for (i = 0; i < vss->capacity; i++) - stor[i].lbid = -1; + for (i = 0; i < vss->capacity; i++) + stor[i].lbid = -1; } -//assumes write lock is held +// assumes write lock is held void VSS::growVSS() { - int allocSize; - key_t newshmkey; - char* newshmseg; + int allocSize; + key_t newshmkey; + char* newshmseg; - if (vssShminfo->allocdSize == 0) - allocSize = VSS_INITIAL_SIZE; - else - allocSize = vssShminfo->allocdSize + VSS_INCREMENT; + if (vssShminfo->allocdSize == 0) + allocSize = VSS_INITIAL_SIZE; + else + allocSize = vssShminfo->allocdSize + VSS_INCREMENT; - newshmkey = chooseShmkey(); - idbassert((allocSize == VSS_INITIAL_SIZE && !fPVSSImpl) || fPVSSImpl); + newshmkey = chooseShmkey(); + idbassert((allocSize == VSS_INITIAL_SIZE && !fPVSSImpl) || fPVSSImpl); - if (fPVSSImpl) - { - BRMShmImpl newShm(newshmkey, allocSize); - newshmseg = static_cast(newShm.fMapreg.get_address()); - memset(newshmseg, 0, allocSize); - idbassert(vss); - VSSShmsegHeader* tmp = reinterpret_cast(newshmseg); - tmp->capacity = vss->capacity + VSSSTORAGE_INCREMENT / sizeof(VSSEntry); - tmp->numHashBuckets = vss->numHashBuckets + VSSTABLE_INCREMENT / sizeof(int); - tmp->LWM = 0; - copyVSS(tmp); - fPVSSImpl->swapout(newShm); - } - else - { - fPVSSImpl = VSSImpl::makeVSSImpl(newshmkey, allocSize); - newshmseg = reinterpret_cast(fPVSSImpl->get()); - memset(newshmseg, 0, allocSize); - } + if (fPVSSImpl) + { + BRMShmImpl newShm(newshmkey, allocSize); + newshmseg = static_cast(newShm.fMapreg.get_address()); + memset(newshmseg, 0, allocSize); + idbassert(vss); + VSSShmsegHeader* tmp = reinterpret_cast(newshmseg); + tmp->capacity = vss->capacity + VSSSTORAGE_INCREMENT / sizeof(VSSEntry); + tmp->numHashBuckets = vss->numHashBuckets + VSSTABLE_INCREMENT / sizeof(int); + tmp->LWM = 0; + copyVSS(tmp); + fPVSSImpl->swapout(newShm); + } + else + { + fPVSSImpl = VSSImpl::makeVSSImpl(newshmkey, allocSize); + newshmseg = reinterpret_cast(fPVSSImpl->get()); + memset(newshmseg, 0, allocSize); + } + vss = fPVSSImpl->get(); + + if (allocSize == VSS_INITIAL_SIZE) + initShmseg(); + + vssShminfo->tableShmkey = newshmkey; + vssShminfo->allocdSize = allocSize; + + if (r_only) + { + fPVSSImpl->makeReadOnly(); vss = fPVSSImpl->get(); + } - if (allocSize == VSS_INITIAL_SIZE) - initShmseg(); - - vssShminfo->tableShmkey = newshmkey; - vssShminfo->allocdSize = allocSize; - - if (r_only) - { - fPVSSImpl->makeReadOnly(); - vss = fPVSSImpl->get(); - } - - newshmseg = reinterpret_cast(vss); - hashBuckets = reinterpret_cast - (&newshmseg[sizeof(VSSShmsegHeader)]); - storage = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + - vss->numHashBuckets * sizeof(int)]); + newshmseg = reinterpret_cast(vss); + hashBuckets = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader)]); + storage = + reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + vss->numHashBuckets * sizeof(int)]); } -//assumes write lock is held +// assumes write lock is held void VSS::growForLoad(int elementCount) { - int allocSize; - key_t newshmkey; - char* newshmseg; - int i; + int allocSize; + key_t newshmkey; + char* newshmseg; + int i; - if (elementCount < VSSSTORAGE_INITIAL_COUNT) - elementCount = VSSSTORAGE_INITIAL_COUNT; + if (elementCount < VSSSTORAGE_INITIAL_COUNT) + elementCount = VSSSTORAGE_INITIAL_COUNT; - /* round up to the next normal increment out of paranoia */ - if (elementCount % VSSSTORAGE_INCREMENT_COUNT) - elementCount = ((elementCount / VSSSTORAGE_INCREMENT_COUNT) + 1) * VSSSTORAGE_INCREMENT_COUNT; + /* round up to the next normal increment out of paranoia */ + if (elementCount % VSSSTORAGE_INCREMENT_COUNT) + elementCount = ((elementCount / VSSSTORAGE_INCREMENT_COUNT) + 1) * VSSSTORAGE_INCREMENT_COUNT; - allocSize = VSS_SIZE(elementCount); + allocSize = VSS_SIZE(elementCount); - newshmkey = chooseShmkey(); + newshmkey = chooseShmkey(); - if (fPVSSImpl) - { - // isn't this the same as makeVSSImpl()? - BRMShmImpl newShm(newshmkey, allocSize); - fPVSSImpl->swapout(newShm); - } - else - { - fPVSSImpl = VSSImpl::makeVSSImpl(newshmkey, allocSize); - } + if (fPVSSImpl) + { + // isn't this the same as makeVSSImpl()? + BRMShmImpl newShm(newshmkey, allocSize); + fPVSSImpl->swapout(newShm); + } + else + { + fPVSSImpl = VSSImpl::makeVSSImpl(newshmkey, allocSize); + } - vss = fPVSSImpl->get(); - vss->capacity = elementCount; - vss->currentSize = 0; - vss->LWM = 0; - vss->numHashBuckets = elementCount / 4; - vss->lockedEntryCount = 0; - undoRecords.clear(); - newshmseg = reinterpret_cast(vss); - hashBuckets = reinterpret_cast - (&newshmseg[sizeof(VSSShmsegHeader)]); - storage = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + - vss->numHashBuckets * sizeof(int)]); + vss = fPVSSImpl->get(); + vss->capacity = elementCount; + vss->currentSize = 0; + vss->LWM = 0; + vss->numHashBuckets = elementCount / 4; + vss->lockedEntryCount = 0; + undoRecords.clear(); + newshmseg = reinterpret_cast(vss); + hashBuckets = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader)]); + storage = + reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + vss->numHashBuckets * sizeof(int)]); - for (i = 0; i < vss->capacity; i++) - storage[i].lbid = -1; + for (i = 0; i < vss->capacity; i++) + storage[i].lbid = -1; - for (i = 0; i < vss->numHashBuckets; i++) - hashBuckets[i] = -1; + for (i = 0; i < vss->numHashBuckets; i++) + hashBuckets[i] = -1; - vssShminfo->tableShmkey = newshmkey; - vssShminfo->allocdSize = allocSize; + vssShminfo->tableShmkey = newshmkey; + vssShminfo->allocdSize = allocSize; } -//assumes write lock is held and the src is vbbm -//and that dest->{numHashBuckets, capacity, LWM} have been set. +// assumes write lock is held and the src is vbbm +// and that dest->{numHashBuckets, capacity, LWM} have been set. void VSS::copyVSS(VSSShmsegHeader* dest) { - int i; - int* newHashtable; - VSSEntry* newStorage; - char* cDest = reinterpret_cast(dest); + int i; + int* newHashtable; + VSSEntry* newStorage; + char* cDest = reinterpret_cast(dest); - // copy metadata - dest->currentSize = vss->currentSize; - dest->lockedEntryCount = vss->lockedEntryCount; + // copy metadata + dest->currentSize = vss->currentSize; + dest->lockedEntryCount = vss->lockedEntryCount; - newHashtable = reinterpret_cast(&cDest[sizeof(VSSShmsegHeader)]); - newStorage = reinterpret_cast(&cDest[sizeof(VSSShmsegHeader) + - dest->numHashBuckets * sizeof(int)]); + newHashtable = reinterpret_cast(&cDest[sizeof(VSSShmsegHeader)]); + newStorage = + reinterpret_cast(&cDest[sizeof(VSSShmsegHeader) + dest->numHashBuckets * sizeof(int)]); - //initialize new storage & hash - for (i = 0; i < dest->numHashBuckets; i++) - newHashtable[i] = -1; + // initialize new storage & hash + for (i = 0; i < dest->numHashBuckets; i++) + newHashtable[i] = -1; - for (i = 0; i < dest->capacity; i++) - newStorage[i].lbid = -1; + for (i = 0; i < dest->capacity; i++) + newStorage[i].lbid = -1; - //walk the storage & re-hash all entries; - for (i = 0; i < vss->currentSize; i++) - if (storage[i].lbid != -1) - { - _insert(storage[i], dest, newHashtable, newStorage, true); - //confirmChanges(); - } + // walk the storage & re-hash all entries; + for (i = 0; i < vss->currentSize; i++) + if (storage[i].lbid != -1) + { + _insert(storage[i], dest, newHashtable, newStorage, true); + // confirmChanges(); + } } key_t VSS::chooseShmkey() const { - int fixedKeys = 1; - key_t ret; + int fixedKeys = 1; + key_t ret; - if (vssShminfo->tableShmkey + 1 == (key_t) (fShmKeys.KEYRANGE_VSS_BASE + - fShmKeys.KEYRANGE_SIZE - 1) || (unsigned)vssShminfo->tableShmkey < fShmKeys.KEYRANGE_VSS_BASE) - ret = fShmKeys.KEYRANGE_VSS_BASE + fixedKeys; - else - ret = vssShminfo->tableShmkey + 1; + if (vssShminfo->tableShmkey + 1 == (key_t)(fShmKeys.KEYRANGE_VSS_BASE + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)vssShminfo->tableShmkey < fShmKeys.KEYRANGE_VSS_BASE) + ret = fShmKeys.KEYRANGE_VSS_BASE + fixedKeys; + else + ret = vssShminfo->tableShmkey + 1; - return ret; + return ret; } void VSS::insert(LBID_t lbid, VER_t verID, bool vbFlag, bool locked, bool loading) { - VSSEntry entry; + VSSEntry entry; #ifdef BRM_DEBUG - if (lbid < 0) - { - log("VSS::insert(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::insert(): lbid must be >= 0"); - } + if (lbid < 0) + { + log("VSS::insert(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::insert(): lbid must be >= 0"); + } - if (verID < 0) - { - log("VSS::insert(): verID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::insert(): verID must be >= 0"); - } + if (verID < 0) + { + log("VSS::insert(): verID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::insert(): verID must be >= 0"); + } #endif - entry.lbid = lbid; - entry.verID = verID; - entry.vbFlag = vbFlag; - entry.locked = locked; + entry.lbid = lbid; + entry.verID = verID; + entry.vbFlag = vbFlag; + entry.locked = locked; - //cerr << "Insert to vss lbid:verID:locked:vbFlag = " << entry.lbid <<":" <currentSize == vss->capacity) - growVSS(); + // cerr << "Insert to vss lbid:verID:locked:vbFlag = " << entry.lbid <<":" <currentSize == vss->capacity) + growVSS(); - _insert(entry, vss, hashBuckets, storage, loading); + _insert(entry, vss, hashBuckets, storage, loading); - if (!loading) - makeUndoRecord(&vss->currentSize, sizeof(vss->currentSize)); + if (!loading) + makeUndoRecord(&vss->currentSize, sizeof(vss->currentSize)); - vss->currentSize++; + vss->currentSize++; - if (locked) - vss->lockedEntryCount++; + if (locked) + vss->lockedEntryCount++; } - - -//assumes write lock is held and that it is properly sized already -//metadata is modified by the caller -void VSS::_insert(VSSEntry& e, VSSShmsegHeader* dest, int* destHash, - VSSEntry* destStorage, bool loading) +// assumes write lock is held and that it is properly sized already +// metadata is modified by the caller +void VSS::_insert(VSSEntry& e, VSSShmsegHeader* dest, int* destHash, VSSEntry* destStorage, bool loading) { - int hashIndex, insertIndex; + int hashIndex, insertIndex; - hashIndex = hasher((char*) &e.lbid, sizeof(e.lbid)) % dest->numHashBuckets; + hashIndex = hasher((char*)&e.lbid, sizeof(e.lbid)) % dest->numHashBuckets; - insertIndex = dest->LWM; + insertIndex = dest->LWM; - while (destStorage[insertIndex].lbid != -1) - { - insertIndex++; + while (destStorage[insertIndex].lbid != -1) + { + insertIndex++; #ifdef BRM_DEBUG - if (insertIndex == dest->capacity) - { - log("VSS:_insert(): There are no empty entries. Check resize condition.", - logging::LOG_TYPE_DEBUG); - throw logic_error("VSS:_insert(): There are no empty entries. Check resize condition."); - } + if (insertIndex == dest->capacity) + { + log("VSS:_insert(): There are no empty entries. Check resize condition.", logging::LOG_TYPE_DEBUG); + throw logic_error("VSS:_insert(): There are no empty entries. Check resize condition."); + } #endif - } + } - if (!loading) - makeUndoRecord(dest, sizeof(VSSShmsegHeader)); + if (!loading) + makeUndoRecord(dest, sizeof(VSSShmsegHeader)); - dest->LWM = insertIndex + 1; + dest->LWM = insertIndex + 1; - if (!loading) - { - makeUndoRecord(&destStorage[insertIndex], sizeof(VSSEntry)); - makeUndoRecord(&destHash[hashIndex], sizeof(int)); - } + if (!loading) + { + makeUndoRecord(&destStorage[insertIndex], sizeof(VSSEntry)); + makeUndoRecord(&destHash[hashIndex], sizeof(int)); + } - e.next = destHash[hashIndex]; - destStorage[insertIndex] = e; - destHash[hashIndex] = insertIndex; + e.next = destHash[hashIndex]; + destStorage[insertIndex] = e; + destHash[hashIndex] = insertIndex; } -//assumes read lock is held -int VSS::lookup(LBID_t lbid, const QueryContext_vss& verInfo, VER_t txnID, VER_t* outVer, - bool* vbFlag, bool vbOnly) const +// assumes read lock is held +int VSS::lookup(LBID_t lbid, const QueryContext_vss& verInfo, VER_t txnID, VER_t* outVer, bool* vbFlag, + bool vbOnly) const { - int hashIndex, maxVersion = -1, minVersion = -1, currentIndex; - VSSEntry* listEntry, *maxEntry = NULL; + int hashIndex, maxVersion = -1, minVersion = -1, currentIndex; + VSSEntry *listEntry, *maxEntry = NULL; #ifdef BRM_DEBUG - if (lbid < 0) - { - log("VSS::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::lookup(): lbid must be >= 0"); - } + if (lbid < 0) + { + log("VSS::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::lookup(): lbid must be >= 0"); + } - if (verInfo.currentScn < 0) - { - log("VSS::lookup(): verID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::lookup(): verID must be >= 0"); - } + if (verInfo.currentScn < 0) + { + log("VSS::lookup(): verID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::lookup(): verID must be >= 0"); + } - if (txnID < 0) - { - log("VSS::lookup(): txnID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::lookup(): txnID must be >= 0"); - } + if (txnID < 0) + { + log("VSS::lookup(): txnID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::lookup(): txnID must be >= 0"); + } #endif - hashIndex = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; + hashIndex = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; - currentIndex = hashBuckets[hashIndex]; + currentIndex = hashBuckets[hashIndex]; - while (currentIndex != -1) + while (currentIndex != -1) + { + listEntry = &storage[currentIndex]; + + if (listEntry->lbid == lbid) { - listEntry = &storage[currentIndex]; + if (vbOnly && !listEntry->vbFlag) + goto next; - if (listEntry->lbid == lbid) - { + /* "if it is/was part of a different transaction, ignore it" + */ + if (txnID != listEntry->verID && (verInfo.txns->find(listEntry->verID) != verInfo.txns->end())) + goto next; - if (vbOnly && !listEntry->vbFlag) - goto next; - - /* "if it is/was part of a different transaction, ignore it" - */ - if (txnID != listEntry->verID && - (verInfo.txns->find(listEntry->verID) != verInfo.txns->end())) - goto next; - - /* fast exit if the exact version is found */ - if (verInfo.currentScn == listEntry->verID) - { - *outVer = listEntry->verID; - *vbFlag = listEntry->vbFlag; - return 0; - } - - /* Track the min version of this LBID */ - if (minVersion > listEntry->verID || minVersion == -1) - minVersion = listEntry->verID; - - /* Pick the highest version <= the SCN of the query */ - if (verInfo.currentScn > listEntry->verID && maxVersion < listEntry->verID) - { - maxVersion = listEntry->verID; - maxEntry = listEntry; - } - } - -next: - currentIndex = listEntry->next; - } - - if (maxEntry != NULL) - { - *outVer = maxVersion; - *vbFlag = maxEntry->vbFlag; + /* fast exit if the exact version is found */ + if (verInfo.currentScn == listEntry->verID) + { + *outVer = listEntry->verID; + *vbFlag = listEntry->vbFlag; return 0; - } - else if (minVersion > verInfo.currentScn) - { - *outVer = 0; - *vbFlag = false; - return ERR_SNAPSHOT_TOO_OLD; + } + + /* Track the min version of this LBID */ + if (minVersion > listEntry->verID || minVersion == -1) + minVersion = listEntry->verID; + + /* Pick the highest version <= the SCN of the query */ + if (verInfo.currentScn > listEntry->verID && maxVersion < listEntry->verID) + { + maxVersion = listEntry->verID; + maxEntry = listEntry; + } } + next: + currentIndex = listEntry->next; + } + + if (maxEntry != NULL) + { + *outVer = maxVersion; + *vbFlag = maxEntry->vbFlag; + return 0; + } + else if (minVersion > verInfo.currentScn) + { *outVer = 0; *vbFlag = false; - return -1; + return ERR_SNAPSHOT_TOO_OLD; + } + + *outVer = 0; + *vbFlag = false; + return -1; } VER_t VSS::getCurrentVersion(LBID_t lbid, bool* isLocked) const { - int hashIndex, currentIndex; - VSSEntry* listEntry; + int hashIndex, currentIndex; + VSSEntry* listEntry; - hashIndex = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; - currentIndex = hashBuckets[hashIndex]; + hashIndex = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; + currentIndex = hashBuckets[hashIndex]; - while (currentIndex != -1) + while (currentIndex != -1) + { + listEntry = &storage[currentIndex]; + + if (listEntry->lbid == lbid && !listEntry->vbFlag) { - listEntry = &storage[currentIndex]; + if (isLocked != NULL) + *isLocked = listEntry->locked; - if (listEntry->lbid == lbid && !listEntry->vbFlag) - { - if (isLocked != NULL) - *isLocked = listEntry->locked; - - return listEntry->verID; - } - - currentIndex = listEntry->next; + return listEntry->verID; } - if (isLocked != NULL) - *isLocked = false; + currentIndex = listEntry->next; + } - return 0; + if (isLocked != NULL) + *isLocked = false; + + return 0; } VER_t VSS::getHighestVerInVB(LBID_t lbid, VER_t max) const { - int hashIndex, currentIndex; - VER_t ret = -1; - VSSEntry* listEntry; + int hashIndex, currentIndex; + VER_t ret = -1; + VSSEntry* listEntry; - hashIndex = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; - currentIndex = hashBuckets[hashIndex]; + hashIndex = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; + currentIndex = hashBuckets[hashIndex]; - while (currentIndex != -1) - { - listEntry = &storage[currentIndex]; + while (currentIndex != -1) + { + listEntry = &storage[currentIndex]; - if ((listEntry->lbid == lbid && listEntry->vbFlag) && - (listEntry->verID <= max && listEntry->verID > ret)) - ret = listEntry->verID; + if ((listEntry->lbid == lbid && listEntry->vbFlag) && (listEntry->verID <= max && listEntry->verID > ret)) + ret = listEntry->verID; - currentIndex = listEntry->next; - } + currentIndex = listEntry->next; + } - return ret; + return ret; } bool VSS::isVersioned(LBID_t lbid, VER_t version) const { - int hashIndex, currentIndex; - VSSEntry* listEntry; + int hashIndex, currentIndex; + VSSEntry* listEntry; - hashIndex = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; - currentIndex = hashBuckets[hashIndex]; + hashIndex = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; + currentIndex = hashBuckets[hashIndex]; - while (currentIndex != -1) - { - listEntry = &storage[currentIndex]; + while (currentIndex != -1) + { + listEntry = &storage[currentIndex]; - if (listEntry->lbid == lbid && listEntry->verID == version) - return listEntry->vbFlag; + if (listEntry->lbid == lbid && listEntry->verID == version) + return listEntry->vbFlag; - currentIndex = listEntry->next; - } + currentIndex = listEntry->next; + } - return false; + return false; } bool VSS::isLocked(const LBIDRange& range, VER_t transID) const { - int hashIndex, currentIndex; - LBID_t currentBlock; - VSSEntry* listEntry; + int hashIndex, currentIndex; + LBID_t currentBlock; + VSSEntry* listEntry; - for (currentBlock = range.start; - currentBlock < range.start + range.size; - currentBlock++) + for (currentBlock = range.start; currentBlock < range.start + range.size; currentBlock++) + { + hashIndex = hasher((char*)¤tBlock, sizeof(currentBlock)) % vss->numHashBuckets; + + currentIndex = hashBuckets[hashIndex]; + + while (currentIndex != -1) { + listEntry = &storage[currentIndex]; - hashIndex = hasher((char*) ¤tBlock, sizeof(currentBlock)) % vss->numHashBuckets; + if (listEntry->lbid == currentBlock && listEntry->locked) + { + if (listEntry->verID == transID) + return false; + else + return true; + } - currentIndex = hashBuckets[hashIndex]; - - while (currentIndex != -1) - { - listEntry = &storage[currentIndex]; - - if (listEntry->lbid == currentBlock && listEntry->locked) - { - if (listEntry->verID == transID) - return false; - else - return true; - } - - currentIndex = listEntry->next; - } + currentIndex = listEntry->next; } + } - return false; + return false; } -//requires write lock +// requires write lock void VSS::removeEntry(LBID_t lbid, VER_t verID, vector* flushList) { - int index, prev, bucket; + int index, prev, bucket; #ifdef BRM_DEBUG - if (lbid < 0) - { - log("VSS::removeEntry(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::removeEntry(): lbid must be >= 0"); - } + if (lbid < 0) + { + log("VSS::removeEntry(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::removeEntry(): lbid must be >= 0"); + } - if (verID < 0) - { - log("VSS::removeEntry(): verID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::removeEntry(): verID must be >= 0"); - } + if (verID < 0) + { + log("VSS::removeEntry(): verID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::removeEntry(): verID must be >= 0"); + } #endif - index = getIndex(lbid, verID, prev, bucket); + index = getIndex(lbid, verID, prev, bucket); - if (index == -1) - { + if (index == -1) + { #ifdef BRM_DEBUG - ostringstream ostr; + ostringstream ostr; - ostr << "VSS::removeEntry(): that entry doesn't exist lbid = " << - lbid << " verID = " << verID; - log(ostr.str(), logging::LOG_TYPE_DEBUG); - throw logic_error(ostr.str()); + ostr << "VSS::removeEntry(): that entry doesn't exist lbid = " << lbid << " verID = " << verID; + log(ostr.str(), logging::LOG_TYPE_DEBUG); + throw logic_error(ostr.str()); #else - return; + return; #endif - } + } - makeUndoRecord(&storage[index], sizeof(VSSEntry)); - storage[index].lbid = -1; + makeUndoRecord(&storage[index], sizeof(VSSEntry)); + storage[index].lbid = -1; - if (prev != -1) - { - makeUndoRecord(&storage[prev], sizeof(VSSEntry)); - storage[prev].next = storage[index].next; - } - else + if (prev != -1) + { + makeUndoRecord(&storage[prev], sizeof(VSSEntry)); + storage[prev].next = storage[index].next; + } + else + { + makeUndoRecord(&hashBuckets[bucket], sizeof(int)); + hashBuckets[bucket] = storage[index].next; + } + + makeUndoRecord(vss, sizeof(VSSShmsegHeader)); + vss->currentSize--; + + if (storage[index].locked && (vss->lockedEntryCount > 0)) + vss->lockedEntryCount--; + + if (index < vss->LWM) + vss->LWM = index; + + // scan the list of entries with that lbid to see if there are others + // to remove. + for (index = hashBuckets[bucket]; index != -1; index = storage[index].next) + if (storage[index].lbid == lbid && (storage[index].vbFlag || storage[index].locked)) + return; + + // if execution gets here, we should be able to remove all entries + // with the given lbid because none point to the VB. + for (prev = -1, index = hashBuckets[bucket]; index != -1; index = storage[index].next) + { + if (storage[index].lbid == lbid) { + makeUndoRecord(&storage[index], sizeof(VSSEntry)); + storage[index].lbid = -1; + + if (prev == -1) + { makeUndoRecord(&hashBuckets[bucket], sizeof(int)); hashBuckets[bucket] = storage[index].next; - } + } + else + { + makeUndoRecord(&storage[prev], sizeof(VSSEntry)); + storage[prev].next = storage[index].next; + } - makeUndoRecord(vss, sizeof(VSSShmsegHeader)); - vss->currentSize--; + vss->currentSize--; - if (storage[index].locked && (vss->lockedEntryCount > 0)) + if (storage[index].locked && (vss->lockedEntryCount > 0)) vss->lockedEntryCount--; - if (index < vss->LWM) + if (index < vss->LWM) vss->LWM = index; - - // scan the list of entries with that lbid to see if there are others - // to remove. - for (index = hashBuckets[bucket]; index != -1; index = storage[index].next) - if (storage[index].lbid == lbid && (storage[index].vbFlag || storage[index].locked)) - return; - - // if execution gets here, we should be able to remove all entries - // with the given lbid because none point to the VB. - for (prev = -1, index = hashBuckets[bucket]; - index != -1; index = storage[index].next) - { - if (storage[index].lbid == lbid) - { - makeUndoRecord(&storage[index], sizeof(VSSEntry)); - storage[index].lbid = -1; - - if (prev == -1) - { - makeUndoRecord(&hashBuckets[bucket], sizeof(int)); - hashBuckets[bucket] = storage[index].next; - } - else - { - makeUndoRecord(&storage[prev], sizeof(VSSEntry)); - storage[prev].next = storage[index].next; - } - - vss->currentSize--; - - if (storage[index].locked && (vss->lockedEntryCount > 0)) - vss->lockedEntryCount--; - - if (index < vss->LWM) - vss->LWM = index; - } - else - prev = index; } + else + prev = index; + } - flushList->push_back(lbid); + flushList->push_back(lbid); } bool VSS::isTooOld(LBID_t lbid, VER_t verID) const { - int index, bucket; - VER_t minVer = 0; - VSSEntry* listEntry; + int index, bucket; + VER_t minVer = 0; + VSSEntry* listEntry; - bucket = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; + bucket = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; - index = hashBuckets[bucket]; + index = hashBuckets[bucket]; - while (index != -1) - { - listEntry = &storage[index]; + while (index != -1) + { + listEntry = &storage[index]; - if (listEntry->lbid == lbid && minVer > listEntry->verID) - minVer = listEntry->verID; + if (listEntry->lbid == lbid && minVer > listEntry->verID) + minVer = listEntry->verID; - index = listEntry->next; - } + index = listEntry->next; + } - return (minVer > verID); + return (minVer > verID); } bool VSS::isEntryLocked(LBID_t lbid, VER_t verID) const { - int index; - int bucket; + int index; + int bucket; - if (lbid == -1) - return false; + if (lbid == -1) + return false; - /* See bug 3287 for info on these code blocks */ + /* See bug 3287 for info on these code blocks */ - /* This version checks for the specific lbid,ver pair - index = getIndex(lbid, verID, prev, bucket); - if (index < 0) - return false; - return storage[index].locked; - */ + /* This version checks for the specific lbid,ver pair + index = getIndex(lbid, verID, prev, bucket); + if (index < 0) + return false; + return storage[index].locked; + */ - /* This version considers any locked entry for an LBID to mean the block is locked. - VSSEntry *listEntry; - bucket = hasher((char *) &lbid, sizeof(lbid)) % vss->numHashBuckets; + /* This version considers any locked entry for an LBID to mean the block is locked. + VSSEntry *listEntry; + bucket = hasher((char *) &lbid, sizeof(lbid)) % vss->numHashBuckets; - index = hashBuckets[bucket]; - while (index != -1) { - listEntry = &storage[index]; - if (listEntry->lbid == lbid && listEntry->locked) - { - ostringstream msg; - msg << " Locked entry information lbid:verID = " << listEntry->lbid << ":" << listEntry->verID << endl; - log(msg.str(), logging::LOG_TYPE_CRITICAL); - return true; - } - index = listEntry->next; - } - return false; - */ + index = hashBuckets[bucket]; + while (index != -1) { + listEntry = &storage[index]; + if (listEntry->lbid == lbid && listEntry->locked) + { + ostringstream msg; + msg << " Locked entry information lbid:verID = " << listEntry->lbid << ":" << + listEntry->verID << endl; log(msg.str(), logging::LOG_TYPE_CRITICAL); return true; + } + index = listEntry->next; + } + return false; + */ -// This version considers the blocks needed for rollback to be locked, -// otherwise they're unlocked. Note, the version with the 'locked' flag set -// will never be a candidate for aging out b/c it's not in the version buffer. -// TODO: Update this when we support multiple transactions at once. Need to -// identify ALL versions needed for rollback. + // This version considers the blocks needed for rollback to be locked, + // otherwise they're unlocked. Note, the version with the 'locked' flag set + // will never be a candidate for aging out b/c it's not in the version buffer. + // TODO: Update this when we support multiple transactions at once. Need to + // identify ALL versions needed for rollback. - VSSEntry* listEntry; - bool hasALockedEntry = false; - VER_t rollbackVersion = 0; + VSSEntry* listEntry; + bool hasALockedEntry = false; + VER_t rollbackVersion = 0; - bucket = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; + bucket = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; - index = hashBuckets[bucket]; + index = hashBuckets[bucket]; - while (index != -1) + while (index != -1) + { + listEntry = &storage[index]; + + if (listEntry->lbid == lbid) { - listEntry = &storage[index]; - - if (listEntry->lbid == lbid) - { - if (listEntry->locked) - hasALockedEntry = true; - else if (rollbackVersion < listEntry->verID) - rollbackVersion = listEntry->verID; - } - - index = listEntry->next; + if (listEntry->locked) + hasALockedEntry = true; + else if (rollbackVersion < listEntry->verID) + rollbackVersion = listEntry->verID; } - return (hasALockedEntry && verID == rollbackVersion); + index = listEntry->next; + } + + return (hasALockedEntry && verID == rollbackVersion); } -//read lock +// read lock int VSS::getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const { - int currentIndex; - VSSEntry* listEntry; + int currentIndex; + VSSEntry* listEntry; - prev = -1; - bucket = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; + prev = -1; + bucket = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; - currentIndex = hashBuckets[bucket]; + currentIndex = hashBuckets[bucket]; - while (currentIndex != -1) - { - listEntry = &storage[currentIndex]; + while (currentIndex != -1) + { + listEntry = &storage[currentIndex]; - if (listEntry->lbid == lbid && listEntry->verID == verID) - return currentIndex; + if (listEntry->lbid == lbid && listEntry->verID == verID) + return currentIndex; - prev = currentIndex; - currentIndex = listEntry->next; - } + prev = currentIndex; + currentIndex = listEntry->next; + } - return -1; + return -1; } -//write lock +// write lock void VSS::setVBFlag(LBID_t lbid, VER_t verID, bool vbFlag) { - int index, prev, bucket; + int index, prev, bucket; #ifdef BRM_DEBUG - if (lbid < 0) - { - log("VSS::setVBFlag(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::setVBFlag(): lbid must be >= 0"); - } + if (lbid < 0) + { + log("VSS::setVBFlag(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::setVBFlag(): lbid must be >= 0"); + } - if (verID < 0) - { - log("VSS::setVBFlag(): verID must be >= 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::setVBFlag(): verID must be >= 0"); - } + if (verID < 0) + { + log("VSS::setVBFlag(): verID must be >= 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::setVBFlag(): verID must be >= 0"); + } #endif - index = getIndex(lbid, verID, prev, bucket); + index = getIndex(lbid, verID, prev, bucket); - if (index == -1) - { - ostringstream ostr; + if (index == -1) + { + ostringstream ostr; - ostr << "VSS::setVBFlag(): that entry doesn't exist lbid=" << lbid << - " ver=" << verID; - log(ostr.str(), logging::LOG_TYPE_DEBUG); - throw logic_error(ostr.str()); - } + ostr << "VSS::setVBFlag(): that entry doesn't exist lbid=" << lbid << " ver=" << verID; + log(ostr.str(), logging::LOG_TYPE_DEBUG); + throw logic_error(ostr.str()); + } - makeUndoRecord(&storage[index], sizeof(VSSEntry)); - storage[index].vbFlag = vbFlag; + makeUndoRecord(&storage[index], sizeof(VSSEntry)); + storage[index].vbFlag = vbFlag; } -//write lock +// write lock void VSS::commit(VER_t txnID) { - int i; + int i; #ifdef BRM_DEBUG - if (txnID < 1) + if (txnID < 1) + { + log("VSS::commit(): txnID must be > 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::commit(): txnID must be > 0"); + } + +#endif + + for (i = 0; i < vss->capacity; i++) + if (storage[i].lbid != -1 && storage[i].verID == txnID) { - log("VSS::commit(): txnID must be > 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::commit(): txnID must be > 0"); - } - -#endif - - for (i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1 && storage[i].verID == txnID) - { #ifdef BRM_DEBUG - if (storage[i].locked != true) - { - ostringstream ostr; - ostr << "VSS::commit(): An entry has already been unlocked..? txnId = " << txnID ; - log(ostr.str(), logging::LOG_TYPE_DEBUG); - throw logic_error(ostr.str()); - } + if (storage[i].locked != true) + { + ostringstream ostr; + ostr << "VSS::commit(): An entry has already been unlocked..? txnId = " << txnID; + log(ostr.str(), logging::LOG_TYPE_DEBUG); + throw logic_error(ostr.str()); + } #endif - makeUndoRecord(&storage[i], sizeof(VSSEntry)); - storage[i].locked = false; + makeUndoRecord(&storage[i], sizeof(VSSEntry)); + storage[i].locked = false; - // @ bug 1426 fix. Decrease the counter when an entry releases its lock. - if (vss->lockedEntryCount > 0) - vss->lockedEntryCount--; - } + // @ bug 1426 fix. Decrease the counter when an entry releases its lock. + if (vss->lockedEntryCount > 0) + vss->lockedEntryCount--; + } } -//read lock +// read lock void VSS::getUncommittedLBIDs(VER_t txnID, vector& lbids) { - int i; + int i; - lbids.clear(); + lbids.clear(); #ifdef BRM_DEBUG - if (txnID < 1) + if (txnID < 1) + { + log("VSS::getUncommittedLBIDs(): txnID must be > 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::getUncommittedLBIDs(): txnID must be > 0"); + } + +#endif + + for (i = 0; i < vss->capacity; i++) + if (storage[i].lbid != -1 && storage[i].verID == txnID) { - log("VSS::getUncommittedLBIDs(): txnID must be > 0", - logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::getUncommittedLBIDs(): txnID must be > 0"); - } - -#endif - - for (i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1 && storage[i].verID == txnID) - { #ifdef BRM_DEBUG - if (storage[i].locked == false) - { - log("VSS::getUncommittedLBIDs(): found an unlocked block with that TxnID", - logging::LOG_TYPE_DEBUG); - throw logic_error("VSS::getUncommittedLBIDs(): found an unlocked block with that TxnID"); - } + if (storage[i].locked == false) + { + log("VSS::getUncommittedLBIDs(): found an unlocked block with that TxnID", logging::LOG_TYPE_DEBUG); + throw logic_error("VSS::getUncommittedLBIDs(): found an unlocked block with that TxnID"); + } - if (storage[i].vbFlag == true) - { - log("VSS::getUncommittedLBIDs(): found a block with that TxnID in the VB", - logging::LOG_TYPE_DEBUG); - throw logic_error("VSS::getUncommittedLBIDs(): found a block with that TxnID in the VB"); - } + if (storage[i].vbFlag == true) + { + log("VSS::getUncommittedLBIDs(): found a block with that TxnID in the VB", logging::LOG_TYPE_DEBUG); + throw logic_error("VSS::getUncommittedLBIDs(): found a block with that TxnID in the VB"); + } #endif - lbids.push_back(storage[i].lbid); - } - + lbids.push_back(storage[i].lbid); + } } void VSS::getUnlockedLBIDs(BlockList_t& lbids) { - lbids.clear(); + lbids.clear(); - for (int i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1 && !storage[i].locked) - lbids.push_back(LVP_t(storage[i].lbid, storage[i].verID)); + for (int i = 0; i < vss->capacity; i++) + if (storage[i].lbid != -1 && !storage[i].locked) + lbids.push_back(LVP_t(storage[i].lbid, storage[i].verID)); } void VSS::getLockedLBIDs(BlockList_t& lbids) { - lbids.clear(); + lbids.clear(); - for (int i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1 && storage[i].locked) - lbids.push_back(LVP_t(storage[i].lbid, storage[i].verID)); + for (int i = 0; i < vss->capacity; i++) + if (storage[i].lbid != -1 && storage[i].locked) + lbids.push_back(LVP_t(storage[i].lbid, storage[i].verID)); } -//write lock +// write lock /* Rewritten on 6/2/10 to be O(n) with the size of range rather than * O(nlogn) with VSS capacity. */ void VSS::removeEntriesFromDB(const LBIDRange& range, VBBM& vbbm, bool use_vbbm) { - int bucket, index, prev; - LBID_t lastLBID, lbid; + int bucket, index, prev; + LBID_t lastLBID, lbid; #ifdef BRM_DEBUG - if (range.start < 0) - { - log("VSS::removeEntriesFromDB(): lbids must be positive.", - logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::removeEntriesFromDB(): lbids must be positive."); - } + if (range.start < 0) + { + log("VSS::removeEntriesFromDB(): lbids must be positive.", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::removeEntriesFromDB(): lbids must be positive."); + } - if (range.size < 1) - { - log("VSS::removeEntriesFromDB(): size must be > 0", logging::LOG_TYPE_DEBUG); - throw invalid_argument("VSS::removeEntriesFromDB(): size must be > 0"); - } + if (range.size < 1) + { + log("VSS::removeEntriesFromDB(): size must be > 0", logging::LOG_TYPE_DEBUG); + throw invalid_argument("VSS::removeEntriesFromDB(): size must be > 0"); + } #endif - makeUndoRecord(vss, sizeof(VSSShmsegHeader)); + makeUndoRecord(vss, sizeof(VSSShmsegHeader)); - lastLBID = range.start + range.size - 1; + lastLBID = range.start + range.size - 1; - for (lbid = range.start; lbid <= lastLBID; lbid++) + for (lbid = range.start; lbid <= lastLBID; lbid++) + { + bucket = hasher((char*)&lbid, sizeof(lbid)) % vss->numHashBuckets; + + for (prev = -1, index = hashBuckets[bucket]; index != -1; index = storage[index].next) { - bucket = hasher((char*) &lbid, sizeof(lbid)) % vss->numHashBuckets; + if (storage[index].lbid == lbid) + { + if (storage[index].vbFlag && use_vbbm) + vbbm.removeEntry(storage[index].lbid, storage[index].verID); - for (prev = -1, index = hashBuckets[bucket]; index != -1; - index = storage[index].next) + makeUndoRecord(&storage[index], sizeof(VSSEntry)); + storage[index].lbid = -1; + + if (prev == -1) { - if (storage[index].lbid == lbid) - { - if (storage[index].vbFlag && use_vbbm) - vbbm.removeEntry(storage[index].lbid, storage[index].verID); - - makeUndoRecord(&storage[index], sizeof(VSSEntry)); - storage[index].lbid = -1; - - if (prev == -1) - { - makeUndoRecord(&hashBuckets[bucket], sizeof(int)); - hashBuckets[bucket] = storage[index].next; - } - else - { - makeUndoRecord(&storage[prev], sizeof(VSSEntry)); - storage[prev].next = storage[index].next; - } - - vss->currentSize--; - - if (storage[index].locked && (vss->lockedEntryCount > 0)) - vss->lockedEntryCount--; - - if (index < vss->LWM) - vss->LWM = index; - } - else - prev = index; + makeUndoRecord(&hashBuckets[bucket], sizeof(int)); + hashBuckets[bucket] = storage[index].next; } + else + { + makeUndoRecord(&storage[prev], sizeof(VSSEntry)); + storage[prev].next = storage[index].next; + } + + vss->currentSize--; + + if (storage[index].locked && (vss->lockedEntryCount > 0)) + vss->lockedEntryCount--; + + if (index < vss->LWM) + vss->LWM = index; + } + else + prev = index; } + } } int VSS::size() const { - int i, ret = 0; + int i, ret = 0; - for (i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1) - ret++; + for (i = 0; i < vss->capacity; i++) + if (storage[i].lbid != -1) + ret++; - if (ret != vss->currentSize) - { - ostringstream ostr; + if (ret != vss->currentSize) + { + ostringstream ostr; - ostr << "VSS: actual size & recorded size disagree. actual size = " - << ret << " recorded size = " << vss->currentSize; - log(ostr.str(), logging::LOG_TYPE_DEBUG); - throw logic_error(ostr.str()); - } + ostr << "VSS: actual size & recorded size disagree. actual size = " << ret + << " recorded size = " << vss->currentSize; + log(ostr.str(), logging::LOG_TYPE_DEBUG); + throw logic_error(ostr.str()); + } - return ret; + return ret; } bool VSS::hashEmpty() const { - int i; + int i; - for (i = 0; i < vss->numHashBuckets; i++) - if (hashBuckets[i] != -1) - return false; + for (i = 0; i < vss->numHashBuckets; i++) + if (hashBuckets[i] != -1) + return false; - return true; + return true; } void VSS::clear() { - int allocSize; - key_t newshmkey; - char* newshmseg; + int allocSize; + key_t newshmkey; + char* newshmseg; - allocSize = VSS_INITIAL_SIZE; + allocSize = VSS_INITIAL_SIZE; - newshmkey = chooseShmkey(); + newshmkey = chooseShmkey(); - idbassert(fPVSSImpl); - idbassert(fPVSSImpl->key() != (unsigned)newshmkey); - fPVSSImpl->clear(newshmkey, allocSize); - vssShminfo->tableShmkey = newshmkey; - vssShminfo->allocdSize = allocSize; + idbassert(fPVSSImpl); + idbassert(fPVSSImpl->key() != (unsigned)newshmkey); + fPVSSImpl->clear(newshmkey, allocSize); + vssShminfo->tableShmkey = newshmkey; + vssShminfo->allocdSize = allocSize; + vss = fPVSSImpl->get(); + initShmseg(); + + if (r_only) + { + fPVSSImpl->makeReadOnly(); vss = fPVSSImpl->get(); - initShmseg(); + } - if (r_only) - { - fPVSSImpl->makeReadOnly(); - vss = fPVSSImpl->get(); - } - - newshmseg = reinterpret_cast(vss); - hashBuckets = reinterpret_cast - (&newshmseg[sizeof(VSSShmsegHeader)]); - storage = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + - vss->numHashBuckets * sizeof(int)]); + newshmseg = reinterpret_cast(vss); + hashBuckets = reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader)]); + storage = + reinterpret_cast(&newshmseg[sizeof(VSSShmsegHeader) + vss->numHashBuckets * sizeof(int)]); } // read lock int VSS::checkConsistency(const VBBM& vbbm, ExtentMap& em) const { - /* - 1. Every valid entry in the VSS has an entry either in the VBBM or in the - EM. Verify that. - 2. Struct consistency checks - a. current size agrees with actual # of used entries - b. there are no empty elements in the hashed lists - c. each hash table entry points to a non-empty element or -1 - d. verify that there are no empty entries below the LWM - e. verify uniqueness of the entries + /* + 1. Every valid entry in the VSS has an entry either in the VBBM or in the + EM. Verify that. + 2. Struct consistency checks + a. current size agrees with actual # of used entries + b. there are no empty elements in the hashed lists + c. each hash table entry points to a non-empty element or -1 + d. verify that there are no empty entries below the LWM + e. verify uniqueness of the entries - */ + */ - int i, j, k, err; + int i, j, k, err; - /* Test 1 */ + /* Test 1 */ - OID_t oid; - uint32_t fbo; + OID_t oid; + uint32_t fbo; - for (i = 0; i < vss->capacity; i++) + for (i = 0; i < vss->capacity; i++) + { + if (storage[i].lbid != -1) { - if (storage[i].lbid != -1) - { - if (storage[i].vbFlag) - { - err = vbbm.lookup(storage[i].lbid, storage[i].verID, oid, - fbo); + if (storage[i].vbFlag) + { + err = vbbm.lookup(storage[i].lbid, storage[i].verID, oid, fbo); - if (err != 0) - { - cerr << "VSS: lbid=" << storage[i].lbid << " verID=" << - storage[i].verID << " vbFlag=true isn't in the VBBM" << - endl; - throw logic_error("VSS::checkConsistency(): a VSS entry with vbflag set is not in the VBBM"); - } - } - else - { + if (err != 0) + { + cerr << "VSS: lbid=" << storage[i].lbid << " verID=" << storage[i].verID + << " vbFlag=true isn't in the VBBM" << endl; + throw logic_error("VSS::checkConsistency(): a VSS entry with vbflag set is not in the VBBM"); + } + } + else + { // This version of em.lookup was made obsolete with multiple files per OID. // If ever want to really use this checkConsistency() function, this section // of code needs to be updated to use the new lookup() API. @@ -1214,147 +1183,143 @@ int VSS::checkConsistency(const VBBM& vbbm, ExtentMap& em) const } #endif - } - } + } } + } - /* Test 2a is already implemented */ - size(); + /* Test 2a is already implemented */ + size(); - /* Tests 2b & 2c - no empty elements reachable from the hash table */ + /* Tests 2b & 2c - no empty elements reachable from the hash table */ - int nextElement; + int nextElement; - for (i = 0; i < vss->numHashBuckets; i++) + for (i = 0; i < vss->numHashBuckets; i++) + { + if (hashBuckets[i] != -1) + for (nextElement = hashBuckets[i]; nextElement != -1; nextElement = storage[nextElement].next) + if (storage[nextElement].lbid == -1) + throw logic_error( + "VSS::checkConsistency(): an empty storage entry is reachable from the hash table"); + } + + /* Test 2d - verify that there are no empty entries below the LWM */ + + for (i = 0; i < vss->LWM; i++) + if (storage[i].lbid == -1) { - if (hashBuckets[i] != -1) - for (nextElement = hashBuckets[i]; nextElement != -1; - nextElement = storage[nextElement].next) - if (storage[nextElement].lbid == -1) - throw logic_error("VSS::checkConsistency(): an empty storage entry is reachable from the hash table"); + cerr << "VSS: LWM=" << vss->LWM << " first empty entry=" << i << endl; + throw logic_error("VSS::checkConsistency(): LWM accounting error"); } - /* Test 2d - verify that there are no empty entries below the LWM */ + /* Test 2e - verify uniqueness of each entry */ - for (i = 0; i < vss->LWM; i++) - if (storage[i].lbid == -1) - { - cerr << "VSS: LWM=" << vss->LWM << " first empty entry=" << i << endl; - throw logic_error("VSS::checkConsistency(): LWM accounting error"); - } + for (i = 0; i < vss->numHashBuckets; i++) + if (hashBuckets[i] != -1) + for (j = hashBuckets[i]; j != -1; j = storage[j].next) + for (k = storage[j].next; k != -1; k = storage[k].next) + if (storage[j].lbid == storage[k].lbid && storage[j].verID == storage[k].verID) + { + cerr << "VSS: lbid=" << storage[j].lbid << " verID=" << storage[j].verID << endl; + throw logic_error("VSS::checkConsistency(): Duplicate entry found"); + } - /* Test 2e - verify uniqueness of each entry */ - - for (i = 0; i < vss->numHashBuckets; i++) - if (hashBuckets[i] != -1) - for (j = hashBuckets[i]; j != -1; j = storage[j].next) - for (k = storage[j].next; k != -1; k = storage[k].next) - if (storage[j].lbid == storage[k].lbid && - storage[j].verID == storage[k].verID) - { - cerr << "VSS: lbid=" << storage[j].lbid << " verID=" << - storage[j].verID << endl; - throw logic_error("VSS::checkConsistency(): Duplicate entry found"); - } - - return 0; + return 0; } void VSS::setReadOnly() { - r_only = true; + r_only = true; } void VSS::getCurrentTxnIDs(set& list) const { - int i; + int i; - for (i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1 && storage[i].locked) - list.insert(storage[i].verID); + for (i = 0; i < vss->capacity; i++) + if (storage[i].lbid != -1 && storage[i].locked) + list.insert(storage[i].verID); } /* File Format: - VSS V1 magic (32-bits) - # of VSS entries in capacity (32-bits) - struct VSSEntry * # + VSS V1 magic (32-bits) + # of VSS entries in capacity (32-bits) + struct VSSEntry * # */ struct Header { - int magic; - int entries; + int magic; + int entries; }; // read lock void VSS::save(string filename) { - int i; - struct Header header; + int i; + struct Header header; - const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", IDBDataFile::USE_VBUF)); + const char* filename_p = filename.c_str(); + scoped_ptr out(IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", IDBDataFile::USE_VBUF)); - if (!out) + if (!out) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to open the file"); + } + + header.magic = VSS_MAGIC_V1; + header.entries = vss->currentSize; + + if (out->write((char*)&header, sizeof(header)) != sizeof(header)) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write header to the file"); + } + + int first = -1, last = -1, err; + size_t progress, writeSize; + for (i = 0; i < vss->capacity; i++) + { + if (storage[i].lbid != -1 && first == -1) + first = i; + else if (storage[i].lbid == -1 && first != -1) { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to open the file"); - } - - header.magic = VSS_MAGIC_V1; - header.entries = vss->currentSize; - - if (out->write((char*)&header, sizeof(header)) != sizeof(header)) - { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to write header to the file"); - } - - int first = -1, last = -1, err; - size_t progress, writeSize; - for (i = 0; i < vss->capacity; i++) - { - if (storage[i].lbid != -1 && first == -1) - first = i; - else if (storage[i].lbid == -1 && first != -1) + last = i; + writeSize = (last - first) * sizeof(VSSEntry); + progress = 0; + char* writePos = (char*)&storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) { - last = i; - writeSize = (last - first) * sizeof(VSSEntry); - progress = 0; - char *writePos = (char *) &storage[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to write the file"); - } - progress += err; - } - first = -1; + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write the file"); } + progress += err; + } + first = -1; } - if (first != -1) + } + if (first != -1) + { + writeSize = (vss->capacity - first) * sizeof(VSSEntry); + progress = 0; + char* writePos = (char*)&storage[first]; + while (progress < writeSize) { - writeSize = (vss->capacity - first) * sizeof(VSSEntry); - progress = 0; - char *writePos = (char *) &storage[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to write the file"); - } - progress += err; - } + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write the file"); + } + progress += err; } - + } } // Ideally, we;d like to get in and out of this fcn as quickly as possible. @@ -1373,119 +1338,115 @@ bool VSS::isEmpty(bool useLock) // high DML/query activity. return (fPVSSImpl->get()->currentSize == 0); #endif - //Should be race-free, but takes along time... - bool rc; + // Should be race-free, but takes along time... + bool rc; - if (useLock) - lock(READ); + if (useLock) + lock(READ); - rc = (fPVSSImpl->get()->currentSize == 0); + rc = (fPVSSImpl->get()->currentSize == 0); - if (useLock) - release(READ); + if (useLock) + release(READ); - return rc; + return rc; } - //#include "boost/date_time/posix_time/posix_time.hpp" -//using namespace boost::posix_time; +// using namespace boost::posix_time; void VSS::load(string filename) { - int i; - struct Header header; - struct VSSEntry entry; - //ptime time1, time2; + int i; + struct Header header; + struct VSSEntry entry; + // ptime time1, time2; - //time1 = microsec_clock::local_time(); - //cout << "loading the VSS " << time1 << endl; - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "rb", 0)); + // time1 = microsec_clock::local_time(); + // cout << "loading the VSS " << time1 << endl; + const char* filename_p = filename.c_str(); + scoped_ptr in( + IDBDataFile::open(IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "rb", 0)); - if (!in) + if (!in) + { + log_errno("VSS::load()"); + throw runtime_error("VSS::load(): Failed to open the file"); + } + + if (in->read((char*)&header, sizeof(header)) != sizeof(header)) + { + log_errno("VSS::load()"); + throw runtime_error("VSS::load(): Failed to read header"); + } + + if (header.magic != VSS_MAGIC_V1) + { + log("VSS::load(): Bad magic. Not a VSS file?"); + throw runtime_error("VSS::load(): Bad magic. Not a VSS file?"); + } + + if (header.entries < 0) + { + log("VSS::load(): Bad size. Not a VSS file?"); + throw runtime_error("VSS::load(): Bad size. Not a VSS file?"); + } + + growForLoad(header.entries); + + /* + for (i = 0; i < vss->capacity; i++) + storage[i].lbid = -1; + for (i = 0; i < vss->numHashBuckets; i++) + hashBuckets[i] = -1; + vss->currentSize = 0; + vss->lockedEntryCount = 0; + vss->LWM = 0; + */ + + size_t readSize = header.entries * sizeof(entry); + char* readBuf = new char[readSize]; + size_t progress = 0; + int err; + while (progress < readSize) + { + err = in->read(readBuf + progress, readSize - progress); + if (err < 0) { - log_errno("VSS::load()"); - throw runtime_error("VSS::load(): Failed to open the file"); + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to load, check the critical log file"); } - - if (in->read((char*)&header, sizeof(header)) != sizeof(header)) + else if (err == 0) { - log_errno("VSS::load()"); - throw runtime_error("VSS::load(): Failed to read header"); + log("VBBM::load(): Got early EOF"); + throw runtime_error("VBBM::load(): Got early EOF"); } + progress += err; + } - if (header.magic != VSS_MAGIC_V1) - { - log("VSS::load(): Bad magic. Not a VSS file?"); - throw runtime_error("VSS::load(): Bad magic. Not a VSS file?"); - } + VSSEntry* loadedEntries = (VSSEntry*)readBuf; + for (i = 0; i < header.entries; i++) + insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbFlag, loadedEntries[i].locked, + true); - if (header.entries < 0) - { - log("VSS::load(): Bad size. Not a VSS file?"); - throw runtime_error("VSS::load(): Bad size. Not a VSS file?"); - } - - growForLoad(header.entries); - - /* - for (i = 0; i < vss->capacity; i++) - storage[i].lbid = -1; - for (i = 0; i < vss->numHashBuckets; i++) - hashBuckets[i] = -1; - vss->currentSize = 0; - vss->lockedEntryCount = 0; - vss->LWM = 0; - */ - - size_t readSize = header.entries * sizeof(entry); - char *readBuf = new char[readSize]; - size_t progress = 0; - int err; - while (progress < readSize) - { - err = in->read(readBuf + progress, readSize - progress); - if (err < 0) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to load, check the critical log file"); - } - else if (err == 0) - { - log("VBBM::load(): Got early EOF"); - throw runtime_error("VBBM::load(): Got early EOF"); - } - progress += err; - } - - VSSEntry *loadedEntries = (VSSEntry *) readBuf; - for (i = 0; i < header.entries; i++) - insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbFlag, - loadedEntries[i].locked, true); - - //time2 = microsec_clock::local_time(); - //cout << "done loading " << time2 << " duration: " << time2-time1 << endl; + // time2 = microsec_clock::local_time(); + // cout << "done loading " << time2 << " duration: " << time2-time1 << endl; } - #ifdef BRM_DEBUG // read lock int VSS::getShmid() const { - return vssShmid; + return vssShmid; } #endif -QueryContext_vss::QueryContext_vss(const QueryContext& qc) : - currentScn(qc.currentScn) +QueryContext_vss::QueryContext_vss(const QueryContext& qc) : currentScn(qc.currentScn) { - txns.reset(new set()); + txns.reset(new set()); - for (uint32_t i = 0; i < qc.currentTxns->size(); i++) - txns->insert((*qc.currentTxns)[i]); + for (uint32_t i = 0; i < qc.currentTxns->size(); i++) + txns->insert((*qc.currentTxns)[i]); } -} //namespace +} // namespace BRM diff --git a/versioning/BRM/vss.h b/versioning/BRM/vss.h index 9595b0d26..4df38ac3e 100644 --- a/versioning/BRM/vss.h +++ b/versioning/BRM/vss.h @@ -52,23 +52,22 @@ // These config parameters need to be loaded -//will get a small hash function performance boost by using powers of 2 +// will get a small hash function performance boost by using powers of 2 #define VSSSTORAGE_INITIAL_COUNT 200000 -#define VSSSTORAGE_INITIAL_SIZE (VSSSTORAGE_INITIAL_COUNT*sizeof(VSSEntry)) +#define VSSSTORAGE_INITIAL_SIZE (VSSSTORAGE_INITIAL_COUNT * sizeof(VSSEntry)) #define VSSSTORAGE_INCREMENT_COUNT 20000 -#define VSSSTORAGE_INCREMENT (VSSSTORAGE_INCREMENT_COUNT*sizeof(VSSEntry)) +#define VSSSTORAGE_INCREMENT (VSSSTORAGE_INCREMENT_COUNT * sizeof(VSSEntry)) // (average list length = 4) -#define VSSTABLE_INITIAL_SIZE (50000*sizeof(int)) -#define VSSTABLE_INCREMENT (5000*sizeof(int)) +#define VSSTABLE_INITIAL_SIZE (50000 * sizeof(int)) +#define VSSTABLE_INCREMENT (5000 * sizeof(int)) -#define VSS_INITIAL_SIZE (sizeof(VSSShmsegHeader) + \ - VSSSTORAGE_INITIAL_SIZE + VSSTABLE_INITIAL_SIZE) +#define VSS_INITIAL_SIZE (sizeof(VSSShmsegHeader) + VSSSTORAGE_INITIAL_SIZE + VSSTABLE_INITIAL_SIZE) #define VSS_INCREMENT (VSSTABLE_INCREMENT + VSSSTORAGE_INCREMENT) #define VSS_SIZE(entries) \ - ((entries*sizeof(VSSEntry)) + (entries/4 * sizeof(int)) + sizeof(VSSShmsegHeader)) + ((entries * sizeof(VSSEntry)) + (entries / 4 * sizeof(int)) + sizeof(VSSShmsegHeader)) #if defined(_MSC_VER) && defined(xxxVSS_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -78,94 +77,93 @@ namespace BRM { - struct VSSEntry { - LBID_t lbid; - VER_t verID; - bool vbFlag; - bool locked; - int next; + LBID_t lbid; + VER_t verID; + bool vbFlag; + bool locked; + int next; #ifndef __LP64__ - uint32_t pad1; + uint32_t pad1; #endif - EXPORT VSSEntry(); + EXPORT VSSEntry(); }; struct VSSShmsegHeader { - int capacity; - int currentSize; - int LWM; - int numHashBuckets; - int lockedEntryCount; + int capacity; + int currentSize; + int LWM; + int numHashBuckets; + int lockedEntryCount; -// the rest of the overlay looks like this -// int hashBuckets[numHashBuckets]; -// VSSEntry storage[capacity]; + // the rest of the overlay looks like this + // int hashBuckets[numHashBuckets]; + // VSSEntry storage[capacity]; }; class QueryContext_vss { -public: - QueryContext_vss() : currentScn(0) - { - txns.reset(new std::set()); - } - QueryContext_vss(const QueryContext& qc); - VER_t currentScn; - boost::shared_ptr > txns; + public: + QueryContext_vss() : currentScn(0) + { + txns.reset(new std::set()); + } + QueryContext_vss(const QueryContext& qc); + VER_t currentScn; + boost::shared_ptr > txns; }; class VSSImpl { -public: - static VSSImpl* makeVSSImpl(unsigned key, off_t size, bool readOnly = false); + public: + static VSSImpl* makeVSSImpl(unsigned key, off_t size, bool readOnly = false); - inline void grow(unsigned key, off_t size) + inline void grow(unsigned key, off_t size) #ifndef NDBUG - { - fVSS.grow(key, size); - } + { + fVSS.grow(key, size); + } #else - { - int rc = fVSS.grow(key, size); - idbassert(rc == 0); - } + { + int rc = fVSS.grow(key, size); + idbassert(rc == 0); + } #endif - inline void makeReadOnly() - { - fVSS.setReadOnly(); - } - inline void clear(unsigned key, off_t size) - { - fVSS.clear(key, size); - } - inline void swapout(BRMShmImpl& rhs) - { - fVSS.swap(rhs); - rhs.destroy(); - } - inline unsigned key() const - { - return fVSS.key(); - } + inline void makeReadOnly() + { + fVSS.setReadOnly(); + } + inline void clear(unsigned key, off_t size) + { + fVSS.clear(key, size); + } + inline void swapout(BRMShmImpl& rhs) + { + fVSS.swap(rhs); + rhs.destroy(); + } + inline unsigned key() const + { + return fVSS.key(); + } - inline VSSShmsegHeader* get() const - { - return reinterpret_cast(fVSS.fMapreg.get_address()); - } + inline VSSShmsegHeader* get() const + { + return reinterpret_cast(fVSS.fMapreg.get_address()); + } -private: - VSSImpl(unsigned key, off_t size, bool readOnly = false); - ~VSSImpl(); - VSSImpl(const VSSImpl& rhs); - VSSImpl& operator=(const VSSImpl& rhs); + private: + VSSImpl(unsigned key, off_t size, bool readOnly = false); + ~VSSImpl(); + VSSImpl(const VSSImpl& rhs); + VSSImpl& operator=(const VSSImpl& rhs); - BRMShmImpl fVSS; + BRMShmImpl fVSS; - static boost::mutex fInstanceMutex; - static VSSImpl* fInstance; + static boost::mutex fInstanceMutex; + static VSSImpl* fInstance; }; class VBBM; @@ -204,99 +202,97 @@ class ExtentMap; class VSS : public Undoable { -public: + public: + enum OPS + { + NONE, + READ, + WRITE + }; - enum OPS - { - NONE, - READ, - WRITE - }; + EXPORT VSS(); + EXPORT ~VSS(); - EXPORT VSS(); - EXPORT ~VSS(); + EXPORT bool isLocked(const LBIDRange& l, VER_t txnID = -1) const; + EXPORT void removeEntry(LBID_t lbid, VER_t verID, std::vector* flushList); - EXPORT bool isLocked(const LBIDRange& l, VER_t txnID = -1) const; - EXPORT void removeEntry(LBID_t lbid, VER_t verID, std::vector* flushList); + // Note, the use_vbbm switch should be used for unit testing the VSS only + EXPORT void removeEntriesFromDB(const LBIDRange& range, VBBM& vbbm, bool use_vbbm = true); + EXPORT int lookup(LBID_t lbid, const QueryContext_vss&, VER_t txnID, VER_t* outVer, bool* vbFlag, + bool vbOnly = false) const; - // Note, the use_vbbm switch should be used for unit testing the VSS only - EXPORT void removeEntriesFromDB(const LBIDRange& range, VBBM& vbbm, bool use_vbbm = true); - EXPORT int lookup(LBID_t lbid, const QueryContext_vss&, VER_t txnID, VER_t* outVer, - bool* vbFlag, bool vbOnly = false) const; + /// Returns the version in the main DB files + EXPORT VER_t getCurrentVersion(LBID_t lbid, bool* isLocked) const; // returns the ver in the main DB files - /// Returns the version in the main DB files - EXPORT VER_t getCurrentVersion(LBID_t lbid, bool* isLocked) const; // returns the ver in the main DB files + /// Returns the highest version in the version buffer, less than max + EXPORT VER_t getHighestVerInVB(LBID_t lbid, VER_t max) const; - /// Returns the highest version in the version buffer, less than max - EXPORT VER_t getHighestVerInVB(LBID_t lbid, VER_t max) const; + /// returns true if that block is in the version buffer, false otherwise + EXPORT bool isVersioned(LBID_t lbid, VER_t version) const; - /// returns true if that block is in the version buffer, false otherwise - EXPORT bool isVersioned(LBID_t lbid, VER_t version) const; + EXPORT void setVBFlag(LBID_t lbid, VER_t verID, bool vbFlag); + EXPORT void insert(LBID_t, VER_t, bool vbFlag, bool locked, bool loading = false); + EXPORT void commit(VER_t txnID); + EXPORT void getUncommittedLBIDs(VER_t txnID, std::vector& lbids); + EXPORT void getUnlockedLBIDs(BlockList_t& lbids); + EXPORT void getLockedLBIDs(BlockList_t& lbids); + EXPORT void lock(OPS op); + EXPORT void release(OPS op); + EXPORT void setReadOnly(); - EXPORT void setVBFlag(LBID_t lbid, VER_t verID, bool vbFlag); - EXPORT void insert(LBID_t, VER_t, bool vbFlag, bool locked, bool loading = false); - EXPORT void commit(VER_t txnID); - EXPORT void getUncommittedLBIDs(VER_t txnID, std::vector& lbids); - EXPORT void getUnlockedLBIDs(BlockList_t& lbids); - EXPORT void getLockedLBIDs(BlockList_t& lbids); - EXPORT void lock(OPS op); - EXPORT void release(OPS op); - EXPORT void setReadOnly(); + EXPORT int checkConsistency(const VBBM& vbbm, ExtentMap& em) const; + EXPORT int size() const; + EXPORT bool hashEmpty() const; + EXPORT void getCurrentTxnIDs(std::set& txnList) const; - EXPORT int checkConsistency(const VBBM& vbbm, ExtentMap& em) const; - EXPORT int size() const; - EXPORT bool hashEmpty() const; - EXPORT void getCurrentTxnIDs(std::set& txnList) const; - - EXPORT void clear(); - EXPORT void load(std::string filename); - EXPORT void save(std::string filename); + EXPORT void clear(); + EXPORT void load(std::string filename); + EXPORT void save(std::string filename); #ifdef BRM_DEBUG - EXPORT int getShmid() const; + EXPORT int getShmid() const; #endif - EXPORT bool isEmpty(bool doLock = true); + EXPORT bool isEmpty(bool doLock = true); - /* Bug 2293. VBBM will use this fcn to determine whether a block is - * currently in use. */ - EXPORT bool isEntryLocked(LBID_t lbid, VER_t verID) const; - EXPORT bool isTooOld(LBID_t lbid, VER_t verID) const; + /* Bug 2293. VBBM will use this fcn to determine whether a block is + * currently in use. */ + EXPORT bool isEntryLocked(LBID_t lbid, VER_t verID) const; + EXPORT bool isTooOld(LBID_t lbid, VER_t verID) const; -private: - VSS(const VSS&); - VSS& operator=(const VSS&); + private: + VSS(const VSS&); + VSS& operator=(const VSS&); - struct VSSShmsegHeader* vss; - int* hashBuckets; - VSSEntry* storage; - bool r_only; - static boost::mutex mutex; // @bug5355 - made mutex static + struct VSSShmsegHeader* vss; + int* hashBuckets; + VSSEntry* storage; + bool r_only; + static boost::mutex mutex; // @bug5355 - made mutex static - key_t currentVSSShmkey; - int vssShmid; - MSTEntry* vssShminfo; - MasterSegmentTable mst; - static const int MAX_IO_RETRIES = 10; + key_t currentVSSShmkey; + int vssShmid; + MSTEntry* vssShminfo; + MasterSegmentTable mst; + static const int MAX_IO_RETRIES = 10; - key_t chooseShmkey() const; - void growVSS(); - void growForLoad(int count); - void initShmseg(); - void copyVSS(VSSShmsegHeader* dest); + key_t chooseShmkey() const; + void growVSS(); + void growForLoad(int count); + void initShmseg(); + void copyVSS(VSSShmsegHeader* dest); - int getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const; - void _insert(VSSEntry& e, VSSShmsegHeader* dest, int* destTable, VSSEntry* - destStorage, bool loading = false); - ShmKeys fShmKeys; + int getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const; + void _insert(VSSEntry& e, VSSShmsegHeader* dest, int* destTable, VSSEntry* destStorage, + bool loading = false); + ShmKeys fShmKeys; - VSSImpl* fPVSSImpl; - utils::Hasher hasher; + VSSImpl* fPVSSImpl; + utils::Hasher hasher; }; -} +} // namespace BRM #undef EXPORT -#endif // _VSS_H_ - +#endif // _VSS_H_ diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index 08d93f5d5..bd7a62ea5 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: cpimport.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: cpimport.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ #include #include @@ -60,140 +60,143 @@ using namespace execplan; namespace { char* pgmName = 0; -const std::string IMPORT_PATH_CWD ("."); -bool bDebug = false; +const std::string IMPORT_PATH_CWD("."); +bool bDebug = false; //@bug 4643: cpimport job ended during setup w/o any err msg. // Added a try/catch with logging to main() in case // the process was dying with an uncaught exception. enum TASK { - TASK_CMD_LINE_PARSING = 1, - TASK_INIT_CONFIG_CACHE = 2, - TASK_BRM_STATE_READY = 3, - TASK_BRM_STATE_READ_WRITE = 4, - TASK_SHUTDOWN_PENDING = 5, - TASK_SUSPEND_PENDING = 6, - TASK_ESTABLISH_JOBFILE = 7, - TASK_LOAD_JOBFILE = 8, - TASK_PROCESS_DATA = 9 + TASK_CMD_LINE_PARSING = 1, + TASK_INIT_CONFIG_CACHE = 2, + TASK_BRM_STATE_READY = 3, + TASK_BRM_STATE_READ_WRITE = 4, + TASK_SHUTDOWN_PENDING = 5, + TASK_SUSPEND_PENDING = 6, + TASK_ESTABLISH_JOBFILE = 7, + TASK_LOAD_JOBFILE = 8, + TASK_PROCESS_DATA = 9 }; -const char* taskLabels[] = -{ - "", - "parsing command line options", - "initializing config cache", - "checking BRM Ready state", - "checking BRM Read/Write state", - "checking for pending shutdown", - "checking for pending suspend", - "establishing job file", - "loading job file", - "processing data" -}; -} - +const char* taskLabels[] = {"", + "parsing command line options", + "initializing config cache", + "checking BRM Ready state", + "checking BRM Read/Write state", + "checking for pending shutdown", + "checking for pending suspend", + "establishing job file", + "loading job file", + "processing data"}; +} // namespace + //------------------------------------------------------------------------------ // Print command line usage //------------------------------------------------------------------------------ void printUsage() { - cout << endl << "Simple usage using positional parameters " - "(no XML job file):" << endl << - " cpimport.bin dbName tblName [loadFile] [-j jobID] " << endl << - " [-h] [-r readers] [-w parsers] [-s c] [-f path] [-b readBufs] " << endl << - " [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl << - " [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] " - "[-d debugLevel] [-i] " << endl << - " [-D] [-N] [-L rejectDir] [-T timeZone]" << endl << - " [-U username]" << endl << endl; + cout << endl + << "Simple usage using positional parameters " + "(no XML job file):" + << endl + << " cpimport.bin dbName tblName [loadFile] [-j jobID] " << endl + << " [-h] [-r readers] [-w parsers] [-s c] [-f path] [-b readBufs] " << endl + << " [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl + << " [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] " + "[-d debugLevel] [-i] " + << endl + << " [-D] [-N] [-L rejectDir] [-T timeZone]" << endl + << " [-U username]" << endl + << endl; - cout << endl << "Traditional usage without positional parameters " - "(XML job file required):" << endl << - " cpimport.bin -j jobID " << endl << - " [-h] [-r readers] [-w parsers] [-s c] [-f path] [-b readBufs] " << endl << - " [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl << - " [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] " - "[-d debugLevel] [-i] " << endl << - " [-p path] [-l loadFile]" << endl << - " [-D] [-N] [-L rejectDir] [-T timeZone]" << endl << - " [-U username]" << endl << endl; + cout << endl + << "Traditional usage without positional parameters " + "(XML job file required):" + << endl + << " cpimport.bin -j jobID " << endl + << " [-h] [-r readers] [-w parsers] [-s c] [-f path] [-b readBufs] " << endl + << " [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl + << " [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] " + "[-d debugLevel] [-i] " + << endl + << " [-p path] [-l loadFile]" << endl + << " [-D] [-N] [-L rejectDir] [-T timeZone]" << endl + << " [-U username]" << endl + << endl; - cout << " Positional parameters:" << endl << - " dbName Name of database to load" << endl << - " tblName Name of table to load" << endl << - " loadFile Optional input file name in current directory, " << - "unless a fully" << endl << - " qualified name is given. If not given, " << - "input read from stdin." << endl << endl; + cout << " Positional parameters:" << endl + << " dbName Name of database to load" << endl + << " tblName Name of table to load" << endl + << " loadFile Optional input file name in current directory, " + << "unless a fully" << endl + << " qualified name is given. If not given, " + << "input read from stdin." << endl + << endl; - cout << " Options:" << endl << - " -b Number of read buffers" << endl << - " -c Application read buffer size (in bytes)" << endl << - " -d Print different level (1-3) debug message " << endl << - " -e Maximum number of allowable errors per table" << endl << - " -f Data file directory path; " << endl << - " In simple usage:" << endl << - " Default is current working directory." << endl << - " -f option only applies if loadFile is specified." << endl << - " In traditional usage: " << endl << - " Default is /data/import." << endl << - " 'STDIN' (all caps) redirects input from stdin." << endl << - " -h Print this message" << endl << - " -i Print extended info to console, else this info only goes " - "to log file." << endl << - " -j Job id. In simple usage, default is the table OID." << endl << - " -l Name of input file to be loaded, relative to -f path," - << endl << - " unless a fully qualified input file name is given." << endl << - " -n NullOption (0-treat the string NULL as data (default);" << - endl << - " 1-treat the string NULL as a NULL value)" << - endl << - " -p Path for XML job description file" << endl << - " -r Number of readers" << endl << - " -s 'c' is the delimiter between column values" << endl << - " -w Number of parsers" << endl << - " -B I/O library read buffer size (in bytes)" << endl << - " -E Enclosed by character if field values are enclosed" << endl << - " -C Escape character used in conjunction with 'enclosed by' " << - "character," << endl << - " or as part of NULL escape sequence ('\\N'); default is '\\'" - << endl << - " -I Binary import; binaryOpt 1-import NULL values" << endl << - " 2-saturate NULL values" << endl << - " -S Treat string truncations as errors" << endl << - " -D Disable timeout when waiting for table lock" << endl << - " -N Disable console output" << endl << - " -L send *.err and *.bad (reject) files here" << endl << - " -T Timezone used for TIMESTAMP datatype" << endl << - " Possible values: \"SYSTEM\" (default)" << endl << - " : Offset in the form +/-HH:MM" << endl << endl << - " -y S3 Authentication Key (for S3 imports)" << endl << - " -K S3 Authentication Secret (for S3 imports)" << endl << - " -t S3 Bucket (for S3 imports)" << endl << - " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl << - " -g S3 Regions (for S3 imports)" << endl << - " -U username of new data files owner. Default is mysql" << endl; + cout << " Options:" << endl + << " -b Number of read buffers" << endl + << " -c Application read buffer size (in bytes)" << endl + << " -d Print different level (1-3) debug message " << endl + << " -e Maximum number of allowable errors per table" << endl + << " -f Data file directory path; " << endl + << " In simple usage:" << endl + << " Default is current working directory." << endl + << " -f option only applies if loadFile is specified." << endl + << " In traditional usage: " << endl + << " Default is /data/import." << endl + << " 'STDIN' (all caps) redirects input from stdin." << endl + << " -h Print this message" << endl + << " -i Print extended info to console, else this info only goes " + "to log file." + << endl + << " -j Job id. In simple usage, default is the table OID." << endl + << " -l Name of input file to be loaded, relative to -f path," << endl + << " unless a fully qualified input file name is given." << endl + << " -n NullOption (0-treat the string NULL as data (default);" << endl + << " 1-treat the string NULL as a NULL value)" << endl + << " -p Path for XML job description file" << endl + << " -r Number of readers" << endl + << " -s 'c' is the delimiter between column values" << endl + << " -w Number of parsers" << endl + << " -B I/O library read buffer size (in bytes)" << endl + << " -E Enclosed by character if field values are enclosed" << endl + << " -C Escape character used in conjunction with 'enclosed by' " + << "character," << endl + << " or as part of NULL escape sequence ('\\N'); default is '\\'" << endl + << " -I Binary import; binaryOpt 1-import NULL values" << endl + << " 2-saturate NULL values" << endl + << " -S Treat string truncations as errors" << endl + << " -D Disable timeout when waiting for table lock" << endl + << " -N Disable console output" << endl + << " -L send *.err and *.bad (reject) files here" << endl + << " -T Timezone used for TIMESTAMP datatype" << endl + << " Possible values: \"SYSTEM\" (default)" << endl + << " : Offset in the form +/-HH:MM" << endl + << endl + << " -y S3 Authentication Key (for S3 imports)" << endl + << " -K S3 Authentication Secret (for S3 imports)" << endl + << " -t S3 Bucket (for S3 imports)" << endl + << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl + << " -g S3 Regions (for S3 imports)" << endl + << " -U username of new data files owner. Default is mysql" << endl; - cout << " Example1:" << endl << - " cpimport.bin -j 1234" << endl << - " Example2: Some column values are enclosed within double quotes." << - endl << - " cpimport.bin -j 3000 -E '\"'" << endl << - " Example3: Import a nation table without a Job XML file" << endl << - " cpimport.bin -j 301 tpch nation nation.tbl" << endl; + cout << " Example1:" << endl + << " cpimport.bin -j 1234" << endl + << " Example2: Some column values are enclosed within double quotes." << endl + << " cpimport.bin -j 3000 -E '\"'" << endl + << " Example3: Import a nation table without a Job XML file" << endl + << " cpimport.bin -j 301 tpch nation nation.tbl" << endl; - exit( EXIT_SUCCESS ); + exit(EXIT_SUCCESS); } - + //------------------------------------------------------------------------------ // Signal handler to catch SIGTERM signal to terminate the process //------------------------------------------------------------------------------ void handleSigTerm(int i) { - std::cout << "Received SIGTERM to terminate the process..." << std::endl; - BulkStatus::setJobStatus( EXIT_FAILURE ); + std::cout << "Received SIGTERM to terminate the process..." << std::endl; + BulkStatus::setJobStatus(EXIT_FAILURE); } //------------------------------------------------------------------------------ @@ -201,25 +204,22 @@ void handleSigTerm(int i) //------------------------------------------------------------------------------ void handleControlC(int i) { - if (!BulkLoad::disableConsoleOutput()) - std::cout << "Received Control-C to terminate the process..." << std::endl; + if (!BulkLoad::disableConsoleOutput()) + std::cout << "Received Control-C to terminate the process..." << std::endl; - BulkStatus::setJobStatus( EXIT_FAILURE ); + BulkStatus::setJobStatus(EXIT_FAILURE); } #ifdef _MSC_VER -BOOL WINAPI HandlerCtrlCRoutine( _In_ DWORD dwCtrlType) +BOOL WINAPI HandlerCtrlCRoutine(_In_ DWORD dwCtrlType) { - // Log to syslog - logging::Message::Args errMsgArgs; - errMsgArgs.add("Received Break to terminate the process"); - SimpleSysLog::instance()->logMsg( - errMsgArgs, - logging::LOG_TYPE_DEBUG, - logging::M0087); + // Log to syslog + logging::Message::Args errMsgArgs; + errMsgArgs.add("Received Break to terminate the process"); + SimpleSysLog::instance()->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0087); - handleControlC(dwCtrlType); - return true; + handleControlC(dwCtrlType); + return true; } #endif @@ -227,39 +227,33 @@ BOOL WINAPI HandlerCtrlCRoutine( _In_ DWORD dwCtrlType) // If error occurs during startup, this function is called to log the specified // message and terminate the process. //------------------------------------------------------------------------------ -void startupError( const std::string& errMsg, bool showHint ) +void startupError(const std::string& errMsg, bool showHint) { - // Log to console + // Log to console + if (!BulkLoad::disableConsoleOutput()) + cerr << errMsg << endl; + + if (showHint) + { + std::ostringstream oss; + oss << "Try '" << pgmName << " -h' for more information."; + if (!BulkLoad::disableConsoleOutput()) - cerr << errMsg << endl; + cerr << oss.str() << endl; + } - if (showHint) - { - std::ostringstream oss; - oss << "Try '" << pgmName << " -h' for more information."; + // Log to syslog + logging::Message::Args errMsgArgs; + errMsgArgs.add(errMsg); + SimpleSysLog::instance()->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0087); - if (!BulkLoad::disableConsoleOutput()) - cerr << oss.str() << endl; - } + std::string jobIdStr("0"); + logging::Message::Args endMsgArgs; + endMsgArgs.add(jobIdStr); + endMsgArgs.add("FAILED"); + SimpleSysLog::instance()->logMsg(endMsgArgs, logging::LOG_TYPE_INFO, logging::M0082); - // Log to syslog - logging::Message::Args errMsgArgs; - errMsgArgs.add(errMsg); - SimpleSysLog::instance()->logMsg( - errMsgArgs, - logging::LOG_TYPE_ERROR, - logging::M0087); - - std::string jobIdStr("0"); - logging::Message::Args endMsgArgs; - endMsgArgs.add(jobIdStr); - endMsgArgs.add("FAILED"); - SimpleSysLog::instance()->logMsg( - endMsgArgs, - logging::LOG_TYPE_INFO, - logging::M0082); - - exit( EXIT_FAILURE ); + exit(EXIT_FAILURE); } //------------------------------------------------------------------------------ @@ -268,701 +262,647 @@ void startupError( const std::string& errMsg, bool showHint ) void setupSignalHandlers() { #ifdef _MSC_VER - BOOL brtn = SetConsoleCtrlHandler(HandlerCtrlCRoutine, true); + BOOL brtn = SetConsoleCtrlHandler(HandlerCtrlCRoutine, true); #else - struct sigaction ign; + struct sigaction ign; - // Ignore SIGPIPE signal - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); + // Ignore SIGPIPE signal + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &ign, 0); - //Ignore SIGHUP signals - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGHUP, &ign, 0); + // Ignore SIGHUP signals + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + sigaction(SIGHUP, &ign, 0); - // @bug 4344 enable Control-C by disabling this section of code - // Ignore SIGINT (Control-C) signal - //memset(&ign, 0, sizeof(ign)); - //ign.sa_handler = SIG_IGN; - //sigaction(SIGINT, &ign, 0); + // @bug 4344 enable Control-C by disabling this section of code + // Ignore SIGINT (Control-C) signal + // memset(&ign, 0, sizeof(ign)); + // ign.sa_handler = SIG_IGN; + // sigaction(SIGINT, &ign, 0); - // @bug 4344 enable Control-C by adding this section of code - // catch Control-C signal to terminate the program - struct sigaction act; - memset(&act, 0, sizeof(act)); - act.sa_handler = handleControlC; - sigaction(SIGINT, &act, 0); + // @bug 4344 enable Control-C by adding this section of code + // catch Control-C signal to terminate the program + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = handleControlC; + sigaction(SIGINT, &act, 0); - // catch SIGTERM signal to terminate the program - memset(&act, 0, sizeof(act)); - act.sa_handler = handleSigTerm; - sigaction(SIGTERM, &act, 0); + // catch SIGTERM signal to terminate the program + memset(&act, 0, sizeof(act)); + act.sa_handler = handleSigTerm; + sigaction(SIGTERM, &act, 0); #endif } //------------------------------------------------------------------------------ // Parse the command line arguments //------------------------------------------------------------------------------ -void parseCmdLineArgs( - int argc, - char** argv, - BulkLoad& curJob, - std::string& sJobIdStr, - std::string& sXMLJobDir, - std::string& sModuleIDandPID, - bool& bLogInfo2ToConsole, - std::string& xmlGenSchema, - std::string& xmlGenTable, - bool& bValidateColumnList ) +void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJobIdStr, + std::string& sXMLJobDir, std::string& sModuleIDandPID, bool& bLogInfo2ToConsole, + std::string& xmlGenSchema, std::string& xmlGenTable, bool& bValidateColumnList) { - std::string importPath; - std::string rptFileName; - int option; - bool bImportFileArg = false; - BulkModeType bulkMode = BULK_MODE_LOCAL; - std::string jobUUID; + std::string importPath; + std::string rptFileName; + int option; + bool bImportFileArg = false; + BulkModeType bulkMode = BULK_MODE_LOCAL; + std::string jobUUID; - while ( (option = getopt( - argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != EOF ) + while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != + EOF) + { + switch (option) { - switch (option) + case 'b': // -b: no. of read buffers + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) { - case 'b': // -b: no. of read buffers - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 1) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -b is invalid or out of range."), true ); - } - - int noOfReadBuffers = lValue; - curJob.setReadBufferCount( noOfReadBuffers ); - break; - } - - case 'c': // -c: read buffer size - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 1) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -c is invalid or out of range."), true ); - } - - int readBufferSize = lValue; - curJob.setReadBufferSize( readBufferSize ); - break; - } - - case 'd': // -d: debug level - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 1) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -d is invalid or out of range."), true ); - } - - int debugLevel = lValue; - - if ( debugLevel > 0 && debugLevel <= 3 ) - { - bDebug = true; - curJob.setAllDebug( (DebugLevel) debugLevel ); - - if (!BulkLoad::disableConsoleOutput()) - cout << "\nDebug level is set to " << debugLevel << endl; - } - - break; - } - - case 'e': // -e: max allowed errors - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 0) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -e is invalid or out of range."), true ); - } - - int maxErrors = lValue; - curJob.setMaxErrorCount( maxErrors ); - break; - } - - case 'f': // -f: import path - { - importPath = optarg; - std::string setAltErrMsg; - - if (curJob.setAlternateImportDir(importPath, - setAltErrMsg) != NO_ERROR) - startupError( setAltErrMsg, false ); - - break; - } - - case 'h': // -h: help - { - printUsage(); - break; - } - - case 'i': // -i: log info to console - { - bLogInfo2ToConsole = true; - break; - } - - case 'j': // -j: jobID - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 0) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -j is invalid or out of range."), true ); - } - - sJobIdStr = optarg; - break; - } - - case 'k': // -k: hidden option to keep (not delete) - { - // bulk rollback meta-data files - curJob.setKeepRbMetaFiles( true ); - break; - } - - case 'l': // -l: import load file(s) - { - bImportFileArg = true; - curJob.addToCmdLineImportFileList( std::string(optarg) ); - break; - } - - case 'm': // -m: bulk load mode - { - bulkMode = (BulkModeType)atoi( optarg ); - - if ((bulkMode != BULK_MODE_REMOTE_SINGLE_SRC) && - (bulkMode != BULK_MODE_REMOTE_MULTIPLE_SRC) && - (bulkMode != BULK_MODE_LOCAL)) - { - startupError ( std::string( - "Invalid bulk mode; can be 1,2, or 3"), true ); - } - - break; - } - - case 'n': // -n: treat "NULL" as null - { - int nullStringMode = atoi( optarg ); - - if ((nullStringMode != 0) && - (nullStringMode != 1)) - { - startupError ( std::string( - "Invalid NULL option; value can be 0 or 1"), true ); - } - - if (nullStringMode) - curJob.setNullStringMode(true); - else - curJob.setNullStringMode(false); - - break; - } - - case 'p': // -p: Job XML path - { - sXMLJobDir = optarg; - break; - } - - case 'r': // -r: num read threads - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 1) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -r is invalid or out of range."), true ); - } - - int numOfReaders = lValue; -#if !defined(__LP64__) && !defined(_MSC_VER) - - if (numOfReaders > 1) - { - cerr << "Note: resetting number of read threads to maximum" - << endl; - numOfReaders = 1; - } - -#endif - curJob.setNoOfReadThreads(numOfReaders); - - if (!BulkLoad::disableConsoleOutput()) - cout << "number of read threads : " << numOfReaders << endl; - - break; - } - - case 's': // -s: column delimiter - { - char delim; - - if (!strcmp(optarg, "\\t")) - { - delim = '\t'; - - if (!BulkLoad::disableConsoleOutput()) - cout << "Column delimiter : " << "\\t" << endl; - } - else - { - delim = optarg[0]; - - if (delim == '\t') // special case to print a - { - if (!BulkLoad::disableConsoleOutput()) - cout << "Column delimiter : '\\t'" << endl; - } - else - { - if (!BulkLoad::disableConsoleOutput()) - cout << "Column delimiter : " << delim << endl; - } - } - - curJob.setColDelimiter( delim ); - break; - } - - case 'u': // -u: import job UUID - { - jobUUID = optarg; - curJob.setJobUUID(jobUUID); - break; - } - - case 'w': // -w: num parse threads - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 1) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -w is invalid or out of range."), true ); - } - - int numOfParser = lValue; -#if !defined(__LP64__) && !defined(_MSC_VER) - - if (numOfParser > 3) - { - cerr << "Note: resetting number of parse threads to maximum" - << endl; - numOfParser = 3; - } - -#endif - curJob.setNoOfParseThreads( numOfParser ); - - if (!BulkLoad::disableConsoleOutput()) - cout << "number of parse threads : " << numOfParser << endl; - - break; - } - - case 'B': // -B: setvbuf read size - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || - (lValue < 1) || (lValue > INT_MAX)) - { - startupError ( std::string( - "Option -B is invalid or out of range."), true ); - } - - int vbufReadSize = lValue; - curJob.setVbufReadSize( vbufReadSize ); - break; - } - - case 'C': // -C: enclosed escape char - { - curJob.setEscapeChar( optarg[0] ); - - if (!BulkLoad::disableConsoleOutput()) - cout << "Escape Character : " << optarg[0] << endl; - - break; - } - - case 'E': // -E: enclosed by char - { - curJob.setEnclosedByChar( optarg[0] ); - - if (!BulkLoad::disableConsoleOutput()) - cout << "Enclosed by Character : " << optarg[0] << endl; - - break; - } - - case 'I': // -I: Binary import mode - { - ImportDataMode importMode = (ImportDataMode)atoi( optarg ); - - if ((importMode != IMPORT_DATA_BIN_ACCEPT_NULL) && - (importMode != IMPORT_DATA_BIN_SAT_NULL)) - { - startupError ( std::string( - "Invalid binary import option; value can be 1" - "(accept NULL values) or 2(saturate NULL values)"), - true ); - } - - curJob.setImportDataMode( importMode ); - break; - } - - case 'L': // -L: Error log directory - { - curJob.setErrorDir( optarg ); - break; - } - - case 'P': // -P: Calling moduleid - { - // and PID - sModuleIDandPID = optarg; - break; - } - - case 'R': // -R: distributed mode - { - // report file - rptFileName = optarg; - break; - } - - case 'S': // -S: Char & VarChar data - { - // greater than col def - curJob.setTruncationAsError(true); // are reported as err - break; - } - - case 'T': - { - std::string timeZone = optarg; - long offset; - - if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - startupError ( std::string( - "Value for option -T is invalid"), true ); - } - - curJob.setTimeZone( timeZone ); - break; - } - - case 'X': // Hidden extra options - { - if (!strcmp(optarg, "AllowMissingColumn")) - bValidateColumnList = false; - - break; - } - - case 'D': // disable table lock waiting timeout - { - curJob.disableTimeOut(true); - break; - } - - case 'N': // silent the output to console - { - BulkLoad::disableConsoleOutput(true); - break; - } - - case 'y': - { - curJob.setS3Key(optarg); - break; - } - - case 'K': - { - curJob.setS3Secret(optarg); - break; - } - - case 't': - { - curJob.setS3Bucket(optarg); - break; - } - - case 'H': - { - curJob.setS3Host(optarg); - break; - } - - case 'g': - { - curJob.setS3Region(optarg); - break; - } - - case 'U': - { - curJob.setUsername(optarg); - break; - } - - default : - { - ostringstream oss; - oss << "Unrecognized command line option (" << option << ")"; - startupError( oss.str(), true ); - } + startupError(std::string("Option -b is invalid or out of range."), true); } - } - curJob.setDefaultJobUUID(); + int noOfReadBuffers = lValue; + curJob.setReadBufferCount(noOfReadBuffers); + break; + } - // Inconsistent to specify -f STDIN with -l importFile - if ((bImportFileArg) && (importPath == "STDIN")) - { - startupError( std::string( - "-f STDIN is invalid with -l importFile."), true ); - } + case 'c': // -c: read buffer size + { + errno = 0; + long lValue = strtol(optarg, 0, 10); - // If distributed mode, make sure report filename is specified and that we - // can create the file using the specified path. - if ((bulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (bulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) - { - if (rptFileName.empty()) + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) { - startupError( std::string( - "Bulk modes 1 and 2 require -R rptFileName."), true ); + startupError(std::string("Option -c is invalid or out of range."), true); + } + + int readBufferSize = lValue; + curJob.setReadBufferSize(readBufferSize); + break; + } + + case 'd': // -d: debug level + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + { + startupError(std::string("Option -d is invalid or out of range."), true); + } + + int debugLevel = lValue; + + if (debugLevel > 0 && debugLevel <= 3) + { + bDebug = true; + curJob.setAllDebug((DebugLevel)debugLevel); + + if (!BulkLoad::disableConsoleOutput()) + cout << "\nDebug level is set to " << debugLevel << endl; + } + + break; + } + + case 'e': // -e: max allowed errors + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 0) || (lValue > INT_MAX)) + { + startupError(std::string("Option -e is invalid or out of range."), true); + } + + int maxErrors = lValue; + curJob.setMaxErrorCount(maxErrors); + break; + } + + case 'f': // -f: import path + { + importPath = optarg; + std::string setAltErrMsg; + + if (curJob.setAlternateImportDir(importPath, setAltErrMsg) != NO_ERROR) + startupError(setAltErrMsg, false); + + break; + } + + case 'h': // -h: help + { + printUsage(); + break; + } + + case 'i': // -i: log info to console + { + bLogInfo2ToConsole = true; + break; + } + + case 'j': // -j: jobID + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 0) || (lValue > INT_MAX)) + { + startupError(std::string("Option -j is invalid or out of range."), true); + } + + sJobIdStr = optarg; + break; + } + + case 'k': // -k: hidden option to keep (not delete) + { + // bulk rollback meta-data files + curJob.setKeepRbMetaFiles(true); + break; + } + + case 'l': // -l: import load file(s) + { + bImportFileArg = true; + curJob.addToCmdLineImportFileList(std::string(optarg)); + break; + } + + case 'm': // -m: bulk load mode + { + bulkMode = (BulkModeType)atoi(optarg); + + if ((bulkMode != BULK_MODE_REMOTE_SINGLE_SRC) && (bulkMode != BULK_MODE_REMOTE_MULTIPLE_SRC) && + (bulkMode != BULK_MODE_LOCAL)) + { + startupError(std::string("Invalid bulk mode; can be 1,2, or 3"), true); + } + + break; + } + + case 'n': // -n: treat "NULL" as null + { + int nullStringMode = atoi(optarg); + + if ((nullStringMode != 0) && (nullStringMode != 1)) + { + startupError(std::string("Invalid NULL option; value can be 0 or 1"), true); + } + + if (nullStringMode) + curJob.setNullStringMode(true); + else + curJob.setNullStringMode(false); + + break; + } + + case 'p': // -p: Job XML path + { + sXMLJobDir = optarg; + break; + } + + case 'r': // -r: num read threads + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + { + startupError(std::string("Option -r is invalid or out of range."), true); + } + + int numOfReaders = lValue; +#if !defined(__LP64__) && !defined(_MSC_VER) + + if (numOfReaders > 1) + { + cerr << "Note: resetting number of read threads to maximum" << endl; + numOfReaders = 1; + } + +#endif + curJob.setNoOfReadThreads(numOfReaders); + + if (!BulkLoad::disableConsoleOutput()) + cout << "number of read threads : " << numOfReaders << endl; + + break; + } + + case 's': // -s: column delimiter + { + char delim; + + if (!strcmp(optarg, "\\t")) + { + delim = '\t'; + + if (!BulkLoad::disableConsoleOutput()) + cout << "Column delimiter : " + << "\\t" << endl; } else { - std::ofstream rptFile( rptFileName.c_str() ); + delim = optarg[0]; - if ( rptFile.fail() ) - { - std::ostringstream oss; - oss << "Unable to open report file " << rptFileName; - startupError( oss.str(), false ); - } - - rptFile.close(); + if (delim == '\t') // special case to print a + { + if (!BulkLoad::disableConsoleOutput()) + cout << "Column delimiter : '\\t'" << endl; + } + else + { + if (!BulkLoad::disableConsoleOutput()) + cout << "Column delimiter : " << delim << endl; + } } - curJob.setBulkLoadMode( bulkMode, rptFileName ); + curJob.setColDelimiter(delim); + break; + } + + case 'u': // -u: import job UUID + { + jobUUID = optarg; + curJob.setJobUUID(jobUUID); + break; + } + + case 'w': // -w: num parse threads + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + { + startupError(std::string("Option -w is invalid or out of range."), true); + } + + int numOfParser = lValue; +#if !defined(__LP64__) && !defined(_MSC_VER) + + if (numOfParser > 3) + { + cerr << "Note: resetting number of parse threads to maximum" << endl; + numOfParser = 3; + } + +#endif + curJob.setNoOfParseThreads(numOfParser); + + if (!BulkLoad::disableConsoleOutput()) + cout << "number of parse threads : " << numOfParser << endl; + + break; + } + + case 'B': // -B: setvbuf read size + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + { + startupError(std::string("Option -B is invalid or out of range."), true); + } + + int vbufReadSize = lValue; + curJob.setVbufReadSize(vbufReadSize); + break; + } + + case 'C': // -C: enclosed escape char + { + curJob.setEscapeChar(optarg[0]); + + if (!BulkLoad::disableConsoleOutput()) + cout << "Escape Character : " << optarg[0] << endl; + + break; + } + + case 'E': // -E: enclosed by char + { + curJob.setEnclosedByChar(optarg[0]); + + if (!BulkLoad::disableConsoleOutput()) + cout << "Enclosed by Character : " << optarg[0] << endl; + + break; + } + + case 'I': // -I: Binary import mode + { + ImportDataMode importMode = (ImportDataMode)atoi(optarg); + + if ((importMode != IMPORT_DATA_BIN_ACCEPT_NULL) && (importMode != IMPORT_DATA_BIN_SAT_NULL)) + { + startupError(std::string("Invalid binary import option; value can be 1" + "(accept NULL values) or 2(saturate NULL values)"), + true); + } + + curJob.setImportDataMode(importMode); + break; + } + + case 'L': // -L: Error log directory + { + curJob.setErrorDir(optarg); + break; + } + + case 'P': // -P: Calling moduleid + { + // and PID + sModuleIDandPID = optarg; + break; + } + + case 'R': // -R: distributed mode + { + // report file + rptFileName = optarg; + break; + } + + case 'S': // -S: Char & VarChar data + { + // greater than col def + curJob.setTruncationAsError(true); // are reported as err + break; + } + + case 'T': + { + std::string timeZone = optarg; + long offset; + + if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + startupError(std::string("Value for option -T is invalid"), true); + } + + curJob.setTimeZone(timeZone); + break; + } + + case 'X': // Hidden extra options + { + if (!strcmp(optarg, "AllowMissingColumn")) + bValidateColumnList = false; + + break; + } + + case 'D': // disable table lock waiting timeout + { + curJob.disableTimeOut(true); + break; + } + + case 'N': // silent the output to console + { + BulkLoad::disableConsoleOutput(true); + break; + } + + case 'y': + { + curJob.setS3Key(optarg); + break; + } + + case 'K': + { + curJob.setS3Secret(optarg); + break; + } + + case 't': + { + curJob.setS3Bucket(optarg); + break; + } + + case 'H': + { + curJob.setS3Host(optarg); + break; + } + + case 'g': + { + curJob.setS3Region(optarg); + break; + } + + case 'U': + { + curJob.setUsername(optarg); + break; + } + + default: + { + ostringstream oss; + oss << "Unrecognized command line option (" << option << ")"; + startupError(oss.str(), true); + } } + } - // Get positional arguments, User can provide: - // 1. no positional parameters - // 2. Two positional parameters (schema and table names) - // 3. Three positional parameters (schema, table, and import file name) - if (optind < argc) // see if db schema name is given + curJob.setDefaultJobUUID(); + + // Inconsistent to specify -f STDIN with -l importFile + if ((bImportFileArg) && (importPath == "STDIN")) + { + startupError(std::string("-f STDIN is invalid with -l importFile."), true); + } + + // If distributed mode, make sure report filename is specified and that we + // can create the file using the specified path. + if ((bulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (bulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + { + if (rptFileName.empty()) { - xmlGenSchema = argv[optind]; // 1st pos parm - optind++; - - if (optind < argc) // see if table name is given - { - // Validate invalid options in conjunction with 2-3 positional - // parameter mode, which means we are using temp Job XML file. - if (bImportFileArg) - { - startupError( std::string( - "-l importFile is invalid with positional parameters"), - true ); - } - - if (!sXMLJobDir.empty()) - { - startupError( std::string( - "-p path is invalid with positional parameters."), true ); - } - - if (importPath == "STDIN") - { - startupError( std::string( - "-f STDIN is invalid with positional parameters."), true ); - } - - xmlGenTable = argv[optind]; // 2nd pos parm - optind++; - - if (optind < argc) // see if input file name is given - { - // 3rd pos parm - curJob.addToCmdLineImportFileList( std::string(argv[optind]) ); - - // Default to CWD if loadfile name given w/o -f path - if (importPath.empty()) - { - std::string setAltErrMsg; - - if (curJob.setAlternateImportDir( - std::string("."), setAltErrMsg) != NO_ERROR) - startupError( setAltErrMsg, false ); - } - } - else - { - // Invalid to specify -f if no load file name given - if (!importPath.empty()) - { - startupError( std::string( - "-f requires 3rd positional parameter (load file name)."), - true ); - } - - // Default to STDIN if no import file name given - std::string setAltErrMsg; - - if (curJob.setAlternateImportDir( - std::string("STDIN"), setAltErrMsg) != NO_ERROR) - startupError( setAltErrMsg, false ); - } - } - else - { - startupError( std::string( - "No table name specified with schema."), true ); - } + startupError(std::string("Bulk modes 1 and 2 require -R rptFileName."), true); } else { - // JobID is a required parameter with no positional parm mode, - // because we need the jobid to identify the input job xml file. - if (sJobIdStr.empty()) - { - startupError( std::string("No JobID specified."), true ); - } + std::ofstream rptFile(rptFileName.c_str()); + + if (rptFile.fail()) + { + std::ostringstream oss; + oss << "Unable to open report file " << rptFileName; + startupError(oss.str(), false); + } + + rptFile.close(); } + + curJob.setBulkLoadMode(bulkMode, rptFileName); + } + + // Get positional arguments, User can provide: + // 1. no positional parameters + // 2. Two positional parameters (schema and table names) + // 3. Three positional parameters (schema, table, and import file name) + if (optind < argc) // see if db schema name is given + { + xmlGenSchema = argv[optind]; // 1st pos parm + optind++; + + if (optind < argc) // see if table name is given + { + // Validate invalid options in conjunction with 2-3 positional + // parameter mode, which means we are using temp Job XML file. + if (bImportFileArg) + { + startupError(std::string("-l importFile is invalid with positional parameters"), true); + } + + if (!sXMLJobDir.empty()) + { + startupError(std::string("-p path is invalid with positional parameters."), true); + } + + if (importPath == "STDIN") + { + startupError(std::string("-f STDIN is invalid with positional parameters."), true); + } + + xmlGenTable = argv[optind]; // 2nd pos parm + optind++; + + if (optind < argc) // see if input file name is given + { + // 3rd pos parm + curJob.addToCmdLineImportFileList(std::string(argv[optind])); + + // Default to CWD if loadfile name given w/o -f path + if (importPath.empty()) + { + std::string setAltErrMsg; + + if (curJob.setAlternateImportDir(std::string("."), setAltErrMsg) != NO_ERROR) + startupError(setAltErrMsg, false); + } + } + else + { + // Invalid to specify -f if no load file name given + if (!importPath.empty()) + { + startupError(std::string("-f requires 3rd positional parameter (load file name)."), true); + } + + // Default to STDIN if no import file name given + std::string setAltErrMsg; + + if (curJob.setAlternateImportDir(std::string("STDIN"), setAltErrMsg) != NO_ERROR) + startupError(setAltErrMsg, false); + } + } + else + { + startupError(std::string("No table name specified with schema."), true); + } + } + else + { + // JobID is a required parameter with no positional parm mode, + // because we need the jobid to identify the input job xml file. + if (sJobIdStr.empty()) + { + startupError(std::string("No JobID specified."), true); + } + } } //------------------------------------------------------------------------------ // Print the path of the input load file(s), and the name of the job xml file. //------------------------------------------------------------------------------ -void printInputSource( - const std::string& alternateImportDir, - const std::string& jobDescFile, - const std::string& S3Bucket) +void printInputSource(const std::string& alternateImportDir, const std::string& jobDescFile, + const std::string& S3Bucket) { - if (!S3Bucket.empty()) + if (!S3Bucket.empty()) + { + cout << "Input file will be read from S3 Bucket : " << S3Bucket << ", file/object : " << jobDescFile + << endl; + } + else if (alternateImportDir.size() > 0) + { + if (alternateImportDir == IMPORT_PATH_CWD) { - cout << "Input file will be read from S3 Bucket : " << S3Bucket << ", file/object : " << jobDescFile << endl; - } - else if (alternateImportDir.size() > 0) - { - if (alternateImportDir == IMPORT_PATH_CWD) - { - char cwdBuf[4096]; - char *bufPtr = &cwdBuf[0]; - bufPtr = ::getcwd(cwdBuf, sizeof(cwdBuf)); + char cwdBuf[4096]; + char* bufPtr = &cwdBuf[0]; + bufPtr = ::getcwd(cwdBuf, sizeof(cwdBuf)); - if (!(BulkLoad::disableConsoleOutput())) - cout << "Input file(s) will be read from : " << bufPtr << endl; - } - else - { - if (!(BulkLoad::disableConsoleOutput())) - cout << "Input file(s) will be read from : " << - alternateImportDir << endl; - } + if (!(BulkLoad::disableConsoleOutput())) + cout << "Input file(s) will be read from : " << bufPtr << endl; } else { - if (!(BulkLoad::disableConsoleOutput())) - cout << "Input file(s) will be read from Bulkload root directory : " << - Config::getBulkRoot() << endl; + if (!(BulkLoad::disableConsoleOutput())) + cout << "Input file(s) will be read from : " << alternateImportDir << endl; } - + } + else + { if (!(BulkLoad::disableConsoleOutput())) - cout << "Job description file : " << jobDescFile << endl; + cout << "Input file(s) will be read from Bulkload root directory : " << Config::getBulkRoot() << endl; + } + + if (!(BulkLoad::disableConsoleOutput())) + cout << "Job description file : " << jobDescFile << endl; } //------------------------------------------------------------------------------ // Get TableOID string for the specified db and table name. //------------------------------------------------------------------------------ -void getTableOID(const std::string& xmlGenSchema, - const std::string& xmlGenTable, - std::string& tableOIDStr) +void getTableOID(const std::string& xmlGenSchema, const std::string& xmlGenTable, std::string& tableOIDStr) { - OID tableOID = 0; + OID tableOID = 0; - execplan::CalpontSystemCatalog::TableName tbl( - xmlGenSchema, xmlGenTable ); - - try - { - boost::shared_ptr cat = - CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - cat->identity(CalpontSystemCatalog::EC); - tableOID = cat->tableRID(tbl).objnum; - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Unable to set default JobID; " << - "Error getting OID for table " << - tbl.schema << '.' << tbl.table << ": " << ex.what(); - startupError( oss.str(), false ); - } - catch (...) - { - std::ostringstream oss; - oss << "Unable to set default JobID; " << - "Unknown error getting OID for table " << - tbl.schema << '.' << tbl.table; - startupError( oss.str(), false ); - } + execplan::CalpontSystemCatalog::TableName tbl(xmlGenSchema, xmlGenTable); + try + { + boost::shared_ptr cat = + CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(CalpontSystemCatalog::EC); + tableOID = cat->tableRID(tbl).objnum; + } + catch (std::exception& ex) + { std::ostringstream oss; - oss << tableOID; - tableOIDStr = oss.str(); + oss << "Unable to set default JobID; " + << "Error getting OID for table " << tbl.schema << '.' << tbl.table << ": " << ex.what(); + startupError(oss.str(), false); + } + catch (...) + { + std::ostringstream oss; + oss << "Unable to set default JobID; " + << "Unknown error getting OID for table " << tbl.schema << '.' << tbl.table; + startupError(oss.str(), false); + } + + std::ostringstream oss; + oss << tableOID; + tableOIDStr = oss.str(); } //------------------------------------------------------------------------------ // Construct temporary Job XML file if user provided schema, job, and @@ -974,114 +914,97 @@ void getTableOID(const std::string& xmlGenSchema, // alternateImportDir - alternate directory for input data files // sFileName(out)-filename path for temporary job xml file that is created //------------------------------------------------------------------------------ -void constructTempXmlFile( - const std::string& tempJobDir, - const std::string& sJobIdStr, - const std::string& xmlGenSchema, - const std::string& xmlGenTable, - const std::string& alternateImportDir, - const std::string& S3Bucket, - boost::filesystem::path& sFileName) +void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJobIdStr, + const std::string& xmlGenSchema, const std::string& xmlGenTable, + const std::string& alternateImportDir, const std::string& S3Bucket, + boost::filesystem::path& sFileName) { - // Construct the job description file name - std::string xmlErrMsg; - int rc = 0; - std::string tableOIDStr; - getTableOID(xmlGenSchema, - xmlGenTable, - tableOIDStr); - rc = XMLJob::genJobXMLFileName( std::string(), - tempJobDir, - sJobIdStr, - true, // using temp job xml file - xmlGenSchema, - xmlGenTable, - sFileName, - xmlErrMsg, - tableOIDStr ); + // Construct the job description file name + std::string xmlErrMsg; + int rc = 0; + std::string tableOIDStr; + getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr); + rc = XMLJob::genJobXMLFileName(std::string(), tempJobDir, sJobIdStr, + true, // using temp job xml file + xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, tableOIDStr); - if (rc != NO_ERROR) + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "cpimport.bin error creating temporary Job XML file name: " << xmlErrMsg; + startupError(oss.str(), false); + } + + printInputSource(alternateImportDir, sFileName.string(), S3Bucket); + + TempXMLGenData genData(sJobIdStr, xmlGenSchema, xmlGenTable); + XMLGenProc genProc(&genData, + false, // don't log to Jobxml_nnn.log + false); // generate XML file (not a syscat report) + + try + { + genProc.startXMLFile(); + execplan::CalpontSystemCatalog::TableName tbl(xmlGenSchema, xmlGenTable); + genProc.makeTableData(tbl); + + if (!genProc.makeColumnData(tbl)) { - std::ostringstream oss; - oss << "cpimport.bin error creating temporary Job XML file name: " << - xmlErrMsg; - startupError( oss.str(), false ); + std::ostringstream oss; + oss << "No columns for " << xmlGenSchema << '.' << xmlGenTable; + startupError(oss.str(), false); } + } + catch (runtime_error& ex) + { + std::ostringstream oss; + oss << "cpimport.bin runtime exception constructing temporary " + "Job XML file: " + << ex.what(); + startupError(oss.str(), false); + } + catch (exception& ex) + { + std::ostringstream oss; + oss << "cpimport.bin exception constructing temporary " + "Job XML file: " + << ex.what(); + startupError(oss.str(), false); + } + catch (...) + { + startupError(std::string("cpimport.bin " + "unknown exception constructing temporary Job XML file"), + false); + } - printInputSource( alternateImportDir, sFileName.string(), S3Bucket ); - - TempXMLGenData genData( sJobIdStr, xmlGenSchema, xmlGenTable ); - XMLGenProc genProc( &genData, - false, // don't log to Jobxml_nnn.log - false ); // generate XML file (not a syscat report) - - try - { - genProc.startXMLFile( ); - execplan::CalpontSystemCatalog::TableName tbl( - xmlGenSchema, xmlGenTable ); - genProc.makeTableData( tbl ); - - if ( !genProc.makeColumnData( tbl ) ) - { - std::ostringstream oss; - oss << "No columns for " << xmlGenSchema << '.' << xmlGenTable; - startupError( oss.str(), false ); - } - } - catch (runtime_error& ex) - { - std::ostringstream oss; - oss << "cpimport.bin runtime exception constructing temporary " - "Job XML file: " << ex.what(); - startupError( oss.str(), false ); - } - catch (exception& ex) - { - std::ostringstream oss; - oss << "cpimport.bin exception constructing temporary " - "Job XML file: " << ex.what(); - startupError( oss.str(), false ); - } - catch (...) - { - startupError( std::string( "cpimport.bin " - "unknown exception constructing temporary Job XML file"), - false ); - } - - genProc.writeXMLFile( sFileName.string() ); + genProc.writeXMLFile(sFileName.string()); } - - //------------------------------------------------------------------------------ // Verify we are running from a PM node. //------------------------------------------------------------------------------ void verifyNode() { - std::string localModuleType = Config::getLocalModuleType(); + std::string localModuleType = Config::getLocalModuleType(); - // Validate running on a PM - if (localModuleType != "pm") - { - startupError( std::string( "Exiting, " - "cpimport.bin can only be run on a PM node"), - true ); - } + // Validate running on a PM + if (localModuleType != "pm") + { + startupError(std::string("Exiting, " + "cpimport.bin can only be run on a PM node"), + true); + } } //------------------------------------------------------------------------------ // Log initiate message //------------------------------------------------------------------------------ -void logInitiateMsg( const char* initText ) +void logInitiateMsg(const char* initText) { - logging::Message::Args initMsgArgs; - initMsgArgs.add( initText ); - SimpleSysLog::instance()->logMsg( - initMsgArgs, - logging::LOG_TYPE_INFO, - logging::M0086); + logging::Message::Args initMsgArgs; + initMsgArgs.add(initText); + SimpleSysLog::instance()->logMsg(initMsgArgs, logging::LOG_TYPE_INFO, logging::M0086); } //------------------------------------------------------------------------------ @@ -1090,388 +1013,355 @@ void logInitiateMsg( const char* initText ) int main(int argc, char** argv) { #ifdef _MSC_VER - _setmaxstdio(2048); + _setmaxstdio(2048); #endif + setupSignalHandlers(); + + // Set locale language + const char* pLoc = setlocale(LC_ALL, ""); + if (pLoc) + { + // Log one line + cout << "Locale = " << pLoc; + } + else + { + cout << "Failed to set locale "; + } + setlocale(LC_NUMERIC, "C"); + + // Initialize singleton instance of syslogging + if (argc > 0) + pgmName = argv[0]; + + logging::IDBErrorInfo::instance(); + SimpleSysLog::instance()->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_BULK)); + + // Log job initiation unless user is asking for help + std::ostringstream ossArgList; + bool bHelpFlag = false; + + for (int m = 1; m < argc; m++) + { + if (strcmp(argv[m], "-h") == 0) + { + bHelpFlag = true; + break; + } + + if (!strcmp(argv[m], "\t")) // special case to print a + ossArgList << "'\\t'" << ' '; + else + ossArgList << argv[m] << ' '; + } + + if (!bHelpFlag) + { + logInitiateMsg(ossArgList.str().c_str()); + } + + BulkLoad curJob; + string sJobIdStr; + string sXMLJobDir; + string sModuleIDandPID; + bool bLogInfo2ToConsole = false; + bool bValidateColumnList = true; + bool bRollback = false; + bool bForce = false; + int rc = NO_ERROR; + std::string exceptionMsg; + TASK task; // track tasks being performed + // set this upfront + curJob.setErrorDir(string(MCSLOGDIR) + "/cpimport/"); + try + { + //-------------------------------------------------------------------------- + // Parse the command line arguments + //-------------------------------------------------------------------------- + task = TASK_CMD_LINE_PARSING; + string xmlGenSchema; + string xmlGenTable; + parseCmdLineArgs(argc, argv, curJob, sJobIdStr, sXMLJobDir, sModuleIDandPID, bLogInfo2ToConsole, + xmlGenSchema, xmlGenTable, bValidateColumnList); + + //-------------------------------------------------------------------------- + // Save basename portion of program path from argv[0] + //-------------------------------------------------------------------------- + string base; + string::size_type startBase = string(argv[0]).rfind('/'); + + if (startBase == string::npos) + base.assign(argv[0]); + else + base.assign(argv[0] + startBase + 1); + + curJob.setProcessName(base); + + if (bDebug) + logInitiateMsg("Command line arguments parsed"); + + //-------------------------------------------------------------------------- + // Init singleton classes (other than syslogging that we already setup) + //-------------------------------------------------------------------------- + task = TASK_INIT_CONFIG_CACHE; + + // Initialize cache used to store configuration parms from Columnstore.xml + Config::initConfigCache(); + + // Setup signal handlers "again" because HDFS plugin seems to be + // changing our settings to ignore ctrl-C and sigterm setupSignalHandlers(); - // Set locale language - const char* pLoc = setlocale(LC_ALL, ""); - if (pLoc) - { - // Log one line - cout << "Locale = " << pLoc; - } - else - { - cout << "Failed to set locale "; - } - setlocale(LC_NUMERIC, "C"); + // initialize singleton BRM Wrapper. Also init ExtentRows (in dbrm) from + // main thread, since ExtentMap::getExtentRows is not thread safe. + BRMWrapper::getInstance()->getInstance()->getExtentRows(); - // Initialize singleton instance of syslogging - if (argc > 0) - pgmName = argv[0]; + //-------------------------------------------------------------------------- + // Validate running on valid node + //-------------------------------------------------------------------------- + verifyNode(); - logging::IDBErrorInfo::instance(); - SimpleSysLog::instance()->setLoggingID( - logging::LoggingID(SUBSYSTEM_ID_WE_BULK) ); - - // Log job initiation unless user is asking for help - std::ostringstream ossArgList; - bool bHelpFlag = false; - - for (int m = 1; m < argc; m++) - { - if (strcmp(argv[m], "-h") == 0) - { - bHelpFlag = true; - break; - } - - if (!strcmp(argv[m], "\t")) // special case to print a - ossArgList << "'\\t'" << ' '; - else - ossArgList << argv[m] << ' '; - } - - if (!bHelpFlag) - { - logInitiateMsg( ossArgList.str().c_str() ); - } - - BulkLoad curJob; - string sJobIdStr; - string sXMLJobDir; - string sModuleIDandPID; - bool bLogInfo2ToConsole = false; - bool bValidateColumnList = true; - bool bRollback = false; - bool bForce = false; - int rc = NO_ERROR; - std::string exceptionMsg; - TASK task; // track tasks being performed - // set this upfront - curJob.setErrorDir(string(MCSLOGDIR) + "/cpimport/"); - try - { - //-------------------------------------------------------------------------- - // Parse the command line arguments - //-------------------------------------------------------------------------- - task = TASK_CMD_LINE_PARSING; - string xmlGenSchema; - string xmlGenTable; - parseCmdLineArgs( argc, argv, - curJob, sJobIdStr, sXMLJobDir, sModuleIDandPID, bLogInfo2ToConsole, - xmlGenSchema, xmlGenTable, bValidateColumnList ); - - //-------------------------------------------------------------------------- - // Save basename portion of program path from argv[0] - //-------------------------------------------------------------------------- - string base; - string::size_type startBase = string(argv[0]).rfind('/'); - - if (startBase == string::npos) - base.assign( argv[0] ); - else - base.assign( argv[0] + startBase + 1 ); - - curJob.setProcessName( base ); - - if (bDebug) - logInitiateMsg( "Command line arguments parsed" ); - - //-------------------------------------------------------------------------- - // Init singleton classes (other than syslogging that we already setup) - //-------------------------------------------------------------------------- - task = TASK_INIT_CONFIG_CACHE; - - // Initialize cache used to store configuration parms from Columnstore.xml - Config::initConfigCache(); - - // Setup signal handlers "again" because HDFS plugin seems to be - // changing our settings to ignore ctrl-C and sigterm - setupSignalHandlers(); - - // initialize singleton BRM Wrapper. Also init ExtentRows (in dbrm) from - // main thread, since ExtentMap::getExtentRows is not thread safe. - BRMWrapper::getInstance()->getInstance()->getExtentRows(); - - //-------------------------------------------------------------------------- - // Validate running on valid node - //-------------------------------------------------------------------------- - verifyNode( ); - - //-------------------------------------------------------------------------- - // Set scheduling priority for this cpimport.bin process - //-------------------------------------------------------------------------- + //-------------------------------------------------------------------------- + // Set scheduling priority for this cpimport.bin process + //-------------------------------------------------------------------------- #ifdef _MSC_VER - //FIXME + // FIXME #else - setpriority( PRIO_PROCESS, 0, Config::getBulkProcessPriority() ); + setpriority(PRIO_PROCESS, 0, Config::getBulkProcessPriority()); #endif - if (bDebug) - logInitiateMsg( "Config cache initialized" ); + if (bDebug) + logInitiateMsg("Config cache initialized"); - //-------------------------------------------------------------------------- - // Make sure DMLProc startup has completed before running a cpimport.bin job - //-------------------------------------------------------------------------- - task = TASK_BRM_STATE_READY; + //-------------------------------------------------------------------------- + // Make sure DMLProc startup has completed before running a cpimport.bin job + //-------------------------------------------------------------------------- + task = TASK_BRM_STATE_READY; - if (!BRMWrapper::getInstance()->isSystemReady()) - { - startupError( std::string( - "System is not ready. Verify that ColumnStore is up and ready " - "before running cpimport."), false ); - } - - if (bDebug) - logInitiateMsg( "BRM state verified: state is Ready" ); - - //-------------------------------------------------------------------------- - // Verify that the state of BRM is read/write - //-------------------------------------------------------------------------- - task = TASK_BRM_STATE_READ_WRITE; - int brmReadWriteStatus = BRMWrapper::getInstance()->isReadWrite(); - - if (brmReadWriteStatus != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << ec.errorString(brmReadWriteStatus) << - " cpimport.bin is terminating."; - startupError( oss.str(), false ); - } - - if (bDebug) - logInitiateMsg( "BRM state is Read/Write" ); - - //-------------------------------------------------------------------------- - // Make sure we're not about to shutdown - //-------------------------------------------------------------------------- - task = TASK_SHUTDOWN_PENDING; - int brmShutdownPending = BRMWrapper::getInstance()->isShutdownPending( - bRollback, bForce); - - if (brmShutdownPending != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << ec.errorString(brmShutdownPending) << - " cpimport.bin is terminating."; - startupError( oss.str(), false ); - } - - if (bDebug) - logInitiateMsg( "Verified no shutdown operation is pending" ); - - //-------------------------------------------------------------------------- - // Make sure we're not write suspended - //-------------------------------------------------------------------------- - task = TASK_SUSPEND_PENDING; - int brmSuspendPending = BRMWrapper::getInstance()->isSuspendPending(); - - if (brmSuspendPending != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << ec.errorString(brmSuspendPending) << - " cpimport.bin is terminating."; - startupError( oss.str(), false ); - } - - if (bDebug) - logInitiateMsg( "Verified no suspend operation is pending" ); - - //-------------------------------------------------------------------------- - // Set some flags - //-------------------------------------------------------------------------- - task = TASK_ESTABLISH_JOBFILE; - BRMWrapper::setUseVb( false ); - Cache::setUseCache ( false ); - - //-------------------------------------------------------------------------- - // Construct temporary Job XML file if user provided schema, job, and - // optional load filename. - //-------------------------------------------------------------------------- - boost::filesystem::path sFileName; - bool bUseTempJobFile = false; - - if (!BulkLoad::disableConsoleOutput()) - cout << std::endl; // print blank line before we start - - // Start tracking time to create/load jobfile; - // The elapsed time for this step is logged at the end of loadJobInfo() - curJob.startTimer(); - - if (!xmlGenSchema.empty()) // create temporary job file name - { - // If JobID is not provided, then default to the table OID - if (sJobIdStr.empty()) - { - std::string tableOIDStr; - getTableOID(xmlGenSchema, - xmlGenTable, - tableOIDStr); - - if ( !(BulkLoad::disableConsoleOutput())) - cout << "Using table OID " << tableOIDStr << - " as the default JOB ID" << std::endl; - - sJobIdStr = tableOIDStr; - } - - // No need to validate column list in job XML file for user errors, - // if cpimport.bin just generated the job XML file on-the-fly. - bValidateColumnList = false; - - bUseTempJobFile = true; - constructTempXmlFile(curJob.getTempJobDir(), - sJobIdStr, - xmlGenSchema, - xmlGenTable, - curJob.getAlternateImportDir(), - curJob.getS3Bucket(), - sFileName); - } - else // create user's persistent job file name - { - // Construct the job description file name - std::string xmlErrMsg; - std::string tableOIdStr(""); - rc = XMLJob::genJobXMLFileName( sXMLJobDir, - curJob.getJobDir(), - sJobIdStr, - bUseTempJobFile, - std::string(), - std::string(), - sFileName, - xmlErrMsg, - tableOIdStr ); - - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "cpimport.bin error creating Job XML file name: " << - xmlErrMsg; - startupError( oss.str(), false ); - } - - printInputSource( curJob.getAlternateImportDir(), sFileName.string(), curJob.getS3Bucket()); - } - - if (bDebug) - logInitiateMsg( "Job xml file is established" ); - - //------------------------------------------------------------------------- - // Bug 5415 Add HDFS MemBuffer vs. FileBuffer decision logic. - // MemoryCheckPercent. This controls at what percent of total memory be - // consumed by all processes before we switch from HdfsRdwrMemBuffer to - // HdfsRdwrFileBuffer. This is only used in Hdfs installations. - //------------------------------------------------------------------------- - config::Config* cf = config::Config::makeConfig(); - int checkPct = 95; - string strCheckPct = cf->getConfig("SystemConfig", "MemoryCheckPercent"); - - if ( strCheckPct.length() != 0 ) - checkPct = cf->uFromText(strCheckPct); - - //-------------------------------------------------------------------------- - // If we're HDFS, start the monitor thread. - // Otherwise, we don't need it, so don't waste the resources. - //-------------------------------------------------------------------------- - if (idbdatafile::IDBPolicy::useHdfs()) - { - new boost::thread(utils::MonitorProcMem(0, checkPct, - SUBSYSTEM_ID_WE_BULK)); - } - - //-------------------------------------------------------------------------- - // This is the real business - //-------------------------------------------------------------------------- - task = TASK_LOAD_JOBFILE; - rc = curJob.loadJobInfo( sFileName.string(), bUseTempJobFile, - argc, argv, bLogInfo2ToConsole, bValidateColumnList ); - - if ( rc != NO_ERROR ) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error in loading job information; " << - ec.errorString(rc) << "; cpimport.bin is terminating."; - startupError( oss.str(), false ); - } - - if (bDebug) - logInitiateMsg( "Job xml file is loaded" ); - - task = TASK_PROCESS_DATA; - - // Log start of job to INFO log - logging::Message::Args startMsgArgs; - startMsgArgs.add(sJobIdStr); - startMsgArgs.add(curJob.getSchema()); - SimpleSysLog::instance()->logMsg( - startMsgArgs, - logging::LOG_TYPE_INFO, - logging::M0081); - - curJob.printJob(); - - rc = curJob.processJob( ); - - if ( rc != NO_ERROR ) - { - if (!BulkLoad::disableConsoleOutput()) - cerr << endl << "Error in loading job data" << endl; - } - } - catch (std::exception& ex) + if (!BRMWrapper::getInstance()->isSystemReady()) { + startupError(std::string("System is not ready. Verify that ColumnStore is up and ready " + "before running cpimport."), + false); + } + + if (bDebug) + logInitiateMsg("BRM state verified: state is Ready"); + + //-------------------------------------------------------------------------- + // Verify that the state of BRM is read/write + //-------------------------------------------------------------------------- + task = TASK_BRM_STATE_READ_WRITE; + int brmReadWriteStatus = BRMWrapper::getInstance()->isReadWrite(); + + if (brmReadWriteStatus != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << ec.errorString(brmReadWriteStatus) << " cpimport.bin is terminating."; + startupError(oss.str(), false); + } + + if (bDebug) + logInitiateMsg("BRM state is Read/Write"); + + //-------------------------------------------------------------------------- + // Make sure we're not about to shutdown + //-------------------------------------------------------------------------- + task = TASK_SHUTDOWN_PENDING; + int brmShutdownPending = BRMWrapper::getInstance()->isShutdownPending(bRollback, bForce); + + if (brmShutdownPending != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << ec.errorString(brmShutdownPending) << " cpimport.bin is terminating."; + startupError(oss.str(), false); + } + + if (bDebug) + logInitiateMsg("Verified no shutdown operation is pending"); + + //-------------------------------------------------------------------------- + // Make sure we're not write suspended + //-------------------------------------------------------------------------- + task = TASK_SUSPEND_PENDING; + int brmSuspendPending = BRMWrapper::getInstance()->isSuspendPending(); + + if (brmSuspendPending != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << ec.errorString(brmSuspendPending) << " cpimport.bin is terminating."; + startupError(oss.str(), false); + } + + if (bDebug) + logInitiateMsg("Verified no suspend operation is pending"); + + //-------------------------------------------------------------------------- + // Set some flags + //-------------------------------------------------------------------------- + task = TASK_ESTABLISH_JOBFILE; + BRMWrapper::setUseVb(false); + Cache::setUseCache(false); + + //-------------------------------------------------------------------------- + // Construct temporary Job XML file if user provided schema, job, and + // optional load filename. + //-------------------------------------------------------------------------- + boost::filesystem::path sFileName; + bool bUseTempJobFile = false; + + if (!BulkLoad::disableConsoleOutput()) + cout << std::endl; // print blank line before we start + + // Start tracking time to create/load jobfile; + // The elapsed time for this step is logged at the end of loadJobInfo() + curJob.startTimer(); + + if (!xmlGenSchema.empty()) // create temporary job file name + { + // If JobID is not provided, then default to the table OID + if (sJobIdStr.empty()) + { + std::string tableOIDStr; + getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr); + + if (!(BulkLoad::disableConsoleOutput())) + cout << "Using table OID " << tableOIDStr << " as the default JOB ID" << std::endl; + + sJobIdStr = tableOIDStr; + } + + // No need to validate column list in job XML file for user errors, + // if cpimport.bin just generated the job XML file on-the-fly. + bValidateColumnList = false; + + bUseTempJobFile = true; + constructTempXmlFile(curJob.getTempJobDir(), sJobIdStr, xmlGenSchema, xmlGenTable, + curJob.getAlternateImportDir(), curJob.getS3Bucket(), sFileName); + } + else // create user's persistent job file name + { + // Construct the job description file name + std::string xmlErrMsg; + std::string tableOIdStr(""); + rc = XMLJob::genJobXMLFileName(sXMLJobDir, curJob.getJobDir(), sJobIdStr, bUseTempJobFile, + std::string(), std::string(), sFileName, xmlErrMsg, tableOIdStr); + + if (rc != NO_ERROR) + { std::ostringstream oss; - oss << "Uncaught exception caught in cpimport.bin main() while " << - taskLabels[ task ] << "; " << - ex.what(); - exceptionMsg = oss.str(); + oss << "cpimport.bin error creating Job XML file name: " << xmlErrMsg; + startupError(oss.str(), false); + } - if (task != TASK_PROCESS_DATA) - { - startupError( exceptionMsg, false ); - } + printInputSource(curJob.getAlternateImportDir(), sFileName.string(), curJob.getS3Bucket()); + } - rc = ERR_UNKNOWN; + if (bDebug) + logInitiateMsg("Job xml file is established"); + + //------------------------------------------------------------------------- + // Bug 5415 Add HDFS MemBuffer vs. FileBuffer decision logic. + // MemoryCheckPercent. This controls at what percent of total memory be + // consumed by all processes before we switch from HdfsRdwrMemBuffer to + // HdfsRdwrFileBuffer. This is only used in Hdfs installations. + //------------------------------------------------------------------------- + config::Config* cf = config::Config::makeConfig(); + int checkPct = 95; + string strCheckPct = cf->getConfig("SystemConfig", "MemoryCheckPercent"); + + if (strCheckPct.length() != 0) + checkPct = cf->uFromText(strCheckPct); + + //-------------------------------------------------------------------------- + // If we're HDFS, start the monitor thread. + // Otherwise, we don't need it, so don't waste the resources. + //-------------------------------------------------------------------------- + if (idbdatafile::IDBPolicy::useHdfs()) + { + new boost::thread(utils::MonitorProcMem(0, checkPct, SUBSYSTEM_ID_WE_BULK)); } //-------------------------------------------------------------------------- - // Log end of job to INFO log + // This is the real business //-------------------------------------------------------------------------- - logging::Message::Args endMsgArgs; - endMsgArgs.add(sJobIdStr); + task = TASK_LOAD_JOBFILE; + rc = curJob.loadJobInfo(sFileName.string(), bUseTempJobFile, argc, argv, bLogInfo2ToConsole, + bValidateColumnList); if (rc != NO_ERROR) { - std::string failMsg("FAILED"); - - if (exceptionMsg.length() > 0) - { - failMsg += "; "; - failMsg += exceptionMsg; - } - - endMsgArgs.add(failMsg.c_str()); - } - else - { - endMsgArgs.add("SUCCESS"); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error in loading job information; " << ec.errorString(rc) << "; cpimport.bin is terminating."; + startupError(oss.str(), false); } - SimpleSysLog::instance()->logMsg( - endMsgArgs, - logging::LOG_TYPE_INFO, - logging::M0082); + if (bDebug) + logInitiateMsg("Job xml file is loaded"); + + task = TASK_PROCESS_DATA; + + // Log start of job to INFO log + logging::Message::Args startMsgArgs; + startMsgArgs.add(sJobIdStr); + startMsgArgs.add(curJob.getSchema()); + SimpleSysLog::instance()->logMsg(startMsgArgs, logging::LOG_TYPE_INFO, logging::M0081); + + curJob.printJob(); + + rc = curJob.processJob(); if (rc != NO_ERROR) - return ( EXIT_FAILURE ); - else - return ( EXIT_SUCCESS ); + { + if (!BulkLoad::disableConsoleOutput()) + cerr << endl << "Error in loading job data" << endl; + } + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Uncaught exception caught in cpimport.bin main() while " << taskLabels[task] << "; " << ex.what(); + exceptionMsg = oss.str(); + + if (task != TASK_PROCESS_DATA) + { + startupError(exceptionMsg, false); + } + + rc = ERR_UNKNOWN; + } + + //-------------------------------------------------------------------------- + // Log end of job to INFO log + //-------------------------------------------------------------------------- + logging::Message::Args endMsgArgs; + endMsgArgs.add(sJobIdStr); + + if (rc != NO_ERROR) + { + std::string failMsg("FAILED"); + + if (exceptionMsg.length() > 0) + { + failMsg += "; "; + failMsg += exceptionMsg; + } + + endMsgArgs.add(failMsg.c_str()); + } + else + { + endMsgArgs.add("SUCCESS"); + } + + SimpleSysLog::instance()->logMsg(endMsgArgs, logging::LOG_TYPE_INFO, logging::M0082); + + if (rc != NO_ERROR) + return (EXIT_FAILURE); + else + return (EXIT_SUCCESS); } - diff --git a/writeengine/bulk/parsetest.cpp b/writeengine/bulk/parsetest.cpp index 40e4dbb3a..530a2f3d8 100644 --- a/writeengine/bulk/parsetest.cpp +++ b/writeengine/bulk/parsetest.cpp @@ -16,13 +16,13 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: cpimport.cpp 33 2006-08-24 14:36:17Z wzhou $ -* -******************************************************************************************/ + * $Id: cpimport.cpp 33 2006-08-24 14:36:17Z wzhou $ + * + ******************************************************************************************/ #include #include -#define ENV_BULK_DIR "CP_BULK_DIR" +#define ENV_BULK_DIR "CP_BULK_DIR" #include #include @@ -31,240 +31,233 @@ using namespace WriteEngine; #define MAXSTRINGS 100000 string Lines[MAXSTRINGS]; -typedef std::vector LineFldList; +typedef std::vector LineFldList; - -const int parseStr( const string& instr, LineFldList fields) +const int parseStr(const string& instr, LineFldList fields) { - typedef boost::tokenizer > - tokenizer; - boost::char_separator sep("|"); - tokenizer tokens(instr, sep); + typedef boost::tokenizer > tokenizer; + boost::char_separator sep("|"); + tokenizer tokens(instr, sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) - { -// std::cout << "<" << *tok_iter << "> "; - fields.push_back( *tok_iter ); - } - - //std::cout << "\n"; - return EXIT_SUCCESS; + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { + // std::cout << "<" << *tok_iter << "> "; + fields.push_back(*tok_iter); + } + // std::cout << "\n"; + return EXIT_SUCCESS; } -int strtok_test( const string& instr ) +int strtok_test(const string& instr) { - char* fragment; - char* search = (char*)malloc( instr.length()); + char* fragment; + char* search = (char*)malloc(instr.length()); - memcpy( search, instr.c_str(), instr.length()); + memcpy(search, instr.c_str(), instr.length()); - fragment = strtok(search, "|"); + fragment = strtok(search, "|"); - do - { - // printf("Token: %s\n", fragment); - fragment = strtok(NULL, "|"); - } - while (fragment); + do + { + // printf("Token: %s\n", fragment); + fragment = strtok(NULL, "|"); + } while (fragment); - free( search ); - return EXIT_SUCCESS; + free(search); + return EXIT_SUCCESS; } -int handrolled_test( const string& instr ) +int handrolled_test(const string& instr) { - char* search = (char*)malloc( instr.length()); - char* pos; - int count = 0; - int span; - string temp; - string results[10]; + char* search = (char*)malloc(instr.length()); + char* pos; + int count = 0; + int span; + string temp; + string results[10]; - memcpy( search, instr.data(), instr.length()); + memcpy(search, instr.data(), instr.length()); - if (search[0] == '|') - { - pos = search + 1; - } - else - { - pos = search; - } - - while (pos) - { - span = strcspn(pos, "|"); - - if (span) - { - temp.assign(pos, span); - results[count++].assign(temp); - } - - pos = index(pos + 1, '|'); - - if (pos) - { - pos++ ; - } - }; - - free( search ); - - //printf("\n%i dips", count); - return EXIT_SUCCESS; -} - -int handrolled_test2( string& instr, string Fields[] ) -{ - - char* search = (char*)malloc( instr.length() + 1 ); - char* pos; // pos is used to step inside the search string - int count = 0; // keeps track of fields found - int charspan; - int num_bars; - - strcpy( search, instr.c_str() ); + if (search[0] == '|') + { + pos = search + 1; + } + else + { pos = search; + } + while (pos) + { + span = strcspn(pos, "|"); - if (search[0] == '|') + if (span) { - pos = search + 1; - Fields[count++].assign(""); // a leading bar indicates an opening blank + temp.assign(pos, span); + results[count++].assign(temp); + } + + pos = index(pos + 1, '|'); + + if (pos) + { + pos++; + } + }; + + free(search); + + // printf("\n%i dips", count); + return EXIT_SUCCESS; +} + +int handrolled_test2(string& instr, string Fields[]) +{ + char* search = (char*)malloc(instr.length() + 1); + char* pos; // pos is used to step inside the search string + int count = 0; // keeps track of fields found + int charspan; + int num_bars; + + strcpy(search, instr.c_str()); + pos = search; + + if (search[0] == '|') + { + pos = search + 1; + Fields[count++].assign(""); // a leading bar indicates an opening blank + } + else + { + pos = search; + } + + while (pos < search + instr.length() - 1) + { + charspan = strcspn(pos, "|"); + + if (charspan) + { + Fields[count++].assign(pos, charspan); + pos += charspan + 1; } else { - pos = search; + Fields[count++].assign(""); + pos++; } - while (pos < search + instr.length() - 1 ) + num_bars = strspn(pos, "|"); + pos += num_bars; + + for (; num_bars > 0; num_bars--) { + Fields[count++].assign(""); + } + }; - charspan = strcspn(pos, "|"); + free(search); - if (charspan) - { - Fields[count++].assign(pos, charspan); - pos += charspan + 1; - } - else - { - Fields[count++].assign(""); - pos++; - } - - num_bars = strspn(pos, "|"); - pos += num_bars; - - for ( ; num_bars > 0; num_bars--) - { - Fields[count++].assign(""); - } - }; - - free( search ); - - return count; + return count; } int parseToken() { - return 1; + return 1; } - int build_data() { - int idx; + int idx; - for (idx = 0; idx < MAXSTRINGS; idx++) - { - //tpch data files are of the form - // item|item|item and the line may end with | - // even though this may wrongly suggest a blank value at the end - Lines[idx] = "12345|abcdef|banana|banana|"; // 'item item item item' - } + for (idx = 0; idx < MAXSTRINGS; idx++) + { + // tpch data files are of the form + // item|item|item and the line may end with | + // even though this may wrongly suggest a blank value at the end + Lines[idx] = "12345|abcdef|banana|banana|"; // 'item item item item' + } - //std::cout << Lines[idx-1] << endl; - return 0; + // std::cout << Lines[idx-1] << endl; + return 0; } int main(int argc, char** argv) { + string sJobIdStr, sBulkDir = "", sDbDir = "", sFileName, sTmp; + int fcount; + string Fields[1000]; + string search; + string searches[] = {"", + "|", + "|||||||||||||||", + "12345|abcdef|banana|", + "123456789012345678901234567890", + "|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|" + "12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|" + "12345678901234567890|12345678901234567890|12345678901234567890", + "|12345|abcdef|banana|bank123", + "|123456789012345678901234567890", + "12345|abcdef|banana|bank123", + "12345||abcdef||banana|bank", + "|12345||abcdef|banana|bank", + "|12345|abcdef|banana|bank|", + "|12345|abcdef|banana||", + "|12345|abcdef|banana|||"}; + // 14 elements + printf("\nAccuracy:"); - string sJobIdStr, sBulkDir = "", sDbDir = "", sFileName, sTmp; - int fcount; - string Fields[1000] ; - string search; - string searches[] = + for (int test = 0; test < 14; test++) + { + printf("\n\nSearch string %i: %s", test, searches[test].c_str()); + fcount = handrolled_test2(searches[test], Fields); + + for (int idx = 0; idx < fcount; idx++) { - "", "|", "|||||||||||||||", "12345|abcdef|banana|", "123456789012345678901234567890", - "|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890", - "|12345|abcdef|banana|bank123", "|123456789012345678901234567890", "12345|abcdef|banana|bank123", - "12345||abcdef||banana|bank", "|12345||abcdef|banana|bank", "|12345|abcdef|banana|bank|", - "|12345|abcdef|banana||", "|12345|abcdef|banana|||" - }; - // 14 elements - printf("\nAccuracy:"); - - - for (int test = 0; test < 14; test++) - { - printf("\n\nSearch string %i: %s", test, searches[test].c_str()); - fcount = handrolled_test2(searches[test], Fields); - - for (int idx = 0; idx < fcount; idx++) - { - printf("\nString %i: %s$", idx, Fields[idx].c_str()); - } + printf("\nString %i: %s$", idx, Fields[idx].c_str()); } + } - printf("\n\nSpeed:\n"); + printf("\n\nSpeed:\n"); - build_data(); - boost::timer t; + build_data(); + boost::timer t; - LineFldList parseFields; + LineFldList parseFields; - for (int idx = 0; idx < MAXSTRINGS; idx++) - { - parseStr(Lines[idx], parseFields); - } + for (int idx = 0; idx < MAXSTRINGS; idx++) + { + parseStr(Lines[idx], parseFields); + } - printf("Boost Parse Timer: %lf\n", t.elapsed()); - t.restart(); + printf("Boost Parse Timer: %lf\n", t.elapsed()); + t.restart(); - for (int idx = 0; idx < MAXSTRINGS; idx++) - { - strtok_test(Lines[idx]); - } + for (int idx = 0; idx < MAXSTRINGS; idx++) + { + strtok_test(Lines[idx]); + } - printf("Strtok Timer: %lf\n", t.elapsed()); + printf("Strtok Timer: %lf\n", t.elapsed()); - t.restart(); + t.restart(); - for (int idx = 0; idx < MAXSTRINGS; idx++) - { - handrolled_test(Lines[idx]); - } + for (int idx = 0; idx < MAXSTRINGS; idx++) + { + handrolled_test(Lines[idx]); + } - printf("Handrolled Timer: %lf\n", t.elapsed()); + printf("Handrolled Timer: %lf\n", t.elapsed()); - t.restart(); + t.restart(); - for (int idx = 0; idx < MAXSTRINGS; idx++) - { - fcount = handrolled_test2(Lines[idx], Fields); - } + for (int idx = 0; idx < MAXSTRINGS; idx++) + { + fcount = handrolled_test2(Lines[idx], Fields); + } - printf("Handrolled2 Timer: %lf\n", t.elapsed()); + printf("Handrolled2 Timer: %lf\n", t.elapsed()); - printf("\n"); - return 0; + printf("\n"); + return 0; } - - - - - diff --git a/writeengine/bulk/resource.h b/writeengine/bulk/resource.h index bf240aab9..c5b88727e 100644 --- a/writeengine/bulk/resource.h +++ b/writeengine/bulk/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/writeengine/bulk/we_brmreporter.cpp b/writeengine/bulk/we_brmreporter.cpp index af36bf154..7589a7070 100644 --- a/writeengine/bulk/we_brmreporter.cpp +++ b/writeengine/bulk/we_brmreporter.cpp @@ -39,20 +39,17 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -BRMReporter::BRMReporter(Log* logger, const std::string& tableName) : - fLog( logger ), - fTableName(tableName ) +BRMReporter::BRMReporter(Log* logger, const std::string& tableName) : fLog(logger), fTableName(tableName) { } //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ -BRMReporter::~BRMReporter( ) +BRMReporter::~BRMReporter() { } @@ -61,7 +58,7 @@ BRMReporter::~BRMReporter( ) //------------------------------------------------------------------------------ void BRMReporter::addToCPInfo(const BRM::CPInfoMerge& cpEntry) { - fCPInfo.push_back( cpEntry ); + fCPInfo.push_back(cpEntry); } //------------------------------------------------------------------------------ @@ -69,7 +66,7 @@ void BRMReporter::addToCPInfo(const BRM::CPInfoMerge& cpEntry) //------------------------------------------------------------------------------ void BRMReporter::addToHWMInfo(const BRM::BulkSetHWMArg& hwmEntry) { - fHWMInfo.push_back( hwmEntry ); + fHWMInfo.push_back(hwmEntry); } //------------------------------------------------------------------------------ @@ -77,7 +74,7 @@ void BRMReporter::addToHWMInfo(const BRM::BulkSetHWMArg& hwmEntry) //------------------------------------------------------------------------------ void BRMReporter::addToFileInfo(const BRM::FileInfo& fileEntry) { - fFileInfo.push_back( fileEntry ); + fFileInfo.push_back(fileEntry); } //------------------------------------------------------------------------------ @@ -85,7 +82,7 @@ void BRMReporter::addToFileInfo(const BRM::FileInfo& fileEntry) //------------------------------------------------------------------------------ void BRMReporter::addToDctnryFileInfo(const BRM::FileInfo& fileEntry) { - fDctnryFileInfo.push_back( fileEntry ); + fDctnryFileInfo.push_back(fileEntry); } //------------------------------------------------------------------------------ @@ -93,7 +90,7 @@ void BRMReporter::addToDctnryFileInfo(const BRM::FileInfo& fileEntry) //------------------------------------------------------------------------------ void BRMReporter::addToErrMsgEntry(const std::string& errCritMsg) { - fCritErrMsgs.push_back(errCritMsg); + fCritErrMsgs.push_back(errCritMsg); } //------------------------------------------------------------------------------ @@ -101,22 +98,22 @@ void BRMReporter::addToErrMsgEntry(const std::string& errCritMsg) //------------------------------------------------------------------------------ void BRMReporter::sendErrMsgToFile(const std::string& rptFileName) { - if ((!rptFileName.empty()) && (fRptFileName.empty())) - fRptFileName = rptFileName; + if ((!rptFileName.empty()) && (fRptFileName.empty())) + fRptFileName = rptFileName; - if ((!fRptFileName.empty()) && (fCritErrMsgs.size())) + if ((!fRptFileName.empty()) && (fCritErrMsgs.size())) + { + fRptFile.open(fRptFileName.c_str(), std::ios_base::app); + + if (fRptFile.good()) { - fRptFile.open( fRptFileName.c_str(), std::ios_base::app); - - if ( fRptFile.good() ) - { - for (unsigned int i = 0; i < fCritErrMsgs.size(); i++) - { - fRptFile << "MERR: " << fCritErrMsgs[i] << std::endl; - //std::cout <<"**********" << fCritErrMsgs[i] << std::endl; - } - } + for (unsigned int i = 0; i < fCritErrMsgs.size(); i++) + { + fRptFile << "MERR: " << fCritErrMsgs[i] << std::endl; + // std::cout <<"**********" << fCritErrMsgs[i] << std::endl; + } } + } } //------------------------------------------------------------------------------ @@ -129,308 +126,288 @@ void BRMReporter::sendErrMsgToFile(const std::string& rptFileName) // store). Any DB file changes should have been "confirmed" prior to calling // sendBRMInfo(). Once PrimProc cache is flushed, we can send the BRM updates. //------------------------------------------------------------------------------ -int BRMReporter::sendBRMInfo(const std::string& rptFileName, - const std::vector& errFiles, +int BRMReporter::sendBRMInfo(const std::string& rptFileName, const std::vector& errFiles, const std::vector& badFiles) { - int rc = NO_ERROR; + int rc = NO_ERROR; - // For HDFS, we need to flush PrimProc cache since we modify HDFS files - // by rewriting the files. - if (idbdatafile::IDBPolicy::useHdfs()) + // For HDFS, we need to flush PrimProc cache since we modify HDFS files + // by rewriting the files. + if (idbdatafile::IDBPolicy::useHdfs()) + { + std::vector allFileInfo; + + if (fFileInfo.size() > 0) { - std::vector allFileInfo; - - if ( fFileInfo.size() > 0 ) - { - for (unsigned k = 0; k < fFileInfo.size(); k++) - { - allFileInfo.push_back( fFileInfo[k] ); - } - } - - std::vector oidsToFlush; - std::set oidSet; - - if (fDctnryFileInfo.size() > 0) - { - for (unsigned k = 0; k < fDctnryFileInfo.size(); k++) - { - allFileInfo.push_back( fDctnryFileInfo[k] ); - oidSet.insert( fDctnryFileInfo[k].oid ); - } - - // Store dictionary oids in std::set first, to eliminate duplicates - if (oidSet.size() > 0) - { - for (std::set::const_iterator iter = oidSet.begin(); - iter != oidSet.end(); - ++iter) - { - oidsToFlush.push_back( *iter ); - } - } - } - - // Flush PrimProc FD cache - if (allFileInfo.size() > 0) - { - cacheutils::purgePrimProcFdCache(allFileInfo, - Config::getLocalModuleID()); - } - - // Flush PrimProc block cache - if (oidsToFlush.size() > 0) - cacheutils::flushOIDsFromCache(oidsToFlush); + for (unsigned k = 0; k < fFileInfo.size(); k++) + { + allFileInfo.push_back(fFileInfo[k]); + } } - // After flushing cache (for HDFS), now we can update BRM - if (rptFileName.empty()) + std::vector oidsToFlush; + std::set oidSet; + + if (fDctnryFileInfo.size() > 0) { - // Set Casual Partition (CP) info for BRM for this column. Be sure to - // do this before we set the HWM. Updating HWM 1st could cause a race - // condition resulting in a query being based on temporary outdated CP - // info. + for (unsigned k = 0; k < fDctnryFileInfo.size(); k++) + { + allFileInfo.push_back(fDctnryFileInfo[k]); + oidSet.insert(fDctnryFileInfo[k].oid); + } - rc = sendHWMandCPToBRM( ); - - // If HWM error occurs, we fail the job. - if (rc != NO_ERROR) + // Store dictionary oids in std::set first, to eliminate duplicates + if (oidSet.size() > 0) + { + for (std::set::const_iterator iter = oidSet.begin(); iter != oidSet.end(); ++iter) { - return rc; + oidsToFlush.push_back(*iter); } - } - else - { - fRptFileName = rptFileName; - - rc = openRptFile( ); - - if (rc != NO_ERROR) - { - return rc; - } - - sendCPToFile ( ); - sendHWMToFile( ); - - // Log the list of *.err and *.bad files - for (unsigned k = 0; k < errFiles.size(); k++) - { - fRptFile << "ERR: " << errFiles[k] << std::endl; - } - - for (unsigned k = 0; k < badFiles.size(); k++) - { - fRptFile << "BAD: " << badFiles[k] << std::endl; - } - + } } - return rc; + // Flush PrimProc FD cache + if (allFileInfo.size() > 0) + { + cacheutils::purgePrimProcFdCache(allFileInfo, Config::getLocalModuleID()); + } + + // Flush PrimProc block cache + if (oidsToFlush.size() > 0) + cacheutils::flushOIDsFromCache(oidsToFlush); + } + + // After flushing cache (for HDFS), now we can update BRM + if (rptFileName.empty()) + { + // Set Casual Partition (CP) info for BRM for this column. Be sure to + // do this before we set the HWM. Updating HWM 1st could cause a race + // condition resulting in a query being based on temporary outdated CP + // info. + + rc = sendHWMandCPToBRM(); + + // If HWM error occurs, we fail the job. + if (rc != NO_ERROR) + { + return rc; + } + } + else + { + fRptFileName = rptFileName; + + rc = openRptFile(); + + if (rc != NO_ERROR) + { + return rc; + } + + sendCPToFile(); + sendHWMToFile(); + + // Log the list of *.err and *.bad files + for (unsigned k = 0; k < errFiles.size(); k++) + { + fRptFile << "ERR: " << errFiles[k] << std::endl; + } + + for (unsigned k = 0; k < badFiles.size(); k++) + { + fRptFile << "BAD: " << badFiles[k] << std::endl; + } + } + + return rc; } //------------------------------------------------------------------------------ // Send HWM and CP update information to BRM //------------------------------------------------------------------------------ -int BRMReporter::sendHWMandCPToBRM( ) +int BRMReporter::sendHWMandCPToBRM() { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (fHWMInfo.size() > 0) + if (fHWMInfo.size() > 0) + { + std::ostringstream oss; + oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table " << fTableName << " to BRM"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + if (fCPInfo.size() > 0) + { + std::ostringstream oss; + oss << "Committing " << fCPInfo.size() << " CP update(s) for table " << fTableName << " to BRM"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + if ((fHWMInfo.size() > 0) || (fCPInfo.size() > 0)) + { + rc = BRMWrapper::getInstance()->bulkSetHWMAndCP(fHWMInfo, fCPInfo); + + if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table " << - fTableName << " to BRM"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error updating BRM with HWM and CP data for table " << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } + } - if (fCPInfo.size() > 0) - { - std::ostringstream oss; - oss << "Committing " << fCPInfo.size() << " CP update(s) for table " << - fTableName << " to BRM"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - if ((fHWMInfo.size() > 0) || (fCPInfo.size() > 0)) - { - rc = BRMWrapper::getInstance()->bulkSetHWMAndCP( fHWMInfo, fCPInfo ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error updating BRM with HWM and CP data for table " << - fTableName << "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - } - - return rc; + return rc; } //------------------------------------------------------------------------------ // Save HWM update information to a file //------------------------------------------------------------------------------ -void BRMReporter::sendHWMToFile( ) +void BRMReporter::sendHWMToFile() { - if (fHWMInfo.size() > 0) - { - std::ostringstream oss; - oss << "Writing " << fHWMInfo.size() << " HWM update(s) for table " << - fTableName << " to report file " << fRptFileName; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + if (fHWMInfo.size() > 0) + { + std::ostringstream oss; + oss << "Writing " << fHWMInfo.size() << " HWM update(s) for table " << fTableName << " to report file " + << fRptFileName; + fLog->logMsg(oss.str(), MSGLVL_INFO2); - for (unsigned int i = 0; i < fHWMInfo.size(); i++) - { - fRptFile << "HWM: " << fHWMInfo[i].oid << ' ' << - fHWMInfo[i].partNum << ' ' << - fHWMInfo[i].segNum << ' ' << - fHWMInfo[i].hwm << std::endl; - } + for (unsigned int i = 0; i < fHWMInfo.size(); i++) + { + fRptFile << "HWM: " << fHWMInfo[i].oid << ' ' << fHWMInfo[i].partNum << ' ' << fHWMInfo[i].segNum << ' ' + << fHWMInfo[i].hwm << std::endl; } + } } //------------------------------------------------------------------------------ // Send Casual Partition update information to BRM //------------------------------------------------------------------------------ -void BRMReporter::sendCPToFile( ) +void BRMReporter::sendCPToFile() { - if (fCPInfo.size() > 0) - { - std::ostringstream oss; - oss << "Writing " << fCPInfo.size() << " CP updates for table " << - fTableName << " to report file " << fRptFileName; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + if (fCPInfo.size() > 0) + { + std::ostringstream oss; + oss << "Writing " << fCPInfo.size() << " CP updates for table " << fTableName << " to report file " + << fRptFileName; + fLog->logMsg(oss.str(), MSGLVL_INFO2); - for (unsigned int i = 0; i < fCPInfo.size(); i++) - { - if (!datatypes::isWideDecimalType(fCPInfo[i].type, fCPInfo[i].colWidth)) - { - fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << - fCPInfo[i].seqNum << ' ' << - fCPInfo[i].type << ' ' << - fCPInfo[i].colWidth << ' ' << - fCPInfo[i].max << ' ' << - fCPInfo[i].min << ' ' << - fCPInfo[i].newExtent << std::endl; - } - else - { - datatypes::TSInt128 bigMin(&fCPInfo[i].bigMin); - datatypes::TSInt128 bigMax(&fCPInfo[i].bigMax); - - fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << - fCPInfo[i].seqNum << ' ' << - fCPInfo[i].type << ' ' << - fCPInfo[i].colWidth << ' ' << - bigMax << ' ' << - bigMin << ' ' << - fCPInfo[i].newExtent << std::endl; - } - } + for (unsigned int i = 0; i < fCPInfo.size(); i++) + { + if (!datatypes::isWideDecimalType(fCPInfo[i].type, fCPInfo[i].colWidth)) + { + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << fCPInfo[i].seqNum << ' ' << fCPInfo[i].type + << ' ' << fCPInfo[i].colWidth << ' ' << fCPInfo[i].max << ' ' << fCPInfo[i].min << ' ' + << fCPInfo[i].newExtent << std::endl; + } + else + { + datatypes::TSInt128 bigMin(&fCPInfo[i].bigMin); + datatypes::TSInt128 bigMax(&fCPInfo[i].bigMax); + + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << fCPInfo[i].seqNum << ' ' << fCPInfo[i].type + << ' ' << fCPInfo[i].colWidth << ' ' << bigMax << ' ' << bigMin << ' ' + << fCPInfo[i].newExtent << std::endl; + } } + } } //------------------------------------------------------------------------------ // Report Summary totals; only applicable if we are generating a report file. //------------------------------------------------------------------------------ void BRMReporter::reportTotals( - uint64_t totalReadRows, - uint64_t totalInsertedRows, - const std::vector >& satCounts) + uint64_t totalReadRows, uint64_t totalInsertedRows, + const std::vector >& + satCounts) { - if (fRptFile.is_open()) + if (fRptFile.is_open()) + { + fRptFile << "ROWS: " << totalReadRows << ' ' << totalInsertedRows << std::endl; + + for (unsigned k = 0; k < satCounts.size(); k++) { - fRptFile << "ROWS: " << totalReadRows << ' ' << - totalInsertedRows << std::endl; - - for (unsigned k = 0; k < satCounts.size(); k++) - { - if (boost::get<0>(satCounts[k]) > 0) - fRptFile << "DATA: " << k << ' ' << boost::get<0>(satCounts[k]) << ' ' << - boost::get<1>(satCounts[k]) << ' ' << boost::get<2>(satCounts[k]) << std::endl; - } - - closeRptFile(); + if (boost::get<0>(satCounts[k]) > 0) + fRptFile << "DATA: " << k << ' ' << boost::get<0>(satCounts[k]) << ' ' << boost::get<1>(satCounts[k]) + << ' ' << boost::get<2>(satCounts[k]) << std::endl; } + + closeRptFile(); + } } //------------------------------------------------------------------------------ // Generate report file indicating that user's import exceeded allowable error // limit. //------------------------------------------------------------------------------ -void BRMReporter::rptMaxErrJob(const std::string& rptFileName, - const std::vector& errFiles, - const std::vector& badFiles ) +void BRMReporter::rptMaxErrJob(const std::string& rptFileName, const std::vector& errFiles, + const std::vector& badFiles) { - // We only write out information if we are generating a report file. - if (!rptFileName.empty()) + // We only write out information if we are generating a report file. + if (!rptFileName.empty()) + { + fRptFileName = rptFileName; + + int rc = openRptFile(); + + // No need to return bad return code; we are already in a job that + // is aborting. openRptFile() at least logged the error. + if (rc != NO_ERROR) { - fRptFileName = rptFileName; - - int rc = openRptFile(); - - // No need to return bad return code; we are already in a job that - // is aborting. openRptFile() at least logged the error. - if (rc != NO_ERROR) - { - return; - } - - // Log the list of *.err and *.bad files - for (unsigned k = 0; k < errFiles.size(); k++) - { - fRptFile << "ERR: " << errFiles[k] << std::endl; - } - - for (unsigned k = 0; k < badFiles.size(); k++) - { - fRptFile << "BAD: " << badFiles[k] << std::endl; - } - - closeRptFile(); + return; } + + // Log the list of *.err and *.bad files + for (unsigned k = 0; k < errFiles.size(); k++) + { + fRptFile << "ERR: " << errFiles[k] << std::endl; + } + + for (unsigned k = 0; k < badFiles.size(); k++) + { + fRptFile << "BAD: " << badFiles[k] << std::endl; + } + + closeRptFile(); + } } //------------------------------------------------------------------------------ // Open BRM report file //------------------------------------------------------------------------------ -int BRMReporter::openRptFile( ) +int BRMReporter::openRptFile() { - fRptFile.open( fRptFileName.c_str() ); + fRptFile.open(fRptFileName.c_str()); - if ( fRptFile.fail() ) - { - int errRc = errno; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << "Error opening BRM report file " << fRptFileName << "; " << eMsg; - int rc = ERR_FILE_OPEN; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } + if (fRptFile.fail()) + { + int errRc = errno; + std::ostringstream oss; + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << "Error opening BRM report file " << fRptFileName << "; " << eMsg; + int rc = ERR_FILE_OPEN; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } - fRptFile << "#CP: startLBID seqnum type colwidth max min newExtent" << std::endl; - fRptFile << "#HWM: oid partition segment hwm" << std::endl; - fRptFile << "#ROWS: numRowsRead numRowsInserted" << std::endl; - fRptFile << "#DATA: columNum columnType columnOid numOutOfRangeValues" << std::endl; - fRptFile << "#ERR: error message file" << std::endl; - fRptFile << "#BAD: bad data file, with rejected rows" << std::endl; - fRptFile << "#MERR: critical error messages in cpimport.bin" << std::endl; + fRptFile << "#CP: startLBID seqnum type colwidth max min newExtent" << std::endl; + fRptFile << "#HWM: oid partition segment hwm" << std::endl; + fRptFile << "#ROWS: numRowsRead numRowsInserted" << std::endl; + fRptFile << "#DATA: columNum columnType columnOid numOutOfRangeValues" << std::endl; + fRptFile << "#ERR: error message file" << std::endl; + fRptFile << "#BAD: bad data file, with rejected rows" << std::endl; + fRptFile << "#MERR: critical error messages in cpimport.bin" << std::endl; - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ // Close BRM report file //------------------------------------------------------------------------------ -void BRMReporter::closeRptFile( ) +void BRMReporter::closeRptFile() { - fRptFile.close(); + fRptFile.close(); } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_brmreporter.h b/writeengine/bulk/we_brmreporter.h index e71310ff1..42caae2dd 100644 --- a/writeengine/bulk/we_brmreporter.h +++ b/writeengine/bulk/we_brmreporter.h @@ -47,101 +47,97 @@ class Log; */ class BRMReporter { -public: + public: + /** @brief Constructor + * @param logger Object used for logging + * @param tableName Name of relevant DB table + */ + BRMReporter(Log* logger, const std::string& tableName); - /** @brief Constructor - * @param logger Object used for logging - * @param tableName Name of relevant DB table - */ - BRMReporter(Log* logger, const std::string& tableName); + /** @brief Destructor + */ + ~BRMReporter(); - /** @brief Destructor - */ - ~BRMReporter( ); + /** @brief Add a CPInfoMerge entry to the output list + * @param cpEntry Casual Partition information to be added to output list + */ + void addToCPInfo(const BRM::CPInfoMerge& cpEntry); - /** @brief Add a CPInfoMerge entry to the output list - * @param cpEntry Casual Partition information to be added to output list - */ - void addToCPInfo(const BRM::CPInfoMerge& cpEntry); + /** @brief Add a BulkSetHWMArg entry to the output list + * @param hwmEntry HWM information to be be added to output list + */ + void addToHWMInfo(const BRM::BulkSetHWMArg& hwmEntry); - /** @brief Add a BulkSetHWMArg entry to the output list - * @param hwmEntry HWM information to be be added to output list - */ - void addToHWMInfo(const BRM::BulkSetHWMArg& hwmEntry); + /** @brief Add a Column FileInfo entry to the output list + * @param fileEntry file information to be be added to output list + */ + void addToFileInfo(const BRM::FileInfo& fileEntry); - /** @brief Add a Column FileInfo entry to the output list - * @param fileEntry file information to be be added to output list - */ - void addToFileInfo(const BRM::FileInfo& fileEntry); + /** @brief Add a Dictionary FileInfo entry to the output list + * @param fileEntry file information to be be added to output list + */ + void addToDctnryFileInfo(const BRM::FileInfo& fileEntry); - /** @brief Add a Dictionary FileInfo entry to the output list - * @param fileEntry file information to be be added to output list - */ - void addToDctnryFileInfo(const BRM::FileInfo& fileEntry); + /** @brief Add a ErrMsg entry to the output list + * @param Critical Error Message + */ + void addToErrMsgEntry(const std::string& errCritMsg); - /** @brief Add a ErrMsg entry to the output list - * @param Critical Error Message - */ - void addToErrMsgEntry(const std::string& errCritMsg); + /* @brief Save Critical error messages to the BRM report file + * @param rptFileName Name of file to save info, else info is dropped + */ + void sendErrMsgToFile(const std::string& rptFileName); - /* @brief Save Critical error messages to the BRM report file - * @param rptFileName Name of file to save info, else info is dropped - */ - void sendErrMsgToFile ( const std::string& rptFileName ); + /** @brief Send HWM and Casual Partition Data to the applicable destination + * @param rptFileName Name of file to save info, else info is sent to BRM. + * @param errFiles List of *.err filenames to record in report file + * @param badFiles List of *.bad filenames to record in report file + */ + int sendBRMInfo(const std::string& rptFileName, const std::vector& errFiles, + const std::vector& badFiles); - /** @brief Send HWM and Casual Partition Data to the applicable destination - * @param rptFileName Name of file to save info, else info is sent to BRM. - * @param errFiles List of *.err filenames to record in report file - * @param badFiles List of *.bad filenames to record in report file - */ - int sendBRMInfo( const std::string& rptFileName, - const std::vector& errFiles, - const std::vector& badFiles ); + /** @brief Report summary totals + * @param totalReadRows Total number of rows read + * @param totalInsertedRows Total number of rows inserted + * @param satCounts Number of out-of-range values for each column, + * Vector is vector of column oid, count pairs. + */ + void reportTotals( + uint64_t totalReadRows, uint64_t totalInsertedRows, + const std::vector >& + satCounts); - /** @brief Report summary totals - * @param totalReadRows Total number of rows read - * @param totalInsertedRows Total number of rows inserted - * @param satCounts Number of out-of-range values for each column, - * Vector is vector of column oid, count pairs. - */ - void reportTotals(uint64_t totalReadRows, - uint64_t totalInsertedRows, - const std::vector >& satCounts); + /** @brief Generate report for job that exceeds error limit + * @param rptFileName Name of file to save info, else info is dropped + * @param errFiles List of *.err filenames to record in report file + * @param badFiles List of *.bad filenames to record in report file + */ + void rptMaxErrJob(const std::string& rptFileName, const std::vector& errFiles, + const std::vector& badFiles); - /** @brief Generate report for job that exceeds error limit - * @param rptFileName Name of file to save info, else info is dropped - * @param errFiles List of *.err filenames to record in report file - * @param badFiles List of *.bad filenames to record in report file - */ - void rptMaxErrJob(const std::string& rptFileName, - const std::vector& errFiles, - const std::vector& badFiles ); + private: + // Disable copy constructor and assignment operator by declaring and + // not defining. + BRMReporter(const BRMReporter&); + BRMReporter& operator=(const BRMReporter&); -private: + int sendHWMandCPToBRM(); // send HWM and CP updates to BRM + void sendHWMToFile(); // save HWM updates to a report file + void sendCPToFile(); // save CP updates to a report file + int openRptFile(); // open BRM Report file + void closeRptFile(); // close BRM Report file - // Disable copy constructor and assignment operator by declaring and - // not defining. - BRMReporter(const BRMReporter&); - BRMReporter& operator=(const BRMReporter&); - - int sendHWMandCPToBRM( ); // send HWM and CP updates to BRM - void sendHWMToFile( ); // save HWM updates to a report file - void sendCPToFile ( ); // save CP updates to a report file - int openRptFile ( ); // open BRM Report file - void closeRptFile ( ); // close BRM Report file - - Log* fLog; // Logger - std::string fTableName; // Name of db table we are saving info for - BRM::CPInfoMergeList_t fCPInfo; // Collection of CP info to send - std::vector fHWMInfo; // Collection of HWM info to send - std::vector fCritErrMsgs; //Collection of CRIT ERRs - std::string fRptFileName; // Name of BRM report file - std::ofstream fRptFile; // BRM report file that is generated - std::vector fFileInfo; // Column files to flush from FDcache - std::vector fDctnryFileInfo;//Dct files to flush from FDcache + Log* fLog; // Logger + std::string fTableName; // Name of db table we are saving info for + BRM::CPInfoMergeList_t fCPInfo; // Collection of CP info to send + std::vector fHWMInfo; // Collection of HWM info to send + std::vector fCritErrMsgs; // Collection of CRIT ERRs + std::string fRptFileName; // Name of BRM report file + std::ofstream fRptFile; // BRM report file that is generated + std::vector fFileInfo; // Column files to flush from FDcache + std::vector fDctnryFileInfo; // Dct files to flush from FDcache }; -} // end of namespace +} // namespace WriteEngine -#endif // _WE_BRMREPORTER_H_ +#endif // _WE_BRMREPORTER_H_ diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index fae55bff8..566b99759 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_bulkload.cpp 4730 2013-08-08 21:41:13Z chao $ -* -*******************************************************************************/ + * $Id: we_bulkload.cpp 4730 2013-08-08 21:41:13Z chao $ + * + *******************************************************************************/ /** @file */ #define WE_BULKLOAD_DLLEXPORT @@ -55,7 +55,6 @@ #include "we_ddlcommandclient.h" #include "mcsconfig.h" - using namespace std; using namespace boost; using namespace dataconvert; @@ -63,21 +62,21 @@ using namespace dataconvert; namespace { const std::string IMPORT_PATH_STDIN("STDIN"); -const std::string IMPORT_PATH_CWD ("."); -const std::string LOG_SUFFIX = ".log"; // Job log file suffix -const std::string ERR_LOG_SUFFIX = ".err"; // Job err log file suffix -} +const std::string IMPORT_PATH_CWD("."); +const std::string LOG_SUFFIX = ".log"; // Job log file suffix +const std::string ERR_LOG_SUFFIX = ".err"; // Job err log file suffix +} // namespace -//extern WriteEngine::BRMWrapper* brmWrapperPtr; +// extern WriteEngine::BRMWrapper* brmWrapperPtr; namespace WriteEngine { /* static */ boost::ptr_vector BulkLoad::fTableInfo; -/* static */ boost::mutex* BulkLoad::fDDLMutex = 0; +/* static */ boost::mutex* BulkLoad::fDDLMutex = 0; -/* static */ const std::string BulkLoad::DIR_BULK_JOB("job"); -/* static */ const std::string BulkLoad::DIR_BULK_TEMP_JOB("tmpjob"); -/* static */ const std::string BulkLoad::DIR_BULK_IMPORT("/data/import/"); -/* static */ bool BulkLoad::fNoConsoleOutput = false; +/* static */ const std::string BulkLoad::DIR_BULK_JOB("job"); +/* static */ const std::string BulkLoad::DIR_BULK_TEMP_JOB("tmpjob"); +/* static */ const std::string BulkLoad::DIR_BULK_IMPORT("/data/import/"); +/* static */ bool BulkLoad::fNoConsoleOutput = false; //------------------------------------------------------------------------------ // A thread to periodically call dbrm to see if a user is @@ -91,82 +90,84 @@ namespace WriteEngine // If FORCE is set, we can't rollback. struct CancellationThread { - CancellationThread(BulkLoad* pBulkLoad) : fpBulkLoad(pBulkLoad) - {} - BulkLoad* fpBulkLoad; - void operator()() + CancellationThread(BulkLoad* pBulkLoad) : fpBulkLoad(pBulkLoad) + { + } + BulkLoad* fpBulkLoad; + void operator()() + { + bool bRollback = false; + bool bForce = false; + int iShutdown; + + while (fpBulkLoad->getContinue()) { - bool bRollback = false; - bool bForce = false; - int iShutdown; + usleep(1000000); // 1 seconds + // Check to see if someone has ordered a shutdown or suspend with + // rollback or force. + iShutdown = BRMWrapper::getInstance()->isShutdownPending(bRollback, bForce); - while (fpBulkLoad->getContinue()) + if (iShutdown != ERR_BRM_GET_SHUTDOWN) + { + if (bRollback) { - usleep(1000000); // 1 seconds - // Check to see if someone has ordered a shutdown or suspend with - // rollback or force. - iShutdown = BRMWrapper::getInstance()->isShutdownPending(bRollback, - bForce); + if (iShutdown == ERR_BRM_SHUTDOWN) + { + if (!BulkLoad::disableConsoleOutput()) + cout << "System stop has been ordered. Rollback" << endl; + } + else + { + if (!BulkLoad::disableConsoleOutput()) + cout << "Database writes have been suspended. Rollback" << endl; + } - if (iShutdown != ERR_BRM_GET_SHUTDOWN) - { - if (bRollback) - { - if (iShutdown == ERR_BRM_SHUTDOWN) - { - if (!BulkLoad::disableConsoleOutput()) - cout << "System stop has been ordered. Rollback" - << endl; - } - else - { - if (!BulkLoad::disableConsoleOutput()) - cout << "Database writes have been suspended. Rollback" - << endl; - } - - BulkStatus::setJobStatus( EXIT_FAILURE ); - } - else if (bForce) - { - if (!BulkLoad::disableConsoleOutput()) - cout << "Immediate system stop has been ordered. " - << "No rollback" - << endl; - } - } + BulkStatus::setJobStatus(EXIT_FAILURE); } + else if (bForce) + { + if (!BulkLoad::disableConsoleOutput()) + cout << "Immediate system stop has been ordered. " + << "No rollback" << endl; + } + } } + } }; //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -BulkLoad::BulkLoad() : - fColOp(new ColumnOpBulk()), - fColDelim('\0'), - fNoOfBuffers(-1), fBufferSize(-1), fFileVbufSize(-1), fMaxErrors(-1), - fNoOfParseThreads(3), fNoOfReadThreads(1), - fKeepRbMetaFiles(false), - fNullStringMode(false), - fEnclosedByChar('\0'), // not enabled unless user overrides enclosed by char - fEscapeChar('\0'), - fTotalTime(0.0), - fBulkMode(BULK_MODE_LOCAL), - fbTruncationAsError(false), - fImportDataMode(IMPORT_DATA_TEXT), - fbContinue(false), - fDisableTimeOut(false), - fUUID(boost::uuids::nil_generator()()), - fTimeZone("SYSTEM"), - fUsername("mysql") // MCOL-4328 default file owner +BulkLoad::BulkLoad() + : fColOp(new ColumnOpBulk()) + , fColDelim('\0') + , fNoOfBuffers(-1) + , fBufferSize(-1) + , fFileVbufSize(-1) + , fMaxErrors(-1) + , fNoOfParseThreads(3) + , fNoOfReadThreads(1) + , fKeepRbMetaFiles(false) + , fNullStringMode(false) + , fEnclosedByChar('\0') + , // not enabled unless user overrides enclosed by char + fEscapeChar('\0') + , fTotalTime(0.0) + , fBulkMode(BULK_MODE_LOCAL) + , fbTruncationAsError(false) + , fImportDataMode(IMPORT_DATA_TEXT) + , fbContinue(false) + , fDisableTimeOut(false) + , fUUID(boost::uuids::nil_generator()()) + , fTimeZone("SYSTEM") + , fUsername("mysql") // MCOL-4328 default file owner { - fTableInfo.clear(); - setDebugLevel( DEBUG_0 ); + fTableInfo.clear(); + setDebugLevel(DEBUG_0); - fDDLMutex = new boost::mutex(); - memset( &fStartTime, 0, sizeof(timeval) ); - memset( &fEndTime, 0, sizeof(timeval) ); + fDDLMutex = new boost::mutex(); + memset(&fStartTime, 0, sizeof(timeval)); + memset(&fEndTime, 0, sizeof(timeval)); } //------------------------------------------------------------------------------ @@ -174,10 +175,10 @@ BulkLoad::BulkLoad() : //------------------------------------------------------------------------------ BulkLoad::~BulkLoad() { - fTableInfo.clear(); - delete fDDLMutex; + fTableInfo.clear(); + delete fDDLMutex; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Set alternate directory path for import data files. If the specified @@ -192,41 +193,39 @@ BulkLoad::~BulkLoad() // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::setAlternateImportDir(const std::string& loadDir, - std::string& errMsg) +int BulkLoad::setAlternateImportDir(const std::string& loadDir, std::string& errMsg) { - if (loadDir == IMPORT_PATH_STDIN) + if (loadDir == IMPORT_PATH_STDIN) + { + fAlternateImportDir = loadDir; + } + else + { + if (access(loadDir.c_str(), R_OK | W_OK) < 0) { - fAlternateImportDir = loadDir; + int errnum = errno; + ostringstream oss; + oss << "Error gaining r/w access to import path " << loadDir << ": " << strerror(errnum); + errMsg = oss.str(); + return ERR_FILE_OPEN; + } + + if (loadDir == IMPORT_PATH_CWD) + { + fAlternateImportDir = loadDir; } else { - if ( access(loadDir.c_str(), R_OK | W_OK) < 0 ) - { - int errnum = errno; - ostringstream oss; - oss << "Error gaining r/w access to import path " << loadDir << - ": " << strerror(errnum); - errMsg = oss.str(); - return ERR_FILE_OPEN; - } - - if (loadDir == IMPORT_PATH_CWD) - { - fAlternateImportDir = loadDir; - } - else - { - if ( loadDir.c_str()[loadDir.size() - 1 ] == '/' ) - fAlternateImportDir = loadDir; - else - fAlternateImportDir = loadDir + "/"; - } + if (loadDir.c_str()[loadDir.size() - 1] == '/') + fAlternateImportDir = loadDir; + else + fAlternateImportDir = loadDir + "/"; } + } - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Load a job information @@ -242,151 +241,139 @@ int BulkLoad::setAlternateImportDir(const std::string& loadDir, // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::loadJobInfo( - const string& fullName, - bool bUseTempJobFile, - int argc, - char** argv, - bool bLogInfo2ToConsole, - bool bValidateColumnList ) +int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc, char** argv, + bool bLogInfo2ToConsole, bool bValidateColumnList) { - fJobFileName = fullName; - fRootDir = Config::getBulkRoot(); - fJobInfo.setTimeZone(fTimeZone); + fJobFileName = fullName; + fRootDir = Config::getBulkRoot(); + fJobInfo.setTimeZone(fTimeZone); - if ( !exists( fullName.c_str() ) ) + if (!exists(fullName.c_str())) + { + fLog.logMsg(" file " + fullName + " does not exist", ERR_FILE_NOT_EXIST, MSGLVL_ERROR); + return ERR_FILE_NOT_EXIST; + } + + std::string errMsg; + int rc = fJobInfo.loadJobXmlFile(fullName, bUseTempJobFile, bValidateColumnList, errMsg); + + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error loading job file " << fullName << "; " << errMsg; + fLog.logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + const Job& curJob = fJobInfo.getJob(); + string logFile, errlogFile; + logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + Convertor::int2Str(curJob.id) + LOG_SUFFIX; + errlogFile = + std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + Convertor::int2Str(curJob.id) + ERR_LOG_SUFFIX; + + if (disableConsoleOutput()) + fLog.setLogFileName(logFile.c_str(), errlogFile.c_str(), false); + else + fLog.setLogFileName(logFile.c_str(), errlogFile.c_str(), (int)bLogInfo2ToConsole); + + if (!(disableConsoleOutput())) + { + if (!BulkLoad::disableConsoleOutput()) + cout << "Log file for this job: " << logFile << std::endl; + + fLog.logMsg("successfully loaded job file " + fullName, MSGLVL_INFO1); + } + + if (argc > 1) + { + std::ostringstream oss; + oss << "Command line options: "; + + for (int k = 1; k < argc; k++) { - fLog.logMsg( " file " + fullName + " does not exist", - ERR_FILE_NOT_EXIST, MSGLVL_ERROR ); - return ERR_FILE_NOT_EXIST; + if (!strcmp(argv[k], "\t")) // special case to print a + oss << "'\\t'" + << " "; + else + oss << argv[k] << " "; } - std::string errMsg; - int rc = fJobInfo.loadJobXmlFile( fullName, bUseTempJobFile, - bValidateColumnList, errMsg ); + fLog.logMsg(oss.str(), MSGLVL_INFO2); + } - if ( rc != NO_ERROR ) + // Validate that each table has 1 or more columns referenced in the xml file + for (unsigned i = 0; i < curJob.jobTableList.size(); i++) + { + if (curJob.jobTableList[i].colList.size() == 0) { + rc = ERR_INVALID_PARAM; + fLog.logMsg( + "No column definitions in job description file for " + "table " + + curJob.jobTableList[i].tblName, + rc, MSGLVL_ERROR); + return rc; + } + } + + // Validate that the user's xml file has been regenerated since the + // required tblOid attribute was added to the Table tag for table locking. + for (unsigned i = 0; i < curJob.jobTableList.size(); i++) + { + if (curJob.jobTableList[i].mapOid == 0) + { + rc = ERR_XML_PARSE; + fLog.logMsg("Outdated job file " + fullName + "; missing required 'tblOid' table attribute." + + " Please regenerate this xml file.", + rc, MSGLVL_ERROR); + return rc; + } + } + + for (unsigned kT = 0; kT < curJob.jobTableList.size(); kT++) + { + for (unsigned kC = 0; kC < curJob.jobTableList[kT].colList.size(); kC++) + { + if (!compress::CompressInterface::isCompressionAvail( + curJob.jobTableList[kT].colList[kC].compressionType)) + { std::ostringstream oss; - oss << "Error loading job file " << fullName << "; " << errMsg; - fLog.logMsg( oss.str(), rc, MSGLVL_ERROR ); + oss << "Specified compression type (" << curJob.jobTableList[kT].colList[kC].compressionType + << ") for table " << curJob.jobTableList[kT].tblName << " and column " + << curJob.jobTableList[kT].colList[kC].colName << " is not available for use."; + rc = ERR_COMP_UNAVAIL_TYPE; + fLog.logMsg(oss.str(), rc, MSGLVL_ERROR); return rc; + } } + } - const Job& curJob = fJobInfo.getJob(); - string logFile, errlogFile; - logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + - Convertor::int2Str( curJob.id ) + LOG_SUFFIX; - errlogFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + - Convertor::int2Str( curJob.id ) + ERR_LOG_SUFFIX; - - if (disableConsoleOutput()) - fLog.setLogFileName(logFile.c_str(), errlogFile.c_str(), false); - else - fLog.setLogFileName(logFile.c_str(), errlogFile.c_str(), (int)bLogInfo2ToConsole); - - if (!(disableConsoleOutput())) - { - if (!BulkLoad::disableConsoleOutput()) - cout << "Log file for this job: " << logFile << std::endl; - - fLog.logMsg( "successfully loaded job file " + fullName, MSGLVL_INFO1 ); - } - - if (argc > 1) - { - std::ostringstream oss; - oss << "Command line options: "; - - for (int k = 1; k < argc; k++) - { - if (!strcmp(argv[k], "\t")) // special case to print a - oss << "'\\t'" << " "; - else - oss << argv[k] << " "; - } - - fLog.logMsg( oss.str(), MSGLVL_INFO2 ); - } - - // Validate that each table has 1 or more columns referenced in the xml file - for (unsigned i = 0; i < curJob.jobTableList.size(); i++) - { - if ( curJob.jobTableList[i].colList.size() == 0) - { - rc = ERR_INVALID_PARAM; - fLog.logMsg( "No column definitions in job description file for " - "table " + curJob.jobTableList[i].tblName, - rc, MSGLVL_ERROR ); - return rc; - } - } - - // Validate that the user's xml file has been regenerated since the - // required tblOid attribute was added to the Table tag for table locking. - for (unsigned i = 0; i < curJob.jobTableList.size(); i++) - { - if (curJob.jobTableList[i].mapOid == 0) - { - rc = ERR_XML_PARSE; - fLog.logMsg( "Outdated job file " + fullName + - "; missing required 'tblOid' table attribute." + - " Please regenerate this xml file.", - rc, MSGLVL_ERROR ); - return rc; - } - } - + // If binary import, do not allow tags in the Job file + if ((fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) || (fImportDataMode == IMPORT_DATA_BIN_SAT_NULL)) + { for (unsigned kT = 0; kT < curJob.jobTableList.size(); kT++) { - for (unsigned kC = 0; kC < curJob.jobTableList[kT].colList.size(); kC++) - { - if (!compress::CompressInterface::isCompressionAvail( - curJob.jobTableList[kT].colList[kC].compressionType)) - { - std::ostringstream oss; - oss << "Specified compression type (" << - curJob.jobTableList[kT].colList[kC].compressionType << - ") for table " << curJob.jobTableList[kT].tblName << - " and column " << - curJob.jobTableList[kT].colList[kC].colName << - " is not available for use."; - rc = ERR_COMP_UNAVAIL_TYPE; - fLog.logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - } + if (curJob.jobTableList[kT].fIgnoredFields.size() > 0) + { + std::ostringstream oss; + oss << " tag present in Job file for table " << curJob.jobTableList[kT].tblName + << "; this is not allowed for binary imports."; + rc = ERR_BULK_BINARY_IGNORE_FLD; + fLog.logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } } + } - // If binary import, do not allow tags in the Job file - if ((fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) || - (fImportDataMode == IMPORT_DATA_BIN_SAT_NULL)) - { - for (unsigned kT = 0; kT < curJob.jobTableList.size(); kT++) - { - if (curJob.jobTableList[kT].fIgnoredFields.size() > 0) - { - std::ostringstream oss; - oss << " tag present in Job file for table " << - curJob.jobTableList[kT].tblName << - "; this is not allowed for binary imports."; - rc = ERR_BULK_BINARY_IGNORE_FLD; - fLog.logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - } - } + stopTimer(); - stopTimer(); + std::ostringstream ossXMLTime; + ossXMLTime << "Job file loaded, run time for this step : " << getTotalRunTime() << " seconds"; + fLog.logMsg(ossXMLTime.str(), MSGLVL_INFO1); - std::ostringstream ossXMLTime; - ossXMLTime << "Job file loaded, run time for this step : " << - getTotalRunTime() << " seconds"; - fLog.logMsg( ossXMLTime.str(), MSGLVL_INFO1 ); - - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Spawns and joins the Read and Parsing threads to import the data. @@ -397,48 +384,46 @@ int BulkLoad::loadJobInfo( //------------------------------------------------------------------------------ void BulkLoad::spawnWorkers() { - // We're fixin' to launch threads. This lets anybody who cares (i.e. - // checkCancellation) know that read and parse threads are running. - fbContinue = true; + // We're fixin' to launch threads. This lets anybody who cares (i.e. + // checkCancellation) know that read and parse threads are running. + fbContinue = true; - // Spawn a thread to check for user cancellation via calpont console - // But only in mode 3 (local mode) - boost::thread cancelThread; - CancellationThread cancelationThread(this); + // Spawn a thread to check for user cancellation via calpont console + // But only in mode 3 (local mode) + boost::thread cancelThread; + CancellationThread cancelationThread(this); - if (getBulkLoadMode() == BULK_MODE_LOCAL) - { - cancelThread = boost::thread(cancelationThread); - } + if (getBulkLoadMode() == BULK_MODE_LOCAL) + { + cancelThread = boost::thread(cancelationThread); + } - // Spawn read threads - for (int i = 0; i < fNoOfReadThreads; ++i) - { - fReadThreads.create_thread(boost::bind(&BulkLoad::read, this, (int)i)); - } + // Spawn read threads + for (int i = 0; i < fNoOfReadThreads; ++i) + { + fReadThreads.create_thread(boost::bind(&BulkLoad::read, this, (int)i)); + } - fLog.logMsg("No of Read Threads Spawned = " + - Convertor::int2Str( fNoOfReadThreads ), MSGLVL_INFO1 ); + fLog.logMsg("No of Read Threads Spawned = " + Convertor::int2Str(fNoOfReadThreads), MSGLVL_INFO1); - // Spawn parse threads - for (int i = 0; i < fNoOfParseThreads; ++i) - { - fParseThreads.create_thread(boost::bind(&BulkLoad::parse, this, (int)i)); - } + // Spawn parse threads + for (int i = 0; i < fNoOfParseThreads; ++i) + { + fParseThreads.create_thread(boost::bind(&BulkLoad::parse, this, (int)i)); + } - fLog.logMsg("No of Parse Threads Spawned = " + - Convertor::int2Str( fNoOfParseThreads ), MSGLVL_INFO1 ); + fLog.logMsg("No of Parse Threads Spawned = " + Convertor::int2Str(fNoOfParseThreads), MSGLVL_INFO1); - fReadThreads.join_all(); - fParseThreads.join_all(); - fbContinue = false; + fReadThreads.join_all(); + fParseThreads.join_all(); + fbContinue = false; - if (getBulkLoadMode() == BULK_MODE_LOCAL) - { - cancelThread.join(); - } + if (getBulkLoadMode() == BULK_MODE_LOCAL) + { + cancelThread.join(); + } } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Pre process job. Determine DBRoot/segment file, HWM etc where we are @@ -461,365 +446,332 @@ void BulkLoad::spawnWorkers() // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::preProcess( Job& job, int tableNo, - TableInfo* tableInfo ) +int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) { - int rc = NO_ERROR, minWidth = 9999; // give a big number - HWM minHWM = 999999; // rp 9/25/07 Bug 473 - ColStruct curColStruct; - execplan::CalpontSystemCatalog::ColDataType colDataType; + int rc = NO_ERROR, minWidth = 9999; // give a big number + HWM minHWM = 999999; // rp 9/25/07 Bug 473 + ColStruct curColStruct; + execplan::CalpontSystemCatalog::ColDataType colDataType; - // Initialize portions of TableInfo object - tableInfo->setBufferSize(fBufferSize); - tableInfo->setFileBufferSize(fFileVbufSize); - tableInfo->setTableId(tableNo); - tableInfo->setColDelimiter(fColDelim); - tableInfo->setJobFileName(fJobFileName); - tableInfo->setJobId(job.id); - tableInfo->setNullStringMode(fNullStringMode); - tableInfo->setEnclosedByChar(fEnclosedByChar); - tableInfo->setEscapeChar(fEscapeChar); - tableInfo->setImportDataMode(fImportDataMode); - tableInfo->setTimeZone(fTimeZone); - tableInfo->setJobUUID(fUUID); + // Initialize portions of TableInfo object + tableInfo->setBufferSize(fBufferSize); + tableInfo->setFileBufferSize(fFileVbufSize); + tableInfo->setTableId(tableNo); + tableInfo->setColDelimiter(fColDelim); + tableInfo->setJobFileName(fJobFileName); + tableInfo->setJobId(job.id); + tableInfo->setNullStringMode(fNullStringMode); + tableInfo->setEnclosedByChar(fEnclosedByChar); + tableInfo->setEscapeChar(fEscapeChar); + tableInfo->setImportDataMode(fImportDataMode); + tableInfo->setTimeZone(fTimeZone); + tableInfo->setJobUUID(fUUID); - // MCOL-4328 Get username gid and uid if they are set - // We inject uid and gid into TableInfo and All ColumnInfo-s later. - struct passwd* pwd = nullptr; - errno = 0; - if (fUsername.length() && (pwd = getpwnam(fUsername.c_str())) == nullptr) + // MCOL-4328 Get username gid and uid if they are set + // We inject uid and gid into TableInfo and All ColumnInfo-s later. + struct passwd* pwd = nullptr; + errno = 0; + if (fUsername.length() && (pwd = getpwnam(fUsername.c_str())) == nullptr) + { + std::ostringstream oss; + oss << "Error getting pwd for " << fUsername << " with errno " << errno; + fLog.logMsg(oss.str(), MSGLVL_ERROR); + return ERR_FILE_CHOWN; + } + + if (pwd) + tableInfo->setUIDGID(pwd->pw_uid, pwd->pw_gid); + + if (fMaxErrors != -1) + tableInfo->setMaxErrorRows(fMaxErrors); + else + tableInfo->setMaxErrorRows(job.jobTableList[tableNo].maxErrNum); + + // @bug 3929: cpimport.bin error messaging using up too much memory. + // Validate that max allowed error count is within valid range + long long maxErrNum = tableInfo->getMaxErrorRows(); + + if (maxErrNum > MAX_ALLOW_ERROR_COUNT) + { + ostringstream oss; + oss << "Max allowed error count specified as " << maxErrNum << " for table " + << job.jobTableList[tableNo].tblName << "; this exceeds limit of " << MAX_ALLOW_ERROR_COUNT + << "; resetting to " << MAX_ALLOW_ERROR_COUNT; + fLog.logMsg(oss.str(), MSGLVL_INFO2); + maxErrNum = MAX_ALLOW_ERROR_COUNT; + } + + tableInfo->setMaxErrorRows(maxErrNum); + + //------------------------------------------------------------------------ + // First loop thru the columns for the "tableNo" table in jobTableList[]. + // Get the HWM information for each column. + //------------------------------------------------------------------------ + std::vector colWidths; + std::vector segFileInfo; + std::vector dbRootExtTrackerVec; + std::vector dbRootHWMInfoColVec(job.jobTableList[tableNo].colList.size()); + DBRootExtentTracker* pRefDBRootExtentTracker = 0; + bool bNoStartExtentOnThisPM = false; + bool bEmptyPM = false; + + for (size_t i = 0; i < job.jobTableList[tableNo].colList.size(); i++) + { + const JobColumn& curJobCol = job.jobTableList[tableNo].colList[i]; + + // convert column data type + if (curJobCol.typeName.length() > 0 && fColOp->getColDataType(curJobCol.typeName.c_str(), colDataType)) { - std::ostringstream oss; - oss << "Error getting pwd for " << fUsername - << " with errno " - << errno; - fLog.logMsg( oss.str(), MSGLVL_ERROR ); - return ERR_FILE_CHOWN; + job.jobTableList[tableNo].colList[i].dataType = curColStruct.colDataType = colDataType; } + else + { + ostringstream oss; + oss << "Column type " << curJobCol.typeName << " is not valid "; + fLog.logMsg(oss.str(), ERR_INVALID_PARAM, MSGLVL_ERROR); + return ERR_INVALID_PARAM; + } + + curColStruct.colWidth = curJobCol.width; + Convertor::convertColType(&curColStruct); + + job.jobTableList[tableNo].colList[i].weType = curColStruct.colType; + // set width to correct column width + job.jobTableList[tableNo].colList[i].width = curColStruct.colWidth; + job.jobTableList[tableNo].colList[i].emptyVal = getEmptyRowValue( + job.jobTableList[tableNo].colList[i].dataType, job.jobTableList[tableNo].colList[i].width); + + // check HWM for column file + rc = BRMWrapper::getInstance()->getDbRootHWMInfo(curJobCol.mapOid, dbRootHWMInfoColVec[i]); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "Error getting last DBRoot/HWMs for column file " << curJobCol.mapOid << "; " + << ec.errorString(rc); + fLog.logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + colWidths.push_back(job.jobTableList[tableNo].colList[i].width); + } // end of 1st for-loop through the list of columns (get starting HWM) + + //-------------------------------------------------------------------------- + // Second loop thru the columns for the "tableNo" table in jobTableList[]. + // Create DBRootExtentTracker, and select starting DBRoot. + // Determine the smallest width column(s), and save that as minHWM. + // We save additional HWM information acquired from BRM, in segFileInfo, + // for later use. + //-------------------------------------------------------------------------- + for (size_t i = 0; i < job.jobTableList[tableNo].colList.size(); i++) + { + const JobColumn& curJobCol = job.jobTableList[tableNo].colList[i]; + + // Find DBRoot/segment file where we want to start adding rows + DBRootExtentTracker* pDBRootExtentTracker = + new DBRootExtentTracker(curJobCol.mapOid, colWidths, dbRootHWMInfoColVec, i, &fLog); + + if (i == 0) + pRefDBRootExtentTracker = pDBRootExtentTracker; + + dbRootExtTrackerVec.push_back(pDBRootExtentTracker); + + // Start adding rows to DBRoot/segment file that is selected + DBRootExtentInfo dbRootExtent; + + if (i == 0) // select starting DBRoot/segment for column[0] + { + std::string trkErrMsg; + rc = + pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bNoStartExtentOnThisPM, bEmptyPM, trkErrMsg); + + if (rc != NO_ERROR) + { + fLog.logMsg(trkErrMsg, rc, MSGLVL_ERROR); + return rc; + } + } + else // select starting DBRoot/segment based on column[0] selection + { + // to ensure all columns start with the same DBRoot/segment + pDBRootExtentTracker->assignFirstSegFile(*pRefDBRootExtentTracker, // reference column[0] tracker + dbRootExtent); + } + + if (job.jobTableList[tableNo].colList[i].width < minWidth) + { + // save the minimum hwm -- rp 9/25/07 Bug 473 + minWidth = job.jobTableList[tableNo].colList[i].width; + minHWM = dbRootExtent.fLocalHwm; + } + + // Save column segment file info for use in subsequent loop + segFileInfo.push_back(dbRootExtent); + } + + //-------------------------------------------------------------------------- + // Validate that the starting HWMs for all the columns are in sync + //-------------------------------------------------------------------------- + rc = tableInfo->validateColumnHWMs(&job.jobTableList[tableNo], segFileInfo, "Starting"); + + if (rc != NO_ERROR) + { + return rc; + } + + //-------------------------------------------------------------------------- + // Create bulk rollback meta data file + //-------------------------------------------------------------------------- + ostringstream oss11; + oss11 << "Initializing import: " + << "Table-" << job.jobTableList[tableNo].tblName << "..."; + fLog.logMsg(oss11.str(), MSGLVL_INFO2); + + rc = saveBulkRollbackMetaData(job, tableInfo, segFileInfo, dbRootHWMInfoColVec); + + if (rc != NO_ERROR) + { + return rc; + } + + //-------------------------------------------------------------------------- + // Third loop thru the columns for the "tableNo" table in jobTableList[]. + // In this pass through the columns we create the ColumnInfo object, + // open the applicable column and dictionary store files, and seek to + // the block where we will begin adding data. + //-------------------------------------------------------------------------- + unsigned int fixedBinaryRecLen = 0; + + for (size_t i = 0; i < job.jobTableList[tableNo].colList.size(); i++) + { + uint16_t dbRoot = segFileInfo[i].fDbRoot; + uint32_t partition = segFileInfo[i].fPartition; + uint16_t segment = segFileInfo[i].fSegment; + HWM oldHwm = segFileInfo[i].fLocalHwm; + + DBRootExtentTracker* pDBRootExtentTracker = 0; + + if (dbRootExtTrackerVec.size() > 0) + pDBRootExtentTracker = dbRootExtTrackerVec[i]; + + // Create a ColumnInfo for the next column, and add to tableInfo + ColumnInfo* info = 0; + + if (job.jobTableList[tableNo].colList[i].compressionType) + info = new ColumnInfoCompressed(&fLog, i, job.jobTableList[tableNo].colList[i], pDBRootExtentTracker, + tableInfo); + // tableInfo->rbMetaWriter()); + else + info = new ColumnInfo(&fLog, i, job.jobTableList[tableNo].colList[i], pDBRootExtentTracker, tableInfo); if (pwd) - tableInfo->setUIDGID(pwd->pw_uid, pwd->pw_gid); + info->setUIDGID(pwd->pw_uid, pwd->pw_gid); - if (fMaxErrors != -1) - tableInfo->setMaxErrorRows(fMaxErrors); + // For auto increment column, we need to get the starting value + if (info->column.autoIncFlag) + { + rc = preProcessAutoInc(job.jobTableList[tableNo].tblName, info); + + if (rc != NO_ERROR) + { + return rc; + } + } + + // For binary input mode, sum up the columns widths to get fixed rec len + if ((fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) || (fImportDataMode == IMPORT_DATA_BIN_SAT_NULL)) + { + if (job.jobTableList[tableNo].fFldRefs[i].fFldColType == BULK_FLDCOL_COLUMN_FIELD) + { + fixedBinaryRecLen += info->column.definedWidth; + } + } + + // Skip minimum blocks before starting import; minwidth columns skip to + // next block. Wider columns skip based on multiple of width. If this + // skipping of blocks requires a new extent, then we extend the column. + HWM hwm = (minHWM + 1) * (info->column.width / minWidth); + info->relativeColWidthFactor(info->column.width / minWidth); + + if ((bEmptyPM) || (bNoStartExtentOnThisPM)) + { + // HWM not found in prev loop; can't get LBID. Will create initial + // extent on this PM later in this job, if we have valid rows to add + if (bEmptyPM) + { + // No starting DB file on this PM + ostringstream oss3; + oss3 << "Currently no extents on dbroot" << dbRoot << " for column OID " << info->column.mapOid + << "; will create starting extent"; + fLog.logMsg(oss3.str(), MSGLVL_INFO2); + } + // Skip to subsequent physical partition if current HWM extent + // for this "dbroot" is disabled. + else // bNoStartExtentOnThisPM is true + { + // Starting DB file on this PM is disabled + ostringstream oss3; + oss3 << "Current HWM extent is disabled on dbroot" << dbRoot << " for column OID " + << info->column.mapOid << "; will create starting extent"; + fLog.logMsg(oss3.str(), MSGLVL_INFO2); + } + + // Pass blocks to be skipped at start of file "if" we decide to + // employ block skipping for the first extent. + hwm = info->column.width / minWidth; + + // We don't have a starting DB file on this PM, or the starting HWM + // extent is disabled. In either case, we will wait and create a + // new DB file to receive any new rows, only after we make sure we + // have rows to insert. + info->setupDelayedFileCreation(dbRoot, partition, segment, hwm, bEmptyPM); + } else - tableInfo->setMaxErrorRows(job.jobTableList[tableNo].maxErrNum); - - // @bug 3929: cpimport.bin error messaging using up too much memory. - // Validate that max allowed error count is within valid range - long long maxErrNum = tableInfo->getMaxErrorRows(); - - if (maxErrNum > MAX_ALLOW_ERROR_COUNT) { - ostringstream oss; - oss << "Max allowed error count specified as " << maxErrNum << - " for table " << job.jobTableList[tableNo].tblName << - "; this exceeds limit of " << MAX_ALLOW_ERROR_COUNT << - "; resetting to " << MAX_ALLOW_ERROR_COUNT; - fLog.logMsg( oss.str(), MSGLVL_INFO2 ); - maxErrNum = MAX_ALLOW_ERROR_COUNT; + // Establish starting HWM and LBID for this job. + // Keep in mind we have initial block skipping to account for. + bool bSkippedToNewExtent = false; + BRM::LBID_t lbid; + + RETURN_ON_ERROR(preProcessHwmLbid(info, minWidth, partition, segment, hwm, lbid, bSkippedToNewExtent)); + + // Setup import to start loading into starting HWM DB file + RETURN_ON_ERROR(info->setupInitialColumnExtent(dbRoot, partition, segment, + job.jobTableList[tableNo].tblName, lbid, oldHwm, hwm, + bSkippedToNewExtent, false)); } - tableInfo->setMaxErrorRows(maxErrNum); + tableInfo->addColumn(info); - //------------------------------------------------------------------------ - // First loop thru the columns for the "tableNo" table in jobTableList[]. - // Get the HWM information for each column. - //------------------------------------------------------------------------ - std::vector colWidths; - std::vector segFileInfo; - std::vector dbRootExtTrackerVec; - std::vector dbRootHWMInfoColVec( - job.jobTableList[tableNo].colList.size() ); - DBRootExtentTracker* pRefDBRootExtentTracker = 0; - bool bNoStartExtentOnThisPM = false; - bool bEmptyPM = false; + } // end of 2nd for-loop through the list of columns - for ( size_t i = 0; i < job.jobTableList[tableNo].colList.size(); i++ ) - { - const JobColumn& curJobCol = job.jobTableList[tableNo].colList[i]; + if ((fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) || (fImportDataMode == IMPORT_DATA_BIN_SAT_NULL)) + { + ostringstream oss12; + oss12 << "Table " << job.jobTableList[tableNo].tblName + << " will be " + "imported in binary mode with fixed record length: " + << fixedBinaryRecLen << " bytes; "; - // convert column data type - if ( curJobCol.typeName.length() > 0 && - fColOp->getColDataType( curJobCol.typeName.c_str(), colDataType )) - { - job.jobTableList[tableNo].colList[i].dataType = - curColStruct.colDataType = colDataType; - } - else - { - ostringstream oss; - oss << "Column type " << curJobCol.typeName << " is not valid "; - fLog.logMsg( oss.str(), ERR_INVALID_PARAM, MSGLVL_ERROR ); - return ERR_INVALID_PARAM; - } + if (fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) + oss12 << "NULL values accepted"; + else + oss12 << "NULL values saturated"; - curColStruct.colWidth = curJobCol.width; - Convertor::convertColType( &curColStruct ); + fLog.logMsg(oss12.str(), MSGLVL_INFO2); + } - job.jobTableList[tableNo].colList[i].weType = curColStruct.colType; - // set width to correct column width - job.jobTableList[tableNo].colList[i].width = curColStruct.colWidth; - job.jobTableList[tableNo].colList[i].emptyVal = - getEmptyRowValue(job.jobTableList[tableNo].colList[i].dataType, - job.jobTableList[tableNo].colList[i].width); + // Initialize BulkLoadBuffers after we have added all the columns + rc = tableInfo->initializeBuffers(fNoOfBuffers, job.jobTableList[tableNo].fFldRefs, fixedBinaryRecLen); + if (rc) + return rc; - // check HWM for column file - rc = BRMWrapper::getInstance()->getDbRootHWMInfo( curJobCol.mapOid, - dbRootHWMInfoColVec[i]); + fTableInfo.push_back(tableInfo); - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "Error getting last DBRoot/HWMs for column file " << - curJobCol.mapOid << "; " << ec.errorString(rc); - fLog.logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - colWidths.push_back( job.jobTableList[tableNo].colList[i].width ); - } // end of 1st for-loop through the list of columns (get starting HWM) - - //-------------------------------------------------------------------------- - // Second loop thru the columns for the "tableNo" table in jobTableList[]. - // Create DBRootExtentTracker, and select starting DBRoot. - // Determine the smallest width column(s), and save that as minHWM. - // We save additional HWM information acquired from BRM, in segFileInfo, - // for later use. - //-------------------------------------------------------------------------- - for ( size_t i = 0; i < job.jobTableList[tableNo].colList.size(); i++ ) - { - const JobColumn& curJobCol = job.jobTableList[tableNo].colList[i]; - - // Find DBRoot/segment file where we want to start adding rows - DBRootExtentTracker* pDBRootExtentTracker = new DBRootExtentTracker( - curJobCol.mapOid, - colWidths, - dbRootHWMInfoColVec, - i, - &fLog ); - - if (i == 0) - pRefDBRootExtentTracker = pDBRootExtentTracker; - - dbRootExtTrackerVec.push_back( pDBRootExtentTracker ); - - // Start adding rows to DBRoot/segment file that is selected - DBRootExtentInfo dbRootExtent; - - if (i == 0) // select starting DBRoot/segment for column[0] - { - std::string trkErrMsg; - rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, - bNoStartExtentOnThisPM, bEmptyPM, trkErrMsg); - - if (rc != NO_ERROR) - { - fLog.logMsg( trkErrMsg, rc, MSGLVL_ERROR ); - return rc; - } - } - else // select starting DBRoot/segment based on column[0] selection - { - // to ensure all columns start with the same DBRoot/segment - pDBRootExtentTracker->assignFirstSegFile( - *pRefDBRootExtentTracker, // reference column[0] tracker - dbRootExtent); - } - - if ( job.jobTableList[tableNo].colList[i].width < minWidth ) - { - // save the minimum hwm -- rp 9/25/07 Bug 473 - minWidth = job.jobTableList[tableNo].colList[i].width; - minHWM = dbRootExtent.fLocalHwm; - } - - // Save column segment file info for use in subsequent loop - segFileInfo.push_back( dbRootExtent ); - } - - //-------------------------------------------------------------------------- - // Validate that the starting HWMs for all the columns are in sync - //-------------------------------------------------------------------------- - rc = tableInfo->validateColumnHWMs( &job.jobTableList[tableNo], - segFileInfo, "Starting" ); - - if (rc != NO_ERROR) - { - return rc; - } - - //-------------------------------------------------------------------------- - // Create bulk rollback meta data file - //-------------------------------------------------------------------------- - ostringstream oss11; - oss11 << "Initializing import: " << - "Table-" << job.jobTableList[tableNo].tblName << "..."; - fLog.logMsg( oss11.str(), MSGLVL_INFO2 ); - - rc = saveBulkRollbackMetaData( job, tableInfo, segFileInfo, - dbRootHWMInfoColVec ); - - if (rc != NO_ERROR) - { - return rc; - } - - //-------------------------------------------------------------------------- - // Third loop thru the columns for the "tableNo" table in jobTableList[]. - // In this pass through the columns we create the ColumnInfo object, - // open the applicable column and dictionary store files, and seek to - // the block where we will begin adding data. - //-------------------------------------------------------------------------- - unsigned int fixedBinaryRecLen = 0; - - for ( size_t i = 0; i < job.jobTableList[tableNo].colList.size(); i++ ) - { - uint16_t dbRoot = segFileInfo[i].fDbRoot; - uint32_t partition = segFileInfo[i].fPartition; - uint16_t segment = segFileInfo[i].fSegment; - HWM oldHwm = segFileInfo[i].fLocalHwm; - - DBRootExtentTracker* pDBRootExtentTracker = 0; - - if (dbRootExtTrackerVec.size() > 0) - pDBRootExtentTracker = dbRootExtTrackerVec[i]; - - // Create a ColumnInfo for the next column, and add to tableInfo - ColumnInfo* info = 0; - - if (job.jobTableList[tableNo].colList[i].compressionType) - info = new ColumnInfoCompressed(&fLog, i, - job.jobTableList[tableNo].colList[i], - pDBRootExtentTracker, - tableInfo); - //tableInfo->rbMetaWriter()); - else - info = new ColumnInfo(&fLog, i, - job.jobTableList[tableNo].colList[i], - pDBRootExtentTracker, - tableInfo); - - if (pwd) - info->setUIDGID(pwd->pw_uid, pwd->pw_gid); - - // For auto increment column, we need to get the starting value - if (info->column.autoIncFlag) - { - rc = preProcessAutoInc( job.jobTableList[tableNo].tblName, info ); - - if (rc != NO_ERROR) - { - return rc; - } - } - - // For binary input mode, sum up the columns widths to get fixed rec len - if ((fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) || - (fImportDataMode == IMPORT_DATA_BIN_SAT_NULL)) - { - if (job.jobTableList[tableNo].fFldRefs[i].fFldColType == - BULK_FLDCOL_COLUMN_FIELD) - { - fixedBinaryRecLen += info->column.definedWidth; - } - } - - // Skip minimum blocks before starting import; minwidth columns skip to - // next block. Wider columns skip based on multiple of width. If this - // skipping of blocks requires a new extent, then we extend the column. - HWM hwm = (minHWM + 1) * ( info->column.width / minWidth ); - info->relativeColWidthFactor( info->column.width / minWidth ); - - if ((bEmptyPM) || (bNoStartExtentOnThisPM)) - { - // HWM not found in prev loop; can't get LBID. Will create initial - // extent on this PM later in this job, if we have valid rows to add - if (bEmptyPM) - { - // No starting DB file on this PM - ostringstream oss3; - oss3 << "Currently no extents on dbroot" << dbRoot << - " for column OID " << info->column.mapOid << - "; will create starting extent"; - fLog.logMsg( oss3.str(), MSGLVL_INFO2 ); - } - // Skip to subsequent physical partition if current HWM extent - // for this "dbroot" is disabled. - else // bNoStartExtentOnThisPM is true - { - // Starting DB file on this PM is disabled - ostringstream oss3; - oss3 << "Current HWM extent is disabled on dbroot" << dbRoot << - " for column OID " << info->column.mapOid << - "; will create starting extent"; - fLog.logMsg( oss3.str(), MSGLVL_INFO2 ); - } - - // Pass blocks to be skipped at start of file "if" we decide to - // employ block skipping for the first extent. - hwm = info->column.width / minWidth; - - // We don't have a starting DB file on this PM, or the starting HWM - // extent is disabled. In either case, we will wait and create a - // new DB file to receive any new rows, only after we make sure we - // have rows to insert. - info->setupDelayedFileCreation( - dbRoot, partition, segment, hwm, bEmptyPM ); - } - else - { - // Establish starting HWM and LBID for this job. - // Keep in mind we have initial block skipping to account for. - bool bSkippedToNewExtent = false; - BRM::LBID_t lbid; - - RETURN_ON_ERROR( preProcessHwmLbid( info, - minWidth, partition, segment, - hwm, lbid, bSkippedToNewExtent ) ); - - // Setup import to start loading into starting HWM DB file - RETURN_ON_ERROR( info->setupInitialColumnExtent( - dbRoot, partition, segment, - job.jobTableList[tableNo].tblName, - lbid, - oldHwm, hwm, - bSkippedToNewExtent, false) ); - } - - tableInfo->addColumn(info); - - } // end of 2nd for-loop through the list of columns - - if ((fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) || - (fImportDataMode == IMPORT_DATA_BIN_SAT_NULL)) - { - ostringstream oss12; - oss12 << "Table " << job.jobTableList[tableNo].tblName << " will be " - "imported in binary mode with fixed record length: " << - fixedBinaryRecLen << " bytes; "; - - if (fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL) - oss12 << "NULL values accepted"; - else - oss12 << "NULL values saturated"; - - fLog.logMsg( oss12.str(), MSGLVL_INFO2 ); - } - - // Initialize BulkLoadBuffers after we have added all the columns - rc = tableInfo->initializeBuffers(fNoOfBuffers, - job.jobTableList[tableNo].fFldRefs, - fixedBinaryRecLen); - if (rc) - return rc; - - fTableInfo.push_back(tableInfo); - - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Saves snapshot of extentmap into a bulk rollback meta data file, for @@ -835,13 +787,11 @@ int BulkLoad::preProcess( Job& job, int tableNo, // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::saveBulkRollbackMetaData( Job& job, - TableInfo* tableInfo, - const std::vector& segFileInfo, - const std::vector& dbRootHWMInfoColVec) +int BulkLoad::saveBulkRollbackMetaData(Job& job, TableInfo* tableInfo, + const std::vector& segFileInfo, + const std::vector& dbRootHWMInfoColVec) { - return tableInfo->saveBulkRollbackMetaData( - job, segFileInfo, dbRootHWMInfoColVec ); + return tableInfo->saveBulkRollbackMetaData(job, segFileInfo, dbRootHWMInfoColVec); } //------------------------------------------------------------------------------ @@ -854,14 +804,13 @@ int BulkLoad::saveBulkRollbackMetaData( Job& job, // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::preProcessAutoInc( const std::string& fullTableName, - ColumnInfo* colInfo) +int BulkLoad::preProcessAutoInc(const std::string& fullTableName, ColumnInfo* colInfo) { - int rc = colInfo->initAutoInc( fullTableName ); + int rc = colInfo->initAutoInc(fullTableName); - return rc; + return rc; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Determine starting HWM and LBID, after applying block skipping to HWM. @@ -880,65 +829,48 @@ int BulkLoad::preProcessAutoInc( const std::string& fullTableName, // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::preProcessHwmLbid( - const ColumnInfo* info, - int minWidth, - uint32_t partition, - uint16_t segment, - HWM& hwm, // input/output - BRM::LBID_t& lbid, // output - bool& bSkippedToNewExtent) // output +int BulkLoad::preProcessHwmLbid(const ColumnInfo* info, int minWidth, uint32_t partition, uint16_t segment, + HWM& hwm, // input/output + BRM::LBID_t& lbid, // output + bool& bSkippedToNewExtent) // output { - int rc = NO_ERROR; - bSkippedToNewExtent = false; + int rc = NO_ERROR; + bSkippedToNewExtent = false; - // Get starting LBID for the HWM block; if we can't get the start- - // ing LBID, it means initial block skipping crossed extent boundary - rc = BRMWrapper::getInstance()->getStartLbid( - info->column.mapOid, - partition, - segment, - (int)hwm, lbid); + // Get starting LBID for the HWM block; if we can't get the start- + // ing LBID, it means initial block skipping crossed extent boundary + rc = BRMWrapper::getInstance()->getStartLbid(info->column.mapOid, partition, segment, (int)hwm, lbid); + + // If HWM Lbid is missing, take alternative action to see what to do. + // Block skipping has caused us to advance out of the current HWM extent. + if (rc != NO_ERROR) + { + bSkippedToNewExtent = true; + + lbid = INVALID_LBID; + + int blocksPerExtent = (BRMWrapper::getInstance()->getExtentRows() * info->column.width) / BYTE_PER_BLOCK; + + // Look for LBID associated with block at end of current extent + uint32_t numBlocks = (((hwm + 1) / blocksPerExtent) * blocksPerExtent); + + hwm = numBlocks - 1; + rc = BRMWrapper::getInstance()->getStartLbid(info->column.mapOid, partition, segment, (int)hwm, lbid); - // If HWM Lbid is missing, take alternative action to see what to do. - // Block skipping has caused us to advance out of the current HWM extent. if (rc != NO_ERROR) { - bSkippedToNewExtent = true; - - lbid = INVALID_LBID; - - int blocksPerExtent = - (BRMWrapper::getInstance()->getExtentRows() * - info->column.width) / BYTE_PER_BLOCK; - - // Look for LBID associated with block at end of current extent - uint32_t numBlocks = (((hwm + 1) / blocksPerExtent) * blocksPerExtent); - - hwm = numBlocks - 1; - rc = BRMWrapper::getInstance()->getStartLbid( - info->column.mapOid, - partition, - segment, - (int)hwm, lbid); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "Error getting HWM start LBID " - "for previous last extent in column file OID-" << - info->column.mapOid << - "; partition-" << partition << - "; segment-" << segment << - "; hwm-" << hwm << - "; " << ec.errorString(rc); - fLog.logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } + WErrorCodes ec; + ostringstream oss; + oss << "Error getting HWM start LBID " + "for previous last extent in column file OID-" + << info->column.mapOid << "; partition-" << partition << "; segment-" << segment << "; hwm-" << hwm + << "; " << ec.errorString(rc); + fLog.logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -946,262 +878,248 @@ int BulkLoad::preProcessHwmLbid( // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::processJob( ) +int BulkLoad::processJob() { #ifdef PROFILE - Stats::enableProfiling( fNoOfReadThreads, fNoOfParseThreads ); + Stats::enableProfiling(fNoOfReadThreads, fNoOfParseThreads); #endif - int rc = NO_ERROR; - Job curJob; - size_t i; + int rc = NO_ERROR; + Job curJob; + size_t i; - curJob = fJobInfo.getJob(); + curJob = fJobInfo.getJob(); - // For the following parms, we use the value read from the Job XML file if - // a cmd line override value was not already assigned by cpimport.cpp. - if (fNoOfBuffers == -1) - fNoOfBuffers = curJob.numberOfReadBuffers; + // For the following parms, we use the value read from the Job XML file if + // a cmd line override value was not already assigned by cpimport.cpp. + if (fNoOfBuffers == -1) + fNoOfBuffers = curJob.numberOfReadBuffers; - if (fBufferSize == -1) - fBufferSize = curJob.readBufferSize; + if (fBufferSize == -1) + fBufferSize = curJob.readBufferSize; - if (fFileVbufSize == -1) - fFileVbufSize = curJob.writeBufferSize; + if (fFileVbufSize == -1) + fFileVbufSize = curJob.writeBufferSize; - if (fColDelim == '\0') - fColDelim = curJob.fDelimiter; + if (fColDelim == '\0') + fColDelim = curJob.fDelimiter; - //std::cout << "bulkload::fEnclosedByChar<" << fEnclosedByChar << '>' << - //std::endl << "bulkload::fEscapeChar<" << fEscapeChar << '>' << - //std::endl << "job.fEnclosedByChar<" <' << - //std::endl << "job.fEscapeChar<" << curJob.fEscapeChar << '>' << - //std::endl; - if (fEnclosedByChar == '\0') + // std::cout << "bulkload::fEnclosedByChar<" << fEnclosedByChar << '>' << + // std::endl << "bulkload::fEscapeChar<" << fEscapeChar << '>' << + // std::endl << "job.fEnclosedByChar<" <' << + // std::endl << "job.fEscapeChar<" << curJob.fEscapeChar << '>' << + // std::endl; + if (fEnclosedByChar == '\0') + { + // std::cout << "Using enclosed char from xml file" << std::endl; + fEnclosedByChar = curJob.fEnclosedByChar; + } + + if (fEscapeChar == '\0') + { + // std::cout << "Using escape char from xml file" << std::endl; + fEscapeChar = curJob.fEscapeChar; + } + + // If EnclosedBy char is given, then we need an escape character. + // We default to '\' if we didn't get one from xml file or cmd line. + if (fEscapeChar == '\0') + { + // std::cout << "Using default escape char" << std::endl; + fEscapeChar = '\\'; + } + + // std::cout << "bulkload::fEnclosedByChar<" << fEnclosedByChar << '>' << + // std::endl << "bulkload::fEscapeChar<" << fEscapeChar << '>' << std::endl; + + // Bug1315 - check whether DBRoots are RW mounted. + std::vector dbRootPathList; + Config::getDBRootPathList(dbRootPathList); + + for (unsigned int counter = 0; counter < dbRootPathList.size(); counter++) + { + if (access(dbRootPathList[counter].c_str(), R_OK | W_OK) < 0) { - // std::cout << "Using enclosed char from xml file" << std::endl; - fEnclosedByChar = curJob.fEnclosedByChar; + rc = ERR_FILE_NOT_EXIST; + ostringstream oss; + oss << "Error accessing DBRoot[" << counter << "] " << dbRootPathList[counter] << "; " + << strerror(errno); + fLog.logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + } + + // Init total cumulative run time with time it took to load xml file + double totalRunTime = getTotalRunTime(); + fLog.logMsg("PreProcessing check starts", MSGLVL_INFO1); + startTimer(); + + //-------------------------------------------------------------------------- + // Validate that only 1 table is specified for import if using STDIN + //-------------------------------------------------------------------------- + if ((fAlternateImportDir == IMPORT_PATH_STDIN) && (curJob.jobTableList.size() > 1)) + { + rc = ERR_INVALID_PARAM; + fLog.logMsg("Only 1 table can be imported per job when using STDIN", rc, MSGLVL_ERROR); + return rc; + } + + //-------------------------------------------------------------------------- + // Validate the existence of the import data files + //-------------------------------------------------------------------------- + std::vector tables; + + for (i = 0; i < curJob.jobTableList.size(); i++) + { + TableInfo* tableInfo = new TableInfo(&fLog, fTxnID, fProcessName, curJob.jobTableList[i].mapOid, + curJob.jobTableList[i].tblName, fKeepRbMetaFiles); + + if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + tableInfo->setBulkLoadMode(fBulkMode, fBRMRptFileName); + + tableInfo->setErrorDir(string(getErrorDir())); + tableInfo->setTruncationAsError(getTruncationAsError()); + rc = manageImportDataFileList(curJob, i, tableInfo); + + if (rc != NO_ERROR) + { + tableInfo->fBRMReporter.sendErrMsgToFile(tableInfo->fBRMRptFileName); + return rc; } - if (fEscapeChar == '\0') + tables.push_back(tableInfo); + } + + //-------------------------------------------------------------------------- + // Before we go any further, we lock all the tables + //-------------------------------------------------------------------------- + for (i = 0; i < curJob.jobTableList.size(); i++) + { + rc = tables[i]->acquireTableLock(fDisableTimeOut); + + if (rc != NO_ERROR) { - // std::cout << "Using escape char from xml file" << std::endl; - fEscapeChar = curJob.fEscapeChar; + // Try releasing the table locks we already acquired. + // Note that loop is kreleaseTableLock(); // ignore return code in this case + } + + return rc; } - // If EnclosedBy char is given, then we need an escape character. - // We default to '\' if we didn't get one from xml file or cmd line. - if (fEscapeChar == '\0') + // If we have a lock, then init MetaWriter, so that it can delete any + // leftover backup meta data files that collide with the ones we are + // going to create. + rc = tables[i]->initBulkRollbackMetaData(); + + if (rc != NO_ERROR) { - //std::cout << "Using default escape char" << std::endl; - fEscapeChar = '\\'; + // Try releasing the table locks we already acquired. + // Note that loop is kreleaseTableLock(); // ignore return code in this case + } + + return rc; } + } - //std::cout << "bulkload::fEnclosedByChar<" << fEnclosedByChar << '>' << - //std::endl << "bulkload::fEscapeChar<" << fEscapeChar << '>' << std::endl; + //-------------------------------------------------------------------------- + // Perform necessary preprocessing for each table + //-------------------------------------------------------------------------- + for (i = 0; i < curJob.jobTableList.size(); i++) + { + // If table already marked as complete then we are skipping the + // table because there were no input files to process. + if (tables[i]->getStatusTI() == WriteEngine::PARSE_COMPLETE) + continue; - //Bug1315 - check whether DBRoots are RW mounted. - std::vector dbRootPathList; - Config::getDBRootPathList( dbRootPathList ); + rc = preProcess(curJob, i, tables[i]); - for (unsigned int counter = 0; counter < dbRootPathList.size(); counter++) + if (rc != NO_ERROR) { - if ( access( dbRootPathList[counter].c_str(), R_OK | W_OK) < 0 ) - { - rc = ERR_FILE_NOT_EXIST; - ostringstream oss; - oss << "Error accessing DBRoot[" << counter << "] " << - dbRootPathList[counter] << "; " << - strerror(errno); - fLog.logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - } + std::string errMsg = "Error in pre-processing the job file for table " + curJob.jobTableList[i].tblName; + tables[i]->fBRMReporter.addToErrMsgEntry(errMsg); + fLog.logMsg(errMsg, rc, MSGLVL_CRITICAL); - // Init total cumulative run time with time it took to load xml file - double totalRunTime = getTotalRunTime(); - fLog.logMsg( "PreProcessing check starts", MSGLVL_INFO1 ); - startTimer(); + // Okay to release the locks for the tables we did not get to + for (unsigned k = i + 1; k < tables.size(); k++) + { + tables[k]->releaseTableLock(); // ignore return code in this case + } - //-------------------------------------------------------------------------- - // Validate that only 1 table is specified for import if using STDIN - //-------------------------------------------------------------------------- - if ((fAlternateImportDir == IMPORT_PATH_STDIN) && - (curJob.jobTableList.size() > 1)) - { - rc = ERR_INVALID_PARAM; - fLog.logMsg("Only 1 table can be imported per job when using STDIN", - rc, MSGLVL_ERROR ); - return rc; - } + // Okay to release the locks for any tables we preprocessed. + // We will not have done anything to change these tables yet, + // so all we need to do is release the locks. + for (unsigned k = 0; k <= i; k++) + { + tables[k]->deleteMetaDataRollbackFile(); + tables[k]->releaseTableLock(); // ignore return code + } - //-------------------------------------------------------------------------- - // Validate the existence of the import data files - //-------------------------------------------------------------------------- - std::vector tables; + // Ignore the return code for now; more important to base rc on the + // success or failure of the previous work - for ( i = 0; i < curJob.jobTableList.size(); i++ ) - { - TableInfo* tableInfo = new TableInfo(&fLog, - fTxnID, - fProcessName, - curJob.jobTableList[i].mapOid, - curJob.jobTableList[i].tblName, - fKeepRbMetaFiles); - - if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) - tableInfo->setBulkLoadMode( fBulkMode, fBRMRptFileName ); - - tableInfo->setErrorDir(string(getErrorDir())); - tableInfo->setTruncationAsError(getTruncationAsError()); - rc = manageImportDataFileList( curJob, i, tableInfo ); - - if (rc != NO_ERROR) - { - tableInfo->fBRMReporter.sendErrMsgToFile(tableInfo->fBRMRptFileName); - return rc; - } - - tables.push_back( tableInfo ); - } - - //-------------------------------------------------------------------------- - // Before we go any further, we lock all the tables - //-------------------------------------------------------------------------- - for ( i = 0; i < curJob.jobTableList.size(); i++ ) - { - rc = tables[i]->acquireTableLock( fDisableTimeOut ); - - if (rc != NO_ERROR) - { - // Try releasing the table locks we already acquired. - // Note that loop is kreleaseTableLock( );//ignore return code in this case - } - - return rc; - } - - // If we have a lock, then init MetaWriter, so that it can delete any - // leftover backup meta data files that collide with the ones we are - // going to create. - rc = tables[i]->initBulkRollbackMetaData( ); - - if (rc != NO_ERROR) - { - // Try releasing the table locks we already acquired. - // Note that loop is kreleaseTableLock( );//ignore return code in this case - } - - return rc; - } - } - - //-------------------------------------------------------------------------- - // Perform necessary preprocessing for each table - //-------------------------------------------------------------------------- - for ( i = 0; i < curJob.jobTableList.size(); i++ ) - { - // If table already marked as complete then we are skipping the - // table because there were no input files to process. - if ( tables[i]->getStatusTI() == WriteEngine::PARSE_COMPLETE) - continue; - - rc = preProcess( curJob, i, tables[i] ); - - if ( rc != NO_ERROR ) - { - std::string errMsg = - "Error in pre-processing the job file for table " + - curJob.jobTableList[i].tblName; - tables[i]->fBRMReporter.addToErrMsgEntry(errMsg); - fLog.logMsg( errMsg, rc, MSGLVL_CRITICAL ); - - // Okay to release the locks for the tables we did not get to - for ( unsigned k = i + 1; k < tables.size(); k++) - { - tables[k]->releaseTableLock( );//ignore return code in this case - } - - // Okay to release the locks for any tables we preprocessed. - // We will not have done anything to change these tables yet, - // so all we need to do is release the locks. - for ( unsigned k = 0; k <= i; k++) - { - tables[k]->deleteMetaDataRollbackFile( ); - tables[k]->releaseTableLock( ); //ignore return code - } - - // Ignore the return code for now; more important to base rc on the - // success or failure of the previous work - - // BUG 4398: distributed cpimport calls takeSnapshot for modes 1 & 2 - if ((fBulkMode != BULK_MODE_REMOTE_SINGLE_SRC) && - (fBulkMode != BULK_MODE_REMOTE_MULTIPLE_SRC)) - { - BRMWrapper::getInstance()->takeSnapshot(); - } - - return rc; - } - } - - stopTimer(); - fLog.logMsg( "PreProcessing check completed", MSGLVL_INFO1 ); - - std::ostringstream ossPrepTime; - ossPrepTime << "preProcess completed, run time for this step : " << - getTotalRunTime() << " seconds"; - fLog.logMsg( ossPrepTime.str(), MSGLVL_INFO1 ); - totalRunTime += getTotalRunTime(); - - startTimer(); - - spawnWorkers(); - - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - rc = ERR_UNKNOWN; - } - - // Regardless of JobStatus, we rollback any tables that are left locked - int rollback_rc = rollbackLockedTables( ); - - if ((rc == NO_ERROR) && (rollback_rc != NO_ERROR)) - { - rc = rollback_rc; - } - - // Ignore the return code for now; more important to base rc on the - // success or failure of the previous work - - // BUG 4398: distributed cpimport now calls takeSnapshot for modes 1 & 2 - if ((fBulkMode != BULK_MODE_REMOTE_SINGLE_SRC) && - (fBulkMode != BULK_MODE_REMOTE_MULTIPLE_SRC)) - { + // BUG 4398: distributed cpimport calls takeSnapshot for modes 1 & 2 + if ((fBulkMode != BULK_MODE_REMOTE_SINGLE_SRC) && (fBulkMode != BULK_MODE_REMOTE_MULTIPLE_SRC)) + { BRMWrapper::getInstance()->takeSnapshot(); + } + + return rc; } + } - stopTimer(); - totalRunTime += getTotalRunTime(); + stopTimer(); + fLog.logMsg("PreProcessing check completed", MSGLVL_INFO1); - std::ostringstream ossTotalRunTime; - ossTotalRunTime << "Bulk load completed, total run time : " << - totalRunTime << " seconds" << std::endl; - fLog.logMsg( ossTotalRunTime.str(), MSGLVL_INFO1 ); + std::ostringstream ossPrepTime; + ossPrepTime << "preProcess completed, run time for this step : " << getTotalRunTime() << " seconds"; + fLog.logMsg(ossPrepTime.str(), MSGLVL_INFO1); + totalRunTime += getTotalRunTime(); + + startTimer(); + + spawnWorkers(); + + if (BulkStatus::getJobStatus() == EXIT_FAILURE) + { + rc = ERR_UNKNOWN; + } + + // Regardless of JobStatus, we rollback any tables that are left locked + int rollback_rc = rollbackLockedTables(); + + if ((rc == NO_ERROR) && (rollback_rc != NO_ERROR)) + { + rc = rollback_rc; + } + + // Ignore the return code for now; more important to base rc on the + // success or failure of the previous work + + // BUG 4398: distributed cpimport now calls takeSnapshot for modes 1 & 2 + if ((fBulkMode != BULK_MODE_REMOTE_SINGLE_SRC) && (fBulkMode != BULK_MODE_REMOTE_MULTIPLE_SRC)) + { + BRMWrapper::getInstance()->takeSnapshot(); + } + + stopTimer(); + totalRunTime += getTotalRunTime(); + + std::ostringstream ossTotalRunTime; + ossTotalRunTime << "Bulk load completed, total run time : " << totalRunTime << " seconds" << std::endl; + fLog.logMsg(ossTotalRunTime.str(), MSGLVL_INFO1); #ifdef PROFILE - Stats::printProfilingResults( ); + Stats::printProfilingResults(); #endif - return rc; + return rc; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Deconstruct the list of 1 or more import files for the specified table, @@ -1214,147 +1132,143 @@ int BulkLoad::processJob( ) // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::manageImportDataFileList(Job& job, - int tableNo, - TableInfo* tableInfo) +int BulkLoad::manageImportDataFileList(Job& job, int tableNo, TableInfo* tableInfo) { - std::vector loadFilesList; - bool bUseStdin = false; + std::vector loadFilesList; + bool bUseStdin = false; - // Take loadFileName from command line argument override "if" one exists, - // else we take from the Job xml file - std::string loadFileName; + // Take loadFileName from command line argument override "if" one exists, + // else we take from the Job xml file + std::string loadFileName; - if (fCmdLineImportFiles.size() > (unsigned)tableNo) - loadFileName = fCmdLineImportFiles[tableNo]; - else - loadFileName = job.jobTableList[tableNo].loadFileName; + if (fCmdLineImportFiles.size() > (unsigned)tableNo) + loadFileName = fCmdLineImportFiles[tableNo]; + else + loadFileName = job.jobTableList[tableNo].loadFileName; - if (fAlternateImportDir == IMPORT_PATH_STDIN) + if (fAlternateImportDir == IMPORT_PATH_STDIN) + { + bUseStdin = true; + fLog.logMsg("Using STDIN for input data", MSGLVL_INFO2); + + int rc = buildImportDataFileList(std::string(), loadFileName, loadFilesList); + + if (rc != NO_ERROR) { - bUseStdin = true; - fLog.logMsg( "Using STDIN for input data", MSGLVL_INFO2 ); + return rc; + } - int rc = buildImportDataFileList(std::string(), - loadFileName, - loadFilesList); + // BUG 4737 - in Mode 1, all data coming from STDIN, ignore input files + if ((loadFilesList.size() > 1) && (fBulkMode != BULK_MODE_REMOTE_SINGLE_SRC)) + { + ostringstream oss; + oss << "Table " << tableInfo->getTableName() + << " specifies multiple " + "load files; This is not allowed when using STDIN"; + fLog.logMsg(oss.str(), ERR_INVALID_PARAM, MSGLVL_ERROR); + tableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + return ERR_INVALID_PARAM; + } + } + else + { + std::string importDir; - if (rc != NO_ERROR) - { - return rc; - } - - // BUG 4737 - in Mode 1, all data coming from STDIN, ignore input files - if ((loadFilesList.size() > 1) && (fBulkMode != BULK_MODE_REMOTE_SINGLE_SRC)) - { - ostringstream oss; - oss << "Table " << tableInfo->getTableName() << - " specifies multiple " - "load files; This is not allowed when using STDIN"; - fLog.logMsg( oss.str(), ERR_INVALID_PARAM, MSGLVL_ERROR ); - tableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return ERR_INVALID_PARAM; - } + if (!fS3Key.empty()) + { + loadFilesList.push_back(loadFileName); } else { - std::string importDir; + if (fAlternateImportDir == IMPORT_PATH_CWD) // current working dir + { + char cwdBuf[4096]; + importDir = ::getcwd(cwdBuf, sizeof(cwdBuf)); + importDir += '/'; + } + else if (fAlternateImportDir.size() > 0) // -f path + { + importDir = fAlternateImportDir; + } + else // /data/import + { + importDir = fRootDir; + importDir += DIR_BULK_IMPORT; + } - if (!fS3Key.empty()) + // Break down loadFileName into vector of file names in case load- + // FileName contains a list of files or 1 or more wildcards. + int rc = buildImportDataFileList(importDir, loadFileName, loadFilesList); + + if (rc != NO_ERROR) + { + return rc; + } + } + + // No filenames is considered a fatal error, except for remote mode2. + // For remote mode2 we just mark the table as complete since we will + // have no data to load, but we don't consider this as an error. + if (loadFilesList.size() == 0) + { + ostringstream oss; + oss << "No import files found. " + << "default dir: " << importDir << " importFileName: " << loadFileName; + + if (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC) + { + tableInfo->setLoadFilesInput(bUseStdin, (!fS3Key.empty()), loadFilesList, fS3Host, fS3Key, fS3Secret, + fS3Bucket, fS3Region); + tableInfo->markTableComplete(); + fLog.logMsg(oss.str(), MSGLVL_INFO1); + return NO_ERROR; + } + else + { + fLog.logMsg(oss.str(), ERR_FILE_NOT_EXIST, MSGLVL_ERROR); + return ERR_FILE_NOT_EXIST; + } + } + + // Verify that input data files exist. + // We also used to check to make sure the input file is not empty, and + // if it were, we threw an error at this point, but we removed that + // check. With shared-nothing, an empty file is now acceptable. + if (fS3Key.empty()) + { + for (unsigned ndx = 0; ndx < loadFilesList.size(); ndx++) + { + // in addition to being more portable due to the use of boost, this change + // actually fixes an inherent bug with cpimport reading from a named pipe. + // Only the first open call gets any data passed through the pipe so the + // here that used to do an open to test for existence meant cpimport would + // never get data from the pipe. + boost::filesystem::path pathFile(loadFilesList[ndx]); + + if (!boost::filesystem::exists(pathFile)) { - loadFilesList.push_back(loadFileName); + ostringstream oss; + oss << "input data file " << loadFilesList[ndx] << " does not exist"; + fLog.logMsg(oss.str(), ERR_FILE_NOT_EXIST, MSGLVL_ERROR); + tableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + return ERR_FILE_NOT_EXIST; } else { - if ( fAlternateImportDir == IMPORT_PATH_CWD ) // current working dir - { - char cwdBuf[4096]; - importDir = ::getcwd(cwdBuf, sizeof(cwdBuf)); - importDir += '/'; - } - else if ( fAlternateImportDir.size() > 0 ) // -f path - { - importDir = fAlternateImportDir; - } - else // /data/import - { - importDir = fRootDir; - importDir += DIR_BULK_IMPORT; - } - - // Break down loadFileName into vector of file names in case load- - // FileName contains a list of files or 1 or more wildcards. - int rc = buildImportDataFileList(importDir, - loadFileName, - loadFilesList); - - if (rc != NO_ERROR) - { - return rc; - } - } - - // No filenames is considered a fatal error, except for remote mode2. - // For remote mode2 we just mark the table as complete since we will - // have no data to load, but we don't consider this as an error. - if (loadFilesList.size() == 0) - { - ostringstream oss; - oss << "No import files found. " << "default dir: " << importDir << - " importFileName: " << loadFileName; - - if (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC) - { - tableInfo->setLoadFilesInput(bUseStdin, (!fS3Key.empty()), loadFilesList, fS3Host, fS3Key, fS3Secret, fS3Bucket, fS3Region); - tableInfo->markTableComplete( ); - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - return NO_ERROR; - } - else - { - fLog.logMsg( oss.str(), ERR_FILE_NOT_EXIST, MSGLVL_ERROR ); - return ERR_FILE_NOT_EXIST; - } - } - - // Verify that input data files exist. - // We also used to check to make sure the input file is not empty, and - // if it were, we threw an error at this point, but we removed that - // check. With shared-nothing, an empty file is now acceptable. - if (fS3Key.empty()) - { - for (unsigned ndx = 0; ndx < loadFilesList.size(); ndx++ ) - { - // in addition to being more portable due to the use of boost, this change - // actually fixes an inherent bug with cpimport reading from a named pipe. - // Only the first open call gets any data passed through the pipe so the - // here that used to do an open to test for existence meant cpimport would - // never get data from the pipe. - boost::filesystem::path pathFile(loadFilesList[ndx]); - - if ( !boost::filesystem::exists( pathFile ) ) - { - ostringstream oss; - oss << "input data file " << loadFilesList[ndx] << " does not exist"; - fLog.logMsg( oss.str(), ERR_FILE_NOT_EXIST, MSGLVL_ERROR ); - tableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return ERR_FILE_NOT_EXIST; - } - else - { - ostringstream oss; - oss << "input data file " << loadFilesList[ndx]; - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - } - } + ostringstream oss; + oss << "input data file " << loadFilesList[ndx]; + fLog.logMsg(oss.str(), MSGLVL_INFO1); } + } } + } - tableInfo->setLoadFilesInput(bUseStdin, (!fS3Key.empty()), loadFilesList, fS3Host, fS3Key, fS3Secret, fS3Bucket, fS3Region); + tableInfo->setLoadFilesInput(bUseStdin, (!fS3Key.empty()), loadFilesList, fS3Host, fS3Key, fS3Secret, + fS3Bucket, fS3Region); - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Break up the filename string (which may contain a list of file names) @@ -1367,140 +1281,137 @@ int BulkLoad::manageImportDataFileList(Job& job, // RETURN: // NO_ERROR if success //------------------------------------------------------------------------------ -int BulkLoad::buildImportDataFileList( - const std::string& location, - const std::string& filename, - std::vector& loadFiles) +int BulkLoad::buildImportDataFileList(const std::string& location, const std::string& filename, + std::vector& loadFiles) { - char* filenames = new char[filename.size() + 1]; - strcpy(filenames, filename.c_str()); + char* filenames = new char[filename.size() + 1]; + strcpy(filenames, filename.c_str()); - char* str; - char* token; + char* str; + char* token; - for (str = filenames; ; str = NULL) + for (str = filenames;; str = NULL) + { +#ifdef _MSC_VER + // On Windows, only comma and vertbar can separate input files + token = strtok(str, ",|"); +#else + token = strtok(str, ", |"); +#endif + + if (token == NULL) + break; + + // If the token (filename) is fully qualified, then use the filename + // as-is, else prepend the location (path prefix) + boost::filesystem::path p(token); + std::string fullPath; + + if (p.has_root_path()) { -#ifdef _MSC_VER - //On Windows, only comma and vertbar can separate input files - token = strtok(str, ",|"); -#else - token = strtok(str, ", |"); -#endif - - if (token == NULL) - break; - - // If the token (filename) is fully qualified, then use the filename - // as-is, else prepend the location (path prefix) - boost::filesystem::path p(token); - std::string fullPath; - - if (p.has_root_path()) - { - fullPath = token; - } - else - { - fullPath = location; - fullPath += token; - } + fullPath = token; + } + else + { + fullPath = location; + fullPath += token; + } #ifdef _MSC_VER - loadFiles.push_back(fullPath); + loadFiles.push_back(fullPath); #else - // If running mode2, then support a filename with wildcards - if (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC) + // If running mode2, then support a filename with wildcards + if (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC) + { + bool bExpandFileName = false; + + size_t fpos = fullPath.find_first_of("[*?"); + + if (fpos != std::string::npos) + { + bExpandFileName = true; + } + else // expand a directory name + { + struct stat curStat; + + if ((stat(fullPath.c_str(), &curStat) == 0) && (S_ISDIR(curStat.st_mode))) { - bool bExpandFileName = false; + bExpandFileName = true; + fullPath += "/*"; + } + } - size_t fpos = fullPath.find_first_of( "[*?" ); + // If wildcard(s) present use glob() function to expand into a list + if (bExpandFileName) + { + glob_t globBuf; + memset(&globBuf, 0, sizeof(globBuf)); + int globFlags = GLOB_ERR | GLOB_MARK; + int rc = glob(fullPath.c_str(), globFlags, 0, &globBuf); - if (fpos != std::string::npos) - { - bExpandFileName = true; - } - else // expand a directory name - { - struct stat curStat; + if (rc != 0) + { + if (rc == GLOB_NOMATCH) + { + continue; + } + else + { + ostringstream oss; + oss << "Error expanding filename " << fullPath; - if ( (stat(fullPath.c_str(), &curStat) == 0) && - (S_ISDIR(curStat.st_mode)) ) - { - bExpandFileName = true; - fullPath += "/*"; - } - } - - // If wildcard(s) present use glob() function to expand into a list - if (bExpandFileName) - { - glob_t globBuf; - memset(&globBuf, 0, sizeof(globBuf)); - int globFlags = GLOB_ERR | GLOB_MARK; - int rc = glob(fullPath.c_str(), globFlags, 0, &globBuf); - - if (rc != 0) - { - if (rc == GLOB_NOMATCH) - { - continue; - } - else - { - ostringstream oss; - oss << "Error expanding filename " << fullPath; - - if (rc == GLOB_NOSPACE) - oss << "; out of memory"; - else if (rc == GLOB_ABORTED) - oss << "; error reading directory"; - else if (rc == GLOB_NOSYS) - oss << "; globbing not implemented"; - else - oss << "; rc-" << rc; - - fLog.logMsg(oss.str(), ERR_FILE_GLOBBING, MSGLVL_ERROR); - - delete [] filenames; - return ERR_FILE_GLOBBING; - } - } - - // Include all non-directory files in the import file list - std::string fullPath2; - - for (unsigned int k = 0; k < globBuf.gl_pathc; k++) - { - fullPath2 = globBuf.gl_pathv[k]; - - if ( !fullPath2.empty() ) - { - if ( fullPath2[ fullPath2.length() - 1 ] != '/' ) - { - loadFiles.push_back( fullPath2 ); - } - } - } - } // wild card present + if (rc == GLOB_NOSPACE) + oss << "; out of memory"; + else if (rc == GLOB_ABORTED) + oss << "; error reading directory"; + else if (rc == GLOB_NOSYS) + oss << "; globbing not implemented"; else - { - loadFiles.push_back(fullPath); - } - } // mode2 - else + oss << "; rc-" << rc; + + fLog.logMsg(oss.str(), ERR_FILE_GLOBBING, MSGLVL_ERROR); + + delete[] filenames; + return ERR_FILE_GLOBBING; + } + } + + // Include all non-directory files in the import file list + std::string fullPath2; + + for (unsigned int k = 0; k < globBuf.gl_pathc; k++) { - loadFiles.push_back(fullPath); - } // not mode2 + fullPath2 = globBuf.gl_pathv[k]; + + if (!fullPath2.empty()) + { + if (fullPath2[fullPath2.length() - 1] != '/') + { + loadFiles.push_back(fullPath2); + } + } + } + } // wild card present + else + { + loadFiles.push_back(fullPath); + } + } // mode2 + else + { + loadFiles.push_back(fullPath); + } // not mode2 #endif - } // loop through filename tokens + } // loop through filename tokens - delete [] filenames; + delete[] filenames; - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Clear table locks, and rollback any tables that are @@ -1511,73 +1422,70 @@ int BulkLoad::buildImportDataFileList( // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::rollbackLockedTables( ) +int BulkLoad::rollbackLockedTables() { - int rc = NO_ERROR; + int rc = NO_ERROR; - // See if there are any DB tables that were left in a locked state - bool lockedTableFound = false; + // See if there are any DB tables that were left in a locked state + bool lockedTableFound = false; - for ( unsigned i = 0; i < fTableInfo.size(); i++ ) + for (unsigned i = 0; i < fTableInfo.size(); i++) + { + if (fTableInfo[i].isTableLocked()) { - if (fTableInfo[i].isTableLocked()) - { - lockedTableFound = true; - break; - } + lockedTableFound = true; + break; + } + } + + // If 1 or more tables failed to load, then report the lock + // state of each table we were importing. + if (lockedTableFound) + { + // Report the tables that were successfully loaded + for (unsigned i = 0; i < fTableInfo.size(); i++) + { + if (!fTableInfo[i].isTableLocked()) + { + ostringstream oss; + oss << "Table " << fTableInfo[i].getTableName() << " was successfully loaded. "; + fLog.logMsg(oss.str(), MSGLVL_INFO1); + } } - // If 1 or more tables failed to load, then report the lock - // state of each table we were importing. - if (lockedTableFound) + // Report the tables that were not successfully loaded + for (unsigned i = 0; i < fTableInfo.size(); i++) { - // Report the tables that were successfully loaded - for ( unsigned i = 0; i < fTableInfo.size(); i++ ) + if (fTableInfo[i].isTableLocked()) + { + if (fTableInfo[i].hasProcessingBegun()) { - if (!fTableInfo[i].isTableLocked()) - { - ostringstream oss; - oss << "Table " << fTableInfo[i].getTableName() << - " was successfully loaded. "; - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - } + ostringstream oss; + oss << "Table " << fTableInfo[i].getTableName() << " (OID-" << fTableInfo[i].getTableOID() << ")" + << " was not successfully loaded. Rolling back."; + fLog.logMsg(oss.str(), MSGLVL_INFO1); + } + else + { + ostringstream oss; + oss << "Table " << fTableInfo[i].getTableName() << " (OID-" << fTableInfo[i].getTableOID() << ")" + << " did not start loading. No rollback necessary."; + fLog.logMsg(oss.str(), MSGLVL_INFO1); } - // Report the tables that were not successfully loaded - for ( unsigned i = 0; i < fTableInfo.size(); i++ ) + rc = rollbackLockedTable(fTableInfo[i]); + + if (rc != NO_ERROR) { - if (fTableInfo[i].isTableLocked()) - { - if (fTableInfo[i].hasProcessingBegun()) - { - ostringstream oss; - oss << "Table " << fTableInfo[i].getTableName() << - " (OID-" << fTableInfo[i].getTableOID() << ")" << - " was not successfully loaded. Rolling back."; - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - } - else - { - ostringstream oss; - oss << "Table " << fTableInfo[i].getTableName() << - " (OID-" << fTableInfo[i].getTableOID() << ")" << - " did not start loading. No rollback necessary."; - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - } - - rc = rollbackLockedTable( fTableInfo[i] ); - - if (rc != NO_ERROR) - { - break; - } - } + break; } + } } + } - return rc; + return rc; } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Clear table lock, and rollback the specified table that is still locked. @@ -1590,11 +1498,11 @@ int BulkLoad::rollbackLockedTables( ) // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::rollbackLockedTable( TableInfo& tableInfo ) +int BulkLoad::rollbackLockedTable(TableInfo& tableInfo) { - return tableInfo.rollbackWork( ); + return tableInfo.rollbackWork(); } - + //------------------------------------------------------------------------------ // DESCRIPTION: // Update next autoincrement value for specified column OID. @@ -1608,37 +1516,37 @@ int BulkLoad::rollbackLockedTable( TableInfo& tableInfo ) /* static */ int BulkLoad::updateNextValue(OID columnOid, uint64_t nextAutoIncVal) { - // The odds of us ever having 2 updateNextValue() calls going on in parallel - // are slim and none. But it's theoretically possible if we had an import - // job for 2 tables; so we put a mutex here just in case the DDLClient code - // won't work well with 2 competing WE_DDLCommandClient objects in the same - // process (ex: if there is any static data in WE_DDLCommandClient). - boost::mutex::scoped_lock lock( *fDDLMutex ); - WE_DDLCommandClient ddlCommandClt; - unsigned int rc = ddlCommandClt.UpdateSyscolumnNextval( - columnOid, nextAutoIncVal ); + // The odds of us ever having 2 updateNextValue() calls going on in parallel + // are slim and none. But it's theoretically possible if we had an import + // job for 2 tables; so we put a mutex here just in case the DDLClient code + // won't work well with 2 competing WE_DDLCommandClient objects in the same + // process (ex: if there is any static data in WE_DDLCommandClient). + boost::mutex::scoped_lock lock(*fDDLMutex); + WE_DDLCommandClient ddlCommandClt; + unsigned int rc = ddlCommandClt.UpdateSyscolumnNextval(columnOid, nextAutoIncVal); - return (int)rc; + return (int)rc; } //------------------------------------------------------------------------------ bool BulkLoad::addErrorMsg2BrmUpdater(const std::string& tablename, const ostringstream& oss) { - int size = fTableInfo.size(); - - if (size == 0) return false; - - for (int tableId = 0; tableId < size; tableId++) - { - if (fTableInfo[tableId].getTableName() == tablename) - { - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - return true; - } - } + int size = fTableInfo.size(); + if (size == 0) return false; + + for (int tableId = 0; tableId < size; tableId++) + { + if (fTableInfo[tableId].getTableName() == tablename) + { + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + return true; + } + } + + return false; } //------------------------------------------------------------------------------ @@ -1652,14 +1560,13 @@ bool BulkLoad::addErrorMsg2BrmUpdater(const std::string& tablename, const ostrin //------------------------------------------------------------------------------ void BulkLoad::setJobUUID(const std::string& jobUUID) { - fUUID = boost::uuids::string_generator()(jobUUID); + fUUID = boost::uuids::string_generator()(jobUUID); } void BulkLoad::setDefaultJobUUID() { - if (fUUID.is_nil()) - fUUID = boost::uuids::random_generator()(); + if (fUUID.is_nil()) + fUUID = boost::uuids::random_generator()(); } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index 168dbda99..99220692b 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_bulkload.h 4631 2013-05-02 15:21:09Z dcathey $ -* -*******************************************************************************/ + * $Id: we_bulkload.h 4631 2013-05-02 15:21:09Z dcathey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_BULKLOAD_H_ @@ -48,7 +48,7 @@ #include #include -#if 0 //defined(_MSC_VER) && defined(WE_BULKLOAD_DLLEXPORT) +#if 0 // defined(_MSC_VER) && defined(WE_BULKLOAD_DLLEXPORT) #define EXPORT __declspec(dllexport) #else #define EXPORT @@ -57,267 +57,248 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class BulkLoad */ class BulkLoad : public FileOp { -public: + public: + /** + * @brief BulkLoad constructor + */ + EXPORT BulkLoad(); - /** - * @brief BulkLoad constructor - */ - EXPORT BulkLoad(); + /** + * @brief BulkLoad destructor + */ + EXPORT ~BulkLoad(); - /** - * @brief BulkLoad destructor - */ - EXPORT ~BulkLoad(); + /** + * @brief Load job information + */ + EXPORT int loadJobInfo(const std::string& fullFileName, bool bUseTempJobFile, int argc, char** argv, + bool bLogInfo2ToConsole, bool bValidateColumnList); - /** - * @brief Load job information - */ - EXPORT int loadJobInfo( const std::string& fullFileName, - bool bUseTempJobFile, - int argc, - char** argv, - bool bLogInfo2ToConsole, - bool bValidateColumnList ); + /** + * @brief Pre process jobs to validate and assign values to the job structure + */ + int preProcess(Job& job, int tableNo, TableInfo* tableInfo); - /** - * @brief Pre process jobs to validate and assign values to the job structure - */ - int preProcess(Job& job, int tableNo, TableInfo* tableInfo); + /** + * @brief Print job information + */ + void printJob(); - /** - * @brief Print job information - */ - void printJob( ); + /** + * @brief Process job + */ + EXPORT int processJob(); - /** - * @brief Process job - */ - EXPORT int processJob( ); + /** + * @brief Set Debug level for this BulkLoad object and any data members + */ + void setAllDebug(DebugLevel level); - /** - * @brief Set Debug level for this BulkLoad object and any data members - */ - void setAllDebug( DebugLevel level ); + /** + * @brief Update next autoincrement value for specified OID. + * @param columnOID oid of autoincrement column to be updated + * @param nextValue next autoincrement value to assign to tableOID + */ + static int updateNextValue(OID columnOID, uint64_t nextValue); - /** - * @brief Update next autoincrement value for specified OID. - * @param columnOID oid of autoincrement column to be updated - * @param nextValue next autoincrement value to assign to tableOID - */ - static int updateNextValue(OID columnOID, uint64_t nextValue); + // Accessors and mutators + void addToCmdLineImportFileList(const std::string& importFile); + const std::string& getAlternateImportDir() const; + const std::string& getErrorDir() const; + const std::string& getTimeZone() const; + const std::string& getJobDir() const; + const std::string& getSchema() const; + const std::string& getTempJobDir() const; + const std::string& getS3Key() const; + const std::string& getS3Secret() const; + const std::string& getS3Bucket() const; + const std::string& getS3Host() const; + const std::string& getS3Region() const; + bool getTruncationAsError() const; + BulkModeType getBulkLoadMode() const; + bool getContinue() const; + boost::uuids::uuid getJobUUID() const + { + return fUUID; + } - // Accessors and mutators - void addToCmdLineImportFileList(const std::string& importFile); - const std::string& getAlternateImportDir( ) const; - const std::string& getErrorDir ( ) const; - const std::string& getTimeZone ( ) const; - const std::string& getJobDir ( ) const; - const std::string& getSchema ( ) const; - const std::string& getTempJobDir ( ) const; - const std::string& getS3Key ( ) const; - const std::string& getS3Secret ( ) const; - const std::string& getS3Bucket ( ) const; - const std::string& getS3Host ( ) const; - const std::string& getS3Region ( ) const; - bool getTruncationAsError ( ) const; - BulkModeType getBulkLoadMode ( ) const; - bool getContinue ( ) const; - boost::uuids::uuid getJobUUID ( ) const - { - return fUUID; - } + EXPORT int setAlternateImportDir(const std::string& loadDir, std::string& errMsg); + void setImportDataMode(ImportDataMode importMode); + void setColDelimiter(char delim); + void setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName); + void setEnclosedByChar(char enChar); + void setEscapeChar(char esChar); + void setKeepRbMetaFiles(bool keepMeta); + void setMaxErrorCount(unsigned int maxErrors); + void setNoOfParseThreads(int parseThreads); + void setNoOfReadThreads(int readThreads); + void setNullStringMode(bool bMode); + void setParseErrorOnTable(int tableId, bool lockParseMutex); + void setParserNum(int parser); + void setProcessName(const std::string& processName); + void setReadBufferCount(int noOfReadBuffers); + void setReadBufferSize(int readBufferSize); + void setTxnID(BRM::TxnID txnID); + void setVbufReadSize(int vbufReadSize); + void setTruncationAsError(bool bTruncationAsError); + void setJobUUID(const std::string& jobUUID); + void setErrorDir(const std::string& errorDir); + void setTimeZone(const std::string& timeZone); + void setS3Key(const std::string& key); + void setS3Secret(const std::string& secret); + void setS3Bucket(const std::string& bucket); + void setS3Host(const std::string& host); + void setS3Region(const std::string& region); + void setUsername(const std::string& username); + // Timer functions + void startTimer(); + void stopTimer(); + double getTotalRunTime() const; - EXPORT int setAlternateImportDir( const std::string& loadDir, - std::string& errMsg); - void setImportDataMode ( ImportDataMode importMode ); - void setColDelimiter ( char delim ); - void setBulkLoadMode ( BulkModeType bulkMode, - const std::string& rptFileName ); - void setEnclosedByChar ( char enChar); - void setEscapeChar ( char esChar); - void setKeepRbMetaFiles ( bool keepMeta ); - void setMaxErrorCount ( unsigned int maxErrors ); - void setNoOfParseThreads ( int parseThreads ); - void setNoOfReadThreads ( int readThreads ); - void setNullStringMode ( bool bMode ); - void setParseErrorOnTable (int tableId, bool lockParseMutex); - void setParserNum ( int parser ); - void setProcessName ( const std::string& processName ); - void setReadBufferCount ( int noOfReadBuffers ); - void setReadBufferSize ( int readBufferSize ); - void setTxnID ( BRM::TxnID txnID ); - void setVbufReadSize ( int vbufReadSize ); - void setTruncationAsError ( bool bTruncationAsError ); - void setJobUUID ( const std::string& jobUUID ); - void setErrorDir ( const std::string& errorDir ); - void setTimeZone ( const std::string& timeZone ); - void setS3Key ( const std::string& key ); - void setS3Secret ( const std::string& secret ); - void setS3Bucket ( const std::string& bucket ); - void setS3Host ( const std::string& host ); - void setS3Region ( const std::string& region ); - void setUsername ( const std::string& username ); - // Timer functions - void startTimer ( ); - void stopTimer ( ); - double getTotalRunTime ( ) const; + void disableTimeOut(const bool disableTimeOut); + bool disableTimeOut() const; - void disableTimeOut ( const bool disableTimeOut); - bool disableTimeOut ( ) const; + static void disableConsoleOutput(const bool noConsoleOutput) + { + fNoConsoleOutput = noConsoleOutput; + } + static bool disableConsoleOutput() + { + return fNoConsoleOutput; + } - static void disableConsoleOutput ( const bool noConsoleOutput) - { - fNoConsoleOutput = noConsoleOutput; - } - static bool disableConsoleOutput ( ) - { - return fNoConsoleOutput; - } + // Add error message into appropriate BRM updater + static bool addErrorMsg2BrmUpdater(const std::string& tablename, const std::ostringstream& oss); + void setDefaultJobUUID(); - // Add error message into appropriate BRM updater - static bool addErrorMsg2BrmUpdater(const std::string& tablename, const std::ostringstream& oss); - void setDefaultJobUUID ( ); + private: + //-------------------------------------------------------------------------- + // Private Data Members + //-------------------------------------------------------------------------- + XMLJob fJobInfo; // current job information -private: + boost::scoped_ptr fColOp; // column operation - //-------------------------------------------------------------------------- - // Private Data Members - //-------------------------------------------------------------------------- - XMLJob fJobInfo; // current job information + std::string fRootDir; // job process root directory + std::string fJobFileName; // job description file name - boost::scoped_ptr fColOp; // column operation + Log fLog; // logger - std::string fRootDir; // job process root directory - std::string fJobFileName; // job description file name + int fNumOfParser; // total number of parser + char fColDelim; // delimits col values within a row - Log fLog; // logger + int fNoOfBuffers; // Number of read buffers + int fBufferSize; // Read buffer size + int fFileVbufSize; // Internal file system buffer size + long long fMaxErrors; // Max allowable errors per job + std::string fAlternateImportDir; // Alternate bulk import directory + std::string fErrorDir; // Opt. where error records record + std::string fProcessName; // Application process name + static boost::ptr_vector fTableInfo; // Vector of Table information + int fNoOfParseThreads; // Number of parse threads + int fNoOfReadThreads; // Number of read threads + boost::thread_group fReadThreads; // Read thread group + boost::thread_group fParseThreads; // Parse thread group + boost::mutex fReadMutex; // Manages table selection by each + // read thread + boost::mutex fParseMutex; // Manages table/buffer/column + // selection by each parsing thread + BRM::TxnID fTxnID; // TransID acquired from SessionMgr + bool fKeepRbMetaFiles; // Keep/delete bulkRB metadata files + bool fNullStringMode; // Treat "NULL" as NULL value + char fEnclosedByChar; // Char used to enclose column value + char fEscapeChar; // Escape char within enclosed value + timeval fStartTime; // job start time + timeval fEndTime; // job end time + double fTotalTime; // elapsed time for current phase + std::vector fCmdLineImportFiles; // Import Files from cmd line + BulkModeType fBulkMode; // Distributed bulk mode (1,2, or 3) + std::string fBRMRptFileName; // Name of distributed mode rpt file + bool fbTruncationAsError; // Treat string truncation as error + ImportDataMode fImportDataMode; // Importing text or binary data + bool fbContinue; // true when read and parse r running + // + static boost::mutex* fDDLMutex; // Insure only 1 DDL op at a time - int fNumOfParser; // total number of parser - char fColDelim; // delimits col values within a row + EXPORT static const std::string DIR_BULK_JOB; // Bulk job directory + EXPORT static const std::string DIR_BULK_TEMP_JOB; // Dir for tmp job files + static const std::string DIR_BULK_IMPORT; // Bulk job import dir + static const std::string DIR_BULK_LOG; // Bulk job log directory + bool fDisableTimeOut; // disable timeout when waiting for table lock + boost::uuids::uuid fUUID; // job UUID + static bool fNoConsoleOutput; // disable output to console + std::string fTimeZone; // Timezone to use for TIMESTAMP data type + std::string fS3Key; // S3 Key + std::string fS3Secret; // S3 Secret + std::string fS3Host; // S3 Host + std::string fS3Bucket; // S3 Bucket + std::string fS3Region; // S3 Region + std::string fUsername; // data files owner name mysql by default - int fNoOfBuffers; // Number of read buffers - int fBufferSize; // Read buffer size - int fFileVbufSize; // Internal file system buffer size - long long fMaxErrors; // Max allowable errors per job - std::string fAlternateImportDir; // Alternate bulk import directory - std::string fErrorDir; // Opt. where error records record - std::string fProcessName; // Application process name - static boost::ptr_vector fTableInfo;// Vector of Table information - int fNoOfParseThreads; // Number of parse threads - int fNoOfReadThreads; // Number of read threads - boost::thread_group fReadThreads; // Read thread group - boost::thread_group fParseThreads; // Parse thread group - boost::mutex fReadMutex; // Manages table selection by each - // read thread - boost::mutex fParseMutex; // Manages table/buffer/column - // selection by each parsing thread - BRM::TxnID fTxnID; // TransID acquired from SessionMgr - bool fKeepRbMetaFiles; // Keep/delete bulkRB metadata files - bool fNullStringMode; // Treat "NULL" as NULL value - char fEnclosedByChar; // Char used to enclose column value - char fEscapeChar; // Escape char within enclosed value - timeval fStartTime; // job start time - timeval fEndTime; // job end time - double fTotalTime; // elapsed time for current phase - std::vector fCmdLineImportFiles; // Import Files from cmd line - BulkModeType fBulkMode; // Distributed bulk mode (1,2, or 3) - std::string fBRMRptFileName; // Name of distributed mode rpt file - bool fbTruncationAsError; // Treat string truncation as error - ImportDataMode fImportDataMode; // Importing text or binary data - bool fbContinue; // true when read and parse r running - // - static boost::mutex* fDDLMutex; // Insure only 1 DDL op at a time + //-------------------------------------------------------------------------- + // Private Functions + //-------------------------------------------------------------------------- - EXPORT static const std::string DIR_BULK_JOB; // Bulk job directory - EXPORT static const std::string DIR_BULK_TEMP_JOB;// Dir for tmp job files - static const std::string DIR_BULK_IMPORT; // Bulk job import dir - static const std::string DIR_BULK_LOG; // Bulk job log directory - bool fDisableTimeOut; // disable timeout when waiting for table lock - boost::uuids::uuid fUUID; // job UUID - static bool fNoConsoleOutput; // disable output to console - std::string fTimeZone; // Timezone to use for TIMESTAMP data type - std::string fS3Key; // S3 Key - std::string fS3Secret; // S3 Secret - std::string fS3Host; // S3 Host - std::string fS3Bucket; // S3 Bucket - std::string fS3Region; // S3 Region - std::string fUsername; // data files owner name mysql by default + // Spawn the worker threads. + void spawnWorkers(); - //-------------------------------------------------------------------------- - // Private Functions - //-------------------------------------------------------------------------- + // Checks if all tables have the status set + bool allTablesDone(Status status); - // Spawn the worker threads. - void spawnWorkers(); + // Lock the table for read. Called by the read thread. + int lockTableForRead(int id); - // Checks if all tables have the status set - bool allTablesDone(Status status); + // Get column for parsing. Called by the parse thread. + // @bug 2099 - Temporary hack to diagnose deadlock. Added report parm below. + bool lockColumnForParse(int id, // thread id + int& tableId, // selected table id + int& columnId, // selected column id + int& myParseBuffer, // selected parse buffer + bool report); - // Lock the table for read. Called by the read thread. - int lockTableForRead(int id); + // Map specified DBRoot to it's first segment file number + int mapDBRootToFirstSegment(OID columnOid, uint16_t dbRoot, uint16_t& segment); - // Get column for parsing. Called by the parse thread. - // @bug 2099 - Temporary hack to diagnose deadlock. Added report parm below. - bool lockColumnForParse(int id, // thread id - int& tableId, // selected table id - int& columnId, // selected column id - int& myParseBuffer, // selected parse buffer - bool report); + // The thread method for the read thread. + void read(int id); - // Map specified DBRoot to it's first segment file number - int mapDBRootToFirstSegment(OID columnOid, - uint16_t dbRoot, - uint16_t& segment); + // The thread method for the parse thread. + void parse(int id); - // The thread method for the read thread. - void read(int id); + // Sleep method + void sleepMS(long int ms); - // The thread method for the parse thread. - void parse(int id); + // Initialize auto-increment column for specified schema and table. + int preProcessAutoInc(const std::string& fullTableName, // schema.table + ColumnInfo* colInfo); // ColumnInfo associated with AI column - // Sleep method - void sleepMS(long int ms); + // Determine starting HWM and LBID after block skipping added to HWM + int preProcessHwmLbid(const ColumnInfo* info, int minWidth, uint32_t partition, uint16_t segment, HWM& hwm, + BRM::LBID_t& lbid, bool& bSkippedToNewExtent); - // Initialize auto-increment column for specified schema and table. - int preProcessAutoInc( - const std::string& fullTableName,// schema.table - ColumnInfo* colInfo); // ColumnInfo associated with AI column + // Rollback any tables that are left in a locked state at EOJ. + int rollbackLockedTables(); - // Determine starting HWM and LBID after block skipping added to HWM - int preProcessHwmLbid( const ColumnInfo* info, - int minWidth, - uint32_t partition, - uint16_t segment, - HWM& hwm, - BRM::LBID_t& lbid, - bool& bSkippedToNewExtent); + // Rollback a table left in a locked state. + int rollbackLockedTable(TableInfo& tableInfo); - // Rollback any tables that are left in a locked state at EOJ. - int rollbackLockedTables( ); + // Save metadata info required for shared-nothing bulk rollback. + int saveBulkRollbackMetaData(Job& job, // current job + TableInfo* tableInfo, // TableInfo for table of interest + const std::vector& segFileInfo, // vector seg file info + const std::vector& dbRootHWMInfoPM); - // Rollback a table left in a locked state. - int rollbackLockedTable( TableInfo& tableInfo ); + // Manage/validate the list of 1 or more import data files + int manageImportDataFileList(Job& job, // current job + int tableNo, // table number of current job + TableInfo* tableInfo); // TableInfo for table of interest - // Save metadata info required for shared-nothing bulk rollback. - int saveBulkRollbackMetaData( Job& job, // current job - TableInfo* tableInfo, // TableInfo for table of interest - const std::vector& segFileInfo, //vector seg file info - const std::vector& dbRootHWMInfoPM); - - // Manage/validate the list of 1 or more import data files - int manageImportDataFileList(Job& job, // current job - int tableNo, // table number of current job - TableInfo* tableInfo); // TableInfo for table of interest - - // Break up list of file names into a vector of filename strings - int buildImportDataFileList( - const std::string& location, - const std::string& filename, - std::vector& importFileNames); + // Break up list of file names into a vector of filename strings + int buildImportDataFileList(const std::string& location, const std::string& filename, + std::vector& importFileNames); }; //------------------------------------------------------------------------------ @@ -325,253 +306,251 @@ private: //------------------------------------------------------------------------------ inline void BulkLoad::addToCmdLineImportFileList(const std::string& importFile) { - fCmdLineImportFiles.push_back( importFile ); + fCmdLineImportFiles.push_back(importFile); } -inline const std::string& BulkLoad::getAlternateImportDir( ) const +inline const std::string& BulkLoad::getAlternateImportDir() const { - return fAlternateImportDir; + return fAlternateImportDir; } -inline const std::string& BulkLoad::getErrorDir( ) const +inline const std::string& BulkLoad::getErrorDir() const { - return fErrorDir; + return fErrorDir; } -inline const std::string& BulkLoad::getTimeZone( ) const +inline const std::string& BulkLoad::getTimeZone() const { - return fTimeZone; + return fTimeZone; } -inline const std::string& BulkLoad::getJobDir( ) const +inline const std::string& BulkLoad::getJobDir() const { - return DIR_BULK_JOB; + return DIR_BULK_JOB; } -inline const std::string& BulkLoad::getSchema( ) const +inline const std::string& BulkLoad::getSchema() const { - return fJobInfo.getJob().schema; + return fJobInfo.getJob().schema; } -inline const std::string& BulkLoad::getTempJobDir( ) const +inline const std::string& BulkLoad::getTempJobDir() const { - return DIR_BULK_TEMP_JOB; + return DIR_BULK_TEMP_JOB; } -inline const std::string& BulkLoad::getS3Key( ) const +inline const std::string& BulkLoad::getS3Key() const { - return fS3Key; + return fS3Key; } -inline const std::string& BulkLoad::getS3Secret( ) const +inline const std::string& BulkLoad::getS3Secret() const { - return fS3Secret; + return fS3Secret; } -inline const std::string& BulkLoad::getS3Bucket( ) const +inline const std::string& BulkLoad::getS3Bucket() const { - return fS3Bucket; + return fS3Bucket; } -inline const std::string& BulkLoad::getS3Host( ) const +inline const std::string& BulkLoad::getS3Host() const { - return fS3Host; + return fS3Host; } -inline const std::string& BulkLoad::getS3Region( ) const +inline const std::string& BulkLoad::getS3Region() const { - return fS3Region; + return fS3Region; } -inline bool BulkLoad::getTruncationAsError ( ) const +inline bool BulkLoad::getTruncationAsError() const { - return fbTruncationAsError; + return fbTruncationAsError; } -inline BulkModeType BulkLoad::getBulkLoadMode ( ) const +inline BulkModeType BulkLoad::getBulkLoadMode() const { - return fBulkMode; + return fBulkMode; } -inline bool BulkLoad::getContinue ( ) const +inline bool BulkLoad::getContinue() const { - return fbContinue; + return fbContinue; } inline void BulkLoad::printJob() { - if (isDebug(DEBUG_1)) - fJobInfo.printJobInfo(fLog); - else - fJobInfo.printJobInfoBrief(fLog); + if (isDebug(DEBUG_1)) + fJobInfo.printJobInfo(fLog); + else + fJobInfo.printJobInfoBrief(fLog); } -inline void BulkLoad::setAllDebug( DebugLevel level ) +inline void BulkLoad::setAllDebug(DebugLevel level) { - setDebugLevel( level ); - fLog.setDebugLevel( level ); + setDebugLevel(level); + fLog.setDebugLevel(level); } -inline void BulkLoad::setColDelimiter( char delim ) +inline void BulkLoad::setColDelimiter(char delim) { - fColDelim = delim; + fColDelim = delim; } -inline void BulkLoad::setBulkLoadMode( - BulkModeType bulkMode, - const std::string& rptFileName ) +inline void BulkLoad::setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName) { - fBulkMode = bulkMode; - fBRMRptFileName = rptFileName; + fBulkMode = bulkMode; + fBRMRptFileName = rptFileName; } -inline void BulkLoad::setEnclosedByChar( char enChar ) +inline void BulkLoad::setEnclosedByChar(char enChar) { - fEnclosedByChar = enChar; + fEnclosedByChar = enChar; } -inline void BulkLoad::setEscapeChar( char esChar ) +inline void BulkLoad::setEscapeChar(char esChar) { - fEscapeChar = esChar; + fEscapeChar = esChar; } inline void BulkLoad::setImportDataMode(ImportDataMode importMode) { - fImportDataMode = importMode; + fImportDataMode = importMode; } -inline void BulkLoad::setKeepRbMetaFiles( bool keepMeta ) +inline void BulkLoad::setKeepRbMetaFiles(bool keepMeta) { - fKeepRbMetaFiles = keepMeta; + fKeepRbMetaFiles = keepMeta; } // Mutator takes an unsigned int, but we store in a long long, because... // TableInfo which eventually needs this attribute, takes an unsigned int, // but we want to be able to init to -1, to indicate when it has not been set. -inline void BulkLoad::setMaxErrorCount( unsigned int maxErrors ) +inline void BulkLoad::setMaxErrorCount(unsigned int maxErrors) { - fMaxErrors = maxErrors; + fMaxErrors = maxErrors; } -inline void BulkLoad::setNoOfParseThreads(int parseThreads ) +inline void BulkLoad::setNoOfParseThreads(int parseThreads) { - fNoOfParseThreads = parseThreads; + fNoOfParseThreads = parseThreads; } -inline void BulkLoad::setNoOfReadThreads( int readThreads ) +inline void BulkLoad::setNoOfReadThreads(int readThreads) { - fNoOfReadThreads = readThreads; + fNoOfReadThreads = readThreads; } -inline void BulkLoad::setNullStringMode( bool bMode ) +inline void BulkLoad::setNullStringMode(bool bMode) { - fNullStringMode = bMode; + fNullStringMode = bMode; } -inline void BulkLoad::setParserNum( int parser ) +inline void BulkLoad::setParserNum(int parser) { - fNumOfParser = parser; + fNumOfParser = parser; } -inline void BulkLoad::setProcessName( const std::string& processName ) +inline void BulkLoad::setProcessName(const std::string& processName) { - fProcessName = processName; + fProcessName = processName; } -inline void BulkLoad::setReadBufferCount( int noOfReadBuffers ) +inline void BulkLoad::setReadBufferCount(int noOfReadBuffers) { - fNoOfBuffers = noOfReadBuffers; + fNoOfBuffers = noOfReadBuffers; } -inline void BulkLoad::setReadBufferSize( int readBufferSize ) +inline void BulkLoad::setReadBufferSize(int readBufferSize) { - fBufferSize = readBufferSize; + fBufferSize = readBufferSize; } -inline void BulkLoad::setTxnID( BRM::TxnID txnID ) +inline void BulkLoad::setTxnID(BRM::TxnID txnID) { - fTxnID = txnID; + fTxnID = txnID; } -inline void BulkLoad::setVbufReadSize( int vbufReadSize ) +inline void BulkLoad::setVbufReadSize(int vbufReadSize) { - fFileVbufSize = vbufReadSize; + fFileVbufSize = vbufReadSize; } inline void BulkLoad::setTruncationAsError(bool bTruncationAsError) { - fbTruncationAsError = bTruncationAsError; + fbTruncationAsError = bTruncationAsError; } -inline void BulkLoad::setErrorDir( const std::string& errorDir ) +inline void BulkLoad::setErrorDir(const std::string& errorDir) { - fErrorDir = errorDir; + fErrorDir = errorDir; } -inline void BulkLoad::setTimeZone( const std::string& timeZone ) +inline void BulkLoad::setTimeZone(const std::string& timeZone) { - fTimeZone = timeZone; + fTimeZone = timeZone; } -inline void BulkLoad::setS3Key( const std::string& key ) +inline void BulkLoad::setS3Key(const std::string& key) { - fS3Key = key; + fS3Key = key; } -inline void BulkLoad::setS3Secret( const std::string& secret ) +inline void BulkLoad::setS3Secret(const std::string& secret) { - fS3Secret = secret; + fS3Secret = secret; } -inline void BulkLoad::setS3Bucket( const std::string& bucket ) +inline void BulkLoad::setS3Bucket(const std::string& bucket) { - fS3Bucket = bucket; + fS3Bucket = bucket; } -inline void BulkLoad::setS3Host( const std::string& host ) +inline void BulkLoad::setS3Host(const std::string& host) { - fS3Host = host; + fS3Host = host; } -inline void BulkLoad::setS3Region( const std::string& region ) +inline void BulkLoad::setS3Region(const std::string& region) { - fS3Region = region; + fS3Region = region; } -inline void BulkLoad::setUsername( const std::string& username ) +inline void BulkLoad::setUsername(const std::string& username) { - fUsername = username; + fUsername = username; } -inline void BulkLoad::startTimer( ) +inline void BulkLoad::startTimer() { - gettimeofday( &fStartTime, 0 ); + gettimeofday(&fStartTime, 0); } inline void BulkLoad::stopTimer() { - gettimeofday( &fEndTime, 0 ); - fTotalTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) - - (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); + gettimeofday(&fEndTime, 0); + fTotalTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) - + (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); } inline double BulkLoad::getTotalRunTime() const { - return fTotalTime; + return fTotalTime; } -inline void BulkLoad::disableTimeOut( const bool disableTimeOut) +inline void BulkLoad::disableTimeOut(const bool disableTimeOut) { - fDisableTimeOut = disableTimeOut; + fDisableTimeOut = disableTimeOut; } inline bool BulkLoad::disableTimeOut() const { - return fDisableTimeOut; + return fDisableTimeOut; } -} // end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_BULKLOAD_H_ +#endif // _WE_BULKLOAD_H_ diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 290d2ae62..9f26a161f 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -26,7 +26,7 @@ #include #include #include -#include // includes on linux +#include // includes on linux #include #include #include @@ -43,7 +43,7 @@ #include "joblisttypes.h" -#include "utils_utf8.h" // utf8_truncate_point() +#include "utils_utf8.h" // utf8_truncate_point() using namespace std; using namespace boost; @@ -55,194 +55,192 @@ namespace { inline int __signbitf(float __x) { - union - { - float __f; - int __i; - } __u; - __u.__f = __x; - return __u.__i < 0; + union + { + float __f; + int __i; + } __u; + __u.__f = __x; + return __u.__i < 0; } inline int __signbit(double __x) { - union - { - double __d; - int __i[2]; - } __u; - __u.__d = __x; - return __u.__i[1] < 0; -} + union + { + double __d; + int __i[2]; + } __u; + __u.__d = __x; + return __u.__i[1] < 0; } +} // namespace #define signbit(x) (sizeof(x) == sizeof(float) ? __signbitf(x) : __signbit(x)) #endif namespace { -const std::string INPUT_ERROR_WRONG_NO_COLUMNS = - "Data contains wrong number of columns"; -const std::string INPUT_ERROR_TOO_LONG = - "Data in wrong format; exceeds max field length; "; -const std::string INPUT_ERROR_NULL_CONSTRAINT = - "Data violates NOT NULL constraint with no default"; -const std::string INPUT_ERROR_ODD_VARBINARY_LENGTH = - "VarBinary column is incomplete; odd number of bytes; "; -const std::string INPUT_ERROR_STRING_TOO_LONG = - "Character data exceeds max field length; "; -const char NULL_CHAR = 'N'; -const char* NULL_VALUE_STRING = "NULL"; -const char NULL_AUTO_INC_0 = '0'; +const std::string INPUT_ERROR_WRONG_NO_COLUMNS = "Data contains wrong number of columns"; +const std::string INPUT_ERROR_TOO_LONG = "Data in wrong format; exceeds max field length; "; +const std::string INPUT_ERROR_NULL_CONSTRAINT = "Data violates NOT NULL constraint with no default"; +const std::string INPUT_ERROR_ODD_VARBINARY_LENGTH = "VarBinary column is incomplete; odd number of bytes; "; +const std::string INPUT_ERROR_STRING_TOO_LONG = "Character data exceeds max field length; "; +const char NULL_CHAR = 'N'; +const char* NULL_VALUE_STRING = "NULL"; +const char NULL_AUTO_INC_0 = '0'; const unsigned long long NULL_AUTO_INC_0_BINARY = 0; -const char NEWLINE_CHAR = '\n'; +const char NEWLINE_CHAR = '\n'; // Enumeration states related to parsing a column value enum FieldParsingState { - FLD_PARSE_LEADING_CHAR_STATE = 1, // parsing leading character - FLD_PARSE_ENCLOSED_STATE = 2, // parsing an enclosed column value - FLD_PARSE_TRAILING_CHAR_STATE = 3, // parsing bytes after an - // enclosed column value - FLD_PARSE_NORMAL_STATE = 4 // parsing non-enclosed column value + FLD_PARSE_LEADING_CHAR_STATE = 1, // parsing leading character + FLD_PARSE_ENCLOSED_STATE = 2, // parsing an enclosed column value + FLD_PARSE_TRAILING_CHAR_STATE = 3, // parsing bytes after an + // enclosed column value + FLD_PARSE_NORMAL_STATE = 4 // parsing non-enclosed column value }; //------------------------------------------------------------------------------ // Expand pRowData to size "newArrayCapacity", preserving contents, and // deleting old pointer //------------------------------------------------------------------------------ -inline void resizeRowDataArray( char** pRowData, - unsigned int dataLength, - unsigned int newArrayCapacity) +inline void resizeRowDataArray(char** pRowData, unsigned int dataLength, unsigned int newArrayCapacity) { - char* tmpRaw = new char[newArrayCapacity]; - memcpy(tmpRaw, *pRowData, dataLength); - delete []*pRowData; - *pRowData = tmpRaw; + char* tmpRaw = new char[newArrayCapacity]; + memcpy(tmpRaw, *pRowData, dataLength); + delete[] * pRowData; + *pRowData = tmpRaw; } -} +} // namespace //#define DEBUG_TOKEN_PARSING 1 namespace WriteEngine { - //------------------------------------------------------------------------------ // BulkLoadBuffer constructor //------------------------------------------------------------------------------ -BulkLoadBuffer::BulkLoadBuffer( - unsigned numberOfCols, unsigned bufferSize, Log* logger, - int bufferId, const std::string& tableName, - const JobFieldRefList& jobFieldRefList ) : - fOverflowSize(0), fParseComplete(0), fTotalRows(0), fStartRow(0), - fStartRowForLogging(0), fAutoIncGenCount(0), - fAutoIncNextValue(0), - fReadSize(0), fLog(logger), fNullStringMode(false), - fEnclosedByChar('\0'), fEscapeChar('\\'), - fBufferId(bufferId), fTableName(tableName), - fbTruncationAsError(false), fImportDataMode(IMPORT_DATA_TEXT), - fTimeZone("SYSTEM"), - fFixedBinaryRecLen(0) +BulkLoadBuffer::BulkLoadBuffer(unsigned numberOfCols, unsigned bufferSize, Log* logger, int bufferId, + const std::string& tableName, const JobFieldRefList& jobFieldRefList) + : fOverflowSize(0) + , fParseComplete(0) + , fTotalRows(0) + , fStartRow(0) + , fStartRowForLogging(0) + , fAutoIncGenCount(0) + , fAutoIncNextValue(0) + , fReadSize(0) + , fLog(logger) + , fNullStringMode(false) + , fEnclosedByChar('\0') + , fEscapeChar('\\') + , fBufferId(bufferId) + , fTableName(tableName) + , fbTruncationAsError(false) + , fImportDataMode(IMPORT_DATA_TEXT) + , fTimeZone("SYSTEM") + , fFixedBinaryRecLen(0) { - fData = new char[bufferSize]; - fOverflowBuf = NULL; - fStatusBLB = WriteEngine::NEW; - fNumberOfColumns = numberOfCols; - fBufferSize = bufferSize; + fData = new char[bufferSize]; + fOverflowBuf = NULL; + fStatusBLB = WriteEngine::NEW; + fNumberOfColumns = numberOfCols; + fBufferSize = bufferSize; - fColumnLocks.clear(); + fColumnLocks.clear(); - fTokens = 0; + fTokens = 0; - fRowStatus.clear(); - fErrRows.clear(); + fRowStatus.clear(); + fErrRows.clear(); - struct LockInfo info; - info.locker = -1; - info.status = WriteEngine::NEW; + struct LockInfo info; + info.locker = -1; + info.status = WriteEngine::NEW; - fColumnLocks.resize(numberOfCols); - fColumnLocks.assign(fNumberOfColumns, info); + fColumnLocks.resize(numberOfCols); + fColumnLocks.assign(fNumberOfColumns, info); - fTotalReadRowsParser = 0; - fStartRowParser = 0; - fDataParser = 0; - fTokensParser = 0; - fStartRowForLoggingParser = 0; - fAutoIncGenCountParser = 0; - fNumFieldsInFile = 0; - fNumColsInFile = 0; + fTotalReadRowsParser = 0; + fStartRowParser = 0; + fDataParser = 0; + fTokensParser = 0; + fStartRowForLoggingParser = 0; + fAutoIncGenCountParser = 0; + fNumFieldsInFile = 0; + fNumColsInFile = 0; - // Count the total number of fields in the input file (fNumFieldsInFile) - // and the number of db columns that will be loaded from those fields - // (fNumColsInFile). Keep in mind that fNumColsInFile may be less than - // fNumFieldsInFile, because there may be fields we are to ignore, and/or - // some db columns may get default loaded without a corresponding field - // in the input file. - fFieldList.resize( jobFieldRefList.size() ); + // Count the total number of fields in the input file (fNumFieldsInFile) + // and the number of db columns that will be loaded from those fields + // (fNumColsInFile). Keep in mind that fNumColsInFile may be less than + // fNumFieldsInFile, because there may be fields we are to ignore, and/or + // some db columns may get default loaded without a corresponding field + // in the input file. + fFieldList.resize(jobFieldRefList.size()); - for (unsigned k = 0; k < jobFieldRefList.size(); k++) + for (unsigned k = 0; k < jobFieldRefList.size(); k++) + { + fFieldList[k] = jobFieldRefList[k]; + + switch (jobFieldRefList[k].fFldColType) { - fFieldList[k] = jobFieldRefList[k]; + case BULK_FLDCOL_COLUMN_FIELD: + { + fNumColsInFile++; + fNumFieldsInFile++; + break; + } - switch (jobFieldRefList[k].fFldColType) - { - case BULK_FLDCOL_COLUMN_FIELD: - { - fNumColsInFile++; - fNumFieldsInFile++; - break; - } + case BULK_FLDCOL_IGNORE_FIELD: + { + fNumFieldsInFile++; + break; + } - case BULK_FLDCOL_IGNORE_FIELD: - { - fNumFieldsInFile++; - break; - } - - case BULK_FLDCOL_COLUMN_DEFAULT: - default: - { - break; - } - } + case BULK_FLDCOL_COLUMN_DEFAULT: + default: + { + break; + } } + } } - + //------------------------------------------------------------------------------ // BulkLoadBuffer destructor //------------------------------------------------------------------------------ BulkLoadBuffer::~BulkLoadBuffer() { - if (fData != NULL) - delete [] fData; + if (fData != NULL) + delete[] fData; - if (fOverflowBuf != NULL) - delete [] fOverflowBuf; + if (fOverflowBuf != NULL) + delete[] fOverflowBuf; - fColumnLocks.clear(); + fColumnLocks.clear(); - if (fTokens != NULL) + if (fTokens != NULL) + { + for (unsigned int i = 0; i < fTotalRows; ++i) { - for (unsigned int i = 0; i < fTotalRows; ++i) - { - delete [] fTokens[i]; - } - - delete [] fTokens; - + delete[] fTokens[i]; } - fRowStatus.clear(); - fErrRows.clear(); + delete[] fTokens; + } + + fRowStatus.clear(); + fErrRows.clear(); } - + //------------------------------------------------------------------------------ // Resets state of buffer. //------------------------------------------------------------------------------ void BulkLoadBuffer::reset() { - fStartRow = fTotalReadRows = fTotalReadRowsForLog = 0; - fAutoIncGenCount = 0; + fStartRow = fTotalReadRows = fTotalReadRowsForLog = 0; + fAutoIncGenCount = 0; } //------------------------------------------------------------------------------ @@ -250,10 +248,10 @@ void BulkLoadBuffer::reset() //------------------------------------------------------------------------------ void BulkLoadBuffer::resetColumnLocks() { - fParseComplete = 0; + fParseComplete = 0; - struct LockInfo info; - fColumnLocks.assign(fNumberOfColumns, info); + struct LockInfo info; + fColumnLocks.assign(fNumberOfColumns, info); } //------------------------------------------------------------------------------ @@ -261,21 +259,21 @@ void BulkLoadBuffer::resetColumnLocks() //------------------------------------------------------------------------------ void BulkLoadBuffer::copyOverflow(const BulkLoadBuffer& buffer) { - if (fOverflowBuf != NULL) - { - delete [] fOverflowBuf; - fOverflowBuf = NULL; - } + if (fOverflowBuf != NULL) + { + delete[] fOverflowBuf; + fOverflowBuf = NULL; + } - fOverflowSize = buffer.fOverflowSize; + fOverflowSize = buffer.fOverflowSize; - if (fOverflowSize != 0) - { - fOverflowBuf = new char[buffer.fOverflowSize]; - memcpy( fOverflowBuf, buffer.fOverflowBuf, buffer.fOverflowSize ); - } + if (fOverflowSize != 0) + { + fOverflowBuf = new char[buffer.fOverflowSize]; + memcpy(fOverflowBuf, buffer.fOverflowBuf, buffer.fOverflowSize); + } } - + //------------------------------------------------------------------------------ // Parse/convert the given "field" value based on the specified length and type. // field (in) - the input field value to be parsed @@ -289,1616 +287,1563 @@ void BulkLoadBuffer::copyOverflow(const BulkLoadBuffer& buffer) // maxBufferVal (in/out) - ongoing max value for the Read buffer we are parsing // satCount (in/out) - ongoing saturation row count for buffer being parsed //------------------------------------------------------------------------------ -void BulkLoadBuffer::convert(char* field, int fieldLength, - bool nullFlag, unsigned char* output, const JobColumn& column, - BLBufferStats& bufStats) +void BulkLoadBuffer::convert(char* field, int fieldLength, bool nullFlag, unsigned char* output, + const JobColumn& column, BLBufferStats& bufStats) { - char biVal; - int iVal; - float fVal; - double dVal; - short siVal; - void* pVal; - int32_t iDate; - char charTmpBuf[MAX_COLUMN_BOUNDARY + 1] = {0}; - long long llVal = 0, llDate = 0; - int128_t bigllVal = 0; - uint64_t tmp64; - uint32_t tmp32; - uint8_t ubiVal; - uint16_t usiVal; - uint32_t uiVal; - uint64_t ullVal; + char biVal; + int iVal; + float fVal; + double dVal; + short siVal; + void* pVal; + int32_t iDate; + char charTmpBuf[MAX_COLUMN_BOUNDARY + 1] = {0}; + long long llVal = 0, llDate = 0; + int128_t bigllVal = 0; + uint64_t tmp64; + uint32_t tmp32; + uint8_t ubiVal; + uint16_t usiVal; + uint32_t uiVal; + uint64_t ullVal; - int width = column.width; + int width = column.width; - //-------------------------------------------------------------------------- - // Parse based on column data type - //-------------------------------------------------------------------------- - switch ( column.weType ) + //-------------------------------------------------------------------------- + // Parse based on column data type + //-------------------------------------------------------------------------- + switch (column.weType) + { + //---------------------------------------------------------------------- + // FLOAT + //---------------------------------------------------------------------- + case WriteEngine::WR_FLOAT: { - //---------------------------------------------------------------------- - // FLOAT - //---------------------------------------------------------------------- - case WriteEngine::WR_FLOAT : + if (nullFlag) + { + if (column.fWithDefault) { - if (nullFlag) - { - if (column.fWithDefault) - { - fVal = column.fDefaultDbl; - pVal = &fVal; - } - else - { - tmp32 = joblist::FLOATNULL; - pVal = &tmp32; - } - } + fVal = column.fDefaultDbl; + pVal = &fVal; + } + else + { + tmp32 = joblist::FLOATNULL; + pVal = &tmp32; + } + } + else + { + float minFltSat = column.fMinDblSat; + float maxFltSat = column.fMaxDblSat; + + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&fVal, field, sizeof(fVal)); + + if (isnan(fVal)) + { + if (signbit(fVal)) + fVal = minFltSat; else + fVal = maxFltSat; + + bufStats.satCount++; + } + else + { + if (fVal > maxFltSat) { - float minFltSat = column.fMinDblSat; - float maxFltSat = column.fMaxDblSat; - - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&fVal, field, sizeof(fVal)); - - if ( isnan(fVal) ) - { - if (signbit(fVal)) - fVal = minFltSat; - else - fVal = maxFltSat; - - bufStats.satCount++; - } - else - { - if ( fVal > maxFltSat ) - { - fVal = maxFltSat; - bufStats.satCount++; - } - else if ( fVal < minFltSat ) - { - fVal = minFltSat; - bufStats.satCount++; - } - } - } - else - { - errno = 0; + fVal = maxFltSat; + bufStats.satCount++; + } + else if (fVal < minFltSat) + { + fVal = minFltSat; + bufStats.satCount++; + } + } + } + else + { + errno = 0; #ifdef _MSC_VER - fVal = (float)strtod( field, 0 ); + fVal = (float)strtod(field, 0); #else - fVal = strtof( field, 0 ); + fVal = strtof(field, 0); #endif - if (errno == ERANGE) - { + if (errno == ERANGE) + { #ifdef _MSC_VER - if ( abs(fVal) == HUGE_VAL ) + if (abs(fVal) == HUGE_VAL) #else - if ( abs(fVal) == HUGE_VALF ) + if (abs(fVal) == HUGE_VALF) #endif - { - if ( fVal > 0 ) - fVal = maxFltSat; - else - fVal = minFltSat; - - bufStats.satCount++; - } - else - fVal = 0; - } - else - { - if ( fVal > maxFltSat ) - { - fVal = maxFltSat; - bufStats.satCount++; - } - else if ( fVal < minFltSat ) - { - fVal = minFltSat; - bufStats.satCount++; - } - if ( fVal == 0 - && isTrueWord(const_cast(field), fieldLength) ) - { - fVal = 1; - } - } - } - - pVal = &fVal; - } - - break; - } - - //---------------------------------------------------------------------- - // DOUBLE - //---------------------------------------------------------------------- - case WriteEngine::WR_DOUBLE : - { - if (nullFlag) { - if (column.fWithDefault) - { - dVal = column.fDefaultDbl; - pVal = &dVal; - } - else - { - tmp64 = joblist::DOUBLENULL; - pVal = &tmp64; - } + if (fVal > 0) + fVal = maxFltSat; + else + fVal = minFltSat; + + bufStats.satCount++; } else + fVal = 0; + } + else + { + if (fVal > maxFltSat) { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&dVal, field, sizeof(dVal)); - - if ( std::isnan(dVal) ) - { - if (signbit(dVal)) - dVal = column.fMinDblSat; - else - dVal = column.fMaxDblSat; - - bufStats.satCount++; - } - else - { - if ( dVal > column.fMaxDblSat ) - { - dVal = column.fMaxDblSat; - bufStats.satCount++; - } - else if ( dVal < column.fMinDblSat ) - { - dVal = column.fMinDblSat; - bufStats.satCount++; - } - } - } - else - { - errno = 0; - - dVal = strtod(field, 0); - - if (errno == ERANGE) - { -#ifdef _MSC_VER - - if ( abs(dVal) == HUGE_VAL ) -#else - if ( abs(dVal) == HUGE_VALL ) -#endif - { - if ( dVal > 0 ) - dVal = column.fMaxDblSat; - else - dVal = column.fMinDblSat; - - bufStats.satCount++; - } - else - dVal = 0; - } - else - { - if ( dVal > column.fMaxDblSat ) - { - dVal = column.fMaxDblSat; - bufStats.satCount++; - } - else if ( dVal < column.fMinDblSat ) - { - dVal = column.fMinDblSat; - bufStats.satCount++; - } - else if (dVal == 0 - && isTrueWord(const_cast(field), fieldLength)) - { - dVal = 1; - } - } - } - - pVal = &dVal; + fVal = maxFltSat; + bufStats.satCount++; } - - break; + else if (fVal < minFltSat) + { + fVal = minFltSat; + bufStats.satCount++; + } + if (fVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + fVal = 1; + } + } } - //---------------------------------------------------------------------- - // CHARACTER - //---------------------------------------------------------------------- - case WriteEngine::WR_CHAR : - { - if (nullFlag) - { - if (column.fWithDefault) - { - int defLen = column.fDefaultChr.size(); - const char* defData = column.fDefaultChr.c_str(); + pVal = &fVal; + } - if (defLen > column.definedWidth) - memcpy( charTmpBuf, defData, column.definedWidth ); - else - memcpy( charTmpBuf, defData, defLen ); - - // fall through to update saturation and min/max - } - else - { - idbassert(width <= 8); - - for (int i = 0; i < width - 1; i++) - { - charTmpBuf[i] = '\377'; - } - - charTmpBuf[width - 1] = '\376'; - - pVal = charTmpBuf; - break; - } - } - else - { - // truncate string if it is too long - // @Bug 3040. Use definedWidth for the data truncation to keep - // from storing characters beyond the column's defined width. - // It contains the column definition width rather than the bytes - // on disk (e.g. 5 for a varchar(5) instead of 8). - if (fieldLength > column.definedWidth) - { - uint8_t truncate_point = utf8::utf8_truncate_point(field, column.definedWidth); - memcpy( charTmpBuf, field, column.definedWidth - truncate_point ); - bufStats.satCount++; - } - else - memcpy( charTmpBuf, field, fieldLength ); - } - - // Swap byte order before comparing character string - // Compare must be unsigned - uint64_t compChar = uint64ToStr( *(reinterpret_cast(charTmpBuf)) ); - int64_t binChar = static_cast( compChar ); - - // Update min/max range - uint64_t minVal = static_cast( bufStats.minBufferVal ); - uint64_t maxVal = static_cast( bufStats.maxBufferVal ); - if (compChar < minVal) - bufStats.minBufferVal = binChar; - if (compChar > maxVal) - bufStats.maxBufferVal = binChar; - - pVal = charTmpBuf; - // cout << "In convert: fieldLength = " << fieldLength < static_cast(column.fMaxIntSat) ) - { - origVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - else if ( origVal == 0 - && isTrueWord(const_cast(field), fieldLength) ) - { - origVal = 1; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - if (origVal < bufStats.minBufferVal) - bufStats.minBufferVal = origVal; - - if (origVal > bufStats.maxBufferVal) - bufStats.maxBufferVal = origVal; - - siVal = origVal; - pVal = &siVal; - - break; - } - - //---------------------------------------------------------------------- - // UNSIGNED SHORT INT - //---------------------------------------------------------------------- - case WriteEngine::WR_USHORT : - { - int64_t origVal = 0; - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - origVal = static_cast(column.fDefaultUInt); - // fall through to update saturation and min/max - } - else - { - usiVal = joblist::USMALLINTNULL; - pVal = &usiVal; - break; - } - } - else - { - origVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - unsigned short int siVal2; - memcpy(&siVal2, field, sizeof(siVal2)); - origVal = siVal2; - } - else - { - errno = 0; - - origVal = strtoll(field, 0, 10); - - if (errno == ERANGE) - bSatVal = true; - } - } - - // Saturate the value (saturates any negative value to 0) - if ( origVal < column.fMinIntSat ) - { - origVal = column.fMinIntSat; - bSatVal = true; - } - else if ( origVal > static_cast(column.fMaxIntSat) ) - { - origVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - else if ( origVal == 0 - && isTrueWord(const_cast(field), fieldLength) ) - { - origVal = 1; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - uint64_t uVal = origVal; - - if (uVal < static_cast(bufStats.minBufferVal)) - bufStats.minBufferVal = origVal; - - if (uVal > static_cast(bufStats.maxBufferVal)) - bufStats.maxBufferVal = origVal; - - usiVal = origVal; - pVal = &usiVal; - - break; - } - - //---------------------------------------------------------------------- - // TINY INT - //---------------------------------------------------------------------- - case WriteEngine::WR_BYTE : - { - long long origVal; - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - origVal = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - biVal = joblist::TINYINTNULL; - pVal = &biVal; - break; - } - } - else - { - origVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - char biVal2; - memcpy(&biVal2, field, sizeof(biVal2)); - origVal = biVal2; - } - else - { - if (isTrueWord(const_cast(field), fieldLength)) - { - strcpy(field, "1"); - fieldLength = 1; - } - - if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) || - (column.dataType == CalpontSystemCatalog::UDECIMAL)) - { - // errno is initialized and set in convertDecimalString - origVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); - } - else - { - errno = 0; - origVal = strtol( field, 0, 10 ); - } - - if (errno == ERANGE) - bSatVal = true; - } - } - - // Saturate the value - if ( origVal < column.fMinIntSat ) - { - origVal = column.fMinIntSat; - bSatVal = true; - } - else if ( origVal > static_cast(column.fMaxIntSat) ) - { - origVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - if (origVal < bufStats.minBufferVal) - bufStats.minBufferVal = origVal; - - if (origVal > bufStats.maxBufferVal) - bufStats.maxBufferVal = origVal; - - biVal = origVal; - pVal = &biVal; - - break; - } - - //---------------------------------------------------------------------- - // UNSIGNED TINY INT - //---------------------------------------------------------------------- - case WriteEngine::WR_UBYTE : - { - int64_t origVal = 0; - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - origVal = static_cast(column.fDefaultUInt); - // fall through to update saturation and min/max - } - else - { - ubiVal = joblist::UTINYINTNULL; - pVal = &ubiVal; - break; - } - } - else - { - origVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - uint8_t biVal2; - memcpy(&biVal2, field, sizeof(biVal2)); - origVal = biVal2; - } - else - { - errno = 0; - - origVal = strtoll(field, 0, 10); - - if (errno == ERANGE) - bSatVal = true; - } - } - - // Saturate the value (saturates any negative value to 0) - if ( origVal < column.fMinIntSat ) - { - origVal = column.fMinIntSat; - bSatVal = true; - } - else if ( origVal > static_cast(column.fMaxIntSat) ) - { - origVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - else if ( origVal == 0 - && isTrueWord(const_cast(field), fieldLength) ) - { - origVal = 1; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - uint64_t uVal = origVal; - - if (uVal < static_cast(bufStats.minBufferVal)) - bufStats.minBufferVal = origVal; - - if (uVal > static_cast(bufStats.maxBufferVal)) - bufStats.maxBufferVal = origVal; - - ubiVal = origVal; - pVal = &ubiVal; - - break; - } - - //---------------------------------------------------------------------- - // BIG INT - //---------------------------------------------------------------------- - case WriteEngine::WR_LONGLONG: - { - bool bSatVal = false; - - if ( column.dataType != CalpontSystemCatalog::DATETIME && - column.dataType != CalpontSystemCatalog::TIMESTAMP && - column.dataType != CalpontSystemCatalog::TIME ) - { - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - llVal = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - llVal = joblist::BIGINTNULL; - pVal = &llVal; - break; - } - } - else - { - llVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&llVal, field, sizeof(llVal)); - } - else - { - if (isTrueWord(const_cast(field), fieldLength)) - { - strcpy(field, "1"); - fieldLength = 1; - } - - if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || - (column.dataType == CalpontSystemCatalog::UDECIMAL)) - { - // errno is initialized and set in convertDecimalString - llVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); - } - else - { - errno = 0; - llVal = strtoll( field, 0, 10 ); - } - } - - if (errno == ERANGE) - bSatVal = true; - } - - // Saturate the value - if ( llVal < column.fMinIntSat ) - { - llVal = column.fMinIntSat; - bSatVal = true; - } - else if ( llVal > static_cast(column.fMaxIntSat) ) - { - // llVal can be > fMaxIntSat if this is a decimal column - llVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - if (llVal < bufStats.minBufferVal) - bufStats.minBufferVal = llVal; - - if (llVal > bufStats.maxBufferVal) - bufStats.maxBufferVal = llVal; - - pVal = &llVal; - } - else if (column.dataType == CalpontSystemCatalog::TIME) - { - // time conversion - int rc = 0; - - if (nullFlag) - { - if (column.fWithDefault) - { - llDate = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - llDate = joblist::TIMENULL; - pVal = &llDate; - break; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&llDate, field, sizeof(llDate)); - - if (!dataconvert::DataConvert::isColumnTimeValid( - llDate)) - rc = -1; - } - else - { - llDate = dataconvert::DataConvert::convertColumnTime( - field, dataconvert::CALPONTTIME_ENUM, - rc, fieldLength ); - } - } - - if (rc == 0) - { - if (llDate < bufStats.minBufferVal) - bufStats.minBufferVal = llDate; - - if (llDate > bufStats.maxBufferVal) - bufStats.maxBufferVal = llDate; - } - else - { - bufStats.satCount++; - } - - pVal = &llDate; - } - else if (column.dataType == CalpontSystemCatalog::TIMESTAMP) - { - // timestamp conversion - int rc = 0; - - if (nullFlag) - { - if (column.fWithDefault) - { - llDate = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - llDate = joblist::TIMESTAMPNULL; - pVal = &llDate; - break; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&llDate, field, sizeof(llDate)); - - if (!dataconvert::DataConvert::isColumnTimeStampValid( - llDate)) - rc = -1; - } - else - { - llDate = dataconvert::DataConvert::convertColumnTimestamp( - field, dataconvert::CALPONTDATETIME_ENUM, - rc, fieldLength, fTimeZone ); - } - } - - if (rc == 0) - { - if (llDate < bufStats.minBufferVal) - bufStats.minBufferVal = llDate; - - if (llDate > bufStats.maxBufferVal) - bufStats.maxBufferVal = llDate; - } - else - { - llDate = 0; - bufStats.satCount++; - } - - pVal = &llDate; - } - else - { - // datetime conversion - int rc = 0; - - if (nullFlag) - { - if (column.fWithDefault) - { - llDate = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - llDate = joblist::DATETIMENULL; - pVal = &llDate; - break; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&llDate, field, sizeof(llDate)); - - if (!dataconvert::DataConvert::isColumnDateTimeValid( - llDate)) - rc = -1; - } - else - { - llDate = dataconvert::DataConvert::convertColumnDatetime( - field, dataconvert::CALPONTDATETIME_ENUM, - rc, fieldLength ); - } - } - - if (rc == 0) - { - if (llDate < bufStats.minBufferVal) - bufStats.minBufferVal = llDate; - - if (llDate > bufStats.maxBufferVal) - bufStats.maxBufferVal = llDate; - } - else - { - llDate = 0; - bufStats.satCount++; - } - - pVal = &llDate; - } - - break; - } - - //---------------------------------------------------------------------- - // WIDE DECIMAL - //---------------------------------------------------------------------- - case WriteEngine::WR_BINARY: - { - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - bigllVal = column.fDefaultWideDecimal; - // fall through to update saturation and min/max - } - else - { - bigllVal = datatypes::Decimal128Null; - pVal = &bigllVal; - break; - } - } - else - { - // TODO MCOL-641 Add support for int128_t version of - // fAutoIncNextValue - bigllVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&bigllVal, field, sizeof(bigllVal)); - } - else - { - if (isTrueWord(const_cast(field), fieldLength)) - { - strcpy(field, "1"); - fieldLength = 1; - } - - bool dummy = false; - // Value saturation to 9999... or -9999... is handled by - // number_int_value(), and the bSatVal flag is set to true - dataconvert::number_int_value(string(field), column.dataType, - datatypes::SystemCatalog::TypeAttributesStd( - column.width, column.scale, column.precision), - dummy, false, bigllVal, &bSatVal); - } - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - if (bigllVal < bufStats.bigMinBufferVal) - bufStats.bigMinBufferVal = bigllVal; - - if (bigllVal > bufStats.bigMaxBufferVal) - bufStats.bigMaxBufferVal = bigllVal; - - pVal = &bigllVal; - - break; - } - - //---------------------------------------------------------------------- - // UNSIGNED BIG INT - //---------------------------------------------------------------------- - case WriteEngine::WR_ULONGLONG: - { - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - ullVal = column.fDefaultUInt; - // fall through to update saturation and min/max - } - else - { - ullVal = joblist::UBIGINTNULL; - pVal = &ullVal; - break; - } - } - else - { - ullVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&ullVal, field, sizeof(ullVal)); - } - else - { - // Check for negative. strtoull doesn't do this for us. - // I considered using boost::trim_left here, but part of the - // exercise is to minimize cpu cycles, so I do it the old - // fashioned way. isspace() uses more cycles than direct - // compare to ' ', '\t', etc. but the payoff is that it - // works with Locale, so it ought to work well with utf-8 - // input. - int idx1; - - for (idx1 = 0; idx1 < fieldLength; idx1++) - { - if (!isspace(field[idx1])) - break; - } - - if ((idx1 < fieldLength) && (field[idx1] == '-')) - { - ullVal = static_cast(column.fMinIntSat); - bSatVal = true; - } - else - { - errno = 0; - - ullVal = strtoull(field, 0, 10); - - if (errno == ERANGE) - bSatVal = true; - } - } - } - - // Saturate the value - if ( ullVal > column.fMaxIntSat ) - { - ullVal = column.fMaxIntSat; - bSatVal = true; - } - else if ( ullVal == 0 - && isTrueWord(const_cast(field), fieldLength) ) - { - ullVal = 1; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - if (ullVal < static_cast(bufStats.minBufferVal)) - bufStats.minBufferVal = static_cast(ullVal); - - if (ullVal > static_cast(bufStats.maxBufferVal)) - bufStats.maxBufferVal = static_cast(ullVal); - - pVal = &ullVal; - break; - } - - //---------------------------------------------------------------------- - // UNSIGNED MEDIUM INTEGER AND UNSIGNED INTEGER - //---------------------------------------------------------------------- - case WriteEngine::WR_UMEDINT : - case WriteEngine::WR_UINT : - { - int64_t origVal; - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - origVal = static_cast(column.fDefaultUInt); - // fall through to update saturation and min/max - } - else - { - uiVal = joblist::UINTNULL; - pVal = &uiVal; - break; - } - } - else - { - origVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - unsigned int iVal2; - memcpy(&iVal2, field, sizeof(iVal2)); - origVal = iVal2; - } - else - { - errno = 0; - - origVal = strtoll(field, 0, 10); - - if (errno == ERANGE) - bSatVal = true; - } - } - - // Saturate the value (saturates any negative value to 0) - if ( origVal < column.fMinIntSat) - { - origVal = column.fMinIntSat; - bSatVal = true; - } - else if ( origVal > static_cast(column.fMaxIntSat) ) - { - origVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - else if ( origVal == 0 - && isTrueWord(const_cast(field), fieldLength) ) - { - origVal = 1; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - uint64_t uVal = origVal; - - if (uVal < static_cast(bufStats.minBufferVal)) - bufStats.minBufferVal = origVal; - - if (uVal > static_cast(bufStats.maxBufferVal)) - bufStats.maxBufferVal = origVal; - - uiVal = origVal; - pVal = &uiVal; - break; - } - - //---------------------------------------------------------------------- - // MEDIUM INTEGER AND INTEGER - //---------------------------------------------------------------------- - case WriteEngine::WR_MEDINT : - case WriteEngine::WR_INT : - default : - { - if ( column.dataType != CalpontSystemCatalog::DATE ) - { - long long origVal; - bool bSatVal = false; - - if (nullFlag) - { - if (!column.autoIncFlag) - { - if (column.fWithDefault) - { - origVal = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - iVal = joblist::INTNULL; - pVal = &iVal; - break; - } - } - else - { - origVal = fAutoIncNextValue++; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - int iVal2; - memcpy(&iVal2, field, sizeof(iVal2)); - origVal = iVal2; - } - else - { - if (isTrueWord(const_cast(field), fieldLength)) - { - strcpy(field, "1"); - fieldLength = 1; - } - - if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || - (column.dataType == CalpontSystemCatalog::UDECIMAL)) - { - // errno is initialized and set in convertDecimalString - origVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); - } - else - { - errno = 0; - origVal = strtol( field, 0, 10 ); - } - - if (errno == ERANGE) - bSatVal = true; - } - } - - // Saturate the value - if ( origVal < column.fMinIntSat ) - { - origVal = column.fMinIntSat; - bSatVal = true; - } - else if ( origVal > static_cast(column.fMaxIntSat) ) - { - origVal = static_cast(column.fMaxIntSat); - bSatVal = true; - } - - if (bSatVal) - bufStats.satCount++; - - // Update min/max range - if (origVal < bufStats.minBufferVal) - bufStats.minBufferVal = origVal; - - if (origVal > bufStats.maxBufferVal) - bufStats.maxBufferVal = origVal; - - iVal = (int)origVal; - pVal = &iVal; - } - else - { - // date conversion - int rc = 0; - - if (nullFlag) - { - if (column.fWithDefault) - { - iDate = column.fDefaultInt; - // fall through to update saturation and min/max - } - else - { - iDate = joblist::DATENULL; - pVal = &iDate; - break; - } - } - else - { - if (fImportDataMode != IMPORT_DATA_TEXT) - { - memcpy(&iDate, field, sizeof(iDate)); - - if (!dataconvert::DataConvert::isColumnDateValid(iDate)) - rc = -1; - } - else - { - iDate = dataconvert::DataConvert::convertColumnDate( - field, dataconvert::CALPONTDATE_ENUM, - rc, fieldLength ); - } - } - - if (rc == 0) - { - if (iDate < bufStats.minBufferVal) - bufStats.minBufferVal = iDate; - - if (iDate > bufStats.maxBufferVal) - bufStats.maxBufferVal = iDate; - } - else - { - iDate = 0; - bufStats.satCount++; - } - - pVal = &iDate; - } - - break; - } + break; } - memcpy(output, pVal, width); + //---------------------------------------------------------------------- + // DOUBLE + //---------------------------------------------------------------------- + case WriteEngine::WR_DOUBLE: + { + if (nullFlag) + { + if (column.fWithDefault) + { + dVal = column.fDefaultDbl; + pVal = &dVal; + } + else + { + tmp64 = joblist::DOUBLENULL; + pVal = &tmp64; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&dVal, field, sizeof(dVal)); + + if (std::isnan(dVal)) + { + if (signbit(dVal)) + dVal = column.fMinDblSat; + else + dVal = column.fMaxDblSat; + + bufStats.satCount++; + } + else + { + if (dVal > column.fMaxDblSat) + { + dVal = column.fMaxDblSat; + bufStats.satCount++; + } + else if (dVal < column.fMinDblSat) + { + dVal = column.fMinDblSat; + bufStats.satCount++; + } + } + } + else + { + errno = 0; + + dVal = strtod(field, 0); + + if (errno == ERANGE) + { +#ifdef _MSC_VER + + if (abs(dVal) == HUGE_VAL) +#else + if (abs(dVal) == HUGE_VALL) +#endif + { + if (dVal > 0) + dVal = column.fMaxDblSat; + else + dVal = column.fMinDblSat; + + bufStats.satCount++; + } + else + dVal = 0; + } + else + { + if (dVal > column.fMaxDblSat) + { + dVal = column.fMaxDblSat; + bufStats.satCount++; + } + else if (dVal < column.fMinDblSat) + { + dVal = column.fMinDblSat; + bufStats.satCount++; + } + else if (dVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + dVal = 1; + } + } + } + + pVal = &dVal; + } + + break; + } + + //---------------------------------------------------------------------- + // CHARACTER + //---------------------------------------------------------------------- + case WriteEngine::WR_CHAR: + { + if (nullFlag) + { + if (column.fWithDefault) + { + int defLen = column.fDefaultChr.size(); + const char* defData = column.fDefaultChr.c_str(); + + if (defLen > column.definedWidth) + memcpy(charTmpBuf, defData, column.definedWidth); + else + memcpy(charTmpBuf, defData, defLen); + + // fall through to update saturation and min/max + } + else + { + idbassert(width <= 8); + + for (int i = 0; i < width - 1; i++) + { + charTmpBuf[i] = '\377'; + } + + charTmpBuf[width - 1] = '\376'; + + pVal = charTmpBuf; + break; + } + } + else + { + // truncate string if it is too long + // @Bug 3040. Use definedWidth for the data truncation to keep + // from storing characters beyond the column's defined width. + // It contains the column definition width rather than the bytes + // on disk (e.g. 5 for a varchar(5) instead of 8). + if (fieldLength > column.definedWidth) + { + uint8_t truncate_point = utf8::utf8_truncate_point(field, column.definedWidth); + memcpy(charTmpBuf, field, column.definedWidth - truncate_point); + bufStats.satCount++; + } + else + memcpy(charTmpBuf, field, fieldLength); + } + + // Swap byte order before comparing character string + // Compare must be unsigned + uint64_t compChar = uint64ToStr(*(reinterpret_cast(charTmpBuf))); + int64_t binChar = static_cast(compChar); + + // Update min/max range + uint64_t minVal = static_cast(bufStats.minBufferVal); + uint64_t maxVal = static_cast(bufStats.maxBufferVal); + if (compChar < minVal) + bufStats.minBufferVal = binChar; + if (compChar > maxVal) + bufStats.maxBufferVal = binChar; + + pVal = charTmpBuf; + // cout << "In convert: fieldLength = " << fieldLength < static_cast(column.fMaxIntSat)) + { + origVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + else if (origVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + origVal = 1; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + if (origVal < bufStats.minBufferVal) + bufStats.minBufferVal = origVal; + + if (origVal > bufStats.maxBufferVal) + bufStats.maxBufferVal = origVal; + + siVal = origVal; + pVal = &siVal; + + break; + } + + //---------------------------------------------------------------------- + // UNSIGNED SHORT INT + //---------------------------------------------------------------------- + case WriteEngine::WR_USHORT: + { + int64_t origVal = 0; + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + origVal = static_cast(column.fDefaultUInt); + // fall through to update saturation and min/max + } + else + { + usiVal = joblist::USMALLINTNULL; + pVal = &usiVal; + break; + } + } + else + { + origVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + unsigned short int siVal2; + memcpy(&siVal2, field, sizeof(siVal2)); + origVal = siVal2; + } + else + { + errno = 0; + + origVal = strtoll(field, 0, 10); + + if (errno == ERANGE) + bSatVal = true; + } + } + + // Saturate the value (saturates any negative value to 0) + if (origVal < column.fMinIntSat) + { + origVal = column.fMinIntSat; + bSatVal = true; + } + else if (origVal > static_cast(column.fMaxIntSat)) + { + origVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + else if (origVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + origVal = 1; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + uint64_t uVal = origVal; + + if (uVal < static_cast(bufStats.minBufferVal)) + bufStats.minBufferVal = origVal; + + if (uVal > static_cast(bufStats.maxBufferVal)) + bufStats.maxBufferVal = origVal; + + usiVal = origVal; + pVal = &usiVal; + + break; + } + + //---------------------------------------------------------------------- + // TINY INT + //---------------------------------------------------------------------- + case WriteEngine::WR_BYTE: + { + long long origVal; + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + origVal = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + biVal = joblist::TINYINTNULL; + pVal = &biVal; + break; + } + } + else + { + origVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + char biVal2; + memcpy(&biVal2, field, sizeof(biVal2)); + origVal = biVal2; + } + else + { + if (isTrueWord(const_cast(field), fieldLength)) + { + strcpy(field, "1"); + fieldLength = 1; + } + + if ((column.dataType == CalpontSystemCatalog::DECIMAL) || + (column.dataType == CalpontSystemCatalog::UDECIMAL)) + { + // errno is initialized and set in convertDecimalString + origVal = Convertor::convertDecimalString(field, fieldLength, column.scale); + } + else + { + errno = 0; + origVal = strtol(field, 0, 10); + } + + if (errno == ERANGE) + bSatVal = true; + } + } + + // Saturate the value + if (origVal < column.fMinIntSat) + { + origVal = column.fMinIntSat; + bSatVal = true; + } + else if (origVal > static_cast(column.fMaxIntSat)) + { + origVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + if (origVal < bufStats.minBufferVal) + bufStats.minBufferVal = origVal; + + if (origVal > bufStats.maxBufferVal) + bufStats.maxBufferVal = origVal; + + biVal = origVal; + pVal = &biVal; + + break; + } + + //---------------------------------------------------------------------- + // UNSIGNED TINY INT + //---------------------------------------------------------------------- + case WriteEngine::WR_UBYTE: + { + int64_t origVal = 0; + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + origVal = static_cast(column.fDefaultUInt); + // fall through to update saturation and min/max + } + else + { + ubiVal = joblist::UTINYINTNULL; + pVal = &ubiVal; + break; + } + } + else + { + origVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + uint8_t biVal2; + memcpy(&biVal2, field, sizeof(biVal2)); + origVal = biVal2; + } + else + { + errno = 0; + + origVal = strtoll(field, 0, 10); + + if (errno == ERANGE) + bSatVal = true; + } + } + + // Saturate the value (saturates any negative value to 0) + if (origVal < column.fMinIntSat) + { + origVal = column.fMinIntSat; + bSatVal = true; + } + else if (origVal > static_cast(column.fMaxIntSat)) + { + origVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + else if (origVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + origVal = 1; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + uint64_t uVal = origVal; + + if (uVal < static_cast(bufStats.minBufferVal)) + bufStats.minBufferVal = origVal; + + if (uVal > static_cast(bufStats.maxBufferVal)) + bufStats.maxBufferVal = origVal; + + ubiVal = origVal; + pVal = &ubiVal; + + break; + } + + //---------------------------------------------------------------------- + // BIG INT + //---------------------------------------------------------------------- + case WriteEngine::WR_LONGLONG: + { + bool bSatVal = false; + + if (column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIMESTAMP && column.dataType != CalpontSystemCatalog::TIME) + { + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + llVal = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llVal = joblist::BIGINTNULL; + pVal = &llVal; + break; + } + } + else + { + llVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llVal, field, sizeof(llVal)); + } + else + { + if (isTrueWord(const_cast(field), fieldLength)) + { + strcpy(field, "1"); + fieldLength = 1; + } + + if ((column.dataType == CalpontSystemCatalog::DECIMAL) || + (column.dataType == CalpontSystemCatalog::UDECIMAL)) + { + // errno is initialized and set in convertDecimalString + llVal = Convertor::convertDecimalString(field, fieldLength, column.scale); + } + else + { + errno = 0; + llVal = strtoll(field, 0, 10); + } + } + + if (errno == ERANGE) + bSatVal = true; + } + + // Saturate the value + if (llVal < column.fMinIntSat) + { + llVal = column.fMinIntSat; + bSatVal = true; + } + else if (llVal > static_cast(column.fMaxIntSat)) + { + // llVal can be > fMaxIntSat if this is a decimal column + llVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + if (llVal < bufStats.minBufferVal) + bufStats.minBufferVal = llVal; + + if (llVal > bufStats.maxBufferVal) + bufStats.maxBufferVal = llVal; + + pVal = &llVal; + } + else if (column.dataType == CalpontSystemCatalog::TIME) + { + // time conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMENULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeValid(llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTime(field, dataconvert::CALPONTTIME_ENUM, rc, + fieldLength); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + bufStats.satCount++; + } + + pVal = &llDate; + } + else if (column.dataType == CalpontSystemCatalog::TIMESTAMP) + { + // timestamp conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMESTAMPNULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeStampValid(llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTimestamp( + field, dataconvert::CALPONTDATETIME_ENUM, rc, fieldLength, fTimeZone); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + llDate = 0; + bufStats.satCount++; + } + + pVal = &llDate; + } + else + { + // datetime conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::DATETIMENULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnDateTimeValid(llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnDatetime(field, dataconvert::CALPONTDATETIME_ENUM, + rc, fieldLength); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + llDate = 0; + bufStats.satCount++; + } + + pVal = &llDate; + } + + break; + } + + //---------------------------------------------------------------------- + // WIDE DECIMAL + //---------------------------------------------------------------------- + case WriteEngine::WR_BINARY: + { + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + bigllVal = column.fDefaultWideDecimal; + // fall through to update saturation and min/max + } + else + { + bigllVal = datatypes::Decimal128Null; + pVal = &bigllVal; + break; + } + } + else + { + // TODO MCOL-641 Add support for int128_t version of + // fAutoIncNextValue + bigllVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&bigllVal, field, sizeof(bigllVal)); + } + else + { + if (isTrueWord(const_cast(field), fieldLength)) + { + strcpy(field, "1"); + fieldLength = 1; + } + + bool dummy = false; + // Value saturation to 9999... or -9999... is handled by + // number_int_value(), and the bSatVal flag is set to true + dataconvert::number_int_value( + string(field), column.dataType, + datatypes::SystemCatalog::TypeAttributesStd(column.width, column.scale, column.precision), + dummy, false, bigllVal, &bSatVal); + } + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + if (bigllVal < bufStats.bigMinBufferVal) + bufStats.bigMinBufferVal = bigllVal; + + if (bigllVal > bufStats.bigMaxBufferVal) + bufStats.bigMaxBufferVal = bigllVal; + + pVal = &bigllVal; + + break; + } + + //---------------------------------------------------------------------- + // UNSIGNED BIG INT + //---------------------------------------------------------------------- + case WriteEngine::WR_ULONGLONG: + { + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + ullVal = column.fDefaultUInt; + // fall through to update saturation and min/max + } + else + { + ullVal = joblist::UBIGINTNULL; + pVal = &ullVal; + break; + } + } + else + { + ullVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&ullVal, field, sizeof(ullVal)); + } + else + { + // Check for negative. strtoull doesn't do this for us. + // I considered using boost::trim_left here, but part of the + // exercise is to minimize cpu cycles, so I do it the old + // fashioned way. isspace() uses more cycles than direct + // compare to ' ', '\t', etc. but the payoff is that it + // works with Locale, so it ought to work well with utf-8 + // input. + int idx1; + + for (idx1 = 0; idx1 < fieldLength; idx1++) + { + if (!isspace(field[idx1])) + break; + } + + if ((idx1 < fieldLength) && (field[idx1] == '-')) + { + ullVal = static_cast(column.fMinIntSat); + bSatVal = true; + } + else + { + errno = 0; + + ullVal = strtoull(field, 0, 10); + + if (errno == ERANGE) + bSatVal = true; + } + } + } + + // Saturate the value + if (ullVal > column.fMaxIntSat) + { + ullVal = column.fMaxIntSat; + bSatVal = true; + } + else if (ullVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + ullVal = 1; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + if (ullVal < static_cast(bufStats.minBufferVal)) + bufStats.minBufferVal = static_cast(ullVal); + + if (ullVal > static_cast(bufStats.maxBufferVal)) + bufStats.maxBufferVal = static_cast(ullVal); + + pVal = &ullVal; + break; + } + + //---------------------------------------------------------------------- + // UNSIGNED MEDIUM INTEGER AND UNSIGNED INTEGER + //---------------------------------------------------------------------- + case WriteEngine::WR_UMEDINT: + case WriteEngine::WR_UINT: + { + int64_t origVal; + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + origVal = static_cast(column.fDefaultUInt); + // fall through to update saturation and min/max + } + else + { + uiVal = joblist::UINTNULL; + pVal = &uiVal; + break; + } + } + else + { + origVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + unsigned int iVal2; + memcpy(&iVal2, field, sizeof(iVal2)); + origVal = iVal2; + } + else + { + errno = 0; + + origVal = strtoll(field, 0, 10); + + if (errno == ERANGE) + bSatVal = true; + } + } + + // Saturate the value (saturates any negative value to 0) + if (origVal < column.fMinIntSat) + { + origVal = column.fMinIntSat; + bSatVal = true; + } + else if (origVal > static_cast(column.fMaxIntSat)) + { + origVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + else if (origVal == 0 && isTrueWord(const_cast(field), fieldLength)) + { + origVal = 1; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + uint64_t uVal = origVal; + + if (uVal < static_cast(bufStats.minBufferVal)) + bufStats.minBufferVal = origVal; + + if (uVal > static_cast(bufStats.maxBufferVal)) + bufStats.maxBufferVal = origVal; + + uiVal = origVal; + pVal = &uiVal; + break; + } + + //---------------------------------------------------------------------- + // MEDIUM INTEGER AND INTEGER + //---------------------------------------------------------------------- + case WriteEngine::WR_MEDINT: + case WriteEngine::WR_INT: + default: + { + if (column.dataType != CalpontSystemCatalog::DATE) + { + long long origVal; + bool bSatVal = false; + + if (nullFlag) + { + if (!column.autoIncFlag) + { + if (column.fWithDefault) + { + origVal = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + iVal = joblist::INTNULL; + pVal = &iVal; + break; + } + } + else + { + origVal = fAutoIncNextValue++; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + int iVal2; + memcpy(&iVal2, field, sizeof(iVal2)); + origVal = iVal2; + } + else + { + if (isTrueWord(const_cast(field), fieldLength)) + { + strcpy(field, "1"); + fieldLength = 1; + } + + if ((column.dataType == CalpontSystemCatalog::DECIMAL) || + (column.dataType == CalpontSystemCatalog::UDECIMAL)) + { + // errno is initialized and set in convertDecimalString + origVal = Convertor::convertDecimalString(field, fieldLength, column.scale); + } + else + { + errno = 0; + origVal = strtol(field, 0, 10); + } + + if (errno == ERANGE) + bSatVal = true; + } + } + + // Saturate the value + if (origVal < column.fMinIntSat) + { + origVal = column.fMinIntSat; + bSatVal = true; + } + else if (origVal > static_cast(column.fMaxIntSat)) + { + origVal = static_cast(column.fMaxIntSat); + bSatVal = true; + } + + if (bSatVal) + bufStats.satCount++; + + // Update min/max range + if (origVal < bufStats.minBufferVal) + bufStats.minBufferVal = origVal; + + if (origVal > bufStats.maxBufferVal) + bufStats.maxBufferVal = origVal; + + iVal = (int)origVal; + pVal = &iVal; + } + else + { + // date conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + iDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + iDate = joblist::DATENULL; + pVal = &iDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&iDate, field, sizeof(iDate)); + + if (!dataconvert::DataConvert::isColumnDateValid(iDate)) + rc = -1; + } + else + { + iDate = dataconvert::DataConvert::convertColumnDate(field, dataconvert::CALPONTDATE_ENUM, rc, + fieldLength); + } + } + + if (rc == 0) + { + if (iDate < bufStats.minBufferVal) + bufStats.minBufferVal = iDate; + + if (iDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = iDate; + } + else + { + iDate = 0; + bufStats.satCount++; + } + + pVal = &iDate; + } + + break; + } + } + + memcpy(output, pVal, width); } - + //------------------------------------------------------------------------------ // Parse the contents of the Read buffer based on whether it is a dictionary // column or not. //------------------------------------------------------------------------------ -int BulkLoadBuffer::parse(ColumnInfo& columnInfo) +int BulkLoadBuffer::parse(ColumnInfo& columnInfo) { - int rc = NO_ERROR; + int rc = NO_ERROR; - // Rather than locking fSyncUpdatesBLB for the entire life of parse(), - // we only briefly lock, and force a synchronization with the relevant - // class variables from reader threads (by copying to Parser specific - // variables). It should be okay to reference a copy of these variables - // as no other thread should be changing them while we are in parse(). - { - boost::mutex::scoped_lock lock(fSyncUpdatesBLB); - fTotalReadRowsParser = fTotalReadRows; - fStartRowParser = fStartRow; - fDataParser = fData; - fTokensParser = fTokens; - fStartRowForLoggingParser = fStartRowForLogging; - fAutoIncGenCountParser = fAutoIncGenCount; - } - - //Bug806 - If buffer is empty then return early. - if ( fTotalReadRowsParser == 0 ) - return rc; - - // If this is the first batch of rows, create the starting DB file - // if this PM did not have a DB file (delayed file creation). - RETURN_ON_ERROR( columnInfo.createDelayedFileIfNeeded(fTableName) ); - - if (columnInfo.column.colType == COL_TYPE_DICT) - rc = parseDict(columnInfo); - else - rc = parseCol(columnInfo); + // Rather than locking fSyncUpdatesBLB for the entire life of parse(), + // we only briefly lock, and force a synchronization with the relevant + // class variables from reader threads (by copying to Parser specific + // variables). It should be okay to reference a copy of these variables + // as no other thread should be changing them while we are in parse(). + { + boost::mutex::scoped_lock lock(fSyncUpdatesBLB); + fTotalReadRowsParser = fTotalReadRows; + fStartRowParser = fStartRow; + fDataParser = fData; + fTokensParser = fTokens; + fStartRowForLoggingParser = fStartRowForLogging; + fAutoIncGenCountParser = fAutoIncGenCount; + } + // Bug806 - If buffer is empty then return early. + if (fTotalReadRowsParser == 0) return rc; + + // If this is the first batch of rows, create the starting DB file + // if this PM did not have a DB file (delayed file creation). + RETURN_ON_ERROR(columnInfo.createDelayedFileIfNeeded(fTableName)); + + if (columnInfo.column.colType == COL_TYPE_DICT) + rc = parseDict(columnInfo); + else + rc = parseCol(columnInfo); + + return rc; } - + //------------------------------------------------------------------------------ // Parse nonDictionary column Read buffer. Parsed row values are added to // fColBufferMgr, which stores them into an output buffer before writing them // out to the applicable column segment file. //------------------------------------------------------------------------------ -int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) +int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) { - int rc = NO_ERROR; + int rc = NO_ERROR; - // Parse the data and fill up a buffer; which is written to output file - uint32_t nRowsParsed; + // Parse the data and fill up a buffer; which is written to output file + uint32_t nRowsParsed; - if (fLog->isDebug( DEBUG_2 )) + if (fLog->isDebug(DEBUG_2)) + { + ostringstream oss; + oss << "ColResSecIn: OID-" << columnInfo.column.mapOid << "; StartRID/Rows: " << fStartRowParser << " " + << fTotalReadRowsParser; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + ColumnBufferSection* section = 0; + RID lastInputRowInExtent; + RETURN_ON_ERROR(columnInfo.fColBufferMgr->reserveSection(fStartRowParser, fTotalReadRowsParser, nRowsParsed, + §ion, lastInputRowInExtent)); + + if (nRowsParsed > 0) + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_PARSE_COL); +#endif + + // Reserve auto-increment numbers we need to generate + if ((columnInfo.column.autoIncFlag) && (fAutoIncGenCountParser > 0)) { + rc = columnInfo.reserveAutoIncNums(fAutoIncGenCountParser, fAutoIncNextValue); + + if (rc != NO_ERROR) + { + WErrorCodes ec; ostringstream oss; - oss << "ColResSecIn: OID-" << columnInfo.column.mapOid << - "; StartRID/Rows: " << fStartRowParser << " " << - fTotalReadRowsParser; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + oss << "parseCol: error generating auto-increment values " + "for table-" + << fTableName << ", column-" << columnInfo.column.colName << "; OID-" << columnInfo.column.mapOid + << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); + return rc; + } } - ColumnBufferSection* section = 0; - RID lastInputRowInExtent; - RETURN_ON_ERROR( columnInfo.fColBufferMgr->reserveSection( - fStartRowParser, fTotalReadRowsParser, nRowsParsed, - §ion, lastInputRowInExtent ) ); + // create a buffer for the size of the rows being written. + unsigned char* buf = new unsigned char[fTotalReadRowsParser * columnInfo.column.width]; + char* field = new char[MAX_FIELD_SIZE + 1]; - if (nRowsParsed > 0) + // Initialize min/max buffer values. We initialize to a sufficient + // range to force the first value to automatically update the range. + // If we are managing char data, minBufferVal and maxBufferVal are + // maintained in reverse byte order to facilitate string comparisons + BLBufferStats bufStats(columnInfo.column.dataType); + bool updateCPInfoPendingFlag = false; + + int tokenLength = 0; + bool tokenNullFlag = false; + + for (uint32_t i = 0; i < fTotalReadRowsParser; ++i) { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_PARSE_COL); -#endif + char* p = fDataParser + fTokensParser[i][columnInfo.id].start; - // Reserve auto-increment numbers we need to generate - if ((columnInfo.column.autoIncFlag) && - (fAutoIncGenCountParser > 0)) + if (fTokensParser[i][columnInfo.id].offset > 0) + { + memcpy(field, p, fTokensParser[i][columnInfo.id].offset); + field[fTokensParser[i][columnInfo.id].offset] = '\0'; + tokenLength = fTokensParser[i][columnInfo.id].offset; + tokenNullFlag = false; + } + else + { + field[0] = '\0'; + tokenLength = 0; + tokenNullFlag = true; + } + + // convert the data into appropriate format and update CP values + convert(field, tokenLength, tokenNullFlag, buf + i * columnInfo.column.width, columnInfo.column, + bufStats); + updateCPInfoPendingFlag = true; + + // Update CP min/max if this is last row in this extent + if ((fStartRowParser + i) == lastInputRowInExtent) + { + if (columnInfo.column.width <= 8) { - rc = columnInfo.reserveAutoIncNums( fAutoIncGenCountParser, - fAutoIncNextValue ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "parseCol: error generating auto-increment values " - "for table-" << fTableName << - ", column-" << columnInfo.column.colName << - "; OID-" << columnInfo.column.mapOid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); - return rc; - } + columnInfo.updateCPInfo(lastInputRowInExtent, bufStats.minBufferVal, bufStats.maxBufferVal, + columnInfo.column.dataType, columnInfo.column.width); } - - // create a buffer for the size of the rows being written. - unsigned char* buf = new unsigned char[fTotalReadRowsParser* - columnInfo.column.width]; - char* field = new char[MAX_FIELD_SIZE + 1]; - - // Initialize min/max buffer values. We initialize to a sufficient - // range to force the first value to automatically update the range. - // If we are managing char data, minBufferVal and maxBufferVal are - // maintained in reverse byte order to facilitate string comparisons - BLBufferStats bufStats(columnInfo.column.dataType); - bool updateCPInfoPendingFlag = false; - - int tokenLength = 0; - bool tokenNullFlag = false; - - for (uint32_t i = 0; i < fTotalReadRowsParser; ++i) + else { - char* p = fDataParser + fTokensParser[i][columnInfo.id].start; - - if ( fTokensParser[i][columnInfo.id].offset > 0) - { - memcpy( field, p, fTokensParser[i][columnInfo.id].offset ); - field[fTokensParser[i][columnInfo.id].offset] = '\0'; - tokenLength = fTokensParser[i][columnInfo.id].offset; - tokenNullFlag = false; - } - else - { - field[0] = '\0'; - tokenLength = 0; - tokenNullFlag = true; - } - - // convert the data into appropriate format and update CP values - convert(field, tokenLength, tokenNullFlag, - buf + i * columnInfo.column.width, - columnInfo.column, bufStats); - updateCPInfoPendingFlag = true; - - // Update CP min/max if this is last row in this extent - if ( (fStartRowParser + i) == lastInputRowInExtent ) - { - if (columnInfo.column.width <= 8) - { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.minBufferVal, - bufStats.maxBufferVal, - columnInfo.column.dataType, - columnInfo.column.width ); - } - else - { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.bigMinBufferVal, - bufStats.bigMaxBufferVal, - columnInfo.column.dataType, - columnInfo.column.width ); - } - - // TODO MCOL-641 Add support here. - if (fLog->isDebug( DEBUG_2 )) - { - ostringstream oss; - oss << "ColRelSecOut: OID-" << columnInfo.column.mapOid - << "; StartRID/Rows1: " << section->startRowId() - << " " << i + 1 - << "; lastExtentRow: " << lastInputRowInExtent; - parseColLogMinMax( oss, - columnInfo.column.dataType, - bufStats.minBufferVal, - bufStats.maxBufferVal ); - - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - lastInputRowInExtent += columnInfo.rowsPerExtent(); - - if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) - { - if (columnInfo.column.width <= 8) - { - bufStats.minBufferVal = static_cast(MAX_UBIGINT); - bufStats.maxBufferVal = static_cast(MIN_UBIGINT); - } - else - { - bufStats.bigMinBufferVal = -1; - bufStats.bigMaxBufferVal = 0; - } - updateCPInfoPendingFlag = false; - } - else - { - if (columnInfo.column.width <= 8) - { - bufStats.minBufferVal = MAX_BIGINT; - bufStats.maxBufferVal = MIN_BIGINT; - } - else - { - utils::int128Max(bufStats.bigMinBufferVal); - utils::int128Min(bufStats.bigMaxBufferVal); - } - updateCPInfoPendingFlag = false; - } - } + columnInfo.updateCPInfo(lastInputRowInExtent, bufStats.bigMinBufferVal, bufStats.bigMaxBufferVal, + columnInfo.column.dataType, columnInfo.column.width); } - if (updateCPInfoPendingFlag) - { - if (columnInfo.column.width <= 8) - { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.minBufferVal, - bufStats.maxBufferVal, - columnInfo.column.dataType, - columnInfo.column.width ); - } - else - { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.bigMinBufferVal, - bufStats.bigMaxBufferVal, - columnInfo.column.dataType, - columnInfo.column.width ); - } - } - - if (bufStats.satCount) // @bug 3504: increment row saturation count - { - // If we don't want to allow saturated values for auto inc columns. - // then this is where we handle it. Too late to reject a single - // row from the parsing thread, so we abort the job. - //if (columnInfo.column.autoIncFlag) - //{ - // rc = ERR_AUTOINC_USER_OUT_OF_RANGE; - // WErrorCodes ec; - // ostringstream oss; - // oss << "parseCol: error with auto-increment values " - // "for table-" << fTableName << - // ", column-" << columnInfo.column.colName << - // "; OID-" << columnInfo.column.mapOid << - // "; " << ec.errorString(rc); - // fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - // return rc; - //} - columnInfo.incSaturatedCnt( bufStats.satCount ); - } - - delete [] field; - section->write(buf, fTotalReadRowsParser); - delete [] buf; -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_PARSE_COL); -#endif - // TODO MCOL-641 Add support here. - if (fLog->isDebug( DEBUG_2 )) + if (fLog->isDebug(DEBUG_2)) { - ostringstream oss; - RID rid1 = section->startRowId(); - RID rid2 = section->endRowId(); - oss << "ColRelSecOut: OID-" << columnInfo.column.mapOid << - "; StartRID/Rows2: " << rid1 << " " << (rid2 - rid1) + 1 << - "; startOffset: " << section->getStartOffset() << - "; lastExtentRow: " << lastInputRowInExtent; - parseColLogMinMax( oss, - columnInfo.column.dataType, - bufStats.minBufferVal, - bufStats.maxBufferVal ); + ostringstream oss; + oss << "ColRelSecOut: OID-" << columnInfo.column.mapOid + << "; StartRID/Rows1: " << section->startRowId() << " " << i + 1 + << "; lastExtentRow: " << lastInputRowInExtent; + parseColLogMinMax(oss, columnInfo.column.dataType, bufStats.minBufferVal, bufStats.maxBufferVal); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + fLog->logMsg(oss.str(), MSGLVL_INFO2); } - RETURN_ON_ERROR(columnInfo.fColBufferMgr->releaseSection(section)); + lastInputRowInExtent += columnInfo.rowsPerExtent(); + + if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) + { + if (columnInfo.column.width <= 8) + { + bufStats.minBufferVal = static_cast(MAX_UBIGINT); + bufStats.maxBufferVal = static_cast(MIN_UBIGINT); + } + else + { + bufStats.bigMinBufferVal = -1; + bufStats.bigMaxBufferVal = 0; + } + updateCPInfoPendingFlag = false; + } + else + { + if (columnInfo.column.width <= 8) + { + bufStats.minBufferVal = MAX_BIGINT; + bufStats.maxBufferVal = MIN_BIGINT; + } + else + { + utils::int128Max(bufStats.bigMinBufferVal); + utils::int128Min(bufStats.bigMaxBufferVal); + } + updateCPInfoPendingFlag = false; + } + } } - return rc; + if (updateCPInfoPendingFlag) + { + if (columnInfo.column.width <= 8) + { + columnInfo.updateCPInfo(lastInputRowInExtent, bufStats.minBufferVal, bufStats.maxBufferVal, + columnInfo.column.dataType, columnInfo.column.width); + } + else + { + columnInfo.updateCPInfo(lastInputRowInExtent, bufStats.bigMinBufferVal, bufStats.bigMaxBufferVal, + columnInfo.column.dataType, columnInfo.column.width); + } + } + + if (bufStats.satCount) // @bug 3504: increment row saturation count + { + // If we don't want to allow saturated values for auto inc columns. + // then this is where we handle it. Too late to reject a single + // row from the parsing thread, so we abort the job. + // if (columnInfo.column.autoIncFlag) + //{ + // rc = ERR_AUTOINC_USER_OUT_OF_RANGE; + // WErrorCodes ec; + // ostringstream oss; + // oss << "parseCol: error with auto-increment values " + // "for table-" << fTableName << + // ", column-" << columnInfo.column.colName << + // "; OID-" << columnInfo.column.mapOid << + // "; " << ec.errorString(rc); + // fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); + // return rc; + //} + columnInfo.incSaturatedCnt(bufStats.satCount); + } + + delete[] field; + section->write(buf, fTotalReadRowsParser); + delete[] buf; +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_PARSE_COL); +#endif + + // TODO MCOL-641 Add support here. + if (fLog->isDebug(DEBUG_2)) + { + ostringstream oss; + RID rid1 = section->startRowId(); + RID rid2 = section->endRowId(); + oss << "ColRelSecOut: OID-" << columnInfo.column.mapOid << "; StartRID/Rows2: " << rid1 << " " + << (rid2 - rid1) + 1 << "; startOffset: " << section->getStartOffset() + << "; lastExtentRow: " << lastInputRowInExtent; + parseColLogMinMax(oss, columnInfo.column.dataType, bufStats.minBufferVal, bufStats.maxBufferVal); + + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + RETURN_ON_ERROR(columnInfo.fColBufferMgr->releaseSection(section)); + } + + return rc; } - + //------------------------------------------------------------------------------ // Log the specified min/max buffer values to the log file. This is straight // forward for numeric types, but for character data, we have to reverse the // order of min/max values, because they are maintained in reverse order to // facilitate the comparison of character strings in an int64_t variable. //------------------------------------------------------------------------------ -void BulkLoadBuffer::parseColLogMinMax( - ostringstream& oss, - ColDataType colDataType, - int64_t minBufferVal, - int64_t maxBufferVal ) const +void BulkLoadBuffer::parseColLogMinMax(ostringstream& oss, ColDataType colDataType, int64_t minBufferVal, + int64_t maxBufferVal) const { - if (isCharType(colDataType)) - { - // Swap/restore byte order before printing character string - int64_t minVal = static_cast( uint64ToStr( - static_cast(minBufferVal) ) ); - int64_t maxVal = static_cast( uint64ToStr( - static_cast(maxBufferVal) ) ); - char minValStr[sizeof(int64_t) + 1]; - char maxValStr[sizeof(int64_t) + 1]; - memcpy(minValStr, &minVal, sizeof(int64_t)); - memcpy(maxValStr, &maxVal, sizeof(int64_t)); - minValStr[sizeof(int64_t)] = '\0'; - maxValStr[sizeof(int64_t)] = '\0'; - oss << "; minVal: " << minVal << "; (" << minValStr << ")" - << "; maxVal: " << maxVal << "; (" << maxValStr << ")"; - } - else if (isUnsigned(colDataType)) - { - oss << "; minVal: " << static_cast(minBufferVal) << - "; maxVal: " << static_cast(maxBufferVal); - } - else - { - oss << "; minVal: " << minBufferVal << - "; maxVal: " << maxBufferVal; - } + if (isCharType(colDataType)) + { + // Swap/restore byte order before printing character string + int64_t minVal = static_cast(uint64ToStr(static_cast(minBufferVal))); + int64_t maxVal = static_cast(uint64ToStr(static_cast(maxBufferVal))); + char minValStr[sizeof(int64_t) + 1]; + char maxValStr[sizeof(int64_t) + 1]; + memcpy(minValStr, &minVal, sizeof(int64_t)); + memcpy(maxValStr, &maxVal, sizeof(int64_t)); + minValStr[sizeof(int64_t)] = '\0'; + maxValStr[sizeof(int64_t)] = '\0'; + oss << "; minVal: " << minVal << "; (" << minValStr << ")" + << "; maxVal: " << maxVal << "; (" << maxValStr << ")"; + } + else if (isUnsigned(colDataType)) + { + oss << "; minVal: " << static_cast(minBufferVal) + << "; maxVal: " << static_cast(maxBufferVal); + } + else + { + oss << "; minVal: " << minBufferVal << "; maxVal: " << maxBufferVal; + } } - + //------------------------------------------------------------------------------ // Parse Dictionary column Read buffer. Parsed row values are added to // fColBufferMgr, which stores them into an output buffer before writing them @@ -1908,164 +1853,148 @@ void BulkLoadBuffer::parseColLogMinMax( // and of course split up the corresponding strings into 2 different dictionary // store files as well. //------------------------------------------------------------------------------ -int BulkLoadBuffer::parseDict(ColumnInfo& columnInfo) +int BulkLoadBuffer::parseDict(ColumnInfo& columnInfo) { - int rc = NO_ERROR; + int rc = NO_ERROR; - uint32_t nRowsParsed1; - rc = parseDictSection( columnInfo, - 0, - fStartRowParser, - fTotalReadRowsParser, - nRowsParsed1 ); + uint32_t nRowsParsed1; + rc = parseDictSection(columnInfo, 0, fStartRowParser, fTotalReadRowsParser, nRowsParsed1); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "parseDict: error parsing section1: " + << " OID-" << columnInfo.curCol.dataFile.fid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + //..If fTotalReadRows != nRowsParsed1 then reserveInSection() had to + // split up our input buffer tokens because they spanned 2 extents. + // After exiting reserveSection() above, we no longer have a mutex + // lock on the sections in the internal buffer, so you might think + // this could cause a race condition with more rows being added to + // the buffer by other parsing threads, while we are busy wrapping + // up the first extent and creating the second. But since reserve- + // Section() only took some of the rows from the Read buffer, any + // other threads should be blocked waiting for us to add the remain- + // ing rows from "this" Read buffer into a new ColumnBufferSection. + // The following condition wait in reserveSection() should be keeping + // things stable: + // while((fMaxRowId + 1) != startRowId) { + // //Making sure that allocation are made in order + // fOutOfSequence.wait(lock); + // } + + if (fTotalReadRowsParser != nRowsParsed1) + { + if (fLog->isDebug(DEBUG_1)) + { + ostringstream oss; + oss << "parseDict breaking up bufsec for OID-" << columnInfo.curCol.dataFile.fid << "; file-" + << columnInfo.curCol.dataFile.fSegFileName << "; totalInRows-" << fTotalReadRowsParser + << "; rowsFlushedToEndExtent-" << nRowsParsed1; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + //..Flush the rows in the buffer that fill up the current extent + rc = columnInfo.fColBufferMgr->intermediateFlush(); if (rc != NO_ERROR) { - WErrorCodes ec; - ostringstream oss; - oss << "parseDict: error parsing section1: " << - " OID-" << columnInfo.curCol.dataFile.fid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); + WErrorCodes ec; + ostringstream oss; + oss << "parseDict: error flushing column: " + << " OID-" << columnInfo.curCol.dataFile.fid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + //..See if we just finished filling in the last extent for this seg- + // ment token file, in which case we can truncate the corresponding + // dictionary store segment file. (this only affects compressed data). + uint16_t root = columnInfo.curCol.dataFile.fDbRoot; + uint32_t pNum = columnInfo.curCol.dataFile.fPartition; + uint16_t sNum = columnInfo.curCol.dataFile.fSegment; + bool bFileComplete = columnInfo.isFileComplete(); + + //..Close the current segment file, and add an extent to the next + // segment file in the rotation sequence. newSegmentFile is a + // FILE* that points to the newly opened segment file. + rc = columnInfo.fColBufferMgr->extendTokenColumn(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "parseDict: error extending column: " + << " OID-" << columnInfo.curCol.dataFile.fid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + //..Close current dictionary store file and open the dictionary + // store file that will match the newly opened column segment file. + rc = columnInfo.closeDctnryStore(false); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "parseDict: error closing store file: " + << " OID-" << columnInfo.column.dctnry.dctnryOid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + rc = columnInfo.openDctnryStore(false); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "parseDict: error opening store file: " + << " OID-" << columnInfo.column.dctnry.dctnryOid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + // Ignore return code from closing file; already in error state + columnInfo.closeDctnryStore(true); // clean up loose ends + return rc; + } + + //..Now we can add the remaining rows in the current Read buffer to + // to the output buffer destined for the next extent we just added. + uint32_t nRowsParsed2; + rc = parseDictSection(columnInfo, nRowsParsed1, (fStartRowParser + nRowsParsed1), + (fTotalReadRowsParser - nRowsParsed1), nRowsParsed2); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "parseDict: error parsing section2: " + << " OID-" << columnInfo.curCol.dataFile.fid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + //..We went ahead and completed all the necessary parsing to free up + // the buffer we were working on, so that any blocked threads can + // continue. In the mean time, this thread can now go back and + // truncate the dctnry store file we just completed, if applicable. + if (bFileComplete) + { + rc = columnInfo.truncateDctnryStore(columnInfo.column.dctnry.dctnryOid, root, pNum, sNum); + + if (rc != NO_ERROR) return rc; } + } - //..If fTotalReadRows != nRowsParsed1 then reserveInSection() had to - // split up our input buffer tokens because they spanned 2 extents. - // After exiting reserveSection() above, we no longer have a mutex - // lock on the sections in the internal buffer, so you might think - // this could cause a race condition with more rows being added to - // the buffer by other parsing threads, while we are busy wrapping - // up the first extent and creating the second. But since reserve- - // Section() only took some of the rows from the Read buffer, any - // other threads should be blocked waiting for us to add the remain- - // ing rows from "this" Read buffer into a new ColumnBufferSection. - // The following condition wait in reserveSection() should be keeping - // things stable: - // while((fMaxRowId + 1) != startRowId) { - // //Making sure that allocation are made in order - // fOutOfSequence.wait(lock); - // } - - if (fTotalReadRowsParser != nRowsParsed1) - { - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - oss << "parseDict breaking up bufsec for OID-" << - columnInfo.curCol.dataFile.fid << - "; file-" << columnInfo.curCol.dataFile.fSegFileName << - "; totalInRows-" << fTotalReadRowsParser << - "; rowsFlushedToEndExtent-" << nRowsParsed1; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - //..Flush the rows in the buffer that fill up the current extent - rc = columnInfo.fColBufferMgr->intermediateFlush(); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "parseDict: error flushing column: " << - " OID-" << columnInfo.curCol.dataFile.fid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - //..See if we just finished filling in the last extent for this seg- - // ment token file, in which case we can truncate the corresponding - // dictionary store segment file. (this only affects compressed data). - uint16_t root = columnInfo.curCol.dataFile.fDbRoot; - uint32_t pNum = columnInfo.curCol.dataFile.fPartition; - uint16_t sNum = columnInfo.curCol.dataFile.fSegment; - bool bFileComplete = columnInfo.isFileComplete(); - - //..Close the current segment file, and add an extent to the next - // segment file in the rotation sequence. newSegmentFile is a - // FILE* that points to the newly opened segment file. - rc = columnInfo.fColBufferMgr->extendTokenColumn( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "parseDict: error extending column: " << - " OID-" << columnInfo.curCol.dataFile.fid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - //..Close current dictionary store file and open the dictionary - // store file that will match the newly opened column segment file. - rc = columnInfo.closeDctnryStore(false); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "parseDict: error closing store file: " << - " OID-" << columnInfo.column.dctnry.dctnryOid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - rc = columnInfo.openDctnryStore( false ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "parseDict: error opening store file: " << - " OID-" << columnInfo.column.dctnry.dctnryOid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - // Ignore return code from closing file; already in error state - columnInfo.closeDctnryStore(true); // clean up loose ends - return rc; - } - - //..Now we can add the remaining rows in the current Read buffer to - // to the output buffer destined for the next extent we just added. - uint32_t nRowsParsed2; - rc = parseDictSection( columnInfo, - nRowsParsed1, - (fStartRowParser + nRowsParsed1), - (fTotalReadRowsParser - nRowsParsed1), - nRowsParsed2 ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "parseDict: error parsing section2: " << - " OID-" << columnInfo.curCol.dataFile.fid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - //..We went ahead and completed all the necessary parsing to free up - // the buffer we were working on, so that any blocked threads can - // continue. In the mean time, this thread can now go back and - // truncate the dctnry store file we just completed, if applicable. - if (bFileComplete) - { - rc = columnInfo.truncateDctnryStore( - columnInfo.column.dctnry.dctnryOid, root, pNum, sNum); - - if (rc != NO_ERROR) - return rc; - } - } - - return rc; + return rc; } - + //------------------------------------------------------------------------------ // Parses all or part of a Dictionary Read buffer into a ColumnBufferSection, // depending on whether the buffer crosses an extent boundary or not. If it @@ -2073,41 +2002,34 @@ int BulkLoadBuffer::parseDict(ColumnInfo& columnInfo) // up to the end of the current extent. A second call to parseDictSection() // should be made to parse the remainder of the buffer into the second extent. //------------------------------------------------------------------------------ -int BulkLoadBuffer::parseDictSection(ColumnInfo& columnInfo, - int tokenPos, - RID startRow, - uint32_t totalReadRows, - uint32_t& nRowsParsed) +int BulkLoadBuffer::parseDictSection(ColumnInfo& columnInfo, int tokenPos, RID startRow, + uint32_t totalReadRows, uint32_t& nRowsParsed) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (fLog->isDebug( DEBUG_2 )) + if (fLog->isDebug(DEBUG_2)) + { + ostringstream oss; + oss << "DctResSecIn: OID-" << columnInfo.column.mapOid << "; StartRID/Rows: " << startRow << " " + << totalReadRows; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + ColumnBufferSection* section = 0; + RID lastInputRowInExtent = 0; + RETURN_ON_ERROR(columnInfo.fColBufferMgr->reserveSection(startRow, totalReadRows, nRowsParsed, §ion, + lastInputRowInExtent)); + + if (nRowsParsed > 0) + { + char* tokenBuf = new char[nRowsParsed * 8]; + + // Pass fDataParser data and fTokensParser meta data to dictionary + // to be parsed and tokenized, with tokens returned in tokenBuf. + rc = columnInfo.updateDctnryStore(fDataParser, &fTokensParser[tokenPos], nRowsParsed, tokenBuf); + + if (rc == NO_ERROR) { - ostringstream oss; - oss << "DctResSecIn: OID-" << columnInfo.column.mapOid << - "; StartRID/Rows: " << startRow << " " << totalReadRows; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - ColumnBufferSection* section = 0; - RID lastInputRowInExtent = 0; - RETURN_ON_ERROR( columnInfo.fColBufferMgr->reserveSection( - startRow, totalReadRows, nRowsParsed, - §ion, lastInputRowInExtent ) ); - - if (nRowsParsed > 0) - { - char* tokenBuf = new char[nRowsParsed * 8]; - - // Pass fDataParser data and fTokensParser meta data to dictionary - // to be parsed and tokenized, with tokens returned in tokenBuf. - rc = columnInfo.updateDctnryStore( fDataParser, - &fTokensParser[tokenPos], - nRowsParsed, - tokenBuf ) ; - - if (rc == NO_ERROR) - { #if 0 int64_t* tokenVals = reinterpret_cast(tokenBuf); @@ -2125,130 +2047,125 @@ int BulkLoadBuffer::parseDictSection(ColumnInfo& columnInfo, } #endif - section->write(tokenBuf, nRowsParsed); - delete [] tokenBuf; + section->write(tokenBuf, nRowsParsed); + delete[] tokenBuf; - if (fLog->isDebug( DEBUG_2 )) - { - ostringstream oss; - RID rid1 = section->startRowId(); - RID rid2 = section->endRowId(); - oss << "DctRelSecOut: OID-" << columnInfo.column.mapOid << - "; StartRID/Rows: " << rid1 << " " << (rid2 - rid1) + 1 << - "; startOffset: " << section->getStartOffset(); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + if (fLog->isDebug(DEBUG_2)) + { + ostringstream oss; + RID rid1 = section->startRowId(); + RID rid2 = section->endRowId(); + oss << "DctRelSecOut: OID-" << columnInfo.column.mapOid << "; StartRID/Rows: " << rid1 << " " + << (rid2 - rid1) + 1 << "; startOffset: " << section->getStartOffset(); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - RETURN_ON_ERROR( - columnInfo.fColBufferMgr->releaseSection(section) ); - } - else - { - delete [] tokenBuf; - } + RETURN_ON_ERROR(columnInfo.fColBufferMgr->releaseSection(section)); } + else + { + delete[] tokenBuf; + } + } - return rc; + return rc; } - -int BulkLoadBuffer::fillFromMemory( - const BulkLoadBuffer& overFlowBufIn, - const char* input, size_t length, size_t *parse_length, RID& totalReadRows, - RID& correctTotalRows, const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall ) +int BulkLoadBuffer::fillFromMemory(const BulkLoadBuffer& overFlowBufIn, const char* input, size_t length, + size_t* parse_length, RID& totalReadRows, RID& correctTotalRows, + const boost::ptr_vector& columnsInfo, + unsigned int allowedErrCntThisCall) { - boost::mutex::scoped_lock lock(fSyncUpdatesBLB); - reset(); - copyOverflow( overFlowBufIn ); - size_t readSize = 0; + boost::mutex::scoped_lock lock(fSyncUpdatesBLB); + reset(); + copyOverflow(overFlowBufIn); + size_t readSize = 0; - // Copy the overflow data from the last buffer, that did not get written - if (fOverflowSize != 0) + // Copy the overflow data from the last buffer, that did not get written + if (fOverflowSize != 0) + { + memcpy(fData, fOverflowBuf, fOverflowSize); + + if (fOverflowBuf != NULL) { - memcpy( fData, fOverflowBuf, fOverflowSize ); + delete[] fOverflowBuf; + fOverflowBuf = NULL; + } + } - if (fOverflowBuf != NULL) - { - delete [] fOverflowBuf; - fOverflowBuf = NULL; - } + readSize = fBufferSize - fOverflowSize; + if (readSize > (length - *parse_length)) + { + readSize = length - *parse_length; + } + memcpy(fData + fOverflowSize, input + *parse_length, readSize); + *parse_length += readSize; + + bool bEndOfData = false; + + if (length == *parse_length) + { + bEndOfData = true; + } + + if (bEndOfData && // @bug 3516: Add '\n' if missing from last record + (fImportDataMode == IMPORT_DATA_TEXT)) // Only applies to ascii mode + { + if ((fOverflowSize > 0) | (readSize > 0)) + { + if (fData[fOverflowSize + readSize - 1] != '\n') + { + // Should be safe to add byte to fData w/o risk of overflowing, + // since we hit EOF. That should mean fread() did not read all + // the bytes we requested, meaning we have room to add a byte. + fData[fOverflowSize + readSize] = '\n'; + readSize++; + } + } + } + + // Lazy allocation of fToken memory as needed + if (fTokens == 0) + { + resizeTokenArray(); + } + + if ((readSize > 0) || (fOverflowSize > 0)) + { + if (fOverflowBuf != NULL) + { + delete[] fOverflowBuf; + fOverflowBuf = NULL; } - readSize = fBufferSize - fOverflowSize; - if (readSize > (length - *parse_length)) + fReadSize = readSize + fOverflowSize; + fStartRow = correctTotalRows; + fStartRowForLogging = totalReadRows; + + if (fImportDataMode == IMPORT_DATA_TEXT) { - readSize = length - *parse_length; + tokenize(columnsInfo, allowedErrCntThisCall); } - memcpy(fData + fOverflowSize, input + *parse_length, readSize); - *parse_length += readSize; - - bool bEndOfData = false; - - if (length == *parse_length) + else { - bEndOfData = true; + int rc = tokenizeBinary(columnsInfo, allowedErrCntThisCall, bEndOfData); + + if (rc != NO_ERROR) + return rc; } - if ( bEndOfData && // @bug 3516: Add '\n' if missing from last record - (fImportDataMode == IMPORT_DATA_TEXT) ) // Only applies to ascii mode + // If we read a full buffer without hitting any new lines, then + // terminate import because row size is greater than read buffer size. + if ((fTotalReadRowsForLog == 0) && (fReadSize == fBufferSize)) { - if ( (fOverflowSize > 0) | (readSize > 0) ) - { - if ( fData[ fOverflowSize + readSize - 1 ] != '\n' ) - { - // Should be safe to add byte to fData w/o risk of overflowing, - // since we hit EOF. That should mean fread() did not read all - // the bytes we requested, meaning we have room to add a byte. - fData[ fOverflowSize + readSize ] = '\n'; - readSize++; - } - } + return ERR_BULK_ROW_FILL_BUFFER; } - // Lazy allocation of fToken memory as needed - if (fTokens == 0) - { - resizeTokenArray(); - } + totalReadRows += fTotalReadRowsForLog; + correctTotalRows += fTotalReadRows; + } - if ((readSize > 0) || (fOverflowSize > 0)) - { - if (fOverflowBuf != NULL) - { - delete [] fOverflowBuf; - fOverflowBuf = NULL; - } - - fReadSize = readSize + fOverflowSize; - fStartRow = correctTotalRows; - fStartRowForLogging = totalReadRows; - - if (fImportDataMode == IMPORT_DATA_TEXT) - { - tokenize( columnsInfo, allowedErrCntThisCall ); - } - else - { - int rc = tokenizeBinary( columnsInfo, allowedErrCntThisCall, - bEndOfData ); - - if (rc != NO_ERROR) - return rc; - } - - // If we read a full buffer without hitting any new lines, then - // terminate import because row size is greater than read buffer size. - if ((fTotalReadRowsForLog == 0) && (fReadSize == fBufferSize)) - { - return ERR_BULK_ROW_FILL_BUFFER; - } - - totalReadRows += fTotalReadRowsForLog; - correctTotalRows += fTotalReadRows; - } - - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -2258,103 +2175,99 @@ int BulkLoadBuffer::fillFromMemory( // correctTotalRows (input/output) - total valid row count from tokenize() // (cumulative) //------------------------------------------------------------------------------ -int BulkLoadBuffer::fillFromFile( - const BulkLoadBuffer& overFlowBufIn, - FILE* handle, RID& totalReadRows, RID& correctTotalRows, - const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall ) +int BulkLoadBuffer::fillFromFile(const BulkLoadBuffer& overFlowBufIn, FILE* handle, RID& totalReadRows, + RID& correctTotalRows, const boost::ptr_vector& columnsInfo, + unsigned int allowedErrCntThisCall) { - boost::mutex::scoped_lock lock(fSyncUpdatesBLB); - reset(); - copyOverflow( overFlowBufIn ); - size_t readSize = 0; + boost::mutex::scoped_lock lock(fSyncUpdatesBLB); + reset(); + copyOverflow(overFlowBufIn); + size_t readSize = 0; - // Copy the overflow data from the last buffer, that did not get written - if (fOverflowSize != 0) + // Copy the overflow data from the last buffer, that did not get written + if (fOverflowSize != 0) + { + memcpy(fData, fOverflowBuf, fOverflowSize); + + if (fOverflowBuf != NULL) { - memcpy( fData, fOverflowBuf, fOverflowSize ); + delete[] fOverflowBuf; + fOverflowBuf = NULL; + } + } - if (fOverflowBuf != NULL) - { - delete [] fOverflowBuf; - fOverflowBuf = NULL; - } + readSize = fread(fData + fOverflowSize, 1, fBufferSize - fOverflowSize, handle); + + if (ferror(handle)) + { + return ERR_FILE_READ_IMPORT; + } + + bool bEndOfData = false; + + if (feof(handle)) + bEndOfData = true; + + if (bEndOfData && // @bug 3516: Add '\n' if missing from last record + (fImportDataMode == IMPORT_DATA_TEXT)) // Only applies to ascii mode + { + if ((fOverflowSize > 0) | (readSize > 0)) + { + if (fData[fOverflowSize + readSize - 1] != '\n') + { + // Should be safe to add byte to fData w/o risk of overflowing, + // since we hit EOF. That should mean fread() did not read all + // the bytes we requested, meaning we have room to add a byte. + fData[fOverflowSize + readSize] = '\n'; + readSize++; + } + } + } + + // Lazy allocation of fToken memory as needed + if (fTokens == 0) + { + resizeTokenArray(); + } + + if ((readSize > 0) || (fOverflowSize > 0)) + { + if (fOverflowBuf != NULL) + { + delete[] fOverflowBuf; + fOverflowBuf = NULL; } - readSize = fread( fData + fOverflowSize, - 1, fBufferSize - fOverflowSize, handle ); + fReadSize = readSize + fOverflowSize; + fStartRow = correctTotalRows; + fStartRowForLogging = totalReadRows; - if ( ferror(handle) ) + if (fImportDataMode == IMPORT_DATA_TEXT) { - return ERR_FILE_READ_IMPORT; + tokenize(columnsInfo, allowedErrCntThisCall); + } + else + { + int rc = tokenizeBinary(columnsInfo, allowedErrCntThisCall, bEndOfData); + + if (rc != NO_ERROR) + return rc; } - bool bEndOfData = false; - - if (feof(handle)) - bEndOfData = true; - - if ( bEndOfData && // @bug 3516: Add '\n' if missing from last record - (fImportDataMode == IMPORT_DATA_TEXT) ) // Only applies to ascii mode + // If we read a full buffer without hitting any new lines, then + // terminate import because row size is greater than read buffer size. + if ((fTotalReadRowsForLog == 0) && (fReadSize == fBufferSize)) { - if ( (fOverflowSize > 0) | (readSize > 0) ) - { - if ( fData[ fOverflowSize + readSize - 1 ] != '\n' ) - { - // Should be safe to add byte to fData w/o risk of overflowing, - // since we hit EOF. That should mean fread() did not read all - // the bytes we requested, meaning we have room to add a byte. - fData[ fOverflowSize + readSize ] = '\n'; - readSize++; - } - } + return ERR_BULK_ROW_FILL_BUFFER; } - // Lazy allocation of fToken memory as needed - if (fTokens == 0) - { - resizeTokenArray(); - } + totalReadRows += fTotalReadRowsForLog; + correctTotalRows += fTotalReadRows; + } - if ((readSize > 0) || (fOverflowSize > 0)) - { - if (fOverflowBuf != NULL) - { - delete [] fOverflowBuf; - fOverflowBuf = NULL; - } - - fReadSize = readSize + fOverflowSize; - fStartRow = correctTotalRows; - fStartRowForLogging = totalReadRows; - - if (fImportDataMode == IMPORT_DATA_TEXT) - { - tokenize( columnsInfo, allowedErrCntThisCall ); - } - else - { - int rc = tokenizeBinary( columnsInfo, allowedErrCntThisCall, - bEndOfData ); - - if (rc != NO_ERROR) - return rc; - } - - // If we read a full buffer without hitting any new lines, then - // terminate import because row size is greater than read buffer size. - if ((fTotalReadRowsForLog == 0) && (fReadSize == fBufferSize)) - { - return ERR_BULK_ROW_FILL_BUFFER; - } - - totalReadRows += fTotalReadRowsForLog; - correctTotalRows += fTotalReadRows; - } - - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Parse the rows of data in "fData", saving the meta information that describes // the parsed data, in fTokens. If the number of read parsing errors for a @@ -2387,695 +2300,641 @@ int BulkLoadBuffer::fillFromFile( // The initial parsing state for each column is LEADING_CHAR or NORMAL, // depending on whether the user has enabled the "enclosed by" feature. //------------------------------------------------------------------------------ -void BulkLoadBuffer::tokenize( - const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall ) +void BulkLoadBuffer::tokenize(const boost::ptr_vector& columnsInfo, + unsigned int allowedErrCntThisCall) { - unsigned offset = 0; // length of field - unsigned curCol = 0; // dest db column counter within a row - unsigned curFld = 0; // src input field counter within a row - unsigned curRowNum = 0; // "total" number of rows read during this call - unsigned curRowNum1 = 0; // number of "valid" rows inserted into fTokens - char* p; // iterates thru each byte in the input buffer - char c; // value of byte at address "p". - char* lastRowHead = 0; // start of latest row being processed - bool bValidRow = true; // track whether current row is valid - bool bRowGenAutoInc = false; //track whether row uses generated auto-inc - std::string validationErrMsg;//validation error msg (if any) for current row - unsigned errorCount = 0; - const char FIELD_DELIM_CHAR = fColDelim; - const char STRING_ENCLOSED_CHAR = fEnclosedByChar; - const char ESCAPE_CHAR = fEscapeChar; - const char LINE_FEED = 0x0D; - const char CARRIAGE_RETURN = 0x0A; + unsigned offset = 0; // length of field + unsigned curCol = 0; // dest db column counter within a row + unsigned curFld = 0; // src input field counter within a row + unsigned curRowNum = 0; // "total" number of rows read during this call + unsigned curRowNum1 = 0; // number of "valid" rows inserted into fTokens + char* p; // iterates thru each byte in the input buffer + char c; // value of byte at address "p". + char* lastRowHead = 0; // start of latest row being processed + bool bValidRow = true; // track whether current row is valid + bool bRowGenAutoInc = false; // track whether row uses generated auto-inc + std::string validationErrMsg; // validation error msg (if any) for current row + unsigned errorCount = 0; + const char FIELD_DELIM_CHAR = fColDelim; + const char STRING_ENCLOSED_CHAR = fEnclosedByChar; + const char ESCAPE_CHAR = fEscapeChar; + const char LINE_FEED = 0x0D; + const char CARRIAGE_RETURN = 0x0A; - // Variables used to store raw data read for a row; needed if we strip out - // enclosed char(s) and later have to print original data in a *.bad file - char* pRawDataRow = 0; - unsigned rawDataRowCapacity = 0; - unsigned rawDataRowLength = 0; - const unsigned MIN_RAW_DATA_CAP = 1024; + // Variables used to store raw data read for a row; needed if we strip out + // enclosed char(s) and later have to print original data in a *.bad file + char* pRawDataRow = 0; + unsigned rawDataRowCapacity = 0; + unsigned rawDataRowLength = 0; + const unsigned MIN_RAW_DATA_CAP = 1024; - // Enable "enclosed by" checking if user specified an "enclosed by" char - FieldParsingState initialState = FLD_PARSE_NORMAL_STATE; + // Enable "enclosed by" checking if user specified an "enclosed by" char + FieldParsingState initialState = FLD_PARSE_NORMAL_STATE; - if (STRING_ENCLOSED_CHAR != '\0') + if (STRING_ENCLOSED_CHAR != '\0') + { + initialState = FLD_PARSE_LEADING_CHAR_STATE; + } + + FieldParsingState fieldState = initialState; + bool bNewLine = false; // Tracks new line + unsigned start = 0; // Where next field starts in fData + unsigned idxFrom = 0; // idxFrom and idxTo are used to strip out + unsigned idxTo = 0; // escape characters in \" and "" + + // Initialize which field values are enclosed + unsigned int enclosedFieldFlag = 0; +#ifdef DEBUG_TOKEN_PARSING + unsigned int enclosedFieldFlags[fNumberOfColumns]; + memset(enclosedFieldFlags, 0, sizeof(unsigned) * fNumberOfColumns); +#endif + + p = lastRowHead = fData; + const char* pEndOfData = p + fReadSize; //@bug3810 set an end-of-data marker + + //-------------------------------------------------------------------------- + // Loop through all the bytes in the read buffer in order to construct + // the meta data stored in fTokens. + //-------------------------------------------------------------------------- + while (p < pEndOfData) + { + c = *p; + + // If we have stripped "enclosed" characters, then save raw data + if (rawDataRowLength > 0) { - initialState = FLD_PARSE_LEADING_CHAR_STATE; + if (rawDataRowLength == rawDataRowCapacity) // resize array if full + { + rawDataRowCapacity = rawDataRowCapacity * 2; + resizeRowDataArray(&pRawDataRow, rawDataRowLength, rawDataRowCapacity); + } + + pRawDataRow[rawDataRowLength] = c; + rawDataRowLength++; } - FieldParsingState fieldState = initialState; - bool bNewLine = false; // Tracks new line - unsigned start = 0; // Where next field starts in fData - unsigned idxFrom = 0; // idxFrom and idxTo are used to strip out - unsigned idxTo = 0; // escape characters in \" and "" - - // Initialize which field values are enclosed - unsigned int enclosedFieldFlag = 0; -#ifdef DEBUG_TOKEN_PARSING - unsigned int enclosedFieldFlags[fNumberOfColumns]; - memset (enclosedFieldFlags, 0, sizeof(unsigned)*fNumberOfColumns); -#endif - - p = lastRowHead = fData; - const char* pEndOfData = p + fReadSize; //@bug3810 set an end-of-data marker - - //-------------------------------------------------------------------------- - // Loop through all the bytes in the read buffer in order to construct - // the meta data stored in fTokens. - //-------------------------------------------------------------------------- - while ( p < pEndOfData ) + //---------------------------------------------------------------------- + // Branch based on current parsing state for this field. + // Note that we fall out of switch/case and do more processing if we + // have hit end of column or line; else we "continue" directly to end + // of loop to process the next byte. + //---------------------------------------------------------------------- + switch (fieldState) { - c = *p; - - // If we have stripped "enclosed" characters, then save raw data - if (rawDataRowLength > 0) + //------------------------------------------------------------------ + // FLD_PARSE_NORMAL_STATE + // Field not enclosed in a string delimiter such as a double quote + //------------------------------------------------------------------ + case FLD_PARSE_NORMAL_STATE: + { + if ((c == FIELD_DELIM_CHAR) || (c == NEWLINE_CHAR)) { - if (rawDataRowLength == rawDataRowCapacity) // resize array if full + start = p - fData - offset; + + if (c == NEWLINE_CHAR) + bNewLine = true; + } + else + { + offset++; + p++; + continue; // process next byte + } + + break; + } + + // If state is something other than FLD_PARSE_NORMAL_STATE, then + // there is extra processing to allow for fields that may be en- + // closed within a string delimiter (such as a double quote) + + //---------------------------------------------------------------- + // FLD_PARSE_LEADING_CHAR_STATE + //---------------------------------------------------------------- + case FLD_PARSE_LEADING_CHAR_STATE: + { + bool bNewColumn = false; + + if (c == STRING_ENCLOSED_CHAR) + { + fieldState = FLD_PARSE_ENCLOSED_STATE; + idxFrom = p - fData + 1; + idxTo = idxFrom; + start = idxTo; + offset = 0; + enclosedFieldFlag = 1; + } + + else if ((c == FIELD_DELIM_CHAR) || (c == NEWLINE_CHAR)) + { + bNewColumn = true; + start = p - fData; + offset = 0; + + if (c == NEWLINE_CHAR) + bNewLine = true; + } + + else + { + fieldState = FLD_PARSE_NORMAL_STATE; + start = p - fData; + offset = 1; + } + + if (!bNewColumn) + { + p++; + continue; // process next byte + } + + break; + } + + //------------------------------------------------------------------ + // FLD_PARSE_ENCLOSED_STATE + //------------------------------------------------------------------ + case FLD_PARSE_ENCLOSED_STATE: + { + char next = *(p + 1); + + if ((p + 1 < pEndOfData) && + (((c == ESCAPE_CHAR) && ((next == STRING_ENCLOSED_CHAR) || (next == ESCAPE_CHAR) || + (next == LINE_FEED) || (next == CARRIAGE_RETURN))) || + ((c == STRING_ENCLOSED_CHAR) && (next == STRING_ENCLOSED_CHAR)))) + { + // Create/save original data before stripping out bytes + if (rawDataRowLength == 0) + { + rawDataRowLength = (p + 1) - lastRowHead + 1; + rawDataRowCapacity = rawDataRowLength * 2; + + if (rawDataRowCapacity < MIN_RAW_DATA_CAP) + rawDataRowCapacity = MIN_RAW_DATA_CAP; + + pRawDataRow = new char[rawDataRowCapacity]; + memcpy(pRawDataRow, lastRowHead, rawDataRowLength); + } + else + { + if (rawDataRowLength == rawDataRowCapacity) { - rawDataRowCapacity = rawDataRowCapacity * 2; - resizeRowDataArray( &pRawDataRow, - rawDataRowLength, rawDataRowCapacity ); + // resize array if full + rawDataRowCapacity = rawDataRowCapacity * 2; + resizeRowDataArray(&pRawDataRow, rawDataRowLength, rawDataRowCapacity); } - pRawDataRow[rawDataRowLength] = c; + pRawDataRow[rawDataRowLength] = *(p + 1); rawDataRowLength++; + } + + fData[idxTo] = *(p + 1); + idxFrom += 2; + idxTo++; + offset++; + p++; } - //---------------------------------------------------------------------- - // Branch based on current parsing state for this field. - // Note that we fall out of switch/case and do more processing if we - // have hit end of column or line; else we "continue" directly to end - // of loop to process the next byte. - //---------------------------------------------------------------------- - switch (fieldState) + else if (c == STRING_ENCLOSED_CHAR) { - //------------------------------------------------------------------ - // FLD_PARSE_NORMAL_STATE - // Field not enclosed in a string delimiter such as a double quote - //------------------------------------------------------------------ - case FLD_PARSE_NORMAL_STATE: - { - if ((c == FIELD_DELIM_CHAR) || (c == NEWLINE_CHAR)) - { - start = p - fData - offset; - - if (c == NEWLINE_CHAR) - bNewLine = true; - } - else - { - offset++; - p++; - continue; // process next byte - } - - break; - } - - // If state is something other than FLD_PARSE_NORMAL_STATE, then - // there is extra processing to allow for fields that may be en- - // closed within a string delimiter (such as a double quote) - - //---------------------------------------------------------------- - // FLD_PARSE_LEADING_CHAR_STATE - //---------------------------------------------------------------- - case FLD_PARSE_LEADING_CHAR_STATE: - { - bool bNewColumn = false; - - if (c == STRING_ENCLOSED_CHAR) - { - fieldState = FLD_PARSE_ENCLOSED_STATE; - idxFrom = p - fData + 1; - idxTo = idxFrom; - start = idxTo; - offset = 0; - enclosedFieldFlag = 1; - } - - else if ((c == FIELD_DELIM_CHAR) || (c == NEWLINE_CHAR)) - { - bNewColumn = true; - start = p - fData; - offset = 0; - - if ( c == NEWLINE_CHAR ) - bNewLine = true; - } - - else - { - fieldState = FLD_PARSE_NORMAL_STATE; - start = p - fData; - offset = 1; - } - - if (!bNewColumn) - { - p++; - continue; // process next byte - } - - break; - } - - //------------------------------------------------------------------ - // FLD_PARSE_ENCLOSED_STATE - //------------------------------------------------------------------ - case FLD_PARSE_ENCLOSED_STATE: - { - char next = *(p + 1); - - if ( (p + 1 < pEndOfData) && - (((c == ESCAPE_CHAR) && - (( next == STRING_ENCLOSED_CHAR) || - ( next == ESCAPE_CHAR) || - ( next == LINE_FEED) || - ( next == CARRIAGE_RETURN))) || - ((c == STRING_ENCLOSED_CHAR ) && - ( next == STRING_ENCLOSED_CHAR)) ) ) - { - // Create/save original data before stripping out bytes - if (rawDataRowLength == 0) - { - rawDataRowLength = (p + 1) - lastRowHead + 1; - rawDataRowCapacity = rawDataRowLength * 2; - - if (rawDataRowCapacity < MIN_RAW_DATA_CAP) - rawDataRowCapacity = MIN_RAW_DATA_CAP; - - pRawDataRow = new char[rawDataRowCapacity]; - memcpy(pRawDataRow, - lastRowHead, - rawDataRowLength); - } - else - { - if (rawDataRowLength == rawDataRowCapacity) - { - // resize array if full - rawDataRowCapacity = rawDataRowCapacity * 2; - resizeRowDataArray( &pRawDataRow, - rawDataRowLength, rawDataRowCapacity ); - } - - pRawDataRow[rawDataRowLength] = *(p + 1); - rawDataRowLength++; - } - - fData[ idxTo ] = *(p + 1); - idxFrom += 2; - idxTo++; - offset++; - p++; - } - - else if (c == STRING_ENCLOSED_CHAR) - { - fieldState = FLD_PARSE_TRAILING_CHAR_STATE; - } - - else - { - if (idxTo != idxFrom) - fData[ idxTo ] = fData[ idxFrom ]; - - idxFrom++; - idxTo++; - offset++; - } - - p++; - continue; // process next byte - } - - //------------------------------------------------------------------ - // FLD_PARSE_TRAILING_CHAR_STATE - // Ignore any trailing chars till we reach field or line delimiter. - //------------------------------------------------------------------ - case FLD_PARSE_TRAILING_CHAR_STATE: - default: - { - if ((c == FIELD_DELIM_CHAR) || (c == NEWLINE_CHAR)) - { - if (c == NEWLINE_CHAR) - bNewLine = true; - } - else - { - p++; - continue; // process next byte - } - - break; - } - } // end of switch on fieldState - - //---------------------------------------------------------------------- - // Finished reading the bytes in the next source field. - // See if source field is to be included (or ignored) - //---------------------------------------------------------------------- - if ((curFld < fNumFieldsInFile) && - (fFieldList[curFld].fFldColType == BULK_FLDCOL_COLUMN_FIELD)) - { - //------------------------------------------------------------------ - // Process destination column or end of row if source field is to - // be included as part of output to database. - //------------------------------------------------------------------ - if (curCol < fNumColsInFile) - { - const JobColumn& jobCol = columnsInfo[curCol].column; - - //tmp code to test trailing space - if (jobCol.dataType == CalpontSystemCatalog::CHAR) - { - //cout << "triming ... " << endl; - char* tmp = p; - - while (*(--tmp) == ' ') - { - //cout << "offset is " << offset <8000. Only reject numeric cols>1000 bytes - else if ((fTokens[curRowNum1][curCol].offset > - MAX_FIELD_SIZE) && - (jobCol.colType != COL_TYPE_DICT) && - (bValidRow)) - { - bValidRow = false; - - ostringstream ossErrMsg; - ossErrMsg << INPUT_ERROR_TOO_LONG << - "field " << (curFld + 1) << - " longer than " << MAX_FIELD_SIZE << - " bytes"; - validationErrMsg = ossErrMsg.str(); - } - - break; - } - } // end of switch on offset - - // @bug 4037: When cmd line option set, treat char - // and varchar fields that are too long as errors - if (getTruncationAsError() && bValidRow && - (fTokens[curRowNum1][curCol].offset != - COLPOSPAIR_NULL_TOKEN_OFFSET)) - { - if ((jobCol.dataType == CalpontSystemCatalog::VARCHAR || - jobCol.dataType == CalpontSystemCatalog::CHAR) && - (fTokens[curRowNum1][curCol].offset > - jobCol.definedWidth)) - { - bValidRow = false; - - ostringstream ossErrMsg; - ossErrMsg << INPUT_ERROR_STRING_TOO_LONG << - "field " << (curFld + 1) << - " longer than " << jobCol.definedWidth << - " bytes"; - validationErrMsg = ossErrMsg.str(); - } - } - - } // end of "if (offset)" - else - { - fTokens[curRowNum1][curCol].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; - - if (jobCol.autoIncFlag) - bRowGenAutoInc = true; - } - - // For non-autoincrement column, - // Validate a NotNull column is supplied a value or a default - if (!bRowGenAutoInc) - { - if ((jobCol.fNotNull) && - (fTokens[curRowNum1][curCol].offset == - COLPOSPAIR_NULL_TOKEN_OFFSET) && - (!jobCol.fWithDefault) && - (bValidRow)) - { - bValidRow = false; - - ostringstream ossErrMsg; - ossErrMsg << INPUT_ERROR_NULL_CONSTRAINT << - "; field " << (curFld + 1); - validationErrMsg = ossErrMsg.str(); - } - } - } // end if curCol < fNumberOfColumns - - curCol++; + fieldState = FLD_PARSE_TRAILING_CHAR_STATE; } - curFld++; - - //---------------------------------------------------------------------- - // End-of-row processing - //---------------------------------------------------------------------- - if (bNewLine) + else { - // Debug: Dump next row that may or may not be accepted as - // valid. Not a typo, that we print "curRowNum" as the row - // number, but we use curRowNum1 as the index into fTokens. + if (idxTo != idxFrom) + fData[idxTo] = fData[idxFrom]; + + idxFrom++; + idxTo++; + offset++; + } + + p++; + continue; // process next byte + } + + //------------------------------------------------------------------ + // FLD_PARSE_TRAILING_CHAR_STATE + // Ignore any trailing chars till we reach field or line delimiter. + //------------------------------------------------------------------ + case FLD_PARSE_TRAILING_CHAR_STATE: + default: + { + if ((c == FIELD_DELIM_CHAR) || (c == NEWLINE_CHAR)) + { + if (c == NEWLINE_CHAR) + bNewLine = true; + } + else + { + p++; + continue; // process next byte + } + + break; + } + } // end of switch on fieldState + + //---------------------------------------------------------------------- + // Finished reading the bytes in the next source field. + // See if source field is to be included (or ignored) + //---------------------------------------------------------------------- + if ((curFld < fNumFieldsInFile) && (fFieldList[curFld].fFldColType == BULK_FLDCOL_COLUMN_FIELD)) + { + //------------------------------------------------------------------ + // Process destination column or end of row if source field is to + // be included as part of output to database. + //------------------------------------------------------------------ + if (curCol < fNumColsInFile) + { + const JobColumn& jobCol = columnsInfo[curCol].column; + + // tmp code to test trailing space + if (jobCol.dataType == CalpontSystemCatalog::CHAR) + { + // cout << "triming ... " << endl; + char* tmp = p; + + while (*(--tmp) == ' ') + { + // cout << "offset is " << offset <" << std::endl; - } - } - + enclosedFieldFlags[curCol] = enclosedFieldFlag; #endif - curRowNum++; // increment total number of rows read - int rowLength = p - lastRowHead + 1; - - // @bug 3146: Allow optional trailing value at end of input file - // Don't count last column if no data after last delimiter, - // and we don't need that last column. - if ((offset == 0) && (curFld == (fNumFieldsInFile + 1))) - { - curFld--; - } - - if ((curFld != fNumFieldsInFile) && - (bValidRow)) + // Would like to refactor this validation logic into a separate + // inline function, but code may be too long for compiler + // to inline. And factoring out into a non-inline function + // slows down the read thread by 10%. So left code here. + if (offset) + { + switch (fTokens[curRowNum1][curCol].offset) + { + // Special auto-increment case; treat '0' as null value + case 1: { + if ((jobCol.autoIncFlag) && (*(fData + fTokens[curRowNum1][curCol].start) == NULL_AUTO_INC_0)) + { + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + bRowGenAutoInc = true; + } + else if (jobCol.dataType == CalpontSystemCatalog::VARBINARY && (bValidRow)) + { bValidRow = false; ostringstream ossErrMsg; - ossErrMsg << INPUT_ERROR_WRONG_NO_COLUMNS << - "; num fields expected-" << fNumFieldsInFile << - "; num fields found-" << curFld; + ossErrMsg << INPUT_ERROR_ODD_VARBINARY_LENGTH << "field " << (curFld + 1) << " has " << offset + << " bytes"; validationErrMsg = ossErrMsg.str(); + } + + break; } - if (bValidRow) + case 2: { - // Initialize fTokens for tags not in input file - if (fNumColsInFile < fNumberOfColumns) - { - for (unsigned int n = fNumColsInFile; n < fNumberOfColumns; n++) - { - fTokens[curRowNum1][n].start = 0; - fTokens[curRowNum1][n].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; + if ((*(fData + fTokens[curRowNum1][curCol].start) == ESCAPE_CHAR) && + (*(fData + fTokens[curRowNum1][curCol].start + 1) == NULL_CHAR)) + { + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; - if (columnsInfo[n].column.autoIncFlag) - bRowGenAutoInc = true; - } - } + if (jobCol.autoIncFlag) + bRowGenAutoInc = true; + } - curRowNum1++; // increment valid row count - - if (bRowGenAutoInc) - fAutoIncGenCount++; // update number of generated auto-incs + break; } - else + + // If enclosedFlag set, then treat "NULL" as data and + // not as a null value + case 4: { - // Store validation error message to be logged - if (rawDataRowLength == 0) + if ((fNullStringMode) && (!enclosedFieldFlag)) + { + if ((*(fData + fTokens[curRowNum1][curCol].start) == NULL_VALUE_STRING[0]) && + (*(fData + fTokens[curRowNum1][curCol].start + 1) == NULL_VALUE_STRING[1]) && + (*(fData + fTokens[curRowNum1][curCol].start + 2) == NULL_VALUE_STRING[2]) && + (*(fData + fTokens[curRowNum1][curCol].start + 3) == NULL_VALUE_STRING[3])) { - string tmp(lastRowHead, rowLength); - fErrRows.push_back( tmp ); - } - else - { - string tmp(pRawDataRow, rawDataRowLength); - fErrRows.push_back( tmp ); - } + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; - fRowStatus.push_back(std::pair( - fStartRowForLogging + curRowNum, - validationErrMsg)); - - errorCount++; - - // Quit if we exceed max allowable errors for this call. - // We set lastRowHead = p, so that the code that follows this - // loop won't try to save any data in fOverflowBuf. - if (errorCount > allowedErrCntThisCall) - { - lastRowHead = p + 1; - p++; - break; + if (jobCol.autoIncFlag) + bRowGenAutoInc = true; } + } + + break; } - curCol = 0; - curFld = 0; - lastRowHead = p + 1; - rawDataRowLength = 0; - - // Resize fTokens array if we are about to fill it up - if ( curRowNum1 >= fTotalRows ) + default: { - resizeTokenArray(); - } + if ((jobCol.dataType == CalpontSystemCatalog::VARBINARY) && ((offset & 1) == 1) && (bValidRow)) + { + bValidRow = false; - bNewLine = false; - bValidRow = true; - bRowGenAutoInc = false; + ostringstream ossErrMsg; + ossErrMsg << INPUT_ERROR_ODD_VARBINARY_LENGTH << "field " << (curFld + 1) << " has " << offset + << " bytes"; + validationErrMsg = ossErrMsg.str(); + } + + // @bug 3478: Truncate instead of rejecting dctnry + // strings>8000. Only reject numeric cols>1000 bytes + else if ((fTokens[curRowNum1][curCol].offset > MAX_FIELD_SIZE) && + (jobCol.colType != COL_TYPE_DICT) && (bValidRow)) + { + bValidRow = false; + + ostringstream ossErrMsg; + ossErrMsg << INPUT_ERROR_TOO_LONG << "field " << (curFld + 1) << " longer than " + << MAX_FIELD_SIZE << " bytes"; + validationErrMsg = ossErrMsg.str(); + } + + break; + } + } // end of switch on offset + + // @bug 4037: When cmd line option set, treat char + // and varchar fields that are too long as errors + if (getTruncationAsError() && bValidRow && + (fTokens[curRowNum1][curCol].offset != COLPOSPAIR_NULL_TOKEN_OFFSET)) + { + if ((jobCol.dataType == CalpontSystemCatalog::VARCHAR || + jobCol.dataType == CalpontSystemCatalog::CHAR) && + (fTokens[curRowNum1][curCol].offset > jobCol.definedWidth)) + { + bValidRow = false; + + ostringstream ossErrMsg; + ossErrMsg << INPUT_ERROR_STRING_TOO_LONG << "field " << (curFld + 1) << " longer than " + << jobCol.definedWidth << " bytes"; + validationErrMsg = ossErrMsg.str(); + } + } + + } // end of "if (offset)" + else + { + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + + if (jobCol.autoIncFlag) + bRowGenAutoInc = true; + } + + // For non-autoincrement column, + // Validate a NotNull column is supplied a value or a default + if (!bRowGenAutoInc) + { + if ((jobCol.fNotNull) && (fTokens[curRowNum1][curCol].offset == COLPOSPAIR_NULL_TOKEN_OFFSET) && + (!jobCol.fWithDefault) && (bValidRow)) + { + bValidRow = false; + + ostringstream ossErrMsg; + ossErrMsg << INPUT_ERROR_NULL_CONSTRAINT << "; field " << (curFld + 1); + validationErrMsg = ossErrMsg.str(); + } + } + } // end if curCol < fNumberOfColumns + + curCol++; + } + + curFld++; + + //---------------------------------------------------------------------- + // End-of-row processing + //---------------------------------------------------------------------- + if (bNewLine) + { + // Debug: Dump next row that may or may not be accepted as + // valid. Not a typo, that we print "curRowNum" as the row + // number, but we use curRowNum1 as the index into fTokens. #ifdef DEBUG_TOKEN_PARSING + std::cout << "Row " << curRowNum + 1 << ". fTokens: " + << "(start,offset,enclosed)" << std::endl; + unsigned kColCount = fNumColsInFile; - if (initialState == FLD_PARSE_LEADING_CHAR_STATE) - memset (enclosedFieldFlags, 0, sizeof(unsigned)*fNumberOfColumns); + if (curCol < kColCount) + kColCount = curCol; + + for (unsigned int k = 0; k < kColCount; k++) + { + std::cout << " (" << fTokens[curRowNum1][k].start << "," << fTokens[curRowNum1][k].offset << "," + << enclosedFieldFlags[k] << ") "; + + if (fTokens[curRowNum1][k].offset != COLPOSPAIR_NULL_TOKEN_OFFSET) + { + std::string outField(fData + fTokens[curRowNum1][k].start, fTokens[curRowNum1][k].offset); + std::cout << " " << outField << std::endl; + } + else + { + std::cout << " " << std::endl; + } + } #endif - } // end of (bNewLine) - offset = 0; - fieldState = initialState; - enclosedFieldFlag = 0; + curRowNum++; // increment total number of rows read + int rowLength = p - lastRowHead + 1; - p++; - } // end of (p < pEndOfData) loop to step thru the read buffer + // @bug 3146: Allow optional trailing value at end of input file + // Don't count last column if no data after last delimiter, + // and we don't need that last column. + if ((offset == 0) && (curFld == (fNumFieldsInFile + 1))) + { + curFld--; + } - // Save any leftover data that we did not yet parse, into fOverflowBuf - if ( p > lastRowHead ) - { - fOverflowSize = p - lastRowHead; - fOverflowBuf = new char[fOverflowSize]; + if ((curFld != fNumFieldsInFile) && (bValidRow)) + { + bValidRow = false; - // If we stripped out any chars, be sure to preserve the original data + ostringstream ossErrMsg; + ossErrMsg << INPUT_ERROR_WRONG_NO_COLUMNS << "; num fields expected-" << fNumFieldsInFile + << "; num fields found-" << curFld; + validationErrMsg = ossErrMsg.str(); + } + + if (bValidRow) + { + // Initialize fTokens for tags not in input file + if (fNumColsInFile < fNumberOfColumns) + { + for (unsigned int n = fNumColsInFile; n < fNumberOfColumns; n++) + { + fTokens[curRowNum1][n].start = 0; + fTokens[curRowNum1][n].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + + if (columnsInfo[n].column.autoIncFlag) + bRowGenAutoInc = true; + } + } + + curRowNum1++; // increment valid row count + + if (bRowGenAutoInc) + fAutoIncGenCount++; // update number of generated auto-incs + } + else + { + // Store validation error message to be logged if (rawDataRowLength == 0) - memcpy( fOverflowBuf, lastRowHead, fOverflowSize ); + { + string tmp(lastRowHead, rowLength); + fErrRows.push_back(tmp); + } else - memcpy( fOverflowBuf, pRawDataRow, fOverflowSize ); - } + { + string tmp(pRawDataRow, rawDataRowLength); + fErrRows.push_back(tmp); + } + + fRowStatus.push_back(std::pair(fStartRowForLogging + curRowNum, validationErrMsg)); + + errorCount++; + + // Quit if we exceed max allowable errors for this call. + // We set lastRowHead = p, so that the code that follows this + // loop won't try to save any data in fOverflowBuf. + if (errorCount > allowedErrCntThisCall) + { + lastRowHead = p + 1; + p++; + break; + } + } + + curCol = 0; + curFld = 0; + lastRowHead = p + 1; + rawDataRowLength = 0; + + // Resize fTokens array if we are about to fill it up + if (curRowNum1 >= fTotalRows) + { + resizeTokenArray(); + } + + bNewLine = false; + bValidRow = true; + bRowGenAutoInc = false; +#ifdef DEBUG_TOKEN_PARSING + + if (initialState == FLD_PARSE_LEADING_CHAR_STATE) + memset(enclosedFieldFlags, 0, sizeof(unsigned) * fNumberOfColumns); + +#endif + } // end of (bNewLine) + + offset = 0; + fieldState = initialState; + enclosedFieldFlag = 0; + + p++; + } // end of (p < pEndOfData) loop to step thru the read buffer + + // Save any leftover data that we did not yet parse, into fOverflowBuf + if (p > lastRowHead) + { + fOverflowSize = p - lastRowHead; + fOverflowBuf = new char[fOverflowSize]; + + // If we stripped out any chars, be sure to preserve the original data + if (rawDataRowLength == 0) + memcpy(fOverflowBuf, lastRowHead, fOverflowSize); else - { - fOverflowSize = 0; - fOverflowBuf = NULL; - } + memcpy(fOverflowBuf, pRawDataRow, fOverflowSize); + } + else + { + fOverflowSize = 0; + fOverflowBuf = NULL; + } - fTotalReadRows = curRowNum1; // number of valid rows read - fTotalReadRowsForLog = curRowNum; // total number of rows read + fTotalReadRows = curRowNum1; // number of valid rows read + fTotalReadRowsForLog = curRowNum; // total number of rows read - if (pRawDataRow) - delete []pRawDataRow; + if (pRawDataRow) + delete[] pRawDataRow; } - + //------------------------------------------------------------------------------ // Resize the fTokens array used to store meta data about the input read buffer. // Used for initial allocation as well. //------------------------------------------------------------------------------ void BulkLoadBuffer::resizeTokenArray() { - unsigned tmpTotalRows = 0; + unsigned tmpTotalRows = 0; + + if (!fTokens) + { + tmpTotalRows = fBufferSize / 100; + + // Estimate the number of rows we can store in + // one buffer by getting length of first record + for (unsigned int k = 0; k < (fBufferSize - fOverflowSize); k++) + { + if (fData[k] == NEWLINE_CHAR) + { + tmpTotalRows = fBufferSize / (k + 1); + break; + } + } + } + else + { + tmpTotalRows = (unsigned int)(fTotalRows * 1.25); + + // @bug 3478: Make sure token array is expanded. + // If rows are loooong, then fTotalRows may be small (< 4), in which + // a 1.25 factor won't increase the row count. So this check is here + // to make sure we increase the row count in this case. + if (tmpTotalRows <= fTotalRows) + tmpTotalRows = fTotalRows * 2; + } + + if (fLog->isDebug(DEBUG_1)) + { + std::string allocLabel("Re-Allocating"); if (!fTokens) - { - tmpTotalRows = fBufferSize / 100; + allocLabel = "Allocating"; - // Estimate the number of rows we can store in - // one buffer by getting length of first record - for (unsigned int k = 0; k < (fBufferSize - fOverflowSize); k++) - { - if (fData[k] == NEWLINE_CHAR) - { - tmpTotalRows = fBufferSize / (k + 1); - break; - } - } - } - else - { - tmpTotalRows = (unsigned int)(fTotalRows * 1.25); + ostringstream oss; + oss << "Table: " << fTableName << "; ReadBuffer: " << fBufferId << "; " << allocLabel + << " ColValue metadata of size " << sizeof(ColPosPair) << " for " << tmpTotalRows << " rows and " + << fNumberOfColumns << " columns "; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - // @bug 3478: Make sure token array is expanded. - // If rows are loooong, then fTotalRows may be small (< 4), in which - // a 1.25 factor won't increase the row count. So this check is here - // to make sure we increase the row count in this case. - if (tmpTotalRows <= fTotalRows) - tmpTotalRows = fTotalRows * 2; - } + ColPosPair** tmp; + tmp = new ColPosPair*[tmpTotalRows]; - if (fLog->isDebug( DEBUG_1 )) - { - std::string allocLabel("Re-Allocating"); + if (fTokens) + { + memcpy(tmp, fTokens, sizeof(ColPosPair*) * fTotalRows); + delete[] fTokens; + } - if (!fTokens) - allocLabel = "Allocating"; + fTokens = tmp; - ostringstream oss; - oss << "Table: " << fTableName << - "; ReadBuffer: " << fBufferId << - "; " << allocLabel << " ColValue metadata of size " << - sizeof(ColPosPair) << " for " << tmpTotalRows << - " rows and " << fNumberOfColumns << " columns "; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + // Allocate a ColPosPair array for each new row + for (unsigned i = fTotalRows; i < tmpTotalRows; ++i) + fTokens[i] = new ColPosPair[fNumberOfColumns]; - ColPosPair** tmp; - tmp = new ColPosPair *[tmpTotalRows]; - - if (fTokens) - { - memcpy(tmp, fTokens, sizeof(ColPosPair*) * fTotalRows); - delete [] fTokens; - } - - fTokens = tmp; - - // Allocate a ColPosPair array for each new row - for (unsigned i = fTotalRows; i < tmpTotalRows ; ++i) - fTokens[i] = new ColPosPair[fNumberOfColumns]; - - fTotalRows = tmpTotalRows; + fTotalRows = tmpTotalRows; } - + //@bug 5027: Add tokenizeBinary() and isBinaryFieldNull() for binary imports //------------------------------------------------------------------------------ // Alternatve version of tokenize() uesd to import fixed length records in @@ -3085,377 +2944,359 @@ void BulkLoadBuffer::resizeTokenArray() // given call to tokenize() should exceed the value of "allowedErrCntThisCall", // then tokenize() will stop reading data and exit. //------------------------------------------------------------------------------ -int BulkLoadBuffer::tokenizeBinary( - const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall, - bool bEndOfData ) +int BulkLoadBuffer::tokenizeBinary(const boost::ptr_vector& columnsInfo, + unsigned int allowedErrCntThisCall, bool bEndOfData) { - unsigned curCol = 0; // dest db column counter within a row - unsigned curRowNum = 0; // "total" number of rows read during this call - unsigned curRowNum1 = 0; // number of "valid" rows inserted into fTokens - char* p; // iterates thru each field in the input buffer - char* lastRowHead = 0; // start of latest row being processed - bool bValidRow = true; // track whether current row is valid - bool bRowGenAutoInc = false; //track whether row uses generated auto-inc - std::string validationErrMsg;//validation error msg (if any) for current row - unsigned errorCount = 0; - int rc = NO_ERROR; + unsigned curCol = 0; // dest db column counter within a row + unsigned curRowNum = 0; // "total" number of rows read during this call + unsigned curRowNum1 = 0; // number of "valid" rows inserted into fTokens + char* p; // iterates thru each field in the input buffer + char* lastRowHead = 0; // start of latest row being processed + bool bValidRow = true; // track whether current row is valid + bool bRowGenAutoInc = false; // track whether row uses generated auto-inc + std::string validationErrMsg; // validation error msg (if any) for current row + unsigned errorCount = 0; + int rc = NO_ERROR; - p = lastRowHead = fData; + p = lastRowHead = fData; - ldiv_t rowcnt = ldiv(fReadSize, fFixedBinaryRecLen); + ldiv_t rowcnt = ldiv(fReadSize, fFixedBinaryRecLen); - //-------------------------------------------------------------------------- - // Loop through all the bytes in the read buffer in order to construct - // the meta data stored in fTokens. - //-------------------------------------------------------------------------- - for (long kRow = 0; kRow < rowcnt.quot; kRow++) + //-------------------------------------------------------------------------- + // Loop through all the bytes in the read buffer in order to construct + // the meta data stored in fTokens. + //-------------------------------------------------------------------------- + for (long kRow = 0; kRow < rowcnt.quot; kRow++) + { + //---------------------------------------------------------------------- + // Manage all the fields in a row + //---------------------------------------------------------------------- + for (unsigned int curFld = 0; curFld < fNumFieldsInFile; curFld++) { - //---------------------------------------------------------------------- - // Manage all the fields in a row - //---------------------------------------------------------------------- - for (unsigned int curFld = 0; curFld < fNumFieldsInFile; curFld++) + if (fFieldList[curFld].fFldColType == BULK_FLDCOL_COLUMN_FIELD) + { + const JobColumn& jobCol = columnsInfo[curCol].column; + + if (curCol < fNumColsInFile) { - if (fFieldList[curFld].fFldColType == BULK_FLDCOL_COLUMN_FIELD) + fTokens[curRowNum1][curCol].start = p - fData; + fTokens[curRowNum1][curCol].offset = jobCol.definedWidth; + + // Special auto-increment case; treat 0 as null value + if (jobCol.autoIncFlag) + { + if (memcmp(p, &NULL_AUTO_INC_0_BINARY, jobCol.definedWidth) == 0) { - const JobColumn& jobCol = columnsInfo[curCol].column; - - if (curCol < fNumColsInFile) - { - fTokens[curRowNum1][curCol].start = p - fData; - fTokens[curRowNum1][curCol].offset = jobCol.definedWidth; - - // Special auto-increment case; treat 0 as null value - if (jobCol.autoIncFlag) - { - if (memcmp(p, &NULL_AUTO_INC_0_BINARY, - jobCol.definedWidth) == 0) - { - fTokens[curRowNum1][curCol].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; - bRowGenAutoInc = true; - } - } - - switch (jobCol.weType) - { - case WR_CHAR: - { - // Detect empty string for CHAR and VARCHAR - if (*p == '\0') - fTokens[curRowNum1][curCol].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; - - break; - } - - case WR_VARBINARY: - { - // Detect empty VARBINARY field - int kk; - - for (kk = 0; kk < jobCol.definedWidth; kk++) - { - if (p[kk] != '\0') - break; - } - - if (kk >= jobCol.definedWidth) - fTokens[curRowNum1][curCol].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; - - break; - } - - default: - { - // In BinaryAcceptNULL mode, check for NULL value - if ((fTokens[curRowNum1][curCol].offset != - COLPOSPAIR_NULL_TOKEN_OFFSET) && - (fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL)) - { - if (isBinaryFieldNull(p, - jobCol.weType, - jobCol.dataType)) - { - fTokens[curRowNum1][curCol].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; - - if (jobCol.autoIncFlag) - bRowGenAutoInc = true; - } - } - - break; - } - } // end of switch (jobCol.weType) - - // Validate NotNull column is supplied a value or a default - if ((jobCol.fNotNull) && - (fTokens[curRowNum1][curCol].offset == - COLPOSPAIR_NULL_TOKEN_OFFSET) && - (!jobCol.fWithDefault) && - (bValidRow)) - { - bValidRow = false; - - ostringstream ossErrMsg; - ossErrMsg << INPUT_ERROR_NULL_CONSTRAINT << - "; field " << (curFld + 1); - validationErrMsg = ossErrMsg.str(); - } - } // end "if (curCol < fNumColsInFile)" - - p += jobCol.definedWidth; - curCol++; + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + bRowGenAutoInc = true; } - else + } + + switch (jobCol.weType) + { + case WR_CHAR: { - // This is where we would handle fields - // if they were supported in Binary Import mode - //p += ? + // Detect empty string for CHAR and VARCHAR + if (*p == '\0') + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + + break; } - } // end of loop through fields in a row - //---------------------------------------------------------------------- - // End-of-row processing - //---------------------------------------------------------------------- - - curRowNum++; // increment total number of rows read - - if (bValidRow) - { - // Initialize fTokens for tags not in input file - if (fNumColsInFile < fNumberOfColumns) + case WR_VARBINARY: { - for (unsigned int n = fNumColsInFile; n < fNumberOfColumns; n++) - { - fTokens[curRowNum1][n].start = 0; - fTokens[curRowNum1][n].offset = - COLPOSPAIR_NULL_TOKEN_OFFSET; + // Detect empty VARBINARY field + int kk; - if (columnsInfo[n].column.autoIncFlag) - bRowGenAutoInc = true; + for (kk = 0; kk < jobCol.definedWidth; kk++) + { + if (p[kk] != '\0') + break; + } + + if (kk >= jobCol.definedWidth) + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + + break; + } + + default: + { + // In BinaryAcceptNULL mode, check for NULL value + if ((fTokens[curRowNum1][curCol].offset != COLPOSPAIR_NULL_TOKEN_OFFSET) && + (fImportDataMode == IMPORT_DATA_BIN_ACCEPT_NULL)) + { + if (isBinaryFieldNull(p, jobCol.weType, jobCol.dataType)) + { + fTokens[curRowNum1][curCol].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; + + if (jobCol.autoIncFlag) + bRowGenAutoInc = true; } + } + + break; } + } // end of switch (jobCol.weType) - curRowNum1++; // increment valid row count + // Validate NotNull column is supplied a value or a default + if ((jobCol.fNotNull) && (fTokens[curRowNum1][curCol].offset == COLPOSPAIR_NULL_TOKEN_OFFSET) && + (!jobCol.fWithDefault) && (bValidRow)) + { + bValidRow = false; - if (bRowGenAutoInc) - fAutoIncGenCount++; // update number of generated auto-incs - } - else - { - // Store validation error message to be logged - string tmp(lastRowHead, fFixedBinaryRecLen); - fErrRows.push_back( tmp ); + ostringstream ossErrMsg; + ossErrMsg << INPUT_ERROR_NULL_CONSTRAINT << "; field " << (curFld + 1); + validationErrMsg = ossErrMsg.str(); + } + } // end "if (curCol < fNumColsInFile)" - fRowStatus.push_back(std::pair( - fStartRowForLogging + curRowNum, - validationErrMsg)); + p += jobCol.definedWidth; + curCol++; + } + else + { + // This is where we would handle fields + // if they were supported in Binary Import mode + // p += ? + } + } // end of loop through fields in a row - errorCount++; + //---------------------------------------------------------------------- + // End-of-row processing + //---------------------------------------------------------------------- - // Quit if we exceed max allowable errors for this call - if (errorCount > allowedErrCntThisCall) - break; - } + curRowNum++; // increment total number of rows read - curCol = 0; - lastRowHead += fFixedBinaryRecLen; - - // Resize fTokens array if we are about to fill it up - if ( curRowNum1 >= fTotalRows ) - { - resizeTokenArray(); - } - - bValidRow = true; - bRowGenAutoInc = false; - } // end of loop through the rows in the read buffer - - // Save any leftover data that we did not yet parse, into fOverflowBuf - if (rowcnt.rem > 0) + if (bValidRow) { - if (bEndOfData) + // Initialize fTokens for tags not in input file + if (fNumColsInFile < fNumberOfColumns) + { + for (unsigned int n = fNumColsInFile; n < fNumberOfColumns; n++) { - rc = ERR_BULK_BINARY_PARTIAL_REC; - ostringstream oss; - oss << "Incomplete record (" << rowcnt.rem << " bytes) at end " - "of import data; expected fixed length records of length " << - fFixedBinaryRecLen << " bytes"; - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - } - else - { - fOverflowSize = rowcnt.rem; - fOverflowBuf = new char[fOverflowSize]; + fTokens[curRowNum1][n].start = 0; + fTokens[curRowNum1][n].offset = COLPOSPAIR_NULL_TOKEN_OFFSET; - memcpy( fOverflowBuf, (fData + fReadSize - rowcnt.rem), - fOverflowSize ); + if (columnsInfo[n].column.autoIncFlag) + bRowGenAutoInc = true; } + } + + curRowNum1++; // increment valid row count + + if (bRowGenAutoInc) + fAutoIncGenCount++; // update number of generated auto-incs } else { - fOverflowSize = 0; - fOverflowBuf = NULL; + // Store validation error message to be logged + string tmp(lastRowHead, fFixedBinaryRecLen); + fErrRows.push_back(tmp); + + fRowStatus.push_back(std::pair(fStartRowForLogging + curRowNum, validationErrMsg)); + + errorCount++; + + // Quit if we exceed max allowable errors for this call + if (errorCount > allowedErrCntThisCall) + break; } - fTotalReadRows = curRowNum1; // number of valid rows read - fTotalReadRowsForLog = curRowNum; // total number of rows read + curCol = 0; + lastRowHead += fFixedBinaryRecLen; - return rc; + // Resize fTokens array if we are about to fill it up + if (curRowNum1 >= fTotalRows) + { + resizeTokenArray(); + } + + bValidRow = true; + bRowGenAutoInc = false; + } // end of loop through the rows in the read buffer + + // Save any leftover data that we did not yet parse, into fOverflowBuf + if (rowcnt.rem > 0) + { + if (bEndOfData) + { + rc = ERR_BULK_BINARY_PARTIAL_REC; + ostringstream oss; + oss << "Incomplete record (" << rowcnt.rem + << " bytes) at end " + "of import data; expected fixed length records of length " + << fFixedBinaryRecLen << " bytes"; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + } + else + { + fOverflowSize = rowcnt.rem; + fOverflowBuf = new char[fOverflowSize]; + + memcpy(fOverflowBuf, (fData + fReadSize - rowcnt.rem), fOverflowSize); + } + } + else + { + fOverflowSize = 0; + fOverflowBuf = NULL; + } + + fTotalReadRows = curRowNum1; // number of valid rows read + fTotalReadRowsForLog = curRowNum; // total number of rows read + + return rc; } - + //------------------------------------------------------------------------------ // Compare the numeric value (val) against the relevant NULL value, based on // column type (ct and dt), to see whether the specified value is NULL. //------------------------------------------------------------------------------ -bool BulkLoadBuffer::isBinaryFieldNull(void* val, - WriteEngine::ColType ct, +bool BulkLoadBuffer::isBinaryFieldNull(void* val, WriteEngine::ColType ct, execplan::CalpontSystemCatalog::ColDataType dt) { - bool isNullFlag = false; + bool isNullFlag = false; - switch (ct) + switch (ct) + { + case WriteEngine::WR_BYTE: { - case WriteEngine::WR_BYTE: - { - if ((*(uint8_t*)val) == joblist::TINYINTNULL) - isNullFlag = true; + if ((*(uint8_t*)val) == joblist::TINYINTNULL) + isNullFlag = true; - break; - } - - case WriteEngine::WR_SHORT: - { - if ((*(uint16_t*)val) == joblist::SMALLINTNULL) - isNullFlag = true; - - break; - } - - case WriteEngine::WR_INT: - { - if (dt == execplan::CalpontSystemCatalog::DATE) - { - if ((*(uint32_t*)val) == joblist::DATENULL) - isNullFlag = true; - } - else - { - if ((*(uint32_t*)val) == joblist::INTNULL) - isNullFlag = true; - } - - break; - } - - case WriteEngine::WR_LONGLONG: - { - if (dt == execplan::CalpontSystemCatalog::DATETIME) - { - if ((*(uint64_t*)val) == joblist::DATETIMENULL) - isNullFlag = true; - } - else if (dt == execplan::CalpontSystemCatalog::TIMESTAMP) - { - if ((*(uint64_t*)val) == joblist::TIMESTAMPNULL) - isNullFlag = true; - } - else if (dt == execplan::CalpontSystemCatalog::TIME) - { - if ((*(uint64_t*)val) == joblist::TIMENULL) - isNullFlag = true; - } - else - { - if ((*(uint64_t*)val) == joblist::BIGINTNULL) - isNullFlag = true; - } - - break; - } - - case WriteEngine::WR_FLOAT: - { - if ((*(uint32_t*)val) == joblist::FLOATNULL) - isNullFlag = true; - - break; - } - - case WriteEngine::WR_DOUBLE: - { - if ((*(uint64_t*)val) == joblist::DOUBLENULL) - isNullFlag = true; - - break; - } - - // Detect empty string for CHAR and VARCHAR - case WriteEngine::WR_CHAR: - { - // not applicable - break; - } - - // Detect empty VARBINARY field - case WriteEngine::WR_VARBINARY: - { - // not applicable - break; - } - - case WriteEngine::WR_UBYTE: - { - if ((*(uint8_t*)val) == joblist::UTINYINTNULL) - isNullFlag = true; - - break; - } - - case WriteEngine::WR_USHORT: - { - if ((*(uint16_t*)val) == joblist::USMALLINTNULL) - isNullFlag = true; - - break; - } - - case WriteEngine::WR_UINT: - { - if ((*(uint32_t*)val) == joblist::UINTNULL) - isNullFlag = true; - - break; - } - - case WriteEngine::WR_ULONGLONG: - { - if ((*(uint64_t*)val) == joblist::UBIGINTNULL) - isNullFlag = true; - - break; - } - - case WriteEngine::WR_BINARY: - { - if ((*((int128_t*)val)) == datatypes::Decimal128Null) - isNullFlag = true; - - break; - } - - default: - { - break; - } + break; } - return isNullFlag; + case WriteEngine::WR_SHORT: + { + if ((*(uint16_t*)val) == joblist::SMALLINTNULL) + isNullFlag = true; + + break; + } + + case WriteEngine::WR_INT: + { + if (dt == execplan::CalpontSystemCatalog::DATE) + { + if ((*(uint32_t*)val) == joblist::DATENULL) + isNullFlag = true; + } + else + { + if ((*(uint32_t*)val) == joblist::INTNULL) + isNullFlag = true; + } + + break; + } + + case WriteEngine::WR_LONGLONG: + { + if (dt == execplan::CalpontSystemCatalog::DATETIME) + { + if ((*(uint64_t*)val) == joblist::DATETIMENULL) + isNullFlag = true; + } + else if (dt == execplan::CalpontSystemCatalog::TIMESTAMP) + { + if ((*(uint64_t*)val) == joblist::TIMESTAMPNULL) + isNullFlag = true; + } + else if (dt == execplan::CalpontSystemCatalog::TIME) + { + if ((*(uint64_t*)val) == joblist::TIMENULL) + isNullFlag = true; + } + else + { + if ((*(uint64_t*)val) == joblist::BIGINTNULL) + isNullFlag = true; + } + + break; + } + + case WriteEngine::WR_FLOAT: + { + if ((*(uint32_t*)val) == joblist::FLOATNULL) + isNullFlag = true; + + break; + } + + case WriteEngine::WR_DOUBLE: + { + if ((*(uint64_t*)val) == joblist::DOUBLENULL) + isNullFlag = true; + + break; + } + + // Detect empty string for CHAR and VARCHAR + case WriteEngine::WR_CHAR: + { + // not applicable + break; + } + + // Detect empty VARBINARY field + case WriteEngine::WR_VARBINARY: + { + // not applicable + break; + } + + case WriteEngine::WR_UBYTE: + { + if ((*(uint8_t*)val) == joblist::UTINYINTNULL) + isNullFlag = true; + + break; + } + + case WriteEngine::WR_USHORT: + { + if ((*(uint16_t*)val) == joblist::USMALLINTNULL) + isNullFlag = true; + + break; + } + + case WriteEngine::WR_UINT: + { + if ((*(uint32_t*)val) == joblist::UINTNULL) + isNullFlag = true; + + break; + } + + case WriteEngine::WR_ULONGLONG: + { + if ((*(uint64_t*)val) == joblist::UBIGINTNULL) + isNullFlag = true; + + break; + } + + case WriteEngine::WR_BINARY: + { + if ((*((int128_t*)val)) == datatypes::Decimal128Null) + isNullFlag = true; + + break; + } + + default: + { + break; + } + } + + return isNullFlag; } - + //------------------------------------------------------------------------------ // Sets the column status. // returns TRUE if all columns in the buffer are complete. @@ -3476,18 +3317,17 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val, // So I added this note and removed the extraneous fSyncUpdatesBLB lock from // setColumnStatus() and tryAndLockColumn(). (dmc-07/19/2009) //------------------------------------------------------------------------------ -bool BulkLoadBuffer::setColumnStatus(const int& columnId, - const Status& status) +bool BulkLoadBuffer::setColumnStatus(const int& columnId, const Status& status) { - fColumnLocks[columnId].status = status; + fColumnLocks[columnId].status = status; - if (status == WriteEngine::PARSE_COMPLETE) - fParseComplete++; + if (status == WriteEngine::PARSE_COMPLETE) + fParseComplete++; - if (fParseComplete == fNumberOfColumns) - return true; + if (fParseComplete == fNumberOfColumns) + return true; - return false; + return false; } //------------------------------------------------------------------------------ @@ -3496,14 +3336,13 @@ bool BulkLoadBuffer::setColumnStatus(const int& columnId, //------------------------------------------------------------------------------ bool BulkLoadBuffer::tryAndLockColumn(const int& columnId, const int& id) { - if ((fColumnLocks[columnId].status != WriteEngine::PARSE_COMPLETE) && - (fColumnLocks[columnId].locker == -1)) - { - fColumnLocks[columnId].locker = id; - return true; - } + if ((fColumnLocks[columnId].status != WriteEngine::PARSE_COMPLETE) && (fColumnLocks[columnId].locker == -1)) + { + fColumnLocks[columnId].locker = id; + return true; + } - return false; + return false; } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index e953c1b82..b45ca80a0 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -39,387 +39,368 @@ class Log; // Used to collect stats about a BulkLoadBuffer buffer that is being parsed class BLBufferStats { -public: - int64_t minBufferVal; - int64_t maxBufferVal; - int64_t satCount; - union + public: + int64_t minBufferVal; + int64_t maxBufferVal; + int64_t satCount; + union + { + int128_t bigMinBufferVal; + int64_t minBufferVal_; + }; + union + { + int128_t bigMaxBufferVal; + int64_t maxBufferVal_; + }; + BLBufferStats(ColDataType colDataType) : satCount(0) + { + if (isUnsigned(colDataType) || isCharType(colDataType)) { - int128_t bigMinBufferVal; - int64_t minBufferVal_; - }; - union - { - int128_t bigMaxBufferVal; - int64_t maxBufferVal_; - }; - BLBufferStats(ColDataType colDataType) : satCount(0) - { - if (isUnsigned(colDataType) || isCharType(colDataType)) - { - minBufferVal = static_cast(MAX_UBIGINT); - maxBufferVal = static_cast(MIN_UBIGINT); - bigMinBufferVal = -1; - bigMaxBufferVal = 0; - } - else - { - minBufferVal = MAX_BIGINT; - maxBufferVal = MIN_BIGINT; - utils::int128Max(bigMinBufferVal); - utils::int128Min(bigMaxBufferVal); - } + minBufferVal = static_cast(MAX_UBIGINT); + maxBufferVal = static_cast(MIN_UBIGINT); + bigMinBufferVal = -1; + bigMaxBufferVal = 0; } + else + { + minBufferVal = MAX_BIGINT; + maxBufferVal = MIN_BIGINT; + utils::int128Max(bigMinBufferVal); + utils::int128Min(bigMaxBufferVal); + } + } }; class BulkLoadBuffer { -private: + private: + //-------------------------------------------------------------------------- + // Private Data Members + //-------------------------------------------------------------------------- - //-------------------------------------------------------------------------- - // Private Data Members - //-------------------------------------------------------------------------- + char* fData; // Buffer with data read from tbl file + char* fDataParser; // for temporary use by parser - char* fData; // Buffer with data read from tbl file - char* fDataParser; // for temporary use by parser + char* fOverflowBuf; // Overflow data held for next buffer + unsigned fOverflowSize; // Current size of fOverflowBuf - char* fOverflowBuf; // Overflow data held for next buffer - unsigned fOverflowSize; // Current size of fOverflowBuf + // Information about the locker and status for each column in this buffer. + // Note that TableInfo::fSyncUpdatesTI mutex is used to synchronize + // access to fColumnLocks and fParseComplete from both read and parse + // threads. Table-scope lock (fSyncUpdatesTI) is used instead of + // buffer-scope lock (fSyncUpdatesBLB), to keep state for all buffers + // static while we are scanning fColumnLocks for the buffers in a table + std::vector fColumnLocks; - // Information about the locker and status for each column in this buffer. - // Note that TableInfo::fSyncUpdatesTI mutex is used to synchronize - // access to fColumnLocks and fParseComplete from both read and parse - // threads. Table-scope lock (fSyncUpdatesTI) is used instead of - // buffer-scope lock (fSyncUpdatesBLB), to keep state for all buffers - // static while we are scanning fColumnLocks for the buffers in a table - std::vector fColumnLocks; + // Note that TableInfo::fSyncUpdatesTI mutex (not fStatusBLB) is used + // to synchronize getting/setting fStatusBLB between threads. + Status fStatusBLB; // Status of buffer - // Note that TableInfo::fSyncUpdatesTI mutex (not fStatusBLB) is used - // to synchronize getting/setting fStatusBLB between threads. - Status fStatusBLB; // Status of buffer + // TableInfo::fSyncUpdatesTI mutex should be locked when accessing + // this data member (see fColumnLocks discussion). + unsigned fParseComplete; // Num of columns that are parseComplete - // TableInfo::fSyncUpdatesTI mutex should be locked when accessing - // this data member (see fColumnLocks discussion). - unsigned fParseComplete; // Num of columns that are parseComplete + unsigned fTotalRows; // Max rows this buffer can now hold; + // size of fTokens array + std::vector > fRowStatus; // Status of bad rows + std::vector fErrRows; // Rejected rows to write to .bad file - unsigned fTotalRows; // Max rows this buffer can now hold; - // size of fTokens array - std::vector< std::pair > fRowStatus; //Status of bad rows - std::vector fErrRows; // Rejected rows to write to .bad file + uint32_t fTotalReadRows; // Total valid rows read into buffer; + // this count excludes rejected rows + uint32_t fTotalReadRowsParser; // for temporary use by parser - uint32_t fTotalReadRows; // Total valid rows read into buffer; - // this count excludes rejected rows - uint32_t fTotalReadRowsParser; // for temporary use by parser + uint32_t fTotalReadRowsForLog; // Total rows read into this buffer + // including invalid rows - uint32_t fTotalReadRowsForLog; // Total rows read into this buffer - // including invalid rows + RID fStartRow; // Starting row id for rows in buffer, + // relative to start of job. + // Rejected rows are excluded. + RID fStartRowParser; // for temporary use by parser - RID fStartRow; // Starting row id for rows in buffer, - // relative to start of job. - // Rejected rows are excluded. - RID fStartRowParser; // for temporary use by parser + RID fStartRowForLogging; // Starting row id for rows in buffer, + // relative to start of current input + // file. All rows are counted. + RID fStartRowForLoggingParser; // for temporary use by parser - RID fStartRowForLogging; // Starting row id for rows in buffer, - // relative to start of current input - // file. All rows are counted. - RID fStartRowForLoggingParser; // for temporary use by parser + uint32_t fAutoIncGenCount; // How many auto-increment values are + // to be generated for current buffer + uint32_t fAutoIncGenCountParser; // for temporary use by parser - uint32_t fAutoIncGenCount; // How many auto-increment values are - // to be generated for current buffer - uint32_t fAutoIncGenCountParser; // for temporary use by parser + uint64_t fAutoIncNextValue; // Next auto-increment value assign to + // a row in this buffer + unsigned fNumberOfColumns; // Number of ColumnInfo objs in table - uint64_t fAutoIncNextValue; // Next auto-increment value assign to - // a row in this buffer - unsigned fNumberOfColumns; // Number of ColumnInfo objs in table + ColPosPair** fTokens; // Vector of start and offsets for the + // column values read from tbl files + ColPosPair** fTokensParser; // for temporary use by parser - ColPosPair** fTokens; // Vector of start and offsets for the - // column values read from tbl files - ColPosPair** fTokensParser; // for temporary use by parser + char fColDelim; // Character to delimit columns in a row + unsigned fBufferSize; // Size of input read buffer (fData) + unsigned fReadSize; // Number of bytes in read buffer(fData) + boost::mutex fSyncUpdatesBLB; // Mutex to synchronize updates + Log* fLog; // Logger object + bool fNullStringMode; // Indicates if "NULL" string is to be + // treated as a NULL value or not + char fEnclosedByChar; // Optional char to enclose col values + char fEscapeChar; // Used to escape enclosed character + int fBufferId; // Id for this read buffer + std::string fTableName; // Table assigned to this read buffer + JobFieldRefList fFieldList; // Complete list of cols and flds + unsigned int fNumFieldsInFile; // Number of fields in input file + // (including fields to be ignored) + unsigned int fNumColsInFile; // Number of flds in input file targeted + // for db cols (omits default cols) + bool fbTruncationAsError; // Treat string truncation as error + ImportDataMode fImportDataMode; // Import data in text or binary mode + std::string fTimeZone; // Timezone used by TIMESTAMP datatype + unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode - char fColDelim; // Character to delimit columns in a row - unsigned fBufferSize; // Size of input read buffer (fData) - unsigned fReadSize; // Number of bytes in read buffer(fData) - boost::mutex fSyncUpdatesBLB; // Mutex to synchronize updates - Log* fLog; // Logger object - bool fNullStringMode; // Indicates if "NULL" string is to be - // treated as a NULL value or not - char fEnclosedByChar; // Optional char to enclose col values - char fEscapeChar; // Used to escape enclosed character - int fBufferId; // Id for this read buffer - std::string fTableName; // Table assigned to this read buffer - JobFieldRefList fFieldList; // Complete list of cols and flds - unsigned int fNumFieldsInFile; // Number of fields in input file - // (including fields to be ignored) - unsigned int fNumColsInFile; // Number of flds in input file targeted - // for db cols (omits default cols) - bool fbTruncationAsError; // Treat string truncation as error - ImportDataMode fImportDataMode; // Import data in text or binary mode - std::string fTimeZone; // Timezone used by TIMESTAMP datatype - unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode + //-------------------------------------------------------------------------- + // Private Functions + //-------------------------------------------------------------------------- - //-------------------------------------------------------------------------- - // Private Functions - //-------------------------------------------------------------------------- + /** @brief Copy constructor + */ + BulkLoadBuffer(const BulkLoadBuffer& buffer); - /** @brief Copy constructor - */ - BulkLoadBuffer(const BulkLoadBuffer& buffer); + /** @brief Assignment operator + */ + BulkLoadBuffer& operator=(const BulkLoadBuffer& buffer); - /** @brief Assignment operator - */ - BulkLoadBuffer& operator =(const BulkLoadBuffer& buffer); + /** @brief Convert the buffer data depending upon the data type + */ + void convert(char* field, int fieldLength, bool nullFlag, unsigned char* output, const JobColumn& column, + BLBufferStats& bufStats); - /** @brief Convert the buffer data depending upon the data type - */ - void convert(char* field, int fieldLength, - bool nullFlag, unsigned char* output, - const JobColumn& column, - BLBufferStats& bufStats); + /** @brief Copy the overflow data + */ + void copyOverflow(const BulkLoadBuffer& buffer); - /** @brief Copy the overflow data - */ - void copyOverflow(const BulkLoadBuffer& buffer); + /** @brief Parse a Read buffer for a nonDictionary column + */ + int parseCol(ColumnInfo& columnInfo); - /** @brief Parse a Read buffer for a nonDictionary column - */ - int parseCol(ColumnInfo& columnInfo); + /** @brief Parse a Read buffer for a nonDictionary column + */ + void parseColLogMinMax(std::ostringstream& oss, ColDataType colDataType, int64_t minBufferVal, + int64_t maxBufferVal) const; - /** @brief Parse a Read buffer for a nonDictionary column - */ - void parseColLogMinMax(std::ostringstream& oss, - ColDataType colDataType, - int64_t minBufferVal, - int64_t maxBufferVal) const; + /** @brief Parse a Read buffer for a Dictionary column + */ + int parseDict(ColumnInfo& columnInfo); - /** @brief Parse a Read buffer for a Dictionary column - */ - int parseDict(ColumnInfo& columnInfo); + /** @brief Parse a Dictionary Read buffer into a ColumnBufferSection. + * + * Parses the Read buffer into a section up to the point at which + * the buffer crosses an extent boundary. + * + * @param columnInfo Column being parsed + * @oaram tokenPos Position of rows to be parsed, in fTokens. + * @param startRow Row id of first row in buffer to be parsed. + * Row id is relative to all the rows in this import. + * @param totalReadRows Number of buffer rows ready to be parsed + * @param nRowsParsed Number of buffer rows that were parsed + */ + int parseDictSection(ColumnInfo& columnInfo, int tokenPos, RID startRow, uint32_t totalReadRows, + uint32_t& nRowsParsed); - /** @brief Parse a Dictionary Read buffer into a ColumnBufferSection. - * - * Parses the Read buffer into a section up to the point at which - * the buffer crosses an extent boundary. - * - * @param columnInfo Column being parsed - * @oaram tokenPos Position of rows to be parsed, in fTokens. - * @param startRow Row id of first row in buffer to be parsed. - * Row id is relative to all the rows in this import. - * @param totalReadRows Number of buffer rows ready to be parsed - * @param nRowsParsed Number of buffer rows that were parsed - */ - int parseDictSection(ColumnInfo& columnInfo, int tokenPos, - RID startRow, uint32_t totalReadRows, - uint32_t& nRowsParsed); + /** @brief Expand the size of the fTokens array + */ + void resizeTokenArray(); - /** @brief Expand the size of the fTokens array - */ - void resizeTokenArray(); + /** @brief tokenize the buffer contents and fill up the token array. + */ + void tokenize(const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall); - /** @brief tokenize the buffer contents and fill up the token array. - */ - void tokenize(const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall); + /** @brief Binary tokenization of the buffer, and fill up the token array. + */ + int tokenizeBinary(const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall, + bool bEndOfData); - /** @brief Binary tokenization of the buffer, and fill up the token array. - */ - int tokenizeBinary(const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall, - bool bEndOfData); + /** @brief Determine if specified value is NULL or not. + */ + bool isBinaryFieldNull(void* val, WriteEngine::ColType ct, execplan::CalpontSystemCatalog::ColDataType dt); - /** @brief Determine if specified value is NULL or not. - */ - bool isBinaryFieldNull(void* val, WriteEngine::ColType ct, - execplan::CalpontSystemCatalog::ColDataType dt); + public: + /** @brief Constructor + * @param noOfCol Number of columns + * @param bufferSize Buffer size + * @param logger The Log object used for logging + * @param bufferId Id assigned to this buffer + * @param tableName Name of table associated with this read buffer + * @param jobFieldRefList Complete list of cols/flds listed in Job XML file + */ + BulkLoadBuffer(unsigned noOfCols, unsigned bufferSize, Log* logger, int bufferId, + const std::string& tableName, const JobFieldRefList& jobFieldRefList); -public: + /** @brief Destructor + */ + ~BulkLoadBuffer(); - /** @brief Constructor - * @param noOfCol Number of columns - * @param bufferSize Buffer size - * @param logger The Log object used for logging - * @param bufferId Id assigned to this buffer - * @param tableName Name of table associated with this read buffer - * @param jobFieldRefList Complete list of cols/flds listed in Job XML file - */ - BulkLoadBuffer(unsigned noOfCols, - unsigned bufferSize, Log* logger, - int bufferId, const std::string& tableName, - const JobFieldRefList& jobFieldRefList); + /** @brief Resets the values of the members (excluding column locks) + */ + void reset(); - /** @brief Destructor - */ - ~BulkLoadBuffer(); + /** @brief Resets the column locks. + * TableInfo::fSyncUpdatesTI mutex should be locked when calling this + * function (see fColumnLocks discussion). + */ + void resetColumnLocks(); - /** @brief Resets the values of the members (excluding column locks) - */ - void reset(); + /** @brief Get the buffer status + */ + Status getStatusBLB() const + { + return fStatusBLB; + } - /** @brief Resets the column locks. - * TableInfo::fSyncUpdatesTI mutex should be locked when calling this - * function (see fColumnLocks discussion). - */ - void resetColumnLocks(); + /** @brief Set the buffer status + */ + void setStatusBLB(const Status& status) + { + fStatusBLB = status; + } - /** @brief Get the buffer status - */ - Status getStatusBLB() const - { - return fStatusBLB; - } + /** @brief Try to lock a column for the buffer + * TableInfo::fSyncUpdatesTI mutex should be locked when calling this + * function (see fColumnLocks discussion). + */ + bool tryAndLockColumn(const int& columnId, const int& id); - /** @brief Set the buffer status - */ - void setStatusBLB(const Status& status) - { - fStatusBLB = status; - } + int fillFromMemory(const BulkLoadBuffer& overFlowBufIn, const char* input, size_t length, + size_t* parse_length, RID& totalReadRows, RID& correctTotalRows, + const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall); - /** @brief Try to lock a column for the buffer - * TableInfo::fSyncUpdatesTI mutex should be locked when calling this - * function (see fColumnLocks discussion). - */ - bool tryAndLockColumn(const int& columnId, const int& id); + /** @brief Read the table data into the buffer + */ + int fillFromFile(const BulkLoadBuffer& overFlowBufIn, FILE* handle, RID& totalRows, RID& correctTotalRows, + const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall); - int fillFromMemory( - const BulkLoadBuffer& overFlowBufIn, - const char* input, size_t length, size_t *parse_length, RID& totalReadRows, - RID& correctTotalRows, const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall ); + /** @brief Get the overflow size + */ + int getOverFlowSize() const + { + return fOverflowSize; + } - /** @brief Read the table data into the buffer - */ - int fillFromFile(const BulkLoadBuffer& overFlowBufIn, - FILE* handle, RID& totalRows, RID& correctTotalRows, - const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall); + /** @brief Parse the buffer data + */ + int parse(ColumnInfo& columnInfo); - /** @brief Get the overflow size - */ - int getOverFlowSize() const - { - return fOverflowSize; - } + /** @brief Set the delimiter used to delimit the columns within a row + */ + void setColDelimiter(const char& delim) + { + fColDelim = delim; + } - /** @brief Parse the buffer data - */ - int parse(ColumnInfo& columnInfo); + /** @brief Set mode to treat "NULL" string as NULL value or not. + */ + void setNullStringMode(bool bMode) + { + fNullStringMode = bMode; + } - /** @brief Set the delimiter used to delimit the columns within a row - */ - void setColDelimiter(const char& delim) - { - fColDelim = delim; - } + /** @brief Set character optionally used to enclose input column values. + */ + void setEnclosedByChar(char enChar) + { + fEnclosedByChar = enChar; + } - /** @brief Set mode to treat "NULL" string as NULL value or not. - */ - void setNullStringMode( bool bMode ) - { - fNullStringMode = bMode; - } + /** @brief Set escape char to use in conjunction with enclosed by char. + */ + void setEscapeChar(char esChar) + { + fEscapeChar = esChar; + } - /** @brief Set character optionally used to enclose input column values. - */ - void setEnclosedByChar( char enChar ) - { - fEnclosedByChar = enChar; - } + /** @brief Get the column status + * TableInfo::fSyncUpdatesTI mutex should be locked when calling this + * function (see fColumnLocks discussion). + */ + Status getColumnStatus(const int& columnId) const + { + return fColumnLocks[columnId].status; + } - /** @brief Set escape char to use in conjunction with enclosed by char. - */ - void setEscapeChar ( char esChar ) - { - fEscapeChar = esChar; - } + /** @brief Set the column status + * TableInfo::fSyncUpdatesTI mutex should be locked when calling this + * function (see fColumnLocks discussion). + * @returns TRUE if all columns in the buffer are complete. + */ + bool setColumnStatus(const int& columnId, const Status& status); - /** @brief Get the column status - * TableInfo::fSyncUpdatesTI mutex should be locked when calling this - * function (see fColumnLocks discussion). - */ - Status getColumnStatus(const int& columnId) const - { - return fColumnLocks[columnId].status; - } + /** @brief Get the error row status's + */ + const std::vector >& getErrorRows() const + { + return fRowStatus; + } - /** @brief Set the column status - * TableInfo::fSyncUpdatesTI mutex should be locked when calling this - * function (see fColumnLocks discussion). - * @returns TRUE if all columns in the buffer are complete. - */ - bool setColumnStatus(const int& columnId, const Status& status); + /** @brief Get the error rows + */ + const std::vector& getExactErrorRows() const + { + return fErrRows; + } - /** @brief Get the error row status's - */ - const std::vector< std::pair >& getErrorRows() const - { - return fRowStatus; - } + void clearErrRows() + { + fRowStatus.clear(); + fErrRows.clear(); + } - /** @brief Get the error rows - */ - const std::vector& getExactErrorRows() const - { - return fErrRows; - } + /** @brief Get the column locker. + * TableInfo::fSyncUpdatesTI mutex should be locked when calling this + * function (see fColumnLocks discussion). + */ + int getColumnLocker(const int& columnId) const + { + return fColumnLocks[columnId].locker; + } - void clearErrRows() - { - fRowStatus.clear(); - fErrRows.clear(); - } + /** @brief set truncation as error for this import. + */ + void setTruncationAsError(bool bTruncationAsError) + { + fbTruncationAsError = bTruncationAsError; + } - /** @brief Get the column locker. - * TableInfo::fSyncUpdatesTI mutex should be locked when calling this - * function (see fColumnLocks discussion). - */ - int getColumnLocker(const int& columnId) const - { - return fColumnLocks[columnId].locker; - } + /** @brief retrieve the tuncation as error setting for this + * import. When set, this causes char and varchar strings + * that are longer than the column definition to be treated + * as errors instead of warnings. + */ + bool getTruncationAsError() const + { + return fbTruncationAsError; + } - /** @brief set truncation as error for this import. - */ - void setTruncationAsError(bool bTruncationAsError) - { - fbTruncationAsError = bTruncationAsError; - } + /** @brief Set text vs binary import mode along with corresponding fixed + * record length that is used if the binary mode is set to TRUE. + */ + void setImportDataMode(ImportDataMode importMode, unsigned int fixedBinaryRecLen) + { + fImportDataMode = importMode; + fFixedBinaryRecLen = fixedBinaryRecLen; + } - /** @brief retrieve the tuncation as error setting for this - * import. When set, this causes char and varchar strings - * that are longer than the column definition to be treated - * as errors instead of warnings. - */ - bool getTruncationAsError() const - { - return fbTruncationAsError; - } - - /** @brief Set text vs binary import mode along with corresponding fixed - * record length that is used if the binary mode is set to TRUE. - */ - void setImportDataMode( ImportDataMode importMode, - unsigned int fixedBinaryRecLen ) - { - fImportDataMode = importMode; - fFixedBinaryRecLen = fixedBinaryRecLen; - } - - /** @brief set timezone. - */ - void setTimeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } + /** @brief set timezone. + */ + void setTimeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } }; -inline bool isTrueWord(const char *field, int fieldLength) +inline bool isTrueWord(const char* field, int fieldLength) { - //return false; - return fieldLength == 4 && ( field[0] == 'T' || field[0] == 't' ) - && ( field[1] == 'R' || field[1] == 'r' ) - && ( field[2] == 'U' || field[2] == 'u' ) - && ( field[3] == 'E' || field[3] == 'e' ); + // return false; + return fieldLength == 4 && (field[0] == 'T' || field[0] == 't') && (field[1] == 'R' || field[1] == 'r') && + (field[2] == 'U' || field[2] == 'u') && (field[3] == 'E' || field[3] == 'e'); } -} +} // namespace WriteEngine #endif diff --git a/writeengine/bulk/we_bulkstatus.cpp b/writeengine/bulk/we_bulkstatus.cpp index b8bc27dde..5d5799a1e 100644 --- a/writeengine/bulk/we_bulkstatus.cpp +++ b/writeengine/bulk/we_bulkstatus.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_bulkstatus.cpp 4648 2013-05-29 21:42:40Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_bulkstatus.cpp 4648 2013-05-29 21:42:40Z rdempsey $ + * + *******************************************************************************/ #include #include @@ -31,4 +31,4 @@ namespace WriteEngine { /*static*/ volatile int BulkStatus::fJobStatus = EXIT_SUCCESS; -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_bulkstatus.h b/writeengine/bulk/we_bulkstatus.h index 320c7291c..b06b18e0d 100644 --- a/writeengine/bulk/we_bulkstatus.h +++ b/writeengine/bulk/we_bulkstatus.h @@ -16,15 +16,15 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_bulkstatus.h 4648 2013-05-29 21:42:40Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_bulkstatus.h 4648 2013-05-29 21:42:40Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_BULKSTATUS_H_ #define _WE_BULKSTATUS_H_ -#if 0 //defined(_MSC_VER) && defined(WE_BULKSTATUS_DLLEXPORT) +#if 0 // defined(_MSC_VER) && defined(WE_BULKSTATUS_DLLEXPORT) #define EXPORT __declspec(dllexport) #else #define EXPORT @@ -32,34 +32,33 @@ namespace WriteEngine { - // Defined this class to hold the global JobStatus flag rather then storing in // BulkLoad, because that would introduce circular dependencies, with the other // classes needing BulkLoad. So put the JobStatus in a separate class. class BulkStatus { -public: - static int getJobStatus() - { - return fJobStatus; - } - static void setJobStatus(int jobStatus) - { - fJobStatus = jobStatus; - } + public: + static int getJobStatus() + { + return fJobStatus; + } + static void setJobStatus(int jobStatus) + { + fJobStatus = jobStatus; + } -private: - /* @brief Global job status flag. - * Declared volatile to insure that all threads see when this flag is - * changed. We don't worry about using a mutex since we are just using - * as a flag. Making the variable volatile should suffice, to make it - * work with multiple threads. - */ - static volatile int fJobStatus; + private: + /* @brief Global job status flag. + * Declared volatile to insure that all threads see when this flag is + * changed. We don't worry about using a mutex since we are just using + * as a flag. Making the variable volatile should suffice, to make it + * work with multiple threads. + */ + static volatile int fJobStatus; }; -} // end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_BULKSTATUS_H_ +#endif // _WE_BULKSTATUS_H_ diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index 79619c856..bebde6b06 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -36,33 +36,32 @@ using namespace idbdatafile; namespace WriteEngine { - -ColumnBuffer::ColumnBuffer(ColumnInfo* pColInfo, Log* logger) : - fBuffer(0), fBufSize(0), fFile(0), fColInfo(pColInfo), fLog(logger) +ColumnBuffer::ColumnBuffer(ColumnInfo* pColInfo, Log* logger) + : fBuffer(0), fBufSize(0), fFile(0), fColInfo(pColInfo), fLog(logger) { } ColumnBuffer::~ColumnBuffer() { - delete[] fBuffer; + delete[] fBuffer; } int ColumnBuffer::finishFile(bool /*bTruncFile*/) { - return NO_ERROR; + return NO_ERROR; } -int ColumnBuffer::resetToBeCompressedColBuf( long long& startFileOffset ) +int ColumnBuffer::resetToBeCompressedColBuf(long long& startFileOffset) { - return NO_ERROR; + return NO_ERROR; } int ColumnBuffer::setDbFile(IDBDataFile* f, HWM startHwm, const char* /*hdrs*/) { - fFile = f; - fStartingHwm = startHwm; + fFile = f; + fStartingHwm = startHwm; - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -74,29 +73,28 @@ int ColumnBuffer::setDbFile(IDBDataFile* f, HWM startHwm, const char* /*hdrs*/) //------------------------------------------------------------------------------ void ColumnBuffer::resizeAndCopy(int newSize, int startOffset, int endOffset) { - unsigned char* old_buffer = fBuffer; - fBuffer = new unsigned char[newSize]; + unsigned char* old_buffer = fBuffer; + fBuffer = new unsigned char[newSize]; - if (startOffset != -1) + if (startOffset != -1) + { + int destBufferOffset = 0; + + // If the data in "buffer" wraps around, then here's where we copy + // the data at the end of the buffer, before copying the data + // at the start of the bufffer. + if (endOffset < startOffset) { - int destBufferOffset = 0; - - // If the data in "buffer" wraps around, then here's where we copy - // the data at the end of the buffer, before copying the data - // at the start of the bufffer. - if (endOffset < startOffset) - { - memcpy(fBuffer, old_buffer + startOffset, fBufSize - startOffset); - destBufferOffset = fBufSize - startOffset; - startOffset = 0; - } - - memcpy(fBuffer + destBufferOffset, - old_buffer + startOffset, endOffset - startOffset + 1); + memcpy(fBuffer, old_buffer + startOffset, fBufSize - startOffset); + destBufferOffset = fBufSize - startOffset; + startOffset = 0; } - fBufSize = newSize; - delete[] old_buffer; + memcpy(fBuffer + destBufferOffset, old_buffer + startOffset, endOffset - startOffset + 1); + } + + fBufSize = newSize; + delete[] old_buffer; } //------------------------------------------------------------------------------ @@ -106,46 +104,42 @@ void ColumnBuffer::resizeAndCopy(int newSize, int startOffset, int endOffset) //------------------------------------------------------------------------------ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmpties) { - if (writeSize == 0) // skip unnecessary write, if 0 bytes given - return NO_ERROR; - - unsigned char *newBuf = NULL; - - if ( fillUpWEmpties ) - { - BlockOp blockOp; - newBuf = new unsigned char[BYTE_PER_BLOCK]; - blockOp.findTypeHandler(fColInfo->column.width, - fColInfo->column.dataType); - const uint8_t* emptyVal = blockOp.getEmptyRowValue(fColInfo->column.dataType, - fColInfo->column.width); - - ::memcpy(static_cast(newBuf), - static_cast(fBuffer + startOffset), writeSize); - blockOp.setEmptyBuf(newBuf + writeSize, BYTE_PER_BLOCK - writeSize, - emptyVal, fColInfo->column.width); - } -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_WRITE_COL); -#endif - size_t nitems; - if ( fillUpWEmpties ) - nitems = fFile->write(newBuf, BYTE_PER_BLOCK) / BYTE_PER_BLOCK; - else - nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; - - if (nitems != 1) - { - delete [] newBuf; - return ERR_FILE_WRITE; - } - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WRITE_COL); -#endif - - delete [] newBuf; + if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + + unsigned char* newBuf = NULL; + + if (fillUpWEmpties) + { + BlockOp blockOp; + newBuf = new unsigned char[BYTE_PER_BLOCK]; + blockOp.findTypeHandler(fColInfo->column.width, fColInfo->column.dataType); + const uint8_t* emptyVal = blockOp.getEmptyRowValue(fColInfo->column.dataType, fColInfo->column.width); + + ::memcpy(static_cast(newBuf), static_cast(fBuffer + startOffset), writeSize); + blockOp.setEmptyBuf(newBuf + writeSize, BYTE_PER_BLOCK - writeSize, emptyVal, fColInfo->column.width); + } +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_WRITE_COL); +#endif + size_t nitems; + if (fillUpWEmpties) + nitems = fFile->write(newBuf, BYTE_PER_BLOCK) / BYTE_PER_BLOCK; + else + nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; + + if (nitems != 1) + { + delete[] newBuf; + return ERR_FILE_WRITE; + } + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_WRITE_COL); +#endif + + delete[] newBuf; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -153,7 +147,7 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmptie //------------------------------------------------------------------------------ void ColumnBuffer::write(const void* const data, int startOffset, int bytes) const { - memcpy(fBuffer + startOffset, data, bytes); + memcpy(fBuffer + startOffset, data, bytes); } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colbuf.h b/writeengine/bulk/we_colbuf.h index 72efa1cee..c9b28c63a 100644 --- a/writeengine/bulk/we_colbuf.h +++ b/writeengine/bulk/we_colbuf.h @@ -44,90 +44,86 @@ class ColumnInfo; */ class ColumnBuffer { + public: + /** @brief default Constructor + */ + ColumnBuffer(ColumnInfo* pColInfo, Log* logger); -public: + /** @brief default Destructor + */ + virtual ~ColumnBuffer(); - /** @brief default Constructor - */ - ColumnBuffer(ColumnInfo* pColInfo, Log* logger); + /** @brief Final flushing of data and headers prior to closing the file. + * This is a no-op for uncompressed columns. + * @param bTruncFile is file to be truncated + * @return NO_ERROR or success + */ + virtual int finishFile(bool bTruncFile); - /** @brief default Destructor - */ - virtual ~ColumnBuffer(); + /** @brief Returns size of the buffer + */ + int getSize() const + { + return fBufSize; + } - /** @brief Final flushing of data and headers prior to closing the file. - * This is a no-op for uncompressed columns. - * @param bTruncFile is file to be truncated - * @return NO_ERROR or success - */ - virtual int finishFile( bool bTruncFile ); + /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the + * next extent. This is a no-op for uncompressed columns. + * @param startFileOffset (output) File offset to start of active chunk + */ + virtual int resetToBeCompressedColBuf(long long& startFileOffset); - /** @brief Returns size of the buffer - */ - int getSize() const - { - return fBufSize; - } + /** @brief Set the IDBDataFile* destination for the applicable col segment file. + * + * @param The destination IDBDataFile stream to which buffer data will be written + * @param Starting HWM for cFile + * @param Headers with ptr information (only applies to compressed files) + */ + virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs); - /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the - * next extent. This is a no-op for uncompressed columns. - * @param startFileOffset (output) File offset to start of active chunk - */ - virtual int resetToBeCompressedColBuf( long long& startFileOffset ); + /** @brief Resize the buffer, also copying the section denoted by the + * offsets to the new buffer. If offsets are -1, then the buffer is + * expanded, but no copy is performed. + * + * @param newSize The new size of the buffer + * @param startOffset The start offset in the current buffer from where the + * previous data need to be copied + * @param endOffset The end offset upto which the data should be copied to + * new buffer + */ + void resizeAndCopy(int newSize, int startOffset, int endOffset); - /** @brief Set the IDBDataFile* destination for the applicable col segment file. - * - * @param The destination IDBDataFile stream to which buffer data will be written - * @param Starting HWM for cFile - * @param Headers with ptr information (only applies to compressed files) - */ - virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs); + /** @brief Write data to buffer + * + * @param data + * @param startOffset + * @param bytes + */ + void write(const void* const data, int startOffset, int bytes) const; - /** @brief Resize the buffer, also copying the section denoted by the - * offsets to the new buffer. If offsets are -1, then the buffer is - * expanded, but no copy is performed. - * - * @param newSize The new size of the buffer - * @param startOffset The start offset in the current buffer from where the - * previous data need to be copied - * @param endOffset The end offset upto which the data should be copied to - * new buffer - */ - void resizeAndCopy(int newSize, int startOffset, int endOffset); + /** @brief Write data to FILE + * + * @param startOffset The buffer offset from where the write should begin + * @param writeSize The number of bytes to be written to the file + * @param fillUpWEmpties The flag to fill the buffer with empty magic values + * up to the block boundary. + */ + virtual int writeToFile(int startOffset, int writeSize, bool fillUpWEmpties = false); - /** @brief Write data to buffer - * - * @param data - * @param startOffset - * @param bytes - */ - void write(const void* const data, int startOffset, int bytes) const; + protected: + // Disable copy constructor and assignment operator by declaring and + // not defining. + ColumnBuffer(const ColumnBuffer&); + ColumnBuffer& operator=(const ColumnBuffer&); - /** @brief Write data to FILE - * - * @param startOffset The buffer offset from where the write should begin - * @param writeSize The number of bytes to be written to the file - * @param fillUpWEmpties The flag to fill the buffer with empty magic values - * up to the block boundary. - */ - virtual int writeToFile(int startOffset, int writeSize, - bool fillUpWEmpties = false); - -protected: - - // Disable copy constructor and assignment operator by declaring and - // not defining. - ColumnBuffer(const ColumnBuffer&); - ColumnBuffer& operator=(const ColumnBuffer&); - - unsigned char* fBuffer; // Internal buffer - int fBufSize; // Size of the internal buffer - IDBDataFile* fFile; // The column file output stream - ColumnInfo* fColInfo; // parent ColumnInfo Object - Log* fLog; // Logger - HWM fStartingHwm; // Starting HWM for current column segment file + unsigned char* fBuffer; // Internal buffer + int fBufSize; // Size of the internal buffer + IDBDataFile* fFile; // The column file output stream + ColumnInfo* fColInfo; // parent ColumnInfo Object + Log* fLog; // Logger + HWM fStartingHwm; // Starting HWM for current column segment file }; -} +} // namespace WriteEngine -#endif //WRITEENGINE_COLUMNBUF_H +#endif // WRITEENGINE_COLUMNBUF_H diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 9131d9ea7..9678501c2 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -50,21 +50,19 @@ using namespace compress; namespace WriteEngine { - //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -ColumnBufferCompressed::ColumnBufferCompressed( ColumnInfo* pColInfo, - Log* logger) : - ColumnBuffer(pColInfo, logger), - fToBeCompressedBuffer(0), - fToBeCompressedCapacity(0), - fNumBytes(0), - fPreLoadHWMChunk(true), - fFlushedStartHwmChunk(false) +ColumnBufferCompressed::ColumnBufferCompressed(ColumnInfo* pColInfo, Log* logger) + : ColumnBuffer(pColInfo, logger) + , fToBeCompressedBuffer(0) + , fToBeCompressedCapacity(0) + , fNumBytes(0) + , fPreLoadHWMChunk(true) + , fFlushedStartHwmChunk(false) { - fUserPaddingBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; - compress::initializeCompressorPool(fCompressorPool, fUserPaddingBytes); + fUserPaddingBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; + compress::initializeCompressorPool(fCompressorPool, fUserPaddingBytes); } //------------------------------------------------------------------------------ @@ -72,12 +70,12 @@ ColumnBufferCompressed::ColumnBufferCompressed( ColumnInfo* pColInfo, //------------------------------------------------------------------------------ ColumnBufferCompressed::~ColumnBufferCompressed() { - if (fToBeCompressedBuffer) - delete []fToBeCompressedBuffer; + if (fToBeCompressedBuffer) + delete[] fToBeCompressedBuffer; - fToBeCompressedBuffer = 0; - fToBeCompressedCapacity = 0; - fNumBytes = 0; + fToBeCompressedBuffer = 0; + fToBeCompressedCapacity = 0; + fNumBytes = 0; } //------------------------------------------------------------------------------ @@ -86,34 +84,33 @@ ColumnBufferCompressed::~ColumnBufferCompressed() //------------------------------------------------------------------------------ int ColumnBufferCompressed::setDbFile(IDBDataFile* f, HWM startHwm, const char* hdrs) { - fFile = f; - fStartingHwm = startHwm; + fFile = f; + fStartingHwm = startHwm; - if (compress::CompressInterface::getPtrList(hdrs, fChunkPtrs) != 0) - { - return ERR_COMP_PARSE_HDRS; - } + if (compress::CompressInterface::getPtrList(hdrs, fChunkPtrs) != 0) + { + return ERR_COMP_PARSE_HDRS; + } - // If we have any orphaned chunk pointers (ex: left over after a DML - // rollback), that fall after the HWM, then drop those trailing ptrs. - unsigned int chunkIndex = 0; - unsigned int blockOffsetWithinChunk = 0; + // If we have any orphaned chunk pointers (ex: left over after a DML + // rollback), that fall after the HWM, then drop those trailing ptrs. + unsigned int chunkIndex = 0; + unsigned int blockOffsetWithinChunk = 0; - auto compressor = compress::getCompressorByType( - fCompressorPool, fColInfo->column.compressionType); - if (!compressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } + auto compressor = compress::getCompressorByType(fCompressorPool, fColInfo->column.compressionType); + if (!compressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } - compressor->locateBlock(fStartingHwm, chunkIndex, blockOffsetWithinChunk); + compressor->locateBlock(fStartingHwm, chunkIndex, blockOffsetWithinChunk); - if ((chunkIndex + 1) < fChunkPtrs.size()) - { - fChunkPtrs.resize(chunkIndex + 1); - } + if ((chunkIndex + 1) < fChunkPtrs.size()) + { + fChunkPtrs.resize(chunkIndex + 1); + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -121,49 +118,41 @@ int ColumnBufferCompressed::setDbFile(IDBDataFile* f, HWM startHwm, const char* // importing the first chunk of the next extent. Returns startFileOffset // which indicates file offset (in bytes) where next extent will be starting. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::resetToBeCompressedColBuf( - long long& startFileOffset ) +int ColumnBufferCompressed::resetToBeCompressedColBuf(long long& startFileOffset) { - // Don't load chunk, once we go to next extent - fPreLoadHWMChunk = false; + // Don't load chunk, once we go to next extent + fPreLoadHWMChunk = false; - // Lazy creation of to-be-compressed buffer - if (!fToBeCompressedBuffer) - { - fToBeCompressedBuffer = - new unsigned char[CompressInterface::UNCOMPRESSED_INBUF_LEN]; - } + // Lazy creation of to-be-compressed buffer + if (!fToBeCompressedBuffer) + { + fToBeCompressedBuffer = new unsigned char[CompressInterface::UNCOMPRESSED_INBUF_LEN]; + } - BlockOp::setEmptyBuf( fToBeCompressedBuffer, - CompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, - fColInfo->column.width ); + BlockOp::setEmptyBuf(fToBeCompressedBuffer, CompressInterface::UNCOMPRESSED_INBUF_LEN, + fColInfo->column.emptyVal, fColInfo->column.width); - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Initializing empty chunk for next extent: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; hwm-" << fStartingHwm; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + if (fLog->isDebug(DEBUG_2)) + { + std::ostringstream oss; + oss << "Initializing empty chunk for next extent: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; hwm-" << fStartingHwm; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; + fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; - // Set file offset past end of last chunk - startFileOffset = CompressInterface::HDR_BUF_LEN * 2; + // Set file offset past end of last chunk + startFileOffset = CompressInterface::HDR_BUF_LEN * 2; - if (fChunkPtrs.size() > 0) - startFileOffset = fChunkPtrs[ fChunkPtrs.size() - 1 ].first + - fChunkPtrs[ fChunkPtrs.size() - 1 ].second; + if (fChunkPtrs.size() > 0) + startFileOffset = fChunkPtrs[fChunkPtrs.size() - 1].first + fChunkPtrs[fChunkPtrs.size() - 1].second; - // Positition ourselves to start of empty to-be-compressed buffer - fNumBytes = 0; + // Positition ourselves to start of empty to-be-compressed buffer + fNumBytes = 0; - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -171,193 +160,175 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( // file, and instead buffer up the data to be compressed in 4M chunks before // writing it out. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, - bool fillUpWEmpties) +int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, bool fillUpWEmpties) { - if (writeSize == 0) // skip unnecessary write, if 0 bytes given - return NO_ERROR; - - int fillUpWEmptiesWriteSize = 0; - if (fillUpWEmpties) - fillUpWEmptiesWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK; - - // If we are starting a new file, we need to reinit the buffer and - // find out what our file offset should be set to. - if (!fToBeCompressedCapacity) - { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_COL_INIT_BUF); -#endif - long long startFileOffset; - int rc = initToBeCompressedBuffer( startFileOffset ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFile: error initializing to-be-compressed buffer " - "for OID " << fColInfo->curCol.dataFile.fid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - rc = fColInfo->colOp->setFileOffset(fFile, startFileOffset, SEEK_SET); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFile: error init compressed file offset for " << - "OID " << fColInfo->curCol.dataFile.fid << - "; " << startFileOffset << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_COL_INIT_BUF); -#endif - } - - unsigned char* bufOffset = fToBeCompressedBuffer + fNumBytes; - - // Expand the compression buffer size if working with an abbrev extent, and - // the bytes we are about to add will overflow the abbreviated extent. - if ((fToBeCompressedCapacity < CompressInterface::UNCOMPRESSED_INBUF_LEN) && - ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) > fToBeCompressedCapacity) ) - { - std::ostringstream oss; - oss << "Expanding abbrev to-be-compressed buffer for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; - } - - if ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) <= fToBeCompressedCapacity) - { - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Buffering data to-be-compressed for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; addBytes-" << writeSize << - "; extraBytes-" << fillUpWEmptiesWriteSize << - "; totBytes-" << (fNumBytes + writeSize); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - memcpy(bufOffset, (fBuffer + startOffset), writeSize); - fNumBytes += writeSize; - fNumBytes += fillUpWEmptiesWriteSize; - } - else // Not enough room to add all the data to the to-be-compressed buffer - { - int startOffsetX = startOffset; - int writeSizeX = writeSize; - - // The number of bytes (in fBuffer) to be written, could be larger than - // our to-be-compressed buffer, so we require a loop to potentially - // iterate thru all the bytes to be compresssed and written from fBuffer - while (writeSizeX > 0) - { - idbassert( (fNumBytes <= fToBeCompressedCapacity) ); // DMC-temp debug - - size_t writeSizeOut = 0; - - if ((fNumBytes + writeSizeX) > fToBeCompressedCapacity) - { - writeSizeOut = fToBeCompressedCapacity - fNumBytes; - - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Buffering data (full) to-be-compressed for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; addBytes-" << writeSizeOut << - "; totBytes-" << (fNumBytes + writeSizeOut); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - if (writeSizeOut > 0) - { - memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); - fNumBytes += writeSizeOut; - } - - //char resp; - //std::cout << "dbg: before writeToFile->compressAndFlush" << - // std::endl; - //std::cin >> resp; - int rc = compressAndFlush( false ); - - //std::cout << "dbg: after writeToFile->compressAndFlush" << - // std::endl; - //std::cin >> resp; - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFile: error compressing and writing chunk " - "for OID " << fColInfo->curCol.dataFile.fid << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - // Start over again loading a new to-be-compressed buffer - BlockOp::setEmptyBuf( fToBeCompressedBuffer, - CompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, - fColInfo->column.width ); - - fToBeCompressedCapacity = - CompressInterface::UNCOMPRESSED_INBUF_LEN; - bufOffset = fToBeCompressedBuffer; - - fNumBytes = 0; - } - else - { - writeSizeOut = writeSizeX; - - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Buffering data (new) to-be-compressed for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; addBytes-" << writeSizeOut << - "; totBytes-" << (fNumBytes + writeSizeOut); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); - fNumBytes += writeSizeOut; - fNumBytes += fillUpWEmptiesWriteSize; - } - - startOffsetX += writeSizeOut; - writeSizeX -= writeSizeOut; - } // end of while loop - } - + if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + + int fillUpWEmptiesWriteSize = 0; + if (fillUpWEmpties) + fillUpWEmptiesWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK; + + // If we are starting a new file, we need to reinit the buffer and + // find out what our file offset should be set to. + if (!fToBeCompressedCapacity) + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_COMPRESS_COL_INIT_BUF); +#endif + long long startFileOffset; + int rc = initToBeCompressedBuffer(startFileOffset); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFile: error initializing to-be-compressed buffer " + "for OID " + << fColInfo->curCol.dataFile.fid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + return rc; + } + + rc = fColInfo->colOp->setFileOffset(fFile, startFileOffset, SEEK_SET); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFile: error init compressed file offset for " + << "OID " << fColInfo->curCol.dataFile.fid << "; " << startFileOffset << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + return rc; + } + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_COMPRESS_COL_INIT_BUF); +#endif + } + + unsigned char* bufOffset = fToBeCompressedBuffer + fNumBytes; + + // Expand the compression buffer size if working with an abbrev extent, and + // the bytes we are about to add will overflow the abbreviated extent. + if ((fToBeCompressedCapacity < CompressInterface::UNCOMPRESSED_INBUF_LEN) && + ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) > fToBeCompressedCapacity)) + { + std::ostringstream oss; + oss << "Expanding abbrev to-be-compressed buffer for: OID-" << fColInfo->curCol.dataFile.fid + << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << "; part-" + << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; + } + + if ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) <= fToBeCompressedCapacity) + { + if (fLog->isDebug(DEBUG_2)) + { + std::ostringstream oss; + oss << "Buffering data to-be-compressed for: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition + << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; addBytes-" << writeSize << "; extraBytes-" + << fillUpWEmptiesWriteSize << "; totBytes-" << (fNumBytes + writeSize); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + memcpy(bufOffset, (fBuffer + startOffset), writeSize); + fNumBytes += writeSize; + fNumBytes += fillUpWEmptiesWriteSize; + } + else // Not enough room to add all the data to the to-be-compressed buffer + { + int startOffsetX = startOffset; + int writeSizeX = writeSize; + + // The number of bytes (in fBuffer) to be written, could be larger than + // our to-be-compressed buffer, so we require a loop to potentially + // iterate thru all the bytes to be compresssed and written from fBuffer + while (writeSizeX > 0) + { + idbassert((fNumBytes <= fToBeCompressedCapacity)); // DMC-temp debug + + size_t writeSizeOut = 0; + + if ((fNumBytes + writeSizeX) > fToBeCompressedCapacity) + { + writeSizeOut = fToBeCompressedCapacity - fNumBytes; + + if (fLog->isDebug(DEBUG_2)) + { + std::ostringstream oss; + oss << "Buffering data (full) to-be-compressed for: OID-" << fColInfo->curCol.dataFile.fid + << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << "; part-" + << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment + << "; addBytes-" << writeSizeOut << "; totBytes-" << (fNumBytes + writeSizeOut); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + if (writeSizeOut > 0) + { + memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); + fNumBytes += writeSizeOut; + } + + // char resp; + // std::cout << "dbg: before writeToFile->compressAndFlush" << + // std::endl; + // std::cin >> resp; + int rc = compressAndFlush(false); + + // std::cout << "dbg: after writeToFile->compressAndFlush" << + // std::endl; + // std::cin >> resp; + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFile: error compressing and writing chunk " + "for OID " + << fColInfo->curCol.dataFile.fid << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + return rc; + } + + // Start over again loading a new to-be-compressed buffer + BlockOp::setEmptyBuf(fToBeCompressedBuffer, CompressInterface::UNCOMPRESSED_INBUF_LEN, + fColInfo->column.emptyVal, fColInfo->column.width); + + fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; + bufOffset = fToBeCompressedBuffer; + + fNumBytes = 0; + } + else + { + writeSizeOut = writeSizeX; + + if (fLog->isDebug(DEBUG_2)) + { + std::ostringstream oss; + oss << "Buffering data (new) to-be-compressed for: OID-" << fColInfo->curCol.dataFile.fid + << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << "; part-" + << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment + << "; addBytes-" << writeSizeOut << "; totBytes-" << (fNumBytes + writeSizeOut); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); + fNumBytes += writeSizeOut; + fNumBytes += fillUpWEmptiesWriteSize; + } + + startOffsetX += writeSizeOut; + writeSizeX -= writeSizeOut; + } // end of while loop + } + + return NO_ERROR; } //------------------------------------------------------------------------------ // Compress and write out the data in the to-be-compressed buffer. @@ -379,115 +350,105 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, // requires the immediate rewriting of the header, because we are modifying // that chunk and adding rows to it. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile ) +int ColumnBufferCompressed::compressAndFlush(bool bFinishingFile) { - auto compressor = compress::getCompressorByType( - fCompressorPool, fColInfo->column.compressionType); - if (!compressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } + auto compressor = compress::getCompressorByType(fCompressorPool, fColInfo->column.compressionType); + if (!compressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } - const size_t OUTPUT_BUFFER_SIZE = - compressor->maxCompressedSize(fToBeCompressedCapacity) + - fUserPaddingBytes + - // Padded len = len + COMPRESSED_SIZE_INCREMENT_CHUNK - (len % - // COMPRESSED_SIZE_INCREMENT_CHUNK) + usePadding - compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; + const size_t OUTPUT_BUFFER_SIZE = compressor->maxCompressedSize(fToBeCompressedCapacity) + + fUserPaddingBytes + + // Padded len = len + COMPRESSED_SIZE_INCREMENT_CHUNK - (len % + // COMPRESSED_SIZE_INCREMENT_CHUNK) + usePadding + compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; - unsigned char* compressedOutBuf = new unsigned char[ OUTPUT_BUFFER_SIZE ]; - boost::scoped_array compressedOutBufPtr(compressedOutBuf); - size_t outputLen = OUTPUT_BUFFER_SIZE; + unsigned char* compressedOutBuf = new unsigned char[OUTPUT_BUFFER_SIZE]; + boost::scoped_array compressedOutBufPtr(compressedOutBuf); + size_t outputLen = OUTPUT_BUFFER_SIZE; #ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_COL_COMPRESS); + Stats::startParseEvent(WE_STATS_COMPRESS_COL_COMPRESS); #endif - int rc = compressor->compressBlock( - reinterpret_cast(fToBeCompressedBuffer), - fToBeCompressedCapacity, compressedOutBuf, outputLen); + int rc = compressor->compressBlock(reinterpret_cast(fToBeCompressedBuffer), fToBeCompressedCapacity, + compressedOutBuf, outputLen); - if (rc != 0) - { - return ERR_COMP_COMPRESS; - } + if (rc != 0) + { + return ERR_COMP_COMPRESS; + } - // Round up the compressed chunk size - rc = compressor->padCompressedChunks( compressedOutBuf, - outputLen, OUTPUT_BUFFER_SIZE ); + // Round up the compressed chunk size + rc = compressor->padCompressedChunks(compressedOutBuf, outputLen, OUTPUT_BUFFER_SIZE); - if (rc != 0) - { - return ERR_COMP_PAD_DATA; - } + if (rc != 0) + { + return ERR_COMP_PAD_DATA; + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_COL_COMPRESS); - Stats::startParseEvent(WE_STATS_WRITE_COL); + Stats::stopParseEvent(WE_STATS_COMPRESS_COL_COMPRESS); + Stats::startParseEvent(WE_STATS_WRITE_COL); #endif - off64_t fileOffset = fFile->tell(); - size_t nitems = fFile->write(compressedOutBuf, outputLen) / outputLen; + off64_t fileOffset = fFile->tell(); + size_t nitems = fFile->write(compressedOutBuf, outputLen) / outputLen; - if (nitems != 1) - return ERR_FILE_WRITE; + if (nitems != 1) + return ERR_FILE_WRITE; - CompChunkPtr compChunk( - (uint64_t)fileOffset, (uint64_t)outputLen); - fChunkPtrs.push_back( compChunk ); + CompChunkPtr compChunk((uint64_t)fileOffset, (uint64_t)outputLen); + fChunkPtrs.push_back(compChunk); - if (fLog->isDebug( DEBUG_2 )) + if (fLog->isDebug(DEBUG_2)) + { + std::ostringstream oss; + oss << "Writing compressed data for: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; bytes-" << outputLen << "; fileOffset-" << fileOffset; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + // We write out the compression headers if we are finished with this file + // (either because we are through with the extent or the data), or because + // this is the first HWM chunk that we may be modifying. + // See the description that precedes this function for more details. + if (bFinishingFile || !fFlushedStartHwmChunk) + { + fileOffset = fFile->tell(); + RETURN_ON_ERROR(saveCompressionHeaders()); + + // If we just updated the chunk header for the starting HWM chunk, + // then we flush our output, to synchronize with compressed chunks, + if (!fFlushedStartHwmChunk) { - std::ostringstream oss; - oss << "Writing compressed data for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; bytes-" << outputLen << - "; fileOffset-" << fileOffset; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + // char resp; + // std::cout << "dbg: before fflush of hdrs" << std::endl; + // std::cin >> resp; + if (fFile->flush() != 0) + return ERR_FILE_FLUSH; + + // std::cout << "dbg: after fflush of hdrs" << std::endl; + // std::cin >> resp; + fFlushedStartHwmChunk = true; } - // We write out the compression headers if we are finished with this file - // (either because we are through with the extent or the data), or because - // this is the first HWM chunk that we may be modifying. - // See the description that precedes this function for more details. - if ( bFinishingFile || !fFlushedStartHwmChunk ) + // After seeking to the top of the file to write the headers, + // we restore the file offset to continue adding more chunks, + // if we are not through with this file. + if (!bFinishingFile) { - fileOffset = fFile->tell(); - RETURN_ON_ERROR( saveCompressionHeaders() ); - - // If we just updated the chunk header for the starting HWM chunk, - // then we flush our output, to synchronize with compressed chunks, - if ( !fFlushedStartHwmChunk ) - { - //char resp; - //std::cout << "dbg: before fflush of hdrs" << std::endl; - //std::cin >> resp; - if (fFile->flush() != 0) - return ERR_FILE_FLUSH; - - //std::cout << "dbg: after fflush of hdrs" << std::endl; - //std::cin >> resp; - fFlushedStartHwmChunk = true; - } - - // After seeking to the top of the file to write the headers, - // we restore the file offset to continue adding more chunks, - // if we are not through with this file. - if ( !bFinishingFile ) - { - RETURN_ON_ERROR( fColInfo->colOp->setFileOffset( - fFile, fileOffset, SEEK_SET) ); - } + RETURN_ON_ERROR(fColInfo->colOp->setFileOffset(fFile, fileOffset, SEEK_SET)); } + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WRITE_COL); + Stats::stopParseEvent(WE_STATS_WRITE_COL); #endif - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -496,144 +457,140 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile ) //------------------------------------------------------------------------------ int ColumnBufferCompressed::finishFile(bool bTruncFile) { - // If capacity is 0, we never got far enough to read in the HWM chunk for - // the current column segment file, so no need to update the file contents. - // But we do continue in case we need to truncate the file before exiting. - // This could happen if our initial block skipping finished an extent. - if (fToBeCompressedCapacity > 0) - { - //char resp; - //std::cout << "dbg: before finishFile->compressAndFlush" << std::endl; - //std::cin >> resp; - // Write out any data still waiting to be compressed - RETURN_ON_ERROR( compressAndFlush( true ) ); - //std::cout << "dbg: after finishFile->compressAndFlush" << std::endl; - //std::cin >> resp; - } + // If capacity is 0, we never got far enough to read in the HWM chunk for + // the current column segment file, so no need to update the file contents. + // But we do continue in case we need to truncate the file before exiting. + // This could happen if our initial block skipping finished an extent. + if (fToBeCompressedCapacity > 0) + { + // char resp; + // std::cout << "dbg: before finishFile->compressAndFlush" << std::endl; + // std::cin >> resp; + // Write out any data still waiting to be compressed + RETURN_ON_ERROR(compressAndFlush(true)); + // std::cout << "dbg: after finishFile->compressAndFlush" << std::endl; + // std::cin >> resp; + } #ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_COL_FINISH_EXTENT); + Stats::startParseEvent(WE_STATS_COMPRESS_COL_FINISH_EXTENT); #endif - // Truncate file (if applicable) based on offset and size of last chunk - if (bTruncFile && (fChunkPtrs.size() > 0)) + // Truncate file (if applicable) based on offset and size of last chunk + if (bTruncFile && (fChunkPtrs.size() > 0)) + { + long long truncateFileSize = + fChunkPtrs[fChunkPtrs.size() - 1].first + fChunkPtrs[fChunkPtrs.size() - 1].second; + + // @bug5769 Don't initialize extents or truncate db files on HDFS + if (idbdatafile::IDBPolicy::useHdfs()) { - long long truncateFileSize = fChunkPtrs[fChunkPtrs.size() - 1].first + - fChunkPtrs[fChunkPtrs.size() - 1].second; - - // @bug5769 Don't initialize extents or truncate db files on HDFS - if (idbdatafile::IDBPolicy::useHdfs()) - { - std::ostringstream oss1; - oss1 << "Finished writing column file" - ": OID-" << fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; size-" << truncateFileSize; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - } - else - { - std::ostringstream oss1; - oss1 << "Truncating column file" - ": OID-" << fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; size-" << truncateFileSize; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - - int rc = NO_ERROR; - - if (truncateFileSize > 0) - rc = fColInfo->colOp->truncateFile( fFile, truncateFileSize ); - else - rc = ERR_COMP_TRUNCATE_ZERO;//@bug3913-Catch truncate to 0 bytes - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss2; - oss2 << "finishFile: error truncating file for " << - "OID " << fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; size-" << truncateFileSize << - "; " << ec.errorString(rc); - fLog->logMsg( oss2.str(), rc, MSGLVL_ERROR ); - - return rc; - } - } + std::ostringstream oss1; + oss1 << "Finished writing column file" + ": OID-" + << fColInfo->curCol.dataFile.fid << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << "; part-" + << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment + << "; size-" << truncateFileSize; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); } - - // Nothing more to do if we are not updating the file contents. - if (fToBeCompressedCapacity == 0) + else { -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_COL_FINISH_EXTENT); -#endif - return NO_ERROR; + std::ostringstream oss1; + oss1 << "Truncating column file" + ": OID-" + << fColInfo->curCol.dataFile.fid << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << "; part-" + << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment + << "; size-" << truncateFileSize; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + + int rc = NO_ERROR; + + if (truncateFileSize > 0) + rc = fColInfo->colOp->truncateFile(fFile, truncateFileSize); + else + rc = ERR_COMP_TRUNCATE_ZERO; //@bug3913-Catch truncate to 0 bytes + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss2; + oss2 << "finishFile: error truncating file for " + << "OID " << fColInfo->curCol.dataFile.fid << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot + << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; size-" << truncateFileSize << "; " + << ec.errorString(rc); + fLog->logMsg(oss2.str(), rc, MSGLVL_ERROR); + + return rc; + } } + } - fToBeCompressedCapacity = 0; - fNumBytes = 0; - fChunkPtrs.clear(); - + // Nothing more to do if we are not updating the file contents. + if (fToBeCompressedCapacity == 0) + { #ifdef PROFILE Stats::stopParseEvent(WE_STATS_COMPRESS_COL_FINISH_EXTENT); #endif - return NO_ERROR; + } + + fToBeCompressedCapacity = 0; + fNumBytes = 0; + fChunkPtrs.clear(); + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_COMPRESS_COL_FINISH_EXTENT); +#endif + + return NO_ERROR; } //------------------------------------------------------------------------------ // Write out the updated compression headers. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::saveCompressionHeaders( ) +int ColumnBufferCompressed::saveCompressionHeaders() { - // Construct the header records - char hdrBuf[CompressInterface::HDR_BUF_LEN * 2]; - RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf)); + // Construct the header records + char hdrBuf[CompressInterface::HDR_BUF_LEN * 2]; + RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf)); - BRM::LBID_t lbid = compress::CompressInterface::getLBIDByIndex(hdrBuf, 0); - compress::CompressInterface::initHdr(hdrBuf, fColInfo->column.width, fColInfo->column.dataType, - fColInfo->column.compressionType); - compress::CompressInterface::setBlockCount(hdrBuf, (fColInfo->getFileSize() / BYTE_PER_BLOCK)); - // If lbid written in the header is not 0 and not equal to `lastupdatedlbid` - we are running - // for the next extent for column segment file. - const auto lastUpdatedLbid = fColInfo->getLastUpdatedLBID(); - if (lbid && lastUpdatedLbid != lbid) - { - // Write back lbid, after header initialization. - compress::CompressInterface::setLBIDByIndex(hdrBuf, lbid, 0); - compress::CompressInterface::setLBIDByIndex(hdrBuf, lastUpdatedLbid, 1); - } - else - compress::CompressInterface::setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 0); + BRM::LBID_t lbid = compress::CompressInterface::getLBIDByIndex(hdrBuf, 0); + compress::CompressInterface::initHdr(hdrBuf, fColInfo->column.width, fColInfo->column.dataType, + fColInfo->column.compressionType); + compress::CompressInterface::setBlockCount(hdrBuf, (fColInfo->getFileSize() / BYTE_PER_BLOCK)); + // If lbid written in the header is not 0 and not equal to `lastupdatedlbid` - we are running + // for the next extent for column segment file. + const auto lastUpdatedLbid = fColInfo->getLastUpdatedLBID(); + if (lbid && lastUpdatedLbid != lbid) + { + // Write back lbid, after header initialization. + compress::CompressInterface::setLBIDByIndex(hdrBuf, lbid, 0); + compress::CompressInterface::setLBIDByIndex(hdrBuf, lastUpdatedLbid, 1); + } + else + compress::CompressInterface::setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 0); - std::vector ptrs; + std::vector ptrs; - for (unsigned i = 0; i < fChunkPtrs.size(); i++) - { - ptrs.push_back( fChunkPtrs[i].first ); - } + for (unsigned i = 0; i < fChunkPtrs.size(); i++) + { + ptrs.push_back(fChunkPtrs[i].first); + } - unsigned lastIdx = fChunkPtrs.size() - 1; - ptrs.push_back( fChunkPtrs[lastIdx].first + fChunkPtrs[lastIdx].second ); - compress::CompressInterface::storePtrs(ptrs, hdrBuf); + unsigned lastIdx = fChunkPtrs.size() - 1; + ptrs.push_back(fChunkPtrs[lastIdx].first + fChunkPtrs[lastIdx].second); + compress::CompressInterface::storePtrs(ptrs, hdrBuf); - // Write out the header records - //char resp; - //std::cout << "dbg: before writeHeaders" << std::endl; - //std::cin >> resp; - RETURN_ON_ERROR( fColInfo->colOp->writeHeaders(fFile, hdrBuf) ); - //std::cout << "dbg: after writeHeaders" << std::endl; - //std::cin >> resp; + // Write out the header records + // char resp; + // std::cout << "dbg: before writeHeaders" << std::endl; + // std::cin >> resp; + RETURN_ON_ERROR(fColInfo->colOp->writeHeaders(fFile, hdrBuf)); + // std::cout << "dbg: after writeHeaders" << std::endl; + // std::cin >> resp; - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -647,172 +604,149 @@ int ColumnBufferCompressed::saveCompressionHeaders( ) //------------------------------------------------------------------------------ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset) { - bool bNewBuffer = false; + bool bNewBuffer = false; - // Lazy initialization of to-be-compressed buffer - if (!fToBeCompressedBuffer) + // Lazy initialization of to-be-compressed buffer + if (!fToBeCompressedBuffer) + { + fToBeCompressedBuffer = new unsigned char[CompressInterface::UNCOMPRESSED_INBUF_LEN]; + BlockOp::setEmptyBuf(fToBeCompressedBuffer, CompressInterface::UNCOMPRESSED_INBUF_LEN, + fColInfo->column.emptyVal, fColInfo->column.width); + bNewBuffer = true; + } + + // Find the chunk containing the starting HWM, as long as our initial + // block skipping has not caused us to exit the HWM chunk; in which + // case we start a new empty chunk. + unsigned int chunkIndex = 0; + unsigned int blockOffsetWithinChunk = 0; + bool bSkipStartingBlks = false; + + auto compressor = compress::getCompressorByType(fCompressorPool, fColInfo->column.compressionType); + if (!compressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } + + if (fPreLoadHWMChunk) + { + if (fChunkPtrs.size() > 0) { - fToBeCompressedBuffer = - new unsigned char[CompressInterface::UNCOMPRESSED_INBUF_LEN]; - BlockOp::setEmptyBuf( fToBeCompressedBuffer, - CompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, - fColInfo->column.width ); - bNewBuffer = true; + compressor->locateBlock(fStartingHwm, chunkIndex, blockOffsetWithinChunk); + + if (chunkIndex < fChunkPtrs.size()) + startFileOffset = fChunkPtrs[chunkIndex].first; + else + fPreLoadHWMChunk = false; + } + // If we are at the start of the job, fPreLoadHWMChunk will be true, + // to preload the old HWM chunk. But if we have no chunk ptrs, then + // we are starting on an empty PM. In this case, we skip starting + // blks if fStartingHwm has been set. + else + { + fPreLoadHWMChunk = false; + bSkipStartingBlks = true; + } + } + + // Preload (read and uncompress) the chunk for the starting HWM extent only + if (fPreLoadHWMChunk) + { + fPreLoadHWMChunk = false; // only preload HWM chunk in the first extent + + std::ostringstream oss; + oss << "Reading HWM chunk for: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; hwm-" << fStartingHwm << "; chunk#-" << chunkIndex + << "; blkInChunk-" << blockOffsetWithinChunk; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + // Read the chunk + RETURN_ON_ERROR(fColInfo->colOp->setFileOffset(fFile, startFileOffset, SEEK_SET)); + + char* compressedOutBuf = new char[fChunkPtrs[chunkIndex].second]; + boost::scoped_array compressedOutBufPtr(compressedOutBuf); + size_t itemsRead = + fFile->read(compressedOutBuf, fChunkPtrs[chunkIndex].second) / fChunkPtrs[chunkIndex].second; + + if (itemsRead != 1) + { + std::ostringstream oss; + oss << "Error reading HWM chunk for: " + << "OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot + << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; hwm-" << fStartingHwm; + fLog->logMsg(oss.str(), ERR_COMP_READ_BLOCK, MSGLVL_ERROR); + + return ERR_COMP_READ_BLOCK; } - // Find the chunk containing the starting HWM, as long as our initial - // block skipping has not caused us to exit the HWM chunk; in which - // case we start a new empty chunk. - unsigned int chunkIndex = 0; - unsigned int blockOffsetWithinChunk = 0; - bool bSkipStartingBlks = false; + // Uncompress the chunk into our 4MB buffer + size_t outLen = CompressInterface::UNCOMPRESSED_INBUF_LEN; + int rc = compressor->uncompressBlock(compressedOutBuf, fChunkPtrs[chunkIndex].second, + fToBeCompressedBuffer, outLen); - auto compressor = compress::getCompressorByType( - fCompressorPool, fColInfo->column.compressionType); - if (!compressor) + if (rc) { - return ERR_COMP_WRONG_COMP_TYPE; + WErrorCodes ec; + std::ostringstream oss; + oss << "Error uncompressing HWM chunk for: " + << "OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot + << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; hwm-" << fStartingHwm << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + return ERR_COMP_UNCOMPRESS; } - if (fPreLoadHWMChunk) - { - if (fChunkPtrs.size() > 0) - { - compressor->locateBlock(fStartingHwm, chunkIndex, - blockOffsetWithinChunk); + fToBeCompressedCapacity = outLen; - if (chunkIndex < fChunkPtrs.size()) - startFileOffset = fChunkPtrs[chunkIndex].first; - else - fPreLoadHWMChunk = false; - } - // If we are at the start of the job, fPreLoadHWMChunk will be true, - // to preload the old HWM chunk. But if we have no chunk ptrs, then - // we are starting on an empty PM. In this case, we skip starting - // blks if fStartingHwm has been set. - else - { - fPreLoadHWMChunk = false; - bSkipStartingBlks = true; - } - } + // Positition ourselves to start adding data to the HWM block + fNumBytes = blockOffsetWithinChunk * BYTE_PER_BLOCK; - // Preload (read and uncompress) the chunk for the starting HWM extent only - if (fPreLoadHWMChunk) - { - fPreLoadHWMChunk = false; // only preload HWM chunk in the first extent - - std::ostringstream oss; - oss << "Reading HWM chunk for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; hwm-" << fStartingHwm << - "; chunk#-" << chunkIndex << - "; blkInChunk-" << blockOffsetWithinChunk; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - - // Read the chunk - RETURN_ON_ERROR( fColInfo->colOp->setFileOffset( - fFile, startFileOffset, SEEK_SET) ); - - char* compressedOutBuf = new char[ fChunkPtrs[chunkIndex].second ]; - boost::scoped_array compressedOutBufPtr(compressedOutBuf); - size_t itemsRead = fFile->read(compressedOutBuf, fChunkPtrs[chunkIndex].second) / fChunkPtrs[chunkIndex].second; - - if (itemsRead != 1) - { - std::ostringstream oss; - oss << "Error reading HWM chunk for: " << - "OID-" << fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; hwm-" << fStartingHwm; - fLog->logMsg( oss.str(), ERR_COMP_READ_BLOCK, MSGLVL_ERROR ); - - return ERR_COMP_READ_BLOCK; - } - - // Uncompress the chunk into our 4MB buffer - size_t outLen = CompressInterface::UNCOMPRESSED_INBUF_LEN; - int rc = compressor->uncompressBlock( - compressedOutBuf, - fChunkPtrs[chunkIndex].second, - fToBeCompressedBuffer, - outLen); - - if (rc) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error uncompressing HWM chunk for: " << - "OID-" << fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; hwm-" << fStartingHwm << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return ERR_COMP_UNCOMPRESS; - } - - fToBeCompressedCapacity = outLen; - - // Positition ourselves to start adding data to the HWM block - fNumBytes = blockOffsetWithinChunk * BYTE_PER_BLOCK; - - // We are going to add data to, and thus re-add, the last chunk; so we - // drop it from our list. - fChunkPtrs.resize( fChunkPtrs.size() - 1 ); - } - else // We have left the HWM chunk; just position file offset, + // We are going to add data to, and thus re-add, the last chunk; so we + // drop it from our list. + fChunkPtrs.resize(fChunkPtrs.size() - 1); + } + else // We have left the HWM chunk; just position file offset, // without reading anything + { + // If it's not a new buffer, we need to initialize, since we won't be + // reading in anything to overlay what's in the to-be-compressed buffer. + if (!bNewBuffer) { - // If it's not a new buffer, we need to initialize, since we won't be - // reading in anything to overlay what's in the to-be-compressed buffer. - if (!bNewBuffer) - { - BlockOp::setEmptyBuf( fToBeCompressedBuffer, - CompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, - fColInfo->column.width ); - } - - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Initializing new empty chunk: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; hwm-" << fStartingHwm; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; - - // Set file offset to start after last current chunk - startFileOffset = CompressInterface::HDR_BUF_LEN * 2; - - if (fChunkPtrs.size() > 0) - startFileOffset = fChunkPtrs[ fChunkPtrs.size() - 1 ].first + - fChunkPtrs[ fChunkPtrs.size() - 1 ].second; - - // Position ourselves to start of empty to-be-compressed buffer. - // If we are starting the first extent on a PM, we may employ blk - // skipping at start of import; adjust fNumBytes accordingly. - // (see ColumnInfo::createDelayedFileIfNeeded() for discussion) - if (bSkipStartingBlks) - fNumBytes = fStartingHwm * BYTE_PER_BLOCK; - else - fNumBytes = 0; + BlockOp::setEmptyBuf(fToBeCompressedBuffer, CompressInterface::UNCOMPRESSED_INBUF_LEN, + fColInfo->column.emptyVal, fColInfo->column.width); } - return NO_ERROR; -} + if (fLog->isDebug(DEBUG_2)) + { + std::ostringstream oss; + oss << "Initializing new empty chunk: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition + << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; hwm-" << fStartingHwm; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + fToBeCompressedCapacity = CompressInterface::UNCOMPRESSED_INBUF_LEN; + + // Set file offset to start after last current chunk + startFileOffset = CompressInterface::HDR_BUF_LEN * 2; + + if (fChunkPtrs.size() > 0) + startFileOffset = fChunkPtrs[fChunkPtrs.size() - 1].first + fChunkPtrs[fChunkPtrs.size() - 1].second; + + // Position ourselves to start of empty to-be-compressed buffer. + // If we are starting the first extent on a PM, we may employ blk + // skipping at start of import; adjust fNumBytes accordingly. + // (see ColumnInfo::createDelayedFileIfNeeded() for discussion) + if (bSkipStartingBlks) + fNumBytes = fStartingHwm * BYTE_PER_BLOCK; + else + fNumBytes = 0; + } + + return NO_ERROR; } +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colbufcompressed.h b/writeengine/bulk/we_colbufcompressed.h index 3eba2c9b1..612de63a5 100644 --- a/writeengine/bulk/we_colbufcompressed.h +++ b/writeengine/bulk/we_colbufcompressed.h @@ -48,74 +48,69 @@ class ColumnInfo; */ class ColumnBufferCompressed : public ColumnBuffer { + public: + /** @brief default Constructor + */ + ColumnBufferCompressed(ColumnInfo* pColInfo, Log* logger); -public: + /** @brief default Destructor + */ + virtual ~ColumnBufferCompressed(); - /** @brief default Constructor - */ - ColumnBufferCompressed( ColumnInfo* pColInfo, Log* logger); + /** @brief Final flushing of data and headers prior to closing the file. + * @param bTruncFile is file to be truncated + * @return NO_ERROR or success + */ + virtual int finishFile(bool bTruncFile); - /** @brief default Destructor - */ - virtual ~ColumnBufferCompressed(); + /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the + * next extent. + * @param startFileOffset Byte offset where next extent chunk will start + */ + virtual int resetToBeCompressedColBuf(long long& startFileOffset); - /** @brief Final flushing of data and headers prior to closing the file. - * @param bTruncFile is file to be truncated - * @return NO_ERROR or success - */ - virtual int finishFile(bool bTruncFile); + /** @brief file mutator + * + * @param cFile Destination FILE stream where buffer data will be written + * @param startHwm Starting HWM for cFile + * @param hdrs Headers with ptr information. + */ + virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs); - /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the - * next extent. - * @param startFileOffset Byte offset where next extent chunk will start - */ - virtual int resetToBeCompressedColBuf(long long& startFileOffset ); + /** @brief Write data to FILE + * + * @param startOffset The buffer offset from where the write should begin + * @param writeSize The number of bytes to be written to the file + * @param fillUpWEmpties The flag to fill the buffer with empty magic + * values up to the block boundary. + */ + virtual int writeToFile(int startOffset, int writeSize, bool fillUpWEmpties = false); - /** @brief file mutator - * - * @param cFile Destination FILE stream where buffer data will be written - * @param startHwm Starting HWM for cFile - * @param hdrs Headers with ptr information. - */ - virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs); + private: + // Disable copy constructor and assignment operator by declaring and + // not defining. + ColumnBufferCompressed(const ColumnBufferCompressed&); + ColumnBufferCompressed& operator=(const ColumnBufferCompressed&); - /** @brief Write data to FILE - * - * @param startOffset The buffer offset from where the write should begin - * @param writeSize The number of bytes to be written to the file - * @param fillUpWEmpties The flag to fill the buffer with empty magic - * values up to the block boundary. - */ - virtual int writeToFile(int startOffset, int writeSize, - bool fillUpWEmpties = false); + // Compress and flush the to-be-compressed buffer; updates header if needed + int compressAndFlush(bool bFinishFile); + int initToBeCompressedBuffer(long long& startFileOffset); + // Initialize the to-be-compressed buffer + int saveCompressionHeaders(); // Saves compression headers to the db file -private: - - // Disable copy constructor and assignment operator by declaring and - // not defining. - ColumnBufferCompressed(const ColumnBufferCompressed&); - ColumnBufferCompressed& operator=(const ColumnBufferCompressed&); - - // Compress and flush the to-be-compressed buffer; updates header if needed - int compressAndFlush(bool bFinishFile); - int initToBeCompressedBuffer( long long& startFileOffset); - // Initialize the to-be-compressed buffer - int saveCompressionHeaders(); // Saves compression headers to the db file - - unsigned char* fToBeCompressedBuffer; // data waiting to be compressed - size_t fToBeCompressedCapacity;//size of comp buffer; - // should always be 4MB, unless - // working with abbrev extent. - size_t fNumBytes; // num Bytes in comp buffer - compress::CompressorPool fCompressorPool; // data compression object pool - compress::CompChunkPtrList - fChunkPtrs; // col file header information - bool fPreLoadHWMChunk; // preload 1st HWM chunk only - unsigned int fUserPaddingBytes; // compressed chunk padding - bool fFlushedStartHwmChunk; // have we rewritten the hdr - // for the starting HWM chunk + unsigned char* fToBeCompressedBuffer; // data waiting to be compressed + size_t fToBeCompressedCapacity; // size of comp buffer; + // should always be 4MB, unless + // working with abbrev extent. + size_t fNumBytes; // num Bytes in comp buffer + compress::CompressorPool fCompressorPool; // data compression object pool + compress::CompChunkPtrList fChunkPtrs; // col file header information + bool fPreLoadHWMChunk; // preload 1st HWM chunk only + unsigned int fUserPaddingBytes; // compressed chunk padding + bool fFlushedStartHwmChunk; // have we rewritten the hdr + // for the starting HWM chunk }; -} +} // namespace WriteEngine -#endif //WRITEENGINE_COLUMNBUFCOMPRESSED_H +#endif // WRITEENGINE_COLUMNBUFCOMPRESSED_H diff --git a/writeengine/bulk/we_colbufmgr.cpp b/writeengine/bulk/we_colbufmgr.cpp index 36f3e5493..09d28763a 100644 --- a/writeengine/bulk/we_colbufmgr.cpp +++ b/writeengine/bulk/we_colbufmgr.cpp @@ -46,25 +46,26 @@ namespace // Minimum time to wait for a condition, so as to periodically wake up and // check the global job status, to see if the job needs to terminate. const int COND_WAIT_SECONDS = 1; -} +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // ColumnBufferManger constructor //------------------------------------------------------------------------------ -ColumnBufferManager::ColumnBufferManager(ColumnInfo* pColInfo, - int width, Log* logger, int compressionType) : - fBufWriteOffset(0), fBufFreeOffset(0), fResizePending(false), - fColWidth(width), - fMaxRowId(std::numeric_limits::max()), - fColInfo(pColInfo), fLog(logger) +ColumnBufferManager::ColumnBufferManager(ColumnInfo* pColInfo, int width, Log* logger, int compressionType) + : fBufWriteOffset(0) + , fBufFreeOffset(0) + , fResizePending(false) + , fColWidth(width) + , fMaxRowId(std::numeric_limits::max()) + , fColInfo(pColInfo) + , fLog(logger) { - if (compressionType) - fCBuf = new ColumnBufferCompressed(pColInfo, logger); - else - fCBuf = new ColumnBuffer(pColInfo, logger); + if (compressionType) + fCBuf = new ColumnBufferCompressed(pColInfo, logger); + else + fCBuf = new ColumnBuffer(pColInfo, logger); } //------------------------------------------------------------------------------ @@ -72,8 +73,8 @@ ColumnBufferManager::ColumnBufferManager(ColumnInfo* pColInfo, //------------------------------------------------------------------------------ ColumnBufferManager::~ColumnBufferManager() { - if (fCBuf) - delete fCBuf; + if (fCBuf) + delete fCBuf; } //------------------------------------------------------------------------------ @@ -115,220 +116,209 @@ ColumnBufferManager::~ColumnBufferManager() // 6. Allocate new ColumnBufferSection // 7. Update fBufFreeOffset, fSectionsInUse, fMaxRowId //------------------------------------------------------------------------------ -int ColumnBufferManager::reserveSection( - RID startRowId, - uint32_t nRowsIn, - uint32_t& secRowCnt, - ColumnBufferSection** cbs, - RID& lastInputRowInExtent) +int ColumnBufferManager::reserveSection(RID startRowId, uint32_t nRowsIn, uint32_t& secRowCnt, + ColumnBufferSection** cbs, RID& lastInputRowInExtent) { #ifdef PROFILE - Stats::startParseEvent(WE_STATS_WAIT_TO_RESERVE_OUT_BUF); + Stats::startParseEvent(WE_STATS_WAIT_TO_RESERVE_OUT_BUF); #endif - *cbs = 0; - boost::posix_time::seconds wait_seconds(COND_WAIT_SECONDS); + *cbs = 0; + boost::posix_time::seconds wait_seconds(COND_WAIT_SECONDS); - boost::mutex::scoped_lock lock(fColInfo->colMutex()); + boost::mutex::scoped_lock lock(fColInfo->colMutex()); - //..Ensure that ColumnBufferSection allocations are made in input row order - bool bWaitedForInSequence = false; + //..Ensure that ColumnBufferSection allocations are made in input row order + bool bWaitedForInSequence = false; - while (1) + while (1) + { + RID startRowTest = (std::numeric_limits::max() == fMaxRowId) ? 0 : fMaxRowId + 1; + + if (startRowTest == startRowId) + break; + + if (fLog->isDebug(DEBUG_3)) { - RID startRowTest = (std::numeric_limits::max() == - fMaxRowId) ? 0 : fMaxRowId + 1; - - if (startRowTest == startRowId) - break; - - if (fLog->isDebug( DEBUG_3 )) - { - bWaitedForInSequence = true; - std::ostringstream oss; - oss << "OID-" << fColInfo->curCol.dataFile.fid << - "; Waiting for in-sequence"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - fOutOfSequence.timed_wait(lock, wait_seconds); - - // See if JobStatus has been set to terminate by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - throw SecondaryShutdownException( "ColumnBufferManager::" - "reserveSection(1) responding to job termination"); - } + bWaitedForInSequence = true; + std::ostringstream oss; + oss << "OID-" << fColInfo->curCol.dataFile.fid << "; Waiting for in-sequence"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } - if (fLog->isDebug( DEBUG_3 )) + fOutOfSequence.timed_wait(lock, wait_seconds); + + // See if JobStatus has been set to terminate by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) { - if (bWaitedForInSequence) - { - std::ostringstream oss; - oss << "OID-" << fColInfo->curCol.dataFile.fid << - "; Resume after waiting for in-sequence"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + throw SecondaryShutdownException( + "ColumnBufferManager::" + "reserveSection(1) responding to job termination"); + } + } + + if (fLog->isDebug(DEBUG_3)) + { + if (bWaitedForInSequence) + { + std::ostringstream oss; + oss << "OID-" << fColInfo->curCol.dataFile.fid << "; Resume after waiting for in-sequence"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + } + + //..Check/wait for any pending output buffer expansion to be completed + bool bWaitedForResize = false; + + while (fResizePending) + { + if (fLog->isDebug(DEBUG_3)) + { + bWaitedForResize = true; + std::ostringstream oss; + oss << "OID-" << fColInfo->curCol.dataFile.fid << "; Waiting for pending resize"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } - //..Check/wait for any pending output buffer expansion to be completed - bool bWaitedForResize = false; + fResizeInProgress.timed_wait(lock, wait_seconds); - while (fResizePending) + // See if JobStatus has been set to terminate by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) { - if (fLog->isDebug( DEBUG_3 )) - { - bWaitedForResize = true; - std::ostringstream oss; - oss << "OID-" << fColInfo->curCol.dataFile.fid << - "; Waiting for pending resize"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - fResizeInProgress.timed_wait(lock, wait_seconds); - - // See if JobStatus has been set to terminate by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - throw SecondaryShutdownException( "ColumnBufferManager::" - "reserveSection(2) responding to job termination"); - } + throw SecondaryShutdownException( + "ColumnBufferManager::" + "reserveSection(2) responding to job termination"); } + } - if (fLog->isDebug( DEBUG_3 )) + if (fLog->isDebug(DEBUG_3)) + { + if (bWaitedForResize) { - if (bWaitedForResize) - { - std::ostringstream oss; - oss << "OID-" << fColInfo->curCol.dataFile.fid << - "; Resume after waiting for pending resize"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + std::ostringstream oss; + oss << "OID-" << fColInfo->curCol.dataFile.fid << "; Resume after waiting for pending resize"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_TO_RESERVE_OUT_BUF); + Stats::stopParseEvent(WE_STATS_WAIT_TO_RESERVE_OUT_BUF); #endif - // Through the use of the mutex lock and the fResizePending flag, nobody - // should be changing the buffer size out from under us; so okay to save in - // local variable till we call resizeColumnBuffer() to expand the buffer. - int bufferSize = fCBuf->getSize(); - int remainingSpace = 0; + // Through the use of the mutex lock and the fResizePending flag, nobody + // should be changing the buffer size out from under us; so okay to save in + // local variable till we call resizeColumnBuffer() to expand the buffer. + int bufferSize = fCBuf->getSize(); + int remainingSpace = 0; - if (bufferSize > 0) + if (bufferSize > 0) + { + // Calculate remaining space + remainingSpace = bufferSize - (fBufFreeOffset + bufferSize - fBufWriteOffset) % bufferSize; + } + + //..Restrict the new section to the extent boundary if applicable. + // We assume here that the colMutex() lock will assure the integrity + // of the values used in evaluating or recalculating spaceRequired. + int nRows = 0; + RETURN_ON_ERROR(rowsExtentCheck(nRowsIn, nRows)); + + int spaceRequired = nRows * fColWidth; + + if (nRows > 0) + { + //..If not enough room to add nRows to output buffer, wait for pending + // sections to be released, so that we can flush and resize the buffer. + //..@bug 3456: compare to remainingSpace-1 and not remainingSpace. + // See note in function description that precedes this function. + if (spaceRequired > (remainingSpace - 1)) { - //Calculate remaining space - remainingSpace = bufferSize - - (fBufFreeOffset + bufferSize - fBufWriteOffset) % bufferSize; - } + //#ifdef PROFILE + // Stats::startParseEvent(WE_STATS_WAIT_TO_RESIZE_OUT_BUF); + //#endif + fResizePending = true; + bool bWaitedForSectionsInUse = false; - //..Restrict the new section to the extent boundary if applicable. - // We assume here that the colMutex() lock will assure the integrity - // of the values used in evaluating or recalculating spaceRequired. - int nRows = 0; - RETURN_ON_ERROR( rowsExtentCheck( nRowsIn, nRows ) ); - - int spaceRequired = nRows * fColWidth; - - if (nRows > 0) - { - //..If not enough room to add nRows to output buffer, wait for pending - // sections to be released, so that we can flush and resize the buffer. - //..@bug 3456: compare to remainingSpace-1 and not remainingSpace. - // See note in function description that precedes this function. - if (spaceRequired > (remainingSpace - 1)) + // Wait for all other threads to finish writing pending sections + // to the output buffer, before we resize the buffer + while (fSectionsInUse.size() > 0) + { + if (fLog->isDebug(DEBUG_3)) { -//#ifdef PROFILE -// Stats::startParseEvent(WE_STATS_WAIT_TO_RESIZE_OUT_BUF); -//#endif - fResizePending = true; - bool bWaitedForSectionsInUse = false; + bWaitedForSectionsInUse = true; + std::ostringstream oss; + oss << "OID-" << fColInfo->curCol.dataFile.fid + << "; Waiting to resize output buffer; " + "sections in-use: " + << fSectionsInUse.size(); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - // Wait for all other threads to finish writing pending sections - // to the output buffer, before we resize the buffer - while (fSectionsInUse.size() > 0) - { - if (fLog->isDebug( DEBUG_3 )) - { - bWaitedForSectionsInUse = true; - std::ostringstream oss; - oss << "OID-" << fColInfo->curCol.dataFile.fid << - "; Waiting to resize output buffer; " - "sections in-use: " << - fSectionsInUse.size(); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + fBufInUse.timed_wait(lock, wait_seconds); - fBufInUse.timed_wait(lock, wait_seconds); + // See if JobStatus has been set to quit by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) + { + throw SecondaryShutdownException( + "ColumnBufferManager::" + "reserveSection(3) responding to job termination"); + } + } - // See if JobStatus has been set to quit by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - throw SecondaryShutdownException( "ColumnBufferManager::" - "reserveSection(3) responding to job termination"); - } - } + //#ifdef PROFILE + // Stats::stopParseEvent(WE_STATS_WAIT_TO_RESIZE_OUT_BUF); + // Stats::startParseEvent(WE_STATS_RESIZE_OUT_BUF); + //#endif -//#ifdef PROFILE -// Stats::stopParseEvent(WE_STATS_WAIT_TO_RESIZE_OUT_BUF); -// Stats::startParseEvent(WE_STATS_RESIZE_OUT_BUF); -//#endif + if (fLog->isDebug(DEBUG_3)) + { + if (bWaitedForSectionsInUse) + { + std::ostringstream oss; + oss << "OID-" << fColInfo->curCol.dataFile.fid << "; Resume after waiting to resize output buffer"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + } - if (fLog->isDebug( DEBUG_3 )) - { - if (bWaitedForSectionsInUse) - { - std::ostringstream oss; - oss << "OID-" << fColInfo->curCol.dataFile.fid << - "; Resume after waiting to resize output buffer"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - } + // @bug 1977 correct problem; writing extra blocks + // Flush remaining data blocks to disk "if" buffer contains data + if (bufferSize > 0) + { + if (fBufFreeOffset != fBufWriteOffset) + RETURN_ON_ERROR(writeToFile((fBufFreeOffset + bufferSize - 1) % bufferSize)); + } - // @bug 1977 correct problem; writing extra blocks - // Flush remaining data blocks to disk "if" buffer contains data - if (bufferSize > 0) - { - if (fBufFreeOffset != fBufWriteOffset) - RETURN_ON_ERROR( writeToFile( - (fBufFreeOffset + bufferSize - 1) % bufferSize) ); - } + resizeColumnBuffer(spaceRequired); + bufferSize = fCBuf->getSize(); // update bufferSize after resize- + // ColumnBuffer() expanded the buffer + fResizePending = false; + fResizeInProgress.notify_all(); + //#ifdef PROFILE + // Stats::stopParseEvent(WE_STATS_RESIZE_OUT_BUF); + //#endif + } // (spaceRequired > remainingSpace-1) - resizeColumnBuffer(spaceRequired); - bufferSize = fCBuf->getSize(); // update bufferSize after resize- - // ColumnBuffer() expanded the buffer - fResizePending = false; - fResizeInProgress.notify_all(); -//#ifdef PROFILE -// Stats::stopParseEvent(WE_STATS_RESIZE_OUT_BUF); -//#endif - } // (spaceRequired > remainingSpace-1) + *cbs = new ColumnBufferSection(fCBuf, startRowId, startRowId + nRows - 1, fColWidth, fBufFreeOffset); + fBufFreeOffset = (fBufFreeOffset + nRows * fColWidth) % bufferSize; + fSectionsInUse.push_back(*cbs); - *cbs = new ColumnBufferSection( - fCBuf, startRowId, startRowId + nRows - 1, - fColWidth, fBufFreeOffset); - fBufFreeOffset = (fBufFreeOffset + nRows * fColWidth) % bufferSize; - fSectionsInUse.push_back(*cbs); + fMaxRowId = startRowId + nRows - 1; + fOutOfSequence.notify_all(); + } // (nRows > 0) - fMaxRowId = startRowId + nRows - 1; - fOutOfSequence.notify_all(); - } // (nRows > 0) + secRowCnt = nRows; - secRowCnt = nRows; + // Get/return last input Row number for the extent this buffer goes in. + // If we determine this set of rows will cross over to the next extent, + // then we tell ColumnInfo to bump the last Row to the end of the next + // extent, in preparation for the next Read buffer going into the next + // extent. We do this even though we have not yet allocated the next + // extent from the extent map. + lastInputRowInExtent = fColInfo->lastInputRowInExtent(); - // Get/return last input Row number for the extent this buffer goes in. - // If we determine this set of rows will cross over to the next extent, - // then we tell ColumnInfo to bump the last Row to the end of the next - // extent, in preparation for the next Read buffer going into the next - // extent. We do this even though we have not yet allocated the next - // extent from the extent map. - lastInputRowInExtent = fColInfo->lastInputRowInExtent( ); + if ((startRowId + nRowsIn) > lastInputRowInExtent) + fColInfo->lastInputRowInExtentInc(); - if ((startRowId + nRowsIn) > lastInputRowInExtent) - fColInfo->lastInputRowInExtentInc( ); - - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -338,42 +328,41 @@ int ColumnBufferManager::reserveSection( int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs) { #ifdef PROFILE - Stats::startParseEvent(WE_STATS_WAIT_TO_RELEASE_OUT_BUF); + Stats::startParseEvent(WE_STATS_WAIT_TO_RELEASE_OUT_BUF); #endif - boost::mutex::scoped_lock lock(fColInfo->colMutex()); + boost::mutex::scoped_lock lock(fColInfo->colMutex()); #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_TO_RELEASE_OUT_BUF); + Stats::stopParseEvent(WE_STATS_WAIT_TO_RELEASE_OUT_BUF); #endif - cbs->setStatus(WRITE_COMPLETE); + cbs->setStatus(WRITE_COMPLETE); - int lastWriteOffset = fBufWriteOffset; + int lastWriteOffset = fBufWriteOffset; - std::list::iterator it = fSectionsInUse.begin(); + std::list::iterator it = fSectionsInUse.begin(); - if (it != fSectionsInUse.end()) + if (it != fSectionsInUse.end()) + { + ColumnBufferSection* cbs_temp = *it; + + while (WRITE_COMPLETE == cbs_temp->getStatus()) { - ColumnBufferSection* cbs_temp = *it; + lastWriteOffset = cbs_temp->getStartOffset() + cbs_temp->getSectionSize() - 1; - while (WRITE_COMPLETE == cbs_temp->getStatus()) - { - lastWriteOffset = cbs_temp->getStartOffset() + - cbs_temp->getSectionSize() - 1; + delete cbs_temp; + it = fSectionsInUse.erase(it); - delete cbs_temp; - it = fSectionsInUse.erase(it); + if (it == fSectionsInUse.end()) + break; - if (it == fSectionsInUse.end()) - break; - - cbs_temp = *it; - } + cbs_temp = *it; } + } - fBufInUse.notify_all(); + fBufInUse.notify_all(); - RETURN_ON_ERROR( writeToFile(lastWriteOffset) ); + RETURN_ON_ERROR(writeToFile(lastWriteOffset)); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -382,77 +371,70 @@ int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs) //------------------------------------------------------------------------------ void ColumnBufferManager::resizeColumnBuffer(int spaceRequired) { - int bufferSize = fCBuf->getSize(); - int bufferSizeOld = bufferSize; - int dataRemaining = (bufferSize > 0) ? - ((fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize) : 0; + int bufferSize = fCBuf->getSize(); + int bufferSizeOld = bufferSize; + int dataRemaining = (bufferSize > 0) ? ((fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize) : 0; - int resizeAction = 0; + int resizeAction = 0; - if (0 == bufferSize) + if (0 == bufferSize) + { + bufferSize = (int)(spaceRequired * 1.2); // Additional 20% to account + // for changes in number of rows + // because of varying line-widths + resizeAction = 1; + } + else + { + if (spaceRequired > bufferSize) { - bufferSize = (int)(spaceRequired * 1.2); //Additional 20% to account - //for changes in number of rows - //because of varying line-widths - resizeAction = 1; + bufferSize = spaceRequired * 2; + resizeAction = 2; } else { - if (spaceRequired > bufferSize) - { - bufferSize = spaceRequired * 2; - resizeAction = 2; - } - else - { - bufferSize *= 2; //Double the buffer size - resizeAction = 3; - } + bufferSize *= 2; // Double the buffer size + resizeAction = 3; } + } - //Round off the bufferSize to size of a disk block - if (bufferSize % BLOCK_SIZE > 0) + // Round off the bufferSize to size of a disk block + if (bufferSize % BLOCK_SIZE > 0) + { + bufferSize = (((int)(bufferSize / BLOCK_SIZE)) + 1) * BLOCK_SIZE; + } + + if (resizeAction > 0) + { + if (fLog->isDebug(DEBUG_2)) { - bufferSize = (((int)(bufferSize / BLOCK_SIZE)) + 1) * BLOCK_SIZE; - } + RID numRowsInBuffer = dataRemaining / fColWidth; + RID firstRid = fMaxRowId - numRowsInBuffer + 1; - if (resizeAction > 0) - { - if (fLog->isDebug( DEBUG_2 )) - { - RID numRowsInBuffer = dataRemaining / fColWidth; - RID firstRid = fMaxRowId - numRowsInBuffer + 1; - - std::ostringstream oss; - oss << "Resizing out buffer (case" << - resizeAction << ") for OID-" << - fColInfo->curCol.dataFile.fid << - "; oldSize-" << bufferSizeOld << - "; freeOff-" << fBufFreeOffset << - "; writeOff-" << fBufWriteOffset << - "; startRID-" << firstRid << - "; rows-" << numRowsInBuffer << - "; reqBytes-" << spaceRequired << - "; newSize-" << bufferSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + std::ostringstream oss; + oss << "Resizing out buffer (case" << resizeAction << ") for OID-" << fColInfo->curCol.dataFile.fid + << "; oldSize-" << bufferSizeOld << "; freeOff-" << fBufFreeOffset << "; writeOff-" + << fBufWriteOffset << "; startRID-" << firstRid << "; rows-" << numRowsInBuffer << "; reqBytes-" + << spaceRequired << "; newSize-" << bufferSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } + } - // @bug 1977 correct problem; writing extra blocks - // If we have no data in buffer, we still call resizeAndCopy() - // to expand the buffer; we just pass -1 for the buffer offsets. - if (fBufFreeOffset == fBufWriteOffset) - { - fCBuf->resizeAndCopy(bufferSize, -1, -1); - } - else - { - int endOffset = (fBufFreeOffset + bufferSize - 1) % bufferSize; - fCBuf->resizeAndCopy(bufferSize, fBufWriteOffset, endOffset); - } + // @bug 1977 correct problem; writing extra blocks + // If we have no data in buffer, we still call resizeAndCopy() + // to expand the buffer; we just pass -1 for the buffer offsets. + if (fBufFreeOffset == fBufWriteOffset) + { + fCBuf->resizeAndCopy(bufferSize, -1, -1); + } + else + { + int endOffset = (fBufFreeOffset + bufferSize - 1) % bufferSize; + fCBuf->resizeAndCopy(bufferSize, fBufWriteOffset, endOffset); + } - fBufFreeOffset = dataRemaining; - fBufWriteOffset = 0; + fBufFreeOffset = dataRemaining; + fBufWriteOffset = 0; } //------------------------------------------------------------------------------ @@ -465,50 +447,44 @@ void ColumnBufferManager::resizeColumnBuffer(int spaceRequired) //------------------------------------------------------------------------------ int ColumnBufferManager::writeToFile(int endOffset) { - int bufferSize = fCBuf->getSize(); - - if (endOffset == fBufWriteOffset) - return NO_ERROR; - - unsigned int writeSize = - (endOffset - fBufWriteOffset + bufferSize) % bufferSize + 1; - - // Don't bother writing anything if we don't at least have a BLOCK_SIZE - // set of bytes to write out; which means we need to be sure to flush - // the buffer at the end, because we could have leftover bytes that we - // have not yet written out. - if (writeSize < BLOCK_SIZE) - return NO_ERROR; - - writeSize = writeSize - writeSize % BLOCK_SIZE; //round down to mult of blksiz - endOffset = (fBufWriteOffset + writeSize - 1) % bufferSize; - - if (fLog->isDebug( DEBUG_3 )) - { - std::ostringstream oss; - oss << "Writing OID-" << fColInfo->curCol.dataFile.fid << - "; bufWriteOff-" << fBufWriteOffset << - "; bufFreeOff-" << fBufFreeOffset << - "; endWrite-" << endOffset << - "; bytesToWrite-" << writeSize << - "; bufSize-" << bufferSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - // Account for circular buffer by making 2 calls to write the data, - // if we are wrapping around at the end of the buffer. - if (endOffset < fBufWriteOffset) - { - RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, bufferSize - fBufWriteOffset) ); - fBufWriteOffset = 0; - } - - RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, endOffset - fBufWriteOffset + 1) ); - fBufWriteOffset = (endOffset + 1) % bufferSize; + int bufferSize = fCBuf->getSize(); + if (endOffset == fBufWriteOffset) return NO_ERROR; + + unsigned int writeSize = (endOffset - fBufWriteOffset + bufferSize) % bufferSize + 1; + + // Don't bother writing anything if we don't at least have a BLOCK_SIZE + // set of bytes to write out; which means we need to be sure to flush + // the buffer at the end, because we could have leftover bytes that we + // have not yet written out. + if (writeSize < BLOCK_SIZE) + return NO_ERROR; + + writeSize = writeSize - writeSize % BLOCK_SIZE; // round down to mult of blksiz + endOffset = (fBufWriteOffset + writeSize - 1) % bufferSize; + + if (fLog->isDebug(DEBUG_3)) + { + std::ostringstream oss; + oss << "Writing OID-" << fColInfo->curCol.dataFile.fid << "; bufWriteOff-" << fBufWriteOffset + << "; bufFreeOff-" << fBufFreeOffset << "; endWrite-" << endOffset << "; bytesToWrite-" << writeSize + << "; bufSize-" << bufferSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + // Account for circular buffer by making 2 calls to write the data, + // if we are wrapping around at the end of the buffer. + if (endOffset < fBufWriteOffset) + { + RETURN_ON_ERROR(writeToFileExtentCheck(fBufWriteOffset, bufferSize - fBufWriteOffset)); + fBufWriteOffset = 0; + } + + RETURN_ON_ERROR(writeToFileExtentCheck(fBufWriteOffset, endOffset - fBufWriteOffset + 1)); + fBufWriteOffset = (endOffset + 1) % bufferSize; + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -530,178 +506,162 @@ int ColumnBufferManager::writeToFile(int endOffset) // second db column file has to be opened to finish writing the // internal buffer, or if an abbreviated extent is expanded. //------------------------------------------------------------------------------ -int ColumnBufferManager::writeToFileExtentCheck( - uint32_t startOffset, uint32_t writeSize, bool fillUpWEmpties) +int ColumnBufferManager::writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize, bool fillUpWEmpties) { + if (fLog->isDebug(DEBUG_3)) + { + std::ostringstream oss; + oss << "Col extent check: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" + << fColInfo->curCol.dataFile.fSegment << "; Wanting to write " << writeSize + << " bytes, with avail space " << fColInfo->availFileSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - if (fLog->isDebug( DEBUG_3 )) + // Don't need a mutex lock here because if writeToFile() is calling + // us, we already have a lock; and if flush() is calling us, then + // all parsing is complete, so we should have no thread contention. + + // If extent out of space, see if this is an abbrev extent we can expand + long long availableFileSize = fColInfo->availFileSize; + + if ((availableFileSize < writeSize) && (fColInfo->isAbbrevExtent())) + { + int rc = fColInfo->expandAbbrevExtent(true); + + if (rc != NO_ERROR) { + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFileExtentCheck: expand extent failed: " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + availableFileSize = fColInfo->availFileSize; + } + + if (availableFileSize >= writeSize) + { + int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWEmpties); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFileExtentCheck: write1 extent failed: " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + // MCOL-498 Fill this block up to its boundary. + if (fillUpWEmpties) + { + writeSize = BLOCK_SIZE; + } + + fColInfo->updateBytesWrittenCounts(writeSize); + } + else + { + // We use ColumnInfo to help us add an extent to the "next" + // segment file, if needed. + // Current extent does not have enough room for buffer, so we + // have to break up the buffer into 2 extents; creating a new + // extent and switching the db column file "on-the-fly". + int writeSize1 = availableFileSize; + + if (writeSize1 > 0) + { + int rc = fCBuf->writeToFile(startOffset, writeSize1); + + if (rc != NO_ERROR) + { + WErrorCodes ec; std::ostringstream oss; - oss << "Col extent check: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; Wanting to write " << writeSize << - " bytes, with avail space " << fColInfo->availFileSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + oss << "writeToFileExtentCheck: write2 extent failed: " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + fColInfo->updateBytesWrittenCounts(writeSize1); } - // Don't need a mutex lock here because if writeToFile() is calling - // us, we already have a lock; and if flush() is calling us, then - // all parsing is complete, so we should have no thread contention. + int rc = fColInfo->extendColumn(true); - // If extent out of space, see if this is an abbrev extent we can expand - long long availableFileSize = fColInfo->availFileSize; - - if ((availableFileSize < writeSize) && (fColInfo->isAbbrevExtent())) + if (rc != NO_ERROR) { - int rc = fColInfo->expandAbbrevExtent(true); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFileExtentCheck: expand extent failed: " << - ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - availableFileSize = fColInfo->availFileSize; + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFileExtentCheck: extend column failed: " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } - if (availableFileSize >= writeSize) + int writeSize2 = writeSize - writeSize1; + rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWEmpties); + + if (rc != NO_ERROR) { - int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWEmpties); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFileExtentCheck: write1 extent failed: " << - ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - // MCOL-498 Fill this block up to its boundary. - if ( fillUpWEmpties ) - { - writeSize = BLOCK_SIZE; - } - - fColInfo->updateBytesWrittenCounts( writeSize ); + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFileExtentCheck: write3 extent failed: " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } - else + + // MCOL-498 Fill this block up to its boundary. + if (fillUpWEmpties) { - // We use ColumnInfo to help us add an extent to the "next" - // segment file, if needed. - // Current extent does not have enough room for buffer, so we - // have to break up the buffer into 2 extents; creating a new - // extent and switching the db column file "on-the-fly". - int writeSize1 = availableFileSize; - - if (writeSize1 > 0) - { - int rc = fCBuf->writeToFile(startOffset, writeSize1); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFileExtentCheck: write2 extent failed: " << - ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - fColInfo->updateBytesWrittenCounts( writeSize1 ); - } - - int rc = fColInfo->extendColumn( true ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFileExtentCheck: extend column failed: " << - ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - int writeSize2 = writeSize - writeSize1; - rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWEmpties); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFileExtentCheck: write3 extent failed: " << - ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - // MCOL-498 Fill this block up to its boundary. - if ( fillUpWEmpties ) - { - writeSize2 = BLOCK_SIZE; - } - - fColInfo->updateBytesWrittenCounts( writeSize2 ); + writeSize2 = BLOCK_SIZE; } - return NO_ERROR; + fColInfo->updateBytesWrittenCounts(writeSize2); + } + + return NO_ERROR; } //------------------------------------------------------------------------------ // Flush the contents of internal fCBuf (column buffer) to disk. If CS flushes // less then BLOCK_SIZE bytes then it propagates this event down the stack. //------------------------------------------------------------------------------ -int ColumnBufferManager::flush( ) +int ColumnBufferManager::flush() { - - if (fBufFreeOffset == fBufWriteOffset) + if (fBufFreeOffset == fBufWriteOffset) + { + if (fLog->isDebug(DEBUG_2)) { - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Skipping write flush for: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; both fBufFreeOffset and fBufWriteOffset = " << - fBufFreeOffset; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - return NO_ERROR; + std::ostringstream oss; + oss << "Skipping write flush for: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition + << "; seg-" << fColInfo->curCol.dataFile.fSegment + << "; both fBufFreeOffset and fBufWriteOffset = " << fBufFreeOffset; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } - int bufferSize = fCBuf->getSize(); - - // MCOL-498 There are less the BLOCK_SIZE bytes in the buffer left - // so propagate this info down the stack to fill the buffer up - // with empty magics. - // Account for circular buffer by making 2 calls to write the data, - // if we are wrapping around at the end of the buffer. - if (fBufFreeOffset < fBufWriteOffset) - { - bool fillUpWEmpties = ( static_cast(bufferSize - fBufWriteOffset) >= BLOCK_SIZE ) - ? false : true; - RETURN_ON_ERROR( writeToFileExtentCheck( fBufWriteOffset, - bufferSize - fBufWriteOffset, fillUpWEmpties) ); - fBufWriteOffset = 0; - } - // MCOL-498 fill the buffer up with empty magics. - RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, fBufFreeOffset - fBufWriteOffset, true) ); - fBufWriteOffset = fBufFreeOffset; - return NO_ERROR; + } + + int bufferSize = fCBuf->getSize(); + + // MCOL-498 There are less the BLOCK_SIZE bytes in the buffer left + // so propagate this info down the stack to fill the buffer up + // with empty magics. + // Account for circular buffer by making 2 calls to write the data, + // if we are wrapping around at the end of the buffer. + if (fBufFreeOffset < fBufWriteOffset) + { + bool fillUpWEmpties = + (static_cast(bufferSize - fBufWriteOffset) >= BLOCK_SIZE) ? false : true; + RETURN_ON_ERROR(writeToFileExtentCheck(fBufWriteOffset, bufferSize - fBufWriteOffset, fillUpWEmpties)); + fBufWriteOffset = 0; + } + // MCOL-498 fill the buffer up with empty magics. + RETURN_ON_ERROR(writeToFileExtentCheck(fBufWriteOffset, fBufFreeOffset - fBufWriteOffset, true)); + fBufWriteOffset = fBufFreeOffset; + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -715,34 +675,35 @@ int ColumnBufferManager::flush( ) //------------------------------------------------------------------------------ int ColumnBufferManager::intermediateFlush() { - boost::posix_time::seconds wait_seconds(COND_WAIT_SECONDS); - boost::mutex::scoped_lock lock(fColInfo->colMutex()); + boost::posix_time::seconds wait_seconds(COND_WAIT_SECONDS); + boost::mutex::scoped_lock lock(fColInfo->colMutex()); - // Wait for all other threads which are currently parsing rows, - // to finish parsing the data in those sections. + // Wait for all other threads which are currently parsing rows, + // to finish parsing the data in those sections. #ifdef PROFILE - Stats::startParseEvent(WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH); + Stats::startParseEvent(WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH); #endif - while (fSectionsInUse.size() > 0) + while (fSectionsInUse.size() > 0) + { + fBufInUse.timed_wait(lock, wait_seconds); + + // See if JobStatus has been set to terminate by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) { - fBufInUse.timed_wait(lock, wait_seconds); - - // See if JobStatus has been set to terminate by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - throw SecondaryShutdownException( "ColumnBufferManager::" - "intermediateFlush() responding to job termination"); - } + throw SecondaryShutdownException( + "ColumnBufferManager::" + "intermediateFlush() responding to job termination"); } + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH); + Stats::stopParseEvent(WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH); #endif - RETURN_ON_ERROR( flush( ) ); + RETURN_ON_ERROR(flush()); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -751,11 +712,11 @@ int ColumnBufferManager::intermediateFlush() // as sections from the output buffer are being copied to the column segment // file(s). //------------------------------------------------------------------------------ -int ColumnBufferManager::rowsExtentCheck( int nRows, int& nRows2 ) +int ColumnBufferManager::rowsExtentCheck(int nRows, int& nRows2) { - nRows2 = nRows; + nRows2 = nRows; - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -766,11 +727,11 @@ int ColumnBufferManager::rowsExtentCheck( int nRows, int& nRows2 ) // ColumnBufferManager; hence the need for this function to be called to employ // a mutex lock around the call to extendColumn(). //------------------------------------------------------------------------------ -int ColumnBufferManager::extendTokenColumn( ) +int ColumnBufferManager::extendTokenColumn() { - boost::mutex::scoped_lock lock(fColInfo->colMutex()); + boost::mutex::scoped_lock lock(fColInfo->colMutex()); - return fColInfo->extendColumn( false ); + return fColInfo->extendColumn(false); } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colbufmgr.h b/writeengine/bulk/we_colbufmgr.h index eb0a51370..6619a0d16 100644 --- a/writeengine/bulk/we_colbufmgr.h +++ b/writeengine/bulk/we_colbufmgr.h @@ -59,205 +59,199 @@ class ColumnInfo; class ColumnBufferManager { -public: + public: + //------------------------------------------------------------------------- + // Public Functions + //------------------------------------------------------------------------- - //------------------------------------------------------------------------- - // Public Functions - //------------------------------------------------------------------------- + /** @brief Constructor + * + * @param ColumnInfo object used to manage the addition of extents to + * @param colWidth Width of the column + * @param logger Logging object used for logging + * @param compressionType Compression type + */ + ColumnBufferManager(ColumnInfo* pColInfo, int colWidth, Log* logger, int compressionType); - /** @brief Constructor - * - * @param ColumnInfo object used to manage the addition of extents to - * @param colWidth Width of the column - * @param logger Logging object used for logging - * @param compressionType Compression type - */ - ColumnBufferManager(ColumnInfo* pColInfo, int colWidth, - Log* logger, int compressionType); + /** @brief Default destructor + */ + virtual ~ColumnBufferManager(); - /** @brief Default destructor - */ - virtual ~ColumnBufferManager(); + /** @brief Reserve section of the buffer for this thread to write to + * + * @param startRowId (in) The starting row-id in the column that this + * section will write to + * @param nRows (in) Requested size of the section (number of rows) + * @param secRowCnt (out) Resulting size of the reserved section. + * May be less than nRows for the case where a group of + * dictionary rows crosses an extent boundary. + * @param ColumnBufferSection (out) ptr with info on reserved section + * @param lastRIDInExtent (out) last RID in relevant extent. + * @return success or fail status + */ + int reserveSection(RID startRowId, uint32_t nRows, uint32_t& secRowCnt, ColumnBufferSection** cbs, + RID& lastRIDInExtent); - /** @brief Reserve section of the buffer for this thread to write to - * - * @param startRowId (in) The starting row-id in the column that this - * section will write to - * @param nRows (in) Requested size of the section (number of rows) - * @param secRowCnt (out) Resulting size of the reserved section. - * May be less than nRows for the case where a group of - * dictionary rows crosses an extent boundary. - * @param ColumnBufferSection (out) ptr with info on reserved section - * @param lastRIDInExtent (out) last RID in relevant extent. - * @return success or fail status - */ - int reserveSection(RID startRowId, uint32_t nRows, - uint32_t& secRowCnt, - ColumnBufferSection** cbs, - RID& lastRIDInExtent); + /** @brief Release a section that has been reserved earlier. + * The ColumnBufferSection pointer will be invalid after + * releaseSection() is called. releaseSection() releases all memory + * associated with the ColumnBufferSection and the client is not + * expected to free any memory. + * + * @param cbs ColumnBufferSection pointer denoting the pointer that + * needs to be released + * @return success or fail status + */ + int releaseSection(ColumnBufferSection* cbs); - /** @brief Release a section that has been reserved earlier. - * The ColumnBufferSection pointer will be invalid after - * releaseSection() is called. releaseSection() releases all memory - * associated with the ColumnBufferSection and the client is not - * expected to free any memory. - * - * @param cbs ColumnBufferSection pointer denoting the pointer that - * needs to be released - * @return success or fail status - */ - int releaseSection(ColumnBufferSection* cbs); + /** @brief Final flushing of data and headers prior to closing the file. + * @param bTruncFile is file to be truncated + * @return NO_ERROR or success + */ + int finishFile(bool bTruncFile) + { + return fCBuf->finishFile(bTruncFile); + } - /** @brief Final flushing of data and headers prior to closing the file. - * @param bTruncFile is file to be truncated - * @return NO_ERROR or success - */ - int finishFile(bool bTruncFile) - { - return fCBuf->finishFile(bTruncFile); - } + /** @brief Method to ensure that all the data in the buffer has been + * written to the file (for uncompressed), or the to-be-compressed buffer + * (for compressed). In the case of compressed data, finishFile() is + * responsible for flushing any remaining data in the to-be-compressed + * buffer out to the db file. + * @return NO_ERROR or success + */ + int flush(); - /** @brief Method to ensure that all the data in the buffer has been - * written to the file (for uncompressed), or the to-be-compressed buffer - * (for compressed). In the case of compressed data, finishFile() is - * responsible for flushing any remaining data in the to-be-compressed - * buffer out to the db file. - * @return NO_ERROR or success - */ - int flush( ); + /** @brief Flush the buffer in the middle of parsing. + * + * This is used when we want to split up a buffer that is about to + * cross an extent boundary. + * @return NO_ERROR or success + */ + int intermediateFlush(); - /** @brief Flush the buffer in the middle of parsing. - * - * This is used when we want to split up a buffer that is about to - * cross an extent boundary. - * @return NO_ERROR or success - */ - int intermediateFlush(); + /** @brief Set the IDBDataFile** destination for the applicable col segment file. + * + * @param cFile IDBDataFile* of the output column segment file. + * @param hwm Starting HWM for the file. + * @param hdrs with ptr information (only applies to compressed files) + */ + int setDbFile(IDBDataFile* const cFile, HWM hwm, const char* hdrs) + { + return fCBuf->setDbFile(cFile, hwm, hdrs); + } - /** @brief Set the IDBDataFile** destination for the applicable col segment file. - * - * @param cFile IDBDataFile* of the output column segment file. - * @param hwm Starting HWM for the file. - * @param hdrs with ptr information (only applies to compressed files) - */ - int setDbFile(IDBDataFile* const cFile, HWM hwm, const char* hdrs) - { - return fCBuf->setDbFile(cFile, hwm, hdrs); - } + /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the + * next extent. + */ + int resetToBeCompressedColBuf(long long& startFileOffset) + { + return fCBuf->resetToBeCompressedColBuf(startFileOffset); + } - /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the - * next extent. - */ - int resetToBeCompressedColBuf(long long& startFileOffset) - { - return fCBuf->resetToBeCompressedColBuf( startFileOffset ); - } + /** @brief Wrapper around extendColumn(), used for dictionary token columns. + */ + int extendTokenColumn(); - /** @brief Wrapper around extendColumn(), used for dictionary token columns. - */ - int extendTokenColumn( ); + protected: + //------------------------------------------------------------------------- + // Protected Functions + //------------------------------------------------------------------------- -protected: + /** @brief Resize the internal column buffer + * + * @param spaceRequired Amount of additional space required + */ + void resizeColumnBuffer(int spaceRequired); - //------------------------------------------------------------------------- - // Protected Functions - //------------------------------------------------------------------------- + /** @brief See if buffer has room for "nRows" without filling up the + * current extent. + * + * @param nRows The number of rows ready to add to the buffer. + * @param nRows2 If there is not room, then nRows2 is the allowable + * number of rows to add, else nRows2 will be "nRows". + * @return NO_ERROR or success + */ + virtual int rowsExtentCheck(int nRows, int& nRows2); - /** @brief Resize the internal column buffer - * - * @param spaceRequired Amount of additional space required - */ - void resizeColumnBuffer(int spaceRequired); + /** @brief Write buffer data to file + * + * @param Highest buffer offset which data should be written to file + * @return NO_ERROR or success + */ + int writeToFile(int endOffset); - /** @brief See if buffer has room for "nRows" without filling up the - * current extent. - * - * @param nRows The number of rows ready to add to the buffer. - * @param nRows2 If there is not room, then nRows2 is the allowable - * number of rows to add, else nRows2 will be "nRows". - * @return NO_ERROR or success - */ - virtual int rowsExtentCheck( int nRows, int& nRows2 ); + /** @brief Write buffer data to file while checking for extent boundary. + * + * WARNING: This function will update contents of ColumnInfo struct that + * owns this ColumnBufferManger if more than 1 extent is required to + * write out the buffer. + * @param startOffset The buffer offset where the write should begin + * @param writeSize The number of bytes to be written to the file + * @param fillUpWEmpties The flag to fill the buffer with NULLs up to + * the block boundary. + * @return success or fail status + */ + virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize, bool fillUpWEmpties = false); - /** @brief Write buffer data to file - * - * @param Highest buffer offset which data should be written to file - * @return NO_ERROR or success - */ - int writeToFile(int endOffset); + //------------------------------------------------------------------------- + // Protected Data Members + //------------------------------------------------------------------------- - /** @brief Write buffer data to file while checking for extent boundary. - * - * WARNING: This function will update contents of ColumnInfo struct that - * owns this ColumnBufferManger if more than 1 extent is required to - * write out the buffer. - * @param startOffset The buffer offset where the write should begin - * @param writeSize The number of bytes to be written to the file - * @param fillUpWEmpties The flag to fill the buffer with NULLs up to - * the block boundary. - * @return success or fail status - */ - virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize, - bool fillUpWEmpties = false); + /** @brief Internal ColumnBuffer + * + * The ColumnBuffer is used by this class as a circular buffer. The + * variables fBufWriteOffset and fBufFreeOffset are used to keep track + * of free and allocated space. + */ + ColumnBuffer* fCBuf; - //------------------------------------------------------------------------- - // Protected Data Members - //------------------------------------------------------------------------- + /** @brief Offset from where the next file write should start + */ + int fBufWriteOffset; - /** @brief Internal ColumnBuffer - * - * The ColumnBuffer is used by this class as a circular buffer. The - * variables fBufWriteOffset and fBufFreeOffset are used to keep track - * of free and allocated space. - */ - ColumnBuffer* fCBuf; + /** @brief Offset from where the next section allocation should start + */ + int fBufFreeOffset; - /** @brief Offset from where the next file write should start - */ - int fBufWriteOffset; + /** @brief List of currently in-use sections + */ + std::list fSectionsInUse; - /** @brief Offset from where the next section allocation should start - */ - int fBufFreeOffset; + /** @brief Flag indicating that an internal buffer resize is in progress + */ + bool fResizePending; - /** @brief List of currently in-use sections - */ - std::list fSectionsInUse; + /** @brief Condition variable for threads waiting for resize to complete + */ + boost::condition fResizeInProgress; - /** @brief Flag indicating that an internal buffer resize is in progress - */ - bool fResizePending; + /** @brief Condition variable for threads waiting for all buffer sections + * to be released + */ + boost::condition fBufInUse; - /** @brief Condition variable for threads waiting for resize to complete - */ - boost::condition fResizeInProgress; + /** @brief Condition variable for threads who have arrived out-of- + * sequence with respect to their row-id + */ + boost::condition fOutOfSequence; - /** @brief Condition variable for threads waiting for all buffer sections - * to be released - */ - boost::condition fBufInUse; + /** @brief Width of the column + */ + int fColWidth; - /** @brief Condition variable for threads who have arrived out-of- - * sequence with respect to their row-id - */ - boost::condition fOutOfSequence; + /** @brief Maximum row-id among all the alocated sections + */ + RID fMaxRowId; - /** @brief Width of the column - */ - int fColWidth; + /** @brief Parent ColumnInfo object used to manage the addition of + * extents to the applicable database column segment files. + */ + ColumnInfo* fColInfo; - /** @brief Maximum row-id among all the alocated sections - */ - RID fMaxRowId; - - /** @brief Parent ColumnInfo object used to manage the addition of - * extents to the applicable database column segment files. - */ - ColumnInfo* fColInfo; - - /** @brief Object used for logging - */ - Log* fLog; + /** @brief Object used for logging + */ + Log* fLog; }; //------------------------------------------------------------------------------ @@ -265,16 +259,14 @@ protected: //------------------------------------------------------------------------------ class ColumnBufferManagerDctnry : public ColumnBufferManager { + public: + ColumnBufferManagerDctnry(ColumnInfo* pColInfo, int colWidth, Log* logger, int compressionType); + virtual ~ColumnBufferManagerDctnry(); -public: - ColumnBufferManagerDctnry(ColumnInfo* pColInfo, int colWidth, - Log* logger, int compressionType); - virtual ~ColumnBufferManagerDctnry(); - - virtual int rowsExtentCheck( int nRows, int& nRows2 ); - using ColumnBufferManager::writeToFileExtentCheck; - virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize); + virtual int rowsExtentCheck(int nRows, int& nRows2); + using ColumnBufferManager::writeToFileExtentCheck; + virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize); }; -} +} // namespace WriteEngine #endif /*WRITEENGINE_COLBUFMGR_H*/ diff --git a/writeengine/bulk/we_colbufmgrdctnry.cpp b/writeengine/bulk/we_colbufmgrdctnry.cpp index 4f25294b1..41a27c1b3 100644 --- a/writeengine/bulk/we_colbufmgrdctnry.cpp +++ b/writeengine/bulk/we_colbufmgrdctnry.cpp @@ -28,14 +28,13 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // ColumnBufferManagerDctnry constructor that takes a ColumnInfo, colWidth, and // FILE*. //------------------------------------------------------------------------------ -ColumnBufferManagerDctnry::ColumnBufferManagerDctnry( - ColumnInfo* pColInfo, int colWidth, Log* logger, int compressionType) : - ColumnBufferManager(pColInfo, colWidth, logger, compressionType) +ColumnBufferManagerDctnry::ColumnBufferManagerDctnry(ColumnInfo* pColInfo, int colWidth, Log* logger, + int compressionType) + : ColumnBufferManager(pColInfo, colWidth, logger, compressionType) { } @@ -56,38 +55,34 @@ ColumnBufferManagerDctnry::~ColumnBufferManagerDctnry() // The data to be written, starts at "startOffset" in the internal buffer and // is "writeSize" bytes long. //------------------------------------------------------------------------------ -int ColumnBufferManagerDctnry::writeToFileExtentCheck( - uint32_t startOffset, uint32_t writeSize) +int ColumnBufferManagerDctnry::writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize) { - if (fLog->isDebug( DEBUG_3 )) - { - std::ostringstream oss; - oss << "Dctnry writeToFileExtentCheck" - ": OID-" << fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; writeSize-" << writeSize << - "; oldAvailFileSize-" << fColInfo->availFileSize << - "; newAvailFileSize-" << (fColInfo->availFileSize - writeSize); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + if (fLog->isDebug(DEBUG_3)) + { + std::ostringstream oss; + oss << "Dctnry writeToFileExtentCheck" + ": OID-" + << fColInfo->curCol.dataFile.fid << "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << "; part-" + << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment + << "; writeSize-" << writeSize << "; oldAvailFileSize-" << fColInfo->availFileSize + << "; newAvailFileSize-" << (fColInfo->availFileSize - writeSize); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - int rc = fCBuf->writeToFile(startOffset, writeSize); + int rc = fCBuf->writeToFile(startOffset, writeSize); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "writeToFileExtentCheck: write token extent failed: " << - ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "writeToFileExtentCheck: write token extent failed: " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } - fColInfo->updateBytesWrittenCounts( writeSize ); + fColInfo->updateBytesWrittenCounts(writeSize); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -102,70 +97,59 @@ int ColumnBufferManagerDctnry::writeToFileExtentCheck( // information in the ColumnInfo struct that owns this // ColumnBufferManagerDctnry. //------------------------------------------------------------------------------ -int ColumnBufferManagerDctnry::rowsExtentCheck( int nRows, int& nRows2 ) +int ColumnBufferManagerDctnry::rowsExtentCheck(int nRows, int& nRows2) { - nRows2 = nRows; + nRows2 = nRows; - int bufferSize = fCBuf->getSize(); - long long spaceRequired = nRows * fColWidth; - long dataInBuffer = 0; + int bufferSize = fCBuf->getSize(); + long long spaceRequired = nRows * fColWidth; + long dataInBuffer = 0; - if (bufferSize > 0) - dataInBuffer = (fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize; + if (bufferSize > 0) + dataInBuffer = (fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize; - // if extent is out of space, see if this is an abbrev extent we can expand - if (((dataInBuffer + spaceRequired) > fColInfo->availFileSize) && - (fColInfo->isAbbrevExtent())) + // if extent is out of space, see if this is an abbrev extent we can expand + if (((dataInBuffer + spaceRequired) > fColInfo->availFileSize) && (fColInfo->isAbbrevExtent())) + { + RETURN_ON_ERROR(fColInfo->expandAbbrevExtent(true)) + } + + if ((dataInBuffer + spaceRequired) > fColInfo->availFileSize) + { + spaceRequired = fColInfo->availFileSize - dataInBuffer; + nRows2 = (int)(spaceRequired / fColWidth); + + if (fLog->isDebug(DEBUG_1)) { - RETURN_ON_ERROR( fColInfo->expandAbbrevExtent(true) ) - } + std::ostringstream oss1; + oss1 << "Dctnry rowsExtentCheck (filling extent): OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition + << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; spaceRequired-" << spaceRequired + << "; dataInBuffer-" << dataInBuffer << "; availSpace-" << fColInfo->availFileSize; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); - if ((dataInBuffer + spaceRequired) > fColInfo->availFileSize) + std::ostringstream oss2; + oss2 << "Dctnry rowsExtentCheck: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition + << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; Changing nRows from " << nRows << " to " + << nRows2; + fLog->logMsg(oss2.str(), MSGLVL_INFO2); + } + } + else + { + if (fLog->isDebug(DEBUG_2)) { - spaceRequired = fColInfo->availFileSize - dataInBuffer; - nRows2 = (int)(spaceRequired / fColWidth); - - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss1; - oss1 << "Dctnry rowsExtentCheck (filling extent): OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; spaceRequired-" << spaceRequired << - "; dataInBuffer-" << dataInBuffer << - "; availSpace-" << fColInfo->availFileSize; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - - std::ostringstream oss2; - oss2 << "Dctnry rowsExtentCheck: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; Changing nRows from " << nRows << " to " << nRows2; - fLog->logMsg( oss2.str(), MSGLVL_INFO2 ); - } - } - else - { - if (fLog->isDebug( DEBUG_2 )) - { - std::ostringstream oss; - oss << "Dctnry rowsExtentCheck: OID-" << - fColInfo->curCol.dataFile.fid << - "; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot << - "; part-" << fColInfo->curCol.dataFile.fPartition << - "; seg-" << fColInfo->curCol.dataFile.fSegment << - "; spaceRequired-" << spaceRequired << - "; dataInBuffer-" << dataInBuffer << - "; availSpace-" << fColInfo->availFileSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + std::ostringstream oss; + oss << "Dctnry rowsExtentCheck: OID-" << fColInfo->curCol.dataFile.fid << "; DBRoot-" + << fColInfo->curCol.dataFile.fDbRoot << "; part-" << fColInfo->curCol.dataFile.fPartition + << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; spaceRequired-" << spaceRequired + << "; dataInBuffer-" << dataInBuffer << "; availSpace-" << fColInfo->availFileSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } + } - return NO_ERROR; + return NO_ERROR; } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colbufsec.cpp b/writeengine/bulk/we_colbufsec.cpp index df6445fa7..7da35fe66 100644 --- a/writeengine/bulk/we_colbufsec.cpp +++ b/writeengine/bulk/we_colbufsec.cpp @@ -28,20 +28,15 @@ namespace WriteEngine { - -ColumnBufferSection::ColumnBufferSection( - ColumnBuffer* const cb, - RID sRowId, - RID eRowId, - int width, - int sOffset) - : fCBuf(cb), - fStartRowId(sRowId), - fEndRowId(eRowId), - fColWidth(width), - fBufStartOffset(sOffset), - fCurrRowId(sRowId), - fStatus(INIT_COMPLETE) +ColumnBufferSection::ColumnBufferSection(ColumnBuffer* const cb, RID sRowId, RID eRowId, int width, + int sOffset) + : fCBuf(cb) + , fStartRowId(sRowId) + , fEndRowId(eRowId) + , fColWidth(width) + , fBufStartOffset(sOffset) + , fCurrRowId(sRowId) + , fStatus(INIT_COMPLETE) { } @@ -51,49 +46,47 @@ ColumnBufferSection::~ColumnBufferSection() void ColumnBufferSection::write(const void* const data, int nRows) { -//Casting void * to unsigned char * without modifying the constness - const unsigned char* const tData = - static_cast(data); + // Casting void * to unsigned char * without modifying the constness + const unsigned char* const tData = static_cast(data); - if (fCurrRowId + nRows + 1 > fEndRowId) - { - //TODO: Handle error (old-dmc) - } + if (fCurrRowId + nRows + 1 > fEndRowId) + { + // TODO: Handle error (old-dmc) + } - int startOffset = (fBufStartOffset + (fCurrRowId - fStartRowId) * - fColWidth) % fCBuf->getSize(); - int nBytes = nRows * fColWidth; - int bytesWritten = 0; + int startOffset = (fBufStartOffset + (fCurrRowId - fStartRowId) * fColWidth) % fCBuf->getSize(); + int nBytes = nRows * fColWidth; + int bytesWritten = 0; - if ((startOffset + nBytes) > fCBuf->getSize()) - { - fCBuf->write(tData, startOffset, fCBuf->getSize() - startOffset); - bytesWritten = fCBuf->getSize() - startOffset; - startOffset = 0; - } + if ((startOffset + nBytes) > fCBuf->getSize()) + { + fCBuf->write(tData, startOffset, fCBuf->getSize() - startOffset); + bytesWritten = fCBuf->getSize() - startOffset; + startOffset = 0; + } - fCBuf->write(tData + bytesWritten, startOffset, nBytes - bytesWritten); - fCurrRowId += nRows; + fCBuf->write(tData + bytesWritten, startOffset, nBytes - bytesWritten); + fCurrRowId += nRows; } void ColumnBufferSection::setStatus(int s) { - fStatus = s; + fStatus = s; } int ColumnBufferSection::getStatus() const { - return fStatus; + return fStatus; } int ColumnBufferSection::getStartOffset() const { - return fBufStartOffset; + return fBufStartOffset; } int ColumnBufferSection::getSectionSize() const { - return (fEndRowId - fStartRowId + 1) * fColWidth; + return (fEndRowId - fStartRowId + 1) * fColWidth; } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colbufsec.h b/writeengine/bulk/we_colbufsec.h index b68becd2c..1744ebec6 100644 --- a/writeengine/bulk/we_colbufsec.h +++ b/writeengine/bulk/we_colbufsec.h @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - /****************************************************************************** * $Id: we_colbufsec.h 4450 2013-01-21 14:13:24Z rdempsey $ * @@ -32,13 +31,12 @@ namespace WriteEngine { - /* @brief Status codes for the ColumnBufferSection */ enum { - INIT_COMPLETE, - WRITE_COMPLETE, + INIT_COMPLETE, + WRITE_COMPLETE, }; /** @@ -47,74 +45,70 @@ enum */ class ColumnBufferSection { + public: + /* @brief Constructor + * + * @param cb Ptr to this section's corresponding CoulmnBuffer + * @param startRowId Starting row-id of the column + * @param endRowId Ending row-id of the column + * @param Width Width of the underlying column + * @param startOffset + */ + ColumnBufferSection(ColumnBuffer* const cb, RID startRowId, RID endRowId, int colWidth, int startOffset); -public: - /* @brief Constructor - * - * @param cb Ptr to this section's corresponding CoulmnBuffer - * @param startRowId Starting row-id of the column - * @param endRowId Ending row-id of the column - * @param Width Width of the underlying column - * @param startOffset - */ - ColumnBufferSection(ColumnBuffer* const cb, RID startRowId, - RID endRowId, int colWidth, int startOffset); + /* @brief Default destructor + */ + ~ColumnBufferSection(); - /* @brief Default destructor - */ - ~ColumnBufferSection(); + /* @brief Updates the status of the section + * + * @param Status value + */ + void setStatus(int status); - /* @brief Updates the status of the section - * - * @param Status value - */ - void setStatus(int status); + /* @brief Write data into the column buffer section + * + * @param data pointer to the data + * @param nRows Number of rows to be written, starting from data pointer + */ + void write(const void* const data, int nRows); - /* @brief Write data into the column buffer section - * - * @param data pointer to the data - * @param nRows Number of rows to be written, starting from data pointer - */ - void write(const void* const data, int nRows); + /* @brief Returns the current status of the section + */ + int getStatus() const; - /* @brief Returns the current status of the section - */ - int getStatus() const; + /* @brief Returns the start offset of this section + */ + int getStartOffset() const; - /* @brief Returns the start offset of this section - */ - int getStartOffset() const; + /* @brief Returns the size of this section + */ + int getSectionSize() const; - /* @brief Returns the size of this section - */ - int getSectionSize() const; + /* @brief Returns the ending row-id of this section + */ + RID endRowId() const + { + return fEndRowId; + } - /* @brief Returns the ending row-id of this section - */ - RID endRowId() const - { - return fEndRowId; - } + /* @brief Returns the starting row-id of this section + */ + RID startRowId() const + { + return fStartRowId; + } - /* @brief Returns the starting row-id of this section - */ - RID startRowId() const - { - return fStartRowId; - } - -private: - - const ColumnBuffer* const fCBuf; //ColumnBuffer associated with this section - RID fStartRowId; // Starting row-id for this section - RID fEndRowId; // Ending row-id for this section - int fColWidth; // Width of column (in bytes) - int fBufStartOffset; // Starting offset of this section in ColumnBuffer - RID fCurrRowId; // row-id which will be written next - int fStatus; // Status of this section + private: + const ColumnBuffer* const fCBuf; // ColumnBuffer associated with this section + RID fStartRowId; // Starting row-id for this section + RID fEndRowId; // Ending row-id for this section + int fColWidth; // Width of column (in bytes) + int fBufStartOffset; // Starting offset of this section in ColumnBuffer + RID fCurrRowId; // row-id which will be written next + int fStatus; // Status of this section }; -} +} // namespace WriteEngine #endif /*WRITEENGINE_COLBUFSEC_H*/ - diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index c65158292..1551cd822 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -39,30 +39,26 @@ namespace { -typedef std::tr1::unordered_map < -WriteEngine::RID, WriteEngine::ColExtInfEntry, WriteEngine::uint64Hasher > -RowExtMap; +typedef std::tr1::unordered_map + RowExtMap; } namespace WriteEngine { - //------------------------------------------------------------------------------ // Add an entry for the pre-existing extent that we start loading data into at // the start of a bulk load. In this case we know the LBID, but have no min/ // max values to start with when adding this first entry to our collection. //------------------------------------------------------------------------------ // @bug 4806: Added bIsNewExtent; Set CP min/max for very first extent on a PM -void ColExtInf::addFirstEntry( RID lastInputRow, - BRM::LBID_t lbid, - bool bIsNewExtent ) +void ColExtInf::addFirstEntry(RID lastInputRow, BRM::LBID_t lbid, bool bIsNewExtent) { - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - ColExtInfEntry entry( lbid, bIsNewExtent ); - fMap[ lastInputRow ] = entry; + ColExtInfEntry entry(lbid, bIsNewExtent); + fMap[lastInputRow] = entry; } - + //------------------------------------------------------------------------------ // Add or update an entry for a new extent that we are adding. In this case // we have completed a Read buffer, and thus have min/max values, but we may @@ -71,291 +67,271 @@ void ColExtInf::addFirstEntry( RID lastInputRow, // extent. //------------------------------------------------------------------------------ template -void ColExtInf::addOrUpdateEntryTemplate( RID lastInputRow, - T minVal, T maxVal, - ColDataType colDataType, - int width ) +void ColExtInf::addOrUpdateEntryTemplate(RID lastInputRow, T minVal, T maxVal, ColDataType colDataType, + int width) { - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - RowExtMap::iterator iter = fMap.find( lastInputRow ); + RowExtMap::iterator iter = fMap.find(lastInputRow); - if (iter == fMap.end()) // Add entry + if (iter == fMap.end()) // Add entry + { + ColExtInfEntry entry(minVal, maxVal); + fMap[lastInputRow] = entry; + + fPendingExtentRows.insert(lastInputRow); + } + else // Update entry + { + // If all rows had null value for this column, then minVal will be + // MAX_INT and maxVal will be MIN_INT (see getCPInfoForBRM()). + + int128_t bigMinValInit; + utils::int128Max(bigMinValInit); + if ((iter->second.fMinVal == LLONG_MIN && width <= 8) || + (iter->second.fbigMinVal == bigMinValInit && width > 8)) // init the range { - ColExtInfEntry entry( minVal, maxVal ); - fMap[ lastInputRow ] = entry; - - fPendingExtentRows.insert( lastInputRow ); + if (width <= 8) + { + iter->second.fMinVal = minVal; + iter->second.fMaxVal = maxVal; + } + else + { + iter->second.fbigMinVal = minVal; + iter->second.fbigMaxVal = maxVal; + } } - else // Update entry + else // Update the range { - // If all rows had null value for this column, then minVal will be - // MAX_INT and maxVal will be MIN_INT (see getCPInfoForBRM()). - - int128_t bigMinValInit; - utils::int128Max(bigMinValInit); - if ((iter->second.fMinVal == LLONG_MIN && width <= 8) || - (iter->second.fbigMinVal == bigMinValInit && width > 8)) // init the range + if (isUnsigned(colDataType) || isCharType(colDataType)) + { + if (width <= 8) { - if (width <= 8) - { - iter->second.fMinVal = minVal; - iter->second.fMaxVal = maxVal; - } - else - { - iter->second.fbigMinVal = minVal; - iter->second.fbigMaxVal = maxVal; - } + if (static_cast(minVal) < static_cast(iter->second.fMinVal)) + iter->second.fMinVal = minVal; + + if (static_cast(maxVal) > static_cast(iter->second.fMaxVal)) + iter->second.fMaxVal = maxVal; } - else // Update the range + else { - if (isUnsigned(colDataType) || isCharType(colDataType)) - { - if (width <= 8) - { - if (static_cast(minVal) - < static_cast(iter->second.fMinVal)) - iter->second.fMinVal = minVal; + if (static_cast(minVal) < static_cast(iter->second.fbigMinVal)) + iter->second.fbigMinVal = minVal; - if (static_cast(maxVal) - > static_cast(iter->second.fMaxVal)) - iter->second.fMaxVal = maxVal; - } - else - { - if (static_cast(minVal) - < static_cast(iter->second.fbigMinVal)) - iter->second.fbigMinVal = minVal; - - if (static_cast(maxVal) - > static_cast(iter->second.fbigMaxVal)) - iter->second.fbigMaxVal = maxVal; - } - } - else - { - if (width <= 8) - { - if (minVal < iter->second.fMinVal) - iter->second.fMinVal = minVal; - - if (maxVal > iter->second.fMaxVal) - iter->second.fMaxVal = maxVal; - } - else - { - if (minVal < iter->second.fbigMinVal) - iter->second.fbigMinVal = minVal; - - if (maxVal > iter->second.fbigMaxVal) - iter->second.fbigMaxVal = maxVal; - } - } + if (static_cast(maxVal) > static_cast(iter->second.fbigMaxVal)) + iter->second.fbigMaxVal = maxVal; } + } + else + { + if (width <= 8) + { + if (minVal < iter->second.fMinVal) + iter->second.fMinVal = minVal; + + if (maxVal > iter->second.fMaxVal) + iter->second.fMaxVal = maxVal; + } + else + { + if (minVal < iter->second.fbigMinVal) + iter->second.fbigMinVal = minVal; + + if (maxVal > iter->second.fbigMaxVal) + iter->second.fbigMaxVal = maxVal; + } + } } + } } - + //------------------------------------------------------------------------------ // After flushing an output buffer and allocating it's extent, this function is // called to save the starting LBID back into the corresponding extent entry. //------------------------------------------------------------------------------ -int ColExtInf::updateEntryLbid( BRM::LBID_t startLbid ) +int ColExtInf::updateEntryLbid(BRM::LBID_t startLbid) { - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - // fPendingExtentRows is a Set carrying a sorted list of the last Row - // number in each extent. We should be allocating/assigning LBIDs in - // row order, so we get the "first" object in fPendingExtentRows, and - // that should be the extent corresponding to the LBID we just got. - std::set::iterator iterPendingExt = fPendingExtentRows.begin(); + // fPendingExtentRows is a Set carrying a sorted list of the last Row + // number in each extent. We should be allocating/assigning LBIDs in + // row order, so we get the "first" object in fPendingExtentRows, and + // that should be the extent corresponding to the LBID we just got. + std::set::iterator iterPendingExt = fPendingExtentRows.begin(); - if (iterPendingExt != fPendingExtentRows.end()) + if (iterPendingExt != fPendingExtentRows.end()) + { + RowExtMap::iterator iter = fMap.find(*iterPendingExt); + + if (iter != fMap.end()) { - RowExtMap::iterator iter = fMap.find( *iterPendingExt ); - - if (iter != fMap.end()) - { - iter->second.fLbid = startLbid; - } - else - { - return ERR_BULK_MISSING_EXTENT_ENTRY; - } - - fPendingExtentRows.erase( iterPendingExt ); + iter->second.fLbid = startLbid; } else { - return ERR_BULK_MISSING_EXTENT_ROW; + return ERR_BULK_MISSING_EXTENT_ENTRY; } - return NO_ERROR; + fPendingExtentRows.erase(iterPendingExt); + } + else + { + return ERR_BULK_MISSING_EXTENT_ROW; + } + + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Get updated Casual Partition (CP) information for BRM for this column at EOJ. //------------------------------------------------------------------------------ -void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) +void ColExtInf::getCPInfoForBRM(JobColumn column, BRMReporter& brmReporter) { - bool bIsChar = ((column.weType == WriteEngine::WR_CHAR) && - (column.colType != COL_TYPE_DICT)); + bool bIsChar = ((column.weType == WriteEngine::WR_CHAR) && (column.colType != COL_TYPE_DICT)); - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - RowExtMap::const_iterator iter = fMap.begin(); + RowExtMap::const_iterator iter = fMap.begin(); - while (iter != fMap.end()) + while (iter != fMap.end()) + { + // If/when we support NULL values, we could have an extent with initial + // value of min=MAX_BIGINT and max=MIN_BIGINT (see + // BulkLoadBuffer::parseCol()). If this occurs, (min>max), we still + // send min/max to BRM so that the isValid flag can be set to CP_VALID + // if applicable (indicating an extent with no non-NULL values). + int64_t minVal = iter->second.fMinVal; + int64_t maxVal = iter->second.fMaxVal; + int128_t bigMinVal = iter->second.fbigMinVal; + int128_t bigMaxVal = iter->second.fbigMaxVal; + + if (bIsChar) { - // If/when we support NULL values, we could have an extent with initial - // value of min=MAX_BIGINT and max=MIN_BIGINT (see - // BulkLoadBuffer::parseCol()). If this occurs, (min>max), we still - // send min/max to BRM so that the isValid flag can be set to CP_VALID - // if applicable (indicating an extent with no non-NULL values). - int64_t minVal = iter->second.fMinVal; - int64_t maxVal = iter->second.fMaxVal; - int128_t bigMinVal = iter->second.fbigMinVal; - int128_t bigMaxVal = iter->second.fbigMaxVal; - - if ( bIsChar ) - { - // If we have added 1 or more rows, then we should have a valid - // range in our RowExtMap object, in which case... - // We swap/restore byte order before sending min/max string to BRM; - // else we leave fMinVal & fMaxVal set to LLONG_MIN and send as-is, - // to let BRM know we added no rows. - if ((iter->second.fMinVal != iter->second.fMaxVal) || - (iter->second.fMinVal != LLONG_MIN)) - { - minVal = static_cast( uint64ToStr( - static_cast(iter->second.fMinVal) ) ); - maxVal = static_cast( uint64ToStr( - static_cast(iter->second.fMaxVal) ) ); - } - } - - // Log for now; may control with debug flag later - //if (fLog->isDebug( DEBUG_1 )) - // TODO MCOL-641 Add support here. - { - std::ostringstream oss; - oss << "Saving CP update for OID-" << fColOid << - "; lbid-" << iter->second.fLbid << - "; type-" << bIsChar << - "; isNew-" << iter->second.fNewExtent; - - if (bIsChar) - { - char minValStr[sizeof(int64_t) + 1]; - char maxValStr[sizeof(int64_t) + 1]; - memcpy(minValStr, &minVal, sizeof(int64_t)); - memcpy(maxValStr, &maxVal, sizeof(int64_t)); - minValStr[sizeof(int64_t)] = '\0'; - maxValStr[sizeof(int64_t)] = '\0'; - oss << "; minVal: " << minVal << "; (" << minValStr << ")" - << "; maxVal: " << maxVal << "; (" << maxValStr << ")"; - } - else if (isUnsigned(column.dataType)) - { - oss << "; min: " << static_cast(minVal) << - "; max: " << static_cast(maxVal); - } - else - { - oss << "; min: " << minVal << - "; max: " << maxVal; - } - - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - BRM::CPInfoMerge cpInfoMerge; - cpInfoMerge.startLbid = iter->second.fLbid; - if (column.width <= 8) - { - cpInfoMerge.max = maxVal; - cpInfoMerge.min = minVal; - } - else - { - cpInfoMerge.bigMax = bigMaxVal; - cpInfoMerge.bigMin = bigMinVal; - } - cpInfoMerge.seqNum = -1; // Not used by mergeExtentsMaxMin - cpInfoMerge.type = column.dataType; - cpInfoMerge.newExtent = iter->second.fNewExtent; - cpInfoMerge.colWidth = column.width; - brmReporter.addToCPInfo( cpInfoMerge ); - - ++iter; + // If we have added 1 or more rows, then we should have a valid + // range in our RowExtMap object, in which case... + // We swap/restore byte order before sending min/max string to BRM; + // else we leave fMinVal & fMaxVal set to LLONG_MIN and send as-is, + // to let BRM know we added no rows. + if ((iter->second.fMinVal != iter->second.fMaxVal) || (iter->second.fMinVal != LLONG_MIN)) + { + minVal = static_cast(uint64ToStr(static_cast(iter->second.fMinVal))); + maxVal = static_cast(uint64ToStr(static_cast(iter->second.fMaxVal))); + } } - fMap.clear(); // don't need map anymore, so release memory + // Log for now; may control with debug flag later + // if (fLog->isDebug( DEBUG_1 )) + // TODO MCOL-641 Add support here. + { + std::ostringstream oss; + oss << "Saving CP update for OID-" << fColOid << "; lbid-" << iter->second.fLbid << "; type-" + << bIsChar << "; isNew-" << iter->second.fNewExtent; + + if (bIsChar) + { + char minValStr[sizeof(int64_t) + 1]; + char maxValStr[sizeof(int64_t) + 1]; + memcpy(minValStr, &minVal, sizeof(int64_t)); + memcpy(maxValStr, &maxVal, sizeof(int64_t)); + minValStr[sizeof(int64_t)] = '\0'; + maxValStr[sizeof(int64_t)] = '\0'; + oss << "; minVal: " << minVal << "; (" << minValStr << ")" + << "; maxVal: " << maxVal << "; (" << maxValStr << ")"; + } + else if (isUnsigned(column.dataType)) + { + oss << "; min: " << static_cast(minVal) << "; max: " << static_cast(maxVal); + } + else + { + oss << "; min: " << minVal << "; max: " << maxVal; + } + + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + BRM::CPInfoMerge cpInfoMerge; + cpInfoMerge.startLbid = iter->second.fLbid; + if (column.width <= 8) + { + cpInfoMerge.max = maxVal; + cpInfoMerge.min = minVal; + } + else + { + cpInfoMerge.bigMax = bigMaxVal; + cpInfoMerge.bigMin = bigMinVal; + } + cpInfoMerge.seqNum = -1; // Not used by mergeExtentsMaxMin + cpInfoMerge.type = column.dataType; + cpInfoMerge.newExtent = iter->second.fNewExtent; + cpInfoMerge.colWidth = column.width; + brmReporter.addToCPInfo(cpInfoMerge); + + ++iter; + } + + fMap.clear(); // don't need map anymore, so release memory } - + //------------------------------------------------------------------------------ // Print contents of this object to the log file. //------------------------------------------------------------------------------ -void ColExtInf::print( const JobColumn& column ) +void ColExtInf::print(const JobColumn& column) { - boost::mutex::scoped_lock lock(fMapMutex); - bool bIsChar = ((column.weType == WriteEngine::WR_CHAR) && - (column.colType != COL_TYPE_DICT)); - std::ostringstream oss; - oss << "ColExtInf Map for OID: " << fColOid; - RowExtMap::const_iterator iter = fMap.begin(); + boost::mutex::scoped_lock lock(fMapMutex); + bool bIsChar = ((column.weType == WriteEngine::WR_CHAR) && (column.colType != COL_TYPE_DICT)); + std::ostringstream oss; + oss << "ColExtInf Map for OID: " << fColOid; + RowExtMap::const_iterator iter = fMap.begin(); - while (iter != fMap.end()) + while (iter != fMap.end()) + { + oss << std::endl << " RowKey-" << iter->first << "; lbid-" << iter->second.fLbid; + + if (iter->second.fLbid == (BRM::LBID_t)INVALID_LBID) + oss << " (unset)"; + + oss << "; newExt-" << iter->second.fNewExtent; + + if (bIsChar) { - oss << std::endl << - " RowKey-" << iter->first << - "; lbid-" << iter->second.fLbid; - - if (iter->second.fLbid == (BRM::LBID_t)INVALID_LBID) - oss << " (unset)"; - - oss << "; newExt-" << iter->second.fNewExtent; - - if ( bIsChar ) - { - // Swap/restore byte order before printing character string - int64_t minVal = static_cast( uint64ToStr( - static_cast(iter->second.fMinVal) ) ); - int64_t maxVal = static_cast( uint64ToStr( - static_cast(iter->second.fMaxVal) ) ); - char minValStr[sizeof(int64_t) + 1]; - char maxValStr[sizeof(int64_t) + 1]; - memcpy(minValStr, &minVal, sizeof(int64_t)); - memcpy(maxValStr, &maxVal, sizeof(int64_t)); - minValStr[sizeof(int64_t)] = '\0'; - maxValStr[sizeof(int64_t)] = '\0'; - oss << "; minVal: " << minVal << "; (" << minValStr << ")" - << "; maxVal: " << maxVal << "; (" << maxValStr << ")"; - } - else if (isUnsigned(column.dataType)) - { - oss << "; min: " << static_cast(iter->second.fMinVal) << - "; max: " << static_cast(iter->second.fMaxVal); - } - else - { - oss << "; min: " << iter->second.fMinVal << - "; max: " << iter->second.fMaxVal; - } - - ++iter; + // Swap/restore byte order before printing character string + int64_t minVal = static_cast(uint64ToStr(static_cast(iter->second.fMinVal))); + int64_t maxVal = static_cast(uint64ToStr(static_cast(iter->second.fMaxVal))); + char minValStr[sizeof(int64_t) + 1]; + char maxValStr[sizeof(int64_t) + 1]; + memcpy(minValStr, &minVal, sizeof(int64_t)); + memcpy(maxValStr, &maxVal, sizeof(int64_t)); + minValStr[sizeof(int64_t)] = '\0'; + maxValStr[sizeof(int64_t)] = '\0'; + oss << "; minVal: " << minVal << "; (" << minValStr << ")" + << "; maxVal: " << maxVal << "; (" << maxValStr << ")"; + } + else if (isUnsigned(column.dataType)) + { + oss << "; min: " << static_cast(iter->second.fMinVal) + << "; max: " << static_cast(iter->second.fMaxVal); + } + else + { + oss << "; min: " << iter->second.fMinVal << "; max: " << iter->second.fMaxVal; } - oss << std::endl << " ColExtInf Rows/Extents waiting LBIDs: "; - std::set::const_iterator iterPendingExt = fPendingExtentRows.begin(); + ++iter; + } - while (iterPendingExt != fPendingExtentRows.end()) - { - oss << *iterPendingExt << ", "; - ++iterPendingExt; - } + oss << std::endl << " ColExtInf Rows/Extents waiting LBIDs: "; + std::set::const_iterator iterPendingExt = fPendingExtentRows.begin(); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + while (iterPendingExt != fPendingExtentRows.end()) + { + oss << *iterPendingExt << ", "; + ++iterPendingExt; + } + + fLog->logMsg(oss.str(), MSGLVL_INFO2); } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index 60930d8ed..a721e8757 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -58,70 +58,66 @@ typedef execplan::CalpontSystemCatalog::ColDataType ColDataType; //------------------------------------------------------------------------------ class ColExtInfEntry { -public: - // Default constructor - ColExtInfEntry() : fLbid(INVALID_LBID), - fMinVal(LLONG_MIN), - fMaxVal(LLONG_MIN), - fNewExtent(true) - { - utils::int128Min(fbigMaxVal); - utils::int128Max(fbigMinVal); - } + public: + // Default constructor + ColExtInfEntry() : fLbid(INVALID_LBID), fMinVal(LLONG_MIN), fMaxVal(LLONG_MIN), fNewExtent(true) + { + utils::int128Min(fbigMaxVal); + utils::int128Max(fbigMinVal); + } - // Used to create entry for an existing extent we are going to add data to. - ColExtInfEntry(BRM::LBID_t lbid, bool bIsNewExtent) : - fLbid(lbid), - fMinVal(LLONG_MIN), - fMaxVal(LLONG_MIN), - fNewExtent(bIsNewExtent) - { - utils::int128Min(fbigMaxVal); - utils::int128Max(fbigMinVal); - } + // Used to create entry for an existing extent we are going to add data to. + ColExtInfEntry(BRM::LBID_t lbid, bool bIsNewExtent) + : fLbid(lbid), fMinVal(LLONG_MIN), fMaxVal(LLONG_MIN), fNewExtent(bIsNewExtent) + { + utils::int128Min(fbigMaxVal); + utils::int128Max(fbigMinVal); + } - // Used to create entry for a new extent, with LBID not yet allocated - ColExtInfEntry(int64_t minVal, int64_t maxVal) : - fLbid(INVALID_LBID), - fMinVal(minVal), - fMaxVal(maxVal), - fNewExtent(true) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(int64_t minVal, int64_t maxVal) + : fLbid(INVALID_LBID), fMinVal(minVal), fMaxVal(maxVal), fNewExtent(true) + { + } - // Used to create entry for a new extent, with LBID not yet allocated - ColExtInfEntry(int128_t bigMinVal, int128_t bigMaxVal) : - fLbid(INVALID_LBID), - fNewExtent(true), - fbigMinVal(bigMinVal), - fbigMaxVal(bigMaxVal) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(int128_t bigMinVal, int128_t bigMaxVal) + : fLbid(INVALID_LBID), fNewExtent(true), fbigMinVal(bigMinVal), fbigMaxVal(bigMaxVal) + { + } - // Used to create entry for a new extent, with LBID not yet allocated - ColExtInfEntry(uint64_t minVal, uint64_t maxVal) : - fLbid(INVALID_LBID), - fMinVal(static_cast(minVal)), - fMaxVal(static_cast(maxVal)), - fNewExtent(true) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(uint64_t minVal, uint64_t maxVal) + : fLbid(INVALID_LBID) + , fMinVal(static_cast(minVal)) + , fMaxVal(static_cast(maxVal)) + , fNewExtent(true) + { + } - // Used to create entry for a new extent, with LBID not yet allocated - ColExtInfEntry(uint128_t bigMinVal, uint128_t bigMaxVal) : - fLbid(INVALID_LBID), - fNewExtent(true), - fbigMinVal(static_cast(bigMinVal)), - fbigMaxVal(static_cast(bigMaxVal)) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(uint128_t bigMinVal, uint128_t bigMaxVal) + : fLbid(INVALID_LBID) + , fNewExtent(true) + , fbigMinVal(static_cast(bigMinVal)) + , fbigMaxVal(static_cast(bigMaxVal)) + { + } - BRM::LBID_t fLbid; // LBID for an extent; should be the starting LBID - int64_t fMinVal; // minimum value for extent associated with LBID - int64_t fMaxVal; // maximum value for extent associated with LBID - bool fNewExtent;// is this a new extent - union - { - int128_t fbigMinVal; - int64_t fMinVal_; - }; - union - { - int128_t fbigMaxVal; - int64_t fMaxVal_; - }; + BRM::LBID_t fLbid; // LBID for an extent; should be the starting LBID + int64_t fMinVal; // minimum value for extent associated with LBID + int64_t fMaxVal; // maximum value for extent associated with LBID + bool fNewExtent; // is this a new extent + union + { + int128_t fbigMinVal; + int64_t fMinVal_; + }; + union + { + int128_t fbigMaxVal; + int64_t fMaxVal_; + }; }; //------------------------------------------------------------------------------ @@ -131,10 +127,10 @@ public: //------------------------------------------------------------------------------ struct uint64Hasher : public std::unary_function { - std::size_t operator()(RID val) const - { - return static_cast(val); - } + std::size_t operator()(RID val) const + { + return static_cast(val); + } }; //------------------------------------------------------------------------------ @@ -144,33 +140,38 @@ struct uint64Hasher : public std::unary_function //------------------------------------------------------------------------------ class ColExtInfBase { -public: - ColExtInfBase( ) { } - virtual ~ColExtInfBase( ) { } + public: + ColExtInfBase() + { + } + virtual ~ColExtInfBase() + { + } - virtual void addFirstEntry ( RID lastInputRow, - BRM::LBID_t lbid, - bool bIsNewExtent) { } + virtual void addFirstEntry(RID lastInputRow, BRM::LBID_t lbid, bool bIsNewExtent) + { + } - virtual void addOrUpdateEntry( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType, - int width ) { } + virtual void addOrUpdateEntry(RID lastInputRow, int64_t minVal, int64_t maxVal, ColDataType colDataType, + int width) + { + } - virtual void addOrUpdateEntry( RID lastInputRow, - int128_t minVal, - int128_t maxVal, - ColDataType colDataType, - int width ) { } + virtual void addOrUpdateEntry(RID lastInputRow, int128_t minVal, int128_t maxVal, ColDataType colDataType, + int width) + { + } - virtual void getCPInfoForBRM ( JobColumn column, - BRMReporter& brmReporter) { } - virtual void print( const JobColumn& column ) { } - virtual int updateEntryLbid( BRM::LBID_t startLbid ) - { - return NO_ERROR; - } + virtual void getCPInfoForBRM(JobColumn column, BRMReporter& brmReporter) + { + } + virtual void print(const JobColumn& column) + { + } + virtual int updateEntryLbid(BRM::LBID_t startLbid) + { + return NO_ERROR; + } }; //------------------------------------------------------------------------------ @@ -189,88 +190,75 @@ public: //------------------------------------------------------------------------------ class ColExtInf : public ColExtInfBase { -public: + public: + /** @brief Constructor + * @param logger Log object using for debug logging. + */ + ColExtInf(OID oid, Log* logger) : fColOid(oid), fLog(logger) + { + } + virtual ~ColExtInf() + { + } - /** @brief Constructor - * @param logger Log object using for debug logging. - */ - ColExtInf( OID oid, Log* logger ) : fColOid(oid), fLog(logger) { } - virtual ~ColExtInf( ) { } + /** @brief Add an entry for first extent, for the specified Row and LBID. + * @param lastInputRow Last input Row for old extent we are adding data to + * @param lbid LBID of the relevant extent. + * @param bIsNewExtent Treat as new or existing extent when CP min/max is + * sent to BRM + */ + virtual void addFirstEntry(RID lastInputRow, BRM::LBID_t lbid, bool bIsNewExtent); - /** @brief Add an entry for first extent, for the specified Row and LBID. - * @param lastInputRow Last input Row for old extent we are adding data to - * @param lbid LBID of the relevant extent. - * @param bIsNewExtent Treat as new or existing extent when CP min/max is - * sent to BRM - */ - virtual void addFirstEntry( RID lastInputRow, - BRM::LBID_t lbid, - bool bIsNewExtent ); + /** @brief Add or update an entry for the specified Row and its min/max val. + * If new extent, LBID will be added later when extent is allocated. + * @param lastInputRow Last input Row for a new extent being loaded. + * @param minVal Minimum value for the latest buffer read + * @param maxVal Maximum value for the latest buffer read + */ + template + void addOrUpdateEntryTemplate(RID lastInputRow, T minVal, T maxVal, ColDataType colDataType, int width); - /** @brief Add or update an entry for the specified Row and its min/max val. - * If new extent, LBID will be added later when extent is allocated. - * @param lastInputRow Last input Row for a new extent being loaded. - * @param minVal Minimum value for the latest buffer read - * @param maxVal Maximum value for the latest buffer read - */ - template - void addOrUpdateEntryTemplate( RID lastInputRow, - T minVal, T maxVal, - ColDataType colDataType, - int width ); + virtual void addOrUpdateEntry(RID lastInputRow, int64_t minVal, int64_t maxVal, ColDataType colDataType, + int width) + { + addOrUpdateEntryTemplate(lastInputRow, minVal, maxVal, colDataType, width); + } - virtual void addOrUpdateEntry( RID lastInputRow, - int64_t minVal, int64_t maxVal, - ColDataType colDataType, - int width ) - { - addOrUpdateEntryTemplate( lastInputRow, - minVal, maxVal, - colDataType, - width ); - } + virtual void addOrUpdateEntry(RID lastInputRow, int128_t minVal, int128_t maxVal, ColDataType colDataType, + int width) + { + addOrUpdateEntryTemplate(lastInputRow, minVal, maxVal, colDataType, width); + } - virtual void addOrUpdateEntry( RID lastInputRow, - int128_t minVal, int128_t maxVal, - ColDataType colDataType, - int width ) - { - addOrUpdateEntryTemplate( lastInputRow, - minVal, maxVal, - colDataType, - width ); - } + /** @brief Send updated Casual Partition (CP) info to BRM. + */ + virtual void getCPInfoForBRM(JobColumn column, BRMReporter& brmReporter); - /** @brief Send updated Casual Partition (CP) info to BRM. - */ - virtual void getCPInfoForBRM ( JobColumn column, - BRMReporter& brmReporter ); + /** @brief Debug print function. + */ + virtual void print(const JobColumn& column); - /** @brief Debug print function. - */ - virtual void print( const JobColumn& column ); + /** @brief Add extent's LBID to the oldest entry that is awaiting an LBID + * @param startLbid Starting LBID for a pending extent. + * @return NO_ERROR upon success; else error if extent entry not found + */ + virtual int updateEntryLbid(BRM::LBID_t startLbid); - /** @brief Add extent's LBID to the oldest entry that is awaiting an LBID - * @param startLbid Starting LBID for a pending extent. - * @return NO_ERROR upon success; else error if extent entry not found - */ - virtual int updateEntryLbid( BRM::LBID_t startLbid ); + private: + OID fColOid; // Column OID for the relevant extents + Log* fLog; // Log used for debug logging + boost::mutex fMapMutex; // protects unordered map access + std::set fPendingExtentRows; // list of lastInputRow entries that + // are awaiting an LBID assignment. -private: - OID fColOid; // Column OID for the relevant extents - Log* fLog; // Log used for debug logging - boost::mutex fMapMutex; // protects unordered map access - std::set fPendingExtentRows; // list of lastInputRow entries that - // are awaiting an LBID assignment. + // unordered map where we collect the min/max values per extent + std::tr1::unordered_map fMap; - // unordered map where we collect the min/max values per extent - std::tr1::unordered_map fMap; - - // disable copy constructor and assignment operator - ColExtInf(const ColExtInf&); - ColExtInf& operator=(const ColExtInf&); + // disable copy constructor and assignment operator + ColExtInf(const ColExtInf&); + ColExtInf& operator=(const ColExtInf&); }; -} //end of namespace +} // namespace WriteEngine -#endif // WE_COLEXTINF_H_ +#endif // WE_COLEXTINF_H_ diff --git a/writeengine/bulk/we_colopbulk.cpp b/writeengine/bulk/we_colopbulk.cpp index d8bb11b17..bb8ba0174 100644 --- a/writeengine/bulk/we_colopbulk.cpp +++ b/writeengine/bulk/we_colopbulk.cpp @@ -34,13 +34,12 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // Default ColumnOpBulk constructor //------------------------------------------------------------------------------ ColumnOpBulk::ColumnOpBulk() { - m_compressionType = 0; + m_compressionType = 0; } //------------------------------------------------------------------------------ @@ -48,7 +47,7 @@ ColumnOpBulk::ColumnOpBulk() //------------------------------------------------------------------------------ ColumnOpBulk::ColumnOpBulk(Log* logger, int compressionType) : ColumnOp(logger) { - m_compressionType = compressionType; + m_compressionType = compressionType; } //------------------------------------------------------------------------------ @@ -62,61 +61,52 @@ ColumnOpBulk::~ColumnOpBulk() // Open specified column file //------------------------------------------------------------------------------ // @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* ColumnOpBulk::openFile(const WriteEngine::Column& column, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - bool useTmpSuffix, - const char* mode, - int ioBuffSize) const +IDBDataFile* ColumnOpBulk::openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition, + uint16_t segment, std::string& segFile, bool useTmpSuffix, + const char* mode, int ioBuffSize) const { - return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, - segFile, mode, column.colWidth, useTmpSuffix); + return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, mode, column.colWidth, + useTmpSuffix); } //------------------------------------------------------------------------------ // Stub for abbreviatedExtent //------------------------------------------------------------------------------ -bool ColumnOpBulk::abbreviatedExtent(IDBDataFile*, int) const +bool ColumnOpBulk::abbreviatedExtent(IDBDataFile*, int) const { - throw std::logic_error( - "Unauthorized use of ColumnOpBulk::abbreviatedExtent"); + throw std::logic_error("Unauthorized use of ColumnOpBulk::abbreviatedExtent"); - return false; + return false; } //------------------------------------------------------------------------------ // Stub for blocksInFile //------------------------------------------------------------------------------ -int ColumnOpBulk::blocksInFile(IDBDataFile*) const +int ColumnOpBulk::blocksInFile(IDBDataFile*) const { - throw std::logic_error( - "Unauthorized use of ColumnOpBulk::blocksInFile"); + throw std::logic_error("Unauthorized use of ColumnOpBulk::blocksInFile"); - return 0; + return 0; } //------------------------------------------------------------------------------ // Stub for readBlock //------------------------------------------------------------------------------ -int ColumnOpBulk::readBlock(IDBDataFile*, unsigned char*, const uint64_t) +int ColumnOpBulk::readBlock(IDBDataFile*, unsigned char*, const uint64_t) { - throw std::logic_error( - "Unauthorized use of ColumnOpBulk::readBlock"); + throw std::logic_error("Unauthorized use of ColumnOpBulk::readBlock"); - return 0; + return 0; } //------------------------------------------------------------------------------ // Stub for writeBlock //------------------------------------------------------------------------------ -int ColumnOpBulk::saveBlock(IDBDataFile*, const unsigned char*, const uint64_t) +int ColumnOpBulk::saveBlock(IDBDataFile*, const unsigned char*, const uint64_t) { - throw std::logic_error( - "Unauthorized use of ColumnOpBulk::saveBlock"); + throw std::logic_error("Unauthorized use of ColumnOpBulk::saveBlock"); - return 0; + return 0; } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/bulk/we_colopbulk.h b/writeengine/bulk/we_colopbulk.h index 53d7b770f..e678aefbf 100644 --- a/writeengine/bulk/we_colopbulk.h +++ b/writeengine/bulk/we_colopbulk.h @@ -42,21 +42,20 @@ class Log; */ class ColumnOpBulk : public ColumnOp { -public: - ColumnOpBulk(); - ColumnOpBulk(Log* logger, int compressionType); - virtual ~ColumnOpBulk(); + public: + ColumnOpBulk(); + ColumnOpBulk(Log* logger, int compressionType); + virtual ~ColumnOpBulk(); - virtual bool abbreviatedExtent(IDBDataFile*, int) const; - virtual int blocksInFile(IDBDataFile*) const; - virtual IDBDataFile* openFile(const WriteEngine::Column& column, - uint16_t dbRoot, uint32_t partition, uint16_t segment, - std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const; - virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t); - virtual int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t); + virtual bool abbreviatedExtent(IDBDataFile*, int) const; + virtual int blocksInFile(IDBDataFile*) const; + virtual IDBDataFile* openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition, + uint16_t segment, std::string& segFile, bool useTmpSuffix, + const char* mode = "r+b", int ioBuffSize = DEFAULT_BUFSIZ) const; + virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t); + virtual int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t); }; -} //end of namespace +} // namespace WriteEngine -#endif // _WE_COLOP_BULK_H_ +#endif // _WE_COLOP_BULK_H_ diff --git a/writeengine/bulk/we_columnautoinc.cpp b/writeengine/bulk/we_columnautoinc.cpp index b788978cc..1d2c0785c 100644 --- a/writeengine/bulk/we_columnautoinc.cpp +++ b/writeengine/bulk/we_columnautoinc.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_columnautoinc.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_columnautoinc.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + */ /** @file * Implementation of the ColumnAutoInc, ColumnAutoIncJob, and @@ -39,14 +39,10 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // ColumnAutoInc constructor. //------------------------------------------------------------------------------ -ColumnAutoInc::ColumnAutoInc( Log* logger ) : - fLog(logger), - fAutoIncLastValue(0), - fColumnOID(0) +ColumnAutoInc::ColumnAutoInc(Log* logger) : fLog(logger), fAutoIncLastValue(0), fColumnOID(0) { } @@ -54,7 +50,7 @@ ColumnAutoInc::ColumnAutoInc( Log* logger ) : // ColumnAutoInc destructor. //------------------------------------------------------------------------------ /* virtual */ -ColumnAutoInc::~ColumnAutoInc( ) +ColumnAutoInc::~ColumnAutoInc() { } @@ -63,57 +59,53 @@ ColumnAutoInc::~ColumnAutoInc( ) // fullTableName - Schema and table name separated by a period. // colInfo - ColumnInfo associated with auto-increment column. //------------------------------------------------------------------------------ -int ColumnAutoInc::init( const std::string& fullTableName, - ColumnInfo* colInfo ) +int ColumnAutoInc::init(const std::string& fullTableName, ColumnInfo* colInfo) { - fMaxIntSat = colInfo->column.fMaxIntSat; - fTableName = fullTableName; - fColumnName = colInfo->column.colName; - fColumnOID = colInfo->column.mapOid; + fMaxIntSat = colInfo->column.fMaxIntSat; + fTableName = fullTableName; + fColumnName = colInfo->column.colName; + fColumnOID = colInfo->column.mapOid; - std::string::size_type periodIdx = fTableName.find('.'); + std::string::size_type periodIdx = fTableName.find('.'); - if (periodIdx == std::string::npos) - { - std::ostringstream oss; - oss << "Error parsing full table name to get auto-increment value for " - << fTableName; - fLog->logMsg( oss.str(), ERR_AUTOINC_TABLE_NAME, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); - return ERR_AUTOINC_TABLE_NAME; - } + if (periodIdx == std::string::npos) + { + std::ostringstream oss; + oss << "Error parsing full table name to get auto-increment value for " << fTableName; + fLog->logMsg(oss.str(), ERR_AUTOINC_TABLE_NAME, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); + return ERR_AUTOINC_TABLE_NAME; + } - uint64_t nextAuto = 0; - int rc = getNextValueFromSysCat( nextAuto ); + uint64_t nextAuto = 0; + int rc = getNextValueFromSysCat(nextAuto); - if (rc != NO_ERROR) - { - return rc; - } + if (rc != NO_ERROR) + { + return rc; + } - std::string errMsg; - rc = BRMWrapper::getInstance()->startAutoIncrementSequence( - fColumnOID, nextAuto, colInfo->column.width, colInfo->column.dataType, errMsg ); + std::string errMsg; + rc = BRMWrapper::getInstance()->startAutoIncrementSequence(fColumnOID, nextAuto, colInfo->column.width, + colInfo->column.dataType, errMsg); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Unable to initialize auto-increment sequence for " << - fTableName << "; " << errMsg; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); - return rc; - } + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Unable to initialize auto-increment sequence for " << fTableName << "; " << errMsg; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); + return rc; + } - std::ostringstream oss2; - oss2 << "Initializing next auto increment for table-" << fTableName << - ", column-" << fColumnName << - "; autoincrement " << nextAuto; - fLog->logMsg( oss2.str(), MSGLVL_INFO2 ); + std::ostringstream oss2; + oss2 << "Initializing next auto increment for table-" << fTableName << ", column-" << fColumnName + << "; autoincrement " << nextAuto; + fLog->logMsg(oss2.str(), MSGLVL_INFO2); - initNextAutoInc( nextAuto ); + initNextAutoInc(nextAuto); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -124,15 +116,15 @@ int ColumnAutoInc::init( const std::string& fullTableName, // for completeness. Using the mutex should not affect performance, since this // function is only called once per table. //------------------------------------------------------------------------------ -void ColumnAutoInc::initNextAutoInc( uint64_t nextValue ) +void ColumnAutoInc::initNextAutoInc(uint64_t nextValue) { - boost::mutex::scoped_lock lock(fAutoIncMutex); + boost::mutex::scoped_lock lock(fAutoIncMutex); - // nextValue is unusable if < 1; probably means we already reached max value - if (nextValue < 1) - fAutoIncLastValue = fMaxIntSat; - else - fAutoIncLastValue = nextValue - 1; + // nextValue is unusable if < 1; probably means we already reached max value + if (nextValue < 1) + fAutoIncLastValue = fMaxIntSat; + else + fAutoIncLastValue = nextValue - 1; } //------------------------------------------------------------------------------ @@ -140,95 +132,90 @@ void ColumnAutoInc::initNextAutoInc( uint64_t nextValue ) // necessary to save or commit changes to the auto-increment nextValue, are // applied here. //------------------------------------------------------------------------------ -int ColumnAutoInc::finish( ) +int ColumnAutoInc::finish() { - int rc = NO_ERROR; + int rc = NO_ERROR; - // We intentionally use a separate DBRM instance in this function. We don't - // use the BRMWrapper singleton. We do this because the BRM call that is - // made to issue a lock is a synchronous call that will block till a lock - // is acquired. Better to do this in a separate BRM instance, rather than - // having this call block any other thread using BRM. - BRM::DBRM dbrm; + // We intentionally use a separate DBRM instance in this function. We don't + // use the BRMWrapper singleton. We do this because the BRM call that is + // made to issue a lock is a synchronous call that will block till a lock + // is acquired. Better to do this in a separate BRM instance, rather than + // having this call block any other thread using BRM. + BRM::DBRM dbrm; - // We grab AI lock in order to access/synchronize DBRM and the system - // catalog as a single operation, to avoid race condition between apps. - try - { - dbrm.getAILock( fColumnOID ); - } - catch (std::exception& ex) + // We grab AI lock in order to access/synchronize DBRM and the system + // catalog as a single operation, to avoid race condition between apps. + try + { + dbrm.getAILock(fColumnOID); + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error locking auto-increment nextValue lock for table " << fTableName << "; column " + << fColumnName << "; " << ex.what(); + fLog->logMsg(oss.str(), ERR_AUTOINC_GET_LOCK, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); + return ERR_AUTOINC_GET_LOCK; + } + + uint64_t sysCatNextAuto = 0; + rc = getNextValueFromSysCat(sysCatNextAuto); + + if (rc == NO_ERROR) + { + // Update system catalog if my latest AI nextValue is > the current + // syscat AI nextValue. max(uint64_t) denotes an AI column that has maxed out. + uint64_t myNextValue = getNextAutoIncToSave(); + + if ((sysCatNextAuto != AUTOINCR_SATURATED) && // do not update if syscat already at max + ((myNextValue > sysCatNextAuto) || (myNextValue == AUTOINCR_SATURATED))) { + std::ostringstream oss2; + oss2 << "Updating next auto increment for table-" << fTableName << ", column-" << fColumnName + << "; autoincrement " << myNextValue; + fLog->logMsg(oss2.str(), MSGLVL_INFO2); + + rc = BulkLoad::updateNextValue(fColumnOID, myNextValue); + + if (rc != NO_ERROR) + { + WErrorCodes ec; std::ostringstream oss; - oss << "Error locking auto-increment nextValue lock for table " << - fTableName << "; column " << fColumnName << "; " << ex.what(); - fLog->logMsg( oss.str(), ERR_AUTOINC_GET_LOCK, MSGLVL_ERROR ); + oss << "Error updating auto-increment nextValue for table " << fTableName << "; column " + << fColumnName << "; rc=" << rc << "; " << ec.errorString(ERR_AUTOINC_UPDATE); + fLog->logMsg(oss.str(), ERR_AUTOINC_UPDATE, MSGLVL_ERROR); BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); - return ERR_AUTOINC_GET_LOCK; + // Don't exit this function yet. We set return code and fall + // through to bottom of the function to release the AI lock. + rc = ERR_AUTOINC_UPDATE; + } } - - uint64_t sysCatNextAuto = 0; - rc = getNextValueFromSysCat( sysCatNextAuto ); - - if (rc == NO_ERROR) + else { - // Update system catalog if my latest AI nextValue is > the current - // syscat AI nextValue. max(uint64_t) denotes an AI column that has maxed out. - uint64_t myNextValue = getNextAutoIncToSave(); - - if ( (sysCatNextAuto != AUTOINCR_SATURATED) && // do not update if syscat already at max - ((myNextValue > sysCatNextAuto) || - (myNextValue == AUTOINCR_SATURATED)) ) - { - std::ostringstream oss2; - oss2 << "Updating next auto increment for table-" << fTableName << - ", column-" << fColumnName << - "; autoincrement " << myNextValue; - fLog->logMsg( oss2.str(), MSGLVL_INFO2 ); - - rc = BulkLoad::updateNextValue( fColumnOID, myNextValue ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error updating auto-increment nextValue for table " << - fTableName << "; column " << fColumnName << "; rc=" << rc << - "; " << ec.errorString(ERR_AUTOINC_UPDATE); - fLog->logMsg( oss.str(), ERR_AUTOINC_UPDATE, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); - // Don't exit this function yet. We set return code and fall - // through to bottom of the function to release the AI lock. - rc = ERR_AUTOINC_UPDATE; - } - } - else - { - std::ostringstream oss2; - oss2 << "Skip updating next auto increment for table-" << fTableName << - ", column-" << fColumnName << - "; autoincrement " << myNextValue << - "; syscat AI already at " << sysCatNextAuto; - fLog->logMsg( oss2.str(), MSGLVL_INFO2 ); - } - } // end of rc==NO_ERROR from getNextValueFromSysCat() - - try - { - dbrm.releaseAILock( fColumnOID ); - } - catch (std::exception& ex) - { - // If we have trouble releasing AI lock, we log it, but we don't - // consider it fatal to the job; so we don't return bad return code. - std::ostringstream oss; - oss << "Error releasing auto-increment nextValue lock for table " << - fTableName << "; column " << fColumnName << "; " << ex.what(); - fLog->logMsg( oss.str(), ERR_AUTOINC_REL_LOCK, MSGLVL_WARNING ); - //return ERR_AUTOINC_REL_LOCK; + std::ostringstream oss2; + oss2 << "Skip updating next auto increment for table-" << fTableName << ", column-" << fColumnName + << "; autoincrement " << myNextValue << "; syscat AI already at " << sysCatNextAuto; + fLog->logMsg(oss2.str(), MSGLVL_INFO2); } + } // end of rc==NO_ERROR from getNextValueFromSysCat() - return rc; + try + { + dbrm.releaseAILock(fColumnOID); + } + catch (std::exception& ex) + { + // If we have trouble releasing AI lock, we log it, but we don't + // consider it fatal to the job; so we don't return bad return code. + std::ostringstream oss; + oss << "Error releasing auto-increment nextValue lock for table " << fTableName << "; column " + << fColumnName << "; " << ex.what(); + fLog->logMsg(oss.str(), ERR_AUTOINC_REL_LOCK, MSGLVL_WARNING); + // return ERR_AUTOINC_REL_LOCK; + } + + return rc; } //------------------------------------------------------------------------------ @@ -236,84 +223,79 @@ int ColumnAutoInc::finish( ) // value tracked by this ColumnInfo object, that can/should be saved back // into the system catalog at the end of the job. //------------------------------------------------------------------------------ -uint64_t ColumnAutoInc::getNextAutoIncToSave( ) +uint64_t ColumnAutoInc::getNextAutoIncToSave() { - uint64_t nextValue = AUTOINCR_SATURATED; + uint64_t nextValue = AUTOINCR_SATURATED; - boost::mutex::scoped_lock lock(fAutoIncMutex); + boost::mutex::scoped_lock lock(fAutoIncMutex); - // nextValue is returned as -1 if we reached max value - if (fAutoIncLastValue < fMaxIntSat) - nextValue = fAutoIncLastValue + 1; + // nextValue is returned as -1 if we reached max value + if (fAutoIncLastValue < fMaxIntSat) + nextValue = fAutoIncLastValue + 1; - return nextValue; + return nextValue; } //------------------------------------------------------------------------------ // Get the current AI nextValue from the system catalog. //------------------------------------------------------------------------------ -int ColumnAutoInc::getNextValueFromSysCat( uint64_t& nextValue ) +int ColumnAutoInc::getNextValueFromSysCat(uint64_t& nextValue) { - std::string::size_type periodIdx = fTableName.find('.'); + std::string::size_type periodIdx = fTableName.find('.'); - std::string sName; - std::string tName; - sName.assign(fTableName, 0, periodIdx); - tName.assign(fTableName, periodIdx + 1, - fTableName.length() - (periodIdx + 1)); - execplan::CalpontSystemCatalog::TableName tbl(sName, tName); - uint64_t nextAuto = 0; + std::string sName; + std::string tName; + sName.assign(fTableName, 0, periodIdx); + tName.assign(fTableName, periodIdx + 1, fTableName.length() - (periodIdx + 1)); + execplan::CalpontSystemCatalog::TableName tbl(sName, tName); + uint64_t nextAuto = 0; - try + try + { + boost::shared_ptr systemCatPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + systemCatPtr->identity(execplan::CalpontSystemCatalog::EC); + + // Handle bad return code or thrown exception from + // system catalog query. + nextAuto = systemCatPtr->nextAutoIncrValue(tbl); + + if (nextAuto == 0) { - boost::shared_ptr systemCatPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - systemCatPtr->identity(execplan::CalpontSystemCatalog::EC); - - // Handle bad return code or thrown exception from - // system catalog query. - nextAuto = systemCatPtr->nextAutoIncrValue( tbl ); - - if (nextAuto == 0) - { - throw std::runtime_error( - "Not an auto-increment column, or column not found"); - } - else if (nextAuto == AUTOINCR_SATURATED) - { - throw std::runtime_error( - "auto-increment max value already reached"); - } - - nextValue = nextAuto; + throw std::runtime_error("Not an auto-increment column, or column not found"); } - catch (std::exception& ex) + else if (nextAuto == AUTOINCR_SATURATED) { - std::ostringstream oss; - oss << "Unable to get current auto-increment value for " << - sName << "." << tName << "; " << ex.what(); - fLog->logMsg( oss.str(), ERR_AUTOINC_INIT1, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(tName, oss); - return ERR_AUTOINC_INIT1; - } - catch (...) - { - std::ostringstream oss; - oss << "Unable to get current auto-increment value for " << - sName << "." << tName << "; unknown exception"; - fLog->logMsg( oss.str(), ERR_AUTOINC_INIT2, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(tName, oss); - return ERR_AUTOINC_INIT2; + throw std::runtime_error("auto-increment max value already reached"); } - return NO_ERROR; + nextValue = nextAuto; + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Unable to get current auto-increment value for " << sName << "." << tName << "; " << ex.what(); + fLog->logMsg(oss.str(), ERR_AUTOINC_INIT1, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(tName, oss); + return ERR_AUTOINC_INIT1; + } + catch (...) + { + std::ostringstream oss; + oss << "Unable to get current auto-increment value for " << sName << "." << tName + << "; unknown exception"; + fLog->logMsg(oss.str(), ERR_AUTOINC_INIT2, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(tName, oss); + return ERR_AUTOINC_INIT2; + } + + return NO_ERROR; } - + //------------------------------------------------------------------------------ // ColumnAutoIncJob constructor. //------------------------------------------------------------------------------ -ColumnAutoIncJob::ColumnAutoIncJob( Log* logger ) : ColumnAutoInc(logger) +ColumnAutoIncJob::ColumnAutoIncJob(Log* logger) : ColumnAutoInc(logger) { } @@ -321,7 +303,7 @@ ColumnAutoIncJob::ColumnAutoIncJob( Log* logger ) : ColumnAutoInc(logger) // ColumnAutoIncJob destructor. //------------------------------------------------------------------------------ /* virtual */ -ColumnAutoIncJob::~ColumnAutoIncJob( ) +ColumnAutoIncJob::~ColumnAutoIncJob() { } @@ -330,29 +312,25 @@ ColumnAutoIncJob::~ColumnAutoIncJob( ) // Returns starting nextValue associated with the reserved range of numbers. //------------------------------------------------------------------------------ /* virtual */ -int ColumnAutoIncJob::reserveNextRange( - uint32_t autoIncCount, - uint64_t& nextValue ) +int ColumnAutoIncJob::reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue) { - boost::mutex::scoped_lock lock(fAutoIncMutex); + boost::mutex::scoped_lock lock(fAutoIncMutex); - if ((fMaxIntSat - autoIncCount) < fAutoIncLastValue) - { - return ERR_AUTOINC_GEN_EXCEED_MAX; - } + if ((fMaxIntSat - autoIncCount) < fAutoIncLastValue) + { + return ERR_AUTOINC_GEN_EXCEED_MAX; + } - nextValue = fAutoIncLastValue + 1; - fAutoIncLastValue += autoIncCount; + nextValue = fAutoIncLastValue + 1; + fAutoIncLastValue += autoIncCount; - return NO_ERROR; + return NO_ERROR; } - //------------------------------------------------------------------------------ // ColumnAutoIncIncremental constructor. //------------------------------------------------------------------------------ -ColumnAutoIncIncremental::ColumnAutoIncIncremental( Log* logger ) : - ColumnAutoInc(logger) +ColumnAutoIncIncremental::ColumnAutoIncIncremental(Log* logger) : ColumnAutoInc(logger) { } @@ -360,7 +338,7 @@ ColumnAutoIncIncremental::ColumnAutoIncIncremental( Log* logger ) : // ColumnAutoIncIncremental destructor. //------------------------------------------------------------------------------ /* virtual */ -ColumnAutoIncIncremental::~ColumnAutoIncIncremental( ) +ColumnAutoIncIncremental::~ColumnAutoIncIncremental() { } @@ -369,45 +347,41 @@ ColumnAutoIncIncremental::~ColumnAutoIncIncremental( ) // Returns starting nextValue associated with the reserved range of numbers. //------------------------------------------------------------------------------ /* virtual */ -int ColumnAutoIncIncremental::reserveNextRange( - uint32_t autoIncCount, - uint64_t& nextValue ) +int ColumnAutoIncIncremental::reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue) { - uint64_t countArg = autoIncCount; - uint64_t nextValArg = 0; - std::string errMsg; - int rc = BRMWrapper::getInstance()->getAutoIncrementRange( - fColumnOID, countArg, nextValArg, errMsg ); + uint64_t countArg = autoIncCount; + uint64_t nextValArg = 0; + std::string errMsg; + int rc = BRMWrapper::getInstance()->getAutoIncrementRange(fColumnOID, countArg, nextValArg, errMsg); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Error reserving auto-increment range (" << countArg << - " numbers) for table " << - fTableName << "; column " << fColumnName << "; " << errMsg; + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error reserving auto-increment range (" << countArg << " numbers) for table " << fTableName + << "; column " << fColumnName << "; " << errMsg; - if (rc == ERR_AUTOINC_GEN_EXCEED_MAX) - oss << " Max allowed value is " << fMaxIntSat << "."; + if (rc == ERR_AUTOINC_GEN_EXCEED_MAX) + oss << " Max allowed value is " << fMaxIntSat << "."; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); - return rc; - } + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss); + return rc; + } - nextValue = nextValArg; - uint64_t autoIncLastValue = nextValue + autoIncCount - 1; + nextValue = nextValArg; + uint64_t autoIncLastValue = nextValue + autoIncCount - 1; - // For efficiency we delay the mutex till now, instead of before the call - // to getAutoIncrementRange(). This means we could theoretically end up - // processing AI ranges out of order, so we don't arbitrarily - // update fAutoIncLastValue. We only update it if the range in question - // exceeds the current value for fAutoIncLastValue. - boost::mutex::scoped_lock lock(fAutoIncMutex); + // For efficiency we delay the mutex till now, instead of before the call + // to getAutoIncrementRange(). This means we could theoretically end up + // processing AI ranges out of order, so we don't arbitrarily + // update fAutoIncLastValue. We only update it if the range in question + // exceeds the current value for fAutoIncLastValue. + boost::mutex::scoped_lock lock(fAutoIncMutex); - if (autoIncLastValue > fAutoIncLastValue) - fAutoIncLastValue = autoIncLastValue; + if (autoIncLastValue > fAutoIncLastValue) + fAutoIncLastValue = autoIncLastValue; - return NO_ERROR; + return NO_ERROR; } -} // end of namespace +} // namespace WriteEngine diff --git a/writeengine/bulk/we_columnautoinc.h b/writeengine/bulk/we_columnautoinc.h index f3d93c557..7751ed824 100644 --- a/writeengine/bulk/we_columnautoinc.h +++ b/writeengine/bulk/we_columnautoinc.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_columnautoinc.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_columnautoinc.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ /** @file we_columnautoinc.h * Contains classes to manage the "next value" for an auto-increment column. @@ -34,7 +34,7 @@ namespace WriteEngine { -class ColumnInfo; +class ColumnInfo; class Log; //------------------------------------------------------------------------------ @@ -43,46 +43,44 @@ class Log; //------------------------------------------------------------------------------ class ColumnAutoInc { -public: - /** @brief ColumnAutoInc constructor. - * @param logger Object use in logging. - */ - explicit ColumnAutoInc(Log* logger); + public: + /** @brief ColumnAutoInc constructor. + * @param logger Object use in logging. + */ + explicit ColumnAutoInc(Log* logger); - /** @brief ColumnAutoInc destructor. - */ - virtual ~ColumnAutoInc(); + /** @brief ColumnAutoInc destructor. + */ + virtual ~ColumnAutoInc(); - /** @brief Initialize auto-increment next value. - * @param fullTableName Name of table (schema.table) containing this column - * @param colInfo The auto-increment column. - */ - int init( const std::string& fullTableName, - ColumnInfo* colInfo ); + /** @brief Initialize auto-increment next value. + * @param fullTableName Name of table (schema.table) containing this column + * @param colInfo The auto-increment column. + */ + int init(const std::string& fullTableName, ColumnInfo* colInfo); - /** @brief Reserve next range of auto-increment numbers. - * @param autoIncCount Number of auto-increment numbers to reserve - * @param nextValue Starting number of reserved range - */ - virtual int reserveNextRange(uint32_t autoIncCount, - uint64_t& nextValue) = 0; + /** @brief Reserve next range of auto-increment numbers. + * @param autoIncCount Number of auto-increment numbers to reserve + * @param nextValue Starting number of reserved range + */ + virtual int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue) = 0; - /** @brief Finished with auto-incrementing; perform any applicable updates. - */ - int finish( ); + /** @brief Finished with auto-incrementing; perform any applicable updates. + */ + int finish(); -protected: - void initNextAutoInc(uint64_t nextValue); - uint64_t getNextAutoIncToSave( ); - int getNextValueFromSysCat(uint64_t& nextValue); + protected: + void initNextAutoInc(uint64_t nextValue); + uint64_t getNextAutoIncToSave(); + int getNextValueFromSysCat(uint64_t& nextValue); - Log* fLog; // import log file - boost::mutex fAutoIncMutex; // Mutex to manage fAutoIncLastValue - uint64_t fAutoIncLastValue;// Tracks latest autoincrement value used - uint64_t fMaxIntSat; // Maximum saturation value - std::string fTableName; // Full table name (schema.table) for AI column - std::string fColumnName; // Name of auto-increment column - OID fColumnOID; // Column OID + Log* fLog; // import log file + boost::mutex fAutoIncMutex; // Mutex to manage fAutoIncLastValue + uint64_t fAutoIncLastValue; // Tracks latest autoincrement value used + uint64_t fMaxIntSat; // Maximum saturation value + std::string fTableName; // Full table name (schema.table) for AI column + std::string fColumnName; // Name of auto-increment column + OID fColumnOID; // Column OID }; //------------------------------------------------------------------------------ @@ -98,12 +96,11 @@ protected: //------------------------------------------------------------------------------ class ColumnAutoIncJob : public ColumnAutoInc { -public: - explicit ColumnAutoIncJob(Log* logger); - virtual ~ColumnAutoIncJob(); + public: + explicit ColumnAutoIncJob(Log* logger); + virtual ~ColumnAutoIncJob(); - virtual int reserveNextRange(uint32_t autoIncCount, - uint64_t& nextValue); + virtual int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue); }; //------------------------------------------------------------------------------ @@ -117,14 +114,13 @@ public: //------------------------------------------------------------------------------ class ColumnAutoIncIncremental : public ColumnAutoInc { -public: - explicit ColumnAutoIncIncremental(Log* logger); - virtual ~ColumnAutoIncIncremental(); + public: + explicit ColumnAutoIncIncremental(Log* logger); + virtual ~ColumnAutoIncIncremental(); - virtual int reserveNextRange(uint32_t autoIncCount, - uint64_t& nextValue); + virtual int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue); }; -} //end of namespace +} // namespace WriteEngine -#endif // WE_COLUMNAUTOINC_H_ +#endif // WE_COLUMNAUTOINC_H_ diff --git a/writeengine/bulk/we_columninfo.cpp b/writeengine/bulk/we_columninfo.cpp index e1e883a39..3f89c28a0 100644 --- a/writeengine/bulk/we_columninfo.cpp +++ b/writeengine/bulk/we_columninfo.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************** -* $Id: we_columninfo.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_columninfo.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + * + *******************************************************************************/ #include #include @@ -42,7 +42,6 @@ using namespace idbdatafile; namespace { - //------------------------------------------------------------------------------ // Do a fast ascii-hex-string to binary data conversion. This is done in-place. // We take bytes 1 and 2 and put them back into byte 1; 3 and 4 into 2; etc. @@ -52,43 +51,43 @@ namespace //------------------------------------------------------------------------------ unsigned int compactVarBinary(char* charTmpBuf, int fieldLength) { - unsigned char* p = reinterpret_cast(charTmpBuf); - char* f = charTmpBuf; - char v = '\0'; + unsigned char* p = reinterpret_cast(charTmpBuf); + char* f = charTmpBuf; + char v = '\0'; - for (int i = 0; i < fieldLength / 2; i++, p++) - { - // Store even number byte in high order 4 bits of next output byte - v = *f; + for (int i = 0; i < fieldLength / 2; i++, p++) + { + // Store even number byte in high order 4 bits of next output byte + v = *f; - if (!isxdigit(v)) - return WriteEngine::COLPOSPAIR_NULL_TOKEN_OFFSET; + if (!isxdigit(v)) + return WriteEngine::COLPOSPAIR_NULL_TOKEN_OFFSET; - if (v <= '9') - *p = v - '0'; - else if (v <= 'F') - *p = v - 'A' + 10; - else //if (v <= 'f') - *p = v - 'a' + 10; + if (v <= '9') + *p = v - '0'; + else if (v <= 'F') + *p = v - 'A' + 10; + else // if (v <= 'f') + *p = v - 'a' + 10; - *p <<= 4; - f++; + *p <<= 4; + f++; - // Store odd number byte in low order 4 bite of next output byte - v = *f; + // Store odd number byte in low order 4 bite of next output byte + v = *f; - if (!isxdigit(v)) - return WriteEngine::COLPOSPAIR_NULL_TOKEN_OFFSET; + if (!isxdigit(v)) + return WriteEngine::COLPOSPAIR_NULL_TOKEN_OFFSET; - if (v <= '9') - *p |= v - '0'; - else if (v <= 'F') - *p |= v - 'A' + 10; - else //if (v <= 'f') - *p |= v - 'a' + 10; + if (v <= '9') + *p |= v - '0'; + else if (v <= 'F') + *p |= v - 'A' + 10; + else // if (v <= 'f') + *p |= v - 'a' + 10; - f++; - } + f++; + } // Changed our mind and decided to have the read thread reject rows with // incomplete (odd length) varbinary fields, so the following check is not @@ -119,112 +118,109 @@ unsigned int compactVarBinary(char* charTmpBuf, int fieldLength) #endif - return (fieldLength / 2); + return (fieldLength / 2); } -} +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // ColumnInfo constructor //------------------------------------------------------------------------------ -ColumnInfo::ColumnInfo(Log* logger, - int idIn, - const JobColumn& columnIn, - DBRootExtentTracker* pDBRootExtTrk, - TableInfo* pTableInfo) : - id(idIn), - lastProcessingTime(0), +ColumnInfo::ColumnInfo(Log* logger, int idIn, const JobColumn& columnIn, DBRootExtentTracker* pDBRootExtTrk, + TableInfo* pTableInfo) + : id(idIn) + , lastProcessingTime(0) + , #ifdef PROFILE - totalProcessingTime(0), + totalProcessingTime(0) + , #endif - fColBufferMgr(0), - availFileSize(0), - fileSize(0), - fLog(logger), - fDelayedFileStartBlksSkipped(0), - fSavedLbid(0), - fLastUpdatedLbid(0), - fSizeWrittenStart(0), - fSizeWritten(0), - fLastInputRowInCurrentExtent(0), - fLoadingAbbreviatedExtent(false), - fColExtInf(0), - fMaxNumRowsPerSegFile(0), - fStore(0), - fAutoIncLastValue(0), - fSaturatedRowCnt(0), - fpTableInfo(pTableInfo), - fAutoIncMgr(0), - fDbRootExtTrk(pDBRootExtTrk), - fColWidthFactor(1), - fDelayedFileCreation(INITIAL_DBFILE_STAT_FILE_EXISTS), - fRowsPerExtent(0) + fColBufferMgr(0) + , availFileSize(0) + , fileSize(0) + , fLog(logger) + , fDelayedFileStartBlksSkipped(0) + , fSavedLbid(0) + , fLastUpdatedLbid(0) + , fSizeWrittenStart(0) + , fSizeWritten(0) + , fLastInputRowInCurrentExtent(0) + , fLoadingAbbreviatedExtent(false) + , fColExtInf(0) + , fMaxNumRowsPerSegFile(0) + , fStore(0) + , fAutoIncLastValue(0) + , fSaturatedRowCnt(0) + , fpTableInfo(pTableInfo) + , fAutoIncMgr(0) + , fDbRootExtTrk(pDBRootExtTrk) + , fColWidthFactor(1) + , fDelayedFileCreation(INITIAL_DBFILE_STAT_FILE_EXISTS) + , fRowsPerExtent(0) { - column = columnIn; + column = columnIn; - fRowsPerExtent = BRMWrapper::getInstance()->getExtentRows(); + fRowsPerExtent = BRMWrapper::getInstance()->getExtentRows(); - // Allocate a ColExtInfBase object for those types that won't track - // min/max CasualPartition info; this is a stub class that won't do - // anything. - switch ( column.weType ) + // Allocate a ColExtInfBase object for those types that won't track + // min/max CasualPartition info; this is a stub class that won't do + // anything. + switch (column.weType) + { + case WriteEngine::WR_FLOAT: + case WriteEngine::WR_DOUBLE: + case WriteEngine::WR_VARBINARY: // treat like char dictionary for now + case WriteEngine::WR_TOKEN: { - case WriteEngine::WR_FLOAT: - case WriteEngine::WR_DOUBLE: - case WriteEngine::WR_VARBINARY: // treat like char dictionary for now - case WriteEngine::WR_TOKEN: - { - fColExtInf = new ColExtInfBase( ); - break; - } - - case WriteEngine::WR_CHAR: - { - if (column.colType == COL_TYPE_DICT) - { - fColExtInf = new ColExtInfBase( ); - } - else - { - fColExtInf = new ColExtInf(column.mapOid, logger); - } - - break; - } - - case WriteEngine::WR_SHORT: - case WriteEngine::WR_BYTE: - case WriteEngine::WR_LONGLONG: - case WriteEngine::WR_MEDINT: - case WriteEngine::WR_INT: - case WriteEngine::WR_USHORT: - case WriteEngine::WR_UBYTE: - case WriteEngine::WR_ULONGLONG: - case WriteEngine::WR_UMEDINT: - case WriteEngine::WR_UINT: - case WriteEngine::WR_BINARY: - default: - { - fColExtInf = new ColExtInf(column.mapOid, logger); - break; - } + fColExtInf = new ColExtInfBase(); + break; } - colOp.reset(new ColumnOpBulk(logger, column.compressionType)); - - fMaxNumRowsPerSegFile = fRowsPerExtent * - Config::getExtentsPerSegmentFile(); - - // Create auto-increment object to manage auto-increment next-value - if (column.autoIncFlag) + case WriteEngine::WR_CHAR: { - fAutoIncMgr = new ColumnAutoIncIncremental(logger); - // formerly used ColumnAutoIncJob for Shared Everything - // fAutoIncMgr = new ColumnAutoIncJob(logger); + if (column.colType == COL_TYPE_DICT) + { + fColExtInf = new ColExtInfBase(); + } + else + { + fColExtInf = new ColExtInf(column.mapOid, logger); + } + + break; } + + case WriteEngine::WR_SHORT: + case WriteEngine::WR_BYTE: + case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_MEDINT: + case WriteEngine::WR_INT: + case WriteEngine::WR_USHORT: + case WriteEngine::WR_UBYTE: + case WriteEngine::WR_ULONGLONG: + case WriteEngine::WR_UMEDINT: + case WriteEngine::WR_UINT: + case WriteEngine::WR_BINARY: + default: + { + fColExtInf = new ColExtInf(column.mapOid, logger); + break; + } + } + + colOp.reset(new ColumnOpBulk(logger, column.compressionType)); + + fMaxNumRowsPerSegFile = fRowsPerExtent * Config::getExtentsPerSegmentFile(); + + // Create auto-increment object to manage auto-increment next-value + if (column.autoIncFlag) + { + fAutoIncMgr = new ColumnAutoIncIncremental(logger); + // formerly used ColumnAutoIncJob for Shared Everything + // fAutoIncMgr = new ColumnAutoIncJob(logger); + } } //------------------------------------------------------------------------------ @@ -232,40 +228,40 @@ ColumnInfo::ColumnInfo(Log* logger, //------------------------------------------------------------------------------ ColumnInfo::~ColumnInfo() { - clearMemory(); + clearMemory(); - // Closing dictionary file also updates the extent map; which we - // don't want to do if we are aborting the job. Besides, the - // application code should be closing the dictionary as needed, - // instead of relying on the destructor, so disabled this code. - //if(fStore != NULL) - //{ - // fStore->closeDctnryStore(); - // delete fStore; - //} + // Closing dictionary file also updates the extent map; which we + // don't want to do if we are aborting the job. Besides, the + // application code should be closing the dictionary as needed, + // instead of relying on the destructor, so disabled this code. + // if(fStore != NULL) + //{ + // fStore->closeDctnryStore(); + // delete fStore; + //} - if (fColExtInf) - delete fColExtInf; + if (fColExtInf) + delete fColExtInf; - if (fAutoIncMgr) - delete fAutoIncMgr; + if (fAutoIncMgr) + delete fAutoIncMgr; - if (fDbRootExtTrk) - delete fDbRootExtTrk; + if (fDbRootExtTrk) + delete fDbRootExtTrk; } //------------------------------------------------------------------------------ // Clear memory consumed by this ColumnInfo object. //------------------------------------------------------------------------------ -void ColumnInfo::clearMemory( ) +void ColumnInfo::clearMemory() { - if (fColBufferMgr) - { - delete fColBufferMgr; - fColBufferMgr = 0; - } + if (fColBufferMgr) + { + delete fColBufferMgr; + fColBufferMgr = 0; + } - fDictBlocks.clear(); + fDictBlocks.clear(); } //------------------------------------------------------------------------------ @@ -275,267 +271,212 @@ void ColumnInfo::clearMemory( ) // A starting DB file will be created if/when we determine that we have rows // to be processed. //------------------------------------------------------------------------------ -void ColumnInfo::setupDelayedFileCreation( - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM hwm, - bool bEmptyPM ) +void ColumnInfo::setupDelayedFileCreation(uint16_t dbRoot, uint32_t partition, uint16_t segment, HWM hwm, + bool bEmptyPM) { - if (bEmptyPM) - fDelayedFileCreation = INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY; - else - fDelayedFileCreation = INITIAL_DBFILE_STAT_CREATE_FILE_ON_DISABLED; + if (bEmptyPM) + fDelayedFileCreation = INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY; + else + fDelayedFileCreation = INITIAL_DBFILE_STAT_CREATE_FILE_ON_DISABLED; - fDelayedFileStartBlksSkipped = hwm; - fSavedLbid = INVALID_LBID; + fDelayedFileStartBlksSkipped = hwm; + fSavedLbid = INVALID_LBID; - colOp->initColumn ( curCol ); - colOp->setColParam( curCol, id, - column.width, - column.dataType, - column.weType, - column.mapOid, - column.compressionType, - dbRoot, partition, segment ); - colOp->findTypeHandler(column.width, column.dataType); + colOp->initColumn(curCol); + colOp->setColParam(curCol, id, column.width, column.dataType, column.weType, column.mapOid, + column.compressionType, dbRoot, partition, segment); + colOp->findTypeHandler(column.width, column.dataType); } //------------------------------------------------------------------------------ // Create a DB file as part of delayed file creation. See setupDelayedFile- // Creation for an explanation. //------------------------------------------------------------------------------ -int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName ) +int ColumnInfo::createDelayedFileIfNeeded(const std::string& tableName) { - int rc = NO_ERROR; + int rc = NO_ERROR; - // For optimization sake, we use a separate mutex (fDelayedFileCreateMutex) - // exclusively reserved to be used as the gatekeeper to this function. - // No sense in waiting for a fColMutex lock, when 99.99% of the time, - // all we need to do is check fDelayedFileCreation, see that it's value - // is INITIAL_DBFILE_STAT_FILE_EXISTS, and exit the function. - boost::mutex::scoped_lock lock(fDelayedFileCreateMutex); + // For optimization sake, we use a separate mutex (fDelayedFileCreateMutex) + // exclusively reserved to be used as the gatekeeper to this function. + // No sense in waiting for a fColMutex lock, when 99.99% of the time, + // all we need to do is check fDelayedFileCreation, see that it's value + // is INITIAL_DBFILE_STAT_FILE_EXISTS, and exit the function. + boost::mutex::scoped_lock lock(fDelayedFileCreateMutex); - if (fDelayedFileCreation == INITIAL_DBFILE_STAT_FILE_EXISTS) - return NO_ERROR; - - // Don't try creating extent again if we are already in error state with a - // previous thread failing to create this extent. - if (fDelayedFileCreation == INITIAL_DBFILE_STAT_ERROR_STATE) - { - rc = ERR_FILE_CREATE; - std::ostringstream oss; - oss << "Previous attempt failed to create initial dbroot" << - curCol.dataFile.fDbRoot << - " extent for column file OID-" << column.mapOid << - "; dbroot-" << curCol.dataFile.fDbRoot << - "; partition-" << curCol.dataFile.fPartition; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - - // Once we get this far, we go ahead and acquire a fColMutex lock. The - // fDelayedFileCreateMutex lock might suffice, but better to explicitly - // lock fColMutex since we are modifying attributes that we typically - // change within the scope of a fColMutex lock. - boost::mutex::scoped_lock lock2(fColMutex); - - uint16_t dbRoot = curCol.dataFile.fDbRoot; - uint32_t partition = curCol.dataFile.fPartition; - - // We don't have a file on this PM, so we create an initial file - ColumnOpBulk tempColOp(fLog, column.compressionType); - - bool createLeaveFileOpen = false; - IDBDataFile* createPFile = 0; - uint16_t createDbRoot = dbRoot; - uint32_t createPartition = partition; - uint16_t createSegment = 0; - std::string createSegFile; - HWM createHwm = 0; //output - BRM::LBID_t createStartLbid = 0; //output - bool createNewFile = true; //output - int createAllocSize = 0; //output - char* createHdrs = 0; //output - - std::string allocErrMsg; - rc = fpTableInfo->allocateBRMColumnExtent( curCol.dataFile.fid, - createDbRoot, - createPartition, - createSegment, - createStartLbid, - createAllocSize, - createHwm, - allocErrMsg ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error creating initial dbroot" << dbRoot << - " BRM extent for OID-" << column.mapOid << - "; dbroot-" << dbRoot << - "; partition-" << partition << - "; " << ec.errorString(rc) << - "; " << allocErrMsg; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; - return rc; - } - - uint16_t segment = createSegment; - partition = createPartition; // update our partition variable in - // case extent was added to a different - // partition than we intended - BRM::LBID_t lbid = createStartLbid; - - rc = tempColOp.extendColumn( - curCol, - createLeaveFileOpen, - createHwm, - createStartLbid, - createAllocSize, - createDbRoot, - createPartition, - createSegment, - createSegFile, - createPFile, - createNewFile, - createHdrs); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error adding initial dbroot" << dbRoot << - " extent to column file OID-" << column.mapOid << - "; dbroot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; - return rc; - } - - // We don't have a file on this PM (or HWM extent is disabled), so we - // create a new file to load - std::ostringstream oss1; - - if (fDelayedFileCreation == INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY) - oss1 << "PM empty; Creating starting column extent"; - else - oss1 << "HWM extent disabled; Creating starting column extent"; - - oss1 << " on DBRoot-" << createDbRoot << - " for OID-" << column.mapOid << - "; part-" << createPartition << - "; seg-" << createSegment << - "; hwm-" << createHwm << - "; LBID-" << createStartLbid << - "; file-" << createSegFile; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - - // Create corresponding dictionary store file if applicable - if (column.colType == COL_TYPE_DICT) - { - std::ostringstream oss; - oss << "Creating starting dictionary extent on dbroot" << dbRoot << - " (segment " << segment << - ") for dictionary OID " << column.dctnry.dctnryOid; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - BRM::LBID_t dLbid; - Dctnry* tempD = 0; - - if (column.dctnry.fCompressionType != 0) - { - DctnryCompress1* tempD1; - tempD1 = new DctnryCompress1(column.dctnry.fCompressionType); - tempD1->setMaxActiveChunkNum(1); - tempD1->setBulkFlag(true); - tempD = tempD1; - } - else - { - tempD = new DctnryCompress0; - } - - boost::scoped_ptr refDctnry(tempD); - // MCOL-4328 Define a file owner uid and gid - refDctnry->setUIDGID(this); - - rc = tempD->createDctnry( - column.dctnry.dctnryOid, - column.dctnryWidth, - dbRoot, - partition, - segment, - dLbid, - true); // creating the store file - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error creating initial dbroot" << dbRoot << - " extent for dictionary file OID-" << - column.dctnry.dctnryOid << - "; dbroot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; - return rc; - } - - rc = tempD->closeDctnry(); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error creating/closing initial dbroot" << - dbRoot << " extent for dictionary file OID-" << - column.dctnry.dctnryOid << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; - return rc; - } - } // end of dictionary column processing - - // Check for special case: where we skip initial blk(s) at the start of - // the "very" 1st file on each PM. - // We are checking to see if the PM is empty, "and" if the partition is 0. - // The PM could be empty if all the existing files on the PM were dropped - // or disabled, but we don't want/need to do block skipping in this case; - // so we also check to see if the partition number is 0, denoting the 1st - // extent for the PM. - // (The reason we are skipping blocks in partition 0, is because import - // does this with the partition 0, segment 0 file created by DDL. - // We skip blocks on the other PMs, so that the 1st file created on each - // PM will employ the same block skipping.) - HWM hwm = 0; - - if ((fDelayedFileCreation == INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY) && - (partition == 0)) - { - hwm = fDelayedFileStartBlksSkipped; - } - - rc = setupInitialColumnExtent( - dbRoot, partition, segment, - tableName, lbid, hwm, hwm, false, true ); - - if (rc == NO_ERROR) - fDelayedFileCreation = INITIAL_DBFILE_STAT_FILE_EXISTS; - else - fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; + if (fDelayedFileCreation == INITIAL_DBFILE_STAT_FILE_EXISTS) + return NO_ERROR; + // Don't try creating extent again if we are already in error state with a + // previous thread failing to create this extent. + if (fDelayedFileCreation == INITIAL_DBFILE_STAT_ERROR_STATE) + { + rc = ERR_FILE_CREATE; + std::ostringstream oss; + oss << "Previous attempt failed to create initial dbroot" << curCol.dataFile.fDbRoot + << " extent for column file OID-" << column.mapOid << "; dbroot-" << curCol.dataFile.fDbRoot + << "; partition-" << curCol.dataFile.fPartition; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); return rc; + } + + // Once we get this far, we go ahead and acquire a fColMutex lock. The + // fDelayedFileCreateMutex lock might suffice, but better to explicitly + // lock fColMutex since we are modifying attributes that we typically + // change within the scope of a fColMutex lock. + boost::mutex::scoped_lock lock2(fColMutex); + + uint16_t dbRoot = curCol.dataFile.fDbRoot; + uint32_t partition = curCol.dataFile.fPartition; + + // We don't have a file on this PM, so we create an initial file + ColumnOpBulk tempColOp(fLog, column.compressionType); + + bool createLeaveFileOpen = false; + IDBDataFile* createPFile = 0; + uint16_t createDbRoot = dbRoot; + uint32_t createPartition = partition; + uint16_t createSegment = 0; + std::string createSegFile; + HWM createHwm = 0; // output + BRM::LBID_t createStartLbid = 0; // output + bool createNewFile = true; // output + int createAllocSize = 0; // output + char* createHdrs = 0; // output + + std::string allocErrMsg; + rc = fpTableInfo->allocateBRMColumnExtent(curCol.dataFile.fid, createDbRoot, createPartition, createSegment, + createStartLbid, createAllocSize, createHwm, allocErrMsg); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error creating initial dbroot" << dbRoot << " BRM extent for OID-" << column.mapOid << "; dbroot-" + << dbRoot << "; partition-" << partition << "; " << ec.errorString(rc) << "; " << allocErrMsg; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; + return rc; + } + + uint16_t segment = createSegment; + partition = createPartition; // update our partition variable in + // case extent was added to a different + // partition than we intended + BRM::LBID_t lbid = createStartLbid; + + rc = tempColOp.extendColumn(curCol, createLeaveFileOpen, createHwm, createStartLbid, createAllocSize, + createDbRoot, createPartition, createSegment, createSegFile, createPFile, + createNewFile, createHdrs); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error adding initial dbroot" << dbRoot << " extent to column file OID-" << column.mapOid + << "; dbroot-" << dbRoot << "; partition-" << partition << "; segment-" << segment << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; + return rc; + } + + // We don't have a file on this PM (or HWM extent is disabled), so we + // create a new file to load + std::ostringstream oss1; + + if (fDelayedFileCreation == INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY) + oss1 << "PM empty; Creating starting column extent"; + else + oss1 << "HWM extent disabled; Creating starting column extent"; + + oss1 << " on DBRoot-" << createDbRoot << " for OID-" << column.mapOid << "; part-" << createPartition + << "; seg-" << createSegment << "; hwm-" << createHwm << "; LBID-" << createStartLbid << "; file-" + << createSegFile; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + + // Create corresponding dictionary store file if applicable + if (column.colType == COL_TYPE_DICT) + { + std::ostringstream oss; + oss << "Creating starting dictionary extent on dbroot" << dbRoot << " (segment " << segment + << ") for dictionary OID " << column.dctnry.dctnryOid; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + BRM::LBID_t dLbid; + Dctnry* tempD = 0; + + if (column.dctnry.fCompressionType != 0) + { + DctnryCompress1* tempD1; + tempD1 = new DctnryCompress1(column.dctnry.fCompressionType); + tempD1->setMaxActiveChunkNum(1); + tempD1->setBulkFlag(true); + tempD = tempD1; + } + else + { + tempD = new DctnryCompress0; + } + + boost::scoped_ptr refDctnry(tempD); + // MCOL-4328 Define a file owner uid and gid + refDctnry->setUIDGID(this); + + rc = tempD->createDctnry(column.dctnry.dctnryOid, column.dctnryWidth, dbRoot, partition, segment, dLbid, + true); // creating the store file + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error creating initial dbroot" << dbRoot << " extent for dictionary file OID-" + << column.dctnry.dctnryOid << "; dbroot-" << dbRoot << "; partition-" << partition << "; segment-" + << segment << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; + return rc; + } + + rc = tempD->closeDctnry(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error creating/closing initial dbroot" << dbRoot << " extent for dictionary file OID-" + << column.dctnry.dctnryOid << "; partition-" << partition << "; segment-" << segment << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; + return rc; + } + } // end of dictionary column processing + + // Check for special case: where we skip initial blk(s) at the start of + // the "very" 1st file on each PM. + // We are checking to see if the PM is empty, "and" if the partition is 0. + // The PM could be empty if all the existing files on the PM were dropped + // or disabled, but we don't want/need to do block skipping in this case; + // so we also check to see if the partition number is 0, denoting the 1st + // extent for the PM. + // (The reason we are skipping blocks in partition 0, is because import + // does this with the partition 0, segment 0 file created by DDL. + // We skip blocks on the other PMs, so that the 1st file created on each + // PM will employ the same block skipping.) + HWM hwm = 0; + + if ((fDelayedFileCreation == INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY) && (partition == 0)) + { + hwm = fDelayedFileStartBlksSkipped; + } + + rc = setupInitialColumnExtent(dbRoot, partition, segment, tableName, lbid, hwm, hwm, false, true); + + if (rc == NO_ERROR) + fDelayedFileCreation = INITIAL_DBFILE_STAT_FILE_EXISTS; + else + fDelayedFileCreation = INITIAL_DBFILE_STAT_ERROR_STATE; + + return rc; } //------------------------------------------------------------------------------ @@ -545,275 +486,232 @@ int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName ) // NOTE: no mutex lock is employed here. It is assumed that the calling // application code is taking care of this, if it is needed. //------------------------------------------------------------------------------ -int ColumnInfo::extendColumn( bool saveLBIDForCP ) +int ColumnInfo::extendColumn(bool saveLBIDForCP) { - //..We assume the applicable file is already open, so... - // the HWM of the current segment file should be set to reference the - // last block in the current file (as specified in curCol.dataFile.pFile). - // - // Prior to adding compression, we used ftell() to set HWM, but that - // would not work for compressed data. Code now assumes that if we - // are adding an extent, that fSizeWritten is a multiple of blksize, - // which it should be. If we are adding an extent, fSizeWritten should - // point to the last byte of a full extent boundary. - HWM hwm = (fSizeWritten / BYTE_PER_BLOCK) - 1; + //..We assume the applicable file is already open, so... + // the HWM of the current segment file should be set to reference the + // last block in the current file (as specified in curCol.dataFile.pFile). + // + // Prior to adding compression, we used ftell() to set HWM, but that + // would not work for compressed data. Code now assumes that if we + // are adding an extent, that fSizeWritten is a multiple of blksize, + // which it should be. If we are adding an extent, fSizeWritten should + // point to the last byte of a full extent boundary. + HWM hwm = (fSizeWritten / BYTE_PER_BLOCK) - 1; - //..Save info about the current segment column file, and close that file. - addToSegFileList( curCol.dataFile, hwm ); + //..Save info about the current segment column file, and close that file. + addToSegFileList(curCol.dataFile, hwm); - // Close current segment column file prior to adding extent to next seg file - int rc = closeColumnFile( true, false ); + // Close current segment column file prior to adding extent to next seg file + int rc = closeColumnFile(true, false); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "extendColumn: error closing extent in " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; hwm-" << hwm; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - // Call Config::initConfigCache() to force the Config class - // to reload config cache "if" the config file has changed. - Config::initConfigCache(); - - bool bChangeFlag = Config::hasLocalDBRootListChanged(); - - //if (fLog->isDebug( DEBUG_1 )) - //{ - // std::ostringstream oss; - // oss << "Checking DBRootListChangeFlag: " << bChangeFlag; - // fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - //} - if (bChangeFlag) - { - rc = ERR_BULK_DBROOT_CHANGE; - - WErrorCodes ec; - std::ostringstream oss; - oss << "extendColumn: DBRoots changed; " << - ec.errorString( rc ); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - //..Declare variables used to advance to the next extent - uint16_t dbRootNext = 0; - uint32_t partitionNext = 0; - uint16_t segmentNext = 0; - HWM hwmNext = 0; - BRM::LBID_t startLbid; - - //..When we finish an extent, we typically should be advancing to the next - // DBRoot to create a "new" extent. But "if" the user has moved a DBRoot - // from another PM to this PM, then we may have a partial extent that we - // need to fill up. Here's where we just fill out such partially filled - // extents with empty values, until we can get back to a "normal" full - // extent boundary case. - bool bAllocNewExtent = false; - - while (!bAllocNewExtent) - { - //..If we have a DBRoot Tracker, then use that to determine next DBRoot - // to rotate to, else the old legacy BRM extent allocator will assign, - // if we pass in a dbroot of 0. - bAllocNewExtent = true; - - if (fDbRootExtTrk) - { - bAllocNewExtent = fDbRootExtTrk->nextSegFile( - dbRootNext, partitionNext, segmentNext, hwmNext, startLbid ); - } - - // If our next extent is a partial extent, then fill out that extent - // to the next full extent boundary, and round up HWM accordingly. - if (!bAllocNewExtent) - { - rc = extendColumnOldExtent( dbRootNext, - partitionNext, segmentNext, hwmNext ); - - if (rc != NO_ERROR) - return rc; - } - } - - // Once we are back on a "normal" full extent boundary, we add a new extent - // to resume adding rows. - rc = extendColumnNewExtent( saveLBIDForCP, dbRootNext, partitionNext ); + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "extendColumn: error closing extent in " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; hwm-" << hwm; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); return rc; + } + + // Call Config::initConfigCache() to force the Config class + // to reload config cache "if" the config file has changed. + Config::initConfigCache(); + + bool bChangeFlag = Config::hasLocalDBRootListChanged(); + + // if (fLog->isDebug( DEBUG_1 )) + //{ + // std::ostringstream oss; + // oss << "Checking DBRootListChangeFlag: " << bChangeFlag; + // fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + //} + if (bChangeFlag) + { + rc = ERR_BULK_DBROOT_CHANGE; + + WErrorCodes ec; + std::ostringstream oss; + oss << "extendColumn: DBRoots changed; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + return rc; + } + + //..Declare variables used to advance to the next extent + uint16_t dbRootNext = 0; + uint32_t partitionNext = 0; + uint16_t segmentNext = 0; + HWM hwmNext = 0; + BRM::LBID_t startLbid; + + //..When we finish an extent, we typically should be advancing to the next + // DBRoot to create a "new" extent. But "if" the user has moved a DBRoot + // from another PM to this PM, then we may have a partial extent that we + // need to fill up. Here's where we just fill out such partially filled + // extents with empty values, until we can get back to a "normal" full + // extent boundary case. + bool bAllocNewExtent = false; + + while (!bAllocNewExtent) + { + //..If we have a DBRoot Tracker, then use that to determine next DBRoot + // to rotate to, else the old legacy BRM extent allocator will assign, + // if we pass in a dbroot of 0. + bAllocNewExtent = true; + + if (fDbRootExtTrk) + { + bAllocNewExtent = + fDbRootExtTrk->nextSegFile(dbRootNext, partitionNext, segmentNext, hwmNext, startLbid); + } + + // If our next extent is a partial extent, then fill out that extent + // to the next full extent boundary, and round up HWM accordingly. + if (!bAllocNewExtent) + { + rc = extendColumnOldExtent(dbRootNext, partitionNext, segmentNext, hwmNext); + + if (rc != NO_ERROR) + return rc; + } + } + + // Once we are back on a "normal" full extent boundary, we add a new extent + // to resume adding rows. + rc = extendColumnNewExtent(saveLBIDForCP, dbRootNext, partitionNext); + + return rc; } //------------------------------------------------------------------------------ // Add a new extent to this column, at the specified DBRoot. Partition may be // used if DBRoot is empty. //------------------------------------------------------------------------------ -int ColumnInfo::extendColumnNewExtent( - bool saveLBIDForCP, - uint16_t dbRootNew, - uint32_t partitionNew ) +int ColumnInfo::extendColumnNewExtent(bool saveLBIDForCP, uint16_t dbRootNew, uint32_t partitionNew) { - //..Declare variables used to advance to the next extent - IDBDataFile* pFileNew = 0; - HWM hwmNew = 0; - bool newFile = false; - std::string segFileNew; + //..Declare variables used to advance to the next extent + IDBDataFile* pFileNew = 0; + HWM hwmNew = 0; + bool newFile = false; + std::string segFileNew; - uint16_t segmentNew = 0; - BRM::LBID_t startLbid; + uint16_t segmentNew = 0; + BRM::LBID_t startLbid; - char hdr[ compress::CompressInterface::HDR_BUF_LEN * 2 ]; + char hdr[compress::CompressInterface::HDR_BUF_LEN * 2]; - // Extend the column by adding an extent to the next - // DBRoot, partition, and segment file in the rotation - int allocsize = 0; - std::string allocErrMsg; - int rc = fpTableInfo->allocateBRMColumnExtent( curCol.dataFile.fid, - dbRootNew, - partitionNew, - segmentNew, - startLbid, - allocsize, - hwmNew, - allocErrMsg ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "extendColumnNewExtent: error creating BRM extent after " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment; - - oss << "; newDBRoot-" << dbRootNew << - "; newpart-" << partitionNew << - "; " << ec.errorString(rc) << - "; " << allocErrMsg; - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - - return rc; - } - - rc = colOp->extendColumn ( curCol, - true, // leave file open - hwmNew, - startLbid, - allocsize, - dbRootNew, - partitionNew, - segmentNew, - segFileNew, - pFileNew, - newFile, - hdr ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "extendColumnNewExtent: error adding file extent after " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment; - - oss << "; newDBRoot-" << dbRootNew << - "; newpart-" << partitionNew << - "; newseg-" << segmentNew << - "; fbo-" << hwmNew << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - - if (pFileNew) - colOp->closeFile( pFileNew ); // clean up loose ends - - return rc; - } + // Extend the column by adding an extent to the next + // DBRoot, partition, and segment file in the rotation + int allocsize = 0; + std::string allocErrMsg; + int rc = fpTableInfo->allocateBRMColumnExtent(curCol.dataFile.fid, dbRootNew, partitionNew, segmentNew, + startLbid, allocsize, hwmNew, allocErrMsg); + if (rc != NO_ERROR) + { + WErrorCodes ec; std::ostringstream oss; - oss << "Add column extent OID-" << curCol.dataFile.fid << - "; DBRoot-" << dbRootNew << - "; part-" << partitionNew << - "; seg-" << segmentNew << - "; hwm-" << hwmNew << - "; LBID-" << startLbid << - "; file-" << segFileNew; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + oss << "extendColumnNewExtent: error creating BRM extent after " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment; - // Update lbid. - fLastUpdatedLbid = startLbid; + oss << "; newDBRoot-" << dbRootNew << "; newpart-" << partitionNew << "; " << ec.errorString(rc) << "; " + << allocErrMsg; + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - // Save the LBID with our CP extent info, so that we can update extent map - if (saveLBIDForCP) + return rc; + } + + rc = colOp->extendColumn(curCol, + true, // leave file open + hwmNew, startLbid, allocsize, dbRootNew, partitionNew, segmentNew, segFileNew, + pFileNew, newFile, hdr); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "extendColumnNewExtent: error adding file extent after " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment; + + oss << "; newDBRoot-" << dbRootNew << "; newpart-" << partitionNew << "; newseg-" << segmentNew + << "; fbo-" << hwmNew << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + + if (pFileNew) + colOp->closeFile(pFileNew); // clean up loose ends + + return rc; + } + + std::ostringstream oss; + oss << "Add column extent OID-" << curCol.dataFile.fid << "; DBRoot-" << dbRootNew << "; part-" + << partitionNew << "; seg-" << segmentNew << "; hwm-" << hwmNew << "; LBID-" << startLbid << "; file-" + << segFileNew; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + // Update lbid. + fLastUpdatedLbid = startLbid; + + // Save the LBID with our CP extent info, so that we can update extent map + if (saveLBIDForCP) + { + int rcLBID = fColExtInf->updateEntryLbid(startLbid); + + // If error occurs, we log WARNING, but we don't fail the job. + if (rcLBID != NO_ERROR) { - int rcLBID = fColExtInf->updateEntryLbid( startLbid ); - - // If error occurs, we log WARNING, but we don't fail the job. - if (rcLBID != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "updateEntryLbid failed for OID-" << curCol.dataFile.fid << - "; LBID-" << startLbid << - "; CasualPartition info may become invalid; " << - ec.errorString(rcLBID); - fLog->logMsg( oss.str(), rcLBID, MSGLVL_WARNING ); - } + WErrorCodes ec; + std::ostringstream oss; + oss << "updateEntryLbid failed for OID-" << curCol.dataFile.fid << "; LBID-" << startLbid + << "; CasualPartition info may become invalid; " << ec.errorString(rcLBID); + fLog->logMsg(oss.str(), rcLBID, MSGLVL_WARNING); } + } - //..Reset data members to reflect where we are in the newly - // opened column segment file. The file may be a new file, or we may - // be adding an extent to an existing column segment file. - curCol.dataFile.hwm = hwmNew; - curCol.dataFile.pFile = pFileNew; - curCol.dataFile.fPartition = partitionNew; - curCol.dataFile.fSegment = segmentNew; - curCol.dataFile.fDbRoot = dbRootNew; - curCol.dataFile.fSegFileName = segFileNew; + //..Reset data members to reflect where we are in the newly + // opened column segment file. The file may be a new file, or we may + // be adding an extent to an existing column segment file. + curCol.dataFile.hwm = hwmNew; + curCol.dataFile.pFile = pFileNew; + curCol.dataFile.fPartition = partitionNew; + curCol.dataFile.fSegment = segmentNew; + curCol.dataFile.fDbRoot = dbRootNew; + curCol.dataFile.fSegFileName = segFileNew; - rc = resetFileOffsetsNewExtent(hdr); + rc = resetFileOffsetsNewExtent(hdr); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "extendColumnNewExtent: error moving to new extent in " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; hwm-" << curCol.dataFile.hwm; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "extendColumnNewExtent: error moving to new extent in " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; hwm-" + << curCol.dataFile.hwm; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - if (pFileNew) - closeColumnFile( false, true ); // clean up loose ends + if (pFileNew) + closeColumnFile(false, true); // clean up loose ends - return rc; - } + return rc; + } - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss2; - oss2 << "Extent added to column OID-" << curCol.dataFile.fid << - "; DBRoot-" << dbRootNew << - "; part-" << partitionNew << - "; seg-" << segmentNew << - "; begByte-" << fSizeWritten << - "; endByte-" << fileSize << - "; freeBytes-" << availFileSize; - fLog->logMsg( oss2.str(), MSGLVL_INFO2 ); - } + if (fLog->isDebug(DEBUG_1)) + { + std::ostringstream oss2; + oss2 << "Extent added to column OID-" << curCol.dataFile.fid << "; DBRoot-" << dbRootNew << "; part-" + << partitionNew << "; seg-" << segmentNew << "; begByte-" << fSizeWritten << "; endByte-" << fileSize + << "; freeBytes-" << availFileSize; + fLog->logMsg(oss2.str(), MSGLVL_INFO2); + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -822,111 +720,93 @@ int ColumnInfo::extendColumnNewExtent( // place when a DBRoot with a partial extent has been moved from one PM to // another. //------------------------------------------------------------------------------ -int ColumnInfo::extendColumnOldExtent( - uint16_t dbRootNext, - uint32_t partitionNext, - uint16_t segmentNext, - HWM hwmNext ) +int ColumnInfo::extendColumnOldExtent(uint16_t dbRootNext, uint32_t partitionNext, uint16_t segmentNext, + HWM hwmNext) { - const unsigned int BLKS_PER_EXTENT = - (fRowsPerExtent * column.width) / BYTE_PER_BLOCK; - HWM hwmNextExtentBoundary = hwmNext; + const unsigned int BLKS_PER_EXTENT = (fRowsPerExtent * column.width) / BYTE_PER_BLOCK; + HWM hwmNextExtentBoundary = hwmNext; - // Round up HWM to the end of the current extent - unsigned int nBlks = hwmNext + 1; - unsigned int nRem = nBlks % BLKS_PER_EXTENT; + // Round up HWM to the end of the current extent + unsigned int nBlks = hwmNext + 1; + unsigned int nRem = nBlks % BLKS_PER_EXTENT; - if (nRem > 0) - hwmNextExtentBoundary = nBlks - nRem + BLKS_PER_EXTENT - 1; - else - hwmNextExtentBoundary = nBlks - 1; + if (nRem > 0) + hwmNextExtentBoundary = nBlks - nRem + BLKS_PER_EXTENT - 1; + else + hwmNextExtentBoundary = nBlks - 1; + std::ostringstream oss; + oss << "Padding partial extent to extent boundary in OID-" << curCol.dataFile.fid << "; DBRoot-" + << dbRootNext << "; part-" << partitionNext << "; seg-" << segmentNext << "; oldhwm-" << hwmNext + << "; newhwm-" << hwmNextExtentBoundary; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + long long fileSizeBytes; + int rc = colOp->getFileSize(curCol.dataFile.fid, dbRootNext, partitionNext, segmentNext, fileSizeBytes); + + if (rc != NO_ERROR) + { std::ostringstream oss; - oss << "Padding partial extent to extent boundary in OID-" << - curCol.dataFile.fid << - "; DBRoot-" << dbRootNext << - "; part-" << partitionNext << - "; seg-" << segmentNext << - "; oldhwm-" << hwmNext << - "; newhwm-" << hwmNextExtentBoundary; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + oss << "extendColumnOldExtent: error padding partial extent for " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; hwm-" + << curCol.dataFile.hwm; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - long long fileSizeBytes; - int rc = colOp->getFileSize( curCol.dataFile.fid, - dbRootNext, partitionNext, segmentNext, fileSizeBytes); + return rc; + } + + curCol.dataFile.pFile = 0; + curCol.dataFile.fDbRoot = dbRootNext; + curCol.dataFile.fPartition = partitionNext; + curCol.dataFile.fSegment = segmentNext; + curCol.dataFile.hwm = hwmNextExtentBoundary; + curCol.dataFile.fSegFileName.clear(); + + // See if we have an abbreviated extent that needs to be expanded on disk + if (fileSizeBytes == (long long)INITIAL_EXTENT_ROWS_TO_DISK * column.width) + { + std::string segFile; + + // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag + IDBDataFile* pFile = colOp->openFile(curCol, dbRootNext, partitionNext, segmentNext, segFile, true); + + if (!pFile) + { + std::ostringstream oss; + rc = ERR_FILE_OPEN; + oss << "extendColumnOldExtent: error padding partial extent for " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; hwm-" + << curCol.dataFile.hwm; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + return rc; + } + + rc = colOp->expandAbbrevColumnExtent(pFile, dbRootNext, column.emptyVal, column.width, column.dataType); if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "extendColumnOldExtent: error padding partial extent for " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; hwm-" << curCol.dataFile.hwm; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); + std::ostringstream oss; + oss << "extendColumnOldExtent: error padding partial extent for " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; hwm-" + << curCol.dataFile.hwm; + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return rc; + colOp->closeFile(pFile); + + return rc; } - curCol.dataFile.pFile = 0; - curCol.dataFile.fDbRoot = dbRootNext; - curCol.dataFile.fPartition = partitionNext; - curCol.dataFile.fSegment = segmentNext; - curCol.dataFile.hwm = hwmNextExtentBoundary; - curCol.dataFile.fSegFileName.clear(); + colOp->closeFile(pFile); + } - // See if we have an abbreviated extent that needs to be expanded on disk - if (fileSizeBytes == (long long)INITIAL_EXTENT_ROWS_TO_DISK * column.width) - { - std::string segFile; + addToSegFileList(curCol.dataFile, hwmNextExtentBoundary); - // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag - IDBDataFile* pFile = colOp->openFile( curCol, - dbRootNext, partitionNext, segmentNext, segFile, true ); - - if ( !pFile ) - { - std::ostringstream oss; - rc = ERR_FILE_OPEN; - oss << "extendColumnOldExtent: error padding partial extent for " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; hwm-" << curCol.dataFile.hwm; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - rc = colOp->expandAbbrevColumnExtent( pFile, dbRootNext, - column.emptyVal, column.width, - column.dataType ); - - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "extendColumnOldExtent: error padding partial extent for " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; hwm-" << curCol.dataFile.hwm; - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - - colOp->closeFile( pFile ); - - return rc; - } - - colOp->closeFile( pFile ); - } - - addToSegFileList( curCol.dataFile, hwmNextExtentBoundary ); - - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -936,51 +816,46 @@ int ColumnInfo::extendColumnOldExtent( // dmc-could optimize later by changing fSegFileUpdateList from a vector // to a map or hashtable with a key consisting of partition and segment. //------------------------------------------------------------------------------ -void ColumnInfo::addToSegFileList( File& dataFile, HWM hwm ) +void ColumnInfo::addToSegFileList(File& dataFile, HWM hwm) { - bool foundFlag = false; + bool foundFlag = false; - for (unsigned int i = 0; i < fSegFileUpdateList.size(); i++) + for (unsigned int i = 0; i < fSegFileUpdateList.size(); i++) + { + if ((fSegFileUpdateList[i].fPartition == dataFile.fPartition) && + (fSegFileUpdateList[i].fSegment == dataFile.fSegment)) { - if ((fSegFileUpdateList[i].fPartition == dataFile.fPartition) && - (fSegFileUpdateList[i].fSegment == dataFile.fSegment)) - { - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss3; - oss3 << "Updating HWM list" - "; column OID-" << dataFile.fid << - "; DBRoot-" << dataFile.fDbRoot << - "; part-" << dataFile.fPartition << - "; seg-" << dataFile.fSegment << - "; oldhwm-" << fSegFileUpdateList[i].hwm << - "; newhwm-" << hwm; - fLog->logMsg( oss3.str(), MSGLVL_INFO2 ); - } + if (fLog->isDebug(DEBUG_1)) + { + std::ostringstream oss3; + oss3 << "Updating HWM list" + "; column OID-" + << dataFile.fid << "; DBRoot-" << dataFile.fDbRoot << "; part-" << dataFile.fPartition + << "; seg-" << dataFile.fSegment << "; oldhwm-" << fSegFileUpdateList[i].hwm << "; newhwm-" + << hwm; + fLog->logMsg(oss3.str(), MSGLVL_INFO2); + } - fSegFileUpdateList[i].hwm = hwm; - foundFlag = true; - break; - } + fSegFileUpdateList[i].hwm = hwm; + foundFlag = true; + break; + } + } + + if (!foundFlag) + { + if (fLog->isDebug(DEBUG_1)) + { + std::ostringstream oss3; + oss3 << "Adding to HWM list" + << "; column OID-" << dataFile.fid << "; DBRoot-" << dataFile.fDbRoot << "; part-" + << dataFile.fPartition << "; seg-" << dataFile.fSegment << "; hwm-" << hwm; + fLog->logMsg(oss3.str(), MSGLVL_INFO2); } - if (!foundFlag) - { - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss3; - oss3 << "Adding to HWM list" << - "; column OID-" << dataFile.fid << - "; DBRoot-" << dataFile.fDbRoot << - "; part-" << dataFile.fPartition << - "; seg-" << dataFile.fSegment << - "; hwm-" << hwm; - fLog->logMsg( oss3.str(), MSGLVL_INFO2 ); - } - - dataFile.hwm = hwm; - fSegFileUpdateList.push_back( dataFile ); - } + dataFile.hwm = hwm; + fSegFileUpdateList.push_back(dataFile); + } } //------------------------------------------------------------------------------ @@ -989,26 +864,23 @@ void ColumnInfo::addToSegFileList( File& dataFile, HWM hwm ) //------------------------------------------------------------------------------ int ColumnInfo::resetFileOffsetsNewExtent(const char* /*hdr*/) { - setFileSize( curCol.dataFile.hwm, false ); - long long byteOffset = (long long)curCol.dataFile.hwm * - (long long)BYTE_PER_BLOCK; - fSizeWritten = byteOffset; - fSizeWrittenStart = fSizeWritten; - availFileSize = fileSize - fSizeWritten; + setFileSize(curCol.dataFile.hwm, false); + long long byteOffset = (long long)curCol.dataFile.hwm * (long long)BYTE_PER_BLOCK; + fSizeWritten = byteOffset; + fSizeWrittenStart = fSizeWritten; + availFileSize = fileSize - fSizeWritten; - // If we are adding an extent as part of preliminary block skipping, then - // we won't have a ColumnBufferManager object yet, but that's okay, because - // we are only adding the empty extent at this point. - if (fColBufferMgr) - { - RETURN_ON_ERROR( fColBufferMgr->setDbFile( - curCol.dataFile.pFile, curCol.dataFile.hwm, 0) ); + // If we are adding an extent as part of preliminary block skipping, then + // we won't have a ColumnBufferManager object yet, but that's okay, because + // we are only adding the empty extent at this point. + if (fColBufferMgr) + { + RETURN_ON_ERROR(fColBufferMgr->setDbFile(curCol.dataFile.pFile, curCol.dataFile.hwm, 0)); - RETURN_ON_ERROR( colOp->setFileOffset(curCol.dataFile.pFile, - byteOffset) ); - } + RETURN_ON_ERROR(colOp->setFileOffset(curCol.dataFile.pFile, byteOffset)); + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1017,32 +889,31 @@ int ColumnInfo::resetFileOffsetsNewExtent(const char* /*hdr*/) // For unabbreviated extents the "logical" file size is calculated by rounding // the hwm up to the nearest multiple of the extent size. //------------------------------------------------------------------------------ -void ColumnInfo::setFileSize( HWM hwm, int abbrevFlag ) +void ColumnInfo::setFileSize(HWM hwm, int abbrevFlag) { - // Must be an abbreviated extent if there is only 1 compressed chunk in - // the db file. Even a 1-byte column would have 2 4MB chunks for an 8M - // row column extent. - if (abbrevFlag) + // Must be an abbreviated extent if there is only 1 compressed chunk in + // the db file. Even a 1-byte column would have 2 4MB chunks for an 8M + // row column extent. + if (abbrevFlag) + { + fileSize = (INITIAL_EXTENT_ROWS_TO_DISK * curCol.colWidth); + } + else + { + const unsigned int ROWS_PER_EXTENT = fRowsPerExtent; + + long long nRows = ((long long)(hwm + 1) * (long long)BYTE_PER_BLOCK) / (long long)curCol.colWidth; + long long nRem = nRows % ROWS_PER_EXTENT; + + if (nRem == 0) { - fileSize = (INITIAL_EXTENT_ROWS_TO_DISK * curCol.colWidth); + fileSize = nRows * curCol.colWidth; } else { - const unsigned int ROWS_PER_EXTENT = fRowsPerExtent; - - long long nRows = ((long long)(hwm + 1) * (long long)BYTE_PER_BLOCK) / - (long long)curCol.colWidth; - long long nRem = nRows % ROWS_PER_EXTENT; - - if (nRem == 0) - { - fileSize = nRows * curCol.colWidth; - } - else - { - fileSize = (nRows - nRem + ROWS_PER_EXTENT) * curCol.colWidth; - } + fileSize = (nRows - nRem + ROWS_PER_EXTENT) * curCol.colWidth; } + } } //------------------------------------------------------------------------------ @@ -1052,29 +923,25 @@ void ColumnInfo::setFileSize( HWM hwm, int abbrevFlag ) // with an abbreviated extent that still needs to be expanded and filled, before // we start adding new extents. //------------------------------------------------------------------------------ -void ColumnInfo::setAbbrevExtentCheck( ) +void ColumnInfo::setAbbrevExtentCheck() { -// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((curCol.dataFile.fPartition == 0) && - (curCol.dataFile.fSegment == 0)) + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((curCol.dataFile.fPartition == 0) && (curCol.dataFile.fSegment == 0)) + { + if (fileSize == (INITIAL_EXTENT_ROWS_TO_DISK * curCol.colWidth)) { - if (fileSize == (INITIAL_EXTENT_ROWS_TO_DISK * curCol.colWidth)) - { - fLoadingAbbreviatedExtent = true; + fLoadingAbbreviatedExtent = true; - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss; - oss << "Importing into abbreviated extent, column OID-" << - curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; fileSize-" << fileSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - } + if (fLog->isDebug(DEBUG_1)) + { + std::ostringstream oss; + oss << "Importing into abbreviated extent, column OID-" << curCol.dataFile.fid << "; DBRoot-" + << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; fileSize-" << fileSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } } + } } //------------------------------------------------------------------------------ @@ -1084,77 +951,70 @@ void ColumnInfo::setAbbrevExtentCheck( ) // upon return from this function; else the file will be positioned at the end // of the file. //------------------------------------------------------------------------------ -int ColumnInfo::expandAbbrevExtent( bool bRetainFilePos ) +int ColumnInfo::expandAbbrevExtent(bool bRetainFilePos) { - if (fLoadingAbbreviatedExtent) + if (fLoadingAbbreviatedExtent) + { + off64_t oldOffset = 0; + + if (bRetainFilePos) { - off64_t oldOffset = 0; - - if (bRetainFilePos) - { - oldOffset = curCol.dataFile.pFile->tell(); - } - - colOp->setFileOffset( curCol.dataFile.pFile, 0, SEEK_END ); - - std::ostringstream oss; - oss << "Expanding first extent to column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; file-" << curCol.dataFile.fSegFileName; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - - int rc = colOp->expandAbbrevExtent ( curCol ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "expandAbbrevExtent: error expanding extent for " << - "OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return rc; - } - - // Update available file size to reflect disk space added by expanding - // the extent. - long long fileSizeBeforeExpand = fileSize; - setFileSize( (fileSizeBeforeExpand / BYTE_PER_BLOCK), false ); - availFileSize += (fileSize - fileSizeBeforeExpand); - - // Restore offset back to where we were before expanding the extent - if (bRetainFilePos) - { - rc = colOp->setFileOffset(curCol.dataFile.pFile, oldOffset, SEEK_SET); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "expandAbbrevExtent: error seeking to new extent for " << - "OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return rc; - } - } - - // We only use abbreviated extents for the very first extent. So after - // expanding a col's abbreviated extent, we should disable this check. - fLoadingAbbreviatedExtent = false; + oldOffset = curCol.dataFile.pFile->tell(); } - return NO_ERROR; + colOp->setFileOffset(curCol.dataFile.pFile, 0, SEEK_END); + + std::ostringstream oss; + oss << "Expanding first extent to column OID-" << curCol.dataFile.fid << "; DBRoot-" + << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; file-" << curCol.dataFile.fSegFileName; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + int rc = colOp->expandAbbrevExtent(curCol); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "expandAbbrevExtent: error expanding extent for " + << "OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + return rc; + } + + // Update available file size to reflect disk space added by expanding + // the extent. + long long fileSizeBeforeExpand = fileSize; + setFileSize((fileSizeBeforeExpand / BYTE_PER_BLOCK), false); + availFileSize += (fileSize - fileSizeBeforeExpand); + + // Restore offset back to where we were before expanding the extent + if (bRetainFilePos) + { + rc = colOp->setFileOffset(curCol.dataFile.pFile, oldOffset, SEEK_SET); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "expandAbbrevExtent: error seeking to new extent for " + << "OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + return rc; + } + } + + // We only use abbreviated extents for the very first extent. So after + // expanding a col's abbreviated extent, we should disable this check. + fLoadingAbbreviatedExtent = false; + } + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1162,13 +1022,13 @@ int ColumnInfo::expandAbbrevExtent( bool bRetainFilePos ) //------------------------------------------------------------------------------ int ColumnInfo::closeColumnFile(bool /*bCompletingExtent*/, bool /*bAbort*/) { - if ( curCol.dataFile.pFile ) - { - colOp->closeFile( curCol.dataFile.pFile ); - curCol.dataFile.pFile = 0; - } + if (curCol.dataFile.pFile) + { + colOp->closeFile(curCol.dataFile.pFile); + curCol.dataFile.pFile = 0; + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1176,56 +1036,54 @@ int ColumnInfo::closeColumnFile(bool /*bCompletingExtent*/, bool /*bAbort*/) // is crossing an extent boundary, so that we can accurately track the min/max // for each extent as the Read buffers are parsed. //------------------------------------------------------------------------------ -void ColumnInfo::lastInputRowInExtentInit( bool bIsNewExtent ) +void ColumnInfo::lastInputRowInExtentInit(bool bIsNewExtent) { - // Reworked initial block skipping for compression: - const unsigned int ROWS_PER_EXTENT = fRowsPerExtent; - RID numRowsLeftInExtent = 0; - RID numRowsWritten = fSizeWritten / curCol.colWidth; + // Reworked initial block skipping for compression: + const unsigned int ROWS_PER_EXTENT = fRowsPerExtent; + RID numRowsLeftInExtent = 0; + RID numRowsWritten = fSizeWritten / curCol.colWidth; - if ((numRowsWritten % ROWS_PER_EXTENT) != 0) - numRowsLeftInExtent = ROWS_PER_EXTENT - - (numRowsWritten % ROWS_PER_EXTENT); + if ((numRowsWritten % ROWS_PER_EXTENT) != 0) + numRowsLeftInExtent = ROWS_PER_EXTENT - (numRowsWritten % ROWS_PER_EXTENT); - bool bRoomToAddToOriginalExtent = true; + bool bRoomToAddToOriginalExtent = true; - if (fSizeWritten > 0) + if (fSizeWritten > 0) + { + // Handle edge case; if numRowsLeftInExtent comes out to be 0, then + // current extent is full. In this case we first bump up row count + // by a full extent before we subtract by 1 to get the last row number + // in extent. + if (numRowsLeftInExtent == 0) { - // Handle edge case; if numRowsLeftInExtent comes out to be 0, then - // current extent is full. In this case we first bump up row count - // by a full extent before we subtract by 1 to get the last row number - // in extent. - if (numRowsLeftInExtent == 0) - { - numRowsLeftInExtent = ROWS_PER_EXTENT;; - bRoomToAddToOriginalExtent = false; - } - } - else - { - // Starting new file with empty extent, so set row count to full extent - numRowsLeftInExtent = ROWS_PER_EXTENT; + numRowsLeftInExtent = ROWS_PER_EXTENT; + ; + bRoomToAddToOriginalExtent = false; } + } + else + { + // Starting new file with empty extent, so set row count to full extent + numRowsLeftInExtent = ROWS_PER_EXTENT; + } - fLastInputRowInCurrentExtent = numRowsLeftInExtent - 1; + fLastInputRowInCurrentExtent = numRowsLeftInExtent - 1; - // If we have a pre-existing extent that we are going to add rows to, - // then we need to add that extent to our ColExtInf object, so that we - // can update the CP min/max at the end of the bulk load job. - if ( bRoomToAddToOriginalExtent ) - { - fColExtInf->addFirstEntry(fLastInputRowInCurrentExtent, - fSavedLbid, - bIsNewExtent ); - } + // If we have a pre-existing extent that we are going to add rows to, + // then we need to add that extent to our ColExtInf object, so that we + // can update the CP min/max at the end of the bulk load job. + if (bRoomToAddToOriginalExtent) + { + fColExtInf->addFirstEntry(fLastInputRowInCurrentExtent, fSavedLbid, bIsNewExtent); + } } //------------------------------------------------------------------------------ // Increment fLastRIDInExtent to the end of the next extent. //------------------------------------------------------------------------------ -void ColumnInfo::lastInputRowInExtentInc( ) +void ColumnInfo::lastInputRowInExtentInc() { - fLastInputRowInCurrentExtent += fRowsPerExtent; + fLastInputRowInCurrentExtent += fRowsPerExtent; } //------------------------------------------------------------------------------ @@ -1233,65 +1091,64 @@ void ColumnInfo::lastInputRowInExtentInc( ) // segment file, and corresponding dictionary store file (if applicable). Also // clears memory taken up by this ColumnInfo object. //------------------------------------------------------------------------------ -int ColumnInfo::finishParsing( ) +int ColumnInfo::finishParsing() { - int rc = NO_ERROR; + int rc = NO_ERROR; - // Close the dctnry file handle. - if (fStore) - { - rc = closeDctnryStore(false); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "finishParsing: close dictionary file error with column " << - column.colName << "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR); - return rc; - } - } - - // We don't need the mutex to protect against concurrent access by other - // threads, since by the time we get to this point, this is the last - // thread working on this column. But, we use the mutex to insure that - // we see the latest state that may have been set by another parsing thread - // working with the same column. - boost::mutex::scoped_lock lock(fColMutex); - - // Force the flushing of remaining data in the output buffer - if (fColBufferMgr) - { - rc = fColBufferMgr->flush( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "finishParsing: flush error with column " << column.colName << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR); - return rc; - } - } - - // Close the column file - rc = closeColumnFile(false, false); + // Close the dctnry file handle. + if (fStore) + { + rc = closeDctnryStore(false); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "finishParsing: close column file error with column " << - column.colName << "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR); - return rc; + WErrorCodes ec; + std::ostringstream oss; + oss << "finishParsing: close dictionary file error with column " << column.colName << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } + } - clearMemory(); + // We don't need the mutex to protect against concurrent access by other + // threads, since by the time we get to this point, this is the last + // thread working on this column. But, we use the mutex to insure that + // we see the latest state that may have been set by another parsing thread + // working with the same column. + boost::mutex::scoped_lock lock(fColMutex); - return NO_ERROR; + // Force the flushing of remaining data in the output buffer + if (fColBufferMgr) + { + rc = fColBufferMgr->flush(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "finishParsing: flush error with column " << column.colName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + } + + // Close the column file + rc = closeColumnFile(false, false); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "finishParsing: close column file error with column " << column.colName << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + clearMemory(); + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1306,103 +1163,102 @@ int ColumnInfo::finishParsing( ) // date values in fSegFileUpdateList, fSizeWritten, etc which may have been // set by another parsing thread. //------------------------------------------------------------------------------ -void ColumnInfo::getBRMUpdateInfo( BRMReporter& brmReporter ) +void ColumnInfo::getBRMUpdateInfo(BRMReporter& brmReporter) { - boost::mutex::scoped_lock lock(fColMutex); - // Useful for debugging - //printCPInfo(column); + boost::mutex::scoped_lock lock(fColMutex); + // Useful for debugging + // printCPInfo(column); - int entriesAdded = getHWMInfoForBRM( brmReporter ); + int entriesAdded = getHWMInfoForBRM(brmReporter); - // If we added any rows (HWM update count > 0), then update corresponding CP - if (entriesAdded > 0) - getCPInfoForBRM( brmReporter ); + // If we added any rows (HWM update count > 0), then update corresponding CP + if (entriesAdded > 0) + getCPInfoForBRM(brmReporter); } //------------------------------------------------------------------------------ // Get updated Casual Partition (CP) information for BRM for this column at EOJ. //------------------------------------------------------------------------------ -void ColumnInfo::getCPInfoForBRM( BRMReporter& brmReporter ) +void ColumnInfo::getCPInfoForBRM(BRMReporter& brmReporter) { - fColExtInf->getCPInfoForBRM(column, brmReporter); + fColExtInf->getCPInfoForBRM(column, brmReporter); } //------------------------------------------------------------------------------ // Get updated HWM information for BRM for this column at EOJ. // Returns count of the number of HWM entries added to the BRMReporter. //------------------------------------------------------------------------------ -int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter ) +int ColumnInfo::getHWMInfoForBRM(BRMReporter& brmReporter) { - //..If we wrote out any data to the last segment file, then - // update HWM for the current (last) segment file we were writing to. + //..If we wrote out any data to the last segment file, then + // update HWM for the current (last) segment file we were writing to. - //Bug1374 - Update HWM when data added to file - if ( fSizeWritten > fSizeWrittenStart ) + // Bug1374 - Update HWM when data added to file + if (fSizeWritten > fSizeWrittenStart) + { + // Bug1372. + HWM hwm = (fSizeWritten - 1) / BYTE_PER_BLOCK; + + addToSegFileList(curCol.dataFile, hwm); + } + + int entriesAdded = 0; + + //..Update HWM for each segment file we touched, including the last one + for (unsigned int iseg = 0; iseg < fSegFileUpdateList.size(); iseg++) + { + // Log for now; may control with debug flag later + // if (fLog->isDebug( DEBUG_1 )) { - //Bug1372. - HWM hwm = (fSizeWritten - 1) / BYTE_PER_BLOCK; + std::ostringstream oss; + oss << "Saving HWM update for OID-" << fSegFileUpdateList[iseg].fid << "; hwm-" + << fSegFileUpdateList[iseg].hwm << "; DBRoot-" << fSegFileUpdateList[iseg].fDbRoot << "; partition-" + << fSegFileUpdateList[iseg].fPartition << "; segment-" << fSegFileUpdateList[iseg].fSegment; - addToSegFileList( curCol.dataFile, hwm ); + fLog->logMsg(oss.str(), MSGLVL_INFO2); } - int entriesAdded = 0; + BRM::BulkSetHWMArg hwmArg; + hwmArg.oid = fSegFileUpdateList[iseg].fid; + hwmArg.partNum = fSegFileUpdateList[iseg].fPartition; + hwmArg.segNum = fSegFileUpdateList[iseg].fSegment; + hwmArg.hwm = fSegFileUpdateList[iseg].hwm; + brmReporter.addToHWMInfo(hwmArg); - //..Update HWM for each segment file we touched, including the last one - for (unsigned int iseg = 0; - iseg < fSegFileUpdateList.size(); iseg++) + // Save list of modified db column files + BRM::FileInfo aFile; + aFile.oid = fSegFileUpdateList[iseg].fid; + aFile.partitionNum = fSegFileUpdateList[iseg].fPartition; + aFile.segmentNum = fSegFileUpdateList[iseg].fSegment; + aFile.dbRoot = fSegFileUpdateList[iseg].fDbRoot; + aFile.compType = curCol.compressionType; + brmReporter.addToFileInfo(aFile); + + // Save list of corresponding modified db dictionary store files + if (column.colType == COL_TYPE_DICT) { - // Log for now; may control with debug flag later - //if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss; - oss << "Saving HWM update for OID-" << - fSegFileUpdateList[iseg].fid << - "; hwm-" << fSegFileUpdateList[iseg].hwm << - "; DBRoot-" << fSegFileUpdateList[iseg].fDbRoot << - "; partition-" << fSegFileUpdateList[iseg].fPartition << - "; segment-" << fSegFileUpdateList[iseg].fSegment; - - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - BRM::BulkSetHWMArg hwmArg; - hwmArg.oid = fSegFileUpdateList[iseg].fid; - hwmArg.partNum = fSegFileUpdateList[iseg].fPartition; - hwmArg.segNum = fSegFileUpdateList[iseg].fSegment; - hwmArg.hwm = fSegFileUpdateList[iseg].hwm; - brmReporter.addToHWMInfo( hwmArg ); - - // Save list of modified db column files - BRM::FileInfo aFile; - aFile.oid = fSegFileUpdateList[iseg].fid; - aFile.partitionNum = fSegFileUpdateList[iseg].fPartition; - aFile.segmentNum = fSegFileUpdateList[iseg].fSegment; - aFile.dbRoot = fSegFileUpdateList[iseg].fDbRoot; - aFile.compType = curCol.compressionType; - brmReporter.addToFileInfo( aFile ); - - // Save list of corresponding modified db dictionary store files - if (column.colType == COL_TYPE_DICT) - { - BRM::FileInfo dFile; - dFile.oid = column.dctnry.dctnryOid; - dFile.partitionNum = fSegFileUpdateList[iseg].fPartition; - dFile.segmentNum = fSegFileUpdateList[iseg].fSegment; - dFile.dbRoot = fSegFileUpdateList[iseg].fDbRoot; - dFile.compType = curCol.compressionType; - brmReporter.addToDctnryFileInfo( dFile ); - } - - entriesAdded++; + BRM::FileInfo dFile; + dFile.oid = column.dctnry.dctnryOid; + dFile.partitionNum = fSegFileUpdateList[iseg].fPartition; + dFile.segmentNum = fSegFileUpdateList[iseg].fSegment; + dFile.dbRoot = fSegFileUpdateList[iseg].fDbRoot; + dFile.compType = curCol.compressionType; + brmReporter.addToDctnryFileInfo(dFile); } - fSegFileUpdateList.clear(); // don't need vector anymore, so release memory + entriesAdded++; + } - return entriesAdded; + fSegFileUpdateList.clear(); // don't need vector anymore, so release memory + + return entriesAdded; } // Returns last updated LBID. -BRM::LBID_t ColumnInfo::getLastUpdatedLBID() const { return fLastUpdatedLbid; } +BRM::LBID_t ColumnInfo::getLastUpdatedLBID() const +{ + return fLastUpdatedLbid; +} //------------------------------------------------------------------------------ // Setup initial extent we will begin loading at start of import. @@ -1411,199 +1267,175 @@ BRM::LBID_t ColumnInfo::getLastUpdatedLBID() const { return fLastUpdatedLbid; } // set things up to point to the last block in the current extent. When we // start adding rows, we will automatically advance to the next extent. //------------------------------------------------------------------------------ -int ColumnInfo::setupInitialColumnExtent( - uint16_t dbRoot, // dbroot of starting extent - uint32_t partition, // partition number of starting extent - uint16_t segment, // segment number of starting extent - const std::string& tblName, // name of table containing this column - BRM::LBID_t lbid, // starting LBID for starting extent - HWM oldHwm, // original HWM - HWM hwm, // new projected HWM after block skipping - bool bSkippedToNewExtent, // blk skipping to next extent - bool bIsNewExtent ) // treat as new extent (for CP updates) +int ColumnInfo::setupInitialColumnExtent(uint16_t dbRoot, // dbroot of starting extent + uint32_t partition, // partition number of starting extent + uint16_t segment, // segment number of starting extent + const std::string& tblName, // name of table containing this column + BRM::LBID_t lbid, // starting LBID for starting extent + HWM oldHwm, // original HWM + HWM hwm, // new projected HWM after block skipping + bool bSkippedToNewExtent, // blk skipping to next extent + bool bIsNewExtent) // treat as new extent (for CP updates) { - // Init the ColumnInfo object - colOp->initColumn( curCol ); - colOp->setColParam( curCol, id, - column.width, - column.dataType, - column.weType, - column.mapOid, - column.compressionType, - dbRoot, partition, segment ); - colOp->findTypeHandler(column.width, column.dataType); + // Init the ColumnInfo object + colOp->initColumn(curCol); + colOp->setColParam(curCol, id, column.width, column.dataType, column.weType, column.mapOid, + column.compressionType, dbRoot, partition, segment); + colOp->findTypeHandler(column.width, column.dataType); - // Open the column file - if (!colOp->exists(column.mapOid, dbRoot, partition, segment) ) - { - std::ostringstream oss; - oss << "Column file does not exist for OID-" << column.mapOid << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment; - fLog->logMsg( oss.str(), ERR_FILE_NOT_EXIST, MSGLVL_ERROR ); - return ERR_FILE_NOT_EXIST; - } + // Open the column file + if (!colOp->exists(column.mapOid, dbRoot, partition, segment)) + { + std::ostringstream oss; + oss << "Column file does not exist for OID-" << column.mapOid << "; DBRoot-" << dbRoot << "; partition-" + << partition << "; segment-" << segment; + fLog->logMsg(oss.str(), ERR_FILE_NOT_EXIST, MSGLVL_ERROR); + return ERR_FILE_NOT_EXIST; + } - std::string segFile; - bool useTmpSuffix = false; + std::string segFile; + bool useTmpSuffix = false; - if (!bIsNewExtent) - useTmpSuffix = true; + if (!bIsNewExtent) + useTmpSuffix = true; - // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag - int rc = colOp->openColumnFile( curCol, segFile, useTmpSuffix ); + // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag + int rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error opening column file for OID-" << column.mapOid << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; filename-" << segFile << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR ); - return ERR_FILE_OPEN; - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error opening column file for OID-" << column.mapOid << "; DBRoot-" << dbRoot << "; partition-" + << partition << "; segment-" << segment << "; filename-" << segFile << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); + return ERR_FILE_OPEN; + } - std::ostringstream oss1; - oss1 << "Initializing import: " << - "Table-" << tblName << - "; Col-" << column.colName; + std::ostringstream oss1; + oss1 << "Initializing import: " + << "Table-" << tblName << "; Col-" << column.colName; - if (curCol.compressionType) - oss1 << " (compressed)"; + if (curCol.compressionType) + oss1 << " (compressed)"; - oss1 << "; OID-" << column.mapOid << - "; hwm-" << hwm; + oss1 << "; OID-" << column.mapOid << "; hwm-" << hwm; - if (bSkippedToNewExtent) - oss1 << " (full; load into next extent)"; + if (bSkippedToNewExtent) + oss1 << " (full; load into next extent)"; - oss1 << "; file-" << curCol.dataFile.fSegFileName; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); + oss1 << "; file-" << curCol.dataFile.fSegFileName; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); - if (column.colType == COL_TYPE_DICT) - { - RETURN_ON_ERROR( openDctnryStore( true ) ); - } + if (column.colType == COL_TYPE_DICT) + { + RETURN_ON_ERROR(openDctnryStore(true)); + } - fSavedLbid = lbid; - fLastUpdatedLbid = lbid; + fSavedLbid = lbid; + fLastUpdatedLbid = lbid; - if (bSkippedToNewExtent) - oldHwm = hwm; + if (bSkippedToNewExtent) + oldHwm = hwm; - rc = setupInitialColumnFile(oldHwm, hwm); + rc = setupInitialColumnFile(oldHwm, hwm); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error reading/positioning column file for OID-" << - column.mapOid << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; filename-" << segFile << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error reading/positioning column file for OID-" << column.mapOid << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; filename-" << segFile << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } - // Reworked initial block skipping for compression: - // Block skipping is causing us to wrap up this extent. We consider - // the current extent to be full, so we "pretend" to fill out the - // last block by adding 8192 bytes to the bytes written count. - // This will help trigger the addition of a new extent when we - // try to store the first section of rows to the db. - if (bSkippedToNewExtent) - { - updateBytesWrittenCounts( BYTE_PER_BLOCK ); - fSizeWrittenStart = fSizeWritten; - } + // Reworked initial block skipping for compression: + // Block skipping is causing us to wrap up this extent. We consider + // the current extent to be full, so we "pretend" to fill out the + // last block by adding 8192 bytes to the bytes written count. + // This will help trigger the addition of a new extent when we + // try to store the first section of rows to the db. + if (bSkippedToNewExtent) + { + updateBytesWrittenCounts(BYTE_PER_BLOCK); + fSizeWrittenStart = fSizeWritten; + } - // Reworked initial block skipping for compression: - // This initializes CP stats for first extent regardless of whether - // we end up adding rows to this extent, or initial block skipping - // ultimately causes us to start with a new extent. - lastInputRowInExtentInit( bIsNewExtent ); + // Reworked initial block skipping for compression: + // This initializes CP stats for first extent regardless of whether + // we end up adding rows to this extent, or initial block skipping + // ultimately causes us to start with a new extent. + lastInputRowInExtentInit(bIsNewExtent); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ // Prepare the initial column segment file for import. //------------------------------------------------------------------------------ -int ColumnInfo::setupInitialColumnFile( HWM oldHwm, HWM hwm ) +int ColumnInfo::setupInitialColumnFile(HWM oldHwm, HWM hwm) { - // Initialize the output buffer manager for the column. - if (column.colType == COL_TYPE_DICT) + // Initialize the output buffer manager for the column. + if (column.colType == COL_TYPE_DICT) + { + fColBufferMgr = new ColumnBufferManagerDctnry(this, 8, fLog, 0); + } + else + { + fColBufferMgr = new ColumnBufferManager(this, column.width, fLog, 0); + } + + RETURN_ON_ERROR(fColBufferMgr->setDbFile(curCol.dataFile.pFile, hwm, 0)); + + RETURN_ON_ERROR(colOp->getFileSize(curCol.dataFile.pFile, fileSize)); + + // See if dealing with abbreviated extent that will need expanding. + // This only applies to the first extent of the first segment file. + setAbbrevExtentCheck(); + + // If we are dealing with initial extent, see if block skipping has + // exceeded disk allocation, in which case we expand to a full extent. + if (isAbbrevExtent()) + { + unsigned int numBlksForFirstExtent = (INITIAL_EXTENT_ROWS_TO_DISK * column.width) / BYTE_PER_BLOCK; + + if (((oldHwm + 1) <= numBlksForFirstExtent) && ((hwm + 1) > numBlksForFirstExtent)) { - fColBufferMgr = new ColumnBufferManagerDctnry(this, 8, fLog, 0); - } - else - { - fColBufferMgr = new ColumnBufferManager(this, column.width, fLog, 0); + RETURN_ON_ERROR(expandAbbrevExtent(false)); } + } - RETURN_ON_ERROR( fColBufferMgr->setDbFile(curCol.dataFile.pFile, hwm, 0) ); + // Seek till the HWM lbid. + // Store the current allocated file size in availFileSize. + long long byteOffset = (long long)hwm * (long long)BYTE_PER_BLOCK; + RETURN_ON_ERROR(colOp->setFileOffset(curCol.dataFile.pFile, byteOffset)); - RETURN_ON_ERROR( colOp->getFileSize(curCol.dataFile.pFile, fileSize) ); + fSizeWritten = byteOffset; + fSizeWrittenStart = fSizeWritten; + availFileSize = fileSize - fSizeWritten; - // See if dealing with abbreviated extent that will need expanding. - // This only applies to the first extent of the first segment file. - setAbbrevExtentCheck(); + if (fLog->isDebug(DEBUG_1)) + { + std::ostringstream oss; + oss << "Init raw data offsets in column file OID-" << curCol.dataFile.fid << "; DBRoot-" + << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; begByte-" << fSizeWritten << "; endByte-" << fileSize + << "; freeBytes-" << availFileSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - // If we are dealing with initial extent, see if block skipping has - // exceeded disk allocation, in which case we expand to a full extent. - if (isAbbrevExtent()) - { - unsigned int numBlksForFirstExtent = - (INITIAL_EXTENT_ROWS_TO_DISK * column.width) / BYTE_PER_BLOCK; - - if ( ((oldHwm + 1) <= numBlksForFirstExtent) && - ((hwm + 1 ) > numBlksForFirstExtent) ) - { - RETURN_ON_ERROR( expandAbbrevExtent(false) ); - } - } - - // Seek till the HWM lbid. - // Store the current allocated file size in availFileSize. - long long byteOffset = (long long)hwm * (long long)BYTE_PER_BLOCK; - RETURN_ON_ERROR( colOp->setFileOffset(curCol.dataFile.pFile, byteOffset) ); - - fSizeWritten = byteOffset; - fSizeWrittenStart = fSizeWritten; - availFileSize = fileSize - fSizeWritten; - - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss; - oss << "Init raw data offsets in column file OID-" << - curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; begByte-" << fSizeWritten << - "; endByte-" << fileSize << - "; freeBytes-" << availFileSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ // Update the number of bytes in the file, and the free space still remaining. //------------------------------------------------------------------------------ -void ColumnInfo::updateBytesWrittenCounts( unsigned int numBytesWritten ) +void ColumnInfo::updateBytesWrittenCounts(unsigned int numBytesWritten) { - availFileSize = availFileSize - numBytesWritten; - fSizeWritten = fSizeWritten + numBytesWritten; + availFileSize = availFileSize - numBytesWritten; + fSizeWritten = fSizeWritten + numBytesWritten; } //------------------------------------------------------------------------------ @@ -1612,43 +1444,43 @@ void ColumnInfo::updateBytesWrittenCounts( unsigned int numBytesWritten ) //------------------------------------------------------------------------------ bool ColumnInfo::isFileComplete() const { - if ((fSizeWritten / column.width) >= fMaxNumRowsPerSegFile) - return true; + if ((fSizeWritten / column.width) >= fMaxNumRowsPerSegFile) + return true; - return false; + return false; } //------------------------------------------------------------------------------ // Initialize last used auto-increment value from the current "next" // auto-increment value taken from the system catalog (or BRM). //------------------------------------------------------------------------------ -int ColumnInfo::initAutoInc( const std::string& fullTableName ) +int ColumnInfo::initAutoInc(const std::string& fullTableName) { - int rc = fAutoIncMgr->init( fullTableName, this ); + int rc = fAutoIncMgr->init(fullTableName, this); - return rc; + return rc; } //------------------------------------------------------------------------------ // Reserves the requested number of auto-increment numbers (autoIncCount). // The starting value of the reserved block of numbers is returned in nextValue. //------------------------------------------------------------------------------ -int ColumnInfo::reserveAutoIncNums(uint32_t autoIncCount, uint64_t& nextValue ) +int ColumnInfo::reserveAutoIncNums(uint32_t autoIncCount, uint64_t& nextValue) { - int rc = fAutoIncMgr->reserveNextRange( autoIncCount, nextValue ); + int rc = fAutoIncMgr->reserveNextRange(autoIncCount, nextValue); - return rc; + return rc; } //------------------------------------------------------------------------------ // Finished using auto-increment. Current value can be committed back to the // system catalog (or BRM). //------------------------------------------------------------------------------ -int ColumnInfo::finishAutoInc( ) +int ColumnInfo::finishAutoInc() { - int rc = fAutoIncMgr->finish( ); + int rc = fAutoIncMgr->finish(); - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -1661,168 +1493,137 @@ int ColumnInfo::finishAutoInc( ) // date values for dbroot, partition, segment, and HWM which may have been // set by another parsing thread. //------------------------------------------------------------------------------ -void ColumnInfo::getSegFileInfo( DBRootExtentInfo& fileInfo ) +void ColumnInfo::getSegFileInfo(DBRootExtentInfo& fileInfo) { - boost::mutex::scoped_lock lock(fColMutex); - fileInfo.fDbRoot = curCol.dataFile.fDbRoot; - fileInfo.fPartition = curCol.dataFile.fPartition; - fileInfo.fSegment = curCol.dataFile.fSegment; + boost::mutex::scoped_lock lock(fColMutex); + fileInfo.fDbRoot = curCol.dataFile.fDbRoot; + fileInfo.fPartition = curCol.dataFile.fPartition; + fileInfo.fSegment = curCol.dataFile.fSegment; - if (fSizeWritten > 0) - fileInfo.fLocalHwm = (fSizeWritten - 1) / BYTE_PER_BLOCK; - else - fileInfo.fLocalHwm = 0; + if (fSizeWritten > 0) + fileInfo.fLocalHwm = (fSizeWritten - 1) / BYTE_PER_BLOCK; + else + fileInfo.fLocalHwm = 0; } //------------------------------------------------------------------------------ // Open a new or existing Dictionary store file based on the DBRoot, // partition, and segment settings in curCol.dataFile. //------------------------------------------------------------------------------ -int ColumnInfo::openDctnryStore( bool bMustExist ) +int ColumnInfo::openDctnryStore(bool bMustExist) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if ( column.dctnry.fCompressionType != 0) + if (column.dctnry.fCompressionType != 0) + { + DctnryCompress1* dctnryCompress1 = new DctnryCompress1(column.dctnry.fCompressionType); + dctnryCompress1->setMaxActiveChunkNum(1); + dctnryCompress1->setBulkFlag(true); + fStore = dctnryCompress1; + } + else + { + fStore = new DctnryCompress0; + } + + fStore->setLogger(fLog); + fStore->setColWidth(column.dctnryWidth); + fStore->setUIDGID(this); + + if (column.fWithDefault) + fStore->setDefault(column.fDefaultChr); + + fStore->setImportDataMode(fpTableInfo->getImportDataMode()); + + // If we are in the process of adding an extent to this column, + // and the extent we are adding is the first extent for the + // relevant column segment file, then the corresponding dictionary + // store file will not exist, in which case we must create + // the store file, else we open the applicable store file. + if ((bMustExist) || (colOp->exists(column.dctnry.dctnryOid, curCol.dataFile.fDbRoot, + curCol.dataFile.fPartition, curCol.dataFile.fSegment))) + { + // Save HWM chunk (for compressed files) if this seg file calls for it + // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag + bool useTmpSuffixDctnry = false; + RETURN_ON_ERROR(saveDctnryStoreHWMChunk(useTmpSuffixDctnry)); + + // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag + rc = fStore->openDctnry(column.dctnry.dctnryOid, curCol.dataFile.fDbRoot, curCol.dataFile.fPartition, + curCol.dataFile.fSegment, useTmpSuffixDctnry); + + if (rc != NO_ERROR) { - DctnryCompress1* dctnryCompress1 = - new DctnryCompress1(column.dctnry.fCompressionType); - dctnryCompress1->setMaxActiveChunkNum(1); - dctnryCompress1->setBulkFlag(true); - fStore = dctnryCompress1; - } - else - { - fStore = new DctnryCompress0; + WErrorCodes ec; + std::ostringstream oss; + oss << "openDctnryStore: error opening existing store file for " + << "OID-" << column.dctnry.dctnryOid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; tmpFlag-" + << useTmpSuffixDctnry << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + // Ignore return code from closing file; already in error state + closeDctnryStore(true); // clean up loose ends + return rc; } - fStore->setLogger(fLog); - fStore->setColWidth( column.dctnryWidth ); - fStore->setUIDGID(this); + if (INVALID_LBID != fStore->getCurLbid()) + fDictBlocks.push_back(fStore->getCurLbid()); - if (column.fWithDefault) - fStore->setDefault( column.fDefaultChr ); + std::ostringstream oss; + oss << "Opening existing store file for " << column.colName << "; OID-" << column.dctnry.dctnryOid + << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; hwm-" << fStore->getHWM() << "; file-" << fStore->getFileName(); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + BRM::LBID_t startLbid; + rc = fStore->createDctnry(column.dctnry.dctnryOid, + column.dctnryWidth, //@bug 3313 - pass string col width + curCol.dataFile.fDbRoot, curCol.dataFile.fPartition, curCol.dataFile.fSegment, + startLbid); - fStore->setImportDataMode( fpTableInfo->getImportDataMode() ); - - // If we are in the process of adding an extent to this column, - // and the extent we are adding is the first extent for the - // relevant column segment file, then the corresponding dictionary - // store file will not exist, in which case we must create - // the store file, else we open the applicable store file. - if ( (bMustExist) || - (colOp->exists(column.dctnry.dctnryOid, - curCol.dataFile.fDbRoot, - curCol.dataFile.fPartition, - curCol.dataFile.fSegment)) ) + if (rc != NO_ERROR) { - // Save HWM chunk (for compressed files) if this seg file calls for it - // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag - bool useTmpSuffixDctnry = false; - RETURN_ON_ERROR( saveDctnryStoreHWMChunk( useTmpSuffixDctnry ) ); + WErrorCodes ec; + std::ostringstream oss; + oss << "openDctnryStore: error creating new store file for " + << "OID-" << column.dctnry.dctnryOid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag - rc = fStore->openDctnry( - column.dctnry.dctnryOid, - curCol.dataFile.fDbRoot, - curCol.dataFile.fPartition, - curCol.dataFile.fSegment, - useTmpSuffixDctnry ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "openDctnryStore: error opening existing store file for " << - "OID-" << column.dctnry.dctnryOid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; tmpFlag-" << useTmpSuffixDctnry << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - // Ignore return code from closing file; already in error state - closeDctnryStore(true); // clean up loose ends - return rc; - } - - if (INVALID_LBID != fStore->getCurLbid()) - fDictBlocks.push_back(fStore->getCurLbid()); - - std::ostringstream oss; - oss << "Opening existing store file for " << column.colName << - "; OID-" << column.dctnry.dctnryOid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; hwm-" << fStore->getHWM() << - "; file-" << fStore->getFileName(); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - BRM::LBID_t startLbid; - rc = fStore->createDctnry( - column.dctnry.dctnryOid, - column.dctnryWidth, //@bug 3313 - pass string col width - curCol.dataFile.fDbRoot, - curCol.dataFile.fPartition, - curCol.dataFile.fSegment, - startLbid); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "openDctnryStore: error creating new store file for " << - "OID-" << column.dctnry.dctnryOid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - - // Ignore return code from closing file; already in error state - closeDctnryStore(true); // clean up loose ends - return rc; - } - - rc = fStore->openDctnry( - column.dctnry.dctnryOid, - curCol.dataFile.fDbRoot, - curCol.dataFile.fPartition, - curCol.dataFile.fSegment, - false ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "openDctnryStore: error opening new store file for " << - "OID-" << column.dctnry.dctnryOid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - // Ignore return code from closing file; already in error state - closeDctnryStore(true); // clean up loose ends - return rc; - } - - std::ostringstream oss; - oss << "Opening new store file for " << column.colName << - "; OID-" << column.dctnry.dctnryOid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; file-" << fStore->getFileName(); - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + // Ignore return code from closing file; already in error state + closeDctnryStore(true); // clean up loose ends + return rc; } - return rc; + rc = fStore->openDctnry(column.dctnry.dctnryOid, curCol.dataFile.fDbRoot, curCol.dataFile.fPartition, + curCol.dataFile.fSegment, false); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "openDctnryStore: error opening new store file for " + << "OID-" << column.dctnry.dctnryOid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + + // Ignore return code from closing file; already in error state + closeDctnryStore(true); // clean up loose ends + return rc; + } + + std::ostringstream oss; + oss << "Opening new store file for " << column.colName << "; OID-" << column.dctnry.dctnryOid + << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; file-" << fStore->getFileName(); + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + return rc; } //------------------------------------------------------------------------------ @@ -1830,97 +1631,89 @@ int ColumnInfo::openDctnryStore( bool bMustExist ) //------------------------------------------------------------------------------ int ColumnInfo::closeDctnryStore(bool bAbort) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (fStore) + if (fStore) + { + if (bAbort) + rc = fStore->closeDctnryOnly(); + else + rc = fStore->closeDctnry(); + + if (rc != NO_ERROR) { - if (bAbort) - rc = fStore->closeDctnryOnly(); - else - rc = fStore->closeDctnry(); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "closeDctnryStore: error closing store file for " << - "OID-" << column.dctnry.dctnryOid << - "; file-" << fStore->getFileName() << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - } - - delete fStore; - fStore = 0; + WErrorCodes ec; + std::ostringstream oss; + oss << "closeDctnryStore: error closing store file for " + << "OID-" << column.dctnry.dctnryOid << "; file-" << fStore->getFileName() << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); } - return rc; + delete fStore; + fStore = 0; + } + + return rc; } //------------------------------------------------------------------------------ // Update dictionary store file with specified strings, and return the assigned // tokens (tokenbuf) to be stored in the corresponding column token file. //------------------------------------------------------------------------------ -int ColumnInfo::updateDctnryStore(char* buf, - ColPosPair** pos, - const int totalRow, - char* tokenBuf) +int ColumnInfo::updateDctnryStore(char* buf, ColPosPair** pos, const int totalRow, char* tokenBuf) { - long long truncCount = 0; // No. of rows with truncated values + long long truncCount = 0; // No. of rows with truncated values - // If this is a VARBINARY column; convert the ascii hex string into binary - // data and fix the length (it's now only half as long). - // Should be safe to modify pos and buf arrays outside a mutex, as no other - // thread should be accessing the strings from the same buffer, for this - // column. - // This only applies to default text mode. This step is bypassed for - // binary imports, because in that case, the data is already true binary. - if (((curCol.colType == WR_VARBINARY) || (curCol.colType == WR_BLOB)) && - (fpTableInfo->getImportDataMode() == IMPORT_DATA_TEXT)) + // If this is a VARBINARY column; convert the ascii hex string into binary + // data and fix the length (it's now only half as long). + // Should be safe to modify pos and buf arrays outside a mutex, as no other + // thread should be accessing the strings from the same buffer, for this + // column. + // This only applies to default text mode. This step is bypassed for + // binary imports, because in that case, the data is already true binary. + if (((curCol.colType == WR_VARBINARY) || (curCol.colType == WR_BLOB)) && + (fpTableInfo->getImportDataMode() == IMPORT_DATA_TEXT)) + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_COMPACT_VARBINARY); +#endif + + for (int i = 0; i < totalRow; i++) { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPACT_VARBINARY); -#endif - - for (int i = 0; i < totalRow; i++) - { - pos[i][id].offset = - compactVarBinary(buf + pos[i][id].start, pos[i][id].offset); - } - -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPACT_VARBINARY); -#endif + pos[i][id].offset = compactVarBinary(buf + pos[i][id].start, pos[i][id].offset); } #ifdef PROFILE - Stats::startParseEvent(WE_STATS_WAIT_TO_PARSE_DCT); + Stats::startParseEvent(WE_STATS_COMPACT_VARBINARY); #endif - boost::mutex::scoped_lock lock(fDictionaryMutex); + } + #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_TO_PARSE_DCT); + Stats::startParseEvent(WE_STATS_WAIT_TO_PARSE_DCT); +#endif + boost::mutex::scoped_lock lock(fDictionaryMutex); +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_WAIT_TO_PARSE_DCT); #endif - int rc = fStore->insertDctnry( buf, pos, totalRow, id, tokenBuf, truncCount ); + int rc = fStore->insertDctnry(buf, pos, totalRow, id, tokenBuf, truncCount); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "updateDctnryStore: error adding rows to store file for " << - "OID-" << column.dctnry.dctnryOid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return rc; - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "updateDctnryStore: error adding rows to store file for " + << "OID-" << column.dctnry.dctnryOid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + return rc; + } - incSaturatedCnt( truncCount ); + incSaturatedCnt(truncCount); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1929,62 +1722,47 @@ int ColumnInfo::updateDctnryStore(char* buf, // @bug 5572 - HDFS usage: add flag used to control *.tmp file usage int ColumnInfo::saveDctnryStoreHWMChunk(bool& needBackup) { - needBackup = false; - return NO_ERROR; + needBackup = false; + return NO_ERROR; } //------------------------------------------------------------------------------ // Truncate specified dictionary store file for this column. // Only applies to compressed columns. //------------------------------------------------------------------------------ -int ColumnInfo::truncateDctnryStore( - OID /*dctnryOid*/, uint16_t /*root*/, uint32_t /*pNum*/, uint16_t /*sNum*/) -const +int ColumnInfo::truncateDctnryStore(OID /*dctnryOid*/, uint16_t /*root*/, uint32_t /*pNum*/, + uint16_t /*sNum*/) const { - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ // utility to convert a Status enumeration to a string //------------------------------------------------------------------------------ /* static */ -void ColumnInfo::convertStatusToString( - WriteEngine::Status status, - std::string& statusString ) +void ColumnInfo::convertStatusToString(WriteEngine::Status status, std::string& statusString) { - static std::string statusStringParseComplete("PARSE_COMPLETE"); - static std::string statusStringReadComplete ("READ_COMPLETE"); - static std::string statusStringReadProgress ("READ_PROGRESS"); - static std::string statusStringNew ("NEW"); - static std::string statusStringErr ("ERR"); - static std::string statusStringUnknown ("OTHER"); + static std::string statusStringParseComplete("PARSE_COMPLETE"); + static std::string statusStringReadComplete("READ_COMPLETE"); + static std::string statusStringReadProgress("READ_PROGRESS"); + static std::string statusStringNew("NEW"); + static std::string statusStringErr("ERR"); + static std::string statusStringUnknown("OTHER"); - switch (status) - { - case PARSE_COMPLETE: - statusString = statusStringParseComplete; - break; + switch (status) + { + case PARSE_COMPLETE: statusString = statusStringParseComplete; break; - case READ_COMPLETE: - statusString = statusStringReadComplete; - break; + case READ_COMPLETE: statusString = statusStringReadComplete; break; - case READ_PROGRESS: - statusString = statusStringReadProgress; - break; + case READ_PROGRESS: statusString = statusStringReadProgress; break; - case NEW: - statusString = statusStringNew; - break; + case NEW: statusString = statusStringNew; break; - case ERR: - statusString = statusStringErr; - break; + case ERR: statusString = statusStringErr; break; - default: - statusString = statusStringUnknown; - break; - } + default: statusString = statusStringUnknown; break; + } } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_columninfo.h b/writeengine/bulk/we_columninfo.h index dbae719b5..7e0fa3537 100644 --- a/writeengine/bulk/we_columninfo.h +++ b/writeengine/bulk/we_columninfo.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_columninfo.h 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_columninfo.h 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ /** @file * Contains main class used to manage column information. @@ -52,11 +52,11 @@ struct DBRootExtentInfo; enum Status { - PARSE_COMPLETE = 0, - READ_COMPLETE, - READ_PROGRESS, - NEW, - ERR + PARSE_COMPLETE = 0, + READ_COMPLETE, + READ_PROGRESS, + NEW, + ERR }; // State of starting db file: @@ -66,17 +66,19 @@ enum Status // ERROR_STATE - error has occurred executing delayed file creation enum InitialDBFileStat { - INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY = 1, - INITIAL_DBFILE_STAT_CREATE_FILE_ON_DISABLED = 2, - INITIAL_DBFILE_STAT_FILE_EXISTS = 3, - INITIAL_DBFILE_STAT_ERROR_STATE = 4 + INITIAL_DBFILE_STAT_CREATE_FILE_ON_EMPTY = 1, + INITIAL_DBFILE_STAT_CREATE_FILE_ON_DISABLED = 2, + INITIAL_DBFILE_STAT_FILE_EXISTS = 3, + INITIAL_DBFILE_STAT_ERROR_STATE = 4 }; struct LockInfo { - int locker; - Status status; - LockInfo() : locker(-1), status(WriteEngine::NEW) { } + int locker; + Status status; + LockInfo() : locker(-1), status(WriteEngine::NEW) + { + } }; //------------------------------------------------------------------------------ @@ -123,399 +125,370 @@ struct LockInfo /** @brief Maintains information about a DB column. */ -class ColumnInfo: public WeUIDGID +class ColumnInfo : public WeUIDGID { -public: - //-------------------------------------------------------------------------- - // Public Data Members - //-------------------------------------------------------------------------- + public: + //-------------------------------------------------------------------------- + // Public Data Members + //-------------------------------------------------------------------------- - /** @brief Current column - */ - Column curCol; + /** @brief Current column + */ + Column curCol; - /** @brief ColumnOp instance - */ - boost::scoped_ptr colOp; + /** @brief ColumnOp instance + */ + boost::scoped_ptr colOp; - /** @brief Column information. - */ - JobColumn column; + /** @brief Column information. + */ + JobColumn column; - /** @brief column id - */ - int id; + /** @brief column id + */ + int id; - /** @brief Processing time for the column. (in milliseconds) - */ - double lastProcessingTime; + /** @brief Processing time for the column. (in milliseconds) + */ + double lastProcessingTime; #ifdef PROFILE - /** @brief Total processing time for the column. (in milliseconds) - */ - double totalProcessingTime; + /** @brief Total processing time for the column. (in milliseconds) + */ + double totalProcessingTime; #endif - /** @brief Instance of the write buffer manager. - */ - ColumnBufferManager* fColBufferMgr; - - /** @brief Freespace (in bytes) at the end of the current db column file - * For compressed data files, this is the "raw" data byte count, - * not the compressed byte count. - */ - long long availFileSize; - - /** @brief Total size capacity of current db column segment file. - * For compressed data files, this is the "raw" data byte count, - * not the compressed byte count. - */ - long long fileSize; - - //-------------------------------------------------------------------------- - // Public Functions - //-------------------------------------------------------------------------- - - /** @brief Constructor. - */ - ColumnInfo(Log* logger, - int id, - const JobColumn& column, - DBRootExtentTracker* pDBRootExtTrk, - TableInfo* pTableInfo); - - /** @brief Destructor - */ - virtual ~ColumnInfo(); - - /** @brief Returns last input Row num in current "logical" extent; used - * to track min/max value per extent, as the data is parsed. 0-based - * where Row 0 is first valid input row in the import. - */ - RID lastInputRowInExtent( ) const; - - /** @brief Increment last input Row num in current "logical" extent, so - * that it references the last row of the next extent; used in tracking - * min/max value extent. 0-based where Row 0 is first valid input row - * in the import. This function is called when a Read buffer crosses - * an extent boundary. - */ - void lastInputRowInExtentInc ( ); - - /** @brief Update dictionary method. - * Parses and stores specified strings into the store file, and - * returns the assigned tokens (tokenBuf) to be stored in the - * corresponding column token file. - */ - int updateDctnryStore(char* buf, - ColPosPair** pos, - const int totalRow, - char* tokenBuf); - - /** @brief Close the current Column file. - * @param bCompletedExtent are we completing an extent - * @param bAbort indicates if job is aborting and file should be - * closed without doing extra work: flushing buffer, etc. - */ - virtual int closeColumnFile(bool bCompletingExtent, bool bAbort); - - /** @brief Close the current Dictionary store file. - * @param bAbort Indicates if job is aborting and file should be - * closed without doing extra work: flushing buffer, updating HWM, etc - */ - int closeDctnryStore(bool bAbort); - - /** @brief utility to convert a Status enumeration to a string - */ - static void convertStatusToString( WriteEngine::Status status, - std::string& statusString ); - - /** @brief Adds an extent to "this" column if needed to contain - * the specified number of rows. (New version, supplants checkAnd- - * ExtendColumn()). Also saves the HWM associated with the current - * extent, and uses it to update the extentmap at the job's end. - * - * The state of ColumnInfo is updated to reflect the new extent. - * For example, curCol is updated with the DBRoot, partition, and - * segment file corresponding to the new extent and segment file. - * - * @param saveLBIDForCP (in) Should new extent's LBID be saved in the - * extent stats we are saving to update Casual Partition. - */ - int extendColumn( bool saveLBIDForCP ); - - /** @brief Get Extent Map updates to send to BRM at EOJ, for this column. - * @param brmReporter Reporter object where BRM updates are to be saved - */ - void getBRMUpdateInfo( BRMReporter& brmReporter ); - - /** @brief Commit/Save auto-increment updates - */ - int finishAutoInc( ); - - /** @brief Get current dbroot, partition, segment and HWM for this column. - */ - void getSegFileInfo( DBRootExtentInfo& fileInfo ); - - /** @brief Get last updated saved LBID. - */ - BRM::LBID_t getLastUpdatedLBID() const; - - /** @brief Initialize autoincrement value from the current "next" value - * taken from the system catalog. - */ - int initAutoInc( const std::string& fullTableName ); - - /** @brief Open a new Dictionary store file based on the setting of the - * DBRoot, partition, and segment settings in curCol.dataFile. - * @param bMustExist Indicates whether store file must already exist - */ - int openDctnryStore( bool bMustExist ); - - /** @brief dictionary blocks that will need to be flushed from cache */ - std::vector fDictBlocks; - - /** @brief Set abbreviated extent flag if this is an abbrev extent */ - void setAbbrevExtentCheck( ); - - /** @brief Is current extent we are loading, an "abbreviated" extent - */ - bool isAbbrevExtent(); - - /** @brief Expand abbreviated extent in current column segment file. - * @param bRetainFilePos controls whether current file position is - * to be retained up return from the function. - */ - int expandAbbrevExtent( bool bRetainFilePos ); - - /** @brief Print extent CP information - */ - void printCPInfo( JobColumn column ); - - /** @brief Set width factor relative to other columns in the same table. - */ - void relativeColWidthFactor( int colWidFactor ); - - /** @brief Update extent CP information - */ - template - void updateCPInfo( RID lastInputRow, - T minVal, T maxVal, - ColDataType colDataType, - int width ); - - /** @brief Setup initial extent we will begin loading at start of import. - * @param dbRoot DBRoot of starting extent - * @param partition Partition number of starting extent - * @param segment Segment file number of starting extent - * @param tblName Name of table holding this column - * @param lbid LBID associated with starting extent - * @param oldHwm HWM associated with current HWM extent - * @param hwm Starting HWM after oldHWM has been incremented to - * account for initial block skipping. - * @param bSkippedtoNewExtent Did block skipping advance to next extent - * @param bIsNewExtent Treat as new extent when updating CP min/max - */ - int setupInitialColumnExtent( uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - const std::string& tblName, - BRM::LBID_t lbid, - HWM oldHwm, - HWM hwm, - bool bSkippedToNewExtent, - bool bIsNewExtent ); - - /** @brief Setup a DB file to be created for starting extent only when needed - * @param dbRoot DBRoot of starting extent - * @param partition Partition number of starting extent - * @param segment Segment file number of starting extent - * @param hwm Starting HWM for new start extent - * @param bEmptyPM Are we setting up delayed file creation because a PM - * has no extents (or is the HWM extent just disabled) - */ - void setupDelayedFileCreation( - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM hwm, - bool bEmptyPM ); - - /** @brief Belatedly create a starting DB file for a PM that has none. - * @param tableName Name of table for which this column belongs - */ - int createDelayedFileIfNeeded( const std::string& tableName ); - - /** @brief Update how many bytes of data are in the column segment file and - * how much room remains in the file (till the current extent is full). - * @param numBytesWritten Number of bytes just added to the column file. - */ - void updateBytesWrittenCounts( unsigned int numBytesWritten ); - - /** @brief Returns the list of HWM dictionary blks to be cached - */ - void getDictFlushBlks( std::vector& blks ) const; - - /** @brief Returns the current file size in bytes - */ - int64_t getFileSize( ) const; - - /** @brief Has file filled up all its extents - */ - bool isFileComplete( ) const; - - /** @brief Reserve block of auto-increment numbers to generate - * @param autoIncCount The number of autoincrement numbers to be reserved. - * @param nextValue Value of the first reserved auto inc number. - */ - int reserveAutoIncNums(uint32_t autoIncCount, uint64_t& nextValue ); - - /** @brief Truncate specified dictionary file. Only applies if compressed. - * @param dctnryOid Dictionary store OID - * @param root DBRoot of relevant dictionary store segment file. - * @param pNum Partition number of relevant dictionary store segment file. - * @param sNum Segment number of relevant dictionary store segment file. - */ - virtual int truncateDctnryStore(OID dctnryOid, - uint16_t root, uint32_t pNum, uint16_t sNum) const; - - /** @brief Increment saturated row count for this column in current import - * @param satIncCnt Increment count to add to the total saturation count. - */ - void incSaturatedCnt( int64_t satIncCnt ); - - /** @brief Get saturated row count for this column. - */ - long long saturatedCnt( ); - - /** @brief When parsing is complete for a column, this function is called - * to finish flushing and closing the current segment file. - */ - int finishParsing( ); - - /** @brief Mutex used to manage access to the output buffers and files. - * This was formerly the fMgrMutex in ColumnBufferManager. See comments - * that precede this class definition for more information. - */ - boost::mutex& colMutex(); - - /** @brief Get number of rows per extent - */ - unsigned rowsPerExtent( ); - - void setUIDGID(const uid_t uid, const gid_t gid) override; - -protected: - - //-------------------------------------------------------------------------- - // Protected Functions - //-------------------------------------------------------------------------- - - void addToSegFileList(File& dataFile, // save HWM info per segment file - HWM hwm ); - void clearMemory(); // clear memory used by this object - void getCPInfoForBRM(BRMReporter& brmReporter);//Get updated CP info for BRM - int getHWMInfoForBRM(BRMReporter& brmReporter);//Get updated HWM inf for BRM - - // Init last input Row number in current "logical" extent; used - // to track min/max value per extent. 0-based where Row 0 is first - // valid input row in the import. - // bIsNewExtent indicates whether to treat as a new extent or not. - void lastInputRowInExtentInit( bool bIsNewExtent ); - - virtual int resetFileOffsetsNewExtent(const char* hdr); - // Reset file; start new extent - void setFileSize( HWM hwm, int abbrevFlag ); // Set fileSize data member - - // Prepare initial column segment file for importing of data. - // oldHWM - Current HWM prior to initial block skipping. This is only - // used for abbreviated extents, to detect when block skipping has - // caused us to require a full expanded extent. - // newHWM - Starting point for adding data after initial blockskipping - virtual int setupInitialColumnFile( HWM oldHWM, // original HWM - HWM newHWM ); // new HWM to start from - - virtual int saveDctnryStoreHWMChunk(bool& needBackup);//Backup Dct HWM Chunk - int extendColumnNewExtent( // extend column; new extent - bool saveLBIDForCP, - uint16_t dbRootNew, - uint32_t partitionNew ); - virtual int extendColumnOldExtent( // extend column; existing extent - uint16_t dbRootNext, - uint32_t partitionNext, - uint16_t segmentNext, - HWM hwmNext ); - - //-------------------------------------------------------------------------- - // Protected Data Members - //-------------------------------------------------------------------------- - - boost::mutex fDictionaryMutex; // Mutex for dicionary updates - boost::mutex fColMutex; // Mutex for column changes - boost::mutex fAutoIncMutex; // Mutex to manage fAutoIncLastValue - boost::mutex fDelayedFileCreateMutex; // Manage delayed file check/create - Log* fLog; // Object used for logging - - // Blocks to skip at start of bulk load, if starting file has to be created - // by createDelayedFileIfNeeded() - HWM fDelayedFileStartBlksSkipped; - - // LBID corresponding to initial HWM saved in fSavedHWM at start of import. - // - // LBID is used, at the end of the import, to identify to DBRM, an - // extent whose CasualPartition stats are to be cleared, because we will - // have written additional rows to that extent as part of an import. - BRM::LBID_t fSavedLbid; - - // The last updated LBID. In case `bulk` creates a new extent with new LBID - // we have to initialize file header with created LBID. - BRM::LBID_t fLastUpdatedLbid; - - // Size of a segment file (in bytes) when the file is opened - // to add the next extent. - // For compressed data files, this is the "raw" data byte count, - // not the compressed byte count. - long long fSizeWrittenStart; - - // Tracks the size of a segment file (in bytes) as rows are added. - // For compressed data files, this is the "raw" data byte count, - // not the compressed byte count. - long long fSizeWritten; - - // Tracks last input Row number in the current "logical" extent, - // where Row number is 0-based, with Row 0 being the first row in the - // import. Used by parsing thread to track when a read buffer crosses - // an extent boundary. We detect when a Read buffer crosses an ex- - // pected extent boundary so that we can track a column's min/max for - // each extent. - RID fLastInputRowInCurrentExtent; - - bool fLoadingAbbreviatedExtent; // Is current extent abbreviated - ColExtInfBase* fColExtInf; // Used to update CP at end of job - long long fMaxNumRowsPerSegFile; // Max num rows per segment file - Dctnry* fStore; // Corresponding dctnry store file - - - - // For autoincrement column only... Tracks latest autoincrement value used - long long fAutoIncLastValue; - - volatile int64_t fSaturatedRowCnt; // No. of rows with saturated values - - // List of segment files updated during an import; used to track infor- - // mation necessary to update the ExtentMap at the "end" of the import. - std::vector fSegFileUpdateList; - - TableInfo* fpTableInfo; // pointer to the table info - ColumnAutoInc* fAutoIncMgr; // Maintains autoIncrement nextValue - DBRootExtentTracker* fDbRootExtTrk; // DBRoot extent tracker - - int fColWidthFactor; // Wid factor relative to other cols - - InitialDBFileStat fDelayedFileCreation; // Denotes when initial DB file is - // to be created after preprocessing - - unsigned fRowsPerExtent; // Number of rows per column extent + /** @brief Instance of the write buffer manager. + */ + ColumnBufferManager* fColBufferMgr; + + /** @brief Freespace (in bytes) at the end of the current db column file + * For compressed data files, this is the "raw" data byte count, + * not the compressed byte count. + */ + long long availFileSize; + + /** @brief Total size capacity of current db column segment file. + * For compressed data files, this is the "raw" data byte count, + * not the compressed byte count. + */ + long long fileSize; + + //-------------------------------------------------------------------------- + // Public Functions + //-------------------------------------------------------------------------- + + /** @brief Constructor. + */ + ColumnInfo(Log* logger, int id, const JobColumn& column, DBRootExtentTracker* pDBRootExtTrk, + TableInfo* pTableInfo); + + /** @brief Destructor + */ + virtual ~ColumnInfo(); + + /** @brief Returns last input Row num in current "logical" extent; used + * to track min/max value per extent, as the data is parsed. 0-based + * where Row 0 is first valid input row in the import. + */ + RID lastInputRowInExtent() const; + + /** @brief Increment last input Row num in current "logical" extent, so + * that it references the last row of the next extent; used in tracking + * min/max value extent. 0-based where Row 0 is first valid input row + * in the import. This function is called when a Read buffer crosses + * an extent boundary. + */ + void lastInputRowInExtentInc(); + + /** @brief Update dictionary method. + * Parses and stores specified strings into the store file, and + * returns the assigned tokens (tokenBuf) to be stored in the + * corresponding column token file. + */ + int updateDctnryStore(char* buf, ColPosPair** pos, const int totalRow, char* tokenBuf); + + /** @brief Close the current Column file. + * @param bCompletedExtent are we completing an extent + * @param bAbort indicates if job is aborting and file should be + * closed without doing extra work: flushing buffer, etc. + */ + virtual int closeColumnFile(bool bCompletingExtent, bool bAbort); + + /** @brief Close the current Dictionary store file. + * @param bAbort Indicates if job is aborting and file should be + * closed without doing extra work: flushing buffer, updating HWM, etc + */ + int closeDctnryStore(bool bAbort); + + /** @brief utility to convert a Status enumeration to a string + */ + static void convertStatusToString(WriteEngine::Status status, std::string& statusString); + + /** @brief Adds an extent to "this" column if needed to contain + * the specified number of rows. (New version, supplants checkAnd- + * ExtendColumn()). Also saves the HWM associated with the current + * extent, and uses it to update the extentmap at the job's end. + * + * The state of ColumnInfo is updated to reflect the new extent. + * For example, curCol is updated with the DBRoot, partition, and + * segment file corresponding to the new extent and segment file. + * + * @param saveLBIDForCP (in) Should new extent's LBID be saved in the + * extent stats we are saving to update Casual Partition. + */ + int extendColumn(bool saveLBIDForCP); + + /** @brief Get Extent Map updates to send to BRM at EOJ, for this column. + * @param brmReporter Reporter object where BRM updates are to be saved + */ + void getBRMUpdateInfo(BRMReporter& brmReporter); + + /** @brief Commit/Save auto-increment updates + */ + int finishAutoInc(); + + /** @brief Get current dbroot, partition, segment and HWM for this column. + */ + void getSegFileInfo(DBRootExtentInfo& fileInfo); + + /** @brief Get last updated saved LBID. + */ + BRM::LBID_t getLastUpdatedLBID() const; + + /** @brief Initialize autoincrement value from the current "next" value + * taken from the system catalog. + */ + int initAutoInc(const std::string& fullTableName); + + /** @brief Open a new Dictionary store file based on the setting of the + * DBRoot, partition, and segment settings in curCol.dataFile. + * @param bMustExist Indicates whether store file must already exist + */ + int openDctnryStore(bool bMustExist); + + /** @brief dictionary blocks that will need to be flushed from cache */ + std::vector fDictBlocks; + + /** @brief Set abbreviated extent flag if this is an abbrev extent */ + void setAbbrevExtentCheck(); + + /** @brief Is current extent we are loading, an "abbreviated" extent + */ + bool isAbbrevExtent(); + + /** @brief Expand abbreviated extent in current column segment file. + * @param bRetainFilePos controls whether current file position is + * to be retained up return from the function. + */ + int expandAbbrevExtent(bool bRetainFilePos); + + /** @brief Print extent CP information + */ + void printCPInfo(JobColumn column); + + /** @brief Set width factor relative to other columns in the same table. + */ + void relativeColWidthFactor(int colWidFactor); + + /** @brief Update extent CP information + */ + template + void updateCPInfo(RID lastInputRow, T minVal, T maxVal, ColDataType colDataType, int width); + + /** @brief Setup initial extent we will begin loading at start of import. + * @param dbRoot DBRoot of starting extent + * @param partition Partition number of starting extent + * @param segment Segment file number of starting extent + * @param tblName Name of table holding this column + * @param lbid LBID associated with starting extent + * @param oldHwm HWM associated with current HWM extent + * @param hwm Starting HWM after oldHWM has been incremented to + * account for initial block skipping. + * @param bSkippedtoNewExtent Did block skipping advance to next extent + * @param bIsNewExtent Treat as new extent when updating CP min/max + */ + int setupInitialColumnExtent(uint16_t dbRoot, uint32_t partition, uint16_t segment, + const std::string& tblName, BRM::LBID_t lbid, HWM oldHwm, HWM hwm, + bool bSkippedToNewExtent, bool bIsNewExtent); + + /** @brief Setup a DB file to be created for starting extent only when needed + * @param dbRoot DBRoot of starting extent + * @param partition Partition number of starting extent + * @param segment Segment file number of starting extent + * @param hwm Starting HWM for new start extent + * @param bEmptyPM Are we setting up delayed file creation because a PM + * has no extents (or is the HWM extent just disabled) + */ + void setupDelayedFileCreation(uint16_t dbRoot, uint32_t partition, uint16_t segment, HWM hwm, + bool bEmptyPM); + + /** @brief Belatedly create a starting DB file for a PM that has none. + * @param tableName Name of table for which this column belongs + */ + int createDelayedFileIfNeeded(const std::string& tableName); + + /** @brief Update how many bytes of data are in the column segment file and + * how much room remains in the file (till the current extent is full). + * @param numBytesWritten Number of bytes just added to the column file. + */ + void updateBytesWrittenCounts(unsigned int numBytesWritten); + + /** @brief Returns the list of HWM dictionary blks to be cached + */ + void getDictFlushBlks(std::vector& blks) const; + + /** @brief Returns the current file size in bytes + */ + int64_t getFileSize() const; + + /** @brief Has file filled up all its extents + */ + bool isFileComplete() const; + + /** @brief Reserve block of auto-increment numbers to generate + * @param autoIncCount The number of autoincrement numbers to be reserved. + * @param nextValue Value of the first reserved auto inc number. + */ + int reserveAutoIncNums(uint32_t autoIncCount, uint64_t& nextValue); + + /** @brief Truncate specified dictionary file. Only applies if compressed. + * @param dctnryOid Dictionary store OID + * @param root DBRoot of relevant dictionary store segment file. + * @param pNum Partition number of relevant dictionary store segment file. + * @param sNum Segment number of relevant dictionary store segment file. + */ + virtual int truncateDctnryStore(OID dctnryOid, uint16_t root, uint32_t pNum, uint16_t sNum) const; + + /** @brief Increment saturated row count for this column in current import + * @param satIncCnt Increment count to add to the total saturation count. + */ + void incSaturatedCnt(int64_t satIncCnt); + + /** @brief Get saturated row count for this column. + */ + long long saturatedCnt(); + + /** @brief When parsing is complete for a column, this function is called + * to finish flushing and closing the current segment file. + */ + int finishParsing(); + + /** @brief Mutex used to manage access to the output buffers and files. + * This was formerly the fMgrMutex in ColumnBufferManager. See comments + * that precede this class definition for more information. + */ + boost::mutex& colMutex(); + + /** @brief Get number of rows per extent + */ + unsigned rowsPerExtent(); + + void setUIDGID(const uid_t uid, const gid_t gid) override; + + protected: + //-------------------------------------------------------------------------- + // Protected Functions + //-------------------------------------------------------------------------- + + void addToSegFileList(File& dataFile, // save HWM info per segment file + HWM hwm); + void clearMemory(); // clear memory used by this object + void getCPInfoForBRM(BRMReporter& brmReporter); // Get updated CP info for BRM + int getHWMInfoForBRM(BRMReporter& brmReporter); // Get updated HWM inf for BRM + + // Init last input Row number in current "logical" extent; used + // to track min/max value per extent. 0-based where Row 0 is first + // valid input row in the import. + // bIsNewExtent indicates whether to treat as a new extent or not. + void lastInputRowInExtentInit(bool bIsNewExtent); + + virtual int resetFileOffsetsNewExtent(const char* hdr); + // Reset file; start new extent + void setFileSize(HWM hwm, int abbrevFlag); // Set fileSize data member + + // Prepare initial column segment file for importing of data. + // oldHWM - Current HWM prior to initial block skipping. This is only + // used for abbreviated extents, to detect when block skipping has + // caused us to require a full expanded extent. + // newHWM - Starting point for adding data after initial blockskipping + virtual int setupInitialColumnFile(HWM oldHWM, // original HWM + HWM newHWM); // new HWM to start from + + virtual int saveDctnryStoreHWMChunk(bool& needBackup); // Backup Dct HWM Chunk + int extendColumnNewExtent( // extend column; new extent + bool saveLBIDForCP, uint16_t dbRootNew, uint32_t partitionNew); + virtual int extendColumnOldExtent( // extend column; existing extent + uint16_t dbRootNext, uint32_t partitionNext, uint16_t segmentNext, HWM hwmNext); + + //-------------------------------------------------------------------------- + // Protected Data Members + //-------------------------------------------------------------------------- + + boost::mutex fDictionaryMutex; // Mutex for dicionary updates + boost::mutex fColMutex; // Mutex for column changes + boost::mutex fAutoIncMutex; // Mutex to manage fAutoIncLastValue + boost::mutex fDelayedFileCreateMutex; // Manage delayed file check/create + Log* fLog; // Object used for logging + + // Blocks to skip at start of bulk load, if starting file has to be created + // by createDelayedFileIfNeeded() + HWM fDelayedFileStartBlksSkipped; + + // LBID corresponding to initial HWM saved in fSavedHWM at start of import. + // + // LBID is used, at the end of the import, to identify to DBRM, an + // extent whose CasualPartition stats are to be cleared, because we will + // have written additional rows to that extent as part of an import. + BRM::LBID_t fSavedLbid; + + // The last updated LBID. In case `bulk` creates a new extent with new LBID + // we have to initialize file header with created LBID. + BRM::LBID_t fLastUpdatedLbid; + + // Size of a segment file (in bytes) when the file is opened + // to add the next extent. + // For compressed data files, this is the "raw" data byte count, + // not the compressed byte count. + long long fSizeWrittenStart; + + // Tracks the size of a segment file (in bytes) as rows are added. + // For compressed data files, this is the "raw" data byte count, + // not the compressed byte count. + long long fSizeWritten; + + // Tracks last input Row number in the current "logical" extent, + // where Row number is 0-based, with Row 0 being the first row in the + // import. Used by parsing thread to track when a read buffer crosses + // an extent boundary. We detect when a Read buffer crosses an ex- + // pected extent boundary so that we can track a column's min/max for + // each extent. + RID fLastInputRowInCurrentExtent; + + bool fLoadingAbbreviatedExtent; // Is current extent abbreviated + ColExtInfBase* fColExtInf; // Used to update CP at end of job + long long fMaxNumRowsPerSegFile; // Max num rows per segment file + Dctnry* fStore; // Corresponding dctnry store file + + // For autoincrement column only... Tracks latest autoincrement value used + long long fAutoIncLastValue; + + volatile int64_t fSaturatedRowCnt; // No. of rows with saturated values + + // List of segment files updated during an import; used to track infor- + // mation necessary to update the ExtentMap at the "end" of the import. + std::vector fSegFileUpdateList; + + TableInfo* fpTableInfo; // pointer to the table info + ColumnAutoInc* fAutoIncMgr; // Maintains autoIncrement nextValue + DBRootExtentTracker* fDbRootExtTrk; // DBRoot extent tracker + + int fColWidthFactor; // Wid factor relative to other cols + + InitialDBFileStat fDelayedFileCreation; // Denotes when initial DB file is + // to be created after preprocessing + + unsigned fRowsPerExtent; // Number of rows per column extent }; //------------------------------------------------------------------------------ @@ -523,72 +496,67 @@ protected: //------------------------------------------------------------------------------ inline void ColumnInfo::setUIDGID(const uid_t p_uid, const gid_t p_gid) { - WeUIDGID::setUIDGID(p_uid, p_gid); - if (colOp) - colOp->setUIDGID(this); + WeUIDGID::setUIDGID(p_uid, p_gid); + if (colOp) + colOp->setUIDGID(this); } inline boost::mutex& ColumnInfo::colMutex() { - return fColMutex; + return fColMutex; } -inline void ColumnInfo::getDictFlushBlks( std::vector& blks ) const +inline void ColumnInfo::getDictFlushBlks(std::vector& blks) const { - blks = fDictBlocks; + blks = fDictBlocks; } -inline int64_t ColumnInfo::getFileSize( ) const +inline int64_t ColumnInfo::getFileSize() const { - return fileSize; + return fileSize; } -inline void ColumnInfo::incSaturatedCnt( int64_t satIncCnt ) +inline void ColumnInfo::incSaturatedCnt(int64_t satIncCnt) { - (void)atomicops::atomicAdd(&fSaturatedRowCnt, satIncCnt); + (void)atomicops::atomicAdd(&fSaturatedRowCnt, satIncCnt); } -inline bool ColumnInfo::isAbbrevExtent( ) +inline bool ColumnInfo::isAbbrevExtent() { - return fLoadingAbbreviatedExtent; + return fLoadingAbbreviatedExtent; } -inline RID ColumnInfo::lastInputRowInExtent( ) const +inline RID ColumnInfo::lastInputRowInExtent() const { - return fLastInputRowInCurrentExtent; + return fLastInputRowInCurrentExtent; } -inline void ColumnInfo::printCPInfo( JobColumn column ) +inline void ColumnInfo::printCPInfo(JobColumn column) { - fColExtInf->print( column ); + fColExtInf->print(column); } -inline long long ColumnInfo::saturatedCnt( ) +inline long long ColumnInfo::saturatedCnt() { - return fSaturatedRowCnt; + return fSaturatedRowCnt; } -inline void ColumnInfo::relativeColWidthFactor( int colWidFactor ) +inline void ColumnInfo::relativeColWidthFactor(int colWidFactor) { - fColWidthFactor = colWidFactor; + fColWidthFactor = colWidFactor; } inline unsigned ColumnInfo::rowsPerExtent() { - return fRowsPerExtent; + return fRowsPerExtent; } template -inline void ColumnInfo::updateCPInfo( - RID lastInputRow, - T minVal, - T maxVal, - ColDataType colDataType, - int width ) +inline void ColumnInfo::updateCPInfo(RID lastInputRow, T minVal, T maxVal, ColDataType colDataType, int width) { - fColExtInf->addOrUpdateEntry( lastInputRow, minVal, maxVal, colDataType, width ); + fColExtInf->addOrUpdateEntry(lastInputRow, minVal, maxVal, colDataType, width); } -} // end of namespace +} // namespace WriteEngine #endif diff --git a/writeengine/bulk/we_columninfocompressed.cpp b/writeengine/bulk/we_columninfocompressed.cpp index e412e62d8..5c653649a 100644 --- a/writeengine/bulk/we_columninfocompressed.cpp +++ b/writeengine/bulk/we_columninfocompressed.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************** -* $Id: we_columninfocompressed.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_columninfocompressed.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ #include "we_columninfocompressed.h" @@ -39,182 +39,161 @@ using namespace compress; namespace WriteEngine { - //------------------------------------------------------------------------------ // ColumnInfoCompressed constructor //------------------------------------------------------------------------------ -ColumnInfoCompressed::ColumnInfoCompressed(Log* logger, - int idIn, - const JobColumn& columnIn, - DBRootExtentTracker* pDBRootExtTrk, - TableInfo* pTableInfo): - //RBMetaWriter* rbMetaWriter) : - ColumnInfo(logger, idIn, columnIn, pDBRootExtTrk, pTableInfo), - fRBMetaWriter(pTableInfo->rbMetaWriter()) +ColumnInfoCompressed::ColumnInfoCompressed(Log* logger, int idIn, const JobColumn& columnIn, + DBRootExtentTracker* pDBRootExtTrk, TableInfo* pTableInfo) + : // RBMetaWriter* rbMetaWriter) : + ColumnInfo(logger, idIn, columnIn, pDBRootExtTrk, pTableInfo) + , fRBMetaWriter(pTableInfo->rbMetaWriter()) { } - + //------------------------------------------------------------------------------ // ColumnInfoCompressed destructor //------------------------------------------------------------------------------ ColumnInfoCompressed::~ColumnInfoCompressed() { } - + //------------------------------------------------------------------------------ // Close the current compressed Column file after first compressing/flushing // any remaining data, and re-writing the headers as well. //------------------------------------------------------------------------------ int ColumnInfoCompressed::closeColumnFile(bool bCompletingExtent, bool bAbort) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if ( curCol.dataFile.pFile ) + if (curCol.dataFile.pFile) + { + if (!bAbort) { - if (!bAbort) + // If we are opening and closing a file in order to add an extent as + // part of preliminary block skipping, then we won't have a Column- + // BufferManger object yet. One will be created when the file is + // reopened to begin importing. + if (fColBufferMgr) + { + rc = fColBufferMgr->finishFile(bCompletingExtent); + + if (rc != NO_ERROR) { - // If we are opening and closing a file in order to add an extent as - // part of preliminary block skipping, then we won't have a Column- - // BufferManger object yet. One will be created when the file is - // reopened to begin importing. - if (fColBufferMgr) - { - rc = fColBufferMgr->finishFile( bCompletingExtent ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error closing compressed file; OID-" << - curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - bAbort = true; - } - } + WErrorCodes ec; + std::ostringstream oss; + oss << "Error closing compressed file; OID-" << curCol.dataFile.fid << "; DBRoot-" + << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + bAbort = true; } - - ColumnInfo::closeColumnFile(bCompletingExtent, bAbort); + } } - return rc; + ColumnInfo::closeColumnFile(bCompletingExtent, bAbort); + } + + return rc; } - + //------------------------------------------------------------------------------ // Prepare the initial compressed column segment file for import. //------------------------------------------------------------------------------ -int ColumnInfoCompressed::setupInitialColumnFile( HWM oldHwm, HWM hwm ) +int ColumnInfoCompressed::setupInitialColumnFile(HWM oldHwm, HWM hwm) { - char hdr[ compress::CompressInterface::HDR_BUF_LEN * 2 ]; - RETURN_ON_ERROR( colOp->readHeaders(curCol.dataFile.pFile, hdr) ); + char hdr[compress::CompressInterface::HDR_BUF_LEN * 2]; + RETURN_ON_ERROR(colOp->readHeaders(curCol.dataFile.pFile, hdr)); - // Initialize the output buffer manager for the column. - WriteEngine::ColumnBufferManager* mgr; + // Initialize the output buffer manager for the column. + WriteEngine::ColumnBufferManager* mgr; - if (column.colType == COL_TYPE_DICT) + if (column.colType == COL_TYPE_DICT) + { + mgr = new ColumnBufferManagerDctnry(this, 8, fLog, column.compressionType); + RETURN_ON_ERROR(mgr->setDbFile(curCol.dataFile.pFile, hwm, hdr)); + } + else + { + mgr = new ColumnBufferManager(this, column.width, fLog, column.compressionType); + RETURN_ON_ERROR(mgr->setDbFile(curCol.dataFile.pFile, hwm, hdr)); + } + + fColBufferMgr = mgr; + + int abbrevFlag = (compress::CompressInterface::getBlockCount(hdr) == + uint64_t(INITIAL_EXTENT_ROWS_TO_DISK * column.width / BYTE_PER_BLOCK)); + setFileSize(hwm, abbrevFlag); + + // See if dealing with abbreviated extent that will need expanding. + // This only applies to the first extent of the first segment file. + setAbbrevExtentCheck(); + + // If we are dealing with initial extent, see if block skipping has + // exceeded disk allocation, in which case we expand to a full extent. + if (isAbbrevExtent()) + { + unsigned int numBlksForFirstExtent = (INITIAL_EXTENT_ROWS_TO_DISK * column.width) / BYTE_PER_BLOCK; + + if (((oldHwm + 1) <= numBlksForFirstExtent) && ((hwm + 1) > numBlksForFirstExtent)) { - mgr = new ColumnBufferManagerDctnry( - this, 8, fLog, column.compressionType); - RETURN_ON_ERROR( mgr->setDbFile(curCol.dataFile.pFile, hwm, hdr) ); - } - else - { - mgr = new ColumnBufferManager( - this, column.width, fLog, column.compressionType); - RETURN_ON_ERROR( mgr->setDbFile(curCol.dataFile.pFile, hwm, hdr) ); + RETURN_ON_ERROR(expandAbbrevExtent(false)); } + } - fColBufferMgr = mgr; + // Store the current allocated file size in availFileSize. + // Keep in mind, these are raw uncompressed offsets. + // NOTE: We don't call setFileOffset() to set the file position in the + // column segment file at this point; we wait till we load the compressed + // buffer later on in ColumnBufferCompressed::initToBeCompressedBuffer() + long long byteOffset = (long long)hwm * (long long)BYTE_PER_BLOCK; - int abbrevFlag = (compress::CompressInterface::getBlockCount(hdr) == - uint64_t(INITIAL_EXTENT_ROWS_TO_DISK * column.width / - BYTE_PER_BLOCK)); - setFileSize( hwm, abbrevFlag ); + fSizeWritten = byteOffset; + fSizeWrittenStart = fSizeWritten; + availFileSize = fileSize - fSizeWritten; - // See if dealing with abbreviated extent that will need expanding. - // This only applies to the first extent of the first segment file. - setAbbrevExtentCheck(); + if (fLog->isDebug(DEBUG_1)) + { + std::ostringstream oss; + oss << "Init raw data offsets in compressed column file OID-" << curCol.dataFile.fid << "; DBRoot-" + << curCol.dataFile.fDbRoot << "; part-" << curCol.dataFile.fPartition << "; seg-" + << curCol.dataFile.fSegment << "; abbrev-" << abbrevFlag << "; begByte-" << fSizeWritten + << "; endByte-" << fileSize << "; freeBytes-" << availFileSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - // If we are dealing with initial extent, see if block skipping has - // exceeded disk allocation, in which case we expand to a full extent. - if (isAbbrevExtent()) - { - unsigned int numBlksForFirstExtent = - (INITIAL_EXTENT_ROWS_TO_DISK * column.width) / BYTE_PER_BLOCK; - - if ( ((oldHwm + 1) <= numBlksForFirstExtent) && - ((hwm + 1 ) > numBlksForFirstExtent) ) - { - RETURN_ON_ERROR( expandAbbrevExtent(false) ); - } - } - - // Store the current allocated file size in availFileSize. - // Keep in mind, these are raw uncompressed offsets. - // NOTE: We don't call setFileOffset() to set the file position in the - // column segment file at this point; we wait till we load the compressed - // buffer later on in ColumnBufferCompressed::initToBeCompressedBuffer() - long long byteOffset = (long long)hwm * (long long)BYTE_PER_BLOCK; - - fSizeWritten = byteOffset; - fSizeWrittenStart = fSizeWritten; - availFileSize = fileSize - fSizeWritten; - - if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss; - oss << "Init raw data offsets in compressed column file OID-" << - curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; abbrev-" << abbrevFlag << - "; begByte-" << fSizeWritten << - "; endByte-" << fileSize << - "; freeBytes-" << availFileSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Reinitializes ColBuf buffer, and resets // file offset data member attributes where new extent will start. //------------------------------------------------------------------------------ int ColumnInfoCompressed::resetFileOffsetsNewExtent(const char* hdr) { - setFileSize( curCol.dataFile.hwm, false ); - long long byteOffset = (long long)curCol.dataFile.hwm * - (long long)BYTE_PER_BLOCK; - fSizeWritten = byteOffset; - fSizeWrittenStart = fSizeWritten; - availFileSize = fileSize - fSizeWritten; + setFileSize(curCol.dataFile.hwm, false); + long long byteOffset = (long long)curCol.dataFile.hwm * (long long)BYTE_PER_BLOCK; + fSizeWritten = byteOffset; + fSizeWrittenStart = fSizeWritten; + availFileSize = fileSize - fSizeWritten; - // If we are adding an extent as part of preliminary block skipping, then - // we won't have a ColumnBufferManager object yet, but that's okay, because - // we are only adding the empty extent at this point. - if (fColBufferMgr) - { - RETURN_ON_ERROR( fColBufferMgr->setDbFile(curCol.dataFile.pFile, - curCol.dataFile.hwm, hdr) ); + // If we are adding an extent as part of preliminary block skipping, then + // we won't have a ColumnBufferManager object yet, but that's okay, because + // we are only adding the empty extent at this point. + if (fColBufferMgr) + { + RETURN_ON_ERROR(fColBufferMgr->setDbFile(curCol.dataFile.pFile, curCol.dataFile.hwm, hdr)); - // Reinitialize ColBuf for the next extent - long long startFileOffset; - RETURN_ON_ERROR( fColBufferMgr->resetToBeCompressedColBuf( - startFileOffset ) ); + // Reinitialize ColBuf for the next extent + long long startFileOffset; + RETURN_ON_ERROR(fColBufferMgr->resetToBeCompressedColBuf(startFileOffset)); - // Set the file offset to point to the chunk we are adding or updating - RETURN_ON_ERROR( colOp->setFileOffset(curCol.dataFile.pFile, - startFileOffset) ); - } + // Set the file offset to point to the chunk we are adding or updating + RETURN_ON_ERROR(colOp->setFileOffset(curCol.dataFile.pFile, startFileOffset)); + } - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Save HWM chunk for compressed dictionary store files, so that the HWM chunk // can be restored by bulk rollback if an error should occur. @@ -223,32 +202,29 @@ int ColumnInfoCompressed::resetFileOffsetsNewExtent(const char* hdr) int ColumnInfoCompressed::saveDctnryStoreHWMChunk(bool& needBackup) { #ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_DCT_BACKUP_CHUNK); + Stats::startParseEvent(WE_STATS_COMPRESS_DCT_BACKUP_CHUNK); #endif - needBackup = false; - int rc = NO_ERROR; + needBackup = false; + int rc = NO_ERROR; - try - { - needBackup = fRBMetaWriter->backupDctnryHWMChunk( - column.dctnry.dctnryOid, - curCol.dataFile.fDbRoot, - curCol.dataFile.fPartition, - curCol.dataFile.fSegment ); - } - catch (WeException& ex) - { - fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR); - rc = ex.errorCode(); - } + try + { + needBackup = fRBMetaWriter->backupDctnryHWMChunk(column.dctnry.dctnryOid, curCol.dataFile.fDbRoot, + curCol.dataFile.fPartition, curCol.dataFile.fSegment); + } + catch (WeException& ex) + { + fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR); + rc = ex.errorCode(); + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_BACKUP_CHUNK); + Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_BACKUP_CHUNK); #endif - return rc; + return rc; } - + //------------------------------------------------------------------------------ // Truncate specified dictionary store file for this column. // Only applies to compressed columns. @@ -268,307 +244,257 @@ int ColumnInfoCompressed::saveDctnryStoreHWMChunk(bool& needBackup) // adding or changing the ChunkManager API to support a flush w/o a close. // That would be more optimum than having to reopen the file for truncation. //------------------------------------------------------------------------------ -int ColumnInfoCompressed::truncateDctnryStore( - OID dctnryOid, uint16_t root, uint32_t pNum, uint16_t sNum) const +int ColumnInfoCompressed::truncateDctnryStore(OID dctnryOid, uint16_t root, uint32_t pNum, + uint16_t sNum) const { - int rc = NO_ERROR; + int rc = NO_ERROR; - // @bug5769 Don't initialize extents or truncate db files on HDFS - if (idbdatafile::IDBPolicy::useHdfs()) + // @bug5769 Don't initialize extents or truncate db files on HDFS + if (idbdatafile::IDBPolicy::useHdfs()) + { + std::ostringstream oss1; + oss1 << "Finished writing dictionary file" + ": OID-" + << dctnryOid << "; DBRoot-" << root << "; part-" << pNum << "; seg-" << sNum; + + // Have to rework this logging if we want to keep it. + // Filesize is not correct when adding data to an "existing" file, + // since in the case of HDFS, we are writing to a *.cdf.tmp file. + // char dctnryFileName[FILE_NAME_SIZE]; + // if (colOp->getFileName(dctnryOid,dctnryFileName, + // root, pNum, sNum) == NO_ERROR) + //{ + // off64_t dctnryFileSize = idbdatafile::IDBFileSystem::getFs( + // IDBDataFile::HDFS).size(dctnryFileName); + // if (dctnryFileSize != -1) + // { + // oss1 << "; size-" << dctnryFileSize; + // } + //} + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + } + else + { + // See if the relevant dictionary store file can/should be truncated + // (to the nearest extent) + std::string segFile; + IDBDataFile* dFile = fTruncateDctnryFileOp.openFile(dctnryOid, root, pNum, sNum, segFile); + + if (dFile == 0) { - std::ostringstream oss1; - oss1 << "Finished writing dictionary file" - ": OID-" << dctnryOid << - "; DBRoot-" << root << - "; part-" << pNum << - "; seg-" << sNum; + rc = ERR_FILE_OPEN; - // Have to rework this logging if we want to keep it. - // Filesize is not correct when adding data to an "existing" file, - // since in the case of HDFS, we are writing to a *.cdf.tmp file. - //char dctnryFileName[FILE_NAME_SIZE]; - //if (colOp->getFileName(dctnryOid,dctnryFileName, - // root, pNum, sNum) == NO_ERROR) - //{ - // off64_t dctnryFileSize = idbdatafile::IDBFileSystem::getFs( - // IDBDataFile::HDFS).size(dctnryFileName); - // if (dctnryFileSize != -1) - // { - // oss1 << "; size-" << dctnryFileSize; - // } - //} - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - } - else - { - // See if the relevant dictionary store file can/should be truncated - // (to the nearest extent) - std::string segFile; - IDBDataFile* dFile = fTruncateDctnryFileOp.openFile(dctnryOid, - root, pNum, sNum, segFile); + std::ostringstream oss; + oss << "Error opening compressed dictionary store segment " + "file for truncation" + << ": OID-" << dctnryOid << "; DbRoot-" << root << "; partition-" << pNum << "; segment-" << sNum; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - if (dFile == 0) - { - rc = ERR_FILE_OPEN; - - std::ostringstream oss; - oss << "Error opening compressed dictionary store segment " - "file for truncation" << - ": OID-" << dctnryOid << - "; DbRoot-" << root << - "; partition-" << pNum << - "; segment-" << sNum; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - char controlHdr[ CompressInterface::HDR_BUF_LEN ]; - rc = fTruncateDctnryFileOp.readFile( dFile, - (unsigned char*)controlHdr, CompressInterface::HDR_BUF_LEN); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error reading compressed dictionary store control hdr " - "for truncation" << - ": OID-" << dctnryOid << - "; DbRoot-" << root << - "; partition-" << pNum << - "; segment-" << sNum << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fTruncateDctnryFileOp.closeFile( dFile ); - - return rc; - } - - int rc1 = compress::CompressInterface::verifyHdr(controlHdr); - - if (rc1 != 0) - { - rc = ERR_COMP_VERIFY_HDRS; - - WErrorCodes ec; - std::ostringstream oss; - oss << "Error verifying compressed dictionary store ptr hdr " - "for truncation" << - ": OID-" << dctnryOid << - "; DbRoot-" << root << - "; partition-" << pNum << - "; segment-" << sNum << - "; (" << rc1 << ")"; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fTruncateDctnryFileOp.closeFile( dFile ); - - return rc; - } - - // No need to perform file truncation if the dictionary file just contains - // a single abbreviated extent. Truncating up to the nearest extent would - // actually grow the file (something we don't want to do), because we have - // not yet reserved a full extent (on disk) for this dictionary store file. - const int PSEUDO_COL_WIDTH = 8; - uint64_t numBlocks = - compress::CompressInterface::getBlockCount(controlHdr); - - if ( numBlocks == uint64_t - (INITIAL_EXTENT_ROWS_TO_DISK * PSEUDO_COL_WIDTH / BYTE_PER_BLOCK) ) - { - std::ostringstream oss1; - oss1 << "Skip truncating abbreviated dictionary file" - ": OID-" << dctnryOid << - "; DBRoot-" << root << - "; part-" << pNum << - "; seg-" << sNum << - "; blocks-" << numBlocks; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - fTruncateDctnryFileOp.closeFile( dFile ); - - return NO_ERROR; - } - - uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr); - uint64_t ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN; - char* pointerHdr = new char[ptrHdrSize]; - - rc = fTruncateDctnryFileOp.readFile(dFile, - (unsigned char*)pointerHdr, ptrHdrSize); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error reading compressed dictionary store pointer hdr " - "for truncation" << - ": OID-" << dctnryOid << - "; DbRoot-" << root << - "; partition-" << pNum << - "; segment-" << sNum << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fTruncateDctnryFileOp.closeFile( dFile ); - - delete [] pointerHdr; - return rc; - } - - CompChunkPtrList chunkPtrs; - rc1 = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize, - chunkPtrs); - delete[] pointerHdr; - - if (rc1 != 0) - { - rc = ERR_COMP_PARSE_HDRS; - - WErrorCodes ec; - std::ostringstream oss; - oss << "Error parsing compressed dictionary store ptr hdr " - "for truncation" << - ": OID-" << dctnryOid << - "; DbRoot-" << root << - "; partition-" << pNum << - "; segment-" << sNum << - "; (" << rc1 << ")"; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fTruncateDctnryFileOp.closeFile( dFile ); - - return rc; - } - - // Truncate the relevant dictionary store file to the nearest extent - if (chunkPtrs.size() > 0) - { - long long dataByteLength = chunkPtrs[chunkPtrs.size() - 1].first + - chunkPtrs[chunkPtrs.size() - 1].second - - hdrSize; - - long long extentBytes = - fRowsPerExtent * PSEUDO_COL_WIDTH; - - long long rem = dataByteLength % extentBytes; - - if (rem > 0) - { - dataByteLength = dataByteLength - rem + extentBytes; - } - - long long truncateFileSize = dataByteLength + hdrSize; - - std::ostringstream oss1; - oss1 << "Truncating dictionary file" - ": OID-" << dctnryOid << - "; DBRoot-" << root << - "; part-" << pNum << - "; seg-" << sNum << - "; size-" << truncateFileSize; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - - if (truncateFileSize > 0) - rc = fTruncateDctnryFileOp.truncateFile(dFile, truncateFileSize); - else - rc = ERR_COMP_TRUNCATE_ZERO;//@bug3913-Catch truncate to 0 bytes - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating compressed dictionary store file" - ": OID-" << dctnryOid << - "; DbRoot-" << root << - "; partition-" << pNum << - "; segment-" << sNum << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - fTruncateDctnryFileOp.closeFile( dFile ); - - return rc; - } - } - - fTruncateDctnryFileOp.closeFile( dFile ); + return rc; } - return NO_ERROR; + char controlHdr[CompressInterface::HDR_BUF_LEN]; + rc = fTruncateDctnryFileOp.readFile(dFile, (unsigned char*)controlHdr, CompressInterface::HDR_BUF_LEN); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error reading compressed dictionary store control hdr " + "for truncation" + << ": OID-" << dctnryOid << "; DbRoot-" << root << "; partition-" << pNum << "; segment-" << sNum + << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fTruncateDctnryFileOp.closeFile(dFile); + + return rc; + } + + int rc1 = compress::CompressInterface::verifyHdr(controlHdr); + + if (rc1 != 0) + { + rc = ERR_COMP_VERIFY_HDRS; + + WErrorCodes ec; + std::ostringstream oss; + oss << "Error verifying compressed dictionary store ptr hdr " + "for truncation" + << ": OID-" << dctnryOid << "; DbRoot-" << root << "; partition-" << pNum << "; segment-" << sNum + << "; (" << rc1 << ")"; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fTruncateDctnryFileOp.closeFile(dFile); + + return rc; + } + + // No need to perform file truncation if the dictionary file just contains + // a single abbreviated extent. Truncating up to the nearest extent would + // actually grow the file (something we don't want to do), because we have + // not yet reserved a full extent (on disk) for this dictionary store file. + const int PSEUDO_COL_WIDTH = 8; + uint64_t numBlocks = compress::CompressInterface::getBlockCount(controlHdr); + + if (numBlocks == uint64_t(INITIAL_EXTENT_ROWS_TO_DISK * PSEUDO_COL_WIDTH / BYTE_PER_BLOCK)) + { + std::ostringstream oss1; + oss1 << "Skip truncating abbreviated dictionary file" + ": OID-" + << dctnryOid << "; DBRoot-" << root << "; part-" << pNum << "; seg-" << sNum << "; blocks-" + << numBlocks; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + fTruncateDctnryFileOp.closeFile(dFile); + + return NO_ERROR; + } + + uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr); + uint64_t ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN; + char* pointerHdr = new char[ptrHdrSize]; + + rc = fTruncateDctnryFileOp.readFile(dFile, (unsigned char*)pointerHdr, ptrHdrSize); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error reading compressed dictionary store pointer hdr " + "for truncation" + << ": OID-" << dctnryOid << "; DbRoot-" << root << "; partition-" << pNum << "; segment-" << sNum + << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fTruncateDctnryFileOp.closeFile(dFile); + + delete[] pointerHdr; + return rc; + } + + CompChunkPtrList chunkPtrs; + rc1 = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize, chunkPtrs); + delete[] pointerHdr; + + if (rc1 != 0) + { + rc = ERR_COMP_PARSE_HDRS; + + WErrorCodes ec; + std::ostringstream oss; + oss << "Error parsing compressed dictionary store ptr hdr " + "for truncation" + << ": OID-" << dctnryOid << "; DbRoot-" << root << "; partition-" << pNum << "; segment-" << sNum + << "; (" << rc1 << ")"; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fTruncateDctnryFileOp.closeFile(dFile); + + return rc; + } + + // Truncate the relevant dictionary store file to the nearest extent + if (chunkPtrs.size() > 0) + { + long long dataByteLength = + chunkPtrs[chunkPtrs.size() - 1].first + chunkPtrs[chunkPtrs.size() - 1].second - hdrSize; + + long long extentBytes = fRowsPerExtent * PSEUDO_COL_WIDTH; + + long long rem = dataByteLength % extentBytes; + + if (rem > 0) + { + dataByteLength = dataByteLength - rem + extentBytes; + } + + long long truncateFileSize = dataByteLength + hdrSize; + + std::ostringstream oss1; + oss1 << "Truncating dictionary file" + ": OID-" + << dctnryOid << "; DBRoot-" << root << "; part-" << pNum << "; seg-" << sNum << "; size-" + << truncateFileSize; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + + if (truncateFileSize > 0) + rc = fTruncateDctnryFileOp.truncateFile(dFile, truncateFileSize); + else + rc = ERR_COMP_TRUNCATE_ZERO; //@bug3913-Catch truncate to 0 bytes + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating compressed dictionary store file" + ": OID-" + << dctnryOid << "; DbRoot-" << root << "; partition-" << pNum << "; segment-" << sNum << "; " + << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fTruncateDctnryFileOp.closeFile(dFile); + + return rc; + } + } + + fTruncateDctnryFileOp.closeFile(dFile); + } + + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Fill out existing partial extent to extent boundary, so that we can resume // inserting rows on an extent boundary basis. This use case should only take // place when a DBRoot with a partial extent has been moved from one PM to // another. //------------------------------------------------------------------------------ -int ColumnInfoCompressed::extendColumnOldExtent( - uint16_t dbRootNext, - uint32_t partitionNext, - uint16_t segmentNext, - HWM hwmNextIn ) +int ColumnInfoCompressed::extendColumnOldExtent(uint16_t dbRootNext, uint32_t partitionNext, + uint16_t segmentNext, HWM hwmNextIn) { - const unsigned int BLKS_PER_EXTENT = - (fRowsPerExtent * column.width) / BYTE_PER_BLOCK; + const unsigned int BLKS_PER_EXTENT = (fRowsPerExtent * column.width) / BYTE_PER_BLOCK; - // Round up HWM to the end of the current extent - unsigned int nBlks = hwmNextIn + 1; - unsigned int nRem = nBlks % BLKS_PER_EXTENT; - HWM hwmNext = 0; + // Round up HWM to the end of the current extent + unsigned int nBlks = hwmNextIn + 1; + unsigned int nRem = nBlks % BLKS_PER_EXTENT; + HWM hwmNext = 0; - if (nRem > 0) - hwmNext = nBlks - nRem + BLKS_PER_EXTENT - 1; - else - hwmNext = nBlks - 1; + if (nRem > 0) + hwmNext = nBlks - nRem + BLKS_PER_EXTENT - 1; + else + hwmNext = nBlks - 1; + std::ostringstream oss; + oss << "Padding compressed partial extent to extent boundary in OID-" << curCol.dataFile.fid << "; DBRoot-" + << dbRootNext << "; part-" << partitionNext << "; seg-" << segmentNext << "; hwm-" << hwmNext; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + curCol.dataFile.pFile = 0; + curCol.dataFile.fDbRoot = dbRootNext; + curCol.dataFile.fPartition = partitionNext; + curCol.dataFile.fSegment = segmentNext; + curCol.dataFile.hwm = hwmNext; + curCol.dataFile.fSegFileName.clear(); + + std::string segFileName; + std::string errTask; + int rc = colOp->fillCompColumnExtentEmptyChunks(curCol.dataFile.fid, curCol.colWidth, column.emptyVal, + curCol.dataFile.fDbRoot, curCol.dataFile.fPartition, + curCol.dataFile.fSegment, curCol.colDataType, + curCol.dataFile.hwm, segFileName, errTask); + + if (rc != NO_ERROR) + { + WErrorCodes ec; std::ostringstream oss; - oss << "Padding compressed partial extent to extent boundary in OID-" << - curCol.dataFile.fid << - "; DBRoot-" << dbRootNext << - "; part-" << partitionNext << - "; seg-" << segmentNext << - "; hwm-" << hwmNext; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + oss << "extendColumnOldExtent: error padding extent (" << errTask << "); " + << "column OID-" << curCol.dataFile.fid << "; DBRoot-" << curCol.dataFile.fDbRoot << "; part-" + << curCol.dataFile.fPartition << "; seg-" << curCol.dataFile.fSegment << "; newHwm-" + << curCol.dataFile.hwm << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); + return rc; + } - curCol.dataFile.pFile = 0; - curCol.dataFile.fDbRoot = dbRootNext; - curCol.dataFile.fPartition = partitionNext; - curCol.dataFile.fSegment = segmentNext; - curCol.dataFile.hwm = hwmNext; - curCol.dataFile.fSegFileName.clear(); + addToSegFileList(curCol.dataFile, hwmNext); - std::string segFileName; - std::string errTask; - int rc = colOp->fillCompColumnExtentEmptyChunks( - curCol.dataFile.fid, - curCol.colWidth, - column.emptyVal, - curCol.dataFile.fDbRoot, - curCol.dataFile.fPartition, - curCol.dataFile.fSegment, - curCol.colDataType, - curCol.dataFile.hwm, - segFileName, - errTask); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "extendColumnOldExtent: error padding extent (" << - errTask << "); " << - "column OID-" << curCol.dataFile.fid << - "; DBRoot-" << curCol.dataFile.fDbRoot << - "; part-" << curCol.dataFile.fPartition << - "; seg-" << curCol.dataFile.fSegment << - "; newHwm-" << curCol.dataFile.hwm << - "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - fpTableInfo->fBRMReporter.addToErrMsgEntry(oss.str()); - return rc; - } - - addToSegFileList( curCol.dataFile, hwmNext ); - - return NO_ERROR; + return NO_ERROR; } -} +} // namespace WriteEngine diff --git a/writeengine/bulk/we_columninfocompressed.h b/writeengine/bulk/we_columninfocompressed.h index 7f8ea039c..276aa22ca 100644 --- a/writeengine/bulk/we_columninfocompressed.h +++ b/writeengine/bulk/we_columninfocompressed.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_columninfocompressed.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_columninfocompressed.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file * Contains main class used to manage compressed column information. @@ -42,60 +42,50 @@ class RBMetaWriter; */ class ColumnInfoCompressed : public ColumnInfo { -public: + public: + /** @brief Constructor. + */ + ColumnInfoCompressed(Log* logger, int id, const JobColumn& column, DBRootExtentTracker* pDBRootExtTrk, + TableInfo* pTableInfo); + // RBMetaWriter* rbMetaWriter); - /** @brief Constructor. - */ - ColumnInfoCompressed(Log* logger, - int id, - const JobColumn& column, - DBRootExtentTracker* pDBRootExtTrk, - TableInfo* pTableInfo); - //RBMetaWriter* rbMetaWriter); + /** @brief Destructor + */ + virtual ~ColumnInfoCompressed(); - /** @brief Destructor - */ - virtual ~ColumnInfoCompressed(); + /** @brief Close the current Column file. + * @param bCompletedExtent are we completing an extent + * @param bAbort indicates if job is aborting and file should be + * closed without doing extra work: flushing buffer, etc. + */ + virtual int closeColumnFile(bool bCompletingExtent, bool bAbort); - /** @brief Close the current Column file. - * @param bCompletedExtent are we completing an extent - * @param bAbort indicates if job is aborting and file should be - * closed without doing extra work: flushing buffer, etc. - */ - virtual int closeColumnFile(bool bCompletingExtent, bool bAbort); + /** @brief Truncate specified dictionary file. Only applies if compressed. + * @param dctnryOid Dictionary store OID + * @param root DBRoot of relevant dictionary store segment file. + * @param pNum Partition number of relevant dictionary store segment file. + * @param sNum Segment number of relevant dictionary store segment file. + */ + virtual int truncateDctnryStore(OID dctnryOid, uint16_t root, uint32_t pNum, uint16_t sNum) const; - /** @brief Truncate specified dictionary file. Only applies if compressed. - * @param dctnryOid Dictionary store OID - * @param root DBRoot of relevant dictionary store segment file. - * @param pNum Partition number of relevant dictionary store segment file. - * @param sNum Segment number of relevant dictionary store segment file. - */ - virtual int truncateDctnryStore(OID dctnryOid, - uint16_t root, uint32_t pNum, uint16_t sNum) const; + private: + virtual int resetFileOffsetsNewExtent(const char* hdr); -private: + // Prepare initial compressed column seg file for importing of data. + // oldHWM - Current HWM prior to initial block skipping. This is only + // used for abbreviated extents, to detect when block skipping has + // caused us to require a full expanded extent. + // newHWM - Starting point for adding data after initial blockskipping + virtual int setupInitialColumnFile(HWM oldHWM, HWM newHWM); - virtual int resetFileOffsetsNewExtent(const char* hdr); - - // Prepare initial compressed column seg file for importing of data. - // oldHWM - Current HWM prior to initial block skipping. This is only - // used for abbreviated extents, to detect when block skipping has - // caused us to require a full expanded extent. - // newHWM - Starting point for adding data after initial blockskipping - virtual int setupInitialColumnFile( HWM oldHWM, HWM newHWM ); - - virtual int saveDctnryStoreHWMChunk(bool& needBackup); - virtual int extendColumnOldExtent( - uint16_t dbRootNext, - uint32_t partitionNext, - uint16_t segmentNext, - HWM hwmNext ); - - RBMetaWriter* fRBMetaWriter; - FileOp fTruncateDctnryFileOp; // Used to truncate dctnry store file + virtual int saveDctnryStoreHWMChunk(bool& needBackup); + virtual int extendColumnOldExtent(uint16_t dbRootNext, uint32_t partitionNext, uint16_t segmentNext, + HWM hwmNext); + RBMetaWriter* fRBMetaWriter; + FileOp fTruncateDctnryFileOp; // Used to truncate dctnry store file }; -} +} // namespace WriteEngine #endif diff --git a/writeengine/bulk/we_extentstripealloc.cpp b/writeengine/bulk/we_extentstripealloc.cpp index b94492f95..55e30ceaf 100644 --- a/writeengine/bulk/we_extentstripealloc.cpp +++ b/writeengine/bulk/we_extentstripealloc.cpp @@ -35,21 +35,20 @@ namespace { -typedef std::tr1::unordered_multimap AllocExtMap; -typedef AllocExtMap::iterator AllocExtMapIter; +typedef std::tr1::unordered_multimap + AllocExtMap; +typedef AllocExtMap::iterator AllocExtMapIter; typedef AllocExtMap::const_iterator ConstAllocExtMapIter; -} +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -ExtentStripeAlloc::ExtentStripeAlloc( OID tableOID, - Log* logger ) : fTableOID(tableOID), fLog(logger), fStripeCount(0) +ExtentStripeAlloc::ExtentStripeAlloc(OID tableOID, Log* logger) + : fTableOID(tableOID), fLog(logger), fStripeCount(0) { } @@ -58,7 +57,7 @@ ExtentStripeAlloc::ExtentStripeAlloc( OID tableOID, // Note: fMap will automatically get cleared by unordered_map destructor, // so no need to explicitly call clear() in "this" destructor. //------------------------------------------------------------------------------ -ExtentStripeAlloc::~ExtentStripeAlloc( ) +ExtentStripeAlloc::~ExtentStripeAlloc() { } @@ -66,14 +65,14 @@ ExtentStripeAlloc::~ExtentStripeAlloc( ) // Add a column to be associated with the "stripe" allocations for "this" // ExtentStripeAlloc object. //------------------------------------------------------------------------------ -void ExtentStripeAlloc::addColumn( OID colOID, int colWidth ) +void ExtentStripeAlloc::addColumn(OID colOID, int colWidth) { - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - fColOIDs.push_back ( colOID ); - fColWidths.push_back( colWidth ); + fColOIDs.push_back(colOID); + fColWidths.push_back(colWidth); } - + //------------------------------------------------------------------------------ // Allocate a "stripe" of column extents for the relevant table associated // with the specified column OID, and at the specified DBRoot. The partition @@ -97,210 +96,191 @@ void ExtentStripeAlloc::addColumn( OID colOID, int colWidth ) // 2. select the lowest allocated stripe, if there should be more than // one column extent with the same DBRoot. //------------------------------------------------------------------------------ -int ExtentStripeAlloc::allocateExtent( OID oid, - uint16_t dbRoot, - uint32_t& partNum, // used as input for empty DBRoot, else output only - uint16_t& segNum, - BRM::LBID_t& startLbid, - int& allocSize, - HWM& hwm, - std::string& errMsg ) +int ExtentStripeAlloc::allocateExtent(OID oid, uint16_t dbRoot, + uint32_t& partNum, // used as input for empty DBRoot, else output only + uint16_t& segNum, BRM::LBID_t& startLbid, int& allocSize, HWM& hwm, + std::string& errMsg) { - int retStatus = NO_ERROR; - bool bFound = false; - AllocExtMapIter extentEntryIter; - errMsg.clear(); + int retStatus = NO_ERROR; + bool bFound = false; + AllocExtMapIter extentEntryIter; + errMsg.clear(); - std::pair iters; + std::pair iters; - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - // Search for an extent matching the requested OID and DBRoot. - // We also filter by selecting the lowest stripe number. See - // function description that precedes this function for more detail. - iters = fMap.equal_range( oid ); + // Search for an extent matching the requested OID and DBRoot. + // We also filter by selecting the lowest stripe number. See + // function description that precedes this function for more detail. + iters = fMap.equal_range(oid); - if (iters.first != iters.second) + if (iters.first != iters.second) + { + for (AllocExtMapIter it = iters.first; it != iters.second; ++it) { - for (AllocExtMapIter it = iters.first; it != iters.second; ++it) + if (it->second.fDbRoot == dbRoot) + { + if ((!bFound) || (it->second.fStripeKey < extentEntryIter->second.fStripeKey)) { - if (it->second.fDbRoot == dbRoot) - { - if ((!bFound) || - (it->second.fStripeKey < - extentEntryIter->second.fStripeKey)) - { - extentEntryIter = it; - } - - bFound = true; - } + extentEntryIter = it; } + + bFound = true; + } + } + } + + // Return selected extent + if (bFound) + { + partNum = extentEntryIter->second.fPartNum; + segNum = extentEntryIter->second.fSegNum; + startLbid = extentEntryIter->second.fStartLbid; + allocSize = extentEntryIter->second.fAllocSize; + hwm = extentEntryIter->second.fHwm; + errMsg = extentEntryIter->second.fStatusMsg; + retStatus = extentEntryIter->second.fStatus; + + fMap.erase(extentEntryIter); + } + else // Allocate "stripe" of extents if there's no entry for this column OID + { + fStripeCount++; + + std::ostringstream oss1; + oss1 << "Allocating next stripe(" << fStripeCount << ") of column extents for table " << fTableOID + << "; DBRoot-" << dbRoot; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + + std::vector cols; + std::vector extents; + + for (unsigned int j = 0; j < fColOIDs.size(); ++j) + { + BRM::CreateStripeColumnExtentsArgIn colEntry; + colEntry.oid = fColOIDs[j]; + colEntry.width = fColWidths[j]; + cols.push_back(colEntry); } - // Return selected extent - if (bFound) + uint32_t allocPartNum = partNum; + uint16_t allocSegNum = 0; + BRM::LBID_t allocStartLbid = 0; + int allocAllocSize = 0; + HWM allocHwm = 0; + int allocStatus = NO_ERROR; + std::string allocStatusMsg; + + int rc = + BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, allocPartNum, allocSegNum, extents); + + // If allocation error occurs, we go ahead and store extent entries + // with error status, to satisfy subsequent allocations in same stripe. + if (rc != NO_ERROR) { - partNum = extentEntryIter->second.fPartNum; - segNum = extentEntryIter->second.fSegNum; - startLbid = extentEntryIter->second.fStartLbid; - allocSize = extentEntryIter->second.fAllocSize; - hwm = extentEntryIter->second.fHwm; - errMsg = extentEntryIter->second.fStatusMsg; - retStatus = extentEntryIter->second.fStatus; - - fMap.erase( extentEntryIter ); - } - else // Allocate "stripe" of extents if there's no entry for this column OID - { - fStripeCount++; - - std::ostringstream oss1; - oss1 << "Allocating next stripe(" << fStripeCount << - ") of column extents for table " << fTableOID << - "; DBRoot-" << dbRoot; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - - std::vector cols; - std::vector extents; - - for (unsigned int j = 0; j < fColOIDs.size(); ++j) + for (unsigned int i = 0; i < fColOIDs.size(); ++i) + { + if (oid != fColOIDs[i]) { - BRM::CreateStripeColumnExtentsArgIn colEntry; - colEntry.oid = fColOIDs[j]; - colEntry.width = fColWidths[j]; - cols.push_back( colEntry ); + allocStatus = rc; + + std::ostringstream oss; + oss << "Previous error allocating extent stripe for " + "table " + << fTableOID << "; DBRoot: " << dbRoot; + allocStatusMsg = oss.str(); + + // For error case, just store 0 for part#,segnum, etc. + AllocExtEntry extentEntry(fColOIDs[i], fColWidths[i], dbRoot, 0, 0, 0, 0, 0, allocStatus, + allocStatusMsg, fStripeCount); + + fMap.insert(AllocExtMap::value_type(fColOIDs[i], extentEntry)); } + } - uint32_t allocPartNum = partNum; - uint16_t allocSegNum = 0; - BRM::LBID_t allocStartLbid = 0; - int allocAllocSize = 0; - HWM allocHwm = 0; - int allocStatus = NO_ERROR; - std::string allocStatusMsg; + std::ostringstream oss; + oss << "Error allocating extent stripe for " + "table " + << fTableOID << "; DBRoot: " << dbRoot; + errMsg = oss.str(); - int rc = BRMWrapper::getInstance()->allocateStripeColExtents( - cols, dbRoot, allocPartNum, allocSegNum, extents ); - - // If allocation error occurs, we go ahead and store extent entries - // with error status, to satisfy subsequent allocations in same stripe. - if (rc != NO_ERROR) - { - for (unsigned int i = 0; i < fColOIDs.size(); ++i) - { - if (oid != fColOIDs[i]) - { - allocStatus = rc; - - std::ostringstream oss; - oss << "Previous error allocating extent stripe for " - "table " << fTableOID << "; DBRoot: " << dbRoot; - allocStatusMsg = oss.str(); - - // For error case, just store 0 for part#,segnum, etc. - AllocExtEntry extentEntry(fColOIDs[i], fColWidths[i], - dbRoot, 0, 0, 0, 0, 0, - allocStatus, allocStatusMsg, fStripeCount ); - - fMap.insert( AllocExtMap::value_type(fColOIDs[i], - extentEntry) ); - } - } - - std::ostringstream oss; - oss << "Error allocating extent stripe for " - "table " << fTableOID << "; DBRoot: " << dbRoot; - errMsg = oss.str(); - - return rc; - } - - // Save allocated extents into fMap for later use. For the OID - // requested by this function call, we just return the extent info. - for (unsigned int i = 0; i < fColOIDs.size(); ++i) - { - allocStartLbid = extents[i].startLbid; - allocAllocSize = extents[i].allocSize; - allocHwm = extents[i].startBlkOffset; - - // Might consider controlling this with debug, but we always - // log out for now. - //if (fLog->isDebug( DEBUG_1 )) - { - std::ostringstream oss; - oss << "Stripe Allocation: OID-" << fColOIDs[i] << - "; DBRoot-" << dbRoot << - "; Part#-" << allocPartNum << - "; Seg#-" << allocSegNum << - "; lbid-" << allocStartLbid << - "; fbo-" << allocHwm << - "; nblks-" << allocAllocSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - // Assign output args for requested column OID - if (oid == fColOIDs[i]) - { - partNum = allocPartNum; - segNum = allocSegNum; - startLbid = allocStartLbid; - allocSize = allocAllocSize; - hwm = allocHwm; - } - else // Add all extents in "stripe" (other than requested column) - { - // to the collection of extents - AllocExtEntry extentEntry(fColOIDs[i], fColWidths[i], - dbRoot, allocPartNum, allocSegNum, - allocStartLbid, allocAllocSize, allocHwm, - allocStatus, allocStatusMsg, fStripeCount ); - - fMap.insert( AllocExtMap::value_type(fColOIDs[i], extentEntry) ); - } - } + return rc; } - return retStatus; + // Save allocated extents into fMap for later use. For the OID + // requested by this function call, we just return the extent info. + for (unsigned int i = 0; i < fColOIDs.size(); ++i) + { + allocStartLbid = extents[i].startLbid; + allocAllocSize = extents[i].allocSize; + allocHwm = extents[i].startBlkOffset; + + // Might consider controlling this with debug, but we always + // log out for now. + // if (fLog->isDebug( DEBUG_1 )) + { + std::ostringstream oss; + oss << "Stripe Allocation: OID-" << fColOIDs[i] << "; DBRoot-" << dbRoot << "; Part#-" << allocPartNum + << "; Seg#-" << allocSegNum << "; lbid-" << allocStartLbid << "; fbo-" << allocHwm << "; nblks-" + << allocAllocSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + // Assign output args for requested column OID + if (oid == fColOIDs[i]) + { + partNum = allocPartNum; + segNum = allocSegNum; + startLbid = allocStartLbid; + allocSize = allocAllocSize; + hwm = allocHwm; + } + else // Add all extents in "stripe" (other than requested column) + { + // to the collection of extents + AllocExtEntry extentEntry(fColOIDs[i], fColWidths[i], dbRoot, allocPartNum, allocSegNum, + allocStartLbid, allocAllocSize, allocHwm, allocStatus, allocStatusMsg, + fStripeCount); + + fMap.insert(AllocExtMap::value_type(fColOIDs[i], extentEntry)); + } + } + } + + return retStatus; } - + //------------------------------------------------------------------------------ // Debug logging function to log contents of the allocated extents that are // pending. //------------------------------------------------------------------------------ -void ExtentStripeAlloc::print( ) +void ExtentStripeAlloc::print() { - boost::mutex::scoped_lock lock(fMapMutex); + boost::mutex::scoped_lock lock(fMapMutex); - std::ostringstream oss; - oss << "Current Pending Extents for table " << fTableOID << ":"; + std::ostringstream oss; + oss << "Current Pending Extents for table " << fTableOID << ":"; - if (fMap.size() > 0) + if (fMap.size() > 0) + { + for (ConstAllocExtMapIter iter = fMap.begin(); iter != fMap.end(); ++iter) { - for (ConstAllocExtMapIter iter = fMap.begin(); - iter != fMap.end(); - ++iter) - { - oss << std::endl; - oss << " oid: " << iter->second.fOid << - "; wid: " << iter->second.fColWidth << - "; root: " << iter->second.fDbRoot << - "; part: " << iter->second.fPartNum << - "; seg: " << iter->second.fSegNum << - "; lbid: " << iter->second.fStartLbid << - "; size: " << iter->second.fAllocSize << - "; hwm: " << iter->second.fHwm << - "; stripe: " << iter->second.fStripeKey << - "; stat: " << iter->second.fStatus << - "; msg: " << iter->second.fStatusMsg; - } - } - else - { - oss << " "; + oss << std::endl; + oss << " oid: " << iter->second.fOid << "; wid: " << iter->second.fColWidth + << "; root: " << iter->second.fDbRoot << "; part: " << iter->second.fPartNum + << "; seg: " << iter->second.fSegNum << "; lbid: " << iter->second.fStartLbid + << "; size: " << iter->second.fAllocSize << "; hwm: " << iter->second.fHwm + << "; stripe: " << iter->second.fStripeKey << "; stat: " << iter->second.fStatus + << "; msg: " << iter->second.fStatusMsg; } + } + else + { + oss << " "; + } - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + fLog->logMsg(oss.str(), MSGLVL_INFO2); } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/bulk/we_extentstripealloc.h b/writeengine/bulk/we_extentstripealloc.h index 1709c4c2b..3b1bfa108 100644 --- a/writeengine/bulk/we_extentstripealloc.h +++ b/writeengine/bulk/we_extentstripealloc.h @@ -50,48 +50,51 @@ class Log; //------------------------------------------------------------------------------ class AllocExtEntry { -public: - // Default constructor - AllocExtEntry() : fOid(0), - fColWidth(0), - fDbRoot(0), - fPartNum(0), - fSegNum(0), - fStartLbid(0), - fAllocSize(0), - fHwm(0), - fStatus(NO_ERROR), - fStripeKey(0) { } + public: + // Default constructor + AllocExtEntry() + : fOid(0) + , fColWidth(0) + , fDbRoot(0) + , fPartNum(0) + , fSegNum(0) + , fStartLbid(0) + , fAllocSize(0) + , fHwm(0) + , fStatus(NO_ERROR) + , fStripeKey(0) + { + } - // Used to create entry for an existing extent we are going to add data to. - AllocExtEntry ( OID& oid, int colWidth, - uint16_t dbRoot, uint32_t partNum, uint16_t segNum, - BRM::LBID_t startLbid, int allocSize, - HWM hwm, int status, const std::string& statusMsg, - unsigned int stripeKey ) : - fOid(oid), - fColWidth(colWidth), - fDbRoot(dbRoot), - fPartNum(partNum), - fSegNum(segNum), - fStartLbid(startLbid), - fAllocSize(allocSize), - fHwm(hwm), - fStatus(status), - fStatusMsg(statusMsg), - fStripeKey(stripeKey) { } + // Used to create entry for an existing extent we are going to add data to. + AllocExtEntry(OID& oid, int colWidth, uint16_t dbRoot, uint32_t partNum, uint16_t segNum, + BRM::LBID_t startLbid, int allocSize, HWM hwm, int status, const std::string& statusMsg, + unsigned int stripeKey) + : fOid(oid) + , fColWidth(colWidth) + , fDbRoot(dbRoot) + , fPartNum(partNum) + , fSegNum(segNum) + , fStartLbid(startLbid) + , fAllocSize(allocSize) + , fHwm(hwm) + , fStatus(status) + , fStatusMsg(statusMsg) + , fStripeKey(stripeKey) + { + } - OID fOid; // column OID - int fColWidth; // colum width (in bytes) - uint16_t fDbRoot; // DBRoot of allocated extent - uint32_t fPartNum; // Partition number of allocated extent - uint16_t fSegNum; // Segment number of allocated extent - BRM::LBID_t fStartLbid; // Starting LBID of allocated extent - int fAllocSize; // Number of allocated LBIDS - HWM fHwm; // Starting fbo or hwm of allocated extent - int fStatus; // Status of extent allocation - std::string fStatusMsg; // Status msg of extent allocation - unsigned int fStripeKey;// "Stripe" identifier for this extent + OID fOid; // column OID + int fColWidth; // colum width (in bytes) + uint16_t fDbRoot; // DBRoot of allocated extent + uint32_t fPartNum; // Partition number of allocated extent + uint16_t fSegNum; // Segment number of allocated extent + BRM::LBID_t fStartLbid; // Starting LBID of allocated extent + int fAllocSize; // Number of allocated LBIDS + HWM fHwm; // Starting fbo or hwm of allocated extent + int fStatus; // Status of extent allocation + std::string fStatusMsg; // Status msg of extent allocation + unsigned int fStripeKey; // "Stripe" identifier for this extent }; //------------------------------------------------------------------------------ @@ -101,10 +104,10 @@ public: //------------------------------------------------------------------------------ struct AllocExtHasher : public std::unary_function { - std::size_t operator()(OID val) const - { - return static_cast(val); - } + std::size_t operator()(OID val) const + { + return static_cast(val); + } }; //------------------------------------------------------------------------------ @@ -115,70 +118,61 @@ struct AllocExtHasher : public std::unary_function //------------------------------------------------------------------------------ class ExtentStripeAlloc { -public: + public: + /** @brief Constructor + * @param tableOID OID of table for which extents will be allocated. + * @param logger Log object used for debug logging. + */ + ExtentStripeAlloc(OID tableOID, Log* logger); - /** @brief Constructor - * @param tableOID OID of table for which extents will be allocated. - * @param logger Log object used for debug logging. - */ - ExtentStripeAlloc ( OID tableOID, - Log* logger ); + /** @brief Destructor + */ + ~ExtentStripeAlloc(); - /** @brief Destructor - */ - ~ExtentStripeAlloc( ); + /** @brief Add the specified column to our "stripe" of extents to allocate. + * @param colOID Column OID to be added to extent allocation list. + * @param colWidth Width of column associated with colOID. + */ + void addColumn(OID colOID, int colWidth); - /** @brief Add the specified column to our "stripe" of extents to allocate. - * @param colOID Column OID to be added to extent allocation list. - * @param colWidth Width of column associated with colOID. - */ - void addColumn( OID colOID, - int colWidth ); + /** @brief Request an extent allocation for the specified OID and DBRoot. + * A "stripe" of extents for the corresponding table will be allocated + * if no extent exists to satisfy the request; else the previously + * allocated extent will be returned. + * @param oid Column OID extent to be allocated + * @param dbRoot Requested DBRoot for the new extent + * @param partNum (in/out) Partition number of the allocated extent, + * Input if empty DBRoot, else only used as output. + * @param segNum (out) Segment number of the allocated extent + * @param startLbid (out) Starting LBID of the allocated extent + * @param allocSize (out) Number of blocks in the allocated extent + * @param hwm (out) Starting FBO or hwm for the allocated extent + * @param errMsg (out) Error msg associated with the extent allocation + * @return NO_ERROR returned upon success + */ + int allocateExtent(OID oid, uint16_t dbRoot, uint32_t& partNum, uint16_t& segNum, BRM::LBID_t& startLbid, + int& allocSize, HWM& hwm, std::string& errMsg); - /** @brief Request an extent allocation for the specified OID and DBRoot. - * A "stripe" of extents for the corresponding table will be allocated - * if no extent exists to satisfy the request; else the previously - * allocated extent will be returned. - * @param oid Column OID extent to be allocated - * @param dbRoot Requested DBRoot for the new extent - * @param partNum (in/out) Partition number of the allocated extent, - * Input if empty DBRoot, else only used as output. - * @param segNum (out) Segment number of the allocated extent - * @param startLbid (out) Starting LBID of the allocated extent - * @param allocSize (out) Number of blocks in the allocated extent - * @param hwm (out) Starting FBO or hwm for the allocated extent - * @param errMsg (out) Error msg associated with the extent allocation - * @return NO_ERROR returned upon success - */ - int allocateExtent( OID oid, - uint16_t dbRoot, - uint32_t& partNum, - uint16_t& segNum, - BRM::LBID_t& startLbid, - int& allocSize, - HWM& hwm, - std::string& errMsg ); + /** @brief Debug print function. + */ + void print(); - /** @brief Debug print function. - */ - void print( ); + private: + OID fTableOID; // Table extents to be allocated + Log* fLog; // Log used for debug logging + unsigned int fStripeCount; // Extent "stripe" counter + boost::mutex fMapMutex; // protects unordered map access + std::vector fColOIDs; // Vector of column OIDs + std::vector fColWidths; // Widths associated with fColOIDs -private: - OID fTableOID; // Table extents to be allocated - Log* fLog; // Log used for debug logging - unsigned int fStripeCount; // Extent "stripe" counter - boost::mutex fMapMutex; // protects unordered map access - std::vector fColOIDs; // Vector of column OIDs - std::vector fColWidths; // Widths associated with fColOIDs + // unordered map where we collect the allocated extents + std::tr1::unordered_multimap fMap; - // unordered map where we collect the allocated extents - std::tr1::unordered_multimap fMap; - - // disable copy constructor and assignment operator - ExtentStripeAlloc(const ExtentStripeAlloc&); - ExtentStripeAlloc& operator=(const ExtentStripeAlloc&); + // disable copy constructor and assignment operator + ExtentStripeAlloc(const ExtentStripeAlloc&); + ExtentStripeAlloc& operator=(const ExtentStripeAlloc&); }; -} //end of namespace +} // namespace WriteEngine -#endif // WE_EXTENTSTRIPEALLOC_H_ +#endif // WE_EXTENTSTRIPEALLOC_H_ diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 25de7047a..29dc7b202 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_tableinfo.cpp 4648 2013-05-29 21:42:40Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_tableinfo.cpp 4648 2013-05-29 21:42:40Z rdempsey $ + * + *******************************************************************************/ /** @file */ #include "we_tableinfo.h" @@ -61,40 +61,35 @@ using namespace querytele; namespace { -const std::string BAD_FILE_SUFFIX = ".bad"; // Reject data file suffix -const std::string ERR_FILE_SUFFIX = ".err"; // Job error file suffix -const std::string BOLD_START = "\033[0;1m"; -const std::string BOLD_STOP = "\033[0;39m"; -} +const std::string BAD_FILE_SUFFIX = ".bad"; // Reject data file suffix +const std::string ERR_FILE_SUFFIX = ".err"; // Job error file suffix +const std::string BOLD_START = "\033[0;1m"; +const std::string BOLD_STOP = "\033[0;39m"; +} // namespace namespace WriteEngine { // Helpers -int TableInfo::compareHWMs(const int smallestColumnId, - const int widerColumnId, - const uint32_t smallerColumnWidth, - const uint32_t widerColumnWidth, - const std::vector& segFileInfo, - int& colIdx) +int TableInfo::compareHWMs(const int smallestColumnId, const int widerColumnId, + const uint32_t smallerColumnWidth, const uint32_t widerColumnWidth, + const std::vector& segFileInfo, int& colIdx) { - int rc = NO_ERROR; - if (widerColumnId < 0) - { - return rc; - } - uint32_t columnDiffMultiplier = widerColumnWidth / smallerColumnWidth; - HWM hwmLo = segFileInfo[smallestColumnId].fLocalHwm * columnDiffMultiplier; - HWM hwmHi = hwmLo + columnDiffMultiplier - 1; - - if ((segFileInfo[widerColumnId].fLocalHwm < hwmLo) || - (segFileInfo[widerColumnId].fLocalHwm > hwmHi)) - { - colIdx = widerColumnId; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - } + int rc = NO_ERROR; + if (widerColumnId < 0) + { return rc; -} + } + uint32_t columnDiffMultiplier = widerColumnWidth / smallerColumnWidth; + HWM hwmLo = segFileInfo[smallestColumnId].fLocalHwm * columnDiffMultiplier; + HWM hwmHi = hwmLo + columnDiffMultiplier - 1; + if ((segFileInfo[widerColumnId].fLocalHwm < hwmLo) || (segFileInfo[widerColumnId].fLocalHwm > hwmHi)) + { + colIdx = widerColumnId; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + } + return rc; +} //------------------------------------------------------------------------------ // Puts the current thread to sleep for the specified number of milliseconds. @@ -102,21 +97,20 @@ int TableInfo::compareHWMs(const int smallestColumnId, //------------------------------------------------------------------------------ void TableInfo::sleepMS(long ms) { - struct timespec rm_ts; + struct timespec rm_ts; - rm_ts.tv_sec = ms / 1000; - rm_ts.tv_nsec = ms % 1000 * 1000000; + rm_ts.tv_sec = ms / 1000; + rm_ts.tv_nsec = ms % 1000 * 1000000; #ifdef _MSC_VER - Sleep(ms); + Sleep(ms); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif } @@ -124,55 +118,66 @@ void TableInfo::sleepMS(long ms) //------------------------------------------------------------------------------ // TableInfo constructor //------------------------------------------------------------------------------ -TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, - const string& processName, - OID tableOID, - const string& tableName, - bool bKeepRbMetaFile) : - fTableId(-1), fBufferSize(0), fFileBufSize(0), - fStatusTI(WriteEngine::NEW), - fReadBufCount(0), fNumberOfColumns(0), - fHandle(NULL), fCurrentReadBuffer(0), fTotalReadRows(0), - fTotalErrRows(0), fMaxErrorRows(5), - fLastBufferId(-1), fFileBuffer(NULL), fCurrentParseBuffer(0), - fNumberOfColsParsed(0), fLocker(-1), fTableName(tableName), - fTableOID(tableOID), fJobId(0), fLog(logger), fTxnID(txnID), - fRBMetaWriter(processName, logger), - fProcessName(processName), - fKeepRbMetaFile(bKeepRbMetaFile), - fbTruncationAsError(false), - fImportDataMode(IMPORT_DATA_TEXT), - fTimeZone("SYSTEM"), - fTableLocked(false), - fReadFromStdin(false), - fReadFromS3(false), - fNullStringMode(false), - fEnclosedByChar('\0'), - fEscapeChar('\\'), - fProcessingBegun(false), - fBulkMode(BULK_MODE_LOCAL), - fBRMReporter(logger, tableName), - fTableLockID(0), - fRejectDataCnt(0), - fRejectErrCnt(0), - fExtentStrAlloc(tableOID, logger), - fOamCachePtr(oam::OamCache::makeOamCache()) +TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, const string& processName, OID tableOID, + const string& tableName, bool bKeepRbMetaFile) + : fTableId(-1) + , fBufferSize(0) + , fFileBufSize(0) + , fStatusTI(WriteEngine::NEW) + , fReadBufCount(0) + , fNumberOfColumns(0) + , fHandle(NULL) + , fCurrentReadBuffer(0) + , fTotalReadRows(0) + , fTotalErrRows(0) + , fMaxErrorRows(5) + , fLastBufferId(-1) + , fFileBuffer(NULL) + , fCurrentParseBuffer(0) + , fNumberOfColsParsed(0) + , fLocker(-1) + , fTableName(tableName) + , fTableOID(tableOID) + , fJobId(0) + , fLog(logger) + , fTxnID(txnID) + , fRBMetaWriter(processName, logger) + , fProcessName(processName) + , fKeepRbMetaFile(bKeepRbMetaFile) + , fbTruncationAsError(false) + , fImportDataMode(IMPORT_DATA_TEXT) + , fTimeZone("SYSTEM") + , fTableLocked(false) + , fReadFromStdin(false) + , fReadFromS3(false) + , fNullStringMode(false) + , fEnclosedByChar('\0') + , fEscapeChar('\\') + , fProcessingBegun(false) + , fBulkMode(BULK_MODE_LOCAL) + , fBRMReporter(logger, tableName) + , fTableLockID(0) + , fRejectDataCnt(0) + , fRejectErrCnt(0) + , fExtentStrAlloc(tableOID, logger) + , fOamCachePtr(oam::OamCache::makeOamCache()) { - fBuffers.clear(); - fColumns.clear(); - fStartTime.tv_sec = 0; - fStartTime.tv_usec = 0; - string teleServerHost(config::Config::makeConfig()->getConfig("QueryTele", "Host")); + fBuffers.clear(); + fColumns.clear(); + fStartTime.tv_sec = 0; + fStartTime.tv_usec = 0; + string teleServerHost(config::Config::makeConfig()->getConfig("QueryTele", "Host")); - if (!teleServerHost.empty()) + if (!teleServerHost.empty()) + { + int teleServerPort = + config::Config::fromText(config::Config::makeConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) { - int teleServerPort = config::Config::fromText(config::Config::makeConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - fQtc.serverParms(QueryTeleServerParms(teleServerHost, teleServerPort)); - } + fQtc.serverParms(QueryTeleServerParms(teleServerHost, teleServerPort)); } + } } //------------------------------------------------------------------------------ @@ -180,8 +185,8 @@ TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, //------------------------------------------------------------------------------ TableInfo::~TableInfo() { - fBRMReporter.sendErrMsgToFile(fBRMRptFileName); - freeProcessingBuffers(); + fBRMReporter.sendErrMsgToFile(fBRMRptFileName); + freeProcessingBuffers(); } //------------------------------------------------------------------------------ @@ -194,12 +199,12 @@ TableInfo::~TableInfo() //------------------------------------------------------------------------------ void TableInfo::freeProcessingBuffers() { - // fLog->logMsg( - // string("Releasing TableInfo Buffer for ")+fTableName, - // MSGLVL_INFO1); - fBuffers.clear(); - fColumns.clear(); - fNumberOfColumns = 0; + // fLog->logMsg( + // string("Releasing TableInfo Buffer for ")+fTableName, + // MSGLVL_INFO1); + fBuffers.clear(); + fColumns.clear(); + fNumberOfColumns = 0; } //------------------------------------------------------------------------------ @@ -219,30 +224,27 @@ void TableInfo::freeProcessingBuffers() //------------------------------------------------------------------------------ void TableInfo::closeOpenDbFiles() { - ostringstream oss; - oss << "Closing DB files for table " << fTableName << - ", left open by abnormal termination."; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + ostringstream oss; + oss << "Closing DB files for table " << fTableName << ", left open by abnormal termination."; + fLog->logMsg(oss.str(), MSGLVL_INFO2); - for (unsigned int k = 0; k < fColumns.size(); k++) + for (unsigned int k = 0; k < fColumns.size(); k++) + { + stringstream oss1; + oss1 << "Closing DB column file for: " << fColumns[k].column.colName << " (OID-" + << fColumns[k].column.mapOid << ")"; + fLog->logMsg(oss1.str(), MSGLVL_INFO2); + fColumns[k].closeColumnFile(false, true); + + if (fColumns[k].column.colType == COL_TYPE_DICT) { - stringstream oss1; - oss1 << "Closing DB column file for: " << - fColumns[k].column.colName << - " (OID-" << fColumns[k].column.mapOid << ")"; - fLog->logMsg( oss1.str(), MSGLVL_INFO2 ); - fColumns[k].closeColumnFile(false, true); - - if (fColumns[k].column.colType == COL_TYPE_DICT) - { - stringstream oss2; - oss2 << "Closing DB store file for: " << - fColumns[k].column.colName << - " (OID-" << fColumns[k].column.dctnry.dctnryOid << ")"; - fLog->logMsg( oss2.str(), MSGLVL_INFO2 ); - fColumns[k].closeDctnryStore(true); - } + stringstream oss2; + oss2 << "Closing DB store file for: " << fColumns[k].column.colName << " (OID-" + << fColumns[k].column.dctnry.dctnryOid << ")"; + fLog->logMsg(oss2.str(), MSGLVL_INFO2); + fColumns[k].closeDctnryStore(true); } + } } //------------------------------------------------------------------------------ @@ -251,377 +253,366 @@ void TableInfo::closeOpenDbFiles() //------------------------------------------------------------------------------ bool TableInfo::lockForRead(const int& locker) { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); + boost::mutex::scoped_lock lock(fSyncUpdatesTI); - if (fLocker == -1) + if (fLocker == -1) + { + if (fStatusTI == WriteEngine::NEW) { - if (fStatusTI == WriteEngine::NEW ) - { - fLocker = locker; - return true; - } + fLocker = locker; + return true; } + } - return false; + return false; } //------------------------------------------------------------------------------ // Loop thru reading the import file(s) assigned to this TableInfo object. //------------------------------------------------------------------------------ -int TableInfo::readTableData( ) +int TableInfo::readTableData() { - RID validTotalRows = 0; - RID totalRowsPerInputFile = 0; - int filesTBProcessed = fLoadFileList.size(); - int fileCounter = 0; - unsigned long long qtSentAt = 0; + RID validTotalRows = 0; + RID totalRowsPerInputFile = 0; + int filesTBProcessed = fLoadFileList.size(); + int fileCounter = 0; + unsigned long long qtSentAt = 0; - if (fHandle == NULL) + if (fHandle == NULL) + { + fFileName = fLoadFileList[fileCounter]; + int rc = openTableFile(); + + if (rc != NO_ERROR) { - - fFileName = fLoadFileList[fileCounter]; - int rc = openTableFile(); - - if (rc != NO_ERROR) - { - // Mark the table status as error and exit. - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStatusTI = WriteEngine::ERR; - return rc; - } - - fileCounter++; + // Mark the table status as error and exit. + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStatusTI = WriteEngine::ERR; + return rc; } - timeval readStart; - gettimeofday(&readStart, NULL); - ostringstream ossStartMsg; - ossStartMsg << "Start reading and loading table " << fTableName; - fLog->logMsg( ossStartMsg.str(), MSGLVL_INFO2 ); - fProcessingBegun = true; + fileCounter++; + } - ImportTeleStats its; - its.job_uuid = fJobUUID; - its.import_uuid = QueryTeleClient::genUUID(); - its.msg_type = ImportTeleStats::IT_START; - its.start_time = QueryTeleClient::timeNowms(); - its.table_list.push_back(fTableName); - its.rows_so_far.push_back(0); - its.system_name = fOamCachePtr->getSystemName(); - its.module_name = fOamCachePtr->getModuleName(); - string tn = getTableName(); - its.schema_name = string(tn, 0, tn.find('.')); - fQtc.postImportTele(its); + timeval readStart; + gettimeofday(&readStart, NULL); + ostringstream ossStartMsg; + ossStartMsg << "Start reading and loading table " << fTableName; + fLog->logMsg(ossStartMsg.str(), MSGLVL_INFO2); + fProcessingBegun = true; - // - // LOOP to read all the import data for this table - // - while (true) + ImportTeleStats its; + its.job_uuid = fJobUUID; + its.import_uuid = QueryTeleClient::genUUID(); + its.msg_type = ImportTeleStats::IT_START; + its.start_time = QueryTeleClient::timeNowms(); + its.table_list.push_back(fTableName); + its.rows_so_far.push_back(0); + its.system_name = fOamCachePtr->getSystemName(); + its.module_name = fOamCachePtr->getModuleName(); + string tn = getTableName(); + its.schema_name = string(tn, 0, tn.find('.')); + fQtc.postImportTele(its); + + // + // LOOP to read all the import data for this table + // + while (true) + { + // See if JobStatus has been set to terminate by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) { - // See if JobStatus has been set to terminate by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStartTime = readStart; - fStatusTI = WriteEngine::ERR; - its.msg_type = ImportTeleStats::IT_TERM; - its.rows_so_far.pop_back(); - its.rows_so_far.push_back(0); - fQtc.postImportTele(its); - throw SecondaryShutdownException( "TableInfo::" - "readTableData(1) responding to job termination"); - } + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStartTime = readStart; + fStatusTI = WriteEngine::ERR; + its.msg_type = ImportTeleStats::IT_TERM; + its.rows_so_far.pop_back(); + its.rows_so_far.push_back(0); + fQtc.postImportTele(its); + throw SecondaryShutdownException( + "TableInfo::" + "readTableData(1) responding to job termination"); + } // @bug 3271: Conditionally compile the thread deadlock debug logging #ifdef DEADLOCK_DEBUG - // @bug2099+. Temp hack to diagnose deadlock. - struct timeval tvStart; - gettimeofday(&tvStart, 0); - bool report = false; - bool reported = false; - // @bug2099- + // @bug2099+. Temp hack to diagnose deadlock. + struct timeval tvStart; + gettimeofday(&tvStart, 0); + bool report = false; + bool reported = false; + // @bug2099- #else - const bool report = false; + const bool report = false; #endif #ifdef PROFILE - Stats::startReadEvent(WE_STATS_WAIT_FOR_READ_BUF); + Stats::startReadEvent(WE_STATS_WAIT_FOR_READ_BUF); #endif - // - // LOOP to wait for, and read, the next avail BulkLoadBuffer object - // - while (!isBufferAvailable(report)) - { - // See if JobStatus has been set to terminate by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStartTime = readStart; - fStatusTI = WriteEngine::ERR; - its.msg_type = ImportTeleStats::IT_TERM; - its.rows_so_far.pop_back(); - its.rows_so_far.push_back(0); - fQtc.postImportTele(its); - throw SecondaryShutdownException( "TableInfo::" - "readTableData(2) responding to job termination"); - } + // + // LOOP to wait for, and read, the next avail BulkLoadBuffer object + // + while (!isBufferAvailable(report)) + { + // See if JobStatus has been set to terminate by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) + { + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStartTime = readStart; + fStatusTI = WriteEngine::ERR; + its.msg_type = ImportTeleStats::IT_TERM; + its.rows_so_far.pop_back(); + its.rows_so_far.push_back(0); + fQtc.postImportTele(its); + throw SecondaryShutdownException( + "TableInfo::" + "readTableData(2) responding to job termination"); + } - // Sleep and check the condition again. - sleepMS(1); + // Sleep and check the condition again. + sleepMS(1); #ifdef DEADLOCK_DEBUG - // @bug2099+ - if (report) report = false; // report one time. + // @bug2099+ + if (report) + report = false; // report one time. - if (!reported) - { - struct timeval tvNow; - gettimeofday(&tvNow, 0); + if (!reported) + { + struct timeval tvNow; + gettimeofday(&tvNow, 0); - if ((tvNow.tv_sec - tvStart.tv_sec) > 100) - { - time_t t = time(0); - char timeString[50]; - ctime_r(&t, timeString); - timeString[ strlen(timeString) - 1 ] = '\0'; - ostringstream oss; - oss << endl << timeString << ": " << - "TableInfo::readTableData: " << fTableName << - "; Diff is " << (tvNow.tv_sec - tvStart.tv_sec) << - endl; - cout << oss.str(); - cout.flush(); - report = true; - reported = true; - } - } - - // @bug2099- -#endif + if ((tvNow.tv_sec - tvStart.tv_sec) > 100) + { + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream oss; + oss << endl + << timeString << ": " + << "TableInfo::readTableData: " << fTableName << "; Diff is " << (tvNow.tv_sec - tvStart.tv_sec) + << endl; + cout << oss.str(); + cout.flush(); + report = true; + reported = true; } + } + + // @bug2099- +#endif + } #ifdef PROFILE - Stats::stopReadEvent(WE_STATS_WAIT_FOR_READ_BUF); - Stats::startReadEvent(WE_STATS_READ_INTO_BUF); + Stats::stopReadEvent(WE_STATS_WAIT_FOR_READ_BUF); + Stats::startReadEvent(WE_STATS_READ_INTO_BUF); #endif - int readBufNo = fCurrentReadBuffer; - int prevReadBuf = (fCurrentReadBuffer - 1); + int readBufNo = fCurrentReadBuffer; + int prevReadBuf = (fCurrentReadBuffer - 1); - if (prevReadBuf < 0) - prevReadBuf = fReadBufCount + prevReadBuf; + if (prevReadBuf < 0) + prevReadBuf = fReadBufCount + prevReadBuf; - // We keep a running total of read errors; fMaxErrorRows specifies - // the error limit. Here's where we see how many more errors we - // still have below the limit, and we pass this to fillFromFile(). - unsigned allowedErrCntThisCall = - ( (fMaxErrorRows > fTotalErrRows) ? - (fMaxErrorRows - fTotalErrRows) : 0 ); + // We keep a running total of read errors; fMaxErrorRows specifies + // the error limit. Here's where we see how many more errors we + // still have below the limit, and we pass this to fillFromFile(). + unsigned allowedErrCntThisCall = ((fMaxErrorRows > fTotalErrRows) ? (fMaxErrorRows - fTotalErrRows) : 0); - // Fill in the specified buffer. - // fTotalReadRowsPerInputFile is ongoing total number of rows read, - // per input file. - // validTotalRows is ongoing total of valid rows read for all files - // pertaining to this DB table. - int readRc; - if (fReadFromS3) + // Fill in the specified buffer. + // fTotalReadRowsPerInputFile is ongoing total number of rows read, + // per input file. + // validTotalRows is ongoing total of valid rows read for all files + // pertaining to this DB table. + int readRc; + if (fReadFromS3) + { + readRc = fBuffers[readBufNo].fillFromMemory(fBuffers[prevReadBuf], fFileBuffer, fS3ReadLength, + &fS3ParseLength, totalRowsPerInputFile, validTotalRows, + fColumns, allowedErrCntThisCall); + } + else + { + readRc = fBuffers[readBufNo].fillFromFile(fBuffers[prevReadBuf], fHandle, totalRowsPerInputFile, + validTotalRows, fColumns, allowedErrCntThisCall); + } + + if (readRc != NO_ERROR) + { + // error occurred. + // need to exit. + // mark the table status as error and exit. + { + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStartTime = readStart; + fStatusTI = WriteEngine::ERR; + fBuffers[readBufNo].setStatusBLB(WriteEngine::ERR); + } + closeTableFile(); + + // Error occurred on next row not read, so increment + // totalRowsPerInputFile row count for the error msg + WErrorCodes ec; + ostringstream oss; + oss << "Error reading import file " << fFileName << "; near line " << totalRowsPerInputFile + 1 << "; " + << ec.errorString(readRc); + fLog->logMsg(oss.str(), readRc, MSGLVL_ERROR); + + its.msg_type = ImportTeleStats::IT_TERM; + its.rows_so_far.pop_back(); + its.rows_so_far.push_back(0); + fQtc.postImportTele(its); + + return readRc; + } + +#ifdef PROFILE + Stats::stopReadEvent(WE_STATS_READ_INTO_BUF); +#endif + its.msg_type = ImportTeleStats::IT_PROGRESS; + its.rows_so_far.pop_back(); + its.rows_so_far.push_back(totalRowsPerInputFile); + unsigned long long thisRows = static_cast(totalRowsPerInputFile); + thisRows /= 1000000; + + if (thisRows > qtSentAt) + { + fQtc.postImportTele(its); + qtSentAt = thisRows; + } + + // Check if there were any errors in the read data. + // if yes, copy it to the error list. + // if the number of errors is greater than the maximum error count + // mark the table status as error and exit. + // call the method to copy the errors + writeErrorList(&fBuffers[readBufNo].getErrorRows(), &fBuffers[readBufNo].getExactErrorRows(), false); + fBuffers[readBufNo].clearErrRows(); + + if (fTotalErrRows > fMaxErrorRows) + { + // flush the reject data file and output the rejected rows + // flush err file and output the rejected row id and the reason. + writeErrorList(0, 0, true); + + // number of errors > maximum allowed. hence return error. + { + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStartTime = readStart; + fStatusTI = WriteEngine::ERR; + fBuffers[readBufNo].setStatusBLB(WriteEngine::ERR); + } + closeTableFile(); + ostringstream oss5; + oss5 << "Actual error row count(" << fTotalErrRows << ") exceeds the max error rows(" << fMaxErrorRows + << ") allowed for table " << fTableName; + fLog->logMsg(oss5.str(), ERR_BULK_MAX_ERR_NUM, MSGLVL_ERROR); + + // List Err and Bad files to report file (if applicable) + fBRMReporter.rptMaxErrJob(fBRMRptFileName, fErrFiles, fBadFiles); + + its.msg_type = ImportTeleStats::IT_TERM; + its.rows_so_far.pop_back(); + its.rows_so_far.push_back(0); + fQtc.postImportTele(its); + + return ERR_BULK_MAX_ERR_NUM; + } + + // mark the buffer status as read complete. + { +#ifdef PROFILE + Stats::startReadEvent(WE_STATS_WAIT_TO_COMPLETE_READ); +#endif + boost::mutex::scoped_lock lock(fSyncUpdatesTI); +#ifdef PROFILE + Stats::stopReadEvent(WE_STATS_WAIT_TO_COMPLETE_READ); + Stats::startReadEvent(WE_STATS_COMPLETING_READ); +#endif + + fStartTime = readStart; + fBuffers[readBufNo].setStatusBLB(WriteEngine::READ_COMPLETE); + + fCurrentReadBuffer = (fCurrentReadBuffer + 1) % fReadBufCount; + + // bufferCount++; + if ((fHandle && feof(fHandle)) || (fReadFromS3 && (fS3ReadLength == fS3ParseLength))) + { + timeval readFinished; + gettimeofday(&readFinished, NULL); + + closeTableFile(); + + if (fReadFromStdin) { - readRc = fBuffers[readBufNo].fillFromMemory( - fBuffers[prevReadBuf], fFileBuffer, fS3ReadLength, &fS3ParseLength, - totalRowsPerInputFile, validTotalRows, fColumns, - allowedErrCntThisCall); + fLog->logMsg("Finished loading " + fTableName + " from STDIN" + ", Time taken = " + + Convertor::int2Str((int)(readFinished.tv_sec - readStart.tv_sec)) + " seconds", + //" seconds; bufferCount-"+Convertor::int2Str(bufferCount), + MSGLVL_INFO2); + } + else if (fReadFromS3) + { + fLog->logMsg("Finished loading " + fTableName + " from S3" + ", Time taken = " + + Convertor::int2Str((int)(readFinished.tv_sec - readStart.tv_sec)) + " seconds", + //" seconds; bufferCount-"+Convertor::int2Str(bufferCount), + MSGLVL_INFO2); } else { - readRc = fBuffers[readBufNo].fillFromFile( - fBuffers[prevReadBuf], fHandle, totalRowsPerInputFile, - validTotalRows, fColumns, allowedErrCntThisCall); + fLog->logMsg("Finished reading file " + fFileName + ", Time taken = " + + Convertor::int2Str((int)(readFinished.tv_sec - readStart.tv_sec)) + " seconds", + //" seconds; bufferCount-"+Convertor::int2Str(bufferCount), + MSGLVL_INFO2); } - if (readRc != NO_ERROR) + // flush the reject data file and output the rejected rows + // flush err file and output the rejected row id and the reason. + writeErrorList(0, 0, true); + + // If > 1 file for this table, then open next file in the list + if (fileCounter < filesTBProcessed) { - // error occurred. - // need to exit. - // mark the table status as error and exit. - { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStartTime = readStart; - fStatusTI = WriteEngine::ERR; - fBuffers[readBufNo].setStatusBLB(WriteEngine::ERR); - } - closeTableFile(); + fFileName = fLoadFileList[fileCounter]; + int rc = openTableFile(); - // Error occurred on next row not read, so increment - // totalRowsPerInputFile row count for the error msg - WErrorCodes ec; - ostringstream oss; - oss << "Error reading import file " << fFileName << - "; near line " << totalRowsPerInputFile + 1 << "; " << - ec.errorString(readRc); - fLog->logMsg( oss.str(), readRc, MSGLVL_ERROR); + if (rc != NO_ERROR) + { + // Mark the table status as error and exit. + fStatusTI = WriteEngine::ERR; + return rc; + } - its.msg_type = ImportTeleStats::IT_TERM; - its.rows_so_far.pop_back(); - its.rows_so_far.push_back(0); - fQtc.postImportTele(its); - - return readRc; + fileCounter++; + fTotalReadRows += totalRowsPerInputFile; + totalRowsPerInputFile = 0; + } + else // All files read for this table; break out of read loop + { + fStatusTI = WriteEngine::READ_COMPLETE; + fLastBufferId = readBufNo; + fTotalReadRows += totalRowsPerInputFile; + break; } -#ifdef PROFILE - Stats::stopReadEvent(WE_STATS_READ_INTO_BUF); -#endif - its.msg_type = ImportTeleStats::IT_PROGRESS; - its.rows_so_far.pop_back(); - its.rows_so_far.push_back(totalRowsPerInputFile); - unsigned long long thisRows = static_cast(totalRowsPerInputFile); - thisRows /= 1000000; - - if (thisRows > qtSentAt) - { - fQtc.postImportTele(its); - qtSentAt = thisRows; - } - - // Check if there were any errors in the read data. - // if yes, copy it to the error list. - // if the number of errors is greater than the maximum error count - // mark the table status as error and exit. - // call the method to copy the errors - writeErrorList( &fBuffers[readBufNo].getErrorRows(), - &fBuffers[readBufNo].getExactErrorRows(), false ); - fBuffers[readBufNo].clearErrRows(); - - if (fTotalErrRows > fMaxErrorRows) - { - // flush the reject data file and output the rejected rows - // flush err file and output the rejected row id and the reason. - writeErrorList( 0, 0, true ); - - // number of errors > maximum allowed. hence return error. - { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStartTime = readStart; - fStatusTI = WriteEngine::ERR; - fBuffers[readBufNo].setStatusBLB(WriteEngine::ERR); - } - closeTableFile(); - ostringstream oss5; - oss5 << "Actual error row count(" << fTotalErrRows << - ") exceeds the max error rows(" << fMaxErrorRows << - ") allowed for table " << fTableName; - fLog->logMsg(oss5.str(), ERR_BULK_MAX_ERR_NUM, MSGLVL_ERROR); - - // List Err and Bad files to report file (if applicable) - fBRMReporter.rptMaxErrJob( fBRMRptFileName, fErrFiles, fBadFiles ); - - its.msg_type = ImportTeleStats::IT_TERM; - its.rows_so_far.pop_back(); - its.rows_so_far.push_back(0); - fQtc.postImportTele(its); - - return ERR_BULK_MAX_ERR_NUM; - } - - // mark the buffer status as read complete. - { -#ifdef PROFILE - Stats::startReadEvent(WE_STATS_WAIT_TO_COMPLETE_READ); -#endif - boost::mutex::scoped_lock lock(fSyncUpdatesTI); -#ifdef PROFILE - Stats::stopReadEvent(WE_STATS_WAIT_TO_COMPLETE_READ); - Stats::startReadEvent(WE_STATS_COMPLETING_READ); -#endif - - fStartTime = readStart; - fBuffers[readBufNo].setStatusBLB(WriteEngine::READ_COMPLETE); - - fCurrentReadBuffer = (fCurrentReadBuffer + 1) % fReadBufCount; - - // bufferCount++; - if ( (fHandle && feof(fHandle)) || (fReadFromS3 && (fS3ReadLength == fS3ParseLength)) ) - { - timeval readFinished; - gettimeofday(&readFinished, NULL); - - closeTableFile(); - - if (fReadFromStdin) - { - fLog->logMsg( "Finished loading " + fTableName + " from STDIN" + - ", Time taken = " + Convertor::int2Str((int) - (readFinished.tv_sec - readStart.tv_sec)) + - " seconds", - //" seconds; bufferCount-"+Convertor::int2Str(bufferCount), - MSGLVL_INFO2 ); - } - else if(fReadFromS3) - { - fLog->logMsg( "Finished loading " + fTableName + " from S3" + - ", Time taken = " + Convertor::int2Str((int) - (readFinished.tv_sec - readStart.tv_sec)) + - " seconds", - //" seconds; bufferCount-"+Convertor::int2Str(bufferCount), - MSGLVL_INFO2 ); - } - else - { - fLog->logMsg( "Finished reading file " + fFileName + - ", Time taken = " + Convertor::int2Str((int) - (readFinished.tv_sec - readStart.tv_sec)) + - " seconds", - //" seconds; bufferCount-"+Convertor::int2Str(bufferCount), - MSGLVL_INFO2 ); - } - - // flush the reject data file and output the rejected rows - // flush err file and output the rejected row id and the reason. - writeErrorList( 0, 0, true ); - - // If > 1 file for this table, then open next file in the list - if ( fileCounter < filesTBProcessed ) - { - fFileName = fLoadFileList[fileCounter]; - int rc = openTableFile(); - - if (rc != NO_ERROR) - { - // Mark the table status as error and exit. - fStatusTI = WriteEngine::ERR; - return rc; - } - - fileCounter++; - fTotalReadRows += totalRowsPerInputFile; - totalRowsPerInputFile = 0; - } - else // All files read for this table; break out of read loop - { - fStatusTI = WriteEngine::READ_COMPLETE; - fLastBufferId = readBufNo; - fTotalReadRows += totalRowsPerInputFile; - break; - } - - gettimeofday(&readStart, NULL); - } // reached EOF + gettimeofday(&readStart, NULL); + } // reached EOF #ifdef PROFILE - Stats::stopReadEvent(WE_STATS_COMPLETING_READ); + Stats::stopReadEvent(WE_STATS_COMPLETING_READ); #endif - } // mark buffer status as read-complete within scope of a mutex - } // loop to read all data for this table + } // mark buffer status as read-complete within scope of a mutex + } // loop to read all data for this table - its.msg_type = ImportTeleStats::IT_SUMMARY; - its.end_time = QueryTeleClient::timeNowms(); - its.rows_so_far.pop_back(); - its.rows_so_far.push_back(fTotalReadRows); - fQtc.postImportTele(its); - fQtc.waitForQueues(); + its.msg_type = ImportTeleStats::IT_SUMMARY; + its.end_time = QueryTeleClient::timeNowms(); + its.rows_so_far.pop_back(); + its.rows_so_far.push_back(fTotalReadRows); + fQtc.postImportTele(its); + fQtc.waitForQueues(); - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -632,56 +623,51 @@ int TableInfo::readTableData( ) // Adds errors pertaining to a specific buffer, to the cumulative list of // errors to be reported to the user. //------------------------------------------------------------------------------ -void TableInfo::writeErrorList(const std::vector< std::pair >* errorRows, - const std::vector* errorDatRows, - bool bCloseFile) +void TableInfo::writeErrorList(const std::vector >* errorRows, + const std::vector* errorDatRows, bool bCloseFile) { - size_t errorRowsCount = 0; - size_t errorDatRowsCount = 0; + size_t errorRowsCount = 0; + size_t errorDatRowsCount = 0; - if (errorRows) - errorRowsCount = errorRows->size(); + if (errorRows) + errorRowsCount = errorRows->size(); - if (errorDatRows) - errorDatRowsCount = errorDatRows->size(); + if (errorDatRows) + errorDatRowsCount = errorDatRows->size(); - if ((errorRowsCount > 0) || - (errorDatRowsCount > 0) || - (bCloseFile)) - { - boost::mutex::scoped_lock lock(fErrorRptInfoMutex); + if ((errorRowsCount > 0) || (errorDatRowsCount > 0) || (bCloseFile)) + { + boost::mutex::scoped_lock lock(fErrorRptInfoMutex); - if ((errorRowsCount > 0) || (bCloseFile)) - writeErrReason(errorRows, bCloseFile); + if ((errorRowsCount > 0) || (bCloseFile)) + writeErrReason(errorRows, bCloseFile); - if ((errorDatRowsCount > 0) || (bCloseFile)) - writeBadRows (errorDatRows, bCloseFile); + if ((errorDatRowsCount > 0) || (bCloseFile)) + writeBadRows(errorDatRows, bCloseFile); - fTotalErrRows += errorRowsCount; - } + fTotalErrRows += errorRowsCount; + } } //------------------------------------------------------------------------------ // Parse the specified column (columnId) in the specified buffer (bufferId). //------------------------------------------------------------------------------ -int TableInfo::parseColumn(const int& columnId, const int& bufferId, - double& processingTime) +int TableInfo::parseColumn(const int& columnId, const int& bufferId, double& processingTime) { - // parse the column - // note the time and update the column's last processing time - timeval parseStart, parseEnd; - gettimeofday(&parseStart, NULL); + // parse the column + // note the time and update the column's last processing time + timeval parseStart, parseEnd; + gettimeofday(&parseStart, NULL); - // Will need to check whether the column needs to extend. - // If size of the file is less than the required size, extend the column - int rc = fBuffers[bufferId].parse(fColumns[columnId]); - gettimeofday(&parseEnd, NULL); + // Will need to check whether the column needs to extend. + // If size of the file is less than the required size, extend the column + int rc = fBuffers[bufferId].parse(fColumns[columnId]); + gettimeofday(&parseEnd, NULL); - processingTime = (parseEnd.tv_usec / 1000 + parseEnd.tv_sec * 1000) - - (parseStart.tv_usec / 1000 + parseStart.tv_sec * 1000); + processingTime = (parseEnd.tv_usec / 1000 + parseEnd.tv_sec * 1000) - + (parseStart.tv_usec / 1000 + parseStart.tv_sec * 1000); - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -694,295 +680,283 @@ int TableInfo::parseColumn(const int& columnId, const int& bufferId, // If this is the last column to finish parsing for this table, then mark the // table status as PARSE_COMPLETE. //------------------------------------------------------------------------------ -int TableInfo::setParseComplete(const int& columnId, - const int& bufferId, - double processingTime) +int TableInfo::setParseComplete(const int& columnId, const int& bufferId, double processingTime) { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); + boost::mutex::scoped_lock lock(fSyncUpdatesTI); - // Check table status in case race condition results in this function - // being called after fStatusTI was set to ERR by another thread. - if (fStatusTI == WriteEngine::ERR) - return ERR_UNKNOWN; + // Check table status in case race condition results in this function + // being called after fStatusTI was set to ERR by another thread. + if (fStatusTI == WriteEngine::ERR) + return ERR_UNKNOWN; - fColumns[columnId].lastProcessingTime = processingTime; + fColumns[columnId].lastProcessingTime = processingTime; #ifdef PROFILE - fColumns[columnId].totalProcessingTime += processingTime; + fColumns[columnId].totalProcessingTime += processingTime; #endif - // Set buffer status to complete if setColumnStatus indicates that - // all the columns are complete - if (fBuffers[bufferId].setColumnStatus( - columnId, WriteEngine::PARSE_COMPLETE)) - fBuffers[bufferId].setStatusBLB( WriteEngine::PARSE_COMPLETE ); + // Set buffer status to complete if setColumnStatus indicates that + // all the columns are complete + if (fBuffers[bufferId].setColumnStatus(columnId, WriteEngine::PARSE_COMPLETE)) + fBuffers[bufferId].setStatusBLB(WriteEngine::PARSE_COMPLETE); - // fLastBufferId != -1 means the Read thread has read the last - // buffer for this table - if (fLastBufferId != -1) + // fLastBufferId != -1 means the Read thread has read the last + // buffer for this table + if (fLastBufferId != -1) + { + // check if the status of the column in all the fBuffers is parse + // complete then update the column status as parse complete. + bool allBuffersDoneForAColumn = true; + + for (int i = 0; i < fReadBufCount; ++i) { - // check if the status of the column in all the fBuffers is parse - // complete then update the column status as parse complete. - bool allBuffersDoneForAColumn = true; + // check the status of the column in this buffer. + Status bufferStatus = fBuffers[i].getStatusBLB(); - for (int i = 0; i < fReadBufCount; ++i) + if ((bufferStatus == WriteEngine::READ_COMPLETE) || (bufferStatus == WriteEngine::PARSE_COMPLETE)) + { + if (fBuffers[i].getColumnStatus(columnId) != WriteEngine::PARSE_COMPLETE) { - // check the status of the column in this buffer. - Status bufferStatus = fBuffers[i].getStatusBLB(); - - if ( (bufferStatus == WriteEngine::READ_COMPLETE) || - (bufferStatus == WriteEngine::PARSE_COMPLETE) ) - { - if (fBuffers[i].getColumnStatus(columnId) != - WriteEngine::PARSE_COMPLETE) - { - allBuffersDoneForAColumn = false; - break; - } - } - } - - // allBuffersDoneForAColumn==TRUE means we are finished parsing columnId - if (allBuffersDoneForAColumn) - { - // Accumulate list of HWM dictionary blocks to be flushed from cache - std::vector dictBlksToFlush; - fColumns[columnId].getDictFlushBlks( dictBlksToFlush ); - - for (unsigned kk = 0; kk < dictBlksToFlush.size(); kk++) - { - fDictFlushBlks.push_back( dictBlksToFlush[kk] ); - } - - int rc = fColumns[columnId].finishParsing( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete completion error; " - "Failed to load table: " << - fTableName << "; " << ec.errorString(rc); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - return rc; - } - - fNumberOfColsParsed++; - - // - // If all columns have been parsed, then finished with this tbl - // - if (fNumberOfColsParsed >= fNumberOfColumns) - { - // After closing the column and dictionary store files, - // flush any updated dictionary blocks in PrimProc. - // We only do this for non-HDFS. For HDFS we don't want - // to flush till "after" we have "confirmed" all the file - // changes, which flushes the changes to disk. - if (!idbdatafile::IDBPolicy::useHdfs()) - { - if (fDictFlushBlks.size() > 0) - { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_FLUSH_PRIMPROC_BLOCKS); -#endif - if (fLog->isDebug(DEBUG_2)) - { - ostringstream oss; - oss << "Dictionary cache flush: "; - for (uint32_t i = 0; i < fDictFlushBlks.size(); i++) - { - oss << fDictFlushBlks[i] << ", "; - } - oss << endl; - fLog->logMsg( oss.str(), MSGLVL_INFO1 ); - } - cacheutils::flushPrimProcAllverBlocks(fDictFlushBlks); -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_FLUSH_PRIMPROC_BLOCKS); -#endif - fDictFlushBlks.clear(); - } - } - - // Update auto-increment next value if applicable. - rc = synchronizeAutoInc( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete: autoInc update error; " - "Failed to load table: " << fTableName << - "; " << ec.errorString(rc); - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - return rc; - } - - //..Validate that all the HWM's are consistent and in-sync - std::vector segFileInfo; - - for (unsigned i = 0; i < fColumns.size(); ++i) - { - DBRootExtentInfo extentInfo; - fColumns[i].getSegFileInfo( extentInfo ); - segFileInfo.push_back( extentInfo ); - } - - rc = validateColumnHWMs( 0, segFileInfo, "Ending" ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete: HWM validation error; " - "Failed to load table: " << fTableName << - "; " << ec.errorString(rc); - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - - ostringstream oss2; - oss2 << "Ending HWMs for table " << fTableName << ": "; - - for (unsigned int n = 0; n < fColumns.size(); n++) - { - oss2 << std::endl; - oss2 << " " << fColumns[n].column.colName << - "; DBRoot/part/seg/hwm: " << - segFileInfo[n].fDbRoot << - "/" << segFileInfo[n].fPartition << - "/" << segFileInfo[n].fSegment << - "/" << segFileInfo[n].fLocalHwm; - } - - fLog->logMsg(oss2.str(), MSGLVL_INFO1); - - return rc; - } - - //..Confirm changes to DB files (necessary for HDFS) - rc = confirmDBFileChanges( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete: Error confirming DB changes; " - "Failed to load table: " << fTableName << - "; " << ec.errorString(rc); - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - return rc; - } - - //..Update BRM with HWM and Casual Partition info, etc. - rc = finishBRM( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete: BRM error; " - "Failed to load table: " << fTableName << - "; " << ec.errorString(rc); - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - return rc; - } - - // Change table lock state to CLEANUP - rc = changeTableLockState( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete: table lock state change error; " - "Table load completed: " << fTableName << "; " << - ec.errorString(rc); - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - return rc; - } - - // Finished with this table, so delete bulk rollback - // meta data file and release the table lock. - deleteTempDBFileChanges(); - deleteMetaDataRollbackFile(); - - rc = releaseTableLock( ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - ostringstream oss; - oss << "setParseComplete: table lock release error; " - "Failed to load table: " << fTableName << "; " << - ec.errorString(rc); - fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - fStatusTI = WriteEngine::ERR; - return rc; - } - -#ifdef PROFILE - - // Loop through columns again to print out the elapsed - // parse times - for (unsigned i = 0; i < fColumns.size(); ++i) - { - ostringstream ossColTime; - ossColTime << "Column " << i << "; OID-" << - fColumns[i].column.mapOid << "; parseTime-" << - (fColumns[i].totalProcessingTime / 1000.0) << - " seconds"; - fLog->logMsg(ossColTime.str(), MSGLVL_INFO1); - } - -#endif - - timeval endTime; - gettimeofday(&endTime, 0); - double elapsedTime = - (endTime.tv_sec + (endTime.tv_usec / 1000000.0)) - - (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); - - fStatusTI = WriteEngine::PARSE_COMPLETE; - reportTotals(elapsedTime); - - // Reduce memory use by allocating and releasing as needed - freeProcessingBuffers(); - - } // end of if (fNumberOfColsParsed >= fNumberOfColumns) - } // end of if (allBuffersDoneForAColumn) - } // end of if (fLastBufferId != -1) - - // If we finished parsing the buffer associated with currentParseBuffer, - // but have not finshed the entire table, then advance currentParseBuffer. - if ((fStatusTI != WriteEngine::PARSE_COMPLETE) && - (fBuffers[bufferId].getStatusBLB() == WriteEngine::PARSE_COMPLETE)) - { - // Find the BulkLoadBuffer object that is next in line to be parsed - // and assign fCurrentParseBuffer accordingly. Break out of the - // loop if we wrap all the way around and catch up with the current- - // Read buffer. - if (bufferId == fCurrentParseBuffer) - { - int currentParseBuffer = fCurrentParseBuffer; - - while (fBuffers[currentParseBuffer].getStatusBLB() == - WriteEngine::PARSE_COMPLETE ) - { - currentParseBuffer = (currentParseBuffer + 1) % - fReadBufCount; - fCurrentParseBuffer = currentParseBuffer; - - if (fCurrentParseBuffer == fCurrentReadBuffer) break; - } + allBuffersDoneForAColumn = false; + break; } + } } - return NO_ERROR; + // allBuffersDoneForAColumn==TRUE means we are finished parsing columnId + if (allBuffersDoneForAColumn) + { + // Accumulate list of HWM dictionary blocks to be flushed from cache + std::vector dictBlksToFlush; + fColumns[columnId].getDictFlushBlks(dictBlksToFlush); + + for (unsigned kk = 0; kk < dictBlksToFlush.size(); kk++) + { + fDictFlushBlks.push_back(dictBlksToFlush[kk]); + } + + int rc = fColumns[columnId].finishParsing(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete completion error; " + "Failed to load table: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + return rc; + } + + fNumberOfColsParsed++; + + // + // If all columns have been parsed, then finished with this tbl + // + if (fNumberOfColsParsed >= fNumberOfColumns) + { + // After closing the column and dictionary store files, + // flush any updated dictionary blocks in PrimProc. + // We only do this for non-HDFS. For HDFS we don't want + // to flush till "after" we have "confirmed" all the file + // changes, which flushes the changes to disk. + if (!idbdatafile::IDBPolicy::useHdfs()) + { + if (fDictFlushBlks.size() > 0) + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_FLUSH_PRIMPROC_BLOCKS); +#endif + if (fLog->isDebug(DEBUG_2)) + { + ostringstream oss; + oss << "Dictionary cache flush: "; + for (uint32_t i = 0; i < fDictFlushBlks.size(); i++) + { + oss << fDictFlushBlks[i] << ", "; + } + oss << endl; + fLog->logMsg(oss.str(), MSGLVL_INFO1); + } + cacheutils::flushPrimProcAllverBlocks(fDictFlushBlks); +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_FLUSH_PRIMPROC_BLOCKS); +#endif + fDictFlushBlks.clear(); + } + } + + // Update auto-increment next value if applicable. + rc = synchronizeAutoInc(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete: autoInc update error; " + "Failed to load table: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + return rc; + } + + //..Validate that all the HWM's are consistent and in-sync + std::vector segFileInfo; + + for (unsigned i = 0; i < fColumns.size(); ++i) + { + DBRootExtentInfo extentInfo; + fColumns[i].getSegFileInfo(extentInfo); + segFileInfo.push_back(extentInfo); + } + + rc = validateColumnHWMs(0, segFileInfo, "Ending"); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete: HWM validation error; " + "Failed to load table: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + + ostringstream oss2; + oss2 << "Ending HWMs for table " << fTableName << ": "; + + for (unsigned int n = 0; n < fColumns.size(); n++) + { + oss2 << std::endl; + oss2 << " " << fColumns[n].column.colName << "; DBRoot/part/seg/hwm: " << segFileInfo[n].fDbRoot + << "/" << segFileInfo[n].fPartition << "/" << segFileInfo[n].fSegment << "/" + << segFileInfo[n].fLocalHwm; + } + + fLog->logMsg(oss2.str(), MSGLVL_INFO1); + + return rc; + } + + //..Confirm changes to DB files (necessary for HDFS) + rc = confirmDBFileChanges(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete: Error confirming DB changes; " + "Failed to load table: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + return rc; + } + + //..Update BRM with HWM and Casual Partition info, etc. + rc = finishBRM(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete: BRM error; " + "Failed to load table: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + return rc; + } + + // Change table lock state to CLEANUP + rc = changeTableLockState(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete: table lock state change error; " + "Table load completed: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + return rc; + } + + // Finished with this table, so delete bulk rollback + // meta data file and release the table lock. + deleteTempDBFileChanges(); + deleteMetaDataRollbackFile(); + + rc = releaseTableLock(); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + ostringstream oss; + oss << "setParseComplete: table lock release error; " + "Failed to load table: " + << fTableName << "; " << ec.errorString(rc); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + fStatusTI = WriteEngine::ERR; + return rc; + } + +#ifdef PROFILE + + // Loop through columns again to print out the elapsed + // parse times + for (unsigned i = 0; i < fColumns.size(); ++i) + { + ostringstream ossColTime; + ossColTime << "Column " << i << "; OID-" << fColumns[i].column.mapOid << "; parseTime-" + << (fColumns[i].totalProcessingTime / 1000.0) << " seconds"; + fLog->logMsg(ossColTime.str(), MSGLVL_INFO1); + } + +#endif + + timeval endTime; + gettimeofday(&endTime, 0); + double elapsedTime = (endTime.tv_sec + (endTime.tv_usec / 1000000.0)) - + (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); + + fStatusTI = WriteEngine::PARSE_COMPLETE; + reportTotals(elapsedTime); + + // Reduce memory use by allocating and releasing as needed + freeProcessingBuffers(); + + } // end of if (fNumberOfColsParsed >= fNumberOfColumns) + } // end of if (allBuffersDoneForAColumn) + } // end of if (fLastBufferId != -1) + + // If we finished parsing the buffer associated with currentParseBuffer, + // but have not finshed the entire table, then advance currentParseBuffer. + if ((fStatusTI != WriteEngine::PARSE_COMPLETE) && + (fBuffers[bufferId].getStatusBLB() == WriteEngine::PARSE_COMPLETE)) + { + // Find the BulkLoadBuffer object that is next in line to be parsed + // and assign fCurrentParseBuffer accordingly. Break out of the + // loop if we wrap all the way around and catch up with the current- + // Read buffer. + if (bufferId == fCurrentParseBuffer) + { + int currentParseBuffer = fCurrentParseBuffer; + + while (fBuffers[currentParseBuffer].getStatusBLB() == WriteEngine::PARSE_COMPLETE) + { + currentParseBuffer = (currentParseBuffer + 1) % fReadBufCount; + fCurrentParseBuffer = currentParseBuffer; + + if (fCurrentParseBuffer == fCurrentReadBuffer) + break; + } + } + } + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -992,125 +966,108 @@ int TableInfo::setParseComplete(const int& columnId, //------------------------------------------------------------------------------ void TableInfo::reportTotals(double elapsedTime) { - ostringstream oss1; - oss1 << "For table " << fTableName << - ": " << fTotalReadRows << " rows processed and " << - (fTotalReadRows - fTotalErrRows) << " rows inserted."; + ostringstream oss1; + oss1 << "For table " << fTableName << ": " << fTotalReadRows << " rows processed and " + << (fTotalReadRows - fTotalErrRows) << " rows inserted."; - fLog->logMsg(oss1.str(), MSGLVL_INFO1); + fLog->logMsg(oss1.str(), MSGLVL_INFO1); - ostringstream oss2; - oss2 << "For table " << fTableName << ": " << - "Elapsed time to load this table: " << - elapsedTime << " secs"; + ostringstream oss2; + oss2 << "For table " << fTableName << ": " + << "Elapsed time to load this table: " << elapsedTime << " secs"; - fLog->logMsg(oss2.str(), MSGLVL_INFO2); + fLog->logMsg(oss2.str(), MSGLVL_INFO2); - // @bug 3504: Loop through columns to print saturation counts - std::vector > satCounts; + // @bug 3504: Loop through columns to print saturation counts + std::vector > satCounts; - for (unsigned i = 0; i < fColumns.size(); ++i) + for (unsigned i = 0; i < fColumns.size(); ++i) + { + // std::string colName(fTableName); + // colName += '.'; + // colName += fColumns[i].column.colName; + long long satCount = fColumns[i].saturatedCnt(); + + satCounts.push_back(boost::make_tuple(fColumns[i].column.dataType, fColumns[i].column.mapOid, satCount)); + + if (satCount > 0) { - //std::string colName(fTableName); - // colName += '.'; - // colName += fColumns[i].column.colName; - long long satCount = fColumns[i].saturatedCnt(); + // @bug 3375: report invalid dates/times set to null + ostringstream ossSatCnt; + ossSatCnt << "Column " << fTableName << '.' << fColumns[i].column.colName << "; Number of "; - satCounts.push_back(boost::make_tuple(fColumns[i].column.dataType, - fColumns[i].column.mapOid, - satCount)); + if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::DATE) + { + ossSatCnt << "invalid dates replaced with zero value : "; + } + else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::DATETIME) + { + // bug5383 + ossSatCnt << "invalid date/times replaced with zero value : "; + } - if (satCount > 0) - { - // @bug 3375: report invalid dates/times set to null - ostringstream ossSatCnt; - ossSatCnt << "Column " << fTableName << '.' << - fColumns[i].column.colName << "; Number of "; + else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + ossSatCnt << "invalid timestamps replaced with zero value : "; + } + else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::TIME) + { + ossSatCnt << "invalid times replaced with zero value : "; + } + else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::CHAR) + ossSatCnt << "character strings truncated: "; + else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::VARCHAR) + ossSatCnt << "character strings truncated: "; + else + ossSatCnt << "rows inserted with saturated values: "; - if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::DATE) - { - ossSatCnt << - "invalid dates replaced with zero value : "; - } - else if (fColumns[i].column.dataType == - execplan::CalpontSystemCatalog::DATETIME) - { - //bug5383 - ossSatCnt << - "invalid date/times replaced with zero value : "; - } - - else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::TIMESTAMP) - { - ossSatCnt << - "invalid timestamps replaced with zero value : "; - } - else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::TIME) - { - ossSatCnt << - "invalid times replaced with zero value : "; - } - else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::CHAR) - ossSatCnt << - "character strings truncated: "; - else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::VARCHAR) - ossSatCnt << "character strings truncated: "; - else - ossSatCnt << - "rows inserted with saturated values: "; - - ossSatCnt << satCount; - fLog->logMsg(ossSatCnt.str(), MSGLVL_WARNING); - } + ossSatCnt << satCount; + fLog->logMsg(ossSatCnt.str(), MSGLVL_WARNING); } + } - logging::Message::Args tblFinishedMsgArgs; - tblFinishedMsgArgs.add( fJobId ); - tblFinishedMsgArgs.add( fTableName ); - tblFinishedMsgArgs.add( (fTotalReadRows - fTotalErrRows) ); - SimpleSysLog::instance()->logMsg( - tblFinishedMsgArgs, - logging::LOG_TYPE_INFO, - logging::M0083); + logging::Message::Args tblFinishedMsgArgs; + tblFinishedMsgArgs.add(fJobId); + tblFinishedMsgArgs.add(fTableName); + tblFinishedMsgArgs.add((fTotalReadRows - fTotalErrRows)); + SimpleSysLog::instance()->logMsg(tblFinishedMsgArgs, logging::LOG_TYPE_INFO, logging::M0083); - //Bug1375 - cpimport.bin did not add entries to the transaction - // log file: data_mods.log - if ((fTotalReadRows - fTotalErrRows) > 0 ) - logToDataMods(fjobFileName, oss1.str()); + // Bug1375 - cpimport.bin did not add entries to the transaction + // log file: data_mods.log + if ((fTotalReadRows - fTotalErrRows) > 0) + logToDataMods(fjobFileName, oss1.str()); - // Log totals in report file if applicable - fBRMReporter.reportTotals( fTotalReadRows, - (fTotalReadRows - fTotalErrRows), - satCounts ); + // Log totals in report file if applicable + fBRMReporter.reportTotals(fTotalReadRows, (fTotalReadRows - fTotalErrRows), satCounts); } //------------------------------------------------------------------------------ // Report BRM updates to a report file or to BRM directly. //------------------------------------------------------------------------------ -int TableInfo::finishBRM( ) +int TableInfo::finishBRM() { - // Collect the CP and HWM information for all the columns - for (unsigned i = 0; i < fColumns.size(); ++i) - { - fColumns[i].getBRMUpdateInfo( fBRMReporter ); - } + // Collect the CP and HWM information for all the columns + for (unsigned i = 0; i < fColumns.size(); ++i) + { + fColumns[i].getBRMUpdateInfo(fBRMReporter); + } - // We use mutex not to synchronize contention among parallel threads, - // because we should be the only thread accessing the fErrFiles and - // fBadFiles at this point. But we do use the mutex as a memory barrier - // to make sure we have the latest copy of the data. - std::vector* errFiles = 0; - std::vector* badFiles = 0; - { - boost::mutex::scoped_lock lock(fErrorRptInfoMutex); - errFiles = &fErrFiles; - badFiles = &fBadFiles; - } + // We use mutex not to synchronize contention among parallel threads, + // because we should be the only thread accessing the fErrFiles and + // fBadFiles at this point. But we do use the mutex as a memory barrier + // to make sure we have the latest copy of the data. + std::vector* errFiles = 0; + std::vector* badFiles = 0; + { + boost::mutex::scoped_lock lock(fErrorRptInfoMutex); + errFiles = &fErrFiles; + badFiles = &fBadFiles; + } - // Save the info just collected, to a report file or send to BRM - int rc = fBRMReporter.sendBRMInfo( fBRMRptFileName, *errFiles, *badFiles ); + // Save the info just collected, to a report file or send to BRM + int rc = fBRMReporter.sendBRMInfo(fBRMRptFileName, *errFiles, *badFiles); - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -1120,10 +1077,10 @@ int TableInfo::finishBRM( ) // the JobStatus has been set to EXIT_FAILURE and/or the table status has been // set to WriteEngine::ERR. //------------------------------------------------------------------------------ -void TableInfo::setParseError( ) +void TableInfo::setParseError() { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStatusTI = WriteEngine::ERR; + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStatusTI = WriteEngine::ERR; } //------------------------------------------------------------------------------ @@ -1133,102 +1090,101 @@ void TableInfo::setParseError( ) //------------------------------------------------------------------------------ // @bug2099. Temporary hack to diagnose deadlock. // Added report parm and couts below. -int TableInfo::getColumnForParse(const int& id, - const int& bufferId, - bool report) +int TableInfo::getColumnForParse(const int& id, const int& bufferId, bool report) { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - double maxTime = 0; - int columnId = -1; + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + double maxTime = 0; + int columnId = -1; - while (true) + while (true) + { + // See if JobStatus has been set to terminate by another thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) { - // See if JobStatus has been set to terminate by another thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - fStatusTI = WriteEngine::ERR; - throw SecondaryShutdownException( "TableInfo::" - "getColumnForParse() responding to job termination"); - } - - if ( !bufferReadyForParse(bufferId, report) ) return -1; - - // @bug2099+ - ostringstream oss; - - if (report) - { -#ifdef _MSC_VER - oss << " ----- " << GetCurrentThreadId() << ":fBuffers[" << bufferId << -#else - oss << " ----- " << pthread_self() << ":fBuffers[" << bufferId << -#endif - "]: (colLocker,status,lasttime)- "; - } - - // @bug2099- - - for (unsigned k = 0; k < fNumberOfColumns; ++k) - { - // @bug2099+ - if (report) - { - Status colStatus = fBuffers[bufferId].getColumnStatus(k); - int colLocker = fBuffers[bufferId].getColumnLocker(k); - - string colStatusStr; - ColumnInfo::convertStatusToString(colStatus, colStatusStr); - - oss << '(' << colLocker << ',' << colStatusStr << ',' << - fColumns[k].lastProcessingTime << ") "; - } - - // @bug2099- - - if (fBuffers[bufferId].getColumnLocker(k) == -1) - { - if (columnId == -1) - columnId = k; - else if (fColumns[k].lastProcessingTime == 0) - { - if (fColumns[k].column.width >= - fColumns[columnId].column.width) - columnId = k; - } - else if (fColumns[k].lastProcessingTime > maxTime) - { - maxTime = fColumns[k].lastProcessingTime; - columnId = k; - } - } - } - - // @bug2099+ - if (report) - { - oss << "; selected colId: " << columnId; - - if (columnId != -1) - oss << "; maxTime: " << maxTime; - - oss << endl; - - if (!BulkLoad::disableConsoleOutput()) - { - cout << oss.str(); - cout.flush(); - } - } - - // @bug2099- - - if (columnId == -1) return -1; - - if (fBuffers[bufferId].tryAndLockColumn(columnId, id)) - { - return columnId; - } + fStatusTI = WriteEngine::ERR; + throw SecondaryShutdownException( + "TableInfo::" + "getColumnForParse() responding to job termination"); } + + if (!bufferReadyForParse(bufferId, report)) + return -1; + + // @bug2099+ + ostringstream oss; + + if (report) + { +#ifdef _MSC_VER + oss << " ----- " << GetCurrentThreadId() << ":fBuffers[" << bufferId << +#else + oss << " ----- " << pthread_self() << ":fBuffers[" << bufferId << +#endif + "]: (colLocker,status,lasttime)- "; + } + + // @bug2099- + + for (unsigned k = 0; k < fNumberOfColumns; ++k) + { + // @bug2099+ + if (report) + { + Status colStatus = fBuffers[bufferId].getColumnStatus(k); + int colLocker = fBuffers[bufferId].getColumnLocker(k); + + string colStatusStr; + ColumnInfo::convertStatusToString(colStatus, colStatusStr); + + oss << '(' << colLocker << ',' << colStatusStr << ',' << fColumns[k].lastProcessingTime << ") "; + } + + // @bug2099- + + if (fBuffers[bufferId].getColumnLocker(k) == -1) + { + if (columnId == -1) + columnId = k; + else if (fColumns[k].lastProcessingTime == 0) + { + if (fColumns[k].column.width >= fColumns[columnId].column.width) + columnId = k; + } + else if (fColumns[k].lastProcessingTime > maxTime) + { + maxTime = fColumns[k].lastProcessingTime; + columnId = k; + } + } + } + + // @bug2099+ + if (report) + { + oss << "; selected colId: " << columnId; + + if (columnId != -1) + oss << "; maxTime: " << maxTime; + + oss << endl; + + if (!BulkLoad::disableConsoleOutput()) + { + cout << oss.str(); + cout.flush(); + } + } + + // @bug2099- + + if (columnId == -1) + return -1; + + if (fBuffers[bufferId].tryAndLockColumn(columnId, id)) + { + return columnId; + } + } } //------------------------------------------------------------------------------ @@ -1238,28 +1194,26 @@ int TableInfo::getColumnForParse(const int& id, //------------------------------------------------------------------------------ bool TableInfo::bufferReadyForParse(const int& bufferId, bool report) const { - if (fBuffers.size() == 0) - return false; + if (fBuffers.size() == 0) + return false; - Status stat = fBuffers[bufferId].getStatusBLB(); + Status stat = fBuffers[bufferId].getStatusBLB(); - if (report) - { - ostringstream oss; - string bufStatusStr; - ColumnInfo::convertStatusToString( stat, - bufStatusStr ); + if (report) + { + ostringstream oss; + string bufStatusStr; + ColumnInfo::convertStatusToString(stat, bufStatusStr); #ifdef _MSC_VER - oss << " --- " << GetCurrentThreadId() << + oss << " --- " << GetCurrentThreadId() << #else - oss << " --- " << pthread_self() << + oss << " --- " << pthread_self() << #endif - ":fBuffers[" << bufferId << "]=" << bufStatusStr << - " (" << stat << ")" << std::endl; - cout << oss.str(); - } + ":fBuffers[" << bufferId << "]=" << bufStatusStr << " (" << stat << ")" << std::endl; + cout << oss.str(); + } - return (stat == WriteEngine::READ_COMPLETE) ? true : false; + return (stat == WriteEngine::READ_COMPLETE) ? true : false; } //------------------------------------------------------------------------------ @@ -1268,54 +1222,50 @@ bool TableInfo::bufferReadyForParse(const int& bufferId, bool report) const // fixedBinaryRecLen is fixed record length for binary imports (it is n/a // for text bulk loads). //------------------------------------------------------------------------------ -int TableInfo::initializeBuffers(int noOfBuffers, - const JobFieldRefList& jobFieldRefList, +int TableInfo::initializeBuffers(int noOfBuffers, const JobFieldRefList& jobFieldRefList, unsigned int fixedBinaryRecLen) { #ifdef _MSC_VER - //@bug 3751 - //When reading from STDIN, Windows doesn't like the huge default buffer of - // 1M, so turn it down. - if (fReadFromStdin) - { - fBufferSize = std::min(10240, fBufferSize); - } + //@bug 3751 + // When reading from STDIN, Windows doesn't like the huge default buffer of + // 1M, so turn it down. + if (fReadFromStdin) + { + fBufferSize = std::min(10240, fBufferSize); + } #endif - fReadBufCount = noOfBuffers; + fReadBufCount = noOfBuffers; - // initialize and populate the buffer vector. - for (int i = 0; i < fReadBufCount; ++i) + // initialize and populate the buffer vector. + for (int i = 0; i < fReadBufCount; ++i) + { + BulkLoadBuffer* buffer = + new BulkLoadBuffer(fNumberOfColumns, fBufferSize, fLog, i, fTableName, jobFieldRefList); + buffer->setColDelimiter(fColDelim); + buffer->setNullStringMode(fNullStringMode); + buffer->setEnclosedByChar(fEnclosedByChar); + buffer->setEscapeChar(fEscapeChar); + buffer->setTruncationAsError(getTruncationAsError()); + buffer->setImportDataMode(fImportDataMode, fixedBinaryRecLen); + buffer->setTimeZone(fTimeZone); + fBuffers.push_back(buffer); + } + if (!fS3Key.empty()) + { + ms3_library_init(); + ms3 = ms3_init(fS3Key.c_str(), fS3Secret.c_str(), fS3Region.c_str(), fS3Host.c_str()); + if (!ms3) { - BulkLoadBuffer* buffer = new BulkLoadBuffer(fNumberOfColumns, - fBufferSize, fLog, - i, fTableName, - jobFieldRefList); - buffer->setColDelimiter (fColDelim); - buffer->setNullStringMode(fNullStringMode); - buffer->setEnclosedByChar(fEnclosedByChar); - buffer->setEscapeChar (fEscapeChar ); - buffer->setTruncationAsError(getTruncationAsError()); - buffer->setImportDataMode(fImportDataMode, - fixedBinaryRecLen); - buffer->setTimeZone(fTimeZone); - fBuffers.push_back(buffer); + ostringstream oss; + oss << "Error initiating S3 library"; + fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); + return ERR_FILE_OPEN; } - if (!fS3Key.empty()) - { - ms3_library_init(); - ms3 = ms3_init(fS3Key.c_str(), fS3Secret.c_str(), fS3Region.c_str(), fS3Host.c_str()); - if (!ms3) - { - ostringstream oss; - oss << "Error initiating S3 library"; - fLog->logMsg( oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR ); - return ERR_FILE_OPEN; - } - } - return 0; + } + return 0; } //------------------------------------------------------------------------------ @@ -1323,11 +1273,10 @@ int TableInfo::initializeBuffers(int noOfBuffers, //------------------------------------------------------------------------------ void TableInfo::addColumn(ColumnInfo* info) { - fColumns.push_back(info); - fNumberOfColumns = fColumns.size(); + fColumns.push_back(info); + fNumberOfColumns = fColumns.size(); - fExtentStrAlloc.addColumn( info->column.mapOid, - info->column.width ); + fExtentStrAlloc.addColumn(info->column.mapOid, info->column.width); } //------------------------------------------------------------------------------ @@ -1337,82 +1286,80 @@ void TableInfo::addColumn(ColumnInfo* info) //------------------------------------------------------------------------------ int TableInfo::openTableFile() { - if (fHandle != NULL) - return NO_ERROR; + if (fHandle != NULL) + return NO_ERROR; - if (fReadFromStdin) - { - fHandle = stdin; + if (fReadFromStdin) + { + fHandle = stdin; #ifdef _MSC_VER - // If this is a binary import from STDIN, then set stdin to binary - if (fImportDataMode != IMPORT_DATA_TEXT) - _setmode(_fileno(stdin), _O_BINARY); + // If this is a binary import from STDIN, then set stdin to binary + if (fImportDataMode != IMPORT_DATA_TEXT) + _setmode(_fileno(stdin), _O_BINARY); - fFileBuffer = 0; + fFileBuffer = 0; #else - // Not 100% sure that calling setvbuf on stdin does much, but in - // some tests, it made a slight difference. - fFileBuffer = new char[fFileBufSize]; - setvbuf(fHandle, fFileBuffer, _IOFBF, fFileBufSize); + // Not 100% sure that calling setvbuf on stdin does much, but in + // some tests, it made a slight difference. + fFileBuffer = new char[fFileBufSize]; + setvbuf(fHandle, fFileBuffer, _IOFBF, fFileBufSize); #endif - ostringstream oss; - oss << BOLD_START << "Reading input from STDIN to import into table " << - fTableName << "..." << BOLD_STOP; - fLog->logMsg( oss.str(), MSGLVL_INFO1 ); - } - else if (fReadFromS3) + ostringstream oss; + oss << BOLD_START << "Reading input from STDIN to import into table " << fTableName << "..." << BOLD_STOP; + fLog->logMsg(oss.str(), MSGLVL_INFO1); + } + else if (fReadFromS3) + { + int res; + res = ms3_get(ms3, fS3Bucket.c_str(), fFileName.c_str(), (uint8_t**)&fFileBuffer, &fS3ReadLength); + fS3ParseLength = 0; + if (res) { - int res; - res = ms3_get(ms3, fS3Bucket.c_str(), fFileName.c_str(), (uint8_t**)&fFileBuffer, &fS3ReadLength); - fS3ParseLength = 0; - if (res) - { - ostringstream oss; - oss << "Error retrieving file " << fFileName << " from S3: "; - if (ms3_server_error(ms3)) - { - oss << ms3_server_error(ms3); - } - else - { - oss << ms3_error(res); - } - fLog->logMsg( oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR ); - return ERR_FILE_OPEN; - } + ostringstream oss; + oss << "Error retrieving file " << fFileName << " from S3: "; + if (ms3_server_error(ms3)) + { + oss << ms3_server_error(ms3); + } + else + { + oss << ms3_error(res); + } + fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); + return ERR_FILE_OPEN; } + } + else + { + if (fImportDataMode == IMPORT_DATA_TEXT) + fHandle = fopen(fFileName.c_str(), "r"); else + fHandle = fopen(fFileName.c_str(), "rb"); + + if (fHandle == NULL) { - if (fImportDataMode == IMPORT_DATA_TEXT) - fHandle = fopen( fFileName.c_str(), "r" ); - else - fHandle = fopen( fFileName.c_str(), "rb" ); + int errnum = errno; + ostringstream oss; + oss << "Error opening import file " << fFileName << ". " << strerror(errnum); + fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); - if (fHandle == NULL) - { - int errnum = errno; - ostringstream oss; - oss << "Error opening import file " << fFileName << ". " << - strerror(errnum); - fLog->logMsg( oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR ); - - // return an error; caller should set fStatusTI if needed - return ERR_FILE_OPEN; - } - - // now the input load file is available for reading the data. - // read the data from the load file into the buffers. - fFileBuffer = new char[fFileBufSize]; - setvbuf(fHandle, fFileBuffer, _IOFBF, fFileBufSize); - - ostringstream oss; - oss << "Opening " << fFileName << " to import into table " << fTableName; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + // return an error; caller should set fStatusTI if needed + return ERR_FILE_OPEN; } - return NO_ERROR; + // now the input load file is available for reading the data. + // read the data from the load file into the buffers. + fFileBuffer = new char[fFileBufSize]; + setvbuf(fHandle, fFileBuffer, _IOFBF, fFileBufSize); + + ostringstream oss; + oss << "Opening " << fFileName << " to import into table " << fTableName; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1420,25 +1367,25 @@ int TableInfo::openTableFile() //------------------------------------------------------------------------------ void TableInfo::closeTableFile() { - if (fHandle) + if (fHandle) + { + // If reading from stdin, we don't delete the buffer out from under + // the file handle, because stdin is still open. This will cause a + // memory leak, but when using stdin, we can only read in 1 table. + // So it's not like we will be leaking multiple buffers for several + // tables over the life of the job. + if (!fReadFromStdin) { - // If reading from stdin, we don't delete the buffer out from under - // the file handle, because stdin is still open. This will cause a - // memory leak, but when using stdin, we can only read in 1 table. - // So it's not like we will be leaking multiple buffers for several - // tables over the life of the job. - if (!fReadFromStdin) - { - fclose(fHandle); - delete [] fFileBuffer; - } + fclose(fHandle); + delete[] fFileBuffer; + } - fHandle = 0; - } - else if (ms3) - { - ms3_free((uint8_t*)fFileBuffer); - } + fHandle = 0; + } + else if (ms3) + { + ms3_free((uint8_t*)fFileBuffer); + } } //------------------------------------------------------------------------------ @@ -1449,547 +1396,503 @@ void TableInfo::closeTableFile() // Added report parm and couts below. bool TableInfo::isBufferAvailable(bool report) { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - Status bufferStatus = fBuffers[fCurrentReadBuffer].getStatusBLB(); + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + Status bufferStatus = fBuffers[fCurrentReadBuffer].getStatusBLB(); - if ( (bufferStatus == WriteEngine::PARSE_COMPLETE) || - (bufferStatus == WriteEngine::NEW) ) - { - // reset buffer status and column locks while we have - // an fSyncUpdatesTI lock - fBuffers[fCurrentReadBuffer].setStatusBLB( - WriteEngine::READ_PROGRESS); - fBuffers[fCurrentReadBuffer].resetColumnLocks(); - return true; - } + if ((bufferStatus == WriteEngine::PARSE_COMPLETE) || (bufferStatus == WriteEngine::NEW)) + { + // reset buffer status and column locks while we have + // an fSyncUpdatesTI lock + fBuffers[fCurrentReadBuffer].setStatusBLB(WriteEngine::READ_PROGRESS); + fBuffers[fCurrentReadBuffer].resetColumnLocks(); + return true; + } - if (report) - { - ostringstream oss; - string bufferStatusStr; - ColumnInfo::convertStatusToString( bufferStatus, bufferStatusStr ); - oss << " Buffer status is " << bufferStatusStr << ". " << endl; - oss << " fCurrentReadBuffer is " << fCurrentReadBuffer << endl; - cout << oss.str(); - cout.flush(); - } + if (report) + { + ostringstream oss; + string bufferStatusStr; + ColumnInfo::convertStatusToString(bufferStatus, bufferStatusStr); + oss << " Buffer status is " << bufferStatusStr << ". " << endl; + oss << " fCurrentReadBuffer is " << fCurrentReadBuffer << endl; + cout << oss.str(); + cout.flush(); + } - return false; + return false; } //------------------------------------------------------------------------------ // Report whether rows were rejected, and if so, then list them out into the // reject file. //------------------------------------------------------------------------------ -void TableInfo::writeBadRows( const std::vector* errorDatRows, - bool bCloseFile ) +void TableInfo::writeBadRows(const std::vector* errorDatRows, bool bCloseFile) { - size_t errorDatRowsCount = 0; + size_t errorDatRowsCount = 0; - if (errorDatRows) - errorDatRowsCount = errorDatRows->size(); + if (errorDatRows) + errorDatRowsCount = errorDatRows->size(); - if (errorDatRowsCount > 0) + if (errorDatRowsCount > 0) + { + if (!fRejectDataFile.is_open()) { - if (!fRejectDataFile.is_open()) - { - ostringstream rejectFileName; + ostringstream rejectFileName; - if (fErrorDir.size() > 0) - { + if (fErrorDir.size() > 0) + { #ifdef _MSC_VER - char filename[_MAX_FNAME]; - char ext[_MAX_EXT]; - _splitpath(const_cast(getFileName().c_str()), - NULL, NULL, filename, ext); - rejectFileName << fErrorDir << "\\" << filename << ext; + char filename[_MAX_FNAME]; + char ext[_MAX_EXT]; + _splitpath(const_cast(getFileName().c_str()), NULL, NULL, filename, ext); + rejectFileName << fErrorDir << "\\" << filename << ext; #else - rejectFileName << fErrorDir << basename(getFileName().c_str()); + rejectFileName << fErrorDir << basename(getFileName().c_str()); #endif - } - else - { - if (fReadFromS3) - { - rejectFileName << basename(getFileName().c_str()); - } - else - { - rejectFileName << getFileName(); - } - } - - rejectFileName << ".Job_" << fJobId << - '_' << ::getpid() << BAD_FILE_SUFFIX; - fRejectDataFileName = rejectFileName.str(); - fRejectDataFile.open( rejectFileName.str().c_str(), - ofstream::out ); - - if ( !fRejectDataFile ) - { - ostringstream oss; - oss << "Unable to create file: " << rejectFileName.str() << - "; Check permission."; - fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); - - return; - } - } - - for (std::vector::const_iterator iter = errorDatRows->begin(); - iter != errorDatRows->end(); ++iter) + } + else + { + if (fReadFromS3) { - fRejectDataFile << *iter; + rejectFileName << basename(getFileName().c_str()); } + else + { + rejectFileName << getFileName(); + } + } - fRejectDataCnt += errorDatRowsCount; + rejectFileName << ".Job_" << fJobId << '_' << ::getpid() << BAD_FILE_SUFFIX; + fRejectDataFileName = rejectFileName.str(); + fRejectDataFile.open(rejectFileName.str().c_str(), ofstream::out); + + if (!fRejectDataFile) + { + ostringstream oss; + oss << "Unable to create file: " << rejectFileName.str() << "; Check permission."; + fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); + + return; + } } - if (bCloseFile) + for (std::vector::const_iterator iter = errorDatRows->begin(); iter != errorDatRows->end(); + ++iter) { - if (fRejectDataFile.is_open()) - fRejectDataFile.close(); - - fRejectDataFile.clear(); - - if (fRejectDataCnt > 0) - { - ostringstream oss; - std::string rejectFileNameToLog; - - // Construct/report complete file name and save in list of files - boost::filesystem::path p(fRejectDataFileName); - - if (!p.has_root_path()) - { - // We could fail here having fixed size buffer - char cwdPath[4096]; - char* buffPtr = &cwdPath[0]; - buffPtr = getcwd(cwdPath, sizeof(cwdPath)); - boost::filesystem::path rejectFileName2( buffPtr ); - rejectFileName2 /= fRejectDataFileName; - fBadFiles.push_back( rejectFileName2.string() ); - - rejectFileNameToLog = rejectFileName2.string(); - } - else - { - fBadFiles.push_back( fRejectDataFileName ); - - rejectFileNameToLog = fRejectDataFileName; - } - - oss << "Number of rows with bad data = " << fRejectDataCnt << - ". Exact rows are listed in file located here: " << - fErrorDir; - fLog->logMsg(oss.str(), MSGLVL_INFO1); - - fRejectDataCnt = 0; - } + fRejectDataFile << *iter; } + + fRejectDataCnt += errorDatRowsCount; + } + + if (bCloseFile) + { + if (fRejectDataFile.is_open()) + fRejectDataFile.close(); + + fRejectDataFile.clear(); + + if (fRejectDataCnt > 0) + { + ostringstream oss; + std::string rejectFileNameToLog; + + // Construct/report complete file name and save in list of files + boost::filesystem::path p(fRejectDataFileName); + + if (!p.has_root_path()) + { + // We could fail here having fixed size buffer + char cwdPath[4096]; + char* buffPtr = &cwdPath[0]; + buffPtr = getcwd(cwdPath, sizeof(cwdPath)); + boost::filesystem::path rejectFileName2(buffPtr); + rejectFileName2 /= fRejectDataFileName; + fBadFiles.push_back(rejectFileName2.string()); + + rejectFileNameToLog = rejectFileName2.string(); + } + else + { + fBadFiles.push_back(fRejectDataFileName); + + rejectFileNameToLog = fRejectDataFileName; + } + + oss << "Number of rows with bad data = " << fRejectDataCnt + << ". Exact rows are listed in file located here: " << fErrorDir; + fLog->logMsg(oss.str(), MSGLVL_INFO1); + + fRejectDataCnt = 0; + } + } } //------------------------------------------------------------------------------ // Report whether rows were rejected, and if so, then list out the row numbers // and error reasons into the error file. //------------------------------------------------------------------------------ -void TableInfo::writeErrReason( const std::vector< std::pair >* errorRows, - bool bCloseFile ) +void TableInfo::writeErrReason(const std::vector >* errorRows, bool bCloseFile) { - size_t errorRowsCount = 0; + size_t errorRowsCount = 0; - if (errorRows) - errorRowsCount = errorRows->size(); + if (errorRows) + errorRowsCount = errorRows->size(); - if (errorRowsCount > 0) + if (errorRowsCount > 0) + { + if (!fRejectErrFile.is_open()) { - if (!fRejectErrFile.is_open()) - { - ostringstream errFileName; + ostringstream errFileName; - if (fErrorDir.size() > 0) - { + if (fErrorDir.size() > 0) + { #ifdef _MSC_VER - char filename[_MAX_FNAME]; - char ext[_MAX_EXT]; - _splitpath(const_cast(getFileName().c_str()), - NULL, NULL, filename, ext); - errFileName << fErrorDir << "\\" << filename << ext; + char filename[_MAX_FNAME]; + char ext[_MAX_EXT]; + _splitpath(const_cast(getFileName().c_str()), NULL, NULL, filename, ext); + errFileName << fErrorDir << "\\" << filename << ext; #else - errFileName << fErrorDir << basename(getFileName().c_str()); + errFileName << fErrorDir << basename(getFileName().c_str()); #endif - } - else - { - if (fReadFromS3) - { - errFileName << basename(getFileName().c_str()); - } - else - { - errFileName << getFileName(); - } - } - - errFileName << ".Job_" << fJobId << - '_' << ::getpid() << ERR_FILE_SUFFIX; - fRejectErrFileName = errFileName.str(); - fRejectErrFile.open( errFileName.str().c_str(), - ofstream::out ); - - if ( !fRejectErrFile ) - { - ostringstream oss; - oss << "Unable to create file: " << errFileName.str() << - "; Check permission."; - fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); - - return; - } - } - - for (std::vector< std::pair >::const_iterator iter = - errorRows->begin(); - iter != errorRows->end(); ++iter) + } + else + { + if (fReadFromS3) { - fRejectErrFile << "Line number " << iter->first << - "; Error: " << iter->second << endl; + errFileName << basename(getFileName().c_str()); } + else + { + errFileName << getFileName(); + } + } - fRejectErrCnt += errorRowsCount; + errFileName << ".Job_" << fJobId << '_' << ::getpid() << ERR_FILE_SUFFIX; + fRejectErrFileName = errFileName.str(); + fRejectErrFile.open(errFileName.str().c_str(), ofstream::out); + + if (!fRejectErrFile) + { + ostringstream oss; + oss << "Unable to create file: " << errFileName.str() << "; Check permission."; + fLog->logMsg(oss.str(), ERR_FILE_OPEN, MSGLVL_ERROR); + + return; + } } - if (bCloseFile) + for (std::vector >::const_iterator iter = errorRows->begin(); + iter != errorRows->end(); ++iter) { - if (fRejectErrFile.is_open()) - fRejectErrFile.close(); - - fRejectErrFile.clear(); - - if (fRejectErrCnt > 0) - { - ostringstream oss; - std::string errFileNameToLog; - - // Construct/report complete file name and save in list of files - boost::filesystem::path p(fRejectErrFileName); - - if (!p.has_root_path()) - { - char cwdPath[4096]; - char* buffPtr = &cwdPath[0]; - buffPtr = getcwd(cwdPath, sizeof(cwdPath)); - boost::filesystem::path errFileName2( buffPtr ); - errFileName2 /= fRejectErrFileName; - fErrFiles.push_back( errFileName2.string() ); - - errFileNameToLog = errFileName2.string(); - } - else - { - fErrFiles.push_back( fRejectErrFileName ); - - errFileNameToLog = fRejectErrFileName; - } - - oss << "Number of rows with errors = " << fRejectDataCnt << - ". Exact rows are listed in file located here: " << - fErrorDir; - fLog->logMsg(oss.str(), MSGLVL_INFO1); - - fRejectErrCnt = 0; - } + fRejectErrFile << "Line number " << iter->first << "; Error: " << iter->second << endl; } + + fRejectErrCnt += errorRowsCount; + } + + if (bCloseFile) + { + if (fRejectErrFile.is_open()) + fRejectErrFile.close(); + + fRejectErrFile.clear(); + + if (fRejectErrCnt > 0) + { + ostringstream oss; + std::string errFileNameToLog; + + // Construct/report complete file name and save in list of files + boost::filesystem::path p(fRejectErrFileName); + + if (!p.has_root_path()) + { + char cwdPath[4096]; + char* buffPtr = &cwdPath[0]; + buffPtr = getcwd(cwdPath, sizeof(cwdPath)); + boost::filesystem::path errFileName2(buffPtr); + errFileName2 /= fRejectErrFileName; + fErrFiles.push_back(errFileName2.string()); + + errFileNameToLog = errFileName2.string(); + } + else + { + fErrFiles.push_back(fRejectErrFileName); + + errFileNameToLog = fRejectErrFileName; + } + + oss << "Number of rows with errors = " << fRejectDataCnt + << ". Exact rows are listed in file located here: " << fErrorDir; + fLog->logMsg(oss.str(), MSGLVL_INFO1); + + fRejectErrCnt = 0; + } + } } //------------------------------------------------------------------------------ // Logs "Bulkload |Job" message along with the specified message text // (messageText) to the critical log. //------------------------------------------------------------------------------ -void TableInfo::logToDataMods(const string& jobFile, const string& messageText) +void TableInfo::logToDataMods(const string& jobFile, const string& messageText) { - logging::Message::Args args; + logging::Message::Args args; - unsigned subsystemId = 19; // writeengine + unsigned subsystemId = 19; // writeengine - logging::LoggingID loggingId(subsystemId, 0, fTxnID.id, 0); - logging::MessageLog messageLog(loggingId, LOG_LOCAL1); + logging::LoggingID loggingId(subsystemId, 0, fTxnID.id, 0); + logging::MessageLog messageLog(loggingId, LOG_LOCAL1); - logging::Message m(8); - args.add("Bulkload |Job: " + jobFile); - args.add("|" + messageText); - m.format(args); - messageLog.logInfoMessage(m); + logging::Message m(8); + args.add("Bulkload |Job: " + jobFile); + args.add("|" + messageText); + m.format(args); + messageLog.logInfoMessage(m); } //------------------------------------------------------------------------------ // Acquires DB table lock for this TableInfo object. // Function employs retry logic based on the SystemConfig/WaitPeriod. //------------------------------------------------------------------------------ -int TableInfo::acquireTableLock( bool disableTimeOut ) +int TableInfo::acquireTableLock(bool disableTimeOut) { - // Save DBRoot list at start of job; used to compare at EOJ. - Config::getRootIdList( fOrigDbRootIds ); + // Save DBRoot list at start of job; used to compare at EOJ. + Config::getRootIdList(fOrigDbRootIds); - // If executing distributed (mode1) or central command (mode2) then - // don't worry about table locks. The client front-end will manage locks. - if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + // If executing distributed (mode1) or central command (mode2) then + // don't worry about table locks. The client front-end will manage locks. + if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + { + if (fLog->isDebug(DEBUG_1)) { - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - oss << "Bypass acquiring table lock in distributed mode, " - "for table" << fTableName << "; OID-" << fTableOID; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - return NO_ERROR; + ostringstream oss; + oss << "Bypass acquiring table lock in distributed mode, " + "for table" + << fTableName << "; OID-" << fTableOID; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } - const int SLEEP_INTERVAL = 100; // sleep 100 milliseconds between checks - const int NUM_TRIES_PER_SEC = 10; // try 10 times per second + return NO_ERROR; + } - int waitSeconds = Config::getWaitPeriod(); - const int NUM_TRIES = NUM_TRIES_PER_SEC * waitSeconds; - std::string tblLockErrMsg; + const int SLEEP_INTERVAL = 100; // sleep 100 milliseconds between checks + const int NUM_TRIES_PER_SEC = 10; // try 10 times per second - // Retry loop to lock the db table associated with this TableInfo object - std::string processName; - uint32_t processId; - int32_t sessionId; - int32_t transId; - ostringstream pmModOss; - pmModOss << " (pm" << Config::getLocalModuleID() << ')'; - bool timeout = false; - //for (int i=0; igetTableLock(fTableOID, processName, processId, sessionId, transId, + fTableLockID, tblLockErrMsg); + + if ((rc == NO_ERROR) && (fTableLockID > 0)) { - processName = fProcessName; - processName += pmModOss.str(); - processId = ::getpid(); - sessionId = -1; - transId = -1; - int rc = BRMWrapper::getInstance()->getTableLock ( - fTableOID, - processName, - processId, - sessionId, - transId, - fTableLockID, - tblLockErrMsg); + fTableLocked = true; - if ((rc == NO_ERROR) && (fTableLockID > 0)) - { - fTableLocked = true; + if (fLog->isDebug(DEBUG_1)) + { + ostringstream oss; + oss << "Table lock acquired for table " << fTableName << "; OID-" << fTableOID << "; lockID-" + << fTableLockID; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - oss << "Table lock acquired for table " << fTableName << - "; OID-" << fTableOID << "; lockID-" << fTableLockID; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + return NO_ERROR; + } + else if (fTableLockID == 0) + { + // sleep and then go back and try getting table lock again + sleepMS(SLEEP_INTERVAL); - return NO_ERROR; - } - else if (fTableLockID == 0) - { - // sleep and then go back and try getting table lock again - sleepMS(SLEEP_INTERVAL); + if (fLog->isDebug(DEBUG_1)) + { + ostringstream oss; + oss << "Retrying to acquire table lock for table " << fTableName << "; OID-" << fTableOID; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + } + else + { + ostringstream oss; + oss << "Error in acquiring table lock for table " << fTableName << "; OID-" << fTableOID << "; " + << tblLockErrMsg; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - oss << "Retrying to acquire table lock for table " << - fTableName << "; OID-" << fTableOID; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - } - else - { - ostringstream oss; - oss << "Error in acquiring table lock for table " << fTableName << - "; OID-" << fTableOID << "; " << tblLockErrMsg; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - - return rc; - } - - // if disableTimeOut is set then no timeout for table lock. Forever wait.... - timeout = (disableTimeOut ? false : (++try_count >= NUM_TRIES)); + return rc; } - ostringstream oss; - oss << "Unable to acquire lock for table " << fTableName << - "; OID-" << fTableOID << "; table currently locked by process-" << - processName << "; pid-" << processId << - "; session-" << sessionId << - "; txn-" << transId; - fLog->logMsg( oss.str(), ERR_TBLLOCK_GET_LOCK_LOCKED, MSGLVL_ERROR ); + // if disableTimeOut is set then no timeout for table lock. Forever wait.... + timeout = (disableTimeOut ? false : (++try_count >= NUM_TRIES)); + } - return ERR_TBLLOCK_GET_LOCK_LOCKED; + ostringstream oss; + oss << "Unable to acquire lock for table " << fTableName << "; OID-" << fTableOID + << "; table currently locked by process-" << processName << "; pid-" << processId << "; session-" + << sessionId << "; txn-" << transId; + fLog->logMsg(oss.str(), ERR_TBLLOCK_GET_LOCK_LOCKED, MSGLVL_ERROR); + + return ERR_TBLLOCK_GET_LOCK_LOCKED; } //------------------------------------------------------------------------------ // Change table lock state (to cleanup) //------------------------------------------------------------------------------ -int TableInfo::changeTableLockState( ) +int TableInfo::changeTableLockState() { - // If executing distributed (mode1) or central command (mode2) then - // don't worry about table locks. The client front-end will manage locks. - if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) - { - return NO_ERROR; - } - - std::string tblLockErrMsg; - bool bChanged = false; - - int rc = BRMWrapper::getInstance()->changeTableLockState ( - fTableLockID, - BRM::CLEANUP, - bChanged, - tblLockErrMsg ); - - if (rc == NO_ERROR) - { - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - - if (bChanged) - { - oss << "Table lock state changed to CLEANUP for table " << - fTableName << - "; OID-" << fTableOID << - "; lockID-" << fTableLockID; - } - else - { - oss << "Table lock state not changed to CLEANUP for table " << - fTableName << - "; OID-" << fTableOID << - "; lockID-" << fTableLockID << - ". Table lot locked."; - } - - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - } - else - { - ostringstream oss; - oss << "Error in changing table state for table " << fTableName << - "; OID-" << fTableOID << - "; lockID-" << fTableLockID << "; " << tblLockErrMsg; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } - + // If executing distributed (mode1) or central command (mode2) then + // don't worry about table locks. The client front-end will manage locks. + if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + { return NO_ERROR; + } + + std::string tblLockErrMsg; + bool bChanged = false; + + int rc = + BRMWrapper::getInstance()->changeTableLockState(fTableLockID, BRM::CLEANUP, bChanged, tblLockErrMsg); + + if (rc == NO_ERROR) + { + if (fLog->isDebug(DEBUG_1)) + { + ostringstream oss; + + if (bChanged) + { + oss << "Table lock state changed to CLEANUP for table " << fTableName << "; OID-" << fTableOID + << "; lockID-" << fTableLockID; + } + else + { + oss << "Table lock state not changed to CLEANUP for table " << fTableName << "; OID-" << fTableOID + << "; lockID-" << fTableLockID << ". Table lot locked."; + } + + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + } + else + { + ostringstream oss; + oss << "Error in changing table state for table " << fTableName << "; OID-" << fTableOID << "; lockID-" + << fTableLockID << "; " << tblLockErrMsg; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + return NO_ERROR; } //------------------------------------------------------------------------------ // Releases DB table lock assigned to this TableInfo object. //------------------------------------------------------------------------------ -int TableInfo::releaseTableLock( ) +int TableInfo::releaseTableLock() { - // If executing distributed (mode1) or central command (mode2) then - // don't worry about table locks. The client front-end will manage locks. - if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + // If executing distributed (mode1) or central command (mode2) then + // don't worry about table locks. The client front-end will manage locks. + if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + { + if (fLog->isDebug(DEBUG_1)) { - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - oss << "Bypass releasing table lock in distributed mode, " - "for table " << fTableName << "; OID-" << fTableOID; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - return NO_ERROR; - } - - std::string tblLockErrMsg; - bool bReleased = false; - - // Unlock the database table - int rc = BRMWrapper::getInstance()->releaseTableLock ( - fTableLockID, - bReleased, - tblLockErrMsg ); - - if (rc == NO_ERROR) - { - fTableLocked = false; - - if (fLog->isDebug( DEBUG_1 )) - { - ostringstream oss; - - if (bReleased) - { - oss << "Table lock released for table " << fTableName << - "; OID-" << fTableOID << - "; lockID-" << fTableLockID; - } - else - { - oss << "Table lock not released for table " << fTableName << - "; OID-" << fTableOID << - "; lockID-" << fTableLockID << - ". Table not locked."; - } - - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - } - else - { - ostringstream oss; - oss << "Error in releasing table lock for table " << fTableName << - "; OID-" << fTableOID << - "; lockID-" << fTableLockID << "; " << tblLockErrMsg; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; + ostringstream oss; + oss << "Bypass releasing table lock in distributed mode, " + "for table " + << fTableName << "; OID-" << fTableOID; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } return NO_ERROR; + } + + std::string tblLockErrMsg; + bool bReleased = false; + + // Unlock the database table + int rc = BRMWrapper::getInstance()->releaseTableLock(fTableLockID, bReleased, tblLockErrMsg); + + if (rc == NO_ERROR) + { + fTableLocked = false; + + if (fLog->isDebug(DEBUG_1)) + { + ostringstream oss; + + if (bReleased) + { + oss << "Table lock released for table " << fTableName << "; OID-" << fTableOID << "; lockID-" + << fTableLockID; + } + else + { + oss << "Table lock not released for table " << fTableName << "; OID-" << fTableOID << "; lockID-" + << fTableLockID << ". Table not locked."; + } + + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + } + else + { + ostringstream oss; + oss << "Error in releasing table lock for table " << fTableName << "; OID-" << fTableOID << "; lockID-" + << fTableLockID << "; " << tblLockErrMsg; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; + } + + return NO_ERROR; } //------------------------------------------------------------------------------ // Delete bulk rollback metadata file. //------------------------------------------------------------------------------ -void TableInfo::deleteMetaDataRollbackFile( ) +void TableInfo::deleteMetaDataRollbackFile() { - // If executing distributed (mode1) or central command (mode2) then - // don't worry about table locks, or deleting meta data files. The - // client front-end will manage these tasks after all imports are finished. - if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) - { - return; - } + // If executing distributed (mode1) or central command (mode2) then + // don't worry about table locks, or deleting meta data files. The + // client front-end will manage these tasks after all imports are finished. + if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + { + return; + } - if (!fKeepRbMetaFile) + if (!fKeepRbMetaFile) + { + // Treat any error as non-fatal, though we log it. + try { - // Treat any error as non-fatal, though we log it. - try - { - fRBMetaWriter.deleteFile(); - } - catch (WeException& ex) - { - ostringstream oss; - oss << "Error deleting meta file; " << ex.what(); - fLog->logMsg(oss.str(), ex.errorCode(), MSGLVL_ERROR); - } + fRBMetaWriter.deleteFile(); } + catch (WeException& ex) + { + ostringstream oss; + oss << "Error deleting meta file; " << ex.what(); + fLog->logMsg(oss.str(), ex.errorCode(), MSGLVL_ERROR); + } + } } //------------------------------------------------------------------------------ @@ -1997,33 +1900,32 @@ void TableInfo::deleteMetaDataRollbackFile( ) // This function triggers this action. //------------------------------------------------------------------------------ // @bug 5572 - Add db file confirmation for HDFS -int TableInfo::confirmDBFileChanges( ) +int TableInfo::confirmDBFileChanges() { - // Unlike deleteTempDBFileChanges(), note that confirmDBFileChanges() - // executes regardless of the import mode. We go ahead and confirm - // the file changes at the end of a successful cpimport.bin. - if (idbdatafile::IDBPolicy::useHdfs()) + // Unlike deleteTempDBFileChanges(), note that confirmDBFileChanges() + // executes regardless of the import mode. We go ahead and confirm + // the file changes at the end of a successful cpimport.bin. + if (idbdatafile::IDBPolicy::useHdfs()) + { + ostringstream oss; + oss << "Confirming DB file changes for " << fTableName; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + std::string errMsg; + ConfirmHdfsDbFile confirmHdfs; + int rc = confirmHdfs.confirmDbFileListFromMetaFile(fTableOID, errMsg); + + if (rc != NO_ERROR) { - ostringstream oss; - oss << "Confirming DB file changes for " << fTableName; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + ostringstream ossErrMsg; + ossErrMsg << "Unable to confirm changes to table " << fTableName << "; " << errMsg; + fLog->logMsg(ossErrMsg.str(), rc, MSGLVL_ERROR); - std::string errMsg; - ConfirmHdfsDbFile confirmHdfs; - int rc = confirmHdfs.confirmDbFileListFromMetaFile( fTableOID, errMsg ); - - if (rc != NO_ERROR) - { - ostringstream ossErrMsg; - ossErrMsg << "Unable to confirm changes to table " << fTableName << - "; " << errMsg; - fLog->logMsg( ossErrMsg.str(), rc, MSGLVL_ERROR ); - - return rc; - } + return rc; } + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -2031,37 +1933,35 @@ int TableInfo::confirmDBFileChanges( ) // This function triggers this action. //------------------------------------------------------------------------------ // @bug 5572 - Add db file confirmation for HDFS -void TableInfo::deleteTempDBFileChanges( ) +void TableInfo::deleteTempDBFileChanges() { - // If executing distributed (mode1) or central command (mode2) then - // no action necessary. The client front-end will initiate the deletion - // of the temp files, only after all the distributed imports have - // successfully completed. - if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || - (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + // If executing distributed (mode1) or central command (mode2) then + // no action necessary. The client front-end will initiate the deletion + // of the temp files, only after all the distributed imports have + // successfully completed. + if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) + { + return; + } + + if (idbdatafile::IDBPolicy::useHdfs()) + { + ostringstream oss; + oss << "Deleting DB temp swap files for " << fTableName; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + + std::string errMsg; + ConfirmHdfsDbFile confirmHdfs; + int rc = confirmHdfs.endDbFileListFromMetaFile(fTableOID, true, errMsg); + + // Treat any error as non-fatal, though we log it. + if (rc != NO_ERROR) { - return; - } - - if (idbdatafile::IDBPolicy::useHdfs()) - { - ostringstream oss; - oss << "Deleting DB temp swap files for " << fTableName; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - - std::string errMsg; - ConfirmHdfsDbFile confirmHdfs; - int rc = confirmHdfs.endDbFileListFromMetaFile(fTableOID, true, errMsg); - - // Treat any error as non-fatal, though we log it. - if (rc != NO_ERROR) - { - ostringstream ossErrMsg; - ossErrMsg << "Unable to delete temp swap files for table " << - fTableName << "; " << errMsg; - fLog->logMsg( ossErrMsg.str(), rc, MSGLVL_ERROR ); - } + ostringstream ossErrMsg; + ossErrMsg << "Unable to delete temp swap files for table " << fTableName << "; " << errMsg; + fLog->logMsg(ossErrMsg.str(), rc, MSGLVL_ERROR); } + } } //------------------------------------------------------------------------------ @@ -2079,274 +1979,239 @@ void TableInfo::deleteTempDBFileChanges( ) // HWM, etc to be validated for the columns belonging to jobTable. // stage - Current stage we are validating. "Starting" or "Ending". //------------------------------------------------------------------------------ -int TableInfo::validateColumnHWMs( - const JobTable* jobTable, - const std::vector& segFileInfo, - const char* stage ) +int TableInfo::validateColumnHWMs(const JobTable* jobTable, const std::vector& segFileInfo, + const char* stage) { - int rc = NO_ERROR; + int rc = NO_ERROR; - // Used to track first 1-byte, 2-byte, 4-byte, and 8-byte columns in table - int byte1First = -1; - int byte2First = -1; - int byte4First = -1; - int byte8First = -1; - int byte16First = -1; + // Used to track first 1-byte, 2-byte, 4-byte, and 8-byte columns in table + int byte1First = -1; + int byte2First = -1; + int byte4First = -1; + int byte8First = -1; + int byte16First = -1; - // Make sure the HWMs for all 1-byte columns match; same for all 2-byte, - // 4-byte, and 8-byte columns as well. - for (unsigned k = 0; k < segFileInfo.size(); k++) + // Make sure the HWMs for all 1-byte columns match; same for all 2-byte, + // 4-byte, and 8-byte columns as well. + for (unsigned k = 0; k < segFileInfo.size(); k++) + { + int k1 = 0; + + // Validate HWMs in jobTable if we have it, else use fColumns. + const JobColumn& jobColK = ((jobTable != 0) ? jobTable->colList[k] : fColumns[k].column); + + // Find the first 1-byte, 2-byte, 4-byte, and 8-byte columns. + // Use those as our reference HWM for the respective column widths. + switch (jobColK.width) { - int k1 = 0; + case 1: + { + if (byte1First == -1) + byte1First = k; - // Validate HWMs in jobTable if we have it, else use fColumns. - const JobColumn& jobColK = - ( (jobTable != 0) ? jobTable->colList[k] : fColumns[k].column ); + k1 = byte1First; + break; + } - // Find the first 1-byte, 2-byte, 4-byte, and 8-byte columns. - // Use those as our reference HWM for the respective column widths. - switch ( jobColK.width ) - { - case 1: - { - if (byte1First == -1) - byte1First = k; + case 2: + { + if (byte2First == -1) + byte2First = k; - k1 = byte1First; - break; - } + k1 = byte2First; + break; + } - case 2: - { - if (byte2First == -1) - byte2First = k; + case 4: + { + if (byte4First == -1) + byte4First = k; - k1 = byte2First; - break; - } + k1 = byte4First; + break; + } - case 4: - { - if (byte4First == -1) - byte4First = k; + case 8: + { + if (byte8First == -1) + byte8First = k; - k1 = byte4First; - break; - } + k1 = byte8First; + break; + } + case 16: + { + if (byte16First == -1) + byte16First = k; - case 8: - { - if (byte8First == -1) - byte8First = k; + k1 = byte16First; + break; + } + default: + { + ostringstream oss; + oss << stage + << " Unsupported width for" + " OID-" + << jobColK.mapOid << "; column-" << jobColK.colName << "; width-" << jobColK.width; + fLog->logMsg(oss.str(), ERR_BRM_UNSUPP_WIDTH, MSGLVL_ERROR); + return ERR_BRM_UNSUPP_WIDTH; + } + } // end of switch based on column width. - k1 = byte8First; - break; - } - case 16: - { - if (byte16First == -1) - byte16First = k; + // Validate HWMs in jobTable if we have it, else use fColumns. + const JobColumn& jobColK1 = ((jobTable != 0) ? jobTable->colList[k1] : fColumns[k1].column); - k1 = byte16First; - break; - } - default: - { - ostringstream oss; - oss << stage << " Unsupported width for" - " OID-" << jobColK.mapOid << - "; column-" << jobColK.colName << - "; width-" << jobColK.width; - fLog->logMsg( oss.str(), ERR_BRM_UNSUPP_WIDTH, MSGLVL_ERROR ); - return ERR_BRM_UNSUPP_WIDTH; - } - } // end of switch based on column width. + // std::cout << "dbg: comparing0 " << stage << " refcol-" << k1 << + // "; wid-" << jobColK1.width << "; hwm-" << segFileInfo[k1].fLocalHwm << + // " col-" << k << + // "; wid-" << jobColK.width << " ; hwm-"<colList[k1] : fColumns[k1].column ); - - //std::cout << "dbg: comparing0 " << stage << " refcol-" << k1 << - // "; wid-" << jobColK1.width << "; hwm-" << segFileInfo[k1].fLocalHwm << - // " col-" << k << - // "; wid-" << jobColK.width << " ; hwm-"<logMsg( oss.str(), ERR_BRM_HWMS_NOT_EQUAL, MSGLVL_ERROR ); - return ERR_BRM_HWMS_NOT_EQUAL; - } - - // HWM DBRoot, partition, and segment number should match for all - // columns; so compare DBRoot, part#, and seg# with first column. - if ((segFileInfo[0].fDbRoot != segFileInfo[k].fDbRoot) || - (segFileInfo[0].fPartition != segFileInfo[k].fPartition) || - (segFileInfo[0].fSegment != segFileInfo[k].fSegment)) - { - const JobColumn& jobCol0 = - ( (jobTable != 0) ? jobTable->colList[0] : fColumns[0].column ); - - ostringstream oss; - oss << stage << " HWM DBRoot,Part#, or Seg# do not match for" - " OID1-" << jobCol0.mapOid << - "; column-" << jobCol0.colName << - "; DBRoot-" << segFileInfo[0].fDbRoot << - "; partition-" << segFileInfo[0].fPartition << - "; segment-" << segFileInfo[0].fSegment << - "; hwm-" << segFileInfo[0].fLocalHwm << - "; width-" << jobCol0.width << ':' << std::endl << - " and OID2-" << jobColK.mapOid << - "; column-" << jobColK.colName << - "; DBRoot-" << segFileInfo[k].fDbRoot << - "; partition-" << segFileInfo[k].fPartition << - "; segment-" << segFileInfo[k].fSegment << - "; hwm-" << segFileInfo[k].fLocalHwm << - "; width-" << jobColK.width; - fLog->logMsg( oss.str(), ERR_BRM_HWMS_NOT_EQUAL, MSGLVL_ERROR ); - return ERR_BRM_HWMS_NOT_EQUAL; - } - } // end of loop to compare all 1-byte HWMs, 2-byte HWMs, etc. - - // Validate/compare HWM for 1-byte column in relation to 2-byte column, etc. - // Without knowing the exact row count, we can't extrapolate the exact HWM - // for the wider column, but we can narrow it down to an expected range. - int refCol = 0; - int colIdx = 0; - - // Validate/compare HWMs given a 1-byte column as a starting point - if (byte1First >= 0) + // Validate that the HWM for this column (k) matches that of the + // corresponding reference column with the same width. + if ((segFileInfo[k1].fDbRoot != segFileInfo[k].fDbRoot) || + (segFileInfo[k1].fPartition != segFileInfo[k].fPartition) || + (segFileInfo[k1].fSegment != segFileInfo[k].fSegment) || + (segFileInfo[k1].fLocalHwm != segFileInfo[k].fLocalHwm)) { - refCol = byte1First; - - if ((rc = compareHWMs(byte1First, byte2First, 1, 2, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } - - if ((rc = compareHWMs(byte1First, byte4First, 1, 4, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } - - if ((rc = compareHWMs(byte1First, byte8First, 1, 8, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } - - if ((rc = compareHWMs(byte1First, byte16First, 1, 16, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } + ostringstream oss; + oss << stage + << " HWMs do not match for" + " OID1-" + << jobColK1.mapOid << "; column-" << jobColK1.colName << "; DBRoot-" << segFileInfo[k1].fDbRoot + << "; partition-" << segFileInfo[k1].fPartition << "; segment-" << segFileInfo[k1].fSegment + << "; hwm-" << segFileInfo[k1].fLocalHwm << "; width-" << jobColK1.width << ':' << std::endl + << " and OID2-" << jobColK.mapOid << "; column-" << jobColK.colName << "; DBRoot-" + << segFileInfo[k].fDbRoot << "; partition-" << segFileInfo[k].fPartition << "; segment-" + << segFileInfo[k].fSegment << "; hwm-" << segFileInfo[k].fLocalHwm << "; width-" << jobColK.width; + fLog->logMsg(oss.str(), ERR_BRM_HWMS_NOT_EQUAL, MSGLVL_ERROR); + return ERR_BRM_HWMS_NOT_EQUAL; } - // Validate/compare HWMs given a 2-byte column as a starting point - if (byte2First >= 0) + // HWM DBRoot, partition, and segment number should match for all + // columns; so compare DBRoot, part#, and seg# with first column. + if ((segFileInfo[0].fDbRoot != segFileInfo[k].fDbRoot) || + (segFileInfo[0].fPartition != segFileInfo[k].fPartition) || + (segFileInfo[0].fSegment != segFileInfo[k].fSegment)) { - refCol = byte2First; + const JobColumn& jobCol0 = ((jobTable != 0) ? jobTable->colList[0] : fColumns[0].column); - if ((rc = compareHWMs(byte2First, byte4First, 2, 4, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } + ostringstream oss; + oss << stage + << " HWM DBRoot,Part#, or Seg# do not match for" + " OID1-" + << jobCol0.mapOid << "; column-" << jobCol0.colName << "; DBRoot-" << segFileInfo[0].fDbRoot + << "; partition-" << segFileInfo[0].fPartition << "; segment-" << segFileInfo[0].fSegment + << "; hwm-" << segFileInfo[0].fLocalHwm << "; width-" << jobCol0.width << ':' << std::endl + << " and OID2-" << jobColK.mapOid << "; column-" << jobColK.colName << "; DBRoot-" + << segFileInfo[k].fDbRoot << "; partition-" << segFileInfo[k].fPartition << "; segment-" + << segFileInfo[k].fSegment << "; hwm-" << segFileInfo[k].fLocalHwm << "; width-" << jobColK.width; + fLog->logMsg(oss.str(), ERR_BRM_HWMS_NOT_EQUAL, MSGLVL_ERROR); + return ERR_BRM_HWMS_NOT_EQUAL; + } + } // end of loop to compare all 1-byte HWMs, 2-byte HWMs, etc. - if ((rc = compareHWMs(byte2First, byte8First, 2, 8, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } + // Validate/compare HWM for 1-byte column in relation to 2-byte column, etc. + // Without knowing the exact row count, we can't extrapolate the exact HWM + // for the wider column, but we can narrow it down to an expected range. + int refCol = 0; + int colIdx = 0; - if ((rc = compareHWMs(byte2First, byte16First, 2, 16, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } + // Validate/compare HWMs given a 1-byte column as a starting point + if (byte1First >= 0) + { + refCol = byte1First; + + if ((rc = compareHWMs(byte1First, byte2First, 1, 2, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; } - // Validate/compare HWMs given a 4-byte column as a starting point - if (byte4First >= 0) + if ((rc = compareHWMs(byte1First, byte4First, 1, 4, segFileInfo, colIdx) != NO_ERROR)) { - refCol = byte4First; - - if ((rc = compareHWMs(byte4First, byte8First, 4, 8, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } - - if ((rc = compareHWMs(byte4First, byte16First, 4, 16, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } - + goto errorCheck; } - if (byte8First >= 0) + + if ((rc = compareHWMs(byte1First, byte8First, 1, 8, segFileInfo, colIdx) != NO_ERROR)) { - refCol = byte8First; - if ((rc = compareHWMs(byte8First, byte16First, 8, 16, - segFileInfo, colIdx) != NO_ERROR)) - { - goto errorCheck; - } + goto errorCheck; } + if ((rc = compareHWMs(byte1First, byte16First, 1, 16, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + } + + // Validate/compare HWMs given a 2-byte column as a starting point + if (byte2First >= 0) + { + refCol = byte2First; + + if ((rc = compareHWMs(byte2First, byte4First, 2, 4, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + + if ((rc = compareHWMs(byte2First, byte8First, 2, 8, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + + if ((rc = compareHWMs(byte2First, byte16First, 2, 16, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + } + + // Validate/compare HWMs given a 4-byte column as a starting point + if (byte4First >= 0) + { + refCol = byte4First; + + if ((rc = compareHWMs(byte4First, byte8First, 4, 8, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + + if ((rc = compareHWMs(byte4First, byte16First, 4, 16, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + } + if (byte8First >= 0) + { + refCol = byte8First; + if ((rc = compareHWMs(byte8First, byte16First, 8, 16, segFileInfo, colIdx) != NO_ERROR)) + { + goto errorCheck; + } + } + // To avoid repeating this message 6 times in the preceding source code, we // use the "dreaded" goto to branch to this single place for error handling. errorCheck: - if (rc != NO_ERROR) - { - const JobColumn& jobColRef = ( (jobTable != 0) ? - jobTable->colList[refCol] : fColumns[refCol].column ); - const JobColumn& jobColIdx = ( (jobTable != 0) ? - jobTable->colList[colIdx] : fColumns[colIdx].column ); + if (rc != NO_ERROR) + { + const JobColumn& jobColRef = ((jobTable != 0) ? jobTable->colList[refCol] : fColumns[refCol].column); + const JobColumn& jobColIdx = ((jobTable != 0) ? jobTable->colList[colIdx] : fColumns[colIdx].column); - ostringstream oss; - oss << stage << " HWMs are not in sync for" - " OID1-" << jobColRef.mapOid << - "; column-" << jobColRef.colName << - "; DBRoot-" << segFileInfo[refCol].fDbRoot << - "; partition-" << segFileInfo[refCol].fPartition << - "; segment-" << segFileInfo[refCol].fSegment << - "; hwm-" << segFileInfo[refCol].fLocalHwm << - "; width-" << jobColRef.width << ':' << std::endl << - " and OID2-" << jobColIdx.mapOid << - "; column-" << jobColIdx.colName << - "; DBRoot-" << segFileInfo[colIdx].fDbRoot << - "; partition-" << segFileInfo[colIdx].fPartition << - "; segment-" << segFileInfo[colIdx].fSegment << - "; hwm-" << segFileInfo[colIdx].fLocalHwm << - "; width-" << jobColIdx.width; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - } + ostringstream oss; + oss << stage + << " HWMs are not in sync for" + " OID1-" + << jobColRef.mapOid << "; column-" << jobColRef.colName << "; DBRoot-" << segFileInfo[refCol].fDbRoot + << "; partition-" << segFileInfo[refCol].fPartition << "; segment-" << segFileInfo[refCol].fSegment + << "; hwm-" << segFileInfo[refCol].fLocalHwm << "; width-" << jobColRef.width << ':' << std::endl + << " and OID2-" << jobColIdx.mapOid << "; column-" << jobColIdx.colName << "; DBRoot-" + << segFileInfo[colIdx].fDbRoot << "; partition-" << segFileInfo[colIdx].fPartition << "; segment-" + << segFileInfo[colIdx].fSegment << "; hwm-" << segFileInfo[colIdx].fLocalHwm << "; width-" + << jobColIdx.width; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -2356,21 +2221,21 @@ errorCheck: // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int TableInfo::initBulkRollbackMetaData( ) +int TableInfo::initBulkRollbackMetaData() { - int rc = NO_ERROR; + int rc = NO_ERROR; - try - { - fRBMetaWriter.init( fTableOID, fTableName ); - } - catch (WeException& ex) - { - fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR); - rc = ex.errorCode(); - } + try + { + fRBMetaWriter.init(fTableOID, fTableName); + } + catch (WeException& ex) + { + fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR); + rc = ex.errorCode(); + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -2391,60 +2256,56 @@ int TableInfo::initBulkRollbackMetaData( ) // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int TableInfo::saveBulkRollbackMetaData( Job& job, - const std::vector& segFileInfo, - const std::vector& dbRootHWMInfoVecCol ) +int TableInfo::saveBulkRollbackMetaData(Job& job, const std::vector& segFileInfo, + const std::vector& dbRootHWMInfoVecCol) { - int rc = NO_ERROR; + int rc = NO_ERROR; - std::vector cols; - std::vector dctnryOids; + std::vector cols; + std::vector dctnryOids; - // Loop through the columns in the specified table - for ( size_t i = 0; i < job.jobTableList[fTableId].colList.size(); i++ ) - { - JobColumn& jobCol = job.jobTableList[fTableId].colList[i]; + // Loop through the columns in the specified table + for (size_t i = 0; i < job.jobTableList[fTableId].colList.size(); i++) + { + JobColumn& jobCol = job.jobTableList[fTableId].colList[i]; - Column col; - col.colNo = i; - col.colWidth = jobCol.width; - col.colType = jobCol.weType; - col.colDataType = jobCol.dataType; - col.dataFile.oid = jobCol.mapOid; - col.dataFile.fid = jobCol.mapOid; - col.dataFile.hwm = segFileInfo[i].fLocalHwm; // starting HWM - col.dataFile.pFile = 0; - col.dataFile.fPartition = segFileInfo[i].fPartition; // starting Part# - col.dataFile.fSegment = segFileInfo[i].fSegment; // starting seg# - col.dataFile.fDbRoot = segFileInfo[i].fDbRoot; // starting DBRoot - col.compressionType = jobCol.compressionType; - cols.push_back( col ); + Column col; + col.colNo = i; + col.colWidth = jobCol.width; + col.colType = jobCol.weType; + col.colDataType = jobCol.dataType; + col.dataFile.oid = jobCol.mapOid; + col.dataFile.fid = jobCol.mapOid; + col.dataFile.hwm = segFileInfo[i].fLocalHwm; // starting HWM + col.dataFile.pFile = 0; + col.dataFile.fPartition = segFileInfo[i].fPartition; // starting Part# + col.dataFile.fSegment = segFileInfo[i].fSegment; // starting seg# + col.dataFile.fDbRoot = segFileInfo[i].fDbRoot; // starting DBRoot + col.compressionType = jobCol.compressionType; + cols.push_back(col); - OID dctnryOid = 0; + OID dctnryOid = 0; - if (jobCol.colType == COL_TYPE_DICT) - dctnryOid = jobCol.dctnry.dctnryOid; + if (jobCol.colType == COL_TYPE_DICT) + dctnryOid = jobCol.dctnry.dctnryOid; - dctnryOids.push_back( dctnryOid ); + dctnryOids.push_back(dctnryOid); - } // end of loop through columns + } // end of loop through columns - fRBMetaWriter.setUIDGID(this); + fRBMetaWriter.setUIDGID(this); - try - { - fRBMetaWriter.saveBulkRollbackMetaData( - cols, - dctnryOids, - dbRootHWMInfoVecCol ); - } - catch (WeException& ex) - { - fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR); - rc = ex.errorCode(); - } + try + { + fRBMetaWriter.saveBulkRollbackMetaData(cols, dctnryOids, dbRootHWMInfoVecCol); + } + catch (WeException& ex) + { + fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR); + rc = ex.errorCode(); + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -2453,28 +2314,28 @@ int TableInfo::saveBulkRollbackMetaData( Job& job, // catalog back in line with the latest auto increment next value generated by // BRM. //------------------------------------------------------------------------------ -int TableInfo::synchronizeAutoInc( ) +int TableInfo::synchronizeAutoInc() { - for (unsigned i = 0; i < fColumns.size(); ++i) + for (unsigned i = 0; i < fColumns.size(); ++i) + { + if (fColumns[i].column.autoIncFlag) { - if (fColumns[i].column.autoIncFlag) - { - // TBD: Do we rollback flush cache error for autoinc. - // Not sure we should bail out and rollback on a - // ERR_BLKCACHE_FLUSH_LIST error, but we currently - // rollback for "any" updateNextValue() error - int rc = fColumns[i].finishAutoInc( ); + // TBD: Do we rollback flush cache error for autoinc. + // Not sure we should bail out and rollback on a + // ERR_BLKCACHE_FLUSH_LIST error, but we currently + // rollback for "any" updateNextValue() error + int rc = fColumns[i].finishAutoInc(); - if (rc != NO_ERROR) - { - return rc; - } + if (rc != NO_ERROR) + { + return rc; + } - break; // okay to break; only 1 autoinc column per table - } + break; // okay to break; only 1 autoinc column per table } + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -2483,109 +2344,92 @@ int TableInfo::synchronizeAutoInc( ) // mode3 import. Table lock and bulk rollbacks are managed by parent cpimport // (file splitter) process for mode1 and mode2. //------------------------------------------------------------------------------ -int TableInfo::rollbackWork( ) +int TableInfo::rollbackWork() { - // Close any column or store files left open by abnormal termination. - // We want to do this before reopening the files and doing a bulk rollback. - closeOpenDbFiles(); + // Close any column or store files left open by abnormal termination. + // We want to do this before reopening the files and doing a bulk rollback. + closeOpenDbFiles(); - // Abort "local" bulk rollback if a DBRoot from the start of the job, is - // now missing. User should run cleartablelock to execute a rollback on - // this PM "and" the PM where the DBRoot was moved to. - std::vector dbRootIds; - Config::getRootIdList( dbRootIds ); + // Abort "local" bulk rollback if a DBRoot from the start of the job, is + // now missing. User should run cleartablelock to execute a rollback on + // this PM "and" the PM where the DBRoot was moved to. + std::vector dbRootIds; + Config::getRootIdList(dbRootIds); - for (unsigned int j = 0; j < fOrigDbRootIds.size(); j++) + for (unsigned int j = 0; j < fOrigDbRootIds.size(); j++) + { + bool bFound = false; + + for (unsigned int k = 0; k < dbRootIds.size(); k++) { - bool bFound = false; - - for (unsigned int k = 0; k < dbRootIds.size(); k++) - { - if (fOrigDbRootIds[j] == dbRootIds[k]) - { - bFound = true; - break; - } - } - - if (!bFound) - { - ostringstream oss; - oss << "Mode3 bulk rollback not performed for table " << - fTableName << "; DBRoot" << fOrigDbRootIds[j] << - " moved from this PM during bulk load. " << - " Run cleartablelock to rollback and release the table lock " << - "across PMs."; - int rc = ERR_BULK_ROLLBACK_MISS_ROOT; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } + if (fOrigDbRootIds[j] == dbRootIds[k]) + { + bFound = true; + break; + } } - // Restore/rollback the DB files if we got far enough to begin processing - // this table. - int rc = NO_ERROR; - - if (hasProcessingBegun()) + if (!bFound) { - BulkRollbackMgr rbMgr( fTableOID, - fTableLockID, - fTableName, - fProcessName, fLog ); - - rc = rbMgr.rollback( fKeepRbMetaFile ); - - if (rc != NO_ERROR) - { - ostringstream oss; - oss << "Error rolling back table " << fTableName << - "; " << rbMgr.getErrorMsg(); - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; - } + ostringstream oss; + oss << "Mode3 bulk rollback not performed for table " << fTableName << "; DBRoot" << fOrigDbRootIds[j] + << " moved from this PM during bulk load. " + << " Run cleartablelock to rollback and release the table lock " + << "across PMs."; + int rc = ERR_BULK_ROLLBACK_MISS_ROOT; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } + } - // Delete the meta data files after rollback is complete - deleteMetaDataRollbackFile( ); + // Restore/rollback the DB files if we got far enough to begin processing + // this table. + int rc = NO_ERROR; - // Release the table lock - rc = releaseTableLock( ); + if (hasProcessingBegun()) + { + BulkRollbackMgr rbMgr(fTableOID, fTableLockID, fTableName, fProcessName, fLog); + + rc = rbMgr.rollback(fKeepRbMetaFile); if (rc != NO_ERROR) { - ostringstream oss; - oss << "Table lock not cleared for table " << fTableName; - fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); - return rc; + ostringstream oss; + oss << "Error rolling back table " << fTableName << "; " << rbMgr.getErrorMsg(); + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); + return rc; } + } + // Delete the meta data files after rollback is complete + deleteMetaDataRollbackFile(); + + // Release the table lock + rc = releaseTableLock(); + + if (rc != NO_ERROR) + { + ostringstream oss; + oss << "Table lock not cleared for table " << fTableName; + fLog->logMsg(oss.str(), rc, MSGLVL_ERROR); return rc; + } + + return rc; } //------------------------------------------------------------------------------ // Allocate extent from BRM (through the stripe allocator). //------------------------------------------------------------------------------ -int TableInfo::allocateBRMColumnExtent(OID columnOID, - uint16_t dbRoot, - uint32_t& partition, - uint16_t& segment, - BRM::LBID_t& startLbid, - int& allocSize, - HWM& hwm, - std::string& errMsg ) +int TableInfo::allocateBRMColumnExtent(OID columnOID, uint16_t dbRoot, uint32_t& partition, uint16_t& segment, + BRM::LBID_t& startLbid, int& allocSize, HWM& hwm, std::string& errMsg) { - int rc = fExtentStrAlloc.allocateExtent( columnOID, - dbRoot, - partition, - segment, - startLbid, - allocSize, - hwm, - errMsg ); - //fExtentStrAlloc.print(); + int rc = fExtentStrAlloc.allocateExtent(columnOID, dbRoot, partition, segment, startLbid, allocSize, hwm, + errMsg); + // fExtentStrAlloc.print(); - return rc; + return rc; } -} +} // namespace WriteEngine // end of namespace diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index 93e03b41c..612a637a4 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_tableinfo.h 4648 2013-05-29 21:42:40Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_tableinfo.h 4648 2013-05-29 21:42:40Z rdempsey $ + * + *******************************************************************************/ #ifndef _WE_TABLEINFO_H #define _WE_TABLEINFO_H @@ -52,445 +52,416 @@ namespace WriteEngine { - /* @brief Class which maintains the information for a table. */ class TableInfo : public WeUIDGID { -private: - - //-------------------------------------------------------------------------- - // Private Data Members - //-------------------------------------------------------------------------- - - int fTableId; // Table id - int fBufferSize; // Size of buffer used by BulkLoadBuffer - size_t fFileBufSize; // Size of fFileBuffer passed to setvbuf - // to read import files. Comes from - // writeBufferSize tag in job xml file - char fColDelim; // Used to delimit col values in a row - volatile Status fStatusTI; // Status of table. Made volatile to - // insure BulkLoad methods can access - // (thru getStatusTI()) correctly w/o - // having to go through a mutex lock. - int fReadBufCount; // Number of read buffers - // (size of fBuffers vector) - unsigned fNumberOfColumns; // Number of ColumnInfo objs in this tbl - // (size of fColumns vector) - FILE* fHandle; // Handle to the input load file - int fCurrentReadBuffer; // Id of current buffer being popu- - // lated by the read thread - RID fTotalReadRows; // Total number of rows read - volatile unsigned fTotalErrRows; // Total error rows among all input - // for this table. Is volatile to - // insure parser & reader threads - // see the latest value. - unsigned fMaxErrorRows; // Maximum error rows - int fLastBufferId; // Id of the last buffer - char* fFileBuffer; // File buffer passed to setvbuf() - int fCurrentParseBuffer; // Id of leading current buffer being - // parsed. There can be more than 1 - // buffer being parsed concurrently. - unsigned fNumberOfColsParsed; // Number of columns completed parsing - boost::ptr_vector fColumns; // Columns of the table - boost::ptr_vector fBuffers; // Array of read buffers. Used - // to pass data from the read - // thread to the write thread(s) - - /* fSyncUpdatesTI is the mutex used to synchronize updates to TableInfo - * (excluding fErrorRows and fErrDataRows) - * - * This mutex is also used to coordinate access to fColumnLocks and - * fParseComplete (in BulkLoadBuffer) for the buffers within a table. - * See bulkloadBuffer.h for more information. - * - * As the controlling class, TableInfo is the one that is always - * getting/setting the status of the BulkLoadBuffer objects, so - * fSyncUpdatesTI is also used to set/get the BulkLoadBuffer status. - */ - boost::mutex fSyncUpdatesTI; - - boost::mutex fErrorRptInfoMutex; // Used to synhronize access to - // fRejectDataFile & fRejectErrFile - int fLocker; // Read thread id reading this table - std::vector fLoadFileList; // Load files - std::string fFileName; // Current load file - std::string fTableName; // File name of the table - OID fTableOID; // OID of the table - std::string fjobFileName; // Job file name - int fJobId; // Job ID number - Log* fLog; // Object used for logging - timeval fStartTime; // Time when reading and processing began for this table - const BRM::TxnID fTxnID; // Transaction id for the build load - RBMetaWriter fRBMetaWriter; // Manages the writing of bulk roll- - // back meta data for this table - std::string fProcessName; // Name of application process used - // in db table locks - bool fKeepRbMetaFile; // Keep or delete bulk rollback meta - // data file - bool fbTruncationAsError; // Treat string truncation as error - ImportDataMode fImportDataMode; // Import data in text or binary mode - std::string fTimeZone; // Timezone used by TIMESTAMP data type - - volatile bool fTableLocked; // Do we have db table lock - - bool fReadFromStdin; // Read import file from STDIN - bool fReadFromS3; // Read import file from S3 - std::string fS3Host; // S3 hostname - std::string fS3Key; // S3 key - std::string fS3Secret; // S3 secret key - std::string fS3Bucket; // S3 bucket - std::string fS3Region; - ms3_st* ms3; // S3 object - size_t fS3ReadLength; - size_t fS3ParseLength; - bool fNullStringMode; // Treat "NULL" as a null value - char fEnclosedByChar; // Character to enclose col values - char fEscapeChar; // Escape character used in conjunc- - // tion with fEnclosedByChar - bool fProcessingBegun; // Has processing begun on this tbl - BulkModeType fBulkMode; // Distributed bulk mode (1,2, or 3) - std::string fBRMRptFileName; // Name of distributed mode rpt file - BRMReporter fBRMReporter; // Object used to report BRM updates - uint64_t fTableLockID; // Unique table lock ID - std::vector fOrigDbRootIds; // List of DBRoots at start of job - - std::string fErrorDir; // Opt dir for *.err and *.bad files - std::vector fErrFiles; // List of *.err files for this table - std::vector fBadFiles; // List of *.bad files for this table - std::ofstream fRejectDataFile; // File containing rejected rows - std::ofstream fRejectErrFile; // File containing errmsgs for bad rows - std::string fRejectDataFileName; // Filename for current fRejectDataFile - std::string fRejectErrFileName; // Filename for current fRejectErrFile - unsigned int fRejectDataCnt; // Running row count in current bad file - unsigned int fRejectErrCnt; // Running count in current err msg file - - ExtentStripeAlloc fExtentStrAlloc; // Extent stripe allocator for this tbl - querytele::QueryTeleClient fQtc; // Query Tele client - - oam::OamCache* fOamCachePtr; // OamCache: ptr is copyable - boost::uuids::uuid fJobUUID; // Job UUID - std::vector fDictFlushBlks;//dict blks to be flushed from cache - - //-------------------------------------------------------------------------- - // Private Functions - //-------------------------------------------------------------------------- - - int changeTableLockState(); // Change state of table lock to cleanup - void closeTableFile(); // Close current tbl file; free buffer - void closeOpenDbFiles(); // Close DB files left open at job's end - int confirmDBFileChanges(); // Confirm DB file changes (on HDFS) - void deleteTempDBFileChanges(); // Delete DB temp swap files (on HDFS) - int finishBRM(); // Finish reporting updates for BRM - void freeProcessingBuffers(); // Free up Processing Buffers - bool isBufferAvailable(bool report);// Is tbl buffer available for reading - int openTableFile(); // Open data file and set the buffer - void reportTotals(double elapsedSec);//Report summary totals - void sleepMS(long int ms); // Sleep method - // Compare column HWM with the examplar HWM. - int compareHWMs(const int smallestColumnId, - const int widerColumnId, - const uint32_t smallerColumnWidth, - const uint32_t widerColumnWidth, - const std::vector& segFileInfo, - int& colIdx); - - int synchronizeAutoInc(); // Sychronize AutoInc in BRM with syscat - - // Write the list of errors for this table - void writeErrorList(const std::vector< std::pair >* errorRows, - const std::vector* errorDatRows, - bool bCloseFile); - - // Write out rejected rows, and corresponding error messages - void writeBadRows( const std::vector* errorDatRows, - bool bCloseFile ); - void writeErrReason( const std::vector< std::pair >* errorRows, - bool bCloseFile ); - - // Disable copy constructor and assignment operator - TableInfo (const TableInfo& tableInfo); // - TableInfo& operator =(const TableInfo& info); - -public: - - //------------------------------------------------------------------------- - // Public Functions - //------------------------------------------------------------------------- - - /** @brief Default constructor - */ - TableInfo(Log* logger, const BRM::TxnID, - const std::string& processName, - OID tableOID, - const std::string& tableName, - bool bKeepRbMetaFile); - - /** @brief Default destructor - */ - ~TableInfo(); - - /** @brief Acquire the DB table lock for this table - */ - int acquireTableLock(bool disableTimeOut = false ); - - /** @brief Get current table lock ID for this table - */ - uint64_t getTableLockID() const; - - /** @brief Release the DB table lock for this table - */ - int releaseTableLock( ); - - /** @brief Allocate an extent for the specified OID and DBRoot, using the - * internal "stripe" allocator. - * @param columnOID Allocate next extent for this column - * @param dbRoot Allocate extent on this DBRoot - * @param partition (in/out) If DBRoot is empty, this is an input arg, - * else it is assigned by BRM and returned as output - * @param segment (out) Segment number of extent created by BRM - * @param startLbid (out) Starting LBID for extent created by BRM - * @param allocSize (out) Num blocks allocated to extent by BRM - * @param hwm (out) FBO for extent created by BRM - * @param errMsg (out) Error message - */ - int allocateBRMColumnExtent(OID columnOID, - uint16_t dbRoot, - uint32_t& partition, - uint16_t& segment, - BRM::LBID_t& startLbid, - int& allocSize, - HWM& hwm, - std::string& errMsg ); - - /** @brief Delete the bulk rollback metadata file. - */ - void deleteMetaDataRollbackFile( ); - - /** @brief Get binary import mode. - */ - ImportDataMode getImportDataMode( ) const; - - /** @brief Get timezone. - */ - const std::string& getTimeZone( ) const; - - /** @brief Get number of buffers - */ - int getNumberOfBuffers() const; - - /** @brief Set the buffer size - * @param Buffer size - */ - void setBufferSize(const int bufSize); - - /** @brief Set the file buffer size. - * @param Buffer size - */ - void setFileBufferSize(const int fileBufSize); - - /** @brief Set the delimiter used to delimit column values within a row - */ - void setColDelimiter(const char delim); - - /** @brief Get table status - */ - Status getStatusTI() const; - - /** @brief Get current parse buffer - */ - int getCurrentParseBuffer() const; - - /** @brief Get the number of columns - */ - int getNumberOfColumns() const; - - /** @brief get the file name - */ - std::string getFileName() const; - - /** @brief Get the number of maximum allowed error rows - */ - unsigned getMaxErrorRows() const; - - /** @brief retrieve the tuncation as error setting for this - * import. When set, this causes char and varchar strings - * that are longer than the column definition to be treated - * as errors instead of warnings. - */ - bool getTruncationAsError() const; - - /** @brief set the maximum number of error rows allowed - */ - void setMaxErrorRows(const unsigned int maxErrorRows); - - /** @brief Set mode to treat "NULL" string as NULL value or not. - */ - void setNullStringMode( bool bMode ); - - /** @brief Set binary import data mode (text or binary). - */ - void setImportDataMode( ImportDataMode importMode ); - - /** @brief Set timezone. - */ - void setTimeZone( const std::string& timeZone ); - - /** @brief Enable distributed mode, saving BRM updates in rptFileName - */ - void setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName); - - /** @brief Set character optionally used to enclose input column values. - */ - void setEnclosedByChar( char enChar ); - - /** @brief Set escape char to use in conjunction with enclosed by char. - */ - void setEscapeChar ( char esChar ); - - /** @brief Has processing begun for this table. - */ - bool hasProcessingBegun( ); - - /** @brief set the table id - */ - void setTableId(const int& id); - - /** @brief get the file name - */ - std::string getTableName() const; - - /** @brief get the table OID - */ - OID getTableOID( ); - - /** @brief Set the directory for *.err and *.bad files. May be - * empty string, in which case we use current dir. - */ - void setErrorDir(const std::string& errorDir); - - /** @brief get the bulk rollback meta data writer object for this table - */ - RBMetaWriter* rbMetaWriter(); - - /** @brief Add column information to the table - */ - void addColumn(ColumnInfo* info); - - /** @brief Initialize the buffer list - * @param noOfBuffers Number of buffers to create for this table - * @param jobFieldRefList List of fields in this import - * @param fixedBinaryRecLen In binary mode, this is the fixed record length - * used to read the buffer; in text mode, this value is not used. - */ - int initializeBuffers(int noOfBuffers, - const JobFieldRefList& jobFieldRefList, - unsigned int fixedBinaryRecLen); - - /** @brief Read the table data into the read buffer - */ - int readTableData( ); - - /** @brief parse method - */ - int parseColumn(const int& columnId, const int& bufferId, - double& processingTime); - - /** @brief update the buffer status for column - */ - int setParseComplete(const int& columnId, - const int& bufferId, - double processingTime); - - /** @brief update the status to reflect a parsing error - */ - void setParseError (); - - /** @brief Check if buffer ready for parsing. - */ - bool bufferReadyForParse(const int& bufferId, bool report) const; - - /** @brief Check if a column is available for parsing in the buffer - * and return the column id if available - */ - int getColumnForParse(const int& id, const int& bufferId, bool report); - - /** @brief Do we have a db lock with the session manager for this table. - */ - bool isTableLocked(); - - /** @brief Lock the table for reading - */ - bool lockForRead(const int& locker); - - /** @brief Rollback changes made to "this" table by the current import job - */ - int rollbackWork( ); - - /** @brief set list of import files and STDIN usage flag - */ - void setLoadFilesInput(bool bReadFromStdin, bool bReadFromS3, - const std::vector& files, const std::string& s3host, - const std::string &s3key, const std::string &s3secret, - const std::string &s3bucket, const std::string &s3region); - - /** @brief set job file name under process. - */ - void setJobFileName(const std::string& jobFileName); - - /** @brief set job ID for this import. - */ - void setJobId(int jobId); - - /** @brief set truncation as error for this import. - * When set, this causes char and varchar strings that are - * longer than the column definition to be treated as errors - * instead of warnings. - */ - void setTruncationAsError(bool bTruncationAsError); - - /** @brief log message to data_mods.log file. - */ - void logToDataMods(const std::string& jobFile, - const std::string& messageText); - - /** @brief Validate consistency of current HWMs for this table's columns. - * If jobTable argument is provided, then it will be used to get additional - * column info, else this table's fColumns vector is used. - * "stage" indicates validation stage ("Starting" or "Ending" HWMs). - */ - int validateColumnHWMs( const JobTable* jobTable, - const std::vector& segFileInfo, - const char* stage ); - - /** @brief Initialize the bulk rollback meta data writer for this table. - */ - int initBulkRollbackMetaData( ); - - /** @brief Save meta data information for bulk rollback. - * This is the Shared-Nothing version of this function. - * @param job Input Job information - * @param segFileInfo vector of starting segment files for each column - * @param dbRootHWMInfoColVec Vector of last local HWMs for each DBRoot - * on this PM. - */ - int saveBulkRollbackMetaData( Job& job, - const std::vector& segFileInfo, - const std::vector& dbRootHWMInfoColVec ); - - /** @brief Mark table as complete - */ - void markTableComplete( ); - - void setJobUUID(const boost::uuids::uuid& jobUUID); - -public: - friend class BulkLoad; - friend class ColumnInfo; - friend class ColumnInfoCompressed; - + private: + //-------------------------------------------------------------------------- + // Private Data Members + //-------------------------------------------------------------------------- + + int fTableId; // Table id + int fBufferSize; // Size of buffer used by BulkLoadBuffer + size_t fFileBufSize; // Size of fFileBuffer passed to setvbuf + // to read import files. Comes from + // writeBufferSize tag in job xml file + char fColDelim; // Used to delimit col values in a row + volatile Status fStatusTI; // Status of table. Made volatile to + // insure BulkLoad methods can access + // (thru getStatusTI()) correctly w/o + // having to go through a mutex lock. + int fReadBufCount; // Number of read buffers + // (size of fBuffers vector) + unsigned fNumberOfColumns; // Number of ColumnInfo objs in this tbl + // (size of fColumns vector) + FILE* fHandle; // Handle to the input load file + int fCurrentReadBuffer; // Id of current buffer being popu- + // lated by the read thread + RID fTotalReadRows; // Total number of rows read + volatile unsigned fTotalErrRows; // Total error rows among all input + // for this table. Is volatile to + // insure parser & reader threads + // see the latest value. + unsigned fMaxErrorRows; // Maximum error rows + int fLastBufferId; // Id of the last buffer + char* fFileBuffer; // File buffer passed to setvbuf() + int fCurrentParseBuffer; // Id of leading current buffer being + // parsed. There can be more than 1 + // buffer being parsed concurrently. + unsigned fNumberOfColsParsed; // Number of columns completed parsing + boost::ptr_vector fColumns; // Columns of the table + boost::ptr_vector fBuffers; // Array of read buffers. Used + // to pass data from the read + // thread to the write thread(s) + + /* fSyncUpdatesTI is the mutex used to synchronize updates to TableInfo + * (excluding fErrorRows and fErrDataRows) + * + * This mutex is also used to coordinate access to fColumnLocks and + * fParseComplete (in BulkLoadBuffer) for the buffers within a table. + * See bulkloadBuffer.h for more information. + * + * As the controlling class, TableInfo is the one that is always + * getting/setting the status of the BulkLoadBuffer objects, so + * fSyncUpdatesTI is also used to set/get the BulkLoadBuffer status. + */ + boost::mutex fSyncUpdatesTI; + + boost::mutex fErrorRptInfoMutex; // Used to synhronize access to + // fRejectDataFile & fRejectErrFile + int fLocker; // Read thread id reading this table + std::vector fLoadFileList; // Load files + std::string fFileName; // Current load file + std::string fTableName; // File name of the table + OID fTableOID; // OID of the table + std::string fjobFileName; // Job file name + int fJobId; // Job ID number + Log* fLog; // Object used for logging + timeval fStartTime; // Time when reading and processing began for this table + const BRM::TxnID fTxnID; // Transaction id for the build load + RBMetaWriter fRBMetaWriter; // Manages the writing of bulk roll- + // back meta data for this table + std::string fProcessName; // Name of application process used + // in db table locks + bool fKeepRbMetaFile; // Keep or delete bulk rollback meta + // data file + bool fbTruncationAsError; // Treat string truncation as error + ImportDataMode fImportDataMode; // Import data in text or binary mode + std::string fTimeZone; // Timezone used by TIMESTAMP data type + + volatile bool fTableLocked; // Do we have db table lock + + bool fReadFromStdin; // Read import file from STDIN + bool fReadFromS3; // Read import file from S3 + std::string fS3Host; // S3 hostname + std::string fS3Key; // S3 key + std::string fS3Secret; // S3 secret key + std::string fS3Bucket; // S3 bucket + std::string fS3Region; + ms3_st* ms3; // S3 object + size_t fS3ReadLength; + size_t fS3ParseLength; + bool fNullStringMode; // Treat "NULL" as a null value + char fEnclosedByChar; // Character to enclose col values + char fEscapeChar; // Escape character used in conjunc- + // tion with fEnclosedByChar + bool fProcessingBegun; // Has processing begun on this tbl + BulkModeType fBulkMode; // Distributed bulk mode (1,2, or 3) + std::string fBRMRptFileName; // Name of distributed mode rpt file + BRMReporter fBRMReporter; // Object used to report BRM updates + uint64_t fTableLockID; // Unique table lock ID + std::vector fOrigDbRootIds; // List of DBRoots at start of job + + std::string fErrorDir; // Opt dir for *.err and *.bad files + std::vector fErrFiles; // List of *.err files for this table + std::vector fBadFiles; // List of *.bad files for this table + std::ofstream fRejectDataFile; // File containing rejected rows + std::ofstream fRejectErrFile; // File containing errmsgs for bad rows + std::string fRejectDataFileName; // Filename for current fRejectDataFile + std::string fRejectErrFileName; // Filename for current fRejectErrFile + unsigned int fRejectDataCnt; // Running row count in current bad file + unsigned int fRejectErrCnt; // Running count in current err msg file + + ExtentStripeAlloc fExtentStrAlloc; // Extent stripe allocator for this tbl + querytele::QueryTeleClient fQtc; // Query Tele client + + oam::OamCache* fOamCachePtr; // OamCache: ptr is copyable + boost::uuids::uuid fJobUUID; // Job UUID + std::vector fDictFlushBlks; // dict blks to be flushed from cache + + //-------------------------------------------------------------------------- + // Private Functions + //-------------------------------------------------------------------------- + + int changeTableLockState(); // Change state of table lock to cleanup + void closeTableFile(); // Close current tbl file; free buffer + void closeOpenDbFiles(); // Close DB files left open at job's end + int confirmDBFileChanges(); // Confirm DB file changes (on HDFS) + void deleteTempDBFileChanges(); // Delete DB temp swap files (on HDFS) + int finishBRM(); // Finish reporting updates for BRM + void freeProcessingBuffers(); // Free up Processing Buffers + bool isBufferAvailable(bool report); // Is tbl buffer available for reading + int openTableFile(); // Open data file and set the buffer + void reportTotals(double elapsedSec); // Report summary totals + void sleepMS(long int ms); // Sleep method + // Compare column HWM with the examplar HWM. + int compareHWMs(const int smallestColumnId, const int widerColumnId, const uint32_t smallerColumnWidth, + const uint32_t widerColumnWidth, const std::vector& segFileInfo, + int& colIdx); + + int synchronizeAutoInc(); // Sychronize AutoInc in BRM with syscat + + // Write the list of errors for this table + void writeErrorList(const std::vector >* errorRows, + const std::vector* errorDatRows, bool bCloseFile); + + // Write out rejected rows, and corresponding error messages + void writeBadRows(const std::vector* errorDatRows, bool bCloseFile); + void writeErrReason(const std::vector >* errorRows, bool bCloseFile); + + // Disable copy constructor and assignment operator + TableInfo(const TableInfo& tableInfo); // + TableInfo& operator=(const TableInfo& info); + + public: + //------------------------------------------------------------------------- + // Public Functions + //------------------------------------------------------------------------- + + /** @brief Default constructor + */ + TableInfo(Log* logger, const BRM::TxnID, const std::string& processName, OID tableOID, + const std::string& tableName, bool bKeepRbMetaFile); + + /** @brief Default destructor + */ + ~TableInfo(); + + /** @brief Acquire the DB table lock for this table + */ + int acquireTableLock(bool disableTimeOut = false); + + /** @brief Get current table lock ID for this table + */ + uint64_t getTableLockID() const; + + /** @brief Release the DB table lock for this table + */ + int releaseTableLock(); + + /** @brief Allocate an extent for the specified OID and DBRoot, using the + * internal "stripe" allocator. + * @param columnOID Allocate next extent for this column + * @param dbRoot Allocate extent on this DBRoot + * @param partition (in/out) If DBRoot is empty, this is an input arg, + * else it is assigned by BRM and returned as output + * @param segment (out) Segment number of extent created by BRM + * @param startLbid (out) Starting LBID for extent created by BRM + * @param allocSize (out) Num blocks allocated to extent by BRM + * @param hwm (out) FBO for extent created by BRM + * @param errMsg (out) Error message + */ + int allocateBRMColumnExtent(OID columnOID, uint16_t dbRoot, uint32_t& partition, uint16_t& segment, + BRM::LBID_t& startLbid, int& allocSize, HWM& hwm, std::string& errMsg); + + /** @brief Delete the bulk rollback metadata file. + */ + void deleteMetaDataRollbackFile(); + + /** @brief Get binary import mode. + */ + ImportDataMode getImportDataMode() const; + + /** @brief Get timezone. + */ + const std::string& getTimeZone() const; + + /** @brief Get number of buffers + */ + int getNumberOfBuffers() const; + + /** @brief Set the buffer size + * @param Buffer size + */ + void setBufferSize(const int bufSize); + + /** @brief Set the file buffer size. + * @param Buffer size + */ + void setFileBufferSize(const int fileBufSize); + + /** @brief Set the delimiter used to delimit column values within a row + */ + void setColDelimiter(const char delim); + + /** @brief Get table status + */ + Status getStatusTI() const; + + /** @brief Get current parse buffer + */ + int getCurrentParseBuffer() const; + + /** @brief Get the number of columns + */ + int getNumberOfColumns() const; + + /** @brief get the file name + */ + std::string getFileName() const; + + /** @brief Get the number of maximum allowed error rows + */ + unsigned getMaxErrorRows() const; + + /** @brief retrieve the tuncation as error setting for this + * import. When set, this causes char and varchar strings + * that are longer than the column definition to be treated + * as errors instead of warnings. + */ + bool getTruncationAsError() const; + + /** @brief set the maximum number of error rows allowed + */ + void setMaxErrorRows(const unsigned int maxErrorRows); + + /** @brief Set mode to treat "NULL" string as NULL value or not. + */ + void setNullStringMode(bool bMode); + + /** @brief Set binary import data mode (text or binary). + */ + void setImportDataMode(ImportDataMode importMode); + + /** @brief Set timezone. + */ + void setTimeZone(const std::string& timeZone); + + /** @brief Enable distributed mode, saving BRM updates in rptFileName + */ + void setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName); + + /** @brief Set character optionally used to enclose input column values. + */ + void setEnclosedByChar(char enChar); + + /** @brief Set escape char to use in conjunction with enclosed by char. + */ + void setEscapeChar(char esChar); + + /** @brief Has processing begun for this table. + */ + bool hasProcessingBegun(); + + /** @brief set the table id + */ + void setTableId(const int& id); + + /** @brief get the file name + */ + std::string getTableName() const; + + /** @brief get the table OID + */ + OID getTableOID(); + + /** @brief Set the directory for *.err and *.bad files. May be + * empty string, in which case we use current dir. + */ + void setErrorDir(const std::string& errorDir); + + /** @brief get the bulk rollback meta data writer object for this table + */ + RBMetaWriter* rbMetaWriter(); + + /** @brief Add column information to the table + */ + void addColumn(ColumnInfo* info); + + /** @brief Initialize the buffer list + * @param noOfBuffers Number of buffers to create for this table + * @param jobFieldRefList List of fields in this import + * @param fixedBinaryRecLen In binary mode, this is the fixed record length + * used to read the buffer; in text mode, this value is not used. + */ + int initializeBuffers(int noOfBuffers, const JobFieldRefList& jobFieldRefList, + unsigned int fixedBinaryRecLen); + + /** @brief Read the table data into the read buffer + */ + int readTableData(); + + /** @brief parse method + */ + int parseColumn(const int& columnId, const int& bufferId, double& processingTime); + + /** @brief update the buffer status for column + */ + int setParseComplete(const int& columnId, const int& bufferId, double processingTime); + + /** @brief update the status to reflect a parsing error + */ + void setParseError(); + + /** @brief Check if buffer ready for parsing. + */ + bool bufferReadyForParse(const int& bufferId, bool report) const; + + /** @brief Check if a column is available for parsing in the buffer + * and return the column id if available + */ + int getColumnForParse(const int& id, const int& bufferId, bool report); + + /** @brief Do we have a db lock with the session manager for this table. + */ + bool isTableLocked(); + + /** @brief Lock the table for reading + */ + bool lockForRead(const int& locker); + + /** @brief Rollback changes made to "this" table by the current import job + */ + int rollbackWork(); + + /** @brief set list of import files and STDIN usage flag + */ + void setLoadFilesInput(bool bReadFromStdin, bool bReadFromS3, const std::vector& files, + const std::string& s3host, const std::string& s3key, const std::string& s3secret, + const std::string& s3bucket, const std::string& s3region); + + /** @brief set job file name under process. + */ + void setJobFileName(const std::string& jobFileName); + + /** @brief set job ID for this import. + */ + void setJobId(int jobId); + + /** @brief set truncation as error for this import. + * When set, this causes char and varchar strings that are + * longer than the column definition to be treated as errors + * instead of warnings. + */ + void setTruncationAsError(bool bTruncationAsError); + + /** @brief log message to data_mods.log file. + */ + void logToDataMods(const std::string& jobFile, const std::string& messageText); + + /** @brief Validate consistency of current HWMs for this table's columns. + * If jobTable argument is provided, then it will be used to get additional + * column info, else this table's fColumns vector is used. + * "stage" indicates validation stage ("Starting" or "Ending" HWMs). + */ + int validateColumnHWMs(const JobTable* jobTable, const std::vector& segFileInfo, + const char* stage); + + /** @brief Initialize the bulk rollback meta data writer for this table. + */ + int initBulkRollbackMetaData(); + + /** @brief Save meta data information for bulk rollback. + * This is the Shared-Nothing version of this function. + * @param job Input Job information + * @param segFileInfo vector of starting segment files for each column + * @param dbRootHWMInfoColVec Vector of last local HWMs for each DBRoot + * on this PM. + */ + int saveBulkRollbackMetaData(Job& job, const std::vector& segFileInfo, + const std::vector& dbRootHWMInfoColVec); + + /** @brief Mark table as complete + */ + void markTableComplete(); + + void setJobUUID(const boost::uuids::uuid& jobUUID); + + public: + friend class BulkLoad; + friend class ColumnInfo; + friend class ColumnInfoCompressed; }; //------------------------------------------------------------------------------ @@ -498,191 +469,188 @@ public: //------------------------------------------------------------------------------ inline int TableInfo::getCurrentParseBuffer() const { - return fCurrentParseBuffer; + return fCurrentParseBuffer; } inline std::string TableInfo::getFileName() const { - return fFileName; + return fFileName; } inline ImportDataMode TableInfo::getImportDataMode() const { - return fImportDataMode; + return fImportDataMode; } inline const std::string& TableInfo::getTimeZone() const { - return fTimeZone; + return fTimeZone; } inline int TableInfo::getNumberOfBuffers() const { - return fReadBufCount; + return fReadBufCount; } inline int TableInfo::getNumberOfColumns() const { - return fNumberOfColumns; + return fNumberOfColumns; } inline Status TableInfo::getStatusTI() const { - return fStatusTI; + return fStatusTI; } inline unsigned TableInfo::getMaxErrorRows() const { - return fMaxErrorRows; + return fMaxErrorRows; } inline uint64_t TableInfo::getTableLockID() const { - return fTableLockID; + return fTableLockID; } inline std::string TableInfo::getTableName() const { - return fTableName; + return fTableName; } -inline OID TableInfo::getTableOID( ) +inline OID TableInfo::getTableOID() { - return fTableOID; + return fTableOID; } inline bool TableInfo::getTruncationAsError() const { - return fbTruncationAsError; + return fbTruncationAsError; } inline bool TableInfo::hasProcessingBegun() { - return fProcessingBegun; + return fProcessingBegun; } inline bool TableInfo::isTableLocked() { - return fTableLocked; + return fTableLocked; } inline void TableInfo::markTableComplete() { - boost::mutex::scoped_lock lock(fSyncUpdatesTI); - fStatusTI = WriteEngine::PARSE_COMPLETE; + boost::mutex::scoped_lock lock(fSyncUpdatesTI); + fStatusTI = WriteEngine::PARSE_COMPLETE; } inline RBMetaWriter* TableInfo::rbMetaWriter() { - return &fRBMetaWriter; + return &fRBMetaWriter; } inline void TableInfo::setBufferSize(const int bufSize) { - fBufferSize = bufSize; + fBufferSize = bufSize; } inline void TableInfo::setColDelimiter(const char delim) { - fColDelim = delim; + fColDelim = delim; } -inline void TableInfo::setBulkLoadMode( - BulkModeType bulkMode, - const std::string& rptFileName ) +inline void TableInfo::setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName) { - fBulkMode = bulkMode, - fBRMRptFileName = rptFileName; + fBulkMode = bulkMode, fBRMRptFileName = rptFileName; } -inline void TableInfo::setEnclosedByChar( char enChar ) +inline void TableInfo::setEnclosedByChar(char enChar) { - fEnclosedByChar = enChar; + fEnclosedByChar = enChar; } -inline void TableInfo::setEscapeChar ( char esChar ) +inline void TableInfo::setEscapeChar(char esChar) { - fEscapeChar = esChar; + fEscapeChar = esChar; } inline void TableInfo::setFileBufferSize(const int fileBufSize) { - fFileBufSize = fileBufSize; + fFileBufSize = fileBufSize; } -inline void TableInfo::setImportDataMode( ImportDataMode importMode ) +inline void TableInfo::setImportDataMode(ImportDataMode importMode) { - fImportDataMode = importMode; + fImportDataMode = importMode; } -inline void TableInfo::setTimeZone( const std::string& timeZone ) +inline void TableInfo::setTimeZone(const std::string& timeZone) { - fTimeZone = timeZone; + fTimeZone = timeZone; } inline void TableInfo::setJobFileName(const std::string& jobFileName) { - fjobFileName = jobFileName; + fjobFileName = jobFileName; } inline void TableInfo::setJobId(int jobId) { - fJobId = jobId; + fJobId = jobId; } -inline void TableInfo::setLoadFilesInput(bool bReadFromStdin, bool bReadFromS3, - const std::vector& files, const std::string& s3host, - const std::string &s3key, const std::string &s3secret, - const std::string &s3bucket, const std::string &s3region) +inline void TableInfo::setLoadFilesInput(bool bReadFromStdin, bool bReadFromS3, + const std::vector& files, const std::string& s3host, + const std::string& s3key, const std::string& s3secret, + const std::string& s3bucket, const std::string& s3region) { - fReadFromStdin = bReadFromStdin; - fReadFromS3 = bReadFromS3; - fLoadFileList = files; - fS3Host = s3host; - fS3Key = s3key; - fS3Secret = s3secret; - fS3Bucket = s3bucket; - fS3Region = s3region; + fReadFromStdin = bReadFromStdin; + fReadFromS3 = bReadFromS3; + fLoadFileList = files; + fS3Host = s3host; + fS3Key = s3key; + fS3Secret = s3secret; + fS3Bucket = s3bucket; + fS3Region = s3region; } inline void TableInfo::setMaxErrorRows(const unsigned int maxErrorRows) { - fMaxErrorRows = maxErrorRows; + fMaxErrorRows = maxErrorRows; } -inline void TableInfo::setNullStringMode( bool bMode ) +inline void TableInfo::setNullStringMode(bool bMode) { - fNullStringMode = bMode; + fNullStringMode = bMode; } inline void TableInfo::setTableId(const int& id) { - fTableId = id; + fTableId = id; } inline void TableInfo::setTruncationAsError(bool bTruncationAsError) { - fbTruncationAsError = bTruncationAsError; + fbTruncationAsError = bTruncationAsError; } inline void TableInfo::setJobUUID(const boost::uuids::uuid& jobUUID) { - fJobUUID = jobUUID; + fJobUUID = jobUUID; } -inline void TableInfo::setErrorDir( const std::string& errorDir ) +inline void TableInfo::setErrorDir(const std::string& errorDir) { - fErrorDir = errorDir; + fErrorDir = errorDir; #ifdef _MSC_VER - if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/' && *(--(fErrorDir.end())) != '\\') - fErrorDir.push_back('\\'); + if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/' && *(--(fErrorDir.end())) != '\\') + fErrorDir.push_back('\\'); } #else - if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/') - fErrorDir.push_back('/'); + if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/') + fErrorDir.push_back('/'); } #endif -} +} // namespace WriteEngine #endif diff --git a/writeengine/bulk/we_tempxmlgendata.cpp b/writeengine/bulk/we_tempxmlgendata.cpp index 31dcf28cc..648c58a51 100644 --- a/writeengine/bulk/we_tempxmlgendata.cpp +++ b/writeengine/bulk/we_tempxmlgendata.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_tempxmlgendata.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_tempxmlgendata.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #include "we_tempxmlgendata.h" @@ -29,26 +29,24 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // TempXMLGenData constructor //------------------------------------------------------------------------------ -TempXMLGenData::TempXMLGenData(const std::string& jobId, - const std::string& schema, - const std::string& table) +TempXMLGenData::TempXMLGenData(const std::string& jobId, const std::string& schema, const std::string& table) { - fParms[ JOBID] = jobId; // add or override default value - fSchema = schema; - execplan::CalpontSystemCatalog::TableName tbl(schema, table); - fTables.push_back( tbl ); + fParms[JOBID] = jobId; // add or override default value + fSchema = schema; + execplan::CalpontSystemCatalog::TableName tbl(schema, table); + fTables.push_back(tbl); } //------------------------------------------------------------------------------ // TempXMLGenData destructor //------------------------------------------------------------------------------ /* virtual */ -TempXMLGenData::~TempXMLGenData( ) -{ } +TempXMLGenData::~TempXMLGenData() +{ +} //------------------------------------------------------------------------------ // TempXMLGenData print function. @@ -56,10 +54,10 @@ TempXMLGenData::~TempXMLGenData( ) /* virtual */ void TempXMLGenData::print(std::ostream& os) const { - os << "Generating runtime job xml file for: schema-" << fSchema; + os << "Generating runtime job xml file for: schema-" << fSchema; - if (fTables.size() > 0) - os << ": table-" << fTables[0]; + if (fTables.size() > 0) + os << ": table-" << fTables[0]; } -} // end of WriteEngine namespace +} // namespace WriteEngine diff --git a/writeengine/bulk/we_tempxmlgendata.h b/writeengine/bulk/we_tempxmlgendata.h index 59c812a2b..824e59e71 100644 --- a/writeengine/bulk/we_tempxmlgendata.h +++ b/writeengine/bulk/we_tempxmlgendata.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_tempxmlgendata.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_tempxmlgendata.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_TEMPXMLGENDATA_H_ @@ -31,26 +31,23 @@ namespace WriteEngine { - /** @brief Class used by cpimport.bin to store data that is saved into a * temporary runtime Job XML file. */ class TempXMLGenData : public XMLGenData { -public: - TempXMLGenData(const std::string& jobId, - const std::string& schema, - const std::string& table ); + public: + TempXMLGenData(const std::string& jobId, const std::string& schema, const std::string& table); - virtual ~TempXMLGenData( ); + virtual ~TempXMLGenData(); - virtual void print(std::ostream& os) const; + virtual void print(std::ostream& os) const; -private: - TempXMLGenData(const TempXMLGenData&); //disable default copy ctor - TempXMLGenData& operator=(const TempXMLGenData&);//disable def assignment + private: + TempXMLGenData(const TempXMLGenData&); // disable default copy ctor + TempXMLGenData& operator=(const TempXMLGenData&); // disable def assignment }; -} // end of WriteEngine namespace +} // namespace WriteEngine -#endif // _WE_TEMPXMLGENDATA_H_ +#endif // _WE_TEMPXMLGENDATA_H_ diff --git a/writeengine/bulk/we_workers.cpp b/writeengine/bulk/we_workers.cpp index e15d380fa..7a2cb186b 100644 --- a/writeengine/bulk/we_workers.cpp +++ b/writeengine/bulk/we_workers.cpp @@ -32,27 +32,25 @@ using namespace dataconvert; namespace WriteEngine { - //------------------------------------------------------------------------------ // Puts the current thread to sleep for the specified number of milliseconds. //------------------------------------------------------------------------------ void BulkLoad::sleepMS(long ms) { - struct timespec rm_ts; + struct timespec rm_ts; - rm_ts.tv_sec = ms / 1000; - rm_ts.tv_nsec = ms % 1000 * 1000000; + rm_ts.tv_sec = ms / 1000; + rm_ts.tv_nsec = ms % 1000 * 1000000; #ifdef _MSC_VER - Sleep(ms); + Sleep(ms); #else - struct timespec abs_ts; + struct timespec abs_ts; - do - { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif } @@ -61,114 +59,107 @@ void BulkLoad::sleepMS(long ms) // This is the main entry point method for each Read thread. // id is the one-up number (starting at 0) associated with each Read thread. //------------------------------------------------------------------------------ -void BulkLoad::read(int id) +void BulkLoad::read(int id) { #ifdef PROFILE - Stats::registerReadProfThread( ); + Stats::registerReadProfThread(); #endif - // First get a table to work on. - // Acquire the read mutex - // Iterate over the table list - // if the table's status is new, set the locker = id - // and then exit. - int tableId = -1; + // First get a table to work on. + // Acquire the read mutex + // Iterate over the table list + // if the table's status is new, set the locker = id + // and then exit. + int tableId = -1; - try + try + { + // + // LOOP to select and read the next table + // + while (true) { - // - // LOOP to select and read the next table - // - while (true) - { - tableId = -1; + tableId = -1; #ifdef PROFILE - Stats::startReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); + Stats::startReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); #endif - if ((tableId = lockTableForRead(id)) == -1) - { - fLog.logMsg( "BulkLoad::ReadOperation No more tables " - "available for processing. Read thread " - + Convertor::int2Str(id) + " exiting...", - MSGLVL_INFO2); + if ((tableId = lockTableForRead(id)) == -1) + { + fLog.logMsg( + "BulkLoad::ReadOperation No more tables " + "available for processing. Read thread " + + Convertor::int2Str(id) + " exiting...", + MSGLVL_INFO2); #ifdef PROFILE - Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); + Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); #endif - return; - } + return; + } #ifdef PROFILE - Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); + Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); #endif - int rc = fTableInfo[tableId].readTableData( ); + int rc = fTableInfo[tableId].readTableData(); - if (rc != NO_ERROR) - { - // Error occurred while reading the data, break out of loop. - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; - oss << "Bulkload Read (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << - ". Terminating this job."; - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - fLog.logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - break; - } - } - } - catch (SecondaryShutdownException& ex) - { - // We are bailing out because another thread set bad job status + if (rc != NO_ERROR) + { + // Error occurred while reading the data, break out of loop. + BulkStatus::setJobStatus(EXIT_FAILURE); ostringstream oss; - - if (tableId != -1) - oss << "Bulkload Read (thread " << id << - ") Stopped reading Table " << - fTableInfo[tableId].getTableName() << ". " << ex.what(); - else - oss << "Bulkload Read (thread " << id << - ") Stopped reading Tables. " << ex.what(); - - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); + oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + << ". Terminating this job."; + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fLog.logMsg(oss.str(), rc, MSGLVL_CRITICAL); + break; + } } - catch (exception& ex) - { - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; + } + catch (SecondaryShutdownException& ex) + { + // We are bailing out because another thread set bad job status + ostringstream oss; - if (tableId != -1) - oss << "Bulkload Read (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << ". " << ex.what() << - ". Terminating this job."; - else - oss << "Bulkload Read (thread " << id << - ") Failed for Table. " << ex.what() << - ". Terminating this job."; + if (tableId != -1) + oss << "Bulkload Read (thread " << id << ") Stopped reading Table " + << fTableInfo[tableId].getTableName() << ". " << ex.what(); + else + oss << "Bulkload Read (thread " << id << ") Stopped reading Tables. " << ex.what(); - if (tableId != -1) fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fLog.logMsg(oss.str(), MSGLVL_INFO1); + } + catch (exception& ex) + { + BulkStatus::setJobStatus(EXIT_FAILURE); + ostringstream oss; - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL ); - } - catch (...) - { - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; + if (tableId != -1) + oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + << ". " << ex.what() << ". Terminating this job."; + else + oss << "Bulkload Read (thread " << id << ") Failed for Table. " << ex.what() + << ". Terminating this job."; - if (tableId != -1) - oss << "Bulkload Read (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << - ". Terminating this job."; - else - oss << "Bulkload Read (thread " << id << - ") Failed for Table. Terminating this job."; + if (tableId != -1) + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - if (tableId != -1) fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL); + } + catch (...) + { + BulkStatus::setJobStatus(EXIT_FAILURE); + ostringstream oss; - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL ); - } + if (tableId != -1) + oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + << ". Terminating this job."; + else + oss << "Bulkload Read (thread " << id << ") Failed for Table. Terminating this job."; + + if (tableId != -1) + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL); + } } //------------------------------------------------------------------------------ @@ -179,15 +170,15 @@ void BulkLoad::read(int id) //------------------------------------------------------------------------------ int BulkLoad::lockTableForRead(int id) { - boost::mutex::scoped_lock lock(fReadMutex); + boost::mutex::scoped_lock lock(fReadMutex); - for (unsigned i = 0; i < fTableInfo.size(); ++i) - { - if (fTableInfo[i].lockForRead(id)) - return i; - } + for (unsigned i = 0; i < fTableInfo.size(); ++i) + { + if (fTableInfo[i].lockForRead(id)) + return i; + } - return -1; + return -1; } //------------------------------------------------------------------------------ @@ -197,238 +188,227 @@ int BulkLoad::lockTableForRead(int id) void BulkLoad::parse(int id) { #ifdef PROFILE - Stats::registerParseProfThread( ); + Stats::registerParseProfThread(); #endif - // Get a column from a buffer to parse. - // The currentParseBuffer will be the buffer to be worked on - int tableId = -1; - int columnId = -1; - int myParseBuffer = -1; + // Get a column from a buffer to parse. + // The currentParseBuffer will be the buffer to be worked on + int tableId = -1; + int columnId = -1; + int myParseBuffer = -1; - try + try + { + // + // LOOP to parse BulkLoadBuffers as they're loaded by Read thread(s) + // + while (true) { - // - // LOOP to parse BulkLoadBuffers as they're loaded by Read thread(s) - // - while (true) - { #ifdef PROFILE - Stats::startParseEvent(WE_STATS_WAIT_TO_SELECT_COL); + Stats::startParseEvent(WE_STATS_WAIT_TO_SELECT_COL); #endif // @bug 3271: Conditionally compile the thread deadlock debug logging #ifdef DEADLOCK_DEBUG - // @bug2099+ Temporary hack. - struct timeval tvStart; - gettimeofday(&tvStart, 0); - bool report = false; - bool reported = false; - // @bug2099- + // @bug2099+ Temporary hack. + struct timeval tvStart; + gettimeofday(&tvStart, 0); + bool report = false; + bool reported = false; + // @bug2099- #else - const bool report = false; + const bool report = false; #endif - // - // LOOP to wait and select table/column/buffers - // (BulkLoadBuffers) as they are loaded by the Read buffer. - // - while (true) - { - tableId = -1; - columnId = -1; - myParseBuffer = -1; + // + // LOOP to wait and select table/column/buffers + // (BulkLoadBuffers) as they are loaded by the Read buffer. + // + while (true) + { + tableId = -1; + columnId = -1; + myParseBuffer = -1; - //See if JobStatus has been set to terminate by other thread - if (BulkStatus::getJobStatus() == EXIT_FAILURE) - { - throw SecondaryShutdownException( "BulkLoad::" - "parse() responding to job termination"); - } + // See if JobStatus has been set to terminate by other thread + if (BulkStatus::getJobStatus() == EXIT_FAILURE) + { + throw SecondaryShutdownException( + "BulkLoad::" + "parse() responding to job termination"); + } - if (allTablesDone(WriteEngine::PARSE_COMPLETE)) - { + if (allTablesDone(WriteEngine::PARSE_COMPLETE)) + { #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_TO_SELECT_COL); + Stats::stopParseEvent(WE_STATS_WAIT_TO_SELECT_COL); #endif - // no column from any of the tables available for - // parsing, hence exit. - return; - } + // no column from any of the tables available for + // parsing, hence exit. + return; + } - if (lockColumnForParse(id, tableId, columnId, - myParseBuffer, report)) - break; + if (lockColumnForParse(id, tableId, columnId, myParseBuffer, report)) + break; - // Sleep and check the condition again. - sleepMS(1); + // Sleep and check the condition again. + sleepMS(1); #ifdef DEADLOCK_DEBUG - // @bug2099+ - if (report) report = false; // report one time. + // @bug2099+ + if (report) + report = false; // report one time. - if (!reported) - { - struct timeval tvNow; - gettimeofday(&tvNow, 0); + if (!reported) + { + struct timeval tvNow; + gettimeofday(&tvNow, 0); - if ((tvNow.tv_sec - tvStart.tv_sec) >= 100) - { - time_t t = time(0); - char timeString[50]; - ctime_r(&t, timeString); - timeString[ strlen(timeString) - 1 ] = '\0'; - ostringstream oss; - oss << endl << endl << timeString << - ": BulkLoad::parse(" << id << "); " << + if ((tvNow.tv_sec - tvStart.tv_sec) >= 100) + { + time_t t = time(0); + char timeString[50]; + ctime_r(&t, timeString); + timeString[strlen(timeString) - 1] = '\0'; + ostringstream oss; + oss << endl + << endl + << timeString << ": BulkLoad::parse(" << id << "); " + << #ifdef _MSC_VER - " Worker Thread " << GetCurrentThreadId() << + " Worker Thread " << GetCurrentThreadId() << #else - " Worker Thread " << pthread_self() << + " Worker Thread " << pthread_self() << #endif - ":" << endl << - "---------------------------------------" - "-------------------" << endl; - cout << oss.str(); - cout.flush(); - report = true; - reported = true; - } - } - - // @bug2099- -#endif - } - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_TO_SELECT_COL); -#endif - // Have obtained the table and column for parsing. - // Start parsing the column data. - double processingTime; - int rc = fTableInfo[tableId].parseColumn(columnId, myParseBuffer, - processingTime); - - if (rc != NO_ERROR) - { - // Error occurred while parsing the data, break out of loop. - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; - oss << "Bulkload Parse (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << - " during parsing. Terminating this job."; - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - fLog.logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - - setParseErrorOnTable( tableId, true ); - return; - } - - // Parsing is complete. Acquire the mutex and increment - // the parsingComplete value for the buffer - if (fTableInfo[tableId].getStatusTI() != WriteEngine::ERR) - { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE); -#endif - boost::mutex::scoped_lock lock(fParseMutex); -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE); - Stats::startParseEvent(WE_STATS_COMPLETING_PARSE); -#endif - rc = fTableInfo[tableId].setParseComplete(columnId, - myParseBuffer, - processingTime); - - if (rc != NO_ERROR) - { - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; - oss << "Bulkload Parse (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << - " during parse completion. Terminating this job."; - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - fLog.logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - - setParseErrorOnTable( tableId, false ); - return; - } - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPLETING_PARSE); -#endif - } + ":" << endl + << "---------------------------------------" + "-------------------" + << endl; + cout << oss.str(); + cout.flush(); + report = true; + reported = true; + } } - } - catch (SecondaryShutdownException& ex) - { - // We are bailing out because another thread set bad job status + + // @bug2099- +#endif + } + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_WAIT_TO_SELECT_COL); +#endif + // Have obtained the table and column for parsing. + // Start parsing the column data. + double processingTime; + int rc = fTableInfo[tableId].parseColumn(columnId, myParseBuffer, processingTime); + + if (rc != NO_ERROR) + { + // Error occurred while parsing the data, break out of loop. + BulkStatus::setJobStatus(EXIT_FAILURE); ostringstream oss; + oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + << " during parsing. Terminating this job."; + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fLog.logMsg(oss.str(), rc, MSGLVL_CRITICAL); - if (tableId != -1) - { - oss << "Bulkload Parse (thread " << id << - ") Stopped parsing Table " << - fTableInfo[tableId].getTableName() << ". " << ex.what(); + setParseErrorOnTable(tableId, true); + return; + } - setParseErrorOnTable( tableId, true ); - } - else + // Parsing is complete. Acquire the mutex and increment + // the parsingComplete value for the buffer + if (fTableInfo[tableId].getStatusTI() != WriteEngine::ERR) + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE); +#endif + boost::mutex::scoped_lock lock(fParseMutex); +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE); + Stats::startParseEvent(WE_STATS_COMPLETING_PARSE); +#endif + rc = fTableInfo[tableId].setParseComplete(columnId, myParseBuffer, processingTime); + + if (rc != NO_ERROR) { - oss << "Bulkload Parse (thread " << id << - ") Stopped parsing Tables. " << ex.what(); + BulkStatus::setJobStatus(EXIT_FAILURE); + ostringstream oss; + oss << "Bulkload Parse (thread " << id << ") Failed for Table " + << fTableInfo[tableId].getTableName() << " during parse completion. Terminating this job."; + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fLog.logMsg(oss.str(), rc, MSGLVL_CRITICAL); + + setParseErrorOnTable(tableId, false); + return; } - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_COMPLETING_PARSE); +#endif + } } - catch (exception& ex) + } + catch (SecondaryShutdownException& ex) + { + // We are bailing out because another thread set bad job status + ostringstream oss; + + if (tableId != -1) { - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; + oss << "Bulkload Parse (thread " << id << ") Stopped parsing Table " + << fTableInfo[tableId].getTableName() << ". " << ex.what(); - if (tableId != -1) - { - oss << "Bulkload Parse (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << ". " << ex.what() << - ". Terminating this job."; - - setParseErrorOnTable( tableId, true ); - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - } - else - { - oss << "Bulkload Parse (thread " << id << - ") Failed for Table. " << ex.what() << - ". Terminating this job."; - } - - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL ); + setParseErrorOnTable(tableId, true); } - catch (...) + else { - BulkStatus::setJobStatus( EXIT_FAILURE ); - ostringstream oss; - - if (tableId != -1) - { - oss << "Bulkload Parse (thread " << id << - ") Failed for Table " << - fTableInfo[tableId].getTableName() << - ". Terminating this job."; - - setParseErrorOnTable( tableId, true ); - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); - } - else - { - oss << "Bulkload Parse (thread " << id << - ") Failed for Table. Terminating this job."; - } - - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL ); + oss << "Bulkload Parse (thread " << id << ") Stopped parsing Tables. " << ex.what(); } + + fLog.logMsg(oss.str(), MSGLVL_INFO1); + } + catch (exception& ex) + { + BulkStatus::setJobStatus(EXIT_FAILURE); + ostringstream oss; + + if (tableId != -1) + { + oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + << ". " << ex.what() << ". Terminating this job."; + + setParseErrorOnTable(tableId, true); + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + } + else + { + oss << "Bulkload Parse (thread " << id << ") Failed for Table. " << ex.what() + << ". Terminating this job."; + } + + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL); + } + catch (...) + { + BulkStatus::setJobStatus(EXIT_FAILURE); + ostringstream oss; + + if (tableId != -1) + { + oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + << ". Terminating this job."; + + setParseErrorOnTable(tableId, true); + fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + } + else + { + oss << "Bulkload Parse (thread " << id << ") Failed for Table. Terminating this job."; + } + + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL); + } } //------------------------------------------------------------------------------ @@ -438,73 +418,63 @@ void BulkLoad::parse(int id) // Return value is -1 if no table/column/buffer is available for parsing. //------------------------------------------------------------------------------ // @bug2099 - Temp hack to diagnose deadlock. Added report parm and couts below. -bool BulkLoad::lockColumnForParse( - int thrdId, - int& tableId, - int& columnId, - int& myParseBuffer, - bool report) +bool BulkLoad::lockColumnForParse(int thrdId, int& tableId, int& columnId, int& myParseBuffer, bool report) { - // Acquire mutex - // Iterate on the table list - // Check if the currentParseBuffer is available for parsing - // If yes, put the locker and fill the tableId and columnId - // else, go to the next table for checking if a column is available - boost::mutex::scoped_lock lock(fParseMutex); + // Acquire mutex + // Iterate on the table list + // Check if the currentParseBuffer is available for parsing + // If yes, put the locker and fill the tableId and columnId + // else, go to the next table for checking if a column is available + boost::mutex::scoped_lock lock(fParseMutex); - for (unsigned i = 0; i < fTableInfo.size(); ++i) + for (unsigned i = 0; i < fTableInfo.size(); ++i) + { + if (fTableInfo[i].getStatusTI() == WriteEngine::PARSE_COMPLETE) + continue; + + int currentParseBuffer = fTableInfo[i].getCurrentParseBuffer(); + myParseBuffer = currentParseBuffer; + + do { - if (fTableInfo[i].getStatusTI() == WriteEngine::PARSE_COMPLETE) - continue; - - int currentParseBuffer = fTableInfo[i].getCurrentParseBuffer(); - myParseBuffer = currentParseBuffer; - - do - { - // @bug2099+ - if (report) - { - ostringstream oss; - std::string bufStatusStr; - Status stat = fTableInfo[i].getStatusTI(); - ColumnInfo::convertStatusToString( stat, - bufStatusStr ); + // @bug2099+ + if (report) + { + ostringstream oss; + std::string bufStatusStr; + Status stat = fTableInfo[i].getStatusTI(); + ColumnInfo::convertStatusToString(stat, bufStatusStr); #ifdef _MSC_VER - oss << " - " << GetCurrentThreadId() << + oss << " - " << GetCurrentThreadId() << #else - oss << " - " << pthread_self() << + oss << " - " << pthread_self() << #endif - ":fTableInfo[" << i << "]" << bufStatusStr << " (" << - stat << ")"; + ":fTableInfo[" << i << "]" << bufStatusStr << " (" << stat << ")"; - if ( stat != WriteEngine::PARSE_COMPLETE ) - { - oss << "; fCurrentParseBuffer is " << myParseBuffer; - } - - oss << endl; - cout << oss.str(); - cout.flush(); - } - - // @bug2099- - - // get a buffer and column to parse if available. - if ((columnId = fTableInfo[i].getColumnForParse( - thrdId, myParseBuffer, report )) != -1) - { - tableId = i; - return true; - } - - myParseBuffer = (myParseBuffer + 1) % - fTableInfo[i].getNumberOfBuffers(); + if (stat != WriteEngine::PARSE_COMPLETE) + { + oss << "; fCurrentParseBuffer is " << myParseBuffer; } - while (myParseBuffer != currentParseBuffer); - } - return false; + oss << endl; + cout << oss.str(); + cout.flush(); + } + + // @bug2099- + + // get a buffer and column to parse if available. + if ((columnId = fTableInfo[i].getColumnForParse(thrdId, myParseBuffer, report)) != -1) + { + tableId = i; + return true; + } + + myParseBuffer = (myParseBuffer + 1) % fTableInfo[i].getNumberOfBuffers(); + } while (myParseBuffer != currentParseBuffer); + } + + return false; } //------------------------------------------------------------------------------ @@ -519,16 +489,16 @@ bool BulkLoad::lockColumnForParse( //------------------------------------------------------------------------------ bool BulkLoad::allTablesDone(Status status) { - for (unsigned i = 0; i < fTableInfo.size(); ++i) - { - if (fTableInfo[i].getStatusTI() == WriteEngine::ERR) - return true; + for (unsigned i = 0; i < fTableInfo.size(); ++i) + { + if (fTableInfo[i].getStatusTI() == WriteEngine::ERR) + return true; - if (fTableInfo[i].getStatusTI() != status) - return false; - } + if (fTableInfo[i].getStatusTI() != status) + return false; + } - return true; + return true; } //------------------------------------------------------------------------------ @@ -536,17 +506,17 @@ bool BulkLoad::allTablesDone(Status status) // Optionally lock fParseMutex (if requested). // May evaluate later whether we need to employ the fParseMutex for this call. //------------------------------------------------------------------------------ -void BulkLoad::setParseErrorOnTable( int tableId, bool lockParseMutex ) +void BulkLoad::setParseErrorOnTable(int tableId, bool lockParseMutex) { - if (lockParseMutex) - { - boost::mutex::scoped_lock lock(fParseMutex); - fTableInfo[tableId].setParseError( ); - } - else - { - fTableInfo[tableId].setParseError( ); - } + if (lockParseMutex) + { + boost::mutex::scoped_lock lock(fParseMutex); + fTableInfo[tableId].setParseError(); + } + else + { + fTableInfo[tableId].setParseError(); + } } -} +} // namespace WriteEngine diff --git a/writeengine/client/we_clients.cpp b/writeengine/client/we_clients.cpp index ea555b3c2..c696dbb6d 100644 --- a/writeengine/client/we_clients.cpp +++ b/writeengine/client/we_clients.cpp @@ -62,500 +62,487 @@ using namespace WriteEngine; namespace { - -void writeToLog(const char* file, int line, const string& msg, LOG_TYPE logto = LOG_TYPE_INFO) +void writeToLog(const char* file, int line, const string& msg, LOG_TYPE logto = LOG_TYPE_INFO) { - LoggingID lid(05); - MessageLog ml(lid); - Message::Args args; - Message m(0); - args.add(file); - args.add("@"); - args.add(line); - args.add(msg); - m.format(args); + LoggingID lid(05); + MessageLog ml(lid); + Message::Args args; + Message m(0); + args.add(file); + args.add("@"); + args.add(line); + args.add(msg); + m.format(args); - switch (logto) - { - case LOG_TYPE_DEBUG: - ml.logDebugMessage(m); - break; + switch (logto) + { + case LOG_TYPE_DEBUG: ml.logDebugMessage(m); break; - case LOG_TYPE_INFO: - ml.logInfoMessage(m); - break; + case LOG_TYPE_INFO: ml.logInfoMessage(m); break; - case LOG_TYPE_WARNING: - ml.logWarningMessage(m); - break; + case LOG_TYPE_WARNING: ml.logWarningMessage(m); break; - case LOG_TYPE_ERROR: - ml.logErrorMessage(m); - break; + case LOG_TYPE_ERROR: ml.logErrorMessage(m); break; - case LOG_TYPE_CRITICAL: - ml.logCriticalMessage(m); - break; - } + case LOG_TYPE_CRITICAL: ml.logCriticalMessage(m); break; + } } -string getModuleNameByIPAddr(oam::ModuleTypeConfig moduletypeconfig, - string ipAddress) +string getModuleNameByIPAddr(oam::ModuleTypeConfig moduletypeconfig, string ipAddress) { - string modulename = "pm1"; - DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); + string modulename = "pm1"; + DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); - for ( ; pt != moduletypeconfig.ModuleNetworkList.end() ; pt++) + for (; pt != moduletypeconfig.ModuleNetworkList.end(); pt++) + { + modulename = (*pt).DeviceName; + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for (; pt1 != (*pt).hostConfigList.end(); pt1++) { - modulename = (*pt).DeviceName; - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) - { - if (ipAddress == (*pt1).IPAddr) - return modulename; - } + if (ipAddress == (*pt1).IPAddr) + return modulename; } + } - return modulename; + return modulename; } struct WEClientRunner { - WEClientRunner(WriteEngine::WEClients* jl, - boost::shared_ptr cl, uint32_t connectionIndex) : jbl(jl), client(cl), - connIndex(connectionIndex) {} - WriteEngine::WEClients* jbl; - boost::shared_ptr client; - uint32_t connIndex; - void operator()() + WEClientRunner(WriteEngine::WEClients* jl, boost::shared_ptr cl, + uint32_t connectionIndex) + : jbl(jl), client(cl), connIndex(connectionIndex) + { + } + WriteEngine::WEClients* jbl; + boost::shared_ptr client; + uint32_t connIndex; + void operator()() + { + // cout << "Listening on client at 0x" << hex << (ptrdiff_t)client << dec << endl; + try { - //cout << "Listening on client at 0x" << hex << (ptrdiff_t)client << dec << endl; - try - { - jbl->Listen(client, connIndex); - //cout << "Listening connIndex " << connIndex << endl; - } - catch (std::exception& ex) - { - string what(ex.what()); - cerr << "exception caught in WEClient: " << what << endl; - - if (what.find("St9bad_alloc") != string::npos) - { - writeToLog(__FILE__, __LINE__, what, LOG_TYPE_CRITICAL); -// abort(); - } - else writeToLog(__FILE__, __LINE__, what); - } - catch (...) - { - string msg("exception caught in WEClientRunner."); - writeToLog(__FILE__, __LINE__, msg); - cerr << msg << endl; - } + jbl->Listen(client, connIndex); + // cout << "Listening connIndex " << connIndex << endl; } + catch (std::exception& ex) + { + string what(ex.what()); + cerr << "exception caught in WEClient: " << what << endl; + + if (what.find("St9bad_alloc") != string::npos) + { + writeToLog(__FILE__, __LINE__, what, LOG_TYPE_CRITICAL); + // abort(); + } + else + writeToLog(__FILE__, __LINE__, what); + } + catch (...) + { + string msg("exception caught in WEClientRunner."); + writeToLog(__FILE__, __LINE__, msg); + cerr << msg << endl; + } + } }; template struct QueueShutdown : public unary_function { - void operator()(T& x) - { - x.shutdown(); - } + void operator()(T& x) + { + x.shutdown(); + } }; -} +} // namespace namespace WriteEngine { -WEClients::WEClients(int PrgmID) : - fPrgmID(PrgmID), - pmCount(0) +WEClients::WEClients(int PrgmID) : fPrgmID(PrgmID), pmCount(0) { - closingConnection = 0; - Setup(); + closingConnection = 0; + Setup(); } WEClients::~WEClients() { - - Close(); + Close(); } void WEClients::Setup() { - makeBusy(true); - joblist::ResourceManager* rm = joblist::ResourceManager::instance(); - oam::Oam oam; - string ipAddress; - ModuleTypeConfig moduletypeconfig; + makeBusy(true); + joblist::ResourceManager* rm = joblist::ResourceManager::instance(); + oam::Oam oam; + string ipAddress; + ModuleTypeConfig moduletypeconfig; + try + { + oam.getSystemConfig("pm", moduletypeconfig); + } + catch (...) + { + writeToLog(__FILE__, __LINE__, "oam.getSystemConfig error, unknown exception", LOG_TYPE_ERROR); + throw runtime_error("Setup failed"); + } + + uint32_t pmCountConfig = moduletypeconfig.ModuleCount; + pmCount = 0; + int moduleID = 1; + + char buff[32]; + ByteStream bs, bsRead; + + if (fPrgmID == DDLPROC) + { + bs << (ByteStream::byte)WE_SVR_DDL_KEEPALIVE; + bs << (ByteStream::octbyte)moduleID; + } + else if (fPrgmID == DMLPROC) + { + bs << (ByteStream::byte)WE_SVR_DML_KEEPALIVE; + bs << (ByteStream::octbyte)moduleID; + } + else if (fPrgmID == SPLITTER) + { + bs << (ByteStream::byte)WE_CLT_SRV_KEEPALIVE; + } + else if (fPrgmID == BATCHINSERTPROC) + { + bs << (ByteStream::byte)WE_SVR_BATCH_KEEPALIVE; + bs << (ByteStream::octbyte)moduleID; + } + + for (unsigned i = 0; i < pmCountConfig; i++) + { + // Find the module id + moduleID = atoi((moduletypeconfig.ModuleNetworkList[i]) + .DeviceName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE) + .c_str()); + // cout << "setting connection to moduleid " << moduleID << endl; + snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", moduleID); + string fServer(buff); + + boost::shared_ptr cl(new MessageQueueClient(fServer, rm->getConfig())); + boost::shared_ptr nl(new boost::mutex()); + + // Bug 5224. Take out the retrys. If connection fails, we assume the server is down. try { - oam.getSystemConfig("pm", moduletypeconfig); + if (cl->connect()) + { + try + { + cl->write(bs); + } + catch (std::exception& ex1) + { + ostringstream oss; + oss << "Write to WES during connect failed due to " << ex1.what(); + throw runtime_error(oss.str()); + } + + try + { + bsRead = cl->read(); + + if (bsRead.length() == 0) + throw runtime_error("Got byte 0 during reading "); + } + catch (std::exception& ex2) + { + ostringstream oss; + oss << "Read from WES during connect failed due to " << ex2.what() << " and this = " << this; + throw runtime_error(oss.str()); + } + + fPmConnections[moduleID] = cl; + // cout << "connection is open. this = " << this << endl; + // cout << "set up connection to mudule " << moduleID << endl; + // assign the module name + // ipAddress = sin_addr2String(cl->serv_addr().sin_addr); + ipAddress = cl->addr2String(); + cl->moduleName(getModuleNameByIPAddr(moduletypeconfig, ipAddress)); + StartClientListener(cl, i); + pmCount++; + // ostringstream oss; + // oss << "WECLIENT: connected to " << fServer + " and this = " << this << " and pmcount is now " << + // pmCount; writeToLog(__FILE__, __LINE__, oss.str() , LOG_TYPE_DEBUG); + } + else + { + throw runtime_error("Connection refused"); + } + } + catch (std::exception& ex) + { + writeToLog(__FILE__, __LINE__, "Could not connect to " + fServer + ": " + ex.what(), LOG_TYPE_ERROR); + cerr << "Could not connect to " << fServer << ": " << ex.what() << endl; } catch (...) { - writeToLog(__FILE__, __LINE__, "oam.getSystemConfig error, unknown exception", LOG_TYPE_ERROR); - throw runtime_error("Setup failed"); + writeToLog(__FILE__, __LINE__, "Could not connect to " + fServer, LOG_TYPE_ERROR); } - - uint32_t pmCountConfig = moduletypeconfig.ModuleCount; - pmCount = 0; - int moduleID = 1; - - char buff[32]; - ByteStream bs, bsRead; - - if (fPrgmID == DDLPROC) - { - bs << (ByteStream::byte) WE_SVR_DDL_KEEPALIVE; - bs << (ByteStream::octbyte) moduleID; - } - else if (fPrgmID == DMLPROC) - { - bs << (ByteStream::byte) WE_SVR_DML_KEEPALIVE; - bs << (ByteStream::octbyte) moduleID; - } - else if (fPrgmID == SPLITTER) - { - bs << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE; - } - else if (fPrgmID == BATCHINSERTPROC) - { - bs << (ByteStream::byte) WE_SVR_BATCH_KEEPALIVE; - bs << (ByteStream::octbyte) moduleID; - } - - for (unsigned i = 0; i < pmCountConfig; i++) - { - //Find the module id - moduleID = atoi((moduletypeconfig.ModuleNetworkList[i]).DeviceName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); - //cout << "setting connection to moduleid " << moduleID << endl; - snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", moduleID); - string fServer (buff); - - boost::shared_ptr - cl(new MessageQueueClient(fServer, rm->getConfig())); - boost::shared_ptr nl(new boost::mutex()); - - //Bug 5224. Take out the retrys. If connection fails, we assume the server is down. - try - { - if (cl->connect()) - { - try - { - cl->write(bs); - } - catch (std::exception& ex1) - { - ostringstream oss; - oss << "Write to WES during connect failed due to " << ex1.what(); - throw runtime_error(oss.str()); - } - - try - { - bsRead = cl->read(); - - if (bsRead.length() == 0) - throw runtime_error("Got byte 0 during reading " ); - } - catch (std::exception& ex2) - { - ostringstream oss; - oss << "Read from WES during connect failed due to " << ex2.what() << " and this = " << this; - throw runtime_error(oss.str()); - } - - fPmConnections[moduleID] = cl; - //cout << "connection is open. this = " << this << endl; - //cout << "set up connection to mudule " << moduleID << endl; - // assign the module name - //ipAddress = sin_addr2String(cl->serv_addr().sin_addr); - ipAddress = cl->addr2String(); - cl->moduleName(getModuleNameByIPAddr(moduletypeconfig, ipAddress)); - StartClientListener(cl, i); - pmCount++; - //ostringstream oss; - //oss << "WECLIENT: connected to " << fServer + " and this = " << this << " and pmcount is now " << pmCount; - //writeToLog(__FILE__, __LINE__, oss.str() , LOG_TYPE_DEBUG); - } - else - { - throw runtime_error("Connection refused"); - } - } - catch (std::exception& ex) - { - writeToLog(__FILE__, __LINE__, "Could not connect to " + fServer + ": " + ex.what(), LOG_TYPE_ERROR); - cerr << "Could not connect to " << fServer << ": " << ex.what() << endl; - } - catch (...) - { - writeToLog(__FILE__, __LINE__, "Could not connect to " + fServer, LOG_TYPE_ERROR); - } - } - + } } int WEClients::Close() { - makeBusy(false); - closingConnection = 1; - ByteStream bs; - bs << (ByteStream::byte) WE_SVR_CLOSE_CONNECTION; - write_to_all(bs); + makeBusy(false); + closingConnection = 1; + ByteStream bs; + bs << (ByteStream::byte)WE_SVR_CLOSE_CONNECTION; + write_to_all(bs); -//cout << "connection is closed. this = " << this << " and closingConnection = " << closingConnection << endl; - for (uint32_t i = 0; i < fWESReader.size(); i++) - { - fWESReader[i]->join(); - } + // cout << "connection is closed. this = " << this << " and closingConnection = " << closingConnection << + // endl; + for (uint32_t i = 0; i < fWESReader.size(); i++) + { + fWESReader[i]->join(); + } - fWESReader.clear(); - fPmConnections.clear(); - pmCount = 0; - //ostringstream oss; - //oss << "WECLIENT: closed connection to wes and this = " << this << " and pmcount is now " << pmCount; - //writeToLog(__FILE__, __LINE__, oss.str() , LOG_TYPE_DEBUG); - return 0; + fWESReader.clear(); + fPmConnections.clear(); + pmCount = 0; + // ostringstream oss; + // oss << "WECLIENT: closed connection to wes and this = " << this << " and pmcount is now " << pmCount; + // writeToLog(__FILE__, __LINE__, oss.str() , LOG_TYPE_DEBUG); + return 0; } -void WEClients::Listen ( boost::shared_ptr client, uint32_t connIndex) +void WEClients::Listen(boost::shared_ptr client, uint32_t connIndex) { - SBS sbs; + SBS sbs; - try + try + { + while (Busy()) { - while ( Busy() ) + // TODO: This call blocks so setting Busy() in another thread doesn't work here... + sbs = client->read(); + + if (sbs->length() != 0) + { + // cout << "adding data to connIndex " << endl; + addDataToOutput(sbs, connIndex); + } + else // got zero bytes on read, nothing more will come + { + if (closingConnection > 0) { - //TODO: This call blocks so setting Busy() in another thread doesn't work here... - sbs = client->read(); - - if ( sbs->length() != 0 ) - { - //cout << "adding data to connIndex " << endl; - addDataToOutput(sbs, connIndex); - } - else // got zero bytes on read, nothing more will come - { - if (closingConnection > 0) - { - return; - } - - cerr << "WEC got 0 byte message for object " << this << endl; - goto Error; - } + return; } - return; - } - catch (std::exception& e) - { - cerr << "WEC Caught EXCEPTION: " << e.what() << endl; - goto Error; - } - catch (...) - { - cerr << "WEC Caught UNKNOWN EXCEPT" << endl; + cerr << "WEC got 0 byte message for object " << this << endl; goto Error; + } } + return; + } + catch (std::exception& e) + { + cerr << "WEC Caught EXCEPTION: " << e.what() << endl; + goto Error; + } + catch (...) + { + cerr << "WEC Caught UNKNOWN EXCEPT" << endl; + goto Error; + } + Error: - // error condition! push 0 length bs to messagequeuemap and - // eventually let jobstep error out. - boost::mutex::scoped_lock lk(fMlock); + // error condition! push 0 length bs to messagequeuemap and + // eventually let jobstep error out. + boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok; - sbs.reset(new ByteStream(0)); + MessageQueueMap::iterator map_tok; + sbs.reset(new ByteStream(0)); - for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok) - { - map_tok->second->queue.clear(); - (void)atomicops::atomicInc(&map_tok->second->unackedWork[0]); - map_tok->second->queue.push(sbs); - } - - lk.unlock(); - - // reset the pmconnection map - { - boost::mutex::scoped_lock onErrLock(fOnErrMutex); - string moduleName = client->moduleName(); - ClientList::iterator itor = fPmConnections.begin(); - - while (itor != fPmConnections.end()) - { - if (moduleName == (itor->second)->moduleName()) - { - (fPmConnections[itor->first]).reset(); - pmCount--; - ostringstream oss; - //oss << "WECLIENT: connection to is reset and this = " << this << " and pmcount is decremented."; - //writeToLog(__FILE__, __LINE__, oss.str() , LOG_TYPE_DEBUG); - } - - itor++; - } - } - return; -} - -void WEClients::addQueue(uint32_t key) -{ - bool b; - - boost::mutex* lock = new boost::mutex(); - condition* cond = new condition(); - boost::shared_ptr mqe(new MQE(pmCount)); - - mqe->queue = WESMsgQueue(lock, cond); - - boost::mutex::scoped_lock lk ( fMlock ); - b = fSessionMessages.insert(pair >(key, mqe)).second; - - if (!b) - { - ostringstream os; - os << "WEClient: attempt to add a queue with a duplicate ID " << key << endl; - throw runtime_error(os.str()); - } -} - -void WEClients::removeQueue(uint32_t key) -{ - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - return; - - map_tok->second->queue.shutdown(); + for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok) + { map_tok->second->queue.clear(); - fSessionMessages.erase(map_tok); -} + (void)atomicops::atomicInc(&map_tok->second->unackedWork[0]); + map_tok->second->queue.push(sbs); + } -void WEClients::shutdownQueue(uint32_t key) -{ - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - return; - - map_tok->second->queue.shutdown(); - map_tok->second->queue.clear(); -} - -void WEClients::read(uint32_t key, SBS& bs) -{ - boost::shared_ptr mqe; - - //Find the StepMsgQueueList for this session - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(key); - - if (map_tok == fSessionMessages.end()) - { - ostringstream os; - //cout << " reading for key " << key << " not found" << endl; - os << "WEClient: attempt to read(bs) from a nonexistent queue\n"; - throw runtime_error(os.str()); - } - - mqe = map_tok->second; - lk.unlock(); - - //this method can block: you can't hold any locks here... - (void)mqe->queue.pop(&bs); - - if (!bs) - bs.reset(new ByteStream()); -} - -void WEClients::write(const messageqcpp::ByteStream& msg, uint32_t connection) -{ - if (pmCount == 0) - { - ostringstream oss; - oss << "WECLIENT: There is no connection to WES and this = " << this ; - writeToLog(__FILE__, __LINE__, oss.str(), LOG_TYPE_DEBUG); - throw runtime_error("There is no WriteEngineServer to send message to."); - } - - if (fPmConnections[connection] != 0) - fPmConnections[connection]->write(msg); - else - { - ostringstream os; - os << "Lost connection to WriteEngineServer on pm" << connection; - throw runtime_error(os.str()); - } -} - -void WEClients::write_to_all(const messageqcpp::ByteStream& msg) -{ - if (pmCount == 0) - { - ostringstream oss; - oss << "WECLIENT: There is no connection to WES and this = " << this ; - writeToLog(__FILE__, __LINE__, oss.str(), LOG_TYPE_DEBUG); - throw runtime_error("There is no WriteEngineServer to send message to."); - } + lk.unlock(); + // reset the pmconnection map + { + boost::mutex::scoped_lock onErrLock(fOnErrMutex); + string moduleName = client->moduleName(); ClientList::iterator itor = fPmConnections.begin(); while (itor != fPmConnections.end()) { - if (itor->second != NULL) - { - itor->second->write(msg); - } + if (moduleName == (itor->second)->moduleName()) + { + (fPmConnections[itor->first]).reset(); + pmCount--; + ostringstream oss; + // oss << "WECLIENT: connection to is reset and this = " << this << " and pmcount is decremented."; + // writeToLog(__FILE__, __LINE__, oss.str() , LOG_TYPE_DEBUG); + } - itor++; + itor++; } + } + return; +} + +void WEClients::addQueue(uint32_t key) +{ + bool b; + + boost::mutex* lock = new boost::mutex(); + condition* cond = new condition(); + boost::shared_ptr mqe(new MQE(pmCount)); + + mqe->queue = WESMsgQueue(lock, cond); + + boost::mutex::scoped_lock lk(fMlock); + b = fSessionMessages.insert(pair >(key, mqe)).second; + + if (!b) + { + ostringstream os; + os << "WEClient: attempt to add a queue with a duplicate ID " << key << endl; + throw runtime_error(os.str()); + } +} + +void WEClients::removeQueue(uint32_t key) +{ + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + return; + + map_tok->second->queue.shutdown(); + map_tok->second->queue.clear(); + fSessionMessages.erase(map_tok); +} + +void WEClients::shutdownQueue(uint32_t key) +{ + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + return; + + map_tok->second->queue.shutdown(); + map_tok->second->queue.clear(); +} + +void WEClients::read(uint32_t key, SBS& bs) +{ + boost::shared_ptr mqe; + + // Find the StepMsgQueueList for this session + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(key); + + if (map_tok == fSessionMessages.end()) + { + ostringstream os; + // cout << " reading for key " << key << " not found" << endl; + os << "WEClient: attempt to read(bs) from a nonexistent queue\n"; + throw runtime_error(os.str()); + } + + mqe = map_tok->second; + lk.unlock(); + + // this method can block: you can't hold any locks here... + (void)mqe->queue.pop(&bs); + + if (!bs) + bs.reset(new ByteStream()); +} + +void WEClients::write(const messageqcpp::ByteStream& msg, uint32_t connection) +{ + if (pmCount == 0) + { + ostringstream oss; + oss << "WECLIENT: There is no connection to WES and this = " << this; + writeToLog(__FILE__, __LINE__, oss.str(), LOG_TYPE_DEBUG); + throw runtime_error("There is no WriteEngineServer to send message to."); + } + + if (fPmConnections[connection] != 0) + fPmConnections[connection]->write(msg); + else + { + ostringstream os; + os << "Lost connection to WriteEngineServer on pm" << connection; + throw runtime_error(os.str()); + } +} + +void WEClients::write_to_all(const messageqcpp::ByteStream& msg) +{ + if (pmCount == 0) + { + ostringstream oss; + oss << "WECLIENT: There is no connection to WES and this = " << this; + writeToLog(__FILE__, __LINE__, oss.str(), LOG_TYPE_DEBUG); + throw runtime_error("There is no WriteEngineServer to send message to."); + } + + ClientList::iterator itor = fPmConnections.begin(); + + while (itor != fPmConnections.end()) + { + if (itor->second != NULL) + { + itor->second->write(msg); + } + + itor++; + } } void WEClients::StartClientListener(boost::shared_ptr cl, uint32_t connIndex) { - boost::thread* thrd = new boost::thread(WEClientRunner(this, cl, connIndex)); - fWESReader.push_back(thrd); + boost::thread* thrd = new boost::thread(WEClientRunner(this, cl, connIndex)); + fWESReader.push_back(thrd); } - void WEClients::addDataToOutput(SBS sbs, uint32_t connIndex) { - // ISMPacketHeader *hdr = (ISMPacketHeader*)(sbs->buf()); - // PrimitiveHeader *p = (PrimitiveHeader *)(hdr+1); - //uint32_t uniqueId = p->UniqueID; - uint64_t uniqueId = 0; - *sbs >> uniqueId; - boost::shared_ptr mqe; + // ISMPacketHeader *hdr = (ISMPacketHeader*)(sbs->buf()); + // PrimitiveHeader *p = (PrimitiveHeader *)(hdr+1); + // uint32_t uniqueId = p->UniqueID; + uint64_t uniqueId = 0; + *sbs >> uniqueId; + boost::shared_ptr mqe; - boost::mutex::scoped_lock lk(fMlock); - MessageQueueMap::iterator map_tok = fSessionMessages.find(uniqueId); + boost::mutex::scoped_lock lk(fMlock); + MessageQueueMap::iterator map_tok = fSessionMessages.find(uniqueId); - if (map_tok == fSessionMessages.end()) - { - return; - } + if (map_tok == fSessionMessages.end()) + { + return; + } - mqe = map_tok->second; - lk.unlock(); + mqe = map_tok->second; + lk.unlock(); - if (pmCount > 0) - { - atomicops::atomicInc(&mqe->unackedWork[connIndex % pmCount]); - } - - (void)mqe->queue.push(sbs); + if (pmCount > 0) + { + atomicops::atomicInc(&mqe->unackedWork[connIndex % pmCount]); + } + (void)mqe->queue.push(sbs); } -} +} // namespace WriteEngine // vim:ts=4 sw=4: diff --git a/writeengine/client/we_clients.h b/writeengine/client/we_clients.h index e7f3b05e2..8028181ae 100644 --- a/writeengine/client/we_clients.h +++ b/writeengine/client/we_clients.h @@ -47,128 +47,137 @@ namespace WriteEngine { class WEClients { -public: - /** - * Constructors - */ - EXPORT WEClients(int PrgmID); - EXPORT ~WEClients(); + public: + /** + * Constructors + */ + EXPORT WEClients(int PrgmID); + EXPORT ~WEClients(); - //static boost::mutex map_mutex; - EXPORT void addQueue(uint32_t key); - EXPORT void removeQueue(uint32_t key); - EXPORT void shutdownQueue(uint32_t key); + // static boost::mutex map_mutex; + EXPORT void addQueue(uint32_t key); + EXPORT void removeQueue(uint32_t key); + EXPORT void shutdownQueue(uint32_t key); - /** @brief read a Write Engine Server response - * - * Returns the next message in the inbound queue for unique ids. - * @param bs A pointer to the ByteStream to fill in. - * @note: saves a copy vs read(uint32_t, uint32_t). - */ - EXPORT void read(uint32_t key, messageqcpp::SBS&); + /** @brief read a Write Engine Server response + * + * Returns the next message in the inbound queue for unique ids. + * @param bs A pointer to the ByteStream to fill in. + * @note: saves a copy vs read(uint32_t, uint32_t). + */ + EXPORT void read(uint32_t key, messageqcpp::SBS&); - /** @brief write function to write to specified PM - */ - EXPORT void write(const messageqcpp::ByteStream& msg, uint32_t connection); + /** @brief write function to write to specified PM + */ + EXPORT void write(const messageqcpp::ByteStream& msg, uint32_t connection); - /** @brief write function to write to all PMs - */ - EXPORT void write_to_all(const messageqcpp::ByteStream& msg); + /** @brief write function to write to all PMs + */ + EXPORT void write_to_all(const messageqcpp::ByteStream& msg); - /** @brief Shutdown this object - * - * Closes all the connections created during Setup() and cleans up other stuff. - */ - EXPORT int Close(); + /** @brief Shutdown this object + * + * Closes all the connections created during Setup() and cleans up other stuff. + */ + EXPORT int Close(); - /** @brief Start listening for Write Engine Server responses - * - * Starts the current thread listening on the client socket for Write Engine Server response messages. Will not return - * until busy() returns false or a zero-length response is received. - */ - EXPORT void Listen(boost::shared_ptr client, uint32_t connIndex); + /** @brief Start listening for Write Engine Server responses + * + * Starts the current thread listening on the client socket for Write Engine Server response messages. Will + * not return until busy() returns false or a zero-length response is received. + */ + EXPORT void Listen(boost::shared_ptr client, uint32_t connIndex); - /** @brief set/unset busy flag - * - * Set or unset the busy flag so Listen() can return. - */ - EXPORT void makeBusy(bool b) + /** @brief set/unset busy flag + * + * Set or unset the busy flag so Listen() can return. + */ + EXPORT void makeBusy(bool b) + { + fBusy = b; + } + + /** @brief fBusy accessor + * + */ + EXPORT bool Busy() const + { + return fBusy; + } + + EXPORT void Setup(); + + uint64_t connectedWEServers() const + { + return fPmConnections.size(); + } + + /** @brief accessor + */ + uint32_t getPmCount() + { + return pmCount; + } + + private: + WEClients(const WEClients& weClient); + WEClients& operator=(const WEClients& weClient); + typedef std::vector ReaderList; + typedef std::map > ClientList; + + // A queue of ByteStreams coming in from Write Engine Server + typedef joblist::ThreadSafeQueue WESMsgQueue; + + /* To keep some state associated with the connection */ + struct MQE + { + MQE(uint32_t pCount) : ackSocketIndex(0), pmCount(pCount) { - fBusy = b; + unackedWork.reset(new volatile uint32_t[pmCount]); + memset((void*)unackedWork.get(), 0, pmCount * sizeof(uint32_t)); } - - /** @brief fBusy accessor - * - */ - EXPORT bool Busy() const - { - return fBusy; - } - - EXPORT void Setup(); - - uint64_t connectedWEServers() const - { - return fPmConnections.size(); - } - - /** @brief accessor - */ - uint32_t getPmCount() - { - return pmCount; - } -private: - WEClients(const WEClients& weClient); - WEClients& operator=(const WEClients& weClient); - typedef std::vector ReaderList; - typedef std::map > ClientList; - - //A queue of ByteStreams coming in from Write Engine Server - typedef joblist::ThreadSafeQueue WESMsgQueue; - - /* To keep some state associated with the connection */ - struct MQE - { - MQE(uint32_t pCount) : ackSocketIndex(0), pmCount(pCount) - { - unackedWork.reset(new volatile uint32_t[pmCount]); - memset((void*) unackedWork.get(), 0, pmCount * sizeof(uint32_t)); - } - WESMsgQueue queue; - uint32_t ackSocketIndex; - boost::scoped_array unackedWork; - uint32_t pmCount; - }; - - //The mapping of session ids to StepMsgQueueLists - typedef std::map > MessageQueueMap; - - void StartClientListener(boost::shared_ptr cl, uint32_t connIndex); - - /** @brief Add a message to the queue - * - */ - void addDataToOutput(messageqcpp::SBS, uint32_t connIndex); - - int fPrgmID; - - ClientList fPmConnections; // all the Write Engine servers - ReaderList fWESReader; // all the reader threads for the pm servers - MessageQueueMap fSessionMessages; // place to put messages from the pm server to be returned by the Read method - boost::mutex fMlock; //sessionMessages mutex - std::vector > fWlock; //WES socket write mutexes - bool fBusy; - volatile uint32_t closingConnection; + WESMsgQueue queue; + uint32_t ackSocketIndex; + boost::scoped_array unackedWork; uint32_t pmCount; - boost::mutex fOnErrMutex; // to lock function scope to reset pmconnections under error condition + }; - boost::mutex ackLock; -public: - enum {DDLPROC = 0, SPLITTER, DMLPROC, BATCHINSERTPROC}; + // The mapping of session ids to StepMsgQueueLists + typedef std::map > MessageQueueMap; + + void StartClientListener(boost::shared_ptr cl, uint32_t connIndex); + + /** @brief Add a message to the queue + * + */ + void addDataToOutput(messageqcpp::SBS, uint32_t connIndex); + + int fPrgmID; + + ClientList fPmConnections; // all the Write Engine servers + ReaderList fWESReader; // all the reader threads for the pm servers + MessageQueueMap + fSessionMessages; // place to put messages from the pm server to be returned by the Read method + boost::mutex fMlock; // sessionMessages mutex + std::vector > fWlock; // WES socket write mutexes + bool fBusy; + volatile uint32_t closingConnection; + uint32_t pmCount; + boost::mutex fOnErrMutex; // to lock function scope to reset pmconnections under error condition + + boost::mutex ackLock; + + public: + enum + { + DDLPROC = 0, + SPLITTER, + DMLPROC, + BATCHINSERTPROC + }; }; -} +} // namespace WriteEngine #undef EXPORT diff --git a/writeengine/client/we_ddlcommandclient.cpp b/writeengine/client/we_ddlcommandclient.cpp index c4f53fed0..e17a0604d 100644 --- a/writeengine/client/we_ddlcommandclient.cpp +++ b/writeengine/client/we_ddlcommandclient.cpp @@ -36,63 +36,62 @@ namespace WriteEngine { WE_DDLCommandClient::WE_DDLCommandClient() { - fWEClient = new WEClients(WEClients::DDLPROC); + fWEClient = new WEClients(WEClients::DDLPROC); } WE_DDLCommandClient::~WE_DDLCommandClient() { - delete fWEClient; - fWEClient = NULL; + delete fWEClient; + fWEClient = NULL; } uint8_t WE_DDLCommandClient::UpdateSyscolumnNextval(uint32_t columnOid, uint64_t nextVal, uint32_t sessionID) { - ByteStream command, response; - uint8_t err = 0; - uint64_t uniqueId = fDbrm.getUnique64(); - fWEClient->addQueue(uniqueId); - command << (ByteStream::byte)WE_UPDATE_NEXTVAL; - command << uniqueId; - command << columnOid; - command << nextVal; - command << sessionID; - uint16_t dbRoot; - BRM::OID_t oid = 1021; - fDbrm.getSysCatDBRoot(oid, dbRoot); - int pmNum = 1; - boost::shared_ptr bsIn; + ByteStream command, response; + uint8_t err = 0; + uint64_t uniqueId = fDbrm.getUnique64(); + fWEClient->addQueue(uniqueId); + command << (ByteStream::byte)WE_UPDATE_NEXTVAL; + command << uniqueId; + command << columnOid; + command << nextVal; + command << sessionID; + uint16_t dbRoot; + BRM::OID_t oid = 1021; + fDbrm.getSysCatDBRoot(oid, dbRoot); + int pmNum = 1; + boost::shared_ptr bsIn; - try + try + { + fOam.getDbrootPmConfig(dbRoot, pmNum); + fWEClient->write(command, pmNum); + + while (1) { - fOam.getDbrootPmConfig (dbRoot, pmNum); - fWEClient->write(command, pmNum); + bsIn.reset(new ByteStream()); + fWEClient->read(uniqueId, bsIn); - while (1) - { - bsIn.reset(new ByteStream()); - fWEClient->read(uniqueId, bsIn); - - if ( bsIn->length() == 0 ) //read error - { - err = 1; - - break; - } - else - { - *bsIn >> err; - break; - } - } - } - catch (...) - { + if (bsIn->length() == 0) // read error + { err = 1; + + break; + } + else + { + *bsIn >> err; + break; + } } + } + catch (...) + { + err = 1; + } - fWEClient->removeQueue(uniqueId); - return err; -} - + fWEClient->removeQueue(uniqueId); + return err; } +} // namespace WriteEngine diff --git a/writeengine/client/we_ddlcommandclient.h b/writeengine/client/we_ddlcommandclient.h index e7bcf06e1..0f9a62aae 100644 --- a/writeengine/client/we_ddlcommandclient.h +++ b/writeengine/client/we_ddlcommandclient.h @@ -40,26 +40,25 @@ namespace WriteEngine { class WE_DDLCommandClient { -public: - EXPORT WE_DDLCommandClient(); - EXPORT ~WE_DDLCommandClient(); - /** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal. - * - * Update SYSCOLUMN nextval column for the columnoid with nexValue. - * @param columnOid (in) The column OID - * @param nextVal (in) The partition number - * @return 0 on success, non-0 on error. - */ - EXPORT uint8_t UpdateSyscolumnNextval(uint32_t columnOid, uint64_t nextVal, uint32_t sessionID = 0); - -private: - BRM::DBRM fDbrm; - WEClients* fWEClient; - oam::Oam fOam; + public: + EXPORT WE_DDLCommandClient(); + EXPORT ~WE_DDLCommandClient(); + /** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal. + * + * Update SYSCOLUMN nextval column for the columnoid with nexValue. + * @param columnOid (in) The column OID + * @param nextVal (in) The partition number + * @return 0 on success, non-0 on error. + */ + EXPORT uint8_t UpdateSyscolumnNextval(uint32_t columnOid, uint64_t nextVal, uint32_t sessionID = 0); + private: + BRM::DBRM fDbrm; + WEClients* fWEClient; + oam::Oam fOam; }; -} +} // namespace WriteEngine #undef EXPORT diff --git a/writeengine/client/we_dmlcommandclient.cpp b/writeengine/client/we_dmlcommandclient.cpp index 582c37bd0..1c2ea9d5a 100644 --- a/writeengine/client/we_dmlcommandclient.cpp +++ b/writeengine/client/we_dmlcommandclient.cpp @@ -29,21 +29,19 @@ using namespace messageqcpp; using namespace dataconvert; using namespace dmlpackage; - #include "we_dmlcommandclient.h" namespace WriteEngine { WE_DMLCommandClient::WE_DMLCommandClient() { - fWEClient = new WEClients(WEClients::DDLPROC); + fWEClient = new WEClients(WEClients::DDLPROC); } WE_DMLCommandClient::~WE_DMLCommandClient() { - delete fWEClient; - fWEClient = NULL; -} - + delete fWEClient; + fWEClient = NULL; } +} // namespace WriteEngine diff --git a/writeengine/client/we_dmlcommandclient.h b/writeengine/client/we_dmlcommandclient.h index 14382bf93..92756034c 100644 --- a/writeengine/client/we_dmlcommandclient.h +++ b/writeengine/client/we_dmlcommandclient.h @@ -40,18 +40,17 @@ namespace WriteEngine { class WE_DMLCommandClient { -public: - EXPORT WE_DMLCommandClient(); - EXPORT ~WE_DMLCommandClient(); - -private: - BRM::DBRM fDbrm; - WEClients* fWEClient; - oam::Oam fOam; + public: + EXPORT WE_DMLCommandClient(); + EXPORT ~WE_DMLCommandClient(); + private: + BRM::DBRM fDbrm; + WEClients* fWEClient; + oam::Oam fOam; }; -} +} // namespace WriteEngine #undef EXPORT diff --git a/writeengine/dictionary/qdctnry.cpp b/writeengine/dictionary/qdctnry.cpp index 403058d3e..42c46453e 100644 --- a/writeengine/dictionary/qdctnry.cpp +++ b/writeengine/dictionary/qdctnry.cpp @@ -32,807 +32,774 @@ using namespace WriteEngine; class DctnryTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DctnryTest); + // Dctnry basic testing + // CPPUNIT_TEST( testDctnryInsertLarge ); + // CPPUNIT_TEST( testDctnryInsertDelete ); + /// CPPUNIT_TEST( testDctnryInsertStress); + // CPPUNIT_TEST( testGetBlockHdr); + // CPPUNIT_TEST( testGetFreeSpace); + CPPUNIT_TEST(testDctnryInsertDeleteStore); + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST_SUITE( DctnryTest ); + private: + Dctnry m_Dctnry; + DctnryStore m_DctnryStore; + int m_oId; -// Dctnry basic testing -//CPPUNIT_TEST( testDctnryInsertLarge ); -//CPPUNIT_TEST( testDctnryInsertDelete ); -///CPPUNIT_TEST( testDctnryInsertStress); -//CPPUNIT_TEST( testGetBlockHdr); -//CPPUNIT_TEST( testGetFreeSpace); - CPPUNIT_TEST( testDctnryInsertDeleteStore); - CPPUNIT_TEST_SUITE_END(); + public: + void setUp() + { + } -private: - Dctnry m_Dctnry; - DctnryStore m_DctnryStore; - int m_oId; -public: - void setUp() + void tearDown() + { + } + DctnryTest() + { + m_oId = 0; + } + void testDctnryInsertLarge() + { + int oId = 2000; + // int blockCount =10; + + FILE* dFile = NULL; + + int rc = 0; + DataBlock curBlock; + // int largeSize=6144; + int largeSize = 2724; + int smallSize = 2032; + + uint16_t freeSpace; + uint64_t nextPtr; + uint16_t offSet0; + uint16_t endHeader; + unsigned char dctnryHeader[14]; + m_Dctnry.setUseSmallSize(false); + m_Dctnry.setDebugLevel(DEBUG_3); + m_oId = oId; + + memset(curBlock.data, 0, sizeof(curBlock.data)); + memset(dctnryHeader, 0, sizeof(dctnryHeader)); + printf("\nRunning testDctnryInsertLarge \n"); + + rc = m_Dctnry.deleteFile(oId); + rc = m_Dctnry.createDctnry(oId, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_Dctnry.openDctnry(oId); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + + rc = m_Dctnry.initDctnryHdr(dFile); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_Dctnry.closeDctnry(); + printf("After initDctnryHdr \n"); + testGetBlockHdr(); + + // dFile= m_Dctnry.openFile( oId); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // 12 for 6 bytes and 14 for 8 bytes + rc = m_Dctnry.readSubBlockEntry(dFile, &curBlock, 0, 0, 0, 14, &dctnryHeader); + CPPUNIT_ASSERT(rc == NO_ERROR); + memcpy(&freeSpace, dctnryHeader, 2); + memcpy(&nextPtr, dctnryHeader + 2, 8); // 8 bytes + memcpy(&offSet0, dctnryHeader + 10, 2); + memcpy(&endHeader, dctnryHeader + 12, 2); + + unsigned char sgnature_value[largeSize]; + memset(sgnature_value, 0, sizeof(sgnature_value)); + int j = 0; + + for (int i = 0; i < largeSize; i++) { + // if (j>255) + // j=0; + sgnature_value[i] = 119; + // j++; } - void tearDown() + // insert a signature value + Token token; + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + CPPUNIT_ASSERT(rc == NO_ERROR); + testGetBlockHdr(); + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + CPPUNIT_ASSERT(rc == NO_ERROR); + testGetBlockHdr(); + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + CPPUNIT_ASSERT(rc == NO_ERROR); + testGetBlockHdr(); + + token.fbo = 0; + token.op = 3; + rc = m_Dctnry.deleteDctnryValue(dFile, token); + testGetBlockHdr(); + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + CPPUNIT_ASSERT(rc == NO_ERROR); + testGetBlockHdr(); + + return; + + // add the same dictionary again. it should not be added + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + testGetBlockHdr(); + // add the same dictionary again. it should not be added to the next block + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + testGetBlockHdr(); + + printf("token fbo = %i", token.fbo); + printf("token op = %i", token.op); + token.fbo = 0; + token.op = 2; + // token.op++; + rc = m_Dctnry.deleteDctnryValue(dFile, token); + testGetBlockHdr(); + + return; + + // add a 2nd dictionary to fillup the whole file block + + unsigned char sgnature_value_s[smallSize]; + memset(sgnature_value_s, 0, sizeof(sgnature_value_s)); + + for (int i = 0; i < smallSize; i++) { + // if (j>255) + // j=0; + sgnature_value_s[i] = 118; + // j++; } - DctnryTest() + + // insert a signature value + rc = m_Dctnry.insertDctnry(dFile, smallSize, sgnature_value_s, token); + CPPUNIT_ASSERT(rc == NO_ERROR); + testGetBlockHdr(); + + return; + // insert a signature value + Token token3; + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token3); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_Dctnry.closeDctnry(dFile); + printf("After insert dictionary \n"); + testGetBlockHdr(); + rc = m_Dctnry.openDctnry(oId); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + rc = m_Dctnry.deleteDctnryValue(dFile, token); + m_Dctnry.closeDctnry(dFile); + printf("After delete dictionary \n"); + testGetBlockHdr(); + + CPPUNIT_ASSERT(rc == NO_ERROR); + j = 255; + + for (int i = 0; i < smallSize; i++) { - m_oId = 0; + if (j == 0) + j = 255; + + sgnature_value[i] = j; + j--; } - void testDctnryInsertLarge() + + // insert another signature value + rc = m_Dctnry.openDctnry(oId); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + Token token2; + rc = m_Dctnry.insertDctnry(dFile, smallSize, sgnature_value, token2); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_Dctnry.closeDctnry(dFile); + printf("After insert dictionary \n"); + testGetBlockHdr(); + m_Dctnry.dropDctnry(oId); + return; + } // testDctnryInsert + + void testDctnryInsertDelete() + { + int oId = 2001; + // int blockCount =10; + + FILE* dFile = NULL; + + int rc = 0; + DataBlock curBlock; + int largeSize = 9000; + int smallSize = 1000; + + unsigned char dctnryHeader[14]; + uint16_t freeSpace; + uint64_t nextPtr; + uint16_t offSet0; + uint16_t endHeader; + + m_Dctnry.setDebugLevel(DEBUG_3); + m_Dctnry.setUseSmallSize(true); + m_oId = oId; + memset(curBlock.data, 0, sizeof(curBlock.data)); + memset(dctnryHeader, 0, sizeof(dctnryHeader)); + printf("\nRunning testDctnryInsertDelete \n"); + + rc = m_Dctnry.deleteFile(oId); + + rc = m_Dctnry.createDctnry((FID)-1); + CPPUNIT_ASSERT(rc == ERR_OPEN_FILE); + rc = m_Dctnry.createDctnry(oId, 10); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_Dctnry.openDctnry(); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + + rc = m_Dctnry.initDctnryHdr(dFile); + CPPUNIT_ASSERT(rc == NO_ERROR); + // CPPUNIT_ASSERT(rc==NO_ERROR); + // m_Dctnry.closeFile(dFile); + m_Dctnry.closeDctnry(); + printf("After initDctnryHdr"); + testGetBlockHdr(); + // testGetFreeSpace(); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // 12 for 6 bytes and 14 for 8 bytes + rc = m_Dctnry.readSubBlockEntry(dFile, &curBlock, 0, 0, 0, 14, &dctnryHeader); + CPPUNIT_ASSERT(rc == NO_ERROR); + memcpy(&freeSpace, dctnryHeader, 2); + memcpy(&nextPtr, dctnryHeader + 2, 8); // 8 bytes + memcpy(&offSet0, dctnryHeader + 10, 2); + memcpy(&endHeader, dctnryHeader + 12, 2); + + unsigned char sgnature_value[largeSize]; + memset(sgnature_value, 0, sizeof(sgnature_value)); + int j = 0; + + for (int i = 0; i < largeSize; i++) { - int oId = 2000; -// int blockCount =10; - - FILE* dFile = NULL; - - int rc = 0; - DataBlock curBlock; -// int largeSize=6144; - int largeSize = 2724; - int smallSize = 2032; - - uint16_t freeSpace; - uint64_t nextPtr; - uint16_t offSet0; - uint16_t endHeader; - unsigned char dctnryHeader[14]; - m_Dctnry.setUseSmallSize(false); - m_Dctnry.setDebugLevel( DEBUG_3 ); - m_oId = oId; - - memset(curBlock.data, 0, sizeof (curBlock.data)); - memset(dctnryHeader, 0, sizeof(dctnryHeader)); - printf("\nRunning testDctnryInsertLarge \n"); - - rc = m_Dctnry.deleteFile(oId); - rc = m_Dctnry.createDctnry(oId, 3); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_Dctnry.openDctnry(oId); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - - - rc = m_Dctnry.initDctnryHdr( dFile); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_Dctnry.closeDctnry(); - printf("After initDctnryHdr \n"); - testGetBlockHdr(); - - //dFile= m_Dctnry.openFile( oId); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //12 for 6 bytes and 14 for 8 bytes - rc = m_Dctnry.readSubBlockEntry( dFile, &curBlock, 0, 0, 0, 14, &dctnryHeader); - CPPUNIT_ASSERT( rc == NO_ERROR ); - memcpy(&freeSpace, dctnryHeader, 2); - memcpy(&nextPtr, dctnryHeader + 2, 8); // 8 bytes - memcpy(&offSet0, dctnryHeader + 10, 2); - memcpy(&endHeader, dctnryHeader + 12, 2); - - unsigned char sgnature_value[largeSize]; - memset(sgnature_value, 0, sizeof(sgnature_value)); - int j = 0; - - for (int i = 0; i < largeSize; i++) - { -// if (j>255) -// j=0; - sgnature_value[i] = 119; -// j++; - } - - //insert a signature value - Token token; - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - CPPUNIT_ASSERT( rc == NO_ERROR ); - testGetBlockHdr(); - - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - CPPUNIT_ASSERT( rc == NO_ERROR ); - testGetBlockHdr(); - - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - CPPUNIT_ASSERT( rc == NO_ERROR ); - testGetBlockHdr(); - - token.fbo = 0; - token.op = 3; - rc = m_Dctnry.deleteDctnryValue( dFile, token); - testGetBlockHdr(); - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - CPPUNIT_ASSERT( rc == NO_ERROR ); - testGetBlockHdr(); - - - return; - - //add the same dictionary again. it should not be added - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - testGetBlockHdr(); - //add the same dictionary again. it should not be added to the next block - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - testGetBlockHdr(); - - printf("token fbo = %i", token.fbo); - printf("token op = %i", token.op); - token.fbo = 0; - token.op = 2; -// token.op++; - rc = m_Dctnry.deleteDctnryValue( dFile, token); - testGetBlockHdr(); - - return; - - //add a 2nd dictionary to fillup the whole file block - - unsigned char sgnature_value_s[smallSize]; - memset(sgnature_value_s, 0, sizeof(sgnature_value_s)); - - for (int i = 0; i < smallSize; i++) - { -// if (j>255) -// j=0; - sgnature_value_s[i] = 118; -// j++; - } - - //insert a signature value - rc = m_Dctnry.insertDctnry(dFile, smallSize, - sgnature_value_s, token); - CPPUNIT_ASSERT( rc == NO_ERROR ); - testGetBlockHdr(); - - - return; - //insert a signature value - Token token3; - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token3); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_Dctnry.closeDctnry(dFile); - printf("After insert dictionary \n"); - testGetBlockHdr(); - rc = m_Dctnry.openDctnry(oId); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - rc = m_Dctnry.deleteDctnryValue( dFile, token); - m_Dctnry.closeDctnry(dFile); - printf("After delete dictionary \n"); - testGetBlockHdr(); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - j = 255; - - for (int i = 0; i < smallSize; i++) - { - if (j == 0) - j = 255; - - sgnature_value[i] = j; - j--; - } - - //insert another signature value - rc = m_Dctnry.openDctnry(oId); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - Token token2; - rc = m_Dctnry.insertDctnry(dFile, smallSize, - sgnature_value, token2); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_Dctnry.closeDctnry(dFile); - printf("After insert dictionary \n"); - testGetBlockHdr(); - m_Dctnry.dropDctnry(oId); - return; - }//testDctnryInsert - - void testDctnryInsertDelete() - { - int oId = 2001; -// int blockCount =10; - - FILE* dFile = NULL; - - int rc = 0; - DataBlock curBlock; - int largeSize = 9000; - int smallSize = 1000; - - unsigned char dctnryHeader[14]; - uint16_t freeSpace; - uint64_t nextPtr; - uint16_t offSet0; - uint16_t endHeader; - - m_Dctnry.setDebugLevel( DEBUG_3 ); - m_Dctnry.setUseSmallSize(true); - m_oId = oId; - memset(curBlock.data, 0, sizeof (curBlock.data)); - memset(dctnryHeader, 0, sizeof(dctnryHeader)); - printf("\nRunning testDctnryInsertDelete \n"); - - rc = m_Dctnry.deleteFile(oId); - - rc = m_Dctnry.createDctnry((FID) - 1); - CPPUNIT_ASSERT( rc == ERR_OPEN_FILE ); - rc = m_Dctnry.createDctnry(oId, 10); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_Dctnry.openDctnry(); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - - rc = m_Dctnry.initDctnryHdr( dFile); - CPPUNIT_ASSERT( rc == NO_ERROR ); - //CPPUNIT_ASSERT(rc==NO_ERROR); - //m_Dctnry.closeFile(dFile); - m_Dctnry.closeDctnry(); - printf("After initDctnryHdr"); - testGetBlockHdr(); - //testGetFreeSpace(); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //12 for 6 bytes and 14 for 8 bytes - rc = m_Dctnry.readSubBlockEntry( dFile, &curBlock, 0, 0, 0, 14, &dctnryHeader); - CPPUNIT_ASSERT( rc == NO_ERROR ); - memcpy(&freeSpace, dctnryHeader, 2); - memcpy(&nextPtr, dctnryHeader + 2, 8); // 8 bytes - memcpy(&offSet0, dctnryHeader + 10, 2); - memcpy(&endHeader, dctnryHeader + 12, 2); - - unsigned char sgnature_value[largeSize]; - memset(sgnature_value, 0, sizeof(sgnature_value)); - int j = 0; - - for (int i = 0; i < largeSize; i++) - { - if (j > 255) - j = 0; - - sgnature_value[i] = j; - j++; - } - - //insert a signature value - Token token; - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - CPPUNIT_ASSERT( rc == ERR_DICT_SIZE_GT_8000); - largeSize = 6000; - Token token3; - - for (int i = 0; i < largeSize; i++) - { - if (j > 255) - j = 0; - - sgnature_value[i] = j; - j++; - } - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token3); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( token3.fbo == 0 ); - CPPUNIT_ASSERT( token3.op == 1 ); - m_Dctnry.closeDctnry(dFile); - printf("After insert token 3 into dictionary \n"); - testGetBlockHdr(); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - j = 255; - - for (int i = 0; i < smallSize; i++) - { - if (j == 0) - j = 255; - - sgnature_value[i] = j; - j--; - } - - //insert another signature value - Token token2; - rc = m_Dctnry.insertDctnry(dFile, smallSize, - sgnature_value, token2); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( token2.fbo == 0 ); - CPPUNIT_ASSERT( token2.op == 2 ); - m_Dctnry.closeDctnry(dFile); - printf("After insert token 2 into dictionary \n"); - testGetBlockHdr(); - //delete - memset(&token, 0, sizeof(token)); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //token dose not exist in dictionary - rc = m_Dctnry.deleteDctnryValue( dFile, token); - CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE ); - - //rc =m_Dctnry.deleteDctnryValue( dFile, token3); - rc = m_Dctnry.deleteDctnryValue( dFile, token2); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_Dctnry.closeDctnry(); - printf("After delete dictionary token2 \n"); - testGetBlockHdr(); - //delete right after - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //rc =m_Dctnry.deleteDctnryValue( dFile, token2); - rc = m_Dctnry.deleteDctnryValue( dFile, token3); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //m_Dctnry.closeFile( dFile ); - m_Dctnry.closeDctnry(); - printf("After delete dictionary token3\n"); - testGetBlockHdr(); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //rc =m_Dctnry.deleteDctnryValue( dFile, token2); - rc = m_Dctnry.deleteDctnryValue( dFile, token3); - CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE ); - rc = m_Dctnry.deleteDctnryValue( dFile, token2); - CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE ); - //CPPUNIT_ASSERT( rc == ERR_DICT_ZERO_LEN ); - m_Dctnry.closeDctnry(); - return; - }//testDctnryInsertDelete - - void testDctnryInsertStress() - { - - int oId = 2002; - int blockCount = 2; - FILE* dFile = NULL; - int rc = 0; - DataBlock curBlock; - int smallSize = 1000; - uint16_t freeSpace; - Offset hdrOffsets[4039]; - int opCount = 0; - - m_Dctnry.setDebugLevel( DEBUG_3); - m_Dctnry.setUseSmallSize(true); - m_oId = oId; - memset(curBlock.data, 0, sizeof (curBlock.data)); - memset(hdrOffsets, 0, sizeof(hdrOffsets)); - printf("\nRunning testDctnryInsertStress \n"); - - rc = m_Dctnry.deleteFile(oId); - rc = m_Dctnry.createFile(oId, blockCount); - - dFile = m_Dctnry.openFile( oId ); - CPPUNIT_ASSERT( dFile != NULL ); - - rc = m_Dctnry.initDctnryHdr( dFile); - CPPUNIT_ASSERT( rc == NO_ERROR ); - //m_Dctnry.closeFile(dFile); - m_Dctnry.closeDctnry(); - printf("After initDctnryHdr"); - testGetBlockHdr(); - dFile = m_Dctnry.openFile( oId); - unsigned char sgnature_value[smallSize]; - Token token[100]; - int i, j, k; - j = 255; - - for (k = 0; k < 18; k++) - { - for (int i = 0; i < smallSize; i++) - { - if (j == 0) - j = 255; - - sgnature_value[i] = k; - j--; - - }// endfor i - - rc = m_Dctnry.insertDctnry(dFile, smallSize, - sgnature_value, token[k]); - - if (rc != NO_ERROR) - { - - printf ("k: %i Error Code is: %i \n", k, rc); - //return; - } - - if (k > 15) - { - CPPUNIT_ASSERT( rc != NO_ERROR ); - printf("294. Error code is: %i \n", rc); - } - else - CPPUNIT_ASSERT( rc == NO_ERROR ); - - }//endfor k - - Offset prevOffset, curOffset; + if (j > 255) j = 0; - unsigned char* value = NULL; - - //This is to get the value out - for (i = 0; i < blockCount; i++) - { - rc = m_Dctnry.readSubBlockEntry( dFile, &curBlock, i, 0, 0, 2, &freeSpace); - CPPUNIT_ASSERT( rc == NO_ERROR ); - j = 0; - - prevOffset.hdrLoc = 2 + 8; // 8 bytes - memcpy(&(prevOffset.offset), &(curBlock.data[prevOffset.hdrLoc]), 2); - curOffset.hdrLoc = prevOffset.hdrLoc + 2; - memcpy(&(curOffset.offset), &(curBlock.data[curOffset.hdrLoc]), 2); - - int op = 1; - int size = prevOffset.offset - curOffset.offset; - - value = (unsigned char*)malloc(sizeof(unsigned char) * size); - memcpy(value, &curBlock.data[curOffset.offset], size ); - - while (curOffset.offset != 0xFFFF) - { - //printf("fbo: %i op: %i starting offset: %i ending offset %i size: %i \n", - // i, op, curOffset.offset, prevOffset.offset, size); - - //printf("value : "); - for (k = 0; k < size; k++) - { - //printf("%u",value[k]); - } - - // printf("\n"); - //start again - free(value); - value = NULL; - prevOffset.hdrLoc = curOffset.hdrLoc; - prevOffset.offset = curOffset.offset; - curOffset.hdrLoc += 2; - memcpy(&(curOffset.offset), &curBlock.data[curOffset.hdrLoc], 2); - size = prevOffset.offset - curOffset.offset; - - if (curOffset.offset != 0xFFFF) - { - value = (unsigned char*)malloc(sizeof(unsigned char) * size); - memcpy(value, &curBlock.data[curOffset.offset], size ); - } - - op++; - - }//end while - - //Get Offset info - m_Dctnry.closeDctnry(); - printf("After insertDctnry"); - testGetBlockHdr(); - dFile = m_Dctnry.openFile( oId ); - CPPUNIT_ASSERT( dFile != NULL ); - m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets); - int opCount2 = 0; - - for (k = 0; k < opCount; k++) - { - Token token; - token.fbo = i; - token.op = k + 1; - rc = m_Dctnry.findTokenValue(dFile, token, sgnature_value, size); - CPPUNIT_ASSERT( rc == NO_ERROR); - CPPUNIT_ASSERT( size == smallSize); - rc = m_Dctnry.deleteDctnryValue( dFile, token); - printf("After deleteDctnryValue fbo %i op %i\n", (int)i, k + 1); - m_Dctnry.getBlockHdr(dFile, i, opCount2, hdrOffsets); - Offset startOffset, endOffset; - printf("Header Info for fbo: %i \n %i %llu ", i, (int)m_Dctnry.getFree(), - m_Dctnry.getNextPtr()); - - endOffset.hdrLoc = 10; - endOffset.offset = 8192; - printf("%i ", endOffset.offset); - - for (int k1 = 0; k1 < opCount2; k1++) - { - startOffset.hdrLoc = hdrOffsets[k1].hdrLoc; - startOffset.offset = hdrOffsets[k1].offset; - printf("%i ", startOffset.offset); - }//end for k1 - - rc = m_Dctnry.deleteDctnryValue( dFile, token); - - printf("%x \n", 0xFFFF); - printf("k-> %i i-> %i error code -> %i \n ", k, i, rc); - - if (k < opCount - 1) - CPPUNIT_ASSERT( rc == ERR_DICT_ZERO_LEN); - else - CPPUNIT_ASSERT( rc == ERR_DICT_NO_OP_DELETE); - }//end for k - } //end for i - - //m_Dctnry.closeFile( dFile ); - char sigString[] = "Hello, I am a string; what are you? I am testing dictionary as a signature value, I don't know how long I am but we will find out in a minute"; - int sigStringSize = strlen(sigString); - char resultString[sigStringSize + 1]; - Token stringToken; - rc = m_Dctnry.insertDctnry(dFile, sigStringSize, - (unsigned char*)sigString, stringToken); - rc = m_Dctnry.findTokenValue(dFile, stringToken, (unsigned char*)resultString, sigStringSize); - resultString[sigStringSize] = '\0'; - printf("result String is %s --->size is %i\n", resultString, strlen(resultString) ); - m_Dctnry.closeDctnry(); - printf("After insert the result string, the header look like the following:\n"); - testGetBlockHdr(); - return; - }//testDctnryInsertStress - - void testGetBlockHdr() - { - - int blockCount = 2; - int oId = 2002; - int i = 0, k = 0; - FILE* dFile; - int opCount; - Offset hdrOffsets[4040]; - memset(hdrOffsets, 0, sizeof(hdrOffsets)); - m_Dctnry.setDebugLevel( DEBUG_3 ); - - - printf("\nRunning testGetBlockHdr \n"); - - dFile = m_Dctnry.openFile( m_oId ); - - if (dFile == NULL) - dFile = m_Dctnry.openFile( oId ); - - CPPUNIT_ASSERT( dFile != NULL ); - blockCount = m_Dctnry.getFileSize( dFile ) / BYTE_PER_BLOCK ; - - for (i = 0; i < blockCount; i++) - { - m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets); - Offset startOffset, endOffset; - int sigSize; - printf("Header Info for fbo: %i \n %i %llu ", i, (int)m_Dctnry.getFree(), - m_Dctnry.getNextPtr()); - //printf(" fbo %i total offset number %i \n", i, opCount); - endOffset.hdrLoc = 10; - endOffset.offset = 8192; - printf("%i ", endOffset.offset); - - for (k = 0; k < opCount; k++) - { - startOffset.hdrLoc = hdrOffsets[k].hdrLoc; - startOffset.offset = hdrOffsets[k].offset; - sigSize = endOffset.offset - startOffset.offset; - - //printf(" OP %i signature size : %i \n from %i to %i \n", k+1, - // sigSize, startOffset.offset,endOffset.offset ); - endOffset.hdrLoc = startOffset.hdrLoc ; - endOffset.offset = startOffset.offset; - printf("%i ", endOffset.offset); - }//end for k - - printf("%x \n", 0xFFFF); - }//endfor i - - m_Dctnry.closeDctnry(); + sgnature_value[i] = j; + j++; } - void testGetFreeSpace() + + // insert a signature value + Token token; + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + CPPUNIT_ASSERT(rc == ERR_DICT_SIZE_GT_8000); + largeSize = 6000; + Token token3; + + for (int i = 0; i < largeSize; i++) { - int oId = 2002; - FILE* dFile; + if (j > 255) + j = 0; - m_Dctnry.setDebugLevel( DEBUG_1 ); - - - printf("\nRunning testGetFreeSpace \n"); - dFile = m_Dctnry.openFile( m_oId ); - - if (dFile == NULL) - dFile = m_Dctnry.openFile( oId ); - - CPPUNIT_ASSERT( dFile != NULL ); - m_Dctnry.getFreeSpaceArray(dFile); - - for (int i = 0; i < m_Dctnry.getNumBlocks(); i++) - { - printf("fbo %i -->free space:%i \n", i, m_Dctnry.m_freeSpaceArray[i]); - } - - m_Dctnry.closeDctnry(); + sgnature_value[i] = j; + j++; } - void testDctnryInsertDeleteStore() + + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token3); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(token3.fbo == 0); + CPPUNIT_ASSERT(token3.op == 1); + m_Dctnry.closeDctnry(dFile); + printf("After insert token 3 into dictionary \n"); + testGetBlockHdr(); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + j = 255; + + for (int i = 0; i < smallSize; i++) { - int rc = 0; - int smallSize = 1000; - FID dctnryOID = 2002; - FID treeOID = 101; - FID listOID = 102; - m_DctnryStore.setDebugLevel( DEBUG_3 ); - printf("\nRunning testDctnryInsertDeleteStore \n"); - rc = m_DctnryStore.dropDctnryStore( treeOID, dctnryOID, listOID); - //CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.dropDctnryStore( treeOID, dctnryOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); + if (j == 0) + j = 255; - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.dropDctnryStore(); - CPPUNIT_ASSERT( rc == NO_ERROR ); + sgnature_value[i] = j; + j--; + } - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // insert another signature value + Token token2; + rc = m_Dctnry.insertDctnry(dFile, smallSize, sgnature_value, token2); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(token2.fbo == 0); + CPPUNIT_ASSERT(token2.op == 2); + m_Dctnry.closeDctnry(dFile); + printf("After insert token 2 into dictionary \n"); + testGetBlockHdr(); + // delete + memset(&token, 0, sizeof(token)); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // token dose not exist in dictionary + rc = m_Dctnry.deleteDctnryValue(dFile, token); + CPPUNIT_ASSERT(rc == ERR_DICT_NO_OP_DELETE); - rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // rc =m_Dctnry.deleteDctnryValue( dFile, token3); + rc = m_Dctnry.deleteDctnryValue(dFile, token2); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_Dctnry.closeDctnry(); + printf("After delete dictionary token2 \n"); + testGetBlockHdr(); + // delete right after + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // rc =m_Dctnry.deleteDctnryValue( dFile, token2); + rc = m_Dctnry.deleteDctnryValue(dFile, token3); + CPPUNIT_ASSERT(rc == NO_ERROR); - m_DctnryStore.closeDctnryStore(); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // m_Dctnry.closeFile( dFile ); + m_Dctnry.closeDctnry(); + printf("After delete dictionary token3\n"); + testGetBlockHdr(); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // rc =m_Dctnry.deleteDctnryValue( dFile, token2); + rc = m_Dctnry.deleteDctnryValue(dFile, token3); + CPPUNIT_ASSERT(rc == ERR_DICT_NO_OP_DELETE); + rc = m_Dctnry.deleteDctnryValue(dFile, token2); + CPPUNIT_ASSERT(rc == ERR_DICT_NO_OP_DELETE); + // CPPUNIT_ASSERT( rc == ERR_DICT_ZERO_LEN ); + m_Dctnry.closeDctnry(); + return; + } // testDctnryInsertDelete - rc = m_DctnryStore.openDctnryStore(); - CPPUNIT_ASSERT( rc == NO_ERROR ); + void testDctnryInsertStress() + { + int oId = 2002; + int blockCount = 2; + FILE* dFile = NULL; + int rc = 0; + DataBlock curBlock; + int smallSize = 1000; + uint16_t freeSpace; + Offset hdrOffsets[4039]; + int opCount = 0; - unsigned char sgnature_value[smallSize]; - Token token[100]; - memset(token, 0, sizeof(token)); - int i, j, k; + m_Dctnry.setDebugLevel(DEBUG_3); + m_Dctnry.setUseSmallSize(true); + m_oId = oId; + memset(curBlock.data, 0, sizeof(curBlock.data)); + memset(hdrOffsets, 0, sizeof(hdrOffsets)); + printf("\nRunning testDctnryInsertStress \n"); - for (int i = 0; i < smallSize; i++) + rc = m_Dctnry.deleteFile(oId); + rc = m_Dctnry.createFile(oId, blockCount); + + dFile = m_Dctnry.openFile(oId); + CPPUNIT_ASSERT(dFile != NULL); + + rc = m_Dctnry.initDctnryHdr(dFile); + CPPUNIT_ASSERT(rc == NO_ERROR); + // m_Dctnry.closeFile(dFile); + m_Dctnry.closeDctnry(); + printf("After initDctnryHdr"); + testGetBlockHdr(); + dFile = m_Dctnry.openFile(oId); + unsigned char sgnature_value[smallSize]; + Token token[100]; + int i, j, k; + j = 255; + + for (k = 0; k < 18; k++) + { + for (int i = 0; i < smallSize; i++) + { + if (j == 0) + j = 255; + + sgnature_value[i] = k; + j--; + + } // endfor i + + rc = m_Dctnry.insertDctnry(dFile, smallSize, sgnature_value, token[k]); + + if (rc != NO_ERROR) + { + printf("k: %i Error Code is: %i \n", k, rc); + // return; + } + + if (k > 15) + { + CPPUNIT_ASSERT(rc != NO_ERROR); + printf("294. Error code is: %i \n", rc); + } + else + CPPUNIT_ASSERT(rc == NO_ERROR); + + } // endfor k + + Offset prevOffset, curOffset; + j = 0; + unsigned char* value = NULL; + + // This is to get the value out + for (i = 0; i < blockCount; i++) + { + rc = m_Dctnry.readSubBlockEntry(dFile, &curBlock, i, 0, 0, 2, &freeSpace); + CPPUNIT_ASSERT(rc == NO_ERROR); + j = 0; + + prevOffset.hdrLoc = 2 + 8; // 8 bytes + memcpy(&(prevOffset.offset), &(curBlock.data[prevOffset.hdrLoc]), 2); + curOffset.hdrLoc = prevOffset.hdrLoc + 2; + memcpy(&(curOffset.offset), &(curBlock.data[curOffset.hdrLoc]), 2); + + int op = 1; + int size = prevOffset.offset - curOffset.offset; + + value = (unsigned char*)malloc(sizeof(unsigned char) * size); + memcpy(value, &curBlock.data[curOffset.offset], size); + + while (curOffset.offset != 0xFFFF) + { + // printf("fbo: %i op: %i starting offset: %i ending offset %i size: %i \n", + // i, op, curOffset.offset, prevOffset.offset, size); + + // printf("value : "); + for (k = 0; k < size; k++) { -// if (j>255) -// j=0; - sgnature_value[i] = 119; -// j++; + // printf("%u",value[k]); } - //insert a signature value - for (k = 1; k < 12; k++) - { + // printf("\n"); + // start again + free(value); + value = NULL; + prevOffset.hdrLoc = curOffset.hdrLoc; + prevOffset.offset = curOffset.offset; + curOffset.hdrLoc += 2; + memcpy(&(curOffset.offset), &curBlock.data[curOffset.hdrLoc], 2); + size = prevOffset.offset - curOffset.offset; - rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token[k]); - printf("token fbo = %i", token[k].fbo); - printf("token op = %i", token[k].op); + if (curOffset.offset != 0xFFFF) + { + value = (unsigned char*)malloc(sizeof(unsigned char) * size); + memcpy(value, &curBlock.data[curOffset.offset], size); } - testGetBlockHdr(); - return; + op++; - rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token[1]); - testGetBlockHdr(); + } // end while + // Get Offset info + m_Dctnry.closeDctnry(); + printf("After insertDctnry"); + testGetBlockHdr(); + dFile = m_Dctnry.openFile(oId); + CPPUNIT_ASSERT(dFile != NULL); + m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets); + int opCount2 = 0; - //Recheck if the tokens are all inserted - for (k = 1; k < 99; k++) + for (k = 0; k < opCount; k++) + { + Token token; + token.fbo = i; + token.op = k + 1; + rc = m_Dctnry.findTokenValue(dFile, token, sgnature_value, size); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(size == smallSize); + rc = m_Dctnry.deleteDctnryValue(dFile, token); + printf("After deleteDctnryValue fbo %i op %i\n", (int)i, k + 1); + m_Dctnry.getBlockHdr(dFile, i, opCount2, hdrOffsets); + Offset startOffset, endOffset; + printf("Header Info for fbo: %i \n %i %llu ", i, (int)m_Dctnry.getFree(), m_Dctnry.getNextPtr()); + + endOffset.hdrLoc = 10; + endOffset.offset = 8192; + printf("%i ", endOffset.offset); + + for (int k1 = 0; k1 < opCount2; k1++) { - for (i = 0; i < smallSize; i++) - { - if (j == 0) - j = 255; + startOffset.hdrLoc = hdrOffsets[k1].hdrLoc; + startOffset.offset = hdrOffsets[k1].offset; + printf("%i ", startOffset.offset); + } // end for k1 - if (k > 10) - sgnature_value[i] = k; - else - { - if (i < 8) - sgnature_value[i] = 1; - else - sgnature_value[i] = k; - } + rc = m_Dctnry.deleteDctnryValue(dFile, token); - j--; - }//endfor i ; second time + printf("%x \n", 0xFFFF); + printf("k-> %i i-> %i error code -> %i \n ", k, i, rc); - rc = m_DctnryStore.updateDctnryStore(sgnature_value, - smallSize, token[k]); + if (k < opCount - 1) + CPPUNIT_ASSERT(rc == ERR_DICT_ZERO_LEN); + else + CPPUNIT_ASSERT(rc == ERR_DICT_NO_OP_DELETE); + } // end for k + } // end for i - if (rc != NO_ERROR) - { - printf("443. Attentione!!! ERROR CODE : %i \n", rc); - } + // m_Dctnry.closeFile( dFile ); + char sigString[] = + "Hello, I am a string; what are you? I am testing dictionary as a signature value, I don't know how " + "long I am but we will find out in a minute"; + int sigStringSize = strlen(sigString); + char resultString[sigStringSize + 1]; + Token stringToken; + rc = m_Dctnry.insertDctnry(dFile, sigStringSize, (unsigned char*)sigString, stringToken); + rc = m_Dctnry.findTokenValue(dFile, stringToken, (unsigned char*)resultString, sigStringSize); + resultString[sigStringSize] = '\0'; + printf("result String is %s --->size is %i\n", resultString, strlen(resultString)); + m_Dctnry.closeDctnry(); + printf("After insert the result string, the header look like the following:\n"); + testGetBlockHdr(); + return; + } // testDctnryInsertStress - CPPUNIT_ASSERT( rc == NO_ERROR ); - } //endof for k; second time + void testGetBlockHdr() + { + int blockCount = 2; + int oId = 2002; + int i = 0, k = 0; + FILE* dFile; + int opCount; + Offset hdrOffsets[4040]; + memset(hdrOffsets, 0, sizeof(hdrOffsets)); + m_Dctnry.setDebugLevel(DEBUG_3); - int tempSize = 7; - rc = m_DctnryStore.updateDctnryStore(sgnature_value, - tempSize, token[k]); + printf("\nRunning testGetBlockHdr \n"); - if (rc != NO_ERROR) + dFile = m_Dctnry.openFile(m_oId); + + if (dFile == NULL) + dFile = m_Dctnry.openFile(oId); + + CPPUNIT_ASSERT(dFile != NULL); + blockCount = m_Dctnry.getFileSize(dFile) / BYTE_PER_BLOCK; + + for (i = 0; i < blockCount; i++) + { + m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets); + Offset startOffset, endOffset; + int sigSize; + printf("Header Info for fbo: %i \n %i %llu ", i, (int)m_Dctnry.getFree(), m_Dctnry.getNextPtr()); + // printf(" fbo %i total offset number %i \n", i, opCount); + endOffset.hdrLoc = 10; + endOffset.offset = 8192; + printf("%i ", endOffset.offset); + + for (k = 0; k < opCount; k++) + { + startOffset.hdrLoc = hdrOffsets[k].hdrLoc; + startOffset.offset = hdrOffsets[k].offset; + sigSize = endOffset.offset - startOffset.offset; + + // printf(" OP %i signature size : %i \n from %i to %i \n", k+1, + // sigSize, startOffset.offset,endOffset.offset ); + endOffset.hdrLoc = startOffset.hdrLoc; + endOffset.offset = startOffset.offset; + printf("%i ", endOffset.offset); + } // end for k + + printf("%x \n", 0xFFFF); + } // endfor i + + m_Dctnry.closeDctnry(); + } + void testGetFreeSpace() + { + int oId = 2002; + FILE* dFile; + + m_Dctnry.setDebugLevel(DEBUG_1); + + printf("\nRunning testGetFreeSpace \n"); + dFile = m_Dctnry.openFile(m_oId); + + if (dFile == NULL) + dFile = m_Dctnry.openFile(oId); + + CPPUNIT_ASSERT(dFile != NULL); + m_Dctnry.getFreeSpaceArray(dFile); + + for (int i = 0; i < m_Dctnry.getNumBlocks(); i++) + { + printf("fbo %i -->free space:%i \n", i, m_Dctnry.m_freeSpaceArray[i]); + } + + m_Dctnry.closeDctnry(); + } + void testDctnryInsertDeleteStore() + { + int rc = 0; + int smallSize = 1000; + FID dctnryOID = 2002; + FID treeOID = 101; + FID listOID = 102; + m_DctnryStore.setDebugLevel(DEBUG_3); + printf("\nRunning testDctnryInsertDeleteStore \n"); + rc = m_DctnryStore.dropDctnryStore(treeOID, dctnryOID, listOID); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_DctnryStore.dropDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_DctnryStore.dropDctnryStore(treeOID, dctnryOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_DctnryStore.dropDctnryStore(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_DctnryStore.closeDctnryStore(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_DctnryStore.openDctnryStore(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + unsigned char sgnature_value[smallSize]; + Token token[100]; + memset(token, 0, sizeof(token)); + int i, j, k; + + for (int i = 0; i < smallSize; i++) + { + // if (j>255) + // j=0; + sgnature_value[i] = 119; + // j++; + } + + // insert a signature value + for (k = 1; k < 12; k++) + { + rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token[k]); + printf("token fbo = %i", token[k].fbo); + printf("token op = %i", token[k].op); + } + + testGetBlockHdr(); + return; + + rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token[1]); + testGetBlockHdr(); + + // Recheck if the tokens are all inserted + for (k = 1; k < 99; k++) + { + for (i = 0; i < smallSize; i++) + { + if (j == 0) + j = 255; + + if (k > 10) + sgnature_value[i] = k; + else { - printf("452. Predicted Error Code should be 1363: The result ERROR CODE : %i \n", rc); + if (i < 8) + sgnature_value[i] = 1; + else + sgnature_value[i] = k; } - CPPUNIT_ASSERT( rc == 1363 ); + j--; + } // endfor i ; second time - tempSize = 8; - rc = m_DctnryStore.updateDctnryStore(sgnature_value, - tempSize, token[k]); + rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token[k]); - if (rc != NO_ERROR) - { - printf("461. Attention!!! ERROR CODE : %i \n", rc); - } + if (rc != NO_ERROR) + { + printf("443. Attentione!!! ERROR CODE : %i \n", rc); + } - CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT(rc == NO_ERROR); + } // endof for k; second time - for (i = 1; i < 99; i++) - { - if ( m_DctnryStore.isDebug( DEBUG_3 )) - { - printf("i : %i token.fbo %i token.op %i \n", i, (int)token[i].fbo, (int)token[i].op); - } - } + int tempSize = 7; + rc = m_DctnryStore.updateDctnryStore(sgnature_value, tempSize, token[k]); - for (i = 1; i < 99; i++) - { - rc = m_DctnryStore.deleteDctnryToken(token[i]); + if (rc != NO_ERROR) + { + printf("452. Predicted Error Code should be 1363: The result ERROR CODE : %i \n", rc); + } - if (rc != NO_ERROR) - { - printf("475 . Attention!!! ERROR CODE : %i \n", rc); - } + CPPUNIT_ASSERT(rc == 1363); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } + tempSize = 8; + rc = m_DctnryStore.updateDctnryStore(sgnature_value, tempSize, token[k]); - m_DctnryStore.closeDctnryStore(); - //rc = m_DctnryStore.dropDctnryStore(); - //CPPUNIT_ASSERT( rc == NO_ERROR ); - return; - }//testDctnryInsertDeleteStore + if (rc != NO_ERROR) + { + printf("461. Attention!!! ERROR CODE : %i \n", rc); + } + + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 1; i < 99; i++) + { + if (m_DctnryStore.isDebug(DEBUG_3)) + { + printf("i : %i token.fbo %i token.op %i \n", i, (int)token[i].fbo, (int)token[i].op); + } + } + + for (i = 1; i < 99; i++) + { + rc = m_DctnryStore.deleteDctnryToken(token[i]); + + if (rc != NO_ERROR) + { + printf("475 . Attention!!! ERROR CODE : %i \n", rc); + } + + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + m_DctnryStore.closeDctnryStore(); + // rc = m_DctnryStore.dropDctnryStore(); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + return; + } // testDctnryInsertDeleteStore }; -CPPUNIT_TEST_SUITE_REGISTRATION( DctnryTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(DctnryTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/writeengine/dictionary/tdctnry.cpp b/writeengine/dictionary/tdctnry.cpp index f9a731661..ecfc2cb05 100644 --- a/writeengine/dictionary/tdctnry.cpp +++ b/writeengine/dictionary/tdctnry.cpp @@ -29,669 +29,640 @@ using namespace boost; #include "we_dctnrystore.h" #include -//using namespace dataconvert; +// using namespace dataconvert; using namespace WriteEngine; class DctnryTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(DctnryTest); + // Dctnry basic testing + CPPUNIT_TEST(testDctnryInsertDelete); + CPPUNIT_TEST(testDctnryInsertDeleteStore); + CPPUNIT_TEST(testDctnryInsertStressStore); + CPPUNIT_TEST(testDctnryDropCreate); + CPPUNIT_TEST(testDctnryOpen); + CPPUNIT_TEST(testDctnryMultipleInsert); - CPPUNIT_TEST_SUITE( DctnryTest ); + CPPUNIT_TEST_SUITE_END(); -// Dctnry basic testing - CPPUNIT_TEST( testDctnryInsertDelete ); - CPPUNIT_TEST( testDctnryInsertDeleteStore); - CPPUNIT_TEST( testDctnryInsertStressStore); - CPPUNIT_TEST(testDctnryDropCreate); - CPPUNIT_TEST(testDctnryOpen); - CPPUNIT_TEST( testDctnryMultipleInsert); + private: + Dctnry m_Dctnry; + DctnryStore m_DctnryStore; + int m_oId; - CPPUNIT_TEST_SUITE_END(); + public: + void setUp() + { + BRMWrapper::setUseVb(false); + } -private: - Dctnry m_Dctnry; - DctnryStore m_DctnryStore; - int m_oId; -public: - void setUp() + void tearDown() + { + } + DctnryTest() + { + m_oId = -1; + } + + void testDctnryInsertDelete() + { + int oId = 2001; + FILE* dFile = NULL; + int rc = 0; + DataBlock curBlock; + int largeSize = 9000; + int smallSize = 1000; + + unsigned char dctnryHeader[DCTNRY_HEADER_SIZE]; + uint16_t freeSpace; + uint64_t nextPtr; + uint16_t offSet0; + uint16_t endHeader; + + m_Dctnry.setDebugLevel(DEBUG_3); + m_Dctnry.setUseSmallSize(true); + m_oId = oId; + memset(curBlock.data, 0, sizeof(curBlock.data)); + memset(dctnryHeader, 0, sizeof(dctnryHeader)); + printf("\nRunning testDctnryInsertDelete \n"); + + rc = m_Dctnry.dropDctnry(oId); + rc = m_Dctnry.createDctnry(oId, 10); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // 12 for 6 bytes and 14 for 8 bytes + uint64_t lbid = 0; + BRMWrapper::getInstance()->getBrmInfo(oId, 0, lbid); + printf("passed brm above \n"); + rc = m_Dctnry.readSubBlockEntry(dFile, &curBlock, lbid, 0, 0, DCTNRY_HEADER_SIZE, &dctnryHeader); + CPPUNIT_ASSERT(rc == NO_ERROR); + + memcpy(&freeSpace, dctnryHeader, HDR_UNIT_SIZE); + memcpy(&nextPtr, dctnryHeader + HDR_UNIT_SIZE, NEXT_PTR_BYTES); // 8 bytes + memcpy(&offSet0, dctnryHeader + NEXT_PTR_BYTES + HDR_UNIT_SIZE, HDR_UNIT_SIZE); + memcpy(&endHeader, dctnryHeader + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE, HDR_UNIT_SIZE); + + unsigned char sgnature_value[largeSize]; + memset(sgnature_value, 0, sizeof(sgnature_value)); + int j = 0; + + // j is a 1 byte character; + for (int i = 0; i < largeSize; i++) { - BRMWrapper::setUseVb(false); + if (j > 255) + j = 0; + + sgnature_value[i] = j; + j++; } - void tearDown() - { + // insert a signature value + Token token; - } - DctnryTest() + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token); + CPPUNIT_ASSERT(rc == ERR_DICT_SIZE_GT_8000); + largeSize = 4000; + Token token3; + + for (int i = 0; i < largeSize; i++) { - m_oId = -1; + if (j > 255) + j = 0; + + sgnature_value[i] = j; + j++; } - void testDctnryInsertDelete() + rc = m_Dctnry.insertDctnry(dFile, largeSize, sgnature_value, token3); + cout << "rc=" << rc << endl; + CPPUNIT_ASSERT(rc == NO_ERROR); + int fboCnt; + BRMWrapper::getInstance()->getFboOffset(token3.fbo, fboCnt); + CPPUNIT_ASSERT(fboCnt == 0); + CPPUNIT_ASSERT(token3.op == 1); + m_Dctnry.closeDctnry(dFile); + printf("After insert token 3 into dictionary \n"); + + testGetBlockHdr(); + + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + j = 255; + + for (int i = 0; i < smallSize; i++) { - int oId = 2001; - FILE* dFile = NULL; - int rc = 0; - DataBlock curBlock; - int largeSize = 9000; - int smallSize = 1000; - - unsigned char dctnryHeader[DCTNRY_HEADER_SIZE]; - uint16_t freeSpace; - uint64_t nextPtr; - uint16_t offSet0; - uint16_t endHeader; - - m_Dctnry.setDebugLevel( DEBUG_3 ); - m_Dctnry.setUseSmallSize(true); - m_oId = oId; - memset(curBlock.data, 0, sizeof (curBlock.data)); - memset(dctnryHeader, 0, sizeof(dctnryHeader)); - printf("\nRunning testDctnryInsertDelete \n"); - - rc = m_Dctnry.dropDctnry(oId); - rc = m_Dctnry.createDctnry(oId, 10); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //12 for 6 bytes and 14 for 8 bytes - uint64_t lbid = 0; - BRMWrapper::getInstance()->getBrmInfo( oId, 0, lbid ); - printf("passed brm above \n"); - rc = m_Dctnry.readSubBlockEntry( dFile, &curBlock, lbid, 0, 0, DCTNRY_HEADER_SIZE, &dctnryHeader); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - memcpy(&freeSpace, dctnryHeader, HDR_UNIT_SIZE); - memcpy(&nextPtr, dctnryHeader + HDR_UNIT_SIZE, NEXT_PTR_BYTES); // 8 bytes - memcpy(&offSet0, dctnryHeader + NEXT_PTR_BYTES + HDR_UNIT_SIZE, HDR_UNIT_SIZE); - memcpy(&endHeader, dctnryHeader + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE, HDR_UNIT_SIZE); - - unsigned char sgnature_value[largeSize]; - memset(sgnature_value, 0, sizeof(sgnature_value)); - int j = 0; - - //j is a 1 byte character; - for (int i = 0; i < largeSize; i++) - { - if (j > 255) - j = 0; - - sgnature_value[i] = j; - j++; - } - - //insert a signature value - Token token; - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token); - CPPUNIT_ASSERT( rc == ERR_DICT_SIZE_GT_8000); - largeSize = 4000; - Token token3; - - for (int i = 0; i < largeSize; i++) - { - if (j > 255) - j = 0; - - sgnature_value[i] = j; - j++; - } - - rc = m_Dctnry.insertDctnry(dFile, largeSize, - sgnature_value, token3); - cout << "rc=" << rc << endl; - CPPUNIT_ASSERT( rc == NO_ERROR ); - int fboCnt; - BRMWrapper::getInstance()->getFboOffset( token3.fbo, fboCnt); - CPPUNIT_ASSERT( fboCnt == 0 ); - CPPUNIT_ASSERT( token3.op == 1 ); - m_Dctnry.closeDctnry(dFile); - printf("After insert token 3 into dictionary \n"); - - testGetBlockHdr(); - - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); + if (j == 0) j = 255; - for (int i = 0; i < smallSize; i++) - { - if (j == 0) - j = 255; + sgnature_value[i] = j; + j--; + } - sgnature_value[i] = j; - j--; + // insert another signature value + Token token2; + rc = m_Dctnry.insertDctnry(dFile, smallSize, sgnature_value, token2); + CPPUNIT_ASSERT(rc == NO_ERROR); + + BRMWrapper::getInstance()->getFboOffset(token2.fbo, fboCnt); + CPPUNIT_ASSERT(fboCnt == 0); + CPPUNIT_ASSERT(token2.op == 2); + m_Dctnry.closeDctnry(dFile); + printf("After insert token 2 into dictionary \n"); + + testGetBlockHdr(); + + // delete + memset(&token, 0, sizeof(token)); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + // token dose not exist in dictionary + rc = m_Dctnry.deleteDctnryValue(dFile, token); + CPPUNIT_ASSERT(rc == ERR_DICT_BAD_TOKEN_OP); + + rc = m_Dctnry.deleteDctnryValue(dFile, token2); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_Dctnry.closeDctnry(); + printf("After delete dictionary token2 \n"); + testGetBlockHdr(); + // delete right after + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + unsigned char sigValue3[8000]; + int sigSize3; + rc = m_Dctnry.findTokenValue(dFile, token3, sigValue3, sigSize3); + CPPUNIT_ASSERT(sigSize3 == 4000); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_Dctnry.deleteDctnryValue(dFile, token3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_Dctnry.closeDctnry(); + printf("After delete dictionary token3\n"); + testGetBlockHdr(); + rc = m_Dctnry.openDctnry(); + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + CPPUNIT_ASSERT(dFile != NULL); + rc = m_Dctnry.deleteDctnryValue(dFile, token3); + CPPUNIT_ASSERT(rc == ERR_DICT_BAD_TOKEN_OP); + rc = m_Dctnry.deleteDctnryValue(dFile, token2); + CPPUNIT_ASSERT(rc == ERR_DICT_BAD_TOKEN_OP); + m_Dctnry.closeDctnry(); + printf("\nSuccessfully Running testDctnryInsertDelete \n"); + return; + } // testDctnryInsertDelete + + void testGetBlockHdr() + { + int blockCount = 2; + int i = 0, k = 0; + FILE* dFile; + int opCount; + int rc; + + Offset hdrOffsets[1024]; + memset(hdrOffsets, 0, sizeof(hdrOffsets)); + m_Dctnry.setDebugLevel(DEBUG_3); + + printf("\nRunning testGetBlockHdr \n"); + rc = m_Dctnry.openDctnry(m_oId); + + CPPUNIT_ASSERT(rc == NO_ERROR); + dFile = m_Dctnry.getDctnryFile(); + blockCount = m_Dctnry.getFileSize(dFile) / BYTE_PER_BLOCK; + + for (i = 0; i < blockCount; i++) + { + int freeSpace; + freeSpace = (int)m_Dctnry.getFree(); + + if (freeSpace == 8178) + break; + + printf("Header Info for fbo: %i %i %llu ", i, freeSpace, m_Dctnry.getNextPtr()); + m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets); + + if (opCount == -1) + { + printf("ERROR BAD DICITONARY FILE|n"); + return; + } + + Offset startOffset, endOffset; + int sigSize; + + printf(" fbo %i total offset number %i \n", i, opCount); + endOffset.hdrLoc = 10; + endOffset.offset = 8192; + + for (k = 0; k < opCount; k++) + { + startOffset.hdrLoc = hdrOffsets[k].hdrLoc; + startOffset.offset = hdrOffsets[k].offset; + sigSize = endOffset.offset - startOffset.offset; + + printf(" OP %i signature size : %i \n from %i to %i \n", k + 1, sigSize, startOffset.offset, + endOffset.offset); + endOffset.hdrLoc = startOffset.hdrLoc; + endOffset.offset = startOffset.offset; + printf("%i ", endOffset.offset); + } // end for k + + // printf("%x \n", 0xFFFF); + } // endfor i + + m_Dctnry.closeDctnry(); + printf("\nSuccessfully Running testGetBlockHdr \n"); + return; + } + void testDctnryDropCreate() + { + int rc = 0; + FID dctnryOID = 700; + FID treeOID = 101; + FID listOID = 102; + + m_DctnryStore.setDebugLevel(DEBUG_3); + + printf("\nRunning testDctnryDropCreate \n"); + + rc = m_DctnryStore.dropDctnryStore(dctnryOID, treeOID, listOID); + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_DctnryStore.closeDctnryStore(); + + return; + } + + void testDctnryOpen() + { + int rc = 0; + FID dctnryOID = 700; + FID treeOID = 101; + FID listOID = 102; + + m_DctnryStore.setDebugLevel(DEBUG_3); + + printf("\nRunning testDctnryOpen \n"); + + rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_DctnryStore.closeDctnryStore(); + + return; + } + void testDctnryInsertDeleteStore() + { + int rc = 0; + int smallSize = 1000; + FID dctnryOID = 100; + FID treeOID = 101; + FID listOID = 102; + Token token2[100]; + + m_DctnryStore.setDebugLevel(DEBUG_3); + + printf("\nRunning testDctnryInsertDeleteStore \n"); + printf("dropDctnryStore\n"); + rc = m_DctnryStore.dropDctnryStore(dctnryOID, treeOID, listOID); + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + unsigned char sgnature_value[smallSize]; + Token token[500]; + memset(token, 0, sizeof(token)); + int i, j, k; + j = 255; + + for (k = 1; k < 499; k++) + { + for (i = 0; i < smallSize; i++) + { + if (j == 0) + j = 255; + + if (k > 10) + sgnature_value[i] = k; + else + { + if (i < 8) + sgnature_value[i] = 1; + else + sgnature_value[i] = k; } - //insert another signature value - Token token2; - rc = m_Dctnry.insertDctnry(dFile, smallSize, - sgnature_value, token2); - CPPUNIT_ASSERT( rc == NO_ERROR ); + j--; + } // endfor i - BRMWrapper::getInstance()->getFboOffset( token2.fbo, fboCnt); - CPPUNIT_ASSERT( fboCnt == 0 ); - CPPUNIT_ASSERT( token2.op == 2 ); - m_Dctnry.closeDctnry(dFile); - printf("After insert token 2 into dictionary \n"); + rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token[k]); - testGetBlockHdr(); + if (rc != NO_ERROR) + printf("rc %i k %i \n", rc, k); - //delete - memset(&token, 0, sizeof(token)); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - //token dose not exist in dictionary - rc = m_Dctnry.deleteDctnryValue( dFile, token); - CPPUNIT_ASSERT( rc == ERR_DICT_BAD_TOKEN_OP ); + CPPUNIT_ASSERT(rc == NO_ERROR); + } // endof for k + printf("first time inserted %i values token\n", k); + // Recheck if the tokens are all inserted + memset(token2, 0, sizeof(token2)); - rc = m_Dctnry.deleteDctnryValue( dFile, token2); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_Dctnry.closeDctnry(); - printf("After delete dictionary token2 \n"); - testGetBlockHdr(); - //delete right after - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - unsigned char sigValue3[8000]; - int sigSize3; - rc = m_Dctnry.findTokenValue (dFile, token3, - sigValue3, - sigSize3 ); - CPPUNIT_ASSERT(sigSize3 == 4000); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_Dctnry.deleteDctnryValue( dFile, token3); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_Dctnry.closeDctnry(); - printf("After delete dictionary token3\n"); - testGetBlockHdr(); - rc = m_Dctnry.openDctnry(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - CPPUNIT_ASSERT( dFile != NULL ); - rc = m_Dctnry.deleteDctnryValue( dFile, token3); - CPPUNIT_ASSERT( rc == ERR_DICT_BAD_TOKEN_OP ); - rc = m_Dctnry.deleteDctnryValue( dFile, token2); - CPPUNIT_ASSERT( rc == ERR_DICT_BAD_TOKEN_OP ); - m_Dctnry.closeDctnry(); - printf("\nSuccessfully Running testDctnryInsertDelete \n"); - return; - }//testDctnryInsertDelete - - - void testGetBlockHdr() + for (k = 1; k < 99; k++) { - int blockCount = 2; - int i = 0, k = 0; - FILE* dFile; - int opCount; - int rc; + for (i = 0; i < smallSize; i++) + { + if (j == 0) + j = 255; - Offset hdrOffsets[1024]; - memset(hdrOffsets, 0, sizeof(hdrOffsets)); - m_Dctnry.setDebugLevel( DEBUG_3 ); - - - printf("\nRunning testGetBlockHdr \n"); - rc = m_Dctnry.openDctnry( m_oId ); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - dFile = m_Dctnry.getDctnryFile(); - blockCount = m_Dctnry.getFileSize( dFile ) / BYTE_PER_BLOCK ; - - for (i = 0; i < blockCount; i++) + if (k > 10) + sgnature_value[i] = k; + else { - int freeSpace; - freeSpace = (int)m_Dctnry.getFree(); + if (i < 8) + sgnature_value[i] = 1; + else + sgnature_value[i] = k; + } - if (freeSpace == 8178) - break; + j--; + } // endfor i ; second time - printf("Header Info for fbo: %i %i %llu ", i, freeSpace, - m_Dctnry.getNextPtr()); - m_Dctnry.getBlockHdr(dFile, i, opCount, hdrOffsets); + rc = m_DctnryStore.updateDctnryStore(sgnature_value, smallSize, token2[k]); - if (opCount == -1) - { - printf("ERROR BAD DICITONARY FILE|n"); - return; - } + if (rc != NO_ERROR) + { + printf("443. Attentione!!! ERROR CODE : %i \n", rc); + } - Offset startOffset, endOffset; - int sigSize; + CPPUNIT_ASSERT(rc == NO_ERROR); + } // endof for k; second time - printf(" fbo %i total offset number %i \n", i, opCount); - endOffset.hdrLoc = 10; - endOffset.offset = 8192; + printf("second time inserted %i values token\n", k); + int tempSize = 7; + rc = m_DctnryStore.updateDctnryStore(sgnature_value, tempSize, token2[k]); - for (k = 0; k < opCount; k++) - { - startOffset.hdrLoc = hdrOffsets[k].hdrLoc; - startOffset.offset = hdrOffsets[k].offset; - sigSize = endOffset.offset - startOffset.offset; - - printf(" OP %i signature size : %i \n from %i to %i \n", k + 1, - sigSize, startOffset.offset, endOffset.offset ); - endOffset.hdrLoc = startOffset.hdrLoc ; - endOffset.offset = startOffset.offset; - printf("%i ", endOffset.offset); - }//end for k - - //printf("%x \n", 0xFFFF); - }//endfor i - - m_Dctnry.closeDctnry(); - printf("\nSuccessfully Running testGetBlockHdr \n"); - return; - } - void testDctnryDropCreate() + if (rc != NO_ERROR) { - int rc = 0; - FID dctnryOID = 700; - FID treeOID = 101; - FID listOID = 102; - - m_DctnryStore.setDebugLevel( DEBUG_3 ); - - printf("\nRunning testDctnryDropCreate \n"); - - rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID); - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_DctnryStore.closeDctnryStore(); - - return; + printf("452. Predicted Error Code should be 1363: The result ERROR CODE : %i \n", rc); } - void testDctnryOpen() + CPPUNIT_ASSERT(rc == NO_ERROR); + + tempSize = 8; + rc = m_DctnryStore.updateDctnryStore(sgnature_value, tempSize, token2[k]); + + if (rc != NO_ERROR) { - int rc = 0; - FID dctnryOID = 700; - FID treeOID = 101; - FID listOID = 102; - - m_DctnryStore.setDebugLevel( DEBUG_3 ); - - printf("\nRunning testDctnryOpen \n"); - - rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_DctnryStore.closeDctnryStore(); - - return; + printf("461. Attention!!! ERROR CODE : %i \n", rc); } - void testDctnryInsertDeleteStore() + + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 1; i < 99; i++) { - int rc = 0; - int smallSize = 1000; - FID dctnryOID = 100; - FID treeOID = 101; - FID listOID = 102; - Token token2[100]; + printf("first time->i : %i token.fbo %i token.op %i \n", i, (int)token[i].fbo, (int)token[i].op); + } - m_DctnryStore.setDebugLevel( DEBUG_3 ); + cout << "finished printing the tokens" << endl; + cout << "start deleting the tokens" << endl; - printf("\nRunning testDctnryInsertDeleteStore \n"); - printf("dropDctnryStore\n"); - rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID); - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + for (i = 1; i < 99; i++) + { + cout << "start deleting the tokens i=" << i << endl; - unsigned char sgnature_value[smallSize]; - Token token[500]; - memset(token, 0, sizeof(token)); - int i, j, k; - j = 255; + rc = m_DctnryStore.deleteDctnryToken(token[i]); - for (k = 1; k < 499; k++) - { - for (i = 0; i < smallSize; i++) - { - if (j == 0) - j = 255; + if (rc != NO_ERROR) + { + printf("475 . Attention!!! ERROR CODE : %i \n", rc); + } - if (k > 10) - sgnature_value[i] = k; - else + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + cout << "finish deleting the tokens" << endl; + m_DctnryStore.closeDctnryStore(); + printf("\nSuccessfully Running testDctnryInsertDelete \n"); + return; + } // testDctnryInsertDeleteStore + void testDctnryInsertStressStore() + { + int rc = 0; + + FID dctnryOID = 897; + FID treeOID = 101; + FID listOID = 102; + int count = 5000; + Token stringToken; + string msg; + string timeStr; + + Token pToken; + int sigStringSize; + + m_DctnryStore.setDebugLevel(DEBUG_3); + m_DctnryStore.setUseHashMap(true); + + printf("\nRunning testDctnryInsertStressStore \n"); + + rc = m_DctnryStore.dropDctnryStore(dctnryOID, treeOID, listOID); + cout << "m_DctnryStore.dropDctnryStore error code=" << rc << " dctnryOID=" << dctnryOID + << " treeOID=" << treeOID << " listOID=" << listOID << endl; + rc = m_DctnryStore.createDctnryStore(dctnryOID, treeOID, listOID); + cout << "m_DctnryStore.createDctnryStore error code=" << rc << " dctnryOID=" << dctnryOID + << " treeOID=" << treeOID << " listOID=" << listOID << endl; + cout << " I am here " << endl; + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + char insertString[] = + "Hello, I am a string; what are you? I am testing dictionary as a signature value, I don't know how " + "long I am but we will find out in a minute"; + char sigString[500]; + memset(sigString, '\0', 500); + + for (int i = 0; i < count; i++) + { + sprintf(sigString, "%d%s", i, insertString); + // sprintf(sigString, "%s%d", insertString, i); + sigStringSize = strlen(sigString); + + m_DctnryStore.setAllTransId(10); + rc = m_DctnryStore.updateDctnryStore((unsigned char*)sigString, sigStringSize, stringToken); + cout << "stringToken token->fbo " << stringToken.fbo << " op->" << stringToken.op << endl; + } + + for (int i = 0; i < count; i++) + { + sprintf(sigString, "%d%s", i, insertString); + sigStringSize = strlen(sigString); + m_DctnryStore.setAllTransId(10); + rc = m_DctnryStore.updateDctnryStore((unsigned char*)sigString, sigStringSize, pToken); + cout << "pToken token->fbo " << pToken.fbo << " op->" << pToken.op << endl; + } + + m_DctnryStore.clearMap(); + m_DctnryStore.closeDctnryStore(); + printf("\nSuccessfully Running testDctnryInsertStressStore \n"); + return; + } // testDctnryInsertStressStore + + void testDctnryMultipleInsert() + { + int rc = 0; + FID dctnryOID = 200; + int totalsize = 1; + int size = 18; // 3,18 + int letterSize = 2; // 27,2 + int mapSize = 1000; + // char base[]={'b', 'b','c','d','e','f','g','h','i','j','k','l', + // 'm','n','o','p','q','r','s','t','u','v','w','x','y','z',' '}; + + m_DctnryStore.setDebugLevel(DEBUG_3); + + printf("\nRunning testDctnryMultipleInsert \n"); + rc = m_Dctnry.dropDctnry(dctnryOID); + rc = m_Dctnry.createDctnry(dctnryOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_Dctnry.openDctnry(dctnryOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (int i = 0; i < (size); i++) + { + totalsize = totalsize * letterSize; + } + + cout << "totalsize=" << totalsize << endl; + // There will not be a leak if the following throws an exception: + + int totalRow = totalsize; + char buf[size + 1]; + ColPosPair** pos = new ColPosPair*[totalRow]; + int loc = 0; + + // ColPosPair* pos[totalRow]; + for (int i = 0; i < totalRow; i++) + { + pos[i] = new ColPosPair[1]; + pos[i][0].start = loc; + pos[i][0].offset = size; + loc = loc + size + 1; + } + + int i = 0; + /* + while (i 10) - sgnature_value[i] = k; - else - { - if (i < 8) - sgnature_value[i] = 1; - else - sgnature_value[i] = k; - } - - j--; - }//endfor i ; second time - - rc = m_DctnryStore.updateDctnryStore(sgnature_value, - smallSize, token2[k]); - - if (rc != NO_ERROR) - { - printf("443. Attentione!!! ERROR CODE : %i \n", rc); - } - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } //endof for k; second time - - printf("second time inserted %i values token\n", k); - int tempSize = 7; - rc = m_DctnryStore.updateDctnryStore(sgnature_value, - tempSize, token2[k]); - - if (rc != NO_ERROR) - { - printf("452. Predicted Error Code should be 1363: The result ERROR CODE : %i \n", rc); - } - - CPPUNIT_ASSERT( rc == NO_ERROR ); - - tempSize = 8; - rc = m_DctnryStore.updateDctnryStore(sgnature_value, - tempSize, token2[k]); - - if (rc != NO_ERROR) - { - printf("461. Attention!!! ERROR CODE : %i \n", rc); - } - - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = 1; i < 99; i++) - { - printf("first time->i : %i token.fbo %i token.op %i \n", i, (int)token[i].fbo, - (int)token[i].op); - } - - cout << "finished printing the tokens" << endl; - cout << "start deleting the tokens" << endl; - - for (i = 1; i < 99; i++) - { - cout << "start deleting the tokens i=" << i << endl; - - rc = m_DctnryStore.deleteDctnryToken(token[i]); - - if (rc != NO_ERROR) - { - printf("475 . Attention!!! ERROR CODE : %i \n", rc); - } - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - cout << "finish deleting the tokens" << endl; - m_DctnryStore.closeDctnryStore(); - printf("\nSuccessfully Running testDctnryInsertDelete \n"); - return; - }//testDctnryInsertDeleteStore - void testDctnryInsertStressStore() + for (int i = 0; i < size; i++) { - int rc = 0; - - FID dctnryOID = 897; - FID treeOID = 101; - FID listOID = 102; - int count = 5000; - Token stringToken; - string msg; - string timeStr; - - Token pToken; - int sigStringSize; - - m_DctnryStore.setDebugLevel( DEBUG_3 ); - m_DctnryStore.setUseHashMap(true); - - printf("\nRunning testDctnryInsertStressStore \n"); - - rc = m_DctnryStore.dropDctnryStore( dctnryOID, treeOID, listOID); - cout << "m_DctnryStore.dropDctnryStore error code=" << rc << " dctnryOID=" << dctnryOID << " treeOID=" << treeOID << " listOID=" << listOID << endl; - rc = m_DctnryStore.createDctnryStore( dctnryOID, treeOID, listOID); - cout << "m_DctnryStore.createDctnryStore error code=" << rc << " dctnryOID=" << dctnryOID << " treeOID=" << treeOID << " listOID=" << listOID << endl; - cout << " I am here " << endl; - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_DctnryStore.openDctnryStore(dctnryOID, treeOID, listOID ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - char insertString[] = "Hello, I am a string; what are you? I am testing dictionary as a signature value, I don't know how long I am but we will find out in a minute"; - char sigString[500]; - memset(sigString, '\0', 500); - - for (int i = 0; i < count ; i++) - { - sprintf(sigString, "%d%s", i, insertString); - //sprintf(sigString, "%s%d", insertString, i); - sigStringSize = strlen(sigString); - - m_DctnryStore.setAllTransId(10); - rc = m_DctnryStore.updateDctnryStore((unsigned char*)sigString, - sigStringSize, - stringToken); - cout << "stringToken token->fbo " << stringToken.fbo << " op->" << stringToken.op << endl; - } - - for (int i = 0; i < count ; i++) - { - sprintf(sigString, "%d%s", i, insertString); - sigStringSize = strlen(sigString); - m_DctnryStore.setAllTransId(10); - rc = m_DctnryStore.updateDctnryStore((unsigned char*)sigString, - sigStringSize, - pToken); - cout << "pToken token->fbo " << pToken.fbo << " op->" << pToken.op << endl; - } - - m_DctnryStore.clearMap(); - m_DctnryStore.closeDctnryStore(); - printf("\nSuccessfully Running testDctnryInsertStressStore \n"); - return; - }//testDctnryInsertStressStore - - - - void testDctnryMultipleInsert() - { - int rc = 0; - FID dctnryOID = 200; - int totalsize = 1; - int size = 18;//3,18 - int letterSize = 2; //27,2 - int mapSize = 1000; - //char base[]={'b', 'b','c','d','e','f','g','h','i','j','k','l', - // 'm','n','o','p','q','r','s','t','u','v','w','x','y','z',' '}; - - - - m_DctnryStore.setDebugLevel( DEBUG_3 ); - - printf("\nRunning testDctnryMultipleInsert \n"); - rc = m_Dctnry.dropDctnry( dctnryOID); - rc = m_Dctnry.createDctnry( dctnryOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_Dctnry.openDctnry(dctnryOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (int i = 0; i < (size); i++) - { - totalsize = totalsize * letterSize; - } - - cout << "totalsize=" << totalsize << endl; - // There will not be a leak if the following throws an exception: - - int totalRow = totalsize; - char buf[size + 1]; - ColPosPair** pos = new ColPosPair*[totalRow]; - int loc = 0; - - //ColPosPair* pos[totalRow]; - for (int i = 0; i < totalRow; i++) - { - - pos[i] = new ColPosPair[1]; - pos[i][0].start = loc; - pos[i][0].offset = size; - loc = loc + size + 1; - } - - int i = 0; - /* - while (i::iterator it; - for (it=m_sigArray.begin(); it!=m_sigArray.end(); it++) - { - Signature sig = *it; - delete [] sig.signature; - sig.signature = 0; - } + std::set::iterator it; + for (it = m_sigArray.begin(); it != m_sigArray.end(); it++) + { + Signature sig = *it; + delete[] sig.signature; + sig.signature = 0; + } - m_arraySize = 0; - m_sigArray.clear(); + m_arraySize = 0; + m_sigArray.clear(); } /******************************************************************************* @@ -150,19 +147,19 @@ void Dctnry::freeStringCache( ) * success - successfully write the header to block * failure - it did not write the header to block ******************************************************************************/ -int Dctnry::init() +int Dctnry::init() { - //cout <<"Init called! m_dctnryOID =" << m_dctnryOID << endl; - m_lastFbo = 0; - m_hwm = 0; - m_newStartOffset = 0; - m_freeSpace = 0; - m_curOp = 0; - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - m_curBlock.lbid = INVALID_LBID; - m_arraySize = 0; + // cout <<"Init called! m_dctnryOID =" << m_dctnryOID << endl; + m_lastFbo = 0; + m_hwm = 0; + m_newStartOffset = 0; + m_freeSpace = 0; + m_curOp = 0; + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + m_curBlock.lbid = INVALID_LBID; + m_arraySize = 0; - return NO_ERROR; + return NO_ERROR; } /******************************************************************************* @@ -187,133 +184,122 @@ int Dctnry::init() * success - successfully created file and/or extent * failure - failed to create file and/or extent ******************************************************************************/ -int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, - const uint16_t dbRoot, const uint32_t partition, const uint16_t segment, - LBID_t& startLbid, bool flag) +int Dctnry::createDctnry(const OID& dctnryOID, int colWidth, const uint16_t dbRoot, const uint32_t partition, + const uint16_t segment, LBID_t& startLbid, bool flag) { - int allocSize = 0; - char fileName[FILE_NAME_SIZE]; - int rc; - std::map oids; + int allocSize = 0; + char fileName[FILE_NAME_SIZE]; + int rc; + std::map oids; #ifdef PROFILE - Stats::startParseEvent(WE_STATS_ALLOC_DCT_EXTENT); + Stats::startParseEvent(WE_STATS_ALLOC_DCT_EXTENT); #endif + if (flag) + { + // Allocate extent before file creation. + // If we got an error while allocating dictionary store extent, + // we do not need to create/close the file, because it was not created + // yet. This logic is the same as column segment file creation - at + // first we allocate an extent, then we create a segment file. + rc = BRMWrapper::getInstance()->allocateDictStoreExtent((OID)dctnryOID, dbRoot, partition, segment, + startLbid, allocSize); + + if (rc != NO_ERROR) + { + return rc; + } + + m_dctnryOID = dctnryOID; + m_partition = partition; + m_segment = segment; + m_dbRoot = dbRoot; + + RETURN_ON_ERROR((rc = oid2FileName(m_dctnryOID, fileName, true, m_dbRoot, m_partition, m_segment))); + m_segFileName = fileName; + + // if obsolete file exists, "w+b" will truncate and write over + m_dFile = createDctnryFile(fileName, colWidth, "w+b", DEFAULT_BUFSIZ, startLbid); + + { + // We presume the path will contain / + std::string filePath(fileName); + if (chownDataPath(filePath)) + { + return ERR_FILE_CHOWN; + } + } + } + else + { + rc = BRMWrapper::getInstance()->allocateDictStoreExtent((OID)m_dctnryOID, m_dbRoot, m_partition, + m_segment, startLbid, allocSize); + + if (rc != NO_ERROR) + { + return rc; + } + + RETURN_ON_ERROR(setFileOffset(m_dFile, 0, SEEK_END)); + } + + // We allocate a full extent from BRM, but only write an abbreviated 256K + // rows to disk for 1st extent in each store file, to conserve disk usage. + int totalSize = allocSize; + + if (flag) + { + totalSize = NUM_BLOCKS_PER_INITIAL_EXTENT; + } + + if (!isDiskSpaceAvail(Config::getDBRootByNum(m_dbRoot), totalSize)) + { if (flag) { - // Allocate extent before file creation. - // If we got an error while allocating dictionary store extent, - // we do not need to create/close the file, because it was not created - // yet. This logic is the same as column segment file creation - at - // first we allocate an extent, then we create a segment file. - rc = BRMWrapper::getInstance()->allocateDictStoreExtent( - (OID) dctnryOID, dbRoot, partition, segment, startLbid, allocSize); - - if (rc != NO_ERROR) - { - return rc; - } - - m_dctnryOID = dctnryOID; - m_partition = partition; - m_segment = segment; - m_dbRoot = dbRoot; - - RETURN_ON_ERROR( ( rc = oid2FileName( m_dctnryOID, fileName, true, - m_dbRoot, m_partition, m_segment ) ) ); - m_segFileName = fileName; - - // if obsolete file exists, "w+b" will truncate and write over - m_dFile = createDctnryFile(fileName, colWidth, "w+b", DEFAULT_BUFSIZ, - startLbid); - - { - // We presume the path will contain / - std::string filePath(fileName); - if (chownDataPath(filePath)) - { - return ERR_FILE_CHOWN; - } - } - } - else - { - rc = BRMWrapper::getInstance()->allocateDictStoreExtent( - (OID) m_dctnryOID, m_dbRoot, m_partition, m_segment, startLbid, - allocSize); - - if (rc != NO_ERROR) - { - return rc; - } - - RETURN_ON_ERROR(setFileOffset(m_dFile, 0, SEEK_END)); - + closeDctnryFile(false, oids); } - // We allocate a full extent from BRM, but only write an abbreviated 256K - // rows to disk for 1st extent in each store file, to conserve disk usage. - int totalSize = allocSize; - - if (flag) - { - totalSize = NUM_BLOCKS_PER_INITIAL_EXTENT; - } - - if ( !isDiskSpaceAvail(Config::getDBRootByNum(m_dbRoot), totalSize) ) - { - if (flag) - { - closeDctnryFile(false, oids); - } - - return ERR_FILE_DISK_SPACE; - } + return ERR_FILE_DISK_SPACE; + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_ALLOC_DCT_EXTENT); + Stats::stopParseEvent(WE_STATS_ALLOC_DCT_EXTENT); #endif - if ( m_dFile != NULL ) + if (m_dFile != NULL) + { + // MCOL-498 CS optimizes abbreviated extent + // creation. + rc = FileOp::initDctnryExtent(m_dFile, m_dbRoot, totalSize, m_dctnryHeader2, m_totalHdrBytes, false, + true, // explicitly optimize + startLbid); + if (rc != NO_ERROR) { - // MCOL-498 CS optimizes abbreviated extent - // creation. - rc = FileOp::initDctnryExtent( m_dFile, - m_dbRoot, - totalSize, - m_dctnryHeader2, - m_totalHdrBytes, - false, - true, // explicitly optimize - startLbid ); - if (rc != NO_ERROR) - { - if (flag) - { - closeDctnryFile(false, oids); - } + if (flag) + { + closeDctnryFile(false, oids); + } - return rc; - } + return rc; } - else - return ERR_FILE_CREATE; + } + else + return ERR_FILE_CREATE; - if (flag) - { - closeDctnryFile(true, oids); - m_numBlocks = totalSize; - m_hwm = 0; - rc = BRMWrapper::getInstance()->setLocalHWM( - m_dctnryOID, m_partition, m_segment, m_hwm); - } - else - { - m_numBlocks = m_numBlocks + totalSize; - } + if (flag) + { + closeDctnryFile(true, oids); + m_numBlocks = totalSize; + m_hwm = 0; + rc = BRMWrapper::getInstance()->setLocalHWM(m_dctnryOID, m_partition, m_segment, m_hwm); + } + else + { + m_numBlocks = m_numBlocks + totalSize; + } - return rc; + return rc; } /******************************************************************************* @@ -328,42 +314,37 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, * success - successfully expanded extent * failure - failed to expand extent ******************************************************************************/ -int Dctnry::expandDctnryExtent() +int Dctnry::expandDctnryExtent() { - RETURN_ON_NULL( m_dFile, ERR_FILE_SEEK ); + RETURN_ON_NULL(m_dFile, ERR_FILE_SEEK); - off64_t oldOffset = m_dFile->tell(); + off64_t oldOffset = m_dFile->tell(); - RETURN_ON_ERROR( setFileOffset(m_dFile, 0, SEEK_END) ); + RETURN_ON_ERROR(setFileOffset(m_dFile, 0, SEEK_END)); - // Based on extent size, see how many blocks to add to fill the extent - int blksToAdd = ( ((int)BRMWrapper::getInstance()->getExtentRows() - - INITIAL_EXTENT_ROWS_TO_DISK) / BYTE_PER_BLOCK ) * PSEUDO_COL_WIDTH; + // Based on extent size, see how many blocks to add to fill the extent + int blksToAdd = + (((int)BRMWrapper::getInstance()->getExtentRows() - INITIAL_EXTENT_ROWS_TO_DISK) / BYTE_PER_BLOCK) * + PSEUDO_COL_WIDTH; - if ( !isDiskSpaceAvail(Config::getDBRootByNum(m_dbRoot), blksToAdd) ) - { - return ERR_FILE_DISK_SPACE; - } + if (!isDiskSpaceAvail(Config::getDBRootByNum(m_dbRoot), blksToAdd)) + { + return ERR_FILE_DISK_SPACE; + } - int rc = FileOp::initDctnryExtent( m_dFile, - m_dbRoot, - blksToAdd, - m_dctnryHeader2, - m_totalHdrBytes, - true, - true ); // explicitly optimize - - if (rc != NO_ERROR) - return rc; - - - // Restore offset back to where we were before expanding the extent - RETURN_ON_ERROR( setFileOffset(m_dFile, oldOffset, SEEK_SET) ); - - // Update block count to reflect disk space added by expanding the extent. - m_numBlocks = m_numBlocks + blksToAdd; + int rc = FileOp::initDctnryExtent(m_dFile, m_dbRoot, blksToAdd, m_dctnryHeader2, m_totalHdrBytes, true, + true); // explicitly optimize + if (rc != NO_ERROR) return rc; + + // Restore offset back to where we were before expanding the extent + RETURN_ON_ERROR(setFileOffset(m_dFile, oldOffset, SEEK_SET)); + + // Update block count to reflect disk space added by expanding the extent. + m_numBlocks = m_numBlocks + blksToAdd; + + return rc; } /******************************************************************************* @@ -378,71 +359,69 @@ int Dctnry::expandDctnryExtent() ******************************************************************************/ int Dctnry::closeDctnry(bool realClose) { - if ( !m_dFile ) - return NO_ERROR; + if (!m_dFile) + return NO_ERROR; - int rc; - CommBlock cb; - cb.file.oid = m_dctnryOID; - cb.file.pFile = m_dFile; - std::map oids; + int rc; + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = m_dFile; + std::map oids; - if (m_curBlock.state == BLK_WRITE) - { - rc = writeDBFile(cb, &m_curBlock, m_curBlock.lbid); - - if (rc != NO_ERROR) - { - closeDctnryFile(false, oids); - return rc; - } - - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - // m_curBlock.state== BLK_INIT; - } - - //@Bug 5572. always close file for uncompressed file. - if (FileOp::compressionType() == 0) - realClose = true; - - if (realClose) - { - //@Bug 5689. Need pass oid to write to the right file. - oids[m_dctnryOID] = m_dctnryOID; - // dmc-error handling (should detect/report error in closing file) - closeDctnryFile(true, oids); - - } - - m_hwm = (HWM)m_lastFbo; - idbassert(utils::is_nonnegative(m_dctnryOID)); - - if (idbdatafile::IDBPolicy::useHdfs() && realClose) - { - BRM::FileInfo aFile; - std::vector oidsToFlush; - oidsToFlush.push_back(m_dctnryOID); - aFile.oid = m_dctnryOID; - aFile.partitionNum = m_partition; - aFile.segmentNum = m_segment; - aFile.dbRoot = m_dbRoot; - aFile.compType = FileOp::compressionType(); - std::vector aFileInfo; - aFileInfo.push_back(aFile); - cacheutils::purgePrimProcFdCache(aFileInfo, Config::getLocalModuleID()); - cacheutils::flushOIDsFromCache(oidsToFlush); - } - - rc = BRMWrapper::getInstance()->setLocalHWM( - m_dctnryOID, m_partition, m_segment, m_hwm); + if (m_curBlock.state == BLK_WRITE) + { + rc = writeDBFile(cb, &m_curBlock, m_curBlock.lbid); if (rc != NO_ERROR) - return rc; + { + closeDctnryFile(false, oids); + return rc; + } - //cout <<"Init called! m_dctnryOID =" << m_dctnryOID << endl; - freeStringCache( ); + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + // m_curBlock.state== BLK_INIT; + } - return NO_ERROR; + //@Bug 5572. always close file for uncompressed file. + if (FileOp::compressionType() == 0) + realClose = true; + + if (realClose) + { + //@Bug 5689. Need pass oid to write to the right file. + oids[m_dctnryOID] = m_dctnryOID; + // dmc-error handling (should detect/report error in closing file) + closeDctnryFile(true, oids); + } + + m_hwm = (HWM)m_lastFbo; + idbassert(utils::is_nonnegative(m_dctnryOID)); + + if (idbdatafile::IDBPolicy::useHdfs() && realClose) + { + BRM::FileInfo aFile; + std::vector oidsToFlush; + oidsToFlush.push_back(m_dctnryOID); + aFile.oid = m_dctnryOID; + aFile.partitionNum = m_partition; + aFile.segmentNum = m_segment; + aFile.dbRoot = m_dbRoot; + aFile.compType = FileOp::compressionType(); + std::vector aFileInfo; + aFileInfo.push_back(aFile); + cacheutils::purgePrimProcFdCache(aFileInfo, Config::getLocalModuleID()); + cacheutils::flushOIDsFromCache(oidsToFlush); + } + + rc = BRMWrapper::getInstance()->setLocalHWM(m_dctnryOID, m_partition, m_segment, m_hwm); + + if (rc != NO_ERROR) + return rc; + + // cout <<"Init called! m_dctnryOID =" << m_dctnryOID << endl; + freeStringCache(); + + return NO_ERROR; } /******************************************************************************* @@ -456,18 +435,18 @@ int Dctnry::closeDctnry(bool realClose) * RETURN: * none ******************************************************************************/ -int Dctnry::closeDctnryOnly( ) +int Dctnry::closeDctnryOnly() { - if ( !m_dFile ) - return NO_ERROR; - - // dmc-error handling (should detect/report error in closing file) - std::map oids; - closeDctnryFile(false, oids); - - freeStringCache( ); - + if (!m_dFile) return NO_ERROR; + + // dmc-error handling (should detect/report error in closing file) + std::map oids; + closeDctnryFile(false, oids); + + freeStringCache(); + + return NO_ERROR; } /******************************************************************************* @@ -480,16 +459,16 @@ int Dctnry::closeDctnryOnly( ) * RETURN: * none ******************************************************************************/ -int Dctnry::dropDctnry( const OID& dctnryOID) +int Dctnry::dropDctnry(const OID& dctnryOID) { - m_dctnryOID = dctnryOID; + m_dctnryOID = dctnryOID; - if (m_dFile) - { - RETURN_ON_ERROR( closeDctnry() ); - } + if (m_dFile) + { + RETURN_ON_ERROR(closeDctnry()); + } - return deleteFile( dctnryOID); + return deleteFile(dctnryOID); } /******************************************************************************* @@ -508,126 +487,119 @@ int Dctnry::dropDctnry( const OID& dctnryOID) * Fail - Error Code ******************************************************************************/ // @bug 5572 - HDFS usage: add *.tmp file backup flag -int Dctnry::openDctnry(const OID& dctnryOID, - const uint16_t dbRoot, - const uint32_t partition, - const uint16_t segment, - const bool useTmpSuffix) +int Dctnry::openDctnry(const OID& dctnryOID, const uint16_t dbRoot, const uint32_t partition, + const uint16_t segment, const bool useTmpSuffix) { #ifdef PROFILE - Stats::startParseEvent(WE_STATS_OPEN_DCT_FILE); + Stats::startParseEvent(WE_STATS_OPEN_DCT_FILE); #endif - int rc = NO_ERROR; - m_dctnryOID = dctnryOID; - m_dbRoot = dbRoot; - m_partition = partition; - m_segment = segment; + int rc = NO_ERROR; + m_dctnryOID = dctnryOID; + m_dbRoot = dbRoot; + m_partition = partition; + m_segment = segment; - m_dFile = openDctnryFile(useTmpSuffix); + m_dFile = openDctnryFile(useTmpSuffix); - if ( m_dFile == NULL ) - { - ostringstream oss; - oss << "oid:partition:segment " << - dctnryOID << ":" << partition << ":" << segment; - logging::Message::Args args; - logging::Message message(1); - args.add("Error opening dictionary file "); - args.add(oss.str()); - args.add(""); - args.add(""); - message.format(args); - logging::LoggingID lid(21); - logging::MessageLog ml(lid); + if (m_dFile == NULL) + { + ostringstream oss; + oss << "oid:partition:segment " << dctnryOID << ":" << partition << ":" << segment; + logging::Message::Args args; + logging::Message message(1); + args.add("Error opening dictionary file "); + args.add(oss.str()); + args.add(""); + args.add(""); + message.format(args); + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - ml.logErrorMessage( message ); - return ERR_FILE_OPEN; - } + ml.logErrorMessage(message); + return ERR_FILE_OPEN; + } - m_numBlocks = numOfBlocksInFile(); - std::map oids; + m_numBlocks = numOfBlocksInFile(); + std::map oids; - //Initialize other misc member variables - init(); + // Initialize other misc member variables + init(); - int extState; - rc = BRMWrapper::getInstance()->getLocalHWM(dctnryOID, - m_partition, m_segment, m_hwm, extState); - - if (rc != NO_ERROR) - { - closeDctnryFile(false, oids); - return rc; - } - - m_lastFbo = (int)m_hwm; - - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - m_curFbo = m_lastFbo; - rc = BRMWrapper::getInstance()->getBrmInfo( m_dctnryOID, - m_partition, m_segment, - m_curFbo, m_curLbid); - - if (rc != NO_ERROR) - { - closeDctnryFile(false, oids); - return rc; - } - - CommBlock cb; - cb.file.oid = m_dctnryOID; - cb.file.pFile = m_dFile; -#ifdef PROFILE - // We omit the call to readDBFile from OPEN_DCT_FILE stats, because com- - // pressed files have separate stats that readDBFile() will capture thru - // ChunkManager::fetchChunkFromFile(). - Stats::stopParseEvent(WE_STATS_OPEN_DCT_FILE); -#endif - rc = readDBFile(cb, m_curBlock.data, m_curLbid); -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_OPEN_DCT_FILE); -#endif - - if (rc != NO_ERROR) - { - closeDctnryFile(false, oids); - return rc; - } - - //@Bug 5567 Don't seek for compressed file. - if (m_compressionType == 0) - { - // Position file to the start of the current block; - // Determine file byte offset based on the current block offset (m_curFbo) - long long byteOffset = ((long long)m_curFbo) * (long)BYTE_PER_BLOCK; - rc = setFileOffset(m_dFile, byteOffset); - - if (rc != NO_ERROR) - { - closeDctnryFile(false, oids); - return rc; - } - } - - m_curBlock.lbid = m_curLbid; - m_curBlock.state = BLK_READ; - int opCnt = 0; - // Get new free space (m_freeSpace) from header too! Here!!!!!!!!!!!!!!! - getBlockOpCount( m_curBlock, opCnt); - m_curOp = opCnt; - - // "If" this store file contains no more than 1 block, then we preload - // the string cache used to recognize duplicates during row insertion. - if (m_hwm == 0) - { - preLoadStringCache( m_curBlock ); - } - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_OPEN_DCT_FILE); -#endif + int extState; + rc = BRMWrapper::getInstance()->getLocalHWM(dctnryOID, m_partition, m_segment, m_hwm, extState); + if (rc != NO_ERROR) + { + closeDctnryFile(false, oids); return rc; + } + + m_lastFbo = (int)m_hwm; + + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + m_curFbo = m_lastFbo; + rc = BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID, m_partition, m_segment, m_curFbo, m_curLbid); + + if (rc != NO_ERROR) + { + closeDctnryFile(false, oids); + return rc; + } + + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = m_dFile; +#ifdef PROFILE + // We omit the call to readDBFile from OPEN_DCT_FILE stats, because com- + // pressed files have separate stats that readDBFile() will capture thru + // ChunkManager::fetchChunkFromFile(). + Stats::stopParseEvent(WE_STATS_OPEN_DCT_FILE); +#endif + rc = readDBFile(cb, m_curBlock.data, m_curLbid); +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_OPEN_DCT_FILE); +#endif + + if (rc != NO_ERROR) + { + closeDctnryFile(false, oids); + return rc; + } + + //@Bug 5567 Don't seek for compressed file. + if (m_compressionType == 0) + { + // Position file to the start of the current block; + // Determine file byte offset based on the current block offset (m_curFbo) + long long byteOffset = ((long long)m_curFbo) * (long)BYTE_PER_BLOCK; + rc = setFileOffset(m_dFile, byteOffset); + + if (rc != NO_ERROR) + { + closeDctnryFile(false, oids); + return rc; + } + } + + m_curBlock.lbid = m_curLbid; + m_curBlock.state = BLK_READ; + int opCnt = 0; + // Get new free space (m_freeSpace) from header too! Here!!!!!!!!!!!!!!! + getBlockOpCount(m_curBlock, opCnt); + m_curOp = opCnt; + + // "If" this store file contains no more than 1 block, then we preload + // the string cache used to recognize duplicates during row insertion. + if (m_hwm == 0) + { + preLoadStringCache(m_curBlock); + } + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_OPEN_DCT_FILE); +#endif + + return rc; } /******************************************************************************* @@ -644,17 +616,20 @@ int Dctnry::openDctnry(const OID& dctnryOID, ******************************************************************************/ bool Dctnry::getTokenFromArray(Signature& sig) { - std::set::iterator it; - it = m_sigArray.find(sig); - if ( it == m_sigArray.end()){ - return false; - }else{ - Signature sigfound = *it; - sig.token = sigfound.token; - return true; - } + std::set::iterator it; + it = m_sigArray.find(sig); + if (it == m_sigArray.end()) + { + return false; + } + else + { + Signature sigfound = *it; + sig.token = sigfound.token; + return true; + } - return false; + return false; } /******************************************************************************* @@ -674,108 +649,100 @@ bool Dctnry::getTokenFromArray(Signature& sig) ******************************************************************************/ int Dctnry::insertDctnry2(Signature& sig) { - int rc = 0; - int write_size; - bool lbid_in_token = false; - size_t origSigSize = sig.size; - unsigned char* origSig = sig.signature; + int rc = 0; + int write_size; + bool lbid_in_token = false; + size_t origSigSize = sig.size; + unsigned char* origSig = sig.signature; - sig.token.bc = 0; + sig.token.bc = 0; - while (sig.size > 0) + while (sig.size > 0) + { + if (sig.size > (m_freeSpace - HDR_UNIT_SIZE)) { - if (sig.size > (m_freeSpace - HDR_UNIT_SIZE)) - { - write_size = (m_freeSpace - HDR_UNIT_SIZE); - } - else - { - write_size = sig.size; - } - - insertDctnryHdr(m_curBlock.data, write_size); - insertSgnture(m_curBlock.data, write_size, (unsigned char*)sig.signature); - - sig.size -= write_size; - sig.signature += write_size; - m_curFbo = m_lastFbo; - - if (!lbid_in_token) - { - sig.token.fbo = m_curLbid; - sig.token.op = m_curOp; - lbid_in_token = true; - } - - if (sig.size > 0) - { - CommBlock cb; - cb.file.oid = m_dctnryOID; - cb.file.pFile = m_dFile; - sig.token.bc++; - - RETURN_ON_ERROR( writeDBFileNoVBCache(cb, &m_curBlock, m_curFbo) ); - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - memcpy( m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); - m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; - m_curBlock.state = BLK_WRITE; - m_curOp = 0; - m_lastFbo++; - m_curFbo = m_lastFbo; - - //...Expand current extent if it is an abbreviated initial extent - if ((m_curFbo == m_numBlocks) && - (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) - { - RETURN_ON_ERROR( expandDctnryExtent() ); - } - - //...Allocate a new extent if we have reached the last block in the - // current extent. - if (m_curFbo == m_numBlocks) - { - //last block - //for roll back the extent to use - //Save those empty extents in case of failure to rollback - std::vector dictExtentInfo; - ExtentInfo info; - info.oid = m_dctnryOID; - info.partitionNum = m_partition; - info.segmentNum = m_segment; - info.dbRoot = m_dbRoot; - info.hwm = m_hwm; - info.newFile = false; - dictExtentInfo.push_back (info); - LBID_t startLbid; - // Add an extent. - rc = createDctnry(m_dctnryOID, - 0, // dummy column width - m_dbRoot, - m_partition, - m_segment, - startLbid, - false) ; - - if ( rc != NO_ERROR ) - { - //roll back the extent - BRMWrapper::getInstance()->deleteEmptyDictStoreExtents( - dictExtentInfo); - return rc; - } - } - - RETURN_ON_ERROR( BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID, - m_partition, m_segment, - m_curFbo, m_curLbid) ); - m_curBlock.lbid = m_curLbid; - - } + write_size = (m_freeSpace - HDR_UNIT_SIZE); + } + else + { + write_size = sig.size; } - sig.size = origSigSize; - sig.signature = origSig; - return NO_ERROR; + insertDctnryHdr(m_curBlock.data, write_size); + insertSgnture(m_curBlock.data, write_size, (unsigned char*)sig.signature); + + sig.size -= write_size; + sig.signature += write_size; + m_curFbo = m_lastFbo; + + if (!lbid_in_token) + { + sig.token.fbo = m_curLbid; + sig.token.op = m_curOp; + lbid_in_token = true; + } + + if (sig.size > 0) + { + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = m_dFile; + sig.token.bc++; + + RETURN_ON_ERROR(writeDBFileNoVBCache(cb, &m_curBlock, m_curFbo)); + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + memcpy(m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); + m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; + m_curBlock.state = BLK_WRITE; + m_curOp = 0; + m_lastFbo++; + m_curFbo = m_lastFbo; + + //...Expand current extent if it is an abbreviated initial extent + if ((m_curFbo == m_numBlocks) && (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) + { + RETURN_ON_ERROR(expandDctnryExtent()); + } + + //...Allocate a new extent if we have reached the last block in the + // current extent. + if (m_curFbo == m_numBlocks) + { + // last block + // for roll back the extent to use + // Save those empty extents in case of failure to rollback + std::vector dictExtentInfo; + ExtentInfo info; + info.oid = m_dctnryOID; + info.partitionNum = m_partition; + info.segmentNum = m_segment; + info.dbRoot = m_dbRoot; + info.hwm = m_hwm; + info.newFile = false; + dictExtentInfo.push_back(info); + LBID_t startLbid; + // Add an extent. + rc = createDctnry(m_dctnryOID, + 0, // dummy column width + m_dbRoot, m_partition, m_segment, startLbid, false); + + if (rc != NO_ERROR) + { + // roll back the extent + BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(dictExtentInfo); + return rc; + } + } + + RETURN_ON_ERROR( + BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID, m_partition, m_segment, m_curFbo, m_curLbid)); + m_curBlock.lbid = m_curLbid; + } + } + + sig.size = origSigSize; + sig.signature = origSig; + return NO_ERROR; } /******************************************************************************* @@ -796,272 +763,254 @@ int Dctnry::insertDctnry2(Signature& sig) * success - successfully write the header to block * failure - it did not write the header to block ******************************************************************************/ -int Dctnry::insertDctnry(const char* buf, - ColPosPair** pos, - const int totalRow, const int col, - char* tokenBuf, +int Dctnry::insertDctnry(const char* buf, ColPosPair** pos, const int totalRow, const int col, char* tokenBuf, long long& truncCount) { #ifdef PROFILE - Stats::startParseEvent(WE_STATS_PARSE_DCT); + Stats::startParseEvent(WE_STATS_PARSE_DCT); #endif - int startPos = 0; - int totalUseSize = 0; + int startPos = 0; + int totalUseSize = 0; - int outOffset = 0; - const char* pIn; - char* pOut = tokenBuf; - Signature curSig; - bool found = false; - bool next = false; - CommBlock cb; - cb.file.oid = m_dctnryOID; - cb.file.pFile = m_dFile; - WriteEngine::Token nullToken; + int outOffset = 0; + const char* pIn; + char* pOut = tokenBuf; + Signature curSig; + bool found = false; + bool next = false; + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = m_dFile; + WriteEngine::Token nullToken; - //...Loop through all the rows for the specified column - while (startPos < totalRow) + //...Loop through all the rows for the specified column + while (startPos < totalRow) + { + found = false; + void* curSigPtr = static_cast(&curSig); + memset(curSigPtr, 0, sizeof(curSig)); + curSig.size = pos[startPos][col].offset; + + // Strip trailing null bytes '\0' (by adjusting curSig.size) if import- + // ing in binary mode. If entire string is binary zeros, then we treat + // as a NULL value. + if (m_importDataMode != IMPORT_DATA_TEXT) { - found = false; - void *curSigPtr = static_cast(&curSig); - memset(curSigPtr, 0, sizeof(curSig)); - curSig.size = pos[startPos][col].offset; + if ((curSig.size > 0) && (curSig.size != COLPOSPAIR_NULL_TOKEN_OFFSET)) + { + char* fld = (char*)buf + pos[startPos][col].start; + int kk = curSig.size - 1; - // Strip trailing null bytes '\0' (by adjusting curSig.size) if import- - // ing in binary mode. If entire string is binary zeros, then we treat - // as a NULL value. - if (m_importDataMode != IMPORT_DATA_TEXT) + for (; kk >= 0; kk--) { - if ((curSig.size > 0) && - (curSig.size != COLPOSPAIR_NULL_TOKEN_OFFSET)) - { - char* fld = (char*)buf + pos[startPos][col].start; - int kk = curSig.size - 1; - - for (; kk >= 0; kk--) - { - if (fld[kk] != '\0') - break; - } - - curSig.size = kk + 1; - } + if (fld[kk] != '\0') + break; } - // Read thread should validate against max size so that the entire row - // can be rejected up front. Once we get here in the parsing thread, - // it is too late to reject the row. However, as a precaution, we - // still check against max size & set to null token if needed. - if ((curSig.size == 0) || - (curSig.size == COLPOSPAIR_NULL_TOKEN_OFFSET) || - (curSig.size > MAX_BLOB_SIZE)) + curSig.size = kk + 1; + } + } + + // Read thread should validate against max size so that the entire row + // can be rejected up front. Once we get here in the parsing thread, + // it is too late to reject the row. However, as a precaution, we + // still check against max size & set to null token if needed. + if ((curSig.size == 0) || (curSig.size == COLPOSPAIR_NULL_TOKEN_OFFSET) || (curSig.size > MAX_BLOB_SIZE)) + { + if (m_defVal.length() > 0) // use default string if available + { + pIn = m_defVal.c_str(); + curSig.signature = (unsigned char*)pIn; + curSig.size = m_defVal.length(); + } + else + { + memcpy(pOut + outOffset, &nullToken, 8); + outOffset += 8; + startPos++; + continue; + } + } + else + { + pIn = (char*)buf + pos[startPos][col].start; + curSig.signature = (unsigned char*)pIn; + } + + // @Bug 2565: Truncate any strings longer than schema's column width + if (curSig.size > m_colWidth) + { + uint8_t truncate_point = utf8::utf8_truncate_point((const char*)curSig.signature, m_colWidth); + curSig.size = m_colWidth - truncate_point; + ++truncCount; + } + + //...Search for the string in our string cache + // if it fits into one block (< 8KB) + if (curSig.size <= MAX_SIGNATURE_SIZE) + { + // Stats::startParseEvent("getTokenFromArray"); + found = getTokenFromArray(curSig); + + if (found) + { + memcpy(pOut + outOffset, &curSig.token, 8); + outOffset += 8; + startPos++; + // Stats::stopParseEvent("getTokenFromArray"); + continue; + } + + // Stats::stopParseEvent("getTokenFromArray"); + } + + totalUseSize = m_totalHdrBytes + curSig.size; + + //...String not found in cache, so proceed. + // If room is available in current block then insert into block. + // @bug 3960: Add MAX_OP_COUNT check to handle case after bulk rollback + if (((totalUseSize <= m_freeSpace - HDR_UNIT_SIZE) || + ((curSig.size > 8176) && (m_freeSpace > HDR_UNIT_SIZE))) && + (m_curOp < (MAX_OP_COUNT - 1))) + { + RETURN_ON_ERROR(insertDctnry2(curSig)); // m_freeSpace updated! + m_curBlock.state = BLK_WRITE; + memcpy(pOut + outOffset, &curSig.token, 8); + outOffset += 8; + startPos++; + found = true; + + //...If we have reached limit for the number of strings allowed in + // a block, then we write the current block so that we can start + // another block. + if (m_curOp >= MAX_OP_COUNT - 1) + { +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_PARSE_DCT); +#endif + RETURN_ON_ERROR(writeDBFileNoVBCache(cb, &m_curBlock, m_curFbo)); + m_curBlock.state = BLK_READ; + next = true; + } + + //...Add string to cache, if we have not exceeded cache limit + // Don't cache big blobs + if ((m_arraySize < MAX_STRING_CACHE_SIZE) && (curSig.size <= MAX_SIGNATURE_SIZE)) + { + addToStringCache(curSig); + } + } + else //...No room for this string in current block, so we write + // out the current block, so we can start another block + { +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_PARSE_DCT); +#endif + RETURN_ON_ERROR(writeDBFileNoVBCache(cb, &m_curBlock, m_curFbo)); + m_curBlock.state = BLK_READ; + next = true; + found = false; + } // if m_freeSpace + + //..."next" flag is used to indicate that we need to advance to the + // next block in the store file. + if (next) + { + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + memcpy(m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); + m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; + m_curBlock.state = BLK_WRITE; + m_curOp = 0; + next = false; + m_lastFbo++; + m_curFbo = m_lastFbo; + + //...Expand current extent if it is an abbreviated initial extent + if ((m_curFbo == m_numBlocks) && (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) + { + RETURN_ON_ERROR(expandDctnryExtent()); + } + + //...Allocate a new extent if we have reached the last block in the + // current extent. + if (m_curFbo == m_numBlocks) + { + // last block + LBID_t startLbid; + + // Add an extent. + RETURN_ON_ERROR( + createDctnry(m_dctnryOID, m_colWidth, m_dbRoot, m_partition, m_segment, startLbid, false)); + + if (m_logger) { - if (m_defVal.length() > 0) // use default string if available - { - pIn = m_defVal.c_str(); - curSig.signature = (unsigned char*)pIn; - curSig.size = m_defVal.length(); - } - else - { - memcpy( pOut + outOffset, &nullToken, 8 ); - outOffset += 8; - startPos++; - continue; - } - } - else - { - pIn = (char*)buf + pos[startPos][col].start; - curSig.signature = (unsigned char*)pIn; + std::ostringstream oss; + oss << "Add dictionary extent OID-" << m_dctnryOID << "; DBRoot-" << m_dbRoot << "; part-" + << m_partition << "; seg-" << m_segment << "; hwm-" << m_curFbo << "; LBID-" << startLbid + << "; file-" << m_segFileName; + m_logger->logMsg(oss.str(), MSGLVL_INFO2); } - // @Bug 2565: Truncate any strings longer than schema's column width - if (curSig.size > m_colWidth) + m_curLbid = startLbid; + + // now seek back to the curFbo, after adding an extent + // @bug5769 For uncompressed only; + // ChunkManager manages the file offset for the compression case + if (m_compressionType == 0) { - uint8_t truncate_point = utf8::utf8_truncate_point((const char*)curSig.signature, m_colWidth); - curSig.size = m_colWidth - truncate_point; - ++truncCount; +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK); +#endif + long long byteOffset = m_curFbo; + byteOffset *= BYTE_PER_BLOCK; + RETURN_ON_ERROR(setFileOffset(m_dFile, byteOffset)); +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK); +#endif } + } + else + { + // LBIDs are numbered collectively and consecutively within an + // extent, so within an extent we can derive the LBID by simply + // incrementing it rather than having to go back to BRM to look + // up the LBID for each FBO. + m_curLbid++; + } - //...Search for the string in our string cache - //if it fits into one block (< 8KB) - if (curSig.size <= MAX_SIGNATURE_SIZE) +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_PARSE_DCT); +#endif + m_curBlock.lbid = m_curLbid; + + //..."found" flag indicates whether the string was already found + // "or" added to the end of the previous block. If false, then + // we need to add the string to the new block. + if (!found) + { + RETURN_ON_ERROR(insertDctnry2(curSig)); // m_freeSpace updated! + m_curBlock.state = BLK_WRITE; + memcpy(pOut + outOffset, &curSig.token, 8); + outOffset += 8; + startPos++; + + //...Add string to cache, if we have not exceeded cache limit + if ((m_arraySize < MAX_STRING_CACHE_SIZE) && (curSig.size <= MAX_SIGNATURE_SIZE)) { - //Stats::startParseEvent("getTokenFromArray"); - found = getTokenFromArray(curSig); - - if (found) - { - memcpy( pOut + outOffset, &curSig.token, 8 ); - outOffset += 8; - startPos++; - //Stats::stopParseEvent("getTokenFromArray"); - continue; - } - - //Stats::stopParseEvent("getTokenFromArray"); + addToStringCache(curSig); } - - totalUseSize = m_totalHdrBytes + curSig.size; - - //...String not found in cache, so proceed. - // If room is available in current block then insert into block. - // @bug 3960: Add MAX_OP_COUNT check to handle case after bulk rollback - if ( ((totalUseSize <= m_freeSpace - HDR_UNIT_SIZE) || - ((curSig.size > 8176) && (m_freeSpace > HDR_UNIT_SIZE))) && - (m_curOp < (MAX_OP_COUNT - 1)) ) - { - RETURN_ON_ERROR(insertDctnry2(curSig)); //m_freeSpace updated! - m_curBlock.state = BLK_WRITE; - memcpy( pOut + outOffset, &curSig.token, 8 ); - outOffset += 8; - startPos++; - found = true; - - //...If we have reached limit for the number of strings allowed in - // a block, then we write the current block so that we can start - // another block. - if (m_curOp >= MAX_OP_COUNT - 1) - { -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_PARSE_DCT); -#endif - RETURN_ON_ERROR(writeDBFileNoVBCache(cb, &m_curBlock, m_curFbo)); - m_curBlock.state = BLK_READ; - next = true; - } - - //...Add string to cache, if we have not exceeded cache limit - // Don't cache big blobs - if ((m_arraySize < MAX_STRING_CACHE_SIZE) && - (curSig.size <= MAX_SIGNATURE_SIZE)) - { - addToStringCache( curSig ); - } - } - else //...No room for this string in current block, so we write - // out the current block, so we can start another block - { -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_PARSE_DCT); -#endif - RETURN_ON_ERROR( writeDBFileNoVBCache(cb, &m_curBlock, m_curFbo) ); - m_curBlock.state = BLK_READ; - next = true; - found = false; - }//if m_freeSpace - - //..."next" flag is used to indicate that we need to advance to the - // next block in the store file. - if (next) - { - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - memcpy( m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); - m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; - m_curBlock.state = BLK_WRITE; - m_curOp = 0; - next = false; - m_lastFbo++; - m_curFbo = m_lastFbo; - - //...Expand current extent if it is an abbreviated initial extent - if ((m_curFbo == m_numBlocks) && - (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) - { - RETURN_ON_ERROR( expandDctnryExtent() ); - } - - //...Allocate a new extent if we have reached the last block in the - // current extent. - if (m_curFbo == m_numBlocks) - { - //last block - LBID_t startLbid; - - // Add an extent. - RETURN_ON_ERROR( createDctnry(m_dctnryOID, - m_colWidth, - m_dbRoot, - m_partition, - m_segment, - startLbid, - false) ); - - if (m_logger) - { - std::ostringstream oss; - oss << "Add dictionary extent OID-" << m_dctnryOID << - "; DBRoot-" << m_dbRoot << - "; part-" << m_partition << - "; seg-" << m_segment << - "; hwm-" << m_curFbo << - "; LBID-" << startLbid << - "; file-" << m_segFileName; - m_logger->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - m_curLbid = startLbid; - - // now seek back to the curFbo, after adding an extent - // @bug5769 For uncompressed only; - // ChunkManager manages the file offset for the compression case - if (m_compressionType == 0) - { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK); -#endif - long long byteOffset = m_curFbo; - byteOffset *= BYTE_PER_BLOCK; - RETURN_ON_ERROR( setFileOffset(m_dFile, byteOffset) ); -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK); -#endif - } - } - else - { - // LBIDs are numbered collectively and consecutively within an - // extent, so within an extent we can derive the LBID by simply - // incrementing it rather than having to go back to BRM to look - // up the LBID for each FBO. - m_curLbid++; - } + } + } // if next + } // end while #ifdef PROFILE - Stats::startParseEvent(WE_STATS_PARSE_DCT); + Stats::stopParseEvent(WE_STATS_PARSE_DCT); #endif - m_curBlock.lbid = m_curLbid; + // Done + // If any data leftover and not written by subsequent call to + // insertDctnry(), then it will be written by closeDctnry(). - //..."found" flag indicates whether the string was already found - // "or" added to the end of the previous block. If false, then - // we need to add the string to the new block. - if (!found) - { - RETURN_ON_ERROR(insertDctnry2(curSig)); //m_freeSpace updated! - m_curBlock.state = BLK_WRITE; - memcpy( pOut + outOffset, &curSig.token, 8 ); - outOffset += 8; - startPos++; - - //...Add string to cache, if we have not exceeded cache limit - if ((m_arraySize < MAX_STRING_CACHE_SIZE) && - (curSig.size <= MAX_SIGNATURE_SIZE)) - { - addToStringCache( curSig ); - } - } - }//if next - }//end while - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_PARSE_DCT); -#endif - //Done - // If any data leftover and not written by subsequent call to - // insertDctnry(), then it will be written by closeDctnry(). - - return NO_ERROR; + return NO_ERROR; } /******************************************************************************* @@ -1097,138 +1046,128 @@ int Dctnry::insertDctnry(const char* buf, * success - successfully insert the signature * failure - it did not insert the signature ******************************************************************************/ -int Dctnry::insertDctnry(const int& sgnature_size, - const unsigned char* sgnature_value, - Token& token) +int Dctnry::insertDctnry(const int& sgnature_size, const unsigned char* sgnature_value, Token& token) { - int rc = 0; - int i; - unsigned char* value = NULL; - int size; - int write_size; - bool lbid_in_token = false; + int rc = 0; + int i; + unsigned char* value = NULL; + int size; + int write_size; + bool lbid_in_token = false; - // Round down for safety. In theory we can take 262143 * 8176 bytes - if (sgnature_size > MAX_BLOB_SIZE) - { - return ERR_DICT_SIZE_GT_2G; - } + // Round down for safety. In theory we can take 262143 * 8176 bytes + if (sgnature_size > MAX_BLOB_SIZE) + { + return ERR_DICT_SIZE_GT_2G; + } - if (sgnature_size == 0) + if (sgnature_size == 0) + { + WriteEngine::Token nullToken; + memcpy(&token, &nullToken, 8); + return NO_ERROR; + } + + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = m_dFile; + + size = sgnature_size; + value = (unsigned char*)sgnature_value; + token.bc = 0; + + for (i = m_lastFbo; i < m_numBlocks; i++) + { + // @bug 3960: Add MAX_OP_COUNT check to handle case after bulk rollback + if (((m_freeSpace - HDR_UNIT_SIZE >= size) || ((size > 8176) && (m_freeSpace > HDR_UNIT_SIZE))) && + (m_curOp < (MAX_OP_COUNT - 1))) { - WriteEngine::Token nullToken; - memcpy( &token, &nullToken, 8 ); + // found the perfect block; signature size fit in this block + if (size > (m_freeSpace - HDR_UNIT_SIZE)) + { + write_size = (m_freeSpace - HDR_UNIT_SIZE); + } + else + { + write_size = size; + } + + insertDctnryHdr(m_curBlock.data, write_size); + insertSgnture(m_curBlock.data, write_size, value); + size -= write_size; + value += write_size; + m_curBlock.state = BLK_WRITE; + + // We only want the start LBID for a multi-block dict in the token + if (!lbid_in_token) + { + token.fbo = m_curLbid; + token.op = m_curOp; + lbid_in_token = true; + } + + if (size > 0) + token.bc++; + + m_lastFbo = i; + m_curFbo = m_lastFbo; + + if ((m_curOp < (MAX_OP_COUNT - 1)) && (size <= 0)) return NO_ERROR; + } // end Found + + //@bug 3832. check error code + RETURN_ON_ERROR(writeDBFile(cb, &m_curBlock, m_curLbid)); + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + memcpy(m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); + m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; + m_curBlock.state = BLK_WRITE; + m_curOp = 0; + m_lastFbo++; + m_curFbo = m_lastFbo; + + //...Expand current extent if it is an abbreviated initial extent + if ((m_curFbo == m_numBlocks) && (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) + { + RETURN_ON_ERROR(expandDctnryExtent()); } - CommBlock cb; - cb.file.oid = m_dctnryOID; - cb.file.pFile = m_dFile; - - size = sgnature_size; - value = (unsigned char*)sgnature_value; - token.bc = 0; - - for (i = m_lastFbo; i < m_numBlocks; i++) + //...Allocate a new extent if we have reached the last block in the + // current extent. + if (m_curFbo == m_numBlocks) { - // @bug 3960: Add MAX_OP_COUNT check to handle case after bulk rollback - if ( ((m_freeSpace - HDR_UNIT_SIZE >= size) || - ((size > 8176) && (m_freeSpace > HDR_UNIT_SIZE))) && - (m_curOp < (MAX_OP_COUNT - 1)) ) - { - // found the perfect block; signature size fit in this block - if (size > (m_freeSpace - HDR_UNIT_SIZE)) - { - write_size = (m_freeSpace - HDR_UNIT_SIZE); - } - else - { - write_size = size; - } + // last block + // for roll back the extent to use + // Save those empty extents in case of failure to rollback + std::vector dictExtentInfo; + ExtentInfo info; + info.oid = m_dctnryOID; + info.partitionNum = m_partition; + info.segmentNum = m_segment; + info.dbRoot = m_dbRoot; + info.hwm = m_hwm; + info.newFile = false; + dictExtentInfo.push_back(info); + LBID_t startLbid; + // Add an extent. + rc = createDctnry(m_dctnryOID, + 0, // dummy column width + m_dbRoot, m_partition, m_segment, startLbid, false); - insertDctnryHdr(m_curBlock.data, write_size); - insertSgnture(m_curBlock.data, write_size, value); - size -= write_size; - value += write_size; - m_curBlock.state = BLK_WRITE; + if (rc != NO_ERROR) + { + // roll back the extent + BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(dictExtentInfo); + return rc; + } + } - // We only want the start LBID for a multi-block dict in the token - if (!lbid_in_token) - { - token.fbo = m_curLbid; - token.op = m_curOp; - lbid_in_token = true; - } + RETURN_ON_ERROR( + BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID, m_partition, m_segment, m_curFbo, m_curLbid)); + m_curBlock.lbid = m_curLbid; + } // end for loop for all of the blocks - if (size > 0) - token.bc++; - - m_lastFbo = i; - m_curFbo = m_lastFbo; - - if ((m_curOp < (MAX_OP_COUNT - 1)) && (size <= 0)) - return NO_ERROR; - }//end Found - - //@bug 3832. check error code - RETURN_ON_ERROR( writeDBFile(cb, &m_curBlock, m_curLbid) ); - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - memcpy( m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); - m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; - m_curBlock.state = BLK_WRITE; - m_curOp = 0; - m_lastFbo++; - m_curFbo = m_lastFbo; - - //...Expand current extent if it is an abbreviated initial extent - if ((m_curFbo == m_numBlocks) && - (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) - { - RETURN_ON_ERROR( expandDctnryExtent() ); - } - - //...Allocate a new extent if we have reached the last block in the - // current extent. - if (m_curFbo == m_numBlocks) - { - //last block - //for roll back the extent to use - //Save those empty extents in case of failure to rollback - std::vector dictExtentInfo; - ExtentInfo info; - info.oid = m_dctnryOID; - info.partitionNum = m_partition; - info.segmentNum = m_segment; - info.dbRoot = m_dbRoot; - info.hwm = m_hwm; - info.newFile = false; - dictExtentInfo.push_back (info); - LBID_t startLbid; - // Add an extent. - rc = createDctnry(m_dctnryOID, - 0, // dummy column width - m_dbRoot, - m_partition, - m_segment, - startLbid, - false) ; - - if ( rc != NO_ERROR ) - { - //roll back the extent - BRMWrapper::getInstance()->deleteEmptyDictStoreExtents( - dictExtentInfo); - return rc; - } - } - - RETURN_ON_ERROR( BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID, - m_partition, m_segment, - m_curFbo, m_curLbid) ); - m_curBlock.lbid = m_curLbid; - }//end for loop for all of the blocks - - return ERR_DICT_NO_SPACE_INSERT; + return ERR_DICT_NO_SPACE_INSERT; } /******************************************************************************* @@ -1247,22 +1186,21 @@ int Dctnry::insertDctnry(const int& sgnature_size, * RETURN: * none ******************************************************************************/ -void Dctnry::insertDctnryHdr( unsigned char* blockBuf, - const int& size) +void Dctnry::insertDctnryHdr(unsigned char* blockBuf, const int& size) { - int endHdrLoc = START_HDR1 + (m_curOp + 1) * HDR_UNIT_SIZE; - int nextOffsetLoc = START_HDR1 + m_curOp * HDR_UNIT_SIZE; - int lastOffsetLoc = START_HDR1 + (m_curOp - 1) * HDR_UNIT_SIZE ; + int endHdrLoc = START_HDR1 + (m_curOp + 1) * HDR_UNIT_SIZE; + int nextOffsetLoc = START_HDR1 + m_curOp * HDR_UNIT_SIZE; + int lastOffsetLoc = START_HDR1 + (m_curOp - 1) * HDR_UNIT_SIZE; - m_freeSpace -= (size + HDR_UNIT_SIZE); - memcpy(&blockBuf[endHdrLoc], &m_endHeader, HDR_UNIT_SIZE); - uint16_t lastOffset = *(uint16_t*)&blockBuf[lastOffsetLoc]; - uint16_t nextOffset = lastOffset - size; + m_freeSpace -= (size + HDR_UNIT_SIZE); + memcpy(&blockBuf[endHdrLoc], &m_endHeader, HDR_UNIT_SIZE); + uint16_t lastOffset = *(uint16_t*)&blockBuf[lastOffsetLoc]; + uint16_t nextOffset = lastOffset - size; - memcpy(&blockBuf[0], &m_freeSpace, HDR_UNIT_SIZE); - memcpy(&blockBuf[nextOffsetLoc], &nextOffset, HDR_UNIT_SIZE); - m_newStartOffset = nextOffset; - m_curOp++; + memcpy(&blockBuf[0], &m_freeSpace, HDR_UNIT_SIZE); + memcpy(&blockBuf[nextOffsetLoc], &nextOffset, HDR_UNIT_SIZE); + m_newStartOffset = nextOffset; + m_curOp++; } /******************************************************************************* @@ -1280,11 +1218,10 @@ void Dctnry::insertDctnryHdr( unsigned char* blockBuf, * RETURN: * none ******************************************************************************/ -void Dctnry::insertSgnture(unsigned char* blockBuf, - const int& size, unsigned char* value) +void Dctnry::insertSgnture(unsigned char* blockBuf, const int& size, unsigned char* value) { - //m_newStartLoc is calculated from the header insertion code - memcpy(&blockBuf[m_newStartOffset], value, size); + // m_newStartLoc is calculated from the header insertion code + memcpy(&blockBuf[m_newStartOffset], value, size); } /******************************************************************************* @@ -1295,29 +1232,29 @@ void Dctnry::insertSgnture(unsigned char* blockBuf, * output * op_count - total op count ******************************************************************************/ -void Dctnry::getBlockOpCount( const DataBlock& fileBlock, int& op_count) +void Dctnry::getBlockOpCount(const DataBlock& fileBlock, int& op_count) { - ByteStream bs; - ByteStream::byte inbuf[BYTE_PER_BLOCK]; - memcpy(inbuf, fileBlock.data, BYTE_PER_BLOCK); - bs.load(inbuf, BYTE_PER_BLOCK); + ByteStream bs; + ByteStream::byte inbuf[BYTE_PER_BLOCK]; + memcpy(inbuf, fileBlock.data, BYTE_PER_BLOCK); + bs.load(inbuf, BYTE_PER_BLOCK); - ByteStream::doublebyte offset; - ByteStream::doublebyte dbyte; - bs >> m_freeSpace; - bs >> dbyte; - bs >> dbyte; - bs >> dbyte; - bs >> dbyte; - bs >> dbyte; - idbassert(dbyte == BYTE_PER_BLOCK); + ByteStream::doublebyte offset; + ByteStream::doublebyte dbyte; + bs >> m_freeSpace; + bs >> dbyte; + bs >> dbyte; + bs >> dbyte; + bs >> dbyte; + bs >> dbyte; + idbassert(dbyte == BYTE_PER_BLOCK); + bs >> offset; + + while (offset < 0xffff) + { + op_count++; bs >> offset; - - while (offset < 0xffff) - { - op_count++; - bs >> offset; - } + } } /******************************************************************************* @@ -1327,52 +1264,51 @@ void Dctnry::getBlockOpCount( const DataBlock& fileBlock, int& op_count) * input * DataBlock& fileBlock -- the file block ******************************************************************************/ -void Dctnry::preLoadStringCache( const DataBlock& fileBlock ) +void Dctnry::preLoadStringCache(const DataBlock& fileBlock) { - int hdrOffsetBeg = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE; - int hdrOffsetEnd = HDR_UNIT_SIZE + NEXT_PTR_BYTES; - uint16_t offBeg = 0; - uint16_t offEnd = 0; - memcpy( &offBeg, &fileBlock.data[hdrOffsetBeg], HDR_UNIT_SIZE ); - memcpy( &offEnd, &fileBlock.data[hdrOffsetEnd], HDR_UNIT_SIZE ); + int hdrOffsetBeg = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE; + int hdrOffsetEnd = HDR_UNIT_SIZE + NEXT_PTR_BYTES; + uint16_t offBeg = 0; + uint16_t offEnd = 0; + memcpy(&offBeg, &fileBlock.data[hdrOffsetBeg], HDR_UNIT_SIZE); + memcpy(&offEnd, &fileBlock.data[hdrOffsetEnd], HDR_UNIT_SIZE); - int op = 1; // ordinal position of the string within the block - Signature aSig; - void *aSigPtr = static_cast(&aSig); - memset(aSigPtr, 0, sizeof(aSig)); + int op = 1; // ordinal position of the string within the block + Signature aSig; + void* aSigPtr = static_cast(&aSig); + memset(aSigPtr, 0, sizeof(aSig)); - while ((offBeg != DCTNRY_END_HEADER) && - (op <= MAX_STRING_CACHE_SIZE)) - { - unsigned int len = offEnd - offBeg; - aSig.size = len; - aSig.signature = new unsigned char[len]; - memcpy(aSig.signature, &fileBlock.data[offBeg], len); - aSig.token.op = op; - aSig.token.fbo = m_curLbid; - m_sigArray.insert(aSig); + while ((offBeg != DCTNRY_END_HEADER) && (op <= MAX_STRING_CACHE_SIZE)) + { + unsigned int len = offEnd - offBeg; + aSig.size = len; + aSig.signature = new unsigned char[len]; + memcpy(aSig.signature, &fileBlock.data[offBeg], len); + aSig.token.op = op; + aSig.token.fbo = m_curLbid; + m_sigArray.insert(aSig); - offEnd = offBeg; - hdrOffsetBeg += HDR_UNIT_SIZE; - memcpy( &offBeg, &fileBlock.data[hdrOffsetBeg], HDR_UNIT_SIZE ); - op++; - } + offEnd = offBeg; + hdrOffsetBeg += HDR_UNIT_SIZE; + memcpy(&offBeg, &fileBlock.data[hdrOffsetBeg], HDR_UNIT_SIZE); + op++; + } - m_arraySize = op - 1; + m_arraySize = op - 1; - //std::cout << "Preloading strings..." << std::endl; - //char strSig[1000]; - //uint64_t tokenVal; - //for (int i=0; i(&asig); - memset(aSigPtr, 0, sizeof(asig)); - asig.signature = new unsigned char[newSig.size]; - memcpy(asig.signature, newSig.signature, newSig.size ); - asig.size = newSig.size; - asig.token = newSig.token; - m_sigArray.insert(asig); - m_arraySize++; + // We better add constructors that sets everything to 0; + Signature asig; + void* aSigPtr = static_cast(&asig); + memset(aSigPtr, 0, sizeof(asig)); + asig.signature = new unsigned char[newSig.size]; + memcpy(asig.signature, newSig.signature, newSig.size); + asig.size = newSig.size; + asig.token = newSig.token; + m_sigArray.insert(asig); + m_arraySize++; } /******************************************************************************* @@ -1408,39 +1344,34 @@ void Dctnry::addToStringCache( const Signature& newSig ) * Success -- found and deleted * Fail -- ERR_DICT_INVALID_DELETE ******************************************************************************/ -int Dctnry::getEndOp(IDBDataFile* dFile, int lbid, int& endOp) +int Dctnry::getEndOp(IDBDataFile* dFile, int lbid, int& endOp) { - DataBlock fileBlock; - Offset newOffset; - int rc; - CommBlock cb; - cb.file.oid = m_dctnryOID; - cb.file.pFile = dFile; - memset( fileBlock.data, 0, sizeof(fileBlock.data)); - m_dFile = dFile; - rc = readSubBlockEntry( cb, &fileBlock, lbid, 0, 0, - HDR_UNIT_SIZE + - NEXT_PTR_BYTES + - HDR_UNIT_SIZE + - HDR_UNIT_SIZE, - &m_dctnryHeader); + DataBlock fileBlock; + Offset newOffset; + int rc; + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = dFile; + memset(fileBlock.data, 0, sizeof(fileBlock.data)); + m_dFile = dFile; + rc = readSubBlockEntry(cb, &fileBlock, lbid, 0, 0, + HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE, &m_dctnryHeader); - memcpy(&m_freeSpace, &fileBlock.data[0], HDR_UNIT_SIZE); - memcpy(&m_nextPtr, &fileBlock.data[HDR_UNIT_SIZE], NEXT_PTR_BYTES); + memcpy(&m_freeSpace, &fileBlock.data[0], HDR_UNIT_SIZE); + memcpy(&m_nextPtr, &fileBlock.data[HDR_UNIT_SIZE], NEXT_PTR_BYTES); - newOffset.hdrLoc = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE ; + newOffset.hdrLoc = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE; + memcpy(&newOffset.offset, &fileBlock.data[newOffset.hdrLoc], HDR_UNIT_SIZE); + endOp = 1; // should be zero counting the end of header then + + while (newOffset.offset != DCTNRY_END_HEADER) + { + newOffset.hdrLoc += HDR_UNIT_SIZE; memcpy(&newOffset.offset, &fileBlock.data[newOffset.hdrLoc], HDR_UNIT_SIZE); - endOp = 1; //should be zero counting the end of header then + endOp++; + } - while ( newOffset.offset != DCTNRY_END_HEADER) - { - newOffset.hdrLoc += HDR_UNIT_SIZE; - memcpy(&newOffset.offset, &fileBlock.data[newOffset.hdrLoc], - HDR_UNIT_SIZE); - endOp++; - } - - return rc; + return rc; } /******************************************************************************* @@ -1449,56 +1380,53 @@ int Dctnry::getEndOp(IDBDataFile* dFile, int lbid, int& endOp) * in our string cache, and returns the corresponding token * if it is found in the cache. ******************************************************************************/ -int Dctnry::updateDctnry(unsigned char* sigValue, int& sigSize, - Token& token) +int Dctnry::updateDctnry(unsigned char* sigValue, int& sigSize, Token& token) { - int rc = NO_ERROR; + int rc = NO_ERROR; + Signature sig; + sig.signature = sigValue; + sig.size = sigSize; + + // Look for string in cache + // As long as the string <= 8000 bytes + if (sigSize <= MAX_SIGNATURE_SIZE) + { + bool found = false; + found = getTokenFromArray(sig); + + if (found) + { + token = sig.token; + return NO_ERROR; + } + } + + // Insert into Dictionary + rc = insertDctnry(sigSize, sigValue, token); + + // Add the new signature and token into cache + // As long as the string is <= 8000 bytes + if ((m_arraySize < MAX_STRING_CACHE_SIZE) && (sigSize <= MAX_SIGNATURE_SIZE)) + { Signature sig; - sig.signature = sigValue; sig.size = sigSize; + sig.signature = new unsigned char[sigSize]; + memcpy(sig.signature, sigValue, sigSize); + sig.token = token; + m_sigArray.insert(sig); + m_arraySize++; + } - // Look for string in cache - // As long as the string <= 8000 bytes - if (sigSize <= MAX_SIGNATURE_SIZE) - { - bool found = false; - found = getTokenFromArray(sig); - - if (found) - { - token = sig.token; - return NO_ERROR; - } - } - - //Insert into Dictionary - rc = insertDctnry(sigSize, sigValue, token); - - //Add the new signature and token into cache - //As long as the string is <= 8000 bytes - if ((m_arraySize < MAX_STRING_CACHE_SIZE) && - (sigSize <= MAX_SIGNATURE_SIZE)) - { - Signature sig; - sig.size = sigSize; - sig.signature = new unsigned char[sigSize]; - memcpy (sig.signature, sigValue, sigSize); - sig.token = token; - m_sigArray.insert(sig); - m_arraySize++; - } - - return rc; + return rc; } /******************************************************************************* * open dictionary file ******************************************************************************/ -IDBDataFile* Dctnry::createDctnryFile( - const char* name, int, const char* mode, int ioBuffSize, LBID_t lbid) +IDBDataFile* Dctnry::createDctnryFile(const char* name, int, const char* mode, int ioBuffSize, LBID_t lbid) { - (void) lbid; - return openFile(name, mode, ioBuffSize, false); + (void)lbid; + return openFile(name, mode, ioBuffSize, false); } /******************************************************************************* @@ -1507,9 +1435,8 @@ IDBDataFile* Dctnry::createDctnryFile( // @bug 5572 - HDFS usage: add *.tmp file backup flag IDBDataFile* Dctnry::openDctnryFile(bool useTmpSuffix) { - return openFile( - m_dctnryOID, m_dbRoot, m_partition, m_segment, m_segFileName, - "r+b", DEFAULT_COLSIZ, useTmpSuffix); + return openFile(m_dctnryOID, m_dbRoot, m_partition, m_segment, m_segFileName, "r+b", DEFAULT_COLSIZ, + useTmpSuffix); } /******************************************************************************* @@ -1517,20 +1444,20 @@ IDBDataFile* Dctnry::openDctnryFile(bool useTmpSuffix) ******************************************************************************/ void Dctnry::closeDctnryFile(bool doFlush, std::map& oids) { - closeFile(m_dFile); - m_dFile = NULL; + closeFile(m_dFile); + m_dFile = NULL; } int Dctnry::numOfBlocksInFile() { - long long fileSizeBytes = 0; - getFileSize(m_dFile, fileSizeBytes); //dmc-error handling (ignoring rc) - return fileSizeBytes / BYTE_PER_BLOCK; + long long fileSizeBytes = 0; + getFileSize(m_dFile, fileSizeBytes); // dmc-error handling (ignoring rc) + return fileSizeBytes / BYTE_PER_BLOCK; } void Dctnry::copyDctnryHeader(void* buf) { - memcpy(buf, m_dctnryHeader2, m_totalHdrBytes); + memcpy(buf, m_dctnryHeader2, m_totalHdrBytes); } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index fd9d1bf3a..d741c4bff 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -51,302 +51,292 @@ namespace WriteEngine //--------------------------------------------------------------------------- typedef struct Signature { - int size; - unsigned char* signature; - Token token; + int size; + unsigned char* signature; + Token token; } Signature; -struct sig_compare { - bool operator() (const Signature& a, const Signature& b) const { - if (a.size == b.size){ - return memcmp(a.signature,b.signature,a.size)<0;} - else if (a.size& oids); + virtual int numOfBlocksInFile(); - // methods to be overriden by compression classes - // (width argument in createDctnryFile() is string width, not token width) - virtual IDBDataFile* createDctnryFile(const char* name, int width, - const char* mode, int ioBuffSize, - BRM::LBID_t lbid = -1); - virtual IDBDataFile* openDctnryFile(bool useTmpSuffix); - virtual void closeDctnryFile(bool doFlush, std::map& oids); - virtual int numOfBlocksInFile(); + std::set m_sigArray; + int m_arraySize; // num strings in m_sigArray - std::set m_sigArray; - int m_arraySize; // num strings in m_sigArray + // m_dctnryHeader used for hdr when readSubBlockEntry is used to read a blk + // m_dctnryHeader2 contains filled in template used to initialize new blocks + unsigned char m_dctnryHeader[DCTNRY_HEADER_SIZE]; // first 14 bytes of hdr + unsigned char m_dctnryHeader2[DCTNRY_HEADER_SIZE]; // first 14 bytes of hdr - // m_dctnryHeader used for hdr when readSubBlockEntry is used to read a blk - // m_dctnryHeader2 contains filled in template used to initialize new blocks - unsigned char m_dctnryHeader[DCTNRY_HEADER_SIZE]; // first 14 bytes of hdr - unsigned char m_dctnryHeader2[DCTNRY_HEADER_SIZE]; // first 14 bytes of hdr + uint64_t m_nextPtr; // next pointer - uint64_t m_nextPtr; // next pointer + // relate to different Dictionary file + FID m_dctnryOID; // OID for the dctnry file + IDBDataFile* m_dFile; // dictionary file + uint32_t m_partition; // partition associated with OID + uint16_t m_segment; // segment associated with OID + uint16_t m_dbRoot; // DBRoot associated with OID + std::string m_segFileName; // current column segment file + int m_numBlocks; // num "raw" uncompressed blocks in file + int m_lastFbo; + HWM m_hwm; + // Need to be initialized for different Dictionary file + int m_newStartOffset; // start offset + uint16_t m_freeSpace; // free space (bytes) within current block + int m_curOp; // current ordinal pointer within m_curFbo + int m_curFbo; // current "raw" (uncompressed) FBO + BRM::LBID_t m_curLbid; // LBID associated with m_curFbo + DataBlock m_curBlock; // current "raw" (uncompressed) data block + Log* m_logger; // logger, mainly for bulk load + int m_colWidth; // width of this dictionary column + std::string m_defVal; // optional default string value + ImportDataMode m_importDataMode; // Import data in text or binary mode - //relate to different Dictionary file - FID m_dctnryOID; // OID for the dctnry file - IDBDataFile* m_dFile; // dictionary file - uint32_t m_partition; // partition associated with OID - uint16_t m_segment; // segment associated with OID - uint16_t m_dbRoot; // DBRoot associated with OID - std::string m_segFileName; // current column segment file - int m_numBlocks; // num "raw" uncompressed blocks in file - int m_lastFbo; - HWM m_hwm; - //Need to be initialized for different Dictionary file - int m_newStartOffset; // start offset - uint16_t m_freeSpace; // free space (bytes) within current block - int m_curOp; // current ordinal pointer within m_curFbo - int m_curFbo; // current "raw" (uncompressed) FBO - BRM::LBID_t m_curLbid; // LBID associated with m_curFbo - DataBlock m_curBlock; // current "raw" (uncompressed) data block - Log* m_logger; // logger, mainly for bulk load - int m_colWidth; // width of this dictionary column - std::string m_defVal; // optional default string value - ImportDataMode m_importDataMode; // Import data in text or binary mode +}; // end of class -};//end of class - -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_DCTNRY_H_ +#endif // _WE_DCTNRY_H_ diff --git a/writeengine/dictionary/we_dctnrystore.cpp b/writeengine/dictionary/we_dctnrystore.cpp index 4a7fefe73..4cdf927ca 100644 --- a/writeengine/dictionary/we_dctnrystore.cpp +++ b/writeengine/dictionary/we_dctnrystore.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_dctnrystore.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_dctnrystore.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ #include #ifndef _MSC_VER #include @@ -29,14 +29,12 @@ namespace WriteEngine { - /*********************************************************** * Constructor ***********************************************************/ -DctnryStore::DctnryStore() - : m_hashMapFlag(true), m_hashMapSize(MAX_HASHMAP_SIZE) +DctnryStore::DctnryStore() : m_hashMapFlag(true), m_hashMapSize(MAX_HASHMAP_SIZE) { - m_dctnry.setUseHashMap(m_hashMapFlag); + m_dctnry.setUseHashMap(m_hashMapFlag); } /*********************************************************** @@ -49,45 +47,40 @@ DctnryStore::~DctnryStore() /*********************************************************** * Open a dictionary store file ***********************************************************/ -const int DctnryStore::openDctnryStore(const OID& dctnryOID, - const OID& treeOID, - const OID& listOID, - const uint16_t dbRoot, - const uint32_t partition, +const int DctnryStore::openDctnryStore(const OID& dctnryOID, const OID& treeOID, const OID& listOID, + const uint16_t dbRoot, const uint32_t partition, const uint16_t segment) { - int rc2; - rc2 = m_dctnry.openDctnry(dctnryOID, dbRoot, partition, segment); - setUseHashMap(true); + int rc2; + rc2 = m_dctnry.openDctnry(dctnryOID, dbRoot, partition, segment); + setUseHashMap(true); - return rc2; + return rc2; } /*********************************************************** * Create a dictionary store file ***********************************************************/ -const int DctnryStore::createDctnryStore( const OID& dctnryOID, - const OID& treeOID, const OID& listOID, int colWidth, const uint16_t dbRoot, - const uint32_t partition, const uint16_t segment ) +const int DctnryStore::createDctnryStore(const OID& dctnryOID, const OID& treeOID, const OID& listOID, + int colWidth, const uint16_t dbRoot, const uint32_t partition, + const uint16_t segment) { - int rc2 ; + int rc2; - rc2 = m_dctnry.createDctnry(dctnryOID, colWidth, dbRoot, partition, segment); + rc2 = m_dctnry.createDctnry(dctnryOID, colWidth, dbRoot, partition, segment); - return rc2; + return rc2; } /*********************************************************** * Drop a dictionary store file ***********************************************************/ -const int DctnryStore::dropDctnryStore( const OID& dctnryOID, - const OID& treeOID, - const OID& listOID) +const int DctnryStore::dropDctnryStore(const OID& dctnryOID, const OID& treeOID, const OID& listOID) { - int rc2; - rc2 = m_dctnry.dropDctnry(dctnryOID); + int rc2; + rc2 = m_dctnry.dropDctnry(dctnryOID); - return rc2; + return rc2; } /*********************************************************** @@ -96,75 +89,73 @@ const int DctnryStore::dropDctnryStore( const OID& dctnryOID, * in our string cache, and returns the corresponding token * if it is found in the cache. ***********************************************************/ -const int DctnryStore::updateDctnryStore(unsigned char* sigValue, - int& sigSize, - Token& token) +const int DctnryStore::updateDctnryStore(unsigned char* sigValue, int& sigSize, Token& token) { - int rc = NO_ERROR; + int rc = NO_ERROR; + Signature sig; + sig.signature = sigValue; + sig.size = sigSize; + + // if String cache is enabled then look for string in cache + if (m_hashMapFlag) + { + if (m_dctnry.m_arraySize < (int)m_hashMapSize) + { + bool found = false; + found = m_dctnry.getTokenFromArray(sig); + + if (found) + { + token = sig.token; + return NO_ERROR; + } + } // end if use hash map and size >0 + } + + // Insert into Dictionary + rc = m_dctnry.insertDctnry(sigSize, sigValue, token); + + // Add the new signature and token into cache if the hashmap flag is on + // (We currently use an array instead of a hashmap.) + if ((m_hashMapFlag) && (m_dctnry.m_arraySize < (int)m_hashMapSize)) + { Signature sig; - sig.signature = sigValue; sig.size = sigSize; + sig.signature = new unsigned char[sigSize]; + memcpy(sig.signature, sigValue, sigSize); + sig.token = token; + m_dctnry.m_sigArray.insert(sig) = sig; + m_dctnry.m_arraySize++; + } - //if String cache is enabled then look for string in cache - if (m_hashMapFlag) - { - if (m_dctnry.m_arraySize < (int)m_hashMapSize) - { - bool found = false; - found = m_dctnry.getTokenFromArray(sig); - - if (found) - { - token = sig.token; - return NO_ERROR; - } - } //end if use hash map and size >0 - } - - //Insert into Dictionary - rc = m_dctnry.insertDctnry(sigSize, sigValue, token); - - //Add the new signature and token into cache if the hashmap flag is on - // (We currently use an array instead of a hashmap.) - if ((m_hashMapFlag) && (m_dctnry.m_arraySize < (int)m_hashMapSize)) - { - Signature sig; - sig.size = sigSize; - sig.signature = new unsigned char[sigSize]; - memcpy (sig.signature, sigValue, sigSize); - sig.token = token; - m_dctnry.m_sigArray.insert(sig) = sig; - m_dctnry.m_arraySize++; - } - - return rc; + return rc; } /*********************************************************** * Delete signature from the dictionary store file ***********************************************************/ -const int DctnryStore::deleteDctnryToken(Token& token) +const int DctnryStore::deleteDctnryToken(Token& token) { - int rc ; - int sigSize; - unsigned char* sigValue = NULL; + int rc; + int sigSize; + unsigned char* sigValue = NULL; - rc = m_dctnry.deleteDctnryValue( token, sigSize, &sigValue); - - if (rc != NO_ERROR) - { - return rc; - } - - if (!sigValue) - { - rc = ERR_DICT_TOKEN_NOT_FOUND; - return rc; - } - - free(sigValue); + rc = m_dctnry.deleteDctnryValue(token, sigSize, &sigValue); + if (rc != NO_ERROR) + { return rc; + } + + if (!sigValue) + { + rc = ERR_DICT_TOKEN_NOT_FOUND; + return rc; + } + + free(sigValue); + + return rc; } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/dictionary/we_dctnrystore.h b/writeengine/dictionary/we_dctnrystore.h index 1c19306b5..9ae00a9a2 100644 --- a/writeengine/dictionary/we_dctnrystore.h +++ b/writeengine/dictionary/we_dctnrystore.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_dctnrystore.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_dctnrystore.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** * Wrapper around the Dctnry class used to maintain a dictionary store file. * When signature(s) are specified to the update functions, the value(s) are @@ -58,173 +58,162 @@ namespace WriteEngine class DctnryStore : public DbFileOp { -public: - /** - * @brief Constructor - */ - EXPORT DctnryStore(); + public: + /** + * @brief Constructor + */ + EXPORT DctnryStore(); - /** - * @brief Destructor - */ - EXPORT ~DctnryStore(); + /** + * @brief Destructor + */ + EXPORT ~DctnryStore(); - /** - * @brief Close a dictionary store after it has been opened. - */ - EXPORT int closeDctnryStore() - { - return m_dctnry.closeDctnry(); - } + /** + * @brief Close a dictionary store after it has been opened. + */ + EXPORT int closeDctnryStore() + { + return m_dctnry.closeDctnry(); + } - /** - * @brief Close a dictionary store without flushing the block buffer or - * updating BRM with HWM. Intended to be used for immediate file closure - * to shut down a job that has encountered an error, and intends to do - * a rollback. - */ - EXPORT int closeDctnryStoreOnly() - { - return m_dctnry.closeDctnryOnly(); - } + /** + * @brief Close a dictionary store without flushing the block buffer or + * updating BRM with HWM. Intended to be used for immediate file closure + * to shut down a job that has encountered an error, and intends to do + * a rollback. + */ + EXPORT int closeDctnryStoreOnly() + { + return m_dctnry.closeDctnryOnly(); + } - /** - * @brief create dictionary store - * - * @param dctnryOID - dictionary file OID - * @param treeOID - index tree OID (not used) - * @param listOID - list OID (not used) - * @param colWidth - column width - * @param dbRoot - DBRoot for store file - * @param partition - partition number for store file - * @param segment - column segment number for store file - */ - EXPORT const int createDctnryStore( - const OID& dctnryOID, const OID& treeOID, - const OID& listOID, int colWidth, const uint16_t dbRoot, - const uint32_t partition, const uint16_t segment ); + /** + * @brief create dictionary store + * + * @param dctnryOID - dictionary file OID + * @param treeOID - index tree OID (not used) + * @param listOID - list OID (not used) + * @param colWidth - column width + * @param dbRoot - DBRoot for store file + * @param partition - partition number for store file + * @param segment - column segment number for store file + */ + EXPORT const int createDctnryStore(const OID& dctnryOID, const OID& treeOID, const OID& listOID, + int colWidth, const uint16_t dbRoot, const uint32_t partition, + const uint16_t segment); - /** - * @brief Delete a token from dictionary store, for maintanance use - * - * @param token - token to be deleted - */ - EXPORT const int deleteDctnryToken(Token& token); + /** + * @brief Delete a token from dictionary store, for maintanance use + * + * @param token - token to be deleted + */ + EXPORT const int deleteDctnryToken(Token& token); - /** - * @brief Drop dictionary store (for DDL/DML use) - * - * @param dctnryOID - dictionary file OID - * @param treeOID - index tree OID (not used) - * @param listOID - list OID (not used) - */ - EXPORT const int dropDctnryStore( const OID& dctnryOID, const OID& treeOID, - const OID& listOID); - /** - * @brief Open a dictionary store after creation - * - * @param dctnryOID - dictionary file OID - * @param treeOID - index tree OID (not used) - * @param listOID - list OID (not used) - * @param dbRoot - DBRoot for store file - * @param partition - partition number for store file - * @param segment - column segment number for store file - */ - EXPORT const int openDctnryStore(const OID& dctnryOID, const OID& treeOID, - const OID& listOID, - const uint16_t dbRoot, - const uint32_t partition, - const uint16_t segment); + /** + * @brief Drop dictionary store (for DDL/DML use) + * + * @param dctnryOID - dictionary file OID + * @param treeOID - index tree OID (not used) + * @param listOID - list OID (not used) + */ + EXPORT const int dropDctnryStore(const OID& dctnryOID, const OID& treeOID, const OID& listOID); + /** + * @brief Open a dictionary store after creation + * + * @param dctnryOID - dictionary file OID + * @param treeOID - index tree OID (not used) + * @param listOID - list OID (not used) + * @param dbRoot - DBRoot for store file + * @param partition - partition number for store file + * @param segment - column segment number for store file + */ + EXPORT const int openDctnryStore(const OID& dctnryOID, const OID& treeOID, const OID& listOID, + const uint16_t dbRoot, const uint32_t partition, const uint16_t segment); - /** - * @brief Update dictionary store with tokenized strings (for DDL/DML use) - * - * @param sigValue - signature value - * @param sigSize - signature size - * @param token - (output) token that was added - */ - EXPORT const int updateDctnryStore(unsigned char* sigValue, - int& sigSize, Token& token); + /** + * @brief Update dictionary store with tokenized strings (for DDL/DML use) + * + * @param sigValue - signature value + * @param sigSize - signature size + * @param token - (output) token that was added + */ + EXPORT const int updateDctnryStore(unsigned char* sigValue, int& sigSize, Token& token); - /** - * @brief Update dictionary store with tokenized strings (for Bulk use) - * - * @param buf - bulk buffer containing strings to be parsed - * @param pos - list of offsets into buf - * @param totalRow - total number of rows in buf - * @param col - the column to be parsed from buf - * @param colWidth - width of the dictionary column being parsed - * @param tokenBuf - (output) list of tokens for the parsed strings - */ - const int updateDctnryStore(const char* buf, - ColPosPair** pos, - const int totalRow, - const int col, - const int colWidth, - char* tokenBuf) - { - return (m_dctnry.insertDctnry( - buf, pos, totalRow, col, colWidth, tokenBuf)); - } + /** + * @brief Update dictionary store with tokenized strings (for Bulk use) + * + * @param buf - bulk buffer containing strings to be parsed + * @param pos - list of offsets into buf + * @param totalRow - total number of rows in buf + * @param col - the column to be parsed from buf + * @param colWidth - width of the dictionary column being parsed + * @param tokenBuf - (output) list of tokens for the parsed strings + */ + const int updateDctnryStore(const char* buf, ColPosPair** pos, const int totalRow, const int col, + const int colWidth, char* tokenBuf) + { + return (m_dctnry.insertDctnry(buf, pos, totalRow, col, colWidth, tokenBuf)); + } - /** - * @brief TransId related function - * - * @param transId - Current transaction id (for DDL/DML use) - */ - void setAllTransId(const TxnID& transId) - { - m_dctnry.setTransId(transId); - } + /** + * @brief TransId related function + * + * @param transId - Current transaction id (for DDL/DML use) + */ + void setAllTransId(const TxnID& transId) + { + m_dctnry.setTransId(transId); + } - /** - * @brief String cache related routines - */ - void clearMap() - { - m_dctnry.clearMap(); - } - void createMap() - { - m_dctnry.createMap(); - } - void setUseHashMap(bool flag) - { - m_hashMapFlag = flag; - m_dctnry.setUseHashMap(flag); - } - void setHashMapSize(int size) - { - if (size < MAX_HASHMAP_SIZE) - m_hashMapSize = size; - else - m_hashMapSize = MAX_HASHMAP_SIZE; + /** + * @brief String cache related routines + */ + void clearMap() + { + m_dctnry.clearMap(); + } + void createMap() + { + m_dctnry.createMap(); + } + void setUseHashMap(bool flag) + { + m_hashMapFlag = flag; + m_dctnry.setUseHashMap(flag); + } + void setHashMapSize(int size) + { + if (size < MAX_HASHMAP_SIZE) + m_hashMapSize = size; + else + m_hashMapSize = MAX_HASHMAP_SIZE; - m_dctnry.setHashMapSize(m_hashMapSize); - } + m_dctnry.setHashMapSize(m_hashMapSize); + } - HWM getHWM() const - { - return m_dctnry.getHWM(); - } - const std::string& getFileName() const - { - return m_dctnry.getFileName(); - } + HWM getHWM() const + { + return m_dctnry.getHWM(); + } + const std::string& getFileName() const + { + return m_dctnry.getFileName(); + } - /** - * @brief public instance - */ - Dctnry m_dctnry; + /** + * @brief public instance + */ + Dctnry m_dctnry; -private: - // Used to configure string cache usage - bool m_hashMapFlag; - int m_hashMapSize; + private: + // Used to configure string cache usage + bool m_hashMapFlag; + int m_hashMapSize; }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_DctnryStore_H_ +#endif // _WE_DctnryStore_H_ diff --git a/writeengine/index/qindex.cpp b/writeengine/index/qindex.cpp index 00899679a..89ed54ac9 100644 --- a/writeengine/index/qindex.cpp +++ b/writeengine/index/qindex.cpp @@ -29,1231 +29,1226 @@ using namespace boost; #include "we_freemgr.h" #include "we_indexlist.h" - using namespace WriteEngine; typedef struct { - IdxRidListHdr hdrRec; - IdxRidListPtr listPtr; - int fbo; - int sbid; - int entry; - int sbSeqNo; - int recCnt; - int sbCnt; - int fullSbCnt; + IdxRidListHdr hdrRec; + IdxRidListPtr listPtr; + int fbo; + int sbid; + int entry; + int sbSeqNo; + int recCnt; + int sbCnt; + int fullSbCnt; } IdxListStat; typedef struct { - int mapSize; - int entryCnt; + int mapSize; + int entryCnt; } IdxFreeSpaceStat; -const int QA_FILE_TYPE_TREE = 1; -const int QA_FILE_TYPE_LIST = 2; - -const int QA_FREEMAP_1ENTRY = 1; -const int QA_FREEMAP_2ENTRY = 2; -const int QA_FREEMAP_4ENTRY = 3; -const int QA_FREEMAP_8ENTRY = 4; -const int QA_FREEMAP_16ENTRY = 5; -const int QA_FREEMAP_32ENTRY = 6; +const int QA_FILE_TYPE_TREE = 1; +const int QA_FILE_TYPE_LIST = 2; +const int QA_FREEMAP_1ENTRY = 1; +const int QA_FREEMAP_2ENTRY = 2; +const int QA_FREEMAP_4ENTRY = 3; +const int QA_FREEMAP_8ENTRY = 4; +const int QA_FREEMAP_16ENTRY = 5; +const int QA_FREEMAP_32ENTRY = 6; class IndexTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(IndexTest); - CPPUNIT_TEST_SUITE( IndexTest ); + /* ==================================================== + Index stress test + These tests will create the maxium number: + 1) index key records, one row ID for each + 2) row IDs for an index key + ======================================================= */ - /* ==================================================== - Index stress test - These tests will create the maxium number: - 1) index key records, one row ID for each - 2) row IDs for an index key - ======================================================= */ + // CPPUNIT_TEST( testCreateIndex ); -//CPPUNIT_TEST( testCreateIndex ); + /* ==================================================== + TTest Suites + ======================================================= */ + CPPUNIT_TEST(qaTSCStressTestMaxRowID); + // CPPUNIT_TEST(qaTSCStressTestMaxKey); + // CPPUNIT_TEST(qaTSCIndexTestUpdate); + // CPPUNIT_TEST(qaTSCIndexTestDelete); + // CPPUNIT_TEST(qaTSCFreeSpaceMapTest); - /* ==================================================== - TTest Suites - ======================================================= */ + // CPPUNIT_TEST(qatestIndexList); - CPPUNIT_TEST(qaTSCStressTestMaxRowID); -//CPPUNIT_TEST(qaTSCStressTestMaxKey); -//CPPUNIT_TEST(qaTSCIndexTestUpdate); -//CPPUNIT_TEST(qaTSCIndexTestDelete); -//CPPUNIT_TEST(qaTSCFreeSpaceMapTest); + CPPUNIT_TEST_SUITE_END(); -//CPPUNIT_TEST(qatestIndexList); + private: + IndexTree m_index; + IndexList m_indexlist; + FILE* curFile; + public: + void setUp() + { + } + void tearDown() + { + } - CPPUNIT_TEST_SUITE_END(); + /* ========================================================== + Create maximum number of row ids for key and verify all + index list header records + ============================================================= */ -private: - IndexTree m_index; - IndexList m_indexlist; - FILE* curFile; + void qaTSCStressTestMaxRowID() + { + IdxFreeSpaceStat mapInfo; -public: - void setUp() + uint64_t idxKey = 1; + int testCaseNo; + int numOfRowID; + int expSbCnt; + int expFullSbCnt; + char testCaseDesc[256]; + + int tokenLen = 1; + int rc; + int execMode; + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + execMode = 0; + + strcpy(testCaseDesc, "8 bit key, max row stress test"); + + tokenLen = 8; + testCaseNo = 10520; + numOfRowID = 256; + expSbCnt = 9; + expFullSbCnt = 8; + + qaSupCreateMaxRowID(idxKey, tokenLen); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + qaDspIndexListByKey(idxKey, tokenLen); + + strcpy(testCaseDesc, "16 bit key, max row stress test"); + + testCaseNo = 10521; + numOfRowID = 65536; + expSbCnt = 2115; + expFullSbCnt = 2114; + + qaSupCreateMaxRowID(idxKey, tokenLen); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + } + /* ========================================================== + Create maximum number key for a given key length and + verify each index list header record + ============================================================= */ + + void qaTSCStressTestMaxKey() + { + IdxFreeSpaceStat mapInfo; + + int testCaseNo; + char testCaseDesc[256]; + int tokenLen; + int entryCnt; + int execMode; + + execMode = 1; + // qaSupCreateMaxHdrRec(tokenLen); + + strcpy(testCaseDesc, "8 bit key, max key stress test"); + + tokenLen = 8; + testCaseNo = 10522; + + qaSupCreateMaxHdrRec(tokenLen); + qaTstStressMaxKeyDriver(testCaseNo, testCaseDesc, tokenLen); + + // qaSupCreateMaxHdrRec(tokenLen); + + strcpy(testCaseDesc, "16 bit key, max key stress test"); + + tokenLen = 16; + testCaseNo = 10523; + + qaSupCreateMaxHdrRec(tokenLen); + qaTstStressMaxKeyDriver(testCaseNo, testCaseDesc, tokenLen); + } + + /* ========================================================== + Test cases for for updating index + ============================================================= */ + void qaTSCIndexTestUpdate() + { + uint64_t idxKey = 1; + int tokenLen = 16; + int testCaseNo; + int startRowID = 1; + int numOfRowID; + int expSbCnt; + int expFullSbCnt; + char testCaseDesc[256]; + + strcpy(testCaseDesc, "Create 1 index key with one row ID, expecting no sub block and no full sub block."); + + testCaseNo = 10524; + numOfRowID = 1; + expSbCnt = 0; + expFullSbCnt = 0; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + + strcpy(testCaseDesc, "Create 2 index key with one row ID, expecting no sub block and no full sub block."); + + testCaseNo = 10525; + numOfRowID = 2; + expSbCnt = 0; + expFullSbCnt = 0; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + + strcpy(testCaseDesc, "Create 3 index key with one row ID, expecting 1 sub block and no full sub block."); + + testCaseNo = 10526; + numOfRowID = 3; + expSbCnt = 1; + expFullSbCnt = 0; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + + strcpy(testCaseDesc, "Create 1 index key with 32 row IDs, expecting 1 sub block and 1 full sub block."); + + testCaseNo = 10527; + numOfRowID = 32; + expSbCnt = 1; + expFullSbCnt = 1; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + + strcpy(testCaseDesc, "Create 1 index key with 33 row IDs, expecting 2 sub block and 1 full sub block."); + + testCaseNo = 10528; + numOfRowID = 33; + expSbCnt = 2; + expFullSbCnt = 1; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + + strcpy(testCaseDesc, "Create 1 index key with 63 row IDs, expecting 2 sub block and 2 full sub block."); + testCaseNo = 10529; + numOfRowID = 63; + expSbCnt = 2; + expFullSbCnt = 2; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + } + /* ========================================================== + Test cases deleting index + ============================================================= */ + void qaTSCIndexTestDelete() + { + uint64_t idxKey = 1; + int tokenLen = 16; + int testCaseNo; + int startRowID = 1; + int numOfRowID; + int expSbCnt; + int expFullSbCnt; + char testCaseDesc[256]; + + int i; + int rc; + + // Create 1 row IDs, then remove it. + // Expecting 0 row IDs, 0 sub blocks and 0 full sub blocks. + + strcpy(testCaseDesc, + "Create 1 row ID, then remove it. Expecting 0 row ID, 0 sub block and 0 full sub block."); + testCaseNo = 10530; + numOfRowID = 1; + expSbCnt = 0; + expFullSbCnt = 0; + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_index.deleteIndex(idxKey, tokenLen, startRowID); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_index.closeIndex(); + + numOfRowID = 0; + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + // qaDspIndexListStatsByKey(idxKey, tokenLen); + + // Create 3 row IDs, then remove the first two. + // Expecting 1 row IDs, 1 sub blocks and 0 full sub blocks. + + strcpy(testCaseDesc, + "Create 3 row IDs, then remove the first two. Expecting 1 row ID, 1 sub block and 0 full sub " + "block."); + testCaseNo = 10531; + numOfRowID = 3; + expSbCnt = 1; + expFullSbCnt = 0; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = startRowID; i < startRowID + numOfRowID - 1; i++) { + rc = m_index.deleteIndex(idxKey, tokenLen, i); + CPPUNIT_ASSERT(rc == NO_ERROR); } - void tearDown() + m_index.closeIndex(); + + numOfRowID = 1; + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + // qaDspIndexListStatsByKey(idxKey, tokenLen); + + // Create 100 row IDs, then remove them. + // Expecting 0 row IDs, 0 sub blocks and 0 full sub blocks. + + strcpy(testCaseDesc, + "Create 100 row IDs, then remove them. Expecting 0 row ID, 0 sub block and 0 full sub block."); + testCaseNo = 10532; + numOfRowID = 100; + expSbCnt = 0; + expFullSbCnt = 0; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = startRowID; i < startRowID + numOfRowID; i++) { + rc = m_index.deleteIndex(idxKey, tokenLen, i); + CPPUNIT_ASSERT(rc == NO_ERROR); } + m_index.closeIndex(); - /* ========================================================== - Create maximum number of row ids for key and verify all - index list header records - ============================================================= */ + numOfRowID = 0; + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + // qaDspIndexListStatsByKey(idxKey, tokenLen); - void qaTSCStressTestMaxRowID() + // create 63 row ids so that two full sub blocks will be allocated. + // delete all row ids, 2 to 32, in the first sub block + + strcpy(testCaseDesc, + "Create 63 row ids, then remove all in the 2nd sub block. Verify that row id sub block gets " + "released when empty."); + testCaseNo = 10533; + numOfRowID = 63; + expSbCnt = 1; + expFullSbCnt = 1; + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 2; i < 33; i++) { - - IdxFreeSpaceStat mapInfo; - - uint64_t idxKey = 1; - int testCaseNo; - int numOfRowID; - int expSbCnt; - int expFullSbCnt; - char testCaseDesc[256]; - - int tokenLen = 1; - int rc; - int execMode; - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - execMode = 0; - - strcpy(testCaseDesc, "8 bit key, max row stress test"); - - tokenLen = 8; - testCaseNo = 10520; - numOfRowID = 256; - expSbCnt = 9; - expFullSbCnt = 8; - - qaSupCreateMaxRowID(idxKey, tokenLen); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - qaDspIndexListByKey(idxKey, tokenLen); - - - strcpy(testCaseDesc, "16 bit key, max row stress test"); - - testCaseNo = 10521; - numOfRowID = 65536; - expSbCnt = 2115; - expFullSbCnt = 2114; - - qaSupCreateMaxRowID(idxKey, tokenLen); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - - } - /* ========================================================== - Create maximum number key for a given key length and - verify each index list header record - ============================================================= */ - - void qaTSCStressTestMaxKey() - { - - IdxFreeSpaceStat mapInfo; - - int testCaseNo; - char testCaseDesc[256]; - int tokenLen; - int entryCnt; - int execMode; - - execMode = 1; -// qaSupCreateMaxHdrRec(tokenLen); - - strcpy(testCaseDesc, "8 bit key, max key stress test"); - - tokenLen = 8; - testCaseNo = 10522; - - qaSupCreateMaxHdrRec(tokenLen); - qaTstStressMaxKeyDriver(testCaseNo, testCaseDesc, tokenLen); - - -// qaSupCreateMaxHdrRec(tokenLen); - - strcpy(testCaseDesc, "16 bit key, max key stress test"); - - tokenLen = 16; - testCaseNo = 10523; - - qaSupCreateMaxHdrRec(tokenLen); - qaTstStressMaxKeyDriver(testCaseNo, testCaseDesc, tokenLen); - + rc = m_index.deleteIndex(idxKey, tokenLen, i); + CPPUNIT_ASSERT(rc == NO_ERROR); } - /* ========================================================== - Test cases for for updating index - ============================================================= */ - void qaTSCIndexTestUpdate() + m_index.closeIndex(); + + numOfRowID = 32; + qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); + } + /* ========================================================== + Test cases for empty maps + ============================================================= */ + + void qaTSCFreeSpaceMapTest() + { + IdxFreeSpaceStat mapInfo; + uint64_t idxKey = 1; + int tokenLen = 16; + int execMode = 0; + int testCaseNo, startKey, numOfKey; + int startRowID, numOfRowID; + int i; + int expMapSize, expEntryCnt; + char testCaseDesc[256]; + + strcpy(testCaseDesc, "No keys created. There should not be a empty list for header record."); + testCaseNo = 10534; + startKey = 1; + numOfKey = 0; + expMapSize = 0; + expEntryCnt = 0; + + qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); + qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); + qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); + + strcpy(testCaseDesc, "Create 1 key. There will be space left for 7 more header records."); + testCaseNo = 10535; + startKey = 1; + numOfKey = 1; + expMapSize = 1; + expEntryCnt = 7; + + qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); + qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); + qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); + + strcpy(testCaseDesc, "Create 8 row ids. There will be space left for 0 more header records."); + testCaseNo = 10536; + startKey = 1; + numOfKey = 8; + expMapSize = 1; + expEntryCnt = 0; + + qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); + qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); + qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); + + strcpy(testCaseDesc, "Create 9 key. There will be space left for 7 more header records."); + testCaseNo = 10537; + startKey = 1; + numOfKey = 1; + expMapSize = 1; + expEntryCnt = 7; + + qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); + qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); + qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); + /* + strcpy(testCaseDesc, "Verify that a sub block holding empty list entries will be released + when empty."); testCaseNo = 10538; startRowID = 1; expMapSize = 0; + expEntryCnt = 0; + //Create files without key records, then get number of 32-entry free blocks. + numOfRowID = 0; + qaSupCreateRangeKey(idxKey, tokenLen, startRowID, numOfRowID); + qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); + int initialSize = mapInfo.mapSize; + int initialEntry = mapInfo.entryCnt; + execMode = 1; + qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); + printf(" size: %d\n", mapInfo.mapSize); + printf(" entry: %d\n", mapInfo.entryCnt); + + numOfRowID = 900; //There are 31 entries in a 32-entry empty list sub block. + //1 ptr allocated for 4-entry free map, 1 + for hdr, 29 entries remaining. + //29 entriess, pointing 29 sub blocks. 29 + x 31 = 899, plus 1 in hdr = 900 entries qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, + numOfRowID); qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); expMapSize + = initialSize - 1 - 1 + 1; // - 1 hdr block, -1 4-entry list, + 1 free 32-entry block. + expEntryCnt = initialEntry - 29 - 1 - 1 + 1; // -29 data sub blocks, - 1 for + 4-entry list ptr, 1 for hdr record ptr, +1 free 32-entry ptr + expEntryCnt++ ; // add 1 for the free 32-entry sub + block. + + qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); + qaDspIndexListByKey(idxKey, tokenLen); + qaDspIndexListStatsByKey(idxKey, tokenLen); + + */ + + execMode = 1; + // qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); + + // qaDspIndexListByKey(idxKey, tokenLen); + // qaDspIndexListStatsByKey(idxKey, tokenLen); + } + + /* ========================================================== + verify each index list header record for a given key length + ============================================================= */ + + void qaTstStressMaxKeyDriver(int testCaseNo, char* testCaseDesc, int tokenLen) + { + bool rc, passed, ; + int checkMode = 2; + int i, maxKey; + int numOfRowID = 1; + int expSbCnt = 0; + int expFullSbCnt = 0; + + qaDspTestStatusHeader(testCaseNo, testCaseDesc); + + maxKey = (1 << tokenLen); + + if (maxKey > 160) + maxKey = 120; + + passed = true; + + for (i = 0; i < maxKey; i++) { + rc = qaSupVerifyIndexByKey(i, tokenLen, checkMode, numOfRowID, expSbCnt, expFullSbCnt); - uint64_t idxKey = 1; - int tokenLen = 16; - int testCaseNo; - int startRowID = 1; - int numOfRowID; - int expSbCnt; - int expFullSbCnt; - char testCaseDesc[256]; - - strcpy(testCaseDesc, "Create 1 index key with one row ID, expecting no sub block and no full sub block."); - - testCaseNo = 10524; - numOfRowID = 1; - expSbCnt = 0; - expFullSbCnt = 0; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - strcpy(testCaseDesc, "Create 2 index key with one row ID, expecting no sub block and no full sub block."); - - testCaseNo = 10525; - numOfRowID = 2; - expSbCnt = 0; - expFullSbCnt = 0; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - strcpy(testCaseDesc, "Create 3 index key with one row ID, expecting 1 sub block and no full sub block."); - - testCaseNo = 10526; - numOfRowID = 3; - expSbCnt = 1; - expFullSbCnt = 0; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - strcpy(testCaseDesc, "Create 1 index key with 32 row IDs, expecting 1 sub block and 1 full sub block."); - - testCaseNo = 10527; - numOfRowID = 32; - expSbCnt = 1; - expFullSbCnt = 1; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - strcpy(testCaseDesc, "Create 1 index key with 33 row IDs, expecting 2 sub block and 1 full sub block."); - - testCaseNo = 10528; - numOfRowID = 33; - expSbCnt = 2; - expFullSbCnt = 1; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - strcpy(testCaseDesc, "Create 1 index key with 63 row IDs, expecting 2 sub block and 2 full sub block."); - testCaseNo = 10529; - numOfRowID = 63; - expSbCnt = 2; - expFullSbCnt = 2; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - } - /* ========================================================== - Test cases deleting index - ============================================================= */ - void qaTSCIndexTestDelete() - { - - - uint64_t idxKey = 1; - int tokenLen = 16; - int testCaseNo; - int startRowID = 1; - int numOfRowID; - int expSbCnt; - int expFullSbCnt; - char testCaseDesc[256]; - - int i; - int rc; - - //Create 1 row IDs, then remove it. - //Expecting 0 row IDs, 0 sub blocks and 0 full sub blocks. - - strcpy(testCaseDesc, "Create 1 row ID, then remove it. Expecting 0 row ID, 0 sub block and 0 full sub block."); - testCaseNo = 10530; - numOfRowID = 1; - expSbCnt = 0; - expFullSbCnt = 0; - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_index.deleteIndex(idxKey, tokenLen, startRowID ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_index.closeIndex(); - - numOfRowID = 0; - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); -// qaDspIndexListStatsByKey(idxKey, tokenLen); - - //Create 3 row IDs, then remove the first two. - //Expecting 1 row IDs, 1 sub blocks and 0 full sub blocks. - - strcpy(testCaseDesc, "Create 3 row IDs, then remove the first two. Expecting 1 row ID, 1 sub block and 0 full sub block."); - testCaseNo = 10531; - numOfRowID = 3; - expSbCnt = 1; - expFullSbCnt = 0; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = startRowID; i < startRowID + numOfRowID - 1; i++) - { - rc = m_index.deleteIndex(idxKey, tokenLen, i ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - numOfRowID = 1; - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); -// qaDspIndexListStatsByKey(idxKey, tokenLen); - - - //Create 100 row IDs, then remove them. - //Expecting 0 row IDs, 0 sub blocks and 0 full sub blocks. - - strcpy(testCaseDesc, "Create 100 row IDs, then remove them. Expecting 0 row ID, 0 sub block and 0 full sub block."); - testCaseNo = 10532; - numOfRowID = 100; - expSbCnt = 0; - expFullSbCnt = 0; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = startRowID; i < startRowID + numOfRowID; i++) - { - rc = m_index.deleteIndex(idxKey, tokenLen, i ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - numOfRowID = 0; - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); -// qaDspIndexListStatsByKey(idxKey, tokenLen); - - //create 63 row ids so that two full sub blocks will be allocated. - //delete all row ids, 2 to 32, in the first sub block - - strcpy(testCaseDesc, "Create 63 row ids, then remove all in the 2nd sub block. Verify that row id sub block gets released when empty."); - testCaseNo = 10533; - numOfRowID = 63; - expSbCnt = 1; - expFullSbCnt = 1; - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = 2; i < 33; i++) - { - rc = m_index.deleteIndex(idxKey, tokenLen, i ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - numOfRowID = 32; - qaTstIndexListDriver(testCaseNo, testCaseDesc, idxKey, tokenLen, numOfRowID, expSbCnt, expFullSbCnt); - - } - /* ========================================================== - Test cases for empty maps - ============================================================= */ - - void qaTSCFreeSpaceMapTest() - { - - IdxFreeSpaceStat mapInfo; - uint64_t idxKey = 1; - int tokenLen = 16; - int execMode = 0; - int testCaseNo, startKey, numOfKey; - int startRowID, numOfRowID; - int i; - int expMapSize, expEntryCnt; - char testCaseDesc[256]; - - strcpy(testCaseDesc, "No keys created. There should not be a empty list for header record."); - testCaseNo = 10534; - startKey = 1; - numOfKey = 0; - expMapSize = 0; - expEntryCnt = 0; - - qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); - qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); - - strcpy(testCaseDesc, "Create 1 key. There will be space left for 7 more header records."); - testCaseNo = 10535; - startKey = 1; - numOfKey = 1; - expMapSize = 1; - expEntryCnt = 7; - - qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); - qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); - - strcpy(testCaseDesc, "Create 8 row ids. There will be space left for 0 more header records."); - testCaseNo = 10536; - startKey = 1; - numOfKey = 8; - expMapSize = 1; - expEntryCnt = 0; - - - qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); - qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); - - strcpy(testCaseDesc, "Create 9 key. There will be space left for 7 more header records."); - testCaseNo = 10537; - startKey = 1; - numOfKey = 1; - expMapSize = 1; - expEntryCnt = 7; - - qaSupCreateRangeKey(idxKey, tokenLen, startKey, numOfKey); - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_4ENTRY, execMode, mapInfo); - qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); - /* - strcpy(testCaseDesc, "Verify that a sub block holding empty list entries will be released when empty."); - testCaseNo = 10538; - startRowID = 1; - expMapSize = 0; - expEntryCnt = 0; - //Create files without key records, then get number of 32-entry free blocks. - numOfRowID = 0; - qaSupCreateRangeKey(idxKey, tokenLen, startRowID, numOfRowID); - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); - int initialSize = mapInfo.mapSize; - int initialEntry = mapInfo.entryCnt; - execMode = 1; - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); - printf(" size: %d\n", mapInfo.mapSize); - printf(" entry: %d\n", mapInfo.entryCnt); - - numOfRowID = 900; //There are 31 entries in a 32-entry empty list sub block. - //1 ptr allocated for 4-entry free map, 1 for hdr, 29 entries remaining. - //29 entriess, pointing 29 sub blocks. 29 x 31 = 899, plus 1 in hdr = 900 entries - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, numOfRowID); - qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); - expMapSize = initialSize - 1 - 1 + 1; // - 1 hdr block, -1 4-entry list, + 1 free 32-entry block. - expEntryCnt = initialEntry - 29 - 1 - 1 + 1; // -29 data sub blocks, - 1 for 4-entry list ptr, 1 for hdr record ptr, +1 free 32-entry ptr - expEntryCnt++ ; // add 1 for the free 32-entry sub block. - - qaTstFreeMapDriver(testCaseNo, testCaseDesc, expMapSize, expEntryCnt, mapInfo); - qaDspIndexListByKey(idxKey, tokenLen); - qaDspIndexListStatsByKey(idxKey, tokenLen); - - */ - - - execMode = 1; -// qaDspFreeSpaceMap(QA_FILE_TYPE_LIST, QA_FREEMAP_32ENTRY, execMode, mapInfo); - -// qaDspIndexListByKey(idxKey, tokenLen); -// qaDspIndexListStatsByKey(idxKey, tokenLen); - - + if (rc != NO_ERROR) + passed = false; } - /* ========================================================== - verify each index list header record for a given key length - ============================================================= */ + qaDspTestStatusFooter(testCaseNo, passed); + } - void qaTstStressMaxKeyDriver(int testCaseNo, char* testCaseDesc, int tokenLen) + /* ========================================================== + A driver function to execution test cases for the + index list. + ============================================================= */ + void qaTstIndexListDriver(int testCaseNo, char* testCaseDesc, uint64_t idxKey, int tokenLen, int numOfRowID, + int expSbCnt, int expFullSbCnt) + { + bool passed; + int checkMode = 2; + + qaDspTestStatusHeader(testCaseNo, testCaseDesc); + passed = qaSupVerifyIndexByKey(idxKey, tokenLen, checkMode, numOfRowID, expSbCnt, expFullSbCnt); + qaDspTestStatusFooter(testCaseNo, passed); + } + + /* ========================================================== + A driver function to verify free map list + ============================================================= */ + void qaTstFreeMapDriver(int testCaseNo, char* testCaseDesc, int expMapSize, int expEntryCnt, + IdxFreeSpaceStat mapInfo) + { + int passed = true; + + qaDspTestStatusHeader(testCaseNo, testCaseDesc); + + if (mapInfo.mapSize != expMapSize) { - bool rc, passed, ; - int checkMode = 2; - int i, maxKey; - int numOfRowID = 1; - int expSbCnt = 0; - int expFullSbCnt = 0; - - qaDspTestStatusHeader(testCaseNo, testCaseDesc); - - maxKey = (1 << tokenLen); - - if (maxKey > 160) maxKey = 120; - - passed = true; - - for (i = 0; i < maxKey; i++) - { - rc = qaSupVerifyIndexByKey(i, tokenLen, checkMode, numOfRowID, expSbCnt, expFullSbCnt); - - if (rc != NO_ERROR) passed = false; - } - - qaDspTestStatusFooter(testCaseNo, passed); + printf(" Free map size mismatch: Expected: %d Returned: %d\n", expMapSize, + (int)mapInfo.mapSize); + passed = false; } - /* ========================================================== - A driver function to execution test cases for the - index list. - ============================================================= */ - void qaTstIndexListDriver(int testCaseNo, char* testCaseDesc, uint64_t idxKey, int tokenLen, int numOfRowID, int expSbCnt, int expFullSbCnt) + if (mapInfo.entryCnt != expEntryCnt) { - - bool passed; - int checkMode = 2; - - qaDspTestStatusHeader(testCaseNo, testCaseDesc); - passed = qaSupVerifyIndexByKey(idxKey, tokenLen, checkMode, numOfRowID, expSbCnt, expFullSbCnt); - qaDspTestStatusFooter(testCaseNo, passed); + printf(" Free map entry mismatch: Expected: %d Returned: %d\n", expEntryCnt, + (int)mapInfo.entryCnt); + passed = false; } - /* ========================================================== - A driver function to verify free map list - ============================================================= */ - void qaTstFreeMapDriver(int testCaseNo, char* testCaseDesc, int expMapSize, int expEntryCnt, IdxFreeSpaceStat mapInfo) + qaDspTestStatusFooter(testCaseNo, passed); + } + /* ========================================================== + Verify index list record with it's integral components + and expected values from the calling program. + ============================================================= */ + bool qaSupVerifyIndexByKey(uint64_t idxKey, int tokenLen, int checkMode, int expRowIDCnt, int expSbCnt, + int expFullSbCnt) + { + IdxListStat idxKeyInfo; + + int execMode = 0; + int passed = true; + + // checkMode != 1 Verify index header information only + // checkMode = 2 verify header information, plus matching expected results. + + qaSupWalkIndexList(idxKey, tokenLen, execMode, idxKeyInfo); + + // verify for matching record keys + if (idxKeyInfo.hdrRec.key != idxKey) { - - int passed = true; - - qaDspTestStatusHeader(testCaseNo, testCaseDesc); - - if (mapInfo.mapSize != expMapSize) - { - printf(" Free map size mismatch: Expected: %d Returned: %d\n", expMapSize, (int)mapInfo.mapSize ); - passed = false; - } - - if (mapInfo.entryCnt != expEntryCnt) - { - printf(" Free map entry mismatch: Expected: %d Returned: %d\n", expEntryCnt, (int)mapInfo.entryCnt ); - passed = false; - } - - qaDspTestStatusFooter(testCaseNo, passed); - } - /* ========================================================== - Verify index list record with it's integral components - and expected values from the calling program. - ============================================================= */ - bool qaSupVerifyIndexByKey(uint64_t idxKey, int tokenLen, int checkMode, int expRowIDCnt, int expSbCnt, int expFullSbCnt) - { - - IdxListStat idxKeyInfo; - - int execMode = 0; - int passed = true; - - //checkMode != 1 Verify index header information only - //checkMode = 2 verify header information, plus matching expected results. - - - qaSupWalkIndexList(idxKey, tokenLen, execMode, idxKeyInfo); - - // verify for matching record keys - if (idxKeyInfo.hdrRec.key != idxKey) - { - printf(" Index key mismatch: Expected: %d Returned: %d\n", (int)idxKey, (int)idxKeyInfo.hdrRec.key ); - passed = false; - } - - // verify row id count in the hdr record against row id count gathered from the index list record. - if (idxKeyInfo.hdrRec.idxRidListSize.size != idxKeyInfo.recCnt) - { - printf(" Index Hdr cnt mismatch: In Hdr : %d In SB: %d\n", idxKeyInfo.hdrRec.idxRidListSize.size, idxKeyInfo.recCnt ); - passed = false; - } - - // verify that when there is not sub blocks for the row IDs, the header's link list pointer should not be used. - if ((idxKeyInfo.sbCnt == 0) && (idxKeyInfo.hdrRec.nextIdxRidListPtr.type == 4)) - { - printf(" Incorrect list pointer: Ptr type: %d SB Cnt: %d\n", idxKeyInfo.hdrRec.idxRidListSize.size, idxKeyInfo.recCnt ); - passed = false; - } - - if (checkMode = 2) - { - - // verify row id count in the hdr record against expected row id count - if (idxKeyInfo.hdrRec.idxRidListSize.size != expRowIDCnt) - { - printf(" Row ID cnt mismatch: Expected: %d Returned: %d\n", expRowIDCnt, idxKeyInfo.hdrRec.idxRidListSize.size); - passed = false; - } - - // verify sub block count gathered from the index list record against the expected sub block count - if (idxKeyInfo.sbCnt != expSbCnt) - { - printf(" Subblock cnt mismatch: Expected: %d Returned: %d\n", expSbCnt, idxKeyInfo.sbCnt ); - passed = false; - } - - // verify full sub block count gathered from the index list record against the expected full sub block count - if (idxKeyInfo.fullSbCnt != expFullSbCnt) - { - printf(" Full subblock cnt mismatch: Expected: %d Returned: %d\n", expFullSbCnt, idxKeyInfo.fullSbCnt ); - passed = false; - } - } - - return passed; - + printf(" Index key mismatch: Expected: %d Returned: %d\n", (int)idxKey, + (int)idxKeyInfo.hdrRec.key); + passed = false; } - - - /* ========================================================== - Display index record for a given key - ============================================================= */ - - void qaGetListSubblockByKey(uint64_t idxKey, int tokenLen, int sbSeqNo, int& fbo, int& sbid, int& entry) + // verify row id count in the hdr record against row id count gathered from the index list record. + if (idxKeyInfo.hdrRec.idxRidListSize.size != idxKeyInfo.recCnt) { - - int p_fbo, p_sbid, p_entry; - int execMode = 1; - bool exist; - -// qaSupWalkIndexList(idxKey, tokenLen, execMode, sbSeqNo, p_fbo, p_sbid, p_entry, hdrCntMatch, hdrRecCnt, hdrSbCnt, hdrFullSbCnt); - fbo = p_fbo; - sbid = p_sbid; - entry = p_entry; + printf(" Index Hdr cnt mismatch: In Hdr : %d In SB: %d\n", + idxKeyInfo.hdrRec.idxRidListSize.size, idxKeyInfo.recCnt); + passed = false; } - - /* ========================================================== - Display tree empty list map for a given block size - ============================================================= */ - void qaDspFreeSpaceMap(int fileType, int mapType, int execMode, IdxFreeSpaceStat& mapInfo) + // verify that when there is not sub blocks for the row IDs, the header's link list pointer should not be + // used. + if ((idxKeyInfo.sbCnt == 0) && (idxKeyInfo.hdrRec.nextIdxRidListPtr.type == 4)) { + printf(" Incorrect list pointer: Ptr type: %d SB Cnt: %d\n", + idxKeyInfo.hdrRec.idxRidListSize.size, idxKeyInfo.recCnt); + passed = false; + } - DataBlock curBlock; - FILE* pFile = NULL; + if (checkMode = 2) + { + // verify row id count in the hdr record against expected row id count + if (idxKeyInfo.hdrRec.idxRidListSize.size != expRowIDCnt) + { + printf(" Row ID cnt mismatch: Expected: %d Returned: %d\n", expRowIDCnt, + idxKeyInfo.hdrRec.idxRidListSize.size); + passed = false; + } - int fileNum; - int p_fbo, p_sbid, p_entry, p_type, p_group; - int i, j; - int p_mapType; - bool done; + // verify sub block count gathered from the index list record against the expected sub block + //count + if (idxKeyInfo.sbCnt != expSbCnt) + { + printf(" Subblock cnt mismatch: Expected: %d Returned: %d\n", expSbCnt, + idxKeyInfo.sbCnt); + passed = false; + } + // verify full sub block count gathered from the index list record against the expected full sub + //block count + if (idxKeyInfo.fullSbCnt != expFullSbCnt) + { + printf(" Full subblock cnt mismatch: Expected: %d Returned: %d\n", expFullSbCnt, + idxKeyInfo.fullSbCnt); + passed = false; + } + } - IdxEmptyListEntry emptyMap; + return passed; + } - if (fileType == QA_FILE_TYPE_TREE) - fileNum = 990; //index tree - else - fileNum = 991; //index list + /* ========================================================== + Display index record for a given key + ============================================================= */ - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( fileNum ); - CPPUNIT_ASSERT( pFile != NULL ); + void qaGetListSubblockByKey(uint64_t idxKey, int tokenLen, int sbSeqNo, int& fbo, int& sbid, int& entry) + { + int p_fbo, p_sbid, p_entry; + int execMode = 1; + bool exist; - //Free Manager - m_indexlist.setUseFreeMgr( true ); + // qaSupWalkIndexList(idxKey, tokenLen, execMode, sbSeqNo, p_fbo, p_sbid, p_entry, hdrCntMatch, + //hdrRecCnt, hdrSbCnt, hdrFullSbCnt); + fbo = p_fbo; + sbid = p_sbid; + entry = p_entry; + } - p_fbo = 0; - p_sbid = 0; - p_entry = mapType; //Entry location in Fb 0 and SB 0; + /* ========================================================== + Display tree empty list map for a given block size + ============================================================= */ + void qaDspFreeSpaceMap(int fileType, int mapType, int execMode, IdxFreeSpaceStat& mapInfo) + { + DataBlock curBlock; + FILE* pFile = NULL; + + int fileNum; + int p_fbo, p_sbid, p_entry, p_type, p_group; + int i, j; + int p_mapType; + bool done; + + IdxEmptyListEntry emptyMap; + + if (fileType == QA_FILE_TYPE_TREE) + fileNum = 990; // index tree + else + fileNum = 991; // index list + + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(fileNum); + CPPUNIT_ASSERT(pFile != NULL); + + // Free Manager + m_indexlist.setUseFreeMgr(true); + + p_fbo = 0; + p_sbid = 0; + p_entry = mapType; // Entry location in Fb 0 and SB 0; + + if (execMode == 1) + { + printf("\n------------------------------------\n"); + + if (fileType == 1) + printf("Index Tree %d-ENTRY free space map\n", 1 << (mapType - 1)); + else + printf("Index list %d-ENTRY free space map\n", 1 << (mapType - 1)); + + printf("------------------------------------\n"); + } + + m_indexlist.readSubBlockEntry(pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &emptyMap); + + if (execMode == 1) + { + // printf("\nList Ptr: Type = %d Group = %d fbo = %d sbid = %d Entry = %d\n", + //p_type, p_group, p_fbo, p_sbid, p_entry); + printf("\nList: Type = %d Group = %d Addr(%d, %d, %d) ----> Addr(%d, %d, %d)\n", + int(emptyMap.type), int(emptyMap.group), p_fbo, p_sbid, p_entry, (int)emptyMap.fbo, + (int)emptyMap.sbid, (int)emptyMap.entry); + printf("-------------------------------------------------------------------------\n"); + } + + p_type = emptyMap.type; + p_group = emptyMap.group; + p_fbo = emptyMap.fbo; + p_sbid = emptyMap.sbid; + p_entry = emptyMap.entry; + + p_mapType = emptyMap.type; + mapInfo.mapSize = 0; + mapInfo.entryCnt = 0; + + done = ((p_mapType == 1) && (p_fbo == 0) && (p_sbid == 0) && (p_entry == 0)); + + while (not done) + { + j = p_entry; + mapInfo.mapSize++; + + for (i = j; i >= 0; i--) + { + m_indexlist.readSubBlockEntry(pFile, &curBlock, p_fbo, p_sbid, i, 8, &emptyMap); if (execMode == 1) + // printf("EntryPtr: %d Type = %d Group = %d fbo = %d sbid = %d Entry = %d\n", + //i, int(emptyMap.type), int(emptyMap.group), (int)emptyMap.fbo, (int)emptyMap.sbid, + //(int)emptyMap.entry); + printf("Entry: %d Type = %d Group = %d Addr(%d, %d, %d) ----> Addr(%d, %d, %d)\n", i, + int(emptyMap.type), int(emptyMap.group), p_fbo, p_sbid, i, (int)emptyMap.fbo, + (int)emptyMap.sbid, (int)emptyMap.entry); + + if (i != 0) + mapInfo.entryCnt++; + } + + p_fbo = emptyMap.fbo; + p_sbid = emptyMap.sbid; + p_entry = emptyMap.entry; + p_mapType = emptyMap.type; + + done = ((p_fbo == 0) && (p_sbid == 0) && (p_entry == 0)); + + if (not done && (execMode == 1)) + { + // printf("\nList Ptr: Type: %d Group: %d FBO: %d SBID: %d ENTRY: %d\n", + //p_type, p_group, p_fbo, p_sbid, p_entry); + printf("\nList: Type = %d Group = %d Addr(%d, %d, %d) ----> Addr(%d, %d, %d)\n", + int(emptyMap.type), int(emptyMap.group), p_fbo, p_sbid, p_entry, (int)emptyMap.fbo, + (int)emptyMap.sbid, (int)emptyMap.entry); + printf("-------------------------------------------------------------------------\n"); + } + } + + m_indexlist.closeFile(pFile); + + } // qaDspFreeSpaceMap + + /* ========================================================== + Show branch list for all keys for a given key length + ============================================================= */ + + void qaDspTreePathAll(uint64_t idxKey, int tokenLen) + { + uint64_t i; + int maxKey; + + maxKey = (1 << tokenLen); + + for (i = 0; i < maxKey; i++) + qaDspTreePathByKey(idxKey, tokenLen); + } + + /******************************************************* + . Display branch list for a key + *******************************************************/ + void qaDspTreePathByKey(uint64_t idxKey, int tokenLen) + { + int p_fbo, p_sbid, p_entry; + bool exist; + int execMode = 1; + + qaSupWalkIndexTree(idxKey, tokenLen, execMode, p_fbo, p_sbid, p_entry, exist); + + } // qaDspTreePathByKey + /* ========================================================== + Verfiy hdr records all keys for a given key length + ============================================================= */ + void qaDspIndexListStatsByKey(uint64_t idxKey, int tokenLen) + { + IdxListStat idxKeyInfo; + + int execMode = 0; + int sbSeqNo = 0; + + qaSupWalkIndexList(idxKey, tokenLen, execMode, idxKeyInfo); + + printf("\nINDEX STATS for key: %d\n", (int)idxKey); + printf("--------------------------------------\n"); + printf("Key: %d\n", (int)idxKeyInfo.hdrRec.key); + printf("Hdr Record Count: %d\n", idxKeyInfo.hdrRec.idxRidListSize.size); + + if (idxKeyInfo.hdrRec.firstIdxRidListEntry.type == 3) + printf("RowID 0 : %d\n", idxKeyInfo.hdrRec.firstIdxRidListEntry.rid); + + if (idxKeyInfo.hdrRec.nextIdxRidListPtr.type == 3) + printf("RowID 1 : %d\n", (int)idxKeyInfo.hdrRec.nextIdxRidListPtr.llp); + + printf("File Block: %d\n", idxKeyInfo.fbo); + printf("Sub Block: %d\n", idxKeyInfo.sbid); + printf("Entry: %d\n", idxKeyInfo.entry); + printf("Actual Row ID Cnt: %d\n", idxKeyInfo.recCnt); + printf("Sub Block Cnt: %d\n", idxKeyInfo.sbCnt); + printf("Full Sub Block Cnt: %d\n", idxKeyInfo.fullSbCnt); + printf("Row ID Cnt Match?: "); + + if (idxKeyInfo.hdrRec.idxRidListSize.size == idxKeyInfo.recCnt) + printf("Yes\n"); + else + printf("No\n"); + } + /* ========================================================== + Display index record all keys for a given key length + ============================================================= */ + + void qaDspIndexListAll(uint64_t, int tokenLen) + { + uint64_t i; + int maxKey; + + maxKey = (1 << tokenLen); + + for (i = 0; i < maxKey; i++) + qaDspIndexListByKey(i, tokenLen); + } + + /* ========================================================== + Display index record for a given key + ============================================================= */ + void qaDspIndexListByKey(uint64_t idxKey, int tokenLen) + { + IdxListStat idxKeyInfo; + + int p_fbo, p_sbid, p_entry; + int execMode = 1; + bool exist; + int sbSeqNo = 0; + + qaSupWalkIndexList(idxKey, tokenLen, execMode, idxKeyInfo); + } + /* ========================================================== + Display test case status header + ============================================================= */ + void qaDspTestStatusHeader(int testCaseNo, char* testCaseDesc) + { + printf("\nBEGIN IdxHdrTestCase %d\n", testCaseNo); + printf(" Desc: %s\n", testCaseDesc); + } + + /* ========================================================== + Display test case status header + ============================================================= */ + void qaDspTestStatusFooter(int testCaseNo, bool passed) + { + if (passed) + printf(" Status: Passed\n"); + else + { + printf(" Status: Failed -- [**ERROR**]\n"); + } + + printf("END IdxHdrTestCase %d\n", testCaseNo); + } + /* ========================================================== + Populate tree with max key for a given bit length + Populate tree row ID for a given bitlength + ============================================================= */ + void qaSupCreateMaxRowID(uint64_t idxKey, int tokenLen) + { + int maxKey; + int startRowID = 0; + + maxKey = (1 << tokenLen); + + qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, maxKey); + + } // qaSupCreateMaxRowID + + /* ========================================================== + Populate tree with a range of row IDs for a key. + ============================================================= */ + void qaSupCreateRangeRowID(uint64_t idxKey, int tokenLen, int startRowID, int numOfRowID) + { + int rc; + int i; + int maxKey; + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + + // here is the correct check + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 0; i < numOfRowID; i++) + { + rc = m_index.updateIndex(idxKey, tokenLen, startRowID + i); // 1 key, multiple rows, multiple row ids + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + m_index.closeIndex(); + + } // qaSupCreateMaxRowID + + /* ========================================================== + Populate tree with max key for a given bit length + Populate tree row ID for a given bitlength + ============================================================= */ + void qaSupCreateMaxHdrRec(int tokenLen) + { + int rc; + uint64_t i; + int maxKey; + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + + // here is the correct check + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + maxKey = (1 << tokenLen); + + if (maxKey > 160) + maxKey = 160; + + for (i = 0; i < maxKey; i++) + rc = m_index.updateIndex(i, tokenLen, i); // 1 row id per key. + + m_index.closeIndex(); + + } // qaSupCreateMaxHdrRec + /* ========================================================== + Populate tree with range of keys + ============================================================= */ + + void qaSupCreateRangeKey(uint64_t idxKey, int tokenLen, int startKey, int numOfKey) + { + int rc; + uint64_t i; + int maxKey; + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + + // here is the correct check + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = startKey; i < startKey + numOfKey; i++) + rc = m_index.updateIndex(i, tokenLen, i); // 1 row id per key. + + m_index.closeIndex(); + + } // qaSupCreateRangeKey + + /******************************************************* + . This function navigate through the index list record. + few global variables are set relating to index list. + *******************************************************/ + void qaSupWalkIndexList(uint64_t idxKey, int tokenLen, int execMode, IdxListStat& idxKeyInfo) + { + DataBlock indexBlock; + FILE* pFileIndexList = NULL; + int file_indexList = 991; + + int p_fbo, p_sbid, p_entry; + int i, ; + int SBRecCnt; + bool endOfList; + bool exist; + int tempExecMode; + + IdxRidListEntry listEntry; + IdxRidListPtr listPtr; + IdxRidListHdr indexListHdrRec; + + // Free Manager + m_indexlist.setUseFreeMgr(true); + tempExecMode = execMode; + execMode = 0; + qaSupWalkIndexTree(idxKey, tokenLen, execMode, p_fbo, p_sbid, p_entry, exist); + execMode = tempExecMode; + + idxKeyInfo.fbo = p_fbo; // Save the address for the header + idxKeyInfo.sbid = p_sbid; + idxKeyInfo.entry = p_entry; + + if (not exist) + printf("Tree node not found: Key = %d\n", (int)idxKey); + else + { + // read index list header record + + i = 0; + + idxKeyInfo.sbCnt = 0; + idxKeyInfo.fullSbCnt = 0; + idxKeyInfo.recCnt = 0; // first entry in the hdr + endOfList = false; + + memset(indexBlock.data, 0, sizeof(indexBlock.data)); + pFileIndexList = m_indexlist.openFile(file_indexList); + CPPUNIT_ASSERT(pFileIndexList != NULL); + + m_indexlist.readSubBlockEntry(pFileIndexList, &indexBlock, p_fbo, p_sbid, p_entry, 32, + &indexListHdrRec); + + idxKeyInfo.hdrRec = indexListHdrRec; + + if (execMode == 1) + { + printf("\n---------------------------------\n"); + printf("Index record for key: %d\n", (int)idxKey); + printf("---------------------------------\n"); + printf("Size: %d\n", (int)indexListHdrRec.idxRidListSize.size); + printf("Key: %d\n", (int)indexListHdrRec.key); + printf("RowID 1: %d Type: %d\n", (int)indexListHdrRec.firstIdxRidListEntry.rid, + (int)indexListHdrRec.firstIdxRidListEntry.type); + + if (indexListHdrRec.nextIdxRidListPtr.type == 3) + printf("RowID 2: %d Type: %d\\nn", (int)indexListHdrRec.nextIdxRidListPtr.llp, + (int)indexListHdrRec.nextIdxRidListPtr.type); + else if (indexListHdrRec.nextIdxRidListPtr.type == 4) + printf("Ptr: Addr(%d, %d, %d) Type = %d\n\n", + (int)((IdxEmptyListEntry*)&(indexListHdrRec.nextIdxRidListPtr))->fbo, + (int)((IdxEmptyListEntry*)&(indexListHdrRec.nextIdxRidListPtr))->sbid, + (int)((IdxEmptyListEntry*)&(indexListHdrRec.nextIdxRidListPtr))->entry, + (int)indexListHdrRec.nextIdxRidListPtr.type); + } + + if (int(indexListHdrRec.firstIdxRidListEntry.type) == 3) + idxKeyInfo.recCnt++; + + if (indexListHdrRec.nextIdxRidListPtr.type == 3) + { + idxKeyInfo.recCnt++; + } + else if (indexListHdrRec.nextIdxRidListPtr.type == 4) + { + p_fbo = ((IdxEmptyListEntry*)&(indexListHdrRec.nextIdxRidListPtr))->fbo; + p_sbid = ((IdxEmptyListEntry*)&(indexListHdrRec.nextIdxRidListPtr))->sbid; + p_entry = ((IdxEmptyListEntry*)&(indexListHdrRec.nextIdxRidListPtr))->entry; + + while (not endOfList) { - printf("\n------------------------------------\n"); + idxKeyInfo.sbCnt++; + SBRecCnt = 0; + m_indexlist.readSubBlockEntry(pFileIndexList, &indexBlock, p_fbo, p_sbid, 31, 8, &listPtr); + printf("Index list subblock: #%d\n", idxKeyInfo.sbCnt); - if (fileType == 1) - printf("Index Tree %d-ENTRY free space map\n", 1 << (mapType - 1)); - else - printf("Index list %d-ENTRY free space map\n", 1 << (mapType - 1)); + for (i = 0; i <= 30; i++) + { + m_indexlist.readSubBlockEntry(pFileIndexList, &indexBlock, p_fbo, p_sbid, i, 8, &listEntry); - printf("------------------------------------\n"); - } - - m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &emptyMap); - - if (execMode == 1) - { -// printf("\nList Ptr: Type = %d Group = %d fbo = %d sbid = %d Entry = %d\n", p_type, p_group, p_fbo, p_sbid, p_entry); - printf("\nList: Type = %d Group = %d Addr(%d, %d, %d) ----> Addr(%d, %d, %d)\n", int(emptyMap.type), int(emptyMap.group), p_fbo, p_sbid, p_entry, (int)emptyMap.fbo, (int)emptyMap.sbid, (int)emptyMap.entry); - printf("-------------------------------------------------------------------------\n"); - } - - p_type = emptyMap.type; - p_group = emptyMap.group; - p_fbo = emptyMap.fbo; - p_sbid = emptyMap.sbid; - p_entry = emptyMap.entry; - - p_mapType = emptyMap.type; - mapInfo.mapSize = 0; - mapInfo.entryCnt = 0; - - done = ((p_mapType == 1) && (p_fbo == 0) && (p_sbid == 0) && (p_entry == 0)); - - while (not done ) - { - - j = p_entry; - mapInfo.mapSize++; - - for (i = j; i >= 0; i--) + if (listEntry.type == 3) { - m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, i, 8, &emptyMap); + SBRecCnt++; - if (execMode == 1) -// printf("EntryPtr: %d Type = %d Group = %d fbo = %d sbid = %d Entry = %d\n", i, int(emptyMap.type), int(emptyMap.group), (int)emptyMap.fbo, (int)emptyMap.sbid, (int)emptyMap.entry); - printf("Entry: %d Type = %d Group = %d Addr(%d, %d, %d) ----> Addr(%d, %d, %d)\n", i, int(emptyMap.type), int(emptyMap.group), p_fbo, p_sbid, i, (int)emptyMap.fbo, (int)emptyMap.sbid, (int)emptyMap.entry); - - if (i != 0) mapInfo.entryCnt++; + if (execMode == 1) + printf("Entry %d: Addr(%d, %d, %d) RowId: %d \n", i, p_fbo, p_sbid, i, listEntry.rid); } + } - p_fbo = emptyMap.fbo; - p_sbid = emptyMap.sbid; - p_entry = emptyMap.entry; - p_mapType = emptyMap.type; + p_fbo = ((IdxEmptyListEntry*)&(listPtr))->fbo; + p_sbid = ((IdxEmptyListEntry*)&(listPtr))->sbid; - done = ( (p_fbo == 0) && (p_sbid == 0) && (p_entry == 0)); + if ((idxKeyInfo.sbSeqNo != 0) && (idxKeyInfo.sbSeqNo == (idxKeyInfo.sbCnt - 1))) + { + idxKeyInfo.fbo = p_fbo; // Save the address for the request block + idxKeyInfo.sbid = p_sbid; + idxKeyInfo.entry = p_entry; + } - if (not done && (execMode == 1)) - { -// printf("\nList Ptr: Type: %d Group: %d FBO: %d SBID: %d ENTRY: %d\n", p_type, p_group, p_fbo, p_sbid, p_entry); - printf("\nList: Type = %d Group = %d Addr(%d, %d, %d) ----> Addr(%d, %d, %d)\n", int(emptyMap.type), int(emptyMap.group), p_fbo, p_sbid, p_entry, (int)emptyMap.fbo, (int)emptyMap.sbid, (int)emptyMap.entry); - printf("-------------------------------------------------------------------------\n"); - } + if (execMode == 1) + printf("Entry 31: Addr(%d, %d, %d) RecCnt = %d\n\n", p_fbo, p_sbid, p_entry, + (int)listPtr.count); + + idxKeyInfo.recCnt += SBRecCnt; + + if (SBRecCnt == 31) + idxKeyInfo.fullSbCnt++; + + if (SBRecCnt != listPtr.count) + printf("[**ERROR**] Sub Block entry count Error: In Ptr Rec: %d In sub block: %d\n", + listPtr.count, SBRecCnt); + + endOfList = (listPtr.type == 0); } - - m_indexlist.closeFile(pFile); - - } // qaDspFreeSpaceMap - - /* ========================================================== - Show branch list for all keys for a given key length - ============================================================= */ - - void qaDspTreePathAll(uint64_t idxKey, int tokenLen) - { - - uint64_t i; - int maxKey; - - maxKey = (1 << tokenLen); - - for (i = 0; i < maxKey; i++) - qaDspTreePathByKey(idxKey, tokenLen); + } } - /******************************************************* - . Display branch list for a key - *******************************************************/ - void qaDspTreePathByKey(uint64_t idxKey, int tokenLen) + m_indexlist.closeFile(pFileIndexList); + } + + /******************************************************* + . This function navigate through the tree, from the + bitmap entry to the leaf. It can be used for many + functions that require access to the hdr record also. + *******************************************************/ + void qaSupWalkIndexTree(uint64_t idxKey, int tokenLen, int execMode, int& fbo, int& sbid, int& entry, + bool& exist) + { + DataBlock curBlock; + FILE* pFile = NULL; + int file_number = 990; + + IdxBitmapPointerEntry bitmapRec; + IdxBitTestEntry treeNodeRec; + int p_fbo, p_sbid, p_entry, p_type, p_group, p_bitTest; + int i; + int maskValue = 31; // mask for 11111. + int bitMapLoc, key1, shiftBits; + int bitCnt; + bool found; + int mask[5] = {1, 3, 7, 15, 31}; + int group[7] = {1, 2, 4, 8, 16, 32, 64}; + + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + + // Free Manager + m_indexlist.setUseFreeMgr(true); + + if (tokenLen > 5) + bitMapLoc = ((idxKey >> tokenLen - 5) & maskValue); + else + bitMapLoc = idxKey; + + p_fbo = 0; + p_sbid = 1; + p_entry = bitMapLoc; + + if (execMode == 1) { - - int p_fbo, p_sbid, p_entry; - bool exist; - int execMode = 1; - - qaSupWalkIndexTree(idxKey, tokenLen, execMode, p_fbo, p_sbid, p_entry, exist); - - } // qaDspTreePathByKey - /* ========================================================== - Verfiy hdr records all keys for a given key length - ============================================================= */ - void qaDspIndexListStatsByKey(uint64_t idxKey, int tokenLen) - { - - IdxListStat idxKeyInfo; - - int execMode = 0; - int sbSeqNo = 0; - - qaSupWalkIndexList (idxKey, tokenLen, execMode, idxKeyInfo); - - printf("\nINDEX STATS for key: %d\n", (int)idxKey); - printf("--------------------------------------\n"); - printf("Key: %d\n", (int)idxKeyInfo.hdrRec.key ); - printf("Hdr Record Count: %d\n", idxKeyInfo.hdrRec.idxRidListSize.size ); - - if (idxKeyInfo.hdrRec.firstIdxRidListEntry.type == 3) - printf("RowID 0 : %d\n", idxKeyInfo.hdrRec.firstIdxRidListEntry.rid); - - if (idxKeyInfo.hdrRec.nextIdxRidListPtr.type == 3) - printf("RowID 1 : %d\n", (int)idxKeyInfo.hdrRec.nextIdxRidListPtr.llp); - - printf("File Block: %d\n", idxKeyInfo.fbo); - printf("Sub Block: %d\n", idxKeyInfo.sbid); - printf("Entry: %d\n", idxKeyInfo.entry); - printf("Actual Row ID Cnt: %d\n", idxKeyInfo.recCnt); - printf("Sub Block Cnt: %d\n", idxKeyInfo.sbCnt); - printf("Full Sub Block Cnt: %d\n", idxKeyInfo.fullSbCnt); - printf("Row ID Cnt Match?: "); - - if (idxKeyInfo.hdrRec.idxRidListSize.size == idxKeyInfo.recCnt) - printf("Yes\n"); - else - printf("No\n"); - } - /* ========================================================== - Display index record all keys for a given key length - ============================================================= */ - - void qaDspIndexListAll(uint64_t, int tokenLen) - { - uint64_t i; - int maxKey; - - maxKey = (1 << tokenLen); - - for (i = 0; i < maxKey; i++) - qaDspIndexListByKey(i, tokenLen); + printf("\n---------------------------------\n"); + printf("Tree branch path for key: %d\n", (int)idxKey); + printf("---------------------------------\n"); } - /* ========================================================== - Display index record for a given key - ============================================================= */ - void qaDspIndexListByKey(uint64_t idxKey, int tokenLen) + m_indexlist.readSubBlockEntry(pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &bitmapRec); + + shiftBits = 1; + p_type = bitmapRec.type; + p_fbo = bitmapRec.fbo; + p_sbid = bitmapRec.sbid; + p_entry = bitmapRec.entry; + + if (execMode == 1) // branch list + printf("Bitmap: Type= %d BitTest= %d FBO= %d SBID= %d Entry= %d\n", int(p_type), + int(p_bitTest), (int)p_fbo, (int)p_sbid, (int)p_entry); + + bitCnt = tokenLen - 5; + + while (bitCnt > 0) { + if (bitCnt > 5) + key1 = ((idxKey >> (bitCnt - 5)) & maskValue); + else + key1 = ((idxKey)&mask[bitCnt - 1]); - IdxListStat idxKeyInfo; + m_indexlist.readSubBlockEntry(pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &treeNodeRec); - int p_fbo, p_sbid, p_entry; - int execMode = 1; - bool exist; - int sbSeqNo = 0; + int groupIdx = treeNodeRec.group; + // printf("group: %d count: %d\n", groupIdx, group[groupIdx]); - qaSupWalkIndexList (idxKey, tokenLen, execMode, idxKeyInfo); - } - /* ========================================================== - Display test case status header - ============================================================= */ - void qaDspTestStatusHeader(int testCaseNo, char* testCaseDesc) - { + found = false; - printf("\nBEGIN IdxHdrTestCase %d\n", testCaseNo); - printf(" Desc: %s\n", testCaseDesc); + for (i = 0; i < group[groupIdx]; i++) + { + m_indexlist.readSubBlockEntry(pFile, &curBlock, p_fbo, p_sbid, p_entry + i, 8, &treeNodeRec); + found = (treeNodeRec.bitTest == key1); + p_type = treeNodeRec.type; + + // printf("key1: %d bitTest: %d\n", key1, treeNodeRec.bitTest); + if (found) + i = group[groupIdx]; + }; + + // m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, + //&treeNodeRec); + + p_type = treeNodeRec.type; + + p_group = treeNodeRec.group; + + p_bitTest = treeNodeRec.bitTest; + + p_fbo = treeNodeRec.fbo; + + p_sbid = treeNodeRec.sbid; + + p_entry = treeNodeRec.entry; + + bitCnt = bitCnt - 5; + + if (execMode == 1) // branch list + printf("Level: Type= %d Group= %d BitTest= %d FBO= %d SBID= %d Entry= %d\n", int(p_type), + int(p_group), int(p_bitTest), (int)p_fbo, (int)p_sbid, (int)p_entry); } - /* ========================================================== - Display test case status header - ============================================================= */ - void qaDspTestStatusFooter(int testCaseNo, bool passed) - { + // set global variables so that calling functions can access to the address of the index header record - if (passed) - printf(" Status: Passed\n"); - else - { - printf(" Status: Failed -- [**ERROR**]\n"); - } - - printf("END IdxHdrTestCase %d\n", testCaseNo); - } - /* ========================================================== - Populate tree with max key for a given bit length - Populate tree row ID for a given bitlength - ============================================================= */ - void qaSupCreateMaxRowID(uint64_t idxKey, int tokenLen) - { - - int maxKey; - int startRowID = 0; - - maxKey = (1 << tokenLen); - - qaSupCreateRangeRowID(idxKey, tokenLen, startRowID, maxKey); - - } //qaSupCreateMaxRowID - - /* ========================================================== - Populate tree with a range of row IDs for a key. - ============================================================= */ - void qaSupCreateRangeRowID(uint64_t idxKey, int tokenLen, int startRowID, int numOfRowID) - { - - int rc; - int i; - int maxKey; - - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - - // here is the correct check - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = 0; i < numOfRowID; i++) - { - rc = m_index.updateIndex( idxKey, tokenLen, startRowID + i ); //1 key, multiple rows, multiple row ids - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - } //qaSupCreateMaxRowID - - /* ========================================================== - Populate tree with max key for a given bit length - Populate tree row ID for a given bitlength - ============================================================= */ - void qaSupCreateMaxHdrRec(int tokenLen) - { - - int rc; - uint64_t i; - int maxKey; - - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - - // here is the correct check - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - maxKey = (1 << tokenLen); - - if (maxKey > 160) maxKey = 160; - - for (i = 0; i < maxKey; i++) - rc = m_index.updateIndex( i, tokenLen, i ); //1 row id per key. - - m_index.closeIndex(); - - } //qaSupCreateMaxHdrRec - /* ========================================================== - Populate tree with range of keys - ============================================================= */ - - void qaSupCreateRangeKey(uint64_t idxKey, int tokenLen, int startKey, int numOfKey) - { - int rc; - uint64_t i; - int maxKey; - - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - - // here is the correct check - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = startKey; i < startKey + numOfKey; i++) - rc = m_index.updateIndex( i, tokenLen, i ); //1 row id per key. - - m_index.closeIndex(); - - } //qaSupCreateRangeKey - - /******************************************************* - . This function navigate through the index list record. - few global variables are set relating to index list. - *******************************************************/ - void qaSupWalkIndexList (uint64_t idxKey, int tokenLen, int execMode, IdxListStat& idxKeyInfo) - { - - DataBlock indexBlock; - FILE* pFileIndexList = NULL; - int file_indexList = 991; - - int p_fbo, p_sbid, p_entry; - int i,; - int SBRecCnt; - bool endOfList; - bool exist; - int tempExecMode; - - - IdxRidListEntry listEntry; - IdxRidListPtr listPtr; - IdxRidListHdr indexListHdrRec; - - - //Free Manager - m_indexlist.setUseFreeMgr( true ); - tempExecMode = execMode; - execMode = 0; - qaSupWalkIndexTree(idxKey, tokenLen, execMode, p_fbo, p_sbid, p_entry, exist); - execMode = tempExecMode; - - idxKeyInfo.fbo = p_fbo; //Save the address for the header - idxKeyInfo.sbid = p_sbid; - idxKeyInfo.entry = p_entry; - - if (not exist) - printf("Tree node not found: Key = %d\n", (int)idxKey); - else - { - //read index list header record - - i = 0; - - idxKeyInfo.sbCnt = 0; - idxKeyInfo.fullSbCnt = 0; - idxKeyInfo.recCnt = 0; //first entry in the hdr - endOfList = false; - - memset( indexBlock.data, 0, sizeof(indexBlock.data)); - pFileIndexList = m_indexlist.openFile( file_indexList ); - CPPUNIT_ASSERT( pFileIndexList != NULL ); - - m_indexlist.readSubBlockEntry( pFileIndexList, &indexBlock, p_fbo, p_sbid, p_entry, 32, &indexListHdrRec); - - idxKeyInfo.hdrRec = indexListHdrRec; - - - if (execMode == 1) - { - printf("\n---------------------------------\n"); - printf("Index record for key: %d\n", (int)idxKey); - printf("---------------------------------\n"); - printf("Size: %d\n", (int)indexListHdrRec.idxRidListSize.size); - printf("Key: %d\n", (int)indexListHdrRec.key); - printf("RowID 1: %d Type: %d\n", (int)indexListHdrRec.firstIdxRidListEntry.rid, (int)indexListHdrRec.firstIdxRidListEntry.type); - - if (indexListHdrRec.nextIdxRidListPtr.type == 3 ) - printf("RowID 2: %d Type: %d\\nn", (int)indexListHdrRec.nextIdxRidListPtr.llp, (int)indexListHdrRec.nextIdxRidListPtr.type); - else if (indexListHdrRec.nextIdxRidListPtr.type == 4 ) - printf("Ptr: Addr(%d, %d, %d) Type = %d\n\n", (int)((IdxEmptyListEntry*) & (indexListHdrRec.nextIdxRidListPtr))->fbo, (int)((IdxEmptyListEntry*) & (indexListHdrRec.nextIdxRidListPtr))->sbid, (int)((IdxEmptyListEntry*) & (indexListHdrRec.nextIdxRidListPtr))->entry, (int)indexListHdrRec.nextIdxRidListPtr.type); - } - - if (int(indexListHdrRec.firstIdxRidListEntry.type) == 3) - idxKeyInfo.recCnt++; - - if (indexListHdrRec.nextIdxRidListPtr.type == 3 ) - { - idxKeyInfo.recCnt++; - } - else if (indexListHdrRec.nextIdxRidListPtr.type == 4 ) - { - p_fbo = ((IdxEmptyListEntry*) & (indexListHdrRec.nextIdxRidListPtr))->fbo; - p_sbid = ((IdxEmptyListEntry*) & (indexListHdrRec.nextIdxRidListPtr))->sbid; - p_entry = ((IdxEmptyListEntry*) & (indexListHdrRec.nextIdxRidListPtr))->entry; - - while ( not endOfList ) - { - idxKeyInfo.sbCnt++; - SBRecCnt = 0; - m_indexlist.readSubBlockEntry( pFileIndexList, &indexBlock, p_fbo, p_sbid, 31, 8, &listPtr); - printf("Index list subblock: #%d\n", idxKeyInfo.sbCnt); - - for (i = 0; i <= 30; i++) - { - m_indexlist.readSubBlockEntry( pFileIndexList, &indexBlock, p_fbo, p_sbid, i, 8, &listEntry); - - if (listEntry.type == 3) - { - SBRecCnt++; - - if (execMode == 1) - printf("Entry %d: Addr(%d, %d, %d) RowId: %d \n", i, p_fbo, p_sbid, i, listEntry.rid); - } - } - - p_fbo = ((IdxEmptyListEntry*) & (listPtr))->fbo; - p_sbid = ((IdxEmptyListEntry*) & (listPtr))->sbid; - - if ((idxKeyInfo.sbSeqNo != 0 ) && (idxKeyInfo.sbSeqNo == (idxKeyInfo.sbCnt - 1))) - { - idxKeyInfo.fbo = p_fbo; //Save the address for the request block - idxKeyInfo.sbid = p_sbid; - idxKeyInfo.entry = p_entry; - } - - if (execMode == 1 ) - printf("Entry 31: Addr(%d, %d, %d) RecCnt = %d\n\n", p_fbo, p_sbid, p_entry, (int)listPtr.count); - - idxKeyInfo.recCnt += SBRecCnt; - - if (SBRecCnt == 31) idxKeyInfo.fullSbCnt++; - - if (SBRecCnt != listPtr.count) - printf("[**ERROR**] Sub Block entry count Error: In Ptr Rec: %d In sub block: %d\n", listPtr.count, SBRecCnt); - - endOfList = (listPtr.type == 0); - - } - } - } - - m_indexlist.closeFile(pFileIndexList); - } - - /******************************************************* - . This function navigate through the tree, from the - bitmap entry to the leaf. It can be used for many - functions that require access to the hdr record also. - *******************************************************/ - void qaSupWalkIndexTree(uint64_t idxKey, int tokenLen, int execMode, int& fbo, int& sbid, int& entry, bool& exist) - { - - DataBlock curBlock; - FILE* pFile = NULL; - int file_number = 990; - - IdxBitmapPointerEntry bitmapRec; - IdxBitTestEntry treeNodeRec; - int p_fbo, p_sbid, p_entry, p_type, p_group, p_bitTest; - int i; - int maskValue = 31; //mask for 11111. - int bitMapLoc, key1, shiftBits; - int bitCnt; - bool found; - int mask[5] = {1, 3, 7, 15, 31}; - int group[7] = {1, 2, 4, 8, 16, 32, 64}; - - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - - //Free Manager - m_indexlist.setUseFreeMgr( true ); - - if (tokenLen > 5) - bitMapLoc = ((idxKey >> tokenLen - 5) & maskValue); - else - bitMapLoc = idxKey; - - p_fbo = 0; - p_sbid = 1; - p_entry = bitMapLoc; - - if (execMode == 1) - { - printf("\n---------------------------------\n"); - printf("Tree branch path for key: %d\n", (int)idxKey); - printf("---------------------------------\n"); - } - - m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &bitmapRec); - - shiftBits = 1; - p_type = bitmapRec.type; - p_fbo = bitmapRec.fbo; - p_sbid = bitmapRec.sbid; - p_entry = bitmapRec.entry; - - if (execMode == 1) // branch list - printf("Bitmap: Type= %d BitTest= %d FBO= %d SBID= %d Entry= %d\n", int(p_type), int(p_bitTest), (int)p_fbo, (int)p_sbid, (int)p_entry); - - bitCnt = tokenLen - 5; - - while (bitCnt > 0) - { - if (bitCnt > 5) - key1 = ((idxKey >> (bitCnt - 5)) & maskValue); - else - key1 = ((idxKey) & mask[bitCnt - 1]); - - m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &treeNodeRec); - - int groupIdx = treeNodeRec.group; -// printf("group: %d count: %d\n", groupIdx, group[groupIdx]); - - found = false; - - for (i = 0; i < group[groupIdx]; i++) - { - m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, p_entry + i, 8, &treeNodeRec); - found = ( treeNodeRec.bitTest == key1); - p_type = treeNodeRec.type; - -// printf("key1: %d bitTest: %d\n", key1, treeNodeRec.bitTest); - if (found) i = group[groupIdx]; - - }; - -// m_indexlist.readSubBlockEntry( pFile, &curBlock, p_fbo, p_sbid, p_entry, 8, &treeNodeRec); - - p_type = treeNodeRec.type; - - p_group = treeNodeRec.group; - - p_bitTest = treeNodeRec.bitTest; - - p_fbo = treeNodeRec.fbo; - - p_sbid = treeNodeRec.sbid; - - p_entry = treeNodeRec.entry; - - bitCnt = bitCnt - 5; - - if (execMode == 1) // branch list - printf("Level: Type= %d Group= %d BitTest= %d FBO= %d SBID= %d Entry= %d\n", int(p_type), int(p_group), int(p_bitTest), (int)p_fbo, (int)p_sbid, (int)p_entry); - } - -//set global variables so that calling functions can access to the address of the index header record - - exist = found; - fbo = p_fbo; - sbid = p_sbid; - entry = p_entry; - - m_indexlist.closeFile(pFile); - } // qaSupWalkIndexTree + exist = found; + fbo = p_fbo; + sbid = p_sbid; + entry = p_entry; + m_indexlist.closeFile(pFile); + } // qaSupWalkIndexTree }; -CPPUNIT_TEST_SUITE_REGISTRATION( IndexTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(IndexTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/writeengine/index/tindex.cpp b/writeengine/index/tindex.cpp index 17a902eb2..bc0755754 100644 --- a/writeengine/index/tindex.cpp +++ b/writeengine/index/tindex.cpp @@ -39,4218 +39,4180 @@ using namespace WriteEngine; class IndexTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(IndexTest); + + // CPPUNIT_TEST( tmpTest ); + + // PUNIT_TEST( multimapdemo ); + // PUNIT_TEST( sortvecdemo ); + // PUNIT_TEST( sortarrdemo ); + + // CPPUNIT_TEST( spacehog ); + // CPPUNIT_TEST( test1 ); + // CPPUNIT_TEST( test2 ); + + // Index basic testing + + CPPUNIT_TEST(testStructIO); + CPPUNIT_TEST(testAddTreeNode); + CPPUNIT_TEST(testSetupBittestArray); + + // Index tree testing + CPPUNIT_TEST(testTreeGetTestbitValue); + CPPUNIT_TEST(testTreeGetTreeNodeInfo); + CPPUNIT_TEST(testTreeGetTreeMatchEntry); + CPPUNIT_TEST(testTreeMoveEntry); + CPPUNIT_TEST(testTreeBuildEmptyTree); + CPPUNIT_TEST(testTreeBuildExistTree1); + CPPUNIT_TEST(testTreeBuildExistTree2); + CPPUNIT_TEST(testTreeIntegration); + // + CPPUNIT_TEST(testIntegrationVolume); + + CPPUNIT_TEST(testCreateAndUpdateIndex); + + // + CPPUNIT_TEST(testDeleteIndex); + // + CPPUNIT_TEST(testResetIndex); + // + CPPUNIT_TEST(testUpdateMutiColIndex); + + // Index free manager testing + CPPUNIT_TEST(testFreeMgrBRM); + CPPUNIT_TEST(testFreeMgrInit); + CPPUNIT_TEST(testFreeMgrInit2); + CPPUNIT_TEST(testFreeMgrAssign); + CPPUNIT_TEST(testFreeMgrAssignList); + CPPUNIT_TEST(testFreeMgrAssignListCk); + CPPUNIT_TEST(testFreeMgrAssignListCk2); + CPPUNIT_TEST(testFreeMgrAssignListLots); + CPPUNIT_TEST(testFreeMgrAssignListBlocks); + CPPUNIT_TEST(testFreeMgrAssignExtend); + CPPUNIT_TEST(testFreeMgrExtendLots); + CPPUNIT_TEST(testFreeMgrRelease); + CPPUNIT_TEST(testFreeMgrFragment); + CPPUNIT_TEST(testFreeMgrChain); + + // Index list testing + CPPUNIT_TEST(testIndexListMultiKey); + CPPUNIT_TEST(testIndexListUpdate); + CPPUNIT_TEST(testIndexListDelete); + CPPUNIT_TEST(testIndexListReleaseMgrBack); + CPPUNIT_TEST(testIndexListMultipleAddHdr); + CPPUNIT_TEST(testIndexListMultipleUpdate); + + CPPUNIT_TEST_SUITE_END(); + + private: + IndexTree m_index; + IndexList m_indexlist; + FreeMgr m_freeMgr; + + public: + void setUp() + { + m_index.setUseFreeMgr(false); + m_index.setUseListMgr(false); + m_index.setTransId(124353); // a dummy transaction id + + // init the 000.dir + // int rc = m_index.createIndex( 20, 21 ); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + BRMWrapper::setUseBrm(true); + m_indexlist.setDebugLevel(DEBUG_0); + m_indexlist.setNarray(true); + } + + void testInit() + { + } + + void tearDown() + { + } + /* + void tmpTest() { + int rc, width = 32, i; + + m_index.setUseFreeMgr( true ); + m_index.setUseListMgr( true ); + BRMWrapper::setUseBrm( true ); + BRMWrapper::setUseVb( false ); + Cache::setUseCache( true ); + Cache::init(); + + m_index.setTransId( 12345678 ); + m_index.dropIndex( 990, 991 ); + rc = m_index.createIndex( 990, 991 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = m_index.openIndex( 990, 991 ); + CPPUNIT_ASSERT( rc == NO_ERROR ); + // m_index.m_freeMgr.setDebugLevel( DEBUG_1 ); + FILE* pFile; + uint64_t key, curkey, lastkey; + RID rid, ridArray[5000]; + int counter = 0, ridCounter = 0; + + pFile = fopen( "test.txt", "r" ); + CPPUNIT_ASSERT( pFile != NULL ); + + while( !feof( pFile ) && rc == NO_ERROR ) + { + fscanf( pFile, "%lld %lld\n", &key, &rid ); + curkey = key; + if( curkey == lastkey || counter == 0 || ridCounter == MAX_IDX_RID - 1 ) + ridArray[ridCounter++] = rid; + else { + m_index.m_multiRid.setMultiRid( ridArray, ridCounter ); + rc = m_index.updateIndex( lastkey, width, ridArray[0] ); + ridCounter = 0; + ridArray[ridCounter++] = rid; + } // end of if( curkey == lastkey ) { + lastkey = curkey; + + // printf( "%d %d\n", (int)key, (int)rid ); + counter++; + if( counter%1000 == 0 ) + printf( "\ncounter=%d", counter ); + } // end of while - CPPUNIT_TEST_SUITE( IndexTest ); - -//CPPUNIT_TEST( tmpTest ); - -//PUNIT_TEST( multimapdemo ); -//PUNIT_TEST( sortvecdemo ); -//PUNIT_TEST( sortarrdemo ); - -// CPPUNIT_TEST( spacehog ); -//CPPUNIT_TEST( test1 ); -// CPPUNIT_TEST( test2 ); - -// Index basic testing - - CPPUNIT_TEST( testStructIO ); - CPPUNIT_TEST( testAddTreeNode ); - CPPUNIT_TEST( testSetupBittestArray ); - -// Index tree testing - CPPUNIT_TEST( testTreeGetTestbitValue ); - CPPUNIT_TEST( testTreeGetTreeNodeInfo ); - CPPUNIT_TEST( testTreeGetTreeMatchEntry ); - CPPUNIT_TEST( testTreeMoveEntry ); - CPPUNIT_TEST( testTreeBuildEmptyTree ); - CPPUNIT_TEST( testTreeBuildExistTree1 ); - CPPUNIT_TEST( testTreeBuildExistTree2 ); - CPPUNIT_TEST( testTreeIntegration ); -// - CPPUNIT_TEST( testIntegrationVolume ); - - CPPUNIT_TEST( testCreateAndUpdateIndex ); - -// - CPPUNIT_TEST( testDeleteIndex ); -// - CPPUNIT_TEST( testResetIndex ); -// - CPPUNIT_TEST( testUpdateMutiColIndex ); - - - -// Index free manager testing - CPPUNIT_TEST( testFreeMgrBRM ); - CPPUNIT_TEST( testFreeMgrInit ); - CPPUNIT_TEST( testFreeMgrInit2 ); - CPPUNIT_TEST( testFreeMgrAssign ); - CPPUNIT_TEST( testFreeMgrAssignList ); - CPPUNIT_TEST( testFreeMgrAssignListCk ); - CPPUNIT_TEST( testFreeMgrAssignListCk2 ); - CPPUNIT_TEST( testFreeMgrAssignListLots ); - CPPUNIT_TEST( testFreeMgrAssignListBlocks ); - CPPUNIT_TEST( testFreeMgrAssignExtend ); - CPPUNIT_TEST( testFreeMgrExtendLots ); - CPPUNIT_TEST( testFreeMgrRelease ); - CPPUNIT_TEST( testFreeMgrFragment ); - CPPUNIT_TEST( testFreeMgrChain ); - -// Index list testing - CPPUNIT_TEST(testIndexListMultiKey); - CPPUNIT_TEST(testIndexListUpdate); - CPPUNIT_TEST(testIndexListDelete); - CPPUNIT_TEST(testIndexListReleaseMgrBack); - CPPUNIT_TEST(testIndexListMultipleAddHdr); - CPPUNIT_TEST(testIndexListMultipleUpdate); - - - CPPUNIT_TEST_SUITE_END(); - -private: - IndexTree m_index; - IndexList m_indexlist; - FreeMgr m_freeMgr; -public: - void setUp() - { - m_index.setUseFreeMgr( false ); - m_index.setUseListMgr( false ); - m_index.setTransId( 124353 ); // a dummy transaction id - - // init the 000.dir -// int rc = m_index.createIndex( 20, 21 ); -// CPPUNIT_ASSERT( rc == NO_ERROR ); - BRMWrapper::setUseBrm(true); - m_indexlist.setDebugLevel(DEBUG_0); - m_indexlist.setNarray(true); - } - - void testInit() - { - - } - - void tearDown() - { - } - /* - void tmpTest() { - int rc, width = 32, i; - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - BRMWrapper::setUseBrm( true ); - BRMWrapper::setUseVb( false ); - Cache::setUseCache( true ); - Cache::init(); - - m_index.setTransId( 12345678 ); - m_index.dropIndex( 990, 991 ); - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - // m_index.m_freeMgr.setDebugLevel( DEBUG_1 ); - FILE* pFile; - uint64_t key, curkey, lastkey; - RID rid, ridArray[5000]; - int counter = 0, ridCounter = 0; - - pFile = fopen( "test.txt", "r" ); - CPPUNIT_ASSERT( pFile != NULL ); - - while( !feof( pFile ) && rc == NO_ERROR ) - { - fscanf( pFile, "%lld %lld\n", &key, &rid ); - curkey = key; - if( curkey == lastkey || counter == 0 || ridCounter == MAX_IDX_RID - 1 ) - ridArray[ridCounter++] = rid; - else { - m_index.m_multiRid.setMultiRid( ridArray, ridCounter ); - rc = m_index.updateIndex( lastkey, width, ridArray[0] ); - ridCounter = 0; - ridArray[ridCounter++] = rid; - } // end of if( curkey == lastkey ) { - lastkey = curkey; - - // printf( "%d %d\n", (int)key, (int)rid ); - counter++; - if( counter%1000 == 0 ) - printf( "\ncounter=%d", counter ); - } // end of while - - - // last piece - if( rc == NO_ERROR && ridCounter > 0 ) { - m_index.m_multiRid.setMultiRid( ridArray, ridCounter ); - rc = m_index.updateIndex( lastkey, width, ridArray[0] ); - } - - m_index.flushCache(); - m_index.closeIndex(); - - Cache::freeMemory(); + // last piece + if( rc == NO_ERROR && ridCounter > 0 ) { + m_index.m_multiRid.setMultiRid( ridArray, ridCounter ); + rc = m_index.updateIndex( lastkey, width, ridArray[0] ); } - */ + m_index.flushCache(); + m_index.closeIndex(); + + Cache::freeMemory(); + } + */ #define NUMELEM 30000000 - void multimapdemo() + void multimapdemo() + { + multimap mul; + + timer t1, t2; + unsigned int idx; + map mm; + + for (idx = 0; idx < NUMELEM; idx++) { - multimap < int, int >mul; + mul.insert(pair(idx, idx)); - timer t1, t2; - unsigned int idx; - map < int, int > mm; - - for ( idx = 0; idx < NUMELEM; idx++) - { - mul.insert( pair(idx, idx) ); - - if ( idx % 10000 == 0) - { - printf("\nMultimap: index is %06i and time is:%lf (elapsed is %lf)", idx, t1.elapsed(), t2.elapsed()); - t1.restart(); - } - } - - mul.clear(); - } - bool pairCompare( const pair& lhs, const pair& rhs) - { - return lhs.second > rhs.second; - } - void sortvecdemo() - { - vector< pair > idxvec; - - timer t1, t2; - unsigned int idx; - - for ( idx = 0; idx < NUMELEM; idx++) - { - idxvec.push_back( pair(idx, idx) ); - - if ( idx % 10000 == 0) - { - printf("Sortvec: index is %06i and time is:%lf (elapsed is %lf)\n", idx, t1.elapsed(), t2.elapsed()); - t1.restart(); - } - } - - sort(idxvec.begin(), idxvec.end()); - printf("Sortvec: time is:%lf (elapsed is %lf)\n", t1.elapsed(), t2.elapsed()); - idxvec.clear(); + if (idx % 10000 == 0) + { + printf("\nMultimap: index is %06i and time is:%lf (elapsed is %lf)", idx, t1.elapsed(), t2.elapsed()); + t1.restart(); + } } - void sortarrdemo() + mul.clear(); + } + bool pairCompare(const pair& lhs, const pair& rhs) + { + return lhs.second > rhs.second; + } + void sortvecdemo() + { + vector > idxvec; + + timer t1, t2; + unsigned int idx; + + for (idx = 0; idx < NUMELEM; idx++) { - int* idxarr; - idxarr = (int*)malloc(sizeof(int) * NUMELEM); - - if (idxarr == 0) - { - printf("\nOOPs..\n"); - exit(-1); - } - - timer t1, t2; - unsigned int idx; - - for ( idx = 0; idx < NUMELEM; idx++) - { - idxarr[idx] = idx; - - if ( idx % 10000 == 0) - { - printf("Sortarr: index is %06i and time is:%lf (elapsed is %lf)\n", idx, t1.elapsed(), t2.elapsed()); - t1.restart(); - } - } - - sort(idxarr, idxarr + NUMELEM); - printf("Sortarr: time is:%lf (elapsed is %lf)\n", t1.elapsed(), t2.elapsed()); + idxvec.push_back(pair(idx, idx)); + if (idx % 10000 == 0) + { + printf("Sortvec: index is %06i and time is:%lf (elapsed is %lf)\n", idx, t1.elapsed(), t2.elapsed()); + t1.restart(); + } } - void spacehog() + sort(idxvec.begin(), idxvec.end()); + printf("Sortvec: time is:%lf (elapsed is %lf)\n", t1.elapsed(), t2.elapsed()); + idxvec.clear(); + } + + void sortarrdemo() + { + int* idxarr; + idxarr = (int*)malloc(sizeof(int) * NUMELEM); + + if (idxarr == 0) { - // test case to estimate space requirements - // generate an index with 200K entries, each unique - - int rc; - uint64_t i; - printf("\nUT: Beginning Space Hog\n"); - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - - rc = m_index.createIndex( 123, 124 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.openIndex( 123, 124 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for (i = 0; i < 800 * 1000; i++) - { - if (i % (1000 * 10) == 0) - { - printf("UT: I is now: %i\n", (int)i); - } - - rc = m_index.updateIndex( i * 10 * rand(), 32, i ); - - if (rc != NO_ERROR) - { - printf("\nShock and horror! i=%llu rc=%i\n", i, rc); - } - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - - m_index.setAssignFbo( 1 ); - m_index.closeIndex(); - - rc = m_index.dropIndex( 123, 124 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( false ); - m_index.setUseListMgr( false ); - - printf("\nUT: Finishing Space Hog\n"); + printf("\nOOPs..\n"); + exit(-1); } - void test1() + timer t1, t2; + unsigned int idx; + + for (idx = 0; idx < NUMELEM; idx++) { - int rc, width = 64, i; + idxarr[idx] = idx; - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - BRMWrapper::setUseBrm( true ); - BRMWrapper::setUseVb( false ); -// Cache::setUseCache( true ); -// Cache::init(); + if (idx % 10000 == 0) + { + printf("Sortarr: index is %06i and time is:%lf (elapsed is %lf)\n", idx, t1.elapsed(), t2.elapsed()); + t1.restart(); + } + } - m_index.setTransId( 12345678 ); - m_index.dropIndex( 990, 991 ); - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); -// m_index.m_freeMgr.setDebugLevel( DEBUG_1 ); - DataBlock curBlock; - char buf[20]; - uint64_t key; - timer t1, t2; + sort(idxarr, idxarr + NUMELEM); + printf("Sortarr: time is:%lf (elapsed is %lf)\n", t1.elapsed(), t2.elapsed()); + } - for ( i = 0; i < 100000; i++ ) - { - sprintf( buf, "%dSuccess", i ); - memcpy( &key, buf, 8 ); + void spacehog() + { + // test case to estimate space requirements + // generate an index with 200K entries, each unique - if (i % (1000) == 0) - { - printf("UT: I is now: %i, elapsed time %lf s, delta %lf s\n", (int)i, t1.elapsed(), t2.elapsed()); - t2.restart(); - } + int rc; + uint64_t i; + printf("\nUT: Beginning Space Hog\n"); + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); - rc = m_index.updateIndex( key, width, i ); + rc = m_index.createIndex(123, 124); + CPPUNIT_ASSERT(rc == NO_ERROR); - if ( rc != NO_ERROR ) - printf( "\n i =%d rc=%d", i, rc ); + rc = m_index.openIndex(123, 124); + CPPUNIT_ASSERT(rc == NO_ERROR); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } + for (i = 0; i < 800 * 1000; i++) + { + if (i % (1000 * 10) == 0) + { + printf("UT: I is now: %i\n", (int)i); + } + rc = m_index.updateIndex(i * 10 * rand(), 32, i); + + if (rc != NO_ERROR) + { + printf("\nShock and horror! i=%llu rc=%i\n", i, rc); + } + + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + m_index.setAssignFbo(1); + m_index.closeIndex(); + + rc = m_index.dropIndex(123, 124); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(false); + m_index.setUseListMgr(false); + + printf("\nUT: Finishing Space Hog\n"); + } + + void test1() + { + int rc, width = 64, i; + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + BRMWrapper::setUseBrm(true); + BRMWrapper::setUseVb(false); + // Cache::setUseCache( true ); + // Cache::init(); + + m_index.setTransId(12345678); + m_index.dropIndex(990, 991); + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + // m_index.m_freeMgr.setDebugLevel( DEBUG_1 ); + DataBlock curBlock; + char buf[20]; + uint64_t key; + timer t1, t2; + + for (i = 0; i < 100000; i++) + { + sprintf(buf, "%dSuccess", i); + memcpy(&key, buf, 8); + + if (i % (1000) == 0) + { printf("UT: I is now: %i, elapsed time %lf s, delta %lf s\n", (int)i, t1.elapsed(), t2.elapsed()); + t2.restart(); + } - curBlock = m_index.getRootBlock(); - printf( "\n block 0 subblock 0 " ); - m_index.printMemSubBlock( &curBlock, 0 ); - printf( "\n block 0 subblock 1 " ); - m_index.printMemSubBlock( &curBlock, 1 ); + rc = m_index.updateIndex(key, width, i); - m_index.printSubBlock( 11, 10 ); - m_index.closeIndex(); + if (rc != NO_ERROR) + printf("\n i =%d rc=%d", i, rc); -// rc = m_index.dropIndex( 990, 991 ); -// CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( false ); - m_index.setUseListMgr( false ); - -// Cache::freeMemory(); + CPPUNIT_ASSERT(rc == NO_ERROR); } - void test2() + printf("UT: I is now: %i, elapsed time %lf s, delta %lf s\n", (int)i, t1.elapsed(), t2.elapsed()); + + curBlock = m_index.getRootBlock(); + printf("\n block 0 subblock 0 "); + m_index.printMemSubBlock(&curBlock, 0); + printf("\n block 0 subblock 1 "); + m_index.printMemSubBlock(&curBlock, 1); + + m_index.printSubBlock(11, 10); + m_index.closeIndex(); + + // rc = m_index.dropIndex( 990, 991 ); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + + m_index.setUseFreeMgr(false); + m_index.setUseListMgr(false); + + // Cache::freeMemory(); + } + + void test2() + { + int rc, width = 16, i; + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + DataBlock curBlock; + + for (i = 10; i < 381; i++) { - int rc, width = 16, i; + curBlock = m_index.getRootBlock(); + printf("\n i=%d", i); - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); + if (i == 130) + { + printf("\n130"); + ; + } - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + if (i == 20 || i == 128 || i == 129 || i == 64 || i == 66) + { + printf("\n******* before call updateIndex (i=%i)", i); + m_index.printMemSubBlock(&curBlock, 0); + m_index.printSubBlock(63, 19); + m_index.printSubBlock(63, 31); + } - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = m_index.updateIndex(2 * i, width, i * 100); + if (rc != NO_ERROR) + printf("\n i =%d rc=%d", i, rc); - DataBlock curBlock; - - for ( i = 10; i < 381; i++ ) - { - - curBlock = m_index.getRootBlock(); - printf( "\n i=%d", i ); - - if (i == 130) - { - printf("\n130");; - } - - if ( i == 20 || i == 128 || i == 129 || i == 64 || i == 66) - { - printf( "\n******* before call updateIndex (i=%i)", i ); - m_index.printMemSubBlock( &curBlock, 0 ); - m_index.printSubBlock( 63, 19 ); - m_index.printSubBlock( 63, 31 ); - } - - rc = m_index.updateIndex( 2 * i, width, i * 100 ); - - if ( rc != NO_ERROR ) - printf( "\n i =%d rc=%d", i, rc ); - - if ( i == 20 || i == 128 || i == 129 || i == 64 || i == 66) - { - printf( "\n******* after call updateIndex (i=%i)", i ); - m_index.printMemSubBlock( &curBlock, 0 ); - m_index.printSubBlock( 63, 19 ); - m_index.printSubBlock( 63, 31 ); - } - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - - m_index.setAssignFbo( 1 ); - m_index.closeIndex(); - - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( false ); - m_index.setUseListMgr( false ); + if (i == 20 || i == 128 || i == 129 || i == 64 || i == 66) + { + printf("\n******* after call updateIndex (i=%i)", i); + m_index.printMemSubBlock(&curBlock, 0); + m_index.printSubBlock(63, 19); + m_index.printSubBlock(63, 31); + } + CPPUNIT_ASSERT(rc == NO_ERROR); } - void testStructIO() + m_index.setAssignFbo(1); + m_index.closeIndex(); + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(false); + m_index.setUseListMgr(false); + } + + void testStructIO() + { + DataBlock curBlock; + FILE* pFile; + int rc, fbo = 1, sbid = 2, entryNo = 5, width = 8, allocSize; + IdxBitmapPointerEntry bitmapPtr; + + memset(&bitmapPtr, 0, 8); + memset(curBlock.data, 0, sizeof(curBlock.data)); + + m_index.deleteFile(999); + CPPUNIT_ASSERT(m_index.exists(999) == false); + + CPPUNIT_ASSERT(m_index.createFile(999, 10, allocSize) == NO_ERROR); + + pFile = m_index.openFile(999); + CPPUNIT_ASSERT(pFile != NULL); + + rc = m_index.readSubBlockEntry(pFile, &curBlock, fbo, sbid, entryNo, width, &bitmapPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + + bitmapPtr.type = 1; + bitmapPtr.fbo = 2; + bitmapPtr.sbid = 8; + bitmapPtr.entry = 7; + + rc = m_index.writeSubBlockEntry(pFile, &curBlock, fbo, sbid, entryNo, width, &bitmapPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_index.closeFile(pFile); + + // reload the value + memset(&bitmapPtr, 0, 8); + memset(curBlock.data, 0, sizeof(curBlock.data)); + + CPPUNIT_ASSERT(m_index.isAddrPtrEmpty(&bitmapPtr, EMPTY_PTR) == true); + + pFile = m_index.openFile(999); + CPPUNIT_ASSERT(pFile != NULL); + + rc = m_index.readDBFile(pFile, curBlock.data, fbo); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.getSubBlockEntry(curBlock.data, sbid, entryNo, width, &bitmapPtr); + + CPPUNIT_ASSERT(bitmapPtr.type == 1); + CPPUNIT_ASSERT(bitmapPtr.fbo == 2); + CPPUNIT_ASSERT(bitmapPtr.sbid == 8); + CPPUNIT_ASSERT(bitmapPtr.entry == 7); + + CPPUNIT_ASSERT(m_index.isAddrPtrEmpty(&bitmapPtr, BITMAP_PTR /*EMPTY_PTR*/) == false); + + uint64_t i; + + for (i = 0; i < 15; i++) { - DataBlock curBlock; - FILE* pFile; - int rc, fbo = 1, sbid = 2, entryNo = 5, width = 8, allocSize; - IdxBitmapPointerEntry bitmapPtr; + bitmapPtr.type = i % 8; + bitmapPtr.fbo = 2 * i; + bitmapPtr.sbid = i; + bitmapPtr.entry = i + 6; - memset( &bitmapPtr, 0, 8 ); - memset( curBlock.data, 0, sizeof( curBlock.data ) ); - - m_index.deleteFile( 999 ); - CPPUNIT_ASSERT( m_index.exists( 999 ) == false ); - - CPPUNIT_ASSERT( m_index.createFile( 999, 10, allocSize ) == NO_ERROR ); - - pFile = m_index.openFile( 999 ); - CPPUNIT_ASSERT( pFile != NULL ); - - rc = m_index.readSubBlockEntry( pFile, &curBlock, fbo, sbid, entryNo, width, &bitmapPtr ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - bitmapPtr.type = 1; - bitmapPtr.fbo = 2; - bitmapPtr.sbid = 8; - bitmapPtr.entry = 7; - - rc = m_index.writeSubBlockEntry( pFile, &curBlock, fbo, sbid, entryNo, width, &bitmapPtr ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_index.closeFile( pFile ); - - // reload the value - memset( &bitmapPtr, 0, 8 ); - memset( curBlock.data, 0, sizeof( curBlock.data ) ); - - CPPUNIT_ASSERT( m_index.isAddrPtrEmpty( &bitmapPtr, EMPTY_PTR ) == true ); - - pFile = m_index.openFile( 999 ); - CPPUNIT_ASSERT( pFile != NULL ); - - rc = m_index.readDBFile( pFile, curBlock.data, fbo ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.getSubBlockEntry( curBlock.data, sbid, entryNo, width, &bitmapPtr ); - - CPPUNIT_ASSERT( bitmapPtr.type == 1 ); - CPPUNIT_ASSERT( bitmapPtr.fbo == 2 ); - CPPUNIT_ASSERT( bitmapPtr.sbid == 8 ); - CPPUNIT_ASSERT( bitmapPtr.entry == 7 ); - - CPPUNIT_ASSERT( m_index.isAddrPtrEmpty( &bitmapPtr, BITMAP_PTR/*EMPTY_PTR*/ ) == false ); - - uint64_t i; - - for ( i = 0; i < 15; i++ ) - { - bitmapPtr.type = i % 8; - bitmapPtr.fbo = 2 * i; - bitmapPtr.sbid = i; - bitmapPtr.entry = i + 6; - - m_index.setSubBlockEntry( curBlock.data, i, 0, 8, &bitmapPtr ); - } - - rc = m_index.writeDBFile( pFile, curBlock.data, fbo ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.closeFile( pFile ); - - for ( i = 0; i < 15; i++ ) - { - m_index.getSubBlockEntry( curBlock.data, i, 0, 8, &bitmapPtr ); - CPPUNIT_ASSERT( bitmapPtr.type == i % 8 ); -// CPPUNIT_ASSERT( bitmapPtr.oid == i ); - CPPUNIT_ASSERT( bitmapPtr.fbo == 2 * i ); - CPPUNIT_ASSERT( bitmapPtr.sbid == i ); - CPPUNIT_ASSERT( bitmapPtr.entry == i + 6 ); - } + m_index.setSubBlockEntry(curBlock.data, i, 0, 8, &bitmapPtr); } - void testAddTreeNode() + rc = m_index.writeDBFile(pFile, curBlock.data, fbo); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.closeFile(pFile); + + for (i = 0; i < 15; i++) { - uint64_t key = 123, rid = 1900, width = 8, testbitVal = 3; - IdxTree myTree; - IdxBitmapPointerEntry bitmapEntry; + m_index.getSubBlockEntry(curBlock.data, i, 0, 8, &bitmapPtr); + CPPUNIT_ASSERT(bitmapPtr.type == i % 8); + // CPPUNIT_ASSERT( bitmapPtr.oid == i ); + CPPUNIT_ASSERT(bitmapPtr.fbo == 2 * i); + CPPUNIT_ASSERT(bitmapPtr.sbid == i); + CPPUNIT_ASSERT(bitmapPtr.entry == i + 6); + } + } - m_index.clearTree( &myTree ); - bitmapEntry.fbo = 6; - bitmapEntry.sbid = 1; - bitmapEntry.entry = 9; + void testAddTreeNode() + { + uint64_t key = 123, rid = 1900, width = 8, testbitVal = 3; + IdxTree myTree; + IdxBitmapPointerEntry bitmapEntry; - m_index.setTreeHeader( &myTree, key, rid, width, testbitVal, bitmapEntry); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.rid == rid ); - CPPUNIT_ASSERT( myTree.maxLevel == ((width / 5) + 1) ); - CPPUNIT_ASSERT( myTree.node[0].level == 0 ); - CPPUNIT_ASSERT( myTree.node[0].allocCount == ENTRY_PER_SUBBLOCK ); -// CPPUNIT_ASSERT( myTree.node[0].group == ENTRY_32 ); - CPPUNIT_ASSERT( myTree.node[0].used == true ); - CPPUNIT_ASSERT( myTree.node[0].next.fbo == bitmapEntry.fbo ); - CPPUNIT_ASSERT( myTree.node[0].next.sbid == bitmapEntry.sbid ); - CPPUNIT_ASSERT( myTree.node[0].next.entry == bitmapEntry.entry ); + m_index.clearTree(&myTree); + bitmapEntry.fbo = 6; + bitmapEntry.sbid = 1; + bitmapEntry.entry = 9; + m_index.setTreeHeader(&myTree, key, rid, width, testbitVal, bitmapEntry); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.rid == rid); + CPPUNIT_ASSERT(myTree.maxLevel == ((width / 5) + 1)); + CPPUNIT_ASSERT(myTree.node[0].level == 0); + CPPUNIT_ASSERT(myTree.node[0].allocCount == ENTRY_PER_SUBBLOCK); + // CPPUNIT_ASSERT( myTree.node[0].group == ENTRY_32 ); + CPPUNIT_ASSERT(myTree.node[0].used == true); + CPPUNIT_ASSERT(myTree.node[0].next.fbo == bitmapEntry.fbo); + CPPUNIT_ASSERT(myTree.node[0].next.sbid == bitmapEntry.sbid); + CPPUNIT_ASSERT(myTree.node[0].next.entry == bitmapEntry.entry); + } + + void testSetupBittestArray() + { + char charVal[] = "abcde"; + short i8Val = 21; + uint16_t i16Val = 1098, curPos = 0; + uint32_t i32Val = 0x0A6E6D8E, compareVal; + uint64_t i64Val = 0xDE1B4213; + int width, testbitVal, rc, i, shiftPos; + + // compare with old code to make sure we can still be consistent + // test 64 bit + width = 8; + + rc = m_index.setBitsetColumn(&i64Val, 1, width * 8, WriteEngine::WR_LONGLONG); + CPPUNIT_ASSERT(rc == NO_ERROR); + // cout << "bitArray = " << m_index.m_multiColKey.bitSet.to_string() << endl; + + m_index.calculateBittestArray(); + + // cout << "\nMax level = " << m_index.m_multiColKey.maxLevel << endl; + for (i = 0; i < m_index.m_multiColKey.maxLevel; i++) + { + rc = m_index.getTestbitValue(i64Val, width * 8, i, &testbitVal); + CPPUNIT_ASSERT(rc == true); + printf("\tOld Level[%d] bit test value : %d \n", i, testbitVal); + shiftPos = i == m_index.m_multiColKey.maxLevel - 1 ? 4 : 5; + compareVal = compareVal << shiftPos | testbitVal; + CPPUNIT_ASSERT(testbitVal == m_index.m_multiColKey.testbitArray[i]); } - void testSetupBittestArray() + // cout << "\nNew bitset array value " << endl; + for (i = 0; i < m_index.m_multiColKey.maxLevel; i++) + printf("\tLevel[%d] bit test value : %d \n", i, m_index.m_multiColKey.testbitArray[i]); + + // test 32 bit + width = 4; + m_index.m_multiColKey.clear(); + rc = m_index.setBitsetColumn(&i32Val, 1, width * 8, WriteEngine::WR_INT); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_index.calculateBittestArray(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 0; i < m_index.m_multiColKey.maxLevel; i++) { - char charVal[] = "abcde"; - short i8Val = 21; - uint16_t i16Val = 1098, curPos = 0; - uint32_t i32Val = 0x0A6E6D8E, compareVal; - uint64_t i64Val = 0xDE1B4213; - int width, testbitVal, rc, i, shiftPos; - - // compare with old code to make sure we can still be consistent - // test 64 bit - width = 8; - - rc = m_index.setBitsetColumn( &i64Val, 1, width * 8, WriteEngine::WR_LONGLONG ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - //cout << "bitArray = " << m_index.m_multiColKey.bitSet.to_string() << endl; - - m_index.calculateBittestArray(); - - //cout << "\nMax level = " << m_index.m_multiColKey.maxLevel << endl; - for ( i = 0; i < m_index.m_multiColKey.maxLevel; i++ ) - { - rc = m_index.getTestbitValue( i64Val, width * 8, i, &testbitVal ); - CPPUNIT_ASSERT( rc == true ); - printf( "\tOld Level[%d] bit test value : %d \n", i, testbitVal ); - shiftPos = i == m_index.m_multiColKey.maxLevel - 1 ? 4 : 5; - compareVal = compareVal << shiftPos | testbitVal; - CPPUNIT_ASSERT( testbitVal == m_index.m_multiColKey.testbitArray[i] ); - } - - //cout << "\nNew bitset array value " << endl; - for ( i = 0; i < m_index.m_multiColKey.maxLevel; i++ ) - printf( "\tLevel[%d] bit test value : %d \n", i, m_index.m_multiColKey.testbitArray[i] ); - - // test 32 bit - width = 4; - m_index.m_multiColKey.clear(); - rc = m_index.setBitsetColumn( &i32Val, 1, width * 8, WriteEngine::WR_INT ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_index.calculateBittestArray(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( i = 0; i < m_index.m_multiColKey.maxLevel; i++ ) - { - rc = m_index.getTestbitValue( i32Val, width * 8, i, &testbitVal ); - CPPUNIT_ASSERT( rc == true ); - CPPUNIT_ASSERT( testbitVal == m_index.m_multiColKey.testbitArray[i] ); - } - - // test 16 bit - width = 2; - m_index.m_multiColKey.clear(); - rc = m_index.setBitsetColumn( &i16Val, 1, width * 8, WriteEngine::WR_SHORT ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_index.calculateBittestArray(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( i = 0; i < m_index.m_multiColKey.maxLevel; i++ ) - { - rc = m_index.getTestbitValue( i16Val, width * 8, i, &testbitVal ); - CPPUNIT_ASSERT( rc == true ); - CPPUNIT_ASSERT( testbitVal == m_index.m_multiColKey.testbitArray[i] ); - } - - // test 8 bit - width = 1; - m_index.m_multiColKey.clear(); - rc = m_index.setBitsetColumn( &i8Val, 1, width * 8, WriteEngine::WR_BYTE ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_index.calculateBittestArray(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( i = 0; i < m_index.m_multiColKey.maxLevel; i++ ) - { - rc = m_index.getTestbitValue( i8Val, width * 8, i, &testbitVal ); - CPPUNIT_ASSERT( rc == true ); - CPPUNIT_ASSERT( testbitVal == m_index.m_multiColKey.testbitArray[i] ); - } - - // here is the real one for multi-column - m_index.m_multiColKey.clear(); - width = 1; - rc = m_index.setBitsetColumn( &i8Val, curPos++, width * 8, WriteEngine::WR_BYTE ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - width = 5; - rc = m_index.setBitsetColumn( charVal, curPos++, width * 8, WriteEngine::WR_CHAR ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - width = 4; - rc = m_index.setBitsetColumn( &i32Val, curPos++, width * 8, WriteEngine::WR_INT ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - width = 9; - strcpy( charVal, "test12345" ); - rc = m_index.setBitsetColumn( charVal, curPos++, width * 8, WriteEngine::WR_CHAR ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.calculateBittestArray(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( m_index.m_multiColKey.maxLevel == 31 ); - CPPUNIT_ASSERT( m_index.m_multiColKey.totalBit == 152 ); - - //cout << "\nMulti-column bitset array value " << endl; - for ( i = 0; i < m_index.m_multiColKey.maxLevel; i++ ) - printf( "\tLevel[%d] bit test value : %d \n", i, m_index.m_multiColKey.testbitArray[i] ); + rc = m_index.getTestbitValue(i32Val, width * 8, i, &testbitVal); + CPPUNIT_ASSERT(rc == true); + CPPUNIT_ASSERT(testbitVal == m_index.m_multiColKey.testbitArray[i]); } - void testCreateAndUpdateIndex() + // test 16 bit + width = 2; + m_index.m_multiColKey.clear(); + rc = m_index.setBitsetColumn(&i16Val, 1, width * 8, WriteEngine::WR_SHORT); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_index.calculateBittestArray(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 0; i < m_index.m_multiColKey.maxLevel; i++) { - int rc; - bool bEmptyFlag; - - BRMWrapper::setUseBrm(false); - m_index.dropIndex( 990, 991 ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // check invalid file not exist - rc = m_index.openIndex( 990, 992 ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - - rc = m_index.openIndex( 992, 991 ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - - // here is the correct check - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - bEmptyFlag = m_index.isTreeEmpty(); - CPPUNIT_ASSERT( bEmptyFlag == true ); - - rc = m_index.updateIndex( 123, 8, 1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - bEmptyFlag = m_index.isTreeEmpty(); - CPPUNIT_ASSERT( bEmptyFlag == false ); - - // test insert multiple rid - RID ridArray[6]; - - ridArray[0] = 23; - ridArray[1] = 26; - ridArray[2] = 27; - ridArray[3] = 31; - ridArray[4] = 39; - ridArray[5] = 40; - - m_index.setUseMultiRid( true ); - m_index.m_multiRid.setMultiRid( ridArray, 6 ); - - rc = m_index.updateIndex( 123, 8, ridArray[0] ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.m_multiRid.clearMultiRid(); - m_index.closeIndex(); + rc = m_index.getTestbitValue(i16Val, width * 8, i, &testbitVal); + CPPUNIT_ASSERT(rc == true); + CPPUNIT_ASSERT(testbitVal == m_index.m_multiColKey.testbitArray[i]); } - void testDeleteIndex() + // test 8 bit + width = 1; + m_index.m_multiColKey.clear(); + rc = m_index.setBitsetColumn(&i8Val, 1, width * 8, WriteEngine::WR_BYTE); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_index.calculateBittestArray(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 0; i < m_index.m_multiColKey.maxLevel; i++) { - int rc; - - - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // 8 bit - rc = m_index.deleteIndex( 123, 8, 1 ); - CPPUNIT_ASSERT( rc == ERR_IDX_LIST_INVALID_DELETE ); - - rc = m_index.updateIndex( 123, 8, 1 ); - printf("383. rc %i \n", rc); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.deleteIndex( 123, 8, 1 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // 16 bit - rc = m_index.deleteIndex( 0xA1C9, 16, 3 ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - - rc = m_index.updateIndex( 0xA1C9, 16, 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.deleteIndex( 0xA1C9, 16, 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // 32 bit - rc = m_index.updateIndex( 0xCA6E6D8E, 32, 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.deleteIndex( 0xCA6E6D8E, 32, 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // repeat one more time, should fail - rc = m_index.deleteIndex( 0xCA6E6D8E, 32, 3 ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - - m_index.closeIndex(); + rc = m_index.getTestbitValue(i8Val, width * 8, i, &testbitVal); + CPPUNIT_ASSERT(rc == true); + CPPUNIT_ASSERT(testbitVal == m_index.m_multiColKey.testbitArray[i]); } - void testResetIndex() + // here is the real one for multi-column + m_index.m_multiColKey.clear(); + width = 1; + rc = m_index.setBitsetColumn(&i8Val, curPos++, width * 8, WriteEngine::WR_BYTE); + CPPUNIT_ASSERT(rc == NO_ERROR); + + width = 5; + rc = m_index.setBitsetColumn(charVal, curPos++, width * 8, WriteEngine::WR_CHAR); + CPPUNIT_ASSERT(rc == NO_ERROR); + + width = 4; + rc = m_index.setBitsetColumn(&i32Val, curPos++, width * 8, WriteEngine::WR_INT); + CPPUNIT_ASSERT(rc == NO_ERROR); + + width = 9; + strcpy(charVal, "test12345"); + rc = m_index.setBitsetColumn(charVal, curPos++, width * 8, WriteEngine::WR_CHAR); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.calculateBittestArray(); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(m_index.m_multiColKey.maxLevel == 31); + CPPUNIT_ASSERT(m_index.m_multiColKey.totalBit == 152); + + // cout << "\nMulti-column bitset array value " << endl; + for (i = 0; i < m_index.m_multiColKey.maxLevel; i++) + printf("\tLevel[%d] bit test value : %d \n", i, m_index.m_multiColKey.testbitArray[i]); + } + + void testCreateAndUpdateIndex() + { + int rc; + bool bEmptyFlag; + + BRMWrapper::setUseBrm(false); + m_index.dropIndex(990, 991); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // check invalid file not exist + rc = m_index.openIndex(990, 992); + CPPUNIT_ASSERT(rc != NO_ERROR); + + rc = m_index.openIndex(992, 991); + CPPUNIT_ASSERT(rc != NO_ERROR); + + // here is the correct check + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + bEmptyFlag = m_index.isTreeEmpty(); + CPPUNIT_ASSERT(bEmptyFlag == true); + + rc = m_index.updateIndex(123, 8, 1); + CPPUNIT_ASSERT(rc == NO_ERROR); + + bEmptyFlag = m_index.isTreeEmpty(); + CPPUNIT_ASSERT(bEmptyFlag == false); + + // test insert multiple rid + RID ridArray[6]; + + ridArray[0] = 23; + ridArray[1] = 26; + ridArray[2] = 27; + ridArray[3] = 31; + ridArray[4] = 39; + ridArray[5] = 40; + + m_index.setUseMultiRid(true); + m_index.m_multiRid.setMultiRid(ridArray, 6); + + rc = m_index.updateIndex(123, 8, ridArray[0]); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.m_multiRid.clearMultiRid(); + m_index.closeIndex(); + } + + void testDeleteIndex() + { + int rc; + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // 8 bit + rc = m_index.deleteIndex(123, 8, 1); + CPPUNIT_ASSERT(rc == ERR_IDX_LIST_INVALID_DELETE); + + rc = m_index.updateIndex(123, 8, 1); + printf("383. rc %i \n", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.deleteIndex(123, 8, 1); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // 16 bit + rc = m_index.deleteIndex(0xA1C9, 16, 3); + CPPUNIT_ASSERT(rc != NO_ERROR); + + rc = m_index.updateIndex(0xA1C9, 16, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.deleteIndex(0xA1C9, 16, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // 32 bit + rc = m_index.updateIndex(0xCA6E6D8E, 32, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.deleteIndex(0xCA6E6D8E, 32, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // repeat one more time, should fail + rc = m_index.deleteIndex(0xCA6E6D8E, 32, 3); + CPPUNIT_ASSERT(rc != NO_ERROR); + + m_index.closeIndex(); + } + + void testResetIndex() + { + int rc, width = 64, i; + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + BRMWrapper::setUseBrm(true); + BRMWrapper::setUseVb(false); + + m_index.setTransId(12345678); + m_index.dropIndex(990, 991); + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.closeIndex(); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + char buf[20]; + uint64_t key; + + for (i = 0; i < 10000; i++) { - int rc, width = 64, i; - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - BRMWrapper::setUseBrm( true ); - BRMWrapper::setUseVb( false ); + sprintf(buf, "%dSuccess", i); + memcpy(&key, buf, 8); - m_index.setTransId( 12345678 ); - m_index.dropIndex( 990, 991 ); - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = m_index.updateIndex(key, width, i); - m_index.closeIndex(); + if (rc != NO_ERROR) + printf("\n i =%d rc=%d", i, rc); - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - char buf[20]; - uint64_t key; - - for ( i = 0; i < 10000; i++ ) - { - sprintf( buf, "%dSuccess", i ); - memcpy( &key, buf, 8 ); - - rc = m_index.updateIndex( key, width, i ); - - if ( rc != NO_ERROR ) - printf( "\n i =%d rc=%d", i, rc ); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - // reset index -// rc = m_index.openIndex( 990, 991 ); -// CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.resetIndexFile( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); -// m_index.closeIndex(); - - printf( "\nre-insert begin\n" ); - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( i = 0; i < 10000; i++ ) - { - sprintf( buf, "%dSuccess", i ); - memcpy( &key, buf, 8 ); - - rc = m_index.updateIndex( key, width, i ); - - if ( rc != NO_ERROR ) - printf( "\n i =%d rc=%d", i, rc ); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT(rc == NO_ERROR); } - void testUpdateMutiColIndex() + m_index.closeIndex(); + + // reset index + // rc = m_index.openIndex( 990, 991 ); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + + rc = m_index.resetIndexFile(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + // m_index.closeIndex(); + + printf("\nre-insert begin\n"); + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 0; i < 10000; i++) { - uint16_t i16Val; - uint32_t i32Val; - uint64_t key; - int rc, width = 64, i, pos = 0, totalWidth = 0; - char charVal[50]; + sprintf(buf, "%dSuccess", i); + memcpy(&key, buf, 8); - m_index.setUseFreeMgr( true ); - m_index.setUseListMgr( true ); - BRMWrapper::setUseBrm( true ); - BRMWrapper::setUseVb( false ); + rc = m_index.updateIndex(key, width, i); - m_index.setTransId( 12345678 ); - m_index.dropIndex( 990, 991 ); - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + if (rc != NO_ERROR) + printf("\n i =%d rc=%d", i, rc); - m_index.closeIndex(); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_index.setUseMultiCol( true ); - - printf( "\nBegin of multicol index insert\n" ); - - for ( i = 0; i < 10000; i++ ) - { - totalWidth = 0; - // init part - m_index.m_multiColKey.clear(); - - // multi colum inserts - // column 1 - width = 4; - totalWidth += width; - i32Val = i; - rc = m_index.setBitsetColumn( &i32Val, pos++, width * 8, WriteEngine::WR_INT ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // column 2 - width = 4; - totalWidth += width; - i32Val = i * 3; - rc = m_index.setBitsetColumn( &i32Val, pos++, width * 8, WriteEngine::WR_INT ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // column 3 - width = 2; - totalWidth += width; - i16Val = i; - rc = m_index.setBitsetColumn( &i16Val, pos++, width * 8, WriteEngine::WR_SHORT ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // column 4 - width = 15; - totalWidth += width; - sprintf( charVal, "%7d success", i ); - rc = m_index.setBitsetColumn( charVal, pos++, width * 8, WriteEngine::WR_CHAR ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // column 5 - width = 7; - totalWidth += width; - sprintf( charVal, "%7d", i ); - rc = m_index.setBitsetColumn( charVal, pos++, width * 8, WriteEngine::WR_CHAR ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // final part - rc = m_index.calculateBittestArray(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - memcpy( &key, m_index.m_multiColKey.keyBuf, 8 ); -// printf( "\ntotalWidth = %d", totalWidth ); - rc = m_index.updateIndex( key, totalWidth * 8, i ); - - if ( rc != NO_ERROR ) - printf( "\n i =%d rc=%d", i, rc ); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - - if ( i % 500 == 0 ) - printf( "\n process i=%d", i ); - } - - m_index.closeIndex(); - - printf( "\nEnd of multicol index insert\n" ); - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + CPPUNIT_ASSERT(rc == NO_ERROR); } - void testIntegrationVolume() + m_index.closeIndex(); + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testUpdateMutiColIndex() + { + uint16_t i16Val; + uint32_t i32Val; + uint64_t key; + int rc, width = 64, i, pos = 0, totalWidth = 0; + char charVal[50]; + + m_index.setUseFreeMgr(true); + m_index.setUseListMgr(true); + BRMWrapper::setUseBrm(true); + BRMWrapper::setUseVb(false); + + m_index.setTransId(12345678); + m_index.dropIndex(990, 991); + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.closeIndex(); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_index.setUseMultiCol(true); + + printf("\nBegin of multicol index insert\n"); + + for (i = 0; i < 10000; i++) { - int rc, width = 8, count = 3, i, j; - uint64_t key = 123; - RID rid = 1; - IdxTree myTree; - BRMWrapper::setUseBrm(false); - m_index.setUseFreeMgr( false ); - m_index.setUseListMgr( true ); - m_index.setDebugLevel( DEBUG_3 ); + totalWidth = 0; + // init part + m_index.m_multiColKey.clear(); - for ( i = 0; i < 3; i++ ) - { - switch ( i ) - { - case 0: - key = 123; - width = 8; - rid = 1; - break; + // multi colum inserts + // column 1 + width = 4; + totalWidth += width; + i32Val = i; + rc = m_index.setBitsetColumn(&i32Val, pos++, width * 8, WriteEngine::WR_INT); + CPPUNIT_ASSERT(rc == NO_ERROR); - case 1: - key = 0xA1C9; - width = 16; - rid = 100; - break; + // column 2 + width = 4; + totalWidth += width; + i32Val = i * 3; + rc = m_index.setBitsetColumn(&i32Val, pos++, width * 8, WriteEngine::WR_INT); + CPPUNIT_ASSERT(rc == NO_ERROR); - case 2: - key = 0xCA6E6D8E; - width = 32; - rid = 1000; - break; - } + // column 3 + width = 2; + totalWidth += width; + i16Val = i; + rc = m_index.setBitsetColumn(&i16Val, pos++, width * 8, WriteEngine::WR_SHORT); + CPPUNIT_ASSERT(rc == NO_ERROR); - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // column 4 + width = 15; + totalWidth += width; + sprintf(charVal, "%7d success", i); + rc = m_index.setBitsetColumn(charVal, pos++, width * 8, WriteEngine::WR_CHAR); + CPPUNIT_ASSERT(rc == NO_ERROR); - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // column 5 + width = 7; + totalWidth += width; + sprintf(charVal, "%7d", i); + rc = m_index.setBitsetColumn(charVal, pos++, width * 8, WriteEngine::WR_CHAR); + CPPUNIT_ASSERT(rc == NO_ERROR); - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); + // final part + rc = m_index.calculateBittestArray(); + CPPUNIT_ASSERT(rc == NO_ERROR); + memcpy(&key, m_index.m_multiColKey.keyBuf, 8); + // printf( "\ntotalWidth = %d", totalWidth ); + rc = m_index.updateIndex(key, totalWidth * 8, i); - for ( j = 0; j < count; j++ ) - { - printf( "\nj=%d", j ); - rc = m_index.updateIndex( key + j, width, rid + j ); - - if ( rc != NO_ERROR ) - printf( "\nrc=%d", rc ); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - myTree = m_index.getTree(); - - m_index.setAssignFbo( 1 + j ); - } - - for ( j = 0; j < count; j++ ) - { - rc = m_index.deleteIndex( key + j, width, rid + j ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - m_index.closeIndex(); - - } + if (rc != NO_ERROR) + printf("\n i =%d rc=%d", i, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + if (i % 500 == 0) + printf("\n process i=%d", i); } + m_index.closeIndex(); - void testTreeGetTestbitValue() + printf("\nEnd of multicol index insert\n"); + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testIntegrationVolume() + { + int rc, width = 8, count = 3, i, j; + uint64_t key = 123; + RID rid = 1; + IdxTree myTree; + BRMWrapper::setUseBrm(false); + m_index.setUseFreeMgr(false); + m_index.setUseListMgr(true); + m_index.setDebugLevel(DEBUG_3); + + for (i = 0; i < 3; i++) { - uint64_t key; - int width, bittestVal; - bool bStatus; - BRMWrapper::setUseBrm(false); - // test 8 bit - width = 8; - key = 35; - bStatus = m_index.getTestbitValue( key, width, 0, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 4 ); - bStatus = m_index.getTestbitValue( key, width, 1, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 3 ); - bStatus = m_index.getTestbitValue( key, width, 2, &bittestVal ); - CPPUNIT_ASSERT( bStatus == false ); - - // test 16 bit - width = 16; - key = 0xA1C9; - bStatus = m_index.getTestbitValue( key, width, 0, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 20 ); - bStatus = m_index.getTestbitValue( key, width, 1, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 7 ); - bStatus = m_index.getTestbitValue( key, width, 2, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 4 ); - bStatus = m_index.getTestbitValue( key, width, 3, &bittestVal ); - CPPUNIT_ASSERT( bittestVal == 1 ); - CPPUNIT_ASSERT( bStatus == true ); - bStatus = m_index.getTestbitValue( key, width, 4, &bittestVal ); - CPPUNIT_ASSERT( bStatus == false ); - - // test 32 bit - width = 32; - key = 0xCA6E6D8E; - bStatus = m_index.getTestbitValue( key, width, 0, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 25 ); - bStatus = m_index.getTestbitValue( key, width, 1, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 9 ); - bStatus = m_index.getTestbitValue( key, width, 2, &bittestVal ); - CPPUNIT_ASSERT( bStatus == true ); - CPPUNIT_ASSERT( bittestVal == 23 ); - bStatus = m_index.getTestbitValue( key, width, 3, &bittestVal ); - CPPUNIT_ASSERT( bittestVal == 6 ); - CPPUNIT_ASSERT( bStatus == true ); - bStatus = m_index.getTestbitValue( key, width, 4, &bittestVal ); - CPPUNIT_ASSERT( bittestVal == 27 ); - CPPUNIT_ASSERT( bStatus == true ); - bStatus = m_index.getTestbitValue( key, width, 5, &bittestVal ); - CPPUNIT_ASSERT( bittestVal == 3 ); - CPPUNIT_ASSERT( bStatus == true ); - bStatus = m_index.getTestbitValue( key, width, 6, &bittestVal ); - CPPUNIT_ASSERT( bittestVal == 2 ); - CPPUNIT_ASSERT( bStatus == true ); - bStatus = m_index.getTestbitValue( key, width, 7, &bittestVal ); - CPPUNIT_ASSERT( bStatus == false ); - - } - - void testTreeGetTreeNodeInfo() - { - int rc, fbo = 6, sbid = 1, entry = 2, testbitVal = 5; - int curSbid = 3, curEntry = 2, allocCount, realCount; - IdxTreeGroupType curGroup; - bool entryMap[ENTRY_PER_SUBBLOCK]; - DataBlock block; - IdxBitTestEntry bittestEntry; - - // test ENTRY_1 - m_index.clearBlock( &block ); - curGroup = ENTRY_1; - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo, sbid, entry ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 3, curGroup, fbo + 6, sbid + 3, entry + 1 ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 1, 8, &bittestEntry ); - - rc = m_index.getTreeNodeInfo( &block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocCount == 1 ); - CPPUNIT_ASSERT( realCount == 1 ); - CPPUNIT_ASSERT( entryMap[0] == true ); - - // test ENTRY_2 - m_index.clearBlock( &block ); - curGroup = ENTRY_2; - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo, sbid, entry ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 3, curGroup, fbo + 6, sbid + 3, entry + 1 ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 1, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo + 6, sbid + 3, entry + 1 ); - bittestEntry.type = EMPTY_LIST; - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 2, 8, &bittestEntry ); - - rc = m_index.getTreeNodeInfo( &block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocCount == 2 ); - CPPUNIT_ASSERT( realCount == 2 ); - CPPUNIT_ASSERT( entryMap[0] == true ); - CPPUNIT_ASSERT( entryMap[1] == true ); - CPPUNIT_ASSERT( entryMap[2] == false ); - - // test ENTRY_8 - m_index.clearBlock( &block ); - curGroup = ENTRY_8; - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo, sbid, entry ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo + 6, sbid + 3, entry + 1 ); - bittestEntry.type = EMPTY_LIST; - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 3, 8, &bittestEntry ); - - rc = m_index.getTreeNodeInfo( &block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == ERR_IDX_TREE_INVALID_TYPE ); - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo + 6, sbid + 3, entry + 1 ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 3, 8, &bittestEntry ); - - bittestEntry.type = EMPTY_ENTRY; - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 2, 2, &bittestEntry ); - - rc = m_index.getTreeNodeInfo( &block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocCount == 8 ); - CPPUNIT_ASSERT( realCount == 2 ); - CPPUNIT_ASSERT( entryMap[0] == true ); - CPPUNIT_ASSERT( entryMap[1] == false ); - CPPUNIT_ASSERT( entryMap[2] == false ); - CPPUNIT_ASSERT( entryMap[3] == true ); - CPPUNIT_ASSERT( entryMap[4] == false ); - } - - void testTreeGetTreeMatchEntry() - { - int rc, fbo = 6, sbid = 1, entry = 2, testbitVal = 5; - int curSbid = 3, curEntry = 2, allocCount, realCount, matchEntry; - IdxTreeGroupType curGroup; - bool entryMap[ENTRY_PER_SUBBLOCK]; - DataBlock block; - IdxBitTestEntry bittestEntry, checkEntry; - bool bFound; - - //ENTRY_8 - m_index.clearBlock( &block ); - curGroup = ENTRY_8; - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo, sbid, entry ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 5, curGroup, fbo + 9, sbid + 6, entry + 2 ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 4, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 9, curGroup, fbo + 6, sbid + 3, entry + 1 ); - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 3, 8, &bittestEntry ); - - bittestEntry.type = EMPTY_ENTRY; - m_index.setSubBlockEntry( block.data, curSbid, curEntry + 2, 2, &bittestEntry ); - - rc = m_index.getTreeNodeInfo( &block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocCount == 8 ); - CPPUNIT_ASSERT( realCount == 3 ); - CPPUNIT_ASSERT( entryMap[0] == true ); - CPPUNIT_ASSERT( entryMap[1] == false ); - CPPUNIT_ASSERT( entryMap[2] == false ); - CPPUNIT_ASSERT( entryMap[3] == true ); - CPPUNIT_ASSERT( entryMap[4] == true ); - CPPUNIT_ASSERT( entryMap[5] == false ); - - memset( &checkEntry, 0, 8 ); - checkEntry.type = BIT_TEST; - checkEntry.bitTest = testbitVal + 4; - bFound = m_index.getTreeMatchEntry( &block, curSbid, curEntry, 8, allocCount, entryMap, &matchEntry, &checkEntry ); - CPPUNIT_ASSERT( bFound == false ); - - checkEntry.bitTest = testbitVal + 9; - bFound = m_index.getTreeMatchEntry( &block, curSbid, curEntry, 8, allocCount, entryMap, &matchEntry, &checkEntry ); - CPPUNIT_ASSERT( bFound == true ); - CPPUNIT_ASSERT( matchEntry == 3 ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 5, curGroup, fbo + 9, sbid + 6, entry + 2 ); - checkEntry.bitTest = testbitVal + 5; - bFound = m_index.getTreeMatchEntry( &block, curSbid, curEntry, 8, allocCount, entryMap, &matchEntry, &checkEntry ); - CPPUNIT_ASSERT( bFound == true ); - CPPUNIT_ASSERT( matchEntry == 4 ); - CPPUNIT_ASSERT( !memcmp( &checkEntry, &bittestEntry, 8 ) ); - - } - - - void testTreeMoveEntry() - { - int rc, fbo = 6, sbid = 1, entry = 2, testbitVal = 5; - int curSbid = 3, curEntry = 2, allocCount, realCount, moveCount; - IdxTreeGroupType curGroup; - bool entryMap[ENTRY_PER_SUBBLOCK]; - DataBlock oldBlock, newBlock, testBlock, blankBlock; - IdxBitTestEntry bittestEntry; - FILE* treeFile; - - if ( m_index.exists( 990 ) ) - { - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - BRMWrapper::setUseBrm(false); - rc = m_index.createIndex( 990, 991, false ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - treeFile = m_index.openFile( 990 ); - CPPUNIT_ASSERT( treeFile != NULL ); - - //ENTRY_8 - m_index.clearBlock( &oldBlock ); - m_index.clearBlock( &newBlock ); - m_index.clearBlock( &testBlock ); - m_index.clearBlock( &blankBlock ); - curGroup = ENTRY_8; - - m_index.setBittestEntry( &bittestEntry, testbitVal, curGroup, fbo, sbid, entry ); - m_index.setSubBlockEntry( oldBlock.data, curSbid, curEntry, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 5, curGroup, fbo + 9, sbid + 6, entry + 2 ); - m_index.setSubBlockEntry( oldBlock.data, curSbid, curEntry + 4, 8, &bittestEntry ); - - m_index.setBittestEntry( &bittestEntry, testbitVal + 9, curGroup, fbo + 6, sbid + 3, entry + 1 ); - m_index.setSubBlockEntry( oldBlock.data, curSbid, curEntry + 3, 8, &bittestEntry ); - - bittestEntry.type = EMPTY_ENTRY; - m_index.setSubBlockEntry( oldBlock.data, curSbid, curEntry + 2, 2, &bittestEntry ); - - rc = m_index.getTreeNodeInfo( &oldBlock, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocCount == 8 ); - CPPUNIT_ASSERT( realCount == 3 ); - CPPUNIT_ASSERT( entryMap[0] == true ); - CPPUNIT_ASSERT( entryMap[1] == false ); - CPPUNIT_ASSERT( entryMap[2] == false ); - CPPUNIT_ASSERT( entryMap[3] == true ); - CPPUNIT_ASSERT( entryMap[4] == true ); - CPPUNIT_ASSERT( entryMap[5] == false ); - - rc = m_index.writeDBFile( treeFile, &oldBlock, 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_index.closeFile( treeFile ); - - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.moveEntry( 3, curSbid, curEntry, 8, - 5, curSbid + 10, curEntry + 1, curGroup, allocCount, entryMap, &moveCount ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( moveCount == 3 ); - - m_index.closeIndex(); - - treeFile = m_index.openFile( 990 ); - CPPUNIT_ASSERT( treeFile != NULL ); - rc = m_index.readDBFile( treeFile, &testBlock, 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( !memcmp( testBlock.data, blankBlock.data, sizeof( testBlock.data ))); - - rc = m_index.readDBFile( treeFile, &testBlock, 5 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.getTreeNodeInfo( &testBlock, curSbid + 10, curEntry + 1, 8, curGroup, &allocCount, &realCount, entryMap ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( allocCount == 8 ); - CPPUNIT_ASSERT( realCount == 3 ); - CPPUNIT_ASSERT( entryMap[0] == true ); - CPPUNIT_ASSERT( entryMap[1] == true ); - CPPUNIT_ASSERT( entryMap[2] == true ); - CPPUNIT_ASSERT( entryMap[3] == false ); - CPPUNIT_ASSERT( entryMap[4] == false ); - CPPUNIT_ASSERT( entryMap[5] == false ); - - m_index.closeFile( treeFile ); - } - - void testTreeNodeAssert( IdxTree myTree, int curLevel, uint64_t curBitTest, uint16_t curGroup, - uint64_t curFbo, uint64_t curSbid, uint64_t curEntry, uint64_t nextFbo, uint64_t nextSbid, uint64_t nextEntry ) - { - CPPUNIT_ASSERT( myTree.node[curLevel].current.bitTest == curBitTest ); - CPPUNIT_ASSERT( myTree.node[curLevel].current.group == curGroup ); - CPPUNIT_ASSERT( myTree.node[curLevel].current.fbo == curFbo ); - CPPUNIT_ASSERT( myTree.node[curLevel].current.sbid == curSbid ); - CPPUNIT_ASSERT( myTree.node[curLevel].current.entry == curEntry ); - - if ( curLevel == 0 ) - CPPUNIT_ASSERT( myTree.node[curLevel].current.type == BITMAP_PTR ); - else if ( curLevel == myTree.maxLevel - 1 ) - CPPUNIT_ASSERT( myTree.node[curLevel].current.type == LEAF_LIST ); - else - CPPUNIT_ASSERT( myTree.node[curLevel].current.type == BIT_TEST ); - - CPPUNIT_ASSERT( myTree.node[curLevel].next.fbo == nextFbo ); - CPPUNIT_ASSERT( myTree.node[curLevel].next.sbid == nextSbid ); - CPPUNIT_ASSERT( myTree.node[curLevel].next.entry == nextEntry ); - - CPPUNIT_ASSERT( myTree.node[curLevel].allocCount == (0x1 << curGroup) ); - - CPPUNIT_ASSERT( myTree.node[curLevel].used == true ); - } - - void testTreeBuildEmptyTree() - { - int rc, width = 8, rid = 3; - uint64_t key = 123; - IdxTree myTree; - BRMWrapper::setUseBrm(false); - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.buildEmptyTreePart( key, width, rid, 0 ); - CPPUNIT_ASSERT( rc == ERR_IDX_TREE_INVALID_LEVEL ); - - - rc = m_index.buildEmptyTreePart( key, 4, rid, 0 ); - CPPUNIT_ASSERT( rc == ERR_IDX_TREE_INVALID_LEVEL ); - - rc = m_index.updateIndex( key, width, rid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // width = 8 - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 2 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) rid ); - - testTreeNodeAssert( myTree, 0, 15, ENTRY_32, 0, 1, 15,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 3, ENTRY_1, 1, 3, 4, /* */ 3, 3, 4 ); - CPPUNIT_ASSERT( myTree.node[2].used == false ); - - // width = 16 - key = 0xA1C9; - width = 16; - rc = m_index.updateIndex( key, width, rid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - - CPPUNIT_ASSERT( myTree.maxLevel == 4 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) rid ); - - testTreeNodeAssert( myTree, 0, 20, ENTRY_32, 0, 1, 20,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4 ); - testTreeNodeAssert( myTree, 2, 4, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4 ); - testTreeNodeAssert( myTree, 3, 1, ENTRY_1, 1, 5, 4,/* */ 3, 3, 6 ); - CPPUNIT_ASSERT( myTree.node[4].used == false ); - - // width = 32 - key = 0xCA6E6D8E; - width = 32; - rc = m_index.updateIndex( key, width, rid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - - CPPUNIT_ASSERT( myTree.maxLevel == 7 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) rid ); - - testTreeNodeAssert( myTree, 0, 25, ENTRY_32, 0, 1, 25,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 9, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4 ); - testTreeNodeAssert( myTree, 2, 23, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4 ); - testTreeNodeAssert( myTree, 3, 6, ENTRY_1, 1, 5, 4,/* */ 1, 6, 4 ); - testTreeNodeAssert( myTree, 4, 27, ENTRY_1, 1, 6, 4, /* */ 1, 7, 4 ); - testTreeNodeAssert( myTree, 5, 3, ENTRY_1, 1, 7, 4,/* */ 1, 8, 4 ); - testTreeNodeAssert( myTree, 6, 2, ENTRY_1, 1, 8, 4,/* */ 3, 3, 9 ); - - CPPUNIT_ASSERT( myTree.node[7].used == false ); - - - m_index.closeIndex(); - } - - - - void testTreeBuildExistTree1() - { - int rc, width = 8, rid = 3; - uint64_t key = 123; - IdxTree myTree; - BRMWrapper::setUseBrm(false); - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.updateIndex( key, width, rid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // width = 8 - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 2 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) rid ); - - testTreeNodeAssert( myTree, 0, 15, ENTRY_32, 0, 1, 15,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 3, ENTRY_1, 1, 3, 4,/* */ 3, 3, 4 ); - CPPUNIT_ASSERT( myTree.node[2].used == false ); - - // insert again - m_index.setAssignFbo( 1 ); - rc = m_index.updateIndex( key, width, rid + 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 2 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 3) ); - - testTreeNodeAssert( myTree, 0, 15, ENTRY_32, 0, 1, 15,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 3, ENTRY_1, 1, 3, 4,/* */ 3, 3, 4 ); - - CPPUNIT_ASSERT( myTree.node[2].used == false ); - - // insert with a little different key - m_index.setAssignFbo( 2 ); - rc = m_index.updateIndex( key + 1, width, rid + 4 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 2 ); - CPPUNIT_ASSERT( myTree.key == key + 1 ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 4) ); - - testTreeNodeAssert( myTree, 0, 15, ENTRY_32, 0, 1, 15,/* */ 3, 4, 4 ); - testTreeNodeAssert( myTree, 1, 4, ENTRY_2, 3, 4, 4,/* */ 3, 3, 4 ); - CPPUNIT_ASSERT( myTree.node[2].used == false ); - - // insert with a different key and different rid - m_index.setAssignFbo( 3 ); - rc = m_index.updateIndex( key + 2, width, rid + 5 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 2 ); - CPPUNIT_ASSERT( myTree.key == key + 2 ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 5) ); - - testTreeNodeAssert( myTree, 0, 15, ENTRY_32, 0, 1, 15,/* */ 4, 4, 4 ); - testTreeNodeAssert( myTree, 1, 5, ENTRY_4, 4, 4, 4,/* */ 3, 3, 4 ); - - CPPUNIT_ASSERT( myTree.node[1].useCount == 3 ); - CPPUNIT_ASSERT( myTree.node[2].used == false ); - - - // insert with a different key and different rid - m_index.setAssignFbo( 4 ); - rc = m_index.updateIndex( key + 3, width, rid + 7 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 2 ); - CPPUNIT_ASSERT( myTree.key == key + 3 ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 7) ); - - testTreeNodeAssert( myTree, 0, 15, ENTRY_32, 0, 1, 15,/* */ 4, 4, 4 ); - testTreeNodeAssert( myTree, 1, 6, ENTRY_4, 4, 4, 4,/* */ 3, 3, 4 ); - - CPPUNIT_ASSERT( myTree.node[2].used == false ); - - m_index.closeIndex(); - - } - - - void testTreeBuildExistTree2() - { - int rc, width = 16, rid = 3; - uint64_t key = 0xA1C9; - IdxTree myTree; - BRMWrapper::setUseBrm(false); - rc = m_index.dropIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.createIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.updateIndex( key, width, rid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // width = 16 - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 4 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) rid ); - - testTreeNodeAssert( myTree, 0, 20, ENTRY_32, 0, 1, 20,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4 ); - testTreeNodeAssert( myTree, 2, 4, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4 ); - testTreeNodeAssert( myTree, 3, 1, ENTRY_1, 1, 5, 4,/* */ 3, 3, 6 ); - - CPPUNIT_ASSERT( myTree.node[4].used == false ); - - // insert again - m_index.setAssignFbo( 1 ); - rc = m_index.updateIndex( key, width, rid + 3 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 4 ); - CPPUNIT_ASSERT( myTree.key == key ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 3)); - testTreeNodeAssert( myTree, 0, 20, ENTRY_32, 0, 1, 20,/* */ 1, 3, 4 ); - testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4 ); - testTreeNodeAssert( myTree, 2, 4, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4 ); - testTreeNodeAssert( myTree, 3, 1, ENTRY_1, 1, 5, 4,/* */ 3, 3, 6 ); - - CPPUNIT_ASSERT( myTree.node[4].used == false ); - - // insert with a little different key - m_index.setAssignFbo( 2 ); - rc = m_index.updateIndex( key + 1, width, rid + 4 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 4 ); - CPPUNIT_ASSERT( myTree.key == key + 1 ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 4)); - - testTreeNodeAssert( myTree, 0, 20, ENTRY_32, 0, 1, 20,/* */ 1, 3, 4 ); - // testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1,3,4, /* */ 3, 5, 5 ); - testTreeNodeAssert( myTree, 2, 5, ENTRY_2, 3, 5, 5, /* */ 3, 3, 4 ); - testTreeNodeAssert( myTree, 3, 0, ENTRY_1, 3, 3, 4,/* */ 3, 3, 6 ); - CPPUNIT_ASSERT( myTree.node[4].used == false ); - - // insert with a different key and different rid - m_index.setAssignFbo( 3 ); - rc = m_index.updateIndex( key + 2, width, rid + 5 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 4 ); - CPPUNIT_ASSERT( myTree.key == key + 2 ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 5)); - - testTreeNodeAssert( myTree, 0, 20, ENTRY_32, 0, 1, 20,/* */ 1, 3, 4 ); -// testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1,3,4, /* */ 3, 5, 5 ); - testTreeNodeAssert( myTree, 2, 5, ENTRY_2, 3, 5, 5, /* */ 4, 6, 4 ); - testTreeNodeAssert( myTree, 3, 1, ENTRY_2, 4, 6, 4,/* */ 3, 3, 6 ); - CPPUNIT_ASSERT( myTree.node[4].used == false ); - - // insert with a different key and different rid - m_index.setAssignFbo( 4 ); - rc = m_index.updateIndex( key + 3, width, rid + 7 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myTree = m_index.getTree(); - CPPUNIT_ASSERT( myTree.maxLevel == 4 ); - CPPUNIT_ASSERT( myTree.key == key + 3 ); - CPPUNIT_ASSERT( myTree.width == width ); - CPPUNIT_ASSERT( myTree.rid == (RID) (rid + 7)); - - testTreeNodeAssert( myTree, 0, 20, ENTRY_32, 0, 1, 20,/* */ 1, 3, 4 ); -// testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1,3,4, /* */ 3, 5, 5 ); - testTreeNodeAssert( myTree, 2, 6, ENTRY_4, 5, 5, 6, /* */ 5, 3, 4 ); - testTreeNodeAssert( myTree, 3, 0, ENTRY_1, 5, 3, 4,/* */ 3, 3, 6 ); - CPPUNIT_ASSERT( myTree.node[4].used == false ); - - m_index.closeIndex(); - - // set a blank entry - IdxBitTestEntry curEntry; - FILE* treeFile; - DataBlock curBlock; - - treeFile = m_index.openFile( 990 ); - CPPUNIT_ASSERT( treeFile != NULL ); - - rc = m_index.readSubBlockEntry( treeFile, &curBlock, 1, 3, 4, MAX_COLUMN_BOUNDARY, &curEntry ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setBlankEntry( &curEntry ); - - rc = m_index.writeSubBlockEntry( treeFile, &curBlock, 1, 3, 4, MAX_COLUMN_BOUNDARY, &curEntry ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.closeFile( treeFile ); - - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.updateIndex( key, width, rid ); - CPPUNIT_ASSERT( rc == ERR_STRUCT_EMPTY ); - - m_index.closeIndex(); - } - - void testTreeIntegration() - { - IdxEmptyListEntry myEntry; - int rc; - BRMWrapper::setUseBrm(false); - rc = m_index.openIndex( 990, 991 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.assignSegment( ENTRY_1, &myEntry, 0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.releaseSegment( ENTRY_1, &myEntry ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.setUseFreeMgr( true ); - rc = m_index.assignSegment( ENTRY_1, &myEntry, 0 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = m_index.releaseSegment( ENTRY_1, &myEntry ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - m_index.closeIndex(); - } - - void testFreeMgrBRM() - { - - int rc; - int allocSize; - CommBlock cb; - uint64_t lbid; - uint64_t fbo; - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("\nUT: Begin testFreeMgrBRM\n"); - - - m_freeMgr.deleteFile( 777 ); - m_freeMgr.createFile( 777, 80, allocSize ); - m_freeMgr.createFile( 778, 80, allocSize ); - - cb.file.oid = 777 ; - fbo = 0; - lbid = m_freeMgr.mapLBID( cb, fbo, rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); - - cb.file.oid = 777 ; - fbo = 10; - lbid = m_freeMgr.mapLBID( cb, fbo, rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); - - cb.file.oid = 777 ; - fbo = 1023; - lbid = m_freeMgr.mapLBID( cb, fbo, rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); - - cb.file.oid = 777 ; - fbo = 1020; - lbid = m_freeMgr.mapLBID( cb, fbo, rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); - - cb.file.oid = 778 ; - fbo = 0; - lbid = m_freeMgr.mapLBID( cb, fbo, rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); - - cb.file.oid = 778 ; - fbo = 1000; - lbid = m_freeMgr.mapLBID( cb, fbo, rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - void testFreeMgrInit() - { - FILE* indexFile; - DataBlock blockZero; - IdxEmptyListEntry emptyMap, assignPtr; - CommBlock cb; - int allocSize, rc; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_1 ); - printf("\nUT: Begin testFreeMgrInit\n"); - - - - assignPtr.fbo = 0; - assignPtr.sbid = 0; - assignPtr.entry = 0; - assignPtr.type = 0; - - - cb.file.pFile = NULL; - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == ERR_INVALID_PARAM ); - m_freeMgr.deleteFile( 777 ); - m_freeMgr.deleteFile( 778 ); - m_freeMgr.createFile( 778, 8, allocSize ); // create a second file to try and force lbid range to not collide with fbo - m_freeMgr.createFile( 777, 8, allocSize ); - - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - -// m_indexlist.createFile(777,800, allocSize); - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, 1234 ) == ERR_INVALID_PARAM ); - - // init a freemgr for a tree - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb.file.pFile, blockZero.data, lbid ); - m_freeMgr.getSubBlockEntry( blockZero.data, 0, 1 + ENTRY_32, 8, &emptyMap ); - // check the entries are as expected - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); - // check the entries are as expected - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_16 ); -// printf("UT: fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - CPPUNIT_ASSERT( emptyMap.fbo == 0 ); - CPPUNIT_ASSERT( emptyMap.entry == 0 ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_8, 8, &emptyMap); - // check the entries are as expected - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_8 ); - CPPUNIT_ASSERT( emptyMap.fbo == 0 ); - CPPUNIT_ASSERT( emptyMap.entry == 0 ); - - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); - // check the entries are as expected - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_4 ); - CPPUNIT_ASSERT( emptyMap.fbo == 0 ); - CPPUNIT_ASSERT( emptyMap.entry == 0 ); - - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_2, 8, &emptyMap); - // check the entries are as expected - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_2 ); - CPPUNIT_ASSERT( emptyMap.fbo == 0 ); - CPPUNIT_ASSERT( emptyMap.entry == 0 ); - - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); - // check the entries are as expected - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_1 ); - CPPUNIT_ASSERT( emptyMap.fbo == 0 ); - CPPUNIT_ASSERT( emptyMap.entry == 0 ); - - // assign a segment and check that entries in sb0 change - - printf("UT: Init: Assign ENTRY_32 fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( assignPtr.entry == 0 ); // points to start of sub-block -// CPPUNIT_ASSERT( assignPtr.fbo == 1 ); // - CPPUNIT_ASSERT( assignPtr.group == ENTRY_32 ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.entry == 30 ); // points to second to last entry in sub-block - - // release segment and see if head pointer moves - rc = m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 31 ); -// CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block - - - // release to wrong list - printf("UT: Put sub-block segment on list of 16 (expect error)\n"); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == ERR_FM_RELEASE_ERR ); - - // tweak type then try to release -// printf("UT: Release EMPTY_LIST entry\n"); -// assignPtr.type = EMPTY_LIST; -// CPPUNIT_ASSERT( freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == ERR_FM_BAD_TYPE ); - - printf("UT: assign ENTRY_16 segment\n"); - // assign ENTRY_16 segment - CPPUNIT_ASSERT_MESSAGE( "Failed assigning ENTRY_16 segment", m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); - - CPPUNIT_ASSERT_MESSAGE( "Empty map type should be EMPTY_LIST", emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_16 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 0 ); -// CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to last entry in sub-block - - - // release ENTRY_16 segment - printf("UT: Release ENTRY_16 segment\n"); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - - // legal segment size but list is empty - causes list to be populated - printf("UT: Assign from ENTRY_4 list - which is empty\n"); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == NO_ERROR ); - printf("UT: After Assign ENTRY_4 fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); - - //Check ENTRY_4 in map - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); - printf("UT: fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - // ENTRY_4 segment size - printf("UT: Assign from ENTRY_4 list - which is no longer empty\n"); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == NO_ERROR ); - printf("UT: Assign from ENTRY_4 fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); - - - // cleanup - m_freeMgr.writeDBFile( indexFile, blockZero.data, lbid ); - m_freeMgr.closeFile( indexFile ); - - } - // list tests - void testFreeMgrInit2() - { - FILE* indexFile; - CommBlock cb; - DataBlock blockZero; - IdxEmptyListEntry emptyMap, assignPtr; - int allocSize, rc; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("\nUT: Begin testFreeMgrInit2\n"); - - - - assignPtr.fbo = 0; - assignPtr.sbid = 0; - assignPtr.entry = 0; - assignPtr.type = 0; - - m_freeMgr.deleteFile( 776 ); - rc = m_freeMgr.createFile( 776, 80, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 776 ); - cb.file.oid = 776; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readSubBlockEntry( cb, &blockZero, lbid, 0, 1 + ENTRY_BLK, 8, &emptyMap ); - - // check the entries are as expected - printf("UT: fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_BLK ); - CPPUNIT_ASSERT( emptyMap.fbo != 0 ); -// CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block - - // assign a segment and check that entries in sb0 change - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_BLK, &assignPtr ) == NO_ERROR ); - - CPPUNIT_ASSERT( assignPtr.group == ENTRY_BLK ); // right group -// CPPUNIT_ASSERT( assignPtr.entry == 0 ); // right position -// CPPUNIT_ASSERT( assignPtr.sbid == 31 ); // right position -// CPPUNIT_ASSERT( assignPtr.fbo == 1 ); // right block - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_BLK, 8, &emptyMap); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_BLK ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 0 ); -// CPPUNIT_ASSERT( emptyMap.entry == 30 ); // points to second to last entry in sub-block - - // illegal segment size - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_16, &assignPtr ) == ERR_INVALID_PARAM ); - - // illegal segment size - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_8, &assignPtr ) == ERR_INVALID_PARAM ); - - // illegal segment size - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_2, &assignPtr ) == ERR_INVALID_PARAM ); - - // illegal segment size - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_1, &assignPtr ) == ERR_INVALID_PARAM ); - - // legal segment size but list is empty - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &assignPtr ) == NO_ERROR ); - - CPPUNIT_ASSERT( assignPtr.group == ENTRY_4 ); // points to start of sub-block - CPPUNIT_ASSERT( assignPtr.entry == 28 ); -// CPPUNIT_ASSERT( assignPtr.sbid == 29 ); -// CPPUNIT_ASSERT( assignPtr.fbo == 1 ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_4 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 30 ); - CPPUNIT_ASSERT( emptyMap.entry == 7 ); // eight segments in list -// CPPUNIT_ASSERT( emptyMap.entry == 31 ); // 32 segments if using blocks for buckets - - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_4, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_4 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 30 ); - CPPUNIT_ASSERT( emptyMap.entry == 8 ); // eight segments in list -// CPPUNIT_ASSERT( (int)emptyMap.entry == 32 ); // 32 segments if using blocks for buckets - - m_freeMgr.writeDBFile( cb, blockZero.data, lbid ); - m_freeMgr.closeFile( indexFile ); - - } - - - void testFreeMgrAssign() - { - /** - * Assign tests - */ - FILE* indexFile; - CommBlock cb; - DataBlock blockZero; - IdxEmptyListEntry assignPtr, assignPtr2; - IdxEmptyListEntry assignPtr3, assignPtr4; - IdxEmptyListEntry emptyMap; - int idx, result; - int allocSize, rc; - IdxEmptyListEntry entries[1500]; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("\nUT: Begin testFreeMgrAssign\n"); - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 8, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - /* m_freeMgr.readDBFile( indexFile, blockZero.data, 0 ); - m_index.printMemSubBlock( &blockZero , 0 );*/ - uint64_t lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( indexFile, blockZero.data, lbid ); -// m_index.printMemSubBlock( &blockZero , 0 ); - printf("\nUT: Cont testFreeMgrAssign\n"); - -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, 0, TREE, ENTRY_32, &assignPtr ) == ERR_INVALID_PARAM ); -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, 0 ) == ERR_INVALID_PARAM ); -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, 1234, ENTRY_32, &assignPtr ) == ERR_INVALID_PARAM ); - - // Assign a segment - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - // Assign a second segment and make sure that we get different pointers - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr2 ) == NO_ERROR ); - printf("UT: sbid: %i sbid2: %i\n", (unsigned int)assignPtr.sbid, (unsigned int)assignPtr2.sbid); - CPPUNIT_ASSERT( assignPtr.sbid != assignPtr2.sbid); // assigned from sb list so cannot have same sbid - CPPUNIT_ASSERT( assignPtr.type = EMPTY_PTR ); - CPPUNIT_ASSERT( assignPtr2.type = EMPTY_PTR ); - // release in 'wrong' order .. - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr2 ) == NO_ERROR ); - // .. then check that the list issues them in the return order - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr3 ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr4 ) == NO_ERROR ); -// CPPUNIT_ASSERT( assignPtr2.sbid == assignPtr3.sbid); -// CPPUNIT_ASSERT( assignPtr.sbid == assignPtr4.sbid); - - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr3 ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr4 ) == NO_ERROR ); - - //assign lots of sub-blocks - printf("UT: Assign lots of sub-blocks\n"); - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %03i fbo: %llu sbid: %02i entry: %02i in 'Assign lots of sub-blocks, assigning'\n", idx, entries[idx].fbo, (unsigned int)(entries[idx].sbid), (unsigned int)(entries[idx].entry)); - - } - - printf("UT: Releasing\n"); - - for (idx --; idx > -1; idx--) - { -// printf("UT: Visit: %i\n", idx); - printf("UT: Visit: %03i fbo: %llu sbid: %02i entry: %02i in 'Assign lots of sub-blocks, releasing'\n", idx, entries[idx].fbo, (unsigned int)(entries[idx].sbid), (unsigned int)(entries[idx].entry)); - - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &entries[idx] ) == NO_ERROR ); - } - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %03i fbo: %llu sbid: %02i entry: %02i in 'Assign lots of sub-blocks, assigning'\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - } - - - //assign lots of smaller segments - printf("UT: Assign lots of smaller segments\n"); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_2, &assignPtr ) == NO_ERROR ); - printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - - printf("UT: Assign lots of segments after init\n"); - - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( indexFile, blockZero.data, lbid ); - - // use up 32 entries - // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend over two blocks - printf("UT: Assign more than 32 segments\n"); - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i in 'Assign more than 32 segments'\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - - } - - printf("UT: Releasing\n"); - - for (idx --; idx > -1; idx--) - { -// printf("UT: Visit: %i\n", idx); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i in 'Assign more than 32 segments, releasing'\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_4, &entries[idx] ) == NO_ERROR ); - } - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i in 'Assign more than 32 segments'\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - } - - // commmenting this next bit breaks things -// CPPUNIT_ASSERT( freeMgr.init( indexFile, TREE ) == NO_ERROR ); -// dbFileOp.readDBFile( indexFile, blockZero.data, 0 ); - - printf("UT: Use almost all free space\n"); - - for (idx = 0; idx < 215; idx++) - { - result = m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i, result = %i\n", idx, assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry, result); - CPPUNIT_ASSERT ( result == NO_ERROR ); - } - - // assign one more than is available, expect error - CPPUNIT_ASSERT( (result = m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr2 )) != ERR_FM_NO_SPACE ); - // release last assigned segment - CPPUNIT_ASSERT( (result = m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr )) == NO_ERROR ); - // assign segment, expect no error - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - // assign segment after using up the list - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr2 ) != ERR_FM_NO_SPACE ); - //release the segment, expect to become first bucket - printf("UT: Putting sub-block back\n"); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - printf("UT: After putting one sb back fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 7 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 0 ); -// CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to second to last entry in sub-block - - printf("UT: Request space after using all space\n"); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) != ERR_FM_NO_SPACE ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) != ERR_FM_NO_SPACE ); -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == NO_ERROR + 1); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_2, &assignPtr ) != ERR_FM_NO_SPACE ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_1, &assignPtr ) != ERR_FM_NO_SPACE ); - - m_freeMgr.writeDBFile( indexFile, blockZero.data, lbid ); - m_freeMgr.closeFile( indexFile ); - printf("UT: End testFreeMgrAssign\n"); - - } - - - void testFreeMgrAssignList() - { - /** - * Assign tests - */ - FILE* indexFile; - - DataBlock blockZero; - IdxEmptyListEntry assignPtr, assignPtr2; - IdxEmptyListEntry assignPtr3, assignPtr4; - IdxEmptyListEntry emptyMap; - int rc; - int allocSize; - CommBlock cb; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrAssignList\n"); - - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 8, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, 0, LIST, ENTRY_32, &assignPtr ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, 0 ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, 1234, ENTRY_32, &assignPtr ) == ERR_INVALID_PARAM ); - - // Assign a segment - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - // Assign a second segment and make sure that we get different pointers - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr2 ) == NO_ERROR ); - printf("UT: sbid: %i sbid2: %i\n", (unsigned int)assignPtr.sbid, (unsigned int)assignPtr2.sbid); - CPPUNIT_ASSERT( assignPtr.sbid != assignPtr2.sbid); // assigned from sb list so cannot have same sbid - CPPUNIT_ASSERT( assignPtr.type = EMPTY_PTR ); - CPPUNIT_ASSERT( assignPtr2.type = EMPTY_PTR ); - // release in 'wrong' order .. - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr2 ) == NO_ERROR ); - // .. then check that the list issues them in the return order - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr3 ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr4 ) == NO_ERROR ); -// CPPUNIT_ASSERT( assignPtr2.sbid == assignPtr3.sbid); -// CPPUNIT_ASSERT( assignPtr.sbid == assignPtr4.sbid); - - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr3 ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr4 ) == NO_ERROR ); - - //assign lots of sub-blocks - printf("UT: Assign lots of sub-blocks\n"); - - int idx; - IdxEmptyListEntry entries[40]; - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &entries[idx] ) == NO_ERROR ); - } - - printf("UT: Releasing\n"); - - for (idx --; idx > -1; idx--) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &entries[idx] ) == NO_ERROR ); - } - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &entries[idx] ) == NO_ERROR ); - } - - //assign lots of smaller segments - printf("UT: Assign lots of smaller segments\n"); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &assignPtr ) == NO_ERROR ); - - printf("UT: Assign lots of segments after init\n"); - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - m_freeMgr.readDBFile( indexFile, blockZero.data, lbid ); - - // use up 32 entries - // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend over two blocks - printf("UT: Assign more than 32 segments\n"); - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &entries[idx] ) == NO_ERROR ); - } - - printf("UT: Releasing\n"); - - for (idx --; idx > -1; idx--) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_4, &entries[idx] ) == NO_ERROR ); - } - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &entries[idx] ) == NO_ERROR ); - } - - // printf("UT: sbid: %i entry: %i \n", (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - printf("UT: sbid: %i entry: %i \n", (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry ); - - // now use up almost all free space - for (idx = 0; idx < 215; idx++) - { - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i\n", idx, assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); - } - - // assign one more than is available, expect error - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) != ERR_FM_NO_SPACE ); - - //check empty map - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - - // check the entries are as expected - printf("UT: Empty - fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 7 ); -// CPPUNIT_ASSERT( emptyMap.entry == 0 ); // points to last entry in sub-block - - // release last assigned segment - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_16, &assignPtr ) == ERR_FM_RELEASE_ERR ); - assignPtr.group = ENTRY_16; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_16, &assignPtr ) == ERR_INVALID_PARAM ); - assignPtr.group = ENTRY_32; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - - // check the entries are as expected - printf("UT: Empty - fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 7 ); -// CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to last entry in sub-block - - // assign segment, expect no error - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - // assign segment after using up the list - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr2 ) != ERR_FM_NO_SPACE); - //release the segment, expect to become first bucket - printf("UT: Putting sub-block back\n"); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_32, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - printf("UT: After putting one sb back fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 7 ); -// CPPUNIT_ASSERT( emptyMap.sbid == 0 ); -// CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to second to last entry in sub-block - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_16, &assignPtr ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_8, &assignPtr ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_2, &assignPtr ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_1, &assignPtr ) == ERR_INVALID_PARAM ); - - m_freeMgr.writeDBFile( cb, blockZero.data, lbid ); - m_freeMgr.closeFile( indexFile ); - printf("UT: End testFreeMgrAssignList\n"); - - } - - void testFreeMgrAssignListCk() - { - /** - * Assign tests - */ - FILE* indexFile; - CommBlock cb; - int rc, allocSize; - DataBlock blockZero; - int idx; - IdxEmptyListEntry entries[100]; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrAssignListCk\n"); - - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 8, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - - printf("UT: Assign lots of segments\n"); - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - // use up 32 entries - // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend over two blocks - printf("UT: Assign more than 32 segments\n"); - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - } - - printf("UT: Releasing segments\n"); - - for (idx --; idx > -1; idx--) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, ENTRY_4, &entries[idx] ) == NO_ERROR ); - } - - printf("UT: Assign again\n"); - - for (idx = 0; idx < 39; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, ENTRY_4, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - } - - - printf("UT: End testFreeMgrAssignListCk\n"); - - } - - void testFreeMgrAssignListLots() - { - /** - * Assign tests - */ - FILE* indexFile; - DbFileOp dbFileOp; - DataBlock blockZero; - DataBlock tempBlock; - int idx, rc, allocSize; - IdxEmptyListEntry entries[4000]; - IdxEmptyListEntry map; - CommBlock cb; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrAssignListLots\n"); - - - int freemgr_type = LIST; - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 8, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, freemgr_type ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - printf("UT: Assign lots of segments\n"); - - // use up 32 entries - // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend over two blocks - printf("UT: Assign more than 32 segments\n"); - - int iters = 172; - IdxTreeGroupType entryType = ENTRY_32; - - for (idx = 0; idx < iters; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, freemgr_type, entryType, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (assigning)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - rc = m_freeMgr.readDBFile( cb, tempBlock.data, entries[idx].fbo ); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &map); - printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (assigning)\n", idx, map.fbo, (unsigned int)map.sbid, (unsigned int)map.entry); - - } - - printf("UT: Releasing segments\n"); - - for (idx = 0; idx < iters; idx++) - { -// for (idx=iters-1; idx>-1; idx--){ - printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, freemgr_type, entryType, &entries[idx] ) == NO_ERROR ); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &map); - printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (releasing)\n", idx, map.fbo, (unsigned int)map.sbid, (unsigned int)map.entry); - - } - - printf("UT: Assign again\n"); - - for (idx = 0; idx < iters; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, freemgr_type, entryType, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (assigning again)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &map); - printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (assigning again)\n", idx, map.fbo, (unsigned int)map.sbid, (unsigned int)map.entry); - - } - -// printf("UT: Releasing segments again\n"); -// -// for (idx=0; idx-1; idx--){ -// printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing again)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, freemgr_type, entryType, &entries[idx] ) == NO_ERROR ); -// getSubBlockEntry( &blockZero, 0, 1+ENTRY_32, 8, &map); -// printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (releasing again )\n", idx, map.fbo, (unsigned int)map.sbid, (unsigned int)map.entry); -// -// } - - - - printf("UT: End testFreeMgrAssignListLots\n"); - - } - - - - void testFreeMgrAssignListBlocks() - { - /** - * Assign tests - */ - FILE* indexFile; - DataBlock blockZero; - int rc, allocSize; - CommBlock cb; - int idx; - IdxEmptyListEntry entries[8000]; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrAssignListBlocks\n"); - - m_freeMgr.deleteFile( 776 ); - rc = m_freeMgr.createFile( 776, 80, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 776 ); - cb.file.oid = 776; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - printf("UT: Assign blocks from list\n"); - - // use up 32 entries - // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend over two blocks - - int iters = 77; - IdxTreeGroupType entryType = ENTRY_BLK; - - for (idx = 0; idx < iters; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, entryType, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (assigning)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - } - - printf("UT: Releasing segments\n"); - - for (idx = 0; idx < iters; idx++) - { -// for (idx=iters-1; idx>-1; idx--){ - printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, entryType, &entries[idx] ) == NO_ERROR ); - entries[idx].fbo = 0; - } - - printf("UT: Assign again\n"); - - for (idx = 0; idx < iters; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, LIST, entryType, &entries[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo: %llu sbid: %02i entry: %02i (assigning again)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - } - - for (idx = 0; idx < iters; idx++) - { -// for (idx=iters-1; idx>-1; idx--){ - printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing again)\n", idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, LIST, entryType, &entries[idx] ) == NO_ERROR ); - entries[idx].fbo = 0; - } - - - printf("UT: End testFreeMgrAssignListBlocks\n"); - - } - - - - void testFreeMgrAssignListCk2() - { - /** - * Assign tests - */ - FILE* indexFile; - DataBlock blockZero; - int rc, allocSize; - CommBlock cb; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrAssignListCk2\n"); - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 8, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - - int idx; - IdxEmptyListEntry e1[3400]; - IdxEmptyListEntry e32[3400]; - - printf("UT: Assign lots of segments\n"); - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - // use up 32 entries - // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend over two blocks - printf("UT: Assign more than 32 segments\n"); - - for (idx = 0; idx < 340; idx++) - { - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_1, &e1[idx] ) == NO_ERROR ); - printf("UT: Visit: %i fbo %llu sbid %i entry %i\n", idx, e1[idx].fbo, (unsigned int)e1[idx].sbid, (unsigned int)e1[idx].entry); - - if (idx % 10 == 0) - { - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &e32[idx] ) == NO_ERROR ); - } - - e1[idx].type = 7; - } - - printf("UT: Releasing segments\n"); - - for (idx = 0; idx < 340; idx++) - { -// printf("UT: Visit: %i\n", idx); - rc = m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_1, &e1[idx] ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - if (idx % 10 == 0) - { - rc = m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &e1[idx] ); - } - } - - printf("UT: Assign again\n"); - - for (idx = 0; idx < 340; idx++) - { - rc = m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_1, &e1[idx] ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - printf("UT: Visit: %i fbo %llu sbid %i entry %i\n", idx, e1[idx].fbo, (unsigned int)e1[idx].sbid, (unsigned int)e1[idx].entry); - } - - printf("UT: Releasing segments\n"); - - for (idx = 0; idx < 340; idx++) - { -// printf("UT: Visit: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_1, &e1[idx] ) == NO_ERROR ); - } - -// - printf("UT: End testFreeMgrAssignListCk2\n"); - - } - - - void testFreeMgrAssignExtend() - { - /** - * Test the Extend method - **/ - FILE* indexFile; - int allocSize; - int rc; - DataBlock blockZero; - CommBlock cb; - uint64_t lbid; - DbFileOp db; - - printf("UT: Begin testFreeMgrAssignExtend\n"); - m_indexlist.deleteFile( 777 ); - m_freeMgr.setDebugLevel( DEBUG_1 ); - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 800, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - CPPUNIT_ASSERT( m_freeMgr.extendFreespace( cb, &blockZero, LIST ) == NO_ERROR ); - - int currSize = db.getFileSize( indexFile ) / BYTE_PER_BLOCK ; - - lbid = m_freeMgr.mapLBID( cb, currSize - 1, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - } - - - void testFreeMgrExtendLots() - { - /** - * Test the Extend method - **/ - FILE* indexFile; - int allocSize; - int idx, rc; - DataBlock blockZero; - CommBlock cb; - uint64_t lbid; - DbFileOp db; - - printf("UT: Begin testFreeMgrExtendLots\n"); - m_indexlist.deleteFile( 777 ); - m_freeMgr.setDebugLevel( DEBUG_1 ); - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 800, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - // CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); - // lbid = m_freeMgr.mapLBID( cb, 0, rc); - // CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - for (idx = 0; idx < 5; idx++) - { - printf("Loop count: %i\n", idx); - CPPUNIT_ASSERT( m_freeMgr.extendFreespace( cb, &blockZero, LIST ) == NO_ERROR ); - } - - //int currSize = db.getFileSize( indexFile )/BYTE_PER_BLOCK ; - - //lbid = m_freeMgr.mapLBID( cb, currSize-1, rc); - // CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - printf("UT: End testFreeMgrExtendLots\n"); - - } - - - void testFreeMgrRelease() - { - FILE* indexFile; - DataBlock blockZero; - IdxEmptyListEntry assignPtr, emptyMap; - CommBlock cb; - int rc, allocSize; - uint64_t lbid; - - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrRelease\n"); - - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 80, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, 0, TREE, ENTRY_32, &assignPtr ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, 0 ) == ERR_INVALID_PARAM ); - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); - - // check the entries are as expected - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); -// CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block - - - //CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - - m_freeMgr.writeDBFile( cb, blockZero.data, 0 ); - m_freeMgr.closeFile( indexFile ); - - printf("UT: End testFreeMgrRelease\n"); - } - void testFreeMgrFragment() - { - - FILE* indexFile; - CommBlock cb; - int rc, idx, allocSize; - DataBlock blockZero; - IdxEmptyListEntry assignPtr, emptyMap; - - // test if possible to split a list of segments and populate smaller lists - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrFragment\n"); - - - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 80, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - - m_freeMgr.readSubBlockEntry( cb, &blockZero, 0, 0, 1 + ENTRY_1, 8, &emptyMap); - - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - - printf("UT: fbo %llu sbid %i entry %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_16; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); - printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_8; - assignPtr.entry = 16; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_8, 8, &emptyMap); - printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_4; - assignPtr.entry = 24; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); - printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_2; - assignPtr.entry = 28; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_2, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_2, 8, &emptyMap); - printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_1; - assignPtr.entry = 30; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_1, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); - printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_1; - assignPtr.entry = 31; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_1, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); - - // check the entries are as expected - printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - CPPUNIT_ASSERT( emptyMap.type == EMPTY_LIST ); - CPPUNIT_ASSERT( emptyMap.group == ENTRY_1 ); -// CPPUNIT_ASSERT( emptyMap.fbo == 1 ); - CPPUNIT_ASSERT( emptyMap.entry == 2 ); // points to second entry in sub-block - - /** - * Now do it many times - **/ - - for (idx = 0; idx < 185; idx++) - { - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); -// printf("\nVisit: %i\n", idx); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); -// printf("UT: Chain: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_16; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); -// printf("UT: Chain: %i fbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_8; - assignPtr.entry = 16; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_8, 8, &emptyMap); -// printf("UT: Chain: %i ffbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_4; - assignPtr.entry = 24; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); -// printf("UT: Chain: %i fbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_2; - assignPtr.entry = 28; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_2, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_2, 8, &emptyMap); -// printf("UT: Chain: %i fbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_1; - assignPtr.entry = 30; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_1, &assignPtr ) == NO_ERROR ); - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); -// printf("UT: Chain: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - assignPtr.type = EMPTY_PTR; - assignPtr.group = ENTRY_1; - assignPtr.entry = 31; - CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_1, &assignPtr ) == NO_ERROR ); - } - - //CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); - - m_freeMgr.writeDBFile( indexFile, blockZero.data, 0 ); - m_freeMgr.closeFile( indexFile ); - - printf("UT: End testFreeMgrFragment\n"); - } - - - void testFreeMgrChain() - { - - FILE* indexFile; - DataBlock blockZero; - IdxEmptyListEntry assignPtr, emptyMap; - int rc, allocSize; - CommBlock cb; - uint64_t lbid; - - // test if possible to split a list of segments and populate smaller lists - m_freeMgr.setDebugLevel( DEBUG_0 ); - printf("UT: Begin testFreeMgrChain\n"); - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 80, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - /** - * Now do it many times - **/ - - int idx; - - for (idx = 0; idx < 208; idx++) - { - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); -// printf("\nVisit: %i\n", idx); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); -// printf("UT: ENTRY_32: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); -// printf("UT: ENTRY_16: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); - - } - - - m_freeMgr.deleteFile( 777 ); - rc = m_freeMgr.createFile( 777, 10, allocSize ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - cb.file.pFile = m_freeMgr.openFile( 777 ); - cb.file.oid = 777; - indexFile = cb.file.pFile; - - CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); - lbid = m_freeMgr.mapLBID( cb, 0, rc); - CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); - - m_freeMgr.readDBFile( cb, blockZero.data, lbid ); - - printf("UT: Assigning from ENTRY_16 chain\n"); - - for (idx = 0; idx < 420; idx++) - { - CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); -// printf("Visit: %i\n", idx); - m_freeMgr.getSubBlockEntry( &blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); - printf("UT: ENTRY_16: fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); - } - - m_freeMgr.writeDBFile( cb, blockZero.data, 0 ); - m_freeMgr.closeFile( indexFile ); - - printf("UT: End testFreeMgrChain\n"); - } - - -// void testFreeMgrEvil() { -// -// FILE* indexFile; -// int rc, allocSize; -// DataBlock blockZero, workBlock; -// IdxEmptyListEntry assignPtr, emptyPtr, emptyEntry; -// CommBlock cb; -// uint64_t lbid; -// -// /** -// * Forget Google.. be evil -// **/ -// -// test if possible to split a list of segments and populate smaller lists -// m_freeMgr.setDebugLevel( DEBUG_0 ); -// printf("UT: Begin testFreeMgrEvil\n"); -// -// -// m_freeMgr.deleteFile( 777 ); -// rc = m_freeMgr.createFile( 777, 80, allocSize ); -// CPPUNIT_ASSERT( rc == NO_ERROR ); -// cb.file.pFile = m_freeMgr.openFile( 777 ); -// cb.file.oid = 777; -// indexFile = cb.file.pFile; -// -// CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); -// lbid = m_freeMgr.mapLBID( cb, 0, rc); -// CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); -// -// m_freeMgr.readDBFile( cb, blockZero.data, lbid ); -// -// m_freeMgr.getSubBlockEntry( &blockZero, 0, 1+ENTRY_32, 8, &emptyPtr); -// -// check the entries are as expected -// CPPUNIT_ASSERT( emptyPtr.type == EMPTY_LIST ); -// CPPUNIT_ASSERT( emptyPtr.group == ENTRY_32 ); -// CPPUNIT_ASSERT( emptyPtr.fbo == 1 ); -// CPPUNIT_ASSERT( emptyPtr.entry == 31 ); // points to last entry in sub-block -// -// printf("UT: Mess up fbo in emptyMap for ENTRY_32\n"); -// emptyPtr.fbo =999999999; -// m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_32, 8, &emptyPtr); -// -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == ERR_FM_BAD_FBO ); -// emptyPtr.fbo =7; -// m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_32, 8, &emptyPtr); -// -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); -// -// CPPUNIT_ASSERT( assignPtr.fbo == 7 ); -// assignPtr.fbo =1234; -// first release with bad fbo then with good -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == ERR_FM_BAD_FBO ); -// -// assignPtr.fbo =7; -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); -// -// -// printf("UT: Mess up fbo in emptyMap for ENTRY_16\n"); -// m_freeMgr.getSubBlockEntry( &blockZero, 0, 1+ENTRY_16, 8, &emptyPtr); -// CPPUNIT_ASSERT( emptyPtr.fbo == 0 ); // zero because nothing in list yet -// -// emptyPtr.fbo =999999999; -// m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_16, 8, &emptyPtr); -// -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == ERR_FM_BAD_FBO ); -// emptyPtr.fbo =0; -// m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_16, 8, &emptyPtr); -// -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); -// -// CPPUNIT_ASSERT( assignPtr.fbo == 7 ); -// assignPtr.fbo =1234; -// first release with bad fbo then with good -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == ERR_FM_BAD_FBO ); -// -// assignPtr.fbo =7; -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); -// -// Mess with type field -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); -// -// m_freeMgr.getSubBlockEntry( blockZero.data, 0, 1+ENTRY_16, 8, &emptyPtr ); -// m_freeMgr.readDBFile( cb, &workBlock, emptyPtr.fbo ); -// m_freeMgr.getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); -// -// CPPUNIT_ASSERT( emptyEntry.type == 2 ); //check is EMPTY_PTR -// CPPUNIT_ASSERT( emptyEntry.fbo == 7 ); //check is last sb -// emptyEntry.type = 7; // set to garbage -// m_freeMgr.writeSubBlockEntry( cb, &workBlock, emptyPtr.fbo, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); -// check assign fails -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == ERR_FM_BAD_TYPE ); -// -// emptyEntry.type = 2; -// emptyEntry.fbo = 0; -// m_freeMgr.writeSubBlockEntry( cb, &workBlock, emptyPtr.fbo, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); -// check assign fails -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == ERR_FM_BAD_TYPE ); -// -// -// fix type field and re-try assign -// emptyEntry.type = 2; -// emptyEntry.fbo = 7; -// m_freeMgr.writeSubBlockEntry( cb, &workBlock, emptyPtr.fbo, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR ); -// -// -// printf("UT: Zero fbo in emptyMap for ENTRY_8\n"); -// //start 8 list -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); -// -// m_freeMgr.getSubBlockEntry( &blockZero, 0, 1+ENTRY_8, 8, &emptyPtr); -// CPPUNIT_ASSERT( emptyPtr.fbo == 7 ); -// CPPUNIT_ASSERT( emptyPtr.type == 2 ); -// -// emptyPtr.fbo = 0; -// m_freeMgr.writeSubBlockEntry( indexFile, &blockZero, 0, 0, 1+ENTRY_8, 8, &emptyPtr); -// -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR + 1 ); -// //cleanup -// emptyPtr.fbo = 7; -// m_freeMgr.writeSubBlockEntry( indexFile, &blockZero, 0, 0, 1+ENTRY_8, 8, &emptyPtr); -// -// check that assign and release works ok -// CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); -// CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR ); -// -// m_freeMgr.writeDBFile( indexFile, blockZero.data, 0 ); -// CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); -// m_freeMgr.closeFile( indexFile ); -// -// printf("UT: End testFreeMgrEvil\n"); -// -// } - void testIndexListMultiKey() - { - - DataBlock curBlock; - FILE* pFile = NULL; - int rc, allocSize; - uint64_t key = 0x123; - int rowId; - int file_number = 20000; - int fbo, sbid, entry ; - IdxEmptyListEntry newIdxListHdrPtr; - IdxRidListHdr newIdxRidListHdr; - - printf("\nRunning multiple keys for adding header \n"); - BRMWrapper::setUseBrm(true); - //Then increment the rowId as it goes - - rowId = 0x567 ; - - /******************************************************* - * TEST with FreeMgr for coverage - * Start here for add header and delete header entries - * - *******************************************************/ - m_indexlist.deleteFile(file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - CommBlock cb; - File file; - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - m_indexlist.init(cb, LIST); - - int loop; - - for (loop = 0; loop < 10 ; loop++) - { - rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); - - if (rc != NO_ERROR) - printf("196. error code:%i \n", rc); - - //printf("The fbo is:%i sbid is %i entry is %i \n",(int)newIdxListHdrPtr.fbo, (int)newIdxListHdrPtr.sbid, (int)newIdxListHdrPtr.entry); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fbo = newIdxListHdrPtr.fbo; //fbo=? from the free manager - sbid = newIdxListHdrPtr.sbid;//sbid =? from the free manager - entry = newIdxListHdrPtr.entry;//entry=? from the free manager - memset( curBlock.data, 0, sizeof(curBlock.data)); - - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, LIST_HDR_SIZE, - &newIdxRidListHdr ); - //cout<< "rowId->" << rowId << " key->" << key << " size->" << - //newIdxRidListHdr.idxRidListSize.size << endl; - rowId++; - key++; - } - - //cout << " \n Successfully ran testIndexListMultiKey \n" << endl; - return; - }//testIndexListMultiKey - - void testIndexListUpdate() - { - - DataBlock curBlock; - FILE* pFile = NULL; - int rc, fbo, sbid, entry ; - uint64_t key = 123; - int count = 40000; - int rowId; - int i; - int file_number = 8001; - - IdxEmptyListEntry newIdxListHdrPtr; - IdxRidListHdr newIdxRidListHdr; - //bool found = false; - //int p_sbid, p_entry, - //uint64_t p_fbo; - int allocSize; - - BRMWrapper::setUseBrm(true); - printf("\nRunning testIndexListUpdate\n"); - - memset( curBlock.data, 0, sizeof(curBlock.data)); - m_indexlist.deleteFile(file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - //Free Manager - - CommBlock cb; - File file; - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - m_indexlist.init(cb, LIST); - - //Initialize first rowId to 0 - - rowId = 0 ; - rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); + switch (i) + { + case 0: + key = 123; + width = 8; + rid = 1; + break; + + case 1: + key = 0xA1C9; + width = 16; + rid = 100; + break; + + case 2: + key = 0xCA6E6D8E; + width = 32; + rid = 1000; + break; + } + + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (j = 0; j < count; j++) + { + printf("\nj=%d", j); + rc = m_index.updateIndex(key + j, width, rid + j); if (rc != NO_ERROR) - //printf("245. error code:%i \n", rc); - CPPUNIT_ASSERT( rc == NO_ERROR ); + printf("\nrc=%d", rc); - fbo = newIdxListHdrPtr.fbo; //fbo=? from the free manager - sbid = newIdxListHdrPtr.sbid;//sbid =? from the free manager - entry = newIdxListHdrPtr.entry;//entry=? from the free manager - memset( curBlock.data, 0, sizeof(curBlock.data)); + CPPUNIT_ASSERT(rc == NO_ERROR); + myTree = m_index.getTree(); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, LIST_HDR_SIZE, - &newIdxRidListHdr ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == 1 ); - rowId++; - //cout << " Single Insert->start Timer :inserting " << count << " rowId " << endl; - m_indexlist.startfTimer(); + m_index.setAssignFbo(1 + j); + } - for (i = 0; i < count; i++) - { - ////cout << " count number-> " << i << endl; - rc = m_indexlist.updateIndexList(cb, rowId, key, &newIdxListHdrPtr); + for (j = 0; j < count; j++) + { + rc = m_index.deleteIndex(key + j, width, rid + j); + CPPUNIT_ASSERT(rc == NO_ERROR); + } - if (rc != NO_ERROR) - { - //cout << "error code->" << rc << " i->" << i << endl; - return; - } + m_index.closeIndex(); + } + } - rowId++; - } + void testTreeGetTestbitValue() + { + uint64_t key; + int width, bittestVal; + bool bStatus; + BRMWrapper::setUseBrm(false); + // test 8 bit + width = 8; + key = 35; + bStatus = m_index.getTestbitValue(key, width, 0, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 4); + bStatus = m_index.getTestbitValue(key, width, 1, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 3); + bStatus = m_index.getTestbitValue(key, width, 2, &bittestVal); + CPPUNIT_ASSERT(bStatus == false); - m_indexlist.stopfTimer(); - //cout << " Single Insert ->End Timer :inserting " << count << " rowId " << endl; - //cout << " Total used time (msec) " << m_indexlist.getTotalfRunTime() << endl; - m_indexlist.closeFile(pFile); - // - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, LIST_HDR_SIZE, - &newIdxRidListHdr ); - //cout<< " key->" << key << " size->" << - //newIdxRidListHdr.idxRidListSize.size << endl; - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)(count + 1)); - RID ridArray[count + 1]; - int size; - m_indexlist.getRIDArrayFromListHdr(pFile, key, + // test 16 bit + width = 16; + key = 0xA1C9; + bStatus = m_index.getTestbitValue(key, width, 0, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 20); + bStatus = m_index.getTestbitValue(key, width, 1, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 7); + bStatus = m_index.getTestbitValue(key, width, 2, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 4); + bStatus = m_index.getTestbitValue(key, width, 3, &bittestVal); + CPPUNIT_ASSERT(bittestVal == 1); + CPPUNIT_ASSERT(bStatus == true); + bStatus = m_index.getTestbitValue(key, width, 4, &bittestVal); + CPPUNIT_ASSERT(bStatus == false); + + // test 32 bit + width = 32; + key = 0xCA6E6D8E; + bStatus = m_index.getTestbitValue(key, width, 0, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 25); + bStatus = m_index.getTestbitValue(key, width, 1, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 9); + bStatus = m_index.getTestbitValue(key, width, 2, &bittestVal); + CPPUNIT_ASSERT(bStatus == true); + CPPUNIT_ASSERT(bittestVal == 23); + bStatus = m_index.getTestbitValue(key, width, 3, &bittestVal); + CPPUNIT_ASSERT(bittestVal == 6); + CPPUNIT_ASSERT(bStatus == true); + bStatus = m_index.getTestbitValue(key, width, 4, &bittestVal); + CPPUNIT_ASSERT(bittestVal == 27); + CPPUNIT_ASSERT(bStatus == true); + bStatus = m_index.getTestbitValue(key, width, 5, &bittestVal); + CPPUNIT_ASSERT(bittestVal == 3); + CPPUNIT_ASSERT(bStatus == true); + bStatus = m_index.getTestbitValue(key, width, 6, &bittestVal); + CPPUNIT_ASSERT(bittestVal == 2); + CPPUNIT_ASSERT(bStatus == true); + bStatus = m_index.getTestbitValue(key, width, 7, &bittestVal); + CPPUNIT_ASSERT(bStatus == false); + } + + void testTreeGetTreeNodeInfo() + { + int rc, fbo = 6, sbid = 1, entry = 2, testbitVal = 5; + int curSbid = 3, curEntry = 2, allocCount, realCount; + IdxTreeGroupType curGroup; + bool entryMap[ENTRY_PER_SUBBLOCK]; + DataBlock block; + IdxBitTestEntry bittestEntry; + + // test ENTRY_1 + m_index.clearBlock(&block); + curGroup = ENTRY_1; + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo, sbid, entry); + m_index.setSubBlockEntry(block.data, curSbid, curEntry, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 3, curGroup, fbo + 6, sbid + 3, entry + 1); + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 1, 8, &bittestEntry); + + rc = m_index.getTreeNodeInfo(&block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(allocCount == 1); + CPPUNIT_ASSERT(realCount == 1); + CPPUNIT_ASSERT(entryMap[0] == true); + + // test ENTRY_2 + m_index.clearBlock(&block); + curGroup = ENTRY_2; + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo, sbid, entry); + m_index.setSubBlockEntry(block.data, curSbid, curEntry, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 3, curGroup, fbo + 6, sbid + 3, entry + 1); + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 1, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo + 6, sbid + 3, entry + 1); + bittestEntry.type = EMPTY_LIST; + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 2, 8, &bittestEntry); + + rc = m_index.getTreeNodeInfo(&block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(allocCount == 2); + CPPUNIT_ASSERT(realCount == 2); + CPPUNIT_ASSERT(entryMap[0] == true); + CPPUNIT_ASSERT(entryMap[1] == true); + CPPUNIT_ASSERT(entryMap[2] == false); + + // test ENTRY_8 + m_index.clearBlock(&block); + curGroup = ENTRY_8; + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo, sbid, entry); + m_index.setSubBlockEntry(block.data, curSbid, curEntry, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo + 6, sbid + 3, entry + 1); + bittestEntry.type = EMPTY_LIST; + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 3, 8, &bittestEntry); + + rc = m_index.getTreeNodeInfo(&block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap); + CPPUNIT_ASSERT(rc == ERR_IDX_TREE_INVALID_TYPE); + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo + 6, sbid + 3, entry + 1); + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 3, 8, &bittestEntry); + + bittestEntry.type = EMPTY_ENTRY; + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 2, 2, &bittestEntry); + + rc = m_index.getTreeNodeInfo(&block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(allocCount == 8); + CPPUNIT_ASSERT(realCount == 2); + CPPUNIT_ASSERT(entryMap[0] == true); + CPPUNIT_ASSERT(entryMap[1] == false); + CPPUNIT_ASSERT(entryMap[2] == false); + CPPUNIT_ASSERT(entryMap[3] == true); + CPPUNIT_ASSERT(entryMap[4] == false); + } + + void testTreeGetTreeMatchEntry() + { + int rc, fbo = 6, sbid = 1, entry = 2, testbitVal = 5; + int curSbid = 3, curEntry = 2, allocCount, realCount, matchEntry; + IdxTreeGroupType curGroup; + bool entryMap[ENTRY_PER_SUBBLOCK]; + DataBlock block; + IdxBitTestEntry bittestEntry, checkEntry; + bool bFound; + + // ENTRY_8 + m_index.clearBlock(&block); + curGroup = ENTRY_8; + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo, sbid, entry); + m_index.setSubBlockEntry(block.data, curSbid, curEntry, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 5, curGroup, fbo + 9, sbid + 6, entry + 2); + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 4, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 9, curGroup, fbo + 6, sbid + 3, entry + 1); + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 3, 8, &bittestEntry); + + bittestEntry.type = EMPTY_ENTRY; + m_index.setSubBlockEntry(block.data, curSbid, curEntry + 2, 2, &bittestEntry); + + rc = m_index.getTreeNodeInfo(&block, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(allocCount == 8); + CPPUNIT_ASSERT(realCount == 3); + CPPUNIT_ASSERT(entryMap[0] == true); + CPPUNIT_ASSERT(entryMap[1] == false); + CPPUNIT_ASSERT(entryMap[2] == false); + CPPUNIT_ASSERT(entryMap[3] == true); + CPPUNIT_ASSERT(entryMap[4] == true); + CPPUNIT_ASSERT(entryMap[5] == false); + + memset(&checkEntry, 0, 8); + checkEntry.type = BIT_TEST; + checkEntry.bitTest = testbitVal + 4; + bFound = m_index.getTreeMatchEntry(&block, curSbid, curEntry, 8, allocCount, entryMap, &matchEntry, + &checkEntry); + CPPUNIT_ASSERT(bFound == false); + + checkEntry.bitTest = testbitVal + 9; + bFound = m_index.getTreeMatchEntry(&block, curSbid, curEntry, 8, allocCount, entryMap, &matchEntry, + &checkEntry); + CPPUNIT_ASSERT(bFound == true); + CPPUNIT_ASSERT(matchEntry == 3); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 5, curGroup, fbo + 9, sbid + 6, entry + 2); + checkEntry.bitTest = testbitVal + 5; + bFound = m_index.getTreeMatchEntry(&block, curSbid, curEntry, 8, allocCount, entryMap, &matchEntry, + &checkEntry); + CPPUNIT_ASSERT(bFound == true); + CPPUNIT_ASSERT(matchEntry == 4); + CPPUNIT_ASSERT(!memcmp(&checkEntry, &bittestEntry, 8)); + } + + void testTreeMoveEntry() + { + int rc, fbo = 6, sbid = 1, entry = 2, testbitVal = 5; + int curSbid = 3, curEntry = 2, allocCount, realCount, moveCount; + IdxTreeGroupType curGroup; + bool entryMap[ENTRY_PER_SUBBLOCK]; + DataBlock oldBlock, newBlock, testBlock, blankBlock; + IdxBitTestEntry bittestEntry; + FILE* treeFile; + + if (m_index.exists(990)) + { + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + BRMWrapper::setUseBrm(false); + rc = m_index.createIndex(990, 991, false); + CPPUNIT_ASSERT(rc == NO_ERROR); + + treeFile = m_index.openFile(990); + CPPUNIT_ASSERT(treeFile != NULL); + + // ENTRY_8 + m_index.clearBlock(&oldBlock); + m_index.clearBlock(&newBlock); + m_index.clearBlock(&testBlock); + m_index.clearBlock(&blankBlock); + curGroup = ENTRY_8; + + m_index.setBittestEntry(&bittestEntry, testbitVal, curGroup, fbo, sbid, entry); + m_index.setSubBlockEntry(oldBlock.data, curSbid, curEntry, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 5, curGroup, fbo + 9, sbid + 6, entry + 2); + m_index.setSubBlockEntry(oldBlock.data, curSbid, curEntry + 4, 8, &bittestEntry); + + m_index.setBittestEntry(&bittestEntry, testbitVal + 9, curGroup, fbo + 6, sbid + 3, entry + 1); + m_index.setSubBlockEntry(oldBlock.data, curSbid, curEntry + 3, 8, &bittestEntry); + + bittestEntry.type = EMPTY_ENTRY; + m_index.setSubBlockEntry(oldBlock.data, curSbid, curEntry + 2, 2, &bittestEntry); + + rc = + m_index.getTreeNodeInfo(&oldBlock, curSbid, curEntry, 8, curGroup, &allocCount, &realCount, entryMap); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(allocCount == 8); + CPPUNIT_ASSERT(realCount == 3); + CPPUNIT_ASSERT(entryMap[0] == true); + CPPUNIT_ASSERT(entryMap[1] == false); + CPPUNIT_ASSERT(entryMap[2] == false); + CPPUNIT_ASSERT(entryMap[3] == true); + CPPUNIT_ASSERT(entryMap[4] == true); + CPPUNIT_ASSERT(entryMap[5] == false); + + rc = m_index.writeDBFile(treeFile, &oldBlock, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_index.closeFile(treeFile); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.moveEntry(3, curSbid, curEntry, 8, 5, curSbid + 10, curEntry + 1, curGroup, allocCount, + entryMap, &moveCount); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(moveCount == 3); + + m_index.closeIndex(); + + treeFile = m_index.openFile(990); + CPPUNIT_ASSERT(treeFile != NULL); + rc = m_index.readDBFile(treeFile, &testBlock, 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(!memcmp(testBlock.data, blankBlock.data, sizeof(testBlock.data))); + + rc = m_index.readDBFile(treeFile, &testBlock, 5); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.getTreeNodeInfo(&testBlock, curSbid + 10, curEntry + 1, 8, curGroup, &allocCount, &realCount, + entryMap); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(allocCount == 8); + CPPUNIT_ASSERT(realCount == 3); + CPPUNIT_ASSERT(entryMap[0] == true); + CPPUNIT_ASSERT(entryMap[1] == true); + CPPUNIT_ASSERT(entryMap[2] == true); + CPPUNIT_ASSERT(entryMap[3] == false); + CPPUNIT_ASSERT(entryMap[4] == false); + CPPUNIT_ASSERT(entryMap[5] == false); + + m_index.closeFile(treeFile); + } + + void testTreeNodeAssert(IdxTree myTree, int curLevel, uint64_t curBitTest, uint16_t curGroup, + uint64_t curFbo, uint64_t curSbid, uint64_t curEntry, uint64_t nextFbo, + uint64_t nextSbid, uint64_t nextEntry) + { + CPPUNIT_ASSERT(myTree.node[curLevel].current.bitTest == curBitTest); + CPPUNIT_ASSERT(myTree.node[curLevel].current.group == curGroup); + CPPUNIT_ASSERT(myTree.node[curLevel].current.fbo == curFbo); + CPPUNIT_ASSERT(myTree.node[curLevel].current.sbid == curSbid); + CPPUNIT_ASSERT(myTree.node[curLevel].current.entry == curEntry); + + if (curLevel == 0) + CPPUNIT_ASSERT(myTree.node[curLevel].current.type == BITMAP_PTR); + else if (curLevel == myTree.maxLevel - 1) + CPPUNIT_ASSERT(myTree.node[curLevel].current.type == LEAF_LIST); + else + CPPUNIT_ASSERT(myTree.node[curLevel].current.type == BIT_TEST); + + CPPUNIT_ASSERT(myTree.node[curLevel].next.fbo == nextFbo); + CPPUNIT_ASSERT(myTree.node[curLevel].next.sbid == nextSbid); + CPPUNIT_ASSERT(myTree.node[curLevel].next.entry == nextEntry); + + CPPUNIT_ASSERT(myTree.node[curLevel].allocCount == (0x1 << curGroup)); + + CPPUNIT_ASSERT(myTree.node[curLevel].used == true); + } + + void testTreeBuildEmptyTree() + { + int rc, width = 8, rid = 3; + uint64_t key = 123; + IdxTree myTree; + BRMWrapper::setUseBrm(false); + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.buildEmptyTreePart(key, width, rid, 0); + CPPUNIT_ASSERT(rc == ERR_IDX_TREE_INVALID_LEVEL); + + rc = m_index.buildEmptyTreePart(key, 4, rid, 0); + CPPUNIT_ASSERT(rc == ERR_IDX_TREE_INVALID_LEVEL); + + rc = m_index.updateIndex(key, width, rid); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // width = 8 + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 2); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)rid); + + testTreeNodeAssert(myTree, 0, 15, ENTRY_32, 0, 1, 15, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 3, ENTRY_1, 1, 3, 4, /* */ 3, 3, 4); + CPPUNIT_ASSERT(myTree.node[2].used == false); + + // width = 16 + key = 0xA1C9; + width = 16; + rc = m_index.updateIndex(key, width, rid); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + + CPPUNIT_ASSERT(myTree.maxLevel == 4); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)rid); + + testTreeNodeAssert(myTree, 0, 20, ENTRY_32, 0, 1, 20, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 7, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4); + testTreeNodeAssert(myTree, 2, 4, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4); + testTreeNodeAssert(myTree, 3, 1, ENTRY_1, 1, 5, 4, /* */ 3, 3, 6); + CPPUNIT_ASSERT(myTree.node[4].used == false); + + // width = 32 + key = 0xCA6E6D8E; + width = 32; + rc = m_index.updateIndex(key, width, rid); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + + CPPUNIT_ASSERT(myTree.maxLevel == 7); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)rid); + + testTreeNodeAssert(myTree, 0, 25, ENTRY_32, 0, 1, 25, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 9, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4); + testTreeNodeAssert(myTree, 2, 23, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4); + testTreeNodeAssert(myTree, 3, 6, ENTRY_1, 1, 5, 4, /* */ 1, 6, 4); + testTreeNodeAssert(myTree, 4, 27, ENTRY_1, 1, 6, 4, /* */ 1, 7, 4); + testTreeNodeAssert(myTree, 5, 3, ENTRY_1, 1, 7, 4, /* */ 1, 8, 4); + testTreeNodeAssert(myTree, 6, 2, ENTRY_1, 1, 8, 4, /* */ 3, 3, 9); + + CPPUNIT_ASSERT(myTree.node[7].used == false); + + m_index.closeIndex(); + } + + void testTreeBuildExistTree1() + { + int rc, width = 8, rid = 3; + uint64_t key = 123; + IdxTree myTree; + BRMWrapper::setUseBrm(false); + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.updateIndex(key, width, rid); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // width = 8 + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 2); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)rid); + + testTreeNodeAssert(myTree, 0, 15, ENTRY_32, 0, 1, 15, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 3, ENTRY_1, 1, 3, 4, /* */ 3, 3, 4); + CPPUNIT_ASSERT(myTree.node[2].used == false); + + // insert again + m_index.setAssignFbo(1); + rc = m_index.updateIndex(key, width, rid + 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 2); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 3)); + + testTreeNodeAssert(myTree, 0, 15, ENTRY_32, 0, 1, 15, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 3, ENTRY_1, 1, 3, 4, /* */ 3, 3, 4); + + CPPUNIT_ASSERT(myTree.node[2].used == false); + + // insert with a little different key + m_index.setAssignFbo(2); + rc = m_index.updateIndex(key + 1, width, rid + 4); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 2); + CPPUNIT_ASSERT(myTree.key == key + 1); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 4)); + + testTreeNodeAssert(myTree, 0, 15, ENTRY_32, 0, 1, 15, /* */ 3, 4, 4); + testTreeNodeAssert(myTree, 1, 4, ENTRY_2, 3, 4, 4, /* */ 3, 3, 4); + CPPUNIT_ASSERT(myTree.node[2].used == false); + + // insert with a different key and different rid + m_index.setAssignFbo(3); + rc = m_index.updateIndex(key + 2, width, rid + 5); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 2); + CPPUNIT_ASSERT(myTree.key == key + 2); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 5)); + + testTreeNodeAssert(myTree, 0, 15, ENTRY_32, 0, 1, 15, /* */ 4, 4, 4); + testTreeNodeAssert(myTree, 1, 5, ENTRY_4, 4, 4, 4, /* */ 3, 3, 4); + + CPPUNIT_ASSERT(myTree.node[1].useCount == 3); + CPPUNIT_ASSERT(myTree.node[2].used == false); + + // insert with a different key and different rid + m_index.setAssignFbo(4); + rc = m_index.updateIndex(key + 3, width, rid + 7); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 2); + CPPUNIT_ASSERT(myTree.key == key + 3); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 7)); + + testTreeNodeAssert(myTree, 0, 15, ENTRY_32, 0, 1, 15, /* */ 4, 4, 4); + testTreeNodeAssert(myTree, 1, 6, ENTRY_4, 4, 4, 4, /* */ 3, 3, 4); + + CPPUNIT_ASSERT(myTree.node[2].used == false); + + m_index.closeIndex(); + } + + void testTreeBuildExistTree2() + { + int rc, width = 16, rid = 3; + uint64_t key = 0xA1C9; + IdxTree myTree; + BRMWrapper::setUseBrm(false); + rc = m_index.dropIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.createIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.updateIndex(key, width, rid); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // width = 16 + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 4); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)rid); + + testTreeNodeAssert(myTree, 0, 20, ENTRY_32, 0, 1, 20, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 7, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4); + testTreeNodeAssert(myTree, 2, 4, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4); + testTreeNodeAssert(myTree, 3, 1, ENTRY_1, 1, 5, 4, /* */ 3, 3, 6); + + CPPUNIT_ASSERT(myTree.node[4].used == false); + + // insert again + m_index.setAssignFbo(1); + rc = m_index.updateIndex(key, width, rid + 3); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 4); + CPPUNIT_ASSERT(myTree.key == key); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 3)); + testTreeNodeAssert(myTree, 0, 20, ENTRY_32, 0, 1, 20, /* */ 1, 3, 4); + testTreeNodeAssert(myTree, 1, 7, ENTRY_1, 1, 3, 4, /* */ 1, 4, 4); + testTreeNodeAssert(myTree, 2, 4, ENTRY_1, 1, 4, 4, /* */ 1, 5, 4); + testTreeNodeAssert(myTree, 3, 1, ENTRY_1, 1, 5, 4, /* */ 3, 3, 6); + + CPPUNIT_ASSERT(myTree.node[4].used == false); + + // insert with a little different key + m_index.setAssignFbo(2); + rc = m_index.updateIndex(key + 1, width, rid + 4); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 4); + CPPUNIT_ASSERT(myTree.key == key + 1); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 4)); + + testTreeNodeAssert(myTree, 0, 20, ENTRY_32, 0, 1, 20, /* */ 1, 3, 4); + // testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1,3,4, /* */ 3, 5, 5 ); + testTreeNodeAssert(myTree, 2, 5, ENTRY_2, 3, 5, 5, /* */ 3, 3, 4); + testTreeNodeAssert(myTree, 3, 0, ENTRY_1, 3, 3, 4, /* */ 3, 3, 6); + CPPUNIT_ASSERT(myTree.node[4].used == false); + + // insert with a different key and different rid + m_index.setAssignFbo(3); + rc = m_index.updateIndex(key + 2, width, rid + 5); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 4); + CPPUNIT_ASSERT(myTree.key == key + 2); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 5)); + + testTreeNodeAssert(myTree, 0, 20, ENTRY_32, 0, 1, 20, /* */ 1, 3, 4); + // testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1,3,4, /* */ 3, 5, 5 ); + testTreeNodeAssert(myTree, 2, 5, ENTRY_2, 3, 5, 5, /* */ 4, 6, 4); + testTreeNodeAssert(myTree, 3, 1, ENTRY_2, 4, 6, 4, /* */ 3, 3, 6); + CPPUNIT_ASSERT(myTree.node[4].used == false); + + // insert with a different key and different rid + m_index.setAssignFbo(4); + rc = m_index.updateIndex(key + 3, width, rid + 7); + CPPUNIT_ASSERT(rc == NO_ERROR); + + myTree = m_index.getTree(); + CPPUNIT_ASSERT(myTree.maxLevel == 4); + CPPUNIT_ASSERT(myTree.key == key + 3); + CPPUNIT_ASSERT(myTree.width == width); + CPPUNIT_ASSERT(myTree.rid == (RID)(rid + 7)); + + testTreeNodeAssert(myTree, 0, 20, ENTRY_32, 0, 1, 20, /* */ 1, 3, 4); + // testTreeNodeAssert( myTree, 1, 7, ENTRY_1, 1,3,4, /* */ 3, 5, 5 ); + testTreeNodeAssert(myTree, 2, 6, ENTRY_4, 5, 5, 6, /* */ 5, 3, 4); + testTreeNodeAssert(myTree, 3, 0, ENTRY_1, 5, 3, 4, /* */ 3, 3, 6); + CPPUNIT_ASSERT(myTree.node[4].used == false); + + m_index.closeIndex(); + + // set a blank entry + IdxBitTestEntry curEntry; + FILE* treeFile; + DataBlock curBlock; + + treeFile = m_index.openFile(990); + CPPUNIT_ASSERT(treeFile != NULL); + + rc = m_index.readSubBlockEntry(treeFile, &curBlock, 1, 3, 4, MAX_COLUMN_BOUNDARY, &curEntry); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setBlankEntry(&curEntry); + + rc = m_index.writeSubBlockEntry(treeFile, &curBlock, 1, 3, 4, MAX_COLUMN_BOUNDARY, &curEntry); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.closeFile(treeFile); + + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.updateIndex(key, width, rid); + CPPUNIT_ASSERT(rc == ERR_STRUCT_EMPTY); + + m_index.closeIndex(); + } + + void testTreeIntegration() + { + IdxEmptyListEntry myEntry; + int rc; + BRMWrapper::setUseBrm(false); + rc = m_index.openIndex(990, 991); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.assignSegment(ENTRY_1, &myEntry, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.releaseSegment(ENTRY_1, &myEntry); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.setUseFreeMgr(true); + rc = m_index.assignSegment(ENTRY_1, &myEntry, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_index.releaseSegment(ENTRY_1, &myEntry); + CPPUNIT_ASSERT(rc == NO_ERROR); + + m_index.closeIndex(); + } + + void testFreeMgrBRM() + { + int rc; + int allocSize; + CommBlock cb; + uint64_t lbid; + uint64_t fbo; + m_freeMgr.setDebugLevel(DEBUG_0); + printf("\nUT: Begin testFreeMgrBRM\n"); + + m_freeMgr.deleteFile(777); + m_freeMgr.createFile(777, 80, allocSize); + m_freeMgr.createFile(778, 80, allocSize); + + cb.file.oid = 777; + fbo = 0; + lbid = m_freeMgr.mapLBID(cb, fbo, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); + + cb.file.oid = 777; + fbo = 10; + lbid = m_freeMgr.mapLBID(cb, fbo, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); + + cb.file.oid = 777; + fbo = 1023; + lbid = m_freeMgr.mapLBID(cb, fbo, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); + + cb.file.oid = 777; + fbo = 1020; + lbid = m_freeMgr.mapLBID(cb, fbo, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); + + cb.file.oid = 778; + fbo = 0; + lbid = m_freeMgr.mapLBID(cb, fbo, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); + + cb.file.oid = 778; + fbo = 1000; + lbid = m_freeMgr.mapLBID(cb, fbo, rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("OID: %u FBO: %llu LBID: %llu\n", cb.file.oid, fbo, lbid); + + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testFreeMgrInit() + { + FILE* indexFile; + DataBlock blockZero; + IdxEmptyListEntry emptyMap, assignPtr; + CommBlock cb; + int allocSize, rc; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_1); + printf("\nUT: Begin testFreeMgrInit\n"); + + assignPtr.fbo = 0; + assignPtr.sbid = 0; + assignPtr.entry = 0; + assignPtr.type = 0; + + cb.file.pFile = NULL; + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == ERR_INVALID_PARAM); + m_freeMgr.deleteFile(777); + m_freeMgr.deleteFile(778); + m_freeMgr.createFile( + 778, 8, allocSize); // create a second file to try and force lbid range to not collide with fbo + m_freeMgr.createFile(777, 8, allocSize); + + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + + // m_indexlist.createFile(777,800, allocSize); + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, 1234) == ERR_INVALID_PARAM); + + // init a freemgr for a tree + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb.file.pFile, blockZero.data, lbid); + m_freeMgr.getSubBlockEntry(blockZero.data, 0, 1 + ENTRY_32, 8, &emptyMap); + // check the entries are as expected + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); + // check the entries are as expected + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_16); + // printf("UT: fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + // (unsigned int)emptyMap.entry); + CPPUNIT_ASSERT(emptyMap.fbo == 0); + CPPUNIT_ASSERT(emptyMap.entry == 0); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_8, 8, &emptyMap); + // check the entries are as expected + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_8); + CPPUNIT_ASSERT(emptyMap.fbo == 0); + CPPUNIT_ASSERT(emptyMap.entry == 0); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); + // check the entries are as expected + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_4); + CPPUNIT_ASSERT(emptyMap.fbo == 0); + CPPUNIT_ASSERT(emptyMap.entry == 0); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_2, 8, &emptyMap); + // check the entries are as expected + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_2); + CPPUNIT_ASSERT(emptyMap.fbo == 0); + CPPUNIT_ASSERT(emptyMap.entry == 0); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); + // check the entries are as expected + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_1); + CPPUNIT_ASSERT(emptyMap.fbo == 0); + CPPUNIT_ASSERT(emptyMap.entry == 0); + + // assign a segment and check that entries in sb0 change + + printf("UT: Init: Assign ENTRY_32 fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, + (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(assignPtr.entry == 0); // points to start of sub-block + // CPPUNIT_ASSERT( assignPtr.fbo == 1 ); // + CPPUNIT_ASSERT(assignPtr.group == ENTRY_32); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.entry == 30 ); // points to second to last entry in sub-block + + // release segment and see if head pointer moves + rc = m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 31 ); + // CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block + + // release to wrong list + printf("UT: Put sub-block segment on list of 16 (expect error)\n"); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == + ERR_FM_RELEASE_ERR); + + // tweak type then try to release + // printf("UT: Release EMPTY_LIST entry\n"); + // assignPtr.type = EMPTY_LIST; + // CPPUNIT_ASSERT( freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == + // ERR_FM_BAD_TYPE ); + + printf("UT: assign ENTRY_16 segment\n"); + // assign ENTRY_16 segment + CPPUNIT_ASSERT_MESSAGE("Failed assigning ENTRY_16 segment", + m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); + + CPPUNIT_ASSERT_MESSAGE("Empty map type should be EMPTY_LIST", emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_16); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 0 ); + // CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to last entry in sub-block + + // release ENTRY_16 segment + printf("UT: Release ENTRY_16 segment\n"); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + + // legal segment size but list is empty - causes list to be populated + printf("UT: Assign from ENTRY_4 list - which is empty\n"); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_4, &assignPtr) == NO_ERROR); + printf("UT: After Assign ENTRY_4 fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, + (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); + + // Check ENTRY_4 in map + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); + printf("UT: fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + // ENTRY_4 segment size + printf("UT: Assign from ENTRY_4 list - which is no longer empty\n"); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_4, &assignPtr) == NO_ERROR); + printf("UT: Assign from ENTRY_4 fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, + (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); + + // cleanup + m_freeMgr.writeDBFile(indexFile, blockZero.data, lbid); + m_freeMgr.closeFile(indexFile); + } + // list tests + void testFreeMgrInit2() + { + FILE* indexFile; + CommBlock cb; + DataBlock blockZero; + IdxEmptyListEntry emptyMap, assignPtr; + int allocSize, rc; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("\nUT: Begin testFreeMgrInit2\n"); + + assignPtr.fbo = 0; + assignPtr.sbid = 0; + assignPtr.entry = 0; + assignPtr.type = 0; + + m_freeMgr.deleteFile(776); + rc = m_freeMgr.createFile(776, 80, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(776); + cb.file.oid = 776; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readSubBlockEntry(cb, &blockZero, lbid, 0, 1 + ENTRY_BLK, 8, &emptyMap); + + // check the entries are as expected + printf("UT: fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_BLK); + CPPUNIT_ASSERT(emptyMap.fbo != 0); + // CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block + + // assign a segment and check that entries in sb0 change + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_BLK, &assignPtr) == NO_ERROR); + + CPPUNIT_ASSERT(assignPtr.group == ENTRY_BLK); // right group + // CPPUNIT_ASSERT( assignPtr.entry == 0 ); // right position + // CPPUNIT_ASSERT( assignPtr.sbid == 31 ); // right position + // CPPUNIT_ASSERT( assignPtr.fbo == 1 ); // right block + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_BLK, 8, &emptyMap); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_BLK); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 0 ); + // CPPUNIT_ASSERT( emptyMap.entry == 30 ); // points to second to last entry in sub-block + + // illegal segment size + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_16, &assignPtr) == ERR_INVALID_PARAM); + + // illegal segment size + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_8, &assignPtr) == ERR_INVALID_PARAM); + + // illegal segment size + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_2, &assignPtr) == ERR_INVALID_PARAM); + + // illegal segment size + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_1, &assignPtr) == ERR_INVALID_PARAM); + + // legal segment size but list is empty + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &assignPtr) == NO_ERROR); + + CPPUNIT_ASSERT(assignPtr.group == ENTRY_4); // points to start of sub-block + CPPUNIT_ASSERT(assignPtr.entry == 28); + // CPPUNIT_ASSERT( assignPtr.sbid == 29 ); + // CPPUNIT_ASSERT( assignPtr.fbo == 1 ); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_4); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 30 ); + CPPUNIT_ASSERT(emptyMap.entry == 7); // eight segments in list + // CPPUNIT_ASSERT( emptyMap.entry == 31 ); // 32 segments if using blocks for buckets + + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_4, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_4); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 30 ); + CPPUNIT_ASSERT(emptyMap.entry == 8); // eight segments in list + // CPPUNIT_ASSERT( (int)emptyMap.entry == 32 ); // 32 segments if using blocks for buckets + + m_freeMgr.writeDBFile(cb, blockZero.data, lbid); + m_freeMgr.closeFile(indexFile); + } + + void testFreeMgrAssign() + { + /** + * Assign tests + */ + FILE* indexFile; + CommBlock cb; + DataBlock blockZero; + IdxEmptyListEntry assignPtr, assignPtr2; + IdxEmptyListEntry assignPtr3, assignPtr4; + IdxEmptyListEntry emptyMap; + int idx, result; + int allocSize, rc; + IdxEmptyListEntry entries[1500]; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("\nUT: Begin testFreeMgrAssign\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 8, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + /* m_freeMgr.readDBFile( indexFile, blockZero.data, 0 ); + m_index.printMemSubBlock( &blockZero , 0 );*/ + uint64_t lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(indexFile, blockZero.data, lbid); + // m_index.printMemSubBlock( &blockZero , 0 ); + printf("\nUT: Cont testFreeMgrAssign\n"); + + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, 0, TREE, ENTRY_32, &assignPtr ) == + // ERR_INVALID_PARAM ); CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, 0 + // ) == ERR_INVALID_PARAM ); CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, 1234, + // ENTRY_32, &assignPtr ) == ERR_INVALID_PARAM ); + + // Assign a segment + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + // Assign a second segment and make sure that we get different pointers + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr2) == NO_ERROR); + printf("UT: sbid: %i sbid2: %i\n", (unsigned int)assignPtr.sbid, (unsigned int)assignPtr2.sbid); + CPPUNIT_ASSERT(assignPtr.sbid != assignPtr2.sbid); // assigned from sb list so cannot have same sbid + CPPUNIT_ASSERT(assignPtr.type = EMPTY_PTR); + CPPUNIT_ASSERT(assignPtr2.type = EMPTY_PTR); + // release in 'wrong' order .. + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr2) == NO_ERROR); + // .. then check that the list issues them in the return order + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr3) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr4) == NO_ERROR); + // CPPUNIT_ASSERT( assignPtr2.sbid == assignPtr3.sbid); + // CPPUNIT_ASSERT( assignPtr.sbid == assignPtr4.sbid); + + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr3) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr4) == NO_ERROR); + + // assign lots of sub-blocks + printf("UT: Assign lots of sub-blocks\n"); + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %03i fbo: %llu sbid: %02i entry: %02i in 'Assign lots of sub-blocks, assigning'\n", + idx, entries[idx].fbo, (unsigned int)(entries[idx].sbid), (unsigned int)(entries[idx].entry)); + } + + printf("UT: Releasing\n"); + + for (idx--; idx > -1; idx--) + { + // printf("UT: Visit: %i\n", idx); + printf("UT: Visit: %03i fbo: %llu sbid: %02i entry: %02i in 'Assign lots of sub-blocks, releasing'\n", + idx, entries[idx].fbo, (unsigned int)(entries[idx].sbid), (unsigned int)(entries[idx].entry)); + + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &entries[idx]) == NO_ERROR); + } + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %03i fbo: %llu sbid: %02i entry: %02i in 'Assign lots of sub-blocks, assigning'\n", + idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + // assign lots of smaller segments + printf("UT: Assign lots of smaller segments\n"); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_4, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_2, &assignPtr) == NO_ERROR); + printf("UT: fbo: %llu sbid: %i entry: %i \n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + + printf("UT: Assign lots of segments after init\n"); + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(indexFile, blockZero.data, lbid); + + // use up 32 entries + // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend + // over two blocks + printf("UT: Assign more than 32 segments\n"); + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_4, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i in 'Assign more than 32 segments'\n", idx, + entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + printf("UT: Releasing\n"); + + for (idx--; idx > -1; idx--) + { + // printf("UT: Visit: %i\n", idx); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i in 'Assign more than 32 segments, releasing'\n", + idx, entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_4, &entries[idx]) == NO_ERROR); + } + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_4, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i in 'Assign more than 32 segments'\n", idx, + entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + // commmenting this next bit breaks things + // CPPUNIT_ASSERT( freeMgr.init( indexFile, TREE ) == NO_ERROR ); + // dbFileOp.readDBFile( indexFile, blockZero.data, 0 ); + + printf("UT: Use almost all free space\n"); + + for (idx = 0; idx < 215; idx++) + { + result = m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i, result = %i\n", idx, assignPtr.fbo, + (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry, result); + CPPUNIT_ASSERT(result == NO_ERROR); + } + + // assign one more than is available, expect error + CPPUNIT_ASSERT((result = m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr2)) != + ERR_FM_NO_SPACE); + // release last assigned segment + CPPUNIT_ASSERT((result = m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr)) == + NO_ERROR); + // assign segment, expect no error + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + // assign segment after using up the list + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr2) != ERR_FM_NO_SPACE); + // release the segment, expect to become first bucket + printf("UT: Putting sub-block back\n"); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + printf("UT: After putting one sb back fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, + (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 7 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 0 ); + // CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to second to last entry in sub-block + + printf("UT: Request space after using all space\n"); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) != ERR_FM_NO_SPACE); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) != ERR_FM_NO_SPACE); + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_4, &assignPtr ) == + // NO_ERROR + 1); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_2, &assignPtr) != ERR_FM_NO_SPACE); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_1, &assignPtr) != ERR_FM_NO_SPACE); + + m_freeMgr.writeDBFile(indexFile, blockZero.data, lbid); + m_freeMgr.closeFile(indexFile); + printf("UT: End testFreeMgrAssign\n"); + } + + void testFreeMgrAssignList() + { + /** + * Assign tests + */ + FILE* indexFile; + + DataBlock blockZero; + IdxEmptyListEntry assignPtr, assignPtr2; + IdxEmptyListEntry assignPtr3, assignPtr4; + IdxEmptyListEntry emptyMap; + int rc; + int allocSize; + CommBlock cb; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrAssignList\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 8, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, 0, LIST, ENTRY_32, &assignPtr) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, 0) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, 1234, ENTRY_32, &assignPtr) == ERR_INVALID_PARAM); + + // Assign a segment + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + // Assign a second segment and make sure that we get different pointers + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr2) == NO_ERROR); + printf("UT: sbid: %i sbid2: %i\n", (unsigned int)assignPtr.sbid, (unsigned int)assignPtr2.sbid); + CPPUNIT_ASSERT(assignPtr.sbid != assignPtr2.sbid); // assigned from sb list so cannot have same sbid + CPPUNIT_ASSERT(assignPtr.type = EMPTY_PTR); + CPPUNIT_ASSERT(assignPtr2.type = EMPTY_PTR); + // release in 'wrong' order .. + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr2) == NO_ERROR); + // .. then check that the list issues them in the return order + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr3) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr4) == NO_ERROR); + // CPPUNIT_ASSERT( assignPtr2.sbid == assignPtr3.sbid); + // CPPUNIT_ASSERT( assignPtr.sbid == assignPtr4.sbid); + + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr3) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr4) == NO_ERROR); + + // assign lots of sub-blocks + printf("UT: Assign lots of sub-blocks\n"); + + int idx; + IdxEmptyListEntry entries[40]; + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &entries[idx]) == NO_ERROR); + } + + printf("UT: Releasing\n"); + + for (idx--; idx > -1; idx--) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &entries[idx]) == NO_ERROR); + } + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &entries[idx]) == NO_ERROR); + } + + // assign lots of smaller segments + printf("UT: Assign lots of smaller segments\n"); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &assignPtr) == NO_ERROR); + + printf("UT: Assign lots of segments after init\n"); + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + m_freeMgr.readDBFile(indexFile, blockZero.data, lbid); + + // use up 32 entries + // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend + // over two blocks + printf("UT: Assign more than 32 segments\n"); + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &entries[idx]) == NO_ERROR); + } + + printf("UT: Releasing\n"); + + for (idx--; idx > -1; idx--) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_4, &entries[idx]) == NO_ERROR); + } + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &entries[idx]) == NO_ERROR); + } + + // printf("UT: sbid: %i entry: %i \n", (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry + // ); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + printf("UT: sbid: %i entry: %i \n", (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); + + // now use up almost all free space + for (idx = 0; idx < 215; idx++) + { + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i\n", idx, assignPtr.fbo, + (unsigned int)assignPtr.sbid, (unsigned int)assignPtr.entry); + } + + // assign one more than is available, expect error + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) != ERR_FM_NO_SPACE); + + // check empty map + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + + // check the entries are as expected + printf("UT: Empty - fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 7 ); + // CPPUNIT_ASSERT( emptyMap.entry == 0 ); // points to last entry in sub-block + + // release last assigned segment + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_16, &assignPtr) == + ERR_FM_RELEASE_ERR); + assignPtr.group = ENTRY_16; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_16, &assignPtr) == ERR_INVALID_PARAM); + assignPtr.group = ENTRY_32; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + + // check the entries are as expected + printf("UT: Empty - fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 7 ); + // CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to last entry in sub-block + + // assign segment, expect no error + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + // assign segment after using up the list + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr2) != ERR_FM_NO_SPACE); + // release the segment, expect to become first bucket + printf("UT: Putting sub-block back\n"); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_32, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + printf("UT: After putting one sb back fbo: %llu sbid: %i entry: %i\n", emptyMap.fbo, + (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 7 ); + // CPPUNIT_ASSERT( emptyMap.sbid == 0 ); + // CPPUNIT_ASSERT( emptyMap.entry == 1 ); // points to second to last entry in sub-block + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_16, &assignPtr) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_8, &assignPtr) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_2, &assignPtr) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_1, &assignPtr) == ERR_INVALID_PARAM); + + m_freeMgr.writeDBFile(cb, blockZero.data, lbid); + m_freeMgr.closeFile(indexFile); + printf("UT: End testFreeMgrAssignList\n"); + } + + void testFreeMgrAssignListCk() + { + /** + * Assign tests + */ + FILE* indexFile; + CommBlock cb; + int rc, allocSize; + DataBlock blockZero; + int idx; + IdxEmptyListEntry entries[100]; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrAssignListCk\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 8, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + printf("UT: Assign lots of segments\n"); + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + // use up 32 entries + // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend + // over two blocks + printf("UT: Assign more than 32 segments\n"); + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + printf("UT: Releasing segments\n"); + + for (idx--; idx > -1; idx--) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, ENTRY_4, &entries[idx]) == NO_ERROR); + } + + printf("UT: Assign again\n"); + + for (idx = 0; idx < 39; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, ENTRY_4, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo: %llu sbid: %i entry: %i\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + printf("UT: End testFreeMgrAssignListCk\n"); + } + + void testFreeMgrAssignListLots() + { + /** + * Assign tests + */ + FILE* indexFile; + DbFileOp dbFileOp; + DataBlock blockZero; + DataBlock tempBlock; + int idx, rc, allocSize; + IdxEmptyListEntry entries[4000]; + IdxEmptyListEntry map; + CommBlock cb; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrAssignListLots\n"); + + int freemgr_type = LIST; + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 8, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, freemgr_type) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + printf("UT: Assign lots of segments\n"); + + // use up 32 entries + // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend + // over two blocks + printf("UT: Assign more than 32 segments\n"); + + int iters = 172; + IdxTreeGroupType entryType = ENTRY_32; + + for (idx = 0; idx < iters; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, freemgr_type, entryType, &entries[idx]) == + NO_ERROR); + printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (assigning)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + rc = m_freeMgr.readDBFile(cb, tempBlock.data, entries[idx].fbo); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &map); + printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (assigning)\n", idx, map.fbo, + (unsigned int)map.sbid, (unsigned int)map.entry); + } + + printf("UT: Releasing segments\n"); + + for (idx = 0; idx < iters; idx++) + { + // for (idx=iters-1; idx>-1; idx--){ + printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, freemgr_type, entryType, &entries[idx]) == + NO_ERROR); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &map); + printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (releasing)\n", idx, map.fbo, + (unsigned int)map.sbid, (unsigned int)map.entry); + } + + printf("UT: Assign again\n"); + + for (idx = 0; idx < iters; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, freemgr_type, entryType, &entries[idx]) == + NO_ERROR); + printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (assigning again)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &map); + printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (assigning again)\n", idx, map.fbo, + (unsigned int)map.sbid, (unsigned int)map.entry); + } + + // printf("UT: Releasing segments again\n"); + // + // for (idx=0; idx-1; idx--){ + // printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing again)\n", idx, + // entries[idx].fbo, (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, freemgr_type, entryType, + // &entries[idx] ) == NO_ERROR ); getSubBlockEntry( &blockZero, 0, 1+ENTRY_32, 8, &map); + // printf("UT: Map is now: %i fbo %llu sbid %02i entry %02i (releasing again )\n", idx, + // map.fbo, (unsigned int)map.sbid, (unsigned int)map.entry); + // + // } + + printf("UT: End testFreeMgrAssignListLots\n"); + } + + void testFreeMgrAssignListBlocks() + { + /** + * Assign tests + */ + FILE* indexFile; + DataBlock blockZero; + int rc, allocSize; + CommBlock cb; + int idx; + IdxEmptyListEntry entries[8000]; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrAssignListBlocks\n"); + + m_freeMgr.deleteFile(776); + rc = m_freeMgr.createFile(776, 80, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(776); + cb.file.oid = 776; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + printf("UT: Assign blocks from list\n"); + + // use up 32 entries + // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend + // over two blocks + + int iters = 77; + IdxTreeGroupType entryType = ENTRY_BLK; + + for (idx = 0; idx < iters; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, entryType, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (assigning)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + printf("UT: Releasing segments\n"); + + for (idx = 0; idx < iters; idx++) + { + // for (idx=iters-1; idx>-1; idx--){ + printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, entryType, &entries[idx]) == NO_ERROR); + entries[idx].fbo = 0; + } + + printf("UT: Assign again\n"); + + for (idx = 0; idx < iters; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, LIST, entryType, &entries[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo: %llu sbid: %02i entry: %02i (assigning again)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + } + + for (idx = 0; idx < iters; idx++) + { + // for (idx=iters-1; idx>-1; idx--){ + printf("UT: Visit: %i fbo %llu sbid %02i entry %02i (releasing again)\n", idx, entries[idx].fbo, + (unsigned int)entries[idx].sbid, (unsigned int)entries[idx].entry); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, LIST, entryType, &entries[idx]) == NO_ERROR); + entries[idx].fbo = 0; + } + + printf("UT: End testFreeMgrAssignListBlocks\n"); + } + + void testFreeMgrAssignListCk2() + { + /** + * Assign tests + */ + FILE* indexFile; + DataBlock blockZero; + int rc, allocSize; + CommBlock cb; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrAssignListCk2\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 8, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + int idx; + IdxEmptyListEntry e1[3400]; + IdxEmptyListEntry e32[3400]; + + printf("UT: Assign lots of segments\n"); + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + // use up 32 entries + // assign more than 32 entries, keeping track of them all, then release them causing the chain to extend + // over two blocks + printf("UT: Assign more than 32 segments\n"); + + for (idx = 0; idx < 340; idx++) + { + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_1, &e1[idx]) == NO_ERROR); + printf("UT: Visit: %i fbo %llu sbid %i entry %i\n", idx, e1[idx].fbo, (unsigned int)e1[idx].sbid, + (unsigned int)e1[idx].entry); + + if (idx % 10 == 0) + { + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &e32[idx]) == NO_ERROR); + } + + e1[idx].type = 7; + } + + printf("UT: Releasing segments\n"); + + for (idx = 0; idx < 340; idx++) + { + // printf("UT: Visit: %i\n", idx); + rc = m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_1, &e1[idx]); + CPPUNIT_ASSERT(rc == NO_ERROR); + + if (idx % 10 == 0) + { + rc = m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &e1[idx]); + } + } + + printf("UT: Assign again\n"); + + for (idx = 0; idx < 340; idx++) + { + rc = m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_1, &e1[idx]); + CPPUNIT_ASSERT(rc == NO_ERROR); + printf("UT: Visit: %i fbo %llu sbid %i entry %i\n", idx, e1[idx].fbo, (unsigned int)e1[idx].sbid, + (unsigned int)e1[idx].entry); + } + + printf("UT: Releasing segments\n"); + + for (idx = 0; idx < 340; idx++) + { + // printf("UT: Visit: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_1, &e1[idx]) == NO_ERROR); + } + + // + printf("UT: End testFreeMgrAssignListCk2\n"); + } + + void testFreeMgrAssignExtend() + { + /** + * Test the Extend method + **/ + FILE* indexFile; + int allocSize; + int rc; + DataBlock blockZero; + CommBlock cb; + uint64_t lbid; + DbFileOp db; + + printf("UT: Begin testFreeMgrAssignExtend\n"); + m_indexlist.deleteFile(777); + m_freeMgr.setDebugLevel(DEBUG_1); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 800, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, LIST) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + CPPUNIT_ASSERT(m_freeMgr.extendFreespace(cb, &blockZero, LIST) == NO_ERROR); + + int currSize = db.getFileSize(indexFile) / BYTE_PER_BLOCK; + + lbid = m_freeMgr.mapLBID(cb, currSize - 1, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + } + + void testFreeMgrExtendLots() + { + /** + * Test the Extend method + **/ + FILE* indexFile; + int allocSize; + int idx, rc; + DataBlock blockZero; + CommBlock cb; + uint64_t lbid; + DbFileOp db; + + printf("UT: Begin testFreeMgrExtendLots\n"); + m_indexlist.deleteFile(777); + m_freeMgr.setDebugLevel(DEBUG_1); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 800, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + // CPPUNIT_ASSERT( m_freeMgr.init( cb, LIST ) == NO_ERROR ); + // lbid = m_freeMgr.mapLBID( cb, 0, rc); + // CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + for (idx = 0; idx < 5; idx++) + { + printf("Loop count: %i\n", idx); + CPPUNIT_ASSERT(m_freeMgr.extendFreespace(cb, &blockZero, LIST) == NO_ERROR); + } + + // int currSize = db.getFileSize( indexFile )/BYTE_PER_BLOCK ; + + // lbid = m_freeMgr.mapLBID( cb, currSize-1, rc); + // CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); + printf("UT: End testFreeMgrExtendLots\n"); + } + + void testFreeMgrRelease() + { + FILE* indexFile; + DataBlock blockZero; + IdxEmptyListEntry assignPtr, emptyMap; + CommBlock cb; + int rc, allocSize; + uint64_t lbid; + + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrRelease\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 80, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, 0, TREE, ENTRY_32, &assignPtr) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, 0) == ERR_INVALID_PARAM); + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + + // check the entries are as expected + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_32); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + // CPPUNIT_ASSERT( emptyMap.entry == 31 ); // points to last entry in sub-block + + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); + + m_freeMgr.writeDBFile(cb, blockZero.data, 0); + m_freeMgr.closeFile(indexFile); + + printf("UT: End testFreeMgrRelease\n"); + } + void testFreeMgrFragment() + { + FILE* indexFile; + CommBlock cb; + int rc, idx, allocSize; + DataBlock blockZero; + IdxEmptyListEntry assignPtr, emptyMap; + + // test if possible to split a list of segments and populate smaller lists + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrFragment\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 80, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + + m_freeMgr.readSubBlockEntry(cb, &blockZero, 0, 0, 1 + ENTRY_1, 8, &emptyMap); + + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + + printf("UT: fbo %llu sbid %i entry %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_16; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); + printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_8; + assignPtr.entry = 16; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_8, 8, &emptyMap); + printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_4; + assignPtr.entry = 24; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_4, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); + printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_2; + assignPtr.entry = 28; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_2, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_2, 8, &emptyMap); + printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_1; + assignPtr.entry = 30; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_1, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); + printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_1; + assignPtr.entry = 31; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_1, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); + + // check the entries are as expected + printf("UT: fbo %llu sbid %i entry %i\n", emptyMap.fbo, (unsigned int)emptyMap.sbid, + (unsigned int)emptyMap.entry); + + CPPUNIT_ASSERT(emptyMap.type == EMPTY_LIST); + CPPUNIT_ASSERT(emptyMap.group == ENTRY_1); + // CPPUNIT_ASSERT( emptyMap.fbo == 1 ); + CPPUNIT_ASSERT(emptyMap.entry == 2); // points to second entry in sub-block + + /** + * Now do it many times + **/ + + for (idx = 0; idx < 185; idx++) + { + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + // printf("\nVisit: %i\n", idx); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + // printf("UT: Chain: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_16; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); + // printf("UT: Chain: %i fbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_8; + assignPtr.entry = 16; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_8, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_8, 8, &emptyMap); + // printf("UT: Chain: %i ffbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_4; + assignPtr.entry = 24; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_4, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_4, 8, &emptyMap); + // printf("UT: Chain: %i fbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_2; + assignPtr.entry = 28; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_2, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_2, 8, &emptyMap); + // printf("UT: Chain: %i fbo %llu sbid %i entry %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_1; + assignPtr.entry = 30; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_1, &assignPtr) == NO_ERROR); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_1, 8, &emptyMap); + // printf("UT: Chain: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + assignPtr.type = EMPTY_PTR; + assignPtr.group = ENTRY_1; + assignPtr.entry = 31; + CPPUNIT_ASSERT(m_freeMgr.releaseSegment(cb, &blockZero, TREE, ENTRY_1, &assignPtr) == NO_ERROR); + } + + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == NO_ERROR ); + + m_freeMgr.writeDBFile(indexFile, blockZero.data, 0); + m_freeMgr.closeFile(indexFile); + + printf("UT: End testFreeMgrFragment\n"); + } + + void testFreeMgrChain() + { + FILE* indexFile; + DataBlock blockZero; + IdxEmptyListEntry assignPtr, emptyMap; + int rc, allocSize; + CommBlock cb; + uint64_t lbid; + + // test if possible to split a list of segments and populate smaller lists + m_freeMgr.setDebugLevel(DEBUG_0); + printf("UT: Begin testFreeMgrChain\n"); + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 80, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + /** + * Now do it many times + **/ + + int idx; + + for (idx = 0; idx < 208; idx++) + { + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_32, &assignPtr) == NO_ERROR); + // printf("\nVisit: %i\n", idx); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_32, 8, &emptyMap); + // printf("UT: ENTRY_32: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); + // printf("UT: ENTRY_16: %i fbo: %llu sbid: %i entry: %i\n", (unsigned int)assignPtr.group, + // emptyMap.fbo, (unsigned int)emptyMap.sbid, (unsigned int)emptyMap.entry); + } + + m_freeMgr.deleteFile(777); + rc = m_freeMgr.createFile(777, 10, allocSize); + CPPUNIT_ASSERT(rc == NO_ERROR); + cb.file.pFile = m_freeMgr.openFile(777); + cb.file.oid = 777; + indexFile = cb.file.pFile; + + CPPUNIT_ASSERT(m_freeMgr.init(cb, TREE) == NO_ERROR); + lbid = m_freeMgr.mapLBID(cb, 0, rc); + CPPUNIT_ASSERT_MESSAGE("Error using mapLBID", rc == NO_ERROR); + + m_freeMgr.readDBFile(cb, blockZero.data, lbid); + + printf("UT: Assigning from ENTRY_16 chain\n"); + + for (idx = 0; idx < 420; idx++) + { + CPPUNIT_ASSERT(m_freeMgr.assignSegment(cb, &blockZero, TREE, ENTRY_16, &assignPtr) == NO_ERROR); + // printf("Visit: %i\n", idx); + m_freeMgr.getSubBlockEntry(&blockZero, 0, 1 + ENTRY_16, 8, &emptyMap); + printf("UT: ENTRY_16: fbo: %llu sbid: %i entry: %i\n", assignPtr.fbo, (unsigned int)assignPtr.sbid, + (unsigned int)assignPtr.entry); + } + + m_freeMgr.writeDBFile(cb, blockZero.data, 0); + m_freeMgr.closeFile(indexFile); + + printf("UT: End testFreeMgrChain\n"); + } + + // void testFreeMgrEvil() { + // + // FILE* indexFile; + // int rc, allocSize; + // DataBlock blockZero, workBlock; + // IdxEmptyListEntry assignPtr, emptyPtr, emptyEntry; + // CommBlock cb; + // uint64_t lbid; + // + // /** + // * Forget Google.. be evil + // **/ + // + // test if possible to split a list of segments and populate smaller lists + // m_freeMgr.setDebugLevel( DEBUG_0 ); + // printf("UT: Begin testFreeMgrEvil\n"); + // + // + // m_freeMgr.deleteFile( 777 ); + // rc = m_freeMgr.createFile( 777, 80, allocSize ); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + // cb.file.pFile = m_freeMgr.openFile( 777 ); + // cb.file.oid = 777; + // indexFile = cb.file.pFile; + // + // CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); + // lbid = m_freeMgr.mapLBID( cb, 0, rc); + // CPPUNIT_ASSERT_MESSAGE( "Error using mapLBID", rc == NO_ERROR ); + // + // m_freeMgr.readDBFile( cb, blockZero.data, lbid ); + // + // m_freeMgr.getSubBlockEntry( &blockZero, 0, 1+ENTRY_32, 8, &emptyPtr); + // + // check the entries are as expected + // CPPUNIT_ASSERT( emptyPtr.type == EMPTY_LIST ); + // CPPUNIT_ASSERT( emptyPtr.group == ENTRY_32 ); + // CPPUNIT_ASSERT( emptyPtr.fbo == 1 ); + // CPPUNIT_ASSERT( emptyPtr.entry == 31 ); // points to last entry in sub-block + // + // printf("UT: Mess up fbo in emptyMap for ENTRY_32\n"); + // emptyPtr.fbo =999999999; + // m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_32, 8, &emptyPtr); + // + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == + // ERR_FM_BAD_FBO ); emptyPtr.fbo =7; m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_32, + // 8, &emptyPtr); + // + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == + // NO_ERROR ); + // + // CPPUNIT_ASSERT( assignPtr.fbo == 7 ); + // assignPtr.fbo =1234; + // first release with bad fbo then with good + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == + // ERR_FM_BAD_FBO ); + // + // assignPtr.fbo =7; + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_32, &assignPtr ) == + // NO_ERROR ); + // + // + // printf("UT: Mess up fbo in emptyMap for ENTRY_16\n"); + // m_freeMgr.getSubBlockEntry( &blockZero, 0, 1+ENTRY_16, 8, &emptyPtr); + // CPPUNIT_ASSERT( emptyPtr.fbo == 0 ); // zero because nothing in list yet + // + // emptyPtr.fbo =999999999; + // m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_16, 8, &emptyPtr); + // + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == + // ERR_FM_BAD_FBO ); emptyPtr.fbo =0; m_freeMgr.writeSubBlockEntry( cb, &blockZero, 0, 0, 1+ENTRY_16, + // 8, &emptyPtr); + // + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR + // ); + // + // CPPUNIT_ASSERT( assignPtr.fbo == 7 ); + // assignPtr.fbo =1234; + // first release with bad fbo then with good + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == + // ERR_FM_BAD_FBO ); + // + // assignPtr.fbo =7; + // CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == + // NO_ERROR ); + // + // Mess with type field + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr ) == NO_ERROR + // ); + // + // m_freeMgr.getSubBlockEntry( blockZero.data, 0, 1+ENTRY_16, 8, &emptyPtr ); + // m_freeMgr.readDBFile( cb, &workBlock, emptyPtr.fbo ); + // m_freeMgr.getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); + // + // CPPUNIT_ASSERT( emptyEntry.type == 2 ); //check is EMPTY_PTR + // CPPUNIT_ASSERT( emptyEntry.fbo == 7 ); //check is last sb + // emptyEntry.type = 7; // set to garbage + // m_freeMgr.writeSubBlockEntry( cb, &workBlock, emptyPtr.fbo, emptyPtr.sbid, emptyPtr.entry, 8, + // &emptyEntry); check assign fails CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, + // ENTRY_16, &assignPtr ) == ERR_FM_BAD_TYPE ); + // + // emptyEntry.type = 2; + // emptyEntry.fbo = 0; + // m_freeMgr.writeSubBlockEntry( cb, &workBlock, emptyPtr.fbo, emptyPtr.sbid, emptyPtr.entry, 8, + // &emptyEntry); check assign fails CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, + // ENTRY_16, &assignPtr ) == ERR_FM_BAD_TYPE ); + // + // + // fix type field and re-try assign + // emptyEntry.type = 2; + // emptyEntry.fbo = 7; + // m_freeMgr.writeSubBlockEntry( cb, &workBlock, emptyPtr.fbo, emptyPtr.sbid, emptyPtr.entry, 8, + // &emptyEntry); CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_16, &assignPtr + // ) == NO_ERROR ); CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_16, + // &assignPtr ) == NO_ERROR ); + // + // + // printf("UT: Zero fbo in emptyMap for ENTRY_8\n"); + // //start 8 list + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR + // ); CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == + // NO_ERROR ); + // + // m_freeMgr.getSubBlockEntry( &blockZero, 0, 1+ENTRY_8, 8, &emptyPtr); + // CPPUNIT_ASSERT( emptyPtr.fbo == 7 ); + // CPPUNIT_ASSERT( emptyPtr.type == 2 ); + // + // emptyPtr.fbo = 0; + // m_freeMgr.writeSubBlockEntry( indexFile, &blockZero, 0, 0, 1+ENTRY_8, 8, &emptyPtr); + // + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR + // + 1 ); + // //cleanup + // emptyPtr.fbo = 7; + // m_freeMgr.writeSubBlockEntry( indexFile, &blockZero, 0, 0, 1+ENTRY_8, 8, &emptyPtr); + // + // check that assign and release works ok + // CPPUNIT_ASSERT( m_freeMgr.assignSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == NO_ERROR + // ); CPPUNIT_ASSERT( m_freeMgr.releaseSegment( cb, &blockZero, TREE, ENTRY_8, &assignPtr ) == + // NO_ERROR ); + // + // m_freeMgr.writeDBFile( indexFile, blockZero.data, 0 ); + // CPPUNIT_ASSERT( m_freeMgr.init( cb, TREE ) == NO_ERROR ); + // m_freeMgr.closeFile( indexFile ); + // + // printf("UT: End testFreeMgrEvil\n"); + // + // } + void testIndexListMultiKey() + { + DataBlock curBlock; + FILE* pFile = NULL; + int rc, allocSize; + uint64_t key = 0x123; + int rowId; + int file_number = 20000; + int fbo, sbid, entry; + IdxEmptyListEntry newIdxListHdrPtr; + IdxRidListHdr newIdxRidListHdr; + + printf("\nRunning multiple keys for adding header \n"); + BRMWrapper::setUseBrm(true); + // Then increment the rowId as it goes + + rowId = 0x567; + + /******************************************************* + * TEST with FreeMgr for coverage + * Start here for add header and delete header entries + * + *******************************************************/ + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + CommBlock cb; + File file; + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + m_indexlist.init(cb, LIST); + + int loop; + + for (loop = 0; loop < 10; loop++) + { + rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); + + if (rc != NO_ERROR) + printf("196. error code:%i \n", rc); + + // printf("The fbo is:%i sbid is %i entry is %i \n",(int)newIdxListHdrPtr.fbo, + // (int)newIdxListHdrPtr.sbid, (int)newIdxListHdrPtr.entry); + CPPUNIT_ASSERT(rc == NO_ERROR); + fbo = newIdxListHdrPtr.fbo; // fbo=? from the free manager + sbid = newIdxListHdrPtr.sbid; // sbid =? from the free manager + entry = newIdxListHdrPtr.entry; // entry=? from the free manager + memset(curBlock.data, 0, sizeof(curBlock.data)); + + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, LIST_HDR_SIZE, &newIdxRidListHdr); + // cout<< "rowId->" << rowId << " key->" << key << " size->" << + // newIdxRidListHdr.idxRidListSize.size << endl; + rowId++; + key++; + } + + // cout << " \n Successfully ran testIndexListMultiKey \n" << endl; + return; + } // testIndexListMultiKey + + void testIndexListUpdate() + { + DataBlock curBlock; + FILE* pFile = NULL; + int rc, fbo, sbid, entry; + uint64_t key = 123; + int count = 40000; + int rowId; + int i; + int file_number = 8001; + + IdxEmptyListEntry newIdxListHdrPtr; + IdxRidListHdr newIdxRidListHdr; + // bool found = false; + // int p_sbid, p_entry, + // uint64_t p_fbo; + int allocSize; + + BRMWrapper::setUseBrm(true); + printf("\nRunning testIndexListUpdate\n"); + + memset(curBlock.data, 0, sizeof(curBlock.data)); + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + // Free Manager + + CommBlock cb; + File file; + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + m_indexlist.init(cb, LIST); + + // Initialize first rowId to 0 + + rowId = 0; + rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); + + if (rc != NO_ERROR) + // printf("245. error code:%i \n", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + + fbo = newIdxListHdrPtr.fbo; // fbo=? from the free manager + sbid = newIdxListHdrPtr.sbid; // sbid =? from the free manager + entry = newIdxListHdrPtr.entry; // entry=? from the free manager + memset(curBlock.data, 0, sizeof(curBlock.data)); + + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, LIST_HDR_SIZE, &newIdxRidListHdr); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == 1); + rowId++; + // cout << " Single Insert->start Timer :inserting " << count << " rowId " << endl; + m_indexlist.startfTimer(); + + for (i = 0; i < count; i++) + { + ////cout << " count number-> " << i << endl; + rc = m_indexlist.updateIndexList(cb, rowId, key, &newIdxListHdrPtr); + + if (rc != NO_ERROR) + { + // cout << "error code->" << rc << " i->" << i << endl; + return; + } + + rowId++; + } + + m_indexlist.stopfTimer(); + // cout << " Single Insert ->End Timer :inserting " << count << " rowId " << endl; + // cout << " Total used time (msec) " << m_indexlist.getTotalfRunTime() << endl; + m_indexlist.closeFile(pFile); + // + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, LIST_HDR_SIZE, &newIdxRidListHdr); + // cout<< " key->" << key << " size->" << + // newIdxRidListHdr.idxRidListSize.size << endl; + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == (uint64_t)(count + 1)); + RID ridArray[count + 1]; + int size; + m_indexlist.getRIDArrayFromListHdr(pFile, key, &newIdxListHdrPtr, ridArray, size); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == (uint64_t)size); + // cout << "size=" << size << endl; + // cout << " newIdxRidListHdr.idxRidListSize.size=" << newIdxRidListHdr.idxRidListSize.size << endl; + // for (int i=0; i< size ; i++) + ////cout<< " ridArray[i]->" << ridArray[i] << " i->" << i << endl; + + uint64_t firstLbid; + rc = m_indexlist.findFirstBlk(pFile, key, &newIdxListHdrPtr, firstLbid); + // cout << " Single RID insert->FirstLbid is ->" << firstLbid << endl; + // cout << " This will print level and counts for Children " << endl; + // if (m_indexlist.getUseNarray()) + // m_indexlist.printBlocks(firstLbid); + CPPUNIT_ASSERT(rc == NO_ERROR); + m_indexlist.closeFile(pFile); + printf("\nSuccessfully Running testIndexListUpdate\n"); + } // testIndexListUpdate + + void testIndexListDelete() + { + DataBlock curBlock; + FILE* pFile = NULL; + int rc, fbo, sbid, entry, width = 32, allocSize; + uint64_t key = 123; + int count = 2890; + int rowId; + int i; + int k = count + 1; + int delete_count; + int file_number = 8002; + IdxEmptyListEntry newIdxListHdrPtr; + IdxRidListHdr newIdxRidListHdr; + bool found = false; + int p_sbid, p_entry; + uint64_t p_fbo; + + // Initialize first rowId to 0 + // Then increment the rowId as it goes + printf("\nRunning testIndexListDelete\n"); + rowId = 0; + delete_count = 0; + BRMWrapper::setUseBrm(true); + /******************************************************* + * TEST with FreeMgr for coverage + * Start here for add header and delete header entries + * + *******************************************************/ + memset(curBlock.data, 0, sizeof(curBlock.data)); + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + CommBlock cb; + File file; + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + m_indexlist.init(cb, LIST); + + rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); + + if (rc != NO_ERROR) + // printf("388. error code:%i \n", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rowId++; + rc = m_indexlist.deleteIndexList(pFile, 0, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + delete_count++; + // delete the same item again, should not be there, return an error + rc = m_indexlist.deleteIndexList(pFile, 0, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc != NO_ERROR); + // delete an rowId which does not exist in the index list + // return a error + rc = m_indexlist.deleteIndexList(pFile, 1, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc != NO_ERROR); + // + rc = m_indexlist.updateIndexList(pFile, 5, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_indexlist.updateIndexList(pFile, 6, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_indexlist.deleteIndexList(pFile, 5, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_indexlist.deleteIndexList(pFile, 6, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc == NO_ERROR); + fbo = newIdxListHdrPtr.fbo; // fbo=? from free manager + sbid = newIdxListHdrPtr.sbid; // sbid =? + entry = newIdxListHdrPtr.entry; // entry=? + // printf("Index List Header: fbo: %i sbid: %i entry: %i\n", (int)fbo, (int)sbid, (int)entry); + m_indexlist.closeFile(pFile); + // Reading the Header out + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, 32, &newIdxRidListHdr); + + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == 0); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.type == 0); + CPPUNIT_ASSERT(newIdxRidListHdr.key == key); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.type == 7); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.rid == (RID)0); + CPPUNIT_ASSERT(newIdxRidListHdr.nextIdxRidListPtr.type == 7); + CPPUNIT_ASSERT(newIdxRidListHdr.nextIdxRidListPtr.llp == 0x0); + m_indexlist.closeFile(pFile); + + // Add Header again and also Update the index list + rowId = 0; + delete_count = 0; + // + memset(curBlock.data, 0, sizeof(curBlock.data)); + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + m_indexlist.init(cb, LIST); + rc = m_indexlist.addIndexListHdr(pFile, rowId, key, &newIdxListHdrPtr); + found = m_indexlist.findRowId(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); + // printf("Find RowId: %i in : fbo: %i sbid: %i entry: %i\n", (int) rowId, (int)p_fbo, (int)p_sbid, + // (int)p_entry); cout << " RowId->" << rowId <<" Fbo->" << p_fbo << " Sbid->" << p_sbid << " Entry->" << + // p_entry << endl; + rowId++; + + CPPUNIT_ASSERT(rc == NO_ERROR); + fbo = newIdxListHdrPtr.fbo; // fbo=? from the free manager + sbid = newIdxListHdrPtr.sbid; // sbid =? from the free manager + entry = newIdxListHdrPtr.entry; // entry=? from the free manager + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, width, &newIdxRidListHdr); + + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == 1); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.type == 0); + CPPUNIT_ASSERT(newIdxRidListHdr.key == key); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.rid == (RID)rowId - 1); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.type == 3); + CPPUNIT_ASSERT(newIdxRidListHdr.nextIdxRidListPtr.llp == 0x0); + CPPUNIT_ASSERT(newIdxRidListHdr.nextIdxRidListPtr.type == 7); + + for (i = 0; i < count; i++) + { + // printf("Index List Count: i: %i Row Id: %i \n", i, rowId); + rc = m_indexlist.updateIndexList(pFile, rowId, key, &newIdxListHdrPtr); + + rowId++; + // if (rc != NO_ERROR) + // printf("ERROR ->2940 i %i \n rowID %i \n", i, rowId); + // CPPUNIT_ASSERT( rc == NO_ERROR ); + } + + for (i = 0; i < count; i++) + { + found = m_indexlist.findRowId(pFile, i, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); + // printf("Find RowId: %i in : fbo: %i sbid: %i entry: %i\n", (int) rowId, (int)p_fbo, (int)p_sbid, + // (int)p_entry); if (found) cout << " Found RowId->" << i <<" Fbo->" << p_fbo << " Sbid->" << p_sbid << + // " Entry->" << p_entry << endl; else cout << " Not Found RowId->" << i << endl; Check Header again, we + // need to read the block again since it is changed + } + + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, width, &newIdxRidListHdr); + + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == (uint64_t)k); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.type == 0); + CPPUNIT_ASSERT(newIdxRidListHdr.key == key); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.rid == (RID)0); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.type == 3); + CPPUNIT_ASSERT(newIdxRidListHdr.nextIdxRidListPtr.type == 4); + // printf("Index List Header: fbo: %i sbid: %i entry: %i\n", (int)fbo, (int)sbid, (int)entry); + m_indexlist.closeFile(pFile); + + // Delete test + // Read now + // Check Header + + delete_count = 0; + fbo = newIdxListHdrPtr.fbo; // fbo=? + sbid = newIdxListHdrPtr.sbid; // sbid =? + entry = newIdxListHdrPtr.entry; // entry=? + + memset(&newIdxRidListHdr, 0, 32); + memset(curBlock.data, 0, sizeof(curBlock.data)); + + // Delete action + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + + rc = m_indexlist.deleteIndexList(pFile, 0, key, &newIdxListHdrPtr); // rowId 0 deleted + + if (rc == NO_ERROR) + delete_count++; + else + { + // cout << "cannot find rowId->" << 0 << endl; + } + + rc = m_indexlist.deleteIndexList(pFile, 100, key, &newIdxListHdrPtr); // rowId 100 deleted + + if (rc == NO_ERROR) + delete_count++; + else + { + // cout << "cannot find rowId->" << 100 << endl; + } + + rc = m_indexlist.deleteIndexList(pFile, 975, key, &newIdxListHdrPtr); // rowId 975 deleted + + if (rc == NO_ERROR) + delete_count++; + else + { + // cout << "cannot find rowId->" << 975 << endl; + } + + rc = m_indexlist.deleteIndexList(pFile, 2000, key, &newIdxListHdrPtr); // rowId 2000 deleted + + if (rc == NO_ERROR) + delete_count++; + else + { + // cout << "cannot find rowId->" << 2000 << endl; + } + + m_indexlist.closeFile(pFile); + + // Check Answers Read File at block fbo=0; + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + width = 32; + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, width, &newIdxRidListHdr); + k = k - delete_count; + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == (uint64_t)k); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.type == 0); + CPPUNIT_ASSERT(newIdxRidListHdr.key == key); + + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.type == 7); + CPPUNIT_ASSERT(newIdxRidListHdr.firstIdxRidListEntry.rid == 0); + m_indexlist.closeFile(pFile); + printf("\nSuccessfully Running testIndexListDelete\n"); + } // testIndexListDelete + + void testIndexListReleaseMgrBack() + { + FILE* pFile = NULL; + int rc, allocSize; + int file_number = 8006; + DataBlock BlockZero; + int i; + IdxEmptyListEntry newIdxListHdrPtr; + IdxRidListHdr idxRidListHdr; + int rowId = 0; + int newCnt = 2000; + CommBlock cb; + File file; + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + BRMWrapper::setUseBrm(true); + m_indexlist.init(cb, LIST); + int key = 123; + printf("\nRunning testIndexListReleaseMgrBack\n"); + + memset(BlockZero.data, 0, sizeof(BlockZero.data)); + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + CPPUNIT_ASSERT(pFile != NULL); + m_indexlist.init(cb, LIST); + rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); + /* + if (rc != NO_ERROR) + { + //cout << "rc -> " << rc << endl; + //cout << "i->"<< i << endl; + //cout << "rowId->" << rowId << endl; + //cout << "key->" << key << endl; + } + */ + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + // if (rc!= NO_ERROR) + // printf("724. error code:%i \n", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + rowId++; + rc = m_indexlist.updateIndexList(pFile, rowId, key + 1, &newIdxListHdrPtr); + /* + if (rc != NO_ERROR) + { + //cout << "rc -> " << rc << endl; + //cout << "rowId->" << rowId << endl; + //cout << "key+1->" << key+1 << endl; + } + */ + CPPUNIT_ASSERT(rc != NO_ERROR); + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + /* + if (rc != NO_ERROR) + { + //cout <<" getHdrInfo has error-> rc " << rc << endl; + } + */ + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (i = 0; i < newCnt; i++) + { + rc = m_indexlist.updateIndexList(pFile, rowId, key, &newIdxListHdrPtr); + /* + if (rc != NO_ERROR) + { + //cout <<" updateIndexList has error->" << endl; + //cout << "rc -> " << rc << endl; + //cout << "i->"<< i << endl; + //cout << "rowId->" << rowId << endl; + //cout << "key->" << key << endl; + } + */ + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + /* + if (rc != NO_ERROR) + { + //cout <<" getHdrInfo has error->" << endl; + //cout << "rc -> " << rc << endl; + //cout << "i->"<< i << endl; + //cout << "rowId->" << rowId << endl; + //cout << "key->" << key << endl; + } + */ + CPPUNIT_ASSERT(rc == NO_ERROR); + rowId++; + } + + rowId--; + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + rc = m_indexlist.deleteIndexList(pFile, rowId, key + 1, &newIdxListHdrPtr); + CPPUNIT_ASSERT(rc != NO_ERROR); + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + + for (i = newCnt; i >= 0; i--) + { + rc = m_indexlist.deleteIndexList(pFile, rowId, key, &newIdxListHdrPtr); + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + CPPUNIT_ASSERT(rc == NO_ERROR); + rowId--; + } + + rowId++; + + for (i = 0; i < newCnt; i++) + { + rc = m_indexlist.updateIndexList(pFile, rowId, key, &newIdxListHdrPtr); + rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr); + /* + if (rc!=NO_ERROR) + { + //cout << " tindex 3099-> i " << i << " rc " << rc << endl; + } + */ + CPPUNIT_ASSERT(rc == NO_ERROR); + rowId++; + } + + m_indexlist.closeFile(pFile); + printf("\nSuccessfully Running testIndexListReleaseMgrBack\n"); + return; + } + void testIndexListMultipleAddHdr() + { + DataBlock curBlock; + DataBlock curBlock2; + DataBlock curBlock3; + DataBlock curBlock4; + FILE* pFile = NULL; + int rc, fbo, sbid, entry; + int fbo2, sbid2, entry2; + int fbo3, sbid3, entry3; + int fbo4, sbid4, entry4; + uint64_t key = 1; + uint64_t key2 = 2; + uint64_t key3 = 3; + uint64_t key4 = 4; + int count = 2000; + int i = 0; + int delete_count; + int file_number = 8010; + RID rowIdArray[count + 1]; + RID rowIdArray2[count + 1]; + RID rowIdArray3[count + 1]; + RID rowIdArray4[count + 1]; + int j, k, l; + RID rowId; + + IdxEmptyListEntry newIdxListHdrPtr; + IdxRidListHdr newIdxRidListHdr; + IdxEmptyListEntry newIdxListHdrPtr2; + IdxRidListHdr newIdxRidListHdr2; + IdxEmptyListEntry newIdxListHdrPtr3; + IdxRidListHdr newIdxRidListHdr3; + IdxEmptyListEntry newIdxListHdrPtr4; + IdxRidListHdr newIdxRidListHdr4; + + bool found = false; + int p_sbid, p_entry, allocSize; + uint64_t p_fbo; + BRMWrapper::setUseBrm(true); + m_indexlist.setUseSortFlag(false); + printf("\nRunning testIndexListMultipleAddHdr\n"); + delete_count = 0; + memset(curBlock.data, 0, sizeof(curBlock.data)); + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + CommBlock cb; + File file; + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + m_indexlist.init(cb, LIST); + + for (i = 0; i < count + 1; i++) + { + rowIdArray[i] = i; + } + + for (j = 0; j < count + 1; j++) + { + rowIdArray2[j] = i; + i++; + } + + for (k = 0; k < count + 1; k++) + { + rowIdArray3[k] = i; + i++; + } + + for (l = 0; l < count + 1; l++) + { + rowIdArray4[l] = i; + i++; + } + + i = count + 1; + // Initialize first rowId to 0 + // cout << " start Timer :inserting " << count << " rowId " << endl; + m_indexlist.startfTimer(); + int count_size = count + 1; + rc = m_indexlist.addIndexListHdr(cb, &rowIdArray[0], count_size, key, &newIdxListHdrPtr); + + rc = m_indexlist.addIndexListHdr(cb, &rowIdArray2[0], count_size, key2, &newIdxListHdrPtr2); + rc = m_indexlist.addIndexListHdr(cb, &rowIdArray3[0], count_size, key3, &newIdxListHdrPtr3); + rc = m_indexlist.addIndexListHdr(cb, &rowIdArray4[0], count_size, key4, &newIdxListHdrPtr4); + + m_indexlist.stopfTimer(); + // cout << " End Timer :inserting " << count << " rowId " << endl; + // cout << " Total used time (msec) " << m_indexlist.getTotalfRunTime() << endl; + // if (rc!= NO_ERROR) + // printf("245. error code:%i \n", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + + fbo = newIdxListHdrPtr.fbo; // fbo=? from the free manager + sbid = newIdxListHdrPtr.sbid; // sbid =? from the free manager + entry = newIdxListHdrPtr.entry; // entry=? from the free manager + + fbo2 = newIdxListHdrPtr2.fbo; // fbo=? from the free manager + sbid2 = newIdxListHdrPtr2.sbid; // sbid =? from the free manager + entry2 = newIdxListHdrPtr2.entry; // entry=? from the free manager + + fbo3 = newIdxListHdrPtr3.fbo; // fbo=? from the free manager + sbid3 = newIdxListHdrPtr3.sbid; // sbid =? from the free manager + entry3 = newIdxListHdrPtr3.entry; // entry=? from the free manager + + fbo4 = newIdxListHdrPtr4.fbo; // fbo=? from the free manager + sbid4 = newIdxListHdrPtr4.sbid; // sbid =? from the free manager + entry4 = newIdxListHdrPtr4.entry; // entry=? from the free manager + + memset(curBlock.data, 0, sizeof(curBlock.data)); + + memset(curBlock2.data, 0, sizeof(curBlock2.data)); + memset(curBlock3.data, 0, sizeof(curBlock.data)); + memset(curBlock4.data, 0, sizeof(curBlock2.data)); + + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, 32, &newIdxRidListHdr); + RID ridArray[count + 1]; + int ridSize; + rc = m_indexlist.getRIDArrayFromListHdr(pFile, key, &newIdxListHdrPtr, ridArray, ridSize); + // cout << "ridSize ->" << ridSize << endl; + CPPUNIT_ASSERT(ridSize == (int)newIdxRidListHdr.idxRidListSize.size); + + rc = m_indexlist.readDBFile(pFile, curBlock2.data, fbo2); + m_indexlist.getSubBlockEntry(curBlock2.data, sbid2, entry2, 32, &newIdxRidListHdr2); + RID ridArray2[count + 1]; + int ridSize2; + rc = m_indexlist.getRIDArrayFromListHdr(pFile, key2, &newIdxListHdrPtr2, ridArray2, ridSize2); + // cout << "ridSize2 ->" << ridSize2 << endl; + CPPUNIT_ASSERT(ridSize2 == (int)newIdxRidListHdr2.idxRidListSize.size); + + rc = m_indexlist.readDBFile(pFile, curBlock3.data, fbo3); + m_indexlist.getSubBlockEntry(curBlock3.data, sbid3, entry3, 32, &newIdxRidListHdr3); + RID ridArray3[count + 1]; + int ridSize3; + rc = m_indexlist.getRIDArrayFromListHdr(pFile, key3, &newIdxListHdrPtr3, ridArray3, ridSize3); + // cout << "ridSize3 ->" << ridSize3 << endl; + CPPUNIT_ASSERT(ridSize3 == (int)newIdxRidListHdr3.idxRidListSize.size); + + rc = m_indexlist.readDBFile(pFile, curBlock4.data, fbo4); + m_indexlist.getSubBlockEntry(curBlock4.data, sbid4, entry4, 32, &newIdxRidListHdr4); + RID ridArray4[count + 1]; + int ridSize4; + rc = m_indexlist.getRIDArrayFromListHdr(pFile, key4, &newIdxListHdrPtr4, ridArray4, ridSize4); + // cout << "ridSize4 ->" << ridSize4 << endl; + CPPUNIT_ASSERT(ridSize4 == (int)newIdxRidListHdr4.idxRidListSize.size); + // m_indexlist.stopTimer(); + + ////cout << " Total used time (sec) " << m_indexlist.getTotalRunTime() << endl; + + uint64_t firstLbid; + uint64_t firstLbid2; + uint64_t firstLbid3; + uint64_t firstLbid4; + rc = m_indexlist.findFirstBlk(pFile, key, &newIdxListHdrPtr, firstLbid); + + rc = m_indexlist.findFirstBlk(pFile, key2, &newIdxListHdrPtr2, firstLbid2); + rc = m_indexlist.findFirstBlk(pFile, key3, &newIdxListHdrPtr3, firstLbid3); + rc = m_indexlist.findFirstBlk(pFile, key4, &newIdxListHdrPtr4, firstLbid4); + + // cout << " Multiple RIDS Header insert->FirstLbid is ->" << firstLbid << endl; + + // cout << " Multiple RIDS Header insert->FirstLbid2 is ->" << firstLbid2 << endl; + // cout << " Multiple RIDS Header insert->FirstLbid3 is ->" << firstLbid3 << endl; + // cout << " Multiple RIDS Header insert->FirstLbid4 is ->" << firstLbid4 << endl; + + // cout << " This will print level and counts for Children " << endl; + /* + if (m_indexlist.getUseNarray()) + { + m_indexlist.printBlocks(firstLbid); + + m_indexlist.printBlocks(firstLbid2); + m_indexlist.printBlocks(firstLbid3); + m_indexlist.printBlocks(firstLbid4); + + } + */ + CPPUNIT_ASSERT(rc == NO_ERROR); + m_indexlist.closeFile(pFile); + // + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + /* + rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); + m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); + CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)(count+1)); + int arraySize=0; + arraySize = newIdxRidListHdr.idxRidListSize.size; + if (arraySize >0) + { + RID ridArray[arraySize]; + int ridSize; + rc= m_indexlist.getRIDArrayFromListHdr(pFile, + key, &newIdxListHdrPtr, - ridArray, size); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)size); - //cout << "size=" << size << endl; - //cout << " newIdxRidListHdr.idxRidListSize.size=" << newIdxRidListHdr.idxRidListSize.size << endl; - //for (int i=0; i< size ; i++) - ////cout<< " ridArray[i]->" << ridArray[i] << " i->" << i << endl; + ridArray, ridSize); + //cout << "ridSize->" << ridSize << endl; + CPPUNIT_ASSERT(ridSize==arraySize); - uint64_t firstLbid; - rc = m_indexlist.findFirstBlk(pFile, key, &newIdxListHdrPtr, firstLbid); - //cout << " Single RID insert->FirstLbid is ->" << firstLbid << endl; - //cout << " This will print level and counts for Children " << endl; - //if (m_indexlist.getUseNarray()) - // m_indexlist.printBlocks(firstLbid); - CPPUNIT_ASSERT(rc == NO_ERROR); - m_indexlist.closeFile(pFile); - printf("\nSuccessfully Running testIndexListUpdate\n"); - }//testIndexListUpdate + if (rc!= NO_ERROR) + { + printf("ERROR CODE :%i \n", rc); - void testIndexListDelete() - { - DataBlock curBlock; - FILE* pFile = NULL; - int rc, fbo, sbid, entry, width = 32, allocSize; - uint64_t key = 123; - int count = 2890; - int rowId; - int i; - int k = count + 1; - int delete_count; - int file_number = 8002; - IdxEmptyListEntry newIdxListHdrPtr; - IdxRidListHdr newIdxRidListHdr; - bool found = false; - int p_sbid, p_entry; - uint64_t p_fbo; - - //Initialize first rowId to 0 - //Then increment the rowId as it goes - printf("\nRunning testIndexListDelete\n"); - rowId = 0 ; - delete_count = 0; - BRMWrapper::setUseBrm(true); - /******************************************************* - * TEST with FreeMgr for coverage - * Start here for add header and delete header entries - * - *******************************************************/ - memset( curBlock.data, 0, sizeof(curBlock.data)); - m_indexlist.deleteFile( file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - CommBlock cb; - File file; - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - m_indexlist.init(cb, LIST); - - rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); - - if (rc != NO_ERROR) - //printf("388. error code:%i \n", rc); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rowId++; - rc = m_indexlist.deleteIndexList( pFile, 0, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc == NO_ERROR ); - delete_count++; - //delete the same item again, should not be there, return an error - rc = m_indexlist.deleteIndexList( pFile, 0, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc != NO_ERROR ); - // delete an rowId which does not exist in the index list - // return a error - rc = m_indexlist.deleteIndexList( pFile, 1, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc != NO_ERROR ); - // - rc = m_indexlist.updateIndexList(pFile, 5, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_indexlist.updateIndexList(pFile, 6, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_indexlist.deleteIndexList( pFile, 5, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_indexlist.deleteIndexList( pFile, 6, key, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc == NO_ERROR ); - fbo = newIdxListHdrPtr.fbo; //fbo=? from free manager - sbid = newIdxListHdrPtr.sbid;//sbid =? - entry = newIdxListHdrPtr.entry;//entry=? - //printf("Index List Header: fbo: %i sbid: %i entry: %i\n", (int)fbo, (int)sbid, (int)entry); - m_indexlist.closeFile(pFile); - // Reading the Header out - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == 0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.type == 0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.key == key ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.type == 7 ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.rid == (RID)0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.nextIdxRidListPtr.type == 7 ); - CPPUNIT_ASSERT( newIdxRidListHdr.nextIdxRidListPtr.llp == 0x0 ); - m_indexlist.closeFile(pFile); - - //Add Header again and also Update the index list - rowId = 0; - delete_count = 0; - // - memset( curBlock.data, 0, sizeof(curBlock.data)); - m_indexlist.deleteFile( file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - - - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - m_indexlist.init(cb, LIST); - rc = m_indexlist.addIndexListHdr(pFile, rowId, key, &newIdxListHdrPtr); - found = m_indexlist.findRowId(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); - //printf("Find RowId: %i in : fbo: %i sbid: %i entry: %i\n", (int) rowId, (int)p_fbo, (int)p_sbid, (int)p_entry); - //cout << " RowId->" << rowId <<" Fbo->" << p_fbo << " Sbid->" << p_sbid << " Entry->" << p_entry << endl; - rowId++; - - CPPUNIT_ASSERT( rc == NO_ERROR ); - fbo = newIdxListHdrPtr.fbo; //fbo=? from the free manager - sbid = newIdxListHdrPtr.sbid;//sbid =? from the free manager - entry = newIdxListHdrPtr.entry;//entry=? from the free manager - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, width, &newIdxRidListHdr ); - - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == 1 ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.type == 0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.key == key ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.rid == (RID)rowId - 1 ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.type == 3 ); - CPPUNIT_ASSERT( newIdxRidListHdr.nextIdxRidListPtr.llp == 0x0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.nextIdxRidListPtr.type == 7 ); - - - for (i = 0; i < count; i++) - { - //printf("Index List Count: i: %i Row Id: %i \n", i, rowId); - rc = m_indexlist.updateIndexList(pFile, rowId, key, &newIdxListHdrPtr); - - rowId++; - // if (rc != NO_ERROR) - // printf("ERROR ->2940 i %i \n rowID %i \n", i, rowId); - //CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - for (i = 0; i < count; i++) - { - found = m_indexlist.findRowId(pFile, i, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); - //printf("Find RowId: %i in : fbo: %i sbid: %i entry: %i\n", (int) rowId, (int)p_fbo, (int)p_sbid, (int)p_entry); - //if (found) - //cout << " Found RowId->" << i <<" Fbo->" << p_fbo << " Sbid->" << p_sbid << " Entry->" << p_entry << endl; - //else - //cout << " Not Found RowId->" << i << endl; - //Check Header again, we need to read the block again - //since it is changed - } - - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, width, &newIdxRidListHdr ); - - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)k ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.type == 0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.key == key ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.rid == (RID)0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.type == 3 ); - CPPUNIT_ASSERT( newIdxRidListHdr.nextIdxRidListPtr.type == 4 ); - //printf("Index List Header: fbo: %i sbid: %i entry: %i\n", (int)fbo, (int)sbid, (int)entry); - m_indexlist.closeFile(pFile); - - //Delete test - //Read now - //Check Header - - delete_count = 0; - fbo = newIdxListHdrPtr.fbo; //fbo=? - sbid = newIdxListHdrPtr.sbid;//sbid =? - entry = newIdxListHdrPtr.entry;//entry=? - - memset( &newIdxRidListHdr, 0, 32 ); - memset( curBlock.data, 0, sizeof( curBlock.data ) ); - - //Delete action - pFile = m_indexlist.openFile( file_number); - CPPUNIT_ASSERT( pFile != NULL ); - - rc = m_indexlist.deleteIndexList( pFile, 0, key, &newIdxListHdrPtr);//rowId 0 deleted - - if (rc == NO_ERROR) - delete_count++; - else - { - //cout << "cannot find rowId->" << 0 << endl; - } - - rc = m_indexlist.deleteIndexList( pFile, 100, key, &newIdxListHdrPtr);//rowId 100 deleted - - if (rc == NO_ERROR) - delete_count++; - else - { - //cout << "cannot find rowId->" << 100 << endl; - } - - rc = m_indexlist.deleteIndexList( pFile, 975, key, &newIdxListHdrPtr);//rowId 975 deleted - - if (rc == NO_ERROR) - delete_count++; - else - { - //cout << "cannot find rowId->" << 975 << endl; - } - - rc = m_indexlist.deleteIndexList( pFile, 2000, key, &newIdxListHdrPtr);//rowId 2000 deleted - - if (rc == NO_ERROR) - delete_count++; - else - { - //cout << "cannot find rowId->" << 2000 << endl; - } - - m_indexlist.closeFile(pFile); - - //Check Answers Read File at block fbo=0; - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo); - width = 32; - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, width, &newIdxRidListHdr ); - k = k - delete_count; - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)k ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.type == 0 ); - CPPUNIT_ASSERT( newIdxRidListHdr.key == key ); - - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.type == 7 ); - CPPUNIT_ASSERT( newIdxRidListHdr.firstIdxRidListEntry.rid == 0 ); - m_indexlist.closeFile(pFile); - printf("\nSuccessfully Running testIndexListDelete\n"); - }//testIndexListDelete - - - void testIndexListReleaseMgrBack() - { - - - FILE* pFile = NULL; - int rc, allocSize; - int file_number = 8006; - DataBlock BlockZero; - int i; - IdxEmptyListEntry newIdxListHdrPtr; - IdxRidListHdr idxRidListHdr; - int rowId = 0; - int newCnt = 2000; - CommBlock cb; - File file; - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - BRMWrapper::setUseBrm(true); - m_indexlist.init(cb, LIST); - int key = 123; - printf("\nRunning testIndexListReleaseMgrBack\n"); - - memset( BlockZero.data, 0, sizeof(BlockZero.data)); - m_indexlist.deleteFile( file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - CPPUNIT_ASSERT( pFile != NULL ); - m_indexlist.init(cb, LIST); - rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); - /* - if (rc != NO_ERROR) - { - //cout << "rc -> " << rc << endl; - //cout << "i->"<< i << endl; - //cout << "rowId->" << rowId << endl; - //cout << "key->" << key << endl; - } - */ - CPPUNIT_ASSERT( rc == NO_ERROR); - - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - //if (rc!= NO_ERROR) - //printf("724. error code:%i \n", rc); - CPPUNIT_ASSERT( rc == NO_ERROR); - rowId++; - rc = m_indexlist.updateIndexList(pFile, rowId, key + 1, &newIdxListHdrPtr); - /* - if (rc != NO_ERROR) - { - //cout << "rc -> " << rc << endl; - //cout << "rowId->" << rowId << endl; - //cout << "key+1->" << key+1 << endl; - } - */ - CPPUNIT_ASSERT( rc != NO_ERROR); - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - /* - if (rc != NO_ERROR) - { - //cout <<" getHdrInfo has error-> rc " << rc << endl; - } - */ - CPPUNIT_ASSERT( rc == NO_ERROR); - - for (i = 0; i < newCnt; i++) - { - rc = m_indexlist.updateIndexList(pFile, rowId, key, &newIdxListHdrPtr); - /* - if (rc != NO_ERROR) - { - //cout <<" updateIndexList has error->" << endl; - //cout << "rc -> " << rc << endl; - //cout << "i->"<< i << endl; - //cout << "rowId->" << rowId << endl; - //cout << "key->" << key << endl; - } - */ - CPPUNIT_ASSERT( rc == NO_ERROR); - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - /* - if (rc != NO_ERROR) - { - //cout <<" getHdrInfo has error->" << endl; - //cout << "rc -> " << rc << endl; - //cout << "i->"<< i << endl; - //cout << "rowId->" << rowId << endl; - //cout << "key->" << key << endl; - } - */ - CPPUNIT_ASSERT( rc == NO_ERROR); - rowId++; - } - - rowId--; - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - rc = m_indexlist.deleteIndexList( pFile, rowId, - key + 1, &newIdxListHdrPtr); - CPPUNIT_ASSERT( rc != NO_ERROR); - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - - for (i = newCnt; i >= 0; i--) - { - rc = m_indexlist.deleteIndexList( pFile, rowId, - key, &newIdxListHdrPtr); - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - CPPUNIT_ASSERT( rc == NO_ERROR); - rowId--; - } - - rowId++; - - for (i = 0; i < newCnt; i++) - { - rc = m_indexlist.updateIndexList(pFile, rowId, key, &newIdxListHdrPtr); - rc = m_indexlist.getHdrInfo(&newIdxListHdrPtr, &idxRidListHdr ); - /* - if (rc!=NO_ERROR) - { - //cout << " tindex 3099-> i " << i << " rc " << rc << endl; - } - */ - CPPUNIT_ASSERT( rc == NO_ERROR); - rowId++; - - } - - m_indexlist.closeFile(pFile); - printf("\nSuccessfully Running testIndexListReleaseMgrBack\n"); - return; + } } - void testIndexListMultipleAddHdr() + */ + // uint64_t p2_fbo ; + // int p2_sbid, p2_entry; + rowId = 0; + int rowId2 = count + 1; + int rowId3 = 2 * (count + 1); + int rowId4 = 3 * (count + 1); + + for (i = 0; i < count; i++) { + found = m_indexlist.findRowId(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); + /* + if (!found) + { + printf(" \n Not Found tindex 3659->i %i rowId %i Not Found \n", i, (int)rowId); + } + else + { + //cout << " findRowId-> count in findRowId line 3663-> " << i <<" rowId " << rowId << endl; + //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; + } + */ + CPPUNIT_ASSERT(found == true); + /*rc = m_indexlist.deleteIndexList( pFile, rowId, + key, &newIdxListHdrPtr, + p2_fbo, p2_sbid, p2_entry); + */ - DataBlock curBlock; - DataBlock curBlock2; - DataBlock curBlock3; - DataBlock curBlock4; - FILE* pFile = NULL; - int rc, fbo, sbid, entry ; - int fbo2, sbid2, entry2; - int fbo3, sbid3, entry3; - int fbo4, sbid4, entry4; - uint64_t key = 1; - uint64_t key2 = 2; - uint64_t key3 = 3; - uint64_t key4 = 4; - int count = 2000; - int i = 0; - int delete_count; - int file_number = 8010; - RID rowIdArray[count + 1]; - RID rowIdArray2[count + 1]; - RID rowIdArray3[count + 1]; - RID rowIdArray4[count + 1]; - int j, k, l; - RID rowId; + // //cout << " deleteIndexList-> p2_fbo " << p2_fbo << " p2_sbid " << p2_sbid << " p2_entry " << + // p2_entry << endl; + // CPPUNIT_ASSERT( rc == NO_ERROR ); - IdxEmptyListEntry newIdxListHdrPtr; - IdxRidListHdr newIdxRidListHdr; - IdxEmptyListEntry newIdxListHdrPtr2; - IdxRidListHdr newIdxRidListHdr2; - IdxEmptyListEntry newIdxListHdrPtr3; - IdxRidListHdr newIdxRidListHdr3; - IdxEmptyListEntry newIdxListHdrPtr4; - IdxRidListHdr newIdxRidListHdr4; + found = m_indexlist.findRowId(pFile, rowId2, key2, &newIdxListHdrPtr2, p_fbo, p_sbid, p_entry); + /* + if (!found) + { + printf(" \n Not Found tindex 3678->i %i rowId2 %i Not Found \n", i, (int)rowId2); + } + else + { + //cout << " findRowId-> count in findRowId line 3682-> " << i <<" rowId2 " << rowId2 << endl; + //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; + } + */ + CPPUNIT_ASSERT(found == true); - bool found = false; - int p_sbid, p_entry, allocSize; - uint64_t p_fbo; - BRMWrapper::setUseBrm(true); - m_indexlist.setUseSortFlag(false); - printf("\nRunning testIndexListMultipleAddHdr\n"); - delete_count = 0; - memset( curBlock.data, 0, sizeof(curBlock.data)); - m_indexlist.deleteFile( file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - CommBlock cb; - File file; - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - m_indexlist.init(cb, LIST); + found = m_indexlist.findRowId(pFile, rowId3, key3, &newIdxListHdrPtr3, p_fbo, p_sbid, p_entry); + /* + if (!found) + { + printf(" \n Not Found tindex 3691->i %i rowId3 %i Not Found \n", i, (int)rowId3); + } + else + { + //cout << " findRowId-> count in findRowId line 3695-> " << i <<" rowId3 " << rowId3 << endl; + //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; + } + */ + CPPUNIT_ASSERT(found == true); - for (i = 0; i < count + 1; i++) - { - rowIdArray[i] = i; - } + found = m_indexlist.findRowId(pFile, rowId4, key4, &newIdxListHdrPtr4, p_fbo, p_sbid, p_entry); + /* + if (!found) + { + printf(" \n Not Found tindex 3704->i %i rowId4 %i Not Found \n", i, (int)rowId4); + } + else + { + //cout << " findRowId-> count in findRowId line 3708-> " << i <<" rowId4 " << rowId4 << endl; + //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; + } + */ + CPPUNIT_ASSERT(found == true); - for (j = 0; j < count + 1; j++) - { - rowIdArray2[j] = i; - i++; - } + rowId++; - for (k = 0; k < count + 1; k++) - { - rowIdArray3[k] = i; - i++; - } + rowId2++; + rowId3++; + rowId4++; + } - for (l = 0; l < count + 1; l++) - { - rowIdArray4[l] = i; - i++; - } + /* + rc = m_indexlist.deleteIndexList( pFile, rowId, + key, &newIdxListHdrPtr, + p_fbo, p_sbid, p_entry); + ////cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - i = count + 1; - //Initialize first rowId to 0 - //cout << " start Timer :inserting " << count << " rowId " << endl; - m_indexlist.startfTimer(); - int count_size = count + 1; - rc = m_indexlist.addIndexListHdr(cb, &rowIdArray[0], count_size, key, &newIdxListHdrPtr); + CPPUNIT_ASSERT( rc == NO_ERROR ); + m_indexlist.closeFile(pFile); - rc = m_indexlist.addIndexListHdr(cb, &rowIdArray2[0], count_size, key2, &newIdxListHdrPtr2); - rc = m_indexlist.addIndexListHdr(cb, &rowIdArray3[0], count_size, key3, &newIdxListHdrPtr3); - rc = m_indexlist.addIndexListHdr(cb, &rowIdArray4[0], count_size, key4, &newIdxListHdrPtr4); + // + memset( curBlock.data, 0, sizeof(curBlock.data)); + pFile= m_indexlist.openFile( file_number ); + CPPUNIT_ASSERT( pFile != NULL ); + rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); + m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); + CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == 0 ); + */ + m_indexlist.closeFile(pFile); + printf("\nSuccessfully Running testIndexListMultipleAddHdr\n"); + } // testIndexListMultipleAddHdr + void testIndexListMultipleUpdate() + { + DataBlock curBlock; + FILE* pFile = NULL; + int rc, fbo, sbid, entry; + uint64_t key = 123; + int count = 10000; + int rowId; + int i; + int delete_count; + int file_number = 8012; + RID rowIdArray[count + 1]; - m_indexlist.stopfTimer(); - //cout << " End Timer :inserting " << count << " rowId " << endl; - //cout << " Total used time (msec) " << m_indexlist.getTotalfRunTime() << endl; - //if (rc!= NO_ERROR) - // printf("245. error code:%i \n", rc); - CPPUNIT_ASSERT( rc == NO_ERROR ); + IdxEmptyListEntry newIdxListHdrPtr; + IdxRidListHdr newIdxRidListHdr; + bool found = false; + int p_sbid, p_entry, allocSize; + uint64_t p_fbo; + BRMWrapper::setUseBrm(true); + m_indexlist.setUseSortFlag(false); + printf("\nRunning testIndexListMultipleUpdate\n"); + rowId = 0; + delete_count = 0; + memset(curBlock.data, 0, sizeof(curBlock.data)); + m_indexlist.deleteFile(file_number); + m_indexlist.createFile(file_number, 10, allocSize); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + // Free Manager - fbo = newIdxListHdrPtr.fbo; //fbo=? from the free manager - sbid = newIdxListHdrPtr.sbid;//sbid =? from the free manager - entry = newIdxListHdrPtr.entry;//entry=? from the free manager + CommBlock cb; + File file; + file.oid = file_number; + file.pFile = pFile; + cb.file = file; + m_indexlist.init(cb, LIST); - fbo2 = newIdxListHdrPtr2.fbo; //fbo=? from the free manager - sbid2 = newIdxListHdrPtr2.sbid;//sbid =? from the free manager - entry2 = newIdxListHdrPtr2.entry;//entry=? from the free manager - - - fbo3 = newIdxListHdrPtr3.fbo; //fbo=? from the free manager - sbid3 = newIdxListHdrPtr3.sbid;//sbid =? from the free manager - entry3 = newIdxListHdrPtr3.entry;//entry=? from the free manager - - fbo4 = newIdxListHdrPtr4.fbo; //fbo=? from the free manager - sbid4 = newIdxListHdrPtr4.sbid;//sbid =? from the free manager - entry4 = newIdxListHdrPtr4.entry;//entry=? from the free manager - - - - memset( curBlock.data, 0, sizeof(curBlock.data)); - - memset( curBlock2.data, 0, sizeof(curBlock2.data)); - memset( curBlock3.data, 0, sizeof(curBlock.data)); - memset( curBlock4.data, 0, sizeof(curBlock2.data)); - - - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - RID ridArray[count + 1]; - int ridSize; - rc = m_indexlist.getRIDArrayFromListHdr(pFile, - key, - &newIdxListHdrPtr, - ridArray, ridSize); - //cout << "ridSize ->" << ridSize << endl; - CPPUNIT_ASSERT( ridSize == (int)newIdxRidListHdr.idxRidListSize.size); - - rc = m_indexlist.readDBFile( pFile, curBlock2.data, fbo2 ); - m_indexlist.getSubBlockEntry( curBlock2.data, sbid2, entry2, 32, &newIdxRidListHdr2 ); - RID ridArray2[count + 1]; - int ridSize2; - rc = m_indexlist.getRIDArrayFromListHdr(pFile, - key2, - &newIdxListHdrPtr2, - ridArray2, ridSize2); - //cout << "ridSize2 ->" << ridSize2 << endl; - CPPUNIT_ASSERT( ridSize2 == (int)newIdxRidListHdr2.idxRidListSize.size); - - rc = m_indexlist.readDBFile( pFile, curBlock3.data, fbo3 ); - m_indexlist.getSubBlockEntry( curBlock3.data, sbid3, entry3, 32, &newIdxRidListHdr3 ); - RID ridArray3[count + 1]; - int ridSize3; - rc = m_indexlist.getRIDArrayFromListHdr(pFile, - key3, - &newIdxListHdrPtr3, - ridArray3, ridSize3); - //cout << "ridSize3 ->" << ridSize3 << endl; - CPPUNIT_ASSERT( ridSize3 == (int)newIdxRidListHdr3.idxRidListSize.size); - - rc = m_indexlist.readDBFile( pFile, curBlock4.data, fbo4 ); - m_indexlist.getSubBlockEntry( curBlock4.data, sbid4, entry4, 32, &newIdxRidListHdr4 ); - RID ridArray4[count + 1]; - int ridSize4; - rc = m_indexlist.getRIDArrayFromListHdr(pFile, - key4, - &newIdxListHdrPtr4, - ridArray4, ridSize4); - //cout << "ridSize4 ->" << ridSize4 << endl; - CPPUNIT_ASSERT( ridSize4 == (int)newIdxRidListHdr4.idxRidListSize.size); - //m_indexlist.stopTimer(); - - ////cout << " Total used time (sec) " << m_indexlist.getTotalRunTime() << endl; - - uint64_t firstLbid; - uint64_t firstLbid2; - uint64_t firstLbid3; - uint64_t firstLbid4; - rc = m_indexlist.findFirstBlk(pFile, key, &newIdxListHdrPtr, firstLbid); - - rc = m_indexlist.findFirstBlk(pFile, key2, &newIdxListHdrPtr2, firstLbid2); - rc = m_indexlist.findFirstBlk(pFile, key3, &newIdxListHdrPtr3, firstLbid3); - rc = m_indexlist.findFirstBlk(pFile, key4, &newIdxListHdrPtr4, firstLbid4); - - //cout << " Multiple RIDS Header insert->FirstLbid is ->" << firstLbid << endl; - - //cout << " Multiple RIDS Header insert->FirstLbid2 is ->" << firstLbid2 << endl; - //cout << " Multiple RIDS Header insert->FirstLbid3 is ->" << firstLbid3 << endl; - //cout << " Multiple RIDS Header insert->FirstLbid4 is ->" << firstLbid4 << endl; - - //cout << " This will print level and counts for Children " << endl; - /* - if (m_indexlist.getUseNarray()) - { - m_indexlist.printBlocks(firstLbid); - - m_indexlist.printBlocks(firstLbid2); - m_indexlist.printBlocks(firstLbid3); - m_indexlist.printBlocks(firstLbid4); - - } - */ - CPPUNIT_ASSERT(rc == NO_ERROR); - m_indexlist.closeFile(pFile); - // - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - /* - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)(count+1)); - int arraySize=0; - arraySize = newIdxRidListHdr.idxRidListSize.size; - if (arraySize >0) - { - RID ridArray[arraySize]; - int ridSize; - rc= m_indexlist.getRIDArrayFromListHdr(pFile, - key, - &newIdxListHdrPtr, - ridArray, ridSize); - //cout << "ridSize->" << ridSize << endl; - CPPUNIT_ASSERT(ridSize==arraySize); - - if (rc!= NO_ERROR) - { - printf("ERROR CODE :%i \n", rc); - - } - } - */ - //uint64_t p2_fbo ; - //int p2_sbid, p2_entry; - rowId = 0; - int rowId2 = count + 1; - int rowId3 = 2 * (count + 1); - int rowId4 = 3 * (count + 1); - - for (i = 0; i < count; i++) - { - found = m_indexlist.findRowId(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); - /* - if (!found) - { - printf(" \n Not Found tindex 3659->i %i rowId %i Not Found \n", i, (int)rowId); - } - else - { - //cout << " findRowId-> count in findRowId line 3663-> " << i <<" rowId " << rowId << endl; - //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - } - */ - CPPUNIT_ASSERT( found == true ); - /*rc = m_indexlist.deleteIndexList( pFile, rowId, - key, &newIdxListHdrPtr, - p2_fbo, p2_sbid, p2_entry); - */ - - // //cout << " deleteIndexList-> p2_fbo " << p2_fbo << " p2_sbid " << p2_sbid << " p2_entry " << p2_entry << endl; - //CPPUNIT_ASSERT( rc == NO_ERROR ); - - found = m_indexlist.findRowId(pFile, rowId2, key2, &newIdxListHdrPtr2, p_fbo, p_sbid, p_entry); - /* - if (!found) - { - printf(" \n Not Found tindex 3678->i %i rowId2 %i Not Found \n", i, (int)rowId2); - } - else - { - //cout << " findRowId-> count in findRowId line 3682-> " << i <<" rowId2 " << rowId2 << endl; - //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - } - */ - CPPUNIT_ASSERT( found == true ); - - found = m_indexlist.findRowId(pFile, rowId3, key3, &newIdxListHdrPtr3, p_fbo, p_sbid, p_entry); - /* - if (!found) - { - printf(" \n Not Found tindex 3691->i %i rowId3 %i Not Found \n", i, (int)rowId3); - } - else - { - //cout << " findRowId-> count in findRowId line 3695-> " << i <<" rowId3 " << rowId3 << endl; - //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - } - */ - CPPUNIT_ASSERT( found == true ); - - found = m_indexlist.findRowId(pFile, rowId4, key4, &newIdxListHdrPtr4, p_fbo, p_sbid, p_entry); - /* - if (!found) - { - printf(" \n Not Found tindex 3704->i %i rowId4 %i Not Found \n", i, (int)rowId4); - } - else - { - //cout << " findRowId-> count in findRowId line 3708-> " << i <<" rowId4 " << rowId4 << endl; - //cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - } - */ - CPPUNIT_ASSERT( found == true ); - - rowId++; - - rowId2++; - rowId3++; - rowId4++; - - } - - /* - rc = m_indexlist.deleteIndexList( pFile, rowId, - key, &newIdxListHdrPtr, - p_fbo, p_sbid, p_entry); - ////cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - - CPPUNIT_ASSERT( rc == NO_ERROR ); - m_indexlist.closeFile(pFile); - - // - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile= m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == 0 ); - */ - m_indexlist.closeFile(pFile); - printf("\nSuccessfully Running testIndexListMultipleAddHdr\n"); - }//testIndexListMultipleAddHdr - void testIndexListMultipleUpdate() + for (int i = 0; i < count + 1; i++) { + rowIdArray[i] = i; + } - DataBlock curBlock; - FILE* pFile = NULL; - int rc, fbo, sbid, entry ; - uint64_t key = 123; - int count = 10000; - int rowId; - int i; - int delete_count; - int file_number = 8012; - RID rowIdArray[count + 1]; + rowId = 0; + // Initialize first rowId to 0 + rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); + /* + if (rc!= NO_ERROR) + printf("245. error code:%i \n", rc); + */ + CPPUNIT_ASSERT(rc == NO_ERROR); - IdxEmptyListEntry newIdxListHdrPtr; - IdxRidListHdr newIdxRidListHdr; - bool found = false; - int p_sbid, p_entry, allocSize; - uint64_t p_fbo; - BRMWrapper::setUseBrm(true); - m_indexlist.setUseSortFlag(false); - printf("\nRunning testIndexListMultipleUpdate\n"); - rowId = 0 ; - delete_count = 0; - memset( curBlock.data, 0, sizeof(curBlock.data)); - m_indexlist.deleteFile( file_number); - m_indexlist.createFile(file_number, 10, allocSize); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - //Free Manager + fbo = newIdxListHdrPtr.fbo; // fbo=? from the free manager + sbid = newIdxListHdrPtr.sbid; // sbid =? from the free manager + entry = newIdxListHdrPtr.entry; // entry=? from the free manager + memset(curBlock.data, 0, sizeof(curBlock.data)); - CommBlock cb; - File file; - file.oid = file_number; - file.pFile = pFile; - cb.file = file; - m_indexlist.init(cb, LIST); + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, 32, &newIdxRidListHdr); + RID ridArray2[count + 5]; + int ridSize2; + rc = m_indexlist.getRIDArrayFromListHdr(pFile, key, &newIdxListHdrPtr, ridArray2, ridSize2); + CPPUNIT_ASSERT(ridSize2 == 1); - for (int i = 0; i < count + 1; i++) - { - rowIdArray[i] = i; - } + // Then increment the rowId as it goes + string timeStr; - rowId = 0; - //Initialize first rowId to 0 - rc = m_indexlist.addIndexListHdr(cb, rowId, key, &newIdxListHdrPtr); - /* - if (rc!= NO_ERROR) - printf("245. error code:%i \n", rc); - */ - CPPUNIT_ASSERT( rc == NO_ERROR ); + m_indexlist.startfTimer(); + // cout << " line 3571->startTimer for inserting "<< count <<" rowId " << endl; - fbo = newIdxListHdrPtr.fbo; //fbo=? from the free manager - sbid = newIdxListHdrPtr.sbid;//sbid =? from the free manager - entry = newIdxListHdrPtr.entry;//entry=? from the free manager - memset( curBlock.data, 0, sizeof(curBlock.data)); + rc = m_indexlist.updateIndexList(cb, &rowIdArray[1], count, key, &newIdxListHdrPtr); + /* + if (rc != NO_ERROR) + { + printf("rc %i i %i \n", rc, i); + } + */ - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - RID ridArray2[count + 5]; - int ridSize2; - rc = m_indexlist.getRIDArrayFromListHdr(pFile, - key, - &newIdxListHdrPtr, - ridArray2, ridSize2); - CPPUNIT_ASSERT( ridSize2 == 1 ); + m_indexlist.stopfTimer(); + // cout << " End Time :inserting "<< count << "rowId " << endl; + // cout << " Total used time (msec) " << m_indexlist.getTotalfRunTime() << endl; - //Then increment the rowId as it goes - string timeStr; + m_indexlist.closeFile(pFile); + // + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, 32, &newIdxRidListHdr); + // cout << "newIdxRidListHdr.idxRidListSize.size->" << newIdxRidListHdr.idxRidListSize.size << endl; + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == (uint64_t)(count + 1)); + // CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (int64_t)ridSize2); - m_indexlist.startfTimer(); - //cout << " line 3571->startTimer for inserting "<< count <<" rowId " << endl; + int arraySize = 0; + rowId = 0; + arraySize = newIdxRidListHdr.idxRidListSize.size; - rc = m_indexlist.updateIndexList(cb, &rowIdArray[1], count, key, &newIdxListHdrPtr); - /* - if (rc != NO_ERROR) - { - printf("rc %i i %i \n", rc, i); - } - */ + if (arraySize > 0) + { + RID ridArray[arraySize]; + int ridSize; + rc = m_indexlist.getRIDArrayFromListHdr(pFile, key, &newIdxListHdrPtr, ridArray, ridSize); + CPPUNIT_ASSERT(ridSize == arraySize); + /* + if (rc!= NO_ERROR) + { + printf("ERROR CODE :%i \n", rc); - m_indexlist.stopfTimer(); - //cout << " End Time :inserting "<< count << "rowId " << endl; - //cout << " Total used time (msec) " << m_indexlist.getTotalfRunTime() << endl; + } + */ + // for (int i=0; i< ridSize ; i++) + // //cout<< " line 3591->ridArray[i]->" << ridArray[i] << " i->" << i << endl; + } - m_indexlist.closeFile(pFile); - // - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - //cout << "newIdxRidListHdr.idxRidListSize.size->" << newIdxRidListHdr.idxRidListSize.size << endl; - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (uint64_t)(count + 1)); - //CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == (int64_t)ridSize2); + uint64_t firstLbid; + rc = m_indexlist.findFirstBlk(pFile, key, &newIdxListHdrPtr, firstLbid); + // cout << " Multiple RIDS insert->FirstLbid is ->" << firstLbid << endl; + // cout << " This will print level and counts for Children " << endl; + /* + if (m_indexlist.getUseNarray()) + m_indexlist.printBlocks(firstLbid); + */ - int arraySize = 0; - rowId = 0; - arraySize = newIdxRidListHdr.idxRidListSize.size; + uint64_t p2_fbo; + int p2_sbid, p2_entry; + rowId = 0; - if (arraySize > 0) - { - RID ridArray[arraySize]; - int ridSize; - rc = m_indexlist.getRIDArrayFromListHdr(pFile, - key, - &newIdxListHdrPtr, - ridArray, ridSize); - CPPUNIT_ASSERT(ridSize == arraySize); - /* - if (rc!= NO_ERROR) - { - printf("ERROR CODE :%i \n", rc); + for (i = 0; i < count + 1; i++) + { + found = m_indexlist.findRowId(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); + /* + if (!found) + { + printf(" tindex 3600->i %i rowId %i Not Found \n", i, rowId); + //return; + } + else + { + ////cout << " Found count in findRowId line 3605-> " << i <<" rowId " << rowId << endl; + ////cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - } - */ - //for (int i=0; i< ridSize ; i++) - // //cout<< " line 3591->ridArray[i]->" << ridArray[i] << " i->" << i << endl; - } + } + */ + CPPUNIT_ASSERT(found == true); + rc = m_indexlist.deleteIndexList(pFile, rowId, key, &newIdxListHdrPtr, p2_fbo, p2_sbid, p2_entry); - uint64_t firstLbid; - rc = m_indexlist.findFirstBlk(pFile, key, &newIdxListHdrPtr, firstLbid); - //cout << " Multiple RIDS insert->FirstLbid is ->" << firstLbid << endl; - //cout << " This will print level and counts for Children " << endl; - /* - if (m_indexlist.getUseNarray()) - m_indexlist.printBlocks(firstLbid); - */ + ////cout << " p2_fbo " << p2_fbo << " p2_sbid " << p2_sbid << " p2_entry " << p2_entry << endl; + CPPUNIT_ASSERT(rc == NO_ERROR); + rowId++; + } - uint64_t p2_fbo ; - int p2_sbid, p2_entry; - rowId = 0; + rc = m_indexlist.deleteIndexList(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); - for (i = 0; i < count + 1; i++) - { - found = m_indexlist.findRowId(pFile, rowId, key, &newIdxListHdrPtr, p_fbo, p_sbid, p_entry); - /* - if (!found) - { - printf(" tindex 3600->i %i rowId %i Not Found \n", i, rowId); - //return; - } - else - { - ////cout << " Found count in findRowId line 3605-> " << i <<" rowId " << rowId << endl; - ////cout << " p_fbo " << p_fbo << " p_sbid " << p_sbid << " p_entry " << p_entry << endl; - - } - */ - CPPUNIT_ASSERT( found == true ); - rc = m_indexlist.deleteIndexList( pFile, rowId, - key, &newIdxListHdrPtr, - p2_fbo, p2_sbid, p2_entry); - - ////cout << " p2_fbo " << p2_fbo << " p2_sbid " << p2_sbid << " p2_entry " << p2_entry << endl; - CPPUNIT_ASSERT( rc == NO_ERROR ); - rowId++; - } - - rc = m_indexlist.deleteIndexList( pFile, rowId, - key, &newIdxListHdrPtr, - p_fbo, p_sbid, p_entry); - - CPPUNIT_ASSERT( rc != NO_ERROR ); - m_indexlist.closeFile(pFile); - // - memset( curBlock.data, 0, sizeof(curBlock.data)); - pFile = m_indexlist.openFile( file_number ); - CPPUNIT_ASSERT( pFile != NULL ); - rc = m_indexlist.readDBFile( pFile, curBlock.data, fbo ); - m_indexlist.getSubBlockEntry( curBlock.data, sbid, entry, 32, &newIdxRidListHdr ); - CPPUNIT_ASSERT( newIdxRidListHdr.idxRidListSize.size == 0 ); - - m_indexlist.closeFile(pFile); - printf("\nSuccessfully Running testIndexListMultipleUpdate\n"); - }//testIndexListMultipleUpdate + CPPUNIT_ASSERT(rc != NO_ERROR); + m_indexlist.closeFile(pFile); + // + memset(curBlock.data, 0, sizeof(curBlock.data)); + pFile = m_indexlist.openFile(file_number); + CPPUNIT_ASSERT(pFile != NULL); + rc = m_indexlist.readDBFile(pFile, curBlock.data, fbo); + m_indexlist.getSubBlockEntry(curBlock.data, sbid, entry, 32, &newIdxRidListHdr); + CPPUNIT_ASSERT(newIdxRidListHdr.idxRidListSize.size == 0); + m_indexlist.closeFile(pFile); + printf("\nSuccessfully Running testIndexListMultipleUpdate\n"); + } // testIndexListMultipleUpdate }; -CPPUNIT_TEST_SUITE_REGISTRATION( IndexTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(IndexTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/writeengine/index/we_freemgr.cpp b/writeengine/index/we_freemgr.cpp index a9295ee18..0f8918102 100644 --- a/writeengine/index/we_freemgr.cpp +++ b/writeengine/index/we_freemgr.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_freemgr.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -******************************************************************************************/ + * $Id: we_freemgr.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + ******************************************************************************************/ /** @file */ #include @@ -34,429 +34,420 @@ namespace WriteEngine */ FreeMgr::FreeMgr() { - //memset( m_workBlock.data, 0, BYTE_PER_BLOCK ); + // memset( m_workBlock.data, 0, BYTE_PER_BLOCK ); - - // use 0 for old style, 1 for new without block list and 2 for new with block list - initType = 2; -// setDebugLevel( DEBUG_1 ); - allowExtend = 1; + // use 0 for old style, 1 for new without block list and 2 for new with block list + initType = 2; + // setDebugLevel( DEBUG_1 ); + allowExtend = 1; } /** * Default Destructor */ FreeMgr::~FreeMgr() -{} - +{ +} /** * * blank out all stats, set types and groups of null pointers in block zero */ -const int FreeMgr::initBlockzero( DataBlock* blockZero) +const int FreeMgr::initBlockzero(DataBlock* blockZero) { - IdxEmptyListEntry Pointer; - uint64_t blankVal = 0; + IdxEmptyListEntry Pointer; + uint64_t blankVal = 0; - memset(blockZero->data, 0, sizeof(blockZero->data)); + memset(blockZero->data, 0, sizeof(blockZero->data)); - nullPtr( &Pointer ); - Pointer.type = EMPTY_LIST; + nullPtr(&Pointer); + Pointer.type = EMPTY_LIST; - Pointer.group = ENTRY_BLK; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_BLK), 8, &Pointer); + Pointer.group = ENTRY_BLK; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_BLK), 8, &Pointer); - Pointer.group = ENTRY_32; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_32), 8, &Pointer); + Pointer.group = ENTRY_32; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_32), 8, &Pointer); - Pointer.group = ENTRY_16; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_16), 8, &Pointer); + Pointer.group = ENTRY_16; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_16), 8, &Pointer); - Pointer.group = ENTRY_8; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_8), 8, &Pointer); + Pointer.group = ENTRY_8; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_8), 8, &Pointer); - Pointer.group = ENTRY_4; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_4), 8, &Pointer); + Pointer.group = ENTRY_4; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_4), 8, &Pointer); - Pointer.group = ENTRY_2; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_2), 8, &Pointer); + Pointer.group = ENTRY_2; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_2), 8, &Pointer); - Pointer.group = ENTRY_1; - setSubBlockEntry( blockZero->data, 0, calcPtrOffset(ENTRY_1 + ENTRY_1), 8, &Pointer); + Pointer.group = ENTRY_1; + setSubBlockEntry(blockZero->data, 0, calcPtrOffset(ENTRY_1 + ENTRY_1), 8, &Pointer); - //all stat locations are contiguous and after the pointers in block zero + // all stat locations are contiguous and after the pointers in block zero - for (int idx = calcStatOffset( ENTRY_1 ); idx < calcStatOffset( ENTRY_32 ) + 1; idx++) - { - setSubBlockEntry( blockZero->data, 0, idx, 8, &blankVal); - } + for (int idx = calcStatOffset(ENTRY_1); idx < calcStatOffset(ENTRY_32) + 1; idx++) + { + setSubBlockEntry(blockZero->data, 0, idx, 8, &blankVal); + } - - return NO_ERROR; + return NO_ERROR; } -const int FreeMgr::init(CommBlock& cb, DataBlock* blockZero, int freemgrType, IdxTreeGroupType chainType, int startBlock, int numberBlocks) +const int FreeMgr::init(CommBlock& cb, DataBlock* blockZero, int freemgrType, IdxTreeGroupType chainType, + int startBlock, int numberBlocks) { + /** + * Use this to allocate a group of blocks or sub blocks to a free list + * startBlock is fbo NOT lbid + * This allows us to use a simple for loop to traverese the full range of blocks + * + **/ - /** - * Use this to allocate a group of blocks or sub blocks to a free list - * startBlock is fbo NOT lbid - * This allows us to use a simple for loop to traverese the full range of blocks - * - **/ + int blkIdx, sbIdx; + int rc; + IdxEmptyListEntry emptyEntry; // populate the chains with pointer to empty entries + FILE* indexFile; - int blkIdx, sbIdx; - int rc; - IdxEmptyListEntry emptyEntry; // populate the chains with pointer to empty entries - FILE* indexFile; + indexFile = cb.file.pFile; - indexFile = cb.file.pFile; + nullPtr(&emptyEntry); + emptyEntry.type = EMPTY_PTR; - nullPtr(&emptyEntry); - emptyEntry.type = EMPTY_PTR; + if (isDebug(DEBUG_3)) + { + printf("\nNew style Init v2, file: %u startBlock: %i num: %i \n", cb.file.oid, (unsigned int)startBlock, + (unsigned int)numberBlocks); + } - if ( isDebug( DEBUG_3 )) + int lastBlock = numberBlocks + startBlock; + + if (chainType == ENTRY_BLK) + { + emptyEntry.group = ENTRY_BLK; + + for (blkIdx = lastBlock; blkIdx > startBlock; blkIdx--) { - printf("\nNew style Init v2, file: %u startBlock: %i num: %i \n", cb.file.oid, (unsigned int)startBlock, (unsigned int)numberBlocks); - } + // if( isDebug( DEBUG_1 )) { printf("Initing block %u\n", blkIdx-1);} + emptyEntry.fbo = mapLBID(cb, blkIdx - 1, rc); // map fbo to lbid before storage - int lastBlock = numberBlocks + startBlock; - - if (chainType == ENTRY_BLK) - { - emptyEntry.group = ENTRY_BLK; - - for ( blkIdx = lastBlock ; blkIdx > startBlock ; blkIdx-- ) - { -// if( isDebug( DEBUG_1 )) { printf("Initing block %u\n", blkIdx-1);} - emptyEntry.fbo = mapLBID( cb, blkIdx - 1, rc ); // map fbo to lbid before storage - - if (rc != NO_ERROR ) - { - printf("DBG: Init 1: Error resolving LBID for OID: %u FBO: %u\n", cb.file.oid, blkIdx - 1 ); - return rc; - } - - if ( isDebug( DEBUG_3 )) - { - cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; - } - - emptyEntry.sbid = 0; - emptyEntry.type = EMPTY_PTR; - rc = releaseSegment( cb, blockZero, freemgrType, chainType, &emptyEntry ); - - if (rc != NO_ERROR) - { - printf("Error releasing sb\n"); - return rc; - } - } - - } - else if (chainType == ENTRY_32 ) - { - - emptyEntry.group = ENTRY_32; - - for ( blkIdx = lastBlock ; blkIdx > startBlock ; blkIdx-- ) - { -// if( isDebug( DEBUG_1 )) { printf("Initing block %u\n", blkIdx-1);} - emptyEntry.fbo = mapLBID( cb, blkIdx - 1, rc ); // map fbo to lbid before storage - - if (rc != NO_ERROR ) - { - printf("DBG: Init 2: Error resolving LBID for OID: %u FBO: %u\n", cb.file.oid, blkIdx - 1 ); - return rc; - } - - if ( isDebug( DEBUG_3 )) - { - cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; - } - - for (sbIdx = BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK - 1; sbIdx > -1; sbIdx--) - { - if ( isDebug( DEBUG_3 )) - { - printf("DBG: Working on subblock %u\n", sbIdx); - } - - emptyEntry.sbid = sbIdx; - emptyEntry.type = EMPTY_PTR; - rc = releaseSubblock( cb, blockZero, freemgrType, &emptyEntry ); - - if (rc != NO_ERROR) - { - printf("Error releasing sb\n"); - return rc; - } - } - } - - } - else - { - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Invalid segment size %i (should be ENTRY_32 or ENTRY_BLK)\n", chainType); - } - - return ERR_INVALID_PARAM; - } - - // now write sb0 back to disk - uint64_t lbid = mapLBID( cb, 0, rc); - - if (rc != NO_ERROR ) - { + if (rc != NO_ERROR) + { + printf("DBG: Init 1: Error resolving LBID for OID: %u FBO: %u\n", cb.file.oid, blkIdx - 1); return rc; - } + } - rc = writeDBFile( cb, blockZero, lbid ); + if (isDebug(DEBUG_3)) + { + cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; + } - if (rc != NO_ERROR) - { + emptyEntry.sbid = 0; + emptyEntry.type = EMPTY_PTR; + rc = releaseSegment(cb, blockZero, freemgrType, chainType, &emptyEntry); + + if (rc != NO_ERROR) + { + printf("Error releasing sb\n"); return rc; + } } - - return NO_ERROR; -} - - -/*********************************************************** -* DESCRIPTION: -* Initializes free space lists in an new index file -* PARAMETERS: -* indexFile - file pointer for new index file -* freemgr_type - type of free mgr to init -* RETURN: -* NO_ERROR if success -* -***********************************************************/ - - -const int FreeMgr::init( CommBlock& cb, int freemgrType) -{ - - /** - * Tree has 6 chains, (1,2,4,8,16,32 entries) starting at entry 1 in sb0 of block0 - * List has 3 chains (4,32,1024 entries) starting at first entry of file - * - * Starting at Block 1, add each subblock after sb0 to the free list in sb0 - * then go back and build the smaller chains on demand. This way we make sure all SBs - * are in a chain somewhere.. but this is not best behaviour when allocating blocks - * - **/ - - int rc; // return code from file ops - DataBlock blockZero; - DataBlock workBlock; - int sbIdx; - uint64_t blkIdx; - IdxEmptyListEntry emptyEntry; // populate the chains with pointer to empty entries - IdxEmptyListEntry nextPointer; // pointer at end of sub-block of pointers to emptyEntries - uint64_t numBlocks; - FILE* indexFile; - - indexFile = cb.file.pFile; - - if (!indexFile) // Make sure that we have non-null filehandle - { -// printf("DBG: File handle is null\n"); - return ERR_INVALID_PARAM; - } - - if (freemgrType != TREE && freemgrType != LIST) - { -// printf("DBG: Bad type in freeMgr Init\n"); - return ERR_INVALID_PARAM; - } - - numBlocks = getFileSize( indexFile ) / BYTE_PER_BLOCK ; - -// printf ("DBG: File size: %lu Total blocks: %llu (%u)\n", getFileSize( indexFile ), numBlocks, BYTE_PER_BLOCK); -// printf ("DBG: Adding sub-blocks: %llu \n", numBlocks * 32); - - // Clear the list of pointers in sb0 - initBlockzero( &blockZero ); - - // initialize the non varying fields - nullPtr(&emptyEntry); - emptyEntry.type = EMPTY_PTR; + } + else if (chainType == ENTRY_32) + { emptyEntry.group = ENTRY_32; - // nextPointer identifies next sub-block with empty pointers - // initially there is no next sub-block so zero it out - nullPtr(&nextPointer); - nextPointer.type = EMPTY_LIST; - nextPointer.group = ENTRY_32; - nextPointer.entry = ENTRY_PER_SUBBLOCK - 1; // last entry on the list.. think of the list as a stack - - if (initType == 0) + for (blkIdx = lastBlock; blkIdx > startBlock; blkIdx--) { - if ( isDebug( DEBUG_3 )) - { - printf("\nOld style init\n"); - } + // if( isDebug( DEBUG_1 )) { printf("Initing block %u\n", blkIdx-1);} + emptyEntry.fbo = mapLBID(cb, blkIdx - 1, rc); // map fbo to lbid before storage - for ( blkIdx = numBlocks - 1; blkIdx > 0; blkIdx-- ) - { - emptyEntry.fbo = blkIdx;// map fbo to lbid before storage -// emptyEntry.fbo = mapLBID( cb, blkIdx, rc );// map fbo to lbid before storage -// if (rc != NO_ERROR ){ printf("DBG: Error resolving LBID for OID: %u FBO: %llu\n", cb.file.oid, blkIdx-1 ); return rc; } - - if ( isDebug( DEBUG_3 )) - { - cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; - } - - memset(workBlock.data, 0, sizeof(workBlock.data)); - - /** - * each block after zeroth block uses sb0 to store entry list - * first entry (#0) is the llp to additional sblks with entries - * entries 1-31 are pointers - **/ - - // sb0 is used for initial map, so start at sb1 - for (sbIdx = 1; sbIdx < BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK; sbIdx++) - { - if ( isDebug( DEBUG_3 )) - { - printf("DBG: Working on subblock %u\n", sbIdx); - } - - emptyEntry.sbid = sbIdx; - emptyEntry.type = EMPTY_PTR; - // store pointer in sb0 - replace this with a releaseSubblock call - setSubBlockEntry( workBlock.data, 0, sbIdx, 8, &emptyEntry); - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Init sb fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry << "\n"; - } - } - - // having stored all pointers, store linkptr - setSubBlockEntry( workBlock.data, 0, 0, 8, &nextPointer); -// nextPointer.fbo = mapLBID( cb, blkIdx, rc ); // remember this block ID - nextPointer.fbo = blkIdx; // remember this block ID -// if (rc != NO_ERROR ){ printf("DBG: Error resolving LBID for OID: %u FBO: %llu\n", cb.file.oid, blkIdx-1 ); return rc; } - - rc = writeDBFile( cb, workBlock.data, emptyEntry.fbo ); - - if (rc != NO_ERROR) - { - return rc; - } - - } - - // chain for segments of sub block size - setSubBlockEntry( blockZero.data, 0, calcPtrOffset(ENTRY_32), 8, &nextPointer); - - /** - * the next algorithm uses the release sub-block method and does not layout the map on particular boundaries. - **/ - - } - else if (initType == 1) // new style - use release - { - if ( isDebug( DEBUG_3 )) - { - printf("\nNew style\n"); - } - - for ( blkIdx = numBlocks - 1; blkIdx > 0; blkIdx-- ) - { - - emptyEntry.fbo = blkIdx;// map fbo to lbid before storage - -// emptyEntry.fbo = mapLBID( cb, blkIdx, rc );// map fbo to lbid before storage -// if (rc != NO_ERROR ){ printf("DBG: Error resolving LBID for OID: %u FBO: %llu\n", cb.file.oid, blkIdx-1 ); return rc; } - - if ( isDebug( DEBUG_3 )) - { - cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; - } - - memset(workBlock.data, 0, sizeof(workBlock.data)); - - for (sbIdx = BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK - 1; sbIdx > -1; sbIdx--) - { - if ( isDebug( DEBUG_3 )) - { - printf("DBG: Working on subblock %u\n", sbIdx); - } - - emptyEntry.sbid = sbIdx; - emptyEntry.type = EMPTY_PTR; - // store pointer in sb0 - replace this with a releaseSubblock call - rc = releaseSubblock( cb, &blockZero, freemgrType, &emptyEntry ); - - if (rc != NO_ERROR) - { - printf("Error releasing sb\n"); - return rc; - } - } - } - - /** - * the next algorithm uses the release sub-block method and does not layout the map on particular boundaries. - * It also allows pieces to be allocated - **/ - - - } - else if (initType == 2) - { - /** The following calls to init accept FBO not LBID.. - * This makes it easier to work on a range of blocks - **/ - // use the first block of the new range for sub-block chain - // and the rest for block chain - rc = init( cb, &blockZero, freemgrType, ENTRY_32, 1, 50); - - if ( rc != NO_ERROR ) - { - return rc; - } - - rc = init( cb, &blockZero, freemgrType, ENTRY_BLK, 51, numBlocks - 52); - - if ( rc != NO_ERROR ) - { - return rc; - } - - } - - // now write sb0 back to disk - if ( isDebug( DEBUG_2 )) - { - printf("Writing SB0 back to disk\n"); - printMemSubBlock( &blockZero, 0 ); - } - - uint64_t lbid = mapLBID( cb, 0, rc); - - if (rc != NO_ERROR ) - { + if (rc != NO_ERROR) + { + printf("DBG: Init 2: Error resolving LBID for OID: %u FBO: %u\n", cb.file.oid, blkIdx - 1); return rc; + } + + if (isDebug(DEBUG_3)) + { + cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; + } + + for (sbIdx = BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK - 1; sbIdx > -1; sbIdx--) + { + if (isDebug(DEBUG_3)) + { + printf("DBG: Working on subblock %u\n", sbIdx); + } + + emptyEntry.sbid = sbIdx; + emptyEntry.type = EMPTY_PTR; + rc = releaseSubblock(cb, blockZero, freemgrType, &emptyEntry); + + if (rc != NO_ERROR) + { + printf("Error releasing sb\n"); + return rc; + } + } + } + } + else + { + if (isDebug(DEBUG_1)) + { + printf("DBG: Invalid segment size %i (should be ENTRY_32 or ENTRY_BLK)\n", chainType); } - rc = writeDBFile( cb, blockZero.data, lbid ); + return ERR_INVALID_PARAM; + } + + // now write sb0 back to disk + uint64_t lbid = mapLBID(cb, 0, rc); + + if (rc != NO_ERROR) + { + return rc; + } + + rc = writeDBFile(cb, blockZero, lbid); + + if (rc != NO_ERROR) + { + return rc; + } + + return NO_ERROR; +} + +/*********************************************************** + * DESCRIPTION: + * Initializes free space lists in an new index file + * PARAMETERS: + * indexFile - file pointer for new index file + * freemgr_type - type of free mgr to init + * RETURN: + * NO_ERROR if success + * + ***********************************************************/ + +const int FreeMgr::init(CommBlock& cb, int freemgrType) +{ + /** + * Tree has 6 chains, (1,2,4,8,16,32 entries) starting at entry 1 in sb0 of block0 + * List has 3 chains (4,32,1024 entries) starting at first entry of file + * + * Starting at Block 1, add each subblock after sb0 to the free list in sb0 + * then go back and build the smaller chains on demand. This way we make sure all SBs + * are in a chain somewhere.. but this is not best behaviour when allocating blocks + * + **/ + + int rc; // return code from file ops + DataBlock blockZero; + DataBlock workBlock; + int sbIdx; + uint64_t blkIdx; + IdxEmptyListEntry emptyEntry; // populate the chains with pointer to empty entries + IdxEmptyListEntry nextPointer; // pointer at end of sub-block of pointers to emptyEntries + uint64_t numBlocks; + FILE* indexFile; + + indexFile = cb.file.pFile; + + if (!indexFile) // Make sure that we have non-null filehandle + { + // printf("DBG: File handle is null\n"); + return ERR_INVALID_PARAM; + } + + if (freemgrType != TREE && freemgrType != LIST) + { + // printf("DBG: Bad type in freeMgr Init\n"); + return ERR_INVALID_PARAM; + } + + numBlocks = getFileSize(indexFile) / BYTE_PER_BLOCK; + + // printf ("DBG: File size: %lu Total blocks: %llu (%u)\n", getFileSize( indexFile ), numBlocks, + // BYTE_PER_BLOCK); printf ("DBG: Adding sub-blocks: %llu \n", numBlocks * 32); + + // Clear the list of pointers in sb0 + initBlockzero(&blockZero); + + // initialize the non varying fields + nullPtr(&emptyEntry); + emptyEntry.type = EMPTY_PTR; + emptyEntry.group = ENTRY_32; + + // nextPointer identifies next sub-block with empty pointers + // initially there is no next sub-block so zero it out + nullPtr(&nextPointer); + nextPointer.type = EMPTY_LIST; + nextPointer.group = ENTRY_32; + nextPointer.entry = ENTRY_PER_SUBBLOCK - 1; // last entry on the list.. think of the list as a stack + + if (initType == 0) + { + if (isDebug(DEBUG_3)) + { + printf("\nOld style init\n"); + } + + for (blkIdx = numBlocks - 1; blkIdx > 0; blkIdx--) + { + emptyEntry.fbo = blkIdx; // map fbo to lbid before storage + // emptyEntry.fbo = mapLBID( cb, blkIdx, rc );// map fbo to lbid before storage + // if (rc != NO_ERROR ){ printf("DBG: Error resolving LBID for OID: %u FBO: %llu\n", + // cb.file.oid, blkIdx-1 ); return rc; } + + if (isDebug(DEBUG_3)) + { + cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; + } + + memset(workBlock.data, 0, sizeof(workBlock.data)); + + /** + * each block after zeroth block uses sb0 to store entry list + * first entry (#0) is the llp to additional sblks with entries + * entries 1-31 are pointers + **/ + + // sb0 is used for initial map, so start at sb1 + for (sbIdx = 1; sbIdx < BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK; sbIdx++) + { + if (isDebug(DEBUG_3)) + { + printf("DBG: Working on subblock %u\n", sbIdx); + } + + emptyEntry.sbid = sbIdx; + emptyEntry.type = EMPTY_PTR; + // store pointer in sb0 - replace this with a releaseSubblock call + setSubBlockEntry(workBlock.data, 0, sbIdx, 8, &emptyEntry); + + if (isDebug(DEBUG_2)) + { + cout << "DBG: Init sb fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " + << emptyEntry.entry << "\n"; + } + } + + // having stored all pointers, store linkptr + setSubBlockEntry(workBlock.data, 0, 0, 8, &nextPointer); + // nextPointer.fbo = mapLBID( cb, blkIdx, rc ); // remember this block ID + nextPointer.fbo = blkIdx; // remember this block ID + // if (rc != NO_ERROR ){ printf("DBG: Error resolving LBID for OID: %u FBO: %llu\n", + // cb.file.oid, blkIdx-1 ); return rc; } + + rc = writeDBFile(cb, workBlock.data, emptyEntry.fbo); + + if (rc != NO_ERROR) + { + return rc; + } + } + + // chain for segments of sub block size + setSubBlockEntry(blockZero.data, 0, calcPtrOffset(ENTRY_32), 8, &nextPointer); + + /** + * the next algorithm uses the release sub-block method and does not layout the map on particular + *boundaries. + **/ + } + else if (initType == 1) // new style - use release + { + if (isDebug(DEBUG_3)) + { + printf("\nNew style\n"); + } + + for (blkIdx = numBlocks - 1; blkIdx > 0; blkIdx--) + { + emptyEntry.fbo = blkIdx; // map fbo to lbid before storage + + // emptyEntry.fbo = mapLBID( cb, blkIdx, rc );// map fbo to lbid before storage + // if (rc != NO_ERROR ){ printf("DBG: Error resolving LBID for OID: %u FBO: %llu\n", + // cb.file.oid, blkIdx-1 ); return rc; } + + if (isDebug(DEBUG_3)) + { + cout << "DBG: Working on block " << emptyEntry.fbo << "\n"; + } + + memset(workBlock.data, 0, sizeof(workBlock.data)); + + for (sbIdx = BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK - 1; sbIdx > -1; sbIdx--) + { + if (isDebug(DEBUG_3)) + { + printf("DBG: Working on subblock %u\n", sbIdx); + } + + emptyEntry.sbid = sbIdx; + emptyEntry.type = EMPTY_PTR; + // store pointer in sb0 - replace this with a releaseSubblock call + rc = releaseSubblock(cb, &blockZero, freemgrType, &emptyEntry); + + if (rc != NO_ERROR) + { + printf("Error releasing sb\n"); + return rc; + } + } + } + + /** + * the next algorithm uses the release sub-block method and does not layout the map on particular + *boundaries. It also allows pieces to be allocated + **/ + } + else if (initType == 2) + { + /** The following calls to init accept FBO not LBID.. + * This makes it easier to work on a range of blocks + **/ + // use the first block of the new range for sub-block chain + // and the rest for block chain + rc = init(cb, &blockZero, freemgrType, ENTRY_32, 1, 50); if (rc != NO_ERROR) { - return rc; + return rc; } - return NO_ERROR; + rc = init(cb, &blockZero, freemgrType, ENTRY_BLK, 51, numBlocks - 52); + if (rc != NO_ERROR) + { + return rc; + } + } + + // now write sb0 back to disk + if (isDebug(DEBUG_2)) + { + printf("Writing SB0 back to disk\n"); + printMemSubBlock(&blockZero, 0); + } + + uint64_t lbid = mapLBID(cb, 0, rc); + + if (rc != NO_ERROR) + { + return rc; + } + + rc = writeDBFile(cb, blockZero.data, lbid); + + if (rc != NO_ERROR) + { + return rc; + } + + return NO_ERROR; } /** @@ -474,383 +465,396 @@ const int FreeMgr::init( CommBlock& cb, int freemgrType) * error no if fail **/ -const int FreeMgr::assignSegment( CommBlock& cb, DataBlock* blockZero, const int freemgr_type, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr ) +const int FreeMgr::assignSegment(CommBlock& cb, DataBlock* blockZero, const int freemgr_type, + const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr) { - int rc; // return code from file ops - DataBlock workBlock; - int listOffset; // entry in block zero of head pointer - IdxEmptyListEntry emptyEntry; - IdxEmptyListEntry emptyPtr; - IdxEmptyListEntry emptyMap; - uint64_t numBlocks; - IdxEmptyListEntry newSb; - FILE* indexFile; + int rc; // return code from file ops + DataBlock workBlock; + int listOffset; // entry in block zero of head pointer + IdxEmptyListEntry emptyEntry; + IdxEmptyListEntry emptyPtr; + IdxEmptyListEntry emptyMap; + uint64_t numBlocks; + IdxEmptyListEntry newSb; + FILE* indexFile; - indexFile = cb.file.pFile; + indexFile = cb.file.pFile; - if ( isDebug( DEBUG_3 )) + if (isDebug(DEBUG_3)) + { + printf("DBG: Assign ENTRY_%i segment in %s\n", 1 << (unsigned int)segmentType, + (freemgr_type == LIST) ? "LIST" : "TREE"); + } + + /* Check all input parameters are ok */ + if (!blockZero) + { + if (isDebug(DEBUG_1)) { - printf("DBG: Assign ENTRY_%i segment in %s\n", 1 << (unsigned int)segmentType, (freemgr_type == LIST) ? "LIST" : "TREE"); + printf("DBG: Bad pointer: blockZero is zero\n"); } - /* Check all input parameters are ok */ - if (!blockZero) - { - if ( isDebug( DEBUG_1 )) - { - printf ("DBG: Bad pointer: blockZero is zero\n"); - } + return ERR_INVALID_PARAM; + } - return ERR_INVALID_PARAM; + if (!assignPtr) + { + if (isDebug(DEBUG_1)) + { + printf("DBG: Bad pointer: assignPtr is zero\n"); } - if (!assignPtr) - { - if ( isDebug( DEBUG_1 )) - { - printf ("DBG: Bad pointer: assignPtr is zero\n"); - } + return ERR_INVALID_PARAM; + } - return ERR_INVALID_PARAM; + if (freemgr_type != TREE && freemgr_type != LIST) + { + if (isDebug(DEBUG_0)) + { + printf("DBG: assignSegment: Must be TREE or LIST\n"); } - if (freemgr_type != TREE && freemgr_type != LIST) - { - if ( isDebug( DEBUG_0 )) - { - printf ("DBG: assignSegment: Must be TREE or LIST\n"); - } + return ERR_INVALID_PARAM; + } - return ERR_INVALID_PARAM; - } + numBlocks = getFileSize(indexFile) / BYTE_PER_BLOCK; - numBlocks = getFileSize( indexFile ) / BYTE_PER_BLOCK ; - - //find the start of the chain - if (segmentType == ENTRY_32) - { - rc = assignSubblock( cb, blockZero, freemgr_type, assignPtr); - - if (rc != NO_ERROR) - { - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Error assigning sb (rc=%i)\n", rc); - } - } - - return rc; - } - - listOffset = calcPtrOffset( segmentType ); - - if (freemgr_type == LIST && !(segmentType == ENTRY_4 || segmentType == ENTRY_BLK)) - { - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Assign segment size illegal %i\n", (unsigned int)segmentType); - } - - return ERR_INVALID_PARAM; // should not have got here so quit with error - } - - // read Empty Map in sb0 - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Empty map ENTRY_" << (1 << segmentType) << " was fbo " << emptyPtr.fbo << " sbid " << (unsigned int)emptyPtr.sbid << " entry " << (unsigned int)emptyPtr.entry << "\n"; - } - - /* if (emptyPtr.fbo > numBlocks) - { - printf("DBG: Weirdness in assignSegment.. emptyPtr.fbo > numBlocks\n"); - return ERR_FM_BAD_FBO; - }*/ - - // check to see if queue has been built - // if not then assign a container block, add LLP in entry 0 pointing to nothing - if (emptyPtr.fbo == 0) - { - if ( isDebug( DEBUG_3 )) - { - printf("DBG: Need to add sb to chain and entries to sb\n"); - } - - // cannot assign more space to block list from a smaller list.. need to extend the file - if ( segmentType == ENTRY_BLK ) - { - if ( isDebug( DEBUG_1 )) - { - printf("Out of space in BLOCK list, quitting\n"); - } - - rc = extendFreespace( cb, blockZero, freemgr_type ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_3 )) - { - printf("DBG: Error extending file\n"); - } - - return rc; - } - } - - rc = assignSubblock( cb, blockZero, freemgr_type, &emptyPtr); - - if (rc != NO_ERROR) - { - printf("DBG: Error extending chain\n"); - return rc; - } - - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); - - if (rc != NO_ERROR) /*printf("DBG: Error reading newly allocated sb\n");*/ - { - return rc; - } - - // update map to point to new bucket - setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr); - - nullPtr(&emptyEntry); - emptyEntry.type = EMPTY_LIST; - emptyEntry.group = segmentType; - - setSubBlockEntry( workBlock.data, emptyPtr.sbid, 0, 8, &emptyEntry); // store head ptr - - rc = writeDBFile( cb, workBlock.data, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - return rc; - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Added fbo " << emptyPtr.fbo << " sbid " << (unsigned int)emptyPtr.sbid << " as bucket to chain ENTRY_" << (1 << segmentType) << "in" << ((freemgr_type == LIST) ? "LIST" : "TREE") << "\n"; - } - } - - // follow the chain to the head container - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); + // find the start of the chain + if (segmentType == ENTRY_32) + { + rc = assignSubblock(cb, blockZero, freemgr_type, assignPtr); if (rc != NO_ERROR) { - if ( isDebug( DEBUG_1 )) + if (isDebug(DEBUG_1)) + { + printf("DBG: Error assigning sb (rc=%i)\n", rc); + } + } + + return rc; + } + + listOffset = calcPtrOffset(segmentType); + + if (freemgr_type == LIST && !(segmentType == ENTRY_4 || segmentType == ENTRY_BLK)) + { + if (isDebug(DEBUG_1)) + { + printf("DBG: Assign segment size illegal %i\n", (unsigned int)segmentType); + } + + return ERR_INVALID_PARAM; // should not have got here so quit with error + } + + // read Empty Map in sb0 + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + + if (isDebug(DEBUG_2)) + { + cout << "DBG: Empty map ENTRY_" << (1 << segmentType) << " was fbo " << emptyPtr.fbo << " sbid " + << (unsigned int)emptyPtr.sbid << " entry " << (unsigned int)emptyPtr.entry << "\n"; + } + + /* if (emptyPtr.fbo > numBlocks) + { + printf("DBG: Weirdness in assignSegment.. emptyPtr.fbo > numBlocks\n"); + return ERR_FM_BAD_FBO; + }*/ + + // check to see if queue has been built + // if not then assign a container block, add LLP in entry 0 pointing to nothing + if (emptyPtr.fbo == 0) + { + if (isDebug(DEBUG_3)) + { + printf("DBG: Need to add sb to chain and entries to sb\n"); + } + + // cannot assign more space to block list from a smaller list.. need to extend the file + if (segmentType == ENTRY_BLK) + { + if (isDebug(DEBUG_1)) + { + printf("Out of space in BLOCK list, quitting\n"); + } + + rc = extendFreespace(cb, blockZero, freemgr_type); + + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_3)) { - cout << "DBG: Error reading block (" << emptyPtr.fbo << ") during segmentAssign: rc is " << rc; + printf("DBG: Error extending file\n"); } return rc; + } } - getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); + rc = assignSubblock(cb, blockZero, freemgr_type, &emptyPtr); - if ((emptyEntry.type != EMPTY_LIST) && (emptyEntry.type != EMPTY_PTR)) + if (rc != NO_ERROR) { - if ( isDebug( DEBUG_0 )) - { - printf("WTF: Bad entry in ENTRY_%i chain - type is %i (expected %i or %i)\n", 1 << segmentType, (unsigned int)emptyEntry.type, EMPTY_PTR, EMPTY_LIST ); - printMemSubBlock( &workBlock, emptyPtr.sbid ); - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: fbo " << emptyEntry.fbo << " sbid " << (unsigned int)emptyEntry.sbid << " entry " << (unsigned int)emptyEntry.entry << " chain ENTRY_" << (1 << segmentType) << "in" << ((freemgr_type == LIST) ? "LIST" : "TREE") << "\n"; - } - - return ERR_FM_BAD_TYPE; + printf("DBG: Error extending chain\n"); + return rc; } - if ((emptyEntry.fbo == 0) && (emptyEntry.type == EMPTY_PTR)) + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) /*printf("DBG: Error reading newly allocated sb\n");*/ { - if ( isDebug( DEBUG_0 )) - { - printf("DBG: Bad entry in %i list - found EMPTY_PTR but indicates block 0\n", 1 << segmentType ); - } - - return ERR_FM_BAD_TYPE; + return rc; } - if ((emptyEntry.fbo == 0) && (emptyEntry.type == EMPTY_LIST)) + // update map to point to new bucket + setSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + + nullPtr(&emptyEntry); + emptyEntry.type = EMPTY_LIST; + emptyEntry.group = segmentType; + + setSubBlockEntry(workBlock.data, emptyPtr.sbid, 0, 8, &emptyEntry); // store head ptr + + rc = writeDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) { - /** - * if at the end of the rainbow, in a bucket with no entries, fill the bucket - * Allocate a sub block and split it into parts - **/ - - // cannot assign more space to block list from a smaller list.. need to extend the file - if ( segmentType == ENTRY_BLK ) - { - if ( isDebug( DEBUG_1 )) - { - printf("\nNeed to extend block\n"); - } - - if ( isDebug( DEBUG_1 )) - { - printf("Out of space in BLOCK list\n"); - } - - rc = extendFreespace( cb, blockZero, freemgr_type ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_3 )) - { - printf("DBG: Error extending file\n"); - } - - return rc; - } - - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); - getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); - - } - else - { - - rc = assignSubblock( cb, blockZero, freemgr_type, &newSb); - - if (rc != NO_ERROR) - { -// printf("DBG: Error extending chain\n"); - return rc; - } - - if (newSb.entry != 0) - { - printf("WTF: Entry should be 0 after assign from sb list, instead is %i", (unsigned int)newSb.entry); - return ERR_FM_ASSIGN_ERR; - } - - if (isDebug(DEBUG_2)) - { - cout << "DBG: added fbo " << newSb.fbo << " sbid " << (unsigned int)newSb.sbid << " entry " << (unsigned) newSb.entry << " to " << segmentType << " list - need to split " << (1 << (ENTRY_32 - segmentType)) << "times\n"; - } - - newSb.entry = 0; - newSb.group = segmentType; - newSb.type = EMPTY_PTR; - emptyEntry = newSb; - - int idx, inc; - inc = 1 << segmentType; - - for (idx = 0; idx < ENTRY_PER_SUBBLOCK - inc; idx += inc) - { - if ( isDebug( DEBUG_3 )) - { - printf ("DBG: split..%i-%i\n", idx, idx + inc - 1 ); - } - - newSb.entry = idx; - releaseSegment( cb, blockZero, freemgr_type, segmentType, &newSb ); - } - - emptyEntry.entry = idx; - - if ( isDebug( DEBUG_3 )) - { - printf ("DBG: split and return..%i-%i\n", idx, idx + inc - 1); - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Assigned fbo " << emptyEntry.fbo << " sbid " << (unsigned int)emptyEntry.sbid << " entry " << (unsigned int)emptyEntry.entry << " to chain ENTRY_" << (1 << segmentType) << "\n"; - } - - memcpy(assignPtr, &emptyEntry, 8); - - uint64_t count; - getSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - count--; - setSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - - return NO_ERROR; - } + return rc; } + if (isDebug(DEBUG_2)) + { + cout << "DBG: Added fbo " << emptyPtr.fbo << " sbid " << (unsigned int)emptyPtr.sbid + << " as bucket to chain ENTRY_" << (1 << segmentType) << "in" + << ((freemgr_type == LIST) ? "LIST" : "TREE") << "\n"; + } + } + + // follow the chain to the head container + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_1)) + { + cout << "DBG: Error reading block (" << emptyPtr.fbo << ") during segmentAssign: rc is " << rc; + } + + return rc; + } + + getSubBlockEntry(&workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); + + if ((emptyEntry.type != EMPTY_LIST) && (emptyEntry.type != EMPTY_PTR)) + { + if (isDebug(DEBUG_0)) + { + printf("WTF: Bad entry in ENTRY_%i chain - type is %i (expected %i or %i)\n", 1 << segmentType, + (unsigned int)emptyEntry.type, EMPTY_PTR, EMPTY_LIST); + printMemSubBlock(&workBlock, emptyPtr.sbid); + } + + if (isDebug(DEBUG_2)) + { + cout << "DBG: fbo " << emptyEntry.fbo << " sbid " << (unsigned int)emptyEntry.sbid << " entry " + << (unsigned int)emptyEntry.entry << " chain ENTRY_" << (1 << segmentType) << "in" + << ((freemgr_type == LIST) ? "LIST" : "TREE") << "\n"; + } + + return ERR_FM_BAD_TYPE; + } + + if ((emptyEntry.fbo == 0) && (emptyEntry.type == EMPTY_PTR)) + { + if (isDebug(DEBUG_0)) + { + printf("DBG: Bad entry in %i list - found EMPTY_PTR but indicates block 0\n", 1 << segmentType); + } + + return ERR_FM_BAD_TYPE; + } + + if ((emptyEntry.fbo == 0) && (emptyEntry.type == EMPTY_LIST)) + { /** - * got here because we didn't need to populate a chain and did not fall into any traps - * so either we are at the end of bucket or we have a valid entry + * if at the end of the rainbow, in a bucket with no entries, fill the bucket + * Allocate a sub block and split it into parts **/ - if (emptyEntry.type == EMPTY_LIST) // reached end of this segment (should release it for re-use) + + // cannot assign more space to block list from a smaller list.. need to extend the file + if (segmentType == ENTRY_BLK) { - /** - * release bucket - **/ - if ( isDebug( DEBUG_2 )) + if (isDebug(DEBUG_1)) + { + printf("\nNeed to extend block\n"); + } + + if (isDebug(DEBUG_1)) + { + printf("Out of space in BLOCK list\n"); + } + + rc = extendFreespace(cb, blockZero, freemgr_type); + + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_3)) { - cout << "DBG: Need to release sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " from chain ENTRY_" << (1 << segmentType) << " in " << ((freemgr_type == LIST) ? "LIST" : "TREE") << "\n"; + printf("DBG: Error extending file\n"); } - // when we stored the ptr in the empty map, we tweaked group, must change it back - emptyPtr.type = EMPTY_PTR; - emptyPtr.group = ENTRY_32; - rc = releaseSubblock( cb, blockZero, freemgr_type, &emptyPtr ); + return rc; + } - if (rc != NO_ERROR) + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + getSubBlockEntry(&workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); + } + else + { + rc = assignSubblock(cb, blockZero, freemgr_type, &newSb); + + if (rc != NO_ERROR) + { + // printf("DBG: Error extending chain\n"); + return rc; + } + + if (newSb.entry != 0) + { + printf("WTF: Entry should be 0 after assign from sb list, instead is %i", (unsigned int)newSb.entry); + return ERR_FM_ASSIGN_ERR; + } + + if (isDebug(DEBUG_2)) + { + cout << "DBG: added fbo " << newSb.fbo << " sbid " << (unsigned int)newSb.sbid << " entry " + << (unsigned)newSb.entry << " to " << segmentType << " list - need to split " + << (1 << (ENTRY_32 - segmentType)) << "times\n"; + } + + newSb.entry = 0; + newSb.group = segmentType; + newSb.type = EMPTY_PTR; + emptyEntry = newSb; + + int idx, inc; + inc = 1 << segmentType; + + for (idx = 0; idx < ENTRY_PER_SUBBLOCK - inc; idx += inc) + { + if (isDebug(DEBUG_3)) { - printf("Error releasing sb\n"); - return rc; + printf("DBG: split..%i-%i\n", idx, idx + inc - 1); } - emptyPtr = emptyEntry; - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); + newSb.entry = idx; + releaseSegment(cb, blockZero, freemgr_type, segmentType, &newSb); + } - if (rc != NO_ERROR) - { - printf("DBG: Error following chain\n"); - return rc; - } + emptyEntry.entry = idx; - getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); + if (isDebug(DEBUG_3)) + { + printf("DBG: split and return..%i-%i\n", idx, idx + inc - 1); + } + if (isDebug(DEBUG_2)) + { + cout << "DBG: Assigned fbo " << emptyEntry.fbo << " sbid " << (unsigned int)emptyEntry.sbid + << " entry " << (unsigned int)emptyEntry.entry << " to chain ENTRY_" << (1 << segmentType) + << "\n"; + } + + memcpy(assignPtr, &emptyEntry, 8); + + uint64_t count; + getSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); + count--; + setSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); + + return NO_ERROR; + } + } + + /** + * got here because we didn't need to populate a chain and did not fall into any traps + * so either we are at the end of bucket or we have a valid entry + **/ + if (emptyEntry.type == EMPTY_LIST) // reached end of this segment (should release it for re-use) + { + /** + * release bucket + **/ + if (isDebug(DEBUG_2)) + { + cout << "DBG: Need to release sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid + << " from chain ENTRY_" << (1 << segmentType) << " in " + << ((freemgr_type == LIST) ? "LIST" : "TREE") << "\n"; } - if (emptyEntry.type == EMPTY_PTR) + // when we stored the ptr in the empty map, we tweaked group, must change it back + emptyPtr.type = EMPTY_PTR; + emptyPtr.group = ENTRY_32; + rc = releaseSubblock(cb, blockZero, freemgr_type, &emptyPtr); + + if (rc != NO_ERROR) { - - emptyPtr.entry--; - blockZero->dirty = 1; - setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - - if ( isDebug( DEBUG_3 )) - { - printf("DBG: Empty entry is now %u\n", (unsigned int)emptyPtr.entry); - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Assigned fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry << " from chain ENTRY_" << (1 << segmentType) << "\n"; - cout << "DBG: Empty map ENTRY_" << (1 << segmentType) << " now fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - memcpy(assignPtr, &emptyEntry, 8); - - // -- workblock may have changed on disk since it was read.. making the writeDBfile dangerous without a readDBfile first - nullPtr(&emptyMap); // zero out the entry - readDBFile( cb, &workBlock, emptyPtr.fbo ); - setSubBlockEntry( workBlock.data, emptyPtr.sbid, emptyPtr.entry + 1, 8, &emptyMap ); - rc = writeDBFile( cb, &workBlock, emptyPtr.fbo ); - // -- - - uint64_t count; - getSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - count--; - setSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - - return NO_ERROR; + printf("Error releasing sb\n"); + return rc; } - return ERR_FM_ASSIGN_ERR; + emptyPtr = emptyEntry; + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) + { + printf("DBG: Error following chain\n"); + return rc; + } + + getSubBlockEntry(&workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); + } + + if (emptyEntry.type == EMPTY_PTR) + { + emptyPtr.entry--; + blockZero->dirty = 1; + setSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + + if (isDebug(DEBUG_3)) + { + printf("DBG: Empty entry is now %u\n", (unsigned int)emptyPtr.entry); + } + + if (isDebug(DEBUG_2)) + { + cout << "DBG: Assigned fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " + << emptyEntry.entry << " from chain ENTRY_" << (1 << segmentType) << "\n"; + cout << "DBG: Empty map ENTRY_" << (1 << segmentType) << " now fbo " << emptyPtr.fbo << " sbid " + << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; + } + + memcpy(assignPtr, &emptyEntry, 8); + + // -- workblock may have changed on disk since it was read.. making the writeDBfile dangerous without a + // readDBfile first + nullPtr(&emptyMap); // zero out the entry + readDBFile(cb, &workBlock, emptyPtr.fbo); + setSubBlockEntry(workBlock.data, emptyPtr.sbid, emptyPtr.entry + 1, 8, &emptyMap); + rc = writeDBFile(cb, &workBlock, emptyPtr.fbo); + // -- + + uint64_t count; + getSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); + count--; + setSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); + + return NO_ERROR; + } + + return ERR_FM_ASSIGN_ERR; } /** @@ -867,224 +871,232 @@ const int FreeMgr::assignSegment( CommBlock& cb, DataBlock* blockZero, const int * error no if fail **/ -const int FreeMgr::assignSubblock( CommBlock& cb, DataBlock* blockZero, const int freemgrType, IdxEmptyListEntry* assignPtr ) +const int FreeMgr::assignSubblock(CommBlock& cb, DataBlock* blockZero, const int freemgrType, + IdxEmptyListEntry* assignPtr) { - int rc; // return code from file ops - DataBlock workBlock, tempBlock; - int listOffset; // entry in block zero of head pointer - IdxEmptyListEntry emptyEntry; - IdxEmptyListEntry emptyPtr, emptyMap; - IdxEmptyListEntry newBlock; - uint64_t numBlocks; - FILE* indexFile; + int rc; // return code from file ops + DataBlock workBlock, tempBlock; + int listOffset; // entry in block zero of head pointer + IdxEmptyListEntry emptyEntry; + IdxEmptyListEntry emptyPtr, emptyMap; + IdxEmptyListEntry newBlock; + uint64_t numBlocks; + FILE* indexFile; - indexFile = cb.file.pFile; + indexFile = cb.file.pFile; - /** - * Separated subblock assignment out from general segment assignment - * Reduces the hoops to jump through - **/ + /** + * Separated subblock assignment out from general segment assignment + * Reduces the hoops to jump through + **/ - numBlocks = getFileSize( indexFile ) / BYTE_PER_BLOCK ; + numBlocks = getFileSize(indexFile) / BYTE_PER_BLOCK; - if ( isDebug( DEBUG_3 )) + if (isDebug(DEBUG_3)) + { + printf("DBG: Assign subblock \n"); + } + + // find the start of the chain + listOffset = calcPtrOffset(ENTRY_32); + + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + + if (isDebug(DEBUG_2)) + { + cout << "DBG: EM (start assign) sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " + << emptyPtr.entry << "\n"; + } + + if (emptyPtr.type != EMPTY_LIST) + { + return ERR_FM_BAD_TYPE; + } + + // if (emptyPtr.fbo > numBlocks) + // { + // if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in assignSubblock.. emptyPtr.fbo > + // numBlocks\n"); } return ERR_FM_BAD_FBO; + // } + + // follow the chain to the empty entry + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) + { + // printf("DBG: RC Weirdness: rc is %i", rc); + return rc; + } + + getSubBlockEntry(&workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry); + + if (isDebug(DEBUG_2)) + { + cout << "DBG: Next avail sb fbo " << emptyEntry.fbo << " sbid " + << " entry " << emptyEntry.entry << "\n"; + } + + if (emptyEntry.fbo == 0) // then nowhere to go.. exit + { + // if( isDebug( DEBUG_1 )) { printf("DBG: No space in subblock list\n"); } + if (isDebug(DEBUG_2)) { - printf("DBG: Assign subblock \n"); + cout << "DBG: fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry + << "\n"; } - //find the start of the chain - listOffset = calcPtrOffset( ENTRY_32 ); + //-- try and assign from BLOCK list + // printf("Go ask for a block\n"); + rc = assignSegment(cb, blockZero, freemgrType, ENTRY_BLK, &newBlock); - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: EM (start assign) sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - if (emptyPtr.type != EMPTY_LIST ) - { - return ERR_FM_BAD_TYPE; - } - -// if (emptyPtr.fbo > numBlocks) -// { -// if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in assignSubblock.. emptyPtr.fbo > numBlocks\n"); } -// return ERR_FM_BAD_FBO; -// } - - // follow the chain to the empty entry - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); + // rc = extendFreespace( indexFile, blockZero, freemgrType ); if (rc != NO_ERROR) { -// printf("DBG: RC Weirdness: rc is %i", rc); + if (isDebug(DEBUG_1)) + { + printf("DBG: Could not get block from block list\n"); + } + + return rc; + } + + // got a block so now split it + newBlock.entry = 0; + newBlock.group = ENTRY_32; + newBlock.type = EMPTY_PTR; + emptyEntry = newBlock; + + //-- assign almost all sub blocks - keep last one + int sbIdx; + + for (sbIdx = BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK - 1; sbIdx > 0; sbIdx--) + { + if (isDebug(DEBUG_3)) + { + cout << "DBG: Working on fbo " << newBlock.fbo << " sbid " << sbIdx << "\n"; + } + + emptyEntry.sbid = sbIdx; + emptyEntry.type = EMPTY_PTR; + rc = releaseSubblock(cb, blockZero, freemgrType, &emptyEntry); + + if (rc != NO_ERROR) + { + printf("DBG: Error releasing sb\n"); return rc; + } } - getSubBlockEntry( &workBlock, emptyPtr.sbid, emptyPtr.entry, 8, &emptyEntry ); + emptyEntry.sbid = 0; + } - if ( isDebug( DEBUG_2 )) + if ((emptyEntry.type != EMPTY_LIST) && (emptyEntry.type != EMPTY_PTR)) + { + if (isDebug(DEBUG_0)) { - cout << "DBG: Next avail sb fbo " << emptyEntry.fbo << " sbid " << " entry " << emptyEntry.entry << "\n"; + printf("WTF: Bad entry in in subblock list- type is %i (expected %i or %i)\n", + (unsigned int)emptyEntry.type, EMPTY_PTR, EMPTY_LIST); } - if (emptyEntry.fbo == 0) // then nowhere to go.. exit + if (isDebug(DEBUG_2)) { - //if( isDebug( DEBUG_1 )) { printf("DBG: No space in subblock list\n"); } - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry << "\n"; - } - - //-- try and assign from BLOCK list -// printf("Go ask for a block\n"); - rc = assignSegment( cb, blockZero, freemgrType, ENTRY_BLK, &newBlock); - -// rc = extendFreespace( indexFile, blockZero, freemgrType ); - - if (rc != NO_ERROR) - { - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Could not get block from block list\n"); - } - - return rc; - } - - // got a block so now split it - newBlock.entry = 0; - newBlock.group = ENTRY_32; - newBlock.type = EMPTY_PTR; - emptyEntry = newBlock; - - //-- assign almost all sub blocks - keep last one - int sbIdx; - - for (sbIdx = BYTE_PER_BLOCK / BYTE_PER_SUBBLOCK - 1; sbIdx > 0; sbIdx--) - { - if ( isDebug( DEBUG_3 )) - { - cout << "DBG: Working on fbo " << newBlock.fbo << " sbid " << sbIdx << "\n"; - } - - emptyEntry.sbid = sbIdx; - emptyEntry.type = EMPTY_PTR; - rc = releaseSubblock( cb, blockZero, freemgrType, &emptyEntry ); - - if (rc != NO_ERROR) - { - printf("DBG: Error releasing sb\n"); - return rc; - } - } - - emptyEntry.sbid = 0; + cout << "DBG: fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry + << "\n"; } - if ((emptyEntry.type != EMPTY_LIST) && (emptyEntry.type != EMPTY_PTR)) - { - if ( isDebug( DEBUG_0 )) - { - printf("WTF: Bad entry in in subblock list- type is %i (expected %i or %i)\n", (unsigned int)emptyEntry.type, EMPTY_PTR, EMPTY_LIST); - } + return ERR_FM_BAD_TYPE; + } - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry << "\n"; - } - - return ERR_FM_BAD_TYPE; - } - - if (emptyEntry.type == EMPTY_PTR) - { - // this is what we expect normally - emptyPtr.entry--; // only decrement if we didn't just drain a bucket - setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - memcpy(assignPtr, &emptyEntry, 8); - - } - else if (emptyEntry.type == EMPTY_LIST && emptyPtr.entry == 0 ) - { - if (emptyPtr.entry > 0) printf("\nWTF!! %i\n", (unsigned int) emptyPtr.entry); - - // reached end of this bucket (should release it for re-use) - // this is not the typical case.. - if ( isDebug( DEBUG_3 )) - { - cout << "DBG: Drained bucket sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - //blank the llp - rc = readDBFile( cb, tempBlock.data, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_1 )) - { - cout << "DBG: File error during releaseSubblock, fbo/lbid: " << emptyPtr.fbo << "\n"; - } - - return rc; - } - - nullPtr(&emptyMap); // zero out the entry - - setSubBlockEntry( tempBlock.data, emptyPtr.sbid, 0, 8, &emptyMap ); - rc = writeDBFile( cb, &tempBlock, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - return rc; - } - - memcpy(assignPtr, &emptyPtr, 8); - assignPtr->type = EMPTY_PTR; - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Change head pointer to fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " << emptyEntry.entry << "\n"; - } - - setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyEntry ); - } - else - { - printf("DBG: Weirdness - not list and not ptr\n"); - } - -// printf("DBG: Assigned sb fbo %llu sbid %u entry %u\n", assignPtr->fbo, (unsigned int)assignPtr->sbid, (unsigned int)assignPtr->entry); - - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyMap ); + if (emptyEntry.type == EMPTY_PTR) + { + // this is what we expect normally + emptyPtr.entry--; // only decrement if we didn't just drain a bucket + setSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + memcpy(assignPtr, &emptyEntry, 8); + } + else if (emptyEntry.type == EMPTY_LIST && emptyPtr.entry == 0) + { + if (emptyPtr.entry > 0) + printf("\nWTF!! %i\n", (unsigned int)emptyPtr.entry); + // reached end of this bucket (should release it for re-use) + // this is not the typical case.. if (isDebug(DEBUG_3)) { - cout << "DBG: EM (sb assign 1) sb fbo " << emptyMap.fbo << " sbid " << emptyMap.sbid << " entry " << emptyMap.entry << "\n"; + cout << "DBG: Drained bucket sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " + << emptyPtr.entry << "\n"; } - blockZero->dirty = 1; + // blank the llp + rc = readDBFile(cb, tempBlock.data, emptyPtr.fbo); - uint64_t count; - getSubBlockEntry( blockZero, 0, calcStatOffset(ENTRY_32), 8, &count ); - count--; - setSubBlockEntry( blockZero, 0, calcStatOffset(ENTRY_32), 8, &count ); + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_1)) + { + cout << "DBG: File error during releaseSubblock, fbo/lbid: " << emptyPtr.fbo << "\n"; + } + return rc; + } - // -- workblock may have changed on disk since it was read.. making the writeDBfile dangerous without a readDBfile first - nullPtr(&emptyMap); // zero out the entry - readDBFile( cb, &workBlock, emptyPtr.fbo ); - setSubBlockEntry( workBlock.data, emptyPtr.sbid, emptyPtr.entry + 1, 8, &emptyMap ); - rc = writeDBFile( cb, &workBlock, emptyPtr.fbo ); - // -- -// if( isDebug( DEBUG_3 )) { printf("DBG: Assign subblock -- all done\n"); } + nullPtr(&emptyMap); // zero out the entry - return NO_ERROR; + setSubBlockEntry(tempBlock.data, emptyPtr.sbid, 0, 8, &emptyMap); + rc = writeDBFile(cb, &tempBlock, emptyPtr.fbo); + if (rc != NO_ERROR) + { + return rc; + } + + memcpy(assignPtr, &emptyPtr, 8); + assignPtr->type = EMPTY_PTR; + + if (isDebug(DEBUG_2)) + { + cout << "DBG: Change head pointer to fbo " << emptyEntry.fbo << " sbid " << emptyEntry.sbid << " entry " + << emptyEntry.entry << "\n"; + } + + setSubBlockEntry(blockZero, 0, listOffset, 8, &emptyEntry); + } + else + { + printf("DBG: Weirdness - not list and not ptr\n"); + } + + // printf("DBG: Assigned sb fbo %llu sbid %u entry %u\n", assignPtr->fbo, (unsigned + // int)assignPtr->sbid, (unsigned int)assignPtr->entry); + + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyMap); + + if (isDebug(DEBUG_3)) + { + cout << "DBG: EM (sb assign 1) sb fbo " << emptyMap.fbo << " sbid " << emptyMap.sbid << " entry " + << emptyMap.entry << "\n"; + } + + blockZero->dirty = 1; + + uint64_t count; + getSubBlockEntry(blockZero, 0, calcStatOffset(ENTRY_32), 8, &count); + count--; + setSubBlockEntry(blockZero, 0, calcStatOffset(ENTRY_32), 8, &count); + + // -- workblock may have changed on disk since it was read.. making the writeDBfile dangerous without a + // readDBfile first + nullPtr(&emptyMap); // zero out the entry + readDBFile(cb, &workBlock, emptyPtr.fbo); + setSubBlockEntry(workBlock.data, emptyPtr.sbid, emptyPtr.entry + 1, 8, &emptyMap); + rc = writeDBFile(cb, &workBlock, emptyPtr.fbo); + // -- + // if( isDebug( DEBUG_3 )) { printf("DBG: Assign subblock -- all done\n"); } + + return NO_ERROR; } - /** * DESCRIPTION: * Release a segment back to the list @@ -1100,410 +1112,419 @@ const int FreeMgr::assignSubblock( CommBlock& cb, DataBlock* blockZero, const in * error no if fail **/ - -const int FreeMgr::releaseSegment( CommBlock& cb, DataBlock* blockZero, const int freemgr_type, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr ) +const int FreeMgr::releaseSegment(CommBlock& cb, DataBlock* blockZero, const int freemgr_type, + const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr) { - int rc; // return code from file ops - DataBlock workBlock; - DataBlock extraBlock; - int listOffset; // entry in block zero of head pointer - IdxEmptyListEntry emptyPtr; - IdxEmptyListEntry newSb; - uint64_t numBlocks; - FILE* indexFile; + int rc; // return code from file ops + DataBlock workBlock; + DataBlock extraBlock; + int listOffset; // entry in block zero of head pointer + IdxEmptyListEntry emptyPtr; + IdxEmptyListEntry newSb; + uint64_t numBlocks; + FILE* indexFile; - indexFile = cb.file.pFile; + indexFile = cb.file.pFile; - if (!assignPtr) + if (!assignPtr) + { + // printf ("DBG: Bad pointer: assignPtr is zero\n"); + return ERR_INVALID_PARAM; + } + + if (!blockZero) + { + // printf ("DBG: Bad pointer: blockZero is zero\n"); + return ERR_INVALID_PARAM; + } + + if (isDebug(DEBUG_3)) + { + printf("DBG: release ENTRY_%i segment \n", 1 << (unsigned int)segmentType); + } + + if (isDebug(DEBUG_2)) + { + cout << "DBG: releasing fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " + << assignPtr->entry << " from " << ((freemgr_type == LIST) ? "LIST" : "TREE") << " chain ENTRY_" + << (1 << segmentType) << " (type is " + << ((assignPtr->type == EMPTY_PTR) ? "EMPTY_PTR" : "Not EMPTY_PTR") << ")\n"; + } + + numBlocks = getFileSize(indexFile) / BYTE_PER_BLOCK; + + /* if (assignPtr->fbo > numBlocks) + { + if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in releaseSegment.. assignPtr.fbo > numBlocks (%llu + %llu)\n", assignPtr->fbo, numBlocks );}; return ERR_FM_BAD_FBO; + }*/ + + if (assignPtr->type != EMPTY_PTR) + { + // printf("DBG: Weirdness in releaseSegment.. tried to return a pointer with type %i (expected + // %i)\n", (unsigned int)assignPtr->type, EMPTY_PTR ); + assignPtr->type = EMPTY_PTR; + // return ERR_FM_BAD_TYPE; // do not exit + } + + if (assignPtr->group != (uint64_t)segmentType) + { + // printf("DBG: Weirdness in releaseSegment.. tried to return a pointer from group %i to group + // %i\n", (unsigned int)assignPtr->group, segmentType ); + return ERR_FM_RELEASE_ERR; + } + + // find the start of the chain + if (segmentType == ENTRY_32) + { + rc = releaseSubblock(cb, blockZero, freemgr_type, assignPtr); + + if (rc != NO_ERROR) { -// printf ("DBG: Bad pointer: assignPtr is zero\n"); - return ERR_INVALID_PARAM; + // printf("DBG: Error releasing sb\n"); } - if (!blockZero) + return rc; + } + + listOffset = calcPtrOffset(segmentType); + + if (freemgr_type == LIST && !(segmentType == ENTRY_4 || segmentType == ENTRY_BLK)) + { + printf("DBG: Assign segment size illegal %i\n", (unsigned int)segmentType); + return ERR_INVALID_PARAM; // should not have got here so quit with error + } + + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + + /* if (emptyPtr.fbo > numBlocks) + { + if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in releaseSegment.. emptyPtr.fbo > numBlocks (%llu + %llu)\n", emptyPtr.fbo, numBlocks );} return ERR_FM_BAD_FBO; + }*/ + + // sub block is full or chain never started + if (emptyPtr.entry == ENTRY_PER_SUBBLOCK - 1 || emptyPtr.fbo == 0) + { + if (isDebug(DEBUG_3)) { -// printf ("DBG: Bad pointer: blockZero is zero\n"); - return ERR_INVALID_PARAM; + printf("DBG: No room in chain %i - need to add a sub-block\n", (unsigned int)segmentType); } - if ( isDebug( DEBUG_3 )) + if (isDebug(DEBUG_2)) { - printf("DBG: release ENTRY_%i segment \n", 1 << (unsigned int)segmentType); + cout << "DBG: Empty ptr fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " + << emptyPtr.entry << "\n"; } - if ( isDebug( DEBUG_2 )) + // ask for a new sb to extend chain + rc = assignSubblock(cb, blockZero, freemgr_type, &newSb); + + if (rc != NO_ERROR) { - cout << "DBG: releasing fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " << assignPtr->entry << " from " << ((freemgr_type == LIST) ? "LIST" : "TREE") << " chain ENTRY_" << (1 << segmentType) << " (type is " << ((assignPtr->type == EMPTY_PTR) ? "EMPTY_PTR" : "Not EMPTY_PTR") << ")\n"; + printf("DBG: Error extending chain\n"); + return rc; } - numBlocks = getFileSize( indexFile ) / BYTE_PER_BLOCK ; - - /* if (assignPtr->fbo > numBlocks) + if (isDebug(DEBUG_2)) { - if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in releaseSegment.. assignPtr.fbo > numBlocks (%llu %llu)\n", assignPtr->fbo, numBlocks );}; - return ERR_FM_BAD_FBO; - }*/ - - if (assignPtr->type != EMPTY_PTR) - { -// printf("DBG: Weirdness in releaseSegment.. tried to return a pointer with type %i (expected %i)\n", (unsigned int)assignPtr->type, EMPTY_PTR ); - assignPtr->type = EMPTY_PTR; - // return ERR_FM_BAD_TYPE; // do not exit + cout << "DBG: release segment, new SB is fbo " << newSb.fbo << " sbid " << newSb.sbid << " entry " + << newSb.entry << "\n"; } - if ( assignPtr->group != (uint64_t)segmentType ) + rc = readDBFile(cb, extraBlock.data, newSb.fbo); + + if (rc != NO_ERROR) { -// printf("DBG: Weirdness in releaseSegment.. tried to return a pointer from group %i to group %i\n", (unsigned int)assignPtr->group, segmentType ); - return ERR_FM_RELEASE_ERR; + if (isDebug(DEBUG_1)) + { + printf("DBG: File error during releaseSegment (3)\n"); + } + + return rc; } - //find the start of the chain - if (segmentType == ENTRY_32) + emptyPtr.type = EMPTY_LIST; // writing into the LLP field so set type accordingly + setSubBlockEntry(extraBlock.data, newSb.sbid, 0, 8, &emptyPtr); + setSubBlockEntry(extraBlock.data, newSb.sbid, 1, 8, assignPtr); + rc = writeDBFile(cb, &extraBlock, newSb.fbo); + + if (rc != NO_ERROR) { - rc = releaseSubblock( cb, blockZero, freemgr_type, assignPtr); + if (isDebug(DEBUG_1)) + { + printf("DBG: File error during releaseSegment (4)\n"); + } - if (rc != NO_ERROR) - { -// printf("DBG: Error releasing sb\n"); - } - - return rc; + return rc; } - listOffset = calcPtrOffset( segmentType ); + newSb.entry = 1; + newSb.type = EMPTY_LIST; + newSb.group = segmentType; - if (freemgr_type == LIST && !(segmentType == ENTRY_4 || segmentType == ENTRY_BLK)) - { - printf("DBG: Assign segment size illegal %i\n", (unsigned int)segmentType); - return ERR_INVALID_PARAM; // should not have got here so quit with error - } - - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - - /* if (emptyPtr.fbo > numBlocks) - { - if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in releaseSegment.. emptyPtr.fbo > numBlocks (%llu %llu)\n", emptyPtr.fbo, numBlocks );} - return ERR_FM_BAD_FBO; - }*/ - - //sub block is full or chain never started - if (emptyPtr.entry == ENTRY_PER_SUBBLOCK - 1 || emptyPtr.fbo == 0) - { - if ( isDebug( DEBUG_3 )) - { - printf("DBG: No room in chain %i - need to add a sub-block\n", (unsigned int)segmentType); - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Empty ptr fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - //ask for a new sb to extend chain - rc = assignSubblock( cb, blockZero, freemgr_type, &newSb); - - if (rc != NO_ERROR) - { - printf("DBG: Error extending chain\n"); - return rc; - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: release segment, new SB is fbo " << newSb.fbo << " sbid " << newSb.sbid << " entry " << newSb.entry << "\n"; - } - - rc = readDBFile( cb, extraBlock.data, newSb.fbo ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_1 )) - { - printf("DBG: File error during releaseSegment (3)\n"); - } - - return rc; - } - - emptyPtr.type = EMPTY_LIST; // writing into the LLP field so set type accordingly - setSubBlockEntry( extraBlock.data, newSb.sbid, 0, 8, &emptyPtr ); - setSubBlockEntry( extraBlock.data, newSb.sbid, 1, 8, assignPtr ); - rc = writeDBFile( cb, &extraBlock, newSb.fbo ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_1 )) - { - printf("DBG: File error during releaseSegment (4)\n"); - } - - return rc; - } - - newSb.entry = 1; - newSb.type = EMPTY_LIST; - newSb.group = segmentType; - - setSubBlockEntry( blockZero, 0, listOffset, 8, &newSb ); - blockZero->dirty = 1; - - uint64_t count; - getSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - count++; - setSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - - - return NO_ERROR ; - - } - else - { - // - emptyPtr.entry++; - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_1 )) - { - printf("DBG: File error during releaseSegment\n"); - } - - return rc; - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Empty map ENTRY_" << ( 1 << segmentType) << " is fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - setSubBlockEntry( workBlock.data, emptyPtr.sbid, emptyPtr.entry, 8, assignPtr ); - rc = writeDBFile( cb, workBlock.data, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - return rc; - } - - emptyPtr.type = EMPTY_LIST; - emptyPtr.group = segmentType; - - setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - blockZero->dirty = 1;//sub block is full or chain never started - - } + setSubBlockEntry(blockZero, 0, listOffset, 8, &newSb); + blockZero->dirty = 1; uint64_t count; - getSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); + getSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); count++; - setSubBlockEntry( blockZero, 0, calcStatOffset(segmentType), 8, &count ); - + setSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); return NO_ERROR; + } + else + { + // + emptyPtr.entry++; + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_1)) + { + printf("DBG: File error during releaseSegment\n"); + } + + return rc; + } + + if (isDebug(DEBUG_2)) + { + cout << "DBG: Empty map ENTRY_" << (1 << segmentType) << " is fbo " << emptyPtr.fbo << " sbid " + << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; + } + + setSubBlockEntry(workBlock.data, emptyPtr.sbid, emptyPtr.entry, 8, assignPtr); + rc = writeDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) + { + return rc; + } + + emptyPtr.type = EMPTY_LIST; + emptyPtr.group = segmentType; + + setSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + blockZero->dirty = 1; // sub block is full or chain never started + } + + uint64_t count; + getSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); + count++; + setSubBlockEntry(blockZero, 0, calcStatOffset(segmentType), 8, &count); + + return NO_ERROR; } - -const int FreeMgr::releaseSubblock( CommBlock& cb, DataBlock* blockZero, const int freemgr_type, IdxEmptyListEntry* assignPtr ) +const int FreeMgr::releaseSubblock(CommBlock& cb, DataBlock* blockZero, const int freemgr_type, + IdxEmptyListEntry* assignPtr) { - int rc; // return code from file ops - DataBlock workBlock; - DataBlock extraBlock; - int listOffset; // entry in block zero of head pointer - IdxEmptyListEntry emptyPtr, emptyMap ; - uint64_t numBlocks; - FILE* indexFile; + int rc; // return code from file ops + DataBlock workBlock; + DataBlock extraBlock; + int listOffset; // entry in block zero of head pointer + IdxEmptyListEntry emptyPtr, emptyMap; + uint64_t numBlocks; + FILE* indexFile; - indexFile = cb.file.pFile; - /** - * Release sub-block - handle de-allocation of only sub-blocks - * This makes the assign/release code for smaller segments simpler and - * separates the tasks of handling the list containers and the list contents - * When called, we look at the bucket indicated as head of chain and if it full - * or not present (no room left in chain) then insert the returned SB as a bucket and - * move the head pointer - **/ + indexFile = cb.file.pFile; + /** + * Release sub-block - handle de-allocation of only sub-blocks + * This makes the assign/release code for smaller segments simpler and + * separates the tasks of handling the list containers and the list contents + * When called, we look at the bucket indicated as head of chain and if it full + * or not present (no room left in chain) then insert the returned SB as a bucket and + * move the head pointer + **/ + // if( isDebug( DEBUG_1 )) { printf("DBG: releaseSubblock\n"); } + if (isDebug(DEBUG_2)) + { + cout << "DBG: releasing sb fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " + << assignPtr->entry << " from " << ((freemgr_type == LIST) ? "LIST" : "TREE") << " (type is " + << ((assignPtr->type == EMPTY_PTR) ? "EMPTY_PTR" : "Not EMPTY_PTR") << ")\n"; + } - //if( isDebug( DEBUG_1 )) { printf("DBG: releaseSubblock\n"); } - if ( isDebug( DEBUG_2 )) + if (!assignPtr) + { + // printf ("DBG: Bad pointer: assignPtr is zero\n"); + return ERR_INVALID_PARAM; + } + + if (!blockZero) + { + printf("DBG: Bad pointer: pointer for blockZero is zero\n"); + return ERR_INVALID_PARAM; + } + + numBlocks = getFileSize(indexFile) / BYTE_PER_BLOCK; + + /* if (assignPtr->fbo > numBlocks) + { + if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in releaseSubblock.. assignPtr.fbo > numBlocks (%llu + %llu)\n", assignPtr->fbo, numBlocks );} return ERR_FM_BAD_FBO; + }*/ + if (assignPtr->type != EMPTY_PTR) + { + printf("DBG: Weirdness in releaseSubblock.. tried to return a pointer with type %i (expected %i)\n", + (unsigned int)assignPtr->type, EMPTY_PTR); + return ERR_FM_BAD_TYPE; + } + + if (assignPtr->group != ENTRY_32) + { + printf("DBG: Weirdness in releaseSubblock.. tried to return a pointer from group %i to subblock group\n", + (unsigned int)assignPtr->group); + return ERR_INVALID_PARAM; + } + + // find the start of the chain + listOffset = calcPtrOffset(ENTRY_32); + + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + + if (isDebug(DEBUG_2)) + { + cout << "DBG: EM (sb release 1) sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " + << emptyPtr.entry << "\n"; + } + + // sub block is full or chain empty + if (emptyPtr.entry == ENTRY_PER_SUBBLOCK - 1 || emptyPtr.fbo == 0) + { + // change type from EMPTY_PTR to EMPTY_LIST + assignPtr->type = EMPTY_LIST; + + // if( isDebug( DEBUG_1 )) { printf("DBG: No room in subblock chain - need to add a sub-block\n"); } + if (isDebug(DEBUG_2)) { - cout << "DBG: releasing sb fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " << assignPtr->entry << " from " << ((freemgr_type == LIST) ? "LIST" : "TREE") << " (type is " << ((assignPtr->type == EMPTY_PTR) ? "EMPTY_PTR" : "Not EMPTY_PTR") << ")\n"; + cout << "DBG: Change head pointer to fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " + << assignPtr->entry << "\n"; } - if (!assignPtr) + // change head pointer to released segment + setSubBlockEntry(blockZero, 0, listOffset, 8, assignPtr); + blockZero->dirty = 1; + + // read in released segment to set llp of new block to point to current head of chain + rc = readDBFile(cb, extraBlock.data, assignPtr->fbo); + + if (rc != NO_ERROR) { -// printf ("DBG: Bad pointer: assignPtr is zero\n"); - return ERR_INVALID_PARAM; + if (isDebug(DEBUG_1)) + { + cout << "DBG: File error during releaseSegment (2), rc: " << rc << " fbo/lbid: " << assignPtr->fbo + << "\n"; + } + + return rc; } - if (!blockZero) + if (isDebug(DEBUG_2)) { - printf ("DBG: Bad pointer: pointer for blockZero is zero\n"); - return ERR_INVALID_PARAM; + cout << "DBG: Set LLP for fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " + << assignPtr->entry << "to fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " + << emptyPtr.entry << "\n"; } - numBlocks = getFileSize( indexFile ) / BYTE_PER_BLOCK ; - - /* if (assignPtr->fbo > numBlocks) - { - if( isDebug( DEBUG_1 )) { printf("DBG: Weirdness in releaseSubblock.. assignPtr.fbo > numBlocks (%llu %llu)\n", assignPtr->fbo, numBlocks );} - return ERR_FM_BAD_FBO; - }*/ - if (assignPtr->type != EMPTY_PTR) + if (isDebug(DEBUG_3)) { - printf("DBG: Weirdness in releaseSubblock.. tried to return a pointer with type %i (expected %i)\n", (unsigned int)assignPtr->type, EMPTY_PTR ); - return ERR_FM_BAD_TYPE; + printf("Before\n"); + printMemSubBlock(&extraBlock, assignPtr->sbid); } - if ( assignPtr->group != ENTRY_32 ) + emptyPtr.type = EMPTY_LIST; + // memset( extraBlock.data, 0, BYTE_PER_SUBBLOCK); + setSubBlockEntry(extraBlock.data, assignPtr->sbid, 0, 8, &emptyPtr); + rc = writeDBFile(cb, &extraBlock, assignPtr->fbo); + + if (rc != NO_ERROR) { - printf("DBG: Weirdness in releaseSubblock.. tried to return a pointer from group %i to subblock group\n", (unsigned int)assignPtr->group ); - return ERR_INVALID_PARAM; + return rc; } - //find the start of the chain - listOffset = calcPtrOffset( ENTRY_32 ); - - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - - if ( isDebug( DEBUG_2 )) + if (isDebug(DEBUG_2)) { - cout << "DBG: EM (sb release 1) sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; + getSubBlockEntry(blockZero, 0, listOffset, 8, &emptyMap); + cout << "DBG: EM (sb release 2) sb fbo " << emptyMap.fbo << " sbid " << emptyMap.sbid << " entry " + << emptyMap.entry << "\n"; } - //sub block is full or chain empty - if (emptyPtr.entry == ENTRY_PER_SUBBLOCK - 1 || emptyPtr.fbo == 0) + if (isDebug(DEBUG_3)) { - // change type from EMPTY_PTR to EMPTY_LIST - assignPtr->type = EMPTY_LIST; - - //if( isDebug( DEBUG_1 )) { printf("DBG: No room in subblock chain - need to add a sub-block\n"); } - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Change head pointer to fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " << assignPtr->entry << "\n"; - } - - // change head pointer to released segment - setSubBlockEntry( blockZero, 0, listOffset, 8, assignPtr ); - blockZero->dirty = 1; - - // read in released segment to set llp of new block to point to current head of chain - rc = readDBFile( cb, extraBlock.data, assignPtr->fbo ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_1 )) - { - cout << "DBG: File error during releaseSegment (2), rc: " << rc << " fbo/lbid: " << assignPtr->fbo << "\n"; - } - - return rc; - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: Set LLP for fbo " << assignPtr->fbo << " sbid " << assignPtr->sbid << " entry " << assignPtr->entry << "to fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - if ( isDebug( DEBUG_3)) - { - printf("Before\n"); - printMemSubBlock( &extraBlock, assignPtr->sbid ); - } - - emptyPtr.type = EMPTY_LIST; - //memset( extraBlock.data, 0, BYTE_PER_SUBBLOCK); - setSubBlockEntry( extraBlock.data, assignPtr->sbid, 0, 8, &emptyPtr ); - rc = writeDBFile( cb, &extraBlock, assignPtr->fbo ); - - if (rc != NO_ERROR) - { - return rc; - } - - if ( isDebug( DEBUG_2 )) - { - getSubBlockEntry( blockZero, 0, listOffset, 8, &emptyMap ); - cout << "DBG: EM (sb release 2) sb fbo " << emptyMap.fbo << " sbid " << emptyMap.sbid << " entry " << emptyMap.entry << "\n"; - } - - if ( isDebug( DEBUG_3)) - { - printf("After\n"); - printMemSubBlock( &extraBlock, assignPtr->sbid ); - } - + printf("After\n"); + printMemSubBlock(&extraBlock, assignPtr->sbid); } - else + } + else + { + // + emptyPtr.entry++; + rc = readDBFile(cb, workBlock.data, emptyPtr.fbo); + + if (rc != NO_ERROR) { - // - emptyPtr.entry++; - rc = readDBFile( cb, workBlock.data, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - if (isDebug( DEBUG_1 )) - { - printf("DBG: File error during releaseSubblock\n"); - } - - return rc; - } - - if ( isDebug( DEBUG_3)) - { - printf("Before\n"); - printMemSubBlock( &workBlock, emptyPtr.sbid ); - } - - setSubBlockEntry( workBlock.data, emptyPtr.sbid, emptyPtr.entry, 8, assignPtr ); - rc = writeDBFile( cb, &workBlock, emptyPtr.fbo ); - - if (rc != NO_ERROR) - { - return rc; - } - - if ( isDebug( DEBUG_2 )) - { - cout << "DBG: setting emptyPtr sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " << emptyPtr.entry << "\n"; - } - - if ( isDebug( DEBUG_3)) - { - printf("After\n"); - printMemSubBlock( &workBlock, emptyPtr.sbid ); - } - - emptyPtr.type = EMPTY_LIST; - emptyPtr.group = ENTRY_32; - - setSubBlockEntry( blockZero, 0, listOffset, 8, &emptyPtr ); - blockZero->dirty = 1; + if (isDebug(DEBUG_1)) + { + printf("DBG: File error during releaseSubblock\n"); + } + return rc; } - uint64_t count; + if (isDebug(DEBUG_3)) + { + printf("Before\n"); + printMemSubBlock(&workBlock, emptyPtr.sbid); + } - getSubBlockEntry( blockZero, 0, calcStatOffset(ENTRY_32), 8, &count ); - count++; - setSubBlockEntry( blockZero, 0, calcStatOffset(ENTRY_32), 8, &count ); + setSubBlockEntry(workBlock.data, emptyPtr.sbid, emptyPtr.entry, 8, assignPtr); + rc = writeDBFile(cb, &workBlock, emptyPtr.fbo); + if (rc != NO_ERROR) + { + return rc; + } - return NO_ERROR; + if (isDebug(DEBUG_2)) + { + cout << "DBG: setting emptyPtr sb fbo " << emptyPtr.fbo << " sbid " << emptyPtr.sbid << " entry " + << emptyPtr.entry << "\n"; + } + + if (isDebug(DEBUG_3)) + { + printf("After\n"); + printMemSubBlock(&workBlock, emptyPtr.sbid); + } + + emptyPtr.type = EMPTY_LIST; + emptyPtr.group = ENTRY_32; + + setSubBlockEntry(blockZero, 0, listOffset, 8, &emptyPtr); + blockZero->dirty = 1; + } + + uint64_t count; + + getSubBlockEntry(blockZero, 0, calcStatOffset(ENTRY_32), 8, &count); + count++; + setSubBlockEntry(blockZero, 0, calcStatOffset(ENTRY_32), 8, &count); + + return NO_ERROR; } -const void FreeMgr::nullPtr(WriteEngine::IdxEmptyListEntry* assignPtr ) const +const void FreeMgr::nullPtr(WriteEngine::IdxEmptyListEntry* assignPtr) const { - - - assignPtr->fbo = 0; - assignPtr->sbid = 0; - assignPtr->entry = 0; - assignPtr->type = 0; - assignPtr->group = 0; - assignPtr->spare = 0; - assignPtr->spare2 = 0; + assignPtr->fbo = 0; + assignPtr->sbid = 0; + assignPtr->entry = 0; + assignPtr->type = 0; + assignPtr->group = 0; + assignPtr->spare = 0; + assignPtr->spare2 = 0; } /** @@ -1515,74 +1536,72 @@ const void FreeMgr::nullPtr(WriteEngine::IdxEmptyListEntry* assignPtr ) const * @param freemgrType * @return */ -const int FreeMgr::extendFreespace( CommBlock& cb, DataBlock* blockZero, int freemgrType ) +const int FreeMgr::extendFreespace(CommBlock& cb, DataBlock* blockZero, int freemgrType) { - DataBlock workBlock; - int rc = NO_ERROR; // result code from file ops - FILE* indexFile; - int allocSize = 0; + DataBlock workBlock; + int rc = NO_ERROR; // result code from file ops + FILE* indexFile; + int allocSize = 0; - indexFile = cb.file.pFile; + indexFile = cb.file.pFile; - if ( isDebug( DEBUG_1 )) + if (isDebug(DEBUG_1)) + { + printf("Extending File\n"); + } + + int numBlocks = 1024; // default number - should ask BRM for extentsize + int currSize = getFileSize(indexFile) / BYTE_PER_BLOCK; + + if (!allowExtend) + { + if (isDebug(DEBUG_1)) { - printf ("Extending File\n"); + printf("DBG: Extension denied\n"); } - int numBlocks = 1024; // default number - should ask BRM for extentsize - int currSize = getFileSize( indexFile ) / BYTE_PER_BLOCK ; + return ERR_FM_EXTEND; + } - if (!allowExtend) - { - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Extension denied\n"); - } + if (isDebug(DEBUG_1)) + { + printf("DBG: Extending free space in file\n"); + } - return ERR_FM_EXTEND; - } - - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Extending free space in file\n"); - } - - if ( isDebug( DEBUG_1 )) - { - printf("DBG: File is currently: %i blocks Asking for: %i blocks\n", currSize, numBlocks); - } + if (isDebug(DEBUG_1)) + { + printf("DBG: File is currently: %i blocks Asking for: %i blocks\n", currSize, numBlocks); + } #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - rc = extendFile( indexFile, cb.file.oid, numBlocks, allocSize, 0, 8, false ); + rc = extendFile(indexFile, cb.file.oid, numBlocks, allocSize, 0, 8, false); #endif - if (rc != NO_ERROR) - { - return rc; - } + if (rc != NO_ERROR) + { + return rc; + } - if ( isDebug( DEBUG_1 )) - { - printf("DBG: Extended file by %i blocks\n", allocSize); - } + if (isDebug(DEBUG_1)) + { + printf("DBG: Extended file by %i blocks\n", allocSize); + } + rc = init(cb, blockZero, freemgrType, ENTRY_32, currSize, 50); - rc = init( cb, blockZero, freemgrType, ENTRY_32, currSize, 50 ); + if (rc != NO_ERROR) + { + return rc; + } - if ( rc != NO_ERROR ) - { - return rc; - } + rc = init(cb, blockZero, freemgrType, ENTRY_BLK, currSize + 50, allocSize - 50); - rc = init( cb, blockZero, freemgrType, ENTRY_BLK, currSize + 50, allocSize - 50 ); + if (rc != NO_ERROR) + { + return rc; + } - if ( rc != NO_ERROR ) - { - return rc; - } - - - return NO_ERROR; + return NO_ERROR; } /** @@ -1591,61 +1610,61 @@ const int FreeMgr::extendFreespace( CommBlock& cb, DataBlock* blockZero, int fre * @param fbo * @return */ -const uint64_t FreeMgr::mapLBID( CommBlock& cb, const uint64_t fbo, int& rc ) +const uint64_t FreeMgr::mapLBID(CommBlock& cb, const uint64_t fbo, int& rc) { - uint64_t lbid = 0; + uint64_t lbid = 0; - rc = NO_ERROR; + rc = NO_ERROR; #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - rc = BRMWrapper::getInstance()->getBrmInfo( cb.file.oid, fbo, lbid ); + rc = BRMWrapper::getInstance()->getBrmInfo(cb.file.oid, fbo, lbid); #endif - if (rc != NO_ERROR) + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_0)) { - if (isDebug( DEBUG_0 )) - { - cout << "BRM Failure looking up lbid for oid: " << cb.file.oid << " fbo: " << fbo << ", returning unchanged value\n"; - } + cout << "BRM Failure looking up lbid for oid: " << cb.file.oid << " fbo: " << fbo + << ", returning unchanged value\n"; + } - return fbo; - } - else - { - return lbid; - } + return fbo; + } + else + { + return lbid; + } } - -const void FreeMgr::printMemSubBlock( DataBlock* curBlock, const int sbid ) +const void FreeMgr::printMemSubBlock(DataBlock* curBlock, const int sbid) { - int off; -// DataBlock curBlock; - unsigned char* curPos; - IdxEmptyListEntry curEntry, testZero; + int off; + // DataBlock curBlock; + unsigned char* curPos; + IdxEmptyListEntry curEntry, testZero; - nullPtr( &testZero ); -// readDBFile( m_pTreeFile, &curBlock, fbo ); - curPos = curBlock->data + BYTE_PER_SUBBLOCK * sbid; - printf( "\n======================== sbid: %i", sbid ); + nullPtr(&testZero); + // readDBFile( m_pTreeFile, &curBlock, fbo ); + curPos = curBlock->data + BYTE_PER_SUBBLOCK * sbid; + printf("\n======================== sbid: %i", sbid); - for ( int i = 0; i < ENTRY_PER_SUBBLOCK; i++ ) - { - memcpy( &curEntry, curPos, MAX_COLUMN_BOUNDARY ); - off = memcmp( &testZero, &curEntry, MAX_COLUMN_BOUNDARY ); -// if( /*bNoZero &&*/ off == 0 ) -// continue; - printf( "\n Entry %2d : ", i ); + for (int i = 0; i < ENTRY_PER_SUBBLOCK; i++) + { + memcpy(&curEntry, curPos, MAX_COLUMN_BOUNDARY); + off = memcmp(&testZero, &curEntry, MAX_COLUMN_BOUNDARY); + // if( /*bNoZero &&*/ off == 0 ) + // continue; + printf("\n Entry %2d : ", i); - for ( int j = 0; j < MAX_COLUMN_BOUNDARY; j++ ) - printf( " %2X", *(curPos + j) ); + for (int j = 0; j < MAX_COLUMN_BOUNDARY; j++) + printf(" %2X", *(curPos + j)); - cout << " fbo=" << curEntry.fbo << " sbid=" << curEntry.sbid << " entry=" << curEntry.entry << " group=" << curEntry.group << " type=" << curEntry.type; + cout << " fbo=" << curEntry.fbo << " sbid=" << curEntry.sbid << " entry=" << curEntry.entry + << " group=" << curEntry.group << " type=" << curEntry.type; - curPos += MAX_COLUMN_BOUNDARY; - } + curPos += MAX_COLUMN_BOUNDARY; + } - printf( "\n" ); + printf("\n"); } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/index/we_freemgr.h b/writeengine/index/we_freemgr.h index 407085e62..50fbb9800 100644 --- a/writeengine/index/we_freemgr.h +++ b/writeengine/index/we_freemgr.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_freemgr.h 33 2006-07-19 17:09:27Z mthomas $ -* -******************************************************************************************/ + * $Id: we_freemgr.h 33 2006-07-19 17:09:27Z mthomas $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_FREEMGR_H_ @@ -32,93 +32,96 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class FreeMgr */ class FreeMgr : public DbFileOp { -public: - /** - * @brief Constructor - */ - FreeMgr(); + public: + /** + * @brief Constructor + */ + FreeMgr(); - /** - * @brief Default Destructor - */ - ~FreeMgr(); + /** + * @brief Default Destructor + */ + ~FreeMgr(); - /** - * @brief init free chains in a new index file - */ - const int init( CommBlock& cb, const int freemgrType ); + /** + * @brief init free chains in a new index file + */ + const int init(CommBlock& cb, const int freemgrType); - /** - * @brief Init free chains in a new file or an existing file - * The start block is an FBO or an LBID depending on whether we are using BRM - **/ - const int init( CommBlock& cb, DataBlock* blockZero, const int freemgrType, const IdxTreeGroupType chainType, const int startBlock, const int numberBlocks ); + /** + * @brief Init free chains in a new file or an existing file + * The start block is an FBO or an LBID depending on whether we are using BRM + **/ + const int init(CommBlock& cb, DataBlock* blockZero, const int freemgrType, const IdxTreeGroupType chainType, + const int startBlock, const int numberBlocks); - /** - * @brief find a free segment and return ptr - */ - const int assignSegment( CommBlock& cb, DataBlock* blockZero, const int freemgr_type, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr ) ; + /** + * @brief find a free segment and return ptr + */ + const int assignSegment(CommBlock& cb, DataBlock* blockZero, const int freemgr_type, + const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr); - /** - * @brief put a free segment back into chain - */ - const int releaseSegment( CommBlock& cb, DataBlock* blockZero, const int freemgr_type, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr ) ; + /** + * @brief put a free segment back into chain + */ + const int releaseSegment(CommBlock& cb, DataBlock* blockZero, const int freemgr_type, + const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr); - /** - * @brief Map an FBO to LBID - */ - const uint64_t mapLBID( CommBlock& cb, const uint64_t fbo, int& rc ); + /** + * @brief Map an FBO to LBID + */ + const uint64_t mapLBID(CommBlock& cb, const uint64_t fbo, int& rc); - /** - * extendFreespace - ran out of space in one of the chains? - * Add blocks via calls to BRM extent mgr and fseek the file to the end - **/ - const int extendFreespace( CommBlock& cb, DataBlock* blockZero, const int freemgr_type ); + /** + * extendFreespace - ran out of space in one of the chains? + * Add blocks via calls to BRM extent mgr and fseek the file to the end + **/ + const int extendFreespace(CommBlock& cb, DataBlock* blockZero, const int freemgr_type); + // private: -//private: + /** + * @brief Create sub block zero for use by free manager + */ + const int initBlockzero(DataBlock* blockZero); - /** - * @brief Create sub block zero for use by free manager - */ - const int initBlockzero( DataBlock* blockZero); + /** + * @brief Handle release of sub-blocks separately - sometimes a sub-block contains list entries and + * sometimes it is a list entry + */ - /** - * @brief Handle release of sub-blocks separately - sometimes a sub-block contains list entries and sometimes it is a list entry - */ + const int releaseSubblock(CommBlock& cb, WriteEngine::DataBlock*, int, WriteEngine::IdxEmptyListEntry*); - const int releaseSubblock( CommBlock& cb, WriteEngine::DataBlock*, int, WriteEngine::IdxEmptyListEntry*) ; + /** + * @brief Handle assignment of sub-blocks separately - sometimes a sub-block contains list entries and + * sometimes it is a list entry + */ - /** - * @brief Handle assignment of sub-blocks separately - sometimes a sub-block contains list entries and sometimes it is a list entry - */ + const int assignSubblock(CommBlock& cb, WriteEngine::DataBlock*, int, WriteEngine::IdxEmptyListEntry*); - const int assignSubblock( CommBlock& cb, WriteEngine::DataBlock*, int, WriteEngine::IdxEmptyListEntry*); + /** + * Blank out the entries in the structure + **/ + const void nullPtr(WriteEngine::IdxEmptyListEntry* assignPtr) const; - /** - * Blank out the entries in the structure - **/ - const void nullPtr( WriteEngine::IdxEmptyListEntry* assignPtr ) const; + const void printMemSubBlock(DataBlock* curBlock, const int sbid); - const void printMemSubBlock( DataBlock* curBlock, const int sbid ); + inline const int calcPtrOffset(const int position) const + { + return 1 + position; + } - inline const int calcPtrOffset( const int position ) const - { - return 1 + position; - } + inline const int calcStatOffset(const int position) const + { + return 8 + position; + } - inline const int calcStatOffset( const int position ) const - { - return 8 + position; - } - - int initType;// decide which algorithm to use to init the chains - int allowExtend; // allow file to be extended + int initType; // decide which algorithm to use to init the chains + int allowExtend; // allow file to be extended }; -} //end of namespace -#endif // _WE_FREEMGR_H_ +} // namespace WriteEngine +#endif // _WE_FREEMGR_H_ diff --git a/writeengine/index/we_indexlist.cpp b/writeengine/index/we_indexlist.cpp index 921b06fe0..ddf813e63 100644 --- a/writeengine/index/we_indexlist.cpp +++ b/writeengine/index/we_indexlist.cpp @@ -37,13 +37,15 @@ namespace WriteEngine * Constructor */ IndexList::IndexList() - : m_oid((OID) INVALID_NUM), m_useNarray(true), - m_curLevel(INVALID_NUM), m_curBlkPos(0), - m_curLevelPos(INVALID_NUM) + : m_oid((OID)INVALID_NUM) + , m_useNarray(true) + , m_curLevel(INVALID_NUM) + , m_curBlkPos(0) + , m_curLevelPos(INVALID_NUM) { - m_freemgr.setDebugLevel(DEBUG_0); - init(); + m_freemgr.setDebugLevel(DEBUG_0); + init(); }; /**************************************************************** * DESCRIPTION: @@ -64,71 +66,70 @@ IndexList::IndexList() * failure - it did not create the index list header * *******************************************************************/ -const int IndexList::addIndexListHdr( FILE* pFile, const RID& rowId, - const uint64_t& key, - IdxEmptyListEntry* newEmptyListPtr) +const int IndexList::addIndexListHdr(FILE* pFile, const RID& rowId, const uint64_t& key, + IdxEmptyListEntry* newEmptyListPtr) { - int rc; - CommBlock cb; - m_pFile = pFile; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - //Set up the header structure - //Initialize header blokcs - rc = resetBlk(&m_hdrBlock); - m_hdrLbid = INVALID_LBID; - m_hdrSbid = INVALID_NUM; - m_hdrEntry = INVALID_NUM; + int rc; + CommBlock cb; + m_pFile = pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + // Set up the header structure + // Initialize header blokcs + rc = resetBlk(&m_hdrBlock); + m_hdrLbid = INVALID_LBID; + m_hdrSbid = INVALID_NUM; + m_hdrEntry = INVALID_NUM; - //Initialize the new Index List header to null - memset( &m_curIdxRidListHdr, 0, LIST_HDR_SIZE ); - //Assign the bit fields for the first entry in the Index List Header - m_curIdxRidListHdr.idxRidListSize.type = LIST_SIZE_TYPE; - m_curIdxRidListHdr.idxRidListSize.spare = 0x0; - m_curIdxRidListHdr.idxRidListSize.size = 1; - //Assign the bit fields for the second entry of the Index List Header - m_curIdxRidListHdr.key = key; - //Assign bit fields for the third entry of the Index List Header - m_curIdxRidListHdr.firstIdxRidListEntry.type = LIST_RID_TYPE; - m_curIdxRidListHdr.firstIdxRidListEntry.spare = 0x0; - m_curIdxRidListHdr.firstIdxRidListEntry.rid = rowId ; - //Assign bit fields for the fourth entry of the Index List Header - m_curIdxRidListHdr.nextIdxRidListPtr.type = LIST_NOT_USED_TYPE; - m_curIdxRidListHdr.nextIdxRidListPtr.spare = 0x0; - m_curIdxRidListHdr.nextIdxRidListPtr.llp = 0x0; - /* Get assigned space for the header from free manager - * Get the new block for the new idx list header - * The header needs LIST_HDR_SIZE bytes - */ - rc = getSegment(pFile, ENTRY_4, newEmptyListPtr); + // Initialize the new Index List header to null + memset(&m_curIdxRidListHdr, 0, LIST_HDR_SIZE); + // Assign the bit fields for the first entry in the Index List Header + m_curIdxRidListHdr.idxRidListSize.type = LIST_SIZE_TYPE; + m_curIdxRidListHdr.idxRidListSize.spare = 0x0; + m_curIdxRidListHdr.idxRidListSize.size = 1; + // Assign the bit fields for the second entry of the Index List Header + m_curIdxRidListHdr.key = key; + // Assign bit fields for the third entry of the Index List Header + m_curIdxRidListHdr.firstIdxRidListEntry.type = LIST_RID_TYPE; + m_curIdxRidListHdr.firstIdxRidListEntry.spare = 0x0; + m_curIdxRidListHdr.firstIdxRidListEntry.rid = rowId; + // Assign bit fields for the fourth entry of the Index List Header + m_curIdxRidListHdr.nextIdxRidListPtr.type = LIST_NOT_USED_TYPE; + m_curIdxRidListHdr.nextIdxRidListPtr.spare = 0x0; + m_curIdxRidListHdr.nextIdxRidListPtr.llp = 0x0; + /* Get assigned space for the header from free manager + * Get the new block for the new idx list header + * The header needs LIST_HDR_SIZE bytes + */ + rc = getSegment(pFile, ENTRY_4, newEmptyListPtr); - if (rc != NO_ERROR) - return rc; - - m_hdrLbid = newEmptyListPtr->fbo; - m_hdrSbid = newEmptyListPtr->sbid; - m_hdrEntry = newEmptyListPtr->entry; - - //Write Index List Header to the file block - //Write LIST_HDR_SIZE bytes in one time. - - rc = readDBFile( cb, m_hdrBlock.data, m_hdrLbid ); - rc = writeSubBlockEntry( cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr ); - - if (rc != NO_ERROR) - { - return rc; - } - - //Wrote Header Block Out already, Start Over next time - //Update the flags to indicate there is data on the header block - m_hdrBlock.dirty = true; - m_hdrBlock.lbid = m_hdrLbid; - m_hdrBlock.state = BLK_READ; - m_lastLbid = INVALID_LBID; - //DONE + if (rc != NO_ERROR) return rc; + + m_hdrLbid = newEmptyListPtr->fbo; + m_hdrSbid = newEmptyListPtr->sbid; + m_hdrEntry = newEmptyListPtr->entry; + + // Write Index List Header to the file block + // Write LIST_HDR_SIZE bytes in one time. + + rc = readDBFile(cb, m_hdrBlock.data, m_hdrLbid); + rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + + if (rc != NO_ERROR) + { + return rc; + } + + // Wrote Header Block Out already, Start Over next time + // Update the flags to indicate there is data on the header block + m_hdrBlock.dirty = true; + m_hdrBlock.lbid = m_hdrLbid; + m_hdrBlock.state = BLK_READ; + m_lastLbid = INVALID_LBID; + // DONE + return rc; }; /**************************************************************** * DESCRIPTION: @@ -138,32 +139,31 @@ const int IndexList::addIndexListHdr( FILE* pFile, const RID& rowId, * success - successfully created the index list header * failure - it did not create the index list header ***********************************************************/ -const int IndexList::updateIndexList(FILE* pFile, const RID& newRid, - const uint64_t& key, +const int IndexList::updateIndexList(FILE* pFile, const RID& newRid, const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr) { - int rc; - m_pFile = pFile; + int rc; + m_pFile = pFile; - //Initialization - if ( (key != m_curIdxRidListHdr.key) || (m_hdrBlock.state == BLK_INIT)) - { - rc = initGetHdr(key, curIdxRidListHdrPtr); + // Initialization + if ((key != m_curIdxRidListHdr.key) || (m_hdrBlock.state == BLK_INIT)) + { + rc = initGetHdr(key, curIdxRidListHdrPtr); - if (key != m_curIdxRidListHdr.key) - return ERR_IDX_LIST_INVALID_KEY; - } + if (key != m_curIdxRidListHdr.key) + return ERR_IDX_LIST_INVALID_KEY; + } - rc = updateIndexList(newRid, key); + rc = updateIndexList(newRid, key); - if (rc != NO_ERROR) - { - return rc; - } - - //Write everything out - rc = updateIndexListWrite(); + if (rc != NO_ERROR) + { return rc; + } + + // Write everything out + rc = updateIndexListWrite(); + return rc; }; /**************************************************************** * DESCRIPTION: @@ -175,26 +175,26 @@ const int IndexList::updateIndexList(FILE* pFile, const RID& newRid, ***********************************************************/ const int IndexList::updateIndexList(const RID& newRid, const uint64_t& key) { - int rc = NO_ERROR; + int rc = NO_ERROR; - //m_lastLbid==0 or not determines if we can skip from the header,the first - //subblock or go to the last inserted block - if (m_lastLbid == (uint64_t)INVALID_LBID) - { - rc = updateHdrSub(newRid, key); - } - else // get the lastLbid info from header - { - //m_lastLbid > 0, space is in some block now - m_lbid = m_lastLbid; - m_sbid = 0; - m_entry = 0; - m_segType = LIST_BLOCK_TYPE; - m_curType = LIST_BLOCK_TYPE; - rc = addRidInBlk(newRid); - } + // m_lastLbid==0 or not determines if we can skip from the header,the first + // subblock or go to the last inserted block + if (m_lastLbid == (uint64_t)INVALID_LBID) + { + rc = updateHdrSub(newRid, key); + } + else // get the lastLbid info from header + { + // m_lastLbid > 0, space is in some block now + m_lbid = m_lastLbid; + m_sbid = 0; + m_entry = 0; + m_segType = LIST_BLOCK_TYPE; + m_curType = LIST_BLOCK_TYPE; + rc = addRidInBlk(newRid); + } - return rc; + return rc; }; /************************************************ @@ -215,24 +215,24 @@ const int IndexList::updateIndexList(const RID& newRid, const uint64_t& key) * Success -- 0 * Fail -- ERR_IDX_LIST_INVALID_DELETE ************************************************/ -const int IndexList::deleteIndexList( FILE* pFile, const RID& rowId, - const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr) +const int IndexList::deleteIndexList(FILE* pFile, const RID& rowId, const uint64_t& key, + IdxEmptyListEntry* curIdxRidListHdrPtr) { - int rc = ERR_IDX_LIST_INVALID_DELETE; - m_pFile = pFile; + int rc = ERR_IDX_LIST_INVALID_DELETE; + m_pFile = pFile; - getHdrInfo(curIdxRidListHdrPtr); + getHdrInfo(curIdxRidListHdrPtr); - if (key != m_curIdxRidListHdr.key) - { - memset( m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - m_hdrBlock.dirty = false; - m_hdrBlock.state = BLK_INIT; - return ERR_IDX_LIST_INVALID_KEY; - } + if (key != m_curIdxRidListHdr.key) + { + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + m_hdrBlock.dirty = false; + m_hdrBlock.state = BLK_INIT; + return ERR_IDX_LIST_INVALID_KEY; + } - rc = deleteIndexList(rowId, key); - return rc; + rc = deleteIndexList(rowId, key); + return rc; } /************************************************ @@ -253,102 +253,93 @@ const int IndexList::deleteIndexList( FILE* pFile, const RID& rowId, * Success -- 0 * Fail -- ERR_IDX_LIST_INVALID_DELETE ************************************************/ -const int IndexList::deleteIndexList( const RID& rowId, const uint64_t& key) +const int IndexList::deleteIndexList(const RID& rowId, const uint64_t& key) { - int rc = ERR_IDX_LIST_INVALID_DELETE; - RID savedRid ; - DataBlock prevDataBlock; + int rc = ERR_IDX_LIST_INVALID_DELETE; + RID savedRid; + DataBlock prevDataBlock; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - //Check the first row location, 3rd entry - //Because it may be deleted from the delete action - //The header size cannot tell us the rowid size on header - if (m_curIdxRidListHdr.firstIdxRidListEntry.type - == (int)LIST_RID_TYPE) + // Check the first row location, 3rd entry + // Because it may be deleted from the delete action + // The header size cannot tell us the rowid size on header + if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) + { + if (m_curIdxRidListHdr.firstIdxRidListEntry.rid == rowId) { - if (m_curIdxRidListHdr.firstIdxRidListEntry.rid - == rowId) - { - m_curIdxRidListHdr.firstIdxRidListEntry.type - = LIST_NOT_USED_TYPE; //not used type - m_curIdxRidListHdr.firstIdxRidListEntry.rid = 0; - m_curIdxRidListHdr.idxRidListSize.size--; - rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, - m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - m_hdrBlock.dirty = false; - m_dLbid = m_hdrLbid; - m_dSbid = m_hdrSbid; - m_dEntry = m_hdrEntry + 2; + m_curIdxRidListHdr.firstIdxRidListEntry.type = LIST_NOT_USED_TYPE; // not used type + m_curIdxRidListHdr.firstIdxRidListEntry.rid = 0; + m_curIdxRidListHdr.idxRidListSize.size--; + rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + m_hdrBlock.dirty = false; + m_dLbid = m_hdrLbid; + m_dSbid = m_hdrSbid; + m_dEntry = m_hdrEntry + 2; - return rc; - } - }; + return rc; + } + }; - //Check Header last entry's type - int type = m_curIdxRidListHdr.nextIdxRidListPtr.type; + // Check Header last entry's type + int type = m_curIdxRidListHdr.nextIdxRidListPtr.type; - switch (type) - { - case LIST_NOT_USED_TYPE://Header is not full, no sub-block linked - //No RowId here - memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - m_hdrBlock.dirty = false; - m_dLbid = -1LL; - m_dSbid = -1; - m_dEntry = -1; - return ERR_IDX_LIST_INVALID_DELETE; //not found, failed + switch (type) + { + case LIST_NOT_USED_TYPE: // Header is not full, no sub-block linked + // No RowId here + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + m_hdrBlock.dirty = false; + m_dLbid = -1LL; + m_dSbid = -1; + m_dEntry = -1; + return ERR_IDX_LIST_INVALID_DELETE; // not found, failed - case LIST_RID_TYPE:// There is a row id here, Check! - savedRid = m_curIdxRidListHdr.nextIdxRidListPtr.llp; + case LIST_RID_TYPE: // There is a row id here, Check! + savedRid = m_curIdxRidListHdr.nextIdxRidListPtr.llp; - if (savedRid == rowId) - { - m_curIdxRidListHdr.nextIdxRidListPtr.type - = LIST_NOT_USED_TYPE; - m_curIdxRidListHdr.nextIdxRidListPtr.llp = 0; - m_curIdxRidListHdr.idxRidListSize.size--; - rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, - m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - m_hdrBlock.dirty = false; - memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - m_dLbid = m_hdrLbid; - m_dSbid = m_hdrSbid; - m_dEntry = 3; + if (savedRid == rowId) + { + m_curIdxRidListHdr.nextIdxRidListPtr.type = LIST_NOT_USED_TYPE; + m_curIdxRidListHdr.nextIdxRidListPtr.llp = 0; + m_curIdxRidListHdr.idxRidListSize.size--; + rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + m_hdrBlock.dirty = false; + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + m_dLbid = m_hdrLbid; + m_dSbid = m_hdrSbid; + m_dEntry = 3; - return rc; - } - else - { - m_hdrBlock.dirty = false; - memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - m_dLbid = -1LL; - m_dSbid = -1; - m_dEntry = -1; - return ERR_IDX_LIST_INVALID_DELETE; - } + return rc; + } + else + { + m_hdrBlock.dirty = false; + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + m_dLbid = -1LL; + m_dSbid = -1; + m_dEntry = -1; + return ERR_IDX_LIST_INVALID_DELETE; + } - case LIST_SUBBLOCK_TYPE://Not found in header, - rc = deleteInSub(rowId); + case LIST_SUBBLOCK_TYPE: // Not found in header, + rc = deleteInSub(rowId); - if (rc == NO_ERROR) - return rc; + if (rc == NO_ERROR) + return rc; - rc = deleteInBlock(rowId); - return rc; - break; + rc = deleteInBlock(rowId); + return rc; + break; - default: - break; - };//end of switch + default: break; + }; // end of switch - return ERR_IDX_LIST_INVALID_DELETE; -} + return ERR_IDX_LIST_INVALID_DELETE; } +} // namespace WriteEngine diff --git a/writeengine/index/we_indexlist.h b/writeengine/index/we_indexlist.h index c558f164b..bdd903588 100644 --- a/writeengine/index/we_indexlist.h +++ b/writeengine/index/we_indexlist.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_indexlist.h 33 2006-07-19 09:18:27Z jhuang $ -* -******************************************************************************************/ + * $Id: we_indexlist.h 33 2006-07-19 09:18:27Z jhuang $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_IndexList_H_ @@ -36,452 +36,420 @@ /** Namespace WriteEngine */ namespace WriteEngine { - class IndexList : public DbFileOp { -public: - /** - * @brief Constructor - */ - IndexList(); - //IndexList(FILE* pFile, IdxEmptyListEntry* newHeaderListPtr); - /** - * @brief Default Destructor - */ - ~IndexList() {}; + public: + /** + * @brief Constructor + */ + IndexList(); + // IndexList(FILE* pFile, IdxEmptyListEntry* newHeaderListPtr); + /** + * @brief Default Destructor + */ + ~IndexList(){}; - /** - * @brief Public index List related functions - */ + /** + * @brief Public index List related functions + */ - /** - * @brief Add a 4 bytes header for a rowid, key in the index list - */ - const int addIndexListHdr( FILE* pFile, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* newHeaderListPtr ); + /** + * @brief Add a 4 bytes header for a rowid, key in the index list + */ + const int addIndexListHdr(FILE* pFile, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* newHeaderListPtr); - const int addIndexListHdr( CommBlock& cbList, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* newHeaderListPtr ) - { - m_oid = cbList.file.oid; - m_pFile = cbList.file.pFile; - int rc = addIndexListHdr( m_pFile, listRid, key, newHeaderListPtr); - return rc; - } - const int addIndexListHdr( FILE* pFile, const RID* ridList, const int size, const uint64_t& key, - IdxEmptyListEntry* newHeaderListPtr ); + const int addIndexListHdr(CommBlock& cbList, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* newHeaderListPtr) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int rc = addIndexListHdr(m_pFile, listRid, key, newHeaderListPtr); + return rc; + } + const int addIndexListHdr(FILE* pFile, const RID* ridList, const int size, const uint64_t& key, + IdxEmptyListEntry* newHeaderListPtr); - const int addIndexListHdr( CommBlock& cbList, RID* ridList, int& size, const uint64_t& key, - IdxEmptyListEntry* newHeaderListPtr ) - { - m_oid = cbList.file.oid; - m_pFile = cbList.file.pFile; - int rc = addIndexListHdr( m_pFile, ridList, size, key, newHeaderListPtr); - return rc; - } - const int addIndexListHdr( CommBlock& cbList, const IdxMultiRid& multiRids, - const uint64_t& key, IdxEmptyListEntry* newHeaderListPtr) - { - m_oid = cbList.file.oid; - m_pFile = cbList.file.pFile; - int size = multiRids.totalRid; - return addIndexListHdr( m_pFile, multiRids.ridArray, size, - key, newHeaderListPtr ); + const int addIndexListHdr(CommBlock& cbList, RID* ridList, int& size, const uint64_t& key, + IdxEmptyListEntry* newHeaderListPtr) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int rc = addIndexListHdr(m_pFile, ridList, size, key, newHeaderListPtr); + return rc; + } + const int addIndexListHdr(CommBlock& cbList, const IdxMultiRid& multiRids, const uint64_t& key, + IdxEmptyListEntry* newHeaderListPtr) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int size = multiRids.totalRid; + return addIndexListHdr(m_pFile, multiRids.ridArray, size, key, newHeaderListPtr); + } - } + /** + * @brief Update the indexlist when a new rowid an key inserted + */ + const int updateIndexList(FILE* pFile, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr); - /** - * @brief Update the indexlist when a new rowid an key inserted - */ - const int updateIndexList( FILE* pFile, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr ); + const int updateIndexList(CommBlock& cbList, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; - const int updateIndexList( CommBlock& cbList, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr ) - { - m_oid = cbList.file.oid; - m_pFile = cbList.file.pFile; + int rc = updateIndexList(m_pFile, listRid, key, oldHeaderListPtr); + return rc; + }; - int rc = updateIndexList( m_pFile, listRid, key, oldHeaderListPtr); - return rc; - }; + const int updateIndexList(FILE* pFile, const RID* ridList, const int size, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr); + const int updateIndexList(CommBlock& cbList, const RID* ridList, const int size, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; - const int updateIndexList( FILE* pFile, const RID* ridList, const int size, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr ); - const int updateIndexList( CommBlock& cbList, const RID* ridList, const int size, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr ) - { - m_oid = cbList.file.oid; - m_pFile = cbList.file.pFile; + int rc = updateIndexList(m_pFile, ridList, size, key, oldHeaderListPtr); + return rc; + } + const int updateIndexList(CommBlock& cbList, const IdxMultiRid& multiRids, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr) + { + return updateIndexList(cbList, multiRids.ridArray, multiRids.totalRid, key, oldHeaderListPtr); + } - int rc = updateIndexList( m_pFile, ridList, size, key, oldHeaderListPtr); - return rc; - } - const int updateIndexList( CommBlock& cbList, const IdxMultiRid& multiRids, - const uint64_t& key, IdxEmptyListEntry* oldHeaderListPtr) - { - return updateIndexList( cbList, multiRids.ridArray, multiRids.totalRid, - key, oldHeaderListPtr ); - } + /** + * @brief Delete the rowid in the key indexlist + */ + const int deleteIndexList(FILE* pFile, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr); - /** - * @brief Delete the rowid in the key indexlist - */ - const int deleteIndexList( FILE* pFile, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr ); + const int deleteIndexList(CommBlock& cbList, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; - const int deleteIndexList( CommBlock& cbList, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr ) - { - m_oid = cbList.file.oid ; - m_pFile = cbList.file.pFile; + int rc = deleteIndexList(m_pFile, listRid, key, oldHeaderListPtr); + return rc; + }; - int rc = deleteIndexList( m_pFile, listRid, key, oldHeaderListPtr ); - return rc; - }; + /** + * @brief delete a row id from the key index list and return the location + */ + const int deleteIndexList(FILE* pFile, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr, uint64_t& lbid, int& sbid, int& entry); - /** - * @brief delete a row id from the key index list and return the location - */ - const int deleteIndexList( FILE* pFile, const RID& listRid, const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr, - uint64_t& lbid, int& sbid, int& entry ); + const int deleteIndexList(CommBlock& cbList, const RID& listRid, const uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr, uint64_t& lbid, int& sbid, int& entry) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int rc = deleteIndexList(m_pFile, listRid, key, oldHeaderListPtr, lbid, sbid, entry); + return rc; + }; - const int deleteIndexList( CommBlock& cbList, const RID& listRid, - const uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr, - uint64_t& lbid, int& sbid, int& entry ) - { - m_oid = cbList.file.oid ; - m_pFile = cbList.file.pFile; - int rc = deleteIndexList( m_pFile, listRid, key, - oldHeaderListPtr, - lbid, sbid, entry); - return rc; - }; + /** + * @brief find a row id from the key index list and return the location + */ + bool findRowId(FILE* pFile, const RID& rid, const uint64_t& key, IdxEmptyListEntry* oldIdxRidListHdrPtr, + uint64_t& lbid, int& sbid, int& entry); - /** - * @brief find a row id from the key index list and return the location - */ - bool findRowId(FILE* pFile, const RID& rid, const uint64_t& key, - IdxEmptyListEntry* oldIdxRidListHdrPtr, - uint64_t& lbid, int& sbid, int& entry); + bool findRowId(CommBlock& cbList, const RID& rid, const uint64_t& key, + IdxEmptyListEntry* oldIdxRidListHdrPtr, uint64_t& lbid, int& sbid, int& entry) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; - bool findRowId(CommBlock& cbList, const RID& rid, const uint64_t& key, - IdxEmptyListEntry* oldIdxRidListHdrPtr, - uint64_t& lbid, int& sbid, int& entry) - { - m_oid = cbList.file.oid ; - m_pFile = cbList.file.pFile; + bool found = findRowId(m_pFile, rid, key, oldIdxRidListHdrPtr, lbid, sbid, entry); + return found; + }; + /** + * @brief get the total row ids or tokens from the index list header + */ + const int getRIDArrayFromListHdr(FILE* pFile, uint64_t& key, IdxEmptyListEntry* oldHeaderListPtr, + RID* ridArrary, int& size); - bool found = findRowId(m_pFile, rid, key, - oldIdxRidListHdrPtr, lbid, - sbid, entry); - return found; - }; - /** - * @brief get the total row ids or tokens from the index list header - */ - const int getRIDArrayFromListHdr(FILE* pFile, uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr, - RID* ridArrary, int& size); + const int getRIDArrayFromListHdr(CommBlock& cbList, uint64_t& key, IdxEmptyListEntry* oldHeaderListPtr, + RID* ridArrary, int& size) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int rc = getRIDArrayFromListHdr(m_pFile, key, oldHeaderListPtr, ridArrary, size); + return rc; + }; + const int getRIDArrayFromListHdrNarray(FILE* pFile, uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr, + RID* ridArray, int& size, bool flag); + const int getRIDArrayFromListHdrNarray(CommBlock& cbList, uint64_t& key, + IdxEmptyListEntry* oldHeaderListPtr, RID* ridArrary, int& size, + bool flag) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int rc = getRIDArrayFromListHdrNarray(m_pFile, key, oldHeaderListPtr, ridArrary, size, flag); + return rc; + }; - const int getRIDArrayFromListHdr(CommBlock& cbList, uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr, - RID* ridArrary, int& size) - { - m_oid = cbList.file.oid ; - m_pFile = cbList.file.pFile; - int rc = getRIDArrayFromListHdr(m_pFile, key, - oldHeaderListPtr, - ridArrary, size); - return rc; - }; - const int getRIDArrayFromListHdrNarray(FILE* pFile, uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr, - RID* ridArray, int& size, bool flag); - const int getRIDArrayFromListHdrNarray(CommBlock& cbList, uint64_t& key, - IdxEmptyListEntry* oldHeaderListPtr, - RID* ridArrary, int& size, bool flag) - { - m_oid = cbList.file.oid ; - m_pFile = cbList.file.pFile; - int rc = getRIDArrayFromListHdrNarray(m_pFile, key, - oldHeaderListPtr, - ridArrary, size, flag); - return rc; - }; + const int init(CommBlock& cbList, const int& freemgr_type) + { + m_oid = cbList.file.oid; + m_pFile = cbList.file.pFile; + int rc = m_freemgr.init(cbList, freemgr_type); + return rc; + }; + const int closeList() + { + int rc = updateIndexListWrite(); + return rc; + } + void setTransId(const TxnID txnId) + { + FileOp::setTransId(txnId); + m_freemgr.setTransId(txnId); + } + IdxRidListHdr m_curIdxRidListHdr; /**@brief current list header */ + const int getHdrInfo(IdxEmptyListEntry* curIdxRidListHdrPtr); + const int getHdrInfo(IdxEmptyListEntry* curIdxRidListHdrPtr, IdxRidListHdr* idxRidListHdr) + { + int rc = 0; + rc = getHdrInfo(curIdxRidListHdrPtr); + memcpy(idxRidListHdr, &m_curIdxRidListHdr, LIST_HDR_SIZE); + return rc; + } + /** + * @brief Timer functions + */ + void startTimer() + { + time(&m_startTime); + } + void stopTimer() + { + time(&m_endTime); + m_totalTime = difftime(m_endTime, m_startTime); + } + double getTotalRunTime() const + { + return m_totalTime; + } + void setDebugLevel(const DebugLevel level) + { + m_freemgr.setDebugLevel(level); + } + void setUseSortFlag(const bool flag) + { + m_useSortFlag = flag; + } + void startfTimer() + { + ftime(&t_start); + }; + void stopfTimer() + { + ftime(&t_current); + }; + int getTotalfRunTime() + { + int t_diff; + t_diff = (int)(1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); + return t_diff; + } + /** + * @brief Private index List member functions + */ + const int updateIndexListWrite(); + const int init(); + const int initBlksGetHdrBlk(); + const int initGetHdr(const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr); + const int resetBlk(DataBlock* dataBlk); + const int resetBlkZero(uint64_t& lbid0); + const int writeBlkZero(uint64_t& lbid0); - const int init (CommBlock& cbList, const int& freemgr_type) - { - m_oid = cbList.file.oid ; - m_pFile = cbList.file.pFile; - int rc = m_freemgr.init(cbList, freemgr_type); - return rc; - }; - const int closeList() - { - int rc = updateIndexListWrite(); - return rc; - } - void setTransId( const TxnID txnId ) - { - FileOp::setTransId( txnId ); - m_freemgr.setTransId(txnId); - } - IdxRidListHdr m_curIdxRidListHdr; /**@brief current list header */ - const int getHdrInfo(IdxEmptyListEntry* curIdxRidListHdrPtr); - const int getHdrInfo(IdxEmptyListEntry* curIdxRidListHdrPtr, IdxRidListHdr* idxRidListHdr) - { - int rc = 0; - rc = getHdrInfo(curIdxRidListHdrPtr); - memcpy(idxRidListHdr, &m_curIdxRidListHdr, LIST_HDR_SIZE); - return rc; - } - /** - * @brief Timer functions - */ - void startTimer() - { - time( &m_startTime ); - } - void stopTimer() - { - time( &m_endTime ); - m_totalTime = difftime( m_endTime, m_startTime); - } - double getTotalRunTime() const - { - return m_totalTime; - } - void setDebugLevel(const DebugLevel level) - { - m_freemgr.setDebugLevel(level); - } - void setUseSortFlag(const bool flag) - { - m_useSortFlag = flag; - } + const int setLastLbid(uint64_t& lastLbid); + const int getLastLbid(); - void startfTimer() - { - ftime(&t_start); - }; - void stopfTimer() - { - ftime(&t_current); - }; - int getTotalfRunTime() - { - int t_diff; - t_diff = (int) (1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); - return t_diff; - } - /** - * @brief Private index List member functions - */ - const int updateIndexListWrite(); - const int init(); - const int initBlksGetHdrBlk(); - const int initGetHdr(const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr); - const int resetBlk(DataBlock* dataBlk) ; - const int resetBlkZero(uint64_t& lbid0); - const int writeBlkZero(uint64_t& lbid0); + const int findLastBlk(int& count); + const int readCurBlk(); + const int writeCurBlk(); + const int addRidInBlk(const RID& newRid); + const int insertRid(const RID& newRid, int& pos); + const int updateCurCount(int frequency = 1); + const int getNextInfo(int& count); + const int getNextInfoFromBlk(IdxRidListPtr& idxRidListPtr); + const int getNextInfoFromBlk() + { + return getNextInfoFromBlk(m_lastIdxRidListPtr); + }; + const int getLastFboPtr(uint64_t& lbid, int& sbid, IdxRidListPtr& lastFboListPtr); + const int setLastFboPtr(uint64_t& lbid, int& sbid, IdxRidListPtr& lastFboListPtr); + const int setNextInfoFromBlk(IdxRidListPtr& idxRidListPtr); + const int setNarray(bool flag) + { + m_useNarray = flag; + return NO_ERROR; + }; + const int findFirstBlk(FILE* pFile, const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr, + uint64_t& lbid); + bool getUseNarray() + { + return m_useNarray; + }; + const int printBlocks(uint64_t& lbid); + const int getBlk(uint64_t& lbid); + const int getSubBlk(); + const int getSubBlk(uint64_t& lbid, int& sbid, int& entry); - const int setLastLbid( uint64_t& lastLbid); - const int getLastLbid(); + private: + struct timeb t_start, t_current; + const int getLastFbo(); + const int setLastFbo(int* fbo); + // const int addRidList(RID* ridList, int & size); + /** + * @brief get a segment from freemanager + */ + const int getSegment(FILE* pfile, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr); + /** + * @brief insert a rowid into the key index list + */ + const int updateIndexList(const RID& listRid, const uint64_t& key, uint64_t& startLbid, int& startSbid, + int& startEntry, uint64_t& endLbid, int& endSbid, int& endEntry); - const int findLastBlk(int& count); - const int readCurBlk(); - const int writeCurBlk(); - const int addRidInBlk( const RID& newRid); - const int insertRid(const RID& newRid, int& pos); - const int updateCurCount(int frequency = 1); - const int getNextInfo(int& count); - const int getNextInfoFromBlk( IdxRidListPtr& idxRidListPtr); - const int getNextInfoFromBlk() - { - return getNextInfoFromBlk(m_lastIdxRidListPtr); - }; - const int getLastFboPtr(uint64_t& lbid, int& sbid, IdxRidListPtr& lastFboListPtr); - const int setLastFboPtr(uint64_t& lbid, int& sbid, IdxRidListPtr& lastFboListPtr); - const int setNextInfoFromBlk( IdxRidListPtr& idxRidListPtr); - const int setNarray(bool flag) - { - m_useNarray = flag; - return NO_ERROR; - }; - const int findFirstBlk(FILE* pFile, const uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr, - uint64_t& lbid); - bool getUseNarray() - { - return m_useNarray; - }; - const int printBlocks(uint64_t& lbid); - const int getBlk(uint64_t& lbid); - const int getSubBlk(); - const int getSubBlk(uint64_t& lbid, int& sbid, int& entry); + /** + * @brief insert a rowid into the key index list + */ + const int updateIndexList(const RID& listRid, const uint64_t& key); + const int updateIndexList(const RID* ridList, const int size, int& startPos); + const int addRid(const RID& newRid, IdxRidListPtr& lastIdxRidListPtr); + const int addRidList(const RID* ridList, const int size, int& startPos); + /** + * @brief write all of the blocks in the same time at the end + */ + /** + * @brief add a new subblock segment to the linked list + */ + const int addNextIdxRidList(FILE* pFile, const RID& rowId, IdxEmptyListEntry* newIdxListEntryPtr); + /** + * @brief find a row id from the key index list and return the location + */ + bool findRowId(const RID& rowId, const uint64_t& key, uint64_t& lbid, int& sbid, int& entry); + /** + * @brief delete a row id from the key index list + */ + const int deleteIndexList(const RID& rowId, const uint64_t& key); + /** + * @brief delete a row id from the key index list and return a location + */ -private: - struct timeb t_start, t_current; - const int getLastFbo(); - const int setLastFbo(int* fbo); - //const int addRidList(RID* ridList, int & size); - /** - * @brief get a segment from freemanager - */ - const int getSegment( FILE* pfile, const IdxTreeGroupType segmentType, - IdxEmptyListEntry* assignPtr ); - /** - * @brief insert a rowid into the key index list - */ - const int updateIndexList( const RID& listRid, const uint64_t& key, - uint64_t& startLbid, int& startSbid, int& startEntry, - uint64_t& endLbid, int& endSbid, int& endEntry - ); + const int deleteIndexList(const RID& rowId, const uint64_t& key, uint64_t& lbid, int& sbid, int& entry); + /** + * @brief insert a new row id to the key index list + */ + const int updateIdxRidList(const RID& rid, DataBlock* dataBlock, const int& sbid, const int& entry, + IdxRidListPtr* idxRidListPtr); - /** - * @brief insert a rowid into the key index list - */ - const int updateIndexList( const RID& listRid, const uint64_t& key); - const int updateIndexList( const RID* ridList, const int size, int& startPos); - const int addRid(const RID& newRid, IdxRidListPtr& lastIdxRidListPtr); - const int addRidList(const RID* ridList, const int size, int& startPos); - /** - * @brief write all of the blocks in the same time at the end - */ + const int deleteInSub(const RID& rowId); + const int deleteInBlock(const RID& newRid); + const int releaseSegment(); + void setListOid(const OID& listOid) + { + m_oid = listOid; + } + const OID getListOid() + { + return m_oid; + } + const int insertRowId(uint64_t& curLbid, int& pos, uint64_t& parentLbid); + const int setupBlock(uint64_t& curLbid, int& pos, uint64_t& parentLbid); - /** - * @brief add a new subblock segment to the linked list - */ - const int addNextIdxRidList(FILE* pFile, const RID& rowId, - IdxEmptyListEntry* newIdxListEntryPtr); - /** - * @brief find a row id from the key index list and return the location - */ - bool findRowId( const RID& rowId, const uint64_t& key, uint64_t& lbid, int& sbid, int& entry); - /** - * @brief delete a row id from the key index list - */ - const int deleteIndexList( const RID& rowId, const uint64_t& key); - /** - * @brief delete a row id from the key index list and return a location - */ + const int updateHdrSub(const RID& newRid, const uint64_t& key); + const int addRidInSub(const RID& newRid, IdxRidListPtr& lastIdxRidListPtr); + const int moveRidsToNewSub(FILE* pFile, const RID& oldRid, const RID& newRid, + IdxEmptyListEntry* newIdxListEntryPtr); + const int updateHdrCount(); - const int deleteIndexList(const RID& rowId, - const uint64_t& key, - uint64_t& lbid, int& sbid, int& entry); - /** - * @brief insert a new row id to the key index list - */ - const int updateIdxRidList(const RID& rid, DataBlock* dataBlock, - const int& sbid, const int& entry, - IdxRidListPtr* idxRidListPtr); + const int setParentStatus(uint64_t& pLbid); + const int setCurBlkNextPtr(uint64_t& nextLbid, int count); + const int initNewBlock(int64_t& lbid, DataBlock* dataBlock, uint64_t& pLbid); + const int initNewBlock(DataBlock* dataBlock, uint64_t& pLbid); + const int initCurBlock(); + const int getNextInfoFromArray(IdxRidNextListPtr& nextIdxListPtr); + const int getParentInfoFromArray(IdxRidParentListPtr& parentIdxListPtr); + const int updateCurCountInArray(int insCnt = 1); + const int updateParentStatus(uint64_t& childLbid); - const int deleteInSub( const RID& rowId) ; - const int deleteInBlock(const RID& newRid); - const int releaseSegment() ; - void setListOid(const OID& listOid) - { - m_oid = listOid; - } - const OID getListOid() - { - return m_oid; - } - const int insertRowId( uint64_t& curLbid, int& pos, uint64_t& parentLbid); - const int setupBlock ( uint64_t& curLbid, int& pos, uint64_t& parentLbid); + const int updateLastPtrAndParent(const int lastCount); + const int updateLastPtr(const int lastCount); + const int updateParent(); - const int updateHdrSub(const RID& newRid, const uint64_t& key); - const int addRidInSub(const RID& newRid, - IdxRidListPtr& lastIdxRidListPtr); - const int moveRidsToNewSub(FILE* pFile, const RID& oldRid, - const RID& newRid, - IdxEmptyListEntry* newIdxListEntryPtr); - const int updateHdrCount(); + /** + * @brief private member data + */ + DataBlock m_curBlock; /**@brief the current block buffer */ + DataBlock m_hdrBlock; /**@brief the header block buffer*/ + DataBlock m_blockZero; /**@brief 0,0,0 for free manager*/ + DataBlock m_parentBlock; + DataBlock m_nextBlock; - const int setParentStatus(uint64_t& pLbid); - const int setCurBlkNextPtr(uint64_t& nextLbid, int count); - const int initNewBlock(int64_t& lbid, DataBlock* dataBlock, uint64_t& pLbid); - const int initNewBlock(DataBlock* dataBlock, uint64_t& pLbid); - const int initCurBlock(); - const int getNextInfoFromArray(IdxRidNextListPtr& nextIdxListPtr); - const int getParentInfoFromArray(IdxRidParentListPtr& parentIdxListPtr); - const int updateCurCountInArray(int insCnt = 1); - const int updateParentStatus(uint64_t& childLbid); + uint64_t m_hdrLbid; /**@brief the header block number */ + int m_hdrSbid; /**@brief the header subblock number */ + int m_hdrEntry; /**@brief the header entry number */ - const int updateLastPtrAndParent(const int lastCount); - const int updateLastPtr(const int lastCount); - const int updateParent(); + uint64_t m_lbid; /**@brief the current block number */ + int m_sbid; /**@brief the current subblock number */ + int m_entry; /**@brief the current entry number */ - /** - * @brief private member data - */ - DataBlock m_curBlock; /**@brief the current block buffer */ - DataBlock m_hdrBlock; /**@brief the header block buffer*/ - DataBlock m_blockZero; /**@brief 0,0,0 for free manager*/ - DataBlock m_parentBlock; - DataBlock m_nextBlock; + uint64_t m_newLbid; + int m_newSbid; + int m_newEntry; - uint64_t m_hdrLbid; /**@brief the header block number */ - int m_hdrSbid; /**@brief the header subblock number */ - int m_hdrEntry; /**@brief the header entry number */ + uint64_t m_parentLbid; + int m_parentSbid; + int m_parentEntry; - uint64_t m_lbid; /**@brief the current block number */ - int m_sbid; /**@brief the current subblock number */ - int m_entry; /**@brief the current entry number */ + uint64_t m_nextLbid; /**@brief the next block number */ + int m_nextSbid; /**@brief the next subblock number */ + int m_nextEntry; /**@brief the next entry number */ - uint64_t m_newLbid; - int m_newSbid; - int m_newEntry; + uint64_t m_dLbid; /**@brief the next block number */ + int m_dSbid; /**@brief the next subblock number */ + int m_dEntry; /**@brief the next entry number */ - uint64_t m_parentLbid; - int m_parentSbid; - int m_parentEntry; + FILE* m_pFile; /**@brief file handle for index list */ + OID m_oid; + int m_llpPos; + bool m_useBlock; + int m_entryGroup; + int m_type; + int m_segType; + int m_curType; + int m_nextType; + int m_totalbytes; + int m_lastFbo; + uint64_t m_lastLbid; + FreeMgr m_freemgr; + time_t m_startTime; // start time + time_t m_endTime; // end time + double m_totalTime; + DebugLevel m_debugLevel; // internal use debug level + bool m_useSortFlag; + bool m_useNarray; + IdxRidChildListPtr m_childIdxRidListPtr[4]; + IdxRidNextListPtr m_nextIdxListPtr; + IdxRidParentListPtr m_parentIdxListPtr; + IdxRidListArrayPtr m_idxRidListArrayPtr; + IdxRidLastListPtr m_lastBLKIdxListPtr; + IdxRidListPtr m_lastIdxRidListPtr; - uint64_t m_nextLbid; /**@brief the next block number */ - int m_nextSbid; /**@brief the next subblock number */ - int m_nextEntry; /**@brief the next entry number */ - - uint64_t m_dLbid; /**@brief the next block number */ - int m_dSbid; /**@brief the next subblock number */ - int m_dEntry; /**@brief the next entry number */ - - FILE* m_pFile; /**@brief file handle for index list */ - OID m_oid; - int m_llpPos; - bool m_useBlock; - int m_entryGroup; - int m_type; - int m_segType; - int m_curType; - int m_nextType; - int m_totalbytes; - int m_lastFbo; - uint64_t m_lastLbid; - FreeMgr m_freemgr; - time_t m_startTime; // start time - time_t m_endTime; // end time - double m_totalTime; - DebugLevel m_debugLevel; // internal use debug level - bool m_useSortFlag; - bool m_useNarray; - IdxRidChildListPtr m_childIdxRidListPtr[4]; - IdxRidNextListPtr m_nextIdxListPtr; - IdxRidParentListPtr m_parentIdxListPtr; - IdxRidListArrayPtr m_idxRidListArrayPtr; - IdxRidLastListPtr m_lastBLKIdxListPtr; - IdxRidListPtr m_lastIdxRidListPtr; - - int m_curLevel; - int m_curBlkPos; - int m_curLevelPos; - int m_curCount; - bool m_flag; + int m_curLevel; + int m_curBlkPos; + int m_curLevelPos; + int m_curCount; + bool m_flag; }; -//std::ostream& operator<<(std::ostream& os, const IndexList& rhs); -} //end of namespace -#endif // _WE_IndexList_H_ +// std::ostream& operator<<(std::ostream& os, const IndexList& rhs); +} // namespace WriteEngine +#endif // _WE_IndexList_H_ diff --git a/writeengine/index/we_indexlist_common.cpp b/writeengine/index/we_indexlist_common.cpp index 2af4a6c70..55fc38e72 100644 --- a/writeengine/index/we_indexlist_common.cpp +++ b/writeengine/index/we_indexlist_common.cpp @@ -30,65 +30,64 @@ #endif #include "we_indexlist.h" - using namespace std; namespace WriteEngine { /**************************************************************** -* DESCRIPTION: -* Private Function for getting the last Fbo on header -* -***************************************************************/ + * DESCRIPTION: + * Private Function for getting the last Fbo on header + * + ***************************************************************/ const int IndexList::init() { - memset( m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - memset( m_nextBlock.data, 0, sizeof(m_nextBlock.data)); - memset( m_blockZero.data, 0, sizeof(m_blockZero.data)); - memset( m_parentBlock.data, 0, sizeof(m_parentBlock.data)); + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + memset(m_nextBlock.data, 0, sizeof(m_nextBlock.data)); + memset(m_blockZero.data, 0, sizeof(m_blockZero.data)); + memset(m_parentBlock.data, 0, sizeof(m_parentBlock.data)); - memset( &m_curIdxRidListHdr, 0, LIST_HDR_SIZE ); - m_curIdxRidListHdr.key = INVALID_KEY; + memset(&m_curIdxRidListHdr, 0, LIST_HDR_SIZE); + m_curIdxRidListHdr.key = INVALID_KEY; - memset( &m_idxRidListArrayPtr, 0, sizeof(IdxRidListArrayPtr) ); - memset( &m_lastIdxRidListPtr, 0, sizeof(IdxRidListPtr) ); + memset(&m_idxRidListArrayPtr, 0, sizeof(IdxRidListArrayPtr)); + memset(&m_lastIdxRidListPtr, 0, sizeof(IdxRidListPtr)); - m_hdrBlock.dirty = false; - m_curBlock.dirty = false; - m_nextBlock.dirty = false; - m_blockZero.dirty = false; - m_parentBlock.dirty = false; + m_hdrBlock.dirty = false; + m_curBlock.dirty = false; + m_nextBlock.dirty = false; + m_blockZero.dirty = false; + m_parentBlock.dirty = false; - m_hdrBlock.state = BLK_INIT; - m_curBlock.state = BLK_INIT; - m_nextBlock.state = BLK_INIT; - m_blockZero.state = BLK_INIT; - m_parentBlock.state = BLK_INIT; + m_hdrBlock.state = BLK_INIT; + m_curBlock.state = BLK_INIT; + m_nextBlock.state = BLK_INIT; + m_blockZero.state = BLK_INIT; + m_parentBlock.state = BLK_INIT; - m_hdrBlock.lbid = INVALID_LBID; - m_curBlock.lbid = INVALID_LBID; - m_nextBlock.lbid = INVALID_LBID; - m_blockZero.lbid = INVALID_LBID; - m_parentBlock.lbid = INVALID_LBID; + m_hdrBlock.lbid = INVALID_LBID; + m_curBlock.lbid = INVALID_LBID; + m_nextBlock.lbid = INVALID_LBID; + m_blockZero.lbid = INVALID_LBID; + m_parentBlock.lbid = INVALID_LBID; - m_lbid = INVALID_LBID; - m_sbid = INVALID_NUM; - m_entry = INVALID_NUM; + m_lbid = INVALID_LBID; + m_sbid = INVALID_NUM; + m_entry = INVALID_NUM; - m_hdrLbid = INVALID_LBID; - m_hdrSbid = INVALID_NUM; - m_hdrEntry = INVALID_NUM; + m_hdrLbid = INVALID_LBID; + m_hdrSbid = INVALID_NUM; + m_hdrEntry = INVALID_NUM; - m_nextLbid = INVALID_LBID; - m_nextSbid = INVALID_NUM; - m_nextEntry = INVALID_NUM; + m_nextLbid = INVALID_LBID; + m_nextSbid = INVALID_NUM; + m_nextEntry = INVALID_NUM; - m_parentLbid = INVALID_LBID; + m_parentLbid = INVALID_LBID; - m_lastLbid = INVALID_LBID; + m_lastLbid = INVALID_LBID; - return NO_ERROR; + return NO_ERROR; } /**************************************************************** * DESCRIPTION: @@ -97,125 +96,123 @@ const int IndexList::init() ***************************************************************/ const int IndexList::initBlksGetHdrBlk() { - int rc = NO_ERROR; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - memset( m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); - memset( m_nextBlock.data, 0, sizeof(m_nextBlock.data)); - memset( m_blockZero.data, 0, sizeof(m_blockZero.data)); - memset( m_parentBlock.data, 0, sizeof(m_curBlock.data)); + int rc = NO_ERROR; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + memset(m_nextBlock.data, 0, sizeof(m_nextBlock.data)); + memset(m_blockZero.data, 0, sizeof(m_blockZero.data)); + memset(m_parentBlock.data, 0, sizeof(m_curBlock.data)); - m_hdrBlock.dirty = false; - m_curBlock.dirty = false; - m_nextBlock.dirty = false; - m_blockZero.dirty = false; - m_parentBlock.dirty = false; + m_hdrBlock.dirty = false; + m_curBlock.dirty = false; + m_nextBlock.dirty = false; + m_blockZero.dirty = false; + m_parentBlock.dirty = false; - m_hdrBlock.state = BLK_INIT; - m_curBlock.state = BLK_INIT; - m_nextBlock.state = BLK_INIT; - m_blockZero.state = BLK_INIT; - m_parentBlock.state = BLK_INIT; + m_hdrBlock.state = BLK_INIT; + m_curBlock.state = BLK_INIT; + m_nextBlock.state = BLK_INIT; + m_blockZero.state = BLK_INIT; + m_parentBlock.state = BLK_INIT; - //Get the header info if the header exist already - if (m_hdrLbid != (uint64_t)INVALID_LBID) - { - memset( &m_curIdxRidListHdr, 0, LIST_HDR_SIZE ); - rc = readSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, &m_curIdxRidListHdr ); - m_hdrBlock.dirty = true; - m_hdrBlock.lbid = m_hdrLbid; - m_hdrBlock.state = BLK_READ; - } + // Get the header info if the header exist already + if (m_hdrLbid != (uint64_t)INVALID_LBID) + { + memset(&m_curIdxRidListHdr, 0, LIST_HDR_SIZE); + rc = readSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + m_hdrBlock.dirty = true; + m_hdrBlock.lbid = m_hdrLbid; + m_hdrBlock.state = BLK_READ; + } - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for getting the last Fbo on header * ***************************************************************/ -const int IndexList::initGetHdr(const uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr) +const int IndexList::initGetHdr(const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr) { - int rc; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - rc = init(); - m_hdrLbid = curIdxRidListHdrPtr->fbo; - m_hdrSbid = curIdxRidListHdrPtr->sbid; - m_hdrEntry = curIdxRidListHdrPtr->entry; - memset( &m_curIdxRidListHdr, 0, LIST_HDR_SIZE ); - rc = readSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, &m_curIdxRidListHdr ); - m_hdrBlock.dirty = true; - m_hdrBlock.lbid = m_hdrLbid; - m_hdrBlock.state = BLK_READ; + rc = init(); + m_hdrLbid = curIdxRidListHdrPtr->fbo; + m_hdrSbid = curIdxRidListHdrPtr->sbid; + m_hdrEntry = curIdxRidListHdrPtr->entry; + memset(&m_curIdxRidListHdr, 0, LIST_HDR_SIZE); + rc = readSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + m_hdrBlock.dirty = true; + m_hdrBlock.lbid = m_hdrLbid; + m_hdrBlock.state = BLK_READ; - if (m_curIdxRidListHdr.nextIdxRidListPtr.type == (int)LIST_SUBBLOCK_TYPE) - rc = getLastLbid(); - else - m_lastLbid = INVALID_LBID; + if (m_curIdxRidListHdr.nextIdxRidListPtr.type == (int)LIST_SUBBLOCK_TYPE) + rc = getLastLbid(); + else + m_lastLbid = INVALID_LBID; - return rc; + return rc; } /**************************************************************** -* DESCRIPTION: -* Private Function for getting the header -* -***************************************************************/ -const int IndexList::getHdrInfo(IdxEmptyListEntry* curIdxRidListHdrPtr) + * DESCRIPTION: + * Private Function for getting the header + * + ***************************************************************/ +const int IndexList::getHdrInfo(IdxEmptyListEntry* curIdxRidListHdrPtr) { - int rc = NO_ERROR; - CommBlock cb; + int rc = NO_ERROR; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - //Get the Header block, sub-block and entry info from Index Tree + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + // Get the Header block, sub-block and entry info from Index Tree - m_hdrLbid = curIdxRidListHdrPtr->fbo; - m_hdrSbid = curIdxRidListHdrPtr->sbid; - m_hdrEntry = curIdxRidListHdrPtr->entry; + m_hdrLbid = curIdxRidListHdrPtr->fbo; + m_hdrSbid = curIdxRidListHdrPtr->sbid; + m_hdrEntry = curIdxRidListHdrPtr->entry; - memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - m_hdrBlock.dirty = false; - m_hdrBlock.state = BLK_INIT; - m_hdrBlock.no = INVALID_NUM; - m_hdrBlock.lbid = INVALID_LBID; - //header is 4 entries LIST_HDR_SIZE bytes - memset( &m_curIdxRidListHdr, 0, LIST_HDR_SIZE ); - //Get the old header out - rc = readDBFile( cb, &m_hdrBlock, m_hdrLbid ); + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); + m_hdrBlock.dirty = false; + m_hdrBlock.state = BLK_INIT; + m_hdrBlock.no = INVALID_NUM; + m_hdrBlock.lbid = INVALID_LBID; + // header is 4 entries LIST_HDR_SIZE bytes + memset(&m_curIdxRidListHdr, 0, LIST_HDR_SIZE); + // Get the old header out + rc = readDBFile(cb, &m_hdrBlock, m_hdrLbid); - m_hdrBlock.dirty = true; - m_hdrBlock.state = BLK_READ; - m_hdrBlock.lbid = m_hdrLbid; - getSubBlockEntry( m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); + m_hdrBlock.dirty = true; + m_hdrBlock.state = BLK_READ; + m_hdrBlock.lbid = m_hdrLbid; + getSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); - if (m_curIdxRidListHdr.nextIdxRidListPtr.type == (int)LIST_SUBBLOCK_TYPE) - rc = getLastLbid(); - else - m_lastLbid = INVALID_LBID; + if (m_curIdxRidListHdr.nextIdxRidListPtr.type == (int)LIST_SUBBLOCK_TYPE) + rc = getLastLbid(); + else + m_lastLbid = INVALID_LBID; - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for getting a segment for LIST * ***************************************************************/ -const int IndexList::resetBlk(DataBlock* dataBlk) +const int IndexList::resetBlk(DataBlock* dataBlk) { - memset( dataBlk->data, 0, sizeof(dataBlk->data)); - dataBlk->dirty = false; - dataBlk->state = BLK_INIT; - dataBlk->lbid = INVALID_LBID; - return NO_ERROR; + memset(dataBlk->data, 0, sizeof(dataBlk->data)); + dataBlk->dirty = false; + dataBlk->state = BLK_INIT; + dataBlk->lbid = INVALID_LBID; + return NO_ERROR; } /**************************************************************** * DESCRIPTION: @@ -223,104 +220,99 @@ const int IndexList::resetBlk(DataBlock* dataBlk) * either it is 0 or get it from BRM * ***************************************************************/ -const int IndexList::resetBlkZero(uint64_t& lbid0) +const int IndexList::resetBlkZero(uint64_t& lbid0) { - int rc = NO_ERROR; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc = NO_ERROR; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - memset( m_blockZero.data, 0, sizeof(m_blockZero.data)); - m_blockZero.dirty = false; - m_blockZero.state = BLK_INIT; + memset(m_blockZero.data, 0, sizeof(m_blockZero.data)); + m_blockZero.dirty = false; + m_blockZero.state = BLK_INIT; #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - rc = BRMWrapper::getInstance()->getBrmInfo( m_oid, 0, lbid0 ); + rc = BRMWrapper::getInstance()->getBrmInfo(m_oid, 0, lbid0); #endif - if (rc != NO_ERROR) - return rc; - - rc = readDBFile( cb, m_blockZero.data, lbid0 ); - m_blockZero.lbid = lbid0; - m_blockZero.state = BLK_READ; - + if (rc != NO_ERROR) return rc; + + rc = readDBFile(cb, m_blockZero.data, lbid0); + m_blockZero.lbid = lbid0; + m_blockZero.state = BLK_READ; + + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for writing block zero * ***************************************************************/ -const int IndexList::writeBlkZero(uint64_t& lbid0) +const int IndexList::writeBlkZero(uint64_t& lbid0) { - int rc; - CommBlock cb; + int rc; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - rc = writeDBFile( cb, m_blockZero.data, lbid0 ); - memset( m_blockZero.data, 0, sizeof(m_blockZero.data)); - m_blockZero.dirty = false; - m_blockZero.state = BLK_INIT; - return rc; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + rc = writeDBFile(cb, m_blockZero.data, lbid0); + memset(m_blockZero.data, 0, sizeof(m_blockZero.data)); + m_blockZero.dirty = false; + m_blockZero.state = BLK_INIT; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for getting a segment for LIST * ***************************************************************/ -const int IndexList::getSegment( FILE* pFile, - const IdxTreeGroupType segmentType, - IdxEmptyListEntry* assignPtr ) +const int IndexList::getSegment(FILE* pFile, const IdxTreeGroupType segmentType, IdxEmptyListEntry* assignPtr) { - int rc = ERR_IDX_LIST_GET_SEGMT; - CommBlock cb; - uint64_t lbid0; + int rc = ERR_IDX_LIST_GET_SEGMT; + CommBlock cb; + uint64_t lbid0; - m_pFile = pFile; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + m_pFile = pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - /* - DataBlock tmpBlock; - IdxEmptyListEntry tmpEntry; - memset(tmpBlock.data,0, 8192); - rc = readSubBlockEntry( pFile, &tmpBlock, 1492798, 31, 31, 8, &tmpEntry ); - cout << "common273->tmpEntry.fbo=" << tmpEntry.fbo << " tmpEntry.sbid=" - << tmpEntry.sbid << " tmpEntry.entry=" << tmpEntry.entry << endl; - */ - if (m_hdrLbid != INVALID_LBID) - { - setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - m_hdrBlock.state = BLK_WRITE; - } + /* + DataBlock tmpBlock; + IdxEmptyListEntry tmpEntry; + memset(tmpBlock.data,0, 8192); + rc = readSubBlockEntry( pFile, &tmpBlock, 1492798, 31, 31, 8, &tmpEntry ); + cout << "common273->tmpEntry.fbo=" << tmpEntry.fbo << " tmpEntry.sbid=" + << tmpEntry.sbid << " tmpEntry.entry=" << tmpEntry.entry << endl; + */ + if (m_hdrLbid != INVALID_LBID) + { + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + m_hdrBlock.state = BLK_WRITE; + } - //write everything out before calling free space manager - rc = updateIndexListWrite(); - rc = resetBlkZero(lbid0); - rc = m_freemgr.assignSegment( cb, &m_blockZero, LIST, segmentType, - assignPtr); + // write everything out before calling free space manager + rc = updateIndexListWrite(); + rc = resetBlkZero(lbid0); + rc = m_freemgr.assignSegment(cb, &m_blockZero, LIST, segmentType, assignPtr); - if (rc != NO_ERROR) - return rc; - - rc = writeBlkZero(lbid0); - - if (segmentType == ENTRY_4) - { - m_hdrLbid = assignPtr->fbo; - m_hdrSbid = assignPtr->sbid; - m_hdrEntry = assignPtr->entry; - return rc; - } - - //get the header back for sure - //recover other blocks as it goes - rc = initBlksGetHdrBlk(); + if (rc != NO_ERROR) return rc; + + rc = writeBlkZero(lbid0); + + if (segmentType == ENTRY_4) + { + m_hdrLbid = assignPtr->fbo; + m_hdrSbid = assignPtr->sbid; + m_hdrEntry = assignPtr->entry; + return rc; + } + + // get the header back for sure + // recover other blocks as it goes + rc = initBlksGetHdrBlk(); + return rc; } /**************************************************************** * DESCRIPTION: @@ -328,188 +320,177 @@ const int IndexList::getSegment( FILE* pFile, * Only block can be released, not subblock or header * ***************************************************************/ -const int IndexList::releaseSegment() +const int IndexList::releaseSegment() { - IdxEmptyListEntry releasePtrEntry; - int entryType = 0; - int rc; - uint64_t lbid0; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + IdxEmptyListEntry releasePtrEntry; + int entryType = 0; + int rc; + uint64_t lbid0; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - memset(&releasePtrEntry, 0, sizeof(IdxEmptyListEntry)); - releasePtrEntry.fbo = m_lbid; - releasePtrEntry.sbid = 0; - releasePtrEntry.entry = 0; - releasePtrEntry.spare = 0; + memset(&releasePtrEntry, 0, sizeof(IdxEmptyListEntry)); + releasePtrEntry.fbo = m_lbid; + releasePtrEntry.sbid = 0; + releasePtrEntry.entry = 0; + releasePtrEntry.spare = 0; - entryType = ENTRY_BLK; - // The following is related to ptr - releasePtrEntry.spare2 = 0 ; - releasePtrEntry.type = EMPTY_PTR; - releasePtrEntry.group = entryType; - //release free manager - rc = resetBlkZero(lbid0); + entryType = ENTRY_BLK; + // The following is related to ptr + releasePtrEntry.spare2 = 0; + releasePtrEntry.type = EMPTY_PTR; + releasePtrEntry.group = entryType; + // release free manager + rc = resetBlkZero(lbid0); - rc = m_freemgr.releaseSegment( cb, &m_blockZero, LIST, - (const IdxTreeGroupType)entryType, - &releasePtrEntry ); - rc = writeBlkZero(lbid0); - return rc; -} -/**************************************************************** -* DESCRIPTION: -* Private Function for setting the last Fbo on header -* -***************************************************************/ -const int IndexList::setLastLbid( uint64_t& lastLbid) -{ - int rc = NO_ERROR; - uint64_t lbid; - int sbid, entry; - IdxRidListPtr lastFboListPtr; - - if (m_curIdxRidListHdr.nextIdxRidListPtr.type != (int)LIST_SUBBLOCK_TYPE) - { - m_lastLbid = INVALID_LBID; - return NO_ERROR; - } - - if (m_curType == LIST_SUBBLOCK_TYPE) - { - m_lastLbid = INVALID_LBID; - return NO_ERROR; - } - else - { - m_lastLbid = lastLbid; - } - - rc = getSubBlk(lbid, sbid, entry); - //First link - memset(&lastFboListPtr, 0, sizeof(lastFboListPtr)); - lastFboListPtr.type = LIST_BLOCK_TYPE ; - ((IdxEmptyListEntry*)&lastFboListPtr)->fbo = m_lastLbid; - ((IdxEmptyListEntry*)&lastFboListPtr)->sbid = 0; - ((IdxEmptyListEntry*)&lastFboListPtr)->entry = 0; - lastFboListPtr.spare = 0x0; - rc = setLastFboPtr(lbid, sbid, lastFboListPtr); - return rc; + rc = m_freemgr.releaseSegment(cb, &m_blockZero, LIST, (const IdxTreeGroupType)entryType, &releasePtrEntry); + rc = writeBlkZero(lbid0); + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::setLastFboPtr(uint64_t& lbid, int& sbid, - IdxRidListPtr& lastFboListPtr) +const int IndexList::setLastLbid(uint64_t& lastLbid) { - int rc = NO_ERROR; - CommBlock cb; - DataBlock dataBlock; - IdxRidListPtr oldFboListPtr; + int rc = NO_ERROR; + uint64_t lbid; + int sbid, entry; + IdxRidListPtr lastFboListPtr; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + if (m_curIdxRidListHdr.nextIdxRidListPtr.type != (int)LIST_SUBBLOCK_TYPE) + { + m_lastLbid = INVALID_LBID; + return NO_ERROR; + } - if ((m_hdrBlock.lbid == lbid) && (m_hdrBlock.state >= BLK_READ)) + if (m_curType == LIST_SUBBLOCK_TYPE) + { + m_lastLbid = INVALID_LBID; + return NO_ERROR; + } + else + { + m_lastLbid = lastLbid; + } + + rc = getSubBlk(lbid, sbid, entry); + // First link + memset(&lastFboListPtr, 0, sizeof(lastFboListPtr)); + lastFboListPtr.type = LIST_BLOCK_TYPE; + ((IdxEmptyListEntry*)&lastFboListPtr)->fbo = m_lastLbid; + ((IdxEmptyListEntry*)&lastFboListPtr)->sbid = 0; + ((IdxEmptyListEntry*)&lastFboListPtr)->entry = 0; + lastFboListPtr.spare = 0x0; + rc = setLastFboPtr(lbid, sbid, lastFboListPtr); + return rc; +} +/**************************************************************** + * DESCRIPTION: + * Private Function for setting the last Fbo on header + * + ***************************************************************/ +const int IndexList::setLastFboPtr(uint64_t& lbid, int& sbid, IdxRidListPtr& lastFboListPtr) +{ + int rc = NO_ERROR; + CommBlock cb; + DataBlock dataBlock; + IdxRidListPtr oldFboListPtr; + + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + + if ((m_hdrBlock.lbid == lbid) && (m_hdrBlock.state >= BLK_READ)) + { + setSubBlockEntry(m_hdrBlock.data, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &lastFboListPtr); + m_hdrBlock.state = BLK_WRITE; + } + else if ((m_curBlock.lbid == lbid) && (m_curBlock.state >= BLK_READ)) + { + setSubBlockEntry(m_curBlock.data, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &lastFboListPtr); + m_curBlock.state = BLK_WRITE; + } + else + { + memset(dataBlock.data, 0, sizeof(dataBlock.data)); + rc = readSubBlockEntry(cb, &dataBlock, lbid, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &oldFboListPtr); + + if (((IdxEmptyListEntry*)&oldFboListPtr)->fbo != m_lastLbid) { - setSubBlockEntry(m_hdrBlock.data, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &lastFboListPtr); - m_hdrBlock.state = BLK_WRITE; - } - else if ((m_curBlock.lbid == lbid) && (m_curBlock.state >= BLK_READ)) - { - setSubBlockEntry(m_curBlock.data, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &lastFboListPtr); - m_curBlock.state = BLK_WRITE; + setSubBlockEntry(dataBlock.data, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &lastFboListPtr); + rc = writeDBFile(cb, dataBlock.data, lbid); } else - { - memset( dataBlock.data, 0, sizeof(dataBlock.data)); - rc = readSubBlockEntry(cb, &dataBlock, lbid, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &oldFboListPtr); + return NO_ERROR; + } - if (((IdxEmptyListEntry*)&oldFboListPtr)->fbo != m_lastLbid) - { - setSubBlockEntry(dataBlock.data, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &lastFboListPtr); - rc = writeDBFile(cb, dataBlock.data, lbid); - } - else - return NO_ERROR; - } - - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for getting the last Fbo on header * ***************************************************************/ -const int IndexList::getLastLbid() +const int IndexList::getLastLbid() { - int rc = NO_ERROR; - uint64_t lbid; - int sbid, entry; - IdxRidListPtr lastFboListPtr; + int rc = NO_ERROR; + uint64_t lbid; + int sbid, entry; + IdxRidListPtr lastFboListPtr; - if (m_curIdxRidListHdr.nextIdxRidListPtr.type != (int)LIST_SUBBLOCK_TYPE) - { - m_lastLbid = INVALID_LBID; - return NO_ERROR; - } + if (m_curIdxRidListHdr.nextIdxRidListPtr.type != (int)LIST_SUBBLOCK_TYPE) + { + m_lastLbid = INVALID_LBID; + return NO_ERROR; + } - rc = getSubBlk(lbid, sbid, entry); + rc = getSubBlk(lbid, sbid, entry); - //First link - rc = getLastFboPtr(lbid, sbid, lastFboListPtr); + // First link + rc = getLastFboPtr(lbid, sbid, lastFboListPtr); - if (lastFboListPtr.type == (int)LIST_BLOCK_TYPE) - { - m_lastLbid = ((IdxEmptyListEntry*)&lastFboListPtr)->fbo; - } - else - m_lastLbid = INVALID_LBID; + if (lastFboListPtr.type == (int)LIST_BLOCK_TYPE) + { + m_lastLbid = ((IdxEmptyListEntry*)&lastFboListPtr)->fbo; + } + else + m_lastLbid = INVALID_LBID; - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::getLastFboPtr(uint64_t& lbid, int& sbid, - IdxRidListPtr& lastFboListPtr) +const int IndexList::getLastFboPtr(uint64_t& lbid, int& sbid, IdxRidListPtr& lastFboListPtr) { - int rc = NO_ERROR; - CommBlock cb; - DataBlock dataBlock; + int rc = NO_ERROR; + CommBlock cb; + DataBlock dataBlock; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - if ((m_hdrBlock.lbid == lbid) && (m_hdrBlock.state >= BLK_READ)) - { - getSubBlockEntry(m_hdrBlock.data, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &lastFboListPtr); - } - else if ((m_curBlock.lbid == lbid) && (m_curBlock.state >= BLK_READ)) - { - getSubBlockEntry(m_curBlock.data, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &lastFboListPtr); - } - else - { - memset( dataBlock.data, 0, sizeof(dataBlock.data)); - dataBlock.dirty = false; - dataBlock.state = BLK_INIT; - rc = readSubBlockEntry(cb, &dataBlock, lbid, sbid, LIST_LAST_LBID_POS, - LIST_ENTRY_WIDTH, &lastFboListPtr); - } + if ((m_hdrBlock.lbid == lbid) && (m_hdrBlock.state >= BLK_READ)) + { + getSubBlockEntry(m_hdrBlock.data, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &lastFboListPtr); + } + else if ((m_curBlock.lbid == lbid) && (m_curBlock.state >= BLK_READ)) + { + getSubBlockEntry(m_curBlock.data, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &lastFboListPtr); + } + else + { + memset(dataBlock.data, 0, sizeof(dataBlock.data)); + dataBlock.dirty = false; + dataBlock.state = BLK_INIT; + rc = readSubBlockEntry(cb, &dataBlock, lbid, sbid, LIST_LAST_LBID_POS, LIST_ENTRY_WIDTH, &lastFboListPtr); + } - return rc; + return rc; } /**************************************************************** @@ -517,76 +498,68 @@ const int IndexList::getLastFboPtr(uint64_t& lbid, int& sbid, * Private Function for getting the sub block * ***************************************************************/ -const int IndexList::getSubBlk() +const int IndexList::getSubBlk() { - int rc = NO_ERROR; - uint64_t lbid; - int sbid, entry; - RID rowIdArray[ENTRY_PER_SUBBLOCK]; - DataBlock dataBlk; - CommBlock cb; + int rc = NO_ERROR; + uint64_t lbid; + int sbid, entry; + RID rowIdArray[ENTRY_PER_SUBBLOCK]; + DataBlock dataBlk; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - memset( dataBlk.data, 0, sizeof(dataBlk.data)); - rc = getSubBlk(lbid, sbid, entry); + memset(dataBlk.data, 0, sizeof(dataBlk.data)); + rc = getSubBlk(lbid, sbid, entry); - if (lbid != INVALID_LBID) - rc = readSubBlockEntry(cb, &dataBlk, lbid, - sbid, 0, BYTE_PER_SUBBLOCK, - rowIdArray); + if (lbid != INVALID_LBID) + rc = readSubBlockEntry(cb, &dataBlk, lbid, sbid, 0, BYTE_PER_SUBBLOCK, rowIdArray); - return rc; + return rc; } /**************************************************************** -* DESCRIPTION: -* Private Function for getting the sub block -* -***************************************************************/ -const int IndexList::getSubBlk(uint64_t& lbid, int& sbid, int& entry) + * DESCRIPTION: + * Private Function for getting the sub block + * + ***************************************************************/ +const int IndexList::getSubBlk(uint64_t& lbid, int& sbid, int& entry) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (m_curIdxRidListHdr.nextIdxRidListPtr.type == (int)LIST_SUBBLOCK_TYPE) - { + if (m_curIdxRidListHdr.nextIdxRidListPtr.type == (int)LIST_SUBBLOCK_TYPE) + { + lbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; + sbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; + entry = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->entry; + } + else + { + lbid = INVALID_LBID; + sbid = INVALID_NUM; + entry = INVALID_NUM; + return ERR_IDX_LIST_GET_SUB_BLK; + } - lbid = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; - sbid = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; - entry = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->entry; - } - else - { - lbid = INVALID_LBID; - sbid = INVALID_NUM; - entry = INVALID_NUM; - return ERR_IDX_LIST_GET_SUB_BLK; - } - - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for getting a particular block * ***************************************************************/ -const int IndexList::getBlk(uint64_t& lbid) +const int IndexList::getBlk(uint64_t& lbid) { - int rc = NO_ERROR; - RID rowIdArray[MAX_BLOCK_ENTRY]; - DataBlock dataBlk; - CommBlock cb; - memset( dataBlk.data, 0, sizeof(dataBlk.data)); - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc = NO_ERROR; + RID rowIdArray[MAX_BLOCK_ENTRY]; + DataBlock dataBlk; + CommBlock cb; + memset(dataBlk.data, 0, sizeof(dataBlk.data)); + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - rc = readSubBlockEntry(cb, &dataBlk, lbid, - 0, 0, BYTE_PER_BLOCK, - rowIdArray); - return rc; + rc = readSubBlockEntry(cb, &dataBlk, lbid, 0, 0, BYTE_PER_BLOCK, rowIdArray); + return rc; } /**************************************************************** @@ -599,373 +572,371 @@ const int IndexList::getBlk(uint64_t& lbid) ***********************************************************/ const int IndexList::updateIndexListWrite() { - int rc = NO_ERROR; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc = NO_ERROR; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - //Now write in one place - if (m_hdrBlock.state == BLK_WRITE) - { - //A - rc = writeDBFile( cb, m_hdrBlock.data, m_hdrLbid ); + // Now write in one place + if (m_hdrBlock.state == BLK_WRITE) + { + // A + rc = writeDBFile(cb, m_hdrBlock.data, m_hdrLbid); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - m_hdrBlock.state = BLK_READ; - } + m_hdrBlock.state = BLK_READ; + } - if (m_curBlock.state == BLK_WRITE) - { - //B - rc = writeDBFile( cb, m_curBlock.data, m_lbid ); + if (m_curBlock.state == BLK_WRITE) + { + // B + rc = writeDBFile(cb, m_curBlock.data, m_lbid); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - m_curBlock.state = BLK_READ; - } + m_curBlock.state = BLK_READ; + } - if (m_parentBlock.state == BLK_WRITE) - { - rc = writeDBFile( cb, m_parentBlock.data, m_parentLbid ); + if (m_parentBlock.state == BLK_WRITE) + { + rc = writeDBFile(cb, m_parentBlock.data, m_parentLbid); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - m_parentBlock.state = BLK_READ; - } + m_parentBlock.state = BLK_READ; + } - if (m_nextBlock.state == BLK_WRITE) - { - rc = writeDBFile( cb, m_nextBlock.data, m_nextLbid ); + if (m_nextBlock.state == BLK_WRITE) + { + rc = writeDBFile(cb, m_nextBlock.data, m_nextLbid); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - m_nextBlock.state = BLK_READ; - } + m_nextBlock.state = BLK_READ; + } - return rc; + return rc; } /************************************************ -* Description: -* Find a entry for the given rowId and Key -* Converted -* input -* pFile -- File Handler -* rowId -- row id -* key -- value -* curIdxRidListHdrPtr - point to the header -* output -* lbid -- File block id -* sbid -- Sub Block id -* entry -- Entry id -* -* -* return value -* true --found -* false--not found -************************************************/ -const int IndexList::findFirstBlk(FILE* pFile, const uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr, - uint64_t& lbid) -{ - int rc; - CommBlock cb; - int count; - - cb.file.oid = m_oid; - cb.file.pFile = pFile; - - m_pFile = pFile; - rc = getHdrInfo(curIdxRidListHdrPtr); - - if (key != m_curIdxRidListHdr.key) - { - return ERR_IDX_LIST_INVALID_KEY;; - } - - int type = m_curIdxRidListHdr.nextIdxRidListPtr.type; - - switch (type) - { - case LIST_NOT_USED_TYPE://Header is not full, no sub-block linked - lbid = INVALID_LBID; - return NO_ERROR; //not found - - case LIST_RID_TYPE:// There is a row id here, Check! - lbid = INVALID_LBID; - return NO_ERROR; //not found - - case LIST_SUBBLOCK_TYPE://Not found in header - //get the lbid sbid and entry out from the header last entry - rc = getSubBlk(m_lbid, m_sbid, m_entry ); - m_curType = type; - - if (m_lbid != m_hdrLbid) - rc = readCurBlk(); - - rc = getNextInfoFromBlk(m_lastIdxRidListPtr); - count = m_lastIdxRidListPtr.count; //current count - type = m_lastIdxRidListPtr.type; //block type - - if (type == LIST_BLOCK_TYPE) - lbid = ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo; - else - lbid = INVALID_LBID; - - return NO_ERROR; - break; - - default: - //printf ("FIND FIRST BLOCK got no where, error out !! \n"); - break; - }; //end switch - - lbid = INVALID_LBID; - - return NO_ERROR; -} //end function - -/**************************************************************** - * DESCRIPTION: - * Private Function for setting the last Fbo on header + * Description: + * Find a entry for the given rowId and Key + * Converted + * input + * pFile -- File Handler + * rowId -- row id + * key -- value + * curIdxRidListHdrPtr - point to the header + * output + * lbid -- File block id + * sbid -- Sub Block id + * entry -- Entry id * - ***************************************************************/ -const int IndexList::readCurBlk() -{ - int rc = NO_ERROR; - CommBlock cb; - - if (m_lbid == m_hdrLbid) - return NO_ERROR; - - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - - if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state != BLK_INIT)) - return NO_ERROR; - - if (m_curBlock.state == BLK_WRITE) - rc = writeCurBlk(); - - if ((m_curBlock.state == BLK_INIT) || (m_curBlock.lbid != m_lbid)) - { - memset(m_curBlock.data, 0, sizeof(m_curBlock.data ) ); - rc = readDBFile(cb, m_curBlock.data, m_lbid); - - if (rc != NO_ERROR) - return rc; - - m_curBlock.dirty = true; - m_curBlock.lbid = m_lbid; - m_curBlock.state = BLK_READ; - } - - return rc; -} -/**************************************************************** - * DESCRIPTION: - * Private Function for setting the last Fbo on header * - ***************************************************************/ -const int IndexList::writeCurBlk() + * return value + * true --found + * false--not found + ************************************************/ +const int IndexList::findFirstBlk(FILE* pFile, const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr, + uint64_t& lbid) { - int rc; - CommBlock cb; + int rc; + CommBlock cb; + int count; - if (m_curBlock.state == BLK_WRITE) - { - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - rc = writeDBFile(cb, m_curBlock.data, m_curBlock.lbid ); - m_curBlock.state = BLK_READ; - return rc; - } - else - return NO_ERROR; -} -/************************************************ - * No Change - * RETURN: - * success - successfully created the index list header - * failure - it did not create the index list header - ***********************************************************/ -const int IndexList::updateLastPtrAndParent(const int lastCount) -{ - int rc = NO_ERROR; - rc = updateLastPtr(lastCount); - rc = updateParent(); - return rc; -} -/************************************************ - * No Change - * RETURN: - * success - successfully created the index list header - * failure - it did not create the index list header - ***********************************************************/ -const int IndexList::updateParent() -{ - int rc; + cb.file.oid = m_oid; + cb.file.pFile = pFile; - if (m_useNarray) - { - IdxRidParentListPtr parentIdxListPtr; - memset(&parentIdxListPtr, 0, sizeof(parentIdxListPtr)); - parentIdxListPtr.parentLbid = (uint64_t)INVALID_LBID; + m_pFile = pFile; + rc = getHdrInfo(curIdxRidListHdrPtr); - rc = getParentInfoFromArray(parentIdxListPtr); - m_parentLbid = (uint64_t)parentIdxListPtr.parentLbid; + if (key != m_curIdxRidListHdr.key) + { + return ERR_IDX_LIST_INVALID_KEY; + ; + } - //If this block has no parent, then it is the first block - if ((m_parentLbid <= (uint64_t)0) || (m_parentLbid == (uint64_t)INVALID_LBID)) - m_parentLbid = m_lbid; //It is truly itself + int type = m_curIdxRidListHdr.nextIdxRidListPtr.type; - //The previous lbid m_lbid is full, so we have to go to nextLbid - // and register it to the parent block - rc = updateParentStatus(m_nextLbid); - return rc; - } - else - return NO_ERROR; -} -/************************************************ - * No Change - * RETURN: - * success - successfully created the index list header - * failure - it did not create the index list header - ***********************************************************/ -const int IndexList::updateLastPtr(const int lastCount) -{ - int rc; + switch (type) + { + case LIST_NOT_USED_TYPE: // Header is not full, no sub-block linked + lbid = INVALID_LBID; + return NO_ERROR; // not found - if (m_curBlock.state == BLK_INIT) - readCurBlk(); + case LIST_RID_TYPE: // There is a row id here, Check! + lbid = INVALID_LBID; + return NO_ERROR; // not found - if (m_useNarray) - { - rc = setCurBlkNextPtr(m_nextLbid, lastCount); - rc = writeCurBlk(); - } - else - { - ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo = m_nextLbid; - ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->sbid = 0; - ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->entry = 0; - m_lastIdxRidListPtr.type = LIST_BLOCK_TYPE; - m_lastIdxRidListPtr.spare = 0x0; - m_lastIdxRidListPtr.count = lastCount; - - setSubBlockEntry( m_curBlock.data, m_sbid, - LIST_BLOCK_LLP_POS, LIST_ENTRY_WIDTH, - &m_lastIdxRidListPtr ); - m_curBlock.state = BLK_WRITE; - rc = writeCurBlk(); - } - - return rc; -} - -/************************************************ - * No Change - * RETURN: - * success - successfully created the index list header - * failure - it did not create the index list header - ***********************************************************/ -const int IndexList::addRidInBlk( const RID& newRid) -{ - int maxCount, count = 0; - int rc = NO_ERROR; - CommBlock cb; - - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - - if (m_useNarray) - maxCount = MAX_BLK_NARRAY_RID_CNT; - else - maxCount = MAX_BLK_RID_CNT; - - //Find the last block that has a space or - //No next block linked - rc = findLastBlk(count); - - if ((count == maxCount) && (m_nextType == LIST_SIZE_TYPE)) - { - //Full, also need a new segment - IdxEmptyListEntry newIdxListEntryPtr; - m_segType = LIST_BLOCK_TYPE; - rc = getSegment( m_pFile, ENTRY_BLK, &newIdxListEntryPtr); - m_nextLbid = ((IdxEmptyListEntry*)&newIdxListEntryPtr)->fbo; - m_nextType = m_segType; - m_lastIdxRidListPtr.llp = ((IdxRidListPtr*)&newIdxListEntryPtr)->llp; - rc = updateLastPtrAndParent(count); - //the new block for insertion and count record - m_lbid = m_nextLbid; - m_sbid = 0; - m_entry = 0; - m_curType = m_nextType; + case LIST_SUBBLOCK_TYPE: // Not found in header + // get the lbid sbid and entry out from the header last entry + rc = getSubBlk(m_lbid, m_sbid, m_entry); + m_curType = type; + if (m_lbid != m_hdrLbid) rc = readCurBlk(); - //free manager puts bad entry type at the last entry for new block - //clean it up! - IdxRidListPtr idxRidListPtr; - memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); - rc = setNextInfoFromBlk( idxRidListPtr); - if (m_useNarray) - { - rc = initCurBlock(); - } + rc = getNextInfoFromBlk(m_lastIdxRidListPtr); + count = m_lastIdxRidListPtr.count; // current count + type = m_lastIdxRidListPtr.type; // block type - //Set the count to the beginning - count = 0; - }//end if FULL get new segment + if (type == LIST_BLOCK_TYPE) + lbid = ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo; + else + lbid = INVALID_LBID; - // insert in the current block at the location - if (m_lastLbid != m_lbid) - { - rc = setLastLbid(m_lbid); - } + return NO_ERROR; + break; - rc = insertRid(newRid, count); - rc = updateCurCount(); - rc = updateHdrCount(); + default: + // printf ("FIND FIRST BLOCK got no where, error out !! \n"); + break; + }; // end switch + + lbid = INVALID_LBID; + + return NO_ERROR; +} // end function + +/**************************************************************** + * DESCRIPTION: + * Private Function for setting the last Fbo on header + * + ***************************************************************/ +const int IndexList::readCurBlk() +{ + int rc = NO_ERROR; + CommBlock cb; + + if (m_lbid == m_hdrLbid) + return NO_ERROR; + + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state != BLK_INIT)) + return NO_ERROR; + + if (m_curBlock.state == BLK_WRITE) + rc = writeCurBlk(); + + if ((m_curBlock.state == BLK_INIT) || (m_curBlock.lbid != m_lbid)) + { + memset(m_curBlock.data, 0, sizeof(m_curBlock.data)); + rc = readDBFile(cb, m_curBlock.data, m_lbid); + + if (rc != NO_ERROR) + return rc; + + m_curBlock.dirty = true; + m_curBlock.lbid = m_lbid; + m_curBlock.state = BLK_READ; + } + + return rc; +} +/**************************************************************** + * DESCRIPTION: + * Private Function for setting the last Fbo on header + * + ***************************************************************/ +const int IndexList::writeCurBlk() +{ + int rc; + CommBlock cb; + + if (m_curBlock.state == BLK_WRITE) + { + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + rc = writeDBFile(cb, m_curBlock.data, m_curBlock.lbid); + m_curBlock.state = BLK_READ; return rc; + } + else + return NO_ERROR; } /************************************************ -* No Change -* RETURN: -* success - successfully created the index list header -* failure - it did not create the index list header -***********************************************************/ -const int IndexList::findLastBlk(int& count) + * No Change + * RETURN: + * success - successfully created the index list header + * failure - it did not create the index list header + ***********************************************************/ +const int IndexList::updateLastPtrAndParent(const int lastCount) { - int maxCount; - int rc ; + int rc = NO_ERROR; + rc = updateLastPtr(lastCount); + rc = updateParent(); + return rc; +} +/************************************************ + * No Change + * RETURN: + * success - successfully created the index list header + * failure - it did not create the index list header + ***********************************************************/ +const int IndexList::updateParent() +{ + int rc; - if (m_useNarray) - maxCount = MAX_BLK_NARRAY_RID_CNT; - else - maxCount = MAX_BLK_RID_CNT; + if (m_useNarray) + { + IdxRidParentListPtr parentIdxListPtr; + memset(&parentIdxListPtr, 0, sizeof(parentIdxListPtr)); + parentIdxListPtr.parentLbid = (uint64_t)INVALID_LBID; + + rc = getParentInfoFromArray(parentIdxListPtr); + m_parentLbid = (uint64_t)parentIdxListPtr.parentLbid; + + // If this block has no parent, then it is the first block + if ((m_parentLbid <= (uint64_t)0) || (m_parentLbid == (uint64_t)INVALID_LBID)) + m_parentLbid = m_lbid; // It is truly itself + + // The previous lbid m_lbid is full, so we have to go to nextLbid + // and register it to the parent block + rc = updateParentStatus(m_nextLbid); + return rc; + } + else + return NO_ERROR; +} +/************************************************ + * No Change + * RETURN: + * success - successfully created the index list header + * failure - it did not create the index list header + ***********************************************************/ +const int IndexList::updateLastPtr(const int lastCount) +{ + int rc; + + if (m_curBlock.state == BLK_INIT) + readCurBlk(); + + if (m_useNarray) + { + rc = setCurBlkNextPtr(m_nextLbid, lastCount); + rc = writeCurBlk(); + } + else + { + ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo = m_nextLbid; + ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->sbid = 0; + ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->entry = 0; + m_lastIdxRidListPtr.type = LIST_BLOCK_TYPE; + m_lastIdxRidListPtr.spare = 0x0; + m_lastIdxRidListPtr.count = lastCount; + + setSubBlockEntry(m_curBlock.data, m_sbid, LIST_BLOCK_LLP_POS, LIST_ENTRY_WIDTH, &m_lastIdxRidListPtr); + m_curBlock.state = BLK_WRITE; + rc = writeCurBlk(); + } + + return rc; +} + +/************************************************ + * No Change + * RETURN: + * success - successfully created the index list header + * failure - it did not create the index list header + ***********************************************************/ +const int IndexList::addRidInBlk(const RID& newRid) +{ + int maxCount, count = 0; + int rc = NO_ERROR; + CommBlock cb; + + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + + if (m_useNarray) + maxCount = MAX_BLK_NARRAY_RID_CNT; + else + maxCount = MAX_BLK_RID_CNT; + + // Find the last block that has a space or + // No next block linked + rc = findLastBlk(count); + + if ((count == maxCount) && (m_nextType == LIST_SIZE_TYPE)) + { + // Full, also need a new segment + IdxEmptyListEntry newIdxListEntryPtr; + m_segType = LIST_BLOCK_TYPE; + rc = getSegment(m_pFile, ENTRY_BLK, &newIdxListEntryPtr); + m_nextLbid = ((IdxEmptyListEntry*)&newIdxListEntryPtr)->fbo; + m_nextType = m_segType; + m_lastIdxRidListPtr.llp = ((IdxRidListPtr*)&newIdxListEntryPtr)->llp; + rc = updateLastPtrAndParent(count); + // the new block for insertion and count record + m_lbid = m_nextLbid; + m_sbid = 0; + m_entry = 0; + m_curType = m_nextType; rc = readCurBlk(); - rc = getNextInfo(count); + // free manager puts bad entry type at the last entry for new block + // clean it up! + IdxRidListPtr idxRidListPtr; + memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); + rc = setNextInfoFromBlk(idxRidListPtr); - while ((count == maxCount) && (m_nextType == LIST_BLOCK_TYPE)) + if (m_useNarray) { - //current is a Full link, No space to insert, go to the next one - m_lbid = m_nextLbid; - m_sbid = 0; - m_entry = 0; - rc = readCurBlk(); - rc = getNextInfo(count); - }//end of while + rc = initCurBlock(); + } - return rc; + // Set the count to the beginning + count = 0; + } // end if FULL get new segment + + // insert in the current block at the location + if (m_lastLbid != m_lbid) + { + rc = setLastLbid(m_lbid); + } + + rc = insertRid(newRid, count); + rc = updateCurCount(); + rc = updateHdrCount(); + return rc; +} +/************************************************ + * No Change + * RETURN: + * success - successfully created the index list header + * failure - it did not create the index list header + ***********************************************************/ +const int IndexList::findLastBlk(int& count) +{ + int maxCount; + int rc; + + if (m_useNarray) + maxCount = MAX_BLK_NARRAY_RID_CNT; + else + maxCount = MAX_BLK_RID_CNT; + + rc = readCurBlk(); + rc = getNextInfo(count); + + while ((count == maxCount) && (m_nextType == LIST_BLOCK_TYPE)) + { + // current is a Full link, No space to insert, go to the next one + m_lbid = m_nextLbid; + m_sbid = 0; + m_entry = 0; + rc = readCurBlk(); + rc = getNextInfo(count); + } // end of while + + return rc; } /************************************************ * No Change @@ -975,39 +946,36 @@ const int IndexList::findLastBlk(int& count) ***********************************************************/ const int IndexList::insertRid(const RID& newRid, int& pos) { - int rc = NO_ERROR; - IdxRidListEntry idxRidListEntry; + int rc = NO_ERROR; + IdxRidListEntry idxRidListEntry; - if (m_curType == LIST_BLOCK_TYPE) - { - m_sbid = 0; - m_entry = 0; - } + if (m_curType == LIST_BLOCK_TYPE) + { + m_sbid = 0; + m_entry = 0; + } - memset(&idxRidListEntry, 0, LIST_ENTRY_WIDTH) ; + memset(&idxRidListEntry, 0, LIST_ENTRY_WIDTH); - idxRidListEntry.type = LIST_RID_TYPE; - idxRidListEntry.spare = 0; - //cout << "line 910:newRid->" << newRid << endl; + idxRidListEntry.type = LIST_RID_TYPE; + idxRidListEntry.spare = 0; + // cout << "line 910:newRid->" << newRid << endl; - idxRidListEntry.rid = newRid; + idxRidListEntry.rid = newRid; - if (m_lbid != m_hdrLbid) - { - rc = readCurBlk(); - setSubBlockEntry( m_curBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, - &idxRidListEntry ); - m_curBlock.state = BLK_WRITE; + if (m_lbid != m_hdrLbid) + { + rc = readCurBlk(); + setSubBlockEntry(m_curBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &idxRidListEntry); + m_curBlock.state = BLK_WRITE; + } + else + { + setSubBlockEntry(m_hdrBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &idxRidListEntry); + m_hdrBlock.state = BLK_WRITE; + } - } - else - { - setSubBlockEntry( m_hdrBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, - &idxRidListEntry ); - m_hdrBlock.state = BLK_WRITE; - } - - return rc; + return rc; } /************************************************ * No Change @@ -1017,146 +985,139 @@ const int IndexList::insertRid(const RID& newRid, int& pos) ***********************************************************/ const int IndexList::updateCurCount(int frequency) { - int rc = NO_ERROR; - int pos = 0; + int rc = NO_ERROR; + int pos = 0; - if (m_curType == LIST_SUBBLOCK_TYPE) - pos = LIST_SUB_LLP_POS; - else if (m_curType == LIST_BLOCK_TYPE) + if (m_curType == LIST_SUBBLOCK_TYPE) + pos = LIST_SUB_LLP_POS; + else if (m_curType == LIST_BLOCK_TYPE) + { + pos = LIST_BLOCK_LLP_POS; + m_sbid = 0; + m_entry = 0; + } + + if ((m_useNarray) && (m_curType == LIST_BLOCK_TYPE)) + rc = updateCurCountInArray(); + else + { + rc = getNextInfoFromBlk(m_lastIdxRidListPtr); + + for (int i = 0; i < frequency; i++) + m_lastIdxRidListPtr.count++; + + if (m_lbid == m_hdrLbid) { - pos = LIST_BLOCK_LLP_POS; - m_sbid = 0; - m_entry = 0; + setSubBlockEntry(m_hdrBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &m_lastIdxRidListPtr); + m_hdrBlock.state = BLK_WRITE; } - - if ((m_useNarray) && (m_curType == LIST_BLOCK_TYPE)) - rc = updateCurCountInArray(); else { - rc = getNextInfoFromBlk(m_lastIdxRidListPtr); + setSubBlockEntry(m_curBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &m_lastIdxRidListPtr); + m_curBlock.state = BLK_WRITE; + }; + } - for (int i = 0; i < frequency; i++) - m_lastIdxRidListPtr.count++; - - if (m_lbid == m_hdrLbid) - { - setSubBlockEntry( m_hdrBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, - &m_lastIdxRidListPtr ); - m_hdrBlock.state = BLK_WRITE; - } - else - { - setSubBlockEntry(m_curBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, - &m_lastIdxRidListPtr ); - m_curBlock.state = BLK_WRITE; - }; - } - - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::getNextInfo(int& count) +const int IndexList::getNextInfo(int& count) { - int rc; + int rc; - if ((!m_useNarray) || (m_curType == LIST_SUBBLOCK_TYPE )) - { - rc = getNextInfoFromBlk(m_lastIdxRidListPtr); - count = m_lastIdxRidListPtr.count; - m_nextType = m_lastIdxRidListPtr.type; - m_nextLbid = ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo; - } - else - { - rc = getNextInfoFromArray(m_nextIdxListPtr ); - count = m_nextIdxListPtr.count; - m_nextType = m_nextIdxListPtr.type; - m_nextLbid = m_nextIdxListPtr.nextLbid; - } + if ((!m_useNarray) || (m_curType == LIST_SUBBLOCK_TYPE)) + { + rc = getNextInfoFromBlk(m_lastIdxRidListPtr); + count = m_lastIdxRidListPtr.count; + m_nextType = m_lastIdxRidListPtr.type; + m_nextLbid = ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo; + } + else + { + rc = getNextInfoFromArray(m_nextIdxListPtr); + count = m_nextIdxListPtr.count; + m_nextType = m_nextIdxListPtr.type; + m_nextLbid = m_nextIdxListPtr.nextLbid; + } - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function non-array * ***************************************************************/ -const int IndexList::getNextInfoFromBlk( IdxRidListPtr& idxRidListPtr) +const int IndexList::getNextInfoFromBlk(IdxRidListPtr& idxRidListPtr) { - int rc = NO_ERROR; - int pos = 0; + int rc = NO_ERROR; + int pos = 0; + memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); + + if (m_curType == LIST_SUBBLOCK_TYPE) + pos = LIST_SUB_LLP_POS; + else if (m_curType == LIST_BLOCK_TYPE) + pos = LIST_BLOCK_LLP_POS; + + if ((m_hdrBlock.lbid == m_lbid) && (m_hdrBlock.state >= BLK_READ)) + getSubBlockEntry(m_hdrBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &idxRidListPtr); + else if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + getSubBlockEntry(m_curBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &idxRidListPtr); + else + return ERR_IDX_LIST_WRONG_BLK; + + if ((idxRidListPtr.type != 0) && (idxRidListPtr.type != 4) && (idxRidListPtr.type != 5)) + { + // cout << "line 1028->m_lbid=" << m_lbid << " m_sbid =" << m_sbid << " m_entry" <idxRidListPtr.type = " << idxRidListPtr.type << endl; memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); + } - if (m_curType == LIST_SUBBLOCK_TYPE) - pos = LIST_SUB_LLP_POS; - else if (m_curType == LIST_BLOCK_TYPE) - pos = LIST_BLOCK_LLP_POS; - - if ((m_hdrBlock.lbid == m_lbid) && (m_hdrBlock.state >= BLK_READ)) - getSubBlockEntry(m_hdrBlock.data, m_sbid, pos, - LIST_ENTRY_WIDTH, &idxRidListPtr); - else if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) - getSubBlockEntry(m_curBlock.data, m_sbid, pos, - LIST_ENTRY_WIDTH, &idxRidListPtr); - else - return ERR_IDX_LIST_WRONG_BLK; - - if ((idxRidListPtr.type != 0) && (idxRidListPtr.type != 4) && (idxRidListPtr.type != 5)) - { - //cout << "line 1028->m_lbid=" << m_lbid << " m_sbid =" << m_sbid << " m_entry" <idxRidListPtr.type = " << idxRidListPtr.type << endl; - memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); - } - - return rc; + return rc; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::setNextInfoFromBlk( IdxRidListPtr& idxRidListPtr) +const int IndexList::setNextInfoFromBlk(IdxRidListPtr& idxRidListPtr) { - int rc = NO_ERROR; - CommBlock cb; + int rc = NO_ERROR; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - int pos = 0; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + int pos = 0; - if (m_curType == LIST_SUBBLOCK_TYPE) - pos = LIST_SUB_LLP_POS; - else if (m_curType == LIST_BLOCK_TYPE) - pos = LIST_BLOCK_LLP_POS; + if (m_curType == LIST_SUBBLOCK_TYPE) + pos = LIST_SUB_LLP_POS; + else if (m_curType == LIST_BLOCK_TYPE) + pos = LIST_BLOCK_LLP_POS; - if (m_lbid == m_hdrLbid) + if (m_lbid == m_hdrLbid) + { + // when sub == hdr + setSubBlockEntry(m_hdrBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &idxRidListPtr); + m_hdrBlock.state = BLK_WRITE; + } + else + { + readCurBlk(); + + if (m_lbid == m_curBlock.lbid) { - //when sub == hdr - setSubBlockEntry( m_hdrBlock.data, m_sbid, - pos, LIST_ENTRY_WIDTH, &idxRidListPtr ); - m_hdrBlock.state = BLK_WRITE; + setSubBlockEntry(m_curBlock.data, m_sbid, pos, LIST_ENTRY_WIDTH, &idxRidListPtr); + rc = writeDBFile(cb, m_curBlock.data, m_lbid); + m_curBlock.state = BLK_READ; } else - { - readCurBlk(); + return ERR_IDX_LIST_WRONG_LBID_WRITE; + } - if (m_lbid == m_curBlock.lbid) - { - setSubBlockEntry( m_curBlock.data, m_sbid, - pos, LIST_ENTRY_WIDTH, - &idxRidListPtr ); - rc = writeDBFile(cb, m_curBlock.data, m_lbid ); - m_curBlock.state = BLK_READ; - } - else - return ERR_IDX_LIST_WRONG_LBID_WRITE; - } - - return rc; + return rc; } -}//end namespace +} // namespace WriteEngine diff --git a/writeengine/index/we_indexlist_find_delete.cpp b/writeengine/index/we_indexlist_find_delete.cpp index b227bebe7..e90bf02d1 100644 --- a/writeengine/index/we_indexlist_find_delete.cpp +++ b/writeengine/index/we_indexlist_find_delete.cpp @@ -31,12 +31,10 @@ #include "we_indextree.h" #include "we_indexlist.h" - using namespace std; namespace WriteEngine { - /************************************************ * Description: * Find a entry for the given rowId and Key @@ -55,443 +53,401 @@ namespace WriteEngine * Success -- 0 * Fail -- ERR_IDX_LIST_INVALID_DELETE ************************************************/ -const int IndexList::deleteInSub( const RID& rowId) +const int IndexList::deleteInSub(const RID& rowId) { + int rc = ERR_IDX_LIST_INVALID_DELETE; + DataBlock prevDataBlock; + int pos = 0, totalbytes = 0; + IdxRidListPtr* lastIdxRidListPtr; + int type; - int rc = ERR_IDX_LIST_INVALID_DELETE; - DataBlock prevDataBlock; - int pos = 0, totalbytes = 0; - IdxRidListPtr* lastIdxRidListPtr; - int type; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + // get thelbid sbid and entry out from the header last entry + // First Sub-block + m_lbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; + m_sbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; + m_entry = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->entry; + // Read the pointer entry at LIST_SUB_LLP_POS location - //get thelbid sbid and entry out from the header last entry - //First Sub-block - m_lbid = ((IdxEmptyListEntry*) & (m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; - m_sbid = ((IdxEmptyListEntry*) & (m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; - m_entry = ((IdxEmptyListEntry*) & (m_curIdxRidListHdr.nextIdxRidListPtr))->entry; - //Read the pointer entry at LIST_SUB_LLP_POS location + IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; + IdxRidListEntry newRowIdArray[MAX_BLOCK_ENTRY]; + memset(rowIdArray, 0, BYTE_PER_BLOCK); + memset(newRowIdArray, 0, BYTE_PER_BLOCK); + // First link + pos = LIST_SUB_LLP_POS; + totalbytes = SUBBLOCK_TOTAL_BYTES; + m_entryGroup = ENTRY_32; - IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; - IdxRidListEntry newRowIdArray[MAX_BLOCK_ENTRY]; - memset(rowIdArray, 0, BYTE_PER_BLOCK); - memset(newRowIdArray, 0, BYTE_PER_BLOCK); - //First link - pos = LIST_SUB_LLP_POS; - totalbytes = SUBBLOCK_TOTAL_BYTES; - m_entryGroup = ENTRY_32; + if (m_lbid != m_hdrLbid) + { + rc = readDBFile(cb, &m_curBlock, m_lbid); - if (m_lbid != m_hdrLbid) - { - rc = readDBFile(cb, &m_curBlock, m_lbid ); + if (rc != NO_ERROR) + return rc; - if (rc != NO_ERROR) - return rc; + rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, m_sbid, 0, totalbytes, rowIdArray); - rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, - m_sbid, 0, totalbytes, - rowIdArray); + if (rc != NO_ERROR) + return rc; - if (rc != NO_ERROR) - return rc; - - m_curBlock.dirty = true; - m_curBlock.lbid = m_lbid; - m_curBlock.state = BLK_READ; - } + m_curBlock.dirty = true; + m_curBlock.lbid = m_lbid; + m_curBlock.state = BLK_READ; + } + else + { + if (m_hdrBlock.state >= BLK_READ) + getSubBlockEntry(m_hdrBlock.data, m_sbid, 0, totalbytes, rowIdArray); else - { - if (m_hdrBlock.state >= BLK_READ) - getSubBlockEntry(m_hdrBlock.data, m_sbid, - 0, totalbytes, rowIdArray ); - else - return ERR_IDX_LIST_INVALID_DELETE; - } + return ERR_IDX_LIST_INVALID_DELETE; + } - lastIdxRidListPtr = (IdxRidListPtr*)&rowIdArray[pos]; - int count; - type = lastIdxRidListPtr->type; //next type - count = lastIdxRidListPtr->count;//current count + lastIdxRidListPtr = (IdxRidListPtr*)&rowIdArray[pos]; + int count; + type = lastIdxRidListPtr->type; // next type + count = lastIdxRidListPtr->count; // current count - for (int i = 0; i < count; i++) + for (int i = 0; i < count; i++) + { + if (rowIdArray[i].rid == rowId) { - if (rowIdArray[i].rid == rowId) + // found + m_dLbid = m_lbid; + m_dSbid = m_sbid; + m_dEntry = i; + rc = NO_ERROR; + memcpy(&newRowIdArray[0], &rowIdArray[0], totalbytes); + lastIdxRidListPtr->count--; + + if (lastIdxRidListPtr->count == 0) + { + if (type == LIST_SIZE_TYPE) { - //found - m_dLbid = m_lbid; - m_dSbid = m_sbid; - m_dEntry = i; - rc = NO_ERROR; - memcpy(&newRowIdArray[0], - &rowIdArray[0], totalbytes); - lastIdxRidListPtr->count--; + // header has no link + m_curIdxRidListHdr.nextIdxRidListPtr.type = LIST_NOT_USED_TYPE; + m_curIdxRidListHdr.nextIdxRidListPtr.llp = 0; + } + } // header's link block has nothing now + else // still have more + { + memcpy(&rowIdArray[i], &newRowIdArray[i + 1], (count - (i + 1)) * LIST_ENTRY_WIDTH); + } - if (lastIdxRidListPtr->count == 0) - { - if (type == LIST_SIZE_TYPE) - { - //header has no link - m_curIdxRidListHdr.nextIdxRidListPtr.type - = LIST_NOT_USED_TYPE; - m_curIdxRidListHdr.nextIdxRidListPtr.llp = 0; + // last row id entry now moved up, so not used + rowIdArray[count - 1].type = LIST_NOT_USED_TYPE; + rowIdArray[count - 1].rid = 0; + rowIdArray[count - 1].spare = 0; + // header update the size + m_curIdxRidListHdr.idxRidListSize.size--; - } - }//header's link block has nothing now - else //still have more - { - memcpy(&rowIdArray[i], &newRowIdArray[i + 1], (count - (i + 1))*LIST_ENTRY_WIDTH); - } + if (m_lbid != m_hdrLbid) + { + setSubBlockEntry(m_curBlock.data, m_sbid, 0, totalbytes, rowIdArray); + rc = writeDBFile(cb, m_curBlock.data, m_lbid); - //last row id entry now moved up, so not used - rowIdArray[count - 1].type = LIST_NOT_USED_TYPE; - rowIdArray[count - 1].rid = 0; - rowIdArray[count - 1].spare = 0; - //header update the size - m_curIdxRidListHdr.idxRidListSize.size--; + if (rc != NO_ERROR) + return rc; - if (m_lbid != m_hdrLbid) - { - setSubBlockEntry( m_curBlock.data, - m_sbid, 0, totalbytes, - rowIdArray ); - rc = writeDBFile( cb, m_curBlock.data, m_lbid); + m_curBlock.state = BLK_READ; + rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - m_curBlock.state = BLK_READ; - rc = writeSubBlockEntry(cb, &m_hdrBlock, - m_hdrLbid, m_hdrSbid, - m_hdrEntry, - LIST_HDR_SIZE, - &m_curIdxRidListHdr ); + m_hdrBlock.state = BLK_READ; + } + else + { + // m_lbid==m_hdrLbid + setSubBlockEntry(m_hdrBlock.data, m_sbid, 0, totalbytes, rowIdArray); + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + m_hdrBlock.state = BLK_WRITE; + rc = writeDBFile(cb, m_hdrBlock.data, m_hdrLbid); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - m_hdrBlock.state = BLK_READ; - } - else - { - //m_lbid==m_hdrLbid - setSubBlockEntry( m_hdrBlock.data, - m_sbid, 0, totalbytes, - rowIdArray ); - setSubBlockEntry( m_hdrBlock.data, - m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, - &m_curIdxRidListHdr); - m_hdrBlock.state = BLK_WRITE; - rc = writeDBFile( cb, m_hdrBlock.data, - m_hdrLbid); + m_hdrBlock.state = BLK_READ; + } // end if m_lbid==m_hdrHdrLbid - if (rc != NO_ERROR) - return rc; + m_dEntry = i; + return rc; + } // endif found + } // end for - m_hdrBlock.state = BLK_READ; - } //end if m_lbid==m_hdrHdrLbid - - m_dEntry = i; - return rc; - }//endif found - }//end for - - return rc; + return rc; } /************************************************ - * Description: - * Find a entry for the given rowId and Key - * Then Delete it from the list - * Move the rest of the row id up in the same - * sub block an decrement the count in that subblock - * decrement the header size - * Converted - * input - * pFile -- File Handler - * rowId -- row id - * key -- value - * curIdxRidListHdrPtr - point to the header - * - * return value - * Success -- 0 - * Fail -- ERR_IDX_LIST_INVALID_DELETE - ************************************************/ + * Description: + * Find a entry for the given rowId and Key + * Then Delete it from the list + * Move the rest of the row id up in the same + * sub block an decrement the count in that subblock + * decrement the header size + * Converted + * input + * pFile -- File Handler + * rowId -- row id + * key -- value + * curIdxRidListHdrPtr - point to the header + * + * return value + * Success -- 0 + * Fail -- ERR_IDX_LIST_INVALID_DELETE + ************************************************/ const int IndexList::deleteInBlock(const RID& rowId) { - int width = LIST_ENTRY_WIDTH; - int rc = ERR_IDX_LIST_INVALID_DELETE; - IdxRidListPtr* lastIdxRidListPtr; - IdxRidListPtr lastSubIdxRidListPtr; - bool found; - int type, count; - IdxRidListPtr prevIdxRidListPtr; - int prevSbid, prevEntry, prevType; - uint64_t prevLbid; - DataBlock prevDataBlock; - int pos = 0, totalbytes = 0; - int preTotalBytes, prevPos ; - //IdxRidNextListPtr *nextIdxListPtr; + int width = LIST_ENTRY_WIDTH; + int rc = ERR_IDX_LIST_INVALID_DELETE; + IdxRidListPtr* lastIdxRidListPtr; + IdxRidListPtr lastSubIdxRidListPtr; + bool found; + int type, count; + IdxRidListPtr prevIdxRidListPtr; + int prevSbid, prevEntry, prevType; + uint64_t prevLbid; + DataBlock prevDataBlock; + int pos = 0, totalbytes = 0; + int preTotalBytes, prevPos; + // IdxRidNextListPtr *nextIdxListPtr; - IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; - IdxRidListEntry newRowIdArray[MAX_BLOCK_ENTRY]; + IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; + IdxRidListEntry newRowIdArray[MAX_BLOCK_ENTRY]; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - //This is the sub block info - prevLbid = m_lbid; - prevSbid = m_sbid; - prevEntry = m_entry; - prevPos = LIST_SUB_LLP_POS; - preTotalBytes = SUBBLOCK_TOTAL_BYTES; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + // This is the sub block info + prevLbid = m_lbid; + prevSbid = m_sbid; + prevEntry = m_entry; + prevPos = LIST_SUB_LLP_POS; + preTotalBytes = SUBBLOCK_TOTAL_BYTES; - if (prevLbid == m_hdrLbid) - { - if (m_hdrBlock.state >= BLK_READ) - getSubBlockEntry(m_hdrBlock.data, m_sbid, - prevPos, LIST_ENTRY_WIDTH, &lastSubIdxRidListPtr ); - else - return ERR_IDX_LIST_INVALID_DELETE; - } + if (prevLbid == m_hdrLbid) + { + if (m_hdrBlock.state >= BLK_READ) + getSubBlockEntry(m_hdrBlock.data, m_sbid, prevPos, LIST_ENTRY_WIDTH, &lastSubIdxRidListPtr); else + return ERR_IDX_LIST_INVALID_DELETE; + } + else + { + if (m_curBlock.state >= BLK_READ) + getSubBlockEntry(m_curBlock.data, m_sbid, prevPos, LIST_ENTRY_WIDTH, &lastSubIdxRidListPtr); + else + return ERR_IDX_LIST_INVALID_DELETE; + } + + found = false; + m_lbid = ((IdxEmptyListEntry*)&lastSubIdxRidListPtr)->fbo; + m_sbid = 0; + m_entry = 0; + + type = lastSubIdxRidListPtr.type; + count = lastSubIdxRidListPtr.count; + pos = LIST_BLOCK_LLP_POS; + totalbytes = BYTE_PER_BLOCK; + + // Not found in the first sub + while ((!found) && (type == LIST_BLOCK_TYPE)) + { + rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, totalbytes, rowIdArray); + + if (rc != NO_ERROR) + return rc; + + m_curBlock.dirty = true; + m_curBlock.state = BLK_READ; + m_curBlock.lbid = m_lbid; + prevType = type; // Save it just in case not found here + lastIdxRidListPtr = (IdxRidListPtr*)&rowIdArray[pos]; + type = lastIdxRidListPtr->type; + count = lastIdxRidListPtr->count; + + // prepared for not found in current block + // find out what is the next type + // Next Type is needed here + for (int i = 0; i < count; i++) { - if (m_curBlock.state >= BLK_READ) - getSubBlockEntry(m_curBlock.data, m_sbid, - prevPos, LIST_ENTRY_WIDTH, &lastSubIdxRidListPtr ); + if (rowIdArray[i].rid == rowId) + { + // found the rowid + memcpy(&newRowIdArray[0], &rowIdArray[0], totalbytes); + found = true; + m_dLbid = m_lbid; + m_dSbid = m_sbid; + m_dEntry = i; + lastIdxRidListPtr->count--; + + if (lastIdxRidListPtr->count == 0) + { + if (!m_useNarray) + { + // get the previous value out, could be a sub block + if (prevLbid == m_hdrLbid) + getSubBlockEntry(m_hdrBlock.data, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr); + else if (prevLbid == m_lbid) + getSubBlockEntry(m_curBlock.data, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr); + else + rc = readSubBlockEntry(cb, &prevDataBlock, prevLbid, prevSbid, prevPos, LIST_ENTRY_WIDTH, + &prevIdxRidListPtr); + + if (rc != NO_ERROR) + return rc; + + // check the type before set + if (type == LIST_BLOCK_TYPE) + { + ((IdxEmptyListEntry*)&prevIdxRidListPtr)->fbo = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; + ((IdxEmptyListEntry*)&prevIdxRidListPtr)->sbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->sbid; + ((IdxEmptyListEntry*)&prevIdxRidListPtr)->entry = + ((IdxEmptyListEntry*)lastIdxRidListPtr)->entry; + // safety check + prevIdxRidListPtr.type = type; + } + else // If no more links, the current one is gone also + { + if (prevIdxRidListPtr.count > 0) + { + prevIdxRidListPtr.type = 0; + prevIdxRidListPtr.llp = 0; + } + else + { + // In case it is a sub block, not released with 0 count + prevIdxRidListPtr.type = LIST_NOT_USED_TYPE; + prevIdxRidListPtr.llp = 0; + } + } // end if type =LIST_SUBBLOCK_TYPE,LIST_BLOCK_TYPE + + //;set to LIST_NOT_USED_TYPE--unused before release + lastIdxRidListPtr->type = LIST_NOT_USED_TYPE; + lastIdxRidListPtr->llp = 0; + + if (prevPos == LIST_BLOCK_LLP_POS) + { + if (prevLbid < m_lastLbid) + rc = setLastLbid(prevLbid); + } + } + } // end if count==0 else - return ERR_IDX_LIST_INVALID_DELETE; - } + { + memcpy(&rowIdArray[i], &newRowIdArray[i + 1], (count - (i + 1)) * LIST_ENTRY_WIDTH); - found = false; - m_lbid = ((IdxEmptyListEntry*)&lastSubIdxRidListPtr)->fbo; - m_sbid = 0; - m_entry = 0; + if (m_lastLbid > m_lbid) + rc = setLastLbid(m_lbid); - type = lastSubIdxRidListPtr.type; - count = lastSubIdxRidListPtr.count; - pos = LIST_BLOCK_LLP_POS; - totalbytes = BYTE_PER_BLOCK; + } // count check - //Not found in the first sub - while ((!found) && (type == LIST_BLOCK_TYPE)) - { - rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, - totalbytes, rowIdArray); + // Found rowId + rowIdArray[count - 1].type = LIST_NOT_USED_TYPE; + rowIdArray[count - 1].rid = 0; - if (rc != NO_ERROR) + m_curIdxRidListHdr.idxRidListSize.size--; + // Write Out Put in another routine + + if ((prevLbid == m_hdrLbid) && (m_lbid != m_hdrLbid)) + { + // AAC --3 + if (!m_useNarray) + { + if (lastIdxRidListPtr->count == 0) + { + setSubBlockEntry(m_hdrBlock.data, prevSbid, prevPos, width, &prevIdxRidListPtr); + } + } + + setSubBlockEntry(m_curBlock.data, m_sbid, 0, totalbytes, rowIdArray); + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + rc = writeDBFile(cb, m_hdrBlock.data, m_hdrLbid); + + if (rc != NO_ERROR) return rc; - m_curBlock.dirty = true; - m_curBlock.state = BLK_READ; - m_curBlock.lbid = m_lbid; - prevType = type; //Save it just in case not found here - lastIdxRidListPtr = (IdxRidListPtr*) &rowIdArray[pos]; - type = lastIdxRidListPtr->type; - count = lastIdxRidListPtr->count; + rc = writeDBFile(cb, m_curBlock.data, m_lbid); - //prepared for not found in current block - //find out what is the next type - //Next Type is needed here - for (int i = 0; i < count; i++) + if (rc != NO_ERROR) + return rc; + + m_hdrBlock.state = BLK_READ; + m_curBlock.state = BLK_READ; + } + else { - if (rowIdArray[i].rid == rowId) + // ABC -- + if (!m_useNarray) + { + if (lastIdxRidListPtr->count == 0) { - //found the rowid - memcpy(&newRowIdArray[0], &rowIdArray[0], totalbytes); - found = true; - m_dLbid = m_lbid; - m_dSbid = m_sbid; - m_dEntry = i; - lastIdxRidListPtr->count--; + setSubBlockEntry(prevDataBlock.data, prevSbid, prevPos, LIST_ENTRY_WIDTH, &prevIdxRidListPtr); + rc = writeDBFile(cb, prevDataBlock.data, prevLbid); - if (lastIdxRidListPtr->count == 0) - { - if (!m_useNarray) - { - //get the previous value out, could be a sub block - if (prevLbid == m_hdrLbid) - getSubBlockEntry(m_hdrBlock.data, prevSbid, - prevPos, LIST_ENTRY_WIDTH, - &prevIdxRidListPtr); - else if (prevLbid == m_lbid) - getSubBlockEntry(m_curBlock.data, prevSbid, - prevPos, LIST_ENTRY_WIDTH, - &prevIdxRidListPtr); - else - rc = readSubBlockEntry(cb, &prevDataBlock, prevLbid, - prevSbid, prevPos, LIST_ENTRY_WIDTH, - &prevIdxRidListPtr); + if (rc != NO_ERROR) + return rc; + } + } - if (rc != NO_ERROR) - return rc; + setSubBlockEntry(m_curBlock.data, m_sbid, 0, totalbytes, rowIdArray); + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + rc = writeDBFile(cb, m_hdrBlock.data, m_hdrLbid); - //check the type before set - if (type == LIST_BLOCK_TYPE) - { - ((IdxEmptyListEntry*)&prevIdxRidListPtr)->fbo - = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - ((IdxEmptyListEntry*)&prevIdxRidListPtr)->sbid - = ((IdxEmptyListEntry*)lastIdxRidListPtr)->sbid; - ((IdxEmptyListEntry*)&prevIdxRidListPtr)->entry - = ((IdxEmptyListEntry*)lastIdxRidListPtr)->entry; - //safety check - prevIdxRidListPtr.type = type; - } - else // If no more links, the current one is gone also - { - if (prevIdxRidListPtr.count > 0) - { - prevIdxRidListPtr.type = 0; - prevIdxRidListPtr.llp = 0; - } - else - { - //In case it is a sub block, not released with 0 count - prevIdxRidListPtr.type = LIST_NOT_USED_TYPE; - prevIdxRidListPtr.llp = 0; - } - }//end if type =LIST_SUBBLOCK_TYPE,LIST_BLOCK_TYPE + if (rc != NO_ERROR) + return rc; - //;set to LIST_NOT_USED_TYPE--unused before release - lastIdxRidListPtr->type = LIST_NOT_USED_TYPE; - lastIdxRidListPtr->llp = 0; + rc = writeDBFile(cb, m_curBlock.data, m_lbid); + memset(m_hdrBlock.data, 0, sizeof(m_hdrBlock.data)); - if (prevPos == LIST_BLOCK_LLP_POS) - { - if (prevLbid < m_lastLbid) - rc = setLastLbid(prevLbid); - } - } - } //end if count==0 - else - { - memcpy(&rowIdArray[i], &newRowIdArray[i + 1], (count - (i + 1))*LIST_ENTRY_WIDTH); + if (rc != NO_ERROR) + return rc; - if (m_lastLbid > m_lbid) - rc = setLastLbid(m_lbid); + m_hdrBlock.state = BLK_READ; + m_curBlock.state = BLK_READ; + } // last case A B C --end 5 - }//count check + // Done with writing to disk + // Now we need to release the segment + if (!m_useNarray) + { + if (lastIdxRidListPtr->count == 0) + { + rc = releaseSegment(); - //Found rowId - rowIdArray[count - 1].type = LIST_NOT_USED_TYPE; - rowIdArray[count - 1].rid = 0; + if (rc != NO_ERROR) + return rc; + } // end release segment when count ==0 + } - m_curIdxRidListHdr.idxRidListSize.size--; - //Write Out Put in another routine + m_entry = i; // for use in findRow ID + return rc; // DONE !!!found then we return, no need to go on + } // FOUND THE ROWID returned !!!! + } // for loop i not found continue to i++ - if ((prevLbid == m_hdrLbid) && (m_lbid != m_hdrLbid)) - { - // AAC --3 - if (!m_useNarray) - { - if (lastIdxRidListPtr->count == 0) - { - setSubBlockEntry( m_hdrBlock.data, prevSbid, - prevPos, width, - &prevIdxRidListPtr ); - } - } + // NOT FOUND in this block go to next block + // assigning the current llp as previous llp:lbid, sbid, entry + prevLbid = m_lbid; + prevSbid = 0; + prevEntry = 0; + prevPos = pos; + preTotalBytes = totalbytes; - setSubBlockEntry( m_curBlock.data, m_sbid, - 0, totalbytes, - rowIdArray ); - setSubBlockEntry( m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - rc = writeDBFile( cb, m_hdrBlock.data, m_hdrLbid); + m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; + m_sbid = 0; + m_entry = 0; + } // end while - if (rc != NO_ERROR) - return rc; + if (!found) + rc = ERR_IDX_LIST_INVALID_DELETE; - rc = writeDBFile( cb, m_curBlock.data, m_lbid); - - if (rc != NO_ERROR) - return rc; - - m_hdrBlock.state = BLK_READ; - m_curBlock.state = BLK_READ; - } - else - { - //ABC -- - if (!m_useNarray) - { - if (lastIdxRidListPtr->count == 0) - { - setSubBlockEntry( prevDataBlock.data, prevSbid, - prevPos, LIST_ENTRY_WIDTH, - &prevIdxRidListPtr ); - rc = writeDBFile( cb, prevDataBlock.data, prevLbid); - - if (rc != NO_ERROR) - return rc; - } - } - - setSubBlockEntry( m_curBlock.data, m_sbid, - 0, totalbytes, - rowIdArray ); - setSubBlockEntry( m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - rc = writeDBFile( cb, m_hdrBlock.data, m_hdrLbid); - - if (rc != NO_ERROR) - return rc; - - rc = writeDBFile( cb, m_curBlock.data, m_lbid); - memset(m_hdrBlock.data, 0, - sizeof(m_hdrBlock.data)); - - if (rc != NO_ERROR) - return rc; - - m_hdrBlock.state = BLK_READ; - m_curBlock.state = BLK_READ; - } //last case A B C --end 5 - - //Done with writing to disk - // Now we need to release the segment - if (!m_useNarray) - { - if (lastIdxRidListPtr->count == 0) - { - rc = releaseSegment(); - - if (rc != NO_ERROR) - return rc; - }// end release segment when count ==0 - } - - m_entry = i; //for use in findRow ID - return rc; //DONE !!!found then we return, no need to go on - }//FOUND THE ROWID returned !!!! - }//for loop i not found continue to i++ - - //NOT FOUND in this block go to next block - //assigning the current llp as previous llp:lbid, sbid, entry - prevLbid = m_lbid; - prevSbid = 0; - prevEntry = 0; - prevPos = pos; - preTotalBytes = totalbytes; - - m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - m_sbid = 0; - m_entry = 0; - }// end while - - if (!found) - rc = ERR_IDX_LIST_INVALID_DELETE; - - return rc; + return rc; } - /************************************************ * Description: * Converted @@ -510,42 +466,42 @@ const int IndexList::deleteInBlock(const RID& rowId) * Success -- 0 * Fail -- ERR_IDX_LIST_INVALID_DELETE ************************************************/ -const int IndexList::deleteIndexList( FILE* pFile, const RID& rowId, - const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr, - uint64_t& lbid, int& sbid, int& entry) +const int IndexList::deleteIndexList(FILE* pFile, const RID& rowId, const uint64_t& key, + IdxEmptyListEntry* curIdxRidListHdrPtr, uint64_t& lbid, int& sbid, + int& entry) { - int rc = ERR_IDX_LIST_INVALID_DELETE; - bool found = false; + int rc = ERR_IDX_LIST_INVALID_DELETE; + bool found = false; - m_pFile = pFile; - getHdrInfo(curIdxRidListHdrPtr); + m_pFile = pFile; + getHdrInfo(curIdxRidListHdrPtr); - if (key != m_curIdxRidListHdr.key) - { - return ERR_IDX_LIST_INVALID_KEY; - } + if (key != m_curIdxRidListHdr.key) + { + return ERR_IDX_LIST_INVALID_KEY; + } - uint64_t dlbid = -1LL; - int dsbid = -1; - int dentry = -1; - rc = deleteIndexList(rowId, key, dlbid, dsbid, dentry); - - if (rc != NO_ERROR) - { - lbid = -1LL; - sbid = -1; - entry = -1; - found = false; - return rc; - } - else - { - lbid = dlbid; - sbid = dsbid; - entry = dentry; - } + uint64_t dlbid = -1LL; + int dsbid = -1; + int dentry = -1; + rc = deleteIndexList(rowId, key, dlbid, dsbid, dentry); + if (rc != NO_ERROR) + { + lbid = -1LL; + sbid = -1; + entry = -1; + found = false; return rc; + } + else + { + lbid = dlbid; + sbid = dsbid; + entry = dentry; + } + + return rc; } /************************************************ * Description: @@ -565,17 +521,16 @@ const int IndexList::deleteIndexList( FILE* pFile, const RID& rowId, * Success -- 0 * Fail -- ERR_IDX_LIST_INVALID_DELETE ************************************************/ -const int IndexList::deleteIndexList(const RID& rowId, - const uint64_t& key, - uint64_t& lbid, int& sbid, int& entry) +const int IndexList::deleteIndexList(const RID& rowId, const uint64_t& key, uint64_t& lbid, int& sbid, + int& entry) { - int rc = ERR_IDX_LIST_INVALID_DELETE; - rc = deleteIndexList(rowId, key); + int rc = ERR_IDX_LIST_INVALID_DELETE; + rc = deleteIndexList(rowId, key); - lbid = m_dLbid; - sbid = m_dSbid; - entry = m_dEntry; - return rc; + lbid = m_dLbid; + sbid = m_dSbid; + entry = m_dEntry; + return rc; } /************************************************ * Description: @@ -589,312 +544,296 @@ const int IndexList::deleteIndexList(const RID& rowId, * Success -- 0 * Fail -- ERR_IDX_LIST_INVALID_DELETE ************************************************/ -const int IndexList::getRIDArrayFromListHdr(FILE* pFile, uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr, - RID* ridArray, int& size) +const int IndexList::getRIDArrayFromListHdr(FILE* pFile, uint64_t& key, + IdxEmptyListEntry* curIdxRidListHdrPtr, RID* ridArray, int& size) { - int rc = NO_ERROR; - int arrayCount = 0; - IdxRidNextListPtr* nextIdxListPtr = NULL; + int rc = NO_ERROR; + int arrayCount = 0; + IdxRidNextListPtr* nextIdxListPtr = NULL; - m_pFile = pFile; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + m_pFile = pFile; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - rc = getHdrInfo(curIdxRidListHdrPtr); + rc = getHdrInfo(curIdxRidListHdrPtr); - if (m_curIdxRidListHdr.idxRidListSize.size == 0) - { - size = 0; - return NO_ERROR; - } + if (m_curIdxRidListHdr.idxRidListSize.size == 0) + { + size = 0; + return NO_ERROR; + } - if (key != m_curIdxRidListHdr.key) - { - return ERR_IDX_LIST_WRONG_KEY; - } + if (key != m_curIdxRidListHdr.key) + { + return ERR_IDX_LIST_WRONG_KEY; + } - // cout << "IndexList::getRIDArrayFromListHdr->KEY ------>" << key << endl; - //Check the first row location, 3rd enty - if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) - { - ridArray[arrayCount] = (RID)m_curIdxRidListHdr.firstIdxRidListEntry.rid; - //cout<<" IndexList::getRIDArrayFromListHdr->header Lbid->" << m_hdrLbid <<" count->" << arrayCount <KEY ------>" << key << endl; + // Check the first row location, 3rd enty + if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) + { + ridArray[arrayCount] = (RID)m_curIdxRidListHdr.firstIdxRidListEntry.rid; + // cout<<" IndexList::getRIDArrayFromListHdr->header Lbid->" << m_hdrLbid <<" count->" << arrayCount + // <" << arrayCount << "rid->" << ridArray[arrayCount]<" << count << endl; + // type should be LIST_BLOCK_TYPE from now on + for (int i = 0; i < count; i++) + { + ridArray[arrayCount] = (RID)(rowIdArray[i].rid); + // cout << "RID =" << (RID)(rowIdArray[i].rid) << endl; + // cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<" << m_lbid ; + // cout << " count->" << count << endl; + m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - switch (type) - { - case LIST_RID_TYPE:// There is a row id here, Check! - ridArray[arrayCount] = (RID)m_curIdxRidListHdr.nextIdxRidListPtr.llp; - //cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<" << m_lbid ; - size = arrayCount; - return NO_ERROR; - - case LIST_SUBBLOCK_TYPE://Not found in header, so go to the sub-block - //get thelbid sbid and entry out from the header last entry - - m_lbid = ((IdxEmptyListEntry*) & (m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; - m_sbid = ((IdxEmptyListEntry*) & (m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; - m_entry = ((IdxEmptyListEntry*) & (m_curIdxRidListHdr.nextIdxRidListPtr))->entry; - m_curType = type; - //Read the pointer entry at LIST_SUB_LLP_POS location - IdxRidListPtr* lastIdxRidListPtr; - IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; - memset(rowIdArray, 0, BYTE_PER_BLOCK); - int pos = 0, totalbytes = 0; - pos = LIST_SUB_LLP_POS; - totalbytes = SUBBLOCK_TOTAL_BYTES; - - if (m_lbid != m_hdrLbid) - { - rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, - m_sbid, 0, totalbytes, - rowIdArray); - m_curBlock.lbid = m_lbid; - m_curBlock.state = BLK_READ; - m_curBlock.dirty = true; - } - else - getSubBlockEntry(m_hdrBlock.data, m_sbid, - 0, totalbytes, - rowIdArray ); - - int type, count; - - lastIdxRidListPtr = (IdxRidListPtr*) &rowIdArray[pos]; - type = lastIdxRidListPtr->type; - count = lastIdxRidListPtr->count; - - //cout << "count->" << count << endl; - //type should be LIST_BLOCK_TYPE from now on - for (int i = 0; i < count; i++) - { - ridArray[arrayCount] = (RID)(rowIdArray[i].rid); - //cout << "RID =" << (RID)(rowIdArray[i].rid) << endl; - //cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<" << m_lbid ; - //cout << " count->" << count << endl; - m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - - while (type == LIST_BLOCK_TYPE) - { - //cout << " Lbid->" << m_lbid ; - - pos = LIST_BLOCK_LLP_POS; - totalbytes = BYTE_PER_BLOCK; - rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, - totalbytes, rowIdArray); - m_curBlock.lbid = m_lbid; - m_curBlock.state = BLK_READ; - m_curBlock.dirty = true; - - if (!m_useNarray) - { - lastIdxRidListPtr = (IdxRidListPtr*) &rowIdArray[pos]; - type = lastIdxRidListPtr->type; - count = lastIdxRidListPtr->count; - } - else - { - nextIdxListPtr = (IdxRidNextListPtr*)&rowIdArray[pos]; - type = nextIdxListPtr->type; - count = nextIdxListPtr->count; - } - - //cout << " count->" << count << endl; - for (int i = 0; i < count; i++) - { - ridArray[arrayCount] = (RID)(rowIdArray[i].rid) ; - //cout << "RID =" << (RID)(rowIdArray[i].rid) << endl; - //cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<nextLbid; - else - m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - } - - }//end while - };//end of switch - - size = arrayCount; - - return rc; -}//end getRIDArrayFromListHdr - -const int IndexList::getRIDArrayFromListHdrNarray(FILE* pFile, uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr, - RID* ridArray, int& size, bool flag) -{ - int rc = NO_ERROR; - IdxRidNextListPtr* nextIdxListPtr; - int pos = 0, totalbytes = 0; - IdxRidListPtr* lastIdxRidListPtr; - IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY * 10]; - int type = 0, count = 0; - - - - m_pFile = pFile; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - - if (flag) - { - rc = getHdrInfo(curIdxRidListHdrPtr); - - if (m_curIdxRidListHdr.idxRidListSize.size == 0) - { - size = 0; - return NO_ERROR; - } - - if (key != m_curIdxRidListHdr.key) - { - return ERR_IDX_LIST_WRONG_KEY; - } - - // cout << "IndexList::getRIDArrayFromListHdr->KEY ------>" << key << endl; - //Check the first row location, 3rd enty - if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) - { - ridArray[size] = (RID)m_curIdxRidListHdr.firstIdxRidListEntry.rid; - //cout<<" IndexList::getRIDArrayFromListHdr->header Lbid->" << m_hdrLbid <<" count->" << arrayCount <" << arrayCount << "rid->" << ridArray[arrayCount]<" << count << endl; - //type should be LIST_BLOCK_TYPE from now on - for (int i = 0; i < count; i++) - { - ridArray[size] = (RID)(rowIdArray[i].rid); - //cout << "RID =" << (RID)(rowIdArray[i].rid) << endl; - //cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<" << m_lbid ; - //cout << " count->" << count << endl; - m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - m_curType = type; - }//end of switch - }//end if flag - - if (m_curType == LIST_BLOCK_TYPE) - { pos = LIST_BLOCK_LLP_POS; totalbytes = BYTE_PER_BLOCK; - rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, - totalbytes, rowIdArray); + rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, totalbytes, rowIdArray); m_curBlock.lbid = m_lbid; m_curBlock.state = BLK_READ; m_curBlock.dirty = true; - nextIdxListPtr = (IdxRidNextListPtr*)&rowIdArray[pos]; - type = nextIdxListPtr->type; - - count = nextIdxListPtr->count; + if (!m_useNarray) + { + lastIdxRidListPtr = (IdxRidListPtr*)&rowIdArray[pos]; + type = lastIdxRidListPtr->type; + count = lastIdxRidListPtr->count; + } + else + { + nextIdxListPtr = (IdxRidNextListPtr*)&rowIdArray[pos]; + type = nextIdxListPtr->type; + count = nextIdxListPtr->count; + } + // cout << " count->" << count << endl; for (int i = 0; i < count; i++) { - ridArray[size] = (RID)(rowIdArray[i].rid) ; - size++; + ridArray[arrayCount] = (RID)(rowIdArray[i].rid); + // cout << "RID =" << (RID)(rowIdArray[i].rid) << endl; + // cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<nextLbid; + else + m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; } - }//end if block + } // end while + }; // end of switch - return rc; -}//end getRIDArrayFromListHdrNarray + size = arrayCount; + return rc; +} // end getRIDArrayFromListHdr + +const int IndexList::getRIDArrayFromListHdrNarray(FILE* pFile, uint64_t& key, + IdxEmptyListEntry* curIdxRidListHdrPtr, RID* ridArray, + int& size, bool flag) +{ + int rc = NO_ERROR; + IdxRidNextListPtr* nextIdxListPtr; + int pos = 0, totalbytes = 0; + IdxRidListPtr* lastIdxRidListPtr; + IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY * 10]; + int type = 0, count = 0; + + m_pFile = pFile; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + + if (flag) + { + rc = getHdrInfo(curIdxRidListHdrPtr); + + if (m_curIdxRidListHdr.idxRidListSize.size == 0) + { + size = 0; + return NO_ERROR; + } + + if (key != m_curIdxRidListHdr.key) + { + return ERR_IDX_LIST_WRONG_KEY; + } + + // cout << "IndexList::getRIDArrayFromListHdr->KEY ------>" << key << endl; + // Check the first row location, 3rd enty + if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) + { + ridArray[size] = (RID)m_curIdxRidListHdr.firstIdxRidListEntry.rid; + // cout<<" IndexList::getRIDArrayFromListHdr->header Lbid->" << m_hdrLbid <<" count->" << arrayCount + // <" << arrayCount << "rid->" << ridArray[arrayCount]<" << count << endl; + // type should be LIST_BLOCK_TYPE from now on + for (int i = 0; i < count; i++) + { + ridArray[size] = (RID)(rowIdArray[i].rid); + // cout << "RID =" << (RID)(rowIdArray[i].rid) << endl; + // cout<<"arrayCount->" << arrayCount << "rid->" << ridArray[arrayCount]<" << m_lbid ; + // cout << " count->" << count << endl; + m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; + m_curType = type; + } // end of switch + } // end if flag + + if (m_curType == LIST_BLOCK_TYPE) + { + pos = LIST_BLOCK_LLP_POS; + totalbytes = BYTE_PER_BLOCK; + rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, 0, 0, totalbytes, rowIdArray); + m_curBlock.lbid = m_lbid; + m_curBlock.state = BLK_READ; + m_curBlock.dirty = true; + + nextIdxListPtr = (IdxRidNextListPtr*)&rowIdArray[pos]; + type = nextIdxListPtr->type; + + count = nextIdxListPtr->count; + + for (int i = 0; i < count; i++) + { + ridArray[size] = (RID)(rowIdArray[i].rid); + size++; + } + + IdxRidListArrayPtr idxRidListArrayPtr; + int curLevel = 0, curCount = 0; + + memset(&idxRidListArrayPtr, 0, sizeof(idxRidListArrayPtr)); + getSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS, LIST_BLK_LLP_ENTRY_WIDTH, + &idxRidListArrayPtr); + curLevel = idxRidListArrayPtr.nextIdxListPtr.curLevel; + curCount = idxRidListArrayPtr.nextIdxListPtr.count; + + for (int i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) + { + m_lbid = idxRidListArrayPtr.childIdxRidListPtr[i].childLbid; + int type = idxRidListArrayPtr.childIdxRidListPtr[i].type; + + if ((m_lbid != (uint64_t)INVALID_LBID) && (type == LIST_BLOCK_TYPE)) + { + m_curType = LIST_BLOCK_TYPE; + getRIDArrayFromListHdrNarray(pFile, key, curIdxRidListHdrPtr, ridArray, size, false); + } + } + + } // end if block + + return rc; +} // end getRIDArrayFromListHdrNarray /*--------------------------------------------------------------------- * Description: Output the info of this company to an abstract stream @@ -909,14 +848,12 @@ const int IndexList::getRIDArrayFromListHdrNarray(FILE* pFile, uint64_t& key, *------------------------------------------------------------------*/ ostream& operator<<(ostream& os, IndexList& rhs) { - os << rhs.m_curIdxRidListHdr.idxRidListSize.size << "\t" - << rhs.m_curIdxRidListHdr.key << "\t" - << rhs.m_curIdxRidListHdr.firstIdxRidListEntry.type << "\t" - << rhs.m_curIdxRidListHdr.firstIdxRidListEntry.rid << "\t" - << rhs.m_curIdxRidListHdr.nextIdxRidListPtr.type << "\t" - << rhs.m_curIdxRidListHdr.nextIdxRidListPtr.llp << "\t" - << endl; - return os; + os << rhs.m_curIdxRidListHdr.idxRidListSize.size << "\t" << rhs.m_curIdxRidListHdr.key << "\t" + << rhs.m_curIdxRidListHdr.firstIdxRidListEntry.type << "\t" + << rhs.m_curIdxRidListHdr.firstIdxRidListEntry.rid << "\t" + << rhs.m_curIdxRidListHdr.nextIdxRidListPtr.type << "\t" << rhs.m_curIdxRidListHdr.nextIdxRidListPtr.llp + << "\t" << endl; + return os; } /************************************************ * Description: @@ -938,22 +875,21 @@ ostream& operator<<(ostream& os, IndexList& rhs) * false--not found ************************************************/ bool IndexList::findRowId(FILE* pFile, const RID& rowId, const uint64_t& key, - IdxEmptyListEntry* curIdxRidListHdrPtr, - uint64_t& lbid, int& sbid, int& entry) + IdxEmptyListEntry* curIdxRidListHdrPtr, uint64_t& lbid, int& sbid, int& entry) { - bool found = false; - int rc; + bool found = false; + int rc; - m_pFile = pFile; - rc = getHdrInfo(curIdxRidListHdrPtr); + m_pFile = pFile; + rc = getHdrInfo(curIdxRidListHdrPtr); - if (key != m_curIdxRidListHdr.key) - { - return false; - } + if (key != m_curIdxRidListHdr.key) + { + return false; + } - found = findRowId(rowId, key, lbid, sbid, entry); - return found; + found = findRowId(rowId, key, lbid, sbid, entry); + return found; } /************************************************ @@ -974,187 +910,174 @@ bool IndexList::findRowId(FILE* pFile, const RID& rowId, const uint64_t& key, * true --found * false--not found ************************************************/ -bool IndexList::findRowId(const RID& rowId, const int64_t& key, - int64_t& lbid, int& sbid, int& entry) +bool IndexList::findRowId(const RID& rowId, const int64_t& key, int64_t& lbid, int& sbid, int& entry) { - bool found = false; - int rc; - RID savedRid; - CommBlock cb; - int count; - uint64_t prevLbid; - int prevType; + bool found = false; + int rc; + RID savedRid; + CommBlock cb; + int count; + uint64_t prevLbid; + int prevType; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; - //Check the first row location, 3rd enty--0,1,2 + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; + // Check the first row location, 3rd enty--0,1,2 - if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) + if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_RID_TYPE) + { + if (m_curIdxRidListHdr.firstIdxRidListEntry.rid == rowId) { - if (m_curIdxRidListHdr.firstIdxRidListEntry.rid == rowId) + lbid = m_hdrLbid; + sbid = m_hdrSbid; + entry = m_hdrEntry + 2; + found = true; + return found; + } + }; // endif type=LIST_RID_TYPE + + // Check Header last entry's type + int type = m_curIdxRidListHdr.nextIdxRidListPtr.type; + + int pos = 0, totalbytes = 0; + + switch (type) + { + case LIST_NOT_USED_TYPE: // Header is not full, no sub-block linked + // No RowId here then on + lbid = -1LL; + sbid = -1; + entry = -1; + found = false; + return found; // not found + + case LIST_RID_TYPE: // There is a row id here, Check! + + savedRid = (RID)m_curIdxRidListHdr.nextIdxRidListPtr.llp; + + if (savedRid == rowId) + { + lbid = m_hdrLbid; + sbid = m_hdrSbid; + entry = m_hdrEntry + 3; + found = true; + return found; + } + else + { + lbid = -1LL; + sbid = -1; + entry = -1; + found = false; + return found; + } + + case LIST_SUBBLOCK_TYPE: // Not found in header + // get the lbid sbid and entry out from the header last entry + + m_lbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; + m_sbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; + m_entry = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->entry; + + // Read the pointer entry at LIST_SUB_LLP_POS + // reserve enough space for rowIdArray + IdxRidListPtr* lastIdxRidListPtr; + IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; + memset(rowIdArray, 0, BYTE_PER_BLOCK); + // first link + pos = LIST_SUB_LLP_POS; + totalbytes = SUBBLOCK_TOTAL_BYTES; + + // check if the sub block is on the header block + if (m_lbid != m_hdrLbid) + { + rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, m_sbid, 0, totalbytes, rowIdArray); + m_curBlock.dirty = true; + m_curBlock.state = BLK_READ; + } + else + { + getSubBlockEntry(m_hdrBlock.data, m_sbid, 0, totalbytes, rowIdArray); + } + + lastIdxRidListPtr = (IdxRidListPtr*)&rowIdArray[pos]; + + prevLbid = m_lbid; // sub block + prevType = type; // sub block + count = lastIdxRidListPtr->count; // current count + type = lastIdxRidListPtr->type; // block + found = false; + + // look inside the first sub block + for (int i = 0; i < count; i++) + { + if (rowIdArray[i].rid == rowId) { - lbid = m_hdrLbid; - sbid = m_hdrSbid; - entry = m_hdrEntry + 2; - found = true; - return found; + found = true; + lbid = m_lbid; + sbid = m_sbid; + entry = i; + return found; } - }; //endif type=LIST_RID_TYPE + } - //Check Header last entry's type - int type = m_curIdxRidListHdr.nextIdxRidListPtr.type; + while ((!found) && (type == LIST_BLOCK_TYPE)) + { + // There are more to check on the next link + m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; + m_sbid = 0; + m_entry = 0; - int pos = 0, totalbytes = 0; + pos = LIST_BLOCK_LLP_POS; + totalbytes = BYTE_PER_BLOCK; - switch (type) - { - case LIST_NOT_USED_TYPE://Header is not full, no sub-block linked - //No RowId here then on - lbid = -1LL; - sbid = -1; - entry = -1; - found = false; - return found; //not found + if ((m_lbid != m_hdrLbid) && (m_lbid != prevLbid)) + { + // the only case for block + rc = readSubBlockEntry(cb, &m_curBlock, m_lbid, m_sbid, 0, totalbytes, rowIdArray); + m_curBlock.dirty = true; + } + else + { + printf("error in findRowID\n"); + return false; + } - case LIST_RID_TYPE:// There is a row id here, Check! + prevType = type; + lastIdxRidListPtr = (IdxRidListPtr*)&rowIdArray[pos]; + type = lastIdxRidListPtr->type; + count = lastIdxRidListPtr->count; + found = false; - savedRid = (RID)m_curIdxRidListHdr.nextIdxRidListPtr.llp; + for (int i = 0; i < count; i++) + { + if (rowIdArray[i].rid == rowId) + { + found = true; + lbid = m_lbid; + sbid = m_sbid; + entry = i; + return found; + } + } // end for i - if (savedRid == rowId) - { - lbid = m_hdrLbid; - sbid = m_hdrSbid; - entry = m_hdrEntry + 3; - found = true; - return found; - } - else - { - lbid = -1LL; - sbid = -1; - entry = -1; - found = false; - return found; - } + prevLbid = m_lbid; + } // end while - case LIST_SUBBLOCK_TYPE://Not found in header - //get the lbid sbid and entry out from the header last entry + break; - m_lbid = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; - m_sbid = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; - m_entry = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->entry; + default: printf("FIND ROWID got no where, error out !! \n"); break; + }; // end switch - //Read the pointer entry at LIST_SUB_LLP_POS - //reserve enough space for rowIdArray - IdxRidListPtr* lastIdxRidListPtr; - IdxRidListEntry rowIdArray[MAX_BLOCK_ENTRY]; - memset(rowIdArray, 0, BYTE_PER_BLOCK); - //first link - pos = LIST_SUB_LLP_POS; - totalbytes = SUBBLOCK_TOTAL_BYTES; + lbid = INVALID_LBID; - //check if the sub block is on the header block - if (m_lbid != m_hdrLbid) - { - rc = readSubBlockEntry( cb, &m_curBlock, m_lbid, m_sbid, 0, - totalbytes, rowIdArray); - m_curBlock.dirty = true; - m_curBlock.state = BLK_READ; - } - else - { - getSubBlockEntry(m_hdrBlock.data, m_sbid, - 0, totalbytes, rowIdArray ); + sbid = INVALID_NUM; - } + entry = INVALID_NUM; - lastIdxRidListPtr = (IdxRidListPtr*) &rowIdArray[pos]; - - prevLbid = m_lbid; //sub block - prevType = type; //sub block - count = lastIdxRidListPtr->count; //current count - type = lastIdxRidListPtr->type; //block - found = false; - - //look inside the first sub block - for (int i = 0; i < count; i++) - { - if (rowIdArray[i].rid == rowId) - { - found = true; - lbid = m_lbid; - sbid = m_sbid; - entry = i; - return found; - } - } - - while ((!found) && (type == LIST_BLOCK_TYPE)) - { - //There are more to check on the next link - m_lbid = ((IdxEmptyListEntry*)lastIdxRidListPtr)->fbo; - m_sbid = 0; - m_entry = 0; - - pos = LIST_BLOCK_LLP_POS; - totalbytes = BYTE_PER_BLOCK; - - if ((m_lbid != m_hdrLbid) && (m_lbid != prevLbid)) - { - // the only case for block - rc = readSubBlockEntry( cb, &m_curBlock, m_lbid, - m_sbid, 0, totalbytes, - rowIdArray); - m_curBlock.dirty = true; - } - else - { - printf("error in findRowID\n"); - return false; - } - - prevType = type; - lastIdxRidListPtr = (IdxRidListPtr*) &rowIdArray[pos]; - type = lastIdxRidListPtr->type; - count = lastIdxRidListPtr->count; - found = false; - - for (int i = 0; i < count; i++) - { - if (rowIdArray[i].rid == rowId) - { - found = true; - lbid = m_lbid; - sbid = m_sbid; - entry = i; - return found; - } - }//end for i - - prevLbid = m_lbid; - } //end while - - break; - - default: - printf ("FIND ROWID got no where, error out !! \n"); - break; - }; //end switch - - lbid = INVALID_LBID; - - sbid = INVALID_NUM; - - entry = INVALID_NUM; - - found = false; - - return found; -} //end function - -} //end of namespace + found = false; + return found; +} // end function +} // namespace WriteEngine diff --git a/writeengine/index/we_indexlist_multiple_narray.cpp b/writeengine/index/we_indexlist_multiple_narray.cpp index 59d3eca64..3c6b06342 100644 --- a/writeengine/index/we_indexlist_multiple_narray.cpp +++ b/writeengine/index/we_indexlist_multiple_narray.cpp @@ -30,7 +30,6 @@ #endif #include "we_indexlist.h" - using namespace std; namespace WriteEngine @@ -53,74 +52,72 @@ namespace WriteEngine * failure - it did not create the index list header ***********************************************************/ -const int IndexList::addIndexListHdr( FILE* pFile, const RID* ridList, - const int size, const uint64_t& key, - IdxEmptyListEntry* newEmptyListPtr) +const int IndexList::addIndexListHdr(FILE* pFile, const RID* ridList, const int size, const uint64_t& key, + IdxEmptyListEntry* newEmptyListPtr) { - int rc; - CommBlock cb; - m_pFile = pFile; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc; + CommBlock cb; + m_pFile = pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - if (size < 1) - return ERR_IDX_LIST_INVALID_ADDHDR; + if (size < 1) + return ERR_IDX_LIST_INVALID_ADDHDR; - //cout << "key =" << key << endl; - //Initialize three blokcs - rc = init(); - //The same as a single rowid - rc = addIndexListHdr( pFile, ridList[0], key, newEmptyListPtr); + // cout << "key =" << key << endl; + // Initialize three blokcs + rc = init(); + // The same as a single rowid + rc = addIndexListHdr(pFile, ridList[0], key, newEmptyListPtr); - if (size == 1) - { - return rc; - } + if (size == 1) + { + return rc; + } - //More than one row id - //Set up the header structure - m_hdrLbid = newEmptyListPtr->fbo; - m_hdrSbid = newEmptyListPtr->sbid; - m_hdrEntry = newEmptyListPtr->entry; - rc = updateIndexList(pFile, ridList[1], key, newEmptyListPtr); + // More than one row id + // Set up the header structure + m_hdrLbid = newEmptyListPtr->fbo; + m_hdrSbid = newEmptyListPtr->sbid; + m_hdrEntry = newEmptyListPtr->entry; + rc = updateIndexList(pFile, ridList[1], key, newEmptyListPtr); - if (rc != NO_ERROR) - return rc; - - if (size == 2) - return rc; - - rc = updateIndexList(pFile, ridList[2], key, newEmptyListPtr); - - if (rc != NO_ERROR) - return rc; - - if (size == 3) - return rc; - - //DONE with the header - //add more row id into the subblock and block - //first segment is a subblock - m_curType = LIST_SUBBLOCK_TYPE ; - m_type = LIST_SUBBLOCK_TYPE ; - int startPos = 3; - rc = getSubBlk(); - - if (size > 3) - rc = addRidList(ridList, size, startPos); - - if (rc != NO_ERROR) - return rc; - - rc = updateIndexListWrite(); - //rc = getSubBlk(); - //cout << "DEBUG INFO====================================="<< endl; - //cout<< " m_oid=" << m_oid << endl; - //cout<< " key= " << key << " header lbid=" << newEmptyListPtr->fbo - // << " header sbid= " << newEmptyListPtr->sbid - // << " header entry= " << newEmptyListPtr->entry < 3) + rc = addRidList(ridList, size, startPos); + + if (rc != NO_ERROR) + return rc; + + rc = updateIndexListWrite(); + // rc = getSubBlk(); + // cout << "DEBUG INFO====================================="<< endl; + // cout<< " m_oid=" << m_oid << endl; + // cout<< " key= " << key << " header lbid=" << newEmptyListPtr->fbo + // << " header sbid= " << newEmptyListPtr->sbid + // << " header entry= " << newEmptyListPtr->entry <type; - maxCount = MAX_SUB_RID_CNT; - m_curLevel = 0; - m_curLevelPos = 0; - m_curBlkPos = 0; - rc = readCurBlk(); - - //startPos means how many has been inserted - while (startPos < size) - { - rc = getNextInfoFromBlk(); - count = m_lastIdxRidListPtr.count; - - if (m_useNarray) - m_curLevel = ((IdxRidNextListPtr*)&m_lastIdxRidListPtr)->curLevel; - - if (remainder > (maxCount - count)) - { - insCnt = maxCount - count; - remainder = remainder - insCnt; - needNextBlk = true; - } - else - { - insCnt = remainder; - remainder = 0; - needNextBlk = false; - } - - //Prepare to write the ridList - //write the current subblock or block - for (int i = count; i < (count + insCnt) ; i++) - { - //cout << "startPos->" << startPos << endl; - RID newRID = ridList[startPos]; - rc = insertRid(newRID, i); - startPos++; - m_curIdxRidListHdr.idxRidListSize.size++; - m_hdrBlock.state = BLK_WRITE; - }//end for - - setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - m_hdrBlock.state = BLK_WRITE; - - //write out the last pointer and it depends on how much rid left - if (!needNextBlk) - { - //No more, the end. Just update the current block - //Not read from existing block, so initiate one - if ((m_useNarray) && (m_curType == LIST_BLOCK_TYPE)) - rc = updateCurCountInArray(insCnt); - else - { - m_lastIdxRidListPtr.count = count + insCnt; - rc = setNextInfoFromBlk( m_lastIdxRidListPtr); - } - - if (m_lastLbid != m_lbid) - rc = setLastLbid(m_lbid); - - if (rc != NO_ERROR) - { - return ERR_IDX_LIST_INVALID_ADD_LIST; - } - - return rc;//Done - } - else//new link - { - //take care the last entry with the new link - int lastCount = 0; - m_segType = LIST_BLOCK_TYPE; - memset(&newEmptyEntry, 0, sizeof(newEmptyEntry)); - rc = getSegment(m_pFile, ENTRY_BLK, &newEmptyEntry); - //handle current block update before move to the new block - lastCount = count + insCnt; - - if ((m_curType == LIST_SUBBLOCK_TYPE ) || (!m_useNarray)) - { - m_lastIdxRidListPtr.llp = ((IdxRidListHdrPtr*)&newEmptyEntry)->llp; - m_lastIdxRidListPtr.type = LIST_BLOCK_TYPE; - m_lastIdxRidListPtr.spare = 0x0; - m_lastIdxRidListPtr.count = lastCount; - } - - m_nextLbid = ((IdxEmptyListEntry*)&newEmptyEntry)->fbo; - m_nextSbid = 0 ; - m_nextEntry = 0 ; - m_nextType = m_segType; - - if ((m_curType == LIST_SUBBLOCK_TYPE ) || (!m_useNarray)) - { - //when current block is a subblock or single child link - rc = setNextInfoFromBlk( m_lastIdxRidListPtr); - } - else - { - rc = updateLastPtrAndParent(lastCount); - } - - //Move on to the new block for insertions - m_lbid = newEmptyEntry.fbo; - m_sbid = 0; - m_entry = 0; - m_curType = m_nextType; - - rc = readCurBlk(); - //make sure no garbage - IdxRidListPtr idxRidListPtr; - memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); - rc = setNextInfoFromBlk( idxRidListPtr); - - if (m_useNarray) - { - maxCount = MAX_BLK_NARRAY_RID_CNT; - rc = initCurBlock(); - } - else - maxCount = MAX_BLK_RID_CNT; - - count = 0; - } //end else if needs next block - }//end while + IdxEmptyListEntry newEmptyEntry; + rc = getSubBlk(m_lbid, m_sbid, m_entry); + if (rc != NO_ERROR) return rc; + + m_curType = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->type; + maxCount = MAX_SUB_RID_CNT; + m_curLevel = 0; + m_curLevelPos = 0; + m_curBlkPos = 0; + rc = readCurBlk(); + + // startPos means how many has been inserted + while (startPos < size) + { + rc = getNextInfoFromBlk(); + count = m_lastIdxRidListPtr.count; + + if (m_useNarray) + m_curLevel = ((IdxRidNextListPtr*)&m_lastIdxRidListPtr)->curLevel; + + if (remainder > (maxCount - count)) + { + insCnt = maxCount - count; + remainder = remainder - insCnt; + needNextBlk = true; + } + else + { + insCnt = remainder; + remainder = 0; + needNextBlk = false; + } + + // Prepare to write the ridList + // write the current subblock or block + for (int i = count; i < (count + insCnt); i++) + { + // cout << "startPos->" << startPos << endl; + RID newRID = ridList[startPos]; + rc = insertRid(newRID, i); + startPos++; + m_curIdxRidListHdr.idxRidListSize.size++; + m_hdrBlock.state = BLK_WRITE; + } // end for + + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + m_hdrBlock.state = BLK_WRITE; + + // write out the last pointer and it depends on how much rid left + if (!needNextBlk) + { + // No more, the end. Just update the current block + // Not read from existing block, so initiate one + if ((m_useNarray) && (m_curType == LIST_BLOCK_TYPE)) + rc = updateCurCountInArray(insCnt); + else + { + m_lastIdxRidListPtr.count = count + insCnt; + rc = setNextInfoFromBlk(m_lastIdxRidListPtr); + } + + if (m_lastLbid != m_lbid) + rc = setLastLbid(m_lbid); + + if (rc != NO_ERROR) + { + return ERR_IDX_LIST_INVALID_ADD_LIST; + } + + return rc; // Done + } + else // new link + { + // take care the last entry with the new link + int lastCount = 0; + m_segType = LIST_BLOCK_TYPE; + memset(&newEmptyEntry, 0, sizeof(newEmptyEntry)); + rc = getSegment(m_pFile, ENTRY_BLK, &newEmptyEntry); + // handle current block update before move to the new block + lastCount = count + insCnt; + + if ((m_curType == LIST_SUBBLOCK_TYPE) || (!m_useNarray)) + { + m_lastIdxRidListPtr.llp = ((IdxRidListHdrPtr*)&newEmptyEntry)->llp; + m_lastIdxRidListPtr.type = LIST_BLOCK_TYPE; + m_lastIdxRidListPtr.spare = 0x0; + m_lastIdxRidListPtr.count = lastCount; + } + + m_nextLbid = ((IdxEmptyListEntry*)&newEmptyEntry)->fbo; + m_nextSbid = 0; + m_nextEntry = 0; + m_nextType = m_segType; + + if ((m_curType == LIST_SUBBLOCK_TYPE) || (!m_useNarray)) + { + // when current block is a subblock or single child link + rc = setNextInfoFromBlk(m_lastIdxRidListPtr); + } + else + { + rc = updateLastPtrAndParent(lastCount); + } + + // Move on to the new block for insertions + m_lbid = newEmptyEntry.fbo; + m_sbid = 0; + m_entry = 0; + m_curType = m_nextType; + + rc = readCurBlk(); + // make sure no garbage + IdxRidListPtr idxRidListPtr; + memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); + rc = setNextInfoFromBlk(idxRidListPtr); + + if (m_useNarray) + { + maxCount = MAX_BLK_NARRAY_RID_CNT; + rc = initCurBlock(); + } + else + maxCount = MAX_BLK_RID_CNT; + + count = 0; + } // end else if needs next block + } // end while + + return rc; } /**************************************************************** * DESCRIPTION: @@ -300,366 +293,342 @@ const int IndexList::addRidList(const RID* ridList, const int size, * success - successfully created the index list header * failure - it did not create the index list header ***********************************************************/ -const int IndexList::updateIndexList(FILE* pFile, const RID* ridList, - const int size, const uint64_t& key, +const int IndexList::updateIndexList(FILE* pFile, const RID* ridList, const int size, const uint64_t& key, IdxEmptyListEntry* curIdxRidListHdrPtr) { - int rc; - CommBlock cb; - m_pFile = pFile; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc; + CommBlock cb; + m_pFile = pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - m_pFile = pFile; - //cout << "key=" << key << endl; - rc = init(); - rc = updateIndexList(pFile, ridList[0], key, curIdxRidListHdrPtr); - - if (size == 1) - return rc; - - rc = updateIndexList(pFile, ridList[1], key, curIdxRidListHdrPtr); - - if (size == 2) - return rc; - - rc = getHdrInfo(curIdxRidListHdrPtr); - - if (rc != NO_ERROR) - { - return rc; - } - - if (key != m_curIdxRidListHdr.key) - { - //cout << "line 829->Error Key ->" << key << endl; - //cout << "m_curIdxRidListHdr.key->"<" << m_hdrLbid << endl; - //cout << "m_hdrSbid->" << m_hdrSbid << endl; - //cout << "m_hdrEntry->" << m_hdrEntry << endl; - //cout << "m_oid->" << m_oid << endl; - return ERR_IDX_LIST_INVALID_KEY; - } - - int startPos = 2; - rc = updateIndexList(ridList, size, startPos); - - if (rc != NO_ERROR) - { - return rc; - } - - rc = updateIndexListWrite(); - - if (rc != NO_ERROR) - { - return rc; - } + m_pFile = pFile; + // cout << "key=" << key << endl; + rc = init(); + rc = updateIndexList(pFile, ridList[0], key, curIdxRidListHdrPtr); + if (size == 1) return rc; + rc = updateIndexList(pFile, ridList[1], key, curIdxRidListHdrPtr); + + if (size == 2) + return rc; + + rc = getHdrInfo(curIdxRidListHdrPtr); + + if (rc != NO_ERROR) + { + return rc; + } + + if (key != m_curIdxRidListHdr.key) + { + // cout << "line 829->Error Key ->" << key << endl; + // cout << "m_curIdxRidListHdr.key->"<" << m_hdrLbid << endl; + // cout << "m_hdrSbid->" << m_hdrSbid << endl; + // cout << "m_hdrEntry->" << m_hdrEntry << endl; + // cout << "m_oid->" << m_oid << endl; + return ERR_IDX_LIST_INVALID_KEY; + } + + int startPos = 2; + rc = updateIndexList(ridList, size, startPos); + + if (rc != NO_ERROR) + { + return rc; + } + + rc = updateIndexListWrite(); + + if (rc != NO_ERROR) + { + return rc; + } + + return rc; } /**************************************************************** -* DESCRIPTION: -* (0) THIS FUNCTION CAN ONLY BE CALLED WITH THE PUBLIC -* -* (1) Given a key value and a row ID, update the link list -* Converted -* PARAMETERS: -* Input rid -* --row ID -* Input key -* -- key value -* -* RETURN: -* success - successfully created the index list header -* failure - it did not create the index list header -***********************************************************/ -const int IndexList::updateIndexList( const RID* ridList, const int size, - int& startPos) + * DESCRIPTION: + * (0) THIS FUNCTION CAN ONLY BE CALLED WITH THE PUBLIC + * + * (1) Given a key value and a row ID, update the link list + * Converted + * PARAMETERS: + * Input rid + * --row ID + * Input key + * -- key value + * + * RETURN: + * success - successfully created the index list header + * failure - it did not create the index list header + ***********************************************************/ +const int IndexList::updateIndexList(const RID* ridList, const int size, int& startPos) { - IdxEmptyListEntry newEmptyEntry; - int rc = NO_ERROR, width = 8; - int pos = 0, totalbytes = 0, maxCount = 0; - int remainder = size - startPos; - int count = 0; - bool needNextBlk = false; + IdxEmptyListEntry newEmptyEntry; + int rc = NO_ERROR, width = 8; + int pos = 0, totalbytes = 0, maxCount = 0; + int remainder = size - startPos; + int count = 0; + bool needNextBlk = false; - IdxRidListEntry idxRidListEntry; - int oldType; + IdxRidListEntry idxRidListEntry; + int oldType; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - if (m_pFile == NULL) - return ERR_IDX_LIST_INVALID_UPDATE; + if (m_pFile == NULL) + return ERR_IDX_LIST_INVALID_UPDATE; - //Header should be available by now - if (((long long)m_hdrLbid == -1LL) || (m_hdrSbid == -1) || (m_hdrEntry == -1)) - return ERR_IDX_LIST_INVALID_UPDATE; + // Header should be available by now + if (((long long)m_hdrLbid == -1LL) || (m_hdrSbid == -1) || (m_hdrEntry == -1)) + return ERR_IDX_LIST_INVALID_UPDATE; - width = LIST_ENTRY_WIDTH; - memset(&m_lastIdxRidListPtr, 0, sizeof(m_lastIdxRidListPtr)); + width = LIST_ENTRY_WIDTH; + memset(&m_lastIdxRidListPtr, 0, sizeof(m_lastIdxRidListPtr)); - if (m_lastLbid == (uint64_t)INVALID_LBID) - { - rc = getSubBlk(m_lbid, m_sbid, m_entry); - m_curType = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->type; - //First link - pos = LIST_SUB_LLP_POS; - totalbytes = SUBBLOCK_TOTAL_BYTES; - maxCount = MAX_SUB_RID_CNT; - m_segType = LIST_SUBBLOCK_TYPE; //This is for next segment type - m_curType = LIST_SUBBLOCK_TYPE; - } + if (m_lastLbid == (uint64_t)INVALID_LBID) + { + rc = getSubBlk(m_lbid, m_sbid, m_entry); + m_curType = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->type; + // First link + pos = LIST_SUB_LLP_POS; + totalbytes = SUBBLOCK_TOTAL_BYTES; + maxCount = MAX_SUB_RID_CNT; + m_segType = LIST_SUBBLOCK_TYPE; // This is for next segment type + m_curType = LIST_SUBBLOCK_TYPE; + } + else + { + m_lbid = m_lastLbid; + m_sbid = 0; + m_entry = 0; + pos = LIST_BLOCK_LLP_POS; + totalbytes = BYTE_PER_BLOCK; + + if (m_useNarray) + maxCount = MAX_BLK_NARRAY_RID_CNT; else + maxCount = MAX_BLK_RID_CNT; + + m_curType = LIST_BLOCK_TYPE; + m_segType = LIST_BLOCK_TYPE; + } + + while (startPos < size) + { + rc = getNextInfoFromBlk(); + count = m_lastIdxRidListPtr.count; + + if (m_useNarray) + m_curLevel = ((IdxRidNextListPtr*)&m_lastIdxRidListPtr)->curLevel; + + int availCount = maxCount - count; + int insCnt = 0; + m_nextType = m_lastIdxRidListPtr.type; + + if (availCount > 0) { - m_lbid = m_lastLbid; - m_sbid = 0; - m_entry = 0; - pos = LIST_BLOCK_LLP_POS; - totalbytes = BYTE_PER_BLOCK; + if (remainder > availCount) + { + insCnt = availCount; + needNextBlk = true; + } + else + { + insCnt = remainder; + needNextBlk = false; + } - if (m_useNarray) - maxCount = MAX_BLK_NARRAY_RID_CNT; - else - maxCount = MAX_BLK_RID_CNT; + for (int i = 0; i < insCnt; i++) + { + idxRidListEntry.type = LIST_RID_TYPE; + idxRidListEntry.spare = 0x0; + idxRidListEntry.rid = ridList[startPos]; + startPos++; + remainder--; + m_lastIdxRidListPtr.count++; + availCount--; - - m_curType = LIST_BLOCK_TYPE; - m_segType = LIST_BLOCK_TYPE; - } - - while (startPos < size) - { - rc = getNextInfoFromBlk(); - count = m_lastIdxRidListPtr.count; - - if (m_useNarray) - m_curLevel = ((IdxRidNextListPtr*)&m_lastIdxRidListPtr)->curLevel; - - int availCount = maxCount - count; - int insCnt = 0; - m_nextType = m_lastIdxRidListPtr.type; - - if (availCount > 0) + if (m_lbid != m_hdrLbid) { - if (remainder > availCount) - { - insCnt = availCount; - needNextBlk = true; - } - else - { - insCnt = remainder; - needNextBlk = false; - } - - for (int i = 0; i < insCnt; i++) - { - idxRidListEntry.type = LIST_RID_TYPE; - idxRidListEntry.spare = 0x0; - idxRidListEntry.rid = ridList[startPos]; - startPos++; - remainder--; - m_lastIdxRidListPtr.count++; - availCount--; - - if (m_lbid != m_hdrLbid) - { - if ((m_curBlock.state != BLK_INIT) && - (m_curBlock.lbid == m_lbid)) - setSubBlockEntry( m_curBlock.data, m_sbid, - i + count, width, &idxRidListEntry ); - else - { - //return ERR_IDX_LIST_INVALID_UPDATE; - rc = readCurBlk(); - - if (rc != NO_ERROR) - return ERR_IDX_LIST_INVALID_BLK_READ; - - setSubBlockEntry( m_curBlock.data, m_sbid, - i + count, width, &idxRidListEntry ); - - } - - m_curBlock.dirty = true; - m_curBlock.state = BLK_WRITE; - } - else - { - setSubBlockEntry( m_hdrBlock.data, m_sbid, - i + count, width, &idxRidListEntry ); - m_hdrBlock.dirty = true; - m_hdrBlock.state = BLK_WRITE; - } - - m_curIdxRidListHdr.idxRidListSize.size++; - m_hdrBlock.state = BLK_WRITE; - }//end for - - setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - m_hdrBlock.state = BLK_WRITE; - }//endif availCount>0 - else if (remainder > 0) - { - needNextBlk = true; - } - else if (remainder == 0) - { - needNextBlk = false; - - } - - if (!needNextBlk) - { - //No more, the end. Just update the current block - if (insCnt > 0) - { - m_lastIdxRidListPtr.count = count + insCnt; - - if (m_lbid != m_hdrLbid) - { - if ((m_curBlock.lbid == m_lbid) && - (m_curBlock.state != BLK_INIT)) - setSubBlockEntry(m_curBlock.data, m_sbid, pos, width, - &m_lastIdxRidListPtr ); - else - return ERR_IDX_LIST_WRONG_LBID_WRITE; - - m_curBlock.state = BLK_WRITE; - } - else - { - setSubBlockEntry( m_hdrBlock.data, m_sbid, pos, width, - &m_lastIdxRidListPtr ); - m_hdrBlock.state = BLK_WRITE; - } - - if (m_curType == LIST_SUBBLOCK_TYPE) - { - if (m_lastLbid != INVALID_LBID) - { - uint64_t zlbid = INVALID_LBID; - rc = setLastLbid(zlbid); - } - } - else - { - if (m_lastLbid != m_lbid) - rc = setLastLbid(m_lbid); - } - - if (rc != NO_ERROR) - { - return ERR_IDX_LIST_INVALID_UPDATE; - } - - return rc;//Done - } - else - return NO_ERROR; - }//no new link - else - { - //take care the last entry with the new link - int lastCount = 0; - lastCount = count + insCnt; - m_lastIdxRidListPtr.type = LIST_BLOCK_TYPE; - m_lastIdxRidListPtr.spare = 0x0; - m_lastIdxRidListPtr.count = lastCount; - - if (m_nextType != LIST_BLOCK_TYPE) - { - m_segType = LIST_BLOCK_TYPE; - rc = getSegment(m_pFile, ENTRY_BLK, &newEmptyEntry); - - if (rc != NO_ERROR) - { - cout << "Indexlist->Free mgr getSegment ERROR CODE rc=" << rc << endl; - return rc; - } - - //handle current block update before move to the new - m_lastIdxRidListPtr.llp = - ((IdxRidListHdrPtr*)&newEmptyEntry)->llp; - //For Narray - m_nextLbid = ((IdxEmptyListEntry*)&newEmptyEntry)->fbo; - m_nextSbid = 0 ; - m_nextEntry = 0 ; - oldType = m_nextType; - m_nextType = m_segType; - - } - else - { - m_nextLbid = ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo; - m_nextSbid = 0 ; - m_nextEntry = 0 ; - oldType = m_nextType; - m_nextType = m_segType; - } - - if (m_curType == LIST_SUBBLOCK_TYPE ) - { - //when current block is a subblock - if (m_lbid == m_hdrLbid) - { - //header should be read already - setSubBlockEntry( m_hdrBlock.data, m_sbid, - pos, width, - &m_lastIdxRidListPtr ); - m_hdrBlock.state = BLK_WRITE; - } - else - { - rc = readCurBlk(); - setSubBlockEntry( m_curBlock.data, m_sbid, - pos, width, - &m_lastIdxRidListPtr ); - m_curBlock.state = BLK_WRITE; - } - } - else - { - if (m_useNarray) - rc = updateLastPtrAndParent(lastCount); - else - rc = updateLastPtr(lastCount); - } - - writeCurBlk(); - m_lbid = m_nextLbid; - m_sbid = 0; - m_entry = 0; - //m_lastLbid = m_lbid; - m_curType = m_nextType; - pos = LIST_BLOCK_LLP_POS; - totalbytes = BYTE_PER_BLOCK; + if ((m_curBlock.state != BLK_INIT) && (m_curBlock.lbid == m_lbid)) + setSubBlockEntry(m_curBlock.data, m_sbid, i + count, width, &idxRidListEntry); + else + { + // return ERR_IDX_LIST_INVALID_UPDATE; rc = readCurBlk(); - if (m_useNarray) - { - maxCount = MAX_BLK_NARRAY_RID_CNT; + if (rc != NO_ERROR) + return ERR_IDX_LIST_INVALID_BLK_READ; - if (oldType != LIST_BLOCK_TYPE) - rc = initCurBlock(); - } - else - maxCount = MAX_BLK_RID_CNT; + setSubBlockEntry(m_curBlock.data, m_sbid, i + count, width, &idxRidListEntry); + } - } //end else if needs next block + m_curBlock.dirty = true; + m_curBlock.state = BLK_WRITE; + } + else + { + setSubBlockEntry(m_hdrBlock.data, m_sbid, i + count, width, &idxRidListEntry); + m_hdrBlock.dirty = true; + m_hdrBlock.state = BLK_WRITE; + } - }//end while + m_curIdxRidListHdr.idxRidListSize.size++; + m_hdrBlock.state = BLK_WRITE; + } // end for - return rc; + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + m_hdrBlock.state = BLK_WRITE; + } // endif availCount>0 + else if (remainder > 0) + { + needNextBlk = true; + } + else if (remainder == 0) + { + needNextBlk = false; + } + + if (!needNextBlk) + { + // No more, the end. Just update the current block + if (insCnt > 0) + { + m_lastIdxRidListPtr.count = count + insCnt; + + if (m_lbid != m_hdrLbid) + { + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state != BLK_INIT)) + setSubBlockEntry(m_curBlock.data, m_sbid, pos, width, &m_lastIdxRidListPtr); + else + return ERR_IDX_LIST_WRONG_LBID_WRITE; + + m_curBlock.state = BLK_WRITE; + } + else + { + setSubBlockEntry(m_hdrBlock.data, m_sbid, pos, width, &m_lastIdxRidListPtr); + m_hdrBlock.state = BLK_WRITE; + } + + if (m_curType == LIST_SUBBLOCK_TYPE) + { + if (m_lastLbid != INVALID_LBID) + { + uint64_t zlbid = INVALID_LBID; + rc = setLastLbid(zlbid); + } + } + else + { + if (m_lastLbid != m_lbid) + rc = setLastLbid(m_lbid); + } + + if (rc != NO_ERROR) + { + return ERR_IDX_LIST_INVALID_UPDATE; + } + + return rc; // Done + } + else + return NO_ERROR; + } // no new link + else + { + // take care the last entry with the new link + int lastCount = 0; + lastCount = count + insCnt; + m_lastIdxRidListPtr.type = LIST_BLOCK_TYPE; + m_lastIdxRidListPtr.spare = 0x0; + m_lastIdxRidListPtr.count = lastCount; + + if (m_nextType != LIST_BLOCK_TYPE) + { + m_segType = LIST_BLOCK_TYPE; + rc = getSegment(m_pFile, ENTRY_BLK, &newEmptyEntry); + + if (rc != NO_ERROR) + { + cout << "Indexlist->Free mgr getSegment ERROR CODE rc=" << rc << endl; + return rc; + } + + // handle current block update before move to the new + m_lastIdxRidListPtr.llp = ((IdxRidListHdrPtr*)&newEmptyEntry)->llp; + // For Narray + m_nextLbid = ((IdxEmptyListEntry*)&newEmptyEntry)->fbo; + m_nextSbid = 0; + m_nextEntry = 0; + oldType = m_nextType; + m_nextType = m_segType; + } + else + { + m_nextLbid = ((IdxEmptyListEntry*)&m_lastIdxRidListPtr)->fbo; + m_nextSbid = 0; + m_nextEntry = 0; + oldType = m_nextType; + m_nextType = m_segType; + } + + if (m_curType == LIST_SUBBLOCK_TYPE) + { + // when current block is a subblock + if (m_lbid == m_hdrLbid) + { + // header should be read already + setSubBlockEntry(m_hdrBlock.data, m_sbid, pos, width, &m_lastIdxRidListPtr); + m_hdrBlock.state = BLK_WRITE; + } + else + { + rc = readCurBlk(); + setSubBlockEntry(m_curBlock.data, m_sbid, pos, width, &m_lastIdxRidListPtr); + m_curBlock.state = BLK_WRITE; + } + } + else + { + if (m_useNarray) + rc = updateLastPtrAndParent(lastCount); + else + rc = updateLastPtr(lastCount); + } + + writeCurBlk(); + m_lbid = m_nextLbid; + m_sbid = 0; + m_entry = 0; + // m_lastLbid = m_lbid; + m_curType = m_nextType; + pos = LIST_BLOCK_LLP_POS; + totalbytes = BYTE_PER_BLOCK; + rc = readCurBlk(); + + if (m_useNarray) + { + maxCount = MAX_BLK_NARRAY_RID_CNT; + + if (oldType != LIST_BLOCK_TYPE) + rc = initCurBlock(); + } + else + maxCount = MAX_BLK_RID_CNT; + + } // end else if needs next block + + } // end while + + return rc; } -}//end namespace - - +} // namespace WriteEngine diff --git a/writeengine/index/we_indexlist_narray.cpp b/writeengine/index/we_indexlist_narray.cpp index cbbb59792..a40ed5ca1 100644 --- a/writeengine/index/we_indexlist_narray.cpp +++ b/writeengine/index/we_indexlist_narray.cpp @@ -36,46 +36,41 @@ using namespace std; namespace WriteEngine { - /**************************************************************** * DESCRIPTION: * Private Function for print the block trees * ***************************************************************/ -const int IndexList::printBlocks(uint64_t& lbid) +const int IndexList::printBlocks(uint64_t& lbid) { - int rc; - IdxRidListArrayPtr idxRidListArrayPtr; - int curLevel = 0, curCount = 0; + int rc; + IdxRidListArrayPtr idxRidListArrayPtr; + int curLevel = 0, curCount = 0; - memset(&idxRidListArrayPtr, 0, sizeof(idxRidListArrayPtr)); - m_lbid = lbid; - rc = readCurBlk(); - getSubBlockEntry(m_curBlock.data, 0, - BEGIN_LIST_BLK_LLP_POS, - LIST_BLK_LLP_ENTRY_WIDTH, - &idxRidListArrayPtr); - curLevel = idxRidListArrayPtr.nextIdxListPtr.curLevel; - curCount = idxRidListArrayPtr.nextIdxListPtr.count; + memset(&idxRidListArrayPtr, 0, sizeof(idxRidListArrayPtr)); + m_lbid = lbid; + rc = readCurBlk(); + getSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS, LIST_BLK_LLP_ENTRY_WIDTH, &idxRidListArrayPtr); + curLevel = idxRidListArrayPtr.nextIdxListPtr.curLevel; + curCount = idxRidListArrayPtr.nextIdxListPtr.count; - for (int i = 0; i < curLevel ; i++) - { - cout << " "; - } + for (int i = 0; i < curLevel; i++) + { + cout << " "; + } - cout << " Lbid->" << lbid << " curLevel->" << curLevel - << " curCount->" << curCount << endl; + cout << " Lbid->" << lbid << " curLevel->" << curLevel << " curCount->" << curCount << endl; - for (int i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) - { - uint64_t lbid; - lbid = idxRidListArrayPtr.childIdxRidListPtr[i].childLbid; + for (int i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) + { + uint64_t lbid; + lbid = idxRidListArrayPtr.childIdxRidListPtr[i].childLbid; - if (lbid != (uint64_t)INVALID_LBID) - printBlocks(lbid); - } + if (lbid != (uint64_t)INVALID_LBID) + printBlocks(lbid); + } - return rc; + return rc; } /**************************************************************** @@ -83,157 +78,151 @@ const int IndexList::printBlocks(uint64_t& lbid) * Private Function for getting the last Fbo on header * ***************************************************************/ -const int IndexList::setCurBlkNextPtr(uint64_t& nextLbid, int count) +const int IndexList::setCurBlkNextPtr(uint64_t& nextLbid, int count) { - int rc ; - CommBlock cb; + int rc; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - if (nextLbid == (uint64_t)INVALID_LBID) - return ERR_IDX_LIST_SET_NEXT_LBID; + if (nextLbid == (uint64_t)INVALID_LBID) + return ERR_IDX_LIST_SET_NEXT_LBID; - m_idxRidListArrayPtr.nextIdxListPtr.type = (int)LIST_BLOCK_TYPE; - m_idxRidListArrayPtr.nextIdxListPtr.curLevel = m_curLevel; - m_idxRidListArrayPtr.nextIdxListPtr.count = count; - m_idxRidListArrayPtr.nextIdxListPtr.nextLbid = nextLbid; - m_idxRidListArrayPtr.nextIdxListPtr.spare = 0; + m_idxRidListArrayPtr.nextIdxListPtr.type = (int)LIST_BLOCK_TYPE; + m_idxRidListArrayPtr.nextIdxListPtr.curLevel = m_curLevel; + m_idxRidListArrayPtr.nextIdxListPtr.count = count; + m_idxRidListArrayPtr.nextIdxListPtr.nextLbid = nextLbid; + m_idxRidListArrayPtr.nextIdxListPtr.spare = 0; - if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + { + rc = writeSubBlockEntry(cb, &m_curBlock, m_lbid, 0, BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, + LIST_ENTRY_WIDTH, &m_idxRidListArrayPtr.nextIdxListPtr); + m_curBlock.state = BLK_READ; + + if (rc != NO_ERROR) + return rc; + } + else + return ERR_IDX_LIST_SET_NEXT_LBID; + + return NO_ERROR; +} + +/**************************************************************** + * DESCRIPTION: + * Private Function for setting the last Fbo on header + * + ***************************************************************/ +const int IndexList::initCurBlock() +{ + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + { + memset(&m_idxRidListArrayPtr, 0, sizeof(m_idxRidListArrayPtr)); + + for (int i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) { - rc = writeSubBlockEntry( cb, &m_curBlock, m_lbid, 0, - BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, - LIST_ENTRY_WIDTH, - &m_idxRidListArrayPtr.nextIdxListPtr ); - m_curBlock.state = BLK_READ; - - if (rc != NO_ERROR) - return rc; + m_idxRidListArrayPtr.childIdxRidListPtr[i].type = LIST_NOT_USED_TYPE; + m_idxRidListArrayPtr.childIdxRidListPtr[i].llpStat = LLP_NOT_FULL; + m_idxRidListArrayPtr.childIdxRidListPtr[i].spare = 0; + m_idxRidListArrayPtr.childIdxRidListPtr[i].childLbid = INVALID_LBID; } - else - return ERR_IDX_LIST_SET_NEXT_LBID; - return NO_ERROR; -} + m_idxRidListArrayPtr.nextIdxListPtr.type = LIST_SIZE_TYPE; + m_idxRidListArrayPtr.nextIdxListPtr.spare = 0; + m_idxRidListArrayPtr.nextIdxListPtr.curLevel = m_curLevel; + m_idxRidListArrayPtr.nextIdxListPtr.count = 0; + m_idxRidListArrayPtr.nextIdxListPtr.nextLbid = INVALID_LBID; -/**************************************************************** - * DESCRIPTION: - * Private Function for setting the last Fbo on header - * - ***************************************************************/ -const int IndexList::initCurBlock() -{ + m_idxRidListArrayPtr.parentIdxListPtr.type = LIST_LLP_TYPE; + m_idxRidListArrayPtr.parentIdxListPtr.spare = 0; + m_idxRidListArrayPtr.parentIdxListPtr.curLevelPos = m_curLevelPos; + m_idxRidListArrayPtr.parentIdxListPtr.curBlkPos = m_curBlkPos; + m_idxRidListArrayPtr.parentIdxListPtr.parentLbid = m_parentLbid; - if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) - { - memset(&m_idxRidListArrayPtr, 0, sizeof(m_idxRidListArrayPtr)); + setSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS, LIST_BLK_LLP_ENTRY_WIDTH, + &m_idxRidListArrayPtr); + } + else + return ERR_IDX_LIST_INIT_LINK_BLKS; - for (int i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) - { - m_idxRidListArrayPtr.childIdxRidListPtr[i].type = LIST_NOT_USED_TYPE; - m_idxRidListArrayPtr.childIdxRidListPtr[i].llpStat = LLP_NOT_FULL; - m_idxRidListArrayPtr.childIdxRidListPtr[i].spare = 0; - m_idxRidListArrayPtr.childIdxRidListPtr[i].childLbid = INVALID_LBID; - } - - m_idxRidListArrayPtr.nextIdxListPtr.type = LIST_SIZE_TYPE; - m_idxRidListArrayPtr.nextIdxListPtr.spare = 0; - m_idxRidListArrayPtr.nextIdxListPtr.curLevel = m_curLevel; - m_idxRidListArrayPtr.nextIdxListPtr.count = 0; - m_idxRidListArrayPtr.nextIdxListPtr.nextLbid = INVALID_LBID; - - - m_idxRidListArrayPtr.parentIdxListPtr.type = LIST_LLP_TYPE; - m_idxRidListArrayPtr.parentIdxListPtr.spare = 0; - m_idxRidListArrayPtr.parentIdxListPtr.curLevelPos = m_curLevelPos; - m_idxRidListArrayPtr.parentIdxListPtr.curBlkPos = m_curBlkPos; - m_idxRidListArrayPtr.parentIdxListPtr.parentLbid = m_parentLbid; - - setSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS, - LIST_BLK_LLP_ENTRY_WIDTH, &m_idxRidListArrayPtr); - } - else - return ERR_IDX_LIST_INIT_LINK_BLKS; - - return NO_ERROR; + return NO_ERROR; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::getNextInfoFromArray(IdxRidNextListPtr& nextIdxListPtr) +const int IndexList::getNextInfoFromArray(IdxRidNextListPtr& nextIdxListPtr) { - if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) - getSubBlockEntry(m_curBlock.data, 0, - BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, - LIST_ENTRY_WIDTH, &nextIdxListPtr); - else - return ERR_IDX_LIST_GET_NEXT; + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + getSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, LIST_ENTRY_WIDTH, + &nextIdxListPtr); + else + return ERR_IDX_LIST_GET_NEXT; - return NO_ERROR; + return NO_ERROR; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::getParentInfoFromArray(IdxRidParentListPtr& parentIdxListPtr) +const int IndexList::getParentInfoFromArray(IdxRidParentListPtr& parentIdxListPtr) { - int rc; - CommBlock cb; + int rc; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) - getSubBlockEntry(m_curBlock.data, 0, - BEGIN_LIST_BLK_LLP_POS + PARENT_PTR_OFFSET, - LIST_ENTRY_WIDTH, &parentIdxListPtr); - else - return ERR_IDX_LIST_GET_PARENT; + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + getSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + PARENT_PTR_OFFSET, LIST_ENTRY_WIDTH, + &parentIdxListPtr); + else + return ERR_IDX_LIST_GET_PARENT; - m_curLevelPos = parentIdxListPtr.curLevelPos; - m_curBlkPos = parentIdxListPtr.curBlkPos; - m_parentLbid = parentIdxListPtr.parentLbid; + m_curLevelPos = parentIdxListPtr.curLevelPos; + m_curBlkPos = parentIdxListPtr.curBlkPos; + m_parentLbid = parentIdxListPtr.parentLbid; - if (m_parentLbid == (uint64_t)INVALID_LBID) - return NO_ERROR; - - memset( m_parentBlock.data, 0, sizeof(m_parentBlock.data)); - rc = readDBFile(cb, &m_parentBlock, m_parentLbid ); - m_parentBlock.lbid = m_parentLbid; - m_parentBlock.state = BLK_READ; - m_parentBlock.dirty = true; + if (m_parentLbid == (uint64_t)INVALID_LBID) return NO_ERROR; + + memset(m_parentBlock.data, 0, sizeof(m_parentBlock.data)); + rc = readDBFile(cb, &m_parentBlock, m_parentLbid); + m_parentBlock.lbid = m_parentLbid; + m_parentBlock.state = BLK_READ; + m_parentBlock.dirty = true; + return NO_ERROR; } /**************************************************************** * DESCRIPTION: * Private Function for setting the last Fbo on header * ***************************************************************/ -const int IndexList::updateCurCountInArray(int insCnt) +const int IndexList::updateCurCountInArray(int insCnt) { - int rc = NO_ERROR; - IdxRidNextListPtr nextIdxListPtr; + int rc = NO_ERROR; + IdxRidNextListPtr nextIdxListPtr; - memset(&nextIdxListPtr, 0, sizeof(IdxRidNextListPtr)); + memset(&nextIdxListPtr, 0, sizeof(IdxRidNextListPtr)); - if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) - { - getSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, - LIST_ENTRY_WIDTH, &nextIdxListPtr); + if ((m_curBlock.lbid == m_lbid) && (m_curBlock.state >= BLK_READ)) + { + getSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, LIST_ENTRY_WIDTH, + &nextIdxListPtr); - nextIdxListPtr.count = nextIdxListPtr.count + insCnt; - setSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, - LIST_ENTRY_WIDTH, &nextIdxListPtr); - m_curBlock.state = BLK_WRITE; - } - else - return ERR_IDX_LIST_GET_COUNT; + nextIdxListPtr.count = nextIdxListPtr.count + insCnt; + setSubBlockEntry(m_curBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + NEXT_BLK_PTR_OFFSET, LIST_ENTRY_WIDTH, + &nextIdxListPtr); + m_curBlock.state = BLK_WRITE; + } + else + return ERR_IDX_LIST_GET_COUNT; - return rc; + return rc; } /**************************************************************** * DESCRIPTION: @@ -244,118 +233,115 @@ const int IndexList::updateCurCountInArray(int insCnt) ***************************************************************/ const int IndexList::updateParentStatus(uint64_t& childLbid) { - int rc; - CommBlock cb; + int rc; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - IdxRidListArrayPtr idxRidListArrayPtr; + IdxRidListArrayPtr idxRidListArrayPtr; + + // Get the parent block read out or get the 4 children pointers out + if ((m_parentBlock.state == BLK_INIT) || (m_parentBlock.lbid != m_parentLbid)) + { + rc = readSubBlockEntry(cb, &m_parentBlock, m_parentLbid, 0, BEGIN_LIST_BLK_LLP_POS, + LIST_BLK_LLP_ENTRY_WIDTH, &idxRidListArrayPtr); + m_parentBlock.dirty = true; + m_parentBlock.state = BLK_READ; + m_parentBlock.lbid = m_parentLbid; + } + else + getSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS, LIST_BLK_LLP_ENTRY_WIDTH, + &idxRidListArrayPtr); + + // update current Child Block to full + // The reason to update parent is becuse CURRENT child link is full + // The CURRENT child lbid is m_lbid, m_curBlkPos is where it was registered + if (m_lbid != m_parentLbid) // The fulled child is not the parent itself + { + // normal case found the child block is full and set it full + + if (idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].type == (int)LIST_BLOCK_TYPE) + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].llpStat = LLP_FULL; + // else if ((m_curBlkPos==0) && (m_flag)) //Next level parent, a new parent, no child + else if (m_curBlkPos == 0) // get here only when a brand new parent given, + // it went up one level and the go back down to the same level, - //Get the parent block read out or get the 4 children pointers out - if ((m_parentBlock.state == BLK_INIT) || (m_parentBlock.lbid != m_parentLbid)) { - rc = readSubBlockEntry(cb, &m_parentBlock, m_parentLbid, - 0, BEGIN_LIST_BLK_LLP_POS, - LIST_BLK_LLP_ENTRY_WIDTH, - &idxRidListArrayPtr); - m_parentBlock.dirty = true; - m_parentBlock.state = BLK_READ; - m_parentBlock.lbid = m_parentLbid; + if (!m_flag) // it is NOT the first block, this flag is useless, cannot be FALSE! + return ERR_IDX_LIST_WRONG_TYPE; + + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].type = LIST_BLOCK_TYPE; + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].llpStat = LLP_NOT_FULL; + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].spare = 0; + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].childLbid = childLbid; + setSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + m_curBlkPos, LIST_ENTRY_WIDTH, + &idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos]); + rc = writeDBFile(cb, m_parentBlock.data, m_parentLbid); + m_parentBlock.state = BLK_READ; + m_curLevelPos++; + return rc; } - else - getSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS, - LIST_BLK_LLP_ENTRY_WIDTH, - &idxRidListArrayPtr); - - //update current Child Block to full - // The reason to update parent is becuse CURRENT child link is full - // The CURRENT child lbid is m_lbid, m_curBlkPos is where it was registered - if (m_lbid != m_parentLbid)//The fulled child is not the parent itself - { - //normal case found the child block is full and set it full - - if (idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].type == (int)LIST_BLOCK_TYPE) - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].llpStat = LLP_FULL; - //else if ((m_curBlkPos==0) && (m_flag)) //Next level parent, a new parent, no child - else if (m_curBlkPos == 0) //get here only when a brand new parent given, - //it went up one level and the go back down to the same level, - - { - if (!m_flag) //it is NOT the first block, this flag is useless, cannot be FALSE! - return ERR_IDX_LIST_WRONG_TYPE; - - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].type = LIST_BLOCK_TYPE; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].llpStat = LLP_NOT_FULL; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].spare = 0; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].childLbid = childLbid; - setSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + m_curBlkPos, - LIST_ENTRY_WIDTH, &idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos]); - rc = writeDBFile( cb, m_parentBlock.data, m_parentLbid ); - m_parentBlock.state = BLK_READ; - m_curLevelPos ++; - return rc; - } - else //m_flag cannot be false since it is not the first block - return ERR_IDX_LIST_WRONG_TYPE; - } - else//This is the first block-> the current block's parent is itself - { - //only done once when first block was full - m_curBlkPos = 0; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].type = (int)LIST_BLOCK_TYPE; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].llpStat = LLP_FULL; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].spare = 0; - idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].childLbid = childLbid; - setSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + m_curBlkPos, - LIST_ENTRY_WIDTH, &idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos]); - rc = writeDBFile( cb, m_parentBlock.data, m_parentLbid ); - m_parentBlock.state = BLK_READ; - m_curLevel = 1; - m_curLevelPos = 0; - return rc; - } - - int i; - - for (i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) - { - if (idxRidListArrayPtr.childIdxRidListPtr[i].type != (int)LIST_BLOCK_TYPE) - { - idxRidListArrayPtr.childIdxRidListPtr[i].type = LIST_BLOCK_TYPE; - idxRidListArrayPtr.childIdxRidListPtr[i].llpStat = LLP_NOT_FULL; - idxRidListArrayPtr.childIdxRidListPtr[i].spare = 0; - idxRidListArrayPtr.childIdxRidListPtr[i].childLbid = childLbid; - setSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + i, - LIST_ENTRY_WIDTH, &idxRidListArrayPtr.childIdxRidListPtr[i]); - rc = writeDBFile( cb, m_parentBlock.data, m_parentLbid ); - m_parentBlock.state = BLK_READ; - m_curBlkPos = i; //Need to figure out this where to put it - m_curLevelPos ++; - return rc; - } - } - - //Parent is full and all children are full Not found any child pointer is available to add - //then go to sibling in the same level or down - //Need to look for the next block on parent block, on level down - //The level will be increment by 1 HERE - //Change parent lbid go to next link - if (m_curLevelPos == (pow(4.0, m_curLevel) - 1)) //if the last node of this level - { - m_curLevel++; - m_curLevelPos = -1; - } - - m_flag = true; // this need to go - // A new parent + else // m_flag cannot be false since it is not the first block + return ERR_IDX_LIST_WRONG_TYPE; + } + else // This is the first block-> the current block's parent is itself + { + // only done once when first block was full m_curBlkPos = 0; - m_parentLbid = idxRidListArrayPtr.nextIdxListPtr.nextLbid; - memset( m_parentBlock.data, 0, sizeof(m_parentBlock.data)); - m_parentBlock.state = BLK_INIT; - m_parentBlock.dirty = false; - - rc = updateParentStatus(childLbid); + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].type = (int)LIST_BLOCK_TYPE; + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].llpStat = LLP_FULL; + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].spare = 0; + idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos].childLbid = childLbid; + setSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + m_curBlkPos, LIST_ENTRY_WIDTH, + &idxRidListArrayPtr.childIdxRidListPtr[m_curBlkPos]); + rc = writeDBFile(cb, m_parentBlock.data, m_parentLbid); + m_parentBlock.state = BLK_READ; + m_curLevel = 1; + m_curLevelPos = 0; return rc; + } + + int i; + + for (i = 0; i < TOTAL_NUM_ARRAY_PTR; i++) + { + if (idxRidListArrayPtr.childIdxRidListPtr[i].type != (int)LIST_BLOCK_TYPE) + { + idxRidListArrayPtr.childIdxRidListPtr[i].type = LIST_BLOCK_TYPE; + idxRidListArrayPtr.childIdxRidListPtr[i].llpStat = LLP_NOT_FULL; + idxRidListArrayPtr.childIdxRidListPtr[i].spare = 0; + idxRidListArrayPtr.childIdxRidListPtr[i].childLbid = childLbid; + setSubBlockEntry(m_parentBlock.data, 0, BEGIN_LIST_BLK_LLP_POS + i, LIST_ENTRY_WIDTH, + &idxRidListArrayPtr.childIdxRidListPtr[i]); + rc = writeDBFile(cb, m_parentBlock.data, m_parentLbid); + m_parentBlock.state = BLK_READ; + m_curBlkPos = i; // Need to figure out this where to put it + m_curLevelPos++; + return rc; + } + } + + // Parent is full and all children are full Not found any child pointer is available to add + // then go to sibling in the same level or down + // Need to look for the next block on parent block, on level down + // The level will be increment by 1 HERE + // Change parent lbid go to next link + if (m_curLevelPos == (pow(4.0, m_curLevel) - 1)) // if the last node of this level + { + m_curLevel++; + m_curLevelPos = -1; + } + + m_flag = true; // this need to go + // A new parent + m_curBlkPos = 0; + m_parentLbid = idxRidListArrayPtr.nextIdxListPtr.nextLbid; + memset(m_parentBlock.data, 0, sizeof(m_parentBlock.data)); + m_parentBlock.state = BLK_INIT; + m_parentBlock.dirty = false; + + rc = updateParentStatus(childLbid); + return rc; } -} +} // namespace WriteEngine diff --git a/writeengine/index/we_indexlist_update_hdr_sub.cpp b/writeengine/index/we_indexlist_update_hdr_sub.cpp index 6a2def7eb..932a20a69 100644 --- a/writeengine/index/we_indexlist_update_hdr_sub.cpp +++ b/writeengine/index/we_indexlist_update_hdr_sub.cpp @@ -41,110 +41,98 @@ namespace WriteEngine ***********************************************************/ const int IndexList::updateHdrSub(const RID& newRid, const uint64_t& key) { - int rc = ERR_IDX_LIST_INVALID_UP_HDR; - CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + int rc = ERR_IDX_LIST_INVALID_UP_HDR; + CommBlock cb; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - if (m_curIdxRidListHdr.firstIdxRidListEntry.type - == (int)LIST_NOT_USED_TYPE) - { - m_curIdxRidListHdr.firstIdxRidListEntry.type = LIST_RID_TYPE; - m_curIdxRidListHdr.firstIdxRidListEntry.rid = newRid; - m_curIdxRidListHdr.idxRidListSize.size++; + if (m_curIdxRidListHdr.firstIdxRidListEntry.type == (int)LIST_NOT_USED_TYPE) + { + m_curIdxRidListHdr.firstIdxRidListEntry.type = LIST_RID_TYPE; + m_curIdxRidListHdr.firstIdxRidListEntry.rid = newRid; + m_curIdxRidListHdr.idxRidListSize.size++; - if (m_hdrBlock.lbid == m_hdrLbid) - rc = writeSubBlockEntry( cb, &m_hdrBlock, m_hdrLbid, - m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - else - return ERR_IDX_LIST_WRONG_LBID_WRITE; - - m_hdrBlock.state = BLK_READ; - return rc; - };//Done - - //Check Header last entry's type and go to different next step - m_nextType = m_curIdxRidListHdr.nextIdxRidListPtr.type; - - switch (m_nextType) - { - case LIST_NOT_USED_TYPE://Header is not full - //insert row id into header last enty - m_curIdxRidListHdr.nextIdxRidListPtr.type = LIST_RID_TYPE; - m_curIdxRidListHdr.nextIdxRidListPtr.llp = newRid; - m_curIdxRidListHdr.idxRidListSize.size++; - - if (m_hdrBlock.lbid == m_hdrLbid) - rc = writeSubBlockEntry( cb, &m_hdrBlock, m_hdrLbid, - m_hdrSbid, m_hdrEntry, - LIST_HDR_SIZE, &m_curIdxRidListHdr); - else - return ERR_IDX_LIST_WRONG_LBID_WRITE; - - m_hdrBlock.state = BLK_READ; - m_lastLbid = INVALID_LBID; - break; - - case LIST_RID_TYPE://Header is full, need a new sub-block - RID oldRid ; - IdxEmptyListEntry newIdxListEntry; - oldRid = m_curIdxRidListHdr.nextIdxRidListPtr.llp; - //need a sub block - m_segType = LIST_SUBBLOCK_TYPE; - - rc = moveRidsToNewSub(m_pFile, oldRid, newRid, &newIdxListEntry); - - if (rc != NO_ERROR) - return rc; - - //update header count twice - m_curIdxRidListHdr.nextIdxRidListPtr.type = m_segType; - m_curIdxRidListHdr.nextIdxRidListPtr.llp = - ((IdxRidListHdrPtr*)&newIdxListEntry)->llp; - m_curIdxRidListHdr.nextIdxRidListPtr.spare = 0; - m_curIdxRidListHdr.idxRidListSize.size++; - setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - rc = writeDBFile( cb, m_hdrBlock.data, m_hdrLbid ); - - if (rc != NO_ERROR) - return rc; - - m_hdrBlock.state = BLK_READ; - m_lastLbid = INVALID_LBID; - - if (m_curBlock.state == BLK_WRITE) - { - rc = writeDBFile( cb, m_curBlock.data, m_lbid ); - m_curBlock.state = BLK_READ; - } - - break; - - case LIST_SUBBLOCK_TYPE: //first one is a sub block - - m_lbid = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; - m_sbid = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; - m_entry = ((IdxEmptyListEntry*) - & (m_curIdxRidListHdr.nextIdxRidListPtr))->entry; - m_curType = m_nextType; - m_segType = LIST_BLOCK_TYPE; - rc = readCurBlk(); - rc = getNextInfoFromBlk(m_lastIdxRidListPtr); - rc = addRidInSub(newRid, m_lastIdxRidListPtr); - break; - - default: - rc = ERR_IDX_LIST_INVALID_UP_HDR; - break; - } //end of switch + if (m_hdrBlock.lbid == m_hdrLbid) + rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + else + return ERR_IDX_LIST_WRONG_LBID_WRITE; + m_hdrBlock.state = BLK_READ; return rc; + }; // Done + + // Check Header last entry's type and go to different next step + m_nextType = m_curIdxRidListHdr.nextIdxRidListPtr.type; + + switch (m_nextType) + { + case LIST_NOT_USED_TYPE: // Header is not full + // insert row id into header last enty + m_curIdxRidListHdr.nextIdxRidListPtr.type = LIST_RID_TYPE; + m_curIdxRidListHdr.nextIdxRidListPtr.llp = newRid; + m_curIdxRidListHdr.idxRidListSize.size++; + + if (m_hdrBlock.lbid == m_hdrLbid) + rc = writeSubBlockEntry(cb, &m_hdrBlock, m_hdrLbid, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, + &m_curIdxRidListHdr); + else + return ERR_IDX_LIST_WRONG_LBID_WRITE; + + m_hdrBlock.state = BLK_READ; + m_lastLbid = INVALID_LBID; + break; + + case LIST_RID_TYPE: // Header is full, need a new sub-block + RID oldRid; + IdxEmptyListEntry newIdxListEntry; + oldRid = m_curIdxRidListHdr.nextIdxRidListPtr.llp; + // need a sub block + m_segType = LIST_SUBBLOCK_TYPE; + + rc = moveRidsToNewSub(m_pFile, oldRid, newRid, &newIdxListEntry); + + if (rc != NO_ERROR) + return rc; + + // update header count twice + m_curIdxRidListHdr.nextIdxRidListPtr.type = m_segType; + m_curIdxRidListHdr.nextIdxRidListPtr.llp = ((IdxRidListHdrPtr*)&newIdxListEntry)->llp; + m_curIdxRidListHdr.nextIdxRidListPtr.spare = 0; + m_curIdxRidListHdr.idxRidListSize.size++; + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + rc = writeDBFile(cb, m_hdrBlock.data, m_hdrLbid); + + if (rc != NO_ERROR) + return rc; + + m_hdrBlock.state = BLK_READ; + m_lastLbid = INVALID_LBID; + + if (m_curBlock.state == BLK_WRITE) + { + rc = writeDBFile(cb, m_curBlock.data, m_lbid); + m_curBlock.state = BLK_READ; + } + + break; + + case LIST_SUBBLOCK_TYPE: // first one is a sub block + + m_lbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->fbo; + m_sbid = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->sbid; + m_entry = ((IdxEmptyListEntry*)&(m_curIdxRidListHdr.nextIdxRidListPtr))->entry; + m_curType = m_nextType; + m_segType = LIST_BLOCK_TYPE; + rc = readCurBlk(); + rc = getNextInfoFromBlk(m_lastIdxRidListPtr); + rc = addRidInSub(newRid, m_lastIdxRidListPtr); + break; + + default: rc = ERR_IDX_LIST_INVALID_UP_HDR; break; + } // end of switch + + return rc; } /************************************************ * No Change @@ -152,87 +140,86 @@ const int IndexList::updateHdrSub(const RID& newRid, const uint64_t& key) * success - successfully created the index list header * failure - it did not create the index list header ***********************************************************/ -const int IndexList::addRidInSub(const RID& newRid, - IdxRidListPtr& lastIdxRidListPtr) +const int IndexList::addRidInSub(const RID& newRid, IdxRidListPtr& lastIdxRidListPtr) { - int rc = NO_ERROR; - int maxCount; - int count; - CommBlock cb; + int rc = NO_ERROR; + int maxCount; + int count; + CommBlock cb; - cb.file.oid = m_oid; - cb.file.pFile = m_pFile; + cb.file.oid = m_oid; + cb.file.pFile = m_pFile; - m_curType = LIST_SUBBLOCK_TYPE; - m_segType = LIST_BLOCK_TYPE; - m_nextType = lastIdxRidListPtr.type; + m_curType = LIST_SUBBLOCK_TYPE; + m_segType = LIST_BLOCK_TYPE; + m_nextType = lastIdxRidListPtr.type; - maxCount = MAX_SUB_RID_CNT; - count = lastIdxRidListPtr.count; - //For n-array - m_curLevel = 0; - m_curLevelPos = 0; - m_curBlkPos = 0; - m_parentLbid = INVALID_LBID; + maxCount = MAX_SUB_RID_CNT; + count = lastIdxRidListPtr.count; + // For n-array + m_curLevel = 0; + m_curLevelPos = 0; + m_curBlkPos = 0; + m_parentLbid = INVALID_LBID; - if ((count == maxCount) && (m_nextType == LIST_SIZE_TYPE)) + if ((count == maxCount) && (m_nextType == LIST_SIZE_TYPE)) + { + // Full, need a new segment + + IdxEmptyListEntry newIdxListEntryPtr; + memset(&newIdxListEntryPtr, 0, sizeof(newIdxListEntryPtr)); + rc = getSegment(m_pFile, ENTRY_BLK, &newIdxListEntryPtr); + + if (rc != NO_ERROR) + return rc; + + lastIdxRidListPtr.type = LIST_BLOCK_TYPE; + lastIdxRidListPtr.llp = ((IdxRidListPtr*)&newIdxListEntryPtr)->llp; + lastIdxRidListPtr.spare = 0x0; + rc = setNextInfoFromBlk(lastIdxRidListPtr); + // New Block initialization + m_lbid = newIdxListEntryPtr.fbo; + m_sbid = 0; + m_entry = 0; + m_curType = m_segType; + rc = readCurBlk(); + // make sure no garbage in the new block last entry + IdxRidListPtr idxRidListPtr; + memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); + rc = setNextInfoFromBlk(idxRidListPtr); + count = 0; + + if (m_useNarray) + rc = initCurBlock(); + + if (m_lastLbid != m_lbid) { - //Full, need a new segment - - IdxEmptyListEntry newIdxListEntryPtr; - memset(&newIdxListEntryPtr, 0, sizeof(newIdxListEntryPtr)); - rc = getSegment( m_pFile, ENTRY_BLK, &newIdxListEntryPtr); - - if (rc != NO_ERROR) - return rc; - - lastIdxRidListPtr.type = LIST_BLOCK_TYPE; - lastIdxRidListPtr.llp = ((IdxRidListPtr*)&newIdxListEntryPtr)->llp; - lastIdxRidListPtr.spare = 0x0; - rc = setNextInfoFromBlk( lastIdxRidListPtr); - //New Block initialization - m_lbid = newIdxListEntryPtr.fbo; - m_sbid = 0; - m_entry = 0; - m_curType = m_segType; - rc = readCurBlk(); - //make sure no garbage in the new block last entry - IdxRidListPtr idxRidListPtr; - memset(&idxRidListPtr, 0, sizeof(idxRidListPtr)); - rc = setNextInfoFromBlk( idxRidListPtr); - count = 0; - - if (m_useNarray) - rc = initCurBlock(); - - if (m_lastLbid != m_lbid) - { - rc = setLastLbid(m_lbid); - } - }//end if FULL - else if (count < maxCount) // if less than maxCount either type =7 or 0 - { - if (m_lastLbid != INVALID_LBID) - { - uint64_t zlbid = INVALID_LBID; - rc = setLastLbid(zlbid); - } - } //endif count - else if ((count == maxCount) && (m_nextType == LIST_BLOCK_TYPE)) - { - m_lbid = ((IdxEmptyListEntry*)&lastIdxRidListPtr)->fbo; - m_sbid = 0; - m_entry = 0; - m_curType = LIST_BLOCK_TYPE; - rc = addRidInBlk(newRid); - return rc; + rc = setLastLbid(m_lbid); } - - rc = insertRid(newRid, count);// count is the position - rc = updateCurCount(); - rc = updateHdrCount(); - + } // end if FULL + else if (count < maxCount) // if less than maxCount either type =7 or 0 + { + if (m_lastLbid != INVALID_LBID) + { + uint64_t zlbid = INVALID_LBID; + rc = setLastLbid(zlbid); + } + } // endif count + else if ((count == maxCount) && (m_nextType == LIST_BLOCK_TYPE)) + { + m_lbid = ((IdxEmptyListEntry*)&lastIdxRidListPtr)->fbo; + m_sbid = 0; + m_entry = 0; + m_curType = LIST_BLOCK_TYPE; + rc = addRidInBlk(newRid); return rc; + } + + rc = insertRid(newRid, count); // count is the position + rc = updateCurCount(); + rc = updateHdrCount(); + + return rc; } /*********************************************************** * DESCRIPTION: @@ -240,34 +227,33 @@ const int IndexList::addRidInSub(const RID& newRid, * success NO_ERROR * fail ***********************************************************/ -const int IndexList::moveRidsToNewSub(FILE* pFile, const RID& oldRid, - const RID& newRid, +const int IndexList::moveRidsToNewSub(FILE* pFile, const RID& oldRid, const RID& newRid, IdxEmptyListEntry* newIdxListEntryPtr) { - int rc; - //Write everything out in getSegment - m_segType = LIST_SUBBLOCK_TYPE; + int rc; + // Write everything out in getSegment + m_segType = LIST_SUBBLOCK_TYPE; - rc = getSegment( m_pFile, ENTRY_32, newIdxListEntryPtr ); - - if (rc != NO_ERROR) - return rc; - - //This is the new segment - m_curType = LIST_SUBBLOCK_TYPE; - m_lbid = newIdxListEntryPtr->fbo; - m_sbid = newIdxListEntryPtr->sbid; - m_entry = newIdxListEntryPtr->entry; - - if (m_lbid != m_hdrLbid) - rc = readCurBlk(); - - rc = insertRid(oldRid, m_entry); - m_entry++; - rc = insertRid(newRid, m_entry); - rc = updateCurCount(2); + rc = getSegment(m_pFile, ENTRY_32, newIdxListEntryPtr); + if (rc != NO_ERROR) return rc; + + // This is the new segment + m_curType = LIST_SUBBLOCK_TYPE; + m_lbid = newIdxListEntryPtr->fbo; + m_sbid = newIdxListEntryPtr->sbid; + m_entry = newIdxListEntryPtr->entry; + + if (m_lbid != m_hdrLbid) + rc = readCurBlk(); + + rc = insertRid(oldRid, m_entry); + m_entry++; + rc = insertRid(newRid, m_entry); + rc = updateCurCount(2); + + return rc; } /**************************************************************************** * RETURN: @@ -276,17 +262,15 @@ const int IndexList::moveRidsToNewSub(FILE* pFile, const RID& oldRid, ***************************************************************************/ const int IndexList::updateHdrCount() { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (m_hdrBlock.state == BLK_INIT) - return ERR_IDX_LIST_UPDATE_HDR_COUNT; + if (m_hdrBlock.state == BLK_INIT) + return ERR_IDX_LIST_UPDATE_HDR_COUNT; - m_curIdxRidListHdr.idxRidListSize.size++; - setSubBlockEntry( m_hdrBlock.data, m_hdrSbid, - m_hdrEntry, LIST_HDR_SIZE, - &m_curIdxRidListHdr ); - m_hdrBlock.state = BLK_WRITE; + m_curIdxRidListHdr.idxRidListSize.size++; + setSubBlockEntry(m_hdrBlock.data, m_hdrSbid, m_hdrEntry, LIST_HDR_SIZE, &m_curIdxRidListHdr); + m_hdrBlock.state = BLK_WRITE; - return rc; -} + return rc; } +} // namespace WriteEngine diff --git a/writeengine/index/we_indextree.cpp b/writeengine/index/we_indextree.cpp index 0b927c9e6..64b1a8e82 100644 --- a/writeengine/index/we_indextree.cpp +++ b/writeengine/index/we_indextree.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_indextree.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -******************************************************************************************/ + * $Id: we_indextree.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + ******************************************************************************************/ /** @file */ #include @@ -30,24 +30,25 @@ namespace WriteEngine { - /** * Constructor */ IndexTree::IndexTree() - : m_useFreeMgr( true ), m_useListMgr( true ), m_useMultiCol( false), m_useMultiRid( false ), m_assignFbo( 0 ) + : m_useFreeMgr(true), m_useListMgr(true), m_useMultiCol(false), m_useMultiRid(false), m_assignFbo(0) { - clearBlock( &m_rootBlock ); + clearBlock(&m_rootBlock); } /** * Default Destructor */ IndexTree::~IndexTree() -{} +{ +} /* - const int IndexTree::getIndexTreeBitTestEntry( uint64_t entry, short* entryType, int* bitTest, int* group, int32_t* treePointer ) + const int IndexTree::getIndexTreeBitTestEntry( uint64_t entry, short* entryType, int* bitTest, int* group, + int32_t* treePointer ) { *treePointer = entry & IDX_PTR_MASK; entry = entry >> IDX_PTR_SIZE + 2; // skip one spare bit and bit-compare bit @@ -63,7 +64,8 @@ IndexTree::~IndexTree() return NO_ERROR; } - const void IndexTree::setIndexTreeBitTestEntry( uint64_t* entry, short entryType, int bitTest, int group, int32_t treePointer ) + const void IndexTree::setIndexTreeBitTestEntry( uint64_t* entry, short entryType, int bitTest, int group, + int32_t treePointer ) { memset( entry, 0, ROW_PER_BYTE ); @@ -82,12 +84,12 @@ IndexTree::~IndexTree() * RETURN: * none ***********************************************************/ -void IndexTree::clearTree( IdxTree* myTree ) +void IndexTree::clearTree(IdxTree* myTree) { - myTree->width = myTree->key = myTree->rid = myTree->maxLevel = 0; + myTree->width = myTree->key = myTree->rid = myTree->maxLevel = 0; - for ( int i = 0; i < IDX_MAX_TREE_LEVEL; i++ ) - clearTreeNode( &myTree->node[i] ); + for (int i = 0; i < IDX_MAX_TREE_LEVEL; i++) + clearTreeNode(&myTree->node[i]); } /*********************************************************** @@ -98,18 +100,17 @@ void IndexTree::clearTree( IdxTree* myTree ) * RETURN: * none ***********************************************************/ -void IndexTree::clearTreeNode( IdxTreeNode* myNode ) +void IndexTree::clearTreeNode(IdxTreeNode* myNode) { - myNode->level = 0; - myNode->allocCount = 0; - myNode->useCount = 0; -// myNode->group = 0; - myNode->used = false; - setBlankEntry( &myNode->next ); - setBlankEntry( &myNode->current ); + myNode->level = 0; + myNode->allocCount = 0; + myNode->useCount = 0; + // myNode->group = 0; + myNode->used = false; + setBlankEntry(&myNode->next); + setBlankEntry(&myNode->current); } - /*********************************************************** * DESCRIPTION: * Assign segment from free manager @@ -121,39 +122,40 @@ void IndexTree::clearTreeNode( IdxTreeNode* myNode ) * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::assignSegment( int segmentType, IdxEmptyListEntry* assignPtr, int no ) +const int IndexTree::assignSegment(int segmentType, IdxEmptyListEntry* assignPtr, int no) { - int rc = NO_ERROR; + int rc = NO_ERROR; - m_rootBlock.dirty = true; + m_rootBlock.dirty = true; - if ( m_useFreeMgr ) + if (m_useFreeMgr) + { + if (isDebug(DEBUG_3)) { - if ( isDebug( DEBUG_3 )) - { - printf( "\n++++++ Before Assign"); - printMemSubBlock( &m_rootBlock, 0, true ); - } - - rc = m_freeMgr.assignSegment( /*m_pTreeFile*/m_cbTree, &m_rootBlock, TREE, (IdxTreeGroupType) segmentType, assignPtr /*, TREE */); - - if ( isDebug( DEBUG_3 )) - { - printf( "\nAssign the pointer, entry segment=%d fbo=%2d sbid=%2d entry=%2d", segmentType, (int)assignPtr->fbo, (int)assignPtr->sbid, (int)assignPtr->entry ); - printMemSubBlock( &m_rootBlock, 0, true ); - } - } - else - { - assignPtr->fbo = 1 + m_assignFbo; - assignPtr->sbid = 3 + no; - assignPtr->entry = 4 ; + printf("\n++++++ Before Assign"); + printMemSubBlock(&m_rootBlock, 0, true); } - return rc; + rc = m_freeMgr.assignSegment(/*m_pTreeFile*/ m_cbTree, &m_rootBlock, TREE, (IdxTreeGroupType)segmentType, + assignPtr /*, TREE */); + + if (isDebug(DEBUG_3)) + { + printf("\nAssign the pointer, entry segment=%d fbo=%2d sbid=%2d entry=%2d", segmentType, + (int)assignPtr->fbo, (int)assignPtr->sbid, (int)assignPtr->entry); + printMemSubBlock(&m_rootBlock, 0, true); + } + } + else + { + assignPtr->fbo = 1 + m_assignFbo; + assignPtr->sbid = 3 + no; + assignPtr->entry = 4; + } + + return rc; } - /*********************************************************** * DESCRIPTION: * Build a complete empty tree branch @@ -166,23 +168,25 @@ const int IndexTree::assignSegment( int segmentType, IdxEmptyListEntry* assignPt * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::buildEmptyIndexTreeBranch( const uint64_t key, const int width, const RID rid, const int rootTestbitVal ) +const int IndexTree::buildEmptyIndexTreeBranch(const uint64_t key, const int width, const RID rid, + const int rootTestbitVal) { - int rc; - IdxBitmapPointerEntry bitmapEntry = {0}; + int rc; + IdxBitmapPointerEntry bitmapEntry = {0}; - rc = buildEmptyTreePart( key, width, rid, 1 ); + rc = buildEmptyTreePart(key, width, rid, 1); - // set the root level bitmapPointerMap - bitmapEntry.type = BITMAP_PTR; //BIT_TEST; - bitmapEntry.fbo = m_tree.node[0].next.fbo; - bitmapEntry.sbid = m_tree.node[0].next.sbid; - bitmapEntry.entry = m_tree.node[0].next.entry; + // set the root level bitmapPointerMap + bitmapEntry.type = BITMAP_PTR; // BIT_TEST; + bitmapEntry.fbo = m_tree.node[0].next.fbo; + bitmapEntry.sbid = m_tree.node[0].next.sbid; + bitmapEntry.entry = m_tree.node[0].next.entry; - setSubBlockEntry( m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, rootTestbitVal, MAX_COLUMN_BOUNDARY, &bitmapEntry ); - m_rootBlock.dirty = true; + setSubBlockEntry(m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, rootTestbitVal, MAX_COLUMN_BOUNDARY, + &bitmapEntry); + m_rootBlock.dirty = true; - return rc; + return rc; } /*********************************************************** @@ -198,202 +202,223 @@ const int IndexTree::buildEmptyIndexTreeBranch( const uint64_t key, const int wi * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::buildExistIndexTreeBranch( const uint64_t key, const int width, const RID rid, const int rootTestbitVal, IdxBitmapPointerEntry bitmapEntry ) +const int IndexTree::buildExistIndexTreeBranch(const uint64_t key, const int width, const RID rid, + const int rootTestbitVal, IdxBitmapPointerEntry bitmapEntry) { - int rc = NO_ERROR, loopCount, testbitVal, i, j, allocCount, realCount, matchPos, moveCount, parentLevel = 0, curLevel, curOffset = 0; - bool bSuccess; - IdxEmptyListEntry assignPtrEntry, releasePtrEntry; - IdxBitTestEntry bittestEntry, matchBitTestEntry, parentBitTestEntry, curEntry; - DataBlock curBlock, parentBlock; - bool bAddFlag = false, bDone = false, bFound, bExitOuterLoop, bExitInnerLoop, entryMap[ENTRY_PER_SUBBLOCK]; + int rc = NO_ERROR, loopCount, testbitVal, i, j, allocCount, realCount, matchPos, moveCount, parentLevel = 0, + curLevel, curOffset = 0; + bool bSuccess; + IdxEmptyListEntry assignPtrEntry, releasePtrEntry; + IdxBitTestEntry bittestEntry, matchBitTestEntry, parentBitTestEntry, curEntry; + DataBlock curBlock, parentBlock; + bool bAddFlag = false, bDone = false, bFound, bExitOuterLoop, bExitInnerLoop, entryMap[ENTRY_PER_SUBBLOCK]; - bExitOuterLoop = false; - loopCount = m_tree.maxLevel; + bExitOuterLoop = false; + loopCount = m_tree.maxLevel; - for ( i = 1; !bExitOuterLoop && i < loopCount; i++ ) + for (i = 1; !bExitOuterLoop && i < loopCount; i++) + { + // load the block + rc = readSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, + MAX_COLUMN_BOUNDARY, &bittestEntry); + + if (rc != NO_ERROR) + return rc; + + if (i == 1 && isAddrPtrEmpty(&bittestEntry, BIT_TEST)) + return ERR_STRUCT_EMPTY; + + rc = getTreeNodeInfo(&curBlock, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, + width, (IdxTreeGroupType)bittestEntry.group, &allocCount, &realCount, entryMap); + + if (rc != NO_ERROR) + return rc; + + matchBitTestEntry = bittestEntry; // assign to the value of the first entry + bSuccess = getTestbitValue(key, width, i, &testbitVal); + + setBittestEntry(&curEntry, testbitVal, bittestEntry.group, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry); + setTreeNode(&m_tree.node[i], i, allocCount, realCount, 0, bittestEntry, curEntry); + + matchBitTestEntry.bitTest = testbitVal; + bFound = + getTreeMatchEntry(&curBlock, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, + width, allocCount, entryMap, &matchPos, &matchBitTestEntry); + + if (!bFound) // this testbit not exist at the current level { - // load the block - rc = readSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, - m_tree.node[parentLevel].next.entry, MAX_COLUMN_BOUNDARY, &bittestEntry ); + bExitOuterLoop = true; // tell to exit the outer loop + bAddFlag = true; - if ( rc != NO_ERROR ) - return rc; + if (allocCount < realCount + 1) // we have enough space to take care of the extra one + { + // we don't have space to take care of the extra one, have to reassign to a big block + if (bittestEntry.group >= ENTRY_32) // it's impossible this condition holds true + return ERR_IDX_TREE_INVALID_GRP; - if ( i == 1 && isAddrPtrEmpty( &bittestEntry, BIT_TEST )) - return ERR_STRUCT_EMPTY; + m_tree.node[i].current.group++; + rc = assignSegment(m_tree.node[i].current.group, &assignPtrEntry, i); - rc = getTreeNodeInfo( &curBlock, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, width, - (IdxTreeGroupType)bittestEntry.group, &allocCount, &realCount, entryMap ); + if (rc != NO_ERROR) + return rc; - if ( rc != NO_ERROR ) - return rc; + m_tree.node[i].allocCount = 0x1 << m_tree.node[i].current.group; - - matchBitTestEntry = bittestEntry; // assign to the value of the first entry - bSuccess = getTestbitValue( key, width, i, &testbitVal ); - - setBittestEntry( &curEntry, testbitVal, bittestEntry.group, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry ); - setTreeNode( &m_tree.node[i], i, allocCount, realCount, 0, bittestEntry, curEntry ); - - matchBitTestEntry.bitTest = testbitVal; - bFound = getTreeMatchEntry( &curBlock, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, - width, allocCount, entryMap, &matchPos, &matchBitTestEntry ); - - if ( !bFound ) // this testbit not exist at the current level + if (isDebug(DEBUG_3)) { - bExitOuterLoop = true; // tell to exit the outer loop - bAddFlag = true; - - if ( allocCount < realCount + 1 ) // we have enough space to take care of the extra one - { - // we don't have space to take care of the extra one, have to reassign to a big block - if ( bittestEntry.group >= ENTRY_32 ) // it's impossible this condition holds true - return ERR_IDX_TREE_INVALID_GRP; - - m_tree.node[i].current.group++; - rc = assignSegment( m_tree.node[i].current.group, &assignPtrEntry, i ); - - if ( rc != NO_ERROR ) - return rc; - - m_tree.node[i].allocCount = 0x1 << m_tree.node[i].current.group; - - if ( isDebug( DEBUG_3 )) - { - printf( "\nEntry starting from %d:%d:%d (type %d ) will move to %d:%d:%d (type %d)", (int)m_tree.node[parentLevel].next.fbo, (int)m_tree.node[parentLevel].next.sbid, (int)m_tree.node[parentLevel].next.entry, (int)(m_tree.node[i].current.group - 1), - (int)assignPtrEntry.fbo, (int)assignPtrEntry.sbid, (int)assignPtrEntry.entry, (int)m_tree.node[i].current.group ); - printf( "\nNew space capacity is %d", (int)m_tree.node[i].allocCount ); - printf( "\nBefore the move" ); - printSubBlock( m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid ); - printSubBlock( assignPtrEntry.fbo, assignPtrEntry.sbid ); - } - - rc = moveEntry( m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, width, - assignPtrEntry.fbo, assignPtrEntry.sbid, assignPtrEntry.entry, m_tree.node[i].current.group, - allocCount, entryMap, &moveCount, m_tree.node[i].allocCount ); - - if ( rc != NO_ERROR ) - return rc; - - if ( isDebug( DEBUG_3 )) - { - printf( "\nAfter the move" ); - printSubBlock( m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid ); - printSubBlock( assignPtrEntry.fbo, assignPtrEntry.sbid ); - } - - if ( moveCount != realCount ) - return ERR_IDX_TREE_MOVE_ENTRY; - - // set release entry - setEmptyListEntry( &releasePtrEntry, bittestEntry.group, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry ); - - if ( i == 1 ) // handle bitmap parent - { - bitmapEntry.fbo = m_tree.node[0].next.fbo = m_tree.node[i].current.fbo = assignPtrEntry.fbo; - bitmapEntry.sbid = m_tree.node[0].next.sbid = m_tree.node[i].current.sbid = assignPtrEntry.sbid; - bitmapEntry.entry = m_tree.node[0].next.entry = m_tree.node[i].current.entry = assignPtrEntry.entry; - setSubBlockEntry( m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, rootTestbitVal, MAX_COLUMN_BOUNDARY, &bitmapEntry ); - } - else // handle parent for the rest of levels in the tree - { - rc = readSubBlockEntry( m_cbTree, &parentBlock, m_tree.node[parentLevel].current.fbo, m_tree.node[parentLevel].current.sbid, - m_tree.node[parentLevel].current.entry, MAX_COLUMN_BOUNDARY, &parentBitTestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - parentBitTestEntry.fbo = m_tree.node[parentLevel].next.fbo = m_tree.node[i].current.fbo = assignPtrEntry.fbo; - parentBitTestEntry.sbid = m_tree.node[parentLevel].next.sbid = m_tree.node[i].current.sbid = assignPtrEntry.sbid; - parentBitTestEntry.entry = m_tree.node[parentLevel].next.entry = m_tree.node[i].current.entry = assignPtrEntry.entry; - - rc = writeSubBlockEntry( m_cbTree, &parentBlock, m_tree.node[parentLevel].current.fbo, m_tree.node[parentLevel].current.sbid, - m_tree.node[parentLevel].current.entry, MAX_COLUMN_BOUNDARY, &parentBitTestEntry ); - - if ( rc != NO_ERROR ) - return rc; - } - - // here's the work to release the ptr - rc = releaseSegment( bittestEntry.group, &releasePtrEntry ); - - if ( rc != NO_ERROR ) - return rc; - - } // end of if( allocCount >= - - // take care of rest of empty part - rc = readSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, - m_tree.node[parentLevel].next.entry, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - m_tree.node[i].current.bitTest = testbitVal; - matchBitTestEntry.group = m_tree.node[i].current.group; - m_tree.node[i].allocCount = 0x1 << m_tree.node[i].current.group; - - bExitInnerLoop = false; - - for ( j = 0; !bExitInnerLoop && j < m_tree.node[i].allocCount; j++ ) - if ( !entryMap[j] ) // here's the empty spot - { - - if ( m_tree.maxLevel > 2 && i != loopCount - 1 ) - { - rc = buildEmptyTreePart( key, width, rid, i + 1, 0 ); - - if ( rc != NO_ERROR ) - return rc; - - bDone = true; - } - - // check out of bound - rc = readSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, - m_tree.node[parentLevel].next.entry + j, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - matchBitTestEntry.fbo = m_tree.node[i + 1].current.fbo; - matchBitTestEntry.sbid = m_tree.node[i + 1].current.sbid; - matchBitTestEntry.entry = m_tree.node[i + 1].current.entry; - - rc = writeSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, - m_tree.node[parentLevel].next.entry + j, MAX_COLUMN_BOUNDARY, &matchBitTestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - m_tree.node[i].useCount++; - - m_tree.node[i].current.entry += j; - entryMap[j] = true; - curOffset = j; - - bExitInnerLoop = true; - } // end of if( !entryMap[j] ) - - } // end of if( !bFound - else - { - m_tree.node[i].current.entry += matchPos; - m_tree.node[parentLevel].next.entry += matchPos; + printf("\nEntry starting from %d:%d:%d (type %d ) will move to %d:%d:%d (type %d)", + (int)m_tree.node[parentLevel].next.fbo, (int)m_tree.node[parentLevel].next.sbid, + (int)m_tree.node[parentLevel].next.entry, (int)(m_tree.node[i].current.group - 1), + (int)assignPtrEntry.fbo, (int)assignPtrEntry.sbid, (int)assignPtrEntry.entry, + (int)m_tree.node[i].current.group); + printf("\nNew space capacity is %d", (int)m_tree.node[i].allocCount); + printf("\nBefore the move"); + printSubBlock(m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid); + printSubBlock(assignPtrEntry.fbo, assignPtrEntry.sbid); } - m_tree.node[i].next = matchBitTestEntry; - parentLevel++; - } // end of for( i = 1; + rc = moveEntry(m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, + m_tree.node[parentLevel].next.entry, width, assignPtrEntry.fbo, assignPtrEntry.sbid, + assignPtrEntry.entry, m_tree.node[i].current.group, allocCount, entryMap, &moveCount, + m_tree.node[i].allocCount); - curLevel = m_tree.maxLevel - 1; + if (rc != NO_ERROR) + return rc; - if ( !bDone ) + if (isDebug(DEBUG_3)) + { + printf("\nAfter the move"); + printSubBlock(m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid); + printSubBlock(assignPtrEntry.fbo, assignPtrEntry.sbid); + } + + if (moveCount != realCount) + return ERR_IDX_TREE_MOVE_ENTRY; + + // set release entry + setEmptyListEntry(&releasePtrEntry, bittestEntry.group, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry); + + if (i == 1) // handle bitmap parent + { + bitmapEntry.fbo = m_tree.node[0].next.fbo = m_tree.node[i].current.fbo = assignPtrEntry.fbo; + bitmapEntry.sbid = m_tree.node[0].next.sbid = m_tree.node[i].current.sbid = assignPtrEntry.sbid; + bitmapEntry.entry = m_tree.node[0].next.entry = m_tree.node[i].current.entry = assignPtrEntry.entry; + setSubBlockEntry(m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, rootTestbitVal, MAX_COLUMN_BOUNDARY, + &bitmapEntry); + } + else // handle parent for the rest of levels in the tree + { + rc = + readSubBlockEntry(m_cbTree, &parentBlock, m_tree.node[parentLevel].current.fbo, + m_tree.node[parentLevel].current.sbid, m_tree.node[parentLevel].current.entry, + MAX_COLUMN_BOUNDARY, &parentBitTestEntry); + + if (rc != NO_ERROR) + return rc; + + parentBitTestEntry.fbo = m_tree.node[parentLevel].next.fbo = m_tree.node[i].current.fbo = + assignPtrEntry.fbo; + parentBitTestEntry.sbid = m_tree.node[parentLevel].next.sbid = m_tree.node[i].current.sbid = + assignPtrEntry.sbid; + parentBitTestEntry.entry = m_tree.node[parentLevel].next.entry = m_tree.node[i].current.entry = + assignPtrEntry.entry; + + rc = writeSubBlockEntry(m_cbTree, &parentBlock, m_tree.node[parentLevel].current.fbo, + m_tree.node[parentLevel].current.sbid, + m_tree.node[parentLevel].current.entry, MAX_COLUMN_BOUNDARY, + &parentBitTestEntry); + + if (rc != NO_ERROR) + return rc; + } + + // here's the work to release the ptr + rc = releaseSegment(bittestEntry.group, &releasePtrEntry); + + if (rc != NO_ERROR) + return rc; + + } // end of if( allocCount >= + + // take care of rest of empty part + rc = readSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, + MAX_COLUMN_BOUNDARY, &bittestEntry); + + if (rc != NO_ERROR) + return rc; + + m_tree.node[i].current.bitTest = testbitVal; + matchBitTestEntry.group = m_tree.node[i].current.group; + m_tree.node[i].allocCount = 0x1 << m_tree.node[i].current.group; + + bExitInnerLoop = false; + + for (j = 0; !bExitInnerLoop && j < m_tree.node[i].allocCount; j++) + if (!entryMap[j]) // here's the empty spot + { + if (m_tree.maxLevel > 2 && i != loopCount - 1) + { + rc = buildEmptyTreePart(key, width, rid, i + 1, 0); + + if (rc != NO_ERROR) + return rc; + + bDone = true; + } + + // check out of bound + rc = readSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry + j, + MAX_COLUMN_BOUNDARY, &bittestEntry); + + if (rc != NO_ERROR) + return rc; + + matchBitTestEntry.fbo = m_tree.node[i + 1].current.fbo; + matchBitTestEntry.sbid = m_tree.node[i + 1].current.sbid; + matchBitTestEntry.entry = m_tree.node[i + 1].current.entry; + + rc = writeSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry + j, + MAX_COLUMN_BOUNDARY, &matchBitTestEntry); + + if (rc != NO_ERROR) + return rc; + + m_tree.node[i].useCount++; + + m_tree.node[i].current.entry += j; + entryMap[j] = true; + curOffset = j; + + bExitInnerLoop = true; + } // end of if( !entryMap[j] ) + + } // end of if( !bFound + else { - rc = updateListFile( key, width, rid, curLevel, m_tree.node[curLevel].current.group, m_tree.node[curLevel].allocCount, m_tree.node[curLevel].useCount, curOffset, bAddFlag ); + m_tree.node[i].current.entry += matchPos; + m_tree.node[parentLevel].next.entry += matchPos; } - return rc; + m_tree.node[i].next = matchBitTestEntry; + parentLevel++; + } // end of for( i = 1; + + curLevel = m_tree.maxLevel - 1; + + if (!bDone) + { + rc = + updateListFile(key, width, rid, curLevel, m_tree.node[curLevel].current.group, + m_tree.node[curLevel].allocCount, m_tree.node[curLevel].useCount, curOffset, bAddFlag); + } + + return rc; } /*********************************************************** @@ -409,71 +434,75 @@ const int IndexTree::buildExistIndexTreeBranch( const uint64_t key, const int wi * error no if fail * retBitTestEntry - return address pointer ***********************************************************/ -const int IndexTree::buildEmptyTreePart( const uint64_t key, const int width, const RID rid, const int startLevel, const int offset ) +const int IndexTree::buildEmptyTreePart(const uint64_t key, const int width, const RID rid, + const int startLevel, const int offset) { - int rc, loopCount, testbitVal, i, parentLevel; - bool bSuccess; - IdxEmptyListEntry assignPtrEntry, childPtrEntry;//, listEntry; - IdxBitTestEntry bittestEntry, curEntry; - DataBlock curBlock; + int rc, loopCount, testbitVal, i, parentLevel; + bool bSuccess; + IdxEmptyListEntry assignPtrEntry, childPtrEntry; //, listEntry; + IdxBitTestEntry bittestEntry, curEntry; + DataBlock curBlock; - if ( startLevel <= 0 || m_tree.maxLevel < 2 ) // the start level must >= 1 and maxLevel >= 2 - return ERR_IDX_TREE_INVALID_LEVEL; + if (startLevel <= 0 || m_tree.maxLevel < 2) // the start level must >= 1 and maxLevel >= 2 + return ERR_IDX_TREE_INVALID_LEVEL; - loopCount = (m_tree.maxLevel - 1) > 1 ? m_tree.maxLevel - 1 : 0; //( width/5 ) - 1; + loopCount = (m_tree.maxLevel - 1) > 1 ? m_tree.maxLevel - 1 : 0; //( width/5 ) - 1; - rc = assignSegment( ENTRY_1, &assignPtrEntry, 0 ); - - if ( rc != NO_ERROR ) - return rc; - - if ( isAddrPtrEmpty( &assignPtrEntry, EMPTY_LIST ) ) - return ERR_STRUCT_EMPTY; - - parentLevel = startLevel - 1; - - // assuming the parent take care of group, bit test value, and type - m_tree.node[parentLevel].next.fbo = assignPtrEntry.fbo; - m_tree.node[parentLevel].next.sbid = assignPtrEntry.sbid; - m_tree.node[parentLevel].next.entry = assignPtrEntry.entry; - - // assign bit test for rest of levels - for ( i = startLevel; i < loopCount; i++ ) - { - // assign another one for child - rc = assignSegment( ENTRY_1, &childPtrEntry, i ); - - if ( rc != NO_ERROR ) - return rc; - - rc = readSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, - m_tree.node[parentLevel].next.entry, /*width*/MAX_COLUMN_BOUNDARY, &bittestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - bSuccess = getTestbitValue( key, width, i, &testbitVal ); - - if ( !bSuccess ) - return ERR_IDX_TREE_INVALID_LEVEL; - - setBittestEntry( &bittestEntry, testbitVal, ENTRY_1, childPtrEntry.fbo, childPtrEntry.sbid, childPtrEntry.entry ); - setSubBlockEntry( &curBlock, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - writeDBFile( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo ); - - setBittestEntry( &curEntry, testbitVal, ENTRY_1, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry ); - setTreeNode( &m_tree.node[i], i, 1, 1, 0, bittestEntry, curEntry ); - parentLevel++; - } - - // assign the bit test for the last level - // load the last piece - rc = updateListFile( key, width, rid, i, ENTRY_1, 1, 1, offset, true ); + rc = assignSegment(ENTRY_1, &assignPtrEntry, 0); + if (rc != NO_ERROR) return rc; -} + if (isAddrPtrEmpty(&assignPtrEntry, EMPTY_LIST)) + return ERR_STRUCT_EMPTY; + + parentLevel = startLevel - 1; + + // assuming the parent take care of group, bit test value, and type + m_tree.node[parentLevel].next.fbo = assignPtrEntry.fbo; + m_tree.node[parentLevel].next.sbid = assignPtrEntry.sbid; + m_tree.node[parentLevel].next.entry = assignPtrEntry.entry; + + // assign bit test for rest of levels + for (i = startLevel; i < loopCount; i++) + { + // assign another one for child + rc = assignSegment(ENTRY_1, &childPtrEntry, i); + + if (rc != NO_ERROR) + return rc; + + rc = readSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, + /*width*/ MAX_COLUMN_BOUNDARY, &bittestEntry); + + if (rc != NO_ERROR) + return rc; + + bSuccess = getTestbitValue(key, width, i, &testbitVal); + + if (!bSuccess) + return ERR_IDX_TREE_INVALID_LEVEL; + + setBittestEntry(&bittestEntry, testbitVal, ENTRY_1, childPtrEntry.fbo, childPtrEntry.sbid, + childPtrEntry.entry); + setSubBlockEntry(&curBlock, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, + MAX_COLUMN_BOUNDARY, &bittestEntry); + + writeDBFile(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo); + + setBittestEntry(&curEntry, testbitVal, ENTRY_1, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry); + setTreeNode(&m_tree.node[i], i, 1, 1, 0, bittestEntry, curEntry); + parentLevel++; + } + + // assign the bit test for the last level + // load the last piece + rc = updateListFile(key, width, rid, i, ENTRY_1, 1, 1, offset, true); + + return rc; +} /*********************************************************** * DESCRIPTION: @@ -485,24 +514,23 @@ const int IndexTree::buildEmptyTreePart( const uint64_t key, const int width, co ***********************************************************/ void IndexTree::closeIndex() { - - if ( m_rootBlock.dirty ) - { - uint64_t lbid0 = 0; + if (m_rootBlock.dirty) + { + uint64_t lbid0 = 0; #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - BRMWrapper::getInstance()->getBrmInfo( m_cbTree.file.oid, 0, lbid0 ); + BRMWrapper::getInstance()->getBrmInfo(m_cbTree.file.oid, 0, lbid0); #endif - writeDBFile( m_cbTree, m_rootBlock.data, lbid0); - } + writeDBFile(m_cbTree, m_rootBlock.data, lbid0); + } - if ( Cache::getUseCache() ) - flushCache(); + if (Cache::getUseCache()) + flushCache(); - m_fileopTree.closeFile( m_cbTree.file.pFile ); - m_fileopList.closeFile( m_cbList.file.pFile ); - m_cbTree.file.pFile = NULL; - m_cbList.file.pFile = NULL; - clear(); + m_fileopTree.closeFile(m_cbTree.file.pFile); + m_fileopList.closeFile(m_cbList.file.pFile); + m_cbTree.file.pFile = NULL; + m_cbList.file.pFile = NULL; + clear(); } /*********************************************************** @@ -516,24 +544,24 @@ void IndexTree::closeIndex() * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::createIndex( const FID treeFid, const FID listFid, const bool useFreeMgrFlag ) +const int IndexTree::createIndex(const FID treeFid, const FID listFid, const bool useFreeMgrFlag) { #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - int allocSize; + int allocSize; - // init -// clear(); + // init + // clear(); - RETURN_ON_ERROR( createFile( treeFid, DEFAULT_TOTAL_BLOCK/* * 10*/, allocSize ) ); - RETURN_ON_ERROR( createFile( listFid, DEFAULT_TOTAL_BLOCK/* * 10*/, allocSize ) ); + RETURN_ON_ERROR(createFile(treeFid, DEFAULT_TOTAL_BLOCK /* * 10*/, allocSize)); + RETURN_ON_ERROR(createFile(listFid, DEFAULT_TOTAL_BLOCK /* * 10*/, allocSize)); #endif - // load index files -// RETURN_ON_ERROR( openIndex( treeFid, listFid ) ); -// rc = initIndex( treeFid, listFid ); -// closeIndex(); + // load index files + // RETURN_ON_ERROR( openIndex( treeFid, listFid ) ); + // rc = initIndex( treeFid, listFid ); + // closeIndex(); - return initIndex( treeFid, listFid ); + return initIndex(treeFid, listFid); } /*********************************************************** @@ -547,14 +575,13 @@ const int IndexTree::createIndex( const FID treeFid, const FID listFid, const bo * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::deleteIndex( const uint64_t key, const int width, const RID rid ) +const int IndexTree::deleteIndex(const uint64_t key, const int width, const RID rid) { - IdxEmptyListEntry listHdrAddr; + IdxEmptyListEntry listHdrAddr; - return processIndex( key, width, rid, listHdrAddr ); + return processIndex(key, width, rid, listHdrAddr); } - /*********************************************************** * DESCRIPTION: * Get test bit values @@ -566,36 +593,36 @@ const int IndexTree::deleteIndex( const uint64_t key, const int width, const RID * True if success, otherwise if out of bound * bittestVal - test bit value ***********************************************************/ -const bool IndexTree::getTestbitValue( const uint64_t key, const int width, const int curTestNo, int* bittestVal ) +const bool IndexTree::getTestbitValue(const uint64_t key, const int width, const int curTestNo, + int* bittestVal) { - int shiftPos, maskPos = 0; - bool bSuccess = true; + int shiftPos, maskPos = 0; + bool bSuccess = true; - if ( !m_useMultiCol ) - { - *bittestVal = 0; - shiftPos = width - ( curTestNo + 1 ) * 5; + if (!m_useMultiCol) + { + *bittestVal = 0; + shiftPos = width - (curTestNo + 1) * 5; - if ( shiftPos > 0 ) - *bittestVal = getBitValue( key, shiftPos, BIT_MASK_ARRAY[5] ); - else - { - if ( shiftPos >= -4 ) - { - maskPos = width - curTestNo * 5 ; - *bittestVal = key & BIT_MASK_ARRAY[maskPos]; - } - else - bSuccess = false; - } - } + if (shiftPos > 0) + *bittestVal = getBitValue(key, shiftPos, BIT_MASK_ARRAY[5]); else - *bittestVal = m_multiColKey.testbitArray[curTestNo]; + { + if (shiftPos >= -4) + { + maskPos = width - curTestNo * 5; + *bittestVal = key & BIT_MASK_ARRAY[maskPos]; + } + else + bSuccess = false; + } + } + else + *bittestVal = m_multiColKey.testbitArray[curTestNo]; - return bSuccess; + return bSuccess; } - /*********************************************************** * DESCRIPTION: * Get the matching entry within the current tree node @@ -610,26 +637,27 @@ const bool IndexTree::getTestbitValue( const uint64_t key, const int width, cons * True if found, False otherwise * checkEntry - if found the ptr got reset ***********************************************************/ -const bool IndexTree::getTreeMatchEntry( DataBlock* block, const uint64_t sbid, const uint64_t entry, const int width, - const int allocCount, const bool* entryMap, int* matchEntry, IdxBitTestEntry* checkEntry ) +const bool IndexTree::getTreeMatchEntry(DataBlock* block, const uint64_t sbid, const uint64_t entry, + const int width, const int allocCount, const bool* entryMap, + int* matchEntry, IdxBitTestEntry* checkEntry) { - IdxBitTestEntry curEntry; - bool bFoundFlag = false; + IdxBitTestEntry curEntry; + bool bFoundFlag = false; - for ( int i = 0; i < allocCount; i++ ) + for (int i = 0; i < allocCount; i++) + { + getSubBlockEntry(block->data, sbid, entry + i, MAX_COLUMN_BOUNDARY, &curEntry); + + if (entryMap[i] && (curEntry.type == checkEntry->type && curEntry.bitTest == checkEntry->bitTest)) { - getSubBlockEntry( block->data, sbid, entry + i, MAX_COLUMN_BOUNDARY, &curEntry ); - - if ( entryMap[i] && ( curEntry.type == checkEntry->type && curEntry.bitTest == checkEntry->bitTest ) ) - { - *checkEntry = curEntry; - *matchEntry = i; - bFoundFlag = true; - break; - } + *checkEntry = curEntry; + *matchEntry = i; + bFoundFlag = true; + break; } + } - return bFoundFlag; + return bFoundFlag; } /*********************************************************** @@ -648,35 +676,36 @@ const bool IndexTree::getTreeMatchEntry( DataBlock* block, const uint64_t sbid, * realCount - the total number of real entries * entryMap - the entry availablibility map ***********************************************************/ -const int IndexTree::getTreeNodeInfo( DataBlock* block, const uint64_t sbid, const uint64_t entry, const int width, - const IdxTreeGroupType group, int* allocCount, int* realCount, bool* entryMap ) +const int IndexTree::getTreeNodeInfo(DataBlock* block, const uint64_t sbid, const uint64_t entry, + const int width, const IdxTreeGroupType group, int* allocCount, + int* realCount, bool* entryMap) { - IdxBitTestEntry curEntry; - int rc = NO_ERROR; + IdxBitTestEntry curEntry; + int rc = NO_ERROR; - memset( entryMap, false, ENTRY_PER_SUBBLOCK ); - *realCount = 0; - *allocCount = 0x1 << group; + memset(entryMap, false, ENTRY_PER_SUBBLOCK); + *realCount = 0; + *allocCount = 0x1 << group; - for ( int i = 0; i < *allocCount; i++ ) + for (int i = 0; i < *allocCount; i++) + { + getSubBlockEntry(block->data, sbid, entry + i, MAX_COLUMN_BOUNDARY, &curEntry); + + if (curEntry.type == BIT_TEST || curEntry.type == LEAF_LIST) // every guy here must have the same type { - getSubBlockEntry( block->data, sbid, entry + i, MAX_COLUMN_BOUNDARY, &curEntry ); - - if ( curEntry.type == BIT_TEST || curEntry.type == LEAF_LIST ) // every guy here must have the same type - { - entryMap[i] = true; - *realCount = *realCount + 1; - } - else if ( curEntry.type == EMPTY_ENTRY ) - entryMap[i] = false; - else - rc = ERR_IDX_TREE_INVALID_TYPE; - - if ( rc != NO_ERROR ) - break; + entryMap[i] = true; + *realCount = *realCount + 1; } + else if (curEntry.type == EMPTY_ENTRY) + entryMap[i] = false; + else + rc = ERR_IDX_TREE_INVALID_TYPE; - return rc; + if (rc != NO_ERROR) + break; + } + + return rc; } /*********************************************************** @@ -688,78 +717,76 @@ const int IndexTree::getTreeNodeInfo( DataBlock* block, const uint64_t sbid, con * RETURN: * True if empty, False otherwise ***********************************************************/ -const bool IndexTree::isAddrPtrEmpty( void* pStruct, const IdxTreeEntryType entryType ) const +const bool IndexTree::isAddrPtrEmpty(void* pStruct, const IdxTreeEntryType entryType) const { - bool bStatus; - IdxBitmapPointerEntry* pBitmap; - IdxBitTestEntry* pBittest; - IdxEmptyListEntry* pEmptyList; + bool bStatus; + IdxBitmapPointerEntry* pBitmap; + IdxBitTestEntry* pBittest; + IdxEmptyListEntry* pEmptyList; + switch (entryType) + { + case BITMAP_PTR /*EMPTY_PTR*/: // this is the case for bitmap pointer address + pBitmap = (IdxBitmapPointerEntry*)pStruct; + bStatus = /*!pBitmap->oid &&*/ !pBitmap->fbo && !pBitmap->sbid && !pBitmap->entry; + break; - switch ( entryType ) - { - case BITMAP_PTR/*EMPTY_PTR*/ : // this is the case for bitmap pointer address - pBitmap = (IdxBitmapPointerEntry*) pStruct; - bStatus = /*!pBitmap->oid &&*/ !pBitmap->fbo && !pBitmap->sbid && !pBitmap->entry; - break; + case BIT_TEST: // this is the case for bittest pointer address + pBittest = (IdxBitTestEntry*)pStruct; + bStatus = /*!pBittest->oid &&*/ !pBittest->fbo && !pBittest->sbid && !pBittest->entry; + break; - case BIT_TEST : // this is the case for bittest pointer address - pBittest = (IdxBitTestEntry*) pStruct; - bStatus = /*!pBittest->oid &&*/ !pBittest->fbo && !pBittest->sbid && !pBittest->entry; - break; + case EMPTY_LIST: // this is the case for bittest pointer address + pEmptyList = (IdxEmptyListEntry*)pStruct; + bStatus = !pEmptyList->fbo && !pEmptyList->sbid && !pEmptyList->entry; + break; - case EMPTY_LIST : // this is the case for bittest pointer address - pEmptyList = (IdxEmptyListEntry*) pStruct; - bStatus = !pEmptyList->fbo && !pEmptyList->sbid && !pEmptyList->entry; - break; + default: bStatus = true; + } - default : - bStatus = true; - } - - return bStatus; + return bStatus; } -const int IndexTree::initIndex( const FID treeFid, const FID listFid ) +const int IndexTree::initIndex(const FID treeFid, const FID listFid) { - int rc = NO_ERROR; - long numOfBlock; - unsigned char writeBuf[BYTE_PER_BLOCK]; + int rc = NO_ERROR; + long numOfBlock; + unsigned char writeBuf[BYTE_PER_BLOCK]; - bool oldUseVb = BRMWrapper::getUseVb(); - BRMWrapper::setUseVb( false ); + bool oldUseVb = BRMWrapper::getUseVb(); + BRMWrapper::setUseVb(false); - clear(); - RETURN_ON_ERROR( openIndex( treeFid, listFid ) ); + clear(); + RETURN_ON_ERROR(openIndex(treeFid, listFid)); - memset( writeBuf, 0, BYTE_PER_BLOCK ); - numOfBlock = getFileSize( m_cbTree.file.pFile ) / BYTE_PER_BLOCK; + memset(writeBuf, 0, BYTE_PER_BLOCK); + numOfBlock = getFileSize(m_cbTree.file.pFile) / BYTE_PER_BLOCK; - for ( int i = 0; i < numOfBlock; i++ ) - fwrite( writeBuf, sizeof( writeBuf ), 1, m_cbTree.file.pFile ); + for (int i = 0; i < numOfBlock; i++) + fwrite(writeBuf, sizeof(writeBuf), 1, m_cbTree.file.pFile); - numOfBlock = getFileSize( m_cbList.file.pFile ) / BYTE_PER_BLOCK; + numOfBlock = getFileSize(m_cbList.file.pFile) / BYTE_PER_BLOCK; - for ( int i = 0; i < numOfBlock; i++ ) - fwrite( writeBuf, sizeof( writeBuf ), 1, m_cbList.file.pFile ); + for (int i = 0; i < numOfBlock; i++) + fwrite(writeBuf, sizeof(writeBuf), 1, m_cbList.file.pFile); - // very weird, have to close before we can call free mgr init - closeIndex(); - RETURN_ON_ERROR( openIndex( treeFid, listFid ) ); + // very weird, have to close before we can call free mgr init + closeIndex(); + RETURN_ON_ERROR(openIndex(treeFid, listFid)); - rc = m_freeMgr.init( m_cbTree, TREE ); + rc = m_freeMgr.init(m_cbTree, TREE); - if ( rc == NO_ERROR ) - rc = m_freeMgr.init( m_cbList, LIST ); + if (rc == NO_ERROR) + rc = m_freeMgr.init(m_cbList, LIST); - closeIndex(); + closeIndex(); - BRMWrapper::setUseVb( oldUseVb ); + BRMWrapper::setUseVb(oldUseVb); - if ( isDebug( DEBUG_1 ) ) - printf( "\nEnd of the init for oid %d\n", m_cbTree.file.oid ); + if (isDebug(DEBUG_1)) + printf("\nEnd of the init for oid %d\n", m_cbTree.file.oid); - return rc; + return rc; } /*********************************************************** @@ -774,65 +801,68 @@ const int IndexTree::initIndex( const FID treeFid, const FID listFid ) * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::moveEntry( const uint64_t oldFbo, const uint64_t oldSbid, const uint64_t oldEntry, const int width, const uint64_t newFbo, - const uint64_t newSbid, const uint64_t newEntry, const int newGroup, const int allocCount, bool* entryMap, int* moveCount, const int newAllocCount ) +const int IndexTree::moveEntry(const uint64_t oldFbo, const uint64_t oldSbid, const uint64_t oldEntry, + const int width, const uint64_t newFbo, const uint64_t newSbid, + const uint64_t newEntry, const int newGroup, const int allocCount, + bool* entryMap, int* moveCount, const int newAllocCount) { - int rc, i; - DataBlock oldBlock, newBlock; - IdxBitTestEntry curEntry, blankEntry; + int rc, i; + DataBlock oldBlock, newBlock; + IdxBitTestEntry curEntry, blankEntry; - rc = readSubBlockEntry( m_cbTree, &oldBlock, oldFbo, oldSbid, oldEntry, /*width*/MAX_COLUMN_BOUNDARY, &curEntry ); + rc = readSubBlockEntry(m_cbTree, &oldBlock, oldFbo, oldSbid, oldEntry, /*width*/ MAX_COLUMN_BOUNDARY, + &curEntry); - if ( rc != NO_ERROR ) - return rc; + if (rc != NO_ERROR) + return rc; - rc = readSubBlockEntry( m_cbTree, &newBlock, newFbo, newSbid, newEntry, /*width*/MAX_COLUMN_BOUNDARY, &curEntry ); + rc = readSubBlockEntry(m_cbTree, &newBlock, newFbo, newSbid, newEntry, /*width*/ MAX_COLUMN_BOUNDARY, + &curEntry); - if ( rc != NO_ERROR ) - return rc; + if (rc != NO_ERROR) + return rc; - setBlankEntry( &blankEntry ); + setBlankEntry(&blankEntry); - for ( i = 0; i < newAllocCount; i++ ) - setSubBlockEntry( newBlock.data, newSbid, newEntry + i, MAX_COLUMN_BOUNDARY, &blankEntry ); + for (i = 0; i < newAllocCount; i++) + setSubBlockEntry(newBlock.data, newSbid, newEntry + i, MAX_COLUMN_BOUNDARY, &blankEntry); - *moveCount = 0; + *moveCount = 0; - for ( i = 0; i < allocCount; i++ ) + for (i = 0; i < allocCount; i++) + { + getSubBlockEntry(oldBlock.data, oldSbid, oldEntry + i, MAX_COLUMN_BOUNDARY, &curEntry); + + if (entryMap[i]) { - getSubBlockEntry( oldBlock.data, oldSbid, oldEntry + i, MAX_COLUMN_BOUNDARY, &curEntry ); - - if ( entryMap[i] ) - { - curEntry.group = newGroup; - setSubBlockEntry( newBlock.data, newSbid, newEntry + *moveCount, MAX_COLUMN_BOUNDARY, &curEntry ); - *moveCount = *moveCount + 1; - - } - - setBlankEntry( &curEntry ); - - if ( newFbo != oldFbo ) - setSubBlockEntry( oldBlock.data, oldSbid, oldEntry + i, MAX_COLUMN_BOUNDARY, &curEntry ); - else - setSubBlockEntry( newBlock.data, oldSbid, oldEntry + i, MAX_COLUMN_BOUNDARY, &curEntry ); + curEntry.group = newGroup; + setSubBlockEntry(newBlock.data, newSbid, newEntry + *moveCount, MAX_COLUMN_BOUNDARY, &curEntry); + *moveCount = *moveCount + 1; } - rc = writeDBFile( m_cbTree, &newBlock, newFbo ); + setBlankEntry(&curEntry); - if ( rc != NO_ERROR ) - return rc; + if (newFbo != oldFbo) + setSubBlockEntry(oldBlock.data, oldSbid, oldEntry + i, MAX_COLUMN_BOUNDARY, &curEntry); + else + setSubBlockEntry(newBlock.data, oldSbid, oldEntry + i, MAX_COLUMN_BOUNDARY, &curEntry); + } - if ( newFbo != oldFbo ) - rc = writeDBFile( m_cbTree, &oldBlock, oldFbo ); - - // reset map - memset( entryMap, false, ENTRY_PER_SUBBLOCK ); - - for ( i = 0; i < *moveCount; i++ ) - entryMap[i] = true; + rc = writeDBFile(m_cbTree, &newBlock, newFbo); + if (rc != NO_ERROR) return rc; + + if (newFbo != oldFbo) + rc = writeDBFile(m_cbTree, &oldBlock, oldFbo); + + // reset map + memset(entryMap, false, ENTRY_PER_SUBBLOCK); + + for (i = 0; i < *moveCount; i++) + entryMap[i] = true; + + return rc; } /*********************************************************** @@ -845,51 +875,51 @@ const int IndexTree::moveEntry( const uint64_t oldFbo, const uint64_t oldSbid, c * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::openIndex( const FID treeFid, const FID listFid ) +const int IndexTree::openIndex(const FID treeFid, const FID listFid) { - int rc = NO_ERROR; - clear(); + int rc = NO_ERROR; + clear(); - // load index files + // load index files #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - m_cbTree.file.pFile = m_fileopTree.openFile( treeFid ); + m_cbTree.file.pFile = m_fileopTree.openFile(treeFid); #endif - if ( m_cbTree.file.pFile == NULL ) - return ERR_FILE_OPEN; + if (m_cbTree.file.pFile == NULL) + return ERR_FILE_OPEN; #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - m_cbList.file.pFile = m_fileopList.openFile( listFid ); + m_cbList.file.pFile = m_fileopList.openFile(listFid); #endif - if ( m_cbList.file.pFile == NULL ) + if (m_cbList.file.pFile == NULL) + { + m_fileopTree.closeFile(m_cbTree.file.pFile); // close the one just open + return ERR_FILE_OPEN; + } + + m_cbTree.file.oid = treeFid; + m_cbList.file.oid = listFid; + + uint64_t lbid0 = 0; +#ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID + rc = BRMWrapper::getInstance()->getBrmInfo(m_cbTree.file.oid, 0, lbid0); +#endif + + if (rc != NO_ERROR) + { + if (isDebug(DEBUG_1)) { - m_fileopTree.closeFile( m_cbTree.file.pFile ); // close the one just open - return ERR_FILE_OPEN; + printf("\nFor oid : %d block zero is %ld", m_cbTree.file.oid, (long)lbid0); + printf("\nIn open index, have problem in get brmInfo, rc=%d", rc); } - m_cbTree.file.oid = treeFid; - m_cbList.file.oid = listFid; - - uint64_t lbid0 = 0; -#ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - rc = BRMWrapper::getInstance()->getBrmInfo( m_cbTree.file.oid, 0, lbid0 ); -#endif - - if ( rc != NO_ERROR ) - { - if ( isDebug( DEBUG_1 ) ) - { - printf( "\nFor oid : %d block zero is %ld", m_cbTree.file.oid, (long)lbid0 ); - printf( "\nIn open index, have problem in get brmInfo, rc=%d", rc ); - } - - return rc; - } - - rc = readDBFile( m_cbTree, m_rootBlock.data, lbid0 ); - return rc; + } + + rc = readDBFile(m_cbTree, m_rootBlock.data, lbid0); + + return rc; } /*********************************************************** @@ -905,107 +935,111 @@ const int IndexTree::openIndex( const FID treeFid, const FID listFid ) * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::processIndex( const uint64_t key, const int width, const RID rid, IdxEmptyListEntry& listHdrAddr, const bool bDelete ) +const int IndexTree::processIndex(const uint64_t key, const int width, const RID rid, + IdxEmptyListEntry& listHdrAddr, const bool bDelete) { - int loopCount, testbitVal, i, allocCount, realCount, matchPos, curFbo, curSbid, curEntry; - bool bSuccess; - IdxEmptyListEntry listEntry; - IdxBitTestEntry bittestEntry, matchBitTestEntry; - DataBlock curBlock; - bool bFound, entryMap[ENTRY_PER_SUBBLOCK]; + int loopCount, testbitVal, i, allocCount, realCount, matchPos, curFbo, curSbid, curEntry; + bool bSuccess; + IdxEmptyListEntry listEntry; + IdxBitTestEntry bittestEntry, matchBitTestEntry; + DataBlock curBlock; + bool bFound, entryMap[ENTRY_PER_SUBBLOCK]; - int rootTestbitVal, rc = NO_ERROR; - IdxBitmapPointerEntry bitmapEntry; + int rootTestbitVal, rc = NO_ERROR; + IdxBitmapPointerEntry bitmapEntry; - getTestbitValue( key, width, 0, &rootTestbitVal ); - getSubBlockEntry( m_rootBlock.data, 1, rootTestbitVal, MAX_COLUMN_BOUNDARY, &bitmapEntry ); + getTestbitValue(key, width, 0, &rootTestbitVal); + getSubBlockEntry(m_rootBlock.data, 1, rootTestbitVal, MAX_COLUMN_BOUNDARY, &bitmapEntry); - if ( isAddrPtrEmpty( &bitmapEntry, /*EMPTY_PTR*/BITMAP_PTR ) ) - return bDelete ? ERR_IDX_LIST_INVALID_DELETE : NOT_FOUND ; + if (isAddrPtrEmpty(&bitmapEntry, /*EMPTY_PTR*/ BITMAP_PTR)) + return bDelete ? ERR_IDX_LIST_INVALID_DELETE : NOT_FOUND; - // get the rootbittestEntry level bitmapPointerMap - curFbo = bitmapEntry.fbo; - curSbid = bitmapEntry.sbid; - curEntry = bitmapEntry.entry; + // get the rootbittestEntry level bitmapPointerMap + curFbo = bitmapEntry.fbo; + curSbid = bitmapEntry.sbid; + curEntry = bitmapEntry.entry; - loopCount = width / 5 + 1; + loopCount = width / 5 + 1; - for ( i = 1; i < loopCount; i++ ) + for (i = 1; i < loopCount; i++) + { + // load the block + rc = readSubBlockEntry(m_cbTree, &curBlock, curFbo, curSbid, curEntry, /*width*/ MAX_COLUMN_BOUNDARY, + &bittestEntry); + + if (rc != NO_ERROR) + return rc; + + if (isDebug(DEBUG_3)) { - // load the block - rc = readSubBlockEntry( m_cbTree, &curBlock, curFbo, curSbid, curEntry, /*width*/MAX_COLUMN_BOUNDARY, &bittestEntry ); + printf("\nIn processIndex, level %d", i); + printSubBlock(curFbo, curSbid); + } - if ( rc != NO_ERROR ) - return rc; + rc = getTreeNodeInfo(&curBlock, curSbid, curEntry, width, (IdxTreeGroupType)bittestEntry.group, + &allocCount, &realCount, entryMap); - if ( isDebug( DEBUG_3 ) ) - { - printf( "\nIn processIndex, level %d", i ); - printSubBlock( curFbo, curSbid ); - } + if (rc != NO_ERROR) + return rc; - rc = getTreeNodeInfo( &curBlock, curSbid, curEntry, width, (IdxTreeGroupType)bittestEntry.group, &allocCount, &realCount, entryMap ); + matchBitTestEntry = bittestEntry; // assign to the value of the first entry + bSuccess = getTestbitValue(key, width, i, &testbitVal); - if ( rc != NO_ERROR ) - return rc; + matchBitTestEntry.bitTest = testbitVal; + bFound = getTreeMatchEntry(&curBlock, curSbid, curEntry, width, allocCount, entryMap, &matchPos, + &matchBitTestEntry); - matchBitTestEntry = bittestEntry; // assign to the value of the first entry - bSuccess = getTestbitValue( key, width, i, &testbitVal ); - - matchBitTestEntry.bitTest = testbitVal; - bFound = getTreeMatchEntry( &curBlock, curSbid, curEntry, width, allocCount, entryMap, &matchPos, &matchBitTestEntry ); - - if ( bFound ) // this test bit exists at the current level - { - if ( i == loopCount - 1 ) - { -// if( loopCount == 2 ) // because of bitmap, it requires a special treatment - curEntry += matchPos; - break; - } - } - else - return NOT_FOUND; - - getSubBlockEntry( &curBlock, curSbid, curEntry + matchPos, MAX_COLUMN_BOUNDARY, &matchBitTestEntry ); - - curFbo = matchBitTestEntry.fbo; - curSbid = matchBitTestEntry.sbid; - curEntry = matchBitTestEntry.entry; - - } // end of for( i = 0; - - // here's the last level - // load the last piece - rc = readSubBlockEntry( m_cbTree, &curBlock, curFbo, curSbid, curEntry /*+ matchPos*/, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - listEntry.fbo = bittestEntry.fbo; - listEntry.sbid = bittestEntry.sbid; - listEntry.entry = bittestEntry.entry; - - if ( bDelete ) + if (bFound) // this test bit exists at the current level { - rc = m_listMgr.deleteIndexList( m_cbList, rid, key, &listEntry ); - - if ( rc != NO_ERROR ) - return rc; - - bSuccess = getTestbitValue( key, width, loopCount - 1, &testbitVal ); - setBittestEntry( &bittestEntry, testbitVal, bittestEntry.group, listEntry.fbo, listEntry.sbid, listEntry.entry, LEAF_LIST ); - setSubBlockEntry( &curBlock, curSbid, curEntry, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - rc = writeDBFile( m_cbTree, &curBlock, curFbo ); - + if (i == loopCount - 1) + { + // if( loopCount == 2 ) // because of bitmap, it requires a special treatment + curEntry += matchPos; + break; + } } else - listHdrAddr = listEntry; + return NOT_FOUND; + getSubBlockEntry(&curBlock, curSbid, curEntry + matchPos, MAX_COLUMN_BOUNDARY, &matchBitTestEntry); + + curFbo = matchBitTestEntry.fbo; + curSbid = matchBitTestEntry.sbid; + curEntry = matchBitTestEntry.entry; + + } // end of for( i = 0; + + // here's the last level + // load the last piece + rc = readSubBlockEntry(m_cbTree, &curBlock, curFbo, curSbid, curEntry /*+ matchPos*/, MAX_COLUMN_BOUNDARY, + &bittestEntry); + + if (rc != NO_ERROR) return rc; -} + listEntry.fbo = bittestEntry.fbo; + listEntry.sbid = bittestEntry.sbid; + listEntry.entry = bittestEntry.entry; + + if (bDelete) + { + rc = m_listMgr.deleteIndexList(m_cbList, rid, key, &listEntry); + + if (rc != NO_ERROR) + return rc; + + bSuccess = getTestbitValue(key, width, loopCount - 1, &testbitVal); + setBittestEntry(&bittestEntry, testbitVal, bittestEntry.group, listEntry.fbo, listEntry.sbid, + listEntry.entry, LEAF_LIST); + setSubBlockEntry(&curBlock, curSbid, curEntry, MAX_COLUMN_BOUNDARY, &bittestEntry); + + rc = writeDBFile(m_cbTree, &curBlock, curFbo); + } + else + listHdrAddr = listEntry; + + return rc; +} /*********************************************************** * DESCRIPTION: @@ -1016,40 +1050,41 @@ const int IndexTree::processIndex( const uint64_t key, const int width, const RI * RETURN: * none ***********************************************************/ -void IndexTree::printSubBlock( const int fbo, const int sbid, const bool bNoZero ) +void IndexTree::printSubBlock(const int fbo, const int sbid, const bool bNoZero) { - DataBlock curBlock; + DataBlock curBlock; - readDBFile( m_cbTree, &curBlock, fbo ); - printf( "\n lbid=%2d sbid=%2d", fbo, sbid ); - printMemSubBlock( &curBlock, sbid ); + readDBFile(m_cbTree, &curBlock, fbo); + printf("\n lbid=%2d sbid=%2d", fbo, sbid); + printMemSubBlock(&curBlock, sbid); } -void IndexTree::printMemSubBlock( DataBlock* curBlock, const int sbid, const bool bNoZero ) +void IndexTree::printMemSubBlock(DataBlock* curBlock, const int sbid, const bool bNoZero) { - int off; - unsigned char* curPos; - IdxBitTestEntry curEntry, testZero; + int off; + unsigned char* curPos; + IdxBitTestEntry curEntry, testZero; - setBlankEntry( &testZero ); - curPos = curBlock->data + BYTE_PER_SUBBLOCK * sbid; - printf( "\n========================" ); + setBlankEntry(&testZero); + curPos = curBlock->data + BYTE_PER_SUBBLOCK * sbid; + printf("\n========================"); - for ( int i = 0; i < ENTRY_PER_SUBBLOCK; i++ ) - { - memcpy( &curEntry, curPos, MAX_COLUMN_BOUNDARY ); - off = memcmp( &testZero, &curEntry, MAX_COLUMN_BOUNDARY ); - printf( "\n Entry %2d : ", i ); + for (int i = 0; i < ENTRY_PER_SUBBLOCK; i++) + { + memcpy(&curEntry, curPos, MAX_COLUMN_BOUNDARY); + off = memcmp(&testZero, &curEntry, MAX_COLUMN_BOUNDARY); + printf("\n Entry %2d : ", i); - for ( int j = 0; j < MAX_COLUMN_BOUNDARY; j++ ) - printf( " %2X", *(curPos + j) ); + for (int j = 0; j < MAX_COLUMN_BOUNDARY; j++) + printf(" %2X", *(curPos + j)); - printf( " fbo=%2d sbid=%2d entry=%2d group=%d bit=%2d type=%2d", (int)curEntry.fbo, (int)curEntry.sbid, (int)curEntry.entry, (int)curEntry.group, (int)curEntry.bitTest, (int)curEntry.type ); + printf(" fbo=%2d sbid=%2d entry=%2d group=%d bit=%2d type=%2d", (int)curEntry.fbo, (int)curEntry.sbid, + (int)curEntry.entry, (int)curEntry.group, (int)curEntry.bitTest, (int)curEntry.type); - curPos += MAX_COLUMN_BOUNDARY; - } + curPos += MAX_COLUMN_BOUNDARY; + } - printf( "\n" ); + printf("\n"); } /*********************************************************** @@ -1063,41 +1098,43 @@ void IndexTree::printMemSubBlock( DataBlock* curBlock, const int sbid, const boo * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::releaseSegment( int segmentType, IdxEmptyListEntry* myPtr ) +const int IndexTree::releaseSegment(int segmentType, IdxEmptyListEntry* myPtr) { - int rc = NO_ERROR; + int rc = NO_ERROR; - m_rootBlock.dirty = true; + m_rootBlock.dirty = true; - if ( m_useFreeMgr ) - { - if ( isDebug( DEBUG_3 ) ) - printf( "\n----------Release the pointer, entry segment=%d fbo=%d sbid=%d entry=%d", segmentType, (int)myPtr->fbo, (int)myPtr->sbid, (int)myPtr->entry ); + if (m_useFreeMgr) + { + if (isDebug(DEBUG_3)) + printf("\n----------Release the pointer, entry segment=%d fbo=%d sbid=%d entry=%d", segmentType, + (int)myPtr->fbo, (int)myPtr->sbid, (int)myPtr->entry); - rc = m_freeMgr.releaseSegment( m_cbTree, &m_rootBlock, TREE, (IdxTreeGroupType) segmentType, myPtr/*, TREE */); + rc = m_freeMgr.releaseSegment(m_cbTree, &m_rootBlock, TREE, (IdxTreeGroupType)segmentType, + myPtr /*, TREE */); - if ( isDebug( DEBUG_3 ) ) - printf("\nReleased" ); - } + if (isDebug(DEBUG_3)) + printf("\nReleased"); + } - return rc; + return rc; } -const int IndexTree::resetIndexFile( const FID treeFid, const FID listFid ) +const int IndexTree::resetIndexFile(const FID treeFid, const FID listFid) { - /* long numOfBlock; - unsigned char writeBuf[BYTE_PER_BLOCK]; + /* long numOfBlock; + unsigned char writeBuf[BYTE_PER_BLOCK]; - memset( writeBuf, 0, BYTE_PER_BLOCK ); - numOfBlock = getFileSize( m_cbTree.file.pFile )/BYTE_PER_BLOCK; - for( int i = 0; i < numOfBlock; i++ ) - fwrite( writeBuf, sizeof( writeBuf ), 1, m_cbTree.file.pFile ); + memset( writeBuf, 0, BYTE_PER_BLOCK ); + numOfBlock = getFileSize( m_cbTree.file.pFile )/BYTE_PER_BLOCK; + for( int i = 0; i < numOfBlock; i++ ) + fwrite( writeBuf, sizeof( writeBuf ), 1, m_cbTree.file.pFile ); - numOfBlock = getFileSize( m_cbList.file.pFile )/BYTE_PER_BLOCK; - for( int i = 0; i < numOfBlock; i++ ) - fwrite( writeBuf, sizeof( writeBuf ), 1, m_cbList.file.pFile ); - */ - return initIndex( treeFid, listFid ); + numOfBlock = getFileSize( m_cbList.file.pFile )/BYTE_PER_BLOCK; + for( int i = 0; i < numOfBlock; i++ ) + fwrite( writeBuf, sizeof( writeBuf ), 1, m_cbList.file.pFile ); + */ + return initIndex(treeFid, listFid); } /*********************************************************** @@ -1111,24 +1148,26 @@ const int IndexTree::resetIndexFile( const FID treeFid, const FID listFid ) ***********************************************************/ const int IndexTree::calculateBittestArray() { - if ( m_multiColKey.totalBit <= 0 ) - return ERR_INVALID_PARAM; + if (m_multiColKey.totalBit <= 0) + return ERR_INVALID_PARAM; - m_multiColKey.maxLevel = m_multiColKey.totalBit / 5 + 1; + m_multiColKey.maxLevel = m_multiColKey.totalBit / 5 + 1; - if ( m_multiColKey.maxLevel > IDX_MAX_MULTI_COL_IDX_LEVEL ) - return ERR_VALUE_OUTOFRANGE; + if (m_multiColKey.maxLevel > IDX_MAX_MULTI_COL_IDX_LEVEL) + return ERR_VALUE_OUTOFRANGE; - for ( int i = 0; i < m_multiColKey.maxLevel - 1; i++ ) - { - m_multiColKey.curBitset = ( m_multiColKey.bitSet & m_multiColKey.curMask ) >> ( IDX_MAX_MULTI_COL_BIT - ( i + 1 ) * 5 ); - m_multiColKey.testbitArray[i] = m_multiColKey.curBitset.to_ulong(); - m_multiColKey.curMask = m_multiColKey.curMask >> 5; - } + for (int i = 0; i < m_multiColKey.maxLevel - 1; i++) + { + m_multiColKey.curBitset = + (m_multiColKey.bitSet & m_multiColKey.curMask) >> (IDX_MAX_MULTI_COL_BIT - (i + 1) * 5); + m_multiColKey.testbitArray[i] = m_multiColKey.curBitset.to_ulong(); + m_multiColKey.curMask = m_multiColKey.curMask >> 5; + } - m_multiColKey.curBitset = ( m_multiColKey.bitSet & m_multiColKey.curMask ) >> ( IDX_MAX_MULTI_COL_BIT - m_multiColKey.totalBit ); - m_multiColKey.testbitArray[m_multiColKey.maxLevel - 1] = m_multiColKey.curBitset.to_ulong(); - return NO_ERROR; + m_multiColKey.curBitset = + (m_multiColKey.bitSet & m_multiColKey.curMask) >> (IDX_MAX_MULTI_COL_BIT - m_multiColKey.totalBit); + m_multiColKey.testbitArray[m_multiColKey.maxLevel - 1] = m_multiColKey.curBitset.to_ulong(); + return NO_ERROR; } /*********************************************************** @@ -1141,58 +1180,58 @@ const int IndexTree::calculateBittestArray() * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::setBitsetColumn( void* val, const int pos, const int width, const ColType colType ) +const int IndexTree::setBitsetColumn(void* val, const int pos, const int width, const ColType colType) { - int copyLen = width / 8; + int copyLen = width / 8; - switch ( colType ) - { - // No body is using float or double for indexing - /* case WriteEngine::WR_FLOAT : m_multiColKey.curBitset = *((float*) val); - break; - case WriteEngine::WR_DOUBLE : m_multiColKey.curBitset = *((double*) val); - break; - */ - case WriteEngine::WR_CHAR : - m_multiColKey.curBitset.reset(); + switch (colType) + { + // No body is using float or double for indexing + /* case WriteEngine::WR_FLOAT : m_multiColKey.curBitset = *((float*) val); + break; + case WriteEngine::WR_DOUBLE : m_multiColKey.curBitset = *((double*) val); + break; + */ + case WriteEngine::WR_CHAR: + m_multiColKey.curBitset.reset(); - for ( int i = 0; i < width / 8; i++ ) - { - uint64_t curChar = ((char*)val)[i]; - m_multiColKey.curBitset = m_multiColKey.curBitset << 8; - m_multiColKey.curBitset |= curChar; - } + for (int i = 0; i < width / 8; i++) + { + uint64_t curChar = ((char*)val)[i]; + m_multiColKey.curBitset = m_multiColKey.curBitset << 8; + m_multiColKey.curBitset |= curChar; + } - memcpy( m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (char*) val, copyLen ); - break; + memcpy(m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (char*)val, copyLen); + break; - case WriteEngine::WR_SHORT : - m_multiColKey.curBitset = *((short*) val); - memcpy( m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (short*) val, copyLen ); - break; + case WriteEngine::WR_SHORT: + m_multiColKey.curBitset = *((short*)val); + memcpy(m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (short*)val, copyLen); + break; - case WriteEngine::WR_BYTE : - m_multiColKey.curBitset = *((char*) val); - memcpy( m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (char*) val, copyLen ); - break; + case WriteEngine::WR_BYTE: + m_multiColKey.curBitset = *((char*)val); + memcpy(m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (char*)val, copyLen); + break; - case WriteEngine::WR_LONGLONG: - m_multiColKey.curBitset = *((long long*) val); - memcpy( m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (long long*) val, copyLen ); - break; + case WriteEngine::WR_LONGLONG: + m_multiColKey.curBitset = *((long long*)val); + memcpy(m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (long long*)val, copyLen); + break; - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - default : - memcpy( m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (int*) val, copyLen ); - m_multiColKey.curBitset = *((int*) val); - break; - } + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + default: + memcpy(m_multiColKey.keyBuf + m_multiColKey.totalBit / 8, (int*)val, copyLen); + m_multiColKey.curBitset = *((int*)val); + break; + } - m_multiColKey.totalBit += width; - m_multiColKey.curBitset = m_multiColKey.curBitset << ( IDX_MAX_MULTI_COL_BIT - m_multiColKey.totalBit ); - m_multiColKey.bitSet |= m_multiColKey.curBitset; - return NO_ERROR; + m_multiColKey.totalBit += width; + m_multiColKey.curBitset = m_multiColKey.curBitset << (IDX_MAX_MULTI_COL_BIT - m_multiColKey.totalBit); + m_multiColKey.bitSet |= m_multiColKey.curBitset; + return NO_ERROR; } /*********************************************************** @@ -1206,17 +1245,19 @@ const int IndexTree::setBitsetColumn( void* val, const int pos, const int width, * RETURN: * none ***********************************************************/ -void IndexTree::setBittestEntry( IdxBitTestEntry* bittestEntry, const uint64_t testbitVal, const uint64_t group, const uint64_t fbo, const uint64_t sbid, const uint64_t entry, const uint64_t entryType ) const +void IndexTree::setBittestEntry(IdxBitTestEntry* bittestEntry, const uint64_t testbitVal, + const uint64_t group, const uint64_t fbo, const uint64_t sbid, + const uint64_t entry, const uint64_t entryType) const { - bittestEntry->type = entryType; - bittestEntry->bitTest = testbitVal; - bittestEntry->group = group; - bittestEntry->bitCompare = BIT_5; - bittestEntry->spare = 0; + bittestEntry->type = entryType; + bittestEntry->bitTest = testbitVal; + bittestEntry->group = group; + bittestEntry->bitCompare = BIT_5; + bittestEntry->spare = 0; - bittestEntry->fbo = fbo; - bittestEntry->sbid = sbid; - bittestEntry->entry = entry; + bittestEntry->fbo = fbo; + bittestEntry->sbid = sbid; + bittestEntry->entry = entry; } /*********************************************************** @@ -1231,15 +1272,16 @@ void IndexTree::setBittestEntry( IdxBitTestEntry* bittestEntry, const uint64_t t * none ***********************************************************/ // todo: need test case -void IndexTree::setEmptyListEntry( IdxEmptyListEntry* myEntry, const uint64_t group, const uint64_t fbo, const uint64_t sbid, const uint64_t entry ) const +void IndexTree::setEmptyListEntry(IdxEmptyListEntry* myEntry, const uint64_t group, const uint64_t fbo, + const uint64_t sbid, const uint64_t entry) const { - myEntry->type = EMPTY_PTR; - myEntry->group = group; - myEntry->spare = 0; - myEntry->spare2 = 0; - myEntry->fbo = fbo; - myEntry->sbid = sbid; - myEntry->entry = entry; + myEntry->type = EMPTY_PTR; + myEntry->group = group; + myEntry->spare = 0; + myEntry->spare2 = 0; + myEntry->fbo = fbo; + myEntry->sbid = sbid; + myEntry->entry = entry; } /*********************************************************** @@ -1255,37 +1297,36 @@ void IndexTree::setEmptyListEntry( IdxEmptyListEntry* myEntry, const uint64_t gr * RETURN: * none ***********************************************************/ -void IndexTree::setTreeHeader( IdxTree* myTree, const uint64_t key, const RID rid, const int width, - const int testbitVal, const IdxBitmapPointerEntry bitmapEntry ) +void IndexTree::setTreeHeader(IdxTree* myTree, const uint64_t key, const RID rid, const int width, + const int testbitVal, const IdxBitmapPointerEntry bitmapEntry) { - IdxBitTestEntry nextEntry, curEntry; + IdxBitTestEntry nextEntry, curEntry; - myTree->width = width; - myTree->key = key; - myTree->rid = rid; - myTree->maxLevel = (width / 5) + 1; + myTree->width = width; + myTree->key = key; + myTree->rid = rid; + myTree->maxLevel = (width / 5) + 1; - setBlankEntry( &nextEntry ); - setBlankEntry( &curEntry ); + setBlankEntry(&nextEntry); + setBlankEntry(&curEntry); -// myEntry.bitTest = testbitVal; - nextEntry.fbo = bitmapEntry.fbo; - nextEntry.sbid = bitmapEntry.sbid; - nextEntry.entry = bitmapEntry.entry; - nextEntry.type = BIT_TEST; + // myEntry.bitTest = testbitVal; + nextEntry.fbo = bitmapEntry.fbo; + nextEntry.sbid = bitmapEntry.sbid; + nextEntry.entry = bitmapEntry.entry; + nextEntry.type = BIT_TEST; -// nextEntry.group = ENTRY_32; + // nextEntry.group = ENTRY_32; - curEntry.fbo = 0; - curEntry.sbid = IDX_BITMAP_SUBBLOCK_NO; - curEntry.entry = testbitVal; - curEntry.type = BITMAP_PTR; - curEntry.group = ENTRY_32; - curEntry.bitTest = testbitVal; + curEntry.fbo = 0; + curEntry.sbid = IDX_BITMAP_SUBBLOCK_NO; + curEntry.entry = testbitVal; + curEntry.type = BITMAP_PTR; + curEntry.group = ENTRY_32; + curEntry.bitTest = testbitVal; - - // at this point useCount not known - setTreeNode( &myTree->node[0], 0, ENTRY_PER_SUBBLOCK, 0, testbitVal, nextEntry, curEntry ); + // at this point useCount not known + setTreeNode(&myTree->node[0], 0, ENTRY_PER_SUBBLOCK, 0, testbitVal, nextEntry, curEntry); } /*********************************************************** @@ -1302,39 +1343,38 @@ void IndexTree::setTreeHeader( IdxTree* myTree, const uint64_t key, const RID ri * RETURN: * none ***********************************************************/ -void IndexTree::setTreeNode( IdxTreeNode* myNode, const int level, const int allocCount, const int useCount, - const int offset, const IdxBitTestEntry nextEntry, const IdxBitTestEntry curEntry ) +void IndexTree::setTreeNode(IdxTreeNode* myNode, const int level, const int allocCount, const int useCount, + const int offset, const IdxBitTestEntry nextEntry, const IdxBitTestEntry curEntry) { - myNode->level = level; - myNode->allocCount = allocCount; - myNode->useCount = useCount; - myNode->offset = offset; - myNode->next = nextEntry; - myNode->current = curEntry; - myNode->used = true; + myNode->level = level; + myNode->allocCount = allocCount; + myNode->useCount = useCount; + myNode->offset = offset; + myNode->next = nextEntry; + myNode->current = curEntry; + myNode->used = true; } - const bool IndexTree::isTreeEmpty() { - bool bEmpty = true; - IdxBitmapPointerEntry curBitmapPointer, emptyPointer; + bool bEmpty = true; + IdxBitmapPointerEntry curBitmapPointer, emptyPointer; - memset( &emptyPointer, 0, MAX_COLUMN_BOUNDARY ); + memset(&emptyPointer, 0, MAX_COLUMN_BOUNDARY); - for ( int i = 0; i < 32; i++ ) + for (int i = 0; i < 32; i++) + { + getSubBlockEntry(m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, i, MAX_COLUMN_BOUNDARY, &curBitmapPointer); + + if (memcmp(&curBitmapPointer, &emptyPointer, MAX_COLUMN_BOUNDARY)) { - getSubBlockEntry( m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, i, MAX_COLUMN_BOUNDARY, &curBitmapPointer ); - - if ( memcmp( &curBitmapPointer, &emptyPointer, MAX_COLUMN_BOUNDARY ) ) - { - bEmpty = false; - break; - } + bEmpty = false; + break; } + } -// printMemSubBlock( &m_rootBlock, IDX_BITMAP_SUBBLOCK_NO ); - return bEmpty; + // printMemSubBlock( &m_rootBlock, IDX_BITMAP_SUBBLOCK_NO ); + return bEmpty; } /*********************************************************** @@ -1347,27 +1387,28 @@ const bool IndexTree::isTreeEmpty() * NO_ERROR if success * rowIdArray - allocation of the row id left here ***********************************************************/ -const int IndexTree::updateIndex( const uint64_t key, const int width, const RID rid ) +const int IndexTree::updateIndex(const uint64_t key, const int width, const RID rid) { - int rootTestbitVal, rc = NO_ERROR; - IdxBitmapPointerEntry curBitmapPointer; + int rootTestbitVal, rc = NO_ERROR; + IdxBitmapPointerEntry curBitmapPointer; - clearTree( &m_tree ); + clearTree(&m_tree); - getTestbitValue( key, width, 0, &rootTestbitVal ); - getSubBlockEntry( m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, rootTestbitVal, MAX_COLUMN_BOUNDARY, &curBitmapPointer ); + getTestbitValue(key, width, 0, &rootTestbitVal); + getSubBlockEntry(m_rootBlock.data, IDX_BITMAP_SUBBLOCK_NO, rootTestbitVal, MAX_COLUMN_BOUNDARY, + &curBitmapPointer); - setTreeHeader( &m_tree, key, rid, width, rootTestbitVal, curBitmapPointer ); + setTreeHeader(&m_tree, key, rid, width, rootTestbitVal, curBitmapPointer); - if ( m_tree.maxLevel > IDX_MAX_TREE_LEVEL ) - return ERR_VALUE_OUTOFRANGE; + if (m_tree.maxLevel > IDX_MAX_TREE_LEVEL) + return ERR_VALUE_OUTOFRANGE; - if ( isAddrPtrEmpty( &curBitmapPointer, (IdxTreeEntryType) curBitmapPointer.type ) ) - rc = buildEmptyIndexTreeBranch( key, width, rid, rootTestbitVal ); - else // at least we have the root level pointer - rc = buildExistIndexTreeBranch( key, width, rid, rootTestbitVal, curBitmapPointer ); + if (isAddrPtrEmpty(&curBitmapPointer, (IdxTreeEntryType)curBitmapPointer.type)) + rc = buildEmptyIndexTreeBranch(key, width, rid, rootTestbitVal); + else // at least we have the root level pointer + rc = buildExistIndexTreeBranch(key, width, rid, rootTestbitVal, curBitmapPointer); - return rc; + return rc; } /*********************************************************** @@ -1383,37 +1424,37 @@ const int IndexTree::updateIndex( const uint64_t key, const int width, const RID * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::updateIndexList( const uint64_t key, const int width, const RID rid, IdxEmptyListEntry* myEntry, const int no, const bool addFlag ) +const int IndexTree::updateIndexList(const uint64_t key, const int width, const RID rid, + IdxEmptyListEntry* myEntry, const int no, const bool addFlag) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if ( m_useListMgr ) + if (m_useListMgr) + { + // doing something here + if (!addFlag) // update list { - // doing something here - if ( !addFlag ) // update list - { - if ( m_useMultiRid ) - rc = m_listMgr.updateIndexList( m_cbList, m_multiRid, key, myEntry ); - else - rc = m_listMgr.updateIndexList( m_cbList, rid, key, myEntry ); - } - else // add list - { - if ( m_useMultiRid ) - rc = m_listMgr.addIndexListHdr( m_cbList, m_multiRid, key, myEntry ); - else - rc = m_listMgr.addIndexListHdr( m_cbList, rid, key, myEntry ); - } - + if (m_useMultiRid) + rc = m_listMgr.updateIndexList(m_cbList, m_multiRid, key, myEntry); + else + rc = m_listMgr.updateIndexList(m_cbList, rid, key, myEntry); } - else + else // add list { - myEntry->fbo = 3; - myEntry->sbid = 3; - myEntry->entry = 3 + no; + if (m_useMultiRid) + rc = m_listMgr.addIndexListHdr(m_cbList, m_multiRid, key, myEntry); + else + rc = m_listMgr.addIndexListHdr(m_cbList, rid, key, myEntry); } + } + else + { + myEntry->fbo = 3; + myEntry->sbid = 3; + myEntry->entry = 3 + no; + } - return rc; + return rc; } /*********************************************************** @@ -1432,42 +1473,48 @@ const int IndexTree::updateIndexList( const uint64_t key, const int width, const * NO_ERROR if success * error no if fail ***********************************************************/ -const int IndexTree::updateListFile( const uint64_t key, const int width, const RID rid, const int curLevel, const uint64_t group, const int allocCount, const int useCount, const int offset, const bool addFlag ) +const int IndexTree::updateListFile(const uint64_t key, const int width, const RID rid, const int curLevel, + const uint64_t group, const int allocCount, const int useCount, + const int offset, const bool addFlag) { - int rc, testbitVal, parentLevel = curLevel - 1; - IdxEmptyListEntry listEntry; - IdxBitTestEntry bittestEntry, curEntry; - bool bSuccess; - DataBlock curBlock; + int rc, testbitVal, parentLevel = curLevel - 1; + IdxEmptyListEntry listEntry; + IdxBitTestEntry bittestEntry, curEntry; + bool bSuccess; + DataBlock curBlock; - rc = readSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, - m_tree.node[parentLevel].next.entry + offset, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - if ( rc != NO_ERROR ) - return rc; - - if ( !addFlag ) - { - listEntry.fbo = bittestEntry.fbo; - listEntry.sbid = bittestEntry.sbid; - listEntry.entry = bittestEntry.entry; - } - - rc = updateIndexList( key, width, rid, &listEntry, curLevel, addFlag ); - - if ( rc != NO_ERROR ) - return rc; - - bSuccess = getTestbitValue( key, width, curLevel, &testbitVal ); - setBittestEntry( &bittestEntry, testbitVal, group, listEntry.fbo, listEntry.sbid, listEntry.entry, LEAF_LIST ); - - rc = writeSubBlockEntry( m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry + offset, MAX_COLUMN_BOUNDARY, &bittestEntry ); - - setBittestEntry( &curEntry, testbitVal, group, m_tree.node[parentLevel].next.fbo, m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, LEAF_LIST ); - setTreeNode( &m_tree.node[curLevel], curLevel, allocCount, useCount, offset, bittestEntry, curEntry ); + rc = readSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry + offset, + MAX_COLUMN_BOUNDARY, &bittestEntry); + if (rc != NO_ERROR) return rc; + + if (!addFlag) + { + listEntry.fbo = bittestEntry.fbo; + listEntry.sbid = bittestEntry.sbid; + listEntry.entry = bittestEntry.entry; + } + + rc = updateIndexList(key, width, rid, &listEntry, curLevel, addFlag); + + if (rc != NO_ERROR) + return rc; + + bSuccess = getTestbitValue(key, width, curLevel, &testbitVal); + setBittestEntry(&bittestEntry, testbitVal, group, listEntry.fbo, listEntry.sbid, listEntry.entry, + LEAF_LIST); + + rc = writeSubBlockEntry(m_cbTree, &curBlock, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry + offset, + MAX_COLUMN_BOUNDARY, &bittestEntry); + + setBittestEntry(&curEntry, testbitVal, group, m_tree.node[parentLevel].next.fbo, + m_tree.node[parentLevel].next.sbid, m_tree.node[parentLevel].next.entry, LEAF_LIST); + setTreeNode(&m_tree.node[curLevel], curLevel, allocCount, useCount, offset, bittestEntry, curEntry); + + return rc; } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/index/we_indextree.h b/writeengine/index/we_indextree.h index 4a454a368..a440fea16 100644 --- a/writeengine/index/we_indextree.h +++ b/writeengine/index/we_indextree.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_indextree.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -******************************************************************************************/ + * $Id: we_indextree.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_INDEXTREE_H_ @@ -40,295 +40,303 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class ColumnOp */ class IndexTree : public DbFileOp { -public: - /** - * @brief Constructor - */ - EXPORT IndexTree(); + public: + /** + * @brief Constructor + */ + EXPORT IndexTree(); - /** - * @brief Default Destructor - */ - EXPORT ~IndexTree(); + /** + * @brief Default Destructor + */ + EXPORT ~IndexTree(); - /** - * @brief A wrapper for the call to free manager - */ - EXPORT const int assignSegment( int segmentType, IdxEmptyListEntry* assignPtr, int no ); + /** + * @brief A wrapper for the call to free manager + */ + EXPORT const int assignSegment(int segmentType, IdxEmptyListEntry* assignPtr, int no); - /** - * @brief Build empty index tree part - */ - EXPORT const int buildEmptyTreePart( const uint64_t key, const int width, const RID rid, const int startBitTestNo, const int offset = 0 ); + /** + * @brief Build empty index tree part + */ + EXPORT const int buildEmptyTreePart(const uint64_t key, const int width, const RID rid, + const int startBitTestNo, const int offset = 0); - /** - * @brief Build empty index branch - */ - EXPORT const int buildEmptyIndexTreeBranch( const uint64_t key, const int width, const RID rid, const int rootTestbitVal ); + /** + * @brief Build empty index branch + */ + EXPORT const int buildEmptyIndexTreeBranch(const uint64_t key, const int width, const RID rid, + const int rootTestbitVal); - /** - * @brief Build exist index branch - */ - EXPORT const int buildExistIndexTreeBranch( const uint64_t key, const int width, const RID rid, const int rootTestbitVal, IdxBitmapPointerEntry bitmapEntry ); + /** + * @brief Build exist index branch + */ + EXPORT const int buildExistIndexTreeBranch(const uint64_t key, const int width, const RID rid, + const int rootTestbitVal, IdxBitmapPointerEntry bitmapEntry); - /** - * @brief Calculate bit test array - */ - EXPORT const int calculateBittestArray(); + /** + * @brief Calculate bit test array + */ + EXPORT const int calculateBittestArray(); - /** - * @brief Close index - */ - EXPORT void closeIndex(); + /** + * @brief Close index + */ + EXPORT void closeIndex(); - /** - * @brief Clear index tree member variables - */ - void clear() - { - clearBlock( &m_rootBlock ); - } + /** + * @brief Clear index tree member variables + */ + void clear() + { + clearBlock(&m_rootBlock); + } - /** - * @brief Create index related files - */ - EXPORT const int createIndex( const FID treeFid, const FID listFid, const bool useFreeMgrFlag = true ); + /** + * @brief Create index related files + */ + EXPORT const int createIndex(const FID treeFid, const FID listFid, const bool useFreeMgrFlag = true); - /** - * @brief Delete a value from an index - */ - EXPORT const int deleteIndex( const uint64_t key, const int width, const RID rid ); + /** + * @brief Delete a value from an index + */ + EXPORT const int deleteIndex(const uint64_t key, const int width, const RID rid); - /** - * @brief Drop index related files - */ - const int dropIndex( const FID treeFid, const FID listFid ) + /** + * @brief Drop index related files + */ + const int dropIndex(const FID treeFid, const FID listFid) #ifdef BROKEN_BY_MULTIPLE_FILES_PER_OID - { - m_cbTree.file.pFile = m_fileopTree.openFile(treeFid); - m_cbList.file.pFile = m_fileopList.openFile(listFid); - closeIndex(); - m_useMultiCol = false; - deleteFile( treeFid ); - deleteFile( listFid ); - return NO_ERROR; - } + { + m_cbTree.file.pFile = m_fileopTree.openFile(treeFid); + m_cbList.file.pFile = m_fileopList.openFile(listFid); + closeIndex(); + m_useMultiCol = false; + deleteFile(treeFid); + deleteFile(listFid); + return NO_ERROR; + } #endif - { - std::string segFile; - m_cbTree.file.pFile = m_fileopTree.openFile( - treeFid, 0, 0, 0, segFile); - m_cbList.file.pFile = m_fileopList.openFile( - listFid, 0, 0, 0, segFile); - closeIndex(); - m_useMultiCol = false; - deleteFile( treeFid ); - deleteFile( listFid ); - return NO_ERROR; - } + { + std::string segFile; + m_cbTree.file.pFile = m_fileopTree.openFile(treeFid, 0, 0, 0, segFile); + m_cbList.file.pFile = m_fileopList.openFile(listFid, 0, 0, 0, segFile); + closeIndex(); + m_useMultiCol = false; + deleteFile(treeFid); + deleteFile(listFid); + return NO_ERROR; + } - /** - * @brief Get the test bit value - */ - EXPORT const bool getTestbitValue( const uint64_t key, const int width, const int curTestNo, int* bittestVal ); + /** + * @brief Get the test bit value + */ + EXPORT const bool getTestbitValue(const uint64_t key, const int width, const int curTestNo, + int* bittestVal); - /** - * @brief Get the match entry in the tree - */ - EXPORT const bool getTreeMatchEntry( DataBlock* block, const uint64_t sbid, const uint64_t entry, const int width, - const int allocCount, const bool* entryMap, int* matchEntry, IdxBitTestEntry* checkEntry ); + /** + * @brief Get the match entry in the tree + */ + EXPORT const bool getTreeMatchEntry(DataBlock* block, const uint64_t sbid, const uint64_t entry, + const int width, const int allocCount, const bool* entryMap, + int* matchEntry, IdxBitTestEntry* checkEntry); - /** - * @brief Get the tree node summary information - */ - EXPORT const int getTreeNodeInfo( DataBlock* block, const uint64_t sbid, const uint64_t entry, const int width, - const IdxTreeGroupType group, int* allocCount, int* realCount, bool* entryMap ); + /** + * @brief Get the tree node summary information + */ + EXPORT const int getTreeNodeInfo(DataBlock* block, const uint64_t sbid, const uint64_t entry, + const int width, const IdxTreeGroupType group, int* allocCount, + int* realCount, bool* entryMap); - /** - * @brief Check index address pointer is empty or not - */ - EXPORT const bool isAddrPtrEmpty( void* pStruct, const IdxTreeEntryType entryType ) const; + /** + * @brief Check index address pointer is empty or not + */ + EXPORT const bool isAddrPtrEmpty(void* pStruct, const IdxTreeEntryType entryType) const; - /** - * @brief Check whether the tree is empty - */ - EXPORT const bool isTreeEmpty(); + /** + * @brief Check whether the tree is empty + */ + EXPORT const bool isTreeEmpty(); - /** - * @brief Init index - */ - EXPORT const int initIndex( const FID treeFid, const FID listFid ); + /** + * @brief Init index + */ + EXPORT const int initIndex(const FID treeFid, const FID listFid); - /** - * @brief Move tree entries - */ - EXPORT const int moveEntry( const uint64_t oldFbo, const uint64_t oldSbid, const uint64_t oldEntry, const int width, const uint64_t newFbo, - const uint64_t newSbid, const uint64_t newEntry, const int newGroup, const int allocCount, bool* entryMap, int* moveCount, const int newAllocCount = 0 ); + /** + * @brief Move tree entries + */ + EXPORT const int moveEntry(const uint64_t oldFbo, const uint64_t oldSbid, const uint64_t oldEntry, + const int width, const uint64_t newFbo, const uint64_t newSbid, + const uint64_t newEntry, const int newGroup, const int allocCount, + bool* entryMap, int* moveCount, const int newAllocCount = 0); - /** - * @brief Open index related files - */ - EXPORT const int openIndex( const FID treeFid, const FID listFid ); + /** + * @brief Open index related files + */ + EXPORT const int openIndex(const FID treeFid, const FID listFid); - /** - * @brief Process index, including delete and search - */ - EXPORT const int processIndex( const uint64_t key, const int width, const RID rid, IdxEmptyListEntry& listHdrAddr, const bool bDelete = true ); + /** + * @brief Process index, including delete and search + */ + EXPORT const int processIndex(const uint64_t key, const int width, const RID rid, + IdxEmptyListEntry& listHdrAddr, const bool bDelete = true); - /** - * @brief A wrapper for the call to free manager - */ - EXPORT const int releaseSegment( int segmentType, IdxEmptyListEntry* myPtr ); + /** + * @brief A wrapper for the call to free manager + */ + EXPORT const int releaseSegment(int segmentType, IdxEmptyListEntry* myPtr); - /** - * @brief Clean up index file - */ - EXPORT const int resetIndexFile( const FID treeFid, const FID listFid ); + /** + * @brief Clean up index file + */ + EXPORT const int resetIndexFile(const FID treeFid, const FID listFid); - /** - * @brief Setup bitset by column - */ - EXPORT const int setBitsetColumn( void* val, const int pos, const int width, const ColType colType ); + /** + * @brief Setup bitset by column + */ + EXPORT const int setBitsetColumn(void* val, const int pos, const int width, const ColType colType); - /** - * @brief Set bit test entry - */ - EXPORT void setBittestEntry( IdxBitTestEntry* bittestEntry, const uint64_t testbitVal, const uint64_t group, const uint64_t fbo, const uint64_t sbid, const uint64_t entry, constu int64_t entryType = BIT_TEST ) const; + /** + * @brief Set bit test entry + */ + EXPORT void setBittestEntry(IdxBitTestEntry* bittestEntry, const uint64_t testbitVal, const uint64_t group, + const uint64_t fbo, const uint64_t sbid, const uint64_t entry, + constu int64_t entryType = BIT_TEST) const; - /** - * @brief Set blank entry - */ - void setBlankEntry( void* pStruct ) const - { - memset( pStruct, 0, 8 ); - } + /** + * @brief Set blank entry + */ + void setBlankEntry(void* pStruct) const + { + memset(pStruct, 0, 8); + } - /** - * @brief Set empty list ptr entry - */ - EXPORT void setEmptyListEntry( IdxEmptyListEntry* myEntry, const uint64_t group, const uint64_t fbo, const uint64_t sbid, const uint64_t entry ) const; + /** + * @brief Set empty list ptr entry + */ + EXPORT void setEmptyListEntry(IdxEmptyListEntry* myEntry, const uint64_t group, const uint64_t fbo, + const uint64_t sbid, const uint64_t entry) const; - /** - * @brief Set transaction Id - */ - EXPORT void setTransId( const TxnID txnId ) - { - FileOp::setTransId( txnId ); - m_cbTree.session.txnid = m_cbList.session.txnid = txnId; - m_freeMgr.setTransId(txnId); - m_listMgr.setTransId(txnId); - } + /** + * @brief Set transaction Id + */ + EXPORT void setTransId(const TxnID txnId) + { + FileOp::setTransId(txnId); + m_cbTree.session.txnid = m_cbList.session.txnid = txnId; + m_freeMgr.setTransId(txnId); + m_listMgr.setTransId(txnId); + } - /** - * @brief Set use brm flag - */ + /** + * @brief Set use brm flag + */ - /** - * @brief Update a value in an index - */ - EXPORT const int updateIndex( const uint64_t key, const int width, const RID rid ); + /** + * @brief Update a value in an index + */ + EXPORT const int updateIndex(const uint64_t key, const int width, const RID rid); - EXPORT const int updateListFile( const uint64_t key, const int width, const RID rid, const int curLevel, const uint64_t group, const int allocCount, const int useCount, const int offset, const bool addFlag = false ); + EXPORT const int updateListFile(const uint64_t key, const int width, const RID rid, const int curLevel, + const uint64_t group, const int allocCount, const int useCount, + const int offset, const bool addFlag = false); + // internal use functions + EXPORT void clearTree(IdxTree* myTree); + EXPORT void clearTreeNode(IdxTreeNode* myNode); + const IdxTree& getTree() + { + return m_tree; + } + const DataBlock& getRootBlock() + { + return m_rootBlock; + } + const bool getUseMultiRid() + { + return m_useMultiRid; + } - // internal use functions - EXPORT void clearTree( IdxTree* myTree ); - EXPORT void clearTreeNode( IdxTreeNode* myNode ); - const IdxTree& getTree() - { - return m_tree; - } - const DataBlock& getRootBlock() - { - return m_rootBlock; - } - const bool getUseMultiRid() - { - return m_useMultiRid; - } + EXPORT void setTreeHeader(IdxTree* myTree, const uint64_t key, const RID rid, const int width, + const int testbitVal, const IdxBitmapPointerEntry bitmapEntry); + EXPORT void setTreeNode(IdxTreeNode* myNode, const int level, const int allocCount, const int useCount, + const int offset, const IdxBitTestEntry nextEntry, const IdxBitTestEntry curEntry); - EXPORT void setTreeHeader( IdxTree* myTree, const uint64_t key, const RID rid, const int width, - const int testbitVal, const IdxBitmapPointerEntry bitmapEntry ); - EXPORT void setTreeNode( IdxTreeNode* myNode, const int level, const int allocCount, const int useCount, - const int offset, const IdxBitTestEntry nextEntry, const IdxBitTestEntry curEntry ); + EXPORT const int updateIndexList(const uint64_t key, const int width, const RID rid, + IdxEmptyListEntry* myEntry, const int no, const bool addFlag = false); - EXPORT const int updateIndexList( const uint64_t key, const int width, const RID rid, IdxEmptyListEntry* myEntry, const int no, const bool addFlag = false); + EXPORT void printMemSubBlock(DataBlock* curBlock, const int sbid, const bool bNoZero = false); + EXPORT void printSubBlock(const int fbo, const int sbid, const bool bNoZero = false); - EXPORT void printMemSubBlock( DataBlock* curBlock, const int sbid, const bool bNoZero = false ); - EXPORT void printSubBlock( const int fbo, const int sbid, const bool bNoZero = false ); + void setAssignFbo(const int fbo) + { + m_assignFbo = fbo; // internal testing purpose only + } + void setUseFreeMgr(const bool val) + { + m_useFreeMgr = val; + } + void setUseListMgr(const bool val) + { + m_useListMgr = val; + } + void setUseMultiCol(const bool val) + { + m_useMultiCol = val; + } + void setUseMultiRid(const bool val) + { + m_useMultiRid = val; + } - void setAssignFbo( const int fbo ) - { - m_assignFbo = fbo; // internal testing purpose only - } - void setUseFreeMgr( const bool val ) - { - m_useFreeMgr = val; - } - void setUseListMgr( const bool val ) - { - m_useListMgr = val; - } - void setUseMultiCol( const bool val ) - { - m_useMultiCol = val; - } - void setUseMultiRid( const bool val ) - { - m_useMultiRid = val; - } + void setCbTree(CommBlock& cb) + { + memcpy(&m_cbTree.session, &cb.session, sizeof(cb.session)); + memcpy(&m_cbTree.file, &cb.file, sizeof(cb.file)); + } + const CommBlock& getCbTree() const + { + return m_cbTree; + } - void setCbTree(CommBlock& cb) - { - memcpy(&m_cbTree.session, &cb.session, - sizeof(cb.session)); - memcpy(&m_cbTree.file, &cb.file, - sizeof(cb.file)); - } - const CommBlock& getCbTree() const - { - return m_cbTree; - } + void setCbList(CommBlock& cb) + { + memcpy(&m_cbList.session, &cb.session, sizeof(cb.session)); + memcpy(&m_cbList.file, &cb.file, sizeof(cb.file)); + } + const CommBlock& getCbList() const + { + return m_cbList; + } - void setCbList(CommBlock& cb) - { - memcpy(&m_cbList.session, &cb.session, - sizeof(cb.session)); - memcpy(&m_cbList.file, &cb.file, - sizeof(cb.file)); - } - const CommBlock& getCbList() const - { - return m_cbList; - } + DataBlock m_rootBlock; /** @brief This block contains sub block 0 and sub block 1 */ + CommBlock m_cbTree; /** @brief index tree */ + CommBlock m_cbList; /** @brief index list */ - DataBlock m_rootBlock; /** @brief This block contains sub block 0 and sub block 1 */ - CommBlock m_cbTree; /** @brief index tree */ - CommBlock m_cbList; /** @brief index list */ + IdxTree m_tree; /** @brief index tree node */ + FreeMgr m_freeMgr; /** @brief index free mgr */ + IndexList m_listMgr; /** @brief index list mgr */ - IdxTree m_tree; /** @brief index tree node */ - FreeMgr m_freeMgr; /** @brief index free mgr */ - IndexList m_listMgr; /** @brief index list mgr */ + IdxMultiColKey m_multiColKey; /** @brief index multi-column key */ + IdxMultiRid m_multiRid; /** @brief index multi rids */ + FileOp m_fileopTree; + FileOp m_fileopList; - IdxMultiColKey m_multiColKey; /** @brief index multi-column key */ - IdxMultiRid m_multiRid; /** @brief index multi rids */ - FileOp m_fileopTree; - FileOp m_fileopList; -private: - // internal testing flags - bool m_useFreeMgr; - bool m_useListMgr; - bool m_useMultiCol; - bool m_useMultiRid; - int m_assignFbo; + private: + // internal testing flags + bool m_useFreeMgr; + bool m_useListMgr; + bool m_useMultiCol; + bool m_useMultiRid; + int m_assignFbo; }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_INDEXTREE_H_ +#endif // _WE_INDEXTREE_H_ diff --git a/writeengine/redistribute/we_redistribute.cpp b/writeengine/redistribute/we_redistribute.cpp index 54aae303e..d155715b5 100644 --- a/writeengine/redistribute/we_redistribute.cpp +++ b/writeengine/redistribute/we_redistribute.cpp @@ -16,9 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistribute.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -*/ - + * $Id: we_redistribute.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + */ #include "messagequeue.h" #include "bytestream.h" @@ -30,36 +29,27 @@ using namespace messageqcpp; namespace redistribute { - void Redistribute::handleRedistributeMessage(ByteStream& bs, IOSocket& ios) { - // consume the WES message id - ByteStream::byte wesMsgId; - bs >> wesMsgId; + // consume the WES message id + ByteStream::byte wesMsgId; + bs >> wesMsgId; - // peek at the message header - const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) bs.buf(); + // peek at the message header + const RedistributeMsgHeader* h = (const RedistributeMsgHeader*)bs.buf(); - switch (h->messageId) - { - case RED_CNTL_START: - case RED_CNTL_STATUS: - case RED_CNTL_STOP: - case RED_CNTL_CLEAR: - case RED_CNTL_RESP: - RedistributeControl::instance()->handleUIMsg(bs, ios); - break; + switch (h->messageId) + { + case RED_CNTL_START: + case RED_CNTL_STATUS: + case RED_CNTL_STOP: + case RED_CNTL_CLEAR: + case RED_CNTL_RESP: RedistributeControl::instance()->handleUIMsg(bs, ios); break; - default: - RedistributeControl::instance()->handleJobMsg(bs, ios); - break; - } + default: RedistributeControl::instance()->handleJobMsg(bs, ios); break; + } } - -} - - +} // namespace redistribute // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistribute.h b/writeengine/redistribute/we_redistribute.h index f3b08fb9f..62c33b833 100644 --- a/writeengine/redistribute/we_redistribute.h +++ b/writeengine/redistribute/we_redistribute.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistribute.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistribute.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ #ifndef WE_REDISTRIBUTE_H #define WE_REDISTRIBUTE_H @@ -28,27 +28,23 @@ namespace messageqcpp { class ByteStream; class IOSocket; -} +} // namespace messageqcpp namespace redistribute { - class Redistribute { -public: - Redistribute(); - ~Redistribute() {}; + public: + Redistribute(); + ~Redistribute(){}; - static void handleRedistributeMessage(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + static void handleRedistributeMessage(messageqcpp::ByteStream&, messageqcpp::IOSocket&); -private: + private: }; - -} // namespace - +} // namespace redistribute #endif // WE_REDISTRIBUTE_H // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistributecontrol.cpp b/writeengine/redistribute/we_redistributecontrol.cpp index 59e42bee5..a54e47ff8 100644 --- a/writeengine/redistribute/we_redistributecontrol.cpp +++ b/writeengine/redistribute/we_redistributecontrol.cpp @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributecontrol.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistributecontrol.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + */ #include #include @@ -65,7 +65,6 @@ using namespace execplan; namespace redistribute { - RedistributeControl* RedistributeControl::fInstance = NULL; boost::mutex instanceMutex; @@ -73,671 +72,630 @@ const string RedistributeDir("/data1/systemFiles/redistribute"); const string InfoFileName("/redistribute.info"); const string PlanFileName("/redistribute.plan"); - RedistributeControl* RedistributeControl::instance() { - // The constructor is protected by instanceMutex lock. - boost::mutex::scoped_lock lock(instanceMutex); + // The constructor is protected by instanceMutex lock. + boost::mutex::scoped_lock lock(instanceMutex); - if (fInstance == NULL) - fInstance = new RedistributeControl(); + if (fInstance == NULL) + fInstance = new RedistributeControl(); - return fInstance; + return fInstance; } - RedistributeControl::RedistributeControl() : fInfoFilePtr(NULL), fPlanFilePtr(NULL) { - // default path /usr/local/mariadb/columnstore/data1/systemFiles/redistribute - fRedistributeDir = "/var/lib/columnstore/" + RedistributeDir; - fInfoFilePath = fRedistributeDir + InfoFileName; - fPlanFilePath = fRedistributeDir + PlanFileName; + // default path /usr/local/mariadb/columnstore/data1/systemFiles/redistribute + fRedistributeDir = "/var/lib/columnstore/" + RedistributeDir; + fInfoFilePath = fRedistributeDir + InfoFileName; + fPlanFilePath = fRedistributeDir + PlanFileName; - fOam.reset(new oam::Oam); - fDbrm.reset(new BRM::DBRM); - fSysLogger.reset(new logging::Logger(32)); //32 - writeengineserver in SubsystemIDs.txt - logging::MsgMap msgMap; - msgMap[logging::M0002] = logging::Message(logging::M0002); - fSysLogger->msgMap(msgMap); + fOam.reset(new oam::Oam); + fDbrm.reset(new BRM::DBRM); + fSysLogger.reset(new logging::Logger(32)); // 32 - writeengineserver in SubsystemIDs.txt + logging::MsgMap msgMap; + msgMap[logging::M0002] = logging::Message(logging::M0002); + fSysLogger->msgMap(msgMap); - //struct stat st; - //if (stat(fRedistributeDir.c_str(), &st) != 0) - //filesystem::path dirPath(fRedistributeDir); - if (filesystem::exists(fRedistributeDir)) + // struct stat st; + // if (stat(fRedistributeDir.c_str(), &st) != 0) + // filesystem::path dirPath(fRedistributeDir); + if (filesystem::exists(fRedistributeDir)) + { + // try to open info file for update if dir exists + RedistributeInfo info; + fInfoFilePtr = fopen(fInfoFilePath.c_str(), "r+"); + + if (fInfoFilePtr != NULL && 1 == fread(&info, sizeof(info), 1, fInfoFilePtr)) { - // try to open info file for update if dir exists - RedistributeInfo info; - fInfoFilePtr = fopen(fInfoFilePath.c_str(), "r+"); + fRedistributeInfo = info; - if (fInfoFilePtr != NULL && 1 == fread(&info, sizeof(info), 1, fInfoFilePtr)) - { - fRedistributeInfo = info; - - // if there was an active session, mark it as failed until support resume. - if (fRedistributeInfo.state == RED_STATE_ACTIVE) - updateState(RED_STATE_FAILED); - } + // if there was an active session, mark it as failed until support resume. + if (fRedistributeInfo.state == RED_STATE_ACTIVE) + updateState(RED_STATE_FAILED); } + } } - RedistributeControl::~RedistributeControl() { - fOam.reset(); - fDbrm.reset(); - delete fInstance; - fInstance = NULL; + fOam.reset(); + fDbrm.reset(); + delete fInstance; + fInstance = NULL; } - int RedistributeControl::handleUIMsg(messageqcpp::ByteStream& bs, messageqcpp::IOSocket& so) { - boost::mutex::scoped_lock sessionLock(fSessionMutex); + boost::mutex::scoped_lock sessionLock(fSessionMutex); - uint32_t status = RED_STATE_UNDEF; - const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) bs.buf(); + uint32_t status = RED_STATE_UNDEF; + const RedistributeMsgHeader* h = (const RedistributeMsgHeader*)bs.buf(); - try + try + { + switch (h->messageId) { - switch (h->messageId) - { - case RED_CNTL_START: - status = handleStartMsg(bs, so); - break; + case RED_CNTL_START: status = handleStartMsg(bs, so); break; - case RED_CNTL_STOP: - status = handleStopMsg(bs, so); - break; + case RED_CNTL_STOP: status = handleStopMsg(bs, so); break; - case RED_CNTL_CLEAR: - status = handleClearMsg(bs, so); - break; + case RED_CNTL_CLEAR: status = handleClearMsg(bs, so); break; - case RED_CNTL_STATUS: - default: - status = handleStatusMsg(bs, so); - break; - } - } - catch (const std::exception& ex) - { - if (fUIResponse.empty()) - fUIResponse = ex.what(); - } - catch (...) - { - if (fUIResponse.empty()) - fUIResponse = "Failed to process the redistribute command."; + case RED_CNTL_STATUS: + default: status = handleStatusMsg(bs, so); break; } + } + catch (const std::exception& ex) + { + if (fUIResponse.empty()) + fUIResponse = ex.what(); + } + catch (...) + { + if (fUIResponse.empty()) + fUIResponse = "Failed to process the redistribute command."; + } - // log the response - logMessage(fUIResponse); + // log the response + logMessage(fUIResponse); - // bs restart() in handlers - bs.restart(); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - bs << status; - bs << fUIResponse; - so.write(bs); + // bs restart() in handlers + bs.restart(); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + bs << status; + bs << fUIResponse; + so.write(bs); - return status; + return status; } - int RedistributeControl::handleStartMsg(messageqcpp::ByteStream& bs, messageqcpp::IOSocket& so) { - ostringstream oss; - uint32_t status = getCurrentState(); - - if (status != RED_STATE_IDLE) - { - if (status == RED_STATE_ACTIVE) - oss << "Redistribute is already running. Command is ignored. You need to stop and clear this active session before starting a new one."; - else - oss << "Redistribute is not in IDLE state. Command is ignored. Please check the status of last session, then reset the state to IDLE using action CLEAR."; - - fUIResponse = oss.str(); - - return status; - } - - // must be IDLE state - try - { - // skip the header part, may need save it. - bs.advance(sizeof(RedistributeMsgHeader)); - - getStartOptions(bs); - RedistributeControlThread::setStopAction(false); - updateState(RED_STATE_ACTIVE); - } - catch (const std::exception& ex) - { - if (fErrorMsg.empty()) - fErrorMsg = ex.what(); - - fRedistributeInfo.state = RED_STATE_FAILED; - } - catch (...) - { - fRedistributeInfo.state = RED_STATE_FAILED; - } - - status = fRedistributeInfo.state; + ostringstream oss; + uint32_t status = getCurrentState(); + if (status != RED_STATE_IDLE) + { if (status == RED_STATE_ACTIVE) - { - oss << "Redistribute is started."; - fControlThread.reset(new boost::thread(RedistributeControlThread(RED_CNTL_START))); - // Let go the new thread unless we want to call interrupt on this thread in future. - // Not going to join() because the redistribution could take very long. - fControlThread->detach(); - fControlThread.reset(); - } + oss << "Redistribute is already running. Command is ignored. You need to stop and clear this active " + "session before starting a new one."; else - { - updateState(RED_STATE_FAILED); - oss << "Starting redistribute failed."; - - if (!fErrorMsg.empty()) - oss << " " << fErrorMsg; - } + oss << "Redistribute is not in IDLE state. Command is ignored. Please check the status of last " + "session, then reset the state to IDLE using action CLEAR."; fUIResponse = oss.str(); return status; -} + } + // must be IDLE state + try + { + // skip the header part, may need save it. + bs.advance(sizeof(RedistributeMsgHeader)); + + getStartOptions(bs); + RedistributeControlThread::setStopAction(false); + updateState(RED_STATE_ACTIVE); + } + catch (const std::exception& ex) + { + if (fErrorMsg.empty()) + fErrorMsg = ex.what(); + + fRedistributeInfo.state = RED_STATE_FAILED; + } + catch (...) + { + fRedistributeInfo.state = RED_STATE_FAILED; + } + + status = fRedistributeInfo.state; + + if (status == RED_STATE_ACTIVE) + { + oss << "Redistribute is started."; + fControlThread.reset(new boost::thread(RedistributeControlThread(RED_CNTL_START))); + // Let go the new thread unless we want to call interrupt on this thread in future. + // Not going to join() because the redistribution could take very long. + fControlThread->detach(); + fControlThread.reset(); + } + else + { + updateState(RED_STATE_FAILED); + oss << "Starting redistribute failed."; + + if (!fErrorMsg.empty()) + oss << " " << fErrorMsg; + } + + fUIResponse = oss.str(); + + return status; +} int RedistributeControl::handleStatusMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket& so) { - ostringstream oss; - uint32_t status = getCurrentState(); - RedistributeInfo info = fRedistributeInfo; + ostringstream oss; + uint32_t status = getCurrentState(); + RedistributeInfo info = fRedistributeInfo; - switch (status) - { - case RED_STATE_IDLE: - oss << "Redistribute is in IDLE state."; - break; + switch (status) + { + case RED_STATE_IDLE: oss << "Redistribute is in IDLE state."; break; - case RED_STATE_ACTIVE: - oss << "Redistribute is in progress: total " << info.planned; + case RED_STATE_ACTIVE: + oss << "Redistribute is in progress: total " << info.planned; - if (info.planned > 1) - oss << " logical partitions are planned to move.\n"; - else - oss << " logical partition is planned to move.\n"; + if (info.planned > 1) + oss << " logical partitions are planned to move.\n"; + else + oss << " logical partition is planned to move.\n"; - if (info.planned > 0) - { - if (info.endTime > 0) - oss << "In " << (info.endTime - info.startTime) << " seconds, "; + if (info.planned > 0) + { + if (info.endTime > 0) + oss << "In " << (info.endTime - info.startTime) << " seconds, "; - oss << info.success << " success, " - << info.skipped << " skipped, " - << info.failed << " failed, " - << ((info.success + info.skipped + info.failed) * 100 / info.planned) << "%."; - } + oss << info.success << " success, " << info.skipped << " skipped, " << info.failed << " failed, " + << ((info.success + info.skipped + info.failed) * 100 / info.planned) << "%."; + } - break; + break; - case RED_STATE_FINISH: - oss << "Redistribute is finished.\n" - << info.success << " success, " - << info.skipped << " skipped, " - << info.failed << " failed.\n"; + case RED_STATE_FINISH: + oss << "Redistribute is finished.\n" + << info.success << " success, " << info.skipped << " skipped, " << info.failed << " failed.\n"; - if (info.endTime > 0) - oss << "Total time: " << (info.endTime - info.startTime) << " seconds.\n"; + if (info.endTime > 0) + oss << "Total time: " << (info.endTime - info.startTime) << " seconds.\n"; - break; + break; - case RED_STATE_FAILED: - oss << "Redistribute is failed.\n"; + case RED_STATE_FAILED: + oss << "Redistribute is failed.\n"; - try - { - size_t l = 0; // message length - size_t n = fread(&l, sizeof(int), 1, fInfoFilePtr); - - if (n == 1) - { - boost::scoped_array buf(new char[l + 1]); - n = fread(buf.get(), 1, l, fInfoFilePtr); - - if (n == l) - { - buf[l] = '\0'; - fErrorMsg += buf.get(); - oss << buf.get(); - } - } - } - catch (const std::exception&) - { - } - catch (...) - { - } - - break; - - case RED_STATE_STOPPED: - oss << "Redistribute is stopped by user.\n"; - - if (info.planned > 0) - { - if (info.endTime > 0) - oss << "In " << (info.endTime - info.startTime) << " seconds, "; - - oss << info.success << " success, " - << info.skipped << " skipped, " - << info.failed << " failed, " - << ((info.success + info.skipped + info.failed) * 100 / info.planned) << "%."; - } - - break; - - default: - oss << "Failed to retrieve redistribute information, the file " - << fInfoFilePath << " may be corrupted."; - break; - } - - fUIResponse = oss.str(); - - return status; -} - - -int RedistributeControl::handleStopMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket& so) -{ - ostringstream oss; - uint32_t status = getCurrentState(); - - if (status != RED_STATE_ACTIVE) - { - oss << "Redistribute is not running. Command is ignored."; - } - else - { - RedistributeControlThread::setStopAction(true); - updateState(RED_STATE_STOPPED); - status = RED_STATE_STOPPED; - boost::thread rct((RedistributeControlThread(RED_CNTL_STOP))); - rct.join(); - oss << "Redistribute is stopped."; - } - - fUIResponse = oss.str(); - - return status; -} - - -int RedistributeControl::handleClearMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket& so) -{ - ostringstream oss; - uint32_t status = getCurrentState(); - - if (status == RED_STATE_ACTIVE) - { - oss << "Redistribute is running. Command is ignored. To CLEAR, you have to wait or stop the running session."; - } - else - { - updateState(RED_STATE_IDLE); - status = RED_STATE_IDLE; - oss << "Cleared."; - } - - fUIResponse = oss.str(); - - return status; -} - - -uint32_t RedistributeControl::getCurrentState() -{ - uint32_t status = RED_STATE_UNDEF; - ostringstream oss; - boost::mutex::scoped_lock lock(fInfoFileMutex); - - if (!fInfoFilePtr) - { - status = RED_STATE_IDLE; - } - else - { - rewind(fInfoFilePtr); - RedistributeInfo info; - size_t n = fread(&info, sizeof(info), 1, fInfoFilePtr); + try + { + size_t l = 0; // message length + size_t n = fread(&l, sizeof(int), 1, fInfoFilePtr); if (n == 1) { - fRedistributeInfo = info; - status = info.state; - } - } + boost::scoped_array buf(new char[l + 1]); + n = fread(buf.get(), 1, l, fInfoFilePtr); - return status; + if (n == l) + { + buf[l] = '\0'; + fErrorMsg += buf.get(); + oss << buf.get(); + } + } + } + catch (const std::exception&) + { + } + catch (...) + { + } + + break; + + case RED_STATE_STOPPED: + oss << "Redistribute is stopped by user.\n"; + + if (info.planned > 0) + { + if (info.endTime > 0) + oss << "In " << (info.endTime - info.startTime) << " seconds, "; + + oss << info.success << " success, " << info.skipped << " skipped, " << info.failed << " failed, " + << ((info.success + info.skipped + info.failed) * 100 / info.planned) << "%."; + } + + break; + + default: + oss << "Failed to retrieve redistribute information, the file " << fInfoFilePath + << " may be corrupted."; + break; + } + + fUIResponse = oss.str(); + + return status; } +int RedistributeControl::handleStopMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket& so) +{ + ostringstream oss; + uint32_t status = getCurrentState(); + + if (status != RED_STATE_ACTIVE) + { + oss << "Redistribute is not running. Command is ignored."; + } + else + { + RedistributeControlThread::setStopAction(true); + updateState(RED_STATE_STOPPED); + status = RED_STATE_STOPPED; + boost::thread rct((RedistributeControlThread(RED_CNTL_STOP))); + rct.join(); + oss << "Redistribute is stopped."; + } + + fUIResponse = oss.str(); + + return status; +} + +int RedistributeControl::handleClearMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket& so) +{ + ostringstream oss; + uint32_t status = getCurrentState(); + + if (status == RED_STATE_ACTIVE) + { + oss << "Redistribute is running. Command is ignored. To CLEAR, you have to wait or stop the running " + "session."; + } + else + { + updateState(RED_STATE_IDLE); + status = RED_STATE_IDLE; + oss << "Cleared."; + } + + fUIResponse = oss.str(); + + return status; +} + +uint32_t RedistributeControl::getCurrentState() +{ + uint32_t status = RED_STATE_UNDEF; + ostringstream oss; + boost::mutex::scoped_lock lock(fInfoFileMutex); + + if (!fInfoFilePtr) + { + status = RED_STATE_IDLE; + } + else + { + rewind(fInfoFilePtr); + RedistributeInfo info; + size_t n = fread(&info, sizeof(info), 1, fInfoFilePtr); + + if (n == 1) + { + fRedistributeInfo = info; + status = info.state; + } + } + + return status; +} bool RedistributeControl::getStartOptions(messageqcpp::ByteStream& bs) { - bool ret = true; - uint32_t n = 0; - uint32_t d = 0; + bool ret = true; + uint32_t n = 0; + uint32_t d = 0; - try + try + { + bs >> fOptions; + + bs >> n; + fSourceList.clear(); + fSourceList.reserve(n); + + for (uint32_t i = 0; i < n; i++) { - bs >> fOptions; - - bs >> n; - fSourceList.clear(); - fSourceList.reserve(n); - - for (uint32_t i = 0; i < n; i++) - { - bs >> d; - fSourceList.push_back(d); - } - - bs >> n; - fDestinationList.clear(); - fDestinationList.reserve(n); - - for (uint32_t i = 0; i < n; i++) - { - bs >> d; - fDestinationList.push_back(d); - } - - if (fSourceList.size() == 0 || fDestinationList.size() == 0) - throw runtime_error("Failed to get dbroot lists."); - } - catch (const std::exception& ex) - { - ret = false; - fErrorMsg = ex.what(); - } - catch (...) - { - ret = false; - fErrorMsg = "Failed to get dbroot lists."; + bs >> d; + fSourceList.push_back(d); } - return ret; + bs >> n; + fDestinationList.clear(); + fDestinationList.reserve(n); + + for (uint32_t i = 0; i < n; i++) + { + bs >> d; + fDestinationList.push_back(d); + } + + if (fSourceList.size() == 0 || fDestinationList.size() == 0) + throw runtime_error("Failed to get dbroot lists."); + } + catch (const std::exception& ex) + { + ret = false; + fErrorMsg = ex.what(); + } + catch (...) + { + ret = false; + fErrorMsg = "Failed to get dbroot lists."; + } + + return ret; } - void RedistributeControl::updateState(uint32_t s) { - boost::mutex::scoped_lock lock(fInfoFileMutex); + boost::mutex::scoped_lock lock(fInfoFileMutex); - // allowed state change: - // idle -> active - // active -> finish - // active -> stopped - // active -> failed - // finish -> idle - // stopped -> idle - // failed -> idle + // allowed state change: + // idle -> active + // active -> finish + // active -> stopped + // active -> failed + // finish -> idle + // stopped -> idle + // failed -> idle + if (s == RED_STATE_IDLE) + { + if (fRedistributeInfo.state == RED_STATE_ACTIVE) + return; - if (s == RED_STATE_IDLE) + // close the files if they are already opened + if (fInfoFilePtr != NULL) { - if (fRedistributeInfo.state == RED_STATE_ACTIVE) - return; - - // close the files if they are already opened - if (fInfoFilePtr != NULL) - { - fclose(fInfoFilePtr); - fInfoFilePtr = NULL; - } - - if (fPlanFilePtr != NULL) - { - fclose(fPlanFilePtr); - fPlanFilePtr = NULL; - } - - // move old files to archive - // zip or compress if the .plan file gets large - time_t t = fRedistributeInfo.startTime; - - if (t == 0) - t = time(NULL); - - ostringstream oss; - struct tm m; - localtime_r(&t, &m); - oss << setfill('0') << setw(4) << (m.tm_year + 1900) << setw(2) << (m.tm_mon + 1) - << setw(2) << (m.tm_mday) << setw(2) << (m.tm_hour) << setw(2) << (m.tm_min) - << setw(2) << (m.tm_sec); - - try - { - if (filesystem::exists(fInfoFilePath) && filesystem::exists(fPlanFilePath)) - { - bool mergeOk = false; - FILE* infoPtr = fopen(fInfoFilePath.c_str(), "r+b"); - FILE* entryPtr = fopen(fPlanFilePath.c_str(), "rb"); - int rc = 1; - - if (infoPtr != NULL && entryPtr != NULL) - { - rc = fseek(infoPtr, sizeof(RedistributeInfo), SEEK_SET); - RedistributePlanEntry entry; - - while (rc == 0) - { - size_t n = fread(&entry, sizeof(entry), 1, entryPtr); - - if (n != 1) - break; - - n = fwrite(&entry, sizeof(entry), 1, infoPtr); - fflush(infoPtr); - - if (n != 1) - rc = -1; - } - - } - - if (rc == 0 && feof(entryPtr)) - mergeOk = true; - - if (infoPtr != NULL) - fclose(infoPtr); - - if (entryPtr != NULL) - fclose(entryPtr); - - if (mergeOk) - filesystem::remove(fPlanFilePath); - } - - if (filesystem::exists(fInfoFilePath)) - { - string newInfoPath = fRedistributeDir + "/archive" + InfoFileName + "." + oss.str(); - filesystem::rename(fInfoFilePath, newInfoPath); - } - - if (filesystem::exists(fPlanFilePath)) - { - string newPlanPath = fRedistributeDir + "/archive" + PlanFileName + "." + oss.str(); - filesystem::rename(fPlanFilePath, newPlanPath); - } - } - catch (const std::exception&) - { - } - catch (...) - { - } - - fRedistributeInfo = RedistributeInfo(); - return; + fclose(fInfoFilePtr); + fInfoFilePtr = NULL; } - // safety check - if (s != RED_STATE_ACTIVE && fRedistributeInfo.state != RED_STATE_ACTIVE) - return; - - // in IDLE state there is no redistribute.info file - if (s == RED_STATE_ACTIVE) + if (fPlanFilePtr != NULL) { -// filesystem::path dirPath(fRedistributeDir); -// if (filesystem::exists(fRedistributeDir) && !filesystem::is_directory(fRedistributeDir)) -// filesystem::remove(fRedistributeDir); - if (!filesystem::exists(fRedistributeDir)) - { - errno = 0; - filesystem::create_directory(fRedistributeDir); - - if (!filesystem::exists(fRedistributeDir)) - { - int e = errno; - ostringstream oss; - oss << "Failed to create redistribute directory: "; - oss << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } - - errno = 0; - filesystem::path archivePath(fRedistributeDir + "/archive"); - filesystem::create_directory(archivePath); - - if (!filesystem::exists(archivePath)) - { - int e = errno; - ostringstream oss; - oss << "Failed to create redistribute archive directory: "; - oss << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } - } - - fRedistributeInfo.startTime = time(NULL); + fclose(fPlanFilePtr); + fPlanFilePtr = NULL; } + // move old files to archive + // zip or compress if the .plan file gets large + time_t t = fRedistributeInfo.startTime; - // open the info file to write - errno = 0; + if (t == 0) + t = time(NULL); - if (fInfoFilePtr == NULL) - fInfoFilePtr = fopen(fInfoFilePath.c_str(), "w+"); - - if (fInfoFilePtr == NULL) - { - int e = errno; - ostringstream oss; - oss << "Failed to open " << fInfoFilePath << ": " << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } - - fRedistributeInfo.state = s; - - if (s == RED_STATE_FINISH) - fRedistributeInfo.endTime = time(NULL); - - rewind(fInfoFilePtr); - size_t n = fwrite(&fRedistributeInfo, sizeof(fRedistributeInfo), 1, fInfoFilePtr); - - if (n != 1) - { - fclose(fInfoFilePtr); - fInfoFilePtr = NULL; - - int e = errno; - ostringstream oss; - oss << "Failed to write into " << fInfoFilePath << ": " << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } - - fflush(fInfoFilePtr); -} - - -void RedistributeControl::setEntryCount(uint32_t entryCount) -{ - boost::mutex::scoped_lock lock(fInfoFileMutex); - fRedistributeInfo.planned = entryCount; - - rewind(fInfoFilePtr); - fwrite(&fRedistributeInfo, sizeof(fRedistributeInfo), 1, fInfoFilePtr); - fflush(fInfoFilePtr); -} - - -void RedistributeControl::updateProgressInfo(uint32_t s, time_t t) -{ - boost::mutex::scoped_lock lock(fInfoFileMutex); - fRedistributeInfo.endTime = t; - - switch (s) - { - case RED_TRANS_SUCCESS: - fRedistributeInfo.success++; - break; - - case RED_TRANS_SKIPPED: - fRedistributeInfo.skipped++; - break; - - default: - fRedistributeInfo.failed++; - break; - } - - rewind(fInfoFilePtr); - fwrite(&fRedistributeInfo, sizeof(fRedistributeInfo), 1, fInfoFilePtr); - fflush(fInfoFilePtr); -} - - -int RedistributeControl::handleJobMsg(messageqcpp::ByteStream& bs, messageqcpp::IOSocket& so) -{ -// boost::mutex::scoped_lock jobLock(fJobMutex); - - uint32_t status = RED_TRANS_SUCCESS; + ostringstream oss; + struct tm m; + localtime_r(&t, &m); + oss << setfill('0') << setw(4) << (m.tm_year + 1900) << setw(2) << (m.tm_mon + 1) << setw(2) + << (m.tm_mday) << setw(2) << (m.tm_hour) << setw(2) << (m.tm_min) << setw(2) << (m.tm_sec); try { - fWorkThread.reset(new boost::thread(RedistributeWorkerThread(bs, so))); - fWorkThread->join(); + if (filesystem::exists(fInfoFilePath) && filesystem::exists(fPlanFilePath)) + { + bool mergeOk = false; + FILE* infoPtr = fopen(fInfoFilePath.c_str(), "r+b"); + FILE* entryPtr = fopen(fPlanFilePath.c_str(), "rb"); + int rc = 1; + + if (infoPtr != NULL && entryPtr != NULL) + { + rc = fseek(infoPtr, sizeof(RedistributeInfo), SEEK_SET); + RedistributePlanEntry entry; + + while (rc == 0) + { + size_t n = fread(&entry, sizeof(entry), 1, entryPtr); + + if (n != 1) + break; + + n = fwrite(&entry, sizeof(entry), 1, infoPtr); + fflush(infoPtr); + + if (n != 1) + rc = -1; + } + } + + if (rc == 0 && feof(entryPtr)) + mergeOk = true; + + if (infoPtr != NULL) + fclose(infoPtr); + + if (entryPtr != NULL) + fclose(entryPtr); + + if (mergeOk) + filesystem::remove(fPlanFilePath); + } + + if (filesystem::exists(fInfoFilePath)) + { + string newInfoPath = fRedistributeDir + "/archive" + InfoFileName + "." + oss.str(); + filesystem::rename(fInfoFilePath, newInfoPath); + } + + if (filesystem::exists(fPlanFilePath)) + { + string newPlanPath = fRedistributeDir + "/archive" + PlanFileName + "." + oss.str(); + filesystem::rename(fPlanFilePath, newPlanPath); + } } - catch (const std::exception& ex) + catch (const std::exception&) { - status = RED_TRANS_FAILED; - logMessage(ex.what()); } catch (...) { - status = RED_TRANS_FAILED; } - return status; + fRedistributeInfo = RedistributeInfo(); + return; + } + + // safety check + if (s != RED_STATE_ACTIVE && fRedistributeInfo.state != RED_STATE_ACTIVE) + return; + + // in IDLE state there is no redistribute.info file + if (s == RED_STATE_ACTIVE) + { + // filesystem::path dirPath(fRedistributeDir); + // if (filesystem::exists(fRedistributeDir) && !filesystem::is_directory(fRedistributeDir)) + // filesystem::remove(fRedistributeDir); + if (!filesystem::exists(fRedistributeDir)) + { + errno = 0; + filesystem::create_directory(fRedistributeDir); + + if (!filesystem::exists(fRedistributeDir)) + { + int e = errno; + ostringstream oss; + oss << "Failed to create redistribute directory: "; + oss << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } + + errno = 0; + filesystem::path archivePath(fRedistributeDir + "/archive"); + filesystem::create_directory(archivePath); + + if (!filesystem::exists(archivePath)) + { + int e = errno; + ostringstream oss; + oss << "Failed to create redistribute archive directory: "; + oss << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } + } + + fRedistributeInfo.startTime = time(NULL); + } + + // open the info file to write + errno = 0; + + if (fInfoFilePtr == NULL) + fInfoFilePtr = fopen(fInfoFilePath.c_str(), "w+"); + + if (fInfoFilePtr == NULL) + { + int e = errno; + ostringstream oss; + oss << "Failed to open " << fInfoFilePath << ": " << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } + + fRedistributeInfo.state = s; + + if (s == RED_STATE_FINISH) + fRedistributeInfo.endTime = time(NULL); + + rewind(fInfoFilePtr); + size_t n = fwrite(&fRedistributeInfo, sizeof(fRedistributeInfo), 1, fInfoFilePtr); + + if (n != 1) + { + fclose(fInfoFilePtr); + fInfoFilePtr = NULL; + + int e = errno; + ostringstream oss; + oss << "Failed to write into " << fInfoFilePath << ": " << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } + + fflush(fInfoFilePtr); } +void RedistributeControl::setEntryCount(uint32_t entryCount) +{ + boost::mutex::scoped_lock lock(fInfoFileMutex); + fRedistributeInfo.planned = entryCount; + + rewind(fInfoFilePtr); + fwrite(&fRedistributeInfo, sizeof(fRedistributeInfo), 1, fInfoFilePtr); + fflush(fInfoFilePtr); +} + +void RedistributeControl::updateProgressInfo(uint32_t s, time_t t) +{ + boost::mutex::scoped_lock lock(fInfoFileMutex); + fRedistributeInfo.endTime = t; + + switch (s) + { + case RED_TRANS_SUCCESS: fRedistributeInfo.success++; break; + + case RED_TRANS_SKIPPED: fRedistributeInfo.skipped++; break; + + default: fRedistributeInfo.failed++; break; + } + + rewind(fInfoFilePtr); + fwrite(&fRedistributeInfo, sizeof(fRedistributeInfo), 1, fInfoFilePtr); + fflush(fInfoFilePtr); +} + +int RedistributeControl::handleJobMsg(messageqcpp::ByteStream& bs, messageqcpp::IOSocket& so) +{ + // boost::mutex::scoped_lock jobLock(fJobMutex); + + uint32_t status = RED_TRANS_SUCCESS; + + try + { + fWorkThread.reset(new boost::thread(RedistributeWorkerThread(bs, so))); + fWorkThread->join(); + } + catch (const std::exception& ex) + { + status = RED_TRANS_FAILED; + logMessage(ex.what()); + } + catch (...) + { + status = RED_TRANS_FAILED; + } + + return status; +} void RedistributeControl::logMessage(const string& msg) { - logging::Message::Args args; - args.add(string("RED:")); - args.add(msg); + logging::Message::Args args; + args.add(string("RED:")); + args.add(msg); - fSysLogger->logMessage( - logging::LOG_TYPE_INFO, logging::M0002, args, logging::LoggingID(32, 0, 0)); + fSysLogger->logMessage(logging::LOG_TYPE_INFO, logging::M0002, args, logging::LoggingID(32, 0, 0)); } - -} // namespace +} // namespace redistribute // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistributecontrol.h b/writeengine/redistribute/we_redistributecontrol.h index c5f492347..655350515 100644 --- a/writeengine/redistribute/we_redistributecontrol.h +++ b/writeengine/redistribute/we_redistributecontrol.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributecontrol.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistributecontrol.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ #ifndef WE_REDISTRIBUTECONTROL_H #define WE_REDISTRIBUTECONTROL_H @@ -35,7 +35,7 @@ namespace messageqcpp { class ByteStream; class IOSocket; -} +} // namespace messageqcpp namespace oam { @@ -54,82 +54,77 @@ class Logger; namespace redistribute { - class RedistributeControl { -public: - ~RedistributeControl(); + public: + ~RedistributeControl(); - int handleUIMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - int handleJobMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleUIMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleJobMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - static RedistributeControl* instance(); - static void destroyInstace(); + static RedistributeControl* instance(); + static void destroyInstace(); -private: - int handleStartMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - int handleStatusMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - int handleStopMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - int handleClearMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + private: + int handleStartMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleStatusMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleStopMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleClearMsg(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - int handleStatusRpt(messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleStatusRpt(messageqcpp::ByteStream&, messageqcpp::IOSocket&); - uint32_t getCurrentState(); - bool getStartOptions(messageqcpp::ByteStream&); + uint32_t getCurrentState(); + bool getStartOptions(messageqcpp::ByteStream&); - void setEntryCount(uint32_t); - void updateState(uint32_t); - void updateProgressInfo(uint32_t, time_t); + void setEntryCount(uint32_t); + void updateState(uint32_t); + void updateProgressInfo(uint32_t, time_t); - void logMessage(const std::string&); + void logMessage(const std::string&); - boost::mutex fSessionMutex; - boost::mutex fInfoFileMutex; + boost::mutex fSessionMutex; + boost::mutex fInfoFileMutex; - boost::scoped_ptr fControlThread; - boost::scoped_ptr fWorkThread; + boost::scoped_ptr fControlThread; + boost::scoped_ptr fWorkThread; - FILE* fInfoFilePtr; - FILE* fPlanFilePtr; - std::string fRedistributeDir; - std::string fInfoFilePath; - std::string fPlanFilePath; - std::string fUIResponse; + FILE* fInfoFilePtr; + FILE* fPlanFilePtr; + std::string fRedistributeDir; + std::string fInfoFilePath; + std::string fPlanFilePath; + std::string fUIResponse; - uint32_t fOptions; - std::vector fSourceList; - std::vector fDestinationList; - std::vector fRedistributePlan; - RedistributeInfo fRedistributeInfo; + uint32_t fOptions; + std::vector fSourceList; + std::vector fDestinationList; + std::vector fRedistributePlan; + RedistributeInfo fRedistributeInfo; - std::string fErrorMsg; + std::string fErrorMsg; - // for work threads, they don't have to create their own. - boost::shared_ptr fOam; - boost::shared_ptr fDbrm; - boost::shared_ptr fSysLogger; + // for work threads, they don't have to create their own. + boost::shared_ptr fOam; + boost::shared_ptr fDbrm; + boost::shared_ptr fSysLogger; - // singleton instance - static RedistributeControl* fInstance; + // singleton instance + static RedistributeControl* fInstance; - // private constructor - RedistributeControl(); + // private constructor + RedistributeControl(); - // disable copy constructor and assignment operator - // private without implementation - RedistributeControl(const RedistributeControl&); - RedistributeControl& operator=(const RedistributeControl&); + // disable copy constructor and assignment operator + // private without implementation + RedistributeControl(const RedistributeControl&); + RedistributeControl& operator=(const RedistributeControl&); - - friend class RedistributeControlThread; - friend class RedistributeWorkerThread; + friend class RedistributeControlThread; + friend class RedistributeWorkerThread; }; - -} // namespace - +} // namespace redistribute #endif // WE_REDISTRIBUTECONTROL_H // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 8b92c430f..ce808eb20 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -17,8 +17,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributecontrolthread.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistributecontrolthread.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + */ #include #include @@ -64,792 +64,778 @@ using namespace execplan; namespace redistribute { - // static variables boost::mutex RedistributeControlThread::fActionMutex; volatile bool RedistributeControlThread::fStopAction = false; string RedistributeControlThread::fWesInUse; - void RedistributeControlThread::setStopAction(bool s) { - boost::mutex::scoped_lock lock(fActionMutex); - fStopAction = s; + boost::mutex::scoped_lock lock(fActionMutex); + fStopAction = s; } - -RedistributeControlThread::RedistributeControlThread(uint32_t act) : - fAction(act), fMaxDbroot(0), fEntryCount(0), fErrorCode(RED_EC_OK) +RedistributeControlThread::RedistributeControlThread(uint32_t act) + : fAction(act), fMaxDbroot(0), fEntryCount(0), fErrorCode(RED_EC_OK) { } - RedistributeControlThread::~RedistributeControlThread() { -// fWEClient->removeQueue(uniqueId); + // fWEClient->removeQueue(uniqueId); } - void RedistributeControlThread::operator()() { - if (fAction == RED_CNTL_START) - doRedistribute(); - else if (fAction == RED_CNTL_STOP) - doStopAction(); + if (fAction == RED_CNTL_START) + doRedistribute(); + else if (fAction == RED_CNTL_STOP) + doStopAction(); } - void RedistributeControlThread::doRedistribute() { - if (setup() != 0) - fErrorCode = RED_EC_CNTL_SETUP_FAIL; - else if (makeRedistributePlan() != 0) - fErrorCode = RED_EC_MAKEPLAN_FAIL; + if (setup() != 0) + fErrorCode = RED_EC_CNTL_SETUP_FAIL; + else if (makeRedistributePlan() != 0) + fErrorCode = RED_EC_MAKEPLAN_FAIL; - try - { - if (fErrorCode == RED_EC_OK && !fStopAction && fEntryCount > 0) - executeRedistributePlan(); - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); - fErrorCode = RED_EC_EXECUTE_FAIL; - } - catch (...) - { - fErrorMsg += "Error when executing the plan."; - fErrorCode = RED_EC_EXECUTE_FAIL; - } + try + { + if (fErrorCode == RED_EC_OK && !fStopAction && fEntryCount > 0) + executeRedistributePlan(); + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); + fErrorCode = RED_EC_EXECUTE_FAIL; + } + catch (...) + { + fErrorMsg += "Error when executing the plan."; + fErrorCode = RED_EC_EXECUTE_FAIL; + } - uint32_t state = RED_STATE_FINISH; + uint32_t state = RED_STATE_FINISH; - if (fErrorCode != RED_EC_OK) - state = RED_STATE_FAILED; + if (fErrorCode != RED_EC_OK) + state = RED_STATE_FAILED; - try - { - if (!fStopAction) - fControl->updateState(state); - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); + try + { + if (!fStopAction) + fControl->updateState(state); + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); - if (fErrorCode == RED_EC_OK) - fErrorCode = RED_EC_UPDATE_STATE; - } - catch (...) - { - fErrorMsg += "Error when updating state."; + if (fErrorCode == RED_EC_OK) + fErrorCode = RED_EC_UPDATE_STATE; + } + catch (...) + { + fErrorMsg += "Error when updating state."; - if (fErrorCode == RED_EC_OK) - fErrorCode = RED_EC_UPDATE_STATE; - } + if (fErrorCode == RED_EC_OK) + fErrorCode = RED_EC_UPDATE_STATE; + } - if (fErrorMsg.empty()) - fControl->logMessage("finished @controlThread::doRedistribute"); - else - fControl->logMessage(fErrorMsg + " @controlThread::doRedistribute"); + if (fErrorMsg.empty()) + fControl->logMessage("finished @controlThread::doRedistribute"); + else + fControl->logMessage(fErrorMsg + " @controlThread::doRedistribute"); - { - boost::mutex::scoped_lock lock(fActionMutex); - fWesInUse.clear(); - } + { + boost::mutex::scoped_lock lock(fActionMutex); + fWesInUse.clear(); + } } - int RedistributeControlThread::setup() { - int ret = 0; + int ret = 0; - try + try + { + // fUniqueId = fDbrm.getUnique64(); + // fWEClient = WriteEngine::WEClients::instance(WriteEngine::WEClients::REDISTRIBUTE); + // fWEClient->addQueue(uniqueId); + fConfig = Config::makeConfig(); + fOamCache = oam::OamCache::makeOamCache(); + fControl = RedistributeControl::instance(); + // fOam.reset(new oam::Oam); + // fDbrm.reset(new BRM::DBRM); + + vector::iterator i = fControl->fSourceList.begin(); + + for (; i != fControl->fSourceList.end(); i++) { -// fUniqueId = fDbrm.getUnique64(); -// fWEClient = WriteEngine::WEClients::instance(WriteEngine::WEClients::REDISTRIBUTE); -// fWEClient->addQueue(uniqueId); - fConfig = Config::makeConfig(); - fOamCache = oam::OamCache::makeOamCache(); - fControl = RedistributeControl::instance(); -// fOam.reset(new oam::Oam); -// fDbrm.reset(new BRM::DBRM); + fSourceSet.insert(*i); + fDbrootSet.insert(*i); - vector::iterator i = fControl->fSourceList.begin(); - - for (; i != fControl->fSourceList.end(); i++) - { - fSourceSet.insert(*i); - fDbrootSet.insert(*i); - - if (*i > fMaxDbroot) - fMaxDbroot = *i; - } - - vector::iterator j = fControl->fDestinationList.begin(); - - for (; j != fControl->fDestinationList.end(); j++) - { - fTargetSet.insert(*j); - - if (fDbrootSet.find(*j) == fDbrootSet.end()) - { - fDbrootSet.insert(*j); - } - -// if (*j > fMaxDbroot) -// fMaxDbroot = *j; - } - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); - ret = 1; - } - catch (...) - { - ret = 1; + if (*i > fMaxDbroot) + fMaxDbroot = *i; } - return ret; + vector::iterator j = fControl->fDestinationList.begin(); + + for (; j != fControl->fDestinationList.end(); j++) + { + fTargetSet.insert(*j); + + if (fDbrootSet.find(*j) == fDbrootSet.end()) + { + fDbrootSet.insert(*j); + } + + // if (*j > fMaxDbroot) + // fMaxDbroot = *j; + } + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); + ret = 1; + } + catch (...) + { + ret = 1; + } + + return ret; } - int RedistributeControlThread::makeRedistributePlan() { - int ret = 0; + int ret = 0; - try + try + { + if (fControl->fPlanFilePtr != NULL) { - if (fControl->fPlanFilePtr != NULL) + // should not happen, just in case. + fclose(fControl->fPlanFilePtr); + fControl->fPlanFilePtr = NULL; + } + + // get all user table oids + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); + vector > tables = csc->getTables(); + vector >::iterator i; + + for (i = tables.begin(); i != tables.end(); i++) + { + // in case, action is cancelled. + if (fStopAction) + break; + + // column oids + CalpontSystemCatalog::RIDList cols = csc->columnRIDs(i->second, true); + typedef std::map PartitionExtentMap; + PartitionExtentMap partitionMap; + vector entries; + + // sample the first column + int rc = fControl->fDbrm->getExtents(cols[0].objnum, entries, false, false, true); + + if (rc != 0 || entries.size() == 0) + { + ostringstream oss; + oss << "Error in DBRM getExtents; oid:" << cols[0].objnum << "; returnCode: " << rc; + throw runtime_error(oss.str()); + } + + for (vector::iterator j = entries.begin(); j != entries.end(); j++) + { + RedistributeExtentEntry redEntry; + redEntry.oid = cols[0].objnum; + redEntry.dbroot = j->dbRoot; + redEntry.partition = j->partitionNum; + redEntry.segment = j->segmentNum; + redEntry.lbid = j->range.start; + redEntry.range = j->range.size * 1024; + + PartitionInfo partInfo(j->dbRoot, j->partitionNum); + partitionMap.insert(make_pair(partInfo, redEntry)); + } + + // sort partitions by dbroot + vector > dbPartVec(fMaxDbroot + 1); + uint64_t totalPartitionCount = 0; + int maxPartitionId = 0; + + for (PartitionExtentMap::iterator j = partitionMap.begin(); j != partitionMap.end(); j++) + { + int dbroot = j->first.dbroot; + + if (fSourceSet.find(dbroot) != fSourceSet.end()) { - // should not happen, just in case. - fclose(fControl->fPlanFilePtr); - fControl->fPlanFilePtr = NULL; + // only dbroot in source list needs attention + dbPartVec[dbroot].push_back(j->first.partition); + + if (j->first.partition > maxPartitionId) + maxPartitionId = j->first.partition; + + totalPartitionCount++; + } + } + + // sort the partitions + for (vector >::iterator k = dbPartVec.begin(); k != dbPartVec.end(); k++) + sort(k->begin(), k->end()); + + // divide the dbroots into the source and target sets + uint64_t average = totalPartitionCount / fTargetSet.size(); + // Remainder is the number of partitions that must be spread across some + // of the dbroots such that no dbroot has more than average+1 partitions. + uint64_t remainder = totalPartitionCount % fTargetSet.size(); + set sourceDbroots; + set targetDbroots; + // list > targetList; // to be ordered by partition size + int64_t extra = remainder; + + for (set::iterator j = fDbrootSet.begin(); j != fDbrootSet.end(); ++j) + { + if (fTargetSet.find(*j) == fTargetSet.end()) + { + // Not a target (removed on command line). Always a source. + sourceDbroots.insert(*j); + continue; } - // get all user table oids - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); - vector > - tables = csc->getTables(); - vector >::iterator i; - - for (i = tables.begin(); i != tables.end(); i++) + // If a dbroot has exactly average+1 partitions and there's extras to be had, + // then it is neither a source nor a target. + if ((dbPartVec[*j].size() == average + 1) && extra) { - // in case, action is cancelled. - if (fStopAction) - break; + --extra; + continue; + } - // column oids - CalpontSystemCatalog::RIDList cols = csc->columnRIDs(i->second, true); - typedef std::map PartitionExtentMap; - PartitionExtentMap partitionMap; - vector entries; + if (dbPartVec[*j].size() > average) + { + // Sources are those dbroots with more than average partitions + sourceDbroots.insert(*j); + } + else + { + // Targets are those with room ( <= average ) + targetDbroots.insert(*j); + } + } - // sample the first column - int rc = fControl->fDbrm->getExtents(cols[0].objnum, entries, false, false, true); + // At this point, there are two concepts of target. (1)Those in fTargetSet, which is the + // set of dbroots the user wants partitions on and (2) those in targetDbroots, a subset of + // fTargetSet, which is those that actually have room, based on average, for more data. - if (rc != 0 || entries.size() == 0) + // After redistribution, partition count for each dbroot is average or average+1. + // When remainder > 0, some targets will have (average+1) partitions. + + // loop through target dbroots and find partitions from sources to move to each. + set::iterator sourceDbroot = sourceDbroots.begin(); + int sourceCnt = sourceDbroots.size(); + + for (set::iterator targetDbroot = targetDbroots.begin(); targetDbroot != targetDbroots.end(); + ++targetDbroot) + { + // check if this target will have average + 1 partitions. + uint64_t e = 0; + + if (extra-- > 0) + e = 1; + + // A set of the partitions already on the target. We try not to move the same partition here. + set targetParts(dbPartVec[*targetDbroot].begin(), dbPartVec[*targetDbroot].end()); + + if (targetParts.size() >= (average + e)) + continue; // Don't move any partitions to this target + + // partitions to be moved to this target + vector planVec; + + // looking for source partitions start from partition 0 + bool done = false; // if target got enough partitions, set to true. + int loop = 0; // avoid infinite loop. maxPartitionId is the last partition of one of the dbroots. + // It's a place to stop if all else fails. + + while (!done && loop <= maxPartitionId) + { + for (int p = loop++; p <= maxPartitionId && !done; ++p) + { + bool found = false; + + if (targetParts.find(p) == + targetParts.end()) // True if the partition is not on the target already { - ostringstream oss; - oss << "Error in DBRM getExtents; oid:" << cols[0].objnum << "; returnCode: " << rc; - throw runtime_error(oss.str()); - } + // try to find partition p in one of the source dbroots + for (int x = 0; x < sourceCnt && !found; ++x) + { + vector& sourceParts = dbPartVec[*sourceDbroot]; + // This partition needs to move if: + // 1) source still has more than average partitions, or + // 2) source is not a listed target (we want to empty source). + bool bNotTarget = + fTargetSet.find(*sourceDbroot) == fTargetSet.end(); // true if source not in target list - for (vector::iterator j = entries.begin(); j != entries.end(); j++) - { - RedistributeExtentEntry redEntry; - redEntry.oid = cols[0].objnum; - redEntry.dbroot = j->dbRoot; - redEntry.partition = j->partitionNum; - redEntry.segment = j->segmentNum; - redEntry.lbid = j->range.start; - redEntry.range = j->range.size * 1024; - - PartitionInfo partInfo(j->dbRoot, j->partitionNum); - partitionMap.insert(make_pair(partInfo, redEntry)); - } - - // sort partitions by dbroot - vector > dbPartVec(fMaxDbroot + 1); - uint64_t totalPartitionCount = 0; - int maxPartitionId = 0; - - for (PartitionExtentMap::iterator j = partitionMap.begin(); - j != partitionMap.end(); j++) - { - int dbroot = j->first.dbroot; - - if (fSourceSet.find(dbroot) != fSourceSet.end()) + if (sourceParts.size() >= average || bNotTarget) { - // only dbroot in source list needs attention - dbPartVec[dbroot].push_back(j->first.partition); + vector::iterator y = find(sourceParts.begin(), sourceParts.end(), p); - if (j->first.partition > maxPartitionId) - maxPartitionId = j->first.partition; - - totalPartitionCount++; - } - } - - // sort the partitions - for (vector >::iterator k = dbPartVec.begin(); k != dbPartVec.end(); k++) - sort(k->begin(), k->end()); - - // divide the dbroots into the source and target sets - uint64_t average = totalPartitionCount / fTargetSet.size(); - // Remainder is the number of partitions that must be spread across some - // of the dbroots such that no dbroot has more than average+1 partitions. - uint64_t remainder = totalPartitionCount % fTargetSet.size(); - set sourceDbroots; - set targetDbroots; -// list > targetList; // to be ordered by partition size - int64_t extra = remainder; - - for (set::iterator j = fDbrootSet.begin(); j != fDbrootSet.end(); ++j) - { - if (fTargetSet.find(*j) == fTargetSet.end()) - { - // Not a target (removed on command line). Always a source. - sourceDbroots.insert(*j); - continue; - } - - // If a dbroot has exactly average+1 partitions and there's extras to be had, - // then it is neither a source nor a target. - if ((dbPartVec[*j].size() == average + 1) && extra) - { - --extra; - continue; - } - - if (dbPartVec[*j].size() > average) - { - // Sources are those dbroots with more than average partitions - sourceDbroots.insert(*j); - } - else - { - // Targets are those with room ( <= average ) - targetDbroots.insert(*j); - } - } - - // At this point, there are two concepts of target. (1)Those in fTargetSet, which is the - // set of dbroots the user wants partitions on and (2) those in targetDbroots, a subset of - // fTargetSet, which is those that actually have room, based on average, for more data. - - // After redistribution, partition count for each dbroot is average or average+1. - // When remainder > 0, some targets will have (average+1) partitions. - - // loop through target dbroots and find partitions from sources to move to each. - set::iterator sourceDbroot = sourceDbroots.begin(); - int sourceCnt = sourceDbroots.size(); - - for (set::iterator targetDbroot = targetDbroots.begin(); - targetDbroot != targetDbroots.end(); - ++targetDbroot) - { - // check if this target will have average + 1 partitions. - uint64_t e = 0; - - if (extra-- > 0) - e = 1; - - // A set of the partitions already on the target. We try not to move the same partition here. - set targetParts(dbPartVec[*targetDbroot].begin(), dbPartVec[*targetDbroot].end()); - - if (targetParts.size() >= (average + e)) - continue; // Don't move any partitions to this target - - // partitions to be moved to this target - vector planVec; - - // looking for source partitions start from partition 0 - bool done = false; // if target got enough partitions, set to true. - int loop = 0; // avoid infinite loop. maxPartitionId is the last partition of one of the dbroots. It's a place to stop if all else fails. - - while (!done && loop <= maxPartitionId) - { - for (int p = loop++; p <= maxPartitionId && !done; ++p) - { - bool found = false; - - if (targetParts.find(p) == targetParts.end()) // True if the partition is not on the target already - { - // try to find partition p in one of the source dbroots - for (int x = 0; x < sourceCnt && !found; ++x) - { - vector& sourceParts = dbPartVec[*sourceDbroot]; - // This partition needs to move if: - // 1) source still has more than average partitions, or - // 2) source is not a listed target (we want to empty source). - bool bNotTarget = fTargetSet.find(*sourceDbroot) == fTargetSet.end(); // true if source not in target list - - if (sourceParts.size() >= average || bNotTarget) - { - vector::iterator y = find(sourceParts.begin(), sourceParts.end(), p); - - if ((y != sourceParts.end()) && - (sourceParts.size() > targetParts.size() || bNotTarget)) - { - targetParts.insert(p); - planVec.push_back(PartitionInfo(*sourceDbroot, p)); - found = true; - - // update the source - sourceParts.erase(y); - } - } - - if (++sourceDbroot == sourceDbroots.end()) - sourceDbroot = sourceDbroots.begin(); - } // for source - - if (targetParts.size() == (average + e)) - done = true; - } // !find p - } // for p - } // while loop - - // dump the plan for the target to file - dumpPlanToFile(i->first, planVec, *targetDbroot); - } // for target - - // It's possible that a source that is "removed" on the command line is not empty. - // This can happen if a partition exists on all dbroots. - - // WCOL-786: use nextDbroot to start the loop looking for a suitible target - // where we left off with the previous partition. This gives each target - // an opportunity to get some of the data. In the case of dbroot removal, - // there is often the same number of partitions on each remaining dbroot. - // This logic tends to roundrobin which dbroot gets the next batch. - set::iterator nextDbroot = targetDbroots.begin(); - set::iterator targetDbroot; - int targetCnt = (int)targetDbroots.size(); - - // Loop through the sources, looking for dbroots that are not targets that also still contain partitions - for (set::iterator sourceDbroot = sourceDbroots.begin(); - sourceDbroot != sourceDbroots.end(); - ++sourceDbroot) - { - // Is this source in target list? If so, do nothing. - if (fTargetSet.find(*sourceDbroot) != fTargetSet.end()) - continue; - - vector& sourceParts = dbPartVec[*sourceDbroot]; // Partitions still on source - - // We can't erase from a vector we're iterating, so we need a kludge: - for (int p = 0; p <= maxPartitionId; ++p) - { - vector::iterator sourcePart = find(sourceParts.begin(), sourceParts.end(), p); - - if (sourcePart == sourceParts.end()) - { - continue; - } - - // Look through targets to see which can accept this partition. Find the one with the least - // number of partitions. Someday we want to put with the dbroot having the fewest segments of - // the partition. - uint64_t partCount = std::numeric_limits::max(); - int tdbroot = 0; - targetDbroot = nextDbroot; - - // MCOL-786. Start at targetDbroot and loop around back to the same spot. - for (int tbd = 0; tbd < targetCnt; ++tbd) - { - if (targetDbroot == targetDbroots.end()) - { - targetDbroot = targetDbroots.begin(); - } - - if (dbPartVec[*targetDbroot].size() < partCount) - { - tdbroot = *targetDbroot; - partCount = dbPartVec[*targetDbroot].size(); - nextDbroot = targetDbroot; - ++nextDbroot; - } - - ++targetDbroot; - } - - if (tdbroot == 0) - { - continue; - } - - set targetParts(dbPartVec[tdbroot].begin(), dbPartVec[tdbroot].end()); - vector planVec; // partitions to be moved to this target + if ((y != sourceParts.end()) && (sourceParts.size() > targetParts.size() || bNotTarget)) + { targetParts.insert(p); planVec.push_back(PartitionInfo(*sourceDbroot, p)); - sourceParts.erase(sourcePart); - dumpPlanToFile(i->first, planVec, tdbroot); - } // for sourceParts - } // for source - } // for tables - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); - ret = 2; - } - catch (...) - { - ret = 2; - } + found = true; - displayPlan(); - return ret; + // update the source + sourceParts.erase(y); + } + } + + if (++sourceDbroot == sourceDbroots.end()) + sourceDbroot = sourceDbroots.begin(); + } // for source + + if (targetParts.size() == (average + e)) + done = true; + } // !find p + } // for p + } // while loop + + // dump the plan for the target to file + dumpPlanToFile(i->first, planVec, *targetDbroot); + } // for target + + // It's possible that a source that is "removed" on the command line is not empty. + // This can happen if a partition exists on all dbroots. + + // WCOL-786: use nextDbroot to start the loop looking for a suitible target + // where we left off with the previous partition. This gives each target + // an opportunity to get some of the data. In the case of dbroot removal, + // there is often the same number of partitions on each remaining dbroot. + // This logic tends to roundrobin which dbroot gets the next batch. + set::iterator nextDbroot = targetDbroots.begin(); + set::iterator targetDbroot; + int targetCnt = (int)targetDbroots.size(); + + // Loop through the sources, looking for dbroots that are not targets that also still contain partitions + for (set::iterator sourceDbroot = sourceDbroots.begin(); sourceDbroot != sourceDbroots.end(); + ++sourceDbroot) + { + // Is this source in target list? If so, do nothing. + if (fTargetSet.find(*sourceDbroot) != fTargetSet.end()) + continue; + + vector& sourceParts = dbPartVec[*sourceDbroot]; // Partitions still on source + + // We can't erase from a vector we're iterating, so we need a kludge: + for (int p = 0; p <= maxPartitionId; ++p) + { + vector::iterator sourcePart = find(sourceParts.begin(), sourceParts.end(), p); + + if (sourcePart == sourceParts.end()) + { + continue; + } + + // Look through targets to see which can accept this partition. Find the one with the least + // number of partitions. Someday we want to put with the dbroot having the fewest segments of + // the partition. + uint64_t partCount = std::numeric_limits::max(); + int tdbroot = 0; + targetDbroot = nextDbroot; + + // MCOL-786. Start at targetDbroot and loop around back to the same spot. + for (int tbd = 0; tbd < targetCnt; ++tbd) + { + if (targetDbroot == targetDbroots.end()) + { + targetDbroot = targetDbroots.begin(); + } + + if (dbPartVec[*targetDbroot].size() < partCount) + { + tdbroot = *targetDbroot; + partCount = dbPartVec[*targetDbroot].size(); + nextDbroot = targetDbroot; + ++nextDbroot; + } + + ++targetDbroot; + } + + if (tdbroot == 0) + { + continue; + } + + set targetParts(dbPartVec[tdbroot].begin(), dbPartVec[tdbroot].end()); + vector planVec; // partitions to be moved to this target + targetParts.insert(p); + planVec.push_back(PartitionInfo(*sourceDbroot, p)); + sourceParts.erase(sourcePart); + dumpPlanToFile(i->first, planVec, tdbroot); + } // for sourceParts + } // for source + } // for tables + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); + ret = 2; + } + catch (...) + { + ret = 2; + } + + displayPlan(); + return ret; } void RedistributeControlThread::dumpPlanToFile(uint64_t oid, vector& vec, int target) { - // open the plan file, if not already opened, to write. + // open the plan file, if not already opened, to write. + if (fControl->fPlanFilePtr == NULL) + { + errno = 0; + fControl->fPlanFilePtr = fopen(fControl->fPlanFilePath.c_str(), "w+"); + if (fControl->fPlanFilePtr == NULL) { - errno = 0; - fControl->fPlanFilePtr = fopen(fControl->fPlanFilePath.c_str(), "w+"); - - if (fControl->fPlanFilePtr == NULL) - { - int e = errno; - ostringstream oss; - oss << "Failed to open redistribute.plan: " << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } + int e = errno; + ostringstream oss; + oss << "Failed to open redistribute.plan: " << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); } + } - size_t entryNum = vec.size(); - scoped_array entries(new RedistributePlanEntry[entryNum]); + size_t entryNum = vec.size(); + scoped_array entries(new RedistributePlanEntry[entryNum]); - for (uint64_t i = 0; i < entryNum; ++i) - { - entries[i].table = oid; - entries[i].source = vec[i].dbroot; - entries[i].partition = vec[i].partition; - entries[i].destination = target; - entries[i].status = RED_TRANS_READY; - } + for (uint64_t i = 0; i < entryNum; ++i) + { + entries[i].table = oid; + entries[i].source = vec[i].dbroot; + entries[i].partition = vec[i].partition; + entries[i].destination = target; + entries[i].status = RED_TRANS_READY; + } - errno = 0; - size_t n = fwrite(entries.get(), sizeof(RedistributePlanEntry), entryNum, fControl->fPlanFilePtr); + errno = 0; + size_t n = fwrite(entries.get(), sizeof(RedistributePlanEntry), entryNum, fControl->fPlanFilePtr); - if (n != entryNum) // need retry - { - int e = errno; - ostringstream oss; - oss << "Failed to write into redistribute.plan: " << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } + if (n != entryNum) // need retry + { + int e = errno; + ostringstream oss; + oss << "Failed to write into redistribute.plan: " << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } - fEntryCount += entryNum; + fEntryCount += entryNum; } void RedistributeControlThread::displayPlan() { - // start from the first entry - try + // start from the first entry + try + { + if (!fControl->fPlanFilePtr) { - if (!fControl->fPlanFilePtr) - { - ostringstream oss; - oss << "No data is schefuled to be moved" << endl; - fControl->logMessage(oss.str()); - return; - } - - rewind(fControl->fPlanFilePtr); - - ByteStream bs; - uint32_t entryId = 0; - long entrySize = sizeof(RedistributePlanEntry); - fControl->logMessage(string("Redistribution Plan:")); - - while (entryId++ < fEntryCount) - { - RedistributePlanEntry entry; - errno = 0; - size_t n = fread(&entry, entrySize, 1, fControl->fPlanFilePtr); - - if (n != 1) - { - int e = errno; - ostringstream oss; - oss << "Failed to read from redistribute.plan: " << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } - - // Print this plan entry - ostringstream oss; - oss << "table oid " << entry.table << " partition " << entry.partition - << " moves from dbroot " << entry.source << " to " << entry.destination << endl; - fControl->logMessage(oss.str()); - } + ostringstream oss; + oss << "No data is schefuled to be moved" << endl; + fControl->logMessage(oss.str()); + return; } - catch (std::exception& e) - { - ostringstream oss; - oss << "exception during display of plan: " << e.what() << endl; - fControl->logMessage(oss.str()); - } - catch (...) - { - ostringstream oss; - oss << "exception during display of plan" << endl; - fControl->logMessage(oss.str()); - } -} -int RedistributeControlThread::executeRedistributePlan() -{ - // update the info with total partitions to move - fControl->setEntryCount(fEntryCount); - - // start from the first entry rewind(fControl->fPlanFilePtr); ByteStream bs; uint32_t entryId = 0; long entrySize = sizeof(RedistributePlanEntry); + fControl->logMessage(string("Redistribution Plan:")); while (entryId++ < fEntryCount) { - try - { - // skip system status check in case no OAM -/* - if (getenv("SKIP_OAM_INIT") == NULL) - { - // make sure system is in active state - bool isActive = false; + RedistributePlanEntry entry; + errno = 0; + size_t n = fread(&entry, entrySize, 1, fControl->fPlanFilePtr); - while (!isActive) - { - bool noExcept = true; - SystemStatus systemstatus; + if (n != 1) + { + int e = errno; + ostringstream oss; + oss << "Failed to read from redistribute.plan: " << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } - try - { - fControl->fOam->getSystemStatus(systemstatus); - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); - noExcept = false; - } - catch (...) - { - noExcept = false; - } - - if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false)) - sleep(1);; - } - } -*/ - - if (fStopAction) - return RED_EC_USER_STOP; - - - RedistributePlanEntry entry; - errno = 0; - size_t n = fread(&entry, entrySize, 1, fControl->fPlanFilePtr); - - if (n != 1) - { - int e = errno; - ostringstream oss; - oss << "Failed to read from redistribute.plan: " << strerror(e) << " (" << e << ")"; - throw runtime_error(oss.str()); - } - - if (entry.status != (int) RED_TRANS_READY) - continue; - - // send the job to source dbroot - size_t headerSize = sizeof(RedistributeMsgHeader); - size_t entrySize = sizeof(RedistributePlanEntry); - RedistributeMsgHeader header(entry.destination, entry.source, entryId, RED_ACTN_REQUEST); - - if (connectToWes(header.source) == 0) - { - bs.restart(); - entry.starttime = time(NULL); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, headerSize); - bs.append((const ByteStream::byte*) &entry, entrySize); - fMsgQueueClient->write(bs); - - SBS sbs = fMsgQueueClient->read(); - entry.status = RED_TRANS_FAILED; - - if (sbs->length() == 0) - { - ostringstream oss; - oss << "Zero byte read, Network error. entryID=" << entryId; - fErrorMsg = oss.str(); - } - else if (sbs->length() < (headerSize + entrySize + 1)) - { - ostringstream oss; - oss << "Short message, length=" << sbs->length() << ". entryID=" << entryId; - fErrorMsg = oss.str(); - } - else - { - ByteStream::byte wesMsgId; - *sbs >> wesMsgId; - // Need check header info - //const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) sbs->buf(); - sbs->advance(headerSize); - const RedistributePlanEntry* e = (const RedistributePlanEntry*) sbs->buf(); - sbs->advance(entrySize); - entry.status = e->status; - entry.endtime = time(NULL); -// if (entry.status == (int32_t) RED_TRANS_FAILED) -// *sbs >> fErrorMsg; - } - - // done with this connection, may consider to reuse. - fMsgQueueClient.reset(); - } - else - { - entry.status = RED_TRANS_FAILED; - ostringstream oss; - oss << "Connect to PM failed." << ". entryID=" << entryId; - fErrorMsg += oss.str(); - } - - if (!fErrorMsg.empty()) - throw runtime_error(fErrorMsg); - - errno = 0; - int rc = fseek(fControl->fPlanFilePtr, -((long)entrySize), SEEK_CUR); - - if (rc != 0) - { - int e = errno; - ostringstream oss; - oss << "fseek is failed: " << strerror(e) << " (" << e << "); entry id=" << entryId; - throw runtime_error(oss.str()); - } - - errno = 0; - n = fwrite(&entry, entrySize, 1, fControl->fPlanFilePtr); - - if (n != 1) // need retry - { - int e = errno; - ostringstream oss; - oss << "Failed to update redistribute.plan: " << strerror(e) << " (" << e - << "); entry id=" << entryId; - throw runtime_error(oss.str()); - } - - fflush(fControl->fPlanFilePtr); - - fControl->updateProgressInfo(entry.status, entry.endtime); - - } - catch (const std::exception& ex) - { - fControl->logMessage(string("got exception when executing plan:") + ex.what()); - } - catch (...) - { - fControl->logMessage("got unknown exception when executing plan."); - } + // Print this plan entry + ostringstream oss; + oss << "table oid " << entry.table << " partition " << entry.partition << " moves from dbroot " + << entry.source << " to " << entry.destination << endl; + fControl->logMessage(oss.str()); } - - return 0; + } + catch (std::exception& e) + { + ostringstream oss; + oss << "exception during display of plan: " << e.what() << endl; + fControl->logMessage(oss.str()); + } + catch (...) + { + ostringstream oss; + oss << "exception during display of plan" << endl; + fControl->logMessage(oss.str()); + } } - -int RedistributeControlThread::connectToWes(int dbroot) +int RedistributeControlThread::executeRedistributePlan() { - int ret = 0; - OamCache::dbRootPMMap_t dbrootToPM = fOamCache->getDBRootToPMMap(); - int pmId = (*dbrootToPM)[dbroot]; - ostringstream oss; - oss << "pm" << pmId << "_WriteEngineServer"; + // update the info with total partitions to move + fControl->setEntryCount(fEntryCount); + // start from the first entry + rewind(fControl->fPlanFilePtr); + + ByteStream bs; + uint32_t entryId = 0; + long entrySize = sizeof(RedistributePlanEntry); + + while (entryId++ < fEntryCount) + { try { - boost::mutex::scoped_lock lock(fActionMutex); - fWesInUse = oss.str(); - fMsgQueueClient.reset(new MessageQueueClient(fWesInUse, fConfig)); + // skip system status check in case no OAM + /* + if (getenv("SKIP_OAM_INIT") == NULL) + { + // make sure system is in active state + bool isActive = false; + + while (!isActive) + { + bool noExcept = true; + SystemStatus systemstatus; + + try + { + fControl->fOam->getSystemStatus(systemstatus); + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); + noExcept = false; + } + catch (...) + { + noExcept = false; + } + + if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false)) + sleep(1);; + } + } + */ + + if (fStopAction) + return RED_EC_USER_STOP; + + RedistributePlanEntry entry; + errno = 0; + size_t n = fread(&entry, entrySize, 1, fControl->fPlanFilePtr); + + if (n != 1) + { + int e = errno; + ostringstream oss; + oss << "Failed to read from redistribute.plan: " << strerror(e) << " (" << e << ")"; + throw runtime_error(oss.str()); + } + + if (entry.status != (int)RED_TRANS_READY) + continue; + + // send the job to source dbroot + size_t headerSize = sizeof(RedistributeMsgHeader); + size_t entrySize = sizeof(RedistributePlanEntry); + RedistributeMsgHeader header(entry.destination, entry.source, entryId, RED_ACTN_REQUEST); + + if (connectToWes(header.source) == 0) + { + bs.restart(); + entry.starttime = time(NULL); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, headerSize); + bs.append((const ByteStream::byte*)&entry, entrySize); + fMsgQueueClient->write(bs); + + SBS sbs = fMsgQueueClient->read(); + entry.status = RED_TRANS_FAILED; + + if (sbs->length() == 0) + { + ostringstream oss; + oss << "Zero byte read, Network error. entryID=" << entryId; + fErrorMsg = oss.str(); + } + else if (sbs->length() < (headerSize + entrySize + 1)) + { + ostringstream oss; + oss << "Short message, length=" << sbs->length() << ". entryID=" << entryId; + fErrorMsg = oss.str(); + } + else + { + ByteStream::byte wesMsgId; + *sbs >> wesMsgId; + // Need check header info + // const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) sbs->buf(); + sbs->advance(headerSize); + const RedistributePlanEntry* e = (const RedistributePlanEntry*)sbs->buf(); + sbs->advance(entrySize); + entry.status = e->status; + entry.endtime = time(NULL); + // if (entry.status == (int32_t) RED_TRANS_FAILED) + // *sbs >> fErrorMsg; + } + + // done with this connection, may consider to reuse. + fMsgQueueClient.reset(); + } + else + { + entry.status = RED_TRANS_FAILED; + ostringstream oss; + oss << "Connect to PM failed." + << ". entryID=" << entryId; + fErrorMsg += oss.str(); + } + + if (!fErrorMsg.empty()) + throw runtime_error(fErrorMsg); + + errno = 0; + int rc = fseek(fControl->fPlanFilePtr, -((long)entrySize), SEEK_CUR); + + if (rc != 0) + { + int e = errno; + ostringstream oss; + oss << "fseek is failed: " << strerror(e) << " (" << e << "); entry id=" << entryId; + throw runtime_error(oss.str()); + } + + errno = 0; + n = fwrite(&entry, entrySize, 1, fControl->fPlanFilePtr); + + if (n != 1) // need retry + { + int e = errno; + ostringstream oss; + oss << "Failed to update redistribute.plan: " << strerror(e) << " (" << e + << "); entry id=" << entryId; + throw runtime_error(oss.str()); + } + + fflush(fControl->fPlanFilePtr); + + fControl->updateProgressInfo(entry.status, entry.endtime); } catch (const std::exception& ex) { - fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- " + ex.what(); - ret = 1; + fControl->logMessage(string("got exception when executing plan:") + ex.what()); } catch (...) { - fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- unknown"; - ret = 2; + fControl->logMessage("got unknown exception when executing plan."); } + } - if (ret != 0) - { - boost::mutex::scoped_lock lock(fActionMutex); - fWesInUse.clear(); - - fMsgQueueClient.reset(); - } - - return ret; + return 0; } +int RedistributeControlThread::connectToWes(int dbroot) +{ + int ret = 0; + OamCache::dbRootPMMap_t dbrootToPM = fOamCache->getDBRootToPMMap(); + int pmId = (*dbrootToPM)[dbroot]; + ostringstream oss; + oss << "pm" << pmId << "_WriteEngineServer"; + + try + { + boost::mutex::scoped_lock lock(fActionMutex); + fWesInUse = oss.str(); + fMsgQueueClient.reset(new MessageQueueClient(fWesInUse, fConfig)); + } + catch (const std::exception& ex) + { + fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- " + ex.what(); + ret = 1; + } + catch (...) + { + fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- unknown"; + ret = 2; + } + + if (ret != 0) + { + boost::mutex::scoped_lock lock(fActionMutex); + fWesInUse.clear(); + + fMsgQueueClient.reset(); + } + + return ret; +} void RedistributeControlThread::doStopAction() { - fConfig = Config::makeConfig(); - fControl = RedistributeControl::instance(); + fConfig = Config::makeConfig(); + fControl = RedistributeControl::instance(); - boost::mutex::scoped_lock lock(fActionMutex); + boost::mutex::scoped_lock lock(fActionMutex); - if (!fWesInUse.empty()) + if (!fWesInUse.empty()) + { + // send the stop message to dbroots + size_t headerSize = sizeof(RedistributeMsgHeader); + RedistributeMsgHeader header(-1, -1, -1, RED_ACTN_STOP); + + try { - // send the stop message to dbroots - size_t headerSize = sizeof(RedistributeMsgHeader); - RedistributeMsgHeader header(-1, -1, -1, RED_ACTN_STOP); + fMsgQueueClient.reset(new MessageQueueClient(fWesInUse, fConfig)); + ByteStream bs; + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, headerSize); + fMsgQueueClient->write(bs); - try - { - fMsgQueueClient.reset(new MessageQueueClient(fWesInUse, fConfig)); - ByteStream bs; - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, headerSize); - fMsgQueueClient->write(bs); - - SBS sbs; - sbs = fMsgQueueClient->read(); - // no retry yet. - } - catch (const std::exception& ex) - { - fErrorMsg = "Caught exception when connecting to " + fWesInUse + " -- " + ex.what(); - } - catch (...) - { - fErrorMsg = "Caught exception when connecting to " + fWesInUse + " -- unknown"; - } + SBS sbs; + sbs = fMsgQueueClient->read(); + // no retry yet. } + catch (const std::exception& ex) + { + fErrorMsg = "Caught exception when connecting to " + fWesInUse + " -- " + ex.what(); + } + catch (...) + { + fErrorMsg = "Caught exception when connecting to " + fWesInUse + " -- unknown"; + } + } - if (!fErrorMsg.empty()) - fControl->logMessage(fErrorMsg + " @controlThread::doStop"); - else - fControl->logMessage("User stop @controlThread::doStop"); + if (!fErrorMsg.empty()) + fControl->logMessage(fErrorMsg + " @controlThread::doStop"); + else + fControl->logMessage("User stop @controlThread::doStop"); - fWesInUse.clear(); - fMsgQueueClient.reset(); + fWesInUse.clear(); + fMsgQueueClient.reset(); } - -} // namespace +} // namespace redistribute // vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrolthread.h b/writeengine/redistribute/we_redistributecontrolthread.h index 9b28fe9d7..8f3116fa7 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.h +++ b/writeengine/redistribute/we_redistributecontrolthread.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributecontrolthread.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistributecontrolthread.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ #ifndef WE_REDISTRIBUTECONTROLTHREAD_H #define WE_REDISTRIBUTECONTROLTHREAD_H @@ -29,7 +29,6 @@ #include "boost/shared_ptr.hpp" #include "boost/thread/mutex.hpp" - // forward reference namespace config { @@ -41,95 +40,92 @@ namespace oam class OamCache; } - namespace messageqcpp { class ByteStream; class IOSocket; -} +} // namespace messageqcpp namespace messagequeue { class MessageQueueClient; } - namespace redistribute { class RedistributeControl; - class RedistributeControlThread { -public: - RedistributeControlThread(uint32_t act); - ~RedistributeControlThread(); + public: + RedistributeControlThread(uint32_t act); + ~RedistributeControlThread(); - int handleJobMsg(RedistributeMsgHeader&, messageqcpp::ByteStream&, messageqcpp::IOSocket&); + int handleJobMsg(RedistributeMsgHeader&, messageqcpp::ByteStream&, messageqcpp::IOSocket&); - void operator()(); + void operator()(); - // used by control to change status - static void setStopAction(bool); + // used by control to change status + static void setStopAction(bool); -private: - // struct for sort partitions - struct PartitionInfo + private: + // struct for sort partitions + struct PartitionInfo + { + int32_t dbroot; + int32_t partition; + + PartitionInfo() : dbroot(0), partition(0) { - int32_t dbroot; - int32_t partition; + } + PartitionInfo(int32_t d, int32_t p) : dbroot(d), partition(p) + { + } - PartitionInfo() : dbroot(0), partition(0) {} - PartitionInfo(int32_t d, int32_t p) : dbroot(d), partition(p) {} + bool operator<(const struct PartitionInfo& rhs) const + { + return ((dbroot < rhs.dbroot) || (dbroot == rhs.dbroot && partition < rhs.partition)); + } - bool operator < (const struct PartitionInfo& rhs) const - { - return ((dbroot < rhs.dbroot) || (dbroot == rhs.dbroot && partition < rhs.partition)); - } + bool operator==(const struct PartitionInfo& rhs) const + { + return (dbroot == rhs.dbroot && partition == rhs.partition); + } + }; - bool operator == (const struct PartitionInfo& rhs) const - { - return (dbroot == rhs.dbroot && partition == rhs.partition); - } + void doRedistribute(); + void doStopAction(); - }; + int setup(); + int makeRedistributePlan(); + int executeRedistributePlan(); - void doRedistribute(); - void doStopAction(); + int connectToWes(int); + void dumpPlanToFile(uint64_t, std::vector&, int); + void displayPlan(); - int setup(); - int makeRedistributePlan(); - int executeRedistributePlan(); + uint32_t fAction; + oam::OamCache* fOamCache; + config::Config* fConfig; + boost::shared_ptr fMsgQueueClient; - int connectToWes(int); - void dumpPlanToFile(uint64_t, std::vector&, int); - void displayPlan(); + std::set fSourceSet; + std::set fTargetSet; + std::set fDbrootSet; // Union of fSourceSet and fTargetSet + int fMaxDbroot; + uint32_t fEntryCount; + std::string fErrorMsg; + int32_t fErrorCode; - uint32_t fAction; - oam::OamCache* fOamCache; - config::Config* fConfig; - boost::shared_ptr fMsgQueueClient; + RedistributeControl* fControl; - std::set fSourceSet; - std::set fTargetSet; - std::set fDbrootSet; // Union of fSourceSet and fTargetSet - int fMaxDbroot; - uint32_t fEntryCount; - std::string fErrorMsg; - int32_t fErrorCode; - - RedistributeControl* fControl; - - static boost::mutex fActionMutex; - static volatile bool fStopAction; - static std::string fWesInUse; + static boost::mutex fActionMutex; + static volatile bool fStopAction; + static std::string fWesInUse; }; - -} // namespace - +} // namespace redistribute #endif // WE_REDISTRIBUTECONTROLTHREAD_H // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistributedef.h b/writeengine/redistribute/we_redistributedef.h index cfc98ec91..fbdb7c954 100644 --- a/writeengine/redistribute/we_redistributedef.h +++ b/writeengine/redistribute/we_redistributedef.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributedef.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistributedef.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ #ifndef WE_REDISTRIBUTEDEF_H #define WE_REDISTRIBUTEDEF_H @@ -27,187 +27,183 @@ namespace redistribute { - // version number for parsing the info and plan files -const uint32_t RED_VERSION_NUM = 1; +const uint32_t RED_VERSION_NUM = 1; // state values -const uint32_t RED_STATE_UNDEF = 0; // indication of failed to retrieve redistribute status. -const uint32_t RED_STATE_IDLE = 1; -const uint32_t RED_STATE_ACTIVE = 2; -const uint32_t RED_STATE_FINISH = 3; +const uint32_t RED_STATE_UNDEF = 0; // indication of failed to retrieve redistribute status. +const uint32_t RED_STATE_IDLE = 1; +const uint32_t RED_STATE_ACTIVE = 2; +const uint32_t RED_STATE_FINISH = 3; const uint32_t RED_STATE_STOPPED = 4; -const uint32_t RED_STATE_FAILED = 5; -const uint32_t RED_STATE_INCOMP = 6; +const uint32_t RED_STATE_FAILED = 5; +const uint32_t RED_STATE_INCOMP = 6; // partition transfer state -const uint32_t RED_TRANS_READY = 20; +const uint32_t RED_TRANS_READY = 20; const uint32_t RED_TRANS_SUCCESS = 21; const uint32_t RED_TRANS_SKIPPED = 22; -const uint32_t RED_TRANS_FAILED = 23; - +const uint32_t RED_TRANS_FAILED = 23; // return code to shell or between redistribute entities enum RED_ErrorCode { - RED_EC_OK, + RED_EC_OK, - // return code for util, for shell script to check exit status value is [0, 255] - RED_EC_HELP, - RED_EC_INVALID_OPTION, - RED_EC_EXTRA_OPERAND, - RED_EC_INVALID_DBROOTID, - RED_EC_GET_DBROOT1, - RED_EC_CONNECT_FAIL, - RED_EC_GET_DBROOT_EXCEPT, - RED_EC_NETWORK_FAIL, - RED_EC_PROTOCOL_ERROR, - RED_EC_NO_ARC_NAME, - RED_EC_ARC_NOT_EXIST, - RED_EC_READ_ARC_FAIL, + // return code for util, for shell script to check exit status value is [0, 255] + RED_EC_HELP, + RED_EC_INVALID_OPTION, + RED_EC_EXTRA_OPERAND, + RED_EC_INVALID_DBROOTID, + RED_EC_GET_DBROOT1, + RED_EC_CONNECT_FAIL, + RED_EC_GET_DBROOT_EXCEPT, + RED_EC_NETWORK_FAIL, + RED_EC_PROTOCOL_ERROR, + RED_EC_NO_ARC_NAME, + RED_EC_ARC_NOT_EXIST, + RED_EC_READ_ARC_FAIL, - // return code for control and workers - RED_EC_CNTL_SETUP_FAIL, - RED_EC_MAKEPLAN_FAIL, - RED_EC_EXECUTE_FAIL, - RED_EC_UPDATE_STATE, - RED_EC_USER_STOP, + // return code for control and workers + RED_EC_CNTL_SETUP_FAIL, + RED_EC_MAKEPLAN_FAIL, + RED_EC_EXECUTE_FAIL, + RED_EC_UPDATE_STATE, + RED_EC_USER_STOP, - RED_EC_WKR_SETUP_FAIL, - RED_EC_WKR_MSG_SHORT, - RED_EC_FILE_LIST_FAIL, - RED_EC_NO_TABLE_LOCK, - RED_EC_IDB_HARD_FAIL, - RED_EC_EXTENT_ERROR, - RED_EC_PART_EXIST_ON_TARGET, - RED_EC_OID_TO_FILENAME, - RED_EC_OPEN_FILE_FAIL, - RED_EC_FSEEK_FAIL, - RED_EC_FREAD_FAIL, - RED_EC_FWRITE_FAIL, - RED_EC_SIZE_NACK, - RED_EC_COPY_FILE_FAIL, - RED_EC_UPDATE_DBRM_FAIL, - RED_EC_BS_TOO_SHORT, - RED_EC_FILE_SIZE_NOT_MATCH, - RED_EC_UNKNOWN_DATA_MSG, - RED_EC_UNKNOWN_JOB_MSG, + RED_EC_WKR_SETUP_FAIL, + RED_EC_WKR_MSG_SHORT, + RED_EC_FILE_LIST_FAIL, + RED_EC_NO_TABLE_LOCK, + RED_EC_IDB_HARD_FAIL, + RED_EC_EXTENT_ERROR, + RED_EC_PART_EXIST_ON_TARGET, + RED_EC_OID_TO_FILENAME, + RED_EC_OPEN_FILE_FAIL, + RED_EC_FSEEK_FAIL, + RED_EC_FREAD_FAIL, + RED_EC_FWRITE_FAIL, + RED_EC_SIZE_NACK, + RED_EC_COPY_FILE_FAIL, + RED_EC_UPDATE_DBRM_FAIL, + RED_EC_BS_TOO_SHORT, + RED_EC_FILE_SIZE_NOT_MATCH, + RED_EC_UNKNOWN_DATA_MSG, + RED_EC_UNKNOWN_JOB_MSG, }; - // redistribute message ID -const uint32_t RED_CNTL_START = 1; -const uint32_t RED_CNTL_STATUS = 2; -const uint32_t RED_CNTL_STOP = 3; -const uint32_t RED_CNTL_CLEAR = 4; -const uint32_t RED_CNTL_RESP = 5; -const uint32_t RED_CNTL_VIEW = 6; +const uint32_t RED_CNTL_START = 1; +const uint32_t RED_CNTL_STATUS = 2; +const uint32_t RED_CNTL_STOP = 3; +const uint32_t RED_CNTL_CLEAR = 4; +const uint32_t RED_CNTL_RESP = 5; +const uint32_t RED_CNTL_VIEW = 6; const uint32_t RED_ACTN_REQUEST = 21; -const uint32_t RED_ACTN_STOP = 22; -const uint32_t RED_ACTN_RESP = 23; -const uint32_t RED_ACTN_REPORT = 24; +const uint32_t RED_ACTN_STOP = 22; +const uint32_t RED_ACTN_RESP = 23; +const uint32_t RED_ACTN_REPORT = 24; -const uint32_t RED_DATA_INIT = 51; -const uint32_t RED_DATA_START = 52; -const uint32_t RED_DATA_CONT = 53; -const uint32_t RED_DATA_FINISH = 54; -const uint32_t RED_DATA_COMMIT = 55; -const uint32_t RED_DATA_ABORT = 56; -const uint32_t RED_DATA_ACK = 57; +const uint32_t RED_DATA_INIT = 51; +const uint32_t RED_DATA_START = 52; +const uint32_t RED_DATA_CONT = 53; +const uint32_t RED_DATA_FINISH = 54; +const uint32_t RED_DATA_COMMIT = 55; +const uint32_t RED_DATA_ABORT = 56; +const uint32_t RED_DATA_ACK = 57; // options for start message -const uint32_t RED_OPTN_REMOVE = 0x00000001; - - +const uint32_t RED_OPTN_REMOVE = 0x00000001; // file transfer chunk size const size_t CHUNK_SIZE = 1024 * 1024; const size_t PRE_ALLOC_SIZE = 4 * 1024; - - // redistribute message header struct RedistributeMsgHeader { - uint32_t destination; - uint32_t source; - uint32_t sequenceNum; - uint32_t messageId; + uint32_t destination; + uint32_t source; + uint32_t sequenceNum; + uint32_t messageId; - RedistributeMsgHeader(uint32_t d = 0, uint32_t s = 0, uint32_t n = 0, uint32_t i = 0) : - destination(d), source(s), sequenceNum(n), messageId(i) {}; + RedistributeMsgHeader(uint32_t d = 0, uint32_t s = 0, uint32_t n = 0, uint32_t i = 0) + : destination(d), source(s), sequenceNum(n), messageId(i){}; }; - // redistribute data transfer control block struct RedistributeDataControl { - uint64_t oid; - uint16_t dbroot; - uint32_t partition; - uint16_t segment; - uint64_t size; + uint64_t oid; + uint16_t dbroot; + uint32_t partition; + uint16_t segment; + uint64_t size; - RedistributeDataControl(uint64_t o = 0, uint16_t d = 0, uint32_t p = 0, uint16_t s = 0, uint32_t z = 0) : - oid(o), dbroot(d), partition(p), segment(s), size(z) {}; + RedistributeDataControl(uint64_t o = 0, uint16_t d = 0, uint32_t p = 0, uint16_t s = 0, uint32_t z = 0) + : oid(o), dbroot(d), partition(p), segment(s), size(z){}; }; - // extent entry struct RedistributeExtentEntry { - int64_t oid; - int16_t dbroot; - int32_t partition; - int16_t segment; - int64_t lbid; - int64_t range; - RedistributeExtentEntry() : - oid(0), dbroot(0), partition(0), segment(0), lbid(0), range(0) {} + int64_t oid; + int16_t dbroot; + int32_t partition; + int16_t segment; + int64_t lbid; + int64_t range; + RedistributeExtentEntry() : oid(0), dbroot(0), partition(0), segment(0), lbid(0), range(0) + { + } }; - // RedistributePlanEntry struct RedistributePlanEntry { - int64_t table; - int32_t source; - int32_t partition; - int32_t destination; - int32_t status; - time_t starttime; - time_t endtime; + int64_t table; + int32_t source; + int32_t partition; + int32_t destination; + int32_t status; + time_t starttime; + time_t endtime; - RedistributePlanEntry() : - table(0), source(0), partition(0), destination(0), status(0), starttime(0), endtime(0) {} + RedistributePlanEntry() + : table(0), source(0), partition(0), destination(0), status(0), starttime(0), endtime(0) + { + } }; - // RedistributeInfo struct RedistributeInfo { - uint64_t version; - uint64_t state; - uint64_t planned; - uint64_t success; - uint64_t skipped; - uint64_t failed; - time_t startTime; - time_t endTime; + uint64_t version; + uint64_t state; + uint64_t planned; + uint64_t success; + uint64_t skipped; + uint64_t failed; + time_t startTime; + time_t endTime; - RedistributeInfo() : - version(RED_VERSION_NUM), state(RED_STATE_UNDEF), - planned(0), success(0), skipped(0), failed(0), - startTime(0), endTime(0) {} + RedistributeInfo() + : version(RED_VERSION_NUM) + , state(RED_STATE_UNDEF) + , planned(0) + , success(0) + , skipped(0) + , failed(0) + , startTime(0) + , endTime(0) + { + } }; - -} // namespace - +} // namespace redistribute #endif // WE_REDISTRIBUTEDEF_H // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistributeworkerthread.cpp b/writeengine/redistribute/we_redistributeworkerthread.cpp index 37fae4e04..67f1f5866 100644 --- a/writeengine/redistribute/we_redistributeworkerthread.cpp +++ b/writeengine/redistribute/we_redistributeworkerthread.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributeworkerthread.cpp 4646 2013-05-23 20:58:08Z xlou $ -*/ + * $Id: we_redistributeworkerthread.cpp 4646 2013-05-23 20:58:08Z xlou $ + */ #include #include @@ -69,303 +69,288 @@ using namespace idbdatafile; #include "we_redistributecontrol.h" #include "we_redistributeworkerthread.h" - - - namespace redistribute { - // static variables boost::mutex RedistributeWorkerThread::fActionMutex; volatile bool RedistributeWorkerThread::fStopAction = false; volatile bool RedistributeWorkerThread::fCommitted = false; string RedistributeWorkerThread::fWesInUse; - -RedistributeWorkerThread::RedistributeWorkerThread(ByteStream& bs, IOSocket& ios) : - fBs(bs), - fIOSocket(ios), - fTableLockId(0), - fErrorCode(RED_EC_OK), - fNewFilePtr(NULL), - fOldFilePtr(NULL) +RedistributeWorkerThread::RedistributeWorkerThread(ByteStream& bs, IOSocket& ios) + : fBs(bs), fIOSocket(ios), fTableLockId(0), fErrorCode(RED_EC_OK), fNewFilePtr(NULL), fOldFilePtr(NULL) { - fWriteBuffer.reset(new char[CHUNK_SIZE]); + fWriteBuffer.reset(new char[CHUNK_SIZE]); } - RedistributeWorkerThread::~RedistributeWorkerThread() { - boost::mutex::scoped_lock lock(fActionMutex); + boost::mutex::scoped_lock lock(fActionMutex); - if (fNewFilePtr) - closeFile(fNewFilePtr); + if (fNewFilePtr) + closeFile(fNewFilePtr); - if (fOldFilePtr) - closeFile(fOldFilePtr); + if (fOldFilePtr) + closeFile(fOldFilePtr); - // make sure releasing the table lock. - if (fTableLockId > 0) - { - fDbrm->releaseTableLock(fTableLockId); + // make sure releasing the table lock. + if (fTableLockId > 0) + { + fDbrm->releaseTableLock(fTableLockId); - // use the interface, line# replaced with lock id. - logMessage("Releasing table lock in destructor. ", fTableLockId); - } + // use the interface, line# replaced with lock id. + logMessage("Releasing table lock in destructor. ", fTableLockId); + } } - void RedistributeWorkerThread::operator()() { - memcpy(&fMsgHeader, fBs.buf(), sizeof(RedistributeMsgHeader)); - fBs.advance(sizeof(RedistributeMsgHeader)); + memcpy(&fMsgHeader, fBs.buf(), sizeof(RedistributeMsgHeader)); + fBs.advance(sizeof(RedistributeMsgHeader)); - if (fMsgHeader.messageId == RED_ACTN_REQUEST) - handleRequest(); - else if (fMsgHeader.messageId == RED_ACTN_STOP) - handleStop(); - else if (fMsgHeader.messageId == RED_DATA_INIT) - handleData(); - else - handleUnknowJobMsg(); + if (fMsgHeader.messageId == RED_ACTN_REQUEST) + handleRequest(); + else if (fMsgHeader.messageId == RED_ACTN_STOP) + handleStop(); + else if (fMsgHeader.messageId == RED_DATA_INIT) + handleData(); + else + handleUnknowJobMsg(); } - void RedistributeWorkerThread::handleRequest() { - try + try + { + // clear stop flag if ever set. { - // clear stop flag if ever set. - { - boost::mutex::scoped_lock lock(fActionMutex); - fStopAction = false; - fCommitted = false; - } + boost::mutex::scoped_lock lock(fActionMutex); + fStopAction = false; + fCommitted = false; + } - if (setup() == 0) + if (setup() == 0) + { + if (fBs.length() >= sizeof(RedistributePlanEntry)) + { + memcpy(&fPlanEntry, fBs.buf(), sizeof(RedistributePlanEntry)); + fBs.advance(sizeof(RedistributePlanEntry)); + OamCache::dbRootPMMap_t dbrootToPM = fOamCache->getDBRootToPMMap(); + fMyId.first = fPlanEntry.source; + fMyId.second = (*dbrootToPM)[fMyId.first]; + fPeerId.first = fPlanEntry.destination; + fPeerId.second = (*dbrootToPM)[fPeerId.first]; + + if (grabTableLock() == 0) { - if (fBs.length() >= sizeof(RedistributePlanEntry)) + // workaround extentmap slow update + sleep(1); + + // build segment & entry list after grabbing the table lock. + if (buildEntryList() == 0) + { + if (sendData() == 0) { - memcpy(&fPlanEntry, fBs.buf(), sizeof(RedistributePlanEntry)); - fBs.advance(sizeof(RedistributePlanEntry)); - OamCache::dbRootPMMap_t dbrootToPM = fOamCache->getDBRootToPMMap(); - fMyId.first = fPlanEntry.source; - fMyId.second = (*dbrootToPM)[fMyId.first]; - fPeerId.first = fPlanEntry.destination; - fPeerId.second = (*dbrootToPM)[fPeerId.first]; - - if (grabTableLock() == 0) - { - // workaround extentmap slow update - sleep(1); - - // build segment & entry list after grabbing the table lock. - if (buildEntryList() == 0) - { - if (sendData() == 0) - { - // do bulk update - updateDbrm(); - } - } - - // conversation to peer after got table lock - // confirm commit or abort - confirmToPeer(); - } + // do bulk update + updateDbrm(); } + } + + // conversation to peer after got table lock + // confirm commit or abort + confirmToPeer(); } + } } - catch (const std::exception&) - { - } - catch (...) - { - } + } + catch (const std::exception&) + { + } + catch (...) + { + } - sendResponse(RED_ACTN_REQUEST); + sendResponse(RED_ACTN_REQUEST); - boost::mutex::scoped_lock lock(fActionMutex); - fWesInUse.clear(); - fMsgQueueClient.reset(); + boost::mutex::scoped_lock lock(fActionMutex); + fWesInUse.clear(); + fMsgQueueClient.reset(); - fStopAction = false; - fCommitted = false; + fStopAction = false; + fCommitted = false; } - int RedistributeWorkerThread::setup() { - int ret = 0; + int ret = 0; - try - { - fConfig = Config::makeConfig(); - fOamCache = oam::OamCache::makeOamCache(); - fDbrm = RedistributeControl::instance()->fDbrm; + try + { + fConfig = Config::makeConfig(); + fOamCache = oam::OamCache::makeOamCache(); + fDbrm = RedistributeControl::instance()->fDbrm; - // for segment file # workaround - // string tmp = fConfig->getConfig("ExtentMap", "FilesPerColumnPartition"); - // int filesPerPartition = fConfig->fromText(tmp); - // if (filesPerPartition == 0) - // filesPerPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; - // int dbrootNum = fOamCache->getDBRootNums().size(); - // if (dbrootNum == 0) - // { - // fErrorMsg = "OamCache->getDBRootNums() failed."; - // logMessage(fErrorMsg, __LINE__); - // return 1; - // } - // if ((filesPerPartition % dbrootNum) != 0) - // { - // fErrorMsg = "ExtentMap::FilesPerColumnPartition is not a multiple of db root number."; - // logMessage(fErrorMsg, __LINE__); - // return 1; - // } - // fSegPerRoot = filesPerPartition / dbrootNum; - } - catch (const std::exception&) - { - ret = 2; - } - catch (...) - { - ret = 2; - } + // for segment file # workaround + // string tmp = fConfig->getConfig("ExtentMap", "FilesPerColumnPartition"); + // int filesPerPartition = fConfig->fromText(tmp); + // if (filesPerPartition == 0) + // filesPerPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; + // int dbrootNum = fOamCache->getDBRootNums().size(); + // if (dbrootNum == 0) + // { + // fErrorMsg = "OamCache->getDBRootNums() failed."; + // logMessage(fErrorMsg, __LINE__); + // return 1; + // } + // if ((filesPerPartition % dbrootNum) != 0) + // { + // fErrorMsg = "ExtentMap::FilesPerColumnPartition is not a multiple of db root number."; + // logMessage(fErrorMsg, __LINE__); + // return 1; + // } + // fSegPerRoot = filesPerPartition / dbrootNum; + } + catch (const std::exception&) + { + ret = 2; + } + catch (...) + { + ret = 2; + } - return ret; + return ret; } - int RedistributeWorkerThread::grabTableLock() { - fTableLockId = 0; + fTableLockId = 0; - try + try + { + vector pms; + pms.push_back(fMyId.second); + + if (fMyId.second != fPeerId.second) + pms.push_back(fPeerId.second); + + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 100 * 1000000; + + while (fTableLockId == 0 && !fStopAction) { - vector pms; - pms.push_back(fMyId.second); + // make sure it's not stopped. + if (fStopAction) + return RED_EC_USER_STOP; - if (fMyId.second != fPeerId.second) - pms.push_back(fPeerId.second); - - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 100 * 1000000; - - while (fTableLockId == 0 && !fStopAction) - { - // make sure it's not stopped. - if (fStopAction) - return RED_EC_USER_STOP; - - // always wait long enough for ddl/dml/cpimport to get table lock - // for now, triple the ddl/dml/cpimport retry interval: 3 * 100ms + // always wait long enough for ddl/dml/cpimport to get table lock + // for now, triple the ddl/dml/cpimport retry interval: 3 * 100ms #ifdef _MSC_VER - Sleep(ts.tv_nsec * 1000); + Sleep(ts.tv_nsec * 1000); #else - struct timespec tmp = ts; + struct timespec tmp = ts; - while (nanosleep(&tmp, &ts) < 0); + while (nanosleep(&tmp, &ts) < 0) + ; - tmp = ts; + tmp = ts; #endif - try - { - uint32_t processID = ::getpid(); - int32_t txnId = 0; - int32_t sessionId = 0; - string processName = "WriteEngineServer"; - fTableLockId = fDbrm->getTableLock(pms, fPlanEntry.table, - &processName, &processID, &sessionId, &txnId, BRM::LOADING ); - } - catch (const std::exception& ex) - { - fErrorCode = RED_EC_IDB_HARD_FAIL; - logMessage(string("getTableLock exception") + ex.what(), __LINE__); - } - catch (...) - { - fErrorCode = RED_EC_IDB_HARD_FAIL; - logMessage("getTableLock exception", __LINE__); + try + { + uint32_t processID = ::getpid(); + int32_t txnId = 0; + int32_t sessionId = 0; + string processName = "WriteEngineServer"; + fTableLockId = fDbrm->getTableLock(pms, fPlanEntry.table, &processName, &processID, &sessionId, + &txnId, BRM::LOADING); + } + catch (const std::exception& ex) + { + fErrorCode = RED_EC_IDB_HARD_FAIL; + logMessage(string("getTableLock exception") + ex.what(), __LINE__); + } + catch (...) + { + fErrorCode = RED_EC_IDB_HARD_FAIL; + logMessage("getTableLock exception", __LINE__); - // no need to throw - // throw IDBExcept(ERR_HARD_FAILURE); - } - } + // no need to throw + // throw IDBExcept(ERR_HARD_FAILURE); + } } - catch (const std::exception& ex) - { - // use the interface, line# replaced with lock id. - logMessage(string(ex.what()) + " when try to get table lock: ", fTableLockId); - } - catch (...) - { - // use the interface, line# replaced with lock id. - logMessage("Unknown exception when try to get table lock: ", fTableLockId); - } - + } + catch (const std::exception& ex) + { // use the interface, line# replaced with lock id. - logMessage("Got table lock: ", fTableLockId); + logMessage(string(ex.what()) + " when try to get table lock: ", fTableLockId); + } + catch (...) + { + // use the interface, line# replaced with lock id. + logMessage("Unknown exception when try to get table lock: ", fTableLockId); + } - return ((fTableLockId > 0) ? 0 : -1); + // use the interface, line# replaced with lock id. + logMessage("Got table lock: ", fTableLockId); + + return ((fTableLockId > 0) ? 0 : -1); } - int RedistributeWorkerThread::buildEntryList() { - int ret = 0; + int ret = 0; - try + try + { + // get all column oids + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); + const CalpontSystemCatalog::TableName table = csc->tableName(fPlanEntry.table); + CalpontSystemCatalog::RIDList cols = csc->columnRIDs(table, true); + + for (CalpontSystemCatalog::RIDList::iterator i = cols.begin(); i != cols.end(); i++) + fOids.push_back(i->objnum); + + CalpontSystemCatalog::DictOIDList dicts = csc->dictOIDs(table); + + for (CalpontSystemCatalog::DictOIDList::iterator i = dicts.begin(); i != dicts.end(); i++) + fOids.push_back(i->dictOID); + + bool firstOid = true; // for adding segments, all columns have the same lay out. + uint16_t source = fPlanEntry.source; + uint16_t target = fPlanEntry.destination; + uint16_t partition = fPlanEntry.partition; + uint32_t minWidth = 8; // column width greater than 8 will be dictionary. + + for (vector::iterator i = fOids.begin(); i != fOids.end(); i++) { - // get all column oids - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0); - const CalpontSystemCatalog::TableName table = csc->tableName(fPlanEntry.table); - CalpontSystemCatalog::RIDList cols = csc->columnRIDs(table, true); + vector entries; + int rc = fDbrm->getExtents(*i, entries, false, false, true); - for (CalpontSystemCatalog::RIDList::iterator i = cols.begin(); i != cols.end(); i++) - fOids.push_back(i->objnum); + if (rc != 0 || entries.size() == 0) + { + ostringstream oss; + oss << "Error in DBRM getExtents; oid:" << *i << "; returnCode: " << rc; + throw runtime_error(oss.str()); + } - CalpontSystemCatalog::DictOIDList dicts = csc->dictOIDs(table); + // same oid has the same column width + uint32_t colWid = entries.front().colWid; + vector::iterator targetHwmEntry = entries.end(); // for HWM_0 workaround - for (CalpontSystemCatalog::DictOIDList::iterator i = dicts.begin(); i != dicts.end(); i++) - fOids.push_back(i->dictOID); + if (colWid > 0 && colWid < minWidth) + minWidth = colWid; - bool firstOid = true; // for adding segments, all columns have the same lay out. - uint16_t source = fPlanEntry.source; - uint16_t target = fPlanEntry.destination; - uint16_t partition = fPlanEntry.partition; - uint32_t minWidth = 8; // column width greater than 8 will be dictionary. - - for (vector::iterator i = fOids.begin(); i != fOids.end(); i++) + for (vector::iterator j = entries.begin(); j != entries.end(); j++) + { + if (j->dbRoot == source && j->partitionNum == partition) { - vector entries; - int rc = fDbrm->getExtents(*i, entries, false, false, true); + fUpdateRtEntries.push_back(BulkUpdateDBRootArg(j->range.start, target)); - if (rc != 0 || entries.size() == 0) - { - ostringstream oss; - oss << "Error in DBRM getExtents; oid:" << *i << "; returnCode: " << rc; - throw runtime_error(oss.str()); - } - - // same oid has the same column width - uint32_t colWid = entries.front().colWid; - vector::iterator targetHwmEntry = entries.end(); // for HWM_0 workaround - - if (colWid > 0 && colWid < minWidth) - minWidth = colWid; - - for (vector::iterator j = entries.begin(); j != entries.end(); j++) - { - if (j->dbRoot == source && j->partitionNum == partition) - { - fUpdateRtEntries.push_back(BulkUpdateDBRootArg(j->range.start, target)); - - if (firstOid) - fSegments.insert(j->segmentNum); - } + if (firstOid) + fSegments.insert(j->segmentNum); + } #if 0 else if (j->dbRoot == target && j->partitionNum == partition) @@ -382,450 +367,428 @@ int RedistributeWorkerThread::buildEntryList() #endif - // workaround for HWM_0 of highest extents of the oid on target dbroot. - if (j->dbRoot == target) - { - if (targetHwmEntry == entries.end()) - { - targetHwmEntry = j; - } - else - { - if (j->partitionNum > targetHwmEntry->partitionNum) - { - targetHwmEntry = j; - } - else if (j->partitionNum == targetHwmEntry->partitionNum && - j->blockOffset > targetHwmEntry->blockOffset) - { - targetHwmEntry = j; - } - else if (j->partitionNum == targetHwmEntry->partitionNum && - j->blockOffset == targetHwmEntry->blockOffset && - j->segmentNum > targetHwmEntry->segmentNum) - { - targetHwmEntry = j; - } - } - } - } // for em entries - - // HWM_0 workaround - // HWM 0 has two possibilities: - // 1. segment file has one extent, and the first block is not full yet. - // 2. segment file has more than one extents, the HWM of the extents other than - // the last extent is set to 0, that is only last extent has none-zero HWM. - // In tuple-bps::makeJob, there is a check to handle last extent has 0 hwm: - // (scannedExtents[i].HWM == 0 && (int) i < lastExtent[scannedExtents[i].dbRoot-1]) - // lbidsToScan = scannedExtents[i].range.size * 1024; - // Based on this check, the number of block to scan is caculated. - // After redistributing the partitions, the original case 1 extent on destination - // may not be the highest extent in the dbroot, and result in a full extent scan. - // This scan will fail because there is no enough blocks if this is an abbreviated - // extent or not enough chunks if the column is compressed. - // The workaround is to bump up the HWM to 1 if moved in partitions are greater. - if (targetHwmEntry != entries.end() && // exclude no extent case - targetHwmEntry->colWid > 0 && // exclude dictionary - targetHwmEntry->HWM == 0 && - targetHwmEntry->partitionNum < partition) - { - BulkSetHWMArg arg; - arg.oid = *i; - arg.partNum = targetHwmEntry->partitionNum; - arg.segNum = targetHwmEntry->segmentNum; - arg.hwm = targetHwmEntry->colWid; // will correct later based on minWidth - - fUpdateHwmEntries.push_back(arg); - } - } // for oids - - - // HWM_0 workaround - // Caculate the min(column width), the HWM(bump up to) for each column. - if (fUpdateHwmEntries.size() > 0) + // workaround for HWM_0 of highest extents of the oid on target dbroot. + if (j->dbRoot == target) { - // update the HWM based in column width, not include dictionary extents - for (vector::iterator j = fUpdateHwmEntries.begin(); - j != fUpdateHwmEntries.end(); j++) + if (targetHwmEntry == entries.end()) + { + targetHwmEntry = j; + } + else + { + if (j->partitionNum > targetHwmEntry->partitionNum) { - if (j->hwm <= 8) - j->hwm /= minWidth; - else - j->hwm = 1; // not needed, but in case + targetHwmEntry = j; } + else if (j->partitionNum == targetHwmEntry->partitionNum && + j->blockOffset > targetHwmEntry->blockOffset) + { + targetHwmEntry = j; + } + else if (j->partitionNum == targetHwmEntry->partitionNum && + j->blockOffset == targetHwmEntry->blockOffset && + j->segmentNum > targetHwmEntry->segmentNum) + { + targetHwmEntry = j; + } + } } - } - catch (const std::exception& ex) - { - fErrorCode = RED_EC_EXTENT_ERROR; - fErrorMsg = ex.what(); - logMessage(fErrorMsg, __LINE__); - ret = fErrorCode; - } - catch (...) - { - fErrorCode = RED_EC_EXTENT_ERROR; - fErrorMsg = "get extent error."; - logMessage(fErrorMsg, __LINE__); - ret = fErrorCode; - } + } // for em entries - return ret; + // HWM_0 workaround + // HWM 0 has two possibilities: + // 1. segment file has one extent, and the first block is not full yet. + // 2. segment file has more than one extents, the HWM of the extents other than + // the last extent is set to 0, that is only last extent has none-zero HWM. + // In tuple-bps::makeJob, there is a check to handle last extent has 0 hwm: + // (scannedExtents[i].HWM == 0 && (int) i < lastExtent[scannedExtents[i].dbRoot-1]) + // lbidsToScan = scannedExtents[i].range.size * 1024; + // Based on this check, the number of block to scan is caculated. + // After redistributing the partitions, the original case 1 extent on destination + // may not be the highest extent in the dbroot, and result in a full extent scan. + // This scan will fail because there is no enough blocks if this is an abbreviated + // extent or not enough chunks if the column is compressed. + // The workaround is to bump up the HWM to 1 if moved in partitions are greater. + if (targetHwmEntry != entries.end() && // exclude no extent case + targetHwmEntry->colWid > 0 && // exclude dictionary + targetHwmEntry->HWM == 0 && targetHwmEntry->partitionNum < partition) + { + BulkSetHWMArg arg; + arg.oid = *i; + arg.partNum = targetHwmEntry->partitionNum; + arg.segNum = targetHwmEntry->segmentNum; + arg.hwm = targetHwmEntry->colWid; // will correct later based on minWidth + + fUpdateHwmEntries.push_back(arg); + } + } // for oids + + // HWM_0 workaround + // Caculate the min(column width), the HWM(bump up to) for each column. + if (fUpdateHwmEntries.size() > 0) + { + // update the HWM based in column width, not include dictionary extents + for (vector::iterator j = fUpdateHwmEntries.begin(); j != fUpdateHwmEntries.end(); + j++) + { + if (j->hwm <= 8) + j->hwm /= minWidth; + else + j->hwm = 1; // not needed, but in case + } + } + } + catch (const std::exception& ex) + { + fErrorCode = RED_EC_EXTENT_ERROR; + fErrorMsg = ex.what(); + logMessage(fErrorMsg, __LINE__); + ret = fErrorCode; + } + catch (...) + { + fErrorCode = RED_EC_EXTENT_ERROR; + fErrorMsg = "get extent error."; + logMessage(fErrorMsg, __LINE__); + ret = fErrorCode; + } + + return ret; } - int RedistributeWorkerThread::sendData() { - WriteEngine::FileOp fileOp; // just to get filename, not for file operations - bool remotePM = (fMyId.second != fPeerId.second); - uint32_t dbroot = fPlanEntry.source; - uint32_t partition = fPlanEntry.partition; - int16_t source = fPlanEntry.source; - int16_t dest = fPlanEntry.destination; + WriteEngine::FileOp fileOp; // just to get filename, not for file operations + bool remotePM = (fMyId.second != fPeerId.second); + uint32_t dbroot = fPlanEntry.source; + uint32_t partition = fPlanEntry.partition; + int16_t source = fPlanEntry.source; + int16_t dest = fPlanEntry.destination; - IDBDataFile::Types fileType = (IDBPolicy::useHdfs() ? IDBDataFile::HDFS : - IDBPolicy::useCloud() ? IDBDataFile::CLOUD : IDBDataFile::UNBUFFERED); + IDBDataFile::Types fileType = (IDBPolicy::useHdfs() ? IDBDataFile::HDFS + : IDBPolicy::useCloud() ? IDBDataFile::CLOUD + : IDBDataFile::UNBUFFERED); - IDBFileSystem& fs = IDBFileSystem::getFs( fileType ); + IDBFileSystem& fs = IDBFileSystem::getFs(fileType); - if ((remotePM) && (fileType != IDBDataFile::HDFS)) + if ((remotePM) && (fileType != IDBDataFile::HDFS)) + { + if (connectToWes(fPeerId.second) != 0) { - if (connectToWes(fPeerId.second) != 0) + fErrorCode = RED_EC_CONNECT_FAIL; + ostringstream oss; + oss << "Failed to connect to PM" << fPeerId.second << " from PM" << fMyId.second; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + + // start to send each segment file + uint32_t seq = 0; + ByteStream bs; + + // start conversion with peer, hand shaking. + RedistributeMsgHeader header(dest, source, seq++, RED_DATA_INIT); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, sizeof(header)); + fMsgQueueClient->write(bs); + + SBS sbs = fMsgQueueClient->read(); + + if (!checkDataTransferAck(sbs, 0)) + return fErrorCode; + + for (vector::iterator i = fOids.begin(); i != fOids.end(); i++) + { + for (set::iterator j = fSegments.begin(); j != fSegments.end(); ++j) + { + char fileName[WriteEngine::FILE_NAME_SIZE]; + int rc = fileOp.oid2FileName(*i, fileName, false, dbroot, partition, *j); + + if (rc == WriteEngine::NO_ERROR) { - fErrorCode = RED_EC_CONNECT_FAIL; + ostringstream oss; + oss << "<=redistributing: " << fileName << ", oid=" << *i << ", db=" << source + << ", part=" << partition << ", seg=" << *j << " to db=" << dest; + logMessage(oss.str(), __LINE__); + } + else + { + fErrorCode = RED_EC_OID_TO_FILENAME; + ostringstream oss; + oss << "Failed to get file name: oid=" << *i << ", dbroot=" << dbroot << ", partition=" << partition + << ", segment=" << *j; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + + if (fOldFilePtr != NULL) + closeFile(fOldFilePtr); + + errno = 0; + FILE* fOldFilePtr = fopen(fileName, "rb"); + + if (fOldFilePtr != NULL) + { + ostringstream oss; + oss << "open " << fileName << ", oid=" << *i << ", dbroot=" << dbroot << ", partition=" << partition + << ", segment=" << *j << ". " << fOldFilePtr; + logMessage(oss.str(), __LINE__); + } + else + { + int e = errno; + fErrorCode = RED_EC_OPEN_FILE_FAIL; + ostringstream oss; + oss << "Failed to open " << fileName << ", oid=" << *i << ", dbroot=" << dbroot + << ", partition=" << partition << ", segment=" << *j << ". " << strerror(e) << " (" << e << ")"; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + + // add to set for remove after commit + addToDirSet(fileName, true); + + char chunk[CHUNK_SIZE]; + errno = 0; + fseek(fOldFilePtr, 0, SEEK_END); // go to end of file + long fileSize = ftell(fOldFilePtr); // get current file size + + if (fileSize < 0) + { + int e = errno; + ostringstream oss; + oss << "Fail to tell file size: " << strerror(e) << " (" << e << ")"; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_FSEEK_FAIL; + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + + // send start message to have the file of fileSize created at target dbroot. + bs.restart(); + RedistributeMsgHeader header(dest, source, seq++, RED_DATA_START); + RedistributeDataControl dataControl(*i, dest, partition, *j, fileSize); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, sizeof(header)); + bs.append((const ByteStream::byte*)&dataControl, sizeof(dataControl)); + fMsgQueueClient->write(bs); + + sbs = fMsgQueueClient->read(); + + if (!checkDataTransferAck(sbs, fileSize)) + return fErrorCode; + + // now send the file chunk by chunk. + rewind(fOldFilePtr); + int64_t bytesLeft = fileSize; + size_t bytesSend = CHUNK_SIZE; + header.messageId = RED_DATA_CONT; + + while (bytesLeft > 0) + { + if (fStopAction) + { + closeFile(fOldFilePtr); + fOldFilePtr = NULL; + return RED_EC_USER_STOP; + } + + if (bytesLeft < (long)CHUNK_SIZE) + bytesSend = bytesLeft; + + errno = 0; + size_t n = fread(chunk, 1, bytesSend, fOldFilePtr); + + if (n != bytesSend) + { + int e = errno; ostringstream oss; - oss << "Failed to connect to PM" << fPeerId.second << " from PM" << fMyId.second; + oss << "Fail to read: " << strerror(e) << " (" << e << ")"; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_FREAD_FAIL; + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + + header.sequenceNum = seq++; + bs.restart(); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, sizeof(header)); + bs << (size_t)bytesSend; + bs.append((const ByteStream::byte*)chunk, bytesSend); + fMsgQueueClient->write(bs); + + sbs = fMsgQueueClient->read(); + + if (!checkDataTransferAck(sbs, bytesSend)) + return fErrorCode; + + bytesLeft -= bytesSend; + } + + closeFile(fOldFilePtr); + fOldFilePtr = NULL; + + header.messageId = RED_DATA_FINISH; + header.sequenceNum = seq++; + bs.restart(); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, sizeof(header)); + bs << (uint64_t)fileSize; + fMsgQueueClient->write(bs); + + sbs = fMsgQueueClient->read(); + + if (!checkDataTransferAck(sbs, fileSize)) + return fErrorCode; + + } // segments + } // for oids + } // remote peer non-hdfs + else // local or HDFS file copy + { + std::map rootToPathMap; + + // use cp, in case failed in middle. May consider to use rename if possible. + for (vector::iterator i = fOids.begin(); i != fOids.end(); i++) + { + for (set::iterator j = fSegments.begin(); j != fSegments.end(); ++j) + { + if (fStopAction) + return RED_EC_USER_STOP; + + if (fileType == IDBDataFile::HDFS) // HDFS file copy + { + string sourceName; + int rc = buildFullHdfsPath(rootToPathMap, // map of root to path + *i, // OID + source, // dbroot + partition, // partition + *j, // segment + sourceName); // full path name + + if (rc != 0) + { + fErrorCode = RED_EC_OID_TO_FILENAME; + ostringstream oss; + oss << "Failed to get src file name: oid=" << *i << ", dbroot=" << source + << ", partition=" << partition << ", segment=" << *j; fErrorMsg = oss.str(); logMessage(fErrorMsg, __LINE__); return fErrorCode; - } + } - // start to send each segment file - uint32_t seq = 0; - ByteStream bs; + string destName; + rc = buildFullHdfsPath(rootToPathMap, // map of root to path + *i, // OID + dest, // dbroot + partition, // partition + *j, // segment + destName); // full path name - // start conversion with peer, hand shaking. - RedistributeMsgHeader header(dest, source, seq++, RED_DATA_INIT); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, sizeof(header)); - fMsgQueueClient->write(bs); - - SBS sbs = fMsgQueueClient->read(); - - if (!checkDataTransferAck(sbs, 0)) + if (rc != 0) + { + fErrorCode = RED_EC_OID_TO_FILENAME; + ostringstream oss; + oss << "Failed to get dest file name: oid=" << *i << ", dbroot=" << dest + << ", partition=" << partition << ", segment=" << *j; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); return fErrorCode; + } - for (vector::iterator i = fOids.begin(); i != fOids.end(); i++) + ostringstream oss; + oss << "<=redistributing(hdfs): " << sourceName << ", oid=" << *i << ", db=" << source + << ", part=" << partition << ", seg=" << *j << " to db=" << dest; + logMessage(oss.str(), __LINE__); + + // add to set for remove after commit/abort + addToDirSet(sourceName.c_str(), true); + addToDirSet(destName.c_str(), false); + + int ret = fs.copyFile(sourceName.c_str(), destName.c_str()); + + if (ret != 0) + { + fErrorCode = RED_EC_COPY_FILE_FAIL; + ostringstream oss; + oss << "Failed to copy " << sourceName << " to " << destName << "; error is: " << strerror(errno); + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + } + else // local file copy { - for (set::iterator j = fSegments.begin(); j != fSegments.end(); ++j) - { - char fileName[WriteEngine::FILE_NAME_SIZE]; - int rc = fileOp.oid2FileName(*i, fileName, false, dbroot, partition, *j); + char sourceName[WriteEngine::FILE_NAME_SIZE]; + int rc = fileOp.oid2FileName(*i, sourceName, false, source, partition, *j); - if (rc == WriteEngine::NO_ERROR) - { - ostringstream oss; - oss << "<=redistributing: " << fileName << ", oid=" << *i << ", db=" - << source << ", part=" << partition << ", seg=" << *j << " to db=" - << dest; - logMessage(oss.str(), __LINE__); - } - else - { - fErrorCode = RED_EC_OID_TO_FILENAME; - ostringstream oss; - oss << "Failed to get file name: oid=" << *i << ", dbroot=" << dbroot - << ", partition=" << partition << ", segment=" << *j; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } + if (rc != WriteEngine::NO_ERROR) + { + fErrorCode = RED_EC_OID_TO_FILENAME; + ostringstream oss; + oss << "Failed to get file name: oid=" << *i << ", dbroot=" << source + << ", partition=" << partition << ", segment=" << *j; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } - if (fOldFilePtr != NULL) - closeFile(fOldFilePtr); + char destName[WriteEngine::FILE_NAME_SIZE]; + rc = fileOp.oid2FileName(*i, destName, true, dest, partition, *j); - errno = 0; - FILE* fOldFilePtr = fopen(fileName, "rb"); + if (rc != WriteEngine::NO_ERROR) + { + fErrorCode = RED_EC_OID_TO_FILENAME; + ostringstream oss; + oss << "Failed to get file name: oid=" << *i << ", dbroot=" << dest << ", partition=" << partition + << ", segment=" << *j; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } - if (fOldFilePtr != NULL) - { - ostringstream oss; - oss << "open " << fileName << ", oid=" << *i << ", dbroot=" << dbroot - << ", partition=" << partition << ", segment=" << *j - << ". " << fOldFilePtr; - logMessage(oss.str(), __LINE__); - } - else - { - int e = errno; - fErrorCode = RED_EC_OPEN_FILE_FAIL; - ostringstream oss; - oss << "Failed to open " << fileName << ", oid=" << *i << ", dbroot=" << dbroot - << ", partition=" << partition << ", segment=" << *j - << ". " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } + ostringstream oss; + oss << "<=redistributing(copy): " << sourceName << ", oid=" << *i << ", db=" << source + << ", part=" << partition << ", seg=" << *j << " to db=" << dest; + logMessage(oss.str(), __LINE__); - // add to set for remove after commit - addToDirSet(fileName, true); + // add to set for remove after commit/abort + addToDirSet(sourceName, true); + addToDirSet(destName, false); - char chunk[CHUNK_SIZE]; - errno = 0; - fseek(fOldFilePtr, 0, SEEK_END); // go to end of file - long fileSize = ftell(fOldFilePtr); // get current file size - - if (fileSize < 0) - { - int e = errno; - ostringstream oss; - oss << "Fail to tell file size: " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_FSEEK_FAIL; - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - - // send start message to have the file of fileSize created at target dbroot. - bs.restart(); - RedistributeMsgHeader header(dest, source, seq++, RED_DATA_START); - RedistributeDataControl dataControl(*i, dest, partition, *j, fileSize); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, sizeof(header)); - bs.append((const ByteStream::byte*) &dataControl, sizeof(dataControl)); - fMsgQueueClient->write(bs); - - sbs = fMsgQueueClient->read(); - - if (!checkDataTransferAck(sbs, fileSize)) - return fErrorCode; - - // now send the file chunk by chunk. - rewind(fOldFilePtr); - int64_t bytesLeft = fileSize; - size_t bytesSend = CHUNK_SIZE; - header.messageId = RED_DATA_CONT; - - while (bytesLeft > 0) - { - if (fStopAction) - { - closeFile(fOldFilePtr); - fOldFilePtr = NULL; - return RED_EC_USER_STOP; - } - - if (bytesLeft < (long) CHUNK_SIZE) - bytesSend = bytesLeft; - - errno = 0; - size_t n = fread(chunk, 1, bytesSend, fOldFilePtr); - - if (n != bytesSend) - { - int e = errno; - ostringstream oss; - oss << "Fail to read: " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_FREAD_FAIL; - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - - header.sequenceNum = seq++; - bs.restart(); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, sizeof(header)); - bs << (size_t) bytesSend; - bs.append((const ByteStream::byte*) chunk, bytesSend); - fMsgQueueClient->write(bs); - - sbs = fMsgQueueClient->read(); - - if (!checkDataTransferAck(sbs, bytesSend)) - return fErrorCode; - - bytesLeft -= bytesSend; - } - - closeFile(fOldFilePtr); - fOldFilePtr = NULL; - - header.messageId = RED_DATA_FINISH; - header.sequenceNum = seq++; - bs.restart(); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, sizeof(header)); - bs << (uint64_t) fileSize; - fMsgQueueClient->write(bs); - - sbs = fMsgQueueClient->read(); - - if (!checkDataTransferAck(sbs, fileSize)) - return fErrorCode; - - } // segments - } // for oids - } // remote peer non-hdfs - else // local or HDFS file copy - { - std::map rootToPathMap; - - // use cp, in case failed in middle. May consider to use rename if possible. - for (vector::iterator i = fOids.begin(); i != fOids.end(); i++) - { - for (set::iterator j = fSegments.begin(); j != fSegments.end(); ++j) - { - if (fStopAction) - return RED_EC_USER_STOP; - - if (fileType == IDBDataFile::HDFS) // HDFS file copy - { - string sourceName; - int rc = buildFullHdfsPath( - rootToPathMap, // map of root to path - *i, // OID - source, // dbroot - partition, // partition - *j, // segment - sourceName ); // full path name - - if (rc != 0) - { - fErrorCode = RED_EC_OID_TO_FILENAME; - ostringstream oss; - oss << "Failed to get src file name: oid=" << *i - << ", dbroot=" << source - << ", partition=" << partition - << ", segment=" << *j; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - - string destName; - rc = buildFullHdfsPath( - rootToPathMap, // map of root to path - *i, // OID - dest, // dbroot - partition, // partition - *j, // segment - destName ); // full path name - - if (rc != 0) - { - fErrorCode = RED_EC_OID_TO_FILENAME; - ostringstream oss; - oss << "Failed to get dest file name: oid=" << *i - << ", dbroot=" << dest - << ", partition=" << partition - << ", segment=" << *j; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - - ostringstream oss; - oss << "<=redistributing(hdfs): " << sourceName << ", oid=" - << *i << ", db=" << source << ", part=" << partition - << ", seg=" << *j << " to db=" << dest; - logMessage(oss.str(), __LINE__); - - // add to set for remove after commit/abort - addToDirSet(sourceName.c_str(), true); - addToDirSet(destName.c_str(), false); - - int ret = fs.copyFile(sourceName.c_str(), destName.c_str()); - - if (ret != 0) - { - fErrorCode = RED_EC_COPY_FILE_FAIL; - ostringstream oss; - oss << "Failed to copy " << sourceName << " to " << - destName << "; error is: " << strerror(errno); - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - } - else // local file copy - { - char sourceName[WriteEngine::FILE_NAME_SIZE]; - int rc = fileOp.oid2FileName(*i, sourceName, false, source, - partition, *j); - - if (rc != WriteEngine::NO_ERROR) - { - fErrorCode = RED_EC_OID_TO_FILENAME; - ostringstream oss; - oss << "Failed to get file name: oid=" << *i - << ", dbroot=" << source - << ", partition=" << partition - << ", segment=" << *j; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - - char destName[WriteEngine::FILE_NAME_SIZE]; - rc = fileOp.oid2FileName(*i, destName, true, - dest, partition, *j); - - if (rc != WriteEngine::NO_ERROR) - { - fErrorCode = RED_EC_OID_TO_FILENAME; - ostringstream oss; - oss << "Failed to get file name: oid=" << *i - << ", dbroot=" << dest - << ", partition=" << partition - << ", segment=" << *j; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - - ostringstream oss; - oss << "<=redistributing(copy): " << sourceName << ", oid=" - << *i << ", db=" << source << ", part=" << partition - << ", seg=" << *j << " to db=" << dest; - logMessage(oss.str(), __LINE__); - - // add to set for remove after commit/abort - addToDirSet(sourceName, true); - addToDirSet(destName, false); - - // Using boost::copy_file() instead of IDBFileSystem::copy- - // File() so we can capture/report any boost exception error - // msg that IDBFileSystem::copyFile() currently swallows. - try - { - filesystem::copy_file(sourceName, destName); - } + // Using boost::copy_file() instead of IDBFileSystem::copy- + // File() so we can capture/report any boost exception error + // msg that IDBFileSystem::copyFile() currently swallows. + try + { + filesystem::copy_file(sourceName, destName); + } #if BOOST_VERSION >= 105200 - catch (filesystem::filesystem_error& e) + catch (filesystem::filesystem_error& e) #else - catch (filesystem::basic_filesystem_error& e) + catch (filesystem::basic_filesystem_error& e) #endif - { - fErrorCode = RED_EC_COPY_FILE_FAIL; - ostringstream oss; - oss << "Failed to copy " << sourceName << " to " << - destName << "; error is: " << e.what(); - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - return fErrorCode; - } - } - } // segment - } // oid - } // !remote + { + fErrorCode = RED_EC_COPY_FILE_FAIL; + ostringstream oss; + oss << "Failed to copy " << sourceName << " to " << destName << "; error is: " << e.what(); + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + return fErrorCode; + } + } + } // segment + } // oid + } // !remote - return 0; + return 0; } - //------------------------------------------------------------------------------ // Construct a full path name based on the given oid, root, partition, and seg. // The rootToPathMap is the map of dbroot to dbrootPath that we are using. We @@ -834,768 +797,727 @@ int RedistributeWorkerThread::sendData() // an HDFS copy, we will be copying files from/to DBRoots that are not on the // local PM. //------------------------------------------------------------------------------ -int RedistributeWorkerThread::buildFullHdfsPath( - std::map& rootToPathMap, - int64_t colOid, - int16_t dbRoot, - uint32_t partition, - int16_t segment, - std::string& fullFileName) +int RedistributeWorkerThread::buildFullHdfsPath(std::map& rootToPathMap, int64_t colOid, + int16_t dbRoot, uint32_t partition, int16_t segment, + std::string& fullFileName) { - std::map::const_iterator iter = rootToPathMap.find(dbRoot); + std::map::const_iterator iter = rootToPathMap.find(dbRoot); - if (iter == rootToPathMap.end()) - { - ostringstream oss; - oss << "DBRoot" << dbRoot; - std::string dbRootPath = fConfig->getConfig("SystemConfig", oss.str()); - - if (dbRootPath.empty()) - { - return 1; - } - - rootToPathMap[ dbRoot ] = dbRootPath; - iter = rootToPathMap.find( dbRoot ); - } - - char tempFileName[WriteEngine::FILE_NAME_SIZE]; - char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; - - int rc = WriteEngine::Convertor::oid2FileName( - colOid, tempFileName, dbDir, partition, segment ); - - if (rc != WriteEngine::NO_ERROR) - { - return 2; - } - - ostringstream fullFileNameOss; - fullFileNameOss << iter->second << '/' << tempFileName; - fullFileName = fullFileNameOss.str(); - - return 0; -} - - -int RedistributeWorkerThread::connectToWes(int pmId) -{ - int ret = 0; + if (iter == rootToPathMap.end()) + { ostringstream oss; - oss << "pm" << pmId << "_WriteEngineServer"; + oss << "DBRoot" << dbRoot; + std::string dbRootPath = fConfig->getConfig("SystemConfig", oss.str()); - try + if (dbRootPath.empty()) { - fMsgQueueClient.reset(new MessageQueueClient(oss.str(), fConfig)); - } - catch (const std::exception& ex) - { - fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- " + ex.what(); - ret = 1; - } - catch (...) - { - fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- unknown"; - ret = 2; + return 1; } - return ret; + rootToPathMap[dbRoot] = dbRootPath; + iter = rootToPathMap.find(dbRoot); + } + + char tempFileName[WriteEngine::FILE_NAME_SIZE]; + char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; + + int rc = WriteEngine::Convertor::oid2FileName(colOid, tempFileName, dbDir, partition, segment); + + if (rc != WriteEngine::NO_ERROR) + { + return 2; + } + + ostringstream fullFileNameOss; + fullFileNameOss << iter->second << '/' << tempFileName; + fullFileName = fullFileNameOss.str(); + + return 0; } - -int RedistributeWorkerThread::updateDbrm() +int RedistributeWorkerThread::connectToWes(int pmId) { - int rc1 = BRM::ERR_OK; - int rc2 = BRM::ERR_OK; - boost::mutex::scoped_lock lock(fActionMutex); + int ret = 0; + ostringstream oss; + oss << "pm" << pmId << "_WriteEngineServer"; - // cannot stop after extent map is updated. - if (!fStopAction) - { - if (fUpdateHwmEntries.size() > 0) - rc1 = fDbrm->bulkSetHWM(fUpdateHwmEntries, 0); + try + { + fMsgQueueClient.reset(new MessageQueueClient(oss.str(), fConfig)); + } + catch (const std::exception& ex) + { + fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- " + ex.what(); + ret = 1; + } + catch (...) + { + fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- unknown"; + ret = 2; + } - if (rc1 == BRM::ERR_OK) - { - int rc2 = fDbrm->bulkUpdateDBRoot(fUpdateRtEntries); - - if (rc2 == 0) - fCommitted = true; - else - fErrorCode = RED_EC_UPDATE_DBRM_FAIL; - } - - // logging for debug - { - if (fUpdateHwmEntries.size() > 0) - { - ostringstream oss; - oss << "HWM_0 workaround, updateHWM(oid,part,seg,hwm)"; - vector::iterator i = fUpdateHwmEntries.begin(); - - for (; i != fUpdateHwmEntries.end(); i++) - { - oss << ":(" << i->oid << "," << i->partNum << "," << i->segNum << "," - << i->hwm << ")"; - } - - oss << ((rc1 == BRM::ERR_OK) ? " success" : " failed"); - logMessage(oss.str(), __LINE__); - } - - if (rc1 == BRM::ERR_OK) - { - ostringstream oss; - oss << "updateDBRoot(startLBID,dbRoot)"; - vector::iterator i = fUpdateRtEntries.begin(); - - for (; i != fUpdateRtEntries.end(); i++) - oss << ":(" << i->startLBID << "," << i->dbRoot << ")"; - - oss << ((rc2 == BRM::ERR_OK) ? " success" : " failed"); - logMessage(oss.str(), __LINE__); - } - } - } - - return ((rc1 == BRM::ERR_OK && rc2 == BRM::ERR_OK) ? 0 : -1); + return ret; } +int RedistributeWorkerThread::updateDbrm() +{ + int rc1 = BRM::ERR_OK; + int rc2 = BRM::ERR_OK; + boost::mutex::scoped_lock lock(fActionMutex); + + // cannot stop after extent map is updated. + if (!fStopAction) + { + if (fUpdateHwmEntries.size() > 0) + rc1 = fDbrm->bulkSetHWM(fUpdateHwmEntries, 0); + + if (rc1 == BRM::ERR_OK) + { + int rc2 = fDbrm->bulkUpdateDBRoot(fUpdateRtEntries); + + if (rc2 == 0) + fCommitted = true; + else + fErrorCode = RED_EC_UPDATE_DBRM_FAIL; + } + + // logging for debug + { + if (fUpdateHwmEntries.size() > 0) + { + ostringstream oss; + oss << "HWM_0 workaround, updateHWM(oid,part,seg,hwm)"; + vector::iterator i = fUpdateHwmEntries.begin(); + + for (; i != fUpdateHwmEntries.end(); i++) + { + oss << ":(" << i->oid << "," << i->partNum << "," << i->segNum << "," << i->hwm << ")"; + } + + oss << ((rc1 == BRM::ERR_OK) ? " success" : " failed"); + logMessage(oss.str(), __LINE__); + } + + if (rc1 == BRM::ERR_OK) + { + ostringstream oss; + oss << "updateDBRoot(startLBID,dbRoot)"; + vector::iterator i = fUpdateRtEntries.begin(); + + for (; i != fUpdateRtEntries.end(); i++) + oss << ":(" << i->startLBID << "," << i->dbRoot << ")"; + + oss << ((rc2 == BRM::ERR_OK) ? " success" : " failed"); + logMessage(oss.str(), __LINE__); + } + } + } + + return ((rc1 == BRM::ERR_OK && rc2 == BRM::ERR_OK) ? 0 : -1); +} bool RedistributeWorkerThread::checkDataTransferAck(SBS& sbs, size_t size) { - if (sbs->length() == 0) - { - ostringstream oss; - oss << "Zero byte read, Network error."; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - fErrorCode = RED_EC_NETWORK_FAIL; - } - else if (sbs->length() < (sizeof(RedistributeMsgHeader) + 1)) - { - ostringstream oss; - oss << "Short message, length=" << sbs->length(); - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - fErrorCode = RED_EC_WKR_MSG_SHORT; - } - else - { - // Need check header info - ByteStream::byte wesMsgId; - *sbs >> wesMsgId; - //const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) sbs->buf(); - sbs->advance(sizeof(RedistributeMsgHeader)); - size_t ack; - *sbs >> ack; + if (sbs->length() == 0) + { + ostringstream oss; + oss << "Zero byte read, Network error."; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + fErrorCode = RED_EC_NETWORK_FAIL; + } + else if (sbs->length() < (sizeof(RedistributeMsgHeader) + 1)) + { + ostringstream oss; + oss << "Short message, length=" << sbs->length(); + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + fErrorCode = RED_EC_WKR_MSG_SHORT; + } + else + { + // Need check header info + ByteStream::byte wesMsgId; + *sbs >> wesMsgId; + // const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) sbs->buf(); + sbs->advance(sizeof(RedistributeMsgHeader)); + size_t ack; + *sbs >> ack; - if (ack != size) - { - ostringstream oss; - oss << "Acked size does not match request: " << ack << "/" << size; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - fErrorCode = RED_EC_SIZE_NACK; - } + if (ack != size) + { + ostringstream oss; + oss << "Acked size does not match request: " << ack << "/" << size; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + fErrorCode = RED_EC_SIZE_NACK; } + } - sbs.reset(); + sbs.reset(); - return (fErrorCode == RED_EC_OK); + return (fErrorCode == RED_EC_OK); } - void RedistributeWorkerThread::confirmToPeer() { - if (fTableLockId > 0) + if (fTableLockId > 0) + { + bool rc = false; + + try { - bool rc = false; + rc = fDbrm->releaseTableLock(fTableLockId); - try - { - rc = fDbrm->releaseTableLock(fTableLockId); + // use the interface, line# replaced with lock id. + logMessage("Releasing table lock... ", fTableLockId); + } + catch (const std::exception& ex) + { + // too bad, the talbe lock is messed up. + fErrorMsg = ex.what(); - // use the interface, line# replaced with lock id. - logMessage("Releasing table lock... ", fTableLockId); - } - catch (const std::exception& ex) - { - // too bad, the talbe lock is messed up. - fErrorMsg = ex.what(); - - // use the interface, line# replaced with lock id. - logMessage("Release table exception: " + fErrorMsg, fTableLockId); - } - catch (...) - { - // use the interface, line# replaced with lock id. - logMessage("Release table lock unknown exception. ", fTableLockId); - } - - if (rc == true) - { - // use the interface, line# replaced with lock id. - logMessage("Release table lock return true. ", fTableLockId); - fTableLockId = 0; - } - else - { - // let destructor try again. - // use the interface, line# replaced with lock id. - logMessage("Release table lock return false. ", fTableLockId); - } + // use the interface, line# replaced with lock id. + logMessage("Release table exception: " + fErrorMsg, fTableLockId); + } + catch (...) + { + // use the interface, line# replaced with lock id. + logMessage("Release table lock unknown exception. ", fTableLockId); } - IDBFileSystem& fs = (IDBPolicy::useHdfs() ? - IDBFileSystem::getFs(IDBDataFile::HDFS) : - IDBPolicy::useCloud() ? - IDBFileSystem::getFs(IDBDataFile::CLOUD) : - IDBFileSystem::getFs(IDBDataFile::BUFFERED)); - - uint32_t confirmCode = RED_DATA_COMMIT; - - if (fErrorCode != RED_EC_OK || fStopAction == true) // fCommitted must be false - confirmCode = RED_DATA_ABORT; - - if (fMyId.second != fPeerId.second) + if (rc == true) { - if (fMsgQueueClient.get() != NULL) - { - ByteStream bs; - RedistributeMsgHeader header(fPeerId.first, fMyId.first, -1, confirmCode); - bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; - bs.append((const ByteStream::byte*) &header, sizeof(header)); - fMsgQueueClient->write(bs); - - // not going to retry for now, ignore the ack and close the connection. - fMsgQueueClient->read(); - fMsgQueueClient.reset(); - } + // use the interface, line# replaced with lock id. + logMessage("Release table lock return true. ", fTableLockId); + fTableLockId = 0; } - else if (confirmCode != RED_DATA_COMMIT) + else { - for (set::iterator i = fNewDirSet.begin(); i != fNewDirSet.end(); i++) - { - fs.remove(i->c_str()); // ignoring return code - } + // let destructor try again. + // use the interface, line# replaced with lock id. + logMessage("Release table lock return false. ", fTableLockId); } + } - // new files committed, remove old ones. - if (confirmCode == RED_DATA_COMMIT) + IDBFileSystem& fs = (IDBPolicy::useHdfs() ? IDBFileSystem::getFs(IDBDataFile::HDFS) + : IDBPolicy::useCloud() ? IDBFileSystem::getFs(IDBDataFile::CLOUD) + : IDBFileSystem::getFs(IDBDataFile::BUFFERED)); + + uint32_t confirmCode = RED_DATA_COMMIT; + + if (fErrorCode != RED_EC_OK || fStopAction == true) // fCommitted must be false + confirmCode = RED_DATA_ABORT; + + if (fMyId.second != fPeerId.second) + { + if (fMsgQueueClient.get() != NULL) { - for (set::iterator i = fOldDirSet.begin(); i != fOldDirSet.end(); i++) - { - fs.remove(i->c_str()); // ignoring return code - } - } + ByteStream bs; + RedistributeMsgHeader header(fPeerId.first, fMyId.first, -1, confirmCode); + bs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; + bs.append((const ByteStream::byte*)&header, sizeof(header)); + fMsgQueueClient->write(bs); - fNewDirSet.clear(); - fOldDirSet.clear(); + // not going to retry for now, ignore the ack and close the connection. + fMsgQueueClient->read(); + fMsgQueueClient.reset(); + } + } + else if (confirmCode != RED_DATA_COMMIT) + { + for (set::iterator i = fNewDirSet.begin(); i != fNewDirSet.end(); i++) + { + fs.remove(i->c_str()); // ignoring return code + } + } + + // new files committed, remove old ones. + if (confirmCode == RED_DATA_COMMIT) + { + for (set::iterator i = fOldDirSet.begin(); i != fOldDirSet.end(); i++) + { + fs.remove(i->c_str()); // ignoring return code + } + } + + fNewDirSet.clear(); + fOldDirSet.clear(); } - void RedistributeWorkerThread::addToDirSet(const char* fileName, bool isSource) { - string path(fileName); - size_t found = path.find_last_of("/\\"); - path = path.substr(0, found); + string path(fileName); + size_t found = path.find_last_of("/\\"); + path = path.substr(0, found); - if (isSource) - fOldDirSet.insert(path); - else - fNewDirSet.insert(path); + if (isSource) + fOldDirSet.insert(path); + else + fNewDirSet.insert(path); } - void RedistributeWorkerThread::handleStop() { - boost::mutex::scoped_lock lock(fActionMutex); + boost::mutex::scoped_lock lock(fActionMutex); - // cannot stop after extent map is updated. - if (!fCommitted) - fStopAction = true; + // cannot stop after extent map is updated. + if (!fCommitted) + fStopAction = true; - lock.unlock(); + lock.unlock(); - logMessage("User stop", __LINE__); - sendResponse(RED_ACTN_STOP); + logMessage("User stop", __LINE__); + sendResponse(RED_ACTN_STOP); } - void RedistributeWorkerThread::sendResponse(uint32_t type) { - uint32_t tmp = fMsgHeader.destination; - fMsgHeader.destination = fMsgHeader.source; - fMsgHeader.source = tmp; - fMsgHeader.messageId = RED_ACTN_RESP; + uint32_t tmp = fMsgHeader.destination; + fMsgHeader.destination = fMsgHeader.source; + fMsgHeader.source = tmp; + fMsgHeader.messageId = RED_ACTN_RESP; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); - if (type == RED_ACTN_REQUEST) - { - if (fErrorCode == RED_EC_OK && fStopAction == false) - fPlanEntry.status = RED_TRANS_SUCCESS; - else if (fErrorCode == RED_EC_PART_EXIST_ON_TARGET) - fPlanEntry.status = RED_TRANS_SKIPPED; - else if (fErrorCode != RED_EC_OK) - fPlanEntry.status = RED_TRANS_FAILED; + if (type == RED_ACTN_REQUEST) + { + if (fErrorCode == RED_EC_OK && fStopAction == false) + fPlanEntry.status = RED_TRANS_SUCCESS; + else if (fErrorCode == RED_EC_PART_EXIST_ON_TARGET) + fPlanEntry.status = RED_TRANS_SKIPPED; + else if (fErrorCode != RED_EC_OK) + fPlanEntry.status = RED_TRANS_FAILED; - // else -- stopped, may try again if support resume + // else -- stopped, may try again if support resume - fBs.append((const ByteStream::byte*) &fPlanEntry, sizeof(fPlanEntry)); - } + fBs.append((const ByteStream::byte*)&fPlanEntry, sizeof(fPlanEntry)); + } - fIOSocket.write(fBs); + fIOSocket.write(fBs); } - void RedistributeWorkerThread::handleData() { - bool done = false; - bool noExcept = true; - SBS sbs; - size_t size = 0; + bool done = false; + bool noExcept = true; + SBS sbs; + size_t size = 0; - try + try + { + do { - do - { - switch (fMsgHeader.messageId) - { - case RED_DATA_INIT: - handleDataInit(); - break; + switch (fMsgHeader.messageId) + { + case RED_DATA_INIT: handleDataInit(); break; - case RED_DATA_START: - handleDataStart(sbs, size); - break; + case RED_DATA_START: handleDataStart(sbs, size); break; - case RED_DATA_CONT: - handleDataCont(sbs, size); - break; + case RED_DATA_CONT: handleDataCont(sbs, size); break; - case RED_DATA_FINISH: - handleDataFinish(sbs, size); - break; + case RED_DATA_FINISH: handleDataFinish(sbs, size); break; - case RED_DATA_COMMIT: - handleDataCommit(sbs, size); - done = true; - break; + case RED_DATA_COMMIT: + handleDataCommit(sbs, size); + done = true; + break; - case RED_DATA_ABORT: - handleDataAbort(sbs, size); - done = true; - break; + case RED_DATA_ABORT: + handleDataAbort(sbs, size); + done = true; + break; - default: - handleUnknowDataMsg(); - done = true; - break; - } + default: + handleUnknowDataMsg(); + done = true; + break; + } - if (!done) - { - // get next message - sbs = fIOSocket.read(); - ByteStream::byte wesMsgId; - *sbs >> wesMsgId; - memcpy(&fMsgHeader, sbs->buf(), sizeof(RedistributeMsgHeader)); - sbs->advance(sizeof(RedistributeMsgHeader)); - } - } - while (!done); // will break after commit/abort or catch an exception - } - catch (const std::exception& ex) - { - noExcept = false; - logMessage(ex.what(), __LINE__); - } - catch (...) - { - noExcept = false; - } + if (!done) + { + // get next message + sbs = fIOSocket.read(); + ByteStream::byte wesMsgId; + *sbs >> wesMsgId; + memcpy(&fMsgHeader, sbs->buf(), sizeof(RedistributeMsgHeader)); + sbs->advance(sizeof(RedistributeMsgHeader)); + } + } while (!done); // will break after commit/abort or catch an exception + } + catch (const std::exception& ex) + { + noExcept = false; + logMessage(ex.what(), __LINE__); + } + catch (...) + { + noExcept = false; + } - if (noExcept == false) - { - // send NACK to peer - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << ((size_t) - 1); - fIOSocket.write(fBs); - } + if (noExcept == false) + { + // send NACK to peer + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << ((size_t)-1); + fIOSocket.write(fBs); + } - fBs.reset(); - fIOSocket.close(); + fBs.reset(); + fIOSocket.close(); } - void RedistributeWorkerThread::handleDataInit() { - uint32_t tmp = fMsgHeader.destination; - fMsgHeader.destination = fMsgHeader.source; - fMsgHeader.source = tmp; - fMsgHeader.messageId = RED_DATA_ACK; - size_t size = 0; + uint32_t tmp = fMsgHeader.destination; + fMsgHeader.destination = fMsgHeader.source; + fMsgHeader.source = tmp; + fMsgHeader.messageId = RED_DATA_ACK; + size_t size = 0; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << size; + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << size; - // finish the hand shaking - fIOSocket.write(fBs); + // finish the hand shaking + fIOSocket.write(fBs); } - void RedistributeWorkerThread::handleDataStart(SBS& sbs, size_t& size) { - char fileName[WriteEngine::FILE_NAME_SIZE]; + char fileName[WriteEngine::FILE_NAME_SIZE]; - try + try + { + // extract the control data for the segment file + RedistributeDataControl dc; + + if (sbs->length() >= sizeof(RedistributeDataControl)) { - // extract the control data for the segment file - RedistributeDataControl dc; - - if (sbs->length() >= sizeof(RedistributeDataControl)) - { - memcpy(&dc, sbs->buf(), sizeof(RedistributeDataControl)); - sbs->advance(sizeof(RedistributeDataControl)); - size = dc.size; - } - else - { - ostringstream oss; - oss << "Short message, length=" << sbs->length(); - fErrorMsg = oss.str(); - fErrorCode = RED_EC_WKR_MSG_SHORT; - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - - // create and open the file for writing. - WriteEngine::FileOp fileOp; // just to get filename, not for file operations - int rc = fileOp.oid2FileName(dc.oid, fileName, true, dc.dbroot, dc.partition, dc.segment); - - if (rc == WriteEngine::NO_ERROR) - { - ostringstream oss; - oss << "=>redistributing: " << fileName << ", oid=" << dc.oid << ", db=" << dc.dbroot - << ", part=" << dc.partition << ", seg=" << dc.segment << " from db=" - << fMsgHeader.source; - logMessage(oss.str(), __LINE__); - } - else - { - fErrorCode = RED_EC_OID_TO_FILENAME; - ostringstream oss; - oss << "Failed to get file name: oid=" << dc.oid << ", dbroot=" << dc.dbroot - << ", partition=" << dc.partition << ", segment=" << dc.segment; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - - if (fNewFilePtr != NULL) - closeFile(fNewFilePtr); - - errno = 0; - fNewFilePtr = fopen(fileName, "wb"); - - if (fNewFilePtr != NULL) - { - ostringstream oss; - oss << "open " << fileName << ", oid=" << dc.oid << ", dbroot=" - << dc.dbroot << ", partition=" << dc.partition << ", segment=" << dc.segment - << ". " << fNewFilePtr; - logMessage(oss.str(), __LINE__); - } - else - { - int e = errno; - fErrorCode = RED_EC_OPEN_FILE_FAIL; - ostringstream oss; - oss << "Failed to open " << fileName << ", oid=" << dc.oid << ", dbroot=" - << dc.dbroot << ", partition=" << dc.partition << ", segment=" << dc.segment - << ". " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - - // set output buffering - errno = 0; - - if (setvbuf(fNewFilePtr, fWriteBuffer.get(), _IOFBF, CHUNK_SIZE)) - { - int e = errno; - ostringstream oss; - oss << "Failed to set i/o buffer: " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - logMessage(fErrorMsg, __LINE__); - - // not throwing an exception now. - } - - // add to set for remove after abort - addToDirSet(fileName, false); - - // do a fseek will show the right size, but will not actually allocate the continuous block. - // do write 4k block till file size. - char buf[PRE_ALLOC_SIZE] = {1}; - size_t nmemb = size / PRE_ALLOC_SIZE; - - while (nmemb-- > 0) - { - errno = 0; - size_t n = fwrite(buf, PRE_ALLOC_SIZE, 1, fNewFilePtr); - - if (n != 1) - { - int e = errno; - ostringstream oss; - oss << "Fail to preallocate file: " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_FWRITE_FAIL; - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - } - - // move back to beging to write real data - fflush(fNewFilePtr); - rewind(fNewFilePtr); + memcpy(&dc, sbs->buf(), sizeof(RedistributeDataControl)); + sbs->advance(sizeof(RedistributeDataControl)); + size = dc.size; } - catch (const std::exception& ex) + else { - // NACK - size = -1; - logMessage(ex.what(), __LINE__); - } - catch (...) - { - // NACK - size = -1; + ostringstream oss; + oss << "Short message, length=" << sbs->length(); + fErrorMsg = oss.str(); + fErrorCode = RED_EC_WKR_MSG_SHORT; + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); } - // ack file size - fMsgHeader.messageId = RED_DATA_ACK; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << size; - fIOSocket.write(fBs); + // create and open the file for writing. + WriteEngine::FileOp fileOp; // just to get filename, not for file operations + int rc = fileOp.oid2FileName(dc.oid, fileName, true, dc.dbroot, dc.partition, dc.segment); - // reset to count the data received - size = 0; - sbs.reset(); + if (rc == WriteEngine::NO_ERROR) + { + ostringstream oss; + oss << "=>redistributing: " << fileName << ", oid=" << dc.oid << ", db=" << dc.dbroot + << ", part=" << dc.partition << ", seg=" << dc.segment << " from db=" << fMsgHeader.source; + logMessage(oss.str(), __LINE__); + } + else + { + fErrorCode = RED_EC_OID_TO_FILENAME; + ostringstream oss; + oss << "Failed to get file name: oid=" << dc.oid << ", dbroot=" << dc.dbroot + << ", partition=" << dc.partition << ", segment=" << dc.segment; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); + } + + if (fNewFilePtr != NULL) + closeFile(fNewFilePtr); + + errno = 0; + fNewFilePtr = fopen(fileName, "wb"); + + if (fNewFilePtr != NULL) + { + ostringstream oss; + oss << "open " << fileName << ", oid=" << dc.oid << ", dbroot=" << dc.dbroot + << ", partition=" << dc.partition << ", segment=" << dc.segment << ". " << fNewFilePtr; + logMessage(oss.str(), __LINE__); + } + else + { + int e = errno; + fErrorCode = RED_EC_OPEN_FILE_FAIL; + ostringstream oss; + oss << "Failed to open " << fileName << ", oid=" << dc.oid << ", dbroot=" << dc.dbroot + << ", partition=" << dc.partition << ", segment=" << dc.segment << ". " << strerror(e) << " (" << e + << ")"; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); + } + + // set output buffering + errno = 0; + + if (setvbuf(fNewFilePtr, fWriteBuffer.get(), _IOFBF, CHUNK_SIZE)) + { + int e = errno; + ostringstream oss; + oss << "Failed to set i/o buffer: " << strerror(e) << " (" << e << ")"; + fErrorMsg = oss.str(); + logMessage(fErrorMsg, __LINE__); + + // not throwing an exception now. + } + + // add to set for remove after abort + addToDirSet(fileName, false); + + // do a fseek will show the right size, but will not actually allocate the continuous block. + // do write 4k block till file size. + char buf[PRE_ALLOC_SIZE] = {1}; + size_t nmemb = size / PRE_ALLOC_SIZE; + + while (nmemb-- > 0) + { + errno = 0; + size_t n = fwrite(buf, PRE_ALLOC_SIZE, 1, fNewFilePtr); + + if (n != 1) + { + int e = errno; + ostringstream oss; + oss << "Fail to preallocate file: " << strerror(e) << " (" << e << ")"; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_FWRITE_FAIL; + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); + } + } + + // move back to beging to write real data + fflush(fNewFilePtr); + rewind(fNewFilePtr); + } + catch (const std::exception& ex) + { + // NACK + size = -1; + logMessage(ex.what(), __LINE__); + } + catch (...) + { + // NACK + size = -1; + } + + // ack file size + fMsgHeader.messageId = RED_DATA_ACK; + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << size; + fIOSocket.write(fBs); + + // reset to count the data received + size = 0; + sbs.reset(); } - void RedistributeWorkerThread::handleDataCont(SBS& sbs, size_t& size) { - size_t ack = 0; + size_t ack = 0; - try + try + { + size_t bytesRcvd = 0; + *sbs >> bytesRcvd; + + if (bytesRcvd != sbs->length()) { - size_t bytesRcvd = 0; - *sbs >> bytesRcvd; - - if (bytesRcvd != sbs->length()) - { - ostringstream oss; - oss << "Incorrect data length: " << sbs->length() << ", expecting " << bytesRcvd; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_BS_TOO_SHORT; - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - - errno = 0; - size_t n = fwrite(sbs->buf(), 1, bytesRcvd, fNewFilePtr); - - if (n != bytesRcvd) - { - int e = errno; - ostringstream oss; - oss << "Fail to write file: " << strerror(e) << " (" << e << ")"; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_FWRITE_FAIL; - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - - ack = bytesRcvd; - size += ack; - } - catch (const std::exception&) - { - // NACK - size = -1; - } - catch (...) - { - // NACK - ack = -1; + ostringstream oss; + oss << "Incorrect data length: " << sbs->length() << ", expecting " << bytesRcvd; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_BS_TOO_SHORT; + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); } - // ack received data - sbs.reset(); - fMsgHeader.messageId = RED_DATA_ACK; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << ack; - fIOSocket.write(fBs); + errno = 0; + size_t n = fwrite(sbs->buf(), 1, bytesRcvd, fNewFilePtr); + + if (n != bytesRcvd) + { + int e = errno; + ostringstream oss; + oss << "Fail to write file: " << strerror(e) << " (" << e << ")"; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_FWRITE_FAIL; + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); + } + + ack = bytesRcvd; + size += ack; + } + catch (const std::exception&) + { + // NACK + size = -1; + } + catch (...) + { + // NACK + ack = -1; + } + + // ack received data + sbs.reset(); + fMsgHeader.messageId = RED_DATA_ACK; + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << ack; + fIOSocket.write(fBs); } - void RedistributeWorkerThread::handleDataFinish(SBS& sbs, size_t& size) { - size_t ack = 0; + size_t ack = 0; - // close open file - closeFile(fNewFilePtr); - fNewFilePtr = NULL; + // close open file + closeFile(fNewFilePtr); + fNewFilePtr = NULL; - try + try + { + size_t fileSize = 0; + *sbs >> fileSize; + + if (fileSize != size) { - size_t fileSize = 0; - *sbs >> fileSize; - - if (fileSize != size) - { - ostringstream oss; - oss << "File size not match: local=" << size << ", remote=" << fileSize; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_FILE_SIZE_NOT_MATCH; - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); - } - - ack = size; - } - catch (const std::exception&) - { - // NACK - size = -1; - } - catch (...) - { - // NACK - ack = -1; + ostringstream oss; + oss << "File size not match: local=" << size << ", remote=" << fileSize; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_FILE_SIZE_NOT_MATCH; + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); } - // ack received data - sbs.reset(); - fMsgHeader.messageId = RED_DATA_ACK; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << ack; - fIOSocket.write(fBs); + ack = size; + } + catch (const std::exception&) + { + // NACK + size = -1; + } + catch (...) + { + // NACK + ack = -1; + } + + // ack received data + sbs.reset(); + fMsgHeader.messageId = RED_DATA_ACK; + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << ack; + fIOSocket.write(fBs); } - void RedistributeWorkerThread::handleDataCommit(SBS& sbs, size_t& size) { - size_t ack = 0; - sbs.reset(); - fMsgHeader.messageId = RED_DATA_ACK; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << ack; - fIOSocket.write(fBs); + size_t ack = 0; + sbs.reset(); + fMsgHeader.messageId = RED_DATA_ACK; + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << ack; + fIOSocket.write(fBs); } - void RedistributeWorkerThread::handleDataAbort(SBS& sbs, size_t& size) { - // close open file - if (fNewFilePtr != NULL) - closeFile(fNewFilePtr); + // close open file + if (fNewFilePtr != NULL) + closeFile(fNewFilePtr); - IDBFileSystem& fs = (IDBPolicy::useHdfs() ? - IDBFileSystem::getFs(IDBDataFile::HDFS) : - IDBPolicy::useCloud() ? - IDBFileSystem::getFs(IDBDataFile::CLOUD) : - IDBFileSystem::getFs(IDBDataFile::BUFFERED)); + IDBFileSystem& fs = (IDBPolicy::useHdfs() ? IDBFileSystem::getFs(IDBDataFile::HDFS) + : IDBPolicy::useCloud() ? IDBFileSystem::getFs(IDBDataFile::CLOUD) + : IDBFileSystem::getFs(IDBDataFile::BUFFERED)); - // remove local files - for (set::iterator i = fNewDirSet.begin(); i != fNewDirSet.end(); i++) - { - fs.remove(i->c_str()); // ignoring return code - } + // remove local files + for (set::iterator i = fNewDirSet.begin(); i != fNewDirSet.end(); i++) + { + fs.remove(i->c_str()); // ignoring return code + } - // send ack - sbs.reset(); - size_t ack = 0; - fMsgHeader.messageId = RED_DATA_ACK; - fBs.restart(); - fBs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. - fBs.append((const ByteStream::byte*) &fMsgHeader, sizeof(fMsgHeader)); - fBs << ack; - fIOSocket.write(fBs); + // send ack + sbs.reset(); + size_t ack = 0; + fMsgHeader.messageId = RED_DATA_ACK; + fBs.restart(); + fBs << (ByteStream::byte)WriteEngine::WE_SVR_REDISTRIBUTE; // dummy, keep for now. + fBs.append((const ByteStream::byte*)&fMsgHeader, sizeof(fMsgHeader)); + fBs << ack; + fIOSocket.write(fBs); } - void RedistributeWorkerThread::handleUnknowDataMsg() { - ostringstream oss; - oss << "Unknown data message: " << fMsgHeader.messageId; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_UNKNOWN_DATA_MSG; - logMessage(fErrorMsg, __LINE__); - throw runtime_error(fErrorMsg); + ostringstream oss; + oss << "Unknown data message: " << fMsgHeader.messageId; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_UNKNOWN_DATA_MSG; + logMessage(fErrorMsg, __LINE__); + throw runtime_error(fErrorMsg); } - void RedistributeWorkerThread::handleUnknowJobMsg() { - ostringstream oss; - oss << "Unknown job message: " << fMsgHeader.messageId; - fErrorMsg = oss.str(); - fErrorCode = RED_EC_UNKNOWN_JOB_MSG; - logMessage(fErrorMsg, __LINE__); + ostringstream oss; + oss << "Unknown job message: " << fMsgHeader.messageId; + fErrorMsg = oss.str(); + fErrorCode = RED_EC_UNKNOWN_JOB_MSG; + logMessage(fErrorMsg, __LINE__); - //protocol error, ignore and close connection. + // protocol error, ignore and close connection. } - void RedistributeWorkerThread::closeFile(FILE* f) { - if (f == NULL) - return; + if (f == NULL) + return; - ostringstream oss; - oss << "close file* " << f << " "; + ostringstream oss; + oss << "close file* " << f << " "; - errno = 0; - int rc = fclose(f); + errno = 0; + int rc = fclose(f); - if (rc != 0) - oss << "error: " << strerror(errno) << " (" << errno << ")"; - else - oss << "OK"; + if (rc != 0) + oss << "error: " << strerror(errno) << " (" << errno << ")"; + else + oss << "OK"; - logMessage(oss.str(), __LINE__); + logMessage(oss.str(), __LINE__); } - void RedistributeWorkerThread::logMessage(const string& msg, int line) { - ostringstream oss; - oss << msg << " @workerThread:" << line; - RedistributeControl::instance()->logMessage(oss.str()); + ostringstream oss; + oss << msg << " @workerThread:" << line; + RedistributeControl::instance()->logMessage(oss.str()); } - -} // namespace +} // namespace redistribute // vim:ts=4 sw=4: - diff --git a/writeengine/redistribute/we_redistributeworkerthread.h b/writeengine/redistribute/we_redistributeworkerthread.h index 447833d1e..cdd3a62aa 100644 --- a/writeengine/redistribute/we_redistributeworkerthread.h +++ b/writeengine/redistribute/we_redistributeworkerthread.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_redistributeworkerthread.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_redistributeworkerthread.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ #ifndef WE_REDISTRIBUTEWORKERTHREAD_H #define WE_REDISTRIBUTEWORKERTHREAD_H @@ -34,7 +34,6 @@ #include "brmtypes.h" #include "we_redistributedef.h" - // forward reference namespace config { @@ -55,105 +54,94 @@ namespace messageqcpp { class ByteStream; class IOSocket; -} +} // namespace messageqcpp namespace messagequeue { class MessageQueueClient; } - namespace redistribute { - - class RedistributeWorkerThread { -public: - RedistributeWorkerThread(messageqcpp::ByteStream& bs, messageqcpp::IOSocket& ios); - ~RedistributeWorkerThread(); + public: + RedistributeWorkerThread(messageqcpp::ByteStream& bs, messageqcpp::IOSocket& ios); + ~RedistributeWorkerThread(); - void operator()(); + void operator()(); -private: + private: + void handleRequest(); + void handleStop(); + void handleData(); + void handleUnknowJobMsg(); - void handleRequest(); - void handleStop(); - void handleData(); - void handleUnknowJobMsg(); + int setup(); + int grabTableLock(); + int buildEntryList(); + int sendData(); + int connectToWes(int); + int updateDbrm(); + void confirmToPeer(); + bool checkDataTransferAck(SBS&, size_t); - int setup(); - int grabTableLock(); - int buildEntryList(); - int sendData(); - int connectToWes(int); - int updateDbrm(); - void confirmToPeer(); - bool checkDataTransferAck(SBS&, size_t); + void sendResponse(uint32_t); - void sendResponse(uint32_t); + void doAbort(); - void doAbort(); + void handleDataInit(); + void handleDataStart(messageqcpp::SBS&, size_t&); + void handleDataCont(messageqcpp::SBS&, size_t&); + void handleDataFinish(messageqcpp::SBS&, size_t&); + void handleDataCommit(messageqcpp::SBS&, size_t&); + void handleDataAbort(messageqcpp::SBS&, size_t&); + void handleUnknowDataMsg(); - void handleDataInit(); - void handleDataStart(messageqcpp::SBS&, size_t&); - void handleDataCont(messageqcpp::SBS&, size_t&); - void handleDataFinish(messageqcpp::SBS&, size_t&); - void handleDataCommit(messageqcpp::SBS&, size_t&); - void handleDataAbort(messageqcpp::SBS&, size_t&); - void handleUnknowDataMsg(); + int buildFullHdfsPath(std::map& rootToPathMap, int64_t colOid, int16_t dbRoot, + uint32_t partition, int16_t segment, std::string& fullFileName); - int buildFullHdfsPath( std::map& rootToPathMap, - int64_t colOid, - int16_t dbRoot, - uint32_t partition, - int16_t segment, - std::string& fullFileName); + void closeFile(FILE*); // for tracing, may remove later. + void addToDirSet(const char*, bool); + void logMessage(const std::string&, int); - void closeFile(FILE*); // for tracing, may remove later. - void addToDirSet(const char*, bool); - void logMessage(const std::string&, int); + oam::OamCache* fOamCache; + config::Config* fConfig; + boost::shared_ptr fMsgQueueClient; - oam::OamCache* fOamCache; - config::Config* fConfig; - boost::shared_ptr fMsgQueueClient; + RedistributeMsgHeader fMsgHeader; + messageqcpp::ByteStream& fBs; + messageqcpp::IOSocket& fIOSocket; + RedistributePlanEntry fPlanEntry; + uint64_t fTableLockId; + int32_t fErrorCode; + std::string fErrorMsg; + std::pair fMyId; // + std::pair fPeerId; // + std::set fSegments; + std::vector fOids; // column oids + std::vector fUpdateRtEntries; // for dbrm update + std::vector fUpdateHwmEntries; // for dbrm update - RedistributeMsgHeader fMsgHeader; - messageqcpp::ByteStream& fBs; - messageqcpp::IOSocket& fIOSocket; - RedistributePlanEntry fPlanEntry; - uint64_t fTableLockId; - int32_t fErrorCode; - std::string fErrorMsg; - std::pair fMyId; // - std::pair fPeerId; // - std::set fSegments; - std::vector fOids; // column oids - std::vector fUpdateRtEntries; // for dbrm update - std::vector fUpdateHwmEntries; // for dbrm update + FILE* fNewFilePtr; + FILE* fOldFilePtr; + std::set fNewDirSet; + std::set fOldDirSet; + boost::shared_array fWriteBuffer; - FILE* fNewFilePtr; - FILE* fOldFilePtr; - std::set fNewDirSet; - std::set fOldDirSet; - boost::shared_array fWriteBuffer; + boost::shared_ptr fDbrm; - boost::shared_ptr fDbrm; + // for segment file # workaround + // uint64_t fSegPerRoot; - // for segment file # workaround - //uint64_t fSegPerRoot; - - - static boost::mutex fActionMutex; - static volatile bool fStopAction; - static volatile bool fCommitted; - static std::string fWesInUse; + static boost::mutex fActionMutex; + static volatile bool fStopAction; + static volatile bool fCommitted; + static std::string fWesInUse; }; -} // namespace - +} // namespace redistribute #endif // WE_REDISTRIBUTEWORKERTHREAD_H // vim:ts=4 sw=4: - diff --git a/writeengine/resource.h b/writeengine/resource.h index acd7800d5..5c447bc23 100644 --- a/writeengine/resource.h +++ b/writeengine/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/writeengine/server/resource.h b/writeengine/server/resource.h index bf240aab9..c5b88727e 100644 --- a/writeengine/server/resource.h +++ b/writeengine/server/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/writeengine/server/we_brmrprtparser.cpp b/writeengine/server/we_brmrprtparser.cpp index 3df21b49c..85ccf652c 100644 --- a/writeengine/server/we_brmrprtparser.cpp +++ b/writeengine/server/we_brmrprtparser.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_brmrprtparser.cpp @@ -35,15 +35,12 @@ using namespace std; namespace WriteEngine { - -BrmReportParser::BrmReportParser(): fRptFile() +BrmReportParser::BrmReportParser() : fRptFile() { - } BrmReportParser::~BrmReportParser() { - } //------------------------------------------------------------------------------ @@ -53,53 +50,52 @@ BrmReportParser::~BrmReportParser() // Function is limited to records that are no longer than 255 bytes. // If a record is longer than 255 bytes, the function misbehaves. //------------------------------------------------------------------------------ -bool BrmReportParser::serialize(std::string RptFileName, - messageqcpp::ByteStream& Bs) +bool BrmReportParser::serialize(std::string RptFileName, messageqcpp::ByteStream& Bs) { - try - { - fRptFile.open(RptFileName.c_str(), ifstream::in); - } - catch (std::exception& ex) - { - cout << "Failed to open BRMRptFile " << RptFileName << endl; - cout << ex.what() << endl; - throw runtime_error(ex.what()); - } + try + { + fRptFile.open(RptFileName.c_str(), ifstream::in); + } + catch (std::exception& ex) + { + cout << "Failed to open BRMRptFile " << RptFileName << endl; + cout << ex.what() << endl; + throw runtime_error(ex.what()); + } - if (fRptFile.good()) - { - char aBuff[10240]; - unsigned int aLen = 0; + if (fRptFile.good()) + { + char aBuff[10240]; + unsigned int aLen = 0; - while (fRptFile.good() && !fRptFile.eof()) + while (fRptFile.good() && !fRptFile.eof()) + { + fRptFile.getline(aBuff, sizeof(aBuff) - 1); + aLen = fRptFile.gcount(); + + if ((aLen != (sizeof(aBuff) - 1)) && (aLen > 0)) + { + // aBuff[aLen-1] = '\n'; + // aBuff[aLen]=0; + // cout << "Data Read " << aBuff < 0)) - { - //aBuff[aLen-1] = '\n'; - //aBuff[aLen]=0; - //cout << "Data Read " << aBuff < 0) + { + strBuff.assign(aBuff, aLen); + Bs << strBuff; + } } - if (fRptFile.good()) - { - char aBuff[8192]; - unsigned int aLen = 0; - std::string strBuff; + fRptFile.close(); + cout << "Closed Report File : " << RptFileName << endl; + } + else + { + std::ostringstream oss; + oss << "Failed to open Report File " << RptFileName << endl; + cout << oss.str() << endl; + return false; + } - while ( fRptFile.good() ) - { - fRptFile.read(aBuff, sizeof(aBuff)); - aLen = fRptFile.gcount(); - - if (aLen > 0) - { - strBuff.assign( aBuff, aLen ); - Bs << strBuff; - } - } - - fRptFile.close(); - cout << "Closed Report File : " << RptFileName << endl; - } - else - { - std::ostringstream oss; - oss << "Failed to open Report File " << RptFileName << endl; - cout << oss.str() << endl; - return false; - } - - return true; + return true; } void BrmReportParser::unserialize(messageqcpp::ByteStream& Bs) { - //TODO to be changed. left it here to understand how to implement - /* - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN); - ReturnedColumn::unserialize(b); // parent class unserialize - b >> (uint32_t&) fOid; - b >> fData; - b >> reinterpret_cast(fReturnAll); - b >> (uint32_t&) fSequence; - */ + // TODO to be changed. left it here to understand how to implement + /* + ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN); + ReturnedColumn::unserialize(b); // parent class unserialize + b >> (uint32_t&) fOid; + b >> fData; + b >> reinterpret_cast(fReturnAll); + b >> (uint32_t&) fSequence; + */ - std::string aStrLine; - - while (Bs.length() > 0) - { - Bs >> aStrLine; - cout << aStrLine; - } + std::string aStrLine; + while (Bs.length() > 0) + { + Bs >> aStrLine; + cout << aStrLine; + } } - - - - } /* namespace WriteEngine */ diff --git a/writeengine/server/we_brmrprtparser.h b/writeengine/server/we_brmrprtparser.h index 9b179f070..b69bb6d6a 100644 --- a/writeengine/server/we_brmrprtparser.h +++ b/writeengine/server/we_brmrprtparser.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_brmrprtparser.h * @@ -53,21 +53,19 @@ namespace WriteEngine { - class BrmReportParser { -public: - BrmReportParser(); - virtual ~BrmReportParser(); + public: + BrmReportParser(); + virtual ~BrmReportParser(); -public: - bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs); - bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs); - void unserialize(messageqcpp::ByteStream& Bs); - -private: - ifstream fRptFile; + public: + bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs); + bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs); + void unserialize(messageqcpp::ByteStream& Bs); + private: + ifstream fRptFile; }; } /* namespace WriteEngine */ diff --git a/writeengine/server/we_cleartablelockcmd.cpp b/writeengine/server/we_cleartablelockcmd.cpp index 39df76973..db103ee4e 100644 --- a/writeengine/server/we_cleartablelockcmd.cpp +++ b/writeengine/server/we_cleartablelockcmd.cpp @@ -31,60 +31,52 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // Process a bulk rollback request based on input from the specified input // bytestream object. //------------------------------------------------------------------------------ -int WE_ClearTableLockCmd::processRollback( - messageqcpp::ByteStream& bs, - std::string& errMsg) +int WE_ClearTableLockCmd::processRollback(messageqcpp::ByteStream& bs, std::string& errMsg) { - uint8_t rc = 0; - errMsg.clear(); + uint8_t rc = 0; + errMsg.clear(); - try - { - uint32_t tableOID; - uint64_t tableLockID; - std::string tableName; - std::string appName; + try + { + uint32_t tableOID; + uint64_t tableLockID; + std::string tableName; + std::string appName; - // May want to eventually comment out this logging to stdout, - // but it shouldn't hurt to keep in here. - std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc; - bs >> tableLockID; - std::cout << ": tableLock-" << tableLockID; + // May want to eventually comment out this logging to stdout, + // but it shouldn't hurt to keep in here. + std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc; + bs >> tableLockID; + std::cout << ": tableLock-" << tableLockID; - bs >> tableOID; - std::cout << "; tableOID-" << tableOID; + bs >> tableOID; + std::cout << "; tableOID-" << tableOID; - bs >> tableName; - std::cout << "; table-" << tableName; + bs >> tableName; + std::cout << "; table-" << tableName; - bs >> appName; - std::cout << "; app-" << appName << std::endl; + bs >> appName; + std::cout << "; app-" << appName << std::endl; - int we_rc = fWEWrapper.bulkRollback( - tableOID, - tableLockID, - tableName, - appName, - false, // no extra debug logging to the console - errMsg ); + int we_rc = fWEWrapper.bulkRollback(tableOID, tableLockID, tableName, appName, + false, // no extra debug logging to the console + errMsg); - if (we_rc != NO_ERROR) - rc = 2; - } - catch (std::exception& ex) - { - std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() << - std::endl; - errMsg = ex.what(); - rc = 1; - } + if (we_rc != NO_ERROR) + rc = 2; + } + catch (std::exception& ex) + { + std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() << std::endl; + errMsg = ex.what(); + rc = 1; + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -95,61 +87,55 @@ int WE_ClearTableLockCmd::processRollback( // We only need this for the "successful" case, as the bulk rollback takes // care of db file cleanup in the "unsuccessful" case. //------------------------------------------------------------------------------ -int WE_ClearTableLockCmd::processCleanup( - messageqcpp::ByteStream& bs, - std::string& errMsg) +int WE_ClearTableLockCmd::processCleanup(messageqcpp::ByteStream& bs, std::string& errMsg) { - uint8_t rc = 0; - errMsg.clear(); + uint8_t rc = 0; + errMsg.clear(); - try + try + { + uint32_t tableOID; + + // May want to eventually comment out this logging to stdout, + // but it shouldn't hurt to keep in here. + std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc; + bs >> tableOID; + std::cout << ": tableOID-" << tableOID << std::endl; + + // On an HDFS system, this is where we delete any DB files + // (ex: *.orig) that we no longer need. + if ((idbdatafile::IDBPolicy::useHdfs()) && (bs.length() >= sizeof(messageqcpp::ByteStream::byte))) { - uint32_t tableOID; + messageqcpp::ByteStream::byte deleteHdfsTempDbFiles; + bs >> deleteHdfsTempDbFiles; - // May want to eventually comment out this logging to stdout, - // but it shouldn't hurt to keep in here. - std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc; - bs >> tableOID; - std::cout << ": tableOID-" << tableOID << std::endl; + if (deleteHdfsTempDbFiles) + { + std::string endDbErrMsg; + ConfirmHdfsDbFile confirmHdfs; - // On an HDFS system, this is where we delete any DB files - // (ex: *.orig) that we no longer need. - if ((idbdatafile::IDBPolicy::useHdfs()) && - (bs.length() >= sizeof(messageqcpp::ByteStream::byte))) - { - messageqcpp::ByteStream::byte deleteHdfsTempDbFiles; - bs >> deleteHdfsTempDbFiles; + // We always pass "true", as this only applies to the "success- + // ful" case. See comments that precede this function. + int endRc = confirmHdfs.endDbFileListFromMetaFile(tableOID, true, endDbErrMsg); - if (deleteHdfsTempDbFiles) - { - std::string endDbErrMsg; - ConfirmHdfsDbFile confirmHdfs; - - // We always pass "true", as this only applies to the "success- - // ful" case. See comments that precede this function. - int endRc = confirmHdfs.endDbFileListFromMetaFile( - tableOID, true, endDbErrMsg); - - // In this case, a deletion error is not fatal. - // so we don't propagate a bad return code. We - // may want to add syslog msg (TBD) - if (endRc != NO_ERROR) - std::cout << "Orig db file deletion error: " << - endDbErrMsg << std::endl; - } - } - - BulkRollbackMgr::deleteMetaFile( tableOID ); - } - catch (std::exception& ex) - { - std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() << - std::endl; - errMsg = ex.what(); - rc = 1; + // In this case, a deletion error is not fatal. + // so we don't propagate a bad return code. We + // may want to add syslog msg (TBD) + if (endRc != NO_ERROR) + std::cout << "Orig db file deletion error: " << endDbErrMsg << std::endl; + } } - return rc; + BulkRollbackMgr::deleteMetaFile(tableOID); + } + catch (std::exception& ex) + { + std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() << std::endl; + errMsg = ex.what(); + rc = 1; + } + + return rc; } -} +} // namespace WriteEngine diff --git a/writeengine/server/we_cleartablelockcmd.h b/writeengine/server/we_cleartablelockcmd.h index 362ece153..7574d9791 100644 --- a/writeengine/server/we_cleartablelockcmd.h +++ b/writeengine/server/we_cleartablelockcmd.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_cleartablelockcmd.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_cleartablelockcmd.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ #ifndef WE_CLEARTABLELOCKCMD_H__ #define WE_CLEARTABLELOCKCMD_H__ @@ -30,35 +30,34 @@ namespace WriteEngine { - /** @brief Process messages from a cleartablelock or filesplitter client */ class WE_ClearTableLockCmd { -public: - /** @brief WE_ClearTableLockCmd constructor - */ - WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc) { } + public: + /** @brief WE_ClearTableLockCmd constructor + */ + WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc) + { + } - /** @brief Process bulk rollback request - * @param ibs Input byte stream - * @param errMsg Return error message - */ - int processRollback(messageqcpp::ByteStream& ibs, - std::string& errMsg); + /** @brief Process bulk rollback request + * @param ibs Input byte stream + * @param errMsg Return error message + */ + int processRollback(messageqcpp::ByteStream& ibs, std::string& errMsg); - /** @brief Process bulk rollback cleanup request - * @param ibs Input byte stream - * @param errMsg Return error message - */ - int processCleanup (messageqcpp::ByteStream& ibs, - std::string& errMsg); + /** @brief Process bulk rollback cleanup request + * @param ibs Input byte stream + * @param errMsg Return error message + */ + int processCleanup(messageqcpp::ByteStream& ibs, std::string& errMsg); -private: - WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object - std::string fUserDesc; + private: + WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object + std::string fUserDesc; }; -} +} // namespace WriteEngine #endif diff --git a/writeengine/server/we_cpifeederthread.cpp b/writeengine/server/we_cpifeederthread.cpp index 309d20212..19d96b7ad 100644 --- a/writeengine/server/we_cpifeederthread.cpp +++ b/writeengine/server/we_cpifeederthread.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_cpifeederthread.cpp @@ -27,7 +27,6 @@ * Author: bpaul */ - #include #include #include @@ -48,148 +47,138 @@ using namespace messageqcpp; namespace WriteEngine { - //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ void WECpiFeederRunner::operator()() { - fOwner.feedData2Cpi(); - cout << "Finished running Feeder Thread!!" << endl; + fOwner.feedData2Cpi(); + cout << "Finished running Feeder Thread!!" << endl; } - //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ -WECpiFeederThread::WECpiFeederThread(WEDataLoader& Ref): - fOwner(Ref), - fpThread(0), - fContinue(true), - fStopped(true) +WECpiFeederThread::WECpiFeederThread(WEDataLoader& Ref) + : fOwner(Ref), fpThread(0), fContinue(true), fStopped(true) { - cout << "Inside WECpiFeederThread constructor" << endl; + cout << "Inside WECpiFeederThread constructor" << endl; } //------------------------------------------------------------------------------ WECpiFeederThread::~WECpiFeederThread() { - if (fpThread) - { - delete fpThread; - } + if (fpThread) + { + delete fpThread; + } - fpThread = 0; + fpThread = 0; } //------------------------------------------------------------------------------ void WECpiFeederThread::startFeederThread() { - fStopped = false; - cout << "Starting Feeder Thread!!" << endl; - fpThread = new boost::thread(WECpiFeederRunner(*this)); + fStopped = false; + cout << "Starting Feeder Thread!!" << endl; + fpThread = new boost::thread(WECpiFeederRunner(*this)); } //------------------------------------------------------------------------------ void WECpiFeederThread::add2MsgQueue(ByteStream& Ibs) { - - //TODO creating copy is NOT good; later read from socket using a SBS - messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs)); - Ibs.reset(); //forcefully clearing it - boost::mutex::scoped_lock aLock(fMsgQMutex); - //cout << "pushing to the MsgQueue" << endl; - fMsgQueue.push(aSbs); - fFeederCond.notify_one(); // as per preference of Damon - aLock.unlock(); - + // TODO creating copy is NOT good; later read from socket using a SBS + messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs)); + Ibs.reset(); // forcefully clearing it + boost::mutex::scoped_lock aLock(fMsgQMutex); + // cout << "pushing to the MsgQueue" << endl; + fMsgQueue.push(aSbs); + fFeederCond.notify_one(); // as per preference of Damon + aLock.unlock(); } //------------------------------------------------------------------------------ void WECpiFeederThread::feedData2Cpi() { - while (isContinue()) + while (isContinue()) + { + boost::mutex::scoped_lock aLock(fMsgQMutex); + + if (fMsgQueue.empty()) { + bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000)); - boost::mutex::scoped_lock aLock(fMsgQMutex); - - if (fMsgQueue.empty()) - { - bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000)); - - if (!isContinue()) - { - aLock.unlock(); - break; - } - - // to handle spurious wake ups and timeout wake ups - if ((fMsgQueue.empty()) || (!aTimedOut)) - { - aLock.unlock(); - continue; - } - } - - messageqcpp::SBS aSbs = fMsgQueue.front(); - fMsgQueue.pop(); - + if (!isContinue()) + { aLock.unlock(); + break; + } - try - { - fOwner.pushData2Cpimport((*aSbs)); - //cout << "Finished PUSHING data " << endl; - } - catch (runtime_error&) - { - //cout << "Caught exception : " << e.what() << endl; - //break; - } - - aSbs.reset(); //forcefully clearing it - - // We start sending data request from here ONLY - if (getQueueSize() == WEDataLoader::MAX_QSIZE) fOwner.sendDataRequest(); + // to handle spurious wake ups and timeout wake ups + if ((fMsgQueue.empty()) || (!aTimedOut)) + { + aLock.unlock(); + continue; + } } - cout << "CpiFeedThread Stopped!! " << endl; - fStopped = true; + messageqcpp::SBS aSbs = fMsgQueue.front(); + fMsgQueue.pop(); + aLock.unlock(); + + try + { + fOwner.pushData2Cpimport((*aSbs)); + // cout << "Finished PUSHING data " << endl; + } + catch (runtime_error&) + { + // cout << "Caught exception : " << e.what() << endl; + // break; + } + + aSbs.reset(); // forcefully clearing it + + // We start sending data request from here ONLY + if (getQueueSize() == WEDataLoader::MAX_QSIZE) + fOwner.sendDataRequest(); + } + + cout << "CpiFeedThread Stopped!! " << endl; + fStopped = true; } //------------------------------------------------------------------------------ bool WECpiFeederThread::isMsgQueueEmpty() { - bool aRet = false; - boost::mutex::scoped_lock aLock(fMsgQMutex); - aRet = fMsgQueue.empty(); - aLock.unlock(); - return aRet; + bool aRet = false; + boost::mutex::scoped_lock aLock(fMsgQMutex); + aRet = fMsgQueue.empty(); + aLock.unlock(); + return aRet; } //------------------------------------------------------------------------------ void WECpiFeederThread::stopThread() { - boost::mutex::scoped_lock aCondLock(fContMutex); - fContinue = false; - aCondLock.unlock(); - fFeederCond.notify_all(); - cout << "Notified all" << endl; + boost::mutex::scoped_lock aCondLock(fContMutex); + fContinue = false; + aCondLock.unlock(); + fFeederCond.notify_all(); + cout << "Notified all" << endl; } //------------------------------------------------------------------------------ bool WECpiFeederThread::isContinue() { - boost::mutex::scoped_lock aCondLock(fContMutex); - return fContinue; + boost::mutex::scoped_lock aCondLock(fContMutex); + return fContinue; } //------------------------------------------------------------------------------ - - } /* namespace WriteEngine */ diff --git a/writeengine/server/we_cpifeederthread.h b/writeengine/server/we_cpifeederthread.h index b0851c172..0a846aca3 100644 --- a/writeengine/server/we_cpifeederthread.h +++ b/writeengine/server/we_cpifeederthread.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_cpifeederthread.h @@ -34,61 +34,62 @@ namespace WriteEngine { - class WEDataLoader; class WECpiFeederThread; class WECpiFeederRunner { -public: - WECpiFeederRunner(WECpiFeederThread& Ref): fOwner(Ref) { /* ctor */ } - virtual ~WECpiFeederRunner() {/* dtor */} - void operator()(); + public: + WECpiFeederRunner(WECpiFeederThread& Ref) : fOwner(Ref) + { /* ctor */ + } + virtual ~WECpiFeederRunner() + { /* dtor */ + } + void operator()(); -public: - WECpiFeederThread& fOwner; + public: + WECpiFeederThread& fOwner; }; - class WECpiFeederThread { -public: - WECpiFeederThread(WEDataLoader& Ref); - virtual ~WECpiFeederThread(); + public: + WECpiFeederThread(WEDataLoader& Ref); + virtual ~WECpiFeederThread(); -public: - void startFeederThread(); - void add2MsgQueue(messageqcpp::ByteStream& Ibs); - void feedData2Cpi(); - void stopThread(); - bool isMsgQueueEmpty(); - //bool isPushing() { return fPushing; } - bool isStopped() - { - return fStopped; - } - int getQueueSize() - { - return fMsgQueue.size(); - } - bool isContinue(); -private: + public: + void startFeederThread(); + void add2MsgQueue(messageqcpp::ByteStream& Ibs); + void feedData2Cpi(); + void stopThread(); + bool isMsgQueueEmpty(); + // bool isPushing() { return fPushing; } + bool isStopped() + { + return fStopped; + } + int getQueueSize() + { + return fMsgQueue.size(); + } + bool isContinue(); - WEDataLoader& fOwner; + private: + WEDataLoader& fOwner; - boost::condition fFeederCond; - boost::mutex fMsgQMutex; - typedef std::queue WEMsgQueue; - WEMsgQueue fMsgQueue; + boost::condition fFeederCond; + boost::mutex fMsgQMutex; + typedef std::queue WEMsgQueue; + WEMsgQueue fMsgQueue; - boost::thread* fpThread; - bool fContinue; - boost::mutex fContMutex; - //bool fPushing; - bool fStopped; + boost::thread* fpThread; + bool fContinue; + boost::mutex fContMutex; + // bool fPushing; + bool fStopped; - - friend class WEDataLoader; + friend class WEDataLoader; }; } /* namespace WriteEngine */ diff --git a/writeengine/server/we_dataloader.cpp b/writeengine/server/we_dataloader.cpp index 94a9c8553..18a616799 100644 --- a/writeengine/server/we_dataloader.cpp +++ b/writeengine/server/we_dataloader.cpp @@ -15,9 +15,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_dataloader.cpp * @@ -25,19 +25,19 @@ * Author: Boby Paul: bpaul@calpont.com */ -#include "mcsconfig.h" // Used to pickup STRERROR_R_CHAR_P definition +#include "mcsconfig.h" // Used to pickup STRERROR_R_CHAR_P definition #include #include #include #include -#include //pipe() && fork() +#include //pipe() && fork() #if defined(__linux__) -#include //wait() +#include //wait() #elif defined(__FreeBSD__) #include -#include // For stat(). +#include // For stat(). #include #include #include @@ -56,7 +56,6 @@ #include using namespace std; - #include #include #include @@ -77,8 +76,7 @@ using namespace messageqcpp; namespace WriteEngine { - -//bool WEDataLoader::fTearDownCpimport=false; // @bug 4267 +// bool WEDataLoader::fTearDownCpimport=false; // @bug 4267 //----------------------------------------------------------------------------- /** @@ -86,39 +84,43 @@ namespace WriteEngine * @brief WEDataLoader::Constructor * **/ -WEDataLoader::WEDataLoader(SplitterReadThread& Srt ): fRef(Srt), - fMode(-1), - fDataDumpFile(), - fTxBytes(0), - fRxBytes(0), - fPmId(0), - fCh_pid(0), - fThis_pid(0), - fP_pid(0), - fpCfThread(0), - fTearDownCpimport(false), // @bug 4267 - fWaitPidRc(0), // @bug 4267 - fWaitPidStatus(0), // @bug 4267 - fForceKill(false), - fPipeErr(false), - fpSysLog(0) +WEDataLoader::WEDataLoader(SplitterReadThread& Srt) + : fRef(Srt) + , fMode(-1) + , fDataDumpFile() + , fTxBytes(0) + , fRxBytes(0) + , fPmId(0) + , fCh_pid(0) + , fThis_pid(0) + , fP_pid(0) + , fpCfThread(0) + , fTearDownCpimport(false) + , // @bug 4267 + fWaitPidRc(0) + , // @bug 4267 + fWaitPidStatus(0) + , // @bug 4267 + fForceKill(false) + , fPipeErr(false) + , fpSysLog(0) { - Config weConfig; - uint16_t localModuleId = weConfig.getLocalModuleID(); - fPmId = static_cast(localModuleId); + Config weConfig; + uint16_t localModuleId = weConfig.getLocalModuleID(); + fPmId = static_cast(localModuleId); - srand ( time(NULL) ); // initialize random seed - int aObjId = rand() % 10000 + 1; // generate a random number + srand(time(NULL)); // initialize random seed + int aObjId = rand() % 10000 + 1; // generate a random number - setObjId(aObjId); + setObjId(aObjId); - setupSignalHandlers(); + setupSignalHandlers(); - if (!fpSysLog) - { - fpSysLog = SimpleSysLog::instance(); - fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SRV)); - } + if (!fpSysLog) + { + fpSysLog = SimpleSysLog::instance(); + fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SRV)); + } } //----------------------------------------------------------------------------- /** @@ -129,36 +131,35 @@ WEDataLoader::WEDataLoader(SplitterReadThread& Srt ): fRef(Srt), WEDataLoader::~WEDataLoader() { + try + { + if (fDataDumpFile.is_open()) + fDataDumpFile.close(); - try + cout << "\tRx Bytes " << getRxBytes() << endl; + cout << "\tTX Bytes " << getTxBytes() << endl; + + cout << "\tChild PID " << getChPid() << endl; + + if (getChPid()) { - if (fDataDumpFile.is_open()) fDataDumpFile.close(); - - cout << "\tRx Bytes " << getRxBytes() << endl; - cout << "\tTX Bytes " << getTxBytes() << endl; - - cout << "\tChild PID " << getChPid() << endl; - - if (getChPid()) - { - if (2 == getMode()) //@bug 5012 - { - kill(getChPid(), SIGINT); - teardownCpimport(fTearDownCpimport); - } - else - { - teardownCpimport(false); // @bug 4267 - } - } - - } - catch (std::exception&) // @bug 4164: exception causing thread to exit - { - cout << "Error tearing down cpimport in WEDataLoader destructor" << endl; + if (2 == getMode()) //@bug 5012 + { + kill(getChPid(), SIGINT); + teardownCpimport(fTearDownCpimport); + } + else + { + teardownCpimport(false); // @bug 4267 + } } + } + catch (std::exception&) // @bug 4164: exception causing thread to exit + { + cout << "Error tearing down cpimport in WEDataLoader destructor" << endl; + } - //cout << "Leaving WEDataLoader destructor" << endl; + // cout << "Leaving WEDataLoader destructor" << endl; } //------------------------------------------------------------------------------ @@ -168,8 +169,8 @@ WEDataLoader::~WEDataLoader() void WEDataLoader::setupSignalHandlers() { #ifndef _MSC_VER - signal(SIGPIPE, SIG_IGN); - signal(SIGCHLD, WEDataLoader::onSigChild); + signal(SIGPIPE, SIG_IGN); + signal(SIGCHLD, WEDataLoader::onSigChild); #endif } //------------------------------------------------------------------------------ @@ -177,20 +178,18 @@ void WEDataLoader::setupSignalHandlers() //------------------------------------------------------------------------------ void WEDataLoader::onSigChild(int aInt) { - std::string aStr = "Received SIGCHLD of terminated process.."; - cout << aStr << endl; - // fTearDownCpimport = true; // @bug 4267 - - // commented out for non-static variables - //ostringstream oss; - //oss << getObjId() <<" : " <logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + std::string aStr = "Received SIGCHLD of terminated process.."; + cout << aStr << endl; + // fTearDownCpimport = true; // @bug 4267 + // commented out for non-static variables + // ostringstream oss; + // oss << getObjId() <<" : " <logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); } - //----------------------------------------------------------------------------- /** * @@ -200,7 +199,7 @@ void WEDataLoader::onSigChild(int aInt) bool WEDataLoader::update(Subject* pSub) { - return true; + return true; } //----------------------------------------------------------------------------- /** @@ -209,112 +208,111 @@ bool WEDataLoader::update(Subject* pSub) * **/ -bool WEDataLoader::setupCpimport() // fork the cpimport +bool WEDataLoader::setupCpimport() // fork the cpimport { - pid_t aChPid; + pid_t aChPid; + + errno = 0; + + if (pipe(fFIFO) == -1) + { + int errnum = errno; + ostringstream oss; + oss << getObjId() << " : Error in creating pipe (errno-" << errnum << "); " << strerror(errnum); + throw runtime_error(oss.str()); + } + + setPid(getpid()); + setPPid(getppid()); + + errno = 0; + aChPid = fork(); + + if (aChPid == -1) // an error caused + { + int errnum = errno; + ostringstream oss; + oss << getObjId() << " : Error in forking cpimport.bin (errno-" << errnum << "); " << strerror(errnum); + throw runtime_error(oss.str()); + } + else if (aChPid == 0) // we are in child + { + int aStartFD = 3; + int aEndFD = fFIFO[1] + 256; + close(fFIFO[1]); // close the WRITER of CHILD + + cout << "Child Process Info: PID = " << getpid() << " (fFIFO[0], fFIFO[1]) = (" << fFIFO[0] << "," + << fFIFO[1] << ")" + << " (StartFD, EndFD) = (" << aStartFD << "," << aEndFD << ")" << endl; + + std::vector Cmds; + // str2Argv(fCmdLineStr, Cmds); // to avoid out-of-scope problem + std::string aCmdLine = fCmdLineStr; + std::istringstream ss(aCmdLine); + std::string arg; + std::vector v2(20, ""); + unsigned int i = 0; + + while (ss >> arg) + { + v2[i++] = arg; + } + + for (unsigned int j = 0; j < i; ++j) + { + Cmds.push_back(const_cast(v2[j].c_str())); + } + + Cmds.push_back(0); // null terminate + // updatePrgmPath(Cmds); + + // NOTE: for debugging + int aSize = Cmds.size(); + + for (int aIdx = 0; aIdx < aSize; ++aIdx) + { + cout << "Args " << Cmds[aIdx] << endl; + } + + cout.flush(); + + close(0); // close stdin for the child + dup2(fFIFO[0], 0); // make stdin be the reading end of the pipe + + // BUG 4410 : hacky solution so that CHLD process get EOF on close of pipe + for (int i = aStartFD; i < aEndFD; i++) + close(i); errno = 0; + int aRet = execvp(Cmds[0], &Cmds[0]); // NOTE - works with full Path + // int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works if $PATH has cpimport - if (pipe(fFIFO) == -1) - { - int errnum = errno; - ostringstream oss; - oss << getObjId() << " : Error in creating pipe (errno-" << - errnum << "); " << strerror(errnum); - throw runtime_error( oss.str() ); - } + int execvErrno = errno; + cout << "Return status of cpimport is " << aRet << endl; + cout.flush(); + close(fFIFO[0]); // will trigger an EOF on stdin + ostringstream oss; + oss << getObjId() << " : execv error: cpimport.bin invocation failed; " + << "(errno-" << errno << "); " << strerror(execvErrno) << "; Check file and try invoking locally."; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - setPid(getpid()); - setPPid(getppid()); + if (aRet == -1) + exit(-1); + } + else // parent + { + setChPid(aChPid); // This is the child PID + cout << "Child PID is " << this->getChPid() << endl; + close(fFIFO[0]); // close the READER of PARENT + // now we can send all the data thru FIFO[1], writer of PARENT + } - errno = 0; - aChPid = fork(); + if (aChPid == 0) + cout << "******** Child finished its work ********" << endl; - if (aChPid == -1) //an error caused - { - int errnum = errno; - ostringstream oss; - oss << getObjId() << " : Error in forking cpimport.bin (errno-" << - errnum << "); " << strerror(errnum); - throw runtime_error( oss.str() ); - } - else if (aChPid == 0) // we are in child - { - int aStartFD = 3; - int aEndFD = fFIFO[1] + 256; - close(fFIFO[1]); //close the WRITER of CHILD - - cout << "Child Process Info: PID = " << getpid() - << " (fFIFO[0], fFIFO[1]) = (" << fFIFO[0] << "," << fFIFO[1] << ")" - << " (StartFD, EndFD) = (" << aStartFD << "," << aEndFD << ")" << endl; - - std::vector Cmds; - //str2Argv(fCmdLineStr, Cmds); // to avoid out-of-scope problem - std::string aCmdLine = fCmdLineStr; - std::istringstream ss(aCmdLine); - std::string arg; - std::vector v2(20, ""); - unsigned int i = 0; - - while (ss >> arg) - { - v2[i++] = arg; - } - - for (unsigned int j = 0; j < i; ++j) - { - Cmds.push_back(const_cast(v2[j].c_str())); - } - - Cmds.push_back(0); //null terminate - //updatePrgmPath(Cmds); - - //NOTE: for debugging - int aSize = Cmds.size(); - - for (int aIdx = 0; aIdx < aSize; ++aIdx) - { - cout << "Args " << Cmds[aIdx] << endl; - } - - cout.flush(); - - close(0); //close stdin for the child - dup2(fFIFO[0], 0); //make stdin be the reading end of the pipe - - //BUG 4410 : hacky solution so that CHLD process get EOF on close of pipe - for (int i = aStartFD; i < aEndFD; i++) close(i); - - errno = 0; - int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works with full Path - //int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works if $PATH has cpimport - - int execvErrno = errno; - cout << "Return status of cpimport is " << aRet << endl; - cout.flush(); - close(fFIFO[0]); // will trigger an EOF on stdin - ostringstream oss; - oss << getObjId() << " : execv error: cpimport.bin invocation failed; " - << "(errno-" << errno << "); " << strerror(execvErrno) << - "; Check file and try invoking locally."; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - - if (aRet == -1) exit(-1); - } - else // parent - { - setChPid(aChPid); // This is the child PID - cout << "Child PID is " << this->getChPid() << endl; - close(fFIFO[0]); //close the READER of PARENT - // now we can send all the data thru FIFO[1], writer of PARENT - } - - if (aChPid == 0) - cout << "******** Child finished its work ********" << endl; - - return true; + return true; } //----------------------------------------------------------------------------- /** @@ -324,120 +322,119 @@ bool WEDataLoader::setupCpimport() // fork the cpimport * **/ -void WEDataLoader::teardownCpimport(bool useStoredWaitPidStatus) // @bug 4267 +void WEDataLoader::teardownCpimport(bool useStoredWaitPidStatus) // @bug 4267 { - fTearDownCpimport = false; //Reset it - //cout << "Tearing down Cpimport" << endl; - int aStatus; + fTearDownCpimport = false; // Reset it + // cout << "Tearing down Cpimport" << endl; + int aStatus; - //cout << "checking fpCfThread value" << endl; - if (fpCfThread) + // cout << "checking fpCfThread value" << endl; + if (fpCfThread) + { + // cout << "checking fpCfThread has a valid value" << endl; + + // wait until we are done with the queued messages + while ((!fpCfThread->isMsgQueueEmpty()) && (!fpCfThread->isStopped())) { - //cout << "checking fpCfThread has a valid value" << endl; - - //wait until we are done with the queued messages - while ((!fpCfThread->isMsgQueueEmpty()) && (!fpCfThread->isStopped())) - { - //cout << "DEBUG : MsgQueue not empty" << endl; - //cannot be too low, since there is a lock in isMsgQueueEmpty() - usleep(2000000); - } - -// while(fpCfThread->isPushing()) -// { -// cout << "DEBUG : still pushing" << endl; -// usleep(100000); -// } - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Message Queue is empty; Stopping CF Thread"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); - } - - fpCfThread->stopThread(); - - while (!fpCfThread->isStopped()) - { - cout << "DEBUG : still not stopped" << endl; - usleep(100000); - } - - delete fpCfThread; - fpCfThread = 0; + // cout << "DEBUG : MsgQueue not empty" << endl; + // cannot be too low, since there is a lock in isMsgQueueEmpty() + usleep(2000000); } + // while(fpCfThread->isPushing()) + // { + // cout << "DEBUG : still pushing" << endl; + // usleep(100000); + // } - closeWritePipe(); - pid_t aPid; - - // @bug 4267 begin: call waitpid() to get job status or use stored job status - // aPid = waitpid(getChPid(), &aStatus, 0); // wait until cpimport finishs - if (useStoredWaitPidStatus) + if (fpSysLog) { - aPid = fWaitPidRc; - aStatus = fWaitPidStatus; + ostringstream oss; + oss << getObjId() << " : Message Queue is empty; Stopping CF Thread"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + fpCfThread->stopThread(); + + while (!fpCfThread->isStopped()) + { + cout << "DEBUG : still not stopped" << endl; + usleep(100000); + } + + delete fpCfThread; + fpCfThread = 0; + } + + closeWritePipe(); + pid_t aPid; + + // @bug 4267 begin: call waitpid() to get job status or use stored job status + // aPid = waitpid(getChPid(), &aStatus, 0); // wait until cpimport finishs + if (useStoredWaitPidStatus) + { + aPid = fWaitPidRc; + aStatus = fWaitPidStatus; + } + else + { + // aPid = waitpid(getChPid(), &aStatus, 0); // wait until cpimport finishs + aPid = waitpid(getChPid(), &aStatus, WNOHANG); // wait until cpimport finishs + int aIdx = 0; + + while ((aPid == 0) && (aIdx < 25 * MAX_QSIZE)) // Do not loop infinitly + { + usleep(2000000); + aPid = waitpid(getChPid(), &aStatus, WNOHANG); + cout << "Inside tearDown waitpid rc[" << aIdx << "] = " << aPid << endl; + ++aIdx; + } + } + + // @bug 4267 end // BP - added -1 as per DMC comment below + if ((aPid == getChPid()) || (aPid == -1)) // @bug 4267 (DMC-shouldn't we check for aPid of -1?) + { + setChPid(0); + + if ((WIFEXITED(aStatus)) && (WEXITSTATUS(aStatus) == 0)) + { + cout << "\tCpimport exit on success" << endl; + ostringstream oss; + oss << getObjId() << " : cpimport exit on success"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + + onCpimportSuccess(); } else { - //aPid = waitpid(getChPid(), &aStatus, 0); // wait until cpimport finishs - aPid = waitpid(getChPid(), &aStatus, WNOHANG); // wait until cpimport finishs - int aIdx = 0; + int termsig = (WIFSIGNALED(aStatus) ? WTERMSIG(aStatus) : -1); - while ((aPid == 0) && (aIdx < 25 * MAX_QSIZE)) //Do not loop infinitly - { - usleep(2000000); - aPid = waitpid(getChPid(), &aStatus, WNOHANG); - cout << "Inside tearDown waitpid rc[" << aIdx << "] = " << aPid << endl; - ++aIdx; - } - } - - // @bug 4267 end // BP - added -1 as per DMC comment below - if ((aPid == getChPid()) || (aPid == -1)) // @bug 4267 (DMC-shouldn't we check for aPid of -1?) - { - setChPid(0); - - if ((WIFEXITED(aStatus)) && (WEXITSTATUS(aStatus) == 0)) - { - cout << "\tCpimport exit on success" << endl; - ostringstream oss; - oss << getObjId() << " : cpimport exit on success"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - - onCpimportSuccess(); - } - else - { - int termsig = (WIFSIGNALED(aStatus) ? WTERMSIG(aStatus) : -1); - - if (!fForceKill) - { - cout << "\tCpimport exit on failure (signal " << termsig << ')' << endl; - ostringstream oss; - oss << getObjId() << " : cpimport exit on failure (signal " << termsig << ')'; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - onCpimportFailure(); - } - else - { - cout << "\tCpimport exit on Force Kill!!" << endl; - ostringstream oss; - oss << getObjId() << " : cpimport exit on Force kill!!"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - onCpimportSuccess(); - } - } + if (!fForceKill) + { + cout << "\tCpimport exit on failure (signal " << termsig << ')' << endl; + ostringstream oss; + oss << getObjId() << " : cpimport exit on failure (signal " << termsig << ')'; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + onCpimportFailure(); + } + else + { + cout << "\tCpimport exit on Force Kill!!" << endl; + ostringstream oss; + oss << getObjId() << " : cpimport exit on Force kill!!"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + onCpimportSuccess(); + } } + } } //----------------------------------------------------------------------------- /** @@ -448,78 +445,76 @@ void WEDataLoader::teardownCpimport(bool useStoredWaitPidStatus) // @bug 4267 // void WEDataLoader::pushData2Cpimport(ByteStream& Ibs) { - if (Ibs.length() > 0) + if (Ibs.length() > 0) + { + int aLen = Ibs.length(); + char* pStart = reinterpret_cast(Ibs.buf()); + char* pEnd = pStart + aLen; + char* pPtr = pStart; + + while (pPtr < pEnd) { - int aLen = Ibs.length(); - char* pStart = reinterpret_cast(Ibs.buf()); - char* pEnd = pStart + aLen; - char* pPtr = pStart; + // if(pEnd > (pPtr + MAX_LEN)) + // { + // int aRet = write(fFIFO[1], pPtr, MAX_LEN); + // if(aRet == -1) throw runtime_error("Pipe write error"); + // //write(fFIFO[1], Ibs.buf(), Ibs.length()); + // pPtr += MAX_LEN; + // } + // else + // { + // int aStrLen = pEnd - pPtr; + // int aRet = write(fFIFO[1], pPtr, aStrLen); + // if(aRet == -1) throw runtime_error("Pipe write error"); + // pPtr += aStrLen; + // } - while (pPtr < pEnd) + try + { + int aRet = write(fFIFO[1], pPtr, pEnd - pPtr); + + if (aRet < 0) { - // if(pEnd > (pPtr + MAX_LEN)) - // { - // int aRet = write(fFIFO[1], pPtr, MAX_LEN); - // if(aRet == -1) throw runtime_error("Pipe write error"); - // //write(fFIFO[1], Ibs.buf(), Ibs.length()); - // pPtr += MAX_LEN; - // } - // else - // { - // int aStrLen = pEnd - pPtr; - // int aRet = write(fFIFO[1], pPtr, aStrLen); - // if(aRet == -1) throw runtime_error("Pipe write error"); - // pPtr += aStrLen; - // } + if (!fPipeErr) + { + int e = errno; + fPipeErr = true; + std::string aStr = "pushing data : PIPE error ........."; - try - { - int aRet = write(fFIFO[1], pPtr, pEnd - pPtr); - - if (aRet < 0) - { - if (!fPipeErr) - { - int e = errno; - fPipeErr = true; - std::string aStr = "pushing data : PIPE error ........."; - - char errMsgBuf[160]; + char errMsgBuf[160]; #if STRERROR_R_CHAR_P - const char* pErrMsg = strerror_r( - e, errMsgBuf, sizeof(errMsgBuf)); + const char* pErrMsg = strerror_r(e, errMsgBuf, sizeof(errMsgBuf)); - if (pErrMsg) - aStr += pErrMsg; + if (pErrMsg) + aStr += pErrMsg; #else - int errMsgRc = strerror_r(e, errMsgBuf, sizeof(errMsgBuf)); + int errMsgRc = strerror_r(e, errMsgBuf, sizeof(errMsgBuf)); - if (errMsgRc == 0) - aStr += errMsgBuf; + if (errMsgRc == 0) + aStr += errMsgBuf; #endif - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - } + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + } - throw runtime_error("Pipe Error - cpimport.bin exited already!!"); - } - - pPtr += aRet; - } - catch (...) - { - //std::string aStr = "pushing data PIPE error ........."; - //logging::Message::Args errMsgArgs; - //errMsgArgs.add(aStr); - //fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - throw runtime_error("Pipe Error - cpimport.bin exited already!!"); - } + throw runtime_error("Pipe Error - cpimport.bin exited already!!"); } + pPtr += aRet; + } + catch (...) + { + // std::string aStr = "pushing data PIPE error ........."; + // logging::Message::Args errMsgArgs; + // errMsgArgs.add(aStr); + // fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + throw runtime_error("Pipe Error - cpimport.bin exited already!!"); + } } + } } //----------------------------------------------------------------------------- @@ -530,12 +525,12 @@ void WEDataLoader::pushData2Cpimport(ByteStream& Ibs) * **/ -void WEDataLoader::closeWritePipe() +void WEDataLoader::closeWritePipe() { - cout << "Going to close call = " << fFIFO[1] << endl; - //NOTE this will flush the file buffer and close it. - int aRet = close(fFIFO[1]); // will trigger EOD - cout << "----- closed both pipes -------- aRet = " << aRet << endl; + cout << "Going to close call = " << fFIFO[1] << endl; + // NOTE this will flush the file buffer and close it. + int aRet = close(fFIFO[1]); // will trigger EOD + cout << "----- closed both pipes -------- aRet = " << aRet << endl; } //----------------------------------------------------------------------------- @@ -550,17 +545,17 @@ void WEDataLoader::closeWritePipe() void WEDataLoader::str2Argv(std::string CmdLine, std::vector& V) { - std::istringstream ss(CmdLine); - std::string arg; - std::vector v2; + std::istringstream ss(CmdLine); + std::string arg; + std::vector v2; - while (ss >> arg) - { - v2.push_back(arg); - V.push_back(const_cast(v2.back().c_str())); - } + while (ss >> arg) + { + v2.push_back(arg); + V.push_back(const_cast(v2.back().c_str())); + } - V.push_back(0); //null terminate + V.push_back(0); // null terminate } //----------------------------------------------------------------------------- @@ -572,91 +567,88 @@ void WEDataLoader::str2Argv(std::string CmdLine, std::vector& V) void WEDataLoader::onCpimportSuccess() { + ByteStream obs; - ByteStream obs; + cout << "Sending BRMRPT" << endl; + obs << (ByteStream::byte)WE_CLT_SRV_BRMRPT; + obs << (ByteStream::byte)fPmId; // PM id + // for testing + // std::string fRptFileName("ReportFile.txt"); + BrmReportParser aBrmRptParser; + bool aRet = aBrmRptParser.serialize(fBrmRptFileName, obs); - cout << "Sending BRMRPT" << endl; - obs << (ByteStream::byte)WE_CLT_SRV_BRMRPT; - obs << (ByteStream::byte)fPmId; // PM id - // for testing - //std::string fRptFileName("ReportFile.txt"); - BrmReportParser aBrmRptParser; - bool aRet = aBrmRptParser.serialize(fBrmRptFileName, obs); - - if (aRet) - { - boost::mutex::scoped_lock aLock(fClntMsgMutex); - //aBrmRptParser.unserialize(obs); - was for testing - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); - cout << "Finished Sending BRMRPT" << endl; - } - else - { - cout << "Failed to serialize BRMRpt " << endl; - } - -// if(remove(fBrmRptFileName.c_str()) != 0) -// cout <<"Failed to delete BRMRpt File "<< fBrmRptFileName << endl; - //usleep(1000000); //sleep 1 second. - - obs.reset(); - obs << (ByteStream::byte)WE_CLT_SRV_CPIPASS; - obs << (ByteStream::byte)fPmId; // PM id + if (aRet) + { boost::mutex::scoped_lock aLock(fClntMsgMutex); + // aBrmRptParser.unserialize(obs); - was for testing updateTxBytes(obs.length()); try { - fRef.fIos.write(obs); + fRef.fIos.write(obs); } catch (...) { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } + cout << "Broken Pipe .." << endl; + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + } } aLock.unlock(); + cout << "Finished Sending BRMRPT" << endl; + } + else + { + cout << "Failed to serialize BRMRpt " << endl; + } - cout << "Sent CPIPASS info" << endl; + // if(remove(fBrmRptFileName.c_str()) != 0) + // cout <<"Failed to delete BRMRpt File "<< fBrmRptFileName << endl; + // usleep(1000000); //sleep 1 second. + + obs.reset(); + obs << (ByteStream::byte)WE_CLT_SRV_CPIPASS; + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : onCpimportSuccess BrmReport Send"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } + aLock.unlock(); + + cout << "Sent CPIPASS info" << endl; + + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : onCpimportSuccess BrmReport Send"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } } //----------------------------------------------------------------------------- @@ -667,56 +659,54 @@ void WEDataLoader::onCpimportSuccess() **/ void WEDataLoader::onCpimportFailure() { - // Send failure notice back to the parent splitter job - sendCpimportFailureNotice(); + // Send failure notice back to the parent splitter job + sendCpimportFailureNotice(); - //Even if we failed, we have failure info in BRMRPT - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_BRMRPT; - obs << (ByteStream::byte)fPmId; // PM id - BrmReportParser aBrmRptParser; - bool aRet = aBrmRptParser.serialize(fBrmRptFileName, obs); + // Even if we failed, we have failure info in BRMRPT + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_BRMRPT; + obs << (ByteStream::byte)fPmId; // PM id + BrmReportParser aBrmRptParser; + bool aRet = aBrmRptParser.serialize(fBrmRptFileName, obs); - if (aRet) + if (aRet) + { + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try { - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (std::exception& ex) - { - cout << "Broken Pipe .." << ex.what() << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed; " << - ex.what(); - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, - logging::M0000); - } - } - - aLock.unlock(); - cout << "Finished Sending BRMRPT" << endl; + fRef.fIos.write(obs); } - - if (remove(fBrmRptFileName.c_str()) != 0) - cout << "Failed to delete BRMRpt File " << fBrmRptFileName << endl; - - if (fpSysLog) + catch (std::exception& ex) { + cout << "Broken Pipe .." << ex.what() << endl; + + if (fpSysLog) + { ostringstream oss; - oss << getObjId() << " : onCpimportFailure BrmReport Send"; + oss << getObjId() << " : Broken Pipe : socket write failed; " << ex.what(); logging::Message::Args errMsgArgs; errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + } } + + aLock.unlock(); + cout << "Finished Sending BRMRPT" << endl; + } + + if (remove(fBrmRptFileName.c_str()) != 0) + cout << "Failed to delete BRMRpt File " << fBrmRptFileName << endl; + + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : onCpimportFailure BrmReport Send"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } } //----------------------------------------------------------------------------- @@ -724,29 +714,29 @@ void WEDataLoader::onCpimportFailure() //----------------------------------------------------------------------------- void WEDataLoader::sendCpimportFailureNotice() { - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_CPIFAIL; - obs << (ByteStream::byte)fPmId; // PM id - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_CPIFAIL; + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - } + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); } + } } //----------------------------------------------------------------------------- @@ -756,117 +746,117 @@ void WEDataLoader::sendCpimportFailureNotice() */ void WEDataLoader::onReceiveKeepAlive(ByteStream& Ibs) { - /* - // TODO comment out when we done with debug - if(fpSysLog) - { - ostringstream oss; - oss << getObjId() <<" : Received KEEPALIVE"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - */ - //cout << "Received KEEPALIVE" << endl; - // NOTE only seldom a KEEPALIVE receives, - // so nothing wrong in responding with a KEEPALIVE. - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_KEEPALIVE; - obs << (ByteStream::byte)fPmId; // PM id - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); + /* + // TODO comment out when we done with debug + if(fpSysLog) + { + ostringstream oss; + oss << getObjId() <<" : Received KEEPALIVE"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + } + */ + // cout << "Received KEEPALIVE" << endl; + // NOTE only seldom a KEEPALIVE receives, + // so nothing wrong in responding with a KEEPALIVE. + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_KEEPALIVE; + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); - try + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; + + if (fpSysLog) { - fRef.fIos.write(obs); + ostringstream oss; + oss << getObjId() << ": Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } - catch (...) + } + + aLock.unlock(); + + // @bug 4267 begin + int aStatus; + pid_t aPid; + + if (getChPid() > 0) + { + aPid = waitpid(getChPid(), &aStatus, WNOHANG); // wait until cpimport finishs + + if (aPid != 0) { + cout << "waitpid(" << getChPid() << "): rc-" << aPid << "; status-" << aStatus << "; exited-" + << (WIFEXITED(aStatus)) << endl; + } + + if ((aPid == getChPid()) || (aPid == -1)) + { + fTearDownCpimport = true; + fWaitPidRc = aPid; + fWaitPidStatus = aStatus; + } + } + + // @bug 4267 end + + if (fTearDownCpimport) + { + // fTearDownCpimport = false; //Reset it // commented out to use the flag in EOD + cout << "Cpimport terminated " << endl; + + if (0 == getMode()) + onReceiveEod(Ibs); + else if (1 == getMode()) // mode 1 has to drive from UM + { + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_EOD; + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try + { + fRef.fIos.write(obs); + } + catch (...) + { cout << "Broken Pipe .." << endl; if (fpSysLog) { - ostringstream oss; - oss << getObjId() << ": Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + ostringstream oss; + oss << getObjId() << ": Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } + + aLock.unlock(); } - - aLock.unlock(); - - // @bug 4267 begin - int aStatus; - pid_t aPid; - - if (getChPid() > 0) + else if (2 == getMode()) { - aPid = waitpid(getChPid(), &aStatus, WNOHANG); // wait until cpimport finishs - - if (aPid != 0) - { - cout << "waitpid(" << getChPid() << "): rc-" << aPid << - "; status-" << aStatus << "; exited-" << - (WIFEXITED(aStatus)) << endl; - } - - if ((aPid == getChPid()) || (aPid == -1)) - { - fTearDownCpimport = true; - fWaitPidRc = aPid; - fWaitPidStatus = aStatus; - } + // if(getChPid()) teardownCpimport(true); // @bug 4267 + if (getChPid()) + teardownCpimport(fTearDownCpimport); // @bug 4267 BP } - - // @bug 4267 end - - if (fTearDownCpimport) - { - //fTearDownCpimport = false; //Reset it // commented out to use the flag in EOD - cout << "Cpimport terminated " << endl; - - if (0 == getMode()) onReceiveEod(Ibs); - else if (1 == getMode()) //mode 1 has to drive from UM - { - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_EOD; - obs << (ByteStream::byte)fPmId; // PM id - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << ": Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); - } - else if (2 == getMode()) - { - //if(getChPid()) teardownCpimport(true); // @bug 4267 - if (getChPid()) teardownCpimport(fTearDownCpimport); // @bug 4267 BP - } - } - else - { - //if(1 == getMode()) - // if(fpCfThread) cout << "Queue Size = " << fpCfThread->getQueueSize() << endl; - } - + } + else + { + // if(1 == getMode()) + // if(fpCfThread) cout << "Queue Size = " << fpCfThread->getQueueSize() << endl; + } } //----------------------------------------------------------------------------- /** @@ -876,58 +866,56 @@ void WEDataLoader::onReceiveKeepAlive(ByteStream& Ibs) void WEDataLoader::onReceiveData(ByteStream& Ibs) { + if ((0 == getMode()) && (fDataDumpFile.is_open())) + { + // Will write to the output file. + fDataDumpFile << Ibs; + sendDataRequest(); + } + else if (1 == getMode()) + { + // commented out since we are going to use seperate thread + // pushData2Cpimport(Ibs); - if ((0 == getMode()) && (fDataDumpFile.is_open())) + if (fpCfThread) { - // Will write to the output file. - fDataDumpFile << Ibs; + fpCfThread->add2MsgQueue(Ibs); + // sendDataRequest(); // Need to control Queue Size + // Bug 5031 : Will only send 1 rqst for a batch to cpimport.bin + // if(fpCfThread->getQueueSize()getQueueSize()getQueueSize() : 0; + + // Bug 5031 : If Q size goes above 100 (2*250); + if (aQsz < MAX_QSIZE) sendDataRequest(); - } - else if ( 1 == getMode()) - { - // commented out since we are going to use seperate thread - //pushData2Cpimport(Ibs); - if (fpCfThread) + if (aQsz > 1.5 * MAX_QSIZE) // > 2*250 + { + cout << "WARNING : Data Queuing up : QSize = " << aQsz << endl; + + if (fpSysLog) { - fpCfThread->add2MsgQueue(Ibs); - //sendDataRequest(); // Need to control Queue Size - // Bug 5031 : Will only send 1 rqst for a batch to cpimport.bin - //if(fpCfThread->getQueueSize()getQueueSize()getQueueSize() : 0; - - // Bug 5031 : If Q size goes above 100 (2*250); - if (aQsz < MAX_QSIZE) sendDataRequest(); - - if (aQsz > 1.5 * MAX_QSIZE) // > 2*250 - { - cout << "WARNING : Data Queuing up : QSize = " << aQsz << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << "WARNING : Data Queuing up : QSize = " << aQsz; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } + ostringstream oss; + oss << getObjId() << "WARNING : Data Queuing up : QSize = " << aQsz; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } - - } - else if ( 2 == getMode()) - { - cout << "onReceiveData : In Mode 2 NO data suppose to receive" << endl; + } } + } + else if (2 == getMode()) + { + cout << "onReceiveData : In Mode 2 NO data suppose to receive" << endl; + } - -// ByteStream obs; -// obs << (ByteStream::byte)WE_CLT_SRV_DATARQST; -// obs << (ByteStream::byte)fPmId; // PM id -// updateTxBytes(obs.length()); -// fRef.fIos.write(obs); + // ByteStream obs; + // obs << (ByteStream::byte)WE_CLT_SRV_DATARQST; + // obs << (ByteStream::byte)fPmId; // PM id + // updateTxBytes(obs.length()); + // fRef.fIos.write(obs); } //----------------------------------------------------------------------------- @@ -937,65 +925,64 @@ void WEDataLoader::onReceiveData(ByteStream& Ibs) */ void WEDataLoader::onReceiveEod(ByteStream& Ibs) { + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : onReceiveEOD : child ID = " << getChPid(); + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + cout << "Received EOD " << endl; + + if (0 == getMode()) + { + fDataDumpFile.close(); + } + + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_EOD; + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; + if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : onReceiveEOD : child ID = " << getChPid(); - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } - cout << "Received EOD " << endl; + aLock.unlock(); - if (0 == getMode()) + // if(( 1 == getMode())||( 2 == getMode())) + if (1 == getMode()) // BUG 4370 - seperated mode 1 & 2 + { + // if(getChPid()) teardownCpimport(false); // @bug 4267 + if (getChPid()) + teardownCpimport(fTearDownCpimport); // @bug 4267 //BP changed to send the correct flag + } + else if (2 == getMode()) // BUG 4370 + { + if (getChPid()) { - fDataDumpFile.close(); + kill(getChPid(), SIGINT); // BUG 4370 + fForceKill = true; + teardownCpimport(fTearDownCpimport); // BUG 4370 } - - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_EOD; - obs << (ByteStream::byte)fPmId; // PM id - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); - - //if(( 1 == getMode())||( 2 == getMode())) - if (1 == getMode()) //BUG 4370 - seperated mode 1 & 2 - { - //if(getChPid()) teardownCpimport(false); // @bug 4267 - if (getChPid()) teardownCpimport(fTearDownCpimport); // @bug 4267 //BP changed to send the correct flag - } - else if (2 == getMode()) //BUG 4370 - { - if (getChPid()) - { - kill(getChPid(), SIGINT); //BUG 4370 - fForceKill = true; - teardownCpimport(fTearDownCpimport); //BUG 4370 - } - } - - + } } //----------------------------------------------------------------------------- /** @@ -1004,9 +991,9 @@ void WEDataLoader::onReceiveEod(ByteStream& Ibs) */ void WEDataLoader::onReceiveCmd(ByteStream& bs) { - //TODO - can be cpimport cmd or server cmd, for now write to a file - ByteStream::byte aCmdId; - bs >> aCmdId; + // TODO - can be cpimport cmd or server cmd, for now write to a file + ByteStream::byte aCmdId; + bs >> aCmdId; } //----------------------------------------------------------------------------- /** @@ -1015,73 +1002,69 @@ void WEDataLoader::onReceiveCmd(ByteStream& bs) */ void WEDataLoader::onReceiveMode(ByteStream& Ibs) { - // Assigning it here since WEDataLoader constructor is called multiple times - // while coping in readthread class. - if (!fpSysLog) - { - fpSysLog = SimpleSysLog::instance(); - fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SRV)); - } + // Assigning it here since WEDataLoader constructor is called multiple times + // while coping in readthread class. + if (!fpSysLog) + { + fpSysLog = SimpleSysLog::instance(); + fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SRV)); + } - Ibs >> (ByteStream::quadbyte&)fMode; - cout << "Setting fMode = " << fMode << endl; + Ibs >> (ByteStream::quadbyte&)fMode; + cout << "Setting fMode = " << fMode << endl; + + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : onReceiveMode() Setting fMode = " << fMode; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + char aName[64]; + snprintf(aName, sizeof(aName), "ModuleDBRootCount%d-3", fPmId); + + string aStrDbRootCnt = config::Config::makeConfig()->getConfig("SystemModuleConfig", aName); + cout << "DbRootCnt = " << aStrDbRootCnt << endl; + ByteStream::byte aDbCnt = (ByteStream::byte)atoi(aStrDbRootCnt.c_str()); + + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : onReceiveMode() DbRoot Count = " + aStrDbRootCnt; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + // Send No. of DBRoots to Client + ByteStream aObs; + aObs << (ByteStream::byte)WE_CLT_SRV_DBRCNT; + aObs << (ByteStream::byte)fPmId; + aObs << (ByteStream::byte)aDbCnt; + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(aObs.length()); + + try + { + fRef.fIos.write(aObs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : onReceiveMode() Setting fMode = " << fMode; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } - char aName[64]; - snprintf(aName, sizeof(aName), "ModuleDBRootCount%d-3", fPmId); - - string aStrDbRootCnt = config::Config::makeConfig()->getConfig( - "SystemModuleConfig", aName); - cout << "DbRootCnt = " << aStrDbRootCnt << endl; - ByteStream::byte aDbCnt = (ByteStream::byte)atoi(aStrDbRootCnt.c_str()); - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : onReceiveMode() DbRoot Count = " + aStrDbRootCnt; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); - } - - - //Send No. of DBRoots to Client - ByteStream aObs; - aObs << (ByteStream::byte)WE_CLT_SRV_DBRCNT; - aObs << (ByteStream::byte)fPmId; - aObs << (ByteStream::byte)aDbCnt; - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(aObs.length()); - - try - { - fRef.fIos.write(aObs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - - } - - aLock.unlock(); - + aLock.unlock(); } //----------------------------------------------------------------------------- /** @@ -1092,8 +1075,8 @@ void WEDataLoader::onReceiveMode(ByteStream& Ibs) void WEDataLoader::onReceiveAck(ByteStream& Ibs) { - // All is good - // update the status + // All is good + // update the status } //----------------------------------------------------------------------------- /** @@ -1103,7 +1086,7 @@ void WEDataLoader::onReceiveAck(ByteStream& Ibs) */ void WEDataLoader::onReceiveNak(ByteStream& Ibs) { - // TODO - handle the problem + // TODO - handle the problem } //----------------------------------------------------------------------------- /** @@ -1113,7 +1096,7 @@ void WEDataLoader::onReceiveNak(ByteStream& Ibs) */ void WEDataLoader::onReceiveError(ByteStream& Ibs) { - // TODO - handle the failure situation. + // TODO - handle the failure situation. } //------------------------------------------------------------------------------ // onReceiveCmdLineArgs - do what ever need to do with command line args @@ -1126,551 +1109,532 @@ void WEDataLoader::onReceiveError(ByteStream& Ibs) void WEDataLoader::onReceiveCmdLineArgs(ByteStream& Ibs) { - Ibs >> fCmdLineStr; - cout << "CMD LINE ARGS came in " << fCmdLineStr << endl; + Ibs >> fCmdLineStr; + cout << "CMD LINE ARGS came in " << fCmdLineStr << endl; + + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : CMD LINE ARGS came in " << fCmdLineStr; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + ByteStream obs; + + // TODO - Need to check all clear for starting CPI + if (0 != getMode()) + { + obs << (ByteStream::byte)WE_CLT_SRV_STARTCPI; + } + else + { + obs << (ByteStream::byte)WE_CLT_SRV_DATARQST; + } + + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : CMD LINE ARGS came in " << fCmdLineStr; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } - ByteStream obs; - - //TODO - Need to check all clear for starting CPI - if (0 != getMode()) - { - obs << (ByteStream::byte)WE_CLT_SRV_STARTCPI; - } - else - { - obs << (ByteStream::byte)WE_CLT_SRV_DATARQST; - } - - obs << (ByteStream::byte) fPmId; // PM id - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); - + aLock.unlock(); } //----------------------------------------------------------------------------- void WEDataLoader::onReceiveStartCpimport() { - cout << "Start Cpimport command reached!!" << endl; + cout << "Start Cpimport command reached!!" << endl; + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : Start Cpimport command reached!!"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + try + { + setupCpimport(); + + if (1 == getMode()) // create a thread to handle the data feeding part + { + fpCfThread = new WECpiFeederThread(*this); + fpCfThread->startFeederThread(); + } + } + catch (std::exception& ex) + { + // send an CPI FAIL command back to splitter if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : Start Cpimport command reached!!"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + logging::Message::Args errMsgArgs; + errMsgArgs.add(ex.what()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + + sendCpimportFailureNotice(); + return; } + } - try - { - setupCpimport(); - - if (1 == getMode()) // create a thread to handle the data feeding part - { - fpCfThread = new WECpiFeederThread(*this); - fpCfThread->startFeederThread(); - } - } - catch (std::exception& ex) - { - // send an CPI FAIL command back to splitter - if (fpSysLog) - { - logging::Message::Args errMsgArgs; - errMsgArgs.add(ex.what()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - - sendCpimportFailureNotice(); - return; - } - } - - if (1 == getMode()) // In mode 2/0 we do not rqst data. - { - sendDataRequest(); - } - - // We need to respond to KEEP ALIVES - //else if(2 == getMode()) // Now we wait till cpimport comes back - //{ - // if(getChPid()) - // teardownCpimport(); - //} - + if (1 == getMode()) // In mode 2/0 we do not rqst data. + { + sendDataRequest(); + } + // We need to respond to KEEP ALIVES + // else if(2 == getMode()) // Now we wait till cpimport comes back + //{ + // if(getChPid()) + // teardownCpimport(); + //} } //----------------------------------------------------------------------------- void WEDataLoader::onReceiveBrmRptFileName(ByteStream& Ibs) { - Ibs >> fBrmRptFileName; - cout << "Brm Rpt Filename Arrived " << fBrmRptFileName << endl; + Ibs >> fBrmRptFileName; + cout << "Brm Rpt Filename Arrived " << fBrmRptFileName << endl; - //BUG 4645 - string::size_type idx = fBrmRptFileName.find_last_of('/'); + // BUG 4645 + string::size_type idx = fBrmRptFileName.find_last_of('/'); - if (idx > 0 && idx < string::npos) + if (idx > 0 && idx < string::npos) + { + string dirname(fBrmRptFileName, 0, idx); + struct stat st; + + if (stat(dirname.c_str(), &st) != 0) { - string dirname(fBrmRptFileName, 0, idx); - struct stat st; - - if (stat(dirname.c_str(), &st) != 0) - { - cout << "Creating directory : " << dirname << endl; - boost::filesystem::create_directories(dirname.c_str()); - } - } - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Brm Rpt Filename Arrived " << fBrmRptFileName; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + cout << "Creating directory : " << dirname << endl; + boost::filesystem::create_directories(dirname.c_str()); } + } + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : Brm Rpt Filename Arrived " << fBrmRptFileName; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } } //----------------------------------------------------------------------------- void WEDataLoader::onReceiveCleanup(ByteStream& Ibs) { + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : OnReceiveCleanup arrived"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + std::string aErrMsg; + + WE_ClearTableLockCmd aClrTblLockCmd("DataLoader"); + int aRet = aClrTblLockCmd.processCleanup(Ibs, aErrMsg); + + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_CLEANUP; + obs << (ByteStream::byte)fPmId; // PM id + + if (aRet == 0) + obs << (ByteStream::byte)1; // cleanup success + else + obs << (ByteStream::byte)0; // cleanup failed + + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; + if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : OnReceiveCleanup arrived"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } - std::string aErrMsg; - - WE_ClearTableLockCmd aClrTblLockCmd("DataLoader"); - int aRet = aClrTblLockCmd.processCleanup(Ibs, aErrMsg); - - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_CLEANUP; - obs << (ByteStream::byte) fPmId; // PM id - - if (aRet == 0) - obs << (ByteStream::byte)1; // cleanup success - else - obs << (ByteStream::byte)0; // cleanup failed - - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - - } - - aLock.unlock(); - + aLock.unlock(); } - //----------------------------------------------------------------------------- - - void WEDataLoader::onReceiveRollback(ByteStream& Ibs) { + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : OnReceiveRollback arrived"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + } + + std::string aErrMsg; + + WE_ClearTableLockCmd aClrTblLockCmd("DataLoader"); + int aRet = aClrTblLockCmd.processRollback(Ibs, aErrMsg); + + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_ROLLBACK; + obs << (ByteStream::byte)fPmId; // PM id + + if (aRet == 0) + obs << (ByteStream::byte)1; // Rollback success + else + obs << (ByteStream::byte)0; // Rollback failed + + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; + if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : OnReceiveRollback arrived"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_DEBUG, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } + aLock.unlock(); +} +//----------------------------------------------------------------------------- - std::string aErrMsg; +void WEDataLoader::onReceiveImportFileName(ByteStream& Ibs) +{ + bool aGoodFile = true; + std::string aFileName; + Ibs >> aFileName; - WE_ClearTableLockCmd aClrTblLockCmd("DataLoader"); - int aRet = aClrTblLockCmd.processRollback(Ibs, aErrMsg); + // BUG 4245 : Need to check the file or path exists + { + std::fstream aFin; + aFin.open(aFileName.c_str(), std::ios::in); - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_ROLLBACK; - obs << (ByteStream::byte) fPmId; // PM id - - if (aRet == 0) - obs << (ByteStream::byte)1; // Rollback success - else - obs << (ByteStream::byte)0; // Rollback failed - - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try + if (aFin.is_open()) // File exist, send an ERROR immediately { + // file exists + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_IMPFILEERROR; + obs << (ByteStream::byte)fPmId; + updateTxBytes(obs.length()); + boost::mutex::scoped_lock aLock(fClntMsgMutex); + + try + { fRef.fIos.write(obs); - } - catch (...) - { + } + catch (...) + { cout << "Broken Pipe .." << endl; if (fpSysLog) { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } + + aGoodFile = false; + aLock.unlock(); } - aLock.unlock(); + aFin.close(); + } -} -//----------------------------------------------------------------------------- + if (aGoodFile) + { + fDataDumpFile.open(aFileName.c_str(), std::ios::app); - -void WEDataLoader::onReceiveImportFileName(ByteStream& Ibs) -{ - bool aGoodFile = true; - std::string aFileName; - Ibs >> aFileName; - - //BUG 4245 : Need to check the file or path exists + // BUG 4245 : If file dir is not existing, we need to fail this import + if (!fDataDumpFile.good()) { - std::fstream aFin; - aFin.open(aFileName.c_str(), std::ios::in); + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_IMPFILEERROR; + obs << (ByteStream::byte)fPmId; // PM id + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); - if (aFin.is_open()) // File exist, send an ERROR immediately + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; + + if (fpSysLog) { - // file exists - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_IMPFILEERROR; - obs << (ByteStream::byte)fPmId; - updateTxBytes(obs.length()); - boost::mutex::scoped_lock aLock(fClntMsgMutex); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - - } - - aGoodFile = false; - aLock.unlock(); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } - aFin.close(); + aLock.unlock(); } - - if (aGoodFile) - { - fDataDumpFile.open(aFileName.c_str(), std::ios::app); - - //BUG 4245 : If file dir is not existing, we need to fail this import - if (!fDataDumpFile.good()) - { - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_IMPFILEERROR; - obs << (ByteStream::byte)fPmId; // PM id - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); - - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); - } - } - + } } //------------------------------------------------------------------------------ void WEDataLoader::onReceiveJobId(ByteStream& Ibs) { - std::string aJobFileName; - Ibs >> aJobFileName; + std::string aJobFileName; + Ibs >> aJobFileName; - cout << "Incoming JobFileName : " << aJobFileName << endl; + cout << "Incoming JobFileName : " << aJobFileName << endl; - //BUG 4645 - string::size_type idx = aJobFileName.find_last_of('/'); + // BUG 4645 + string::size_type idx = aJobFileName.find_last_of('/'); - if (idx > 0 && idx < string::npos) + if (idx > 0 && idx < string::npos) + { + string dirname(aJobFileName, 0, idx); + struct stat st; + + if (stat(dirname.c_str(), &st) != 0) { - string dirname(aJobFileName, 0, idx); - struct stat st; - - if (stat(dirname.c_str(), &st) != 0) - { - cout << "Creating directory : " << dirname << endl; - boost::filesystem::create_directories(dirname.c_str()); - } + cout << "Creating directory : " << dirname << endl; + boost::filesystem::create_directories(dirname.c_str()); } + } - fJobFile.open(aJobFileName.c_str()); - + fJobFile.open(aJobFileName.c_str()); } //------------------------------------------------------------------------------ void WEDataLoader::onReceiveJobData(ByteStream& Ibs) { - // Will write to the output file. - std::string aData; - Ibs >> aData; - fJobFile << aData; - fJobFile.close(); + // Will write to the output file. + std::string aData; + Ibs >> aData; + fJobFile << aData; + fJobFile.close(); } //------------------------------------------------------------------------------ void WEDataLoader::onReceiveErrFileRqst(ByteStream& Ibs) { - std::string aErrFileName; - Ibs >> aErrFileName; - cout << "Error Filename Arrived " << aErrFileName << endl; + std::string aErrFileName; + Ibs >> aErrFileName; + cout << "Error Filename Arrived " << aErrFileName << endl; - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_ERRLOG; - obs << (ByteStream::byte) fPmId; // PM id - obs << aErrFileName; - BrmReportParser aErrFileParser; - bool aRet = aErrFileParser.serialize(aErrFileName, obs); + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_ERRLOG; + obs << (ByteStream::byte)fPmId; // PM id + obs << aErrFileName; + BrmReportParser aErrFileParser; + bool aRet = aErrFileParser.serialize(aErrFileName, obs); - if (aRet) + if (aRet) + { + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try { - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + } } - // delete the temp files - if (remove(aErrFileName.c_str()) != 0 ) - cout << "Failed in removing Error file: " << aErrFileName << endl; + aLock.unlock(); + } + + // delete the temp files + if (remove(aErrFileName.c_str()) != 0) + cout << "Failed in removing Error file: " << aErrFileName << endl; } - - //------------------------------------------------------------------------------ // Process the receipt of a msg containing the contents of a *.bad file. //------------------------------------------------------------------------------ void WEDataLoader::onReceiveBadFileRqst(ByteStream& Ibs) { - std::string aBadFileName; - Ibs >> aBadFileName; - cout << "Error Filename Arrived " << aBadFileName << endl; + std::string aBadFileName; + Ibs >> aBadFileName; + cout << "Error Filename Arrived " << aBadFileName << endl; - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_BADLOG; - obs << (ByteStream::byte) fPmId; // PM id - obs << aBadFileName; - BrmReportParser aBadFileParser; - bool aRet = aBadFileParser.serializeBlocks(aBadFileName, obs); + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_BADLOG; + obs << (ByteStream::byte)fPmId; // PM id + obs << aBadFileName; + BrmReportParser aBadFileParser; + bool aRet = aBadFileParser.serializeBlocks(aBadFileName, obs); - if (aRet) + if (aRet) + { + boost::mutex::scoped_lock aLock(fClntMsgMutex); + updateTxBytes(obs.length()); + + try { - boost::mutex::scoped_lock aLock(fClntMsgMutex); - updateTxBytes(obs.length()); + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; - try - { - fRef.fIos.write(obs); - } - catch (...) - { - cout << "Broken Pipe .." << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); + if (fpSysLog) + { + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + } } - // delete the temp files - if ( remove(aBadFileName.c_str()) != 0) - cout << "Failed in removing Error file: " << aBadFileName << endl; -} + aLock.unlock(); + } + // delete the temp files + if (remove(aBadFileName.c_str()) != 0) + cout << "Failed in removing Error file: " << aBadFileName << endl; +} //------------------------------------------------------------------------------ void WEDataLoader::sendDataRequest() { - int aQsz = (fpCfThread) ? fpCfThread->getQueueSize() : 0; + int aQsz = (fpCfThread) ? fpCfThread->getQueueSize() : 0; - //if(aQsz>MIN_QSIZE) - // Bug 5031 : If Q size goes above 100 (2*50); there is some thing wrong - // will put a warning in info log. Controlled in Cpimport init data rqst cnt - if (aQsz > MAX_QSIZE) // >250 + // if(aQsz>MIN_QSIZE) + // Bug 5031 : If Q size goes above 100 (2*50); there is some thing wrong + // will put a warning in info log. Controlled in Cpimport init data rqst cnt + if (aQsz > MAX_QSIZE) // >250 + { + cout << "WARNING : Data Queuing up : QSize = " << aQsz << endl; + + if (fpSysLog) { - cout << "WARNING : Data Queuing up : QSize = " << aQsz << endl; - - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << "WARNING : Data Queuing up : QSize = " << aQsz; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } + ostringstream oss; + oss << getObjId() << "WARNING : Data Queuing up : QSize = " << aQsz; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } + } - boost::mutex::scoped_lock aLock(fClntMsgMutex); - ByteStream obs; - obs << (ByteStream::byte)WE_CLT_SRV_DATARQST; - obs << (ByteStream::byte)fPmId; // PM id - updateTxBytes(obs.length()); + boost::mutex::scoped_lock aLock(fClntMsgMutex); + ByteStream obs; + obs << (ByteStream::byte)WE_CLT_SRV_DATARQST; + obs << (ByteStream::byte)fPmId; // PM id + updateTxBytes(obs.length()); - try + try + { + fRef.fIos.write(obs); + } + catch (...) + { + cout << "Broken Pipe .." << endl; + + if (fpSysLog) { - fRef.fIos.write(obs); + ostringstream oss; + oss << getObjId() << " : Broken Pipe : socket write failed "; + logging::Message::Args errMsgArgs; + errMsgArgs.add(oss.str()); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); } - catch (...) - { - cout << "Broken Pipe .." << endl; + } - if (fpSysLog) - { - ostringstream oss; - oss << getObjId() << " : Broken Pipe : socket write failed "; - logging::Message::Args errMsgArgs; - errMsgArgs.add(oss.str()); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } - } - - aLock.unlock(); + aLock.unlock(); } - //------------------------------------------------------------------------------ void WEDataLoader::serialize(messageqcpp::ByteStream& b) const { - //TODO to be changed. left it here to understand how to implement - /* - b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN; - ReturnedColumn::serialize(b); // parent class serialize - b << (uint32_t) fOid; - b << fData; - b << static_cast(fReturnAll); - b << (uint32_t) fSequence; - */ + // TODO to be changed. left it here to understand how to implement + /* + b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN; + ReturnedColumn::serialize(b); // parent class serialize + b << (uint32_t) fOid; + b << fData; + b << static_cast(fReturnAll); + b << (uint32_t) fSequence; + */ } //----------------------------------------------------------------------------- void WEDataLoader::unserialize(messageqcpp::ByteStream& b) { - //TODO to be changed. left it here to understand how to implement - /* - ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN); - ReturnedColumn::unserialize(b); // parent class unserialize - b >> (uint32_t&) fOid; - b >> fData; - b >> reinterpret_cast(fReturnAll); - b >> (uint32_t&) fSequence; - */ + // TODO to be changed. left it here to understand how to implement + /* + ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN); + ReturnedColumn::unserialize(b); // parent class unserialize + b >> (uint32_t&) fOid; + b >> fData; + b >> reinterpret_cast(fReturnAll); + b >> (uint32_t&) fSequence; + */ } //----------------------------------------------------------------------------- -} // namespace WriteEngine +} // namespace WriteEngine diff --git a/writeengine/server/we_dataloader.h b/writeengine/server/we_dataloader.h index 6a1117528..73a069de5 100644 --- a/writeengine/server/we_dataloader.h +++ b/writeengine/server/we_dataloader.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_dataloader.h @@ -28,8 +28,6 @@ * */ - - #ifndef WE_DATALOADER_H_ #define WE_DATALOADER_H_ @@ -45,7 +43,6 @@ namespace WriteEngine { - class SplitterReadThread; // This class will go with the read thread & client socket @@ -53,169 +50,174 @@ class SplitterReadThread; class WEDataLoader : public Observer { -public: - explicit WEDataLoader(SplitterReadThread& pSrt ); - virtual ~WEDataLoader(); + public: + explicit WEDataLoader(SplitterReadThread& pSrt); + virtual ~WEDataLoader(); - virtual bool update(Subject* pSub); + virtual bool update(Subject* pSub); -public: - bool setupCpimport(); // fork the cpimport - void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267 - void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue - void closeWritePipe(); - void str2Argv(std::string CmdLine, std::vector& V); + public: + bool setupCpimport(); // fork the cpimport + void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267 + void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue + void closeWritePipe(); + void str2Argv(std::string CmdLine, std::vector& V); + public: + void onReceiveKeepAlive(ByteStream& Ibs); + void onReceiveData(ByteStream& Ibs); + void onReceiveEod(ByteStream& Ibs); // end of data + void onReceiveMode(ByteStream& Ibs); + // void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs); + void onReceiveCmd(ByteStream& bs); // {(ByteStream& Ibs); + void onReceiveAck(ByteStream& Ibs); + void onReceiveNak(ByteStream& Ibs); + void onReceiveError(ByteStream& Ibs); -public: - void onReceiveKeepAlive(ByteStream& Ibs); - void onReceiveData(ByteStream& Ibs); - void onReceiveEod(ByteStream& Ibs); //end of data - void onReceiveMode(ByteStream& Ibs); - //void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs); - void onReceiveCmd(ByteStream& bs);// {(ByteStream& Ibs); - void onReceiveAck(ByteStream& Ibs); - void onReceiveNak(ByteStream& Ibs); - void onReceiveError(ByteStream& Ibs); + void onReceiveJobId(ByteStream& Ibs); + void onReceiveJobData(ByteStream& Ibs); + void onReceiveImportFileName(ByteStream& Ibs); + void onReceiveCmdLineArgs(ByteStream& Ibs); + void onReceiveStartCpimport(); + void onReceiveBrmRptFileName(ByteStream& Ibs); + void onReceiveCleanup(ByteStream& Ibs); + void onReceiveRollback(ByteStream& Ibs); - void onReceiveJobId(ByteStream& Ibs); - void onReceiveJobData(ByteStream& Ibs); - void onReceiveImportFileName(ByteStream& Ibs); - void onReceiveCmdLineArgs(ByteStream& Ibs); - void onReceiveStartCpimport(); - void onReceiveBrmRptFileName(ByteStream& Ibs); - void onReceiveCleanup(ByteStream& Ibs); - void onReceiveRollback(ByteStream& Ibs); + void onReceiveErrFileRqst(ByteStream& Ibs); + void onReceiveBadFileRqst(ByteStream& Ibs); - void onReceiveErrFileRqst(ByteStream& Ibs); - void onReceiveBadFileRqst(ByteStream& Ibs); + void onCpimportSuccess(); + void onCpimportFailure(); - void onCpimportSuccess(); - void onCpimportFailure(); + void sendDataRequest(); + void sendCpimportFailureNotice(); - void sendDataRequest(); - void sendCpimportFailureNotice(); + void serialize(messageqcpp::ByteStream& b) const; + void unserialize(messageqcpp::ByteStream& b); - void serialize(messageqcpp::ByteStream& b) const; - void unserialize(messageqcpp::ByteStream& b); + // setup the signal handlers for the main app + void setupSignalHandlers(); + static void onSigChild(int aInt); + public: + void setMode(int Mode) + { + fMode = Mode; + } + void updateTxBytes(unsigned int Tx) + { + fTxBytes += Tx; + } + void updateRxBytes(unsigned int Rx) + { + fRxBytes += Rx; + } + void setChPid(pid_t pid) + { + fCh_pid = pid; + } + void setPid(pid_t pid) + { + fThis_pid = pid; + } + void setPPid(pid_t pid) + { + fP_pid = pid; + } + void setCmdLineStr(std::string& Str) + { + fCmdLineStr = Str; + } + void setObjId(int ObjId) + { + fObjId = ObjId; + } - // setup the signal handlers for the main app - void setupSignalHandlers(); - static void onSigChild(int aInt); + unsigned int getTxBytes() + { + return fTxBytes; + } + unsigned int getRxBytes() + { + return fRxBytes; + } -public: - void setMode(int Mode) - { - fMode = Mode; - } - void updateTxBytes(unsigned int Tx) - { - fTxBytes += Tx; - } - void updateRxBytes(unsigned int Rx) - { - fRxBytes += Rx; - } - void setChPid(pid_t pid) - { - fCh_pid = pid; - } - void setPid(pid_t pid) - { - fThis_pid = pid; - } - void setPPid(pid_t pid) - { - fP_pid = pid; - } - void setCmdLineStr(std::string& Str) - { - fCmdLineStr = Str; - } - void setObjId(int ObjId) - { - fObjId = ObjId; - } + int getObjId() + { + return fObjId; + } + int getMode() + { + return fMode; + } + pid_t getChPid() + { + return fCh_pid; + } + pid_t getPid() + { + return fThis_pid; + } + pid_t getPPid() + { + return fP_pid; + } + std::string getCmdLineStr() + { + return fCmdLineStr; + } - unsigned int getTxBytes() - { - return fTxBytes; - } - unsigned int getRxBytes() - { - return fRxBytes; - } + private: + SplitterReadThread& fRef; - int getObjId() - { - return fObjId; - } - int getMode() - { - return fMode; - } - pid_t getChPid() - { - return fCh_pid; - } - pid_t getPid() - { - return fThis_pid; - } - pid_t getPPid() - { - return fP_pid; - } - std::string getCmdLineStr() - { - return fCmdLineStr; - } + int fMode; + std::ofstream fDataDumpFile; + std::ofstream fJobFile; + unsigned int fTxBytes; + unsigned int fRxBytes; + char fPmId; + int fObjId; // Object Identifier for logging -private: - SplitterReadThread& fRef; + // CpImport related Member variables + int fFIFO[2]; // I/O Pipes + pid_t fCh_pid; + pid_t fThis_pid; + pid_t fP_pid; + bool fCpIStarted; + std::string fCmdLineStr; + std::string fBrmRptFileName; - int fMode; - std::ofstream fDataDumpFile; - std::ofstream fJobFile; - unsigned int fTxBytes; - unsigned int fRxBytes; - char fPmId; - int fObjId; // Object Identifier for logging + // CPI Feeder Thread + WECpiFeederThread* fpCfThread; - // CpImport related Member variables - int fFIFO[2]; //I/O Pipes - pid_t fCh_pid; - pid_t fThis_pid; - pid_t fP_pid; - bool fCpIStarted; - std::string fCmdLineStr; - std::string fBrmRptFileName; + boost::mutex fClntMsgMutex; // mutex in sending messages to client. - //CPI Feeder Thread - WECpiFeederThread* fpCfThread; + // static bool fTearDownCpimport; // @bug 4267 + bool fTearDownCpimport; // @bug 4267 + pid_t fWaitPidRc; // @bug 4267 + int fWaitPidStatus; // @bug 4267 - boost::mutex fClntMsgMutex; //mutex in sending messages to client. + bool fForceKill; + bool fPipeErr; // Err Flag to restrict err msgs logging. - //static bool fTearDownCpimport; // @bug 4267 - bool fTearDownCpimport; // @bug 4267 - pid_t fWaitPidRc; // @bug 4267 - int fWaitPidStatus; // @bug 4267 + private: + // more enums follow + enum CmdId + { + BULKFILENAME + }; - bool fForceKill; - bool fPipeErr; // Err Flag to restrict err msgs logging. - -private: - // more enums follow - enum CmdId { BULKFILENAME }; -public: - enum { MIN_QSIZE = 25, MAX_QSIZE = 250}; - -public: - SimpleSysLog* fpSysLog; + public: + enum + { + MIN_QSIZE = 25, + MAX_QSIZE = 250 + }; + public: + SimpleSysLog* fpSysLog; }; -} +} // namespace WriteEngine #endif /* WE_DATALOADER_H_ */ diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index ec44557f3..69373fff6 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -49,23 +49,23 @@ namespace WriteEngine { WE_DDLCommandProc::WE_DDLCommandProc() { - filesPerColumnPartition = 8; - extentsPerSegmentFile = 1; - dbrootCnt = 1; - extentRows = 0x800000; - config::Config* cf = config::Config::makeConfig(); - string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); + filesPerColumnPartition = 8; + extentsPerSegmentFile = 1; + dbrootCnt = 1; + extentRows = 0x800000; + config::Config* cf = config::Config::makeConfig(); + string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); - if (fpc.length() != 0) - filesPerColumnPartition = cf->uFromText(fpc); + if (fpc.length() != 0) + filesPerColumnPartition = cf->uFromText(fpc); - // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). - extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; - string dbct = cf->getConfig("SystemConfig", "DBRootCount"); + string dbct = cf->getConfig("SystemConfig", "DBRootCount"); - if (dbct.length() != 0) - dbrootCnt = cf->uFromText(dbct); + if (dbct.length() != 0) + dbrootCnt = cf->uFromText(dbct); } WE_DDLCommandProc::WE_DDLCommandProc(const WE_DDLCommandProc& rhs) { @@ -75,1691 +75,1266 @@ WE_DDLCommandProc::~WE_DDLCommandProc() } uint8_t WE_DDLCommandProc::updateSyscolumnNextval(ByteStream& bs, std::string& err) { - uint32_t columnOid, sessionID; - uint64_t nextVal; - int rc = 0; - bs >> columnOid; - bs >> nextVal; - bs >> sessionID; - uint16_t dbRoot; - BRM::OID_t oid = 1021; - fDbrm.getSysCatDBRoot(oid, dbRoot); - std::map oids; - //std::vector oidsToFlush; - oids[columnOid] = columnOid; + uint32_t columnOid, sessionID; + uint64_t nextVal; + int rc = 0; + bs >> columnOid; + bs >> nextVal; + bs >> sessionID; + uint16_t dbRoot; + BRM::OID_t oid = 1021; + fDbrm.getSysCatDBRoot(oid, dbRoot); + std::map oids; + // std::vector oidsToFlush; + oids[columnOid] = columnOid; - //oidsToFlush.push_back(columnOid); - if (idbdatafile::IDBPolicy::useHdfs()) - fWEWrapper.startTransaction(sessionID); + // oidsToFlush.push_back(columnOid); + if (idbdatafile::IDBPolicy::useHdfs()) + fWEWrapper.startTransaction(sessionID); - rc = fWEWrapper.updateNextValue(sessionID, columnOid, nextVal, sessionID, dbRoot); + rc = fWEWrapper.updateNextValue(sessionID, columnOid, nextVal, sessionID, dbRoot); - if (rc != 0) - { - err = "Error in WE::updateNextValue"; - rc = 1; - } + if (rc != 0) + { + err = "Error in WE::updateNextValue"; + rc = 1; + } - if (idbdatafile::IDBPolicy::useHdfs()) - { - fWEWrapper.flushDataFiles(rc, sessionID, oids); - fWEWrapper.confirmTransaction(sessionID); + if (idbdatafile::IDBPolicy::useHdfs()) + { + fWEWrapper.flushDataFiles(rc, sessionID, oids); + fWEWrapper.confirmTransaction(sessionID); - if ( rc == 0) - fWEWrapper.endTransaction(sessionID, true); - else - fWEWrapper.endTransaction(sessionID, false); - } + if (rc == 0) + fWEWrapper.endTransaction(sessionID, true); + else + fWEWrapper.endTransaction(sessionID, false); + } - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - return rc; + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; } uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - int txnID, tableOID; - uint32_t tableWithAutoi; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> tmp32; - tableOID = tmp32; - bs >> tmp32; - tableWithAutoi = tmp32; - bs >> tmp32; - uint16_t dbroot = tmp32; - ddlpackage::TableDef tableDef; - tableDef.unserialize(bs); + int rc = 0; + uint32_t sessionID, tmp32; + int txnID, tableOID; + uint32_t tableWithAutoi; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> tmp32; + tableOID = tmp32; + bs >> tmp32; + tableWithAutoi = tmp32; + bs >> tmp32; + uint16_t dbroot = tmp32; + ddlpackage::TableDef tableDef; + tableDef.unserialize(bs); - WriteEngine::ColTuple colTuple; - WriteEngine::ColStruct colStruct; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::ColTupleList colTuples; - WriteEngine::dictStr dctColTuples; - WriteEngine::DctnryStruct dctnryStruct; - WriteEngine::ColValueList colValuesList; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DictStrList dctnryValueList; - WriteEngine::RIDList ridList; - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::ROPair sysTableROPair; - boost::shared_ptr systemCatalogPtr; - ColumnList columns; - ColumnList::const_iterator column_iterator; - DDLColumn column; - int error = 0; + WriteEngine::ColTuple colTuple; + WriteEngine::ColStruct colStruct; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::ColTupleList colTuples; + WriteEngine::dictStr dctColTuples; + WriteEngine::DctnryStruct dctnryStruct; + WriteEngine::ColValueList colValuesList; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DictStrList dctnryValueList; + WriteEngine::RIDList ridList; + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::ROPair sysTableROPair; + boost::shared_ptr systemCatalogPtr; + ColumnList columns; + ColumnList::const_iterator column_iterator; + DDLColumn column; + int error = 0; - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSTABLE_TABLE; + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSTABLE_TABLE; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - std::map oids; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + std::map oids; - //std::vector oidsToFlush; - try + // std::vector oidsToFlush; + try + { + sysTableROPair = systemCatalogPtr->tableRID(tableName); + + getColumnsForTable(sessionID, tableName.schema, tableName.table, columns); + + column_iterator = columns.begin(); + std::string tmpStr(""); + + while (column_iterator != columns.end()) { - sysTableROPair = systemCatalogPtr->tableRID(tableName); + column = *column_iterator; + boost::to_lower(column.tableColName.column); - getColumnsForTable(sessionID, tableName.schema, tableName.table, columns); + if (TABLENAME_COL == column.tableColName.column) + { + std::string tablename = tableDef.fQualifiedName->fName; + colTuple.data = tablename; + tmpStr = tablename; + } + else if (SCHEMA_COL == column.tableColName.column) + { + std::string schema = tableDef.fQualifiedName->fSchema; + colTuple.data = schema; + tmpStr = schema; + } + else if (OBJECTID_COL == column.tableColName.column) + { + colTuple.data = tableOID; + } + else if (CREATEDATE_COL == column.tableColName.column) + { + time_t t; + struct tm tmp; + Date aDay; - column_iterator = columns.begin(); - std::string tmpStr(""); + t = time(NULL); + gmtime_r(&t, &tmp); + aDay.year = tmp.tm_year + 1900; + aDay.month = tmp.tm_mon + 1; + aDay.day = tmp.tm_mday; - while (column_iterator != columns.end()) - { + colTuple.data = *(reinterpret_cast(&aDay)); + } + else if (INIT_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + } + else if (NEXT_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + } + else if (AUTOINC_COL == column.tableColName.column) + { + colTuple.data = tableWithAutoi; + } + else + { + colTuple.data = column.colType.getNullValueForType(); + } - column = *column_iterator; - boost::to_lower(column.tableColName.column); + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbroot; - if (TABLENAME_COL == column.tableColName.column) - { - std::string tablename = tableDef.fQualifiedName->fName; - colTuple.data = tablename; - tmpStr = tablename; - } - else if (SCHEMA_COL == column.tableColName.column) - { - std::string schema = tableDef.fQualifiedName->fSchema; - colTuple.data = schema; - tmpStr = schema; - } - else if (OBJECTID_COL == column.tableColName.column) - { - colTuple.data = tableOID; - } - else if (CREATEDATE_COL == column.tableColName.column) - { - time_t t; - struct tm tmp; - Date aDay; + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } - t = time(NULL); - gmtime_r(&t, &tmp); - aDay.year = tmp.tm_year + 1900; - aDay.month = tmp.tm_mon + 1; - aDay.day = tmp.tm_mday; + dctnryStruct.fColDbRoot = dbroot; - colTuple.data = *(reinterpret_cast (&aDay)); + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = column.oid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = column.oid; + } - } - else if (INIT_COL == column.tableColName.column) - { + colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); + oids[colStruct.dataOid] = colStruct.dataOid; - colTuple.data = column.colType.getNullValueForType(); - } - else if (NEXT_COL == column.tableColName.column) - { + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } - colTuple.data = column.colType.getNullValueForType(); - } - else if (AUTOINC_COL == column.tableColName.column) - { - colTuple.data = tableWithAutoi; - } - else - { - colTuple.data = column.colType.getNullValueForType(); - } + colTuples.push_back(colTuple); - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbroot; + dctColTuples.push_back(tmpStr); - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } + colValuesList.push_back(colTuples); - dctnryStruct.fColDbRoot = dbroot; + dctnryStructList.push_back(dctnryStruct); - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = column.oid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = column.oid; - } + dctnryValueList.push_back(dctColTuples); - colStructs.push_back(colStruct); - cscColTypeList.push_back(column.colType); - oids[colStruct.dataOid] = colStruct.dataOid; + colTuples.pop_back(); + dctColTuples.pop_back(); - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - colTuples.push_back(colTuple); - - dctColTuples.push_back (tmpStr); - - colValuesList.push_back(colTuples); - - dctnryStructList.push_back (dctnryStruct); - - dctnryValueList.push_back (dctColTuples); - - - colTuples.pop_back(); - dctColTuples.pop_back(); - - ++column_iterator; - } - - //fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - - if (0 != colStructs.size()) - { - // MCOL-66 The DBRM can't handle concurrent transactions to sys tables - // TODO: This may be redundant - static boost::mutex dbrmMutex; - boost::mutex::scoped_lock lk(dbrmMutex); - error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, - dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); - - if (error != WriteEngine::NO_ERROR) - { - if (error == ERR_BRM_WR_VB_ENTRY) - { - throw std::runtime_error("WE: Error writing to BRM."); - } - else - { - WErrorCodes ec; - throw std::runtime_error("WE: Error updating calpontsys.systable:" + ec.errorString(error)); - } - } - - if (idbdatafile::IDBPolicy::useHdfs()) - { - int rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - } - } - catch (exception& ex) - { - err += ex.what(); - rc = 1; - } - catch (...) - { - err += "Unknown exception caught"; - rc = 1; + ++column_iterator; } - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - - fWEWrapper.setIsInsert(false); + // fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(false); - return rc; + fWEWrapper.startTransaction(txnID); + + if (0 != colStructs.size()) + { + // MCOL-66 The DBRM can't handle concurrent transactions to sys tables + // TODO: This may be redundant + static boost::mutex dbrmMutex; + boost::mutex::scoped_lock lk(dbrmMutex); + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, + dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); + + if (error != WriteEngine::NO_ERROR) + { + if (error == ERR_BRM_WR_VB_ENTRY) + { + throw std::runtime_error("WE: Error writing to BRM."); + } + else + { + WErrorCodes ec; + throw std::runtime_error("WE: Error updating calpontsys.systable:" + ec.errorString(error)); + } + } + + if (idbdatafile::IDBPolicy::useHdfs()) + { + int rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + } + } + catch (exception& ex) + { + err += ex.what(); + rc = 1; + } + catch (...) + { + err += "Unknown exception caught"; + rc = 1; + } + + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + return rc; } uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32, columnSize, dictSize, i; - uint8_t tmp8; - int txnID, colpos; - bs >> sessionID; + int rc = 0; + uint32_t sessionID, tmp32, columnSize, dictSize, i; + uint8_t tmp8; + int txnID, colpos; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> columnSize; + + // deserialize column Oid and dictionary oid + vector coloids; + vector dictoids; + + for (i = 0; i < columnSize; ++i) + { bs >> tmp32; - txnID = tmp32; - bs >> columnSize; + coloids.push_back(tmp32); + } - //deserialize column Oid and dictionary oid - vector coloids; - vector dictoids; + bs >> dictSize; - for (i = 0; i < columnSize; ++i) - { - bs >> tmp32; - coloids.push_back(tmp32); - } - - bs >> dictSize; - - for (i = 0; i < dictSize; ++i) - { - bs >> tmp32; - dictoids.push_back(tmp32); - } - - bool alterFlag = 0; - bs >> tmp8; - alterFlag = (tmp8 != 0); + for (i = 0; i < dictSize; ++i) + { bs >> tmp32; - colpos = tmp32; - bs >> tmp32; - uint16_t dbroot = tmp32; - ddlpackage::TableDef tableDef; - tableDef.unserialize(bs); + dictoids.push_back(tmp32); + } - WriteEngine::ColTuple colTuple; - WriteEngine::ColStruct colStruct; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::ColTupleList colTuples; - WriteEngine::dictStr dctColTuples; - WriteEngine::DctnryStruct dctnryStruct; - WriteEngine::ColValueList colValuesList; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DictStrList dctnryValueList; - WriteEngine::RIDList ridList; - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::ROPair sysTableROPair; - boost::shared_ptr systemCatalogPtr; - ColumnList columns; - ColumnList::const_iterator column_iterator; - DDLColumn column; - int error = 0; + bool alterFlag = 0; + bs >> tmp8; + alterFlag = (tmp8 != 0); + bs >> tmp32; + colpos = tmp32; + bs >> tmp32; + uint16_t dbroot = tmp32; + ddlpackage::TableDef tableDef; + tableDef.unserialize(bs); - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - std::map oids; - int rc1 = 0; - ColumnDef* colDefPtr = 0; - ColumnDefList::const_iterator iter; + WriteEngine::ColTuple colTuple; + WriteEngine::ColStruct colStruct; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::ColTupleList colTuples; + WriteEngine::dictStr dctColTuples; + WriteEngine::DctnryStruct dctnryStruct; + WriteEngine::ColValueList colValuesList; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DictStrList dctnryValueList; + WriteEngine::RIDList ridList; + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::ROPair sysTableROPair; + boost::shared_ptr systemCatalogPtr; + ColumnList columns; + ColumnList::const_iterator column_iterator; + DDLColumn column; + int error = 0; - int startPos = colpos; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + std::map oids; + int rc1 = 0; + ColumnDef* colDefPtr = 0; + ColumnDefList::const_iterator iter; - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSCOLUMN_TABLE; + int startPos = colpos; - getColumnsForTable(sessionID, tableName.schema, tableName.table, columns); - unsigned int numCols = columns.size(); - //WriteEngine::ColTupleList colList[numCols]; - //ColTupleList is NOT POD, so let's try this: - std::vector colList; - //WriteEngine::dictStr dctColList[numCols]; - std::vector dctColList; - ColumnDefList tableDefCols = tableDef.fColumns; + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSCOLUMN_TABLE; - ddlpackage::QualifiedName qualifiedName = *(tableDef.fQualifiedName); - iter = tableDefCols.begin(); - //colpos = 0; - std::string tmpStr(""); + getColumnsForTable(sessionID, tableName.schema, tableName.table, columns); + unsigned int numCols = columns.size(); + // WriteEngine::ColTupleList colList[numCols]; + // ColTupleList is NOT POD, so let's try this: + std::vector colList; + // WriteEngine::dictStr dctColList[numCols]; + std::vector dctColList; + ColumnDefList tableDefCols = tableDef.fColumns; - for (unsigned int ii = 0; ii < numCols; ii++) + ddlpackage::QualifiedName qualifiedName = *(tableDef.fQualifiedName); + iter = tableDefCols.begin(); + // colpos = 0; + std::string tmpStr(""); + + for (unsigned int ii = 0; ii < numCols; ii++) + { + colList.push_back(WriteEngine::ColTupleList()); + dctColList.push_back(WriteEngine::dictStr()); + } + + try + { + unsigned int col = 0; + unsigned int dictcol = 0; + + while (iter != tableDefCols.end()) { - colList.push_back(WriteEngine::ColTupleList()); - dctColList.push_back(WriteEngine::dictStr()); - } + colDefPtr = *iter; - try - { - unsigned int col = 0; - unsigned int dictcol = 0; + DictOID dictOID = {0, 0, 0, 0, 0}; - while (iter != tableDefCols.end()) + int dataType; + dataType = convertDataType(colDefPtr->fType->fType); + + if (dataType == CalpontSystemCatalog::DECIMAL || dataType == CalpontSystemCatalog::UDECIMAL) + { + if (colDefPtr->fType->fPrecision > 38) // precision cannot be over 38. { - colDefPtr = *iter; - - DictOID dictOID = {0, 0, 0, 0, 0}; - - int dataType; - dataType = convertDataType(colDefPtr->fType->fType); - - if (dataType == CalpontSystemCatalog::DECIMAL || - dataType == CalpontSystemCatalog::UDECIMAL) - { - if (colDefPtr->fType->fPrecision > 38) // precision cannot be over 38. - { - ostringstream os; - os << "Syntax error: The maximum precision (total number of digits) that can be specified is 38"; - throw std::runtime_error(os.str()); - } - else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) - { - ostringstream os; - os << "Syntax error: scale should be less than precision, precision: " << colDefPtr->fType->fPrecision << " scale: " << colDefPtr->fType->fScale; - throw std::runtime_error(os.str()); - } - - colDefPtr->convertDecimal(); - } - - bool hasDict = false; - - if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || - (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::TEXT && colDefPtr->fType->fLength > 7) ) - { - hasDict = true; - dictOID.compressionType = colDefPtr->fType->fCompressiontype; - dictOID.colWidth = colDefPtr->fType->fLength; - dictOID.dictOID = dictoids[dictcol]; - dictcol++; - - //@Bug 2534. Take away the limit of 255 and set the limit to 8000. - if ((colDefPtr->fType->fLength > 8000) && - (dataType != CalpontSystemCatalog::BLOB) && - (dataType != CalpontSystemCatalog::TEXT)) - { - ostringstream os; - os << "char, varchar and varbinary length may not exceed 8000"; - throw std::runtime_error(os.str()); - } - } - else if ((dataType == CalpontSystemCatalog::VARBINARY - || dataType == CalpontSystemCatalog::BLOB - || dataType == CalpontSystemCatalog::TEXT) - && colDefPtr->fType->fLength <= 7) - { - ostringstream os; - os << "varbinary and blob length may not be less than 8"; - throw std::runtime_error(os.str()); - } - - unsigned int i = 0; - column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - column = *column_iterator; - boost::to_lower(column.tableColName.column); - - if (SCHEMA_COL == column.tableColName.column) - { - colTuple.data = qualifiedName.fSchema; - tmpStr = qualifiedName.fSchema; - } - else if (TABLENAME_COL == column.tableColName.column) - { - colTuple.data = qualifiedName.fName; - tmpStr = qualifiedName.fName; - } - else if (COLNAME_COL == column.tableColName.column) - { - boost::to_lower(colDefPtr->fName); - colTuple.data = colDefPtr->fName; - tmpStr = colDefPtr->fName; - } - else if (OBJECTID_COL == column.tableColName.column) - { - if (alterFlag) - colTuple.data = coloids[col]; - else - colTuple.data = coloids[col]; - } - else if (DATATYPE_COL == column.tableColName.column) - { - colTuple.data = dataType; - } - else if (COLUMNLEN_COL == column.tableColName.column) - { - //@Bug 2089 Disallow zero length char and varch column to be created - if (dataType == CalpontSystemCatalog::CHAR || - dataType == CalpontSystemCatalog::VARCHAR || - dataType == CalpontSystemCatalog::VARBINARY || - dataType == CalpontSystemCatalog::BLOB || - dataType == CalpontSystemCatalog::TEXT) - { - if (colDefPtr->fType->fLength <= 0) - { - ostringstream os; - os << "char, varchar and varbinary length must be greater than zero"; - throw std::runtime_error(os.str()); - } - } - - colTuple.data = colDefPtr->fType->fLength; - } - else if (COLUMNPOS_COL == column.tableColName.column) - { - colTuple.data = colpos; - } - else if (DEFAULTVAL_COL == column.tableColName.column) - { - if (colDefPtr->fDefaultValue) - { - colTuple.data = colDefPtr->fDefaultValue->fValue; - tmpStr = colDefPtr->fDefaultValue->fValue; - } - else - { - tmpStr = ""; - //colTuple.data = column.colType.getNullValueForType(); - } - - } - else if (NULLABLE_COL == column.tableColName.column) - { - int nullable = 1; - ColumnConstraintList& colConstraints = colDefPtr->fConstraints; - ColumnConstraintList::const_iterator constraint_iter = colConstraints.begin(); - - while (constraint_iter != colConstraints.end()) - { - ColumnConstraintDef* consDefPtr = *constraint_iter; - - if (consDefPtr->fConstraintType == ddlpackage::DDL_NOT_NULL) - { - nullable = 0; - break; - } - - ++constraint_iter; - } - - colTuple.data = nullable; - } - else if (SCALE_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fScale; - } - else if (PRECISION_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fPrecision; - } - else if (DICTOID_COL == column.tableColName.column) - { - if (hasDict) - { - colTuple.data = dictOID.dictOID; - } - else - { - colTuple.data = column.colType.getNullValueForType(); - } - } - else if (LISTOBJID_COL == column.tableColName.column) - { - colTuple.data = column.colType.getNullValueForType(); - } - else if (TREEOBJID_COL == column.tableColName.column) - { - colTuple.data = column.colType.getNullValueForType(); - } - else if (MINVAL_COL == column.tableColName.column) - { - tmpStr = ""; - } - else if (MAXVAL_COL == column.tableColName.column) - { - tmpStr = ""; - } - else if (COMPRESSIONTYPE_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fCompressiontype; - } - else if (AUTOINC_COL == column.tableColName.column) - { - //cout << "autoincrement= " << colDefPtr->fType->fAutoincrement << endl; - colTuple.data = colDefPtr->fType->fAutoincrement; - - } - else if (NEXTVALUE_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fNextvalue; - } - else - { - colTuple.data = column.colType.getNullValueForType(); - } - - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbroot; - dctnryStruct.fColDbRoot = dbroot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = column.oid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = column.oid; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - if (colpos == startPos) - { - colStructs.push_back(colStruct); - dctnryStructList.push_back (dctnryStruct); - cscColTypeList.push_back(column.colType); - } - - colList[i].push_back(colTuple); - //colList.push_back(WriteEngine::ColTupleList()); - //colList.back().push_back(colTuple); - dctColList[i].push_back(tmpStr); - //dctColList.push_back(WriteEngine::dictStr()); - //dctColList.back().push_back(tmpStr); - ++i; - ++column_iterator; - } - - ++colpos; - col++; - ++iter; + ostringstream os; + os << "Syntax error: The maximum precision (total number of digits) that can be specified is 38"; + throw std::runtime_error(os.str()); + } + else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) + { + ostringstream os; + os << "Syntax error: scale should be less than precision, precision: " + << colDefPtr->fType->fPrecision << " scale: " << colDefPtr->fType->fScale; + throw std::runtime_error(os.str()); } - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); + colDefPtr->convertDecimal(); + } - if (0 != colStructs.size()) + bool hasDict = false; + + if ((dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || + (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::TEXT && colDefPtr->fType->fLength > 7)) + { + hasDict = true; + dictOID.compressionType = colDefPtr->fType->fCompressiontype; + dictOID.colWidth = colDefPtr->fType->fLength; + dictOID.dictOID = dictoids[dictcol]; + dictcol++; + + //@Bug 2534. Take away the limit of 255 and set the limit to 8000. + if ((colDefPtr->fType->fLength > 8000) && (dataType != CalpontSystemCatalog::BLOB) && + (dataType != CalpontSystemCatalog::TEXT)) { - for (unsigned int n = 0; n < numCols; n++) - { - colValuesList.push_back(colList[n]); - dctnryValueList.push_back(dctColList[n]); - } - - //fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); - error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, - dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (error != WriteEngine::NO_ERROR) - { - if (error == ERR_BRM_WR_VB_ENTRY) - { - throw std::runtime_error( - "writeSysColumnMetaData WE: Error writing to BRM."); - } - else - { - WErrorCodes ec; - throw std::runtime_error( - "WE: Error updating calpontsys.syscolumn. " + ec.errorString(error)); - } - } - else - error = rc1; + ostringstream os; + os << "char, varchar and varbinary length may not exceed 8000"; + throw std::runtime_error(os.str()); } - } - catch (exception& ex) - { - err += ex.what(); - rc = 1; - } - catch (...) - { - err += "Unknown exception caught"; - rc = 1; + } + else if ((dataType == CalpontSystemCatalog::VARBINARY || dataType == CalpontSystemCatalog::BLOB || + dataType == CalpontSystemCatalog::TEXT) && + colDefPtr->fType->fLength <= 7) + { + ostringstream os; + os << "varbinary and blob length may not be less than 8"; + throw std::runtime_error(os.str()); + } + + unsigned int i = 0; + column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + column = *column_iterator; + boost::to_lower(column.tableColName.column); + + if (SCHEMA_COL == column.tableColName.column) + { + colTuple.data = qualifiedName.fSchema; + tmpStr = qualifiedName.fSchema; + } + else if (TABLENAME_COL == column.tableColName.column) + { + colTuple.data = qualifiedName.fName; + tmpStr = qualifiedName.fName; + } + else if (COLNAME_COL == column.tableColName.column) + { + boost::to_lower(colDefPtr->fName); + colTuple.data = colDefPtr->fName; + tmpStr = colDefPtr->fName; + } + else if (OBJECTID_COL == column.tableColName.column) + { + if (alterFlag) + colTuple.data = coloids[col]; + else + colTuple.data = coloids[col]; + } + else if (DATATYPE_COL == column.tableColName.column) + { + colTuple.data = dataType; + } + else if (COLUMNLEN_COL == column.tableColName.column) + { + //@Bug 2089 Disallow zero length char and varch column to be created + if (dataType == CalpontSystemCatalog::CHAR || dataType == CalpontSystemCatalog::VARCHAR || + dataType == CalpontSystemCatalog::VARBINARY || dataType == CalpontSystemCatalog::BLOB || + dataType == CalpontSystemCatalog::TEXT) + { + if (colDefPtr->fType->fLength <= 0) + { + ostringstream os; + os << "char, varchar and varbinary length must be greater than zero"; + throw std::runtime_error(os.str()); + } + } + + colTuple.data = colDefPtr->fType->fLength; + } + else if (COLUMNPOS_COL == column.tableColName.column) + { + colTuple.data = colpos; + } + else if (DEFAULTVAL_COL == column.tableColName.column) + { + if (colDefPtr->fDefaultValue) + { + colTuple.data = colDefPtr->fDefaultValue->fValue; + tmpStr = colDefPtr->fDefaultValue->fValue; + } + else + { + tmpStr = ""; + // colTuple.data = column.colType.getNullValueForType(); + } + } + else if (NULLABLE_COL == column.tableColName.column) + { + int nullable = 1; + ColumnConstraintList& colConstraints = colDefPtr->fConstraints; + ColumnConstraintList::const_iterator constraint_iter = colConstraints.begin(); + + while (constraint_iter != colConstraints.end()) + { + ColumnConstraintDef* consDefPtr = *constraint_iter; + + if (consDefPtr->fConstraintType == ddlpackage::DDL_NOT_NULL) + { + nullable = 0; + break; + } + + ++constraint_iter; + } + + colTuple.data = nullable; + } + else if (SCALE_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fScale; + } + else if (PRECISION_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fPrecision; + } + else if (DICTOID_COL == column.tableColName.column) + { + if (hasDict) + { + colTuple.data = dictOID.dictOID; + } + else + { + colTuple.data = column.colType.getNullValueForType(); + } + } + else if (LISTOBJID_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + } + else if (TREEOBJID_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + } + else if (MINVAL_COL == column.tableColName.column) + { + tmpStr = ""; + } + else if (MAXVAL_COL == column.tableColName.column) + { + tmpStr = ""; + } + else if (COMPRESSIONTYPE_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fCompressiontype; + } + else if (AUTOINC_COL == column.tableColName.column) + { + // cout << "autoincrement= " << colDefPtr->fType->fAutoincrement << endl; + colTuple.data = colDefPtr->fType->fAutoincrement; + } + else if (NEXTVALUE_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fNextvalue; + } + else + { + colTuple.data = column.colType.getNullValueForType(); + } + + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbroot; + dctnryStruct.fColDbRoot = dbroot; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = column.oid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = column.oid; + } + + oids[colStruct.dataOid] = colStruct.dataOid; + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + if (colpos == startPos) + { + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); + } + + colList[i].push_back(colTuple); + // colList.push_back(WriteEngine::ColTupleList()); + // colList.back().push_back(colTuple); + dctColList[i].push_back(tmpStr); + // dctColList.push_back(WriteEngine::dictStr()); + // dctColList.back().push_back(tmpStr); + ++i; + ++column_iterator; + } + + ++colpos; + col++; + ++iter; } - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - - fWEWrapper.setIsInsert(false); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(false); - return rc; + fWEWrapper.startTransaction(txnID); + + if (0 != colStructs.size()) + { + for (unsigned int n = 0; n < numCols; n++) + { + colValuesList.push_back(colList[n]); + dctnryValueList.push_back(dctColList[n]); + } + + // fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, + dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error != WriteEngine::NO_ERROR) + { + if (error == ERR_BRM_WR_VB_ENTRY) + { + throw std::runtime_error("writeSysColumnMetaData WE: Error writing to BRM."); + } + else + { + WErrorCodes ec; + throw std::runtime_error("WE: Error updating calpontsys.syscolumn. " + ec.errorString(error)); + } + } + else + error = rc1; + } + } + catch (exception& ex) + { + err += ex.what(); + rc = 1; + } + catch (...) + { + err += "Unknown exception caught"; + rc = 1; + } + + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + return rc; } uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32, coloid, dictoid; - int txnID, startPos; - string schema, tablename; - uint8_t tmp8; - bool isAlter = false; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - bs >> coloid; - bs >> dictoid; - bs >> tmp8; //alterFlag - bs >> tmp32; - startPos = tmp32; - isAlter = (tmp8 != 0); - boost::scoped_ptr colDefPtr(new ddlpackage::ColumnDef()); - colDefPtr->unserialize(bs); + int rc = 0; + uint32_t sessionID, tmp32, coloid, dictoid; + int txnID, startPos; + string schema, tablename; + uint8_t tmp8; + bool isAlter = false; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + bs >> coloid; + bs >> dictoid; + bs >> tmp8; // alterFlag + bs >> tmp32; + startPos = tmp32; + isAlter = (tmp8 != 0); + boost::scoped_ptr colDefPtr(new ddlpackage::ColumnDef()); + colDefPtr->unserialize(bs); - WriteEngine::ColStruct colStruct; - WriteEngine::ColTuple colTuple; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::ColTupleList colTuples; - WriteEngine::DctColTupleList dctColTuples; - WriteEngine::ColValueList colValuesList; - WriteEngine::RIDList ridList; - WriteEngine::DctnryStruct dctnryStruct; - WriteEngine::dictStr dctnryTuple; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DictStrList dctnryValueList; + WriteEngine::ColStruct colStruct; + WriteEngine::ColTuple colTuple; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::ColTupleList colTuples; + WriteEngine::DctColTupleList dctColTuples; + WriteEngine::ColValueList colValuesList; + WriteEngine::RIDList ridList; + WriteEngine::DctnryStruct dctnryStruct; + WriteEngine::dictStr dctnryTuple; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DictStrList dctnryValueList; - CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::TableName tableName; - ColumnList columns; - ColumnList::const_iterator column_iterator; + ColumnList columns; + ColumnList::const_iterator column_iterator; - DDLColumn column; - int error = 0; + DDLColumn column; + int error = 0; - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSCOLUMN_TABLE; + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSCOLUMN_TABLE; - getColumnsForTable(sessionID, tableName.schema, tableName.table, columns); - unsigned int numCols = columns.size(); - //WriteEngine::ColTupleList colList[numCols]; - //ColTupleList is NOT POD, so let's try this: - std::vector colList; - //WriteEngine::dictStr dctColList[numCols]; - std::vector dctColList; - std::map oids; - std::vector oidsToFlush; - //colpos = 0; - std::string tmpStr(""); + getColumnsForTable(sessionID, tableName.schema, tableName.table, columns); + unsigned int numCols = columns.size(); + // WriteEngine::ColTupleList colList[numCols]; + // ColTupleList is NOT POD, so let's try this: + std::vector colList; + // WriteEngine::dictStr dctColList[numCols]; + std::vector dctColList; + std::map oids; + std::vector oidsToFlush; + // colpos = 0; + std::string tmpStr(""); - for (unsigned int ii = 0; ii < numCols; ii++) + for (unsigned int ii = 0; ii < numCols; ii++) + { + colList.push_back(WriteEngine::ColTupleList()); + dctColList.push_back(WriteEngine::dictStr()); + } + + try + { + DictOID dictOID = {0, 0, 0, 0, 0}; + + int dataType = convertDataType(colDefPtr->fType->fType); + + if (dataType == CalpontSystemCatalog::DECIMAL || dataType == CalpontSystemCatalog::UDECIMAL) { - colList.push_back(WriteEngine::ColTupleList()); - dctColList.push_back(WriteEngine::dictStr()); + if (colDefPtr->fType->fPrecision > 38) //@Bug 5717 precision cannot be over 38. + { + ostringstream os; + os << "Syntax error: The maximum precision (total number of digits) that can be specified is 38"; + throw std::runtime_error(os.str()); + } + else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) + { + ostringstream os; + os << "Syntax error: scale should be less than precision, precision: " << colDefPtr->fType->fPrecision + << " scale: " << colDefPtr->fType->fScale; + throw std::runtime_error(os.str()); + } + + colDefPtr->convertDecimal(); } - try + if (dictoid > 0) { - DictOID dictOID = {0, 0, 0, 0, 0}; + dictOID.compressionType = colDefPtr->fType->fCompressiontype; + dictOID.colWidth = colDefPtr->fType->fLength; + dictOID.dictOID = dictoid; - int dataType = convertDataType(colDefPtr->fType->fType); + //@Bug 2534. Take away the limit of 255 and set the limit to 8000. + if ((colDefPtr->fType->fLength > 8000) && (dataType != CalpontSystemCatalog::BLOB) && + (dataType != CalpontSystemCatalog::TEXT)) + { + ostringstream os; + os << "char, varchar and varbinary length may not exceed 8000"; + throw std::runtime_error(os.str()); + } + } + else if ((dataType == CalpontSystemCatalog::VARBINARY || dataType == CalpontSystemCatalog::BLOB || + dataType == CalpontSystemCatalog::TEXT) && + colDefPtr->fType->fLength <= 7) + { + ostringstream os; + os << "varbinary and blob length may not be less than 8"; + throw std::runtime_error(os.str()); + } - if (dataType == CalpontSystemCatalog::DECIMAL || - dataType == CalpontSystemCatalog::UDECIMAL) - { - if (colDefPtr->fType->fPrecision > 38) //@Bug 5717 precision cannot be over 38. - { - ostringstream os; - os << "Syntax error: The maximum precision (total number of digits) that can be specified is 38"; - throw std::runtime_error(os.str()); - } - else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) - { - ostringstream os; - os << "Syntax error: scale should be less than precision, precision: " << colDefPtr->fType->fPrecision << " scale: " << colDefPtr->fType->fScale; - throw std::runtime_error(os.str()); - } - - colDefPtr->convertDecimal(); - } - - if (dictoid > 0) - { - dictOID.compressionType = colDefPtr->fType->fCompressiontype; - dictOID.colWidth = colDefPtr->fType->fLength; - dictOID.dictOID = dictoid; - - //@Bug 2534. Take away the limit of 255 and set the limit to 8000. - if ((colDefPtr->fType->fLength > 8000) && - (dataType != CalpontSystemCatalog::BLOB) && - (dataType != CalpontSystemCatalog::TEXT)) - { - ostringstream os; - os << "char, varchar and varbinary length may not exceed 8000"; - throw std::runtime_error(os.str()); - } - } - else if ((dataType == CalpontSystemCatalog::VARBINARY - || dataType == CalpontSystemCatalog::BLOB - || dataType == CalpontSystemCatalog::TEXT) - && colDefPtr->fType->fLength <= 7) + unsigned int i = 0; + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + column = *column_iterator; + boost::to_lower(column.tableColName.column); + + if (SCHEMA_COL == column.tableColName.column) + { + colTuple.data = schema; + tmpStr = schema; + } + else if (TABLENAME_COL == column.tableColName.column) + { + colTuple.data = tablename; + tmpStr = tablename; + } + else if (COLNAME_COL == column.tableColName.column) + { + boost::to_lower(colDefPtr->fName); + colTuple.data = colDefPtr->fName; + tmpStr = colDefPtr->fName; + } + else if (OBJECTID_COL == column.tableColName.column) + { + colTuple.data = coloid; + } + else if (DATATYPE_COL == column.tableColName.column) + { + colTuple.data = dataType; + } + else if (COLUMNLEN_COL == column.tableColName.column) + { + //@Bug 2089 Disallow zero length char and varch column to be created + if (dataType == CalpontSystemCatalog::CHAR || dataType == CalpontSystemCatalog::VARCHAR || + dataType == CalpontSystemCatalog::VARBINARY || dataType == CalpontSystemCatalog::BLOB || + dataType == CalpontSystemCatalog::TEXT) { + if (colDefPtr->fType->fLength <= 0) + { ostringstream os; - os << "varbinary and blob length may not be less than 8"; + os << "char, varchar and varbinary length must be greater than zero"; throw std::runtime_error(os.str()); + } } - unsigned int i = 0; - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); - column_iterator = columns.begin(); - - while (column_iterator != columns.end()) + colTuple.data = colDefPtr->fType->fLength; + } + else if (COLUMNPOS_COL == column.tableColName.column) + { + colTuple.data = startPos; + } + else if (DEFAULTVAL_COL == column.tableColName.column) + { + if (colDefPtr->fDefaultValue) { - column = *column_iterator; - boost::to_lower(column.tableColName.column); - - if (SCHEMA_COL == column.tableColName.column) - { - colTuple.data = schema; - tmpStr = schema; - } - else if (TABLENAME_COL == column.tableColName.column) - { - colTuple.data = tablename; - tmpStr = tablename; - } - else if (COLNAME_COL == column.tableColName.column) - { - boost::to_lower(colDefPtr->fName); - colTuple.data = colDefPtr->fName; - tmpStr = colDefPtr->fName; - } - else if (OBJECTID_COL == column.tableColName.column) - { - colTuple.data = coloid; - } - else if (DATATYPE_COL == column.tableColName.column) - { - colTuple.data = dataType; - } - else if (COLUMNLEN_COL == column.tableColName.column) - { - //@Bug 2089 Disallow zero length char and varch column to be created - if (dataType == CalpontSystemCatalog::CHAR || - dataType == CalpontSystemCatalog::VARCHAR || - dataType == CalpontSystemCatalog::VARBINARY || - dataType == CalpontSystemCatalog::BLOB || - dataType == CalpontSystemCatalog::TEXT) - { - if (colDefPtr->fType->fLength <= 0) - { - ostringstream os; - os << "char, varchar and varbinary length must be greater than zero"; - throw std::runtime_error(os.str()); - } - } - - colTuple.data = colDefPtr->fType->fLength; - } - else if (COLUMNPOS_COL == column.tableColName.column) - { - colTuple.data = startPos; - } - else if (DEFAULTVAL_COL == column.tableColName.column) - { - if (colDefPtr->fDefaultValue) - { - colTuple.data = colDefPtr->fDefaultValue->fValue; - tmpStr = colDefPtr->fDefaultValue->fValue; - } - else - { - tmpStr = ""; - //colTuple.data = column.colType.getNullValueForType(); - } - - } - else if (NULLABLE_COL == column.tableColName.column) - { - int nullable = 1; - ColumnConstraintList& colConstraints = colDefPtr->fConstraints; - ColumnConstraintList::const_iterator constraint_iter = colConstraints.begin(); - - while (constraint_iter != colConstraints.end()) - { - ColumnConstraintDef* consDefPtr = *constraint_iter; - - if (consDefPtr->fConstraintType == ddlpackage::DDL_NOT_NULL) - { - nullable = 0; - break; - } - - ++constraint_iter; - } - - colTuple.data = nullable; - } - else if (SCALE_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fScale; - } - else if (PRECISION_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fPrecision; - } - else if (DICTOID_COL == column.tableColName.column) - { - if (dictoid > 0) - { - colTuple.data = dictOID.dictOID; - } - else - { - colTuple.data = column.colType.getNullValueForType(); - } - } - else if (LISTOBJID_COL == column.tableColName.column) - { - colTuple.data = column.colType.getNullValueForType(); - } - else if (TREEOBJID_COL == column.tableColName.column) - { - colTuple.data = column.colType.getNullValueForType(); - } - else if (MINVAL_COL == column.tableColName.column) - { - tmpStr = ""; - } - else if (MAXVAL_COL == column.tableColName.column) - { - tmpStr = ""; - } - else if (COMPRESSIONTYPE_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fCompressiontype; - } - else if (AUTOINC_COL == column.tableColName.column) - { - //cout << "autoincrement= " << colDefPtr->fType->fAutoincrement << endl; - colTuple.data = colDefPtr->fType->fAutoincrement; - - } - else if (NEXTVALUE_COL == column.tableColName.column) - { - colTuple.data = colDefPtr->fType->fNextvalue; - } - else - { - colTuple.data = column.colType.getNullValueForType(); - } - - colStruct.dataOid = column.oid; - oids[column.oid] = column.oid; - oidsToFlush.push_back(column.oid); - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.fColDbRoot = dbRoot; - colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; - colStruct.colDataType = column.colType.colDataType; - dctnryStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = column.oid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = column.oid; - } - - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back (dctnryStruct); - cscColTypeList.push_back(column.colType); - colList[i].push_back(colTuple); - //colList.push_back(WriteEngine::ColTupleList()); - //colList.back().push_back(colTuple); - dctColList[i].push_back(tmpStr); - //dctColList.push_back(WriteEngine::dictStr()); - //dctColList.back().push_back(tmpStr); - ++i; - ++column_iterator; - } - - - if (0 != colStructs.size()) - { - //FIXME: Is there a cleaner way to do this? Isn't colValuesList the same as colList after this? - for (unsigned int n = 0; n < numCols; n++) - { - colValuesList.push_back(colList[n]); - dctnryValueList.push_back(dctColList[n]); - } - - //fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - int rc1 = 0; - - error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, - dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (error != WriteEngine::NO_ERROR) - { - if (error == ERR_BRM_WR_VB_ENTRY) - { - throw std::runtime_error( - "writeSysColumnMetaData WE: Error writing to BRM."); - } - else - { - WErrorCodes ec; - throw std::runtime_error( - "WE: Error updating calpontsys.syscolumn. " + ec.errorString(error)); - } - } - else - error = rc1; - } - - } - catch (exception& ex) - { - err += ex.what(); - rc = 1; - } - catch (...) - { - err += "Unknown exception caught"; - rc = 1; - } - - purgeFDCache(); - - if (isAlter) - { - if (idbdatafile::IDBPolicy::useHdfs()) - cacheutils::flushOIDsFromCache(oidsToFlush); - } - - return rc; -} - - - -uint8_t WE_DDLCommandProc::createtablefiles(ByteStream& bs, std::string& err) -{ - int rc = 0; - uint32_t size, i; - uint16_t tmp16; - uint32_t tmp32; - uint8_t tmp8; - OID dataOid; - int colWidth; - bool tokenFlag; - int txnID; - CalpontSystemCatalog::ColDataType colDataType; - uint16_t colDbRoot; - int compressionType; - bs >> tmp32; - txnID = tmp32; - bs >> size; - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - std::map oids; - - for (i = 0; i < size; ++i) - { - bs >> tmp32; - dataOid = tmp32; - bs >> tmp8; - colDataType = (CalpontSystemCatalog::ColDataType)tmp8; - bs >> tmp8; - tokenFlag = (tmp8 != 0); - bs >> tmp32; - colWidth = tmp32; - bs >> tmp16; - colDbRoot = tmp16; - bs >> tmp32; - compressionType = tmp32; - oids[dataOid] = dataOid; - - if (tokenFlag) - { - rc = fWEWrapper.createDctnry(0, dataOid, colWidth, colDbRoot, 0, 0, compressionType); + colTuple.data = colDefPtr->fDefaultValue->fValue; + tmpStr = colDefPtr->fDefaultValue->fValue; } else { - rc = fWEWrapper.createColumn(0, dataOid, colDataType, colWidth, colDbRoot, 0, compressionType); + tmpStr = ""; + // colTuple.data = column.colType.getNullValueForType(); + } + } + else if (NULLABLE_COL == column.tableColName.column) + { + int nullable = 1; + ColumnConstraintList& colConstraints = colDefPtr->fConstraints; + ColumnConstraintList::const_iterator constraint_iter = colConstraints.begin(); + + while (constraint_iter != colConstraints.end()) + { + ColumnConstraintDef* consDefPtr = *constraint_iter; + + if (consDefPtr->fConstraintType == ddlpackage::DDL_NOT_NULL) + { + nullable = 0; + break; + } + + ++constraint_iter; } - if (rc != 0) - break; + colTuple.data = nullable; + } + else if (SCALE_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fScale; + } + else if (PRECISION_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fPrecision; + } + else if (DICTOID_COL == column.tableColName.column) + { + if (dictoid > 0) + { + colTuple.data = dictOID.dictOID; + } + else + { + colTuple.data = column.colType.getNullValueForType(); + } + } + else if (LISTOBJID_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + } + else if (TREEOBJID_COL == column.tableColName.column) + { + colTuple.data = column.colType.getNullValueForType(); + } + else if (MINVAL_COL == column.tableColName.column) + { + tmpStr = ""; + } + else if (MAXVAL_COL == column.tableColName.column) + { + tmpStr = ""; + } + else if (COMPRESSIONTYPE_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fCompressiontype; + } + else if (AUTOINC_COL == column.tableColName.column) + { + // cout << "autoincrement= " << colDefPtr->fType->fAutoincrement << endl; + colTuple.data = colDefPtr->fType->fAutoincrement; + } + else if (NEXTVALUE_COL == column.tableColName.column) + { + colTuple.data = colDefPtr->fType->fNextvalue; + } + else + { + colTuple.data = column.colType.getNullValueForType(); + } + + colStruct.dataOid = column.oid; + oids[column.oid] = column.oid; + oidsToFlush.push_back(column.oid); + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.fColDbRoot = dbRoot; + colStruct.tokenFlag = column.colType.colWidth > 8 ? true : false; + colStruct.colDataType = column.colType.colDataType; + dctnryStruct.fColDbRoot = dbRoot; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = column.oid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = column.oid; + } + + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); + colList[i].push_back(colTuple); + // colList.push_back(WriteEngine::ColTupleList()); + // colList.back().push_back(colTuple); + dctColList[i].push_back(tmpStr); + // dctColList.push_back(WriteEngine::dictStr()); + // dctColList.back().push_back(tmpStr); + ++i; + ++column_iterator; } - //cout << "creating column file got error code " << rc << endl; - if (rc != 0) + if (0 != colStructs.size()) { - WErrorCodes ec; - ostringstream oss; - oss << "WE: Error creating column file for oid " << dataOid << "; " << ec.errorString(rc) << endl; - err = oss.str(); + // FIXME: Is there a cleaner way to do this? Isn't colValuesList the same as colList after this? + for (unsigned int n = 0; n < numCols; n++) + { + colValuesList.push_back(colList[n]); + dctnryValueList.push_back(dctColList[n]); + } + + // fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + int rc1 = 0; + + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, + dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error != WriteEngine::NO_ERROR) + { + if (error == ERR_BRM_WR_VB_ENTRY) + { + throw std::runtime_error("writeSysColumnMetaData WE: Error writing to BRM."); + } + else + { + WErrorCodes ec; + throw std::runtime_error("WE: Error updating calpontsys.syscolumn. " + ec.errorString(error)); + } + } + else + error = rc1; + } + } + catch (exception& ex) + { + err += ex.what(); + rc = 1; + } + catch (...) + { + err += "Unknown exception caught"; + rc = 1; + } + + purgeFDCache(); + + if (isAlter) + { + if (idbdatafile::IDBPolicy::useHdfs()) + cacheutils::flushOIDsFromCache(oidsToFlush); + } + + return rc; +} + +uint8_t WE_DDLCommandProc::createtablefiles(ByteStream& bs, std::string& err) +{ + int rc = 0; + uint32_t size, i; + uint16_t tmp16; + uint32_t tmp32; + uint8_t tmp8; + OID dataOid; + int colWidth; + bool tokenFlag; + int txnID; + CalpontSystemCatalog::ColDataType colDataType; + uint16_t colDbRoot; + int compressionType; + bs >> tmp32; + txnID = tmp32; + bs >> size; + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + std::map oids; + + for (i = 0; i < size; ++i) + { + bs >> tmp32; + dataOid = tmp32; + bs >> tmp8; + colDataType = (CalpontSystemCatalog::ColDataType)tmp8; + bs >> tmp8; + tokenFlag = (tmp8 != 0); + bs >> tmp32; + colWidth = tmp32; + bs >> tmp16; + colDbRoot = tmp16; + bs >> tmp32; + compressionType = tmp32; + oids[dataOid] = dataOid; + + if (tokenFlag) + { + rc = fWEWrapper.createDctnry(0, dataOid, colWidth, colDbRoot, 0, 0, compressionType); + } + else + { + rc = fWEWrapper.createColumn(0, dataOid, colDataType, colWidth, colDbRoot, 0, compressionType); } - //if (idbdatafile::IDBPolicy::useHdfs()) - fWEWrapper.flushDataFiles(rc, txnID, oids); - purgeFDCache(); - return rc; + if (rc != 0) + break; + } + + // cout << "creating column file got error code " << rc << endl; + if (rc != 0) + { + WErrorCodes ec; + ostringstream oss; + oss << "WE: Error creating column file for oid " << dataOid << "; " << ec.errorString(rc) << endl; + err = oss.str(); + } + + // if (idbdatafile::IDBPolicy::useHdfs()) + fWEWrapper.flushDataFiles(rc, txnID, oids); + purgeFDCache(); + return rc; } uint8_t WE_DDLCommandProc::commitVersion(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tmp32; - int txnID; + int rc = 0; + uint32_t tmp32; + int txnID; - bs >> tmp32; - txnID = tmp32; + bs >> tmp32; + txnID = tmp32; - rc = fWEWrapper.commit(txnID); + rc = fWEWrapper.commit(txnID); - if (rc != 0) - { - WErrorCodes ec; - ostringstream oss; - oss << "WE: Error commiting transaction; " << txnID << ec.errorString(rc) << endl; - err = oss.str(); - } + if (rc != 0) + { + WErrorCodes ec; + ostringstream oss; + oss << "WE: Error commiting transaction; " << txnID << ec.errorString(rc) << endl; + err = oss.str(); + } - return rc; + return rc; } uint8_t WE_DDLCommandProc::rollbackBlocks(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32;; - int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - rc = fWEWrapper.rollbackBlocks(txnID, sessionID); + int rc = 0; + uint32_t sessionID, tmp32; + ; + int txnID; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + rc = fWEWrapper.rollbackBlocks(txnID, sessionID); - if (rc != 0) - { - WErrorCodes ec; - ostringstream oss; - oss << "WE: Error rolling back files " << txnID << " for session " << sessionID << "; " << ec.errorString(rc) << endl; - err = oss.str(); - } + if (rc != 0) + { + WErrorCodes ec; + ostringstream oss; + oss << "WE: Error rolling back files " << txnID << " for session " << sessionID << "; " + << ec.errorString(rc) << endl; + err = oss.str(); + } - std::map oids; + std::map oids; - if (idbdatafile::IDBPolicy::useHdfs()) - fWEWrapper.flushDataFiles(rc, txnID, oids); + if (idbdatafile::IDBPolicy::useHdfs()) + fWEWrapper.flushDataFiles(rc, txnID, oids); - purgeFDCache(); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - return rc; + purgeFDCache(); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + return rc; } uint8_t WE_DDLCommandProc::rollbackVersion(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; + int rc = 0; + uint32_t sessionID, tmp32; + int txnID; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; - rc = fWEWrapper.rollbackVersion(txnID, sessionID); + rc = fWEWrapper.rollbackVersion(txnID, sessionID); - if (rc != 0) - { - WErrorCodes ec; - ostringstream oss; - oss << "WE: Error rolling back transaction " << txnID << " for session " << sessionID << "; " << ec.errorString(rc) << endl; - err = oss.str(); - } + if (rc != 0) + { + WErrorCodes ec; + ostringstream oss; + oss << "WE: Error rolling back transaction " << txnID << " for session " << sessionID << "; " + << ec.errorString(rc) << endl; + err = oss.str(); + } - purgeFDCache(); - return rc; + purgeFDCache(); + return rc; } uint8_t WE_DDLCommandProc::deleteSyscolumn(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32;; - int txnID; - string schema, tablename; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; + int rc = 0; + uint32_t sessionID, tmp32; + ; + int txnID; + string schema, tablename; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + + ddlpackage::QualifiedName sysCatalogTableName; + sysCatalogTableName.fSchema = CALPONT_SCHEMA; + sysCatalogTableName.fName = SYSCOLUMN_TABLE; + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = schema; + userTableName.table = tablename; + + boost::shared_ptr systemCatalogPtr; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + fWEWrapper.setTransId(txnID); + fWEWrapper.startTransaction(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + + std::map oids; + // std::vector oidsToFlush; + + try + { + CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(userTableName); - ddlpackage::QualifiedName sysCatalogTableName; - sysCatalogTableName.fSchema = CALPONT_SCHEMA; - sysCatalogTableName.fName = SYSCOLUMN_TABLE; - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = schema; - userTableName.table = tablename; - - boost::shared_ptr systemCatalogPtr; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); - fWEWrapper.setTransId(txnID); - fWEWrapper.startTransaction(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - - std::map oids; - //std::vector oidsToFlush; - - try - { - CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(userTableName); - - WriteEngine::ColStruct colStruct; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector colValuesList; - WriteEngine::RIDList ridList; - std::vector ridLists; - DDLColumn column; - CalpontSystemCatalog::RIDList::const_iterator colrid_iterator = colRidList.begin(); - - while (colrid_iterator != colRidList.end()) - { - WriteEngine::RID rid = (*colrid_iterator).rid; - ridList.push_back(rid); - ++colrid_iterator; - } - - ColumnList columns; - getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); - - ColumnList::const_iterator column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - column = *column_iterator; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - colStructs.push_back(colStruct); - cscColTypeList.push_back(column.colType); - - ++column_iterator; - } - - colExtentsStruct.push_back(colStructs); - colExtentsColType.push_back(cscColTypeList); - ridLists.push_back(ridList); - - - if (0 != colStructs.size() && 0 != ridLists[0].size()) - { - int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if ( error == NO_ERROR) - rc = rc1; - else rc = error; - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } - catch (...) - { - err = "Unknown exception caught"; - rc = 1; - } - - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - return rc; -} - -uint8_t WE_DDLCommandProc::deleteSyscolumnRow(ByteStream& bs, std::string& err) -{ - int rc = 0; - uint32_t sessionID, tmp32;; - int txnID; - string schema, tablename, columnname; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - bs >> columnname; - - ddlpackage::QualifiedName sysCatalogTableName; - sysCatalogTableName.fSchema = CALPONT_SCHEMA; - sysCatalogTableName.fName = SYSCOLUMN_TABLE; - - CalpontSystemCatalog::TableColName tableColName; - tableColName.schema = schema; - tableColName.table = tablename; - tableColName.column = columnname; - - boost::shared_ptr systemCatalogPtr; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - std::map oids; - //std::vector oidsToFlush; - - try - { - CalpontSystemCatalog::ROPair colRO = systemCatalogPtr->columnRID(tableColName); - - if (colRO.objnum < 0) - { - err = "Column not found:" + tableColName.table + "." + tableColName.column; - throw std::runtime_error(err); - } - - WriteEngine::ColStruct colStruct; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector colValuesList; - WriteEngine::RIDList ridList; - std::vector ridLists; - DDLColumn column; - - ridList.push_back(colRO.rid); - - ColumnList columns; - getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); - - ColumnList::const_iterator column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - column = *column_iterator; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - colStructs.push_back(colStruct); - cscColTypeList.push_back(column.colType); - - ++column_iterator; - } - - colExtentsStruct.push_back(colStructs); - colExtentsColType.push_back(cscColTypeList); - ridLists.push_back(ridList); - - - if (0 != colStructs.size() && 0 != ridLists[0].size()) - { - int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if ( error == NO_ERROR) - rc = rc1; - else rc = error; - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } - catch (...) - { - err = "Unknown exception caught"; - rc = 1; - } - - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - - return rc; -} - -uint8_t WE_DDLCommandProc::deleteSystable(ByteStream& bs, std::string& err) -{ - int rc = 0; - uint32_t sessionID, tmp32;; - int txnID; - string schema, tablename; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - - WriteEngine::WriteEngineWrapper writeEngine; - ddlpackage::QualifiedName sysCatalogTableName; - sysCatalogTableName.fSchema = CALPONT_SCHEMA; - sysCatalogTableName.fName = SYSTABLE_TABLE; - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = schema; - userTableName.table = tablename; - - boost::shared_ptr systemCatalogPtr; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - - uint16_t dbRoot; - BRM::OID_t sysOid = 1001; - - //Find out where systcolumn is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); - fWEWrapper.setTransId(txnID); - - fWEWrapper.startTransaction(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - std::map oids; - //std::vector oidsToFlush; - - try - { - CalpontSystemCatalog::ROPair userTableROPair = systemCatalogPtr->tableRID(userTableName); - - if (userTableROPair.rid == std::numeric_limits::max()) - { - err = "RowID is not valid "; - throw std::runtime_error(err); - } - - WriteEngine::ColStruct colStruct; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector colValuesList; - WriteEngine::RIDList ridList; - std::vector ridLists; - DDLColumn column; - ridList.push_back(userTableROPair.rid); - - ColumnList columns; - getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); - - ColumnList::const_iterator column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - column = *column_iterator; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - colStructs.push_back(colStruct); - cscColTypeList.push_back(column.colType); - - ++column_iterator; - } - - colExtentsStruct.push_back(colStructs); - colExtentsColType.push_back(cscColTypeList); - ridLists.push_back(ridList); - - - if (0 != colStructs.size() && 0 != ridLists[0].size()) - { - int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if ( error == NO_ERROR) - rc = rc1; - else rc = error; - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } - catch (...) - { - err = "Unknown exception caught"; - rc = 1; - } - - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - - return rc; -} - -uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) -{ - int rc = 0; - uint32_t sessionID, tmp32;; - int txnID; - string schema, tablename; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - - WriteEngine::WriteEngineWrapper writeEngine; - ddlpackage::QualifiedName sysCatalogTableName; - sysCatalogTableName.fSchema = CALPONT_SCHEMA; - sysCatalogTableName.fName = SYSTABLE_TABLE; - - CalpontSystemCatalog::TableName userTableName; - userTableName.schema = schema; - userTableName.table = tablename; - - boost::shared_ptr systemCatalogPtr; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; WriteEngine::CSCTypesList cscColTypeList; @@ -1769,3247 +1344,3610 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) WriteEngine::RIDList ridList; std::vector ridLists; DDLColumn column; - uint16_t dbRoot; - BRM::OID_t sysOid = 1003; - //Find out where systable is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - std::map oids; + CalpontSystemCatalog::RIDList::const_iterator colrid_iterator = colRidList.begin(); - //std::vector oidsToFlush; - try + while (colrid_iterator != colRidList.end()) { - CalpontSystemCatalog::ROPair userTableROPair = systemCatalogPtr->tableRID(userTableName); - - if (userTableROPair.rid == std::numeric_limits::max()) - { - err = "RowID is not valid "; - throw std::runtime_error(err); - } - - ridList.push_back(userTableROPair.rid); - - ColumnList columns; - getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); - - ColumnList::const_iterator column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - column = *column_iterator; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - colStructs.push_back(colStruct); - cscColTypeList.push_back(column.colType); - - ++column_iterator; - } - - colExtentsStruct.push_back(colStructs); - colExtentsColType.push_back(cscColTypeList); - ridLists.push_back(ridList); - - - { - int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if ( error == NO_ERROR) - rc = rc1; - else rc = error; - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } - catch (...) - { - err = "Unknown exception caught"; - rc = 1; + WriteEngine::RID rid = (*colrid_iterator).rid; + ridList.push_back(rid); + ++colrid_iterator; } - if (rc != 0) - return rc; + ColumnList columns; + getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); - //deleting from SYSCOLUMN - sysCatalogTableName.fSchema = CALPONT_SCHEMA; - sysCatalogTableName.fName = SYSCOLUMN_TABLE; - sysOid = 1021; - //Find out where syscolumn is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + ColumnList::const_iterator column_iterator = columns.begin(); - try + while (column_iterator != columns.end()) { - CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(userTableName); + column = *column_iterator; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbRoot; - colStructs.clear(); - cscColTypeList.clear(); - colExtentsStruct.clear(); - colExtentsColType.clear(); - colValuesList.clear(); - ridList.clear(); - ridLists.clear(); - oids.clear(); - DDLColumn column; - CalpontSystemCatalog::RIDList::const_iterator colrid_iterator = colRidList.begin(); + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + } - while (colrid_iterator != colRidList.end()) - { - WriteEngine::RID rid = (*colrid_iterator).rid; - ridList.push_back(rid); - ++colrid_iterator; - } + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); - ColumnList columns; - getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); - - ColumnList::const_iterator column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - column = *column_iterator; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.colDataType = column.colType.colDataType; - colStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column.colType); - //oidsToFlush.push_back(colStruct.dataOid); - ++column_iterator; - } - - colExtentsStruct.push_back(colStructs); - colExtentsColType.push_back(cscColTypeList); - ridLists.push_back(ridList); - - - if (0 != colStructs.size() && 0 != ridLists[0].size()) - { - int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); - - if ((error == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if ( error == NO_ERROR) - rc = rc1; - else rc = error; - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } - catch (...) - { - err = "Unknown exception caught"; - rc = 1; + ++column_iterator; } - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); + colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); + ridLists.push_back(ridList); + + if (0 != colStructs.size() && 0 != ridLists[0].size()) + { + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, + SYSCOLUMN_BASE); + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error == NO_ERROR) + rc = rc1; + else + rc = error; + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } + catch (...) + { + err = "Unknown exception caught"; + rc = 1; + } + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; +} + +uint8_t WE_DDLCommandProc::deleteSyscolumnRow(ByteStream& bs, std::string& err) +{ + int rc = 0; + uint32_t sessionID, tmp32; + ; + int txnID; + string schema, tablename, columnname; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + bs >> columnname; + + ddlpackage::QualifiedName sysCatalogTableName; + sysCatalogTableName.fSchema = CALPONT_SCHEMA; + sysCatalogTableName.fName = SYSCOLUMN_TABLE; + + CalpontSystemCatalog::TableColName tableColName; + tableColName.schema = schema; + tableColName.table = tablename; + tableColName.column = columnname; + + boost::shared_ptr systemCatalogPtr; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + std::map oids; + // std::vector oidsToFlush; + + try + { + CalpontSystemCatalog::ROPair colRO = systemCatalogPtr->columnRID(tableColName); + + if (colRO.objnum < 0) + { + err = "Column not found:" + tableColName.table + "." + tableColName.column; + throw std::runtime_error(err); + } + + WriteEngine::ColStruct colStruct; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector colValuesList; + WriteEngine::RIDList ridList; + std::vector ridLists; + DDLColumn column; + + ridList.push_back(colRO.rid); + + ColumnList columns; + getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); + + ColumnList::const_iterator column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + column = *column_iterator; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbRoot; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + } + + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); + + ++column_iterator; + } + + colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); + ridLists.push_back(ridList); + + if (0 != colStructs.size() && 0 != ridLists[0].size()) + { + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, + SYSCOLUMN_BASE); + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error == NO_ERROR) + rc = rc1; + else + rc = error; + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } + catch (...) + { + err = "Unknown exception caught"; + rc = 1; + } + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + + return rc; +} + +uint8_t WE_DDLCommandProc::deleteSystable(ByteStream& bs, std::string& err) +{ + int rc = 0; + uint32_t sessionID, tmp32; + ; + int txnID; + string schema, tablename; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + + WriteEngine::WriteEngineWrapper writeEngine; + ddlpackage::QualifiedName sysCatalogTableName; + sysCatalogTableName.fSchema = CALPONT_SCHEMA; + sysCatalogTableName.fName = SYSTABLE_TABLE; + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = schema; + userTableName.table = tablename; + + boost::shared_ptr systemCatalogPtr; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + + uint16_t dbRoot; + BRM::OID_t sysOid = 1001; + + // Find out where systcolumn is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + fWEWrapper.setTransId(txnID); + + fWEWrapper.startTransaction(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + std::map oids; + // std::vector oidsToFlush; + + try + { + CalpontSystemCatalog::ROPair userTableROPair = systemCatalogPtr->tableRID(userTableName); + + if (userTableROPair.rid == std::numeric_limits::max()) + { + err = "RowID is not valid "; + throw std::runtime_error(err); + } + + WriteEngine::ColStruct colStruct; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector colValuesList; + WriteEngine::RIDList ridList; + std::vector ridLists; + DDLColumn column; + ridList.push_back(userTableROPair.rid); + + ColumnList columns; + getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); + + ColumnList::const_iterator column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + column = *column_iterator; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbRoot; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + } + + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); + + ++column_iterator; + } + + colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); + ridLists.push_back(ridList); + + if (0 != colStructs.size() && 0 != ridLists[0].size()) + { + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, + SYSCOLUMN_BASE); + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error == NO_ERROR) + rc = rc1; + else + rc = error; + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } + catch (...) + { + err = "Unknown exception caught"; + rc = 1; + } + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + + return rc; +} + +uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) +{ + int rc = 0; + uint32_t sessionID, tmp32; + ; + int txnID; + string schema, tablename; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + + WriteEngine::WriteEngineWrapper writeEngine; + ddlpackage::QualifiedName sysCatalogTableName; + sysCatalogTableName.fSchema = CALPONT_SCHEMA; + sysCatalogTableName.fName = SYSTABLE_TABLE; + + CalpontSystemCatalog::TableName userTableName; + userTableName.schema = schema; + userTableName.table = tablename; + + boost::shared_ptr systemCatalogPtr; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + WriteEngine::ColStruct colStruct; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector colValuesList; + WriteEngine::RIDList ridList; + std::vector ridLists; + DDLColumn column; + uint16_t dbRoot; + BRM::OID_t sysOid = 1003; + // Find out where systable is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + std::map oids; + + // std::vector oidsToFlush; + try + { + CalpontSystemCatalog::ROPair userTableROPair = systemCatalogPtr->tableRID(userTableName); + + if (userTableROPair.rid == std::numeric_limits::max()) + { + err = "RowID is not valid "; + throw std::runtime_error(err); + } + + ridList.push_back(userTableROPair.rid); + + ColumnList columns; + getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); + + ColumnList::const_iterator column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + column = *column_iterator; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbRoot; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + } + + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); + + ++column_iterator; + } + + colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); + ridLists.push_back(ridList); + + { + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, + SYSCOLUMN_BASE); + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error == NO_ERROR) + rc = rc1; + else + rc = error; + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } + catch (...) + { + err = "Unknown exception caught"; + rc = 1; + } + + if (rc != 0) return rc; + + // deleting from SYSCOLUMN + sysCatalogTableName.fSchema = CALPONT_SCHEMA; + sysCatalogTableName.fName = SYSCOLUMN_TABLE; + sysOid = 1021; + // Find out where syscolumn is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + + try + { + CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(userTableName); + + colStructs.clear(); + cscColTypeList.clear(); + colExtentsStruct.clear(); + colExtentsColType.clear(); + colValuesList.clear(); + ridList.clear(); + ridLists.clear(); + oids.clear(); + DDLColumn column; + CalpontSystemCatalog::RIDList::const_iterator colrid_iterator = colRidList.begin(); + + while (colrid_iterator != colRidList.end()) + { + WriteEngine::RID rid = (*colrid_iterator).rid; + ridList.push_back(rid); + ++colrid_iterator; + } + + ColumnList columns; + getColumnsForTable(sessionID, sysCatalogTableName.fSchema, sysCatalogTableName.fName, columns); + + ColumnList::const_iterator column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + column = *column_iterator; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.colDataType = column.colType.colDataType; + colStruct.fColDbRoot = dbRoot; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + } + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); + // oidsToFlush.push_back(colStruct.dataOid); + ++column_iterator; + } + + colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); + ridLists.push_back(ridList); + + if (0 != colStructs.size() && 0 != ridLists[0].size()) + { + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, + SYSCOLUMN_BASE); + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(error, txnID, oids); + + if ((error == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (error == NO_ERROR) + rc = rc1; + else + rc = error; + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } + catch (...) + { + err = "Unknown exception caught"; + rc = 1; + } + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; } uint8_t WE_DDLCommandProc::dropFiles(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t size, i; - uint32_t tmp32; - std::vector dataOids; + int rc = 0; + uint32_t size, i; + uint32_t tmp32; + std::vector dataOids; - bs >> size; + bs >> size; - for (i = 0; i < size; ++i) - { - bs >> tmp32; - dataOids.push_back(tmp32); - } + for (i = 0; i < size; ++i) + { + bs >> tmp32; + dataOids.push_back(tmp32); + } - try - { - rc = fWEWrapper.dropFiles(0, dataOids); - } - catch (...) - { - err = "WE: Error removing files "; - rc = 1; - } + try + { + rc = fWEWrapper.dropFiles(0, dataOids); + } + catch (...) + { + err = "WE: Error removing files "; + rc = 1; + } - purgeFDCache(); - return rc; + purgeFDCache(); + return rc; } uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - std::string schema, tablename; - int txnID; - uint8_t tmp8; - bool autoIncrement = false; + int rc = 0; + uint32_t sessionID, tmp32; + std::string schema, tablename; + int txnID; + uint8_t tmp8; + bool autoIncrement = false; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - bs >> tmp8; - autoIncrement = true; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + bs >> tmp8; + autoIncrement = true; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = tablename; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = tablename; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; - CalpontSystemCatalog::RIDList roList; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - - try - { - roList = systemCatalogPtr->columnRIDs(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - //Build colStructs for SYSTABLE - std::vector ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - std::map oids; - //std::vector oidsToFlush; - - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSCOLUMN_TABLE; - DDLColumn column; - WriteEngine::ColTuple colTuple; - - findColumnData(sessionID, tableName, AUTOINC_COL, column); - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.colDataType = column.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - string s1("y"), s2("n"); - boost::any datavalue1 = s1; - boost::any datavalue2 = s2; - - if (autoIncrement) - colTuple.data = datavalue1; - else - colTuple.data = datavalue2; - - colStruct.colDataType = column.colType.colDataType; - - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - dctnryStructList.push_back(dctnryStruct); - cscColTypeList.push_back(column.colType); - - for (unsigned int i = 0; i < roList.size(); i++) - { - aColList.push_back(colTuple); - } - - colValuesList.push_back(aColList); - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector dctnryExtentsStruct; - std::vector extentsinfo; - extentInfo aExtentinfo; - CalpontSystemCatalog::OID oid = 1021; - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.setTransId(txnID); - - for (unsigned int i = 0; i < roList.size(); i++) - { - convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); - - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - if (extentsinfo.empty()) - extentsinfo.push_back(aExtentinfo); - else if (extentsinfo.back() != aExtentinfo) - extentsinfo.push_back(aExtentinfo); - - ridList.push_back(roList[i].rid); - } - - std::vector ridLists; - ridLists.push_back(ridList); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - if (idbdatafile::IDBPolicy::useHdfs()) - fWEWrapper.startTransaction(txnID); - - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - } - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; + CalpontSystemCatalog::RIDList roList; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + try + { + roList = systemCatalogPtr->columnRIDs(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + // Build colStructs for SYSTABLE + std::vector ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + std::map oids; + // std::vector oidsToFlush; + + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSCOLUMN_TABLE; + DDLColumn column; + WriteEngine::ColTuple colTuple; + + findColumnData(sessionID, tableName, AUTOINC_COL, column); + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.colDataType = column.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + } + + string s1("y"), s2("n"); + boost::any datavalue1 = s1; + boost::any datavalue2 = s2; + + if (autoIncrement) + colTuple.data = datavalue1; + else + colTuple.data = datavalue2; + + colStruct.colDataType = column.colType.colDataType; + + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); + + for (unsigned int i = 0; i < roList.size(); i++) + { + aColList.push_back(colTuple); + } + + colValuesList.push_back(aColList); + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector dctnryExtentsStruct; + std::vector extentsinfo; + extentInfo aExtentinfo; + CalpontSystemCatalog::OID oid = 1021; + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.setTransId(txnID); + + for (unsigned int i = 0; i < roList.size(); i++) + { + convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); + + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + if (extentsinfo.empty()) + extentsinfo.push_back(aExtentinfo); + else if (extentsinfo.back() != aExtentinfo) + extentsinfo.push_back(aExtentinfo); + + ridList.push_back(roList[i].rid); + } + + std::vector ridLists; + ridLists.push_back(ridList); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } + + // call the write engine to update the row + if (idbdatafile::IDBPolicy::useHdfs()) + fWEWrapper.startTransaction(txnID); + + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + + return rc; } uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - std::string schema, tablename; - int txnID; - uint8_t tmp8; - bool autoIncrement = false; + int rc = 0; + uint32_t sessionID, tmp32; + std::string schema, tablename; + int txnID; + uint8_t tmp8; + bool autoIncrement = false; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - bs >> tmp8; - autoIncrement = true; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + bs >> tmp8; + autoIncrement = true; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = tablename; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = tablename; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; - CalpontSystemCatalog::RIDList roList; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::RIDList roList; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - try - { - roList = systemCatalogPtr->columnRIDs(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - //Build colStructs for SYSTABLE - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSCOLUMN_TABLE; - DDLColumn column; - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - WriteEngine::ColTuple colTuple; - std::vector ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - std::map oids; - //std::vector oidsToFlush; - - boost::any datavalue; - findColumnData(sessionID, tableName, AUTOINC_COL, column); - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.colDataType = column.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - string ystr("y"); - string nstr("n"); - - if (autoIncrement) - colTuple.data = ystr; - else - colTuple.data = nstr; - - colStruct.colDataType = column.colType.colDataType; - - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - cscColTypeList.push_back(column.colType); - - for (unsigned int i = 0; i < roList.size(); i++) - { - aColList.push_back(colTuple); - } - - colValuesList.push_back(aColList); - - - //get start dbroot for this PM. - //int PMNum = Config::getLocalModuleID(); - std::vector extentsinfo; - extentInfo aExtentinfo; - - - //oam.getDbroots(PMNum); - //dbRoot will be the first dbroot on this pm. dbrootCnt will be how many dbroots on this PM. - CalpontSystemCatalog::OID oid = 1021; - - for (unsigned int i = 0; i < roList.size(); i++) - { - convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); - - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - if (extentsinfo.empty()) - extentsinfo.push_back(aExtentinfo); - else if (extentsinfo.back() != aExtentinfo) - extentsinfo.push_back(aExtentinfo); - - ridList.push_back(roList[i].rid); - } - - std::vector ridLists; - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector dctnryExtentsStruct; - ridLists.push_back(ridList); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - fWEWrapper.setTransId(txnID); - fWEWrapper.startTransaction(txnID); - - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - } - - if (idbdatafile::IDBPolicy::useHdfs()) - { - fWEWrapper.flushDataFiles(rc, txnID, oids); - fWEWrapper.confirmTransaction(txnID); - - if ( rc == 0) - fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - - systemCatalogPtr->flushCache(); - purgeFDCache(); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); + try + { + roList = systemCatalogPtr->columnRIDs(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + // Build colStructs for SYSTABLE + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSCOLUMN_TABLE; + DDLColumn column; + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + WriteEngine::ColTuple colTuple; + std::vector ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + std::map oids; + // std::vector oidsToFlush; + + boost::any datavalue; + findColumnData(sessionID, tableName, AUTOINC_COL, column); + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.colDataType = column.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + string ystr("y"); + string nstr("n"); + + if (autoIncrement) + colTuple.data = ystr; + else + colTuple.data = nstr; + + colStruct.colDataType = column.colType.colDataType; + + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); + + for (unsigned int i = 0; i < roList.size(); i++) + { + aColList.push_back(colTuple); + } + + colValuesList.push_back(aColList); + + // get start dbroot for this PM. + // int PMNum = Config::getLocalModuleID(); + std::vector extentsinfo; + extentInfo aExtentinfo; + + // oam.getDbroots(PMNum); + // dbRoot will be the first dbroot on this pm. dbrootCnt will be how many dbroots on this PM. + CalpontSystemCatalog::OID oid = 1021; + + for (unsigned int i = 0; i < roList.size(); i++) + { + convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); + + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + if (extentsinfo.empty()) + extentsinfo.push_back(aExtentinfo); + else if (extentsinfo.back() != aExtentinfo) + extentsinfo.push_back(aExtentinfo); + + ridList.push_back(roList[i].rid); + } + + std::vector ridLists; + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector dctnryExtentsStruct; + ridLists.push_back(ridList); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } + + // call the write engine to update the row + fWEWrapper.setTransId(txnID); + fWEWrapper.startTransaction(txnID); + + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; + } + + if (idbdatafile::IDBPolicy::useHdfs()) + { + fWEWrapper.flushDataFiles(rc, txnID, oids); + fWEWrapper.confirmTransaction(txnID); + + if (rc == 0) + fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + + systemCatalogPtr->flushCache(); + purgeFDCache(); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + return rc; } uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - std::string schema, oldTablename, newTablename; - int txnID; + int rc = 0; + uint32_t sessionID, tmp32; + std::string schema, oldTablename, newTablename; + int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> oldTablename; - bs >> newTablename; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> oldTablename; + bs >> newTablename; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = oldTablename; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; - WriteEngine::ColTuple colTuple; - std::map oids; - //std::vector oidsToFlush; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = oldTablename; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; + WriteEngine::ColTuple colTuple; + std::map oids; + // std::vector oidsToFlush; - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; - CalpontSystemCatalog::RIDList roList; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::RIDList roList; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - try - { - roList = systemCatalogPtr->columnRIDs(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - //Build colStructs for SYSTABLE - std::vector ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSCOLUMN_TABLE; - DDLColumn column; - findColumnData(sessionID, tableName, TABLENAME_COL, column); - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - WriteEngine::DctnryTuple dictTuple; - dictTuple.isNull = false; - - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - - if ( (column.colType.colDataType == CalpontSystemCatalog::CHAR - && column.colType.colWidth > 8) - || (column.colType.colDataType == CalpontSystemCatalog::VARCHAR - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::BLOB - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::TEXT - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL - && column.colType.precision > 18) - || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL - && column.colType.precision > 18) )//token - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = column.colType.colWidth; - } - - colStruct.colDataType = column.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - cscColTypeList.push_back(column.colType); - - for (unsigned int i = 0; i < roList.size(); i++) - { - aColList.push_back(colTuple); - } - - colValuesList.push_back(aColList); - - - //It's the same string for each column, so we just need one dictionary struct - void *dictTuplePtr = static_cast(&dictTuple); - memset(dictTuplePtr, 0, sizeof(dictTuple)); - dictTuple.sigValue = (unsigned char*)newTablename.c_str(); - dictTuple.sigSize = newTablename.length(); - dictTuple.isNull = false; - dctColList = dictTuple; - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - CalpontSystemCatalog::OID oid = 1021; - std::vector extentsinfo; - extentInfo aExtentinfo; - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector dctnryExtentsStruct; - - for (unsigned int i = 0; i < roList.size(); i++) - { - convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); - - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - if (extentsinfo.empty()) - extentsinfo.push_back(aExtentinfo); - else if (extentsinfo.back() != aExtentinfo) - extentsinfo.push_back(aExtentinfo); - - ridList.push_back(roList[i].rid); - } - - std::vector ridLists; - ridLists.push_back(ridList); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - } - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (rc == 0 ) - rc = rc1; - - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - //cout << "rename:syscolumn is updated" << endl; + try + { + roList = systemCatalogPtr->columnRIDs(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + // Build colStructs for SYSTABLE + std::vector ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSCOLUMN_TABLE; + DDLColumn column; + findColumnData(sessionID, tableName, TABLENAME_COL, column); + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + WriteEngine::DctnryTuple dictTuple; + dictTuple.isNull = false; + + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + + if ((column.colType.colDataType == CalpontSystemCatalog::CHAR && column.colType.colWidth > 8) || + (column.colType.colDataType == CalpontSystemCatalog::VARCHAR && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::VARBINARY && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::BLOB && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::TEXT && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::DECIMAL && column.colType.precision > 18) || + (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL && + column.colType.precision > 18)) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = column.colType.colWidth; + } + + colStruct.colDataType = column.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + oids[colStruct.dataOid] = colStruct.dataOid; + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); + + for (unsigned int i = 0; i < roList.size(); i++) + { + aColList.push_back(colTuple); + } + + colValuesList.push_back(aColList); + + // It's the same string for each column, so we just need one dictionary struct + void* dictTuplePtr = static_cast(&dictTuple); + memset(dictTuplePtr, 0, sizeof(dictTuple)); + dictTuple.sigValue = (unsigned char*)newTablename.c_str(); + dictTuple.sigSize = newTablename.length(); + dictTuple.isNull = false; + dctColList = dictTuple; + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + CalpontSystemCatalog::OID oid = 1021; + std::vector extentsinfo; + extentInfo aExtentinfo; + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector dctnryExtentsStruct; + + for (unsigned int i = 0; i < roList.size(); i++) + { + convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); + + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + if (extentsinfo.empty()) + extentsinfo.push_back(aExtentinfo); + else if (extentsinfo.back() != aExtentinfo) + extentsinfo.push_back(aExtentinfo); + + ridList.push_back(roList[i].rid); + } + + std::vector ridLists; + ridLists.push_back(ridList); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } + + // call the write engine to update the row + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + // cout << "rename:syscolumn is updated" << endl; + return rc; } uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32, autoVal; - std::string schema, tablename; - int txnID; + int rc = 0; + uint32_t sessionID, tmp32, autoVal; + std::string schema, tablename; + int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tablename; - bs >> autoVal; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tablename; + bs >> autoVal; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = tablename; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = tablename; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; - CalpontSystemCatalog::ROPair ropair; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair ropair; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - try - { - ropair = systemCatalogPtr->tableRID(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - if (ropair.objnum < 0) - { - err = "No such table: " + tableName.table; - rc = 1; - return rc; - } - - // now we have to prepare the various structures for the WE to update the column. - - std::vector ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - std::map oids; - //std::vector oidsToFlush; - boost::any datavalue; - datavalue = autoVal; - - WriteEngine::ColTuple colTuple; - - //Build colStructs for SYSTABLE - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSTABLE_TABLE; - DDLColumn column; - findColumnData (sessionID, tableName, AUTOINC_COL, column); - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - - colStruct.colDataType = column.colType.colDataType; - - colTuple.data = datavalue; - - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - colStructs.push_back(colStruct); - cscColTypeList.push_back(column.colType); - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - dctnryStructList.push_back(dctnryStruct); - aColList.push_back(colTuple); - colValuesList.push_back(aColList); - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector dctnryExtentsStruct; - - - WriteEngine::DctnryTuple dctnryTuple; - dctColList = dctnryTuple; - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //In this case, there's only 1 row, so only one one extent, but keep it generic... - std::vector extentsinfo; - extentInfo aExtentinfo; - CalpontSystemCatalog::OID oid = 1003; - convertRidToColumn(ropair.rid, dbRoot, partition, segment, oid); - - ridList.push_back(ropair.rid); - std::vector ridLists; - ridLists.push_back(ridList); - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - extentsinfo.push_back(aExtentinfo); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - colExtentsColType.push_back(cscColTypeList); - dctnryExtentsStruct.push_back(dctnryStructList); - } - - // call the write engine to update the row - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - } - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (rc == 0 ) - rc = rc1; - - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); + try + { + ropair = systemCatalogPtr->tableRID(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + if (ropair.objnum < 0) + { + err = "No such table: " + tableName.table; + rc = 1; + return rc; + } + + // now we have to prepare the various structures for the WE to update the column. + + std::vector ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + std::map oids; + // std::vector oidsToFlush; + boost::any datavalue; + datavalue = autoVal; + + WriteEngine::ColTuple colTuple; + + // Build colStructs for SYSTABLE + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSTABLE_TABLE; + DDLColumn column; + findColumnData(sessionID, tableName, AUTOINC_COL, column); + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + + colStruct.colDataType = column.colType.colDataType; + + colTuple.data = datavalue; + + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + dctnryStructList.push_back(dctnryStruct); + aColList.push_back(colTuple); + colValuesList.push_back(aColList); + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector dctnryExtentsStruct; + + WriteEngine::DctnryTuple dctnryTuple; + dctColList = dctnryTuple; + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // In this case, there's only 1 row, so only one one extent, but keep it generic... + std::vector extentsinfo; + extentInfo aExtentinfo; + CalpontSystemCatalog::OID oid = 1003; + convertRidToColumn(ropair.rid, dbRoot, partition, segment, oid); + + ridList.push_back(ropair.rid); + std::vector ridLists; + ridLists.push_back(ridList); + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + extentsinfo.push_back(aExtentinfo); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); + dctnryExtentsStruct.push_back(dctnryStructList); + } + + // call the write engine to update the row + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; } uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - std::string schema, oldTablename, newTablename; - int txnID; + int rc = 0; + uint32_t sessionID, tmp32; + std::string schema, oldTablename, newTablename; + int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> oldTablename; - bs >> newTablename; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> oldTablename; + bs >> newTablename; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = oldTablename; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = oldTablename; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; - CalpontSystemCatalog::ROPair ropair; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair ropair; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - try + try + { + ropair = systemCatalogPtr->tableRID(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + if (ropair.objnum < 0) + { + err = "No such table: " + tableName.table; + return 1; + } + + // now we have to prepare the various structures for the WE to update the column. + + std::vector ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + std::map oids; + // std::vector oidsToFlush; + boost::any datavalue; + datavalue = newTablename; + + WriteEngine::ColTuple colTuple; + + // Build colStructs for SYSTABLE + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSTABLE_TABLE; + DDLColumn column; + findColumnData(sessionID, tableName, TABLENAME_COL, column); + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = true; + + colStruct.colDataType = column.colType.colDataType; + // Tokenize the data value + WriteEngine::DctnryStruct dictStruct; + dictStruct.dctnryOid = column.colType.ddn.dictOID; + dictStruct.columnOid = column.colType.columnOID; + WriteEngine::DctnryTuple dictTuple; + dictTuple.isNull = false; + dictTuple.sigValue = (unsigned char*)newTablename.c_str(); + dictTuple.sigSize = newTablename.length(); + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + aColList.push_back(colTuple); + colValuesList.push_back(aColList); + std::vector colExtentsStruct; + std::vector dctnryExtentsStruct; + std::vector colExtentsColType; + + dctColList = dictTuple; + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // In this case, there's only 1 row, so only one one extent, but keep it generic... + std::vector extentsinfo; + extentInfo aExtentinfo; + CalpontSystemCatalog::OID oid = 1003; + convertRidToColumn(ropair.rid, dbRoot, partition, segment, oid); + + ridList.push_back(ropair.rid); + std::vector ridLists; + ridLists.push_back(ridList); + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + extentsinfo.push_back(aExtentinfo); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) { - ropair = systemCatalogPtr->tableRID(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; } - if (ropair.objnum < 0) - { - err = "No such table: " + tableName.table; - return 1; - } + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } - // now we have to prepare the various structures for the WE to update the column. + // call the write engine to update the row + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); - std::vector ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - std::map oids; - //std::vector oidsToFlush; - boost::any datavalue; - datavalue = newTablename; - - WriteEngine::ColTuple colTuple; - - //Build colStructs for SYSTABLE - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSTABLE_TABLE; - DDLColumn column; - findColumnData (sessionID, tableName, TABLENAME_COL, column); - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = true; - - colStruct.colDataType = column.colType.colDataType; - //Tokenize the data value - WriteEngine::DctnryStruct dictStruct; - dictStruct.dctnryOid = column.colType.ddn.dictOID; - dictStruct.columnOid = column.colType.columnOID; - WriteEngine::DctnryTuple dictTuple; - dictTuple.isNull = false; - dictTuple.sigValue = (unsigned char*)newTablename.c_str(); - dictTuple.sigSize = newTablename.length(); - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - aColList.push_back(colTuple); - colValuesList.push_back(aColList); - std::vector colExtentsStruct; - std::vector dctnryExtentsStruct; - std::vector colExtentsColType; - - dctColList = dictTuple; - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //In this case, there's only 1 row, so only one one extent, but keep it generic... - std::vector extentsinfo; - extentInfo aExtentinfo; - CalpontSystemCatalog::OID oid = 1003; - convertRidToColumn(ropair.rid, dbRoot, partition, segment, oid); - - ridList.push_back(ropair.rid); - std::vector ridLists; - ridLists.push_back(ridList); - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - extentsinfo.push_back(aExtentinfo); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (rc == 0 ) - rc = rc1; - - if (rc != 0) - return rc; - } + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - if ((rc == 0) && ( rc1 == 0)) - { + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); else - { - fWEWrapper.endTransaction(txnID, false); - } + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } } - if (rc == 0 ) - rc = rc1; + if (rc == 0) + rc = rc1; - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - //cout << "rename:syscolumn is updated" << endl; - return rc; + if (rc != 0) + return rc; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + // cout << "rename:syscolumn is updated" << endl; + return rc; } uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - std::string schema, oldTablename, newTablename; - int txnID; + int rc = 0; + uint32_t sessionID, tmp32; + std::string schema, oldTablename, newTablename; + int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> oldTablename; - bs >> newTablename; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> oldTablename; + bs >> newTablename; - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = oldTablename; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = oldTablename; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; - CalpontSystemCatalog::ROPair ropair; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair ropair; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - //@bug 4592 Error handling for syscat call + //@bug 4592 Error handling for syscat call - try + try + { + ropair = systemCatalogPtr->tableRID(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + if (ropair.objnum < 0) + { + err = "No such table: " + tableName.table; + return 1; + } + + // now we have to prepare the various structures for the WE to update the column. + + std::vector ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + std::map oids; + // std::vector oidsToFlush; + boost::any datavalue; + datavalue = newTablename; + + WriteEngine::ColTuple colTuple; + + // Build colStructs for SYSTABLE + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSTABLE_TABLE; + DDLColumn column; + findColumnData(sessionID, tableName, TABLENAME_COL, column); + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = true; + + colStruct.colDataType = column.colType.colDataType; + // Tokenize the data value + WriteEngine::DctnryStruct dictStruct; + dictStruct.dctnryOid = column.colType.ddn.dictOID; + dictStruct.columnOid = column.colType.columnOID; + WriteEngine::DctnryTuple dictTuple; + dictTuple.isNull = false; + dictTuple.sigValue = (unsigned char*)newTablename.c_str(); + dictTuple.sigSize = newTablename.length(); + // int error = NO_ERROR; + // if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple))) + //{ + // WErrorCodes ec; + // throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); + //} + // WriteEngine::Token aToken = dictTuple.token; + + // colTuple.data = aToken; + // cout << "token value for new table name is op:fbo = " << aToken.op <<":" << aToken.fbo << " null flag = " + // << (uint32_t)dictTuple.isNull<< endl; + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + aColList.push_back(colTuple); + colValuesList.push_back(aColList); + std::vector colExtentsStruct; + std::vector dctnryExtentsStruct; + std::vector colExtentsColType; + + dctColList = dictTuple; + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // In this case, there's only 1 row, so only one one extent, but keep it generic... + std::vector extentsinfo; + extentInfo aExtentinfo; + CalpontSystemCatalog::OID oid = 1003; + convertRidToColumn(ropair.rid, dbRoot, partition, segment, oid); + + ridList.push_back(ropair.rid); + std::vector ridLists; + ridLists.push_back(ridList); + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + extentsinfo.push_back(aExtentinfo); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) { - ropair = systemCatalogPtr->tableRID(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; } - if (ropair.objnum < 0) - { - err = "No such table: " + tableName.table; - return 1; - } + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } - // now we have to prepare the various structures for the WE to update the column. + // call the write engine to update the row + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); - std::vector ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - std::map oids; - //std::vector oidsToFlush; - boost::any datavalue; - datavalue = newTablename; - - WriteEngine::ColTuple colTuple; - - //Build colStructs for SYSTABLE - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSTABLE_TABLE; - DDLColumn column; - findColumnData (sessionID, tableName, TABLENAME_COL, column); - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = true; - - colStruct.colDataType = column.colType.colDataType; - //Tokenize the data value - WriteEngine::DctnryStruct dictStruct; - dictStruct.dctnryOid = column.colType.ddn.dictOID; - dictStruct.columnOid = column.colType.columnOID; - WriteEngine::DctnryTuple dictTuple; - dictTuple.isNull = false; - dictTuple.sigValue = (unsigned char*)newTablename.c_str(); - dictTuple.sigSize = newTablename.length(); - //int error = NO_ERROR; - //if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple))) - //{ - // WErrorCodes ec; - // throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); - //} - //WriteEngine::Token aToken = dictTuple.token; - - //colTuple.data = aToken; - //cout << "token value for new table name is op:fbo = " << aToken.op <<":" << aToken.fbo << " null flag = " << (uint32_t)dictTuple.isNull<< endl; - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - aColList.push_back(colTuple); - colValuesList.push_back(aColList); - std::vector colExtentsStruct; - std::vector dctnryExtentsStruct; - std::vector colExtentsColType; - - dctColList = dictTuple; - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //In this case, there's only 1 row, so only one one extent, but keep it generic... - std::vector extentsinfo; - extentInfo aExtentinfo; - CalpontSystemCatalog::OID oid = 1003; - convertRidToColumn(ropair.rid, dbRoot, partition, segment, oid); - - ridList.push_back(ropair.rid); - std::vector ridLists; - ridLists.push_back(ridList); - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - extentsinfo.push_back(aExtentinfo); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (rc == 0 ) - rc = rc1; - - if (rc != 0) - return rc; - } - - //cout << "rename:systable is updated to " << newTablename << " for rid " << ropair.rid << endl; - //Update SYSCOLUMN table - tableName.schema = schema; - tableName.table = oldTablename; - dctnryStructList.clear(); - dctnryValueList.clear(); - dctRowList.clear(); - - CalpontSystemCatalog::RIDList roList; - - try - { - roList = systemCatalogPtr->columnRIDs(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - //Build colStructs for SYSCOLUMN - ridList.clear(); - colValuesList.clear(); - aColList.clear(); - colStructs.clear(); - cscColTypeList.clear(); - colOldValuesList.clear(); - oids.clear(); - tableName.schema = CALPONT_SCHEMA; - tableName.table = SYSCOLUMN_TABLE; - findColumnData(sessionID, tableName, TABLENAME_COL, column); - - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - - if ( (column.colType.colDataType == CalpontSystemCatalog::CHAR - && column.colType.colWidth > 8) - || (column.colType.colDataType == CalpontSystemCatalog::VARCHAR - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::BLOB - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::TEXT - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL - && column.colType.precision > 18) - || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL - && column.colType.precision > 18) )//token - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = column.colType.colWidth; - } - - colStruct.colDataType = column.colType.colDataType; - - //Tokenize the data value - dictStruct.dctnryOid = column.colType.ddn.dictOID; - dictStruct.columnOid = column.colType.columnOID; - dictTuple.sigValue = (unsigned char*)newTablename.c_str(); - dictTuple.sigSize = newTablename.length(); - dictTuple.isNull = false; - /* - if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple))) - { - WErrorCodes ec; - throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); - } - aToken = dictTuple.token; - colTuple.data = aToken; */ - - colStruct.colDataType = column.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - oids[colStruct.dataOid] = colStruct.dataOid; - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - cscColTypeList.push_back(column.colType); - - for (unsigned int i = 0; i < roList.size(); i++) - { - aColList.push_back(colTuple); - } - - colValuesList.push_back(aColList); - - - //It's the same string for each column, so we just need one dictionary struct - void *dictTuplePtr = static_cast(&dictTuple); - memset(dictTuplePtr, 0, sizeof(dictTuple)); - dictTuple.sigValue = (unsigned char*)newTablename.c_str(); - dictTuple.sigSize = newTablename.length(); - dictTuple.isNull = false; - dctColList = dictTuple; - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - extentsinfo.clear(); - colExtentsStruct.clear(); - colExtentsColType.clear(); - dctnryExtentsStruct.clear(); - oid = 1021; - - for (unsigned int i = 0; i < roList.size(); i++) - { - convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); - - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - if (extentsinfo.empty()) - extentsinfo.push_back(aExtentinfo); - else if (extentsinfo.back() != aExtentinfo) - extentsinfo.push_back(aExtentinfo); - - ridList.push_back(roList[i].rid); - } - - ridLists.clear(); - ridLists.push_back(ridList); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); - - if (rc != NO_ERROR) - { - // build the logging message - err = "WE: Update failed on: " + tableName.table; - } + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - if ((rc == 0) && ( rc1 == 0)) - { + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); else - { - fWEWrapper.endTransaction(txnID, false); - } + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } } - if (rc == 0 ) - rc = rc1; + if (rc == 0) + rc = rc1; - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - //cout << "rename:syscolumn is updated" << endl; + if (rc != 0) + return rc; + } + + // cout << "rename:systable is updated to " << newTablename << " for rid " << ropair.rid << endl; + // Update SYSCOLUMN table + tableName.schema = schema; + tableName.table = oldTablename; + dctnryStructList.clear(); + dctnryValueList.clear(); + dctRowList.clear(); + + CalpontSystemCatalog::RIDList roList; + + try + { + roList = systemCatalogPtr->columnRIDs(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + // Build colStructs for SYSCOLUMN + ridList.clear(); + colValuesList.clear(); + aColList.clear(); + colStructs.clear(); + cscColTypeList.clear(); + colOldValuesList.clear(); + oids.clear(); + tableName.schema = CALPONT_SCHEMA; + tableName.table = SYSCOLUMN_TABLE; + findColumnData(sessionID, tableName, TABLENAME_COL, column); + + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + + if ((column.colType.colDataType == CalpontSystemCatalog::CHAR && column.colType.colWidth > 8) || + (column.colType.colDataType == CalpontSystemCatalog::VARCHAR && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::VARBINARY && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::BLOB && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::TEXT && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::DECIMAL && column.colType.precision > 18) || + (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL && + column.colType.precision > 18)) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = column.colType.colWidth; + } + + colStruct.colDataType = column.colType.colDataType; + + // Tokenize the data value + dictStruct.dctnryOid = column.colType.ddn.dictOID; + dictStruct.columnOid = column.colType.columnOID; + dictTuple.sigValue = (unsigned char*)newTablename.c_str(); + dictTuple.sigSize = newTablename.length(); + dictTuple.isNull = false; + /* + if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple))) + { + WErrorCodes ec; + throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); + } + aToken = dictTuple.token; + colTuple.data = aToken; */ + + colStruct.colDataType = column.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + oids[colStruct.dataOid] = colStruct.dataOid; + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); + + for (unsigned int i = 0; i < roList.size(); i++) + { + aColList.push_back(colTuple); + } + + colValuesList.push_back(aColList); + + // It's the same string for each column, so we just need one dictionary struct + void* dictTuplePtr = static_cast(&dictTuple); + memset(dictTuplePtr, 0, sizeof(dictTuple)); + dictTuple.sigValue = (unsigned char*)newTablename.c_str(); + dictTuple.sigSize = newTablename.length(); + dictTuple.isNull = false; + dctColList = dictTuple; + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + extentsinfo.clear(); + colExtentsStruct.clear(); + colExtentsColType.clear(); + dctnryExtentsStruct.clear(); + oid = 1021; + + for (unsigned int i = 0; i < roList.size(); i++) + { + convertRidToColumn(roList[i].rid, dbRoot, partition, segment, oid); + + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + if (extentsinfo.empty()) + extentsinfo.push_back(aExtentinfo); + else if (extentsinfo.back() != aExtentinfo) + extentsinfo.push_back(aExtentinfo); + + ridList.push_back(roList[i].rid); + } + + ridLists.clear(); + ridLists.push_back(ridList); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } + + // call the write engine to update the row + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, + ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); + + if (rc != NO_ERROR) + { + // build the logging message + err = "WE: Update failed on: " + tableName.table; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + // cout << "rename:syscolumn is updated" << endl; + return rc; } uint8_t WE_DDLCommandProc::updateSyscolumnColumnposCol(messageqcpp::ByteStream& bs, std::string& err) { - int rc = 0; - int colPos; - string schema, atableName; - uint32_t sessionID, tmp32; - int txnID; + int rc = 0; + int colPos; + string schema, atableName; + uint32_t sessionID, tmp32; + int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> atableName; - bs >> tmp32; - colPos = tmp32; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> atableName; + bs >> tmp32; + colPos = tmp32; - WriteEngine::RIDList ridList; - WriteEngine::ColValueList colValuesList; - WriteEngine::ColValueList colOldValuesList; - CalpontSystemCatalog::TableName tableName; - tableName.table = atableName; - tableName.schema = schema; - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::RIDList rids; + WriteEngine::RIDList ridList; + WriteEngine::ColValueList colValuesList; + WriteEngine::ColValueList colOldValuesList; + CalpontSystemCatalog::TableName tableName; + tableName.table = atableName; + tableName.schema = schema; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::RIDList rids; - try + try + { + rids = systemCatalogPtr->columnRIDs(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + CalpontSystemCatalog::RIDList::const_iterator rid_iter = rids.begin(); + boost::any value; + WriteEngine::ColTupleList colTuples; + CalpontSystemCatalog::ColType columnType; + CalpontSystemCatalog::ROPair colRO; + + // cout << "colpos is " << colPos << endl; + try + { + while (rid_iter != rids.end()) { - rids = systemCatalogPtr->columnRIDs(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; + // look up colType + colRO = *rid_iter; + columnType = systemCatalogPtr->colType(colRO.objnum); + + if (columnType.colPosition < colPos) + { + ++rid_iter; + continue; + } + + ridList.push_back(colRO.rid); + value = columnType.colPosition - 1; + WriteEngine::ColTuple colTuple; + colTuple.data = value; + colTuples.push_back(colTuple); + ++rid_iter; } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } - CalpontSystemCatalog::RIDList::const_iterator rid_iter = rids.begin(); - boost::any value; - WriteEngine::ColTupleList colTuples; - CalpontSystemCatalog::ColType columnType; - CalpontSystemCatalog::ROPair colRO; + colValuesList.push_back(colTuples); + uint16_t dbRoot; + BRM::OID_t sysOid = 1021; + // Find out where systable is + rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + std::map oids; - //cout << "colpos is " << colPos << endl; - try - { - while (rid_iter != rids.end()) - { - //look up colType - colRO = *rid_iter; - columnType = systemCatalogPtr->colType(colRO.objnum); - - if (columnType.colPosition < colPos) - { - ++rid_iter; - continue; - } - - ridList.push_back(colRO.rid); - value = columnType.colPosition - 1; - WriteEngine::ColTuple colTuple; - colTuple.data = value; - colTuples.push_back(colTuple); - ++rid_iter; - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - colValuesList.push_back(colTuples); - uint16_t dbRoot; - BRM::OID_t sysOid = 1021; - //Find out where systable is - rc = fDbrm.getSysCatDBRoot(sysOid, dbRoot); - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - std::map oids; - - //std::vector oidsToFlush; - if (colTuples.size() > 0) - { - WriteEngine::ColStructList colStructs; - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::CSCTypesList cscColTypeList; - CalpontSystemCatalog::ColType colType; - //Build column structure for COLUMNPOS_COL - colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; - colType.colWidth = colStruct.colWidth = 4; - colStruct.tokenFlag = false; - colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::INT; - colStruct.fColDbRoot = dbRoot; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - } - - colStructs.push_back(colStruct); - cscColTypeList.push_back(colType); - oids[colStruct.dataOid] = colStruct.dataOid; - //oidsToFlush.push_back(colStruct.dataOid); - rc = fWEWrapper.updateColumnRecs( txnID, cscColTypeList, colStructs, colValuesList, ridList, SYSCOLUMN_BASE ); - } - - int rc1 = 0; + // std::vector oidsToFlush; + if (colTuples.size() > 0) + { + WriteEngine::ColStructList colStructs; + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::CSCTypesList cscColTypeList; + CalpontSystemCatalog::ColType colType; + // Build column structure for COLUMNPOS_COL + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; + colType.colWidth = colStruct.colWidth = 4; + colStruct.tokenFlag = false; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::INT; + colStruct.fColDbRoot = dbRoot; if (idbdatafile::IDBPolicy::useHdfs()) { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } + colStruct.fCompressionType = 2; } - if (rc == 0 ) - rc = rc1; + colStructs.push_back(colStruct); + cscColTypeList.push_back(colType); + oids[colStruct.dataOid] = colStruct.dataOid; + // oidsToFlush.push_back(colStruct.dataOid); + rc = fWEWrapper.updateColumnRecs(txnID, cscColTypeList, colStructs, colValuesList, ridList, + SYSCOLUMN_BASE); + } - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - return rc; + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; } uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tmp32; - uint8_t tmp8; - int txnID; - OID dataOid, dictOid, refColOID; - CalpontSystemCatalog::ColDataType dataType, refColDataType; - bool autoincrement; - int dataWidth, scale, precision, compressionType, refColWidth, refCompressionType; - string defaultValStr; - ColTuple defaultVal; - string timeZone; + int rc = 0; + uint32_t tmp32; + uint8_t tmp8; + int txnID; + OID dataOid, dictOid, refColOID; + CalpontSystemCatalog::ColDataType dataType, refColDataType; + bool autoincrement; + int dataWidth, scale, precision, compressionType, refColWidth, refCompressionType; + string defaultValStr; + ColTuple defaultVal; + string timeZone; - bs >> tmp32; - txnID = tmp32; - bs >> tmp32; - dataOid = tmp32; - bs >> tmp32; - dictOid = tmp32; - bs >> tmp8; - dataType = (CalpontSystemCatalog::ColDataType) tmp8; - bs >> tmp8; - autoincrement = (tmp8 != 0); - bs >> tmp32; - dataWidth = tmp32; - bs >> tmp32; - scale = tmp32; - bs >> tmp32; - precision = tmp32; - bs >> defaultValStr; - bs >> tmp8; - compressionType = tmp8; - bs >> tmp32; - refColOID = tmp32; - bs >> tmp8; - refColDataType = (CalpontSystemCatalog::ColDataType) tmp8; - bs >> tmp32; - refColWidth = tmp32; - bs >> tmp8; - refCompressionType = tmp8; - bs >> timeZone; - //Find the fill in value - bool isNULL = false; + bs >> tmp32; + txnID = tmp32; + bs >> tmp32; + dataOid = tmp32; + bs >> tmp32; + dictOid = tmp32; + bs >> tmp8; + dataType = (CalpontSystemCatalog::ColDataType)tmp8; + bs >> tmp8; + autoincrement = (tmp8 != 0); + bs >> tmp32; + dataWidth = tmp32; + bs >> tmp32; + scale = tmp32; + bs >> tmp32; + precision = tmp32; + bs >> defaultValStr; + bs >> tmp8; + compressionType = tmp8; + bs >> tmp32; + refColOID = tmp32; + bs >> tmp8; + refColDataType = (CalpontSystemCatalog::ColDataType)tmp8; + bs >> tmp32; + refColWidth = tmp32; + bs >> tmp8; + refCompressionType = tmp8; + bs >> timeZone; + // Find the fill in value + bool isNULL = false; - if (defaultValStr == "") - isNULL = true; + if (defaultValStr == "") + isNULL = true; - CalpontSystemCatalog::ColType colType; - colType.colDataType = static_cast(dataType); - colType.colWidth = dataWidth; - colType.scale = scale; - colType.precision = precision; - bool pushWarning = false; - defaultVal.data = colType.convertColumnData(defaultValStr, pushWarning, timeZone, isNULL, false, false); - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - std::map oids; - oids[dataOid] = dataOid; - oids[refColOID] = refColOID; - rc = fWEWrapper.fillColumn(txnID, dataOid, colType, defaultVal, refColOID, refColDataType, - refColWidth, refCompressionType, isNULL, compressionType, defaultValStr, dictOid, autoincrement); + CalpontSystemCatalog::ColType colType; + colType.colDataType = static_cast(dataType); + colType.colWidth = dataWidth; + colType.scale = scale; + colType.precision = precision; + bool pushWarning = false; + defaultVal.data = colType.convertColumnData(defaultValStr, pushWarning, timeZone, isNULL, false, false); + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + std::map oids; + oids[dataOid] = dataOid; + oids[refColOID] = refColOID; + rc = fWEWrapper.fillColumn(txnID, dataOid, colType, defaultVal, refColOID, refColDataType, refColWidth, + refCompressionType, isNULL, compressionType, defaultValStr, dictOid, + autoincrement); - if ( rc != 0 ) - { - WErrorCodes ec; - err = ec.errorString(rc); - } + if (rc != 0) + { + WErrorCodes ec; + err = ec.errorString(rc); + } - purgeFDCache(); - return rc; + purgeFDCache(); + return rc; } uint8_t WE_DDLCommandProc::writeTruncateLog(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tableOid, numOid, tmp32; - bs >> tableOid; - bs >> numOid; - std::vector oids; + int rc = 0; + uint32_t tableOid, numOid, tmp32; + bs >> tableOid; + bs >> numOid; + std::vector oids; - for (uint32_t i = 0; i < numOid; i++) - { - bs >> tmp32; - oids.push_back(tmp32); - } + for (uint32_t i = 0; i < numOid; i++) + { + bs >> tmp32; + oids.push_back(tmp32); + } - string prefix; - config::Config* config = config::Config::makeConfig(); - prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - err = "Need a valid DBRMRoot entry in Calpont configuation file"; - rc = 1; - return rc; - } - - uint64_t pos = prefix.find_last_of ("/") ; - std::string DDLLogFileName; - - if (pos != string::npos) - { - DDLLogFileName = prefix.substr(0, pos + 1); //Get the file path - } - else - { - err = "Cannot find the dbrm directory for the DDL log file"; - rc = 1; - return rc; - } - - std::ostringstream oss; - oss << tableOid; - DDLLogFileName += "DDL_TRUNCATETABLE_Log_" + oss.str(); - boost::scoped_ptr DDLLogFile(IDBDataFile::open( - IDBPolicy::getType(DDLLogFileName.c_str(), IDBPolicy::WRITEENG), - DDLLogFileName.c_str(), "w", 0)); - - if (!DDLLogFile) - { - err = "DDL truncate table log file cannot be created"; - rc = 1; - return rc; - } - - std::ostringstream buf; - - for (unsigned i = 0; i < oids.size(); i++) - buf << oids[i] << std::endl; - - std::string tmp(buf.str()); - DDLLogFile->write(tmp.c_str(), tmp.size()); - - // DDLLogFile is a scoped_ptr, will be closed after return. + string prefix; + config::Config* config = config::Config::makeConfig(); + prefix = config->getConfig("SystemConfig", "DBRMRoot"); + if (prefix.length() == 0) + { + err = "Need a valid DBRMRoot entry in Calpont configuation file"; + rc = 1; return rc; + } + + uint64_t pos = prefix.find_last_of("/"); + std::string DDLLogFileName; + + if (pos != string::npos) + { + DDLLogFileName = prefix.substr(0, pos + 1); // Get the file path + } + else + { + err = "Cannot find the dbrm directory for the DDL log file"; + rc = 1; + return rc; + } + + std::ostringstream oss; + oss << tableOid; + DDLLogFileName += "DDL_TRUNCATETABLE_Log_" + oss.str(); + boost::scoped_ptr DDLLogFile(IDBDataFile::open( + IDBPolicy::getType(DDLLogFileName.c_str(), IDBPolicy::WRITEENG), DDLLogFileName.c_str(), "w", 0)); + + if (!DDLLogFile) + { + err = "DDL truncate table log file cannot be created"; + rc = 1; + return rc; + } + + std::ostringstream buf; + + for (unsigned i = 0; i < oids.size(); i++) + buf << oids[i] << std::endl; + + std::string tmp(buf.str()); + DDLLogFile->write(tmp.c_str(), tmp.size()); + + // DDLLogFile is a scoped_ptr, will be closed after return. + + return rc; } uint8_t WE_DDLCommandProc::writeDropPartitionLog(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tableOid, numParts, numOid, tmp32; - bs >> tableOid; - std::set partitionNums; - bs >> numParts; - BRM::LogicalPartition lp; + int rc = 0; + uint32_t tableOid, numParts, numOid, tmp32; + bs >> tableOid; + std::set partitionNums; + bs >> numParts; + BRM::LogicalPartition lp; - for (uint32_t i = 0; i < numParts; i++) - { - lp.unserialize(bs); - partitionNums.insert(lp); - } + for (uint32_t i = 0; i < numParts; i++) + { + lp.unserialize(bs); + partitionNums.insert(lp); + } - bs >> numOid; - std::vector oids; + bs >> numOid; + std::vector oids; - for (uint32_t i = 0; i < numOid; i++) - { - bs >> tmp32; - oids.push_back(tmp32); - } + for (uint32_t i = 0; i < numOid; i++) + { + bs >> tmp32; + oids.push_back(tmp32); + } - string prefix; - config::Config* config = config::Config::makeConfig(); - prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - err = "Need a valid DBRMRoot entry in Calpont configuation file"; - rc = 1; - return rc; - } - - uint64_t pos = prefix.find_last_of ("/") ; - std::string DDLLogFileName; - - if (pos != string::npos) - { - DDLLogFileName = prefix.substr(0, pos + 1); //Get the file path - } - else - { - err = "Cannot find the dbrm directory for the DDL drop partitions log file"; - rc = 1; - return rc; - } - - std::ostringstream oss; - oss << tableOid; - DDLLogFileName += "DDL_DROPPARTITION_Log_" + oss.str(); - boost::scoped_ptr DDLLogFile(IDBDataFile::open( - IDBPolicy::getType(DDLLogFileName.c_str(), IDBPolicy::WRITEENG), - DDLLogFileName.c_str(), "w", 0)); - - if (!DDLLogFile) - { - err = "DDL drop partitions log file cannot be created"; - rc = 1; - return rc; - } - - std::ostringstream buf; - // @SN write partition numbers to the log file, separated by space - set::const_iterator it; - - for (it = partitionNums.begin(); it != partitionNums.end(); ++it) - buf << (*it) << endl; - - - // -1 indicates the end of partition list - BRM::LogicalPartition end(-1, -1, -1); - buf << end << endl; - - for (unsigned i = 0; i < oids.size(); i++) - buf << oids[i] << std::endl; - - std::string tmp(buf.str()); - DDLLogFile->write(tmp.c_str(), tmp.size()); + string prefix; + config::Config* config = config::Config::makeConfig(); + prefix = config->getConfig("SystemConfig", "DBRMRoot"); + if (prefix.length() == 0) + { + err = "Need a valid DBRMRoot entry in Calpont configuation file"; + rc = 1; return rc; + } + + uint64_t pos = prefix.find_last_of("/"); + std::string DDLLogFileName; + + if (pos != string::npos) + { + DDLLogFileName = prefix.substr(0, pos + 1); // Get the file path + } + else + { + err = "Cannot find the dbrm directory for the DDL drop partitions log file"; + rc = 1; + return rc; + } + + std::ostringstream oss; + oss << tableOid; + DDLLogFileName += "DDL_DROPPARTITION_Log_" + oss.str(); + boost::scoped_ptr DDLLogFile(IDBDataFile::open( + IDBPolicy::getType(DDLLogFileName.c_str(), IDBPolicy::WRITEENG), DDLLogFileName.c_str(), "w", 0)); + + if (!DDLLogFile) + { + err = "DDL drop partitions log file cannot be created"; + rc = 1; + return rc; + } + + std::ostringstream buf; + // @SN write partition numbers to the log file, separated by space + set::const_iterator it; + + for (it = partitionNums.begin(); it != partitionNums.end(); ++it) + buf << (*it) << endl; + + // -1 indicates the end of partition list + BRM::LogicalPartition end(-1, -1, -1); + buf << end << endl; + + for (unsigned i = 0; i < oids.size(); i++) + buf << oids[i] << std::endl; + + std::string tmp(buf.str()); + DDLLogFile->write(tmp.c_str(), tmp.size()); + + return rc; } uint8_t WE_DDLCommandProc::writeDropTableLog(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tableOid, numOid, tmp32; - bs >> tableOid; + int rc = 0; + uint32_t tableOid, numOid, tmp32; + bs >> tableOid; - bs >> numOid; - std::vector oids; + bs >> numOid; + std::vector oids; - for (uint32_t i = 0; i < numOid; i++) - { - bs >> tmp32; - oids.push_back(tmp32); - } + for (uint32_t i = 0; i < numOid; i++) + { + bs >> tmp32; + oids.push_back(tmp32); + } - string prefix; - config::Config* config = config::Config::makeConfig(); - prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - err = "Need a valid DBRMRoot entry in Calpont configuation file"; - rc = 1; - return rc; - } - - uint64_t pos = prefix.find_last_of ("/") ; - std::string DDLLogFileName; - - if (pos != string::npos) - { - DDLLogFileName = prefix.substr(0, pos + 1); //Get the file path - } - else - { - err = "Cannot find the dbrm directory for the DDL drop partitions log file"; - rc = 1; - return rc; - } - - std::ostringstream oss; - oss << tableOid; - DDLLogFileName += "DDL_DROPTABLE_Log_" + oss.str(); - boost::scoped_ptr DDLLogFile(IDBDataFile::open( - IDBPolicy::getType(DDLLogFileName.c_str(), IDBPolicy::WRITEENG), - DDLLogFileName.c_str(), "w", 0)); - - if (!DDLLogFile) - { - err = "DDL drop table log file cannot be created"; - rc = 1; - return rc; - } - - std::ostringstream buf; - - for (unsigned i = 0; i < oids.size(); i++) - buf << oids[i] << std::endl; - - std::string tmp(buf.str()); - DDLLogFile->write(tmp.c_str(), tmp.size()); + string prefix; + config::Config* config = config::Config::makeConfig(); + prefix = config->getConfig("SystemConfig", "DBRMRoot"); + if (prefix.length() == 0) + { + err = "Need a valid DBRMRoot entry in Calpont configuation file"; + rc = 1; return rc; + } + + uint64_t pos = prefix.find_last_of("/"); + std::string DDLLogFileName; + + if (pos != string::npos) + { + DDLLogFileName = prefix.substr(0, pos + 1); // Get the file path + } + else + { + err = "Cannot find the dbrm directory for the DDL drop partitions log file"; + rc = 1; + return rc; + } + + std::ostringstream oss; + oss << tableOid; + DDLLogFileName += "DDL_DROPTABLE_Log_" + oss.str(); + boost::scoped_ptr DDLLogFile(IDBDataFile::open( + IDBPolicy::getType(DDLLogFileName.c_str(), IDBPolicy::WRITEENG), DDLLogFileName.c_str(), "w", 0)); + + if (!DDLLogFile) + { + err = "DDL drop table log file cannot be created"; + rc = 1; + return rc; + } + + std::ostringstream buf; + + for (unsigned i = 0; i < oids.size(); i++) + buf << oids[i] << std::endl; + + std::string tmp(buf.str()); + DDLLogFile->write(tmp.c_str(), tmp.size()); + + return rc; } uint8_t WE_DDLCommandProc::deleteDDLLog(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tableOid, fileType; - bs >> fileType; - bs >> tableOid; - string prefix; - config::Config* config = config::Config::makeConfig(); - prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - err = "Need a valid DBRMRoot entry in Calpont configuation file"; - rc = 1; - return rc; - } - - uint64_t pos = prefix.find_last_of ("/") ; - std::string DDLLogFileName; - - if (pos != string::npos) - { - DDLLogFileName = prefix.substr(0, pos + 1); //Get the file path - } - else - { - err = "Cannot find the dbrm directory for the DDL drop partitions log file"; - rc = 1; - return rc; - } - - std::ostringstream oss; - oss << tableOid; - - switch (fileType) - { - case DROPTABLE_LOG: - { - DDLLogFileName += "DDL_DROPTABLE_Log_" + oss.str(); - break; - } - - case DROPPART_LOG: - { - DDLLogFileName += "DDL_DROPPARTITION_Log_" + oss.str(); - break; - } - - case TRUNCATE_LOG: - { - DDLLogFileName += "DDL_TRUNCATETABLE_Log_" + oss.str(); - break; - } - - default: - break; - } - - IDBPolicy::remove(DDLLogFileName.c_str()); + int rc = 0; + uint32_t tableOid, fileType; + bs >> fileType; + bs >> tableOid; + string prefix; + config::Config* config = config::Config::makeConfig(); + prefix = config->getConfig("SystemConfig", "DBRMRoot"); + if (prefix.length() == 0) + { + err = "Need a valid DBRMRoot entry in Calpont configuation file"; + rc = 1; return rc; + } + + uint64_t pos = prefix.find_last_of("/"); + std::string DDLLogFileName; + + if (pos != string::npos) + { + DDLLogFileName = prefix.substr(0, pos + 1); // Get the file path + } + else + { + err = "Cannot find the dbrm directory for the DDL drop partitions log file"; + rc = 1; + return rc; + } + + std::ostringstream oss; + oss << tableOid; + + switch (fileType) + { + case DROPTABLE_LOG: + { + DDLLogFileName += "DDL_DROPTABLE_Log_" + oss.str(); + break; + } + + case DROPPART_LOG: + { + DDLLogFileName += "DDL_DROPPARTITION_Log_" + oss.str(); + break; + } + + case TRUNCATE_LOG: + { + DDLLogFileName += "DDL_TRUNCATETABLE_Log_" + oss.str(); + break; + } + + default: break; + } + + IDBPolicy::remove(DDLLogFileName.c_str()); + + return rc; } uint8_t WE_DDLCommandProc::fetchDDLLog(ByteStream& bs, std::string& err) { - int rc = 0; + int rc = 0; - //Find the ddl log files under DBRMRoot directory - string prefix, ddlLogDir; - config::Config* config = config::Config::makeConfig(); - prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - rc = 1; - err = "Need a valid DBRMRoot entry in Calpont configuation file"; - return rc; - } - - uint64_t pos = prefix.find_last_of ("/"); - - if ( pos != string::npos ) - { - ddlLogDir = prefix.substr(0, pos + 1); //Get the file path - } - else - { - rc = 1; - err = "Cannot find the dbrm directory for the DDL log file"; - return rc; - } - - boost::filesystem::path filePath; - filePath = fs::system_complete( fs::path( ddlLogDir ) ); - - if ( !fs::exists( filePath ) ) - { - rc = 1; - err = "\nDDL log file path is Not found: "; - return rc; - } - - std::vector fileNames; - - if ( fs::is_directory( filePath ) ) - { - fs::directory_iterator end_iter; - - for ( fs::directory_iterator dir_itr( filePath ); dir_itr != end_iter; ++dir_itr ) - { - try - { - if ( !fs::is_directory( *dir_itr ) ) - { -#if BOOST_VERSION >= 105200 - fileNames.push_back ( dir_itr->path().generic_string() ); -#else - fileNames.push_back ( dir_itr->string() ); -#endif - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - } - - CalpontSystemCatalog::OID fileoid; - string tableName; - bs.restart(); - - for ( unsigned i = 0; i < fileNames.size(); i++ ) - { - pos = fileNames[i].find ("DDL_DROPTABLE_Log_") ; - - if ( pos != string::npos ) - { - //Read the file to get oids - //cout << "Found file " << fileNames[i] << endl; - boost::scoped_ptr ddlLogFile(IDBDataFile::open( - IDBPolicy::getType(fileNames[i].c_str(), IDBPolicy::WRITEENG), - fileNames[i].c_str(), "r", 0)); - - if ( !ddlLogFile ) - continue; - - //find the table oid - pos = fileNames[i].find_last_of( "_"); - string tableOidStr = fileNames[i].substr(pos + 1, fileNames[i].length() - pos - 1); - char* ep = NULL; - uint32_t tableOid = strtoll(tableOidStr.c_str(), &ep, 10); - bs << tableOid; - bs << (uint32_t)DROPTABLE_LOG; - std::vector oidList; - - ssize_t fileSize = ddlLogFile->size(); - boost::scoped_array buf(new char[fileSize]); - - if (ddlLogFile->read(buf.get(), fileSize) != fileSize) - return (uint8_t) ERR_FILE_READ; - - std::istringstream strbuf(string(buf.get(), fileSize)); - - while (strbuf >> fileoid) - oidList.push_back(fileoid); - - bs << (uint32_t)oidList.size(); - - for ( unsigned j = 0; j < oidList.size(); j++ ) - { - bs << (uint32_t)oidList[j]; - } - - bs << (uint32_t) 0; - } - else //Find drop partition log file - { - pos = fileNames[i].find ("DDL_DROPPARTITION_Log_") ; - - if ( pos != string::npos ) - { - boost::scoped_ptr ddlLogFile(IDBDataFile::open( - IDBPolicy::getType(fileNames[i].c_str(), IDBPolicy::WRITEENG), - fileNames[i].c_str(), "r", 0)); - BRM::LogicalPartition partition; - vector partitionNums; - //find the table oid - pos = fileNames[i].find_last_of( "_"); - string tableOidStr = fileNames[i].substr(pos + 1, fileNames[i].length() - pos - 1); - char* ep = NULL; - uint32_t tableOid = strtoll(tableOidStr.c_str(), &ep, 10); - bs << tableOid; - bs << (uint32_t)DROPPART_LOG; - - ssize_t fileSize = ddlLogFile->size(); - boost::scoped_array buf(new char[fileSize]); - - if (ddlLogFile->read(buf.get(), fileSize) != fileSize) - return (uint8_t) ERR_FILE_READ; - - std::istringstream strbuf(string(buf.get(), fileSize)); - - while (strbuf >> partition) - { - if (partition.dbroot == (uint16_t) - 1) - break; - - partitionNums.push_back(partition); - } - - std::vector oidPartList; - - while (strbuf >> fileoid) - oidPartList.push_back( fileoid ); - - bs << (uint32_t)oidPartList.size(); - - for ( unsigned j = 0; j < oidPartList.size(); j++ ) - { - bs << (uint32_t)oidPartList[j]; - } - - bs << (uint32_t) partitionNums.size(); - - for ( unsigned j = 0; j < partitionNums.size(); j++ ) - { - partitionNums[j].serialize(bs); - } - } - else //find truncate table log file - { - pos = fileNames[i].find ("DDL_TRUNCATETABLE_Log_") ; - - if ( pos != string::npos ) - { - boost::scoped_ptr ddlLogFile(IDBDataFile::open( - IDBPolicy::getType(fileNames[i].c_str(), IDBPolicy::WRITEENG), - fileNames[i].c_str(), "r", 0)); - - if ( !ddlLogFile ) - { - continue; - } - - //find the table oid - pos = fileNames[i].find_last_of( "_"); - string tableOidStr = fileNames[i].substr(pos + 1, fileNames[i].length() - pos - 1); - char* ep = NULL; - uint32_t tableOid = strtoll(tableOidStr.c_str(), &ep, 10); - bs << tableOid; - bs << (uint32_t)TRUNCATE_LOG; - std::vector oidList; - - ssize_t fileSize = ddlLogFile->size(); - boost::scoped_array buf(new char[fileSize]); - - if (ddlLogFile->read(buf.get(), fileSize) != fileSize) - return (uint8_t) ERR_FILE_READ; - - std::istringstream strbuf(string(buf.get(), fileSize)); - - while (strbuf >> fileoid) - oidList.push_back(fileoid); - - bs << (uint32_t)oidList.size(); - - for ( unsigned j = 0; j < oidList.size(); j++ ) - { - bs << (uint32_t)oidList[j]; - } - - bs << (uint32_t) 0; - } - } - } - } + // Find the ddl log files under DBRMRoot directory + string prefix, ddlLogDir; + config::Config* config = config::Config::makeConfig(); + prefix = config->getConfig("SystemConfig", "DBRMRoot"); + if (prefix.length() == 0) + { + rc = 1; + err = "Need a valid DBRMRoot entry in Calpont configuation file"; return rc; + } + + uint64_t pos = prefix.find_last_of("/"); + + if (pos != string::npos) + { + ddlLogDir = prefix.substr(0, pos + 1); // Get the file path + } + else + { + rc = 1; + err = "Cannot find the dbrm directory for the DDL log file"; + return rc; + } + + boost::filesystem::path filePath; + filePath = fs::system_complete(fs::path(ddlLogDir)); + + if (!fs::exists(filePath)) + { + rc = 1; + err = "\nDDL log file path is Not found: "; + return rc; + } + + std::vector fileNames; + + if (fs::is_directory(filePath)) + { + fs::directory_iterator end_iter; + + for (fs::directory_iterator dir_itr(filePath); dir_itr != end_iter; ++dir_itr) + { + try + { + if (!fs::is_directory(*dir_itr)) + { +#if BOOST_VERSION >= 105200 + fileNames.push_back(dir_itr->path().generic_string()); +#else + fileNames.push_back(dir_itr->string()); +#endif + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + } + + CalpontSystemCatalog::OID fileoid; + string tableName; + bs.restart(); + + for (unsigned i = 0; i < fileNames.size(); i++) + { + pos = fileNames[i].find("DDL_DROPTABLE_Log_"); + + if (pos != string::npos) + { + // Read the file to get oids + // cout << "Found file " << fileNames[i] << endl; + boost::scoped_ptr ddlLogFile(IDBDataFile::open( + IDBPolicy::getType(fileNames[i].c_str(), IDBPolicy::WRITEENG), fileNames[i].c_str(), "r", 0)); + + if (!ddlLogFile) + continue; + + // find the table oid + pos = fileNames[i].find_last_of("_"); + string tableOidStr = fileNames[i].substr(pos + 1, fileNames[i].length() - pos - 1); + char* ep = NULL; + uint32_t tableOid = strtoll(tableOidStr.c_str(), &ep, 10); + bs << tableOid; + bs << (uint32_t)DROPTABLE_LOG; + std::vector oidList; + + ssize_t fileSize = ddlLogFile->size(); + boost::scoped_array buf(new char[fileSize]); + + if (ddlLogFile->read(buf.get(), fileSize) != fileSize) + return (uint8_t)ERR_FILE_READ; + + std::istringstream strbuf(string(buf.get(), fileSize)); + + while (strbuf >> fileoid) + oidList.push_back(fileoid); + + bs << (uint32_t)oidList.size(); + + for (unsigned j = 0; j < oidList.size(); j++) + { + bs << (uint32_t)oidList[j]; + } + + bs << (uint32_t)0; + } + else // Find drop partition log file + { + pos = fileNames[i].find("DDL_DROPPARTITION_Log_"); + + if (pos != string::npos) + { + boost::scoped_ptr ddlLogFile(IDBDataFile::open( + IDBPolicy::getType(fileNames[i].c_str(), IDBPolicy::WRITEENG), fileNames[i].c_str(), "r", 0)); + BRM::LogicalPartition partition; + vector partitionNums; + // find the table oid + pos = fileNames[i].find_last_of("_"); + string tableOidStr = fileNames[i].substr(pos + 1, fileNames[i].length() - pos - 1); + char* ep = NULL; + uint32_t tableOid = strtoll(tableOidStr.c_str(), &ep, 10); + bs << tableOid; + bs << (uint32_t)DROPPART_LOG; + + ssize_t fileSize = ddlLogFile->size(); + boost::scoped_array buf(new char[fileSize]); + + if (ddlLogFile->read(buf.get(), fileSize) != fileSize) + return (uint8_t)ERR_FILE_READ; + + std::istringstream strbuf(string(buf.get(), fileSize)); + + while (strbuf >> partition) + { + if (partition.dbroot == (uint16_t)-1) + break; + + partitionNums.push_back(partition); + } + + std::vector oidPartList; + + while (strbuf >> fileoid) + oidPartList.push_back(fileoid); + + bs << (uint32_t)oidPartList.size(); + + for (unsigned j = 0; j < oidPartList.size(); j++) + { + bs << (uint32_t)oidPartList[j]; + } + + bs << (uint32_t)partitionNums.size(); + + for (unsigned j = 0; j < partitionNums.size(); j++) + { + partitionNums[j].serialize(bs); + } + } + else // find truncate table log file + { + pos = fileNames[i].find("DDL_TRUNCATETABLE_Log_"); + + if (pos != string::npos) + { + boost::scoped_ptr ddlLogFile(IDBDataFile::open( + IDBPolicy::getType(fileNames[i].c_str(), IDBPolicy::WRITEENG), fileNames[i].c_str(), "r", 0)); + + if (!ddlLogFile) + { + continue; + } + + // find the table oid + pos = fileNames[i].find_last_of("_"); + string tableOidStr = fileNames[i].substr(pos + 1, fileNames[i].length() - pos - 1); + char* ep = NULL; + uint32_t tableOid = strtoll(tableOidStr.c_str(), &ep, 10); + bs << tableOid; + bs << (uint32_t)TRUNCATE_LOG; + std::vector oidList; + + ssize_t fileSize = ddlLogFile->size(); + boost::scoped_array buf(new char[fileSize]); + + if (ddlLogFile->read(buf.get(), fileSize) != fileSize) + return (uint8_t)ERR_FILE_READ; + + std::istringstream strbuf(string(buf.get(), fileSize)); + + while (strbuf >> fileoid) + oidList.push_back(fileoid); + + bs << (uint32_t)oidList.size(); + + for (unsigned j = 0; j < oidList.size(); j++) + { + bs << (uint32_t)oidList[j]; + } + + bs << (uint32_t)0; + } + } + } + } + + return rc; } uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs, std::string& err) { - //Will update five columns: columnname, defaultvalue, nullable, autoincrement, nextvalue. - int rc = 0; - uint32_t tmp32; - string schema, tableName, colName, defaultvalue; - int txnID; - uint32_t sessionID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tableName; - bs >> colName; - bs >> defaultvalue; + // Will update five columns: columnname, defaultvalue, nullable, autoincrement, nextvalue. + int rc = 0; + uint32_t tmp32; + string schema, tableName, colName, defaultvalue; + int txnID; + uint32_t sessionID; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tableName; + bs >> colName; + bs >> defaultvalue; - boost::shared_ptr systemCatalogPtr; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + boost::shared_ptr systemCatalogPtr; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::TableName atableName; - CalpontSystemCatalog::TableColName tableColName; - tableColName.schema = schema; - tableColName.table = tableName; - tableColName.column = colName; - CalpontSystemCatalog::ROPair ropair; + CalpontSystemCatalog::TableName atableName; + CalpontSystemCatalog::TableColName tableColName; + tableColName.schema = schema; + tableColName.table = tableName; + tableColName.column = colName; + CalpontSystemCatalog::ROPair ropair; - try + try + { + ropair = systemCatalogPtr->columnRID(tableColName); + + if (ropair.objnum < 0) { - ropair = systemCatalogPtr->columnRID(tableColName); - - if (ropair.objnum < 0) - { - ostringstream oss; - oss << "No such column: " << tableColName; - throw std::runtime_error(oss.str().c_str()); - } + ostringstream oss; + oss << "No such column: " << tableColName; + throw std::runtime_error(oss.str().c_str()); } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - catch (...) - { - err = "renameColumn:Unknown exception caught"; - rc = 1; - return rc; - } - - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; - - std::vector ridList; - ridList.push_back(ropair.rid); - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList1; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; - std::map oids; - //std::vector oidsToFlush; - - WriteEngine::ColTuple colTuple; - - //Build colStructs for SYSCOLUMN - atableName.schema = CALPONT_SCHEMA; - atableName.table = SYSCOLUMN_TABLE; - DDLColumn column; - findColumnData (sessionID, atableName, DEFAULTVAL_COL, column); //DEFAULTVAL_COL column - - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.setTransId(txnID); - fWEWrapper.startTransaction(txnID); - //Build DEFAULTVAL_COL structure - WriteEngine::ColTupleList aColList; - colStruct.dataOid = column.oid; - colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; - colStruct.tokenFlag = false; - - if ( (column.colType.colDataType == CalpontSystemCatalog::CHAR - && column.colType.colWidth > 8) - || (column.colType.colDataType == CalpontSystemCatalog::VARCHAR - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::BLOB - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::TEXT - && column.colType.colWidth > 7) - || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL - && column.colType.precision > 18) - || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL - && column.colType.precision > 18) )//token - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = column.colType.colWidth; - } - - colStruct.colDataType = column.colType.colDataType; - - if (colStruct.tokenFlag) - { - WriteEngine::DctnryStruct dictStruct; - dictStruct.dctnryOid = column.colType.ddn.dictOID; - dictStruct.columnOid = column.colType.columnOID; - - if (defaultvalue.length() <= 0) //null token - { - WriteEngine::Token nullToken; - colTuple.data = nullToken; - } - else - { - WriteEngine::DctnryTuple dictTuple; - dictTuple.sigValue = (unsigned char*)defaultvalue.c_str(); - dictTuple.sigSize = defaultvalue.length(); - dictTuple.isNull = false; - int error = NO_ERROR; - - if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file - { - WErrorCodes ec; - throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); - } - - WriteEngine::Token aToken = dictTuple.token; - colTuple.data = aToken; - } - } - - colStruct.colDataType = column.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - dctnryStructList.push_back(dctnryStruct); - aColList.push_back(colTuple); - colValuesList.push_back(aColList); - WriteEngine::DctnryTuple dctnryTuple; - - if (defaultvalue.length() > 0) - { - dctnryTuple.sigValue = (unsigned char*)defaultvalue.c_str(); - dctnryTuple.sigSize = defaultvalue.length(); - dctnryTuple.isNull = false; - } - else - { - dctnryTuple.isNull = true; - } - - dctColList = dctnryTuple; - dctRowList.clear(); - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - std::vector colExtentsStruct; - std::vector dctnryExtentsStruct; - std::vector colExtentsColType; - std::vector ridLists; - ridLists.push_back(ridList); - - //In this case, there's only 1 row, so only one one extent, but keep it generic... - std::vector extentsinfo; - extentInfo aExtentinfo; - - convertRidToColumn(ropair.rid, dbRoot, partition, segment, 1021); - - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - extentsinfo.push_back(aExtentinfo); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - - if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE)) - { - err = "WE: Update failed on: " + atableName.table; - rc = 1; - } - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (rc == 0 ) - rc = rc1; - - //flush syscat cahche - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + catch (...) + { + err = "renameColumn:Unknown exception caught"; + rc = 1; + return rc; + } + + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; + + std::vector ridList; + ridList.push_back(ropair.rid); + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList1; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; + std::map oids; + // std::vector oidsToFlush; + + WriteEngine::ColTuple colTuple; + + // Build colStructs for SYSCOLUMN + atableName.schema = CALPONT_SCHEMA; + atableName.table = SYSCOLUMN_TABLE; + DDLColumn column; + findColumnData(sessionID, atableName, DEFAULTVAL_COL, column); // DEFAULTVAL_COL column + + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.setTransId(txnID); + fWEWrapper.startTransaction(txnID); + // Build DEFAULTVAL_COL structure + WriteEngine::ColTupleList aColList; + colStruct.dataOid = column.oid; + colStruct.colWidth = column.colType.colWidth > 8 ? 8 : column.colType.colWidth; + colStruct.tokenFlag = false; + + if ((column.colType.colDataType == CalpontSystemCatalog::CHAR && column.colType.colWidth > 8) || + (column.colType.colDataType == CalpontSystemCatalog::VARCHAR && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::VARBINARY && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::BLOB && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::TEXT && column.colType.colWidth > 7) || + (column.colType.colDataType == CalpontSystemCatalog::DECIMAL && column.colType.precision > 18) || + (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL && + column.colType.precision > 18)) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = column.colType.colWidth; + } + + colStruct.colDataType = column.colType.colDataType; + + if (colStruct.tokenFlag) + { + WriteEngine::DctnryStruct dictStruct; + dictStruct.dctnryOid = column.colType.ddn.dictOID; + dictStruct.columnOid = column.colType.columnOID; + + if (defaultvalue.length() <= 0) // null token + { + WriteEngine::Token nullToken; + colTuple.data = nullToken; + } + else + { + WriteEngine::DctnryTuple dictTuple; + dictTuple.sigValue = (unsigned char*)defaultvalue.c_str(); + dictTuple.sigSize = defaultvalue.length(); + dictTuple.isNull = false; + int error = NO_ERROR; + + if (NO_ERROR != + (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file + { + WErrorCodes ec; + throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); + } + + WriteEngine::Token aToken = dictTuple.token; + colTuple.data = aToken; + } + } + + colStruct.colDataType = column.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + dctnryStructList.push_back(dctnryStruct); + aColList.push_back(colTuple); + colValuesList.push_back(aColList); + WriteEngine::DctnryTuple dctnryTuple; + + if (defaultvalue.length() > 0) + { + dctnryTuple.sigValue = (unsigned char*)defaultvalue.c_str(); + dctnryTuple.sigSize = defaultvalue.length(); + dctnryTuple.isNull = false; + } + else + { + dctnryTuple.isNull = true; + } + + dctColList = dctnryTuple; + dctRowList.clear(); + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + std::vector colExtentsStruct; + std::vector dctnryExtentsStruct; + std::vector colExtentsColType; + std::vector ridLists; + ridLists.push_back(ridList); + + // In this case, there's only 1 row, so only one one extent, but keep it generic... + std::vector extentsinfo; + extentInfo aExtentinfo; + + convertRidToColumn(ropair.rid, dbRoot, partition, segment, 1021); + + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + extentsinfo.push_back(aExtentinfo); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } + + // call the write engine to update the row + + if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, + colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, + SYSCOLUMN_BASE)) + { + err = "WE: Update failed on: " + atableName.table; + rc = 1; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + // flush syscat cahche + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + + return rc; } uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& bs, std::string& err) { - //Will update five columns: columnname, defaultvalue, nullable, autoincrement, nextvalue. - int rc = 0; - uint64_t nextVal; - uint32_t tmp32, nullable; - string schema, tableName, colOldname, autoinc, colNewName, defaultvalue; - int txnID; - uint32_t sessionID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - bs >> schema; - bs >> tableName; - bs >> colOldname; - bs >> colNewName; - bs >> autoinc; - bs >> nextVal; - bs >> nullable; - bs >> defaultvalue; + // Will update five columns: columnname, defaultvalue, nullable, autoincrement, nextvalue. + int rc = 0; + uint64_t nextVal; + uint32_t tmp32, nullable; + string schema, tableName, colOldname, autoinc, colNewName, defaultvalue; + int txnID; + uint32_t sessionID; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + bs >> schema; + bs >> tableName; + bs >> colOldname; + bs >> colNewName; + bs >> autoinc; + bs >> nextVal; + bs >> nullable; + bs >> defaultvalue; - boost::shared_ptr systemCatalogPtr; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + boost::shared_ptr systemCatalogPtr; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::TableName atableName; - CalpontSystemCatalog::TableColName tableColName; - tableColName.schema = schema; - tableColName.table = tableName; - tableColName.column = colOldname; - CalpontSystemCatalog::ROPair ropair; + CalpontSystemCatalog::TableName atableName; + CalpontSystemCatalog::TableColName tableColName; + tableColName.schema = schema; + tableColName.table = tableName; + tableColName.column = colOldname; + CalpontSystemCatalog::ROPair ropair; - try + try + { + ropair = systemCatalogPtr->columnRID(tableColName); + + if (ropair.objnum < 0) { - ropair = systemCatalogPtr->columnRID(tableColName); - - if (ropair.objnum < 0) - { - ostringstream oss; - oss << "No such column: " << tableColName; - throw std::runtime_error(oss.str().c_str()); - } + ostringstream oss; + oss << "No such column: " << tableColName; + throw std::runtime_error(oss.str().c_str()); } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - catch (...) - { - err = "renameColumn:Unknown exception caught"; - rc = 1; - return rc; - } - - uint16_t dbRoot = 0; - uint16_t segment; - uint32_t partition; - - std::vector ridList; - ridList.push_back(ropair.rid); - WriteEngine::ColValueList colValuesList; - WriteEngine::ColTupleList aColList1; - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - std::vector colOldValuesList; - std::map oids; - //std::vector oidsToFlush; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::DctColTupleList dctRowList; - WriteEngine::DctnryTuple dctColList; - boost::any datavalue; - datavalue = colNewName; - - WriteEngine::ColTuple colTuple; - - //Build colStructs for SYSCOLUMN - atableName.schema = CALPONT_SCHEMA; - atableName.table = SYSCOLUMN_TABLE; - DDLColumn column1, column2, column3, column4, column5; - findColumnData (sessionID, atableName, COLNAME_COL, column1); //COLNAME_COL column - findColumnData (sessionID, atableName, AUTOINC_COL, column2); //AUTOINC_COL column - findColumnData (sessionID, atableName, NEXTVALUE_COL, column3); //NEXTVALUE_COL column - findColumnData (sessionID, atableName, NULLABLE_COL, column4); //NULLABLE_COL column - findColumnData (sessionID, atableName, DEFAULTVAL_COL, column5); //DEFAULTVAL_COL column - - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - - fWEWrapper.setTransId(txnID); - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(txnID); - //Build COLNAME_COL structure - colStruct.dataOid = column1.oid; - colStruct.colWidth = column1.colType.colWidth > 8 ? 8 : column1.colType.colWidth; - colStruct.tokenFlag = false; - - if ( (column1.colType.colDataType == CalpontSystemCatalog::CHAR - && column1.colType.colWidth > 8) - || (column1.colType.colDataType == CalpontSystemCatalog::VARCHAR - && column1.colType.colWidth > 7) - || (column1.colType.colDataType == CalpontSystemCatalog::VARBINARY - && column1.colType.colWidth > 7) - || (column1.colType.colDataType == CalpontSystemCatalog::BLOB - && column1.colType.colWidth > 7) - || (column1.colType.colDataType == CalpontSystemCatalog::TEXT - && column1.colType.colWidth > 7) - || (column1.colType.colDataType == CalpontSystemCatalog::DECIMAL - && column1.colType.precision > 18) - || (column1.colType.colDataType == CalpontSystemCatalog::UDECIMAL - && column1.colType.precision > 18) )//token - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = column1.colType.colWidth; - } - - colStruct.colDataType = column1.colType.colDataType; - - if (colStruct.tokenFlag) - { - WriteEngine::DctnryStruct dictStruct; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - dictStruct.fCompressionType = 2; - } - - dictStruct.dctnryOid = column1.colType.ddn.dictOID; - dictStruct.columnOid = column1.colType.columnOID; - WriteEngine::DctnryTuple dictTuple; - dictTuple.sigValue = (unsigned char*)colNewName.c_str(); - dictTuple.sigSize = colNewName.length(); - dictTuple.isNull = false; - int error = NO_ERROR; - - if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file - { - WErrorCodes ec; - err = ec.errorString(error); - rc = error; - return rc; - } - - WriteEngine::Token aToken = dictTuple.token; - colTuple.data = aToken; - } - else - colTuple.data = datavalue; - - colStruct.colDataType = column1.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column1.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column1.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - dctnryStructList.push_back(dctnryStruct); - aColList1.push_back(colTuple); - colValuesList.push_back(aColList1); - WriteEngine::DctnryTuple dctnryTuple; - boost::to_lower(colNewName); - dctnryTuple.sigValue = (unsigned char*)colNewName.c_str(); - dctnryTuple.sigSize = colNewName.length(); - dctnryTuple.isNull = false; - dctColList = dctnryTuple; - dctRowList.clear(); - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //Build AUTOINC_COL structure - WriteEngine::ColTupleList aColList2; - colStruct.dataOid = column2.oid; - colStruct.colWidth = column2.colType.colWidth > 8 ? 8 : column2.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.colDataType = column2.colType.colDataType; - - colTuple.data = autoinc; - - colStruct.colDataType = column2.colType.colDataType; - - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column2.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - dctnryStructList.push_back(dctnryStruct); - aColList2.push_back(colTuple); - colValuesList.push_back(aColList2); - dctnryTuple.isNull = true; - dctColList = dctnryTuple; - dctRowList.clear(); - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //Build NEXTVALUE_COL structure - WriteEngine::ColTupleList aColList3; - colStruct.dataOid = column3.oid; - colStruct.colWidth = column3.colType.colWidth > 8 ? 8 : column3.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.colDataType = column3.colType.colDataType; - - colTuple.data = nextVal; - - colStruct.colDataType = column3.colType.colDataType; - - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column3.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - dctnryStructList.push_back(dctnryStruct); - aColList3.push_back(colTuple); - colValuesList.push_back(aColList3); - - dctnryTuple.isNull = true; - dctColList = dctnryTuple; - dctRowList.clear(); - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //Build NULLABLE_COL structure - WriteEngine::ColTupleList aColList4; - colStruct.dataOid = column4.oid; - colStruct.colWidth = column4.colType.colWidth > 8 ? 8 : column4.colType.colWidth; - colStruct.tokenFlag = false; - colStruct.colDataType = column4.colType.colDataType; - - colTuple.data = nullable; - - colStruct.colDataType = column4.colType.colDataType; - - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - - colStructs.push_back(colStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column4.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - dctnryStructList.push_back(dctnryStruct); - aColList4.push_back(colTuple); - colValuesList.push_back(aColList4); - dctnryTuple.isNull = true; - dctColList = dctnryTuple; - dctRowList.clear(); - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - - //Build DEFAULTVAL_COL structure - WriteEngine::ColTupleList aColList5; - colStruct.dataOid = column5.oid; - colStruct.colWidth = column5.colType.colWidth > 8 ? 8 : column5.colType.colWidth; - colStruct.tokenFlag = false; - - if ( (column5.colType.colDataType == CalpontSystemCatalog::CHAR - && column5.colType.colWidth > 8) - || (column5.colType.colDataType == CalpontSystemCatalog::VARCHAR - && column5.colType.colWidth > 7) - || (column5.colType.colDataType == CalpontSystemCatalog::VARBINARY - && column5.colType.colWidth > 7) - || (column5.colType.colDataType == CalpontSystemCatalog::BLOB - && column5.colType.colWidth > 7) - || (column5.colType.colDataType == CalpontSystemCatalog::TEXT - && column5.colType.colWidth > 7) - || (column5.colType.colDataType == CalpontSystemCatalog::DECIMAL - && column5.colType.precision > 18) - || (column5.colType.colDataType == CalpontSystemCatalog::UDECIMAL - && column5.colType.precision > 18) )//token - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = column5.colType.colWidth; - } - - colStruct.colDataType = column5.colType.colDataType; - - if (colStruct.tokenFlag) - { - WriteEngine::DctnryStruct dictStruct; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dictStruct.fCompressionType = 2; - } - - dictStruct.dctnryOid = column5.colType.ddn.dictOID; - dictStruct.columnOid = column5.colType.columnOID; - - if (defaultvalue.length() <= 0) //null token - { - WriteEngine::Token nullToken; - colTuple.data = nullToken; - } - else - { - WriteEngine::DctnryTuple dictTuple; - dictTuple.sigValue = (unsigned char*)defaultvalue.c_str(); - dictTuple.sigSize = defaultvalue.length(); - dictTuple.isNull = false; - int error = NO_ERROR; - - if (NO_ERROR != (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file - { - WErrorCodes ec; - throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); - } - - WriteEngine::Token aToken = dictTuple.token; - colTuple.data = aToken; - } - } - - fWEWrapper.flushDataFiles(rc, txnID, oids); - - colStruct.colDataType = column5.colType.colDataType; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - colStruct.fCompressionType = 2; - dctnryStruct.fCompressionType = 2; - } - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = column5.colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - oids[colStruct.dataOid] = colStruct.dataOid; - cscColTypeList.push_back(column5.colType); - - //oidsToFlush.push_back(colStruct.dataOid); - if (dctnryStruct.dctnryOid > 0) - { - oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; - //oidsToFlush.push_back(dctnryStruct.dctnryOid); - } - - aColList5.push_back(colTuple); - colValuesList.push_back(aColList5); - - if (defaultvalue.length() > 0) - { - dctnryTuple.sigValue = (unsigned char*)defaultvalue.c_str(); - dctnryTuple.sigSize = defaultvalue.length(); - dctnryTuple.isNull = false; - } - else - { - dctnryTuple.isNull = true; - } - - dctColList = dctnryTuple; - dctRowList.clear(); - dctRowList.push_back(dctColList); - dctnryValueList.push_back(dctRowList); - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector dctnryExtentsStruct; - std::vector ridLists; - ridLists.push_back(ridList); - - //In this case, there's only 1 row, so only one one extent, but keep it generic... - std::vector extentsinfo; - extentInfo aExtentinfo; - - convertRidToColumn(ropair.rid, dbRoot, partition, segment, 1021); - - aExtentinfo.dbRoot = dbRoot; - aExtentinfo.partition = partition; - aExtentinfo.segment = segment; - - extentsinfo.push_back(aExtentinfo); - - //build colExtentsStruct - for (unsigned i = 0; i < extentsinfo.size(); i++) - { - for (unsigned j = 0; j < colStructs.size(); j++) - { - colStructs[j].fColPartition = extentsinfo[i].partition; - colStructs[j].fColSegment = extentsinfo[i].segment; - colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; - dctnryStructList[j].fColPartition = extentsinfo[i].partition; - dctnryStructList[j].fColSegment = extentsinfo[i].segment; - dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; - } - - colExtentsStruct.push_back(colStructs); - dctnryExtentsStruct.push_back(dctnryStructList); - colExtentsColType.push_back(cscColTypeList); - } - - // call the write engine to update the row - if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, - ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE)) - { - err = "WE: Update failed on: " + atableName.table; - rc = 1; - } - - int rc1 = 0; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); - - if ((rc == 0) && ( rc1 == 0)) - { - - rc1 = fWEWrapper.confirmTransaction(txnID); - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(txnID, true); - else - fWEWrapper.endTransaction(txnID, false); - } - else - { - fWEWrapper.endTransaction(txnID, false); - } - } - - if (rc == 0 ) - rc = rc1; - - //flush syscat cahche - systemCatalogPtr->flushCache(); - purgeFDCache(); - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + catch (...) + { + err = "renameColumn:Unknown exception caught"; + rc = 1; + return rc; + } + + uint16_t dbRoot = 0; + uint16_t segment; + uint32_t partition; + + std::vector ridList; + ridList.push_back(ropair.rid); + WriteEngine::ColValueList colValuesList; + WriteEngine::ColTupleList aColList1; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + std::vector colOldValuesList; + std::map oids; + // std::vector oidsToFlush; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::DctColTupleList dctRowList; + WriteEngine::DctnryTuple dctColList; + boost::any datavalue; + datavalue = colNewName; + + WriteEngine::ColTuple colTuple; + + // Build colStructs for SYSCOLUMN + atableName.schema = CALPONT_SCHEMA; + atableName.table = SYSCOLUMN_TABLE; + DDLColumn column1, column2, column3, column4, column5; + findColumnData(sessionID, atableName, COLNAME_COL, column1); // COLNAME_COL column + findColumnData(sessionID, atableName, AUTOINC_COL, column2); // AUTOINC_COL column + findColumnData(sessionID, atableName, NEXTVALUE_COL, column3); // NEXTVALUE_COL column + findColumnData(sessionID, atableName, NULLABLE_COL, column4); // NULLABLE_COL column + findColumnData(sessionID, atableName, DEFAULTVAL_COL, column5); // DEFAULTVAL_COL column + + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + fWEWrapper.setTransId(txnID); + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(txnID); + // Build COLNAME_COL structure + colStruct.dataOid = column1.oid; + colStruct.colWidth = column1.colType.colWidth > 8 ? 8 : column1.colType.colWidth; + colStruct.tokenFlag = false; + + if ((column1.colType.colDataType == CalpontSystemCatalog::CHAR && column1.colType.colWidth > 8) || + (column1.colType.colDataType == CalpontSystemCatalog::VARCHAR && column1.colType.colWidth > 7) || + (column1.colType.colDataType == CalpontSystemCatalog::VARBINARY && column1.colType.colWidth > 7) || + (column1.colType.colDataType == CalpontSystemCatalog::BLOB && column1.colType.colWidth > 7) || + (column1.colType.colDataType == CalpontSystemCatalog::TEXT && column1.colType.colWidth > 7) || + (column1.colType.colDataType == CalpontSystemCatalog::DECIMAL && column1.colType.precision > 18) || + (column1.colType.colDataType == CalpontSystemCatalog::UDECIMAL && + column1.colType.precision > 18)) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = column1.colType.colWidth; + } + + colStruct.colDataType = column1.colType.colDataType; + + if (colStruct.tokenFlag) + { + WriteEngine::DctnryStruct dictStruct; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + dictStruct.fCompressionType = 2; + } + + dictStruct.dctnryOid = column1.colType.ddn.dictOID; + dictStruct.columnOid = column1.colType.columnOID; + WriteEngine::DctnryTuple dictTuple; + dictTuple.sigValue = (unsigned char*)colNewName.c_str(); + dictTuple.sigSize = colNewName.length(); + dictTuple.isNull = false; + int error = NO_ERROR; + + if (NO_ERROR != + (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file + { + WErrorCodes ec; + err = ec.errorString(error); + rc = error; + return rc; + } + + WriteEngine::Token aToken = dictTuple.token; + colTuple.data = aToken; + } + else + colTuple.data = datavalue; + + colStruct.colDataType = column1.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column1.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column1.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + dctnryStructList.push_back(dctnryStruct); + aColList1.push_back(colTuple); + colValuesList.push_back(aColList1); + WriteEngine::DctnryTuple dctnryTuple; + boost::to_lower(colNewName); + dctnryTuple.sigValue = (unsigned char*)colNewName.c_str(); + dctnryTuple.sigSize = colNewName.length(); + dctnryTuple.isNull = false; + dctColList = dctnryTuple; + dctRowList.clear(); + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // Build AUTOINC_COL structure + WriteEngine::ColTupleList aColList2; + colStruct.dataOid = column2.oid; + colStruct.colWidth = column2.colType.colWidth > 8 ? 8 : column2.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.colDataType = column2.colType.colDataType; + + colTuple.data = autoinc; + + colStruct.colDataType = column2.colType.colDataType; + + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column2.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + dctnryStructList.push_back(dctnryStruct); + aColList2.push_back(colTuple); + colValuesList.push_back(aColList2); + dctnryTuple.isNull = true; + dctColList = dctnryTuple; + dctRowList.clear(); + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // Build NEXTVALUE_COL structure + WriteEngine::ColTupleList aColList3; + colStruct.dataOid = column3.oid; + colStruct.colWidth = column3.colType.colWidth > 8 ? 8 : column3.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.colDataType = column3.colType.colDataType; + + colTuple.data = nextVal; + + colStruct.colDataType = column3.colType.colDataType; + + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column3.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + dctnryStructList.push_back(dctnryStruct); + aColList3.push_back(colTuple); + colValuesList.push_back(aColList3); + + dctnryTuple.isNull = true; + dctColList = dctnryTuple; + dctRowList.clear(); + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // Build NULLABLE_COL structure + WriteEngine::ColTupleList aColList4; + colStruct.dataOid = column4.oid; + colStruct.colWidth = column4.colType.colWidth > 8 ? 8 : column4.colType.colWidth; + colStruct.tokenFlag = false; + colStruct.colDataType = column4.colType.colDataType; + + colTuple.data = nullable; + + colStruct.colDataType = column4.colType.colDataType; + + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + + colStructs.push_back(colStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column4.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + dctnryStructList.push_back(dctnryStruct); + aColList4.push_back(colTuple); + colValuesList.push_back(aColList4); + dctnryTuple.isNull = true; + dctColList = dctnryTuple; + dctRowList.clear(); + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + + // Build DEFAULTVAL_COL structure + WriteEngine::ColTupleList aColList5; + colStruct.dataOid = column5.oid; + colStruct.colWidth = column5.colType.colWidth > 8 ? 8 : column5.colType.colWidth; + colStruct.tokenFlag = false; + + if ((column5.colType.colDataType == CalpontSystemCatalog::CHAR && column5.colType.colWidth > 8) || + (column5.colType.colDataType == CalpontSystemCatalog::VARCHAR && column5.colType.colWidth > 7) || + (column5.colType.colDataType == CalpontSystemCatalog::VARBINARY && column5.colType.colWidth > 7) || + (column5.colType.colDataType == CalpontSystemCatalog::BLOB && column5.colType.colWidth > 7) || + (column5.colType.colDataType == CalpontSystemCatalog::TEXT && column5.colType.colWidth > 7) || + (column5.colType.colDataType == CalpontSystemCatalog::DECIMAL && column5.colType.precision > 18) || + (column5.colType.colDataType == CalpontSystemCatalog::UDECIMAL && + column5.colType.precision > 18)) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = column5.colType.colWidth; + } + + colStruct.colDataType = column5.colType.colDataType; + + if (colStruct.tokenFlag) + { + WriteEngine::DctnryStruct dictStruct; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dictStruct.fCompressionType = 2; + } + + dictStruct.dctnryOid = column5.colType.ddn.dictOID; + dictStruct.columnOid = column5.colType.columnOID; + + if (defaultvalue.length() <= 0) // null token + { + WriteEngine::Token nullToken; + colTuple.data = nullToken; + } + else + { + WriteEngine::DctnryTuple dictTuple; + dictTuple.sigValue = (unsigned char*)defaultvalue.c_str(); + dictTuple.sigSize = defaultvalue.length(); + dictTuple.isNull = false; + int error = NO_ERROR; + + if (NO_ERROR != + (error = fWEWrapper.tokenize(txnID, dictStruct, dictTuple, false))) // @bug 5572 HDFS tmp file + { + WErrorCodes ec; + throw std::runtime_error("WE: Tokenization failed " + ec.errorString(error)); + } + + WriteEngine::Token aToken = dictTuple.token; + colTuple.data = aToken; + } + } + + fWEWrapper.flushDataFiles(rc, txnID, oids); + + colStruct.colDataType = column5.colType.colDataType; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + colStruct.fCompressionType = 2; + dctnryStruct.fCompressionType = 2; + } + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = column5.colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column5.colType); + + // oidsToFlush.push_back(colStruct.dataOid); + if (dctnryStruct.dctnryOid > 0) + { + oids[dctnryStruct.dctnryOid] = dctnryStruct.dctnryOid; + // oidsToFlush.push_back(dctnryStruct.dctnryOid); + } + + aColList5.push_back(colTuple); + colValuesList.push_back(aColList5); + + if (defaultvalue.length() > 0) + { + dctnryTuple.sigValue = (unsigned char*)defaultvalue.c_str(); + dctnryTuple.sigSize = defaultvalue.length(); + dctnryTuple.isNull = false; + } + else + { + dctnryTuple.isNull = true; + } + + dctColList = dctnryTuple; + dctRowList.clear(); + dctRowList.push_back(dctColList); + dctnryValueList.push_back(dctRowList); + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector dctnryExtentsStruct; + std::vector ridLists; + ridLists.push_back(ridList); + + // In this case, there's only 1 row, so only one one extent, but keep it generic... + std::vector extentsinfo; + extentInfo aExtentinfo; + + convertRidToColumn(ropair.rid, dbRoot, partition, segment, 1021); + + aExtentinfo.dbRoot = dbRoot; + aExtentinfo.partition = partition; + aExtentinfo.segment = segment; + + extentsinfo.push_back(aExtentinfo); + + // build colExtentsStruct + for (unsigned i = 0; i < extentsinfo.size(); i++) + { + for (unsigned j = 0; j < colStructs.size(); j++) + { + colStructs[j].fColPartition = extentsinfo[i].partition; + colStructs[j].fColSegment = extentsinfo[i].segment; + colStructs[j].fColDbRoot = extentsinfo[i].dbRoot; + dctnryStructList[j].fColPartition = extentsinfo[i].partition; + dctnryStructList[j].fColSegment = extentsinfo[i].segment; + dctnryStructList[j].fColDbRoot = extentsinfo[i].dbRoot; + } + + colExtentsStruct.push_back(colStructs); + dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); + } + + // call the write engine to update the row + if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, + colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, + SYSCOLUMN_BASE)) + { + err = "WE: Update failed on: " + atableName.table; + rc = 1; + } + + int rc1 = 0; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + rc1 = fWEWrapper.flushDataFiles(rc, txnID, oids); + + if ((rc == 0) && (rc1 == 0)) + { + rc1 = fWEWrapper.confirmTransaction(txnID); + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(txnID, true); + else + fWEWrapper.endTransaction(txnID, false); + } + else + { + fWEWrapper.endTransaction(txnID, false); + } + } + + if (rc == 0) + rc = rc1; + + // flush syscat cahche + systemCatalogPtr->flushCache(); + purgeFDCache(); + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; } uint8_t WE_DDLCommandProc::dropPartitions(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t size, i; - uint32_t tmp32; - std::vector dataOids; - std::vector partitions; + int rc = 0; + uint32_t size, i; + uint32_t tmp32; + std::vector dataOids; + std::vector partitions; - bs >> size; + bs >> size; - for (i = 0; i < size; ++i) - { - bs >> tmp32; - dataOids.push_back(tmp32); - } + for (i = 0; i < size; ++i) + { + bs >> tmp32; + dataOids.push_back(tmp32); + } - bs >> size; - BRM::PartitionInfo pi; + bs >> size; + BRM::PartitionInfo pi; - for (i = 0; i < size; ++i) - { - pi.unserialize(bs); - partitions.push_back(pi); - } + for (i = 0; i < size; ++i) + { + pi.unserialize(bs); + partitions.push_back(pi); + } - try - { - rc = fWEWrapper.deletePartitions(dataOids, partitions); - } - catch (...) - { - err = "WE: Error removing files "; - rc = 1; - } + try + { + rc = fWEWrapper.deletePartitions(dataOids, partitions); + } + catch (...) + { + err = "WE: Error removing files "; + rc = 1; + } - return rc; + return rc; } void WE_DDLCommandProc::purgeFDCache() { - if (idbdatafile::IDBPolicy::useHdfs()) + if (idbdatafile::IDBPolicy::useHdfs()) + { + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(SYSCOLUMN_BASE); + ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); + ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); + ColExtsInfo::iterator aIt; + std::vector files; + BRM::FileInfo aFile; + vector lbidList; + BRM::LBID_t startLbid; + + while (it != colsExtsInfoMap.end()) { - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(SYSCOLUMN_BASE); - ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - std::vector files; - BRM::FileInfo aFile; - vector lbidList; - BRM::LBID_t startLbid; + aIt = (it->second).begin(); + aFile.oid = it->first; - while (it != colsExtsInfoMap.end()) - { - aIt = (it->second).begin(); - aFile.oid = it->first; + while (aIt != (it->second).end()) + { + aFile.partitionNum = aIt->partNum; + aFile.dbRoot = aIt->dbRoot; + aFile.segmentNum = aIt->segNum; + aFile.compType = aIt->compType; + files.push_back(aFile); + fDbrm.lookupLocalStartLbid(aFile.oid, aFile.partitionNum, aFile.segmentNum, aIt->hwm, startLbid); + // cout <<"Added to files oid:dbroot:part:seg:compType = " << + // aFile.oid<<":"<second).end()) - { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot = aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - fDbrm.lookupLocalStartLbid(aFile.oid, aFile.partitionNum, aFile.segmentNum, aIt->hwm, startLbid); - //cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"< template -bool from_string(T& t, - const std::string& s, - std::ios_base & (*f)(std::ios_base&)) +bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&)) { - std::istringstream iss(s); - return !(iss >> f >> t).fail(); + std::istringstream iss(s); + return !(iss >> f >> t).fail(); } namespace WriteEngine { struct DDLColumn { - execplan::CalpontSystemCatalog::OID oid; - execplan::CalpontSystemCatalog::ColType colType; - execplan::CalpontSystemCatalog::TableColName tableColName; + execplan::CalpontSystemCatalog::OID oid; + execplan::CalpontSystemCatalog::ColType colType; + execplan::CalpontSystemCatalog::TableColName tableColName; }; typedef std::vector ColumnList; struct DictOID { - int dictOID; - int listOID; - int treeOID; - int colWidth; - int compressionType; + int dictOID; + int listOID; + int treeOID; + int colWidth; + int compressionType; }; struct extentInfo { - uint16_t dbRoot; - uint32_t partition; - uint16_t segment; - bool operator==(const extentInfo& rhs) const - { - return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment); - } - bool operator!=(const extentInfo& rhs) const - { - return !(*this == rhs); - } + uint16_t dbRoot; + uint32_t partition; + uint16_t segment; + bool operator==(const extentInfo& rhs) const + { + return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment); + } + bool operator!=(const extentInfo& rhs) const + { + return !(*this == rhs); + } }; -inline void getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, - ColumnList& colList) +inline void getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, ColumnList& colList) { - execplan::CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = table; - std::string err; + tableName.schema = schema; + tableName.table = table; + std::string err; - try + try + { + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); + + const execplan::CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName); + + execplan::CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); + + while (rid_iterator != ridList.end()) { - boost::shared_ptr systemCatalogPtr = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); + execplan::CalpontSystemCatalog::ROPair roPair = *rid_iterator; - const execplan::CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName); + DDLColumn column; + column.oid = roPair.objnum; + column.colType = systemCatalogPtr->colType(column.oid); + column.tableColName = systemCatalogPtr->colName(column.oid); - execplan::CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); - - while (rid_iterator != ridList.end()) - { - execplan::CalpontSystemCatalog::ROPair roPair = *rid_iterator; - - DDLColumn column; - column.oid = roPair.objnum; - column.colType = systemCatalogPtr->colType(column.oid); - column.tableColName = systemCatalogPtr->colName(column.oid); - - colList.push_back(column); - - ++rid_iterator; - } + colList.push_back(column); + ++rid_iterator; } - catch (exception& ex) - { - - err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " + schema + '.' + table + ": " + ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!" ; - throw std::runtime_error(err); - } - + } + catch (exception& ex) + { + err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " + schema + '.' + table + + ": " + ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!"; + throw std::runtime_error(err); + } } - inline int convertDataType(int dataType) { - int calpontDataType; + int calpontDataType; - switch (dataType) - { - case ddlpackage::DDL_CHAR: - calpontDataType = execplan::CalpontSystemCatalog::CHAR; - break; + switch (dataType) + { + case ddlpackage::DDL_CHAR: calpontDataType = execplan::CalpontSystemCatalog::CHAR; break; - case ddlpackage::DDL_VARCHAR: - calpontDataType = execplan::CalpontSystemCatalog::VARCHAR; - break; + case ddlpackage::DDL_VARCHAR: calpontDataType = execplan::CalpontSystemCatalog::VARCHAR; break; - case ddlpackage::DDL_VARBINARY: - calpontDataType = execplan::CalpontSystemCatalog::VARBINARY; - break; + case ddlpackage::DDL_VARBINARY: calpontDataType = execplan::CalpontSystemCatalog::VARBINARY; break; - case ddlpackage::DDL_BIT: - calpontDataType = execplan::CalpontSystemCatalog::BIT; - break; + case ddlpackage::DDL_BIT: calpontDataType = execplan::CalpontSystemCatalog::BIT; break; - case ddlpackage::DDL_REAL: - case ddlpackage::DDL_DECIMAL: - case ddlpackage::DDL_NUMERIC: - case ddlpackage::DDL_NUMBER: - calpontDataType = execplan::CalpontSystemCatalog::DECIMAL; - break; + case ddlpackage::DDL_REAL: + case ddlpackage::DDL_DECIMAL: + case ddlpackage::DDL_NUMERIC: + case ddlpackage::DDL_NUMBER: calpontDataType = execplan::CalpontSystemCatalog::DECIMAL; break; - case ddlpackage::DDL_FLOAT: - calpontDataType = execplan::CalpontSystemCatalog::FLOAT; - break; + case ddlpackage::DDL_FLOAT: calpontDataType = execplan::CalpontSystemCatalog::FLOAT; break; - case ddlpackage::DDL_DOUBLE: - calpontDataType = execplan::CalpontSystemCatalog::DOUBLE; - break; + case ddlpackage::DDL_DOUBLE: calpontDataType = execplan::CalpontSystemCatalog::DOUBLE; break; - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: - calpontDataType = execplan::CalpontSystemCatalog::INT; - break; + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: calpontDataType = execplan::CalpontSystemCatalog::INT; break; - case ddlpackage::DDL_BIGINT: - calpontDataType = execplan::CalpontSystemCatalog::BIGINT; - break; + case ddlpackage::DDL_BIGINT: calpontDataType = execplan::CalpontSystemCatalog::BIGINT; break; - case ddlpackage::DDL_MEDINT: - calpontDataType = execplan::CalpontSystemCatalog::MEDINT; - break; + case ddlpackage::DDL_MEDINT: calpontDataType = execplan::CalpontSystemCatalog::MEDINT; break; - case ddlpackage::DDL_SMALLINT: - calpontDataType = execplan::CalpontSystemCatalog::SMALLINT; - break; + case ddlpackage::DDL_SMALLINT: calpontDataType = execplan::CalpontSystemCatalog::SMALLINT; break; - case ddlpackage::DDL_TINYINT: - calpontDataType = execplan::CalpontSystemCatalog::TINYINT; - break; + case ddlpackage::DDL_TINYINT: calpontDataType = execplan::CalpontSystemCatalog::TINYINT; break; - case ddlpackage::DDL_DATE: - calpontDataType = execplan::CalpontSystemCatalog::DATE; - break; + case ddlpackage::DDL_DATE: calpontDataType = execplan::CalpontSystemCatalog::DATE; break; - case ddlpackage::DDL_DATETIME: - calpontDataType = execplan::CalpontSystemCatalog::DATETIME; - break; + case ddlpackage::DDL_DATETIME: calpontDataType = execplan::CalpontSystemCatalog::DATETIME; break; - case ddlpackage::DDL_TIME: - calpontDataType = execplan::CalpontSystemCatalog::TIME; - break; + case ddlpackage::DDL_TIME: calpontDataType = execplan::CalpontSystemCatalog::TIME; break; - case ddlpackage::DDL_TIMESTAMP: - calpontDataType = execplan::CalpontSystemCatalog::TIMESTAMP; - break; + case ddlpackage::DDL_TIMESTAMP: calpontDataType = execplan::CalpontSystemCatalog::TIMESTAMP; break; - case ddlpackage::DDL_CLOB: - calpontDataType = execplan::CalpontSystemCatalog::CLOB; - break; + case ddlpackage::DDL_CLOB: calpontDataType = execplan::CalpontSystemCatalog::CLOB; break; - case ddlpackage::DDL_BLOB: - calpontDataType = execplan::CalpontSystemCatalog::BLOB; - break; + case ddlpackage::DDL_BLOB: calpontDataType = execplan::CalpontSystemCatalog::BLOB; break; - case ddlpackage::DDL_TEXT: - calpontDataType = execplan::CalpontSystemCatalog::TEXT; - break; + case ddlpackage::DDL_TEXT: calpontDataType = execplan::CalpontSystemCatalog::TEXT; break; - case ddlpackage::DDL_UNSIGNED_TINYINT: - calpontDataType = execplan::CalpontSystemCatalog::UTINYINT; - break; + case ddlpackage::DDL_UNSIGNED_TINYINT: calpontDataType = execplan::CalpontSystemCatalog::UTINYINT; break; - case ddlpackage::DDL_UNSIGNED_SMALLINT: - calpontDataType = execplan::CalpontSystemCatalog::USMALLINT; - break; + case ddlpackage::DDL_UNSIGNED_SMALLINT: + calpontDataType = execplan::CalpontSystemCatalog::USMALLINT; + break; - case ddlpackage::DDL_UNSIGNED_MEDINT: - calpontDataType = execplan::CalpontSystemCatalog::UMEDINT; - break; + case ddlpackage::DDL_UNSIGNED_MEDINT: calpontDataType = execplan::CalpontSystemCatalog::UMEDINT; break; - case ddlpackage::DDL_UNSIGNED_INT: - calpontDataType = execplan::CalpontSystemCatalog::UINT; - break; + case ddlpackage::DDL_UNSIGNED_INT: calpontDataType = execplan::CalpontSystemCatalog::UINT; break; - case ddlpackage::DDL_UNSIGNED_BIGINT: - calpontDataType = execplan::CalpontSystemCatalog::UBIGINT; - break; + case ddlpackage::DDL_UNSIGNED_BIGINT: calpontDataType = execplan::CalpontSystemCatalog::UBIGINT; break; - case ddlpackage::DDL_UNSIGNED_DECIMAL: - case ddlpackage::DDL_UNSIGNED_NUMERIC: - calpontDataType = execplan::CalpontSystemCatalog::UDECIMAL; - break; + case ddlpackage::DDL_UNSIGNED_DECIMAL: + case ddlpackage::DDL_UNSIGNED_NUMERIC: calpontDataType = execplan::CalpontSystemCatalog::UDECIMAL; break; - case ddlpackage::DDL_UNSIGNED_FLOAT: - calpontDataType = execplan::CalpontSystemCatalog::UFLOAT; - break; + case ddlpackage::DDL_UNSIGNED_FLOAT: calpontDataType = execplan::CalpontSystemCatalog::UFLOAT; break; - case ddlpackage::DDL_UNSIGNED_DOUBLE: - calpontDataType = execplan::CalpontSystemCatalog::UDOUBLE; - break; - - default: - throw runtime_error("Unsupported datatype!"); + case ddlpackage::DDL_UNSIGNED_DOUBLE: calpontDataType = execplan::CalpontSystemCatalog::UDOUBLE; break; - } + default: throw runtime_error("Unsupported datatype!"); + } - return calpontDataType; + return calpontDataType; } inline void findColumnData(uint32_t sessionID, execplan::CalpontSystemCatalog::TableName& systableName, const std::string& colName, DDLColumn& sysCol) { - ColumnList columns; - ColumnList::const_iterator column_iterator; - std::string err; + ColumnList columns; + ColumnList::const_iterator column_iterator; + std::string err; - try + try + { + getColumnsForTable(sessionID, systableName.schema, systableName.table, columns); + column_iterator = columns.begin(); + + while (column_iterator != columns.end()) { - getColumnsForTable(sessionID, systableName.schema, systableName.table, columns); - column_iterator = columns.begin(); + sysCol = *column_iterator; + boost::to_lower(sysCol.tableColName.column); - while (column_iterator != columns.end()) - { - sysCol = *column_iterator; - boost::to_lower(sysCol.tableColName.column); + if (colName == sysCol.tableColName.column) + { + break; + } - if (colName == sysCol.tableColName.column) - { - break; - } - - ++column_iterator; - } - } - catch (exception& ex) - { - err = ex.what(); - throw std::runtime_error(err); - } - catch (...) - { - err = "findColumnData:Unknown exception caught"; - throw std::runtime_error(err); + ++column_iterator; } + } + catch (exception& ex) + { + err = ex.what(); + throw std::runtime_error(err); + } + catch (...) + { + err = "findColumnData:Unknown exception caught"; + throw std::runtime_error(err); + } } -inline void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, - unsigned& segment, unsigned filesPerColumnPartition, - unsigned extentsPerSegmentFile, unsigned extentRows, - unsigned startDBRoot, unsigned dbrootCnt) +inline void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, unsigned& segment, + unsigned filesPerColumnPartition, unsigned extentsPerSegmentFile, + unsigned extentRows, unsigned startDBRoot, unsigned dbrootCnt) { - partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); + partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); - segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % filesPerColumnPartition; + segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % + filesPerColumnPartition; - dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; + dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; - //Calculate the relative rid for this segment file - uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); - idbassert(relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); - uint32_t numExtentsInThisPart = relRidInPartition / extentRows; - unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; - uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; - rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; + // Calculate the relative rid for this segment file + uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * + (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); + idbassert(relRidInPartition <= + (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows); + uint32_t numExtentsInThisPart = relRidInPartition / extentRows; + unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; + uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; + rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; } -} +} // namespace WriteEngine #undef EXPORT #endif - diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index fcbe80759..d183e3b0f 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -57,2392 +57,2392 @@ using namespace std; namespace WriteEngine { -//StopWatch timer; +// StopWatch timer; WE_DMLCommandProc::WE_DMLCommandProc() { - fIsFirstBatchPm = true; - filesPerColumnPartition = 8; - extentsPerSegmentFile = 1; - dbrootCnt = 1; - extentRows = 0x800000; - config::Config* cf = config::Config::makeConfig(); - string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); + fIsFirstBatchPm = true; + filesPerColumnPartition = 8; + extentsPerSegmentFile = 1; + dbrootCnt = 1; + extentRows = 0x800000; + config::Config* cf = config::Config::makeConfig(); + string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); - if (fpc.length() != 0) - filesPerColumnPartition = cf->uFromText(fpc); + if (fpc.length() != 0) + filesPerColumnPartition = cf->uFromText(fpc); - // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). - extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; - string dbct = cf->getConfig("SystemConfig", "DBRootCount"); + string dbct = cf->getConfig("SystemConfig", "DBRootCount"); - if (dbct.length() != 0) - dbrootCnt = cf->uFromText(dbct); + if (dbct.length() != 0) + dbrootCnt = cf->uFromText(dbct); } WE_DMLCommandProc::WE_DMLCommandProc(const WE_DMLCommandProc& rhs) { - fIsFirstBatchPm = rhs.fIsFirstBatchPm; - fRBMetaWriter.reset(new RBMetaWriter("BatchInsert", NULL)); + fIsFirstBatchPm = rhs.fIsFirstBatchPm; + fRBMetaWriter.reset(new RBMetaWriter("BatchInsert", NULL)); } WE_DMLCommandProc::~WE_DMLCommandProc() { - dbRootExtTrackerVec.clear(); + dbRootExtTrackerVec.clear(); } uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - err.clear(); - InsertDMLPackage insertPkg; - ByteStream::quadbyte tmp32; - bs >> tmp32; - BRM::TxnID txnid; - txnid.valid = true; - txnid.id = tmp32; - bs >> tmp32; - uint32_t dbroot = tmp32; + uint8_t rc = 0; + err.clear(); + InsertDMLPackage insertPkg; + ByteStream::quadbyte tmp32; + bs >> tmp32; + BRM::TxnID txnid; + txnid.valid = true; + txnid.id = tmp32; + bs >> tmp32; + uint32_t dbroot = tmp32; - //cout << "processSingleInsert received bytestream length " << bs.length() << endl; + // cout << "processSingleInsert received bytestream length " << bs.length() << endl; - messageqcpp::ByteStream::byte packageType; - bs >> packageType; - insertPkg.read( bs); - uint32_t sessionId = insertPkg.get_SessionID(); - //cout << " processSingleInsert for session " << sessionId << endl; - DMLTable* tablePtr = insertPkg.get_Table(); - RowList rows = tablePtr->get_RowList(); + messageqcpp::ByteStream::byte packageType; + bs >> packageType; + insertPkg.read(bs); + uint32_t sessionId = insertPkg.get_SessionID(); + // cout << " processSingleInsert for session " << sessionId << endl; + DMLTable* tablePtr = insertPkg.get_Table(); + RowList rows = tablePtr->get_RowList(); - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::ColValueList colValuesList; - WriteEngine::DictStrList dicStringList ; - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::TableColName tableColName; - tableName.table = tableColName.table = tablePtr->get_TableName(); - tableName.schema = tableColName.schema = tablePtr->get_SchemaName(); + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::ColValueList colValuesList; + WriteEngine::DictStrList dicStringList; + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::TableColName tableColName; + tableName.table = tableColName.table = tablePtr->get_TableName(); + tableName.schema = tableColName.schema = tablePtr->get_SchemaName(); - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::ROPair tableRoPair; - std::vector colNames; - bool isWarningSet = false; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair tableRoPair; + std::vector colNames; + bool isWarningSet = false; + + try + { + tableRoPair = systemCatalogPtr->tableRID(tableName); + + if (rows.size()) + { + Row* rowPtr = rows.at(0); + ColumnList columns = rowPtr->get_ColumnList(); + unsigned int numcols = rowPtr->get_NumberOfColumns(); + cscColTypeList.reserve(numcols); + // WIP + // We presume that DictCols number is low + colStructs.reserve(numcols); + ColumnList::const_iterator column_iterator = columns.begin(); + + while (column_iterator != columns.end()) + { + DMLColumn* columnPtr = *column_iterator; + tableColName.column = columnPtr->get_Name(); + // TODO MCOL-641 replace with getColRidsOidsTypes() + CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName); + + CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); + + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(oid); + + WriteEngine::ColStruct colStruct; + colStruct.fColDbRoot = dbroot; + WriteEngine::DctnryStruct dctnryStruct; + dctnryStruct.fColDbRoot = dbroot; + colStruct.dataOid = roPair.objnum; + colStruct.tokenFlag = false; + colStruct.fCompressionType = colType.compressionType; + + // Token + if (isDictCol(colType)) + { + // WIP Hardcoded value + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = colType.colWidth; + } + + colStruct.colDataType = colType.colDataType; + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(colType); + + ++column_iterator; + } + + std::string tmpStr(""); + + for (unsigned int i = 0; i < numcols; i++) + { + WriteEngine::ColTupleList colTuples; + WriteEngine::DctColTupleList dctColTuples; + RowList::const_iterator row_iterator = rows.begin(); + + while (row_iterator != rows.end()) + { + Row* rowPtr = *row_iterator; + const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i); + + tableColName.column = columnPtr->get_Name(); + // TODO MCOL-641 remove these calls + CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); + + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(oid); + + boost::any datavalue; + bool isNULL = false; + bool pushWarning = false; + std::vector origVals; + origVals = columnPtr->get_DataVector(); + WriteEngine::dictStr dicStrings; + + // token + if (isDictCol(colType)) + { + for (uint32_t i = 0; i < origVals.size(); i++) + { + tmpStr = origVals[i]; + + isNULL = columnPtr->get_isnull(); + + if (isNULL || (tmpStr.length() == 0)) + isNULL = true; + else + isNULL = false; + + if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (isNULL && colType.defaultValue.empty()) // error out + { + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + rc = 1; + return rc; + } + else if (isNULL && !(colType.defaultValue.empty())) + { + tmpStr = colType.defaultValue; + } + } + + if (tmpStr.length() > (unsigned int)colType.colWidth) + { + tmpStr = tmpStr.substr(0, colType.colWidth); + + if (!pushWarning) + { + pushWarning = true; + isWarningSet = true; + + if ((rc != NO_ERROR) && (rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + + colNames.push_back(tableColName.column); + } + } + + WriteEngine::ColTuple colTuple; + colTuple.data = datavalue; + + colTuples.push_back(colTuple); + //@Bug 2515. Only pass string values to write engine + dicStrings.push_back(tmpStr); + } + + colValuesList.push_back(colTuples); + //@Bug 2515. Only pass string values to write engine + dicStringList.push_back(dicStrings); + } + else + { + string x; + std::string indata; + + for (uint32_t i = 0; i < origVals.size(); i++) + { + indata = origVals[i]; + + isNULL = columnPtr->get_isnull(); + + if (isNULL || (indata.length() == 0)) + isNULL = true; + else + isNULL = false; + + // check if autoincrement column and value is 0 or null + uint64_t nextVal = 1; + + if (colType.autoincrement) + { + try + { + // WIP What if we combine this and previous loop and fail + // after get nextAIValue ? + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + + if (colType.autoincrement && (isNULL || (indata.compare("0") == 0))) + { + try + { + bool reserved = fDbrm.getAIRange(oid, 1, &nextVal); + + if (!reserved) + { + err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); + rc = 1; + return rc; + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + ostringstream oss; + oss << nextVal; + indata = oss.str(); + isNULL = false; + } + + if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (isNULL && colType.defaultValue.empty()) // error out + { + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + rc = 1; + return rc; + } + else if (isNULL && !(colType.defaultValue.empty())) + { + indata = colType.defaultValue; + isNULL = false; + } + } + + try + { + datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, + false, false); + } + catch (exception&) + { + rc = 1; + Message::Args args; + args.add(string("'") + indata + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + } + + //@Bug 1806 + if (rc != NO_ERROR && rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + { + return rc; + } + + if (pushWarning) + { + if (!isWarningSet) + isWarningSet = true; + + if (rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + + colNames.push_back(tableColName.column); + } + + WriteEngine::ColTuple colTuple; + colTuple.data = datavalue; + + colTuples.push_back(colTuple); + //@Bug 2515. Only pass string values to write engine + dicStrings.push_back(tmpStr); + } + + colValuesList.push_back(colTuples); + dicStringList.push_back(dicStrings); + } + + ++row_iterator; + } + } + } + } + catch (exception& ex) + { + rc = 1; + err = ex.what(); + return rc; + } + + // call the write engine to write the rows + int error = NO_ERROR; + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + fWEWrapper.setTransId(txnid.id); + // For hdfs use only + uint32_t tblOid = tableRoPair.objnum; + + // WIP are we saving HDFS? + if (idbdatafile::IDBPolicy::useHdfs()) + { + std::vector columns; + DctnryStructList dctnryList; + CalpontSystemCatalog::ColType colType; + std::vector colDBRootExtentInfo; + Convertor convertor; + dbRootExtTrackerVec.clear(); + fRBMetaWriter.reset(new RBMetaWriter("SingleInsert", NULL)); + CalpontSystemCatalog::RIDList ridList; try { - tableRoPair = systemCatalogPtr->tableRID(tableName); + ridList = systemCatalogPtr->columnRIDs(tableName, true); + std::vector dctnryStoreOids(ridList.size()); + std::vector dbRootHWMInfoColVec(ridList.size()); + bool bFirstExtentOnThisPM = false; - if (rows.size()) + // First gather HWM BRM information for all columns + std::vector colWidths; + + for (unsigned i = 0; i < ridList.size(); i++) + { + rc = BRMWrapper::getInstance()->getDbRootHWMInfo(ridList[i].objnum, dbRootHWMInfoColVec[i]); + // need handle error + + CalpontSystemCatalog::ColType colType2 = systemCatalogPtr->colType(ridList[i].objnum); + colWidths.push_back(convertor.getCorrectRowWidth(colType2.colDataType, colType2.colWidth)); + } + + for (unsigned i = 0; i < ridList.size(); i++) + { + // Find DBRoot/segment file where we want to start adding rows + colType = systemCatalogPtr->colType(ridList[i].objnum); + boost::shared_ptr pDBRootExtentTracker( + new DBRootExtentTracker(ridList[i].objnum, colWidths, dbRootHWMInfoColVec, i, 0)); + dbRootExtTrackerVec.push_back(pDBRootExtentTracker); + DBRootExtentInfo dbRootExtent; + std::string trkErrMsg; + bool bEmptyPM; + + if (i == 0) + rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, + trkErrMsg); + else + pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent); + + colDBRootExtentInfo.push_back(dbRootExtent); + + Column aColumn; + aColumn.colWidth = convertor.getCorrectRowWidth(colType.colDataType, colType.colWidth); + aColumn.colDataType = colType.colDataType; + aColumn.compressionType = colType.compressionType; + aColumn.dataFile.oid = ridList[i].objnum; + aColumn.dataFile.fPartition = dbRootExtent.fPartition; + aColumn.dataFile.fSegment = dbRootExtent.fSegment; + aColumn.dataFile.fDbRoot = dbRootExtent.fDbRoot; + aColumn.dataFile.hwm = dbRootExtent.fLocalHwm; + columns.push_back(aColumn); + + if ((colType.compressionType > 0) && (colType.ddn.dictOID > 0)) { - Row* rowPtr = rows.at(0); - ColumnList columns = rowPtr->get_ColumnList(); - unsigned int numcols = rowPtr->get_NumberOfColumns(); - cscColTypeList.reserve(numcols); - // WIP - // We presume that DictCols number is low - colStructs.reserve(numcols); - ColumnList::const_iterator column_iterator = columns.begin(); - - while (column_iterator != columns.end()) - { - DMLColumn* columnPtr = *column_iterator; - tableColName.column = columnPtr->get_Name(); - // TODO MCOL-641 replace with getColRidsOidsTypes() - CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName); - - CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); - - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType(oid); - - WriteEngine::ColStruct colStruct; - colStruct.fColDbRoot = dbroot; - WriteEngine::DctnryStruct dctnryStruct; - dctnryStruct.fColDbRoot = dbroot; - colStruct.dataOid = roPair.objnum; - colStruct.tokenFlag = false; - colStruct.fCompressionType = colType.compressionType; - - // Token - if (isDictCol(colType) ) - { - // WIP Hardcoded value - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = colType.colWidth; - } - - colStruct.colDataType = colType.colDataType; - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - cscColTypeList.push_back(colType); - - ++column_iterator; - } - - std::string tmpStr(""); - - for (unsigned int i = 0; i < numcols; i++) - { - - WriteEngine::ColTupleList colTuples; - WriteEngine::DctColTupleList dctColTuples; - RowList::const_iterator row_iterator = rows.begin(); - - while (row_iterator != rows.end()) - { - Row* rowPtr = *row_iterator; - const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i); - - tableColName.column = columnPtr->get_Name(); - // TODO MCOL-641 remove these calls - CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); - - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType(oid); - - boost::any datavalue; - bool isNULL = false; - bool pushWarning = false; - std::vector origVals; - origVals = columnPtr->get_DataVector(); - WriteEngine::dictStr dicStrings; - - // token - if ( isDictCol(colType) ) - { - for ( uint32_t i = 0; i < origVals.size(); i++ ) - { - tmpStr = origVals[i]; - - isNULL = columnPtr->get_isnull(); - - if ( isNULL || ( tmpStr.length() == 0 ) ) - isNULL = true; - else - isNULL = false; - - if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (isNULL && colType.defaultValue.empty()) //error out - { - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - rc = 1; - return rc; - } - else if (isNULL && !(colType.defaultValue.empty())) - { - tmpStr = colType.defaultValue; - } - } - - if ( tmpStr.length() > (unsigned int)colType.colWidth ) - { - tmpStr = tmpStr.substr(0, colType.colWidth); - - if ( !pushWarning ) - { - pushWarning = true; - isWarningSet = true; - - if ((rc != NO_ERROR) && (rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - - colNames.push_back(tableColName.column); - } - } - - WriteEngine::ColTuple colTuple; - colTuple.data = datavalue; - - colTuples.push_back(colTuple); - //@Bug 2515. Only pass string values to write engine - dicStrings.push_back( tmpStr ); - } - - colValuesList.push_back(colTuples); - //@Bug 2515. Only pass string values to write engine - dicStringList.push_back( dicStrings ); - } - else - { - string x; - std::string indata; - - for ( uint32_t i = 0; i < origVals.size(); i++ ) - { - indata = origVals[i]; - - isNULL = columnPtr->get_isnull(); - - if ( isNULL || ( indata.length() == 0 ) ) - isNULL = true; - else - isNULL = false; - - //check if autoincrement column and value is 0 or null - uint64_t nextVal = 1; - - if (colType.autoincrement) - { - try - { - // WIP What if we combine this and previous loop and fail - // after get nextAIValue ? - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - - if (colType.autoincrement && ( isNULL || (indata.compare("0") == 0))) - { - try - { - bool reserved = fDbrm.getAIRange(oid, 1, &nextVal); - - if (!reserved) - { - err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); - rc = 1; - return rc; - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - ostringstream oss; - oss << nextVal; - indata = oss.str(); - isNULL = false; - } - - if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (isNULL && colType.defaultValue.empty()) //error out - { - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - rc = 1; - return rc; - } - else if (isNULL && !(colType.defaultValue.empty())) - { - indata = colType.defaultValue; - isNULL = false; - } - } - - try - { - datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); - } - catch (exception&) - { - rc = 1; - Message::Args args; - args.add(string("'") + indata + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - } - - //@Bug 1806 - if (rc != NO_ERROR && rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - { - return rc; - } - - if (pushWarning) - { - if (!isWarningSet) - isWarningSet = true; - - if ( rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING ) - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - - colNames.push_back(tableColName.column); - } - - WriteEngine::ColTuple colTuple; - colTuple.data = datavalue; - - colTuples.push_back(colTuple); - //@Bug 2515. Only pass string values to write engine - dicStrings.push_back( tmpStr ); - } - - colValuesList.push_back(colTuples); - dicStringList.push_back( dicStrings ); - } - - ++row_iterator; - } - } + DctnryStruct aDctnry; + aDctnry.dctnryOid = colType.ddn.dictOID; + aDctnry.fColPartition = dbRootExtent.fPartition; + aDctnry.fColSegment = dbRootExtent.fSegment; + aDctnry.fColDbRoot = dbRootExtent.fDbRoot; + dctnryList.push_back(aDctnry); } - } - catch (exception& ex) - { - rc = 1; - err = ex.what(); - return rc; - } - // call the write engine to write the rows - int error = NO_ERROR; - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - fWEWrapper.setTransId(txnid.id); - //For hdfs use only - uint32_t tblOid = tableRoPair.objnum; - - // WIP are we saving HDFS? - if (idbdatafile::IDBPolicy::useHdfs()) - { - - std::vector columns; - DctnryStructList dctnryList; - CalpontSystemCatalog::ColType colType; - std::vector colDBRootExtentInfo; - Convertor convertor; - dbRootExtTrackerVec.clear(); - fRBMetaWriter.reset(new RBMetaWriter("SingleInsert", NULL)); - CalpontSystemCatalog::RIDList ridList; - - try + if (colType.ddn.dictOID > 0) { - ridList = systemCatalogPtr->columnRIDs(tableName, true); - std::vector dctnryStoreOids(ridList.size()) ; - std::vector dbRootHWMInfoColVec(ridList.size()); - bool bFirstExtentOnThisPM = false; - - // First gather HWM BRM information for all columns - std::vector colWidths; - - for (unsigned i = 0; i < ridList.size(); i++) - { - rc = BRMWrapper::getInstance()->getDbRootHWMInfo(ridList[i].objnum, dbRootHWMInfoColVec[i]); - //need handle error - - CalpontSystemCatalog::ColType colType2 = systemCatalogPtr->colType(ridList[i].objnum); - colWidths.push_back( convertor.getCorrectRowWidth( - colType2.colDataType, colType2.colWidth) ); - } - - for (unsigned i = 0; i < ridList.size(); i++) - { - // Find DBRoot/segment file where we want to start adding rows - colType = systemCatalogPtr->colType(ridList[i].objnum); - boost::shared_ptr pDBRootExtentTracker (new DBRootExtentTracker(ridList[i].objnum, - colWidths, dbRootHWMInfoColVec, i, 0) ); - dbRootExtTrackerVec.push_back( pDBRootExtentTracker ); - DBRootExtentInfo dbRootExtent; - std::string trkErrMsg; - bool bEmptyPM; - - if (i == 0) - rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg); - else - pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent); - - colDBRootExtentInfo.push_back(dbRootExtent); - - Column aColumn; - aColumn.colWidth = convertor.getCorrectRowWidth(colType.colDataType, colType.colWidth); - aColumn.colDataType = colType.colDataType; - aColumn.compressionType = colType.compressionType; - aColumn.dataFile.oid = ridList[i].objnum; - aColumn.dataFile.fPartition = dbRootExtent.fPartition; - aColumn.dataFile.fSegment = dbRootExtent.fSegment; - aColumn.dataFile.fDbRoot = dbRootExtent.fDbRoot; - aColumn.dataFile.hwm = dbRootExtent.fLocalHwm; - columns.push_back(aColumn); - - if ((colType.compressionType > 0) && (colType.ddn.dictOID > 0)) - { - DctnryStruct aDctnry; - aDctnry.dctnryOid = colType.ddn.dictOID; - aDctnry.fColPartition = dbRootExtent.fPartition; - aDctnry.fColSegment = dbRootExtent.fSegment; - aDctnry.fColDbRoot = dbRootExtent.fDbRoot; - dctnryList.push_back(aDctnry); - } - - if (colType.ddn.dictOID > 0) - { - dctnryStoreOids[i] = colType.ddn.dictOID; - } - else - { - dctnryStoreOids[i] = 0; - } - } - - fRBMetaWriter->init(tblOid, tableName.table); - fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); - - //cout << "Backing up hwm chunks" << endl; - for (unsigned i = 0; i < dctnryList.size(); i++) //back up chunks for compressed dictionary - { - // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context - fRBMetaWriter->backupDctnryHWMChunk( - dctnryList[i].dctnryOid, dctnryList[i].fColDbRoot, dctnryList[i].fColPartition, dctnryList[i].fColSegment); - } + dctnryStoreOids[i] = colType.ddn.dictOID; } - catch (std::exception& ex) + else { - err = ex.what(); - rc = 1; - return rc; + dctnryStoreOids[i] = 0; } + } + + fRBMetaWriter->init(tblOid, tableName.table); + fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); + + // cout << "Backing up hwm chunks" << endl; + for (unsigned i = 0; i < dctnryList.size(); i++) // back up chunks for compressed dictionary + { + // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context + fRBMetaWriter->backupDctnryHWMChunk(dctnryList[i].dctnryOid, dctnryList[i].fColDbRoot, + dctnryList[i].fColPartition, dctnryList[i].fColSegment); + } } - - if (colValuesList[0].size() > 0) + catch (std::exception& ex) { - if (NO_ERROR != - (error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) - { - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - else if (error == ERR_BRM_VB_OVERFLOW) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } - else - { - rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - } + err = ex.what(); + rc = 1; + return rc; } + } - std::map oids; - std::vector oidsToFlush; - - for (unsigned i = 0; i < colStructs.size(); i++) - { - oids[colStructs[i].dataOid] = colStructs[i].dataOid; - oidsToFlush.push_back(colStructs[i].dataOid); - } - - for (unsigned i = 0; i < dctnryStructList.size(); i++) - { - oids[dctnryStructList[i].dctnryOid] = dctnryStructList[i].dctnryOid; - oidsToFlush.push_back(dctnryStructList[i].dctnryOid); - } - - fWEWrapper.setTransId(txnid.id); - vector lbidList; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - // XXX THIS IS WRONG - //save the extent info to mark them invalid, after flush, the meta file will be gone. - std::tr1::unordered_map m_txnLBIDMap = fWEWrapper.getTxnMap(); - - try - { - auto mapIter = m_txnLBIDMap.find(txnid.id); - - if (mapIter != m_txnLBIDMap.end()) - { - SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; - lbidList = spTxnLBIDRec->m_LBIDs; - } - } - catch (...) {} - } - - //flush files - // @bug5333, up to here, rc may have an error code already, don't overwrite it. - int flushChunksRc = fWEWrapper.flushChunks(0, oids); // why not pass rc to flushChunks? - - if (flushChunksRc != NO_ERROR) + if (colValuesList[0].size() > 0) + { + if (NO_ERROR != + (error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypeList, colStructs, colValuesList, + dctnryStructList, dicStringList, tableRoPair.objnum))) { + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; WErrorCodes ec; - std::ostringstream ossErr; - ossErr << "Error flushing chunks for table " << tableName << - "; " << ec.errorString(flushChunksRc); + err = ec.errorString(error); + } + else if (error == ERR_BRM_VB_OVERFLOW) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + else + { + rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + WErrorCodes ec; + err = ec.errorString(error); + } + } + } - // Append to errmsg in case we already have an error - if (err.length() > 0) - err += "; "; + std::map oids; + std::vector oidsToFlush; - err += ossErr.str(); + for (unsigned i = 0; i < colStructs.size(); i++) + { + oids[colStructs[i].dataOid] = colStructs[i].dataOid; + oidsToFlush.push_back(colStructs[i].dataOid); + } - if (error == NO_ERROR) - error = flushChunksRc; + for (unsigned i = 0; i < dctnryStructList.size(); i++) + { + oids[dctnryStructList[i].dctnryOid] = dctnryStructList[i].dctnryOid; + oidsToFlush.push_back(dctnryStructList[i].dctnryOid); + } - if ((rc == NO_ERROR) || (rc == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) - rc = 1; // return hardcoded 1 as the above + fWEWrapper.setTransId(txnid.id); + vector lbidList; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + // XXX THIS IS WRONG + // save the extent info to mark them invalid, after flush, the meta file will be gone. + std::tr1::unordered_map m_txnLBIDMap = fWEWrapper.getTxnMap(); + + try + { + auto mapIter = m_txnLBIDMap.find(txnid.id); + + if (mapIter != m_txnLBIDMap.end()) + { + SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; + lbidList = spTxnLBIDRec->m_LBIDs; + } + } + catch (...) + { + } + } + + // flush files + // @bug5333, up to here, rc may have an error code already, don't overwrite it. + int flushChunksRc = fWEWrapper.flushChunks(0, oids); // why not pass rc to flushChunks? + + if (flushChunksRc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream ossErr; + ossErr << "Error flushing chunks for table " << tableName << "; " << ec.errorString(flushChunksRc); + + // Append to errmsg in case we already have an error + if (err.length() > 0) + err += "; "; + + err += ossErr.str(); + + if (error == NO_ERROR) + error = flushChunksRc; + + if ((rc == NO_ERROR) || (rc == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + rc = 1; // return hardcoded 1 as the above + } + + // Confirm HDFS DB file changes "only" if no error up to this point + if (idbdatafile::IDBPolicy::useHdfs()) + { + if (error == NO_ERROR) + { + std::string eMsg; + ConfirmHdfsDbFile confirmHdfs; + error = confirmHdfs.confirmDbFileListFromMetaFile(tblOid, eMsg); + + if (error != NO_ERROR) + { + ostringstream ossErr; + ossErr << "Error confirming changes to table " << tableName << "; " << eMsg; + err = ossErr.str(); + rc = 1; + } + else // Perform extra cleanup that is necessary for HDFS + { + std::string eMsg; + ConfirmHdfsDbFile confirmHdfs; + int confirmRc2 = confirmHdfs.endDbFileListFromMetaFile(tblOid, true, eMsg); + + if (confirmRc2 != NO_ERROR) + { + // Might want to log this error, but don't think we need + // to report as fatal, since all changes were confirmed. + } + + // flush PrimProc FD cache + TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tblOid); + ColsExtsInfoMap colsExtsInfoMap = aTableMetaData->getColsExtsInfoMap(); + ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); + ColExtsInfo::iterator aIt; + std::vector files; + BRM::FileInfo aFile; + + while (it != colsExtsInfoMap.end()) + { + aIt = (it->second).begin(); + aFile.oid = it->first; + + while (aIt != (it->second).end()) + { + aFile.partitionNum = aIt->partNum; + aFile.dbRoot = aIt->dbRoot; + aFile.segmentNum = aIt->segNum; + aFile.compType = aIt->compType; + files.push_back(aFile); + aIt++; + } + + it++; + } + + if (files.size() > 0) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + + cacheutils::flushOIDsFromCache(oidsToFlush); + fDbrm.invalidateUncommittedExtentLBIDs(0, false, &lbidList); + + try + { + BulkRollbackMgr::deleteMetaFile(tblOid); + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } + } + } // (error == NO_ERROR) through call to flushChunks() + + if (error != NO_ERROR) // rollback + { + string applName("SingleInsert"); + fWEWrapper.bulkRollback(tblOid, txnid.id, tableName.toString(), applName, false, err); + BulkRollbackMgr::deleteMetaFile(tblOid); + } + } // extra hdfs steps + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + TableMetaData::removeTableMetaData(tblOid); + + if ((rc == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) || isWarningSet) + { + if (rc == NO_ERROR) + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + + Message::Args args; + string cols = "'" + colNames[0] + "'"; + + for (unsigned i = 1; i < colNames.size(); i++) + { + cols = cols + ", " + "'" + colNames[i] + "'"; } - // Confirm HDFS DB file changes "only" if no error up to this point - if (idbdatafile::IDBPolicy::useHdfs()) + args.add(cols); + err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); + + // Strict mode enabled, so rollback on warning + if (insertPkg.get_isWarnToError()) { - if (error == NO_ERROR) - { - std::string eMsg; - ConfirmHdfsDbFile confirmHdfs; - error = confirmHdfs.confirmDbFileListFromMetaFile(tblOid, eMsg); - - if (error != NO_ERROR) - { - ostringstream ossErr; - ossErr << "Error confirming changes to table " << - tableName << "; " << eMsg; - err = ossErr.str(); - rc = 1; - } - else // Perform extra cleanup that is necessary for HDFS - { - std::string eMsg; - ConfirmHdfsDbFile confirmHdfs; - int confirmRc2 = confirmHdfs.endDbFileListFromMetaFile( - tblOid, true, eMsg); - - if (confirmRc2 != NO_ERROR) - { - // Might want to log this error, but don't think we need - // to report as fatal, since all changes were confirmed. - } - - //flush PrimProc FD cache - TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tblOid); - ColsExtsInfoMap colsExtsInfoMap = aTableMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - std::vector files; - BRM::FileInfo aFile; - - while (it != colsExtsInfoMap.end()) - { - aIt = (it->second).begin(); - aFile.oid = it->first; - - while (aIt != (it->second).end()) - { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot = aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - aIt++; - } - - it++; - } - - if (files.size() > 0) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - - cacheutils::flushOIDsFromCache(oidsToFlush); - fDbrm.invalidateUncommittedExtentLBIDs(0, false, &lbidList); - - try - { - BulkRollbackMgr::deleteMetaFile( tblOid ); - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } - } - } // (error == NO_ERROR) through call to flushChunks() - - if (error != NO_ERROR) // rollback - { - string applName ("SingleInsert"); - fWEWrapper.bulkRollback(tblOid, txnid.id, tableName.toString(), - applName, false, err); - BulkRollbackMgr::deleteMetaFile( tblOid ); - } - } // extra hdfs steps - - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - TableMetaData::removeTableMetaData(tblOid); - - if ((rc == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) || isWarningSet) - { - if (rc == NO_ERROR) - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - - Message::Args args; - string cols = "'" + colNames[0] + "'"; - - for (unsigned i = 1; i < colNames.size(); i++) - { - cols = cols + ", " + "'" + colNames[i] + "'"; - } - - args.add(cols); - err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); - - // Strict mode enabled, so rollback on warning - if (insertPkg.get_isWarnToError()) - { - string applName ("SingleInsert"); - fWEWrapper.bulkRollback(tblOid, txnid.id, tableName.toString(), - applName, false, err); - BulkRollbackMgr::deleteMetaFile( tblOid ); - } + string applName("SingleInsert"); + fWEWrapper.bulkRollback(tblOid, txnid.id, tableName.toString(), applName, false, err); + BulkRollbackMgr::deleteMetaFile(tblOid); } + } - // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); + // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); - return rc; + return rc; } uint8_t WE_DMLCommandProc::commitVersion(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tmp32; - int txnID; + int rc = 0; + uint32_t tmp32; + int txnID; - bs >> tmp32; - txnID = tmp32; - //cout << "processing commit txnid = " << txnID << endl; - rc = fWEWrapper.commit(txnID); + bs >> tmp32; + txnID = tmp32; + // cout << "processing commit txnid = " << txnID << endl; + rc = fWEWrapper.commit(txnID); - if (rc != 0) - { - WErrorCodes ec; - ostringstream oss; - oss << "WE: Error commiting transaction; " << txnID << ec.errorString(rc) << endl; - err = oss.str(); - } + if (rc != 0) + { + WErrorCodes ec; + ostringstream oss; + oss << "WE: Error commiting transaction; " << txnID << ec.errorString(rc) << endl; + err = oss.str(); + } - return rc; + return rc; } uint8_t WE_DMLCommandProc::rollbackBlocks(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32;; - int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; + int rc = 0; + uint32_t sessionID, tmp32; + ; + int txnID; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; - //cout << "processing rollbackBlocks txnid = " << txnID << endl; - try - { - rc = fWEWrapper.rollbackBlocks(txnID, sessionID); - } - catch (std::exception& ex) - { - rc = 1; - err = ex.what(); - } + // cout << "processing rollbackBlocks txnid = " << txnID << endl; + try + { + rc = fWEWrapper.rollbackBlocks(txnID, sessionID); + } + catch (std::exception& ex) + { + rc = 1; + err = ex.what(); + } - return rc; + return rc; } uint8_t WE_DMLCommandProc::rollbackVersion(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t sessionID, tmp32; - int txnID; - bs >> sessionID; - bs >> tmp32; - txnID = tmp32; - //cout << "processing rollbackVersion txnid = " << txnID << endl; - rc = fWEWrapper.rollbackVersion(txnID, sessionID); + int rc = 0; + uint32_t sessionID, tmp32; + int txnID; + bs >> sessionID; + bs >> tmp32; + txnID = tmp32; + // cout << "processing rollbackVersion txnid = " << txnID << endl; + rc = fWEWrapper.rollbackVersion(txnID, sessionID); + + if (rc != 0) + { + WErrorCodes ec; + ostringstream oss; + oss << "WE: Error rolling back transaction " << txnID << " for session " << sessionID << "; " + << ec.errorString(rc) << endl; + err = oss.str(); + } + + return rc; +} + +uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId) +{ + int rc = 0; + + InsertDMLPackage insertPkg; + ByteStream::quadbyte tmp32; + bs >> tmp32; + bs >> PMId; + insertPkg.read(bs); + uint32_t sessionId = insertPkg.get_SessionID(); + DMLTable* tablePtr = insertPkg.get_Table(); + bool isAutocommitOn = insertPkg.get_isAutocommitOn(); + + if (idbdatafile::IDBPolicy::useHdfs()) + isAutocommitOn = true; + + BRM::TxnID txnid; + txnid.id = tmp32; + txnid.valid = true; + RowList rows = tablePtr->get_RowList(); + bool isInsertSelect = insertPkg.get_isInsertSelect(); + + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::ColValueList colValuesList; + WriteEngine::DictStrList dicStringList; + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::TableColName tableColName; + tableName.table = tableColName.table = tablePtr->get_TableName(); + tableName.schema = tableColName.schema = tablePtr->get_SchemaName(); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair roPair; + CalpontSystemCatalog::RIDList ridList; + CalpontSystemCatalog::DictOIDList dictOids; + + try + { + ridList = systemCatalogPtr->columnRIDs(tableName, true); + roPair = systemCatalogPtr->tableRID(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + std::vector dctnryStoreOids(ridList.size()); + std::vector columns; + DctnryStructList dctnryList; + std::vector dbRootHWMInfoColVec(ridList.size()); + + uint32_t tblOid = roPair.objnum; + CalpontSystemCatalog::ColType colType; + std::vector colDBRootExtentInfo; + bool bFirstExtentOnThisPM = false; + Convertor convertor; + + if (fIsFirstBatchPm) + { + dbRootExtTrackerVec.clear(); + + if (isAutocommitOn || ((fRBMetaWriter.get() == NULL) && (!isAutocommitOn))) + fRBMetaWriter.reset(new RBMetaWriter("BatchInsert", NULL)); + + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + fWEWrapper.setTransId(txnid.id); + + try + { + // First gather HWM BRM information for all columns + std::vector colWidths; + + for (unsigned i = 0; i < ridList.size(); i++) + { + rc = BRMWrapper::getInstance()->getDbRootHWMInfo(ridList[i].objnum, dbRootHWMInfoColVec[i]); + // need handle error + + CalpontSystemCatalog::ColType colType2 = systemCatalogPtr->colType(ridList[i].objnum); + colWidths.push_back(convertor.getCorrectRowWidth(colType2.colDataType, colType2.colWidth)); + } + + for (unsigned i = 0; i < ridList.size(); i++) + { + // Find DBRoot/segment file where we want to start adding rows + colType = systemCatalogPtr->colType(ridList[i].objnum); + boost::shared_ptr pDBRootExtentTracker( + new DBRootExtentTracker(ridList[i].objnum, colWidths, dbRootHWMInfoColVec, i, 0)); + dbRootExtTrackerVec.push_back(pDBRootExtentTracker); + DBRootExtentInfo dbRootExtent; + std::string trkErrMsg; + bool bEmptyPM; + + if (i == 0) + { + rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, + trkErrMsg); + } + else + pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent); + + colDBRootExtentInfo.push_back(dbRootExtent); + + Column aColumn; + aColumn.colWidth = convertor.getCorrectRowWidth(colType.colDataType, colType.colWidth); + aColumn.colDataType = colType.colDataType; + aColumn.compressionType = colType.compressionType; + aColumn.dataFile.oid = ridList[i].objnum; + aColumn.dataFile.fPartition = dbRootExtent.fPartition; + aColumn.dataFile.fSegment = dbRootExtent.fSegment; + aColumn.dataFile.fDbRoot = dbRootExtent.fDbRoot; + aColumn.dataFile.hwm = dbRootExtent.fLocalHwm; + columns.push_back(aColumn); + + if ((colType.compressionType > 0) && (colType.ddn.dictOID > 0)) + { + DctnryStruct aDctnry; + aDctnry.dctnryOid = colType.ddn.dictOID; + aDctnry.fColPartition = dbRootExtent.fPartition; + aDctnry.fColSegment = dbRootExtent.fSegment; + aDctnry.fColDbRoot = dbRootExtent.fDbRoot; + dctnryList.push_back(aDctnry); + } + + if (colType.ddn.dictOID > 0) + { + dctnryStoreOids[i] = colType.ddn.dictOID; + } + else + { + dctnryStoreOids[i] = 0; + } + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + //@Bug 5996 validate hwm before starts + rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Starting"); if (rc != 0) { + WErrorCodes ec; + err = ec.errorString(rc); + err += " Check err.log for detailed information."; + fIsFirstBatchPm = false; + rc = 1; + return rc; + } + } + + std::vector rangeList; + + // use of MetaFile for bulk rollback support + if (fIsFirstBatchPm && isAutocommitOn) + { + // save meta data, version last block for each dbroot at the start of batch insert + try + { + fRBMetaWriter->init(tblOid, tableName.table); + fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); + + if (!bFirstExtentOnThisPM) + { + for (unsigned i = 0; i < dctnryList.size(); i++) // back up chunks for compressed dictionary + { + // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context + fRBMetaWriter->backupDctnryHWMChunk(dctnryList[i].dctnryOid, dctnryList[i].fColDbRoot, + dctnryList[i].fColPartition, dctnryList[i].fColSegment); + } + } + } + catch (WeException& ex) // catch exception to close file, then rethrow + { + rc = 1; + err = ex.what(); + } + + // Do versioning. Currently, we only version columns, not strings. If there is a design change, this will + // need to be re-visited + if (rc != 0) + return rc; + } + + std::vector colNames; + bool isWarningSet = false; + + if (rows.size()) + { + Row* rowPtr = rows.at(0); + ColumnList columns = rowPtr->get_ColumnList(); + ColumnList::const_iterator column_iterator = columns.begin(); + + try + { + while (column_iterator != columns.end()) + { + DMLColumn* columnPtr = *column_iterator; + tableColName.column = columnPtr->get_Name(); + CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName); + + CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); + + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(oid); + + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + colStruct.dataOid = roPair.objnum; + colStruct.tokenFlag = false; + colStruct.fCompressionType = colType.compressionType; + + // Token + if (isDictCol(colType)) + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = colType.colWidth; + } + + colStruct.colDataType = colType.colDataType; + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(colType); + + ++column_iterator; + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + unsigned int numcols = rowPtr->get_NumberOfColumns(); + std::string tmpStr(""); + + try + { + for (unsigned int i = 0; i < numcols; i++) + { + WriteEngine::ColTupleList colTuples; + WriteEngine::DctColTupleList dctColTuples; + RowList::const_iterator row_iterator = rows.begin(); + bool pushWarning = false; + + while (row_iterator != rows.end()) + { + Row* rowPtr = *row_iterator; + const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i); + + tableColName.column = columnPtr->get_Name(); + CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); + + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(oid); + + boost::any datavalue; + bool isNULL = false; + std::vector origVals; + origVals = columnPtr->get_DataVector(); + WriteEngine::dictStr dicStrings; + + // token + if (isDictCol(colType)) + { + for (uint32_t i = 0; i < origVals.size(); i++) + { + tmpStr = origVals[i]; + + if (tmpStr.length() == 0) + isNULL = true; + else + isNULL = false; + + if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (isNULL && colType.defaultValue.empty()) // error out + { + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + rc = 1; + return rc; + } + else if (isNULL && !(colType.defaultValue.empty())) + { + tmpStr = colType.defaultValue; + } + } + + if (tmpStr.length() > (unsigned int)colType.colWidth) + { + tmpStr = tmpStr.substr(0, colType.colWidth); + + if (!pushWarning) + pushWarning = true; + } + + WriteEngine::ColTuple colTuple; + colTuple.data = datavalue; + + colTuples.push_back(colTuple); + //@Bug 2515. Only pass string values to write engine + dicStrings.push_back(tmpStr); + } + + colValuesList.push_back(colTuples); + //@Bug 2515. Only pass string values to write engine + dicStringList.push_back(dicStrings); + } + else + { + string x; + std::string indata; + // scan once to check how many autoincrement value needed + uint32_t nextValNeeded = 0; + uint64_t nextVal = 1; + + if (colType.autoincrement) + { + try + { + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + for (uint32_t i = 0; i < origVals.size(); i++) + { + indata = origVals[i]; + + if (indata.length() == 0) + isNULL = true; + else + isNULL = false; + + if (isNULL || (indata.compare("0") == 0)) + nextValNeeded++; + } + } + + if (nextValNeeded > 0) // reserve next value + { + try + { + bool reserved = fDbrm.getAIRange(oid, nextValNeeded, &nextVal); + + if (!reserved) + { + err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); + rc = 1; + return rc; + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + + for (uint32_t i = 0; i < origVals.size(); i++) + { + indata = origVals[i]; + + if (indata.length() == 0) + isNULL = true; + else + isNULL = false; + + // check if autoincrement column and value is 0 or null + if (colType.autoincrement && (isNULL || (indata.compare("0") == 0))) + { + ostringstream oss; + oss << nextVal++; + indata = oss.str(); + isNULL = false; + } + + if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (isNULL && colType.defaultValue.empty()) // error out + { + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + rc = 1; + return rc; + } + else if (isNULL && !(colType.defaultValue.empty())) + { + indata = colType.defaultValue; + isNULL = false; + } + } + + try + { + datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, + false, false); + } + catch (exception&) + { + rc = 1; + Message::Args args; + args.add(string("'") + indata + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + } + + //@Bug 1806 + if (rc != NO_ERROR && rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + { + return rc; + } + + if (pushWarning && (rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + + WriteEngine::ColTuple colTuple; + colTuple.data = datavalue; + + colTuples.push_back(colTuple); + //@Bug 2515. Only pass string values to write engine + dicStrings.push_back(tmpStr); + } + + colValuesList.push_back(colTuples); + dicStringList.push_back(dicStrings); + } + + ++row_iterator; + } + + if (pushWarning) + { + colNames.push_back(tableColName.column); + isWarningSet = true; + } + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + + // call the write engine to write the rows + int error = NO_ERROR; + + if (colValuesList.size() > 0) + { + if (colValuesList[0].size() > 0) + { + /* Begin-Disable use of MetaFile for bulk rollback support; + Use alternate call below that passes 0 ptr for RBMetaWriter + if (NO_ERROR != + (error = fWEWrapper.insertColumnRecs(txnid.id, colStructs, colValuesList, + dctnryStructList, dicStringList, dbRootExtTrackerVec, fRBMetaWriter.get(), bFirstExtentOnThisPM, + isInsertSelect, 0, roPair.objnum, fIsFirstBatchPm))) End-Disable use of MetaFile for bulk rollback + support + */ + + if (NO_ERROR != (error = fWEWrapper.insertColumnRecs( + txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, + dicStringList, dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, + isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) + { + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; + WErrorCodes ec; + err = ec.errorString(error); + } + else if (error == ERR_BRM_VB_OVERFLOW) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + else + { + rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + WErrorCodes ec; + err = ec.errorString(error); + } + } + } + } + + if (fIsFirstBatchPm && isAutocommitOn) + { + // fWEWrapper.writeVBEnd(txnid.id, rangeList); + fIsFirstBatchPm = false; + } + else if (fIsFirstBatchPm) + { + fIsFirstBatchPm = false; + } + + if (isWarningSet && (rc == NO_ERROR)) + { + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + Message::Args args; + string cols = "'" + colNames[0] + "'"; + + for (unsigned i = 1; i < colNames.size(); i++) + { + cols = cols + ", " + "'" + colNames[i] + "'"; + } + + args.add(cols); + err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); + + // Strict mode enabled, so rollback on warning + if (insertPkg.get_isWarnToError()) + { + string applName("BatchInsert"); + fWEWrapper.bulkRollback(tblOid, txnid.id, tableName.toString(), applName, false, err); + BulkRollbackMgr::deleteMetaFile(tblOid); + } + } + + // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); + return rc; +} + +uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId) +{ + int rc = 0; + // cout << "processBatchInsert received bytestream length " << bs.length() << endl; + + ByteStream::quadbyte tmp32; + ByteStream::byte tmp8; + bs >> tmp32; + // cout << "processBatchInsert got transaction id " << tmp32 << endl; + bs >> PMId; + // cout << "processBatchInsert gor PMId " << PMId << endl; + uint32_t sessionId; + bs >> sessionId; + // cout << " processBatchInsert for session " << sessionId << endl; + bool isAutocommitOn; + bs >> tmp8; + isAutocommitOn = tmp8; + + if (idbdatafile::IDBPolicy::useHdfs()) + isAutocommitOn = true; + + // cout << "This session isAutocommitOn is " << isAutocommitOn << endl; + BRM::TxnID txnid; + txnid.id = tmp32; + txnid.valid = true; + bool isInsertSelect; + bs >> tmp8; + // For insert select, skip the hwm block and start inserting from the next block + // to avoid self insert issue. + // For batch insert: if not first batch, use the saved last rid to start adding rows. + isInsertSelect = tmp8; + + WriteEngine::ColStructList colStructs; + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryValueList dctnryValueList; + std::vector colValuesList; + WriteEngine::DictStrList dicStringList; + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::TableColName tableColName; + bs >> tableColName.table; + bs >> tableColName.schema; + tableName.table = tableColName.table; + tableName.schema = tableColName.schema; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair roPair; + CalpontSystemCatalog::RIDList ridList; + CalpontSystemCatalog::DictOIDList dictOids; + + try + { + ridList = systemCatalogPtr->columnRIDs(tableName, true); + roPair = systemCatalogPtr->tableRID(tableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + std::vector dctnryStoreOids(ridList.size()); + std::vector columns; + DctnryStructList dctnryList; + std::vector dbRootHWMInfoColVec(ridList.size()); + + uint32_t tblOid = roPair.objnum; + CalpontSystemCatalog::ColType colType; + std::vector colDBRootExtentInfo; + bool bFirstExtentOnThisPM = false; + Convertor convertor; + + if (fIsFirstBatchPm) + { + dbRootExtTrackerVec.clear(); + + if (isAutocommitOn || ((fRBMetaWriter.get() == NULL) && (!isAutocommitOn))) + fRBMetaWriter.reset(new RBMetaWriter("BatchInsert", NULL)); + + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + fWEWrapper.setTransId(txnid.id); + + try + { + // First gather HWM BRM information for all columns + std::vector colWidths; + + for (unsigned i = 0; i < ridList.size(); i++) + { + rc = BRMWrapper::getInstance()->getDbRootHWMInfo(ridList[i].objnum, dbRootHWMInfoColVec[i]); + // need handle error + + CalpontSystemCatalog::ColType colType2 = systemCatalogPtr->colType(ridList[i].objnum); + colWidths.push_back(convertor.getCorrectRowWidth(colType2.colDataType, colType2.colWidth)); + } + + for (unsigned i = 0; i < ridList.size(); i++) + { + // Find DBRoot/segment file where we want to start adding rows + colType = systemCatalogPtr->colType(ridList[i].objnum); + boost::shared_ptr pDBRootExtentTracker( + new DBRootExtentTracker(ridList[i].objnum, colWidths, dbRootHWMInfoColVec, i, 0)); + dbRootExtTrackerVec.push_back(pDBRootExtentTracker); + DBRootExtentInfo dbRootExtent; + std::string trkErrMsg; + bool bEmptyPM; + + if (i == 0) + { + rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, + trkErrMsg); + /* cout << "bEmptyPM = " << (int) bEmptyPM << " bFirstExtentOnThisPM= " << + (int)bFirstExtentOnThisPM << " oid:dbroot:hwm = " << ridList[i].objnum << + ":"< rangeList; + + // use of MetaFile for bulk rollback support + if (fIsFirstBatchPm && isAutocommitOn) + { + // save meta data, version last block for each dbroot at the start of batch insert + try + { + fRBMetaWriter->init(tblOid, tableName.table); + fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); + + // cout << "Saved meta files" << endl; + if (!bFirstExtentOnThisPM) + { + // cout << "Backing up hwm chunks" << endl; + for (unsigned i = 0; i < dctnryList.size(); i++) // back up chunks for compressed dictionary + { + // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context + fRBMetaWriter->backupDctnryHWMChunk(dctnryList[i].dctnryOid, dctnryList[i].fColDbRoot, + dctnryList[i].fColPartition, dctnryList[i].fColSegment); + } + } + } + catch (WeException& ex) // catch exception to close file, then rethrow + { + rc = 1; + err = ex.what(); + } + + // Do versioning. Currently, we only version columns, not strings. If there is a design change, this will + // need to be re-visited + if (rc != 0) + return rc; + } + + std::vector colNames; + bool isWarningSet = false; + uint32_t columnCount; + bs >> columnCount; + + if (columnCount) + { + try + { + for (uint32_t current_column = 0; current_column < columnCount; current_column++) + { + uint32_t tmp32; + std::string colName; + bs >> tmp32; + bs >> colName; + colNames.push_back(colName); + CalpontSystemCatalog::OID oid = tmp32; + + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(oid); + + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStruct dctnryStruct; + colStruct.dataOid = oid; + colStruct.tokenFlag = false; + colStruct.fCompressionType = colType.compressionType; + + // Token + if (isDictCol(colType)) + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = colType.colWidth; + } + + colStruct.colDataType = colType.colDataType; + + if (colStruct.tokenFlag) + { + dctnryStruct.dctnryOid = colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + } + else + { + dctnryStruct.dctnryOid = 0; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + } + + colStructs.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + std::string tmpStr(""); + uint32_t valuesPerColumn; + bs >> valuesPerColumn; + colValuesList.reserve(columnCount * valuesPerColumn); + + try + { + bool pushWarning = false; + + for (uint32_t j = 0; j < columnCount; j++) + { + WriteEngine::DctColTupleList dctColTuples; + tableColName.column = colNames[j]; + CalpontSystemCatalog::OID oid = colStructs[j].dataOid; + + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(oid); + + bool isNULL = false; + WriteEngine::dictStr dicStrings; + + // token + if (isDictCol(colType)) + { + for (uint32_t i = 0; i < valuesPerColumn; i++) + { + bs >> tmp8; + isNULL = tmp8; + bs >> tmpStr; + + if (tmpStr.length() == 0) + isNULL = true; + + if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (isNULL && colType.defaultValue.empty()) // error out + { + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + rc = 1; + return rc; + } + else if (isNULL && !(colType.defaultValue.empty())) + { + tmpStr = colType.defaultValue; + } + } + + if (tmpStr.length() > (unsigned int)colType.colWidth) + { + tmpStr = tmpStr.substr(0, colType.colWidth); + + if (!pushWarning) + pushWarning = true; + } + + colValuesList.push_back(0); + //@Bug 2515. Only pass string values to write engine + dicStrings.push_back(tmpStr); + } + + //@Bug 2515. Only pass string values to write engine + dicStringList.push_back(dicStrings); + } + else + { + string x; + // scan once to check how many autoincrement value needed + uint32_t nextValNeeded = 0; + uint64_t nextVal = 1; + + if (colType.autoincrement) + { + try + { + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + + for (uint32_t i = 0; i < valuesPerColumn; i++) + { + bs >> tmp8; + isNULL = tmp8; + + uint8_t val8; + uint16_t val16; + uint32_t val32; + uint64_t val64; + uint64_t colValue; + float valF; + double valD; + std::string valStr; + bool valZero = false; // Needed for autoinc check + + switch (colType.colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::UTINYINT: + bs >> val8; + + if (val8 == 0) + valZero = true; + + colValue = val8; + break; + + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::USMALLINT: + bs >> val16; + + if (val16 == 0) + valZero = true; + + colValue = val16; + break; + + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + bs >> val32; + + if (val32 == 0) + valZero = true; + + colValue = val32; + break; + + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::UBIGINT: + bs >> val64; + + if (val64 == 0) + valZero = true; + + colValue = val64; + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + switch (colType.colWidth) + { + case 1: + { + bs >> val8; + colValue = val8; + break; + } + + case 2: + { + bs >> val16; + colValue = val16; + break; + } + + case 4: + { + bs >> val32; + colValue = val32; + break; + } + + default: + { + bs >> val64; + colValue = val64; + break; + } + } + + break; + + case execplan::CalpontSystemCatalog::UDECIMAL: + + // UDECIMAL numbers may not be negative + if (colType.colWidth == 1) + { + bs >> val8; + + // FIXME: IDK what would it mean if valN are unsigned + if (utils::is_negative(val8) && val8 != joblist::TINYINTEMPTYROW && + val8 != joblist::TINYINTNULL) + { + val8 = 0; + pushWarning = true; + } + + colValue = val8; + } + else if (colType.colWidth == 2) + { + bs >> val16; + + if (utils::is_negative(val16) && val16 != joblist::SMALLINTEMPTYROW && + val16 != joblist::SMALLINTNULL) + { + val16 = 0; + pushWarning = true; + } + + colValue = val16; + } + else if (colType.colWidth == 4) + { + bs >> val32; + + if (utils::is_negative(val32) && val32 != joblist::INTEMPTYROW && val32 != joblist::INTNULL) + { + val32 = 0; + pushWarning = true; + } + + colValue = val32; + } + else if (colType.colWidth == 8) + { + bs >> val64; + + if (utils::is_negative(val64) && val64 != joblist::BIGINTEMPTYROW && + val64 != joblist::BIGINTNULL) + { + val64 = 0; + pushWarning = true; + } + + colValue = val64; + } + + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + bs >> val64; + colValue = val64; + break; + + case execplan::CalpontSystemCatalog::UDOUBLE: + bs >> val64; + memcpy(&valD, &val64, 8); + + if (valD < 0.0 && valD != static_cast(joblist::DOUBLEEMPTYROW) && + valD != static_cast(joblist::DOUBLENULL)) + { + valD = 0.0; + pushWarning = true; + } + + colValue = val64; + break; + + case execplan::CalpontSystemCatalog::FLOAT: + bs >> val32; + colValue = val32; + break; + + case execplan::CalpontSystemCatalog::UFLOAT: + bs >> val32; + memcpy(&valF, &val32, 4); + + if (valF < 0.0 && valF != static_cast(joblist::FLOATEMPTYROW) && + valF != static_cast(joblist::FLOATNULL)) + { + valF = 0.0; + pushWarning = true; + } + + colValue = val32; + break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::BLOB: + bs >> valStr; + + if (valStr.length() > (unsigned int)colType.colWidth) + { + valStr = valStr.substr(0, colType.colWidth); + pushWarning = true; + } + else + { + if ((unsigned int)colType.colWidth > valStr.length()) + { + // Pad null character to the string + valStr.resize(colType.colWidth, 0); + } + } + + // FIXME: colValue is uint64_t (8 bytes) + memcpy(&colValue, valStr.c_str(), valStr.length()); + break; + + default: + rc = 1; + err = IDBErrorInfo::instance()->errorMsg(ERR_DATATYPE_NOT_SUPPORT); + break; + } + + // check if autoincrement column and value is 0 or null + if (colType.autoincrement && (isNULL || valZero)) + { + ostringstream oss; + oss << nextVal++; + isNULL = false; + + try + { + nextValNeeded++; + bool reserved = fDbrm.getAIRange(oid, nextValNeeded, &nextVal); + + if (!reserved) + { + err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); + rc = 1; + return rc; + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + colValue = nextVal; + } + + if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + if (isNULL && colType.defaultValue.empty()) // error out + { + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + rc = 1; + return rc; + } + else if (isNULL && !(colType.defaultValue.empty())) + { + memcpy(&colValue, colType.defaultValue.c_str(), colType.defaultValue.length()); + isNULL = false; + } + } + + //@Bug 1806 + if (rc != NO_ERROR && rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) + { + return rc; + } + + if (pushWarning && (rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING)) + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + + colValuesList.push_back(colValue); + //@Bug 2515. Only pass string values to write engine + dicStrings.push_back(valStr); + } + + dicStringList.push_back(dicStrings); + } + + if (pushWarning) + { + colNames.push_back(tableColName.column); + isWarningSet = true; + } + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + + // call the write engine to write the rows + int error = NO_ERROR; + + // fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); + // cout << "Batch inserting a row with transaction id " << txnid.id << endl; + if (colValuesList.size() > 0) + { + if (NO_ERROR != + (error = fWEWrapper.insertColumnRecsBinary( + txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, dbRootExtTrackerVec, 0, + bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) + { + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; WErrorCodes ec; - ostringstream oss; - oss << "WE: Error rolling back transaction " << txnID << " for session " << sessionID << "; " << ec.errorString(rc) << endl; - err = oss.str(); + err = ec.errorString(error); + } + else if (error == ERR_BRM_VB_OVERFLOW) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + else + { + rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; + WErrorCodes ec; + err = ec.errorString(error); + } + } + } + + if (fIsFirstBatchPm && isAutocommitOn) + { + // fWEWrapper.writeVBEnd(txnid.id, rangeList); + fIsFirstBatchPm = false; + } + else if (fIsFirstBatchPm) + { + fIsFirstBatchPm = false; + } + + if (isWarningSet && (rc == NO_ERROR)) + { + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + // cout << "Got warning" << endl; + Message::Args args; + string cols = "'" + colNames[0] + "'"; + + for (unsigned i = 1; i < colNames.size(); i++) + { + cols = cols + ", " + "'" + colNames[i] + "'"; } - return rc; + args.add(cols); + err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); + } + + // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); + return rc; } -uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId) -{ - int rc = 0; - - InsertDMLPackage insertPkg; - ByteStream::quadbyte tmp32; - bs >> tmp32; - bs >> PMId; - insertPkg.read( bs); - uint32_t sessionId = insertPkg.get_SessionID(); - DMLTable* tablePtr = insertPkg.get_Table(); - bool isAutocommitOn = insertPkg.get_isAutocommitOn(); - - if (idbdatafile::IDBPolicy::useHdfs()) - isAutocommitOn = true; - - BRM::TxnID txnid; - txnid.id = tmp32; - txnid.valid = true; - RowList rows = tablePtr->get_RowList(); - bool isInsertSelect = insertPkg.get_isInsertSelect(); - - WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - WriteEngine::ColValueList colValuesList; - WriteEngine::DictStrList dicStringList ; - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::TableColName tableColName; - tableName.table = tableColName.table = tablePtr->get_TableName(); - tableName.schema = tableColName.schema = tablePtr->get_SchemaName(); - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::ROPair roPair; - CalpontSystemCatalog::RIDList ridList; - CalpontSystemCatalog::DictOIDList dictOids; - - try - { - ridList = systemCatalogPtr->columnRIDs(tableName, true); - roPair = systemCatalogPtr->tableRID(tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - std::vector dctnryStoreOids(ridList.size()) ; - std::vector columns; - DctnryStructList dctnryList; - std::vector dbRootHWMInfoColVec(ridList.size()); - - uint32_t tblOid = roPair.objnum; - CalpontSystemCatalog::ColType colType; - std::vector colDBRootExtentInfo; - bool bFirstExtentOnThisPM = false; - Convertor convertor; - - if ( fIsFirstBatchPm ) - { - dbRootExtTrackerVec.clear(); - - if (isAutocommitOn || ((fRBMetaWriter.get() == NULL) && (!isAutocommitOn))) - fRBMetaWriter.reset(new RBMetaWriter("BatchInsert", NULL)); - - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - fWEWrapper.setTransId(txnid.id); - - try - { - // First gather HWM BRM information for all columns - std::vector colWidths; - - for (unsigned i = 0; i < ridList.size(); i++) - { - rc = BRMWrapper::getInstance()->getDbRootHWMInfo(ridList[i].objnum, dbRootHWMInfoColVec[i]); - //need handle error - - CalpontSystemCatalog::ColType colType2 = systemCatalogPtr->colType(ridList[i].objnum); - colWidths.push_back( convertor.getCorrectRowWidth( - colType2.colDataType, colType2.colWidth) ); - } - - for (unsigned i = 0; i < ridList.size(); i++) - { - // Find DBRoot/segment file where we want to start adding rows - colType = systemCatalogPtr->colType(ridList[i].objnum); - boost::shared_ptr pDBRootExtentTracker (new DBRootExtentTracker(ridList[i].objnum, - colWidths, dbRootHWMInfoColVec, i, 0) ); - dbRootExtTrackerVec.push_back( pDBRootExtentTracker ); - DBRootExtentInfo dbRootExtent; - std::string trkErrMsg; - bool bEmptyPM; - - if (i == 0) - { - rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg); - } - else - pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent); - - colDBRootExtentInfo.push_back(dbRootExtent); - - Column aColumn; - aColumn.colWidth = convertor.getCorrectRowWidth(colType.colDataType, colType.colWidth); - aColumn.colDataType = colType.colDataType; - aColumn.compressionType = colType.compressionType; - aColumn.dataFile.oid = ridList[i].objnum; - aColumn.dataFile.fPartition = dbRootExtent.fPartition; - aColumn.dataFile.fSegment = dbRootExtent.fSegment; - aColumn.dataFile.fDbRoot = dbRootExtent.fDbRoot; - aColumn.dataFile.hwm = dbRootExtent.fLocalHwm; - columns.push_back(aColumn); - - if ((colType.compressionType > 0) && (colType.ddn.dictOID > 0)) - { - DctnryStruct aDctnry; - aDctnry.dctnryOid = colType.ddn.dictOID; - aDctnry.fColPartition = dbRootExtent.fPartition; - aDctnry.fColSegment = dbRootExtent.fSegment; - aDctnry.fColDbRoot = dbRootExtent.fDbRoot; - dctnryList.push_back(aDctnry); - } - - if (colType.ddn.dictOID > 0) - { - dctnryStoreOids[i] = colType.ddn.dictOID; - } - else - { - dctnryStoreOids[i] = 0; - } - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - //@Bug 5996 validate hwm before starts - rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Starting"); - - if ( rc != 0) - { - WErrorCodes ec; - err = ec.errorString(rc); - err += " Check err.log for detailed information."; - fIsFirstBatchPm = false; - rc = 1; - return rc; - } - } - - std::vector rangeList; - - // use of MetaFile for bulk rollback support - if (fIsFirstBatchPm && isAutocommitOn) - { - //save meta data, version last block for each dbroot at the start of batch insert - try - { - fRBMetaWriter->init(tblOid, tableName.table); - fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); - - if (!bFirstExtentOnThisPM) - { - for (unsigned i = 0; i < dctnryList.size(); i++) //back up chunks for compressed dictionary - { - // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context - fRBMetaWriter->backupDctnryHWMChunk( - dctnryList[i].dctnryOid, dctnryList[i].fColDbRoot, dctnryList[i].fColPartition, dctnryList[i].fColSegment); - } - } - } - catch (WeException& ex) // catch exception to close file, then rethrow - { - rc = 1; - err = ex.what(); - } - - //Do versioning. Currently, we only version columns, not strings. If there is a design change, this will need to be re-visited - if ( rc != 0) - return rc; - - } - - std::vector colNames; - bool isWarningSet = false; - - if (rows.size()) - { - Row* rowPtr = rows.at(0); - ColumnList columns = rowPtr->get_ColumnList(); - ColumnList::const_iterator column_iterator = columns.begin(); - - try - { - while (column_iterator != columns.end()) - { - DMLColumn* columnPtr = *column_iterator; - tableColName.column = columnPtr->get_Name(); - CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName); - - CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); - - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType(oid); - - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - colStruct.dataOid = roPair.objnum; - colStruct.tokenFlag = false; - colStruct.fCompressionType = colType.compressionType; - - // Token - if (isDictCol(colType) ) - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = colType.colWidth; - } - - colStruct.colDataType = colType.colDataType; - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - cscColTypeList.push_back(colType); - - ++column_iterator; - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - unsigned int numcols = rowPtr->get_NumberOfColumns(); - std::string tmpStr(""); - - try - { - for (unsigned int i = 0; i < numcols; i++) - { - WriteEngine::ColTupleList colTuples; - WriteEngine::DctColTupleList dctColTuples; - RowList::const_iterator row_iterator = rows.begin(); - bool pushWarning = false; - - while (row_iterator != rows.end()) - { - Row* rowPtr = *row_iterator; - const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i); - - tableColName.column = columnPtr->get_Name(); - CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); - - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType(oid); - - boost::any datavalue; - bool isNULL = false; - std::vector origVals; - origVals = columnPtr->get_DataVector(); - WriteEngine::dictStr dicStrings; - - // token - if ( isDictCol(colType) ) - { - for ( uint32_t i = 0; i < origVals.size(); i++ ) - { - tmpStr = origVals[i]; - - if ( tmpStr.length() == 0 ) - isNULL = true; - else - isNULL = false; - - if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (isNULL && colType.defaultValue.empty()) //error out - { - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - rc = 1; - return rc; - } - else if (isNULL && !(colType.defaultValue.empty())) - { - tmpStr = colType.defaultValue; - } - } - - if ( tmpStr.length() > (unsigned int)colType.colWidth ) - { - tmpStr = tmpStr.substr(0, colType.colWidth); - - if ( !pushWarning ) - pushWarning = true; - } - - WriteEngine::ColTuple colTuple; - colTuple.data = datavalue; - - colTuples.push_back(colTuple); - //@Bug 2515. Only pass string values to write engine - dicStrings.push_back( tmpStr ); - } - - colValuesList.push_back(colTuples); - //@Bug 2515. Only pass string values to write engine - dicStringList.push_back( dicStrings ); - } - else - { - string x; - std::string indata; - //scan once to check how many autoincrement value needed - uint32_t nextValNeeded = 0; - uint64_t nextVal = 1; - - if (colType.autoincrement) - { - try - { - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - for ( uint32_t i = 0; i < origVals.size(); i++ ) - { - indata = origVals[i]; - - if ( indata.length() == 0 ) - isNULL = true; - else - isNULL = false; - - if ( isNULL || (indata.compare("0") == 0)) - nextValNeeded++; - } - } - - if (nextValNeeded > 0) //reserve next value - { - try - { - bool reserved = fDbrm.getAIRange(oid, nextValNeeded, &nextVal); - - if (!reserved) - { - err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); - rc = 1; - return rc; - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - - for ( uint32_t i = 0; i < origVals.size(); i++ ) - { - indata = origVals[i]; - - if ( indata.length() == 0 ) - isNULL = true; - else - isNULL = false; - - //check if autoincrement column and value is 0 or null - if (colType.autoincrement && ( isNULL || (indata.compare("0") == 0))) - { - ostringstream oss; - oss << nextVal++; - indata = oss.str(); - isNULL = false; - } - - if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (isNULL && colType.defaultValue.empty()) //error out - { - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - rc = 1; - return rc; - } - else if (isNULL && !(colType.defaultValue.empty())) - { - indata = colType.defaultValue; - isNULL = false; - } - } - - try - { - datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); - } - catch (exception&) - { - rc = 1; - Message::Args args; - args.add(string("'") + indata + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - } - - //@Bug 1806 - if (rc != NO_ERROR && rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - { - return rc; - } - - if ( pushWarning && ( rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING ) ) - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - - WriteEngine::ColTuple colTuple; - colTuple.data = datavalue; - - colTuples.push_back(colTuple); - //@Bug 2515. Only pass string values to write engine - dicStrings.push_back( tmpStr ); - } - - colValuesList.push_back(colTuples); - dicStringList.push_back( dicStrings ); - } - - ++row_iterator; - } - - if (pushWarning) - { - colNames.push_back(tableColName.column); - isWarningSet = true; - } - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - - // call the write engine to write the rows - int error = NO_ERROR; - - if (colValuesList.size() > 0) - { - if (colValuesList[0].size() > 0) - { - /* Begin-Disable use of MetaFile for bulk rollback support; - Use alternate call below that passes 0 ptr for RBMetaWriter - if (NO_ERROR != - (error = fWEWrapper.insertColumnRecs(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, - dbRootExtTrackerVec, fRBMetaWriter.get(), bFirstExtentOnThisPM, isInsertSelect, 0, roPair.objnum, fIsFirstBatchPm))) - End-Disable use of MetaFile for bulk rollback support - */ - - if (NO_ERROR != - (error = fWEWrapper.insertColumnRecs(txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, dicStringList, - dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) - { - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - else if ( error == ERR_BRM_VB_OVERFLOW ) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } - else - { - rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - } - } - } - - if (fIsFirstBatchPm && isAutocommitOn) - { - //fWEWrapper.writeVBEnd(txnid.id, rangeList); - fIsFirstBatchPm = false; - } - else if (fIsFirstBatchPm) - { - fIsFirstBatchPm = false; - } - - if ( isWarningSet && ( rc == NO_ERROR ) ) - { - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - Message::Args args; - string cols = "'" + colNames[0] + "'"; - - for (unsigned i = 1; i < colNames.size(); i++) - { - cols = cols + ", " + "'" + colNames[i] + "'"; - } - - args.add(cols); - err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); - - // Strict mode enabled, so rollback on warning - if (insertPkg.get_isWarnToError()) - { - string applName ("BatchInsert"); - fWEWrapper.bulkRollback(tblOid, txnid.id, tableName.toString(), - applName, false, err); - BulkRollbackMgr::deleteMetaFile( tblOid ); - } - } - - // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); - return rc; -} - -uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId) -{ - int rc = 0; - //cout << "processBatchInsert received bytestream length " << bs.length() << endl; - - ByteStream::quadbyte tmp32; - ByteStream::byte tmp8; - bs >> tmp32; - //cout << "processBatchInsert got transaction id " << tmp32 << endl; - bs >> PMId; - //cout << "processBatchInsert gor PMId " << PMId << endl; - uint32_t sessionId; - bs >> sessionId; - //cout << " processBatchInsert for session " << sessionId << endl; - bool isAutocommitOn; - bs >> tmp8; - isAutocommitOn = tmp8; - - if (idbdatafile::IDBPolicy::useHdfs()) - isAutocommitOn = true; - - //cout << "This session isAutocommitOn is " << isAutocommitOn << endl; - BRM::TxnID txnid; - txnid.id = tmp32; - txnid.valid = true; - bool isInsertSelect; - bs >> tmp8; - // For insert select, skip the hwm block and start inserting from the next block - // to avoid self insert issue. - //For batch insert: if not first batch, use the saved last rid to start adding rows. - isInsertSelect = tmp8; - - WriteEngine::ColStructList colStructs; - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryValueList dctnryValueList; - std::vector colValuesList; - WriteEngine::DictStrList dicStringList ; - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::TableColName tableColName; - bs >> tableColName.table; - bs >> tableColName.schema; - tableName.table = tableColName.table; - tableName.schema = tableColName.schema; - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::ROPair roPair; - CalpontSystemCatalog::RIDList ridList; - CalpontSystemCatalog::DictOIDList dictOids; - - try - { - ridList = systemCatalogPtr->columnRIDs(tableName, true); - roPair = systemCatalogPtr->tableRID( tableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - - std::vector dctnryStoreOids(ridList.size()) ; - std::vector columns; - DctnryStructList dctnryList; - std::vector dbRootHWMInfoColVec(ridList.size()); - - uint32_t tblOid = roPair.objnum; - CalpontSystemCatalog::ColType colType; - std::vector colDBRootExtentInfo; - bool bFirstExtentOnThisPM = false; - Convertor convertor; - - if ( fIsFirstBatchPm ) - { - dbRootExtTrackerVec.clear(); - - if (isAutocommitOn || ((fRBMetaWriter.get() == NULL) && (!isAutocommitOn))) - fRBMetaWriter.reset(new RBMetaWriter("BatchInsert", NULL)); - - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - fWEWrapper.setTransId(txnid.id); - - try - { - // First gather HWM BRM information for all columns - std::vector colWidths; - - for (unsigned i = 0; i < ridList.size(); i++) - { - rc = BRMWrapper::getInstance()->getDbRootHWMInfo(ridList[i].objnum, dbRootHWMInfoColVec[i]); - //need handle error - - CalpontSystemCatalog::ColType colType2 = systemCatalogPtr->colType(ridList[i].objnum); - colWidths.push_back( convertor.getCorrectRowWidth( - colType2.colDataType, colType2.colWidth) ); - } - - for (unsigned i = 0; i < ridList.size(); i++) - { - // Find DBRoot/segment file where we want to start adding rows - colType = systemCatalogPtr->colType(ridList[i].objnum); - boost::shared_ptr pDBRootExtentTracker (new DBRootExtentTracker(ridList[i].objnum, - colWidths, dbRootHWMInfoColVec, i, 0) ); - dbRootExtTrackerVec.push_back( pDBRootExtentTracker ); - DBRootExtentInfo dbRootExtent; - std::string trkErrMsg; - bool bEmptyPM; - - if (i == 0) - { - rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg); - /* cout << "bEmptyPM = " << (int) bEmptyPM << " bFirstExtentOnThisPM= " << (int)bFirstExtentOnThisPM << - " oid:dbroot:hwm = " << ridList[i].objnum << ":"< rangeList; - - // use of MetaFile for bulk rollback support - if ( fIsFirstBatchPm && isAutocommitOn) - { - //save meta data, version last block for each dbroot at the start of batch insert - try - { - fRBMetaWriter->init(tblOid, tableName.table); - fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); - - //cout << "Saved meta files" << endl; - if (!bFirstExtentOnThisPM) - { - //cout << "Backing up hwm chunks" << endl; - for (unsigned i = 0; i < dctnryList.size(); i++) //back up chunks for compressed dictionary - { - // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context - fRBMetaWriter->backupDctnryHWMChunk( - dctnryList[i].dctnryOid, dctnryList[i].fColDbRoot, dctnryList[i].fColPartition, dctnryList[i].fColSegment); - } - } - } - catch (WeException& ex) // catch exception to close file, then rethrow - { - rc = 1; - err = ex.what(); - } - - //Do versioning. Currently, we only version columns, not strings. If there is a design change, this will need to be re-visited - if ( rc != 0) - return rc; - - } - - std::vector colNames; - bool isWarningSet = false; - uint32_t columnCount; - bs >> columnCount; - - if (columnCount) - { - try - { - for (uint32_t current_column = 0; current_column < columnCount; current_column++) - { - uint32_t tmp32; - std::string colName; - bs >> tmp32; - bs >> colName; - colNames.push_back(colName); - CalpontSystemCatalog::OID oid = tmp32; - - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType(oid); - - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStruct dctnryStruct; - colStruct.dataOid = oid; - colStruct.tokenFlag = false; - colStruct.fCompressionType = colType.compressionType; - - // Token - if ( isDictCol(colType) ) - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - } - else - { - colStruct.colWidth = colType.colWidth; - } - - colStruct.colDataType = colType.colDataType; - - if (colStruct.tokenFlag) - { - dctnryStruct.dctnryOid = colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - } - else - { - dctnryStruct.dctnryOid = 0; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - } - - colStructs.push_back(colStruct); - dctnryStructList.push_back(dctnryStruct); - - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - std::string tmpStr(""); - uint32_t valuesPerColumn; - bs >> valuesPerColumn; - colValuesList.reserve(columnCount * valuesPerColumn); - - try - { - bool pushWarning = false; - - for (uint32_t j = 0; j < columnCount; j++) - { - WriteEngine::DctColTupleList dctColTuples; - tableColName.column = colNames[j]; - CalpontSystemCatalog::OID oid = colStructs[j].dataOid; - - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType(oid); - - bool isNULL = false; - WriteEngine::dictStr dicStrings; - - // token - if ( isDictCol(colType) ) - { - for ( uint32_t i = 0; i < valuesPerColumn; i++ ) - { - bs >> tmp8; - isNULL = tmp8; - bs >> tmpStr; - - if ( tmpStr.length() == 0 ) - isNULL = true; - - if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (isNULL && colType.defaultValue.empty()) //error out - { - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - rc = 1; - return rc; - } - else if (isNULL && !(colType.defaultValue.empty())) - { - tmpStr = colType.defaultValue; - } - } - - if ( tmpStr.length() > (unsigned int)colType.colWidth ) - { - tmpStr = tmpStr.substr(0, colType.colWidth); - - if ( !pushWarning ) - pushWarning = true; - } - - colValuesList.push_back(0); - //@Bug 2515. Only pass string values to write engine - dicStrings.push_back( tmpStr ); - } - - //@Bug 2515. Only pass string values to write engine - dicStringList.push_back( dicStrings ); - } - else - { - string x; - //scan once to check how many autoincrement value needed - uint32_t nextValNeeded = 0; - uint64_t nextVal = 1; - - if (colType.autoincrement) - { - try - { - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - - for ( uint32_t i = 0; i < valuesPerColumn; i++ ) - { - bs >> tmp8; - isNULL = tmp8; - - uint8_t val8; - uint16_t val16; - uint32_t val32; - uint64_t val64; - uint64_t colValue; - float valF; - double valD; - std::string valStr; - bool valZero = false; // Needed for autoinc check - - switch (colType.colDataType) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::UTINYINT: - bs >> val8; - - if (val8 == 0) - valZero = true; - - colValue = val8; - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::USMALLINT: - bs >> val16; - - if (val16 == 0) - valZero = true; - - colValue = val16; - break; - - case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - bs >> val32; - - if (val32 == 0) - valZero = true; - - colValue = val32; - break; - - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: - case execplan::CalpontSystemCatalog::TIMESTAMP: - case execplan::CalpontSystemCatalog::UBIGINT: - bs >> val64; - - if (val64 == 0) - valZero = true; - - colValue = val64; - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - switch (colType.colWidth) - { - case 1: - { - bs >> val8; - colValue = val8; - break; - } - - case 2: - { - bs >> val16; - colValue = val16; - break; - } - - case 4: - { - bs >> val32; - colValue = val32; - break; - } - - default: - { - bs >> val64; - colValue = val64; - break; - } - } - - break; - - case execplan::CalpontSystemCatalog::UDECIMAL: - - // UDECIMAL numbers may not be negative - if (colType.colWidth == 1) - { - bs >> val8; - - // FIXME: IDK what would it mean if valN are unsigned - if (utils::is_negative(val8) && - val8 != joblist::TINYINTEMPTYROW && - val8 != joblist::TINYINTNULL) - { - val8 = 0; - pushWarning = true; - } - - colValue = val8; - } - else if (colType.colWidth == 2) - { - bs >> val16; - - if (utils::is_negative(val16) && - val16 != joblist::SMALLINTEMPTYROW && - val16 != joblist::SMALLINTNULL) - { - val16 = 0; - pushWarning = true; - } - - colValue = val16; - } - else if (colType.colWidth == 4) - { - bs >> val32; - - if (utils::is_negative(val32) && - val32 != joblist::INTEMPTYROW && - val32 != joblist::INTNULL) - { - val32 = 0; - pushWarning = true; - } - - colValue = val32; - } - else if (colType.colWidth == 8) - { - bs >> val64; - - if (utils::is_negative(val64) && - val64 != joblist::BIGINTEMPTYROW && - val64 != joblist::BIGINTNULL) - { - val64 = 0; - pushWarning = true; - } - - colValue = val64; - } - - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - bs >> val64; - colValue = val64; - break; - - case execplan::CalpontSystemCatalog::UDOUBLE: - bs >> val64; - memcpy(&valD, &val64, 8); - - if (valD < 0.0 && valD != static_cast(joblist::DOUBLEEMPTYROW) - && valD != static_cast(joblist::DOUBLENULL)) - { - valD = 0.0; - pushWarning = true; - } - - colValue = val64; - break; - - case execplan::CalpontSystemCatalog::FLOAT: - bs >> val32; - colValue = val32; - break; - - case execplan::CalpontSystemCatalog::UFLOAT: - bs >> val32; - memcpy(&valF, &val32, 4); - - if (valF < 0.0 && valF != static_cast(joblist::FLOATEMPTYROW) - && valF != static_cast(joblist::FLOATNULL)) - { - valF = 0.0; - pushWarning = true; - } - - colValue = val32; - break; - - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::BLOB: - bs >> valStr; - - if (valStr.length() > (unsigned int)colType.colWidth) - { - valStr = valStr.substr(0, colType.colWidth); - pushWarning = true; - } - else - { - if ( (unsigned int)colType.colWidth > valStr.length()) - { - //Pad null character to the string - valStr.resize(colType.colWidth, 0); - } - } - - // FIXME: colValue is uint64_t (8 bytes) - memcpy(&colValue, valStr.c_str(), valStr.length()); - break; - - default: - rc = 1; - err = IDBErrorInfo::instance()->errorMsg(ERR_DATATYPE_NOT_SUPPORT); - break; - } - - //check if autoincrement column and value is 0 or null - if (colType.autoincrement && ( isNULL || valZero)) - { - ostringstream oss; - oss << nextVal++; - isNULL = false; - - try - { - nextValNeeded++; - bool reserved = fDbrm.getAIRange(oid, nextValNeeded, &nextVal); - - if (!reserved) - { - err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); - rc = 1; - return rc; - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - colValue = nextVal; - } - - if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - if (isNULL && colType.defaultValue.empty()) //error out - { - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - rc = 1; - return rc; - } - else if (isNULL && !(colType.defaultValue.empty())) - { - memcpy(&colValue, colType.defaultValue.c_str(), colType.defaultValue.length()); - isNULL = false; - } - } - - //@Bug 1806 - if (rc != NO_ERROR && rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) - { - return rc; - } - - if ( pushWarning && ( rc != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING ) ) - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - - - colValuesList.push_back(colValue); - //@Bug 2515. Only pass string values to write engine - dicStrings.push_back( valStr ); - } - - dicStringList.push_back( dicStrings ); - } - - if (pushWarning) - { - colNames.push_back(tableColName.column); - isWarningSet = true; - } - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - - // call the write engine to write the rows - int error = NO_ERROR; - - //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); - //cout << "Batch inserting a row with transaction id " << txnid.id << endl; - if (colValuesList.size() > 0) - { - if (NO_ERROR != - (error = fWEWrapper.insertColumnRecsBinary(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, - dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) - { - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - else if ( error == ERR_BRM_VB_OVERFLOW ) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } - else - { - rc = dmlpackageprocessor::DMLPackageProcessor::INSERT_ERROR; - WErrorCodes ec; - err = ec.errorString(error); - } - } - } - - if (fIsFirstBatchPm && isAutocommitOn) - { - //fWEWrapper.writeVBEnd(txnid.id, rangeList); - fIsFirstBatchPm = false; - } - else if (fIsFirstBatchPm) - { - fIsFirstBatchPm = false; - } - - if ( isWarningSet && ( rc == NO_ERROR ) ) - { - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - //cout << "Got warning" << endl; - Message::Args args; - string cols = "'" + colNames[0] + "'"; - - for (unsigned i = 1; i < colNames.size(); i++) - { - cols = cols + ", " + "'" + colNames[i] + "'"; - } - - args.add(cols); - err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); - - } - - // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); - return rc; -} - - uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - //need to commit the versioned blocks, set hwm, update casual partition, send back to DMLProc to set them - //cout << " in commiting autocommit on batch insert " << endl; - uint32_t tmp32, tableOid, sessionId; - int txnID; + uint8_t rc = 0; + // need to commit the versioned blocks, set hwm, update casual partition, send back to DMLProc to set them + // cout << " in commiting autocommit on batch insert " << endl; + uint32_t tmp32, tableOid, sessionId; + int txnID; - bs >> tmp32; - txnID = tmp32; - bs >> tmp32; - tableOid = tmp32; - bs >> tmp32; - sessionId = tmp32; + bs >> tmp32; + txnID = tmp32; + bs >> tmp32; + tableOid = tmp32; + bs >> tmp32; + sessionId = tmp32; - BRM::DBRM dbrm; + BRM::DBRM dbrm; - std::vector setHWMArgs; - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - BulkSetHWMArg aArg; - std::vector files; - BRM::FileInfo aFile; + std::vector setHWMArgs; + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); + ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); + ColExtsInfo::iterator aIt; + BulkSetHWMArg aArg; + std::vector files; + BRM::FileInfo aFile; - while (it != colsExtsInfoMap.end()) + while (it != colsExtsInfoMap.end()) + { + aIt = (it->second).begin(); + aArg.oid = it->first; + aFile.oid = it->first; + + // cout << "OID:" << aArg.oid; + while (aIt != (it->second).end()) { - aIt = (it->second).begin(); - aArg.oid = it->first; - aFile.oid = it->first; + aArg.partNum = aIt->partNum; + aArg.segNum = aIt->segNum; + aArg.hwm = aIt->hwm; - //cout << "OID:" << aArg.oid; - while (aIt != (it->second).end()) - { - aArg.partNum = aIt->partNum; - aArg.segNum = aIt->segNum; - aArg.hwm = aIt->hwm; + if (!aIt->isDict) + setHWMArgs.push_back(aArg); - if (!aIt->isDict) - setHWMArgs.push_back(aArg); - - aFile.partitionNum = aIt->partNum; - aFile.dbRoot = aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - //cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"<partNum; + aFile.dbRoot = aIt->dbRoot; + aFile.segmentNum = aIt->segNum; + aFile.compType = aIt->compType; + // cout <<"Added to files oid:dbroot:part:seg:compType = " << + // aFile.oid<<":"<tableName(tableOid); - CalpontSystemCatalog::RIDList ridList; + std::map oids; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); - try - { - ridList = systemCatalogPtr->columnRIDs(aTableName, true); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } + CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(tableOid); + CalpontSystemCatalog::RIDList ridList; - for (unsigned i = 0; i < ridList.size(); i++) - { - oids[ridList[i].objnum] = ridList[i].objnum; - } - - CalpontSystemCatalog::DictOIDList dictOids; - - try - { - dictOids = systemCatalogPtr->dictOIDs(aTableName); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - for (unsigned i = 0; i < dictOids.size(); i++) - { - oids[dictOids[i].dictOID] = dictOids[i].dictOID; - } - - fWEWrapper.setTransId(txnID); - - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - - fIsFirstBatchPm = true; - - if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0) ) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - - TableMetaData::removeTableMetaData(tableOid); - - // MCOL-1160 For API bulk insert flush the PrimProc cached dictionary - // blocks tounched - std::tr1::unordered_map::iterator mapIter; - mapIter = fWEWrapper.getDictMap().find(txnID); - - if (mapIter != fWEWrapper.getDictMap().end()) - { - std::set::iterator lbidIter; - std::vector dictFlushBlks; - cerr << "API Flushing blocks: "; - for (lbidIter = (*mapIter).second.begin(); lbidIter != (*mapIter).second.end(); lbidIter++) - { - cerr << *lbidIter << ", "; - dictFlushBlks.push_back((*lbidIter)); - } - cerr << endl; - cacheutils::flushPrimProcAllverBlocks(dictFlushBlks); - fWEWrapper.getDictMap().erase(txnID); - } - - // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); + try + { + ridList = systemCatalogPtr->columnRIDs(aTableName, true); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + for (unsigned i = 0; i < ridList.size(); i++) + { + oids[ridList[i].objnum] = ridList[i].objnum; + } + + CalpontSystemCatalog::DictOIDList dictOids; + + try + { + dictOids = systemCatalogPtr->dictOIDs(aTableName); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + for (unsigned i = 0; i < dictOids.size(); i++) + { + oids[dictOids[i].dictOID] = dictOids[i].dictOID; + } + + fWEWrapper.setTransId(txnID); + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + + fIsFirstBatchPm = true; + + if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + + TableMetaData::removeTableMetaData(tableOid); + + // MCOL-1160 For API bulk insert flush the PrimProc cached dictionary + // blocks tounched + std::tr1::unordered_map::iterator mapIter; + mapIter = fWEWrapper.getDictMap().find(txnID); + + if (mapIter != fWEWrapper.getDictMap().end()) + { + std::set::iterator lbidIter; + std::vector dictFlushBlks; + cerr << "API Flushing blocks: "; + for (lbidIter = (*mapIter).second.begin(); lbidIter != (*mapIter).second.end(); lbidIter++) + { + cerr << *lbidIter << ", "; + dictFlushBlks.push_back((*lbidIter)); + } + cerr << endl; + cacheutils::flushPrimProcAllverBlocks(dictFlushBlks); + fWEWrapper.getDictMap().erase(txnID); + } + + // MCOL-1495 Remove fCatalogMap entries CS won't use anymore. + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); + return rc; } uint8_t WE_DMLCommandProc::processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t tmp8, rc = 0; - err.clear(); - //set hwm for autocommit off - uint32_t tmp32, tableOid; - int txnID; - bool isAutoCommitOn; + uint8_t tmp8, rc = 0; + err.clear(); + // set hwm for autocommit off + uint32_t tmp32, tableOid; + int txnID; + bool isAutoCommitOn; - bs >> tmp32; - txnID = tmp32; - bs >> tmp8; - isAutoCommitOn = (tmp8 != 0); - bs >> tmp32; - tableOid = tmp32; - bs >> tmp8; - //cout << "processBatchInsertHwm: tranid:isAutoCommitOn = " < files; - std::vector oidsToFlush; + bs >> tmp32; + txnID = tmp32; + bs >> tmp8; + isAutoCommitOn = (tmp8 != 0); + bs >> tmp32; + tableOid = tmp32; + bs >> tmp8; + // cout << "processBatchInsertHwm: tranid:isAutoCommitOn = " < files; + std::vector oidsToFlush; - BRM::FileInfo aFile; - //BRM::FileInfo curFile; - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - CalpontSystemCatalog::RIDList ridList; - CalpontSystemCatalog::ROPair roPair; - std::vector colDBRootExtentInfo; - DBRootExtentInfo aExtentInfo; + BRM::FileInfo aFile; + // BRM::FileInfo curFile; + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); + ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); + ColExtsInfo::iterator aIt; + CalpontSystemCatalog::RIDList ridList; + CalpontSystemCatalog::ROPair roPair; + std::vector colDBRootExtentInfo; + DBRootExtentInfo aExtentInfo; - while (it != colsExtsInfoMap.end()) + while (it != colsExtsInfoMap.end()) + { + aIt = (it->second).begin(); + aFile.oid = it->first; + oidsToFlush.push_back(aFile.oid); + roPair.objnum = aFile.oid; + aExtentInfo.fPartition = 0; + aExtentInfo.fDbRoot = 0; + aExtentInfo.fSegment = 0; + aExtentInfo.fLocalHwm = 0; + bool isDict = false; + + while (aIt != (it->second).end()) { - aIt = (it->second).begin(); - aFile.oid = it->first; - oidsToFlush.push_back(aFile.oid); - roPair.objnum = aFile.oid; - aExtentInfo.fPartition = 0; - aExtentInfo.fDbRoot = 0; - aExtentInfo.fSegment = 0; - aExtentInfo.fLocalHwm = 0; - bool isDict = false; + aFile.partitionNum = aIt->partNum; + aFile.dbRoot = aIt->dbRoot; + aFile.segmentNum = aIt->segNum; + aFile.compType = aIt->compType; + files.push_back(aFile); - while (aIt != (it->second).end()) + if (!aIt->isDict) + { + if ((aIt->partNum > aExtentInfo.fPartition) || + ((aIt->partNum == aExtentInfo.fPartition) && (aIt->segNum > aExtentInfo.fSegment)) || + ((aIt->partNum == aExtentInfo.fPartition) && (aIt->segNum == aExtentInfo.fSegment) && + (aIt->segNum > aExtentInfo.fLocalHwm))) { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot = aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - - if (!aIt->isDict) - { - if ((aIt->partNum > aExtentInfo.fPartition) || ((aIt->partNum == aExtentInfo.fPartition) && (aIt->segNum > aExtentInfo.fSegment)) || - ((aIt->partNum == aExtentInfo.fPartition) && (aIt->segNum == aExtentInfo.fSegment) && (aIt->segNum > aExtentInfo.fLocalHwm ))) - { - aExtentInfo.fPartition = aIt->partNum; - aExtentInfo.fDbRoot = aIt->dbRoot; - aExtentInfo.fSegment = aIt->segNum; - aExtentInfo.fLocalHwm = aIt->hwm; - } - } - else - { - isDict = true; - } - - aIt++; + aExtentInfo.fPartition = aIt->partNum; + aExtentInfo.fDbRoot = aIt->dbRoot; + aExtentInfo.fSegment = aIt->segNum; + aExtentInfo.fLocalHwm = aIt->hwm; } + } + else + { + isDict = true; + } - if (!isDict) - { - ridList.push_back(roPair); - colDBRootExtentInfo.push_back(aExtentInfo); - } - - it++; + aIt++; } - //@Bug 5996. Validate hwm before set them - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - - try + if (!isDict) { - CalpontSystemCatalog::TableName tableName = systemCatalogPtr->tableName(tableOid); - ridList = systemCatalogPtr->columnRIDs(tableName); - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - TableMetaData::removeTableMetaData(tableOid); - - fIsFirstBatchPm = true; - //cout << "flush files when autocommit off" << endl; - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - return rc; + ridList.push_back(roPair); + colDBRootExtentInfo.push_back(aExtentInfo); } - rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Ending"); + it++; + } - if ( rc != 0) - { - WErrorCodes ec; - err = ec.errorString(rc); - err += " Check err.log for detailed information."; - TableMetaData::removeTableMetaData(tableOid); - - fIsFirstBatchPm = true; - fWEWrapper.setIsInsert(true); - fWEWrapper.setBulkFlag(true); - rc = 1; - return rc; - } - - try - { - if (isAutoCommitOn) - { - bs.restart(); - - if (fWEWrapper.getIsInsert()) - { - // @bug5333, up to here, rc == 0, but flushchunk may fail. - rc = processBatchInsertHwmFlushChunks(tableOid, txnID, files, oidsToFlush, err); - } - - if ( tmp8 != 0) - TableMetaData::removeTableMetaData(tableOid); - - return rc; // will set hwm with version commit. - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - // Handle case where isAutoCommitOn is false - BRM::DBRM dbrm; - //cout << " In processBatchInsertHwm setting hwm" << endl; - std::vector setHWMArgs; - it = colsExtsInfoMap.begin(); - BulkSetHWMArg aArg; - - while (it != colsExtsInfoMap.end()) - { - aIt = (it->second).begin(); - aArg.oid = it->first; - - //cout << "for oid " << aArg.oid << endl; - while (aIt != (it->second).end()) - { - aArg.partNum = aIt->partNum; - aArg.segNum = aIt->segNum; - aArg.hwm = aIt->hwm; - - //@Bug 6029 dictionary store files already set hwm. - if (!aIt->isDict) - setHWMArgs.push_back(aArg); - - aIt++; - } - - it++; - } + //@Bug 5996. Validate hwm before set them + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(0); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + try + { + CalpontSystemCatalog::TableName tableName = systemCatalogPtr->tableName(tableOid); + ridList = systemCatalogPtr->columnRIDs(tableName); + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; TableMetaData::removeTableMetaData(tableOid); fIsFirstBatchPm = true; - //cout << "flush files when autocommit off" << endl; + // cout << "flush files when autocommit off" << endl; fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); - - rc = processBatchInsertHwmFlushChunks(tableOid, txnID, - files, oidsToFlush, err); - bs.restart(); - - try - { - serializeInlineVector (bs, setHWMArgs); - } - catch (exception& ex) - { - // Append to errmsg in case we already have an error - if (err.length() > 0) - err += "; "; - - err += ex.what(); - rc = 1; - return rc; - } - - //cout << "flush is called for transaction " << txnID << endl; - return rc; + } + + rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Ending"); + + if (rc != 0) + { + WErrorCodes ec; + err = ec.errorString(rc); + err += " Check err.log for detailed information."; + TableMetaData::removeTableMetaData(tableOid); + + fIsFirstBatchPm = true; + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + rc = 1; + return rc; + } + + try + { + if (isAutoCommitOn) + { + bs.restart(); + + if (fWEWrapper.getIsInsert()) + { + // @bug5333, up to here, rc == 0, but flushchunk may fail. + rc = processBatchInsertHwmFlushChunks(tableOid, txnID, files, oidsToFlush, err); + } + + if (tmp8 != 0) + TableMetaData::removeTableMetaData(tableOid); + + return rc; // will set hwm with version commit. + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + // Handle case where isAutoCommitOn is false + BRM::DBRM dbrm; + // cout << " In processBatchInsertHwm setting hwm" << endl; + std::vector setHWMArgs; + it = colsExtsInfoMap.begin(); + BulkSetHWMArg aArg; + + while (it != colsExtsInfoMap.end()) + { + aIt = (it->second).begin(); + aArg.oid = it->first; + + // cout << "for oid " << aArg.oid << endl; + while (aIt != (it->second).end()) + { + aArg.partNum = aIt->partNum; + aArg.segNum = aIt->segNum; + aArg.hwm = aIt->hwm; + + //@Bug 6029 dictionary store files already set hwm. + if (!aIt->isDict) + setHWMArgs.push_back(aArg); + + aIt++; + } + + it++; + } + + TableMetaData::removeTableMetaData(tableOid); + + fIsFirstBatchPm = true; + // cout << "flush files when autocommit off" << endl; + fWEWrapper.setIsInsert(true); + fWEWrapper.setBulkFlag(true); + + rc = processBatchInsertHwmFlushChunks(tableOid, txnID, files, oidsToFlush, err); + bs.restart(); + + try + { + serializeInlineVector(bs, setHWMArgs); + } + catch (exception& ex) + { + // Append to errmsg in case we already have an error + if (err.length() > 0) + err += "; "; + + err += ex.what(); + rc = 1; + return rc; + } + + // cout << "flush is called for transaction " << txnID << endl; + + return rc; } //------------------------------------------------------------------------------ @@ -2456,2090 +2456,2071 @@ uint8_t WE_DMLCommandProc::processBatchInsertHwm(messageqcpp::ByteStream& bs, st // fIsFirstBatchPm = true; // returns 0 for success; returns 1 if error occurs //------------------------------------------------------------------------------ -uint8_t WE_DMLCommandProc::processBatchInsertHwmFlushChunks( - uint32_t tblOid, int txnID, - const std::vector& files, - const std::vector& oidsToFlush, - std::string& err) +uint8_t WE_DMLCommandProc::processBatchInsertHwmFlushChunks(uint32_t tblOid, int txnID, + const std::vector& files, + const std::vector& oidsToFlush, + std::string& err) { - uint8_t rc = 0; - std::map oids; - CalpontSystemCatalog::TableName aTableName; - CalpontSystemCatalog::RIDList ridList; - CalpontSystemCatalog::DictOIDList dictOids; + uint8_t rc = 0; + std::map oids; + CalpontSystemCatalog::TableName aTableName; + CalpontSystemCatalog::RIDList ridList; + CalpontSystemCatalog::DictOIDList dictOids; - try - { - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(txnID); - aTableName = systemCatalogPtr->tableName(tblOid); - ridList = systemCatalogPtr->columnRIDs(aTableName, true); - dictOids = systemCatalogPtr->dictOIDs(aTableName); - } - catch (exception& ex) - { - std::ostringstream ossErr; - ossErr << "System Catalog error for table OID " << tblOid; + try + { + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(txnID); + aTableName = systemCatalogPtr->tableName(tblOid); + ridList = systemCatalogPtr->columnRIDs(aTableName, true); + dictOids = systemCatalogPtr->dictOIDs(aTableName); + } + catch (exception& ex) + { + std::ostringstream ossErr; + ossErr << "System Catalog error for table OID " << tblOid; - // Include tbl name in msg unless exception occurred before we got it - if (aTableName.table.length() > 0) - ossErr << '(' << aTableName << ')'; - - ossErr << "; " << ex.what(); - err = ossErr.str(); - rc = 1; - return rc; - } - - for (unsigned i = 0; i < ridList.size(); i++) - { - oids[ridList[i].objnum] = ridList[i].objnum; - } - - for (unsigned i = 0; i < dictOids.size(); i++) - { - oids[dictOids[i].dictOID] = dictOids[i].dictOID; - } - - fWEWrapper.setTransId(txnID); - - // @bug5333, up to here, rc == 0, but flushchunk may fail. - rc = fWEWrapper.flushChunks(0, oids); - - if (rc == NO_ERROR) - { - // Confirm changes to db files "only" if no error up to this point - if (idbdatafile::IDBPolicy::useHdfs()) - { - std::string eMsg; - ConfirmHdfsDbFile confirmHdfs; - int confirmDbRc = confirmHdfs.confirmDbFileListFromMetaFile( - tblOid, eMsg); - - if (confirmDbRc == NO_ERROR) - { - int endDbRc = confirmHdfs.endDbFileListFromMetaFile( - tblOid, true, eMsg); - - if (endDbRc != NO_ERROR) - { - // Might want to log this error, but don't think we - // need to report as fatal, as all changes were confirmed. - } - - if (files.size() > 0) - cacheutils::purgePrimProcFdCache(files, - Config::getLocalModuleID()); - - cacheutils::flushOIDsFromCache(oidsToFlush); - } - else - { - ostringstream ossErr; - ossErr << "Error confirming changes to table " << - aTableName << "; " << eMsg; - err = ossErr.str(); - rc = 1; // reset to 1 - } - } - } - else // flushChunks error - { - WErrorCodes ec; - std::ostringstream ossErr; - ossErr << "Error flushing chunks for table " << aTableName << - "; " << ec.errorString(rc); - err = ossErr.str(); - rc = 1; // reset to 1 - } - - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fIsFirstBatchPm = true; + // Include tbl name in msg unless exception occurred before we got it + if (aTableName.table.length() > 0) + ossErr << '(' << aTableName << ')'; + ossErr << "; " << ex.what(); + err = ossErr.str(); + rc = 1; return rc; + } + + for (unsigned i = 0; i < ridList.size(); i++) + { + oids[ridList[i].objnum] = ridList[i].objnum; + } + + for (unsigned i = 0; i < dictOids.size(); i++) + { + oids[dictOids[i].dictOID] = dictOids[i].dictOID; + } + + fWEWrapper.setTransId(txnID); + + // @bug5333, up to here, rc == 0, but flushchunk may fail. + rc = fWEWrapper.flushChunks(0, oids); + + if (rc == NO_ERROR) + { + // Confirm changes to db files "only" if no error up to this point + if (idbdatafile::IDBPolicy::useHdfs()) + { + std::string eMsg; + ConfirmHdfsDbFile confirmHdfs; + int confirmDbRc = confirmHdfs.confirmDbFileListFromMetaFile(tblOid, eMsg); + + if (confirmDbRc == NO_ERROR) + { + int endDbRc = confirmHdfs.endDbFileListFromMetaFile(tblOid, true, eMsg); + + if (endDbRc != NO_ERROR) + { + // Might want to log this error, but don't think we + // need to report as fatal, as all changes were confirmed. + } + + if (files.size() > 0) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + + cacheutils::flushOIDsFromCache(oidsToFlush); + } + else + { + ostringstream ossErr; + ossErr << "Error confirming changes to table " << aTableName << "; " << eMsg; + err = ossErr.str(); + rc = 1; // reset to 1 + } + } + } + else // flushChunks error + { + WErrorCodes ec; + std::ostringstream ossErr; + ossErr << "Error flushing chunks for table " << aTableName << "; " << ec.errorString(rc); + err = ossErr.str(); + rc = 1; // reset to 1 + } + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fIsFirstBatchPm = true; + + return rc; } uint8_t WE_DMLCommandProc::commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - //commit all versioned blocks, set hwm, update casual partition - return rc; + uint8_t rc = 0; + // commit all versioned blocks, set hwm, update casual partition + return rc; } uint8_t WE_DMLCommandProc::rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - uint32_t tmp32, tableOid, sessionID; - uint64_t lockID; - bs >> sessionID; - bs >> lockID; - bs >> tmp32; - tableOid = tmp32; - //Bulkrollback - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - CalpontSystemCatalog::TableName aTableName; + uint8_t rc = 0; + uint32_t tmp32, tableOid, sessionID; + uint64_t lockID; + bs >> sessionID; + bs >> lockID; + bs >> tmp32; + tableOid = tmp32; + // Bulkrollback + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::TableName aTableName; - try - { - aTableName = systemCatalogPtr->tableName(tableOid); - } - catch ( ... ) - { - err = std::string("No such table for oid ") + std::to_string(tableOid); - rc = 1; - return rc; - } - - string table = aTableName.schema + "." + aTableName.table; - string applName ("BatchInsert"); - rc = fWEWrapper.bulkRollback(tableOid, lockID, table, applName, false, err); - fIsFirstBatchPm = true; - TableMetaData::removeTableMetaData(tableOid); + try + { + aTableName = systemCatalogPtr->tableName(tableOid); + } + catch (...) + { + err = std::string("No such table for oid ") + std::to_string(tableOid); + rc = 1; return rc; + } + + string table = aTableName.schema + "." + aTableName.table; + string applName("BatchInsert"); + rc = fWEWrapper.bulkRollback(tableOid, lockID, table, applName, false, err); + fIsFirstBatchPm = true; + TableMetaData::removeTableMetaData(tableOid); + return rc; } uint8_t WE_DMLCommandProc::rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - //Rollbacked all versioned blocks - return rc; + uint8_t rc = 0; + // Rollbacked all versioned blocks + return rc; } -uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, - std::string& err, - ByteStream::quadbyte& PMId, - uint64_t& blocksChanged) +uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId, uint64_t& blocksChanged) { - uint8_t rc = 0; - uint32_t tmp32, sessionID; - TxnID txnId; - bs >> PMId; - bs >> tmp32; - txnId = tmp32; - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.setTransId(txnId); + uint8_t rc = 0; + uint32_t tmp32, sessionID; + TxnID txnId; + bs >> PMId; + bs >> tmp32; + txnId = tmp32; + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.setTransId(txnId); - if (!rowGroups[txnId]) //meta data + if (!rowGroups[txnId]) // meta data + { + rowGroups[txnId] = new rowgroup::RowGroup(); + rowGroups[txnId]->deserialize(bs); + uint8_t pkgType; + bs >> pkgType; + cpackages[txnId].read(bs); + + rc = fWEWrapper.startTransaction(txnId); + + if (rc != NO_ERROR) { - rowGroups[txnId] = new rowgroup::RowGroup(); - rowGroups[txnId]->deserialize(bs); - uint8_t pkgType; - bs >> pkgType; - cpackages[txnId].read(bs); - - rc = fWEWrapper.startTransaction(txnId); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - err = ec.errorString(rc); - } - - return rc; + WErrorCodes ec; + err = ec.errorString(rc); } - bool pushWarning = false; - rowgroup::RGData rgData; - rgData.deserialize(bs); - rowGroups[txnId]->setData(&rgData); - //rowGroups[txnId]->setData(const_cast(bs.buf())); - //get rows and values - rowgroup::Row row; - rowGroups[txnId]->initRow(&row); - string value(""); - uint32_t rowsThisRowgroup = rowGroups[txnId]->getRowCount(); - uint32_t columnsSelected = rowGroups[txnId]->getColumnCount(); - std::vector fetchColTypes = rowGroups[txnId]->getColTypes(); - std::vector fetchColScales = rowGroups[txnId]->getScale(); - std::vector fetchColColwidths; + return rc; + } - for (uint32_t i = 0; i < columnsSelected; i++) - { - fetchColColwidths.push_back(rowGroups[txnId]->getColumnWidth(i)); - } + bool pushWarning = false; + rowgroup::RGData rgData; + rgData.deserialize(bs); + rowGroups[txnId]->setData(&rgData); + // rowGroups[txnId]->setData(const_cast(bs.buf())); + // get rows and values + rowgroup::Row row; + rowGroups[txnId]->initRow(&row); + string value(""); + uint32_t rowsThisRowgroup = rowGroups[txnId]->getRowCount(); + uint32_t columnsSelected = rowGroups[txnId]->getColumnCount(); + std::vector fetchColTypes = rowGroups[txnId]->getColTypes(); + std::vector fetchColScales = rowGroups[txnId]->getScale(); + std::vector fetchColColwidths; - WriteEngine::ColTupleList aColList; - WriteEngine::ColTuple colTuple; - WriteEngine::ColStructList colStructList; - WriteEngine::ColStruct colStruct; - WriteEngine::ColValueList colValueList; - WriteEngine::RIDList rowIDLists; - WriteEngine::CSCTypesList cscColTypeList; + for (uint32_t i = 0; i < columnsSelected; i++) + { + fetchColColwidths.push_back(rowGroups[txnId]->getColumnWidth(i)); + } - WriteEngine::DctnryStructList dctnryStructList; - WriteEngine::DctnryStruct dctnryStruct; - WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::ColTupleList aColList; + WriteEngine::ColTuple colTuple; + WriteEngine::ColStructList colStructList; + WriteEngine::ColStruct colStruct; + WriteEngine::ColValueList colValueList; + WriteEngine::RIDList rowIDLists; + WriteEngine::CSCTypesList cscColTypeList; - CalpontSystemCatalog::TableName tableName; - CalpontSystemCatalog::TableColName tableColName; - DMLTable* tablePtr = cpackages[txnId].get_Table(); - RowList rows = tablePtr->get_RowList(); - dmlpackage::ColumnList columnsUpdated = rows[0]->get_ColumnList(); - tableColName.table = tableName.table = tablePtr->get_TableName(); - tableColName.schema = tableName.schema = tablePtr->get_SchemaName(); - tableColName.column = columnsUpdated[0]->get_Name(); + WriteEngine::DctnryStructList dctnryStructList; + WriteEngine::DctnryStruct dctnryStruct; + WriteEngine::DctnryValueList dctnryValueList; - sessionID = cpackages[txnId].get_SessionID(); + CalpontSystemCatalog::TableName tableName; + CalpontSystemCatalog::TableColName tableColName; + DMLTable* tablePtr = cpackages[txnId].get_Table(); + RowList rows = tablePtr->get_RowList(); + dmlpackage::ColumnList columnsUpdated = rows[0]->get_ColumnList(); + tableColName.table = tableName.table = tablePtr->get_TableName(); + tableColName.schema = tableName.schema = tablePtr->get_SchemaName(); + tableColName.column = columnsUpdated[0]->get_Name(); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - CalpontSystemCatalog::OID oid = 0; - CalpontSystemCatalog::ROPair tableRO; + sessionID = cpackages[txnId].get_SessionID(); - std::string timeZone = cpackages[txnId].get_TimeZone(); + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::OID oid = 0; + CalpontSystemCatalog::ROPair tableRO; + + std::string timeZone = cpackages[txnId].get_TimeZone(); + + try + { + tableRO = systemCatalogPtr->tableRID(tableName); + oid = systemCatalogPtr->lookupOID(tableColName); + } + catch (std::exception& ex) + { + rc = 1; + ostringstream oss; + oss << "lookupOID got exception " << ex.what() << " with column " << tableColName.schema + << "." + tableColName.table << "." << tableColName.column; + err = oss.str(); + } + catch (...) + { + rc = 1; + ostringstream oss; + oss << "lookupOID got unknown exception with column " << tableColName.schema << "." << tableColName.table + << "." << tableColName.column; + err = oss.str(); + } + + if (rc != 0) + return rc; + + rowGroups[txnId]->getRow(0, &row); + CalpontSystemCatalog::RID rid = row.getRid(); + uint16_t dbRoot, segment, blockNum; + uint32_t partition; + uint8_t extentNum; + // Get the file information from rowgroup + dbRoot = rowGroups[txnId]->getDBRoot(); + rowGroups[txnId]->getLocation(&partition, &segment, &extentNum, &blockNum); + colStruct.fColPartition = partition; + colStruct.fColSegment = segment; + colStruct.fColDbRoot = dbRoot; + dctnryStruct.fColPartition = partition; + dctnryStruct.fColSegment = segment; + dctnryStruct.fColDbRoot = dbRoot; + TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tableRO.objnum); + // Build to be updated column structure and values + int error = 0; + unsigned fetchColPos = 0; + bool ridsFetched = false; + bool isNull = false; + boost::any datavalue; + int64_t intColVal = 0; + // timer.start("fetch values"); + std::vector colNames; + + // for query stats + boost::scoped_array colTypes( + new CalpontSystemCatalog::ColType[columnsUpdated.size()]); + boost::scoped_array preBlkNums(new int[columnsUpdated.size()]); + boost::scoped_array oids(new OID[columnsUpdated.size()]); + + BRMWrapper::setUseVb(true); + for (unsigned int j = 0; j < columnsUpdated.size(); j++) + { + // timer.start("lookupsyscat"); + tableColName.column = columnsUpdated[j]->get_Name(); try { - tableRO = systemCatalogPtr->tableRID(tableName); - oid = systemCatalogPtr->lookupOID(tableColName); + oids[j] = systemCatalogPtr->lookupOID(tableColName); + colTypes[j] = systemCatalogPtr->colType(oids[j]); } - catch (std::exception& ex) + catch (std::exception& ex) { - rc = 1; - ostringstream oss; - oss << "lookupOID got exception " << ex.what() << " with column " << tableColName.schema << "." + tableColName.table << "." << tableColName.column; - err = oss.str(); + rc = 1; + ostringstream oss; + oss << "colType got exception " << ex.what() << " with column oid " << oid; + err = oss.str(); } - catch ( ... ) + catch (...) { - rc = 1; - ostringstream oss; - oss << "lookupOID got unknown exception with column " << tableColName.schema << "." << tableColName.table << "." << tableColName.column; - err = oss.str(); + rc = 1; + ostringstream oss; + oss << "colType got unknown exception with column oid " << oid; + err = oss.str(); } if (rc != 0) - return rc; + return rc; - rowGroups[txnId]->getRow(0, &row); - CalpontSystemCatalog::RID rid = row.getRid(); - uint16_t dbRoot, segment, blockNum; - uint32_t partition; - uint8_t extentNum; - //Get the file information from rowgroup - dbRoot = rowGroups[txnId]->getDBRoot(); - rowGroups[txnId]->getLocation(&partition, &segment, &extentNum, &blockNum); - colStruct.fColPartition = partition; - colStruct.fColSegment = segment; - colStruct.fColDbRoot = dbRoot; - dctnryStruct.fColPartition = partition; - dctnryStruct.fColSegment = segment; - dctnryStruct.fColDbRoot = dbRoot; - TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tableRO.objnum); - //Build to be updated column structure and values - int error = 0; - unsigned fetchColPos = 0; - bool ridsFetched = false; - bool isNull = false; - boost::any datavalue; - int64_t intColVal = 0; - //timer.start("fetch values"); - std::vector colNames; + preBlkNums[j] = -1; + } - // for query stats - boost::scoped_array colTypes(new CalpontSystemCatalog::ColType[columnsUpdated.size()]); - boost::scoped_array preBlkNums(new int[columnsUpdated.size()]); - boost::scoped_array oids(new OID[columnsUpdated.size()]); + for (unsigned int j = 0; j < columnsUpdated.size(); j++) + { + WriteEngine::ColTupleList colTupleList; + CalpontSystemCatalog::ColType colType = colTypes[j]; + oid = oids[j]; + colStruct.dataOid = oid; + colStruct.colDataType = colType.colDataType; + colStruct.tokenFlag = false; + colStruct.fCompressionType = colType.compressionType; + tableColName.column = columnsUpdated[j]->get_Name(); - BRMWrapper::setUseVb(true); - for (unsigned int j = 0; j < columnsUpdated.size(); j++) + if (!ridsFetched) { - //timer.start("lookupsyscat"); - tableColName.column = columnsUpdated[j]->get_Name(); + // querystats + uint64_t relativeRID = 0; - try - { - oids[j] = systemCatalogPtr->lookupOID(tableColName); - colTypes[j] = systemCatalogPtr->colType(oids[j]); - } - catch (std::exception& ex) - { - rc = 1; - ostringstream oss; - oss << "colType got exception " << ex.what() << " with column oid " << oid; - err = oss.str(); - } - catch ( ... ) - { - rc = 1; - ostringstream oss; - oss << "colType got unknown exception with column oid " << oid; - err = oss.str(); - } - - if (rc != 0) - return rc; - - preBlkNums[j] = -1; - } - - for (unsigned int j = 0; j < columnsUpdated.size(); j++) - { - WriteEngine::ColTupleList colTupleList; - CalpontSystemCatalog::ColType colType = colTypes[j]; - oid = oids[j]; - colStruct.dataOid = oid; - colStruct.colDataType = colType.colDataType; - colStruct.tokenFlag = false; - colStruct.fCompressionType = colType.compressionType; - tableColName.column = columnsUpdated[j]->get_Name(); - - if (!ridsFetched) - { - // querystats - uint64_t relativeRID = 0; - - for (unsigned i = 0; i < rowsThisRowgroup; i++) - { - rowGroups[txnId]->getRow(i, &row); - rid = row.getRid(); - relativeRID = rid - rowGroups[txnId]->getBaseRid(); - rid = relativeRID; - convertToRelativeRid (rid, extentNum, blockNum); - rowIDLists.push_back(rid); - - uint32_t colWidth = colTypes[j].colWidth; - - if (colWidth > 8 && - !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || - colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) - { - colWidth = 8; - } - else if (colWidth >= datatypes::MAXDECIMALWIDTH) - { - colWidth = datatypes::MAXDECIMALWIDTH; - } - - int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth); - // populate stats.blocksChanged - if (rrid > preBlkNums[j]) - { - preBlkNums[j] = rrid ; - blocksChanged++; - } - - } - - ridsFetched = true; - } - - bool pushWarn = false; - bool nameNeeded = false; - - if (isDictCol(colType)) - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - dctnryStruct.dctnryOid = colType.ddn.dictOID; - dctnryStruct.columnOid = colStruct.dataOid; - dctnryStruct.fCompressionType = colType.compressionType; - dctnryStruct.colWidth = colType.colWidth; - - if (NO_ERROR != (error = fWEWrapper.openDctnry (txnId, dctnryStruct, false))) // @bug 5572 HDFS tmp file - { - WErrorCodes ec; - err = ec.errorString(error); - rc = error; - return rc; - } - - ColExtsInfo aColExtsInfo = aTableMetaData->getColExtsInfo(dctnryStruct.dctnryOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == dctnryStruct.fColDbRoot) && (it->partNum == dctnryStruct.fColPartition) && (it->segNum == dctnryStruct.fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = dctnryStruct.fColDbRoot; - aExt.partNum = dctnryStruct.fColPartition; - aExt.segNum = dctnryStruct.fColSegment; - aExt.compType = dctnryStruct.fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - } - - aTableMetaData->setColExtsInfo(dctnryStruct.dctnryOid, aColExtsInfo); - - - if (columnsUpdated[j]->get_isFromCol()) - { - for (unsigned i = 0; i < rowsThisRowgroup; i++) - { - - rowGroups[txnId]->getRow(i, &row); - - if (row.isNullValue(fetchColPos)) - { - if ((colType.defaultValue.length() <= 0) && (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - rc = 1; - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - return rc; - } - else if (colType.defaultValue.length() > 0) - { - value = colType.defaultValue; - - if (value.length() > (unsigned int)colType.colWidth) - { - value = value.substr(0, colType.colWidth); - pushWarn = true; - - if (!pushWarning) - { - pushWarning = true; - } - - if (pushWarn) - nameNeeded = true; - } - - WriteEngine::DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)value.c_str(); - dctTuple.sigSize = value.length(); - dctTuple.isNull = false; - - error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); - - if (error != NO_ERROR) - { - fWEWrapper.closeDctnry(txnId, colType.compressionType); - return false; - } - - colTuple.data = dctTuple.token; - colTupleList.push_back (colTuple); - } - else - { - WriteEngine::Token nullToken; - colTuple.data = nullToken; - colTupleList.push_back (colTuple); - } - - continue; - } - - switch (fetchColTypes[fetchColPos]) - { - case CalpontSystemCatalog::DATE: - { - intColVal = row.getUintField<4>(fetchColPos); - value = DataConvert::dateToString(intColVal); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal, colType.precision); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::timestampToString(intColVal, timeZone, colType.precision); - break; - } - - case CalpontSystemCatalog::TIME: - { - intColVal = row.getIntField<8>(fetchColPos); - value = DataConvert::timeToString(intColVal, colType.precision); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - value = row.getStringField(fetchColPos); - unsigned i = strlen(value.c_str()); - value = value.substr(0, i); - break; - } - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - value = row.getVarBinaryStringField(fetchColPos); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) - { - datatypes::Decimal dec(0, - fetchColScales[fetchColPos], - rowGroups[txnId]->getPrecision()[fetchColPos], - row.getBinaryField(fetchColPos)); - value = dec.toString(true); - break; - } - } - /* fall through */ - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::UTINYINT: - { - { - intColVal = row.getIntField(fetchColPos); - - if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDECIMAL - && intColVal < 0) - { - intColVal = 0; - } - - if (fetchColScales[fetchColPos] <= 0) - { - ostringstream os; - - if (isUnsigned(fetchColTypes[fetchColPos])) - os << static_cast(intColVal); - else - os << intColVal; - - value = os.str(); - } - else - { - datatypes::Decimal dec(intColVal, - fetchColScales[fetchColPos], - rowGroups[txnId]->getPrecision()[fetchColPos]); - value = dec.toString(); - } - } - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float dl = row.getFloatField(fetchColPos); - - if (dl == std::numeric_limits::infinity()) - continue; - - if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UFLOAT && dl < 0.0) - { - dl = 0.0; - } - - ostringstream os; - //@Bug 3350 fix the precision. - os << setprecision(7) << dl; - value = os.str(); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dl = row.getDoubleField(fetchColPos); - - if (dl == std::numeric_limits::infinity()) - continue; - - if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDOUBLE && dl < 0.0) - { - dl = 0.0; - } - - ostringstream os; - //@Bug 3350 fix the precision. - os << setprecision(16) << dl; - value = os.str(); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dll = row.getLongDoubleField(fetchColPos); - - if (dll == std::numeric_limits::infinity()) - continue; - - ostringstream os; - //@Bug 3350 fix the precision. - os << setprecision(19) << dll; - value = os.str(); - break; - } - - default: // treat as int64 - { - ostringstream os; - intColVal = row.getUintField<8>(fetchColPos); - os << intColVal; - value = os.str(); - break; - } - } - - uint32_t funcScale = columnsUpdated[j]->get_funcScale(); - - if (funcScale != 0) - { - string::size_type pos = value.find_first_of("."); //decimal point - - if ( pos >= value.length() ) - value.insert(value.length(), "."); - - //padding 0 if needed - pos = value.find_first_of("."); - uint32_t digitsAfterPoint = value.length() - pos - 1; - - if (digitsAfterPoint < funcScale) - { - for (uint32_t i = 0; i < (funcScale - digitsAfterPoint); i++) - value += "0"; - } - } - - //check data length - //trim the string if needed - if (value.length() > (unsigned int)colType.colWidth) - { - value = value.substr(0, colType.colWidth); - - if (!pushWarn) - pushWarn = true; - - if (!pushWarning) - pushWarning = true; - - if (pushWarn) - nameNeeded = true; - } - - WriteEngine::DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)value.c_str(); - dctTuple.sigSize = value.length(); - dctTuple.isNull = false; - - error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); - - if (error != NO_ERROR) - { - fWEWrapper.closeDctnry(txnId, colType.compressionType); - rc = error; - WErrorCodes ec; - err = ec.errorString(error); - return rc; - } - - colTuple.data = dctTuple.token; - colTupleList.push_back (colTuple); - } - - if (colType.compressionType == 0) - fWEWrapper.closeDctnry(txnId, colType.compressionType, true); - else - fWEWrapper.closeDctnry(txnId, colType.compressionType, false); - - fetchColPos++; - } - else //constant - { - if (columnsUpdated[j]->get_isnull()) - { - if ((colType.defaultValue.length() <= 0) && (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - rc = 1; - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - return rc; - } - else if (colType.defaultValue.length() > 0) - { - value = colType.defaultValue; - - if (value.length() > (unsigned int)colType.colWidth) - { - value = value.substr(0, colType.colWidth); - pushWarn = true; - - if (!pushWarning) - { - pushWarning = true; - } - - if (pushWarn) - nameNeeded = true; - } - - WriteEngine::DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)value.c_str(); - dctTuple.sigSize = value.length(); - dctTuple.isNull = false; - error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); - - if (error != NO_ERROR) - { - fWEWrapper.closeDctnry(txnId, colType.compressionType); - rc = error; - WErrorCodes ec; - err = ec.errorString(error); - return rc; - } - - colTuple.data = dctTuple.token; - - if (colType.compressionType == 0) - fWEWrapper.closeDctnry(txnId, colType.compressionType, true); - else - fWEWrapper.closeDctnry(txnId, colType.compressionType, false); // Constant only need to tokenize once. - } - else - { - WriteEngine::Token nullToken; - colTuple.data = nullToken; - } - } - else - { - value = columnsUpdated[j]->get_Data(); - - if (value.length() > (unsigned int)colType.colWidth) - { - value = value.substr(0, colType.colWidth); - pushWarn = true; - - if (!pushWarning) - { - pushWarning = true; - } - - if (pushWarn) - nameNeeded = true; - } - - WriteEngine::DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)value.c_str(); - dctTuple.sigSize = value.length(); - dctTuple.isNull = false; - error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); - - if (error != NO_ERROR) - { - fWEWrapper.closeDctnry(txnId, colType.compressionType); - rc = error; - WErrorCodes ec; - err = ec.errorString(error); - return rc; - } - - colTuple.data = dctTuple.token; - - if (colType.compressionType == 0) - fWEWrapper.closeDctnry(txnId, colType.compressionType, true); - else - fWEWrapper.closeDctnry(txnId, colType.compressionType, false); // Constant only need to tokenize once. - } - - for (unsigned row = 0; row < rowsThisRowgroup; row++) - colTupleList.push_back (colTuple); - } - } - else //Non dictionary column - { - colStruct.colWidth = colType.colWidth; - - if (columnsUpdated[j]->get_isFromCol()) - { - for (unsigned i = 0; i < rowsThisRowgroup; i++) - { - rowGroups[txnId]->getRow(i, &row); - - if (row.isNullValue(fetchColPos)) - { - isNull = true; - value = ""; - } - else - { - isNull = false; - - switch (fetchColTypes[fetchColPos]) - { - case CalpontSystemCatalog::DATE: - { - intColVal = row.getUintField<4>(fetchColPos); - value = DataConvert::dateToString(intColVal); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal, colType.precision); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::timestampToString(intColVal, timeZone, colType.precision); - break; - } - - case CalpontSystemCatalog::TIME: - { - intColVal = row.getIntField<8>(fetchColPos); - value = DataConvert::timeToString(intColVal, colType.precision); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - value = row.getStringField(fetchColPos); - unsigned i = strlen(value.c_str()); - value = value.substr(0, i); - break; - } - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - value = row.getVarBinaryStringField(fetchColPos); - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) - { - datatypes::Decimal dec(0, - fetchColScales[fetchColPos], - rowGroups[txnId]->getPrecision()[fetchColPos], - row.getBinaryField(fetchColPos)); - value = dec.toString(true); - break; - } - } - /* fall through */ - - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::UTINYINT: - { - { - intColVal = row.getIntField(fetchColPos); - - if (fetchColTypes[fetchColPos] == - CalpontSystemCatalog::UDECIMAL - && intColVal < 0) - { - intColVal = 0; - } - - if (fetchColScales[fetchColPos] <= 0) - { - ostringstream os; - - if (isUnsigned(fetchColTypes[fetchColPos])) - os << static_cast(intColVal); - else - os << intColVal; - - value = os.str(); - } - else - { - datatypes::Decimal dec(intColVal, - fetchColScales[fetchColPos], - rowGroups[txnId]->getPrecision()[fetchColPos]); - value = dec.toString(); - } - } - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float dl = row.getFloatField(fetchColPos); - - if (dl == std::numeric_limits::infinity()) - continue; - - if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UFLOAT && dl < 0.0) - { - dl = 0.0; - } - - ostringstream os; - //@Bug 3350 fix the precision. - os << setprecision(7) << dl; - value = os.str(); - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dl = row.getDoubleField(fetchColPos); - - if (dl == std::numeric_limits::infinity()) - continue; - - if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDOUBLE && dl < 0.0) - { - dl = 0.0; - } - - ostringstream os; - //@Bug 3350 fix the precision. - os << setprecision(16) << dl; - value = os.str(); - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dll = row.getLongDoubleField(fetchColPos); - - if (dll == std::numeric_limits::infinity()) - continue; - - ostringstream os; - //@Bug 3350 fix the precision. - os << setprecision(19) << dll; - value = os.str(); - break; - } - - default: // treat as int64 - { - ostringstream os; - intColVal = row.getUintField<8>(fetchColPos); - os << intColVal; - value = os.str(); - break; - } - } - } - - uint32_t funcScale = columnsUpdated[j]->get_funcScale(); - - if (funcScale != 0) - { - string::size_type pos = value.find_first_of("."); //decimal point - - if ( pos >= value.length() ) - value.insert(value.length(), "."); - - //padding 0 if needed - pos = value.find_first_of("."); - uint32_t digitsAfterPoint = value.length() - pos - 1; - - if (digitsAfterPoint < funcScale) - { - for (uint32_t i = 0; i < (funcScale - digitsAfterPoint); i++) - value += "0"; - } - } - - //Check NOT NULL constraint and default value - if ((isNull) && (colType.defaultValue.length() <= 0) && (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - rc = 1; - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - return rc; - } - else if ((isNull) && (colType.defaultValue.length() > 0)) - { - isNull = false; - bool oneWarn = false; - - try - { - datavalue = colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); - } - catch (exception&) - { - //@Bug 2624. Error out on conversion failure - rc = 1; - Message::Args args; - args.add(string("'") + colType.defaultValue + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - } - - if ((pushWarn) && (!oneWarn)) - oneWarn = true; - - colTuple.data = datavalue; - colTupleList.push_back (colTuple); - - if (oneWarn) - pushWarn = true; - - if (!pushWarning) - { - pushWarning = pushWarn; - } - - if (pushWarn) - nameNeeded = true; - } - else - { - try - { - datavalue = colType.convertColumnData(value, pushWarn, timeZone, isNull, false, false); - } - catch (exception&) - { - //@Bug 2624. Error out on conversion failure - rc = 1; - Message::Args args; - args.add(string("'") + value + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - return rc; - } - - colTuple.data = datavalue; - colTupleList.push_back (colTuple); - - if (!pushWarning) - { - pushWarning = pushWarn; - } - - if (pushWarn) - nameNeeded = true; - } - } - - fetchColPos++; - } - else //constant column - { - if (columnsUpdated[j]->get_isnull()) - { - isNull = true; - } - else - { - isNull = false; - } - - string inData (columnsUpdated[j]->get_Data()); - - if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (inData == "0000-00-00")) || - ((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (inData == "0000-00-00 00:00:00")) || - ((colType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) && (inData == "0000-00-00 00:00:00"))) - { - isNull = true; - } - - uint64_t nextVal = 0; - - if (colType.autoincrement) - { - try - { - nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); - fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - } - - if (colType.autoincrement && ( isNull || (inData.compare("0") == 0))) - { - //reserve nextVal - try - { - bool reserved = fDbrm.getAIRange(oid, rowsThisRowgroup, &nextVal); - - if (!reserved) - { - err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); - rc = 1; - return rc; - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - } - - isNull = false; - bool oneWarn = false; - - for (unsigned row = 0; row < rowsThisRowgroup; row++) - { - - ostringstream oss; - oss << nextVal++; - inData = oss.str(); - - try - { - datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, false); - } - catch (exception&) - { - //@Bug 2624. Error out on conversion failure - rc = 1; - Message::Args args; - args.add(string("'") + inData + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - } - - if ((pushWarn) && (!oneWarn)) - oneWarn = true; - - colTuple.data = datavalue; - colTupleList.push_back (colTuple); - } - - if (oneWarn) - pushWarn = true; - - if (!pushWarning) - { - pushWarning = pushWarn; - } - - if (pushWarn) - nameNeeded = true; - } - else if (isNull && (colType.defaultValue.length() <= 0) && (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - rc = 1; - Message::Args args; - args.add(tableColName.column); - err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); - return rc; - } - else if (isNull && (colType.defaultValue.length() > 0)) - { - isNull = false; - bool oneWarn = false; - - for (unsigned row = 0; row < rowsThisRowgroup; row++) - { - try - { - datavalue = colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); - } - catch (exception&) - { - //@Bug 2624. Error out on conversion failure - rc = 1; - Message::Args args; - args.add(string("'") + colType.defaultValue + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - } - - if ((pushWarn) && (!oneWarn)) - oneWarn = true; - - colTuple.data = datavalue; - colTupleList.push_back (colTuple); - } - - if (oneWarn) - pushWarn = true; - - if (!pushWarning) - { - pushWarning = pushWarn; - } - - if (pushWarn) - nameNeeded = true; - } - else - { - try - { - datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, true); - } - catch (exception& ex) - { - //@Bug 2624. Error out on conversion failure - rc = 1; - cout << ex.what() << endl; - Message::Args args; - args.add(string("'") + inData + string("'")); - err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); - return rc; - } - - colTuple.data = datavalue; - - if (!pushWarning) - { - pushWarning = pushWarn; - } - - if (pushWarn) - nameNeeded = true; - - for (unsigned row = 0; row < rowsThisRowgroup; row++) - colTupleList.push_back (colTuple); - } - } - } - - if (nameNeeded) - { - colNames.push_back(tableColName.column); - } - - colStructList.push_back(colStruct); - colValueList.push_back (colTupleList); - cscColTypeList.push_back(colType); - } //end of bulding values and column structure. - - //timer.stop("fetch values"); - if (rowIDLists.size() > 0) - { - error = fWEWrapper.updateColumnRecs(txnId, cscColTypeList, colStructList, colValueList, rowIDLists, tableRO.objnum); - } - - if (error != NO_ERROR) - { - rc = error; - WErrorCodes ec; - err = ec.errorString(error); - - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - } - else if (error == ERR_BRM_VB_OVERFLOW) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } - } - - if (pushWarning) - { - rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - Message::Args args; - string cols = "'" + colNames[0] + "'"; - - for (unsigned i = 1; i < colNames.size(); i++) - { - cols = cols + ", " + "'" + colNames[i] + "'"; - } - - args.add(cols); - err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); - } - - return rc; -} - -uint8_t WE_DMLCommandProc::getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId) -{ - uint8_t rc = 0; - uint32_t txnId; - vector lbidList; - - bs >> txnId; - std::tr1::unordered_map m_txnLBIDMap = fWEWrapper.getTxnMap(); - - try - { - auto mapIter = m_txnLBIDMap.find(txnId); - - if (mapIter != m_txnLBIDMap.end()) - { - SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; - lbidList = spTxnLBIDRec->m_LBIDs; - } - } - catch (...) {} - - bs.restart(); - - try - { - serializeInlineVector (bs, lbidList); - } - catch (exception& ex) - { - // Append to errmsg in case we already have an error - if (err.length() > 0) - err += "; "; - - err += ex.what(); - rc = 1; - return rc; - } - - return rc; -} - -uint8_t WE_DMLCommandProc::processFlushFiles(messageqcpp::ByteStream& bs, std::string& err) -{ - uint8_t rc = 0; - uint32_t flushCode, txnId, tableOid; - int error; - bs >> flushCode; - bs >> txnId; - bs >> tableOid; - std::map oids; - CalpontSystemCatalog::TableName aTableName; - CalpontSystemCatalog::RIDList ridList; - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(txnId); - // execplan::CalpontSystemCatalog::ColType colType; - CalpontSystemCatalog::DictOIDList dictOids; - - if (tableOid >= 3000) - { - try - { - aTableName = systemCatalogPtr->tableName(tableOid); - } - catch ( ... ) - { - err = std::string("Systemcatalog error for tableoid ") + std::to_string(tableOid); - rc = 1; - return rc; - } - - dictOids = systemCatalogPtr->dictOIDs(aTableName); - - for (unsigned i = 0; i < dictOids.size(); i++) - { - oids[dictOids[i].dictOID] = dictOids[i].dictOID; - } - - //if (dictOids.size() > 0) - // colType = systemCatalogPtr->colTypeDct(dictOids[0].dictOID); - } - - - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - vector lbidList; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - // XXX THIS IS WRONG!!! - //save the extent info to mark them invalid, after flush, the meta file will be gone. - std::tr1::unordered_map m_txnLBIDMap = fWEWrapper.getTxnMap(); - - try - { - auto mapIter = m_txnLBIDMap.find(txnId); - - if (mapIter != m_txnLBIDMap.end()) - { - SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; - lbidList = spTxnLBIDRec->m_LBIDs; - - } - } - catch (...) {} - } - - error = fWEWrapper.flushDataFiles(flushCode, txnId, oids); - -//No need to close files, flushDataFile will close them. - //if (((colType.compressionType > 0 ) && (dictOids.size() > 0)) || (idbdatafile::IDBPolicy::useHdfs())) - // fWEWrapper.closeDctnry(txnId, colType.compressionType, true); - if (error != NO_ERROR) - { - rc = error; - WErrorCodes ec; - err = ec.errorString(error); - } - - //erase rowgroup from the rowGroup map - if (rowGroups[txnId]) - { - delete rowGroups[txnId]; - rowGroups[txnId] = 0; - } - - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - std::vector files; - std::vector oidsToFlush; - BRM::FileInfo aFile; - - while (it != colsExtsInfoMap.end()) - { - aIt = (it->second).begin(); - aFile.oid = it->first; - oidsToFlush.push_back(aFile.oid); - - while (aIt != (it->second).end()) - { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot = aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - //cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"<> PMId; - bs >> sessionID; - bs >> tmp32; - txnId = tmp32; - string schema, tableName; - bs >> schema; - bs >> tableName; - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.setTransId(txnId); - - if (!rowGroups[txnId]) //meta data - { - rowGroups[txnId] = new rowgroup::RowGroup(); - rowGroups[txnId]->deserialize(bs); - //If hdfs, call chunkmanager to set up - - rc = fWEWrapper.startTransaction(txnId); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - err = ec.errorString(rc); - } - - return rc; - } - - rowgroup::RGData rgData; - rgData.deserialize(bs); - rowGroups[txnId]->setData(&rgData); - //rowGroups[txnId]->setData(const_cast(bs.buf())); - //get row ids - rowgroup::Row row; - rowGroups[txnId]->initRow(&row); - WriteEngine::RIDList rowIDList; - CalpontSystemCatalog::RID rid; - uint32_t rowsThisRowgroup = rowGroups[txnId]->getRowCount(); - uint16_t dbRoot, segment, blockNum; - uint32_t partition; - uint8_t extentNum; - CalpontSystemCatalog::TableName aTableName; - aTableName.schema = schema; - aTableName.table = tableName; - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - CalpontSystemCatalog::ROPair roPair; - - CalpontSystemCatalog::RIDList tableRidList; - - try - { - roPair = systemCatalogPtr->tableRID( aTableName); - tableRidList = systemCatalogPtr->columnRIDs(aTableName, true); - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - - } - - // querystats - uint64_t relativeRID = 0; - boost::scoped_array preBlkNums(new int[row.getColumnCount()]); - boost::scoped_array colWidth(new uint32_t[row.getColumnCount()]); - - // initialize - for (uint32_t j = 0; j < row.getColumnCount(); j++) - { - preBlkNums[j] = -1; - colWidth[j] = row.getColumnWidth(j); - execplan::CalpontSystemCatalog::ColDataType colDataType = row.getColType(j); - if (colWidth[j] >= 8 && - !(colDataType == execplan::CalpontSystemCatalog::DECIMAL || - colDataType == execplan::CalpontSystemCatalog::UDECIMAL)) - { - colWidth[j] = 8; - } - else if (colWidth[j] >= datatypes::MAXDECIMALWIDTH) - { - colWidth[j] = datatypes::MAXDECIMALWIDTH; - } - } - - //Get the file information from rowgroup - dbRoot = rowGroups[txnId]->getDBRoot(); - rowGroups[txnId]->getLocation(&partition, &segment, &extentNum, &blockNum); - WriteEngine::ColStructList colStructList; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::ColStruct colStruct; - colStruct.fColPartition = partition; - colStruct.fColSegment = segment; - colStruct.fColDbRoot = dbRoot; - - for (unsigned i = 0; i < rowsThisRowgroup; i++) - { + for (unsigned i = 0; i < rowsThisRowgroup; i++) + { rowGroups[txnId]->getRow(i, &row); rid = row.getRid(); relativeRID = rid - rowGroups[txnId]->getBaseRid(); rid = relativeRID; - convertToRelativeRid (rid, extentNum, blockNum); - rowIDList.push_back(rid); + convertToRelativeRid(rid, extentNum, blockNum); + rowIDLists.push_back(rid); - // populate stats.blocksChanged - for (uint32_t j = 0; j < row.getColumnCount(); j++) + uint32_t colWidth = colTypes[j].colWidth; + + if (colWidth > 8 && !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || + colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) { - if ((int)(relativeRID / (BYTE_PER_BLOCK / colWidth[j])) > preBlkNums[j]) - { - blocksChanged++; - preBlkNums[j] = relativeRID / (BYTE_PER_BLOCK / colWidth[j]); - } + colWidth = 8; } + else if (colWidth >= datatypes::MAXDECIMALWIDTH) + { + colWidth = datatypes::MAXDECIMALWIDTH; + } + + int rrid = (int)relativeRID / (BYTE_PER_BLOCK / colWidth); + // populate stats.blocksChanged + if (rrid > preBlkNums[j]) + { + preBlkNums[j] = rrid; + blocksChanged++; + } + } + + ridsFetched = true; } - try - { - for (unsigned i = 0; i < tableRidList.size(); i++) - { - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType( tableRidList[i].objnum ); - colStruct.dataOid = tableRidList[i].objnum; - colStruct.tokenFlag = false; - colStruct.fCompressionType = colType.compressionType; + bool pushWarn = false; + bool nameNeeded = false; - if (colType.colWidth > 8 && - !(colType.colDataType == CalpontSystemCatalog::DECIMAL || - colType.colDataType == CalpontSystemCatalog::UDECIMAL)) //token + if (isDictCol(colType)) + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + dctnryStruct.dctnryOid = colType.ddn.dictOID; + dctnryStruct.columnOid = colStruct.dataOid; + dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.colWidth = colType.colWidth; + + if (NO_ERROR != (error = fWEWrapper.openDctnry(txnId, dctnryStruct, false))) // @bug 5572 HDFS tmp file + { + WErrorCodes ec; + err = ec.errorString(error); + rc = error; + return rc; + } + + ColExtsInfo aColExtsInfo = aTableMetaData->getColExtsInfo(dctnryStruct.dctnryOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == dctnryStruct.fColDbRoot) && (it->partNum == dctnryStruct.fColPartition) && + (it->segNum == dctnryStruct.fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = dctnryStruct.fColDbRoot; + aExt.partNum = dctnryStruct.fColPartition; + aExt.segNum = dctnryStruct.fColSegment; + aExt.compType = dctnryStruct.fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + } + + aTableMetaData->setColExtsInfo(dctnryStruct.dctnryOid, aColExtsInfo); + + if (columnsUpdated[j]->get_isFromCol()) + { + for (unsigned i = 0; i < rowsThisRowgroup; i++) + { + rowGroups[txnId]->getRow(i, &row); + + if (row.isNullValue(fetchColPos)) + { + if ((colType.defaultValue.length() <= 0) && + (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; + rc = 1; + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + return rc; + } + else if (colType.defaultValue.length() > 0) + { + value = colType.defaultValue; + + if (value.length() > (unsigned int)colType.colWidth) + { + value = value.substr(0, colType.colWidth); + pushWarn = true; + + if (!pushWarning) + { + pushWarning = true; + } + + if (pushWarn) + nameNeeded = true; + } + + WriteEngine::DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)value.c_str(); + dctTuple.sigSize = value.length(); + dctTuple.isNull = false; + + error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); + + if (error != NO_ERROR) + { + fWEWrapper.closeDctnry(txnId, colType.compressionType); + return false; + } + + colTuple.data = dctTuple.token; + colTupleList.push_back(colTuple); } else { - colStruct.colWidth = colType.colWidth; + WriteEngine::Token nullToken; + colTuple.data = nullToken; + colTupleList.push_back(colTuple); } - colStruct.colDataType = colType.colDataType; + continue; + } - colStructList.push_back(colStruct); - cscColTypeList.push_back(colType); + switch (fetchColTypes[fetchColPos]) + { + case CalpontSystemCatalog::DATE: + { + intColVal = row.getUintField<4>(fetchColPos); + value = DataConvert::dateToString(intColVal); + break; + } + + case CalpontSystemCatalog::DATETIME: + { + intColVal = row.getUintField<8>(fetchColPos); + value = DataConvert::datetimeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + intColVal = row.getUintField<8>(fetchColPos); + value = DataConvert::timestampToString(intColVal, timeZone, colType.precision); + break; + } + + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + { + value = row.getStringField(fetchColPos); + unsigned i = strlen(value.c_str()); + value = value.substr(0, i); + break; + } + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + value = row.getVarBinaryStringField(fetchColPos); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) + { + datatypes::Decimal dec(0, fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos], + row.getBinaryField(fetchColPos)); + value = dec.toString(true); + break; + } + } + /* fall through */ + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::UTINYINT: + { + { + intColVal = row.getIntField(fetchColPos); + + if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDECIMAL && intColVal < 0) + { + intColVal = 0; + } + + if (fetchColScales[fetchColPos] <= 0) + { + ostringstream os; + + if (isUnsigned(fetchColTypes[fetchColPos])) + os << static_cast(intColVal); + else + os << intColVal; + + value = os.str(); + } + else + { + datatypes::Decimal dec(intColVal, fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos]); + value = dec.toString(); + } + } + break; + } + + // In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float dl = row.getFloatField(fetchColPos); + + if (dl == std::numeric_limits::infinity()) + continue; + + if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UFLOAT && dl < 0.0) + { + dl = 0.0; + } + + ostringstream os; + //@Bug 3350 fix the precision. + os << setprecision(7) << dl; + value = os.str(); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dl = row.getDoubleField(fetchColPos); + + if (dl == std::numeric_limits::infinity()) + continue; + + if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDOUBLE && dl < 0.0) + { + dl = 0.0; + } + + ostringstream os; + //@Bug 3350 fix the precision. + os << setprecision(16) << dl; + value = os.str(); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double dll = row.getLongDoubleField(fetchColPos); + + if (dll == std::numeric_limits::infinity()) + continue; + + ostringstream os; + //@Bug 3350 fix the precision. + os << setprecision(19) << dll; + value = os.str(); + break; + } + + default: // treat as int64 + { + ostringstream os; + intColVal = row.getUintField<8>(fetchColPos); + os << intColVal; + value = os.str(); + break; + } + } + + uint32_t funcScale = columnsUpdated[j]->get_funcScale(); + + if (funcScale != 0) + { + string::size_type pos = value.find_first_of("."); // decimal point + + if (pos >= value.length()) + value.insert(value.length(), "."); + + // padding 0 if needed + pos = value.find_first_of("."); + uint32_t digitsAfterPoint = value.length() - pos - 1; + + if (digitsAfterPoint < funcScale) + { + for (uint32_t i = 0; i < (funcScale - digitsAfterPoint); i++) + value += "0"; + } + } + + // check data length + // trim the string if needed + if (value.length() > (unsigned int)colType.colWidth) + { + value = value.substr(0, colType.colWidth); + + if (!pushWarn) + pushWarn = true; + + if (!pushWarning) + pushWarning = true; + + if (pushWarn) + nameNeeded = true; + } + + WriteEngine::DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)value.c_str(); + dctTuple.sigSize = value.length(); + dctTuple.isNull = false; + + error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); + + if (error != NO_ERROR) + { + fWEWrapper.closeDctnry(txnId, colType.compressionType); + rc = error; + WErrorCodes ec; + err = ec.errorString(error); + return rc; + } + + colTuple.data = dctTuple.token; + colTupleList.push_back(colTuple); } + + if (colType.compressionType == 0) + fWEWrapper.closeDctnry(txnId, colType.compressionType, true); + else + fWEWrapper.closeDctnry(txnId, colType.compressionType, false); + + fetchColPos++; + } + else // constant + { + if (columnsUpdated[j]->get_isnull()) + { + if ((colType.defaultValue.length() <= 0) && + (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + rc = 1; + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + return rc; + } + else if (colType.defaultValue.length() > 0) + { + value = colType.defaultValue; + + if (value.length() > (unsigned int)colType.colWidth) + { + value = value.substr(0, colType.colWidth); + pushWarn = true; + + if (!pushWarning) + { + pushWarning = true; + } + + if (pushWarn) + nameNeeded = true; + } + + WriteEngine::DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)value.c_str(); + dctTuple.sigSize = value.length(); + dctTuple.isNull = false; + error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); + + if (error != NO_ERROR) + { + fWEWrapper.closeDctnry(txnId, colType.compressionType); + rc = error; + WErrorCodes ec; + err = ec.errorString(error); + return rc; + } + + colTuple.data = dctTuple.token; + + if (colType.compressionType == 0) + fWEWrapper.closeDctnry(txnId, colType.compressionType, true); + else + fWEWrapper.closeDctnry(txnId, colType.compressionType, + false); // Constant only need to tokenize once. + } + else + { + WriteEngine::Token nullToken; + colTuple.data = nullToken; + } + } + else + { + value = columnsUpdated[j]->get_Data(); + + if (value.length() > (unsigned int)colType.colWidth) + { + value = value.substr(0, colType.colWidth); + pushWarn = true; + + if (!pushWarning) + { + pushWarning = true; + } + + if (pushWarn) + nameNeeded = true; + } + + WriteEngine::DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)value.c_str(); + dctTuple.sigSize = value.length(); + dctTuple.isNull = false; + error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType); + + if (error != NO_ERROR) + { + fWEWrapper.closeDctnry(txnId, colType.compressionType); + rc = error; + WErrorCodes ec; + err = ec.errorString(error); + return rc; + } + + colTuple.data = dctTuple.token; + + if (colType.compressionType == 0) + fWEWrapper.closeDctnry(txnId, colType.compressionType, true); + else + fWEWrapper.closeDctnry(txnId, colType.compressionType, + false); // Constant only need to tokenize once. + } + + for (unsigned row = 0; row < rowsThisRowgroup; row++) + colTupleList.push_back(colTuple); + } } - catch (exception& ex) + else // Non dictionary column { - err = ex.what(); - rc = 1; - return rc; + colStruct.colWidth = colType.colWidth; + + if (columnsUpdated[j]->get_isFromCol()) + { + for (unsigned i = 0; i < rowsThisRowgroup; i++) + { + rowGroups[txnId]->getRow(i, &row); + + if (row.isNullValue(fetchColPos)) + { + isNull = true; + value = ""; + } + else + { + isNull = false; + + switch (fetchColTypes[fetchColPos]) + { + case CalpontSystemCatalog::DATE: + { + intColVal = row.getUintField<4>(fetchColPos); + value = DataConvert::dateToString(intColVal); + break; + } + + case CalpontSystemCatalog::DATETIME: + { + intColVal = row.getUintField<8>(fetchColPos); + value = DataConvert::datetimeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::TIMESTAMP: + { + intColVal = row.getUintField<8>(fetchColPos); + value = DataConvert::timestampToString(intColVal, timeZone, colType.precision); + break; + } + + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + { + value = row.getStringField(fetchColPos); + unsigned i = strlen(value.c_str()); + value = value.substr(0, i); + break; + } + + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + value = row.getVarBinaryStringField(fetchColPos); + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) + { + datatypes::Decimal dec(0, fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos], + row.getBinaryField(fetchColPos)); + value = dec.toString(true); + break; + } + } + /* fall through */ + + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::UTINYINT: + { + { + intColVal = row.getIntField(fetchColPos); + + if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDECIMAL && intColVal < 0) + { + intColVal = 0; + } + + if (fetchColScales[fetchColPos] <= 0) + { + ostringstream os; + + if (isUnsigned(fetchColTypes[fetchColPos])) + os << static_cast(intColVal); + else + os << intColVal; + + value = os.str(); + } + else + { + datatypes::Decimal dec(intColVal, fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos]); + value = dec.toString(); + } + } + break; + } + + // In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float dl = row.getFloatField(fetchColPos); + + if (dl == std::numeric_limits::infinity()) + continue; + + if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UFLOAT && dl < 0.0) + { + dl = 0.0; + } + + ostringstream os; + //@Bug 3350 fix the precision. + os << setprecision(7) << dl; + value = os.str(); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dl = row.getDoubleField(fetchColPos); + + if (dl == std::numeric_limits::infinity()) + continue; + + if (fetchColTypes[fetchColPos] == CalpontSystemCatalog::UDOUBLE && dl < 0.0) + { + dl = 0.0; + } + + ostringstream os; + //@Bug 3350 fix the precision. + os << setprecision(16) << dl; + value = os.str(); + break; + } + + case CalpontSystemCatalog::LONGDOUBLE: + { + long double dll = row.getLongDoubleField(fetchColPos); + + if (dll == std::numeric_limits::infinity()) + continue; + + ostringstream os; + //@Bug 3350 fix the precision. + os << setprecision(19) << dll; + value = os.str(); + break; + } + + default: // treat as int64 + { + ostringstream os; + intColVal = row.getUintField<8>(fetchColPos); + os << intColVal; + value = os.str(); + break; + } + } + } + + uint32_t funcScale = columnsUpdated[j]->get_funcScale(); + + if (funcScale != 0) + { + string::size_type pos = value.find_first_of("."); // decimal point + + if (pos >= value.length()) + value.insert(value.length(), "."); + + // padding 0 if needed + pos = value.find_first_of("."); + uint32_t digitsAfterPoint = value.length() - pos - 1; + + if (digitsAfterPoint < funcScale) + { + for (uint32_t i = 0; i < (funcScale - digitsAfterPoint); i++) + value += "0"; + } + } + + // Check NOT NULL constraint and default value + if ((isNull) && (colType.defaultValue.length() <= 0) && + (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + rc = 1; + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + return rc; + } + else if ((isNull) && (colType.defaultValue.length() > 0)) + { + isNull = false; + bool oneWarn = false; + + try + { + datavalue = + colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); + } + catch (exception&) + { + //@Bug 2624. Error out on conversion failure + rc = 1; + Message::Args args; + args.add(string("'") + colType.defaultValue + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + } + + if ((pushWarn) && (!oneWarn)) + oneWarn = true; + + colTuple.data = datavalue; + colTupleList.push_back(colTuple); + + if (oneWarn) + pushWarn = true; + + if (!pushWarning) + { + pushWarning = pushWarn; + } + + if (pushWarn) + nameNeeded = true; + } + else + { + try + { + datavalue = colType.convertColumnData(value, pushWarn, timeZone, isNull, false, false); + } + catch (exception&) + { + //@Bug 2624. Error out on conversion failure + rc = 1; + Message::Args args; + args.add(string("'") + value + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + return rc; + } + + colTuple.data = datavalue; + colTupleList.push_back(colTuple); + + if (!pushWarning) + { + pushWarning = pushWarn; + } + + if (pushWarn) + nameNeeded = true; + } + } + + fetchColPos++; + } + else // constant column + { + if (columnsUpdated[j]->get_isnull()) + { + isNull = true; + } + else + { + isNull = false; + } + + string inData(columnsUpdated[j]->get_Data()); + + if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (inData == "0000-00-00")) || + ((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && + (inData == "0000-00-00 00:00:00")) || + ((colType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) && + (inData == "0000-00-00 00:00:00"))) + { + isNull = true; + } + + uint64_t nextVal = 0; + + if (colType.autoincrement) + { + try + { + nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); + fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + } + + if (colType.autoincrement && (isNull || (inData.compare("0") == 0))) + { + // reserve nextVal + try + { + bool reserved = fDbrm.getAIRange(oid, rowsThisRowgroup, &nextVal); + + if (!reserved) + { + err = IDBErrorInfo::instance()->errorMsg(ERR_EXCEED_LIMIT); + rc = 1; + return rc; + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + isNull = false; + bool oneWarn = false; + + for (unsigned row = 0; row < rowsThisRowgroup; row++) + { + ostringstream oss; + oss << nextVal++; + inData = oss.str(); + + try + { + datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, false); + } + catch (exception&) + { + //@Bug 2624. Error out on conversion failure + rc = 1; + Message::Args args; + args.add(string("'") + inData + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + } + + if ((pushWarn) && (!oneWarn)) + oneWarn = true; + + colTuple.data = datavalue; + colTupleList.push_back(colTuple); + } + + if (oneWarn) + pushWarn = true; + + if (!pushWarning) + { + pushWarning = pushWarn; + } + + if (pushWarn) + nameNeeded = true; + } + else if (isNull && (colType.defaultValue.length() <= 0) && + (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + rc = 1; + Message::Args args; + args.add(tableColName.column); + err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args); + return rc; + } + else if (isNull && (colType.defaultValue.length() > 0)) + { + isNull = false; + bool oneWarn = false; + + for (unsigned row = 0; row < rowsThisRowgroup; row++) + { + try + { + datavalue = + colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); + } + catch (exception&) + { + //@Bug 2624. Error out on conversion failure + rc = 1; + Message::Args args; + args.add(string("'") + colType.defaultValue + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + } + + if ((pushWarn) && (!oneWarn)) + oneWarn = true; + + colTuple.data = datavalue; + colTupleList.push_back(colTuple); + } + + if (oneWarn) + pushWarn = true; + + if (!pushWarning) + { + pushWarning = pushWarn; + } + + if (pushWarn) + nameNeeded = true; + } + else + { + try + { + datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, true); + } + catch (exception& ex) + { + //@Bug 2624. Error out on conversion failure + rc = 1; + cout << ex.what() << endl; + Message::Args args; + args.add(string("'") + inData + string("'")); + err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args); + return rc; + } + + colTuple.data = datavalue; + + if (!pushWarning) + { + pushWarning = pushWarn; + } + + if (pushWarn) + nameNeeded = true; + + for (unsigned row = 0; row < rowsThisRowgroup; row++) + colTupleList.push_back(colTuple); + } + } } - std::vector colExtentsStruct; - std::vector colExtentsColType; - std::vector colOldValueList; - std::vector ridLists; - colExtentsStruct.push_back(colStructList); - colExtentsColType.push_back(cscColTypeList); - ridLists.push_back(rowIDList); - int error = 0; - - error = fWEWrapper.deleteRow(txnId, colExtentsColType, colExtentsStruct, colOldValueList, ridLists, roPair.objnum); - - if (error != NO_ERROR) + if (nameNeeded) { - rc = error; - //cout << "WE Error code " << error << endl; - WErrorCodes ec; - err = ec.errorString(error); + colNames.push_back(tableColName.column); + } - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - } - else if (error == ERR_BRM_VB_OVERFLOW) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } + colStructList.push_back(colStruct); + colValueList.push_back(colTupleList); + cscColTypeList.push_back(colType); + } // end of bulding values and column structure. + + // timer.stop("fetch values"); + if (rowIDLists.size() > 0) + { + error = fWEWrapper.updateColumnRecs(txnId, cscColTypeList, colStructList, colValueList, rowIDLists, + tableRO.objnum); + } + + if (error != NO_ERROR) + { + rc = error; + WErrorCodes ec; + err = ec.errorString(error); + + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; + } + else if (error == ERR_BRM_VB_OVERFLOW) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + } + + if (pushWarning) + { + rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; + Message::Args args; + string cols = "'" + colNames[0] + "'"; + + for (unsigned i = 1; i < colNames.size(); i++) + { + cols = cols + ", " + "'" + colNames[i] + "'"; + } + + args.add(cols); + err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args); + } + + return rc; +} + +uint8_t WE_DMLCommandProc::getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId) +{ + uint8_t rc = 0; + uint32_t txnId; + vector lbidList; + + bs >> txnId; + std::tr1::unordered_map m_txnLBIDMap = fWEWrapper.getTxnMap(); + + try + { + auto mapIter = m_txnLBIDMap.find(txnId); + + if (mapIter != m_txnLBIDMap.end()) + { + SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; + lbidList = spTxnLBIDRec->m_LBIDs; + } + } + catch (...) + { + } + + bs.restart(); + + try + { + serializeInlineVector(bs, lbidList); + } + catch (exception& ex) + { + // Append to errmsg in case we already have an error + if (err.length() > 0) + err += "; "; + + err += ex.what(); + rc = 1; + return rc; + } + + return rc; +} + +uint8_t WE_DMLCommandProc::processFlushFiles(messageqcpp::ByteStream& bs, std::string& err) +{ + uint8_t rc = 0; + uint32_t flushCode, txnId, tableOid; + int error; + bs >> flushCode; + bs >> txnId; + bs >> tableOid; + std::map oids; + CalpontSystemCatalog::TableName aTableName; + CalpontSystemCatalog::RIDList ridList; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(txnId); + // execplan::CalpontSystemCatalog::ColType colType; + CalpontSystemCatalog::DictOIDList dictOids; + + if (tableOid >= 3000) + { + try + { + aTableName = systemCatalogPtr->tableName(tableOid); + } + catch (...) + { + err = std::string("Systemcatalog error for tableoid ") + std::to_string(tableOid); + rc = 1; + return rc; + } + + dictOids = systemCatalogPtr->dictOIDs(aTableName); + + for (unsigned i = 0; i < dictOids.size(); i++) + { + oids[dictOids[i].dictOID] = dictOids[i].dictOID; + } + + // if (dictOids.size() > 0) + // colType = systemCatalogPtr->colTypeDct(dictOids[0].dictOID); + } + + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + vector lbidList; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + // XXX THIS IS WRONG!!! + // save the extent info to mark them invalid, after flush, the meta file will be gone. + std::tr1::unordered_map m_txnLBIDMap = fWEWrapper.getTxnMap(); + + try + { + auto mapIter = m_txnLBIDMap.find(txnId); + + if (mapIter != m_txnLBIDMap.end()) + { + SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; + lbidList = spTxnLBIDRec->m_LBIDs; + } + } + catch (...) + { + } + } + + error = fWEWrapper.flushDataFiles(flushCode, txnId, oids); + + // No need to close files, flushDataFile will close them. + // if (((colType.compressionType > 0 ) && (dictOids.size() > 0)) || (idbdatafile::IDBPolicy::useHdfs())) + // fWEWrapper.closeDctnry(txnId, colType.compressionType, true); + if (error != NO_ERROR) + { + rc = error; + WErrorCodes ec; + err = ec.errorString(error); + } + + // erase rowgroup from the rowGroup map + if (rowGroups[txnId]) + { + delete rowGroups[txnId]; + rowGroups[txnId] = 0; + } + + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); + ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); + ColExtsInfo::iterator aIt; + std::vector files; + std::vector oidsToFlush; + BRM::FileInfo aFile; + + while (it != colsExtsInfoMap.end()) + { + aIt = (it->second).begin(); + aFile.oid = it->first; + oidsToFlush.push_back(aFile.oid); + + while (aIt != (it->second).end()) + { + aFile.partitionNum = aIt->partNum; + aFile.dbRoot = aIt->dbRoot; + aFile.segmentNum = aIt->segNum; + aFile.compType = aIt->compType; + files.push_back(aFile); + // cout <<"Added to files oid:dbroot:part:seg:compType = " << + // aFile.oid<<":"<> PMId; + bs >> sessionID; + bs >> tmp32; + txnId = tmp32; + string schema, tableName; + bs >> schema; + bs >> tableName; + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.setTransId(txnId); + + if (!rowGroups[txnId]) // meta data + { + rowGroups[txnId] = new rowgroup::RowGroup(); + rowGroups[txnId]->deserialize(bs); + // If hdfs, call chunkmanager to set up + + rc = fWEWrapper.startTransaction(txnId); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + err = ec.errorString(rc); } return rc; + } + + rowgroup::RGData rgData; + rgData.deserialize(bs); + rowGroups[txnId]->setData(&rgData); + // rowGroups[txnId]->setData(const_cast(bs.buf())); + // get row ids + rowgroup::Row row; + rowGroups[txnId]->initRow(&row); + WriteEngine::RIDList rowIDList; + CalpontSystemCatalog::RID rid; + uint32_t rowsThisRowgroup = rowGroups[txnId]->getRowCount(); + uint16_t dbRoot, segment, blockNum; + uint32_t partition; + uint8_t extentNum; + CalpontSystemCatalog::TableName aTableName; + aTableName.schema = schema; + aTableName.table = tableName; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::ROPair roPair; + + CalpontSystemCatalog::RIDList tableRidList; + + try + { + roPair = systemCatalogPtr->tableRID(aTableName); + tableRidList = systemCatalogPtr->columnRIDs(aTableName, true); + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + // querystats + uint64_t relativeRID = 0; + boost::scoped_array preBlkNums(new int[row.getColumnCount()]); + boost::scoped_array colWidth(new uint32_t[row.getColumnCount()]); + + // initialize + for (uint32_t j = 0; j < row.getColumnCount(); j++) + { + preBlkNums[j] = -1; + colWidth[j] = row.getColumnWidth(j); + execplan::CalpontSystemCatalog::ColDataType colDataType = row.getColType(j); + if (colWidth[j] >= 8 && !(colDataType == execplan::CalpontSystemCatalog::DECIMAL || + colDataType == execplan::CalpontSystemCatalog::UDECIMAL)) + { + colWidth[j] = 8; + } + else if (colWidth[j] >= datatypes::MAXDECIMALWIDTH) + { + colWidth[j] = datatypes::MAXDECIMALWIDTH; + } + } + + // Get the file information from rowgroup + dbRoot = rowGroups[txnId]->getDBRoot(); + rowGroups[txnId]->getLocation(&partition, &segment, &extentNum, &blockNum); + WriteEngine::ColStructList colStructList; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::ColStruct colStruct; + colStruct.fColPartition = partition; + colStruct.fColSegment = segment; + colStruct.fColDbRoot = dbRoot; + + for (unsigned i = 0; i < rowsThisRowgroup; i++) + { + rowGroups[txnId]->getRow(i, &row); + rid = row.getRid(); + relativeRID = rid - rowGroups[txnId]->getBaseRid(); + rid = relativeRID; + convertToRelativeRid(rid, extentNum, blockNum); + rowIDList.push_back(rid); + + // populate stats.blocksChanged + for (uint32_t j = 0; j < row.getColumnCount(); j++) + { + if ((int)(relativeRID / (BYTE_PER_BLOCK / colWidth[j])) > preBlkNums[j]) + { + blocksChanged++; + preBlkNums[j] = relativeRID / (BYTE_PER_BLOCK / colWidth[j]); + } + } + } + + try + { + for (unsigned i = 0; i < tableRidList.size(); i++) + { + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(tableRidList[i].objnum); + colStruct.dataOid = tableRidList[i].objnum; + colStruct.tokenFlag = false; + colStruct.fCompressionType = colType.compressionType; + + if (colType.colWidth > 8 && !(colType.colDataType == CalpontSystemCatalog::DECIMAL || + colType.colDataType == CalpontSystemCatalog::UDECIMAL)) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + } + else + { + colStruct.colWidth = colType.colWidth; + } + + colStruct.colDataType = colType.colDataType; + + colStructList.push_back(colStruct); + cscColTypeList.push_back(colType); + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + std::vector colExtentsStruct; + std::vector colExtentsColType; + std::vector colOldValueList; + std::vector ridLists; + colExtentsStruct.push_back(colStructList); + colExtentsColType.push_back(cscColTypeList); + ridLists.push_back(rowIDList); + int error = 0; + + error = fWEWrapper.deleteRow(txnId, colExtentsColType, colExtentsStruct, colOldValueList, ridLists, + roPair.objnum); + + if (error != NO_ERROR) + { + rc = error; + // cout << "WE Error code " << error << endl; + WErrorCodes ec; + err = ec.errorString(error); + + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; + } + else if (error == ERR_BRM_VB_OVERFLOW) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + } + + return rc; } uint8_t WE_DMLCommandProc::processRemoveMeta(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - uint32_t tableOID; + uint8_t rc = 0; + uint32_t tableOID; - try - { - bs >> tableOID; - //std::cout << ": tableOID-" << tableOID << std::endl; + try + { + bs >> tableOID; + // std::cout << ": tableOID-" << tableOID << std::endl; - BulkRollbackMgr::deleteMetaFile( tableOID ); - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - } + BulkRollbackMgr::deleteMetaFile(tableOID); + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Process bulk rollback command //------------------------------------------------------------------------------ -uint8_t WE_DMLCommandProc::processBulkRollback(messageqcpp::ByteStream& bs, - std::string& err) +uint8_t WE_DMLCommandProc::processBulkRollback(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - err.clear(); + uint8_t rc = 0; + err.clear(); - try + try + { + uint32_t tableOID; + uint64_t tableLockID; + std::string tableName; + std::string appName; + + // May want to eventually comment out this logging to stdout, + // but it shouldn't hurt to keep in here. + std::cout << "processBulkRollback"; + bs >> tableLockID; + // std::cout << ": tableLock-"<< tableLockID; + + bs >> tableOID; + // std::cout << "; tableOID-" << tableOID; + + bs >> tableName; + + if (tableName.length() == 0) { - uint32_t tableOID; - uint64_t tableLockID; - std::string tableName; - std::string appName; - - // May want to eventually comment out this logging to stdout, - // but it shouldn't hurt to keep in here. - std::cout << "processBulkRollback"; - bs >> tableLockID; - //std::cout << ": tableLock-"<< tableLockID; - - bs >> tableOID; - //std::cout << "; tableOID-" << tableOID; - - bs >> tableName; - - if (tableName.length() == 0) - { - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(0); - CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(tableOID); - tableName = aTableName.toString(); - - } - - std::cout << "; table-" << tableName; - - bs >> appName; - std::cout << "; app-" << appName << std::endl; - - int we_rc = fWEWrapper.bulkRollback( - tableOID, - tableLockID, - tableName, - appName, - false, // no extra debug logging to the console - err ); - - if (we_rc != NO_ERROR) - rc = 2; - } - catch (exception& ex) - { - std::cout << "processBulkRollback: exception-" << ex.what() << std::endl; - err = ex.what(); - rc = 1; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(0); + CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(tableOID); + tableName = aTableName.toString(); } - return rc; + std::cout << "; table-" << tableName; + + bs >> appName; + std::cout << "; app-" << appName << std::endl; + + int we_rc = fWEWrapper.bulkRollback(tableOID, tableLockID, tableName, appName, + false, // no extra debug logging to the console + err); + + if (we_rc != NO_ERROR) + rc = 2; + } + catch (exception& ex) + { + std::cout << "processBulkRollback: exception-" << ex.what() << std::endl; + err = ex.what(); + rc = 1; + } + + return rc; } //------------------------------------------------------------------------------ // Process bulk rollback cleanup command (deletes bulk rollback meta data files) //------------------------------------------------------------------------------ -uint8_t WE_DMLCommandProc::processBulkRollbackCleanup( - messageqcpp::ByteStream& bs, - std::string& err) +uint8_t WE_DMLCommandProc::processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string& err) { - uint8_t rc = 0; - err.clear(); + uint8_t rc = 0; + err.clear(); - try - { - uint32_t tableOID; + try + { + uint32_t tableOID; - // May want to eventually comment out this logging to stdout, - // but it shouldn't hurt to keep in here. - std::cout << "processBulkRollbackCleanup"; - bs >> tableOID; - std::cout << ": tableOID-" << tableOID << std::endl; + // May want to eventually comment out this logging to stdout, + // but it shouldn't hurt to keep in here. + std::cout << "processBulkRollbackCleanup"; + bs >> tableOID; + std::cout << ": tableOID-" << tableOID << std::endl; - BulkRollbackMgr::deleteMetaFile( tableOID ); - } - catch (exception& ex) - { - std::cout << "processBulkRollbackCleanup: exception-" << ex.what() << - std::endl; - err = ex.what(); - rc = 1; - } + BulkRollbackMgr::deleteMetaFile(tableOID); + } + catch (exception& ex) + { + std::cout << "processBulkRollbackCleanup: exception-" << ex.what() << std::endl; + err = ex.what(); + rc = 1; + } - return rc; + return rc; } uint8_t WE_DMLCommandProc::updateSyscolumnNextval(ByteStream& bs, std::string& err) { - uint32_t columnOid, sessionID; - uint64_t nextVal; - int rc = 0; - bs >> columnOid; - bs >> nextVal; - bs >> sessionID; - uint16_t dbRoot; - std::map oids; - //std::vector oidsToFlush; - oids[columnOid] = columnOid; - //oidsToFlush.push_back(columnOid); - BRM::OID_t oid = 1021; - fDbrm.getSysCatDBRoot(oid, dbRoot); - fWEWrapper.setTransId(sessionID); - fWEWrapper.setBulkFlag(false); - fWEWrapper.startTransaction(sessionID); - //cout << "updateSyscolumnNextval startTransaction id " << sessionID << endl; - rc = fWEWrapper.updateNextValue(sessionID, columnOid, nextVal, sessionID, dbRoot); + uint32_t columnOid, sessionID; + uint64_t nextVal; + int rc = 0; + bs >> columnOid; + bs >> nextVal; + bs >> sessionID; + uint16_t dbRoot; + std::map oids; + // std::vector oidsToFlush; + oids[columnOid] = columnOid; + // oidsToFlush.push_back(columnOid); + BRM::OID_t oid = 1021; + fDbrm.getSysCatDBRoot(oid, dbRoot); + fWEWrapper.setTransId(sessionID); + fWEWrapper.setBulkFlag(false); + fWEWrapper.startTransaction(sessionID); + // cout << "updateSyscolumnNextval startTransaction id " << sessionID << endl; + rc = fWEWrapper.updateNextValue(sessionID, columnOid, nextVal, sessionID, dbRoot); - if (rc != 0) + if (rc != 0) + { + err = "Error in WE::updateNextValue"; + rc = 1; + } + + if (idbdatafile::IDBPolicy::useHdfs()) + { + cout << "updateSyscolumnNextval flushDataFiles " << endl; + int rc1 = fWEWrapper.flushDataFiles(rc, sessionID, oids); + + if ((rc == 0) && (rc1 == 0)) { - err = "Error in WE::updateNextValue"; - rc = 1; + cout << "updateSyscolumnNextval confirmTransaction rc =0 " << endl; + rc1 = fWEWrapper.confirmTransaction(sessionID); + cout << "updateSyscolumnNextval confirmTransaction return code is " << rc1 << endl; + + if (rc1 == NO_ERROR) + rc1 = fWEWrapper.endTransaction(sessionID, true); + else + fWEWrapper.endTransaction(sessionID, false); + } + else + { + cout << "updateSyscolumnNextval endTransaction with error " << endl; + fWEWrapper.endTransaction(sessionID, false); } - if (idbdatafile::IDBPolicy::useHdfs()) - { - cout << "updateSyscolumnNextval flushDataFiles " << endl; - int rc1 = fWEWrapper.flushDataFiles(rc, sessionID, oids); + if (rc == NO_ERROR) + rc = rc1; + } - if ((rc == 0) && ( rc1 == 0)) - { - cout << "updateSyscolumnNextval confirmTransaction rc =0 " << endl; - rc1 = fWEWrapper.confirmTransaction(sessionID); - cout << "updateSyscolumnNextval confirmTransaction return code is " << rc1 << endl; - - if ( rc1 == NO_ERROR) - rc1 = fWEWrapper.endTransaction(sessionID, true); - else - fWEWrapper.endTransaction(sessionID, false); - } - else - { - cout << "updateSyscolumnNextval endTransaction with error " << endl; - fWEWrapper.endTransaction(sessionID, false); - - } - - if ( rc == NO_ERROR) - rc = rc1; - } - - //if (idbdatafile::IDBPolicy::useHdfs()) - // cacheutils::flushOIDsFromCache(oidsToFlush); - return rc; + // if (idbdatafile::IDBPolicy::useHdfs()) + // cacheutils::flushOIDsFromCache(oidsToFlush); + return rc; } uint8_t WE_DMLCommandProc::processPurgeFDCache(ByteStream& bs, std::string& err) { - int rc = 0; - uint32_t tableOid; - bs >> tableOid; - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - std::vector files; - BRM::FileInfo aFile; + int rc = 0; + uint32_t tableOid; + bs >> tableOid; + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + ColsExtsInfoMap colsExtsInfoMap = aTbaleMetaData->getColsExtsInfoMap(); + ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); + ColExtsInfo::iterator aIt; + std::vector files; + BRM::FileInfo aFile; - while (it != colsExtsInfoMap.end()) + while (it != colsExtsInfoMap.end()) + { + aIt = (it->second).begin(); + aFile.oid = it->first; + + while (aIt != (it->second).end()) { - aIt = (it->second).begin(); - aFile.oid = it->first; - - while (aIt != (it->second).end()) - { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot = aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - //cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"<partNum; + aFile.dbRoot = aIt->dbRoot; + aFile.segmentNum = aIt->segNum; + aFile.compType = aIt->compType; + files.push_back(aFile); + // cout <<"Added to files oid:dbroot:part:seg:compType = " << + // aFile.oid<<":"< 0) ) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + it++; + } - TableMetaData::removeTableMetaData(tableOid); - return rc; + if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + + TableMetaData::removeTableMetaData(tableOid); + return rc; } -uint8_t WE_DMLCommandProc::processFixRows(messageqcpp::ByteStream& bs, - std::string& err, - ByteStream::quadbyte& PMId) +uint8_t WE_DMLCommandProc::processFixRows(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId) { - uint8_t rc = 0; - //cout << " In processFixRows" << endl; - uint32_t tmp32; - uint64_t sessionID; - uint16_t dbRoot, segment; - uint32_t partition; - string schema, tableName; - TxnID txnId; - uint8_t tmp8; - bool firstBatch = false; - WriteEngine::RIDList rowIDList; - bs >> PMId; - bs >> tmp8; - firstBatch = (tmp8 != 0); - bs >> sessionID; - bs >> tmp32; - txnId = tmp32; + uint8_t rc = 0; + // cout << " In processFixRows" << endl; + uint32_t tmp32; + uint64_t sessionID; + uint16_t dbRoot, segment; + uint32_t partition; + string schema, tableName; + TxnID txnId; + uint8_t tmp8; + bool firstBatch = false; + WriteEngine::RIDList rowIDList; + bs >> PMId; + bs >> tmp8; + firstBatch = (tmp8 != 0); + bs >> sessionID; + bs >> tmp32; + txnId = tmp32; - bs >> schema; - bs >> tableName; - bs >> dbRoot; - bs >> partition; - bs >> segment; + bs >> schema; + bs >> tableName; + bs >> dbRoot; + bs >> partition; + bs >> segment; - deserializeInlineVector(bs, rowIDList); + deserializeInlineVector(bs, rowIDList); - //Need to identify whether this is the first batch to start transaction. - if (firstBatch) + // Need to identify whether this is the first batch to start transaction. + if (firstBatch) + { + rc = fWEWrapper.startTransaction(txnId); + + if (rc != NO_ERROR) { - rc = fWEWrapper.startTransaction(txnId); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - err = ec.errorString(rc); - return rc; - } - - fWEWrapper.setIsInsert(false); - fWEWrapper.setBulkFlag(false); - fWEWrapper.setTransId(txnId); - fWEWrapper.setFixFlag(true); + WErrorCodes ec; + err = ec.errorString(rc); + return rc; } + fWEWrapper.setIsInsert(false); + fWEWrapper.setBulkFlag(false); + fWEWrapper.setTransId(txnId); + fWEWrapper.setFixFlag(true); + } - CalpontSystemCatalog::TableName aTableName; - aTableName.schema = schema; - aTableName.table = tableName; - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::TableName aTableName; + aTableName.schema = schema; + aTableName.table = tableName; + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - CalpontSystemCatalog::ROPair roPair; + CalpontSystemCatalog::ROPair roPair; - CalpontSystemCatalog::RIDList tableRidList; + CalpontSystemCatalog::RIDList tableRidList; - try - { - roPair = systemCatalogPtr->tableRID( aTableName); - tableRidList = systemCatalogPtr->columnRIDs(aTableName, true); - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - - } - - WriteEngine::ColStructList colStructList; - WriteEngine::ColStruct colStruct; - WriteEngine::DctnryStructList dctnryStructList; - /* colStruct.fColPartition = partition; - colStruct.fColSegment = segment; - colStruct.fColDbRoot = dbRoot; */ - colStruct.fColPartition = 0; - colStruct.fColSegment = 0; - colStruct.fColDbRoot = 3; - //should we always scan dictionary store files? - - try - { - for (unsigned i = 0; i < tableRidList.size(); i++) - { - CalpontSystemCatalog::ColType colType; - colType = systemCatalogPtr->colType( tableRidList[i].objnum ); - colStruct.dataOid = tableRidList[i].objnum; - colStruct.tokenFlag = false; - colStruct.fCompressionType = colType.compressionType; - WriteEngine::DctnryStruct dctnryStruct; - dctnryStruct.fColDbRoot = colStruct.fColDbRoot; - dctnryStruct.fColPartition = colStruct.fColPartition; - dctnryStruct.fColSegment = colStruct.fColSegment; - dctnryStruct.fCompressionType = colStruct.fCompressionType; - dctnryStruct.dctnryOid = 0; - - if (colType.colWidth > 8) //token - { - colStruct.colWidth = 8; - colStruct.tokenFlag = true; - dctnryStruct.dctnryOid = colType.ddn.dictOID; - } - else - { - colStruct.colWidth = colType.colWidth; - } - - colStruct.colDataType = colType.colDataType; - - colStructList.push_back( colStruct ); - dctnryStructList.push_back(dctnryStruct); - } - } - catch (exception& ex) - { - err = ex.what(); - rc = 1; - return rc; - - } - - int error = 0; - - try - { - error = fWEWrapper.deleteBadRows( txnId, colStructList, rowIDList, dctnryStructList); - - if (error != NO_ERROR) - { - rc = error; - //cout << "WE Error code " << error << endl; - WErrorCodes ec; - err = ec.errorString(error); - - if (error == ERR_BRM_DEAD_LOCK) - { - rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; - } - else if (error == ERR_BRM_VB_OVERFLOW) - { - rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; - err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); - } - } - } - catch (std::exception& ex) - { - err = ex.what(); - rc = 1; - } - - //cout << "WES return rc " << (int)rc << " with msg " << err << endl; + try + { + roPair = systemCatalogPtr->tableRID(aTableName); + tableRidList = systemCatalogPtr->columnRIDs(aTableName, true); + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; return rc; + } + + WriteEngine::ColStructList colStructList; + WriteEngine::ColStruct colStruct; + WriteEngine::DctnryStructList dctnryStructList; + /* colStruct.fColPartition = partition; + colStruct.fColSegment = segment; + colStruct.fColDbRoot = dbRoot; */ + colStruct.fColPartition = 0; + colStruct.fColSegment = 0; + colStruct.fColDbRoot = 3; + // should we always scan dictionary store files? + + try + { + for (unsigned i = 0; i < tableRidList.size(); i++) + { + CalpontSystemCatalog::ColType colType; + colType = systemCatalogPtr->colType(tableRidList[i].objnum); + colStruct.dataOid = tableRidList[i].objnum; + colStruct.tokenFlag = false; + colStruct.fCompressionType = colType.compressionType; + WriteEngine::DctnryStruct dctnryStruct; + dctnryStruct.fColDbRoot = colStruct.fColDbRoot; + dctnryStruct.fColPartition = colStruct.fColPartition; + dctnryStruct.fColSegment = colStruct.fColSegment; + dctnryStruct.fCompressionType = colStruct.fCompressionType; + dctnryStruct.dctnryOid = 0; + + if (colType.colWidth > 8) // token + { + colStruct.colWidth = 8; + colStruct.tokenFlag = true; + dctnryStruct.dctnryOid = colType.ddn.dictOID; + } + else + { + colStruct.colWidth = colType.colWidth; + } + + colStruct.colDataType = colType.colDataType; + + colStructList.push_back(colStruct); + dctnryStructList.push_back(dctnryStruct); + } + } + catch (exception& ex) + { + err = ex.what(); + rc = 1; + return rc; + } + + int error = 0; + + try + { + error = fWEWrapper.deleteBadRows(txnId, colStructList, rowIDList, dctnryStructList); + + if (error != NO_ERROR) + { + rc = error; + // cout << "WE Error code " << error << endl; + WErrorCodes ec; + err = ec.errorString(error); + + if (error == ERR_BRM_DEAD_LOCK) + { + rc = dmlpackageprocessor::DMLPackageProcessor::DEAD_LOCK_ERROR; + } + else if (error == ERR_BRM_VB_OVERFLOW) + { + rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; + err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); + } + } + } + catch (std::exception& ex) + { + err = ex.what(); + rc = 1; + } + + // cout << "WES return rc " << (int)rc << " with msg " << err << endl; + return rc; } uint8_t WE_DMLCommandProc::processEndTransaction(ByteStream& bs, std::string& err) { - int rc = 0; - ByteStream::byte tmp8; - bool success; - uint32_t txnid; - bs >> txnid; - bs >> tmp8; - success = (tmp8 != 0); - rc = fWEWrapper.endTransaction(txnid, success); + int rc = 0; + ByteStream::byte tmp8; + bool success; + uint32_t txnid; + bs >> txnid; + bs >> tmp8; + success = (tmp8 != 0); + rc = fWEWrapper.endTransaction(txnid, success); - if (rc != NO_ERROR) - { - WErrorCodes ec; - err = ec.errorString(rc); - } - - return rc; + if (rc != NO_ERROR) + { + WErrorCodes ec; + err = ec.errorString(rc); + } + return rc; } //------------------------------------------------------------------------------ // Validates the correctness of the current HWMs for this table. @@ -4552,277 +4533,255 @@ uint8_t WE_DMLCommandProc::processEndTransaction(ByteStream& bs, std::string& er // HWM, etc to be validated for the columns belonging to jobTable. // stage - Current stage we are validating. "Starting" or "Ending". //------------------------------------------------------------------------------ -int WE_DMLCommandProc::validateColumnHWMs( - CalpontSystemCatalog::RIDList& ridList, - boost::shared_ptr systemCatalogPtr, - const std::vector& segFileInfo, - const char* stage ) +int WE_DMLCommandProc::validateColumnHWMs(CalpontSystemCatalog::RIDList& ridList, + boost::shared_ptr systemCatalogPtr, + const std::vector& segFileInfo, const char* stage) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if ((!fIsFirstBatchPm) && (strcmp(stage, "Starting") == 0)) - return rc; + if ((!fIsFirstBatchPm) && (strcmp(stage, "Starting") == 0)) + return rc; - // Used to track first 1-byte, 2-byte, 4-byte, and 8-byte columns in table - int byte1First = -1; - int byte2First = -1; - int byte4First = -1; - int byte8First = -1; + // Used to track first 1-byte, 2-byte, 4-byte, and 8-byte columns in table + int byte1First = -1; + int byte2First = -1; + int byte4First = -1; + int byte8First = -1; - // Make sure the HWMs for all 1-byte columns match; same for all 2-byte, - // 4-byte, and 8-byte columns as well. - CalpontSystemCatalog::ColType colType; - Convertor convertor; + // Make sure the HWMs for all 1-byte columns match; same for all 2-byte, + // 4-byte, and 8-byte columns as well. + CalpontSystemCatalog::ColType colType; + Convertor convertor; - for (unsigned k = 0; k < segFileInfo.size(); k++) + for (unsigned k = 0; k < segFileInfo.size(); k++) + { + int k1 = 0; + + // Find out column width + colType = systemCatalogPtr->colType(ridList[k].objnum); + colType.colWidth = convertor.getCorrectRowWidth(colType.colDataType, colType.colWidth); + + // Find the first 1-byte, 2-byte, 4-byte, and 8-byte columns. + // Use those as our reference HWM for the respective column widths. + switch (colType.colWidth) { - int k1 = 0; + case 1: + { + if (byte1First == -1) + byte1First = k; - // Find out column width - colType = systemCatalogPtr->colType(ridList[k].objnum); - colType.colWidth = convertor.getCorrectRowWidth(colType.colDataType, colType.colWidth); + k1 = byte1First; + break; + } - // Find the first 1-byte, 2-byte, 4-byte, and 8-byte columns. - // Use those as our reference HWM for the respective column widths. - switch ( colType.colWidth ) - { - case 1: - { - if (byte1First == -1) - byte1First = k; + case 2: + { + if (byte2First == -1) + byte2First = k; - k1 = byte1First; - break; - } + k1 = byte2First; + break; + } - case 2: - { - if (byte2First == -1) - byte2First = k; + case 4: + { + if (byte4First == -1) + byte4First = k; - k1 = byte2First; - break; - } + k1 = byte4First; + break; + } - case 4: - { - if (byte4First == -1) - byte4First = k; + case 8: + default: + { + if (byte8First == -1) + byte8First = k; - k1 = byte4First; - break; - } + k1 = byte8First; + break; + } + } // end of switch based on column width (1,2,4, or 8) - case 8: - default: - { - if (byte8First == -1) - byte8First = k; + // std::cout << "dbg: comparing0 " << stage << " refcol-" << k1 << + // "; wid-" << jobColK1.width << "; hwm-" << segFileInfo[k1].fLocalHwm << + // " col-" << k << + // "; wid-" << jobColK.width << " ; hwm-"< col-" << k << -// "; wid-" << jobColK.width << " ; hwm-"<colType(ridList[k1].objnum); - ostringstream oss; - oss << stage << " HWMs do not match for" - " OID1-" << ridList[k1].objnum << - "; DBRoot-" << segFileInfo[k1].fDbRoot << - "; partition-" << segFileInfo[k1].fPartition << - "; segment-" << segFileInfo[k1].fSegment << - "; hwm-" << segFileInfo[k1].fLocalHwm << - "; width-" << colType2.colWidth << ':' << std::endl << - " and OID2-" << ridList[k].objnum << - "; DBRoot-" << segFileInfo[k].fDbRoot << - "; partition-" << segFileInfo[k].fPartition << - "; segment-" << segFileInfo[k].fSegment << - "; hwm-" << segFileInfo[k].fLocalHwm << - "; width-" << colType.colWidth; - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_ERROR ); - return ERR_BRM_HWMS_NOT_EQUAL; - } - - // HWM DBRoot, partition, and segment number should match for all - // columns; so compare DBRoot, part#, and seg# with first column. - if ((segFileInfo[0].fDbRoot != segFileInfo[k].fDbRoot) || - (segFileInfo[0].fPartition != segFileInfo[k].fPartition) || - (segFileInfo[0].fSegment != segFileInfo[k].fSegment)) - { - CalpontSystemCatalog::ColType colType2; - colType2 = systemCatalogPtr->colType(ridList[0].objnum); - ostringstream oss; - oss << stage << " HWM DBRoot,Part#, or Seg# do not match for" - " OID1-" << ridList[0].objnum << - "; DBRoot-" << segFileInfo[0].fDbRoot << - "; partition-" << segFileInfo[0].fPartition << - "; segment-" << segFileInfo[0].fSegment << - "; hwm-" << segFileInfo[0].fLocalHwm << - "; width-" << colType2.colWidth << ':' << std::endl << - " and OID2-" << ridList[k].objnum << - "; DBRoot-" << segFileInfo[k].fDbRoot << - "; partition-" << segFileInfo[k].fPartition << - "; segment-" << segFileInfo[k].fSegment << - "; hwm-" << segFileInfo[k].fLocalHwm << - "; width-" << colType.colWidth; - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_ERROR ); - return ERR_BRM_HWMS_NOT_EQUAL; - } - } // end of loop to compare all 1-byte HWMs, 2-byte HWMs, etc. - - // Validate/compare HWM for 1-byte column in relation to 2-byte column, etc. - // Without knowing the exact row count, we can't extrapolate the exact HWM - // for the wider column, but we can narrow it down to an expected range. - int refCol = 0; - int colIdx = 0; - - // Validate/compare HWMs given a 1-byte column as a starting point - if (byte1First >= 0) + // Validate that the HWM for this column (k) matches that of the + // corresponding reference column with the same width. + if ((segFileInfo[k1].fDbRoot != segFileInfo[k].fDbRoot) || + (segFileInfo[k1].fPartition != segFileInfo[k].fPartition) || + (segFileInfo[k1].fSegment != segFileInfo[k].fSegment) || + (segFileInfo[k1].fLocalHwm != segFileInfo[k].fLocalHwm)) { - refCol = byte1First; - - if (byte2First >= 0) - { - HWM hwmLo = segFileInfo[byte1First].fLocalHwm * 2; - HWM hwmHi = hwmLo + 1; - - if ((segFileInfo[byte2First].fLocalHwm < hwmLo) || - (segFileInfo[byte2First].fLocalHwm > hwmHi)) - { - colIdx = byte2First; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - goto errorCheck; - } - } - - if (byte4First >= 0) - { - HWM hwmLo = segFileInfo[byte1First].fLocalHwm * 4; - HWM hwmHi = hwmLo + 3; - - if ((segFileInfo[byte4First].fLocalHwm < hwmLo) || - (segFileInfo[byte4First].fLocalHwm > hwmHi)) - { - colIdx = byte4First; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - goto errorCheck; - } - } - - if (byte8First >= 0) - { - HWM hwmLo = segFileInfo[byte1First].fLocalHwm * 8; - HWM hwmHi = hwmLo + 7; - - if ((segFileInfo[byte8First].fLocalHwm < hwmLo) || - (segFileInfo[byte8First].fLocalHwm > hwmHi)) - { - colIdx = byte8First; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - goto errorCheck; - } - } + CalpontSystemCatalog::ColType colType2; + colType2 = systemCatalogPtr->colType(ridList[k1].objnum); + ostringstream oss; + oss << stage + << " HWMs do not match for" + " OID1-" + << ridList[k1].objnum << "; DBRoot-" << segFileInfo[k1].fDbRoot << "; partition-" + << segFileInfo[k1].fPartition << "; segment-" << segFileInfo[k1].fSegment << "; hwm-" + << segFileInfo[k1].fLocalHwm << "; width-" << colType2.colWidth << ':' << std::endl + << " and OID2-" << ridList[k].objnum << "; DBRoot-" << segFileInfo[k].fDbRoot << "; partition-" + << segFileInfo[k].fPartition << "; segment-" << segFileInfo[k].fSegment << "; hwm-" + << segFileInfo[k].fLocalHwm << "; width-" << colType.colWidth; + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_ERROR); + return ERR_BRM_HWMS_NOT_EQUAL; } - // Validate/compare HWMs given a 2-byte column as a starting point + // HWM DBRoot, partition, and segment number should match for all + // columns; so compare DBRoot, part#, and seg# with first column. + if ((segFileInfo[0].fDbRoot != segFileInfo[k].fDbRoot) || + (segFileInfo[0].fPartition != segFileInfo[k].fPartition) || + (segFileInfo[0].fSegment != segFileInfo[k].fSegment)) + { + CalpontSystemCatalog::ColType colType2; + colType2 = systemCatalogPtr->colType(ridList[0].objnum); + ostringstream oss; + oss << stage + << " HWM DBRoot,Part#, or Seg# do not match for" + " OID1-" + << ridList[0].objnum << "; DBRoot-" << segFileInfo[0].fDbRoot << "; partition-" + << segFileInfo[0].fPartition << "; segment-" << segFileInfo[0].fSegment << "; hwm-" + << segFileInfo[0].fLocalHwm << "; width-" << colType2.colWidth << ':' << std::endl + << " and OID2-" << ridList[k].objnum << "; DBRoot-" << segFileInfo[k].fDbRoot << "; partition-" + << segFileInfo[k].fPartition << "; segment-" << segFileInfo[k].fSegment << "; hwm-" + << segFileInfo[k].fLocalHwm << "; width-" << colType.colWidth; + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_ERROR); + return ERR_BRM_HWMS_NOT_EQUAL; + } + } // end of loop to compare all 1-byte HWMs, 2-byte HWMs, etc. + + // Validate/compare HWM for 1-byte column in relation to 2-byte column, etc. + // Without knowing the exact row count, we can't extrapolate the exact HWM + // for the wider column, but we can narrow it down to an expected range. + int refCol = 0; + int colIdx = 0; + + // Validate/compare HWMs given a 1-byte column as a starting point + if (byte1First >= 0) + { + refCol = byte1First; + if (byte2First >= 0) { - refCol = byte2First; + HWM hwmLo = segFileInfo[byte1First].fLocalHwm * 2; + HWM hwmHi = hwmLo + 1; - if (byte4First >= 0) - { - HWM hwmLo = segFileInfo[byte2First].fLocalHwm * 2; - HWM hwmHi = hwmLo + 1; - - if ((segFileInfo[byte4First].fLocalHwm < hwmLo) || - (segFileInfo[byte4First].fLocalHwm > hwmHi)) - { - colIdx = byte4First; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - goto errorCheck; - } - } - - if (byte8First >= 0) - { - HWM hwmLo = segFileInfo[byte2First].fLocalHwm * 4; - HWM hwmHi = hwmLo + 3; - - if ((segFileInfo[byte8First].fLocalHwm < hwmLo) || - (segFileInfo[byte8First].fLocalHwm > hwmHi)) - { - colIdx = byte8First; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - goto errorCheck; - } - } + if ((segFileInfo[byte2First].fLocalHwm < hwmLo) || (segFileInfo[byte2First].fLocalHwm > hwmHi)) + { + colIdx = byte2First; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + goto errorCheck; + } } - // Validate/compare HWMs given a 4-byte column as a starting point if (byte4First >= 0) { - refCol = byte4First; + HWM hwmLo = segFileInfo[byte1First].fLocalHwm * 4; + HWM hwmHi = hwmLo + 3; - if (byte8First >= 0) - { - HWM hwmLo = segFileInfo[byte4First].fLocalHwm * 2; - HWM hwmHi = hwmLo + 1; - - if ((segFileInfo[byte8First].fLocalHwm < hwmLo) || - (segFileInfo[byte8First].fLocalHwm > hwmHi)) - { - colIdx = byte8First; - rc = ERR_BRM_HWMS_OUT_OF_SYNC; - goto errorCheck; - } - } + if ((segFileInfo[byte4First].fLocalHwm < hwmLo) || (segFileInfo[byte4First].fLocalHwm > hwmHi)) + { + colIdx = byte4First; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + goto errorCheck; + } } + if (byte8First >= 0) + { + HWM hwmLo = segFileInfo[byte1First].fLocalHwm * 8; + HWM hwmHi = hwmLo + 7; + + if ((segFileInfo[byte8First].fLocalHwm < hwmLo) || (segFileInfo[byte8First].fLocalHwm > hwmHi)) + { + colIdx = byte8First; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + goto errorCheck; + } + } + } + + // Validate/compare HWMs given a 2-byte column as a starting point + if (byte2First >= 0) + { + refCol = byte2First; + + if (byte4First >= 0) + { + HWM hwmLo = segFileInfo[byte2First].fLocalHwm * 2; + HWM hwmHi = hwmLo + 1; + + if ((segFileInfo[byte4First].fLocalHwm < hwmLo) || (segFileInfo[byte4First].fLocalHwm > hwmHi)) + { + colIdx = byte4First; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + goto errorCheck; + } + } + + if (byte8First >= 0) + { + HWM hwmLo = segFileInfo[byte2First].fLocalHwm * 4; + HWM hwmHi = hwmLo + 3; + + if ((segFileInfo[byte8First].fLocalHwm < hwmLo) || (segFileInfo[byte8First].fLocalHwm > hwmHi)) + { + colIdx = byte8First; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + goto errorCheck; + } + } + } + + // Validate/compare HWMs given a 4-byte column as a starting point + if (byte4First >= 0) + { + refCol = byte4First; + + if (byte8First >= 0) + { + HWM hwmLo = segFileInfo[byte4First].fLocalHwm * 2; + HWM hwmHi = hwmLo + 1; + + if ((segFileInfo[byte8First].fLocalHwm < hwmLo) || (segFileInfo[byte8First].fLocalHwm > hwmHi)) + { + colIdx = byte8First; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + goto errorCheck; + } + } + } + // To avoid repeating this message 6 times in the preceding source code, we // use the "dreaded" goto to branch to this single place for error handling. errorCheck: - if (rc != NO_ERROR) - { - CalpontSystemCatalog::ColType colType1, colType2; - colType1 = systemCatalogPtr->colType(ridList[refCol].objnum); - colType1.colWidth = convertor.getCorrectRowWidth( - colType1.colDataType, colType1.colWidth); + if (rc != NO_ERROR) + { + CalpontSystemCatalog::ColType colType1, colType2; + colType1 = systemCatalogPtr->colType(ridList[refCol].objnum); + colType1.colWidth = convertor.getCorrectRowWidth(colType1.colDataType, colType1.colWidth); - colType2 = systemCatalogPtr->colType(ridList[colIdx].objnum); - colType2.colWidth = convertor.getCorrectRowWidth( - colType2.colDataType, colType2.colWidth); + colType2 = systemCatalogPtr->colType(ridList[colIdx].objnum); + colType2.colWidth = convertor.getCorrectRowWidth(colType2.colDataType, colType2.colWidth); - ostringstream oss; - oss << stage << " HWMs are not in sync for" - " OID1-" << ridList[refCol].objnum << - "; DBRoot-" << segFileInfo[refCol].fDbRoot << - "; partition-" << segFileInfo[refCol].fPartition << - "; segment-" << segFileInfo[refCol].fSegment << - "; hwm-" << segFileInfo[refCol].fLocalHwm << - "; width-" << colType1.colWidth << ':' << std::endl << - " and OID2-" << ridList[colIdx].objnum << - "; DBRoot-" << segFileInfo[colIdx].fDbRoot << - "; partition-" << segFileInfo[colIdx].fPartition << - "; segment-" << segFileInfo[colIdx].fSegment << - "; hwm-" << segFileInfo[colIdx].fLocalHwm << - "; width-" << colType2.colWidth; - fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_ERROR ); - } + ostringstream oss; + oss << stage + << " HWMs are not in sync for" + " OID1-" + << ridList[refCol].objnum << "; DBRoot-" << segFileInfo[refCol].fDbRoot << "; partition-" + << segFileInfo[refCol].fPartition << "; segment-" << segFileInfo[refCol].fSegment << "; hwm-" + << segFileInfo[refCol].fLocalHwm << "; width-" << colType1.colWidth << ':' << std::endl + << " and OID2-" << ridList[colIdx].objnum << "; DBRoot-" << segFileInfo[colIdx].fDbRoot + << "; partition-" << segFileInfo[colIdx].fPartition << "; segment-" << segFileInfo[colIdx].fSegment + << "; hwm-" << segFileInfo[colIdx].fLocalHwm << "; width-" << colType2.colWidth; + fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_ERROR); + } - return rc; -} + return rc; } +} // namespace WriteEngine diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index 4265aa59c..11781233a 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $ -* -*******************************************************************************/ + * $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $ + * + *******************************************************************************/ #ifndef WE_DMLCOMMANDPROC_H__ #define WE_DMLCOMMANDPROC_H__ @@ -49,92 +49,93 @@ namespace WriteEngine { - class WE_DMLCommandProc { -public: - typedef std::vector ColValues; + public: + typedef std::vector ColValues; - EXPORT WE_DMLCommandProc(); - EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs); - EXPORT ~WE_DMLCommandProc(); - inline void isFirstBatchPm (bool firstBatch) + EXPORT WE_DMLCommandProc(); + EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs); + EXPORT ~WE_DMLCommandProc(); + inline void isFirstBatchPm(bool firstBatch) + { + fIsFirstBatchPm = firstBatch; + } + + inline bool isFirstBatchPm() + { + return fIsFirstBatchPm; + } + + // Convert rid from logical block rid to file relative rid + inline void convertToRelativeRid(uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum) + { + rid = rid + extentNum * extentRows + blockNum * 8192; + } + + EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId); + EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, + ByteStream::quadbyte& PMId); + EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, + uint64_t& blocksChanged); + EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, + uint64_t& blocksChanged); + EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string& err); + EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string& err); + EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string& err); + EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string& err); + EXPORT uint8_t processFixRows(ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId); + EXPORT uint8_t getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId); + int validateColumnHWMs(execplan::CalpontSystemCatalog::RIDList& ridList, + boost::shared_ptr systemCatalogPtr, + const std::vector& segFileInfo, const char* stage); + + private: + WriteEngineWrapper fWEWrapper; + boost::scoped_ptr fRBMetaWriter; + std::vector > dbRootExtTrackerVec; + inline bool isDictCol(execplan::CalpontSystemCatalog::ColType& colType) + { + if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || + ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || + ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) || + ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 38)) || + (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || + (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || + (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) { - fIsFirstBatchPm = firstBatch; + return true; } + else + return false; + } + uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID, + const std::vector& files, + const std::vector& oidsToFlush, std::string& err); - inline bool isFirstBatchPm () - { - return fIsFirstBatchPm; - } - - //Convert rid from logical block rid to file relative rid - inline void convertToRelativeRid (uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum) - { - rid = rid + extentNum * extentRows + blockNum * 8192; - } - - EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId); - EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId); - EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged); - EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged); - EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string& err); - EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string& err); - EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string& err); - EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string& err); - EXPORT uint8_t processFixRows(ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId); - EXPORT uint8_t getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId); - int validateColumnHWMs( - execplan::CalpontSystemCatalog::RIDList& ridList, - boost::shared_ptr systemCatalogPtr, - const std::vector& segFileInfo, - const char* stage ); -private: - WriteEngineWrapper fWEWrapper; - boost::scoped_ptr fRBMetaWriter; - std::vector > dbRootExtTrackerVec; - inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType &colType ) - { - if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 38)) - || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) - || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) - || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) - { - return true; - } - else - return false; - } - uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID, - const std::vector& files, - const std::vector& oidsToFlush, - std::string& err); - - bool fIsFirstBatchPm; - std::map rowGroups; - std::map cpackages; - BRM::DBRM fDbrm; - unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt; - Log fLog; - static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; + bool fIsFirstBatchPm; + std::map rowGroups; + std::map cpackages; + BRM::DBRM fDbrm; + unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt; + Log fLog; + static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; }; -} +} // namespace WriteEngine #undef EXPORT #endif diff --git a/writeengine/server/we_getfilesizes.cpp b/writeengine/server/we_getfilesizes.cpp index aa374755f..194b260e3 100644 --- a/writeengine/server/we_getfilesizes.cpp +++ b/writeengine/server/we_getfilesizes.cpp @@ -46,19 +46,19 @@ namespace WriteEngine { struct FileInfo { - uint32_t partition; /** @brief Partition for a file*/ - uint16_t segment; /** @brief Segment for a file */ - uint16_t dbRoot; /** @brief DbRoot for a file */ - std::string segFileName; /** @brief seg file path */ - double fileSize; /** @brief seg file size in giga bytes */ - void serialize(messageqcpp::ByteStream& bs) - { - bs << partition; - bs << segment; - bs << dbRoot; - bs << segFileName; - bs << (*(uint64_t*)(&fileSize)); - } + uint32_t partition; /** @brief Partition for a file*/ + uint16_t segment; /** @brief Segment for a file */ + uint16_t dbRoot; /** @brief DbRoot for a file */ + std::string segFileName; /** @brief seg file path */ + double fileSize; /** @brief seg file size in giga bytes */ + void serialize(messageqcpp::ByteStream& bs) + { + bs << partition; + bs << segment; + bs << dbRoot; + bs << segFileName; + bs << (*(uint64_t*)(&fileSize)); + } }; typedef std::vector Files; typedef std::map columnMap; @@ -67,358 +67,349 @@ allColumnMap wholeMap; boost::mutex columnMapLock; ActiveThreadCounter* activeThreadCounter; -size_t readFillBuffer( - idbdatafile::IDBDataFile* pFile, - char* buffer, - size_t bytesReq) +size_t readFillBuffer(idbdatafile::IDBDataFile* pFile, char* buffer, size_t bytesReq) { - char* pBuf = buffer; - ssize_t nBytes; - size_t bytesToRead = bytesReq; - size_t totalBytesRead = 0; + char* pBuf = buffer; + ssize_t nBytes; + size_t bytesToRead = bytesReq; + size_t totalBytesRead = 0; - while (1) - { - nBytes = pFile->read(pBuf, bytesToRead); + while (1) + { + nBytes = pFile->read(pBuf, bytesToRead); - if (nBytes > 0) - totalBytesRead += nBytes; - else - break; + if (nBytes > 0) + totalBytesRead += nBytes; + else + break; - if ((size_t)nBytes == bytesToRead) - break; + if ((size_t)nBytes == bytesToRead) + break; - pBuf += nBytes; - bytesToRead = bytesToRead - (size_t)nBytes; - } + pBuf += nBytes; + bytesToRead = bytesToRead - (size_t)nBytes; + } - return totalBytesRead; + return totalBytesRead; } static off64_t getCompressedDataSize(string& fileName) { - off64_t dataSize = 0; - IDBDataFile* pFile = 0; - size_t nBytes; - // Some IDBPolicy functions can throw exceptions, caller will catch it - IDBPolicy::configIDBPolicy(); - bool bHdfsFile = IDBPolicy::useHdfs(); + off64_t dataSize = 0; + IDBDataFile* pFile = 0; + size_t nBytes; + // Some IDBPolicy functions can throw exceptions, caller will catch it + IDBPolicy::configIDBPolicy(); + bool bHdfsFile = IDBPolicy::useHdfs(); - if (bHdfsFile) - pFile = IDBDataFile::open(IDBDataFile::HDFS, - fileName.c_str(), "r", 0); - else - pFile = IDBDataFile::open(IDBDataFile::BUFFERED, - fileName.c_str(), "r", 0); + if (bHdfsFile) + pFile = IDBDataFile::open(IDBDataFile::HDFS, fileName.c_str(), "r", 0); + else + pFile = IDBDataFile::open(IDBDataFile::BUFFERED, fileName.c_str(), "r", 0); - if (!pFile) - { - std::ostringstream oss; - oss << "Cannot open file " << fileName << " for read."; - throw std::runtime_error(oss.str()); - } + if (!pFile) + { + std::ostringstream oss; + oss << "Cannot open file " << fileName << " for read."; + throw std::runtime_error(oss.str()); + } - //-------------------------------------------------------------------------- - // Read headers and extract compression pointers - //-------------------------------------------------------------------------- - char hdr1[CompressInterface::HDR_BUF_LEN]; - nBytes = readFillBuffer( pFile, hdr1, CompressInterface::HDR_BUF_LEN); + //-------------------------------------------------------------------------- + // Read headers and extract compression pointers + //-------------------------------------------------------------------------- + char hdr1[CompressInterface::HDR_BUF_LEN]; + nBytes = readFillBuffer(pFile, hdr1, CompressInterface::HDR_BUF_LEN); - if ( nBytes != CompressInterface::HDR_BUF_LEN ) - { - std::ostringstream oss; - oss << "Error reading first header from file " << fileName; - throw std::runtime_error(oss.str()); - } + if (nBytes != CompressInterface::HDR_BUF_LEN) + { + std::ostringstream oss; + oss << "Error reading first header from file " << fileName; + throw std::runtime_error(oss.str()); + } - int64_t ptrSecSize = compress::CompressInterface::getHdrSize(hdr1) - - CompressInterface::HDR_BUF_LEN; - char* hdr2 = new char[ptrSecSize]; - nBytes = readFillBuffer( pFile, hdr2, ptrSecSize); + int64_t ptrSecSize = compress::CompressInterface::getHdrSize(hdr1) - CompressInterface::HDR_BUF_LEN; + char* hdr2 = new char[ptrSecSize]; + nBytes = readFillBuffer(pFile, hdr2, ptrSecSize); - if ( (int64_t)nBytes != ptrSecSize ) - { - std::ostringstream oss; - oss << "Error reading second header from file " << fileName; - throw std::runtime_error(oss.str()); - } + if ((int64_t)nBytes != ptrSecSize) + { + std::ostringstream oss; + oss << "Error reading second header from file " << fileName; + throw std::runtime_error(oss.str()); + } - CompChunkPtrList chunkPtrs; - int rc = - compress::CompressInterface::getPtrList(hdr2, ptrSecSize, chunkPtrs); - delete[] hdr2; + CompChunkPtrList chunkPtrs; + int rc = compress::CompressInterface::getPtrList(hdr2, ptrSecSize, chunkPtrs); + delete[] hdr2; - if (rc != 0) - { - std::ostringstream oss; - oss << "Error decompressing second header from file " << fileName; - throw std::runtime_error(oss.str()); - } + if (rc != 0) + { + std::ostringstream oss; + oss << "Error decompressing second header from file " << fileName; + throw std::runtime_error(oss.str()); + } - unsigned k = chunkPtrs.size(); - // last header's offset + length will be the data bytes - dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second; - delete pFile; - return dataSize; + unsigned k = chunkPtrs.size(); + // last header's offset + length will be the data bytes + dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second; + delete pFile; + return dataSize; } struct ColumnThread { - ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key) - : fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key) - {} - void operator()() + ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key) + : fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key) + { + } + void operator()() + { + Config config; + config.initConfigCache(); + std::vector rootList; + config.getRootIdList(rootList); + FileOp fileOp; + Files aFiles; + + // This function relies on IDBPolicy being initialized by + // IDBPolicy::init(). This is done when WriteEngineServer main() calls + // IDBPolicy::configIDBPolicy(); + IDBDataFile::Types fileType; + bool bUsingHdfs = IDBPolicy::useHdfs(); + + if (bUsingHdfs) + fileType = IDBDataFile::HDFS; + else if (IDBPolicy::useCloud()) + fileType = IDBDataFile::CLOUD; + else + fileType = IDBDataFile::UNBUFFERED; + + IDBFileSystem& fs = IDBFileSystem::getFs(fileType); + + for (uint32_t i = 0; i < rootList.size(); i++) { - Config config; - config.initConfigCache(); - std::vector rootList; - config.getRootIdList( rootList ); - FileOp fileOp; - Files aFiles; + std::vector entries; + (void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]); + std::vector::const_iterator iter = entries.begin(); - // This function relies on IDBPolicy being initialized by - // IDBPolicy::init(). This is done when WriteEngineServer main() calls - // IDBPolicy::configIDBPolicy(); - IDBDataFile::Types fileType; - bool bUsingHdfs = IDBPolicy::useHdfs(); + while (iter != entries.end()) // organize extents into files + { + // Find the size of this file + // string fileName; + char fileName[200]; + (void)fileOp.getFileName(fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum); + string aFile(fileName); // convert between char* and string + off64_t fileSize = 0; - if (bUsingHdfs) - fileType = IDBDataFile::HDFS; - else if (IDBPolicy::useCloud()) - fileType = IDBDataFile::CLOUD; + if (fReportRealUse && (fCompressionType > 0)) + { + try + { + fileSize = getCompressedDataSize(aFile); + } + catch (std::exception& ex) + { + cerr << ex.what(); + } + } else - fileType = IDBDataFile::UNBUFFERED; + fileSize = fs.size(fileName); - IDBFileSystem& fs = IDBFileSystem::getFs( fileType ); - - for (uint32_t i = 0; i < rootList.size(); i++) + if (fileSize > 0) // File exists, add to list { - std::vector entries; - (void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]); - std::vector::const_iterator iter = entries.begin(); - - while ( iter != entries.end() ) //organize extents into files - { - //Find the size of this file - //string fileName; - char fileName[200]; - (void)fileOp.getFileName( fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum); - string aFile(fileName); //convert between char* and string - off64_t fileSize = 0; - - if (fReportRealUse && (fCompressionType > 0)) - { - try - { - fileSize = getCompressedDataSize(aFile); - } - catch (std::exception& ex) - { - cerr << ex.what(); - } - } - else - fileSize = fs.size( fileName ); - - if (fileSize > 0) // File exists, add to list - { - FileInfo aFileInfo; - aFileInfo.partition = entries[0].partitionNum; - aFileInfo.segment = entries[0].segmentNum; - aFileInfo.dbRoot = rootList[i]; - aFileInfo.segFileName = aFile; - aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024); - aFiles.push_back(aFileInfo); - //cout.precision(15); - //cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << "GB" << endl; - } - - //erase the entries from this dbroot. - std::vector entriesTrimed; - - for (uint32_t m = 0; m < entries.size(); m++) - { - if ((entries[0].partitionNum != entries[m].partitionNum) || (entries[0].segmentNum != entries[m].segmentNum)) - entriesTrimed.push_back(entries[m]); - } - - entriesTrimed.swap(entries); - iter = entries.begin(); - } + FileInfo aFileInfo; + aFileInfo.partition = entries[0].partitionNum; + aFileInfo.segment = entries[0].segmentNum; + aFileInfo.dbRoot = rootList[i]; + aFileInfo.segFileName = aFile; + aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024); + aFiles.push_back(aFileInfo); + // cout.precision(15); + // cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << + // "GB" << endl; } - boost::mutex::scoped_lock lk(columnMapLock); - //cout << "Current size of columnsMap is " << columnsMap.size() << endl; - allColumnMap::iterator colMapiter = wholeMap.find(fKey); + // erase the entries from this dbroot. + std::vector entriesTrimed; - if (colMapiter != wholeMap.end()) + for (uint32_t m = 0; m < entries.size(); m++) { - (colMapiter->second)->insert(make_pair(fOid, aFiles)); - activeThreadCounter->decr(); - //cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl; + if ((entries[0].partitionNum != entries[m].partitionNum) || + (entries[0].segmentNum != entries[m].segmentNum)) + entriesTrimed.push_back(entries[m]); } + + entriesTrimed.swap(entries); + iter = entries.begin(); + } } - uint32_t fOid; - int32_t fCompressionType; - bool fReportRealUse; - int fKey; + + boost::mutex::scoped_lock lk(columnMapLock); + // cout << "Current size of columnsMap is " << columnsMap.size() << endl; + allColumnMap::iterator colMapiter = wholeMap.find(fKey); + + if (colMapiter != wholeMap.end()) + { + (colMapiter->second)->insert(make_pair(fOid, aFiles)); + activeThreadCounter->decr(); + // cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl; + } + } + uint32_t fOid; + int32_t fCompressionType; + bool fReportRealUse; + int fKey; }; //------------------------------------------------------------------------------ // Get file size from file name in bytestream object //------------------------------------------------------------------------------ -int WE_GetFileSizes::processFileName( - messageqcpp::ByteStream& bs, - std::string& errMsg, int key) +int WE_GetFileSizes::processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key) { - uint8_t rc = 0; - off_t fileSize = 0; - off_t compressedFileSize = 0; - errMsg.clear(); + uint8_t rc = 0; + off_t fileSize = 0; + off_t compressedFileSize = 0; + errMsg.clear(); - try - { - std::string fileName; + try + { + std::string fileName; - bs >> fileName; - fileSize = IDBPolicy::size(fileName.c_str()); - compressedFileSize = IDBPolicy::compressedSize(fileName.c_str()); - } - catch (std::exception& ex) - { - errMsg = ex.what(); - rc = 1; - } + bs >> fileName; + fileSize = IDBPolicy::size(fileName.c_str()); + compressedFileSize = IDBPolicy::compressedSize(fileName.c_str()); + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = 1; + } - bs.reset(); - bs << fileSize; - bs << compressedFileSize; - return rc; + bs.reset(); + bs << fileSize; + bs << compressedFileSize; + return rc; } //------------------------------------------------------------------------------ // Process a table size based on input from the // bytestream object. //------------------------------------------------------------------------------ -int WE_GetFileSizes::processTable( - messageqcpp::ByteStream& bs, - std::string& errMsg, int key) +int WE_GetFileSizes::processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key) { - uint8_t rc = 0; - errMsg.clear(); + uint8_t rc = 0; + errMsg.clear(); - try + try + { + std::string aTableName; + std::string schemaName; + bool reportRealUse = false; + ByteStream::byte tmp8; + bs >> schemaName; + // cout << "schema: "<< schemaName << endl; + + bs >> aTableName; + // cout << "tableName: " << aTableName << endl; + bs >> tmp8; + reportRealUse = (tmp8 != 0); + + // get column oids + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(0); + CalpontSystemCatalog::TableName tableName; + tableName.schema = schemaName; + tableName.table = aTableName; + CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName); + CalpontSystemCatalog::ColType colType; + CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName); + int serverThreads = 20; + int serverQueueSize = serverThreads * 100; + threadpool::ThreadPool tp(serverThreads, serverQueueSize); + int totalSize = columnList.size() + dictOidList.size(); + activeThreadCounter = new ActiveThreadCounter(totalSize); + + columnMap* columnsMap = new columnMap(); { - std::string aTableName; - std::string schemaName; - bool reportRealUse = false; - ByteStream::byte tmp8; - bs >> schemaName; - //cout << "schema: "<< schemaName << endl; + boost::mutex::scoped_lock lk(columnMapLock); + wholeMap[key] = columnsMap; + } - bs >> aTableName; - //cout << "tableName: " << aTableName << endl; - bs >> tmp8; - reportRealUse = (tmp8 != 0); + for (uint32_t i = 0; i < columnList.size(); i++) + { + colType = systemCatalogPtr->colType(columnList[i].objnum); + tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key)); - //get column oids - boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0); - CalpontSystemCatalog::TableName tableName; - tableName.schema = schemaName; - tableName.table = aTableName; - CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName); - CalpontSystemCatalog::ColType colType; - CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName); - int serverThreads = 20; - int serverQueueSize = serverThreads * 100; - threadpool::ThreadPool tp(serverThreads, serverQueueSize); - int totalSize = columnList.size() + dictOidList.size(); - activeThreadCounter = new ActiveThreadCounter(totalSize); + if (colType.ddn.dictOID > 0) + tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key)); + } - columnMap* columnsMap = new columnMap(); - { - boost::mutex::scoped_lock lk(columnMapLock); - wholeMap[key] = columnsMap; - } - - for (uint32_t i = 0; i < columnList.size(); i++) - { - colType = systemCatalogPtr->colType(columnList[i].objnum); - tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key)); - - if (colType.ddn.dictOID > 0) - tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key)); - } - - /* for (uint32_t i=0; i < dictOidList.size(); i++) - { - tp.invoke(ColumnThread(dictOidList[i].dictOID)); - } */ - //check whether all threads finish - int sleepTime = 100; // sleep 100 milliseconds between checks - struct timespec rm_ts; - - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - uint32_t currentActiveThreads = 10; - - while (currentActiveThreads > 0) - { -#ifdef _MSC_VER - Sleep(sleepTime); -#else - struct timespec abs_ts; - - do + /* for (uint32_t i=0; i < dictOidList.size(); i++) { - abs_ts.tv_sec = rm_ts.tv_sec; - abs_ts.tv_nsec = rm_ts.tv_nsec; - } - while (nanosleep(&abs_ts, &rm_ts) < 0); + tp.invoke(ColumnThread(dictOidList[i].dictOID)); + } */ + // check whether all threads finish + int sleepTime = 100; // sleep 100 milliseconds between checks + struct timespec rm_ts; + + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + uint32_t currentActiveThreads = 10; + + while (currentActiveThreads > 0) + { +#ifdef _MSC_VER + Sleep(sleepTime); +#else + struct timespec abs_ts; + + do + { + abs_ts.tv_sec = rm_ts.tv_sec; + abs_ts.tv_nsec = rm_ts.tv_nsec; + } while (nanosleep(&abs_ts, &rm_ts) < 0); #endif - currentActiveThreads = activeThreadCounter->cur(); - } + currentActiveThreads = activeThreadCounter->cur(); } - catch (std::exception& ex) + } + catch (std::exception& ex) + { + // cout << "WE_GetFileSizes got exception-" << ex.what() << + // std::endl; + errMsg = ex.what(); + rc = 1; + } + + // Build the message to send to the caller + bs.reset(); + boost::mutex::scoped_lock lk(columnMapLock); + allColumnMap::iterator colMapiter = wholeMap.find(key); + + if (colMapiter != wholeMap.end()) + { + columnMap::iterator iter = colMapiter->second->begin(); + uint64_t size; + Files::iterator it; + + while (iter != colMapiter->second->end()) { - //cout << "WE_GetFileSizes got exception-" << ex.what() << - // std::endl; - errMsg = ex.what(); - rc = 1; + bs << iter->first; + // cout << "processTable::coloid = " << iter->first << endl; + + size = iter->second.size(); + bs << size; + + for (it = iter->second.begin(); it != iter->second.end(); it++) + it->serialize(bs); + + // cout << "length now is " << bs.length() << endl; + iter++; } - //Build the message to send to the caller - bs.reset(); - boost::mutex::scoped_lock lk(columnMapLock); - allColumnMap::iterator colMapiter = wholeMap.find(key); + wholeMap.erase(colMapiter); + } - if (colMapiter != wholeMap.end()) - { - columnMap::iterator iter = colMapiter->second->begin(); - uint64_t size; - Files::iterator it; - - while ( iter != colMapiter->second->end()) - { - bs << iter->first; - //cout << "processTable::coloid = " << iter->first << endl; - - size = iter->second.size(); - bs << size; - - for (it = iter->second.begin(); it != iter->second.end(); it++) - it->serialize(bs); - - //cout << "length now is " << bs.length() << endl; - iter++; - } - - wholeMap.erase(colMapiter); - } - - return rc; + return rc; } - -} +} // namespace WriteEngine diff --git a/writeengine/server/we_getfilesizes.h b/writeengine/server/we_getfilesizes.h index 7545955a1..8f246133f 100644 --- a/writeengine/server/we_getfilesizes.h +++ b/writeengine/server/we_getfilesizes.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_getfilesizes.h 4450 2013-04-15 14:13:24Z chao $ -* -*******************************************************************************/ + * $Id: we_getfilesizes.h 4450 2013-04-15 14:13:24Z chao $ + * + *******************************************************************************/ #ifndef WE_GETFILESIZES_H__ #define WE_GETFILESIZES_H__ @@ -31,54 +31,56 @@ namespace WriteEngine { - /** @brief Get all file sizes for the given table */ class WE_GetFileSizes { -public: - - static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key); - static int processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key); + public: + static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key); + static int processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key); }; class ActiveThreadCounter { -public: - ActiveThreadCounter(int size) : factiveThreadCount(size) {} - virtual ~ActiveThreadCounter() {} + public: + ActiveThreadCounter(int size) : factiveThreadCount(size) + { + } + virtual ~ActiveThreadCounter() + { + } - void decr() + void decr() + { + int atc; + + for (;;) { - int atc; + atomicops::atomicMb(); + atc = factiveThreadCount; - for (;;) - { - atomicops::atomicMb(); - atc = factiveThreadCount; + if (atc <= 0) // hopefully atc will never be < 0! + return; - if (atc <= 0) //hopefully atc will never be < 0! - return; + if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1))) + return; - if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1))) - return; - - atomicops::atomicYield(); - } + atomicops::atomicYield(); } + } - uint32_t cur() - { - return factiveThreadCount; - } + uint32_t cur() + { + return factiveThreadCount; + } -private: - ActiveThreadCounter(const ActiveThreadCounter& rhs); - ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs); + private: + ActiveThreadCounter(const ActiveThreadCounter& rhs); + ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs); - volatile int32_t factiveThreadCount; + volatile int32_t factiveThreadCount; }; -} +} // namespace WriteEngine #endif diff --git a/writeengine/server/we_message_handlers.h b/writeengine/server/we_message_handlers.h index ddee04a92..8bb21668c 100644 --- a/writeengine/server/we_message_handlers.h +++ b/writeengine/server/we_message_handlers.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_message_handlers.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_message_handlers.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ #ifndef WE_MESSAGE_HANDLERS_H__ #define WE_MESSAGE_HANDLERS_H__ @@ -26,11 +26,9 @@ namespace WriteEngine { - extern messageqcpp::ByteStream::byte doMsg1(messageqcpp::ByteStream& bs, std::string err); extern messageqcpp::ByteStream::byte doMsg2(messageqcpp::ByteStream& bs, std::string err); extern messageqcpp::ByteStream::byte doUpdateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string err); -} +} // namespace WriteEngine #endif - diff --git a/writeengine/server/we_messages.h b/writeengine/server/we_messages.h index 46870ca4a..e0475c9fc 100644 --- a/writeengine/server/we_messages.h +++ b/writeengine/server/we_messages.h @@ -16,107 +16,102 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_messages.h 4609 2013-04-19 15:32:02Z chao $ -* -*******************************************************************************/ + * $Id: we_messages.h 4609 2013-04-19 15:32:02Z chao $ + * + *******************************************************************************/ #ifndef WE_MESSAGES_H__ #define WE_MESSAGES_H__ namespace WriteEngine { - enum ServerMessages { - WE_SVR_LOOPBACK, - WE_SVR_DDL_KEEPALIVE, - WE_SVR_DML_KEEPALIVE, - WE_UPDATE_NEXTVAL, - WE_SVR_WRITE_SYSTABLE, - WE_SVR_WRITE_SYSCOLUMN, - WE_SVR_WRITE_CREATETABLEFILES, - WE_SVR_COMMIT_VERSION, - WE_SVR_ROLLBACK_BLOCKS, - WE_SVR_ROLLBACK_VERSION, - WE_SVR_DELETE_SYSTABLE, - WE_SVR_DELETE_SYSTABLES, - WE_SVR_DELETE_SYSCOLUMN, - WE_SVR_DELETE_SYSCOLUMN_ROW, - WE_SVR_WRITE_DROPFILES, - WE_SVR_UPDATE_SYSTABLE_AUTO, - WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL, - WE_SVR_UPDATE_SYSCOLUMN_TABLENAME, - WE_SVR_UPDATE_SYSCOLUMN_AUTO, - WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL, - WE_SVR_UPDATE_SYSCOLUMN_COLPOS, - WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN, - WE_SVR_UPDATE_SYSTABLE_TABLENAME, - WE_SVR_UPDATE_SYSTABLES_TABLENAME, - WE_SVR_DROP_PARTITIONS, - WE_SVR_SINGLE_INSERT, - WE_SVR_BATCH_KEEPALIVE, - WE_SVR_BATCH_INSERT, - WE_SVR_BATCH_INSERT_END, - WE_SVR_COMMIT_BATCH_AUTO_ON, - WE_SVR_ROLLBACK_BATCH_AUTO_ON, - WE_SVR_COMMIT_BATCH_AUTO_OFF, - WE_SVR_ROLLBACK_BATCH_AUTO_OFF, - WE_SVR_BATCH_AUTOON_REMOVE_META, - WE_SVR_UPDATE, - WE_SVR_FLUSH_FILES, //35 - WE_SVR_DELETE, - WE_SVR_DML_BULKROLLBACK, - WE_SVR_DML_BULKROLLBACK_CLEANUP, - WE_SVR_FILL_COLUMN, - WE_SVR_WRITE_TRUNCATE, - WE_SVR_WRITE_DROPPARTITION, - WE_SVR_WRITE_DROPTABLE, - WE_SVR_DELETE_DDLLOG, - WE_SVR_FETCH_DDL_LOGS, - WE_SVR_REMOVE_TABLEDATA, - WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL, - WE_SVR_REDISTRIBUTE, - WE_SVR_CLOSE_CONNECTION, - WE_SVR_GET_FILESIZES, - WE_SVR_GET_FILESIZE, - WE_SVR_PURGEFD, - WE_END_TRANSACTION, - WE_SRV_FIX_ROWS, - WE_SVR_WRITE_CREATE_SYSCOLUMN, - WE_SVR_BATCH_INSERT_BINARY, - WE_SVR_GET_WRITTEN_LBIDS, + WE_SVR_LOOPBACK, + WE_SVR_DDL_KEEPALIVE, + WE_SVR_DML_KEEPALIVE, + WE_UPDATE_NEXTVAL, + WE_SVR_WRITE_SYSTABLE, + WE_SVR_WRITE_SYSCOLUMN, + WE_SVR_WRITE_CREATETABLEFILES, + WE_SVR_COMMIT_VERSION, + WE_SVR_ROLLBACK_BLOCKS, + WE_SVR_ROLLBACK_VERSION, + WE_SVR_DELETE_SYSTABLE, + WE_SVR_DELETE_SYSTABLES, + WE_SVR_DELETE_SYSCOLUMN, + WE_SVR_DELETE_SYSCOLUMN_ROW, + WE_SVR_WRITE_DROPFILES, + WE_SVR_UPDATE_SYSTABLE_AUTO, + WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL, + WE_SVR_UPDATE_SYSCOLUMN_TABLENAME, + WE_SVR_UPDATE_SYSCOLUMN_AUTO, + WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL, + WE_SVR_UPDATE_SYSCOLUMN_COLPOS, + WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN, + WE_SVR_UPDATE_SYSTABLE_TABLENAME, + WE_SVR_UPDATE_SYSTABLES_TABLENAME, + WE_SVR_DROP_PARTITIONS, + WE_SVR_SINGLE_INSERT, + WE_SVR_BATCH_KEEPALIVE, + WE_SVR_BATCH_INSERT, + WE_SVR_BATCH_INSERT_END, + WE_SVR_COMMIT_BATCH_AUTO_ON, + WE_SVR_ROLLBACK_BATCH_AUTO_ON, + WE_SVR_COMMIT_BATCH_AUTO_OFF, + WE_SVR_ROLLBACK_BATCH_AUTO_OFF, + WE_SVR_BATCH_AUTOON_REMOVE_META, + WE_SVR_UPDATE, + WE_SVR_FLUSH_FILES, // 35 + WE_SVR_DELETE, + WE_SVR_DML_BULKROLLBACK, + WE_SVR_DML_BULKROLLBACK_CLEANUP, + WE_SVR_FILL_COLUMN, + WE_SVR_WRITE_TRUNCATE, + WE_SVR_WRITE_DROPPARTITION, + WE_SVR_WRITE_DROPTABLE, + WE_SVR_DELETE_DDLLOG, + WE_SVR_FETCH_DDL_LOGS, + WE_SVR_REMOVE_TABLEDATA, + WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL, + WE_SVR_REDISTRIBUTE, + WE_SVR_CLOSE_CONNECTION, + WE_SVR_GET_FILESIZES, + WE_SVR_GET_FILESIZE, + WE_SVR_PURGEFD, + WE_END_TRANSACTION, + WE_SRV_FIX_ROWS, + WE_SVR_WRITE_CREATE_SYSCOLUMN, + WE_SVR_BATCH_INSERT_BINARY, + WE_SVR_GET_WRITTEN_LBIDS, - WE_CLT_SRV_DATA = 100, - WE_CLT_SRV_EOD, - WE_CLT_SRV_CMD, - WE_CLT_SRV_ACK, - WE_CLT_SRV_NAK, - WE_CLT_SRV_PM_ERROR, - WE_CLT_SRV_KEEPALIVE, - WE_CLT_SRV_IMPFILENAME, - WE_CLT_SRV_IMPFILEERROR, - WE_CLT_SRV_CMDLINEARGS, - WE_CLT_SRV_STARTCPI, - WE_CLT_SRV_CPIPASS, - WE_CLT_SRV_CPIFAIL, - WE_CLT_SRV_ROLLBACK, - WE_CLT_SRV_CLEANUP, - WE_CLT_SRV_DATARQST, - WE_CLT_SRV_MODE, - WE_CLT_SRV_DBRCNT, - WE_CLT_SRV_BRMRPT, - WE_CLT_SRV_JOBID, - WE_CLT_SRV_JOBDATA, - WE_CLT_SRV_ERRLOG, - WE_CLT_SRV_BADLOG, - WE_CLT_SRV_CLEAR_TABLE_LOCK, - WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP + WE_CLT_SRV_DATA = 100, + WE_CLT_SRV_EOD, + WE_CLT_SRV_CMD, + WE_CLT_SRV_ACK, + WE_CLT_SRV_NAK, + WE_CLT_SRV_PM_ERROR, + WE_CLT_SRV_KEEPALIVE, + WE_CLT_SRV_IMPFILENAME, + WE_CLT_SRV_IMPFILEERROR, + WE_CLT_SRV_CMDLINEARGS, + WE_CLT_SRV_STARTCPI, + WE_CLT_SRV_CPIPASS, + WE_CLT_SRV_CPIFAIL, + WE_CLT_SRV_ROLLBACK, + WE_CLT_SRV_CLEANUP, + WE_CLT_SRV_DATARQST, + WE_CLT_SRV_MODE, + WE_CLT_SRV_DBRCNT, + WE_CLT_SRV_BRMRPT, + WE_CLT_SRV_JOBID, + WE_CLT_SRV_JOBDATA, + WE_CLT_SRV_ERRLOG, + WE_CLT_SRV_BADLOG, + WE_CLT_SRV_CLEAR_TABLE_LOCK, + WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP }; - - - } #endif - diff --git a/writeengine/server/we_msg1.cpp b/writeengine/server/we_msg1.cpp index 36d1f2b51..f4d124237 100644 --- a/writeengine/server/we_msg1.cpp +++ b/writeengine/server/we_msg1.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ #include @@ -30,11 +30,9 @@ using namespace messageqcpp; namespace WriteEngine { - ByteStream::byte doMsg1(ByteStream& bs, std::string err) { - return 0; -} - + return 0; } +} // namespace WriteEngine diff --git a/writeengine/server/we_msg2.cpp b/writeengine/server/we_msg2.cpp index dff1e995b..4ca03029c 100644 --- a/writeengine/server/we_msg2.cpp +++ b/writeengine/server/we_msg2.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ #include @@ -30,11 +30,9 @@ using namespace messageqcpp; namespace WriteEngine { - ByteStream::byte doMsg2(ByteStream& bs, std::string err) { - return 0; -} - + return 0; } +} // namespace WriteEngine diff --git a/writeengine/server/we_observer.cpp b/writeengine/server/we_observer.cpp index 0629749f9..2761e662f 100644 --- a/writeengine/server/we_observer.cpp +++ b/writeengine/server/we_observer.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * Observer.cpp @@ -31,77 +31,71 @@ #include "we_observer.h" - namespace WriteEngine { - //----------------------------------------------------------------------------- // ctor Observer::Observer() { - } //----------------------------------------------------------------------------- -//dtor +// dtor Observer::~Observer() { - // + // } //----------------------------------------------------------------------------- -//ctor +// ctor Subject::Subject() { - } //----------------------------------------------------------------------------- -//dtor +// dtor Subject::~Subject() { - } //----------------------------------------------------------------------------- void Subject::attach(Observer* Obs) { - boost::mutex::scoped_lock aLstLock; - fObs.push_back(Obs); + boost::mutex::scoped_lock aLstLock; + fObs.push_back(Obs); } //----------------------------------------------------------------------------- void Subject::detach(Observer* Obs) { - boost::mutex::scoped_lock aLstLock; - Observers::iterator aIt = fObs.begin(); + boost::mutex::scoped_lock aLstLock; + Observers::iterator aIt = fObs.begin(); - while (aIt != fObs.end()) + while (aIt != fObs.end()) + { + if ((*aIt) == Obs) { - if ((*aIt) == Obs) - { - fObs.erase(aIt); - break; - } + fObs.erase(aIt); + break; } + } } //----------------------------------------------------------------------------- void Subject::notify() { - boost::mutex::scoped_lock aLstLock; - Observers::iterator aIt = fObs.begin(); + boost::mutex::scoped_lock aLstLock; + Observers::iterator aIt = fObs.begin(); - while (aIt != fObs.end()) - { - (*aIt)->update(this); - } + while (aIt != fObs.end()) + { + (*aIt)->update(this); + } } //----------------------------------------------------------------------------- -}// namespace WriteEngine - +} // namespace WriteEngine diff --git a/writeengine/server/we_observer.h b/writeengine/server/we_observer.h index ac3039318..0f57451f4 100644 --- a/writeengine/server/we_observer.h +++ b/writeengine/server/we_observer.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * Observer.h @@ -32,38 +32,35 @@ #include - namespace WriteEngine { - -class Subject; // forward deceleration +class Subject; // forward deceleration class Observer { -public: - virtual ~Observer(); - virtual bool update(Subject* pSub) = 0; + public: + virtual ~Observer(); + virtual bool update(Subject* pSub) = 0; -protected: - Observer(); + protected: + Observer(); }; class Subject { -public: - Subject(); - virtual ~Subject(); + public: + Subject(); + virtual ~Subject(); - virtual void attach(Observer* Obs); - virtual void detach(Observer* Obs); - virtual void notify(); + virtual void attach(Observer* Obs); + virtual void detach(Observer* Obs); + virtual void notify(); - -private: - typedef std::list Observers; - Observers fObs; + private: + typedef std::list Observers; + Observers fObs; }; -} +} // namespace WriteEngine #endif /* OBSERVER_H_ */ diff --git a/writeengine/server/we_readthread.cpp b/writeengine/server/we_readthread.cpp index f262817f5..9da77a3a2 100644 --- a/writeengine/server/we_readthread.cpp +++ b/writeengine/server/we_readthread.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_readthread.cpp 4609 2013-04-19 15:32:02Z chao $ -* -*******************************************************************************/ + * $Id: we_readthread.cpp 4609 2013-04-19 15:32:02Z chao $ + * + *******************************************************************************/ #include @@ -40,465 +40,460 @@ using namespace threadpool; #include "stopwatch.h" using namespace logging; using namespace WriteEngine; -//StopWatch timer; +// StopWatch timer; namespace WriteEngine { - -ReadThread::ReadThread(const IOSocket& ios): fIos(ios) +ReadThread::ReadThread(const IOSocket& ios) : fIos(ios) { - } ReadThread::~ReadThread() { - } -void ReadThread::operator ()() +void ReadThread::operator()() { - // We should never come here + // We should never come here } //----------------------------------------------------------------------------- -//ctor -DmlReadThread::DmlReadThread(const messageqcpp::IOSocket& ios, - ByteStream& Ibs ): ReadThread(ios), fWeDMLprocessor(new WE_DMLCommandProc), fWeDDLprocessor(new WE_DDLCommandProc) +// ctor +DmlReadThread::DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs) + : ReadThread(ios), fWeDMLprocessor(new WE_DMLCommandProc), fWeDDLprocessor(new WE_DDLCommandProc) { - fIbs = Ibs; + fIbs = Ibs; } -//dtor +// dtor DmlReadThread::~DmlReadThread() { - //cout << "in DmlReadThread destructor" << endl; + // cout << "in DmlReadThread destructor" << endl; } void DmlReadThread::operator()() { - // DCH Since fIbs is a class member, there's no reason to make a copy here. - // Why waste the CPU? Note that Splitter thread doesn't make a copy. - // The only reason I can think of to make such a copy is to guarantee a - // strong exception, but that doesn't appear to be in play here. - ByteStream ibs = fIbs; - ByteStream obs; - ByteStream::byte msgId; - ByteStream::octbyte uniqueID; - ByteStream::quadbyte PMId; - ByteStream::byte rc = 0; - std::string errMsg; - //cout << "DmlReadThread created ..." << endl; - // queryStats.blocksChanged for delete/update - uint64_t blocksChanged = 0; + // DCH Since fIbs is a class member, there's no reason to make a copy here. + // Why waste the CPU? Note that Splitter thread doesn't make a copy. + // The only reason I can think of to make such a copy is to guarantee a + // strong exception, but that doesn't appear to be in play here. + ByteStream ibs = fIbs; + ByteStream obs; + ByteStream::byte msgId; + ByteStream::octbyte uniqueID; + ByteStream::quadbyte PMId; + ByteStream::byte rc = 0; + std::string errMsg; + // cout << "DmlReadThread created ..." << endl; + // queryStats.blocksChanged for delete/update + uint64_t blocksChanged = 0; - while (ibs.length() > 0) + while (ibs.length() > 0) + { + try { - try + errMsg.clear(); + + // do work here... + ibs >> msgId; + + if (msgId != WE_SVR_CLOSE_CONNECTION) + ibs >> uniqueID; + + // cout << "DmlReadThread " << pthread_self () << " received message id " << (uint32_t)msgId << " and + // bytestream length " << ibs.length() << endl; + switch (msgId) + { + case WE_SVR_SINGLE_INSERT: { - errMsg.clear(); - - //do work here... - ibs >> msgId; - - if (msgId != WE_SVR_CLOSE_CONNECTION) - ibs >> uniqueID; - - //cout << "DmlReadThread " << pthread_self () << " received message id " << (uint32_t)msgId << " and bytestream length " << ibs.length() << endl; - switch (msgId) - { - case WE_SVR_SINGLE_INSERT: - { - rc = fWeDMLprocessor->processSingleInsert(ibs, errMsg); - break; - } - - case WE_SVR_COMMIT_VERSION: - { - rc = fWeDMLprocessor->commitVersion(ibs, errMsg); - break; - } - - case WE_SVR_ROLLBACK_BLOCKS: - { - rc = fWeDMLprocessor->rollbackBlocks(ibs, errMsg); - break; - } - - case WE_SVR_ROLLBACK_VERSION: - { - rc = fWeDMLprocessor->rollbackVersion(ibs, errMsg); - break; - } - - case WE_SVR_COMMIT_BATCH_AUTO_ON: - { - rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg); - break; - } - - case WE_SVR_ROLLBACK_BATCH_AUTO_ON: - { - rc = fWeDMLprocessor->rollbackBatchAutoOn(ibs, errMsg); - break; - } - - case WE_SVR_COMMIT_BATCH_AUTO_OFF: - { - rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg); - break; - } - - case WE_SVR_ROLLBACK_BATCH_AUTO_OFF: - { - rc = fWeDMLprocessor->rollbackBatchAutoOff(ibs, errMsg); - break; - } - - case WE_SVR_BATCH_INSERT: - { - rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId); - break; - } - - case WE_SVR_BATCH_INSERT_BINARY: - { - rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId); - break; - } - - case WE_SVR_GET_WRITTEN_LBIDS: - { - rc = fWeDMLprocessor->getWrittenLbids(ibs, errMsg, PMId); - break; - } - - case WE_SVR_BATCH_INSERT_END: - { - rc = fWeDMLprocessor->processBatchInsertHwm(ibs, errMsg); - //timer.finish(); - break; - } - - case WE_SVR_UPDATE: - { - rc = fWeDMLprocessor->processUpdate(ibs, errMsg, PMId, blocksChanged); - break; - } - - case WE_SVR_FLUSH_FILES: - { - rc = fWeDMLprocessor->processFlushFiles(ibs, errMsg); - break; - } - - case WE_SVR_DELETE: - { - rc = fWeDMLprocessor->processDelete(ibs, errMsg, PMId, blocksChanged); - break; - } - - case WE_SVR_BATCH_AUTOON_REMOVE_META: - { - rc = fWeDMLprocessor->processRemoveMeta(ibs, errMsg); - break; - } - - case WE_SVR_DML_BULKROLLBACK: - { - rc = fWeDMLprocessor->processBulkRollback(ibs, errMsg); - break; - } - - case WE_SVR_DML_BULKROLLBACK_CLEANUP: - { - rc = fWeDMLprocessor->processBulkRollbackCleanup(ibs, errMsg); - break; - } - - case WE_UPDATE_NEXTVAL: - { - rc = fWeDMLprocessor->updateSyscolumnNextval(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_SYSTABLE: - { - rc = fWeDDLprocessor->writeSystable(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_SYSCOLUMN: - { - rc = fWeDDLprocessor->writeSyscolumn(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_CREATE_SYSCOLUMN: - { - rc = fWeDDLprocessor->writeCreateSyscolumn(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_CREATETABLEFILES: - { - rc = fWeDDLprocessor->createtablefiles(ibs, errMsg); - break; - } - - case WE_SVR_DELETE_SYSCOLUMN: - { - rc = fWeDDLprocessor->deleteSyscolumn(ibs, errMsg); - break; - } - - case WE_SVR_DELETE_SYSCOLUMN_ROW: - { - rc = fWeDDLprocessor->deleteSyscolumnRow(ibs, errMsg); - break; - } - - case WE_SVR_DELETE_SYSTABLE: - { - rc = fWeDDLprocessor->deleteSystable(ibs, errMsg); - break; - } - - case WE_SVR_DELETE_SYSTABLES: - { - rc = fWeDDLprocessor->deleteSystables(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_DROPFILES: - { - rc = fWeDDLprocessor->dropFiles(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_AUTO: - { - rc = fWeDDLprocessor->updateSyscolumnAuto(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL: - { - rc = fWeDDLprocessor->updateSyscolumnNextvalCol(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL: - { - rc = fWeDDLprocessor->updateSyscolumnNextval(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL: - { - rc = fWeDDLprocessor->updateSyscolumnSetDefault(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_TABLENAME: - { - rc = fWeDDLprocessor->updateSyscolumnTablename(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN: - { - rc = fWeDDLprocessor->updateSyscolumnRenameColumn(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSCOLUMN_COLPOS: - { - rc = fWeDDLprocessor->updateSyscolumnColumnposCol(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSTABLE_AUTO: - { - rc = fWeDDLprocessor->updateSystableAuto(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSTABLE_TABLENAME: - { - rc = fWeDDLprocessor->updateSystableTablename(ibs, errMsg); - break; - } - - case WE_SVR_UPDATE_SYSTABLES_TABLENAME: - { - rc = fWeDDLprocessor->updateSystablesTablename(ibs, errMsg); - break; - } - - case WE_SVR_FILL_COLUMN: - { - rc = fWeDDLprocessor->fillNewColumn(ibs, errMsg); - break; - } - - case WE_SVR_DROP_PARTITIONS: - { - rc = fWeDDLprocessor->dropPartitions(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_TRUNCATE: - { - rc = fWeDDLprocessor->writeTruncateLog(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_DROPPARTITION: - { - rc = fWeDDLprocessor->writeDropPartitionLog(ibs, errMsg); - break; - } - - case WE_SVR_WRITE_DROPTABLE: - { - rc = fWeDDLprocessor->writeDropTableLog(ibs, errMsg); - break; - } - - case WE_SVR_DELETE_DDLLOG: - { - rc = fWeDDLprocessor->deleteDDLLog(ibs, errMsg); - break; - } - - case WE_SVR_FETCH_DDL_LOGS: - { - rc = fWeDDLprocessor->fetchDDLLog(ibs, errMsg); - break; - } - - case WE_SVR_PURGEFD: - { - rc = fWeDMLprocessor->processPurgeFDCache(ibs, errMsg); - break; - } - - case WE_END_TRANSACTION: - { - rc = fWeDMLprocessor->processEndTransaction(ibs, errMsg); - break; - } - - case WE_SRV_FIX_ROWS: - { - rc = fWeDMLprocessor->processFixRows(ibs, errMsg, PMId); - break; - } - - case WE_SVR_CLOSE_CONNECTION: - { - break; - } - - default: - break; - } - } - catch (std::exception& ex) - { - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("we_readthread caught exception "); - args.add(ex.what()); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); - rc = 1; - errMsg = msg.msg(); - } - catch (...) - { - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("we_readthread caught ... exception "); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); - rc = 1; - errMsg = msg.msg(); + rc = fWeDMLprocessor->processSingleInsert(ibs, errMsg); + break; } - if (msgId != WE_SVR_CLOSE_CONNECTION) + case WE_SVR_COMMIT_VERSION: { - //send response - obs.restart(); - obs << uniqueID; - obs << rc; - obs << errMsg; + rc = fWeDMLprocessor->commitVersion(ibs, errMsg); + break; } - if ((msgId == WE_SVR_COMMIT_BATCH_AUTO_ON) || (msgId == WE_SVR_BATCH_INSERT_END) || (msgId == WE_SVR_FETCH_DDL_LOGS) || (msgId == WE_SVR_GET_WRITTEN_LBIDS)) + case WE_SVR_ROLLBACK_BLOCKS: { - obs += ibs; - //cout << " sending back hwm info with ibs length " << endl; - } - else if ((msgId == WE_SVR_BATCH_INSERT) || (msgId == WE_SVR_UPDATE) || (msgId == WE_SVR_DELETE)) - { - obs << PMId; - } - else if ((msgId == WE_SVR_DML_BULKROLLBACK) || - (msgId == WE_SVR_DML_BULKROLLBACK_CLEANUP)) - { - obs << Config::getLocalModuleID(); + rc = fWeDMLprocessor->rollbackBlocks(ibs, errMsg); + break; } - if (msgId == WE_SVR_UPDATE || msgId == WE_SVR_DELETE) - obs << blocksChanged; // send stats back to DMLProc - - blocksChanged = 0; // reset - - if (msgId == WE_SVR_CLOSE_CONNECTION) + case WE_SVR_ROLLBACK_VERSION: { - //cout << "received request. closing connection ..." << endl; - break; + rc = fWeDMLprocessor->rollbackVersion(ibs, errMsg); + break; } - else + + case WE_SVR_COMMIT_BATCH_AUTO_ON: { - try - { - fIos.write(obs); - //cout << "dmlthread sent back response for msgid " << (uint32_t)msgId << " with uniqueID:rc= " - //<< (uint32_t)uniqueID<<":"<< (uint32_t)rc<<" and error message is " << errMsg <commitBatchAutoOn(ibs, errMsg); + break; } + + case WE_SVR_ROLLBACK_BATCH_AUTO_ON: + { + rc = fWeDMLprocessor->rollbackBatchAutoOn(ibs, errMsg); + break; + } + + case WE_SVR_COMMIT_BATCH_AUTO_OFF: + { + rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg); + break; + } + + case WE_SVR_ROLLBACK_BATCH_AUTO_OFF: + { + rc = fWeDMLprocessor->rollbackBatchAutoOff(ibs, errMsg); + break; + } + + case WE_SVR_BATCH_INSERT: + { + rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId); + break; + } + + case WE_SVR_BATCH_INSERT_BINARY: + { + rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId); + break; + } + + case WE_SVR_GET_WRITTEN_LBIDS: + { + rc = fWeDMLprocessor->getWrittenLbids(ibs, errMsg, PMId); + break; + } + + case WE_SVR_BATCH_INSERT_END: + { + rc = fWeDMLprocessor->processBatchInsertHwm(ibs, errMsg); + // timer.finish(); + break; + } + + case WE_SVR_UPDATE: + { + rc = fWeDMLprocessor->processUpdate(ibs, errMsg, PMId, blocksChanged); + break; + } + + case WE_SVR_FLUSH_FILES: + { + rc = fWeDMLprocessor->processFlushFiles(ibs, errMsg); + break; + } + + case WE_SVR_DELETE: + { + rc = fWeDMLprocessor->processDelete(ibs, errMsg, PMId, blocksChanged); + break; + } + + case WE_SVR_BATCH_AUTOON_REMOVE_META: + { + rc = fWeDMLprocessor->processRemoveMeta(ibs, errMsg); + break; + } + + case WE_SVR_DML_BULKROLLBACK: + { + rc = fWeDMLprocessor->processBulkRollback(ibs, errMsg); + break; + } + + case WE_SVR_DML_BULKROLLBACK_CLEANUP: + { + rc = fWeDMLprocessor->processBulkRollbackCleanup(ibs, errMsg); + break; + } + + case WE_UPDATE_NEXTVAL: + { + rc = fWeDMLprocessor->updateSyscolumnNextval(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_SYSTABLE: + { + rc = fWeDDLprocessor->writeSystable(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_SYSCOLUMN: + { + rc = fWeDDLprocessor->writeSyscolumn(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_CREATE_SYSCOLUMN: + { + rc = fWeDDLprocessor->writeCreateSyscolumn(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_CREATETABLEFILES: + { + rc = fWeDDLprocessor->createtablefiles(ibs, errMsg); + break; + } + + case WE_SVR_DELETE_SYSCOLUMN: + { + rc = fWeDDLprocessor->deleteSyscolumn(ibs, errMsg); + break; + } + + case WE_SVR_DELETE_SYSCOLUMN_ROW: + { + rc = fWeDDLprocessor->deleteSyscolumnRow(ibs, errMsg); + break; + } + + case WE_SVR_DELETE_SYSTABLE: + { + rc = fWeDDLprocessor->deleteSystable(ibs, errMsg); + break; + } + + case WE_SVR_DELETE_SYSTABLES: + { + rc = fWeDDLprocessor->deleteSystables(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_DROPFILES: + { + rc = fWeDDLprocessor->dropFiles(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_AUTO: + { + rc = fWeDDLprocessor->updateSyscolumnAuto(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL: + { + rc = fWeDDLprocessor->updateSyscolumnNextvalCol(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL: + { + rc = fWeDDLprocessor->updateSyscolumnNextval(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL: + { + rc = fWeDDLprocessor->updateSyscolumnSetDefault(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_TABLENAME: + { + rc = fWeDDLprocessor->updateSyscolumnTablename(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN: + { + rc = fWeDDLprocessor->updateSyscolumnRenameColumn(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSCOLUMN_COLPOS: + { + rc = fWeDDLprocessor->updateSyscolumnColumnposCol(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSTABLE_AUTO: + { + rc = fWeDDLprocessor->updateSystableAuto(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSTABLE_TABLENAME: + { + rc = fWeDDLprocessor->updateSystableTablename(ibs, errMsg); + break; + } + + case WE_SVR_UPDATE_SYSTABLES_TABLENAME: + { + rc = fWeDDLprocessor->updateSystablesTablename(ibs, errMsg); + break; + } + + case WE_SVR_FILL_COLUMN: + { + rc = fWeDDLprocessor->fillNewColumn(ibs, errMsg); + break; + } + + case WE_SVR_DROP_PARTITIONS: + { + rc = fWeDDLprocessor->dropPartitions(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_TRUNCATE: + { + rc = fWeDDLprocessor->writeTruncateLog(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_DROPPARTITION: + { + rc = fWeDDLprocessor->writeDropPartitionLog(ibs, errMsg); + break; + } + + case WE_SVR_WRITE_DROPTABLE: + { + rc = fWeDDLprocessor->writeDropTableLog(ibs, errMsg); + break; + } + + case WE_SVR_DELETE_DDLLOG: + { + rc = fWeDDLprocessor->deleteDDLLog(ibs, errMsg); + break; + } + + case WE_SVR_FETCH_DDL_LOGS: + { + rc = fWeDDLprocessor->fetchDDLLog(ibs, errMsg); + break; + } + + case WE_SVR_PURGEFD: + { + rc = fWeDMLprocessor->processPurgeFDCache(ibs, errMsg); + break; + } + + case WE_END_TRANSACTION: + { + rc = fWeDMLprocessor->processEndTransaction(ibs, errMsg); + break; + } + + case WE_SRV_FIX_ROWS: + { + rc = fWeDMLprocessor->processFixRows(ibs, errMsg, PMId); + break; + } + + case WE_SVR_CLOSE_CONNECTION: + { + break; + } + + default: break; + } + } + catch (std::exception& ex) + { + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("we_readthread caught exception "); + args.add(ex.what()); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + rc = 1; + errMsg = msg.msg(); + } + catch (...) + { + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("we_readthread caught ... exception "); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + rc = 1; + errMsg = msg.msg(); } - //cout << "closing connection for thread " << pthread_self () << endl; - fIos.close(); -} + if (msgId != WE_SVR_CLOSE_CONNECTION) + { + // send response + obs.restart(); + obs << uniqueID; + obs << rc; + obs << errMsg; + } + if ((msgId == WE_SVR_COMMIT_BATCH_AUTO_ON) || (msgId == WE_SVR_BATCH_INSERT_END) || + (msgId == WE_SVR_FETCH_DDL_LOGS) || (msgId == WE_SVR_GET_WRITTEN_LBIDS)) + { + obs += ibs; + // cout << " sending back hwm info with ibs length " << endl; + } + else if ((msgId == WE_SVR_BATCH_INSERT) || (msgId == WE_SVR_UPDATE) || (msgId == WE_SVR_DELETE)) + { + obs << PMId; + } + else if ((msgId == WE_SVR_DML_BULKROLLBACK) || (msgId == WE_SVR_DML_BULKROLLBACK_CLEANUP)) + { + obs << Config::getLocalModuleID(); + } -//----------------------------------------------------------------------------- -//ctor -SplitterReadThread::SplitterReadThread(const messageqcpp::IOSocket& ios, - ByteStream& Ibs): ReadThread(ios), fWeDataLoader(*this) -{ - fIbs = Ibs; + if (msgId == WE_SVR_UPDATE || msgId == WE_SVR_DELETE) + obs << blocksChanged; // send stats back to DMLProc + + blocksChanged = 0; // reset + + if (msgId == WE_SVR_CLOSE_CONNECTION) + { + // cout << "received request. closing connection ..." << endl; + break; + } + else + { + try + { + fIos.write(obs); + // cout << "dmlthread sent back response for msgid " << (uint32_t)msgId << " with uniqueID:rc= " + //<< (uint32_t)uniqueID<<":"<< (uint32_t)rc<<" and error message is " << errMsg < 0) + while (fIbs.length() > 0) + { + fWeDataLoader.updateRxBytes(fIbs.length()); + + // do work here... + fIbs >> msgId; + + // cout << (int)msgId << endl; + + switch (msgId) { - fWeDataLoader.updateRxBytes(fIbs.length()); + case WE_CLT_SRV_KEEPALIVE: + { + fWeDataLoader.onReceiveKeepAlive(fIbs); + break; + } - //do work here... - fIbs >> msgId; + case WE_CLT_SRV_DATA: + { + fWeDataLoader.onReceiveData(fIbs); + break; + } - //cout << (int)msgId << endl; + case WE_CLT_SRV_EOD: + { + fWeDataLoader.onReceiveEod(fIbs); + break; + } - switch (msgId) - { - case WE_CLT_SRV_KEEPALIVE: - { - fWeDataLoader.onReceiveKeepAlive(fIbs); - break; - } + case WE_CLT_SRV_MODE: + { + fWeDataLoader.onReceiveMode(fIbs); + break; + } - case WE_CLT_SRV_DATA: - { - fWeDataLoader.onReceiveData(fIbs); - break; - } + case WE_CLT_SRV_IMPFILENAME: + { + fWeDataLoader.onReceiveImportFileName(fIbs); + break; + } - case WE_CLT_SRV_EOD: - { - fWeDataLoader.onReceiveEod(fIbs); - break; - } + case WE_CLT_SRV_CMDLINEARGS: + { + fWeDataLoader.onReceiveCmdLineArgs(fIbs); + break; + } - case WE_CLT_SRV_MODE: - { - fWeDataLoader.onReceiveMode(fIbs); - break; - } + case WE_CLT_SRV_CMD: + { + fWeDataLoader.onReceiveCmd(fIbs); // fig out share_ptr on BS& is better + break; + } - case WE_CLT_SRV_IMPFILENAME: - { - fWeDataLoader.onReceiveImportFileName(fIbs); - break; - } + case WE_CLT_SRV_ACK: + { + fWeDataLoader.onReceiveAck(fIbs); + break; + } - case WE_CLT_SRV_CMDLINEARGS: - { - fWeDataLoader.onReceiveCmdLineArgs(fIbs); - break; - } + case WE_CLT_SRV_NAK: + { + fWeDataLoader.onReceiveNak(fIbs); + break; + } - case WE_CLT_SRV_CMD: - { - fWeDataLoader.onReceiveCmd(fIbs); //fig out share_ptr on BS& is better - break; - } + case WE_CLT_SRV_PM_ERROR: + { + fWeDataLoader.onReceiveError(fIbs); + break; + } - case WE_CLT_SRV_ACK: - { - fWeDataLoader.onReceiveAck(fIbs); - break; - } + case WE_CLT_SRV_STARTCPI: + { + fWeDataLoader.onReceiveStartCpimport(); + break; + } - case WE_CLT_SRV_NAK: - { - fWeDataLoader.onReceiveNak(fIbs); - break; - } + case WE_CLT_SRV_BRMRPT: + { + fWeDataLoader.onReceiveBrmRptFileName(fIbs); + break; + } - case WE_CLT_SRV_PM_ERROR: - { - fWeDataLoader.onReceiveError(fIbs); - break; - } + case WE_CLT_SRV_CLEANUP: + { + fWeDataLoader.onReceiveCleanup(fIbs); + break; + } - case WE_CLT_SRV_STARTCPI: - { - fWeDataLoader.onReceiveStartCpimport(); - break; - } + case WE_CLT_SRV_ROLLBACK: + { + fWeDataLoader.onReceiveRollback(fIbs); + break; + } - case WE_CLT_SRV_BRMRPT: - { - fWeDataLoader.onReceiveBrmRptFileName(fIbs); - break; - } + case WE_CLT_SRV_JOBID: + { + fWeDataLoader.onReceiveJobId(fIbs); + break; + } - case WE_CLT_SRV_CLEANUP: - { - fWeDataLoader.onReceiveCleanup(fIbs); - break; - } + case WE_CLT_SRV_JOBDATA: + { + fWeDataLoader.onReceiveJobData(fIbs); + break; + } - case WE_CLT_SRV_ROLLBACK: - { - fWeDataLoader.onReceiveRollback(fIbs); - break; - } + case WE_CLT_SRV_ERRLOG: + { + fWeDataLoader.onReceiveErrFileRqst(fIbs); + break; + } - case WE_CLT_SRV_JOBID: - { - fWeDataLoader.onReceiveJobId(fIbs); - break; - } + case WE_CLT_SRV_BADLOG: + { + fWeDataLoader.onReceiveBadFileRqst(fIbs); + break; + } - case WE_CLT_SRV_JOBDATA: - { - fWeDataLoader.onReceiveJobData(fIbs); - break; - } - - case WE_CLT_SRV_ERRLOG: - { - fWeDataLoader.onReceiveErrFileRqst(fIbs); - break; - } - - case WE_CLT_SRV_BADLOG: - { - fWeDataLoader.onReceiveBadFileRqst(fIbs); - break; - } - - default: - break; - - } - - fIbs.restart(); - - try - { - //get next message - fIbs = fIos.read(); - } - catch (...) - { - fIbs.restart(); //setting length=0, get out of loop - std::cout << "Broken Pipe" << std::endl; - - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("SplitterReadThread::operator: Broken Pipe "); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); - } + default: break; } - fIos.close(); + fIbs.restart(); + try + { + // get next message + fIbs = fIos.read(); + } + catch (...) + { + fIbs.restart(); // setting length=0, get out of loop + std::cout << "Broken Pipe" << std::endl; + + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("SplitterReadThread::operator: Broken Pipe "); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + } + } + + fIos.close(); } //------------------------------------------------------------------------------ // ClearTableLockReadThread constructor. //------------------------------------------------------------------------------ -ClearTableLockReadThread::ClearTableLockReadThread( - const messageqcpp::IOSocket& ios, - ByteStream& Ibs ): ReadThread(ios), - fWeClearTableLockCmd(new WE_ClearTableLockCmd("ClearTableLockTool")) +ClearTableLockReadThread::ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs) + : ReadThread(ios), fWeClearTableLockCmd(new WE_ClearTableLockCmd("ClearTableLockTool")) { - fIbs = Ibs; + fIbs = Ibs; } //------------------------------------------------------------------------------ @@ -688,72 +678,71 @@ ClearTableLockReadThread::~ClearTableLockReadThread() //------------------------------------------------------------------------------ void ClearTableLockReadThread::operator()() { - ByteStream::byte msgId; - ByteStream obs; - ByteStream::byte rc = 0; - std::string errMsg; + ByteStream::byte msgId; + ByteStream obs; + ByteStream::byte rc = 0; + std::string errMsg; - // Read msgid from ByteStream and forward to applicable processing function - while (fIbs.length() > 0) + // Read msgid from ByteStream and forward to applicable processing function + while (fIbs.length() > 0) + { + fIbs >> msgId; + + switch (msgId) { - fIbs >> msgId; + case WE_CLT_SRV_CLEAR_TABLE_LOCK: + { + rc = fWeClearTableLockCmd->processRollback(fIbs, errMsg); + break; + } - switch (msgId) - { - case WE_CLT_SRV_CLEAR_TABLE_LOCK: - { - rc = fWeClearTableLockCmd->processRollback(fIbs, errMsg); - break; - } + case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP: + { + rc = fWeClearTableLockCmd->processCleanup(fIbs, errMsg); + break; + } - case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP: - { - rc = fWeClearTableLockCmd->processCleanup(fIbs, errMsg); - break; - } - - default: - { - break; - } - } - - // Send response - obs.restart(); - obs << rc; - obs << errMsg; - - try - { - fIos.write(obs); - - // Get next message - fIbs = fIos.read(); - } - catch (...) - { - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("ClearTableLockReadThread::operator: Broken Pipe "); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); - break; - } + default: + { + break; + } } - fIos.close(); -} + // Send response + obs.restart(); + obs << rc; + obs << errMsg; + try + { + fIos.write(obs); + + // Get next message + fIbs = fIos.read(); + } + catch (...) + { + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("ClearTableLockReadThread::operator: Broken Pipe "); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + break; + } + } + + fIos.close(); +} //------------------------------------------------------------------------------ // RedistributeReadThread constructor. //------------------------------------------------------------------------------ RedistributeReadThread::RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs) - : ReadThread(ios) + : ReadThread(ios) { - fIbs = Ibs; + fIbs = Ibs; } //------------------------------------------------------------------------------ @@ -769,31 +758,31 @@ RedistributeReadThread::~RedistributeReadThread() //------------------------------------------------------------------------------ void RedistributeReadThread::operator()() { - try - { - redistribute::Redistribute::handleRedistributeMessage(fIbs, fIos); - } - catch (...) - { - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("RedistributeReadThread::operator exception handled "); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); - } + try + { + redistribute::Redistribute::handleRedistributeMessage(fIbs, fIos); + } + catch (...) + { + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("RedistributeReadThread::operator exception handled "); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + } - fIos.close(); + fIos.close(); } //------------------------------------------------------------------------------ // GetFileSizeThread constructor. //------------------------------------------------------------------------------ GetFileSizeThread::GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs, BRM::DBRM& dbrm) - : ReadThread(ios), fWeGetFileSizes(new WE_GetFileSizes()) + : ReadThread(ios), fWeGetFileSizes(new WE_GetFileSizes()) { - fIbs = Ibs; - key = dbrm.getUnique32(); + fIbs = Ibs; + key = dbrm.getUnique32(); } //------------------------------------------------------------------------------ @@ -808,162 +797,159 @@ GetFileSizeThread::~GetFileSizeThread() //------------------------------------------------------------------------------ void GetFileSizeThread::operator()() { - ByteStream::byte msgId; - ByteStream obs; - ByteStream::byte rc = 0; - std::string errMsg; + ByteStream::byte msgId; + ByteStream obs; + ByteStream::byte rc = 0; + std::string errMsg; - // Read msgid from ByteStream and forward to applicable processing function - while (fIbs.length() > 0) + // Read msgid from ByteStream and forward to applicable processing function + while (fIbs.length() > 0) + { + fIbs >> msgId; + + switch (msgId) { - fIbs >> msgId; + case WE_SVR_GET_FILESIZES: + { + rc = fWeGetFileSizes->processTable(fIbs, errMsg, key); + break; + } - switch (msgId) - { - case WE_SVR_GET_FILESIZES: - { - rc = fWeGetFileSizes->processTable(fIbs, errMsg, key); - break; - } + case WE_SVR_GET_FILESIZE: + { + rc = fWeGetFileSizes->processFileName(fIbs, errMsg, key); + break; + } - case WE_SVR_GET_FILESIZE: - { - rc = fWeGetFileSizes->processFileName(fIbs, errMsg, key); - break; - } - - default: - { - break; - } - } - - // Send response - obs.restart(); - obs << rc; - obs << errMsg; - obs += fIbs; - - try - { - fIos.write(obs); - - // Get next message - fIbs = fIos.read(); - } - catch (...) - { - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("GetFileSizeThread::operator: Broken Pipe "); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); - break; - } + default: + { + break; + } } - fIos.close(); + // Send response + obs.restart(); + obs << rc; + obs << errMsg; + obs += fIbs; + + try + { + fIos.write(obs); + + // Get next message + fIbs = fIos.read(); + } + catch (...) + { + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("GetFileSizeThread::operator: Broken Pipe "); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + break; + } + } + + fIos.close(); } //----------------------------------------------------------------------------- void ReadThreadFactory::CreateReadThread(ThreadPool& Tp, IOSocket& Ios, BRM::DBRM& dbrm) { - struct timespec rm_ts; - int sleepTime = 20000; // wait for 20 seconds - rm_ts.tv_sec = sleepTime / 1000; - rm_ts.tv_nsec = sleepTime % 1000 * 1000000; - bool isTimeOut = false; + struct timespec rm_ts; + int sleepTime = 20000; // wait for 20 seconds + rm_ts.tv_sec = sleepTime / 1000; + rm_ts.tv_nsec = sleepTime % 1000 * 1000000; + bool isTimeOut = false; - ByteStream::byte msgId; - ByteStream aBs; + ByteStream::byte msgId; + ByteStream aBs; - try + try + { + aBs = Ios.read(&rm_ts, &isTimeOut); + } + catch (std::exception& ex) + { + std::cout << "Handled : " << ex.what() << std::endl; + logging::LoggingID logid(19, 0, 0); + logging::Message::Args args; + logging::Message msg(1); + args.add("ReadThreadFactory::CreateReadThread: read() error"); + args.add(ex.what()); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + } + + if ((aBs.length() <= 0) || (isTimeOut)) + { + Ios.close(); + return; + } + + aBs.peek(msgId); + + switch (msgId) + { + case WE_SVR_DDL_KEEPALIVE: + case WE_SVR_DML_KEEPALIVE: { - aBs = Ios.read(&rm_ts, &isTimeOut); + DmlReadThread dmlReadThread(Ios, aBs); + boost::thread t(dmlReadThread); + // cout << "starting DML thread id " << t.get_id() << endl; } - catch (std::exception& ex) + break; + + case WE_CLT_SRV_KEEPALIVE: + case WE_CLT_SRV_MODE: + case WE_CLT_SRV_DATA: + case WE_CLT_SRV_CMD: + case WE_CLT_SRV_ACK: + case WE_CLT_SRV_NAK: + case WE_CLT_SRV_PM_ERROR: + case WE_CLT_SRV_CMDLINEARGS: { - std::cout << "Handled : " << ex.what() << std::endl; - logging::LoggingID logid(19, 0, 0); - logging::Message::Args args; - logging::Message msg(1); - args.add("ReadThreadFactory::CreateReadThread: read() error"); - args.add(ex.what()); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid); + // SplitterReadThread aSpReadThread(Ios, aBs); + // fOwner.attach(reinterpret_cast(&(aSpReadThread.fWeDataLoader))); + // Tp.invoke(aSpReadThread); + Tp.invoke(SplitterReadThread(Ios, aBs)); } + break; - if ((aBs.length() <= 0) || (isTimeOut)) + case WE_CLT_SRV_CLEAR_TABLE_LOCK: + case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP: { - Ios.close(); - return; + ClearTableLockReadThread clearTableLockThread(Ios, aBs); + Tp.invoke(clearTableLockThread); } + break; - aBs.peek(msgId); - - switch (msgId) + case WE_SVR_REDISTRIBUTE: { - case WE_SVR_DDL_KEEPALIVE: - case WE_SVR_DML_KEEPALIVE: - { - DmlReadThread dmlReadThread(Ios, aBs); - boost::thread t(dmlReadThread); - //cout << "starting DML thread id " << t.get_id() << endl; - } - break; - - case WE_CLT_SRV_KEEPALIVE: - case WE_CLT_SRV_MODE: - case WE_CLT_SRV_DATA: - case WE_CLT_SRV_CMD: - case WE_CLT_SRV_ACK: - case WE_CLT_SRV_NAK: - case WE_CLT_SRV_PM_ERROR: - case WE_CLT_SRV_CMDLINEARGS: - { - //SplitterReadThread aSpReadThread(Ios, aBs); - //fOwner.attach(reinterpret_cast(&(aSpReadThread.fWeDataLoader))); - //Tp.invoke(aSpReadThread); - Tp.invoke(SplitterReadThread(Ios, aBs)); - } - break; - - case WE_CLT_SRV_CLEAR_TABLE_LOCK: - case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP: - { - ClearTableLockReadThread clearTableLockThread(Ios, aBs); - Tp.invoke( clearTableLockThread ); - } - break; - - case WE_SVR_REDISTRIBUTE: - { - RedistributeReadThread RedistributeReadThread(Ios, aBs); - Tp.invoke(RedistributeReadThread); - } - break; - - case WE_SVR_GET_FILESIZES: - case WE_SVR_GET_FILESIZE: - { - GetFileSizeThread getFileSizeThread(Ios, aBs, dbrm); - Tp.invoke(getFileSizeThread); - } - break; - - default: - { - Ios.close(); // don't know who is this - } - break; - + RedistributeReadThread RedistributeReadThread(Ios, aBs); + Tp.invoke(RedistributeReadThread); } + break; + case WE_SVR_GET_FILESIZES: + case WE_SVR_GET_FILESIZE: + { + GetFileSizeThread getFileSizeThread(Ios, aBs, dbrm); + Tp.invoke(getFileSizeThread); + } + break; + + default: + { + Ios.close(); // don't know who is this + } + break; + } } -} - +} // namespace WriteEngine diff --git a/writeengine/server/we_readthread.h b/writeengine/server/we_readthread.h index 39f6267dd..8a56d0dc4 100644 --- a/writeengine/server/we_readthread.h +++ b/writeengine/server/we_readthread.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_readthread.h 4609 2013-04-19 15:32:02Z chao $ -* -*******************************************************************************/ + * $Id: we_readthread.h 4609 2013-04-19 15:32:02Z chao $ + * + *******************************************************************************/ #ifndef WE_SRV_READTHREAD_H__ #define WE_SRV_READTHREAD_H__ @@ -37,92 +37,82 @@ namespace WriteEngine { - - class ReadThread { -public: - explicit ReadThread(const messageqcpp::IOSocket& ios); - virtual ~ReadThread(); + public: + explicit ReadThread(const messageqcpp::IOSocket& ios); + virtual ~ReadThread(); - virtual void operator()(); + virtual void operator()(); + // protected: + public: + messageqcpp::IOSocket fIos; + messageqcpp::ByteStream fIbs; -//protected: -public: - messageqcpp::IOSocket fIos; - messageqcpp::ByteStream fIbs; - -private: - //defaults okay - //ReadThread(const ReadThread& rhs); - //ReadThread& operator=(const ReadThread& rhs); - + private: + // defaults okay + // ReadThread(const ReadThread& rhs); + // ReadThread& operator=(const ReadThread& rhs); }; - -class DmlReadThread: public ReadThread +class DmlReadThread : public ReadThread { -public: - explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs ); - virtual ~DmlReadThread(); + public: + explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs); + virtual ~DmlReadThread(); - virtual void operator()(); - - -private: - boost::shared_ptr fWeDMLprocessor; - boost::shared_ptr fWeDDLprocessor; + virtual void operator()(); + private: + boost::shared_ptr fWeDMLprocessor; + boost::shared_ptr fWeDDLprocessor; }; -class SplitterReadThread: public ReadThread +class SplitterReadThread : public ReadThread { -public: - SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs); - SplitterReadThread(const SplitterReadThread& rhs); - virtual ~SplitterReadThread(); + public: + SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs); + SplitterReadThread(const SplitterReadThread& rhs); + virtual ~SplitterReadThread(); - virtual void operator()(); + virtual void operator()(); + private: + // WEDataLoader* fpWeDataLoader; + WEDataLoader fWeDataLoader; -private: - //WEDataLoader* fpWeDataLoader; - WEDataLoader fWeDataLoader; - - friend class ReadThreadFactory; -//friend class WEDataLoader; + friend class ReadThreadFactory; + // friend class WEDataLoader; }; - //------------------------------------------------------------------------------ // Thread object that receives a cleartablelock tool command. //------------------------------------------------------------------------------ class ClearTableLockReadThread : public ReadThread { -public: - ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs); - virtual ~ClearTableLockReadThread(); + public: + ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs); + virtual ~ClearTableLockReadThread(); - virtual void operator()(); -private: - boost::shared_ptr fWeClearTableLockCmd; + virtual void operator()(); + + private: + boost::shared_ptr fWeClearTableLockCmd; }; - //------------------------------------------------------------------------------ // Thread object that receives a redistributeDB tool command. //------------------------------------------------------------------------------ class RedistributeReadThread : public ReadThread { -public: - RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs); - virtual ~RedistributeReadThread(); + public: + RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs); + virtual ~RedistributeReadThread(); - virtual void operator()(); - -private: + virtual void operator()(); + private: }; //------------------------------------------------------------------------------ @@ -130,31 +120,31 @@ private: //------------------------------------------------------------------------------ class GetFileSizeThread : public ReadThread { -public: - GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM& dbrm); - virtual ~GetFileSizeThread(); + public: + GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM& dbrm); + virtual ~GetFileSizeThread(); - virtual void operator()(); + virtual void operator()(); -private: - boost::shared_ptr fWeGetFileSizes; - int key; + private: + boost::shared_ptr fWeGetFileSizes; + int key; }; class ReadThreadFactory { -public: - ReadThreadFactory() {} - virtual ~ReadThreadFactory() {} - -public: - static void CreateReadThread(threadpool::ThreadPool& Tp, IOSocket& ios, BRM::DBRM& dbrm); - + public: + ReadThreadFactory() + { + } + virtual ~ReadThreadFactory() + { + } + public: + static void CreateReadThread(threadpool::ThreadPool& Tp, IOSocket& ios, BRM::DBRM& dbrm); }; - -} +} // namespace WriteEngine #endif - diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index 397eb013e..b7c9fc194 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_server.cpp 4700 2013-07-08 16:43:49Z bpaul $ -* -*******************************************************************************/ + * $Id: we_server.cpp 4700 2013-07-08 16:43:49Z bpaul $ + * + *******************************************************************************/ #include #include @@ -59,295 +59,280 @@ using namespace WriteEngine; namespace { - class Opt { -public: - int m_debug; - bool m_fg; - Opt(int argc, char *argv[]) - :m_debug(0), - m_fg(false) + public: + int m_debug; + bool m_fg; + Opt(int argc, char* argv[]) : m_debug(0), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "df")) != EOF) { - int c; - while ((c = getopt(argc, argv, "df")) != EOF) - { - switch(c) - { - case 'd': - m_debug++; - break; - case 'f': - m_fg= true; - break; - case '?': - default: - break; - } - } + switch (c) + { + case 'd': m_debug++; break; + case 'f': m_fg = true; break; + case '?': + default: break; + } + } } }; - -class ServiceWriteEngine: public Service, public Opt +class ServiceWriteEngine : public Service, public Opt { - void log(logging::LOG_TYPE type, const std::string &str) - { - logging::LoggingID logid(SUBSYSTEM_ID_WE_SRV); - logging::Message::Args args; - logging::Message msg(1); - args.add(str); - msg.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(type, msg, logid); - } - int setupResources(); - void setupChildSignalHandlers(); -public: - ServiceWriteEngine(const Opt &opt) - :Service("WriteEngine"), Opt(opt) - { } - void LogErrno() override - { - log(logging::LOG_TYPE_CRITICAL, strerror(errno)); - } - void ParentLogChildMessage(const std::string &str) override - { - log(logging::LOG_TYPE_INFO, str); - } - int Child() override; - int Run() { return m_fg ? Child() : RunForking(); } -}; + void log(logging::LOG_TYPE type, const std::string& str) + { + logging::LoggingID logid(SUBSYSTEM_ID_WE_SRV); + logging::Message::Args args; + logging::Message msg(1); + args.add(str); + msg.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(type, msg, logid); + } + int setupResources(); + void setupChildSignalHandlers(); + public: + ServiceWriteEngine(const Opt& opt) : Service("WriteEngine"), Opt(opt) + { + } + void LogErrno() override + { + log(logging::LOG_TYPE_CRITICAL, strerror(errno)); + } + void ParentLogChildMessage(const std::string& str) override + { + log(logging::LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } +}; void added_a_pm(int) { - logging::LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("we_server caught SIGHUP. Resetting connections"); - msg.format( args1 ); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); - joblist::DistributedEngineComm::reset(); -} + logging::LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("we_server caught SIGHUP. Resetting connections"); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); + joblist::DistributedEngineComm::reset(); } +} // namespace int ServiceWriteEngine::setupResources() { #ifndef _MSC_VER - struct rlimit rlim; + struct rlimit rlim; - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } - rlim.rlim_cur = rlim.rlim_max = 65536; + rlim.rlim_cur = rlim.rlim_max = 65536; - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } - if (rlim.rlim_cur != 65536) - { - return -4; - } + if (rlim.rlim_cur != 65536) + { + return -4; + } #endif - return 0; + return 0; } - void ServiceWriteEngine::setupChildSignalHandlers() { #ifndef _MSC_VER - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = added_a_pm; - sigaction(SIGHUP, &sa, 0); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, 0); + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = added_a_pm; + sigaction(SIGHUP, &sa, 0); + sa.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sa, 0); - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = fatalHandler; - sigaction(SIGSEGV, &sa, 0); - sigaction(SIGABRT, &sa, 0); - sigaction(SIGFPE, &sa, 0); + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = fatalHandler; + sigaction(SIGSEGV, &sa, 0); + sigaction(SIGABRT, &sa, 0); + sigaction(SIGFPE, &sa, 0); #endif } - int ServiceWriteEngine::Child() { - setupChildSignalHandlers(); + setupChildSignalHandlers(); - // Init WriteEngine Wrapper (including Config Columnstore.xml cache) - WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_WE_SRV ); + // Init WriteEngine Wrapper (including Config Columnstore.xml cache) + WriteEngine::WriteEngineWrapper::init(WriteEngine::SUBSYSTEM_ID_WE_SRV); #ifdef _MSC_VER - // In windows, initializing the wrapper (A dll) does not set the static variables - // in the main program - idbdatafile::IDBPolicy::configIDBPolicy(); + // In windows, initializing the wrapper (A dll) does not set the static variables + // in the main program + idbdatafile::IDBPolicy::configIDBPolicy(); #endif - Config weConfig; + Config weConfig; - ostringstream serverParms; - serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer"; + ostringstream serverParms; + serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer"; - // Create MessageQueueServer, with one retry in case the call to bind the - // known port fails with "Address already in use". - boost::scoped_ptr mqs; - bool tellUser = true; + // Create MessageQueueServer, with one retry in case the call to bind the + // known port fails with "Address already in use". + boost::scoped_ptr mqs; + bool tellUser = true; - for (;;) + for (;;) + { + try { - try - { - mqs.reset(new MessageQueueServer(serverParms.str())); - break; - } - // @bug4393 Error Handling for MessageQueueServer constructor exception - catch (runtime_error& re) - { - string what = re.what(); - - if (what.find("Address already in use") != string::npos) - { - if (tellUser) - { - cerr << "Address already in use, retrying..." << endl; - tellUser = false; - } - - sleep(5); - } - else - { - // If/when a common logging class or function is added to the - // WriteEngineServer, we should use that. In the mean time, - // I will log this errmsg with inline calls to the logging. - logging::Message::Args args; - logging::Message message; - string errMsg("WriteEngineServer failed to initiate: "); - errMsg += what; - args.add( errMsg ); - message.format(args); - logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); - logging::MessageLog ml(lid); - ml.logCriticalMessage( message ); - NotifyServiceInitializationFailed(); - return 2; - } - } + mqs.reset(new MessageQueueServer(serverParms.str())); + break; } - - int err = 0; - if (!m_debug) - err = setupResources(); - string errMsg; - - switch (err) + // @bug4393 Error Handling for MessageQueueServer constructor exception + catch (runtime_error& re) { - case -1: - case -3: - errMsg = "Error getting file limits, please see non-root install documentation"; - break; + string what = re.what(); - case -2: - errMsg = "Error setting file limits, please see non-root install documentation"; - break; + if (what.find("Address already in use") != string::npos) + { + if (tellUser) + { + cerr << "Address already in use, retrying..." << endl; + tellUser = false; + } - case -4: - errMsg = "Could not install file limits to required value, please see non-root install documentation"; - break; - - default: - break; - } - - if (err < 0) - { - Oam oam; + sleep(5); + } + else + { + // If/when a common logging class or function is added to the + // WriteEngineServer, we should use that. In the mean time, + // I will log this errmsg with inline calls to the logging. logging::Message::Args args; logging::Message message; - args.add( errMsg ); + string errMsg("WriteEngineServer failed to initiate: "); + errMsg += what; + args.add(errMsg); message.format(args); logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); logging::MessageLog ml(lid); - ml.logCriticalMessage( message ); - cerr << errMsg << endl; - + ml.logCriticalMessage(message); NotifyServiceInitializationFailed(); return 2; + } } + } + int err = 0; + if (!m_debug) + err = setupResources(); + string errMsg; - IOSocket ios; - size_t mt = 20; - size_t qs = mt * 100; - ThreadPool tp(mt, qs); + switch (err) + { + case -1: + case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; - cout << "WriteEngineServer is ready" << endl; - NotifyServiceStarted(); + case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; - BRM::DBRM dbrm; + case -4: + errMsg = "Could not install file limits to required value, please see non-root install documentation"; + break; - for (;;) + default: break; + } + + if (err < 0) + { + Oam oam; + logging::Message::Args args; + logging::Message message; + args.add(errMsg); + message.format(args); + logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); + logging::MessageLog ml(lid); + ml.logCriticalMessage(message); + cerr << errMsg << endl; + + NotifyServiceInitializationFailed(); + return 2; + } + + IOSocket ios; + size_t mt = 20; + size_t qs = mt * 100; + ThreadPool tp(mt, qs); + + cout << "WriteEngineServer is ready" << endl; + NotifyServiceStarted(); + + BRM::DBRM dbrm; + + for (;;) + { + try // BUG 4834 - { - try // BUG 4834 - - { - ios = mqs->accept(); - //tp.invoke(ReadThread(ios)); - ReadThreadFactory::CreateReadThread(tp, ios, dbrm); - { - /* logging::Message::Args args; - logging::Message message; - string aMsg("WriteEngineServer : New incoming connection"); - args.add(aMsg); - message.format(args); - logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); - logging::MessageLog ml(lid); - ml.logInfoMessage( message ); */ - } - } - catch (std::exception& ex) // BUG 4834 - log the exception - { - logging::Message::Args args; - logging::Message message; - string errMsg("WriteEngineServer : Exception caught on accept(): "); - errMsg += ex.what(); - args.add( errMsg ); - message.format(args); - logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); - logging::MessageLog ml(lid); - ml.logCriticalMessage( message ); - break; - } + ios = mqs->accept(); + // tp.invoke(ReadThread(ios)); + ReadThreadFactory::CreateReadThread(tp, ios, dbrm); + { + /* logging::Message::Args args; + logging::Message message; + string aMsg("WriteEngineServer : New incoming connection"); + args.add(aMsg); + message.format(args); + logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); + logging::MessageLog ml(lid); + ml.logInfoMessage( message ); */ + } } + catch (std::exception& ex) // BUG 4834 - log the exception + { + logging::Message::Args args; + logging::Message message; + string errMsg("WriteEngineServer : Exception caught on accept(): "); + errMsg += ex.what(); + args.add(errMsg); + message.format(args); + logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV); + logging::MessageLog ml(lid); + ml.logCriticalMessage(message); + break; + } + } - //It is an error to reach here... - return 1; + // It is an error to reach here... + return 1; } - int main(int argc, char** argv) { - Opt opt(argc, argv); + Opt opt(argc, argv); - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - // This is unset due to the way we start it - program_invocation_short_name = const_cast("WriteEngineServ"); - // Initialize the charset library - MY_INIT(argv[0]); + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("WriteEngineServ"); + // Initialize the charset library + MY_INIT(argv[0]); - return ServiceWriteEngine(opt).Run(); + return ServiceWriteEngine(opt).Run(); } diff --git a/writeengine/shared/tconfig.cpp b/writeengine/shared/tconfig.cpp index e202f2a4b..a15703719 100644 --- a/writeengine/shared/tconfig.cpp +++ b/writeengine/shared/tconfig.cpp @@ -13,92 +13,73 @@ using namespace WriteEngine; void test() { - std::cout << "getDBRootByIdx(1): " << - Config::getDBRootByIdx(1) << std::endl; - std::cout << "getDBRootByIdx(3): " << - Config::getDBRootByIdx(3) << std::endl; + std::cout << "getDBRootByIdx(1): " << Config::getDBRootByIdx(1) << std::endl; + std::cout << "getDBRootByIdx(3): " << Config::getDBRootByIdx(3) << std::endl; - std::cout << "getDBRootByNum(1): " << - Config::getDBRootByNum(1) << std::endl; - std::cout << "getDBRootByNum(3): " << - Config::getDBRootByNum(3) << std::endl; + std::cout << "getDBRootByNum(1): " << Config::getDBRootByNum(1) << std::endl; + std::cout << "getDBRootByNum(3): " << Config::getDBRootByNum(3) << std::endl; - std::vector dbRootIds; - Config::getRootIdList( dbRootIds ); - std::cout << "getRootIdList: "; + std::vector dbRootIds; + Config::getRootIdList(dbRootIds); + std::cout << "getRootIdList: "; - for (unsigned k = 0; k < dbRootIds.size(); k++) - { - std::cout << dbRootIds[k] << ' '; - } + for (unsigned k = 0; k < dbRootIds.size(); k++) + { + std::cout << dbRootIds[k] << ' '; + } - std::cout << std::endl; + std::cout << std::endl; - std::vector dbRootPathList; - Config::getDBRootPathList( dbRootPathList ); - std::cout << "getDBRootPathList: " << std::endl; + std::vector dbRootPathList; + Config::getDBRootPathList(dbRootPathList); + std::cout << "getDBRootPathList: " << std::endl; - for (unsigned k = 0; k < dbRootPathList.size(); k++) - std::cout << " " << k << ". " << dbRootPathList[k] << std::endl; + for (unsigned k = 0; k < dbRootPathList.size(); k++) + std::cout << " " << k << ". " << dbRootPathList[k] << std::endl; - std::cout << "getBulkRoot(): " << - Config::getBulkRoot() << std::endl; - std::cout << "DBRootCount(): " << - Config::DBRootCount() << std::endl; - std::cout << "totalDBRootCount(): " << - Config::totalDBRootCount() << std::endl; - std::cout << "getWaitPeriod(): " << - Config::getWaitPeriod() << std::endl; - std::cout << "getFilePerColumnPartition(): " << - Config::getFilesPerColumnPartition() << std::endl; - std::cout << "getExtentsPerSegmentFile(): " << - Config::getExtentsPerSegmentFile() << std::endl; - std::cout << "getBulkProcessPriority(): " << - Config::getBulkProcessPriority() << std::endl; - std::cout << "getBulkRollbackDir(): " << - Config::getBulkRollbackDir() << std::endl; - std::cout << "getMaxFileSystemDiskUsage(): " << - Config::getMaxFileSystemDiskUsage() << std::endl; - std::cout << "getNumCompressedPadBlks(): " << - Config::getNumCompressedPadBlks() << std::endl; - std::cout << "getParentOAMModuleFlag(): " << - Config::getParentOAMModuleFlag() << std::endl; - std::cout << "getLocalModuleType(): " << - Config::getLocalModuleType() << std::endl; - std::cout << "getLocalModuleID(): " << - Config::getLocalModuleID() << std::endl; - std::cout << "getVBRoot(): " << - Config::getVBRoot() << std::endl; + std::cout << "getBulkRoot(): " << Config::getBulkRoot() << std::endl; + std::cout << "DBRootCount(): " << Config::DBRootCount() << std::endl; + std::cout << "totalDBRootCount(): " << Config::totalDBRootCount() << std::endl; + std::cout << "getWaitPeriod(): " << Config::getWaitPeriod() << std::endl; + std::cout << "getFilePerColumnPartition(): " << Config::getFilesPerColumnPartition() << std::endl; + std::cout << "getExtentsPerSegmentFile(): " << Config::getExtentsPerSegmentFile() << std::endl; + std::cout << "getBulkProcessPriority(): " << Config::getBulkProcessPriority() << std::endl; + std::cout << "getBulkRollbackDir(): " << Config::getBulkRollbackDir() << std::endl; + std::cout << "getMaxFileSystemDiskUsage(): " << Config::getMaxFileSystemDiskUsage() << std::endl; + std::cout << "getNumCompressedPadBlks(): " << Config::getNumCompressedPadBlks() << std::endl; + std::cout << "getParentOAMModuleFlag(): " << Config::getParentOAMModuleFlag() << std::endl; + std::cout << "getLocalModuleType(): " << Config::getLocalModuleType() << std::endl; + std::cout << "getLocalModuleID(): " << Config::getLocalModuleID() << std::endl; + std::cout << "getVBRoot(): " << Config::getVBRoot() << std::endl; } int main() { - Config::initConfigCache(); - char resp; + Config::initConfigCache(); + char resp; - int nTest = 0; + int nTest = 0; + std::cout << std::endl; + + while (1) + { + std::cout << "test" << nTest << "..." << std::endl; + test(); + std::cout << "Pause..." << std::endl; + std::cin >> resp; std::cout << std::endl; - while (1) + if (resp == 'c') { - std::cout << "test" << nTest << "..." << std::endl; - test(); - std::cout << "Pause..." << std::endl; - std::cin >> resp; - std::cout << std::endl; - - if (resp == 'c') - { - std::cout << "Has local DBRootList changed: " << - (bool)Config::hasLocalDBRootListChanged() << std::endl; - } - else if (resp == 'q') - { - break; - } - - nTest++; + std::cout << "Has local DBRootList changed: " << (bool)Config::hasLocalDBRootListChanged() << std::endl; + } + else if (resp == 'q') + { + break; } - return 0; + nTest++; + } + + return 0; } diff --git a/writeengine/shared/we_blockop.cpp b/writeengine/shared/we_blockop.cpp index ae2f29fff..e37810b8e 100644 --- a/writeengine/shared/we_blockop.cpp +++ b/writeengine/shared/we_blockop.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_blockop.cpp 4500 2013-01-31 20:25:42Z dhall $ -* -*******************************************************************************/ + * $Id: we_blockop.cpp 4500 2013-01-31 20:25:42Z dhall $ + * + *******************************************************************************/ /** @file */ #include @@ -37,18 +37,19 @@ using namespace execplan; namespace WriteEngine { - /** * Constructor */ -BlockOp::BlockOp(): m_typeHandler(nullptr) -{} +BlockOp::BlockOp() : m_typeHandler(nullptr) +{ +} /** * Default Destructor */ BlockOp::~BlockOp() -{} +{ +} /*********************************************************** * DESCRIPTION: @@ -62,16 +63,15 @@ BlockOp::~BlockOp() * bio - Block internal offset * true if success, false otherwise ***********************************************************/ -bool BlockOp::calculateRowId( - RID rowId, const int epb, const int width, int& fbo, int& bio ) const +bool BlockOp::calculateRowId(RID rowId, const int epb, const int width, int& fbo, int& bio) const { - if ( std::numeric_limits::max() == rowId ) - return false; + if (std::numeric_limits::max() == rowId) + return false; - fbo = (int)( rowId / epb ); - bio = ( rowId & ( epb - 1 )) * width; + fbo = (int)(rowId / epb); + bio = (rowId & (epb - 1)) * width; - return true; + return true; } /*********************************************************** @@ -83,20 +83,18 @@ bool BlockOp::calculateRowId( * RETURN: * emptyVal - the value of empty row ***********************************************************/ -const uint8_t* BlockOp::getEmptyRowValue( - const CalpontSystemCatalog::ColDataType colDataType, - const int width) const +const uint8_t* BlockOp::getEmptyRowValue(const CalpontSystemCatalog::ColDataType colDataType, + const int width) const { - auto attrs = datatypes::SystemCatalog::TypeAttributesStd(width, 0, -1); - // Bulk operation runtime should have m_typeHandler nullptr calling this - // Non-bulk operations runtime branch - if (m_typeHandler) - return m_typeHandler->getEmptyValueForType(attrs); + auto attrs = datatypes::SystemCatalog::TypeAttributesStd(width, 0, -1); + // Bulk operation runtime should have m_typeHandler nullptr calling this + // Non-bulk operations runtime branch + if (m_typeHandler) + return m_typeHandler->getEmptyValueForType(attrs); - // Bulk operation branch - auto* typeHandler = datatypes::TypeHandler::find(colDataType, - attrs); - return typeHandler->getEmptyValueForType(attrs); + // Bulk operation branch + auto* typeHandler = datatypes::TypeHandler::find(colDataType, attrs); + return typeHandler->getEmptyValueForType(attrs); } /*********************************************************** @@ -108,10 +106,9 @@ const uint8_t* BlockOp::getEmptyRowValue( * RETURN: * emptyVal - the value of empty row ***********************************************************/ -int BlockOp::getCorrectRowWidth( - const CalpontSystemCatalog::ColDataType colDataType, const int width ) const +int BlockOp::getCorrectRowWidth(const CalpontSystemCatalog::ColDataType colDataType, const int width) const { - return Convertor::getCorrectRowWidth(colDataType, width); + return Convertor::getCorrectRowWidth(colDataType, width); } /*********************************************************** @@ -124,12 +121,11 @@ int BlockOp::getCorrectRowWidth( * RETURN: * row id ***********************************************************/ -RID BlockOp::getRowId( - const long fbo, const int width, const int rowPos - /*const int bio, const int bbo*/ ) const +RID BlockOp::getRowId(const long fbo, const int width, const int rowPos + /*const int bio, const int bbo*/) const { -// return fbo*BYTE_PER_BLOCK*ROW_PER_BYTE + bio*ROW_PER_BYTE + bbo; - return (BYTE_PER_BLOCK / width) * fbo + rowPos; + // return fbo*BYTE_PER_BLOCK*ROW_PER_BYTE + bio*ROW_PER_BYTE + bbo; + return (BYTE_PER_BLOCK / width) * fbo + rowPos; } /*********************************************************** @@ -141,10 +137,9 @@ RID BlockOp::getRowId( * RETURN: * val - buffer value ***********************************************************/ -void BlockOp::readBufValue( - const unsigned char* buf, void* val, const short width ) const +void BlockOp::readBufValue(const unsigned char* buf, void* val, const short width) const { - memcpy( val, buf, width ); + memcpy(val, buf, width); } /*********************************************************** @@ -156,9 +151,9 @@ void BlockOp::readBufValue( * RETURN: * none ***********************************************************/ -void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const +void BlockOp::resetBuf(unsigned char* buf, const int bufSize) const { - memset( buf, 0, bufSize ); + memset(buf, 0, bufSize); } /*********************************************************** @@ -173,45 +168,38 @@ void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const * none ***********************************************************/ /* static */ -void BlockOp::setEmptyBuf( - unsigned char* buf, - const int bufSize, - const uint8_t* emptyVal, - const int width ) +void BlockOp::setEmptyBuf(unsigned char* buf, const int bufSize, const uint8_t* emptyVal, const int width) { - const int ARRAY_COUNT = 128; - const int NBYTES_IN_ARRAY = width * ARRAY_COUNT; - //unsigned char emptyValArray[NBYTES_IN_ARRAY]; - unsigned char* emptyValArray = (unsigned char*)alloca(NBYTES_IN_ARRAY); + const int ARRAY_COUNT = 128; + const int NBYTES_IN_ARRAY = width * ARRAY_COUNT; + // unsigned char emptyValArray[NBYTES_IN_ARRAY]; + unsigned char* emptyValArray = (unsigned char*)alloca(NBYTES_IN_ARRAY); - // Optimize buffer initialization by constructing and copying in an array - // instead of individual values. This reduces the number of calls to - // memcpy(). - - for(uint8_t* pos = emptyValArray, * end = pos + NBYTES_IN_ARRAY; pos < end; pos += width) //FIXME for no loop - { - memcpy(pos, emptyVal, width); - } + // Optimize buffer initialization by constructing and copying in an array + // instead of individual values. This reduces the number of calls to + // memcpy(). - int countFull128 = (bufSize / width) / ARRAY_COUNT; - int countRemain = (bufSize / width) % ARRAY_COUNT; + for (uint8_t *pos = emptyValArray, *end = pos + NBYTES_IN_ARRAY; pos < end; + pos += width) // FIXME for no loop + { + memcpy(pos, emptyVal, width); + } - // Copy in the 128 element array into "buf" as many times as needed - if (countFull128 > 0) - { - for ( int i = 0; i < countFull128; i++ ) - memcpy( buf + (i * (NBYTES_IN_ARRAY)), - emptyValArray, - NBYTES_IN_ARRAY ); - } + int countFull128 = (bufSize / width) / ARRAY_COUNT; + int countRemain = (bufSize / width) % ARRAY_COUNT; - // Initialize the remainder of "buf" that is leftover - if (countRemain > 0) - { - memcpy( buf + (countFull128 * NBYTES_IN_ARRAY), - emptyValArray, - width * countRemain ); - } + // Copy in the 128 element array into "buf" as many times as needed + if (countFull128 > 0) + { + for (int i = 0; i < countFull128; i++) + memcpy(buf + (i * (NBYTES_IN_ARRAY)), emptyValArray, NBYTES_IN_ARRAY); + } + + // Initialize the remainder of "buf" that is leftover + if (countRemain > 0) + { + memcpy(buf + (countFull128 * NBYTES_IN_ARRAY), emptyValArray, width * countRemain); + } } /*********************************************************** @@ -224,25 +212,20 @@ void BlockOp::setEmptyBuf( * RETURN: * none ***********************************************************/ -void BlockOp::writeBufValue( - unsigned char* buf, const void* val, const size_t width, const bool clear ) const +void BlockOp::writeBufValue(unsigned char* buf, const void* val, const size_t width, const bool clear) const { - if ( clear ) - memset( buf, 0, width ); + if (clear) + memset(buf, 0, width); - memcpy( buf, val, width ); + memcpy(buf, val, width); } void BlockOp::findTypeHandler(const int colWidth, const execplan::CalpontSystemCatalog::ColDataType colDataType) { - auto attrs = datatypes::SystemCatalog::TypeAttributesStd(colWidth, - 0, - -1); - m_typeHandler = datatypes::TypeHandler::find(colDataType, - attrs); + auto attrs = datatypes::SystemCatalog::TypeAttributesStd(colWidth, 0, -1); + m_typeHandler = datatypes::TypeHandler::find(colDataType, attrs); } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_blockop.h b/writeengine/shared/we_blockop.h index b9bc870ea..9e0670a82 100644 --- a/writeengine/shared/we_blockop.h +++ b/writeengine/shared/we_blockop.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_blockop.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_blockop.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_BLOCKOP_H_ @@ -35,107 +35,90 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class BlockOp */ class BlockOp : public WEObj { -public: - /** - * @brief Constructor - */ - EXPORT BlockOp(); + public: + /** + * @brief Constructor + */ + EXPORT BlockOp(); - /** - * @brief Default Destructor - */ - EXPORT ~BlockOp(); + /** + * @brief Default Destructor + */ + EXPORT ~BlockOp(); + /** + * @brief Calculate the location of Row ID + */ + EXPORT bool calculateRowId(RID rowId, const int epb, const int width, int& fbo, int& bio) const; - /** - * @brief Calculate the location of Row ID - */ - EXPORT bool calculateRowId( RID rowId, - const int epb, - const int width, - int& fbo, - int& bio ) const; + /** + * @brief Calculate the location of Row ID + */ + void clearBlock(DataBlock* block) + { + memset(block->data, 0, sizeof(block->data)); + block->no = -1; + block->dirty = false; + } - /** - * @brief Calculate the location of Row ID - */ - void clearBlock( DataBlock* block ) - { - memset(block->data, 0, sizeof(block->data)); - block->no = -1; - block->dirty = false; - } + /** + * @brief Get bit value after shift + */ + uint64_t getBitValue(uint64_t val, int shiftBit, uint64_t mask) const + { + return (val >> shiftBit) & mask; + } - /** - * @brief Get bit value after shift - */ - uint64_t getBitValue( uint64_t val, - int shiftBit, - uint64_t mask ) const - { - return ( val >> shiftBit ) & mask ; - } + /** + * @brief Get correct row width + */ + EXPORT int getCorrectRowWidth(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width) const; - /** - * @brief Get correct row width - */ - EXPORT int getCorrectRowWidth( const execplan::CalpontSystemCatalog::ColDataType colDataType, - const int width ) const; + /** + * @brief Get an empty row value + */ + EXPORT const uint8_t* getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width) const; - /** - * @brief Get an empty row value - */ - EXPORT const uint8_t* getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, - const int width) const; + /** + * @brief Calculate row id + */ + EXPORT RID getRowId(const long fbo, const int width, const int rowPos) const; + /** + * @brief Get buffer value + */ + EXPORT void readBufValue(const unsigned char* buf, void* val, const short width) const; - /** - * @brief Calculate row id - */ - EXPORT RID getRowId( const long fbo, - const int width, - const int rowPos ) const; + /** + * @brief Reset a buffer + */ + EXPORT void resetBuf(unsigned char* buf, const int bufSize) const; - /** - * @brief Get buffer value - */ - EXPORT void readBufValue( const unsigned char* buf, - void* val, const short width ) const; + /** + * @brief Fill buffer with empty values + */ + EXPORT void static setEmptyBuf(unsigned char* buf, const int bufSize, const uint8_t* emptyVal, + const int width); - /** - * @brief Reset a buffer - */ - EXPORT void resetBuf( unsigned char* buf, - const int bufSize ) const; - - /** - * @brief Fill buffer with empty values - */ - EXPORT void static setEmptyBuf( unsigned char* buf, - const int bufSize, - const uint8_t* emptyVal, - const int width ); - - /** - * @brief Set a value in a buffer - */ - EXPORT void writeBufValue( unsigned char* buf, - const void* val, - const size_t width, - const bool clear = false ) const; - EXPORT void findTypeHandler( const int colWidth, + /** + * @brief Set a value in a buffer + */ + EXPORT void writeBufValue(unsigned char* buf, const void* val, const size_t width, + const bool clear = false) const; + EXPORT void findTypeHandler(const int colWidth, const execplan::CalpontSystemCatalog::ColDataType colDataType); -private: - const datatypes::TypeHandler* m_typeHandler; + + private: + const datatypes::TypeHandler* m_typeHandler; }; - -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_BLOCKOP_H_ +#endif // _WE_BLOCKOP_H_ diff --git a/writeengine/shared/we_brm.cpp b/writeengine/shared/we_brm.cpp index 2dc696254..a049806b1 100644 --- a/writeengine/shared/we_brm.cpp +++ b/writeengine/shared/we_brm.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_brm.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_brm.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + * + *******************************************************************************/ /** @file */ #include @@ -63,158 +63,140 @@ boost::mutex BRMWrapper::m_instanceCreateMutex; __declspec(dllexport) #endif -bool BRMWrapper::m_useVb = true; -OID BRMWrapper::m_curVBOid = INVALID_NUM; + bool BRMWrapper::m_useVb = true; +OID BRMWrapper::m_curVBOid = INVALID_NUM; IDBDataFile* BRMWrapper::m_curVBFile = NULL; boost::mutex vbFileLock; -struct fileInfoCompare // lt operator +struct fileInfoCompare // lt operator { - bool operator()(const File& lhs, const File& rhs) const + bool operator()(const File& lhs, const File& rhs) const + { + if (lhs.oid < rhs.oid) { - if (lhs.oid < rhs.oid) - { - return true; - } + return true; + } - if ((lhs.oid == rhs.oid) && (lhs.fDbRoot < rhs.fDbRoot)) - { - return true; - } + if ((lhs.oid == rhs.oid) && (lhs.fDbRoot < rhs.fDbRoot)) + { + return true; + } - if ((lhs.oid == rhs.oid) && (lhs.fDbRoot == rhs.fDbRoot) && (lhs.fPartition < rhs.fPartition)) - { - return true; - } + if ((lhs.oid == rhs.oid) && (lhs.fDbRoot == rhs.fDbRoot) && (lhs.fPartition < rhs.fPartition)) + { + return true; + } - if ((lhs.oid == rhs.oid) && (lhs.fDbRoot == rhs.fDbRoot) && (lhs.fPartition == rhs.fPartition) && (lhs.fSegment < rhs.fSegment)) - { - return true; - } + if ((lhs.oid == rhs.oid) && (lhs.fDbRoot == rhs.fDbRoot) && (lhs.fPartition == rhs.fPartition) && + (lhs.fSegment < rhs.fSegment)) + { + return true; + } - return false; - } // operator -}; // struct + return false; + } // operator +}; // struct -typedef std::map< File, IDBDataFile*, fileInfoCompare > FileOpenMap; +typedef std::map FileOpenMap; //------------------------------------------------------------------------------ // Set up an Auto-increment sequence for specified Column OID, starting at // startNextValue. Column width is required to monitor if/when the sequence // reaches the max integer value for the given column width. //------------------------------------------------------------------------------ -int BRMWrapper::startAutoIncrementSequence( - OID colOID, - uint64_t startNextValue, - uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType, - std::string& errMsg) +int BRMWrapper::startAutoIncrementSequence(OID colOID, uint64_t startNextValue, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType, + std::string& errMsg) { - int rc = NO_ERROR; + int rc = NO_ERROR; - try - { - blockRsltnMgrPtr->startAISequence( colOID, startNextValue, colWidth, colDataType ); - } - catch (std::exception& ex) - { - errMsg = ex.what(); - rc = ERR_AUTOINC_START_SEQ; - } + try + { + blockRsltnMgrPtr->startAISequence(colOID, startNextValue, colWidth, colDataType); + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = ERR_AUTOINC_START_SEQ; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Reserve a range of auto-increment numbers for the specified column OID. //------------------------------------------------------------------------------ -int BRMWrapper::getAutoIncrementRange( - OID colOID, - uint64_t count, - uint64_t& firstNum, - std::string& errMsg) +int BRMWrapper::getAutoIncrementRange(OID colOID, uint64_t count, uint64_t& firstNum, std::string& errMsg) { - int rc = NO_ERROR; + int rc = NO_ERROR; - try + try + { + uint64_t firstNumArg = 0; + bool gotFullRange = blockRsltnMgrPtr->getAIRange(colOID, count, &firstNumArg); + + if (gotFullRange) { - uint64_t firstNumArg = 0; - bool gotFullRange = blockRsltnMgrPtr->getAIRange( - colOID, count, &firstNumArg ); - - if (gotFullRange) - { - firstNum = firstNumArg; - } - else - { - rc = ERR_AUTOINC_GEN_EXCEED_MAX; - WriteEngine::WErrorCodes ec; - errMsg = ec.errorString(rc); - } + firstNum = firstNumArg; } - catch (std::exception& ex) + else { - errMsg = ex.what(); - rc = ERR_AUTOINC_GET_RANGE; + rc = ERR_AUTOINC_GEN_EXCEED_MAX; + WriteEngine::WErrorCodes ec; + errMsg = ec.errorString(rc); } + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = ERR_AUTOINC_GET_RANGE; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Allocate a stripe of column extents for a table //------------------------------------------------------------------------------ -int BRMWrapper::allocateStripeColExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partition, - uint16_t& segmentNum, - std::vector& extents) +int BRMWrapper::allocateStripeColExtents(const std::vector& cols, + uint16_t dbRoot, uint32_t& partition, uint16_t& segmentNum, + std::vector& extents) { - int rc = blockRsltnMgrPtr->createStripeColumnExtents( - cols, dbRoot, partition, segmentNum, extents ); - rc = getRC( rc, ERR_BRM_ALLOC_EXTEND ); + int rc = blockRsltnMgrPtr->createStripeColumnExtents(cols, dbRoot, partition, segmentNum, extents); + rc = getRC(rc, ERR_BRM_ALLOC_EXTEND); - if (rc == NO_ERROR) - { - if (cols.size() != extents.size()) - rc = ERR_BRM_BAD_STRIPE_CNT; - } + if (rc == NO_ERROR) + { + if (cols.size() != extents.size()) + rc = ERR_BRM_BAD_STRIPE_CNT; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Allocate an extent to the exact file specified by the column OID, DBRoot, // partition, and segment. //------------------------------------------------------------------------------ -int BRMWrapper::allocateColExtentExactFile( - const OID oid, - const uint32_t colWidth, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - LBID_t& startLbid, - int& allocSize, - uint32_t& startBlock) +int BRMWrapper::allocateColExtentExactFile(const OID oid, const uint32_t colWidth, uint16_t dbRoot, + uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + LBID_t& startLbid, int& allocSize, uint32_t& startBlock) { - int rc = blockRsltnMgrPtr->createColumnExtentExactFile( - (int)oid, colWidth, dbRoot, partition, segment, colDataType, - startLbid, allocSize, startBlock); + int rc = blockRsltnMgrPtr->createColumnExtentExactFile((int)oid, colWidth, dbRoot, partition, segment, + colDataType, startLbid, allocSize, startBlock); - //std::ostringstream oss; - //oss << "Allocated column extent: oid-" << oid << - // "; wid-" << colWidth << - // "; DBRoot-" << dbRoot << - // "; part-" << partition << - // "; seg-" << segment << - // "; lbid-" << startLbid << - // "; allocSize-" << allocSize << - // "; block-" << startBlock; - //std::cout << oss.str() << std::endl; + // std::ostringstream oss; + // oss << "Allocated column extent: oid-" << oid << + // "; wid-" << colWidth << + // "; DBRoot-" << dbRoot << + // "; part-" << partition << + // "; seg-" << segment << + // "; lbid-" << startLbid << + // "; allocSize-" << allocSize << + // "; block-" << startBlock; + // std::cout << oss.str() << std::endl; - return getRC(rc, ERR_BRM_ALLOC_EXTEND); + return getRC(rc, ERR_BRM_ALLOC_EXTEND); } //------------------------------------------------------------------------------ @@ -223,27 +205,22 @@ int BRMWrapper::allocateColExtentExactFile( // specified, else the selected DBRoot is returned. The selected partition // and segment numbers are always returned. //------------------------------------------------------------------------------ -int BRMWrapper::allocateDictStoreExtent( - const OID oid, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - LBID_t& startLbid, - int& allocSize) +int BRMWrapper::allocateDictStoreExtent(const OID oid, uint16_t dbRoot, uint32_t partition, uint16_t segment, + LBID_t& startLbid, int& allocSize) { - int rc = blockRsltnMgrPtr->createDictStoreExtent( - (int)oid, dbRoot, partition, segment, startLbid, allocSize); + int rc = + blockRsltnMgrPtr->createDictStoreExtent((int)oid, dbRoot, partition, segment, startLbid, allocSize); - //std::ostringstream oss; - //oss << "Allocated dict extent: oid-" << oid << - // "; DBRoot-" << dbRoot << - // "; part-" << partition << - // "; seg-" << segment << - // "; lbid-" << startLbid << - // "; allocSize-" << allocSize; - //std::cout << oss.str() << std::endl; + // std::ostringstream oss; + // oss << "Allocated dict extent: oid-" << oid << + // "; DBRoot-" << dbRoot << + // "; part-" << partition << + // "; seg-" << segment << + // "; lbid-" << startLbid << + // "; allocSize-" << allocSize; + // std::cout << oss.str() << std::endl; - return getRC(rc, ERR_BRM_ALLOC_EXTEND); + return getRC(rc, ERR_BRM_ALLOC_EXTEND); } //------------------------------------------------------------------------------ @@ -251,8 +228,8 @@ int BRMWrapper::allocateDictStoreExtent( //------------------------------------------------------------------------------ int BRMWrapper::deleteOid(const OID oid) { - int rc = blockRsltnMgrPtr->deleteOID(oid); - return getRC(rc, ERR_BRM_DEL_OID); + int rc = blockRsltnMgrPtr->deleteOID(oid); + return getRC(rc, ERR_BRM_DEL_OID); } //------------------------------------------------------------------------------ @@ -260,78 +237,61 @@ int BRMWrapper::deleteOid(const OID oid) //------------------------------------------------------------------------------ int BRMWrapper::deleteOIDsFromExtentMap(const std::vector& oids) { - int rc = blockRsltnMgrPtr->deleteOIDs(oids); - return getRC(rc, ERR_BRM_DEL_OID); + int rc = blockRsltnMgrPtr->deleteOIDs(oids); + return getRC(rc, ERR_BRM_DEL_OID); } //------------------------------------------------------------------------------ // Get BRM information based on a specfic OID, DBRoot, partition, and segment. //------------------------------------------------------------------------------ -int BRMWrapper::getBrmInfo(const OID oid, - const uint32_t partition, - const uint16_t segment, - const int fbo, - LBID_t& lbid) +int BRMWrapper::getBrmInfo(const OID oid, const uint32_t partition, const uint16_t segment, const int fbo, + LBID_t& lbid) { - // SHARED_NOTHING: lookupLocal() usage okay if segNum unique. - // If segment number is not unique across physical partition, then would - // need to pass DBRoot to lookupLocal(). - int rc = blockRsltnMgrPtr->lookupLocal((int)oid, - partition, segment, - (uint32_t)fbo, lbid); - return getRC(rc, ERR_BRM_LOOKUP_LBID); + // SHARED_NOTHING: lookupLocal() usage okay if segNum unique. + // If segment number is not unique across physical partition, then would + // need to pass DBRoot to lookupLocal(). + int rc = blockRsltnMgrPtr->lookupLocal((int)oid, partition, segment, (uint32_t)fbo, lbid); + return getRC(rc, ERR_BRM_LOOKUP_LBID); }; //------------------------------------------------------------------------------ // Get starting LBID from BRM for a specfic OID, partition, segment, and // block offset. //------------------------------------------------------------------------------ -int BRMWrapper::getStartLbid(const OID oid, - const uint32_t partition, - const uint16_t segment, - const int fbo, - LBID_t& lbid) +int BRMWrapper::getStartLbid(const OID oid, const uint32_t partition, const uint16_t segment, const int fbo, + LBID_t& lbid) { - // SHARED_NOTHING: lookupLocalStartLbid() usage okay if segNum unique. - // If segment number is not unique across physical partition, then would - // need to pass DBRoot to lookupLocalStartLbid(). - int rc = blockRsltnMgrPtr->lookupLocalStartLbid((int)oid, - partition, segment, - (uint32_t)fbo, lbid); + // SHARED_NOTHING: lookupLocalStartLbid() usage okay if segNum unique. + // If segment number is not unique across physical partition, then would + // need to pass DBRoot to lookupLocalStartLbid(). + int rc = blockRsltnMgrPtr->lookupLocalStartLbid((int)oid, partition, segment, (uint32_t)fbo, lbid); - return getRC(rc, ERR_BRM_LOOKUP_START_LBID); + return getRC(rc, ERR_BRM_LOOKUP_START_LBID); } //------------------------------------------------------------------------------ // Get the real physical offset based on the LBID //------------------------------------------------------------------------------ -int BRMWrapper::getFboOffset(const uint64_t lbid, - uint16_t& dbRoot, - uint32_t& partition, - uint16_t& segment, - int& fbo) +int BRMWrapper::getFboOffset(const uint64_t lbid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, + int& fbo) { - int oid; - // according to Patric, extendmap don't need vbflag, thus verid =0 -// int rc = blockRsltnMgrPtr->lookup((uint64_t)lbid, 0, false, oid, (uint32_t&)fbo); -// return getRC(rc, ERR_BRM_LOOKUP_FBO); - return getFboOffset(lbid, oid, dbRoot, partition, segment, fbo); + int oid; + // according to Patric, extendmap don't need vbflag, thus verid =0 + // int rc = blockRsltnMgrPtr->lookup((uint64_t)lbid, 0, false, oid, (uint32_t&)fbo); + // return getRC(rc, ERR_BRM_LOOKUP_FBO); + return getFboOffset(lbid, oid, dbRoot, partition, segment, fbo); } //------------------------------------------------------------------------------ // Get the real physical offset based on the LBID //------------------------------------------------------------------------------ -int BRMWrapper::getFboOffset(const uint64_t lbid, int& oid, - uint16_t& dbRoot, - uint32_t& partition, - uint16_t& segment, - int& fbo) +int BRMWrapper::getFboOffset(const uint64_t lbid, int& oid, uint16_t& dbRoot, uint32_t& partition, + uint16_t& segment, int& fbo) { - // according to Patric, extendmap don't need vbflag, thus verid =0 - int rc = blockRsltnMgrPtr->lookupLocal((uint64_t)lbid, 0, false, (BRM::OID_t&)oid, - dbRoot, partition, segment, - (uint32_t&)fbo); - return getRC(rc, ERR_BRM_LOOKUP_FBO); + // according to Patric, extendmap don't need vbflag, thus verid =0 + int rc = blockRsltnMgrPtr->lookupLocal((uint64_t)lbid, 0, false, (BRM::OID_t&)oid, dbRoot, partition, + segment, (uint32_t&)fbo); + return getRC(rc, ERR_BRM_LOOKUP_FBO); } //------------------------------------------------------------------------------ @@ -340,37 +300,35 @@ int BRMWrapper::getFboOffset(const uint64_t lbid, int& oid, //------------------------------------------------------------------------------ BRMWrapper* BRMWrapper::getInstance() { + if (m_instance == 0) + { + boost::mutex::scoped_lock lock(m_instanceCreateMutex); + if (m_instance == 0) { - boost::mutex::scoped_lock lock(m_instanceCreateMutex); + BRMWrapper* tmp = new BRMWrapper(); - if (m_instance == 0) - { - BRMWrapper* tmp = new BRMWrapper(); - - // Memory barrier makes sure the m_instance assignment is not - // mingled with the constructor code - atomicops::atomicMb(); - m_instance = tmp; - } + // Memory barrier makes sure the m_instance assignment is not + // mingled with the constructor code + atomicops::atomicMb(); + m_instance = tmp; } + } - return m_instance; + return m_instance; } //------------------------------------------------------------------------------ // Get HWM/extent information for each DBRoot in the specified column OID, // for the current PM. //------------------------------------------------------------------------------ -int BRMWrapper::getDbRootHWMInfo( const OID oid, - BRM::EmDbRootHWMInfo_v& emDbRootHwmInfos) +int BRMWrapper::getDbRootHWMInfo(const OID oid, BRM::EmDbRootHWMInfo_v& emDbRootHwmInfos) { - int rc = NO_ERROR; - uint16_t localModuleID = Config::getLocalModuleID(); - rc = blockRsltnMgrPtr->getDbRootHWMInfo( - oid, localModuleID, emDbRootHwmInfos); + int rc = NO_ERROR; + uint16_t localModuleID = Config::getLocalModuleID(); + rc = blockRsltnMgrPtr->getDbRootHWMInfo(oid, localModuleID, emDbRootHwmInfos); - // Temporary code for testing shared nothing + // Temporary code for testing shared nothing #if 0 EmDbRootHWMInfo info; info.dbRoot = 1; @@ -409,7 +367,7 @@ int BRMWrapper::getDbRootHWMInfo( const OID oid, info.totalBlocks = 0; // 0, 1 emDbRootHwmInfos.push_back( info ); #endif - return getRC( rc, ERR_BRM_DBROOT_HWMS ); + return getRC(rc, ERR_BRM_DBROOT_HWMS); } //------------------------------------------------------------------------------ @@ -417,14 +375,14 @@ int BRMWrapper::getDbRootHWMInfo( const OID oid, //------------------------------------------------------------------------------ int BRMWrapper::isReadWrite() { - int rc = blockRsltnMgrPtr->isReadWrite(); + int rc = blockRsltnMgrPtr->isReadWrite(); - if (rc == BRM::ERR_OK) - return NO_ERROR; - else if (rc == BRM::ERR_READONLY) - return ERR_BRM_READ_ONLY; - else - return ERR_BRM_GET_READ_WRITE; + if (rc == BRM::ERR_OK) + return NO_ERROR; + else if (rc == BRM::ERR_READONLY) + return ERR_BRM_READ_ONLY; + else + return ERR_BRM_GET_READ_WRITE; } //------------------------------------------------------------------------------ @@ -432,14 +390,14 @@ int BRMWrapper::isReadWrite() //------------------------------------------------------------------------------ int BRMWrapper::isShutdownPending(bool& bRollback, bool& bForce) { - int rc = blockRsltnMgrPtr->getSystemShutdownPending(bRollback, bForce); + int rc = blockRsltnMgrPtr->getSystemShutdownPending(bRollback, bForce); - if (rc < 0) - return ERR_BRM_GET_SHUTDOWN; - else if (rc > 0) - return ERR_BRM_SHUTDOWN; + if (rc < 0) + return ERR_BRM_GET_SHUTDOWN; + else if (rc > 0) + return ERR_BRM_SHUTDOWN; - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -447,22 +405,22 @@ int BRMWrapper::isShutdownPending(bool& bRollback, bool& bForce) //------------------------------------------------------------------------------ int BRMWrapper::isSuspendPending() { - bool bRollback; - int rc = blockRsltnMgrPtr->getSystemSuspendPending(bRollback); + bool bRollback; + int rc = blockRsltnMgrPtr->getSystemSuspendPending(bRollback); - if (rc < 0) - return ERR_BRM_GET_SUSPEND; - else if (rc > 0) - return ERR_BRM_SUSPEND; + if (rc < 0) + return ERR_BRM_GET_SUSPEND; + else if (rc > 0) + return ERR_BRM_SUSPEND; - rc = blockRsltnMgrPtr->getSystemSuspended(); + rc = blockRsltnMgrPtr->getSystemSuspended(); - if (rc < 0) - return ERR_BRM_GET_SUSPEND; - else if (rc > 0) - return ERR_BRM_SUSPEND; + if (rc < 0) + return ERR_BRM_GET_SUSPEND; + else if (rc > 0) + return ERR_BRM_SUSPEND; - return NO_ERROR; + return NO_ERROR; } // //------------------------------------------------------------------------------ @@ -470,15 +428,14 @@ int BRMWrapper::isSuspendPending() //------------------------------------------------------------------------------ int BRMWrapper::saveState() { - int rc = 0; + int rc = 0; - rc = blockRsltnMgrPtr->saveState(); + rc = blockRsltnMgrPtr->saveState(); - if (rc != NO_ERROR) - rc = ERR_BRM_SAVE_STATE; - - return rc; + if (rc != NO_ERROR) + rc = ERR_BRM_SAVE_STATE; + return rc; } //------------------------------------------------------------------------------ @@ -486,17 +443,17 @@ int BRMWrapper::saveState() //------------------------------------------------------------------------------ void BRMWrapper::saveBrmRc(int brmRc) { - int* dataPtr = m_ThreadDataPtr.get(); + int* dataPtr = m_ThreadDataPtr.get(); - if (dataPtr == 0) - { - dataPtr = new int(brmRc); - m_ThreadDataPtr.reset(dataPtr); - } - else - { - *dataPtr = brmRc; - } + if (dataPtr == 0) + { + dataPtr = new int(brmRc); + m_ThreadDataPtr.reset(dataPtr); + } + else + { + *dataPtr = brmRc; + } } //------------------------------------------------------------------------------ @@ -504,104 +461,92 @@ void BRMWrapper::saveBrmRc(int brmRc) // Resulting lock is returned in lockID. If table is already locked, then the // owner, pid, session id, and transaction of the current lock are returned. //------------------------------------------------------------------------------ -int BRMWrapper::getTableLock( OID tableOid, - std::string& ownerName, - uint32_t& processID, - int32_t& sessionID, - int32_t& transID, - uint64_t& lockID, - std::string& errMsg) +int BRMWrapper::getTableLock(OID tableOid, std::string& ownerName, uint32_t& processID, int32_t& sessionID, + int32_t& transID, uint64_t& lockID, std::string& errMsg) { - int rc = NO_ERROR; - lockID = 0; + int rc = NO_ERROR; + lockID = 0; - std::vector pmList; - pmList.push_back( Config::getLocalModuleID() ); + std::vector pmList; + pmList.push_back(Config::getLocalModuleID()); - try - { - lockID = blockRsltnMgrPtr->getTableLock( - pmList, tableOid, &ownerName, &processID, &sessionID, &transID, - BRM::LOADING); - } - catch (std::exception& ex) - { - errMsg = ex.what(); - rc = ERR_TBLLOCK_GET_LOCK; - } + try + { + lockID = blockRsltnMgrPtr->getTableLock(pmList, tableOid, &ownerName, &processID, &sessionID, &transID, + BRM::LOADING); + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = ERR_TBLLOCK_GET_LOCK; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Change the state of the specified lock to the indicated lock state. //------------------------------------------------------------------------------ -int BRMWrapper::changeTableLockState( uint64_t lockID, - BRM::LockState lockState, - bool& bChanged, - std::string& errMsg ) +int BRMWrapper::changeTableLockState(uint64_t lockID, BRM::LockState lockState, bool& bChanged, + std::string& errMsg) { - int rc = NO_ERROR; - bChanged = false; + int rc = NO_ERROR; + bChanged = false; - try - { - bChanged = blockRsltnMgrPtr->changeState( lockID, lockState ); - } - catch (std::exception& ex) - { - errMsg = ex.what(); - rc = ERR_TBLLOCK_CHANGE_STATE; - } + try + { + bChanged = blockRsltnMgrPtr->changeState(lockID, lockState); + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = ERR_TBLLOCK_CHANGE_STATE; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Release the table lock associated with the specified lockID. // bReleased will indicate whether the lock was released or not. //------------------------------------------------------------------------------ -int BRMWrapper::releaseTableLock( uint64_t lockID, - bool& bReleased, - std::string& errMsg ) +int BRMWrapper::releaseTableLock(uint64_t lockID, bool& bReleased, std::string& errMsg) { - int rc = NO_ERROR; - bReleased = false; + int rc = NO_ERROR; + bReleased = false; - try - { - bReleased = blockRsltnMgrPtr->releaseTableLock( lockID ); - } - catch (std::exception& ex) - { - errMsg = ex.what(); - rc = ERR_TBLLOCK_RELEASE_LOCK; - } + try + { + bReleased = blockRsltnMgrPtr->releaseTableLock(lockID); + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = ERR_TBLLOCK_RELEASE_LOCK; + } - return rc; + return rc; } //------------------------------------------------------------------------------ // Get information about the specified table lock. //------------------------------------------------------------------------------ -int BRMWrapper::getTableLockInfo( uint64_t lockID, - BRM::TableLockInfo* lockInfo, - bool& bLockExists, - std::string& errMsg ) +int BRMWrapper::getTableLockInfo(uint64_t lockID, BRM::TableLockInfo* lockInfo, bool& bLockExists, + std::string& errMsg) { - int rc = NO_ERROR; + int rc = NO_ERROR; - try - { - bLockExists = blockRsltnMgrPtr->getTableLockInfo( lockID, lockInfo ); - } - catch (std::exception& ex) - { - errMsg = ex.what(); - rc = ERR_TBLLOCK_GET_INFO; - } + try + { + bLockExists = blockRsltnMgrPtr->getTableLockInfo(lockID, lockInfo); + } + catch (std::exception& ex) + { + errMsg = ex.what(); + rc = ERR_TBLLOCK_GET_INFO; + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -612,15 +557,15 @@ int BRMWrapper::getTableLockInfo( uint64_t lockID, /* static */ int BRMWrapper::getBrmRc(bool reset) { - if (m_ThreadDataPtr.get() == 0) - return BRM::ERR_OK; + if (m_ThreadDataPtr.get() == 0) + return BRM::ERR_OK; - int brmRc = *m_ThreadDataPtr; + int brmRc = *m_ThreadDataPtr; - if (reset) - m_ThreadDataPtr.reset(new int(BRM::ERR_OK)); + if (reset) + m_ThreadDataPtr.reset(new int(BRM::ERR_OK)); - return brmRc; + return brmRc; } //------------------------------------------------------------------------------ @@ -633,978 +578,992 @@ int BRMWrapper::getBrmRc(bool reset) #define MAX_VERSION_BUFFER_SIZE 1024 -int BRMWrapper::copyVBBlock (IDBDataFile* pSourceFile, const OID sourceOid, - IDBDataFile* pTargetFile, const OID targetOid, - const std::vector < uint32_t >& fboList, - const BRM::VBRange& freeList, - size_t& nBlocksProcessed, - DbFileOp* pFileOp, - const size_t fboCurrentOffset) +int BRMWrapper::copyVBBlock(IDBDataFile* pSourceFile, const OID sourceOid, IDBDataFile* pTargetFile, + const OID targetOid, const std::vector& fboList, + const BRM::VBRange& freeList, size_t& nBlocksProcessed, DbFileOp* pFileOp, + const size_t fboCurrentOffset) { - size_t bufferSize = MAX_VERSION_BUFFER_SIZE; + size_t bufferSize = MAX_VERSION_BUFFER_SIZE; - if (freeList.size < bufferSize) bufferSize = freeList.size; + if (freeList.size < bufferSize) + bufferSize = freeList.size; - if ((fboList.size() - fboCurrentOffset) < bufferSize) bufferSize = fboList.size() - fboCurrentOffset; + if ((fboList.size() - fboCurrentOffset) < bufferSize) + bufferSize = fboList.size() - fboCurrentOffset; - unsigned char* buffer = (unsigned char*) malloc(bufferSize * BYTE_PER_BLOCK); + unsigned char* buffer = (unsigned char*)malloc(bufferSize * BYTE_PER_BLOCK); - if (buffer == NULL) + if (buffer == NULL) + { + return ERR_NO_MEM; + } + + size_t outputFileWritePointer = 0; + + while (outputFileWritePointer < freeList.size) + { + size_t numBlocksAvailableForWriting = freeList.size - outputFileWritePointer; + + if (bufferSize < numBlocksAvailableForWriting) + numBlocksAvailableForWriting = bufferSize; + + size_t numBlocksToBeWritten = 0; + // size_t startOffsetInInput = nBlocksProcessed; + size_t startOffsetInInput = fboCurrentOffset + nBlocksProcessed; + + // std::cout << "for oid " << sourceOid << " startOffsetInInput is " << startOffsetInInput << endl; + // Consume whole of the freeList + while ((numBlocksToBeWritten < numBlocksAvailableForWriting) && (startOffsetInInput < fboList.size())) { - return ERR_NO_MEM; + // determine how many contiguous source blocks are availale + size_t spaceAvailableInBuffer = numBlocksAvailableForWriting - numBlocksToBeWritten; + + size_t numContiguousBlocksAvaliableForReading = 1; + size_t tmp = startOffsetInInput; + + while (1) + { + if (numContiguousBlocksAvaliableForReading == spaceAvailableInBuffer) + break; + + if (tmp == (fboList.size() - 1)) + break; + + if ((fboList[tmp] + 1) != fboList[tmp + 1]) + break; + + tmp++; + numContiguousBlocksAvaliableForReading++; + } + + numContiguousBlocksAvaliableForReading = tmp - startOffsetInInput + 1; + + // determine how many contiguous blocks can be read from source file into buffer + size_t numCopyBlocks = (numContiguousBlocksAvaliableForReading < spaceAvailableInBuffer) + ? numContiguousBlocksAvaliableForReading + : spaceAvailableInBuffer; + + if (0 == numCopyBlocks) + break; + + // read source blocks into buffer + unsigned char* bufferOffset = buffer + (numBlocksToBeWritten * BYTE_PER_BLOCK); + ColumnOp* colOp = dynamic_cast(pFileOp); + Dctnry* dctnry = dynamic_cast(pFileOp); + + if (colOp != NULL) + pFileOp->chunkManager(colOp->chunkManager()); + else if (dctnry != NULL) + pFileOp->chunkManager(dctnry->chunkManager()); + else + pFileOp->chunkManager(NULL); + + size_t rwSize = + pFileOp->readDbBlocks(pSourceFile, bufferOffset, fboList[startOffsetInInput], numCopyBlocks); + + if (rwSize != numCopyBlocks) + { + if (buffer) + free(buffer); + + // std::cout << " error when processing startOffsetInInput:fbo = " << startOffsetInInput + // <<":"< 0) { - size_t numBlocksAvailableForWriting = freeList.size - outputFileWritePointer; + // Seek into target file + size_t tgtOffset = (freeList.vbFBO + outputFileWritePointer) * BYTE_PER_BLOCK; + int wc = pTargetFile->seek(tgtOffset, 0); - if (bufferSize < numBlocksAvailableForWriting) numBlocksAvailableForWriting = bufferSize; + if (wc != NO_ERROR) + { + std::string errMsgStr; + Convertor::mapErrnoToString(errno, errMsgStr); + logging::Message::Args args; + args.add((uint64_t)targetOid); + args.add((uint64_t)tgtOffset); + args.add(std::string(errMsgStr)); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_CRITICAL, logging::M0079); - size_t numBlocksToBeWritten = 0; - // size_t startOffsetInInput = nBlocksProcessed; - size_t startOffsetInInput = fboCurrentOffset + nBlocksProcessed; + if (buffer) + free(buffer); - //std::cout << "for oid " << sourceOid << " startOffsetInInput is " << startOffsetInInput << endl; - // Consume whole of the freeList - while ((numBlocksToBeWritten < numBlocksAvailableForWriting) - && (startOffsetInInput < fboList.size())) - { + return ERR_BRM_VB_COPY_SEEK_VB; + } - // determine how many contiguous source blocks are availale - size_t spaceAvailableInBuffer = numBlocksAvailableForWriting - numBlocksToBeWritten; + size_t rwSize = pTargetFile->write(buffer, BYTE_PER_BLOCK * numBlocksToBeWritten) / BYTE_PER_BLOCK; - size_t numContiguousBlocksAvaliableForReading = 1; - size_t tmp = startOffsetInInput; + if (rwSize != numBlocksToBeWritten) + { + if (buffer) + free(buffer); - while (1) - { - if (numContiguousBlocksAvaliableForReading == spaceAvailableInBuffer) break; + return ERR_BRM_VB_COPY_WRITE; + } - if (tmp == (fboList.size() - 1)) break; - - if ((fboList[tmp] + 1) != fboList[tmp + 1]) break; - - tmp++; - numContiguousBlocksAvaliableForReading++; - } - - numContiguousBlocksAvaliableForReading = tmp - startOffsetInInput + 1; - - // determine how many contiguous blocks can be read from source file into buffer - size_t numCopyBlocks = (numContiguousBlocksAvaliableForReading < spaceAvailableInBuffer) ? - numContiguousBlocksAvaliableForReading : spaceAvailableInBuffer; - - if (0 == numCopyBlocks) break; - - // read source blocks into buffer - unsigned char* bufferOffset = buffer + (numBlocksToBeWritten * BYTE_PER_BLOCK); - ColumnOp* colOp = dynamic_cast(pFileOp); - Dctnry* dctnry = dynamic_cast(pFileOp); - - if (colOp != NULL) - pFileOp->chunkManager(colOp->chunkManager()); - else if (dctnry != NULL) - pFileOp->chunkManager(dctnry->chunkManager()); - else - pFileOp->chunkManager(NULL); - - size_t rwSize = pFileOp->readDbBlocks(pSourceFile, bufferOffset, - fboList[startOffsetInInput], numCopyBlocks); - - if (rwSize != numCopyBlocks) - { - if (buffer) free(buffer); - - //std::cout << " error when processing startOffsetInInput:fbo = " << startOffsetInInput <<":"< 0) - { - // Seek into target file - size_t tgtOffset = (freeList.vbFBO + outputFileWritePointer) * BYTE_PER_BLOCK; - int wc = pTargetFile->seek(tgtOffset, 0); - - if (wc != NO_ERROR) - { - std::string errMsgStr; - Convertor::mapErrnoToString(errno, errMsgStr); - logging::Message::Args args; - args.add((uint64_t)targetOid); - args.add((uint64_t)tgtOffset); - args.add(std::string(errMsgStr)); - SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_CRITICAL, logging::M0079); - - if (buffer) free(buffer); - - return ERR_BRM_VB_COPY_SEEK_VB; - } - - size_t rwSize = pTargetFile->write(buffer, BYTE_PER_BLOCK * numBlocksToBeWritten) / BYTE_PER_BLOCK; - - if (rwSize != numBlocksToBeWritten) - { - if (buffer) free(buffer); - - return ERR_BRM_VB_COPY_WRITE; - } - - outputFileWritePointer += numBlocksToBeWritten; - nBlocksProcessed += numBlocksToBeWritten; - - } - else // There was nothing in the buffer, either source list or free list is finished - { - if (buffer) free(buffer); - - return 0; - } + outputFileWritePointer += numBlocksToBeWritten; + nBlocksProcessed += numBlocksToBeWritten; } + else // There was nothing in the buffer, either source list or free list is finished + { + if (buffer) + free(buffer); - if (buffer) free(buffer); + return 0; + } + } - return 0; + if (buffer) + free(buffer); + + return 0; } -int BRMWrapper::copyVBBlock(IDBDataFile* pSourceFile, IDBDataFile* pTargetFile, - const uint64_t sourceFbo, const uint64_t targetFbo, - DbFileOp* fileOp, const Column& column) +int BRMWrapper::copyVBBlock(IDBDataFile* pSourceFile, IDBDataFile* pTargetFile, const uint64_t sourceFbo, + const uint64_t targetFbo, DbFileOp* fileOp, const Column& column) { - size_t rwSize; - unsigned char buf[BYTE_PER_BLOCK]; - //add new error code for versioning error - rwSize = pSourceFile->pread(buf, sourceFbo * BYTE_PER_BLOCK, BYTE_PER_BLOCK); + size_t rwSize; + unsigned char buf[BYTE_PER_BLOCK]; + // add new error code for versioning error + rwSize = pSourceFile->pread(buf, sourceFbo * BYTE_PER_BLOCK, BYTE_PER_BLOCK); - if ((int) rwSize != BYTE_PER_BLOCK) - return ERR_BRM_VB_COPY_READ; + if ((int)rwSize != BYTE_PER_BLOCK) + return ERR_BRM_VB_COPY_READ; - rwSize = fileOp->restoreBlock(pTargetFile, buf, targetFbo); + rwSize = fileOp->restoreBlock(pTargetFile, buf, targetFbo); - if ((int) rwSize != BYTE_PER_BLOCK) - return ERR_BRM_VB_COPY_WRITE; - else - return NO_ERROR; + if ((int)rwSize != BYTE_PER_BLOCK) + return ERR_BRM_VB_COPY_WRITE; + else + return NO_ERROR; } int BRMWrapper::commit(const VER_t transID) { - int rc = blockRsltnMgrPtr->vbCommit(transID); - return getRC(rc, ERR_BRM_COMMIT); + int rc = blockRsltnMgrPtr->vbCommit(transID); + return getRC(rc, ERR_BRM_COMMIT); } IDBDataFile* BRMWrapper::openFile(const File& fileInfo, const char* mode, const bool bCache) { - IDBDataFile* pFile; - char fileName[FILE_NAME_SIZE]; + IDBDataFile* pFile; + char fileName[FILE_NAME_SIZE]; - if (bCache && fileInfo.oid == m_curVBOid && m_curVBFile != NULL) - return m_curVBFile; + if (bCache && fileInfo.oid == m_curVBOid && m_curVBFile != NULL) + return m_curVBFile; - FileOp fileOp; + FileOp fileOp; - if (fileInfo.oid < 1000) //Cannot have more than 999 version buffer files tp prevent oid collision + if (fileInfo.oid < 1000) // Cannot have more than 999 version buffer files tp prevent oid collision + { + RETURN_ON_WE_ERROR(fileOp.getVBFileName(fileInfo.oid, fileName), NULL); + } + else + { + RETURN_ON_WE_ERROR( + fileOp.getFileName(fileInfo.oid, fileName, fileInfo.fDbRoot, fileInfo.fPartition, fileInfo.fSegment), + NULL); + } + + // disable buffering for versionbuffer file by passing USE_NOVBUF + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, mode, + IDBDataFile::USE_NOVBUF); + + if (pFile && bCache) + { + if (m_curVBOid != (OID)INVALID_NUM) { - RETURN_ON_WE_ERROR(fileOp.getVBFileName (fileInfo.oid, fileName), NULL); - } - else - { - RETURN_ON_WE_ERROR(fileOp.getFileName (fileInfo.oid, fileName, fileInfo.fDbRoot, - fileInfo.fPartition, fileInfo.fSegment), NULL); + if (m_curVBOid != fileInfo.oid && m_curVBFile != NULL) + { + delete m_curVBFile; + m_curVBFile = 0; + } } - // disable buffering for versionbuffer file by passing USE_NOVBUF - pFile = IDBDataFile::open( - IDBPolicy::getType( fileName, IDBPolicy::WRITEENG ), - fileName, - mode, - IDBDataFile::USE_NOVBUF ); + m_curVBOid = fileInfo.oid; + m_curVBFile = pFile; + } - if (pFile && bCache) - { - if (m_curVBOid != (OID)INVALID_NUM) - { - if (m_curVBOid != fileInfo.oid && m_curVBFile != NULL) - { - delete m_curVBFile; - m_curVBFile = 0; - } - } - - m_curVBOid = fileInfo.oid; - m_curVBFile = pFile; - } - - return pFile; + return pFile; } int BRMWrapper::rollBack(const VER_t transID, int sessionId) { - std::vector lbidList; - std::vector lbidRangeList; - LBIDRange range; - OID_t vbOid, weOid, currentVbOid; - uint32_t vbFbo, weFbo; - size_t i; - bool vbFlag; - uint16_t vbDbRoot, weDbRoot, vbSegmentNum, weSegmentNum; - uint32_t vbPartitionNum, wePartitionNum; - File sourceFileInfo; - File targetFileInfo; - int rc = 0; - std::map columnOids; - //Check BRM status before processing. - rc = blockRsltnMgrPtr->isReadWrite(); + std::vector lbidList; + std::vector lbidRangeList; + LBIDRange range; + OID_t vbOid, weOid, currentVbOid; + uint32_t vbFbo, weFbo; + size_t i; + bool vbFlag; + uint16_t vbDbRoot, weDbRoot, vbSegmentNum, weSegmentNum; + uint32_t vbPartitionNum, wePartitionNum; + File sourceFileInfo; + File targetFileInfo; + int rc = 0; + std::map columnOids; + // Check BRM status before processing. + rc = blockRsltnMgrPtr->isReadWrite(); - if (rc != 0 ) - return ERR_BRM_READ_ONLY; + if (rc != 0) + return ERR_BRM_READ_ONLY; - rc = blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList); + rc = blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList); - if ( rc != 0 ) - { - if (rc == BRM::ERR_READONLY) - return ERR_BRM_READ_ONLY; - - return rc; - - } - - //RETURN_ON_WE_ERROR(blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList), ERR_BRM_GET_UNCOMM_LBID); - - if (isDebug(DEBUG_3)) - { - printf("\nIn rollBack, the transID is %d", transID); - printf("\n\t the size of umcommittedLBIDs is " -#if __LP64__ - "%lu", -#else - "%u", -#endif - lbidList.size()); - } - - //@Bug 2314. Optimize the version buffer open times. - currentVbOid = vbOid = 0; - vbOid = currentVbOid; - sourceFileInfo.oid = currentVbOid; - sourceFileInfo.fPartition = 0; - sourceFileInfo.fSegment = 0; - size_t rootCnt = Config::DBRootCount(); - sourceFileInfo.fDbRoot = (vbOid % rootCnt) + 1; - IDBDataFile* pSourceFile; - IDBDataFile* pTargetFile; - RETURN_ON_NULL((pSourceFile = openFile(sourceFileInfo, "r+b")), ERR_VB_FILE_NOT_EXIST); - - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); - - DbFileOp fileOp; - fileOp.setTransId(transID); - ChunkManager chunkManager; - chunkManager.fileOp(&fileOp); - FileOpenMap fileOpenList; - //@bug3224, sort lbidList based on lbid - sort(lbidList.begin(), lbidList.end()); - - try - { - for (i = 0; i < lbidList.size(); i++) - { - QueryContext verID(transID); - VER_t outVer; - range.start = lbidList[i]; - range.size = 1; - lbidRangeList.push_back(range); - //timer.start("vssLookup"); - // get version id - RETURN_ON_WE_ERROR( - blockRsltnMgrPtr->vssLookup(lbidList[i], verID, transID, &outVer, &vbFlag, true), - ERR_BRM_LOOKUP_VERSION); - //timer.stop("vssLookup"); - // copy buffer back - //look for the block in extentmap - //timer.start("lookupLocalEX"); - RETURN_ON_WE_ERROR( - blockRsltnMgrPtr->lookupLocal(lbidList[i], outVer, false, weOid, - weDbRoot, wePartitionNum, weSegmentNum, weFbo), ERR_EXTENTMAP_LOOKUP); - //timer.stop("lookupLocalEX"); - Column column; - execplan::CalpontSystemCatalog::ColType colType = systemCatalogPtr->colType(weOid); - columnOids[weOid] = weOid; - - //This must be a dict oid - if (colType.columnOID == 0) - { - colType = systemCatalogPtr->colTypeDct(weOid); - - idbassert(colType.columnOID != 0); - idbassert(colType.ddn.dictOID == weOid); - } - - CalpontSystemCatalog::ColDataType colDataType = colType.colDataType; - ColType weColType; - Convertor::convertColType(colDataType, colType.colWidth, weColType); - column.colWidth = Convertor::getCorrectRowWidth(colDataType, colType.colWidth); - column.colType = weColType; - column.colDataType = colDataType; - column.dataFile.fid = weOid; - column.dataFile.fDbRoot = weDbRoot; - column.dataFile.fPartition = wePartitionNum; - column.dataFile.fSegment = weSegmentNum; - column.compressionType = colType.compressionType; - - if (colType.compressionType == 0) - fileOp.chunkManager(NULL); - else - fileOp.chunkManager(&chunkManager); - - if (isDebug(DEBUG_3)) -#ifndef __LP64__ - printf( - "\n\tuncommitted lbid - lbidList[i]=%lld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", - lbidList[i], weOid, weFbo, outVer, weDbRoot); - -#else - printf( - "\n\tuncommitted lbid - lbidList[i]=%ld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", - lbidList[i], weOid, weFbo, outVer, weDbRoot); -#endif - //look for the block in the version buffer - //timer.start("lookupLocalVB"); - RETURN_ON_WE_ERROR(blockRsltnMgrPtr->lookupLocal(lbidList[i], outVer, true, vbOid, - vbDbRoot, vbPartitionNum, vbSegmentNum, vbFbo), ERR_BRM_LOOKUP_FBO); - -//timer.stop("lookupLocalVB"); - if (isDebug(DEBUG_3)) -#ifndef __LP64__ - printf("\n\tuncommitted lbid - lbidList[i]=%lld vbOid =%d vbFbo=%d\n", - lbidList[i], vbOid, vbFbo); - -#else - printf("\n\tuncommitted lbid - lbidList[i]=%ld vbOid =%d vbFbo=%d\n", - lbidList[i], vbOid, vbFbo); -#endif - - //@Bug 2293 Version buffer file information cannot be obtained from lookupLocal - if (vbOid != currentVbOid) - { - currentVbOid = vbOid; - //cout << "VB file changed to " << vbOid << endl; - delete pSourceFile; - sourceFileInfo.oid = currentVbOid; - sourceFileInfo.fPartition = 0; - sourceFileInfo.fSegment = 0; - sourceFileInfo.fDbRoot = (vbOid % rootCnt) + 1; - RETURN_ON_NULL((pSourceFile = openFile(sourceFileInfo, "r+b")), ERR_VB_FILE_NOT_EXIST); - } - - targetFileInfo.oid = weOid; - targetFileInfo.fPartition = wePartitionNum; - targetFileInfo.fSegment = weSegmentNum; - targetFileInfo.fDbRoot = weDbRoot; - -// printf("\n\tsource file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", sourceFileInfo.oid, sourceFileInfo.fPartition, sourceFileInfo.fSegment, sourceFileInfo.fDbRoot); -// printf("\n\ttarget file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", weOid, wePartitionNum, weSegmentNum, weDbRoot); - if (column.compressionType != 0) - { - pTargetFile = fileOp.getFilePtr(column, false); // @bug 5572 HDFS tmp file - } - else if (fileOpenList.find(targetFileInfo) != fileOpenList.end()) - { - pTargetFile = fileOpenList[targetFileInfo]; - } - else - { - pTargetFile = openFile(targetFileInfo, "r+b"); - - if (pTargetFile != NULL) - fileOpenList[targetFileInfo] = pTargetFile; - } - - if (pTargetFile == NULL) - { - rc = ERR_FILE_NOT_EXIST; - goto cleanup; - } - -//timer.start("copyVBBlock"); - rc = copyVBBlock(pSourceFile, pTargetFile, vbFbo, weFbo, &fileOp, column); - -//timer.stop("copyVBBlock"); - if (rc != NO_ERROR) - { - //@bug 4012, log an error to crit.log - logging::Message::MessageID msgId = 6; - SimpleSysLog* slog = SimpleSysLog::instance(); - logging::Message m( msgId ); - logging::Message::Args args; - - std::ostringstream oss; - WriteEngine::WErrorCodes ec; - oss << "Error in rolling back the block. lbid:oid:dbroot:partition:segment: " << lbidList[i] << ":" << weOid << ":" - << weDbRoot << ":" << wePartitionNum << ":" << weSegmentNum << " The error message is " << ec.errorString(rc); - args.add( oss.str() ); - slog->logMsg(args, logging::LOG_TYPE_CRITICAL, msgId); - goto cleanup; - } - } - } - catch ( runtime_error& ) - { - rc = ERR_TBL_SYSCAT_ERROR; - } - -//timer.start("vbRollback"); - // rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); -// timer.stop("vbRollback"); - if (rc != 0) - { - if (rc == BRM::ERR_READONLY) - return ERR_BRM_READ_ONLY; - else - return rc; - } - else - { - rc = NO_ERROR; - } - -cleanup: - delete pSourceFile; - - //Close all target files - // -- chunkManager managed files -// timer.start("flushChunks"); - if (rc == NO_ERROR) - { - rc = chunkManager.flushChunks(rc, columnOids); // write all active chunks to disk - rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); - } - else - chunkManager.cleanUp(columnOids); // close file w/o writing data to disk - -// timer.stop("flushChunks"); - // -- other files - FileOpenMap::const_iterator itor; - - for (itor = fileOpenList.begin(); itor != fileOpenList.end(); itor++) - { - delete itor->second; - } + if (rc != 0) + { + if (rc == BRM::ERR_READONLY) + return ERR_BRM_READ_ONLY; return rc; + } + + // RETURN_ON_WE_ERROR(blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList), ERR_BRM_GET_UNCOMM_LBID); + + if (isDebug(DEBUG_3)) + { + printf("\nIn rollBack, the transID is %d", transID); + printf( + "\n\t the size of umcommittedLBIDs is " +#if __LP64__ + "%lu", +#else + "%u", +#endif + lbidList.size()); + } + + //@Bug 2314. Optimize the version buffer open times. + currentVbOid = vbOid = 0; + vbOid = currentVbOid; + sourceFileInfo.oid = currentVbOid; + sourceFileInfo.fPartition = 0; + sourceFileInfo.fSegment = 0; + size_t rootCnt = Config::DBRootCount(); + sourceFileInfo.fDbRoot = (vbOid % rootCnt) + 1; + IDBDataFile* pSourceFile; + IDBDataFile* pTargetFile; + RETURN_ON_NULL((pSourceFile = openFile(sourceFileInfo, "r+b")), ERR_VB_FILE_NOT_EXIST); + + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); + + DbFileOp fileOp; + fileOp.setTransId(transID); + ChunkManager chunkManager; + chunkManager.fileOp(&fileOp); + FileOpenMap fileOpenList; + //@bug3224, sort lbidList based on lbid + sort(lbidList.begin(), lbidList.end()); + + try + { + for (i = 0; i < lbidList.size(); i++) + { + QueryContext verID(transID); + VER_t outVer; + range.start = lbidList[i]; + range.size = 1; + lbidRangeList.push_back(range); + // timer.start("vssLookup"); + // get version id + RETURN_ON_WE_ERROR(blockRsltnMgrPtr->vssLookup(lbidList[i], verID, transID, &outVer, &vbFlag, true), + ERR_BRM_LOOKUP_VERSION); + // timer.stop("vssLookup"); + // copy buffer back + // look for the block in extentmap + // timer.start("lookupLocalEX"); + RETURN_ON_WE_ERROR(blockRsltnMgrPtr->lookupLocal(lbidList[i], outVer, false, weOid, weDbRoot, + wePartitionNum, weSegmentNum, weFbo), + ERR_EXTENTMAP_LOOKUP); + // timer.stop("lookupLocalEX"); + Column column; + execplan::CalpontSystemCatalog::ColType colType = systemCatalogPtr->colType(weOid); + columnOids[weOid] = weOid; + + // This must be a dict oid + if (colType.columnOID == 0) + { + colType = systemCatalogPtr->colTypeDct(weOid); + + idbassert(colType.columnOID != 0); + idbassert(colType.ddn.dictOID == weOid); + } + + CalpontSystemCatalog::ColDataType colDataType = colType.colDataType; + ColType weColType; + Convertor::convertColType(colDataType, colType.colWidth, weColType); + column.colWidth = Convertor::getCorrectRowWidth(colDataType, colType.colWidth); + column.colType = weColType; + column.colDataType = colDataType; + column.dataFile.fid = weOid; + column.dataFile.fDbRoot = weDbRoot; + column.dataFile.fPartition = wePartitionNum; + column.dataFile.fSegment = weSegmentNum; + column.compressionType = colType.compressionType; + + if (colType.compressionType == 0) + fileOp.chunkManager(NULL); + else + fileOp.chunkManager(&chunkManager); + + if (isDebug(DEBUG_3)) +#ifndef __LP64__ + printf("\n\tuncommitted lbid - lbidList[i]=%lld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", + lbidList[i], weOid, weFbo, outVer, weDbRoot); + +#else + printf("\n\tuncommitted lbid - lbidList[i]=%ld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", lbidList[i], + weOid, weFbo, outVer, weDbRoot); +#endif + // look for the block in the version buffer + // timer.start("lookupLocalVB"); + RETURN_ON_WE_ERROR(blockRsltnMgrPtr->lookupLocal(lbidList[i], outVer, true, vbOid, vbDbRoot, + vbPartitionNum, vbSegmentNum, vbFbo), + ERR_BRM_LOOKUP_FBO); + + // timer.stop("lookupLocalVB"); + if (isDebug(DEBUG_3)) +#ifndef __LP64__ + printf("\n\tuncommitted lbid - lbidList[i]=%lld vbOid =%d vbFbo=%d\n", lbidList[i], vbOid, vbFbo); + +#else + printf("\n\tuncommitted lbid - lbidList[i]=%ld vbOid =%d vbFbo=%d\n", lbidList[i], vbOid, vbFbo); +#endif + + //@Bug 2293 Version buffer file information cannot be obtained from lookupLocal + if (vbOid != currentVbOid) + { + currentVbOid = vbOid; + // cout << "VB file changed to " << vbOid << endl; + delete pSourceFile; + sourceFileInfo.oid = currentVbOid; + sourceFileInfo.fPartition = 0; + sourceFileInfo.fSegment = 0; + sourceFileInfo.fDbRoot = (vbOid % rootCnt) + 1; + RETURN_ON_NULL((pSourceFile = openFile(sourceFileInfo, "r+b")), ERR_VB_FILE_NOT_EXIST); + } + + targetFileInfo.oid = weOid; + targetFileInfo.fPartition = wePartitionNum; + targetFileInfo.fSegment = weSegmentNum; + targetFileInfo.fDbRoot = weDbRoot; + + // printf("\n\tsource file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", + // sourceFileInfo.oid, sourceFileInfo.fPartition, sourceFileInfo.fSegment, sourceFileInfo.fDbRoot); + // printf("\n\ttarget file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", weOid, + // wePartitionNum, weSegmentNum, weDbRoot); + if (column.compressionType != 0) + { + pTargetFile = fileOp.getFilePtr(column, false); // @bug 5572 HDFS tmp file + } + else if (fileOpenList.find(targetFileInfo) != fileOpenList.end()) + { + pTargetFile = fileOpenList[targetFileInfo]; + } + else + { + pTargetFile = openFile(targetFileInfo, "r+b"); + + if (pTargetFile != NULL) + fileOpenList[targetFileInfo] = pTargetFile; + } + + if (pTargetFile == NULL) + { + rc = ERR_FILE_NOT_EXIST; + goto cleanup; + } + + // timer.start("copyVBBlock"); + rc = copyVBBlock(pSourceFile, pTargetFile, vbFbo, weFbo, &fileOp, column); + + // timer.stop("copyVBBlock"); + if (rc != NO_ERROR) + { + //@bug 4012, log an error to crit.log + logging::Message::MessageID msgId = 6; + SimpleSysLog* slog = SimpleSysLog::instance(); + logging::Message m(msgId); + logging::Message::Args args; + + std::ostringstream oss; + WriteEngine::WErrorCodes ec; + oss << "Error in rolling back the block. lbid:oid:dbroot:partition:segment: " << lbidList[i] << ":" + << weOid << ":" << weDbRoot << ":" << wePartitionNum << ":" << weSegmentNum + << " The error message is " << ec.errorString(rc); + args.add(oss.str()); + slog->logMsg(args, logging::LOG_TYPE_CRITICAL, msgId); + goto cleanup; + } + } + } + catch (runtime_error&) + { + rc = ERR_TBL_SYSCAT_ERROR; + } + + // timer.start("vbRollback"); + // rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); + // timer.stop("vbRollback"); + if (rc != 0) + { + if (rc == BRM::ERR_READONLY) + return ERR_BRM_READ_ONLY; + else + return rc; + } + else + { + rc = NO_ERROR; + } + +cleanup: + delete pSourceFile; + + // Close all target files + // -- chunkManager managed files + // timer.start("flushChunks"); + if (rc == NO_ERROR) + { + rc = chunkManager.flushChunks(rc, columnOids); // write all active chunks to disk + rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); + } + else + chunkManager.cleanUp(columnOids); // close file w/o writing data to disk + + // timer.stop("flushChunks"); + // -- other files + FileOpenMap::const_iterator itor; + + for (itor = fileOpenList.begin(); itor != fileOpenList.end(); itor++) + { + delete itor->second; + } + + return rc; } int BRMWrapper::rollBackBlocks(const VER_t transID, int sessionId) { - if (idbdatafile::IDBPolicy::useHdfs()) - return 0; + if (idbdatafile::IDBPolicy::useHdfs()) + return 0; - std::vector lbidList; - OID_t vbOid; - OID_t weOid; - OID_t currentVbOid = static_cast(-1); - uint32_t vbFbo, weFbo; - size_t i; - VER_t verID = (VER_t) transID; + std::vector lbidList; + OID_t vbOid; + OID_t weOid; + OID_t currentVbOid = static_cast(-1); + uint32_t vbFbo, weFbo; + size_t i; + VER_t verID = (VER_t)transID; - uint16_t vbDbRoot, weDbRoot, vbSegmentNum, weSegmentNum; - uint32_t vbPartitionNum, wePartitionNum; - File sourceFileInfo; - File targetFileInfo; - Config config; - config.initConfigCache(); - std::vector rootList; - config.getRootIdList( rootList ); - std::map dbrootPmMap; + uint16_t vbDbRoot, weDbRoot, vbSegmentNum, weSegmentNum; + uint32_t vbPartitionNum, wePartitionNum; + File sourceFileInfo; + File targetFileInfo; + Config config; + config.initConfigCache(); + std::vector rootList; + config.getRootIdList(rootList); + std::map dbrootPmMap; - for (i = 0; i < rootList.size(); i++) - { - dbrootPmMap[rootList[i]] = rootList[i]; - } + for (i = 0; i < rootList.size(); i++) + { + dbrootPmMap[rootList[i]] = rootList[i]; + } - int rc = 0; - std::map columnOids; - //Check BRM status before processing. - rc = blockRsltnMgrPtr->isReadWrite(); + int rc = 0; + std::map columnOids; + // Check BRM status before processing. + rc = blockRsltnMgrPtr->isReadWrite(); - if (rc != 0 ) - return ERR_BRM_READ_ONLY; + if (rc != 0) + return ERR_BRM_READ_ONLY; - rc = blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList); + rc = blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList); - if ( rc != 0 ) - { - if (rc == BRM::ERR_READONLY) - return ERR_BRM_READ_ONLY; + if (rc != 0) + { + if (rc == BRM::ERR_READONLY) + return ERR_BRM_READ_ONLY; - return rc; + return rc; + } - } + // std::cout << "rollBackBlocks get uncommited lbid " << lbidList.size() << std::endl; + // RETURN_ON_WE_ERROR(blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList), ERR_BRM_GET_UNCOMM_LBID); - //std::cout << "rollBackBlocks get uncommited lbid " << lbidList.size() << std::endl; - //RETURN_ON_WE_ERROR(blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList), ERR_BRM_GET_UNCOMM_LBID); - - if (isDebug(DEBUG_3)) - { - printf("\nIn rollBack, the transID is %d", transID); - printf("\n\t the size of umcommittedLBIDs is " + if (isDebug(DEBUG_3)) + { + printf("\nIn rollBack, the transID is %d", transID); + printf( + "\n\t the size of umcommittedLBIDs is " #if __LP64__ - "%lu", + "%lu", #else - "%u", + "%u", #endif - lbidList.size()); - } + lbidList.size()); + } - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionId); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC); - DbFileOp fileOp; - fileOp.setTransId(transID); - ChunkManager chunkManager; - chunkManager.fileOp(&fileOp); - FileOpenMap fileOpenList; - //@bug3224, sort lbidList based on lbid - sort(lbidList.begin(), lbidList.end()); + DbFileOp fileOp; + fileOp.setTransId(transID); + ChunkManager chunkManager; + chunkManager.fileOp(&fileOp); + FileOpenMap fileOpenList; + //@bug3224, sort lbidList based on lbid + sort(lbidList.begin(), lbidList.end()); - IDBDataFile* pSourceFile = 0; - IDBDataFile* pTargetFile = 0; - std::map::const_iterator dbrootPmMapItor; - std::string errorMsg; + IDBDataFile* pSourceFile = 0; + IDBDataFile* pTargetFile = 0; + std::map::const_iterator dbrootPmMapItor; + std::string errorMsg; - std::vector files; + std::vector files; - for (i = 0; i < lbidList.size(); i++) + for (i = 0; i < lbidList.size(); i++) + { + verID = (VER_t)transID; + // timer.start("vssLookup"); + // get version id + + verID = blockRsltnMgrPtr->getHighestVerInVB(lbidList[i], transID); + + if (verID < 0) { - verID = (VER_t) transID; - //timer.start("vssLookup"); - // get version id - - verID = blockRsltnMgrPtr->getHighestVerInVB(lbidList[i], transID); - - if (verID < 0) - { - std::ostringstream oss; - BRM::errString(verID, errorMsg); - oss << "vssLookup error encountered while looking up lbid " << lbidList[i] << " and error code is " << verID << " with message " << errorMsg; - throw std::runtime_error(oss.str()); - } - - //timer.stop("vssLookup"); - // copy buffer back - //look for the block in extentmap - //timer.start("lookupLocalEX"); - rc = blockRsltnMgrPtr->lookupLocal(lbidList[i], /*transID*/verID, false, weOid, - weDbRoot, wePartitionNum, weSegmentNum, weFbo); - - if ( rc != 0) - { - std::ostringstream oss; - BRM::errString(rc, errorMsg); - oss << "lookupLocal from extent map error encountered while looking up lbid:verID " << lbidList[i] << ":" - << (uint32_t)verID << " and error code is " << rc << " with message " << errorMsg; - throw std::runtime_error(oss.str()); - } - - //Check whether this lbid is on this PM. - dbrootPmMapItor = dbrootPmMap.find(weDbRoot); - - if (dbrootPmMapItor == dbrootPmMap.end()) - continue; - - //timer.stop("lookupLocalEX"); - Column column; - execplan::CalpontSystemCatalog::ColType colType = systemCatalogPtr->colType(weOid); - columnOids[weOid] = weOid; - - //This must be a dict oid - if (colType.columnOID == 0) - { - colType = systemCatalogPtr->colTypeDct(weOid); - - idbassert(colType.columnOID != 0); - idbassert(colType.ddn.dictOID == weOid); - } - - CalpontSystemCatalog::ColDataType colDataType = colType.colDataType; - ColType weColType; - Convertor::convertColType(colDataType, colType.colWidth, weColType); - column.colWidth = Convertor::getCorrectRowWidth(colDataType, colType.colWidth); - column.colType = weColType; - column.colDataType = colDataType; - column.dataFile.fid = weOid; - column.dataFile.fDbRoot = weDbRoot; - column.dataFile.fPartition = wePartitionNum; - column.dataFile.fSegment = weSegmentNum; - column.compressionType = colType.compressionType; - - BRM::FileInfo aFile; - aFile.oid = weOid; - aFile.partitionNum = wePartitionNum; - aFile.dbRoot = weDbRoot; - aFile.segmentNum = weSegmentNum; - aFile.compType = colType.compressionType; - files.push_back(aFile); - - if (colType.compressionType == 0) - fileOp.chunkManager(NULL); - else - fileOp.chunkManager(&chunkManager); - - if (isDebug(DEBUG_3)) -#ifndef __LP64__ - printf( - "\n\tuncommitted lbid - lbidList[i]=%lld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", - lbidList[i], weOid, weFbo, verID, weDbRoot); - -#else - printf( - "\n\tuncommitted lbid - lbidList[i]=%ld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", - lbidList[i], weOid, weFbo, verID, weDbRoot); -#endif - //look for the block in the version buffer - //timer.start("lookupLocalVB"); - rc = blockRsltnMgrPtr->lookupLocal(lbidList[i], verID, true, vbOid, - vbDbRoot, vbPartitionNum, vbSegmentNum, vbFbo); - - if ( rc != 0) - { - std::ostringstream oss; - BRM::errString(rc, errorMsg); - oss << "lookupLocal from version buffer error encountered while looking up lbid:verID " << lbidList[i] << ":" - << (uint32_t)verID << " and error code is " << rc << " with message " << errorMsg; - throw std::runtime_error(oss.str()); - } - - if (pSourceFile == 0) //@Bug 2314. Optimize the version buffer open times. - { - currentVbOid = vbOid; - sourceFileInfo.oid = currentVbOid; - sourceFileInfo.fPartition = 0; - sourceFileInfo.fSegment = 0; - sourceFileInfo.fDbRoot = weDbRoot; - errno = 0; - pSourceFile = openFile(sourceFileInfo, "r+b"); - - if (pSourceFile == NULL) - { - std::ostringstream oss; - Convertor::mapErrnoToString(errno, errorMsg); - oss << "Error encountered while opening version buffer file oid:dbroot = " << currentVbOid << ":" - << weDbRoot << " and error message:" << errorMsg; - throw std::runtime_error(oss.str()); - } - } - -//timer.stop("lookupLocalVB"); - if (isDebug(DEBUG_3)) -#ifndef __LP64__ - printf("\n\tuncommitted lbid - lbidList[i]=%lld vbOid =%d vbFbo=%d\n", - lbidList[i], vbOid, vbFbo); - -#else - printf("\n\tuncommitted lbid - lbidList[i]=%ld vbOid =%d vbFbo=%d\n", - lbidList[i], vbOid, vbFbo); -#endif - - //@Bug 2293 Version buffer file information cannot be obtained from lookupLocal - if (vbOid != currentVbOid) - { - currentVbOid = vbOid; - //cout << "VB file changed to " << vbOid << endl; - delete pSourceFile; - sourceFileInfo.oid = currentVbOid; - sourceFileInfo.fPartition = 0; - sourceFileInfo.fSegment = 0; - sourceFileInfo.fDbRoot = weDbRoot; - errno = 0; - pSourceFile = openFile(sourceFileInfo, "r+b"); - - if (pSourceFile == NULL) - { - std::ostringstream oss; - Convertor::mapErrnoToString(errno, errorMsg); - oss << "Error encountered while opening version buffer file oid:dbroot = " << currentVbOid << ":" - << weDbRoot << " and error message:" << errorMsg; - throw std::runtime_error(oss.str()); - } - } - - targetFileInfo.oid = weOid; - targetFileInfo.fPartition = wePartitionNum; - targetFileInfo.fSegment = weSegmentNum; - targetFileInfo.fDbRoot = weDbRoot; -// printf("\n\tsource file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", sourceFileInfo.oid, sourceFileInfo.fPartition, sourceFileInfo.fSegment, sourceFileInfo.fDbRoot); -// printf("\n\ttarget file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", weOid, wePartitionNum, weSegmentNum, weDbRoot); - //Check whether the file is on this pm. - - if (column.compressionType != 0) - { - pTargetFile = fileOp.getFilePtr(column, false); // @bug 5572 HDFS tmp file - } - else if (fileOpenList.find(targetFileInfo) != fileOpenList.end()) - { - pTargetFile = fileOpenList[targetFileInfo]; - } - else - { - pTargetFile = openFile(targetFileInfo, "r+b"); - - if (pTargetFile != NULL) - fileOpenList[targetFileInfo] = pTargetFile; - } - - if (pTargetFile == NULL) - { - std::ostringstream oss; - Convertor::mapErrnoToString(errno, errorMsg); - oss << "Error encountered while opening source file oid:dbroot:partition:segment = " << weOid << ":" - << weDbRoot << ":" << wePartitionNum << ":" << weSegmentNum << " and error message:" << errorMsg; - errorMsg = oss.str(); - goto cleanup; - } - -//timer.start("copyVBBlock"); - std::vector lbidRangeList; - BRM::LBIDRange range; - range.start = lbidList[i]; - range.size = 1; - lbidRangeList.push_back(range); - rc = blockRsltnMgrPtr->dmlLockLBIDRanges(lbidRangeList, transID); - - if (rc != 0 ) - { - BRM::errString(rc, errorMsg); - goto cleanup; - } - - rc = copyVBBlock(pSourceFile, pTargetFile, vbFbo, weFbo, &fileOp, column); - - //cout << "WES rolled block " << lbidList[i] << endl; - if (rc != 0) - { - std::ostringstream oss; - oss << "Error encountered while copying lbid " << lbidList[i] << " to source file oid:dbroot:partition:segment = " << weOid << ":" - << weDbRoot << ":" << wePartitionNum << ":" << weSegmentNum; - errorMsg = oss.str(); - goto cleanup; - } - - pTargetFile->flush(); - rc = blockRsltnMgrPtr->dmlReleaseLBIDRanges(lbidRangeList); - - if (rc != 0 ) - { - BRM::errString(rc, errorMsg); - goto cleanup; - } - -//timer.stop("copyVBBlock"); - if (rc != NO_ERROR) - goto cleanup; + std::ostringstream oss; + BRM::errString(verID, errorMsg); + oss << "vssLookup error encountered while looking up lbid " << lbidList[i] << " and error code is " + << verID << " with message " << errorMsg; + throw std::runtime_error(oss.str()); } -//timer.start("vbRollback"); - // rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); -// timer.stop("vbRollback"); + // timer.stop("vssLookup"); + // copy buffer back + // look for the block in extentmap + // timer.start("lookupLocalEX"); + rc = blockRsltnMgrPtr->lookupLocal(lbidList[i], /*transID*/ verID, false, weOid, weDbRoot, wePartitionNum, + weSegmentNum, weFbo); + if (rc != 0) { - if (rc == BRM::ERR_READONLY) - return ERR_BRM_READ_ONLY; - else - return rc; + std::ostringstream oss; + BRM::errString(rc, errorMsg); + oss << "lookupLocal from extent map error encountered while looking up lbid:verID " << lbidList[i] + << ":" << (uint32_t)verID << " and error code is " << rc << " with message " << errorMsg; + throw std::runtime_error(oss.str()); + } + + // Check whether this lbid is on this PM. + dbrootPmMapItor = dbrootPmMap.find(weDbRoot); + + if (dbrootPmMapItor == dbrootPmMap.end()) + continue; + + // timer.stop("lookupLocalEX"); + Column column; + execplan::CalpontSystemCatalog::ColType colType = systemCatalogPtr->colType(weOid); + columnOids[weOid] = weOid; + + // This must be a dict oid + if (colType.columnOID == 0) + { + colType = systemCatalogPtr->colTypeDct(weOid); + + idbassert(colType.columnOID != 0); + idbassert(colType.ddn.dictOID == weOid); + } + + CalpontSystemCatalog::ColDataType colDataType = colType.colDataType; + ColType weColType; + Convertor::convertColType(colDataType, colType.colWidth, weColType); + column.colWidth = Convertor::getCorrectRowWidth(colDataType, colType.colWidth); + column.colType = weColType; + column.colDataType = colDataType; + column.dataFile.fid = weOid; + column.dataFile.fDbRoot = weDbRoot; + column.dataFile.fPartition = wePartitionNum; + column.dataFile.fSegment = weSegmentNum; + column.compressionType = colType.compressionType; + + BRM::FileInfo aFile; + aFile.oid = weOid; + aFile.partitionNum = wePartitionNum; + aFile.dbRoot = weDbRoot; + aFile.segmentNum = weSegmentNum; + aFile.compType = colType.compressionType; + files.push_back(aFile); + + if (colType.compressionType == 0) + fileOp.chunkManager(NULL); + else + fileOp.chunkManager(&chunkManager); + + if (isDebug(DEBUG_3)) +#ifndef __LP64__ + printf("\n\tuncommitted lbid - lbidList[i]=%lld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", lbidList[i], + weOid, weFbo, verID, weDbRoot); + +#else + printf("\n\tuncommitted lbid - lbidList[i]=%ld weOid =%d weFbo=%d verID=%d, weDbRoot=%d", lbidList[i], + weOid, weFbo, verID, weDbRoot); +#endif + // look for the block in the version buffer + // timer.start("lookupLocalVB"); + rc = blockRsltnMgrPtr->lookupLocal(lbidList[i], verID, true, vbOid, vbDbRoot, vbPartitionNum, + vbSegmentNum, vbFbo); + + if (rc != 0) + { + std::ostringstream oss; + BRM::errString(rc, errorMsg); + oss << "lookupLocal from version buffer error encountered while looking up lbid:verID " << lbidList[i] + << ":" << (uint32_t)verID << " and error code is " << rc << " with message " << errorMsg; + throw std::runtime_error(oss.str()); + } + + if (pSourceFile == 0) //@Bug 2314. Optimize the version buffer open times. + { + currentVbOid = vbOid; + sourceFileInfo.oid = currentVbOid; + sourceFileInfo.fPartition = 0; + sourceFileInfo.fSegment = 0; + sourceFileInfo.fDbRoot = weDbRoot; + errno = 0; + pSourceFile = openFile(sourceFileInfo, "r+b"); + + if (pSourceFile == NULL) + { + std::ostringstream oss; + Convertor::mapErrnoToString(errno, errorMsg); + oss << "Error encountered while opening version buffer file oid:dbroot = " << currentVbOid << ":" + << weDbRoot << " and error message:" << errorMsg; + throw std::runtime_error(oss.str()); + } + } + + // timer.stop("lookupLocalVB"); + if (isDebug(DEBUG_3)) +#ifndef __LP64__ + printf("\n\tuncommitted lbid - lbidList[i]=%lld vbOid =%d vbFbo=%d\n", lbidList[i], vbOid, vbFbo); + +#else + printf("\n\tuncommitted lbid - lbidList[i]=%ld vbOid =%d vbFbo=%d\n", lbidList[i], vbOid, vbFbo); +#endif + + //@Bug 2293 Version buffer file information cannot be obtained from lookupLocal + if (vbOid != currentVbOid) + { + currentVbOid = vbOid; + // cout << "VB file changed to " << vbOid << endl; + delete pSourceFile; + sourceFileInfo.oid = currentVbOid; + sourceFileInfo.fPartition = 0; + sourceFileInfo.fSegment = 0; + sourceFileInfo.fDbRoot = weDbRoot; + errno = 0; + pSourceFile = openFile(sourceFileInfo, "r+b"); + + if (pSourceFile == NULL) + { + std::ostringstream oss; + Convertor::mapErrnoToString(errno, errorMsg); + oss << "Error encountered while opening version buffer file oid:dbroot = " << currentVbOid << ":" + << weDbRoot << " and error message:" << errorMsg; + throw std::runtime_error(oss.str()); + } + } + + targetFileInfo.oid = weOid; + targetFileInfo.fPartition = wePartitionNum; + targetFileInfo.fSegment = weSegmentNum; + targetFileInfo.fDbRoot = weDbRoot; + // printf("\n\tsource file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", sourceFileInfo.oid, + // sourceFileInfo.fPartition, sourceFileInfo.fSegment, sourceFileInfo.fDbRoot); printf("\n\ttarget + // file info - oid =%d fPartition=%d fSegment=%d, fDbRoot=%d", weOid, wePartitionNum, weSegmentNum, + // weDbRoot); + // Check whether the file is on this pm. + + if (column.compressionType != 0) + { + pTargetFile = fileOp.getFilePtr(column, false); // @bug 5572 HDFS tmp file + } + else if (fileOpenList.find(targetFileInfo) != fileOpenList.end()) + { + pTargetFile = fileOpenList[targetFileInfo]; } else { - rc = NO_ERROR; + pTargetFile = openFile(targetFileInfo, "r+b"); + + if (pTargetFile != NULL) + fileOpenList[targetFileInfo] = pTargetFile; } + if (pTargetFile == NULL) + { + std::ostringstream oss; + Convertor::mapErrnoToString(errno, errorMsg); + oss << "Error encountered while opening source file oid:dbroot:partition:segment = " << weOid << ":" + << weDbRoot << ":" << wePartitionNum << ":" << weSegmentNum << " and error message:" << errorMsg; + errorMsg = oss.str(); + goto cleanup; + } + + // timer.start("copyVBBlock"); + std::vector lbidRangeList; + BRM::LBIDRange range; + range.start = lbidList[i]; + range.size = 1; + lbidRangeList.push_back(range); + rc = blockRsltnMgrPtr->dmlLockLBIDRanges(lbidRangeList, transID); + + if (rc != 0) + { + BRM::errString(rc, errorMsg); + goto cleanup; + } + + rc = copyVBBlock(pSourceFile, pTargetFile, vbFbo, weFbo, &fileOp, column); + + // cout << "WES rolled block " << lbidList[i] << endl; + if (rc != 0) + { + std::ostringstream oss; + oss << "Error encountered while copying lbid " << lbidList[i] + << " to source file oid:dbroot:partition:segment = " << weOid << ":" << weDbRoot << ":" + << wePartitionNum << ":" << weSegmentNum; + errorMsg = oss.str(); + goto cleanup; + } + + pTargetFile->flush(); + rc = blockRsltnMgrPtr->dmlReleaseLBIDRanges(lbidRangeList); + + if (rc != 0) + { + BRM::errString(rc, errorMsg); + goto cleanup; + } + + // timer.stop("copyVBBlock"); + if (rc != NO_ERROR) + goto cleanup; + } + + // timer.start("vbRollback"); + // rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); + // timer.stop("vbRollback"); + if (rc != 0) + { + if (rc == BRM::ERR_READONLY) + return ERR_BRM_READ_ONLY; + else + return rc; + } + else + { + rc = NO_ERROR; + } + cleanup: - if (pSourceFile) - { - delete pSourceFile; - } + if (pSourceFile) + { + delete pSourceFile; + } - //Close all target files - // -- chunkManager managed files -// timer.start("flushChunks"); - if (rc == NO_ERROR) - { - rc = chunkManager.flushChunks(rc, columnOids); // write all active chunks to disk - } - else - chunkManager.cleanUp(columnOids); // close file w/o writing data to disk + // Close all target files + // -- chunkManager managed files + // timer.start("flushChunks"); + if (rc == NO_ERROR) + { + rc = chunkManager.flushChunks(rc, columnOids); // write all active chunks to disk + } + else + chunkManager.cleanUp(columnOids); // close file w/o writing data to disk - //@Bug 5466 need to purge PrimProc FD cache - if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + //@Bug 5466 need to purge PrimProc FD cache + if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); -// timer.stop("flushChunks"); - // -- other files - FileOpenMap::const_iterator itor; + // timer.stop("flushChunks"); + // -- other files + FileOpenMap::const_iterator itor; - for (itor = fileOpenList.begin(); itor != fileOpenList.end(); itor++) - { - delete itor->second; - } + for (itor = fileOpenList.begin(); itor != fileOpenList.end(); itor++) + { + delete itor->second; + } - if ( rc != 0) - throw std::runtime_error(errorMsg); + if (rc != 0) + throw std::runtime_error(errorMsg); - return rc; + return rc; } int BRMWrapper::rollBackVersion(const VER_t transID, int sessionId) { - std::vector lbidList; - std::vector lbidRangeList; - LBIDRange range; - int rc = 0; + std::vector lbidList; + std::vector lbidRangeList; + LBIDRange range; + int rc = 0; - //Check BRM status before processing. - rc = blockRsltnMgrPtr->isReadWrite(); + // Check BRM status before processing. + rc = blockRsltnMgrPtr->isReadWrite(); - if (rc != 0 ) - return ERR_BRM_READ_ONLY; + if (rc != 0) + return ERR_BRM_READ_ONLY; - rc = blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList); + rc = blockRsltnMgrPtr->getUncommittedLBIDs(transID, lbidList); - if ( rc != 0 ) - { - if (rc == BRM::ERR_READONLY) - return ERR_BRM_READ_ONLY; + if (rc != 0) + { + if (rc == BRM::ERR_READONLY) + return ERR_BRM_READ_ONLY; - return rc; - - } - - //std::cout << "rollBackVersion get uncommited lbid " << lbidList.size() << std::endl; - for (size_t i = 0; i < lbidList.size(); i++) - { - range.start = lbidList[i]; - range.size = 1; - lbidRangeList.push_back(range); - } - - rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); return rc; -} + } + // std::cout << "rollBackVersion get uncommited lbid " << lbidList.size() << std::endl; + for (size_t i = 0; i < lbidList.size(); i++) + { + range.start = lbidList[i]; + range.size = 1; + lbidRangeList.push_back(range); + } + + rc = blockRsltnMgrPtr->vbRollback(transID, lbidRangeList); + return rc; +} int BRMWrapper::writeVB(IDBDataFile* pFile, const VER_t transID, const OID oid, const uint64_t lbid, DbFileOp* pFileOp) { - int fbo; - LBIDRange lbidRange; - std::vector fboList; - std::vector rangeList; + int fbo; + LBIDRange lbidRange; + std::vector fboList; + std::vector rangeList; - lbidRange.start = lbid; - lbidRange.size = 1; - rangeList.push_back(lbidRange); + lbidRange.start = lbid; + lbidRange.size = 1; + rangeList.push_back(lbidRange); - uint16_t dbRoot; - uint32_t partition; - uint16_t segment; - RETURN_ON_ERROR(getFboOffset(lbid, dbRoot, partition, segment, fbo)); + uint16_t dbRoot; + uint32_t partition; + uint16_t segment; + RETURN_ON_ERROR(getFboOffset(lbid, dbRoot, partition, segment, fbo)); - fboList.push_back(fbo); - std::vector freeList; - int rc = writeVB(pFile, transID, oid, fboList, rangeList, pFileOp, freeList, dbRoot); - //writeVBEnd(transID,rangeList); - return rc; + fboList.push_back(fbo); + std::vector freeList; + int rc = writeVB(pFile, transID, oid, fboList, rangeList, pFileOp, freeList, dbRoot); + // writeVBEnd(transID,rangeList); + return rc; } // Eliminates blocks that have already been versioned by transaction transID -void BRMWrapper::pruneLBIDList(VER_t transID, vector* rangeList, - vector* fboList) const +void BRMWrapper::pruneLBIDList(VER_t transID, vector* rangeList, vector* fboList) const { - vector lbids; - vector vssData; - BRM::QueryContext verID(transID); - uint32_t i; - int rc; - vector newrangeList; - vector newfboList; + vector lbids; + vector vssData; + BRM::QueryContext verID(transID); + uint32_t i; + int rc; + vector newrangeList; + vector newfboList; - for (i = 0; i < rangeList->size(); i++) - lbids.push_back((*rangeList)[i].start); + for (i = 0; i < rangeList->size(); i++) + lbids.push_back((*rangeList)[i].start); - rc = blockRsltnMgrPtr->bulkVSSLookup(lbids, verID, transID, &vssData); + rc = blockRsltnMgrPtr->bulkVSSLookup(lbids, verID, transID, &vssData); - if (rc != 0) - return; // catch the error in a more appropriate place + if (rc != 0) + return; // catch the error in a more appropriate place - for (i = 0; i < vssData.size(); i++) + for (i = 0; i < vssData.size(); i++) + { + BRM::VSSData& vd = vssData[i]; + + // Check whether this transaction has already versioned this block + if (vd.returnCode != 0 || vd.verID != transID) { - BRM::VSSData& vd = vssData[i]; - - // Check whether this transaction has already versioned this block - if (vd.returnCode != 0 || vd.verID != transID) - { - newrangeList.push_back((*rangeList)[i]); - newfboList.push_back((*fboList)[i]); - } + newrangeList.push_back((*rangeList)[i]); + newfboList.push_back((*fboList)[i]); } + } - /* if (newrangeList.size() != rangeList->size()) { - cout << "Lbidlist is pruned, and the original list is: " << endl; - for (uint32_t i = 0; i < rangeList->size(); i++) - { - cout << "lbid : " << (*rangeList)[i].start << endl; - } - } */ - newrangeList.swap(*rangeList); - newfboList.swap(*fboList); + /* if (newrangeList.size() != rangeList->size()) { + cout << "Lbidlist is pruned, and the original list is: " << endl; + for (uint32_t i = 0; i < rangeList->size(); i++) + { + cout << "lbid : " << (*rangeList)[i].start << endl; + } + } */ + newrangeList.swap(*rangeList); + newfboList.swap(*fboList); } int BRMWrapper::writeVB(IDBDataFile* pSourceFile, const VER_t transID, const OID weOid, - std::vector& fboList, std::vector& rangeList, DbFileOp* pFileOp, std::vector& freeList, uint16_t dbRoot, bool skipBeginVBCopy) + std::vector& fboList, std::vector& rangeList, DbFileOp* pFileOp, + std::vector& freeList, uint16_t dbRoot, bool skipBeginVBCopy) { - if (idbdatafile::IDBPolicy::useHdfs()) - return 0; + if (idbdatafile::IDBPolicy::useHdfs()) + return 0; - int rc; - size_t i; - size_t processedBlocks; - size_t rangeListCount; - size_t k = 0; - //std::vector freeList; - IDBDataFile* pTargetFile; - int32_t vbOid; + int rc; + size_t i; + size_t processedBlocks; + size_t rangeListCount; + size_t k = 0; + // std::vector freeList; + IDBDataFile* pTargetFile; + int32_t vbOid; - if (isDebug(DEBUG_3)) + if (isDebug(DEBUG_3)) + { + cout << "\nIn writeVB" << endl; + cout << "\n\tTransId=" << transID << endl; + cout << "\t weOid : " << weOid << endl; + cout << "\trangeList size=" << rangeList.size(); + + for (i = 0; i < rangeList.size(); i++) { - cout << "\nIn writeVB" << endl; - cout << "\n\tTransId=" << transID << endl; - cout << "\t weOid : " << weOid << endl; - cout << "\trangeList size=" << rangeList.size(); - - for (i = 0; i < rangeList.size(); i++) - { - cout << "\t weLBID start : " << rangeList[i].start << endl; - cout << " weSize : " << rangeList[i].size << endl; - } - - cout << "\tfboList size=" << fboList.size() << endl; - - for (i = 0; i < fboList.size(); i++) - cout << "\t weFbo : " << fboList[i] << endl; + cout << "\t weLBID start : " << rangeList[i].start << endl; + cout << " weSize : " << rangeList[i].size << endl; } + cout << "\tfboList size=" << fboList.size() << endl; + + for (i = 0; i < fboList.size(); i++) + cout << "\t weFbo : " << fboList[i] << endl; + } + + /* cout << "\nIn writeVB" << endl; + cout << "\n\tTransId=" << transID << endl; + cout << "\t weOid : " << weOid << endl; + cout << "\trangeList size=" << rangeList.size(); + for (i = 0; i < rangeList.size(); i++) + { + cout << "\t weLBID start : " << rangeList[i].start << endl; + } + */ + if (!skipBeginVBCopy) + { + pruneLBIDList(transID, &rangeList, &fboList); + /* cout << "\nIn writeVB" << endl; cout << "\n\tTransId=" << transID << endl; cout << "\t weOid : " << weOid << endl; @@ -1614,201 +1573,172 @@ int BRMWrapper::writeVB(IDBDataFile* pSourceFile, const VER_t transID, const OID cout << "\t weLBID start : " << rangeList[i].start << endl; } */ - if (!skipBeginVBCopy) + if (rangeList.empty()) // all blocks have already been versioned + return NO_ERROR; + + // Find the dbroot for a lbid + // OID_t oid; + // uint16_t segmentNum; + // uint32_t partitionNum, fileBlockOffset; + // rc = blockRsltnMgrPtr->lookupLocal(rangeList[0].start, transID, false, oid, dbRoot, partitionNum, + // segmentNum, fileBlockOffset); if (rc != NO_ERROR) return rc; + + rc = blockRsltnMgrPtr->beginVBCopy(transID, dbRoot, rangeList, freeList); + + if (rc != NO_ERROR) { - pruneLBIDList(transID, &rangeList, &fboList); + switch (rc) + { + case ERR_DEADLOCK: return ERR_BRM_DEAD_LOCK; - /* cout << "\nIn writeVB" << endl; - cout << "\n\tTransId=" << transID << endl; - cout << "\t weOid : " << weOid << endl; - cout << "\trangeList size=" << rangeList.size(); - for (i = 0; i < rangeList.size(); i++) - { - cout << "\t weLBID start : " << rangeList[i].start << endl; - } - */ - if (rangeList.empty()) // all blocks have already been versioned - return NO_ERROR; + case ERR_VBBM_OVERFLOW: return ERR_BRM_VB_OVERFLOW; - //Find the dbroot for a lbid - //OID_t oid; - //uint16_t segmentNum; - //uint32_t partitionNum, fileBlockOffset; - //rc = blockRsltnMgrPtr->lookupLocal(rangeList[0].start, transID, false, oid, dbRoot, partitionNum, segmentNum, fileBlockOffset); - //if (rc != NO_ERROR) - // return rc; + case ERR_NETWORK: return ERR_BRM_NETWORK; - rc = blockRsltnMgrPtr->beginVBCopy(transID, dbRoot, rangeList, freeList); + case ERR_READONLY: return ERR_BRM_READONLY; - if (rc != NO_ERROR) - { - switch (rc) - { - case ERR_DEADLOCK: - return ERR_BRM_DEAD_LOCK; - - case ERR_VBBM_OVERFLOW: - return ERR_BRM_VB_OVERFLOW; - - case ERR_NETWORK: - return ERR_BRM_NETWORK; - - case ERR_READONLY: - return ERR_BRM_READONLY; - - default: - return ERR_BRM_BEGIN_COPY; - } - } + default: return ERR_BRM_BEGIN_COPY; + } } + } - if (isDebug(DEBUG_3)) + if (isDebug(DEBUG_3)) + { + cout << "\nAfter beginCopy and get a freeList=" << freeList.size() << endl; + cout << "\tfreeList size=" << freeList.size() << endl; + + for (i = 0; i < freeList.size(); i++) { - cout << "\nAfter beginCopy and get a freeList=" << freeList.size() << endl; - cout << "\tfreeList size=" << freeList.size() << endl; - - for (i = 0; i < freeList.size(); i++) - { - cout << "\t VBOid : " << freeList[i].vbOID ; - cout << " VBFBO : " << freeList[i].vbFBO ; - cout << " Size : " << freeList[i].size << endl; - } + cout << "\t VBOid : " << freeList[i].vbOID; + cout << " VBFBO : " << freeList[i].vbFBO; + cout << " Size : " << freeList[i].size << endl; } + } - /* for (i = 0; i < freeList.size(); i++) - { - cout << "\t VBOid : " << freeList[i].vbOID ; - cout << " VBFBO : " << freeList[i].vbFBO ; - cout << " Size : " << freeList[i].size << endl; - } - */ - //@Bug 2371 The assumption of all entries in the freelist belong to the same version buffer file is wrong - // Open the first version buffer file - File fileInfo; -// size_t rootCnt = Config::DBRootCount(); - fileInfo.oid = freeList[0].vbOID; - fileInfo.fPartition = 0; - fileInfo.fSegment = 0; -// fileInfo.fDbRoot = (freeList[0].vbOID % rootCnt) + 1; - fileInfo.fDbRoot = dbRoot; - boost::mutex::scoped_lock lk(vbFileLock); - pTargetFile = openFile(fileInfo, "r+b", true); + /* for (i = 0; i < freeList.size(); i++) + { + cout << "\t VBOid : " << freeList[i].vbOID ; + cout << " VBFBO : " << freeList[i].vbFBO ; + cout << " Size : " << freeList[i].size << endl; + } + */ + //@Bug 2371 The assumption of all entries in the freelist belong to the same version buffer file is wrong + // Open the first version buffer file + File fileInfo; + // size_t rootCnt = Config::DBRootCount(); + fileInfo.oid = freeList[0].vbOID; + fileInfo.fPartition = 0; + fileInfo.fSegment = 0; + // fileInfo.fDbRoot = (freeList[0].vbOID % rootCnt) + 1; + fileInfo.fDbRoot = dbRoot; + boost::mutex::scoped_lock lk(vbFileLock); + pTargetFile = openFile(fileInfo, "r+b", true); + + if (pTargetFile == NULL) + { + pTargetFile = openFile(fileInfo, "w+b"); if (pTargetFile == NULL) { - pTargetFile = openFile(fileInfo, "w+b"); - - if (pTargetFile == NULL) - { - rc = ERR_FILE_NOT_EXIST; - goto cleanup; - } - else - { - delete pTargetFile; - pTargetFile = openFile(fileInfo, "r+b", true); - - if (pTargetFile == NULL) - { - rc = ERR_FILE_NOT_EXIST; - goto cleanup; - } - } + rc = ERR_FILE_NOT_EXIST; + goto cleanup; } - - k = 0; - vbOid = freeList[0].vbOID; - rangeListCount = 0; - - //cout << "writeVBEntry is putting the follwing lbids into VSS and freelist size is " << freeList.size() << endl; - for (i = 0; i < freeList.size(); i++) + else { - rangeListCount += k; - processedBlocks = rangeListCount; // store the number of blocks processed till now for this file + delete pTargetFile; + pTargetFile = openFile(fileInfo, "r+b", true); - if (vbOid == freeList[i].vbOID) - { - // This call to copyVBBlock will consume whole of the freeList[i] - k = 0; - rc = copyVBBlock(pSourceFile, weOid, pTargetFile, fileInfo.oid, - fboList, freeList[i], k, pFileOp, rangeListCount); - //cout << "processedBlocks:k = " << processedBlocks <<":"< lbids(k); - std::vector vbFBOs(k); - size_t idx = 0; - - for (; processedBlocks < (k + rangeListCount); processedBlocks++, idx++) - { - lbids[idx] = rangeList[processedBlocks].start; - vbFBOs[idx] = freeList[i].vbFBO + (processedBlocks - rangeListCount); - } - - rc = blockRsltnMgrPtr->bulkWriteVBEntry(transID, lbids, freeList[i].vbOID, - vbFBOs); - - if (rc != NO_ERROR) - { - switch (rc) - { - case ERR_DEADLOCK: - rc = ERR_BRM_DEAD_LOCK; - break; - - case ERR_VBBM_OVERFLOW: - rc = ERR_BRM_VB_OVERFLOW; - break; - - case ERR_NETWORK: - rc = ERR_BRM_NETWORK; - break; - - case ERR_READONLY: - rc = ERR_BRM_READONLY; - break; - - default: - rc = ERR_BRM_WR_VB_ENTRY; - } - - goto cleanup; - } - } + if (pTargetFile == NULL) + { + rc = ERR_FILE_NOT_EXIST; + goto cleanup; + } } + } - if (pTargetFile) + k = 0; + vbOid = freeList[0].vbOID; + rangeListCount = 0; + + // cout << "writeVBEntry is putting the follwing lbids into VSS and freelist size is " << freeList.size() << + // endl; + for (i = 0; i < freeList.size(); i++) + { + rangeListCount += k; + processedBlocks = rangeListCount; // store the number of blocks processed till now for this file + + if (vbOid == freeList[i].vbOID) { - pTargetFile->flush(); - } + // This call to copyVBBlock will consume whole of the freeList[i] + k = 0; + rc = copyVBBlock(pSourceFile, weOid, pTargetFile, fileInfo.oid, fboList, freeList[i], k, pFileOp, + rangeListCount); + // cout << "processedBlocks:k = " << processedBlocks <<":"< lbids(k); + std::vector vbFBOs(k); + size_t idx = 0; + + for (; processedBlocks < (k + rangeListCount); processedBlocks++, idx++) + { + lbids[idx] = rangeList[processedBlocks].start; + vbFBOs[idx] = freeList[i].vbFBO + (processedBlocks - rangeListCount); + } + + rc = blockRsltnMgrPtr->bulkWriteVBEntry(transID, lbids, freeList[i].vbOID, vbFBOs); + + if (rc != NO_ERROR) + { + switch (rc) + { + case ERR_DEADLOCK: rc = ERR_BRM_DEAD_LOCK; break; + + case ERR_VBBM_OVERFLOW: rc = ERR_BRM_VB_OVERFLOW; break; + + case ERR_NETWORK: rc = ERR_BRM_NETWORK; break; + + case ERR_READONLY: rc = ERR_BRM_READONLY; break; + + default: rc = ERR_BRM_WR_VB_ENTRY; + } + + goto cleanup; + } + } + } + + if (pTargetFile) + { + pTargetFile->flush(); + } + + return rc; cleanup: - if (pTargetFile) - { - pTargetFile->flush(); - } + if (pTargetFile) + { + pTargetFile->flush(); + } - writeVBEnd(transID, rangeList); - return rc; + writeVBEnd(transID, rangeList); + return rc; } void BRMWrapper::writeVBEnd(const VER_t transID, std::vector& rangeList) { - if (idbdatafile::IDBPolicy::useHdfs()) - return; + if (idbdatafile::IDBPolicy::useHdfs()) + return; - blockRsltnMgrPtr->endVBCopy(transID, rangeList); + blockRsltnMgrPtr->endVBCopy(transID, rangeList); } int BRMWrapper::getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMin) { - int rc = blockRsltnMgrPtr->getExtentCPMaxMin(lbid, cpMaxMin); - return getRC(rc, ERR_BRM_GET_EXTENT_CP); + int rc = blockRsltnMgrPtr->getExtentCPMaxMin(lbid, cpMaxMin); + return getRC(rc, ERR_BRM_GET_EXTENT_CP); } -} //end of namespace +} // namespace WriteEngine // vim:ts=4 sw=4: - diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index 2b43397b4..c69aa7380 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_brm.h 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_brm.h 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_BRM_H_ @@ -31,7 +31,7 @@ #include "brm.h" #include "we_obj.h" -#include +#include #include "brmtypes.h" #include "mcs_datatype.h" #include "dataconvert.h" @@ -53,511 +53,415 @@ class DbFileOp; /** @brief Extended CPInfo - with type handler for all type-related information */ struct ExtCPInfo { - execplan::CalpontSystemCatalog::ColDataType fColType; - int fColWidth; - BRM::CPInfo fCPInfo; - ExtCPInfo(execplan::CalpontSystemCatalog::ColDataType colType, int colWidth) - : fColType(colType), fColWidth(colWidth) - { - fCPInfo.isBinaryColumn = (unsigned int)colWidth > datatypes::MAXLEGACYWIDTH; - } - void toInvalid() - { - auto mm = datatypes::MinMaxInfo::invalidRange(fColType); - fCPInfo.max = mm.max; - fCPInfo.min = mm.min; - fCPInfo.bigMax = mm.int128Max; - fCPInfo.bigMin = mm.int128Min; - } + execplan::CalpontSystemCatalog::ColDataType fColType; + int fColWidth; + BRM::CPInfo fCPInfo; + ExtCPInfo(execplan::CalpontSystemCatalog::ColDataType colType, int colWidth) + : fColType(colType), fColWidth(colWidth) + { + fCPInfo.isBinaryColumn = (unsigned int)colWidth > datatypes::MAXLEGACYWIDTH; + } + void toInvalid() + { + auto mm = datatypes::MinMaxInfo::invalidRange(fColType); + fCPInfo.max = mm.max; + fCPInfo.min = mm.min; + fCPInfo.bigMax = mm.int128Max; + fCPInfo.bigMin = mm.int128Min; + } - bool isInvalid() - { - datatypes::MinMaxInfo mm; - mm.max = fCPInfo.max; - mm.min = fCPInfo.min; - mm.int128Max = fCPInfo.bigMax; - mm.int128Min = fCPInfo.bigMin; - return datatypes::MinMaxInfo::isRangeInvalid(mm, fColType, fColWidth); - } - void fromToChars() - { - fCPInfo.max = static_cast(uint64ToStr(fCPInfo.max)); - fCPInfo.min = static_cast(uint64ToStr(fCPInfo.min)); - } - bool isBinaryColumn() - { - return fCPInfo.isBinaryColumn; - } + bool isInvalid() + { + datatypes::MinMaxInfo mm; + mm.max = fCPInfo.max; + mm.min = fCPInfo.min; + mm.int128Max = fCPInfo.bigMax; + mm.int128Min = fCPInfo.bigMin; + return datatypes::MinMaxInfo::isRangeInvalid(mm, fColType, fColWidth); + } + void fromToChars() + { + fCPInfo.max = static_cast(uint64ToStr(fCPInfo.max)); + fCPInfo.min = static_cast(uint64ToStr(fCPInfo.min)); + } + bool isBinaryColumn() + { + return fCPInfo.isBinaryColumn; + } }; typedef std::vector ExtCPInfoList; /** Class BRMWrapper */ class BRMWrapper : public WEObj { -public: - /** - * @brief Initialize an Auto Increment sequence for the specified OID - * @param colOID Column OID of interest - * @param startNextValue Starting next value for the AI sequence - * @param colWidth Width of the relevant column (in bytes) - * @param errMsg Applicable error message. - */ - EXPORT int startAutoIncrementSequence( OID colOID, - uint64_t startNextValue, - uint32_t colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType, - std::string& errMsg); + public: + /** + * @brief Initialize an Auto Increment sequence for the specified OID + * @param colOID Column OID of interest + * @param startNextValue Starting next value for the AI sequence + * @param colWidth Width of the relevant column (in bytes) + * @param errMsg Applicable error message. + */ + EXPORT int startAutoIncrementSequence(OID colOID, uint64_t startNextValue, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType, + std::string& errMsg); - /** - * @brief Reserve a range of Auto Increment numbers for the specified OID - * @param colOID Column OID of interest - * @param count Requested range of auto increment numbers - * @param firstNum (out) First number of range that is reserved - * @param errMsg Applicable error message. - */ - EXPORT int getAutoIncrementRange( OID colOID, - uint64_t count, - uint64_t& firstNum, - std::string& errMsg); + /** + * @brief Reserve a range of Auto Increment numbers for the specified OID + * @param colOID Column OID of interest + * @param count Requested range of auto increment numbers + * @param firstNum (out) First number of range that is reserved + * @param errMsg Applicable error message. + */ + EXPORT int getAutoIncrementRange(OID colOID, uint64_t count, uint64_t& firstNum, std::string& errMsg); - /** - * @brief Inform BRM to add an extent to each of the requested OIDs at - * the specified DBRoot (and partition number if the DBRoot is empty). - * @param cols (in) List of column OIDs and column widths - * @param dbRoot (in) DBRoot for requested extents - * @param partition (in/out) Physical partition number in file path. - * If allocating OID's first extent for this DBRoot, then - * partition is input, else it is only for output. - * @param segmentNum (out) Segment number for new extents - * @param extents (out) List of lbids, numBlks, and fbo for new extents - */ - EXPORT int allocateStripeColExtents( - const std::vector& cols, - uint16_t dbRoot, - uint32_t& partition, - uint16_t& segmentNum, - std::vector& extents); + /** + * @brief Inform BRM to add an extent to each of the requested OIDs at + * the specified DBRoot (and partition number if the DBRoot is empty). + * @param cols (in) List of column OIDs and column widths + * @param dbRoot (in) DBRoot for requested extents + * @param partition (in/out) Physical partition number in file path. + * If allocating OID's first extent for this DBRoot, then + * partition is input, else it is only for output. + * @param segmentNum (out) Segment number for new extents + * @param extents (out) List of lbids, numBlks, and fbo for new extents + */ + EXPORT int allocateStripeColExtents(const std::vector& cols, + uint16_t dbRoot, uint32_t& partition, uint16_t& segmentNum, + std::vector& extents); - /** - * @brief Inform BRM to add extent to the exact segment file specified by - * OID, DBRoot, partition, and segment. - */ - EXPORT int allocateColExtentExactFile( const OID oid, - const uint32_t colWidth, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - BRM::LBID_t& startLbid, - int& allocSize, - uint32_t& startBlock); + /** + * @brief Inform BRM to add extent to the exact segment file specified by + * OID, DBRoot, partition, and segment. + */ + EXPORT int allocateColExtentExactFile(const OID oid, const uint32_t colWidth, uint16_t dbRoot, + uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + BRM::LBID_t& startLbid, int& allocSize, uint32_t& startBlock); - /** - * @brief Inform BRM to add a dictionary store extent to the specified OID - */ - EXPORT int allocateDictStoreExtent( const OID oid, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - BRM::LBID_t& startLbid, - int& allocSize ); + /** + * @brief Inform BRM to add a dictionary store extent to the specified OID + */ + EXPORT int allocateDictStoreExtent(const OID oid, uint16_t dbRoot, uint32_t partition, uint16_t segment, + BRM::LBID_t& startLbid, int& allocSize); - /** - * @brief Inform BRM to delete certain oid - */ - EXPORT int deleteOid( const OID oid ); + /** + * @brief Inform BRM to delete certain oid + */ + EXPORT int deleteOid(const OID oid); - /** - * @brief Inform BRM to delete list of oids - */ - EXPORT int deleteOIDsFromExtentMap (const std::vector& oids); + /** + * @brief Inform BRM to delete list of oids + */ + EXPORT int deleteOIDsFromExtentMap(const std::vector& oids); - /** - * @brief Get BRM information based on a specfic OID, DBRoot, partition, - * and segment - */ - EXPORT int getBrmInfo( const OID oid, - const uint32_t partition, - const uint16_t segment, - const int fbo, - BRM::LBID_t& lbid ); + /** + * @brief Get BRM information based on a specfic OID, DBRoot, partition, + * and segment + */ + EXPORT int getBrmInfo(const OID oid, const uint32_t partition, const uint16_t segment, const int fbo, + BRM::LBID_t& lbid); - /** - * @brief Get starting LBID from BRM for a specfic OID, DBRoot, partition, - * segment, and block offset. - */ - EXPORT int getStartLbid( const OID oid, - const uint32_t partition, - const uint16_t segment, - const int fbo, - BRM::LBID_t& startLbid ); + /** + * @brief Get starting LBID from BRM for a specfic OID, DBRoot, partition, + * segment, and block offset. + */ + EXPORT int getStartLbid(const OID oid, const uint32_t partition, const uint16_t segment, const int fbo, + BRM::LBID_t& startLbid); - /** - * @brief Get the real physical offset based on the LBID - */ - EXPORT int getFboOffset( const uint64_t lbid, - uint16_t& dbRoot, - uint32_t& partition, - uint16_t& segment, - int& fbo ); - EXPORT int getFboOffset( const uint64_t lbid, int& oid, - uint16_t& dbRoot, - uint32_t& partition, - uint16_t& segment, - int& fbo ); + /** + * @brief Get the real physical offset based on the LBID + */ + EXPORT int getFboOffset(const uint64_t lbid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, + int& fbo); + EXPORT int getFboOffset(const uint64_t lbid, int& oid, uint16_t& dbRoot, uint32_t& partition, + uint16_t& segment, int& fbo); - /** - * @brief Get last "local" HWM, partition, and segment for an OID and DBRoot - */ - EXPORT int getLastHWM_DBroot( OID oid, - uint16_t dbRoot, - uint32_t& partition, - uint16_t& segment, - HWM& hwm, - int& status, - bool& bFound); + /** + * @brief Get last "local" HWM, partition, and segment for an OID and DBRoot + */ + EXPORT int getLastHWM_DBroot(OID oid, uint16_t dbRoot, uint32_t& partition, uint16_t& segment, HWM& hwm, + int& status, bool& bFound); - /** - * @brief Get HWM for a specific OID, partition, and segment - */ - int getLocalHWM( OID oid, - uint32_t partition, - uint16_t segment, - HWM& hwm, - int& status); + /** + * @brief Get HWM for a specific OID, partition, and segment + */ + int getLocalHWM(OID oid, uint32_t partition, uint16_t segment, HWM& hwm, int& status); - /** - * @brief Get HWM info for a specific OID and PM - */ - EXPORT int getDbRootHWMInfo( const OID oid, - BRM::EmDbRootHWMInfo_v& emDbRootHwmInfos); + /** + * @brief Get HWM info for a specific OID and PM + */ + EXPORT int getDbRootHWMInfo(const OID oid, BRM::EmDbRootHWMInfo_v& emDbRootHwmInfos); - /** - * @brief Get status or state of the extents in the specified segment file. - * bFound flag indicates whether an extent was found or not. - */ - int getExtentState( OID oid, - uint32_t partition, - uint16_t segment, - bool& bFound, - int& status); + /** + * @brief Get status or state of the extents in the specified segment file. + * bFound flag indicates whether an extent was found or not. + */ + int getExtentState(OID oid, uint32_t partition, uint16_t segment, bool& bFound, int& status); - /** - * @brief Get extentRows - */ - unsigned getExtentRows(); + /** + * @brief Get extentRows + */ + unsigned getExtentRows(); - /** - * @brief Return the extents info for specified OID - */ - int getExtents( int oid, - std::vector& entries, - bool sorted, bool notFoundErr, - bool incOutOfService ); + /** + * @brief Return the extents info for specified OID + */ + int getExtents(int oid, std::vector& entries, bool sorted, bool notFoundErr, + bool incOutOfService); - /** - * @brief Return the extents info for specified OID and dbroot - */ - int getExtents_dbroot( int oid, - std::vector& entries, - const uint16_t dbroot); + /** + * @brief Return the extents info for specified OID and dbroot + */ + int getExtents_dbroot(int oid, std::vector& entries, const uint16_t dbroot); - /** - * @brief Return the read/write status of DBRM (helps detect if DBRM is up) - */ - EXPORT int isReadWrite(); + /** + * @brief Return the read/write status of DBRM (helps detect if DBRM is up) + */ + EXPORT int isReadWrite(); - /** - * @brief Return the state of the system state shutdown pending - * flags - */ - EXPORT int isShutdownPending(bool& bRollback, bool& bForce); + /** + * @brief Return the state of the system state shutdown pending + * flags + */ + EXPORT int isShutdownPending(bool& bRollback, bool& bForce); - /** - * @brief Return the state of the system state suspend pending - * flags - */ - EXPORT int isSuspendPending(); + /** + * @brief Return the state of the system state suspend pending + * flags + */ + EXPORT int isSuspendPending(); - /** - * @brief Is InfiniDB system ready (completed startup) - */ - bool isSystemReady(); + /** + * @brief Is InfiniDB system ready (completed startup) + */ + bool isSystemReady(); - /** - * @brief Lookup LBID ranges for column specified OID - */ - int lookupLbidRanges(OID oid, BRM::LBIDRange_v& lbidRanges); + /** + * @brief Lookup LBID ranges for column specified OID + */ + int lookupLbidRanges(OID oid, BRM::LBIDRange_v& lbidRanges); - /** - * @brief Mark extent invalid for causal partioning - */ - int markExtentInvalid(const uint64_t lbid, - const execplan::CalpontSystemCatalog::ColDataType colDataType); + /** + * @brief Mark extent invalid for causal partioning + */ + int markExtentInvalid(const uint64_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); - /** - * @brief Mark multiple extents invalid for causal partioning - */ - int markExtentsInvalid(std::vector& lbids, - const std::vector& - colDataTypes); + /** + * @brief Mark multiple extents invalid for causal partioning + */ + int markExtentsInvalid(std::vector& lbids, + const std::vector& colDataTypes); - /** - * @brief set extents CP min/max info into extent map - */ - int setExtentsMaxMin(const ExtCPInfoList& cpinfoList); + /** + * @brief set extents CP min/max info into extent map + */ + int setExtentsMaxMin(const ExtCPInfoList& cpinfoList); - /** - * @brief Perform bulk rollback of any column extents that logically follow - * the specified HWM for the given column OID and DBRoot. The HWM for the - * last local extent is reset to the specified hwm as well. Any extents in - * subsequent partitions are deleted. If bDeleteAll is true, then all - * extents for the specified oid and dbroot are deleted. - */ - int rollbackColumnExtents_DBroot( const OID oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - BRM::HWM_t hwm ); + /** + * @brief Perform bulk rollback of any column extents that logically follow + * the specified HWM for the given column OID and DBRoot. The HWM for the + * last local extent is reset to the specified hwm as well. Any extents in + * subsequent partitions are deleted. If bDeleteAll is true, then all + * extents for the specified oid and dbroot are deleted. + */ + int rollbackColumnExtents_DBroot(const OID oid, bool bDeleteAll, uint16_t dbRoot, uint32_t partition, + uint16_t segment, BRM::HWM_t hwm); - /** - * @brief Perform bulk rollback of the extents that follow the specified - * dictionary extents for the given column OID and DBRoot. The HWM for - * the last retained extents, are reset as well. Any trailing segment - * files for the same parition, that are not specified in the hwm list, - * are deleted. Any extents in subsequent partitions are deleted. If - * segNums and hwms vector are empty, then all extents for the specified - * oid and dbroot are deleted. - */ - int rollbackDictStoreExtents_DBroot( OID oid, - uint16_t dbRoot, - uint32_t partition, - const std::vector& segNums, - const std::vector& hwms ); + /** + * @brief Perform bulk rollback of the extents that follow the specified + * dictionary extents for the given column OID and DBRoot. The HWM for + * the last retained extents, are reset as well. Any trailing segment + * files for the same parition, that are not specified in the hwm list, + * are deleted. Any extents in subsequent partitions are deleted. If + * segNums and hwms vector are empty, then all extents for the specified + * oid and dbroot are deleted. + */ + int rollbackDictStoreExtents_DBroot(OID oid, uint16_t dbRoot, uint32_t partition, + const std::vector& segNums, + const std::vector& hwms); - /** - * @brief Perform delete column extents - */ - int deleteEmptyColExtents(const std::vector& extentsInfo); + /** + * @brief Perform delete column extents + */ + int deleteEmptyColExtents(const std::vector& extentsInfo); - /** - * @brief Perform delete dictionary extents - */ - int deleteEmptyDictStoreExtents( - const std::vector& extentsInfo ); + /** + * @brief Perform delete dictionary extents + */ + int deleteEmptyDictStoreExtents(const std::vector& extentsInfo); - /** - * @brief Set HWM for a specific OID, partition, and segment - */ - int setLocalHWM( OID oid, - uint32_t partition, - uint16_t segment, - const HWM hwm ); + /** + * @brief Set HWM for a specific OID, partition, and segment + */ + int setLocalHWM(OID oid, uint32_t partition, uint16_t segment, const HWM hwm); - //Set hwm for all columns in a table - int bulkSetHWM( const std::vector& vec, - BRM::VER_t transID); + // Set hwm for all columns in a table + int bulkSetHWM(const std::vector& vec, BRM::VER_t transID); - /** - * @brief Atomically apply a batch of HWM and CP updates within the scope - * of a single BRM lock. CP info is merged with current min/max range. - * @param hwmArgs Vector of HWM updates - * @param mergeCPDataArgs Vector of Casual Partition updates - */ - int bulkSetHWMAndCP( const std::vector& hwmArgs, - const std::vector& mergeCPDataArgs); + /** + * @brief Atomically apply a batch of HWM and CP updates within the scope + * of a single BRM lock. CP info is merged with current min/max range. + * @param hwmArgs Vector of HWM updates + * @param mergeCPDataArgs Vector of Casual Partition updates + */ + int bulkSetHWMAndCP(const std::vector& hwmArgs, + const std::vector& mergeCPDataArgs); - /** - * @brief Acquire a table lock for the specified table OID. - * If nonzero lockID is returned, then the table is already locked. - * @param tableOID Table to be locked. - * @param ownerName Requested (in) and current (out) owner for the lock. - * @param processID Requested (in) and current (out) pid for the lock. - * @param sessionID Requested (in) and current (out) session ID for the lock - * @param transID Requested (in) and current (out) transacton of the lock - * @param lockID Assigned or current lock for the specified table. - * @param errMsg Applicable error message. - */ - EXPORT int getTableLock ( OID tableOid, - std::string& ownerName, - uint32_t& processID, - int32_t& sessionID, - int32_t& transID, - uint64_t& lockID, - std::string& errMsg); + /** + * @brief Acquire a table lock for the specified table OID. + * If nonzero lockID is returned, then the table is already locked. + * @param tableOID Table to be locked. + * @param ownerName Requested (in) and current (out) owner for the lock. + * @param processID Requested (in) and current (out) pid for the lock. + * @param sessionID Requested (in) and current (out) session ID for the lock + * @param transID Requested (in) and current (out) transacton of the lock + * @param lockID Assigned or current lock for the specified table. + * @param errMsg Applicable error message. + */ + EXPORT int getTableLock(OID tableOid, std::string& ownerName, uint32_t& processID, int32_t& sessionID, + int32_t& transID, uint64_t& lockID, std::string& errMsg); - /** - * @brief Change the state of the specified table lock ID. - * @param lockID Lock for which the status is to be changed. - * @param lockState New state to be assigned to the specified lock. - * @param bChanged Indicates whether lock state was changed. - * @param errMsg Applicable error message. - */ - EXPORT int changeTableLockState ( uint64_t lockID, - BRM::LockState lockState, - bool& bChanged, - std::string& errMsg); + /** + * @brief Change the state of the specified table lock ID. + * @param lockID Lock for which the status is to be changed. + * @param lockState New state to be assigned to the specified lock. + * @param bChanged Indicates whether lock state was changed. + * @param errMsg Applicable error message. + */ + EXPORT int changeTableLockState(uint64_t lockID, BRM::LockState lockState, bool& bChanged, + std::string& errMsg); - /** - * @brief Release the specified table lock ID. - * @param lockID Lock to be released. - * @param bReleased Indicates whether lock was released. - * @param errMsg Applicable error message. - */ - EXPORT int releaseTableLock( uint64_t lockID, - bool& bReleased, - std::string& errMsg); + /** + * @brief Release the specified table lock ID. + * @param lockID Lock to be released. + * @param bReleased Indicates whether lock was released. + * @param errMsg Applicable error message. + */ + EXPORT int releaseTableLock(uint64_t lockID, bool& bReleased, std::string& errMsg); - /** - * @brief Get current table lock information for the specified lock ID. - * @param lockID Lock to be retrieved. - * @param lockInfo Current lock information for the specified lock. - * @param blockExists Indicates whether lock was found. - * @param errMsg Applicable error message. - */ - EXPORT int getTableLockInfo( uint64_t lockID, - BRM::TableLockInfo* lockInfo, - bool& bLockExists, - std::string& errMsg); + /** + * @brief Get current table lock information for the specified lock ID. + * @param lockID Lock to be retrieved. + * @param lockInfo Current lock information for the specified lock. + * @param blockExists Indicates whether lock was found. + * @param errMsg Applicable error message. + */ + EXPORT int getTableLockInfo(uint64_t lockID, BRM::TableLockInfo* lockInfo, bool& bLockExists, + std::string& errMsg); - /** - * @brief Tell BRM to make a snapshot of it's current state to disk. - */ - int takeSnapshot(); + /** + * @brief Tell BRM to make a snapshot of it's current state to disk. + */ + int takeSnapshot(); - /** - * @brief Save brm structures to file - */ - EXPORT int saveState(); + /** + * @brief Save brm structures to file + */ + EXPORT int saveState(); - //-------------------------------------------------------------------------- - // Non-inline Versioning Functions Start Here - //-------------------------------------------------------------------------- - /** - * @brief Commit the transaction - */ - EXPORT int commit( const BRM::VER_t transID ); + //-------------------------------------------------------------------------- + // Non-inline Versioning Functions Start Here + //-------------------------------------------------------------------------- + /** + * @brief Commit the transaction + */ + EXPORT int commit(const BRM::VER_t transID); - /** - * @brief Copy blocks between write engine and version buffer - */ - EXPORT int copyVBBlock( IDBDataFile* pSourceFile, - IDBDataFile* pTargetFile, - const uint64_t sourceFbo, - const uint64_t targetFbo, - DbFileOp* fileOp, - const Column& column ); - EXPORT int copyVBBlock( IDBDataFile* pSourceFile, - const OID sourceOid, - IDBDataFile* pTargetFile, - const OID targetOid, - const std::vector& fboList, - const BRM::VBRange& freeList, - size_t& nBlocksProcessed, - DbFileOp* pFileOp, - const size_t fboCurrentOffset = 0 ); + /** + * @brief Copy blocks between write engine and version buffer + */ + EXPORT int copyVBBlock(IDBDataFile* pSourceFile, IDBDataFile* pTargetFile, const uint64_t sourceFbo, + const uint64_t targetFbo, DbFileOp* fileOp, const Column& column); + EXPORT int copyVBBlock(IDBDataFile* pSourceFile, const OID sourceOid, IDBDataFile* pTargetFile, + const OID targetOid, const std::vector& fboList, + const BRM::VBRange& freeList, size_t& nBlocksProcessed, DbFileOp* pFileOp, + const size_t fboCurrentOffset = 0); - /** - * @brief Rollback the specified transaction - */ - EXPORT int rollBack( const BRM::VER_t transID, int sessionId ); + /** + * @brief Rollback the specified transaction + */ + EXPORT int rollBack(const BRM::VER_t transID, int sessionId); - /** - * @brief Rollback the specified transaction - */ - EXPORT int rollBackVersion( const BRM::VER_t transID, int sessionId ); + /** + * @brief Rollback the specified transaction + */ + EXPORT int rollBackVersion(const BRM::VER_t transID, int sessionId); - /** - * @brief Rollback the specified transaction - */ - EXPORT int rollBackBlocks( const BRM::VER_t transID, int sessionId ); + /** + * @brief Rollback the specified transaction + */ + EXPORT int rollBackBlocks(const BRM::VER_t transID, int sessionId); - /** - * @brief Write specified LBID to version buffer - */ - EXPORT int writeVB( IDBDataFile* pFile, - const BRM::VER_t transID, - const OID oid, - const uint64_t lbid, - DbFileOp* pFileOp ); - int writeVB( IDBDataFile* pFile, - const BRM::VER_t transID, - const OID weOid, - std::vector& fboList, - std::vector& rangeList, - DbFileOp* pFileOp, - std::vector& freeList, - uint16_t dbRoot, - bool skipBeginVBCopy = false); - void writeVBEnd(const BRM::VER_t transID, - std::vector& rangeList); + /** + * @brief Write specified LBID to version buffer + */ + EXPORT int writeVB(IDBDataFile* pFile, const BRM::VER_t transID, const OID oid, const uint64_t lbid, + DbFileOp* pFileOp); + int writeVB(IDBDataFile* pFile, const BRM::VER_t transID, const OID weOid, std::vector& fboList, + std::vector& rangeList, DbFileOp* pFileOp, std::vector& freeList, + uint16_t dbRoot, bool skipBeginVBCopy = false); + void writeVBEnd(const BRM::VER_t transID, std::vector& rangeList); - BRM::DBRM* getDbrmObject(); - void pruneLBIDList(BRM::VER_t transID, - std::vector* rangeList, - std::vector* fboList) const; + BRM::DBRM* getDbrmObject(); + void pruneLBIDList(BRM::VER_t transID, std::vector* rangeList, + std::vector* fboList) const; - //-------------------------------------------------------------------------- - // Non-inline Versioning Functions End Here - //-------------------------------------------------------------------------- + //-------------------------------------------------------------------------- + // Non-inline Versioning Functions End Here + //-------------------------------------------------------------------------- - /** - * @brief static functions - */ - EXPORT static BRMWrapper* getInstance(); - EXPORT static int getBrmRc(bool reset = true); - static bool getUseVb() - { - return m_useVb; - } - static void setUseVb( const bool val ) - { - m_useVb = val; - } + /** + * @brief static functions + */ + EXPORT static BRMWrapper* getInstance(); + EXPORT static int getBrmRc(bool reset = true); + static bool getUseVb() + { + return m_useVb; + } + static void setUseVb(const bool val) + { + m_useVb = val; + } - int getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMin); + int getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMin); -private: - //-------------------------------------------------------------------------- - // Private methods - //-------------------------------------------------------------------------- - BRMWrapper(); - ~BRMWrapper(); + private: + //-------------------------------------------------------------------------- + // Private methods + //-------------------------------------------------------------------------- + BRMWrapper(); + ~BRMWrapper(); - // disable copy constructor and assignment operator - BRMWrapper(const BRMWrapper&); - BRMWrapper& operator= ( const BRMWrapper& wrapper ); + // disable copy constructor and assignment operator + BRMWrapper(const BRMWrapper&); + BRMWrapper& operator=(const BRMWrapper& wrapper); - // Convert BRM return code to WE return code - int getRC( int brmRc, int errRc ); + // Convert BRM return code to WE return code + int getRC(int brmRc, int errRc); - EXPORT void saveBrmRc( int brmRc ); + EXPORT void saveBrmRc(int brmRc); - IDBDataFile* openFile( const File& fileInfo, - const char* mode, - const bool bCache = false ); + IDBDataFile* openFile(const File& fileInfo, const char* mode, const bool bCache = false); + //-------------------------------------------------------------------------- + // Private data members + //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - // Private data members - //-------------------------------------------------------------------------- - - static BRMWrapper* volatile m_instance; - static boost::thread_specific_ptr m_ThreadDataPtr; - static boost::mutex m_instanceCreateMutex; + static BRMWrapper* volatile m_instance; + static boost::thread_specific_ptr m_ThreadDataPtr; + static boost::mutex m_instanceCreateMutex; #if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT) - __declspec(dllimport) + __declspec(dllimport) #endif - EXPORT static bool m_useVb; + EXPORT static bool m_useVb; - static OID m_curVBOid; - static IDBDataFile* m_curVBFile; + static OID m_curVBOid; + static IDBDataFile* m_curVBFile; - BRM::DBRM* blockRsltnMgrPtr; + BRM::DBRM* blockRsltnMgrPtr; }; //------------------------------------------------------------------------------ @@ -565,207 +469,170 @@ private: //------------------------------------------------------------------------------ inline BRMWrapper::BRMWrapper() { - blockRsltnMgrPtr = new BRM::DBRM(); + blockRsltnMgrPtr = new BRM::DBRM(); } inline BRMWrapper::~BRMWrapper() { - if (blockRsltnMgrPtr) - delete blockRsltnMgrPtr; + if (blockRsltnMgrPtr) + delete blockRsltnMgrPtr; - blockRsltnMgrPtr = 0; + blockRsltnMgrPtr = 0; } -inline BRM::DBRM* BRMWrapper::getDbrmObject() +inline BRM::DBRM* BRMWrapper::getDbrmObject() { - return blockRsltnMgrPtr; + return blockRsltnMgrPtr; } -inline int BRMWrapper::getRC( int brmRc, int errRc ) +inline int BRMWrapper::getRC(int brmRc, int errRc) { - if (brmRc == BRM::ERR_OK) - return NO_ERROR; + if (brmRc == BRM::ERR_OK) + return NO_ERROR; - saveBrmRc( brmRc ); - return errRc; + saveBrmRc(brmRc); + return errRc; } -inline int BRMWrapper::getLastHWM_DBroot( OID oid, - uint16_t dbRoot, - uint32_t& partition, - uint16_t& segment, - HWM& hwm, - int& status, - bool& bFound) +inline int BRMWrapper::getLastHWM_DBroot(OID oid, uint16_t dbRoot, uint32_t& partition, uint16_t& segment, + HWM& hwm, int& status, bool& bFound) { - int rc = blockRsltnMgrPtr->getLastHWM_DBroot( - (BRM::OID_t)oid, dbRoot, partition, segment, hwm, - status, bFound); - return getRC( rc, ERR_BRM_GET_HWM ); + int rc = + blockRsltnMgrPtr->getLastHWM_DBroot((BRM::OID_t)oid, dbRoot, partition, segment, hwm, status, bFound); + return getRC(rc, ERR_BRM_GET_HWM); } -inline int BRMWrapper::getLocalHWM( OID oid, - uint32_t partition, - uint16_t segment, - HWM& hwm, - int& status) +inline int BRMWrapper::getLocalHWM(OID oid, uint32_t partition, uint16_t segment, HWM& hwm, int& status) { - int rc = blockRsltnMgrPtr->getLocalHWM( - (BRM::OID_t)oid, partition, segment, hwm, status); - return getRC( rc, ERR_BRM_GET_HWM ); + int rc = blockRsltnMgrPtr->getLocalHWM((BRM::OID_t)oid, partition, segment, hwm, status); + return getRC(rc, ERR_BRM_GET_HWM); } -inline int BRMWrapper::getExtentState( OID oid, - uint32_t partition, - uint16_t segment, - bool& bFound, - int& status) +inline int BRMWrapper::getExtentState(OID oid, uint32_t partition, uint16_t segment, bool& bFound, + int& status) { - int rc = blockRsltnMgrPtr->getExtentState( - (BRM::OID_t)oid, partition, segment, bFound, status); - return getRC( rc, ERR_BRM_GET_EXT_STATE ); + int rc = blockRsltnMgrPtr->getExtentState((BRM::OID_t)oid, partition, segment, bFound, status); + return getRC(rc, ERR_BRM_GET_EXT_STATE); } inline unsigned BRMWrapper::getExtentRows() { - return blockRsltnMgrPtr->getExtentRows( ); + return blockRsltnMgrPtr->getExtentRows(); } -inline int BRMWrapper::getExtents( int oid, - std::vector& entries, - bool sorted, bool notFoundErr, - bool incOutOfService ) +inline int BRMWrapper::getExtents(int oid, std::vector& entries, bool sorted, + bool notFoundErr, bool incOutOfService) { - int rc = blockRsltnMgrPtr->getExtents( - oid, entries, sorted, notFoundErr, incOutOfService); - return rc; + int rc = blockRsltnMgrPtr->getExtents(oid, entries, sorted, notFoundErr, incOutOfService); + return rc; } -inline int BRMWrapper::getExtents_dbroot( int oid, - std::vector& entries, - const uint16_t dbroot ) +inline int BRMWrapper::getExtents_dbroot(int oid, std::vector& entries, + const uint16_t dbroot) { - int rc = blockRsltnMgrPtr->getExtents_dbroot( - oid, entries, dbroot); - return rc; + int rc = blockRsltnMgrPtr->getExtents_dbroot(oid, entries, dbroot); + return rc; } inline bool BRMWrapper::isSystemReady() { - return blockRsltnMgrPtr->getSystemReady() > 0 ? true : false; + return blockRsltnMgrPtr->getSystemReady() > 0 ? true : false; } -inline int BRMWrapper::lookupLbidRanges( OID oid, BRM::LBIDRange_v& lbidRanges) +inline int BRMWrapper::lookupLbidRanges(OID oid, BRM::LBIDRange_v& lbidRanges) { - int rc = blockRsltnMgrPtr->lookup( oid, lbidRanges ); - return getRC( rc, ERR_BRM_LOOKUP_LBID_RANGES ); + int rc = blockRsltnMgrPtr->lookup(oid, lbidRanges); + return getRC(rc, ERR_BRM_LOOKUP_LBID_RANGES); } -inline int BRMWrapper::markExtentInvalid( const uint64_t lbid, - const execplan::CalpontSystemCatalog::ColDataType colDataType ) +inline int BRMWrapper::markExtentInvalid(const uint64_t lbid, + const execplan::CalpontSystemCatalog::ColDataType colDataType) { - int rc = blockRsltnMgrPtr->markExtentInvalid( lbid, colDataType ); - return getRC( rc, ERR_BRM_MARK_INVALID ); + int rc = blockRsltnMgrPtr->markExtentInvalid(lbid, colDataType); + return getRC(rc, ERR_BRM_MARK_INVALID); } -inline int BRMWrapper::markExtentsInvalid(std::vector& lbids, - const std::vector& - colDataTypes) +inline int BRMWrapper::markExtentsInvalid( + std::vector& lbids, + const std::vector& colDataTypes) { - int rc = 0; + int rc = 0; - if (idbdatafile::IDBPolicy::useHdfs()) - return rc; + if (idbdatafile::IDBPolicy::useHdfs()) + return rc; - rc = blockRsltnMgrPtr->markExtentsInvalid(lbids, colDataTypes); - return getRC( rc, ERR_BRM_MARK_INVALID ); + rc = blockRsltnMgrPtr->markExtentsInvalid(lbids, colDataTypes); + return getRC(rc, ERR_BRM_MARK_INVALID); } -inline int BRMWrapper::bulkSetHWMAndCP( - const std::vector& hwmArgs, - const std::vector& mergeCPDataArgs) +inline int BRMWrapper::bulkSetHWMAndCP(const std::vector& hwmArgs, + const std::vector& mergeCPDataArgs) { - std::vector setCPDataArgs; // not used - BRM::VER_t transID = 0; // n/a - int rc = blockRsltnMgrPtr->bulkSetHWMAndCP( - hwmArgs, setCPDataArgs, mergeCPDataArgs, transID ); + std::vector setCPDataArgs; // not used + BRM::VER_t transID = 0; // n/a + int rc = blockRsltnMgrPtr->bulkSetHWMAndCP(hwmArgs, setCPDataArgs, mergeCPDataArgs, transID); - return getRC( rc, ERR_BRM_BULK_UPDATE ); + return getRC(rc, ERR_BRM_BULK_UPDATE); } inline int BRMWrapper::setExtentsMaxMin(const ExtCPInfoList& extCPInfoList) { - BRM::CPInfoList_t toSet; - toSet.reserve(extCPInfoList.size()); - for (const auto& extCPInfo : extCPInfoList) - { - toSet.push_back(extCPInfo.fCPInfo); - } - int rc = blockRsltnMgrPtr->setExtentsMaxMin(toSet); - return getRC( rc, ERR_BRM_SET_EXTENTS_CP ); + BRM::CPInfoList_t toSet; + toSet.reserve(extCPInfoList.size()); + for (const auto& extCPInfo : extCPInfoList) + { + toSet.push_back(extCPInfo.fCPInfo); + } + int rc = blockRsltnMgrPtr->setExtentsMaxMin(toSet); + return getRC(rc, ERR_BRM_SET_EXTENTS_CP); } -inline int BRMWrapper::rollbackColumnExtents_DBroot( const OID oid, - bool bDeleteAll, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - BRM::HWM_t hwm ) +inline int BRMWrapper::rollbackColumnExtents_DBroot(const OID oid, bool bDeleteAll, uint16_t dbRoot, + uint32_t partition, uint16_t segment, BRM::HWM_t hwm) { - int rc = blockRsltnMgrPtr->rollbackColumnExtents_DBroot ( - oid, bDeleteAll, dbRoot, partition, segment, hwm ); - return getRC( rc, ERR_BRM_BULK_RB_COLUMN ); + int rc = blockRsltnMgrPtr->rollbackColumnExtents_DBroot(oid, bDeleteAll, dbRoot, partition, segment, hwm); + return getRC(rc, ERR_BRM_BULK_RB_COLUMN); } -inline int BRMWrapper::rollbackDictStoreExtents_DBroot( OID oid, - uint16_t dbRoot, - uint32_t partition, - const std::vector& segNums, - const std::vector& hwms ) +inline int BRMWrapper::rollbackDictStoreExtents_DBroot(OID oid, uint16_t dbRoot, uint32_t partition, + const std::vector& segNums, + const std::vector& hwms) { - int rc = blockRsltnMgrPtr->rollbackDictStoreExtents_DBroot ( - oid, dbRoot, partition, segNums, hwms ); - return getRC( rc, ERR_BRM_BULK_RB_DCTNRY ); + int rc = blockRsltnMgrPtr->rollbackDictStoreExtents_DBroot(oid, dbRoot, partition, segNums, hwms); + return getRC(rc, ERR_BRM_BULK_RB_DCTNRY); } -inline int BRMWrapper::deleteEmptyColExtents( - const std::vector& extentsInfo ) +inline int BRMWrapper::deleteEmptyColExtents(const std::vector& extentsInfo) { - int rc = blockRsltnMgrPtr->deleteEmptyColExtents ( extentsInfo ); - return getRC( rc, ERR_BRM_DELETE_EXTENT_COLUMN ); + int rc = blockRsltnMgrPtr->deleteEmptyColExtents(extentsInfo); + return getRC(rc, ERR_BRM_DELETE_EXTENT_COLUMN); } -inline int BRMWrapper::deleteEmptyDictStoreExtents( - const std::vector& extentsInfo ) +inline int BRMWrapper::deleteEmptyDictStoreExtents(const std::vector& extentsInfo) { - int rc = blockRsltnMgrPtr->deleteEmptyDictStoreExtents ( extentsInfo ); - return getRC( rc, ERR_BRM_DELETE_EXTENT_DCTNRY ); + int rc = blockRsltnMgrPtr->deleteEmptyDictStoreExtents(extentsInfo); + return getRC(rc, ERR_BRM_DELETE_EXTENT_DCTNRY); } -inline int BRMWrapper::setLocalHWM( OID oid, - uint32_t partition, - uint16_t segment, - const HWM hwm ) +inline int BRMWrapper::setLocalHWM(OID oid, uint32_t partition, uint16_t segment, const HWM hwm) { - int rc = blockRsltnMgrPtr->setLocalHWM( - (int)oid, partition, segment, hwm); - return getRC( rc, ERR_BRM_SET_HWM ); + int rc = blockRsltnMgrPtr->setLocalHWM((int)oid, partition, segment, hwm); + return getRC(rc, ERR_BRM_SET_HWM); } -inline int BRMWrapper::bulkSetHWM( const std::vector& vec, - BRM::VER_t transID = 0) +inline int BRMWrapper::bulkSetHWM(const std::vector& vec, BRM::VER_t transID = 0) { - int rc = blockRsltnMgrPtr->bulkSetHWM( vec, transID); - return getRC( rc, ERR_BRM_SET_HWM ); + int rc = blockRsltnMgrPtr->bulkSetHWM(vec, transID); + return getRC(rc, ERR_BRM_SET_HWM); } inline int BRMWrapper::takeSnapshot() { - int rc = blockRsltnMgrPtr->takeSnapshot(); - return getRC( rc, ERR_BRM_TAKE_SNAPSHOT ); + int rc = blockRsltnMgrPtr->takeSnapshot(); + return getRC(rc, ERR_BRM_TAKE_SNAPSHOT); } -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_BRM_H_ +#endif // _WE_BRM_H_ diff --git a/writeengine/shared/we_bulkrollbackfile.cpp b/writeengine/shared/we_bulkrollbackfile.cpp index 9bf39fa30..6e9fb6764 100644 --- a/writeengine/shared/we_bulkrollbackfile.cpp +++ b/writeengine/shared/we_bulkrollbackfile.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_bulkrollbackfile.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -*/ + * $Id: we_bulkrollbackfile.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + */ #include "we_bulkrollbackfile.h" #include "we_bulkrollbackmgr.h" @@ -34,26 +34,23 @@ using namespace execplan; namespace WriteEngine { - //------------------------------------------------------------------------------ // BulkRollbackFile constructor //------------------------------------------------------------------------------ BulkRollbackFile::BulkRollbackFile(BulkRollbackMgr* mgr) : fMgr(mgr) { - // Initialize empty dictionary header block used when reinitializing - // dictionary store extents. - const uint16_t freeSpace = BYTE_PER_BLOCK - - (HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE); - const uint64_t nextPtr = NOT_USED_PTR; - const uint16_t offSetZero = BYTE_PER_BLOCK; - const uint16_t endHeader = DCTNRY_END_HEADER; + // Initialize empty dictionary header block used when reinitializing + // dictionary store extents. + const uint16_t freeSpace = + BYTE_PER_BLOCK - (HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE); + const uint64_t nextPtr = NOT_USED_PTR; + const uint16_t offSetZero = BYTE_PER_BLOCK; + const uint16_t endHeader = DCTNRY_END_HEADER; - memcpy(fDctnryHdr, &freeSpace, HDR_UNIT_SIZE); - memcpy(fDctnryHdr + HDR_UNIT_SIZE, &nextPtr, NEXT_PTR_BYTES); - memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES, - &offSetZero, HDR_UNIT_SIZE); - memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE, - &endHeader, HDR_UNIT_SIZE); + memcpy(fDctnryHdr, &freeSpace, HDR_UNIT_SIZE); + memcpy(fDctnryHdr + HDR_UNIT_SIZE, &nextPtr, NEXT_PTR_BYTES); + memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES, &offSetZero, HDR_UNIT_SIZE); + memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE, &endHeader, HDR_UNIT_SIZE); } //------------------------------------------------------------------------------ @@ -73,35 +70,24 @@ BulkRollbackFile::~BulkRollbackFile() // segNum - Segment number of segment file to be found // segFileName (out) - Name of segment file //------------------------------------------------------------------------------ -void BulkRollbackFile::buildSegmentFileName( - OID columnOID, - bool fileTypeFlag, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - std::string& segFileName ) +void BulkRollbackFile::buildSegmentFileName(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, + uint32_t partNum, uint32_t segNum, std::string& segFileName) { - char fileName[FILE_NAME_SIZE]; - int rc = fDbFile.getFileName( columnOID, fileName, - dbRoot, partNum, segNum ); + char fileName[FILE_NAME_SIZE]; + int rc = fDbFile.getFileName(columnOID, fileName, dbRoot, partNum, segNum); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error constructing " << - (fileTypeFlag ? "column" : "dictionary store") << - " filename for deletion" << - "; columnOID-" << columnOID << - "; dbRoot-" << dbRoot << - "; partNum-" << partNum << - "; segNum-" << segNum << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error constructing " << (fileTypeFlag ? "column" : "dictionary store") << " filename for deletion" + << "; columnOID-" << columnOID << "; dbRoot-" << dbRoot << "; partNum-" << partNum << "; segNum-" + << segNum << "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } + throw WeException(oss.str(), rc); + } - segFileName = fileName; + segFileName = fileName; } //------------------------------------------------------------------------------ @@ -114,41 +100,31 @@ void BulkRollbackFile::buildSegmentFileName( // segNum - Segment number of segment file to be deleted // segFileName - Name of file to be deleted //------------------------------------------------------------------------------ -void BulkRollbackFile::deleteSegmentFile( - OID columnOID, - bool fileTypeFlag, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - const std::string& segFileName ) +void BulkRollbackFile::deleteSegmentFile(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, const std::string& segFileName) { - std::ostringstream msgText; - msgText << "Deleting " << (fileTypeFlag ? "column" : "dictionary store") << - " file: dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Deleting " << (fileTypeFlag ? "column" : "dictionary store") << " file: dbRoot-" << dbRoot + << "; part#-" << partNum << "; seg#-" << segNum; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str()); - // delete the db segment file if it exists - int rc = fDbFile.deleteFile( segFileName.c_str() ); + // delete the db segment file if it exists + int rc = fDbFile.deleteFile(segFileName.c_str()); - if (rc != NO_ERROR) + if (rc != NO_ERROR) + { + if (rc != ERR_FILE_NOT_EXIST) { - if (rc != ERR_FILE_NOT_EXIST) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error deleting segment file" - "; columnOID-" << columnOID << - "; dbRoot-" << dbRoot << - "; partNum-" << partNum << - "; segNum-" << segNum << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error deleting segment file" + "; columnOID-" + << columnOID << "; dbRoot-" << dbRoot << "; partNum-" << partNum << "; segNum-" << segNum << "; " + << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } + throw WeException(oss.str(), rc); } + } } //------------------------------------------------------------------------------ @@ -161,59 +137,47 @@ void BulkRollbackFile::deleteSegmentFile( // fileSizeBlocks - Number of blocks to be left in the file. Remainder of file // is to be truncated. //------------------------------------------------------------------------------ -void BulkRollbackFile::truncateSegmentFile( - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long fileSizeBlocks ) +void BulkRollbackFile::truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long fileSizeBlocks) { - long long fileSizeBytes = fileSizeBlocks * BYTE_PER_BLOCK; + long long fileSizeBytes = fileSizeBlocks * BYTE_PER_BLOCK; - std::ostringstream msgText; - msgText << "Truncating column file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; totBlks-" << fileSizeBlocks << - "; fileSize(bytes)-" << fileSizeBytes; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Truncating column file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; totBlks-" << fileSizeBlocks + << "; fileSize(bytes)-" << fileSizeBytes; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str()); - std::string segFile; - IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); + std::string segFile; + IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); - if (pFile == 0) - { - std::ostringstream oss; - oss << "Error opening column segment file to rollback extents " - "from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; + if (pFile == 0) + { + std::ostringstream oss; + oss << "Error opening column segment file to rollback extents " + "from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; - throw WeException( oss.str(), ERR_FILE_OPEN ); - } + throw WeException(oss.str(), ERR_FILE_OPEN); + } - int rc = fDbFile.truncateFile( pFile, fileSizeBytes ); + int rc = fDbFile.truncateFile(pFile, fileSizeBytes); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating column extents from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating column extents from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } - fDbFile.closeFile( pFile ); + fDbFile.closeFile(pFile); } //------------------------------------------------------------------------------ @@ -233,122 +197,98 @@ void BulkRollbackFile::truncateSegmentFile( // uncompressed, but defined in this base class for the // compressed derived class. //------------------------------------------------------------------------------ -void BulkRollbackFile::reInitTruncColumnExtent( - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks, - CalpontSystemCatalog::ColDataType colType, - uint32_t colWidth, - bool /*restoreHwmChk*/ ) +void BulkRollbackFile::reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long startOffsetBlk, int nBlocks, + CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, + bool /*restoreHwmChk*/) { - long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; + long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; - std::ostringstream msgText; - msgText << "Reinit HWM column extent in db file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; offset(bytes)-" << startOffset << - "; freeBlks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Reinit HWM column extent in db file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" << startOffset + << "; freeBlks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str()); - std::string segFile; - IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); + std::string segFile; + IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); - if (pFile == 0) + if (pFile == 0) + { + std::ostringstream oss; + oss << "Error opening HWM column segment file to rollback extents " + "from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + + throw WeException(oss.str(), ERR_FILE_OPEN); + } + + // nBlocks is based on full extents, but if the database file only has an + // abbreviated extent, then we reset nBlocks to reflect the size of the file + // (Only the 1st extent in part0, seg0 employs an abbreviated extent.) + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((partNum == 0) && (segNum == 0)) + { + long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * colWidth; + + if (startOffset <= nBytesInAbbrevExtent) { - std::ostringstream oss; - oss << "Error opening HWM column segment file to rollback extents " - "from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; + // This check would prevent us from truncating back to an + // abbreviated extent if the failed import expanded the initial + // extent; but when adding compression, decided to go ahead and + // truncate back to an abbreviated extent. + // long long fileSizeBytes; + // int rc = fDbFile.getFileSize2(pFile,fileSizeBytes); + // if (fileSizeBytes == nBytesInAbbrevExtent) + { + nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; - throw WeException( oss.str(), ERR_FILE_OPEN ); + std::ostringstream msgText2; + msgText2 << "Reinit (abbrev) HWM column extent in db file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" + << startOffset << "; freeBlks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText2.str()); + } } + } - // nBlocks is based on full extents, but if the database file only has an - // abbreviated extent, then we reset nBlocks to reflect the size of the file - // (Only the 1st extent in part0, seg0 employs an abbreviated extent.) -// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((partNum == 0) && (segNum == 0)) - { - long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * colWidth; + // Initialize the remainder of the extent after the HWM block + const uint8_t* emptyVal = fDbFile.getEmptyRowValue(colType, colWidth); - if (startOffset <= nBytesInAbbrevExtent) - { - // This check would prevent us from truncating back to an - // abbreviated extent if the failed import expanded the initial - // extent; but when adding compression, decided to go ahead and - // truncate back to an abbreviated extent. - //long long fileSizeBytes; - //int rc = fDbFile.getFileSize2(pFile,fileSizeBytes); - //if (fileSizeBytes == nBytesInAbbrevExtent) - { - nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; + int rc = fDbFile.reInitPartialColumnExtent(pFile, startOffset, nBlocks, emptyVal, colWidth); - std::ostringstream msgText2; - msgText2 << "Reinit (abbrev) HWM column extent in db file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; offset(bytes)-" << startOffset << - "; freeBlks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText2.str() ); - } - } - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error rolling back HWM column extent from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - // Initialize the remainder of the extent after the HWM block - const uint8_t* emptyVal = fDbFile.getEmptyRowValue( colType, colWidth ); + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } - int rc = fDbFile.reInitPartialColumnExtent( pFile, - startOffset, - nBlocks, - emptyVal, - colWidth ); + // Truncate the remainder of the file + rc = fDbFile.truncateFile(pFile, pFile->tell()); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error rolling back HWM column extent from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating post-HWM column extents " + "from HWM segment DB file for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } - // Truncate the remainder of the file - rc = fDbFile.truncateFile( pFile, pFile->tell() ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating post-HWM column extents " - "from HWM segment DB file for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - - fDbFile.closeFile( pFile ); + fDbFile.closeFile(pFile); } //------------------------------------------------------------------------------ @@ -363,116 +303,92 @@ void BulkRollbackFile::reInitTruncColumnExtent( // to be reinitialized. Value is in raw data blocks. // nBlocks - Number of blocks to be reinitialized //------------------------------------------------------------------------------ -void BulkRollbackFile::reInitTruncDctnryExtent( - OID dStoreOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks ) +void BulkRollbackFile::reInitTruncDctnryExtent(OID dStoreOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long startOffsetBlk, int nBlocks) { - long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; + long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; - std::ostringstream msgText; - msgText << "Reinit dictionary store extent in db file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; offset(bytes)-" << startOffset << - "; numblks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, dStoreOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Reinit dictionary store extent in db file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" << startOffset + << "; numblks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText.str()); - std::string segFile; - IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile); + std::string segFile; + IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile); - if (pFile == 0) + if (pFile == 0) + { + std::ostringstream oss; + oss << "Error opening dictionary store segment file to rollback extents" + " from DB for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + + throw WeException(oss.str(), ERR_FILE_OPEN); + } + + // nBlocks is based on full extents, but if the database file only has an + // abbreviated extent, then we reset nBlocks to reflect the size of the file + // (Unlike column files which only employ an abbreviated extent for the + // 1st extent in part0, seg0, all new store files start with abbrev extent) + const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dictionary + long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * PSEUDO_COL_WIDTH; + + if (startOffset <= nBytesInAbbrevExtent) + { + // This check would prevent us from truncating back to an + // abbreviated extent if the failed import expanded the initial + // extent; but when adding compression, decided to go ahead and + // truncate back to an abbreviated extent. + // long long fileSizeBytes; + // int rc = fDbFile.getFileSize2(pFile,fileSizeBytes); + // if (fileSizeBytes == nBytesInAbbrevExtent) { - std::ostringstream oss; - oss << "Error opening dictionary store segment file to rollback extents" - " from DB for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; + nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; - throw WeException( oss.str(), ERR_FILE_OPEN ); + std::ostringstream msgText2; + msgText2 << "Reinit (abbrev) dictionary store extent in db file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" << startOffset + << "; numblks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText2.str()); } + } - // nBlocks is based on full extents, but if the database file only has an - // abbreviated extent, then we reset nBlocks to reflect the size of the file - // (Unlike column files which only employ an abbreviated extent for the - // 1st extent in part0, seg0, all new store files start with abbrev extent) - const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dictionary - long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * - PSEUDO_COL_WIDTH; + // Initialize the remainder of the extent after the HWM block + int rc = fDbFile.reInitPartialDctnryExtent(pFile, startOffset, nBlocks, fDctnryHdr, DCTNRY_HEADER_SIZE); - if (startOffset <= nBytesInAbbrevExtent) - { - // This check would prevent us from truncating back to an - // abbreviated extent if the failed import expanded the initial - // extent; but when adding compression, decided to go ahead and - // truncate back to an abbreviated extent. - //long long fileSizeBytes; - //int rc = fDbFile.getFileSize2(pFile,fileSizeBytes); - //if (fileSizeBytes == nBytesInAbbrevExtent) - { - nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error rolling back HWM dictionary store extent from DB for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - std::ostringstream msgText2; - msgText2 << "Reinit (abbrev) dictionary store extent in db file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; offset(bytes)-" << startOffset << - "; numblks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, dStoreOID, msgText2.str() ); - } - } + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } - // Initialize the remainder of the extent after the HWM block - int rc = fDbFile.reInitPartialDctnryExtent( pFile, - startOffset, - nBlocks, - fDctnryHdr, - DCTNRY_HEADER_SIZE ); + // Truncate the remainder of the file + rc = fDbFile.truncateFile(pFile, pFile->tell()); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error rolling back HWM dictionary store extent from DB for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating post-HWM dictionary store extents " + "from DB file for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } - // Truncate the remainder of the file - rc = fDbFile.truncateFile( pFile, pFile->tell() ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating post-HWM dictionary store extents " - "from DB file for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - - fDbFile.closeFile( pFile ); + fDbFile.closeFile(pFile); } //------------------------------------------------------------------------------ @@ -483,12 +399,10 @@ void BulkRollbackFile::reInitTruncDctnryExtent( // class can override this functionality, and return true or false depending // on whether the HWM chunk was modified and backed up to disk. //------------------------------------------------------------------------------ -bool BulkRollbackFile::doWeReInitExtent( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) const +bool BulkRollbackFile::doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum) const { - return true; + return true; } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_bulkrollbackfile.h b/writeengine/shared/we_bulkrollbackfile.h index 52ee066e3..6db55e761 100644 --- a/writeengine/shared/we_bulkrollbackfile.h +++ b/writeengine/shared/we_bulkrollbackfile.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_bulkrollbackfile.h 4675 2013-06-13 15:20:37Z dcathey $ -*/ + * $Id: we_bulkrollbackfile.h 4675 2013-06-13 15:20:37Z dcathey $ + */ /** @file * Contains class to restore db files on behalf of BulkRollBackMgr. @@ -46,162 +46,132 @@ class BulkRollbackMgr; //------------------------------------------------------------------------------ class BulkRollbackFile { -public: + public: + /** @brief BulkRollbackFile constructor + * @param mgr The controlling BulkRollbackMgr object. + */ + BulkRollbackFile(BulkRollbackMgr* mgr); - /** @brief BulkRollbackFile constructor - * @param mgr The controlling BulkRollbackMgr object. - */ - BulkRollbackFile(BulkRollbackMgr* mgr); + /** @brief BulkRollbackFile destructor + */ + virtual ~BulkRollbackFile(); - /** @brief BulkRollbackFile destructor - */ - virtual ~BulkRollbackFile(); + /** @brief Construct the relevant db filename. + * Warning: This function may throw a WeException. + * + * @param columnOID OID of the segment file to be deleted + * @param fileTypeFlag file type (true->column; false->dictionary) + * @param dbRoot DBRoot of the segment file to be deleted + * @param partNum Partition number of the segment file to be deleted + * @param segNum Segment number of the segment file to be deleted + * @param segFileName (out) Name of segment file + */ + void buildSegmentFileName(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, std::string& segFileName); - /** @brief Construct the relevant db filename. - * Warning: This function may throw a WeException. - * - * @param columnOID OID of the segment file to be deleted - * @param fileTypeFlag file type (true->column; false->dictionary) - * @param dbRoot DBRoot of the segment file to be deleted - * @param partNum Partition number of the segment file to be deleted - * @param segNum Segment number of the segment file to be deleted - * @param segFileName (out) Name of segment file - */ - void buildSegmentFileName(OID columnOID, - bool fileTypeFlag, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - std::string& segFileName); + /** @brief Delete a segment file. + * Warning: This function may throw a WeException. + * + * @param columnOID OID of the segment file to be deleted + * @param fileTypeFlag file type (true->column; false->dictionary) + * @param dbRoot DBRoot of the segment file to be deleted + * @param partNum Partition number of the segment file to be deleted + * @param segNum Segment number of the segment file to be deleted + * @param segFileName Name of segment file to be deleted + */ + void deleteSegmentFile(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + const std::string& segFileName); - /** @brief Delete a segment file. - * Warning: This function may throw a WeException. - * - * @param columnOID OID of the segment file to be deleted - * @param fileTypeFlag file type (true->column; false->dictionary) - * @param dbRoot DBRoot of the segment file to be deleted - * @param partNum Partition number of the segment file to be deleted - * @param segNum Segment number of the segment file to be deleted - * @param segFileName Name of segment file to be deleted - */ - void deleteSegmentFile(OID columnOID, - bool fileTypeFlag, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - const std::string& segFileName ); + /** @brief Construct a directory path. + * + * @param oid (in) OID to use in constructing directory path + * @param dbRoot (in) DBRoot to use in constructing directory path + * @param partition (in) Partition number to use in constructing dir path + * @param dirName (out)Directory path constructed from input arguments + * @return returns NO_ERROR if success + */ + int buildDirName(OID oid, uint16_t dbRoot, uint32_t partition, std::string& dirName); - /** @brief Construct a directory path. - * - * @param oid (in) OID to use in constructing directory path - * @param dbRoot (in) DBRoot to use in constructing directory path - * @param partition (in) Partition number to use in constructing dir path - * @param dirName (out)Directory path constructed from input arguments - * @return returns NO_ERROR if success - */ - int buildDirName( OID oid, - uint16_t dbRoot, - uint32_t partition, - std::string& dirName); + /** @brief Do we reinit trailing blocks in the HWM extent for the specified + * segment file + * + * The base behavior of this function always returns true, to reinit + * any trailing blocks in the HWM extent (for uncompressed data) to + * empty values. + * + * @param columnOID OID of the segment file in question + * @param dbRoot DBRoot for the segment file in question + * @param partNum Partition number for the segment file in question + * @param segNum Segment number for the segment file in question + */ + virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const; - /** @brief Do we reinit trailing blocks in the HWM extent for the specified - * segment file - * - * The base behavior of this function always returns true, to reinit - * any trailing blocks in the HWM extent (for uncompressed data) to - * empty values. - * - * @param columnOID OID of the segment file in question - * @param dbRoot DBRoot for the segment file in question - * @param partNum Partition number for the segment file in question - * @param segNum Segment number for the segment file in question - */ - virtual bool doWeReInitExtent( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) const; + /** @brief Reinitialize the specified column segment file starting at + * startOffsetBlk, and truncate trailing extents. + * Warning: This function may throw a WeException. + * + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param startOffsetBlk Starting block offset where file is to be + * reinitialized + * @param nBlocks Number of blocks to be reinitialized + * @param colType Column type of the relevant segment file + * @param colWidth Width in bytes of column. + * @param restoreHwmChk Restore HWM chunk (n/a to uncompressed) + */ + virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks, + execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, + bool restoreHwmChk); - /** @brief Reinitialize the specified column segment file starting at - * startOffsetBlk, and truncate trailing extents. - * Warning: This function may throw a WeException. - * - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param startOffsetBlk Starting block offset where file is to be - * reinitialized - * @param nBlocks Number of blocks to be reinitialized - * @param colType Column type of the relevant segment file - * @param colWidth Width in bytes of column. - * @param restoreHwmChk Restore HWM chunk (n/a to uncompressed) - */ - virtual void reInitTruncColumnExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks, - execplan::CalpontSystemCatalog::ColDataType colType, - uint32_t colWidth, - bool restoreHwmChk ); + /** @brief Reinitialize the specified dictionary store segment file starting + * at startOffsetBlk, and truncate trailing extents. + * Warning: This function may throw a WeException. + * + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param startOffsetBlk Starting block offset where file is to be + * reinitialized + * @param nBlocks Number of blocks to be reinitialized + */ + virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks); - /** @brief Reinitialize the specified dictionary store segment file starting - * at startOffsetBlk, and truncate trailing extents. - * Warning: This function may throw a WeException. - * - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param startOffsetBlk Starting block offset where file is to be - * reinitialized - * @param nBlocks Number of blocks to be reinitialized - */ - virtual void reInitTruncDctnryExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks ); + /** @brief Truncate the specified segment file to a specified num of bytes + * Warning: This function may throw a WeException. + * + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param fileSizeBlocks Number of blocks to retain in the file + */ + virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long filesSizeBlocks); - /** @brief Truncate the specified segment file to a specified num of bytes - * Warning: This function may throw a WeException. - * - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param fileSizeBlocks Number of blocks to retain in the file - */ - virtual void truncateSegmentFile( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long filesSizeBlocks ); + protected: + BulkRollbackMgr* fMgr; // Bulk Rollback controller + FileOp fDbFile; // interface to DB file + unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk -protected: - BulkRollbackMgr* fMgr; // Bulk Rollback controller - FileOp fDbFile; // interface to DB file - unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk - -private: - // Disable unnecessary copy constructor and assignment operator - BulkRollbackFile(const BulkRollbackFile& rhs); - BulkRollbackFile& operator=(const BulkRollbackFile& rhs); + private: + // Disable unnecessary copy constructor and assignment operator + BulkRollbackFile(const BulkRollbackFile& rhs); + BulkRollbackFile& operator=(const BulkRollbackFile& rhs); }; //------------------------------------------------------------------------------ // Inline functions //------------------------------------------------------------------------------ -inline int BulkRollbackFile::buildDirName( OID oid, - uint16_t dbRoot, - uint32_t partition, - std::string& dirName) +inline int BulkRollbackFile::buildDirName(OID oid, uint16_t dbRoot, uint32_t partition, std::string& dirName) { - return fDbFile.getDirName( oid, dbRoot, partition, dirName ); + return fDbFile.getDirName(oid, dbRoot, partition, dirName); } -} //end of namespace +} // namespace WriteEngine -#endif // WE_BULKROLLBACKFILE_H_ +#endif // WE_BULKROLLBACKFILE_H_ diff --git a/writeengine/shared/we_bulkrollbackfilecompressed.cpp b/writeengine/shared/we_bulkrollbackfilecompressed.cpp index f15b090e9..7e2b8c509 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressed.cpp +++ b/writeengine/shared/we_bulkrollbackfilecompressed.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_bulkrollbackfilecompressed.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -*/ + * $Id: we_bulkrollbackfilecompressed.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + */ #include "we_bulkrollbackfilecompressed.h" @@ -44,14 +44,12 @@ const char* DATA_DIR_SUFFIX = "_data"; namespace WriteEngine { - //------------------------------------------------------------------------------ // BulkRollbackFileCompressed constructor //------------------------------------------------------------------------------ -BulkRollbackFileCompressed::BulkRollbackFileCompressed(BulkRollbackMgr* mgr) : - BulkRollbackFile(mgr) +BulkRollbackFileCompressed::BulkRollbackFileCompressed(BulkRollbackMgr* mgr) : BulkRollbackFile(mgr) { - compress::initializeCompressorPool(fCompressorPool); + compress::initializeCompressorPool(fCompressorPool); } //------------------------------------------------------------------------------ @@ -60,7 +58,7 @@ BulkRollbackFileCompressed::BulkRollbackFileCompressed(BulkRollbackMgr* mgr) : BulkRollbackFileCompressed::~BulkRollbackFileCompressed() { } - + //------------------------------------------------------------------------------ // Truncate the specified database segment file to the extent specified by // the given file offset. Also updates the header(s) as well. @@ -72,144 +70,122 @@ BulkRollbackFileCompressed::~BulkRollbackFileCompressed() // fileSizeBlocks - Number of raw data blocks to be left in the file. // Remainder of file is to be truncated. //------------------------------------------------------------------------------ -void BulkRollbackFileCompressed::truncateSegmentFile( - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long fileSizeBlocks ) +void BulkRollbackFileCompressed::truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long fileSizeBlocks) { - std::ostringstream msgText1; - msgText1 << "Truncating compressed column file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; rawTotBlks-" << fileSizeBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText1.str() ); + std::ostringstream msgText1; + msgText1 << "Truncating compressed column file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawTotBlks-" << fileSizeBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText1.str()); - std::string segFile; - IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); + std::string segFile; + IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); - if (pFile == 0) + if (pFile == 0) + { + std::ostringstream oss; + oss << "Error opening compressed column segment file to rollback " + "extents from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + + throw WeException(oss.str(), ERR_FILE_OPEN); + } + + // Read and parse the header pointers + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + ; + CompChunkPtrList chunkPtrs; + std::string errMsg; + int rc = loadColumnHdrPtrs(pFile, hdrs, chunkPtrs, errMsg); + + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error reading compressed column ptr headers from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << errMsg; + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + + // Locate the chunk containing the last block we intend to keep + unsigned int blockOffset = fileSizeBlocks - 1; + unsigned int chunkIndex = 0; + unsigned int blkOffsetInChunk = 0; + + auto fCompressor = + compress::getCompressorByType(fCompressorPool, compress::CompressInterface::getCompressionType(hdrs)); + if (!fCompressor) + { + std::ostringstream oss; + oss << "Error, wrong compression type for segment file" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << ";"; + throw WeException(oss.str(), ERR_COMP_WRONG_COMP_TYPE); + } + + fCompressor->locateBlock(blockOffset, chunkIndex, blkOffsetInChunk); + + // Truncate the extra extents that are to be aborted + if (chunkIndex < chunkPtrs.size()) + { + long long fileSizeBytes = chunkPtrs[chunkIndex].first + chunkPtrs[chunkIndex].second; + + std::ostringstream msgText2; + msgText2 << "Compressed column file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; truncated to " << fileSizeBytes + << " bytes"; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText2.str()); + + // Drop off any trailing pointers (that point beyond the last block) + compress::CompressInterface::setBlockCount(hdrs, fileSizeBlocks); + std::vector ptrs; + + for (unsigned i = 0; i <= chunkIndex; i++) { - std::ostringstream oss; - oss << "Error opening compressed column segment file to rollback " - "extents from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; - - throw WeException( oss.str(), ERR_FILE_OPEN ); + ptrs.push_back(chunkPtrs[i].first); } - // Read and parse the header pointers - char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ];; - CompChunkPtrList chunkPtrs; - std::string errMsg; - int rc = loadColumnHdrPtrs(pFile, hdrs, chunkPtrs, errMsg); + ptrs.push_back(chunkPtrs[chunkIndex].first + chunkPtrs[chunkIndex].second); + compress::CompressInterface::storePtrs(ptrs, hdrs); + + rc = fDbFile.writeHeaders(pFile, hdrs); if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Error reading compressed column ptr headers from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << errMsg; + WErrorCodes ec; + std::ostringstream oss; + oss << "Error writing compressed column headers to DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); } - // Locate the chunk containing the last block we intend to keep - unsigned int blockOffset = fileSizeBlocks - 1; - unsigned int chunkIndex = 0; - unsigned int blkOffsetInChunk = 0; + // Finally, we truncate the data base column segment file + rc = fDbFile.truncateFile(pFile, fileSizeBytes); - auto fCompressor = compress::getCompressorByType( - fCompressorPool, - compress::CompressInterface::getCompressionType(hdrs)); - if (!fCompressor) + if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Error, wrong compression type for segment file" - << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" - << partNum << "; segment-" << segNum << ";"; - throw WeException(oss.str(), ERR_COMP_WRONG_COMP_TYPE); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating compressed column extents from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); } + } // end of (chunkIndex < chunkPtrs.size()) - fCompressor->locateBlock(blockOffset, chunkIndex, blkOffsetInChunk); - - // Truncate the extra extents that are to be aborted - if (chunkIndex < chunkPtrs.size()) - { - long long fileSizeBytes = chunkPtrs[chunkIndex].first + - chunkPtrs[chunkIndex].second; - - std::ostringstream msgText2; - msgText2 << "Compressed column file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; truncated to " << fileSizeBytes << " bytes"; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText2.str() ); - - // Drop off any trailing pointers (that point beyond the last block) - compress::CompressInterface::setBlockCount(hdrs, fileSizeBlocks); - std::vector ptrs; - - for (unsigned i = 0; i <= chunkIndex; i++) - { - ptrs.push_back( chunkPtrs[i].first ); - } - - ptrs.push_back( chunkPtrs[chunkIndex].first + - chunkPtrs[chunkIndex].second ); - compress::CompressInterface::storePtrs(ptrs, hdrs); - - rc = fDbFile.writeHeaders( pFile, hdrs ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error writing compressed column headers to DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - - // Finally, we truncate the data base column segment file - rc = fDbFile.truncateFile( pFile, fileSizeBytes ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating compressed column extents from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - } // end of (chunkIndex < chunkPtrs.size()) - - fDbFile.closeFile( pFile ); + fDbFile.closeFile(pFile); } - + //------------------------------------------------------------------------------ // Reinitialize a column segment extent (in the db file) to empty values, // following the HWM. Remaining extents in the file are truncated. @@ -226,262 +202,223 @@ void BulkRollbackFileCompressed::truncateSegmentFile( // colWidth - Width in bytes, of the applicable column // restoreHwmChk - Specifies whether HWM chunk is to be restored. //------------------------------------------------------------------------------ -void BulkRollbackFileCompressed::reInitTruncColumnExtent( - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks, - CalpontSystemCatalog::ColDataType colType, - uint32_t colWidth, - bool restoreHwmChk ) +void BulkRollbackFileCompressed::reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long startOffsetBlk, + int nBlocks, + CalpontSystemCatalog::ColDataType colType, + uint32_t colWidth, bool restoreHwmChk) { - long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; + long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; - std::ostringstream msgText1; - msgText1 << "Reinit HWM compressed column extent in db file" << - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; rawOffset(bytes)-" << startOffset << - "; rawFreeBlks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText1.str() ); + std::ostringstream msgText1; + msgText1 << "Reinit HWM compressed column extent in db file" + << ": dbRoot-" << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawOffset(bytes)-" + << startOffset << "; rawFreeBlks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText1.str()); - std::string segFile; - IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); + std::string segFile; + IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile); - if (pFile == 0) + if (pFile == 0) + { + std::ostringstream oss; + oss << "Error opening compressed column segment file to rollback " + "extents from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + + throw WeException(oss.str(), ERR_FILE_OPEN); + } + + // Read and parse the header pointers + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + CompChunkPtrList chunkPtrs; + std::string errMsg; + int rc = loadColumnHdrPtrs(pFile, hdrs, chunkPtrs, errMsg); + + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error reading compressed column ptr headers from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << errMsg; + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + + // Locate the chunk containing the last block we intend to keep + unsigned int blockOffset = startOffsetBlk - 1; + unsigned int chunkIndex = 0; + unsigned int blkOffsetInChunk = 0; + + auto fCompressor = + compress::getCompressorByType(fCompressorPool, compress::CompressInterface::getCompressionType(hdrs)); + if (!fCompressor) + { + std::ostringstream oss; + oss << "Error, wrong compression type for segment file" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << ";"; + throw WeException(oss.str(), ERR_COMP_WRONG_COMP_TYPE); + } + + fCompressor->locateBlock(blockOffset, chunkIndex, blkOffsetInChunk); + + if (chunkIndex < chunkPtrs.size()) + { + // Read backup copy of HWM chunk and restore it's contents + uint64_t restoredChunkLen = 0; + uint64_t restoredFileSize = 0; + + if (restoreHwmChk) { - std::ostringstream oss; - oss << "Error opening compressed column segment file to rollback " - "extents from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; + rc = restoreHWMChunk(pFile, columnOID, partNum, segNum, chunkPtrs[chunkIndex].first, restoredChunkLen, + restoredFileSize, errMsg); - throw WeException( oss.str(), ERR_FILE_OPEN ); + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error restoring HWM chunk for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; blkoff-" << blockOffset << "; " << errMsg; + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + } + else + { + restoredChunkLen = chunkPtrs[chunkIndex].second; + + // leave truncated to last chunk if no extra blocks needed + if (nBlocks == 0) + restoredFileSize = chunkPtrs[chunkIndex].first + chunkPtrs[chunkIndex].second; + else + restoredFileSize = (chunkPtrs[chunkIndex].first + chunkPtrs[chunkIndex].second) + + (uint64_t)(nBlocks * BYTE_PER_BLOCK); } - // Read and parse the header pointers - char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ]; - CompChunkPtrList chunkPtrs; - std::string errMsg; - int rc = loadColumnHdrPtrs(pFile, hdrs, chunkPtrs, errMsg); + // nBlocks is based on full extents, but if database file only has an + // abbreviated extent, then we reset nBlocks to reflect the size of a + // file with a single abbreviated extent. + // (Only the 1st extent in part0, seg0 employs an abbreviated extent.) + bool bAbbreviatedExtent = false; + + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((partNum == 0) && (segNum == 0)) + { + long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * colWidth; + + if (startOffset <= nBytesInAbbrevExtent) + { + nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; + bAbbreviatedExtent = true; + } + } + + long long fileSizeBytes = restoredFileSize; + + std::ostringstream msgText2; + msgText2 << "HWM compressed column file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum; + + if (bAbbreviatedExtent) // log adjusted nBlock count for abbrev extent + msgText2 << "; rawFreeBlks-" << nBlocks << " (abbrev)"; + + msgText2 << "; restoredChunk-" << restoredChunkLen << " bytes"; + + if (!restoreHwmChk) + msgText2 << " (no change)"; + + msgText2 << "; truncated to " << fileSizeBytes << " bytes"; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText2.str()); + + // Initialize the remainder of the extent after the HWM chunk. + // Just doing an ftruncate() reinits the file to 0's, which may or may + // not actually reserve disk space if ftruncate is growing the file. + // So reinit the blocks by calling reInitPartialColumnExtent() to help + // avoid disk fragmentation. Be careful not to init > 1 extent, be- + // cause that is the limit on what that function was intended to do. + const unsigned BLKS_PER_EXTENT = (BRMWrapper::getInstance()->getExtentRows() * colWidth) / BYTE_PER_BLOCK; + long long nBlocksToInit = + (fileSizeBytes - (chunkPtrs[chunkIndex].first + restoredChunkLen)) / BYTE_PER_BLOCK; + + if (nBlocksToInit > BLKS_PER_EXTENT) + nBlocksToInit = BLKS_PER_EXTENT; // don't init > 1 full extent + + if (nBlocksToInit > 0) + { + const uint8_t* emptyVal = fDbFile.getEmptyRowValue(colType, colWidth); + rc = fDbFile.reInitPartialColumnExtent(pFile, (chunkPtrs[chunkIndex].first + restoredChunkLen), + nBlocksToInit, emptyVal, colWidth); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error clearing HWM column extent from DB for" + ": OID-" + << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" << segNum + << "; " << ec.errorString(rc); + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + } + + // Drop off any trailing pointers (that point beyond the last block). + // Watch for the special case where we are restoring a db file as an + // empty file (chunkindex=0 and restoredChunkLen=0); in this case we + // just restore the first pointer (set to 8192). + compress::CompressInterface::setBlockCount(hdrs, (startOffsetBlk + nBlocks)); + std::vector newPtrs; + + if ((chunkIndex > 0) || (restoredChunkLen > 0)) + { + for (unsigned int i = 0; i <= chunkIndex; i++) + { + newPtrs.push_back(chunkPtrs[i].first); + } + } + + newPtrs.push_back(chunkPtrs[chunkIndex].first + restoredChunkLen); + compress::CompressInterface::storePtrs(newPtrs, hdrs); + + rc = fDbFile.writeHeaders(pFile, hdrs); if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Error reading compressed column ptr headers from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << errMsg; + WErrorCodes ec; + std::ostringstream oss; + oss << "Error writing compressed column headers to DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); } - // Locate the chunk containing the last block we intend to keep - unsigned int blockOffset = startOffsetBlk - 1; - unsigned int chunkIndex = 0; - unsigned int blkOffsetInChunk = 0; + // Finally, we truncate the data base column segment file + rc = fDbFile.truncateFile(pFile, fileSizeBytes); - auto fCompressor = compress::getCompressorByType( - fCompressorPool, - compress::CompressInterface::getCompressionType(hdrs)); - if (!fCompressor) + if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Error, wrong compression type for segment file" - << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" - << partNum << "; segment-" << segNum << ";"; - throw WeException(oss.str(), ERR_COMP_WRONG_COMP_TYPE); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating compressed column extents from DB for" + << ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); } + } // end of (chunkIndex < chunkPtrs.size()) - fCompressor->locateBlock(blockOffset, chunkIndex, blkOffsetInChunk); - - if (chunkIndex < chunkPtrs.size()) - { - // Read backup copy of HWM chunk and restore it's contents - uint64_t restoredChunkLen = 0; - uint64_t restoredFileSize = 0; - - if (restoreHwmChk) - { - rc = restoreHWMChunk(pFile, columnOID, partNum, segNum, - chunkPtrs[chunkIndex].first, - restoredChunkLen, restoredFileSize, errMsg); - - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Error restoring HWM chunk for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; blkoff-" << blockOffset << - "; " << errMsg; - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - } - else - { - restoredChunkLen = chunkPtrs[chunkIndex].second; - - // leave truncated to last chunk if no extra blocks needed - if (nBlocks == 0) - restoredFileSize = chunkPtrs[chunkIndex].first + - chunkPtrs[chunkIndex].second; - else - restoredFileSize = (chunkPtrs[chunkIndex].first + - chunkPtrs[chunkIndex].second) + - (uint64_t)(nBlocks * BYTE_PER_BLOCK); - } - - // nBlocks is based on full extents, but if database file only has an - // abbreviated extent, then we reset nBlocks to reflect the size of a - // file with a single abbreviated extent. - // (Only the 1st extent in part0, seg0 employs an abbreviated extent.) - bool bAbbreviatedExtent = false; - -// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((partNum == 0) && (segNum == 0)) - { - long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * - colWidth; - - if (startOffset <= nBytesInAbbrevExtent) - { - nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; - bAbbreviatedExtent = true; - } - } - - long long fileSizeBytes = restoredFileSize; - - std::ostringstream msgText2; - msgText2 << "HWM compressed column file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum; - - if (bAbbreviatedExtent) // log adjusted nBlock count for abbrev extent - msgText2 << "; rawFreeBlks-" << nBlocks << " (abbrev)"; - - msgText2 << "; restoredChunk-" << restoredChunkLen << " bytes"; - - if (!restoreHwmChk) - msgText2 << " (no change)"; - - msgText2 << "; truncated to " << fileSizeBytes << " bytes"; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText2.str() ); - - // Initialize the remainder of the extent after the HWM chunk. - // Just doing an ftruncate() reinits the file to 0's, which may or may - // not actually reserve disk space if ftruncate is growing the file. - // So reinit the blocks by calling reInitPartialColumnExtent() to help - // avoid disk fragmentation. Be careful not to init > 1 extent, be- - // cause that is the limit on what that function was intended to do. - const unsigned BLKS_PER_EXTENT = - (BRMWrapper::getInstance()->getExtentRows() * colWidth) / - BYTE_PER_BLOCK; - long long nBlocksToInit = (fileSizeBytes - - (chunkPtrs[chunkIndex].first + restoredChunkLen)) / BYTE_PER_BLOCK; - - if (nBlocksToInit > BLKS_PER_EXTENT) - nBlocksToInit = BLKS_PER_EXTENT; // don't init > 1 full extent - - if (nBlocksToInit > 0) - { - const uint8_t* emptyVal = fDbFile.getEmptyRowValue( colType, colWidth ); - rc = fDbFile.reInitPartialColumnExtent( pFile, - (chunkPtrs[chunkIndex].first + restoredChunkLen), - nBlocksToInit, - emptyVal, - colWidth ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error clearing HWM column extent from DB for" - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - } - - // Drop off any trailing pointers (that point beyond the last block). - // Watch for the special case where we are restoring a db file as an - // empty file (chunkindex=0 and restoredChunkLen=0); in this case we - // just restore the first pointer (set to 8192). - compress::CompressInterface::setBlockCount(hdrs, - (startOffsetBlk + nBlocks)); - std::vector newPtrs; - - if ((chunkIndex > 0) || (restoredChunkLen > 0)) - { - for (unsigned int i = 0; i <= chunkIndex; i++) - { - newPtrs.push_back( chunkPtrs[i].first ); - } - } - - newPtrs.push_back( chunkPtrs[chunkIndex].first + restoredChunkLen ); - compress::CompressInterface::storePtrs(newPtrs, hdrs); - - rc = fDbFile.writeHeaders( pFile, hdrs ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error writing compressed column headers to DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - - // Finally, we truncate the data base column segment file - rc = fDbFile.truncateFile( pFile, fileSizeBytes ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating compressed column extents from DB for" << - ": OID-" << columnOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - } // end of (chunkIndex < chunkPtrs.size()) - - fDbFile.closeFile( pFile ); + fDbFile.closeFile(pFile); } - + //------------------------------------------------------------------------------ // Load header pointer data for compressed column file. // @@ -490,43 +427,40 @@ void BulkRollbackFileCompressed::reInitTruncColumnExtent( // chunkPtrs - (out) Chunk ptrs extracted from raw header data. // errMsg - (out) Error message if applicable. //------------------------------------------------------------------------------ -int BulkRollbackFileCompressed::loadColumnHdrPtrs( - IDBDataFile* pFile, - char* hdrs, - CompChunkPtrList& chunkPtrs, - std::string& errMsg) const +int BulkRollbackFileCompressed::loadColumnHdrPtrs(IDBDataFile* pFile, char* hdrs, CompChunkPtrList& chunkPtrs, + std::string& errMsg) const { - // Read the header pointers - int rc = fDbFile.readHeaders( pFile, hdrs ); + // Read the header pointers + int rc = fDbFile.readHeaders(pFile, hdrs); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Header read error: " << ec.errorString(rc); - errMsg = oss.str(); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Header read error: " << ec.errorString(rc); + errMsg = oss.str(); - return rc; - } + return rc; + } - // Parse the header pointers - int rc1 = compress::CompressInterface::getPtrList(hdrs, chunkPtrs); + // Parse the header pointers + int rc1 = compress::CompressInterface::getPtrList(hdrs, chunkPtrs); - if (rc1 != 0) - { - rc = ERR_METADATABKUP_COMP_PARSE_HDRS; + if (rc1 != 0) + { + rc = ERR_METADATABKUP_COMP_PARSE_HDRS; - WErrorCodes ec; - std::ostringstream oss; - oss << "Header parsing error (" << rc1 << "): " << ec.errorString(rc); - errMsg = oss.str(); + WErrorCodes ec; + std::ostringstream oss; + oss << "Header parsing error (" << rc1 << "): " << ec.errorString(rc); + errMsg = oss.str(); - return rc; - } + return rc; + } - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Reinitialize a dictionary segment extent (in the db file) to empty blocks, // following the HWM. Remaining extents in the file are truncated. @@ -540,251 +474,213 @@ int BulkRollbackFileCompressed::loadColumnHdrPtrs( // to be reinitialized. Value is in raw data blocks. // nBlocks - Number of blocks to be reinitialized //------------------------------------------------------------------------------ -void BulkRollbackFileCompressed::reInitTruncDctnryExtent( - OID dStoreOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks ) +void BulkRollbackFileCompressed::reInitTruncDctnryExtent(OID dStoreOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long startOffsetBlk, + int nBlocks) { - long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; + long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; - std::ostringstream msgText1; - msgText1 << "Reinit HWM compressed dictionary store extent in db file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; rawOffset(bytes)-" << startOffset << - "; rawFreeBlks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, dStoreOID, msgText1.str() ); + std::ostringstream msgText1; + msgText1 << "Reinit HWM compressed dictionary store extent in db file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawOffset(bytes)-" << startOffset + << "; rawFreeBlks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText1.str()); - std::string segFile; - IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile); + std::string segFile; + IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile); - if (pFile == 0) + if (pFile == 0) + { + std::ostringstream oss; + oss << "Error opening compressed dictionary store segment file to " + "rollback extents from DB for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + + throw WeException(oss.str(), ERR_FILE_OPEN); + } + + char controlHdr[CompressInterface::HDR_BUF_LEN]; + CompChunkPtrList chunkPtrs; + uint64_t ptrHdrSize; + std::string errMsg; + int rc = loadDctnryHdrPtrs(pFile, controlHdr, chunkPtrs, ptrHdrSize, errMsg); + + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error reading compressed dctnry ptr headers from DB for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << errMsg; + + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + + // Locate the chunk containing the last block we intend to keep + unsigned int blockOffset = startOffsetBlk - 1; + unsigned int chunkIndex = 0; + unsigned int blkOffsetInChunk = 0; + + auto fCompressor = compress::getCompressorByType( + fCompressorPool, compress::CompressInterface::getCompressionType(controlHdr)); + if (!fCompressor) + { + std::ostringstream oss; + oss << "Error, wrong compression type for segment file" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << ";"; + throw WeException(oss.str(), ERR_COMP_WRONG_COMP_TYPE); + } + + fCompressor->locateBlock(blockOffset, chunkIndex, blkOffsetInChunk); + + if (chunkIndex < chunkPtrs.size()) + { + // Read backup copy of HWM chunk and restore it's contents + uint64_t restoredChunkLen = 0; + uint64_t restoredFileSize = 0; + rc = restoreHWMChunk(pFile, dStoreOID, partNum, segNum, chunkPtrs[chunkIndex].first, restoredChunkLen, + restoredFileSize, errMsg); + + if (rc == ERR_FILE_NOT_EXIST) { - std::ostringstream oss; - oss << "Error opening compressed dictionary store segment file to " - "rollback extents from DB for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; + std::ostringstream msgText3; + msgText3 << "No restore needed to Compressed dictionary file" + << ": dbRoot-" << dbRoot << "; part#-" << partNum << "; seg#-" << segNum; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText3.str()); - throw WeException( oss.str(), ERR_FILE_OPEN ); + fDbFile.closeFile(pFile); + return; } - char controlHdr[ CompressInterface::HDR_BUF_LEN ]; - CompChunkPtrList chunkPtrs; - uint64_t ptrHdrSize; - std::string errMsg; - int rc = loadDctnryHdrPtrs(pFile, controlHdr, chunkPtrs, ptrHdrSize, errMsg); - if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Error reading compressed dctnry ptr headers from DB for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << errMsg; + std::ostringstream oss; + oss << "Error restoring HWM chunk for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; blkoff-" << blockOffset << "; " << errMsg; - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); } - // Locate the chunk containing the last block we intend to keep - unsigned int blockOffset = startOffsetBlk - 1; - unsigned int chunkIndex = 0; - unsigned int blkOffsetInChunk = 0; + // nBlocks is based on full extents, but if database file only has an + // abbreviated extent, then we reset nBlocks to reflect the file size. + // (Unlike column files which only employ an abbreviated extent for the + // 1st extent in part0, seg0, all store files start with abbrev extent) + bool bAbbreviatedExtent = false; + const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dctnry + long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * PSEUDO_COL_WIDTH; - auto fCompressor = compress::getCompressorByType( - fCompressorPool, - compress::CompressInterface::getCompressionType(controlHdr)); - if (!fCompressor) + if (startOffset <= nBytesInAbbrevExtent) { - std::ostringstream oss; - oss << "Error, wrong compression type for segment file" - << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" - << partNum << "; segment-" << segNum << ";"; - throw WeException(oss.str(), ERR_COMP_WRONG_COMP_TYPE); + nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; + bAbbreviatedExtent = true; } - fCompressor->locateBlock(blockOffset, chunkIndex, blkOffsetInChunk); + long long fileSizeBytes = restoredFileSize; - if (chunkIndex < chunkPtrs.size()) + std::ostringstream msgText2; + msgText2 << "HWM compressed dictionary file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum; + + if (bAbbreviatedExtent) // log adjusted nBlock count for abbrev extent + msgText2 << "; rawFreeBlks-" << nBlocks << " (abbrev)"; + + msgText2 << "; restoredChunk-" << restoredChunkLen << " bytes" + << "; truncated to " << fileSizeBytes << " bytes"; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText2.str()); + + // Initialize the remainder of the extent after the HWM chunk + // Just doing an ftruncate() reinits the file to 0's, which may or may + // not actually reserve disk space if ftruncate is growing the file. + // So reinit the blocks by calling reInitPartialDctnryExtent() to help + // avoid disk fragmentation. Be careful not to init > 1 extent, be- + // cause that is the limit on what that function was intended to do. + const unsigned BLKS_PER_EXTENT = + (BRMWrapper::getInstance()->getExtentRows() * PSEUDO_COL_WIDTH) / BYTE_PER_BLOCK; + long long nBlocksToInit = + (fileSizeBytes - (chunkPtrs[chunkIndex].first + restoredChunkLen)) / BYTE_PER_BLOCK; + + if (nBlocksToInit > BLKS_PER_EXTENT) + nBlocksToInit = BLKS_PER_EXTENT; // don't init > 1 full extent + + if (nBlocksToInit > 0) { - // Read backup copy of HWM chunk and restore it's contents - uint64_t restoredChunkLen = 0; - uint64_t restoredFileSize = 0; - rc = restoreHWMChunk(pFile, dStoreOID, partNum, segNum, - chunkPtrs[chunkIndex].first, - restoredChunkLen, restoredFileSize, errMsg); + rc = fDbFile.reInitPartialDctnryExtent(pFile, (chunkPtrs[chunkIndex].first + restoredChunkLen), + nBlocksToInit, fDctnryHdr, DCTNRY_HEADER_SIZE); - if (rc == ERR_FILE_NOT_EXIST) - { - std::ostringstream msgText3; - msgText3 << "No restore needed to Compressed dictionary file" << - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, dStoreOID, msgText3.str() ); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error clearing HWM dictionary store extent from DB for" + ": OID-" + << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" << segNum + << "; " << ec.errorString(rc); - fDbFile.closeFile( pFile ); - return; - } + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + } - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Error restoring HWM chunk for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; blkoff-" << blockOffset << - "; " << errMsg; + // Drop off any trailing pointers (that point beyond the last block). + // Watch for the special case where we are restoring a db file as an + // empty file (chunkindex=0 and restoredChunkLen=0); in this case we + // just restore the first pointer (set to 8192). + compress::CompressInterface::setBlockCount(controlHdr, (startOffsetBlk + nBlocks)); + std::vector newPtrs; - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } + if ((chunkIndex > 0) || (restoredChunkLen > 0)) + { + for (unsigned int i = 0; i <= chunkIndex; i++) + { + newPtrs.push_back(chunkPtrs[i].first); + } + } - // nBlocks is based on full extents, but if database file only has an - // abbreviated extent, then we reset nBlocks to reflect the file size. - // (Unlike column files which only employ an abbreviated extent for the - // 1st extent in part0, seg0, all store files start with abbrev extent) - bool bAbbreviatedExtent = false; - const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dctnry - long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * - PSEUDO_COL_WIDTH; + newPtrs.push_back(chunkPtrs[chunkIndex].first + restoredChunkLen); + char* pointerHdr = new char[ptrHdrSize]; + compress::CompressInterface::storePtrs(newPtrs, pointerHdr, ptrHdrSize); - if (startOffset <= nBytesInAbbrevExtent) - { - nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK; - bAbbreviatedExtent = true; - } + rc = fDbFile.writeHeaders(pFile, controlHdr, pointerHdr, ptrHdrSize); + delete[] pointerHdr; - long long fileSizeBytes = restoredFileSize; + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error writing compressed dictionary headers to DB for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - std::ostringstream msgText2; - msgText2 << "HWM compressed dictionary file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum; + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } - if (bAbbreviatedExtent) // log adjusted nBlock count for abbrev extent - msgText2 << "; rawFreeBlks-" << nBlocks << " (abbrev)"; + // Finally, we truncate the data base dictionary store segment file + rc = fDbFile.truncateFile(pFile, fileSizeBytes); - msgText2 << "; restoredChunk-" << restoredChunkLen << " bytes" << - "; truncated to " << fileSizeBytes << " bytes"; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, dStoreOID, msgText2.str() ); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error truncating compressed dictionary store extents " + "from DB file for" + << ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum << "; " << ec.errorString(rc); - // Initialize the remainder of the extent after the HWM chunk - // Just doing an ftruncate() reinits the file to 0's, which may or may - // not actually reserve disk space if ftruncate is growing the file. - // So reinit the blocks by calling reInitPartialDctnryExtent() to help - // avoid disk fragmentation. Be careful not to init > 1 extent, be- - // cause that is the limit on what that function was intended to do. - const unsigned BLKS_PER_EXTENT = - (BRMWrapper::getInstance()->getExtentRows() * PSEUDO_COL_WIDTH) / - BYTE_PER_BLOCK; - long long nBlocksToInit = (fileSizeBytes - - (chunkPtrs[chunkIndex].first + restoredChunkLen)) / BYTE_PER_BLOCK; + fDbFile.closeFile(pFile); + throw WeException(oss.str(), rc); + } + } // end of (chunkIndex < chunkPtrs.size()) - if (nBlocksToInit > BLKS_PER_EXTENT) - nBlocksToInit = BLKS_PER_EXTENT; // don't init > 1 full extent - - if (nBlocksToInit > 0) - { - rc = fDbFile.reInitPartialDctnryExtent( pFile, - (chunkPtrs[chunkIndex].first + restoredChunkLen), - nBlocksToInit, - fDctnryHdr, - DCTNRY_HEADER_SIZE ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error clearing HWM dictionary store extent from DB for" - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - } - - // Drop off any trailing pointers (that point beyond the last block). - // Watch for the special case where we are restoring a db file as an - // empty file (chunkindex=0 and restoredChunkLen=0); in this case we - // just restore the first pointer (set to 8192). - compress::CompressInterface::setBlockCount(controlHdr, - (startOffsetBlk + nBlocks)); - std::vector newPtrs; - - if ((chunkIndex > 0) || (restoredChunkLen > 0)) - { - for (unsigned int i = 0; i <= chunkIndex; i++) - { - newPtrs.push_back( chunkPtrs[i].first ); - } - } - - newPtrs.push_back( chunkPtrs[chunkIndex].first + restoredChunkLen ); - char* pointerHdr = new char[ptrHdrSize]; - compress::CompressInterface::storePtrs(newPtrs, pointerHdr, - ptrHdrSize); - - rc = fDbFile.writeHeaders( pFile, controlHdr, pointerHdr, ptrHdrSize ); - delete[] pointerHdr; - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error writing compressed dictionary headers to DB for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - - // Finally, we truncate the data base dictionary store segment file - rc = fDbFile.truncateFile( pFile, fileSizeBytes ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error truncating compressed dictionary store extents " - "from DB file for" << - ": OID-" << dStoreOID << - "; DbRoot-" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << ec.errorString(rc); - - fDbFile.closeFile( pFile ); - throw WeException( oss.str(), rc ); - } - } // end of (chunkIndex < chunkPtrs.size()) - - fDbFile.closeFile( pFile ); + fDbFile.closeFile(pFile); } - + //------------------------------------------------------------------------------ // Load header pointer data for compressed dictionary file. // @@ -794,79 +690,72 @@ void BulkRollbackFileCompressed::reInitTruncDctnryExtent( // ptrHdrSize- (out) Size of pointer header. // errMsg - (out) Error message if applicable. //------------------------------------------------------------------------------ -int BulkRollbackFileCompressed::loadDctnryHdrPtrs( - IDBDataFile* pFile, - char* controlHdr, - CompChunkPtrList& chunkPtrs, - uint64_t& ptrHdrSize, - std::string& errMsg) const +int BulkRollbackFileCompressed::loadDctnryHdrPtrs(IDBDataFile* pFile, char* controlHdr, + CompChunkPtrList& chunkPtrs, uint64_t& ptrHdrSize, + std::string& errMsg) const { - int rc = fDbFile.readFile( - pFile, (unsigned char*)controlHdr, CompressInterface::HDR_BUF_LEN); + int rc = fDbFile.readFile(pFile, (unsigned char*)controlHdr, CompressInterface::HDR_BUF_LEN); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Control header read error: " << ec.errorString(rc); - errMsg = oss.str(); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Control header read error: " << ec.errorString(rc); + errMsg = oss.str(); - return rc; - } + return rc; + } - int rc1 = compress::CompressInterface::verifyHdr(controlHdr); + int rc1 = compress::CompressInterface::verifyHdr(controlHdr); - if (rc1 != 0) - { - rc = ERR_METADATABKUP_COMP_VERIFY_HDRS; + if (rc1 != 0) + { + rc = ERR_METADATABKUP_COMP_VERIFY_HDRS; - WErrorCodes ec; - std::ostringstream oss; - oss << "Control header verify error (" << rc1 << "): " << - ec.errorString(rc); - errMsg = oss.str(); + WErrorCodes ec; + std::ostringstream oss; + oss << "Control header verify error (" << rc1 << "): " << ec.errorString(rc); + errMsg = oss.str(); - return rc; - } + return rc; + } - uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr); - ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN; - char* pointerHdr = new char[ptrHdrSize]; + uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr); + ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN; + char* pointerHdr = new char[ptrHdrSize]; - rc = fDbFile.readFile(pFile, (unsigned char*)pointerHdr, ptrHdrSize); + rc = fDbFile.readFile(pFile, (unsigned char*)pointerHdr, ptrHdrSize); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Pointer header read error: " << ec.errorString(rc); - errMsg = oss.str(); - delete[] pointerHdr; - - return rc; - } - - // Parse the header pointers - rc1 = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize, - chunkPtrs); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Pointer header read error: " << ec.errorString(rc); + errMsg = oss.str(); delete[] pointerHdr; - if (rc1 != 0) - { - rc = ERR_METADATABKUP_COMP_PARSE_HDRS; + return rc; + } - WErrorCodes ec; - std::ostringstream oss; - oss << "Pointer header parsing error (" << rc1 << "): " << - ec.errorString(rc); - errMsg = oss.str(); + // Parse the header pointers + rc1 = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize, chunkPtrs); + delete[] pointerHdr; - return rc; - } + if (rc1 != 0) + { + rc = ERR_METADATABKUP_COMP_PARSE_HDRS; - return NO_ERROR; + WErrorCodes ec; + std::ostringstream oss; + oss << "Pointer header parsing error (" << rc1 << "): " << ec.errorString(rc); + errMsg = oss.str(); + + return rc; + } + + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Restore the HWM chunk back to the contents saved in the backup file. // @@ -880,168 +769,150 @@ int BulkRollbackFileCompressed::loadDctnryHdrPtrs( // restoredFileSize (out) - Size of file (in bytes) when backup was made // errMsg (out) - Error msg if error returned //------------------------------------------------------------------------------ -int BulkRollbackFileCompressed::restoreHWMChunk( - IDBDataFile* pFile, - OID columnOID, - uint32_t partNum, - uint32_t segNum, - uint64_t fileOffsetByteForRestoredChunk, - uint64_t& restoredChunkLen, - uint64_t& restoredFileSize, - std::string& errMsg) +int BulkRollbackFileCompressed::restoreHWMChunk(IDBDataFile* pFile, OID columnOID, uint32_t partNum, + uint32_t segNum, uint64_t fileOffsetByteForRestoredChunk, + uint64_t& restoredChunkLen, uint64_t& restoredFileSize, + std::string& errMsg) { - restoredChunkLen = 0; - restoredFileSize = 0; + restoredChunkLen = 0; + restoredFileSize = 0; - // Open the backup HWM chunk file + // Open the backup HWM chunk file + std::ostringstream oss; + oss << "/" << columnOID << ".p" << partNum << ".s" << segNum; + std::string bulkRollbackSubPath(fMgr->getMetaFileName()); + bulkRollbackSubPath += DATA_DIR_SUFFIX; + bulkRollbackSubPath += oss.str(); + + if (!IDBPolicy::exists(bulkRollbackSubPath.c_str())) + { std::ostringstream oss; - oss << "/" << columnOID << ".p" << partNum << ".s" << segNum; - std::string bulkRollbackSubPath( fMgr->getMetaFileName() ); - bulkRollbackSubPath += DATA_DIR_SUFFIX; - bulkRollbackSubPath += oss.str(); + oss << "Backup file does not exist: " << bulkRollbackSubPath; + errMsg = oss.str(); - if ( !IDBPolicy::exists( bulkRollbackSubPath.c_str() ) ) + return ERR_FILE_NOT_EXIST; + } + + IDBDataFile* backupFile = + IDBDataFile::open(IDBPolicy::getType(bulkRollbackSubPath.c_str(), IDBPolicy::WRITEENG), + bulkRollbackSubPath.c_str(), "rb", 0, pFile->colWidth()); + + if (!backupFile) + { + int errrc = errno; + + std::string eMsg; + Convertor::mapErrnoToString(errrc, eMsg); + std::ostringstream oss; + oss << "Error opening backup file " << bulkRollbackSubPath << "; " << eMsg; + errMsg = oss.str(); + + return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP; + } + + // Read the chunk length and file size + uint64_t sizeHdr[2]; + size_t bytesRead = readFillBuffer(backupFile, (char*)sizeHdr, sizeof(uint64_t) * 2); + + if (bytesRead != sizeof(uint64_t) * 2) + { + int errrc = errno; + + std::string eMsg; + Convertor::mapErrnoToString(errrc, eMsg); + std::ostringstream oss; + oss << "Error reading chunk length from backup file " << bulkRollbackSubPath << "; " << eMsg; + errMsg = oss.str(); + + delete backupFile; + return ERR_METADATABKUP_COMP_READ_BULK_BKUP; + } + + restoredChunkLen = sizeHdr[0]; + restoredFileSize = sizeHdr[1]; + + // Position the destination offset in the DB file + int rc = fDbFile.setFileOffset(pFile, fileOffsetByteForRestoredChunk, SEEK_SET); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error setting column file offset" + << "; offset-" << fileOffsetByteForRestoredChunk << "; " << ec.errorString(rc); + errMsg = oss.str(); + + delete backupFile; + return rc; + } + + // Copy backup version of chunk back to DB, unless chunk length is 0 + // in which case we have nothing to copy. + if (restoredChunkLen > 0) + { + // Read the HWM chunk to be restored + unsigned char* chunk = new unsigned char[restoredChunkLen]; + boost::scoped_array scopedChunk(chunk); + bytesRead = readFillBuffer(backupFile, (char*)chunk, restoredChunkLen); + + if (bytesRead != restoredChunkLen) { - std::ostringstream oss; - oss << "Backup file does not exist: " << bulkRollbackSubPath; - errMsg = oss.str(); + int errrc = errno; - return ERR_FILE_NOT_EXIST; + std::string eMsg; + Convertor::mapErrnoToString(errrc, eMsg); + std::ostringstream oss; + oss << "Error reading chunk data from backup file " << bulkRollbackSubPath << "; size-" + << restoredChunkLen << ": " << eMsg; + errMsg = oss.str(); + + delete backupFile; + return ERR_METADATABKUP_COMP_READ_BULK_BKUP; } - IDBDataFile* backupFile = IDBDataFile::open( - IDBPolicy::getType( bulkRollbackSubPath.c_str(), IDBPolicy::WRITEENG ), - bulkRollbackSubPath.c_str(), - "rb", - 0, - pFile->colWidth() ); - - if (!backupFile) - { - int errrc = errno; - - std::string eMsg; - Convertor::mapErrnoToString(errrc, eMsg); - std::ostringstream oss; - oss << "Error opening backup file " << - bulkRollbackSubPath << "; " << eMsg; - errMsg = oss.str(); - - return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP; - } - - // Read the chunk length and file size - uint64_t sizeHdr[2]; - size_t bytesRead = readFillBuffer(backupFile, (char*)sizeHdr, - sizeof(uint64_t) * 2); - - if (bytesRead != sizeof(uint64_t) * 2) - { - int errrc = errno; - - std::string eMsg; - Convertor::mapErrnoToString(errrc, eMsg); - std::ostringstream oss; - oss << "Error reading chunk length from backup file " << - bulkRollbackSubPath << "; " << eMsg; - errMsg = oss.str(); - - delete backupFile; - return ERR_METADATABKUP_COMP_READ_BULK_BKUP; - } - - restoredChunkLen = sizeHdr[0]; - restoredFileSize = sizeHdr[1]; - - // Position the destination offset in the DB file - int rc = fDbFile.setFileOffset(pFile, fileOffsetByteForRestoredChunk, - SEEK_SET); + // Write/restore the HWM chunk to the applicable database file + rc = fDbFile.writeFile(pFile, chunk, restoredChunkLen); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error setting column file offset" << - "; offset-" << fileOffsetByteForRestoredChunk << - "; " << ec.errorString(rc); - errMsg = oss.str(); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error writing to column file" + << "; offset-" << fileOffsetByteForRestoredChunk << "; bytes-" << restoredChunkLen << "; " + << ec.errorString(rc); + errMsg = oss.str(); - delete backupFile; - return rc; + delete backupFile; + return rc; } + } - // Copy backup version of chunk back to DB, unless chunk length is 0 - // in which case we have nothing to copy. - if (restoredChunkLen > 0) - { - // Read the HWM chunk to be restored - unsigned char* chunk = new unsigned char[restoredChunkLen]; - boost::scoped_array scopedChunk( chunk ); - bytesRead = readFillBuffer(backupFile, (char*)chunk, restoredChunkLen); + delete backupFile; - if (bytesRead != restoredChunkLen) - { - int errrc = errno; - - std::string eMsg; - Convertor::mapErrnoToString(errrc, eMsg); - std::ostringstream oss; - oss << "Error reading chunk data from backup file " << - bulkRollbackSubPath << - "; size-" << restoredChunkLen << - ": " << eMsg; - errMsg = oss.str(); - - delete backupFile; - return ERR_METADATABKUP_COMP_READ_BULK_BKUP; - } - - // Write/restore the HWM chunk to the applicable database file - rc = fDbFile.writeFile(pFile, chunk, restoredChunkLen); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error writing to column file" << - "; offset-" << fileOffsetByteForRestoredChunk << - "; bytes-" << restoredChunkLen << - "; " << ec.errorString(rc); - errMsg = oss.str(); - - delete backupFile; - return rc; - } - } - - delete backupFile; - - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Return true/false depending on whether the applicable backup chunk file can // be found to restore a backed up compressed chunk back into a db file. If // the backup file is not found, we assume that it's because one was not created // and thus not needed. //------------------------------------------------------------------------------ -bool BulkRollbackFileCompressed::doWeReInitExtent( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) const +bool BulkRollbackFileCompressed::doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum) const { - std::ostringstream oss; - oss << "/" << columnOID << ".p" << partNum << ".s" << segNum; - std::string bulkRollbackSubPath( fMgr->getMetaFileName() ); - bulkRollbackSubPath += DATA_DIR_SUFFIX; - bulkRollbackSubPath += oss.str(); + std::ostringstream oss; + oss << "/" << columnOID << ".p" << partNum << ".s" << segNum; + std::string bulkRollbackSubPath(fMgr->getMetaFileName()); + bulkRollbackSubPath += DATA_DIR_SUFFIX; + bulkRollbackSubPath += oss.str(); - if ( !IDBPolicy::exists( bulkRollbackSubPath.c_str() ) ) - { - return false; - } + if (!IDBPolicy::exists(bulkRollbackSubPath.c_str())) + { + return false; + } - return true; + return true; } //------------------------------------------------------------------------------ @@ -1049,32 +920,29 @@ bool BulkRollbackFileCompressed::doWeReInitExtent( OID columnOID, // Added this function as part of hdfs port, because IDBDataFile::read() // may not return all the requested data in the first call to read(). //------------------------------------------------------------------------------ -size_t BulkRollbackFileCompressed::readFillBuffer( - IDBDataFile* pFile, - char* buffer, - size_t bytesReq) const +size_t BulkRollbackFileCompressed::readFillBuffer(IDBDataFile* pFile, char* buffer, size_t bytesReq) const { - char* pBuf = buffer; - ssize_t nBytes; - size_t bytesToRead = bytesReq; - size_t totalBytesRead = 0; + char* pBuf = buffer; + ssize_t nBytes; + size_t bytesToRead = bytesReq; + size_t totalBytesRead = 0; - while (1) - { - nBytes = pFile->read(pBuf, bytesToRead); + while (1) + { + nBytes = pFile->read(pBuf, bytesToRead); - if (nBytes > 0) - totalBytesRead += nBytes; - else - break; + if (nBytes > 0) + totalBytesRead += nBytes; + else + break; - if ((size_t)nBytes == bytesToRead) - break; + if ((size_t)nBytes == bytesToRead) + break; - pBuf += nBytes; - bytesToRead = bytesToRead - (size_t)nBytes; - } + pBuf += nBytes; + bytesToRead = bytesToRead - (size_t)nBytes; + } - return totalBytesRead; + return totalBytesRead; } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_bulkrollbackfilecompressed.h b/writeengine/shared/we_bulkrollbackfilecompressed.h index ab9e8439c..2d44f25bc 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressed.h +++ b/writeengine/shared/we_bulkrollbackfilecompressed.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_bulkrollbackfilecompressed.h 4726 2013-08-07 03:38:36Z bwilkinson $ -*/ + * $Id: we_bulkrollbackfilecompressed.h 4726 2013-08-07 03:38:36Z bwilkinson $ + */ /** @file * Contains class to restore compressed db files on behalf of BulkRollBackMgr. @@ -46,112 +46,84 @@ class BulkRollbackMgr; //------------------------------------------------------------------------------ class BulkRollbackFileCompressed : public BulkRollbackFile { -public: + public: + /** @brief BulkRollbackFile constructor + * @param mgr The controlling BulkRollbackMgr object. + */ + BulkRollbackFileCompressed(BulkRollbackMgr* mgr); - /** @brief BulkRollbackFile constructor - * @param mgr The controlling BulkRollbackMgr object. - */ - BulkRollbackFileCompressed(BulkRollbackMgr* mgr); + /** @brief BulkRollbackFile destructor + */ + virtual ~BulkRollbackFileCompressed(); - /** @brief BulkRollbackFile destructor - */ - virtual ~BulkRollbackFileCompressed(); + /** @brief Do we reinit trailing blocks in the HWM extent for the specified + * segment file + * + * @param columnOID OID of the segment file in question + * @param dbRoot DBRoot for the segment file in question + * @param partNum Partition number for the segment file in question + * @param segNum Segment number for the segment file in question + */ + virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const; - /** @brief Do we reinit trailing blocks in the HWM extent for the specified - * segment file - * - * @param columnOID OID of the segment file in question - * @param dbRoot DBRoot for the segment file in question - * @param partNum Partition number for the segment file in question - * @param segNum Segment number for the segment file in question - */ - virtual bool doWeReInitExtent( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) const; + /** @brief Reinitialize the specified column segment file starting at + * startOffsetBlk, and truncate trailing extents. + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param startOffsetBlk Starting block offset where file is to be + * reinitialized + * @param nBlocks Number of blocks to be reinitialized + * @param colType Column type of the relevant segment file + * @param colWidth Width in bytes of column. + * @param restoreHwmChk Restore HWM chunk + */ + virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks, + execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, + bool restoreHwmChk); - /** @brief Reinitialize the specified column segment file starting at - * startOffsetBlk, and truncate trailing extents. - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param startOffsetBlk Starting block offset where file is to be - * reinitialized - * @param nBlocks Number of blocks to be reinitialized - * @param colType Column type of the relevant segment file - * @param colWidth Width in bytes of column. - * @param restoreHwmChk Restore HWM chunk - */ - virtual void reInitTruncColumnExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks, - execplan::CalpontSystemCatalog::ColDataType colType, - uint32_t colWidth, - bool restoreHwmChk ); + /** @brief Reinitialize the specified dictionary store segment file starting + * at startOffsetBlk, and truncate trailing extents. + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param startOffsetBlk Starting block offset where file is to be + * reinitialized + * @param nBlocks Number of blocks to be reinitialized + */ + virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks); - /** @brief Reinitialize the specified dictionary store segment file starting - * at startOffsetBlk, and truncate trailing extents. - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param startOffsetBlk Starting block offset where file is to be - * reinitialized - * @param nBlocks Number of blocks to be reinitialized - */ - virtual void reInitTruncDctnryExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks ); + /** @brief Truncate the specified segment file to a specified num of bytes + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param fileSizeBlocks Number of blocks to retain in the file + */ + virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long filesSizeBlocks); - /** @brief Truncate the specified segment file to a specified num of bytes - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param fileSizeBlocks Number of blocks to retain in the file - */ - virtual void truncateSegmentFile( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long filesSizeBlocks ); + private: + // Disable unnecessary copy constructor and assignment operator + BulkRollbackFileCompressed(const BulkRollbackFileCompressed& rhs); + BulkRollbackFileCompressed& operator=(const BulkRollbackFileCompressed& rhs); -private: - // Disable unnecessary copy constructor and assignment operator - BulkRollbackFileCompressed(const BulkRollbackFileCompressed& rhs); - BulkRollbackFileCompressed& operator=(const BulkRollbackFileCompressed& rhs); + size_t readFillBuffer(IDBDataFile* pFile, char* buffer, size_t bytesReq) const; + int restoreHWMChunk(IDBDataFile* pFile, OID columnOID, uint32_t partNum, uint32_t segNum, + uint64_t fileOffsetByteForRestoredChunk, uint64_t& restoredChunkLen, + uint64_t& restoredFileSize, std::string& errMsg); + int loadColumnHdrPtrs(IDBDataFile* pFile, char* hdrs, compress::CompChunkPtrList& chunkPtrs, + std::string& errMsg) const; + int loadDctnryHdrPtrs(IDBDataFile* pFile, char* controlHdr, compress::CompChunkPtrList& chunkPtrs, + uint64_t& ptrHdrSize, std::string& errMsg) const; - size_t readFillBuffer ( IDBDataFile* pFile, - char* buffer, - size_t bytesReq) const; - int restoreHWMChunk ( IDBDataFile* pFile, - OID columnOID, - uint32_t partNum, - uint32_t segNum, - uint64_t fileOffsetByteForRestoredChunk, - uint64_t& restoredChunkLen, - uint64_t& restoredFileSize, - std::string& errMsg ); - int loadColumnHdrPtrs ( IDBDataFile* pFile, - char* hdrs, - compress::CompChunkPtrList& chunkPtrs, - std::string& errMsg) const; - int loadDctnryHdrPtrs ( IDBDataFile* pFile, - char* controlHdr, - compress::CompChunkPtrList& chunkPtrs, - uint64_t& ptrHdrSize, - std::string& errMsg ) const; - - compress::CompressorPool fCompressorPool; + compress::CompressorPool fCompressorPool; }; -} //end of namespace +} // namespace WriteEngine -#endif // WE_BULKROLLBACKFILECOMPRESSED_H_ +#endif // WE_BULKROLLBACKFILECOMPRESSED_H_ diff --git a/writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp b/writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp index c85bfbfad..61f8b61f7 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp +++ b/writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp @@ -38,17 +38,14 @@ namespace { const char* DATA_DIR_SUFFIX = "_data"; const char* OLD_FILE_SUFFIX = ".old_bulk"; -} +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // BulkRollbackFileCompressedHdfs constructor //------------------------------------------------------------------------------ -BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs( - BulkRollbackMgr* mgr) : - BulkRollbackFile(mgr) +BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr) : BulkRollbackFile(mgr) { } @@ -58,7 +55,7 @@ BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs( BulkRollbackFileCompressedHdfs::~BulkRollbackFileCompressedHdfs() { } - + //------------------------------------------------------------------------------ // Truncate the specified database segment file to the extent specified by // the given file offset. Also updates the header(s) as well. @@ -70,25 +67,18 @@ BulkRollbackFileCompressedHdfs::~BulkRollbackFileCompressedHdfs() // fileSizeBlocks - Number of raw data blocks to be left in the file. // Remainder of file is to be truncated. //------------------------------------------------------------------------------ -void BulkRollbackFileCompressedHdfs::truncateSegmentFile( - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long fileSizeBlocks ) +void BulkRollbackFileCompressedHdfs::truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long fileSizeBlocks) { - std::ostringstream msgText; - msgText << "Truncating compressed HDFS column file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; rawTotBlks-" << fileSizeBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Truncating compressed HDFS column file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawTotBlks-" << fileSizeBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str()); - restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum ); + restoreFromBackup("column", columnOID, dbRoot, partNum, segNum); } - + //------------------------------------------------------------------------------ // Reinitialize a column segment extent (in the db file) to empty values, // following the HWM. Remaining extents in the file are truncated. @@ -105,32 +95,23 @@ void BulkRollbackFileCompressedHdfs::truncateSegmentFile( // colWidth - Width in bytes, of the applicable column // restoreHwmChk - Specifies whether HWM chunk is to be restored. //------------------------------------------------------------------------------ -void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent( - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks, - CalpontSystemCatalog::ColDataType colType, - uint32_t colWidth, - bool restoreHwmChk ) +void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long startOffsetBlk, + int nBlocks, + CalpontSystemCatalog::ColDataType colType, + uint32_t colWidth, bool restoreHwmChk) { - long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; + long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; - std::ostringstream msgText; - msgText << "Reinit HWM compressed column extent in HDFS db file" << - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; rawOffset(bytes)-" << startOffset << - "; rawFreeBlks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, columnOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Reinit HWM compressed column extent in HDFS db file" + << ": dbRoot-" << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawOffset(bytes)-" + << startOffset << "; rawFreeBlks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str()); - restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum ); + restoreFromBackup("column", columnOID, dbRoot, partNum, segNum); } - + //------------------------------------------------------------------------------ // Reinitialize a dictionary segment extent (in the db file) to empty blocks, // following the HWM. Remaining extents in the file are truncated. @@ -144,42 +125,33 @@ void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent( // to be reinitialized. Value is in raw data blocks. // nBlocks - Number of blocks to be reinitialized //------------------------------------------------------------------------------ -void BulkRollbackFileCompressedHdfs::reInitTruncDctnryExtent( - OID dStoreOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks ) +void BulkRollbackFileCompressedHdfs::reInitTruncDctnryExtent(OID dStoreOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, long long startOffsetBlk, + int nBlocks) { - long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; + long long startOffset = startOffsetBlk * BYTE_PER_BLOCK; - std::ostringstream msgText; - msgText << "Reinit HWM compressed dictionary store extent in HDFS db file" - ": dbRoot-" << dbRoot << - "; part#-" << partNum << - "; seg#-" << segNum << - "; rawOffset(bytes)-" << startOffset << - "; rawFreeBlks-" << nBlocks; - fMgr->logAMessage( logging::LOG_TYPE_INFO, - logging::M0075, dStoreOID, msgText.str() ); + std::ostringstream msgText; + msgText << "Reinit HWM compressed dictionary store extent in HDFS db file" + ": dbRoot-" + << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawOffset(bytes)-" << startOffset + << "; rawFreeBlks-" << nBlocks; + fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText.str()); - restoreFromBackup( "dictionary store", dStoreOID, dbRoot, partNum, segNum ); + restoreFromBackup("dictionary store", dStoreOID, dbRoot, partNum, segNum); } - + //------------------------------------------------------------------------------ // For HDFS system, just always return true. // Let ConfirmHdfsDbFile later determine when/if/how to restore from any // existing backup file. //------------------------------------------------------------------------------ -bool BulkRollbackFileCompressedHdfs::doWeReInitExtent( OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) const +bool BulkRollbackFileCompressedHdfs::doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum) const { - return true; + return true; } - + //------------------------------------------------------------------------------ // Replace the currently specified db file with it's corresponding backup file. // The backup file is a complete backup, not just a backup of a single chunk. @@ -193,96 +165,79 @@ bool BulkRollbackFileCompressedHdfs::doWeReInitExtent( OID columnOID, // in case the user did not upgrade cleanly, and we have to restore using an // old leftover backup file. //------------------------------------------------------------------------------ -void BulkRollbackFileCompressedHdfs::restoreFromBackup(const char* colType, - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) +void BulkRollbackFileCompressedHdfs::restoreFromBackup(const char* colType, OID columnOID, uint32_t dbRoot, + uint32_t partNum, uint32_t segNum) { - // Construct file name for db file to be restored - char dbFileName[FILE_NAME_SIZE]; - int rc = fDbFile.getFileName( columnOID, dbFileName, - dbRoot, partNum, segNum ); + // Construct file name for db file to be restored + char dbFileName[FILE_NAME_SIZE]; + int rc = fDbFile.getFileName(columnOID, dbFileName, dbRoot, partNum, segNum); - if (rc != NO_ERROR) + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error restoring " << colType << " HDFS file for OID " << columnOID + << "; Can't construct file name for DBRoot" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + throw WeException(oss.str(), rc); + } + + // Construct file name for backup copy of db file + std::ostringstream ossFile; + ossFile << "/" << columnOID << ".p" << partNum << ".s" << segNum; + std::string backupFileName(fMgr->getMetaFileName()); + backupFileName += DATA_DIR_SUFFIX; + backupFileName += ossFile.str(); + + std::string dbFileNameTmp = dbFileName; + dbFileNameTmp += OLD_FILE_SUFFIX; + + // For backwards compatibility... + // Restore from backup file used in initial HDFS release, in case the user + // upgraded without going down cleanly. In that case we might need to + // rollback using an old backup file left from previous release. + if (IDBPolicy::exists(backupFileName.c_str())) + { + // Rename current db file to make room for restored file + rc = IDBPolicy::rename(dbFileName, dbFileNameTmp.c_str()); + + if (rc != 0) { - std::ostringstream oss; - oss << "Error restoring " << colType << - " HDFS file for OID " << columnOID << - "; Can't construct file name for DBRoot" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; - throw WeException( oss.str(), rc ); + std::ostringstream oss; + oss << "Error restoring " << colType << " HDFS file for OID " << columnOID + << "; Can't move old file for DBRoot" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + throw WeException(oss.str(), ERR_COMP_RENAME_FILE); } - // Construct file name for backup copy of db file - std::ostringstream ossFile; - ossFile << "/" << columnOID << ".p" << partNum << ".s" << segNum; - std::string backupFileName( fMgr->getMetaFileName() ); - backupFileName += DATA_DIR_SUFFIX; - backupFileName += ossFile.str(); + // Rename backup file to replace current db file + rc = IDBPolicy::rename(backupFileName.c_str(), dbFileName); - std::string dbFileNameTmp = dbFileName; - dbFileNameTmp += OLD_FILE_SUFFIX; - - // For backwards compatibility... - // Restore from backup file used in initial HDFS release, in case the user - // upgraded without going down cleanly. In that case we might need to - // rollback using an old backup file left from previous release. - if ( IDBPolicy::exists(backupFileName.c_str()) ) + if (rc != 0) { - // Rename current db file to make room for restored file - rc = IDBPolicy::rename( dbFileName, dbFileNameTmp.c_str() ); - - if (rc != 0) - { - std::ostringstream oss; - oss << "Error restoring " << colType << - " HDFS file for OID " << columnOID << - "; Can't move old file for DBRoot" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; - throw WeException( oss.str(), ERR_COMP_RENAME_FILE ); - } - - // Rename backup file to replace current db file - rc = IDBPolicy::rename( backupFileName.c_str(), dbFileName ); - - if (rc != 0) - { - std::ostringstream oss; - oss << "Error restoring " << colType << - " HDFS file for OID " << columnOID << - "; Can't rename backup file for DBRoot" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum; - throw WeException( oss.str(), ERR_METADATABKUP_COMP_RENAME ); - } - - // Delete db file we just replaced with backup - IDBPolicy::remove( dbFileNameTmp.c_str() ); + std::ostringstream oss; + oss << "Error restoring " << colType << " HDFS file for OID " << columnOID + << "; Can't rename backup file for DBRoot" << dbRoot << "; partition-" << partNum << "; segment-" + << segNum; + throw WeException(oss.str(), ERR_METADATABKUP_COMP_RENAME); } - else // Restore from HDFS temp swap backup file; This is the normal case + + // Delete db file we just replaced with backup + IDBPolicy::remove(dbFileNameTmp.c_str()); + } + else // Restore from HDFS temp swap backup file; This is the normal case + { + std::string errMsg; + ConfirmHdfsDbFile confirmHdfs; + rc = confirmHdfs.endDbFileChange(std::string("tmp"), dbFileName, false, errMsg); + + if (rc != 0) { - std::string errMsg; - ConfirmHdfsDbFile confirmHdfs; - rc = confirmHdfs.endDbFileChange( std::string("tmp"), - dbFileName, - false, - errMsg); - - if (rc != 0) - { - std::ostringstream oss; - oss << "Error restoring " << colType << - " HDFS file for OID " << columnOID << - "; DBRoot" << dbRoot << - "; partition-" << partNum << - "; segment-" << segNum << - "; " << errMsg; - throw WeException( oss.str(), rc ); - } + std::ostringstream oss; + oss << "Error restoring " << colType << " HDFS file for OID " << columnOID << "; DBRoot" << dbRoot + << "; partition-" << partNum << "; segment-" << segNum << "; " << errMsg; + throw WeException(oss.str(), rc); } + } } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h b/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h index fc90e4b99..d254bc3a0 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h +++ b/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h @@ -44,96 +44,76 @@ class BulkRollbackMgr; //------------------------------------------------------------------------------ class BulkRollbackFileCompressedHdfs : public BulkRollbackFile { -public: + public: + /** @brief BulkRollbackFile constructor + * @param mgr The controlling BulkRollbackMgr object. + */ + BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr); - /** @brief BulkRollbackFile constructor - * @param mgr The controlling BulkRollbackMgr object. - */ - BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr); + /** @brief BulkRollbackFile destructor + */ + virtual ~BulkRollbackFileCompressedHdfs(); - /** @brief BulkRollbackFile destructor - */ - virtual ~BulkRollbackFileCompressedHdfs(); + /** @brief Do we reinit trailing blocks in the HWM extent for the specified + * segment file + * + * @param columnOID OID of the segment file in question + * @param dbRoot DBRoot for the segment file in question + * @param partNum Partition number for the segment file in question + * @param segNum Segment number for the segment file in question + */ + virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const; - /** @brief Do we reinit trailing blocks in the HWM extent for the specified - * segment file - * - * @param columnOID OID of the segment file in question - * @param dbRoot DBRoot for the segment file in question - * @param partNum Partition number for the segment file in question - * @param segNum Segment number for the segment file in question - */ - virtual bool doWeReInitExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum) const; + /** @brief Reinitialize the specified column segment file starting at + * startOffsetBlk, and truncate trailing extents. + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param startOffsetBlk Starting block offset where file is to be + * reinitialized + * @param nBlocks Number of blocks to be reinitialized + * @param colType Column type of the relevant segment file + * @param colWidth Width in bytes of column. + * @param restoreHwmChk Restore HWM chunk + */ + virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks, + execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, + bool restoreHwmChk); - /** @brief Reinitialize the specified column segment file starting at - * startOffsetBlk, and truncate trailing extents. - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param startOffsetBlk Starting block offset where file is to be - * reinitialized - * @param nBlocks Number of blocks to be reinitialized - * @param colType Column type of the relevant segment file - * @param colWidth Width in bytes of column. - * @param restoreHwmChk Restore HWM chunk - */ - virtual void reInitTruncColumnExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks, - execplan::CalpontSystemCatalog::ColDataType colType, - uint32_t colWidth, - bool restoreHwmChk ); + /** @brief Reinitialize the specified dictionary store segment file starting + * at startOffsetBlk, and truncate trailing extents. + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param startOffsetBlk Starting block offset where file is to be + * reinitialized + * @param nBlocks Number of blocks to be reinitialized + */ + virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks); - /** @brief Reinitialize the specified dictionary store segment file starting - * at startOffsetBlk, and truncate trailing extents. - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param startOffsetBlk Starting block offset where file is to be - * reinitialized - * @param nBlocks Number of blocks to be reinitialized - */ - virtual void reInitTruncDctnryExtent(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long startOffsetBlk, - int nBlocks ); + /** @brief Truncate the specified segment file to a specified num of bytes + * @param columnOID OID of the relevant segment file + * @param dbRoot DBRoot of the relevant segment file + * @param partNum Partition number of the relevant segment file + * @param segNum Segment number of the relevant segment file + * @param fileSizeBlocks Number of blocks to retain in the file + */ + virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long filesSizeBlocks); - /** @brief Truncate the specified segment file to a specified num of bytes - * @param columnOID OID of the relevant segment file - * @param dbRoot DBRoot of the relevant segment file - * @param partNum Partition number of the relevant segment file - * @param segNum Segment number of the relevant segment file - * @param fileSizeBlocks Number of blocks to retain in the file - */ - virtual void truncateSegmentFile(OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - long long filesSizeBlocks ); + private: + // Disable unnecessary copy constructor and assignment operator + BulkRollbackFileCompressedHdfs(const BulkRollbackFileCompressedHdfs& rhs); + BulkRollbackFileCompressedHdfs& operator=(const BulkRollbackFileCompressedHdfs& rhs); -private: - // Disable unnecessary copy constructor and assignment operator - BulkRollbackFileCompressedHdfs(const BulkRollbackFileCompressedHdfs& rhs); - BulkRollbackFileCompressedHdfs& operator=( - const BulkRollbackFileCompressedHdfs& rhs); - - void restoreFromBackup(const char* colType, - OID columnOID, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum ); + void restoreFromBackup(const char* colType, OID columnOID, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum); }; -} //end of namespace +} // namespace WriteEngine -#endif // WE_BULKROLLBACKFILECOMPRESSEDHDFS_H_ +#endif // WE_BULKROLLBACKFILECOMPRESSEDHDFS_H_ diff --git a/writeengine/shared/we_bulkrollbackmgr.cpp b/writeengine/shared/we_bulkrollbackmgr.cpp index e9872da50..0fc93729f 100644 --- a/writeengine/shared/we_bulkrollbackmgr.cpp +++ b/writeengine/shared/we_bulkrollbackmgr.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_bulkrollbackmgr.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -*/ + * $Id: we_bulkrollbackmgr.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + */ #include #include @@ -60,39 +60,36 @@ const char* TMP_FILE_SUFFIX = ".tmp"; const int BUF_SIZE = 1024; // size of buffer used to read meta data records -const std::string DB_FILE_PREFIX ("FILE"); +const std::string DB_FILE_PREFIX("FILE"); const std::string DB_FILE_EXTENSION(".cdf"); const std::string DB_FILE_EXTENSION_ORIG(".orig"); -const std::string DB_FILE_EXTENSION_TMP (".tmp" ); -} +const std::string DB_FILE_EXTENSION_TMP(".tmp"); +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // BulkRollbackMgr constructor // // tableOID - OID of the table to be rolled back. //------------------------------------------------------------------------------ -BulkRollbackMgr::BulkRollbackMgr ( OID tableOID, - uint64_t lockID, - const std::string& tableName, - const std::string& applName, Log* logger ) : - fTableOID(tableOID), - fLockID(lockID), - fTableName(tableName), - fProcessId(0), - fMetaFile(NULL), - fPendingDctnryStoreOID(0), - fPendingDctnryStoreDbRoot(0), - fSysLogger( logging::LoggingID( SUBSYSTEM_ID_WE ) ), - fDebugConsole( false ), - fLog(logger), - fApplName(applName), - fVersion(4) +BulkRollbackMgr::BulkRollbackMgr(OID tableOID, uint64_t lockID, const std::string& tableName, + const std::string& applName, Log* logger) + : fTableOID(tableOID) + , fLockID(lockID) + , fTableName(tableName) + , fProcessId(0) + , fMetaFile(NULL) + , fPendingDctnryStoreOID(0) + , fPendingDctnryStoreDbRoot(0) + , fSysLogger(logging::LoggingID(SUBSYSTEM_ID_WE)) + , fDebugConsole(false) + , fLog(logger) + , fApplName(applName) + , fVersion(4) { } - + //------------------------------------------------------------------------------ // Rolls back the state of the extentmap and database files for the table OID // specified to the constructor, using the previously written meta-data file. @@ -104,169 +101,160 @@ BulkRollbackMgr::BulkRollbackMgr ( OID tableOID, // returns: // NO_ERROR if rollback completed successfully //------------------------------------------------------------------------------ -int BulkRollbackMgr::rollback ( bool keepMetaFile ) +int BulkRollbackMgr::rollback(bool keepMetaFile) { - logAMessage( logging::LOG_TYPE_INFO, - logging::M0084, 0, fApplName ); + logAMessage(logging::LOG_TYPE_INFO, logging::M0084, 0, fApplName); - int rc = NO_ERROR; - int dbRootRollbackCount = 0; + int rc = NO_ERROR; + int dbRootRollbackCount = 0; - try + try + { + // validate that BRM is in read/write mode so we can update extentmap + rc = BRMWrapper::getInstance()->isReadWrite(); + + if (rc != NO_ERROR) { - // validate that BRM is in read/write mode so we can update extentmap - rc = BRMWrapper::getInstance()->isReadWrite(); + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback for table " << fTableName << " (OID-" << fTableOID << ") not performed; " + << ec.errorString(rc); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback for table " << fTableName << " (OID-" << - fTableOID << ") not performed; " << ec.errorString(rc); - - throw WeException( oss.str(), rc ); - } - - std::vector dbRoots; - Config::getRootIdList( dbRoots ); - - std::string emptyText0072; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0072, 0, emptyText0072 ); - - // Loop through DBRoots for this PM - for (unsigned m = 0; m < dbRoots.size(); m++) - { - std::istringstream metaDataStream; - bool bPerformRollback = openMetaDataFile ( dbRoots[m], - metaDataStream ); - - // Call function to: - // 1. read bulk rollback meta-data file - // 2. rollback applicable extents from extentmap - // 3. delete applicable extents from database files - // 4. reinitialize blocks trailing the HWM block in the last extent - // of each segment file - // ... - if (bPerformRollback) - { - dbRootRollbackCount++; - deleteExtents ( metaDataStream ); - closeMetaDataFile ( ); - } - else // Skip any DBRoot not having a meta-data file - { - std::ostringstream msg0090Text; - msg0090Text << dbRoots[m]; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0090, 0, msg0090Text.str() ); - } - } - - if (dbRootRollbackCount > 0) - { - // Notify PrimProc to flush FD cache. If error occurs, we tell - // the user but keep going. - int flushFd_rc = cacheutils::dropPrimProcFdCache(); - - if (flushFd_rc != 0) - { - std::ostringstream oss; - oss << "ClearTableLock: Error flushing PrimProc " - "FD cache after rolling back data for table " << - fTableName << - " (OID-" << fTableOID << "); rc-" << flushFd_rc; - - // If we have a logger, then use it to log to syslog, etc - if (fLog) - { - fLog->logMsg( oss.str(), MSGLVL_ERROR ); - } - else // log message ourselves - { - std::cout << oss.str() << std::endl; - - logging::Message m( logging::M0010 ); - logging::Message::Args args; - args.add( oss.str() ); - m.format( args ); - fSysLogger.logErrorMessage( m ); - } - } - - // Notify PrimProc to flush block cache. If error occurs, we tell - // the user but keep going. - std::vector allOIDs; - std::set::const_iterator iter = fAllColDctOIDs.begin(); - cerr << "Rollback flushing: "; - while (iter != fAllColDctOIDs.end()) - { - cerr << *iter << ", "; - //std::cout << "Flushing OID from PrimProc cache " << *iter << - // std::endl; - allOIDs.push_back(*iter); - ++iter; - } - cerr << endl; - - int cache_rc = cacheutils::flushOIDsFromCache( allOIDs ); - - if (cache_rc != 0) - { - std::ostringstream oss; - oss << "ClearTableLock: Error flushing " - "PrimProc cache after rolling back data for table " << - fTableName << - " (OID-" << fTableOID << "); rc-" << cache_rc; - - // If we have a logger, then use it to log to syslog, etc - if (fLog) - { - fLog->logMsg( oss.str(), MSGLVL_ERROR ); - } - else // log message ourselves - { - std::cout << oss.str() << std::endl; - - logging::Message m( logging::M0010 ); - logging::Message::Args args; - args.add( oss.str() ); - m.format( args ); - fSysLogger.logErrorMessage( m ); - } - } - } - } - catch (WeException& ex) - { - std::string msgText(fApplName); - msgText += ". (rollback failed; "; - msgText += ex.what(); - msgText += ')'; - logAMessage( logging::LOG_TYPE_ERROR, - logging::M0085, 0, msgText ); - - fErrorMsg = ex.what(); - return ex.errorCode(); + throw WeException(oss.str(), rc); } - if (!keepMetaFile) - deleteMetaDataFiles ( ); + std::vector dbRoots; + Config::getRootIdList(dbRoots); + + std::string emptyText0072; + logAMessage(logging::LOG_TYPE_INFO, logging::M0072, 0, emptyText0072); + + // Loop through DBRoots for this PM + for (unsigned m = 0; m < dbRoots.size(); m++) + { + std::istringstream metaDataStream; + bool bPerformRollback = openMetaDataFile(dbRoots[m], metaDataStream); + + // Call function to: + // 1. read bulk rollback meta-data file + // 2. rollback applicable extents from extentmap + // 3. delete applicable extents from database files + // 4. reinitialize blocks trailing the HWM block in the last extent + // of each segment file + // ... + if (bPerformRollback) + { + dbRootRollbackCount++; + deleteExtents(metaDataStream); + closeMetaDataFile(); + } + else // Skip any DBRoot not having a meta-data file + { + std::ostringstream msg0090Text; + msg0090Text << dbRoots[m]; + logAMessage(logging::LOG_TYPE_INFO, logging::M0090, 0, msg0090Text.str()); + } + } if (dbRootRollbackCount > 0) { - logAMessage( logging::LOG_TYPE_INFO, - logging::M0085, 0, fApplName ); - } - else - { - std::string msgText(fApplName); - msgText += ". (Nothing to rollback)"; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0085, 0, msgText ); - } + // Notify PrimProc to flush FD cache. If error occurs, we tell + // the user but keep going. + int flushFd_rc = cacheutils::dropPrimProcFdCache(); - return rc; + if (flushFd_rc != 0) + { + std::ostringstream oss; + oss << "ClearTableLock: Error flushing PrimProc " + "FD cache after rolling back data for table " + << fTableName << " (OID-" << fTableOID << "); rc-" << flushFd_rc; + + // If we have a logger, then use it to log to syslog, etc + if (fLog) + { + fLog->logMsg(oss.str(), MSGLVL_ERROR); + } + else // log message ourselves + { + std::cout << oss.str() << std::endl; + + logging::Message m(logging::M0010); + logging::Message::Args args; + args.add(oss.str()); + m.format(args); + fSysLogger.logErrorMessage(m); + } + } + + // Notify PrimProc to flush block cache. If error occurs, we tell + // the user but keep going. + std::vector allOIDs; + std::set::const_iterator iter = fAllColDctOIDs.begin(); + cerr << "Rollback flushing: "; + while (iter != fAllColDctOIDs.end()) + { + cerr << *iter << ", "; + // std::cout << "Flushing OID from PrimProc cache " << *iter << + // std::endl; + allOIDs.push_back(*iter); + ++iter; + } + cerr << endl; + + int cache_rc = cacheutils::flushOIDsFromCache(allOIDs); + + if (cache_rc != 0) + { + std::ostringstream oss; + oss << "ClearTableLock: Error flushing " + "PrimProc cache after rolling back data for table " + << fTableName << " (OID-" << fTableOID << "); rc-" << cache_rc; + + // If we have a logger, then use it to log to syslog, etc + if (fLog) + { + fLog->logMsg(oss.str(), MSGLVL_ERROR); + } + else // log message ourselves + { + std::cout << oss.str() << std::endl; + + logging::Message m(logging::M0010); + logging::Message::Args args; + args.add(oss.str()); + m.format(args); + fSysLogger.logErrorMessage(m); + } + } + } + } + catch (WeException& ex) + { + std::string msgText(fApplName); + msgText += ". (rollback failed; "; + msgText += ex.what(); + msgText += ')'; + logAMessage(logging::LOG_TYPE_ERROR, logging::M0085, 0, msgText); + + fErrorMsg = ex.what(); + return ex.errorCode(); + } + + if (!keepMetaFile) + deleteMetaDataFiles(); + + if (dbRootRollbackCount > 0) + { + logAMessage(logging::LOG_TYPE_INFO, logging::M0085, 0, fApplName); + } + else + { + std::string msgText(fApplName); + msgText += ". (Nothing to rollback)"; + logAMessage(logging::LOG_TYPE_INFO, logging::M0085, 0, msgText); + } + + return rc; } //------------------------------------------------------------------------------ @@ -285,29 +273,27 @@ int BulkRollbackMgr::rollback ( bool keepMetaFile ) // and skip performing a bulk rollback for a dbroot that does not // have a meta-data file. Kept the function around for the time being. //------------------------------------------------------------------------------ -void BulkRollbackMgr::validateAllMetaFilesExist ( - const std::vector& dbRoots ) const +void BulkRollbackMgr::validateAllMetaFilesExist(const std::vector& dbRoots) const { - // Loop through DBRoots for this PM - for (unsigned m = 0; m < dbRoots.size(); m++) + // Loop through DBRoots for this PM + for (unsigned m = 0; m < dbRoots.size(); m++) + { + std::string bulkRollbackPath(Config::getDBRootByNum(dbRoots[m])); + + // Construct file name and check for it's existence + std::ostringstream oss; + oss << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << fTableOID; + std::string metaFileName = bulkRollbackPath; + metaFileName += oss.str(); + + if (!IDBPolicy::exists(metaFileName.c_str())) { - std::string bulkRollbackPath( Config::getDBRootByNum(dbRoots[m]) ); + std::ostringstream oss; + oss << "Error opening bulk rollback meta-data file " << metaFileName << "; File does not exist."; - // Construct file name and check for it's existence - std::ostringstream oss; - oss << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << fTableOID; - std::string metaFileName = bulkRollbackPath; - metaFileName += oss.str(); - - if ( !IDBPolicy::exists( metaFileName.c_str() ) ) - { - std::ostringstream oss; - oss << "Error opening bulk rollback meta-data file " << - metaFileName << "; File does not exist."; - - throw WeException( oss.str(), ERR_FILE_OPEN ); - } + throw WeException(oss.str(), ERR_FILE_OPEN); } + } } //------------------------------------------------------------------------------ @@ -316,231 +302,222 @@ void BulkRollbackMgr::validateAllMetaFilesExist ( // Returns true/false to indicate whether execution should continue if the // meta-data file is missing. //------------------------------------------------------------------------------ -bool BulkRollbackMgr::openMetaDataFile ( uint16_t dbRoot, - std::istringstream& metaDataStream ) +bool BulkRollbackMgr::openMetaDataFile(uint16_t dbRoot, std::istringstream& metaDataStream) { - std::string bulkRollbackPath( Config::getDBRootByNum( dbRoot ) ); + std::string bulkRollbackPath(Config::getDBRootByNum(dbRoot)); - // Construct file name and check for it's existence + // Construct file name and check for it's existence + std::ostringstream oss; + oss << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << fTableOID; + fMetaFileName = bulkRollbackPath; + fMetaFileName += oss.str(); + + // Return if the meta-data file does not exist. This could happen if we + // are executing distributed rollback on several PMs, some of which may + // have not even executed an import. + // Also could happen if DBRoots are moved from 1 PM to another during a job. + if (!IDBPolicy::exists(fMetaFileName.c_str())) + { + return false; + } + + // Open the file + fMetaFile = IDBDataFile::open(IDBPolicy::getType(fMetaFileName.c_str(), IDBPolicy::WRITEENG), + fMetaFileName.c_str(), "rb", 0); + + if (!fMetaFile) + { + int errRc = errno; std::ostringstream oss; - oss << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << fTableOID; - fMetaFileName = bulkRollbackPath; - fMetaFileName += oss.str(); + oss << "Error opening bulk rollback meta-data file " << fMetaFileName << "; err-" << errRc << "; " + << strerror(errRc); - // Return if the meta-data file does not exist. This could happen if we - // are executing distributed rollback on several PMs, some of which may - // have not even executed an import. - // Also could happen if DBRoots are moved from 1 PM to another during a job. - if ( !IDBPolicy::exists( fMetaFileName.c_str() ) ) - { - return false; - } + throw WeException(oss.str(), ERR_FILE_OPEN); + } - // Open the file - fMetaFile = IDBDataFile::open( IDBPolicy::getType(fMetaFileName.c_str(), - IDBPolicy::WRITEENG), - fMetaFileName.c_str(), "rb", 0); + fMetaFileNames.push_back(fMetaFileName); - if ( !fMetaFile ) - { - int errRc = errno; - std::ostringstream oss; - oss << "Error opening bulk rollback meta-data file " << - fMetaFileName << "; err-" << - errRc << "; " << strerror( errRc ); + // First record in the file must be a Version record. + char inBuf[BUF_SIZE]; + ssize_t metaFileSize = IDBPolicy::size(fMetaFileName.c_str()); + boost::scoped_array buf(new char[metaFileSize]); + // retry 10 times for partial reads, just in case + ssize_t readSofar = 0; // bytes read so far + ssize_t bytes = 0; // bytes read by one pread + char* p = buf.get(); - throw WeException( oss.str(), ERR_FILE_OPEN ); - } + for (int i = 0; i < 10 && readSofar < metaFileSize; i++) + { + bytes = fMetaFile->pread(p + readSofar, readSofar, metaFileSize - readSofar); - fMetaFileNames.push_back( fMetaFileName ); + if (bytes < 0) + break; - // First record in the file must be a Version record. - char inBuf[ BUF_SIZE ]; - ssize_t metaFileSize = IDBPolicy::size( fMetaFileName.c_str() ); - boost::scoped_array buf( new char[ metaFileSize ] ); - // retry 10 times for partial reads, just in case - ssize_t readSofar = 0; // bytes read so far - ssize_t bytes = 0; // bytes read by one pread - char* p = buf.get(); + readSofar += bytes; + } - for (int i = 0; i < 10 && readSofar < metaFileSize; i++) - { - bytes = fMetaFile->pread( p + readSofar, readSofar, metaFileSize - readSofar); + if (readSofar != metaFileSize) + { + int errRc = errno; + std::ostringstream oss; + oss << "Error reading bulk rollback meta-data file " << fMetaFileName << "; read/expect:" << readSofar + << "/" << metaFileSize << "; err-" << errRc << "; " << strerror(errRc); - if (bytes < 0) - break; + throw WeException(oss.str(), ERR_FILE_READ); + } - readSofar += bytes; - } + // put the data in a string stream + metaDataStream.str(string(p, metaFileSize)); + buf.reset(); - if ( readSofar != metaFileSize ) - { - int errRc = errno; - std::ostringstream oss; - oss << "Error reading bulk rollback meta-data file " - << fMetaFileName << "; read/expect:" << readSofar << "/" << metaFileSize - << "; err-" << errRc << "; " << strerror( errRc ); + // read data + metaDataStream.getline(inBuf, BUF_SIZE); - throw WeException( oss.str(), ERR_FILE_READ ); - } + if (RBMetaWriter::verifyVersion3(inBuf)) + { + fVersion = 3; + } + else if (RBMetaWriter::verifyVersion4(inBuf)) + { + fVersion = 4; + } + else + { + std::ostringstream oss; + oss << "Invalid version record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; - // put the data in a string stream - metaDataStream.str( string( p, metaFileSize ) ); - buf.reset(); + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - // read data - metaDataStream.getline( inBuf, BUF_SIZE ); - - if (RBMetaWriter::verifyVersion3(inBuf)) - { - fVersion = 3; - } - else if (RBMetaWriter::verifyVersion4(inBuf)) - { - fVersion = 4; - } - else - { - std::ostringstream oss; - oss << "Invalid version record in meta-data file " << fMetaFileName - << "; record-<" << inBuf << ">" << std::endl; - - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } - - return true; + return true; } //------------------------------------------------------------------------------ // Close the current meta-data file used in rolling back fTableOID. //------------------------------------------------------------------------------ -void BulkRollbackMgr::closeMetaDataFile ( ) +void BulkRollbackMgr::closeMetaDataFile() { - delete fMetaFile; - fMetaFile = NULL; + delete fMetaFile; + fMetaFile = NULL; } //------------------------------------------------------------------------------ // Delete all the local meta-data files used in rolling back fTableOID. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteMetaDataFiles ( ) +void BulkRollbackMgr::deleteMetaDataFiles() { - for (unsigned k = 0; k < fMetaFileNames.size(); k++) - { - IDBPolicy::remove( fMetaFileNames[k].c_str() ) ; + for (unsigned k = 0; k < fMetaFileNames.size(); k++) + { + IDBPolicy::remove(fMetaFileNames[k].c_str()); - // Unlink corresponding tmp file created by RBMetaWriter. - std::string tmpMetaFileName = fMetaFileNames[k]; - tmpMetaFileName += TMP_FILE_SUFFIX; - IDBPolicy::remove( tmpMetaFileName.c_str() ); + // Unlink corresponding tmp file created by RBMetaWriter. + std::string tmpMetaFileName = fMetaFileNames[k]; + tmpMetaFileName += TMP_FILE_SUFFIX; + IDBPolicy::remove(tmpMetaFileName.c_str()); - deleteSubDir( fMetaFileNames[k] ); - } + deleteSubDir(fMetaFileNames[k]); + } } //------------------------------------------------------------------------------ // Delete the subdirectory used to backup data needed for rollback. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteSubDir( const std::string& metaFileName ) +void BulkRollbackMgr::deleteSubDir(const std::string& metaFileName) { - std::string bulkRollbackSubPath( metaFileName ); - bulkRollbackSubPath += DATA_DIR_SUFFIX; + std::string bulkRollbackSubPath(metaFileName); + bulkRollbackSubPath += DATA_DIR_SUFFIX; - if ( IDBPolicy::remove( bulkRollbackSubPath.c_str() ) != 0 ) - { - std::ostringstream oss; - oss << "Warning: Error deleting bulk rollback data subdirectory " << - bulkRollbackSubPath << ";"; + if (IDBPolicy::remove(bulkRollbackSubPath.c_str()) != 0) + { + std::ostringstream oss; + oss << "Warning: Error deleting bulk rollback data subdirectory " << bulkRollbackSubPath << ";"; - if (fLog) - fLog->logMsg( oss.str(), MSGLVL_WARNING ); - else - std::cout << oss.str() << std::endl; - } + if (fLog) + fLog->logMsg(oss.str(), MSGLVL_WARNING); + else + std::cout << oss.str() << std::endl; + } } //------------------------------------------------------------------------------ // Function that drives the rolling back or deletion of extents for a given // database table as specified in a meta-data bulk rollback file. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteExtents ( std::istringstream& metaDataStream ) +void BulkRollbackMgr::deleteExtents(std::istringstream& metaDataStream) { - char inBuf[ BUF_SIZE ]; - OID columnOID = 0; - OID storeOID = 0; - uint32_t dbRoot = 0; + char inBuf[BUF_SIZE]; + OID columnOID = 0; + OID storeOID = 0; + uint32_t dbRoot = 0; - // Loop through the records in the meta-data file - while (metaDataStream.getline( inBuf, BUF_SIZE )) + // Loop through the records in the meta-data file + while (metaDataStream.getline(inBuf, BUF_SIZE)) + { + // Restore extents for a DBRoot + if (RBMetaWriter::verifyColumn1Rec(inBuf)) { - // Restore extents for a DBRoot - if (RBMetaWriter::verifyColumn1Rec(inBuf)) - { - // Process any pending dictionary deletes - if (fPendingDctnryExtents.size() > 0) - { - deleteDctnryExtents ( ); - deleteDbFiles( ); - } + // Process any pending dictionary deletes + if (fPendingDctnryExtents.size() > 0) + { + deleteDctnryExtents(); + deleteDbFiles(); + } - deleteColumn1Extents ( inBuf ); - deleteDbFiles( ); - } - // Delete all extents from a formerly empty DBRoot - else if (RBMetaWriter::verifyColumn2Rec(inBuf)) - { - // Process any pending dictionary deletes - if (fPendingDctnryExtents.size() > 0) - { - deleteDctnryExtents ( ); - deleteDbFiles( ); - } - - deleteColumn2Extents ( inBuf ); - deleteDbFiles( ); - } - else if (RBMetaWriter::verifyDStore1Rec(inBuf) || - RBMetaWriter::verifyDStore2Rec(inBuf)) - { - if (fPendingDctnryExtents.size() > 0) - { - char recType[100]; - int numFields = sscanf( - inBuf, "%s %u %u %d", - recType, &columnOID, &storeOID, &dbRoot ); - - if (numFields != 4) - { - std::ostringstream oss; - oss << "Invalid record in meta-data file " << fMetaFileName - << "; record-<" << inBuf << ">" << std::endl; - - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } - - // Process any pending dictionary deletes - if ((storeOID != fPendingDctnryStoreOID) || - (dbRoot != fPendingDctnryStoreDbRoot)) - { - deleteDctnryExtents ( ); - deleteDbFiles( ); - } - } - - readMetaDataRecDctnry ( inBuf ); - } - else - { - // ignore unrecognized record type - } - } // end of loop through all the records in the meta-data input file - - // Process any pending dictionary deletes - if (fPendingDctnryExtents.size() > 0) - { - deleteDctnryExtents ( ); - deleteDbFiles( ); + deleteColumn1Extents(inBuf); + deleteDbFiles(); } + // Delete all extents from a formerly empty DBRoot + else if (RBMetaWriter::verifyColumn2Rec(inBuf)) + { + // Process any pending dictionary deletes + if (fPendingDctnryExtents.size() > 0) + { + deleteDctnryExtents(); + deleteDbFiles(); + } + + deleteColumn2Extents(inBuf); + deleteDbFiles(); + } + else if (RBMetaWriter::verifyDStore1Rec(inBuf) || RBMetaWriter::verifyDStore2Rec(inBuf)) + { + if (fPendingDctnryExtents.size() > 0) + { + char recType[100]; + int numFields = sscanf(inBuf, "%s %u %u %d", recType, &columnOID, &storeOID, &dbRoot); + + if (numFields != 4) + { + std::ostringstream oss; + oss << "Invalid record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; + + throw WeException(oss.str(), ERR_INVALID_PARAM); + } + + // Process any pending dictionary deletes + if ((storeOID != fPendingDctnryStoreOID) || (dbRoot != fPendingDctnryStoreDbRoot)) + { + deleteDctnryExtents(); + deleteDbFiles(); + } + } + + readMetaDataRecDctnry(inBuf); + } + else + { + // ignore unrecognized record type + } + } // end of loop through all the records in the meta-data input file + + // Process any pending dictionary deletes + if (fPendingDctnryExtents.size() > 0) + { + deleteDctnryExtents(); + deleteDbFiles(); + } } //------------------------------------------------------------------------------ @@ -550,97 +527,95 @@ void BulkRollbackMgr::deleteExtents ( std::istringstream& metaDataStream ) // // inBuf - latest dictionary extent record to be parsed from meta-data file //------------------------------------------------------------------------------ -void BulkRollbackMgr::readMetaDataRecDctnry ( const char* inBuf ) +void BulkRollbackMgr::readMetaDataRecDctnry(const char* inBuf) { - char recType[100]; - OID dColumnOID; - OID dStoreOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM localHwm; - int compressionType = 0; // optional parameter + char recType[100]; + OID dColumnOID; + OID dStoreOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM localHwm; + int compressionType = 0; // optional parameter - sscanf(inBuf, "%s", recType); - RollbackData rbData; + sscanf(inBuf, "%s", recType); + RollbackData rbData; - // Process DSTORE1 records representing segment files with an HWM - if (RBMetaWriter::verifyDStore1Rec(recType)) + // Process DSTORE1 records representing segment files with an HWM + if (RBMetaWriter::verifyDStore1Rec(recType)) + { + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d", recType, &dColumnOID, &dStoreOID, &dbRootHwm, + &partNumHwm, &segNumHwm, &localHwm, &compressionType); + + if (numFields < 7) // compressionType optional { - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d", - recType, &dColumnOID, &dStoreOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &localHwm, &compressionType ); + std::ostringstream oss; + oss << "Invalid DSTOR1 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; - if (numFields < 7) // compressionType optional - { - std::ostringstream oss; - oss << "Invalid DSTOR1 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">" << std::endl; - - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } - - rbData.fWithHwm = true; - rbData.fHwm = localHwm; + throw WeException(oss.str(), ERR_INVALID_PARAM); } - // Process DSTORE2 records representing segment files w/o HWM; meaning that - // segment file did not previously exist and can be deleted if it was added - else + rbData.fWithHwm = true; + rbData.fHwm = localHwm; + } + + // Process DSTORE2 records representing segment files w/o HWM; meaning that + // segment file did not previously exist and can be deleted if it was added + else + { + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d", recType, &dColumnOID, &dStoreOID, &dbRootHwm, + &partNumHwm, &segNumHwm, &compressionType); + + if (numFields < 6) // compressionType optional { - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d", - recType, &dColumnOID, &dStoreOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &compressionType ); + std::ostringstream oss; + oss << "Invalid DSTOR2 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; - if (numFields < 6) // compressionType optional - { - std::ostringstream oss; - oss << "Invalid DSTOR2 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">" << std::endl; - - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } - - rbData.fWithHwm = false; - rbData.fHwm = 0; + throw WeException(oss.str(), ERR_INVALID_PARAM); } - rbData.fDbRoot = dbRootHwm; - rbData.fPartNum = partNumHwm; - rbData.fSegNum = segNumHwm; + rbData.fWithHwm = false; + rbData.fHwm = 0; + } - fPendingDctnryExtents.push_back( rbData ); + rbData.fDbRoot = dbRootHwm; + rbData.fPartNum = partNumHwm; + rbData.fSegNum = segNumHwm; - // OID and compression type should be the same for all store files relating - // to the same dictionary column, but they are set for each record nonethe- - // less since the format of the meta data file is a flat file format. - // Likewise, the DBRoot will be the same for each collection of dictionary - // extents that are processed as a group. - fPendingDctnryStoreOID = dStoreOID; - fPendingDctnryStoreCompressionType = compressionType; - fPendingDctnryStoreDbRoot = dbRootHwm; + fPendingDctnryExtents.push_back(rbData); + + // OID and compression type should be the same for all store files relating + // to the same dictionary column, but they are set for each record nonethe- + // less since the format of the meta data file is a flat file format. + // Likewise, the DBRoot will be the same for each collection of dictionary + // extents that are processed as a group. + fPendingDctnryStoreOID = dStoreOID; + fPendingDctnryStoreCompressionType = compressionType; + fPendingDctnryStoreDbRoot = dbRootHwm; } //------------------------------------------------------------------------------ // Delete column extents based on COLUMN1 record input //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteColumn1Extents ( const char* inBuf ) +void BulkRollbackMgr::deleteColumn1Extents(const char* inBuf) { - if (fVersion == 3) - deleteColumn1ExtentsV3( inBuf ); - else - deleteColumn1ExtentsV4( inBuf ); + if (fVersion == 3) + deleteColumn1ExtentsV3(inBuf); + else + deleteColumn1ExtentsV4(inBuf); } //------------------------------------------------------------------------------ // Delete column extents based on COLUMN2 record input //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteColumn2Extents ( const char* inBuf ) +void BulkRollbackMgr::deleteColumn2Extents(const char* inBuf) { - if (fVersion == 3) - deleteColumn2ExtentsV3( inBuf ); - else - deleteColumn2ExtentsV4( inBuf ); + if (fVersion == 3) + deleteColumn2ExtentsV3(inBuf); + else + deleteColumn2ExtentsV4(inBuf); } //------------------------------------------------------------------------------ @@ -662,9 +637,9 @@ void BulkRollbackMgr::deleteColumn2Extents ( const char* inBuf ) // In the case of a COLUMN1 record, V3 and V4 are handled the same, so this // adaptor function is a pass-thru to the V4 function. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteColumn1ExtentsV3 ( const char* inBuf ) +void BulkRollbackMgr::deleteColumn1ExtentsV3(const char* inBuf) { - deleteColumn1ExtentsV4( inBuf ); + deleteColumn1ExtentsV4(inBuf); } //------------------------------------------------------------------------------ @@ -689,48 +664,40 @@ void BulkRollbackMgr::deleteColumn1ExtentsV3 ( const char* inBuf ) // mined. So this adaptor function changes any segment number for the COLUMN2 // record to a 0, to be 100% compatible with the Version4 format. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteColumn2ExtentsV3 ( const char* inBuf ) +void BulkRollbackMgr::deleteColumn2ExtentsV3(const char* inBuf) { - char recType[100]; - OID columnOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - int colTypeInt; - char colTypeName[100]; - uint32_t colWidth; - int compressionType = 0; // optional parameter + char recType[100]; + OID columnOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + int colTypeInt; + char colTypeName[100]; + uint32_t colWidth; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %d %s %u %d", - recType, &columnOID, - &dbRootHwm, &partNumHwm, &segNumHwm, - &colTypeInt, colTypeName, &colWidth, &compressionType ); + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %d %s %u %d", recType, &columnOID, &dbRootHwm, &partNumHwm, + &segNumHwm, &colTypeInt, colTypeName, &colWidth, &compressionType); - if (numFields < 8) // compressionType is optional - { - std::ostringstream oss; - oss << "Invalid COLUM2 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">" << std::endl; + if (numFields < 8) // compressionType is optional + { + std::ostringstream oss; + oss << "Invalid COLUM2 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - std::ostringstream revisedBuf; - uint32_t revisedSegNumHwm = 0; - revisedBuf << recType << ' ' << - columnOID << ' ' << - dbRootHwm << ' ' << - partNumHwm << ' ' << - revisedSegNumHwm << ' ' << - colTypeInt << ' ' << - colTypeName << ' ' << - colWidth << ' '; + std::ostringstream revisedBuf; + uint32_t revisedSegNumHwm = 0; + revisedBuf << recType << ' ' << columnOID << ' ' << dbRootHwm << ' ' << partNumHwm << ' ' + << revisedSegNumHwm << ' ' << colTypeInt << ' ' << colTypeName << ' ' << colWidth << ' '; - if (numFields > 8) - revisedBuf << compressionType; + if (numFields > 8) + revisedBuf << compressionType; - deleteColumn2ExtentsV4( revisedBuf.str().c_str() ); + deleteColumn2ExtentsV4(revisedBuf.str().c_str()); } //@bug 4091: V4 support for adding DBRoot @@ -742,272 +709,224 @@ void BulkRollbackMgr::deleteColumn2ExtentsV3 ( const char* inBuf ) // // inBuf - latest column extent record to be parsed from meta-data file //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteColumn1ExtentsV4 ( const char* inBuf ) +void BulkRollbackMgr::deleteColumn1ExtentsV4(const char* inBuf) { - char recType[100]; - OID columnOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM lastLocalHwm; - int colTypeInt; - CalpontSystemCatalog::ColDataType colType; - char colTypeName[100]; - uint32_t colWidth; - int compressionType = 0; // optional parameter + char recType[100]; + OID columnOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM lastLocalHwm; + int colTypeInt; + CalpontSystemCatalog::ColDataType colType; + char colTypeName[100]; + uint32_t colWidth; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d", - recType, &columnOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &lastLocalHwm, - &colTypeInt, colTypeName, &colWidth, &compressionType ); - colType = (CalpontSystemCatalog::ColDataType)colTypeInt; + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d", recType, &columnOID, &dbRootHwm, &partNumHwm, + &segNumHwm, &lastLocalHwm, &colTypeInt, colTypeName, &colWidth, &compressionType); + colType = (CalpontSystemCatalog::ColDataType)colTypeInt; - if (numFields < 9) // compressionType is optional - { - std::ostringstream oss; - oss << "Invalid COLUM1 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">" << std::endl; + if (numFields < 9) // compressionType is optional + { + std::ostringstream oss; + oss << "Invalid COLUM1 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - std::ostringstream msg0073Text; - msg0073Text << "DBRoot-" << dbRootHwm << " (column extent)"; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0073, columnOID, msg0073Text.str() ); + std::ostringstream msg0073Text; + msg0073Text << "DBRoot-" << dbRootHwm << " (column extent)"; + logAMessage(logging::LOG_TYPE_INFO, logging::M0073, columnOID, msg0073Text.str()); - // Delete extents from the extentmap - std::ostringstream msg0074Text; - msg0074Text << "Restoring HWM column extent: " << - "dbRoot-" << dbRootHwm << - "; part#-" << partNumHwm << - "; seg#-" << segNumHwm << - "; hwm-" << lastLocalHwm; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0074, columnOID, msg0074Text.str() ); - fAllColDctOIDs.insert( columnOID ); + // Delete extents from the extentmap + std::ostringstream msg0074Text; + msg0074Text << "Restoring HWM column extent: " + << "dbRoot-" << dbRootHwm << "; part#-" << partNumHwm << "; seg#-" << segNumHwm << "; hwm-" + << lastLocalHwm; + logAMessage(logging::LOG_TYPE_INFO, logging::M0074, columnOID, msg0074Text.str()); + fAllColDctOIDs.insert(columnOID); - // Create the object responsible for restoring the extents in the db files. - BulkRollbackFile* fileRestorer = makeFileRestorer(compressionType); - boost::scoped_ptr refBulkRollbackFile(fileRestorer); + // Create the object responsible for restoring the extents in the db files. + BulkRollbackFile* fileRestorer = makeFileRestorer(compressionType); + boost::scoped_ptr refBulkRollbackFile(fileRestorer); - // DMC-We should probably change this to build up a list of BRM changes, - // and wait to make the call(s) to rollback the BRM changes "after" we - // have restored the db files, and purged PrimProc FD and block cache. - int rc = BRMWrapper::getInstance()->rollbackColumnExtents_DBroot ( - columnOID, - false, // false -> Don't delete all extents (rollback - (uint16_t)dbRootHwm, // to specified dbroot, partition, etc.) - partNumHwm, - (uint16_t)segNumHwm, - lastLocalHwm ); + // DMC-We should probably change this to build up a list of BRM changes, + // and wait to make the call(s) to rollback the BRM changes "after" we + // have restored the db files, and purged PrimProc FD and block cache. + int rc = BRMWrapper::getInstance()->rollbackColumnExtents_DBroot( + columnOID, + false, // false -> Don't delete all extents (rollback + (uint16_t)dbRootHwm, // to specified dbroot, partition, etc.) + partNumHwm, (uint16_t)segNumHwm, lastLocalHwm); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error rolling back column extents from extent map for " << columnOID << "; dbRoot-" << dbRootHwm + << "; partition-" << partNumHwm << "; segment-" << segNumHwm << "; hwm-" << lastLocalHwm << "; " + << ec.errorString(rc); + + throw WeException(oss.str(), ERR_BRM_BULK_RB_COLUMN); + } + + // Determine the exact rollback point for the extents we are rolling back to + const unsigned BLKS_PER_EXTENT = (BRMWrapper::getInstance()->getExtentRows() * colWidth) / BYTE_PER_BLOCK; + uint32_t lastBlkOfCurrStripe = 0; + uint32_t lastBlkOfPrevStripe = 0; + + if ((lastLocalHwm + 1) <= BLKS_PER_EXTENT) + { + lastBlkOfCurrStripe = BLKS_PER_EXTENT - 1; + } + else + { + lastBlkOfPrevStripe = lastLocalHwm - (lastLocalHwm % BLKS_PER_EXTENT) - 1; + lastBlkOfCurrStripe = lastBlkOfPrevStripe + BLKS_PER_EXTENT; + } + + uint32_t dbRoot = dbRootHwm; + uint32_t partNum = partNumHwm; + std::string segFileListErrMsg; + + // Delete extents from the database files. + // Loop through all partitions (starting with the HWM partition partNumHwm), + // deleting or restoring applicable extents. We stay in loop till we + // reach a partition that has no column segment files to roll back. + bool useHdfs = IDBPolicy::useHdfs(); + + while (1) + { + std::vector segList; + std::string dirName; + rc = fileRestorer->buildDirName(columnOID, dbRoot, partNum, dirName); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error rolling back column extents from extent map for " << - columnOID << - "; dbRoot-" << dbRootHwm << - "; partition-" << partNumHwm << - "; segment-" << segNumHwm << - "; hwm-" << lastLocalHwm << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error constructing path for column " << columnOID << "; dbRoot-" << dbRoot + << "; partition-" << partNum << "; " << ec.errorString(rc); - throw WeException( oss.str(), ERR_BRM_BULK_RB_COLUMN ); + throw WeException(oss.str(), rc); } - // Determine the exact rollback point for the extents we are rolling back to - const unsigned BLKS_PER_EXTENT = - (BRMWrapper::getInstance()->getExtentRows() * colWidth) / BYTE_PER_BLOCK; - uint32_t lastBlkOfCurrStripe = 0; - uint32_t lastBlkOfPrevStripe = 0; + rc = getSegFileList(dirName, useHdfs, segList, segFileListErrMsg); - if ((lastLocalHwm + 1) <= BLKS_PER_EXTENT) + if (rc != NO_ERROR) { - lastBlkOfCurrStripe = BLKS_PER_EXTENT - 1; - } - else - { - lastBlkOfPrevStripe = lastLocalHwm - - (lastLocalHwm % BLKS_PER_EXTENT) - 1; - lastBlkOfCurrStripe = lastBlkOfPrevStripe + BLKS_PER_EXTENT; + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error for column " << columnOID << "; directory-" << dirName << "; " + << segFileListErrMsg << "; " << ec.errorString(rc); + + throw WeException(oss.str(), rc); } - uint32_t dbRoot = dbRootHwm; - uint32_t partNum = partNumHwm; - std::string segFileListErrMsg; + if (segList.size() == 0) + break; // Exit loop when we reach empty partition - // Delete extents from the database files. - // Loop through all partitions (starting with the HWM partition partNumHwm), - // deleting or restoring applicable extents. We stay in loop till we - // reach a partition that has no column segment files to roll back. - bool useHdfs = IDBPolicy::useHdfs(); - - while ( 1 ) + for (unsigned int kk = 0; kk < segList.size(); kk++) { - std::vector segList; - std::string dirName; - rc = fileRestorer->buildDirName( columnOID, dbRoot, partNum, dirName ); + uint32_t segNum = segList[kk]; + + if (partNum == partNumHwm) + { + // Don't rollback an OutOfService extent in the HWM partition + bool bFound; + int extState; + rc = BRMWrapper::getInstance()->getExtentState(columnOID, partNum, segNum, bFound, extState); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error constructing path for column " << - columnOID << - "; dbRoot-" << dbRoot << - "; partition-" << partNum << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error for column " << columnOID << "; Unable to get extent state for part-" + << partNum << "; seg-" << segNum << "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); + throw WeException(oss.str(), rc); } - rc = getSegFileList( dirName, useHdfs, segList, segFileListErrMsg ); + if ((bFound) && (extState == BRM::EXTENTOUTOFSERVICE)) + continue; - if (rc != NO_ERROR) + if (segNum < segNumHwm) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error for column " << columnOID << - "; directory-" << dirName << - "; " << segFileListErrMsg << - "; " << ec.errorString(rc); + fileRestorer->truncateSegmentFile(columnOID, dbRoot, partNum, segNum, (lastBlkOfCurrStripe + 1)); + } // end of (segNum < segNumHwm) - throw WeException( oss.str(), rc ); - } - - if (segList.size() == 0) - break; // Exit loop when we reach empty partition - - for (unsigned int kk = 0; kk < segList.size(); kk++) + else if (segNum > segNumHwm) { - uint32_t segNum = segList[kk]; + if (lastBlkOfPrevStripe > 0) + { + fileRestorer->truncateSegmentFile(columnOID, dbRoot, partNum, segNum, (lastBlkOfPrevStripe + 1)); + } + // lastBlkOfPrevStripe = 0, means there was no previous + // stripe in this partition. The HWM block was in the + // first stripe. In this case we can delete any segment + // files added to this partition that follow segNumHwm. + else + { + std::string segFileName; + fileRestorer->buildSegmentFileName(columnOID, + true, // column segment file + dbRoot, partNum, segNum, segFileName); - if ( partNum == partNumHwm ) - { - // Don't rollback an OutOfService extent in the HWM partition - bool bFound; - int extState; - rc = BRMWrapper::getInstance()->getExtentState( - columnOID, partNum, segNum, bFound, extState ); + createFileDeletionEntry(columnOID, + true, // column segment file + dbRoot, partNum, segNum, segFileName); + } + } // end of (segNum > segNumHwm) - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error for column " << columnOID << - "; Unable to get extent state for part-" << partNum << - "; seg-" << segNum << - "; " << ec.errorString(rc); + else // segNum == segNumHwm + { + if (lastBlkOfCurrStripe == lastLocalHwm) + { + fileRestorer->truncateSegmentFile(columnOID, dbRoot, partNum, segNum, (lastBlkOfCurrStripe + 1)); + } + else + { + bool restoreChunk = fileRestorer->doWeReInitExtent(columnOID, dbRoot, partNum, segNum); - throw WeException( oss.str(), rc ); - } + // For compressed data, if there is no backup chunk to + // restore (restoreChunk is false), we still restore + // the compressed headers to their previous setting. + // This would happen if DBRoot HWM was not on full ex- + // tent boundary when it was moved. If/when cpimport + // reaches this migrated DBRoot in the middle of an + // import, we do not create a backup chunk file. We + // instead only fill-in the HWM extent with empty row + // markers. So no backup and restore is necessary. Only + // need to truncate any extra extents that were added. - if ((bFound) && (extState == BRM::EXTENTOUTOFSERVICE)) - continue; + // Reinit last extent and truncate the remainder, + // starting with the next block following the HWM block. + fileRestorer->reInitTruncColumnExtent(columnOID, dbRoot, partNum, segNum, (lastLocalHwm + 1), + (lastBlkOfCurrStripe - lastLocalHwm), colType, colWidth, + restoreChunk); + } + } // end of (segNum == segNumHwm) + } + else // ( partNum > partNumHwm ) + { + // Delete any files added to subsequent partitions + std::string segFileName; + fileRestorer->buildSegmentFileName(columnOID, + true, // column segment file + dbRoot, partNum, segNum, segFileName); - if ( segNum < segNumHwm ) - { - fileRestorer->truncateSegmentFile ( columnOID, - dbRoot, - partNum, - segNum, - (lastBlkOfCurrStripe + 1) ); - } // end of (segNum < segNumHwm) + createFileDeletionEntry(columnOID, + true, // column segment file + dbRoot, partNum, segNum, segFileName); + } + } // loop thru all the potential segment files in a partition - else if ( segNum > segNumHwm ) - { - if (lastBlkOfPrevStripe > 0) - { - fileRestorer->truncateSegmentFile ( columnOID, - dbRoot, - partNum, - segNum, - (lastBlkOfPrevStripe + 1) ); - } - // lastBlkOfPrevStripe = 0, means there was no previous - // stripe in this partition. The HWM block was in the - // first stripe. In this case we can delete any segment - // files added to this partition that follow segNumHwm. - else - { - std::string segFileName; - fileRestorer->buildSegmentFileName ( columnOID, - true, // column segment file - dbRoot, - partNum, - segNum, - segFileName ); + partNum++; - createFileDeletionEntry( columnOID, - true, // column segment file - dbRoot, partNum, segNum, segFileName ); - } - } // end of (segNum > segNumHwm) - - else // segNum == segNumHwm - { - if (lastBlkOfCurrStripe == lastLocalHwm) - { - fileRestorer->truncateSegmentFile ( columnOID, - dbRoot, - partNum, - segNum, - (lastBlkOfCurrStripe + 1) ); - } - else - { - bool restoreChunk = - fileRestorer->doWeReInitExtent(columnOID, - dbRoot, partNum, segNum); - - // For compressed data, if there is no backup chunk to - // restore (restoreChunk is false), we still restore - // the compressed headers to their previous setting. - // This would happen if DBRoot HWM was not on full ex- - // tent boundary when it was moved. If/when cpimport - // reaches this migrated DBRoot in the middle of an - // import, we do not create a backup chunk file. We - // instead only fill-in the HWM extent with empty row - // markers. So no backup and restore is necessary. Only - // need to truncate any extra extents that were added. - - // Reinit last extent and truncate the remainder, - // starting with the next block following the HWM block. - fileRestorer->reInitTruncColumnExtent ( columnOID, - dbRoot, - partNum, - segNum, - (lastLocalHwm + 1), - (lastBlkOfCurrStripe - lastLocalHwm), - colType, - colWidth, - restoreChunk ); - } - } // end of (segNum == segNumHwm) - } - else // ( partNum > partNumHwm ) - { - // Delete any files added to subsequent partitions - std::string segFileName; - fileRestorer->buildSegmentFileName ( columnOID, - true, // column segment file - dbRoot, - partNum, - segNum, - segFileName ); - - createFileDeletionEntry( columnOID, - true, // column segment file - dbRoot, partNum, segNum, segFileName ); - } - } // loop thru all the potential segment files in a partition - - partNum++; - - } // end of loop to go thru all partitions till we find last segment file + } // end of loop to go thru all partitions till we find last segment file } //@bug 4091: V4 support for adding DBRoot @@ -1019,194 +938,170 @@ void BulkRollbackMgr::deleteColumn1ExtentsV4 ( const char* inBuf ) // // inBuf - latest column extent record to be parsed from meta-data file //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteColumn2ExtentsV4 ( const char* inBuf ) +void BulkRollbackMgr::deleteColumn2ExtentsV4(const char* inBuf) { - char recType[100]; - OID columnOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM lastLocalHwm = 0; - int colTypeInt; - char colTypeName[100]; - uint32_t colWidth; - int compressionType = 0; // optional parameter + char recType[100]; + OID columnOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM lastLocalHwm = 0; + int colTypeInt; + char colTypeName[100]; + uint32_t colWidth; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %d %s %u %d", - recType, &columnOID, - &dbRootHwm, &partNumHwm, &segNumHwm, - &colTypeInt, colTypeName, &colWidth, &compressionType ); + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %d %s %u %d", recType, &columnOID, &dbRootHwm, &partNumHwm, + &segNumHwm, &colTypeInt, colTypeName, &colWidth, &compressionType); - if (numFields < 8) // compressionType is optional + if (numFields < 8) // compressionType is optional + { + std::ostringstream oss; + oss << "Invalid COLUM2 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">" + << std::endl; + + throw WeException(oss.str(), ERR_INVALID_PARAM); + } + + std::ostringstream msg0073Text; + msg0073Text << "DBRoot-" << dbRootHwm << " (column extent)"; + logAMessage(logging::LOG_TYPE_INFO, logging::M0073, columnOID, msg0073Text.str()); + + // @bug 5644 - If user dropped all partitions in a dbroot, partNumHwm will + // be 0, but we may start importing into part# > 0 (to line up with other + // DBRoots). Use extent map to find first partition added by this import. + std::vector extEntries; + int rc = BRMWrapper::getInstance()->getExtents_dbroot(columnOID, extEntries, dbRootHwm); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error getting extent list from extent map for " << columnOID << "; dbRoot-" << dbRootHwm << "; " + << ec.errorString(rc); + + throw WeException(oss.str(), ERR_BRM_BULK_RB_COLUMN); + } + + uint32_t part1 = partNumHwm; // lowest part# for column and DBRoot + + if (extEntries.size() > 0) + { + part1 = extEntries[0].partitionNum; + + for (unsigned int kk = 0; kk < extEntries.size(); kk++) { - std::ostringstream oss; - oss << "Invalid COLUM2 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">" << std::endl; - - throw WeException( oss.str(), ERR_INVALID_PARAM ); + if (extEntries[kk].partitionNum < part1) + part1 = extEntries[kk].partitionNum; } + } - std::ostringstream msg0073Text; - msg0073Text << "DBRoot-" << dbRootHwm << " (column extent)"; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0073, columnOID, msg0073Text.str() ); + // Delete extents from the extentmap + std::ostringstream msg0074Text; + msg0074Text << "Restoring empty DBRoot. " + "dbRoot-" + << dbRootHwm << "; part#-" << partNumHwm << "; seg#-" << segNumHwm << "; hwm-" << lastLocalHwm + << "; delete starting at part#-" << part1; + logAMessage(logging::LOG_TYPE_INFO, logging::M0074, columnOID, msg0074Text.str()); + fAllColDctOIDs.insert(columnOID); - // @bug 5644 - If user dropped all partitions in a dbroot, partNumHwm will - // be 0, but we may start importing into part# > 0 (to line up with other - // DBRoots). Use extent map to find first partition added by this import. - std::vector extEntries; - int rc = BRMWrapper::getInstance()->getExtents_dbroot( columnOID, - extEntries, dbRootHwm ); + // Reset partNumHwm to partNum taken from extent map + partNumHwm = part1; + + // Create the object responsible for restoring the extents in the db files. + BulkRollbackFile* fileRestorer = makeFileRestorer(compressionType); + boost::scoped_ptr refBulkRollbackFile(fileRestorer); + + // DMC-We should probably change this to build up a list of BRM changes, + // and wait to make the call(s) to rollback the BRM changes "after" we + // have restored the db files, and purged PrimProc FD and block cache. + rc = BRMWrapper::getInstance()->rollbackColumnExtents_DBroot( + columnOID, + true, // true -> delete all extents (restore to empty DBRoot) + (uint16_t)dbRootHwm, partNumHwm, (uint16_t)segNumHwm, lastLocalHwm); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error rolling back column extents from extent map for " << columnOID << "; dbRoot-" << dbRootHwm + << "; partition-" << partNumHwm << "; segment-" << segNumHwm << "; hwm-" << lastLocalHwm << "; " + << ec.errorString(rc); + + throw WeException(oss.str(), ERR_BRM_BULK_RB_COLUMN); + } + + uint32_t dbRoot = dbRootHwm; + uint32_t partNum = partNumHwm; + std::string segFileListErrMsg; + + // Delete extents from the database files. + // Loop through all partitions (starting with the HWM partition partNumHwm), + // deleting or restoring applicable extents. We stay in loop till we + // reach a partition that has no column segment files to roll back. + bool useHdfs = IDBPolicy::useHdfs(); + + while (1) + { + std::vector segList; + std::string dirName; + rc = fileRestorer->buildDirName(columnOID, dbRoot, partNum, dirName); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error getting extent list from extent map for " << - columnOID << - "; dbRoot-" << dbRootHwm << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error constructing path for column " << columnOID << "; dbRoot-" << dbRoot + << "; partition-" << partNum << "; " << ec.errorString(rc); - throw WeException( oss.str(), ERR_BRM_BULK_RB_COLUMN ); + throw WeException(oss.str(), rc); } - uint32_t part1 = partNumHwm; // lowest part# for column and DBRoot - - if (extEntries.size() > 0) - { - part1 = extEntries[0].partitionNum; - - for (unsigned int kk = 0; kk < extEntries.size(); kk++) - { - if (extEntries[kk].partitionNum < part1) - part1 = extEntries[kk].partitionNum; - } - } - - // Delete extents from the extentmap - std::ostringstream msg0074Text; - msg0074Text << "Restoring empty DBRoot. " - "dbRoot-" << dbRootHwm << - "; part#-" << partNumHwm << - "; seg#-" << segNumHwm << - "; hwm-" << lastLocalHwm << - "; delete starting at part#-" << part1; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0074, columnOID, msg0074Text.str() ); - fAllColDctOIDs.insert( columnOID ); - - // Reset partNumHwm to partNum taken from extent map - partNumHwm = part1; - - // Create the object responsible for restoring the extents in the db files. - BulkRollbackFile* fileRestorer = makeFileRestorer(compressionType); - boost::scoped_ptr refBulkRollbackFile(fileRestorer); - - // DMC-We should probably change this to build up a list of BRM changes, - // and wait to make the call(s) to rollback the BRM changes "after" we - // have restored the db files, and purged PrimProc FD and block cache. - rc = BRMWrapper::getInstance()->rollbackColumnExtents_DBroot ( - columnOID, - true, // true -> delete all extents (restore to empty DBRoot) - (uint16_t)dbRootHwm, - partNumHwm, - (uint16_t)segNumHwm, - lastLocalHwm ); + rc = getSegFileList(dirName, useHdfs, segList, segFileListErrMsg); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error rolling back column extents from extent map for " << - columnOID << - "; dbRoot-" << dbRootHwm << - "; partition-" << partNumHwm << - "; segment-" << segNumHwm << - "; hwm-" << lastLocalHwm << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error for column " << columnOID << "; directory-" << dirName << "; " + << segFileListErrMsg << "; " << ec.errorString(rc); - throw WeException( oss.str(), ERR_BRM_BULK_RB_COLUMN ); + throw WeException(oss.str(), rc); } - uint32_t dbRoot = dbRootHwm; - uint32_t partNum = partNumHwm; - std::string segFileListErrMsg; + if (segList.size() == 0) + break; // Exit loop when we reach empty partition - // Delete extents from the database files. - // Loop through all partitions (starting with the HWM partition partNumHwm), - // deleting or restoring applicable extents. We stay in loop till we - // reach a partition that has no column segment files to roll back. - bool useHdfs = IDBPolicy::useHdfs(); - - while ( 1 ) + for (unsigned int kk = 0; kk < segList.size(); kk++) { - std::vector segList; - std::string dirName; - rc = fileRestorer->buildDirName( columnOID, dbRoot, partNum, dirName ); + uint32_t segNum = segList[kk]; - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error constructing path for column " << - columnOID << - "; dbRoot-" << dbRoot << - "; partition-" << partNum << - "; " << ec.errorString(rc); + // Delete any files added to subsequent partitions + std::string segFileName; + fileRestorer->buildSegmentFileName(columnOID, + true, // column segment file + dbRoot, partNum, segNum, segFileName); - throw WeException( oss.str(), rc ); - } + createFileDeletionEntry(columnOID, + true, // column segment file + dbRoot, partNum, segNum, segFileName); + } // loop thru all the potential segment files in a partition - rc = getSegFileList( dirName, useHdfs, segList, segFileListErrMsg ); + partNum++; - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error for column " << columnOID << - "; directory-" << dirName << - "; " << segFileListErrMsg << - "; " << ec.errorString(rc); - - throw WeException( oss.str(), rc ); - } - - if (segList.size() == 0) - break; // Exit loop when we reach empty partition - - for (unsigned int kk = 0; kk < segList.size(); kk++) - { - uint32_t segNum = segList[kk]; - - // Delete any files added to subsequent partitions - std::string segFileName; - fileRestorer->buildSegmentFileName ( columnOID, - true, // column segment file - dbRoot, - partNum, - segNum, - segFileName ); - - createFileDeletionEntry( columnOID, - true, // column segment file - dbRoot, partNum, segNum, segFileName ); - } // loop thru all the potential segment files in a partition - - partNum++; - - } // end of loop to go thru all partitions till we find last segment file + } // end of loop to go thru all partitions till we find last segment file } //------------------------------------------------------------------------------ // Delete dictionary store extents based on COLUMN1 record input //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteDctnryExtents ( ) +void BulkRollbackMgr::deleteDctnryExtents() { - if (fVersion == 3) - deleteDctnryExtentsV3( ); - else - deleteDctnryExtentsV4( ); + if (fVersion == 3) + deleteDctnryExtentsV3(); + else + deleteDctnryExtentsV4(); } //------------------------------------------------------------------------------ @@ -1233,27 +1128,27 @@ void BulkRollbackMgr::deleteDctnryExtents ( ) // So this adaptor function strips out the unnecessary trailing DSTORE2 // records, to be compatible with the Version4 format. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteDctnryExtentsV3 ( ) +void BulkRollbackMgr::deleteDctnryExtentsV3() { - for (unsigned i = 0; i < fPendingDctnryExtents.size(); ++i) + for (unsigned i = 0; i < fPendingDctnryExtents.size(); ++i) + { + if (!fPendingDctnryExtents[i].fWithHwm) { - if (!fPendingDctnryExtents[i].fWithHwm) - { - if (i == 0) - { - fPendingDctnryExtents[i].fSegNum = 0; - fPendingDctnryExtents.resize(1); - } - else - { - fPendingDctnryExtents.resize(i); - } + if (i == 0) + { + fPendingDctnryExtents[i].fSegNum = 0; + fPendingDctnryExtents.resize(1); + } + else + { + fPendingDctnryExtents.resize(i); + } - break; - } + break; } + } - deleteDctnryExtentsV4( ); + deleteDctnryExtentsV4(); } //@bug 4091: V4 support for adding DBRoot @@ -1264,260 +1159,224 @@ void BulkRollbackMgr::deleteDctnryExtentsV3 ( ) // specific column OID), read from a meta-data bulk rollback file. This // function is limited to rolling back the extent changes to a specific DBRoot. //------------------------------------------------------------------------------ -void BulkRollbackMgr::deleteDctnryExtentsV4 ( ) +void BulkRollbackMgr::deleteDctnryExtentsV4() { - std::ostringstream msg0073Text; - msg0073Text << "DBRoot-" << fPendingDctnryStoreDbRoot << - " (dictionary extent)"; - logAMessage( logging::LOG_TYPE_INFO, - logging::M0073, fPendingDctnryStoreOID, msg0073Text.str() ); + std::ostringstream msg0073Text; + msg0073Text << "DBRoot-" << fPendingDctnryStoreDbRoot << " (dictionary extent)"; + logAMessage(logging::LOG_TYPE_INFO, logging::M0073, fPendingDctnryStoreOID, msg0073Text.str()); - if (fPendingDctnryExtents.size() == 0) - return; + if (fPendingDctnryExtents.size() == 0) + return; - std::vector segNums; - std::vector hwms; + std::vector segNums; + std::vector hwms; - // Build up list of HWM's to be sent to DBRM for extentmap rollback - for (unsigned i = 0; i < fPendingDctnryExtents.size(); i++) - { - if ( !fPendingDctnryExtents[i].fWithHwm ) - break; + // Build up list of HWM's to be sent to DBRM for extentmap rollback + for (unsigned i = 0; i < fPendingDctnryExtents.size(); i++) + { + if (!fPendingDctnryExtents[i].fWithHwm) + break; - segNums.push_back( fPendingDctnryExtents[i].fSegNum ); - hwms.push_back( fPendingDctnryExtents[i].fHwm ); - } + segNums.push_back(fPendingDctnryExtents[i].fSegNum); + hwms.push_back(fPendingDctnryExtents[i].fHwm); + } - // @bug 5644 - If user dropped all partitions in a dbroot, fPartNum will - // be 0, but we may start importing into part# > 0 (to line up with other - // DBRoots). Use extent map to find first partition added by this import. - uint32_t part1 = fPendingDctnryExtents[0].fPartNum; // lowest part# for + // @bug 5644 - If user dropped all partitions in a dbroot, fPartNum will + // be 0, but we may start importing into part# > 0 (to line up with other + // DBRoots). Use extent map to find first partition added by this import. + uint32_t part1 = fPendingDctnryExtents[0].fPartNum; // lowest part# for - // OID and DBRoot - if (hwms.size() == 0) // empty DBRoot case - { - std::vector extEntries; - int rc = BRMWrapper::getInstance()->getExtents_dbroot( - fPendingDctnryStoreOID, - extEntries, - fPendingDctnryStoreDbRoot ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error getting extent list from extent map for " << - fPendingDctnryStoreOID << - "; dbRoot-" << fPendingDctnryStoreDbRoot << - "; " << ec.errorString(rc); - - throw WeException( oss.str(), ERR_BRM_BULK_RB_COLUMN ); - } - - if (extEntries.size() > 0) - { - part1 = extEntries[0].partitionNum; - - for (unsigned int kk = 0; kk < extEntries.size(); kk++) - { - if (extEntries[kk].partitionNum < part1) - part1 = extEntries[kk].partitionNum; - } - } - } - - // Delete extents from the extentmap using hwms vector - std::ostringstream msg0074Text; - msg0074Text << "Restoring HWM dictionary store extents: " - "dbRoot-" << fPendingDctnryStoreDbRoot << - "; part#-" << fPendingDctnryExtents[0].fPartNum << - "; HWM(s): "; - - for (unsigned int k = 0; k < hwms.size(); k++) - { - if (k > 0) - msg0074Text << ", "; - - msg0074Text << hwms[k]; - } - - if (hwms.size() == 0) - msg0074Text << "; delete starting at part#-" << part1; - - logAMessage( logging::LOG_TYPE_INFO, - logging::M0074, fPendingDctnryStoreOID, msg0074Text.str() ); - fAllColDctOIDs.insert( fPendingDctnryStoreOID ); - - // Reset partNum to partNum taken from extent map - uint32_t partNum = part1; - - // Create the object responsible for restoring the extents in the db files. - BulkRollbackFile* fileRestorer = makeFileRestorer( - fPendingDctnryStoreCompressionType); - boost::scoped_ptr refBulkRollbackFile(fileRestorer); - - // DMC-We should probably change this to build up a list of BRM changes, - // and wait to make the call(s) to rollback the BRM changes "after" we - // have restored the db files, and purged PrimProc FD and block cache. - int rc = BRMWrapper::getInstance()->rollbackDictStoreExtents_DBroot ( - fPendingDctnryStoreOID, - (uint16_t)fPendingDctnryStoreDbRoot, - partNum, - segNums, - hwms ); + // OID and DBRoot + if (hwms.size() == 0) // empty DBRoot case + { + std::vector extEntries; + int rc = BRMWrapper::getInstance()->getExtents_dbroot(fPendingDctnryStoreOID, extEntries, + fPendingDctnryStoreDbRoot); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error rolling back dictionary extents from extent map for " << - fPendingDctnryStoreOID << - "; partNum-" << partNum << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Error getting extent list from extent map for " << fPendingDctnryStoreOID << "; dbRoot-" + << fPendingDctnryStoreDbRoot << "; " << ec.errorString(rc); - throw WeException( oss.str(), ERR_BRM_BULK_RB_DCTNRY ); + throw WeException(oss.str(), ERR_BRM_BULK_RB_COLUMN); } - // Assign constants used later in calculating exact rollback point - const unsigned COL_WIDTH = 8; - const unsigned ROWS_PER_EXTENT = BRMWrapper::getInstance()->getExtentRows(); - const unsigned BLKS_PER_EXTENT = - (ROWS_PER_EXTENT * COL_WIDTH) / BYTE_PER_BLOCK; - - uint32_t dbRoot = fPendingDctnryStoreDbRoot; - std::string segFileListErrMsg; - - // Delete extents from the database files. - // Loop through all partitions (starting with the HWM partition fPartNum), - // deleting or restoring applicable extents. We stay in loop till we - // reach a partition that has no dctnry store segment files to roll back. - bool useHdfs = IDBPolicy::useHdfs(); - - while ( 1 ) + if (extEntries.size() > 0) { - std::vector segList; - std::string dirName; - rc = fileRestorer->buildDirName( fPendingDctnryStoreOID, - dbRoot, partNum, dirName ); + part1 = extEntries[0].partitionNum; + + for (unsigned int kk = 0; kk < extEntries.size(); kk++) + { + if (extEntries[kk].partitionNum < part1) + part1 = extEntries[kk].partitionNum; + } + } + } + + // Delete extents from the extentmap using hwms vector + std::ostringstream msg0074Text; + msg0074Text << "Restoring HWM dictionary store extents: " + "dbRoot-" + << fPendingDctnryStoreDbRoot << "; part#-" << fPendingDctnryExtents[0].fPartNum << "; HWM(s): "; + + for (unsigned int k = 0; k < hwms.size(); k++) + { + if (k > 0) + msg0074Text << ", "; + + msg0074Text << hwms[k]; + } + + if (hwms.size() == 0) + msg0074Text << "; delete starting at part#-" << part1; + + logAMessage(logging::LOG_TYPE_INFO, logging::M0074, fPendingDctnryStoreOID, msg0074Text.str()); + fAllColDctOIDs.insert(fPendingDctnryStoreOID); + + // Reset partNum to partNum taken from extent map + uint32_t partNum = part1; + + // Create the object responsible for restoring the extents in the db files. + BulkRollbackFile* fileRestorer = makeFileRestorer(fPendingDctnryStoreCompressionType); + boost::scoped_ptr refBulkRollbackFile(fileRestorer); + + // DMC-We should probably change this to build up a list of BRM changes, + // and wait to make the call(s) to rollback the BRM changes "after" we + // have restored the db files, and purged PrimProc FD and block cache. + int rc = BRMWrapper::getInstance()->rollbackDictStoreExtents_DBroot( + fPendingDctnryStoreOID, (uint16_t)fPendingDctnryStoreDbRoot, partNum, segNums, hwms); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error rolling back dictionary extents from extent map for " << fPendingDctnryStoreOID + << "; partNum-" << partNum << "; " << ec.errorString(rc); + + throw WeException(oss.str(), ERR_BRM_BULK_RB_DCTNRY); + } + + // Assign constants used later in calculating exact rollback point + const unsigned COL_WIDTH = 8; + const unsigned ROWS_PER_EXTENT = BRMWrapper::getInstance()->getExtentRows(); + const unsigned BLKS_PER_EXTENT = (ROWS_PER_EXTENT * COL_WIDTH) / BYTE_PER_BLOCK; + + uint32_t dbRoot = fPendingDctnryStoreDbRoot; + std::string segFileListErrMsg; + + // Delete extents from the database files. + // Loop through all partitions (starting with the HWM partition fPartNum), + // deleting or restoring applicable extents. We stay in loop till we + // reach a partition that has no dctnry store segment files to roll back. + bool useHdfs = IDBPolicy::useHdfs(); + + while (1) + { + std::vector segList; + std::string dirName; + rc = fileRestorer->buildDirName(fPendingDctnryStoreOID, dbRoot, partNum, dirName); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error constructing path for dictionary " << fPendingDctnryStoreOID << "; dbRoot-" + << dbRoot << "; partition-" << partNum << "; " << ec.errorString(rc); + + throw WeException(oss.str(), rc); + } + + rc = getSegFileList(dirName, useHdfs, segList, segFileListErrMsg); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error for dictionary " << fPendingDctnryStoreOID << "; directory-" << dirName + << "; " << segFileListErrMsg << "; " << ec.errorString(rc); + + throw WeException(oss.str(), rc); + } + + if (segList.size() == 0) + break; + + for (unsigned int kk = 0; kk < segList.size(); kk++) + { + uint32_t segNum = segList[kk]; + bool reInit = false; + uint32_t segIdx = 0; + + // For each segment file found in dirName, see if the file is in + // the list of pending dictionary files to be rolled back; if the + // file is found, then roll it back, if not found, then delete it. + for (unsigned nn = 0; nn < fPendingDctnryExtents.size(); nn++) + { + if ((fPendingDctnryExtents[nn].fPartNum == partNum) && (fPendingDctnryExtents[nn].fSegNum == segNum)) + { + if (fPendingDctnryExtents[nn].fWithHwm) + { + segIdx = nn; // found corresponding file in pending list + reInit = true; + } + + break; + } + } + + if (reInit) + { + // Don't rollback an OutOfService extent in the HWM partition + bool bFound; + int extState; + rc = BRMWrapper::getInstance()->getExtentState(fPendingDctnryStoreOID, partNum, segNum, bFound, + extState); if (rc != NO_ERROR) { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error constructing path for dictionary " << - fPendingDctnryStoreOID << - "; dbRoot-" << dbRoot << - "; partition-" << partNum << - "; " << ec.errorString(rc); + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error for dctnry store " << fPendingDctnryStoreOID + << "; Unable to get extent state for part-" << partNum << "; seg-" << segNum << "; " + << ec.errorString(rc); - throw WeException( oss.str(), rc ); + throw WeException(oss.str(), rc); } - rc = getSegFileList( dirName, useHdfs, segList, segFileListErrMsg ); + if ((bFound) && (extState == BRM::EXTENTOUTOFSERVICE)) + continue; - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error for dictionary " << - fPendingDctnryStoreOID << - "; directory-" << dirName << - "; " << segFileListErrMsg << - "; " << ec.errorString(rc); + HWM hwm = fPendingDctnryExtents[segIdx].fHwm; - throw WeException( oss.str(), rc ); - } + // Determine the exact rollback point for the extent + // we are rolling back to + uint32_t lastBlkOfCurrStripe = hwm - (hwm % BLKS_PER_EXTENT) + BLKS_PER_EXTENT - 1; - if (segList.size() == 0) - break; + // Reinit last extent and truncate the remainder, + // starting with the next block following the HWM block. + fileRestorer->reInitTruncDctnryExtent(fPendingDctnryStoreOID, dbRoot, partNum, segNum, (hwm + 1), + (lastBlkOfCurrStripe - hwm)); + } + else // don't keep this segment file + { + std::string segFileName; + fileRestorer->buildSegmentFileName(fPendingDctnryStoreOID, + false, // not a column segment file + dbRoot, partNum, segNum, segFileName); - for (unsigned int kk = 0; kk < segList.size(); kk++) - { - uint32_t segNum = segList[kk]; - bool reInit = false; - uint32_t segIdx = 0; + createFileDeletionEntry(fPendingDctnryStoreOID, + false, // not a column segment file + dbRoot, partNum, segNum, segFileName); + } + } // loop thru all the potential segment files in a partition - // For each segment file found in dirName, see if the file is in - // the list of pending dictionary files to be rolled back; if the - // file is found, then roll it back, if not found, then delete it. - for (unsigned nn = 0; nn < fPendingDctnryExtents.size(); nn++) - { - if ((fPendingDctnryExtents[nn].fPartNum == partNum) && - (fPendingDctnryExtents[nn].fSegNum == segNum)) - { - if (fPendingDctnryExtents[nn].fWithHwm) - { - segIdx = nn; // found corresponding file in pending list - reInit = true; - } + partNum++; - break; - } - } + } // end of loop to go thru all partitions till we find last segment file - if (reInit) - { - // Don't rollback an OutOfService extent in the HWM partition - bool bFound; - int extState; - rc = BRMWrapper::getInstance()->getExtentState( - fPendingDctnryStoreOID, partNum, segNum, bFound, extState ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error for dctnry store " << - fPendingDctnryStoreOID << - "; Unable to get extent state for part-" << partNum << - "; seg-" << segNum << - "; " << ec.errorString(rc); - - throw WeException( oss.str(), rc ); - } - - if ((bFound) && (extState == BRM::EXTENTOUTOFSERVICE)) - continue; - - HWM hwm = fPendingDctnryExtents[segIdx].fHwm; - - // Determine the exact rollback point for the extent - // we are rolling back to - uint32_t lastBlkOfCurrStripe = hwm - - (hwm % BLKS_PER_EXTENT) + BLKS_PER_EXTENT - 1; - - // Reinit last extent and truncate the remainder, - // starting with the next block following the HWM block. - fileRestorer->reInitTruncDctnryExtent ( - fPendingDctnryStoreOID, - dbRoot, - partNum, - segNum, - (hwm + 1), - (lastBlkOfCurrStripe - hwm)); - } - else // don't keep this segment file - { - std::string segFileName; - fileRestorer->buildSegmentFileName ( fPendingDctnryStoreOID, - false, // not a column segment file - dbRoot, - partNum, - segNum, - segFileName ); - - createFileDeletionEntry( fPendingDctnryStoreOID, - false, // not a column segment file - dbRoot, partNum, segNum, - segFileName ); - } - } // loop thru all the potential segment files in a partition - - partNum++; - - } //end of loop to go thru all partitions till we find last segment file - - fPendingDctnryExtents.clear ( ); + fPendingDctnryExtents.clear(); } //------------------------------------------------------------------------------ @@ -1526,22 +1385,18 @@ void BulkRollbackMgr::deleteDctnryExtentsV4 ( ) // order. (See deleteDbFiles()). //------------------------------------------------------------------------------ //@bug 4241 Delete files in reverse order -void BulkRollbackMgr::createFileDeletionEntry( - OID columnOID, - bool fileTypeFlag, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - const std::string& segFileName ) +void BulkRollbackMgr::createFileDeletionEntry(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, + uint32_t partNum, uint32_t segNum, + const std::string& segFileName) { - File f; - f.oid = columnOID; - f.fid = ((fileTypeFlag) ? 1 : 0); // use fid for file type flag - f.fPartition = partNum; - f.fSegment = segNum; - f.fDbRoot = dbRoot; - f.fSegFileName = segFileName; - fPendingFilesToDelete.push_back( f ); + File f; + f.oid = columnOID; + f.fid = ((fileTypeFlag) ? 1 : 0); // use fid for file type flag + f.fPartition = partNum; + f.fSegment = segNum; + f.fDbRoot = dbRoot; + f.fSegFileName = segFileName; + fPendingFilesToDelete.push_back(f); } //------------------------------------------------------------------------------ @@ -1550,28 +1405,25 @@ void BulkRollbackMgr::createFileDeletionEntry( // bulk rollback failures and retries. //------------------------------------------------------------------------------ //@bug 4241 Delete files in reverse order -void BulkRollbackMgr::deleteDbFiles( ) +void BulkRollbackMgr::deleteDbFiles() { - // Okay to use a BulkRollbackFile object, because we are only calling the - // deleteSegmentFile() method which is a base class function with no - // polymorphic behavior. (In other words, we don't need to worry about - // employing a BulkRollbackFileCompressed object for a compressed column.) - BulkRollbackFile fileRestorer( this ); + // Okay to use a BulkRollbackFile object, because we are only calling the + // deleteSegmentFile() method which is a base class function with no + // polymorphic behavior. (In other words, we don't need to worry about + // employing a BulkRollbackFileCompressed object for a compressed column.) + BulkRollbackFile fileRestorer(this); - unsigned int fileCount = fPendingFilesToDelete.size(); + unsigned int fileCount = fPendingFilesToDelete.size(); - for (int i = fileCount - 1; i >= 0; --i) - { - fileRestorer.deleteSegmentFile( - fPendingFilesToDelete[i].oid, - ((fPendingFilesToDelete[i].fid > 0) ? true : false), - fPendingFilesToDelete[i].fDbRoot, - fPendingFilesToDelete[i].fPartition, - fPendingFilesToDelete[i].fSegment, - fPendingFilesToDelete[i].fSegFileName ); - } + for (int i = fileCount - 1; i >= 0; --i) + { + fileRestorer.deleteSegmentFile(fPendingFilesToDelete[i].oid, + ((fPendingFilesToDelete[i].fid > 0) ? true : false), + fPendingFilesToDelete[i].fDbRoot, fPendingFilesToDelete[i].fPartition, + fPendingFilesToDelete[i].fSegment, fPendingFilesToDelete[i].fSegFileName); + } - fPendingFilesToDelete.clear(); + fPendingFilesToDelete.clear(); } //------------------------------------------------------------------------------ @@ -1583,131 +1435,124 @@ void BulkRollbackMgr::deleteDbFiles( ) // The latter case (flag set to true) is used on connection with HDFS. //------------------------------------------------------------------------------ /* static */ -int BulkRollbackMgr::getSegFileList( - const std::string& dirName, - bool bIncludeAlternateSegFileNames, - std::vector& segList, - std::string& errMsg ) +int BulkRollbackMgr::getSegFileList(const std::string& dirName, bool bIncludeAlternateSegFileNames, + std::vector& segList, std::string& errMsg) { - const unsigned int DB_FILE_PREFIX_LEN = DB_FILE_PREFIX.length(); - segList.clear(); - std::set segSet; + const unsigned int DB_FILE_PREFIX_LEN = DB_FILE_PREFIX.length(); + segList.clear(); + std::set segSet; - // Return no segment files if partition directory path does not exist - if ( !IDBPolicy::isDir( dirName.c_str() ) ) - return NO_ERROR; + // Return no segment files if partition directory path does not exist + if (!IDBPolicy::isDir(dirName.c_str())) + return NO_ERROR; - list dircontents; + list dircontents; - if ( IDBPolicy::listDirectory( dirName.c_str(), dircontents ) == 0 ) + if (IDBPolicy::listDirectory(dirName.c_str(), dircontents) == 0) + { + list::iterator iend = dircontents.end(); + + for (list::iterator i = dircontents.begin(); i != iend; ++i) { - list::iterator iend = dircontents.end(); - - for ( list::iterator i = dircontents.begin(); i != iend; ++i ) - { - boost::filesystem::path filepath( *i ); + boost::filesystem::path filepath(*i); #if BOOST_VERSION >= 105200 - //@bug 4989 - stem() and extension() return a temp path object by - // value so be sure to store in a string and not a string reference. - const std::string fileBase = filepath.stem().generic_string(); - const std::string fileExt = filepath.extension().generic_string(); + //@bug 4989 - stem() and extension() return a temp path object by + // value so be sure to store in a string and not a string reference. + const std::string fileBase = filepath.stem().generic_string(); + const std::string fileExt = filepath.extension().generic_string(); #else - const std::string& fileBase = filepath.stem(); - const std::string& fileExt = filepath.extension(); + const std::string& fileBase = filepath.stem(); + const std::string& fileExt = filepath.extension(); #endif - //std::cout << "getSegFileList: " << fileBase << " / " << - // fileExt << std::endl; + // std::cout << "getSegFileList: " << fileBase << " / " << + // fileExt << std::endl; - // Select files of interest ("FILE*.cdf") - bool bMatchFound = false; - unsigned int segNumStrLen = 0; + // Select files of interest ("FILE*.cdf") + bool bMatchFound = false; + unsigned int segNumStrLen = 0; - if (fileBase.compare(0, DB_FILE_PREFIX_LEN, DB_FILE_PREFIX) == 0) - { - segNumStrLen = fileBase.length() - DB_FILE_PREFIX_LEN; + if (fileBase.compare(0, DB_FILE_PREFIX_LEN, DB_FILE_PREFIX) == 0) + { + segNumStrLen = fileBase.length() - DB_FILE_PREFIX_LEN; - // Select primary "FILE*.cdf" files - if (fileExt == DB_FILE_EXTENSION) - { - bMatchFound = true; - //std::cout << "getSegFileList: match *.cdf" << std::endl; - } - // Select alternate files of interest ("FILE*.cdf.orig" and - // "FILE*.cdf.tmp") used for HDFS backup and rollback. - else if (bIncludeAlternateSegFileNames) - { - if ((fileExt == DB_FILE_EXTENSION_ORIG) || - (fileExt == DB_FILE_EXTENSION_TMP)) - { - //std::cout << "getSegFileList: match *.tmp or *.orig"<< - // std::endl; - unsigned int extLen = DB_FILE_EXTENSION.length(); - - if ((fileBase.length() >= extLen) && - (fileBase.compare((fileBase.length() - extLen), - extLen, - DB_FILE_EXTENSION) == 0)) - { - //std::cout << "getSegFileList: match *cdf.tmp or " - // "*cdf.orig" << std::endl; - bMatchFound = true; - segNumStrLen -= extLen; - } - } - } - } // if fileBase.compare() shows filename starting with "FILE" - - if (bMatchFound) - { - const std::string& fileSeg = fileBase.substr( - DB_FILE_PREFIX_LEN, segNumStrLen); - bool bDbFile = true; - - const unsigned fileSegLen = fileSeg.length(); - - if (fileSegLen >= 3) - { - for (unsigned int k = 0; k < fileSegLen; k++) - { - if ( !isdigit(fileSeg[k]) ) - { - bDbFile = false; - break; - } - } - - if (bDbFile) - { - uint32_t segNum = atoi( fileSeg.c_str() ); - segSet.insert( segNum ); - } - } // filename must have 3 or more digits representing seg number - } // found "FILE*.cdf", "FILE*.cdf.orig", or "FILE*.cdf.tmp" file - } // for each file in directory - - if (segSet.size() > 0) + // Select primary "FILE*.cdf" files + if (fileExt == DB_FILE_EXTENSION) { - std::set::const_iterator iter = segSet.begin(); - - while (iter != segSet.end()) - { - //std::cout << "getSegFileList: Adding segnum " << *iter << - // " to the segment list" << std::endl; - segList.push_back(*iter); - ++iter; - } + bMatchFound = true; + // std::cout << "getSegFileList: match *.cdf" << std::endl; } + // Select alternate files of interest ("FILE*.cdf.orig" and + // "FILE*.cdf.tmp") used for HDFS backup and rollback. + else if (bIncludeAlternateSegFileNames) + { + if ((fileExt == DB_FILE_EXTENSION_ORIG) || (fileExt == DB_FILE_EXTENSION_TMP)) + { + // std::cout << "getSegFileList: match *.tmp or *.orig"<< + // std::endl; + unsigned int extLen = DB_FILE_EXTENSION.length(); - return NO_ERROR; + if ((fileBase.length() >= extLen) && + (fileBase.compare((fileBase.length() - extLen), extLen, DB_FILE_EXTENSION) == 0)) + { + // std::cout << "getSegFileList: match *cdf.tmp or " + // "*cdf.orig" << std::endl; + bMatchFound = true; + segNumStrLen -= extLen; + } + } + } + } // if fileBase.compare() shows filename starting with "FILE" - } // if listDirectory() success - else + if (bMatchFound) + { + const std::string& fileSeg = fileBase.substr(DB_FILE_PREFIX_LEN, segNumStrLen); + bool bDbFile = true; + + const unsigned fileSegLen = fileSeg.length(); + + if (fileSegLen >= 3) + { + for (unsigned int k = 0; k < fileSegLen; k++) + { + if (!isdigit(fileSeg[k])) + { + bDbFile = false; + break; + } + } + + if (bDbFile) + { + uint32_t segNum = atoi(fileSeg.c_str()); + segSet.insert(segNum); + } + } // filename must have 3 or more digits representing seg number + } // found "FILE*.cdf", "FILE*.cdf.orig", or "FILE*.cdf.tmp" file + } // for each file in directory + + if (segSet.size() > 0) { - segList.clear(); - errMsg = "Unable to listDirectory in getSegFileList()"; - return ERR_BULK_ROLLBACK_SEG_LIST; + std::set::const_iterator iter = segSet.begin(); + + while (iter != segSet.end()) + { + // std::cout << "getSegFileList: Adding segnum " << *iter << + // " to the segment list" << std::endl; + segList.push_back(*iter); + ++iter; + } } + + return NO_ERROR; + + } // if listDirectory() success + else + { + segList.clear(); + errMsg = "Unable to listDirectory in getSegFileList()"; + return ERR_BULK_ROLLBACK_SEG_LIST; + } } //------------------------------------------------------------------------------ @@ -1716,21 +1561,21 @@ int BulkRollbackMgr::getSegFileList( //------------------------------------------------------------------------------ BulkRollbackFile* BulkRollbackMgr::makeFileRestorer(int compressionType) { - BulkRollbackFile* fileRestorer = 0; + BulkRollbackFile* fileRestorer = 0; - if (compressionType) - { - if (IDBPolicy::useHdfs()) - fileRestorer = new BulkRollbackFileCompressedHdfs(this); - else - fileRestorer = new BulkRollbackFileCompressed(this); - } + if (compressionType) + { + if (IDBPolicy::useHdfs()) + fileRestorer = new BulkRollbackFileCompressedHdfs(this); else - { - fileRestorer = new BulkRollbackFile(this); - } + fileRestorer = new BulkRollbackFileCompressed(this); + } + else + { + fileRestorer = new BulkRollbackFile(this); + } - return fileRestorer; + return fileRestorer; } //------------------------------------------------------------------------------ @@ -1741,255 +1586,239 @@ BulkRollbackFile* BulkRollbackMgr::makeFileRestorer(int compressionType) // columnOID - column OID associated with this rollback message // text - message text //------------------------------------------------------------------------------ -void BulkRollbackMgr::logAMessage ( - logging::LOG_TYPE logType, - logging::Message::MessageID msgId, - OID columnOID, - const std::string& text ) +void BulkRollbackMgr::logAMessage(logging::LOG_TYPE logType, logging::Message::MessageID msgId, OID columnOID, + const std::string& text) { - logging::Message m( msgId ); - logging::Message::Args args; + logging::Message m(msgId); + logging::Message::Args args; - std::ostringstream ossTbl; - ossTbl << fTableName << " (OID-" << fTableOID << ")"; - args.add( ossTbl.str() ); + std::ostringstream ossTbl; + ossTbl << fTableName << " (OID-" << fTableOID << ")"; + args.add(ossTbl.str()); - if (msgId >= logging::M0073) - { - switch (msgId) - { - case logging::M0073: - case logging::M0074: - case logging::M0075: - { - args.add( (uint64_t)columnOID ); - break; - } - - case logging::M0084: - { - args.add( (uint64_t)fLockID ); - break; - } - - case logging::M0085: - { - args.add( (uint64_t)fLockID ); - break; - } - - case logging::M0090: - { - // no other arg applicable for this message - break; - } - } - - args.add( text ); - } - - m.format( args ); - - // Log to syslog - // Note that WARNING, ERROR and CRITICAL are logged to INFO as well as - // their respective log files, so that the message will appear in context - // with all the other INFO msgs used to track the flow of the rollback. - switch (logType) - { - case logging::LOG_TYPE_DEBUG: - { - fSysLogger.logDebugMessage( m ); - break; - } - - case logging::LOG_TYPE_INFO: - { - fSysLogger.logInfoMessage( m ); - break; - } - - case logging::LOG_TYPE_WARNING: - { - fSysLogger.logWarningMessage( m ); - fSysLogger.logInfoMessage ( m ); - break; - } - - case logging::LOG_TYPE_ERROR: - { - fSysLogger.logErrorMessage( m ); - fSysLogger.logInfoMessage ( m ); - break; - } - - default: // LOG_TYPE_CRITICAL - { - fSysLogger.logCriticalMessage( m ); - fSysLogger.logInfoMessage ( m ); - break; - } - } - - // If fLog is defined then log to there, else log to cout. - // Currently log msg0074 and msg0075 to console only if debug is enabled + if (msgId >= logging::M0073) + { switch (msgId) { - // Log the name of the table to be rolled back or restored - case logging::M0072: - { - if (fLog) - { - std::ostringstream oss; - oss << "Rolling back extents for table " << - fTableName << " (OID-" << fTableOID << ")"; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - std::cout << "Rolling back extents for table " << - fTableName << " (OID-" << fTableOID << ")" << std::endl; - } + case logging::M0073: + case logging::M0074: + case logging::M0075: + { + args.add((uint64_t)columnOID); + break; + } - break; - } + case logging::M0084: + { + args.add((uint64_t)fLockID); + break; + } - // Log the name of the table and column to be rolled back or restored - case logging::M0073: - { - if (fLog) - { - std::ostringstream oss; - oss << "Rolling back extents for table " << - fTableName << " (OID-" << fTableOID << - "); column " << columnOID << "; " << text; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - std::cout << "Rolling back extents for table " << - fTableName << " (OID-" << fTableOID << - "); column " << columnOID << "; " << text << std::endl; - } + case logging::M0085: + { + args.add((uint64_t)fLockID); + break; + } - break; - } - - // Log the rolling back of extent(s) from the extent map - case logging::M0074: - { - if (fLog) - { - std::ostringstream oss; - oss << "Rolling back extent map for table " << - fTableName << " (OID-" << fTableOID << - "); column " << columnOID << "; " << text; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - if ( fDebugConsole ) - { - std::cout << "Rolling back extent map for table " << - fTableName << " (OID-" << fTableOID << - "); column " << columnOID << "; " << text << std::endl; - } - } - - break; - } - - // Log the rolling back of extent(s) from the DB - case logging::M0075: - { - if (fLog) - { - std::ostringstream oss; - oss << "Rolling back db file for table " << - fTableName << " (OID-" << fTableOID << - "); column " << columnOID << "; " << text; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - if ( fDebugConsole ) - { - std::cout << "Rolling back db file for table " << - fTableName << " (OID-" << fTableOID << - "); column " << columnOID << "; " << text << std::endl; - } - } - - break; - } - - // Log the start of a bulk rollback - case logging::M0084: - { - if (fLog) - { - std::ostringstream oss; - oss << "Starting bulk rollback for table " << - fTableName << " (OID-" << fTableOID << - ", Lock-" << fLockID << ") in " << text; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - std::cout << "Starting bulk rollback for table " << - fTableName << " (OID-" << fTableOID << - ", Lock-" << fLockID << ") in " << text << std::endl; - } - - break; - } - - // Log the end of a bulk rollback - case logging::M0085: - { - if (fLog) - { - std::ostringstream oss; - oss << "Ending bulk rollback for table " << - fTableName << " (OID-" << fTableOID << - ", Lock-" << fLockID << ") in " << text; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - std::cout << "Ending bulk rollback for table " << - fTableName << " (OID-" << fTableOID << - ", Lock-" << fLockID << ") in " << text << std::endl; - } - - break; - } - - // Log skipping a DBRoot rollback because no meta-data file - case logging::M0090: - { - if (fLog) - { - std::ostringstream oss; - oss << "Nothing to rollback for table " << - fTableName << " (OID-" << fTableOID << - ") on DBRoot" << text; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - std::cout << "Nothing to rollback for table " << - fTableName << " (OID-" << fTableOID << - ") on DBRoot" << text << std::endl; - } - - break; - } - - default: - { - break; - } + case logging::M0090: + { + // no other arg applicable for this message + break; + } } + + args.add(text); + } + + m.format(args); + + // Log to syslog + // Note that WARNING, ERROR and CRITICAL are logged to INFO as well as + // their respective log files, so that the message will appear in context + // with all the other INFO msgs used to track the flow of the rollback. + switch (logType) + { + case logging::LOG_TYPE_DEBUG: + { + fSysLogger.logDebugMessage(m); + break; + } + + case logging::LOG_TYPE_INFO: + { + fSysLogger.logInfoMessage(m); + break; + } + + case logging::LOG_TYPE_WARNING: + { + fSysLogger.logWarningMessage(m); + fSysLogger.logInfoMessage(m); + break; + } + + case logging::LOG_TYPE_ERROR: + { + fSysLogger.logErrorMessage(m); + fSysLogger.logInfoMessage(m); + break; + } + + default: // LOG_TYPE_CRITICAL + { + fSysLogger.logCriticalMessage(m); + fSysLogger.logInfoMessage(m); + break; + } + } + + // If fLog is defined then log to there, else log to cout. + // Currently log msg0074 and msg0075 to console only if debug is enabled + switch (msgId) + { + // Log the name of the table to be rolled back or restored + case logging::M0072: + { + if (fLog) + { + std::ostringstream oss; + oss << "Rolling back extents for table " << fTableName << " (OID-" << fTableOID << ")"; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + std::cout << "Rolling back extents for table " << fTableName << " (OID-" << fTableOID << ")" + << std::endl; + } + + break; + } + + // Log the name of the table and column to be rolled back or restored + case logging::M0073: + { + if (fLog) + { + std::ostringstream oss; + oss << "Rolling back extents for table " << fTableName << " (OID-" << fTableOID << "); column " + << columnOID << "; " << text; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + std::cout << "Rolling back extents for table " << fTableName << " (OID-" << fTableOID << "); column " + << columnOID << "; " << text << std::endl; + } + + break; + } + + // Log the rolling back of extent(s) from the extent map + case logging::M0074: + { + if (fLog) + { + std::ostringstream oss; + oss << "Rolling back extent map for table " << fTableName << " (OID-" << fTableOID << "); column " + << columnOID << "; " << text; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + if (fDebugConsole) + { + std::cout << "Rolling back extent map for table " << fTableName << " (OID-" << fTableOID + << "); column " << columnOID << "; " << text << std::endl; + } + } + + break; + } + + // Log the rolling back of extent(s) from the DB + case logging::M0075: + { + if (fLog) + { + std::ostringstream oss; + oss << "Rolling back db file for table " << fTableName << " (OID-" << fTableOID << "); column " + << columnOID << "; " << text; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + if (fDebugConsole) + { + std::cout << "Rolling back db file for table " << fTableName << " (OID-" << fTableOID + << "); column " << columnOID << "; " << text << std::endl; + } + } + + break; + } + + // Log the start of a bulk rollback + case logging::M0084: + { + if (fLog) + { + std::ostringstream oss; + oss << "Starting bulk rollback for table " << fTableName << " (OID-" << fTableOID << ", Lock-" + << fLockID << ") in " << text; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + std::cout << "Starting bulk rollback for table " << fTableName << " (OID-" << fTableOID << ", Lock-" + << fLockID << ") in " << text << std::endl; + } + + break; + } + + // Log the end of a bulk rollback + case logging::M0085: + { + if (fLog) + { + std::ostringstream oss; + oss << "Ending bulk rollback for table " << fTableName << " (OID-" << fTableOID << ", Lock-" + << fLockID << ") in " << text; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + std::cout << "Ending bulk rollback for table " << fTableName << " (OID-" << fTableOID << ", Lock-" + << fLockID << ") in " << text << std::endl; + } + + break; + } + + // Log skipping a DBRoot rollback because no meta-data file + case logging::M0090: + { + if (fLog) + { + std::ostringstream oss; + oss << "Nothing to rollback for table " << fTableName << " (OID-" << fTableOID << ") on DBRoot" + << text; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + std::cout << "Nothing to rollback for table " << fTableName << " (OID-" << fTableOID << ") on DBRoot" + << text << std::endl; + } + + break; + } + + default: + { + break; + } + } } //------------------------------------------------------------------------------ @@ -1998,35 +1827,35 @@ void BulkRollbackMgr::logAMessage ( // remove_all() ) //------------------------------------------------------------------------------ /* static */ -void BulkRollbackMgr::deleteMetaFile( OID tableOID ) +void BulkRollbackMgr::deleteMetaFile(OID tableOID) { - std::vector dbRoots; - Config::getRootIdList( dbRoots ); + std::vector dbRoots; + Config::getRootIdList(dbRoots); - // Loop through DBRoots for this PM - for (unsigned m = 0; m < dbRoots.size(); m++) - { - std::string bulkRollbackPath( Config::getDBRootByNum(dbRoots[m]) ); + // Loop through DBRoots for this PM + for (unsigned m = 0; m < dbRoots.size(); m++) + { + std::string bulkRollbackPath(Config::getDBRootByNum(dbRoots[m])); - std::ostringstream oss; - oss << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << tableOID; - std::string metaFileName = bulkRollbackPath; - metaFileName += oss.str(); + std::ostringstream oss; + oss << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << tableOID; + std::string metaFileName = bulkRollbackPath; + metaFileName += oss.str(); - // Delete the main bulk rollback file - IDBPolicy::remove( metaFileName.c_str() ); + // Delete the main bulk rollback file + IDBPolicy::remove(metaFileName.c_str()); - // Unlink corresponding tmp file created by RBMetaWriter. - std::string tmpMetaFileName = metaFileName; - tmpMetaFileName += TMP_FILE_SUFFIX; - IDBPolicy::remove( tmpMetaFileName.c_str() ); + // Unlink corresponding tmp file created by RBMetaWriter. + std::string tmpMetaFileName = metaFileName; + tmpMetaFileName += TMP_FILE_SUFFIX; + IDBPolicy::remove(tmpMetaFileName.c_str()); - // Recursively delete any HWM chunk backup files - std::string bulkRollbackSubPath( metaFileName ); - bulkRollbackSubPath += DATA_DIR_SUFFIX; + // Recursively delete any HWM chunk backup files + std::string bulkRollbackSubPath(metaFileName); + bulkRollbackSubPath += DATA_DIR_SUFFIX; - IDBPolicy::remove( bulkRollbackSubPath.c_str() ); - } + IDBPolicy::remove(bulkRollbackSubPath.c_str()); + } } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_bulkrollbackmgr.h b/writeengine/shared/we_bulkrollbackmgr.h index 433dcdadd..c4c9331cf 100644 --- a/writeengine/shared/we_bulkrollbackmgr.h +++ b/writeengine/shared/we_bulkrollbackmgr.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_bulkrollbackmgr.h 4726 2013-08-07 03:38:36Z bwilkinson $ -*/ + * $Id: we_bulkrollbackmgr.h 4726 2013-08-07 03:38:36Z bwilkinson $ + */ /** @file * Contains class to clear a database table lock, and rolls back extents @@ -60,178 +60,167 @@ class BulkRollbackFile; //------------------------------------------------------------------------------ class BulkRollbackMgr { - enum BulkRollbackVersion - { - BULK_RB_VERSION_OTHER, - BULK_RB_VERSION3 = 3, - BULK_RB_VERSION4 = 4 - }; + enum BulkRollbackVersion + { + BULK_RB_VERSION_OTHER, + BULK_RB_VERSION3 = 3, + BULK_RB_VERSION4 = 4 + }; -public: - /** - * @brief BulkRollbackMgr constructor - * @param tableOID table to be rolled back. - * @param lockID Table lock id of the table to be rolled back. - * Currently used for logging only. - * @param tableName name of table associated with tableOID. - * Currently used for logging only. - * @param applName application that is driving this bulk rollback. - * Currently used for logging only. - */ - EXPORT BulkRollbackMgr(OID tableOID, - uint64_t lockID, - const std::string& tableName, - const std::string& applName, - Log* logger = 0); + public: + /** + * @brief BulkRollbackMgr constructor + * @param tableOID table to be rolled back. + * @param lockID Table lock id of the table to be rolled back. + * Currently used for logging only. + * @param tableName name of table associated with tableOID. + * Currently used for logging only. + * @param applName application that is driving this bulk rollback. + * Currently used for logging only. + */ + EXPORT BulkRollbackMgr(OID tableOID, uint64_t lockID, const std::string& tableName, + const std::string& applName, Log* logger = 0); - /** - * @brief BulkRollbackMgr destructor - */ - EXPORT ~BulkRollbackMgr( ) - { - closeMetaDataFile ( ); - } + /** + * @brief BulkRollbackMgr destructor + */ + EXPORT ~BulkRollbackMgr() + { + closeMetaDataFile(); + } - /** - * @brief Clear table lock and rollback extents for fTableOID - * @param keepMetaFile controls whether the meta data file is deleted - * @return NO_ERROR upon success - */ - EXPORT int rollback ( bool keepMetaFile ); + /** + * @brief Clear table lock and rollback extents for fTableOID + * @param keepMetaFile controls whether the meta data file is deleted + * @return NO_ERROR upon success + */ + EXPORT int rollback(bool keepMetaFile); - /** - * @brief Accessor to any error msg related to a bad return code. - * @return error message if rollback rejected or failed. - */ - const std::string& getErrorMsg( ) const - { - return fErrorMsg; - } + /** + * @brief Accessor to any error msg related to a bad return code. + * @return error message if rollback rejected or failed. + */ + const std::string& getErrorMsg() const + { + return fErrorMsg; + } - /** - * @brief Accessor to the name of the meta file we are processing - */ - const std::string& getMetaFileName() const - { - return fMetaFileName; - } + /** + * @brief Accessor to the name of the meta file we are processing + */ + const std::string& getMetaFileName() const + { + return fMetaFileName; + } - /** - * @brief Mutator to enable/disable debug logging to console. - */ - void setDebugConsole ( bool debug ) - { - fDebugConsole = debug; - } + /** + * @brief Mutator to enable/disable debug logging to console. + */ + void setDebugConsole(bool debug) + { + fDebugConsole = debug; + } - /** - * @brief Log the specified message. - * @param logType type of message to be logged - * @param msgId message id to be used - * @param columnOID column OID - * @param text message text to be logged - */ - void logAMessage ( logging::LOG_TYPE logType, //log a message - logging::Message::MessageID msgId, - OID columnOID, - const std::string& text ); - /** - * @brief Standalone utility that can be used to delete the bulk rollback - * meta data files. Caution: this function can throw an exception. - * @param tableOID Bulk rollback files for this table are to be deleted - */ - EXPORT static void deleteMetaFile( OID tableOID ); + /** + * @brief Log the specified message. + * @param logType type of message to be logged + * @param msgId message id to be used + * @param columnOID column OID + * @param text message text to be logged + */ + void logAMessage(logging::LOG_TYPE logType, // log a message + logging::Message::MessageID msgId, OID columnOID, const std::string& text); + /** + * @brief Standalone utility that can be used to delete the bulk rollback + * meta data files. Caution: this function can throw an exception. + * @param tableOID Bulk rollback files for this table are to be deleted + */ + EXPORT static void deleteMetaFile(OID tableOID); - /* - * @brief Get list of segment file numbers found in dirName directory - * @param dirName Directory path to be searched - * @param bIncludeAlternateSegFileNames Include *.orig and *.tmp in search - * @param segList List of segment files found in dirName - * @param errMsg Error msg if return code is not NO_ERROR - */ - EXPORT static int getSegFileList( const std::string& dirName, - bool bIncludeAlternateSegFileNames, - std::vector& segList, - std::string& errMsg ); + /* + * @brief Get list of segment file numbers found in dirName directory + * @param dirName Directory path to be searched + * @param bIncludeAlternateSegFileNames Include *.orig and *.tmp in search + * @param segList List of segment files found in dirName + * @param errMsg Error msg if return code is not NO_ERROR + */ + EXPORT static int getSegFileList(const std::string& dirName, bool bIncludeAlternateSegFileNames, + std::vector& segList, std::string& errMsg); -private: - // Declare but don't define copy constructor and assignment operator - BulkRollbackMgr(const BulkRollbackMgr& rhs); - BulkRollbackMgr& operator=(const BulkRollbackMgr& rhs); + private: + // Declare but don't define copy constructor and assignment operator + BulkRollbackMgr(const BulkRollbackMgr& rhs); + BulkRollbackMgr& operator=(const BulkRollbackMgr& rhs); - // Structure used to store info for the list of dictionary store - // segment files in the last partition. - struct RollbackData - { - uint32_t fDbRoot; - uint32_t fPartNum; - uint32_t fSegNum; - HWM fHwm; - bool fWithHwm; - }; + // Structure used to store info for the list of dictionary store + // segment files in the last partition. + struct RollbackData + { + uint32_t fDbRoot; + uint32_t fPartNum; + uint32_t fSegNum; + HWM fHwm; + bool fWithHwm; + }; - void createFileDeletionEntry( OID columnOID, - bool fileTypeFlag, - uint32_t dbRoot, - uint32_t partNum, - uint32_t segNum, - const std::string& segFileName ); - void deleteColumn1Extents ( const char* inBuf ); // delete col extents - void deleteColumn1ExtentsV3(const char* inBuf ); - void deleteColumn1ExtentsV4(const char* inBuf ); - void deleteColumn2Extents ( const char* inBuf ); // delete col extents - void deleteColumn2ExtentsV3(const char* inBuf ); - void deleteColumn2ExtentsV4(const char* inBuf ); - void deleteDbFiles ( ); // delete DB files waiting to be deleted - void deleteDctnryExtents ( ); // delete dictionary store extents - void deleteDctnryExtentsV3( ); - void deleteDctnryExtentsV4( ); - void deleteExtents ( std::istringstream& metaDataStream ); - // function that drives extent deletion - void readMetaDataRecDctnry(const char* inBuf );//read meta-data dct rec + void createFileDeletionEntry(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum, + uint32_t segNum, const std::string& segFileName); + void deleteColumn1Extents(const char* inBuf); // delete col extents + void deleteColumn1ExtentsV3(const char* inBuf); + void deleteColumn1ExtentsV4(const char* inBuf); + void deleteColumn2Extents(const char* inBuf); // delete col extents + void deleteColumn2ExtentsV3(const char* inBuf); + void deleteColumn2ExtentsV4(const char* inBuf); + void deleteDbFiles(); // delete DB files waiting to be deleted + void deleteDctnryExtents(); // delete dictionary store extents + void deleteDctnryExtentsV3(); + void deleteDctnryExtentsV4(); + void deleteExtents(std::istringstream& metaDataStream); + // function that drives extent deletion + void readMetaDataRecDctnry(const char* inBuf); // read meta-data dct rec - void deleteSubDir ( const std::string& metaFileName ); // delete - // subdirectory used for backup chunks - EXPORT void closeMetaDataFile ( ); // close a metafile - void deleteMetaDataFiles ( ); // delete metafiles - int metaDataFileExists ( bool& exists ); // does meta-data file exists - BulkRollbackFile* makeFileRestorer(int compressionType); - bool openMetaDataFile ( uint16_t dbRoot, // open a metadata file - std::istringstream& metaDataStream ); - void validateAllMetaFilesExist(const std::vector& dbRoots) const; + void deleteSubDir(const std::string& metaFileName); // delete + // subdirectory used for backup chunks + EXPORT void closeMetaDataFile(); // close a metafile + void deleteMetaDataFiles(); // delete metafiles + int metaDataFileExists(bool& exists); // does meta-data file exists + BulkRollbackFile* makeFileRestorer(int compressionType); + bool openMetaDataFile(uint16_t dbRoot, // open a metadata file + std::istringstream& metaDataStream); + void validateAllMetaFilesExist(const std::vector& dbRoots) const; - // Data members - OID fTableOID; // table to be rolled back - uint64_t fLockID; // unique lock ID associated with table lock - std::string fTableName; // name of table associated with fTableOID - uint32_t fProcessId; // pid associated with current table lock - std::string fProcessName; // processName associated with fProcessId - IDBDataFile* fMetaFile; // current meta data file we are reading - std::string fMetaFileName;// name of current meta data file - std::vector fMetaFileNames; // all relevant meta data files - std::string fErrorMsg; - unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk + // Data members + OID fTableOID; // table to be rolled back + uint64_t fLockID; // unique lock ID associated with table lock + std::string fTableName; // name of table associated with fTableOID + uint32_t fProcessId; // pid associated with current table lock + std::string fProcessName; // processName associated with fProcessId + IDBDataFile* fMetaFile; // current meta data file we are reading + std::string fMetaFileName; // name of current meta data file + std::vector fMetaFileNames; // all relevant meta data files + std::string fErrorMsg; + unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk - // Dictionary store extents for an OID are read in and managed as a - // group. The following data members are used to collect this info. - OID fPendingDctnryStoreOID;// Dctnry OID of pending dctnry extents - uint32_t fPendingDctnryStoreDbRoot; // DbRoot of pending dctnry extents - int fPendingDctnryStoreCompressionType; // Dctnry compression type - std::vector fPendingDctnryExtents; - std::set fAllColDctOIDs; // List of all affected col and dctnry OIDS + // Dictionary store extents for an OID are read in and managed as a + // group. The following data members are used to collect this info. + OID fPendingDctnryStoreOID; // Dctnry OID of pending dctnry extents + uint32_t fPendingDctnryStoreDbRoot; // DbRoot of pending dctnry extents + int fPendingDctnryStoreCompressionType; // Dctnry compression type + std::vector fPendingDctnryExtents; + std::set fAllColDctOIDs; // List of all affected col and dctnry OIDS - // List of DB Files to be deleted. Files are deleted in reverse order. - std::vector fPendingFilesToDelete; + // List of DB Files to be deleted. Files are deleted in reverse order. + std::vector fPendingFilesToDelete; - logging::MessageLog fSysLogger; // Used for syslogging - bool fDebugConsole; // control debug logging to console - Log* fLog; // optional logger object - std::string fApplName; // application initiating the bulk rollback - int fVersion; // version of meta data file being read + logging::MessageLog fSysLogger; // Used for syslogging + bool fDebugConsole; // control debug logging to console + Log* fLog; // optional logger object + std::string fApplName; // application initiating the bulk rollback + int fVersion; // version of meta data file being read }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // WE_BULKROLLBACKMGR_H_ +#endif // WE_BULKROLLBACKMGR_H_ diff --git a/writeengine/shared/we_cache.cpp b/writeengine/shared/we_cache.cpp index 227f19927..b82a7a45c 100644 --- a/writeengine/shared/we_cache.cpp +++ b/writeengine/shared/we_cache.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_cache.cpp 33 2006-10-30 13:45:13Z wzhou $ -* -******************************************************************************************/ + * $Id: we_cache.cpp 33 2006-10-30 13:45:13Z wzhou $ + * + ******************************************************************************************/ /** @file */ #include @@ -27,15 +27,14 @@ using namespace std; namespace WriteEngine { - -CacheControl* Cache::m_cacheParam = NULL; -FreeBufList* Cache::m_freeList = NULL; -CacheMap* Cache::m_lruList = NULL; -CacheMap* Cache::m_writeList = NULL; +CacheControl* Cache::m_cacheParam = NULL; +FreeBufList* Cache::m_freeList = NULL; +CacheMap* Cache::m_lruList = NULL; +CacheMap* Cache::m_writeList = NULL; #ifdef _MSC_VER __declspec(dllexport) #endif -bool Cache::m_useCache = false; + bool Cache::m_useCache = false; /*********************************************************** * DESCRIPTION: * Clear all list and free memory @@ -44,47 +43,47 @@ bool Cache::m_useCache = false; * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -void Cache::clear() +void Cache::clear() { - CacheMapIt it; - BlockBuffer* block; - size_t i; + CacheMapIt it; + BlockBuffer* block; + size_t i; - // free list - if ( m_freeList != NULL ) + // free list + if (m_freeList != NULL) + { + for (i = 0; i < m_freeList->size(); i++) { - for ( i = 0; i < m_freeList->size(); i++ ) - { - block = m_freeList->at(i); - block->clear(); - } + block = m_freeList->at(i); + block->clear(); + } + } + + // LRU list + if (m_lruList != NULL) + { + for (it = m_lruList->begin(); it != m_lruList->end(); it++) + { + block = it->second; + block->clear(); + m_freeList->push_back(block); } - // LRU list - if ( m_lruList != NULL ) - { - for ( it = m_lruList->begin(); it != m_lruList->end(); it++ ) - { - block = it->second; - block->clear(); - m_freeList->push_back( block ); - } + m_lruList->clear(); + } - m_lruList->clear(); + // Write list + if (m_writeList != NULL) + { + for (it = m_writeList->begin(); it != m_writeList->end(); it++) + { + block = it->second; + block->clear(); + m_freeList->push_back(block); } - // Write list - if ( m_writeList != NULL ) - { - for ( it = m_writeList->begin(); it != m_writeList->end(); it++ ) - { - block = it->second; - block->clear(); - m_freeList->push_back( block ); - } - - m_writeList->clear(); - } + m_writeList->clear(); + } } /*********************************************************** @@ -95,50 +94,50 @@ void Cache::clear() * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -int Cache::flushCache() +int Cache::flushCache() { - bool bHasReadBlock = false; - BlockBuffer* curBuf; + bool bHasReadBlock = false; + BlockBuffer* curBuf; - // add lock here - if ( m_lruList && m_lruList->size() > 0 ) + // add lock here + if (m_lruList && m_lruList->size() > 0) + { + bHasReadBlock = true; + + for (CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++) { - bHasReadBlock = true; - - for ( CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++ ) - { - curBuf = it->second; - curBuf->clear(); - m_freeList->push_back( curBuf ); - } - - m_lruList->clear(); + curBuf = it->second; + curBuf->clear(); + m_freeList->push_back(curBuf); } - // must write to disk first - if ( m_writeList && m_writeList->size() > 0 ) - { - if ( !bHasReadBlock ) - for ( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ ) - { - curBuf = it->second; - curBuf->clear(); - m_freeList->push_back( curBuf ); - } - else - for ( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ ) - { - curBuf = it->second; - (*curBuf).block.dirty = false; - processCacheMap( m_lruList, curBuf, INSERT ); - } + m_lruList->clear(); + } - m_writeList->clear(); + // must write to disk first + if (m_writeList && m_writeList->size() > 0) + { + if (!bHasReadBlock) + for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++) + { + curBuf = it->second; + curBuf->clear(); + m_freeList->push_back(curBuf); + } + else + for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++) + { + curBuf = it->second; + (*curBuf).block.dirty = false; + processCacheMap(m_lruList, curBuf, INSERT); + } - } // end of if( m_writeList->size() + m_writeList->clear(); - // add unlock here - return NO_ERROR; + } // end of if( m_writeList->size() + + // add unlock here + return NO_ERROR; } /*********************************************************** @@ -149,63 +148,63 @@ int Cache::flushCache() * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -void Cache::freeMemory() +void Cache::freeMemory() { - CacheMapIt it; - BlockBuffer* block; - size_t i; + CacheMapIt it; + BlockBuffer* block; + size_t i; - // free list - if ( m_freeList != NULL ) + // free list + if (m_freeList != NULL) + { + for (i = 0; i < m_freeList->size(); i++) { - for ( i = 0; i < m_freeList->size(); i++ ) - { - block = m_freeList->at(i); - block->freeMem(); - delete block; - } - - m_freeList->clear(); - delete m_freeList; - m_freeList = NULL; + block = m_freeList->at(i); + block->freeMem(); + delete block; } - // LRU list - if ( m_lruList != NULL ) - { - for ( it = m_lruList->begin(); it != m_lruList->end(); it++ ) - { - block = it->second; - block->freeMem(); - delete block; - } + m_freeList->clear(); + delete m_freeList; + m_freeList = NULL; + } - m_lruList->clear(); - delete m_lruList; - m_lruList = NULL; + // LRU list + if (m_lruList != NULL) + { + for (it = m_lruList->begin(); it != m_lruList->end(); it++) + { + block = it->second; + block->freeMem(); + delete block; } - // Write list - if ( m_writeList != NULL ) - { - for ( it = m_writeList->begin(); it != m_writeList->end(); it++ ) - { - block = it->second; - block->freeMem(); - delete block; - } + m_lruList->clear(); + delete m_lruList; + m_lruList = NULL; + } - m_writeList->clear(); - delete m_writeList; - m_writeList = NULL; + // Write list + if (m_writeList != NULL) + { + for (it = m_writeList->begin(); it != m_writeList->end(); it++) + { + block = it->second; + block->freeMem(); + delete block; } - // param - if ( m_cacheParam != NULL ) - { - delete m_cacheParam; - m_cacheParam = NULL; - } + m_writeList->clear(); + delete m_writeList; + m_writeList = NULL; + } + + // param + if (m_cacheParam != NULL) + { + delete m_cacheParam; + m_cacheParam = NULL; + } } /*********************************************************** @@ -216,30 +215,24 @@ void Cache::freeMemory() * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -int Cache::getListSize( const CacheListType listType ) +int Cache::getListSize(const CacheListType listType) { - int size = 0; - - if ( !m_useCache ) - return size; - - switch ( listType ) - { - case FREE_LIST: - size = m_freeList->size(); - break; - - case LRU_LIST: - size = m_lruList->size(); - break; - - case WRITE_LIST: - default: - size = m_writeList->size(); - break; - } + int size = 0; + if (!m_useCache) return size; + + switch (listType) + { + case FREE_LIST: size = m_freeList->size(); break; + + case LRU_LIST: size = m_lruList->size(); break; + + case WRITE_LIST: + default: size = m_writeList->size(); break; + } + + return size; } /*********************************************************** @@ -252,28 +245,28 @@ int Cache::getListSize( const CacheListType listType ) * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -void Cache::init( const int totalBlock, const int chkPoint, const int pctFree ) +void Cache::init(const int totalBlock, const int chkPoint, const int pctFree) { - BlockBuffer* buffer; + BlockBuffer* buffer; - if ( m_cacheParam && m_freeList && m_lruList && m_writeList ) - return; + if (m_cacheParam && m_freeList && m_lruList && m_writeList) + return; - m_cacheParam = new CacheControl(); - m_cacheParam->totalBlock = totalBlock; - m_cacheParam->checkInterval = chkPoint; - m_cacheParam->pctFree = pctFree; + m_cacheParam = new CacheControl(); + m_cacheParam->totalBlock = totalBlock; + m_cacheParam->checkInterval = chkPoint; + m_cacheParam->pctFree = pctFree; - m_freeList = new FreeBufList(); - m_lruList = new CacheMap(); - m_writeList = new CacheMap(); + m_freeList = new FreeBufList(); + m_lruList = new CacheMap(); + m_writeList = new CacheMap(); - for ( int i = 0; i < m_cacheParam->totalBlock; i++ ) - { - buffer = new BlockBuffer(); - buffer->init(); - m_freeList->push_back( buffer ); - } + for (int i = 0; i < m_cacheParam->totalBlock; i++) + { + buffer = new BlockBuffer(); + buffer->init(); + m_freeList->push_back(buffer); + } } /*********************************************************** @@ -287,30 +280,30 @@ void Cache::init( const int totalBlock, const int chkPoint, const int pctFree ) * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -int Cache::insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf ) +int Cache::insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf) { - BlockBuffer* buffer; - vector::iterator it; + BlockBuffer* buffer; + vector::iterator it; - if ( m_freeList->size() == 0 ) - return ERR_FREE_LIST_EMPTY; + if (m_freeList->size() == 0) + return ERR_FREE_LIST_EMPTY; - // make sure flush first if necessary - it = m_freeList->begin(); - buffer = *it; - memcpy( (*buffer).block.data, buf, BYTE_PER_BLOCK ); - (*buffer).listType = LRU_LIST; - (*buffer).block.lbid = lbid; - (*buffer).block.fbo = fbo; - (*buffer).block.dirty = false; - (*buffer).block.hitCount = 1; - (*buffer).cb.file.oid = cb.file.oid; - (*buffer).cb.file.pFile = cb.file.pFile; + // make sure flush first if necessary + it = m_freeList->begin(); + buffer = *it; + memcpy((*buffer).block.data, buf, BYTE_PER_BLOCK); + (*buffer).listType = LRU_LIST; + (*buffer).block.lbid = lbid; + (*buffer).block.fbo = fbo; + (*buffer).block.dirty = false; + (*buffer).block.hitCount = 1; + (*buffer).cb.file.oid = cb.file.oid; + (*buffer).cb.file.pFile = cb.file.pFile; - RETURN_ON_ERROR( processCacheMap( m_lruList, buffer, INSERT ) ); - m_freeList->erase( it ); + RETURN_ON_ERROR(processCacheMap(m_lruList, buffer, INSERT)); + m_freeList->erase(it); - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -322,29 +315,29 @@ int Cache::insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fb * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -int Cache::loadCacheBlock( const CacheKey& key, unsigned char* buf ) +int Cache::loadCacheBlock(const CacheKey& key, unsigned char* buf) { - BlockBuffer* buffer; - CacheMapIt iter; + BlockBuffer* buffer; + CacheMapIt iter; - iter = m_lruList->find( key ); + iter = m_lruList->find(key); - if ( iter != m_lruList->end() ) - buffer = iter->second; + if (iter != m_lruList->end()) + buffer = iter->second; + else + { + iter = m_writeList->find(key); + + if (iter != m_writeList->end()) + buffer = iter->second; else - { - iter = m_writeList->find( key ); + return ERR_CACHE_KEY_NOT_EXIST; + } - if ( iter != m_writeList->end() ) - buffer = iter->second; - else - return ERR_CACHE_KEY_NOT_EXIST; - } + memcpy(buf, (*buffer).block.data, BYTE_PER_BLOCK); + (*buffer).block.hitCount++; - memcpy( buf, (*buffer).block.data, BYTE_PER_BLOCK ); - (*buffer).block.hitCount++; - - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -356,37 +349,36 @@ int Cache::loadCacheBlock( const CacheKey& key, unsigned char* buf ) * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -int Cache::modifyCacheBlock( const CacheKey& key, const unsigned char* buf ) +int Cache::modifyCacheBlock(const CacheKey& key, const unsigned char* buf) { - BlockBuffer* buffer; - CacheMapIt iter; + BlockBuffer* buffer; + CacheMapIt iter; - iter = m_lruList->find( key ); + iter = m_lruList->find(key); - if ( iter != m_lruList->end() ) - { - buffer = iter->second; - (*buffer).listType = WRITE_LIST; - (*buffer).block.dirty = true; + if (iter != m_lruList->end()) + { + buffer = iter->second; + (*buffer).listType = WRITE_LIST; + (*buffer).block.dirty = true; - (*m_writeList)[key] = iter->second; - m_lruList->erase( iter ); + (*m_writeList)[key] = iter->second; + m_lruList->erase(iter); + } + else + { + iter = m_writeList->find(key); - } + if (iter != m_writeList->end()) + buffer = iter->second; else - { - iter = m_writeList->find( key ); + return ERR_CACHE_KEY_NOT_EXIST; + } - if ( iter != m_writeList->end() ) - buffer = iter->second; - else - return ERR_CACHE_KEY_NOT_EXIST; - } + memcpy((*buffer).block.data, buf, BYTE_PER_BLOCK); + (*buffer).block.hitCount++; - memcpy( (*buffer).block.data, buf, BYTE_PER_BLOCK ); - (*buffer).block.hitCount++; - - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -397,35 +389,35 @@ int Cache::modifyCacheBlock( const CacheKey& key, const unsigned char* buf ) * RETURN: * none ***********************************************************/ -void Cache::printCacheList() +void Cache::printCacheList() { - BlockBuffer* buffer; - int i = 0; + BlockBuffer* buffer; + int i = 0; - if ( !m_useCache ) - return; + if (!m_useCache) + return; - cout << "\nFree List has " << m_freeList->size() << " elements" << endl; - cout << "LRU List has " << m_lruList->size() << " elements" << endl; + cout << "\nFree List has " << m_freeList->size() << " elements" << endl; + cout << "LRU List has " << m_lruList->size() << " elements" << endl; - for ( CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++ ) - { - buffer = it->second; - cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType - << " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo - << " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl; - } + for (CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++) + { + buffer = it->second; + cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType + << " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo + << " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl; + } - i = 0; - cout << "Write List has " << m_writeList->size() << " elements" << endl; + i = 0; + cout << "Write List has " << m_writeList->size() << " elements" << endl; - for ( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ ) - { - buffer = it->second; - cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType - << " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo - << " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl; - } + for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++) + { + buffer = it->second; + cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType + << " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo + << " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl; + } } /*********************************************************** @@ -437,34 +429,31 @@ void Cache::printCacheList() * RETURN: * NO_ERROR if success, other otherwise ***********************************************************/ -int Cache::processCacheMap( CacheMap* map, BlockBuffer* buffer, OpType opType ) +int Cache::processCacheMap(CacheMap* map, BlockBuffer* buffer, OpType opType) { - RETURN_ON_NULL( buffer, ERR_NULL_BLOCK ); - CacheMapIt iter; + RETURN_ON_NULL(buffer, ERR_NULL_BLOCK); + CacheMapIt iter; - CacheKey key = getCacheKey( buffer ); - iter = map->find( key ); + CacheKey key = getCacheKey(buffer); + iter = map->find(key); - // only handle insert and delete - if ( iter == map->end() ) - { - if ( opType == INSERT ) - (*map)[key] = buffer; - else - return ERR_CACHE_KEY_NOT_EXIST; - } + // only handle insert and delete + if (iter == map->end()) + { + if (opType == INSERT) + (*map)[key] = buffer; else - { - if ( opType == INSERT ) - return ERR_CACHE_KEY_EXIST; - else - map->erase( iter ); - } + return ERR_CACHE_KEY_NOT_EXIST; + } + else + { + if (opType == INSERT) + return ERR_CACHE_KEY_EXIST; + else + map->erase(iter); + } - return NO_ERROR; + return NO_ERROR; } - - -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_cache.h b/writeengine/shared/we_cache.h index 70d302851..71fe96bfd 100644 --- a/writeengine/shared/we_cache.h +++ b/writeengine/shared/we_cache.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_cache.h 33 2006-10-30 13:45:13Z wzhou $ -* -******************************************************************************************/ + * $Id: we_cache.h 33 2006-10-30 13:45:13Z wzhou $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_CACHE_H_ @@ -46,185 +46,193 @@ /** Namespace WriteEngine */ namespace WriteEngine { -typedef std::vector FreeBufList; /** @brief Free buffer list */ -//typedef std::string CacheKey; /** @brief Key definition */ -typedef uint64_t CacheKey; /** @brief Key definition */ +typedef std::vector FreeBufList; /** @brief Free buffer list */ +// typedef std::string CacheKey; /** @brief Key definition */ +typedef uint64_t CacheKey; /** @brief Key definition */ -//typedef std::map > CacheMap; /** @brief Cache map */ -//typedef CacheMap::iterator CacheMapIt; /** @brief CacheMap iterator */ +// typedef std::map > CacheMap; /** @brief Cache map */ +// typedef CacheMap::iterator CacheMapIt; /** @brief CacheMap iterator */ -templatestruct hashCacheKey { }; -template<> struct hashCacheKey +template +struct hashCacheKey { - size_t - operator()(CacheKey __x) const - { - return __x; - } +}; +template <> +struct hashCacheKey +{ + size_t operator()(CacheKey __x) const + { + return __x; + } }; struct eqCacheKey { - bool operator() (const CacheKey k1, const CacheKey k2 ) const - { - return k1 == k2; - } + bool operator()(const CacheKey k1, const CacheKey k2) const + { + return k1 == k2; + } }; -//typedef hash_map, eqSig> DCTNRYHASHMAP; +// typedef hash_map, eqSig> DCTNRYHASHMAP; #if __GNUC__ == 4 && __GNUC_MINOR__ < 2 typedef __gnu_cxx::hash_map, eqCacheKey> CacheMap; #else typedef std::tr1::unordered_map, eqCacheKey> CacheMap; #endif -//typedef __gnu_cxx::hash_map CacheMap; -typedef CacheMap::iterator CacheMapIt; +// typedef __gnu_cxx::hash_map CacheMap; +typedef CacheMap::iterator CacheMapIt; -//typedef CacheMap LRUBufList; /** @brief Least Recent Used Buffer list */ -//typedef CacheMap WriteBufList; /** @brief Write buffer list */ +// typedef CacheMap LRUBufList; /** @brief Least Recent Used Buffer list */ +// typedef CacheMap WriteBufList; /** @brief Write buffer list */ /** Class Cache */ class Cache { -public: - /** - * @brief Constructor - */ - Cache() {} + public: + /** + * @brief Constructor + */ + Cache() + { + } - /** - * @brief Default Destructor - */ - ~Cache() {} + /** + * @brief Default Destructor + */ + ~Cache() + { + } - /** - * @brief Check whether cache key exists - */ - static bool cacheKeyExist( CacheMap* map, const OID oid, const uint64_t lbid ) - { - CacheKey key = getCacheKey( oid, lbid ); - return map->find(key) == map->end() ? false : true; - } - static bool cacheKeyExist( CacheMap* map, BlockBuffer* buffer ) - { - return cacheKeyExist( map, (*buffer).cb.file.oid, (*buffer).block.lbid ); - } - static bool cacheKeyExist( const OID oid, const uint64_t lbid ) - { - return cacheKeyExist( m_lruList, oid, lbid ) || cacheKeyExist( m_writeList, oid, lbid ); - } + /** + * @brief Check whether cache key exists + */ + static bool cacheKeyExist(CacheMap* map, const OID oid, const uint64_t lbid) + { + CacheKey key = getCacheKey(oid, lbid); + return map->find(key) == map->end() ? false : true; + } + static bool cacheKeyExist(CacheMap* map, BlockBuffer* buffer) + { + return cacheKeyExist(map, (*buffer).cb.file.oid, (*buffer).block.lbid); + } + static bool cacheKeyExist(const OID oid, const uint64_t lbid) + { + return cacheKeyExist(m_lruList, oid, lbid) || cacheKeyExist(m_writeList, oid, lbid); + } - /** - * @brief Clear the buffer - */ - EXPORT static void clear(); + /** + * @brief Clear the buffer + */ + EXPORT static void clear(); - /** - * @brief Free the buffer memory - */ - EXPORT static void freeMemory(); + /** + * @brief Free the buffer memory + */ + EXPORT static void freeMemory(); - /** - * @brief Flush the write cache - */ - EXPORT static int flushCache(); + /** + * @brief Flush the write cache + */ + EXPORT static int flushCache(); - /** - * @brief Get the cache key - */ - static CacheKey getCacheKey( const OID oid, const uint64_t lbid ) - { - CacheKey key = lbid; /*Convertor::int2Str( oid ) + "|" + Convertor::int2Str(lbid)*/; - return key; - } - static CacheKey getCacheKey( const BlockBuffer* buffer ) - { - return getCacheKey( (*buffer).cb.file.oid, (*buffer).block.lbid ); - } + /** + * @brief Get the cache key + */ + static CacheKey getCacheKey(const OID oid, const uint64_t lbid) + { + CacheKey key = lbid; /*Convertor::int2Str( oid ) + "|" + Convertor::int2Str(lbid)*/ + ; + return key; + } + static CacheKey getCacheKey(const BlockBuffer* buffer) + { + return getCacheKey((*buffer).cb.file.oid, (*buffer).block.lbid); + } - EXPORT static int getListSize( const CacheListType listType ); + EXPORT static int getListSize(const CacheListType listType); - /** - * @brief Init the buffers - */ - EXPORT static void init( const int totalBlock, const int chkPoint, const int pctFree ); - static void init() - { - init( DEFAULT_CACHE_BLOCK, DEFAULT_CHK_INTERVAL, DEFAULT_CACHE_PCT_FREE ); - } + /** + * @brief Init the buffers + */ + EXPORT static void init(const int totalBlock, const int chkPoint, const int pctFree); + static void init() + { + init(DEFAULT_CACHE_BLOCK, DEFAULT_CHK_INTERVAL, DEFAULT_CACHE_PCT_FREE); + } - /** - * @brief Insert into LRU list - */ - EXPORT static int insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf ); - static int insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const DataBlock& block ) - { - return insertLRUList( cb, lbid, fbo, block.data ); - } + /** + * @brief Insert into LRU list + */ + EXPORT static int insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo, + const unsigned char* buf); + static int insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const DataBlock& block) + { + return insertLRUList(cb, lbid, fbo, block.data); + } - /** - * @brief Insert into Write list - */ -// static const int insertWriteList( const CacheKey& key ); + /** + * @brief Insert into Write list + */ + // static const int insertWriteList( const CacheKey& key ); - /** - * @brief Load cache block to a buffer - */ - static int loadCacheBlock( const CacheKey& key, DataBlock& block ) - { - return loadCacheBlock( key, block.data ); - } - EXPORT static int loadCacheBlock( const CacheKey& key, unsigned char* buf ); + /** + * @brief Load cache block to a buffer + */ + static int loadCacheBlock(const CacheKey& key, DataBlock& block) + { + return loadCacheBlock(key, block.data); + } + EXPORT static int loadCacheBlock(const CacheKey& key, unsigned char* buf); - /** - * @brief Modify a cache block - */ - static int modifyCacheBlock( const CacheKey& key, const DataBlock& block ) - { - return modifyCacheBlock( key, block.data ); - } - EXPORT static int modifyCacheBlock( const CacheKey& key, const unsigned char* buf ); + /** + * @brief Modify a cache block + */ + static int modifyCacheBlock(const CacheKey& key, const DataBlock& block) + { + return modifyCacheBlock(key, block.data); + } + EXPORT static int modifyCacheBlock(const CacheKey& key, const unsigned char* buf); - /** - * @brief Print - */ - EXPORT static void printCacheMapList( const CacheMap* map ); - EXPORT static void printCacheList(); + /** + * @brief Print + */ + EXPORT static void printCacheMapList(const CacheMap* map); + EXPORT static void printCacheList(); - /** - * @brief Insert/Delete an element in cache map - */ - EXPORT static int processCacheMap( CacheMap* map, BlockBuffer* buffer, OpType opType ); + /** + * @brief Insert/Delete an element in cache map + */ + EXPORT static int processCacheMap(CacheMap* map, BlockBuffer* buffer, OpType opType); - // accessory - static int getTotalBlock() - { - return m_cacheParam->totalBlock; - } - static bool getUseCache() - { - return m_useCache; - } - static void setUseCache( const bool flag ) - { - m_useCache = flag; - } + // accessory + static int getTotalBlock() + { + return m_cacheParam->totalBlock; + } + static bool getUseCache() + { + return m_useCache; + } + static void setUseCache(const bool flag) + { + m_useCache = flag; + } - static CacheControl* m_cacheParam; // Cache parameters - static FreeBufList* m_freeList; // free buffer list - static CacheMap* m_lruList; // LRU buffer list - static CacheMap* m_writeList; // Write buffer list + static CacheControl* m_cacheParam; // Cache parameters + static FreeBufList* m_freeList; // free buffer list + static CacheMap* m_lruList; // LRU buffer list + static CacheMap* m_writeList; // Write buffer list #if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT) - __declspec(dllimport) + __declspec(dllimport) #endif - EXPORT static bool m_useCache; // Use cache flag -private: - + EXPORT static bool m_useCache; // Use cache flag + private: }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_CACHE_H_ +#endif // _WE_CACHE_H_ diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index ea100d5de..d2ab6d324 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -52,67 +52,68 @@ using namespace idbdatafile; namespace { - // Function to compare 2 ChunkData pointers. bool chunkDataPtrLessCompare(WriteEngine::ChunkData* p1, WriteEngine::ChunkData* p2) { - return (p1->fChunkId) < (p2->fChunkId); + return (p1->fChunkId) < (p2->fChunkId); } -} +} // namespace namespace WriteEngine { - -extern int NUM_BLOCKS_PER_INITIAL_EXTENT; // defined in we_dctnry.cpp -extern WErrorCodes ec; // defined in we_log.cpp +extern int NUM_BLOCKS_PER_INITIAL_EXTENT; // defined in we_dctnry.cpp +extern WErrorCodes ec; // defined in we_log.cpp //------------------------------------------------------------------------------ // Search for the specified chunk in fChunkList. //------------------------------------------------------------------------------ ChunkData* CompFileData::findChunk(int64_t id) const { - ChunkData* pChunkData = NULL; + ChunkData* pChunkData = NULL; - for (list::const_iterator lit = fChunkList.begin(); lit != fChunkList.end(); ++lit) + for (list::const_iterator lit = fChunkList.begin(); lit != fChunkList.end(); ++lit) + { + if ((*lit)->fChunkId == id) { - if ((*lit)->fChunkId == id) - { - pChunkData = *lit; - break; - } + pChunkData = *lit; + break; } + } - return pChunkData; + return pChunkData; } //------------------------------------------------------------------------------ // ChunkManager constructor //------------------------------------------------------------------------------ ChunkManager::ChunkManager() - : fMaxActiveChunkNum(100), fLenCompressed(0), fIsBulkLoad(false), - fDropFdCache(false), fIsInsert(false), fIsHdfs(IDBPolicy::useHdfs()), - fFileOp(0), fSysLogger(NULL), fTransId(-1), - fLocalModuleId(Config::getLocalModuleID()), - fFs(fIsHdfs ? IDBFileSystem::getFs(IDBDataFile::HDFS) - : IDBPolicy::useCloud() - ? IDBFileSystem::getFs(IDBDataFile::CLOUD) - : IDBFileSystem::getFs(IDBDataFile::BUFFERED)) + : fMaxActiveChunkNum(100) + , fLenCompressed(0) + , fIsBulkLoad(false) + , fDropFdCache(false) + , fIsInsert(false) + , fIsHdfs(IDBPolicy::useHdfs()) + , fFileOp(0) + , fSysLogger(NULL) + , fTransId(-1) + , fLocalModuleId(Config::getLocalModuleID()) + , fFs(fIsHdfs ? IDBFileSystem::getFs(IDBDataFile::HDFS) + : IDBPolicy::useCloud() ? IDBFileSystem::getFs(IDBDataFile::CLOUD) + : IDBFileSystem::getFs(IDBDataFile::BUFFERED)) { - fUserPaddings = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; - compress::initializeCompressorPool(fCompressorPool, fUserPaddings); + fUserPaddings = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; + compress::initializeCompressorPool(fCompressorPool, fUserPaddings); - COMPRESSED_CHUNK_SIZE = - compress::CompressInterface::getMaxCompressedSizeGeneric( - UNCOMPRESSED_CHUNK_SIZE) + - 64 + 3 + 8 * 1024; + COMPRESSED_CHUNK_SIZE = + compress::CompressInterface::getMaxCompressedSizeGeneric(UNCOMPRESSED_CHUNK_SIZE) + 64 + 3 + 8 * 1024; - fMaxCompressedBufSize = COMPRESSED_CHUNK_SIZE + fUserPaddings; - fBufCompressed = new char[fMaxCompressedBufSize]; - fSysLogger = new logging::Logger(SUBSYSTEM_ID_WE); - logging::MsgMap msgMap; - msgMap[logging::M0080] = logging::Message(logging::M0080); - fSysLogger->msgMap( msgMap ); + fMaxCompressedBufSize = COMPRESSED_CHUNK_SIZE + fUserPaddings; + fBufCompressed = new char[fMaxCompressedBufSize]; + fSysLogger = new logging::Logger(SUBSYSTEM_ID_WE); + logging::MsgMap msgMap; + msgMap[logging::M0080] = logging::Message(logging::M0080); + fSysLogger->msgMap(msgMap); } //------------------------------------------------------------------------------ @@ -120,128 +121,122 @@ ChunkManager::ChunkManager() //------------------------------------------------------------------------------ ChunkManager::~ChunkManager() { - std::map columnOids; - cleanUp(columnOids); + std::map columnOids; + cleanUp(columnOids); - delete [] fBufCompressed; - fBufCompressed = NULL; + delete[] fBufCompressed; + fBufCompressed = NULL; - delete fSysLogger; - fSysLogger = NULL; + delete fSysLogger; + fSysLogger = NULL; } - //------------------------------------------------------------------------------ // Log a message into the DML recovery log. //------------------------------------------------------------------------------ -int ChunkManager::writeLog(TxnID txnId, string backUpFileType, string filename, - string& aDMLLogFileName, int64_t size, int64_t offset) const +int ChunkManager::writeLog(TxnID txnId, string backUpFileType, string filename, string& aDMLLogFileName, + int64_t size, int64_t offset) const { - //Get log file name - if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) - return ERR_DML_LOG_NAME; + // Get log file name + if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) + return ERR_DML_LOG_NAME; - //Open file - boost::scoped_ptr aDMLLogFile; + // Open file + boost::scoped_ptr aDMLLogFile; - try + try + { + aDMLLogFile.reset(IDBDataFile::open(IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), + aDMLLogFileName.c_str(), "a+b", 0)); + + if (!aDMLLogFile) { - aDMLLogFile.reset(IDBDataFile::open( - IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), - aDMLLogFileName.c_str(), "a+b", 0)); - - if (!aDMLLogFile) - { - ostringstream oss; - oss << "trans " << txnId << ":File " << aDMLLogFileName - << " can't be opened (no exception thrown)"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_OPEN_DML_LOG; - } - } - catch (exception& e) - { - ostringstream oss; - oss << "trans " << txnId << ":File " << aDMLLogFileName - << " can't be opened: " << e.what(); - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_OPEN_DML_LOG; + ostringstream oss; + oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened (no exception thrown)"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_OPEN_DML_LOG; } + } + catch (exception& e) + { + ostringstream oss; + oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened: " << e.what(); + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_OPEN_DML_LOG; + } - //Write the log - ostringstream entry; - entry << backUpFileType << '\n' << filename << '\n' << size << '\n' << offset << '\n'; - string tmp = entry.str(); - aDMLLogFile->seek(0, SEEK_END); - aDMLLogFile->tell(); - aDMLLogFile->write(tmp.c_str(), tmp.size()); + // Write the log + ostringstream entry; + entry << backUpFileType << '\n' << filename << '\n' << size << '\n' << offset << '\n'; + string tmp = entry.str(); + aDMLLogFile->seek(0, SEEK_END); + aDMLLogFile->tell(); + aDMLLogFile->write(tmp.c_str(), tmp.size()); - return NO_ERROR; + return NO_ERROR; } int ChunkManager::removeBackups(TxnID txnId) { - // HDFS update/delete is handled differently - if (fIsHdfs || fIsBulkLoad) - return NO_ERROR; + // HDFS update/delete is handled differently + if (fIsHdfs || fIsBulkLoad) + return NO_ERROR; - string aDMLLogFileName; + string aDMLLogFileName; - if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) - return ERR_DML_LOG_NAME; + if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) + return ERR_DML_LOG_NAME; - if (IDBPolicy::exists(aDMLLogFileName.c_str())) + if (IDBPolicy::exists(aDMLLogFileName.c_str())) + { + boost::scoped_ptr aDMLLogFile(IDBDataFile::open( + IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), aDMLLogFileName.c_str(), "r", 0)); + + if (aDMLLogFile) // need recover { - boost::scoped_ptr aDMLLogFile(IDBDataFile::open( - IDBPolicy::getType(aDMLLogFileName.c_str(), - IDBPolicy::WRITEENG), - aDMLLogFileName.c_str(), "r", 0)); + ssize_t fileSize = aDMLLogFile->size(); + boost::scoped_array buf(new char[fileSize]); - if (aDMLLogFile) //need recover + if (aDMLLogFile->read(buf.get(), fileSize) != fileSize) + return ERR_FILE_READ; + + std::istringstream strstream(string(buf.get(), fileSize)); + std::string backUpFileType; + std::string filename; + int64_t size; + int64_t offset; + + while (strstream >> backUpFileType >> filename >> size >> offset) + { + if (backUpFileType.compare("tmp") == 0) { - ssize_t fileSize = aDMLLogFile->size(); - boost::scoped_array buf(new char[fileSize]); - - if (aDMLLogFile->read(buf.get(), fileSize) != fileSize) - return ERR_FILE_READ; - - std::istringstream strstream(string(buf.get(), fileSize)); - std::string backUpFileType; - std::string filename; - int64_t size; - int64_t offset; - - while (strstream >> backUpFileType >> filename >> size >> offset) - { - if (backUpFileType.compare("tmp") == 0 ) - { - filename += ".tmp"; - IDBPolicy::remove(filename.c_str()); - } - else - { - std::string backFileName(filename); - - if (backUpFileType.compare("chk") == 0 ) - backFileName += ".chk"; - else - backFileName += ".hdr"; - - IDBPolicy::remove(backFileName.c_str()); - } - } - - aDMLLogFile.reset(); // closes the file in IDBDataFile destructor. - - IDBPolicy::remove(aDMLLogFileName.c_str()); + filename += ".tmp"; + IDBPolicy::remove(filename.c_str()); } else { - return ERR_OPEN_DML_LOG; - } - } + std::string backFileName(filename); - return NO_ERROR; + if (backUpFileType.compare("chk") == 0) + backFileName += ".chk"; + else + backFileName += ".hdr"; + + IDBPolicy::remove(backFileName.c_str()); + } + } + + aDMLLogFile.reset(); // closes the file in IDBDataFile destructor. + + IDBPolicy::remove(aDMLLogFileName.c_str()); + } + else + { + return ERR_OPEN_DML_LOG; + } + } + + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -255,18 +250,13 @@ int ChunkManager::removeBackups(TxnID txnId) // If useTmpSuffix flag is set, then IDBDataFile will use *.tmp for output. //------------------------------------------------------------------------------ // @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* ChunkManager::getFilePtr(const Column& column, - uint16_t root, - uint32_t partition, - uint16_t segment, - string& filename, - const char* mode, - int size, +IDBDataFile* ChunkManager::getFilePtr(const Column& column, uint16_t root, uint32_t partition, + uint16_t segment, string& filename, const char* mode, int size, bool useTmpSuffix) const { - CompFileData* fileData = getFileData(column.dataFile.fid, root, partition, segment, - filename, mode, size, column.colDataType, column.colWidth, useTmpSuffix); - return (fileData ? fileData->fFilePtr : NULL); + CompFileData* fileData = getFileData(column.dataFile.fid, root, partition, segment, filename, mode, size, + column.colDataType, column.colWidth, useTmpSuffix); + return (fileData ? fileData->fFilePtr : NULL); } //------------------------------------------------------------------------------ @@ -280,18 +270,13 @@ IDBDataFile* ChunkManager::getFilePtr(const Column& column, // If useTmpSuffix flag is set, then IDBDataFile will use *.tmp for output. //------------------------------------------------------------------------------ // @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* ChunkManager::getFilePtr(const FID& fid, - uint16_t root, - uint32_t partition, - uint16_t segment, - string& filename, - const char* mode, - int size, - bool useTmpSuffix) const +IDBDataFile* ChunkManager::getFilePtr(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + string& filename, const char* mode, int size, bool useTmpSuffix) const { - CompFileData* fileData = getFileData(fid, root, partition, segment, filename, mode, size, - CalpontSystemCatalog::VARCHAR, 8, useTmpSuffix, true); // hard code (varchar, 8) are dummy values for dictionary file - return (fileData ? fileData->fFilePtr : NULL); + CompFileData* fileData = + getFileData(fid, root, partition, segment, filename, mode, size, CalpontSystemCatalog::VARCHAR, 8, + useTmpSuffix, true); // hard code (varchar, 8) are dummy values for dictionary file + return (fileData ? fileData->fFilePtr : NULL); } //------------------------------------------------------------------------------ @@ -301,16 +286,14 @@ IDBDataFile* ChunkManager::getFilePtr(const FID& fid, // the mode (mode) and I/O buffer size (size) that is given. Name of the // resulting file is returned in filename. //------------------------------------------------------------------------------ -IDBDataFile* ChunkManager::getSegmentFilePtr( - FID& fid, uint16_t root, uint32_t partition, uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, - std::string& filename, const char* mode, int32_t size, bool useTmpSuffix, - bool isDict) const +IDBDataFile* ChunkManager::getSegmentFilePtr(FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth, std::string& filename, const char* mode, + int32_t size, bool useTmpSuffix, bool isDict) const { - CompFileData* fileData = - getFileData(fid, root, partition, segment, filename, mode, size, - colDataType, colWidth, useTmpSuffix, isDict); - return (fileData ? fileData->fFilePtr : NULL); + CompFileData* fileData = getFileData(fid, root, partition, segment, filename, mode, size, colDataType, + colWidth, useTmpSuffix, isDict); + return (fileData ? fileData->fFilePtr : NULL); } //------------------------------------------------------------------------------ @@ -325,104 +308,96 @@ IDBDataFile* ChunkManager::getSegmentFilePtr( // If useTmpSuffix flag is set, then IDBDataFile will use *.tmp for output. //------------------------------------------------------------------------------ // @bug 5572 - HDFS usage: add *.tmp file backup flag -CompFileData* ChunkManager::getFileData(const FID& fid, - uint16_t root, - uint32_t partition, - uint16_t segment, - string& filename, - const char* mode, - int size, - const CalpontSystemCatalog::ColDataType colDataType, - int colWidth, - bool useTmpSuffix, - bool dctnry) const +CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + string& filename, const char* mode, int size, + const CalpontSystemCatalog::ColDataType colDataType, int colWidth, + bool useTmpSuffix, bool dctnry) const { - FileID fileID(fid, root, partition, segment); - map::const_iterator mit = fFileMap.find(fileID); + FileID fileID(fid, root, partition, segment); + map::const_iterator mit = fFileMap.find(fileID); - WE_COMP_DBG(cout << "getFileData: fid:" << fid << " root:" << root << " part:" << partition - << " seg:" << segment << " file* " << ((mit != fFileMap.end()) ? "" : "not ") - << "found." << endl;) + WE_COMP_DBG(cout << "getFileData: fid:" << fid << " root:" << root << " part:" << partition << " seg:" + << segment << " file* " << ((mit != fFileMap.end()) ? "" : "not ") << "found." << endl;) - // Get CompFileData pointer for existing Column or Dictionary store file - if (mit != fFileMap.end()) - { - filename = mit->second->fFileName; - return mit->second; - } + // Get CompFileData pointer for existing Column or Dictionary store file + if (mit != fFileMap.end()) + { + filename = mit->second->fFileName; + return mit->second; + } - // New CompFileData pointer needs to be created - char name[FILE_NAME_SIZE]; + // New CompFileData pointer needs to be created + char name[FILE_NAME_SIZE]; - if (fFileOp->getFileName(fid, name, root, partition, segment) != NO_ERROR) - return NULL; + if (fFileOp->getFileName(fid, name, root, partition, segment) != NO_ERROR) + return NULL; - CompFileData* fileData = new CompFileData(fileID, fid, colDataType, colWidth); - fileData->fFileName = filename = name; + CompFileData* fileData = new CompFileData(fileID, fid, colDataType, colWidth); + fileData->fFileName = filename = name; - if (openFile(fileData, mode, colWidth, useTmpSuffix, __LINE__) != NO_ERROR) - { - WE_COMP_DBG(cout << "Failed to open " << fileData->fFileName << " ." << endl;) - delete fileData; - return NULL; - } + if (openFile(fileData, mode, colWidth, useTmpSuffix, __LINE__) != NO_ERROR) + { + WE_COMP_DBG(cout << "Failed to open " << fileData->fFileName << " ." << endl;) + delete fileData; + return NULL; + } - fileData->fIoBuffer.reset(new char[size]); - fileData->fIoBSize = size; -// TODO-There is no current way to make this setvbuf call as IDBDataFile only -// accepts the USE_VBUF at construction time and then uses a buffer that it manages -// Can either propagate an option through the openFile() call above and let -// IDBDataFile manage it internally or expose a new setBuffer() option. -// setvbuf(fileData->fFilePtr, fileData->fIoBuffer.get(), _IOFBF, size); - fileData->fDctnryCol = dctnry; - WE_COMP_DBG(cout << "open file* " << name << endl;) + fileData->fIoBuffer.reset(new char[size]); + fileData->fIoBSize = size; + // TODO-There is no current way to make this setvbuf call as IDBDataFile only + // accepts the USE_VBUF at construction time and then uses a buffer that it manages + // Can either propagate an option through the openFile() call above and let + // IDBDataFile manage it internally or expose a new setBuffer() option. + // setvbuf(fileData->fFilePtr, fileData->fIoBuffer.get(), _IOFBF, size); + fileData->fDctnryCol = dctnry; + WE_COMP_DBG(cout << "open file* " << name << endl;) - // get the control data in header. - if (readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__) != NO_ERROR) - { - WE_COMP_DBG(cout << "Failed to read control header." << endl;) - delete fileData; - return NULL; - } + // get the control data in header. + if (readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fControlData, + COMPRESSED_FILE_HEADER_UNIT, __LINE__) != NO_ERROR) + { + WE_COMP_DBG(cout << "Failed to read control header." << endl;) + delete fileData; + return NULL; + } - // make sure the header is valid - if (compress::CompressInterface::verifyHdr(fileData->fFileHeader.fControlData) != 0) - { - WE_COMP_DBG(cout << "Invalid header." << endl;) - delete fileData; - return NULL; - } + // make sure the header is valid + if (compress::CompressInterface::verifyHdr(fileData->fFileHeader.fControlData) != 0) + { + WE_COMP_DBG(cout << "Invalid header." << endl;) + delete fileData; + return NULL; + } - int headerSize = compress::CompressInterface::getHdrSize( - fileData->fFileHeader.fControlData); - int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; + int headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); + int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - // Save segment file compression type. - uint32_t compressionType = compress::CompressInterface::getCompressionType( - fileData->fFileHeader.fControlData); - fileData->fCompressionType = compressionType; + // Save segment file compression type. + uint32_t compressionType = + compress::CompressInterface::getCompressionType(fileData->fFileHeader.fControlData); + fileData->fCompressionType = compressionType; - if (ptrSecSize > COMPRESSED_FILE_HEADER_UNIT) - { - // >8K header, dictionary width > 128 - fileData->fFileHeader.fPtrSection = new char[ptrSecSize]; - fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection); - } + if (ptrSecSize > COMPRESSED_FILE_HEADER_UNIT) + { + // >8K header, dictionary width > 128 + fileData->fFileHeader.fPtrSection = new char[ptrSecSize]; + fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection); + } - // read in the pointer section in header - if (readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__) != NO_ERROR) - { - WE_COMP_DBG(cout << "Failed to read pointer header." << endl;) - delete fileData; - return NULL; - } + // read in the pointer section in header + if (readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, + __LINE__) != NO_ERROR) + { + WE_COMP_DBG(cout << "Failed to read pointer header." << endl;) + delete fileData; + return NULL; + } - fFileMap.insert(make_pair(fileID, fileData)); - //cout << "Insert into fFilemap root:partition:seg:fileID = " <fFilePtr, fileData)); - return fileData; + fFileMap.insert(make_pair(fileID, fileData)); + // cout << "Insert into fFilemap root:partition:seg:fileID = " <fFilePtr, fileData)); + return fileData; } //------------------------------------------------------------------------------ @@ -433,64 +408,58 @@ CompFileData* ChunkManager::getFileData(const FID& fid, // A corresponding CompFileData* is created and inserted into fFileMap and // fFilePtrMap for later use. //------------------------------------------------------------------------------ -IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, - int64_t width, - uint16_t root, - uint32_t partition, - uint16_t segment, - const char* filename, - const char* mode, - int size, - BRM::LBID_t lbid) +IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, int64_t width, uint16_t root, uint32_t partition, + uint16_t segment, const char* filename, const char* mode, + int size, BRM::LBID_t lbid) { - FileID fileID(fid, root, partition, segment); - CompFileData* fileData = new CompFileData(fileID, fid, CalpontSystemCatalog::VARCHAR, width); - fileData->fFileName = filename; + FileID fileID(fid, root, partition, segment); + CompFileData* fileData = new CompFileData(fileID, fid, CalpontSystemCatalog::VARCHAR, width); + fileData->fFileName = filename; - if (openFile(fileData, mode, width, false, __LINE__) != NO_ERROR) // @bug 5572 HDFS tmp file - { - WE_COMP_DBG(cout << "Failed to open " << fileData->fFileName << " ." << endl;) - delete fileData; - return NULL; - } + if (openFile(fileData, mode, width, false, __LINE__) != NO_ERROR) // @bug 5572 HDFS tmp file + { + WE_COMP_DBG(cout << "Failed to open " << fileData->fFileName << " ." << endl;) + delete fileData; + return NULL; + } - fileData->fIoBuffer.reset(new char[size]); - fileData->fIoBSize = size; -// see TODO- comment above -// setvbuf(fileData->fFilePtr, fileData->fIoBuffer.get(), _IOFBF, size); - fileData->fDctnryCol = true; - WE_COMP_DBG(cout << "create file* " << filename << endl;) - int hdrSize = calculateHeaderSize(width); - int ptrSecSize = hdrSize - COMPRESSED_FILE_HEADER_UNIT; + fileData->fIoBuffer.reset(new char[size]); + fileData->fIoBSize = size; + // see TODO- comment above + // setvbuf(fileData->fFilePtr, fileData->fIoBuffer.get(), _IOFBF, size); + fileData->fDctnryCol = true; + WE_COMP_DBG(cout << "create file* " << filename << endl;) + int hdrSize = calculateHeaderSize(width); + int ptrSecSize = hdrSize - COMPRESSED_FILE_HEADER_UNIT; - if (ptrSecSize > COMPRESSED_FILE_HEADER_UNIT) - { - // >8K header, dictionary width > 128 - fileData->fFileHeader.fPtrSection = new char[ptrSecSize]; - fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection); - } + if (ptrSecSize > COMPRESSED_FILE_HEADER_UNIT) + { + // >8K header, dictionary width > 128 + fileData->fFileHeader.fPtrSection = new char[ptrSecSize]; + fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection); + } - // Dictionary store extent width == 0. See more details in function - // `createDictStoreExtent`. - compress::CompressInterface::initHdr( - fileData->fFileHeader.fControlData, fileData->fFileHeader.fPtrSection, - /*colWidth=*/0, fileData->fColDataType, fFileOp->compressionType(), hdrSize); - compress::CompressInterface::setLBIDByIndex(fileData->fFileHeader.fControlData, lbid, 0); - // Save compression type. - fileData->fCompressionType = fFileOp->compressionType(); + // Dictionary store extent width == 0. See more details in function + // `createDictStoreExtent`. + compress::CompressInterface::initHdr(fileData->fFileHeader.fControlData, fileData->fFileHeader.fPtrSection, + /*colWidth=*/0, fileData->fColDataType, fFileOp->compressionType(), + hdrSize); + compress::CompressInterface::setLBIDByIndex(fileData->fFileHeader.fControlData, lbid, 0); + // Save compression type. + fileData->fCompressionType = fFileOp->compressionType(); - if (writeHeader(fileData, __LINE__) != NO_ERROR) - { - WE_COMP_DBG(cout << "Failed to write header." << endl;) - delete fileData; - return NULL; - } + if (writeHeader(fileData, __LINE__) != NO_ERROR) + { + WE_COMP_DBG(cout << "Failed to write header." << endl;) + delete fileData; + return NULL; + } - //@Bug 4977 remove log file - removeBackups(fTransId); - fFileMap.insert(make_pair(fileID, fileData)); - fFilePtrMap.insert(make_pair(fileData->fFilePtr, fileData)); - return fileData->fFilePtr; + //@Bug 4977 remove log file + removeBackups(fTransId); + fFileMap.insert(make_pair(fileID, fileData)); + fFilePtrMap.insert(make_pair(fileData->fFilePtr, fileData)); + return fileData->fFilePtr; } //------------------------------------------------------------------------------ @@ -499,34 +468,34 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, //------------------------------------------------------------------------------ int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo) { - map::iterator fpIt = fFilePtrMap.find(pFile); + map::iterator fpIt = fFilePtrMap.find(pFile); - if (fpIt == fFilePtrMap.end()) - { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; - } + if (fpIt == fFilePtrMap.end()) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } - // find the chunk ID and offset in the chunk - lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); - ChunkData* chunkData = (fpIt->second)->findChunk(offset.quot); + // find the chunk ID and offset in the chunk + lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); + ChunkData* chunkData = (fpIt->second)->findChunk(offset.quot); - WE_COMP_DBG(cout << "fbo:" << fbo << " chunk id:" << offset.quot << " offset:" << offset.rem - << " chunkData*:" << chunkData << endl;) + WE_COMP_DBG(cout << "fbo:" << fbo << " chunk id:" << offset.quot << " offset:" << offset.rem + << " chunkData*:" << chunkData << endl;) - int rc = NO_ERROR; + int rc = NO_ERROR; - // chunk is not already uncompressed - if (chunkData == NULL) - rc = fetchChunkFromFile(pFile, offset.quot, chunkData); + // chunk is not already uncompressed + if (chunkData == NULL) + rc = fetchChunkFromFile(pFile, offset.quot, chunkData); - if (rc == NO_ERROR) - { - // copy the data at fbo to readBuf - memcpy(readBuf, chunkData->fBufUnCompressed + offset.rem, BYTE_PER_BLOCK); - } + if (rc == NO_ERROR) + { + // copy the data at fbo to readBuf + memcpy(readBuf, chunkData->fBufUnCompressed + offset.rem, BYTE_PER_BLOCK); + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -535,47 +504,47 @@ int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t //------------------------------------------------------------------------------ int ChunkManager::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo) { - WE_COMP_DBG(cout << "save block fbo:" << fbo << endl;) - map::iterator fpIt = fFilePtrMap.find(pFile); + WE_COMP_DBG(cout << "save block fbo:" << fbo << endl;) + map::iterator fpIt = fFilePtrMap.find(pFile); - if (fpIt == fFilePtrMap.end()) - { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; - } + if (fpIt == fFilePtrMap.end()) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } - // find the chunk ID and offset in the chunk - lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); - ChunkData* chunkData = (fpIt->second)->findChunk(offset.quot); + // find the chunk ID and offset in the chunk + lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); + ChunkData* chunkData = (fpIt->second)->findChunk(offset.quot); - int rc = NO_ERROR; - - // chunk is not already read in - if ((chunkData == NULL) && ((rc = fetchChunkFromFile(pFile, offset.quot, chunkData)) != NO_ERROR)) - return rc; - - WE_COMP_DBG(cout << "fbo:" << fbo << " chunk id:" << offset.quot << " offset:" << offset.rem - << " saved @" << (&(chunkData->fBufUnCompressed) + offset.rem) << endl;) - - memcpy(chunkData->fBufUnCompressed + offset.rem, writeBuf, BYTE_PER_BLOCK); - chunkData->fWriteToFile = true; - -// if the chunk is full for insert, flush it -//cout << "current offset.rem/8192 = " << offset.rem/8192 << endl; - if (fIsInsert && (offset.rem == MAXOFFSET_PER_CHUNK)) - { - if (((rc = writeChunkToFile(fpIt->second, chunkData)) == NO_ERROR) && - ((rc = writeHeader(fpIt->second, __LINE__)) == NO_ERROR)) - { - //cout << "saveblock flushed the full chunk"<flush(); - - //@Bug 4977 remove log file - removeBackups(fTransId); - } - } + int rc = NO_ERROR; + // chunk is not already read in + if ((chunkData == NULL) && ((rc = fetchChunkFromFile(pFile, offset.quot, chunkData)) != NO_ERROR)) return rc; + + WE_COMP_DBG(cout << "fbo:" << fbo << " chunk id:" << offset.quot << " offset:" << offset.rem << " saved @" + << (&(chunkData->fBufUnCompressed) + offset.rem) << endl;) + + memcpy(chunkData->fBufUnCompressed + offset.rem, writeBuf, BYTE_PER_BLOCK); + chunkData->fWriteToFile = true; + + // if the chunk is full for insert, flush it + // cout << "current offset.rem/8192 = " << offset.rem/8192 << endl; + if (fIsInsert && (offset.rem == MAXOFFSET_PER_CHUNK)) + { + if (((rc = writeChunkToFile(fpIt->second, chunkData)) == NO_ERROR) && + ((rc = writeHeader(fpIt->second, __LINE__)) == NO_ERROR)) + { + // cout << "saveblock flushed the full chunk"<flush(); + + //@Bug 4977 remove log file + removeBackups(fTransId); + } + } + + return rc; } //------------------------------------------------------------------------------ @@ -583,104 +552,104 @@ int ChunkManager::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, u //------------------------------------------------------------------------------ int ChunkManager::flushChunks(int rc, const std::map& columOids) { - // shall fail the the statement if failed here - WE_COMP_DBG(cout << "flushChunks." << endl;) + // shall fail the the statement if failed here + WE_COMP_DBG(cout << "flushChunks." << endl;) - int k = fFilePtrMap.size(); - std::map::const_iterator it; + int k = fFilePtrMap.size(); + std::map::const_iterator it; - if ((rc == NO_ERROR) && fIsInsert) + if ((rc == NO_ERROR) && fIsInsert) + { + while (k-- > 0 && rc == NO_ERROR) { - while (k-- > 0 && rc == NO_ERROR) + map::iterator i = fFilePtrMap.begin(); + // sort the chunk list first + CompFileData* fileData = i->second; + it = columOids.find(fileData->fFid); + + if (it != columOids.end()) + { + list& chunkList = fileData->fChunkList; + chunkList.sort(chunkDataPtrLessCompare); + list::iterator j = chunkList.begin(); + + while (j != chunkList.end()) { - map::iterator i = fFilePtrMap.begin(); - // sort the chunk list first - CompFileData* fileData = i->second; - it = columOids.find (fileData->fFid); + if ((rc = writeChunkToFile(fileData, *j)) != NO_ERROR) + break; - if (it != columOids.end()) - { - list& chunkList = fileData->fChunkList; - chunkList.sort(chunkDataPtrLessCompare); - list::iterator j = chunkList.begin(); - - while (j != chunkList.end()) - { - if ((rc = writeChunkToFile(fileData, *j)) != NO_ERROR) - break; - - // write chunk to file removes the written chunk from the list - j = chunkList.begin(); - } - - if (rc != NO_ERROR) - break; - - // finally update the header - if ((rc = writeHeader(fileData, __LINE__)) != NO_ERROR) - break; - - //@Bug 4977 remove log file - removeBackups(fTransId); - - // closeFile invalidates the iterator - closeFile(fileData); - } + // write chunk to file removes the written chunk from the list + j = chunkList.begin(); } + + if (rc != NO_ERROR) + break; + + // finally update the header + if ((rc = writeHeader(fileData, __LINE__)) != NO_ERROR) + break; + + //@Bug 4977 remove log file + removeBackups(fTransId); + + // closeFile invalidates the iterator + closeFile(fileData); + } } - else if (rc == NO_ERROR) + } + else if (rc == NO_ERROR) + { + while (k-- > 0 && rc == NO_ERROR) { - while (k-- > 0 && rc == NO_ERROR) - { - map::iterator i = fFilePtrMap.begin(); - // sort the chunk list first - CompFileData* fileData = i->second; + map::iterator i = fFilePtrMap.begin(); + // sort the chunk list first + CompFileData* fileData = i->second; - list& chunkList = fileData->fChunkList; - chunkList.sort(chunkDataPtrLessCompare); - list::iterator j = chunkList.begin(); + list& chunkList = fileData->fChunkList; + chunkList.sort(chunkDataPtrLessCompare); + list::iterator j = chunkList.begin(); - while (j != chunkList.end()) - { - if ((rc = writeChunkToFile(fileData, *j)) != NO_ERROR) - break; + while (j != chunkList.end()) + { + if ((rc = writeChunkToFile(fileData, *j)) != NO_ERROR) + break; - // write chunk to file removes the written chunk from the list - j = chunkList.begin(); - } + // write chunk to file removes the written chunk from the list + j = chunkList.begin(); + } - if (rc != NO_ERROR) - break; + if (rc != NO_ERROR) + break; - // finally update the header - if ((rc = writeHeader(fileData, __LINE__)) != NO_ERROR) - break; + // finally update the header + if ((rc = writeHeader(fileData, __LINE__)) != NO_ERROR) + break; - //@Bug 4977 remove log file - removeBackups(fTransId); + //@Bug 4977 remove log file + removeBackups(fTransId); - // closeFile invalidates the iterator - closeFile(fileData); - } + // closeFile invalidates the iterator + closeFile(fileData); } + } - if (rc != NO_ERROR) - { - cleanUp(columOids); - return rc; - } + if (rc != NO_ERROR) + { + cleanUp(columOids); + return rc; + } - //fActiveChunks.clear(); - //fFileMap.clear(); - //fFilePtrMap.clear(); + // fActiveChunks.clear(); + // fFileMap.clear(); + // fFilePtrMap.clear(); - if (fDropFdCache) - { - cacheutils::dropPrimProcFdCache(); - fDropFdCache = false; - } + if (fDropFdCache) + { + cacheutils::dropPrimProcFdCache(); + fDropFdCache = false; + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -694,169 +663,165 @@ int ChunkManager::flushChunks(int rc, const std::map& columOids) //------------------------------------------------------------------------------ int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData) { - // return value - int rc = NO_ERROR; + // return value + int rc = NO_ERROR; - // remove the oldest one if the max active chunk number is reached. - WE_COMP_DBG(cout << "fActiveChunks.size:" << fActiveChunks.size() << endl;) - //cout << "fetchChunkFromFile1: pFile = " << pFile << endl; - map::iterator fpIt = fFilePtrMap.find(pFile); + // remove the oldest one if the max active chunk number is reached. + WE_COMP_DBG(cout << "fActiveChunks.size:" << fActiveChunks.size() << endl;) + // cout << "fetchChunkFromFile1: pFile = " << pFile << endl; + map::iterator fpIt = fFilePtrMap.find(pFile); - if (fpIt == fFilePtrMap.end()) + if (fpIt == fFilePtrMap.end()) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } + + CompFileData* fileData = fpIt->second; + + if (fActiveChunks.size() >= fMaxActiveChunkNum) + { + list >::iterator lIt = fActiveChunks.begin(); + + if (!fIsBulkLoad && !(fpIt->second->fDctnryCol)) { + while ((lIt->first == fpIt->second->fFileID) && (lIt != fActiveChunks.end())) + lIt++; + } + + if (lIt != fActiveChunks.end()) + { + map::iterator fIt = fFileMap.find(lIt->first); + + if (fIt == fFileMap.end()) + { logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); return ERR_COMP_FILE_NOT_FOUND; + } + + if ((rc = writeChunkToFile(fIt->second, lIt->second)) != NO_ERROR) + { + ostringstream oss; + oss << "write inactive chunk to file failed:" << fIt->second->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return rc; + } + + if ((rc = writeHeader(fIt->second, __LINE__)) != NO_ERROR) + { + // logged by writeHeader + return rc; + } + + //@Bug 4977 remove the log files + removeBackups(fTransId); } - - CompFileData* fileData = fpIt->second; - - if (fActiveChunks.size() >= fMaxActiveChunkNum) - { - list >::iterator lIt = fActiveChunks.begin(); - - if (!fIsBulkLoad && !(fpIt->second->fDctnryCol)) - { - while ((lIt->first == fpIt->second->fFileID) && (lIt != fActiveChunks.end())) - lIt++; - } - - if (lIt != fActiveChunks.end()) - { - map::iterator fIt = fFileMap.find(lIt->first); - - if (fIt == fFileMap.end()) - { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; - } - - if ((rc = writeChunkToFile(fIt->second, lIt->second)) != NO_ERROR) - { - ostringstream oss; - oss << "write inactive chunk to file failed:" << fIt->second->fFileName << "@" - << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return rc; - } - - if ((rc = writeHeader(fIt->second, __LINE__)) != NO_ERROR) - { - // logged by writeHeader - return rc; - } - - //@Bug 4977 remove the log files - removeBackups(fTransId); - } - } + } #ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_DCT_INIT_BUF); + Stats::startParseEvent(WE_STATS_COMPRESS_DCT_INIT_BUF); #endif - // get a new ChunkData object - chunkData = new ChunkData(id); - pFile = fileData->fFilePtr; //update to get the reopened file ptr. - fileData->fChunkList.push_back(chunkData); - fActiveChunks.push_back(make_pair(fileData->fFileID, chunkData)); + // get a new ChunkData object + chunkData = new ChunkData(id); + pFile = fileData->fFilePtr; // update to get the reopened file ptr. + fileData->fChunkList.push_back(chunkData); + fActiveChunks.push_back(make_pair(fileData->fFileID, chunkData)); - // read the compressed chunk from file - uint64_t* ptrs = reinterpret_cast(fileData->fFileHeader.fPtrSection); + // read the compressed chunk from file + uint64_t* ptrs = reinterpret_cast(fileData->fFileHeader.fPtrSection); - if (ptrs[id] && ptrs[id + 1]) // compressed chunk data exists + if (ptrs[id] && ptrs[id + 1]) // compressed chunk data exists + { + // safety check + if (ptrs[id] >= ptrs[id + 1]) { - // safety check - if (ptrs[id] >= ptrs[id + 1]) - { - logMessage(ERR_COMP_WRONG_PTR, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_WRONG_PTR; - } - - unsigned int chunkSize = (ptrs[id + 1] - ptrs[id]); - - if ((rc = setFileOffset(pFile, fileData->fFileName, ptrs[id], __LINE__)) != NO_ERROR || - (rc = readFile(pFile, fileData->fFileName, fBufCompressed, chunkSize, __LINE__)) != - NO_ERROR) - { - // logged by setFileOffset/readFile - return rc; - } - - // uncompress the read in buffer - size_t dataLen = sizeof(chunkData->fBufUnCompressed); - - auto fCompressor = compress::getCompressorByType( - fCompressorPool, fileData->fCompressionType); - if (!fCompressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } - - if (fCompressor->uncompressBlock((char*)fBufCompressed, chunkSize, - (unsigned char*)chunkData->fBufUnCompressed, dataLen) != 0) - { - if (fIsFix) - { - uint64_t blocks = 512; - - if (id == 0) - { - char* hdr = fileData->fFileHeader.fControlData; - - if (compress::CompressInterface::getBlockCount(hdr) < 512) - blocks = 256; - } - - dataLen = 8192 * blocks; - - // load the uncompressed buffer with empty values. - char* buf = chunkData->fBufUnCompressed; - chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; - - if (fileData->fDctnryCol) - initializeDctnryChunk(buf, UNCOMPRESSED_CHUNK_SIZE); - else - initializeColumnChunk(buf, fileData); - } - else - { - logMessage(ERR_COMP_UNCOMPRESS, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_UNCOMPRESS; - } - } - -//@bug 3313-Remove validation that incorrectly fails for long string store files -// WE_COMP_DBG(cout << "chunk uncompressed to " << dataLen << endl;) -// if (dataLen < (id+1) * BYTE_PER_BLOCK) -// { -// logMessage(ERR_COMP_UNCOMPRESS, logging::LOG_TYPE_ERROR, __LINE__); -// return ERR_COMP_UNCOMPRESS; -// } - - chunkData->fLenUnCompressed = dataLen; + logMessage(ERR_COMP_WRONG_PTR, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_WRONG_PTR; } - else // new chunk + + unsigned int chunkSize = (ptrs[id + 1] - ptrs[id]); + + if ((rc = setFileOffset(pFile, fileData->fFileName, ptrs[id], __LINE__)) != NO_ERROR || + (rc = readFile(pFile, fileData->fFileName, fBufCompressed, chunkSize, __LINE__)) != NO_ERROR) { - if (id == 0 && ptrs[id] == 0) // if the 1st ptr is not set for new extent + // logged by setFileOffset/readFile + return rc; + } + + // uncompress the read in buffer + size_t dataLen = sizeof(chunkData->fBufUnCompressed); + + auto fCompressor = compress::getCompressorByType(fCompressorPool, fileData->fCompressionType); + if (!fCompressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } + + if (fCompressor->uncompressBlock((char*)fBufCompressed, chunkSize, + (unsigned char*)chunkData->fBufUnCompressed, dataLen) != 0) + { + if (fIsFix) + { + uint64_t blocks = 512; + + if (id == 0) { - ptrs[0] = compress::CompressInterface::getHdrSize( - fileData->fFileHeader.fControlData); + char* hdr = fileData->fFileHeader.fControlData; + + if (compress::CompressInterface::getBlockCount(hdr) < 512) + blocks = 256; } + dataLen = 8192 * blocks; + // load the uncompressed buffer with empty values. char* buf = chunkData->fBufUnCompressed; chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; if (fileData->fDctnryCol) - initializeDctnryChunk(buf, UNCOMPRESSED_CHUNK_SIZE); + initializeDctnryChunk(buf, UNCOMPRESSED_CHUNK_SIZE); else - initializeColumnChunk(buf, fileData); + initializeColumnChunk(buf, fileData); + } + else + { + logMessage(ERR_COMP_UNCOMPRESS, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_UNCOMPRESS; + } } + //@bug 3313-Remove validation that incorrectly fails for long string store files + // WE_COMP_DBG(cout << "chunk uncompressed to " << dataLen << endl;) + // if (dataLen < (id+1) * BYTE_PER_BLOCK) + // { + // logMessage(ERR_COMP_UNCOMPRESS, logging::LOG_TYPE_ERROR, __LINE__); + // return ERR_COMP_UNCOMPRESS; + // } + + chunkData->fLenUnCompressed = dataLen; + } + else // new chunk + { + if (id == 0 && ptrs[id] == 0) // if the 1st ptr is not set for new extent + { + ptrs[0] = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); + } + + // load the uncompressed buffer with empty values. + char* buf = chunkData->fBufUnCompressed; + chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; + + if (fileData->fDctnryCol) + initializeDctnryChunk(buf, UNCOMPRESSED_CHUNK_SIZE); + else + initializeColumnChunk(buf, fileData); + } + #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_INIT_BUF); + Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_INIT_BUF); #endif - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -864,10 +829,9 @@ int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& //------------------------------------------------------------------------------ void ChunkManager::initializeColumnChunk(char* buf, CompFileData* fileData) { - int size = UNCOMPRESSED_CHUNK_SIZE; - const uint8_t* emptyVal = fFileOp->getEmptyRowValue(fileData->fColDataType, - fileData->fColWidth); - fFileOp->setEmptyBuf((unsigned char*)buf, size, emptyVal, fileData->fColWidth); + int size = UNCOMPRESSED_CHUNK_SIZE; + const uint8_t* emptyVal = fFileOp->getEmptyRowValue(fileData->fColDataType, fileData->fColWidth); + fFileOp->setEmptyBuf((unsigned char*)buf, size, emptyVal, fileData->fColWidth); } //------------------------------------------------------------------------------ @@ -875,15 +839,15 @@ void ChunkManager::initializeColumnChunk(char* buf, CompFileData* fileData) //------------------------------------------------------------------------------ void ChunkManager::initializeDctnryChunk(char* buf, int size) { - Dctnry* dctnry = dynamic_cast(fFileOp); - memset(buf, 0, size); - char* end = buf + size; + Dctnry* dctnry = dynamic_cast(fFileOp); + memset(buf, 0, size); + char* end = buf + size; - while (buf < end) - { - dctnry->copyDctnryHeader(buf); - buf += BYTE_PER_BLOCK; - } + while (buf < end) + { + dctnry->copyDctnryHeader(buf); + buf += BYTE_PER_BLOCK; + } } //------------------------------------------------------------------------------ @@ -892,15 +856,15 @@ void ChunkManager::initializeDctnryChunk(char* buf, int size) //------------------------------------------------------------------------------ int ChunkManager::writeChunkToFile(CompFileData* fileData, int64_t id) { - ChunkData* chunkData = fileData->findChunk(id); + ChunkData* chunkData = fileData->findChunk(id); - if (!chunkData) - { - logMessage(ERR_COMP_CHUNK_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_CHUNK_NOT_FOUND; - } + if (!chunkData) + { + logMessage(ERR_COMP_CHUNK_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_CHUNK_NOT_FOUND; + } - return writeChunkToFile(fileData, chunkData); + return writeChunkToFile(fileData, chunkData); } //------------------------------------------------------------------------------ @@ -911,144 +875,139 @@ int ChunkManager::writeChunkToFile(CompFileData* fileData, int64_t id) //------------------------------------------------------------------------------ int ChunkManager::writeChunkToFile(CompFileData* fileData, ChunkData* chunkData) { - WE_COMP_DBG(cout << "write chunk id=" << chunkData->fChunkId << " data " - << ((chunkData->fWriteToFile) ? "changed" : "NOT changed") << endl;) + WE_COMP_DBG(cout << "write chunk id=" << chunkData->fChunkId << " data " + << ((chunkData->fWriteToFile) ? "changed" : "NOT changed") << endl;) - int rc = NO_ERROR; // return value - bool needReallocateChunks = false; - int64_t spaceAvl = 0; + int rc = NO_ERROR; // return value + bool needReallocateChunks = false; + int64_t spaceAvl = 0; - if (chunkData->fWriteToFile) + if (chunkData->fWriteToFile) + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_COMPRESS_DCT_COMPRESS); +#endif + // compress the chunk before writing it to file + fLenCompressed = fMaxCompressedBufSize; + + auto fCompressor = compress::getCompressorByType(fCompressorPool, fileData->fCompressionType); + if (!fCompressor) { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_DCT_COMPRESS); -#endif - // compress the chunk before writing it to file - fLenCompressed = fMaxCompressedBufSize; - - auto fCompressor = compress::getCompressorByType( - fCompressorPool, fileData->fCompressionType); - if (!fCompressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } - - if (fCompressor->compressBlock((char*) chunkData->fBufUnCompressed, - chunkData->fLenUnCompressed, - (unsigned char*) fBufCompressed, - fLenCompressed) != 0) - { - logMessage(ERR_COMP_COMPRESS, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_COMPRESS; - } - - WE_COMP_DBG(cout << "Chunk compressed from " << chunkData->fLenUnCompressed << " to " - << fLenCompressed;) - - // Removed padding code here, will add padding for the last chunk. - // The existing chunks are already correctly aligned, use the padding to absort chunk - // size increase when update. This improves the performance with less chunk shifting. - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_COMPRESS); -#endif - - // need more work if the new compressed buffer is larger - uint64_t* ptrs = reinterpret_cast(fileData->fFileHeader.fPtrSection); - ChunkId chunkId = chunkData->fChunkId; - - if (ptrs[chunkId + 1] > 0) - spaceAvl = (ptrs[chunkId + 1] - ptrs[chunkId]); - - WE_COMP_DBG(cout << ", available space:" << spaceAvl;) - - bool lastChunk = true; - // usable chunkIds are 0 .. POINTERS_IN_HEADER-2 - // [chunkId+0] is the start offset of current chunk. - // [chunkId+1] is the start offset of next chunk, the offset diff is current chunk size. - // [chunkId+2] is 0 or not indicates if the next chunk exists. - int headerSize = compress::CompressInterface::getHdrSize( - fileData->fFileHeader.fControlData); - int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - int64_t usablePtrIds = (ptrSecSize / sizeof(uint64_t)) - 2; - - if (chunkId < usablePtrIds) // make sure [chunkId+2] has valid value - lastChunk = (ptrs[(chunkId + 2)] == 0); - - WE_COMP_DBG(cout << ", last chunk:" << (lastChunk ? "true" : "false") << endl;) - - if (spaceAvl < 0) - { - logMessage(ERR_COMP_WRONG_PTR, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_WRONG_PTR; - } - - if ((int64_t)fLenCompressed <= spaceAvl) - { - // There is enough sapce. - if ((rc = writeCompressedChunk(fileData, ptrs[chunkId], spaceAvl)) != NO_ERROR) - { - // log in writeCompressedChunk by setFileOffset and writeFile - return rc; - } - } - else if (lastChunk) - { - // add padding space if the chunk is written first time - if (fCompressor->padCompressedChunks( - (unsigned char*)fBufCompressed, fLenCompressed, fMaxCompressedBufSize) != 0) - { - WE_COMP_DBG(cout << "Last chunk:" << chunkId << ", padding failed." << endl;) - - logMessage(ERR_COMP_PAD_DATA, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_PAD_DATA; - } - - WE_COMP_DBG(cout << "Last chunk:" << chunkId << ", padded to " << fLenCompressed;) - - // This is the last chunk, safe to write any length of data. - //@Bug 3888. Assign the error code - if ((rc = writeCompressedChunk(fileData, ptrs[chunkId], spaceAvl)) != NO_ERROR) - { - // log in writeCompressedChunk by setFileOffset and writeFile - return rc; - } - - // Update the current chunk size. - ptrs[chunkId + 1] = ptrs[chunkId] + fLenCompressed; - } - else - { - needReallocateChunks = true; - } + return ERR_COMP_WRONG_COMP_TYPE; } - if (!needReallocateChunks) + if (fCompressor->compressBlock((char*)chunkData->fBufUnCompressed, chunkData->fLenUnCompressed, + (unsigned char*)fBufCompressed, fLenCompressed) != 0) { - fActiveChunks.remove(make_pair(fileData->fFileID, chunkData)); - fileData->fChunkList.remove(chunkData); - delete chunkData; + logMessage(ERR_COMP_COMPRESS, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_COMPRESS; + } + + WE_COMP_DBG(cout << "Chunk compressed from " << chunkData->fLenUnCompressed << " to " << fLenCompressed;) + + // Removed padding code here, will add padding for the last chunk. + // The existing chunks are already correctly aligned, use the padding to absort chunk + // size increase when update. This improves the performance with less chunk shifting. + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_COMPRESS); +#endif + + // need more work if the new compressed buffer is larger + uint64_t* ptrs = reinterpret_cast(fileData->fFileHeader.fPtrSection); + ChunkId chunkId = chunkData->fChunkId; + + if (ptrs[chunkId + 1] > 0) + spaceAvl = (ptrs[chunkId + 1] - ptrs[chunkId]); + + WE_COMP_DBG(cout << ", available space:" << spaceAvl;) + + bool lastChunk = true; + // usable chunkIds are 0 .. POINTERS_IN_HEADER-2 + // [chunkId+0] is the start offset of current chunk. + // [chunkId+1] is the start offset of next chunk, the offset diff is current chunk size. + // [chunkId+2] is 0 or not indicates if the next chunk exists. + int headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); + int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; + int64_t usablePtrIds = (ptrSecSize / sizeof(uint64_t)) - 2; + + if (chunkId < usablePtrIds) // make sure [chunkId+2] has valid value + lastChunk = (ptrs[(chunkId + 2)] == 0); + + WE_COMP_DBG(cout << ", last chunk:" << (lastChunk ? "true" : "false") << endl;) + + if (spaceAvl < 0) + { + logMessage(ERR_COMP_WRONG_PTR, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_WRONG_PTR; + } + + if ((int64_t)fLenCompressed <= spaceAvl) + { + // There is enough sapce. + if ((rc = writeCompressedChunk(fileData, ptrs[chunkId], spaceAvl)) != NO_ERROR) + { + // log in writeCompressedChunk by setFileOffset and writeFile + return rc; + } + } + else if (lastChunk) + { + // add padding space if the chunk is written first time + if (fCompressor->padCompressedChunks((unsigned char*)fBufCompressed, fLenCompressed, + fMaxCompressedBufSize) != 0) + { + WE_COMP_DBG(cout << "Last chunk:" << chunkId << ", padding failed." << endl;) + + logMessage(ERR_COMP_PAD_DATA, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_PAD_DATA; + } + + WE_COMP_DBG(cout << "Last chunk:" << chunkId << ", padded to " << fLenCompressed;) + + // This is the last chunk, safe to write any length of data. + //@Bug 3888. Assign the error code + if ((rc = writeCompressedChunk(fileData, ptrs[chunkId], spaceAvl)) != NO_ERROR) + { + // log in writeCompressedChunk by setFileOffset and writeFile + return rc; + } + + // Update the current chunk size. + ptrs[chunkId + 1] = ptrs[chunkId] + fLenCompressed; } else { - ostringstream oss; - oss << "Compressed data does not fit, caused a chunk shifting @line:" << __LINE__ - << " filename:" << fileData->fFileName << ", chunkId:" << chunkData->fChunkId - << " data size:" << fLenCompressed << "/available:" << spaceAvl << " -- shifting "; - - if ((rc = reallocateChunks(fileData)) == NO_ERROR) - { - oss << "SUCCESS"; - logMessage(oss.str(), logging::LOG_TYPE_INFO); - } - else - { - oss << "FAILED"; - logMessage(oss.str(), logging::LOG_TYPE_CRITICAL); - } + needReallocateChunks = true; } + } - return rc; + if (!needReallocateChunks) + { + fActiveChunks.remove(make_pair(fileData->fFileID, chunkData)); + fileData->fChunkList.remove(chunkData); + delete chunkData; + } + else + { + ostringstream oss; + oss << "Compressed data does not fit, caused a chunk shifting @line:" << __LINE__ + << " filename:" << fileData->fFileName << ", chunkId:" << chunkData->fChunkId + << " data size:" << fLenCompressed << "/available:" << spaceAvl << " -- shifting "; + + if ((rc = reallocateChunks(fileData)) == NO_ERROR) + { + oss << "SUCCESS"; + logMessage(oss.str(), logging::LOG_TYPE_INFO); + } + else + { + oss << "FAILED"; + logMessage(oss.str(), logging::LOG_TYPE_CRITICAL); + } + } + + return rc; } //------------------------------------------------------------------------------ @@ -1059,73 +1018,66 @@ int ChunkManager::writeChunkToFile(CompFileData* fileData, ChunkData* chunkData) //------------------------------------------------------------------------------ int ChunkManager::writeCompressedChunk(CompFileData* fileData, int64_t offset, int64_t size) { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (!fIsBulkLoad && !fIsHdfs) + if (!fIsBulkLoad && !fIsHdfs) + { + // backup current chunk to chk file + string chkFileName(fileData->fFileName + ".chk"); + string aDMLLogFileName; + unsigned char* buf = new unsigned char[size]; + + if (((rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, offset, __LINE__)) == NO_ERROR) && + ((rc = readFile(fileData->fFilePtr, fileData->fFileName, buf, size, __LINE__)) == NO_ERROR)) { - // backup current chunk to chk file - string chkFileName(fileData->fFileName + ".chk"); - string aDMLLogFileName; - unsigned char* buf = new unsigned char[size]; + IDBDataFile* chkFilePtr = IDBDataFile::open( + IDBPolicy::getType(chkFileName.c_str(), IDBPolicy::WRITEENG), chkFileName.c_str(), "w+b", 0); - if (((rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, offset, __LINE__)) == - NO_ERROR) - && - ((rc = readFile(fileData->fFilePtr, fileData->fFileName, buf, size, __LINE__)) == - NO_ERROR)) - { - IDBDataFile* chkFilePtr = IDBDataFile::open( - IDBPolicy::getType(chkFileName.c_str(), - IDBPolicy::WRITEENG), - chkFileName.c_str(), - "w+b", - 0 ); + if (chkFilePtr) + { + rc = writeFile(chkFilePtr, chkFileName, buf, size, __LINE__); + delete chkFilePtr; + } - if (chkFilePtr) - { - rc = writeFile(chkFilePtr, chkFileName, buf, size, __LINE__); - delete chkFilePtr; - } + delete[] buf; - delete [] buf; + if (rc != NO_ERROR) + { + IDBPolicy::remove(chkFileName.c_str()); + return rc; + } - if (rc != NO_ERROR) - { - IDBPolicy::remove(chkFileName.c_str()); - return rc; - } + // log the chunk information for recovery + rc = writeLog(fTransId, "chk", fileData->fFileName, aDMLLogFileName, size, offset); - // log the chunk information for recovery - rc = writeLog(fTransId, "chk", fileData->fFileName, aDMLLogFileName, size, offset); - - if (rc != NO_ERROR) - { - ostringstream oss; - oss << "log " << fileData->fFileName << ".chk to DML logfile failed."; - logMessage(oss.str(), logging::LOG_TYPE_INFO); - return rc; - } - } - - // write out the compressed data + padding - if ((rc == NO_ERROR) && ((rc = writeCompressedChunk_(fileData, offset)) == NO_ERROR)) - { - if ((fileData->fFilePtr)->flush() != 0) //@Bug3162. - { - rc = ERR_FILE_WRITE; - ostringstream oss; - oss << "Failed to flush " << fileData->fFileName << " @line: " << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } - } - } - else - { - // write out the compressed data + padding - rc = writeCompressedChunk_(fileData, offset); + if (rc != NO_ERROR) + { + ostringstream oss; + oss << "log " << fileData->fFileName << ".chk to DML logfile failed."; + logMessage(oss.str(), logging::LOG_TYPE_INFO); + return rc; + } } - return rc; + // write out the compressed data + padding + if ((rc == NO_ERROR) && ((rc = writeCompressedChunk_(fileData, offset)) == NO_ERROR)) + { + if ((fileData->fFilePtr)->flush() != 0) //@Bug3162. + { + rc = ERR_FILE_WRITE; + ostringstream oss; + oss << "Failed to flush " << fileData->fFileName << " @line: " << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + } + } + } + else + { + // write out the compressed data + padding + rc = writeCompressedChunk_(fileData, offset); + } + + return rc; } //------------------------------------------------------------------------------ @@ -1134,13 +1086,12 @@ int ChunkManager::writeCompressedChunk(CompFileData* fileData, int64_t offset, i //------------------------------------------------------------------------------ inline int ChunkManager::writeCompressedChunk_(CompFileData* fileData, int64_t offset) { - int rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, offset, __LINE__); + int rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, offset, __LINE__); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - return writeFile(fileData->fFilePtr, fileData->fFileName, - fBufCompressed, fLenCompressed, __LINE__); + return writeFile(fileData->fFilePtr, fileData->fFileName, fBufCompressed, fLenCompressed, __LINE__); } //------------------------------------------------------------------------------ @@ -1155,49 +1106,47 @@ inline int ChunkManager::writeCompressedChunk_(CompFileData* fileData, int64_t o // // @bug 5572 - HDFS usage: add *.tmp file backup flag to API //------------------------------------------------------------------------------ -int ChunkManager::openFile(CompFileData* fileData, const char* mode, int colWidth, - bool useTmpSuffix, int ln) const +int ChunkManager::openFile(CompFileData* fileData, const char* mode, int colWidth, bool useTmpSuffix, + int ln) const { - int rc = NO_ERROR; - unsigned opts = IDBDataFile::USE_VBUF; + int rc = NO_ERROR; + unsigned opts = IDBDataFile::USE_VBUF; - if (fIsHdfs) + if (fIsHdfs) + { + if (useTmpSuffix) { - if (useTmpSuffix) + if (!fIsBulkLoad) + { + // keep a DML log for confirm or cleanup the .tmp file + string aDMLLogFileName; + + if ((rc = writeLog(fTransId, "tmp", fileData->fFileName, aDMLLogFileName, 0)) != NO_ERROR) { - if (!fIsBulkLoad) - { - // keep a DML log for confirm or cleanup the .tmp file - string aDMLLogFileName; + ostringstream oss; + oss << "Failed to put " << fileData->fFileName << " into DML log."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); - if ((rc = writeLog(fTransId, "tmp", fileData->fFileName, - aDMLLogFileName, 0)) != NO_ERROR) - { - ostringstream oss; - oss << "Failed to put " << fileData->fFileName << " into DML log."; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - return rc; - } - } - - opts |= IDBDataFile::USE_TMPFILE; + return rc; } + } + + opts |= IDBDataFile::USE_TMPFILE; } + } - fileData->fFilePtr = IDBDataFile::open( - IDBPolicy::getType( fileData->fFileName.c_str(), IDBPolicy::WRITEENG ), - fileData->fFileName.c_str(), mode, opts, colWidth); + fileData->fFilePtr = IDBDataFile::open(IDBPolicy::getType(fileData->fFileName.c_str(), IDBPolicy::WRITEENG), + fileData->fFileName.c_str(), mode, opts, colWidth); - if (fileData->fFilePtr == NULL) - { - ostringstream oss; - oss << "Failed to open compressed data file " << fileData->fFileName << " @line: " << ln; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - rc = ERR_COMP_OPEN_FILE; - } + if (fileData->fFilePtr == NULL) + { + ostringstream oss; + oss << "Failed to open compressed data file " << fileData->fFileName << " @line: " << ln; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + rc = ERR_COMP_OPEN_FILE; + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -1209,19 +1158,20 @@ int ChunkManager::openFile(CompFileData* fileData, const char* mode, int colWidt //------------------------------------------------------------------------------ int ChunkManager::setFileOffset(IDBDataFile* pFile, const string& fileName, off64_t offset, int ln) const { - int rc = NO_ERROR; + int rc = NO_ERROR; - if (pFile->seek(offset, SEEK_SET) != 0) rc = ERR_COMP_SET_OFFSET; + if (pFile->seek(offset, SEEK_SET) != 0) + rc = ERR_COMP_SET_OFFSET; - if (rc != NO_ERROR) - { - ostringstream oss; - oss << "Failed to set offset in compressed data file " << fileName - << " @line: " << ln << " offset:" << offset; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } + if (rc != NO_ERROR) + { + ostringstream oss; + oss << "Failed to set offset in compressed data file " << fileName << " @line: " << ln + << " offset:" << offset; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -1232,18 +1182,18 @@ int ChunkManager::setFileOffset(IDBDataFile* pFile, const string& fileName, off6 //------------------------------------------------------------------------------ int ChunkManager::readFile(IDBDataFile* pFile, const string& fileName, void* buf, size_t size, int ln) const { - size_t bytes = pFile->read(buf, size); + size_t bytes = pFile->read(buf, size); - if (bytes != size) - { - ostringstream oss; - oss << "Failed to read from compressed data file " << fileName - << " @line: " << ln << " read/expect:" << bytes << "/" << size; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_COMP_READ_FILE; - } + if (bytes != size) + { + ostringstream oss; + oss << "Failed to read from compressed data file " << fileName << " @line: " << ln + << " read/expect:" << bytes << "/" << size; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_COMP_READ_FILE; + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1254,18 +1204,18 @@ int ChunkManager::readFile(IDBDataFile* pFile, const string& fileName, void* buf //------------------------------------------------------------------------------ int ChunkManager::writeFile(IDBDataFile* pFile, const string& fileName, void* buf, size_t size, int ln) const { - size_t bytes = pFile->write(buf, size); + size_t bytes = pFile->write(buf, size); - if (bytes != size) - { - ostringstream oss; - oss << "Failed to write to compressed data file " << fileName - << " @line: " << ln << " written/expect:" << bytes << "/" << size; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_COMP_WRITE_FILE; - } + if (bytes != size) + { + ostringstream oss; + oss << "Failed to write to compressed data file " << fileName << " @line: " << ln + << " written/expect:" << bytes << "/" << size; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_COMP_WRITE_FILE; + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1274,19 +1224,19 @@ int ChunkManager::writeFile(IDBDataFile* pFile, const string& fileName, void* bu //------------------------------------------------------------------------------ int ChunkManager::closeFile(CompFileData* fileData) { - int rc = NO_ERROR; + int rc = NO_ERROR; - WE_COMP_DBG(cout << "closing file:" << fileData->fFileName << endl;) - fFileMap.erase(fileData->fFileID); - fFilePtrMap.erase(fileData->fFilePtr); + WE_COMP_DBG(cout << "closing file:" << fileData->fFileName << endl;) + fFileMap.erase(fileData->fFileID); + fFilePtrMap.erase(fileData->fFilePtr); - if (fileData->fFilePtr) - delete fileData->fFilePtr; + if (fileData->fFilePtr) + delete fileData->fFilePtr; - delete fileData; - fileData = NULL; + delete fileData; + fileData = NULL; - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -1298,74 +1248,67 @@ int ChunkManager::closeFile(CompFileData* fileData) //------------------------------------------------------------------------------ int ChunkManager::writeHeader(CompFileData* fileData, int ln) { - int rc = NO_ERROR; - int headerSize = compress::CompressInterface::getHdrSize( - fileData->fFileHeader.fControlData); - int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; + int rc = NO_ERROR; + int headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); + int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - if (!fIsHdfs && !fIsBulkLoad) + if (!fIsHdfs && !fIsBulkLoad) + { + // write a backup header + string hdrFileName(fileData->fFileName + ".hdr"); + string aDMLLogFileName; + IDBDataFile* hdrFilePtr = IDBDataFile::open(IDBPolicy::getType(hdrFileName.c_str(), IDBPolicy::WRITEENG), + hdrFileName.c_str(), "w+b", 0, fileData->fColWidth); + + if (hdrFilePtr) { - // write a backup header - string hdrFileName(fileData->fFileName + ".hdr"); - string aDMLLogFileName; - IDBDataFile* hdrFilePtr = IDBDataFile::open( - IDBPolicy::getType(hdrFileName.c_str(), - IDBPolicy::WRITEENG), - hdrFileName.c_str(), - "w+b", - 0, - fileData->fColWidth ); + rc = writeFile(hdrFilePtr, hdrFileName, fileData->fFileHeader.fControlData, COMPRESSED_FILE_HEADER_UNIT, + __LINE__); - if (hdrFilePtr) - { - rc = writeFile(hdrFilePtr, hdrFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__); + if (rc == NO_ERROR) + rc = writeFile(hdrFilePtr, hdrFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, __LINE__); - if (rc == NO_ERROR) - rc = writeFile(hdrFilePtr, hdrFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__); + delete hdrFilePtr; + } - delete hdrFilePtr; - } + if (rc == NO_ERROR) + { + // log the header information for recovery + rc = writeLog(fTransId, "hdr", fileData->fFileName, aDMLLogFileName, headerSize); - if (rc == NO_ERROR) - { - // log the header information for recovery - rc = writeLog(fTransId, "hdr", fileData->fFileName, aDMLLogFileName, headerSize); + if (rc != NO_ERROR) + { + ostringstream oss; + oss << "log " << fileData->fFileName << ".hdr to DML logfile failed."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + } - if (rc != NO_ERROR) - { - ostringstream oss; - oss << "log " << fileData->fFileName << ".hdr to DML logfile failed."; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } - - if ((rc == NO_ERROR) && (rc = writeHeader_(fileData, ptrSecSize)) == NO_ERROR) - { - (fileData->fFilePtr)->flush(); - } - } - else - { - IDBPolicy::remove(hdrFileName.c_str()); - } + if ((rc == NO_ERROR) && (rc = writeHeader_(fileData, ptrSecSize)) == NO_ERROR) + { + (fileData->fFilePtr)->flush(); + } } else { - if ((rc = writeHeader_(fileData, ptrSecSize)) == NO_ERROR) - { - (fileData->fFilePtr)->flush(); - } + IDBPolicy::remove(hdrFileName.c_str()); } - - if (rc != NO_ERROR) + } + else + { + if ((rc = writeHeader_(fileData, ptrSecSize)) == NO_ERROR) { - ostringstream oss; - oss << "write header failed: " << fileData->fFileName << "call from line:" << ln; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); + (fileData->fFilePtr)->flush(); } + } - return rc; + if (rc != NO_ERROR) + { + ostringstream oss; + oss << "write header failed: " << fileData->fFileName << "call from line:" << ln; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + } + + return rc; } //------------------------------------------------------------------------------ @@ -1377,19 +1320,17 @@ int ChunkManager::writeHeader(CompFileData* fileData, int ln) //------------------------------------------------------------------------------ inline int ChunkManager::writeHeader_(CompFileData* fileData, int ptrSecSize) { - int rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, 0, __LINE__); + int rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, 0, __LINE__); - if (rc == NO_ERROR) - rc = writeFile(fileData->fFilePtr, fileData->fFileName, - fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__); + if (rc == NO_ERROR) + rc = writeFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fControlData, + COMPRESSED_FILE_HEADER_UNIT, __LINE__); - if (rc == NO_ERROR) - rc = writeFile(fileData->fFilePtr, fileData->fFileName, - fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__); + if (rc == NO_ERROR) + rc = writeFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, + __LINE__); - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -1398,30 +1339,30 @@ inline int ChunkManager::writeHeader_(CompFileData* fileData, int ptrSecSize) //------------------------------------------------------------------------------ int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width) { - map::iterator i = fFilePtrMap.find(pFile); + map::iterator i = fFilePtrMap.find(pFile); - if (i == fFilePtrMap.end()) - { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; - } + if (i == fFilePtrMap.end()) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } - int rc = NO_ERROR; - // fetch the initial chunk if not already done. - ChunkData* chunkData = (i->second)->findChunk(0); + int rc = NO_ERROR; + // fetch the initial chunk if not already done. + ChunkData* chunkData = (i->second)->findChunk(0); - if ((chunkData == NULL) && ((rc = fetchChunkFromFile(pFile, 0, chunkData)) != NO_ERROR)) - return rc; + if ((chunkData == NULL) && ((rc = fetchChunkFromFile(pFile, 0, chunkData)) != NO_ERROR)) + return rc; - // buf points to the end of existing data - char* buf = chunkData->fBufUnCompressed + chunkData->fLenUnCompressed; - int size = UNCOMPRESSED_CHUNK_SIZE - chunkData->fLenUnCompressed; - fFileOp->setEmptyBuf((unsigned char*)buf, size, emptyVal, width); - chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; - chunkData->fWriteToFile = true; - //(writeChunkToFile(i->second, chunkData)); - //(writeHeader(i->second, __LINE__)); - return NO_ERROR; + // buf points to the end of existing data + char* buf = chunkData->fBufUnCompressed + chunkData->fLenUnCompressed; + int size = UNCOMPRESSED_CHUNK_SIZE - chunkData->fLenUnCompressed; + fFileOp->setEmptyBuf((unsigned char*)buf, size, emptyVal, width); + chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; + chunkData->fWriteToFile = true; + //(writeChunkToFile(i->second, chunkData)); + //(writeHeader(i->second, __LINE__)); + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -1432,52 +1373,52 @@ int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* em // same here as for dict. int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid) { - map::iterator i = fFilePtrMap.find(pFile); + map::iterator i = fFilePtrMap.find(pFile); - if (i == fFilePtrMap.end()) + if (i == fFilePtrMap.end()) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } + + CompFileData* pFileData = i->second; + + if (!pFileData) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } + + int rc = NO_ERROR; + char* hdr = pFileData->fFileHeader.fControlData; + compress::CompressInterface::setBlockCount(hdr, + compress::CompressInterface::getBlockCount(hdr) + addBlockCount); + compress::CompressInterface::setLBIDByIndex(hdr, lbid, 1); + + ChunkData* chunkData = (pFileData)->findChunk(0); + + if (chunkData != NULL) + { + if ((rc = writeChunkToFile(pFileData, chunkData)) == NO_ERROR) { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; + rc = writeHeader(pFileData, __LINE__); + + if (rc == NO_ERROR) + { + //@Bug 4977 remove log files + removeBackups(fTransId); + } } - - CompFileData* pFileData = i->second; - - if (!pFileData) + else { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; + ostringstream oss; + oss << "write chunk to file failed when updateColumnExtent: " << pFileData->fFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); } + } - int rc = NO_ERROR; - char* hdr = pFileData->fFileHeader.fControlData; - compress::CompressInterface::setBlockCount( - hdr, compress::CompressInterface::getBlockCount(hdr) + addBlockCount); - compress::CompressInterface::setLBIDByIndex(hdr, lbid, 1); - - ChunkData* chunkData = (pFileData)->findChunk(0); - - if (chunkData != NULL) - { - if ((rc = writeChunkToFile(pFileData, chunkData)) == NO_ERROR) - { - rc = writeHeader(pFileData, __LINE__); - - if ( rc == NO_ERROR) - { - //@Bug 4977 remove log files - removeBackups(fTransId); - } - } - else - { - ostringstream oss; - oss << "write chunk to file failed when updateColumnExtent: " << pFileData->fFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } - } - - pFile->flush(); - return rc; + pFile->flush(); + return rc; } //------------------------------------------------------------------------------ @@ -1485,73 +1426,71 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int6 // Increment the block count stored in the chunk header used to track how many // blocks are allocated to the corresponding segment file. //------------------------------------------------------------------------------ -int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, - BRM::LBID_t lbid) +int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, BRM::LBID_t lbid) { - map::iterator i = fFilePtrMap.find(pFile); + map::iterator i = fFilePtrMap.find(pFile); - if (i == fFilePtrMap.end()) + if (i == fFilePtrMap.end()) + { + logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); + return ERR_COMP_FILE_NOT_FOUND; + } + + int rc = NO_ERROR; + // fetch the initial chunk if not already done. + ChunkData* chunkData = (i->second)->findChunk(0); + + if ((chunkData == NULL) && ((rc = fetchChunkFromFile(pFile, 0, chunkData)) != NO_ERROR)) + return rc; // logged by fetchChunkFromFile + + char* hdr = i->second->fFileHeader.fControlData; + char* uncompressedBuf = chunkData->fBufUnCompressed; + int currentBlockCount = compress::CompressInterface::getBlockCount(hdr); + + // Bug 3203, write out the compressed initial extent. + if (currentBlockCount == 0) + { + int initSize = NUM_BLOCKS_PER_INITIAL_EXTENT * BYTE_PER_BLOCK; + initializeDctnryChunk(uncompressedBuf, initSize); + chunkData->fWriteToFile = true; + + if ((rc = writeChunkToFile(i->second, chunkData)) == NO_ERROR) { - logMessage(ERR_COMP_FILE_NOT_FOUND, logging::LOG_TYPE_ERROR, __LINE__); - return ERR_COMP_FILE_NOT_FOUND; + rc = writeHeader(i->second, __LINE__); + + if (rc == NO_ERROR) + { + //@Bug 4977 remove the log file + removeBackups(fTransId); + } } - - int rc = NO_ERROR; - // fetch the initial chunk if not already done. - ChunkData* chunkData = (i->second)->findChunk(0); - - if ((chunkData == NULL) && ((rc = fetchChunkFromFile(pFile, 0, chunkData)) != NO_ERROR)) - return rc; // logged by fetchChunkFromFile - - char* hdr = i->second->fFileHeader.fControlData; - char* uncompressedBuf = chunkData->fBufUnCompressed; - int currentBlockCount = compress::CompressInterface::getBlockCount(hdr); - - // Bug 3203, write out the compressed initial extent. - if (currentBlockCount == 0) + else { - int initSize = NUM_BLOCKS_PER_INITIAL_EXTENT * BYTE_PER_BLOCK; - initializeDctnryChunk(uncompressedBuf, initSize); - chunkData->fWriteToFile = true; - - if ((rc = writeChunkToFile(i->second, chunkData)) == NO_ERROR) - { - rc = writeHeader(i->second, __LINE__); - - if ( rc == NO_ERROR) - { - //@Bug 4977 remove the log file - removeBackups(fTransId); - } - } - else - { - ostringstream oss; - oss << "write chunk to file failed when updateDctnryExtent: " << i->second->fFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } - } - else if (currentBlockCount == NUM_BLOCKS_PER_INITIAL_EXTENT) - { - int initSize = NUM_BLOCKS_PER_INITIAL_EXTENT * BYTE_PER_BLOCK; - int incrSize = UNCOMPRESSED_CHUNK_SIZE - initSize; - initializeDctnryChunk(uncompressedBuf + initSize, incrSize); - uint64_t* ptrs = reinterpret_cast(i->second->fFileHeader.fPtrSection); - ptrs[1] = 0; // the compressed chunk size is unknown + ostringstream oss; + oss << "write chunk to file failed when updateDctnryExtent: " << i->second->fFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); } + } + else if (currentBlockCount == NUM_BLOCKS_PER_INITIAL_EXTENT) + { + int initSize = NUM_BLOCKS_PER_INITIAL_EXTENT * BYTE_PER_BLOCK; + int incrSize = UNCOMPRESSED_CHUNK_SIZE - initSize; + initializeDctnryChunk(uncompressedBuf + initSize, incrSize); + uint64_t* ptrs = reinterpret_cast(i->second->fFileHeader.fPtrSection); + ptrs[1] = 0; // the compressed chunk size is unknown + } - if (rc == NO_ERROR) - compress::CompressInterface::setBlockCount( - hdr, - compress::CompressInterface::getBlockCount(hdr) + addBlockCount); + if (rc == NO_ERROR) + compress::CompressInterface::setBlockCount( + hdr, compress::CompressInterface::getBlockCount(hdr) + addBlockCount); - if (currentBlockCount) - { - // Append to the end. - uint64_t lbidCount = compress::CompressInterface::getLBIDCount(hdr); - compress::CompressInterface::setLBIDByIndex(hdr, lbid, lbidCount); - } - return rc; + if (currentBlockCount) + { + // Append to the end. + uint64_t lbidCount = compress::CompressInterface::getLBIDCount(hdr); + compress::CompressInterface::setLBIDByIndex(hdr, lbid, lbidCount); + } + return rc; } //------------------------------------------------------------------------------ @@ -1559,54 +1498,53 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, //------------------------------------------------------------------------------ void ChunkManager::cleanUp(const std::map& columOids) { - WE_COMP_DBG(cout << "cleanUp with " << fActiveChunks.size() << " active chunk(s)." << endl;) - std::map::const_iterator it; - map::iterator i = fFilePtrMap.begin(); + WE_COMP_DBG(cout << "cleanUp with " << fActiveChunks.size() << " active chunk(s)." << endl;) + std::map::const_iterator it; + map::iterator i = fFilePtrMap.begin(); - while ( i != fFilePtrMap.end() ) + while (i != fFilePtrMap.end()) + { + CompFileData* fileData = i->second; + + it = columOids.find(fileData->fFid); + + if (fIsInsert && it != columOids.end()) { - CompFileData* fileData = i->second; + list& chunks = fileData->fChunkList; - it = columOids.find (fileData->fFid); + for (list::iterator j = chunks.begin(); j != chunks.end(); ++j) + delete *j; - if (fIsInsert && it != columOids.end()) - { - list& chunks = fileData->fChunkList; + delete fileData->fFilePtr; + fFileMap.erase(fileData->fFileID); + fFilePtrMap.erase(i++); - for (list::iterator j = chunks.begin(); j != chunks.end(); ++j) - delete *j; - - delete fileData->fFilePtr; - fFileMap.erase(fileData->fFileID); - fFilePtrMap.erase(i++); - - delete fileData; - - } - else if (!fIsInsert || (columOids.size() == 0)) - { - list& chunks = fileData->fChunkList; - - for (list::iterator j = chunks.begin(); j != chunks.end(); ++j) - delete *j; - - delete fileData->fFilePtr; - fFileMap.erase(fileData->fFileID); - fFilePtrMap.erase(i++); - - delete fileData; - } - else - { - i++; - } + delete fileData; } - - if (fDropFdCache) + else if (!fIsInsert || (columOids.size() == 0)) { - cacheutils::dropPrimProcFdCache(); - fDropFdCache = false; + list& chunks = fileData->fChunkList; + + for (list::iterator j = chunks.begin(); j != chunks.end(); ++j) + delete *j; + + delete fileData->fFilePtr; + fFileMap.erase(fileData->fFileID); + fFilePtrMap.erase(i++); + + delete fileData; } + else + { + i++; + } + } + + if (fDropFdCache) + { + cacheutils::dropPrimProcFdCache(); + fDropFdCache = false; + } } //------------------------------------------------------------------------------ @@ -1614,60 +1552,58 @@ void ChunkManager::cleanUp(const std::map& columOids) //------------------------------------------------------------------------------ int ChunkManager::readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n) { - WE_COMP_DBG(cout << "backup blocks fbo:" << fbo << " num:" << n << " file:" << pFile << endl;) + WE_COMP_DBG(cout << "backup blocks fbo:" << fbo << " num:" << n << " file:" << pFile << endl;) - // safety check - if (pFile == NULL || n < 1) + // safety check + if (pFile == NULL || n < 1) + { + return -1; + } + + map::iterator fpIt = fFilePtrMap.find(pFile); + + if (fpIt == fFilePtrMap.end()) + { + return -1; + } + + // the n blocks may cross more than one chunk + // find the chunk ID and offset of the 1st fbo + lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); + int idx = offset.quot; // current chunk id + int rem = offset.rem; // offset in current chunk + int num = UNCOMPRESSED_CHUNK_SIZE - rem; // # of bytes available in current chunk + int left = n * BYTE_PER_BLOCK; // # of bytest to be read + // # of bytes to be read from current chunk + num = (left > num) ? num : left; + + do + { + ChunkData* chunkData = (fpIt->second)->findChunk(idx); + + WE_COMP_DBG(cout << "id:" << idx << " ofst:" << rem << " num:" << num << " left:" << left << endl;) + + // chunk is not already uncompressed + if (chunkData == NULL) { + if (fetchChunkFromFile(pFile, idx, chunkData) != NO_ERROR) + { return -1; + } } - map::iterator fpIt = fFilePtrMap.find(pFile); + // copy the data at fbo to readBuf + memcpy(readBuf, chunkData->fBufUnCompressed + rem, num); - if (fpIt == fFilePtrMap.end()) - { - return -1; - } + // prepare for the next read + readBuf += num; + rem = 0; + left -= num; + num = (left > UNCOMPRESSED_CHUNK_SIZE) ? UNCOMPRESSED_CHUNK_SIZE : left; + idx++; + } while (left > 0); - // the n blocks may cross more than one chunk - // find the chunk ID and offset of the 1st fbo - lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); - int idx = offset.quot; // current chunk id - int rem = offset.rem; // offset in current chunk - int num = UNCOMPRESSED_CHUNK_SIZE - rem; // # of bytes available in current chunk - int left = n * BYTE_PER_BLOCK; // # of bytest to be read - // # of bytes to be read from current chunk - num = (left > num) ? num : left; - - do - { - ChunkData* chunkData = (fpIt->second)->findChunk(idx); - - WE_COMP_DBG(cout << "id:" << idx << " ofst:" << rem << " num:" << num << - " left:" << left << endl;) - - // chunk is not already uncompressed - if (chunkData == NULL) - { - if (fetchChunkFromFile(pFile, idx, chunkData) != NO_ERROR) - { - return -1; - } - } - - // copy the data at fbo to readBuf - memcpy(readBuf, chunkData->fBufUnCompressed + rem, num); - - // prepare for the next read - readBuf += num; - rem = 0; - left -= num; - num = (left > UNCOMPRESSED_CHUNK_SIZE) ? UNCOMPRESSED_CHUNK_SIZE : left; - idx++; - } - while (left > 0); - - return n; + return n; } //------------------------------------------------------------------------------ @@ -1677,35 +1613,35 @@ int ChunkManager::readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_ //------------------------------------------------------------------------------ int ChunkManager::restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo) { - WE_COMP_DBG(cout << "restore blocks fbo:" << fbo << " file:" << pFile << endl;) + WE_COMP_DBG(cout << "restore blocks fbo:" << fbo << " file:" << pFile << endl;) - // safety check - if (pFile == NULL) - return -1; + // safety check + if (pFile == NULL) + return -1; - map::iterator fpIt = fFilePtrMap.find(pFile); + map::iterator fpIt = fFilePtrMap.find(pFile); - if (fpIt == fFilePtrMap.end()) - return -1; + if (fpIt == fFilePtrMap.end()) + return -1; - // the n blocks may cross more than one chunk - // find the chunk ID and offset of the 1st fbo - lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); - ChunkData* chunkData = (fpIt->second)->findChunk(offset.quot); - WE_COMP_DBG(cout << "id:" << offset.quot << " ofst:" << offset.rem << endl;) + // the n blocks may cross more than one chunk + // find the chunk ID and offset of the 1st fbo + lldiv_t offset = lldiv(fbo * BYTE_PER_BLOCK, UNCOMPRESSED_CHUNK_SIZE); + ChunkData* chunkData = (fpIt->second)->findChunk(offset.quot); + WE_COMP_DBG(cout << "id:" << offset.quot << " ofst:" << offset.rem << endl;) - // chunk is not already uncompressed - if (chunkData == NULL) - { - if (fetchChunkFromFile(pFile, offset.quot, chunkData) != NO_ERROR) - return -1; - } + // chunk is not already uncompressed + if (chunkData == NULL) + { + if (fetchChunkFromFile(pFile, offset.quot, chunkData) != NO_ERROR) + return -1; + } - // copy the data to chunk buffer - memcpy(chunkData->fBufUnCompressed + offset.rem, writeBuf, BYTE_PER_BLOCK); - chunkData->fWriteToFile = true; + // copy the data to chunk buffer + memcpy(chunkData->fBufUnCompressed + offset.rem, writeBuf, BYTE_PER_BLOCK); + chunkData->fWriteToFile = true; - return BYTE_PER_BLOCK; + return BYTE_PER_BLOCK; } //------------------------------------------------------------------------------ @@ -1713,11 +1649,10 @@ int ChunkManager::restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf //------------------------------------------------------------------------------ int ChunkManager::getBlockCount(IDBDataFile* pFile) { - map::iterator fpIt = fFilePtrMap.find(pFile); - idbassert(fpIt != fFilePtrMap.end()); + map::iterator fpIt = fFilePtrMap.find(pFile); + idbassert(fpIt != fFilePtrMap.end()); - return compress::CompressInterface::getBlockCount( - fpIt->second->fFileHeader.fControlData); + return compress::CompressInterface::getBlockCount(fpIt->second->fFileHeader.fControlData); } //------------------------------------------------------------------------------ @@ -1725,12 +1660,12 @@ int ChunkManager::getBlockCount(IDBDataFile* pFile) //------------------------------------------------------------------------------ void ChunkManager::fileOp(FileOp* fileOp) { - fFileOp = fileOp; + fFileOp = fileOp; - if (fileOp) - { - setTransId(fileOp->getTransId()); - } + if (fileOp) + { + setTransId(fileOp->getTransId()); + } } //------------------------------------------------------------------------------ @@ -1739,38 +1674,38 @@ void ChunkManager::fileOp(FileOp* fileOp) //------------------------------------------------------------------------------ int ChunkManager::calculateHeaderSize(int width) { - int headerUnits = 1; + int headerUnits = 1; - // dictionary columns may need variable length header - if (width > 8) - { - int extentsPerFile = Config::getExtentsPerSegmentFile(); - int rowsPerExtent = BRMWrapper::getInstance()->getExtentRows(); - int rowsPerFile = rowsPerExtent * extentsPerFile; - int stringsPerBlock = 8180 / (width + 2); // 8180 = 8192 - 12 + // dictionary columns may need variable length header + if (width > 8) + { + int extentsPerFile = Config::getExtentsPerSegmentFile(); + int rowsPerExtent = BRMWrapper::getInstance()->getExtentRows(); + int rowsPerFile = rowsPerExtent * extentsPerFile; + int stringsPerBlock = 8180 / (width + 2); // 8180 = 8192 - 12 - // BLOB is 1 string per block - if (stringsPerBlock == 0) - stringsPerBlock = 1; + // BLOB is 1 string per block + if (stringsPerBlock == 0) + stringsPerBlock = 1; - int blocksNeeded = rowsPerFile / stringsPerBlock; - int blocksPerChunk = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK; - lldiv_t chunks = lldiv(blocksNeeded, blocksPerChunk); - int chunksNeeded = chunks.quot + (chunks.rem ? 1 : 0); // round up - int ptrsNeeded = chunksNeeded + 1; // 1 more ptr for 0 ptr marking end - int ptrsIn4K = (4 * 1024) / sizeof(uint64_t); - lldiv_t hdrs = lldiv(ptrsNeeded, ptrsIn4K); - headerUnits = hdrs.quot + (hdrs.rem ? 1 : 0); // round up + int blocksNeeded = rowsPerFile / stringsPerBlock; + int blocksPerChunk = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK; + lldiv_t chunks = lldiv(blocksNeeded, blocksPerChunk); + int chunksNeeded = chunks.quot + (chunks.rem ? 1 : 0); // round up + int ptrsNeeded = chunksNeeded + 1; // 1 more ptr for 0 ptr marking end + int ptrsIn4K = (4 * 1024) / sizeof(uint64_t); + lldiv_t hdrs = lldiv(ptrsNeeded, ptrsIn4K); + headerUnits = hdrs.quot + (hdrs.rem ? 1 : 0); // round up - // Always include odd number of 4K ptr headers, so that when we add the - // single 4K control header, the cumulative header space will be an even - // multiple of an 8K boundary. - if ((headerUnits % 2) == 0) - headerUnits++; - } + // Always include odd number of 4K ptr headers, so that when we add the + // single 4K control header, the cumulative header space will be an even + // multiple of an 8K boundary. + if ((headerUnits % 2) == 0) + headerUnits++; + } - headerUnits++; // add the control data block - return (headerUnits * COMPRESSED_FILE_HEADER_UNIT); + headerUnits++; // add the control data block + return (headerUnits * COMPRESSED_FILE_HEADER_UNIT); } //------------------------------------------------------------------------------ @@ -1779,493 +1714,465 @@ int ChunkManager::calculateHeaderSize(int width) //------------------------------------------------------------------------------ int ChunkManager::reallocateChunks(CompFileData* fileData) { - WE_COMP_DBG(cout << "reallocate chunks in " << fileData->fFileName - << " (" << fileData->fFilePtr << ")" << endl;) + WE_COMP_DBG(cout << "reallocate chunks in " << fileData->fFileName << " (" << fileData->fFilePtr << ")" + << endl;) - // return value - int rc = NO_ERROR; + // return value + int rc = NO_ERROR; - // original file info - string origFileName = fileData->fFileName; - IDBDataFile* origFilePtr = fileData->fFilePtr; - origFilePtr->flush(); + // original file info + string origFileName = fileData->fFileName; + IDBDataFile* origFilePtr = fileData->fFilePtr; + origFilePtr->flush(); - // back out the current pointers - int headerSize = compress::CompressInterface::getHdrSize( - fileData->fFileHeader.fControlData); - int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - compress::CompChunkPtrList origPtrs; + // back out the current pointers + int headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); + int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; + compress::CompChunkPtrList origPtrs; - if (compress::CompressInterface::getPtrList( - fileData->fFileHeader.fPtrSection, ptrSecSize, origPtrs) != 0) + if (compress::CompressInterface::getPtrList(fileData->fFileHeader.fPtrSection, ptrSecSize, origPtrs) != 0) + { + ostringstream oss; + oss << "Chunk shifting failed, file:" << origFileName << " -- invalid header."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_COMP_PARSE_HDRS; + } + + // get the chunks already in memory + list& chunkList = fileData->fChunkList; + chunkList.sort(chunkDataPtrLessCompare); + list::iterator j = chunkList.begin(); + int numOfChunks = origPtrs.size(); // number of chunks that contain user data + vector chunksTouched; // chunk data is being modified, and in memory + + for (int i = 0; i < numOfChunks; i++) + chunksTouched.push_back(NULL); + + // mark touched chunks + while (j != chunkList.end()) + { + chunksTouched[(*j)->fChunkId] = *j; + j++; + } + + // new file name and pointer + string rlcFileName(fileData->fFileName + ".rlc"); + IDBDataFile* rlcFilePtr = IDBDataFile::open(IDBPolicy::getType(rlcFileName.c_str(), IDBPolicy::WRITEENG), + rlcFileName.c_str(), "w+b", 0, fileData->fColWidth); + + if (!rlcFilePtr) + { + ostringstream oss; + oss << "Chunk shifting failed, file:" << origFileName << " -- cannot open rlc file."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_FILE_OPEN; + } + + // log the recover information here + string aDMLLogFileName; + rc = writeLog(fTransId, "rlc", fileData->fFileName, aDMLLogFileName); + + if (rc != NO_ERROR) + { + ostringstream oss; + oss << "log " << fileData->fFileName << ".rlc to DML logfile failed."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + + delete rlcFilePtr; + rlcFilePtr = NULL; + IDBPolicy::remove(rlcFileName.c_str()); + return rc; + } + + // !!! May conside to use mmap to speed up the copy !!! + // !!! copy the whole file and update the shifting part !!! + + // store updated chunk pointers + uint64_t* ptrs = reinterpret_cast(fileData->fFileHeader.fPtrSection); + ptrs[0] = origPtrs[0].first; // the first chunk offset has no change. + + // bug3913, file size 0 after reallocate. + // write the header, to be updated later, make sure there is someing in the file + if ((rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fControlData, + COMPRESSED_FILE_HEADER_UNIT, __LINE__)) == NO_ERROR) + rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, __LINE__); + + int k = 0; + + for (; k < numOfChunks && rc == NO_ERROR; k++) + { + uint64_t chunkSize = 0; // size of current chunk + unsigned char* buf = NULL; // output buffer + + // Find the current chunk size, and allocate the data -- buf point to the data. + if (chunksTouched[k] == NULL) { + // Chunks not touched will be copied to new file without being uncompressed first. + // cout << "reallocateChunks: chunk has not been updated" << endl; + chunkSize = origPtrs[k].second; + + // read disk data into compressed data buffer + buf = (unsigned char*)fBufCompressed; + + if ((rc = setFileOffset(origFilePtr, origFileName, origPtrs[k].first, __LINE__)) != NO_ERROR) + { ostringstream oss; - oss << "Chunk shifting failed, file:" << origFileName << " -- invalid header."; + oss << "set file offset failed @line:" << __LINE__ << "with retCode:" << rc + << " filename:" << origFileName; logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_COMP_PARSE_HDRS; - } + continue; + } - // get the chunks already in memory - list& chunkList = fileData->fChunkList; - chunkList.sort(chunkDataPtrLessCompare); - list::iterator j = chunkList.begin(); - int numOfChunks = origPtrs.size(); // number of chunks that contain user data - vector chunksTouched; // chunk data is being modified, and in memory - - for (int i = 0; i < numOfChunks; i++) - chunksTouched.push_back(NULL); - - // mark touched chunks - while (j != chunkList.end()) - { - chunksTouched[(*j)->fChunkId] = *j; - j++; - } - - // new file name and pointer - string rlcFileName(fileData->fFileName + ".rlc"); - IDBDataFile* rlcFilePtr = IDBDataFile::open( - IDBPolicy::getType( rlcFileName.c_str(), IDBPolicy::WRITEENG ), - rlcFileName.c_str(), - "w+b", - 0, - fileData->fColWidth ); - - if (!rlcFilePtr) - { + if ((rc = readFile(origFilePtr, origFileName, buf, chunkSize, __LINE__)) != NO_ERROR) + { ostringstream oss; - oss << "Chunk shifting failed, file:" << origFileName << " -- cannot open rlc file."; + oss << "readfile failed @line:" << __LINE__ << "with retCode:" << rc << " filename:" << origFileName; logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_FILE_OPEN; + continue; + } + } + else // chunksTouched[k] != NULL + { + // chunk has been updated, and in memory. + // cout << "reallocateChunks: chunk has been updated" << endl; + ChunkData* chunkData = chunksTouched[k]; + fLenCompressed = fMaxCompressedBufSize; + + auto fCompressor = compress::getCompressorByType(fCompressorPool, fileData->fCompressionType); + if (!fCompressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } + + if ((rc = fCompressor->compressBlock((char*)chunkData->fBufUnCompressed, chunkData->fLenUnCompressed, + (unsigned char*)fBufCompressed, fLenCompressed)) != 0) + { + ostringstream oss; + oss << "Compress data failed @line:" << __LINE__ << "with retCode:" << rc + << " filename:" << rlcFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + + rc = ERR_COMP_COMPRESS; + continue; + } + + WE_COMP_DBG(cout << "Chunk compressed from " << chunkData->fLenUnCompressed << " to " + << fLenCompressed;) + + // shifting chunk, add padding space + if ((rc = fCompressor->padCompressedChunks((unsigned char*)fBufCompressed, fLenCompressed, + fMaxCompressedBufSize)) != 0) + { + WE_COMP_DBG(cout << ", but padding failed." << endl;) + ostringstream oss; + oss << "Compress data failed @line:" << __LINE__ << "with retCode:" << rc + << " filename:" << rlcFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + + rc = ERR_COMP_PAD_DATA; + continue; + } + + WE_COMP_DBG(cout << ", and padded to " << fLenCompressed;) + + buf = (unsigned char*)fBufCompressed; + chunkSize = fLenCompressed; } - // log the recover information here - string aDMLLogFileName; - rc = writeLog(fTransId, "rlc", fileData->fFileName, aDMLLogFileName); + // write is in sequence, no need to call setFileOffset + // cout << "reallocateChunks: writing to temp file " << rlcFileName << " with fileptr " << rlcFilePtr << + // endl; + rc = writeFile(rlcFilePtr, rlcFileName, buf, chunkSize, __LINE__); if (rc != NO_ERROR) { - ostringstream oss; - oss << "log " << fileData->fFileName << ".rlc to DML logfile failed."; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - delete rlcFilePtr; - rlcFilePtr = NULL; - IDBPolicy::remove(rlcFileName.c_str()); - return rc; + // cout << "reallocateChunks: writing to temp file " << rlcFileName << " with fileptr " << rlcFilePtr << + // " failed" << endl; + ostringstream oss; + oss << "write file failed @line:" << __LINE__ << "with retCode:" << rc << " filename:" << rlcFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + continue; } - // !!! May conside to use mmap to speed up the copy !!! - // !!! copy the whole file and update the shifting part !!! + // Update the current chunk size. + ptrs[k + 1] = ptrs[k] + chunkSize; + } - // store updated chunk pointers - uint64_t* ptrs = reinterpret_cast(fileData->fFileHeader.fPtrSection); - ptrs[0] = origPtrs[0].first; // the first chunk offset has no change. + // up to now, everything OK: rc == NO_ERROR + // remove all written chunks from active chunk list. + j = chunkList.begin(); - // bug3913, file size 0 after reallocate. - // write the header, to be updated later, make sure there is someing in the file - if ((rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__)) == NO_ERROR) - rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__); + while (j != chunkList.end()) + { + ChunkData* chunkData = *j; + fActiveChunks.remove(make_pair(fileData->fFileID, chunkData)); + fileData->fChunkList.remove(chunkData); + delete chunkData; - int k = 0; - - for (; k < numOfChunks && rc == NO_ERROR; k++) - { - uint64_t chunkSize = 0; // size of current chunk - unsigned char* buf = NULL; // output buffer - - // Find the current chunk size, and allocate the data -- buf point to the data. - if (chunksTouched[k] == NULL) - { - // Chunks not touched will be copied to new file without being uncompressed first. - //cout << "reallocateChunks: chunk has not been updated" << endl; - chunkSize = origPtrs[k].second; - - // read disk data into compressed data buffer - buf = (unsigned char*)fBufCompressed; - - if ((rc = setFileOffset(origFilePtr, origFileName, origPtrs[k].first, __LINE__)) != NO_ERROR) - { - ostringstream oss; - oss << "set file offset failed @line:" << __LINE__ << "with retCode:" << rc - << " filename:" << origFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - continue; - } - - if ((rc = readFile(origFilePtr, origFileName, buf, chunkSize, __LINE__)) != NO_ERROR) - { - ostringstream oss; - oss << "readfile failed @line:" << __LINE__ << "with retCode:" << rc - << " filename:" << origFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - continue; - } - } - else // chunksTouched[k] != NULL - { - // chunk has been updated, and in memory. - //cout << "reallocateChunks: chunk has been updated" << endl; - ChunkData* chunkData = chunksTouched[k]; - fLenCompressed = fMaxCompressedBufSize; - - auto fCompressor = compress::getCompressorByType( - fCompressorPool, fileData->fCompressionType); - if (!fCompressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } - - if ((rc = fCompressor->compressBlock((char*)chunkData->fBufUnCompressed, - chunkData->fLenUnCompressed, - (unsigned char*)fBufCompressed, - fLenCompressed)) != 0) - { - ostringstream oss; - oss << "Compress data failed @line:" << __LINE__ << "with retCode:" << rc - << " filename:" << rlcFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - rc = ERR_COMP_COMPRESS; - continue; - } - - WE_COMP_DBG(cout << "Chunk compressed from " << chunkData->fLenUnCompressed << " to " - << fLenCompressed;) - - // shifting chunk, add padding space - if ((rc = fCompressor->padCompressedChunks( - (unsigned char*)fBufCompressed, fLenCompressed, fMaxCompressedBufSize)) != 0) - { - WE_COMP_DBG(cout << ", but padding failed." << endl;) - ostringstream oss; - oss << "Compress data failed @line:" << __LINE__ << "with retCode:" << rc - << " filename:" << rlcFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - rc = ERR_COMP_PAD_DATA; - continue; - } - - WE_COMP_DBG(cout << ", and padded to " << fLenCompressed;) - - buf = (unsigned char*)fBufCompressed; - chunkSize = fLenCompressed; - } - - // write is in sequence, no need to call setFileOffset - //cout << "reallocateChunks: writing to temp file " << rlcFileName << " with fileptr " << rlcFilePtr << endl; - rc = writeFile(rlcFilePtr, rlcFileName, buf, chunkSize, __LINE__); - - if (rc != NO_ERROR) - { - //cout << "reallocateChunks: writing to temp file " << rlcFileName << " with fileptr " << rlcFilePtr << " failed" << endl; - ostringstream oss; - oss << "write file failed @line:" << __LINE__ << "with retCode:" << rc - << " filename:" << rlcFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - continue; - } - - // Update the current chunk size. - ptrs[k + 1] = ptrs[k] + chunkSize; - } - - // up to now, everything OK: rc == NO_ERROR - // remove all written chunks from active chunk list. j = chunkList.begin(); + } - while (j != chunkList.end()) - { - ChunkData* chunkData = *j; - fActiveChunks.remove(make_pair(fileData->fFileID, chunkData)); - fileData->fChunkList.remove(chunkData); - delete chunkData; + // finally update the header + if (rc == NO_ERROR) + rc = setFileOffset(rlcFilePtr, rlcFileName, 0, __LINE__); - j = chunkList.begin(); - } + if (rc == NO_ERROR) + rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fControlData, COMPRESSED_FILE_HEADER_UNIT, + __LINE__); - // finally update the header - if (rc == NO_ERROR) - rc = setFileOffset(rlcFilePtr, rlcFileName, 0, __LINE__); + if (rc == NO_ERROR) + rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, __LINE__); - if (rc == NO_ERROR) - rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__); - - if (rc == NO_ERROR) - rc = writeFile(rlcFilePtr, rlcFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__); - - if (rc != NO_ERROR) - { - struct timeval tv; - gettimeofday(&tv, 0); - struct tm ltm; - localtime_r(reinterpret_cast(&tv.tv_sec), <m); - char tmText[24]; + if (rc != NO_ERROR) + { + struct timeval tv; + gettimeofday(&tv, 0); + struct tm ltm; + localtime_r(reinterpret_cast(&tv.tv_sec), <m); + char tmText[24]; // this snprintf call causes a compiler warning b/c buffer size is less // then maximum string size. #if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" - snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", - ltm.tm_year + 1900, ltm.tm_mon + 1, - ltm.tm_mday, ltm.tm_hour, ltm.tm_min, - ltm.tm_sec, tv.tv_usec); + snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", ltm.tm_year + 1900, ltm.tm_mon + 1, + ltm.tm_mday, ltm.tm_hour, ltm.tm_min, ltm.tm_sec, tv.tv_usec); #pragma GCC diagnostic pop #else - snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", - ltm.tm_year + 1900, ltm.tm_mon + 1, - ltm.tm_mday, ltm.tm_hour, ltm.tm_min, - ltm.tm_sec, tv.tv_usec); + snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", ltm.tm_year + 1900, ltm.tm_mon + 1, + ltm.tm_mday, ltm.tm_hour, ltm.tm_min, ltm.tm_sec, tv.tv_usec); #endif - string dbgFileName(rlcFileName + tmText); + string dbgFileName(rlcFileName + tmText); - ostringstream oss; - oss << "Chunk shifting failed, file:" << origFileName; + ostringstream oss; + oss << "Chunk shifting failed, file:" << origFileName; - if (IDBPolicy::rename(rlcFileName.c_str(), dbgFileName.c_str()) == 0) - oss << ", rlc file is:" << dbgFileName; + if (IDBPolicy::rename(rlcFileName.c_str(), dbgFileName.c_str()) == 0) + oss << ", rlc file is:" << dbgFileName; - // write out the header for debugging in case the header in rlc file is bad or not updated. - string rlcPtrFileName(dbgFileName + ".ptr"); + // write out the header for debugging in case the header in rlc file is bad or not updated. + string rlcPtrFileName(dbgFileName + ".ptr"); - IDBDataFile* rlcPtrFilePtr = IDBDataFile::open( - IDBPolicy::getType(rlcPtrFileName.c_str(), - IDBPolicy::WRITEENG), - rlcPtrFileName.c_str(), - "w+b", - 0, - fileData->fColWidth); + IDBDataFile* rlcPtrFilePtr = + IDBDataFile::open(IDBPolicy::getType(rlcPtrFileName.c_str(), IDBPolicy::WRITEENG), + rlcPtrFileName.c_str(), "w+b", 0, fileData->fColWidth); - if (rlcPtrFilePtr && - (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__) == NO_ERROR) && - (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__) == NO_ERROR)) - { - oss << ", rlc file header in memory: " << rlcPtrFileName; - } - else - { - oss << ", possible incomplete rlc file header in memory: " << rlcPtrFileName; - } - - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - closeFile(fileData); - delete rlcFilePtr; - rlcFilePtr = NULL; - - if (rlcPtrFilePtr != NULL) - { - delete rlcPtrFilePtr; - rlcPtrFilePtr = NULL; - } - - return rc; + if (rlcPtrFilePtr && + (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fControlData, + COMPRESSED_FILE_HEADER_UNIT, __LINE__) == NO_ERROR) && + (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, __LINE__) == + NO_ERROR)) + { + oss << ", rlc file header in memory: " << rlcPtrFileName; + } + else + { + oss << ", possible incomplete rlc file header in memory: " << rlcPtrFileName; } - // update the file pointer map w/ new file pointer - //cout << "realloc1: remove ptr = " << fileData->fFilePtr << endl; - fFilePtrMap.erase(fileData->fFilePtr); - delete fileData->fFilePtr; - fileData->fFilePtr = NULL; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + + closeFile(fileData); delete rlcFilePtr; rlcFilePtr = NULL; - // put reallocated file size here for logging purpose. - uint64_t fileSize = 0; + if (rlcPtrFilePtr != NULL) + { + delete rlcPtrFilePtr; + rlcPtrFilePtr = NULL; + } + + return rc; + } + + // update the file pointer map w/ new file pointer + // cout << "realloc1: remove ptr = " << fileData->fFilePtr << endl; + fFilePtrMap.erase(fileData->fFilePtr); + delete fileData->fFilePtr; + fileData->fFilePtr = NULL; + delete rlcFilePtr; + rlcFilePtr = NULL; + + // put reallocated file size here for logging purpose. + uint64_t fileSize = 0; + + if (rc == NO_ERROR) + { +#ifdef _MSC_VER + // We need to do this early on so the ::rename() call below will work on Windows + // we'll do it again later on, but that's life... + // FIXME: there's a race here that a query will re-open the files before we can jigger + // them around. We need to make sure PP is opening these files with the right perms + // to allow another process to delete them. + cacheutils::dropPrimProcFdCache(); +#endif + + // @bug3913, keep the original file until the new file is properly renamed. + // 1. check the new file size is NOT 0, matching ptr[k]. + // 2. mv the current to be backup. + // 3. rename the rlc file. + // 4. check the file size again. + // 5. verify each chunk. + // 5. rm the bak file or mv bak file back. + + // check the new file size using two methods mostly for curiosity on 0 size file. + // They can be removed because all chunks are to be verified after rename. + if (IDBPolicy::size(rlcFileName.c_str()) != (int64_t)ptrs[k]) + { + ostringstream oss; + oss << "Incorrect file size, expect:" << ptrs[k] << ", stat:" << fileSize + << ", filename:" << rlcFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + rc = ERR_COMP_RENAME_FILE; + } if (rc == NO_ERROR) { -#ifdef _MSC_VER - //We need to do this early on so the ::rename() call below will work on Windows - // we'll do it again later on, but that's life... - //FIXME: there's a race here that a query will re-open the files before we can jigger - // them around. We need to make sure PP is opening these files with the right perms - // to allow another process to delete them. - cacheutils::dropPrimProcFdCache(); -#endif + if (fIsHdfs) + rc = swapTmpFile(rlcFileName, fileData->fFileName + ".tmp"); + else + rc = swapTmpFile(rlcFileName, fileData->fFileName); + } - // @bug3913, keep the original file until the new file is properly renamed. - // 1. check the new file size is NOT 0, matching ptr[k]. - // 2. mv the current to be backup. - // 3. rename the rlc file. - // 4. check the file size again. - // 5. verify each chunk. - // 5. rm the bak file or mv bak file back. + if ((rc == NO_ERROR) && (rc = openFile(fileData, "r+b", fileData->fColWidth, true, __LINE__)) == + NO_ERROR) // @bug 5572 HDFS tmp file + { + // see TODO- above regarding setvbuf + // setvbuf(fileData->fFilePtr, fileData->fIoBuffer.get(), _IOFBF, fileData->fIoBSize); + fileSize = fileData->fFilePtr->size(); - // check the new file size using two methods mostly for curiosity on 0 size file. - // They can be removed because all chunks are to be verified after rename. - if ( IDBPolicy::size( rlcFileName.c_str() ) != (int64_t) ptrs[k] ) - { - ostringstream oss; - oss << "Incorrect file size, expect:" << ptrs[k] << ", stat:" << fileSize - << ", filename:" << rlcFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - rc = ERR_COMP_RENAME_FILE; - } + if (fileSize == ptrs[k]) + { + rc = verifyChunksAfterRealloc(fileData); + } + else + { + ostringstream oss; + oss << "Incorrect file size, expect:" << ptrs[k] << ", stat:" << fileSize + << ", filename:" << fileData->fFileName; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + rc = ERR_COMP_RENAME_FILE; + } - if (rc == NO_ERROR) - { - if (fIsHdfs) - rc = swapTmpFile(rlcFileName, fileData->fFileName + ".tmp"); - else - rc = swapTmpFile(rlcFileName, fileData->fFileName); - } - - if ((rc == NO_ERROR) && - (rc = openFile(fileData, "r+b", fileData->fColWidth, true, __LINE__)) == NO_ERROR) // @bug 5572 HDFS tmp file - { -// see TODO- above regarding setvbuf -// setvbuf(fileData->fFilePtr, fileData->fIoBuffer.get(), _IOFBF, fileData->fIoBSize); - fileSize = fileData->fFilePtr->size(); - - if (fileSize == ptrs[k]) - { - rc = verifyChunksAfterRealloc(fileData); - } - else - { - ostringstream oss; - oss << "Incorrect file size, expect:" << ptrs[k] << ", stat:" << fileSize - << ", filename:" << fileData->fFileName; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - rc = ERR_COMP_RENAME_FILE; - } - - if (rc == NO_ERROR) - { - fFilePtrMap.insert(make_pair(fileData->fFilePtr, fileData)); - //cout << "realloc2: insert ptr = " << fileData->fFilePtr << endl; - // notify the PrimProc of unlinking original data file - fDropFdCache = true; - } - } - - if (!fIsHdfs) - { - string bakFileName(fileData->fFileName + ".orig"); - - if (rc == NO_ERROR) - { - // unlink the original file (remove is portable) - if (fFs.remove(bakFileName.c_str()) != 0) - { - ostringstream oss; - oss << "remove backup file " << bakFileName << " failed: " << strerror(errno); - - // not much we can do, log an info message for manual cleanup - logMessage(oss.str(), logging::LOG_TYPE_INFO); - } - } - else - { - // keep the bad file for debugging purpose - if (fFs.rename(fileData->fFileName.c_str(), rlcFileName.c_str()) == 0) - { - ostringstream oss; - oss << "data file after chunk shifting failed verification."; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } - - // roll back the bak file - if (fFs.rename(bakFileName.c_str(), fileData->fFileName.c_str()) != 0) - { - ostringstream oss; - oss << "rename " << bakFileName << " to " << fileData->fFileName << " failed: " - << strerror(errno); - - // must manually move it back - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - } - } - } + if (rc == NO_ERROR) + { + fFilePtrMap.insert(make_pair(fileData->fFilePtr, fileData)); + // cout << "realloc2: insert ptr = " << fileData->fFilePtr << endl; + // notify the PrimProc of unlinking original data file + fDropFdCache = true; + } } if (!fIsHdfs) { - if (rc == NO_ERROR) + string bakFileName(fileData->fFileName + ".orig"); + + if (rc == NO_ERROR) + { + // unlink the original file (remove is portable) + if (fFs.remove(bakFileName.c_str()) != 0) { - // remove the log file - fFs.remove(aDMLLogFileName.c_str()); + ostringstream oss; + oss << "remove backup file " << bakFileName << " failed: " << strerror(errno); + + // not much we can do, log an info message for manual cleanup + logMessage(oss.str(), logging::LOG_TYPE_INFO); } - else + } + else + { + // keep the bad file for debugging purpose + if (fFs.rename(fileData->fFileName.c_str(), rlcFileName.c_str()) == 0) { - struct timeval tv; - gettimeofday(&tv, 0); - struct tm ltm; - localtime_r(reinterpret_cast(&tv.tv_sec), <m); - char tmText[24]; - // this snprintf call causes a compiler warning b/c buffer size is less - // then maximum string size. + ostringstream oss; + oss << "data file after chunk shifting failed verification."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + } + + // roll back the bak file + if (fFs.rename(bakFileName.c_str(), fileData->fFileName.c_str()) != 0) + { + ostringstream oss; + oss << "rename " << bakFileName << " to " << fileData->fFileName << " failed: " << strerror(errno); + + // must manually move it back + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + } + } + } + } + + if (!fIsHdfs) + { + if (rc == NO_ERROR) + { + // remove the log file + fFs.remove(aDMLLogFileName.c_str()); + } + else + { + struct timeval tv; + gettimeofday(&tv, 0); + struct tm ltm; + localtime_r(reinterpret_cast(&tv.tv_sec), <m); + char tmText[24]; + // this snprintf call causes a compiler warning b/c buffer size is less + // then maximum string size. #if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" - snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", - ltm.tm_year + 1900, ltm.tm_mon + 1, - ltm.tm_mday, ltm.tm_hour, ltm.tm_min, - ltm.tm_sec, tv.tv_usec); + snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", ltm.tm_year + 1900, ltm.tm_mon + 1, + ltm.tm_mday, ltm.tm_hour, ltm.tm_min, ltm.tm_sec, tv.tv_usec); #pragma GCC diagnostic pop #else - snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", - ltm.tm_year + 1900, ltm.tm_mon + 1, - ltm.tm_mday, ltm.tm_hour, ltm.tm_min, - ltm.tm_sec, tv.tv_usec); + snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", ltm.tm_year + 1900, ltm.tm_mon + 1, + ltm.tm_mday, ltm.tm_hour, ltm.tm_min, ltm.tm_sec, tv.tv_usec); #endif - string dbgFileName(rlcFileName + tmText); + string dbgFileName(rlcFileName + tmText); - ostringstream oss; - oss << "Chunk shifting failed, file:" << origFileName; + ostringstream oss; + oss << "Chunk shifting failed, file:" << origFileName; - if (IDBPolicy::rename(rlcFileName.c_str(), dbgFileName.c_str()) == 0) - oss << ", rlc file is:" << dbgFileName; + if (IDBPolicy::rename(rlcFileName.c_str(), dbgFileName.c_str()) == 0) + oss << ", rlc file is:" << dbgFileName; - // write out the header for debugging in case the header in rlc file is bad. - string rlcPtrFileName(dbgFileName + ".hdr"); - IDBDataFile* rlcPtrFilePtr = IDBDataFile::open( - IDBPolicy::getType(rlcPtrFileName.c_str(), - IDBPolicy::WRITEENG), - rlcPtrFileName.c_str(), - "w+b", - 0); + // write out the header for debugging in case the header in rlc file is bad. + string rlcPtrFileName(dbgFileName + ".hdr"); + IDBDataFile* rlcPtrFilePtr = IDBDataFile::open( + IDBPolicy::getType(rlcPtrFileName.c_str(), IDBPolicy::WRITEENG), rlcPtrFileName.c_str(), "w+b", 0); - if (rlcPtrFilePtr && - (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__) == NO_ERROR) && - (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__) == NO_ERROR)) - { - oss << ", rlc file header in memory: " << rlcPtrFileName; - } - else - { - oss << ", possible incomplete rlc file header in memory: " << rlcPtrFileName; - } + if (rlcPtrFilePtr && + (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fControlData, + COMPRESSED_FILE_HEADER_UNIT, __LINE__) == NO_ERROR) && + (writeFile(rlcPtrFilePtr, rlcPtrFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, + __LINE__) == NO_ERROR)) + { + oss << ", rlc file header in memory: " << rlcPtrFileName; + } + else + { + oss << ", possible incomplete rlc file header in memory: " << rlcPtrFileName; + } - logMessage(oss.str(), logging::LOG_TYPE_ERROR); + logMessage(oss.str(), logging::LOG_TYPE_ERROR); - closeFile(fileData); + closeFile(fileData); - if (rlcFilePtr != NULL) - { - delete rlcFilePtr; - rlcFilePtr = NULL; - } + if (rlcFilePtr != NULL) + { + delete rlcFilePtr; + rlcFilePtr = NULL; + } - if (rlcPtrFilePtr != NULL) - { - delete rlcPtrFilePtr; - rlcPtrFilePtr = NULL; - } - } + if (rlcPtrFilePtr != NULL) + { + delete rlcPtrFilePtr; + rlcPtrFilePtr = NULL; + } } + } - return rc; + return rc; } //------------------------------------------------------------------------------ @@ -2273,103 +2180,98 @@ int ChunkManager::reallocateChunks(CompFileData* fileData) //------------------------------------------------------------------------------ int ChunkManager::verifyChunksAfterRealloc(CompFileData* fileData) { - int rc = NO_ERROR; + int rc = NO_ERROR; - // read in the header - if ((rc = readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fControlData, - COMPRESSED_FILE_HEADER_UNIT, __LINE__)) != NO_ERROR) - { - ostringstream oss; - oss << "Failed to read control header from new " << fileData->fFileName << ", roll back"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - return rc; - } - - // make sure the header is valid - if ((rc = compress::CompressInterface::verifyHdr( - fileData->fFileHeader.fControlData)) != 0) - { - ostringstream oss; - oss << "Invalid header in new " << fileData->fFileName << ", roll back"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - - return rc; - } - - int headerSize = compress::CompressInterface::getHdrSize( - fileData->fFileHeader.fControlData); - int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - - // read in the pointer section in header - if ((rc = readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fPtrSection, - ptrSecSize, __LINE__)) != NO_ERROR) - { - ostringstream oss; - oss << "Failed to read pointer header from new " << fileData->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return rc; - } - - // get pointer list - compress::CompChunkPtrList ptrs; - - if (compress::CompressInterface::getPtrList( - fileData->fFileHeader.fPtrSection, ptrSecSize, ptrs) != 0) - { - ostringstream oss; - oss << "Failed to parse pointer list from new " << fileData->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_COMP_PARSE_HDRS; - } - - // now verify each chunk - ChunkData chunkData; - int numOfChunks = ptrs.size(); // number of chunks in the file - - auto fCompressor = compress::getCompressorByType( - fCompressorPool, fileData->fCompressionType); - if (!fCompressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } - - for (int i = 0; i < numOfChunks && rc == NO_ERROR; i++) - { - unsigned int chunkSize = ptrs[i].second; - - if ((rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, ptrs[i].first, __LINE__))) - { - ostringstream oss; - oss << "Failed to setFileOffset new " << fileData->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - continue; - } - - if ((rc = readFile(fileData->fFilePtr, fileData->fFileName, - fBufCompressed, chunkSize, __LINE__))) - { - ostringstream oss; - oss << "Failed to read chunk from new " << fileData->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - continue; - } - - // uncompress the read in buffer - size_t dataLen = sizeof(chunkData.fBufUnCompressed); - - if (fCompressor->uncompressBlock((char*)fBufCompressed, chunkSize, - (unsigned char*)chunkData.fBufUnCompressed, dataLen) != 0) - { - ostringstream oss; - oss << "Failed to uncompress chunk new " << fileData->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - rc = ERR_COMP_UNCOMPRESS; - continue; - } - } + // read in the header + if ((rc = readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fControlData, + COMPRESSED_FILE_HEADER_UNIT, __LINE__)) != NO_ERROR) + { + ostringstream oss; + oss << "Failed to read control header from new " << fileData->fFileName << ", roll back"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); return rc; + } + + // make sure the header is valid + if ((rc = compress::CompressInterface::verifyHdr(fileData->fFileHeader.fControlData)) != 0) + { + ostringstream oss; + oss << "Invalid header in new " << fileData->fFileName << ", roll back"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + + return rc; + } + + int headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); + int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; + + // read in the pointer section in header + if ((rc = readFile(fileData->fFilePtr, fileData->fFileName, fileData->fFileHeader.fPtrSection, ptrSecSize, + __LINE__)) != NO_ERROR) + { + ostringstream oss; + oss << "Failed to read pointer header from new " << fileData->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return rc; + } + + // get pointer list + compress::CompChunkPtrList ptrs; + + if (compress::CompressInterface::getPtrList(fileData->fFileHeader.fPtrSection, ptrSecSize, ptrs) != 0) + { + ostringstream oss; + oss << "Failed to parse pointer list from new " << fileData->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_COMP_PARSE_HDRS; + } + + // now verify each chunk + ChunkData chunkData; + int numOfChunks = ptrs.size(); // number of chunks in the file + + auto fCompressor = compress::getCompressorByType(fCompressorPool, fileData->fCompressionType); + if (!fCompressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } + + for (int i = 0; i < numOfChunks && rc == NO_ERROR; i++) + { + unsigned int chunkSize = ptrs[i].second; + + if ((rc = setFileOffset(fileData->fFilePtr, fileData->fFileName, ptrs[i].first, __LINE__))) + { + ostringstream oss; + oss << "Failed to setFileOffset new " << fileData->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + continue; + } + + if ((rc = readFile(fileData->fFilePtr, fileData->fFileName, fBufCompressed, chunkSize, __LINE__))) + { + ostringstream oss; + oss << "Failed to read chunk from new " << fileData->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + continue; + } + + // uncompress the read in buffer + size_t dataLen = sizeof(chunkData.fBufUnCompressed); + + if (fCompressor->uncompressBlock((char*)fBufCompressed, chunkSize, + (unsigned char*)chunkData.fBufUnCompressed, dataLen) != 0) + { + ostringstream oss; + oss << "Failed to uncompress chunk new " << fileData->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + rc = ERR_COMP_UNCOMPRESS; + continue; + } + } + + return rc; } //------------------------------------------------------------------------------ @@ -2377,13 +2279,13 @@ int ChunkManager::verifyChunksAfterRealloc(CompFileData* fileData) //------------------------------------------------------------------------------ void ChunkManager::logMessage(int code, int level, int lineNum, int fromLine) const { - ostringstream oss; - oss << ec.errorString(code) << " @line:" << lineNum; + ostringstream oss; + oss << ec.errorString(code) << " @line:" << lineNum; - if (fromLine != -1) - oss << " called from line:" << fromLine; + if (fromLine != -1) + oss << " called from line:" << fromLine; - logMessage(oss.str(), level); + logMessage(oss.str(), level); } //------------------------------------------------------------------------------ @@ -2391,12 +2293,12 @@ void ChunkManager::logMessage(int code, int level, int lineNum, int fromLine) co //------------------------------------------------------------------------------ void ChunkManager::logMessage(const string& msg, int level) const { - logging::Message::Args args; - args.add(msg); + logging::Message::Args args; + args.add(msg); - fSysLogger->logMessage((logging::LOG_TYPE) level, logging::M0080, args, - //FIXME: store session id in class to pass on to LogginID... - logging::LoggingID(SUBSYSTEM_ID_WE, 0, fTransId)); + fSysLogger->logMessage((logging::LOG_TYPE)level, logging::M0080, args, + // FIXME: store session id in class to pass on to LogginID... + logging::LoggingID(SUBSYSTEM_ID_WE, 0, fTransId)); } //------------------------------------------------------------------------------ @@ -2404,185 +2306,178 @@ void ChunkManager::logMessage(const string& msg, int level) const //------------------------------------------------------------------------------ int ChunkManager::swapTmpFile(const string& src, const string& dest) { - // return value - int rc = NO_ERROR; - - // if no change to the cdf, the tmp may not exist, no need to swap. - if (!fFs.exists(src.c_str())) - return rc; + // return value + int rc = NO_ERROR; - ssize_t srcFileSize = IDBPolicy::size(src.c_str()); + // if no change to the cdf, the tmp may not exist, no need to swap. + if (!fFs.exists(src.c_str())) + return rc; - if (srcFileSize <= 0) - { - ostringstream oss; - oss << "swapTmpFile aborted. Source file size = " << srcFileSize; - logMessage(oss.str(), logging::LOG_TYPE_CRITICAL); - rc = ERR_COMP_RENAME_FILE; + ssize_t srcFileSize = IDBPolicy::size(src.c_str()); - return rc; - } - - errno = 0; - // save the original file - string orig(dest + ".orig"); - fFs.remove(orig.c_str()); // remove left overs - - if (fFs.rename(dest.c_str(), orig.c_str()) != 0) - { - ostringstream oss; - oss << "rename " << dest << " to " << orig << " failed: " << strerror(errno); - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - rc = ERR_COMP_RENAME_FILE; - } - - // rename the new file - if (rc == NO_ERROR && fFs.rename(src.c_str(), dest.c_str()) != 0) - { - ostringstream oss; - oss << "rename " << src << " to " << dest << " failed: " << strerror(errno); - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - rc = ERR_COMP_RENAME_FILE; - } - - if (rc == NO_ERROR && fFs.remove(orig.c_str()) != 0) - rc = ERR_COMP_REMOVE_FILE; + if (srcFileSize <= 0) + { + ostringstream oss; + oss << "swapTmpFile aborted. Source file size = " << srcFileSize; + logMessage(oss.str(), logging::LOG_TYPE_CRITICAL); + rc = ERR_COMP_RENAME_FILE; return rc; -} + } + errno = 0; + // save the original file + string orig(dest + ".orig"); + fFs.remove(orig.c_str()); // remove left overs + + if (fFs.rename(dest.c_str(), orig.c_str()) != 0) + { + ostringstream oss; + oss << "rename " << dest << " to " << orig << " failed: " << strerror(errno); + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + rc = ERR_COMP_RENAME_FILE; + } + + // rename the new file + if (rc == NO_ERROR && fFs.rename(src.c_str(), dest.c_str()) != 0) + { + ostringstream oss; + oss << "rename " << src << " to " << dest << " failed: " << strerror(errno); + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + rc = ERR_COMP_RENAME_FILE; + } + + if (rc == NO_ERROR && fFs.remove(orig.c_str()) != 0) + rc = ERR_COMP_REMOVE_FILE; + + return rc; +} //------------------------------------------------------------------------------ // Construct a DML log file name based on transaction ID, etc. //------------------------------------------------------------------------------ int ChunkManager::getDMLLogFileName(string& aDMLLogFileName, const TxnID& txnId) const { - config::Config* config = config::Config::makeConfig(); - string prefix = config->getConfig("SystemConfig", "DBRMRoot"); - - if (prefix.length() == 0) - { - ostringstream oss; - oss << "trans " << txnId << ":Need a valid DBRMRoot entry in Calpont configuation file"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_DML_LOG_NAME; - } - - uint64_t pos = prefix.find_last_of ("/") ; - - if (pos != string::npos) - { - aDMLLogFileName = prefix.substr(0, pos + 1); //Get the file path - } - else - { - ostringstream oss; - oss << "trans " << txnId << ":Cannot find the dbrm directory (" - << prefix.c_str() << ") for the DML log file"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_DML_LOG_NAME; - } + config::Config* config = config::Config::makeConfig(); + string prefix = config->getConfig("SystemConfig", "DBRMRoot"); + if (prefix.length() == 0) + { ostringstream oss; - oss << txnId << "_" << fLocalModuleId; - aDMLLogFileName += "DMLLog_" + oss.str(); + oss << "trans " << txnId << ":Need a valid DBRMRoot entry in Calpont configuation file"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_DML_LOG_NAME; + } - return NO_ERROR; + uint64_t pos = prefix.find_last_of("/"); + + if (pos != string::npos) + { + aDMLLogFileName = prefix.substr(0, pos + 1); // Get the file path + } + else + { + ostringstream oss; + oss << "trans " << txnId << ":Cannot find the dbrm directory (" << prefix.c_str() + << ") for the DML log file"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_DML_LOG_NAME; + } + + ostringstream oss; + oss << txnId << "_" << fLocalModuleId; + aDMLLogFileName += "DMLLog_" + oss.str(); + + return NO_ERROR; } - //------------------------------------------------------------------------------ // clear the DML log file //------------------------------------------------------------------------------ int ChunkManager::startTransaction(const TxnID& txnId) const { - // this step is for HDFS update/delete only. - if (!fIsHdfs || fIsBulkLoad) - return NO_ERROR; - - // Construct the DML log file name - string aDMLLogFileName; - - if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) - return ERR_DML_LOG_NAME; - - // truncate the existing file - boost::scoped_ptr aDMLLogFile(IDBDataFile::open( - IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), - aDMLLogFileName.c_str(), "wb", 0)); - - if (!aDMLLogFile) - { - ostringstream oss; - oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened."; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_OPEN_DML_LOG; - } - + // this step is for HDFS update/delete only. + if (!fIsHdfs || fIsBulkLoad) return NO_ERROR; -} + // Construct the DML log file name + string aDMLLogFileName; + + if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) + return ERR_DML_LOG_NAME; + + // truncate the existing file + boost::scoped_ptr aDMLLogFile(IDBDataFile::open( + IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), aDMLLogFileName.c_str(), "wb", 0)); + + if (!aDMLLogFile) + { + ostringstream oss; + oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened."; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_OPEN_DML_LOG; + } + + return NO_ERROR; +} //------------------------------------------------------------------------------ // Backup cdf file and replace the with the updated tmp file. //------------------------------------------------------------------------------ int ChunkManager::confirmTransaction(const TxnID& txnId) const { - // return value - int rc = NO_ERROR; - - // this step is for HDFS update/delete only. - if (!fIsHdfs || fIsBulkLoad) - return rc; - - string aDMLLogFileName; - - if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) - return ERR_DML_LOG_NAME; - - //Open log file - boost::scoped_ptr aDMLLogFile; - aDMLLogFile.reset(IDBDataFile::open( - IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), - aDMLLogFileName.c_str(), "r", 0)); - - if (!aDMLLogFile) - { - ostringstream oss; - oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_OPEN_DML_LOG; - } - - ssize_t logSize = fFs.size(aDMLLogFileName.c_str()); - boost::scoped_array buf(new char[logSize]); - - if (aDMLLogFile->read(buf.get(), logSize) != logSize) - return ERR_FILE_READ; - - std::istringstream strstream(string(buf.get(), logSize)); - std::string backUpFileType; - std::string filename; - int64_t size; - int64_t offset; - ConfirmHdfsDbFile confirmHdfs; - - while (strstream >> backUpFileType >> filename >> size >> offset) - { - std::string confirmErrMsg; - rc = confirmHdfs.confirmDbFileChange( backUpFileType, - filename, confirmErrMsg ); - - if (rc != NO_ERROR) - { - logMessage(confirmErrMsg, logging::LOG_TYPE_ERROR); - break; - } - } + // return value + int rc = NO_ERROR; + // this step is for HDFS update/delete only. + if (!fIsHdfs || fIsBulkLoad) return rc; -} + string aDMLLogFileName; + + if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) + return ERR_DML_LOG_NAME; + + // Open log file + boost::scoped_ptr aDMLLogFile; + aDMLLogFile.reset(IDBDataFile::open(IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), + aDMLLogFileName.c_str(), "r", 0)); + + if (!aDMLLogFile) + { + ostringstream oss; + oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_OPEN_DML_LOG; + } + + ssize_t logSize = fFs.size(aDMLLogFileName.c_str()); + boost::scoped_array buf(new char[logSize]); + + if (aDMLLogFile->read(buf.get(), logSize) != logSize) + return ERR_FILE_READ; + + std::istringstream strstream(string(buf.get(), logSize)); + std::string backUpFileType; + std::string filename; + int64_t size; + int64_t offset; + ConfirmHdfsDbFile confirmHdfs; + + while (strstream >> backUpFileType >> filename >> size >> offset) + { + std::string confirmErrMsg; + rc = confirmHdfs.confirmDbFileChange(backUpFileType, filename, confirmErrMsg); + + if (rc != NO_ERROR) + { + logMessage(confirmErrMsg, logging::LOG_TYPE_ERROR); + break; + } + } + + return rc; +} //------------------------------------------------------------------------------ // Finalize the chages @@ -2591,169 +2486,158 @@ int ChunkManager::confirmTransaction(const TxnID& txnId) const //------------------------------------------------------------------------------ int ChunkManager::endTransaction(const TxnID& txnId, bool success) const { - // return value - int rc = NO_ERROR; - - // this step is for HDFS update/delete only. - if (!fIsHdfs || fIsBulkLoad) - return rc; - - string aDMLLogFileName; - - if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) - return ERR_DML_LOG_NAME; - - //Open log file - boost::scoped_ptr aDMLLogFile; - aDMLLogFile.reset(IDBDataFile::open( - IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), - aDMLLogFileName.c_str(), "r", 0)); - - if (!aDMLLogFile) - { - ostringstream oss; - oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened"; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_OPEN_DML_LOG; - } - - ssize_t logSize = fFs.size(aDMLLogFileName.c_str()); - ssize_t logRead = 0; - boost::scoped_array buf(new char[logSize]); - - if ((logRead = aDMLLogFile->read(buf.get(), logSize)) != logSize) - { - ostringstream oss; - oss << "trans " << txnId << ":File " << aDMLLogFileName << " filed to read: " - << logRead << "/" << logSize; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_FILE_READ; - } - - std::istringstream strstream(string(buf.get(), logSize)); - std::string backUpFileType; - std::string filename; - int64_t size; - int64_t offset; - ConfirmHdfsDbFile confirmHdfs; - - while (strstream >> backUpFileType >> filename >> size >> offset) - { - std::string finalizeErrMsg; - rc = confirmHdfs.endDbFileChange( backUpFileType, - filename, success, finalizeErrMsg ); - - if (rc != NO_ERROR) - { - logMessage(finalizeErrMsg, logging::LOG_TYPE_ERROR); - break; - } - } - - // final clean up or recover - if (rc == NO_ERROR) - rc = fFs.remove(aDMLLogFileName.c_str()); + // return value + int rc = NO_ERROR; + // this step is for HDFS update/delete only. + if (!fIsHdfs || fIsBulkLoad) return rc; + + string aDMLLogFileName; + + if (getDMLLogFileName(aDMLLogFileName, txnId) != NO_ERROR) + return ERR_DML_LOG_NAME; + + // Open log file + boost::scoped_ptr aDMLLogFile; + aDMLLogFile.reset(IDBDataFile::open(IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), + aDMLLogFileName.c_str(), "r", 0)); + + if (!aDMLLogFile) + { + ostringstream oss; + oss << "trans " << txnId << ":File " << aDMLLogFileName << " can't be opened"; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_OPEN_DML_LOG; + } + + ssize_t logSize = fFs.size(aDMLLogFileName.c_str()); + ssize_t logRead = 0; + boost::scoped_array buf(new char[logSize]); + + if ((logRead = aDMLLogFile->read(buf.get(), logSize)) != logSize) + { + ostringstream oss; + oss << "trans " << txnId << ":File " << aDMLLogFileName << " filed to read: " << logRead << "/" + << logSize; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_FILE_READ; + } + + std::istringstream strstream(string(buf.get(), logSize)); + std::string backUpFileType; + std::string filename; + int64_t size; + int64_t offset; + ConfirmHdfsDbFile confirmHdfs; + + while (strstream >> backUpFileType >> filename >> size >> offset) + { + std::string finalizeErrMsg; + rc = confirmHdfs.endDbFileChange(backUpFileType, filename, success, finalizeErrMsg); + + if (rc != NO_ERROR) + { + logMessage(finalizeErrMsg, logging::LOG_TYPE_ERROR); + break; + } + } + + // final clean up or recover + if (rc == NO_ERROR) + rc = fFs.remove(aDMLLogFileName.c_str()); + + return rc; } -int ChunkManager::checkFixLastDictChunk(const FID& fid, - uint16_t root, - uint32_t partition, - uint16_t segment) +int ChunkManager::checkFixLastDictChunk(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment) { + int rc = 0; + // Find the file info + FileID fileID(fid, root, partition, segment); + map::const_iterator mit = fFileMap.find(fileID); - int rc = 0; - //Find the file info - FileID fileID(fid, root, partition, segment); - map::const_iterator mit = fFileMap.find(fileID); + WE_COMP_DBG(cout << "getFileData: fid:" << fid << " root:" << root << " part:" << partition << " seg:" + << segment << " file* " << ((mit != fFileMap.end()) ? "" : "not ") << "found." << endl;) - WE_COMP_DBG(cout << "getFileData: fid:" << fid << " root:" << root << " part:" << partition - << " seg:" << segment << " file* " << ((mit != fFileMap.end()) ? "" : "not ") - << "found." << endl;) + // Get CompFileData pointer for existing Dictionary store file mit->second is CompFileData + if (mit != fFileMap.end()) + { + int headerSize = compress::CompressInterface::getHdrSize(mit->second->fFileHeader.fControlData); + int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - // Get CompFileData pointer for existing Dictionary store file mit->second is CompFileData - if (mit != fFileMap.end()) + // get pointer list + compress::CompChunkPtrList ptrs; + + if (compress::CompressInterface::getPtrList(mit->second->fFileHeader.fPtrSection, ptrSecSize, ptrs) != 0) { - - int headerSize = compress::CompressInterface::getHdrSize( - mit->second->fFileHeader.fControlData); - int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; - - // get pointer list - compress::CompChunkPtrList ptrs; - - if (compress::CompressInterface::getPtrList( - mit->second->fFileHeader.fPtrSection, ptrSecSize, ptrs) != 0) - { - ostringstream oss; - oss << "Failed to parse pointer list from new " << mit->second->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return ERR_COMP_PARSE_HDRS; - } - - // now verify last chunk - ChunkData* chunkData; - int numOfChunks = ptrs.size(); // number of chunks in the file - unsigned int chunkSize = ptrs[numOfChunks - 1].second; - - if ((rc = setFileOffset(mit->second->fFilePtr, mit->second->fFileName, ptrs[numOfChunks - 1].first, __LINE__))) - { - ostringstream oss; - oss << "Failed to setFileOffset new " << mit->second->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return rc; - } - - if ((rc = readFile(mit->second->fFilePtr, mit->second->fFileName, - fBufCompressed, chunkSize, __LINE__))) - { - ostringstream oss; - oss << "Failed to read chunk from new " << mit->second->fFileName << "@" << __LINE__; - logMessage(oss.str(), logging::LOG_TYPE_ERROR); - return rc; - } - - // uncompress the read in buffer - chunkData = new ChunkData(numOfChunks - 1); - size_t dataLen = sizeof(chunkData->fBufUnCompressed); - - auto fCompressor = compress::getCompressorByType( - fCompressorPool, mit->second->fCompressionType); - if (!fCompressor) - { - return ERR_COMP_WRONG_COMP_TYPE; - } - - if (fCompressor->uncompressBlock((char*)fBufCompressed, chunkSize, - (unsigned char*)chunkData->fBufUnCompressed, dataLen) != 0) - { - mit->second->fChunkList.push_back(chunkData); - fActiveChunks.push_back(make_pair(mit->second->fFileID, chunkData)); - //replace this chunk with empty chunk - uint64_t blocks = 512; - - if ((numOfChunks - 1) == 0) - { - char* hdr = mit->second->fFileHeader.fControlData; - - if (compress::CompressInterface::getBlockCount(hdr) < 512) - blocks = 256; - } - - dataLen = 8192 * blocks; - - // load the uncompressed buffer with empty values. - char* buf = chunkData->fBufUnCompressed; - chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; - initializeDctnryChunk(buf, UNCOMPRESSED_CHUNK_SIZE); - chunkData->fLenUnCompressed = dataLen; - chunkData->fWriteToFile = true; - } + ostringstream oss; + oss << "Failed to parse pointer list from new " << mit->second->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return ERR_COMP_PARSE_HDRS; } - return rc; -} + // now verify last chunk + ChunkData* chunkData; + int numOfChunks = ptrs.size(); // number of chunks in the file + unsigned int chunkSize = ptrs[numOfChunks - 1].second; + + if ((rc = setFileOffset(mit->second->fFilePtr, mit->second->fFileName, ptrs[numOfChunks - 1].first, + __LINE__))) + { + ostringstream oss; + oss << "Failed to setFileOffset new " << mit->second->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return rc; + } + + if ((rc = readFile(mit->second->fFilePtr, mit->second->fFileName, fBufCompressed, chunkSize, __LINE__))) + { + ostringstream oss; + oss << "Failed to read chunk from new " << mit->second->fFileName << "@" << __LINE__; + logMessage(oss.str(), logging::LOG_TYPE_ERROR); + return rc; + } + + // uncompress the read in buffer + chunkData = new ChunkData(numOfChunks - 1); + size_t dataLen = sizeof(chunkData->fBufUnCompressed); + + auto fCompressor = compress::getCompressorByType(fCompressorPool, mit->second->fCompressionType); + if (!fCompressor) + { + return ERR_COMP_WRONG_COMP_TYPE; + } + + if (fCompressor->uncompressBlock((char*)fBufCompressed, chunkSize, + (unsigned char*)chunkData->fBufUnCompressed, dataLen) != 0) + { + mit->second->fChunkList.push_back(chunkData); + fActiveChunks.push_back(make_pair(mit->second->fFileID, chunkData)); + // replace this chunk with empty chunk + uint64_t blocks = 512; + + if ((numOfChunks - 1) == 0) + { + char* hdr = mit->second->fFileHeader.fControlData; + + if (compress::CompressInterface::getBlockCount(hdr) < 512) + blocks = 256; + } + + dataLen = 8192 * blocks; + + // load the uncompressed buffer with empty values. + char* buf = chunkData->fBufUnCompressed; + chunkData->fLenUnCompressed = UNCOMPRESSED_CHUNK_SIZE; + initializeDctnryChunk(buf, UNCOMPRESSED_CHUNK_SIZE); + chunkData->fLenUnCompressed = dataLen; + chunkData->fWriteToFile = true; + } + } + + return rc; } +} // namespace WriteEngine // vim:ts=4 sw=4: diff --git a/writeengine/shared/we_chunkmanager.h b/writeengine/shared/we_chunkmanager.h index 198c67cce..807d03290 100644 --- a/writeengine/shared/we_chunkmanager.h +++ b/writeengine/shared/we_chunkmanager.h @@ -17,7 +17,6 @@ // $Id: we_chunkmanager.h 4726 2013-08-07 03:38:36Z bwilkinson $ - /** @file */ #ifndef CHUNK_MANAGER_H @@ -42,13 +41,20 @@ #endif #ifdef _MSC_VER -#define WE_COMP_DBG(x) {} +#define WE_COMP_DBG(x) \ + { \ + } #else //#define IDB_COMP_DEBUG #ifdef IDB_COMP_DEBUG -#define WE_COMP_DBG(x) {x} +#define WE_COMP_DBG(x) \ + { \ + x \ + } #else -#define WE_COMP_DBG(x) {} +#define WE_COMP_DBG(x) \ + { \ + } #endif #endif @@ -56,11 +62,10 @@ namespace logging { // use Logger (not we_log) for now. class Logger; -} +} // namespace logging namespace WriteEngine { - // forward reference class FileOp; @@ -76,322 +81,296 @@ const int MAXOFFSET_PER_CHUNK = 511 * BYTE_PER_BLOCK; typedef int64_t ChunkId; struct ChunkData { - ChunkId fChunkId; - unsigned int fLenUnCompressed; - char fBufUnCompressed[UNCOMPRESSED_CHUNK_SIZE]; - bool fWriteToFile; + ChunkId fChunkId; + unsigned int fLenUnCompressed; + char fBufUnCompressed[UNCOMPRESSED_CHUNK_SIZE]; + bool fWriteToFile; - ChunkData(ChunkId id = 0) : fChunkId(id), fLenUnCompressed(0), fWriteToFile(false) {} - bool operator < (const ChunkData& rhs) const - { - return fChunkId < rhs.fChunkId; - } + ChunkData(ChunkId id = 0) : fChunkId(id), fLenUnCompressed(0), fWriteToFile(false) + { + } + bool operator<(const ChunkData& rhs) const + { + return fChunkId < rhs.fChunkId; + } }; // compressed DB file header information struct CompFileHeader { - char fHeaderData[COMPRESSED_FILE_HEADER_UNIT * 2]; - char* fControlData; - char* fPtrSection; - boost::scoped_array fLongPtrSectData; + char fHeaderData[COMPRESSED_FILE_HEADER_UNIT * 2]; + char* fControlData; + char* fPtrSection; + boost::scoped_array fLongPtrSectData; - CompFileHeader() : - fControlData(fHeaderData), fPtrSection(fHeaderData + COMPRESSED_FILE_HEADER_UNIT) {} + CompFileHeader() : fControlData(fHeaderData), fPtrSection(fHeaderData + COMPRESSED_FILE_HEADER_UNIT) + { + } }; - // unique ID of a DB file struct FileID { - FID fFid; - uint32_t fDbRoot; - uint32_t fPartition; - uint32_t fSegment; + FID fFid; + uint32_t fDbRoot; + uint32_t fPartition; + uint32_t fSegment; - FileID(FID f, uint32_t r, uint32_t p, uint32_t s) : - fFid(f), fDbRoot(r), fPartition(p), fSegment(s) {} + FileID(FID f, uint32_t r, uint32_t p, uint32_t s) : fFid(f), fDbRoot(r), fPartition(p), fSegment(s) + { + } - bool operator < (const FileID& rhs) const - { - return ( - (fFid < rhs.fFid) || - (fFid == rhs.fFid && fDbRoot < rhs.fDbRoot) || - (fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition < rhs.fPartition) || - (fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && fSegment < rhs.fSegment)); - } - - bool operator == (const FileID& rhs) const - { - return ( - fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && fSegment == rhs.fSegment); - } + bool operator<(const FileID& rhs) const + { + return ((fFid < rhs.fFid) || (fFid == rhs.fFid && fDbRoot < rhs.fDbRoot) || + (fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition < rhs.fPartition) || + (fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && + fSegment < rhs.fSegment)); + } + bool operator==(const FileID& rhs) const + { + return (fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && + fSegment == rhs.fSegment); + } }; - // compressed DB file information class CompFileData { -public: - CompFileData(const FileID& id, const FID& fid, const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) : - fFileID(id), fFid(fid), fColDataType(colDataType), fColWidth(colWidth), fDctnryCol(false), - fFilePtr(NULL), fIoBSize(0), fCompressionType(1) {} + public: + CompFileData(const FileID& id, const FID& fid, + const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) + : fFileID(id) + , fFid(fid) + , fColDataType(colDataType) + , fColWidth(colWidth) + , fDctnryCol(false) + , fFilePtr(NULL) + , fIoBSize(0) + , fCompressionType(1) + { + } - ChunkData* findChunk(int64_t cid) const; + ChunkData* findChunk(int64_t cid) const; -protected: - FileID fFileID; - FID fFid; - execplan::CalpontSystemCatalog::ColDataType fColDataType; - int fColWidth; - bool fDctnryCol; - IDBDataFile* fFilePtr; - std::string fFileName; - CompFileHeader fFileHeader; - std::list fChunkList; - boost::scoped_array fIoBuffer; - size_t fIoBSize; - uint32_t fCompressionType; + protected: + FileID fFileID; + FID fFid; + execplan::CalpontSystemCatalog::ColDataType fColDataType; + int fColWidth; + bool fDctnryCol; + IDBDataFile* fFilePtr; + std::string fFileName; + CompFileHeader fFileHeader; + std::list fChunkList; + boost::scoped_array fIoBuffer; + size_t fIoBSize; + uint32_t fCompressionType; - friend class ChunkManager; + friend class ChunkManager; }; - - class ChunkManager { -public: - // @brief constructor - EXPORT ChunkManager(); + public: + // @brief constructor + EXPORT ChunkManager(); - // @brief destructor - EXPORT virtual ~ChunkManager(); + // @brief destructor + EXPORT virtual ~ChunkManager(); - // @brief Retrieve a file pointer in the chunk manager. - // for column file - IDBDataFile* getFilePtr(const Column& column, - uint16_t root, - uint32_t partition, - uint16_t segment, - std::string& filename, - const char* mode, - int size, - bool useTmpSuffix) const; + // @brief Retrieve a file pointer in the chunk manager. + // for column file + IDBDataFile* getFilePtr(const Column& column, uint16_t root, uint32_t partition, uint16_t segment, + std::string& filename, const char* mode, int size, bool useTmpSuffix) const; - // @brief Retrieve a file pointer in the chunk manager. - // for dictionary file - IDBDataFile* getFilePtr(const FID& fid, - uint16_t root, - uint32_t partition, - uint16_t segment, - std::string& filename, - const char* mode, - int size, - bool useTmpSuffix) const; + // @brief Retrieve a file pointer in the chunk manager. + // for dictionary file + IDBDataFile* getFilePtr(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + std::string& filename, const char* mode, int size, bool useTmpSuffix) const; - // @brief Retrieve a file pointer in the chunk manager. - // for column/dict segment file - IDBDataFile* getSegmentFilePtr( - FID& fid, uint16_t root, uint32_t partition, uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint32_t colWidth, std::string& filename, const char* mode, - int32_t size, bool useTmpSuffix, bool isDict) const; + // @brief Retrieve a file pointer in the chunk manager. + // for column/dict segment file + IDBDataFile* getSegmentFilePtr(FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, + std::string& filename, const char* mode, int32_t size, bool useTmpSuffix, + bool isDict) const; - // @brief Create a compressed dictionary file with an appropriate header. - IDBDataFile* createDctnryFile(const FID& fid, - int64_t width, - uint16_t root, - uint32_t partition, - uint16_t segment, - const char* filename, - const char* mode, - int size, - int64_t lbid); + // @brief Create a compressed dictionary file with an appropriate header. + IDBDataFile* createDctnryFile(const FID& fid, int64_t width, uint16_t root, uint32_t partition, + uint16_t segment, const char* filename, const char* mode, int size, + int64_t lbid); - // @brief Read a block from pFile at offset fbo. - // The data may copied from memory if the chunk it belongs to is already available. - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo); + // @brief Read a block from pFile at offset fbo. + // The data may copied from memory if the chunk it belongs to is already available. + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo); - // @brief Save a block to a chunk in pFile. - // The block is not written to disk immediately, will be delayed until flush. - int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo); + // @brief Save a block to a chunk in pFile. + // The block is not written to disk immediately, will be delayed until flush. + int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo); - // @brief Write all active chunks to disk, and reset all repository. - EXPORT int flushChunks(int rc, const std::map& columOids); + // @brief Write all active chunks to disk, and reset all repository. + EXPORT int flushChunks(int rc, const std::map& columOids); - // @brief Reset all repository without writing anything to disk. - void cleanUp(const std::map& columOids); + // @brief Reset all repository without writing anything to disk. + void cleanUp(const std::map& columOids); - // @brief Expand an initial column, not dictionary, extent to a full extent. - int expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width); + // @brief Expand an initial column, not dictionary, extent to a full extent. + int expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width); - // @brief Update column extent - int updateColumnExtent(IDBDataFile* pFile, int addBlockCount, - int64_t lbid); + // @brief Update column extent + int updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid); - // @brief Update dictionary extent - int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, - int64_t lbid); + // @brief Update dictionary extent + int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid); - // @brief Read in n continuous blocks to read buffer. - // for backing up blocks to version buffer - int readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n); + // @brief Read in n continuous blocks to read buffer. + // for backing up blocks to version buffer + int readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n); - // @brief Restore the data block at offset fbo from version buffer - // for rollback - int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo); + // @brief Restore the data block at offset fbo from version buffer + // for rollback + int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo); - // @brief Retrieve the total block count of a DB file. - int getBlockCount(IDBDataFile* pFile); + // @brief Retrieve the total block count of a DB file. + int getBlockCount(IDBDataFile* pFile); - // @brief Set FileOp pointer (for compression type, empty value, txnId, etc.) - void fileOp(FileOp* fileOp); + // @brief Set FileOp pointer (for compression type, empty value, txnId, etc.) + void fileOp(FileOp* fileOp); - // @brief Control the number of active chunks being stored in memory - void setMaxActiveChunkNum(unsigned int maxActiveChunkNum) - { - fMaxActiveChunkNum = maxActiveChunkNum; - } + // @brief Control the number of active chunks being stored in memory + void setMaxActiveChunkNum(unsigned int maxActiveChunkNum) + { + fMaxActiveChunkNum = maxActiveChunkNum; + } - // @brief Use this flag to avoid logging and backing up chunks, tmp files. - void setBulkFlag(bool isBulkLoad) - { - fIsBulkLoad = isBulkLoad; - } + // @brief Use this flag to avoid logging and backing up chunks, tmp files. + void setBulkFlag(bool isBulkLoad) + { + fIsBulkLoad = isBulkLoad; + } - // @brief Use this flag to flush chunk when is full. - void setIsInsert(bool isInsert) - { - fIsInsert = isInsert; - } - bool getIsInsert() - { - return fIsInsert; - } + // @brief Use this flag to flush chunk when is full. + void setIsInsert(bool isInsert) + { + fIsInsert = isInsert; + } + bool getIsInsert() + { + return fIsInsert; + } - void setTransId(const TxnID& transId) - { - fTransId = transId; - } + void setTransId(const TxnID& transId) + { + fTransId = transId; + } - // @brief bug5504, Use non transactional DML for InfiniDB with HDFS - EXPORT int startTransaction(const TxnID& transId) const; - EXPORT int confirmTransaction(const TxnID& transId) const; - EXPORT int endTransaction(const TxnID& transId, bool success) const; - // @brief Use this flag to fix bad chunk. - void setFixFlag(bool isFix) - { - fIsFix = isFix; - } + // @brief bug5504, Use non transactional DML for InfiniDB with HDFS + EXPORT int startTransaction(const TxnID& transId) const; + EXPORT int confirmTransaction(const TxnID& transId) const; + EXPORT int endTransaction(const TxnID& transId, bool success) const; + // @brief Use this flag to fix bad chunk. + void setFixFlag(bool isFix) + { + fIsFix = isFix; + } - EXPORT int checkFixLastDictChunk(const FID& fid, - uint16_t root, - uint32_t partition, - uint16_t segment); + EXPORT int checkFixLastDictChunk(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment); -protected: - // @brief Retrieve pointer to a compressed DB file. - CompFileData* getFileData(const FID& fid, - uint16_t root, - uint32_t partition, - uint16_t segment, - std::string& filename, - const char* mode, - int size, - const execplan::CalpontSystemCatalog::ColDataType colDataType, - int colWidth, - bool useTmpSuffix, - bool dictnry = false) const; + protected: + // @brief Retrieve pointer to a compressed DB file. + CompFileData* getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + std::string& filename, const char* mode, int size, + const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth, + bool useTmpSuffix, bool dictnry = false) const; - // @brief Retrieve a chunk of pFile from disk. - int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData); + // @brief Retrieve a chunk of pFile from disk. + int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData); - // @brief Compress a chunk and write it to file. - int writeChunkToFile(CompFileData* fileData, int64_t id); - int writeChunkToFile(CompFileData* fileData, ChunkData* chunkData); + // @brief Compress a chunk and write it to file. + int writeChunkToFile(CompFileData* fileData, int64_t id); + int writeChunkToFile(CompFileData* fileData, ChunkData* chunkData); - // @brief Write the compressed data to file and log a recover entry. - int writeCompressedChunk(CompFileData* fileData, int64_t offset, int64_t size); - inline int writeCompressedChunk_(CompFileData* fileData, int64_t offset); + // @brief Write the compressed data to file and log a recover entry. + int writeCompressedChunk(CompFileData* fileData, int64_t offset, int64_t size); + inline int writeCompressedChunk_(CompFileData* fileData, int64_t offset); - // @brief Write the file header to disk. - int writeHeader(CompFileData* fileData, int ln); - inline int writeHeader_(CompFileData* fileData, int ptrSecSize); + // @brief Write the file header to disk. + int writeHeader(CompFileData* fileData, int ln); + inline int writeHeader_(CompFileData* fileData, int ptrSecSize); - // @brief open a compressed DB file. - int openFile(CompFileData* fileData, const char* mode, int colWidth, - bool useTmpSuffix, int ln) const; + // @brief open a compressed DB file. + int openFile(CompFileData* fileData, const char* mode, int colWidth, bool useTmpSuffix, int ln) const; - // @brief set offset in a compressed DB file from beginning. - int setFileOffset(IDBDataFile* pFile, const std::string& fileName, off64_t offset, int ln) const; + // @brief set offset in a compressed DB file from beginning. + int setFileOffset(IDBDataFile* pFile, const std::string& fileName, off64_t offset, int ln) const; - // @brief read from a compressed DB file. - int readFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const; + // @brief read from a compressed DB file. + int readFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const; - // @brief write to a compressed DB file. - int writeFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const; + // @brief write to a compressed DB file. + int writeFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const; - // @brief Close a compressed DB file. - int closeFile(CompFileData* fileData); + // @brief Close a compressed DB file. + int closeFile(CompFileData* fileData); - // @brief Set empty values to a chunk. - void initializeColumnChunk(char* buf, CompFileData* fileData); - void initializeDctnryChunk(char* buf, int size); + // @brief Set empty values to a chunk. + void initializeColumnChunk(char* buf, CompFileData* fileData); + void initializeDctnryChunk(char* buf, int size); - // @brief Calculate the header size based on column width. - int calculateHeaderSize(int width); + // @brief Calculate the header size based on column width. + int calculateHeaderSize(int width); - // @brief Moving chunks as a result of expanding a chunk. - int reallocateChunks(CompFileData* fileData); + // @brief Moving chunks as a result of expanding a chunk. + int reallocateChunks(CompFileData* fileData); - // @brief verify chunks in the file are OK - int verifyChunksAfterRealloc(CompFileData* fileData); + // @brief verify chunks in the file are OK + int verifyChunksAfterRealloc(CompFileData* fileData); - // @brief log a message to the syslog - void logMessage(int code, int level, int lineNum, int fromLine = -1) const; - void logMessage(const std::string& msg, int level) const; + // @brief log a message to the syslog + void logMessage(int code, int level, int lineNum, int fromLine = -1) const; + void logMessage(const std::string& msg, int level) const; - // @brief Write a DML recovery log - int writeLog(TxnID txnId, std::string backUpFileType, std::string filename, - std::string& aDMLLogFileName, int64_t size = 0, int64_t offset = 0) const; + // @brief Write a DML recovery log + int writeLog(TxnID txnId, std::string backUpFileType, std::string filename, std::string& aDMLLogFileName, + int64_t size = 0, int64_t offset = 0) const; - // @brief remove DML recovery logs - int removeBackups(TxnID txnId); + // @brief remove DML recovery logs + int removeBackups(TxnID txnId); - // @brief swap the src file to dest file - int swapTmpFile(const std::string& src, const std::string& dest); + // @brief swap the src file to dest file + int swapTmpFile(const std::string& src, const std::string& dest); - // @brief construnct a DML log file name - int getDMLLogFileName(std::string& aDMLLogFileName, const TxnID& txnId) const; + // @brief construnct a DML log file name + int getDMLLogFileName(std::string& aDMLLogFileName, const TxnID& txnId) const; - mutable std::map fFileMap; - mutable std::map fFilePtrMap; - std::list > fActiveChunks; - unsigned int fMaxActiveChunkNum; // max active chunks per file - char* fBufCompressed; - size_t fLenCompressed; - size_t fMaxCompressedBufSize; - size_t fUserPaddings; - bool fIsBulkLoad; - bool fDropFdCache; - bool fIsInsert; - bool fIsHdfs; - FileOp* fFileOp; - compress::CompressorPool fCompressorPool; - logging::Logger* fSysLogger; - TxnID fTransId; - int fLocalModuleId; - idbdatafile::IDBFileSystem& fFs; - bool fIsFix; - size_t COMPRESSED_CHUNK_SIZE; + mutable std::map fFileMap; + mutable std::map fFilePtrMap; + std::list > fActiveChunks; + unsigned int fMaxActiveChunkNum; // max active chunks per file + char* fBufCompressed; + size_t fLenCompressed; + size_t fMaxCompressedBufSize; + size_t fUserPaddings; + bool fIsBulkLoad; + bool fDropFdCache; + bool fIsInsert; + bool fIsHdfs; + FileOp* fFileOp; + compress::CompressorPool fCompressorPool; + logging::Logger* fSysLogger; + TxnID fTransId; + int fLocalModuleId; + idbdatafile::IDBFileSystem& fFs; + bool fIsFix; + size_t COMPRESSED_CHUNK_SIZE; - private: + private: }; -} +} // namespace WriteEngine #undef EXPORT #endif // CHUNK_MANAGER_H - diff --git a/writeengine/shared/we_config.cpp b/writeengine/shared/we_config.cpp index b75f1cb1a..2e1bd3bf6 100644 --- a/writeengine/shared/we_config.cpp +++ b/writeengine/shared/we_config.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_config.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_config.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + * + *******************************************************************************/ /** @file */ #include @@ -41,42 +41,39 @@ using namespace idbdatafile; namespace WriteEngine { -const int DEFAULT_WAIT_PERIOD = 10; -const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 4; -const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; -const int DEFAULT_BULK_PROCESS_PRIORITY = -1; -const unsigned DEFAULT_MAX_FILESYSTEM_DISK_USAGE = 98; // allow 98% full -const unsigned DEFAULT_COMPRESSED_PADDING_BLKS = 1; -const int DEFAULT_LOCAL_MODULE_ID = 1; -const bool DEFAULT_PARENT_OAM = true; -const char* DEFAULT_LOCAL_MODULE_TYPE = "pm"; +const int DEFAULT_WAIT_PERIOD = 10; +const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 4; +const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; +const int DEFAULT_BULK_PROCESS_PRIORITY = -1; +const unsigned DEFAULT_MAX_FILESYSTEM_DISK_USAGE = 98; // allow 98% full +const unsigned DEFAULT_COMPRESSED_PADDING_BLKS = 1; +const int DEFAULT_LOCAL_MODULE_ID = 1; +const bool DEFAULT_PARENT_OAM = true; +const char* DEFAULT_LOCAL_MODULE_TYPE = "pm"; -int Config::m_dbRootCount = 0; +int Config::m_dbRootCount = 0; Config::strvec_t Config::m_dbRootPath; Config::intstrmap_t Config::m_dbRootPathMap; Config::uint16vec_t Config::m_dbRootId; -string Config::m_bulkRoot; +string Config::m_bulkRoot; -unsigned long Config::fDBRootChangeCount = 0; -time_t Config::fCacheTime = 0; -boost::mutex Config::fCacheLock; +unsigned long Config::fDBRootChangeCount = 0; +time_t Config::fCacheTime = 0; +boost::mutex Config::fCacheLock; #ifdef SHARED_NOTHING_DEMO_2 -boost::mutex Config::m_bulkRoot_lk; +boost::mutex Config::m_bulkRoot_lk; #endif -int Config::m_WaitPeriod = DEFAULT_WAIT_PERIOD; -unsigned Config::m_FilesPerColumnPartition = - DEFAULT_FILES_PER_COLUMN_PARTITION; -unsigned Config::m_ExtentsPerSegmentFile = - DEFAULT_EXTENTS_PER_SEGMENT_FILE; -int Config::m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY; -string Config::m_BulkRollbackDir; -unsigned Config::m_MaxFileSystemDiskUsage = - DEFAULT_MAX_FILESYSTEM_DISK_USAGE; -unsigned Config::m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS; -bool Config::m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM; -string Config::m_LocalModuleType; -int Config::m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID; -string Config::m_VersionBufferDir; +int Config::m_WaitPeriod = DEFAULT_WAIT_PERIOD; +unsigned Config::m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; +unsigned Config::m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; +int Config::m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY; +string Config::m_BulkRollbackDir; +unsigned Config::m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE; +unsigned Config::m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS; +bool Config::m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM; +string Config::m_LocalModuleType; +int Config::m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID; +string Config::m_VersionBufferDir; /******************************************************************************* * DESCRIPTION: @@ -88,8 +85,8 @@ string Config::m_VersionBufferDir; ******************************************************************************/ void Config::initConfigCache() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); } /******************************************************************************* @@ -98,221 +95,218 @@ void Config::initConfigCache() * PARAMETERS: * none ******************************************************************************/ -void Config::checkReload( ) +void Config::checkReload() { - bool bFirstLoad = false; + bool bFirstLoad = false; - if (fCacheTime == 0) - bFirstLoad = true; + if (fCacheTime == 0) + bFirstLoad = true; - config::Config* cf = config::Config::makeConfig(); + config::Config* cf = config::Config::makeConfig(); - // Immediately return if Columnstore.xml timestamp has not changed - if (cf->getCurrentMTime() == fCacheTime) - return; + // Immediately return if Columnstore.xml timestamp has not changed + if (cf->getCurrentMTime() == fCacheTime) + return; - //std::cout << "RELOADING cache..." << std::endl; + // std::cout << "RELOADING cache..." << std::endl; - //-------------------------------------------------------------------------- - // Initialize bulk root directory - //-------------------------------------------------------------------------- - m_bulkRoot = cf->getConfig("WriteEngine", "BulkRoot"); + //-------------------------------------------------------------------------- + // Initialize bulk root directory + //-------------------------------------------------------------------------- + m_bulkRoot = cf->getConfig("WriteEngine", "BulkRoot"); - if ( m_bulkRoot.length() == 0 ) - { - m_bulkRoot = "/var/log/mariadb/columnstore"; + if (m_bulkRoot.length() == 0) + { + m_bulkRoot = "/var/log/mariadb/columnstore"; #ifndef _MSC_VER - m_bulkRoot += "/data"; + m_bulkRoot += "/data"; #endif - m_bulkRoot += "/bulk"; - } + m_bulkRoot += "/bulk"; + } - // Get latest Columnstore.xml timestamp after first access forced a reload - fCacheTime = cf ->getLastMTime(); + // Get latest Columnstore.xml timestamp after first access forced a reload + fCacheTime = cf->getLastMTime(); - //-------------------------------------------------------------------------- - // Initialize time interval (in seconds) between retries - //-------------------------------------------------------------------------- - m_WaitPeriod = DEFAULT_WAIT_PERIOD; - string waitPeriodStr = cf->getConfig("SystemConfig", "WaitPeriod"); + //-------------------------------------------------------------------------- + // Initialize time interval (in seconds) between retries + //-------------------------------------------------------------------------- + m_WaitPeriod = DEFAULT_WAIT_PERIOD; + string waitPeriodStr = cf->getConfig("SystemConfig", "WaitPeriod"); - if ( waitPeriodStr.length() != 0 ) - m_WaitPeriod = static_cast(config::Config::fromText( - waitPeriodStr)); + if (waitPeriodStr.length() != 0) + m_WaitPeriod = static_cast(config::Config::fromText(waitPeriodStr)); - //-------------------------------------------------------------------------- - // Initialize files per column partition - //-------------------------------------------------------------------------- - m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; - string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); + //-------------------------------------------------------------------------- + // Initialize files per column partition + //-------------------------------------------------------------------------- + m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; + string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); - if ( fpc.length() != 0 ) - m_FilesPerColumnPartition = cf->uFromText(fpc); + if (fpc.length() != 0) + m_FilesPerColumnPartition = cf->uFromText(fpc); - //-------------------------------------------------------------------------- - // Initialize extents per segment file - //-------------------------------------------------------------------------- - m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; + //-------------------------------------------------------------------------- + // Initialize extents per segment file + //-------------------------------------------------------------------------- + m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; - //-------------------------------------------------------------------------- - // Initialize bulk load process priority - //-------------------------------------------------------------------------- - m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY; - string prior = cf->getConfig("WriteEngine", "Priority"); + //-------------------------------------------------------------------------- + // Initialize bulk load process priority + //-------------------------------------------------------------------------- + m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY; + string prior = cf->getConfig("WriteEngine", "Priority"); - if ( prior.length() != 0 ) - { - int initialBPP = cf->fromText(prior); + if (prior.length() != 0) + { + int initialBPP = cf->fromText(prior); - // config file priority is 40..1 (highest..lowest) - // convert config file value to setpriority(2) value(-20..19, -1 is the - // default) - if (initialBPP > 0) - m_BulkProcessPriority = 20 - initialBPP; - else if (initialBPP < 0) - m_BulkProcessPriority = 19; + // config file priority is 40..1 (highest..lowest) + // convert config file value to setpriority(2) value(-20..19, -1 is the + // default) + if (initialBPP > 0) + m_BulkProcessPriority = 20 - initialBPP; + else if (initialBPP < 0) + m_BulkProcessPriority = 19; - if (m_BulkProcessPriority < -20) - m_BulkProcessPriority = -20; - } + if (m_BulkProcessPriority < -20) + m_BulkProcessPriority = -20; + } - //-------------------------------------------------------------------------- - // Initialize bulk rollback directory - // Note this uses m_bulkRoot, so this init section must be after the section - // that sets m_bulkRoot. - //-------------------------------------------------------------------------- - m_BulkRollbackDir = cf->getConfig("WriteEngine", "BulkRollbackDir"); + //-------------------------------------------------------------------------- + // Initialize bulk rollback directory + // Note this uses m_bulkRoot, so this init section must be after the section + // that sets m_bulkRoot. + //-------------------------------------------------------------------------- + m_BulkRollbackDir = cf->getConfig("WriteEngine", "BulkRollbackDir"); - if (m_BulkRollbackDir.length() == 0) - { - m_BulkRollbackDir.assign( m_bulkRoot ); - m_BulkRollbackDir += "/rollback"; - } + if (m_BulkRollbackDir.length() == 0) + { + m_BulkRollbackDir.assign(m_bulkRoot); + m_BulkRollbackDir += "/rollback"; + } - //-------------------------------------------------------------------------- - // Initialize max disk usage - //-------------------------------------------------------------------------- + //-------------------------------------------------------------------------- + // Initialize max disk usage + //-------------------------------------------------------------------------- + m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE; + string usg = cf->getConfig("WriteEngine", "MaxFileSystemDiskUsagePct"); + + if (usg.length() != 0) + m_MaxFileSystemDiskUsage = cf->uFromText(usg); + + if (m_MaxFileSystemDiskUsage > 100) m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE; - string usg = cf->getConfig("WriteEngine", "MaxFileSystemDiskUsagePct"); - if ( usg.length() != 0 ) - m_MaxFileSystemDiskUsage = cf->uFromText(usg); + //-------------------------------------------------------------------------- + // Number of compressed padding blocks + //-------------------------------------------------------------------------- + m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS; + string ncpb = cf->getConfig("WriteEngine", "CompressedPaddingBlocks"); - if (m_MaxFileSystemDiskUsage > 100) - m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE; + if (ncpb.length() != 0) + m_NumCompressedPadBlks = cf->uFromText(ncpb); - //-------------------------------------------------------------------------- - // Number of compressed padding blocks - //-------------------------------------------------------------------------- - m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS; - string ncpb = cf->getConfig("WriteEngine", "CompressedPaddingBlocks"); + IDBPolicy::configIDBPolicy(); - if ( ncpb.length() != 0 ) - m_NumCompressedPadBlks = cf->uFromText(ncpb); + //-------------------------------------------------------------------------- + // Initialize Parent OAM Module flag + // Initialize Module Type + // Initialize Local Module ID + //-------------------------------------------------------------------------- + oam::Oam oam; + oam::oamModuleInfo_t t; - IDBPolicy::configIDBPolicy(); + try + { + t = oam.getModuleInfo(); + m_ParentOAMModuleFlag = boost::get<4>(t); + m_LocalModuleType = boost::get<1>(t); + m_LocalModuleID = boost::get<2>(t); + } + catch (exception&) + { + m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM; + m_LocalModuleType.assign(DEFAULT_LOCAL_MODULE_TYPE); + m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID; + } - //-------------------------------------------------------------------------- - // Initialize Parent OAM Module flag - // Initialize Module Type - // Initialize Local Module ID - //-------------------------------------------------------------------------- - oam::Oam oam; - oam::oamModuleInfo_t t; + //-------------------------------------------------------------------------- + // Initialize Version Buffer + //-------------------------------------------------------------------------- + m_VersionBufferDir = cf->getConfig("SystemConfig", "DBRMRoot"); + + if (m_VersionBufferDir.length() == 0) + { + m_VersionBufferDir = "/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves"; + } + + //-------------------------------------------------------------------------- + // Initialize m_dbRootCount, m_dbRootPath, m_dbRootPathMap, m_dbRootId. + // Note this uses m_localModuleType and m_LocalModuleID, so this init + // section must be after the section(s) that set m_localModuleType and + // m_LocalModuleID. + //-------------------------------------------------------------------------- + uint16vec_t dbRootIdPrevious(m_dbRootId); // save current settings + strvec_t dbRootPathPrevious(m_dbRootPath); // save current setttings + + m_dbRootPath.clear(); + m_dbRootPathMap.clear(); + m_dbRootId.clear(); + + if (m_LocalModuleType == "pm") + { + oam::DBRootConfigList oamRootList; try { - t = oam.getModuleInfo(); - m_ParentOAMModuleFlag = boost::get<4>(t); - m_LocalModuleType = boost::get<1>(t); - m_LocalModuleID = boost::get<2>(t); + oam.getPmDbrootConfig(m_LocalModuleID, oamRootList); + + std::sort(oamRootList.begin(), oamRootList.end()); + + m_dbRootCount = oamRootList.size(); + + for (unsigned int idx = 0; idx < oamRootList.size(); idx++) + { + ostringstream oss; + oss << "DBRoot" << oamRootList[idx]; + std::string DbRootPath = cf->getConfig("SystemConfig", oss.str()); + m_dbRootPath.push_back(DbRootPath); + m_dbRootPathMap[oamRootList[idx]] = DbRootPath; + m_dbRootId.push_back(oamRootList[idx]); + } } catch (exception&) { - m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM; - m_LocalModuleType.assign( DEFAULT_LOCAL_MODULE_TYPE ); - m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID; + m_dbRootCount = 0; } + } + else + { + m_dbRootCount = 0; + } - //-------------------------------------------------------------------------- - // Initialize Version Buffer - //-------------------------------------------------------------------------- - m_VersionBufferDir = cf->getConfig("SystemConfig", "DBRMRoot"); - - if ( m_VersionBufferDir.length() == 0 ) + // Update counter used to track changes to local PM DBRoot list + if (!bFirstLoad) + { + if ((dbRootIdPrevious != m_dbRootId) || (dbRootPathPrevious != m_dbRootPath)) { - m_VersionBufferDir = "/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves"; + fDBRootChangeCount++; } + } - //-------------------------------------------------------------------------- - // Initialize m_dbRootCount, m_dbRootPath, m_dbRootPathMap, m_dbRootId. - // Note this uses m_localModuleType and m_LocalModuleID, so this init - // section must be after the section(s) that set m_localModuleType and - // m_LocalModuleID. - //-------------------------------------------------------------------------- - uint16vec_t dbRootIdPrevious( m_dbRootId ); // save current settings - strvec_t dbRootPathPrevious( m_dbRootPath ); // save current setttings - - m_dbRootPath.clear(); - m_dbRootPathMap.clear(); - m_dbRootId.clear(); - - if (m_LocalModuleType == "pm") - { - oam::DBRootConfigList oamRootList; - - try - { - oam.getPmDbrootConfig( m_LocalModuleID, oamRootList ); - - std::sort( oamRootList.begin(), oamRootList.end() ); - - m_dbRootCount = oamRootList.size(); - - for (unsigned int idx = 0; idx < oamRootList.size(); idx++) - { - ostringstream oss; - oss << "DBRoot" << oamRootList[idx]; - std::string DbRootPath = - cf->getConfig("SystemConfig", oss.str()); - m_dbRootPath.push_back( DbRootPath ); - m_dbRootPathMap[ oamRootList[idx] ] = DbRootPath; - m_dbRootId.push_back( oamRootList[idx] ); - } - } - catch (exception&) - { - m_dbRootCount = 0; - } - } - else - { - m_dbRootCount = 0; - } - - // Update counter used to track changes to local PM DBRoot list - if (!bFirstLoad) - { - if ((dbRootIdPrevious != m_dbRootId) || - (dbRootPathPrevious != m_dbRootPath)) - { - fDBRootChangeCount++; - } - } - -// for (unsigned int n=0; nfirst << "," << k->second << std::endl; -// } + // for (unsigned int n=0; nfirst << "," << k->second << std::endl; + // } } /******************************************************************************* @@ -325,10 +319,10 @@ void Config::checkReload( ) ******************************************************************************/ size_t Config::DBRootCount() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_dbRootCount; + return m_dbRootCount; } /******************************************************************************* @@ -341,16 +335,16 @@ size_t Config::DBRootCount() ******************************************************************************/ std::string Config::getDBRootByIdx(unsigned idx) { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - if (idx >= m_dbRootPath.size()) - { - std::string emptyResult; - return emptyResult; - } + if (idx >= m_dbRootPath.size()) + { + std::string emptyResult; + return emptyResult; + } - return m_dbRootPath[idx]; + return m_dbRootPath[idx]; } /******************************************************************************* @@ -361,13 +355,13 @@ std::string Config::getDBRootByIdx(unsigned idx) * RETURN: * none ******************************************************************************/ -void Config::getDBRootPathList( std::vector& dbRootPathList ) +void Config::getDBRootPathList(std::vector& dbRootPathList) { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - dbRootPathList.clear(); - dbRootPathList = m_dbRootPath; + dbRootPathList.clear(); + dbRootPathList = m_dbRootPath; } /******************************************************************************* @@ -380,18 +374,18 @@ void Config::getDBRootPathList( std::vector& dbRootPathList ) ******************************************************************************/ std::string Config::getDBRootByNum(unsigned num) { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - Config::intstrmap_t::const_iterator iter = m_dbRootPathMap.find( num ); + Config::intstrmap_t::const_iterator iter = m_dbRootPathMap.find(num); - if (iter == m_dbRootPathMap.end()) - { - std::string emptyResult; - return emptyResult; - } + if (iter == m_dbRootPathMap.end()) + { + std::string emptyResult; + return emptyResult; + } - return iter->second; + return iter->second; } /******************************************************************************* @@ -402,12 +396,12 @@ std::string Config::getDBRootByNum(unsigned num) * RETURN: * The list of DBRoot ids ******************************************************************************/ -void Config::getRootIdList( std::vector& rootIds ) +void Config::getRootIdList(std::vector& rootIds) { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - rootIds = m_dbRootId; + rootIds = m_dbRootId; } /******************************************************************************* @@ -420,21 +414,20 @@ void Config::getRootIdList( std::vector& rootIds ) ******************************************************************************/ std::string Config::getBulkRoot() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_bulkRoot; + return m_bulkRoot; } #ifdef SHARED_NOTHING_DEMO_2 void Config::getSharedNothingRoot(char* ret) { - string root; - boost::mutex::scoped_lock lk(m_bulkRoot_lk); + string root; + boost::mutex::scoped_lock lk(m_bulkRoot_lk); - root = config::Config::makeConfig()->getConfig( - "WriteEngine", "SharedNothingRoot"); - strncpy(ret, root.c_str(), FILE_NAME_SIZE); + root = config::Config::makeConfig()->getConfig("WriteEngine", "SharedNothingRoot"); + strncpy(ret, root.c_str(), FILE_NAME_SIZE); } #endif @@ -448,10 +441,10 @@ void Config::getSharedNothingRoot(char* ret) ******************************************************************************/ int Config::getWaitPeriod() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_WaitPeriod; + return m_WaitPeriod; } /******************************************************************************* @@ -464,10 +457,10 @@ int Config::getWaitPeriod() ******************************************************************************/ unsigned Config::getFilesPerColumnPartition() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_FilesPerColumnPartition; + return m_FilesPerColumnPartition; } /******************************************************************************* @@ -480,10 +473,10 @@ unsigned Config::getFilesPerColumnPartition() ******************************************************************************/ unsigned Config::getExtentsPerSegmentFile() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_ExtentsPerSegmentFile; + return m_ExtentsPerSegmentFile; } /******************************************************************************* @@ -503,10 +496,10 @@ unsigned Config::getExtentsPerSegmentFile() ******************************************************************************/ int Config::getBulkProcessPriority() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_BulkProcessPriority; + return m_BulkProcessPriority; } /******************************************************************************* @@ -517,10 +510,10 @@ int Config::getBulkProcessPriority() ******************************************************************************/ std::string Config::getBulkRollbackDir() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_BulkRollbackDir; + return m_BulkRollbackDir; } /******************************************************************************* @@ -531,10 +524,10 @@ std::string Config::getBulkRollbackDir() ******************************************************************************/ unsigned Config::getMaxFileSystemDiskUsage() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_MaxFileSystemDiskUsage; + return m_MaxFileSystemDiskUsage; } /******************************************************************************* @@ -546,10 +539,10 @@ unsigned Config::getMaxFileSystemDiskUsage() ******************************************************************************/ unsigned Config::getNumCompressedPadBlks() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_NumCompressedPadBlks; + return m_NumCompressedPadBlks; } /******************************************************************************* @@ -560,10 +553,10 @@ unsigned Config::getNumCompressedPadBlks() ******************************************************************************/ bool Config::getParentOAMModuleFlag() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_ParentOAMModuleFlag; + return m_ParentOAMModuleFlag; } /******************************************************************************* @@ -574,10 +567,10 @@ bool Config::getParentOAMModuleFlag() ******************************************************************************/ std::string Config::getLocalModuleType() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_LocalModuleType; + return m_LocalModuleType; } /******************************************************************************* @@ -588,10 +581,10 @@ std::string Config::getLocalModuleType() ******************************************************************************/ uint16_t Config::getLocalModuleID() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_LocalModuleID; + return m_LocalModuleID; } /******************************************************************************* @@ -604,10 +597,10 @@ uint16_t Config::getLocalModuleID() ******************************************************************************/ std::string Config::getVBRoot() { - boost::mutex::scoped_lock lk(fCacheLock); - checkReload( ); + boost::mutex::scoped_lock lk(fCacheLock); + checkReload(); - return m_VersionBufferDir; + return m_VersionBufferDir; } /******************************************************************************* @@ -621,16 +614,15 @@ std::string Config::getVBRoot() ******************************************************************************/ bool Config::hasLocalDBRootListChanged() { - boost::mutex::scoped_lock lk(fCacheLock); + boost::mutex::scoped_lock lk(fCacheLock); - if (fDBRootChangeCount > 0) - { - fDBRootChangeCount = 0; - return true; - } + if (fDBRootChangeCount > 0) + { + fDBRootChangeCount = 0; + return true; + } - return false; + return false; } - -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_config.h b/writeengine/shared/we_config.h index d2ad2ae50..11ad122bb 100644 --- a/writeengine/shared/we_config.h +++ b/writeengine/shared/we_config.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_config.h 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_config.h 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ /** @file */ #ifndef WE_CONFIG_H_ @@ -42,160 +42,162 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class Config */ class Config { -public: - /** - * @brief Constructor - */ - Config() {} + public: + /** + * @brief Constructor + */ + Config() + { + } - /** - * @brief Default Destructor - */ - ~Config() {} + /** + * @brief Default Destructor + */ + ~Config() + { + } - /** - * @brief Get DB root (for local PM) - * @param idx Index of the DBRootn entry to fetch (0 fetches DBRoot[0],etc.) - */ - EXPORT static std::string getDBRootByIdx(unsigned idx); + /** + * @brief Get DB root (for local PM) + * @param idx Index of the DBRootn entry to fetch (0 fetches DBRoot[0],etc.) + */ + EXPORT static std::string getDBRootByIdx(unsigned idx); - /** - * @brief Get complete DBRoot path list for the local PM - * @param dbRootPathList vector of DBRoot paths - */ - EXPORT static void getDBRootPathList( - std::vector& dbRootPathList ); + /** + * @brief Get complete DBRoot path list for the local PM + * @param dbRootPathList vector of DBRoot paths + */ + EXPORT static void getDBRootPathList(std::vector& dbRootPathList); - /** - * @brief Get DB root (for local PM) - * @param num DBRootN entry to fetch (1 fetches DBRoot1, etc.) - */ - EXPORT static std::string getDBRootByNum(unsigned num); + /** + * @brief Get DB root (for local PM) + * @param num DBRootN entry to fetch (1 fetches DBRoot1, etc.) + */ + EXPORT static std::string getDBRootByNum(unsigned num); - /** - * @brief Get list of applicable DBRoot ids for this job. - */ - EXPORT static void getRootIdList( std::vector& dbRootIds ); + /** + * @brief Get list of applicable DBRoot ids for this job. + */ + EXPORT static void getRootIdList(std::vector& dbRootIds); #ifdef SHARED_NOTHING_DEMO_2 - EXPORT static void getSharedNothingRoot(char*); // pass in an char[FILE_NAME_SIZE] + EXPORT static void getSharedNothingRoot(char*); // pass in an char[FILE_NAME_SIZE] #endif - /** - * @brief Bulkload DB root - */ - EXPORT static std::string getBulkRoot(); + /** + * @brief Bulkload DB root + */ + EXPORT static std::string getBulkRoot(); - /** - * @brief DBRoot count for local PM - */ - EXPORT static size_t DBRootCount(); + /** + * @brief DBRoot count for local PM + */ + EXPORT static size_t DBRootCount(); - /** - * @brief Wait Period - */ - EXPORT static int getWaitPeriod(); + /** + * @brief Wait Period + */ + EXPORT static int getWaitPeriod(); - /** - * @brief FilesPerColumnPartition - */ - EXPORT static unsigned getFilesPerColumnPartition(); + /** + * @brief FilesPerColumnPartition + */ + EXPORT static unsigned getFilesPerColumnPartition(); - /** - * @brief ExtentsPerSegmentFile - */ - EXPORT static unsigned getExtentsPerSegmentFile(); + /** + * @brief ExtentsPerSegmentFile + */ + EXPORT static unsigned getExtentsPerSegmentFile(); - /** - * @brief Process Priority for cpimport.bin - * Return value is in range -20..19 (highest...lowest, 0=normal) - */ - EXPORT static int getBulkProcessPriority(); + /** + * @brief Process Priority for cpimport.bin + * Return value is in range -20..19 (highest...lowest, 0=normal) + */ + EXPORT static int getBulkProcessPriority(); - /** - * @brief Directory carrying Bulk Rollback meta data files - */ - EXPORT static std::string getBulkRollbackDir(); + /** + * @brief Directory carrying Bulk Rollback meta data files + */ + EXPORT static std::string getBulkRollbackDir(); - /** - * @brief Max percentage of allowable file system disk usage for each DBRoot - */ - EXPORT static unsigned getMaxFileSystemDiskUsage(); + /** + * @brief Max percentage of allowable file system disk usage for each DBRoot + */ + EXPORT static unsigned getMaxFileSystemDiskUsage(); - /** - * @brief Number of Blocks to pad each compressed chunk. - */ - EXPORT static unsigned getNumCompressedPadBlks(); + /** + * @brief Number of Blocks to pad each compressed chunk. + */ + EXPORT static unsigned getNumCompressedPadBlks(); - /** - * @brief Parent OAM Module flag (is this the parent OAM node, ex: pm1) - */ - EXPORT static bool getParentOAMModuleFlag(); + /** + * @brief Parent OAM Module flag (is this the parent OAM node, ex: pm1) + */ + EXPORT static bool getParentOAMModuleFlag(); - /** - * @brief Local Module Type (ex: "pm") - */ - EXPORT static std::string getLocalModuleType(); + /** + * @brief Local Module Type (ex: "pm") + */ + EXPORT static std::string getLocalModuleType(); - /** - * @brief Local Module ID (ex: 1 ) - */ - EXPORT static uint16_t getLocalModuleID(); + /** + * @brief Local Module ID (ex: 1 ) + */ + EXPORT static uint16_t getLocalModuleID(); - /** - * @brief Version Buffer root - */ - EXPORT static std::string getVBRoot(); + /** + * @brief Version Buffer root + */ + EXPORT static std::string getVBRoot(); - /** - * @brief Cache the config parameters locally - * Initialize Config cache. Cache will be updated as needed. - */ - EXPORT static void initConfigCache(); + /** + * @brief Cache the config parameters locally + * Initialize Config cache. Cache will be updated as needed. + */ + EXPORT static void initConfigCache(); - /** - * @brief Has Local PM DBRoot info changed since last time this function - * was called. Can be used to monitor changes to DBRoot info. - */ - EXPORT static bool hasLocalDBRootListChanged(); + /** + * @brief Has Local PM DBRoot info changed since last time this function + * was called. Can be used to monitor changes to DBRoot info. + */ + EXPORT static bool hasLocalDBRootListChanged(); -private: - typedef std::vector strvec_t; - typedef std::map intstrmap_t; - typedef std::vector uint16vec_t; + private: + typedef std::vector strvec_t; + typedef std::map intstrmap_t; + typedef std::vector uint16vec_t; - static void checkReload(); + static void checkReload(); - static int m_dbRootCount; // num DBRoots for local PM - static strvec_t m_dbRootPath; // root paths for open files - static intstrmap_t m_dbRootPathMap; // map of root id to root paths - static uint16vec_t m_dbRootId; // list of root ids - static std::string m_bulkRoot; // root path for bulk operation - static unsigned long fDBRootChangeCount; // track recent DBRoot changes - static time_t fCacheTime; // timestamp associated w/cache - static boost::mutex fCacheLock; // mutex for m_dbRoot sync + static int m_dbRootCount; // num DBRoots for local PM + static strvec_t m_dbRootPath; // root paths for open files + static intstrmap_t m_dbRootPathMap; // map of root id to root paths + static uint16vec_t m_dbRootId; // list of root ids + static std::string m_bulkRoot; // root path for bulk operation + static unsigned long fDBRootChangeCount; // track recent DBRoot changes + static time_t fCacheTime; // timestamp associated w/cache + static boost::mutex fCacheLock; // mutex for m_dbRoot sync #ifdef SHARED_NOTHING_DEMO_2 - static boost::mutex m_bulkRoot_lk; // mutex for m_bulkRoot sync + static boost::mutex m_bulkRoot_lk; // mutex for m_bulkRoot sync #endif - static int m_WaitPeriod; // secs to wait for transaction - static unsigned m_FilesPerColumnPartition;//# seg files per partition - static unsigned m_ExtentsPerSegmentFile; // # extents per segment file - static int m_BulkProcessPriority; // cpimport.bin proc priority - static std::string m_BulkRollbackDir; // bulk rollback meta data dir - static unsigned m_MaxFileSystemDiskUsage;// max file system % disk usage - static unsigned m_NumCompressedPadBlks; // num blks to pad comp chunks - static bool m_ParentOAMModuleFlag; // are we running on parent PM - static std::string m_LocalModuleType; // local node type (ex: "pm") - static int m_LocalModuleID; // local node id (ex: 1 ) - static std::string m_VersionBufferDir; // Version buffer directory + static int m_WaitPeriod; // secs to wait for transaction + static unsigned m_FilesPerColumnPartition; //# seg files per partition + static unsigned m_ExtentsPerSegmentFile; // # extents per segment file + static int m_BulkProcessPriority; // cpimport.bin proc priority + static std::string m_BulkRollbackDir; // bulk rollback meta data dir + static unsigned m_MaxFileSystemDiskUsage; // max file system % disk usage + static unsigned m_NumCompressedPadBlks; // num blks to pad comp chunks + static bool m_ParentOAMModuleFlag; // are we running on parent PM + static std::string m_LocalModuleType; // local node type (ex: "pm") + static int m_LocalModuleID; // local node id (ex: 1 ) + static std::string m_VersionBufferDir; // Version buffer directory }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // WE_CONFIG_H_ +#endif // WE_CONFIG_H_ diff --git a/writeengine/shared/we_confirmhdfsdbfile.cpp b/writeengine/shared/we_confirmhdfsdbfile.cpp index ec1483e12..49dcac841 100644 --- a/writeengine/shared/we_confirmhdfsdbfile.cpp +++ b/writeengine/shared/we_confirmhdfsdbfile.cpp @@ -40,7 +40,6 @@ const int BUF_SIZE = 1024; // size of buffer used to read meta data records namespace WriteEngine { - //------------------------------------------------------------------------------ // Constructor // This class should typically only be used on an HDFS system, so we could @@ -48,808 +47,714 @@ namespace WriteEngine // to be able to execute this class on a non-HDFS stack as well. So I rely // on useHdfs() to tell me which FileSystem reference to get. //------------------------------------------------------------------------------ -ConfirmHdfsDbFile::ConfirmHdfsDbFile() : - fFs(idbdatafile::IDBPolicy::useHdfs() ? - idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::HDFS) : - idbdatafile::IDBPolicy::useCloud() ? - idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::CLOUD) : - idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::BUFFERED)) +ConfirmHdfsDbFile::ConfirmHdfsDbFile() + : fFs(idbdatafile::IDBPolicy::useHdfs() ? idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::HDFS) + : idbdatafile::IDBPolicy::useCloud() + ? idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::CLOUD) + : idbdatafile::IDBFileSystem::getFs(idbdatafile::IDBDataFile::BUFFERED)) { } - + //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ ConfirmHdfsDbFile::~ConfirmHdfsDbFile() { } - + //------------------------------------------------------------------------------ // Backup a cdf file and replace it with the updated tmp file. //------------------------------------------------------------------------------ -int ConfirmHdfsDbFile::confirmDbFileChange( - const std::string& backUpFileType, - const std::string& filename, - std::string& errMsg) const +int ConfirmHdfsDbFile::confirmDbFileChange(const std::string& backUpFileType, const std::string& filename, + std::string& errMsg) const { - // return value - int rc = NO_ERROR; + // return value + int rc = NO_ERROR; - // This rlc file should be renamed if success, just skip it - if (backUpFileType.compare("rlc") == 0) - { - return rc; - } + // This rlc file should be renamed if success, just skip it + if (backUpFileType.compare("rlc") == 0) + { + return rc; + } - if (backUpFileType.compare("tmp") != 0 ) - { - std::ostringstream oss; - oss << backUpFileType << " is a bad type to confirm DbFile change: " << - filename; - errMsg = oss.str(); - rc = ERR_HDFS_BACKUP; - - return rc; - } - - // add safety checks, just in case - std::string tmp(filename + ".tmp"); - - if (!fFs.exists(tmp.c_str())) // file already swapped - return rc; - - if (fFs.size(tmp.c_str()) <= 0) - { - std::ostringstream oss; - oss << "tmp file " << tmp << " has bad size" << fFs.size(tmp.c_str()); - errMsg = oss.str(); - rc = ERR_COMP_RENAME_FILE; - - return rc; - } - - // remove the old orig if exists - std::string orig(filename + ".orig"); - errno = 0; - - if ((fFs.exists(orig.c_str())) && - (fFs.remove(orig.c_str())) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "remove old " << orig << " failed: " << strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_REMOVE_FILE; - - return rc; - } - - // backup the original - errno = 0; - - if (fFs.rename(filename.c_str(), orig.c_str()) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "rename " << filename << " to " << orig << " failed: " << - strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_RENAME_FILE; - - return rc; - } - - // rename the new file - errno = 0; - - if (fFs.rename(tmp.c_str(), filename.c_str()) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "rename " << tmp << " to " << filename << " failed: " << - strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_RENAME_FILE; - - return rc; - } + if (backUpFileType.compare("tmp") != 0) + { + std::ostringstream oss; + oss << backUpFileType << " is a bad type to confirm DbFile change: " << filename; + errMsg = oss.str(); + rc = ERR_HDFS_BACKUP; return rc; + } + + // add safety checks, just in case + std::string tmp(filename + ".tmp"); + + if (!fFs.exists(tmp.c_str())) // file already swapped + return rc; + + if (fFs.size(tmp.c_str()) <= 0) + { + std::ostringstream oss; + oss << "tmp file " << tmp << " has bad size" << fFs.size(tmp.c_str()); + errMsg = oss.str(); + rc = ERR_COMP_RENAME_FILE; + + return rc; + } + + // remove the old orig if exists + std::string orig(filename + ".orig"); + errno = 0; + + if ((fFs.exists(orig.c_str())) && (fFs.remove(orig.c_str())) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "remove old " << orig << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_REMOVE_FILE; + + return rc; + } + + // backup the original + errno = 0; + + if (fFs.rename(filename.c_str(), orig.c_str()) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "rename " << filename << " to " << orig << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_RENAME_FILE; + + return rc; + } + + // rename the new file + errno = 0; + + if (fFs.rename(tmp.c_str(), filename.c_str()) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "rename " << tmp << " to " << filename << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_RENAME_FILE; + + return rc; + } + + return rc; } - + //------------------------------------------------------------------------------ // Finalize the changes to a db file. // If success flag is true, then remove the orig // otherwise, move the orig back to cdf //------------------------------------------------------------------------------ -int ConfirmHdfsDbFile::endDbFileChange( - const std::string& backUpFileType, - const std::string& filename, - bool success, - std::string& errMsg) const +int ConfirmHdfsDbFile::endDbFileChange(const std::string& backUpFileType, const std::string& filename, + bool success, std::string& errMsg) const { - // return value - int rc = NO_ERROR; + // return value + int rc = NO_ERROR; - // This rlc file should be renamed if success, it is useless if failed. - if (backUpFileType.compare("rlc") == 0) - { - std::string rlc(filename + ".rlc"); + // This rlc file should be renamed if success, it is useless if failed. + if (backUpFileType.compare("rlc") == 0) + { + std::string rlc(filename + ".rlc"); - if (fFs.exists(rlc.c_str())) - fFs.remove(rlc.c_str()); // TBD-okay to ignore failed removal? - - return rc; - } - - if (backUpFileType.compare("tmp") != 0) - { - std::ostringstream oss; - oss << backUpFileType << " is a bad type to finalize DbFile change: " << - filename; - errMsg = oss.str(); - rc = ERR_HDFS_BACKUP; - - return rc; - } - - std::string orig(filename + ".orig"); - - if (success) - { - // remove the orig file - errno = 0; - - if ((fFs.exists(orig.c_str())) && - (fFs.remove(orig.c_str())) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "remove " << orig << " failed: " << strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_REMOVE_FILE; - - return rc; - } - } - else - { - // restore the orig file - if (fFs.exists(orig.c_str())) - { - errno = 0; - - // Try to remove file only if it exists - if ((fFs.exists(filename.c_str())) && - (fFs.remove(filename.c_str()) != 0)) - { - int errNum = errno; - std::ostringstream oss; - oss << "failed restore; remove " << filename << " failed: " << - strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_REMOVE_FILE; - - return rc; - } - - errno = 0; - - if (fFs.rename(orig.c_str(), filename.c_str()) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "failed restore; rename " << orig << " failed: " << - strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_RENAME_FILE; - - return rc; - } - } - - // remove the tmp file - std::string tmp(filename + ".tmp"); - errno = 0; - - if ((fFs.exists(tmp.c_str())) && - (fFs.remove(tmp.c_str())) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "failed restore; remove " << tmp << " failed: " << - strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_REMOVE_FILE; - - return rc; - } - - // remove the chunk shifting helper - std::string rlc(filename + ".rlc"); - errno = 0; - - if ((fFs.exists(rlc.c_str())) && - (fFs.remove(rlc.c_str())) != 0) - { - int errNum = errno; - std::ostringstream oss; - oss << "failed restore; remove " << rlc << " failed: " << - strerror(errNum); - errMsg = oss.str(); - rc = ERR_COMP_REMOVE_FILE; - - return rc; - } - } + if (fFs.exists(rlc.c_str())) + fFs.remove(rlc.c_str()); // TBD-okay to ignore failed removal? return rc; + } + + if (backUpFileType.compare("tmp") != 0) + { + std::ostringstream oss; + oss << backUpFileType << " is a bad type to finalize DbFile change: " << filename; + errMsg = oss.str(); + rc = ERR_HDFS_BACKUP; + + return rc; + } + + std::string orig(filename + ".orig"); + + if (success) + { + // remove the orig file + errno = 0; + + if ((fFs.exists(orig.c_str())) && (fFs.remove(orig.c_str())) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "remove " << orig << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_REMOVE_FILE; + + return rc; + } + } + else + { + // restore the orig file + if (fFs.exists(orig.c_str())) + { + errno = 0; + + // Try to remove file only if it exists + if ((fFs.exists(filename.c_str())) && (fFs.remove(filename.c_str()) != 0)) + { + int errNum = errno; + std::ostringstream oss; + oss << "failed restore; remove " << filename << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_REMOVE_FILE; + + return rc; + } + + errno = 0; + + if (fFs.rename(orig.c_str(), filename.c_str()) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "failed restore; rename " << orig << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_RENAME_FILE; + + return rc; + } + } + + // remove the tmp file + std::string tmp(filename + ".tmp"); + errno = 0; + + if ((fFs.exists(tmp.c_str())) && (fFs.remove(tmp.c_str())) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "failed restore; remove " << tmp << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_REMOVE_FILE; + + return rc; + } + + // remove the chunk shifting helper + std::string rlc(filename + ".rlc"); + errno = 0; + + if ((fFs.exists(rlc.c_str())) && (fFs.remove(rlc.c_str())) != 0) + { + int errNum = errno; + std::ostringstream oss; + oss << "failed restore; remove " << rlc << " failed: " << strerror(errNum); + errMsg = oss.str(); + rc = ERR_COMP_REMOVE_FILE; + + return rc; + } + } + + return rc; } - + //------------------------------------------------------------------------------ // Confirm the changes to the hwm DB files listed in the bulk rollback meta // data file corresponding to the specified table OID. //------------------------------------------------------------------------------ -int ConfirmHdfsDbFile::confirmDbFileListFromMetaFile( - OID tableOID, - std::string& errMsg) +int ConfirmHdfsDbFile::confirmDbFileListFromMetaFile(OID tableOID, std::string& errMsg) { - int rc = NO_ERROR; + int rc = NO_ERROR; - try + try + { + std::vector dbRoots; + Config::getRootIdList(dbRoots); + + for (unsigned m = 0; m < dbRoots.size(); m++) { - std::vector dbRoots; - Config::getRootIdList( dbRoots ); + std::istringstream metaDataStream; + openMetaDataFile(tableOID, dbRoots[m], metaDataStream); - for (unsigned m = 0; m < dbRoots.size(); m++) - { - std::istringstream metaDataStream; - openMetaDataFile ( tableOID, - dbRoots[m], metaDataStream ); + confirmDbFiles(metaDataStream); + } + } + catch (WeException& ex) + { + std::ostringstream oss; + oss << "Error confirming changes to table " << tableOID << "; " << ex.what(); + errMsg = oss.str(); + rc = ex.errorCode(); + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error confirming changes to table " << tableOID << "; " << ex.what(); + errMsg = oss.str(); + rc = ERR_UNKNOWN; + } - confirmDbFiles( metaDataStream ); - } - } - catch (WeException& ex) - { - std::ostringstream oss; - oss << "Error confirming changes to table " << tableOID << - "; " << ex.what(); - errMsg = oss.str(); - rc = ex.errorCode(); - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Error confirming changes to table " << tableOID << - "; " << ex.what(); - errMsg = oss.str(); - rc = ERR_UNKNOWN; - } - - return rc; + return rc; } - + //------------------------------------------------------------------------------ // Confirm the changes to the hwm DB files listed in the bulk rollback meta // data file stream stored in metaDataStream. //------------------------------------------------------------------------------ void ConfirmHdfsDbFile::confirmDbFiles(std::istringstream& metaDataStream) const { - char inBuf[ BUF_SIZE ]; + char inBuf[BUF_SIZE]; - // Loop through the records in the meta-data file - while (metaDataStream.getline( inBuf, BUF_SIZE )) + // Loop through the records in the meta-data file + while (metaDataStream.getline(inBuf, BUF_SIZE)) + { + // Restore Files for current DBRoot + if (RBMetaWriter::verifyColumn1Rec(inBuf)) { - // Restore Files for current DBRoot - if (RBMetaWriter::verifyColumn1Rec(inBuf)) - { - confirmColumnDbFile(inBuf); - } - else if (RBMetaWriter::verifyDStore1Rec(inBuf)) - { - confirmDctnryStoreDbFile(inBuf); - } + confirmColumnDbFile(inBuf); } + else if (RBMetaWriter::verifyDStore1Rec(inBuf)) + { + confirmDctnryStoreDbFile(inBuf); + } + } } - + //------------------------------------------------------------------------------ // Confirm the changes to the hwm column DB file described in the bulk // rollback meta data file record stored in inBuf. //------------------------------------------------------------------------------ void ConfirmHdfsDbFile::confirmColumnDbFile(const char* inBuf) const { - char recType[100]; - OID columnOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM lastLocalHwm; - int colTypeInt; - char colTypeName[100]; - uint32_t colWidth; - int compressionType = 0; // optional parameter + char recType[100]; + OID columnOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM lastLocalHwm; + int colTypeInt; + char colTypeName[100]; + uint32_t colWidth; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d", - recType, &columnOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &lastLocalHwm, - &colTypeInt, colTypeName, &colWidth, &compressionType ); + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d", recType, &columnOID, &dbRootHwm, &partNumHwm, + &segNumHwm, &lastLocalHwm, &colTypeInt, colTypeName, &colWidth, &compressionType); - if (numFields < 9) // compressionType is optional - { - std::ostringstream oss; - oss << "Invalid COLUM1 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">"; + if (numFields < 9) // compressionType is optional + { + std::ostringstream oss; + oss << "Invalid COLUM1 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">"; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - // Construct the DB file name - char dbFileName[FILE_NAME_SIZE]; - FileOp dbFile(false); - int rc = dbFile.getFileName( columnOID, - dbFileName, - dbRootHwm, - partNumHwm, - segNumHwm ); + // Construct the DB file name + char dbFileName[FILE_NAME_SIZE]; + FileOp dbFile(false); + int rc = dbFile.getFileName(columnOID, dbFileName, dbRootHwm, partNumHwm, segNumHwm); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error constructing column filename to confirm changes" << - "; columnOID-" << columnOID << - "; dbRoot-" << dbRootHwm << - "; partNum-" << partNumHwm << - "; segNum-" << segNumHwm << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error constructing column filename to confirm changes" + << "; columnOID-" << columnOID << "; dbRoot-" << dbRootHwm << "; partNum-" << partNumHwm + << "; segNum-" << segNumHwm << "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } + throw WeException(oss.str(), rc); + } - // Confirm the changes to the DB file name - std::string errMsg; - rc = confirmDbFileChange( std::string("tmp"), - dbFileName, - errMsg ); + // Confirm the changes to the DB file name + std::string errMsg; + rc = confirmDbFileChange(std::string("tmp"), dbFileName, errMsg); - if (rc != NO_ERROR) - { - throw WeException( errMsg, rc ); - } + if (rc != NO_ERROR) + { + throw WeException(errMsg, rc); + } } - + //------------------------------------------------------------------------------ // Confirm the changes to the hwm dctnry store DB file described in the bulk // rollback meta data file record stored in inBuf. //------------------------------------------------------------------------------ void ConfirmHdfsDbFile::confirmDctnryStoreDbFile(const char* inBuf) const { - char recType[100]; - OID dColumnOID; - OID dStoreOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM localHwm; - int compressionType = 0; // optional parameter + char recType[100]; + OID dColumnOID; + OID dStoreOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM localHwm; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d", - recType, &dColumnOID, &dStoreOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &localHwm, &compressionType ); + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d", recType, &dColumnOID, &dStoreOID, &dbRootHwm, + &partNumHwm, &segNumHwm, &localHwm, &compressionType); - if (numFields < 7) // compressionType optional - { - std::ostringstream oss; - oss << "Invalid DSTOR1 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">"; + if (numFields < 7) // compressionType optional + { + std::ostringstream oss; + oss << "Invalid DSTOR1 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">"; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - // Construct the DB file name - char dbFileName[FILE_NAME_SIZE]; - FileOp dbFile(false); - int rc = dbFile.getFileName( dStoreOID, - dbFileName, - dbRootHwm, - partNumHwm, - segNumHwm ); + // Construct the DB file name + char dbFileName[FILE_NAME_SIZE]; + FileOp dbFile(false); + int rc = dbFile.getFileName(dStoreOID, dbFileName, dbRootHwm, partNumHwm, segNumHwm); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error constructing dictionary store filename to confirm changes" << - "; columnOID-" << dStoreOID << - "; dbRoot-" << dbRootHwm << - "; partNum-" << partNumHwm << - "; segNum-" << segNumHwm << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error constructing dictionary store filename to confirm changes" + << "; columnOID-" << dStoreOID << "; dbRoot-" << dbRootHwm << "; partNum-" << partNumHwm + << "; segNum-" << segNumHwm << "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } + throw WeException(oss.str(), rc); + } - // Confirm the changes to the DB file name - std::string errMsg; - rc = confirmDbFileChange( std::string("tmp"), - dbFileName, - errMsg ); + // Confirm the changes to the DB file name + std::string errMsg; + rc = confirmDbFileChange(std::string("tmp"), dbFileName, errMsg); - if (rc != NO_ERROR) - { - throw WeException( errMsg, rc ); - } + if (rc != NO_ERROR) + { + throw WeException(errMsg, rc); + } } - + //------------------------------------------------------------------------------ // End the changes to the hwm DB files listed in the bulk rollback meta // data file corresponding to the specified table OID. Delete temp files. //------------------------------------------------------------------------------ -int ConfirmHdfsDbFile::endDbFileListFromMetaFile( - OID tableOID, - bool success, - std::string& errMsg) +int ConfirmHdfsDbFile::endDbFileListFromMetaFile(OID tableOID, bool success, std::string& errMsg) { - int rc = NO_ERROR; - errMsg.clear(); + int rc = NO_ERROR; + errMsg.clear(); - std::vector dbRoots; - Config::getRootIdList( dbRoots ); + std::vector dbRoots; + Config::getRootIdList(dbRoots); - for (unsigned m = 0; m < dbRoots.size(); m++) + for (unsigned m = 0; m < dbRoots.size(); m++) + { + std::istringstream metaDataStream; + + try { - std::istringstream metaDataStream; + std::istringstream metaDataStream; + openMetaDataFile(tableOID, dbRoots[m], metaDataStream); - try - { - std::istringstream metaDataStream; - openMetaDataFile ( tableOID, - dbRoots[m], metaDataStream ); - - endDbFiles( metaDataStream, success ); - } - // We catch any errors, but not deleting a temp file is not fatal, - // so we capture the error msg and keep going if a problem occurs. - // We return a concatenated list of error msgs if multiple errors - // take place. - catch (WeException& ex) - { - if (errMsg.size() == 0) - { - std::ostringstream oss; - oss << "Error deleting temp files for table " << tableOID << - "; " << ex.what(); - errMsg = oss.str(); - rc = ex.errorCode(); - } - else - { - errMsg += "; "; - errMsg += ex.what(); - } - } - catch (std::exception& ex) - { - if (errMsg.size() == 0) - { - std::ostringstream oss; - oss << "Error deleting temp files for table " << tableOID << - "; " << ex.what(); - errMsg = oss.str(); - rc = ERR_UNKNOWN; - } - else - { - errMsg += "; "; - errMsg += ex.what(); - } - } + endDbFiles(metaDataStream, success); } + // We catch any errors, but not deleting a temp file is not fatal, + // so we capture the error msg and keep going if a problem occurs. + // We return a concatenated list of error msgs if multiple errors + // take place. + catch (WeException& ex) + { + if (errMsg.size() == 0) + { + std::ostringstream oss; + oss << "Error deleting temp files for table " << tableOID << "; " << ex.what(); + errMsg = oss.str(); + rc = ex.errorCode(); + } + else + { + errMsg += "; "; + errMsg += ex.what(); + } + } + catch (std::exception& ex) + { + if (errMsg.size() == 0) + { + std::ostringstream oss; + oss << "Error deleting temp files for table " << tableOID << "; " << ex.what(); + errMsg = oss.str(); + rc = ERR_UNKNOWN; + } + else + { + errMsg += "; "; + errMsg += ex.what(); + } + } + } - return rc; + return rc; } - + //------------------------------------------------------------------------------ // End the changes to the hwm DB files listed in the bulk rollback meta // data file stream stored in metaDataStream. Delete temp files. //------------------------------------------------------------------------------ -void ConfirmHdfsDbFile::endDbFiles( - std::istringstream& metaDataStream, - bool success) const +void ConfirmHdfsDbFile::endDbFiles(std::istringstream& metaDataStream, bool success) const { - char inBuf[ BUF_SIZE ]; - std::string errMsg; - int rc = NO_ERROR; + char inBuf[BUF_SIZE]; + std::string errMsg; + int rc = NO_ERROR; - // Loop through the records in the meta-data file - while (metaDataStream.getline( inBuf, BUF_SIZE )) + // Loop through the records in the meta-data file + while (metaDataStream.getline(inBuf, BUF_SIZE)) + { + try { - try - { - // Delete Temp Files for current DBRoot - if (RBMetaWriter::verifyColumn1Rec(inBuf)) - { - endColumnDbFile(inBuf, success); - } - else if (RBMetaWriter::verifyDStore1Rec(inBuf)) - { - endDctnryStoreDbFile(inBuf, success); - } - } - // We catch any errors, but not deleting a temp file is not fatal, - // so we capture the error msg and keep going if a problem occurs. - // We return a concatenated list of error msgs if multiple errors - // take place. - catch (WeException& ex) - { - if (errMsg.size() == 0) - { - rc = ex.errorCode(); - } - else - { - errMsg += "; "; - } - - errMsg += ex.what(); - } - catch (std::exception& ex) - { - if (errMsg.size() == 0) - { - rc = ERR_UNKNOWN; - } - else - { - errMsg += "; "; - } - - errMsg += ex.what(); - } + // Delete Temp Files for current DBRoot + if (RBMetaWriter::verifyColumn1Rec(inBuf)) + { + endColumnDbFile(inBuf, success); + } + else if (RBMetaWriter::verifyDStore1Rec(inBuf)) + { + endDctnryStoreDbFile(inBuf, success); + } } - - // Throw exception with cumulative list of any error msgs - if (errMsg.size() > 0) + // We catch any errors, but not deleting a temp file is not fatal, + // so we capture the error msg and keep going if a problem occurs. + // We return a concatenated list of error msgs if multiple errors + // take place. + catch (WeException& ex) { - throw WeException( errMsg, rc ); + if (errMsg.size() == 0) + { + rc = ex.errorCode(); + } + else + { + errMsg += "; "; + } + + errMsg += ex.what(); } + catch (std::exception& ex) + { + if (errMsg.size() == 0) + { + rc = ERR_UNKNOWN; + } + else + { + errMsg += "; "; + } + + errMsg += ex.what(); + } + } + + // Throw exception with cumulative list of any error msgs + if (errMsg.size() > 0) + { + throw WeException(errMsg, rc); + } } - + //------------------------------------------------------------------------------ // End the changes to the hwm column DB file described in the bulk // rollback meta data file record stored in inBuf. Delete the temp file. //------------------------------------------------------------------------------ -void ConfirmHdfsDbFile::endColumnDbFile( - const char* inBuf, - bool success) const +void ConfirmHdfsDbFile::endColumnDbFile(const char* inBuf, bool success) const { - char recType[100]; - OID columnOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM lastLocalHwm; - int colTypeInt; - char colTypeName[100]; - uint32_t colWidth; - int compressionType = 0; // optional parameter + char recType[100]; + OID columnOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM lastLocalHwm; + int colTypeInt; + char colTypeName[100]; + uint32_t colWidth; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d", - recType, &columnOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &lastLocalHwm, - &colTypeInt, colTypeName, &colWidth, &compressionType ); + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %d %s %u %d", recType, &columnOID, &dbRootHwm, &partNumHwm, + &segNumHwm, &lastLocalHwm, &colTypeInt, colTypeName, &colWidth, &compressionType); - if (numFields < 9) // compressionType is optional - { - std::ostringstream oss; - oss << "Invalid COLUM1 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">"; + if (numFields < 9) // compressionType is optional + { + std::ostringstream oss; + oss << "Invalid COLUM1 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">"; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - // Construct the DB file name - char dbFileName[FILE_NAME_SIZE]; - FileOp dbFile(false); - int rc = dbFile.getFileName( columnOID, - dbFileName, - dbRootHwm, - partNumHwm, - segNumHwm ); + // Construct the DB file name + char dbFileName[FILE_NAME_SIZE]; + FileOp dbFile(false); + int rc = dbFile.getFileName(columnOID, dbFileName, dbRootHwm, partNumHwm, segNumHwm); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error constructing column filename to end changes" << - "; columnOID-" << columnOID << - "; dbRoot-" << dbRootHwm << - "; partNum-" << partNumHwm << - "; segNum-" << segNumHwm << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error constructing column filename to end changes" + << "; columnOID-" << columnOID << "; dbRoot-" << dbRootHwm << "; partNum-" << partNumHwm + << "; segNum-" << segNumHwm << "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } + throw WeException(oss.str(), rc); + } - // Confirm the changes to the DB file name - std::string errMsg; - rc = endDbFileChange( std::string("tmp"), - dbFileName, - success, - errMsg ); + // Confirm the changes to the DB file name + std::string errMsg; + rc = endDbFileChange(std::string("tmp"), dbFileName, success, errMsg); - if (rc != NO_ERROR) - { - throw WeException( errMsg, rc ); - } + if (rc != NO_ERROR) + { + throw WeException(errMsg, rc); + } } - + //------------------------------------------------------------------------------ // End the changes to the hwm dctnry store DB file described in the bulk // rollback meta data file record stored in inBuf. Delete the temp file. //------------------------------------------------------------------------------ -void ConfirmHdfsDbFile::endDctnryStoreDbFile( - const char* inBuf, - bool success) const +void ConfirmHdfsDbFile::endDctnryStoreDbFile(const char* inBuf, bool success) const { - char recType[100]; - OID dColumnOID; - OID dStoreOID; - uint32_t dbRootHwm; - uint32_t partNumHwm; - uint32_t segNumHwm; - HWM localHwm; - int compressionType = 0; // optional parameter + char recType[100]; + OID dColumnOID; + OID dStoreOID; + uint32_t dbRootHwm; + uint32_t partNumHwm; + uint32_t segNumHwm; + HWM localHwm; + int compressionType = 0; // optional parameter - // Read meta-data record - int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d", - recType, &dColumnOID, &dStoreOID, - &dbRootHwm, &partNumHwm, &segNumHwm, &localHwm, &compressionType ); + // Read meta-data record + int numFields = sscanf(inBuf, "%s %u %u %u %u %u %u %d", recType, &dColumnOID, &dStoreOID, &dbRootHwm, + &partNumHwm, &segNumHwm, &localHwm, &compressionType); - if (numFields < 7) // compressionType optional - { - std::ostringstream oss; - oss << "Invalid DSTOR1 record in meta-data file " << - fMetaFileName << "; record-<" << inBuf << ">"; + if (numFields < 7) // compressionType optional + { + std::ostringstream oss; + oss << "Invalid DSTOR1 record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">"; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } - // Construct the DB file name - char dbFileName[FILE_NAME_SIZE]; - FileOp dbFile(false); - int rc = dbFile.getFileName( dStoreOID, - dbFileName, - dbRootHwm, - partNumHwm, - segNumHwm ); + // Construct the DB file name + char dbFileName[FILE_NAME_SIZE]; + FileOp dbFile(false); + int rc = dbFile.getFileName(dStoreOID, dbFileName, dbRootHwm, partNumHwm, segNumHwm); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error constructing dictionary store filename to end changes" << - "; columnOID-" << dStoreOID << - "; dbRoot-" << dbRootHwm << - "; partNum-" << partNumHwm << - "; segNum-" << segNumHwm << - "; " << ec.errorString(rc); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error constructing dictionary store filename to end changes" + << "; columnOID-" << dStoreOID << "; dbRoot-" << dbRootHwm << "; partNum-" << partNumHwm + << "; segNum-" << segNumHwm << "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } + throw WeException(oss.str(), rc); + } - // Confirm the changes to the DB file name - std::string errMsg; - rc = endDbFileChange( std::string("tmp"), - dbFileName, - success, - errMsg ); + // Confirm the changes to the DB file name + std::string errMsg; + rc = endDbFileChange(std::string("tmp"), dbFileName, success, errMsg); - if (rc != NO_ERROR) - { - throw WeException( errMsg, rc ); - } + if (rc != NO_ERROR) + { + throw WeException(errMsg, rc); + } } - + //------------------------------------------------------------------------------ // Open and read the bulk rollback metadata file for the specified table OID // and DBRoot. The contents of the metadata file is returned in the meta- // DataStream argument. //------------------------------------------------------------------------------ -void ConfirmHdfsDbFile::openMetaDataFile(OID tableOID, - uint16_t dbRoot, - std::istringstream& metaDataStream) +void ConfirmHdfsDbFile::openMetaDataFile(OID tableOID, uint16_t dbRoot, std::istringstream& metaDataStream) { - std::string bulkRollbackPath( Config::getDBRootByNum( dbRoot ) ); + std::string bulkRollbackPath(Config::getDBRootByNum(dbRoot)); - // Construct file name and check for it's existence - std::ostringstream ossFileName; - ossFileName << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << tableOID; - fMetaFileName = bulkRollbackPath; - fMetaFileName += ossFileName.str(); + // Construct file name and check for it's existence + std::ostringstream ossFileName; + ossFileName << '/' << DBROOT_BULK_ROLLBACK_SUBDIR << '/' << tableOID; + fMetaFileName = bulkRollbackPath; + fMetaFileName += ossFileName.str(); - // Return if the meta-data file does not exist. - if ( !fFs.exists( fMetaFileName.c_str() ) ) - { - std::ostringstream oss; - oss << "Bulk rollback meta-data file " << - fMetaFileName << " does not exist."; + // Return if the meta-data file does not exist. + if (!fFs.exists(fMetaFileName.c_str())) + { + std::ostringstream oss; + oss << "Bulk rollback meta-data file " << fMetaFileName << " does not exist."; - throw WeException( oss.str(), ERR_FILE_NOT_EXIST ); - } + throw WeException(oss.str(), ERR_FILE_NOT_EXIST); + } - // Open the file - boost::scoped_ptr metaFile; + // Open the file + boost::scoped_ptr metaFile; + errno = 0; + metaFile.reset(idbdatafile::IDBDataFile::open( + idbdatafile::IDBPolicy::getType(fMetaFileName.c_str(), idbdatafile::IDBPolicy::WRITEENG), + fMetaFileName.c_str(), "rb", 0)); + + if (!metaFile) + { + int errRc = errno; + std::ostringstream oss; + oss << "Error opening bulk rollback meta-data file " << fMetaFileName << "; err-" << errRc << "; " + << strerror(errRc); + + throw WeException(oss.str(), ERR_FILE_OPEN); + } + + // First record in the file must be a Version record. + char inBuf[BUF_SIZE]; + ssize_t metaFileSize = fFs.size(fMetaFileName.c_str()); + boost::scoped_array buf(new char[metaFileSize]); + // retry 10 times for partial reads, just in case + ssize_t readSofar = 0; // bytes read so far + ssize_t bytes = 0; // bytes read by one pread + char* p = buf.get(); + + for (int i = 0; i < 10 && readSofar < metaFileSize; i++) + { errno = 0; - metaFile.reset(idbdatafile::IDBDataFile::open( - idbdatafile::IDBPolicy::getType(fMetaFileName.c_str(), - idbdatafile::IDBPolicy::WRITEENG), - fMetaFileName.c_str(), "rb", 0) ); + bytes = metaFile->pread(p + readSofar, readSofar, metaFileSize - readSofar); - if ( !metaFile ) - { - int errRc = errno; - std::ostringstream oss; - oss << "Error opening bulk rollback meta-data file " << - fMetaFileName << "; err-" << - errRc << "; " << strerror( errRc ); + if (bytes < 0) + break; - throw WeException( oss.str(), ERR_FILE_OPEN ); - } + readSofar += bytes; + } - // First record in the file must be a Version record. - char inBuf[ BUF_SIZE ]; - ssize_t metaFileSize = fFs.size( fMetaFileName.c_str() ); - boost::scoped_array buf( new char[ metaFileSize ] ); - // retry 10 times for partial reads, just in case - ssize_t readSofar = 0; // bytes read so far - ssize_t bytes = 0; // bytes read by one pread - char* p = buf.get(); + if (readSofar != metaFileSize) + { + int errRc = errno; + std::ostringstream oss; + oss << "Error reading bulk rollback meta-data file " << fMetaFileName << "; read/expect:" << readSofar + << "/" << metaFileSize << "; err-" << errRc << "; " << strerror(errRc); - for (int i = 0; i < 10 && readSofar < metaFileSize; i++) - { - errno = 0; - bytes = metaFile->pread( p + readSofar, - readSofar, - metaFileSize - readSofar); + throw WeException(oss.str(), ERR_FILE_READ); + } - if (bytes < 0) - break; + // put the data in a string stream + metaDataStream.str(std::string(p, metaFileSize)); + buf.reset(); - readSofar += bytes; - } + // read data + metaDataStream.getline(inBuf, BUF_SIZE); - if ( readSofar != metaFileSize ) - { - int errRc = errno; - std::ostringstream oss; - oss << "Error reading bulk rollback meta-data file " - << fMetaFileName << "; read/expect:" << readSofar << "/" - << metaFileSize - << "; err-" << errRc << "; " << strerror( errRc ); + if (!RBMetaWriter::verifyVersion4(inBuf)) + { + std::ostringstream oss; + oss << "Invalid version record in meta-data file " << fMetaFileName << "; record-<" << inBuf << ">"; - throw WeException( oss.str(), ERR_FILE_READ ); - } - - // put the data in a string stream - metaDataStream.str( std::string( p, metaFileSize ) ); - buf.reset(); - - // read data - metaDataStream.getline( inBuf, BUF_SIZE ); - - if (!RBMetaWriter::verifyVersion4(inBuf)) - { - std::ostringstream oss; - oss << "Invalid version record in meta-data file " << fMetaFileName - << "; record-<" << inBuf << ">"; - - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } + throw WeException(oss.str(), ERR_INVALID_PARAM); + } } -} // end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_confirmhdfsdbfile.h b/writeengine/shared/we_confirmhdfsdbfile.h index f0d51a312..f853f6531 100644 --- a/writeengine/shared/we_confirmhdfsdbfile.h +++ b/writeengine/shared/we_confirmhdfsdbfile.h @@ -33,7 +33,6 @@ namespace WriteEngine { - /** @brief Encapsulates logic to confirm and finalize (or abort) changes * to an HDFS db file. Class should only be used for HDFS db files. * @@ -47,92 +46,84 @@ namespace WriteEngine */ class ConfirmHdfsDbFile { -public: - EXPORT ConfirmHdfsDbFile( ); - EXPORT ~ConfirmHdfsDbFile( ); + public: + EXPORT ConfirmHdfsDbFile(); + EXPORT ~ConfirmHdfsDbFile(); - /** @brief Confirm changes to the specified db file - * @param backUpFileType Backup file type to confirm. Types: - * "rlc" - reallocated chunk file - * "tmp" - updated db file - * @param filename Name of db file to be confirmed. - * @param errMsg (out) Error msg associated with a bad return code - * @return Returns NO_ERROR if call is successful - */ - EXPORT int confirmDbFileChange( const std::string& backUpFileType, - const std::string& filename, - std::string& errMsg ) const; + /** @brief Confirm changes to the specified db file + * @param backUpFileType Backup file type to confirm. Types: + * "rlc" - reallocated chunk file + * "tmp" - updated db file + * @param filename Name of db file to be confirmed. + * @param errMsg (out) Error msg associated with a bad return code + * @return Returns NO_ERROR if call is successful + */ + EXPORT int confirmDbFileChange(const std::string& backUpFileType, const std::string& filename, + std::string& errMsg) const; - /** @brief Finalize changes to the specified db file - * - * If success flag is true: - * The old version of the db file is deleted. - * If success flag is false: - * The old version is retained, and any temporary file with pending - * changes is deleted. - * - * @param backUpFileType Backup file type to finalize. Types: - * "rlc" - reallocated chunk file - * "tmp" - updated db file - * @param filename Name of db file to be finalized. - * @param success Final success/fail status of db file changes - * @param errMsg (out) Error msg associated with a bad return code - * @return Returns NO_ERROR if call is successful - */ - EXPORT int endDbFileChange( const std::string& backUpFileType, - const std::string& filename, - bool success, - std::string& errMsg ) const; + /** @brief Finalize changes to the specified db file + * + * If success flag is true: + * The old version of the db file is deleted. + * If success flag is false: + * The old version is retained, and any temporary file with pending + * changes is deleted. + * + * @param backUpFileType Backup file type to finalize. Types: + * "rlc" - reallocated chunk file + * "tmp" - updated db file + * @param filename Name of db file to be finalized. + * @param success Final success/fail status of db file changes + * @param errMsg (out) Error msg associated with a bad return code + * @return Returns NO_ERROR if call is successful + */ + EXPORT int endDbFileChange(const std::string& backUpFileType, const std::string& filename, bool success, + std::string& errMsg) const; - /** @brief Confirm changes to the db files modified for tableOID - * - * The HWM db file for each DBRoot, as listed in the bulk rollback meta - * data file (for the specified tableOID), is confirmed. - * - * @param tableOID Table that has changes to be confirmed - * @param errMsg (out) Error msg associated with a bad return code - * @return Returns NO_ERROR if call is successful - */ - EXPORT int confirmDbFileListFromMetaFile( OID tableOID, - std::string& errMsg ); + /** @brief Confirm changes to the db files modified for tableOID + * + * The HWM db file for each DBRoot, as listed in the bulk rollback meta + * data file (for the specified tableOID), is confirmed. + * + * @param tableOID Table that has changes to be confirmed + * @param errMsg (out) Error msg associated with a bad return code + * @return Returns NO_ERROR if call is successful + */ + EXPORT int confirmDbFileListFromMetaFile(OID tableOID, std::string& errMsg); - /** @brief Finalize changes to the db files modified for tableOID - * - * The HWM db file for each DBRoot, as listed in the bulk rollback meta - * data file (for the specified tableOID), is finalized. - * - * If success flag is true: - * The old version of the db files are deleted. - * If success flag is false: - * The old versions are retained, and any temporary files with pending - * changes are deleted. - * - * @param tableOID Table that has changes to be confirmed - * @param errMsg (out) Error msg associated with a bad return code - * @return Returns NO_ERROR if call is successful - */ - EXPORT int endDbFileListFromMetaFile( OID tableOID, - bool success, - std::string& errMsg ); + /** @brief Finalize changes to the db files modified for tableOID + * + * The HWM db file for each DBRoot, as listed in the bulk rollback meta + * data file (for the specified tableOID), is finalized. + * + * If success flag is true: + * The old version of the db files are deleted. + * If success flag is false: + * The old versions are retained, and any temporary files with pending + * changes are deleted. + * + * @param tableOID Table that has changes to be confirmed + * @param errMsg (out) Error msg associated with a bad return code + * @return Returns NO_ERROR if call is successful + */ + EXPORT int endDbFileListFromMetaFile(OID tableOID, bool success, std::string& errMsg); -private: - void openMetaDataFile( OID tableOID, - uint16_t dbRoot, - std::istringstream& metaDataStream ); + private: + void openMetaDataFile(OID tableOID, uint16_t dbRoot, std::istringstream& metaDataStream); - void confirmDbFiles( std::istringstream& metaDataStream ) const; - void confirmColumnDbFile( const char* inBuf ) const; - void confirmDctnryStoreDbFile( const char* inBuf ) const; + void confirmDbFiles(std::istringstream& metaDataStream) const; + void confirmColumnDbFile(const char* inBuf) const; + void confirmDctnryStoreDbFile(const char* inBuf) const; - void endDbFiles( std::istringstream& metaDataStream, bool success ) const; - void endColumnDbFile( const char* inBuf, bool success ) const; - void endDctnryStoreDbFile( const char* inBuf, bool success ) const; + void endDbFiles(std::istringstream& metaDataStream, bool success) const; + void endColumnDbFile(const char* inBuf, bool success) const; + void endDctnryStoreDbFile(const char* inBuf, bool success) const; - idbdatafile::IDBFileSystem& fFs; - std::string fMetaFileName; + idbdatafile::IDBFileSystem& fFs; + std::string fMetaFileName; }; -} // end of namespace +} // namespace WriteEngine #undef EXPORT diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 79a522282..65633ac3e 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_convertor.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_convertor.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ /** @file */ #include @@ -61,21 +61,21 @@ const char CS_FILE_FORMAT[] = "FILE[0-9][0-9][0-9].cdf"; ******************************************************************************/ int _doDir(char* pBuffer, int blen, unsigned int val) { - int rc; + int rc; - if (!pBuffer) - { - rc = -1; - } - else - { - rc = snprintf(pBuffer, blen, "%03u.dir", val); - pBuffer[blen - 1] = (char)0; - } + if (!pBuffer) + { + rc = -1; + } + else + { + rc = snprintf(pBuffer, blen, "%03u.dir", val); + pBuffer[blen - 1] = (char)0; + } - return rc; + return rc; } - + /******************************************************************************* * DESCRIPTION: * Takes an 8-bit value and converts it into a file name. @@ -89,19 +89,19 @@ int _doDir(char* pBuffer, int blen, unsigned int val) ******************************************************************************/ int _doFile(char* pBuffer, int blen, unsigned char val) { - int rc; + int rc; - if (!pBuffer) - { - rc = -1; - } - else - { - rc = snprintf(pBuffer, blen, "FILE%03d.cdf", val); - pBuffer[blen - 1] = (char)0; - } + if (!pBuffer) + { + rc = -1; + } + else + { + rc = snprintf(pBuffer, blen, "FILE%03d.cdf", val); + pBuffer[blen - 1] = (char)0; + } - return rc; + return rc; } /******************************************************************************* @@ -116,25 +116,23 @@ int _doFile(char* pBuffer, int blen, unsigned char val) * RETURN: * 0 is returned on success, -1 is returned on error. ******************************************************************************/ -int32_t _fromText(const char* buffer, const char* fmt, uint32_t offset, - uint32_t& val) +int32_t _fromText(const char* buffer, const char* fmt, uint32_t offset, uint32_t& val) { - int32_t rc = -1; - // Number length in characters. - const uint32_t numberLen = 3; + int32_t rc = -1; + // Number length in characters. + const uint32_t numberLen = 3; - // Check that buffer is in the correct `directory` format. - if (buffer && (fnmatch(fmt, buffer, 0) == 0)) - { - datatypes::DataCondition error; - val = literal::UnsignedInteger(buffer + offset, numberLen) - .toXIntPositive(error); - // The number cannot exceed 0xff. - if (!error && val < 256) - rc = 0; - } + // Check that buffer is in the correct `directory` format. + if (buffer && (fnmatch(fmt, buffer, 0) == 0)) + { + datatypes::DataCondition error; + val = literal::UnsignedInteger(buffer + offset, numberLen).toXIntPositive(error); + // The number cannot exceed 0xff. + if (!error && val < 256) + rc = 0; + } - return rc; + return rc; } /******************************************************************************* @@ -149,7 +147,7 @@ int32_t _fromText(const char* buffer, const char* fmt, uint32_t offset, ******************************************************************************/ int32_t _fromDir(const char* buffer, uint32_t& val) { - return _fromText(buffer, CS_DIR_FORMAT, 0, val); + return _fromText(buffer, CS_DIR_FORMAT, 0, val); } /******************************************************************************* @@ -164,35 +162,34 @@ int32_t _fromDir(const char* buffer, uint32_t& val) ******************************************************************************/ int32_t _fromFile(const char* buffer, uint32_t& val) { - return _fromText(buffer, CS_FILE_FORMAT, 4, val); + return _fromText(buffer, CS_FILE_FORMAT, 4, val); } -} - +} // namespace + namespace WriteEngine { - // This struct represents a path to a segment file. struct Convertor::dmFilePathArgs_t { - struct dmFilePathPart_t - { - char* pName; // Name buffer. - int32_t len; // Size in bytes of a buffer. - }; - // A, B, C, D, E and file name. - dmFilePathPart_t pathPart[6]; + struct dmFilePathPart_t + { + char* pName; // Name buffer. + int32_t len; // Size in bytes of a buffer. + }; + // A, B, C, D, E and file name. + dmFilePathPart_t pathPart[6]; - // Initialize `dmFilePathArgs_t` from the given buffer. - void initialize(char buff[6][MAX_DB_DIR_NAME_SIZE]) + // Initialize `dmFilePathArgs_t` from the given buffer. + void initialize(char buff[6][MAX_DB_DIR_NAME_SIZE]) + { + // A, B, C, D, E directories. + for (uint32_t i = 0; i < 6; ++i) { - // A, B, C, D, E directories. - for (uint32_t i = 0; i < 6; ++i) - { - pathPart[i].pName = buff[i]; - pathPart[i].len = sizeof(buff[i]); - } + pathPart[i].pName = buff[i]; + pathPart[i].len = sizeof(buff[i]); } + } }; /******************************************************************************* @@ -206,21 +203,20 @@ struct Convertor::dmFilePathArgs_t /* static */ const std::string Convertor::getTimeStr() { - char buf[sizeof(DATE_TIME_FORMAT) + 10] = {0}; - time_t curTime = time(NULL); - struct tm pTime; - localtime_r(&curTime, &pTime); - string timeStr; + char buf[sizeof(DATE_TIME_FORMAT) + 10] = {0}; + time_t curTime = time(NULL); + struct tm pTime; + localtime_r(&curTime, &pTime); + string timeStr; - snprintf(buf, sizeof(buf), DATE_TIME_FORMAT, pTime.tm_year + 1900, - pTime.tm_mon + 1, pTime.tm_mday, - pTime.tm_hour, pTime.tm_min, pTime.tm_sec); + snprintf(buf, sizeof(buf), DATE_TIME_FORMAT, pTime.tm_year + 1900, pTime.tm_mon + 1, pTime.tm_mday, + pTime.tm_hour, pTime.tm_min, pTime.tm_sec); - timeStr = buf; + timeStr = buf; - return timeStr; + return timeStr; } - + /******************************************************************************* * DESCRIPTION: * Convert int value to string @@ -232,15 +228,15 @@ const std::string Convertor::getTimeStr() /* static */ const std::string Convertor::int2Str(int val) { - char buf[12]; - string myStr; + char buf[12]; + string myStr; - snprintf(buf, sizeof(buf), "%d", val); - myStr = buf; + snprintf(buf, sizeof(buf), "%d", val); + myStr = buf; - return myStr; + return myStr; } - + /******************************************************************************* * DESCRIPTION: * Convert a numeric string to a decimal long long, given the specified @@ -255,42 +251,37 @@ const std::string Convertor::int2Str(int val) ******************************************************************************/ /* static */ -long long Convertor::convertDecimalString( - const char* field, - int fieldLength, - int scale) +long long Convertor::convertDecimalString(const char* field, int fieldLength, int scale) { - long double dval = strtold(field, NULL); - long long ret = 0; + long double dval = strtold(field, NULL); + long long ret = 0; - - // move scale digits to the left of the decimal point - for (int i = 0; i < scale; i++) - dval *= 10; + // move scale digits to the left of the decimal point + for (int i = 0; i < scale; i++) + dval *= 10; - - // range check against int64 - if (dval > LLONG_MAX) - { - errno = ERANGE; - return LLONG_MAX; - } - if (dval < LLONG_MIN) - { - errno = ERANGE; - return LLONG_MIN; - } - errno = 0; + // range check against int64 + if (dval > LLONG_MAX) + { + errno = ERANGE; + return LLONG_MAX; + } + if (dval < LLONG_MIN) + { + errno = ERANGE; + return LLONG_MIN; + } + errno = 0; - ret = dval; + ret = dval; - // get the fractional part of what's left & round ret up or down. - dval -= ret; - if (dval >= 0.5 && ret < LLONG_MAX) - ++ret; - else if (dval <= -0.5 && ret > LLONG_MIN) - --ret; - return ret; + // get the fractional part of what's left & round ret up or down. + dval -= ret; + if (dval >= 0.5 && ret < LLONG_MAX) + ++ret; + else if (dval <= -0.5 && ret > LLONG_MIN) + --ret; + return ret; } /******************************************************************************* @@ -307,36 +298,29 @@ long long Convertor::convertDecimalString( * NO_ERROR if success, other if fail ******************************************************************************/ /* static */ -int Convertor::oid2FileName(FID fid, - char* fullFileName, - char dbDirName[][MAX_DB_DIR_NAME_SIZE], - uint32_t partition, - uint16_t segment) +int Convertor::oid2FileName(FID fid, char* fullFileName, char dbDirName[][MAX_DB_DIR_NAME_SIZE], + uint32_t partition, uint16_t segment) { - dmFilePathArgs_t args; - int rc; + dmFilePathArgs_t args; + int rc; - // Initialize. - char buff[6][MAX_DB_DIR_NAME_SIZE]; - args.initialize(buff); + // Initialize. + char buff[6][MAX_DB_DIR_NAME_SIZE]; + args.initialize(buff); - RETURN_ON_WE_ERROR( - (rc = dmOid2FPath(fid, partition, segment, &args)), - ERR_DM_CONVERT_OID); - sprintf(fullFileName, "%s/%s/%s/%s/%s/%s", args.pathPart[0].pName, - args.pathPart[1].pName, args.pathPart[2].pName, - args.pathPart[3].pName, args.pathPart[4].pName, - args.pathPart[5].pName); + RETURN_ON_WE_ERROR((rc = dmOid2FPath(fid, partition, segment, &args)), ERR_DM_CONVERT_OID); + sprintf(fullFileName, "%s/%s/%s/%s/%s/%s", args.pathPart[0].pName, args.pathPart[1].pName, + args.pathPart[2].pName, args.pathPart[3].pName, args.pathPart[4].pName, args.pathPart[5].pName); - for (uint32_t i = 0; i < 6; ++i) - { - strcpy(dbDirName[i], args.pathPart[i].pName); - } + for (uint32_t i = 0; i < 6; ++i) + { + strcpy(dbDirName[i], args.pathPart[i].pName); + } - // std::cout << "OID: " << fid << - // " mapping to file: " << fullFileName < dirNames; + // We need exact 6 instances. + dirNames.reserve(6); + + uint32_t end = fullFileNameLen; + // Signed integer for `index` since it could be less than zero. + int32_t index = fullFileNameLen - 1; + + // Iterate over `fullFileName` starting from the end and split it by + // directory separator. Since we starting from the end we need just 6 + // instances to match ColumnStore file name format specification. + while (index >= 0 && dirNames.size() < dirNamesMaxSize) + { + while (index >= 0 && fullFileName[index] != dirSep) { - return -1; + --index; } - std::vector dirNames; - // We need exact 6 instances. - dirNames.reserve(6); - - uint32_t end = fullFileNameLen; - // Signed integer for `index` since it could be less than zero. - int32_t index = fullFileNameLen - 1; - - // Iterate over `fullFileName` starting from the end and split it by - // directory separator. Since we starting from the end we need just 6 - // instances to match ColumnStore file name format specification. - while (index >= 0 && dirNames.size() < dirNamesMaxSize) + // Begin is a `dirSep` index + 1. + uint32_t begin = index + 1; + const uint32_t dirNameLen = end - begin; + // We already checked the `fullFileName` format, + // but this check is only intended to make sure that this algo works + // correctly on any input, if something changes. + if (dirNameLen > 0 && dirNameLen < MAX_DB_DIR_NAME_SIZE) { - while (index >= 0 && fullFileName[index] != dirSep) - { - --index; - } - - // Begin is a `dirSep` index + 1. - uint32_t begin = index + 1; - const uint32_t dirNameLen = end - begin; - // We already checked the `fullFileName` format, - // but this check is only intended to make sure that this algo works - // correctly on any input, if something changes. - if (dirNameLen > 0 && dirNameLen < MAX_DB_DIR_NAME_SIZE) - { - dirNames.push_back(fullFileName.substr(begin, dirNameLen)); - } - else - { - // Something wrong with filename, just return an error. - return -1; - } - // Set `end` to the last directory separator index. - end = index; - // Skip current directory separator. - --index; + dirNames.push_back(fullFileName.substr(begin, dirNameLen)); } - - // Make sure we parsed 6 instances. - if (dirNames.size() != 6) + else { - return -1; + // Something wrong with filename, just return an error. + return -1; } + // Set `end` to the last directory separator index. + end = index; + // Skip current directory separator. + --index; + } - // Initialize `dmFilePathArgs_t` struct. - char buff[6][MAX_DB_DIR_NAME_SIZE]; + // Make sure we parsed 6 instances. + if (dirNames.size() != 6) + { + return -1; + } - dmFilePathArgs_t args; - args.initialize(buff); + // Initialize `dmFilePathArgs_t` struct. + char buff[6][MAX_DB_DIR_NAME_SIZE]; - // Populate `dmFilePathArgs_t` struct with the given names. - // Starting from the E directory. - for (uint32_t i = 0, dirCount = 5; i <= dirCount; ++i) - { - strcpy(args.pathPart[dirCount - i].pName, dirNames[i].c_str()); - } + dmFilePathArgs_t args; + args.initialize(buff); - // FIXME: Currently used ERR_DM_CONVERT_OID, should we introduce new error - // code? - RETURN_ON_WE_ERROR(dmFPath2Oid(args, oid, partition, segment), - ERR_DM_CONVERT_OID); + // Populate `dmFilePathArgs_t` struct with the given names. + // Starting from the E directory. + for (uint32_t i = 0, dirCount = 5; i <= dirCount; ++i) + { + strcpy(args.pathPart[dirCount - i].pName, dirNames[i].c_str()); + } - return NO_ERROR; + // FIXME: Currently used ERR_DM_CONVERT_OID, should we introduce new error + // code? + RETURN_ON_WE_ERROR(dmFPath2Oid(args, oid, partition, segment), ERR_DM_CONVERT_OID); + + return NO_ERROR; } /******************************************************************************* @@ -451,26 +434,26 @@ int Convertor::fileName2Oid(const std::string& fullFileName, uint32_t& oid, /* static */ void Convertor::mapErrnoToString(int errNum, std::string& errString) { - char errnoMsgBuf[1024]; + char errnoMsgBuf[1024]; #if STRERROR_R_CHAR_P - char* errnoMsg = strerror_r(errNum, errnoMsgBuf, sizeof(errnoMsgBuf)); + char* errnoMsg = strerror_r(errNum, errnoMsgBuf, sizeof(errnoMsgBuf)); - if (errnoMsg) - errString = errnoMsg; - else - errString.clear(); + if (errnoMsg) + errString = errnoMsg; + else + errString.clear(); #else - int errnoMsg = strerror_r(errNum, errnoMsgBuf, sizeof(errnoMsgBuf)); + int errnoMsg = strerror_r(errNum, errnoMsgBuf, sizeof(errnoMsgBuf)); - if (errnoMsg == 0) - errString = errnoMsgBuf; - else - errString.clear(); + if (errnoMsg == 0) + errString = errnoMsgBuf; + else + errString.clear(); #endif } - + /******************************************************************************* * DESCRIPTION: * Convert specified ColDataType to internal storage type (ColType). @@ -481,140 +464,96 @@ void Convertor::mapErrnoToString(int errNum, std::string& errString) * none ******************************************************************************/ /* static */ -void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, - int colWidth, ColType& internalType, bool isToken) +void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, int colWidth, + ColType& internalType, bool isToken) { - if (isToken) + if (isToken) + { + internalType = WriteEngine::WR_TOKEN; + return; + } + + switch (dataType) + { + // Map BIT and TINYINT to WR_BYTE + case CalpontSystemCatalog::BIT: + case CalpontSystemCatalog::TINYINT: internalType = WriteEngine::WR_BYTE; break; + + // Map SMALLINT to WR_SHORT + case CalpontSystemCatalog::SMALLINT: internalType = WriteEngine::WR_SHORT; break; + + // Map MEDINT to WR_MEDINT + case CalpontSystemCatalog::MEDINT: internalType = WriteEngine::WR_MEDINT; break; + + // Map INT, and DATE to WR_INT + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DATE: internalType = WriteEngine::WR_INT; break; + + // Map FLOAT and UFLOAT to WR_FLOAT + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: internalType = WriteEngine::WR_FLOAT; break; + + // Map BIGINT, DATETIME, TIMESTAMP, and TIME to WR_LONGLONG + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: internalType = WriteEngine::WR_LONGLONG; break; + + // Map DOUBLE and UDOUBLE to WR_DOUBLE + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: internalType = WriteEngine::WR_DOUBLE; break; + + // Map DECIMAL to applicable integer type + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - internalType = WriteEngine::WR_TOKEN; - return; + switch (colWidth) + { + case 1: internalType = WriteEngine::WR_BYTE; break; + + case 2: internalType = WriteEngine::WR_SHORT; break; + + case 4: internalType = WriteEngine::WR_INT; break; + + case 8: internalType = WriteEngine::WR_LONGLONG; break; + + default: internalType = WriteEngine::WR_BINARY; break; + } + + break; } - switch (dataType) - { - // Map BIT and TINYINT to WR_BYTE - case CalpontSystemCatalog::BIT : - case CalpontSystemCatalog::TINYINT : - internalType = WriteEngine::WR_BYTE; - break; + // Map BLOB to WR_BLOB + case CalpontSystemCatalog::BLOB: internalType = WriteEngine::WR_BLOB; break; - // Map SMALLINT to WR_SHORT - case CalpontSystemCatalog::SMALLINT : - internalType = WriteEngine::WR_SHORT; - break; + // Map TEXT to WR_TEXT + case CalpontSystemCatalog::TEXT: internalType = WriteEngine::WR_TEXT; break; - // Map MEDINT to WR_MEDINT - case CalpontSystemCatalog::MEDINT : - internalType = WriteEngine::WR_MEDINT; - break; + // Map VARBINARY to WR_VARBINARY + case CalpontSystemCatalog::VARBINARY: internalType = WriteEngine::WR_VARBINARY; break; - // Map INT, and DATE to WR_INT - case CalpontSystemCatalog::INT : - case CalpontSystemCatalog::DATE : - internalType = WriteEngine::WR_INT; - break; + // Map CHAR, VARCHAR, and CLOB to WR_CHAR + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CLOB: internalType = WriteEngine::WR_CHAR; break; - // Map FLOAT and UFLOAT to WR_FLOAT - case CalpontSystemCatalog::FLOAT : - case CalpontSystemCatalog::UFLOAT: - internalType = WriteEngine::WR_FLOAT; - break; + // Map UTINYINT to WR_UBYTE + case CalpontSystemCatalog::UTINYINT: internalType = WriteEngine::WR_UBYTE; break; - // Map BIGINT, DATETIME, TIMESTAMP, and TIME to WR_LONGLONG - case CalpontSystemCatalog::BIGINT : - case CalpontSystemCatalog::DATETIME : - case CalpontSystemCatalog::TIMESTAMP : - case CalpontSystemCatalog::TIME : - internalType = WriteEngine::WR_LONGLONG; - break; + // Map USMALLINT to WR_USHORT + case CalpontSystemCatalog::USMALLINT: internalType = WriteEngine::WR_USHORT; break; - // Map DOUBLE and UDOUBLE to WR_DOUBLE - case CalpontSystemCatalog::DOUBLE : - case CalpontSystemCatalog::UDOUBLE: - internalType = WriteEngine::WR_DOUBLE; - break; + // Map UMEDINT to WR_UMEDINT + case CalpontSystemCatalog::UMEDINT: internalType = WriteEngine::WR_UMEDINT; break; - // Map DECIMAL to applicable integer type - case CalpontSystemCatalog::DECIMAL : - case CalpontSystemCatalog::UDECIMAL : - { - switch (colWidth) - { - case 1 : - internalType = WriteEngine::WR_BYTE; - break; + // Map UINT to WR_UINT + case CalpontSystemCatalog::UINT: internalType = WriteEngine::WR_UINT; break; - case 2 : - internalType = WriteEngine::WR_SHORT; - break; + // Map UBIGINT, TIMESTAMP to WR_ULONGLONG + case CalpontSystemCatalog::UBIGINT: internalType = WriteEngine::WR_ULONGLONG; break; - case 4 : - internalType = WriteEngine::WR_INT; - break; - - case 8: - internalType = WriteEngine::WR_LONGLONG; - break; - - default: - internalType = WriteEngine::WR_BINARY; - break; - } - - break; - } - - // Map BLOB to WR_BLOB - case CalpontSystemCatalog::BLOB : - internalType = WriteEngine::WR_BLOB; - break; - - // Map TEXT to WR_TEXT - case CalpontSystemCatalog::TEXT : - internalType = WriteEngine::WR_TEXT; - break; - - // Map VARBINARY to WR_VARBINARY - case CalpontSystemCatalog::VARBINARY: - internalType = WriteEngine::WR_VARBINARY; - break; - - // Map CHAR, VARCHAR, and CLOB to WR_CHAR - case CalpontSystemCatalog::CHAR : - case CalpontSystemCatalog::VARCHAR : - case CalpontSystemCatalog::CLOB : - internalType = WriteEngine::WR_CHAR; - break; - - // Map UTINYINT to WR_UBYTE - case CalpontSystemCatalog::UTINYINT: - internalType = WriteEngine::WR_UBYTE; - break; - - // Map USMALLINT to WR_USHORT - case CalpontSystemCatalog::USMALLINT: - internalType = WriteEngine::WR_USHORT; - break; - - // Map UMEDINT to WR_UMEDINT - case CalpontSystemCatalog::UMEDINT: - internalType = WriteEngine::WR_UMEDINT; - break; - - // Map UINT to WR_UINT - case CalpontSystemCatalog::UINT: - internalType = WriteEngine::WR_UINT; - break; - - // Map UBIGINT, TIMESTAMP to WR_ULONGLONG - case CalpontSystemCatalog::UBIGINT: - internalType = WriteEngine::WR_ULONGLONG; - break; - - default: - internalType = WriteEngine::WR_CHAR; - break; - } + default: internalType = WriteEngine::WR_CHAR; break; + } } /******************************************************************************* @@ -628,97 +567,62 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, * none ******************************************************************************/ /* static */ -void Convertor::convertWEColType(ColType internalType, - CalpontSystemCatalog::ColDataType& dataType) +void Convertor::convertWEColType(ColType internalType, CalpontSystemCatalog::ColDataType& dataType) { - switch (internalType) - { - // Map BIT and TINYINT to WR_BYTE - case WriteEngine::WR_BYTE : - dataType = CalpontSystemCatalog::TINYINT; - break; + switch (internalType) + { + // Map BIT and TINYINT to WR_BYTE + case WriteEngine::WR_BYTE: dataType = CalpontSystemCatalog::TINYINT; break; - // Map SMALLINT to WR_SHORT - case WriteEngine::WR_SHORT : - dataType = CalpontSystemCatalog::SMALLINT; - break; + // Map SMALLINT to WR_SHORT + case WriteEngine::WR_SHORT: dataType = CalpontSystemCatalog::SMALLINT; break; - // Map MEDINT to WR_MEDINT - case WriteEngine::WR_MEDINT : - dataType = CalpontSystemCatalog::MEDINT; - break; + // Map MEDINT to WR_MEDINT + case WriteEngine::WR_MEDINT: dataType = CalpontSystemCatalog::MEDINT; break; - // Map INT, and DATE to WR_INT - case WriteEngine::WR_INT : - dataType = CalpontSystemCatalog::INT; - break; + // Map INT, and DATE to WR_INT + case WriteEngine::WR_INT: dataType = CalpontSystemCatalog::INT; break; - // Map FLOAT and UFLOAT to WR_FLOAT - case WriteEngine::WR_FLOAT: - dataType = CalpontSystemCatalog::FLOAT; - break; + // Map FLOAT and UFLOAT to WR_FLOAT + case WriteEngine::WR_FLOAT: dataType = CalpontSystemCatalog::FLOAT; break; - // Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG - case WriteEngine::WR_LONGLONG : - dataType = CalpontSystemCatalog::BIGINT; - break; + // Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG + case WriteEngine::WR_LONGLONG: dataType = CalpontSystemCatalog::BIGINT; break; - // Map DOUBLE and UDOUBLE to WR_DOUBLE - case WriteEngine::WR_DOUBLE : - dataType = CalpontSystemCatalog::DOUBLE; - break; + // Map DOUBLE and UDOUBLE to WR_DOUBLE + case WriteEngine::WR_DOUBLE: dataType = CalpontSystemCatalog::DOUBLE; break; - // Map BLOB to WR_BLOB - case WriteEngine::WR_BLOB : - dataType = CalpontSystemCatalog::BLOB; - break; + // Map BLOB to WR_BLOB + case WriteEngine::WR_BLOB: dataType = CalpontSystemCatalog::BLOB; break; - // Map TEXT to WR_TEXT - case WriteEngine::WR_TEXT : - dataType = CalpontSystemCatalog::TEXT; - break; + // Map TEXT to WR_TEXT + case WriteEngine::WR_TEXT: dataType = CalpontSystemCatalog::TEXT; break; - // Map VARBINARY to WR_VARBINARY - case WriteEngine::WR_VARBINARY: - dataType = CalpontSystemCatalog::VARBINARY; - break; + // Map VARBINARY to WR_VARBINARY + case WriteEngine::WR_VARBINARY: dataType = CalpontSystemCatalog::VARBINARY; break; - // Map CHAR, VARCHAR, and CLOB to WR_CHAR - case WriteEngine::WR_CHAR : - dataType = CalpontSystemCatalog::CHAR; - break; + // Map CHAR, VARCHAR, and CLOB to WR_CHAR + case WriteEngine::WR_CHAR: dataType = CalpontSystemCatalog::CHAR; break; - // Map UTINYINT to WR_UBYTE - case WriteEngine::WR_UBYTE: - dataType = CalpontSystemCatalog::UTINYINT; - break; + // Map UTINYINT to WR_UBYTE + case WriteEngine::WR_UBYTE: dataType = CalpontSystemCatalog::UTINYINT; break; - // Map USMALLINT to WR_USHORT - case WriteEngine::WR_USHORT: - dataType = CalpontSystemCatalog::USMALLINT; - break; + // Map USMALLINT to WR_USHORT + case WriteEngine::WR_USHORT: dataType = CalpontSystemCatalog::USMALLINT; break; - // Map UMEDINT to WR_UMEDINT - case WriteEngine::WR_UMEDINT: - dataType = CalpontSystemCatalog::UMEDINT; - break; + // Map UMEDINT to WR_UMEDINT + case WriteEngine::WR_UMEDINT: dataType = CalpontSystemCatalog::UMEDINT; break; - // Map UINT to WR_UINT - case WriteEngine::WR_UINT: - dataType = CalpontSystemCatalog::UINT; - break; + // Map UINT to WR_UINT + case WriteEngine::WR_UINT: dataType = CalpontSystemCatalog::UINT; break; - // Map UBIGINT and TIMESTAMP to WR_ULONGLONG - case WriteEngine::WR_ULONGLONG: - dataType = CalpontSystemCatalog::UBIGINT; - break; + // Map UBIGINT and TIMESTAMP to WR_ULONGLONG + case WriteEngine::WR_ULONGLONG: dataType = CalpontSystemCatalog::UBIGINT; break; - default: - dataType = CalpontSystemCatalog::CHAR; - break; - } + default: dataType = CalpontSystemCatalog::CHAR; break; + } } - + /******************************************************************************* * DESCRIPTION: * Convert curStruct from an interface-type struct to an internal-type @@ -731,151 +635,106 @@ void Convertor::convertWEColType(ColType internalType, /* static */ void Convertor::convertColType(ColStruct* curStruct) { - CalpontSystemCatalog::ColDataType dataType // This will be updated later, - = CalpontSystemCatalog::CHAR; // CHAR used only for initialization. - ColType* internalType = NULL; - bool bTokenFlag = false; - int* width = NULL; + CalpontSystemCatalog::ColDataType dataType // This will be updated later, + = CalpontSystemCatalog::CHAR; // CHAR used only for initialization. + ColType* internalType = NULL; + bool bTokenFlag = false; + int* width = NULL; - dataType = curStruct->colDataType; - internalType = &(curStruct->colType); - bTokenFlag = curStruct->tokenFlag; - width = &(curStruct->colWidth); + dataType = curStruct->colDataType; + internalType = &(curStruct->colType); + bTokenFlag = curStruct->tokenFlag; + width = &(curStruct->colWidth); - switch (dataType) + switch (dataType) + { + // Map BIT and TINYINT to WR_BYTE + case CalpontSystemCatalog::BIT: + case CalpontSystemCatalog::TINYINT: *internalType = WriteEngine::WR_BYTE; break; + + // Map SMALLINT to WR_SHORT + case CalpontSystemCatalog::SMALLINT: *internalType = WriteEngine::WR_SHORT; break; + + // Map MEDINT to WR_MEDINT + case CalpontSystemCatalog::MEDINT: *internalType = WriteEngine::WR_MEDINT; break; + + // Map INT, and DATE to WR_INT + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::DATE: *internalType = WriteEngine::WR_INT; break; + + // Map FLOAT and UFLOAT to WR_FLOAT + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: *internalType = WriteEngine::WR_FLOAT; break; + + // Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: + case CalpontSystemCatalog::TIME: *internalType = WriteEngine::WR_LONGLONG; break; + + // Map DOUBLE and UDOUBLE to WR_DOUBLE + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: *internalType = WriteEngine::WR_DOUBLE; break; + + // Map DECIMAL to applicable integer type + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - // Map BIT and TINYINT to WR_BYTE - case CalpontSystemCatalog::BIT : - case CalpontSystemCatalog::TINYINT : - *internalType = WriteEngine::WR_BYTE; - break; + switch (*width) + { + case 1: *internalType = WriteEngine::WR_BYTE; break; - // Map SMALLINT to WR_SHORT - case CalpontSystemCatalog::SMALLINT : - *internalType = WriteEngine::WR_SHORT; - break; + case 2: *internalType = WriteEngine::WR_SHORT; break; - // Map MEDINT to WR_MEDINT - case CalpontSystemCatalog::MEDINT : - *internalType = WriteEngine::WR_MEDINT; - break; + case 4: *internalType = WriteEngine::WR_INT; break; - // Map INT, and DATE to WR_INT - case CalpontSystemCatalog::INT : - case CalpontSystemCatalog::DATE : - *internalType = WriteEngine::WR_INT; - break; + case 8: *internalType = WriteEngine::WR_LONGLONG; break; - // Map FLOAT and UFLOAT to WR_FLOAT - case CalpontSystemCatalog::FLOAT : - case CalpontSystemCatalog::UFLOAT : - *internalType = WriteEngine::WR_FLOAT; - break; + default: *internalType = WriteEngine::WR_BINARY; break; + } - // Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG - case CalpontSystemCatalog::BIGINT : - case CalpontSystemCatalog::DATETIME : - case CalpontSystemCatalog::TIMESTAMP : - case CalpontSystemCatalog::TIME : - *internalType = WriteEngine::WR_LONGLONG; - break; - - // Map DOUBLE and UDOUBLE to WR_DOUBLE - case CalpontSystemCatalog::DOUBLE : - case CalpontSystemCatalog::UDOUBLE : - *internalType = WriteEngine::WR_DOUBLE; - break; - - // Map DECIMAL to applicable integer type - case CalpontSystemCatalog::DECIMAL : - case CalpontSystemCatalog::UDECIMAL : - { - switch (*width) - { - case 1 : - *internalType = WriteEngine::WR_BYTE; - break; - - case 2 : - *internalType = WriteEngine::WR_SHORT; - break; - - case 4 : - *internalType = WriteEngine::WR_INT; - break; - - case 8: - *internalType = WriteEngine::WR_LONGLONG; - break; - - default: - *internalType = WriteEngine::WR_BINARY; - break; - } - - break; - } - - // Map BLOB to WR_BLOB - case CalpontSystemCatalog::BLOB : - *internalType = WriteEngine::WR_BLOB; - break; - - // Map TEXT to WR_TEXT - case CalpontSystemCatalog::TEXT : - *internalType = WriteEngine::WR_TEXT; - break; - - // Map VARBINARY to WR_VARBINARY - case CalpontSystemCatalog::VARBINARY: - *internalType = WriteEngine::WR_VARBINARY; - break; - - // Map CHAR, VARCHAR, and CLOB to WR_CHAR - case CalpontSystemCatalog::CHAR : - case CalpontSystemCatalog::VARCHAR : - case CalpontSystemCatalog::CLOB : - *internalType = WriteEngine::WR_CHAR; - break; - - // Map UTINYINT to WR_UBYTE - case CalpontSystemCatalog::UTINYINT: - *internalType = WriteEngine::WR_UBYTE; - break; - - // Map USMALLINT to WR_USHORT - case CalpontSystemCatalog::USMALLINT: - *internalType = WriteEngine::WR_USHORT; - break; - - // Map UMEDINT to WR_UMEDINT - case CalpontSystemCatalog::UMEDINT: - *internalType = WriteEngine::WR_UMEDINT; - break; - - // Map UINT to WR_UINT - case CalpontSystemCatalog::UINT: - *internalType = WriteEngine::WR_UINT; - break; - - // Map UBIGINT to WR_ULONGLONG - case CalpontSystemCatalog::UBIGINT: - *internalType = WriteEngine::WR_ULONGLONG; - break; - - default: - *internalType = WriteEngine::WR_CHAR; - break; + break; } - if (bTokenFlag) // token overwrite any other types - *internalType = WriteEngine::WR_TOKEN; + // Map BLOB to WR_BLOB + case CalpontSystemCatalog::BLOB: *internalType = WriteEngine::WR_BLOB; break; - // check whether width is in sync with the requirement - *width = getCorrectRowWidth(dataType, *width); + // Map TEXT to WR_TEXT + case CalpontSystemCatalog::TEXT: *internalType = WriteEngine::WR_TEXT; break; + + // Map VARBINARY to WR_VARBINARY + case CalpontSystemCatalog::VARBINARY: *internalType = WriteEngine::WR_VARBINARY; break; + + // Map CHAR, VARCHAR, and CLOB to WR_CHAR + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::CLOB: *internalType = WriteEngine::WR_CHAR; break; + + // Map UTINYINT to WR_UBYTE + case CalpontSystemCatalog::UTINYINT: *internalType = WriteEngine::WR_UBYTE; break; + + // Map USMALLINT to WR_USHORT + case CalpontSystemCatalog::USMALLINT: *internalType = WriteEngine::WR_USHORT; break; + + // Map UMEDINT to WR_UMEDINT + case CalpontSystemCatalog::UMEDINT: *internalType = WriteEngine::WR_UMEDINT; break; + + // Map UINT to WR_UINT + case CalpontSystemCatalog::UINT: *internalType = WriteEngine::WR_UINT; break; + + // Map UBIGINT to WR_ULONGLONG + case CalpontSystemCatalog::UBIGINT: *internalType = WriteEngine::WR_ULONGLONG; break; + + default: *internalType = WriteEngine::WR_CHAR; break; + } + + if (bTokenFlag) // token overwrite any other types + *internalType = WriteEngine::WR_TOKEN; + + // check whether width is in sync with the requirement + *width = getCorrectRowWidth(dataType, *width); } - /******************************************************************************* * DESCRIPTION: * Get the correct width for a row @@ -888,84 +747,68 @@ void Convertor::convertColType(ColStruct* curStruct) /* static */ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, int width) { - int offset, newWidth = 4; + int offset, newWidth = 4; - switch (dataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::UTINYINT: - newWidth = 1; - break; + switch (dataType) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::UTINYINT: newWidth = 1; break; - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::USMALLINT: - newWidth = 2; - break; + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::USMALLINT: newWidth = 2; break; - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - newWidth = 4; - break; + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: newWidth = 4; break; - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::UBIGINT: - newWidth = 8; - break; + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::UBIGINT: newWidth = 8; break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - newWidth = 4; - break; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: newWidth = 4; break; - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - newWidth = 8; - break; + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: newWidth = 8; break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (width == 1) - newWidth = 1; - else if (width == 2) - newWidth = 2; - else if (width <= 4) - newWidth = 4; - else if (width <= 8) - newWidth = 8; - else - newWidth = 16; - break; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (width == 1) + newWidth = 1; + else if (width == 2) + newWidth = 2; + else if (width <= 4) + newWidth = 4; + else if (width <= 8) + newWidth = 8; + else + newWidth = 16; + break; - case CalpontSystemCatalog::DATE: - newWidth = 4; - break; + case CalpontSystemCatalog::DATE: newWidth = 4; break; - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::TIMESTAMP: - newWidth = 8; - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: // treat same as varchar for now - default: - offset = (dataType == CalpontSystemCatalog::VARCHAR) ? -1 : 0; - newWidth = 1; + case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: newWidth = 8; break; - if (width == (2 + offset)) - newWidth = 2; - else if (width >= (3 + offset) && width <= (4 + offset)) - newWidth = 4; - else if (width >= (5 + offset)) - newWidth = 8; + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: // treat same as varchar for now + default: + offset = (dataType == CalpontSystemCatalog::VARCHAR) ? -1 : 0; + newWidth = 1; - break; - } + if (width == (2 + offset)) + newWidth = 2; + else if (width >= (3 + offset) && width <= (4 + offset)) + newWidth = 4; + else if (width >= (5 + offset)) + newWidth = 8; - return newWidth; + break; + } + + return newWidth; } /******************************************************************************* @@ -1009,59 +852,51 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in * or equal to the buffer's length argument, length is too small ******************************************************************************/ /*static*/ -int Convertor::dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment, - dmFilePathArgs_t* pArgs) +int Convertor::dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment, dmFilePathArgs_t* pArgs) { - int32_t retCodes[6]; - // A, B, C directories. - for (uint32_t i = 0, shift = 24, mask = 0xff000000; i < 3; - ++i, shift -= 8, mask >>= 8) - { - retCodes[i] = _doDir(pArgs->pathPart[i].pName, pArgs->pathPart[i].len, - (uint32_t)((oid & mask) >> shift)); - } + int32_t retCodes[6]; + // A, B, C directories. + for (uint32_t i = 0, shift = 24, mask = 0xff000000; i < 3; ++i, shift -= 8, mask >>= 8) + { + retCodes[i] = _doDir(pArgs->pathPart[i].pName, pArgs->pathPart[i].len, (uint32_t)((oid & mask) >> shift)); + } - // include partition and seg num in the file path if they are present - if (pArgs->pathPart[3].len > 0) - { - // D directory. - retCodes[3] = _doDir(pArgs->pathPart[3].pName, pArgs->pathPart[3].len, - (uint32_t)(oid & 0x000000ff)); + // include partition and seg num in the file path if they are present + if (pArgs->pathPart[3].len > 0) + { + // D directory. + retCodes[3] = _doDir(pArgs->pathPart[3].pName, pArgs->pathPart[3].len, (uint32_t)(oid & 0x000000ff)); - // E directory - partition. - retCodes[4] = _doDir(pArgs->pathPart[4].pName, pArgs->pathPart[4].len, - partition); - // File - segment. - retCodes[5] = - _doFile(pArgs->pathPart[5].pName, pArgs->pathPart[5].len, segment); + // E directory - partition. + retCodes[4] = _doDir(pArgs->pathPart[4].pName, pArgs->pathPart[4].len, partition); + // File - segment. + retCodes[5] = _doFile(pArgs->pathPart[5].pName, pArgs->pathPart[5].len, segment); - // D.rc < 0 || E.rc < 0 - if ((retCodes[3] < 0) || (retCodes[4] < 0)) - return -1; + // D.rc < 0 || E.rc < 0 + if ((retCodes[3] < 0) || (retCodes[4] < 0)) + return -1; - // D.rc >= A.len || E.rc >= A.len - if ((retCodes[3] >= pArgs->pathPart[0].len) || - (retCodes[4] >= pArgs->pathPart[0].len)) - return -1; - } - else - { - retCodes[5] = _doFile(pArgs->pathPart[5].pName, pArgs->pathPart[5].len, - (uint32_t)(oid & 0x000000ff)); - } + // D.rc >= A.len || E.rc >= A.len + if ((retCodes[3] >= pArgs->pathPart[0].len) || (retCodes[4] >= pArgs->pathPart[0].len)) + return -1; + } + else + { + retCodes[5] = _doFile(pArgs->pathPart[5].pName, pArgs->pathPart[5].len, (uint32_t)(oid & 0x000000ff)); + } - // A.rc < 0 || B.rc < 0 || C.rc < 0 - // A.rc >= A.len || B.rc >= B.len || C.rc >= C.len - for (uint32_t i = 0; i < 3; ++i) - { - if ((retCodes[i] < 0) || (retCodes[i] >= pArgs->pathPart[i].len)) - return -1; - } - // F.rc < 0 || F.rc >= F.len - if ((retCodes[5] < 0) || (retCodes[5] >= pArgs->pathPart[5].len)) - return -1; + // A.rc < 0 || B.rc < 0 || C.rc < 0 + // A.rc >= A.len || B.rc >= B.len || C.rc >= C.len + for (uint32_t i = 0; i < 3; ++i) + { + if ((retCodes[i] < 0) || (retCodes[i] >= pArgs->pathPart[i].len)) + return -1; + } + // F.rc < 0 || F.rc >= F.len + if ((retCodes[5] < 0) || (retCodes[5] >= pArgs->pathPart[5].len)) + return -1; - return 0; + return 0; } /******************************************************************************* @@ -1078,32 +913,31 @@ int Convertor::dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment, * RETURN: * return 0 if everything went OK. -1 if an error occured. ******************************************************************************/ -int32_t Convertor::dmFPath2Oid(const dmFilePathArgs_t& pArgs, uint32_t& oid, - uint32_t& partition, uint32_t& segment) +int32_t Convertor::dmFPath2Oid(const dmFilePathArgs_t& pArgs, uint32_t& oid, uint32_t& partition, + uint32_t& segment) { - oid = 0; - // A, B, C, D - directories. - for (uint32_t i = 0, shift = 24; i < 4; ++i, shift -= 8) - { - uint32_t val = 0; - auto rc = _fromDir(pArgs.pathPart[i].pName, val); - if (rc == -1) - return rc; - - oid |= val << shift; - } - - // Partition. - auto rc = _fromDir(pArgs.pathPart[4].pName, partition); + oid = 0; + // A, B, C, D - directories. + for (uint32_t i = 0, shift = 24; i < 4; ++i, shift -= 8) + { + uint32_t val = 0; + auto rc = _fromDir(pArgs.pathPart[i].pName, val); if (rc == -1) - return rc; + return rc; - // Segment. - rc = _fromFile(pArgs.pathPart[5].pName, segment); - if (rc == -1) - return rc; + oid |= val << shift; + } - return 0; + // Partition. + auto rc = _fromDir(pArgs.pathPart[4].pName, partition); + if (rc == -1) + return rc; + + // Segment. + rc = _fromFile(pArgs.pathPart[5].pName, segment); + if (rc == -1) + return rc; + + return 0; } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_convertor.h b/writeengine/shared/we_convertor.h index 78c926f7c..e381b8710 100644 --- a/writeengine/shared/we_convertor.h +++ b/writeengine/shared/we_convertor.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_convertor.h 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_convertor.h 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_CONVERTOR_H_ @@ -45,109 +45,105 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class Convertor */ class Convertor { -public: - /** - * @brief Default Constructor - */ - Convertor() {} + public: + /** + * @brief Default Constructor + */ + Convertor() + { + } - /** - * @brief Destructor - */ - ~Convertor() {} + /** + * @brief Destructor + */ + ~Convertor() + { + } - /** - * @brief Get date/time string based on current date and time - */ - EXPORT static const std::string getTimeStr(); + /** + * @brief Get date/time string based on current date and time + */ + EXPORT static const std::string getTimeStr(); - /** - * @brief Convert specified integer value to a string - * - * @param val Integer value to be converted to a string - */ - EXPORT static const std::string int2Str(int val); + /** + * @brief Convert specified integer value to a string + * + * @param val Integer value to be converted to a string + */ + EXPORT static const std::string int2Str(int val); - /** - * @brief Convert an oid to a full file name (with partition and segment - * being included in the filename). This is used for all column and - * dictionary store db files. If dealing with a version buffer file, - * a partition and segment number of 0 should be used. - */ - EXPORT static int oid2FileName(FID fid, char* fullFileName, - char dbDirName[][MAX_DB_DIR_NAME_SIZE], - uint32_t partition, uint16_t segment); + /** + * @brief Convert an oid to a full file name (with partition and segment + * being included in the filename). This is used for all column and + * dictionary store db files. If dealing with a version buffer file, + * a partition and segment number of 0 should be used. + */ + EXPORT static int oid2FileName(FID fid, char* fullFileName, char dbDirName[][MAX_DB_DIR_NAME_SIZE], + uint32_t partition, uint16_t segment); - EXPORT static int fileName2Oid(const std::string &fullFileName, - uint32_t &oid, uint32_t &partition, - uint32_t &segment); - /** - * @brief Convert specified errno to associated error msg string - * - * @param errNum System errno to be converted. - * @param errString Error msg string associated with the specified errno. - */ - EXPORT static void mapErrnoToString(int errNum, std::string& errString); + EXPORT static int fileName2Oid(const std::string& fullFileName, uint32_t& oid, uint32_t& partition, + uint32_t& segment); + /** + * @brief Convert specified errno to associated error msg string + * + * @param errNum System errno to be converted. + * @param errString Error msg string associated with the specified errno. + */ + EXPORT static void mapErrnoToString(int errNum, std::string& errString); - /** - * @brief Convert specified ColDataType to internal storage type (ColType) - * - * @param dataType Interface data-type - * @param internalType Internal data-type used for storing - */ - //BUG931 - EXPORT static void convertColType(execplan::CalpontSystemCatalog::ColDataType dataType, - int colWidth, ColType& internalType, - bool isToken = false); - /** - * @brief Convert specified internal storage type (ColType) to - * ColDataType - * - * @param internalType Internal data-type used for storing - * @param dataType Interface data-type - */ - EXPORT static void convertWEColType(ColType internalType, - execplan::CalpontSystemCatalog::ColDataType& dataType); + /** + * @brief Convert specified ColDataType to internal storage type (ColType) + * + * @param dataType Interface data-type + * @param internalType Internal data-type used for storing + */ + // BUG931 + EXPORT static void convertColType(execplan::CalpontSystemCatalog::ColDataType dataType, int colWidth, + ColType& internalType, bool isToken = false); + /** + * @brief Convert specified internal storage type (ColType) to + * ColDataType + * + * @param internalType Internal data-type used for storing + * @param dataType Interface data-type + */ + EXPORT static void convertWEColType(ColType internalType, + execplan::CalpontSystemCatalog::ColDataType& dataType); - /** - * @brief Convert interface column type to a internal column type. - * curStruct is interpreted as a ColStruct. - */ - EXPORT static void convertColType(ColStruct* curStruct); + /** + * @brief Convert interface column type to a internal column type. + * curStruct is interpreted as a ColStruct. + */ + EXPORT static void convertColType(ColStruct* curStruct); - /* - * @brief Get the correct width for a row - */ - EXPORT static int getCorrectRowWidth( execplan::CalpontSystemCatalog::ColDataType dataType, int width ); + /* + * @brief Get the correct width for a row + */ + EXPORT static int getCorrectRowWidth(execplan::CalpontSystemCatalog::ColDataType dataType, int width); - /* - * @brief Convert a Decimal string to it's equivalent integer value. - * errno can be checked upon return to see if input value was - * out of range (ERANGE). - * - * field decimal string to be converted - * fieldLengh length of "field" in bytes - * scale decimal scale to be applied to value - */ - EXPORT static long long convertDecimalString ( const char* field, - int fieldLength, - int scale ); + /* + * @brief Convert a Decimal string to it's equivalent integer value. + * errno can be checked upon return to see if input value was + * out of range (ERANGE). + * + * field decimal string to be converted + * fieldLengh length of "field" in bytes + * scale decimal scale to be applied to value + */ + EXPORT static long long convertDecimalString(const char* field, int fieldLength, int scale); -private: - - struct dmFilePathArgs_t; - static int dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment, - dmFilePathArgs_t* pArgs); - static int32_t dmFPath2Oid(const dmFilePathArgs_t& pArgs, uint32_t& oid, - uint32_t& partition, uint32_t& segment); + private: + struct dmFilePathArgs_t; + static int dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment, dmFilePathArgs_t* pArgs); + static int32_t dmFPath2Oid(const dmFilePathArgs_t& pArgs, uint32_t& oid, uint32_t& partition, + uint32_t& segment); }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_CONVERTOR_H_ +#endif // _WE_CONVERTOR_H_ diff --git a/writeengine/shared/we_dbfileop.cpp b/writeengine/shared/we_dbfileop.cpp index bfc7107f0..0d179821d 100644 --- a/writeengine/shared/we_dbfileop.cpp +++ b/writeengine/shared/we_dbfileop.cpp @@ -36,18 +36,19 @@ using namespace BRM; namespace WriteEngine { - /** * Constructor */ DbFileOp::DbFileOp() : m_chunkManager(NULL) -{} +{ +} /** * Default Destructor */ DbFileOp::~DbFileOp() -{} +{ +} /*********************************************************** * DESCRIPTION: @@ -59,24 +60,20 @@ DbFileOp::~DbFileOp() ***********************************************************/ int DbFileOp::flushCache() { - BlockBuffer* curBuf; + BlockBuffer* curBuf; - if ( !Cache::getUseCache() ) - return NO_ERROR; - - for ( CacheMapIt it = Cache::m_writeList->begin(); - it != Cache::m_writeList->end(); it++ ) - { - curBuf = it->second; - RETURN_ON_ERROR( writeDBFile( (*curBuf).cb.file.pFile, - (*curBuf).block.data, - (*curBuf).block.lbid ) ); - } - - RETURN_ON_ERROR( Cache::flushCache() ); + if (!Cache::getUseCache()) return NO_ERROR; -} + for (CacheMapIt it = Cache::m_writeList->begin(); it != Cache::m_writeList->end(); it++) + { + curBuf = it->second; + RETURN_ON_ERROR(writeDBFile((*curBuf).cb.file.pFile, (*curBuf).block.data, (*curBuf).block.lbid)); + } + + RETURN_ON_ERROR(Cache::flushCache()); + return NO_ERROR; +} /*********************************************************** * DESCRIPTION: @@ -93,14 +90,13 @@ int DbFileOp::flushCache() * RETURN: * none ***********************************************************/ -void DbFileOp::getSubBlockEntry( unsigned char* blockBuf, - const int sbid, const int entryNo, - const int width, void* pStruct ) +void DbFileOp::getSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width, + void* pStruct) { - unsigned char* pBlock; + unsigned char* pBlock; - pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY; - memcpy( pStruct, pBlock, width ); + pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY; + memcpy(pStruct, pBlock, width); } /*********************************************************** @@ -114,89 +110,80 @@ void DbFileOp::getSubBlockEntry( unsigned char* blockBuf, * NO_ERROR if success * other number if something wrong ***********************************************************/ -int DbFileOp::readDBFile( IDBDataFile* pFile, - unsigned char* readBuf, - const uint64_t lbid, - const bool isFbo ) +int DbFileOp::readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, const bool isFbo) { - long long fboOffset = 0; + long long fboOffset = 0; - if ( !isFbo ) - { - RETURN_ON_ERROR( setFileOffsetBlock( pFile, lbid ) ); - } - else - { - fboOffset = (lbid) * (long)BYTE_PER_BLOCK; - RETURN_ON_ERROR( setFileOffset( pFile, fboOffset ) ); - } + if (!isFbo) + { + RETURN_ON_ERROR(setFileOffsetBlock(pFile, lbid)); + } + else + { + fboOffset = (lbid) * (long)BYTE_PER_BLOCK; + RETURN_ON_ERROR(setFileOffset(pFile, fboOffset)); + } - return readFile( pFile, readBuf, BYTE_PER_BLOCK ); + return readFile(pFile, readBuf, BYTE_PER_BLOCK); } -int DbFileOp::readDBFile( IDBDataFile* pFile, - DataBlock* block, - const uint64_t lbid, - const bool isFbo ) +int DbFileOp::readDBFile(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const bool isFbo) { - block->dirty = false; - block->no = lbid; + block->dirty = false; + block->no = lbid; - Stats::incIoBlockRead(); + Stats::incIoBlockRead(); - return readDBFile( pFile, block->data, lbid, isFbo ); + return readDBFile(pFile, block->data, lbid, isFbo); } -int DbFileOp::readDBFile( CommBlock& cb, - unsigned char* readBuf, - const uint64_t lbid ) +int DbFileOp::readDBFile(CommBlock& cb, unsigned char* readBuf, const uint64_t lbid) { - CacheKey key; + CacheKey key; - if ( Cache::getUseCache() ) + if (Cache::getUseCache()) + { + if (Cache::cacheKeyExist(cb.file.oid, lbid)) { - if ( Cache::cacheKeyExist( cb.file.oid, lbid ) ) - { - key = Cache::getCacheKey( cb.file.oid, lbid ); - RETURN_ON_ERROR( Cache::loadCacheBlock( key, readBuf ) ); - return NO_ERROR; - } + key = Cache::getCacheKey(cb.file.oid, lbid); + RETURN_ON_ERROR(Cache::loadCacheBlock(key, readBuf)); + return NO_ERROR; + } + } + + RETURN_ON_ERROR(readDBFile(cb.file.pFile, readBuf, lbid)); + + if (Cache::getUseCache()) + { + int fbo = lbid; + + uint16_t dbRoot; + uint32_t partition; + uint16_t segment; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getFboOffset(lbid, dbRoot, partition, segment, fbo)); + + if (Cache::getListSize(FREE_LIST) == 0) + { + if (isDebug(DEBUG_1)) + { + printf("\nBefore flushing cache "); + Cache::printCacheList(); + } + + // flush cache to give up more space + RETURN_ON_ERROR(flushCache()); + + if (isDebug(DEBUG_1)) + { + printf("\nAfter flushing cache "); + Cache::printCacheList(); + } } - RETURN_ON_ERROR( readDBFile( cb.file.pFile, readBuf, lbid ) ); + RETURN_ON_ERROR(Cache::insertLRUList(cb, lbid, fbo, readBuf)); + } - if ( Cache::getUseCache() ) - { - int fbo = lbid; - - uint16_t dbRoot; - uint32_t partition; - uint16_t segment; - RETURN_ON_ERROR( BRMWrapper::getInstance()->getFboOffset( - lbid, dbRoot, partition, segment, fbo ) ); - - if ( Cache::getListSize( FREE_LIST ) == 0 ) - { - if ( isDebug( DEBUG_1 ) ) - { - printf( "\nBefore flushing cache " ); - Cache::printCacheList(); - } - - // flush cache to give up more space - RETURN_ON_ERROR( flushCache() ); - - if ( isDebug( DEBUG_1 ) ) - { - printf( "\nAfter flushing cache " ); - Cache::printCacheList(); - } - } - - RETURN_ON_ERROR( Cache::insertLRUList( cb, lbid, fbo, readBuf ) ); - } - - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -217,30 +204,24 @@ int DbFileOp::readDBFile( CommBlock& cb, * NO_ERROR if success * other number if something wrong ***********************************************************/ -int DbFileOp::readSubBlockEntry( IDBDataFile* pFile, DataBlock* block, - const uint64_t lbid, const int sbid, - const int entryNo, const int width, - void* pStruct ) +int DbFileOp::readSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct) { - RETURN_ON_ERROR( readDBFile( pFile, block->data, lbid ) ); - getSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); + RETURN_ON_ERROR(readDBFile(pFile, block->data, lbid)); + getSubBlockEntry(block->data, sbid, entryNo, width, pStruct); - return NO_ERROR; + return NO_ERROR; } - -int DbFileOp::readSubBlockEntry( CommBlock& cb, DataBlock* block, - const uint64_t lbid, const int sbid, - const int entryNo, const int width, - void* pStruct ) +int DbFileOp::readSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct) { - RETURN_ON_ERROR( readDBFile( cb, block->data, lbid ) ); - getSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); + RETURN_ON_ERROR(readDBFile(cb, block->data, lbid)); + getSubBlockEntry(block->data, sbid, entryNo, width, pStruct); - return NO_ERROR; + return NO_ERROR; } - /*********************************************************** * DESCRIPTION: * Set an entry within a sub block @@ -256,14 +237,13 @@ int DbFileOp::readSubBlockEntry( CommBlock& cb, DataBlock* block, * RETURN: * none ***********************************************************/ -void DbFileOp::setSubBlockEntry( unsigned char* blockBuf, const int sbid, - const int entryNo, const int width, - const void* pStruct ) +void DbFileOp::setSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width, + const void* pStruct) { - unsigned char* pBlock; + unsigned char* pBlock; - pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY; - memcpy( pBlock, pStruct, width ); + pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY; + memcpy(pBlock, pStruct, width); } /*********************************************************** @@ -278,47 +258,45 @@ void DbFileOp::setSubBlockEntry( unsigned char* blockBuf, const int sbid, * NO_ERROR if success * other number if something wrong ***********************************************************/ -int DbFileOp::writeDBFile( CommBlock& cb, const unsigned char* writeBuf, - const uint64_t lbid, const int numOfBlock ) +int DbFileOp::writeDBFile(CommBlock& cb, const unsigned char* writeBuf, const uint64_t lbid, + const int numOfBlock) { - CacheKey key; - int ret; - if ( Cache::getUseCache() ) + CacheKey key; + int ret; + if (Cache::getUseCache()) + { + if (Cache::cacheKeyExist(cb.file.oid, lbid)) { - if ( Cache::cacheKeyExist( cb.file.oid, lbid ) ) - { - key = Cache::getCacheKey( cb.file.oid, lbid ); - RETURN_ON_ERROR( Cache::modifyCacheBlock( key, writeBuf ) ); - return NO_ERROR; - } + key = Cache::getCacheKey(cb.file.oid, lbid); + RETURN_ON_ERROR(Cache::modifyCacheBlock(key, writeBuf)); + return NO_ERROR; } + } - if (BRMWrapper::getUseVb()) - { - RETURN_ON_ERROR( writeVB( cb.file.pFile, cb.file.oid, lbid ) ); - } + if (BRMWrapper::getUseVb()) + { + RETURN_ON_ERROR(writeVB(cb.file.pFile, cb.file.oid, lbid)); + } - ret = writeDBFile( cb.file.pFile, writeBuf, lbid, numOfBlock ); + ret = writeDBFile(cb.file.pFile, writeBuf, lbid, numOfBlock); - if (BRMWrapper::getUseVb()) - { - LBIDRange_v ranges; - LBIDRange range; - range.start = lbid; - range.size = 1; - ranges.push_back(range); - BRMWrapper::getInstance()->writeVBEnd(getTransId(), ranges); - } + if (BRMWrapper::getUseVb()) + { + LBIDRange_v ranges; + LBIDRange range; + range.start = lbid; + range.size = 1; + ranges.push_back(range); + BRMWrapper::getInstance()->writeVBEnd(getTransId(), ranges); + } - return ret; + return ret; } -int DbFileOp::writeDBFileNoVBCache(CommBlock& cb, - const unsigned char* writeBuf, - const int fbo, +int DbFileOp::writeDBFileNoVBCache(CommBlock& cb, const unsigned char* writeBuf, const int fbo, const int numOfBlock) { - return writeDBFileNoVBCache( cb.file.pFile, writeBuf, fbo, numOfBlock ); + return writeDBFileNoVBCache(cb.file.pFile, writeBuf, fbo, numOfBlock); } /*********************************************************** @@ -326,65 +304,63 @@ int DbFileOp::writeDBFileNoVBCache(CommBlock& cb, * Core function for writing data w/o using VB cache * (bulk load dictionary store inserts) ***********************************************************/ -int DbFileOp::writeDBFileNoVBCache( IDBDataFile* pFile, - const unsigned char* writeBuf, - const int fbo, - const int numOfBlock ) +int DbFileOp::writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo, + const int numOfBlock) { #ifdef PROFILE - // This function is only used by bulk load for dictionary store files, - // so we log as such. - Stats::startParseEvent(WE_STATS_WRITE_DCT); + // This function is only used by bulk load for dictionary store files, + // so we log as such. + Stats::startParseEvent(WE_STATS_WRITE_DCT); #endif - for ( int i = 0; i < numOfBlock; i++ ) - { - Stats::incIoBlockWrite(); - RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) ); - } + for (int i = 0; i < numOfBlock; i++) + { + Stats::incIoBlockWrite(); + RETURN_ON_ERROR(writeFile(pFile, writeBuf, BYTE_PER_BLOCK)); + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_WRITE_DCT); + Stats::stopParseEvent(WE_STATS_WRITE_DCT); #endif - return NO_ERROR; + return NO_ERROR; } /*********************************************************** * DESCRIPTION: * Core function for writing data using VB cache ***********************************************************/ -int DbFileOp::writeDBFile( IDBDataFile* pFile, const unsigned char* writeBuf, - const uint64_t lbid, const int numOfBlock ) +int DbFileOp::writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid, + const int numOfBlock) { - RETURN_ON_ERROR( setFileOffsetBlock( pFile, lbid ) ); + RETURN_ON_ERROR(setFileOffsetBlock(pFile, lbid)); - for ( int i = 0; i < numOfBlock; i++ ) - { - Stats::incIoBlockWrite(); - RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) ); - } + for (int i = 0; i < numOfBlock; i++) + { + Stats::incIoBlockWrite(); + RETURN_ON_ERROR(writeFile(pFile, writeBuf, BYTE_PER_BLOCK)); + } - return NO_ERROR; + return NO_ERROR; } // just don't have a good solution to consolidate with above functions // Note: This is used with absolute FBO, no lbid involved -int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf, - const uint64_t fbo, const int numOfBlock ) +int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo, + const int numOfBlock) { - long long fboOffset = 0; + long long fboOffset = 0; - fboOffset = (fbo) * (long)BYTE_PER_BLOCK; - RETURN_ON_ERROR( setFileOffset( pFile, fboOffset ) ); + fboOffset = (fbo) * (long)BYTE_PER_BLOCK; + RETURN_ON_ERROR(setFileOffset(pFile, fboOffset)); - for ( int i = 0; i < numOfBlock; i++ ) - { - Stats::incIoBlockWrite(); - RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) ); - } + for (int i = 0; i < numOfBlock; i++) + { + Stats::incIoBlockWrite(); + RETURN_ON_ERROR(writeFile(pFile, writeBuf, BYTE_PER_BLOCK)); + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -405,26 +381,22 @@ int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf, * NO_ERROR if success * other number if something wrong ***********************************************************/ -int DbFileOp::writeSubBlockEntry( IDBDataFile* pFile, DataBlock* block, - const uint64_t lbid, const int sbid, - const int entryNo, const int width, - void* pStruct ) +int DbFileOp::writeSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct) { - setSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); - block->dirty = false; + setSubBlockEntry(block->data, sbid, entryNo, width, pStruct); + block->dirty = false; - return writeDBFile( pFile, block->data, lbid ); + return writeDBFile(pFile, block->data, lbid); } -int DbFileOp::writeSubBlockEntry( CommBlock& cb, DataBlock* block, - const uint64_t lbid, const int sbid, - const int entryNo, const int width, - void* pStruct ) +int DbFileOp::writeSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct) { - setSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); - block->dirty = false; + setSubBlockEntry(block->data, sbid, entryNo, width, pStruct); + block->dirty = false; - return writeDBFile( cb, block->data, lbid ); + return writeDBFile(cb, block->data, lbid); } /*********************************************************** @@ -437,89 +409,77 @@ int DbFileOp::writeSubBlockEntry( CommBlock& cb, DataBlock* block, * NO_ERROR if success * other number if something wrong ***********************************************************/ -int DbFileOp::writeVB( IDBDataFile* pFile, const OID oid, const uint64_t lbid ) +int DbFileOp::writeVB(IDBDataFile* pFile, const OID oid, const uint64_t lbid) { - if ( !BRMWrapper::getUseVb() ) - return NO_ERROR; - - int rc; - const TxnID transId = getTransId(); - - if (transId != ((TxnID)INVALID_NUM)) - { - rc = BRMWrapper::getInstance()->writeVB( pFile, - (VER_t)transId, - oid, lbid, this ); -//@Bug 4671. The error is already logged by worker node. - /* if (rc != NO_ERROR) - { - char msg[2048]; - snprintf(msg, 2048, - "we_dbfileop->BRMWrapper::getInstance()->writeVB " - "transId %i oid %i lbid " - #if __LP64__ - "%lu" - #else - "%llu" - #endif - " Error Code %i", transId, oid, lbid, rc); - puts(msg); - { - logging::MessageLog ml(logging::LoggingID(19)); - logging::Message m; - logging::Message::Args args; - args.add(msg); - m.format(args); - ml.logCriticalMessage(m); - } - return rc; - } */ - return rc; - } - + if (!BRMWrapper::getUseVb()) return NO_ERROR; + + int rc; + const TxnID transId = getTransId(); + + if (transId != ((TxnID)INVALID_NUM)) + { + rc = BRMWrapper::getInstance()->writeVB(pFile, (VER_t)transId, oid, lbid, this); + //@Bug 4671. The error is already logged by worker node. + /* if (rc != NO_ERROR) + { + char msg[2048]; + snprintf(msg, 2048, + "we_dbfileop->BRMWrapper::getInstance()->writeVB " + "transId %i oid %i lbid " + #if __LP64__ + "%lu" + #else + "%llu" + #endif + " Error Code %i", transId, oid, lbid, rc); + puts(msg); + { + logging::MessageLog ml(logging::LoggingID(19)); + logging::Message m; + logging::Message::Args args; + args.add(msg); + m.format(args); + ml.logCriticalMessage(m); + } + return rc; + } */ + return rc; + } + + return NO_ERROR; } -int DbFileOp::readDbBlocks(IDBDataFile* pFile, - unsigned char* readBuf, - uint64_t fbo, - size_t n) +int DbFileOp::readDbBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n) { - if (m_chunkManager) - { - return m_chunkManager->readBlocks(pFile, readBuf, fbo, n); - } + if (m_chunkManager) + { + return m_chunkManager->readBlocks(pFile, readBuf, fbo, n); + } - if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR) - return -1; + if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR) + return -1; - return pFile->read(readBuf, BYTE_PER_BLOCK * n) / BYTE_PER_BLOCK; + return pFile->read(readBuf, BYTE_PER_BLOCK * n) / BYTE_PER_BLOCK; } int DbFileOp::restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo) { - if (m_chunkManager) - return m_chunkManager->restoreBlock(pFile, writeBuf, fbo); + if (m_chunkManager) + return m_chunkManager->restoreBlock(pFile, writeBuf, fbo); - if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR) - return -1; + if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR) + return -1; - return pFile->write(writeBuf, BYTE_PER_BLOCK); + return pFile->write(writeBuf, BYTE_PER_BLOCK); } // @bug 5572 - HDFS usage: add *.tmp file backup flag IDBDataFile* DbFileOp::getFilePtr(const Column& column, bool useTmpSuffix) { - string filename; - return m_chunkManager->getFilePtr(column, - column.dataFile.fDbRoot, - column.dataFile.fPartition, - column.dataFile.fSegment, - filename, - "r+b", - column.colWidth, - useTmpSuffix); + string filename; + return m_chunkManager->getFilePtr(column, column.dataFile.fDbRoot, column.dataFile.fPartition, + column.dataFile.fSegment, filename, "r+b", column.colWidth, useTmpSuffix); } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_dbfileop.h b/writeengine/shared/we_dbfileop.h index 1ec160573..242aa3d04 100644 --- a/writeengine/shared/we_dbfileop.h +++ b/writeengine/shared/we_dbfileop.h @@ -27,7 +27,6 @@ #include "we_blockop.h" #include "we_cache.h" - #if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT) #define EXPORT __declspec(dllexport) #else @@ -37,218 +36,149 @@ /** Namespace WriteEngine */ namespace WriteEngine { - // forward reference class ChunkManager; - /** Class DbFileOp */ class DbFileOp : public FileOp { -public: - /** - * @brief Constructor - */ - EXPORT DbFileOp(); + public: + /** + * @brief Constructor + */ + EXPORT DbFileOp(); - /** - * @brief Default Destructor - */ - EXPORT virtual ~DbFileOp(); + /** + * @brief Default Destructor + */ + EXPORT virtual ~DbFileOp(); - EXPORT virtual int flushCache(); + EXPORT virtual int flushCache(); - /** - * @brief Get an entry within a subblock - */ - EXPORT void getSubBlockEntry( unsigned char* blockBuf, - const int sbid, - const int entryNo, - const int width, - void* pStruct ) ; + /** + * @brief Get an entry within a subblock + */ + EXPORT void getSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width, + void* pStruct); - /** - * @brief Get an entry within a subblock using block information - */ - void getSubBlockEntry( DataBlock* block, - const int sbid, - const int entryNo, - const int width, - void* pStruct ) - { - getSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); - } + /** + * @brief Get an entry within a subblock using block information + */ + void getSubBlockEntry(DataBlock* block, const int sbid, const int entryNo, const int width, void* pStruct) + { + getSubBlockEntry(block->data, sbid, entryNo, width, pStruct); + } - /** - * @brief Read DB file to a buffer - */ - EXPORT virtual int readDBFile( IDBDataFile* pFile, - unsigned char* readBuf, - const uint64_t lbid, - const bool isFbo = false ); - EXPORT int readDBFile( CommBlock& cb, - unsigned char* readBuf, - const uint64_t lbid ); + /** + * @brief Read DB file to a buffer + */ + EXPORT virtual int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, + const bool isFbo = false); + EXPORT int readDBFile(CommBlock& cb, unsigned char* readBuf, const uint64_t lbid); - EXPORT int readDBFile( IDBDataFile* pFile, - DataBlock* block, - const uint64_t lbid, - const bool isFbo = false ); - int readDBFile( CommBlock& cb, - DataBlock* block, - const uint64_t lbid ) - { - return readDBFile( cb, block->data, lbid ); - } + EXPORT int readDBFile(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const bool isFbo = false); + int readDBFile(CommBlock& cb, DataBlock* block, const uint64_t lbid) + { + return readDBFile(cb, block->data, lbid); + } - /** - * @brief Get an entry within a subblock and also populate block buffer - * - */ - EXPORT int readSubBlockEntry(IDBDataFile* pFile, - DataBlock* block, - const uint64_t lbid, - const int sbid, - const int entryNo, - const int width, - void* pStruct ) ; + /** + * @brief Get an entry within a subblock and also populate block buffer + * + */ + EXPORT int readSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct); - EXPORT int readSubBlockEntry(CommBlock& cb, - DataBlock* block, - const uint64_t lbid, - const int sbid, - const int entryNo, - const int width, - void* pStruct ); + EXPORT int readSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct); - /** - * @brief Set an entry within a subblock - */ - EXPORT void setSubBlockEntry( unsigned char* blockBuf, - const int sbid, - const int entryNo, - const int width, - const void* pStruct ) ; + /** + * @brief Set an entry within a subblock + */ + EXPORT void setSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width, + const void* pStruct); - /** - * @brief Set an entry within a subblock using block information - */ - void setSubBlockEntry( DataBlock* block, - const int sbid, - const int entryNo, - const int width, - const void* pStruct ) - { - block->dirty = true; - setSubBlockEntry( block->data, sbid, entryNo, width, pStruct ); - } + /** + * @brief Set an entry within a subblock using block information + */ + void setSubBlockEntry(DataBlock* block, const int sbid, const int entryNo, const int width, + const void* pStruct) + { + block->dirty = true; + setSubBlockEntry(block->data, sbid, entryNo, width, pStruct); + } - /** - * @brief Lbid Write a buffer to a DB file - */ - EXPORT virtual int writeDBFile( IDBDataFile* pFile, - const unsigned char* writeBuf, - const uint64_t lbid, - const int numOfBlock = 1 ); - EXPORT int writeDBFile( CommBlock& cb, - const unsigned char* writeBuf, - const uint64_t lbid, - const int numOfBlock = 1 ); + /** + * @brief Lbid Write a buffer to a DB file + */ + EXPORT virtual int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid, + const int numOfBlock = 1); + EXPORT int writeDBFile(CommBlock& cb, const unsigned char* writeBuf, const uint64_t lbid, + const int numOfBlock = 1); - /** - * @brief Write designated block(s) w/o writing to Version Buffer or cache. - */ - EXPORT int writeDBFileNoVBCache(CommBlock& cb, - const unsigned char* writeBuf, - const int fbo, - const int numOfBlock = 1); - EXPORT virtual int writeDBFileNoVBCache(IDBDataFile* pFile, - const unsigned char* writeBuf, - const int fbo, - const int numOfBlock = 1); + /** + * @brief Write designated block(s) w/o writing to Version Buffer or cache. + */ + EXPORT int writeDBFileNoVBCache(CommBlock& cb, const unsigned char* writeBuf, const int fbo, + const int numOfBlock = 1); + EXPORT virtual int writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo, + const int numOfBlock = 1); - int writeDBFile( IDBDataFile* pFile, - DataBlock* block, - const uint64_t lbid ) - { - block->dirty = false; - return writeDBFile( pFile, block->data, lbid ); - } - int writeDBFile( CommBlock& cb, - DataBlock* block, - const uint64_t lbid ) - { - return writeDBFile( cb, block->data, lbid ); - } + int writeDBFile(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid) + { + block->dirty = false; + return writeDBFile(pFile, block->data, lbid); + } + int writeDBFile(CommBlock& cb, DataBlock* block, const uint64_t lbid) + { + return writeDBFile(cb, block->data, lbid); + } - EXPORT virtual int writeDBFileFbo( IDBDataFile* pFile, - const unsigned char* writeBuf, - const uint64_t fbo, - const int numOfBlock ); + EXPORT virtual int writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo, + const int numOfBlock); - int writeDBFileNoVBCache(CommBlock& cb, - DataBlock* block, - const int fbo) - { - return writeDBFileNoVBCache(cb, block->data, fbo); - } + int writeDBFileNoVBCache(CommBlock& cb, DataBlock* block, const int fbo) + { + return writeDBFileNoVBCache(cb, block->data, fbo); + } - /** - * @brief Write a sub block entry directly to a DB file - */ - EXPORT int writeSubBlockEntry(IDBDataFile* pFile, - DataBlock* block, - const uint64_t lbid, - const int sbid, - const int entryNo, - const int width, - void* pStruct ); + /** + * @brief Write a sub block entry directly to a DB file + */ + EXPORT int writeSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct); - EXPORT int writeSubBlockEntry(CommBlock& cb, - DataBlock* block, - const uint64_t lbid, - const int sbid, - const int entryNo, - const int width, - void* pStruct ) ; + EXPORT int writeSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid, + const int entryNo, const int width, void* pStruct); - /** - * @brief Write to version buffer - */ - EXPORT int writeVB( IDBDataFile* pFile, - const OID oid, - const uint64_t lbid ); + /** + * @brief Write to version buffer + */ + EXPORT int writeVB(IDBDataFile* pFile, const OID oid, const uint64_t lbid); - EXPORT virtual int readDbBlocks( IDBDataFile* pFile, - unsigned char* readBuf, - uint64_t fbo, - size_t n); + EXPORT virtual int readDbBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n); - EXPORT virtual int restoreBlock( IDBDataFile* pFile, - const unsigned char* writeBuf, - uint64_t fbo); + EXPORT virtual int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo); - EXPORT virtual IDBDataFile* getFilePtr(const Column& column, - bool useTmpSuffix); + EXPORT virtual IDBDataFile* getFilePtr(const Column& column, bool useTmpSuffix); - virtual void chunkManager(ChunkManager* ptr) - { - m_chunkManager = ptr; - } - virtual ChunkManager* chunkManager() - { - return m_chunkManager; - } + virtual void chunkManager(ChunkManager* ptr) + { + m_chunkManager = ptr; + } + virtual ChunkManager* chunkManager() + { + return m_chunkManager; + } -protected: - ChunkManager* m_chunkManager; - -private: + protected: + ChunkManager* m_chunkManager; + private: }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_DBFILEOP_H_ +#endif // _WE_DBFILEOP_H_ diff --git a/writeengine/shared/we_dbrootextenttracker.cpp b/writeengine/shared/we_dbrootextenttracker.cpp index 17ef902fd..c07e64bd1 100644 --- a/writeengine/shared/we_dbrootextenttracker.cpp +++ b/writeengine/shared/we_dbrootextenttracker.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_dbrootextenttracker.cpp 4631 2013-05-02 15:21:09Z dcathey $ -*/ + * $Id: we_dbrootextenttracker.cpp 4631 2013-05-02 15:21:09Z dcathey $ + */ #include "we_dbrootextenttracker.h" #include @@ -29,199 +29,167 @@ namespace { -const char* stateStrings[] = { "initState", - "PartialExtent", - "EmptyDbRoot", - "ExtentBoundary", - "OutOfService" - }; +const char* stateStrings[] = {"initState", "PartialExtent", "EmptyDbRoot", "ExtentBoundary", "OutOfService"}; } namespace WriteEngine { - //------------------------------------------------------------------------------ // DBRootExtentInfo constructor //------------------------------------------------------------------------------ -DBRootExtentInfo::DBRootExtentInfo( - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - BRM::LBID_t startLbid, - HWM localHwm, - uint64_t dbrootTotalBlocks, - DBRootExtentInfoState state) : - fPartition(partition), - fDbRoot(dbRoot), - fSegment(segment), - fStartLbid(startLbid), - fLocalHwm(localHwm), - fDBRootTotalBlocks(dbrootTotalBlocks), - fState(state) +DBRootExtentInfo::DBRootExtentInfo(uint16_t dbRoot, uint32_t partition, uint16_t segment, + BRM::LBID_t startLbid, HWM localHwm, uint64_t dbrootTotalBlocks, + DBRootExtentInfoState state) + : fPartition(partition) + , fDbRoot(dbRoot) + , fSegment(segment) + , fStartLbid(startLbid) + , fLocalHwm(localHwm) + , fDBRootTotalBlocks(dbrootTotalBlocks) + , fState(state) { } - + //------------------------------------------------------------------------------ // LessThan operator used to sort DBRootExtentInfo objects by DBRoot. //------------------------------------------------------------------------------ -bool DBRootExtentInfo::operator<( - const DBRootExtentInfo& entry) const +bool DBRootExtentInfo::operator<(const DBRootExtentInfo& entry) const { - if (fDbRoot < entry.fDbRoot) - return true; + if (fDbRoot < entry.fDbRoot) + return true; - return false; + return false; } - + //------------------------------------------------------------------------------ // DBRootExtentTracker constructor // // Mutex lock not needed in this function as it is only called from main thread // before processing threads are spawned. //------------------------------------------------------------------------------ -DBRootExtentTracker::DBRootExtentTracker ( OID oid, - const std::vector& colWidths, - const std::vector& dbRootHWMInfoColVec, - unsigned int columnIdx, - Log* logger ) : - fOID(oid), - fLog(logger), - fCurrentDBRootIdx(-1), - fEmptyOrDisabledPM(false), - fEmptyPM(true), - fDisabledHWM(false) +DBRootExtentTracker::DBRootExtentTracker(OID oid, const std::vector& colWidths, + const std::vector& dbRootHWMInfoColVec, + unsigned int columnIdx, Log* logger) + : fOID(oid) + , fLog(logger) + , fCurrentDBRootIdx(-1) + , fEmptyOrDisabledPM(false) + , fEmptyPM(true) + , fDisabledHWM(false) { - const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo = - dbRootHWMInfoColVec[columnIdx]; - int colWidth = colWidths[columnIdx]; + const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo = dbRootHWMInfoColVec[columnIdx]; + int colWidth = colWidths[columnIdx]; - fBlksPerExtent = (long long)BRMWrapper::getInstance()->getExtentRows() * - (long long)colWidth / (long long)BYTE_PER_BLOCK; + fBlksPerExtent = + (long long)BRMWrapper::getInstance()->getExtentRows() * (long long)colWidth / (long long)BYTE_PER_BLOCK; - std::vector resetState; + std::vector resetState; - for (unsigned int i = 0; i < emDbRootHWMInfo.size(); i++) + for (unsigned int i = 0; i < emDbRootHWMInfo.size(); i++) + { + resetState.push_back(false); + DBRootExtentInfoState state = determineState(colWidths[columnIdx], emDbRootHWMInfo[i].localHWM, + emDbRootHWMInfo[i].totalBlocks, emDbRootHWMInfo[i].status); + + // For a full extent... + // check to see if any of the column HWMs are partially full, in which + // case we consider all the columns for that DBRoot to be partially + // full. (This can happen if a table has columns with varying widths, + // as the HWM may be at the last extent block for a shorter column, and + // still have free blocks for wider columns.) + if (state == DBROOT_EXTENT_EXTENT_BOUNDARY) { - resetState.push_back(false); - DBRootExtentInfoState state = determineState( - colWidths[columnIdx], - emDbRootHWMInfo[i].localHWM, - emDbRootHWMInfo[i].totalBlocks, - emDbRootHWMInfo[i].status); + for (unsigned int kCol = 0; kCol < dbRootHWMInfoColVec.size(); kCol++) + { + const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo2 = dbRootHWMInfoColVec[kCol]; + DBRootExtentInfoState state2 = + determineState(colWidths[kCol], emDbRootHWMInfo2[i].localHWM, emDbRootHWMInfo2[i].totalBlocks, + emDbRootHWMInfo2[i].status); - // For a full extent... - // check to see if any of the column HWMs are partially full, in which - // case we consider all the columns for that DBRoot to be partially - // full. (This can happen if a table has columns with varying widths, - // as the HWM may be at the last extent block for a shorter column, and - // still have free blocks for wider columns.) - if (state == DBROOT_EXTENT_EXTENT_BOUNDARY) + if (state2 == DBROOT_EXTENT_PARTIAL_EXTENT) { - for (unsigned int kCol = 0; kCol < dbRootHWMInfoColVec.size(); kCol++) - { - const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo2 = - dbRootHWMInfoColVec[kCol]; - DBRootExtentInfoState state2 = determineState( - colWidths[kCol], - emDbRootHWMInfo2[i].localHWM, - emDbRootHWMInfo2[i].totalBlocks, - emDbRootHWMInfo2[i].status); - - if (state2 == DBROOT_EXTENT_PARTIAL_EXTENT) - { - state = DBROOT_EXTENT_PARTIAL_EXTENT; - resetState[ resetState.size() - 1 ] = true; - break; - } - } + state = DBROOT_EXTENT_PARTIAL_EXTENT; + resetState[resetState.size() - 1] = true; + break; } - - DBRootExtentInfo dbRootExtent( - emDbRootHWMInfo[i].dbRoot, - emDbRootHWMInfo[i].partitionNum, - emDbRootHWMInfo[i].segmentNum, - emDbRootHWMInfo[i].startLbid, - emDbRootHWMInfo[i].localHWM, - emDbRootHWMInfo[i].totalBlocks, - state); - - fDBRootExtentList.push_back( dbRootExtent ); + } } - std::sort( fDBRootExtentList.begin(), fDBRootExtentList.end() ); + DBRootExtentInfo dbRootExtent(emDbRootHWMInfo[i].dbRoot, emDbRootHWMInfo[i].partitionNum, + emDbRootHWMInfo[i].segmentNum, emDbRootHWMInfo[i].startLbid, + emDbRootHWMInfo[i].localHWM, emDbRootHWMInfo[i].totalBlocks, state); - if (fLog) + fDBRootExtentList.push_back(dbRootExtent); + } + + std::sort(fDBRootExtentList.begin(), fDBRootExtentList.end()); + + if (fLog) + { + // Always log this info for now; may control with debug later + // if (fLog->isDebug(DEBUG_1)) { - // Always log this info for now; may control with debug later - //if (fLog->isDebug(DEBUG_1)) - { - std::ostringstream oss; - oss << "Starting DBRoot info for OID " << fOID; + std::ostringstream oss; + oss << "Starting DBRoot info for OID " << fOID; - for (unsigned int k = 0; k < fDBRootExtentList.size(); k++) - { - oss << std::endl; - oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot << - ", part/seg/hwm/LBID/totBlks/state: " << - fDBRootExtentList[k].fPartition << - "/" << fDBRootExtentList[k].fSegment << - "/" << fDBRootExtentList[k].fLocalHwm << - "/" << fDBRootExtentList[k].fStartLbid << - "/" << fDBRootExtentList[k].fDBRootTotalBlocks << - "/" << stateStrings[ fDBRootExtentList[k].fState ]; + for (unsigned int k = 0; k < fDBRootExtentList.size(); k++) + { + oss << std::endl; + oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot + << ", part/seg/hwm/LBID/totBlks/state: " << fDBRootExtentList[k].fPartition << "/" + << fDBRootExtentList[k].fSegment << "/" << fDBRootExtentList[k].fLocalHwm << "/" + << fDBRootExtentList[k].fStartLbid << "/" << fDBRootExtentList[k].fDBRootTotalBlocks << "/" + << stateStrings[fDBRootExtentList[k].fState]; - if (resetState[k]) - oss << "."; - } + if (resetState[k]) + oss << "."; + } - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + fLog->logMsg(oss.str(), MSGLVL_INFO2); } + } } - + //------------------------------------------------------------------------------ // Determines the state of the HWM extent (for a DBRoot); considering the // current BRM status, HWM, and total block count for the DBRoot. //------------------------------------------------------------------------------ -DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth, - HWM localHwm, - uint64_t dbRootTotalBlocks, - int16_t status) +DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth, HWM localHwm, + uint64_t dbRootTotalBlocks, int16_t status) { - DBRootExtentInfoState extentState; + DBRootExtentInfoState extentState; - if (status == BRM::EXTENTOUTOFSERVICE) + if (status == BRM::EXTENTOUTOFSERVICE) + { + extentState = DBROOT_EXTENT_OUT_OF_SERVICE; + } + else + { + if (dbRootTotalBlocks == 0) { - extentState = DBROOT_EXTENT_OUT_OF_SERVICE; + extentState = DBROOT_EXTENT_EMPTY_DBROOT; } else { - if (dbRootTotalBlocks == 0) - { - extentState = DBROOT_EXTENT_EMPTY_DBROOT; - } - else - { - extentState = DBROOT_EXTENT_PARTIAL_EXTENT; + extentState = DBROOT_EXTENT_PARTIAL_EXTENT; - // See if local hwm is on an extent bndry,in which case the extent - // is full and we won't be adding rows to the current HWM extent; - // we will instead need to allocate a new extent in order to begin - // adding any rows. - long long nRows = ((long long)(localHwm + 1) * - (long long)BYTE_PER_BLOCK) / (long long)colWidth; - long long nRem = nRows % BRMWrapper::getInstance()->getExtentRows(); + // See if local hwm is on an extent bndry,in which case the extent + // is full and we won't be adding rows to the current HWM extent; + // we will instead need to allocate a new extent in order to begin + // adding any rows. + long long nRows = ((long long)(localHwm + 1) * (long long)BYTE_PER_BLOCK) / (long long)colWidth; + long long nRem = nRows % BRMWrapper::getInstance()->getExtentRows(); - if (nRem == 0) - { - extentState = DBROOT_EXTENT_EXTENT_BOUNDARY; - } - } + if (nRem == 0) + { + extentState = DBROOT_EXTENT_EXTENT_BOUNDARY; + } } + } - return extentState; + return extentState; } - + //------------------------------------------------------------------------------ // Select the first segment file to add rows to for the local PM. // Function will first try to find the HWM extent with the fewest blocks. @@ -232,126 +200,118 @@ DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth, // Mutex lock not needed in this function as it is only called from main thread // before processing threads are spawned. //------------------------------------------------------------------------------ -int DBRootExtentTracker::selectFirstSegFile( - DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM, - bool& bEmptyPM, - std::string& errMsg ) +int DBRootExtentTracker::selectFirstSegFile(DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM, + bool& bEmptyPM, std::string& errMsg) { - int startExtentIdx = -1; - int fewestLocalBlocksIdx = -1; // track HWM extent with fewest blocks - int fewestTotalBlocksIdx = -1; // track DBRoot with fewest total blocks - bNoStartExtentOnThisPM = false; + int startExtentIdx = -1; + int fewestLocalBlocksIdx = -1; // track HWM extent with fewest blocks + int fewestTotalBlocksIdx = -1; // track DBRoot with fewest total blocks + bNoStartExtentOnThisPM = false; - unsigned int fewestTotalBlks = UINT_MAX; - unsigned int fewestLocalBlks = UINT_MAX; - uint16_t fewestTotalBlkSegNum = USHRT_MAX; - uint16_t fewestLocalBlkSegNum = USHRT_MAX; + unsigned int fewestTotalBlks = UINT_MAX; + unsigned int fewestLocalBlks = UINT_MAX; + uint16_t fewestTotalBlkSegNum = USHRT_MAX; + uint16_t fewestLocalBlkSegNum = USHRT_MAX; - // Find DBRoot having HWM extent with fewest blocks. If all HWM extents - // are full (remblks=0), then fall-back on selecting the DBRoot with fewest - // total blks. - // - // Selecting HWM extent with fewest blocks should be straight forward, be- - // cause all the DBRoots on a PM should end on an extent boundary except - // for the current last extent. But if the user has moved a DBRoot, then - // we can end up with 2 partially filled HWM extents on 2 DBRoots, on the - // same PM. That's why we loop through the DBRoots to see if we have more - // than 1 partially filled HWM extent. - for (unsigned int iroot = 0; - iroot < fDBRootExtentList.size(); - iroot++) + // Find DBRoot having HWM extent with fewest blocks. If all HWM extents + // are full (remblks=0), then fall-back on selecting the DBRoot with fewest + // total blks. + // + // Selecting HWM extent with fewest blocks should be straight forward, be- + // cause all the DBRoots on a PM should end on an extent boundary except + // for the current last extent. But if the user has moved a DBRoot, then + // we can end up with 2 partially filled HWM extents on 2 DBRoots, on the + // same PM. That's why we loop through the DBRoots to see if we have more + // than 1 partially filled HWM extent. + for (unsigned int iroot = 0; iroot < fDBRootExtentList.size(); iroot++) + { + // Skip over DBRoots which have no extents + if (fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) + continue; + + fEmptyPM = false; + + // Find DBRoot and segment file with most incomplete extent. + // Break a tie by selecting the lowest segment number. + long long remBlks = (long long)(fDBRootExtentList[iroot].fLocalHwm + 1) % fBlksPerExtent; + + if (remBlks > 0) { - // Skip over DBRoots which have no extents - if (fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) - continue; - - fEmptyPM = false; - - // Find DBRoot and segment file with most incomplete extent. - // Break a tie by selecting the lowest segment number. - long long remBlks = (long long)(fDBRootExtentList[iroot].fLocalHwm + 1) % - fBlksPerExtent; - - if (remBlks > 0) - { - if ( (remBlks < fewestLocalBlks) || - ((remBlks == fewestLocalBlks) && - (fDBRootExtentList[iroot].fSegment < fewestLocalBlkSegNum)) ) - { - fewestLocalBlocksIdx = iroot; - fewestLocalBlks = remBlks; - fewestLocalBlkSegNum = fDBRootExtentList[iroot].fSegment; - } - } - - // Find DBRoot with fewest total of blocks. - // Break a tie by selecting the highest segment number. - if ( (fDBRootExtentList[iroot].fDBRootTotalBlocks < fewestTotalBlks) || - ((fDBRootExtentList[iroot].fDBRootTotalBlocks == fewestTotalBlks) && - (fDBRootExtentList[iroot].fSegment > fewestTotalBlkSegNum)) ) - { - fewestTotalBlocksIdx = iroot; - fewestTotalBlks = fDBRootExtentList[iroot].fDBRootTotalBlocks; - fewestTotalBlkSegNum = fDBRootExtentList[iroot].fSegment; - } + if ((remBlks < fewestLocalBlks) || + ((remBlks == fewestLocalBlks) && (fDBRootExtentList[iroot].fSegment < fewestLocalBlkSegNum))) + { + fewestLocalBlocksIdx = iroot; + fewestLocalBlks = remBlks; + fewestLocalBlkSegNum = fDBRootExtentList[iroot].fSegment; + } } - // Select HWM extent with fewest number of blocks; - // If chosen extent is disabled, then treat like an empty PM, - // meaning we have to allocate a new extent before adding any rows - if (fewestLocalBlocksIdx != -1) + // Find DBRoot with fewest total of blocks. + // Break a tie by selecting the highest segment number. + if ((fDBRootExtentList[iroot].fDBRootTotalBlocks < fewestTotalBlks) || + ((fDBRootExtentList[iroot].fDBRootTotalBlocks == fewestTotalBlks) && + (fDBRootExtentList[iroot].fSegment > fewestTotalBlkSegNum))) { - startExtentIdx = fewestLocalBlocksIdx; - - if (fDBRootExtentList[startExtentIdx].fState == - DBROOT_EXTENT_OUT_OF_SERVICE) - { - fDisabledHWM = true; - } + fewestTotalBlocksIdx = iroot; + fewestTotalBlks = fDBRootExtentList[iroot].fDBRootTotalBlocks; + fewestTotalBlkSegNum = fDBRootExtentList[iroot].fSegment; } + } - // If the HWM on each DBRoot ends on an extent boundary, then - // select the DBRoot with the fewest total number of blocks; - // If chosen extent is disabled, then treat like an empty PM, - // meaning we have to allocate a new extent before adding any rows - else if (fewestTotalBlocksIdx != -1) + // Select HWM extent with fewest number of blocks; + // If chosen extent is disabled, then treat like an empty PM, + // meaning we have to allocate a new extent before adding any rows + if (fewestLocalBlocksIdx != -1) + { + startExtentIdx = fewestLocalBlocksIdx; + + if (fDBRootExtentList[startExtentIdx].fState == DBROOT_EXTENT_OUT_OF_SERVICE) { - startExtentIdx = fewestTotalBlocksIdx; - - if (fDBRootExtentList[startExtentIdx].fState == - DBROOT_EXTENT_OUT_OF_SERVICE) - { - fDisabledHWM = true; - } + fDisabledHWM = true; } + } - // PM with no extents (or all extents disabled), so select DBRoot/segment - // file from DBRoot list, where we will start inserting rows. - // Select lowest segment file number. - else + // If the HWM on each DBRoot ends on an extent boundary, then + // select the DBRoot with the fewest total number of blocks; + // If chosen extent is disabled, then treat like an empty PM, + // meaning we have to allocate a new extent before adding any rows + else if (fewestTotalBlocksIdx != -1) + { + startExtentIdx = fewestTotalBlocksIdx; + + if (fDBRootExtentList[startExtentIdx].fState == DBROOT_EXTENT_OUT_OF_SERVICE) { - RETURN_ON_ERROR( selectFirstSegFileForEmptyPM( errMsg ) ); - - startExtentIdx = fCurrentDBRootIdx; + fDisabledHWM = true; } + } - if ((fEmptyOrDisabledPM) || (fDisabledHWM)) - bNoStartExtentOnThisPM = true; + // PM with no extents (or all extents disabled), so select DBRoot/segment + // file from DBRoot list, where we will start inserting rows. + // Select lowest segment file number. + else + { + RETURN_ON_ERROR(selectFirstSegFileForEmptyPM(errMsg)); - bEmptyPM = fEmptyPM; - fCurrentDBRootIdx = startExtentIdx; + startExtentIdx = fCurrentDBRootIdx; + } - // Finish Initializing DBRootExtentList for empty DBRoots w/o any extents - initEmptyDBRoots( ); + if ((fEmptyOrDisabledPM) || (fDisabledHWM)) + bNoStartExtentOnThisPM = true; - logFirstDBRootSelection( ); + bEmptyPM = fEmptyPM; + fCurrentDBRootIdx = startExtentIdx; - dbRootExtent = fDBRootExtentList[startExtentIdx]; - fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY; + // Finish Initializing DBRootExtentList for empty DBRoots w/o any extents + initEmptyDBRoots(); - return NO_ERROR; + logFirstDBRootSelection(); + + dbRootExtent = fDBRootExtentList[startExtentIdx]; + fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY; + + return NO_ERROR; } - + //------------------------------------------------------------------------------ // If we have encountered a PM with no extents (or all extent disabled), then // this function can be called to determine the DBRoot to be used for the 1st @@ -361,73 +321,66 @@ int DBRootExtentTracker::selectFirstSegFile( // Mutex lock not needed in this function as it is only called from main thread // before processing threads are spawned. //------------------------------------------------------------------------------ -int DBRootExtentTracker::selectFirstSegFileForEmptyPM( std::string& errMsg ) +int DBRootExtentTracker::selectFirstSegFileForEmptyPM(std::string& errMsg) { - fEmptyOrDisabledPM = true; + fEmptyOrDisabledPM = true; - fCurrentDBRootIdx = 0; // Start with first DBRoot for this PM + fCurrentDBRootIdx = 0; // Start with first DBRoot for this PM - // Always start empty PM with partition number 0. If the DBRoot has a HWM - // extent that is disabled, then BRM will override this partition number. - fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0; + // Always start empty PM with partition number 0. If the DBRoot has a HWM + // extent that is disabled, then BRM will override this partition number. + fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0; - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Finish Initializing fDBRootExtentList for any empty DBRoots w/o any extents. //------------------------------------------------------------------------------ -void DBRootExtentTracker::initEmptyDBRoots( ) +void DBRootExtentTracker::initEmptyDBRoots() { - int startExtentIdx = fCurrentDBRootIdx; - bool bAnyChanges = false; // If fDBRootExtentList changes, log the contents + int startExtentIdx = fCurrentDBRootIdx; + bool bAnyChanges = false; // If fDBRootExtentList changes, log the contents - // Fill in starting partition for any DBRoots having no extents - for (unsigned int iroot = 0; - iroot < fDBRootExtentList.size(); - iroot++) + // Fill in starting partition for any DBRoots having no extents + for (unsigned int iroot = 0; iroot < fDBRootExtentList.size(); iroot++) + { + if ((fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) && + ((int)iroot != startExtentIdx)) // skip over selected dbroot { - if ((fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) && - ((int)iroot != startExtentIdx)) // skip over selected dbroot - { - if (fDBRootExtentList[iroot].fPartition != - fDBRootExtentList[startExtentIdx].fPartition) - { - bAnyChanges = true; + if (fDBRootExtentList[iroot].fPartition != fDBRootExtentList[startExtentIdx].fPartition) + { + bAnyChanges = true; - fDBRootExtentList[iroot].fPartition = - fDBRootExtentList[startExtentIdx].fPartition; - } - } + fDBRootExtentList[iroot].fPartition = fDBRootExtentList[startExtentIdx].fPartition; + } } + } - // Log fDBRootExtentList if modifications were made - if ((bAnyChanges) && (fLog)) + // Log fDBRootExtentList if modifications were made + if ((bAnyChanges) && (fLog)) + { + // Always log this info for now; may control with debug later + // if (fLog->isDebug(DEBUG_1)) { - // Always log this info for now; may control with debug later - //if (fLog->isDebug(DEBUG_1)) - { - std::ostringstream oss; - oss << "Updated starting (empty) DBRoot info for OID " << fOID; + std::ostringstream oss; + oss << "Updated starting (empty) DBRoot info for OID " << fOID; - for (unsigned int k = 0; k < fDBRootExtentList.size(); k++) - { - oss << std::endl; - oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot << - ", part/seg/hwm/LBID/totBlks/state: " << - fDBRootExtentList[k].fPartition << - "/" << fDBRootExtentList[k].fSegment << - "/" << fDBRootExtentList[k].fLocalHwm << - "/" << fDBRootExtentList[k].fStartLbid << - "/" << fDBRootExtentList[k].fDBRootTotalBlocks << - "/" << stateStrings[ fDBRootExtentList[k].fState ]; - } + for (unsigned int k = 0; k < fDBRootExtentList.size(); k++) + { + oss << std::endl; + oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot + << ", part/seg/hwm/LBID/totBlks/state: " << fDBRootExtentList[k].fPartition << "/" + << fDBRootExtentList[k].fSegment << "/" << fDBRootExtentList[k].fLocalHwm << "/" + << fDBRootExtentList[k].fStartLbid << "/" << fDBRootExtentList[k].fDBRootTotalBlocks << "/" + << stateStrings[fDBRootExtentList[k].fState]; + } - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + fLog->logMsg(oss.str(), MSGLVL_INFO2); } + } } - + //------------------------------------------------------------------------------ // Assign the DBRoot/segment file to be used for extent loading based on the // setting in the specified reference tracker (set for a reference column). @@ -435,79 +388,75 @@ void DBRootExtentTracker::initEmptyDBRoots( ) // Mutex lock not needed in this function as it is only called from main thread // before processing threads are spawned. //------------------------------------------------------------------------------ -void DBRootExtentTracker::assignFirstSegFile( - const DBRootExtentTracker& refTracker, - DBRootExtentInfo& dbRootExtent) +void DBRootExtentTracker::assignFirstSegFile(const DBRootExtentTracker& refTracker, + DBRootExtentInfo& dbRootExtent) { - // Start with the same DBRoot index as the reference tracker; assumes that - // DBRoots for each column are listed in same order in fDBRootExtentList. - // That should be a safe assumption since DBRootExtentTracker constructor - // sorts the entries in fDBRootExtentList by fDbRoot. - int startExtentIdx = refTracker.fCurrentDBRootIdx; - fEmptyOrDisabledPM = refTracker.fEmptyOrDisabledPM; - fEmptyPM = refTracker.fEmptyPM; - fDisabledHWM = refTracker.fDisabledHWM; + // Start with the same DBRoot index as the reference tracker; assumes that + // DBRoots for each column are listed in same order in fDBRootExtentList. + // That should be a safe assumption since DBRootExtentTracker constructor + // sorts the entries in fDBRootExtentList by fDbRoot. + int startExtentIdx = refTracker.fCurrentDBRootIdx; + fEmptyOrDisabledPM = refTracker.fEmptyOrDisabledPM; + fEmptyPM = refTracker.fEmptyPM; + fDisabledHWM = refTracker.fDisabledHWM; - // Always start empty PM with partition number 0. If the DBRoot has a HWM - // extent that is disabled, then BRM will override this partition number. - if (fEmptyOrDisabledPM) - { - fDBRootExtentList[startExtentIdx].fPartition = 0; - } + // Always start empty PM with partition number 0. If the DBRoot has a HWM + // extent that is disabled, then BRM will override this partition number. + if (fEmptyOrDisabledPM) + { + fDBRootExtentList[startExtentIdx].fPartition = 0; + } - fCurrentDBRootIdx = startExtentIdx; + fCurrentDBRootIdx = startExtentIdx; - // Finish Initializing DBRootExtentList for empty DBRoots w/o any extents - initEmptyDBRoots( ); + // Finish Initializing DBRootExtentList for empty DBRoots w/o any extents + initEmptyDBRoots(); - logFirstDBRootSelection( ); + logFirstDBRootSelection(); - dbRootExtent = fDBRootExtentList[startExtentIdx]; - fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY; + dbRootExtent = fDBRootExtentList[startExtentIdx]; + fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY; } - + //------------------------------------------------------------------------------ // Log information about the first DBRoot/segment file that is selected. //------------------------------------------------------------------------------ -void DBRootExtentTracker::logFirstDBRootSelection( ) const +void DBRootExtentTracker::logFirstDBRootSelection() const { - if (fLog) - { - int extentIdx = fCurrentDBRootIdx; + if (fLog) + { + int extentIdx = fCurrentDBRootIdx; - if (fEmptyOrDisabledPM) - { - std::ostringstream oss; - oss << "No active extents; will add partition to start adding " - "rows for oid-" << fOID << - "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else if (fDisabledHWM) - { - std::ostringstream oss; - oss << "HWM extent disabled; will add partition to start adding " - "rows for oid-" << fOID << - "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } - else - { - std::ostringstream oss; - oss << "Selecting existing segFile to begin adding rows: oid-" << fOID << - "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot << - ", part/seg/hwm/LBID/totBlks/state: " << - fDBRootExtentList[extentIdx].fPartition << - "/" << fDBRootExtentList[extentIdx].fSegment << - "/" << fDBRootExtentList[extentIdx].fLocalHwm << - "/" << fDBRootExtentList[extentIdx].fStartLbid << - "/" << fDBRootExtentList[extentIdx].fDBRootTotalBlocks << - "/" << stateStrings[ fDBRootExtentList[extentIdx].fState ]; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + if (fEmptyOrDisabledPM) + { + std::ostringstream oss; + oss << "No active extents; will add partition to start adding " + "rows for oid-" + << fOID << "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot; + fLog->logMsg(oss.str(), MSGLVL_INFO2); } + else if (fDisabledHWM) + { + std::ostringstream oss; + oss << "HWM extent disabled; will add partition to start adding " + "rows for oid-" + << fOID << "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + else + { + std::ostringstream oss; + oss << "Selecting existing segFile to begin adding rows: oid-" << fOID << "; DBRoot-" + << fDBRootExtentList[extentIdx].fDbRoot + << ", part/seg/hwm/LBID/totBlks/state: " << fDBRootExtentList[extentIdx].fPartition << "/" + << fDBRootExtentList[extentIdx].fSegment << "/" << fDBRootExtentList[extentIdx].fLocalHwm << "/" + << fDBRootExtentList[extentIdx].fStartLbid << "/" << fDBRootExtentList[extentIdx].fDBRootTotalBlocks + << "/" << stateStrings[fDBRootExtentList[extentIdx].fState]; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } + } } - + //------------------------------------------------------------------------------ // Iterate/return next DBRoot to be used for the next extent // @@ -522,57 +471,52 @@ void DBRootExtentTracker::logFirstDBRootSelection( ) const // Returns true if new extent needs to be allocated, else false indicates that // the extent is partially full. //------------------------------------------------------------------------------ -bool DBRootExtentTracker::nextSegFile( - uint16_t& dbRoot, - uint32_t& partition, - uint16_t& segment, - HWM& localHwm, - BRM::LBID_t& startLbid) +bool DBRootExtentTracker::nextSegFile(uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, HWM& localHwm, + BRM::LBID_t& startLbid) { - boost::mutex::scoped_lock lock(fDBRootExtTrkMutex); + boost::mutex::scoped_lock lock(fDBRootExtTrkMutex); - fCurrentDBRootIdx++; + fCurrentDBRootIdx++; - if ((unsigned int)fCurrentDBRootIdx >= fDBRootExtentList.size()) - fCurrentDBRootIdx = 0; + if ((unsigned int)fCurrentDBRootIdx >= fDBRootExtentList.size()) + fCurrentDBRootIdx = 0; - dbRoot = fDBRootExtentList[fCurrentDBRootIdx].fDbRoot; - segment = fDBRootExtentList[fCurrentDBRootIdx].fSegment; - partition = fDBRootExtentList[fCurrentDBRootIdx].fPartition; - localHwm = fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm; - startLbid = fDBRootExtentList[fCurrentDBRootIdx].fStartLbid; -//std::cout << "NextSegFile: Current idx: " << fCurrentDBRootIdx << -//"; new dbroot: " << dbRoot << -//"; segment: " << segment << -//"; partition: " << partition << -//"; localHwm: " << localHwm << -//"; startLbid: " << startLbid << -//"; state: " << stateStrings[fDBRootExtentList[fCurrentDBRootIdx].fState] -// << std::endl; + dbRoot = fDBRootExtentList[fCurrentDBRootIdx].fDbRoot; + segment = fDBRootExtentList[fCurrentDBRootIdx].fSegment; + partition = fDBRootExtentList[fCurrentDBRootIdx].fPartition; + localHwm = fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm; + startLbid = fDBRootExtentList[fCurrentDBRootIdx].fStartLbid; + // std::cout << "NextSegFile: Current idx: " << fCurrentDBRootIdx << + //"; new dbroot: " << dbRoot << + //"; segment: " << segment << + //"; partition: " << partition << + //"; localHwm: " << localHwm << + //"; startLbid: " << startLbid << + //"; state: " << stateStrings[fDBRootExtentList[fCurrentDBRootIdx].fState] + // << std::endl; - bool bAllocExtentFlag = true; + bool bAllocExtentFlag = true; - if (fDBRootExtentList[fCurrentDBRootIdx].fState == - DBROOT_EXTENT_PARTIAL_EXTENT) - bAllocExtentFlag = false; + if (fDBRootExtentList[fCurrentDBRootIdx].fState == DBROOT_EXTENT_PARTIAL_EXTENT) + bAllocExtentFlag = false; - // After we have taken care of the "first" extent for each DBRoot, we can - // zero out everything. The only thing we need to continue rotating thru - // the DBRoots, is the DBRoot number itself. - fDBRootExtentList[fCurrentDBRootIdx].fSegment = 0; - fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0; - fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm = 0; - fDBRootExtentList[fCurrentDBRootIdx].fStartLbid = 0; - fDBRootExtentList[fCurrentDBRootIdx].fDBRootTotalBlocks = 0; - fDBRootExtentList[fCurrentDBRootIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY; + // After we have taken care of the "first" extent for each DBRoot, we can + // zero out everything. The only thing we need to continue rotating thru + // the DBRoots, is the DBRoot number itself. + fDBRootExtentList[fCurrentDBRootIdx].fSegment = 0; + fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0; + fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm = 0; + fDBRootExtentList[fCurrentDBRootIdx].fStartLbid = 0; + fDBRootExtentList[fCurrentDBRootIdx].fDBRootTotalBlocks = 0; + fDBRootExtentList[fCurrentDBRootIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY; - return bAllocExtentFlag; + return bAllocExtentFlag; } const std::vector& DBRootExtentTracker::getDBRootExtentList() { - boost::mutex::scoped_lock lock(fDBRootExtTrkMutex); - return fDBRootExtentList; + boost::mutex::scoped_lock lock(fDBRootExtTrkMutex); + return fDBRootExtentList; } -} // end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_dbrootextenttracker.h b/writeengine/shared/we_dbrootextenttracker.h index b7d0c3618..2623c6472 100644 --- a/writeengine/shared/we_dbrootextenttracker.h +++ b/writeengine/shared/we_dbrootextenttracker.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_dbrootextenttracker.h 4631 2013-05-02 15:21:09Z dcathey $ -*/ + * $Id: we_dbrootextenttracker.h 4631 2013-05-02 15:21:09Z dcathey $ + */ /** @file we_dbrootextenttracker.h * Contains classes to track the order of placement (rotation) of extents as @@ -62,10 +62,10 @@ class Log; // enum DBRootExtentInfoState { - DBROOT_EXTENT_PARTIAL_EXTENT = 1, - DBROOT_EXTENT_EMPTY_DBROOT = 2, - DBROOT_EXTENT_EXTENT_BOUNDARY = 3, - DBROOT_EXTENT_OUT_OF_SERVICE = 4 + DBROOT_EXTENT_PARTIAL_EXTENT = 1, + DBROOT_EXTENT_EMPTY_DBROOT = 2, + DBROOT_EXTENT_EXTENT_BOUNDARY = 3, + DBROOT_EXTENT_OUT_OF_SERVICE = 4 }; //------------------------------------------------------------------------------ @@ -74,33 +74,29 @@ enum DBRootExtentInfoState //------------------------------------------------------------------------------ struct DBRootExtentInfo { - uint32_t fPartition; - uint16_t fDbRoot; - uint16_t fSegment; - BRM::LBID_t fStartLbid; - HWM fLocalHwm; - uint64_t fDBRootTotalBlocks; - DBRootExtentInfoState fState; + uint32_t fPartition; + uint16_t fDbRoot; + uint16_t fSegment; + BRM::LBID_t fStartLbid; + HWM fLocalHwm; + uint64_t fDBRootTotalBlocks; + DBRootExtentInfoState fState; - DBRootExtentInfo() : - fPartition(0), - fDbRoot(0), - fSegment(0), - fStartLbid(0), - fLocalHwm(0), - fDBRootTotalBlocks(0), - fState(DBROOT_EXTENT_PARTIAL_EXTENT) { } + DBRootExtentInfo() + : fPartition(0) + , fDbRoot(0) + , fSegment(0) + , fStartLbid(0) + , fLocalHwm(0) + , fDBRootTotalBlocks(0) + , fState(DBROOT_EXTENT_PARTIAL_EXTENT) + { + } - DBRootExtentInfo( - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - BRM::LBID_t startLbid, - HWM localHwm, - uint64_t dbrootTotalBlocks, - DBRootExtentInfoState state); + DBRootExtentInfo(uint16_t dbRoot, uint32_t partition, uint16_t segment, BRM::LBID_t startLbid, HWM localHwm, + uint64_t dbrootTotalBlocks, DBRootExtentInfoState state); - bool operator<(const DBRootExtentInfo& entry) const; + bool operator<(const DBRootExtentInfo& entry) const; }; //------------------------------------------------------------------------------ @@ -110,131 +106,120 @@ struct DBRootExtentInfo //------------------------------------------------------------------------------ class DBRootExtentTracker { -public: + public: + /** @brief DBRootExtentTracker constructor + * @param oid Column OID of interest. + * @param colWidths Widths (in bytes) of all the columns in the table. + * @param dbRootHWMInfoColVec Column HWM, DBRoots, etc for this table. + * @param columnIdx Index (into colWidths and dbRootHWMInfoColVec) + * referencing the column that applies to this ExtentTracker. + * @param logger Logger to be used for logging messages. + */ + EXPORT DBRootExtentTracker(OID oid, const std::vector& colWidths, + const std::vector& dbRootHWMInfoColVec, + unsigned int columnIdx, Log* logger); - /** @brief DBRootExtentTracker constructor - * @param oid Column OID of interest. - * @param colWidths Widths (in bytes) of all the columns in the table. - * @param dbRootHWMInfoColVec Column HWM, DBRoots, etc for this table. - * @param columnIdx Index (into colWidths and dbRootHWMInfoColVec) - * referencing the column that applies to this ExtentTracker. - * @param logger Logger to be used for logging messages. - */ - EXPORT DBRootExtentTracker ( OID oid, - const std::vector& colWidths, - const std::vector& dbRootHWMInfoColVec, - unsigned int columnIdx, - Log* logger ); + /** @brief Select the first DBRoot/segment file to add rows to, for this PM. + * @param dbRootExtent Dbroot/segment file selected for first set of rows. + * @param bNoStartExtentOnThisPM Is starting HWM extent missing or disabled. + * If HWM extent is missing or disabled, the app will have to allo- + * cate a new extent (at the DBRoot returned in dbRootExtent)) in + * order to add any rows. + * @param bEmptyPM Is this PM void of any available extents + * @return Returns NO_ERROR if success, else returns error code. + */ + EXPORT int selectFirstSegFile(DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM, bool& bEmptyPM, + std::string& errMsg); - /** @brief Select the first DBRoot/segment file to add rows to, for this PM. - * @param dbRootExtent Dbroot/segment file selected for first set of rows. - * @param bNoStartExtentOnThisPM Is starting HWM extent missing or disabled. - * If HWM extent is missing or disabled, the app will have to allo- - * cate a new extent (at the DBRoot returned in dbRootExtent)) in - * order to add any rows. - * @param bEmptyPM Is this PM void of any available extents - * @return Returns NO_ERROR if success, else returns error code. - */ - EXPORT int selectFirstSegFile ( DBRootExtentInfo& dbRootExtent, - bool& bNoStartExtentOnThisPM, - bool& bEmptyPM, - std::string& errMsg ); + /** @brief Set up this Tracker to select the same first DBRoot/segment file + * as the reference DBRootExtentTracker that is specified from a ref column. + * + * Application code should call selectFirstSegFile for a reference column, + * and assignFirstSegFile for all other columns in the same table. + * @param refTracker Tracker object used to assign first DBRoot/segment. + * @param dbRootExtent Dbroot/segment file selected for first set of rows. + */ + EXPORT void assignFirstSegFile(const DBRootExtentTracker& refTracker, DBRootExtentInfo& dbRootExtent); - /** @brief Set up this Tracker to select the same first DBRoot/segment file - * as the reference DBRootExtentTracker that is specified from a ref column. - * - * Application code should call selectFirstSegFile for a reference column, - * and assignFirstSegFile for all other columns in the same table. - * @param refTracker Tracker object used to assign first DBRoot/segment. - * @param dbRootExtent Dbroot/segment file selected for first set of rows. - */ - EXPORT void assignFirstSegFile( const DBRootExtentTracker& refTracker, - DBRootExtentInfo& dbRootExtent ); + /** @brief Iterate/return next DBRoot to be used for the next extent. + * + * Case 1) + * If it is the "very" first extent for the specified DBRoot, then the + * applicable partition to be used for the first extent is also returned. + * + * Case 2) + * If the user moves a DBRoot to a different PM, then the next cpimport.bin + * job on the recepient PM may encounter 2 partially filled in extents. + * This differs from the norm, where we only have 1 partially filled extent + * at any given time, on a PM. When a DBRoot is moved, we may finish an ex- + * tent on 1 DBRoot, and instead of advancing to start a new extent, we ro- + * tate to the recently moved DBRoot, and have to first fill in a partilly + * filled in extent instead of adding a new extent. Case 2 is intended to + * cover this use case. + * In this case, in the middle of an import, if the next extent to receive + * rows is a partially filled in extent, then the DBRoot, partition, and + * segment number for the partial extent are returned. In addition, the + * current HWM and starting LBID for the relevant extent are returned. + * + * Case 3) + * If we are just finishing one extent and adding the next extent, then + * only the DBRoot argument is relevant, telling us where to create the + * next extent. Return value will be true. This case also applies to + * the instance where the HWM extent for the next DBRoot is disabled. + * + * @param dbRoot DBRoot for the next extent + * @param partition If first extent on dbRoot (or partial extent), then + * this is the partition # + * @param segment If partially full extent, then this is the segment # + * @param localHwm If partially full extent, then this is current HWM. + * @param startLbid If partially full extent, then this is starting LBID of + * the current HWM extent. + * + * @return Returns true if new extent needs to be allocated, returns false + * if extent is partially full, and has room for more rows. + */ + EXPORT bool nextSegFile(uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, HWM& localHwm, + BRM::LBID_t& startLbid); - /** @brief Iterate/return next DBRoot to be used for the next extent. - * - * Case 1) - * If it is the "very" first extent for the specified DBRoot, then the - * applicable partition to be used for the first extent is also returned. - * - * Case 2) - * If the user moves a DBRoot to a different PM, then the next cpimport.bin - * job on the recepient PM may encounter 2 partially filled in extents. - * This differs from the norm, where we only have 1 partially filled extent - * at any given time, on a PM. When a DBRoot is moved, we may finish an ex- - * tent on 1 DBRoot, and instead of advancing to start a new extent, we ro- - * tate to the recently moved DBRoot, and have to first fill in a partilly - * filled in extent instead of adding a new extent. Case 2 is intended to - * cover this use case. - * In this case, in the middle of an import, if the next extent to receive - * rows is a partially filled in extent, then the DBRoot, partition, and - * segment number for the partial extent are returned. In addition, the - * current HWM and starting LBID for the relevant extent are returned. - * - * Case 3) - * If we are just finishing one extent and adding the next extent, then - * only the DBRoot argument is relevant, telling us where to create the - * next extent. Return value will be true. This case also applies to - * the instance where the HWM extent for the next DBRoot is disabled. - * - * @param dbRoot DBRoot for the next extent - * @param partition If first extent on dbRoot (or partial extent), then - * this is the partition # - * @param segment If partially full extent, then this is the segment # - * @param localHwm If partially full extent, then this is current HWM. - * @param startLbid If partially full extent, then this is starting LBID of - * the current HWM extent. - * - * @return Returns true if new extent needs to be allocated, returns false - * if extent is partially full, and has room for more rows. - */ - EXPORT bool nextSegFile( uint16_t& dbRoot, - uint32_t& partition, - uint16_t& segment, - HWM& localHwm, - BRM::LBID_t& startLbid ); + /** @brief get the DBRootExtentInfo list + */ + const std::vector& getDBRootExtentList(); - /** @brief get the DBRootExtentInfo list - */ - const std::vector& getDBRootExtentList(); + /** @brief get the CurrentDBRootIdx + */ + inline int getCurrentDBRootIdx() + { + boost::mutex::scoped_lock lock(fDBRootExtTrkMutex); + return fCurrentDBRootIdx; + } - /** @brief get the CurrentDBRootIdx - */ - inline int getCurrentDBRootIdx() - { - boost::mutex::scoped_lock lock(fDBRootExtTrkMutex); - return fCurrentDBRootIdx; - } + private: + DBRootExtentInfoState determineState(int colWidth, HWM localHwm, uint64_t dbRootTotalBlocks, + int16_t status); + // Select First DBRoot/segment file on a PM having no extents for fOID + int selectFirstSegFileForEmptyPM(std::string& errMsg); + void initEmptyDBRoots(); // init ExtentList for empty DBRoots + void logFirstDBRootSelection() const; -private: - DBRootExtentInfoState determineState(int colWidth, - HWM localHwm, - uint64_t dbRootTotalBlocks, - int16_t status); - // Select First DBRoot/segment file on a PM having no extents for fOID - int selectFirstSegFileForEmptyPM ( std::string& errMsg ); - void initEmptyDBRoots(); // init ExtentList for empty DBRoots - void logFirstDBRootSelection() const; - - OID fOID; // applicable colunn OID - long long fBlksPerExtent; // blocks per extent for fOID - Log* fLog; // logger - boost::mutex fDBRootExtTrkMutex; // mutex to access fDBRootExtentList - int fCurrentDBRootIdx; // Index into fDBRootExtentList, - // DBRoot where current extent is - // being added - std::vector fDBRootExtentList; // List of current pending - // DBRoot/extents for each DBRoot - // assigned to the local PM. - bool fEmptyOrDisabledPM; // true if PM has no extents or all - // extents are disabled - bool fEmptyPM; // true if PM has no available or - // disabled extents - bool fDisabledHWM; // Did job start with disabled HWM + OID fOID; // applicable colunn OID + long long fBlksPerExtent; // blocks per extent for fOID + Log* fLog; // logger + boost::mutex fDBRootExtTrkMutex; // mutex to access fDBRootExtentList + int fCurrentDBRootIdx; // Index into fDBRootExtentList, + // DBRoot where current extent is + // being added + std::vector fDBRootExtentList; // List of current pending + // DBRoot/extents for each DBRoot + // assigned to the local PM. + bool fEmptyOrDisabledPM; // true if PM has no extents or all + // extents are disabled + bool fEmptyPM; // true if PM has no available or + // disabled extents + bool fDisabledHWM; // Did job start with disabled HWM }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // WE_DBROOTEXTENTTRACKER_H_ +#endif // WE_DBROOTEXTENTTRACKER_H_ diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index d0774ba58..9d8abcdc0 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_define.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ -* -*******************************************************************************/ + * $Id: we_define.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ + * + *******************************************************************************/ /** @file" **/ #include @@ -33,269 +33,277 @@ /** Namespace WriteEngine **/ namespace WriteEngine { - WErrorCodes::WErrorCodes() : fErrorCodes() { - fErrorCodes[ERR_UNKNOWN] = " a Generic (unknown) error"; - fErrorCodes[ERR_INVALID_PARAM] = " due to Invalid parameters"; - fErrorCodes[ERR_STRUCT_EMPTY] = " because the Structure is empty"; - fErrorCodes[ERR_VALUE_OUTOFRANGE] = " because a Value is out of range"; - fErrorCodes[ERR_PARSING] = " a Value is out of range"; - fErrorCodes[ERR_NO_MEM] = " out of memory"; - fErrorCodes[ERR_DML_LOG_NAME] = " construct DML log filename failed"; - fErrorCodes[ERR_OPEN_DML_LOG] = " open DML log file failed"; - fErrorCodes[ERR_HDFS_BACKUP] = " DML backup error in HDFS"; + fErrorCodes[ERR_UNKNOWN] = " a Generic (unknown) error"; + fErrorCodes[ERR_INVALID_PARAM] = " due to Invalid parameters"; + fErrorCodes[ERR_STRUCT_EMPTY] = " because the Structure is empty"; + fErrorCodes[ERR_VALUE_OUTOFRANGE] = " because a Value is out of range"; + fErrorCodes[ERR_PARSING] = " a Value is out of range"; + fErrorCodes[ERR_NO_MEM] = " out of memory"; + fErrorCodes[ERR_DML_LOG_NAME] = " construct DML log filename failed"; + fErrorCodes[ERR_OPEN_DML_LOG] = " open DML log file failed"; + fErrorCodes[ERR_HDFS_BACKUP] = " DML backup error in HDFS"; - // File level error - fErrorCodes[ERR_FILE_CREATE] = " The column file could not be created; it may already exist or be inaccessible."; - fErrorCodes[ERR_FILE_OPEN] = " opening a column file. The file was not found or was inaccessible."; - fErrorCodes[ERR_FILE_DELETE] = " it can not delete the file, because it does not exist or was inaccessible. "; - fErrorCodes[ERR_FILE_EXIST] = " The File already exists. "; - fErrorCodes[ERR_FILE_NOT_EXIST] = " The File does not exist. " ; - fErrorCodes[ERR_FILE_NULL] = " The FILE pointer is null." ; - fErrorCodes[ERR_FILE_WRITE] = " Error writing to a database file. "; - fErrorCodes[ERR_FILE_READ] = " Error reading from a database file. "; - fErrorCodes[ERR_FILE_SEEK] = " Error in positioning file handle. "; - fErrorCodes[ERR_FILE_READ_IMPORT] = " Error reading import source file. "; - fErrorCodes[ERR_DIR_CREATE] = " Error in creating a directory. "; - fErrorCodes[ERR_FILE_NEW_EXTENT_FBO] = " New extent FBO too high for current file. "; - fErrorCodes[ERR_FILE_FBO_NEG] = " Specified file FBO is negative. "; - fErrorCodes[ERR_FILE_TRUNCATE] = " Error truncating db file. "; - fErrorCodes[ERR_FILE_DISK_SPACE] = "Not able to add extent; adding extent " - "would exceed max file system disk usage. "; - fErrorCodes[ERR_FILE_STAT] = " Error getting stats on db file. "; - fErrorCodes[ERR_VB_FILE_NOT_EXIST] = " Version buffer file does not exists."; - fErrorCodes[ERR_FILE_FLUSH] = " Error flushing db file. "; - fErrorCodes[ERR_FILE_GLOBBING] = " Error globbing a file name. "; + // File level error + fErrorCodes[ERR_FILE_CREATE] = + " The column file could not be created; it may already exist or be inaccessible."; + fErrorCodes[ERR_FILE_OPEN] = " opening a column file. The file was not found or was inaccessible."; + fErrorCodes[ERR_FILE_DELETE] = + " it can not delete the file, because it does not exist or was inaccessible. "; + fErrorCodes[ERR_FILE_EXIST] = " The File already exists. "; + fErrorCodes[ERR_FILE_NOT_EXIST] = " The File does not exist. "; + fErrorCodes[ERR_FILE_NULL] = " The FILE pointer is null."; + fErrorCodes[ERR_FILE_WRITE] = " Error writing to a database file. "; + fErrorCodes[ERR_FILE_READ] = " Error reading from a database file. "; + fErrorCodes[ERR_FILE_SEEK] = " Error in positioning file handle. "; + fErrorCodes[ERR_FILE_READ_IMPORT] = " Error reading import source file. "; + fErrorCodes[ERR_DIR_CREATE] = " Error in creating a directory. "; + fErrorCodes[ERR_FILE_NEW_EXTENT_FBO] = " New extent FBO too high for current file. "; + fErrorCodes[ERR_FILE_FBO_NEG] = " Specified file FBO is negative. "; + fErrorCodes[ERR_FILE_TRUNCATE] = " Error truncating db file. "; + fErrorCodes[ERR_FILE_DISK_SPACE] = + "Not able to add extent; adding extent " + "would exceed max file system disk usage. "; + fErrorCodes[ERR_FILE_STAT] = " Error getting stats on db file. "; + fErrorCodes[ERR_VB_FILE_NOT_EXIST] = " Version buffer file does not exists."; + fErrorCodes[ERR_FILE_FLUSH] = " Error flushing db file. "; + fErrorCodes[ERR_FILE_GLOBBING] = " Error globbing a file name. "; - // XML level error - fErrorCodes[ERR_XML_FILE] = " An xml file error, usually because the file does not exist"; - fErrorCodes[ERR_XML_ROOT_ELEM] = " An xml file Root element error "; - fErrorCodes[ERR_XML_EMPTY] = " An Empty XML file "; - fErrorCodes[ERR_XML_PARSE] = " An XML Parsing error"; + // XML level error + fErrorCodes[ERR_XML_FILE] = " An xml file error, usually because the file does not exist"; + fErrorCodes[ERR_XML_ROOT_ELEM] = " An xml file Root element error "; + fErrorCodes[ERR_XML_EMPTY] = " An Empty XML file "; + fErrorCodes[ERR_XML_PARSE] = " An XML Parsing error"; - // table lock level error - fErrorCodes[ERR_TBLLOCK_LOCK_NOT_FOUND] = "Table is not locked."; - fErrorCodes[ERR_TBLLOCK_GET_LOCK] = "Error getting table lock."; - fErrorCodes[ERR_TBLLOCK_GET_LOCK_LOCKED] = "Table locked by another user."; - fErrorCodes[ERR_TBLLOCK_RELEASE_LOCK] = "Error releasing table lock."; - fErrorCodes[ERR_TBLLOCK_CHANGE_STATE] = "Error changing table lock state."; - fErrorCodes[ERR_TBLLOCK_GET_INFO] = "Error getting table lock info."; - fErrorCodes[ERR_TBLLOCK_LOCKID_CONFLICT] = "Table LockID for different table than expected."; + // table lock level error + fErrorCodes[ERR_TBLLOCK_LOCK_NOT_FOUND] = "Table is not locked."; + fErrorCodes[ERR_TBLLOCK_GET_LOCK] = "Error getting table lock."; + fErrorCodes[ERR_TBLLOCK_GET_LOCK_LOCKED] = "Table locked by another user."; + fErrorCodes[ERR_TBLLOCK_RELEASE_LOCK] = "Error releasing table lock."; + fErrorCodes[ERR_TBLLOCK_CHANGE_STATE] = "Error changing table lock state."; + fErrorCodes[ERR_TBLLOCK_GET_INFO] = "Error getting table lock info."; + fErrorCodes[ERR_TBLLOCK_LOCKID_CONFLICT] = "Table LockID for different table than expected."; - // DDL/DML Interface level error - fErrorCodes[ERR_STRUCT_VALUE_NOT_MATCH] = " the number of structs does not match with the number of value sets"; - fErrorCodes[ERR_ROWID_VALUE_NOT_MATCH] = " the number of rowids does not match with the number of values"; - fErrorCodes[ERR_TBL_SYSCAT_ERROR] = "Error occured when querying systemcatalog."; + // DDL/DML Interface level error + fErrorCodes[ERR_STRUCT_VALUE_NOT_MATCH] = + " the number of structs does not match with the number of value sets"; + fErrorCodes[ERR_ROWID_VALUE_NOT_MATCH] = " the number of rowids does not match with the number of values"; + fErrorCodes[ERR_TBL_SYSCAT_ERROR] = "Error occured when querying systemcatalog."; - // index error - fErrorCodes[ERR_IDX_TREE_MOVE_ENTRY] = " an error in moving part of an index tree to a new subblock"; - fErrorCodes[ERR_IDX_TREE_INVALID_TYPE] = " an Invalid index tree entry type"; - fErrorCodes[ERR_IDX_TREE_BITTEST_VAL] = " a Wrong bit test value in the index tree entry"; - fErrorCodes[ERR_IDX_TREE_INVALID_LEVEL] = " an Invalid testbit index tree level"; - fErrorCodes[ERR_IDX_TREE_INVALID_GRP] = "an Invalid index tree group type "; - fErrorCodes[ERR_IDX_TREE_LISTPTR_CHANGE] = " an index tree List pointer change"; - //index list error - fErrorCodes[ERR_IDX_LIST_INVALID_ADDHDR] = " a Create indexlist header error"; - fErrorCodes[ERR_IDX_LIST_INVALID_UPDATE] = " an pdate Index List error "; - fErrorCodes[ERR_IDX_LIST_INVALID_DELETE] = " a Delete rowid in indexlist err"; - fErrorCodes[ERR_IDX_LIST_INVALID_KEY] = " an Invalid index listbppseeder.cpp Key passed"; - fErrorCodes[ERR_IDX_LIST_GET_RID_ARRARY] = " an index list RID array"; - fErrorCodes[ERR_IDX_LIST_WRONG_KEY ] = " a not matched Key passed to an index list"; - fErrorCodes[ERR_IDX_LIST_HDR_EMPTY] = " an empty index list header"; - fErrorCodes[ERR_IDX_LIST_GET_SEGMT] = " in an index list Get Segment"; - fErrorCodes[ERR_IDX_LIST_WRONG_LBID_WRITE] = " an index list incorrect LBID write"; - fErrorCodes[ERR_IDX_LIST_UPDATE_SUB] = " in an index list update sub"; - fErrorCodes[ERR_IDX_LIST_UPDATE_NARRAY] = " in an index list update narray"; - fErrorCodes[ERR_IDX_LIST_LAST_FBO_NEG] = " the last index list FBO neg"; - fErrorCodes[ERR_IDX_LIST_INIT_NEW_BLKS] = " in an index list initialize new blocks"; - fErrorCodes[ERR_IDX_LIST_INIT_LINK_BLKS] = " in an index list initialize link blocks"; - fErrorCodes[ERR_IDX_LIST_UPDATE_COUNT] = " in an index list update count"; - fErrorCodes[ERR_IDX_LIST_SET_NEXT_LBID] = " in an index list set next LBID"; - fErrorCodes[ERR_IDX_LIST_INVALID_LBID] = "an index list invalid LBID"; - fErrorCodes[ERR_IDX_LIST_INVALID_BLK_READ] = " in an index list invalid LBID read"; - fErrorCodes[ERR_IDX_LIST_UPDATE_HDR_COUNT] = " in an index list update header count"; - fErrorCodes[ERR_IDX_LIST_WRONG_BLK] = " an index list wrong block"; - fErrorCodes[ERR_IDX_LIST_WRONG_TYPE] = " an index list wrong type"; - fErrorCodes[ERR_IDX_LIST_GET_COUNT] = " in an index list get count"; - fErrorCodes[ERR_IDX_LIST_GET_NEXT] = " in an index list get next"; - fErrorCodes[ERR_IDX_LIST_GET_PARENT] = " in an index list get parent"; - fErrorCodes[ERR_IDX_LIST_GET_SUB_BLK] = " in an index list get sub block"; - fErrorCodes[ERR_IDX_LIST_INVALID_UP_HDR] = " an invalid Update Index List header "; - fErrorCodes[ERR_IDX_LIST_INVALID_ADD_LIST] = " an invalid add Index List"; - fErrorCodes[ERR_IDX_LIST_INVALID_UP] = " an invalid Update Index List"; + // index error + fErrorCodes[ERR_IDX_TREE_MOVE_ENTRY] = " an error in moving part of an index tree to a new subblock"; + fErrorCodes[ERR_IDX_TREE_INVALID_TYPE] = " an Invalid index tree entry type"; + fErrorCodes[ERR_IDX_TREE_BITTEST_VAL] = " a Wrong bit test value in the index tree entry"; + fErrorCodes[ERR_IDX_TREE_INVALID_LEVEL] = " an Invalid testbit index tree level"; + fErrorCodes[ERR_IDX_TREE_INVALID_GRP] = "an Invalid index tree group type "; + fErrorCodes[ERR_IDX_TREE_LISTPTR_CHANGE] = " an index tree List pointer change"; + // index list error + fErrorCodes[ERR_IDX_LIST_INVALID_ADDHDR] = " a Create indexlist header error"; + fErrorCodes[ERR_IDX_LIST_INVALID_UPDATE] = " an pdate Index List error "; + fErrorCodes[ERR_IDX_LIST_INVALID_DELETE] = " a Delete rowid in indexlist err"; + fErrorCodes[ERR_IDX_LIST_INVALID_KEY] = " an Invalid index listbppseeder.cpp Key passed"; + fErrorCodes[ERR_IDX_LIST_GET_RID_ARRARY] = " an index list RID array"; + fErrorCodes[ERR_IDX_LIST_WRONG_KEY] = " a not matched Key passed to an index list"; + fErrorCodes[ERR_IDX_LIST_HDR_EMPTY] = " an empty index list header"; + fErrorCodes[ERR_IDX_LIST_GET_SEGMT] = " in an index list Get Segment"; + fErrorCodes[ERR_IDX_LIST_WRONG_LBID_WRITE] = " an index list incorrect LBID write"; + fErrorCodes[ERR_IDX_LIST_UPDATE_SUB] = " in an index list update sub"; + fErrorCodes[ERR_IDX_LIST_UPDATE_NARRAY] = " in an index list update narray"; + fErrorCodes[ERR_IDX_LIST_LAST_FBO_NEG] = " the last index list FBO neg"; + fErrorCodes[ERR_IDX_LIST_INIT_NEW_BLKS] = " in an index list initialize new blocks"; + fErrorCodes[ERR_IDX_LIST_INIT_LINK_BLKS] = " in an index list initialize link blocks"; + fErrorCodes[ERR_IDX_LIST_UPDATE_COUNT] = " in an index list update count"; + fErrorCodes[ERR_IDX_LIST_SET_NEXT_LBID] = " in an index list set next LBID"; + fErrorCodes[ERR_IDX_LIST_INVALID_LBID] = "an index list invalid LBID"; + fErrorCodes[ERR_IDX_LIST_INVALID_BLK_READ] = " in an index list invalid LBID read"; + fErrorCodes[ERR_IDX_LIST_UPDATE_HDR_COUNT] = " in an index list update header count"; + fErrorCodes[ERR_IDX_LIST_WRONG_BLK] = " an index list wrong block"; + fErrorCodes[ERR_IDX_LIST_WRONG_TYPE] = " an index list wrong type"; + fErrorCodes[ERR_IDX_LIST_GET_COUNT] = " in an index list get count"; + fErrorCodes[ERR_IDX_LIST_GET_NEXT] = " in an index list get next"; + fErrorCodes[ERR_IDX_LIST_GET_PARENT] = " in an index list get parent"; + fErrorCodes[ERR_IDX_LIST_GET_SUB_BLK] = " in an index list get sub block"; + fErrorCodes[ERR_IDX_LIST_INVALID_UP_HDR] = " an invalid Update Index List header "; + fErrorCodes[ERR_IDX_LIST_INVALID_ADD_LIST] = " an invalid add Index List"; + fErrorCodes[ERR_IDX_LIST_INVALID_UP] = " an invalid Update Index List"; - //freemgr error - fErrorCodes[ERR_FM_ASSIGN_ERR] = " in an assignment"; - fErrorCodes[ERR_FM_RELEASE_ERR] = " in a release"; - fErrorCodes[ERR_FM_BAD_FBO] = " an invalid File Block Offset"; - fErrorCodes[ERR_FM_BAD_TYPE] = "an invalid type that must be pointer or list"; - fErrorCodes[ERR_FM_NO_SPACE] = " that No blocks are available"; - fErrorCodes[ERR_FM_EXTEND] = " while extending a file"; + // freemgr error + fErrorCodes[ERR_FM_ASSIGN_ERR] = " in an assignment"; + fErrorCodes[ERR_FM_RELEASE_ERR] = " in a release"; + fErrorCodes[ERR_FM_BAD_FBO] = " an invalid File Block Offset"; + fErrorCodes[ERR_FM_BAD_TYPE] = "an invalid type that must be pointer or list"; + fErrorCodes[ERR_FM_NO_SPACE] = " that No blocks are available"; + fErrorCodes[ERR_FM_EXTEND] = " while extending a file"; - // Dictionary error - fErrorCodes[ERR_DICT_NO_SPACE_INSERT] = " no space for a dictionary insert"; - fErrorCodes[ERR_DICT_SIZE_GT_2G] = " the dictionary size was > 2GB"; - fErrorCodes[ERR_DICT_NO_OP_DELETE] = " in the dictionary no op delete"; - fErrorCodes[ERR_DICT_NO_OFFSET_DELETE] = " a dictionary bad Delete offset"; - fErrorCodes[ERR_DICT_INVALID_HDR] = " a dictionary bad Delete Hdr"; - fErrorCodes[ERR_DICT_ZERO_LEN] = " a dictionary zero len"; - fErrorCodes[ERR_DICT_TOKEN_NOT_FOUND] = " a dictionary token not found"; - fErrorCodes[ERR_DICT_FILE_NOT_FOUND] = " a dictionary file not found"; - fErrorCodes[ERR_DICT_BAD_TOKEN_LBID] = " a dictionary token lbid is bad"; - fErrorCodes[ERR_DICT_BAD_TOKEN_OP] = " a dictionary token op is bad"; + // Dictionary error + fErrorCodes[ERR_DICT_NO_SPACE_INSERT] = " no space for a dictionary insert"; + fErrorCodes[ERR_DICT_SIZE_GT_2G] = " the dictionary size was > 2GB"; + fErrorCodes[ERR_DICT_NO_OP_DELETE] = " in the dictionary no op delete"; + fErrorCodes[ERR_DICT_NO_OFFSET_DELETE] = " a dictionary bad Delete offset"; + fErrorCodes[ERR_DICT_INVALID_HDR] = " a dictionary bad Delete Hdr"; + fErrorCodes[ERR_DICT_ZERO_LEN] = " a dictionary zero len"; + fErrorCodes[ERR_DICT_TOKEN_NOT_FOUND] = " a dictionary token not found"; + fErrorCodes[ERR_DICT_FILE_NOT_FOUND] = " a dictionary file not found"; + fErrorCodes[ERR_DICT_BAD_TOKEN_LBID] = " a dictionary token lbid is bad"; + fErrorCodes[ERR_DICT_BAD_TOKEN_OP] = " a dictionary token op is bad"; - // Bulk error - fErrorCodes[ERR_BULK_MAX_ERR_NUM] = " the Maximum number of error rows reached"; - fErrorCodes[ERR_BULK_DATA_COL_NUM] = " the total number of data column not match with column definitions"; - fErrorCodes[ERR_BULK_SEND_MSG_ERR] = " in a bulk load send msg"; - fErrorCodes[ERR_BULK_MISSING_EXTENT_ENTRY] = " missing Extent Entry when trying to save LBID info for CP"; - fErrorCodes[ERR_BULK_MISSING_EXTENT_ROW] = " missing Extent Row when trying to save LBID info for CP"; - fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer."; - fErrorCodes[ERR_BULK_DBROOT_CHANGE] = " Local PM DBRoot settings changed during bulk load."; - fErrorCodes[ERR_BULK_ROLLBACK_MISS_ROOT] = " Mode3 automatic rollback not performed. DBRoot missing."; - fErrorCodes[ERR_BULK_ROLLBACK_SEG_LIST] = " Error building segment file list in a directory."; - fErrorCodes[ERR_BULK_BINARY_PARTIAL_REC] = " Binary import did not end on fixed length record boundary."; - fErrorCodes[ERR_BULK_BINARY_IGNORE_FLD] = " tag not supported for binary imports."; + // Bulk error + fErrorCodes[ERR_BULK_MAX_ERR_NUM] = " the Maximum number of error rows reached"; + fErrorCodes[ERR_BULK_DATA_COL_NUM] = " the total number of data column not match with column definitions"; + fErrorCodes[ERR_BULK_SEND_MSG_ERR] = " in a bulk load send msg"; + fErrorCodes[ERR_BULK_MISSING_EXTENT_ENTRY] = " missing Extent Entry when trying to save LBID info for CP"; + fErrorCodes[ERR_BULK_MISSING_EXTENT_ROW] = " missing Extent Row when trying to save LBID info for CP"; + fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer."; + fErrorCodes[ERR_BULK_DBROOT_CHANGE] = " Local PM DBRoot settings changed during bulk load."; + fErrorCodes[ERR_BULK_ROLLBACK_MISS_ROOT] = " Mode3 automatic rollback not performed. DBRoot missing."; + fErrorCodes[ERR_BULK_ROLLBACK_SEG_LIST] = " Error building segment file list in a directory."; + fErrorCodes[ERR_BULK_BINARY_PARTIAL_REC] = " Binary import did not end on fixed length record boundary."; + fErrorCodes[ERR_BULK_BINARY_IGNORE_FLD] = " tag not supported for binary imports."; - // BRM error - fErrorCodes[ERR_BRM_LOOKUP_LBID] = " a BRM Lookup LBID error."; - fErrorCodes[ERR_BRM_LOOKUP_FBO] = " a BRM Lookup FBO error."; - fErrorCodes[ERR_BRM_ALLOC_EXTEND] = " a BRM Allocate extent error."; - fErrorCodes[ERR_BRM_COMMIT] = " a BRM Commit error."; - fErrorCodes[ERR_BRM_ROLLBACK] = " a BRM Rollback error."; - fErrorCodes[ERR_BRM_GET_UNCOMM_LBID] = " a BRM get uncommitted lbid list error."; - fErrorCodes[ERR_BRM_DEL_OID] = " a BRM Delete oid error."; - fErrorCodes[ERR_BRM_BEGIN_COPY] = " a BRM Begin copy error."; - fErrorCodes[ERR_BRM_END_COPY] = " a BRM End copy error."; - fErrorCodes[ERR_BRM_GET_HWM] = " a BRM get hwm error."; - fErrorCodes[ERR_BRM_SET_HWM] = " a BRM Set hwm error."; - fErrorCodes[ERR_BRM_WR_VB_ENTRY] = " a BRM VB entry error."; - fErrorCodes[ERR_BRM_VB_COPY_READ] = " a BRM VB copy read error."; - fErrorCodes[ERR_BRM_VB_COPY_SEEK_DB] = " a BRM VB copy seek error against DB file."; - fErrorCodes[ERR_BRM_VB_COPY_SEEK_VB] = " a BRM VB copy seek error against VB file."; - fErrorCodes[ERR_BRM_VB_COPY_WRITE] = " a BRM VB copy write."; - fErrorCodes[ERR_BRM_DEAD_LOCK] = " a BRM DEAD lock error."; - fErrorCodes[ERR_BRM_MARK_INVALID] = " a BRM Mark extent invalid error from casual paritioning."; - fErrorCodes[ERR_BRM_SAVE_STATE] = " a BRM Save state error."; - fErrorCodes[ERR_BRM_GET_START_EXTENT] = " a BRM get start Extent error."; - fErrorCodes[ERR_BRM_VB_OVERFLOW] = "BRM block version buffer overflow error."; - fErrorCodes[ERR_BRM_READ_ONLY] = "BRM is in read-only state."; - fErrorCodes[ERR_BRM_GET_READ_WRITE] = "BRM error getting read-write state."; - fErrorCodes[ERR_BRM_BULK_RB_COLUMN] = "BRM error performing bulk rollback of column extents."; - fErrorCodes[ERR_BRM_BULK_RB_DCTNRY] = "BRM error performing bulk rollback of dictionary store extents."; - fErrorCodes[ERR_BRM_DELETE_EXTENT_COLUMN] = "BRM error deleting column extents."; - fErrorCodes[ERR_BRM_DELETE_EXTENT_DCTNRY] = "BRM error deleting dictionary extents."; - fErrorCodes[ERR_BRM_TAKE_SNAPSHOT] = "BRM error requesting snapshot of BRM state."; - fErrorCodes[ERR_BRM_LOOKUP_START_LBID] = "BRM start LBID lookup error."; - fErrorCodes[ERR_BRM_BULK_UPDATE] = "BRM error executing bulk update of HWM and CP."; - fErrorCodes[ERR_BRM_GET_EXT_STATE] = "BRM error getting segment file extent state."; - fErrorCodes[ERR_EXTENTMAP_LOOKUP] = " a extent map Lookup error."; - fErrorCodes[ERR_BRM_LOOKUP_VERSION] = " a vssLookup version info error."; - fErrorCodes[ERR_BRM_LOOKUP_LBID_RANGES] = " BRM error getting LBID ranges."; - fErrorCodes[ERR_BRM_HWMS_NOT_EQUAL] = " HWMs for same width columns not equal. "; - fErrorCodes[ERR_BRM_HWMS_OUT_OF_SYNC] = " HWMs for different width columns not in sync. "; - fErrorCodes[ERR_BRM_DBROOT_HWMS] = " BRM error getting HWMs for DBRoots. "; - fErrorCodes[ERR_BRM_NETWORK] = " Network error in DBRM call. "; - fErrorCodes[ERR_BRM_READONLY] = " DBRM is read only. "; - fErrorCodes[ERR_INVALID_VBOID] = " The VB oid is invalid "; - fErrorCodes[ERR_BRM_SET_EXTENTS_CP] = " BRM error setting extents min/max "; - fErrorCodes[ERR_BRM_SHUTDOWN] = " The system is being shutdown "; - fErrorCodes[ERR_BRM_GET_SHUTDOWN] = " BRM error get the system shutdown flag "; - fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode"; - fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag "; - fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe"; - fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width"; - fErrorCodes[ERR_BRM_GET_EXTENT_CP] = " BRM error getting extent max/min"; + // BRM error + fErrorCodes[ERR_BRM_LOOKUP_LBID] = " a BRM Lookup LBID error."; + fErrorCodes[ERR_BRM_LOOKUP_FBO] = " a BRM Lookup FBO error."; + fErrorCodes[ERR_BRM_ALLOC_EXTEND] = " a BRM Allocate extent error."; + fErrorCodes[ERR_BRM_COMMIT] = " a BRM Commit error."; + fErrorCodes[ERR_BRM_ROLLBACK] = " a BRM Rollback error."; + fErrorCodes[ERR_BRM_GET_UNCOMM_LBID] = " a BRM get uncommitted lbid list error."; + fErrorCodes[ERR_BRM_DEL_OID] = " a BRM Delete oid error."; + fErrorCodes[ERR_BRM_BEGIN_COPY] = " a BRM Begin copy error."; + fErrorCodes[ERR_BRM_END_COPY] = " a BRM End copy error."; + fErrorCodes[ERR_BRM_GET_HWM] = " a BRM get hwm error."; + fErrorCodes[ERR_BRM_SET_HWM] = " a BRM Set hwm error."; + fErrorCodes[ERR_BRM_WR_VB_ENTRY] = " a BRM VB entry error."; + fErrorCodes[ERR_BRM_VB_COPY_READ] = " a BRM VB copy read error."; + fErrorCodes[ERR_BRM_VB_COPY_SEEK_DB] = " a BRM VB copy seek error against DB file."; + fErrorCodes[ERR_BRM_VB_COPY_SEEK_VB] = " a BRM VB copy seek error against VB file."; + fErrorCodes[ERR_BRM_VB_COPY_WRITE] = " a BRM VB copy write."; + fErrorCodes[ERR_BRM_DEAD_LOCK] = " a BRM DEAD lock error."; + fErrorCodes[ERR_BRM_MARK_INVALID] = " a BRM Mark extent invalid error from casual paritioning."; + fErrorCodes[ERR_BRM_SAVE_STATE] = " a BRM Save state error."; + fErrorCodes[ERR_BRM_GET_START_EXTENT] = " a BRM get start Extent error."; + fErrorCodes[ERR_BRM_VB_OVERFLOW] = "BRM block version buffer overflow error."; + fErrorCodes[ERR_BRM_READ_ONLY] = "BRM is in read-only state."; + fErrorCodes[ERR_BRM_GET_READ_WRITE] = "BRM error getting read-write state."; + fErrorCodes[ERR_BRM_BULK_RB_COLUMN] = "BRM error performing bulk rollback of column extents."; + fErrorCodes[ERR_BRM_BULK_RB_DCTNRY] = "BRM error performing bulk rollback of dictionary store extents."; + fErrorCodes[ERR_BRM_DELETE_EXTENT_COLUMN] = "BRM error deleting column extents."; + fErrorCodes[ERR_BRM_DELETE_EXTENT_DCTNRY] = "BRM error deleting dictionary extents."; + fErrorCodes[ERR_BRM_TAKE_SNAPSHOT] = "BRM error requesting snapshot of BRM state."; + fErrorCodes[ERR_BRM_LOOKUP_START_LBID] = "BRM start LBID lookup error."; + fErrorCodes[ERR_BRM_BULK_UPDATE] = "BRM error executing bulk update of HWM and CP."; + fErrorCodes[ERR_BRM_GET_EXT_STATE] = "BRM error getting segment file extent state."; + fErrorCodes[ERR_EXTENTMAP_LOOKUP] = " a extent map Lookup error."; + fErrorCodes[ERR_BRM_LOOKUP_VERSION] = " a vssLookup version info error."; + fErrorCodes[ERR_BRM_LOOKUP_LBID_RANGES] = " BRM error getting LBID ranges."; + fErrorCodes[ERR_BRM_HWMS_NOT_EQUAL] = " HWMs for same width columns not equal. "; + fErrorCodes[ERR_BRM_HWMS_OUT_OF_SYNC] = " HWMs for different width columns not in sync. "; + fErrorCodes[ERR_BRM_DBROOT_HWMS] = " BRM error getting HWMs for DBRoots. "; + fErrorCodes[ERR_BRM_NETWORK] = " Network error in DBRM call. "; + fErrorCodes[ERR_BRM_READONLY] = " DBRM is read only. "; + fErrorCodes[ERR_INVALID_VBOID] = " The VB oid is invalid "; + fErrorCodes[ERR_BRM_SET_EXTENTS_CP] = " BRM error setting extents min/max "; + fErrorCodes[ERR_BRM_SHUTDOWN] = " The system is being shutdown "; + fErrorCodes[ERR_BRM_GET_SHUTDOWN] = " BRM error get the system shutdown flag "; + fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode"; + fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag "; + fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe"; + fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width"; + fErrorCodes[ERR_BRM_GET_EXTENT_CP] = " BRM error getting extent max/min"; - // DM error - fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error"; + // DM error + fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error"; - // Cache error - fErrorCodes[ERR_CACHE_KEY_EXIST ] = " a Cache key exists"; - fErrorCodes[ERR_CACHE_KEY_NOT_EXIST] = " a Cache key does not exist"; - fErrorCodes[ERR_NULL_BLOCK] = " a Block is NULL"; - fErrorCodes[ERR_FREE_LIST_EMPTY] = " a Free list is empty"; + // Cache error + fErrorCodes[ERR_CACHE_KEY_EXIST] = " a Cache key exists"; + fErrorCodes[ERR_CACHE_KEY_NOT_EXIST] = " a Cache key does not exist"; + fErrorCodes[ERR_NULL_BLOCK] = " a Block is NULL"; + fErrorCodes[ERR_FREE_LIST_EMPTY] = " a Free list is empty"; - // Compression error - fErrorCodes[ERR_COMP_COMPRESS] = " Error in compressing data. "; - fErrorCodes[ERR_COMP_UNCOMPRESS] = " Error in uncompressing data. "; - fErrorCodes[ERR_COMP_PARSE_HDRS] = " Error parsing compression headers. "; - fErrorCodes[ERR_COMP_VERIFY_HDRS] = " Error verifying compression headers. "; - fErrorCodes[ERR_COMP_PAD_DATA] = " Error in padding compressed data. "; - fErrorCodes[ERR_COMP_READ_BLOCK] = " Error in reading a data block. "; - fErrorCodes[ERR_COMP_SAVE_BLOCK] = " Error in saving a data block. "; - fErrorCodes[ERR_COMP_WRONG_PTR] = " Invalid pointer in compression headers. "; - fErrorCodes[ERR_COMP_FILE_NOT_FOUND] = " Error searching for a compressed file. "; - fErrorCodes[ERR_COMP_CHUNK_NOT_FOUND] = " Error searching for a compressed chunk. "; - fErrorCodes[ERR_COMP_UNAVAIL_TYPE] = " Unavailable compressino type. "; - fErrorCodes[ERR_COMP_REMOVE_FILE] = " Failed to remove a file. "; - fErrorCodes[ERR_COMP_RENAME_FILE] = " Failed to rename a file. "; - fErrorCodes[ERR_COMP_OPEN_FILE] = " Failed to open a compressed data file. "; - fErrorCodes[ERR_COMP_SET_OFFSET] = " Failed to set offset in a compressed data file. "; - fErrorCodes[ERR_COMP_READ_FILE] = " Failed to read from a compressed data file. "; - fErrorCodes[ERR_COMP_WRITE_FILE] = " Failed to write to a compresssed data file. "; - fErrorCodes[ERR_COMP_CLOSE_FILE] = " Failed to close a compressed data file. "; - fErrorCodes[ERR_COMP_TRUNCATE_ZERO] = " Attempting to truncate compressed file to 0 bytes. "; + // Compression error + fErrorCodes[ERR_COMP_COMPRESS] = " Error in compressing data. "; + fErrorCodes[ERR_COMP_UNCOMPRESS] = " Error in uncompressing data. "; + fErrorCodes[ERR_COMP_PARSE_HDRS] = " Error parsing compression headers. "; + fErrorCodes[ERR_COMP_VERIFY_HDRS] = " Error verifying compression headers. "; + fErrorCodes[ERR_COMP_PAD_DATA] = " Error in padding compressed data. "; + fErrorCodes[ERR_COMP_READ_BLOCK] = " Error in reading a data block. "; + fErrorCodes[ERR_COMP_SAVE_BLOCK] = " Error in saving a data block. "; + fErrorCodes[ERR_COMP_WRONG_PTR] = " Invalid pointer in compression headers. "; + fErrorCodes[ERR_COMP_FILE_NOT_FOUND] = " Error searching for a compressed file. "; + fErrorCodes[ERR_COMP_CHUNK_NOT_FOUND] = " Error searching for a compressed chunk. "; + fErrorCodes[ERR_COMP_UNAVAIL_TYPE] = " Unavailable compressino type. "; + fErrorCodes[ERR_COMP_REMOVE_FILE] = " Failed to remove a file. "; + fErrorCodes[ERR_COMP_RENAME_FILE] = " Failed to rename a file. "; + fErrorCodes[ERR_COMP_OPEN_FILE] = " Failed to open a compressed data file. "; + fErrorCodes[ERR_COMP_SET_OFFSET] = " Failed to set offset in a compressed data file. "; + fErrorCodes[ERR_COMP_READ_FILE] = " Failed to read from a compressed data file. "; + fErrorCodes[ERR_COMP_WRITE_FILE] = " Failed to write to a compresssed data file. "; + fErrorCodes[ERR_COMP_CLOSE_FILE] = " Failed to close a compressed data file. "; + fErrorCodes[ERR_COMP_TRUNCATE_ZERO] = " Attempting to truncate compressed file to 0 bytes. "; - // Auto-increment error - fErrorCodes[ERR_AUTOINC_GEN_EXCEED_MAX] = " Generated auto-increment value " - "exceeds maximum value for the column type."; - fErrorCodes[ERR_AUTOINC_USER_OUT_OF_RANGE] = " User specified auto-" - "increment value is out of range for the column type."; - fErrorCodes[ERR_AUTOINC_TABLE_NAME] = " Invalid schema/tablename for auto increment. "; - fErrorCodes[ERR_AUTOINC_INIT1] = " Unable to initialize auto-increment value. "; - fErrorCodes[ERR_AUTOINC_INIT2] = " Unable to initialize auto-increment value. Unknown exception. "; - fErrorCodes[ERR_AUTOINC_RID] = " Failed to get row information from calpontsystemcatalog."; - fErrorCodes[ERR_AUTOINC_START_SEQ] = " Unable to setup AI sequence in BRM."; - fErrorCodes[ERR_AUTOINC_GET_RANGE] = " Unable to reserve AI range from BRM."; - fErrorCodes[ERR_AUTOINC_GET_LOCK] = " Unable to lock AI column in BRM."; - fErrorCodes[ERR_AUTOINC_REL_LOCK] = " Unable to release AI column in BRM."; - fErrorCodes[ERR_AUTOINC_UPDATE] = " Unable to update nextValue in system catalog."; + // Auto-increment error + fErrorCodes[ERR_AUTOINC_GEN_EXCEED_MAX] = + " Generated auto-increment value " + "exceeds maximum value for the column type."; + fErrorCodes[ERR_AUTOINC_USER_OUT_OF_RANGE] = + " User specified auto-" + "increment value is out of range for the column type."; + fErrorCodes[ERR_AUTOINC_TABLE_NAME] = " Invalid schema/tablename for auto increment. "; + fErrorCodes[ERR_AUTOINC_INIT1] = " Unable to initialize auto-increment value. "; + fErrorCodes[ERR_AUTOINC_INIT2] = " Unable to initialize auto-increment value. Unknown exception. "; + fErrorCodes[ERR_AUTOINC_RID] = " Failed to get row information from calpontsystemcatalog."; + fErrorCodes[ERR_AUTOINC_START_SEQ] = " Unable to setup AI sequence in BRM."; + fErrorCodes[ERR_AUTOINC_GET_RANGE] = " Unable to reserve AI range from BRM."; + fErrorCodes[ERR_AUTOINC_GET_LOCK] = " Unable to lock AI column in BRM."; + fErrorCodes[ERR_AUTOINC_REL_LOCK] = " Unable to release AI column in BRM."; + fErrorCodes[ERR_AUTOINC_UPDATE] = " Unable to update nextValue in system catalog."; - // Block cache flush error - fErrorCodes[ERR_BLKCACHE_FLUSH_LIST] = " Failed to flush list of blocks from PrimProc cache. "; + // Block cache flush error + fErrorCodes[ERR_BLKCACHE_FLUSH_LIST] = " Failed to flush list of blocks from PrimProc cache. "; - // Backup bulk meta data file error - fErrorCodes[ERR_METADATABKUP_FILE_RENAME] = " Unable to rename temporary bulk meta data file. "; - fErrorCodes[ERR_METADATABKUP_COMP_PARSE_HDRS] = " Error parsing compression headers in bulk backup file. "; - fErrorCodes[ERR_METADATABKUP_COMP_VERIFY_HDRS] = " Error verifying compression headers in bulk backup file. "; - fErrorCodes[ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND] = " Error searching for compressed chunk in db file being backed up. "; - fErrorCodes[ERR_METADATABKUP_COMP_OPEN_BULK_BKUP] = " Error opening compressed chunk in bulk backup file. "; - fErrorCodes[ERR_METADATABKUP_COMP_WRITE_BULK_BKUP] = " Error writing compressed chunk to bulk backup file. "; - fErrorCodes[ERR_METADATABKUP_COMP_READ_BULK_BKUP] = " Error reading compressed chunk from bulk backup file. "; - fErrorCodes[ERR_METADATABKUP_COMP_RENAME] = " Unable to rename compressed chunk bulk backup file. "; + // Backup bulk meta data file error + fErrorCodes[ERR_METADATABKUP_FILE_RENAME] = " Unable to rename temporary bulk meta data file. "; + fErrorCodes[ERR_METADATABKUP_COMP_PARSE_HDRS] = " Error parsing compression headers in bulk backup file. "; + fErrorCodes[ERR_METADATABKUP_COMP_VERIFY_HDRS] = + " Error verifying compression headers in bulk backup file. "; + fErrorCodes[ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND] = + " Error searching for compressed chunk in db file being backed up. "; + fErrorCodes[ERR_METADATABKUP_COMP_OPEN_BULK_BKUP] = " Error opening compressed chunk in bulk backup file. "; + fErrorCodes[ERR_METADATABKUP_COMP_WRITE_BULK_BKUP] = + " Error writing compressed chunk to bulk backup file. "; + fErrorCodes[ERR_METADATABKUP_COMP_READ_BULK_BKUP] = + " Error reading compressed chunk from bulk backup file. "; + fErrorCodes[ERR_METADATABKUP_COMP_RENAME] = " Unable to rename compressed chunk bulk backup file. "; } std::string WErrorCodes::errorString(int code) { - // Look for error message overrides from system-wide error messages - switch (code) + // Look for error message overrides from system-wide error messages + switch (code) + { + case ERR_FILE_DISK_SPACE: { - case ERR_FILE_DISK_SPACE: - { - logging::Message::Args args; - std::string msgArg; // empty str arg; no extra info in this context - args.add( msgArg ); - return logging::IDBErrorInfo::instance()->errorMsg( - logging::ERR_EXTENT_DISK_SPACE, args); - break; - } + logging::Message::Args args; + std::string msgArg; // empty str arg; no extra info in this context + args.add(msgArg); + return logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_EXTENT_DISK_SPACE, args); + break; } + } - int brmRc = BRMWrapper::getBrmRc(); + int brmRc = BRMWrapper::getBrmRc(); - if (brmRc == BRM::ERR_OK) - return (fErrorCodes[code]); + if (brmRc == BRM::ERR_OK) + return (fErrorCodes[code]); - std::string errMsg( fErrorCodes[code] ); - std::string brmMsg; - errMsg += " [BRM error status: "; - BRM::errString(brmRc, brmMsg); - errMsg += brmMsg; - errMsg += "]"; + std::string errMsg(fErrorCodes[code]); + std::string brmMsg; + errMsg += " [BRM error status: "; + BRM::errString(brmRc, brmMsg); + errMsg += brmMsg; + errMsg += "]"; - return errMsg; + return errMsg; } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index bf987c31c..aaadc9367 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -38,364 +38,376 @@ namespace WriteEngine { // Max column size is 16 bytes since MCOL-641. However left this value // for backward compatibility -const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column -const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val -const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val -const int MAX_DB_DIR_LEVEL = 6; // Max lvl of db dir struct -const int MAX_DB_DIR_NAME_SIZE = 20; // Max len of db dir size -const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file -const int BYTE_PER_BLOCK = 8192; // Num bytes per data block -const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block -const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block -const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation -const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col. +const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column +const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val +const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val +const int MAX_DB_DIR_LEVEL = 6; // Max lvl of db dir struct +const int MAX_DB_DIR_NAME_SIZE = 20; // Max len of db dir size +const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file +const int BYTE_PER_BLOCK = 8192; // Num bytes per data block +const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block +const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block +const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation +const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col. // Num rows reserved to disk for 'initial' extent -const int FILE_NAME_SIZE = 200; // Max size of file name -const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count +const int FILE_NAME_SIZE = 200; // Max size of file name +const long long MAX_ALLOW_ERROR_COUNT = 100000; // Max allowable error count //-------------------------------------------------------------------------- // Dictionary related constants //-------------------------------------------------------------------------- -const uint16_t DCTNRY_END_HEADER = 0xffff ; // end of header -const uint64_t NOT_USED_PTR = 0x0 ; // not continuous ptr -const int HDR_UNIT_SIZE = 2; // hdr unit size -const int NEXT_PTR_BYTES = 8; // const ptr size -const int MAX_OP_COUNT = 1024; // op max size -const int DCTNRY_HEADER_SIZE = 14; // header total size -const int MAX_STRING_CACHE_SIZE = 1000; +const uint16_t DCTNRY_END_HEADER = 0xffff; // end of header +const uint64_t NOT_USED_PTR = 0x0; // not continuous ptr +const int HDR_UNIT_SIZE = 2; // hdr unit size +const int NEXT_PTR_BYTES = 8; // const ptr size +const int MAX_OP_COUNT = 1024; // op max size +const int DCTNRY_HEADER_SIZE = 14; // header total size +const int MAX_STRING_CACHE_SIZE = 1000; // End of Dictionary related constants -const int COLPOSPAIR_NULL_TOKEN_OFFSET = -1; // offset value denoting a null token -const uint32_t BULK_SYSCAT_SESSION_ID = 0; // SessionID for syscat queries +const int COLPOSPAIR_NULL_TOKEN_OFFSET = -1; // offset value denoting a null token +const uint32_t BULK_SYSCAT_SESSION_ID = 0; // SessionID for syscat queries -const char COL_TYPE_DICT = 'D'; // Dictionary type +const char COL_TYPE_DICT = 'D'; // Dictionary type -const uint64_t INVALID_LBID = 0xFFFFFFFFFULL; // 2**36 - 1 +const uint64_t INVALID_LBID = 0xFFFFFFFFFULL; // 2**36 - 1 const unsigned int SUBSYSTEM_ID_DDLPROC = 15; const unsigned int SUBSYSTEM_ID_DMLPROC = 20; -const unsigned int SUBSYSTEM_ID_WE = 19; -const unsigned int SUBSYSTEM_ID_WE_SRV = 32; +const unsigned int SUBSYSTEM_ID_WE = 19; +const unsigned int SUBSYSTEM_ID_WE_SRV = 32; const unsigned int SUBSYSTEM_ID_WE_SPLIT = 33; const unsigned int SUBSYSTEM_ID_WE_BULK = 34; //-------------------------------------------------------------------------- // Default definitions //-------------------------------------------------------------------------- -const int DEFAULT_CACHE_BLOCK = 256; // Max num of cache blocks -const int DEFAULT_CHK_INTERVAL = 3; // Checkpoint in seconds -const int DEFAULT_CACHE_PCT_FREE = 25; // Min % of free cache -const int DEFAULT_BUFSIZ = 1 * 1024 * 1024; // setvbuf buffer size -const int DEFAULT_COLSIZ = 8; // col size for hdfs rdwr buf +const int DEFAULT_CACHE_BLOCK = 256; // Max num of cache blocks +const int DEFAULT_CHK_INTERVAL = 3; // Checkpoint in seconds +const int DEFAULT_CACHE_PCT_FREE = 25; // Min % of free cache +const int DEFAULT_BUFSIZ = 1 * 1024 * 1024; // setvbuf buffer size +const int DEFAULT_COLSIZ = 8; // col size for hdfs rdwr buf -const int BLK_INIT = 0; -const int BLK_READ = 1; -const int BLK_WRITE = 2; +const int BLK_INIT = 0; +const int BLK_READ = 1; +const int BLK_WRITE = 2; //-------------------------------------------------------------------------- // Return code definitions //-------------------------------------------------------------------------- -const int NO_ERROR = 0; // No error -const int NOT_FOUND = -1; // Not found -const int INVALID_NUM = -1; // Invalid number +const int NO_ERROR = 0; // No error +const int NOT_FOUND = -1; // Not found +const int INVALID_NUM = -1; // Invalid number //-------------------------------------------------------------------------- // Error code definition //-------------------------------------------------------------------------- -const int ERR_CODEBASE = 1000; // Generic error codes -const int ERR_FILEBASE = 1050; // File-related error codes -const int ERR_XMLBASE = 1150; // XML job file error codes -const int ERR_TBLLOCKBASE = 1200; // Table-lock error codes -const int ERR_WRAPPERBASE = 1250; // DDL/DML API related errors -const int ERR_INDEXBASE = 1300; // Index-related error codes -const int ERR_FMGRBASE = 1350; // Freemgr errors -const int ERR_DCTNRYBASE = 1400; // Dictionary errors -const int ERR_BULKBASE = 1450; // Bulk specific errors -const int ERR_BRMBASE = 1500; // BRM errors -const int ERR_DMBASE = 1550; // Disk manager errors -const int ERR_CACHEBASE = 1600; // Cche management errors -const int ERR_COMPBASE = 1650; // Compression errors -const int ERR_AUTOINCBASE = 1700; // Auto-increment errors -const int ERR_BLKCACHEBASE = 1750; // Block cache flush errors -const int ERR_METABKUPBASE = 1800; // Backup bulk meta file errors +const int ERR_CODEBASE = 1000; // Generic error codes +const int ERR_FILEBASE = 1050; // File-related error codes +const int ERR_XMLBASE = 1150; // XML job file error codes +const int ERR_TBLLOCKBASE = 1200; // Table-lock error codes +const int ERR_WRAPPERBASE = 1250; // DDL/DML API related errors +const int ERR_INDEXBASE = 1300; // Index-related error codes +const int ERR_FMGRBASE = 1350; // Freemgr errors +const int ERR_DCTNRYBASE = 1400; // Dictionary errors +const int ERR_BULKBASE = 1450; // Bulk specific errors +const int ERR_BRMBASE = 1500; // BRM errors +const int ERR_DMBASE = 1550; // Disk manager errors +const int ERR_CACHEBASE = 1600; // Cche management errors +const int ERR_COMPBASE = 1650; // Compression errors +const int ERR_AUTOINCBASE = 1700; // Auto-increment errors +const int ERR_BLKCACHEBASE = 1750; // Block cache flush errors +const int ERR_METABKUPBASE = 1800; // Backup bulk meta file errors //-------------------------------------------------------------------------- // Generic error //-------------------------------------------------------------------------- -const int ERR_UNKNOWN = ERR_CODEBASE + 1; // Generic error -const int ERR_INVALID_PARAM = ERR_CODEBASE + 2; // Invalid parms -const int ERR_STRUCT_EMPTY = ERR_CODEBASE + 3; // Struct is empty -const int ERR_VALUE_OUTOFRANGE = ERR_CODEBASE + 4; // Val out of range -const int ERR_PARSING = ERR_CODEBASE + 5; // Parsing error -const int ERR_NO_MEM = ERR_CODEBASE + 6; // Mem alloc error -const int ERR_DML_LOG_NAME = ERR_CODEBASE + 7; // DML log filename error -const int ERR_OPEN_DML_LOG = ERR_CODEBASE + 8; // Open DML log file error -const int ERR_HDFS_BACKUP = ERR_CODEBASE + 9; // HDFS backup error +const int ERR_UNKNOWN = ERR_CODEBASE + 1; // Generic error +const int ERR_INVALID_PARAM = ERR_CODEBASE + 2; // Invalid parms +const int ERR_STRUCT_EMPTY = ERR_CODEBASE + 3; // Struct is empty +const int ERR_VALUE_OUTOFRANGE = ERR_CODEBASE + 4; // Val out of range +const int ERR_PARSING = ERR_CODEBASE + 5; // Parsing error +const int ERR_NO_MEM = ERR_CODEBASE + 6; // Mem alloc error +const int ERR_DML_LOG_NAME = ERR_CODEBASE + 7; // DML log filename error +const int ERR_OPEN_DML_LOG = ERR_CODEBASE + 8; // Open DML log file error +const int ERR_HDFS_BACKUP = ERR_CODEBASE + 9; // HDFS backup error //-------------------------------------------------------------------------- // File level error //-------------------------------------------------------------------------- -const int ERR_FILE_CREATE = ERR_FILEBASE + 1; // File creation error, mostly because file has already existed -const int ERR_FILE_OPEN = ERR_FILEBASE + 2; // Can not open the file, mostly because file not found -const int ERR_FILE_DELETE = ERR_FILEBASE + 3; // Can not delete the file, common reason is file not exist -const int ERR_FILE_EXIST = ERR_FILEBASE + 4; // File alreay exists -const int ERR_FILE_NOT_EXIST = ERR_FILEBASE + 5; // File not exists -const int ERR_FILE_NULL = ERR_FILEBASE + 6; // File is empty -const int ERR_FILE_WRITE = ERR_FILEBASE + 7; // Error writing to a DB file -const int ERR_FILE_READ = ERR_FILEBASE + 8; // Error reading from a DB file -const int ERR_FILE_SEEK = ERR_FILEBASE + 9; // Error in positioning file handle -const int ERR_FILE_READ_IMPORT = ERR_FILEBASE + 10;// Error reading import source file -const int ERR_DIR_CREATE = ERR_FILEBASE + 11;// Error in creating directory -const int ERR_FILE_NEW_EXTENT_FBO = ERR_FILEBASE + 12;// New extent fbo too large -const int ERR_FILE_FBO_NEG = ERR_FILEBASE + 13;// File FBO is negative -const int ERR_FILE_TRUNCATE = ERR_FILEBASE + 14;// Error truncating file -const int ERR_FILE_DISK_SPACE = ERR_FILEBASE + 15;// Out of space on file system -const int ERR_FILE_STAT = ERR_FILEBASE + 16;// Error getting stats on file -const int ERR_VB_FILE_NOT_EXIST = ERR_FILEBASE + 17;// Version buffer file not exists -const int ERR_FILE_FLUSH = ERR_FILEBASE + 18;// Error flushing file -const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19;// Error globbing a file name -const int ERR_FILE_EOF = ERR_FILEBASE + 20;// EOF -const int ERR_FILE_CHOWN = ERR_FILEBASE + 21;// EOF +const int ERR_FILE_CREATE = ERR_FILEBASE + 1; // File creation error, mostly because file has already existed +const int ERR_FILE_OPEN = ERR_FILEBASE + 2; // Can not open the file, mostly because file not found +const int ERR_FILE_DELETE = ERR_FILEBASE + 3; // Can not delete the file, common reason is file not exist +const int ERR_FILE_EXIST = ERR_FILEBASE + 4; // File alreay exists +const int ERR_FILE_NOT_EXIST = ERR_FILEBASE + 5; // File not exists +const int ERR_FILE_NULL = ERR_FILEBASE + 6; // File is empty +const int ERR_FILE_WRITE = ERR_FILEBASE + 7; // Error writing to a DB file +const int ERR_FILE_READ = ERR_FILEBASE + 8; // Error reading from a DB file +const int ERR_FILE_SEEK = ERR_FILEBASE + 9; // Error in positioning file handle +const int ERR_FILE_READ_IMPORT = ERR_FILEBASE + 10; // Error reading import source file +const int ERR_DIR_CREATE = ERR_FILEBASE + 11; // Error in creating directory +const int ERR_FILE_NEW_EXTENT_FBO = ERR_FILEBASE + 12; // New extent fbo too large +const int ERR_FILE_FBO_NEG = ERR_FILEBASE + 13; // File FBO is negative +const int ERR_FILE_TRUNCATE = ERR_FILEBASE + 14; // Error truncating file +const int ERR_FILE_DISK_SPACE = ERR_FILEBASE + 15; // Out of space on file system +const int ERR_FILE_STAT = ERR_FILEBASE + 16; // Error getting stats on file +const int ERR_VB_FILE_NOT_EXIST = ERR_FILEBASE + 17; // Version buffer file not exists +const int ERR_FILE_FLUSH = ERR_FILEBASE + 18; // Error flushing file +const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19; // Error globbing a file name +const int ERR_FILE_EOF = ERR_FILEBASE + 20; // EOF +const int ERR_FILE_CHOWN = ERR_FILEBASE + 21; // EOF //-------------------------------------------------------------------------- // XML level error //-------------------------------------------------------------------------- -const int ERR_XML_FILE = ERR_XMLBASE + 1; // File error, probably because file does not exist -const int ERR_XML_ROOT_ELEM = ERR_XMLBASE + 2; // Root element err -const int ERR_XML_EMPTY = ERR_XMLBASE + 3; // Empty XML file -const int ERR_XML_PARSE = ERR_XMLBASE + 4; // Parsing error +const int ERR_XML_FILE = ERR_XMLBASE + 1; // File error, probably because file does not exist +const int ERR_XML_ROOT_ELEM = ERR_XMLBASE + 2; // Root element err +const int ERR_XML_EMPTY = ERR_XMLBASE + 3; // Empty XML file +const int ERR_XML_PARSE = ERR_XMLBASE + 4; // Parsing error //-------------------------------------------------------------------------- // table lock level error //-------------------------------------------------------------------------- -const int ERR_TBLLOCK_LOCK_NOT_FOUND = ERR_TBLLOCKBASE + 1; // table has no lock -const int ERR_TBLLOCK_GET_LOCK = ERR_TBLLOCKBASE + 2; // error acquiring a table lock -const int ERR_TBLLOCK_GET_LOCK_LOCKED = ERR_TBLLOCKBASE + 3; // table currently locked -const int ERR_TBLLOCK_RELEASE_LOCK = ERR_TBLLOCKBASE + 4; // error releasing a table lock -const int ERR_TBLLOCK_CHANGE_STATE = ERR_TBLLOCKBASE + 5; // error changing state of lock -const int ERR_TBLLOCK_GET_INFO = ERR_TBLLOCKBASE + 6; // error getting info about a lock -const int ERR_TBLLOCK_LOCKID_CONFLICT = ERR_TBLLOCKBASE + 7; // lockID for different table than expected +const int ERR_TBLLOCK_LOCK_NOT_FOUND = ERR_TBLLOCKBASE + 1; // table has no lock +const int ERR_TBLLOCK_GET_LOCK = ERR_TBLLOCKBASE + 2; // error acquiring a table lock +const int ERR_TBLLOCK_GET_LOCK_LOCKED = ERR_TBLLOCKBASE + 3; // table currently locked +const int ERR_TBLLOCK_RELEASE_LOCK = ERR_TBLLOCKBASE + 4; // error releasing a table lock +const int ERR_TBLLOCK_CHANGE_STATE = ERR_TBLLOCKBASE + 5; // error changing state of lock +const int ERR_TBLLOCK_GET_INFO = ERR_TBLLOCKBASE + 6; // error getting info about a lock +const int ERR_TBLLOCK_LOCKID_CONFLICT = ERR_TBLLOCKBASE + 7; // lockID for different table than expected //-------------------------------------------------------------------------- // DDL/DML Interface level error //-------------------------------------------------------------------------- -const int ERR_STRUCT_VALUE_NOT_MATCH = ERR_WRAPPERBASE + 1; // The number of struct not match with the number of value set -const int ERR_ROWID_VALUE_NOT_MATCH = ERR_WRAPPERBASE + 2; // The number of rowid not match with the number of values -const int ERR_TBL_SYSCAT_ERROR = ERR_WRAPPERBASE + 3; /** @brief Syscatalog query error */ +const int ERR_STRUCT_VALUE_NOT_MATCH = + ERR_WRAPPERBASE + 1; // The number of struct not match with the number of value set +const int ERR_ROWID_VALUE_NOT_MATCH = + ERR_WRAPPERBASE + 2; // The number of rowid not match with the number of values +const int ERR_TBL_SYSCAT_ERROR = ERR_WRAPPERBASE + 3; /** @brief Syscatalog query error */ //-------------------------------------------------------------------------- // index error //-------------------------------------------------------------------------- -const int ERR_IDX_TREE_MOVE_ENTRY = ERR_INDEXBASE + 1; // The error in move part of tree to a new subblock -const int ERR_IDX_TREE_INVALID_TYPE = ERR_INDEXBASE + 2; // Invalid tree entry type -const int ERR_IDX_TREE_BITTEST_VAL = ERR_INDEXBASE + 3; // Wrong bit test value in the entry -const int ERR_IDX_TREE_INVALID_LEVEL = ERR_INDEXBASE + 4; // Invalid testbit treel level -const int ERR_IDX_TREE_INVALID_GRP = ERR_INDEXBASE + 5; // Invalid group type -const int ERR_IDX_TREE_LISTPTR_CHANGE = ERR_INDEXBASE + 6; // List pointer change -//index list error -const int ERR_IDX_LIST_INVALID_ADDHDR = ERR_INDEXBASE + 7; // Create indexlist header error -const int ERR_IDX_LIST_INVALID_UPDATE = ERR_INDEXBASE + 8; // Update Index List error -const int ERR_IDX_LIST_INVALID_DELETE = ERR_INDEXBASE + 9; // Delete rowid in indexlist err*/ -const int ERR_IDX_LIST_INVALID_KEY = ERR_INDEXBASE + 10;// Invalid Key passed -const int ERR_IDX_LIST_GET_RID_ARRARY = ERR_INDEXBASE + 11;// RID array -const int ERR_IDX_LIST_WRONG_KEY = ERR_INDEXBASE + 12;// not matched Key passed -const int ERR_IDX_LIST_HDR_EMPTY = ERR_INDEXBASE + 13;// Delete rowid in indexlist err -const int ERR_IDX_LIST_GET_SEGMT = ERR_INDEXBASE + 14;// Get Segment -const int ERR_IDX_LIST_WRONG_LBID_WRITE = ERR_INDEXBASE + 15; -const int ERR_IDX_LIST_UPDATE_SUB = ERR_INDEXBASE + 16; -const int ERR_IDX_LIST_UPDATE_NARRAY = ERR_INDEXBASE + 17; -const int ERR_IDX_LIST_LAST_FBO_NEG = ERR_INDEXBASE + 18; -const int ERR_IDX_LIST_INIT_NEW_BLKS = ERR_INDEXBASE + 19; -const int ERR_IDX_LIST_INIT_LINK_BLKS = ERR_INDEXBASE + 20; -const int ERR_IDX_LIST_UPDATE_COUNT = ERR_INDEXBASE + 21; -const int ERR_IDX_LIST_SET_NEXT_LBID = ERR_INDEXBASE + 22; -const int ERR_IDX_LIST_INVALID_LBID = ERR_INDEXBASE + 23; -const int ERR_IDX_LIST_INVALID_BLK_READ = ERR_INDEXBASE + 24; -const int ERR_IDX_LIST_UPDATE_HDR_COUNT = ERR_INDEXBASE + 25; -const int ERR_IDX_LIST_WRONG_BLK = ERR_INDEXBASE + 26; -const int ERR_IDX_LIST_WRONG_TYPE = ERR_INDEXBASE + 27; -const int ERR_IDX_LIST_GET_COUNT = ERR_INDEXBASE + 28; -const int ERR_IDX_LIST_GET_NEXT = ERR_INDEXBASE + 29; -const int ERR_IDX_LIST_GET_PARENT = ERR_INDEXBASE + 30; -const int ERR_IDX_LIST_GET_SUB_BLK = ERR_INDEXBASE + 31; -const int ERR_IDX_LIST_INVALID_UP_HDR = ERR_INDEXBASE + 32;// Update Index List error -const int ERR_IDX_LIST_INVALID_ADD_LIST = ERR_INDEXBASE + 33; // Update Index List error -const int ERR_IDX_LIST_INVALID_UP = ERR_INDEXBASE + 34;// Update Index List error +const int ERR_IDX_TREE_MOVE_ENTRY = ERR_INDEXBASE + 1; // The error in move part of tree to a new subblock +const int ERR_IDX_TREE_INVALID_TYPE = ERR_INDEXBASE + 2; // Invalid tree entry type +const int ERR_IDX_TREE_BITTEST_VAL = ERR_INDEXBASE + 3; // Wrong bit test value in the entry +const int ERR_IDX_TREE_INVALID_LEVEL = ERR_INDEXBASE + 4; // Invalid testbit treel level +const int ERR_IDX_TREE_INVALID_GRP = ERR_INDEXBASE + 5; // Invalid group type +const int ERR_IDX_TREE_LISTPTR_CHANGE = ERR_INDEXBASE + 6; // List pointer change +// index list error +const int ERR_IDX_LIST_INVALID_ADDHDR = ERR_INDEXBASE + 7; // Create indexlist header error +const int ERR_IDX_LIST_INVALID_UPDATE = ERR_INDEXBASE + 8; // Update Index List error +const int ERR_IDX_LIST_INVALID_DELETE = ERR_INDEXBASE + 9; // Delete rowid in indexlist err*/ +const int ERR_IDX_LIST_INVALID_KEY = ERR_INDEXBASE + 10; // Invalid Key passed +const int ERR_IDX_LIST_GET_RID_ARRARY = ERR_INDEXBASE + 11; // RID array +const int ERR_IDX_LIST_WRONG_KEY = ERR_INDEXBASE + 12; // not matched Key passed +const int ERR_IDX_LIST_HDR_EMPTY = ERR_INDEXBASE + 13; // Delete rowid in indexlist err +const int ERR_IDX_LIST_GET_SEGMT = ERR_INDEXBASE + 14; // Get Segment +const int ERR_IDX_LIST_WRONG_LBID_WRITE = ERR_INDEXBASE + 15; +const int ERR_IDX_LIST_UPDATE_SUB = ERR_INDEXBASE + 16; +const int ERR_IDX_LIST_UPDATE_NARRAY = ERR_INDEXBASE + 17; +const int ERR_IDX_LIST_LAST_FBO_NEG = ERR_INDEXBASE + 18; +const int ERR_IDX_LIST_INIT_NEW_BLKS = ERR_INDEXBASE + 19; +const int ERR_IDX_LIST_INIT_LINK_BLKS = ERR_INDEXBASE + 20; +const int ERR_IDX_LIST_UPDATE_COUNT = ERR_INDEXBASE + 21; +const int ERR_IDX_LIST_SET_NEXT_LBID = ERR_INDEXBASE + 22; +const int ERR_IDX_LIST_INVALID_LBID = ERR_INDEXBASE + 23; +const int ERR_IDX_LIST_INVALID_BLK_READ = ERR_INDEXBASE + 24; +const int ERR_IDX_LIST_UPDATE_HDR_COUNT = ERR_INDEXBASE + 25; +const int ERR_IDX_LIST_WRONG_BLK = ERR_INDEXBASE + 26; +const int ERR_IDX_LIST_WRONG_TYPE = ERR_INDEXBASE + 27; +const int ERR_IDX_LIST_GET_COUNT = ERR_INDEXBASE + 28; +const int ERR_IDX_LIST_GET_NEXT = ERR_INDEXBASE + 29; +const int ERR_IDX_LIST_GET_PARENT = ERR_INDEXBASE + 30; +const int ERR_IDX_LIST_GET_SUB_BLK = ERR_INDEXBASE + 31; +const int ERR_IDX_LIST_INVALID_UP_HDR = ERR_INDEXBASE + 32; // Update Index List error +const int ERR_IDX_LIST_INVALID_ADD_LIST = ERR_INDEXBASE + 33; // Update Index List error +const int ERR_IDX_LIST_INVALID_UP = ERR_INDEXBASE + 34; // Update Index List error //-------------------------------------------------------------------------- // freemgr error //-------------------------------------------------------------------------- -const int ERR_FM_ASSIGN_ERR = ERR_FMGRBASE + 1; // General assignment error -const int ERR_FM_RELEASE_ERR = ERR_FMGRBASE + 2; // General release error -const int ERR_FM_BAD_FBO = ERR_FMGRBASE + 3; // File Block Offset err -const int ERR_FM_BAD_TYPE = ERR_FMGRBASE + 4; // type must be pointer or list -const int ERR_FM_NO_SPACE = ERR_FMGRBASE + 5; // No blocks available -const int ERR_FM_EXTEND = ERR_FMGRBASE + 6; // Error extending file +const int ERR_FM_ASSIGN_ERR = ERR_FMGRBASE + 1; // General assignment error +const int ERR_FM_RELEASE_ERR = ERR_FMGRBASE + 2; // General release error +const int ERR_FM_BAD_FBO = ERR_FMGRBASE + 3; // File Block Offset err +const int ERR_FM_BAD_TYPE = ERR_FMGRBASE + 4; // type must be pointer or list +const int ERR_FM_NO_SPACE = ERR_FMGRBASE + 5; // No blocks available +const int ERR_FM_EXTEND = ERR_FMGRBASE + 6; // Error extending file //-------------------------------------------------------------------------- // Dictionary error //-------------------------------------------------------------------------- -const int ERR_DICT_NO_SPACE_INSERT = ERR_DCTNRYBASE + 1; // ins no space -const int ERR_DICT_SIZE_GT_2G = ERR_DCTNRYBASE + 2; // ins size >8000 -const int ERR_DICT_NO_OP_DELETE = ERR_DCTNRYBASE + 3; // del no op -const int ERR_DICT_NO_OFFSET_DELETE = ERR_DCTNRYBASE + 4; // del bad offset -const int ERR_DICT_INVALID_HDR = ERR_DCTNRYBASE + 5; // Delete Hdr -const int ERR_DICT_ZERO_LEN = ERR_DCTNRYBASE + 6; // Delete zero len -const int ERR_DICT_TOKEN_NOT_FOUND = ERR_DCTNRYBASE + 7; // token not found -const int ERR_DICT_FILE_NOT_FOUND = ERR_DCTNRYBASE + 8; // dict file not found -const int ERR_DICT_BAD_TOKEN_LBID = ERR_DCTNRYBASE + 9; // bad token lbid -const int ERR_DICT_BAD_TOKEN_OP = ERR_DCTNRYBASE + 10; // token op is bad +const int ERR_DICT_NO_SPACE_INSERT = ERR_DCTNRYBASE + 1; // ins no space +const int ERR_DICT_SIZE_GT_2G = ERR_DCTNRYBASE + 2; // ins size >8000 +const int ERR_DICT_NO_OP_DELETE = ERR_DCTNRYBASE + 3; // del no op +const int ERR_DICT_NO_OFFSET_DELETE = ERR_DCTNRYBASE + 4; // del bad offset +const int ERR_DICT_INVALID_HDR = ERR_DCTNRYBASE + 5; // Delete Hdr +const int ERR_DICT_ZERO_LEN = ERR_DCTNRYBASE + 6; // Delete zero len +const int ERR_DICT_TOKEN_NOT_FOUND = ERR_DCTNRYBASE + 7; // token not found +const int ERR_DICT_FILE_NOT_FOUND = ERR_DCTNRYBASE + 8; // dict file not found +const int ERR_DICT_BAD_TOKEN_LBID = ERR_DCTNRYBASE + 9; // bad token lbid +const int ERR_DICT_BAD_TOKEN_OP = ERR_DCTNRYBASE + 10; // token op is bad //-------------------------------------------------------------------------- // Bulk error //-------------------------------------------------------------------------- -const int ERR_BULK_MAX_ERR_NUM = ERR_BULKBASE + 1; // Maximum number of error rows reached -const int ERR_BULK_DATA_COL_NUM = ERR_BULKBASE + 2; // The total number of data column not match with column definitions -const int ERR_BULK_SEND_MSG_ERR = ERR_BULKBASE + 3; // send msg to primproc to flush cache -const int ERR_BULK_MISSING_EXTENT_ENTRY = ERR_BULKBASE + 4; // Missing Extent Entry when trying to save LBID info -const int ERR_BULK_MISSING_EXTENT_ROW = ERR_BULKBASE + 5; // Missing Extent Row when trying to save LBID info -const int ERR_BULK_ROW_FILL_BUFFER = ERR_BULKBASE + 6; // Single row fills read buffer -const int ERR_BULK_DBROOT_CHANGE = ERR_BULKBASE + 7; // Local DBRoot settings changed during an import -const int ERR_BULK_ROLLBACK_MISS_ROOT = ERR_BULKBASE + 8; // Mode3 automatic rollback skipped with missing DBRoot -const int ERR_BULK_ROLLBACK_SEG_LIST = ERR_BULKBASE + 9; // Error building segment file list in a directory -const int ERR_BULK_BINARY_PARTIAL_REC = ERR_BULKBASE + 10;// Binary input did not end on fixed length record boundary -const int ERR_BULK_BINARY_IGNORE_FLD = ERR_BULKBASE + 11;// tag not supported for binary import +const int ERR_BULK_MAX_ERR_NUM = ERR_BULKBASE + 1; // Maximum number of error rows reached +const int ERR_BULK_DATA_COL_NUM = + ERR_BULKBASE + 2; // The total number of data column not match with column definitions +const int ERR_BULK_SEND_MSG_ERR = ERR_BULKBASE + 3; // send msg to primproc to flush cache +const int ERR_BULK_MISSING_EXTENT_ENTRY = + ERR_BULKBASE + 4; // Missing Extent Entry when trying to save LBID info +const int ERR_BULK_MISSING_EXTENT_ROW = ERR_BULKBASE + 5; // Missing Extent Row when trying to save LBID info +const int ERR_BULK_ROW_FILL_BUFFER = ERR_BULKBASE + 6; // Single row fills read buffer +const int ERR_BULK_DBROOT_CHANGE = ERR_BULKBASE + 7; // Local DBRoot settings changed during an import +const int ERR_BULK_ROLLBACK_MISS_ROOT = + ERR_BULKBASE + 8; // Mode3 automatic rollback skipped with missing DBRoot +const int ERR_BULK_ROLLBACK_SEG_LIST = ERR_BULKBASE + 9; // Error building segment file list in a directory +const int ERR_BULK_BINARY_PARTIAL_REC = + ERR_BULKBASE + 10; // Binary input did not end on fixed length record boundary +const int ERR_BULK_BINARY_IGNORE_FLD = + ERR_BULKBASE + 11; // tag not supported for binary import //-------------------------------------------------------------------------- // BRM error //-------------------------------------------------------------------------- -const int ERR_BRM_LOOKUP_LBID = ERR_BRMBASE + 1; // Lookup LBID error -const int ERR_BRM_LOOKUP_FBO = ERR_BRMBASE + 2; // Lookup FBO error -const int ERR_BRM_ALLOC_EXTEND = ERR_BRMBASE + 3; // Allocate extent error -const int ERR_BRM_COMMIT = ERR_BRMBASE + 4; // Commit error -const int ERR_BRM_ROLLBACK = ERR_BRMBASE + 5; // Rollback error -const int ERR_BRM_GET_UNCOMM_LBID = ERR_BRMBASE + 6; // Get uncommitted lbid list error -const int ERR_BRM_DEL_OID = ERR_BRMBASE + 7; // Delete oid error -const int ERR_BRM_BEGIN_COPY = ERR_BRMBASE + 8; // Begin copy error -const int ERR_BRM_END_COPY = ERR_BRMBASE + 9; // End copy error -const int ERR_BRM_GET_HWM = ERR_BRMBASE + 10;// Get hwm error -const int ERR_BRM_SET_HWM = ERR_BRMBASE + 11;// Set hwm error -const int ERR_BRM_WR_VB_ENTRY = ERR_BRMBASE + 12;// Write VB entry error -const int ERR_BRM_VB_COPY_READ = ERR_BRMBASE + 13;// VB copy read error -const int ERR_BRM_VB_COPY_SEEK_DB = ERR_BRMBASE + 14;// VB copy seek error to DB file -const int ERR_BRM_VB_COPY_SEEK_VB = ERR_BRMBASE + 15;// VB copy seek error to VB file -const int ERR_BRM_VB_COPY_WRITE = ERR_BRMBASE + 16;// VB copy write -const int ERR_BRM_DEAD_LOCK = ERR_BRMBASE + 17;// DEAD lock error -const int ERR_BRM_MARK_INVALID = ERR_BRMBASE + 18;// Mark extent invalid error from casual paritioning -const int ERR_BRM_SAVE_STATE = ERR_BRMBASE + 19;// Save state error -const int ERR_BRM_GET_START_EXTENT = ERR_BRMBASE + 20; // Get starting Extent error -const int ERR_BRM_VB_OVERFLOW = ERR_BRMBASE + 21;// Version buffer overflow -const int ERR_BRM_READ_ONLY = ERR_BRMBASE + 22;// BRM is in READ-ONLY state -const int ERR_BRM_GET_READ_WRITE = ERR_BRMBASE + 23;// error getting BRM READ/WRITE state -const int ERR_BRM_BULK_RB_COLUMN = ERR_BRMBASE + 24;// error during column bulk rollback -const int ERR_BRM_BULK_RB_DCTNRY = ERR_BRMBASE + 25;// error during dctnry bulk rollback -const int ERR_BRM_DELETE_EXTENT_COLUMN = ERR_BRMBASE + 26; // error during delete column extents -const int ERR_BRM_DELETE_EXTENT_DCTNRY = ERR_BRMBASE + 27; // error during delete dictionary extents -const int ERR_BRM_TAKE_SNAPSHOT = ERR_BRMBASE + 28;// Taking snapshot of BRM state -const int ERR_BRM_LOOKUP_START_LBID = ERR_BRMBASE + 29; // Lookup starting LBID error -const int ERR_BRM_BULK_UPDATE = ERR_BRMBASE + 30;// Error with bulk update of HWM and CP -const int ERR_BRM_GET_EXT_STATE = ERR_BRMBASE + 31;// Error getting extent state -const int ERR_EXTENTMAP_LOOKUP = ERR_BRMBASE + 32;// Lookup extent map error -const int ERR_BRM_LOOKUP_VERSION = ERR_BRMBASE + 33;// Lookup version error -const int ERR_BRM_LOOKUP_LBID_RANGES = ERR_BRMBASE + 34;// Lookup LBID Ranges error -const int ERR_BRM_HWMS_NOT_EQUAL = ERR_BRMBASE + 35;// HWMs of same col width not equal -const int ERR_BRM_HWMS_OUT_OF_SYNC = ERR_BRMBASE + 36; // HWMs for dif col width not in sync -const int ERR_BRM_DBROOT_HWMS = ERR_BRMBASE + 37;// Error getting HWMs for each DBRoot -const int ERR_BRM_NETWORK = ERR_BRMBASE + 38;// Network error when calling BRM functions -const int ERR_BRM_READONLY = ERR_BRMBASE + 39;// DBRM is readonly -const int ERR_INVALID_VBOID = ERR_BRMBASE + 40;// returned if the given vboid is invalid -const int ERR_BRM_SET_EXTENTS_CP = ERR_BRMBASE + 41;// Error setting extents min/max -const int ERR_BRM_SHUTDOWN = ERR_BRMBASE + 42;// BRM is set to shutdown -const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43;// error getting BRM Shutdown flag -const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44;// BRM is set to Suspend writes -const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45;// error getting BRM Suspend flag -const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46;// Incorrect num of cols allocated in stripe -const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47;// Non-dict column Width > allowed MAX. -const int ERR_BRM_GET_EXTENT_CP = ERR_BRMBASE + 48;// Error getting extent's CPInfo +const int ERR_BRM_LOOKUP_LBID = ERR_BRMBASE + 1; // Lookup LBID error +const int ERR_BRM_LOOKUP_FBO = ERR_BRMBASE + 2; // Lookup FBO error +const int ERR_BRM_ALLOC_EXTEND = ERR_BRMBASE + 3; // Allocate extent error +const int ERR_BRM_COMMIT = ERR_BRMBASE + 4; // Commit error +const int ERR_BRM_ROLLBACK = ERR_BRMBASE + 5; // Rollback error +const int ERR_BRM_GET_UNCOMM_LBID = ERR_BRMBASE + 6; // Get uncommitted lbid list error +const int ERR_BRM_DEL_OID = ERR_BRMBASE + 7; // Delete oid error +const int ERR_BRM_BEGIN_COPY = ERR_BRMBASE + 8; // Begin copy error +const int ERR_BRM_END_COPY = ERR_BRMBASE + 9; // End copy error +const int ERR_BRM_GET_HWM = ERR_BRMBASE + 10; // Get hwm error +const int ERR_BRM_SET_HWM = ERR_BRMBASE + 11; // Set hwm error +const int ERR_BRM_WR_VB_ENTRY = ERR_BRMBASE + 12; // Write VB entry error +const int ERR_BRM_VB_COPY_READ = ERR_BRMBASE + 13; // VB copy read error +const int ERR_BRM_VB_COPY_SEEK_DB = ERR_BRMBASE + 14; // VB copy seek error to DB file +const int ERR_BRM_VB_COPY_SEEK_VB = ERR_BRMBASE + 15; // VB copy seek error to VB file +const int ERR_BRM_VB_COPY_WRITE = ERR_BRMBASE + 16; // VB copy write +const int ERR_BRM_DEAD_LOCK = ERR_BRMBASE + 17; // DEAD lock error +const int ERR_BRM_MARK_INVALID = ERR_BRMBASE + 18; // Mark extent invalid error from casual paritioning +const int ERR_BRM_SAVE_STATE = ERR_BRMBASE + 19; // Save state error +const int ERR_BRM_GET_START_EXTENT = ERR_BRMBASE + 20; // Get starting Extent error +const int ERR_BRM_VB_OVERFLOW = ERR_BRMBASE + 21; // Version buffer overflow +const int ERR_BRM_READ_ONLY = ERR_BRMBASE + 22; // BRM is in READ-ONLY state +const int ERR_BRM_GET_READ_WRITE = ERR_BRMBASE + 23; // error getting BRM READ/WRITE state +const int ERR_BRM_BULK_RB_COLUMN = ERR_BRMBASE + 24; // error during column bulk rollback +const int ERR_BRM_BULK_RB_DCTNRY = ERR_BRMBASE + 25; // error during dctnry bulk rollback +const int ERR_BRM_DELETE_EXTENT_COLUMN = ERR_BRMBASE + 26; // error during delete column extents +const int ERR_BRM_DELETE_EXTENT_DCTNRY = ERR_BRMBASE + 27; // error during delete dictionary extents +const int ERR_BRM_TAKE_SNAPSHOT = ERR_BRMBASE + 28; // Taking snapshot of BRM state +const int ERR_BRM_LOOKUP_START_LBID = ERR_BRMBASE + 29; // Lookup starting LBID error +const int ERR_BRM_BULK_UPDATE = ERR_BRMBASE + 30; // Error with bulk update of HWM and CP +const int ERR_BRM_GET_EXT_STATE = ERR_BRMBASE + 31; // Error getting extent state +const int ERR_EXTENTMAP_LOOKUP = ERR_BRMBASE + 32; // Lookup extent map error +const int ERR_BRM_LOOKUP_VERSION = ERR_BRMBASE + 33; // Lookup version error +const int ERR_BRM_LOOKUP_LBID_RANGES = ERR_BRMBASE + 34; // Lookup LBID Ranges error +const int ERR_BRM_HWMS_NOT_EQUAL = ERR_BRMBASE + 35; // HWMs of same col width not equal +const int ERR_BRM_HWMS_OUT_OF_SYNC = ERR_BRMBASE + 36; // HWMs for dif col width not in sync +const int ERR_BRM_DBROOT_HWMS = ERR_BRMBASE + 37; // Error getting HWMs for each DBRoot +const int ERR_BRM_NETWORK = ERR_BRMBASE + 38; // Network error when calling BRM functions +const int ERR_BRM_READONLY = ERR_BRMBASE + 39; // DBRM is readonly +const int ERR_INVALID_VBOID = ERR_BRMBASE + 40; // returned if the given vboid is invalid +const int ERR_BRM_SET_EXTENTS_CP = ERR_BRMBASE + 41; // Error setting extents min/max +const int ERR_BRM_SHUTDOWN = ERR_BRMBASE + 42; // BRM is set to shutdown +const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43; // error getting BRM Shutdown flag +const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44; // BRM is set to Suspend writes +const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45; // error getting BRM Suspend flag +const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46; // Incorrect num of cols allocated in stripe +const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47; // Non-dict column Width > allowed MAX. +const int ERR_BRM_GET_EXTENT_CP = ERR_BRMBASE + 48; // Error getting extent's CPInfo //-------------------------------------------------------------------------- // DM error //-------------------------------------------------------------------------- -const int ERR_DM_CONVERT_OID = ERR_DMBASE + 1; // Conversion error +const int ERR_DM_CONVERT_OID = ERR_DMBASE + 1; // Conversion error //-------------------------------------------------------------------------- // Cache error //-------------------------------------------------------------------------- -const int ERR_CACHE_KEY_EXIST = ERR_CACHEBASE + 1; // Cache key exist -const int ERR_CACHE_KEY_NOT_EXIST = ERR_CACHEBASE + 2; // Cache key not exist -const int ERR_NULL_BLOCK = ERR_CACHEBASE + 3; // Block is NULL -const int ERR_FREE_LIST_EMPTY = ERR_CACHEBASE + 4; // Empty Free list +const int ERR_CACHE_KEY_EXIST = ERR_CACHEBASE + 1; // Cache key exist +const int ERR_CACHE_KEY_NOT_EXIST = ERR_CACHEBASE + 2; // Cache key not exist +const int ERR_NULL_BLOCK = ERR_CACHEBASE + 3; // Block is NULL +const int ERR_FREE_LIST_EMPTY = ERR_CACHEBASE + 4; // Empty Free list //-------------------------------------------------------------------------- // Compression error //-------------------------------------------------------------------------- -const int ERR_COMP_COMPRESS = ERR_COMPBASE + 1; // Error compressing data -const int ERR_COMP_UNCOMPRESS = ERR_COMPBASE + 2; // Error uncompressing data -const int ERR_COMP_PARSE_HDRS = ERR_COMPBASE + 3; // Error parsing compression headers -const int ERR_COMP_VERIFY_HDRS = ERR_COMPBASE + 4; // Error verifying compression headers -const int ERR_COMP_PAD_DATA = ERR_COMPBASE + 5; // Pad compressed data failed -const int ERR_COMP_READ_BLOCK = ERR_COMPBASE + 6; // Failed to read a block -const int ERR_COMP_SAVE_BLOCK = ERR_COMPBASE + 7; // Failed to save a block -const int ERR_COMP_WRONG_PTR = ERR_COMPBASE + 8; // Pointer in header is wrong -const int ERR_COMP_FILE_NOT_FOUND = ERR_COMPBASE + 9; // File not found in map -const int ERR_COMP_CHUNK_NOT_FOUND = ERR_COMPBASE + 10; // Chunk not found in map -const int ERR_COMP_UNAVAIL_TYPE = ERR_COMPBASE + 11;// Unavailable compression type -const int ERR_COMP_REMOVE_FILE = ERR_COMPBASE + 12;// Failed to remove a file -const int ERR_COMP_RENAME_FILE = ERR_COMPBASE + 13;// Failed to rename a file -const int ERR_COMP_OPEN_FILE = ERR_COMPBASE + 14;// Failed to open a compressed data file -const int ERR_COMP_SET_OFFSET = ERR_COMPBASE + 15;// Failed to set offset in a compressed data file -const int ERR_COMP_READ_FILE = ERR_COMPBASE + 16;// Failed to read from a compressed data file -const int ERR_COMP_WRITE_FILE = ERR_COMPBASE + 17;// Failed to write to a compresssed data file -const int ERR_COMP_CLOSE_FILE = ERR_COMPBASE + 18;// Failed to close a compressed data file -const int ERR_COMP_TRUNCATE_ZERO = ERR_COMPBASE + 19;// Invalid attempt to truncate file to 0 bytes -const int ERR_COMP_WRONG_COMP_TYPE = ERR_COMPBASE + 20;// Invalid compression type. +const int ERR_COMP_COMPRESS = ERR_COMPBASE + 1; // Error compressing data +const int ERR_COMP_UNCOMPRESS = ERR_COMPBASE + 2; // Error uncompressing data +const int ERR_COMP_PARSE_HDRS = ERR_COMPBASE + 3; // Error parsing compression headers +const int ERR_COMP_VERIFY_HDRS = ERR_COMPBASE + 4; // Error verifying compression headers +const int ERR_COMP_PAD_DATA = ERR_COMPBASE + 5; // Pad compressed data failed +const int ERR_COMP_READ_BLOCK = ERR_COMPBASE + 6; // Failed to read a block +const int ERR_COMP_SAVE_BLOCK = ERR_COMPBASE + 7; // Failed to save a block +const int ERR_COMP_WRONG_PTR = ERR_COMPBASE + 8; // Pointer in header is wrong +const int ERR_COMP_FILE_NOT_FOUND = ERR_COMPBASE + 9; // File not found in map +const int ERR_COMP_CHUNK_NOT_FOUND = ERR_COMPBASE + 10; // Chunk not found in map +const int ERR_COMP_UNAVAIL_TYPE = ERR_COMPBASE + 11; // Unavailable compression type +const int ERR_COMP_REMOVE_FILE = ERR_COMPBASE + 12; // Failed to remove a file +const int ERR_COMP_RENAME_FILE = ERR_COMPBASE + 13; // Failed to rename a file +const int ERR_COMP_OPEN_FILE = ERR_COMPBASE + 14; // Failed to open a compressed data file +const int ERR_COMP_SET_OFFSET = ERR_COMPBASE + 15; // Failed to set offset in a compressed data file +const int ERR_COMP_READ_FILE = ERR_COMPBASE + 16; // Failed to read from a compressed data file +const int ERR_COMP_WRITE_FILE = ERR_COMPBASE + 17; // Failed to write to a compresssed data file +const int ERR_COMP_CLOSE_FILE = ERR_COMPBASE + 18; // Failed to close a compressed data file +const int ERR_COMP_TRUNCATE_ZERO = ERR_COMPBASE + 19; // Invalid attempt to truncate file to 0 bytes +const int ERR_COMP_WRONG_COMP_TYPE = ERR_COMPBASE + 20; // Invalid compression type. //-------------------------------------------------------------------------- // Auto-increment error //-------------------------------------------------------------------------- -const int ERR_AUTOINC_GEN_EXCEED_MAX = ERR_AUTOINCBASE + 1; // Generated autoinc value exceeds max auto increment value/ -const int ERR_AUTOINC_USER_OUT_OF_RANGE = ERR_AUTOINCBASE + 2; // User specified autoinc value is out of range -const int ERR_AUTOINC_TABLE_NAME = ERR_AUTOINCBASE + 3; // Invalid schema/tablename for auto increment -const int ERR_AUTOINC_INIT1 = ERR_AUTOINCBASE + 4; // Error initializing auto increment (known exception) -const int ERR_AUTOINC_INIT2 = ERR_AUTOINCBASE + 5; // Error initializing auto increment (unknown exception) -const int ERR_AUTOINC_RID = ERR_AUTOINCBASE + 6; // Error initializing auto increment (unknown exception) -const int ERR_AUTOINC_START_SEQ = ERR_AUTOINCBASE + 7; // Error setting up an auto-increment sequence -const int ERR_AUTOINC_GET_RANGE = ERR_AUTOINCBASE + 8; // Error reserving an auto-increment range -const int ERR_AUTOINC_GET_LOCK = ERR_AUTOINCBASE + 9; // Error getting a lock to update auto-inc next value -const int ERR_AUTOINC_REL_LOCK = ERR_AUTOINCBASE + 10; // Error releasing lock to update auto-inc next value -const int ERR_AUTOINC_UPDATE = ERR_AUTOINCBASE + 11; // Error updating nextValue in system catalog +const int ERR_AUTOINC_GEN_EXCEED_MAX = + ERR_AUTOINCBASE + 1; // Generated autoinc value exceeds max auto increment value/ +const int ERR_AUTOINC_USER_OUT_OF_RANGE = + ERR_AUTOINCBASE + 2; // User specified autoinc value is out of range +const int ERR_AUTOINC_TABLE_NAME = ERR_AUTOINCBASE + 3; // Invalid schema/tablename for auto increment +const int ERR_AUTOINC_INIT1 = ERR_AUTOINCBASE + 4; // Error initializing auto increment (known exception) +const int ERR_AUTOINC_INIT2 = ERR_AUTOINCBASE + 5; // Error initializing auto increment (unknown exception) +const int ERR_AUTOINC_RID = ERR_AUTOINCBASE + 6; // Error initializing auto increment (unknown exception) +const int ERR_AUTOINC_START_SEQ = ERR_AUTOINCBASE + 7; // Error setting up an auto-increment sequence +const int ERR_AUTOINC_GET_RANGE = ERR_AUTOINCBASE + 8; // Error reserving an auto-increment range +const int ERR_AUTOINC_GET_LOCK = ERR_AUTOINCBASE + 9; // Error getting a lock to update auto-inc next value +const int ERR_AUTOINC_REL_LOCK = ERR_AUTOINCBASE + 10; // Error releasing lock to update auto-inc next value +const int ERR_AUTOINC_UPDATE = ERR_AUTOINCBASE + 11; // Error updating nextValue in system catalog //-------------------------------------------------------------------------- // Block cache flush error //-------------------------------------------------------------------------- -const int ERR_BLKCACHE_FLUSH_LIST = ERR_BLKCACHEBASE + 1; // Error flushing list of blocks to PrimProc +const int ERR_BLKCACHE_FLUSH_LIST = ERR_BLKCACHEBASE + 1; // Error flushing list of blocks to PrimProc //-------------------------------------------------------------------------- // Bulk backup metadata file and corresponding HWM compressed chunk files //-------------------------------------------------------------------------- -const int ERR_METADATABKUP_FILE_RENAME = ERR_METABKUPBASE + 1; // Error renaming meta file */ -const int ERR_METADATABKUP_COMP_PARSE_HDRS = ERR_METABKUPBASE + 2; // Error parsing compression headers */ -const int ERR_METADATABKUP_COMP_VERIFY_HDRS = ERR_METABKUPBASE + 3; // Error verifying compression headers */ -const int ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND = ERR_METABKUPBASE + 4; // Chunk not found in file */ -const int ERR_METADATABKUP_COMP_OPEN_BULK_BKUP = ERR_METABKUPBASE + 5; // Error opening backup chunk file */ -const int ERR_METADATABKUP_COMP_WRITE_BULK_BKUP = ERR_METABKUPBASE + 6; // Error writing to backup chunk file */ -const int ERR_METADATABKUP_COMP_READ_BULK_BKUP = ERR_METABKUPBASE + 7; // Error reading from backup chunk file */ -const int ERR_METADATABKUP_COMP_RENAME = ERR_METABKUPBASE + 8; // Error renaming chunk file */ +const int ERR_METADATABKUP_FILE_RENAME = ERR_METABKUPBASE + 1; // Error renaming meta file */ +const int ERR_METADATABKUP_COMP_PARSE_HDRS = ERR_METABKUPBASE + 2; // Error parsing compression headers */ +const int ERR_METADATABKUP_COMP_VERIFY_HDRS = ERR_METABKUPBASE + 3; // Error verifying compression headers */ +const int ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND = ERR_METABKUPBASE + 4; // Chunk not found in file */ +const int ERR_METADATABKUP_COMP_OPEN_BULK_BKUP = ERR_METABKUPBASE + 5; // Error opening backup chunk file */ +const int ERR_METADATABKUP_COMP_WRITE_BULK_BKUP = + ERR_METABKUPBASE + 6; // Error writing to backup chunk file */ +const int ERR_METADATABKUP_COMP_READ_BULK_BKUP = + ERR_METABKUPBASE + 7; // Error reading from backup chunk file */ +const int ERR_METADATABKUP_COMP_RENAME = ERR_METABKUPBASE + 8; // Error renaming chunk file */ //------------------------------------------------------------------------------ // Class used to convert an error code to a corresponding error message string //------------------------------------------------------------------------------ struct WErrorCodes { - EXPORT WErrorCodes(); - EXPORT std::string errorString(int code); -private: - typedef std::map CodeMap; - CodeMap fErrorCodes; + EXPORT WErrorCodes(); + EXPORT std::string errorString(int code); + + private: + typedef std::map CodeMap; + CodeMap fErrorCodes; }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_DEFINE_H_ +#endif // _WE_DEFINE_H_ diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index f8046deed..1db08a92f 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -58,25 +58,23 @@ using namespace idbdatafile; namespace WriteEngine { - -/*static*/ boost::mutex FileOp::m_createDbRootMutexes; -/*static*/ boost::mutex FileOp::m_mkdirMutex; +/*static*/ boost::mutex FileOp::m_createDbRootMutexes; +/*static*/ boost::mutex FileOp::m_mkdirMutex; /*static*/ std::map FileOp::m_DbRootAddExtentMutexes; // in 1 call to fwrite(), during initialization -//StopWatch timer; +// StopWatch timer; /** * Constructor */ -FileOp::FileOp(bool doAlloc) : m_compressionType(0), - m_transId((TxnID)INVALID_NUM), m_buffer(0) +FileOp::FileOp(bool doAlloc) : m_compressionType(0), m_transId((TxnID)INVALID_NUM), m_buffer(0) { - if (doAlloc) - { - m_buffer = new char[DEFAULT_BUFSIZ]; - memset(m_buffer, '\0', DEFAULT_BUFSIZ); - } + if (doAlloc) + { + m_buffer = new char[DEFAULT_BUFSIZ]; + memset(m_buffer, '\0', DEFAULT_BUFSIZ); + } } /** @@ -84,12 +82,12 @@ FileOp::FileOp(bool doAlloc) : m_compressionType(0), */ FileOp::~FileOp() { - if (m_buffer) - { - delete [] m_buffer; - } + if (m_buffer) + { + delete[] m_buffer; + } - m_buffer = 0; + m_buffer = 0; } /*********************************************************** @@ -100,9 +98,9 @@ FileOp::~FileOp() * RETURN: * none ***********************************************************/ -void FileOp::closeFile( IDBDataFile* pFile ) const +void FileOp::closeFile(IDBDataFile* pFile) const { - delete pFile; + delete pFile; } /*********************************************************** @@ -116,32 +114,31 @@ void FileOp::closeFile( IDBDataFile* pFile ) const * RETURN: * NO_ERROR if success, otherwise if fail ***********************************************************/ -int FileOp::createDir( const char* dirName, mode_t mode ) const +int FileOp::createDir(const char* dirName, mode_t mode) const { - boost::mutex::scoped_lock lk(m_mkdirMutex); - int rc = IDBPolicy::mkdir( dirName ); + boost::mutex::scoped_lock lk(m_mkdirMutex); + int rc = IDBPolicy::mkdir(dirName); - if ( rc != 0 ) + if (rc != 0) + { + int errRc = errno; + + if (errRc == EEXIST) + return NO_ERROR; // ignore "File exists" error + + if (getLogger()) { - int errRc = errno; - - if (errRc == EEXIST) - return NO_ERROR; // ignore "File exists" error - - if ( getLogger() ) - { - std::ostringstream oss; - std::string errnoMsg; - Convertor::mapErrnoToString(errRc, errnoMsg); - oss << "Error creating directory " << dirName << "; err-" << - errRc << "; " << errnoMsg; - getLogger()->logMsg( oss.str(), ERR_DIR_CREATE, MSGLVL_ERROR ); - } - - return ERR_DIR_CREATE; + std::ostringstream oss; + std::string errnoMsg; + Convertor::mapErrnoToString(errRc, errnoMsg); + oss << "Error creating directory " << dirName << "; err-" << errRc << "; " << errnoMsg; + getLogger()->logMsg(oss.str(), ERR_DIR_CREATE, MSGLVL_ERROR); } - return NO_ERROR; + return ERR_DIR_CREATE; + } + + return NO_ERROR; } /*********************************************************** @@ -160,54 +157,35 @@ int FileOp::createDir( const char* dirName, mode_t mode ) const * ERR_FILE_EXIST if file exists * ERR_FILE_CREATE if can not create the file ***********************************************************/ -int FileOp::createFile( const char* fileName, int numOfBlock, - const uint8_t* emptyVal, int width, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint16_t dbRoot, - BRM::LBID_t startLbid ) +int FileOp::createFile(const char* fileName, int numOfBlock, const uint8_t* emptyVal, int width, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint16_t dbRoot, + BRM::LBID_t startLbid) { - IDBDataFile* pFile = - IDBDataFile::open( - IDBPolicy::getType( fileName, IDBPolicy::WRITEENG ), - fileName, - "w+b", - IDBDataFile::USE_VBUF, - width); - int rc = 0; + IDBDataFile* pFile = IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "w+b", + IDBDataFile::USE_VBUF, width); + int rc = 0; - if ( pFile != NULL ) + if (pFile != NULL) + { + // Initialize the contents of the extent. + if (m_compressionType) { - - // Initialize the contents of the extent. - if (m_compressionType) - { - rc = initAbbrevCompColumnExtent( pFile, - dbRoot, - numOfBlock, - emptyVal, - width, - startLbid, - colDataType ); - } - else - { - rc = initColumnExtent( pFile, - dbRoot, - numOfBlock, - emptyVal, - width, - colDataType, - true, // new file - false, // don't expand; add new extent - true ); // add abbreviated extent - } - - closeFile( pFile ); + rc = initAbbrevCompColumnExtent(pFile, dbRoot, numOfBlock, emptyVal, width, startLbid, colDataType); } else - return ERR_FILE_CREATE; + { + rc = initColumnExtent(pFile, dbRoot, numOfBlock, emptyVal, width, colDataType, + true, // new file + false, // don't expand; add new extent + true); // add abbreviated extent + } - return rc; + closeFile(pFile); + } + else + return ERR_FILE_CREATE; + + return rc; } /*********************************************************** @@ -228,66 +206,60 @@ int FileOp::createFile( const char* fileName, int numOfBlock, * ERR_FILE_EXIST if file exists * ERR_FILE_CREATE if can not create the file ***********************************************************/ -int FileOp::createFile(FID fid, - int& allocSize, - uint16_t dbRoot, - uint32_t partition, - execplan::CalpontSystemCatalog::ColDataType colDataType, - const uint8_t* emptyVal, - int width) +int FileOp::createFile(FID fid, int& allocSize, uint16_t dbRoot, uint32_t partition, + execplan::CalpontSystemCatalog::ColDataType colDataType, const uint8_t* emptyVal, + int width) { - //std::cout << "Creating file oid: " << fid << - // "; compress: " << m_compressionType << std::endl; - char fileName[FILE_NAME_SIZE]; - int rc; + // std::cout << "Creating file oid: " << fid << + // "; compress: " << m_compressionType << std::endl; + char fileName[FILE_NAME_SIZE]; + int rc; - uint16_t segment = 0; // should always be 0 when starting a new column - RETURN_ON_ERROR( ( rc = oid2FileName( fid, fileName, true, - dbRoot, partition, segment ) ) ); + uint16_t segment = 0; // should always be 0 when starting a new column + RETURN_ON_ERROR((rc = oid2FileName(fid, fileName, true, dbRoot, partition, segment))); - //@Bug 3196 - if ( exists( fileName ) ) - return ERR_FILE_EXIST; + //@Bug 3196 + if (exists(fileName)) + return ERR_FILE_EXIST; - // allocatColExtent() treats dbRoot and partition as in/out - // arguments, so we need to pass in a non-const variable. - uint16_t dbRootx = dbRoot; - uint32_t partitionx = partition; + // allocatColExtent() treats dbRoot and partition as in/out + // arguments, so we need to pass in a non-const variable. + uint16_t dbRootx = dbRoot; + uint32_t partitionx = partition; - // Since we are creating a new column OID, we know partition - // and segment are 0, so we ignore their output values. -//timer.start( "allocateColExtent" ); + // Since we are creating a new column OID, we know partition + // and segment are 0, so we ignore their output values. + // timer.start( "allocateColExtent" ); - BRM::LBID_t startLbid; - uint32_t startBlock; - RETURN_ON_ERROR( BRMWrapper::getInstance()->allocateColExtentExactFile( - (OID)fid, (uint32_t)width, dbRootx, partitionx, segment, colDataType, - startLbid, allocSize, startBlock) ); + BRM::LBID_t startLbid; + uint32_t startBlock; + RETURN_ON_ERROR(BRMWrapper::getInstance()->allocateColExtentExactFile((OID)fid, (uint32_t)width, dbRootx, + partitionx, segment, colDataType, + startLbid, allocSize, startBlock)); - // We allocate a full extent from BRM, but only write an abbreviated 256K - // rows to disk for 1st extent, to conserve disk usage for small tables. - // One exception here is if we have rolled off partition 0, and we are - // adding a column to an existing table, then we are adding a column - // whose first partition is not 0. In this case, we know we are not - // dealing with a small table, so we init a full extent for 1st extent. - int totalSize = 0; + // We allocate a full extent from BRM, but only write an abbreviated 256K + // rows to disk for 1st extent, to conserve disk usage for small tables. + // One exception here is if we have rolled off partition 0, and we are + // adding a column to an existing table, then we are adding a column + // whose first partition is not 0. In this case, we know we are not + // dealing with a small table, so we init a full extent for 1st extent. + int totalSize = 0; - if (partition == 0) - totalSize = (INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK) * width; - else - totalSize = allocSize; // full extent if starting partition > 0 + if (partition == 0) + totalSize = (INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK) * width; + else + totalSize = allocSize; // full extent if starting partition > 0 - // Note we can't pass full file name to isDiskSpaceAvail() because the - // file does not exist yet, but passing DBRoot directory should suffice. - if ( !isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), totalSize) ) - { - return ERR_FILE_DISK_SPACE; - } + // Note we can't pass full file name to isDiskSpaceAvail() because the + // file does not exist yet, but passing DBRoot directory should suffice. + if (!isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), totalSize)) + { + return ERR_FILE_DISK_SPACE; + } -//timer.stop( "allocateColExtent" ); + // timer.stop( "allocateColExtent" ); - return createFile(fileName, totalSize, emptyVal, width, colDataType, - dbRoot, startLbid); + return createFile(fileName, totalSize, emptyVal, width, colDataType, dbRoot, startLbid); } /*********************************************************** @@ -300,12 +272,12 @@ int FileOp::createFile(FID fid, * ERR_FILE_NOT_EXIST if file does not exist * ERR_FILE_DELETE if can not delete a file ***********************************************************/ -int FileOp::deleteFile( const char* fileName ) const +int FileOp::deleteFile(const char* fileName) const { - if ( !exists( fileName ) ) - return ERR_FILE_NOT_EXIST; + if (!exists(fileName)) + return ERR_FILE_NOT_EXIST; - return ( IDBPolicy::remove( fileName ) == -1 ) ? ERR_FILE_DELETE : NO_ERROR; + return (IDBPolicy::remove(fileName) == -1) ? ERR_FILE_DELETE : NO_ERROR; } /*********************************************************** @@ -318,41 +290,38 @@ int FileOp::deleteFile( const char* fileName ) const * NO_ERROR if success * ERR_DM_CONVERT_OID if error occurs converting OID to file name ***********************************************************/ -int FileOp::deleteFile( FID fid ) const +int FileOp::deleteFile(FID fid) const { - char tempFileName[FILE_NAME_SIZE]; - char oidDirName [FILE_NAME_SIZE]; - char dbDir [MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; + char tempFileName[FILE_NAME_SIZE]; + char oidDirName[FILE_NAME_SIZE]; + char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; - RETURN_ON_ERROR((Convertor::oid2FileName( - fid, tempFileName, dbDir, 0, 0))); - sprintf(oidDirName, "%s/%s/%s/%s", - dbDir[0], dbDir[1], dbDir[2], dbDir[3]); - //std::cout << "Deleting files for OID " << fid << - // "; dirpath: " << oidDirName << std::endl; - //need check return code. - RETURN_ON_ERROR(BRMWrapper::getInstance()->deleteOid(fid)); + RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, 0, 0))); + sprintf(oidDirName, "%s/%s/%s/%s", dbDir[0], dbDir[1], dbDir[2], dbDir[3]); + // std::cout << "Deleting files for OID " << fid << + // "; dirpath: " << oidDirName << std::endl; + // need check return code. + RETURN_ON_ERROR(BRMWrapper::getInstance()->deleteOid(fid)); - std::vector dbRootPathList; - Config::getDBRootPathList( dbRootPathList ); + std::vector dbRootPathList; + Config::getDBRootPathList(dbRootPathList); - int rc; + int rc; - for (unsigned i = 0; i < dbRootPathList.size(); i++) + for (unsigned i = 0; i < dbRootPathList.size(); i++) + { + char rootOidDirName[FILE_NAME_SIZE]; + rc = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", dbRootPathList[i].c_str(), oidDirName); + + if (rc == FILE_NAME_SIZE || IDBPolicy::remove(rootOidDirName) != 0) { - char rootOidDirName[FILE_NAME_SIZE]; - rc = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", - dbRootPathList[i].c_str(), oidDirName); - - if ( rc == FILE_NAME_SIZE || IDBPolicy::remove( rootOidDirName ) != 0 ) - { - ostringstream oss; - oss << "Unable to remove " << rootOidDirName; - throw std::runtime_error( oss.str() ); - } + ostringstream oss; + oss << "Unable to remove " << rootOidDirName; + throw std::runtime_error(oss.str()); } + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -365,40 +334,37 @@ int FileOp::deleteFile( FID fid ) const * NO_ERROR if success * ERR_DM_CONVERT_OID if error occurs converting OID to file name ***********************************************************/ -int FileOp::deleteFiles( const std::vector& fids ) const +int FileOp::deleteFiles(const std::vector& fids) const { - char tempFileName[FILE_NAME_SIZE]; - char oidDirName [FILE_NAME_SIZE]; - char dbDir [MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; - std::vector dbRootPathList; - Config::getDBRootPathList( dbRootPathList ); - int rc; + char tempFileName[FILE_NAME_SIZE]; + char oidDirName[FILE_NAME_SIZE]; + char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; + std::vector dbRootPathList; + Config::getDBRootPathList(dbRootPathList); + int rc; - for ( unsigned n = 0; n < fids.size(); n++ ) + for (unsigned n = 0; n < fids.size(); n++) + { + RETURN_ON_ERROR((Convertor::oid2FileName(fids[n], tempFileName, dbDir, 0, 0))); + sprintf(oidDirName, "%s/%s/%s/%s", dbDir[0], dbDir[1], dbDir[2], dbDir[3]); + // std::cout << "Deleting files for OID " << fid << + // "; dirpath: " << oidDirName << std::endl; + + for (unsigned i = 0; i < dbRootPathList.size(); i++) { - RETURN_ON_ERROR((Convertor::oid2FileName( - fids[n], tempFileName, dbDir, 0, 0))); - sprintf(oidDirName, "%s/%s/%s/%s", - dbDir[0], dbDir[1], dbDir[2], dbDir[3]); - //std::cout << "Deleting files for OID " << fid << - // "; dirpath: " << oidDirName << std::endl; + char rootOidDirName[FILE_NAME_SIZE]; + rc = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", dbRootPathList[i].c_str(), oidDirName); - for (unsigned i = 0; i < dbRootPathList.size(); i++) - { - char rootOidDirName[FILE_NAME_SIZE]; - rc = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", dbRootPathList[i].c_str(), - oidDirName); - - if ( rc == FILE_NAME_SIZE || IDBPolicy::remove( rootOidDirName ) != 0 ) - { - ostringstream oss; - oss << "Unable to remove " << rootOidDirName; - throw std::runtime_error( oss.str() ); - } - } + if (rc == FILE_NAME_SIZE || IDBPolicy::remove(rootOidDirName) != 0) + { + ostringstream oss; + oss << "Unable to remove " << rootOidDirName; + throw std::runtime_error(oss.str()); + } } + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -412,57 +378,52 @@ int FileOp::deleteFiles( const std::vector& fids ) const * NO_ERROR if success * ERR_DM_CONVERT_OID if error occurs converting OID to file name ***********************************************************/ -int FileOp::deletePartitions( const std::vector& fids, - const std::vector& partitions ) const +int FileOp::deletePartitions(const std::vector& fids, + const std::vector& partitions) const { - char tempFileName[FILE_NAME_SIZE]; - char oidDirName [FILE_NAME_SIZE]; - char dbDir [MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; - char rootOidDirName[FILE_NAME_SIZE]; - char partitionDirName[FILE_NAME_SIZE]; - int rcd, rcp; + char tempFileName[FILE_NAME_SIZE]; + char oidDirName[FILE_NAME_SIZE]; + char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; + char rootOidDirName[FILE_NAME_SIZE]; + char partitionDirName[FILE_NAME_SIZE]; + int rcd, rcp; - for (uint32_t i = 0; i < partitions.size(); i++) + for (uint32_t i = 0; i < partitions.size(); i++) + { + RETURN_ON_ERROR((Convertor::oid2FileName(partitions[i].oid, tempFileName, dbDir, partitions[i].lp.pp, + partitions[i].lp.seg))); + sprintf(oidDirName, "%s/%s/%s/%s/%s", dbDir[0], dbDir[1], dbDir[2], dbDir[3], dbDir[4]); + // config expects dbroot starting from 0 + std::string rt(Config::getDBRootByNum(partitions[i].lp.dbroot)); + rcd = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", rt.c_str(), tempFileName); + rcp = snprintf(partitionDirName, FILE_NAME_SIZE, "%s/%s", rt.c_str(), oidDirName); + + if (rcd == FILE_NAME_SIZE || rcp == FILE_NAME_SIZE || IDBPolicy::remove(rootOidDirName) != 0) { - RETURN_ON_ERROR((Convertor::oid2FileName( - partitions[i].oid, tempFileName, dbDir, - partitions[i].lp.pp, partitions[i].lp.seg))); - sprintf(oidDirName, "%s/%s/%s/%s/%s", - dbDir[0], dbDir[1], dbDir[2], dbDir[3], dbDir[4]); - // config expects dbroot starting from 0 - std::string rt( Config::getDBRootByNum(partitions[i].lp.dbroot) ); - rcd = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", - rt.c_str(), tempFileName); - rcp = snprintf(partitionDirName, FILE_NAME_SIZE, "%s/%s", - rt.c_str(), oidDirName); - - if ( rcd == FILE_NAME_SIZE || rcp == FILE_NAME_SIZE - || IDBPolicy::remove( rootOidDirName ) != 0 ) - { - ostringstream oss; - oss << "Unable to remove " << rootOidDirName; - throw std::runtime_error( oss.str() ); - } - - list dircontents; - - if ( IDBPolicy::listDirectory( partitionDirName, dircontents ) == 0 ) - { - // the directory exists, now check if empty - if ( dircontents.size() == 0 ) - { - // empty directory - if ( IDBPolicy::remove( partitionDirName ) != 0 ) - { - ostringstream oss; - oss << "Unable to remove " << rootOidDirName; - throw std::runtime_error( oss.str() ); - } - } - } + ostringstream oss; + oss << "Unable to remove " << rootOidDirName; + throw std::runtime_error(oss.str()); } - return NO_ERROR; + list dircontents; + + if (IDBPolicy::listDirectory(partitionDirName, dircontents) == 0) + { + // the directory exists, now check if empty + if (dircontents.size() == 0) + { + // empty directory + if (IDBPolicy::remove(partitionDirName) != 0) + { + ostringstream oss; + oss << "Unable to remove " << rootOidDirName; + throw std::runtime_error(oss.str()); + } + } + } + } + + return NO_ERROR; } /*********************************************************** @@ -476,15 +437,13 @@ int FileOp::deletePartitions( const std::vector& fids, * RETURN: * NO_ERROR if success ***********************************************************/ -int FileOp::deleteFile( FID fid, uint16_t dbRoot, - uint32_t partition, uint16_t segment ) const +int FileOp::deleteFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const { - char fileName[FILE_NAME_SIZE]; + char fileName[FILE_NAME_SIZE]; - RETURN_ON_ERROR( getFileName( fid, fileName, - dbRoot, partition, segment) ); + RETURN_ON_ERROR(getFileName(fid, fileName, dbRoot, partition, segment)); - return ( deleteFile( fileName ) ); + return (deleteFile(fileName)); } /*********************************************************** @@ -495,9 +454,9 @@ int FileOp::deleteFile( FID fid, uint16_t dbRoot, * RETURN: * true if exists, false otherwise ***********************************************************/ -bool FileOp::exists( const char* fileName ) const +bool FileOp::exists(const char* fileName) const { - return IDBPolicy::exists( fileName ); + return IDBPolicy::exists(fileName); } /*********************************************************** @@ -511,16 +470,14 @@ bool FileOp::exists( const char* fileName ) const * RETURN: * true if exists, false otherwise ***********************************************************/ -bool FileOp::exists( FID fid, uint16_t dbRoot, - uint32_t partition, uint16_t segment ) const +bool FileOp::exists(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const { - char fileName[FILE_NAME_SIZE]; + char fileName[FILE_NAME_SIZE]; - if (getFileName(fid, fileName, dbRoot, partition, - segment) != NO_ERROR) - return false; + if (getFileName(fid, fileName, dbRoot, partition, segment) != NO_ERROR) + return false; - return exists( fileName ); + return exists(fileName); } /*********************************************************** @@ -531,16 +488,16 @@ bool FileOp::exists( FID fid, uint16_t dbRoot, * RETURN: * true if exists, false otherwise ***********************************************************/ -bool FileOp::existsOIDDir( FID fid ) const +bool FileOp::existsOIDDir(FID fid) const { - char fileName[FILE_NAME_SIZE]; + char fileName[FILE_NAME_SIZE]; - if (oid2DirName( fid, fileName ) != NO_ERROR) - { - return false; - } + if (oid2DirName(fid, fileName) != NO_ERROR) + { + return false; + } - return exists( fileName ); + return exists(fileName); } /*********************************************************** @@ -573,294 +530,243 @@ bool FileOp::existsOIDDir( FID fid ) const * NO_ERROR if success * else the applicable error code is returned ***********************************************************/ -int FileOp::extendFile( - OID oid, - const uint8_t* emptyVal, - int width, - execplan::CalpontSystemCatalog::ColDataType colDataType, - HWM hwm, - BRM::LBID_t startLbid, - int allocSize, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - IDBDataFile*& pFile, - bool& newFile, - char* hdrs) +int FileOp::extendFile(OID oid, const uint8_t* emptyVal, int width, + execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm, + BRM::LBID_t startLbid, int allocSize, uint16_t dbRoot, uint32_t partition, + uint16_t segment, std::string& segFile, IDBDataFile*& pFile, bool& newFile, char* hdrs) { - int rc = NO_ERROR; - pFile = 0; - segFile.clear(); - newFile = false; - char fileName[FILE_NAME_SIZE]; + int rc = NO_ERROR; + pFile = 0; + segFile.clear(); + newFile = false; + char fileName[FILE_NAME_SIZE]; - // If starting hwm or fbo is 0 then this is the first extent of a new file, - // else we are adding an extent to an existing segment file - if (hwm > 0) // db segment file should exist + // If starting hwm or fbo is 0 then this is the first extent of a new file, + // else we are adding an extent to an existing segment file + if (hwm > 0) // db segment file should exist + { + RETURN_ON_ERROR(oid2FileName(oid, fileName, false, dbRoot, partition, segment)); + segFile = fileName; + + if (!exists(fileName)) { - RETURN_ON_ERROR( oid2FileName(oid, fileName, false, - dbRoot, partition, segment) ); - segFile = fileName; - - if (!exists(fileName)) - { - ostringstream oss; - oss << "oid: " << oid << " with path " << segFile; - logging::Message::Args args; - args.add("File not found "); - args.add(oss.str()); - args.add(""); - args.add(""); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0001); - return ERR_FILE_NOT_EXIST; - } - - pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" );//old file - - if (pFile == 0) - { - ostringstream oss; - oss << "oid: " << oid << " with path " << segFile; - logging::Message::Args args; - args.add("Error opening file "); - args.add(oss.str()); - args.add(""); - args.add(""); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0001); - return ERR_FILE_OPEN; - } - - if ( isDebug(DEBUG_1) && getLogger() ) - { - std::ostringstream oss; - oss << "Opening existing column file (extendFile)" << - ": OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; LBID-" << startLbid << - "; hwm-" << hwm << - "; file-" << segFile; - getLogger()->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - // @bug 5349: check that new extent's fbo is not past current EOF - if (m_compressionType) - { - char hdrsIn[ compress::CompressInterface::HDR_BUF_LEN * 2 ]; - RETURN_ON_ERROR( readHeaders(pFile, hdrsIn) ); - - std::unique_ptr compressor( - compress::getCompressInterfaceByType( - compress::CompressInterface::getCompressionType(hdrsIn))); - - unsigned int ptrCount = - compress::CompressInterface::getPtrCount(hdrsIn); - unsigned int chunkIndex = 0; - unsigned int blockOffsetWithinChunk = 0; - compressor->locateBlock((hwm - 1), chunkIndex, - blockOffsetWithinChunk); - - //std::ostringstream oss1; - //oss1 << "Extending compressed column file"<< - // ": OID-" << oid << - // "; LBID-" << startLbid << - // "; fbo-" << hwm << - // "; file-" << segFile << - // "; chkidx-" << chunkIndex<< - // "; numPtrs-"<< ptrCount; - //getLogger()->logMsg( oss1.str(), MSGLVL_INFO2 ); - - if (chunkIndex >= ptrCount) - { - ostringstream oss; - oss << "oid: " << oid << " with path " << segFile << - "; new extent fbo " << hwm << "; number of " - "compressed chunks " << ptrCount << - "; chunkIndex " << chunkIndex; - logging::Message::Args args; - args.add("compressed"); - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0103); - - // Expand the partial extent to full with emptyVal - // Since fillCompColumnExtentEmptyChunks() messes with the - // file on disk, we need to close it and reopen after or - // the cache isn't updated. - if ((pFile)) - closeFile( pFile ); - - pFile = NULL; - string failedTask; // For return error message, if any. - rc = FileOp::fillCompColumnExtentEmptyChunks( - oid, width, emptyVal, dbRoot, partition, segment, - colDataType, hwm, segFile, failedTask); - - if (rc != NO_ERROR) - { - if (getLogger()) - { - std::ostringstream oss; - oss << "FileOp::extendFile: error padding partial compressed extent for " << - "column OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; hwm-" << hwm << - " " << failedTask.c_str(); - getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - } - - return rc; - } - - pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" ); // modified file - } - - // Get the latest file header for the caller. If a partial extent was filled out, - // this will be different than when we first read the headers. - if (hdrs) - { - RETURN_ON_ERROR( readHeaders(pFile, hdrs) ); - } - } - else - { - long long fileSize; - RETURN_ON_ERROR( getFileSize(pFile, fileSize) ); - long long calculatedFileSize = ((long long)hwm) * BYTE_PER_BLOCK; - - //std::ostringstream oss2; - //oss2 << "Extending uncompressed column file"<< - // ": OID-" << oid << - // "; LBID-" << startLbid << - // "; fbo-" << hwm << - // "; file-" << segFile << - // "; filesize-"<logMsg( oss2.str(), MSGLVL_INFO2 ); - - if (calculatedFileSize > fileSize) - { - ostringstream oss; - oss << "oid: " << oid << " with path " << segFile << - "; new extent fbo " << hwm << "; file size (bytes) " << - fileSize; - logging::Message::Args args; - args.add("uncompressed"); - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0103); - // Expand the partial extent to full with emptyVal - // This generally won't ever happen, as uncompressed files - // are created with full extents. - rc = FileOp::expandAbbrevColumnExtent( pFile, dbRoot, - emptyVal, width, - colDataType ); - - if (rc != NO_ERROR) - { - if (getLogger()) - { - std::ostringstream oss; - oss << "FileOp::extendFile: error padding partial uncompressed extent for " << - "column OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; hwm-" << hwm; - getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); - } - - return rc; - } - } - } + ostringstream oss; + oss << "oid: " << oid << " with path " << segFile; + logging::Message::Args args; + args.add("File not found "); + args.add(oss.str()); + args.add(""); + args.add(""); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0001); + return ERR_FILE_NOT_EXIST; } - else // db segment file should not exist + + pFile = openFile(oid, dbRoot, partition, segment, segFile, "r+b"); // old file + + if (pFile == 0) { - RETURN_ON_ERROR( oid2FileName(oid, fileName, true, - dbRoot, partition, segment) ); - segFile = fileName; - - // if obsolete file exists, "w+b" will truncate and write over - pFile = openFile( fileName, "w+b" );//new file - if (pFile == 0) - return ERR_FILE_CREATE; - - { - // We presume the path will contain / - std::string filePath(fileName); - if (chownDataPath(filePath)) - return ERR_FILE_CHOWN; - } - - - newFile = true; - - if ( isDebug(DEBUG_1) && getLogger() ) - { - std::ostringstream oss; - oss << "Opening new column file" << - ": OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; LBID-" << startLbid << - "; hwm-" << hwm << - "; file-" << segFile; - getLogger()->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - if ((m_compressionType) && (hdrs)) - { - compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); - compress::CompressInterface::setLBIDByIndex(hdrs, startLbid, 0); - } + ostringstream oss; + oss << "oid: " << oid << " with path " << segFile; + logging::Message::Args args; + args.add("Error opening file "); + args.add(oss.str()); + args.add(""); + args.add(""); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0001); + return ERR_FILE_OPEN; } + if (isDebug(DEBUG_1) && getLogger()) + { + std::ostringstream oss; + oss << "Opening existing column file (extendFile)" + << ": OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" << segment + << "; LBID-" << startLbid << "; hwm-" << hwm << "; file-" << segFile; + getLogger()->logMsg(oss.str(), MSGLVL_INFO2); + } + + // @bug 5349: check that new extent's fbo is not past current EOF + if (m_compressionType) + { + char hdrsIn[compress::CompressInterface::HDR_BUF_LEN * 2]; + RETURN_ON_ERROR(readHeaders(pFile, hdrsIn)); + + std::unique_ptr compressor( + compress::getCompressInterfaceByType(compress::CompressInterface::getCompressionType(hdrsIn))); + + unsigned int ptrCount = compress::CompressInterface::getPtrCount(hdrsIn); + unsigned int chunkIndex = 0; + unsigned int blockOffsetWithinChunk = 0; + compressor->locateBlock((hwm - 1), chunkIndex, blockOffsetWithinChunk); + + // std::ostringstream oss1; + // oss1 << "Extending compressed column file"<< + // ": OID-" << oid << + // "; LBID-" << startLbid << + // "; fbo-" << hwm << + // "; file-" << segFile << + // "; chkidx-" << chunkIndex<< + // "; numPtrs-"<< ptrCount; + // getLogger()->logMsg( oss1.str(), MSGLVL_INFO2 ); + + if (chunkIndex >= ptrCount) + { + ostringstream oss; + oss << "oid: " << oid << " with path " << segFile << "; new extent fbo " << hwm + << "; number of " + "compressed chunks " + << ptrCount << "; chunkIndex " << chunkIndex; + logging::Message::Args args; + args.add("compressed"); + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0103); + + // Expand the partial extent to full with emptyVal + // Since fillCompColumnExtentEmptyChunks() messes with the + // file on disk, we need to close it and reopen after or + // the cache isn't updated. + if ((pFile)) + closeFile(pFile); + + pFile = NULL; + string failedTask; // For return error message, if any. + rc = FileOp::fillCompColumnExtentEmptyChunks(oid, width, emptyVal, dbRoot, partition, segment, + colDataType, hwm, segFile, failedTask); + + if (rc != NO_ERROR) + { + if (getLogger()) + { + std::ostringstream oss; + oss << "FileOp::extendFile: error padding partial compressed extent for " + << "column OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" + << segment << "; hwm-" << hwm << " " << failedTask.c_str(); + getLogger()->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + } + + return rc; + } + + pFile = openFile(oid, dbRoot, partition, segment, segFile, "r+b"); // modified file + } + + // Get the latest file header for the caller. If a partial extent was filled out, + // this will be different than when we first read the headers. + if (hdrs) + { + RETURN_ON_ERROR(readHeaders(pFile, hdrs)); + } + } + else + { + long long fileSize; + RETURN_ON_ERROR(getFileSize(pFile, fileSize)); + long long calculatedFileSize = ((long long)hwm) * BYTE_PER_BLOCK; + + // std::ostringstream oss2; + // oss2 << "Extending uncompressed column file"<< + // ": OID-" << oid << + // "; LBID-" << startLbid << + // "; fbo-" << hwm << + // "; file-" << segFile << + // "; filesize-"<logMsg( oss2.str(), MSGLVL_INFO2 ); + + if (calculatedFileSize > fileSize) + { + ostringstream oss; + oss << "oid: " << oid << " with path " << segFile << "; new extent fbo " << hwm + << "; file size (bytes) " << fileSize; + logging::Message::Args args; + args.add("uncompressed"); + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0103); + // Expand the partial extent to full with emptyVal + // This generally won't ever happen, as uncompressed files + // are created with full extents. + rc = FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width, colDataType); + + if (rc != NO_ERROR) + { + if (getLogger()) + { + std::ostringstream oss; + oss << "FileOp::extendFile: error padding partial uncompressed extent for " + << "column OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" + << segment << "; hwm-" << hwm; + getLogger()->logMsg(oss.str(), rc, MSGLVL_CRITICAL); + } + + return rc; + } + } + } + } + else // db segment file should not exist + { + RETURN_ON_ERROR(oid2FileName(oid, fileName, true, dbRoot, partition, segment)); + segFile = fileName; + + // if obsolete file exists, "w+b" will truncate and write over + pFile = openFile(fileName, "w+b"); // new file + if (pFile == 0) + return ERR_FILE_CREATE; + + { + // We presume the path will contain / + std::string filePath(fileName); + if (chownDataPath(filePath)) + return ERR_FILE_CHOWN; + } + + newFile = true; + + if (isDebug(DEBUG_1) && getLogger()) + { + std::ostringstream oss; + oss << "Opening new column file" + << ": OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" << segment + << "; LBID-" << startLbid << "; hwm-" << hwm << "; file-" << segFile; + getLogger()->logMsg(oss.str(), MSGLVL_INFO2); + } + + if ((m_compressionType) && (hdrs)) + { + compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); + compress::CompressInterface::setLBIDByIndex(hdrs, startLbid, 0); + } + } + #ifdef _MSC_VER - //Need to call the win version with a dir, not a file - if (!isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), allocSize)) + // Need to call the win version with a dir, not a file + if (!isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), allocSize)) #else - if ( !isDiskSpaceAvail(segFile, allocSize) ) + if (!isDiskSpaceAvail(segFile, allocSize)) #endif - { - return ERR_FILE_DISK_SPACE; - } + { + return ERR_FILE_DISK_SPACE; + } - // We set to EOF just before we start adding the blocks for the new extent. - // At one time, I considered changing this to seek to the HWM block, but - // with compressed files, this is murky; do I find and seek to the chunk - // containing the HWM block? So I left as-is for now, seeking to EOF. - rc = setFileOffset(pFile, 0, SEEK_END); - - if (rc != NO_ERROR) - return rc; - - // Initialize the contents of the extent. - // MCOL-498 optimize full extent creation. - rc = initColumnExtent( pFile, - dbRoot, - allocSize, - emptyVal, - width, - colDataType, - newFile, // new or existing file - false, // don't expand; new extent - false, // add full (not abbreviated) extent - true, // try to optimize extent creation - startLbid ); + // We set to EOF just before we start adding the blocks for the new extent. + // At one time, I considered changing this to seek to the HWM block, but + // with compressed files, this is murky; do I find and seek to the chunk + // containing the HWM block? So I left as-is for now, seeking to EOF. + rc = setFileOffset(pFile, 0, SEEK_END); + if (rc != NO_ERROR) return rc; + + // Initialize the contents of the extent. + // MCOL-498 optimize full extent creation. + rc = initColumnExtent(pFile, dbRoot, allocSize, emptyVal, width, colDataType, + newFile, // new or existing file + false, // don't expand; new extent + false, // add full (not abbreviated) extent + true, // try to optimize extent creation + startLbid); + + return rc; } /*********************************************************** @@ -882,145 +788,117 @@ int FileOp::extendFile( * RETURN: * none ***********************************************************/ -int FileOp::addExtentExactFile( - OID oid, - const uint8_t* emptyVal, - int width, - int& allocSize, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - std::string& segFile, - BRM::LBID_t& startLbid, - bool& newFile, - char* hdrs) +int FileOp::addExtentExactFile(OID oid, const uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, + uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, std::string& segFile, + BRM::LBID_t& startLbid, bool& newFile, char* hdrs) { - int rc = NO_ERROR; - IDBDataFile* pFile = 0; - segFile.clear(); - newFile = false; - HWM hwm; + int rc = NO_ERROR; + IDBDataFile* pFile = 0; + segFile.clear(); + newFile = false; + HWM hwm; - // Allocate the new extent in the ExtentMap - RETURN_ON_ERROR( BRMWrapper::getInstance()->allocateColExtentExactFile( - oid, width, dbRoot, partition, segment, colDataType, startLbid, allocSize, hwm)); + // Allocate the new extent in the ExtentMap + RETURN_ON_ERROR(BRMWrapper::getInstance()->allocateColExtentExactFile( + oid, width, dbRoot, partition, segment, colDataType, startLbid, allocSize, hwm)); - // Determine the existence of the "next" segment file, and either open - // or create the segment file accordingly. - if (exists(oid, dbRoot, partition, segment)) + // Determine the existence of the "next" segment file, and either open + // or create the segment file accordingly. + if (exists(oid, dbRoot, partition, segment)) + { + pFile = openFile(oid, dbRoot, partition, segment, segFile, "r+b"); // old file + + if (pFile == 0) { - pFile = openFile( oid, dbRoot, partition, segment, - segFile, "r+b" );//old file - - if (pFile == 0) - { - ostringstream oss; - oss << "oid: " << oid << " with path " << segFile; - logging::Message::Args args; - args.add("Error opening file "); - args.add(oss.str()); - args.add(""); - args.add(""); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0001); - return ERR_FILE_OPEN; - } - - if ( isDebug(DEBUG_1) && getLogger() ) - { - std::ostringstream oss; - oss << "Opening existing column file" << - ": OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; LBID-" << startLbid << - "; hwm-" << hwm << - "; file-" << segFile; - getLogger()->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - if ((m_compressionType) && (hdrs)) - { - rc = readHeaders(pFile, hdrs); - - if (rc != NO_ERROR) - return rc; - } + ostringstream oss; + oss << "oid: " << oid << " with path " << segFile; + logging::Message::Args args; + args.add("Error opening file "); + args.add(oss.str()); + args.add(""); + args.add(""); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0001); + return ERR_FILE_OPEN; } - else + + if (isDebug(DEBUG_1) && getLogger()) { - char fileName[FILE_NAME_SIZE]; - RETURN_ON_ERROR( oid2FileName(oid, fileName, true, - dbRoot, partition, segment) ); - segFile = fileName; - - pFile = openFile( fileName, "w+b" );//new file - - if (pFile == 0) - return ERR_FILE_CREATE; - - newFile = true; - - if ( isDebug(DEBUG_1) && getLogger() ) - { - std::ostringstream oss; - oss << "Opening new column file" << - ": OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; LBID-" << startLbid << - "; hwm-" << hwm << - "; file-" << segFile; - getLogger()->logMsg( oss.str(), MSGLVL_INFO2 ); - } - - if ((m_compressionType) && (hdrs)) - { - compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); - compress::CompressInterface::setLBIDByIndex(hdrs, startLbid, 0); - } + std::ostringstream oss; + oss << "Opening existing column file" + << ": OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" << segment + << "; LBID-" << startLbid << "; hwm-" << hwm << "; file-" << segFile; + getLogger()->logMsg(oss.str(), MSGLVL_INFO2); } + if ((m_compressionType) && (hdrs)) + { + rc = readHeaders(pFile, hdrs); + + if (rc != NO_ERROR) + return rc; + } + } + else + { + char fileName[FILE_NAME_SIZE]; + RETURN_ON_ERROR(oid2FileName(oid, fileName, true, dbRoot, partition, segment)); + segFile = fileName; + + pFile = openFile(fileName, "w+b"); // new file + + if (pFile == 0) + return ERR_FILE_CREATE; + + newFile = true; + + if (isDebug(DEBUG_1) && getLogger()) + { + std::ostringstream oss; + oss << "Opening new column file" + << ": OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" << segment + << "; LBID-" << startLbid << "; hwm-" << hwm << "; file-" << segFile; + getLogger()->logMsg(oss.str(), MSGLVL_INFO2); + } + + if ((m_compressionType) && (hdrs)) + { + compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); + compress::CompressInterface::setLBIDByIndex(hdrs, startLbid, 0); + } + } + #ifdef _MSC_VER - //Need to call the win version with a dir, not a file - if (!isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), allocSize)) + // Need to call the win version with a dir, not a file + if (!isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), allocSize)) #else - if ( !isDiskSpaceAvail(segFile, allocSize) ) + if (!isDiskSpaceAvail(segFile, allocSize)) #endif - { - return ERR_FILE_DISK_SPACE; - } + { + return ERR_FILE_DISK_SPACE; + } - // We set to EOF just before we start adding the blocks for the new extent. - // At one time, I considered changing this to seek to the HWM block, but - // with compressed files, this is murky; do I find and seek to the chunk - // containing the HWM block? So I left as-is for now, seeking to EOF. - rc = setFileOffset(pFile, 0, SEEK_END); + // We set to EOF just before we start adding the blocks for the new extent. + // At one time, I considered changing this to seek to the HWM block, but + // with compressed files, this is murky; do I find and seek to the chunk + // containing the HWM block? So I left as-is for now, seeking to EOF. + rc = setFileOffset(pFile, 0, SEEK_END); - if (rc != NO_ERROR) - return rc; - - // Initialize the contents of the extent. - // CS doesn't optimize file operations to have a valid - // segment files with empty magics - rc = initColumnExtent( pFile, - dbRoot, - allocSize, - emptyVal, - width, - colDataType, - newFile, // new or existing file - false, // don't expand; new extent - false, // add full (not abbreviated) extent - startLbid ); - - closeFile( pFile ); + if (rc != NO_ERROR) return rc; + + // Initialize the contents of the extent. + // CS doesn't optimize file operations to have a valid + // segment files with empty magics + rc = initColumnExtent(pFile, dbRoot, allocSize, emptyVal, width, colDataType, + newFile, // new or existing file + false, // don't expand; new extent + false, // add full (not abbreviated) extent + startLbid); + + closeFile(pFile); + return rc; } /*********************************************************** @@ -1052,171 +930,160 @@ int FileOp::addExtentExactFile( * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. ***********************************************************/ -int FileOp::initColumnExtent( - IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - const uint8_t* emptyVal, - int width, - execplan::CalpontSystemCatalog::ColDataType colDataType, - bool bNewFile, - bool bExpandExtent, - bool bAbbrevExtent, - bool bOptExtension, - int64_t lbid) +int FileOp::initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, + int width, execplan::CalpontSystemCatalog::ColDataType colDataType, + bool bNewFile, bool bExpandExtent, bool bAbbrevExtent, bool bOptExtension, + int64_t lbid) { - if ((bNewFile) && (m_compressionType)) - { - char hdrs[CompressInterface::HDR_BUF_LEN * 2]; - compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); - compress::CompressInterface::setLBIDByIndex(hdrs, lbid, 0); - if (bAbbrevExtent) - compress::CompressInterface::setBlockCount(hdrs, nBlocks); + if ((bNewFile) && (m_compressionType)) + { + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); + compress::CompressInterface::setLBIDByIndex(hdrs, lbid, 0); + if (bAbbrevExtent) + compress::CompressInterface::setBlockCount(hdrs, nBlocks); - RETURN_ON_ERROR(writeHeaders(pFile, hdrs)); + RETURN_ON_ERROR(writeHeaders(pFile, hdrs)); + } + + // @bug5769 Don't initialize extents or truncate db files on HDFS + if (idbdatafile::IDBPolicy::useHdfs()) + { + //@Bug 3219. update the compression header after the extent is expanded. + if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) + { + updateColumnExtent(pFile, nBlocks, lbid); } - // @bug5769 Don't initialize extents or truncate db files on HDFS - if (idbdatafile::IDBPolicy::useHdfs()) - { - //@Bug 3219. update the compression header after the extent is expanded. - if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) - { - updateColumnExtent(pFile, nBlocks, lbid); - } + // @bug 2378. Synchronize here to avoid write buffer pile up too much, + // which could cause controllernode to timeout later when it needs to + // save a snapshot. + pFile->flush(); + } + else + { + // Create vector of mutexes used to serialize extent access per DBRoot + initDbRootExtentMutexes(); - // @bug 2378. Synchronize here to avoid write buffer pile up too much, - // which could cause controllernode to timeout later when it needs to - // save a snapshot. - pFile->flush(); + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. + if (bOptExtension) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpaceDisabled(dbRoot)) ? bOptExtension : false; } + // Reduce number of blocks allocated for abbreviated extents thus + // CS writes less when creates a new table. This couldn't be zero + // b/c Snappy compressed file format doesn't tolerate empty files. + int realNBlocks = (bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK) ? 3 : nBlocks; + + // Determine the number of blocks in each call to fwrite(), and the + // number of fwrite() calls to make, based on this. In other words, + // we put a cap on the "writeSize" so that we don't allocate and write + // an entire extent at once for the 64M row extents. If we are + // expanding an abbreviated 64M extent, we may not have an even + // multiple of MAX_NBLOCKS to write; remWriteSize is the number of + // blocks above and beyond loopCount*MAX_NBLOCKS. + int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int loopCount = 1; + int remWriteSize = 0; + + if (realNBlocks > MAX_NBLOCKS) // 64M row extent size + { + writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; + loopCount = realNBlocks / MAX_NBLOCKS; + remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); + } + + // Allocate a buffer, initialize it, and use it to create the extent + idbassert(dbRoot > 0); +#ifdef PROFILE + + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_WAIT_TO_EXPAND_COL_EXTENT); else + Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); + +#endif + boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); +#ifdef PROFILE + + if (bExpandExtent) + Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_COL_EXTENT); + else + Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); +#endif + // Skip space preallocation if configured so + // fallback to sequential write otherwise. + // Couldn't avoid preallocation for full extents, + // e.g. ADD COLUMN DDL b/c CS has to fill the file + // with empty magics. + if (!bOptExtension || !m_compressionType) { - // Create vector of mutexes used to serialize extent access per DBRoot - initDbRootExtentMutexes( ); - - // MCOL-498 Skip the huge preallocations if the option is set - // for the dbroot. This check is skiped for abbreviated extent. - // IMO it is better to check bool then to call a function. - if ( bOptExtension ) - { - bOptExtension = (idbdatafile::IDBPolicy::PreallocSpaceDisabled(dbRoot)) - ? bOptExtension : false; - } - // Reduce number of blocks allocated for abbreviated extents thus - // CS writes less when creates a new table. This couldn't be zero - // b/c Snappy compressed file format doesn't tolerate empty files. - int realNBlocks = ( bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK ) ? 3 : nBlocks; - - // Determine the number of blocks in each call to fwrite(), and the - // number of fwrite() calls to make, based on this. In other words, - // we put a cap on the "writeSize" so that we don't allocate and write - // an entire extent at once for the 64M row extents. If we are - // expanding an abbreviated 64M extent, we may not have an even - // multiple of MAX_NBLOCKS to write; remWriteSize is the number of - // blocks above and beyond loopCount*MAX_NBLOCKS. - int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size - int loopCount = 1; - int remWriteSize = 0; - - if (realNBlocks > MAX_NBLOCKS) // 64M row extent size - { - writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = realNBlocks / MAX_NBLOCKS; - remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); - } - - // Allocate a buffer, initialize it, and use it to create the extent - idbassert(dbRoot > 0); #ifdef PROFILE + Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); +#endif + // Allocate buffer, store it in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + { + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr(writeBuf); + + setEmptyBuf(writeBuf, writeSize, emptyVal, width); + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_INIT_COL_EXTENT); if (bExpandExtent) - Stats::startParseEvent(WE_STATS_WAIT_TO_EXPAND_COL_EXTENT); + Stats::startParseEvent(WE_STATS_EXPAND_COL_EXTENT); else - Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); + Stats::startParseEvent(WE_STATS_CREATE_COL_EXTENT); #endif - boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); -#ifdef PROFILE - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_COL_EXTENT); - else - Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); -#endif - // Skip space preallocation if configured so - // fallback to sequential write otherwise. - // Couldn't avoid preallocation for full extents, - // e.g. ADD COLUMN DDL b/c CS has to fill the file - // with empty magics. - if ( !bOptExtension || !m_compressionType ) + // std::ostringstream oss; + // oss << "initColExtent: width-" << width << + //"; loopCount-" << loopCount << + //"; writeSize-" << writeSize; + // std::cout << oss.str() << std::endl; + if (remWriteSize > 0) { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); -#endif - // Allocate buffer, store it in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. - { - - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); - - setEmptyBuf( writeBuf, writeSize, emptyVal, width ); - - #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_INIT_COL_EXTENT); - - if (bExpandExtent) - Stats::startParseEvent(WE_STATS_EXPAND_COL_EXTENT); - else - Stats::startParseEvent(WE_STATS_CREATE_COL_EXTENT); - - #endif - - //std::ostringstream oss; - //oss << "initColExtent: width-" << width << - //"; loopCount-" << loopCount << - //"; writeSize-" << writeSize; - //std::cout << oss.str() << std::endl; - if (remWriteSize > 0) - { - if ( pFile->write( writeBuf, remWriteSize ) != remWriteSize ) - { - return ERR_FILE_WRITE; - } - } - - for (int j = 0; j < loopCount; j++) - { - if ( pFile->write( writeBuf, writeSize ) != writeSize ) - { - return ERR_FILE_WRITE; - } - } - } - - //@Bug 3219. update the compression header after the extent is expanded. - if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) - { - updateColumnExtent(pFile, nBlocks, lbid); - } - - // @bug 2378. Synchronize here to avoid write buffer pile up too much, - // which could cause controllernode to timeout later when it needs to - // save a snapshot. - pFile->flush(); - -#ifdef PROFILE - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_COL_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); -#endif - + if (pFile->write(writeBuf, remWriteSize) != remWriteSize) + { + return ERR_FILE_WRITE; + } } + + for (int j = 0; j < loopCount; j++) + { + if (pFile->write(writeBuf, writeSize) != writeSize) + { + return ERR_FILE_WRITE; + } + } + } + + //@Bug 3219. update the compression header after the extent is expanded. + if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) + { + updateColumnExtent(pFile, nBlocks, lbid); + } + + // @bug 2378. Synchronize here to avoid write buffer pile up too much, + // which could cause controllernode to timeout later when it needs to + // save a snapshot. + pFile->flush(); + +#ifdef PROFILE + if (bExpandExtent) + Stats::stopParseEvent(WE_STATS_EXPAND_COL_EXTENT); + else + Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); +#endif } + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -1233,56 +1100,40 @@ int FileOp::initColumnExtent( * returns ERR_FILE_WRITE or ERR_FILE_SEEK if an error occurs, * else returns NO_ERROR. ***********************************************************/ -int FileOp::initAbbrevCompColumnExtent( - IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - const uint8_t* emptyVal, - int width, - BRM::LBID_t startLBID, - execplan::CalpontSystemCatalog::ColDataType colDataType) +int FileOp::initAbbrevCompColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, + const uint8_t* emptyVal, int width, BRM::LBID_t startLBID, + execplan::CalpontSystemCatalog::ColDataType colDataType) { - // Reserve disk space for optimized abbreviated extent - int rc = initColumnExtent( pFile, - dbRoot, - nBlocks, - emptyVal, - width, - colDataType, - true, // new file - false, // don't expand; add new extent - true, // add abbreviated extent - true, // optimize the initial extent - startLBID); - if (rc != NO_ERROR) - { - return rc; - } + // Reserve disk space for optimized abbreviated extent + int rc = initColumnExtent(pFile, dbRoot, nBlocks, emptyVal, width, colDataType, + true, // new file + false, // don't expand; add new extent + true, // add abbreviated extent + true, // optimize the initial extent + startLBID); + if (rc != NO_ERROR) + { + return rc; + } #ifdef PROFILE - Stats::startParseEvent(WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT); + Stats::startParseEvent(WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT); #endif - char hdrs[CompressInterface::HDR_BUF_LEN * 2]; - rc = writeInitialCompColumnChunk( pFile, - nBlocks, - INITIAL_EXTENT_ROWS_TO_DISK, - emptyVal, - width, - startLBID, - colDataType, - hdrs ); + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + rc = writeInitialCompColumnChunk(pFile, nBlocks, INITIAL_EXTENT_ROWS_TO_DISK, emptyVal, width, startLBID, + colDataType, hdrs); - if (rc != NO_ERROR) - { - return rc; - } + if (rc != NO_ERROR) + { + return rc; + } #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT); + Stats::stopParseEvent(WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT); #endif - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -1299,79 +1150,67 @@ int FileOp::initAbbrevCompColumnExtent( * RETURN: * returns NO_ERROR on success. ***********************************************************/ -int FileOp::writeInitialCompColumnChunk( - IDBDataFile* pFile, - int nBlocksAllocated, - int nRows, - const uint8_t* emptyVal, - int width, - BRM::LBID_t startLBID, - execplan::CalpontSystemCatalog::ColDataType colDataType, - char* hdrs) +int FileOp::writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows, + const uint8_t* emptyVal, int width, BRM::LBID_t startLBID, + execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs) { - const size_t INPUT_BUFFER_SIZE = nRows * width; - char* toBeCompressedInput = new char[INPUT_BUFFER_SIZE]; - unsigned int userPaddingBytes = Config::getNumCompressedPadBlks() * - BYTE_PER_BLOCK; - // Compress an initialized abbreviated extent - // Initially m_compressionType == 0, but this function is used under - // condtion where m_compressionType > 0. - std::unique_ptr compressor( - compress::getCompressInterfaceByType(m_compressionType, - userPaddingBytes)); - const size_t OUTPUT_BUFFER_SIZE = - compressor->maxCompressedSize(INPUT_BUFFER_SIZE) + userPaddingBytes + - compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; + const size_t INPUT_BUFFER_SIZE = nRows * width; + char* toBeCompressedInput = new char[INPUT_BUFFER_SIZE]; + unsigned int userPaddingBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; + // Compress an initialized abbreviated extent + // Initially m_compressionType == 0, but this function is used under + // condtion where m_compressionType > 0. + std::unique_ptr compressor( + compress::getCompressInterfaceByType(m_compressionType, userPaddingBytes)); + const size_t OUTPUT_BUFFER_SIZE = compressor->maxCompressedSize(INPUT_BUFFER_SIZE) + userPaddingBytes + + compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; - unsigned char* compressedOutput = new unsigned char[OUTPUT_BUFFER_SIZE]; - size_t outputLen = OUTPUT_BUFFER_SIZE; - boost::scoped_array toBeCompressedInputPtr( toBeCompressedInput ); - boost::scoped_array compressedOutputPtr(compressedOutput); + unsigned char* compressedOutput = new unsigned char[OUTPUT_BUFFER_SIZE]; + size_t outputLen = OUTPUT_BUFFER_SIZE; + boost::scoped_array toBeCompressedInputPtr(toBeCompressedInput); + boost::scoped_array compressedOutputPtr(compressedOutput); - setEmptyBuf( (unsigned char*)toBeCompressedInput, - INPUT_BUFFER_SIZE, emptyVal, width); + setEmptyBuf((unsigned char*)toBeCompressedInput, INPUT_BUFFER_SIZE, emptyVal, width); - int rc = compressor->compressBlock(toBeCompressedInput, INPUT_BUFFER_SIZE, - compressedOutput, outputLen); + int rc = compressor->compressBlock(toBeCompressedInput, INPUT_BUFFER_SIZE, compressedOutput, outputLen); - if (rc != 0) - { - return ERR_COMP_COMPRESS; - } + if (rc != 0) + { + return ERR_COMP_COMPRESS; + } - // Round up the compressed chunk size - rc = compressor->padCompressedChunks(compressedOutput, outputLen, - OUTPUT_BUFFER_SIZE); + // Round up the compressed chunk size + rc = compressor->padCompressedChunks(compressedOutput, outputLen, OUTPUT_BUFFER_SIZE); - if (rc != 0) - { - return ERR_COMP_PAD_DATA; - } + if (rc != 0) + { + return ERR_COMP_PAD_DATA; + } -// std::cout << "Uncompressed rowCount: " << nRows << -// "; colWidth: " << width << -// "; uncompByteCnt: " << INPUT_BUFFER_SIZE << -// "; blkAllocCnt: " << nBlocksAllocated << -// "; compressedByteCnt: " << outputLen << std::endl; + // std::cout << "Uncompressed rowCount: " << nRows << + // "; colWidth: " << width << + // "; uncompByteCnt: " << INPUT_BUFFER_SIZE << + // "; blkAllocCnt: " << nBlocksAllocated << + // "; compressedByteCnt: " << outputLen << std::endl; - compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); - compress::CompressInterface::setBlockCount(hdrs, nBlocksAllocated); - compress::CompressInterface::setLBIDByIndex(hdrs, startLBID, 0); + compress::CompressInterface::initHdr(hdrs, width, colDataType, m_compressionType); + compress::CompressInterface::setBlockCount(hdrs, nBlocksAllocated); + compress::CompressInterface::setLBIDByIndex(hdrs, startLBID, 0); - // Store compression pointers in the header - std::vector ptrs; - ptrs.push_back( CompressInterface::HDR_BUF_LEN * 2 ); - ptrs.push_back( outputLen + (CompressInterface::HDR_BUF_LEN * 2) ); - compress::CompressInterface::storePtrs(ptrs, hdrs); + // Store compression pointers in the header + std::vector ptrs; + ptrs.push_back(CompressInterface::HDR_BUF_LEN * 2); + ptrs.push_back(outputLen + (CompressInterface::HDR_BUF_LEN * 2)); + compress::CompressInterface::storePtrs(ptrs, hdrs); - RETURN_ON_ERROR( writeHeaders(pFile, hdrs) ); + RETURN_ON_ERROR(writeHeaders(pFile, hdrs)); - // Write the compressed data - size_t writtenLen = pFile->write(compressedOutput, outputLen); - if (writtenLen != outputLen) - return ERR_FILE_WRITE; + // Write the compressed data + size_t writtenLen = pFile->write(compressedOutput, outputLen); + if (writtenLen != outputLen) + return ERR_FILE_WRITE; - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -1391,152 +1230,130 @@ int FileOp::writeInitialCompColumnChunk( * RETURN: * returns NO_ERROR if success. ***********************************************************/ -int FileOp::fillCompColumnExtentEmptyChunks(OID oid, - int colWidth, - const uint8_t* emptyVal, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - HWM hwm, - std::string& segFile, - std::string& failedTask) +int FileOp::fillCompColumnExtentEmptyChunks(OID oid, int colWidth, const uint8_t* emptyVal, uint16_t dbRoot, + uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm, + std::string& segFile, std::string& failedTask) { - int rc = NO_ERROR; - segFile.clear(); - failedTask.clear(); + int rc = NO_ERROR; + segFile.clear(); + failedTask.clear(); - // Open the file and read the headers with the compression chunk pointers - // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag - IDBDataFile* pFile = openFile( oid, dbRoot, partition, segment, segFile, - "r+b", DEFAULT_COLSIZ, true ); + // Open the file and read the headers with the compression chunk pointers + // @bug 5572 - HDFS usage: incorporate *.tmp file backup flag + IDBDataFile* pFile = openFile(oid, dbRoot, partition, segment, segFile, "r+b", DEFAULT_COLSIZ, true); - if (!pFile) + if (!pFile) + { + failedTask = "Opening file"; + ostringstream oss; + oss << "oid: " << oid << " with path " << segFile; + logging::Message::Args args; + args.add("Error opening file "); + args.add(oss.str()); + args.add(""); + args.add(""); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0001); + return ERR_FILE_OPEN; + } + + char hdrs[CompressInterface::HDR_BUF_LEN * 2]; + rc = readHeaders(pFile, hdrs); + + if (rc != NO_ERROR) + { + failedTask = "Reading headers"; + closeFile(pFile); + return rc; + } + + int userPadBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; + + std::unique_ptr compressor(compress::getCompressInterfaceByType( + compress::CompressInterface::getCompressionType(hdrs), userPadBytes)); + + CompChunkPtrList chunkPtrs; + int rcComp = compress::CompressInterface::getPtrList(hdrs, chunkPtrs); + + if (rcComp != 0) + { + failedTask = "Getting header ptrs"; + closeFile(pFile); + return ERR_COMP_PARSE_HDRS; + } + + // Nothing to do if the proposed HWM is < the current block count + uint64_t blkCount = compress::CompressInterface::getBlockCount(hdrs); + + if (blkCount > (hwm + 1)) + { + closeFile(pFile); + return NO_ERROR; + } + + const unsigned int ROWS_PER_EXTENT = BRMWrapper::getInstance()->getInstance()->getExtentRows(); + const unsigned int ROWS_PER_CHUNK = CompressInterface::UNCOMPRESSED_INBUF_LEN / colWidth; + const unsigned int CHUNKS_PER_EXTENT = ROWS_PER_EXTENT / ROWS_PER_CHUNK; + + // If this is an abbreviated extent, we first expand to a full extent + // @bug 4340 - support moving the DBRoot with a single abbrev extent + if ((chunkPtrs.size() == 1) && + ((blkCount * BYTE_PER_BLOCK) == (uint64_t)(INITIAL_EXTENT_ROWS_TO_DISK * colWidth))) + { + if (getLogger()) { - failedTask = "Opening file"; - ostringstream oss; - oss << "oid: " << oid << " with path " << segFile; - logging::Message::Args args; - args.add("Error opening file "); - args.add(oss.str()); - args.add(""); - args.add(""); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0001); - return ERR_FILE_OPEN; + std::ostringstream oss; + oss << "Converting abbreviated partial extent to full extent for" + << ": OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << "; seg-" << segment + << "; file-" << segFile << "; wid-" << colWidth << "; oldBlkCnt-" << blkCount << "; newBlkCnt-" + << ((ROWS_PER_EXTENT * colWidth) / BYTE_PER_BLOCK); + getLogger()->logMsg(oss.str(), MSGLVL_INFO2); } - char hdrs[ CompressInterface::HDR_BUF_LEN * 2 ]; - rc = readHeaders( pFile, hdrs ); + off64_t endHdrsOffset = pFile->tell(); + rc = expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, colWidth, colDataType); if (rc != NO_ERROR) { - failedTask = "Reading headers"; - closeFile ( pFile ); - return rc; + failedTask = "Expanding abbreviated extent"; + closeFile(pFile); + return rc; } - int userPadBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; + CompChunkPtr chunkOutPtr; + rc = expandAbbrevColumnChunk(pFile, emptyVal, colWidth, chunkPtrs[0], chunkOutPtr, hdrs); - std::unique_ptr compressor( - compress::getCompressInterfaceByType( - compress::CompressInterface::getCompressionType(hdrs), - userPadBytes)); - - CompChunkPtrList chunkPtrs; - int rcComp = compress::CompressInterface::getPtrList(hdrs, chunkPtrs); - - if (rcComp != 0) + if (rc != NO_ERROR) { - failedTask = "Getting header ptrs"; - closeFile ( pFile ); - return ERR_COMP_PARSE_HDRS; + failedTask = "Expanding abbreviated chunk"; + closeFile(pFile); + return rc; } - // Nothing to do if the proposed HWM is < the current block count - uint64_t blkCount = compress::CompressInterface::getBlockCount(hdrs); + chunkPtrs[0] = chunkOutPtr; // update chunkPtrs with new chunk size - if (blkCount > (hwm + 1)) + rc = setFileOffset(pFile, endHdrsOffset); + + if (rc != NO_ERROR) { - closeFile ( pFile ); - return NO_ERROR; + failedTask = "Positioning file to end of headers"; + closeFile(pFile); + return rc; } - const unsigned int ROWS_PER_EXTENT = - BRMWrapper::getInstance()->getInstance()->getExtentRows(); - const unsigned int ROWS_PER_CHUNK = - CompressInterface::UNCOMPRESSED_INBUF_LEN / colWidth; - const unsigned int CHUNKS_PER_EXTENT = ROWS_PER_EXTENT / ROWS_PER_CHUNK; + // Update block count to reflect a full extent + blkCount = (ROWS_PER_EXTENT * colWidth) / BYTE_PER_BLOCK; + compress::CompressInterface::setBlockCount(hdrs, blkCount); + } - // If this is an abbreviated extent, we first expand to a full extent - // @bug 4340 - support moving the DBRoot with a single abbrev extent - if ( (chunkPtrs.size() == 1) && - ((blkCount * BYTE_PER_BLOCK) == - (uint64_t)(INITIAL_EXTENT_ROWS_TO_DISK * colWidth)) ) - { - if ( getLogger() ) - { - std::ostringstream oss; - oss << "Converting abbreviated partial extent to full extent for" << - ": OID-" << oid << - "; DBRoot-" << dbRoot << - "; part-" << partition << - "; seg-" << segment << - "; file-" << segFile << - "; wid-" << colWidth << - "; oldBlkCnt-" << blkCount << - "; newBlkCnt-" << - ((ROWS_PER_EXTENT * colWidth) / BYTE_PER_BLOCK); - getLogger()->logMsg( oss.str(), MSGLVL_INFO2 ); - } + // Calculate the number of empty chunks we need to add to fill this extent + unsigned numChunksToFill = 0; + ldiv_t ldivResult = ldiv(chunkPtrs.size(), CHUNKS_PER_EXTENT); - off64_t endHdrsOffset = pFile->tell(); - rc = expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, colWidth, - colDataType); - - if (rc != NO_ERROR) - { - failedTask = "Expanding abbreviated extent"; - closeFile ( pFile ); - return rc; - } - - CompChunkPtr chunkOutPtr; - rc = expandAbbrevColumnChunk( pFile, emptyVal, colWidth, - chunkPtrs[0], chunkOutPtr, hdrs ); - - if (rc != NO_ERROR) - { - failedTask = "Expanding abbreviated chunk"; - closeFile ( pFile ); - return rc; - } - - chunkPtrs[0] = chunkOutPtr; // update chunkPtrs with new chunk size - - rc = setFileOffset( pFile, endHdrsOffset ); - - if (rc != NO_ERROR) - { - failedTask = "Positioning file to end of headers"; - closeFile ( pFile ); - return rc; - } - - // Update block count to reflect a full extent - blkCount = (ROWS_PER_EXTENT * colWidth) / BYTE_PER_BLOCK; - compress::CompressInterface::setBlockCount(hdrs, blkCount); - } - - // Calculate the number of empty chunks we need to add to fill this extent - unsigned numChunksToFill = 0; - ldiv_t ldivResult = ldiv(chunkPtrs.size(), CHUNKS_PER_EXTENT); - - if (ldivResult.rem != 0) - { - numChunksToFill = CHUNKS_PER_EXTENT - ldivResult.rem; - } + if (ldivResult.rem != 0) + { + numChunksToFill = CHUNKS_PER_EXTENT - ldivResult.rem; + } #if 0 std::cout << "Number of allocated blocks: " << @@ -1557,139 +1374,129 @@ int FileOp::fillCompColumnExtentEmptyChunks(OID oid, std::endl << std::endl; #endif - off64_t endOffset = 0; + off64_t endOffset = 0; - // Fill in or add necessary remaining empty chunks - if (numChunksToFill > 0) + // Fill in or add necessary remaining empty chunks + if (numChunksToFill > 0) + { + const int IN_BUF_LEN = CompressInterface::UNCOMPRESSED_INBUF_LEN; + const int OUT_BUF_LEN = compressor->maxCompressedSize(IN_BUF_LEN) + userPadBytes + + compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; + + // Allocate buffer, and store in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of buffers { - const int IN_BUF_LEN = CompressInterface::UNCOMPRESSED_INBUF_LEN; - const int OUT_BUF_LEN = - compressor->maxCompressedSize(IN_BUF_LEN) + userPadBytes + - compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; + char* toBeCompressedBuf = new char[IN_BUF_LEN]; + unsigned char* compressedBuf = new unsigned char[OUT_BUF_LEN]; + boost::scoped_array toBeCompressedInputPtr(toBeCompressedBuf); + boost::scoped_array compressedOutputPtr(compressedBuf); - // Allocate buffer, and store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of buffers - { - char* toBeCompressedBuf = new char [ IN_BUF_LEN ]; - unsigned char* compressedBuf = new unsigned char[ OUT_BUF_LEN ]; - boost::scoped_array toBeCompressedInputPtr(toBeCompressedBuf); - boost::scoped_array - compressedOutputPtr(compressedBuf); + // Compress and then pad the compressed chunk + setEmptyBuf((unsigned char*)toBeCompressedBuf, IN_BUF_LEN, emptyVal, colWidth); + size_t outputLen = OUT_BUF_LEN; + rcComp = compressor->compressBlock(toBeCompressedBuf, IN_BUF_LEN, compressedBuf, outputLen); - // Compress and then pad the compressed chunk - setEmptyBuf( (unsigned char*)toBeCompressedBuf, - IN_BUF_LEN, emptyVal, colWidth ); - size_t outputLen = OUT_BUF_LEN; - rcComp = compressor->compressBlock(toBeCompressedBuf, IN_BUF_LEN, - compressedBuf, outputLen); + if (rcComp != 0) + { + failedTask = "Compressing chunk"; + closeFile(pFile); + return ERR_COMP_COMPRESS; + } - if (rcComp != 0) - { - failedTask = "Compressing chunk"; - closeFile ( pFile ); - return ERR_COMP_COMPRESS; - } + toBeCompressedInputPtr.reset(); // release memory - toBeCompressedInputPtr.reset(); // release memory + rcComp = compressor->padCompressedChunks(compressedBuf, outputLen, OUT_BUF_LEN); - rcComp = compressor->padCompressedChunks(compressedBuf, outputLen, - OUT_BUF_LEN); + if (rcComp != 0) + { + failedTask = "Padding compressed chunk"; + closeFile(pFile); + return ERR_COMP_PAD_DATA; + } - if (rcComp != 0) - { - failedTask = "Padding compressed chunk"; - closeFile ( pFile ); - return ERR_COMP_PAD_DATA; - } + // Position file to write empty chunks; default to end of headers + // in case there are no chunks listed in the header + off64_t startOffset = pFile->tell(); - // Position file to write empty chunks; default to end of headers - // in case there are no chunks listed in the header - off64_t startOffset = pFile->tell(); - - if (chunkPtrs.size() > 0) - { - startOffset = chunkPtrs[chunkPtrs.size() - 1].first + - chunkPtrs[chunkPtrs.size() - 1].second; - rc = setFileOffset( pFile, startOffset ); - - if (rc != NO_ERROR) - { - failedTask = "Positioning file to begin filling chunks"; - closeFile ( pFile ); - return rc; - } - } - - // Write chunks needed to fill out the current extent, add chunk ptr - for (unsigned k = 0; k < numChunksToFill; k++) - { - rc = writeFile( pFile, - (unsigned char*)compressedBuf, - outputLen ); - - if (rc != NO_ERROR) - { - failedTask = "Writing a chunk"; - closeFile ( pFile ); - return rc; - } - - CompChunkPtr compChunk( startOffset, outputLen ); - chunkPtrs.push_back( compChunk ); - startOffset = pFile->tell(); - } - } // end of scope for boost scoped array pointers - - endOffset = pFile->tell(); - - // Update the compressed chunk pointers in the header - std::vector ptrs; - - for (unsigned i = 0; i < chunkPtrs.size(); i++) - { - ptrs.push_back( chunkPtrs[i].first ); - } - - ptrs.push_back( chunkPtrs[chunkPtrs.size() - 1].first + - chunkPtrs[chunkPtrs.size() - 1].second ); - compress::CompressInterface::storePtrs(ptrs, hdrs); - - rc = writeHeaders( pFile, hdrs ); + if (chunkPtrs.size() > 0) + { + startOffset = chunkPtrs[chunkPtrs.size() - 1].first + chunkPtrs[chunkPtrs.size() - 1].second; + rc = setFileOffset(pFile, startOffset); if (rc != NO_ERROR) { - failedTask = "Writing headers"; - closeFile ( pFile ); - return rc; + failedTask = "Positioning file to begin filling chunks"; + closeFile(pFile); + return rc; } - } // end of "numChunksToFill > 0" - else - { - // if no chunks to add, then set endOffset to truncate the db file - // strictly based on the chunks that are already in the file - if (chunkPtrs.size() > 0) - { - endOffset = chunkPtrs[chunkPtrs.size() - 1].first + - chunkPtrs[chunkPtrs.size() - 1].second; - } - } + } - // Truncate the file to release unused space for the extent we just filled - if (endOffset > 0) - { - rc = truncateFile(pFile, endOffset); + // Write chunks needed to fill out the current extent, add chunk ptr + for (unsigned k = 0; k < numChunksToFill; k++) + { + rc = writeFile(pFile, (unsigned char*)compressedBuf, outputLen); if (rc != NO_ERROR) { - failedTask = "Truncating file"; - closeFile ( pFile ); - return rc; + failedTask = "Writing a chunk"; + closeFile(pFile); + return rc; } + + CompChunkPtr compChunk(startOffset, outputLen); + chunkPtrs.push_back(compChunk); + startOffset = pFile->tell(); + } + } // end of scope for boost scoped array pointers + + endOffset = pFile->tell(); + + // Update the compressed chunk pointers in the header + std::vector ptrs; + + for (unsigned i = 0; i < chunkPtrs.size(); i++) + { + ptrs.push_back(chunkPtrs[i].first); } - closeFile ( pFile ); + ptrs.push_back(chunkPtrs[chunkPtrs.size() - 1].first + chunkPtrs[chunkPtrs.size() - 1].second); + compress::CompressInterface::storePtrs(ptrs, hdrs); - return NO_ERROR; + rc = writeHeaders(pFile, hdrs); + + if (rc != NO_ERROR) + { + failedTask = "Writing headers"; + closeFile(pFile); + return rc; + } + } // end of "numChunksToFill > 0" + else + { + // if no chunks to add, then set endOffset to truncate the db file + // strictly based on the chunks that are already in the file + if (chunkPtrs.size() > 0) + { + endOffset = chunkPtrs[chunkPtrs.size() - 1].first + chunkPtrs[chunkPtrs.size() - 1].second; + } + } + + // Truncate the file to release unused space for the extent we just filled + if (endOffset > 0) + { + rc = truncateFile(pFile, endOffset); + + if (rc != NO_ERROR) + { + failedTask = "Truncating file"; + closeFile(pFile); + return rc; + } + } + + closeFile(pFile); + + return NO_ERROR; } /*********************************************************** @@ -1705,86 +1512,75 @@ int FileOp::fillCompColumnExtentEmptyChunks(OID oid, * RETURN: * returns NO_ERROR if success. ***********************************************************/ -int FileOp::expandAbbrevColumnChunk( - IDBDataFile* pFile, - const uint8_t* emptyVal, - int colWidth, - const CompChunkPtr& chunkInPtr, - CompChunkPtr& chunkOutPtr, - const char *hdrs ) +int FileOp::expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal, int colWidth, + const CompChunkPtr& chunkInPtr, CompChunkPtr& chunkOutPtr, + const char* hdrs) { - int userPadBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; - auto realCompressionType = m_compressionType; - if (hdrs) - { - realCompressionType = - compress::CompressInterface::getCompressionType(hdrs); - } - std::unique_ptr compressor( - compress::getCompressInterfaceByType(realCompressionType, - userPadBytes)); + int userPadBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK; + auto realCompressionType = m_compressionType; + if (hdrs) + { + realCompressionType = compress::CompressInterface::getCompressionType(hdrs); + } + std::unique_ptr compressor( + compress::getCompressInterfaceByType(realCompressionType, userPadBytes)); - const int IN_BUF_LEN = CompressInterface::UNCOMPRESSED_INBUF_LEN; - const int OUT_BUF_LEN = - compressor->maxCompressedSize(IN_BUF_LEN) + userPadBytes + - compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; + const int IN_BUF_LEN = CompressInterface::UNCOMPRESSED_INBUF_LEN; + const int OUT_BUF_LEN = compressor->maxCompressedSize(IN_BUF_LEN) + userPadBytes + + compress::CompressInterface::COMPRESSED_CHUNK_INCREMENT_SIZE; - char* toBeCompressedBuf = new char[ IN_BUF_LEN ]; - boost::scoped_array toBeCompressedPtr(toBeCompressedBuf); + char* toBeCompressedBuf = new char[IN_BUF_LEN]; + boost::scoped_array toBeCompressedPtr(toBeCompressedBuf); - setEmptyBuf( (unsigned char*)toBeCompressedBuf, - IN_BUF_LEN, emptyVal, colWidth ); + setEmptyBuf((unsigned char*)toBeCompressedBuf, IN_BUF_LEN, emptyVal, colWidth); - RETURN_ON_ERROR( setFileOffset(pFile, chunkInPtr.first, SEEK_SET) ); + RETURN_ON_ERROR(setFileOffset(pFile, chunkInPtr.first, SEEK_SET)); - char* compressedInBuf = new char[ chunkInPtr.second ]; - boost::scoped_array compressedInBufPtr(compressedInBuf); - RETURN_ON_ERROR( readFile(pFile, (unsigned char*)compressedInBuf, - chunkInPtr.second) ); + char* compressedInBuf = new char[chunkInPtr.second]; + boost::scoped_array compressedInBufPtr(compressedInBuf); + RETURN_ON_ERROR(readFile(pFile, (unsigned char*)compressedInBuf, chunkInPtr.second)); - // Uncompress an "abbreviated" chunk into our 4MB buffer - size_t outputLen = IN_BUF_LEN; - int rc = compressor->uncompressBlock(compressedInBuf, chunkInPtr.second, - (unsigned char*) toBeCompressedBuf, - outputLen); + // Uncompress an "abbreviated" chunk into our 4MB buffer + size_t outputLen = IN_BUF_LEN; + int rc = compressor->uncompressBlock(compressedInBuf, chunkInPtr.second, (unsigned char*)toBeCompressedBuf, + outputLen); - if (rc != 0) - { - return ERR_COMP_UNCOMPRESS; - } + if (rc != 0) + { + return ERR_COMP_UNCOMPRESS; + } - compressedInBufPtr.reset(); // release memory + compressedInBufPtr.reset(); // release memory - RETURN_ON_ERROR( setFileOffset(pFile, chunkInPtr.first, SEEK_SET) ); + RETURN_ON_ERROR(setFileOffset(pFile, chunkInPtr.first, SEEK_SET)); - unsigned char* compressedOutBuf = new unsigned char[ OUT_BUF_LEN ]; - boost::scoped_array compressedOutBufPtr(compressedOutBuf); + unsigned char* compressedOutBuf = new unsigned char[OUT_BUF_LEN]; + boost::scoped_array compressedOutBufPtr(compressedOutBuf); - // Compress the data we just read, as a "full" 4MB chunk - outputLen = OUT_BUF_LEN; - rc = compressor->compressBlock(reinterpret_cast(toBeCompressedBuf), - IN_BUF_LEN, compressedOutBuf, outputLen); + // Compress the data we just read, as a "full" 4MB chunk + outputLen = OUT_BUF_LEN; + rc = compressor->compressBlock(reinterpret_cast(toBeCompressedBuf), IN_BUF_LEN, compressedOutBuf, + outputLen); - if (rc != 0) - { - return ERR_COMP_COMPRESS; - } + if (rc != 0) + { + return ERR_COMP_COMPRESS; + } - // Round up the compressed chunk size - rc = compressor->padCompressedChunks(compressedOutBuf, outputLen, - OUT_BUF_LEN); + // Round up the compressed chunk size + rc = compressor->padCompressedChunks(compressedOutBuf, outputLen, OUT_BUF_LEN); - if (rc != 0) - { - return ERR_COMP_PAD_DATA; - } + if (rc != 0) + { + return ERR_COMP_PAD_DATA; + } - RETURN_ON_ERROR( writeFile(pFile, compressedOutBuf, outputLen) ); + RETURN_ON_ERROR(writeFile(pFile, compressedOutBuf, outputLen)); - chunkOutPtr.first = chunkInPtr.first; - chunkOutPtr.second = outputLen; + chunkOutPtr.first = chunkInPtr.first; + chunkOutPtr.second = outputLen; - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -1799,15 +1595,15 @@ int FileOp::expandAbbrevColumnChunk( ***********************************************************/ int FileOp::writeHeaders(IDBDataFile* pFile, const char* hdr) const { - RETURN_ON_ERROR( setFileOffset(pFile, 0, SEEK_SET) ); + RETURN_ON_ERROR(setFileOffset(pFile, 0, SEEK_SET)); - // Write the headers - if (pFile->write( hdr, CompressInterface::HDR_BUF_LEN * 2 ) != CompressInterface::HDR_BUF_LEN * 2) - { - return ERR_FILE_WRITE; - } + // Write the headers + if (pFile->write(hdr, CompressInterface::HDR_BUF_LEN * 2) != CompressInterface::HDR_BUF_LEN * 2) + { + return ERR_FILE_WRITE; + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -1822,24 +1618,24 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* hdr) const * returns ERR_FILE_WRITE or ERR_FILE_SEEK if an error occurs, * else returns NO_ERROR. ***********************************************************/ -int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, - const char* pointerHdr, uint64_t ptrHdrSize) const +int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, const char* pointerHdr, + uint64_t ptrHdrSize) const { - RETURN_ON_ERROR( setFileOffset(pFile, 0, SEEK_SET) ); + RETURN_ON_ERROR(setFileOffset(pFile, 0, SEEK_SET)); - // Write the control header - if (pFile->write( controlHdr, CompressInterface::HDR_BUF_LEN ) != CompressInterface::HDR_BUF_LEN) - { - return ERR_FILE_WRITE; - } + // Write the control header + if (pFile->write(controlHdr, CompressInterface::HDR_BUF_LEN) != CompressInterface::HDR_BUF_LEN) + { + return ERR_FILE_WRITE; + } - // Write the pointer header - if (pFile->write( pointerHdr, ptrHdrSize ) != (ssize_t) ptrHdrSize) - { - return ERR_FILE_WRITE; - } + // Write the pointer header + if (pFile->write(pointerHdr, ptrHdrSize) != (ssize_t)ptrHdrSize) + { + return ERR_FILE_WRITE; + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -1868,151 +1664,143 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. ***********************************************************/ -int FileOp::initDctnryExtent( - IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - unsigned char* blockHdrInit, - int blockHdrInitSize, - bool bExpandExtent, - bool bOptExtension, - int64_t lbid) +int FileOp::initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit, + int blockHdrInitSize, bool bExpandExtent, bool bOptExtension, int64_t lbid) { - // @bug5769 Don't initialize extents or truncate db files on HDFS - if (idbdatafile::IDBPolicy::useHdfs()) - { - if (m_compressionType) - updateDctnryExtent(pFile, nBlocks, lbid); + // @bug5769 Don't initialize extents or truncate db files on HDFS + if (idbdatafile::IDBPolicy::useHdfs()) + { + if (m_compressionType) + updateDctnryExtent(pFile, nBlocks, lbid); - // Synchronize to avoid write buffer pile up too much, which could cause - // controllernode to timeout later when it needs to save a snapshot. - pFile->flush(); + // Synchronize to avoid write buffer pile up too much, which could cause + // controllernode to timeout later when it needs to save a snapshot. + pFile->flush(); + } + else + { + // Create vector of mutexes used to serialize extent access per DBRoot + initDbRootExtentMutexes(); + + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. + // CS uses non-compressed dict files for its system catalog so + // CS doesn't optimize non-compressed dict creation. + if (bOptExtension) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpaceDisabled(dbRoot) && m_compressionType) + ? bOptExtension + : false; } + // Reduce number of blocks allocated for abbreviated extents thus + // CS writes less when creates a new table. This couldn't be zero + // b/c Snappy compressed file format doesn't tolerate empty files. + int realNBlocks = (bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK) ? 1 : nBlocks; + + // Determine the number of blocks in each call to fwrite(), and the + // number of fwrite() calls to make, based on this. In other words, + // we put a cap on the "writeSize" so that we don't allocate and write + // an entire extent at once for the 64M row extents. If we are + // expanding an abbreviated 64M extent, we may not have an even + // multiple of MAX_NBLOCKS to write; remWriteSize is the number of + // blocks above and beyond loopCount*MAX_NBLOCKS. + int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int loopCount = 1; + int remWriteSize = 0; + + if (realNBlocks > MAX_NBLOCKS) // 64M row extent size + { + writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; + loopCount = realNBlocks / MAX_NBLOCKS; + remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); + } + + // Allocate a buffer, initialize it, and use it to create the extent + idbassert(dbRoot > 0); + +#ifdef PROFILE + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); else + Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); +#endif + + boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); + +#ifdef PROFILE + if (bExpandExtent) + Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); + else + Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); +#endif + // Skip space preallocation if configured so + // fallback to sequential write otherwise. + // Couldn't avoid preallocation for full extents, + // e.g. ADD COLUMN DDL b/c CS has to fill the file + // with empty magics. + if (!bOptExtension) { - // Create vector of mutexes used to serialize extent access per DBRoot - initDbRootExtentMutexes( ); + // Allocate buffer, and store in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_INIT_DCT_EXTENT); +#endif - // MCOL-498 Skip the huge preallocations if the option is set - // for the dbroot. This check is skiped for abbreviated extent. - // IMO it is better to check bool then to call a function. - // CS uses non-compressed dict files for its system catalog so - // CS doesn't optimize non-compressed dict creation. - if ( bOptExtension ) + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr(writeBuf); + + memset(writeBuf, 0, writeSize); + + for (int i = 0; i < realNBlocks; i++) { - bOptExtension = (idbdatafile::IDBPolicy::PreallocSpaceDisabled(dbRoot) - && m_compressionType) ? bOptExtension : false; - } - // Reduce number of blocks allocated for abbreviated extents thus - // CS writes less when creates a new table. This couldn't be zero - // b/c Snappy compressed file format doesn't tolerate empty files. - int realNBlocks = ( bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK ) ? 1 : nBlocks; - - // Determine the number of blocks in each call to fwrite(), and the - // number of fwrite() calls to make, based on this. In other words, - // we put a cap on the "writeSize" so that we don't allocate and write - // an entire extent at once for the 64M row extents. If we are - // expanding an abbreviated 64M extent, we may not have an even - // multiple of MAX_NBLOCKS to write; remWriteSize is the number of - // blocks above and beyond loopCount*MAX_NBLOCKS. - int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size - int loopCount = 1; - int remWriteSize = 0; - - if (realNBlocks > MAX_NBLOCKS) // 64M row extent size - { - writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = realNBlocks / MAX_NBLOCKS; - remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); + memcpy(writeBuf + (i * BYTE_PER_BLOCK), blockHdrInit, blockHdrInitSize); } - // Allocate a buffer, initialize it, and use it to create the extent - idbassert(dbRoot > 0); +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_INIT_DCT_EXTENT); + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_EXPAND_DCT_EXTENT); + else + Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); +#endif + + if (remWriteSize > 0) + { + if (pFile->write(writeBuf, remWriteSize) != remWriteSize) + { + return ERR_FILE_WRITE; + } + } + + for (int j = 0; j < loopCount; j++) + { + if (pFile->write(writeBuf, writeSize) != writeSize) + { + return ERR_FILE_WRITE; + } + } + // CS doesn't account flush timings. #ifdef PROFILE if (bExpandExtent) - Stats::startParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); + Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); else - Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); + Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); #endif + } + } // preallocation fallback end - boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); - -#ifdef PROFILE - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); - else - Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); -#endif - // Skip space preallocation if configured so - // fallback to sequential write otherwise. - // Couldn't avoid preallocation for full extents, - // e.g. ADD COLUMN DDL b/c CS has to fill the file - // with empty magics. - if ( !bOptExtension ) - { - // Allocate buffer, and store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. - { -#ifdef PROFILE - Stats::startParseEvent(WE_STATS_INIT_DCT_EXTENT); -#endif - - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); - - memset(writeBuf, 0, writeSize); - - for (int i = 0; i < realNBlocks; i++) - { - memcpy( writeBuf + (i * BYTE_PER_BLOCK), - blockHdrInit, - blockHdrInitSize ); - } - -#ifdef PROFILE - Stats::stopParseEvent(WE_STATS_INIT_DCT_EXTENT); - - if (bExpandExtent) - Stats::startParseEvent(WE_STATS_EXPAND_DCT_EXTENT); - else - Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); -#endif - - if (remWriteSize > 0) - { - if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) - { - return ERR_FILE_WRITE; - } - } - - for (int j = 0; j < loopCount; j++) - { - if (pFile->write( writeBuf, writeSize ) != writeSize) - { - return ERR_FILE_WRITE; - } - } - // CS doesn't account flush timings. -#ifdef PROFILE - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); -#endif - } - } // preallocation fallback end - - // MCOL-498 CS has to set a number of blocs in the chunk header - if ( m_compressionType ) - { - updateDctnryExtent(pFile, nBlocks, lbid); - } - pFile->flush(); + // MCOL-498 CS has to set a number of blocs in the chunk header + if (m_compressionType) + { + updateDctnryExtent(pFile, nBlocks, lbid); } + pFile->flush(); + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2022,21 +1810,21 @@ int FileOp::initDctnryExtent( * helps to prevent disk fragmentation. ***********************************************************/ /* static */ -void FileOp::initDbRootExtentMutexes( ) +void FileOp::initDbRootExtentMutexes() { - boost::mutex::scoped_lock lk(m_createDbRootMutexes); + boost::mutex::scoped_lock lk(m_createDbRootMutexes); - if ( m_DbRootAddExtentMutexes.size() == 0 ) + if (m_DbRootAddExtentMutexes.size() == 0) + { + std::vector rootIds; + Config::getRootIdList(rootIds); + + for (size_t i = 0; i < rootIds.size(); i++) { - std::vector rootIds; - Config::getRootIdList( rootIds ); - - for (size_t i = 0; i < rootIds.size(); i++) - { - boost::mutex* pM = new boost::mutex; - m_DbRootAddExtentMutexes[ rootIds[i] ] = pM; - } + boost::mutex* pM = new boost::mutex; + m_DbRootAddExtentMutexes[rootIds[i]] = pM; } + } } /*********************************************************** @@ -2047,17 +1835,17 @@ void FileOp::initDbRootExtentMutexes( ) * free up memory at the end of program execution. ***********************************************************/ /* static */ -void FileOp::removeDbRootExtentMutexes( ) +void FileOp::removeDbRootExtentMutexes() { - boost::mutex::scoped_lock lk(m_createDbRootMutexes); + boost::mutex::scoped_lock lk(m_createDbRootMutexes); - std::map::iterator k = m_DbRootAddExtentMutexes.begin(); + std::map::iterator k = m_DbRootAddExtentMutexes.begin(); - while (k != m_DbRootAddExtentMutexes.end() ) - { - delete k->second; - ++k; - } + while (k != m_DbRootAddExtentMutexes.end()) + { + delete k->second; + ++k; + } } /*********************************************************** @@ -2076,66 +1864,62 @@ void FileOp::removeDbRootExtentMutexes( ) * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. ***********************************************************/ -int FileOp::reInitPartialColumnExtent( - IDBDataFile* pFile, - long long startOffset, - int nBlocks, - const uint8_t* emptyVal, - int width ) +int FileOp::reInitPartialColumnExtent(IDBDataFile* pFile, long long startOffset, int nBlocks, + const uint8_t* emptyVal, int width) { - int rc = setFileOffset( pFile, startOffset, SEEK_SET ); + int rc = setFileOffset(pFile, startOffset, SEEK_SET); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - if (nBlocks == 0) - return NO_ERROR; - - // Determine the number of blocks in each call to fwrite(), and the - // number of fwrite() calls to make, based on this. In other words, - // we put a cap on the "writeSize" so that we don't allocate and write - // an entire extent at once for the 64M row extents. - int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size - int loopCount = 0; - int remainderSize = writeSize; - - if (nBlocks > MAX_NBLOCKS) // 64M row extent size - { - writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = nBlocks / MAX_NBLOCKS; - remainderSize = nBlocks - (loopCount * MAX_NBLOCKS); - } - - // Allocate a buffer, initialize it, and use it to initialize the extent - // Store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. - { - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); - - setEmptyBuf( writeBuf, writeSize, emptyVal, width ); - - for (int j = 0; j < loopCount; j++) - { - if (pFile->write( writeBuf, writeSize ) != writeSize) - { - return ERR_FILE_WRITE; - } - } - - if (remainderSize > 0) - { - if (pFile->write( writeBuf, remainderSize ) != remainderSize) - { - return ERR_FILE_WRITE; - } - } - } - - // Synchronize here to avoid write buffer pile up too much, which could - // cause controllernode to timeout later when it needs to save a snapshot. - pFile->flush(); + if (nBlocks == 0) return NO_ERROR; + + // Determine the number of blocks in each call to fwrite(), and the + // number of fwrite() calls to make, based on this. In other words, + // we put a cap on the "writeSize" so that we don't allocate and write + // an entire extent at once for the 64M row extents. + int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int loopCount = 0; + int remainderSize = writeSize; + + if (nBlocks > MAX_NBLOCKS) // 64M row extent size + { + writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; + loopCount = nBlocks / MAX_NBLOCKS; + remainderSize = nBlocks - (loopCount * MAX_NBLOCKS); + } + + // Allocate a buffer, initialize it, and use it to initialize the extent + // Store in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + { + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr(writeBuf); + + setEmptyBuf(writeBuf, writeSize, emptyVal, width); + + for (int j = 0; j < loopCount; j++) + { + if (pFile->write(writeBuf, writeSize) != writeSize) + { + return ERR_FILE_WRITE; + } + } + + if (remainderSize > 0) + { + if (pFile->write(writeBuf, remainderSize) != remainderSize) + { + return ERR_FILE_WRITE; + } + } + } + + // Synchronize here to avoid write buffer pile up too much, which could + // cause controllernode to timeout later when it needs to save a snapshot. + pFile->flush(); + return NO_ERROR; } /*********************************************************** @@ -2154,74 +1938,68 @@ int FileOp::reInitPartialColumnExtent( * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. ***********************************************************/ -int FileOp::reInitPartialDctnryExtent( - IDBDataFile* pFile, - long long startOffset, - int nBlocks, - unsigned char* blockHdrInit, - int blockHdrInitSize ) +int FileOp::reInitPartialDctnryExtent(IDBDataFile* pFile, long long startOffset, int nBlocks, + unsigned char* blockHdrInit, int blockHdrInitSize) { - int rc = setFileOffset( pFile, startOffset, SEEK_SET ); + int rc = setFileOffset(pFile, startOffset, SEEK_SET); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - if (nBlocks == 0) - return NO_ERROR; - - // Determine the number of blocks in each call to fwrite(), and the - // number of fwrite() calls to make, based on this. In other words, - // we put a cap on the "writeSize" so that we don't allocate and write - // an entire extent at once for the 64M row extents. - int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size - int loopCount = 0; - int remainderSize = writeSize; - - if (nBlocks > MAX_NBLOCKS) // 64M row extent size - { - writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = nBlocks / MAX_NBLOCKS; - remainderSize = nBlocks - (loopCount * MAX_NBLOCKS); - nBlocks = MAX_NBLOCKS; - } - - // Allocate a buffer, initialize it, and use it to initialize the extent - // Store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. - { - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); - - memset(writeBuf, 0, writeSize); - - for (int i = 0; i < nBlocks; i++) - { - memcpy( writeBuf + (i * BYTE_PER_BLOCK), - blockHdrInit, - blockHdrInitSize ); - } - - for (int j = 0; j < loopCount; j++) - { - if (pFile->write( writeBuf, writeSize ) != writeSize) - { - return ERR_FILE_WRITE; - } - } - - if (remainderSize > 0) - { - if (pFile->write( writeBuf, remainderSize ) != remainderSize) - { - return ERR_FILE_WRITE; - } - } - } - - // Synchronize here to avoid write buffer pile up too much, which could - // cause controllernode to timeout later when it needs to save a snapshot. - pFile->flush(); + if (nBlocks == 0) return NO_ERROR; + + // Determine the number of blocks in each call to fwrite(), and the + // number of fwrite() calls to make, based on this. In other words, + // we put a cap on the "writeSize" so that we don't allocate and write + // an entire extent at once for the 64M row extents. + int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int loopCount = 0; + int remainderSize = writeSize; + + if (nBlocks > MAX_NBLOCKS) // 64M row extent size + { + writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; + loopCount = nBlocks / MAX_NBLOCKS; + remainderSize = nBlocks - (loopCount * MAX_NBLOCKS); + nBlocks = MAX_NBLOCKS; + } + + // Allocate a buffer, initialize it, and use it to initialize the extent + // Store in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + { + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr(writeBuf); + + memset(writeBuf, 0, writeSize); + + for (int i = 0; i < nBlocks; i++) + { + memcpy(writeBuf + (i * BYTE_PER_BLOCK), blockHdrInit, blockHdrInitSize); + } + + for (int j = 0; j < loopCount; j++) + { + if (pFile->write(writeBuf, writeSize) != writeSize) + { + return ERR_FILE_WRITE; + } + } + + if (remainderSize > 0) + { + if (pFile->write(writeBuf, remainderSize) != remainderSize) + { + return ERR_FILE_WRITE; + } + } + } + + // Synchronize here to avoid write buffer pile up too much, which could + // cause controllernode to timeout later when it needs to save a snapshot. + pFile->flush(); + return NO_ERROR; } /*********************************************************** @@ -2232,22 +2010,22 @@ int FileOp::reInitPartialDctnryExtent( * RETURN: * error code ***********************************************************/ -int FileOp::getFileSize( IDBDataFile* pFile, long long& fileSize ) const +int FileOp::getFileSize(IDBDataFile* pFile, long long& fileSize) const { + fileSize = 0; + + if (pFile == NULL) + return ERR_FILE_NULL; + + fileSize = pFile->size(); + + if (fileSize < 0) + { fileSize = 0; + return ERR_FILE_STAT; + } - if ( pFile == NULL ) - return ERR_FILE_NULL; - - fileSize = pFile->size(); - - if ( fileSize < 0 ) - { - fileSize = 0; - return ERR_FILE_STAT; - } - - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2262,25 +2040,23 @@ int FileOp::getFileSize( IDBDataFile* pFile, long long& fileSize ) const * RETURN: * NO_ERROR if okay, else an error return code. ***********************************************************/ -int FileOp::getFileSize( FID fid, uint16_t dbRoot, - uint32_t partition, uint16_t segment, - long long& fileSize ) const +int FileOp::getFileSize(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment, + long long& fileSize) const { + fileSize = 0; + + char fileName[FILE_NAME_SIZE]; + RETURN_ON_ERROR(getFileName(fid, fileName, dbRoot, partition, segment)); + + fileSize = IDBPolicy::size(fileName); + + if (fileSize < 0) + { fileSize = 0; + return ERR_FILE_STAT; + } - char fileName[FILE_NAME_SIZE]; - RETURN_ON_ERROR( getFileName(fid, fileName, - dbRoot, partition, segment) ); - - fileSize = IDBPolicy::size( fileName ); - - if ( fileSize < 0 ) - { - fileSize = 0; - return ERR_FILE_STAT; - } - - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2291,9 +2067,9 @@ int FileOp::getFileSize( FID fid, uint16_t dbRoot, * RETURN: * true if it is, false otherwise ***********************************************************/ -bool FileOp::isDir( const char* dirName ) const +bool FileOp::isDir(const char* dirName) const { - return IDBPolicy::isDir( dirName ); + return IDBPolicy::isDir(dirName); } /*********************************************************** @@ -2313,148 +2089,136 @@ bool FileOp::isDir( const char* dirName ) const * RETURN: * NO_ERROR if success, other if fail ***********************************************************/ -int FileOp::oid2FileName( FID fid, - char* fullFileName, - bool bCreateDir, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment) const +int FileOp::oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t dbRoot, uint32_t partition, + uint16_t segment) const { #ifdef SHARED_NOTHING_DEMO_2 - if (fid >= 10000) - { - char root[FILE_NAME_SIZE]; - Config::getSharedNothingRoot(root); - sprintf(fullFileName, "%s/FILE%d", root, fid); - return NO_ERROR; - } + if (fid >= 10000) + { + char root[FILE_NAME_SIZE]; + Config::getSharedNothingRoot(root); + sprintf(fullFileName, "%s/FILE%d", root, fid); + return NO_ERROR; + } #endif // Need this stub to use ColumnOp::writeRow in the unit tests #ifdef WITH_UNIT_TESTS - if (fid == 42) - { - sprintf(fullFileName, "./versionbuffer"); - return NO_ERROR; - } + if (fid == 42) + { + sprintf(fullFileName, "./versionbuffer"); + return NO_ERROR; + } #endif - /* If is a version buffer file, the format is different. */ - if (fid < 1000) + /* If is a version buffer file, the format is different. */ + if (fid < 1000) + { + /* Get the dbroot # + * Get the root of that dbroot + * Add "/versionbuffer.cdf" + */ + BRM::DBRM dbrm; + int _dbroot = dbrm.getDBRootOfVBOID(fid); + + if (_dbroot < 0) + return ERR_INVALID_VBOID; + + snprintf(fullFileName, FILE_NAME_SIZE, "%s/versionbuffer.cdf", Config::getDBRootByNum(_dbroot).c_str()); + return NO_ERROR; + } + + // Get hashed part of the filename. This is the tail-end of the filename path, + // excluding the DBRoot. + char tempFileName[FILE_NAME_SIZE]; + char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; + RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, partition, segment))); + + // see if file exists in specified DBRoot; return if found + if (dbRoot > 0) + { + sprintf(fullFileName, "%s/%s", Config::getDBRootByNum(dbRoot).c_str(), tempFileName); + + // std::cout << "oid2FileName() OID: " << fid << + // " searching for file: " << fullFileName < dbRootPathList; + Config::getDBRootPathList(dbRootPathList); + + for (unsigned i = 0; i < dbRootPathList.size(); i++) { - /* Get the dbroot # - * Get the root of that dbroot - * Add "/versionbuffer.cdf" - */ - BRM::DBRM dbrm; - int _dbroot = dbrm.getDBRootOfVBOID(fid); + sprintf(fullFileName, "%s/%s", dbRootPathList[i].c_str(), tempFileName); - if (_dbroot < 0) - return ERR_INVALID_VBOID; - - snprintf(fullFileName, FILE_NAME_SIZE, - "%s/versionbuffer.cdf", Config::getDBRootByNum(_dbroot).c_str()); + // found it, nothing more to do, return + // if (access(fullFileName, R_OK) == 0) return NO_ERROR; + //@Bug 5397 + if (IDBPolicy::exists(fullFileName)) return NO_ERROR; } -//Get hashed part of the filename. This is the tail-end of the filename path, -// excluding the DBRoot. - char tempFileName[FILE_NAME_SIZE]; - char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; - RETURN_ON_ERROR((Convertor::oid2FileName( - fid, tempFileName, dbDir, partition, segment))); + // file wasn't found, user didn't specify DBRoot so we can't create + return ERR_FILE_NOT_EXIST; + } - // see if file exists in specified DBRoot; return if found - if (dbRoot > 0) + std::stringstream aDirName; + for (size_t i = 0; i < MaxDirLevels; i++) + { + if (i == 0) { - sprintf(fullFileName, "%s/%s", Config::getDBRootByNum(dbRoot).c_str(), - tempFileName); - - //std::cout << "oid2FileName() OID: " << fid << - // " searching for file: " << fullFileName < dbRootPathList; - Config::getDBRootPathList( dbRootPathList ); - - for (unsigned i = 0; i < dbRootPathList.size(); i++) - { - sprintf(fullFileName, "%s/%s", dbRootPathList[i].c_str(), - tempFileName); - - //found it, nothing more to do, return - //if (access(fullFileName, R_OK) == 0) return NO_ERROR; - //@Bug 5397 - if (IDBPolicy::exists( fullFileName )) - return NO_ERROR; - } - - //file wasn't found, user didn't specify DBRoot so we can't create - return ERR_FILE_NOT_EXIST; + aDirName << "/" << dbDir[i]; } + if (!isDir(aDirName.str().c_str())) + RETURN_ON_ERROR(createDir(aDirName.str().c_str())); - std::stringstream aDirName; - for (size_t i = 0; i < MaxDirLevels; i++) { - if (i == 0) - { - aDirName << Config::getDBRootByNum(dbRoot).c_str() - << "/" << dbDir[i]; - } - else - { - aDirName << "/" << dbDir[i]; - } - if (!isDir(aDirName.str().c_str())) - RETURN_ON_ERROR( createDir(aDirName.str().c_str()) ); - - { - std::ostringstream ossChown; - if (chownDataPath(aDirName.str())) - return ERR_FILE_CHOWN; - } + std::ostringstream ossChown; + if (chownDataPath(aDirName.str())) + return ERR_FILE_CHOWN; } + } - return NO_ERROR; + return NO_ERROR; } -void FileOp::getFileNameForPrimProc(FID fid, - char* fullFileName, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment) const -{ - string dbRootPath = Config::getDBRootByNum(dbRoot); - if (dbRootPath.empty()) - { - ostringstream oss; - oss << "(dbroot " << dbRoot << " offline)"; - dbRootPath = oss.str(); - } - - // different filenames for the version buffer files - if (fid < 1000) - snprintf(fullFileName, FILE_NAME_SIZE, "%s/versionbuffer.cdf", dbRootPath.c_str()); - else - snprintf(fullFileName, FILE_NAME_SIZE, "%s/%03u.dir/%03u.dir/%03u.dir/%03u.dir/%03u.dir/FILE%03d.cdf", - dbRootPath.c_str(), fid >> 24, (fid & 0x00ff0000) >> 16, (fid & 0x0000ff00) >> 8, - fid & 0x000000ff, partition, segment); -} +void FileOp::getFileNameForPrimProc(FID fid, char* fullFileName, uint16_t dbRoot, uint32_t partition, + uint16_t segment) const +{ + string dbRootPath = Config::getDBRootByNum(dbRoot); + if (dbRootPath.empty()) + { + ostringstream oss; + oss << "(dbroot " << dbRoot << " offline)"; + dbRootPath = oss.str(); + } + + // different filenames for the version buffer files + if (fid < 1000) + snprintf(fullFileName, FILE_NAME_SIZE, "%s/versionbuffer.cdf", dbRootPath.c_str()); + else + snprintf(fullFileName, FILE_NAME_SIZE, "%s/%03u.dir/%03u.dir/%03u.dir/%03u.dir/%03u.dir/FILE%03d.cdf", + dbRootPath.c_str(), fid >> 24, (fid & 0x00ff0000) >> 16, (fid & 0x0000ff00) >> 8, + fid & 0x000000ff, partition, segment); +} /*********************************************************** * DESCRIPTION: @@ -2467,48 +2231,44 @@ void FileOp::getFileNameForPrimProc(FID fid, * RETURN: * NO_ERROR if OID dir path found, else returns ERR_FILE_NOT_EXIST ***********************************************************/ -int FileOp::oid2DirName( FID fid, char* oidDirName ) const +int FileOp::oid2DirName(FID fid, char* oidDirName) const { - char tempFileName[FILE_NAME_SIZE]; - char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; + char tempFileName[FILE_NAME_SIZE]; + char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; - /* If is a version buffer file, the format is different. */ - if (fid < 1000) - { - /* Get the dbroot # - * Get the root of that dbroot - */ - BRM::DBRM dbrm; - int _dbroot = dbrm.getDBRootOfVBOID(fid); + /* If is a version buffer file, the format is different. */ + if (fid < 1000) + { + /* Get the dbroot # + * Get the root of that dbroot + */ + BRM::DBRM dbrm; + int _dbroot = dbrm.getDBRootOfVBOID(fid); - if (_dbroot < 0) - return ERR_INVALID_VBOID; + if (_dbroot < 0) + return ERR_INVALID_VBOID; - snprintf(oidDirName, FILE_NAME_SIZE, "%s", - Config::getDBRootByNum(_dbroot).c_str()); - return NO_ERROR; - } + snprintf(oidDirName, FILE_NAME_SIZE, "%s", Config::getDBRootByNum(_dbroot).c_str()); + return NO_ERROR; + } - RETURN_ON_ERROR((Convertor::oid2FileName( - fid, tempFileName, dbDir, 0, 0))); + RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, 0, 0))); - //Now try to find the directory in each of the DBRoots. - std::vector dbRootPathList; - Config::getDBRootPathList( dbRootPathList ); + // Now try to find the directory in each of the DBRoots. + std::vector dbRootPathList; + Config::getDBRootPathList(dbRootPathList); - for (unsigned i = 0; i < dbRootPathList.size(); i++) - { - sprintf(oidDirName, "%s/%s/%s/%s/%s", - dbRootPathList[i].c_str(), - dbDir[0], dbDir[1], dbDir[2], dbDir[3]); + for (unsigned i = 0; i < dbRootPathList.size(); i++) + { + sprintf(oidDirName, "%s/%s/%s/%s/%s", dbRootPathList[i].c_str(), dbDir[0], dbDir[1], dbDir[2], dbDir[3]); - //found it, nothing more to do, return - //@Bug 5397. use the new way to check - if (IDBPolicy::exists( oidDirName )) - return NO_ERROR; - } + // found it, nothing more to do, return + //@Bug 5397. use the new way to check + if (IDBPolicy::exists(oidDirName)) + return NO_ERROR; + } - return ERR_FILE_NOT_EXIST; + return ERR_FILE_NOT_EXIST; } /*********************************************************** @@ -2523,27 +2283,20 @@ int FileOp::oid2DirName( FID fid, char* oidDirName ) const * RETURN: * NO_ERROR if path is successfully constructed. ***********************************************************/ -int FileOp::getDirName( FID fid, uint16_t dbRoot, - uint32_t partition, - std::string& dirName) const +int FileOp::getDirName(FID fid, uint16_t dbRoot, uint32_t partition, std::string& dirName) const { - char tempFileName[FILE_NAME_SIZE]; - char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; + char tempFileName[FILE_NAME_SIZE]; + char dbDir[MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE]; - RETURN_ON_ERROR((Convertor::oid2FileName( - fid, tempFileName, dbDir, partition, 0))); + RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, partition, 0))); - std::string rootPath = Config::getDBRootByNum( dbRoot ); - std::ostringstream oss; - oss << rootPath << '/' << - dbDir[0] << '/' << - dbDir[1] << '/' << - dbDir[2] << '/' << - dbDir[3] << '/' << - dbDir[4]; - dirName = oss.str(); + std::string rootPath = Config::getDBRootByNum(dbRoot); + std::ostringstream oss; + oss << rootPath << '/' << dbDir[0] << '/' << dbDir[1] << '/' << dbDir[2] << '/' << dbDir[3] << '/' + << dbDir[4]; + dirName = oss.str(); - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2556,78 +2309,61 @@ int FileOp::getDirName( FID fid, uint16_t dbRoot, * true if exists, false otherwise ***********************************************************/ // @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* FileOp::openFile( const char* fileName, - const char* mode, - const int ioColSize, - bool useTmpSuffix ) const +IDBDataFile* FileOp::openFile(const char* fileName, const char* mode, const int ioColSize, + bool useTmpSuffix) const { - IDBDataFile* pFile; - errno = 0; + IDBDataFile* pFile; + errno = 0; - unsigned opts; + unsigned opts; - if (ioColSize > 0) - opts = IDBDataFile::USE_VBUF; - else - opts = IDBDataFile::USE_NOVBUF; + if (ioColSize > 0) + opts = IDBDataFile::USE_VBUF; + else + opts = IDBDataFile::USE_NOVBUF; - if ((useTmpSuffix) && idbdatafile::IDBPolicy::useHdfs()) - opts |= IDBDataFile::USE_TMPFILE; + if ((useTmpSuffix) && idbdatafile::IDBPolicy::useHdfs()) + opts |= IDBDataFile::USE_TMPFILE; - pFile = IDBDataFile::open( - IDBPolicy::getType( fileName, IDBPolicy::WRITEENG ), - fileName, - mode, - opts, - ioColSize ); + pFile = + IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, mode, opts, ioColSize); - if (pFile == NULL) - { - int errRc = errno; - std::ostringstream oss; - std::string errnoMsg; - Convertor::mapErrnoToString(errRc, errnoMsg); - oss << "FileOp::openFile(): fopen(" << fileName << - ", " << mode << "): errno = " << errRc << - ": " << errnoMsg; - logging::Message::Args args; - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_CRITICAL, - logging::M0006); - SimpleSysLog::instance()->logMsg(args, - logging::LOG_TYPE_ERROR, - logging::M0006); - } + if (pFile == NULL) + { + int errRc = errno; + std::ostringstream oss; + std::string errnoMsg; + Convertor::mapErrnoToString(errRc, errnoMsg); + oss << "FileOp::openFile(): fopen(" << fileName << ", " << mode << "): errno = " << errRc << ": " + << errnoMsg; + logging::Message::Args args; + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_CRITICAL, logging::M0006); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0006); + } - return pFile; + return pFile; } // @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* FileOp::openFile( FID fid, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - const char* mode, int ioColSize, - bool useTmpSuffix ) const +IDBDataFile* FileOp::openFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, const char* mode, int ioColSize, bool useTmpSuffix) const { - char fileName[FILE_NAME_SIZE]; - int rc; + char fileName[FILE_NAME_SIZE]; + int rc; - //fid2FileName( fileName, fid ); - RETURN_ON_WE_ERROR( ( rc = getFileName( fid, fileName, - dbRoot, partition, segment ) ), NULL ); + // fid2FileName( fileName, fid ); + RETURN_ON_WE_ERROR((rc = getFileName(fid, fileName, dbRoot, partition, segment)), NULL); - // disable buffering for versionbuffer file - if (fid < 1000) - ioColSize = 0; + // disable buffering for versionbuffer file + if (fid < 1000) + ioColSize = 0; - IDBDataFile* pF = openFile( fileName, mode, ioColSize, useTmpSuffix ); + IDBDataFile* pF = openFile(fileName, mode, ioColSize, useTmpSuffix); - segFile = fileName; + segFile = fileName; - return pF; + return pF; } /*********************************************************** @@ -2642,60 +2378,56 @@ IDBDataFile* FileOp::openFile( FID fid, * ERR_FILE_NULL if file handle is NULL * ERR_FILE_READ if something wrong in reading the file ***********************************************************/ -int FileOp::readFile( IDBDataFile* pFile, unsigned char* readBuf, - int readSize ) const +int FileOp::readFile(IDBDataFile* pFile, unsigned char* readBuf, int readSize) const { - if ( pFile != NULL ) + if (pFile != NULL) + { + int bc = pFile->read(readBuf, readSize); + if (bc != readSize) { - int bc = pFile->read( readBuf, readSize ); - if (bc != readSize) - { - // MCOL-498 EOF if a next block is empty - if (bc == 0) - { - return ERR_FILE_EOF; - } - return ERR_FILE_READ; - } + // MCOL-498 EOF if a next block is empty + if (bc == 0) + { + return ERR_FILE_EOF; + } + return ERR_FILE_READ; } - else - return ERR_FILE_NULL; + } + else + return ERR_FILE_NULL; - return NO_ERROR; + return NO_ERROR; } /*********************************************************** * Reads contents of headers from "pFile" and stores into "hdrs". ***********************************************************/ -int FileOp::readHeaders( IDBDataFile* pFile, char* hdrs ) const +int FileOp::readHeaders(IDBDataFile* pFile, char* hdrs) const { - RETURN_ON_ERROR( setFileOffset(pFile, 0) ); - RETURN_ON_ERROR( readFile( pFile, reinterpret_cast(hdrs), - (CompressInterface::HDR_BUF_LEN * 2) ) ); - int rc = compress::CompressInterface::verifyHdr(hdrs); + RETURN_ON_ERROR(setFileOffset(pFile, 0)); + RETURN_ON_ERROR( + readFile(pFile, reinterpret_cast(hdrs), (CompressInterface::HDR_BUF_LEN * 2))); + int rc = compress::CompressInterface::verifyHdr(hdrs); - if (rc != 0) - { - return ERR_COMP_VERIFY_HDRS; - } + if (rc != 0) + { + return ERR_COMP_VERIFY_HDRS; + } - return NO_ERROR; + return NO_ERROR; } /*********************************************************** * Reads contents of headers from "pFile" and stores into "hdr1" and "hdr2". ***********************************************************/ -int FileOp::readHeaders( IDBDataFile* pFile, char* hdr1, char* hdr2 ) const +int FileOp::readHeaders(IDBDataFile* pFile, char* hdr1, char* hdr2) const { - unsigned char* hdrPtr = reinterpret_cast(hdr1); - RETURN_ON_ERROR( setFileOffset(pFile, 0) ); - RETURN_ON_ERROR( readFile( pFile, hdrPtr, - CompressInterface::HDR_BUF_LEN )); + unsigned char* hdrPtr = reinterpret_cast(hdr1); + RETURN_ON_ERROR(setFileOffset(pFile, 0)); + RETURN_ON_ERROR(readFile(pFile, hdrPtr, CompressInterface::HDR_BUF_LEN)); - int ptrSecSize = compress::CompressInterface::getHdrSize(hdrPtr) - - CompressInterface::HDR_BUF_LEN; - return readFile( pFile, reinterpret_cast(hdr2), - ptrSecSize ); + int ptrSecSize = compress::CompressInterface::getHdrSize(hdrPtr) - CompressInterface::HDR_BUF_LEN; + return readFile(pFile, reinterpret_cast(hdr2), ptrSecSize); } /*********************************************************** @@ -2710,23 +2442,23 @@ int FileOp::readHeaders( IDBDataFile* pFile, char* hdr1, char* hdr2 ) const * ERR_FILE_NULL if file handle is NULL * ERR_FILE_SEEK if something wrong in setting the position ***********************************************************/ -int FileOp::setFileOffset( IDBDataFile* pFile, long long offset, int origin ) const +int FileOp::setFileOffset(IDBDataFile* pFile, long long offset, int origin) const { - int rc; - long long fboOffset = offset; // workaround solution to pass leakcheck error + int rc; + long long fboOffset = offset; // workaround solution to pass leakcheck error - if ( pFile == NULL ) - return ERR_FILE_NULL; + if (pFile == NULL) + return ERR_FILE_NULL; - if ( offset < 0 ) - return ERR_FILE_FBO_NEG; + if (offset < 0) + return ERR_FILE_FBO_NEG; - rc = pFile->seek( fboOffset, origin ); + rc = pFile->seek(fboOffset, origin); - if (rc) - return ERR_FILE_SEEK; + if (rc) + return ERR_FILE_SEEK; - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2741,20 +2473,19 @@ int FileOp::setFileOffset( IDBDataFile* pFile, long long offset, int origin ) co * ERR_FILE_NULL if file handle is NULL * ERR_FILE_SEEK if something wrong in setting the position ***********************************************************/ -int FileOp::setFileOffsetBlock( IDBDataFile* pFile, uint64_t lbid, int origin) const +int FileOp::setFileOffsetBlock(IDBDataFile* pFile, uint64_t lbid, int origin) const { - long long fboOffset = 0; - int fbo = 0; + long long fboOffset = 0; + int fbo = 0; - // only when fboFlag is false, we get in here - uint16_t dbRoot; - uint32_t partition; - uint16_t segment; - RETURN_ON_ERROR( BRMWrapper::getInstance()->getFboOffset( - lbid, dbRoot, partition, segment, fbo ) ); - fboOffset = ((long long)fbo) * (long)BYTE_PER_BLOCK; + // only when fboFlag is false, we get in here + uint16_t dbRoot; + uint32_t partition; + uint16_t segment; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getFboOffset(lbid, dbRoot, partition, segment, fbo)); + fboOffset = ((long long)fbo) * (long)BYTE_PER_BLOCK; - return setFileOffset( pFile, fboOffset, origin ); + return setFileOffset(pFile, fboOffset, origin); } /*********************************************************** @@ -2768,15 +2499,15 @@ int FileOp::setFileOffsetBlock( IDBDataFile* pFile, uint64_t lbid, int origin) c * ERR_FILE_NULL if file handle is NULL * ERR_FILE_SEEK if something wrong in setting the position ***********************************************************/ -int FileOp::truncateFile( IDBDataFile* pFile, long long fileSize ) const +int FileOp::truncateFile(IDBDataFile* pFile, long long fileSize) const { - if ( pFile == NULL ) - return ERR_FILE_NULL; + if (pFile == NULL) + return ERR_FILE_NULL; - if ( pFile->truncate( fileSize ) != 0 ) - return ERR_FILE_TRUNCATE; + if (pFile->truncate(fileSize) != 0) + return ERR_FILE_TRUNCATE; - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2791,18 +2522,17 @@ int FileOp::truncateFile( IDBDataFile* pFile, long long fileSize ) const * ERR_FILE_NULL if file handle is NULL * ERR_FILE_WRITE if something wrong in writing to the file ***********************************************************/ -int FileOp::writeFile( IDBDataFile* pFile, const unsigned char* writeBuf, - int writeSize ) const +int FileOp::writeFile(IDBDataFile* pFile, const unsigned char* writeBuf, int writeSize) const { - if ( pFile != NULL ) - { - if ( pFile->write( writeBuf, writeSize ) != writeSize ) - return ERR_FILE_WRITE; - } - else - return ERR_FILE_NULL; + if (pFile != NULL) + { + if (pFile->write(writeBuf, writeSize) != writeSize) + return ERR_FILE_WRITE; + } + else + return ERR_FILE_NULL; - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -2820,58 +2550,56 @@ int FileOp::writeFile( IDBDataFile* pFile, const unsigned char* writeBuf, ***********************************************************/ bool FileOp::isDiskSpaceAvail(const std::string& fileName, int nBlocks) const { - bool bSpaceAvail = true; + bool bSpaceAvail = true; - unsigned maxDiskUsage = Config::getMaxFileSystemDiskUsage(); + unsigned maxDiskUsage = Config::getMaxFileSystemDiskUsage(); - if (maxDiskUsage < 100) // 100% means to disable the check - { + if (maxDiskUsage < 100) // 100% means to disable the check + { #ifdef _MSC_VER - ULARGE_INTEGER freeBytesAvail; - ULARGE_INTEGER totalBytesAvail; + ULARGE_INTEGER freeBytesAvail; + ULARGE_INTEGER totalBytesAvail; - if (GetDiskFreeSpaceEx(fileName.c_str(), &freeBytesAvail, - &totalBytesAvail, 0) != 0) - { - double avail = (double)freeBytesAvail.QuadPart; - double total = (double)totalBytesAvail.QuadPart; - double wanted = (double)nBlocks * (double)BYTE_PER_BLOCK; + if (GetDiskFreeSpaceEx(fileName.c_str(), &freeBytesAvail, &totalBytesAvail, 0) != 0) + { + double avail = (double)freeBytesAvail.QuadPart; + double total = (double)totalBytesAvail.QuadPart; + double wanted = (double)nBlocks * (double)BYTE_PER_BLOCK; - //If we want more than there is, return an error - if (wanted > avail) - bSpaceAvail = false; - //If the remaining bytes would be too few, return an error - else if ((total - (avail - wanted)) / total * 100.0 > maxDiskUsage) - bSpaceAvail = false; - } - -#else - struct statfs fStats; - int rc = statfs( fileName.c_str(), &fStats ); - - if (rc == 0) - { - double totalBlocks = fStats.f_blocks; - double blksToAlloc = (double)(nBlocks * BYTE_PER_BLOCK) / - fStats.f_bsize; - double freeBlocks = fStats.f_bavail - blksToAlloc; - - if ((((totalBlocks - freeBlocks) / totalBlocks) * 100.0) > maxDiskUsage) - bSpaceAvail = false; - - //std::cout << "isDiskSpaceAvail" << - //": totalBlocks: " << totalBlocks << - //"; blkSize: " << fStats.f_bsize << - //"; nBlocks: " << nBlocks << - //"; freeBlks: " << freeBlocks << - //"; pctUsed: " << (((totalBlocks-freeBlocks)/totalBlocks)*100.0) << - //"; bAvail: " << bSpaceAvail << std::endl; - } - -#endif + // If we want more than there is, return an error + if (wanted > avail) + bSpaceAvail = false; + // If the remaining bytes would be too few, return an error + else if ((total - (avail - wanted)) / total * 100.0 > maxDiskUsage) + bSpaceAvail = false; } - return bSpaceAvail; +#else + struct statfs fStats; + int rc = statfs(fileName.c_str(), &fStats); + + if (rc == 0) + { + double totalBlocks = fStats.f_blocks; + double blksToAlloc = (double)(nBlocks * BYTE_PER_BLOCK) / fStats.f_bsize; + double freeBlocks = fStats.f_bavail - blksToAlloc; + + if ((((totalBlocks - freeBlocks) / totalBlocks) * 100.0) > maxDiskUsage) + bSpaceAvail = false; + + // std::cout << "isDiskSpaceAvail" << + //": totalBlocks: " << totalBlocks << + //"; blkSize: " << fStats.f_bsize << + //"; nBlocks: " << nBlocks << + //"; freeBlks: " << freeBlocks << + //"; pctUsed: " << (((totalBlocks-freeBlocks)/totalBlocks)*100.0) << + //"; bAvail: " << bSpaceAvail << std::endl; + } + +#endif + } + + return bSpaceAvail; } //------------------------------------------------------------------------------ @@ -2885,85 +2613,84 @@ bool FileOp::isDiskSpaceAvail(const std::string& fileName, int nBlocks) const // extent is expanded. //------------------------------------------------------------------------------ int FileOp::expandAbbrevColumnExtent( - IDBDataFile* pFile, // FILE ptr to file where abbrev extent is to be expanded - uint16_t dbRoot, // The DBRoot of the file with the abbreviated extent - const uint8_t* emptyVal,// Empty value to be used in expanding the extent - int width, // Width of the column (in bytes) - execplan::CalpontSystemCatalog::ColDataType colDataType) // Column data type. + IDBDataFile* pFile, // FILE ptr to file where abbrev extent is to be expanded + uint16_t dbRoot, // The DBRoot of the file with the abbreviated extent + const uint8_t* emptyVal, // Empty value to be used in expanding the extent + int width, // Width of the column (in bytes) + execplan::CalpontSystemCatalog::ColDataType colDataType) // Column data type. { - // Based on extent size, see how many blocks to add to fill the extent - int blksToAdd = ( ((int)BRMWrapper::getInstance()->getExtentRows() - - INITIAL_EXTENT_ROWS_TO_DISK) / BYTE_PER_BLOCK ) * width; + // Based on extent size, see how many blocks to add to fill the extent + int blksToAdd = + (((int)BRMWrapper::getInstance()->getExtentRows() - INITIAL_EXTENT_ROWS_TO_DISK) / BYTE_PER_BLOCK) * + width; - // Make sure there is enough disk space to expand the extent. - RETURN_ON_ERROR( setFileOffset( pFile, 0, SEEK_END ) ); + // Make sure there is enough disk space to expand the extent. + RETURN_ON_ERROR(setFileOffset(pFile, 0, SEEK_END)); - // TODO-will have to address this DiskSpaceAvail check at some point - if ( !isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), blksToAdd) ) - { - return ERR_FILE_DISK_SPACE; - } + // TODO-will have to address this DiskSpaceAvail check at some point + if (!isDiskSpaceAvail(Config::getDBRootByNum(dbRoot), blksToAdd)) + { + return ERR_FILE_DISK_SPACE; + } - // Add blocks to turn the abbreviated extent into a full extent. - int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, - width, colDataType, - false, // existing file - true, // expand existing extent - false, // n/a since not adding new extent - true); // optimize segment file extension + // Add blocks to turn the abbreviated extent into a full extent. + int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width, colDataType, + false, // existing file + true, // expand existing extent + false, // n/a since not adding new extent + true); // optimize segment file extension - return rc; + return rc; } void FileOp::setTransId(const TxnID& transId) { - m_transId = transId; + m_transId = transId; } void FileOp::setBulkFlag(bool isBulkLoad) { - m_isBulk = isBulkLoad; + m_isBulk = isBulkLoad; } int FileOp::flushFile(int rc, std::map& oids) { - return NO_ERROR; + return NO_ERROR; } int FileOp::updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { - return NO_ERROR; + return NO_ERROR; } int FileOp::updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { - return NO_ERROR; + return NO_ERROR; } void FileOp::setFixFlag(bool isFix) { - m_isFix = isFix; + m_isFix = isFix; } // Small note. We call chownFileDir in couple places to chown of the // target file and call in oid2Filename() chowns directories created bool FileOp::chownDataPath(const std::string& fileName) const { - std::ostringstream error; - idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(fileName); - if (chownPath(error, fileName, fs)) - { - logging::Message::Args args; - logging::Message message(1); - args.add(error.str()); - message.format(args); - logging::LoggingID lid(SUBSYSTEM_ID_WE_BULK); - logging::MessageLog ml(lid); - ml.logErrorMessage( message ); - return true; - } - return false; + std::ostringstream error; + idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(fileName); + if (chownPath(error, fileName, fs)) + { + logging::Message::Args args; + logging::Message message(1); + args.add(error.str()); + message.format(args); + logging::LoggingID lid(SUBSYSTEM_ID_WE_BULK); + logging::MessageLog ml(lid); + ml.logErrorMessage(message); + return true; + } + return false; } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 267bff6d4..83758dfe5 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -58,510 +58,425 @@ /** Namespace WriteEngine */ namespace WriteEngine { - constexpr size_t MaxDirLevels = 5; /** Class FileOp */ class FileOp : public BlockOp, public WeUIDGID { -public: - /** - * @brief Constructor - */ - EXPORT explicit FileOp(bool doAlloc = true); + public: + /** + * @brief Constructor + */ + EXPORT explicit FileOp(bool doAlloc = true); - /** - * @brief Destructor - */ - EXPORT virtual ~FileOp(); + /** + * @brief Destructor + */ + EXPORT virtual ~FileOp(); - /** - * @brief Close a file - */ - EXPORT void closeFile( IDBDataFile* pFile ) const; + /** + * @brief Close a file + */ + EXPORT void closeFile(IDBDataFile* pFile) const; - /** - * @brief Create a directory - */ - EXPORT int createDir( const char* dirName, mode_t mode ) const; - int createDir( const char* dirName ) const; + /** + * @brief Create a directory + */ + EXPORT int createDir(const char* dirName, mode_t mode) const; + int createDir(const char* dirName) const; - /** - * @brief Create a file with a fixed file size and file id - */ - EXPORT int createFile( FID fid, - int& allocSize, - uint16_t dbRoot, uint32_t partition, - execplan::CalpontSystemCatalog::ColDataType colDataType, - const uint8_t* emptyVal, int width = 1 ) ; + /** + * @brief Create a file with a fixed file size and file id + */ + EXPORT int createFile(FID fid, int& allocSize, uint16_t dbRoot, uint32_t partition, + execplan::CalpontSystemCatalog::ColDataType colDataType, const uint8_t* emptyVal, + int width = 1); + /** + * @brief Create a file with a fixed file size by its name. + * Changed to public for UT. + */ + int createFile(const char* fileName, int fileSize, const uint8_t* emptyVal, int width, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint16_t dbRoot, + BRM::LBID_t lbid = -1); - /** - * @brief Create a file with a fixed file size by its name. - * Changed to public for UT. - */ - int createFile( const char* fileName, int fileSize, - const uint8_t* emptyVal, int width, - execplan::CalpontSystemCatalog::ColDataType colDataType, - uint16_t dbRoot , - BRM::LBID_t lbid = -1 ); + /** + * @brief Delete a file + */ + EXPORT int deleteFile(const char* fileName) const; - /** - * @brief Delete a file - */ - EXPORT int deleteFile( const char* fileName ) const; + /** + * @brief Delete the db files corresponding to the specified file id + */ + EXPORT int deleteFile(FID fid) const; - /** - * @brief Delete the db files corresponding to the specified file id - */ - EXPORT int deleteFile( FID fid ) const; + /** + * @brief Delete the db files corresponding to the specified file id + */ + EXPORT int deleteFiles(const std::vector& fids) const; - /** - * @brief Delete the db files corresponding to the specified file id - */ - EXPORT int deleteFiles( const std::vector& fids ) const; + /** + * @brief Delete db files corresponding to specified file id and partition + */ + EXPORT int deletePartitions(const std::vector& fids, + const std::vector& partitions) const; + /** + * @brief Delete a specific database segment file. + */ + EXPORT int deleteFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; - /** - * @brief Delete db files corresponding to specified file id and partition - */ - EXPORT int deletePartitions( const std::vector& fids, - const std::vector& partitions ) - const; + /** + * @brief Check whether a file exists or not + */ + EXPORT bool exists(const char* fileName) const; - /** - * @brief Delete a specific database segment file. - */ - EXPORT int deleteFile( FID fid, uint16_t dbRoot, - uint32_t partition, - uint16_t segment ) const; + /** + * @brief @brief Check whether file exists or not by using file id, DBRoot, + * partition, and segment number. + */ + EXPORT bool exists(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; - /** - * @brief Check whether a file exists or not - */ - EXPORT bool exists( const char* fileName ) const; + /** + * @brief Check whether a column exists or not by using file id. Since this + * is not enough to fully qualify a db filename, all it can do is to verify + * that the OID directory exists on one or more of the DBRoots. + */ + EXPORT bool existsOIDDir(FID fid) const; - /** - * @brief @brief Check whether file exists or not by using file id, DBRoot, - * partition, and segment number. - */ - EXPORT bool exists( FID fid, uint16_t dbRoot, - uint32_t partition, uint16_t segment ) const; + /** + * @brief Expand current abbreviated extent for this column to a full extent + * + * @param pFile FILE ptr of segment file we are updating. + * @param dbRoot DBRoot of the file being updated. + * @param emptyVal Empty value used in initializing extents for this column + * @param width Width of this column (in bytes) + * @param colDataType Column data type. + */ + EXPORT virtual int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, + int width, + execplan::CalpontSystemCatalog::ColDataType colDataType); - /** - * @brief Check whether a column exists or not by using file id. Since this - * is not enough to fully qualify a db filename, all it can do is to verify - * that the OID directory exists on one or more of the DBRoots. - */ - EXPORT bool existsOIDDir( FID fid ) const; + /** + * @brief Add an extent to the specified Column OID and DBRoot. + * The extent must already exist in the extentmap prior to calling this fctn. + * + * The partition, segment, and HWM of the column file where the + * extent is added is returned. If needed, the applicable column segment + * file will be created. This extendFile1 function should supplant other + * extendFile functions with Multiple-File-per-OID enhancement, "but" we + * may want to rethink when we do Shared-Nothing. When this function + * returns, the file position will be located at the end of the file. + * For shared-everything DBRoot was an output argument, as BRM selected the + * the DBRoot. For shared-nothing DBRoot is an input argument, as the + * application code must track/control the DBRoot selection. + * If this is the very first file for the specified DBRoot, then the + * partition and segment number must be specified, else the selected + * partition and segment numbers are returned. + * + * @param oid OID of the column to be extended + * @param emptyVal Empty value to be used for oid + * @param width Width of the column + * @param hwm The fbo of the column segment file where the new extent begins + * @param startLbid The starting LBID for the new extent + * @param allocSize Number of blocks allocated to the extent. + * @param dbRoot The DBRoot of the file with the new extent. + * @param partition The partnum of the file with the new extent. + * @param segment The segnum of the file with the new extent. + * @param segFile (out) Name of the segment file where extent was added. + * @param pFile (out) FILE ptr to the file where the extent is added. + * @param newFile (out) Indicates if a new file was created for the extent + * @param hdrs (in/out) Contents of headers, if file is compressed. + * @return returns NO_ERROR if success. + */ + EXPORT int extendFile(OID oid, const uint8_t* emptyVal, int width, + execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm, + BRM::LBID_t startLbid, int allocSize, uint16_t dbRoot, uint32_t partition, + uint16_t segment, std::string& segFile, IDBDataFile*& pFile, bool& newFile, + char* hdrs); - /** - * @brief Expand current abbreviated extent for this column to a full extent - * - * @param pFile FILE ptr of segment file we are updating. - * @param dbRoot DBRoot of the file being updated. - * @param emptyVal Empty value used in initializing extents for this column - * @param width Width of this column (in bytes) - * @param colDataType Column data type. - */ - EXPORT virtual int expandAbbrevColumnExtent( - IDBDataFile* pFile, - uint16_t dbRoot, - const uint8_t* emptyVal, - int width, - execplan::CalpontSystemCatalog::ColDataType colDataType); + /** + * @brief For alter table add column; add an extent to a specific file + * + * @param oid OID of the column to be extended + * @param emptyVal Empty value to be used for oid + * @param width Width of the column + * @param allocSize (out) Number of blocks allocated to the extent. + * @param dbRoot The DBRoot of the file with the new extent. + * @param partition The partnum of the file with the new extent. + * @param segment The segnum of the file with the new extent. + * @param segFile (out) Name of the segment file where extent was added. + * @param startLbid (out) The starting LBID for the new extent + * @param newFile (out) Indicates if a new file was created for the extent + * @param hdrs (in/out) Contents of headers, if file is compressed. + */ + EXPORT int addExtentExactFile(OID oid, const uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, + uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, std::string& segFile, + BRM::LBID_t& startLbid, bool& newFile, char* hdrs); - /** - * @brief Add an extent to the specified Column OID and DBRoot. - * The extent must already exist in the extentmap prior to calling this fctn. - * - * The partition, segment, and HWM of the column file where the - * extent is added is returned. If needed, the applicable column segment - * file will be created. This extendFile1 function should supplant other - * extendFile functions with Multiple-File-per-OID enhancement, "but" we - * may want to rethink when we do Shared-Nothing. When this function - * returns, the file position will be located at the end of the file. - * For shared-everything DBRoot was an output argument, as BRM selected the - * the DBRoot. For shared-nothing DBRoot is an input argument, as the - * application code must track/control the DBRoot selection. - * If this is the very first file for the specified DBRoot, then the - * partition and segment number must be specified, else the selected - * partition and segment numbers are returned. - * - * @param oid OID of the column to be extended - * @param emptyVal Empty value to be used for oid - * @param width Width of the column - * @param hwm The fbo of the column segment file where the new extent begins - * @param startLbid The starting LBID for the new extent - * @param allocSize Number of blocks allocated to the extent. - * @param dbRoot The DBRoot of the file with the new extent. - * @param partition The partnum of the file with the new extent. - * @param segment The segnum of the file with the new extent. - * @param segFile (out) Name of the segment file where extent was added. - * @param pFile (out) FILE ptr to the file where the extent is added. - * @param newFile (out) Indicates if a new file was created for the extent - * @param hdrs (in/out) Contents of headers, if file is compressed. - * @return returns NO_ERROR if success. - */ - EXPORT int extendFile(OID oid, const uint8_t* emptyVal, - int width, - execplan::CalpontSystemCatalog::ColDataType colDataType, - HWM hwm, - BRM::LBID_t startLbid, - int allocSize, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - IDBDataFile*& pFile, - bool& newFile, - char* hdrs); + /** + * @brief Pad the specified compressed extent with empty chunks + * @param oid OID of relevant column + * @param width Width in bytes of this column + * @param emptyVal Empty value to be employed in filling the chunks + * @param dbRoot DBRoot of the extent to be filled + * @param partition Partition of the extent to be filled + * @param segment Segment file number of the extent to be filled + * @param colDataType Column data type + * @param hwm New HWM blk setting for the segment file after extent is padded + * @param segFile (out) Name of updated segment file + * @param errTask (out) Task that failed if error occurs + * @return returns NO_ERROR if success. + */ + EXPORT int fillCompColumnExtentEmptyChunks(OID oid, int colWidth, const uint8_t* emptyVal, uint16_t dbRoot, + uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm, + std::string& segFile, std::string& errTask); - /** - * @brief For alter table add column; add an extent to a specific file - * - * @param oid OID of the column to be extended - * @param emptyVal Empty value to be used for oid - * @param width Width of the column - * @param allocSize (out) Number of blocks allocated to the extent. - * @param dbRoot The DBRoot of the file with the new extent. - * @param partition The partnum of the file with the new extent. - * @param segment The segnum of the file with the new extent. - * @param segFile (out) Name of the segment file where extent was added. - * @param startLbid (out) The starting LBID for the new extent - * @param newFile (out) Indicates if a new file was created for the extent - * @param hdrs (in/out) Contents of headers, if file is compressed. - */ - EXPORT int addExtentExactFile(OID oid, const uint8_t* emptyVal, - int width, - int& allocSize, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - std::string& segFile, - BRM::LBID_t& startLbid, - bool& newFile, - char* hdrs); + /** + * @brief Write the specified header info to compressed column file pFile. + * + * @param pFile Column file to be written to + * @param hdr Header info to be written + */ + EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const; - /** - * @brief Pad the specified compressed extent with empty chunks - * @param oid OID of relevant column - * @param width Width in bytes of this column - * @param emptyVal Empty value to be employed in filling the chunks - * @param dbRoot DBRoot of the extent to be filled - * @param partition Partition of the extent to be filled - * @param segment Segment file number of the extent to be filled - * @param colDataType Column data type - * @param hwm New HWM blk setting for the segment file after extent is padded - * @param segFile (out) Name of updated segment file - * @param errTask (out) Task that failed if error occurs - * @return returns NO_ERROR if success. - */ - EXPORT int fillCompColumnExtentEmptyChunks(OID oid, - int colWidth, - const uint8_t* emptyVal, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - execplan::CalpontSystemCatalog::ColDataType colDataType, - HWM hwm, - std::string& segFile, - std::string& errTask); + /** + * @brief Write the specified header info to compressed column or + * dictionary file pFile. + * + * @param pFile Column file to be written to + * @param controlHdr Control header info to be written + * @param pointerHdr Pointer header info to be written + * @param ptrHdrSize Size (in bytes) of pointerHdr + */ + EXPORT int writeHeaders(IDBDataFile* pFile, const char* controlHdr, const char* pointerHdr, + uint64_t ptrHdrSize) const; - /** - * @brief Write the specified header info to compressed column file pFile. - * - * @param pFile Column file to be written to - * @param hdr Header info to be written - */ - EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const; + /** + * @brief Get the Version Buffer filename for the specified fid (OID). + * + * This version of getFileName automatically uses 0 for the partition and + * segment numbers. The applicable DBRoot is assigned based on the OID. + * + * @param fid (in) OID of the Version Buffer DB file of interest + * @param fileName (out) the name of the pertinent file that was found + * + * @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found + */ + int getVBFileName(FID fid, char* fileName) const; - /** - * @brief Write the specified header info to compressed column or - * dictionary file pFile. - * - * @param pFile Column file to be written to - * @param controlHdr Control header info to be written - * @param pointerHdr Pointer header info to be written - * @param ptrHdrSize Size (in bytes) of pointerHdr - */ - EXPORT int writeHeaders(IDBDataFile* pFile, - const char* controlHdr, - const char* pointerHdr, - uint64_t ptrHdrSize) const; + /** + * @brief Get the filename for the specified fid (OID). DBRoot, partition, + * and segment number. + * + * @param fid (in) OID of the DB file of interest + * @param fileName (out) the name of the pertinent file that was found + * @param dbRoot (in) DBRoot of the file of interest. If 0, then all the + * DBRoots will be searched. + * @param partition (in) partition number of the file of interest + * @param segment (in) segment number of the file of interest + */ + int getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; - /** - * @brief Get the Version Buffer filename for the specified fid (OID). - * - * This version of getFileName automatically uses 0 for the partition and - * segment numbers. The applicable DBRoot is assigned based on the OID. - * - * @param fid (in) OID of the Version Buffer DB file of interest - * @param fileName (out) the name of the pertinent file that was found - * - * @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found - */ - int getVBFileName( FID fid, char* fileName ) const; + /* Added for MCOL-3251 */ + void getFileNameForPrimProc(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, + uint16_t segment) const; - /** - * @brief Get the filename for the specified fid (OID). DBRoot, partition, - * and segment number. - * - * @param fid (in) OID of the DB file of interest - * @param fileName (out) the name of the pertinent file that was found - * @param dbRoot (in) DBRoot of the file of interest. If 0, then all the - * DBRoots will be searched. - * @param partition (in) partition number of the file of interest - * @param segment (in) segment number of the file of interest - */ - int getFileName( FID fid, char* fileName, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment ) const; + /** + * @brief Construct directory path for the specified fid (OID), DBRoot, and + * partition number. Directory does not have to exist, nor is it created. + */ + int getDirName(FID fid, uint16_t dbRoot, uint32_t partition, std::string& dirName) const; - /* Added for MCOL-3251 */ - void getFileNameForPrimProc(FID fid, char* fileName, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment ) const; + /** + * @brief Get the file size + */ + EXPORT int getFileSize(IDBDataFile* pFile, long long& fileSize) const; + EXPORT int getFileSize(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment, + long long& fileSize) const; - /** - * @brief Construct directory path for the specified fid (OID), DBRoot, and - * partition number. Directory does not have to exist, nor is it created. - */ - int getDirName( FID fid, uint16_t dbRoot, - uint32_t partition, - std::string& dirName) const; + /** + * @brief Initialize an extent in a dictionary store file + * @param pFile (in) IDBDataFile* of dictionary store file to be written to + * @param dbRoot (in) - DBRoot of pFile + * @param nBlocks (in) - number of blocks to be written for an extent + * @param blockHdrInit(in) - data used to initialize each block header + * @param blockHdrInitSize(in) - number of bytes in blockHdrInit + * @param bExpandExtent (in) - Expand existing extent, or initialize new one + * @param bOptExtension (in) - skip or optimize full extent preallocation + */ + EXPORT int initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit, + int blockHdrInitSize, bool bExpandExtent, bool bOptExtension = false, + int64_t lbid = 0); - /** - * @brief Get the file size - */ - EXPORT int getFileSize( IDBDataFile* pFile, long long& fileSize ) const; - EXPORT int getFileSize( FID fid, uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - long long& fileSize ) const; + /** + * @brief Check whether it is an directory + */ + EXPORT bool isDir(const char* dirName) const; - /** - * @brief Initialize an extent in a dictionary store file - * @param pFile (in) IDBDataFile* of dictionary store file to be written to - * @param dbRoot (in) - DBRoot of pFile - * @param nBlocks (in) - number of blocks to be written for an extent - * @param blockHdrInit(in) - data used to initialize each block header - * @param blockHdrInitSize(in) - number of bytes in blockHdrInit - * @param bExpandExtent (in) - Expand existing extent, or initialize new one - * @param bOptExtension (in) - skip or optimize full extent preallocation - */ - EXPORT int initDctnryExtent( IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - unsigned char* blockHdrInit, - int blockHdrInitSize, - bool bExpandExtent, - bool bOptExtension = false, - int64_t lbid = 0); + /** + * @brief See if there is room in the file system for specific number of blks + * @param fileName Name of file to extend (does not have to be full name) + * @param nBlocks Number of 8192-byte blocks to be added + * @return returns TRUE if file system has room for 'nBlocks', else FALSE + */ + EXPORT bool isDiskSpaceAvail(const std::string& fileName, int nBlocks) const; - /** - * @brief Check whether it is an directory - */ - EXPORT bool isDir( const char* dirName ) const; + /** + * @brief Convert an oid to a full file name + */ + EXPORT int oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t dbRoot, uint32_t partition, + uint16_t segment) const; + EXPORT int oid2DirName(FID fid, char* oidDirName) const; - /** - * @brief See if there is room in the file system for specific number of blks - * @param fileName Name of file to extend (does not have to be full name) - * @param nBlocks Number of 8192-byte blocks to be added - * @return returns TRUE if file system has room for 'nBlocks', else FALSE - */ - EXPORT bool isDiskSpaceAvail(const std::string& fileName, - int nBlocks) const; + /** + * @brief Open a file using a filename. + * @param fileName Name of the file to open. + * @param mode Mode to use in opening the file (ex: "r+b"). + * @param ioBuffSize Buffer size to be employed by setvbuf(). + * @return returns the IDBDataFile* of the opened file. + */ + EXPORT IDBDataFile* openFile(const char* fileName, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ, + bool useTmpSuffix = false) const; - /** - * @brief Convert an oid to a full file name - */ - EXPORT int oid2FileName( FID fid, char* fullFileName, - bool bCreateDir, uint16_t dbRoot, - uint32_t partition, uint16_t segment ) const; - EXPORT int oid2DirName( FID fid, char* oidDirName ) const; + /** + * @brief Open a file using an OID, dbroot, partition, and segment number. + * @param fid OID of the file to be opened. + * @param dbRoot DBRoot of the file to be opened. + * @param partition Partition number of the file to be opened. + * @param segment Segment number of the file to be opened. + * @param mode Mode to use in opening the file (default of "r+b" will open + * an existing binary file as read/write. + * @param ioBuffSize Buffer size to be employed by setvbuf(). + * @return returns the IDBDataFile* of the opened file. + */ + EXPORT IDBDataFile* openFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ, + bool useTmpSuffix = false) const; - /** - * @brief Open a file using a filename. - * @param fileName Name of the file to open. - * @param mode Mode to use in opening the file (ex: "r+b"). - * @param ioBuffSize Buffer size to be employed by setvbuf(). - * @return returns the IDBDataFile* of the opened file. - */ - EXPORT IDBDataFile* openFile( const char* fileName, - const char* mode = "r+b", - int ioColSize = DEFAULT_COLSIZ, - bool useTmpSuffix = false) const; + /** + * @brief Read to a buffer from a file at current location + */ + EXPORT int readFile(IDBDataFile* pFile, unsigned char* readBuf, int readSize) const; - /** - * @brief Open a file using an OID, dbroot, partition, and segment number. - * @param fid OID of the file to be opened. - * @param dbRoot DBRoot of the file to be opened. - * @param partition Partition number of the file to be opened. - * @param segment Segment number of the file to be opened. - * @param mode Mode to use in opening the file (default of "r+b" will open - * an existing binary file as read/write. - * @param ioBuffSize Buffer size to be employed by setvbuf(). - * @return returns the IDBDataFile* of the opened file. - */ - EXPORT IDBDataFile* openFile( FID fid, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - const char* mode = "r+b", - int ioColSize = DEFAULT_COLSIZ, - bool useTmpSuffix = false) const; + /** + * @brief Reads in 2 compression header blocks from a column segment file. + * IDBDataFile* points to start of data when function returns. + * @param pFile (in) IDBDataFile* of column segment file to be read. + * @param hdrs (out) Contents of headers that are read. + */ + EXPORT int readHeaders(IDBDataFile* pFile, char* hdrs) const; + EXPORT int readHeaders(IDBDataFile* pFile, char* hdr1, char* hdr2) const; - /** - * @brief Read to a buffer from a file at current location - */ - EXPORT int readFile( IDBDataFile* pFile, unsigned char* readBuf, - int readSize ) const; + /** + * @brief Reinitialize a partial extent in a column segment file + * @param pFile (in) IDBDataFile* of column segment file to be written to + * @param startOffset (in) - file offset where blocks are to be written + * @param nBlocks (in) - number of blocks to be written to the extent + * @param emptyVal(in) - empty value to be used for column data values + * width (in) - width of the applicable column + */ + EXPORT int reInitPartialColumnExtent(IDBDataFile* pFile, long long startOffset, int nBlocks, + const uint8_t* emptyVal, int width); - /** - * @brief Reads in 2 compression header blocks from a column segment file. - * IDBDataFile* points to start of data when function returns. - * @param pFile (in) IDBDataFile* of column segment file to be read. - * @param hdrs (out) Contents of headers that are read. - */ - EXPORT int readHeaders( IDBDataFile* pFile, char* hdrs ) const; - EXPORT int readHeaders( IDBDataFile* pFile, char* hdr1, char* hdr2 )const; + /** + * @brief Reinitialize an extent in a dictionary store file + * @param pFile (in) IDBDataFile* of dictionary store file to be written to + * @param startOffset (in) - file offset where blocks are to be written + * @param nBlocks (in) - number of blocks to be written to the extent + * @param blockHdrInit(in) - data used to initialize each block header + * @param blockHdrInitSize(in) - number of bytes in blockHdrInit + */ + EXPORT int reInitPartialDctnryExtent(IDBDataFile* pFile, long long startOffset, int nBlocks, + unsigned char* blockHdrInit, int blockHdrInitSize); - /** - * @brief Reinitialize a partial extent in a column segment file - * @param pFile (in) IDBDataFile* of column segment file to be written to - * @param startOffset (in) - file offset where blocks are to be written - * @param nBlocks (in) - number of blocks to be written to the extent - * @param emptyVal(in) - empty value to be used for column data values - * width (in) - width of the applicable column - */ - EXPORT int reInitPartialColumnExtent( IDBDataFile* pFile, - long long startOffset, - int nBlocks, - const uint8_t* emptyVal, - int width ); + /** + * @brief Set the file to specified location based on the offset + */ + EXPORT int setFileOffset(IDBDataFile* pFile, long long offset, int origin = SEEK_SET) const; + EXPORT int setFileOffsetBlock(IDBDataFile* pFile, uint64_t lbid, int origin = SEEK_SET) const; - /** - * @brief Reinitialize an extent in a dictionary store file - * @param pFile (in) IDBDataFile* of dictionary store file to be written to - * @param startOffset (in) - file offset where blocks are to be written - * @param nBlocks (in) - number of blocks to be written to the extent - * @param blockHdrInit(in) - data used to initialize each block header - * @param blockHdrInitSize(in) - number of bytes in blockHdrInit - */ - EXPORT int reInitPartialDctnryExtent( IDBDataFile* pFile, - long long startOffset, - int nBlocks, - unsigned char* blockHdrInit, - int blockHdrInitSize ); + /** + * @brief Truncate the file to the specified file size + */ + EXPORT int truncateFile(IDBDataFile* pFile, long long fileSize) const; - /** - * @brief Set the file to specified location based on the offset - */ - EXPORT int setFileOffset( IDBDataFile* pFile, - long long offset, - int origin = SEEK_SET ) const; - EXPORT int setFileOffsetBlock( IDBDataFile* pFile, - uint64_t lbid, - int origin = SEEK_SET ) const; + /** + * @brief Write a buffer to a file at current location + */ + EXPORT int writeFile(IDBDataFile* pFile, const unsigned char* buf, int bufSize) const; - /** - * @brief Truncate the file to the specified file size - */ - EXPORT int truncateFile( IDBDataFile* pFile, - long long fileSize ) const; + /** + * @brief set the flag to use the instance to access the brm wrapper class + */ + EXPORT virtual void setTransId(const TxnID& transId); + EXPORT virtual void setBulkFlag(bool isBulkLoad); + EXPORT virtual void setFixFlag(bool isFix); + TxnID getTransId() const; - /** - * @brief Write a buffer to a file at current location - */ - EXPORT int writeFile( IDBDataFile* pFile, - const unsigned char* buf, int bufSize ) const; + void compressionType(int t); + int compressionType() const; - /** - * @brief set the flag to use the instance to access the brm wrapper class - */ - EXPORT virtual void setTransId( const TxnID& transId); - EXPORT virtual void setBulkFlag(bool isBulkLoad); - EXPORT virtual void setFixFlag(bool isFix); - TxnID getTransId() const; + EXPORT virtual int flushFile(int rc, std::map& oids); + // Initialize an extent in a column segment file + // pFile (in) IDBDataFile* of column segment file to be written to + // dbRoot (in) - DBRoot of pFile + // nBlocks (in) - number of blocks to be written for an extent + // emptyVal(in) - empty value to be used for column data values + // width (in) - width of the applicable column + // bNewFile (in) - Adding extent to new file + // bExpandExtent (in) - Expand existing extent, or initialize new one + // bAbbrevExtent (in) - If adding new extent, is it abbreviated + // bOptExtension(in) - skip or optimize full extent preallocation + int initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, int width, + execplan::CalpontSystemCatalog::ColDataType colDataType, bool bNewFile, + bool bExpandExtent, bool bAbbrevExtent, bool bOptExtension = false, int64_t lbid = 0); - void compressionType(int t); - int compressionType() const; + // Calls a chown and logs an error message + bool chownDataPath(const std::string& fileName) const; - EXPORT virtual int flushFile(int rc, std::map& oids); - // Initialize an extent in a column segment file - // pFile (in) IDBDataFile* of column segment file to be written to - // dbRoot (in) - DBRoot of pFile - // nBlocks (in) - number of blocks to be written for an extent - // emptyVal(in) - empty value to be used for column data values - // width (in) - width of the applicable column - // bNewFile (in) - Adding extent to new file - // bExpandExtent (in) - Expand existing extent, or initialize new one - // bAbbrevExtent (in) - If adding new extent, is it abbreviated - // bOptExtension(in) - skip or optimize full extent preallocation - int initColumnExtent( IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - const uint8_t* emptyVal, - int width, - execplan::CalpontSystemCatalog::ColDataType colDataType, - bool bNewFile, - bool bExpandExtent, - bool bAbbrevExtent, - bool bOptExtension=false, - int64_t lbid = 0 ); + protected: + EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); + EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); - // Calls a chown and logs an error message - bool chownDataPath(const std::string& fileName) const; + int m_compressionType; // compresssion type -protected: - EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); - EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); + private: + // not copyable + FileOp(const FileOp& rhs); + FileOp& operator=(const FileOp& rhs); - int m_compressionType; // compresssion type + int expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal, int colWidth, + const compress::CompChunkPtr& chunkInPtr, compress::CompChunkPtr& chunkOutPt, + const char* hdrs = nullptr); -private: - //not copyable - FileOp(const FileOp& rhs); - FileOp& operator=(const FileOp& rhs); + int initAbbrevCompColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, + int width, BRM::LBID_t lbid, + execplan::CalpontSystemCatalog::ColDataType colDataType); - int expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal, - int colWidth, - const compress::CompChunkPtr& chunkInPtr, - compress::CompChunkPtr& chunkOutPt, - const char* hdrs = nullptr); + static void initDbRootExtentMutexes(); + static void removeDbRootExtentMutexes(); - int initAbbrevCompColumnExtent( - IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - const uint8_t* emptyVal, int width, BRM::LBID_t lbid, - execplan::CalpontSystemCatalog::ColDataType colDataType); + int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows, + const uint8_t* emptyVal, int width, BRM::LBID_t lbid, + execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs); - static void initDbRootExtentMutexes(); - static void removeDbRootExtentMutexes(); + TxnID m_transId; + bool m_isBulk; + bool m_isFix; - int writeInitialCompColumnChunk( - IDBDataFile* pFile, int nBlocksAllocated, int nRows, - const uint8_t* emptyVal, int width, BRM::LBID_t lbid, - execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs); + // protect creation of m_DbRootAddExtentMutexes + static boost::mutex m_createDbRootMutexes; - TxnID m_transId; - bool m_isBulk; - bool m_isFix; + // Mutexes used to serialize extent creation within each DBRoot + static std::map m_DbRootAddExtentMutexes; - // protect creation of m_DbRootAddExtentMutexes - static boost::mutex m_createDbRootMutexes; + // protect race condition in creating directories + static boost::mutex m_mkdirMutex; - // Mutexes used to serialize extent creation within each DBRoot - static std::map m_DbRootAddExtentMutexes; - - // protect race condition in creating directories - static boost::mutex m_mkdirMutex; - - char* m_buffer; // buffer used with setvbuf() + char* m_buffer; // buffer used with setvbuf() }; //------------------------------------------------------------------------------ @@ -569,43 +484,41 @@ private: //------------------------------------------------------------------------------ inline void FileOp::compressionType(int t) { - m_compressionType = t; + m_compressionType = t; } inline int FileOp::compressionType() const { - return m_compressionType; + return m_compressionType; } -inline int FileOp::createDir( const char* dirName ) const +inline int FileOp::createDir(const char* dirName) const { - return createDir( dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ); + return createDir(dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); } -inline int FileOp::getVBFileName( FID fid, char* fileName ) const +inline int FileOp::getVBFileName(FID fid, char* fileName) const { - uint16_t dbRoot = 0; - uint32_t partition = 0; - uint16_t segment = 0; + uint16_t dbRoot = 0; + uint32_t partition = 0; + uint16_t segment = 0; - return oid2FileName( fid, fileName, true, dbRoot, partition, segment ); + return oid2FileName(fid, fileName, true, dbRoot, partition, segment); } -inline int FileOp::getFileName( FID fid, char* fileName, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment ) const +inline int FileOp::getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, + uint16_t segment) const { - return oid2FileName( fid, fileName, false, dbRoot, partition, segment ); + return oid2FileName(fid, fileName, false, dbRoot, partition, segment); } inline TxnID FileOp::getTransId() const { - return m_transId; + return m_transId; } -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_FILEOP_H_ +#endif // _WE_FILEOP_H_ diff --git a/writeengine/shared/we_index.h b/writeengine/shared/we_index.h index bcd88e5e9..85ab6da90 100644 --- a/writeengine/shared/we_index.h +++ b/writeengine/shared/we_index.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_index.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -******************************************************************************************/ + * $Id: we_index.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_INDEX_H_ @@ -28,117 +28,114 @@ #include "we_type.h" - - /** Namespace WriteEngine */ namespace WriteEngine { - /***************************************************** -* index definition -******************************************************/ -const int IDX_BITTEST_SIZE = 10; /** @brief The bit size of bit test */ -const int IDX_GROUP_SIZE = 3; /** @brief The bit size of group */ -const int IDX_INSTRU_SIZE = 4; /** @brief The bit size of instruction */ -const int IDX_PTR_SIZE = 46; /** @brief The bit size of address pointer */ -const int IDX_TYPE_SIZE = 3; /** @brief The bit size of type */ + * index definition + ******************************************************/ +const int IDX_BITTEST_SIZE = 10; /** @brief The bit size of bit test */ +const int IDX_GROUP_SIZE = 3; /** @brief The bit size of group */ +const int IDX_INSTRU_SIZE = 4; /** @brief The bit size of instruction */ +const int IDX_PTR_SIZE = 46; /** @brief The bit size of address pointer */ +const int IDX_TYPE_SIZE = 3; /** @brief The bit size of type */ -const int IDX_BITMAP_SUBBLOCK_NO = 1; /** @brief Subblock 1 of root block is for bitmap pointer*/ -const int IDX_MAX_TREE_LEVEL = 128; /** @brief The maximum depth of a tree */ -const int IDX_MAX_MULTI_COL_BIT = 256; /** @brief The maximum bits of a multi-column tree (256 bit)*/ -const int IDX_MAX_MULTI_COL_IDX_LEVEL = 52; /** @brief The maximum depth of a multi-column tree */ -const int IDX_MAX_MULTI_COL_IDX_NUM = 64; /** @brief The maximum number of columns for a multi-column index */ -const int MAX_IDX_RID = 1024; /** @brief Maximum index rids for one shot */ -const int IDX_DEFAULT_READ_ROW = 10000; /** @brief Default number of rows for one read for index */ +const int IDX_BITMAP_SUBBLOCK_NO = 1; /** @brief Subblock 1 of root block is for bitmap pointer*/ +const int IDX_MAX_TREE_LEVEL = 128; /** @brief The maximum depth of a tree */ +const int IDX_MAX_MULTI_COL_BIT = 256; /** @brief The maximum bits of a multi-column tree (256 bit)*/ +const int IDX_MAX_MULTI_COL_IDX_LEVEL = 52; /** @brief The maximum depth of a multi-column tree */ +const int IDX_MAX_MULTI_COL_IDX_NUM = 64; /** @brief The maximum number of columns for a multi-column index */ +const int MAX_IDX_RID = 1024; /** @brief Maximum index rids for one shot */ +const int IDX_DEFAULT_READ_ROW = 10000; /** @brief Default number of rows for one read for index */ // todo: need to move a higher level share file for dictionary -const int RID_SIZE = 46; +const int RID_SIZE = 46; // const int OID_SIZE = 24; /** @brief The bit size of object id */ -const int FBO_SIZE = 36; /** @brief The bit size of file block offset */ -const int SBID_SIZE = 5; /** @brief The bit size of sub block id */ -const int ENTRY_SIZE = 5; /** @brief The bit size of entry location with a sub block */ +const int FBO_SIZE = 36; /** @brief The bit size of file block offset */ +const int SBID_SIZE = 5; /** @brief The bit size of sub block id */ +const int ENTRY_SIZE = 5; /** @brief The bit size of entry location with a sub block */ -const int LIST_SIZE_TYPE = 0; -const int LIST_RID_TYPE = 3; -const int LIST_NOT_USED_TYPE = 7; -const int LIST_HDR_SIZE = 32; -const int LIST_SUBBLOCK_TYPE = 4 ; -const int LIST_BLOCK_TYPE = 5 ; -const int LIST_LLP_TYPE = 6 ; -const int SUBBLOCK_TOTAL_BYTES = 256; -const int LIST_SUB_LLP_POS = 31; -const int LIST_LAST_LBID_POS = 30; -const int LIST_BLOCK_LLP_POS = 1023; -const int MAX_BLOCK_ENTRY = 1024; -const int MAX_SUB_RID_CNT = 30; -const int MAX_BLK_RID_CNT = 1023; -const int MAX_BLK_NARRAY_RID_CNT = 1018; -const int LBID_SBID_ENTRY = 46; -const int RID_COUNT_SIZE = 10; -const int CUR_BLK_POS_WIDTH = 2; -const int LLP_STATUS_WIDTH = 2; -const int LIST_ENTRY_WIDTH = 8; -const int LIST_BLK_LLP_ENTRY_WIDTH = 48; -const int BEGIN_LIST_BLK_LLP_POS = 1018; -const int NEXT_BLK_PTR_OFFSET = 5; -const int PARENT_PTR_OFFSET = 4; -const int TOTAL_NUM_ARRAY_PTR = 4; -const int ARRAY_LLP_EXIST = 1; -const int LLP_NOT_FULL = 0; -const int LLP_FULL = 1; -const int TOTAL_CUR_LEVEL = 10; -const int CUR_LEVEL_POS_WIDTH = 20; -const uint64_t INVALID_KEY = -1LL; /** @brief Invalid number */ +const int LIST_SIZE_TYPE = 0; +const int LIST_RID_TYPE = 3; +const int LIST_NOT_USED_TYPE = 7; +const int LIST_HDR_SIZE = 32; +const int LIST_SUBBLOCK_TYPE = 4; +const int LIST_BLOCK_TYPE = 5; +const int LIST_LLP_TYPE = 6; +const int SUBBLOCK_TOTAL_BYTES = 256; +const int LIST_SUB_LLP_POS = 31; +const int LIST_LAST_LBID_POS = 30; +const int LIST_BLOCK_LLP_POS = 1023; +const int MAX_BLOCK_ENTRY = 1024; +const int MAX_SUB_RID_CNT = 30; +const int MAX_BLK_RID_CNT = 1023; +const int MAX_BLK_NARRAY_RID_CNT = 1018; +const int LBID_SBID_ENTRY = 46; +const int RID_COUNT_SIZE = 10; +const int CUR_BLK_POS_WIDTH = 2; +const int LLP_STATUS_WIDTH = 2; +const int LIST_ENTRY_WIDTH = 8; +const int LIST_BLK_LLP_ENTRY_WIDTH = 48; +const int BEGIN_LIST_BLK_LLP_POS = 1018; +const int NEXT_BLK_PTR_OFFSET = 5; +const int PARENT_PTR_OFFSET = 4; +const int TOTAL_NUM_ARRAY_PTR = 4; +const int ARRAY_LLP_EXIST = 1; +const int LLP_NOT_FULL = 0; +const int LLP_FULL = 1; +const int TOTAL_CUR_LEVEL = 10; +const int CUR_LEVEL_POS_WIDTH = 20; +const uint64_t INVALID_KEY = -1LL; /** @brief Invalid number */ /***************************************************** -* mask definition -******************************************************/ -const int BIT_MASK_ARRAY[] = { 0x0, - 0x01, /** @brief 1 bit mask */ - 0x03, /** @brief 2 bit mask */ - 0x07, /** @brief 3 bit mask */ - 0x0F, /** @brief 4 bit mask */ - 0x1F, /** @brief 5 bit mask */ - 0x3F /** @brief 6 bit mask */ - }; + * mask definition + ******************************************************/ +const int BIT_MASK_ARRAY[] = { + 0x0, 0x01, /** @brief 1 bit mask */ + 0x03, /** @brief 2 bit mask */ + 0x07, /** @brief 3 bit mask */ + 0x0F, /** @brief 4 bit mask */ + 0x1F, /** @brief 5 bit mask */ + 0x3F /** @brief 6 bit mask */ +}; /************************************************************************ * Type enumerations ************************************************************************/ -enum IdxTreeEntryType /** @brief Index tree entry types */ +enum IdxTreeEntryType /** @brief Index tree entry types */ { - EMPTY_ENTRY = 0, /** @brief Empty entry */ - UNIQUE_VAL = 7, /** @brief Unique value */ - EMPTY_LIST = 1, /** @brief Empty list pointer entry */ - EMPTY_PTR = 2, /** @brief Empty pointer entry */ - BIT_TEST = 3, /** @brief Bit test entry */ - LEAF_LIST = 4, /** @brief Leaf list pointer */ - BITMAP_PTR = 5, /** @brief Bitmap pointer */ -// SORT_LIST = 5, /** @brief Sorted list pointer */ - MULTI_COL = 6 /** @brief Multi-column index pointer */ + EMPTY_ENTRY = 0, /** @brief Empty entry */ + UNIQUE_VAL = 7, /** @brief Unique value */ + EMPTY_LIST = 1, /** @brief Empty list pointer entry */ + EMPTY_PTR = 2, /** @brief Empty pointer entry */ + BIT_TEST = 3, /** @brief Bit test entry */ + LEAF_LIST = 4, /** @brief Leaf list pointer */ + BITMAP_PTR = 5, /** @brief Bitmap pointer */ + // SORT_LIST = 5, /** @brief Sorted list pointer */ + MULTI_COL = 6 /** @brief Multi-column index pointer */ }; -enum IdxTreeGroupType /** @brief Index tree group types */ +enum IdxTreeGroupType /** @brief Index tree group types */ { - ENTRY_1 = 0, /** @brief 1 entry per group */ - ENTRY_2 = 1, /** @brief 2 entry per group */ - ENTRY_4 = 2, /** @brief 4 entry per group */ - ENTRY_8 = 3, /** @brief 8 entry per group */ - ENTRY_16 = 4, /** @brief 16 entry per group */ - ENTRY_32 = 5, /** @brief 32 entry per group */ - ENTRY_BLK = 6 /** @brief 1k entry per group */ + ENTRY_1 = 0, /** @brief 1 entry per group */ + ENTRY_2 = 1, /** @brief 2 entry per group */ + ENTRY_4 = 2, /** @brief 4 entry per group */ + ENTRY_8 = 3, /** @brief 8 entry per group */ + ENTRY_16 = 4, /** @brief 16 entry per group */ + ENTRY_32 = 5, /** @brief 32 entry per group */ + ENTRY_BLK = 6 /** @brief 1k entry per group */ }; -enum IdxBitCompareType /** @brief Index bit compare types */ +enum IdxBitCompareType /** @brief Index bit compare types */ { - BIT_5 = 0, /** @brief 5-bit compare */ - BIT_10 = 1 /** @brief 10-bit compare */ + BIT_5 = 0, /** @brief 5-bit compare */ + BIT_10 = 1 /** @brief 10-bit compare */ }; -enum IdxFreeMgrType /** @brief Index free manager types */ +enum IdxFreeMgrType /** @brief Index free manager types */ { - TREE = 0, /** @brief Index tree type */ - LIST = 1 /** @brief Index list type */ + TREE = 0, /** @brief Index tree type */ + LIST = 1 /** @brief Index list type */ }; /************************************************************************ @@ -146,164 +143,119 @@ enum IdxFreeMgrType /** @brief Index free manager types */ ************************************************************************/ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 12; /** @brief spare bits */ -uint64_t group : - IDX_GROUP_SIZE; /** @brief entry group type */ - // The following is related to ptr -uint64_t fbo : - FBO_SIZE; /** @brief file block offset */ -uint64_t sbid : - SBID_SIZE; /** @brief sub block id */ -uint64_t entry : - ENTRY_SIZE; /** @brief entry within sub block */ -} IdxStartSubBlockEntry; /** @brief Index start block entry structure */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 12; /** @brief spare bits */ + uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */ + // The following is related to ptr + uint64_t fbo : FBO_SIZE; /** @brief file block offset */ + uint64_t sbid : SBID_SIZE; /** @brief sub block id */ + uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */ +} IdxStartSubBlockEntry; /** @brief Index start block entry structure */ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 2; /** @brief spare bits */ -uint64_t group : - IDX_GROUP_SIZE; /** @brief entry group type */ - // The following is related to ptr - uint64_t spare2 : 10; /** @brief spare bits */ -uint64_t fbo : - FBO_SIZE; /** @brief file block offset */ -uint64_t sbid : - SBID_SIZE; /** @brief sub block id */ -uint64_t entry : - ENTRY_SIZE; /** @brief entry within sub block */ -} IdxEmptyListEntry; /** @brief Index empty list entry structure */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 2; /** @brief spare bits */ + uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */ + // The following is related to ptr + uint64_t spare2 : 10; /** @brief spare bits */ + uint64_t fbo : FBO_SIZE; /** @brief file block offset */ + uint64_t sbid : SBID_SIZE; /** @brief sub block id */ + uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */ +} IdxEmptyListEntry; /** @brief Index empty list entry structure */ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 15; /** @brief spare bits */ - // The following is related to ptr -uint64_t fbo : - FBO_SIZE; /** @brief file block offset */ -uint64_t sbid : - SBID_SIZE; /** @brief sub block id */ -uint64_t entry : - ENTRY_SIZE; /** @brief entry within sub block */ -} IdxBitmapPointerEntry; /** @brief Index bitmap pointer entry structure */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 15; /** @brief spare bits */ + // The following is related to ptr + uint64_t fbo : FBO_SIZE; /** @brief file block offset */ + uint64_t sbid : SBID_SIZE; /** @brief sub block id */ + uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */ +} IdxBitmapPointerEntry; /** @brief Index bitmap pointer entry structure */ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ -uint64_t bitTest : - IDX_BITTEST_SIZE; /** @brief index bittest */ -uint64_t group : - IDX_GROUP_SIZE; /** @brief entry group type */ - uint64_t bitCompare : 1; - uint64_t spare : 1; /** @brief spare bits */ - // The following is related to ptr -uint64_t fbo : - FBO_SIZE; /** @brief file block offset */ -uint64_t sbid : - SBID_SIZE; /** @brief sub block id */ -uint64_t entry : - ENTRY_SIZE; /** @brief entry within sub block */ -} IdxBitTestEntry; /** @brief Index bit test entry structure */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t bitTest : IDX_BITTEST_SIZE; /** @brief index bittest */ + uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */ + uint64_t bitCompare : 1; + uint64_t spare : 1; /** @brief spare bits */ + // The following is related to ptr + uint64_t fbo : FBO_SIZE; /** @brief file block offset */ + uint64_t sbid : SBID_SIZE; /** @brief sub block id */ + uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */ +} IdxBitTestEntry; /** @brief Index bit test entry structure */ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 15; /** @brief spare bits */ - // The following is related to ptr -uint64_t fbo : - FBO_SIZE; /** @brief file block offset */ -uint64_t sbid : - SBID_SIZE; /** @brief sub block id */ -uint64_t entry : - ENTRY_SIZE; /** @brief entry within sub block */ -} IdxTreePointerEntry; /** @brief Index tree pointer entry structure */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 15; /** @brief spare bits */ + // The following is related to ptr + uint64_t fbo : FBO_SIZE; /** @brief file block offset */ + uint64_t sbid : SBID_SIZE; /** @brief sub block id */ + uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */ +} IdxTreePointerEntry; /** @brief Index tree pointer entry structure */ /************************************************************************ * @brief index list node defintions ************************************************************************/ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type 3 */ - uint64_t spare : 15; /** @brief spare bits */ -RID rid : - RID_SIZE; /** @brief row id */ -} IdxRidListEntry; /** @brief Index rid list entry structure */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 3 */ + uint64_t spare : 15; /** @brief spare bits */ + RID rid : RID_SIZE; /** @brief row id */ +} IdxRidListEntry; /** @brief Index rid list entry structure */ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 5; -uint64_t count : - RID_COUNT_SIZE; /** the count of rids on the current blk */ -uint64_t llp : - LBID_SBID_ENTRY; /** @brief size */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 5; + uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */ + uint64_t llp : LBID_SBID_ENTRY; /** @brief size */ } IdxRidListPtr; typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 5; -uint64_t count : - RID_COUNT_SIZE; /** the count of rids on the current blk */ -uint64_t lbid : - FBO_SIZE; /** @brief size */ -uint64_t sbid : - SBID_SIZE; /** @brief sub block id */ -uint64_t entry : - ENTRY_SIZE; /** @brief entry within sub block */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 5; + uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */ + uint64_t lbid : FBO_SIZE; /** @brief size */ + uint64_t sbid : SBID_SIZE; /** @brief sub block id */ + uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */ } IdxRidLastListPtr; typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 13; -uint64_t llpStat : - LLP_STATUS_WIDTH; /** llp status */ -uint64_t childLbid : - FBO_SIZE; /** @brief file block offset */ - uint64_t spare2 : 10; + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 13; + uint64_t llpStat : LLP_STATUS_WIDTH; /** llp status */ + uint64_t childLbid : FBO_SIZE; /** @brief file block offset */ + uint64_t spare2 : 10; } IdxRidChildListPtr; typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type 0 or 6 */ - uint64_t spare : 5; -uint64_t count : - RID_COUNT_SIZE; /** the count of rids on the current blk */ -uint64_t nextLbid : - FBO_SIZE; /** @brief file block offset */ -uint64_t curLevel : - TOTAL_CUR_LEVEL; + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 0 or 6 */ + uint64_t spare : 5; + uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */ + uint64_t nextLbid : FBO_SIZE; /** @brief file block offset */ + uint64_t curLevel : TOTAL_CUR_LEVEL; } IdxRidNextListPtr; typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type 6*/ - uint64_t spare : 3; /** @brief spare bits */ -uint64_t curLevelPos : - CUR_LEVEL_POS_WIDTH; -uint64_t curBlkPos : - CUR_BLK_POS_WIDTH; /** the position of current blk */ -uint64_t parentLbid : - FBO_SIZE; /** @brief file block offset */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 6*/ + uint64_t spare : 3; /** @brief spare bits */ + uint64_t curLevelPos : CUR_LEVEL_POS_WIDTH; + uint64_t curBlkPos : CUR_BLK_POS_WIDTH; /** the position of current blk */ + uint64_t parentLbid : FBO_SIZE; /** @brief file block offset */ } IdxRidParentListPtr; typedef struct { - IdxRidChildListPtr childIdxRidListPtr[4]; - IdxRidParentListPtr parentIdxListPtr; - IdxRidNextListPtr nextIdxListPtr; + IdxRidChildListPtr childIdxRidListPtr[4]; + IdxRidParentListPtr parentIdxListPtr; + IdxRidNextListPtr nextIdxListPtr; } IdxRidListArrayPtr; /************************************************************************ @@ -311,145 +263,145 @@ typedef struct ************************************************************************/ typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 15; /** @brief spare bits */ -uint64_t size : - RID_SIZE; /** @brief size */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 15; /** @brief spare bits */ + uint64_t size : RID_SIZE; /** @brief size */ } IdxRidListHdrSize; typedef struct { -uint64_t type : - IDX_TYPE_SIZE; /** @brief entry type */ - uint64_t spare : 15; /** @brief spare bits */ -uint64_t llp : - RID_SIZE; /** @brief size */ + uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */ + uint64_t spare : 15; /** @brief spare bits */ + uint64_t llp : RID_SIZE; /** @brief size */ } IdxRidListHdrPtr; typedef struct { - IdxRidListHdrSize idxRidListSize; - uint64_t key; - IdxRidListEntry firstIdxRidListEntry; - IdxRidListHdrPtr nextIdxRidListPtr; + IdxRidListHdrSize idxRidListSize; + uint64_t key; + IdxRidListEntry firstIdxRidListEntry; + IdxRidListHdrPtr nextIdxRidListPtr; } IdxRidListHdr; typedef struct { - uint64_t part1 : 15; /** @brief entry type */ - uint64_t part2 : 15; /** @brief spare bits */ - uint64_t spare : 34; /** @brief size */ + uint64_t part1 : 15; /** @brief entry type */ + uint64_t part2 : 15; /** @brief spare bits */ + uint64_t spare : 34; /** @brief size */ } IdxRidListOffSet; /************************************************************************ * @brief index tree node defintions ************************************************************************/ typedef struct { - IdxBitTestEntry next; /** @brief next in the node */ - IdxBitTestEntry current; /** @brief current addr */ - uint16_t level; /** @brief tree level */ - uint16_t allocCount; /** @brief allocated entry cound from free mgr */ - uint16_t useCount; /** @brief actual use entry count */ - uint16_t offset; /** @brief entry offset */ - bool used; /** @brief used flag */ -} IdxTreeNode; /** @brief Index tree node */ + IdxBitTestEntry next; /** @brief next in the node */ + IdxBitTestEntry current; /** @brief current addr */ + uint16_t level; /** @brief tree level */ + uint16_t allocCount; /** @brief allocated entry cound from free mgr */ + uint16_t useCount; /** @brief actual use entry count */ + uint16_t offset; /** @brief entry offset */ + bool used; /** @brief used flag */ +} IdxTreeNode; /** @brief Index tree node */ typedef struct { - IdxTreeNode node[IDX_MAX_TREE_LEVEL]; /** @brief node array */ - uint16_t maxLevel; /** @brief max level */ - RID rid; /** @brief current row id */ - uint64_t key; /** @brief current key */ - uint16_t width; /** @brief current width */ -} IdxTree; /** @brief Index tree */ + IdxTreeNode node[IDX_MAX_TREE_LEVEL]; /** @brief node array */ + uint16_t maxLevel; /** @brief max level */ + RID rid; /** @brief current row id */ + uint64_t key; /** @brief current key */ + uint16_t width; /** @brief current width */ +} IdxTree; /** @brief Index tree */ struct IdxTreeCacheNode { - RID rid; /** @brief RID */ - uint64_t key; /** @brief Key */ - IdxEmptyListEntry entry; /** @brief List pointer */ - bool used; /** @brief Used flag */ - IdxTreeCacheNode() - { - used = false; - } + RID rid; /** @brief RID */ + uint64_t key; /** @brief Key */ + IdxEmptyListEntry entry; /** @brief List pointer */ + bool used; /** @brief Used flag */ + IdxTreeCacheNode() + { + used = false; + } }; struct IdxMultiColKey { - std::bitset bitSet; /** @brief BitArray for all bits */ - std::bitset curBitset;/** @brief Current working column */ - std::bitset curMask; /** @brief Current bitset mask */ - unsigned char keyBuf[IDX_MAX_MULTI_COL_BIT / 8]; /** @brief Key buffer */ - int curLevel; /** @brief Current index level */ - int maxLevel; /** @brief Maximum index level */ - int totalBit; /** @brief Total bits */ - int testbitArray[IDX_MAX_MULTI_COL_IDX_LEVEL]; /** @brief Test bit array */ - void clear() - { - bitSet.reset(); - curBitset.reset(); - curMask.reset(); - curLevel = maxLevel = 0; - totalBit = 0; - memset( testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL * sizeof(testbitArray[0])); - memset( keyBuf, 0, IDX_MAX_MULTI_COL_BIT / 8 ); - curMask = 0x1F; - curMask = curMask << (IDX_MAX_MULTI_COL_BIT - 5); - } - IdxMultiColKey() - { - clear(); - } + std::bitset bitSet; /** @brief BitArray for all bits */ + std::bitset curBitset; /** @brief Current working column */ + std::bitset curMask; /** @brief Current bitset mask */ + unsigned char keyBuf[IDX_MAX_MULTI_COL_BIT / 8]; /** @brief Key buffer */ + int curLevel; /** @brief Current index level */ + int maxLevel; /** @brief Maximum index level */ + int totalBit; /** @brief Total bits */ + int testbitArray[IDX_MAX_MULTI_COL_IDX_LEVEL]; /** @brief Test bit array */ + void clear() + { + bitSet.reset(); + curBitset.reset(); + curMask.reset(); + curLevel = maxLevel = 0; + totalBit = 0; + memset(testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL * sizeof(testbitArray[0])); + memset(keyBuf, 0, IDX_MAX_MULTI_COL_BIT / 8); + curMask = 0x1F; + curMask = curMask << (IDX_MAX_MULTI_COL_BIT - 5); + } + IdxMultiColKey() + { + clear(); + } }; struct IdxMultiRid { - RID* ridArray; /** @brief RID array */ - int totalRid; /** @brief Total number of row id */ - IdxMultiRid() - { - totalRid = 0; - ridArray = NULL; - } - void setMultiRid( RID* rids, const int size ) - { - totalRid = size; - ridArray = rids; - /* ridArray = new RID[size]; - memcpy( ridArray, rids, size * sizeof( RID ) ); */ - } - void clearMultiRid() { /*if( ridArray != NULL ) delete [] ridArray; ridArray = NULL;*/ } // we don't want to get into this mem business + RID* ridArray; /** @brief RID array */ + int totalRid; /** @brief Total number of row id */ + IdxMultiRid() + { + totalRid = 0; + ridArray = NULL; + } + void setMultiRid(RID* rids, const int size) + { + totalRid = size; + ridArray = rids; + /* ridArray = new RID[size]; + memcpy( ridArray, rids, size * sizeof( RID ) ); */ + } + void clearMultiRid() + { /*if( ridArray != NULL ) delete [] ridArray; ridArray = NULL;*/ + } // we don't want to get into this mem business }; struct IdxLoadParam { - File sourceFile; /** @brief Source file contatin values */ + File sourceFile; /** @brief Source file contatin values */ - OID indexTreeOid; /** @brief Target index tree oid */ - OID indexListOid; /** @brief Target index list oid */ - execplan::CalpontSystemCatalog::ColDataType indexColDataType; /** @brief Target index column type */ - int indexWidth; /** @brief Target index width */ + OID indexTreeOid; /** @brief Target index tree oid */ + OID indexListOid; /** @brief Target index list oid */ + execplan::CalpontSystemCatalog::ColDataType indexColDataType; /** @brief Target index column type */ + int indexWidth; /** @brief Target index width */ - int maxLoadRow; /** @brief Max rows for one load */ + int maxLoadRow; /** @brief Max rows for one load */ - void setIdxLoadParam( const OID treeOid, const OID listOid, const execplan::CalpontSystemCatalog::ColDataType colDataType, const int width, const int maxRow ) - { - indexTreeOid = treeOid; - indexListOid = listOid; - indexColDataType = colDataType; - indexWidth = width; - maxLoadRow = maxRow; - } - bool isValid() - { - return indexTreeOid && indexListOid && indexWidth && maxLoadRow; - } - IdxLoadParam() - { - indexTreeOid = indexListOid = indexWidth = maxLoadRow = 0; - } + void setIdxLoadParam(const OID treeOid, const OID listOid, + const execplan::CalpontSystemCatalog::ColDataType colDataType, const int width, + const int maxRow) + { + indexTreeOid = treeOid; + indexListOid = listOid; + indexColDataType = colDataType; + indexWidth = width; + maxLoadRow = maxRow; + } + bool isValid() + { + return indexTreeOid && indexListOid && indexWidth && maxLoadRow; + } + IdxLoadParam() + { + indexTreeOid = indexListOid = indexWidth = maxLoadRow = 0; + } }; -} //end of namespace -#endif // _WE_INDEX_H_ +} // namespace WriteEngine +#endif // _WE_INDEX_H_ diff --git a/writeengine/shared/we_log.cpp b/writeengine/shared/we_log.cpp index 9ee2c2c90..1fe6f8210 100644 --- a/writeengine/shared/we_log.cpp +++ b/writeengine/shared/we_log.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_log.cpp 4504 2013-02-02 00:07:43Z bpaul $ -* -*******************************************************************************/ + * $Id: we_log.cpp 4504 2013-02-02 00:07:43Z bpaul $ + * + *******************************************************************************/ #include "we_log.h" @@ -29,16 +29,14 @@ namespace WriteEngine { -WriteEngine::WErrorCodes ec; // referenced as extern by chunkmanager +WriteEngine::WErrorCodes ec; // referenced as extern by chunkmanager //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -Log::Log() : m_bConsoleOutput( true ), - m_logFileName( "" ), - m_errlogFileName( "" ) +Log::Log() : m_bConsoleOutput(true), m_logFileName(""), m_errlogFileName("") { - m_pid = ::getpid(); + m_pid = ::getpid(); } //------------------------------------------------------------------------------ @@ -46,8 +44,8 @@ Log::Log() : m_bConsoleOutput( true ), //------------------------------------------------------------------------------ Log::~Log() { - m_logFile.close(); - m_errLogFile.close(); + m_logFile.close(); + m_errLogFile.close(); } //------------------------------------------------------------------------------ @@ -61,35 +59,31 @@ Log::~Log() // RETURN: // none //------------------------------------------------------------------------------ -void Log::formatMsg( const std::string& msg, - MsgLevel level, - std::ostringstream& oss, - int code ) const +void Log::formatMsg(const std::string& msg, MsgLevel level, std::ostringstream& oss, int code) const { - // Constructing and logging the entire message as one string, should - // help avoid any thread contention that could cause logging output - // to be interweaved between threads. - oss << Convertor::getTimeStr(); + // Constructing and logging the entire message as one string, should + // help avoid any thread contention that could cause logging output + // to be interweaved between threads. + oss << Convertor::getTimeStr(); - // Include thread id in log message based on debug level - if (isDebug( DEBUG_2 )) - { - oss << " (" << m_pid << ":" << + // Include thread id in log message based on debug level + if (isDebug(DEBUG_2)) + { + oss << " (" << m_pid << ":" << #ifdef _MSC_VER - GetCurrentThreadId() + GetCurrentThreadId() #else - pthread_self() + pthread_self() #endif - << ") " << - MSG_LEVEL_STR[level] << " : " << msg ; - } - else - { - oss << " (" << m_pid << ") " << MSG_LEVEL_STR[level] << " : " << msg ; - } + << ") " << MSG_LEVEL_STR[level] << " : " << msg; + } + else + { + oss << " (" << m_pid << ") " << MSG_LEVEL_STR[level] << " : " << msg; + } - if ( code > 0 ) - oss << " [" << code << "]"; + if (code > 0) + oss << " [" << code << "]"; } //------------------------------------------------------------------------------ @@ -107,46 +101,44 @@ void Log::formatMsg( const std::string& msg, // RETURN: // none //------------------------------------------------------------------------------ -void Log::logMsg( const char* msg, - int code, - MsgLevel level ) +void Log::logMsg(const char* msg, int code, MsgLevel level) { - std::ostringstream oss; - formatMsg( msg, level, oss, code ); + std::ostringstream oss; + formatMsg(msg, level, oss, code); - // log error and critical msgs to syslog - if ( level == MSGLVL_ERROR || level == MSGLVL_CRITICAL ) + // log error and critical msgs to syslog + if (level == MSGLVL_ERROR || level == MSGLVL_CRITICAL) + { { - { - //log to log file and error log file within scope of mutex lock. - //logSyslog uses SimpleSyslog which has it's own lock. - boost::mutex::scoped_lock lk(m_WriteLockMutex); + // log to log file and error log file within scope of mutex lock. + // logSyslog uses SimpleSyslog which has it's own lock. + boost::mutex::scoped_lock lk(m_WriteLockMutex); - m_errLogFile << oss.str() << std::endl; - m_logFile << oss.str() << std::endl; + m_errLogFile << oss.str() << std::endl; + m_logFile << oss.str() << std::endl; - std::cerr << oss.str() << std::endl; - } - - logSyslog( std::string(msg), code ); + std::cerr << oss.str() << std::endl; } - else - { - std::ostringstream oss2; - // Format msg again without including the status code. - // Only log INFO2 msgs to console if m_bConsoleOutput is TRUE; - // All other msg levels always go to console. - if ( (level != MSGLVL_INFO2) || (m_bConsoleOutput) ) - formatMsg ( msg, level, oss2 ); + logSyslog(std::string(msg), code); + } + else + { + std::ostringstream oss2; - boost::mutex::scoped_lock lk(m_WriteLockMutex); + // Format msg again without including the status code. + // Only log INFO2 msgs to console if m_bConsoleOutput is TRUE; + // All other msg levels always go to console. + if ((level != MSGLVL_INFO2) || (m_bConsoleOutput)) + formatMsg(msg, level, oss2); - m_logFile << oss.str() << std::endl; + boost::mutex::scoped_lock lk(m_WriteLockMutex); - if ( (level != MSGLVL_INFO2) || (m_bConsoleOutput) ) - std::cout << oss2.str() << std::endl; - } + m_logFile << oss.str() << std::endl; + + if ((level != MSGLVL_INFO2) || (m_bConsoleOutput)) + std::cout << oss2.str() << std::endl; + } } //------------------------------------------------------------------------------ @@ -158,30 +150,26 @@ void Log::logMsg( const char* msg, // RETURN: // none //------------------------------------------------------------------------------ -void Log::setLogFileName( const char* logfile, - const char* errlogfile, - bool consoleFlag ) +void Log::setLogFileName(const char* logfile, const char* errlogfile, bool consoleFlag) { - m_logFileName = logfile; - m_errlogFileName = errlogfile; - m_bConsoleOutput = consoleFlag; + m_logFileName = logfile; + m_errlogFileName = errlogfile; + m_bConsoleOutput = consoleFlag; #ifdef _MSC_VER - // cpimport.bin calls BulkLoad::loadJobInfo() before calling - // BulkLoad::processJob(). loadJobInfo() attempts to write to this log - // before it's opened (by processJob()). This doesn't seem to bother Linux - // but puts Windows in a bad state. Once this logic is fixed, this hack can - // go away. - // This code probably wouldn't hurt if run on Linux, but I'll leave this - // here as a reminder to fix the logic for all platforms. - m_logFile.close(); - m_logFile.clear(); - m_errLogFile.close(); - m_errLogFile.clear(); + // cpimport.bin calls BulkLoad::loadJobInfo() before calling + // BulkLoad::processJob(). loadJobInfo() attempts to write to this log + // before it's opened (by processJob()). This doesn't seem to bother Linux + // but puts Windows in a bad state. Once this logic is fixed, this hack can + // go away. + // This code probably wouldn't hurt if run on Linux, but I'll leave this + // here as a reminder to fix the logic for all platforms. + m_logFile.close(); + m_logFile.clear(); + m_errLogFile.close(); + m_errLogFile.clear(); #endif - m_logFile.open( m_logFileName.c_str(), - std::ofstream::out | std::ofstream::app ); - m_errLogFile.open(m_errlogFileName.c_str(), - std::ofstream::out | std::ofstream::app); + m_logFile.open(m_logFileName.c_str(), std::ofstream::out | std::ofstream::app); + m_errLogFile.open(m_errlogFileName.c_str(), std::ofstream::out | std::ofstream::app); } //------------------------------------------------------------------------------ @@ -193,38 +181,34 @@ void Log::setLogFileName( const char* logfile, // RETURN: // none //------------------------------------------------------------------------------ -void Log::logSyslog( const std::string& msg, - int statusCode ) +void Log::logSyslog(const std::string& msg, int statusCode) { - logging::Message::MessageID msgId = logging::M0087; + logging::Message::MessageID msgId = logging::M0087; - switch (statusCode) + switch (statusCode) + { + case ERR_FILE_DISK_SPACE: { - case ERR_FILE_DISK_SPACE: - { - msgId = logging::M0076; - break; - } - - case ERR_UNKNOWN: - { - msgId = logging::M0017; - break; - } - - default: - { - msgId = logging::M0087; - break; - } + msgId = logging::M0076; + break; } - logging::Message::Args errMsgArgs; - errMsgArgs.add( msg ); - SimpleSysLog::instance()->logMsg( - errMsgArgs, - logging::LOG_TYPE_ERROR, - msgId); + case ERR_UNKNOWN: + { + msgId = logging::M0017; + break; + } + + default: + { + msgId = logging::M0087; + break; + } + } + + logging::Message::Args errMsgArgs; + errMsgArgs.add(msg); + SimpleSysLog::instance()->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, msgId); } //------------------------------------------------------------------------------ // DESCRIPTION: @@ -239,9 +223,8 @@ void Log::logSyslog( const std::string& msg, void Log::closeLog() { - m_logFile.close(); - m_errLogFile.close(); + m_logFile.close(); + m_errLogFile.close(); } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_log.h b/writeengine/shared/we_log.h index bc226789d..a60c35816 100644 --- a/writeengine/shared/we_log.h +++ b/writeengine/shared/we_log.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_log.h 4504 2013-02-02 00:07:43Z bpaul $ -* -*******************************************************************************/ + * $Id: we_log.h 4504 2013-02-02 00:07:43Z bpaul $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_LOG_H_ @@ -45,84 +45,70 @@ /** Namespace WriteEngine */ namespace WriteEngine { -const std::string MSG_LEVEL_STR[] = -{ - "INFO", - "INFO2", - "WARN", - "ERR ", - "CRIT" -}; +const std::string MSG_LEVEL_STR[] = {"INFO", "INFO2", "WARN", "ERR ", "CRIT"}; /** @brief Class is used to format and write log messages to cpimport.bin log * file. When applicable, messages are also logged to syslog logs as well. */ class Log : public WEObj { -public: - /** - * @brief Constructor - */ - EXPORT Log(); + public: + /** + * @brief Constructor + */ + EXPORT Log(); - /** - * @brief Destructor - */ - EXPORT ~Log(); + /** + * @brief Destructor + */ + EXPORT ~Log(); - /** - * @brief Log a cpimport.bin logfile message; logs errors to syslog as well - */ - EXPORT void logMsg( const char* msg, int code, MsgLevel level ); - EXPORT void logMsg( const char* msg, MsgLevel level ) - { - logMsg( msg, 0, level ); - } - EXPORT void logMsg( const std::string& msg, MsgLevel level ) - { - logMsg( msg.c_str(), level ); - } - EXPORT void logMsg( const std::string& msg, int code, MsgLevel level ) - { - logMsg( msg.c_str(), code, level ); - } + /** + * @brief Log a cpimport.bin logfile message; logs errors to syslog as well + */ + EXPORT void logMsg(const char* msg, int code, MsgLevel level); + EXPORT void logMsg(const char* msg, MsgLevel level) + { + logMsg(msg, 0, level); + } + EXPORT void logMsg(const std::string& msg, MsgLevel level) + { + logMsg(msg.c_str(), level); + } + EXPORT void logMsg(const std::string& msg, int code, MsgLevel level) + { + logMsg(msg.c_str(), code, level); + } - /** - * @brief Set log file name - */ - EXPORT void setLogFileName( const char* logfile, - const char* errlogfile, - bool consoleFlag = true ); + /** + * @brief Set log file name + */ + EXPORT void setLogFileName(const char* logfile, const char* errlogfile, bool consoleFlag = true); - // BUG 5022 - /** - * @brief Set log files close other than calling d'tor - */ - EXPORT void closeLog(); + // BUG 5022 + /** + * @brief Set log files close other than calling d'tor + */ + EXPORT void closeLog(); + private: + void logSyslog(const std::string& msg, int statusCode); + void formatMsg(const std::string& msg, MsgLevel level, std::ostringstream& oss, int code = 0) const; -private: - void logSyslog ( const std::string& msg, - int statusCode); - void formatMsg( const std::string& msg, - MsgLevel level, - std::ostringstream& oss, - int code = 0 ) const; + bool m_bConsoleOutput; // flag allowing INFO2 msg + // to display to console + std::string m_logFileName; // log file name + std::string m_errlogFileName; // error log file name + pid_t m_pid; // current pid - bool m_bConsoleOutput; // flag allowing INFO2 msg - // to display to console - std::string m_logFileName; // log file name - std::string m_errlogFileName; // error log file name - pid_t m_pid; // current pid + std::ofstream m_logFile; // log file stream + std::ofstream m_errLogFile; // error log file stream - std::ofstream m_logFile; // log file stream - std::ofstream m_errLogFile; // error log file stream - - boost::mutex m_WriteLockMutex; // logging mutex + boost::mutex m_WriteLockMutex; // logging mutex }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_LOG_H_ +#endif // _WE_LOG_H_ diff --git a/writeengine/shared/we_macro.h b/writeengine/shared/we_macro.h index da85b5dec..31f73a97f 100644 --- a/writeengine/shared/we_macro.h +++ b/writeengine/shared/we_macro.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_macro.h 33 2006-10-18 14:37:12Z wzhou $ -* -******************************************************************************************/ + * $Id: we_macro.h 33 2006-10-18 14:37:12Z wzhou $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_MACRO_H_ @@ -26,28 +26,29 @@ #include - /** Namespace WriteEngine */ namespace WriteEngine { -#define RETURN_ON_ERROR( statement ) \ - { int rcVal = (statement); \ - if( rcVal != NO_ERROR ) \ - return rcVal; } +#define RETURN_ON_ERROR(statement) \ + { \ + int rcVal = (statement); \ + if (rcVal != NO_ERROR) \ + return rcVal; \ + } -#define RETURN_ON_NULL( obj, rc ) \ - if( obj == NULL ) \ - return rc; +#define RETURN_ON_NULL(obj, rc) \ + if (obj == NULL) \ + return rc; -#define RETURN_ON_WE_ERROR( oldRc, newRc ) \ - if( oldRc != NO_ERROR ) \ - return newRc; +#define RETURN_ON_WE_ERROR(oldRc, newRc) \ + if (oldRc != NO_ERROR) \ + return newRc; -#define RETURN_RC( oldRc, newRc ) \ - if( oldRc != NO_ERROR ) \ - return newRc; \ - else \ - return NO_ERROR; +#define RETURN_RC(oldRc, newRc) \ + if (oldRc != NO_ERROR) \ + return newRc; \ + else \ + return NO_ERROR; -} //end of namespace -#endif // _WE_MACRO_H_ +} // namespace WriteEngine +#endif // _WE_MACRO_H_ diff --git a/writeengine/shared/we_obj.h b/writeengine/shared/we_obj.h index 52019d23b..e4aa35d7d 100644 --- a/writeengine/shared/we_obj.h +++ b/writeengine/shared/we_obj.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /****************************************************************************************** -* $Id: we_obj.h 33 2006-08-17 10:31:20Z wzhou $ -* -******************************************************************************************/ + * $Id: we_obj.h 33 2006-08-17 10:31:20Z wzhou $ + * + ******************************************************************************************/ /** @file */ #ifndef _WE_OBJ_H_ @@ -35,62 +35,65 @@ class Log; /** Class WEObj */ class WEObj { -public: - /** - * @brief Constructor - */ - WEObj() : m_debugLevel( DEBUG_0 ), m_log( 0 ) {} + public: + /** + * @brief Constructor + */ + WEObj() : m_debugLevel(DEBUG_0), m_log(0) + { + } - /** - * @brief Default Destructor - */ - ~WEObj() {} + /** + * @brief Default Destructor + */ + ~WEObj() + { + } - /** - * @brief Is it required to debug - */ - bool isDebug( const DebugLevel level ) const - { - return level <= m_debugLevel; - } + /** + * @brief Is it required to debug + */ + bool isDebug(const DebugLevel level) const + { + return level <= m_debugLevel; + } - /** - * @brief Get debug level - */ - DebugLevel getDebugLevel() const - { - return m_debugLevel; - } + /** + * @brief Get debug level + */ + DebugLevel getDebugLevel() const + { + return m_debugLevel; + } - /** - * @brief Get Logger object - */ - Log* getLogger() const - { - return m_log; - } + /** + * @brief Get Logger object + */ + Log* getLogger() const + { + return m_log; + } - /** - * @brief Set debug level - */ - void setDebugLevel( const DebugLevel level ) - { - m_debugLevel = level; - } + /** + * @brief Set debug level + */ + void setDebugLevel(const DebugLevel level) + { + m_debugLevel = level; + } - /** - * @brief Set debug logger and debug level - */ - void setLogger( Log* logger ) - { - m_log = logger; - } + /** + * @brief Set debug logger and debug level + */ + void setLogger(Log* logger) + { + m_log = logger; + } -private: - DebugLevel m_debugLevel; // internal use debug level - Log* m_log; // logger object for debug output + private: + DebugLevel m_debugLevel; // internal use debug level + Log* m_log; // logger object for debug output }; - -} //end of namespace -#endif // _WE_OBJ_H_ +} // namespace WriteEngine +#endif // _WE_OBJ_H_ diff --git a/writeengine/shared/we_rbmetawriter.cpp b/writeengine/shared/we_rbmetawriter.cpp index 8cc2bd7a1..98bf2cdce 100644 --- a/writeengine/shared/we_rbmetawriter.cpp +++ b/writeengine/shared/we_rbmetawriter.cpp @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_rbmetawriter.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -*/ + * $Id: we_rbmetawriter.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ + */ #include "we_rbmetawriter.h" @@ -46,109 +46,100 @@ namespace const char* DATA_DIR_SUFFIX = "_data"; const char* TMP_FILE_SUFFIX = ".tmp"; -const char* VERSION3_REC = "# VERSION: 3"; -const int VERSION3_REC_LEN = 12; -const char* VERSION4_REC = "# VERSION: 4"; -const int VERSION4_REC_LEN = 12; -const char* COLUMN1_REC = "COLUM1"; // HWM extent for a DBRoot -const int COLUMN1_REC_LEN = 6; -const char* COLUMN2_REC = "COLUM2"; // Placeholder for empty DBRoot -const int COLUMN2_REC_LEN = 6; -const char* DSTORE1_REC = "DSTOR1"; // HWM extent for a DBRoot -const int DSTORE1_REC_LEN = 6; -const char* DSTORE2_REC = "DSTOR2"; // Placeholder for empty DBRoot -const int DSTORE2_REC_LEN = 6; +const char* VERSION3_REC = "# VERSION: 3"; +const int VERSION3_REC_LEN = 12; +const char* VERSION4_REC = "# VERSION: 4"; +const int VERSION4_REC_LEN = 12; +const char* COLUMN1_REC = "COLUM1"; // HWM extent for a DBRoot +const int COLUMN1_REC_LEN = 6; +const char* COLUMN2_REC = "COLUM2"; // Placeholder for empty DBRoot +const int COLUMN2_REC_LEN = 6; +const char* DSTORE1_REC = "DSTOR1"; // HWM extent for a DBRoot +const int DSTORE1_REC_LEN = 6; +const char* DSTORE2_REC = "DSTOR2"; // Placeholder for empty DBRoot +const int DSTORE2_REC_LEN = 6; //-------------------------------------------------------------------------- // Local Function that prints contents of an RBChunkInfo object //-------------------------------------------------------------------------- -std::ostream& operator<<(std::ostream& os, - const WriteEngine::RBChunkInfo& chk) +std::ostream& operator<<(std::ostream& os, const WriteEngine::RBChunkInfo& chk) { - os << "OID-" << chk.fOid << - "; DBRoot-" << chk.fDbRoot << - "; Part-" << chk.fPartition << - "; Seg-" << chk.fSegment << - "; HWM-" << chk.fHwm; + os << "OID-" << chk.fOid << "; DBRoot-" << chk.fDbRoot << "; Part-" << chk.fPartition << "; Seg-" + << chk.fSegment << "; HWM-" << chk.fHwm; - return os; -} + return os; } +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // Compare function used for set of RBChunkInfo objects. //------------------------------------------------------------------------------ -bool RBChunkInfoCompare::operator() -(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const +bool RBChunkInfoCompare::operator()(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const { - if (lhs.fOid < rhs.fOid) - { - return true; - } + if (lhs.fOid < rhs.fOid) + { + return true; + } - if ((lhs.fOid == rhs.fOid) && (lhs.fSegment < rhs.fSegment)) - { - return true; - } + if ((lhs.fOid == rhs.fOid) && (lhs.fSegment < rhs.fSegment)) + { + return true; + } - return false; + return false; } - + //------------------------------------------------------------------------------ // RBMetaWriter constructor //------------------------------------------------------------------------------ -RBMetaWriter::RBMetaWriter ( - const std::string& appDesc, - Log* logger ) : fMetaDataFile(NULL), fAppDesc(appDesc), fLog(logger), fCreatedSubDir(false) +RBMetaWriter::RBMetaWriter(const std::string& appDesc, Log* logger) + : fMetaDataFile(NULL), fAppDesc(appDesc), fLog(logger), fCreatedSubDir(false) { } - + //------------------------------------------------------------------------------ // Initialize this meta data file object using the specified table OID and name. // We assume the application code calling this function, was able to acquire a // table lock, meaning if there should happen to be any leftoever metadata files // from a previous job, they can be deleted. //------------------------------------------------------------------------------ -void RBMetaWriter::init ( - OID tableOID, - const std::string& tableName ) +void RBMetaWriter::init(OID tableOID, const std::string& tableName) { - fTableOID = tableOID; - fTableName = tableName; + fTableOID = tableOID; + fTableName = tableName; - std::vector dbRoots; - Config::getRootIdList( dbRoots ); + std::vector dbRoots; + Config::getRootIdList(dbRoots); - std::string metaFileName; - std::ostringstream oss; - oss << "/" << fTableOID; + std::string metaFileName; + std::ostringstream oss; + oss << "/" << fTableOID; - // Delete any files that collide with the file names we are going to need. - // Construct the filenames; we will use a temporary file name until we are - // finished creating, at which time we will rename the temp files. - for (unsigned m = 0; m < dbRoots.size(); m++) - { - std::string bulkRollbackPath( Config::getDBRootByNum( dbRoots[m] ) ); - bulkRollbackPath += '/'; - bulkRollbackPath += DBROOT_BULK_ROLLBACK_SUBDIR; - metaFileName = bulkRollbackPath; - metaFileName += oss.str(); + // Delete any files that collide with the file names we are going to need. + // Construct the filenames; we will use a temporary file name until we are + // finished creating, at which time we will rename the temp files. + for (unsigned m = 0; m < dbRoots.size(); m++) + { + std::string bulkRollbackPath(Config::getDBRootByNum(dbRoots[m])); + bulkRollbackPath += '/'; + bulkRollbackPath += DBROOT_BULK_ROLLBACK_SUBDIR; + metaFileName = bulkRollbackPath; + metaFileName += oss.str(); - std::string tmpMetaFileName = metaFileName; - tmpMetaFileName += TMP_FILE_SUFFIX; + std::string tmpMetaFileName = metaFileName; + tmpMetaFileName += TMP_FILE_SUFFIX; - // Delete any files that collide with the filenames we intend to use - IDBPolicy::remove( metaFileName.c_str() ); - IDBPolicy::remove( tmpMetaFileName.c_str() ); + // Delete any files that collide with the filenames we intend to use + IDBPolicy::remove(metaFileName.c_str()); + IDBPolicy::remove(tmpMetaFileName.c_str()); - // Clear out any data subdirectory - deleteSubDir( metaFileName ); - } + // Clear out any data subdirectory + deleteSubDir(metaFileName); + } } - + //------------------------------------------------------------------------------ // Saves snapshot of extentmap into a bulk rollback meta data file, for // use in a bulk rollback. Function was closely modeled after function @@ -162,373 +153,331 @@ void RBMetaWriter::init ( // dbRootHWMInfoVecCol - vector of last local HWM info for each DBRoot // (asssigned to current PM) for each column in tblOid. //------------------------------------------------------------------------------ -void RBMetaWriter::saveBulkRollbackMetaData( - const std::vector& columns, - const std::vector& dctnryStoreOids, - const std::vector& dbRootHWMInfoVecCol ) +void RBMetaWriter::saveBulkRollbackMetaData(const std::vector& columns, + const std::vector& dctnryStoreOids, + const std::vector& dbRootHWMInfoVecCol) { - int rc = NO_ERROR; - bool bOpenedFile = false; + int rc = NO_ERROR; + bool bOpenedFile = false; + try + { + std::vector dbRoots; + Config::getRootIdList(dbRoots); + + // Loop through DBRoot HWMs for this PM + for (unsigned m = 0; m < dbRoots.size(); m++) + { + std::string metaFileName = openMetaFile(dbRoots[m]); + bOpenedFile = true; + fCreatedSubDir = false; + + // Loop through the columns in the specified table + for (size_t i = 0; i < columns.size(); i++) + { + const BRM::EmDbRootHWMInfo_v& dbRootHWMInfo = dbRootHWMInfoVecCol[i]; + + // Select dbRootHWMInfo that matches DBRoot for this iteration + unsigned k = 0; + + for (; k < dbRootHWMInfo.size(); k++) + { + if (dbRoots[m] == dbRootHWMInfo[k].dbRoot) + break; + } + + if (k >= dbRootHWMInfo.size()) // logic error; should not happen + { + std::ostringstream oss; + oss << "Error creating meta file; DBRoot" << dbRoots[m] + << " listed in Calpont config file, but not in extentmap" + " for OID " + << columns[i].dataFile.oid; + throw WeException(oss.str(), ERR_INVALID_PARAM); + } + + uint16_t dbRoot = dbRootHWMInfo[k].dbRoot; + uint32_t partition = 0; + uint16_t segment = 0; + HWM localHWM = 0; + bool bExtentWithData = false; + + // For empty DBRoot (totalBlocks == 0), + // leave partition, segment, and HWM set to 0 + if ((dbRootHWMInfo[k].totalBlocks > 0) || (dbRootHWMInfo[k].status == BRM::EXTENTOUTOFSERVICE)) + { + partition = dbRootHWMInfo[k].partitionNum; + segment = dbRootHWMInfo[k].segmentNum; + localHWM = dbRootHWMInfo[k].localHWM; + bExtentWithData = true; + } + + // Save column meta-data info to support bulk rollback + writeColumnMetaData(metaFileName, bExtentWithData, columns[i].dataFile.oid, dbRoot, partition, + segment, localHWM, columns[i].colDataType, ColDataTypeStr[columns[i].colDataType], + columns[i].colWidth, columns[i].compressionType); + + // Save dctnry store meta-data info to support bulk rollback + if (dctnryStoreOids[i] > 0) + { + std::vector segList; + std::string segFileListErrMsg; + + if (bExtentWithData) + { + std::string dirName; + FileOp fileOp(false); + rc = fileOp.getDirName(dctnryStoreOids[i], dbRoot, partition, dirName); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error constructing path " + "for dictionary " + << dctnryStoreOids[i] << "; dbRoot-" << dbRoot << "; partition-" << partition << "; " + << ec.errorString(rc); + + throw WeException(oss.str(), rc); + } + + rc = BulkRollbackMgr::getSegFileList(dirName, false, segList, segFileListErrMsg); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Bulk rollback error for dictionary " << dctnryStoreOids[i] << "; directory-" << dirName + << "; " << segFileListErrMsg << "; " << ec.errorString(rc); + + throw WeException(oss.str(), rc); + } + } // end of "if (bExtentWithData)" + + if (segList.size() == 0) + { + writeDictionaryStoreMetaNoDataMarker(columns[i].dataFile.oid, dctnryStoreOids[i], dbRoot, + partition, + 0, // segment + columns[i].compressionType); + } + else + { + // Loop thru dictionary store seg files for this DBRoot + for (unsigned int kk = 0; kk < segList.size(); kk++) + { + unsigned int segDictionary = segList[kk]; + + // check HWM for dictionary store file + HWM dictHWMStore; + int extState; + rc = BRMWrapper::getInstance()->getLocalHWM(dctnryStoreOids[i], partition, segDictionary, + dictHWMStore, extState); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Error getting rollback HWM for " + "dictionary file " + << dctnryStoreOids[i] << "; partition-" << partition << "; segment-" << segDictionary + << "; " << ec.errorString(rc); + throw WeException(oss.str(), rc); + } + + writeDictionaryStoreMetaData(columns[i].dataFile.oid, dctnryStoreOids[i], dbRoot, partition, + segDictionary, dictHWMStore, columns[i].compressionType); + + } // loop thru dictionary store seg files in this DBRoot + } // dictionary OID has 1 or more seg files in partition + } // if dictionary column + + // For a compressed column, backup the starting HWM chunk if the + // starting HWM block is not on an empty DBRoot (or outOfSrvc) + if ((columns[i].compressionType) && (columns[i].dataFile.fDbRoot == dbRootHWMInfo[k].dbRoot) && + (dbRootHWMInfo[k].totalBlocks > 0) && (dbRootHWMInfo[k].status != BRM::EXTENTOUTOFSERVICE)) + { + backupColumnHWMChunk(columns[i].dataFile.oid, columns[i].dataFile.fDbRoot, + columns[i].dataFile.fPartition, columns[i].dataFile.fSegment, + columns[i].dataFile.hwm); + } + + } // End of loop through columns + + // time to dump the string stream to file + std::string data(fMetaDataStream.str()); + + // this is to cover partical writes + // no need for retry if low layer takes care partial writes. + const char* p = data.c_str(); // buffer contents + size_t s = data.size(); // buffer size + size_t w = 0; // total bytes written so far + ssize_t n = 0; // bytes written in one write + + for (int i = 0; i < 10 && w < s; i++) + { + n = fMetaDataFile->write(p + w, s - w); + + if (n < 0) + break; + + w += n; + } + + if (w != s) + { + int errRc = errno; + std::ostringstream oss; + oss << "Error writing bulk rollback meta-data file " << metaFileName << "; written/expect:" << w + << "/" << s << "; err-" << errRc << "; " << strerror(errRc); + throw WeException(oss.str(), ERR_FILE_WRITE); + } + + fMetaDataStream.str(""); + closeMetaFile(); + bOpenedFile = false; + + } // End of loop through DBRoot HWMs for this PM + + renameMetaFile(); // rename meta files from temporary filenames + } + catch (WeException& ex) // catch exception to close file, then rethrow + { + if (bOpenedFile) + closeMetaFile(); + + // If any error occurred, then go back and try to delete all meta files. + // We catch and drop any exception, and return the original exception, + // since we are already in error-mode at this point. try { - std::vector dbRoots; - Config::getRootIdList( dbRoots ); - - // Loop through DBRoot HWMs for this PM - for (unsigned m = 0; m < dbRoots.size(); m++) - { - std::string metaFileName = openMetaFile( dbRoots[m] ); - bOpenedFile = true; - fCreatedSubDir = false; - - // Loop through the columns in the specified table - for ( size_t i = 0; i < columns.size(); i++ ) - { - const BRM::EmDbRootHWMInfo_v& dbRootHWMInfo = - dbRootHWMInfoVecCol[i]; - - // Select dbRootHWMInfo that matches DBRoot for this iteration - unsigned k = 0; - - for (; k < dbRootHWMInfo.size(); k++) - { - if (dbRoots[m] == dbRootHWMInfo[k].dbRoot) - break; - } - - if (k >= dbRootHWMInfo.size()) // logic error; should not happen - { - std::ostringstream oss; - oss << "Error creating meta file; DBRoot" << dbRoots[m] << - " listed in Calpont config file, but not in extentmap" - " for OID " << columns[i].dataFile.oid; - throw WeException( oss.str(), ERR_INVALID_PARAM ); - } - - uint16_t dbRoot = dbRootHWMInfo[k].dbRoot; - uint32_t partition = 0; - uint16_t segment = 0; - HWM localHWM = 0; - bool bExtentWithData = false; - - // For empty DBRoot (totalBlocks == 0), - // leave partition, segment, and HWM set to 0 - if ((dbRootHWMInfo[k].totalBlocks > 0) || - (dbRootHWMInfo[k].status == BRM::EXTENTOUTOFSERVICE)) - { - partition = dbRootHWMInfo[k].partitionNum; - segment = dbRootHWMInfo[k].segmentNum; - localHWM = dbRootHWMInfo[k].localHWM; - bExtentWithData = true; - } - - // Save column meta-data info to support bulk rollback - writeColumnMetaData( - metaFileName, - bExtentWithData, - columns[i].dataFile.oid, - dbRoot, - partition, - segment, - localHWM, - columns[i].colDataType, - ColDataTypeStr[ columns[i].colDataType ], - columns[i].colWidth, - columns[i].compressionType ); - - // Save dctnry store meta-data info to support bulk rollback - if ( dctnryStoreOids[i] > 0 ) - { - std::vector segList; - std::string segFileListErrMsg; - - if (bExtentWithData) - { - std::string dirName; - FileOp fileOp(false); - rc = fileOp.getDirName( dctnryStoreOids[i], - dbRoot, partition, dirName ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error constructing path " - "for dictionary " << dctnryStoreOids[i] << - "; dbRoot-" << dbRoot << - "; partition-" << partition << - "; " << ec.errorString(rc); - - throw WeException( oss.str(), rc ); - } - - rc = BulkRollbackMgr::getSegFileList(dirName, false, - segList, - segFileListErrMsg); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Bulk rollback error for dictionary " << - dctnryStoreOids[i] << - "; directory-" << dirName << - "; " << segFileListErrMsg << - "; " << ec.errorString(rc); - - throw WeException( oss.str(), rc ); - } - } // end of "if (bExtentWithData)" - - if (segList.size() == 0) - { - writeDictionaryStoreMetaNoDataMarker( - columns[i].dataFile.oid, - dctnryStoreOids[i], - dbRoot, - partition, - 0, // segment - columns[i].compressionType ); - } - else - { - // Loop thru dictionary store seg files for this DBRoot - for (unsigned int kk = 0; kk < segList.size(); kk++) - { - unsigned int segDictionary = segList[kk]; - - // check HWM for dictionary store file - HWM dictHWMStore; - int extState; - rc = BRMWrapper::getInstance()->getLocalHWM( - dctnryStoreOids[i], - partition, - segDictionary, - dictHWMStore, - extState ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Error getting rollback HWM for " - "dictionary file " << dctnryStoreOids[i] << - "; partition-" << partition << - "; segment-" << segDictionary << - "; " << ec.errorString(rc); - throw WeException( oss.str(), rc ); - } - - writeDictionaryStoreMetaData( - columns[i].dataFile.oid, - dctnryStoreOids[i], - dbRoot, - partition, - segDictionary, - dictHWMStore, - columns[i].compressionType ); - - } // loop thru dictionary store seg files in this DBRoot - } // dictionary OID has 1 or more seg files in partition - } // if dictionary column - - // For a compressed column, backup the starting HWM chunk if the - // starting HWM block is not on an empty DBRoot (or outOfSrvc) - if ( (columns[i].compressionType) && - (columns[i].dataFile.fDbRoot == dbRootHWMInfo[k].dbRoot) && - (dbRootHWMInfo[k].totalBlocks > 0) && - (dbRootHWMInfo[k].status != BRM::EXTENTOUTOFSERVICE) ) - { - backupColumnHWMChunk( - columns[i].dataFile.oid, - columns[i].dataFile.fDbRoot, - columns[i].dataFile.fPartition, - columns[i].dataFile.fSegment, - columns[i].dataFile.hwm ); - } - - } // End of loop through columns - - // time to dump the string stream to file - std::string data(fMetaDataStream.str()); - - // this is to cover partical writes - // no need for retry if low layer takes care partial writes. - const char* p = data.c_str(); // buffer contents - size_t s = data.size(); // buffer size - size_t w = 0; // total bytes written so far - ssize_t n = 0; // bytes written in one write - - for (int i = 0; i < 10 && w < s; i++) - { - n = fMetaDataFile->write(p + w, s - w); - - if (n < 0) - break; - - w += n; - } - - if (w != s) - { - int errRc = errno; - std::ostringstream oss; - oss << "Error writing bulk rollback meta-data file " - << metaFileName << "; written/expect:" << w << "/" << s - << "; err-" << errRc << "; " << strerror( errRc ); - throw WeException(oss.str(), ERR_FILE_WRITE); - } - - fMetaDataStream.str(""); - closeMetaFile( ); - bOpenedFile = false; - - } // End of loop through DBRoot HWMs for this PM - - renameMetaFile( ); // rename meta files from temporary filenames + deleteFile(); } - catch (WeException& ex) // catch exception to close file, then rethrow + catch (...) { - if (bOpenedFile) - closeMetaFile( ); - - // If any error occurred, then go back and try to delete all meta files. - // We catch and drop any exception, and return the original exception, - // since we are already in error-mode at this point. - try - { - deleteFile( ); - } - catch (...) - { - } - - throw WeException( ex.what(), ex.errorCode() ); } + + throw WeException(ex.what(), ex.errorCode()); + } } - + //------------------------------------------------------------------------------ // Open a meta data file to save info about the specified table OID. //------------------------------------------------------------------------------ -std::string RBMetaWriter::openMetaFile ( uint16_t dbRoot ) +std::string RBMetaWriter::openMetaFile(uint16_t dbRoot) { - std::string bulkRollbackPath( Config::getDBRootByNum( dbRoot ) ); - bulkRollbackPath += '/'; - bulkRollbackPath += DBROOT_BULK_ROLLBACK_SUBDIR; + std::string bulkRollbackPath(Config::getDBRootByNum(dbRoot)); + bulkRollbackPath += '/'; + bulkRollbackPath += DBROOT_BULK_ROLLBACK_SUBDIR; - if ( !IDBPolicy::exists( bulkRollbackPath.c_str() ) ) + if (!IDBPolicy::exists(bulkRollbackPath.c_str())) + { + if (IDBPolicy::mkdir(bulkRollbackPath.c_str()) != 0) { - if ( IDBPolicy::mkdir( bulkRollbackPath.c_str() ) != 0 ) - { - std::ostringstream oss; - oss << "Error creating bulk rollback directory " << - bulkRollbackPath << ";" << std::endl; - throw WeException( oss.str(), ERR_DIR_CREATE ); - } + std::ostringstream oss; + oss << "Error creating bulk rollback directory " << bulkRollbackPath << ";" << std::endl; + throw WeException(oss.str(), ERR_DIR_CREATE); } + } - // Open the file + // Open the file + std::ostringstream oss; + oss << "/" << fTableOID; + std::string metaFileName(bulkRollbackPath); + metaFileName += oss.str(); + fMetaFileNames.insert(make_pair(dbRoot, metaFileName)); + + std::string tmpMetaFileName(metaFileName); + tmpMetaFileName += TMP_FILE_SUFFIX; + fMetaDataFile = IDBDataFile::open(IDBPolicy::getType(tmpMetaFileName.c_str(), IDBPolicy::WRITEENG), + tmpMetaFileName.c_str(), "wb", 0); + + if (!fMetaDataFile) + { + int errRc = errno; std::ostringstream oss; - oss << "/" << fTableOID; - std::string metaFileName( bulkRollbackPath ); - metaFileName += oss.str(); - fMetaFileNames.insert( make_pair(dbRoot, metaFileName) ); + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << "Error opening bulk rollback file " << tmpMetaFileName << "; " << eMsg; + throw WeException(oss.str(), ERR_FILE_OPEN); + } - std::string tmpMetaFileName( metaFileName ); - tmpMetaFileName += TMP_FILE_SUFFIX; - fMetaDataFile = IDBDataFile::open(IDBPolicy::getType(tmpMetaFileName.c_str(), - IDBPolicy::WRITEENG), - tmpMetaFileName.c_str(), "wb", 0); - - if ( !fMetaDataFile ) + { + std::ostringstream ossChown; + idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(tmpMetaFileName.c_str()); + if (chownPath(ossChown, tmpMetaFileName, fs) || chownPath(ossChown, bulkRollbackPath, fs)) { - int errRc = errno; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << "Error opening bulk rollback file " << - tmpMetaFileName << "; " << eMsg; - throw WeException( oss.str(), ERR_FILE_OPEN ); + throw WeException(ossChown.str(), ERR_FILE_CHOWN); } + } - { - std::ostringstream ossChown; - idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(tmpMetaFileName.c_str()); - if (chownPath(ossChown, tmpMetaFileName, fs) - || chownPath(ossChown, bulkRollbackPath, fs)) - { - throw WeException(ossChown.str(), ERR_FILE_CHOWN); - } - } + fMetaDataStream << "# VERSION: 4" << std::endl + << "# APPLICATION: " << fAppDesc << std::endl + << "# PID: " << ::getpid() << std::endl + << "# TABLE: " << fTableName << std::endl + << "# COLUM1: coloid," + "dbroot,part,seg,lastLocalHWM,type,typename,width,comp" + << std::endl + << "# COLUM2: coloid," + "dbroot,part,seg,type,typename,width,comp" + << std::endl + << "# DSTOR1: coloid,dctoid," + "dbroot,part,seg,localHWM,comp" + << std::endl + << "# DSTOR2: coloid,dctoid," + "dbroot,part,seg,comp" + << std::endl; - fMetaDataStream << - "# VERSION: 4" << std::endl << - "# APPLICATION: " << fAppDesc << std::endl << - "# PID: " << ::getpid() << std::endl << - "# TABLE: " << fTableName << std::endl << - "# COLUM1: coloid," - "dbroot,part,seg,lastLocalHWM,type,typename,width,comp" << - std::endl << - "# COLUM2: coloid," - "dbroot,part,seg,type,typename,width,comp" << - std::endl << - "# DSTOR1: coloid,dctoid," - "dbroot,part,seg,localHWM,comp" << std::endl << - "# DSTOR2: coloid,dctoid," - "dbroot,part,seg,comp" << std::endl; + // Clear out any data subdirectory + // This is redundant because init() also calls deleteSubDir(), but it can't + // hurt to call twice. We "really" want to make sure we start with a clean + // slate (no leftover backup chunk files from a previous import job). + deleteSubDir(metaFileName); - // Clear out any data subdirectory - // This is redundant because init() also calls deleteSubDir(), but it can't - // hurt to call twice. We "really" want to make sure we start with a clean - // slate (no leftover backup chunk files from a previous import job). - deleteSubDir( metaFileName ); - - return metaFileName; + return metaFileName; } - + //------------------------------------------------------------------------------ // Close the currently open "temporary named" meta data file used during // construction. We will rename all the meta data files (for the various // dbroots) to their eventual file names later, in renameMetaFile(). //------------------------------------------------------------------------------ -void RBMetaWriter::closeMetaFile ( ) +void RBMetaWriter::closeMetaFile() { - delete fMetaDataFile; - fMetaDataFile = NULL; + delete fMetaDataFile; + fMetaDataFile = NULL; } - + //------------------------------------------------------------------------------ // Rename temporary metafile names to their permanent name, taking file names // from fMetaFileNames. In the normal case there will be one file name per // DBRoot for the local PM we are running on. //------------------------------------------------------------------------------ -void RBMetaWriter::renameMetaFile ( ) +void RBMetaWriter::renameMetaFile() { - for (std::map::const_iterator iter = - fMetaFileNames.begin(); iter != fMetaFileNames.end(); ++iter) + for (std::map::const_iterator iter = fMetaFileNames.begin(); + iter != fMetaFileNames.end(); ++iter) + { + const std::string& metaFileName = iter->second; + + if (!metaFileName.empty()) { - const std::string& metaFileName = iter->second; + std::string tmpMetaFileName = metaFileName; + tmpMetaFileName += TMP_FILE_SUFFIX; - if (!metaFileName.empty()) - { - std::string tmpMetaFileName = metaFileName; - tmpMetaFileName += TMP_FILE_SUFFIX; - - if ( IDBPolicy::rename(tmpMetaFileName.c_str(), metaFileName.c_str()) ) - { - int errRc = errno; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << "Error renaming meta data file-" << - tmpMetaFileName << "; will be deleted; " << eMsg; - throw WeException( oss.str(), ERR_METADATABKUP_FILE_RENAME ); - } - } + if (IDBPolicy::rename(tmpMetaFileName.c_str(), metaFileName.c_str())) + { + int errRc = errno; + std::ostringstream oss; + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << "Error renaming meta data file-" << tmpMetaFileName << "; will be deleted; " << eMsg; + throw WeException(oss.str(), ERR_METADATABKUP_FILE_RENAME); + } } + } } - + //------------------------------------------------------------------------------ // Delete the meta data files for the specified table OID. We loop through all // the DBRoots for the local PM, deleting the applicable meta data files. @@ -538,203 +487,156 @@ void RBMetaWriter::renameMetaFile ( ) // The application code can then decide whether they want to consider this // condition as fatal or not. //------------------------------------------------------------------------------ -void RBMetaWriter::deleteFile ( ) +void RBMetaWriter::deleteFile() { - for (std::map::const_iterator iter = - fMetaFileNames.begin(); iter != fMetaFileNames.end(); ++iter) + for (std::map::const_iterator iter = fMetaFileNames.begin(); + iter != fMetaFileNames.end(); ++iter) + { + const std::string& metaFileName = iter->second; + + if (!metaFileName.empty()) { - const std::string& metaFileName = iter->second; + std::string tmpMetaFileName = metaFileName; + tmpMetaFileName += TMP_FILE_SUFFIX; - if (!metaFileName.empty()) - { - std::string tmpMetaFileName = metaFileName; - tmpMetaFileName += TMP_FILE_SUFFIX; + IDBPolicy::remove(metaFileName.c_str()); + IDBPolicy::remove(tmpMetaFileName.c_str()); - IDBPolicy::remove( metaFileName.c_str() ); - IDBPolicy::remove( tmpMetaFileName.c_str() ); - - deleteSubDir( metaFileName ); // can throw an exception - } + deleteSubDir(metaFileName); // can throw an exception } + } - fMetaFileNames.clear( ); + fMetaFileNames.clear(); } - + //------------------------------------------------------------------------------ // New version of writeColumnMetaData for Shared-Nothing //------------------------------------------------------------------------------ -void RBMetaWriter::writeColumnMetaData ( - const std::string& metaFileName, - bool withHWM, - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM lastLocalHwm, - CalpontSystemCatalog::ColDataType colType, - const std::string& colTypeName, - int colWidth, - int compressionType ) +void RBMetaWriter::writeColumnMetaData(const std::string& metaFileName, bool withHWM, OID columnOID, + uint16_t dbRoot, uint32_t partition, uint16_t segment, + HWM lastLocalHwm, CalpontSystemCatalog::ColDataType colType, + const std::string& colTypeName, int colWidth, int compressionType) { - if (withHWM) - { - fMetaDataStream << "COLUM1: " << - columnOID << ' ' << - dbRoot << ' ' << - partition << ' ' << - segment << ' ' << - lastLocalHwm << ' ' << - colType << ' ' << - colTypeName << ' ' << - colWidth; - } - else - { - fMetaDataStream << "COLUM2: " << - columnOID << ' ' << - dbRoot << ' ' << - partition << ' ' << - segment << ' ' << - colType << ' ' << - colTypeName << ' ' << - colWidth; - } + if (withHWM) + { + fMetaDataStream << "COLUM1: " << columnOID << ' ' << dbRoot << ' ' << partition << ' ' << segment << ' ' + << lastLocalHwm << ' ' << colType << ' ' << colTypeName << ' ' << colWidth; + } + else + { + fMetaDataStream << "COLUM2: " << columnOID << ' ' << dbRoot << ' ' << partition << ' ' << segment << ' ' + << colType << ' ' << colTypeName << ' ' << colWidth; + } - if (compressionType) - fMetaDataStream << ' ' << compressionType << ' '; + if (compressionType) + fMetaDataStream << ' ' << compressionType << ' '; - fMetaDataStream << std::endl; + fMetaDataStream << std::endl; - // If column is compressed, then create directory for storing HWM chunks - if (compressionType) + // If column is compressed, then create directory for storing HWM chunks + if (compressionType) + { + if (!fCreatedSubDir) { - if (!fCreatedSubDir) - { - // @bug 5572 - Don't need db backup files for HDFS; - // use hdfs buffer file - if (!IDBPolicy::useHdfs()) - createSubDir( metaFileName ); - } + // @bug 5572 - Don't need db backup files for HDFS; + // use hdfs buffer file + if (!IDBPolicy::useHdfs()) + createSubDir(metaFileName); } + } } - + //------------------------------------------------------------------------------ // New version of writeDictionaryStoreMetaData for Shared-Nothing. //------------------------------------------------------------------------------ -void RBMetaWriter::writeDictionaryStoreMetaData ( - OID columnOID, - OID dictionaryStoreOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM localHwm, - int compressionType ) +void RBMetaWriter::writeDictionaryStoreMetaData(OID columnOID, OID dictionaryStoreOID, uint16_t dbRoot, + uint32_t partition, uint16_t segment, HWM localHwm, + int compressionType) { - fMetaDataStream << "DSTOR1: " << - columnOID << ' ' << - dictionaryStoreOID << ' ' << - dbRoot << ' ' << - partition << ' ' << - segment << ' ' << - localHwm; + fMetaDataStream << "DSTOR1: " << columnOID << ' ' << dictionaryStoreOID << ' ' << dbRoot << ' ' << partition + << ' ' << segment << ' ' << localHwm; - if (compressionType) - fMetaDataStream << ' ' << compressionType << ' '; + if (compressionType) + fMetaDataStream << ' ' << compressionType << ' '; - fMetaDataStream << std::endl; + fMetaDataStream << std::endl; - // Save dictionary meta data for later use in backing up the HWM chunks - if (compressionType) - { - RBChunkInfo chunkInfo( - dictionaryStoreOID, dbRoot, partition, segment, localHwm); - fRBChunkDctnrySet.insert( chunkInfo ); + // Save dictionary meta data for later use in backing up the HWM chunks + if (compressionType) + { + RBChunkInfo chunkInfo(dictionaryStoreOID, dbRoot, partition, segment, localHwm); + fRBChunkDctnrySet.insert(chunkInfo); - if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) - printDctnryChunkList(chunkInfo, "after adding "); - } + if ((fLog) && (fLog->isDebug(DEBUG_1))) + printDctnryChunkList(chunkInfo, "after adding "); + } } - + //------------------------------------------------------------------------------ // New version of writeDictionaryStoreMetaNoDataMarker for Shared-Nothing. //------------------------------------------------------------------------------ -void RBMetaWriter::writeDictionaryStoreMetaNoDataMarker ( - OID columnOID, - OID dictionaryStoreOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - int compressionType ) +void RBMetaWriter::writeDictionaryStoreMetaNoDataMarker(OID columnOID, OID dictionaryStoreOID, + uint16_t dbRoot, uint32_t partition, uint16_t segment, + int compressionType) { - fMetaDataStream << "DSTOR2: " << - columnOID << ' ' << - dictionaryStoreOID << ' ' << - dbRoot << ' ' << - partition << ' ' << - segment; + fMetaDataStream << "DSTOR2: " << columnOID << ' ' << dictionaryStoreOID << ' ' << dbRoot << ' ' << partition + << ' ' << segment; - if (compressionType) - fMetaDataStream << ' ' << compressionType << ' '; + if (compressionType) + fMetaDataStream << ' ' << compressionType << ' '; - fMetaDataStream << std::endl; + fMetaDataStream << std::endl; } - + //------------------------------------------------------------------------------ // Create the subdirectory we will use to backup data needed for rollback. //------------------------------------------------------------------------------ -void RBMetaWriter::createSubDir( const std::string& metaFileName ) +void RBMetaWriter::createSubDir(const std::string& metaFileName) { - std::string bulkRollbackSubPath( metaFileName ); - bulkRollbackSubPath += DATA_DIR_SUFFIX; + std::string bulkRollbackSubPath(metaFileName); + bulkRollbackSubPath += DATA_DIR_SUFFIX; - if ( IDBPolicy::mkdir( bulkRollbackSubPath.c_str() ) != 0 ) - { - std::ostringstream oss; - oss << "Error creating bulk rollback data subdirectory " << - bulkRollbackSubPath << ";"; - throw WeException( oss.str(), ERR_DIR_CREATE ); - } + if (IDBPolicy::mkdir(bulkRollbackSubPath.c_str()) != 0) + { + std::ostringstream oss; + oss << "Error creating bulk rollback data subdirectory " << bulkRollbackSubPath << ";"; + throw WeException(oss.str(), ERR_DIR_CREATE); + } - fCreatedSubDir = true; + fCreatedSubDir = true; } - + //------------------------------------------------------------------------------ // Delete the subdirectory used to backup data needed for rollback. //------------------------------------------------------------------------------ -void RBMetaWriter::deleteSubDir( const std::string& metaFileName ) +void RBMetaWriter::deleteSubDir(const std::string& metaFileName) { - std::string bulkRollbackSubPath( metaFileName ); - bulkRollbackSubPath += DATA_DIR_SUFFIX; + std::string bulkRollbackSubPath(metaFileName); + bulkRollbackSubPath += DATA_DIR_SUFFIX; - if ( IDBPolicy::remove( bulkRollbackSubPath.c_str() ) != 0 ) - { - std::ostringstream oss; - oss << "Error deleting bulk rollback data subdirectory " << - bulkRollbackSubPath << ";"; - throw WeException( oss.str(), ERR_FILE_DELETE ); - } + if (IDBPolicy::remove(bulkRollbackSubPath.c_str()) != 0) + { + std::ostringstream oss; + oss << "Error deleting bulk rollback data subdirectory " << bulkRollbackSubPath << ";"; + throw WeException(oss.str(), ERR_FILE_DELETE); + } } - + //------------------------------------------------------------------------------ // Backup the contents of the HWM chunk for the specified column OID extent, // so that the chunk is available for bulk rollback. // This operation is only performed for compressed columns. //------------------------------------------------------------------------------ -void RBMetaWriter::backupColumnHWMChunk( - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM startingHWM) +void RBMetaWriter::backupColumnHWMChunk(OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment, + HWM startingHWM) { - // @bug 5572 - Don't need db backup file for HDFS; we use hdfs buffer file - if (!IDBPolicy::useHdfs()) - { - backupHWMChunk( true, - columnOID, dbRoot, partition, segment, startingHWM ); - } + // @bug 5572 - Don't need db backup file for HDFS; we use hdfs buffer file + if (!IDBPolicy::useHdfs()) + { + backupHWMChunk(true, columnOID, dbRoot, partition, segment, startingHWM); + } } - + //------------------------------------------------------------------------------ // Backup the contents of the HWM chunk for the specified dictionary store OID // extent, so that the chunk is available for bulk rollback. @@ -749,78 +651,72 @@ void RBMetaWriter::backupColumnHWMChunk( // dictionary column. //------------------------------------------------------------------------------ // @bug 5572 - HDFS usage: add return flag to indicate backup status -bool RBMetaWriter::backupDctnryHWMChunk( - OID dctnryOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment) +bool RBMetaWriter::backupDctnryHWMChunk(OID dctnryOID, uint16_t dbRoot, uint32_t partition, uint16_t segment) { - bool bBackupApplies = false; + bool bBackupApplies = false; + + if (fRBChunkDctnrySet.size() > 0) + { + RBChunkInfo chunkInfo(dctnryOID, 0, partition, segment, 0); + RBChunkInfo chunkInfoFound(0, 0, 0, 0, 0); + bool bFound = false; - if (fRBChunkDctnrySet.size() > 0) { - RBChunkInfo chunkInfo( - dctnryOID, 0, partition, segment, 0); - RBChunkInfo chunkInfoFound(0, 0, 0, 0, 0); - bool bFound = false; + // Use scoped lock to perform "find" + boost::mutex::scoped_lock lock(fRBChunkDctnryMutex); - { - // Use scoped lock to perform "find" - boost::mutex::scoped_lock lock( fRBChunkDctnryMutex ); + if ((fLog) && (fLog->isDebug(DEBUG_1))) + printDctnryChunkList(chunkInfo, "when searching "); - if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) - printDctnryChunkList(chunkInfo, "when searching "); + RBChunkSet::iterator iter = fRBChunkDctnrySet.find(chunkInfo); - RBChunkSet::iterator iter = fRBChunkDctnrySet.find ( chunkInfo ); - - if (iter != fRBChunkDctnrySet.end()) - { - bFound = true; - chunkInfoFound = *iter; - } - } - - if (bFound) - { - if (chunkInfoFound.fPartition == partition) - { - // @bug 5572 - Don't need db backup file for HDFS; - // we use hdfs buffer file. Set backup flag - // so application knows to use tmp buffer file. - bBackupApplies = true; - - if (!IDBPolicy::useHdfs()) - { - backupHWMChunk(false, dctnryOID, - dbRoot, partition, segment, chunkInfoFound.fHwm); - } - } - else - { - // How could this happen? Ended up asking for different - // partition than expected for the first instance of this - // OID and segment file. Perhaps initial blockskipping - // or something caused us to advance to another segment file - // without ever changing the expected extent. At any rate - // we still fall through and delete our entry because we - // apparently did not end up changing the chunk referenced - // by this RBChunkInfo object. - } - - { - // Use scoped lock to perform "erase" - boost::mutex::scoped_lock lock( fRBChunkDctnryMutex ); - fRBChunkDctnrySet.erase( chunkInfoFound ); - - if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) - printDctnryChunkList(chunkInfoFound, "after deleting "); - } - } + if (iter != fRBChunkDctnrySet.end()) + { + bFound = true; + chunkInfoFound = *iter; + } } - return bBackupApplies; + if (bFound) + { + if (chunkInfoFound.fPartition == partition) + { + // @bug 5572 - Don't need db backup file for HDFS; + // we use hdfs buffer file. Set backup flag + // so application knows to use tmp buffer file. + bBackupApplies = true; + + if (!IDBPolicy::useHdfs()) + { + backupHWMChunk(false, dctnryOID, dbRoot, partition, segment, chunkInfoFound.fHwm); + } + } + else + { + // How could this happen? Ended up asking for different + // partition than expected for the first instance of this + // OID and segment file. Perhaps initial blockskipping + // or something caused us to advance to another segment file + // without ever changing the expected extent. At any rate + // we still fall through and delete our entry because we + // apparently did not end up changing the chunk referenced + // by this RBChunkInfo object. + } + + { + // Use scoped lock to perform "erase" + boost::mutex::scoped_lock lock(fRBChunkDctnryMutex); + fRBChunkDctnrySet.erase(chunkInfoFound); + + if ((fLog) && (fLog->isDebug(DEBUG_1))) + printDctnryChunkList(chunkInfoFound, "after deleting "); + } + } + } + + return bBackupApplies; } - + //------------------------------------------------------------------------------ // Backup entire contents of HWM file for the specified columnOID,dbRoot,etc, // so that the file is available for bulk rollback. This function is used for @@ -833,117 +729,100 @@ bool RBMetaWriter::backupDctnryHWMChunk( //------------------------------------------------------------------------------ // @bug 5572 - Stopped using backupHWMFile(). // Don't need db backup file for HDFS; we use hdfs buffer file -void RBMetaWriter::backupHWMFile( - bool bColumnFile, // is this a column (vs dictionary) file - OID columnOID, // OID of column or dictionary store - uint16_t dbRoot, // DB Root for db segment file - uint32_t partition, // partition for db segment file - uint16_t segment, // segment for db segment file - HWM startingHWM) // starting HWM for db segment file +void RBMetaWriter::backupHWMFile(bool bColumnFile, // is this a column (vs dictionary) file + OID columnOID, // OID of column or dictionary store + uint16_t dbRoot, // DB Root for db segment file + uint32_t partition, // partition for db segment file + uint16_t segment, // segment for db segment file + HWM startingHWM) // starting HWM for db segment file { - std::string fileType("column"); + std::string fileType("column"); - if (!bColumnFile) - fileType = "dictionary"; + if (!bColumnFile) + fileType = "dictionary"; - FileOp fileOp; // @bug 4960: to keep thread-safe, we use local FileOp + FileOp fileOp; // @bug 4960: to keep thread-safe, we use local FileOp - // Construct file name for db file to be backed up - char dbFileName[FILE_NAME_SIZE]; - int rc = fileOp.getFileName( columnOID, dbFileName, - dbRoot, partition, segment ); + // Construct file name for db file to be backed up + char dbFileName[FILE_NAME_SIZE]; + int rc = fileOp.getFileName(columnOID, dbFileName, dbRoot, partition, segment); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Error creating backup " << fileType << - " file for OID " << columnOID << - "; Can't construct file name for DBRoot" << dbRoot << - "; partition-" << partition << - "; segment-" << segment; - throw WeException( oss.str(), rc ); - } + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error creating backup " << fileType << " file for OID " << columnOID + << "; Can't construct file name for DBRoot" << dbRoot << "; partition-" << partition << "; segment-" + << segment; + throw WeException(oss.str(), rc); + } - // Construct file name for backup copy of db file - std::ostringstream ossFile; - ossFile << "/" << columnOID << ".p" << partition << ".s" << segment; - std::string backupFileName; - rc = getSubDirPath( dbRoot, backupFileName ); + // Construct file name for backup copy of db file + std::ostringstream ossFile; + ossFile << "/" << columnOID << ".p" << partition << ".s" << segment; + std::string backupFileName; + rc = getSubDirPath(dbRoot, backupFileName); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Error creating backup " << fileType << - " file for OID " << columnOID << - "; Can't find matching meta file for DBRoot" << dbRoot; - throw WeException( oss.str(), rc ); - } + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error creating backup " << fileType << " file for OID " << columnOID + << "; Can't find matching meta file for DBRoot" << dbRoot; + throw WeException(oss.str(), rc); + } - backupFileName += ossFile.str(); + backupFileName += ossFile.str(); - std::string backupFileNameTmp = backupFileName; - backupFileNameTmp += TMP_FILE_SUFFIX; + std::string backupFileNameTmp = backupFileName; + backupFileNameTmp += TMP_FILE_SUFFIX; - //if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) - if (fLog) - { - std::ostringstream oss; - oss << "Backing up HWM file for " << fileType << - " file for OID " << columnOID << - "; file-" << backupFileNameTmp << - "; HWM-" << startingHWM; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + // if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) + if (fLog) + { + std::ostringstream oss; + oss << "Backing up HWM file for " << fileType << " file for OID " << columnOID << "; file-" + << backupFileNameTmp << "; HWM-" << startingHWM; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - // Copy the db file to a temporary name - IDBFileSystem& fs = IDBPolicy::getFs( backupFileNameTmp.c_str() ); + // Copy the db file to a temporary name + IDBFileSystem& fs = IDBPolicy::getFs(backupFileNameTmp.c_str()); - if ( !fs.exists(dbFileName) ) - { - std::ostringstream oss; - oss << "Error creating backup " << fileType << - " file for OID " << columnOID << - "; dbfile does not exist for DBRoot" << dbRoot << - "; partition-" << partition << - "; segment-" << segment; - throw WeException( oss.str(), ERR_FILE_NOT_EXIST ); - } + if (!fs.exists(dbFileName)) + { + std::ostringstream oss; + oss << "Error creating backup " << fileType << " file for OID " << columnOID + << "; dbfile does not exist for DBRoot" << dbRoot << "; partition-" << partition << "; segment-" + << segment; + throw WeException(oss.str(), ERR_FILE_NOT_EXIST); + } - rc = fs.copyFile( dbFileName, backupFileNameTmp.c_str() ); + rc = fs.copyFile(dbFileName, backupFileNameTmp.c_str()); - if (rc != 0) - { - std::ostringstream oss; - oss << "Error copying backup for " << fileType << - " OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; rc-" << rc; + if (rc != 0) + { + std::ostringstream oss; + oss << "Error copying backup for " << fileType << " OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; rc-" << rc; - fs.remove( backupFileNameTmp.c_str() ); - throw WeException( oss.str(), ERR_METADATABKUP_COMP_WRITE_BULK_BKUP ); - } + fs.remove(backupFileNameTmp.c_str()); + throw WeException(oss.str(), ERR_METADATABKUP_COMP_WRITE_BULK_BKUP); + } - // Rename temporary named backup file to final name - rc = fs.rename( backupFileNameTmp.c_str(), backupFileName.c_str() ); + // Rename temporary named backup file to final name + rc = fs.rename(backupFileNameTmp.c_str(), backupFileName.c_str()); - if (rc != 0) - { - std::ostringstream oss; - oss << "Error renaming temp backup for " << fileType << - " OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; rc-" << rc; + if (rc != 0) + { + std::ostringstream oss; + oss << "Error renaming temp backup for " << fileType << " OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; rc-" << rc; - fs.remove( backupFileNameTmp.c_str() ); - fs.remove( backupFileName.c_str() ); - throw WeException( oss.str(), ERR_METADATABKUP_COMP_RENAME ); - } + fs.remove(backupFileNameTmp.c_str()); + fs.remove(backupFileName.c_str()); + throw WeException(oss.str(), ERR_METADATABKUP_COMP_RENAME); + } } - + //------------------------------------------------------------------------------ // Backup the contents of the HWM chunk for the specified columnOID,dbRoot,etc, // so that the chunk is available for bulk rollback. This function is used for @@ -954,242 +833,193 @@ void RBMetaWriter::backupHWMFile( // backupHWMChunk() has to have a local FileOp object. We can't share/reuse // a FileOp data member variable unless we want to employ a mutex. //------------------------------------------------------------------------------ -void RBMetaWriter::backupHWMChunk( - bool bColumnFile, // is this a column (vs dictionary) file - OID columnOID, // OID of column or dictionary store - uint16_t dbRoot, // DB Root for db segment file - uint32_t partition, // partition for db segment file - uint16_t segment, // segment for db segment file - HWM startingHWM) // starting HWM for db segment file +void RBMetaWriter::backupHWMChunk(bool bColumnFile, // is this a column (vs dictionary) file + OID columnOID, // OID of column or dictionary store + uint16_t dbRoot, // DB Root for db segment file + uint32_t partition, // partition for db segment file + uint16_t segment, // segment for db segment file + HWM startingHWM) // starting HWM for db segment file { - std::string fileType("column"); + std::string fileType("column"); - if (!bColumnFile) - fileType = "dictionary"; + if (!bColumnFile) + fileType = "dictionary"; - // Open the applicable database column segment file - std::string segFile; - FileOp fileOp; // @bug 4960: to keep thread-safe, we use local FileOp - IDBDataFile* dbFile = fileOp.openFile( columnOID, - dbRoot, - partition, - segment, - segFile, - "rb" ); + // Open the applicable database column segment file + std::string segFile; + FileOp fileOp; // @bug 4960: to keep thread-safe, we use local FileOp + IDBDataFile* dbFile = fileOp.openFile(columnOID, dbRoot, partition, segment, segFile, "rb"); - if ( !dbFile ) - { - std::ostringstream oss; - oss << "Backup error opening " << fileType << - " file for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment; - throw WeException( oss.str(), ERR_FILE_OPEN ); - } + if (!dbFile) + { + std::ostringstream oss; + oss << "Backup error opening " << fileType << " file for OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment; + throw WeException(oss.str(), ERR_FILE_OPEN); + } - // Get the size of the file, so we know where to truncate back to. - long long fileSizeBytes; - int rc = fileOp.getFileSize( dbFile, fileSizeBytes); + // Get the size of the file, so we know where to truncate back to. + long long fileSizeBytes; + int rc = fileOp.getFileSize(dbFile, fileSizeBytes); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error getting file size for " << fileType << - " OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error getting file size for " << fileType << " OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; " << ec.errorString(rc); + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } - // Read Control header - char controlHdr[ CompressInterface::HDR_BUF_LEN ]; - rc = fileOp.readFile( dbFile, (unsigned char*)controlHdr, - CompressInterface::HDR_BUF_LEN ); + // Read Control header + char controlHdr[CompressInterface::HDR_BUF_LEN]; + rc = fileOp.readFile(dbFile, (unsigned char*)controlHdr, CompressInterface::HDR_BUF_LEN); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error reading " << fileType << - " file control hdr for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error reading " << fileType << " file control hdr for OID-" << columnOID << "; DBRoot-" + << dbRoot << "; partition-" << partition << "; segment-" << segment << "; " << ec.errorString(rc); + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } - int rc1 = compress::CompressInterface::verifyHdr(controlHdr); + int rc1 = compress::CompressInterface::verifyHdr(controlHdr); - if (rc1 != 0) - { - rc = ERR_METADATABKUP_COMP_VERIFY_HDRS; + if (rc1 != 0) + { + rc = ERR_METADATABKUP_COMP_VERIFY_HDRS; - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error verifying " << fileType << - " file control hdr for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc) << - "; rc: " << rc1; - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error verifying " << fileType << " file control hdr for OID-" << columnOID << "; DBRoot-" + << dbRoot << "; partition-" << partition << "; segment-" << segment << "; " << ec.errorString(rc) + << "; rc: " << rc1; + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } - auto compressionType = - compress::CompressInterface::getCompressionType(controlHdr); - std::unique_ptr compressor( - compress::getCompressInterfaceByType(compressionType)); + auto compressionType = compress::CompressInterface::getCompressionType(controlHdr); + std::unique_ptr compressor( + compress::getCompressInterfaceByType(compressionType)); - if (!compressor) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Ivalid compression type " << compressionType; - fileOp.closeFile( dbFile ); - throw WeException(oss.str(), rc); - } + if (!compressor) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Ivalid compression type " << compressionType; + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } - // Read Pointer header data - uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr); - uint64_t ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN; - char* pointerHdr = new char[ptrHdrSize]; - rc = fileOp.readFile( dbFile, (unsigned char*)pointerHdr, ptrHdrSize ); + // Read Pointer header data + uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr); + uint64_t ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN; + char* pointerHdr = new char[ptrHdrSize]; + rc = fileOp.readFile(dbFile, (unsigned char*)pointerHdr, ptrHdrSize); - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error reading " << fileType << - " file pointer hdr for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - delete[] pointerHdr; - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } - - CompChunkPtrList chunkPtrs; - rc = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize, - chunkPtrs); + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error reading " << fileType << " file pointer hdr for OID-" << columnOID << "; DBRoot-" + << dbRoot << "; partition-" << partition << "; segment-" << segment << "; " << ec.errorString(rc); delete[] pointerHdr; + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } - if (rc != 0) - { - std::ostringstream oss; - oss << "Backup error getting " << fileType << - " file hdr for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment; - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), ERR_METADATABKUP_COMP_PARSE_HDRS ); - } + CompChunkPtrList chunkPtrs; + rc = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize, chunkPtrs); + delete[] pointerHdr; - // Locate HWM chunk - unsigned int chunkIndex = 0; - unsigned int blockOffsetWithinChunk = 0; - unsigned char* buffer = 0; - uint64_t chunkSize = 0; - compressor->locateBlock(startingHWM, chunkIndex, blockOffsetWithinChunk); + if (rc != 0) + { + std::ostringstream oss; + oss << "Backup error getting " << fileType << " file hdr for OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment; + fileOp.closeFile(dbFile); + throw WeException(oss.str(), ERR_METADATABKUP_COMP_PARSE_HDRS); + } - if (chunkIndex < chunkPtrs.size()) - { - chunkSize = chunkPtrs[chunkIndex].second; + // Locate HWM chunk + unsigned int chunkIndex = 0; + unsigned int blockOffsetWithinChunk = 0; + unsigned char* buffer = 0; + uint64_t chunkSize = 0; + compressor->locateBlock(startingHWM, chunkIndex, blockOffsetWithinChunk); - // Read the HWM chunk - rc = fileOp.setFileOffset(dbFile, chunkPtrs[chunkIndex].first, SEEK_SET); + if (chunkIndex < chunkPtrs.size()) + { + chunkSize = chunkPtrs[chunkIndex].second; - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error seeking in " << fileType << - " file for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } - - buffer = new unsigned char[ chunkPtrs[chunkIndex].second ]; - rc = fileOp.readFile( dbFile, buffer, chunkPtrs[chunkIndex].second ); - - if (rc != NO_ERROR) - { - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error reading in " << fileType << - " file for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << ec.errorString(rc); - delete []buffer; - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } - } - else if (startingHWM == 0) - { - // Okay to proceed. Empty file with no chunks. Save 0 length chunk. - } - else - { - rc = ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND; - - WErrorCodes ec; - std::ostringstream oss; - oss << "Backup error for " << fileType << - " file for OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; hwm-" << startingHWM << - "; chunkIdx-" << chunkIndex << - "; numPtrs-" << chunkPtrs.size() << - "; not in hdrPtrs" << - "; " << ec.errorString(rc); - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); - } - - // Backup the HWM chunk - std::string errMsg; - rc = writeHWMChunk(bColumnFile, columnOID, dbRoot, partition, segment, - buffer, chunkSize, fileSizeBytes, startingHWM, errMsg); + // Read the HWM chunk + rc = fileOp.setFileOffset(dbFile, chunkPtrs[chunkIndex].first, SEEK_SET); if (rc != NO_ERROR) { - std::ostringstream oss; - oss << "Backup error writing backup for " << fileType << - " OID-" << columnOID << - "; DBRoot-" << dbRoot << - "; partition-" << partition << - "; segment-" << segment << - "; " << errMsg; - delete []buffer; - fileOp.closeFile( dbFile ); - throw WeException( oss.str(), rc ); + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error seeking in " << fileType << " file for OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; " << ec.errorString(rc); + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); } - // Close the applicable database column segment file and free memory - delete []buffer; - fileOp.closeFile( dbFile ); + buffer = new unsigned char[chunkPtrs[chunkIndex].second]; + rc = fileOp.readFile(dbFile, buffer, chunkPtrs[chunkIndex].second); + + if (rc != NO_ERROR) + { + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error reading in " << fileType << " file for OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; " << ec.errorString(rc); + delete[] buffer; + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } + } + else if (startingHWM == 0) + { + // Okay to proceed. Empty file with no chunks. Save 0 length chunk. + } + else + { + rc = ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND; + + WErrorCodes ec; + std::ostringstream oss; + oss << "Backup error for " << fileType << " file for OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; hwm-" << startingHWM << "; chunkIdx-" + << chunkIndex << "; numPtrs-" << chunkPtrs.size() << "; not in hdrPtrs" + << "; " << ec.errorString(rc); + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } + + // Backup the HWM chunk + std::string errMsg; + rc = writeHWMChunk(bColumnFile, columnOID, dbRoot, partition, segment, buffer, chunkSize, fileSizeBytes, + startingHWM, errMsg); + + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Backup error writing backup for " << fileType << " OID-" << columnOID << "; DBRoot-" << dbRoot + << "; partition-" << partition << "; segment-" << segment << "; " << errMsg; + delete[] buffer; + fileOp.closeFile(dbFile); + throw WeException(oss.str(), rc); + } + + // Close the applicable database column segment file and free memory + delete[] buffer; + fileOp.closeFile(dbFile); } - + //------------------------------------------------------------------------------ // Writes out the specified HWM chunk to disk, in case we need it for bulk // rollback. If an error occurs, errMsg will contain the error message. @@ -1205,164 +1035,152 @@ void RBMetaWriter::backupHWMChunk( // This function MUST be kept thread-safe in support of backupDctnryHWMChunk(). // See that function description for more details. //------------------------------------------------------------------------------ -int RBMetaWriter::writeHWMChunk( - bool bColumnFile, // is this a column (vs dictionary) file - OID columnOID, // OID of column or dictionary store - uint16_t dbRoot, // dbroot for db segment file - uint32_t partition, // partition for db segment file - uint16_t segment, // segment for db segment file - const unsigned char* compressedOutBuf, // compressed chunk to be written - uint64_t chunkSize, // number of bytes in compressedOutBuf - uint64_t fileSize, // size of file in bytes - HWM chunkHWM, // HWM in the chunk being written - std::string& errMsg) const// error msg if error occurs +int RBMetaWriter::writeHWMChunk(bool bColumnFile, // is this a column (vs dictionary) file + OID columnOID, // OID of column or dictionary store + uint16_t dbRoot, // dbroot for db segment file + uint32_t partition, // partition for db segment file + uint16_t segment, // segment for db segment file + const unsigned char* compressedOutBuf, // compressed chunk to be written + uint64_t chunkSize, // number of bytes in compressedOutBuf + uint64_t fileSize, // size of file in bytes + HWM chunkHWM, // HWM in the chunk being written + std::string& errMsg) const // error msg if error occurs { - std::ostringstream ossFile; - ossFile << "/" << columnOID << ".p" << partition << ".s" << segment; - std::string fileName; - std::string dirPath; - int rc = getSubDirPath( dbRoot, fileName ); + std::ostringstream ossFile; + ossFile << "/" << columnOID << ".p" << partition << ".s" << segment; + std::string fileName; + std::string dirPath; + int rc = getSubDirPath(dbRoot, fileName); - if (rc != NO_ERROR) - { - std::ostringstream oss; - oss << "Error creating backup file for OID " << columnOID << - "; Can't find matching meta file for DBRoot" << dbRoot; - errMsg = oss.str(); - return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP; - } + if (rc != NO_ERROR) + { + std::ostringstream oss; + oss << "Error creating backup file for OID " << columnOID << "; Can't find matching meta file for DBRoot" + << dbRoot; + errMsg = oss.str(); + return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP; + } - dirPath = fileName; + dirPath = fileName; - fileName += ossFile.str(); + fileName += ossFile.str(); - std::string fileNameTmp = fileName; - fileNameTmp += TMP_FILE_SUFFIX; + std::string fileNameTmp = fileName; + fileNameTmp += TMP_FILE_SUFFIX; - //if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) - if (fLog) - { - std::string fileType("column"); + // if ( (fLog) && (fLog->isDebug(DEBUG_1)) ) + if (fLog) + { + std::string fileType("column"); - if (!bColumnFile) - fileType = "dictionary"; + if (!bColumnFile) + fileType = "dictionary"; - std::ostringstream oss; - oss << "Backing up HWM chunk for " << fileType << - " OID-" << columnOID << - "; file-" << fileNameTmp << - "; HWM-" << chunkHWM << - "; bytes-" << chunkSize << - "; fileSize-" << fileSize; - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); - } + std::ostringstream oss; + oss << "Backing up HWM chunk for " << fileType << " OID-" << columnOID << "; file-" << fileNameTmp + << "; HWM-" << chunkHWM << "; bytes-" << chunkSize << "; fileSize-" << fileSize; + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } - IDBDataFile* backupFile = IDBDataFile::open( - IDBPolicy::getType( fileNameTmp.c_str(), IDBPolicy::WRITEENG ), - fileNameTmp.c_str(), - "w+b", - 0 ); + IDBDataFile* backupFile = IDBDataFile::open(IDBPolicy::getType(fileNameTmp.c_str(), IDBPolicy::WRITEENG), + fileNameTmp.c_str(), "w+b", 0); - if (!backupFile) - { - int errRc = errno; - WErrorCodes ec; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << ec.errorString(ERR_METADATABKUP_COMP_OPEN_BULK_BKUP) << - "; " << eMsg; - errMsg = oss.str(); - return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP; - } + if (!backupFile) + { + int errRc = errno; + WErrorCodes ec; + std::ostringstream oss; + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << ec.errorString(ERR_METADATABKUP_COMP_OPEN_BULK_BKUP) << "; " << eMsg; + errMsg = oss.str(); + return ERR_METADATABKUP_COMP_OPEN_BULK_BKUP; + } - IDBFileSystem& fs = IDBPolicy::getFs( fileNameTmp.c_str() ); + IDBFileSystem& fs = IDBPolicy::getFs(fileNameTmp.c_str()); - // Format of backup compressed chunk file: - // 8 byte unsigned int carrying chunk size - // 8 byte unsigned int carrying original file size - // N bytes containing compressed chunk - uint64_t sizeHdr[2]; - sizeHdr[0] = chunkSize; - sizeHdr[1] = fileSize; - size_t itemsWritten = backupFile->write(sizeHdr, sizeof(uint64_t) * 2) / (sizeof(uint64_t) * 2); + // Format of backup compressed chunk file: + // 8 byte unsigned int carrying chunk size + // 8 byte unsigned int carrying original file size + // N bytes containing compressed chunk + uint64_t sizeHdr[2]; + sizeHdr[0] = chunkSize; + sizeHdr[1] = fileSize; + size_t itemsWritten = backupFile->write(sizeHdr, sizeof(uint64_t) * 2) / (sizeof(uint64_t) * 2); + + if (itemsWritten != 1) + { + int errRc = errno; + WErrorCodes ec; + std::ostringstream oss; + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << ec.errorString(ERR_METADATABKUP_COMP_WRITE_BULK_BKUP) << "; " << eMsg; + errMsg = oss.str(); + + delete backupFile; + fs.remove(fileNameTmp.c_str()); + return ERR_METADATABKUP_COMP_WRITE_BULK_BKUP; + } + + if (chunkSize > 0) + { + itemsWritten = backupFile->write(compressedOutBuf, chunkSize) / chunkSize; if (itemsWritten != 1) { - int errRc = errno; - WErrorCodes ec; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << ec.errorString(ERR_METADATABKUP_COMP_WRITE_BULK_BKUP) << - "; " << eMsg; - errMsg = oss.str(); + int errRc = errno; + WErrorCodes ec; + std::ostringstream oss; + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << ec.errorString(ERR_METADATABKUP_COMP_WRITE_BULK_BKUP) << "; " << eMsg; + errMsg = oss.str(); - delete backupFile; - fs.remove( fileNameTmp.c_str() ); - return ERR_METADATABKUP_COMP_WRITE_BULK_BKUP; + delete backupFile; + fs.remove(fileNameTmp.c_str()); + return ERR_METADATABKUP_COMP_WRITE_BULK_BKUP; } + } - if (chunkSize > 0) - { - itemsWritten = backupFile->write(compressedOutBuf, chunkSize ) / chunkSize; - - if (itemsWritten != 1) - { - int errRc = errno; - WErrorCodes ec; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << ec.errorString(ERR_METADATABKUP_COMP_WRITE_BULK_BKUP) << - "; " << eMsg; - errMsg = oss.str(); - - delete backupFile; - fs.remove( fileNameTmp.c_str() ); - return ERR_METADATABKUP_COMP_WRITE_BULK_BKUP; - } - } - - backupFile->flush(); -// IDBDataFile flush() does a sync where appropriate - delete backupFile; + backupFile->flush(); + // IDBDataFile flush() does a sync where appropriate + delete backupFile; #ifdef _MSC_VER - //Windows rename() behaves differently from Linux: it will return an error - // if the target exists - //FIXME: The Linux version seems a bit safer, perhaps implement a better - // Windows port? - unlink(fileName.c_str()); + // Windows rename() behaves differently from Linux: it will return an error + // if the target exists + // FIXME: The Linux version seems a bit safer, perhaps implement a better + // Windows port? + unlink(fileName.c_str()); #endif - // Rename HWM backup file to final name. - if ( fs.rename(fileNameTmp.c_str(), fileName.c_str()) ) + // Rename HWM backup file to final name. + if (fs.rename(fileNameTmp.c_str(), fileName.c_str())) + { + int errRc = errno; + WErrorCodes ec; + std::ostringstream oss; + std::string eMsg; + Convertor::mapErrnoToString(errRc, eMsg); + oss << ec.errorString(ERR_METADATABKUP_COMP_RENAME) << "; " << eMsg; + errMsg = oss.str(); + + fs.remove(fileNameTmp.c_str()); + fs.remove(fileName.c_str()); + return ERR_METADATABKUP_COMP_RENAME; + } + + { + std::ostringstream ossChown; + idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(fileName.c_str()); + if (chownPath(ossChown, fileName, fs) || chownPath(ossChown, dirPath, fs)) { - int errRc = errno; - WErrorCodes ec; - std::ostringstream oss; - std::string eMsg; - Convertor::mapErrnoToString(errRc, eMsg); - oss << ec.errorString(ERR_METADATABKUP_COMP_RENAME) << "; " << eMsg; - errMsg = oss.str(); - - fs.remove( fileNameTmp.c_str() ); - fs.remove( fileName.c_str() ); - return ERR_METADATABKUP_COMP_RENAME; + throw WeException(ossChown.str(), ERR_FILE_CHOWN); } + } - { - std::ostringstream ossChown; - idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(fileName.c_str()); - if (chownPath(ossChown, fileName, fs) - || chownPath(ossChown, dirPath, fs)) - { - throw WeException(ossChown.str(), ERR_FILE_CHOWN); - } - } - - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ // Returns the directory path to be used for storing any backup data files. @@ -1370,60 +1188,55 @@ int RBMetaWriter::writeHWMChunk( // This function MUST be kept thread-safe in support of backupDctnryHWMChunk(). // See that function description for more details. //------------------------------------------------------------------------------ -int RBMetaWriter::getSubDirPath( uint16_t dbRoot, - std::string& bulkRollbackSubPath ) const +int RBMetaWriter::getSubDirPath(uint16_t dbRoot, std::string& bulkRollbackSubPath) const { - std::map::const_iterator iter = - fMetaFileNames.find( dbRoot ); + std::map::const_iterator iter = fMetaFileNames.find(dbRoot); - if (iter == fMetaFileNames.end()) - { - return ERR_INVALID_PARAM; - } + if (iter == fMetaFileNames.end()) + { + return ERR_INVALID_PARAM; + } - bulkRollbackSubPath = iter->second; - bulkRollbackSubPath += DATA_DIR_SUFFIX; + bulkRollbackSubPath = iter->second; + bulkRollbackSubPath += DATA_DIR_SUFFIX; - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Prints list of compressed dictionary HWM chunks that we are tracking, // in order to backup to disk as needed, before we start adding rows to a // previously existing chunk. //------------------------------------------------------------------------------ -void RBMetaWriter::printDctnryChunkList( - const RBChunkInfo& rbChk, - const char* assocAction) +void RBMetaWriter::printDctnryChunkList(const RBChunkInfo& rbChk, const char* assocAction) { - if (fLog) + if (fLog) + { + std::ostringstream oss; + oss << "Dumping metaDictHWMChunks " << assocAction << rbChk << ":"; + + if (fRBChunkDctnrySet.size() > 0) { - std::ostringstream oss; - oss << "Dumping metaDictHWMChunks " << assocAction << - rbChk << ":"; + RBChunkSet::iterator iter = fRBChunkDctnrySet.begin(); + int k = 1; - if (fRBChunkDctnrySet.size() > 0) - { - RBChunkSet::iterator iter = fRBChunkDctnrySet.begin(); - int k = 1; + while (iter != fRBChunkDctnrySet.end()) + { + oss << std::endl; + oss << '\t' << k << ". " << *iter; - while (iter != fRBChunkDctnrySet.end()) - { - oss << std::endl; - oss << '\t' << k << ". " << *iter; - - ++k; - ++iter; - } - } - else - { - oss << std::endl; - oss << '\t' << "Empty list"; - } - - fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + ++k; + ++iter; + } } + else + { + oss << std::endl; + oss << '\t' << "Empty list"; + } + + fLog->logMsg(oss.str(), MSGLVL_INFO2); + } } //------------------------------------------------------------------------------ @@ -1432,10 +1245,10 @@ void RBMetaWriter::printDctnryChunkList( /* static */ bool RBMetaWriter::verifyVersion3(const char* versionRec) { - if (strncmp(versionRec, VERSION3_REC, VERSION3_REC_LEN) == 0) - return true; - else - return false; + if (strncmp(versionRec, VERSION3_REC, VERSION3_REC_LEN) == 0) + return true; + else + return false; } //------------------------------------------------------------------------------ @@ -1444,10 +1257,10 @@ bool RBMetaWriter::verifyVersion3(const char* versionRec) /* static */ bool RBMetaWriter::verifyVersion4(const char* versionRec) { - if (strncmp(versionRec, VERSION4_REC, VERSION4_REC_LEN) == 0) - return true; - else - return false; + if (strncmp(versionRec, VERSION4_REC, VERSION4_REC_LEN) == 0) + return true; + else + return false; } //------------------------------------------------------------------------------ @@ -1456,10 +1269,10 @@ bool RBMetaWriter::verifyVersion4(const char* versionRec) /* static */ bool RBMetaWriter::verifyColumn1Rec(const char* recType) { - if (strncmp(recType, COLUMN1_REC, COLUMN1_REC_LEN) == 0) - return true; - else - return false; + if (strncmp(recType, COLUMN1_REC, COLUMN1_REC_LEN) == 0) + return true; + else + return false; } //------------------------------------------------------------------------------ @@ -1468,10 +1281,10 @@ bool RBMetaWriter::verifyColumn1Rec(const char* recType) /* static */ bool RBMetaWriter::verifyColumn2Rec(const char* recType) { - if (strncmp(recType, COLUMN2_REC, COLUMN2_REC_LEN) == 0) - return true; - else - return false; + if (strncmp(recType, COLUMN2_REC, COLUMN2_REC_LEN) == 0) + return true; + else + return false; } //------------------------------------------------------------------------------ @@ -1480,10 +1293,10 @@ bool RBMetaWriter::verifyColumn2Rec(const char* recType) /* static */ bool RBMetaWriter::verifyDStore1Rec(const char* recType) { - if (strncmp(recType, DSTORE1_REC, DSTORE1_REC_LEN) == 0) - return true; - else - return false; + if (strncmp(recType, DSTORE1_REC, DSTORE1_REC_LEN) == 0) + return true; + else + return false; } //------------------------------------------------------------------------------ @@ -1492,10 +1305,10 @@ bool RBMetaWriter::verifyDStore1Rec(const char* recType) /* static */ bool RBMetaWriter::verifyDStore2Rec(const char* recType) { - if (strncmp(recType, DSTORE2_REC, DSTORE2_REC_LEN) == 0) - return true; - else - return false; + if (strncmp(recType, DSTORE2_REC, DSTORE2_REC_LEN) == 0) + return true; + else + return false; } -} // end of namespace +} // namespace WriteEngine diff --git a/writeengine/shared/we_rbmetawriter.h b/writeengine/shared/we_rbmetawriter.h index e3b7a1bd3..15f22a69a 100644 --- a/writeengine/shared/we_rbmetawriter.h +++ b/writeengine/shared/we_rbmetawriter.h @@ -16,8 +16,8 @@ MA 02110-1301, USA. */ /* -* $Id: we_rbmetawriter.h 4450 2013-01-21 14:13:24Z rdempsey $ -*/ + * $Id: we_rbmetawriter.h 4450 2013-01-21 14:13:24Z rdempsey $ + */ /** @file we_rbmetawriter.h * Contains class to write HWM-related information used to rollback a @@ -60,24 +60,24 @@ class Log; //------------------------------------------------------------------------------ struct RBChunkInfo { - OID fOid; // dctnry store OID containing relevant chunk - uint16_t fDbRoot; // dbroot, partition, segment of file - uint32_t fPartition; // containing relevant HWM chunk - uint16_t fSegment; // - HWM fHwm; // HWM block of interest - RBChunkInfo(OID oid, uint16_t dbRoot, uint32_t partition, - uint16_t segment, HWM hwm ) : - fOid(oid), fDbRoot(dbRoot), fPartition(partition), - fSegment(segment), fHwm(hwm) { } + OID fOid; // dctnry store OID containing relevant chunk + uint16_t fDbRoot; // dbroot, partition, segment of file + uint32_t fPartition; // containing relevant HWM chunk + uint16_t fSegment; // + HWM fHwm; // HWM block of interest + RBChunkInfo(OID oid, uint16_t dbRoot, uint32_t partition, uint16_t segment, HWM hwm) + : fOid(oid), fDbRoot(dbRoot), fPartition(partition), fSegment(segment), fHwm(hwm) + { + } }; class RBChunkInfoCompare { -public: - bool operator()(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const; + public: + bool operator()(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const; }; -typedef std::set< RBChunkInfo, RBChunkInfoCompare > RBChunkSet; +typedef std::set RBChunkSet; //------------------------------------------------------------------------------ /** @brief Class to write HWM-related information to support bulk rollbacks. @@ -136,246 +136,196 @@ typedef std::set< RBChunkInfo, RBChunkInfoCompare > RBChunkSet; * parallel by several threads for different dictionary columns. */ //------------------------------------------------------------------------------ -class RBMetaWriter: public WeUIDGID +class RBMetaWriter : public WeUIDGID { -public: + public: + /** @brief RBMetaWriter constructor + * @param appDesc Description of application that is using RBMetaWriter + * @param logger Logger to be used for logging messages. + */ + EXPORT RBMetaWriter(const std::string& appDesc, Log* logger); - /** @brief RBMetaWriter constructor - * @param appDesc Description of application that is using RBMetaWriter - * @param logger Logger to be used for logging messages. - */ - EXPORT RBMetaWriter ( const std::string& appDesc, - Log* logger ); + /** @brief RBMetaWriter destructor + */ + EXPORT ~RBMetaWriter() + { + closeMetaFile(); + } - /** @brief RBMetaWriter destructor - */ - EXPORT ~RBMetaWriter ( ) - { - closeMetaFile ( ); - } + /** @brief Initialize this RBMetaWriter object + * Warning: This function may throw a WeException. + * + * @param tableOID OID of the table whose state is to be saved. + * @param tableName Name of the table associated with tableOID. + */ + EXPORT void init(OID tableOID, const std::string& tableName); - /** @brief Initialize this RBMetaWriter object - * Warning: This function may throw a WeException. - * - * @param tableOID OID of the table whose state is to be saved. - * @param tableName Name of the table associated with tableOID. - */ - EXPORT void init ( OID tableOID, - const std::string& tableName ); + /** @brief Make a backup copy of the specified HWM dictionary store chunk. + * This operation only applies to compressed columns. Backup may not be + * necessary. Return value indicates whether the specified file needs to + * be backed up or not. + * Warning: This function may throw a WeException. + * + * This function is thread-safe since concurrent calls could be made by + * different threads, each for a different dictionary column. + * + * @param dctnryOID column OID to be saved + * @param dbRoot current dbRoot of last local HWM for columnOID + * @param partition current partition of last local HWM for columnOID + * @param segment current segment of last local HWM for columnOID + * @return Indicates whether it is necessary to perform backup + */ + EXPORT bool backupDctnryHWMChunk(OID dctnryOID, uint16_t dbRoot, uint32_t partition, uint16_t segment); - /** @brief Make a backup copy of the specified HWM dictionary store chunk. - * This operation only applies to compressed columns. Backup may not be - * necessary. Return value indicates whether the specified file needs to - * be backed up or not. - * Warning: This function may throw a WeException. - * - * This function is thread-safe since concurrent calls could be made by - * different threads, each for a different dictionary column. - * - * @param dctnryOID column OID to be saved - * @param dbRoot current dbRoot of last local HWM for columnOID - * @param partition current partition of last local HWM for columnOID - * @param segment current segment of last local HWM for columnOID - * @return Indicates whether it is necessary to perform backup - */ - EXPORT bool backupDctnryHWMChunk ( - OID dctnryOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment ); + /** @brief Delete the rollback meta files associated with this table + * Warning: This function may throw a WeException. + */ + EXPORT void deleteFile(); - /** @brief Delete the rollback meta files associated with this table - * Warning: This function may throw a WeException. - */ - EXPORT void deleteFile ( ); + /** @brief Helper function that creates the primary meta data file. + * Warning: This function may throw a WeException. + * + * See class description for more details. + * @param columns Vector of column information. The dataFile member in + * each columns entry should be filled in with HWM information about + * the start extent where rows will be added. This information is + * needed so that the HWM chunk in that extent can be backed up. + * @param dctnryStoreOids Vector of dictionary store OID associated with + * columns vector. dctnryStoreOid[n] should be 0 if columns[n] is + * not a dictionary column. + * @param dbRootHwmInfoVecCol Vector of EmDbRootHWMInfo_v objects obtained + * from multiple calls to DBRM::getDbRootHWMInfo(). There is one + * EmDbRootHWMInfo_v entry per column. Each + * EmDbRootHWMInfo_v object carries a vector of DBRoot, HWM, etc + * objects representing the current HWM extents for a column's + * DBRoots on the local PM. + */ + EXPORT void saveBulkRollbackMetaData(const std::vector& columns, + const std::vector& dctnryStoreOids, + const std::vector& dbRootHWMInfoVecCol); - /** @brief Helper function that creates the primary meta data file. - * Warning: This function may throw a WeException. - * - * See class description for more details. - * @param columns Vector of column information. The dataFile member in - * each columns entry should be filled in with HWM information about - * the start extent where rows will be added. This information is - * needed so that the HWM chunk in that extent can be backed up. - * @param dctnryStoreOids Vector of dictionary store OID associated with - * columns vector. dctnryStoreOid[n] should be 0 if columns[n] is - * not a dictionary column. - * @param dbRootHwmInfoVecCol Vector of EmDbRootHWMInfo_v objects obtained - * from multiple calls to DBRM::getDbRootHWMInfo(). There is one - * EmDbRootHWMInfo_v entry per column. Each - * EmDbRootHWMInfo_v object carries a vector of DBRoot, HWM, etc - * objects representing the current HWM extents for a column's - * DBRoots on the local PM. - */ - EXPORT void saveBulkRollbackMetaData( - const std::vector& columns, - const std::vector& dctnryStoreOids, - const std::vector& dbRootHWMInfoVecCol ); + /** @brief Verify that specified version record is for Version 3 */ + static bool verifyVersion3(const char* versionRec); - /** @brief Verify that specified version record is for Version 3 */ - static bool verifyVersion3(const char* versionRec); + /** @brief Verify that specified version record is for Version 4 */ + static bool verifyVersion4(const char* versionRec); - /** @brief Verify that specified version record is for Version 4 */ - static bool verifyVersion4(const char* versionRec); + /** @brief Verify that specified record type is a Column1 record */ + static bool verifyColumn1Rec(const char* recType); - /** @brief Verify that specified record type is a Column1 record */ - static bool verifyColumn1Rec(const char* recType); + /** @brief Verify that specified record type is a Column2 record */ + static bool verifyColumn2Rec(const char* recType); - /** @brief Verify that specified record type is a Column2 record */ - static bool verifyColumn2Rec(const char* recType); + /** @brief Verify that specified record type is a DStore1 record */ + static bool verifyDStore1Rec(const char* recType); - /** @brief Verify that specified record type is a DStore1 record */ - static bool verifyDStore1Rec(const char* recType); + /** @brief Verify that specified record type is a DStore2 record */ + static bool verifyDStore2Rec(const char* recType); - /** @brief Verify that specified record type is a DStore2 record */ - static bool verifyDStore2Rec(const char* recType); - -private: - // disable copy constructor and assignment operator - RBMetaWriter(const RBMetaWriter&); - RBMetaWriter& operator=(const RBMetaWriter&); + private: + // disable copy constructor and assignment operator + RBMetaWriter(const RBMetaWriter&); + RBMetaWriter& operator=(const RBMetaWriter&); - // Make a backup copy of the specified HWM column chunk. - // This operation only applies to compressed columns. - // Warning: This function may throw a WeException. - // columnOID column OID to be saved - // dbRoot current dbRoot of last local HWM for columnOID - // partition current partition of last local HWM for columnOID - // segment current segment of last local HWM for columnOID - // lastLocalHwm current last local for column OID - void backupColumnHWMChunk ( - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM lastLocalHwm ); + // Make a backup copy of the specified HWM column chunk. + // This operation only applies to compressed columns. + // Warning: This function may throw a WeException. + // columnOID column OID to be saved + // dbRoot current dbRoot of last local HWM for columnOID + // partition current partition of last local HWM for columnOID + // segment current segment of last local HWM for columnOID + // lastLocalHwm current last local for column OID + void backupColumnHWMChunk(OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment, + HWM lastLocalHwm); - // This function must be thread-safe since it is called directly by - // backupDctnryHWMChunk(). Employed by non-hdfs. - void backupHWMChunk ( - bool bColumnFile, - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM lastLocalHwm ); + // This function must be thread-safe since it is called directly by + // backupDctnryHWMChunk(). Employed by non-hdfs. + void backupHWMChunk(bool bColumnFile, OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment, + HWM lastLocalHwm); - // This function must be thread-safe since it is called directly by - // backupDctnryHWMFile(). Employed by hdfs. - void backupHWMFile ( - bool bColumnFile, - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM lastLocalHwm ); + // This function must be thread-safe since it is called directly by + // backupDctnryHWMFile(). Employed by hdfs. + void backupHWMFile(bool bColumnFile, OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment, + HWM lastLocalHwm); - // Close the current meta data file. - EXPORT void closeMetaFile ( ); + // Close the current meta data file. + EXPORT void closeMetaFile(); - void createSubDir( const std::string& metaFileName ); - void deleteSubDir( const std::string& metaFileName ); - int getSubDirPath(const uint16_t dbRoot, - std::string& subDirPath ) const; + void createSubDir(const std::string& metaFileName); + void deleteSubDir(const std::string& metaFileName); + int getSubDirPath(const uint16_t dbRoot, std::string& subDirPath) const; - // Open a meta data file to save HWM bulk rollback info for tableOID - // Warning: This function may throw a WeException. - // dbRoot is the DBRoot of interest for the applicable table. - std::string openMetaFile ( uint16_t dbRoot ); + // Open a meta data file to save HWM bulk rollback info for tableOID + // Warning: This function may throw a WeException. + // dbRoot is the DBRoot of interest for the applicable table. + std::string openMetaFile(uint16_t dbRoot); - // Rename temporary metadata control file(s) to the permanent name. - // Filenames are taken from fMetaFileNames. - // Warning: This function may throw a WeException. - void renameMetaFile( ); + // Rename temporary metadata control file(s) to the permanent name. + // Filenames are taken from fMetaFileNames. + // Warning: This function may throw a WeException. + void renameMetaFile(); - // Save column meta data to the currently open file. - // This is the Shared-Nothing version of this function. - // Warning: This function may throw a WeException. - // metaFileName name of metafile to be written - // columnOID column OID to be saved - // dbRoot current dbRoot of last local HWM for columnOID - // partition current partition of last local HWM for columnOID - // segment current segment of last local HWM for columnOID - // lastLocalHwm current last local for column OID - // colType type of columnOID - // colTypeName type name of columnOID - // colWidth width (in bytes) of columnOID - // compressionType compression type - void writeColumnMetaData ( - const std::string& metaFileName, - bool withHWM, - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM lastLocalHwm, - execplan::CalpontSystemCatalog::ColDataType colType, - const std::string& colTypeName, - int colWidth, - int compressionType ); + // Save column meta data to the currently open file. + // This is the Shared-Nothing version of this function. + // Warning: This function may throw a WeException. + // metaFileName name of metafile to be written + // columnOID column OID to be saved + // dbRoot current dbRoot of last local HWM for columnOID + // partition current partition of last local HWM for columnOID + // segment current segment of last local HWM for columnOID + // lastLocalHwm current last local for column OID + // colType type of columnOID + // colTypeName type name of columnOID + // colWidth width (in bytes) of columnOID + // compressionType compression type + void writeColumnMetaData(const std::string& metaFileName, bool withHWM, OID columnOID, uint16_t dbRoot, + uint32_t partition, uint16_t segment, HWM lastLocalHwm, + execplan::CalpontSystemCatalog::ColDataType colType, + const std::string& colTypeName, int colWidth, int compressionType); - // Save dictionary store meta data to the currently open file. - // This is the Shared-Nothing version of this function. - // dictionaryStoreOID dictionary store OID to be saved - // dbRoot dbRoot of store file - // partition partition of store file - // segment segment of store file - // localHwm current local HWM for specified partition and seg file - // compressionType compression type - void writeDictionaryStoreMetaData ( - OID columnOID, - OID dictionaryStoreOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - HWM localHwm, - int compressionType ); + // Save dictionary store meta data to the currently open file. + // This is the Shared-Nothing version of this function. + // dictionaryStoreOID dictionary store OID to be saved + // dbRoot dbRoot of store file + // partition partition of store file + // segment segment of store file + // localHwm current local HWM for specified partition and seg file + // compressionType compression type + void writeDictionaryStoreMetaData(OID columnOID, OID dictionaryStoreOID, uint16_t dbRoot, + uint32_t partition, uint16_t segment, HWM localHwm, int compressionType); - // For first extent stripe in a partition, this function is used to - // to log a marker to denote a trailing segment file that does not exist. - // This is the Shared-Nothing version of this function. - // dictionaryStoreOID dictionary store OID to be saved - // dbRoot dbRoot of store file - // partition partition of store file - // segment segment of store file - // compressionType compression type - void writeDictionaryStoreMetaNoDataMarker ( - OID columnOID, - OID dictionaryStoreOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - int compressionType ); + // For first extent stripe in a partition, this function is used to + // to log a marker to denote a trailing segment file that does not exist. + // This is the Shared-Nothing version of this function. + // dictionaryStoreOID dictionary store OID to be saved + // dbRoot dbRoot of store file + // partition partition of store file + // segment segment of store file + // compressionType compression type + void writeDictionaryStoreMetaNoDataMarker(OID columnOID, OID dictionaryStoreOID, uint16_t dbRoot, + uint32_t partition, uint16_t segment, int compressionType); - // This function must be thread-safe since it is called indirectly by - // backupDctnryHWMChunk() (through backupHWMChunk()). - int writeHWMChunk ( - bool bColumnFile, - OID columnOID, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - const unsigned char* compressedOutBuf, - uint64_t chunkSize, - uint64_t fileSize, - HWM chunkHwm, - std::string& errMsg) const; - void printDctnryChunkList(const RBChunkInfo& rbChk, const char* action); + // This function must be thread-safe since it is called indirectly by + // backupDctnryHWMChunk() (through backupHWMChunk()). + int writeHWMChunk(bool bColumnFile, OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment, + const unsigned char* compressedOutBuf, uint64_t chunkSize, uint64_t fileSize, + HWM chunkHwm, std::string& errMsg) const; + void printDctnryChunkList(const RBChunkInfo& rbChk, const char* action); - IDBDataFile* fMetaDataFile; // current meta data file to write - std::ostringstream fMetaDataStream; // adapter for IDBDataFile - std::map fMetaFileNames;//map of dbroots to metafiles - std::string fAppDesc; // description of application user - Log* fLog; // import log file - bool fCreatedSubDir; // has subdir path been created - RBChunkSet fRBChunkDctnrySet; // Dctnry HWM chunk info - boost::mutex fRBChunkDctnryMutex;//Mutex lock for RBChunkSet - OID fTableOID; // OID of relevant table - std::string fTableName; // Name of relevant table + IDBDataFile* fMetaDataFile; // current meta data file to write + std::ostringstream fMetaDataStream; // adapter for IDBDataFile + std::map fMetaFileNames; // map of dbroots to metafiles + std::string fAppDesc; // description of application user + Log* fLog; // import log file + bool fCreatedSubDir; // has subdir path been created + RBChunkSet fRBChunkDctnrySet; // Dctnry HWM chunk info + boost::mutex fRBChunkDctnryMutex; // Mutex lock for RBChunkSet + OID fTableOID; // OID of relevant table + std::string fTableName; // Name of relevant table }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // WE_RBMETAWRITER_H_ +#endif // WE_RBMETAWRITER_H_ diff --git a/writeengine/shared/we_simplesyslog.cpp b/writeengine/shared/we_simplesyslog.cpp index 7d7a8ff43..e27100afc 100644 --- a/writeengine/shared/we_simplesyslog.cpp +++ b/writeengine/shared/we_simplesyslog.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_simplesyslog.cpp 4607 2013-04-11 21:38:09Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_simplesyslog.cpp 4607 2013-04-11 21:38:09Z rdempsey $ + * + *******************************************************************************/ #include "we_simplesyslog.h" @@ -37,75 +37,73 @@ namespace WriteEngine /* static */ SimpleSysLog* SimpleSysLog::instance() { - if ( !fSysLogger ) - fSysLogger = new SimpleSysLog(); + if (!fSysLogger) + fSysLogger = new SimpleSysLog(); - return fSysLogger; + return fSysLogger; } //------------------------------------------------------------------------------ // SimpleSysLog constructor. //------------------------------------------------------------------------------ -SimpleSysLog::SimpleSysLog() : fLoggingID( SUBSYSTEM_ID_WE ) +SimpleSysLog::SimpleSysLog() : fLoggingID(SUBSYSTEM_ID_WE) { } //------------------------------------------------------------------------------ // Reset LoggingID (in order to set the subsystem id) //------------------------------------------------------------------------------ -void SimpleSysLog::setLoggingID( const logging::LoggingID& loggingID ) +void SimpleSysLog::setLoggingID(const logging::LoggingID& loggingID) { - fLoggingID = loggingID; + fLoggingID = loggingID; } //------------------------------------------------------------------------------ // Log arguments (msgArgs) for specified msgId to the requested log (logType). //------------------------------------------------------------------------------ -void SimpleSysLog::logMsg( const logging::Message::Args& msgArgs, - logging::LOG_TYPE logType, - logging::Message::MessageID msgId ) +void SimpleSysLog::logMsg(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType, + logging::Message::MessageID msgId) { - logging::MessageLog ml( fLoggingID ); + logging::MessageLog ml(fLoggingID); - logging::Message m(msgId); - m.format(msgArgs); + logging::Message m(msgId); + m.format(msgArgs); - boost::mutex::scoped_lock lk(fWriteLockMutex); + boost::mutex::scoped_lock lk(fWriteLockMutex); - switch (logType) + switch (logType) + { + case logging::LOG_TYPE_DEBUG: { - case logging::LOG_TYPE_DEBUG: - { - ml.logDebugMessage(m); - break; - } - - case logging::LOG_TYPE_INFO: - default: - { - ml.logInfoMessage(m); - break; - } - - case logging::LOG_TYPE_WARNING: - { - ml.logWarningMessage(m); - break; - } - - case logging::LOG_TYPE_ERROR: - { - ml.logErrorMessage(m); - break; - } - - case logging::LOG_TYPE_CRITICAL: - { - ml.logCriticalMessage(m); - break; - } + ml.logDebugMessage(m); + break; } + + case logging::LOG_TYPE_INFO: + default: + { + ml.logInfoMessage(m); + break; + } + + case logging::LOG_TYPE_WARNING: + { + ml.logWarningMessage(m); + break; + } + + case logging::LOG_TYPE_ERROR: + { + ml.logErrorMessage(m); + break; + } + + case logging::LOG_TYPE_CRITICAL: + { + ml.logCriticalMessage(m); + break; + } + } } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_simplesyslog.h b/writeengine/shared/we_simplesyslog.h index d7668f6b2..35ab9c638 100644 --- a/writeengine/shared/we_simplesyslog.h +++ b/writeengine/shared/we_simplesyslog.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_simplesyslog.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_simplesyslog.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_SIMPLESYSLOG_H_ @@ -38,7 +38,6 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** * @brief SimpleSysLog class is a simple logger that only logs to syslog. * @@ -47,36 +46,35 @@ namespace WriteEngine */ class SimpleSysLog { -public: - /** - * @brief Singleton accessor. - */ - EXPORT static SimpleSysLog* instance(); + public: + /** + * @brief Singleton accessor. + */ + EXPORT static SimpleSysLog* instance(); - /** - * @brief Modify the LoggingID to be used. Mainly used to control the - * subsystem ID. - */ - EXPORT void setLoggingID( const logging::LoggingID& loggingID ); + /** + * @brief Modify the LoggingID to be used. Mainly used to control the + * subsystem ID. + */ + EXPORT void setLoggingID(const logging::LoggingID& loggingID); - /** - * @brief Function that logs a syslog msg. - */ - EXPORT void logMsg( const logging::Message::Args& msgArgs, - logging::LOG_TYPE logType, - logging::Message::MessageID msgId ); + /** + * @brief Function that logs a syslog msg. + */ + EXPORT void logMsg(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType, + logging::Message::MessageID msgId); -private: - SimpleSysLog( ); - SimpleSysLog( const SimpleSysLog& ); - SimpleSysLog& operator= ( const SimpleSysLog& ); + private: + SimpleSysLog(); + SimpleSysLog(const SimpleSysLog&); + SimpleSysLog& operator=(const SimpleSysLog&); - static SimpleSysLog* fSysLogger; - logging::LoggingID fLoggingID; - boost::mutex fWriteLockMutex; // logging mutex + static SimpleSysLog* fSysLogger; + logging::LoggingID fLoggingID; + boost::mutex fWriteLockMutex; // logging mutex }; #undef EXPORT -} //end of namespace -#endif // _WE_SIMPLESYSLOG_H_ +} // namespace WriteEngine +#endif // _WE_SIMPLESYSLOG_H_ diff --git a/writeengine/shared/we_stats.cpp b/writeengine/shared/we_stats.cpp index 2ed0aad72..54d11b451 100644 --- a/writeengine/shared/we_stats.cpp +++ b/writeengine/shared/we_stats.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_stats.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_stats.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #include @@ -28,16 +28,16 @@ using namespace std; namespace WriteEngine { #ifdef PROFILE -/* static */ bool Stats::fProfiling = false; -/* static */ boost::mutex Stats::fRegisterReaderMutex; -/* static */ boost::mutex Stats::fRegisterParseMutex; -/* static */ std::vector Stats::fReadProfThreads; -/* static */ std::vector Stats::fParseProfThreads; +/* static */ bool Stats::fProfiling = false; +/* static */ boost::mutex Stats::fRegisterReaderMutex; +/* static */ boost::mutex Stats::fRegisterParseMutex; +/* static */ std::vector Stats::fReadProfThreads; +/* static */ std::vector Stats::fParseProfThreads; /* static */ std::vector Stats::fReadStopWatch; /* static */ std::vector Stats::fParseStopWatch; #endif -struct IoStats Stats::m_ioStats = { 0, 0 }; +struct IoStats Stats::m_ioStats = {0, 0}; bool Stats::m_bUseStats = false; /*********************************************************** * DESCRIPTION: @@ -47,12 +47,12 @@ bool Stats::m_bUseStats = false; * RETURN: * none ***********************************************************/ -void Stats::incIoBlockRead( const int blockNum ) +void Stats::incIoBlockRead(const int blockNum) { - if ( !m_bUseStats ) - return; + if (!m_bUseStats) + return; - m_ioStats.blockRead += blockNum; + m_ioStats.blockRead += blockNum; } /*********************************************************** @@ -63,12 +63,12 @@ void Stats::incIoBlockRead( const int blockNum ) * RETURN: * none ***********************************************************/ -void Stats::incIoBlockWrite( const int blockNum ) +void Stats::incIoBlockWrite(const int blockNum) { - if ( !m_bUseStats ) - return; + if (!m_bUseStats) + return; - m_ioStats.blockWrite += blockNum; + m_ioStats.blockWrite += blockNum; } #ifdef PROFILE @@ -87,18 +87,18 @@ void Stats::incIoBlockWrite( const int blockNum ) ***********************************************************/ void Stats::enableProfiling(int nReadThreads, int nParseThreads) { - fProfiling = true; + fProfiling = true; - // @bug 2625: pre-reserve space for our vectors; else we could have a race - // condition whereby one parsing thread is adding itself to the vectors - // and thus "growing" the vector (in registerParseProfThread), at the - // same time that another parsing thread is reading the vector in parse- - // Event(). By pre-reserving the space, the vectors won't be growing, - // thus eliminating the problem with this race condition. - fReadProfThreads.reserve ( nReadThreads ); - fReadStopWatch.reserve ( nReadThreads ); - fParseProfThreads.reserve( nParseThreads ); - fParseStopWatch.reserve ( nParseThreads ); + // @bug 2625: pre-reserve space for our vectors; else we could have a race + // condition whereby one parsing thread is adding itself to the vectors + // and thus "growing" the vector (in registerParseProfThread), at the + // same time that another parsing thread is reading the vector in parse- + // Event(). By pre-reserving the space, the vectors won't be growing, + // thus eliminating the problem with this race condition. + fReadProfThreads.reserve(nReadThreads); + fReadStopWatch.reserve(nReadThreads); + fParseProfThreads.reserve(nParseThreads); + fParseStopWatch.reserve(nParseThreads); } /*********************************************************** @@ -109,13 +109,13 @@ void Stats::enableProfiling(int nReadThreads, int nParseThreads) * RETURN: * none ***********************************************************/ -void Stats::registerReadProfThread( ) +void Stats::registerReadProfThread() { - boost::mutex::scoped_lock lk(fRegisterReaderMutex); + boost::mutex::scoped_lock lk(fRegisterReaderMutex); - fReadProfThreads.push_back( pthread_self() ); - logging::StopWatch readStopWatch; - fReadStopWatch.push_back ( readStopWatch ); + fReadProfThreads.push_back(pthread_self()); + logging::StopWatch readStopWatch; + fReadStopWatch.push_back(readStopWatch); } /*********************************************************** @@ -126,13 +126,13 @@ void Stats::registerReadProfThread( ) * RETURN: * none ***********************************************************/ -void Stats::registerParseProfThread( ) +void Stats::registerParseProfThread() { - boost::mutex::scoped_lock lk(fRegisterParseMutex); + boost::mutex::scoped_lock lk(fRegisterParseMutex); - fParseProfThreads.push_back( pthread_self() ); - logging::StopWatch parseStopWatch; - fParseStopWatch.push_back ( parseStopWatch ); + fParseProfThreads.push_back(pthread_self()); + logging::StopWatch parseStopWatch; + fParseStopWatch.push_back(parseStopWatch); } /*********************************************************** @@ -145,25 +145,25 @@ void Stats::registerParseProfThread( ) * RETURN: * none ***********************************************************/ -void Stats::readEvent ( const std::string& eventString, bool start ) +void Stats::readEvent(const std::string& eventString, bool start) { - if (fProfiling) + if (fProfiling) + { + pthread_t thread = pthread_self(); + + for (unsigned i = 0; i < fReadProfThreads.size(); i++) { - pthread_t thread = pthread_self(); + if (fReadProfThreads[i] == thread) + { + if (start) + fReadStopWatch[i].start(eventString); + else + fReadStopWatch[i].stop(eventString); - for (unsigned i = 0; i < fReadProfThreads.size(); i++) - { - if (fReadProfThreads[i] == thread) - { - if (start) - fReadStopWatch[i].start( eventString ); - else - fReadStopWatch[i].stop ( eventString ); - - break; - } - } + break; + } } + } } /*********************************************************** @@ -176,25 +176,25 @@ void Stats::readEvent ( const std::string& eventString, bool start ) * RETURN: * none ***********************************************************/ -void Stats::parseEvent ( const std::string& eventString, bool start ) +void Stats::parseEvent(const std::string& eventString, bool start) { - if (fProfiling) + if (fProfiling) + { + pthread_t thread = pthread_self(); + + for (unsigned i = 0; i < fParseProfThreads.size(); i++) { - pthread_t thread = pthread_self(); + if (fParseProfThreads[i] == thread) + { + if (start) + fParseStopWatch[i].start(eventString); + else + fParseStopWatch[i].stop(eventString); - for (unsigned i = 0; i < fParseProfThreads.size(); i++) - { - if (fParseProfThreads[i] == thread) - { - if (start) - fParseStopWatch[i].start( eventString ); - else - fParseStopWatch[i].stop ( eventString ); - - break; - } - } + break; + } } + } } /*********************************************************** @@ -205,33 +205,30 @@ void Stats::parseEvent ( const std::string& eventString, bool start ) * RETURN: * none ***********************************************************/ -void Stats::printProfilingResults ( ) +void Stats::printProfilingResults() { - if (fProfiling) + if (fProfiling) + { + std::cout << endl; + + for (unsigned j = 0; j < fReadStopWatch.size(); j++) { - std::cout << endl; - - for (unsigned j = 0; j < fReadStopWatch.size(); j++) - { - std::cout << "Execution Stats for Read Thread " << j << " (" << - fReadProfThreads[j] << ")" << std::endl << - "-------------------------------" << std::endl; - fReadStopWatch[j].finish(); - std::cout << std::endl; - } - - for (unsigned j = 0; j < fParseStopWatch.size(); j++) - { - std::cout << "Execution Stats for Parse Thread " << j << " (" << - fParseProfThreads[j] << ")" << std::endl << - "--------------------------------" << std::endl; - fParseStopWatch[j].finish(); - std::cout << std::endl; - } - + std::cout << "Execution Stats for Read Thread " << j << " (" << fReadProfThreads[j] << ")" << std::endl + << "-------------------------------" << std::endl; + fReadStopWatch[j].finish(); + std::cout << std::endl; } + + for (unsigned j = 0; j < fParseStopWatch.size(); j++) + { + std::cout << "Execution Stats for Parse Thread " << j << " (" << fParseProfThreads[j] << ")" + << std::endl + << "--------------------------------" << std::endl; + fParseStopWatch[j].finish(); + std::cout << std::endl; + } + } } #endif -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/shared/we_stats.h b/writeengine/shared/we_stats.h index f5e8d3c84..6297c4f26 100644 --- a/writeengine/shared/we_stats.h +++ b/writeengine/shared/we_stats.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_stats.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_stats.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_STATS_H_ @@ -34,150 +34,153 @@ namespace WriteEngine { struct IoStats { - long blockRead; - long blockWrite; + long blockRead; + long blockWrite; }; /** Class Stats */ class Stats { -public: - /** - * @brief Constructor - */ - Stats() {} + public: + /** + * @brief Constructor + */ + Stats() + { + } - /** - * @brief Default Destructor - */ - ~Stats() {} + /** + * @brief Default Destructor + */ + ~Stats() + { + } - /** - * @brief I/O - */ - static long getIoBlockRead() - { - return m_ioStats.blockRead; - } - static long getIoBlockWrite() - { - return m_ioStats.blockWrite; - } + /** + * @brief I/O + */ + static long getIoBlockRead() + { + return m_ioStats.blockRead; + } + static long getIoBlockWrite() + { + return m_ioStats.blockWrite; + } - static void incIoBlockRead( const int blockNum = 1 ); - static void incIoBlockWrite( const int blockNum = 1 ); + static void incIoBlockRead(const int blockNum = 1); + static void incIoBlockWrite(const int blockNum = 1); - static bool getUseStats() - { - return m_bUseStats; - } - static void setUseStats( const bool flag ) - { - m_bUseStats = flag; - } + static bool getUseStats() + { + return m_bUseStats; + } + static void setUseStats(const bool flag) + { + m_bUseStats = flag; + } - static IoStats m_ioStats; // IO + static IoStats m_ioStats; // IO #ifdef PROFILE - // Prefined event labels -#define WE_STATS_ALLOC_DCT_EXTENT "AllocDctExtent" -#define WE_STATS_COMPACT_VARBINARY "CompactingVarBinary" -#define WE_STATS_COMPLETING_PARSE "CompletingParse" -#define WE_STATS_COMPLETING_READ "CompletingRead" + // Prefined event labels +#define WE_STATS_ALLOC_DCT_EXTENT "AllocDctExtent" +#define WE_STATS_COMPACT_VARBINARY "CompactingVarBinary" +#define WE_STATS_COMPLETING_PARSE "CompletingParse" +#define WE_STATS_COMPLETING_READ "CompletingRead" #define WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT "CmpColInitAbbrevExtent" -#define WE_STATS_COMPRESS_COL_INIT_BUF "CmpColInitBuf" -#define WE_STATS_COMPRESS_COL_COMPRESS "CmpColCompress" -#define WE_STATS_COMPRESS_COL_FINISH_EXTENT "CmpColFinishExtent" -#define WE_STATS_COMPRESS_DCT_INIT_BUF "CmpDctInitBuf" -#define WE_STATS_COMPRESS_DCT_COMPRESS "CmpDctCompress" -#define WE_STATS_COMPRESS_DCT_SEEKO_CHUNK "CmpDctSeekOutputChunk" -#define WE_STATS_COMPRESS_DCT_WRITE_CHUNK "CmpDctWriteChunk" -#define WE_STATS_COMPRESS_DCT_SEEKO_HDR "CmpDctSeekOutputHdr" -#define WE_STATS_COMPRESS_DCT_WRITE_HDR "CmpDctWriteHdr" -#define WE_STATS_COMPRESS_DCT_BACKUP_CHUNK "CmpDctBackupChunk" -#define WE_STATS_CREATE_COL_EXTENT "CreateColExtent" -#define WE_STATS_CREATE_DCT_EXTENT "CreateDctExtent" -#define WE_STATS_EXPAND_COL_EXTENT "ExpandColExtent" -#define WE_STATS_EXPAND_DCT_EXTENT "ExpandDctExtent" -#define WE_STATS_FLUSH_PRIMPROC_BLOCKS "FlushPrimProcBlocks" -#define WE_STATS_INIT_COL_EXTENT "InitColExtent" -#define WE_STATS_INIT_DCT_EXTENT "InitDctExtent" -#define WE_STATS_OPEN_DCT_FILE "OpenDctFile" -#define WE_STATS_PARSE_COL "ParseCol" -#define WE_STATS_PARSE_DCT "ParseDct" -#define WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK "ParseDctSeekExtentBlk" -#define WE_STATS_READ_INTO_BUF "ReadIntoBuf" -#define WE_STATS_RESIZE_OUT_BUF "ResizeOutBuf" -#define WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH "WaitForIntermediateFlush" -#define WE_STATS_WAIT_FOR_READ_BUF "WaitForReadBuf" -#define WE_STATS_WAIT_TO_COMPLETE_PARSE "WaitCompleteParse" -#define WE_STATS_WAIT_TO_COMPLETE_READ "WaitCompleteRead" -#define WE_STATS_WAIT_TO_CREATE_COL_EXTENT "WaitCreateColExtent" -#define WE_STATS_WAIT_TO_CREATE_DCT_EXTENT "WaitCreateDctExtent" -#define WE_STATS_WAIT_TO_EXPAND_COL_EXTENT "WaitExpandColExtent" -#define WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT "WaitExpandDctExtent" -#define WE_STATS_WAIT_TO_PARSE_DCT "WaitParseDct" -#define WE_STATS_WAIT_TO_RELEASE_OUT_BUF "WaitReleaseOutBuf" -#define WE_STATS_WAIT_TO_RESERVE_OUT_BUF "WaitReserveOutBuf" -#define WE_STATS_WAIT_TO_RESIZE_OUT_BUF "WaitResizeOutBuf" -#define WE_STATS_WAIT_TO_SELECT_COL "WaitSelectCol" -#define WE_STATS_WAIT_TO_SELECT_TBL "WaitSelectTbl" -#define WE_STATS_WRITE_COL "WriteCol" -#define WE_STATS_WRITE_DCT "WriteDct" +#define WE_STATS_COMPRESS_COL_INIT_BUF "CmpColInitBuf" +#define WE_STATS_COMPRESS_COL_COMPRESS "CmpColCompress" +#define WE_STATS_COMPRESS_COL_FINISH_EXTENT "CmpColFinishExtent" +#define WE_STATS_COMPRESS_DCT_INIT_BUF "CmpDctInitBuf" +#define WE_STATS_COMPRESS_DCT_COMPRESS "CmpDctCompress" +#define WE_STATS_COMPRESS_DCT_SEEKO_CHUNK "CmpDctSeekOutputChunk" +#define WE_STATS_COMPRESS_DCT_WRITE_CHUNK "CmpDctWriteChunk" +#define WE_STATS_COMPRESS_DCT_SEEKO_HDR "CmpDctSeekOutputHdr" +#define WE_STATS_COMPRESS_DCT_WRITE_HDR "CmpDctWriteHdr" +#define WE_STATS_COMPRESS_DCT_BACKUP_CHUNK "CmpDctBackupChunk" +#define WE_STATS_CREATE_COL_EXTENT "CreateColExtent" +#define WE_STATS_CREATE_DCT_EXTENT "CreateDctExtent" +#define WE_STATS_EXPAND_COL_EXTENT "ExpandColExtent" +#define WE_STATS_EXPAND_DCT_EXTENT "ExpandDctExtent" +#define WE_STATS_FLUSH_PRIMPROC_BLOCKS "FlushPrimProcBlocks" +#define WE_STATS_INIT_COL_EXTENT "InitColExtent" +#define WE_STATS_INIT_DCT_EXTENT "InitDctExtent" +#define WE_STATS_OPEN_DCT_FILE "OpenDctFile" +#define WE_STATS_PARSE_COL "ParseCol" +#define WE_STATS_PARSE_DCT "ParseDct" +#define WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK "ParseDctSeekExtentBlk" +#define WE_STATS_READ_INTO_BUF "ReadIntoBuf" +#define WE_STATS_RESIZE_OUT_BUF "ResizeOutBuf" +#define WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH "WaitForIntermediateFlush" +#define WE_STATS_WAIT_FOR_READ_BUF "WaitForReadBuf" +#define WE_STATS_WAIT_TO_COMPLETE_PARSE "WaitCompleteParse" +#define WE_STATS_WAIT_TO_COMPLETE_READ "WaitCompleteRead" +#define WE_STATS_WAIT_TO_CREATE_COL_EXTENT "WaitCreateColExtent" +#define WE_STATS_WAIT_TO_CREATE_DCT_EXTENT "WaitCreateDctExtent" +#define WE_STATS_WAIT_TO_EXPAND_COL_EXTENT "WaitExpandColExtent" +#define WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT "WaitExpandDctExtent" +#define WE_STATS_WAIT_TO_PARSE_DCT "WaitParseDct" +#define WE_STATS_WAIT_TO_RELEASE_OUT_BUF "WaitReleaseOutBuf" +#define WE_STATS_WAIT_TO_RESERVE_OUT_BUF "WaitReserveOutBuf" +#define WE_STATS_WAIT_TO_RESIZE_OUT_BUF "WaitResizeOutBuf" +#define WE_STATS_WAIT_TO_SELECT_COL "WaitSelectCol" +#define WE_STATS_WAIT_TO_SELECT_TBL "WaitSelectTbl" +#define WE_STATS_WRITE_COL "WriteCol" +#define WE_STATS_WRITE_DCT "WriteDct" - // Functions used to support performance profiling - static void enableProfiling(int nReadThreads, int nParseThreads); - static void registerReadProfThread ( ); - static void registerParseProfThread( ); - static void startReadEvent ( const std::string& eventString ) - { - readEvent ( eventString, true ); - } - static void stopReadEvent ( const std::string& eventString ) - { - readEvent ( eventString, false ); - } - static void startParseEvent( const std::string& eventString ) - { - parseEvent( eventString, true ); - } - static void stopParseEvent ( const std::string& eventString ) - { - parseEvent( eventString, false ); - } - static void printProfilingResults( ); + // Functions used to support performance profiling + static void enableProfiling(int nReadThreads, int nParseThreads); + static void registerReadProfThread(); + static void registerParseProfThread(); + static void startReadEvent(const std::string& eventString) + { + readEvent(eventString, true); + } + static void stopReadEvent(const std::string& eventString) + { + readEvent(eventString, false); + } + static void startParseEvent(const std::string& eventString) + { + parseEvent(eventString, true); + } + static void stopParseEvent(const std::string& eventString) + { + parseEvent(eventString, false); + } + static void printProfilingResults(); #endif -private: - static bool m_bUseStats; // Use statistic flag + private: + static bool m_bUseStats; // Use statistic flag #ifdef PROFILE - // Data members and functions used to support performance profiling - static bool fProfiling; // Is profiling enabled + // Data members and functions used to support performance profiling + static bool fProfiling; // Is profiling enabled - // Protect concurrent addition of Readers - static boost::mutex fRegisterReaderMutex; + // Protect concurrent addition of Readers + static boost::mutex fRegisterReaderMutex; - // Protect concurrent addition of Parsers - static boost::mutex fRegisterParseMutex; + // Protect concurrent addition of Parsers + static boost::mutex fRegisterParseMutex; - // Read threads to be profiled - static std::vector fReadProfThreads; + // Read threads to be profiled + static std::vector fReadProfThreads; - // Parse threads to be profiled - static std::vector fParseProfThreads; + // Parse threads to be profiled + static std::vector fParseProfThreads; - // Track/profile Read events - static std::vector fReadStopWatch; + // Track/profile Read events + static std::vector fReadStopWatch; - // Track/profile Parse events - static std::vector fParseStopWatch; + // Track/profile Parse events + static std::vector fParseStopWatch; - static void readEvent ( const std::string& eventString, bool start ); - static void parseEvent ( const std::string& eventString, bool start ); + static void readEvent(const std::string& eventString, bool start); + static void parseEvent(const std::string& eventString, bool start); #endif }; - -} //end of namespace -#endif // _WE_STATIS_H_ +} // namespace WriteEngine +#endif // _WE_STATIS_H_ diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index e1f5eaf7a..76cbe677c 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -20,7 +20,6 @@ /** @file */ - #ifndef _WE_TYPE_H_ #define _WE_TYPE_H_ @@ -54,156 +53,169 @@ typedef idbdatafile::IDBDataFile IDBDataFile; /************************************************************************ * Type definitions ************************************************************************/ -typedef uint32_t OID; /** @brief Object ID */ -typedef uint32_t FID; /** @brief File ID */ -typedef uint64_t RID; /** @brief Row ID */ -typedef uint32_t TxnID; /** @brief Transaction ID (New)*/ -typedef uint32_t HWM; /** @brief high water mark */ +typedef uint32_t OID; /** @brief Object ID */ +typedef uint32_t FID; /** @brief File ID */ +typedef uint64_t RID; /** @brief Row ID */ +typedef uint32_t TxnID; /** @brief Transaction ID (New)*/ +typedef uint32_t HWM; /** @brief high water mark */ /************************************************************************ * Type enumerations ************************************************************************/ -enum DebugLevel /** @brief Debug level type */ +enum DebugLevel /** @brief Debug level type */ { - DEBUG_0 = 0, /** @brief No debug info */ - DEBUG_1 = 1, /** @brief Summary level debug */ - DEBUG_2 = 2, /** @brief Moderate debug */ - DEBUG_3 = 3, /** @brief Detail debug */ + DEBUG_0 = 0, /** @brief No debug info */ + DEBUG_1 = 1, /** @brief Summary level debug */ + DEBUG_2 = 2, /** @brief Moderate debug */ + DEBUG_3 = 3, /** @brief Detail debug */ }; // INFO2 only goes to log file unless '-i' cmd line arg is specified, // in which case the msg will also get logged to the console. // All other messages always get logged to the log file and the console. -enum MsgLevel /** @brief Message level */ +enum MsgLevel /** @brief Message level */ { - MSGLVL_INFO1 = 0, /** @brief Basic Information level*/ - MSGLVL_INFO2 = 1, /** @brief More Information level */ - MSGLVL_WARNING = 2, /** @brief Warning level */ - MSGLVL_ERROR = 3, /** @brief Error level */ - MSGLVL_CRITICAL = 4, /** @brief Critical level */ + MSGLVL_INFO1 = 0, /** @brief Basic Information level*/ + MSGLVL_INFO2 = 1, /** @brief More Information level */ + MSGLVL_WARNING = 2, /** @brief Warning level */ + MSGLVL_ERROR = 3, /** @brief Error level */ + MSGLVL_CRITICAL = 4, /** @brief Critical level */ }; -enum OpType /** @brief Operation type */ +enum OpType /** @brief Operation type */ { - NOOP = 0, /** @brief No oper */ - INSERT = 1, /** @brief Insert */ - UPDATE = 2, /** @brief Update */ - DELETE = 4, /** @brief Delete */ - QUERY = 8, /** @brief Query */ + NOOP = 0, /** @brief No oper */ + INSERT = 1, /** @brief Insert */ + UPDATE = 2, /** @brief Update */ + DELETE = 4, /** @brief Delete */ + QUERY = 8, /** @brief Query */ }; -enum ColType /** @brief Column type enumeration*/ +enum ColType /** @brief Column type enumeration*/ { -// WR_BIT = 1, /** @brief Bit */ - WR_BYTE = 2, /** @brief Byte */ - WR_SHORT = 3, /** @brief Short */ - WR_INT = 4, /** @brief Int */ -// WR_LONG = 5, /** @brief Long */ - WR_LONGLONG = 6, /** @brief Long long*/ - WR_FLOAT = 7, /** @brief Float */ - WR_DOUBLE = 8, /** @brief Double */ - WR_CHAR = 9, /** @brief Char */ - WR_TOKEN = 10, /** @brief Token */ - WR_BLOB = 11, /** @brief BLOB */ - WR_VARBINARY = 12, /** @brief VARBINARY */ - WR_UBYTE = 13, /** @brief Unsigned Byte */ - WR_USHORT = 14, /** @brief Unsigned Short */ - WR_UINT = 15, /** @brief Unsigned Int */ - WR_ULONGLONG = 16, /** @brief Unsigned Long long*/ - WR_TEXT = 17, /** @brief TEXT */ - WR_MEDINT = 18, /** @brief Medium Int */ - WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ - WR_BINARY = 20 /** @brief BINARY */ - // WIP - //WR_INT128 + // WR_BIT = 1, /** @brief Bit */ + WR_BYTE = 2, /** @brief Byte */ + WR_SHORT = 3, /** @brief Short */ + WR_INT = 4, /** @brief Int */ + // WR_LONG = 5, /** @brief Long */ + WR_LONGLONG = 6, /** @brief Long long*/ + WR_FLOAT = 7, /** @brief Float */ + WR_DOUBLE = 8, /** @brief Double */ + WR_CHAR = 9, /** @brief Char */ + WR_TOKEN = 10, /** @brief Token */ + WR_BLOB = 11, /** @brief BLOB */ + WR_VARBINARY = 12, /** @brief VARBINARY */ + WR_UBYTE = 13, /** @brief Unsigned Byte */ + WR_USHORT = 14, /** @brief Unsigned Short */ + WR_UINT = 15, /** @brief Unsigned Int */ + WR_ULONGLONG = 16, /** @brief Unsigned Long long*/ + WR_TEXT = 17, /** @brief TEXT */ + WR_MEDINT = 18, /** @brief Medium Int */ + WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ + WR_BINARY = 20 /** @brief BINARY */ + // WIP + // WR_INT128 }; // Describes relation of field to column for a bulk load -enum BulkFldColRel { BULK_FLDCOL_COLUMN_FIELD, // map input field to db col - BULK_FLDCOL_COLUMN_DEFAULT,// import def val to db col - BULK_FLDCOL_IGNORE_FIELD - };// ignore fld in import file +enum BulkFldColRel +{ + BULK_FLDCOL_COLUMN_FIELD, // map input field to db col + BULK_FLDCOL_COLUMN_DEFAULT, // import def val to db col + BULK_FLDCOL_IGNORE_FIELD +}; // ignore fld in import file // Bulk Load Mode (ex: local vs remote, single src vs multiple src files) -enum BulkModeType { BULK_MODE_REMOTE_SINGLE_SRC = 1, - BULK_MODE_REMOTE_MULTIPLE_SRC = 2, - BULK_MODE_LOCAL = 3 - }; +enum BulkModeType +{ + BULK_MODE_REMOTE_SINGLE_SRC = 1, + BULK_MODE_REMOTE_MULTIPLE_SRC = 2, + BULK_MODE_LOCAL = 3 +}; // Import Mode 0-text Import (default) // 1-Binary Import with NULL values // 2-Binary Import with saturated NULL values -enum ImportDataMode { IMPORT_DATA_TEXT = 0, - IMPORT_DATA_BIN_ACCEPT_NULL = 1, - IMPORT_DATA_BIN_SAT_NULL = 2 - }; +enum ImportDataMode +{ + IMPORT_DATA_TEXT = 0, + IMPORT_DATA_BIN_ACCEPT_NULL = 1, + IMPORT_DATA_BIN_SAT_NULL = 2 +}; /** * the set of Calpont column data type names; MUST match ColDataType in * calpontsystemcatalog.h. */ -const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE][20] = +const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE][20] = {"bit", + "tinyint", + "char", + "smallint", + "decimal", + "medint", + "integer", + "float", + "date", + "bigint", + "double", + "datetime", + "varchar", + "varbinary", + "clob", + "blob", + "unsigned-tinyint", + "unsigned-smallint", + "unsigned-decimal", + "unsigned-med int", + "unsigned-int", + "unsigned-float", + "unsigned-bigint", + "unsigned-double", + "text", + "time", + "timestamp"}; + +enum FuncType { - "bit", - "tinyint", - "char", - "smallint", - "decimal", - "medint", - "integer", - "float", - "date", - "bigint", - "double", - "datetime", - "varchar", - "varbinary", - "clob", - "blob", - "unsigned-tinyint", - "unsigned-smallint", - "unsigned-decimal", - "unsigned-med int", - "unsigned-int", - "unsigned-float", - "unsigned-bigint", - "unsigned-double", - "text", - "time", - "timestamp" + FUNC_WRITE_ENGINE, + FUNC_INDEX, + FUNC_DICTIONARY }; -enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY }; - -enum CacheListType { FREE_LIST, LRU_LIST, WRITE_LIST }; /** @brief List type */ +enum CacheListType +{ + FREE_LIST, + LRU_LIST, + WRITE_LIST +}; /** @brief List type */ /************************************************************************ * struct data block structure ************************************************************************/ -struct DataBlock /** @brief Data block structure */ +struct DataBlock /** @brief Data block structure */ { - long no; /** @brief block number */ - uint64_t lbid; /** @brief lbid */ - bool dirty; /** @brief block dirty flag */ - int state; /** @brief initialized 0, read 1 , modified 2 */ - unsigned char data[BYTE_PER_BLOCK];/** @brief data buffer */ - DataBlock() - { - dirty = false; /** @brief constructor */ - memset( data, 0, BYTE_PER_BLOCK ); - } + long no; /** @brief block number */ + uint64_t lbid; /** @brief lbid */ + bool dirty; /** @brief block dirty flag */ + int state; /** @brief initialized 0, read 1 , modified 2 */ + unsigned char data[BYTE_PER_BLOCK]; /** @brief data buffer */ + DataBlock() + { + dirty = false; /** @brief constructor */ + memset(data, 0, BYTE_PER_BLOCK); + } }; -struct DataSubBlock /** @brief Data subblock structure*/ +struct DataSubBlock /** @brief Data subblock structure*/ { - long no; /** @brief sub block number */ - bool dirty; /** @brief block dirty flag */ - unsigned char data[BYTE_PER_SUBBLOCK]; /** @brief data buffer */ - DataSubBlock() - { - dirty = false; /** @brief constructor */ - memset( data, 0, BYTE_PER_SUBBLOCK ); - } + long no; /** @brief sub block number */ + bool dirty; /** @brief block dirty flag */ + unsigned char data[BYTE_PER_SUBBLOCK]; /** @brief data buffer */ + DataSubBlock() + { + dirty = false; /** @brief constructor */ + memset(data, 0, BYTE_PER_SUBBLOCK); + } }; /************************************************************************ @@ -213,303 +225,358 @@ struct DataSubBlock /** @brief Data subblock structure*/ * oid and fid replicate one another. oid mostly used by index, cache, * and dictionary. fid mostly used by colop and bulk. ************************************************************************/ -struct File /** @brief File structure */ +struct File /** @brief File structure */ { - OID oid; /** @brief Oid */ - FID fid; /** @brief File id */ - HWM hwm; /** @brief High water mark */ - IDBDataFile* pFile; /** @brief File handle */ - uint32_t fPartition; /** @brief Partition for pFile*/ - uint16_t fSegment; /** @brief Segment for pFile */ - uint16_t fDbRoot; /** @brief DbRoot for pFile */ - std::string fSegFileName; /** @brief Current seg file path */ - File() - { - clear(); /** @brief constructor */ - } - void clear() - { - pFile = NULL; - oid = fid = hwm = 0; - fPartition = fSegment = fDbRoot = 0; - fSegFileName.clear(); - } + OID oid; /** @brief Oid */ + FID fid; /** @brief File id */ + HWM hwm; /** @brief High water mark */ + IDBDataFile* pFile; /** @brief File handle */ + uint32_t fPartition; /** @brief Partition for pFile*/ + uint16_t fSegment; /** @brief Segment for pFile */ + uint16_t fDbRoot; /** @brief DbRoot for pFile */ + std::string fSegFileName; /** @brief Current seg file path */ + File() + { + clear(); /** @brief constructor */ + } + void clear() + { + pFile = NULL; + oid = fid = hwm = 0; + fPartition = fSegment = fDbRoot = 0; + fSegFileName.clear(); + } }; /************************************************************************ * @brief Internal communication block structure ************************************************************************/ -struct CommBlock /** @brief Communication Block */ +struct CommBlock /** @brief Communication Block */ { - File file; /** @brief File structure */ - void clear() - { - file.clear(); - } + File file; /** @brief File structure */ + void clear() + { + file.clear(); + } }; /************************************************************************ * @brief column structure used to pass data in/out of we_colop functions ************************************************************************/ -struct Column /** @brief Column structure */ +struct Column /** @brief Column structure */ { - int colNo; /** @brief column number */ - int colWidth; /** @brief column width */ - ColType colType; /** @brief column type (internal use)*/ - execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (from interface)*/ - File dataFile; /** @brief column data file */ - int compressionType; /** @brief column compression type*/ - Column() : colNo(0), colWidth(0), colType(WR_INT), - colDataType(execplan::CalpontSystemCatalog::INT), - compressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) { } + int colNo; /** @brief column number */ + int colWidth; /** @brief column width */ + ColType colType; /** @brief column type (internal use)*/ + execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (from interface)*/ + File dataFile; /** @brief column data file */ + int compressionType; /** @brief column compression type*/ + Column() + : colNo(0) + , colWidth(0) + , colType(WR_INT) + , colDataType(execplan::CalpontSystemCatalog::INT) + , compressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) + { + } }; /************************************************************************ * @brief dictionary related structures (Token struct is defined in * we_typeext.h to facilitate its use in dbcon and utils/dataconvert). ************************************************************************/ -typedef struct offset_ /** @brief Offset structure */ +typedef struct offset_ /** @brief Offset structure */ { - int hdrLoc; /** @brief offset postion in hdr */ - uint16_t offset; /** @brief offset in block */ + int hdrLoc; /** @brief offset postion in hdr */ + uint16_t offset; /** @brief offset in block */ } Offset; /************************************************************************ * @brief interfaces with DDL/DML ************************************************************************/ -typedef struct colTuple_struct /** @brief Column Tuple definition*/ +typedef struct colTuple_struct /** @brief Column Tuple definition*/ { - boost::any data; /** @brief column value */ + boost::any data; /** @brief column value */ } ColTuple; -typedef std::vector ColTupleList; /** @brief column value list */ +typedef std::vector ColTupleList; /** @brief column value list */ -struct ColStruct /** @brief Column Interface Struct*/ +struct ColStruct /** @brief Column Interface Struct*/ { - OID dataOid; /** @brief column data file object id */ - int colWidth; /** @brief column width */ - bool tokenFlag; /** @brief column token flag, must be set to true if it is a token column */ - execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (for interface)*/ - ColType colType; /** @brief column type (internal use for write engine)*/ - uint32_t fColPartition; /** @brief Partition for column file */ - uint16_t fColSegment; /** @brief Segment for column file*/ - uint16_t fColDbRoot; /** @brief DBRoot for column file */ - int fCompressionType; /** @brief Compression tpye for column file */ - ColStruct() : dataOid(0), colWidth(0), /** @brief constructor */ - tokenFlag(false), colDataType(execplan::CalpontSystemCatalog::INT), colType(WR_INT), - fColPartition(0), fColSegment(0), fColDbRoot(0), - fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) { } + OID dataOid; /** @brief column data file object id */ + int colWidth; /** @brief column width */ + bool tokenFlag; /** @brief column token flag, must be set to true if it is a token column */ + execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (for interface)*/ + ColType colType; /** @brief column type (internal use for write engine)*/ + uint32_t fColPartition; /** @brief Partition for column file */ + uint16_t fColSegment; /** @brief Segment for column file*/ + uint16_t fColDbRoot; /** @brief DBRoot for column file */ + int fCompressionType; /** @brief Compression tpye for column file */ + ColStruct() + : dataOid(0) + , colWidth(0) + , /** @brief constructor */ + tokenFlag(false) + , colDataType(execplan::CalpontSystemCatalog::INT) + , colType(WR_INT) + , fColPartition(0) + , fColSegment(0) + , fColDbRoot(0) + , fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) + { + } }; -typedef std::vector ColStructList; /** @brief column struct list */ -typedef std::vector ColValueList; /** @brief column value list */ -typedef std::vector RIDList; /** @brief RID list */ -typedef std::vector CSCTypesList; /** @brief CSC column types list */ +typedef std::vector ColStructList; /** @brief column struct list */ +typedef std::vector ColValueList; /** @brief column value list */ +typedef std::vector RIDList; /** @brief RID list */ +typedef std::vector CSCTypesList; /** @brief CSC column types list */ typedef std::vector dictStr; typedef std::vector DictStrList; // dictionary -struct DctnryStruct /** @brief Dctnry Interface Struct*/ +struct DctnryStruct /** @brief Dctnry Interface Struct*/ { - OID dctnryOid; /** @brief dictionary signature file */ - OID columnOid; /** @brief corresponding column file */ - int colWidth; /** @brief string width for the dictionary column*/ - uint32_t fColPartition; /** @brief Partition for column file */ - uint16_t fColSegment; /** @brief Segment for column file */ - uint16_t fColDbRoot; /** @brief DBRoot for column file */ - int fCompressionType; /** @brief Compression tpye for column file */ - DctnryStruct() : dctnryOid(0), columnOid(0), /** @brief constructor */ - colWidth(0), - fColPartition(0), fColSegment(0), - fColDbRoot(0), fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) { } + OID dctnryOid; /** @brief dictionary signature file */ + OID columnOid; /** @brief corresponding column file */ + int colWidth; /** @brief string width for the dictionary column*/ + uint32_t fColPartition; /** @brief Partition for column file */ + uint16_t fColSegment; /** @brief Segment for column file */ + uint16_t fColDbRoot; /** @brief DBRoot for column file */ + int fCompressionType; /** @brief Compression tpye for column file */ + DctnryStruct() + : dctnryOid(0) + , columnOid(0) + , /** @brief constructor */ + colWidth(0) + , fColPartition(0) + , fColSegment(0) + , fColDbRoot(0) + , fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) + { + } }; -struct DctnryTuple /** @brief Dictionary Tuple struct*/ +struct DctnryTuple /** @brief Dictionary Tuple struct*/ { - unsigned char* sigValue; /** @brief dictionary signature value*/ - int sigSize; /** @brief dictionary signature size */ - Token token; /** @brief dictionary token */ - bool isNull; - DctnryTuple() { } - ~DctnryTuple() { } + unsigned char* sigValue; /** @brief dictionary signature value*/ + int sigSize; /** @brief dictionary signature size */ + Token token; /** @brief dictionary token */ + bool isNull; + DctnryTuple() + { + } + ~DctnryTuple() + { + } }; typedef std::vector DctColTupleList; -typedef std::vector DctnryStructList; /** @brief column struct list */ -typedef std::vector DctnryValueList; /** @brief column value list */ +typedef std::vector DctnryStructList; /** @brief column struct list */ +typedef std::vector DctnryValueList; /** @brief column value list */ /************************************************************************ * @brief Used by Bulk Load to describe a column ************************************************************************/ -struct JobColumn /** @brief Job Column Structure */ +struct JobColumn /** @brief Job Column Structure */ { - std::string colName; /** @brief column name */ - OID mapOid; /** @brief column OID */ - execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */ - ColType weType; /** @brief write engine data type */ - std::string typeName; /** @brief data type name */ - const uint8_t* emptyVal; /** @brief default empty value */ - int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */ - int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */ - int dctnryWidth; /** @brief dictionary width */ - int precision; /** @brief precision of decimal */ - int scale; /** @brief scale of decimal */ - bool fNotNull; /** @brief not null flag */ - BulkFldColRel fFldColRelation; /** @brief type of field/col relation*/ - char colType; /** @brief column type, blank is regular, D is dictionary */ - int compressionType; /** @brief compression type */ - bool autoIncFlag; /** @brief auto increment flag */ - DctnryStruct dctnry; /** @brief dictionary structure */ - int64_t fMinIntSat; /** @brief For integer type, the min saturation value */ - uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */ - double fMinDblSat; /** @brief for float/double, the min saturation value */ - double fMaxDblSat; /** @brief for float/double, the max saturation value */ - bool fWithDefault; /** @brief With default */ - long long fDefaultInt; /** @brief Integer column default */ - unsigned long long fDefaultUInt; /** @brief UnsignedInt col default*/ - double fDefaultDbl; /** @brief Dbl/Flt column default */ - int128_t fDefaultWideDecimal; /** @brief Wide decimal column default */ - std::string fDefaultChr; /** @brief Char column default */ - JobColumn() : mapOid(0), dataType(execplan::CalpontSystemCatalog::INT), weType(WR_INT), - typeName("integer"), emptyVal(nullptr), - width(0), definedWidth(0), dctnryWidth(0), - precision(0), scale(0), fNotNull(false), - fFldColRelation(BULK_FLDCOL_COLUMN_FIELD), colType(' '), - compressionType(0), autoIncFlag(false), - fMinIntSat(0), fMaxIntSat(0), - fMinDblSat(0), fMaxDblSat(0), fWithDefault(false), - fDefaultInt(0), fDefaultUInt(0), fDefaultDbl(0.0), - fDefaultWideDecimal(0) - { } + std::string colName; /** @brief column name */ + OID mapOid; /** @brief column OID */ + execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */ + ColType weType; /** @brief write engine data type */ + std::string typeName; /** @brief data type name */ + const uint8_t* emptyVal; /** @brief default empty value */ + int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */ + int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */ + int dctnryWidth; /** @brief dictionary width */ + int precision; /** @brief precision of decimal */ + int scale; /** @brief scale of decimal */ + bool fNotNull; /** @brief not null flag */ + BulkFldColRel fFldColRelation; /** @brief type of field/col relation*/ + char colType; /** @brief column type, blank is regular, D is dictionary */ + int compressionType; /** @brief compression type */ + bool autoIncFlag; /** @brief auto increment flag */ + DctnryStruct dctnry; /** @brief dictionary structure */ + int64_t fMinIntSat; /** @brief For integer type, the min saturation value */ + uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */ + double fMinDblSat; /** @brief for float/double, the min saturation value */ + double fMaxDblSat; /** @brief for float/double, the max saturation value */ + bool fWithDefault; /** @brief With default */ + long long fDefaultInt; /** @brief Integer column default */ + unsigned long long fDefaultUInt; /** @brief UnsignedInt col default*/ + double fDefaultDbl; /** @brief Dbl/Flt column default */ + int128_t fDefaultWideDecimal; /** @brief Wide decimal column default */ + std::string fDefaultChr; /** @brief Char column default */ + JobColumn() + : mapOid(0) + , dataType(execplan::CalpontSystemCatalog::INT) + , weType(WR_INT) + , typeName("integer") + , emptyVal(nullptr) + , width(0) + , definedWidth(0) + , dctnryWidth(0) + , precision(0) + , scale(0) + , fNotNull(false) + , fFldColRelation(BULK_FLDCOL_COLUMN_FIELD) + , colType(' ') + , compressionType(0) + , autoIncFlag(false) + , fMinIntSat(0) + , fMaxIntSat(0) + , fMinDblSat(0) + , fMaxDblSat(0) + , fWithDefault(false) + , fDefaultInt(0) + , fDefaultUInt(0) + , fDefaultDbl(0.0) + , fDefaultWideDecimal(0) + { + } }; -typedef std::vector JobColList; /** @brief column value list */ +typedef std::vector JobColList; /** @brief column value list */ -struct JobFieldRef // references field/column in JobTable +struct JobFieldRef // references field/column in JobTable { - BulkFldColRel fFldColType; // type of field or column - unsigned fArrayIndex; // index into colList or fIgnoredFields - // in JobTable based on fFldColType. - JobFieldRef( ) : fFldColType(BULK_FLDCOL_COLUMN_FIELD), fArrayIndex(0) { } - JobFieldRef( BulkFldColRel fldColType, unsigned idx ) : - fFldColType( fldColType ), fArrayIndex( idx ) { } + BulkFldColRel fFldColType; // type of field or column + unsigned fArrayIndex; // index into colList or fIgnoredFields + // in JobTable based on fFldColType. + JobFieldRef() : fFldColType(BULK_FLDCOL_COLUMN_FIELD), fArrayIndex(0) + { + } + JobFieldRef(BulkFldColRel fldColType, unsigned idx) : fFldColType(fldColType), fArrayIndex(idx) + { + } }; -typedef std::vector JobFieldRefList; +typedef std::vector JobFieldRefList; -struct JobTable /** @brief Job Table Structure */ +struct JobTable /** @brief Job Table Structure */ { - std::string tblName; /** @brief table name */ - OID mapOid; /** @brief table OID */ - std::string loadFileName; /** @brief table load file name */ - uint64_t maxErrNum; /** @brief max number of error rows before abort */ - JobColList colList; /** @brief list of columns to be loaded; followed by default columns to be loaded */ - JobColList fIgnoredFields; /** @brief list of fields in input file to be ignored */ - JobFieldRefList fFldRefs; /** @brief Combined list of refs to entries in colList and fIgnoredFields */ - JobTable() : mapOid(0), maxErrNum(0) { } + std::string tblName; /** @brief table name */ + OID mapOid; /** @brief table OID */ + std::string loadFileName; /** @brief table load file name */ + uint64_t maxErrNum; /** @brief max number of error rows before abort */ + JobColList colList; /** @brief list of columns to be loaded; followed by default columns to be loaded */ + JobColList fIgnoredFields; /** @brief list of fields in input file to be ignored */ + JobFieldRefList fFldRefs; /** @brief Combined list of refs to entries in colList and fIgnoredFields */ + JobTable() : mapOid(0), maxErrNum(0) + { + } }; -typedef std::vector JobTableList;/** @brief table list */ +typedef std::vector JobTableList; /** @brief table list */ -struct Job /** @brief Job Structure */ +struct Job /** @brief Job Structure */ { - int id; /** @brief job id */ - std::string schema; /** @brief database name */ - std::string name; /** @brief job name */ - std::string desc; /** @brief job description */ - std::string userName; /** @brief user name */ - JobTableList jobTableList; /** @brief job table list */ + int id; /** @brief job id */ + std::string schema; /** @brief database name */ + std::string name; /** @brief job name */ + std::string desc; /** @brief job description */ + std::string userName; /** @brief user name */ + JobTableList jobTableList; /** @brief job table list */ - std::string createDate; /** @brief job create date */ - std::string createTime; /** @brief job create time */ + std::string createDate; /** @brief job create date */ + std::string createTime; /** @brief job create time */ - char fDelimiter; - char fEnclosedByChar; - char fEscapeChar; - int numberOfReadBuffers; - unsigned readBufferSize; - unsigned writeBufferSize; - Job() : id(0), fDelimiter('|'), - fEnclosedByChar('\0'), fEscapeChar('\0'), - numberOfReadBuffers(0), readBufferSize(0), writeBufferSize(0) { } + char fDelimiter; + char fEnclosedByChar; + char fEscapeChar; + int numberOfReadBuffers; + unsigned readBufferSize; + unsigned writeBufferSize; + Job() + : id(0) + , fDelimiter('|') + , fEnclosedByChar('\0') + , fEscapeChar('\0') + , numberOfReadBuffers(0) + , readBufferSize(0) + , writeBufferSize(0) + { + } }; /************************************************************************ * @brief Cache memory ************************************************************************/ -struct CacheBlock /** @brief Cache block structure */ +struct CacheBlock /** @brief Cache block structure */ { - uint64_t fbo; /** @brief file fbo */ - uint64_t lbid; /** @brief lbid */ - bool dirty; /** @brief dirty flag */ - int hitCount; /** @brief hit count */ - unsigned char* data; /** @brief block buffer */ - CacheBlock() - { - data = NULL; /** @brief constructor */ - clear(); - } - void clear() - { - fbo = lbid = hitCount = 0; - dirty = false; + uint64_t fbo; /** @brief file fbo */ + uint64_t lbid; /** @brief lbid */ + bool dirty; /** @brief dirty flag */ + int hitCount; /** @brief hit count */ + unsigned char* data; /** @brief block buffer */ + CacheBlock() + { + data = NULL; /** @brief constructor */ + clear(); + } + void clear() + { + fbo = lbid = hitCount = 0; + dirty = false; - if ( data ) memset( data, 0, BYTE_PER_BLOCK); - } /** @brief clear, NOTE: buf must be free by caller first */ - void init() - { - data = (unsigned char*)malloc(BYTE_PER_BLOCK); - } - void freeMem() - { - if ( data ) free( data ); - } + if (data) + memset(data, 0, BYTE_PER_BLOCK); + } /** @brief clear, NOTE: buf must be free by caller first */ + void init() + { + data = (unsigned char*)malloc(BYTE_PER_BLOCK); + } + void freeMem() + { + if (data) + free(data); + } }; -struct BlockBuffer /** @brief Block buffer */ +struct BlockBuffer /** @brief Block buffer */ { - CommBlock cb; /** @brief Communication block structure */ - CacheBlock block; /** @brief Cache block strucutre */ - CacheListType listType; /** @brief List number, 0 - free, 1 - LRU, 2 - write */ - BlockBuffer() - { - clear(); /** @brief constructor */ - } - void init() - { - block.init(); - } - void freeMem() - { - block.freeMem(); - } - void clear() - { - cb.clear(); - block.clear(); - listType = FREE_LIST; - } + CommBlock cb; /** @brief Communication block structure */ + CacheBlock block; /** @brief Cache block strucutre */ + CacheListType listType; /** @brief List number, 0 - free, 1 - LRU, 2 - write */ + BlockBuffer() + { + clear(); /** @brief constructor */ + } + void init() + { + block.init(); + } + void freeMem() + { + block.freeMem(); + } + void clear() + { + cb.clear(); + block.clear(); + listType = FREE_LIST; + } }; -struct CacheControl /** @brief Cache control structure */ +struct CacheControl /** @brief Cache control structure */ { - int totalBlock; /** @brief The toal number of allocated blocks */ - int pctFree; /** @brief The percentage of free blocks when some blocks must be aged out */ - int checkInterval; /** @brief A check point interval in seconds */ - CacheControl() - { - totalBlock = pctFree = checkInterval = 0; /** @brief constructor */ - } + int totalBlock; /** @brief The toal number of allocated blocks */ + int pctFree; /** @brief The percentage of free blocks when some blocks must be aged out */ + int checkInterval; /** @brief A check point interval in seconds */ + CacheControl() + { + totalBlock = pctFree = checkInterval = 0; /** @brief constructor */ + } }; /************************************************************************ * @brief Bulk parse meta data describing data in a read buffer. * An offset of COLPOSPAIR_NULL_TOKEN_OFFSET represents a null token. ************************************************************************/ -struct ColPosPair /** @brief Column position pair structure */ +struct ColPosPair /** @brief Column position pair structure */ { - int start; /** @brief start position */ - int offset; /** @brief length of token*/ + int start; /** @brief start position */ + int offset; /** @brief length of token*/ }; /************************************************************************ @@ -518,9 +585,10 @@ struct ColPosPair /** @brief Column position pair structure */ ************************************************************************/ class SecondaryShutdownException : public std::runtime_error { -public: - SecondaryShutdownException(const std::string& msg) : - std::runtime_error(msg) { } + public: + SecondaryShutdownException(const std::string& msg) : std::runtime_error(msg) + { + } }; /************************************************************************ @@ -529,21 +597,23 @@ public: ************************************************************************/ class WeException : public std::runtime_error { -public: - WeException(const std::string& msg, int err = 0) : - std::runtime_error(msg), fErrorCode(err) { } - void errorCode(int code) - { - fErrorCode = code; - } - int errorCode() const - { - return fErrorCode; - } -private: - int fErrorCode; + public: + WeException(const std::string& msg, int err = 0) : std::runtime_error(msg), fErrorCode(err) + { + } + void errorCode(int code) + { + fErrorCode = code; + } + int errorCode() const + { + return fErrorCode; + } + + private: + int fErrorCode; }; -} //end of namespace +} // namespace WriteEngine -#endif // _WE_TYPE_H_ +#endif // _WE_TYPE_H_ diff --git a/writeengine/shared/we_typeext.h b/writeengine/shared/we_typeext.h index 59e69e266..34ccb0bfb 100644 --- a/writeengine/shared/we_typeext.h +++ b/writeengine/shared/we_typeext.h @@ -35,80 +35,79 @@ #include #include "IDBFileSystem.h" - /** Namespace WriteEngine */ namespace WriteEngine { /************************************************************************ * Type definitions ************************************************************************/ -typedef uint64_t RID; // Row ID +typedef uint64_t RID; // Row ID /************************************************************************ * Dictionary related structure ************************************************************************/ struct Token { - uint64_t op : 10; // ordinal position within a block - uint64_t fbo : 36; // file block number - uint64_t bc : 18; // block count - Token() // constructor, set to null value - { - op = 0x3FE; - fbo = 0xFFFFFFFFFLL; - bc = 0x3FFFF; - } + uint64_t op : 10; // ordinal position within a block + uint64_t fbo : 36; // file block number + uint64_t bc : 18; // block count + Token() // constructor, set to null value + { + op = 0x3FE; + fbo = 0xFFFFFFFFFLL; + bc = 0x3FFFF; + } }; -constexpr uid_t UID_NONE = (uid_t) -1; -constexpr gid_t GID_NONE = (gid_t) -1; +constexpr uid_t UID_NONE = (uid_t)-1; +constexpr gid_t GID_NONE = (gid_t)-1; class WeUIDGID { -public: - WeUIDGID(): uid(UID_NONE), gid(GID_NONE) {} - virtual ~WeUIDGID() {}; - virtual void setUIDGID(const uid_t uid, const gid_t gid); - void setUIDGID(const WeUIDGID* id); - bool chownPath(std::ostringstream& error, - const std::string& fileName, - const idbdatafile::IDBFileSystem& fs) const; -; + public: + WeUIDGID() : uid(UID_NONE), gid(GID_NONE) + { + } + virtual ~WeUIDGID(){}; + virtual void setUIDGID(const uid_t uid, const gid_t gid); + void setUIDGID(const WeUIDGID* id); + bool chownPath(std::ostringstream& error, const std::string& fileName, + const idbdatafile::IDBFileSystem& fs) const; + ; -private: - uid_t uid; - gid_t gid; + private: + uid_t uid; + gid_t gid; }; inline void WeUIDGID::setUIDGID(const uid_t p_uid, const gid_t p_gid) { - uid = p_uid; gid = p_gid; + uid = p_uid; + gid = p_gid; } inline void WeUIDGID::setUIDGID(const WeUIDGID* id) { - if (id->uid != UID_NONE) - *this = *id; + if (id->uid != UID_NONE) + *this = *id; } -inline bool WeUIDGID::chownPath(std::ostringstream& error, - const std::string& fileName, - const idbdatafile::IDBFileSystem& fs) const +inline bool WeUIDGID::chownPath(std::ostringstream& error, const std::string& fileName, + const idbdatafile::IDBFileSystem& fs) const { - if (uid != UID_NONE) + if (uid != UID_NONE) + { + int funcErrno = 0; + if (fs.chown(fileName.c_str(), uid, gid, funcErrno) == -1) { - int funcErrno = 0; - if (fs.chown(fileName.c_str(), uid, gid, funcErrno) == -1) - { - error << "Error calling chown() with uid " << uid - << " and gid " << gid << " with the file " - << fileName << " with errno " << funcErrno; - return true; - } + error << "Error calling chown() with uid " << uid << " and gid " << gid << " with the file " << fileName + << " with errno " << funcErrno; + return true; } - return false; + } + return false; } -} //end of namespace +} // namespace WriteEngine -#endif // _WE_TYPEEXT_H_ +#endif // _WE_TYPEEXT_H_ diff --git a/writeengine/splitter/resource.h b/writeengine/splitter/resource.h index bf240aab9..c5b88727e 100644 --- a/writeengine/splitter/resource.h +++ b/writeengine/splitter/resource.h @@ -6,9 +6,9 @@ // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 #endif #endif diff --git a/writeengine/splitter/we_brmupdater.cpp b/writeengine/splitter/we_brmupdater.cpp index 4d3d2f142..f6ea954e8 100644 --- a/writeengine/splitter/we_brmupdater.cpp +++ b/writeengine/splitter/we_brmupdater.cpp @@ -16,10 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ - + * $Id$ + * + *******************************************************************************/ /* * we_brmupdater.cpp @@ -28,9 +27,6 @@ * Author: bpaul */ - - - #include #include using namespace std; @@ -49,586 +45,559 @@ using namespace BRM; #include "we_brmupdater.h" - namespace WriteEngine { - - //----------------------------------------------------------------------------- bool WEBrmUpdater::updateCasualPartitionAndHighWaterMarkInBRM() { + try + { + bool aGood = prepareCasualPartitionInfo(); - try - { - bool aGood = prepareCasualPartitionInfo(); + if (!aGood) + cout << "prepareCasualPartitionInfo Failed" << endl; + } + catch (std::exception& ex) + { + std::string aStr = "Exception in prepareCasualPartitionInfo(); Error Ignored"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + errMsgArgs.add(ex.what()); + fRef.sysLog(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + cout << aStr << endl; + } - if (!aGood) - cout << "prepareCasualPartitionInfo Failed" << endl; - } - catch (std::exception& ex) - { - std::string aStr = "Exception in prepareCasualPartitionInfo(); Error Ignored"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - errMsgArgs.add(ex.what()); - fRef.sysLog(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - cout << aStr << endl; - } + try + { + bool aSuccess = prepareHighWaterMarkInfo(); + if (!aSuccess) + throw(std::runtime_error("prepareHWMInfo Failed")); + } + catch (std::exception& ex) + { + std::string aStr = "prepareHWMInfo() failed... Bailing out!!"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + aStr = "Need to rollback bulk upload"; + errMsgArgs.add(aStr); + errMsgArgs.add(ex.what()); + fRef.sysLog(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + cout << aStr << endl; - try - { - bool aSuccess = prepareHighWaterMarkInfo(); + return false; + } - if (!aSuccess) - throw(std::runtime_error("prepareHWMInfo Failed")); - } - catch (std::exception& ex) - { - std::string aStr = "prepareHWMInfo() failed... Bailing out!!"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - aStr = "Need to rollback bulk upload"; - errMsgArgs.add(aStr); - errMsgArgs.add(ex.what()); - fRef.sysLog(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - cout << aStr << endl; + // If we are here, we packaged informations properly + // Lets connect to BRM - return false; - } + if (!createBrmConnection()) + { + cout << "Brm Connection FAILED" << endl; + return false; + } - // If we are here, we packaged informations properly - // Lets connect to BRM + int aRc = updateCPAndHWMInBRM(); + if (aRc != 0) + { + cout << "Updating High Water Mark Failed" << endl; + } + else + { + if (fRef.getDebugLvl()) + cout << "Updating High Water Mark Successful!!" << endl << endl; + } - if (!createBrmConnection()) - { - cout << "Brm Connection FAILED" << endl; - return false; - } + /* + int cpRc = updateCasualPartitionInBRM(); + if(cpRc != 0) + cout << "Updating Casual Partition Failed" << endl; + else + { + if(fRef.getDebugLvl()) + cout << "Updating Casual Partition Successful" << endl; + } + int hwmRc = updateHighWaterMarkInBRM(); + if(hwmRc != 0) + cout << "Updating High Water Mark Failed" << endl; + else + { + if(fRef.getDebugLvl()) + cout << "Updating High Water Mark Successful!!" << endl << endl; + } + */ - int aRc = updateCPAndHWMInBRM(); + releaseBrmConnection(); - if (aRc != 0) - { - cout << "Updating High Water Mark Failed" << endl; - } - else - { - if (fRef.getDebugLvl()) - cout << "Updating High Water Mark Successful!!" << endl << endl; - } - - - /* - int cpRc = updateCasualPartitionInBRM(); - if(cpRc != 0) - cout << "Updating Casual Partition Failed" << endl; - else - { - if(fRef.getDebugLvl()) - cout << "Updating Casual Partition Successful" << endl; - } - - int hwmRc = updateHighWaterMarkInBRM(); - if(hwmRc != 0) - cout << "Updating High Water Mark Failed" << endl; - else - { - if(fRef.getDebugLvl()) - cout << "Updating High Water Mark Successful!!" << endl << endl; - } - */ - - - releaseBrmConnection(); - - - //return(!hwmRc)? true: false; - return (!aRc) ? true : false; + // return(!hwmRc)? true: false; + return (!aRc) ? true : false; } - //------------------------------------------------------------------------------ // Update Casual Partition information in BRM //------------------------------------------------------------------------------ int WEBrmUpdater::updateCasualPartitionInBRM() { - int rc = 0; + int rc = 0; - if (fCPInfo.size() > 0) + if (fCPInfo.size() > 0) + { + // std::ostringstream oss; + // oss << "Committing " << fCPInfo.size() << " CP updates for table " << + // fRef.getTableName() << " to BRM"; + // cout << endl << oss.str() << endl; + + // TODO - NOTE. later make this Objection creation once for both CP & HWM + if (fpBrm) { - //std::ostringstream oss; - //oss << "Committing " << fCPInfo.size() << " CP updates for table " << - // fRef.getTableName() << " to BRM"; - //cout << endl << oss.str() << endl; + rc = fpBrm->mergeExtentsMaxMin(fCPInfo); + if (rc != BRM::ERR_OK) + { + std::string errStr; + BRM::errString(rc, errStr); + cout << "BRM ERROR is ***** " << errStr << endl; - //TODO - NOTE. later make this Objection creation once for both CP & HWM - if (fpBrm) - { - rc = fpBrm->mergeExtentsMaxMin(fCPInfo); + std::ostringstream oss; + oss << "Error updating BRM with CP data for table " << fRef.getTableName() << " Error: " << errStr + << endl; - if (rc != BRM::ERR_OK) - { - std::string errStr; - BRM::errString(rc, errStr); - cout << "BRM ERROR is ***** " << errStr << endl; - - std::ostringstream oss; - oss << "Error updating BRM with CP data for table " << - fRef.getTableName() << " Error: " << errStr << endl; - - cout << endl << oss.str() << endl; - } - } + cout << endl << oss.str() << endl; + } } + } - return rc; + return rc; } - - //------------------------------------------------------------------------------ // Send HWM update information to BRM //------------------------------------------------------------------------------ int WEBrmUpdater::updateHighWaterMarkInBRM() { - int rc = 0; + int rc = 0; - if (fHWMInfo.size() > 0) + if (fHWMInfo.size() > 0) + { + // std::ostringstream oss; + // oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table "<< + // fRef.getTableName() << " to BRM"; + // cout << endl << oss.str() << endl; + + if (fpBrm) { - //std::ostringstream oss; - //oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table "<< - // fRef.getTableName() << " to BRM"; - //cout << endl << oss.str() << endl; + rc = fpBrm->bulkSetHWM(fHWMInfo, 0); - if (fpBrm) - { - rc = fpBrm->bulkSetHWM(fHWMInfo, 0); + if (rc != BRM::ERR_OK) + { + std::string errStr; + BRM::errString(rc, errStr); + cout << "BRM ERROR is ***** " << errStr << endl; - if (rc != BRM::ERR_OK) - { - std::string errStr; - BRM::errString(rc, errStr); - cout << "BRM ERROR is ***** " << errStr << endl; - - std::ostringstream oss; - oss << "Error updating BRM with HWM data for table " << - fRef.getTableName() << "error: " << errStr << endl; - cout << endl << oss.str() << endl; - return rc; - } - } + std::ostringstream oss; + oss << "Error updating BRM with HWM data for table " << fRef.getTableName() << "error: " << errStr + << endl; + cout << endl << oss.str() << endl; + return rc; + } } + } - return rc; + return rc; } - //----------------------------------------------------------------------------- int WEBrmUpdater::updateCPAndHWMInBRM() { - int rc = 0; + int rc = 0; - //BUG 4232. some imports may not contain CP but HWM - if ((fCPInfo.size() > 0) || (fHWMInfo.size() > 0)) + // BUG 4232. some imports may not contain CP but HWM + if ((fCPInfo.size() > 0) || (fHWMInfo.size() > 0)) + { + // TODO - NOTE. later make this Objection creation once for both CP & HWM + if (fpBrm) { - //TODO - NOTE. later make this Objection creation once for both CP & HWM - if (fpBrm) - { - /* - rc = bulkSetHWMAndCP(const std::vector &, - const std::vector & setCPDataArgs, - const std::vector & mergeCPDataArgs, - VER_t transID = 0) DBRM_THROW; - */ - rc = fpBrm->bulkSetHWMAndCP(fHWMInfo, fCPInfoData, fCPInfo, 0); + /* + rc = bulkSetHWMAndCP(const std::vector &, + const std::vector & setCPDataArgs, + const std::vector & mergeCPDataArgs, + VER_t transID = 0) DBRM_THROW; + */ + rc = fpBrm->bulkSetHWMAndCP(fHWMInfo, fCPInfoData, fCPInfo, 0); - //rc = fpBrm->mergeExtentsMaxMin(fCPInfo); - //rc = fpBrm->bulkSetHWM(fHWMInfo, 0); - if (rc != BRM::ERR_OK) - { - std::string errStr; - BRM::errString(rc, errStr); - cout << "BRM ERROR is ***** " << errStr << endl; + // rc = fpBrm->mergeExtentsMaxMin(fCPInfo); + // rc = fpBrm->bulkSetHWM(fHWMInfo, 0); + if (rc != BRM::ERR_OK) + { + std::string errStr; + BRM::errString(rc, errStr); + cout << "BRM ERROR is ***** " << errStr << endl; - std::ostringstream oss; - oss << "Error updating BRM with HWM data for table " << - fRef.getTableName() << "error: " << errStr << endl; - cout << endl << oss.str() << endl; - cout << "ERROR: HWM and CP set failed!!" << endl; - //throw runtime_error("ERROR: bulSetHWMAndCp Failed!!"); - return rc; - } - } - else - return rc; + std::ostringstream oss; + oss << "Error updating BRM with HWM data for table " << fRef.getTableName() << "error: " << errStr + << endl; + cout << endl << oss.str() << endl; + cout << "ERROR: HWM and CP set failed!!" << endl; + // throw runtime_error("ERROR: bulSetHWMAndCp Failed!!"); + return rc; + } } + else + return rc; + } - return rc; + return rc; } //------------------------------------------------------------------------------ - bool WEBrmUpdater::prepareCasualPartitionInfo() { - //cout << "Started prepareCasualPartitionInfo()!!" << endl; - //CP: 275456 6000000 4776193 -1 0 1 - WESDHandler::StrVec::iterator aIt = fRef.fBrmRptVec.begin(); + // cout << "Started prepareCasualPartitionInfo()!!" << endl; + // CP: 275456 6000000 4776193 -1 0 1 + WESDHandler::StrVec::iterator aIt = fRef.fBrmRptVec.begin(); - while (aIt != fRef.fBrmRptVec.end()) + while (aIt != fRef.fBrmRptVec.end()) + { + std::string aEntry = *aIt; + + if ((!aEntry.empty()) && (aEntry.at(0) == 'C')) { - std::string aEntry = *aIt; + BRM::CPInfoMerge cpInfoMerge; + const int BUFLEN = 128; + char aBuff[BUFLEN]; + strncpy(aBuff, aEntry.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; - if ((!aEntry.empty()) && (aEntry.at(0) == 'C')) + char* pTok = strtok(aBuff, " "); + + if (!pTok) // ignore the Msg Body + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad Body in CP entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.startLbid = boost::lexical_cast(pTok); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad startLbid in CP entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.seqNum = atoi(pTok); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad seqNUM in CP entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.type = (execplan::CalpontSystemCatalog::ColDataType)atoi(pTok); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad type in CP entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.colWidth = boost::lexical_cast(pTok); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad column width in CP entry string")); + } + + if (datatypes::isWideDecimalType(cpInfoMerge.type, cpInfoMerge.colWidth)) + { + datatypes::SystemCatalog::TypeAttributesStd tyAttr(cpInfoMerge.colWidth, 0, + datatypes::INT128MAXPRECISION); + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.bigMax = tyAttr.decimal128FromString(std::string(pTok), NULL); + else { - BRM::CPInfoMerge cpInfoMerge; - const int BUFLEN = 128; - char aBuff[BUFLEN]; - strncpy(aBuff, aEntry.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; - - char* pTok = strtok(aBuff, " "); - - if (!pTok) // ignore the Msg Body - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad Body in CP entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.startLbid = boost::lexical_cast(pTok); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad startLbid in CP entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.seqNum = atoi(pTok); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad seqNUM in CP entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.type = (execplan::CalpontSystemCatalog::ColDataType)atoi(pTok); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad type in CP entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.colWidth = boost::lexical_cast(pTok); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad column width in CP entry string")); - } - - if (datatypes::isWideDecimalType(cpInfoMerge.type, cpInfoMerge.colWidth)) - { - datatypes::SystemCatalog::TypeAttributesStd tyAttr(cpInfoMerge.colWidth, - 0, - datatypes::INT128MAXPRECISION); - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.bigMax = tyAttr.decimal128FromString(std::string(pTok), NULL); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad MAX in CP entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.bigMin = tyAttr.decimal128FromString(std::string(pTok), NULL); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad MIN in CP entry string")); - } - } - else - { - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.max = boost::lexical_cast(pTok); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad MAX in CP entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.min = boost::lexical_cast(pTok); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad MIN in CP entry string")); - } - } - - pTok = strtok(NULL, " "); - - if (pTok) - cpInfoMerge.newExtent = (atoi(pTok) != 0); - else - { - //cout << "CP Entry : " << aEntry << endl; - throw (runtime_error("Bad newExtent in CP entry string")); - } - - fCPInfo.push_back(cpInfoMerge); + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad MAX in CP entry string")); } - ++aIt; + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.bigMin = tyAttr.decimal128FromString(std::string(pTok), NULL); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad MIN in CP entry string")); + } + } + else + { + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.max = boost::lexical_cast(pTok); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad MAX in CP entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.min = boost::lexical_cast(pTok); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad MIN in CP entry string")); + } + } + + pTok = strtok(NULL, " "); + + if (pTok) + cpInfoMerge.newExtent = (atoi(pTok) != 0); + else + { + // cout << "CP Entry : " << aEntry << endl; + throw(runtime_error("Bad newExtent in CP entry string")); + } + + fCPInfo.push_back(cpInfoMerge); } - if (fRef.getDebugLvl()) - cout << "Finished prepareCasualPartitionInfo()!!" << endl; + ++aIt; + } - return true; + if (fRef.getDebugLvl()) + cout << "Finished prepareCasualPartitionInfo()!!" << endl; + + return true; } - //----------------------------------------------------------------------------- bool WEBrmUpdater::prepareHighWaterMarkInfo() { - //HWM: 3056 0 0 8191 - WESDHandler::StrVec::iterator aIt = fRef.fBrmRptVec.begin(); + // HWM: 3056 0 0 8191 + WESDHandler::StrVec::iterator aIt = fRef.fBrmRptVec.begin(); - while (aIt != fRef.fBrmRptVec.end()) + while (aIt != fRef.fBrmRptVec.end()) + { + std::string aEntry = *aIt; + + if ((!aEntry.empty()) && (aEntry.at(0) == 'H')) { - std::string aEntry = *aIt; + BRM::BulkSetHWMArg hwmArg; + const int BUFLEN = 128; + char aBuff[BUFLEN]; + strncpy(aBuff, aEntry.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; - if ((!aEntry.empty()) && (aEntry.at(0) == 'H')) - { - BRM::BulkSetHWMArg hwmArg; - const int BUFLEN = 128; - char aBuff[BUFLEN]; - strncpy(aBuff, aEntry.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; + char* pTok = strtok(aBuff, " "); - char* pTok = strtok(aBuff, " "); + if (!pTok) // ignore the Msg Body + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad Body in HWM entry string")); + } - if (!pTok) // ignore the Msg Body - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad Body in HWM entry string")); - } + pTok = strtok(NULL, " "); - pTok = strtok(NULL, " "); + if (pTok) + hwmArg.oid = atoi(pTok); + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad OID in HWM entry string")); + } - if (pTok) - hwmArg.oid = atoi(pTok); - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad OID in HWM entry string")); - } + pTok = strtok(NULL, " "); - pTok = strtok(NULL, " "); + if (pTok) + hwmArg.partNum = atoi(pTok); + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad partNum in HWM entry string")); + } - if (pTok) - hwmArg.partNum = atoi(pTok); - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad partNum in HWM entry string")); - } + pTok = strtok(NULL, " "); - pTok = strtok(NULL, " "); + if (pTok) + hwmArg.segNum = atoi(pTok); + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad partNum in HWM entry string")); + } - if (pTok) - hwmArg.segNum = atoi(pTok); - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad partNum in HWM entry string")); - } + pTok = strtok(NULL, " "); - pTok = strtok(NULL, " "); - - if (pTok) - hwmArg.hwm = atoi(pTok); - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad partNum in HWM entry string")); - } - - fHWMInfo.push_back( hwmArg ); - - } - - ++aIt; + if (pTok) + hwmArg.hwm = atoi(pTok); + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad partNum in HWM entry string")); + } + fHWMInfo.push_back(hwmArg); } - if (fRef.getDebugLvl()) - cout << "prepareHighWaterMarkInfo() finished" << endl; + ++aIt; + } - return true; + if (fRef.getDebugLvl()) + cout << "prepareHighWaterMarkInfo() finished" << endl; + + return true; } - //------------------------------------------------------------------------------ //#ROWS: numRowsRead numRowsInserted -bool WEBrmUpdater::prepareRowsInsertedInfo(std::string Entry, - int64_t& TotRows, int64_t& InsRows) +bool WEBrmUpdater::prepareRowsInsertedInfo(std::string Entry, int64_t& TotRows, int64_t& InsRows) { - bool aFound = false; + bool aFound = false; - //ROWS: 3 1 - if ((!Entry.empty()) && (Entry.at(0) == 'R')) + // ROWS: 3 1 + if ((!Entry.empty()) && (Entry.at(0) == 'R')) + { + aFound = true; + const int BUFLEN = 128; + char aBuff[BUFLEN]; + strncpy(aBuff, Entry.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; + + char* pTok = strtok(aBuff, " "); + + if (!pTok) // ignore the Msg Body { - aFound = true; - const int BUFLEN = 128; - char aBuff[BUFLEN]; - strncpy(aBuff, Entry.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; - - char* pTok = strtok(aBuff, " "); - - if (!pTok) // ignore the Msg Body - { - //cout << "ROWS Entry : " << aEntry << endl; - throw (runtime_error("Bad Body in entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - TotRows = strtol(pTok, NULL, 10); - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad Tot ROWS entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - InsRows = strtol(pTok, NULL, 10); - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad inserted ROWS in entry string")); - } - + // cout << "ROWS Entry : " << aEntry << endl; + throw(runtime_error("Bad Body in entry string")); } - return aFound; -} + pTok = strtok(NULL, " "); + if (pTok) + TotRows = strtol(pTok, NULL, 10); + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad Tot ROWS entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + InsRows = strtol(pTok, NULL, 10); + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad inserted ROWS in entry string")); + } + } + + return aFound; +} //------------------------------------------------------------------------------ //#DATA: columnNumber columnType columnName numOutOfRangeValues -bool WEBrmUpdater::prepareColumnOutOfRangeInfo(std::string Entry, - int& ColNum, - execplan::CalpontSystemCatalog::ColDataType& ColType, - std::string& ColName, - int& OorValues) +bool WEBrmUpdater::prepareColumnOutOfRangeInfo(std::string Entry, int& ColNum, + execplan::CalpontSystemCatalog::ColDataType& ColType, + std::string& ColName, int& OorValues) { - bool aFound = false; - boost::shared_ptr systemCatalogPtr = + bool aFound = false; + boost::shared_ptr systemCatalogPtr = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(); - //DATA: 3 1 - if ((!Entry.empty()) && (Entry.at(0) == 'D')) + // DATA: 3 1 + if ((!Entry.empty()) && (Entry.at(0) == 'D')) + { + aFound = true; + const int BUFLEN = 128; + char aBuff[BUFLEN]; + strncpy(aBuff, Entry.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; + + char* pTok = strtok(aBuff, " "); + + if (!pTok) // ignore the Msg Body { - aFound = true; - const int BUFLEN = 128; - char aBuff[BUFLEN]; - strncpy(aBuff, Entry.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; - - char* pTok = strtok(aBuff, " "); - - if (!pTok) // ignore the Msg Body - { - //cout << "ROWS Entry : " << aEntry << endl; - throw (runtime_error("Bad OOR entry info")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - { - ColNum = atoi(pTok); - } - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad Oor Column Number entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - { - ColType = (execplan::CalpontSystemCatalog::ColDataType)atoi(pTok); - } - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad Oor Column Type entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - { - uint64_t columnOid = strtol(pTok, NULL, 10); - execplan::CalpontSystemCatalog::TableColName colname = systemCatalogPtr->colName(columnOid); - ColName = colname.schema + "." + colname.table + "." + colname.column; - } - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad Column Name entry string")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - { - OorValues = atoi(pTok); - } - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad OorValues entry string")); - } + // cout << "ROWS Entry : " << aEntry << endl; + throw(runtime_error("Bad OOR entry info")); } - return aFound; + pTok = strtok(NULL, " "); + + if (pTok) + { + ColNum = atoi(pTok); + } + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad Oor Column Number entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + { + ColType = (execplan::CalpontSystemCatalog::ColDataType)atoi(pTok); + } + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad Oor Column Type entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + { + uint64_t columnOid = strtol(pTok, NULL, 10); + execplan::CalpontSystemCatalog::TableColName colname = systemCatalogPtr->colName(columnOid); + ColName = colname.schema + "." + colname.table + "." + colname.column; + } + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad Column Name entry string")); + } + + pTok = strtok(NULL, " "); + + if (pTok) + { + OorValues = atoi(pTok); + } + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad OorValues entry string")); + } + } + + return aFound; } //------------------------------------------------------------------------------ @@ -636,87 +605,83 @@ bool WEBrmUpdater::prepareColumnOutOfRangeInfo(std::string Entry, bool WEBrmUpdater::prepareErrorFileInfo(std::string Entry, std::string& ErrFileName) { - bool aFound = false; + bool aFound = false; - if ((!Entry.empty()) && (Entry.at(0) == 'E')) + if ((!Entry.empty()) && (Entry.at(0) == 'E')) + { + aFound = true; + const int BUFLEN = 128; + char aBuff[BUFLEN]; + strncpy(aBuff, Entry.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; + + char* pTok = strtok(aBuff, " "); + + if (!pTok) // ignore the Msg Body { - aFound = true; - const int BUFLEN = 128; - char aBuff[BUFLEN]; - strncpy(aBuff, Entry.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; - - char* pTok = strtok(aBuff, " "); - - if (!pTok) // ignore the Msg Body - { - //cout << "ROWS Entry : " << aEntry << endl; - throw (runtime_error("Bad ERR File entry info")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - { - ErrFileName = pTok; - //int aLen = ErrFileName.length(); - //if(aLen>0) ErrFileName.insert(aLen-1, 1, 0); - } - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad Error Filename entry string")); - } - + // cout << "ROWS Entry : " << aEntry << endl; + throw(runtime_error("Bad ERR File entry info")); } - return aFound; -} + pTok = strtok(NULL, " "); + if (pTok) + { + ErrFileName = pTok; + // int aLen = ErrFileName.length(); + // if(aLen>0) ErrFileName.insert(aLen-1, 1, 0); + } + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad Error Filename entry string")); + } + } + + return aFound; +} //------------------------------------------------------------------------------ //#BAD: bad data file, with rejected rows bool WEBrmUpdater::prepareBadDataFileInfo(std::string Entry, std::string& BadFileName) { - bool aFound = false; + bool aFound = false; - if ((!Entry.empty()) && (Entry.at(0) == 'B')) + if ((!Entry.empty()) && (Entry.at(0) == 'B')) + { + aFound = true; + const int BUFLEN = 128; + char aBuff[BUFLEN]; + strncpy(aBuff, Entry.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; + + char* pTok = strtok(aBuff, " "); + + if (!pTok) // ignore the Msg Body { - aFound = true; - const int BUFLEN = 128; - char aBuff[BUFLEN]; - strncpy(aBuff, Entry.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; - - char* pTok = strtok(aBuff, " "); - - if (!pTok) // ignore the Msg Body - { - //cout << "ROWS Entry : " << aEntry << endl; - throw (runtime_error("Bad BAD Filename entry ")); - } - - pTok = strtok(NULL, " "); - - if (pTok) - { - BadFileName = pTok; - //int aLen = BadFileName.length(); - //if(aLen>0) BadFileName.insert(aLen-1, 1, 0); - } - else - { - //cout << "HWM Entry : " << aEntry << endl; - throw (runtime_error("Bad BAD Filename entry string")); - } + // cout << "ROWS Entry : " << aEntry << endl; + throw(runtime_error("Bad BAD Filename entry ")); } - return aFound; -} + pTok = strtok(NULL, " "); + if (pTok) + { + BadFileName = pTok; + // int aLen = BadFileName.length(); + // if(aLen>0) BadFileName.insert(aLen-1, 1, 0); + } + else + { + // cout << "HWM Entry : " << aEntry << endl; + throw(runtime_error("Bad BAD Filename entry string")); + } + } + + return aFound; +} //------------------------------------------------------------------------------ - } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_brmupdater.h b/writeengine/splitter/we_brmupdater.h index c575b3bd1..1ea265129 100644 --- a/writeengine/splitter/we_brmupdater.h +++ b/writeengine/splitter/we_brmupdater.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_brmupdater.h @@ -30,60 +30,55 @@ #ifndef WE_BRMUPDATER_H_ #define WE_BRMUPDATER_H_ - namespace WriteEngine { - -class WESDHandler; // forward deceleration - +class WESDHandler; // forward deceleration class WEBrmUpdater { -public: - WEBrmUpdater(WESDHandler& Ref): fRef(Ref), fpBrm(0) {} - virtual ~WEBrmUpdater() {} + public: + WEBrmUpdater(WESDHandler& Ref) : fRef(Ref), fpBrm(0) + { + } + virtual ~WEBrmUpdater() + { + } -public: - bool updateCasualPartitionAndHighWaterMarkInBRM(); - int updateCPAndHWMInBRM(); - int updateCasualPartitionInBRM(); - int updateHighWaterMarkInBRM(); - bool prepareCasualPartitionInfo(); - bool prepareHighWaterMarkInfo(); + public: + bool updateCasualPartitionAndHighWaterMarkInBRM(); + int updateCPAndHWMInBRM(); + int updateCasualPartitionInBRM(); + int updateHighWaterMarkInBRM(); + bool prepareCasualPartitionInfo(); + bool prepareHighWaterMarkInfo(); - bool createBrmConnection() - { - fpBrm = new BRM::DBRM(); - return (fpBrm) ? true : false; - } - void releaseBrmConnection() - { - delete fpBrm; - fpBrm = 0; - } + bool createBrmConnection() + { + fpBrm = new BRM::DBRM(); + return (fpBrm) ? true : false; + } + void releaseBrmConnection() + { + delete fpBrm; + fpBrm = 0; + } + public: + static bool prepareRowsInsertedInfo(std::string Entry, int64_t& TotRows, int64_t& InsRows); + static bool prepareColumnOutOfRangeInfo(std::string Entry, int& ColNum, + execplan::CalpontSystemCatalog::ColDataType& ColType, + std::string& ColName, int& OorValues); + static bool prepareErrorFileInfo(std::string Entry, std::string& ErrFileName); + static bool prepareBadDataFileInfo(std::string Entry, std::string& BadFileName); + private: + WESDHandler& fRef; + BRM::DBRM* fpBrm; -public: - static bool prepareRowsInsertedInfo(std::string Entry, int64_t& TotRows, - int64_t& InsRows); - static bool prepareColumnOutOfRangeInfo(std::string Entry, int& ColNum, - execplan::CalpontSystemCatalog::ColDataType& ColType, - std::string& ColName, int& OorValues); - static bool prepareErrorFileInfo(std::string Entry, std::string& ErrFileName); - static bool prepareBadDataFileInfo(std::string Entry, std::string& BadFileName); - -private: - WESDHandler& fRef; - BRM::DBRM* fpBrm; - - - //BRM::CPInfoMergeList_t fCPInfo; - std::vector fCPInfo; - std::vector fHWMInfo; - std::vector fCPInfoData; - - + // BRM::CPInfoMergeList_t fCPInfo; + std::vector fCPInfo; + std::vector fHWMInfo; + std::vector fCPInfoData; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index be01e9e5b..27efb1d37 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -47,1328 +47,1316 @@ using namespace oam; namespace WriteEngine { - //---------------------------------------------------------------------- //---------------------------------------------------------------------- -WECmdArgs::WECmdArgs(int argc, char** argv) : - fMultiTableCount(0), - fJobLogOnly(false), - fHelp(false), - fMode(1), - fArgMode(-1), - fQuiteMode(true), - fConsoleLog(false), - fVerbose(0), - fBatchQty(10000), - fNoOfReadThrds(0), - fDebugLvl(0), - fMaxErrors(-1), - fReadBufSize(0), - fIOReadBufSize(0), - fSetBufSize(0), - fColDelim('|'), - fEnclosedChar(0), - fEscChar(0), - fNoOfWriteThrds(0), - fNullStrMode(false), - fImportDataMode(IMPORT_DATA_TEXT), - fCpiInvoke(false), - fBlockMode3(false), - fbTruncationAsError(false), - fUUID(boost::uuids::nil_generator()()), - fConsoleOutput(true), - fTimeZone("SYSTEM"), - fErrorDir(string(MCSLOGDIR)+"/cpimport/") +WECmdArgs::WECmdArgs(int argc, char** argv) + : fMultiTableCount(0) + , fJobLogOnly(false) + , fHelp(false) + , fMode(1) + , fArgMode(-1) + , fQuiteMode(true) + , fConsoleLog(false) + , fVerbose(0) + , fBatchQty(10000) + , fNoOfReadThrds(0) + , fDebugLvl(0) + , fMaxErrors(-1) + , fReadBufSize(0) + , fIOReadBufSize(0) + , fSetBufSize(0) + , fColDelim('|') + , fEnclosedChar(0) + , fEscChar(0) + , fNoOfWriteThrds(0) + , fNullStrMode(false) + , fImportDataMode(IMPORT_DATA_TEXT) + , fCpiInvoke(false) + , fBlockMode3(false) + , fbTruncationAsError(false) + , fUUID(boost::uuids::nil_generator()()) + , fConsoleOutput(true) + , fTimeZone("SYSTEM") + , fErrorDir(string(MCSLOGDIR) + "/cpimport/") { - try - { - appTestFunction(); - parseCmdLineArgs(argc, argv); - } - catch (std::exception& exp) - { - std::string exceptMsg( exp.what() ); - exceptMsg += "\nTry 'cpimport -h' for more information."; - throw (runtime_error( exceptMsg )); - } + try + { + appTestFunction(); + parseCmdLineArgs(argc, argv); + } + catch (std::exception& exp) + { + std::string exceptMsg(exp.what()); + exceptMsg += "\nTry 'cpimport -h' for more information."; + throw(runtime_error(exceptMsg)); + } } //---------------------------------------------------------------------- void WECmdArgs::appTestFunction() { + // testing begins + // std::string aJobFile("/home/bpaul/Calpont/data/bulk/job/Job_1234.xml"); + // std::string aJobFile("/usr/local/mariadb/columnstore/data/bulk/job/Job_1234.xml"); + // setSchemaAndTableFromJobFile(aJobFile); + // setEnclByAndEscCharFromJobFile(aJobFile); + // exit(1); - // testing begins - //std::string aJobFile("/home/bpaul/Calpont/data/bulk/job/Job_1234.xml"); - //std::string aJobFile("/usr/local/mariadb/columnstore/data/bulk/job/Job_1234.xml"); - //setSchemaAndTableFromJobFile(aJobFile); - //setEnclByAndEscCharFromJobFile(aJobFile); - //exit(1); - - //testing ends - return; + // testing ends + return; } //---------------------------------------------------------------------- std::string WECmdArgs::getCpImportCmdLine() { - std::ostringstream aSS; - std::string aCmdLine; + std::ostringstream aSS; + std::string aCmdLine; + aSS << fPrgmName; // prgm name as arg[0] - aSS << fPrgmName; //prgm name as arg[0] - - if (fHelp) - { - aSS << " -h "; - aCmdLine = aSS.str(); - return aCmdLine; - } - - //checkJobIdCase(); // check if JobID - - - if ((fPmFile.length() > 0) && (0 == getMode())) - aSS << " -l " << fPmFile; - - //BUG 4379 if -m is not given while prep args, default is m=1 but - //on single node -m will set it to 3, after prep args. - //if((fPmFilePath.length()>0)&&(1!=getMode())) - // aSS << " -f " << fPmFilePath; - if ((fPmFilePath.length() > 0) && (1 != getMode())) - { - if (fPmFilePath == "STDIN") //if stdin, must pass it - aSS << " -f " << fPmFilePath; - else if (2 == getArgMode()) //Mode 2 we need to pass the -f option - aSS << " -f " << fPmFilePath; - else if (3 == getArgMode()) //-m given, -f built in already. - aSS << " -f " << fPmFilePath; - else if (0 == fLocFile.length()) //No filename given, from job file - aSS << " -f " << fPmFilePath; - } - - if (fErrorDir.length() > 0) - aSS << " -L " << fErrorDir; - - if (fUsername.length() > 0) - aSS << " -U " << fUsername; - - if (fJobId.length() > 0) - aSS << " -j " << fJobId; - - if (fNoOfReadThrds > 0) - aSS << " -r " << fNoOfReadThrds; - - if (fNoOfWriteThrds > 0) - aSS << " -w " << fNoOfWriteThrds; - - if (fMaxErrors >= 0) - aSS << " -e " << fMaxErrors; - - // BUG 5088 - if (fDebugLvl > 0) - aSS << " -d " << fDebugLvl; - - if (fSetBufSize > 0) - aSS << " -B " << fSetBufSize; - - if (fColDelim != '|') - { - if (fColDelim == '\t') - aSS << " -s " << "\\t"; //- working with user '\t' - // NONE of the following will work. - //aSS << " -s " << "\t"; //aSS << " -s " << "\"\\t\""; //aSS << " -s " << "'\\t'"; - else - aSS << " -s " << fColDelim; - } - - if (fEnclosedChar != 0) - aSS << " -E " << fEnclosedChar; - - if (fEscChar != 0) - aSS << " -C " << fEscChar; - - if (fNullStrMode) - aSS << " -n " << '1'; - - if (fImportDataMode != IMPORT_DATA_TEXT) - aSS << " -I " << fImportDataMode; - - //if(fConfig.length()>0) - // aSS << " -c " << fConfig; - if (fReadBufSize > 0) - { - cout << "setting fReadBufSize = " << fReadBufSize; - aSS << " -c " << fReadBufSize; - } - - if (fIOReadBufSize > 0) - aSS << " -b " << fIOReadBufSize; - - - if ((fJobPath.length() > 0) && (fMode == 3)) - aSS << " -p " << fJobPath; - - - if (fConsoleLog) - aSS << " -i "; - - if ((fMode == 1) || (fMode == 2)) - { - aSS << " -R " << getBrmRptFileName(); - aSS << " -m " << fMode; - } - - aSS << " -P " << getModuleID(); - aSS << " -T " << fTimeZone; - - if (fbTruncationAsError) - aSS << " -S "; - - if (!fS3Key.empty() && !(fMode == 0 || fMode == 1)) - { - if (fS3Secret.empty() || fS3Bucket.empty() || fS3Region.empty()) - throw (runtime_error("Not all required S3 options provided")); - aSS << " -y " << fS3Key; - aSS << " -K " << fS3Secret; - aSS << " -t " << fS3Bucket; - aSS << " -g " << fS3Region; - - if (!fS3Host.empty()) - { - aSS << " -H " << fS3Host; - } - } - - if ((fJobId.length() > 0) && (fMode == 1) && (!fJobLogOnly)) - { - // if JobPath provided, make it w.r.t WES - aSS << " -p " << fTmpFileDir; - aSS << " -fSTDIN"; - } - else if ((fJobId.length() > 0) && (fMode == 2) && (!fJobLogOnly)) - { - // if JobPath provided, make it w.r.t WES - aSS << " -p " << fTmpFileDir; - - if (fPmFile.length() > 0) - aSS << " -l " << fPmFile; - - if (fPmFilePath.length() > 0) - aSS << " -f " << fPmFilePath; - } - else // do not provide schema & table with JobId - { - - if (!fUUID.is_nil()) - aSS << " -u" << boost::uuids::to_string(fUUID); - - if (fSchema.length() > 0) - aSS << " " << fSchema; - //else if((fMode != 0)||(fMode==3)) //TODO make it mode3 + jobID - else if (fJobId.length() > 0) - { }// may or may not contain Schema. - //else if((fMode == 1)||(fMode==2)) //TODO make it mode3 + jobID - else if (fMode != 0) - throw (runtime_error("Schema not available")); - - if (fTable.length() > 0) - aSS << " " << fTable; - else if (fJobId.length() > 0) - {} //may or may not contain Table. - else if (fMode != 0) - throw (runtime_error("Tablename not available")); - - //else if((fMode != 0)||(fMode==3)) //TODO make it mode3 + jobID - //else if((fMode == 1)||(fMode == 2)) //TODO make it mode3 + jobID - // throw (runtime_error("Tablename not available")); - if ((fPmFile.length() > 0) && (2 == getMode())) - { - //if(fPmFilePath.length()>0) - // aSS << " " << fPmFilePath; - aSS << " " << fPmFile; - } - else if (2 == getMode()) - throw (runtime_error("loadFile [-l ] not available")); - - } - - if ((fLocFile.length() > 0) && (fLocFile != "STDIN") && (3 == getMode())) - { - //Bug 4342 multi-files mode 3 support - //convert all the spaces into 'commas' - if (fLocFile.find_first_of(' ') == string::npos) - aSS << " " << fLocFile; - else - { - std::string aLocFiles = replaceCharInStr(fLocFile, ' ', ','); - aSS << " " << aLocFiles; - } - } - - try - { - aCmdLine = aSS.str(); - } - catch (exception&) - { - throw runtime_error("getcpImportCmdLine failed"); - } - + if (fHelp) + { + aSS << " -h "; + aCmdLine = aSS.str(); return aCmdLine; -} + } + // checkJobIdCase(); // check if JobID + + if ((fPmFile.length() > 0) && (0 == getMode())) + aSS << " -l " << fPmFile; + + // BUG 4379 if -m is not given while prep args, default is m=1 but + // on single node -m will set it to 3, after prep args. + // if((fPmFilePath.length()>0)&&(1!=getMode())) + // aSS << " -f " << fPmFilePath; + if ((fPmFilePath.length() > 0) && (1 != getMode())) + { + if (fPmFilePath == "STDIN") // if stdin, must pass it + aSS << " -f " << fPmFilePath; + else if (2 == getArgMode()) // Mode 2 we need to pass the -f option + aSS << " -f " << fPmFilePath; + else if (3 == getArgMode()) //-m given, -f built in already. + aSS << " -f " << fPmFilePath; + else if (0 == fLocFile.length()) // No filename given, from job file + aSS << " -f " << fPmFilePath; + } + + if (fErrorDir.length() > 0) + aSS << " -L " << fErrorDir; + + if (fUsername.length() > 0) + aSS << " -U " << fUsername; + + if (fJobId.length() > 0) + aSS << " -j " << fJobId; + + if (fNoOfReadThrds > 0) + aSS << " -r " << fNoOfReadThrds; + + if (fNoOfWriteThrds > 0) + aSS << " -w " << fNoOfWriteThrds; + + if (fMaxErrors >= 0) + aSS << " -e " << fMaxErrors; + + // BUG 5088 + if (fDebugLvl > 0) + aSS << " -d " << fDebugLvl; + + if (fSetBufSize > 0) + aSS << " -B " << fSetBufSize; + + if (fColDelim != '|') + { + if (fColDelim == '\t') + aSS << " -s " + << "\\t"; //- working with user '\t' + // NONE of the following will work. + // aSS << " -s " << "\t"; //aSS << " -s " << "\"\\t\""; //aSS << " -s " << "'\\t'"; + else + aSS << " -s " << fColDelim; + } + + if (fEnclosedChar != 0) + aSS << " -E " << fEnclosedChar; + + if (fEscChar != 0) + aSS << " -C " << fEscChar; + + if (fNullStrMode) + aSS << " -n " << '1'; + + if (fImportDataMode != IMPORT_DATA_TEXT) + aSS << " -I " << fImportDataMode; + + // if(fConfig.length()>0) + // aSS << " -c " << fConfig; + if (fReadBufSize > 0) + { + cout << "setting fReadBufSize = " << fReadBufSize; + aSS << " -c " << fReadBufSize; + } + + if (fIOReadBufSize > 0) + aSS << " -b " << fIOReadBufSize; + + if ((fJobPath.length() > 0) && (fMode == 3)) + aSS << " -p " << fJobPath; + + if (fConsoleLog) + aSS << " -i "; + + if ((fMode == 1) || (fMode == 2)) + { + aSS << " -R " << getBrmRptFileName(); + aSS << " -m " << fMode; + } + + aSS << " -P " << getModuleID(); + aSS << " -T " << fTimeZone; + + if (fbTruncationAsError) + aSS << " -S "; + + if (!fS3Key.empty() && !(fMode == 0 || fMode == 1)) + { + if (fS3Secret.empty() || fS3Bucket.empty() || fS3Region.empty()) + throw(runtime_error("Not all required S3 options provided")); + aSS << " -y " << fS3Key; + aSS << " -K " << fS3Secret; + aSS << " -t " << fS3Bucket; + aSS << " -g " << fS3Region; + + if (!fS3Host.empty()) + { + aSS << " -H " << fS3Host; + } + } + + if ((fJobId.length() > 0) && (fMode == 1) && (!fJobLogOnly)) + { + // if JobPath provided, make it w.r.t WES + aSS << " -p " << fTmpFileDir; + aSS << " -fSTDIN"; + } + else if ((fJobId.length() > 0) && (fMode == 2) && (!fJobLogOnly)) + { + // if JobPath provided, make it w.r.t WES + aSS << " -p " << fTmpFileDir; + + if (fPmFile.length() > 0) + aSS << " -l " << fPmFile; + + if (fPmFilePath.length() > 0) + aSS << " -f " << fPmFilePath; + } + else // do not provide schema & table with JobId + { + if (!fUUID.is_nil()) + aSS << " -u" << boost::uuids::to_string(fUUID); + + if (fSchema.length() > 0) + aSS << " " << fSchema; + // else if((fMode != 0)||(fMode==3)) //TODO make it mode3 + jobID + else if (fJobId.length() > 0) + { + } // may or may not contain Schema. + // else if((fMode == 1)||(fMode==2)) //TODO make it mode3 + jobID + else if (fMode != 0) + throw(runtime_error("Schema not available")); + + if (fTable.length() > 0) + aSS << " " << fTable; + else if (fJobId.length() > 0) + { + } // may or may not contain Table. + else if (fMode != 0) + throw(runtime_error("Tablename not available")); + + // else if((fMode != 0)||(fMode==3)) //TODO make it mode3 + jobID + // else if((fMode == 1)||(fMode == 2)) //TODO make it mode3 + jobID + // throw (runtime_error("Tablename not available")); + if ((fPmFile.length() > 0) && (2 == getMode())) + { + // if(fPmFilePath.length()>0) + // aSS << " " << fPmFilePath; + aSS << " " << fPmFile; + } + else if (2 == getMode()) + throw(runtime_error("loadFile [-l ] not available")); + } + + if ((fLocFile.length() > 0) && (fLocFile != "STDIN") && (3 == getMode())) + { + // Bug 4342 multi-files mode 3 support + // convert all the spaces into 'commas' + if (fLocFile.find_first_of(' ') == string::npos) + aSS << " " << fLocFile; + else + { + std::string aLocFiles = replaceCharInStr(fLocFile, ' ', ','); + aSS << " " << aLocFiles; + } + } + + try + { + aCmdLine = aSS.str(); + } + catch (exception&) + { + throw runtime_error("getcpImportCmdLine failed"); + } + + return aCmdLine; +} //---------------------------------------------------------------------- -//BUG 4205 (part FIX) - need to implement more into it +// BUG 4205 (part FIX) - need to implement more into it bool WECmdArgs::checkForCornerCases() { - //BUG 4210 - this->checkJobIdCase(); //Need to do this before we go further + // BUG 4210 + this->checkJobIdCase(); // Need to do this before we go further - - if (fMode == 0) + if (fMode == 0) + { + if (!fJobId.empty()) { - if (!fJobId.empty()) - { - //cout << "Invalid option mode 0 with a Job File" << endl; - throw (runtime_error("Mode 0 with a Job file option is not valid!!" - "\nTry 'cpimport -h' for more information.")); - } - else if (!fJobPath.empty()) - { - cout << "Invalid option mode 0 with a Job Path" << endl; - throw (runtime_error("Mismatched options" - "\nTry 'cpimport -h' for more information.")); - } - else if (!fSchema.empty()) - { - cout << "Invalid option in mode 0 with a schema name" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (!fTable.empty()) - { - cout << "Invalid option in mode 0 with a table name" << endl; - throw (runtime_error("Mismatched options.")); - } - else if ((!fPmFilePath.empty()) && (fPmFilePath != "STDIN")) - { - cout << "Invalid option -f in Mode 0 with value other than STDIN" << endl; - throw (runtime_error("Mismatched options.")); - } - - if (fSetBufSize) - { - cout << "Invalid option -B with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (fIOReadBufSize) - { - cout << "Invalid option -b with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (fMaxErrors >= 0) - { - cout << "Invalid option -e with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (fConsoleLog) - { - cout << "Invalid option -i with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (fReadBufSize) - { - cout << "Invalid option -c with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (fNoOfReadThrds) - { - cout << "Invalid option -r with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - else if (fNoOfWriteThrds) - { - cout << "Invalid option -w with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - - if (fImportDataMode != IMPORT_DATA_TEXT) - { - cout << "Invalid option -I with Mode 0" << endl; - throw (runtime_error("Mismatched options.")); - } - + // cout << "Invalid option mode 0 with a Job File" << endl; + throw( + runtime_error("Mode 0 with a Job file option is not valid!!" + "\nTry 'cpimport -h' for more information.")); + } + else if (!fJobPath.empty()) + { + cout << "Invalid option mode 0 with a Job Path" << endl; + throw( + runtime_error("Mismatched options" + "\nTry 'cpimport -h' for more information.")); + } + else if (!fSchema.empty()) + { + cout << "Invalid option in mode 0 with a schema name" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (!fTable.empty()) + { + cout << "Invalid option in mode 0 with a table name" << endl; + throw(runtime_error("Mismatched options.")); + } + else if ((!fPmFilePath.empty()) && (fPmFilePath != "STDIN")) + { + cout << "Invalid option -f in Mode 0 with value other than STDIN" << endl; + throw(runtime_error("Mismatched options.")); } - if (fMode == 1) + if (fSetBufSize) { - if (!fJobId.empty()) - { - if ((!fPmFilePath.empty()) && (fPmFilePath == "STDIN")) - { - // do not do anything.. this is good. - } - } - // Mode 1, if Input Path is existing and input file is not existing - // it is an error, bce it assumes all the files in directory. - // In mode 2, we are passing info to cpimport.bin, which will take care - // of it, as in Mode 3. - else if ((!fPmFilePath.empty()) && (fPmFile.empty())) - { - // assumed since position param is missing - if ((fLocFile == "STDIN") && (fPmFilePath != "STDIN")) - { - cout << "Invalid options in Mode 1 : option -l " << endl; - cout << " or input file position parameter needed" << endl; - //cout << "\tOption (-j) should follow with option -l option or " - // "an input file position parameter" << endl; - throw (runtime_error("In Mode 1 Error occurred!! " - "\nTry 'cpimport -h' for more information.")); - } - } + cout << "Invalid option -B with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (fIOReadBufSize) + { + cout << "Invalid option -b with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (fMaxErrors >= 0) + { + cout << "Invalid option -e with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (fConsoleLog) + { + cout << "Invalid option -i with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (fReadBufSize) + { + cout << "Invalid option -c with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (fNoOfReadThrds) + { + cout << "Invalid option -r with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); + } + else if (fNoOfWriteThrds) + { + cout << "Invalid option -w with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); } - if (fMode == 2) + if (fImportDataMode != IMPORT_DATA_TEXT) { - if (fPmFile.empty()) - throw(runtime_error("Mode 2 require PM based filename [-l]" - "\nTry 'cpimport -h' for more information.")); - - if ((fPmFilePath.empty()) && (fPmFile.at(0) != '/')) - throw(runtime_error("Mode 2 require remote file opts -f and -l or "\ - "a fully qualified path for the remote file." - "\nTry 'cpimport -h' for more information.")); - if (!fS3Key.empty()) - throw(runtime_error("Mode 2 & an input file from S3 does not make sense.")); + cout << "Invalid option -I with Mode 0" << endl; + throw(runtime_error("Mismatched options.")); } + } - if (fMode == 3) + if (fMode == 1) + { + if (!fJobId.empty()) { - if (fPmVec.size()) - { - cout << "Invalid option -P with Mode 3" << endl; - throw (runtime_error("Mismatched options.")); - } - + if ((!fPmFilePath.empty()) && (fPmFilePath == "STDIN")) + { + // do not do anything.. this is good. + } } + // Mode 1, if Input Path is existing and input file is not existing + // it is an error, bce it assumes all the files in directory. + // In mode 2, we are passing info to cpimport.bin, which will take care + // of it, as in Mode 3. + else if ((!fPmFilePath.empty()) && (fPmFile.empty())) + { + // assumed since position param is missing + if ((fLocFile == "STDIN") && (fPmFilePath != "STDIN")) + { + cout << "Invalid options in Mode 1 : option -l " << endl; + cout << " or input file position parameter needed" << endl; + // cout << "\tOption (-j) should follow with option -l option or " + // "an input file position parameter" << endl; + throw( + runtime_error("In Mode 1 Error occurred!! " + "\nTry 'cpimport -h' for more information.")); + } + } + } - return true; + if (fMode == 2) + { + if (fPmFile.empty()) + throw( + runtime_error("Mode 2 require PM based filename [-l]" + "\nTry 'cpimport -h' for more information.")); + + if ((fPmFilePath.empty()) && (fPmFile.at(0) != '/')) + throw( + runtime_error("Mode 2 require remote file opts -f and -l or " + "a fully qualified path for the remote file." + "\nTry 'cpimport -h' for more information.")); + if (!fS3Key.empty()) + throw(runtime_error("Mode 2 & an input file from S3 does not make sense.")); + } + + if (fMode == 3) + { + if (fPmVec.size()) + { + cout << "Invalid option -P with Mode 3" << endl; + throw(runtime_error("Mismatched options.")); + } + } + + return true; } //---------------------------------------------------------------------- bool WECmdArgs::str2PmList(std::string& PmList, VecInts& V) { - const int BUFLEN = 512; - char aBuff[BUFLEN]; + const int BUFLEN = 512; + char aBuff[BUFLEN]; + int aLen = PmList.length(); - int aLen = PmList.length(); + if (aLen > 0) + { + strncpy(aBuff, PmList.c_str(), BUFLEN); + aBuff[BUFLEN - 1] = 0; + } + else + return false; - if (aLen > 0) + char* pTok = strtok(aBuff, ","); + + while (pTok != NULL) + { + int aPmId = 0; + + try { - strncpy(aBuff, PmList.c_str(), BUFLEN); - aBuff[BUFLEN - 1] = 0; + aPmId = atoi(pTok); + V.push_back(aPmId); } - else - return false; - - char* pTok = strtok(aBuff, ","); - - while (pTok != NULL) + catch (exception& ex) { - int aPmId = 0; - - try - { - aPmId = atoi(pTok); - V.push_back(aPmId); - } - catch (exception& ex) - { - std::stringstream aErr; - aErr << "Wrong PM id format : " << ex.what(); - //cout << "Wrong PM id format : "<< ex.what() << endl; - throw (runtime_error(aErr.str())); - } - - pTok = strtok(NULL, ","); + std::stringstream aErr; + aErr << "Wrong PM id format : " << ex.what(); + // cout << "Wrong PM id format : "<< ex.what() << endl; + throw(runtime_error(aErr.str())); } - return true; + pTok = strtok(NULL, ","); + } + + return true; } //---------------------------------------------------------------------- void WECmdArgs::usage() { - cout << "Simple usage using positional parameters (no XML job file):\n"; - cout << "\tcpimport dbName tblName [loadFile] [-h] [-m mode]\n"; - cout << "\t\t [-f path] [-d debugLevel] [-c readbufSize] [-b readBufs] \n"; - cout << "\t\t [-r readers] [-j JobID] [-e maxErrs] [-B libBufSize] [-w parsers]\n"; - cout << "\t\t [-s c] [-E enclosedChar] [-C escapeChar] [-n NullOption]\n"; - cout << "\t\t [-q batchQty] [-p jobPath] [-P list of PMs] [-S] [-i] [-v verbose]\n"; - cout << "\t\t [-I binaryOpt] [-T timeZone]\n"; + cout << "Simple usage using positional parameters (no XML job file):\n"; + cout << "\tcpimport dbName tblName [loadFile] [-h] [-m mode]\n"; + cout << "\t\t [-f path] [-d debugLevel] [-c readbufSize] [-b readBufs] \n"; + cout << "\t\t [-r readers] [-j JobID] [-e maxErrs] [-B libBufSize] [-w parsers]\n"; + cout << "\t\t [-s c] [-E enclosedChar] [-C escapeChar] [-n NullOption]\n"; + cout << "\t\t [-q batchQty] [-p jobPath] [-P list of PMs] [-S] [-i] [-v verbose]\n"; + cout << "\t\t [-I binaryOpt] [-T timeZone]\n"; + cout << "Traditional usage without positional parameters (XML job file required):\n"; + cout << "\tcpimport -j jobID\n"; + cout << "\t\t [-h] [-m mode] [-r readers] [-w parsers] [-s c] [-f path]\n"; + cout << "\t\t [-b readBufs] [-p path] [-c readBufSize] [-e maxErrs] [-B libBufSize]\n"; + cout << "\t\t [-n NullOption] [-E encloseChar] [-C escapeChar] [-i] [-v verbose]\n"; + cout << "\t\t [-d debugLevel] [-q batchQty] [-l loadFile] [-P list of PMs] [-S]\n"; + cout << "\t\t [-I binaryOpt] [-T timeZone]\n"; - cout << "Traditional usage without positional parameters (XML job file required):\n"; - cout << "\tcpimport -j jobID\n"; - cout << "\t\t [-h] [-m mode] [-r readers] [-w parsers] [-s c] [-f path]\n"; - cout << "\t\t [-b readBufs] [-p path] [-c readBufSize] [-e maxErrs] [-B libBufSize]\n"; - cout << "\t\t [-n NullOption] [-E encloseChar] [-C escapeChar] [-i] [-v verbose]\n"; - cout << "\t\t [-d debugLevel] [-q batchQty] [-l loadFile] [-P list of PMs] [-S]\n"; - cout << "\t\t [-I binaryOpt] [-T timeZone]\n"; + cout << "\n\nPositional parameters:\n"; + cout << "\tdbName Name of the database to load\n"; + cout << "\ttblName Name of table to load\n"; + cout << "\tloadFile Optional input file name in current directory,\n"; + cout << "\t\t\tunless a fully qualified name is given.\n"; + cout << "\t\t\tIf not given, input read from STDIN.\n"; - cout << "\n\nPositional parameters:\n"; - cout << "\tdbName Name of the database to load\n"; - cout << "\ttblName Name of table to load\n"; - cout << "\tloadFile Optional input file name in current directory,\n"; - cout << "\t\t\tunless a fully qualified name is given.\n"; - cout << "\t\t\tIf not given, input read from STDIN.\n"; + cout << "\n\nOptions:\n" + << "\t-b\tNumber of read buffers\n" + << "\t-c\tApplication read buffer size(in bytes)\n" + << "\t-d\tPrint different level(1-3) debug message\n" + << "\t-e\tMax number of allowable error per table per PM\n" + << "\t-f\tData file directory path.\n" + << "\t\t\tDefault is current working directory.\n" + << "\t\t\tIn Mode 1, -f represents the local input file path.\n" + << "\t\t\tIn Mode 2, -f represents the PM based input file path.\n" + << "\t\t\tIn Mode 3, -f represents the local input file path.\n" + << "\t-l\tName of import file to be loaded, relative to -f path,\n" + << "\t-h\tPrint this message.\n" + << "\t-q\tBatch Quantity, Number of rows distributed per batch in Mode 1\n" + << "\t-i\tPrint extended info to console in Mode 3.\n" + << "\t-j\tJob ID. In simple usage, default is the table OID.\n" + << "\t\t\tunless a fully qualified input file name is given.\n" + << "\t-n\tNullOption (0-treat the string NULL as data (default);\n" + << "\t\t\t1-treat the string NULL as a NULL value)\n" + << "\t-p\tPath for XML job description file.\n" + << "\t-r\tNumber of readers.\n" + << "\t-s\t'c' is the delimiter between column values.\n" + << "\t-B\tI/O library read buffer size (in bytes)\n" + << "\t-w\tNumber of parsers.\n" + << "\t-E\tEnclosed by character if field values are enclosed.\n" + << "\t-C\tEscape character used in conjunction with 'enclosed by'\n" + << "\t\t\tcharacter, or as part of NULL escape sequence ('\\N');\n" + << "\t\t\tdefault is '\\'\n" + << "\t-I\tImport binary data; how to treat NULL values:\n" + << "\t\t\t1 - import NULL values\n" + << "\t\t\t2 - saturate NULL values\n" + << "\t-P\tList of PMs ex: -P 1,2,3. Default is all PMs.\n" + << "\t-S\tTreat string truncations as errors.\n" + << "\t-m\tmode\n" + << "\t\t\t1 - rows will be loaded in a distributed manner across PMs.\n" + << "\t\t\t2 - PM based input files loaded onto their respective PM.\n" + << "\t\t\t3 - input files will be loaded on the local PM.\n" + << "\t-T\tTimezone used for TIMESTAMP datatype.\n" + << "\t\tPossible values: \"SYSTEM\" (default)\n" + << "\t\t : Offset in the form +/-HH:MM\n" + << "\t-y\tS3 Authentication Key (for S3 imports)\n" + << "\t-K\tS3 Authentication Secret (for S3 imports)\n" + << "\t-t\tS3 Bucket (for S3 imports)\n" + << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" + << "\t-g\tS3 Region (for S3 imports)\n" + << "\t-L\tDirectory for the output .err and .bad files.\n" + << "\t\tDefault is " << string(MCSLOGDIR); - cout << "\n\nOptions:\n" - << "\t-b\tNumber of read buffers\n" - << "\t-c\tApplication read buffer size(in bytes)\n" - << "\t-d\tPrint different level(1-3) debug message\n" - << "\t-e\tMax number of allowable error per table per PM\n" - << "\t-f\tData file directory path.\n" - << "\t\t\tDefault is current working directory.\n" - << "\t\t\tIn Mode 1, -f represents the local input file path.\n" - << "\t\t\tIn Mode 2, -f represents the PM based input file path.\n" - << "\t\t\tIn Mode 3, -f represents the local input file path.\n" - << "\t-l\tName of import file to be loaded, relative to -f path,\n" - << "\t-h\tPrint this message.\n" - << "\t-q\tBatch Quantity, Number of rows distributed per batch in Mode 1\n" - << "\t-i\tPrint extended info to console in Mode 3.\n" - << "\t-j\tJob ID. In simple usage, default is the table OID.\n" - << "\t\t\tunless a fully qualified input file name is given.\n" - << "\t-n\tNullOption (0-treat the string NULL as data (default);\n" - << "\t\t\t1-treat the string NULL as a NULL value)\n" - << "\t-p\tPath for XML job description file.\n" - << "\t-r\tNumber of readers.\n" - << "\t-s\t'c' is the delimiter between column values.\n" - << "\t-B\tI/O library read buffer size (in bytes)\n" - << "\t-w\tNumber of parsers.\n" - << "\t-E\tEnclosed by character if field values are enclosed.\n" - << "\t-C\tEscape character used in conjunction with 'enclosed by'\n" - << "\t\t\tcharacter, or as part of NULL escape sequence ('\\N');\n" - << "\t\t\tdefault is '\\'\n" - << "\t-I\tImport binary data; how to treat NULL values:\n" - << "\t\t\t1 - import NULL values\n" - << "\t\t\t2 - saturate NULL values\n" - << "\t-P\tList of PMs ex: -P 1,2,3. Default is all PMs.\n" - << "\t-S\tTreat string truncations as errors.\n" - << "\t-m\tmode\n" - << "\t\t\t1 - rows will be loaded in a distributed manner across PMs.\n" - << "\t\t\t2 - PM based input files loaded onto their respective PM.\n" - << "\t\t\t3 - input files will be loaded on the local PM.\n" - << "\t-T\tTimezone used for TIMESTAMP datatype.\n" - << "\t\tPossible values: \"SYSTEM\" (default)\n" - << "\t\t : Offset in the form +/-HH:MM\n" - << "\t-y\tS3 Authentication Key (for S3 imports)\n" - << "\t-K\tS3 Authentication Secret (for S3 imports)\n" - << "\t-t\tS3 Bucket (for S3 imports)\n" - << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" - << "\t-g\tS3 Region (for S3 imports)\n" - << "\t-L\tDirectory for the output .err and .bad files.\n" - << "\t\tDefault is " << string(MCSLOGDIR); + cout << "\nExample1: Traditional usage\n" + << "\tcpimport -j 1234"; + cout << "\nExample2: Some column values are enclosed within double quotes.\n" + << "\tcpimport -j 3000 -E '\"'"; + cout << "\nExample3: Import a nation table without a Job XML file\n" + << "\tcpimport -j 301 tpch nation nation.tbl"; + cout << "\nExample4: Import a nation table to all PMs in Mode 1\n" + << "\tcpimport -m 1 tpch nation nation.tbl"; + cout << "\nExample4: Import a nation table to only PM1 and PM2 in Mode 1\n" + << "\tcpimport -m 1 -P 1,2 tpch nation nation.tbl"; + cout << "\nExample5: Import nation.tbl from PMs to nation table in Mode 2\n" + << "\tcpimport -m 2 tpch nation -f /var/lib/columnstore/data/bulk/data/import/ -l nation.tbl"; + cout << "\nExample6: Import nation.tbl in mode 3\n" + << "\tcpimport -m 3 tpch nation nation.tbl\n\n"; - - cout << "\nExample1: Traditional usage\n" - << "\tcpimport -j 1234"; - cout << "\nExample2: Some column values are enclosed within double quotes.\n" - << "\tcpimport -j 3000 -E '\"'"; - cout << "\nExample3: Import a nation table without a Job XML file\n" - << "\tcpimport -j 301 tpch nation nation.tbl"; - cout << "\nExample4: Import a nation table to all PMs in Mode 1\n" - << "\tcpimport -m 1 tpch nation nation.tbl"; - cout << "\nExample4: Import a nation table to only PM1 and PM2 in Mode 1\n" - << "\tcpimport -m 1 -P 1,2 tpch nation nation.tbl"; - cout << "\nExample5: Import nation.tbl from PMs to nation table in Mode 2\n" - << "\tcpimport -m 2 tpch nation -f /var/lib/columnstore/data/bulk/data/import/ -l nation.tbl"; - cout << "\nExample6: Import nation.tbl in mode 3\n" - << "\tcpimport -m 3 tpch nation nation.tbl\n\n"; - - - exit(1); + exit(1); } - //----------------------------------------------------------------------------- void WECmdArgs::parseCmdLineArgs(int argc, char** argv) { - int aCh; - std::string importPath; - bool aJobType = false; + int aCh; + std::string importPath; + bool aJobType = false; + if (argc > 0) + fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; // argv[0] is splitter but we need cpimport - if (argc > 0) - fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; //argv[0] is splitter but we need cpimport - - while ((aCh = getopt(argc, argv, - "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) - != EOF) + while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF) + { + switch (aCh) { - switch (aCh) + case 'm': + { + fArgMode = atoi(optarg); + + // cout << "Mode level set to " << fMode << endl; + if ((fArgMode > -1) && (fArgMode <= 3)) { - case 'm': - { - fArgMode = atoi(optarg); - - //cout << "Mode level set to " << fMode << endl; - if ((fArgMode > -1) && (fArgMode <= 3)) {} - else - throw runtime_error("Wrong Mode level"); - - break; - } - - case 'B': - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) - throw runtime_error("Option -B is invalid or out of range"); - - fSetBufSize = lValue; - break; - } - - case 'b': - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) - throw runtime_error("Option -b is invalid or out of range"); - - fIOReadBufSize = lValue; - break; - } - - case 'e': - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 0) || (lValue > INT_MAX)) - throw runtime_error("Option -e is invalid or out of range"); - - fMaxErrors = lValue; - break; - } - - case 'i': - { - fConsoleLog = true; - break; - } - - case 'c': - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) - throw runtime_error("Option -c is invalid or out of range"); - - fReadBufSize = lValue; - break; - } - - case 'j': // -j: jobID - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 0) || (lValue > INT_MAX)) - throw runtime_error("Option -j is invalid or out of range"); - - fJobId = optarg; - fOrigJobId = fJobId; // in case if we need to split it. - - if (0 == fJobId.length()) throw runtime_error("Wrong JobID Value"); - - aJobType = true; - break; - } - - case 'v': // verbose - { - string aVerbLen = optarg; - fVerbose = aVerbLen.length(); - fDebugLvl = fVerbose; - break; - } - - case 'd': // -d debug - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) - throw runtime_error("Option -d is invalid or out of range"); - - fDebugLvl = lValue; - - if (fDebugLvl > 0 && fDebugLvl <= 3) - { - cout << "\nDebug level set to " << fDebugLvl << endl; - } - else - { - throw runtime_error("Wrong Debug level"); - } - - break; - } - - case 'r': // -r: num read threads - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) - throw runtime_error("Option -r is invalid or out of range"); - - fNoOfReadThrds = lValue; - break; - } - - case 'w': // -w: num parse threads - { - errno = 0; - long lValue = strtol(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) - throw runtime_error("Option -w is invalid or out of range"); - - fNoOfWriteThrds = lValue; - break; - } - - case 's': // -s: column delimiter - { - if (!strcmp(optarg, "\\t")) - { - fColDelim = '\t'; - - if (fDebugLvl) cout << "Column delimiter : " << "\\t" << endl; - } - else - { - fColDelim = optarg[0]; - - if (fDebugLvl) cout << "Column delimiter : " << fColDelim << endl; - } - - break; - } - - case 'l': // -l: if JobId (-j), it can be input file - { - fPmFile = optarg; - - if (0 == fPmFile.length()) throw runtime_error("Wrong local filename"); - - break; - } - - case 'f': // -f: import file path - { - fPmFilePath = optarg; - break; - } - - case 'n': // -n: treat "NULL" as null - { - // default is 0, ie it is equal to not giving this option - int nullStringMode = atoi(optarg); - - if ((nullStringMode != 0) && (nullStringMode != 1)) - { - throw (runtime_error( - "Invalid NULL option; value can be 0 or 1")); - } - - if (nullStringMode) - fNullStrMode = true; - else - fNullStrMode = false; // This is default - - break; - } - - case 'P': // -p: list of PM's - { - try - { - std::string aPmList = optarg; - - if (!str2PmList(aPmList, fPmVec)) - throw(runtime_error("PM list is wrong")); - } - catch (runtime_error& ex) - { - throw (ex); - } - - break; - } - - case 'p': - { - fJobPath = optarg; - break; - } - - case 'E': // -E: enclosed by char - { - fEnclosedChar = optarg[0]; - //cout << "Enclosed by Character : " << optarg[0] << endl; - break; - } - - case 'C': // -C: enclosed escape char - { - fEscChar = optarg[0]; - //cout << "Escape Character : " << optarg[0] << endl; - break; - } - - case 'h': // -h: help - { - //usage(); // will exit(1) here - fHelp = true; - break; - } - - case 'I': // -I: binary mode (null handling) - { - // default is text mode, unless -I option is specified - int binaryMode = atoi(optarg); - - if (binaryMode == 1) - { - fImportDataMode = IMPORT_DATA_BIN_ACCEPT_NULL; - } - else if (binaryMode == 2) - { - fImportDataMode = IMPORT_DATA_BIN_SAT_NULL; - } - else - { - throw (runtime_error( - "Invalid Binary mode; value can be 1 or 2")); - } - - break; - } - - case 'S': // -S: Treat string truncations as errors - { - setTruncationAsError(true); - //cout << "TruncationAsError : true" << endl; - break; - } - - case 'T': - { - std::string timeZone = optarg; - long offset; - - if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - throw (runtime_error( - "Value for option -T is invalid")); - } - - fTimeZone = timeZone; - break; - } - - case 'q': // -q: batch quantity - default value is 10000 - { - errno = 0; - long long lValue = strtoll(optarg, 0, 10); - - if ((errno != 0) || (lValue < 1) || (lValue > UINT_MAX)) - throw runtime_error("Option -q is invalid or out of range"); - - fBatchQty = lValue; - - if (fBatchQty < 10000) fBatchQty = 10000; - else if (fBatchQty > 100000) fBatchQty = 10000; - - break; - } - - case 'N': //-N no console output - { - fConsoleOutput = false; - break; - } - - case 'y': //-y S3 Key - { - fS3Key = optarg; - break; - } - - case 'K': //-K S3 Secret - { - fS3Secret = optarg; - break; - } - - case 'H': //-H S3 Host - { - fS3Host = optarg; - break; - } - - case 't': //-t S3 bucket - { - fS3Bucket = optarg; - break; - } - - case 'g': //-g S3 Region - { - fS3Region = optarg; - break; - } - - case 'U': //-U username of the files owner - { - fUsername = optarg; - break; - } - - case 'L': // -L set the output location of .bad/.err files - { - fErrorDir = optarg; - break; - } - - default: - { - std::string aErr = std::string("Unknown command line option ") + std::to_string(aCh); - //cout << "Unknown command line option " << aCh << endl; - throw (runtime_error(aErr)); - } } - } + else + throw runtime_error("Wrong Mode level"); - if (fHelp) usage(); //BUG 4210 + break; + } - if (fArgMode != -1) fMode = fArgMode; //BUG 4210 + case 'B': + { + errno = 0; + long lValue = strtol(optarg, 0, 10); - std::string bulkRootPath = getBulkRootDir(); + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + throw runtime_error("Option -B is invalid or out of range"); - checkForBulkLogDir(bulkRootPath); + fSetBufSize = lValue; + break; + } - if (2 == fArgMode && fPmFilePath.empty()) - throw runtime_error("-f option is mandatory with mode 2."); + case 'b': + { + errno = 0; + long lValue = strtol(optarg, 0, 10); - if (aJobType) - { - if (0 == fArgMode) throw runtime_error("Incompatible mode and option types"); + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + throw runtime_error("Option -b is invalid or out of range"); - if (optind < argc) + fIOReadBufSize = lValue; + break; + } + + case 'e': + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 0) || (lValue > INT_MAX)) + throw runtime_error("Option -e is invalid or out of range"); + + fMaxErrors = lValue; + break; + } + + case 'i': + { + fConsoleLog = true; + break; + } + + case 'c': + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + throw runtime_error("Option -c is invalid or out of range"); + + fReadBufSize = lValue; + break; + } + + case 'j': // -j: jobID + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 0) || (lValue > INT_MAX)) + throw runtime_error("Option -j is invalid or out of range"); + + fJobId = optarg; + fOrigJobId = fJobId; // in case if we need to split it. + + if (0 == fJobId.length()) + throw runtime_error("Wrong JobID Value"); + + aJobType = true; + break; + } + + case 'v': // verbose + { + string aVerbLen = optarg; + fVerbose = aVerbLen.length(); + fDebugLvl = fVerbose; + break; + } + + case 'd': // -d debug + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + throw runtime_error("Option -d is invalid or out of range"); + + fDebugLvl = lValue; + + if (fDebugLvl > 0 && fDebugLvl <= 3) { - fSchema = argv[optind]; // 1st pos parm - optind++; - - if (optind < argc) - { - fTable = argv[optind]; // 2nd pos parm - optind++; - } - else - { - // if schema is there, table name should be there - throw runtime_error("No table name specified with schema."); - } - - if (optind < argc) // see if input file name is given - { - // 3rd pos parm - fLocFile = argv[optind]; - - if ((fLocFile.at(0) != '/') && (fLocFile != "STDIN")) - { - std::string aTmp = fLocFile; - - // BUG 4379 -f given? use that - if ((!fPmFilePath.empty()) && (fMode == 1)) - fLocFile = fPmFilePath + "/" + aTmp; - else if (fPmFilePath.empty()) - fLocFile = bulkRootPath + "/data/import/" + aTmp; - } - } - else - { - if (!fPmFile.empty()) - fLocFile = fPmFile; - //BUG 4186 - //else // else take it from the jobxml file - // fLocFile = "STDIN"; - //Historically cpimport works with jobfile as - // -l && -f or - // -fSTDIN as the stdin, it will override colxml loadfile entry - // if -fSTDIN is not provided get i/p file from jobfile - else if ((!fPmFilePath.empty()) && (fPmFilePath == "STDIN")) - fLocFile = "STDIN"; - - // else take it from the jobxml file - } - - if ((fSchema.length() > 0) && (fTable.length() > 0) && (fLocFile.length() > 0)) - fJobLogOnly = true; + cout << "\nDebug level set to " << fDebugLvl << endl; } else { - if (!fPmFile.empty()) - { - fLocFile = fPmFile; - - if (!fPmFilePath.empty()) - { - if (fPmFilePath == "STDIN") - { - throw runtime_error("Conflicting options -l and -fSTDIN"); - } - else - { - std::string aTmp = fLocFile; - - if ((!fPmFilePath.empty()) && (fMode == 1)) //BUG 4379 -f given? use that - fLocFile = fPmFilePath + "/" + aTmp; - else if (!fPmFilePath.empty()) - fLocFile = bulkRootPath + "/data/import/" + aTmp; - } - - } - - if ((fLocFile.at(0) != '/') && (fLocFile != "STDIN") && (fPmFilePath.empty())) - { - std::string aTmp = fLocFile; - fLocFile = bulkRootPath + "/data/import/" + aTmp; - } - } - //BUG 4186 - //else - // fLocFile = "STDIN"; - //Historically cpimport works with jobfile as - // -l && -f or - // -fSTDIN as the stdin, it will override colxml loadfile entry - // if -fSTDIN is not provided get i/p file from jobfile - else if ((!fPmFilePath.empty()) && (fPmFilePath == "STDIN")) - fLocFile = "STDIN"; - - // else take it from the jobxml file + throw runtime_error("Wrong Debug level"); } - } - // Get positional arguments, User can provide: - // 1. no positional parameters - Mode 0 & stdin - // 2. Two positional parameters (schema and table names) - Mode 1/2, stdin - // 3. Three positional parameters (schema, table, and import file name) - else if (optind < argc) // see if db schema name is given - { - if (fArgMode == 0) + break; + } + + case 'r': // -r: num read threads + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + throw runtime_error("Option -r is invalid or out of range"); + + fNoOfReadThrds = lValue; + break; + } + + case 'w': // -w: num parse threads + { + errno = 0; + long lValue = strtol(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > INT_MAX)) + throw runtime_error("Option -w is invalid or out of range"); + + fNoOfWriteThrds = lValue; + break; + } + + case 's': // -s: column delimiter + { + if (!strcmp(optarg, "\\t")) { - //added the code as per BUG 4245 - if (!fPmFilePath.empty()) - { - fLocFile = fPmFilePath; + fColDelim = '\t'; - if (fLocFile != "STDIN") - throw(runtime_error("ERROR: In Mode 0, -f option can only have value STDIN")); - } - else - { - fLocFile = argv[optind]; - optind++; - } - - if (optind < argc) //dest filename provided - { - fPmFile = argv[optind]; - - if ((fPmFile.at(0) != '/') && (fS3Key.empty())) - { - std::string aTmp = fPmFile; - fPmFile = bulkRootPath + "/data/import/" + aTmp; - } - } - else // no dest filename - { - if (fLocFile == "STDIN") - throw(runtime_error("ERROR: Destination file name required!!")); - - if (fLocFile.at(0) == '/') //local FQ-filename,parse out filename - fPmFile = getFileNameFromPath(fLocFile); - else - fPmFile = fLocFile; - - if ((fPmFile.at(0) != '/') && (fS3Key.empty())) //should be true all the time - { - std::string aTmp = fPmFile; - fPmFile = bulkRootPath + "/data/import/" + aTmp; - } - } - - /* commented out for BUG 4245 - if(fPmFilePath.empty()) - fLocFile = argv[optind]; - else - fLocFile = fPmFilePath +"/"+ argv[optind]; - - if (fPmFile.empty()) //BUG 4200 - { - //if(fLocFile.at(0)== '/') - // fPmFile = fLocFile; - //else - if(fLocFile.at(0)!='/') - fPmFile = bulkRootPath + "/data/import/"+ fLocFile; - } - else - { - if(fPmFile.at(0)!='/') - { - std::string aTmp = fPmFile; - fPmFile = bulkRootPath + "/data/import/"+aTmp; - } - } - */ + if (fDebugLvl) + cout << "Column delimiter : " + << "\\t" << endl; } else - fSchema = argv[optind]; // 1st pos parm + { + fColDelim = optarg[0]; + if (fDebugLvl) + cout << "Column delimiter : " << fColDelim << endl; + } + + break; + } + + case 'l': // -l: if JobId (-j), it can be input file + { + fPmFile = optarg; + + if (0 == fPmFile.length()) + throw runtime_error("Wrong local filename"); + + break; + } + + case 'f': // -f: import file path + { + fPmFilePath = optarg; + break; + } + + case 'n': // -n: treat "NULL" as null + { + // default is 0, ie it is equal to not giving this option + int nullStringMode = atoi(optarg); + + if ((nullStringMode != 0) && (nullStringMode != 1)) + { + throw(runtime_error("Invalid NULL option; value can be 0 or 1")); + } + + if (nullStringMode) + fNullStrMode = true; + else + fNullStrMode = false; // This is default + + break; + } + + case 'P': // -p: list of PM's + { + try + { + std::string aPmList = optarg; + + if (!str2PmList(aPmList, fPmVec)) + throw(runtime_error("PM list is wrong")); + } + catch (runtime_error& ex) + { + throw(ex); + } + + break; + } + + case 'p': + { + fJobPath = optarg; + break; + } + + case 'E': // -E: enclosed by char + { + fEnclosedChar = optarg[0]; + // cout << "Enclosed by Character : " << optarg[0] << endl; + break; + } + + case 'C': // -C: enclosed escape char + { + fEscChar = optarg[0]; + // cout << "Escape Character : " << optarg[0] << endl; + break; + } + + case 'h': // -h: help + { + // usage(); // will exit(1) here + fHelp = true; + break; + } + + case 'I': // -I: binary mode (null handling) + { + // default is text mode, unless -I option is specified + int binaryMode = atoi(optarg); + + if (binaryMode == 1) + { + fImportDataMode = IMPORT_DATA_BIN_ACCEPT_NULL; + } + else if (binaryMode == 2) + { + fImportDataMode = IMPORT_DATA_BIN_SAT_NULL; + } + else + { + throw(runtime_error("Invalid Binary mode; value can be 1 or 2")); + } + + break; + } + + case 'S': // -S: Treat string truncations as errors + { + setTruncationAsError(true); + // cout << "TruncationAsError : true" << endl; + break; + } + + case 'T': + { + std::string timeZone = optarg; + long offset; + + if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + throw(runtime_error("Value for option -T is invalid")); + } + + fTimeZone = timeZone; + break; + } + + case 'q': // -q: batch quantity - default value is 10000 + { + errno = 0; + long long lValue = strtoll(optarg, 0, 10); + + if ((errno != 0) || (lValue < 1) || (lValue > UINT_MAX)) + throw runtime_error("Option -q is invalid or out of range"); + + fBatchQty = lValue; + + if (fBatchQty < 10000) + fBatchQty = 10000; + else if (fBatchQty > 100000) + fBatchQty = 10000; + + break; + } + + case 'N': //-N no console output + { + fConsoleOutput = false; + break; + } + + case 'y': //-y S3 Key + { + fS3Key = optarg; + break; + } + + case 'K': //-K S3 Secret + { + fS3Secret = optarg; + break; + } + + case 'H': //-H S3 Host + { + fS3Host = optarg; + break; + } + + case 't': //-t S3 bucket + { + fS3Bucket = optarg; + break; + } + + case 'g': //-g S3 Region + { + fS3Region = optarg; + break; + } + + case 'U': //-U username of the files owner + { + fUsername = optarg; + break; + } + + case 'L': // -L set the output location of .bad/.err files + { + fErrorDir = optarg; + break; + } + + default: + { + std::string aErr = std::string("Unknown command line option ") + std::to_string(aCh); + // cout << "Unknown command line option " << aCh << endl; + throw(runtime_error(aErr)); + } + } + } + + if (fHelp) + usage(); // BUG 4210 + + if (fArgMode != -1) + fMode = fArgMode; // BUG 4210 + + std::string bulkRootPath = getBulkRootDir(); + + checkForBulkLogDir(bulkRootPath); + + if (2 == fArgMode && fPmFilePath.empty()) + throw runtime_error("-f option is mandatory with mode 2."); + + if (aJobType) + { + if (0 == fArgMode) + throw runtime_error("Incompatible mode and option types"); + + if (optind < argc) + { + fSchema = argv[optind]; // 1st pos parm + optind++; + + if (optind < argc) + { + fTable = argv[optind]; // 2nd pos parm optind++; + } + else + { + // if schema is there, table name should be there + throw runtime_error("No table name specified with schema."); + } - if (optind < argc) // see if table name is given + if (optind < argc) // see if input file name is given + { + // 3rd pos parm + fLocFile = argv[optind]; + + if ((fLocFile.at(0) != '/') && (fLocFile != "STDIN")) { - fTable = argv[optind]; // 2nd pos parm - optind++; + std::string aTmp = fLocFile; - if (optind < argc) // see if input file name is given - { - // 3rd pos parm - fLocFile = argv[optind]; - - //BUG 4379 if -f option given we need to use that path, - //over riding bug 4231. look at the code below - //BUG 4231 - This bug over writes 4199 and commenting out changes - //BUG 4199 - //Path not provided, not fully qualified, Look in import dir - //if((fLocFile.at(0)!= '/')&&(fLocFile != "STDIN")) - //{ - // std::string aTmp = fLocFile; - // fLocFile = bulkRootPath + "/data/import/"+ aTmp; - //} - //BUG 4379 if -f option given we need to use that path - if ((fLocFile.at(0) != '/') && (fLocFile != "STDIN")) - { - std::string aTmp = fLocFile; - - //if -f given? use that otherwise just go ahead with CWD - if ((!fPmFilePath.empty()) && (fMode == 1)) - fLocFile = fPmFilePath + "/" + aTmp; - - // TODO - if -f option is given and a list of files are - // are provided, we need to be able to import all that. - } - - - } - else - { - if (fPmFile.length() > 0) - { - // BUG 4210 - //if (fPmFilePath.length() > 0) - //{ - // fLocFile = fPmFilePath +"/"+ fPmFile; - //} - //else - if (fPmFilePath.empty()) - { - //NOTE - un-commenting with an if statement for Mode 2 - //BUG 4231 makes it comment out the below changes, - //This will not change even though directly, to be - //on safer side, we should take out this too. - //check path fully qualified? then set as data import - if (2 == fArgMode) - { - //BUG 4342 - if ((fPmFile.at(0) != '/') && (fS3Key.empty())) - { - std::string aTmp = fPmFile; - fPmFile = PrepMode2ListOfFiles(aTmp); - } - else - { - if (fPmFile.find_first_of(' ') != string::npos) - { - std::string aPmFiles = replaceCharInStr(fPmFile, ' ', ','); - fPmFile = aPmFiles; - } - } - } - - fLocFile = fPmFile; - } - } - else - { - fLocFile = "STDIN"; - } - - //cout << "LocFile set as stdin" << endl; - } - } - else - { - // If Mode is not 0 and table name is a required argument - if (fArgMode != 0) - throw(runtime_error("No table name specified with schema.")); + // BUG 4379 -f given? use that + if ((!fPmFilePath.empty()) && (fMode == 1)) + fLocFile = fPmFilePath + "/" + aTmp; + else if (fPmFilePath.empty()) + fLocFile = bulkRootPath + "/data/import/" + aTmp; } + } + else + { + if (!fPmFile.empty()) + fLocFile = fPmFile; + // BUG 4186 + // else // else take it from the jobxml file + // fLocFile = "STDIN"; + // Historically cpimport works with jobfile as + // -l && -f or + // -fSTDIN as the stdin, it will override colxml loadfile entry + // if -fSTDIN is not provided get i/p file from jobfile + else if ((!fPmFilePath.empty()) && (fPmFilePath == "STDIN")) + fLocFile = "STDIN"; + // else take it from the jobxml file + } + + if ((fSchema.length() > 0) && (fTable.length() > 0) && (fLocFile.length() > 0)) + fJobLogOnly = true; } else { - // for testing we are allowing data from stdin even with Mode 0 - // that is without LocFileName - if (0 == fArgMode) + if (!fPmFile.empty()) + { + fLocFile = fPmFile; + + if (!fPmFilePath.empty()) { - fLocFile = "STDIN"; //cout << "LocFile set as stdin" << endl; + if (fPmFilePath == "STDIN") + { + throw runtime_error("Conflicting options -l and -fSTDIN"); + } + else + { + std::string aTmp = fLocFile; + + if ((!fPmFilePath.empty()) && (fMode == 1)) // BUG 4379 -f given? use that + fLocFile = fPmFilePath + "/" + aTmp; + else if (!fPmFilePath.empty()) + fLocFile = bulkRootPath + "/data/import/" + aTmp; + } + } + + if ((fLocFile.at(0) != '/') && (fLocFile != "STDIN") && (fPmFilePath.empty())) + { + std::string aTmp = fLocFile; + fLocFile = bulkRootPath + "/data/import/" + aTmp; + } + } + // BUG 4186 + // else + // fLocFile = "STDIN"; + // Historically cpimport works with jobfile as + // -l && -f or + // -fSTDIN as the stdin, it will override colxml loadfile entry + // if -fSTDIN is not provided get i/p file from jobfile + else if ((!fPmFilePath.empty()) && (fPmFilePath == "STDIN")) + fLocFile = "STDIN"; + + // else take it from the jobxml file + } + } + // Get positional arguments, User can provide: + // 1. no positional parameters - Mode 0 & stdin + // 2. Two positional parameters (schema and table names) - Mode 1/2, stdin + // 3. Three positional parameters (schema, table, and import file name) + else if (optind < argc) // see if db schema name is given + { + if (fArgMode == 0) + { + // added the code as per BUG 4245 + if (!fPmFilePath.empty()) + { + fLocFile = fPmFilePath; + + if (fLocFile != "STDIN") + throw(runtime_error("ERROR: In Mode 0, -f option can only have value STDIN")); + } + else + { + fLocFile = argv[optind]; + optind++; + } + + if (optind < argc) // dest filename provided + { + fPmFile = argv[optind]; + + if ((fPmFile.at(0) != '/') && (fS3Key.empty())) + { + std::string aTmp = fPmFile; + fPmFile = bulkRootPath + "/data/import/" + aTmp; + } + } + else // no dest filename + { + if (fLocFile == "STDIN") + throw(runtime_error("ERROR: Destination file name required!!")); + + if (fLocFile.at(0) == '/') // local FQ-filename,parse out filename + fPmFile = getFileNameFromPath(fLocFile); + else + fPmFile = fLocFile; + + if ((fPmFile.at(0) != '/') && (fS3Key.empty())) // should be true all the time + { + std::string aTmp = fPmFile; + fPmFile = bulkRootPath + "/data/import/" + aTmp; + } + } + + /* commented out for BUG 4245 + if(fPmFilePath.empty()) + fLocFile = argv[optind]; + else + fLocFile = fPmFilePath +"/"+ argv[optind]; + + if (fPmFile.empty()) //BUG 4200 + { + //if(fLocFile.at(0)== '/') + // fPmFile = fLocFile; + //else + if(fLocFile.at(0)!='/') + fPmFile = bulkRootPath + "/data/import/"+ fLocFile; + } + else + { + if(fPmFile.at(0)!='/') + { + std::string aTmp = fPmFile; + fPmFile = bulkRootPath + "/data/import/"+aTmp; + } + } + */ + } + else + fSchema = argv[optind]; // 1st pos parm + + optind++; + + if (optind < argc) // see if table name is given + { + fTable = argv[optind]; // 2nd pos parm + optind++; + + if (optind < argc) // see if input file name is given + { + // 3rd pos parm + fLocFile = argv[optind]; + + // BUG 4379 if -f option given we need to use that path, + // over riding bug 4231. look at the code below + // BUG 4231 - This bug over writes 4199 and commenting out changes + // BUG 4199 + // Path not provided, not fully qualified, Look in import dir + // if((fLocFile.at(0)!= '/')&&(fLocFile != "STDIN")) + //{ + // std::string aTmp = fLocFile; + // fLocFile = bulkRootPath + "/data/import/"+ aTmp; + //} + // BUG 4379 if -f option given we need to use that path + if ((fLocFile.at(0) != '/') && (fLocFile != "STDIN")) + { + std::string aTmp = fLocFile; + + // if -f given? use that otherwise just go ahead with CWD + if ((!fPmFilePath.empty()) && (fMode == 1)) + fLocFile = fPmFilePath + "/" + aTmp; + + // TODO - if -f option is given and a list of files are + // are provided, we need to be able to import all that. + } + } + else + { + if (fPmFile.length() > 0) + { + // BUG 4210 + // if (fPmFilePath.length() > 0) + //{ + // fLocFile = fPmFilePath +"/"+ fPmFile; + //} + // else + if (fPmFilePath.empty()) + { + // NOTE - un-commenting with an if statement for Mode 2 + // BUG 4231 makes it comment out the below changes, + // This will not change even though directly, to be + // on safer side, we should take out this too. + // check path fully qualified? then set as data import + if (2 == fArgMode) + { + // BUG 4342 + if ((fPmFile.at(0) != '/') && (fS3Key.empty())) + { + std::string aTmp = fPmFile; + fPmFile = PrepMode2ListOfFiles(aTmp); + } + else + { + if (fPmFile.find_first_of(' ') != string::npos) + { + std::string aPmFiles = replaceCharInStr(fPmFile, ' ', ','); + fPmFile = aPmFiles; + } + } + } + + fLocFile = fPmFile; + } } else { - // If Mode 0, LocFileName is reqd and otherwies Schema is required - throw (runtime_error("No schema or local filename specified.")); + fLocFile = "STDIN"; } - } - /* check for all-or-nothing cmdline args to enable S3 import */ - int s3Tmp = (fS3Key.empty() ? 0 : 1) + (fS3Bucket.empty() ? 0 : 1) + - (fS3Secret.empty() ? 0 : 1) + (fS3Region.empty() ? 0 : 1); - if (s3Tmp != 0 && s3Tmp != 4) - throw runtime_error("The access key, secret, bucket, and region are all required to import from S3"); + // cout << "LocFile set as stdin" << endl; + } + } + else + { + // If Mode is not 0 and table name is a required argument + if (fArgMode != 0) + throw(runtime_error("No table name specified with schema.")); + } + } + else + { + // for testing we are allowing data from stdin even with Mode 0 + // that is without LocFileName + if (0 == fArgMode) + { + fLocFile = "STDIN"; // cout << "LocFile set as stdin" << endl; + } + else + { + // If Mode 0, LocFileName is reqd and otherwies Schema is required + throw(runtime_error("No schema or local filename specified.")); + } + } + + /* check for all-or-nothing cmdline args to enable S3 import */ + int s3Tmp = (fS3Key.empty() ? 0 : 1) + (fS3Bucket.empty() ? 0 : 1) + (fS3Secret.empty() ? 0 : 1) + + (fS3Region.empty() ? 0 : 1); + if (s3Tmp != 0 && s3Tmp != 4) + throw runtime_error("The access key, secret, bucket, and region are all required to import from S3"); } std::string WECmdArgs::getJobFileName() { - std::ostringstream aSS; - string aJobIdFileName; + std::ostringstream aSS; + string aJobIdFileName; - if (fJobId.length() > 0) + if (fJobId.length() > 0) + { + if (fJobPath.length() > 0) + aSS << fJobPath; + else { - if (fJobPath.length() > 0) - aSS << fJobPath; - else - { - fJobPath = config::Config::makeConfig()->getConfig("WriteEngine", - "BulkRoot") + "/Job"; - aSS << fJobPath; - } - - aSS << "/Job_" << fJobId << ".xml"; - aJobIdFileName = aSS.str(); + fJobPath = config::Config::makeConfig()->getConfig("WriteEngine", "BulkRoot") + "/Job"; + aSS << fJobPath; } - return aJobIdFileName; + aSS << "/Job_" << fJobId << ".xml"; + aJobIdFileName = aSS.str(); + } + + return aJobIdFileName; } bool WECmdArgs::getPmStatus(int Id) { - // if no PMID's provided on cmdline, return true; - if (0 == fPmVec.size()) return true; + // if no PMID's provided on cmdline, return true; + if (0 == fPmVec.size()) + return true; - VecInts::iterator aIt = fPmVec.begin(); + VecInts::iterator aIt = fPmVec.begin(); - while (aIt != fPmVec.end()) - { - if (*aIt == static_cast(Id)) - return true; + while (aIt != fPmVec.end()) + { + if (*aIt == static_cast(Id)) + return true; - ++aIt; - } + ++aIt; + } - return false; + return false; } - //------------------------------------------------------------------------------ // It is a recursive call. std::string WECmdArgs::getBrmRptFileName() { - if (!fBrmRptFile.empty()) - return fBrmRptFile; + if (!fBrmRptFile.empty()) + return fBrmRptFile; - string brmRptFileName = getTmpFileDir(); + string brmRptFileName = getTmpFileDir(); - if (!brmRptFileName.empty()) - { - fTmpFileDir = brmRptFileName; - char aBuff[64]; - time_t aTime; - struct tm pTm; - time(&aTime); - localtime_r(&aTime, &pTm); + if (!brmRptFileName.empty()) + { + fTmpFileDir = brmRptFileName; + char aBuff[64]; + time_t aTime; + struct tm pTm; + time(&aTime); + localtime_r(&aTime, &pTm); - // BUG 4424 - // M D H M S - snprintf(aBuff, sizeof(aBuff), "/BrmRpt%02d%02d%02d%02d%02d%d.rpt", - pTm.tm_mon, pTm.tm_mday, pTm.tm_hour, - pTm.tm_min, pTm.tm_sec, getpid()); - brmRptFileName += aBuff; - } - else - { - //cout << "ERROR: Could not find TempFileDir in Columnstore.xml" << endl; - throw (runtime_error("Could not find TempFileDir in Columnstore.xml")); - } + // BUG 4424 + // M D H M S + snprintf(aBuff, sizeof(aBuff), "/BrmRpt%02d%02d%02d%02d%02d%d.rpt", pTm.tm_mon, pTm.tm_mday, pTm.tm_hour, + pTm.tm_min, pTm.tm_sec, getpid()); + brmRptFileName += aBuff; + } + else + { + // cout << "ERROR: Could not find TempFileDir in Columnstore.xml" << endl; + throw(runtime_error("Could not find TempFileDir in Columnstore.xml")); + } - setBrmRptFileName(brmRptFileName); - - return brmRptFileName; + setBrmRptFileName(brmRptFileName); + return brmRptFileName; } //------------------------------------------------------------------------------ void WECmdArgs::addJobFilesToVector(std::string& JobName) { - //if((!fSchema.empty())&&(!fTable.empty())&&(!fLocFile.empty())) return; + // if((!fSchema.empty())&&(!fTable.empty())&&(!fLocFile.empty())) return; - WEXmlgetter aXmlGetter(JobName); - vector aSections; - aSections.push_back("BulkJob"); - aSections.push_back("Schema"); - aSections.push_back("Table"); + WEXmlgetter aXmlGetter(JobName); + vector aSections; + aSections.push_back("BulkJob"); + aSections.push_back("Schema"); + aSections.push_back("Table"); - //BUG 4163 - typedef std::vector TableVec; - TableVec aTableVec; - aXmlGetter.getConfig(aSections[1], aSections[2], aTableVec); - setMultiTableCount(aTableVec.size()); - - if (getMultiTableCount() > 1) - { - splitConfigFilePerTable(JobName, aTableVec.size()); - } - else - { - fVecJobFiles.push_back(JobName); - } + // BUG 4163 + typedef std::vector TableVec; + TableVec aTableVec; + aXmlGetter.getConfig(aSections[1], aSections[2], aTableVec); + setMultiTableCount(aTableVec.size()); + if (getMultiTableCount() > 1) + { + splitConfigFilePerTable(JobName, aTableVec.size()); + } + else + { + fVecJobFiles.push_back(JobName); + } } //------------------------------------------------------------------------------ @@ -1378,444 +1366,438 @@ void WECmdArgs::addJobFilesToVector(std::string& JobName) //------------------------------------------------------------------------------ void WECmdArgs::setSchemaAndTableFromJobFile(std::string& JobName) { - if (((fVecJobFiles.size() == 1) && (!fSchema.empty()) && - (!fTable.empty()) && (!fLocFile.empty())) && - (fImportDataMode == IMPORT_DATA_TEXT)) return; + if (((fVecJobFiles.size() == 1) && (!fSchema.empty()) && (!fTable.empty()) && (!fLocFile.empty())) && + (fImportDataMode == IMPORT_DATA_TEXT)) + return; - WEXmlgetter aXmlGetter(JobName); - vector aSections; - aSections.push_back("BulkJob"); - aSections.push_back("Schema"); - aSections.push_back("Table"); + WEXmlgetter aXmlGetter(JobName); + vector aSections; + aSections.push_back("BulkJob"); + aSections.push_back("Schema"); + aSections.push_back("Table"); - // Reset the fSchema, fTable, and FLocFile - if ((fVecJobFiles.size() > 1) || - (fSchema.empty()) || (fTable.empty()) || (fLocFile.empty())) + // Reset the fSchema, fTable, and FLocFile + if ((fVecJobFiles.size() > 1) || (fSchema.empty()) || (fTable.empty()) || (fLocFile.empty())) + { + std::string aSchemaTable; + std::string aInputFile; + + aSchemaTable = aXmlGetter.getAttribute(aSections, "tblName"); + + if (getDebugLvl() > 1) + cout << "schema.table = " << aSchemaTable << endl; + + aInputFile = aXmlGetter.getAttribute(aSections, "loadName"); + + if (getDebugLvl() > 1) + cout << "xml::InputFile = " << aInputFile << endl; + + if (aSchemaTable.length() > 0) { - std::string aSchemaTable; - std::string aInputFile; + char aSchema[64]; + char aTable[64]; + int aRet = aSchemaTable.find('.'); - aSchemaTable = aXmlGetter.getAttribute(aSections, "tblName"); + if (aRet > 0) + { + int aLen = aSchemaTable.copy(aSchema, aRet); - if (getDebugLvl() > 1) cout << "schema.table = " << aSchemaTable << endl; + if (getDebugLvl() > 1) + cout << "Schema: " << aSchema << endl; - aInputFile = aXmlGetter.getAttribute(aSections, "loadName"); + aSchema[aLen] = 0; - if (getDebugLvl() > 1) cout << "xml::InputFile = " << aInputFile << endl; + if (fSchema.empty()) + fSchema = aSchema; - if (aSchemaTable.length() > 0) - { - char aSchema[64]; - char aTable[64]; - int aRet = aSchemaTable.find('.'); + aLen = aSchemaTable.copy(aTable, aSchemaTable.length(), aRet + 1); + aTable[aLen] = 0; - if (aRet > 0) - { - int aLen = aSchemaTable.copy(aSchema, aRet); + if (getDebugLvl() > 1) + cout << "Table: " << aTable << endl; - if (getDebugLvl() > 1) cout << "Schema: " << aSchema << endl; - - aSchema[aLen] = 0; - - if (fSchema.empty()) fSchema = aSchema; - - aLen = aSchemaTable.copy(aTable, aSchemaTable.length(), aRet + 1 ); - aTable[aLen] = 0; - - if (getDebugLvl() > 1) cout << "Table: " << aTable << endl; - - fTable = aTable; - } - else - throw runtime_error( - "JobFile ERROR: Can't get Schema and Table Name"); - } - else - { - throw runtime_error( - "JobFile ERROR: Can't get Schema and Table Name"); - } - - if ((fLocFile.empty()) && (!aInputFile.empty())) - { - string bulkRootPath = config::Config::makeConfig()->getConfig( - "WriteEngine", "BulkRoot"); - - if (aInputFile.at(0) == '/') - fLocFile = aInputFile; - else if ((!fPmFilePath.empty()) && (fMode == 1)) - fLocFile = fPmFilePath + "/" + aInputFile; - else if ((!bulkRootPath.empty()) && (fPmFilePath.empty())) - fLocFile = bulkRootPath + "/data/import/" + aInputFile; - else - fLocFile = aInputFile; - - if (fArgMode == 2) fPmFile = fLocFile; - } - - if (getDebugLvl() > 1) cout << "schema = " << fSchema << endl; - - if (getDebugLvl() > 1) cout << "TableName = " << fTable << endl; - - if (getDebugLvl() > 1) cout << "Input File = " << fLocFile << endl; + fTable = aTable; + } + else + throw runtime_error("JobFile ERROR: Can't get Schema and Table Name"); + } + else + { + throw runtime_error("JobFile ERROR: Can't get Schema and Table Name"); } - // Reset the list of columns we will be importing from the input data - fColFldsFromJobFile.clear(); - - if (fImportDataMode != IMPORT_DATA_TEXT) + if ((fLocFile.empty()) && (!aInputFile.empty())) { - aSections.push_back("Column"); - aXmlGetter.getAttributeListForAllChildren( - aSections, "colName", fColFldsFromJobFile); + string bulkRootPath = config::Config::makeConfig()->getConfig("WriteEngine", "BulkRoot"); + + if (aInputFile.at(0) == '/') + fLocFile = aInputFile; + else if ((!fPmFilePath.empty()) && (fMode == 1)) + fLocFile = fPmFilePath + "/" + aInputFile; + else if ((!bulkRootPath.empty()) && (fPmFilePath.empty())) + fLocFile = bulkRootPath + "/data/import/" + aInputFile; + else + fLocFile = aInputFile; + + if (fArgMode == 2) + fPmFile = fLocFile; } + + if (getDebugLvl() > 1) + cout << "schema = " << fSchema << endl; + + if (getDebugLvl() > 1) + cout << "TableName = " << fTable << endl; + + if (getDebugLvl() > 1) + cout << "Input File = " << fLocFile << endl; + } + + // Reset the list of columns we will be importing from the input data + fColFldsFromJobFile.clear(); + + if (fImportDataMode != IMPORT_DATA_TEXT) + { + aSections.push_back("Column"); + aXmlGetter.getAttributeListForAllChildren(aSections, "colName", fColFldsFromJobFile); + } } //------------------------------------------------------------------------------ void WECmdArgs::checkJobIdCase() { - if ((fJobId.empty()) || (fJobLogOnly) || (fMode == 3) || (fMode == 0)) return; + if ((fJobId.empty()) || (fJobLogOnly) || (fMode == 3) || (fMode == 0)) + return; - if (fJobPath.empty()) - { - string bulkRootPath = config::Config::makeConfig()->getConfig( - "WriteEngine", "BulkRoot"); - //cout << "checkJobIdCase::BulkRoot: " << bulkRootPath << endl; + if (fJobPath.empty()) + { + string bulkRootPath = config::Config::makeConfig()->getConfig("WriteEngine", "BulkRoot"); + // cout << "checkJobIdCase::BulkRoot: " << bulkRootPath << endl; - if (!bulkRootPath.empty()) - fJobPath = bulkRootPath + "/job"; - else - throw runtime_error("Config Error: BulkRoot not found in Columnstore.xml"); - } + if (!bulkRootPath.empty()) + fJobPath = bulkRootPath + "/job"; + else + throw runtime_error("Config Error: BulkRoot not found in Columnstore.xml"); + } - char aBuff[256]; + char aBuff[256]; - if (!fJobPath.empty()) - snprintf(aBuff, sizeof(aBuff), "%s/Job_%s.xml", fJobPath.c_str(), - fJobId.c_str()); - else // for time being - snprintf(aBuff, sizeof(aBuff), "/var/log/mariadb/columnstore/data/bulk/job/Job_%s.xml", - fJobId.c_str()); + if (!fJobPath.empty()) + snprintf(aBuff, sizeof(aBuff), "%s/Job_%s.xml", fJobPath.c_str(), fJobId.c_str()); + else // for time being + snprintf(aBuff, sizeof(aBuff), "/var/log/mariadb/columnstore/data/bulk/job/Job_%s.xml", fJobId.c_str()); - std::string aJobFileName(aBuff); + std::string aJobFileName(aBuff); - //cout << "checkJobIdCase::aJobFileName: " << aJobFileName << endl; + // cout << "checkJobIdCase::aJobFileName: " << aJobFileName << endl; + // BUG 4171 + addJobFilesToVector(aJobFileName); - //BUG 4171 - addJobFilesToVector(aJobFileName); - - aJobFileName = fVecJobFiles[0]; - setSchemaAndTableFromJobFile(aJobFileName); - setEnclByAndEscCharFromJobFile(aJobFileName); - + aJobFileName = fVecJobFiles[0]; + setSchemaAndTableFromJobFile(aJobFileName); + setEnclByAndEscCharFromJobFile(aJobFileName); } //------------------------------------------------------------------------------ std::string WECmdArgs::getTmpFileDir() { - if (!fTmpFileDir.empty()) return fTmpFileDir; + if (!fTmpFileDir.empty()) + return fTmpFileDir; - fTmpFileDir = startup::StartUp::tmpDir(); + fTmpFileDir = startup::StartUp::tmpDir(); - if (fTmpFileDir.empty()) - throw( runtime_error("Config ERROR: TmpFileDir not found!!")); - else - return fTmpFileDir; + if (fTmpFileDir.empty()) + throw(runtime_error("Config ERROR: TmpFileDir not found!!")); + else + return fTmpFileDir; } //------------------------------------------------------------------------------ std::string WECmdArgs::getBulkRootDir() { - if (!fBulkRoot.empty()) return fBulkRoot; + if (!fBulkRoot.empty()) + return fBulkRoot; - fBulkRoot = config::Config::makeConfig()->getConfig("WriteEngine", - "BulkRoot"); + fBulkRoot = config::Config::makeConfig()->getConfig("WriteEngine", "BulkRoot"); - if (fBulkRoot.empty()) - throw( runtime_error("Config ERROR: not found!!")); - else - return fBulkRoot; + if (fBulkRoot.empty()) + throw(runtime_error("Config ERROR: not found!!")); + else + return fBulkRoot; } //------------------------------------------------------------------------------ unsigned int WECmdArgs::getBatchQuantity() { - return (fBatchQty >= 10000) ? fBatchQty : 10000; //default Batch Qty is 10000 + return (fBatchQty >= 10000) ? fBatchQty : 10000; // default Batch Qty is 10000 } //------------------------------------------------------------------------------ void WECmdArgs::setEnclByAndEscCharFromJobFile(std::string& JobName) { - if (fEnclosedChar == 0) // check anything in Jobxml file + if (fEnclosedChar == 0) // check anything in Jobxml file + { + WEXmlgetter aXmlGetter(JobName); + vector aSections; + aSections.push_back("BulkJob"); + aSections.push_back("EnclosedByChar"); + + try { - WEXmlgetter aXmlGetter(JobName); - vector aSections; - aSections.push_back("BulkJob"); - aSections.push_back("EnclosedByChar"); + // std::string aTable = aXmlGetter.getConfig(aSection, aElement); + std::string aEnclosedBy = aXmlGetter.getValue(aSections); - try - { - //std::string aTable = aXmlGetter.getConfig(aSection, aElement); - std::string aEnclosedBy = aXmlGetter.getValue(aSections); + if (getDebugLvl() > 1) + cout << "aEncloseBy = " << aEnclosedBy << endl; - if (getDebugLvl() > 1)cout << "aEncloseBy = " << aEnclosedBy << endl; - - if (!aEnclosedBy.empty()) - { - fEnclosedChar = aEnclosedBy.at(0); - } - } - catch (std::runtime_error&) - { - // do not do anything - } + if (!aEnclosedBy.empty()) + { + fEnclosedChar = aEnclosedBy.at(0); + } } - - if (fEscChar == 0) // check anything in Jobxml file + catch (std::runtime_error&) { - WEXmlgetter aXmlGetter(JobName); - vector aSections; - aSections.push_back("BulkJob"); - aSections.push_back("EscapeChar"); - - try - { - //std::string aTable = aXmlGetter.getConfig(aSection, aElement); - std::string aEscChar = aXmlGetter.getValue(aSections); - - if (getDebugLvl() > 1) cout << "aEscapeChar = " << aEscChar << endl; - - if (!aEscChar.empty()) - { - fEscChar = aEscChar.at(0); - } - } - catch (std::runtime_error&) - { - // do not do anything - } + // do not do anything } + } + if (fEscChar == 0) // check anything in Jobxml file + { + WEXmlgetter aXmlGetter(JobName); + vector aSections; + aSections.push_back("BulkJob"); + aSections.push_back("EscapeChar"); + + try + { + // std::string aTable = aXmlGetter.getConfig(aSection, aElement); + std::string aEscChar = aXmlGetter.getValue(aSections); + + if (getDebugLvl() > 1) + cout << "aEscapeChar = " << aEscChar << endl; + + if (!aEscChar.empty()) + { + fEscChar = aEscChar.at(0); + } + } + catch (std::runtime_error&) + { + // do not do anything + } + } } //------------------------------------------------------------------------------ std::string WECmdArgs::getFileNameFromPath(const std::string& Path) const { - char aBuff[64]; - int iDx = Path.find_last_of('/'); - iDx++; // compensate for the forward slash - int aCx = Path.size() - iDx; - Path.copy(aBuff, aCx, iDx); - aBuff[aCx] = 0; - return aBuff; + char aBuff[64]; + int iDx = Path.find_last_of('/'); + iDx++; // compensate for the forward slash + int aCx = Path.size() - iDx; + Path.copy(aBuff, aCx, iDx); + aBuff[aCx] = 0; + return aBuff; } //------------------------------------------------------------------------------ std::string WECmdArgs::getModuleID() { - oam::Oam oam; - oam::oamModuleInfo_t sModInfo; - std::string sModuleID; - char szModuleIDandPID[64]; - int nModuleNumber; + oam::Oam oam; + oam::oamModuleInfo_t sModInfo; + std::string sModuleID; + char szModuleIDandPID[64]; + int nModuleNumber; - try - { - sModInfo = oam.getModuleInfo(); - sModuleID = boost::get < 1 > (sModInfo); - nModuleNumber = boost::get < 2 > (sModInfo); - snprintf(szModuleIDandPID, sizeof(szModuleIDandPID), "%s%d-%d", - sModuleID.c_str(), nModuleNumber, getpid()); - sModuleID = szModuleIDandPID; - } - catch (exception&) - { - sModuleID = "unknown"; - } + try + { + sModInfo = oam.getModuleInfo(); + sModuleID = boost::get<1>(sModInfo); + nModuleNumber = boost::get<2>(sModInfo); + snprintf(szModuleIDandPID, sizeof(szModuleIDandPID), "%s%d-%d", sModuleID.c_str(), nModuleNumber, + getpid()); + sModuleID = szModuleIDandPID; + } + catch (exception&) + { + sModuleID = "unknown"; + } - return sModuleID; + return sModuleID; } //------------------------------------------------------------------------------ - void WECmdArgs::splitConfigFilePerTable(std::string& ConfigName, int tblCount) { - std::string aOpenTag = "

aVecFiles; + std::vector aVecFiles; - //std::vector aVecConfigs; - for (int aIdx = 1; aIdx <= tblCount; aIdx++) + // std::vector aVecConfigs; + for (int aIdx = 1; aIdx <= tblCount; aIdx++) + { + char aConfName[128]; + snprintf(aConfName, sizeof(aConfName), "%s_%d.xml", ConfigName.c_str(), aIdx); + // aVecConfigs.push_back(aConfName); + fVecJobFiles.push_back(aConfName); + std::ofstream* pCopy = new std::ofstream; + // pCopy->open(aConfName, std::ios_base::app); + pCopy->open(aConfName); + aVecFiles.push_back(pCopy); + } + + std::ifstream aMaster; + aMaster.open(ConfigName.c_str()); + + if (aMaster.is_open()) + { + char aBuff[256]; + int aTblNo = 0; + size_t aStrPos = std::string::npos; + bool aOpenFound = false; + bool aCloseFound = false; + + while (!aMaster.eof()) { - char aConfName[128]; - snprintf(aConfName, sizeof(aConfName), "%s_%d.xml", ConfigName.c_str(), aIdx); - //aVecConfigs.push_back(aConfName); - fVecJobFiles.push_back(aConfName); - std::ofstream* pCopy = new std::ofstream; - //pCopy->open(aConfName, std::ios_base::app); - pCopy->open(aConfName); - aVecFiles.push_back(pCopy); - } + aMaster.getline(aBuff, sizeof(aBuff) - 1); + unsigned int aLen = aMaster.gcount(); + if ((aLen < (sizeof(aBuff) - 2)) && (aLen > 0)) + { + aBuff[aLen - 1] = '\n'; + aBuff[aLen] = 0; + string aData = aBuff; + // cout << "Data Read " << aBuff; - std::ifstream aMaster; - aMaster.open(ConfigName.c_str()); - - if (aMaster.is_open()) - { - char aBuff[256]; - int aTblNo = 0; - size_t aStrPos = std::string::npos; - bool aOpenFound = false; - bool aCloseFound = false; - - while (!aMaster.eof()) + if (!aOpenFound) { - aMaster.getline(aBuff, sizeof(aBuff) - 1); - unsigned int aLen = aMaster.gcount(); + aStrPos = aData.find(aOpenTag); - if ((aLen < (sizeof(aBuff) - 2)) && (aLen > 0)) + if (aStrPos != std::string::npos) + { + aOpenFound = true; + aTblNo++; + write2ConfigFiles(aVecFiles, aBuff, aTblNo); + } + else + { + if ((!aOpenFound) && (aCloseFound)) { - aBuff[aLen - 1] = '\n'; - aBuff[aLen] = 0; - string aData = aBuff; - //cout << "Data Read " << aBuff; + aStrPos = aData.find(aCloseSchemaTag); - if (!aOpenFound) - { - aStrPos = aData.find(aOpenTag); - - if (aStrPos != std::string::npos) - { - aOpenFound = true; - aTblNo++; - write2ConfigFiles(aVecFiles, aBuff, aTblNo); - } - else - { - if ((!aOpenFound) && (aCloseFound)) - { - aStrPos = aData.find(aCloseSchemaTag); - - if (aStrPos != std::string::npos) - { - aOpenFound = false; - aCloseFound = false; - aTblNo = 0; - } - } - - write2ConfigFiles(aVecFiles, aBuff, aTblNo); - } - } - else - { - aStrPos = aData.find(aCloseTag); - - if (aStrPos != std::string::npos) - { - aOpenFound = false; - aCloseFound = true; - write2ConfigFiles(aVecFiles, aBuff, aTblNo); - } - else - { - write2ConfigFiles(aVecFiles, aBuff, aTblNo); - } - } + if (aStrPos != std::string::npos) + { + aOpenFound = false; + aCloseFound = false; + aTblNo = 0; + } } - }//while Master.eof - } - else - { - throw runtime_error("Could not open Job Config file"); - } + write2ConfigFiles(aVecFiles, aBuff, aTblNo); + } + } + else + { + aStrPos = aData.find(aCloseTag); - for (unsigned int Idx = 0; Idx < aVecFiles.size(); Idx++) - { - aVecFiles[Idx]->close(); - delete aVecFiles[Idx]; - } + if (aStrPos != std::string::npos) + { + aOpenFound = false; + aCloseFound = true; + write2ConfigFiles(aVecFiles, aBuff, aTblNo); + } + else + { + write2ConfigFiles(aVecFiles, aBuff, aTblNo); + } + } + } + } // while Master.eof + } + else + { + throw runtime_error("Could not open Job Config file"); + } - aVecFiles.clear(); + for (unsigned int Idx = 0; Idx < aVecFiles.size(); Idx++) + { + aVecFiles[Idx]->close(); + delete aVecFiles[Idx]; + } + aVecFiles.clear(); } //------------------------------------------------------------------------------ -void WECmdArgs::write2ConfigFiles(std::vector& Files, - char* pBuff, int FileIdx) +void WECmdArgs::write2ConfigFiles(std::vector& Files, char* pBuff, int FileIdx) { + if (FileIdx == 0) + { + std::vector::iterator aIt = Files.begin(); - if (FileIdx == 0) + while (aIt != Files.end()) { - std::vector::iterator aIt = Files.begin(); - - while (aIt != Files.end()) - { - std::ofstream* pCopy = (*aIt); - pCopy->write(pBuff, strlen(pBuff)); - ++aIt; - } - } - else - { - Files[FileIdx - 1]->write(pBuff, strlen(pBuff)); + std::ofstream* pCopy = (*aIt); + pCopy->write(pBuff, strlen(pBuff)); + ++aIt; } + } + else + { + Files[FileIdx - 1]->write(pBuff, strlen(pBuff)); + } } //------------------------------------------------------------------------------ void WECmdArgs::updateWithJobFile(int Idx) { - setLocFile(""); // resetting the from the previous import - std::string aJobFileName = fVecJobFiles[Idx]; - setSchemaAndTableFromJobFile(aJobFileName); - setEnclByAndEscCharFromJobFile(aJobFileName); - setJobFileName(aJobFileName); + setLocFile(""); // resetting the from the previous import + std::string aJobFileName = fVecJobFiles[Idx]; + setSchemaAndTableFromJobFile(aJobFileName); + setEnclByAndEscCharFromJobFile(aJobFileName); + setJobFileName(aJobFileName); - std::ostringstream aSS; - aSS << fOrigJobId << ".xml_" << (Idx + 1); - fJobId = aSS.str(); + std::ostringstream aSS; + aSS << fOrigJobId << ".xml_" << (Idx + 1); + fJobId = aSS.str(); } - //------------------------------------------------------------------------------ std::string WECmdArgs::replaceCharInStr(const std::string& Str, char C, char R) { - std::stringstream aSs; + std::stringstream aSs; - size_t start = 0, end = 0; - end = Str.find_first_of(C); + size_t start = 0, end = 0; + end = Str.find_first_of(C); - do + do + { + if (end != string::npos) { - if (end != string::npos) - { - aSs << Str.substr(start, end - start) << R; - start = end + 1; - } - else - { - aSs << Str.substr(start, end - start); - break; - } - - end = Str.find_first_of(C, start); + aSs << Str.substr(start, end - start) << R; + start = end + 1; + } + else + { + aSs << Str.substr(start, end - start); + break; } - while (start != end); - return aSs.str(); + end = Str.find_first_of(C, start); + } while (start != end); + + return aSs.str(); } //------------------------------------------------------------------------------ @@ -1823,84 +1805,84 @@ std::string WECmdArgs::replaceCharInStr(const std::string& Str, char C, char R) std::string WECmdArgs::PrepMode2ListOfFiles(std::string& FileName) { - VecArgs aInfileList; - std::string bulkRootPath = getBulkRootDir(); - //cout << "Inside PrepMode2ListOfFiles("<< FileName << ")" << endl; - std::string aFileName = FileName; + VecArgs aInfileList; + std::string bulkRootPath = getBulkRootDir(); + // cout << "Inside PrepMode2ListOfFiles("<< FileName << ")" << endl; + std::string aFileName = FileName; - istringstream iss(aFileName); - size_t start = 0, end = 0; - const char* sep = " ,|"; + istringstream iss(aFileName); + size_t start = 0, end = 0; + const char* sep = " ,|"; - end = aFileName.find_first_of(sep); + end = aFileName.find_first_of(sep); - do + do + { + if (end != string::npos) { - if (end != string::npos) - { - std::string aFile = aFileName.substr(start, end - start); + std::string aFile = aFileName.substr(start, end - start); - if (getDebugLvl() > 1) - cout << "File: " << aFileName.substr(start, end - start) << endl; + if (getDebugLvl() > 1) + cout << "File: " << aFileName.substr(start, end - start) << endl; - start = end + 1; - aInfileList.push_back(aFile); - } - else - { - std::string aFile = aFileName.substr(start, end - start); - - if (getDebugLvl() > 1) - cout << "Next Input File " << aFileName.substr(start, end - start) << endl; - - aInfileList.push_back(aFile); - break; - } - - end = aFileName.find_first_of(sep, start); + start = end + 1; + aInfileList.push_back(aFile); } - while (start != end); - - std::ostringstream aSS; - int aVecSize = aInfileList.size(); - int aVecIdx = 0; - - // Take file list one by one and append it to one string - while (aVecIdx < aVecSize) + else { - std::string aNextFile = aInfileList[aVecIdx]; - aVecIdx++; + std::string aFile = aFileName.substr(start, end - start); - //aInfileList.pop_front(); - if (aNextFile.at(0) != '/') - { - aSS << bulkRootPath << "/data/import/" + aNextFile; - } - else - { - aSS << aNextFile; - } + if (getDebugLvl() > 1) + cout << "Next Input File " << aFileName.substr(start, end - start) << endl; - if (aVecIdx < aVecSize) aSS << ","; + aInfileList.push_back(aFile); + break; } - //cout << "File list are = " << aSS.str() << endl; + end = aFileName.find_first_of(sep, start); + } while (start != end); - return aSS.str(); + std::ostringstream aSS; + int aVecSize = aInfileList.size(); + int aVecIdx = 0; + + // Take file list one by one and append it to one string + while (aVecIdx < aVecSize) + { + std::string aNextFile = aInfileList[aVecIdx]; + aVecIdx++; + + // aInfileList.pop_front(); + if (aNextFile.at(0) != '/') + { + aSS << bulkRootPath << "/data/import/" + aNextFile; + } + else + { + aSS << aNextFile; + } + + if (aVecIdx < aVecSize) + aSS << ","; + } + + // cout << "File list are = " << aSS.str() << endl; + + return aSS.str(); } //------------------------------------------------------------------------------ // Get set of column names in the "current" table being processed from the // Job xml file. //------------------------------------------------------------------------------ -void WECmdArgs::getColumnList( std::set& columnList ) const +void WECmdArgs::getColumnList(std::set& columnList) const { - columnList.clear(); + columnList.clear(); - for (unsigned k = 0; k < fColFldsFromJobFile.size(); k++) - { - columnList.insert( fColFldsFromJobFile[k] ); - } + for (unsigned k = 0; k < fColFldsFromJobFile.size(); k++) + { + columnList.insert(fColFldsFromJobFile[k]); + } } //----------------------------------------------------------------------------- @@ -1910,38 +1892,37 @@ void WECmdArgs::getColumnList( std::set& columnList ) const void WECmdArgs::checkForBulkLogDir(const std::string& BulkRoot) { - if ( !boost::filesystem::exists(BulkRoot.c_str()) ) - { - cout << "Creating directory : " << BulkRoot << endl; - boost::filesystem::create_directories(BulkRoot.c_str()); - } + if (!boost::filesystem::exists(BulkRoot.c_str())) + { + cout << "Creating directory : " << BulkRoot << endl; + boost::filesystem::create_directories(BulkRoot.c_str()); + } - if ( boost::filesystem::exists(BulkRoot.c_str()) ) - { - // create the job directory also if not existing - std::ostringstream aSS; - aSS << BulkRoot; - aSS << "/job"; - std::string jobDir = aSS.str(); + if (boost::filesystem::exists(BulkRoot.c_str())) + { + // create the job directory also if not existing + std::ostringstream aSS; + aSS << BulkRoot; + aSS << "/job"; + std::string jobDir = aSS.str(); - if ( !boost::filesystem::exists(jobDir.c_str()) ) - { - cout << "Creating directory : " << jobDir << endl; - bool aSuccess = boost::filesystem::create_directories(jobDir.c_str()); - - if (!aSuccess) - { - cout << "\nFailed to create job directory, check permissions\n" << endl; - throw runtime_error("Failed to create job directory, check permissions"); - } - } - } - else + if (!boost::filesystem::exists(jobDir.c_str())) { - cout << "\nFailed to create bulk directory, check permissions\n" << endl; - throw runtime_error("Failed to create bulk directory, check permissions"); + cout << "Creating directory : " << jobDir << endl; + bool aSuccess = boost::filesystem::create_directories(jobDir.c_str()); + + if (!aSuccess) + { + cout << "\nFailed to create job directory, check permissions\n" << endl; + throw runtime_error("Failed to create job directory, check permissions"); + } } + } + else + { + cout << "\nFailed to create bulk directory, check permissions\n" << endl; + throw runtime_error("Failed to create bulk directory, check permissions"); + } } - } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 2fcdf9b89..b4193db20 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ #ifndef WE_CMDARGS_H_ #define WE_CMDARGS_H_ @@ -31,326 +31,325 @@ namespace WriteEngine { - class WECmdArgs { -public: - WECmdArgs(int argc, char** argv); - virtual ~WECmdArgs() {} + public: + WECmdArgs(int argc, char** argv); + virtual ~WECmdArgs() + { + } - typedef std::vector VecInts; - typedef std::vector VecArgs; + typedef std::vector VecInts; + typedef std::vector VecArgs; - void appTestFunction(); - void parseCmdLineArgs(int argc, char** argv); - std::string getCpImportCmdLine(); - void setSchemaAndTableFromJobFile(std::string& JobName); - void setEnclByAndEscCharFromJobFile(std::string& JobName); - void usage(); - void usageMode3(); - bool checkForCornerCases(); - void checkForBulkLogDir(const std::string& BulkRoot); + void appTestFunction(); + void parseCmdLineArgs(int argc, char** argv); + std::string getCpImportCmdLine(); + void setSchemaAndTableFromJobFile(std::string& JobName); + void setEnclByAndEscCharFromJobFile(std::string& JobName); + void usage(); + void usageMode3(); + bool checkForCornerCases(); + void checkForBulkLogDir(const std::string& BulkRoot); - void addJobFilesToVector(std::string& JobName); - void splitConfigFilePerTable(std::string& ConfigName, int tblCount); - void write2ConfigFiles(std::vector& Files, - char* pBuff, int FileIdx); - void updateWithJobFile(int Idx); + void addJobFilesToVector(std::string& JobName); + void splitConfigFilePerTable(std::string& ConfigName, int tblCount); + void write2ConfigFiles(std::vector& Files, char* pBuff, int FileIdx); + void updateWithJobFile(int Idx); - std::string getJobFileName(); - std::string getBrmRptFileName(); - std::string getTmpFileDir(); - std::string getBulkRootDir(); - unsigned int getBatchQuantity(); - void checkJobIdCase(); - std::string getFileNameFromPath(const std::string& Path) const; - std::string getModuleID(); - std::string replaceCharInStr(const std::string& Str, char C, char R); + std::string getJobFileName(); + std::string getBrmRptFileName(); + std::string getTmpFileDir(); + std::string getBulkRootDir(); + unsigned int getBatchQuantity(); + void checkJobIdCase(); + std::string getFileNameFromPath(const std::string& Path) const; + std::string getModuleID(); + std::string replaceCharInStr(const std::string& Str, char C, char R); - std::string getJobId() const - { - return fJobId; - } - std::string getOrigJobId() const - { - return fOrigJobId; - } - std::string getLocFile() const - { - return fLocFile; - } - int getReadBufSize() - { - return fReadBufSize; - } - int getMode() - { - return fMode; - } - int getArgMode() const - { - return fArgMode; - } - bool isHelpMode() - { - return fHelp; - } - int getDebugLvl() - { - return fDebugLvl; - } - char getEnclChar() - { - return fEnclosedChar; - } - char getEscChar() - { - return fEscChar; - } - char getDelimChar() - { - return fColDelim; - } - ImportDataMode getImportDataMode() const - { - return fImportDataMode; - } - bool getConsoleLog() - { - return fConsoleLog; - } + std::string getJobId() const + { + return fJobId; + } + std::string getOrigJobId() const + { + return fOrigJobId; + } + std::string getLocFile() const + { + return fLocFile; + } + int getReadBufSize() + { + return fReadBufSize; + } + int getMode() + { + return fMode; + } + int getArgMode() const + { + return fArgMode; + } + bool isHelpMode() + { + return fHelp; + } + int getDebugLvl() + { + return fDebugLvl; + } + char getEnclChar() + { + return fEnclosedChar; + } + char getEscChar() + { + return fEscChar; + } + char getDelimChar() + { + return fColDelim; + } + ImportDataMode getImportDataMode() const + { + return fImportDataMode; + } + bool getConsoleLog() + { + return fConsoleLog; + } - bool isCpimportInvokeMode() - { - return (fBlockMode3) ? false : fCpiInvoke; - } - bool isQuiteMode() const - { - return fQuiteMode; - } - void setJobId(std::string fJobId) - { - this->fJobId = fJobId; - } - void setOrigJobId(std::string fOrigJobId) - { - this->fOrigJobId = fJobId; - } - void setLocFile(std::string fLocFile) - { - this->fLocFile = fLocFile; - } - void setMode(int fMode) - { - this->fMode = fMode; - } - void setArgMode(int ArgMode) - { - this->fArgMode = ArgMode; - } - void setPmFile(std::string fPmFile) - { - this->fPmFile = fPmFile; - } - void setQuiteMode(bool fQuiteMode) - { - this->fQuiteMode = fQuiteMode; - } - void setVerbose(int fVerbose) - { - this->fVerbose = fVerbose; - } - void setJobFileName(std::string& JobFileName) - { - fJobFile = JobFileName; - } - void setBrmRptFileName(std::string& BrmFileName) - { - this->fBrmRptFile = BrmFileName; - } - void setCpiInvoke(bool CpiInvoke = true) - { - this->fCpiInvoke = CpiInvoke; - } - void setBlockMode3(bool Block) - { - this->fBlockMode3 = Block; - } - void setTruncationAsError(bool bTruncationAsError) - { - fbTruncationAsError = bTruncationAsError; - } - void setUsername(const std::string& username); - - bool isJobLogOnly() const - { - return fJobLogOnly; - } - void setJobUUID(const boost::uuids::uuid& jobUUID) - { - fUUID = jobUUID; - } - bool getConsoleOutput( ) - { - return fConsoleOutput; - } - const std::string& getTimeZone() const - { - return fTimeZone; - } + bool isCpimportInvokeMode() + { + return (fBlockMode3) ? false : fCpiInvoke; + } + bool isQuiteMode() const + { + return fQuiteMode; + } + void setJobId(std::string fJobId) + { + this->fJobId = fJobId; + } + void setOrigJobId(std::string fOrigJobId) + { + this->fOrigJobId = fJobId; + } + void setLocFile(std::string fLocFile) + { + this->fLocFile = fLocFile; + } + void setMode(int fMode) + { + this->fMode = fMode; + } + void setArgMode(int ArgMode) + { + this->fArgMode = ArgMode; + } + void setPmFile(std::string fPmFile) + { + this->fPmFile = fPmFile; + } + void setQuiteMode(bool fQuiteMode) + { + this->fQuiteMode = fQuiteMode; + } + void setVerbose(int fVerbose) + { + this->fVerbose = fVerbose; + } + void setJobFileName(std::string& JobFileName) + { + fJobFile = JobFileName; + } + void setBrmRptFileName(std::string& BrmFileName) + { + this->fBrmRptFile = BrmFileName; + } + void setCpiInvoke(bool CpiInvoke = true) + { + this->fCpiInvoke = CpiInvoke; + } + void setBlockMode3(bool Block) + { + this->fBlockMode3 = Block; + } + void setTruncationAsError(bool bTruncationAsError) + { + fbTruncationAsError = bTruncationAsError; + } + void setUsername(const std::string& username); - bool getPmStatus(int Id); - bool str2PmList(std::string& PmList, VecInts& V); - int getPmVecSize() - { - return fPmVec.size(); - } - void add2PmVec(int PmId) - { - fPmVec.push_back(PmId); - } + bool isJobLogOnly() const + { + return fJobLogOnly; + } + void setJobUUID(const boost::uuids::uuid& jobUUID) + { + fUUID = jobUUID; + } + bool getConsoleOutput() + { + return fConsoleOutput; + } + const std::string& getTimeZone() const + { + return fTimeZone; + } - WECmdArgs::VecInts& getPmVec() - { - return fPmVec; - } - std::string getPmFile() const - { - return fPmFile; - } - int getVerbose() const - { - return fVerbose; - } - std::string getTableName() const - { - return fTable; - } - std::string getSchemaName() const - { - return fSchema; - } - bool getTruncationAsError() const - { - return fbTruncationAsError; - } + bool getPmStatus(int Id); + bool str2PmList(std::string& PmList, VecInts& V); + int getPmVecSize() + { + return fPmVec.size(); + } + void add2PmVec(int PmId) + { + fPmVec.push_back(PmId); + } - int getMultiTableCount() const - { - return fMultiTableCount; - } - void setMultiTableCount(int Count) - { - fMultiTableCount = Count; - } - - bool isS3Import() const - { - return !fS3Key.empty(); - } - std::string getS3Key() const - { - return fS3Key; - } - std::string getS3Bucket() const - { - return fS3Bucket; - } - std::string getS3Host() const - { - return fS3Host; - } - std::string getS3Secret() const - { - return fS3Secret; - } - std::string getS3Region() const - { - return fS3Region; - } - std::string getErrorDir() const - { - return fErrorDir; - } - void setErrorDir(std::string fErrorDir) - { - this->fErrorDir = fErrorDir; - } - std::string& getUsername(); - std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342 - void getColumnList( std::set& columnList ) const; + WECmdArgs::VecInts& getPmVec() + { + return fPmVec; + } + std::string getPmFile() const + { + return fPmFile; + } + int getVerbose() const + { + return fVerbose; + } + std::string getTableName() const + { + return fTable; + } + std::string getSchemaName() const + { + return fSchema; + } + bool getTruncationAsError() const + { + return fbTruncationAsError; + } -private: // variables for SplitterApp - VecArgs fVecArgs; - VecInts fPmVec; + int getMultiTableCount() const + { + return fMultiTableCount; + } + void setMultiTableCount(int Count) + { + fMultiTableCount = Count; + } - VecArgs fVecJobFiles; //JobFiles splitter from master JobFile - int fMultiTableCount; //MultiTable count - VecArgs fColFldsFromJobFile;//List of columns from any job file, that - // represent fields in the import data + bool isS3Import() const + { + return !fS3Key.empty(); + } + std::string getS3Key() const + { + return fS3Key; + } + std::string getS3Bucket() const + { + return fS3Bucket; + } + std::string getS3Host() const + { + return fS3Host; + } + std::string getS3Secret() const + { + return fS3Secret; + } + std::string getS3Region() const + { + return fS3Region; + } + std::string getErrorDir() const + { + return fErrorDir; + } + void setErrorDir(std::string fErrorDir) + { + this->fErrorDir = fErrorDir; + } + std::string& getUsername(); + std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342 + void getColumnList(std::set& columnList) const; - std::string fJobId; // JobID - std::string fOrigJobId; // Original JobID, in case we have to split it - bool fJobLogOnly; // Job number is only for log filename only - bool fHelp; // Help mode - int fMode; // splitter Mode - int fArgMode; // Argument mode, dep. on this fMode is decided. - bool fQuiteMode; // in quite mode or not - bool fConsoleLog; // Log everything to console - w.r.t cpimport - int fVerbose; // how many v's - std::string fPmFile; // FileName at PM - std::string fPmFilePath;// Path of input file in PM - std::string fLocFile; // Local file name - std::string fBrmRptFile;// BRM report file - std::string fJobPath; // Path to Job File - std::string fTmpFileDir;// Temp file directory. - std::string fBulkRoot; // Bulk Root path - std::string fJobFile; // Job File Name - std::string fS3Key; // S3 key - std::string fS3Secret; // S3 Secret - std::string fS3Bucket; // S3 Bucket - std::string fS3Host; // S3 Host - std::string fS3Region; // S3 Region + private: // variables for SplitterApp + VecArgs fVecArgs; + VecInts fPmVec; - unsigned int fBatchQty; // No. of batch Qty. - int fNoOfReadThrds; // No. of read buffers - //std::string fConfig; // config filename - int fDebugLvl; // Debug level - int fMaxErrors; // Max allowable errors - int fReadBufSize; // Read buffer size - int fIOReadBufSize; // I/O read buffer size - int fSetBufSize; // Buff size w/setvbuf - char fColDelim; // column delimiter - char fEnclosedChar; // enclosed by char - char fEscChar; // esc char - int fNoOfWriteThrds; // No. of write threads - bool fNullStrMode; // set null string mode - treat null as null - ImportDataMode fImportDataMode; // Importing text or binary data - std::string fPrgmName; // argv[0] - std::string fSchema; // Schema name - positional parmater - std::string fTable; // Table name - table name parameter + VecArgs fVecJobFiles; // JobFiles splitter from master JobFile + int fMultiTableCount; // MultiTable count + VecArgs fColFldsFromJobFile; // List of columns from any job file, that + // represent fields in the import data - bool fCpiInvoke; // invoke cpimport in mode 3 - bool fBlockMode3; // Do not allow Mode 3 - bool fbTruncationAsError; // Treat string truncation as error - boost::uuids::uuid fUUID; - bool fConsoleOutput; // If false, no output to console. - std::string fTimeZone; // Timezone to use for TIMESTAMP datatype - std::string fUsername; // Username of the data files owner - std::string fErrorDir; + std::string fJobId; // JobID + std::string fOrigJobId; // Original JobID, in case we have to split it + bool fJobLogOnly; // Job number is only for log filename only + bool fHelp; // Help mode + int fMode; // splitter Mode + int fArgMode; // Argument mode, dep. on this fMode is decided. + bool fQuiteMode; // in quite mode or not + bool fConsoleLog; // Log everything to console - w.r.t cpimport + int fVerbose; // how many v's + std::string fPmFile; // FileName at PM + std::string fPmFilePath; // Path of input file in PM + std::string fLocFile; // Local file name + std::string fBrmRptFile; // BRM report file + std::string fJobPath; // Path to Job File + std::string fTmpFileDir; // Temp file directory. + std::string fBulkRoot; // Bulk Root path + std::string fJobFile; // Job File Name + std::string fS3Key; // S3 key + std::string fS3Secret; // S3 Secret + std::string fS3Bucket; // S3 Bucket + std::string fS3Host; // S3 Host + std::string fS3Region; // S3 Region + + unsigned int fBatchQty; // No. of batch Qty. + int fNoOfReadThrds; // No. of read buffers + // std::string fConfig; // config filename + int fDebugLvl; // Debug level + int fMaxErrors; // Max allowable errors + int fReadBufSize; // Read buffer size + int fIOReadBufSize; // I/O read buffer size + int fSetBufSize; // Buff size w/setvbuf + char fColDelim; // column delimiter + char fEnclosedChar; // enclosed by char + char fEscChar; // esc char + int fNoOfWriteThrds; // No. of write threads + bool fNullStrMode; // set null string mode - treat null as null + ImportDataMode fImportDataMode; // Importing text or binary data + std::string fPrgmName; // argv[0] + std::string fSchema; // Schema name - positional parmater + std::string fTable; // Table name - table name parameter + + bool fCpiInvoke; // invoke cpimport in mode 3 + bool fBlockMode3; // Do not allow Mode 3 + bool fbTruncationAsError; // Treat string truncation as error + boost::uuids::uuid fUUID; + bool fConsoleOutput; // If false, no output to console. + std::string fTimeZone; // Timezone to use for TIMESTAMP datatype + std::string fUsername; // Username of the data files owner + std::string fErrorDir; }; //---------------------------------------------------------------------- inline void WECmdArgs::setUsername(const std::string& username) { - fUsername = username; + fUsername = username; } inline std::string& WECmdArgs::getUsername() { - return fUsername; + return fUsername; } - -} +} // namespace WriteEngine #endif /* WE_CMDARGS_H_ */ diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index f6aa8f3b1..4faab3fa6 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_filereadthread.cpp @@ -27,7 +27,6 @@ * Author: bpaul */ - #include "we_messages.h" #include "we_sdhandler.h" #include "we_splitterapp.h" @@ -53,631 +52,627 @@ using namespace std; namespace WriteEngine { - -void WEReadThreadRunner::operator ()() +void WEReadThreadRunner::operator()() { - try - { - fRef.feedData(); - } - catch (std::exception& ex) - { - throw runtime_error(ex.what()); - } + try + { + fRef.feedData(); + } + catch (std::exception& ex) + { + throw runtime_error(ex.what()); + } } - //------------------------------------------------------------------------------ // //------------------------------------------------------------------------------ - - -WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh), - fpThread(0), - fFileMutex(), - fContinue(true), - fInFileName(), - fInFile(std::cin.rdbuf()), //@BUG 4326 - fTgtPmId(0), - fBatchQty(0), - fEnclEsc(false), - fEncl('\0'), - fEsc('\\'), - fDelim('|') +WEFileReadThread::WEFileReadThread(WESDHandler& aSdh) + : fSdh(aSdh) + , fpThread(0) + , fFileMutex() + , fContinue(true) + , fInFileName() + , fInFile(std::cin.rdbuf()) + , //@BUG 4326 + fTgtPmId(0) + , fBatchQty(0) + , fEnclEsc(false) + , fEncl('\0') + , fEsc('\\') + , fDelim('|') { - //TODO batch qty to get from config - fBatchQty = 10000; + // TODO batch qty to get from config + fBatchQty = 10000; - if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE) - { - fBuffSize = DEFAULTBUFFSIZE; - } - else - { - fBuffSize = fSdh.getReadBufSize(); - } + if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE) + { + fBuffSize = DEFAULTBUFFSIZE; + } + else + { + fBuffSize = fSdh.getReadBufSize(); + } - fBuff = new char [fBuffSize]; + fBuff = new char[fBuffSize]; - const WECmdArgs &args = fSdh.fRef.fCmdArgs; - initS3Connection(args); + const WECmdArgs& args = fSdh.fRef.fCmdArgs; + initS3Connection(args); } -//WEFileReadThread::WEFileReadThread(const WEFileReadThread& rhs):fSdh(rhs.fSdh) +// WEFileReadThread::WEFileReadThread(const WEFileReadThread& rhs):fSdh(rhs.fSdh) //{ // // TODO copy constructor //} - WEFileReadThread::~WEFileReadThread() { - //if(fInFile.is_open()) fInFile.close(); //@BUG 4326 - if (fIfFile.is_open()) fIfFile.close(); + // if(fInFile.is_open()) fInFile.close(); //@BUG 4326 + if (fIfFile.is_open()) + fIfFile.close(); - setTgtPmId(0); + setTgtPmId(0); - if (fpThread) + if (fpThread) + { + delete fpThread; + } + + fpThread = 0; + delete[] fBuff; + // cout << "WEFileReadThread destructor called" << endl; + + if (doS3Import) + { + ms3_deinit(s3Connection); + ms3_library_deinit(); + if (buf) { - delete fpThread; - } - - fpThread = 0; - delete []fBuff; - //cout << "WEFileReadThread destructor called" << endl; - - if (doS3Import) - { - ms3_deinit(s3Connection); - ms3_library_deinit(); - if (buf) - { - s3Stream.reset(); - arrSource.reset(); - free(buf); - } + s3Stream.reset(); + arrSource.reset(); + free(buf); } + } } //------------------------------------------------------------------------------ void WEFileReadThread::reset() { - //if(fInFile.is_open()) fInFile.close(); //@BUG 4326 - if (fIfFile.is_open()) fIfFile.close(); + // if(fInFile.is_open()) fInFile.close(); //@BUG 4326 + if (fIfFile.is_open()) + fIfFile.close(); - setTgtPmId(0); + setTgtPmId(0); - if (fpThread) - { - delete fpThread; - } + if (fpThread) + { + delete fpThread; + } - fpThread = 0; - //cout << "WEFileReadThread destructor called" << endl; - this->setContinue(true); - - if (buf) - { - arrSource.reset(); - s3Stream.reset(); - free(buf); - buf = NULL; - } + fpThread = 0; + // cout << "WEFileReadThread destructor called" << endl; + this->setContinue(true); + + if (buf) + { + arrSource.reset(); + s3Stream.reset(); + free(buf); + buf = NULL; + } } //------------------------------------------------------------------------------ void WEFileReadThread::setup(std::string FileName) { - if (fSdh.getDebugLvl()) - cout << "WEFileReadThread::setup : *** Input files = " << FileName << endl; + if (fSdh.getDebugLvl()) + cout << "WEFileReadThread::setup : *** Input files = " << FileName << endl; - reset(); + reset(); - try - { - char aEncl = fSdh.getEnclChar(); - char aEsc = fSdh.getEscChar(); - char aDelim = fSdh.getDelimChar(); + try + { + char aEncl = fSdh.getEnclChar(); + char aEsc = fSdh.getEscChar(); + char aDelim = fSdh.getDelimChar(); - if (aEncl) fEncl = aEncl; + if (aEncl) + fEncl = aEncl; - if (aEsc) fEsc = aEsc; + if (aEsc) + fEsc = aEsc; - if (aDelim) fDelim = aDelim; + if (aDelim) + fDelim = aDelim; - if (aEncl != 0) fEnclEsc = true; + if (aEncl != 0) + fEnclEsc = true; - //BUG 4342 - Need to support "list of infiles" - //chkForListOfFiles(FileName); - List prepared in sdhandler. + // BUG 4342 - Need to support "list of infiles" + // chkForListOfFiles(FileName); - List prepared in sdhandler. - string aStrName = getNextInputDataFile(); + string aStrName = getNextInputDataFile(); - if (fSdh.getDebugLvl() > 2) - cout << "Next InFileName = " << aStrName << endl; + if (fSdh.getDebugLvl() > 2) + cout << "Next InFileName = " << aStrName << endl; - setInFileName(aStrName); - //setInFileName(FileName); - openInFile(); - //set the target PM - fpThread = new boost::thread(WEReadThreadRunner(*this)); - } - catch (std::exception& ex) - { - //cout << ex.what() << endl; - //throw runtime_error("Exception occured in WEFileReadThread\n"); - throw runtime_error(ex.what()); - } + setInFileName(aStrName); + // setInFileName(FileName); + openInFile(); + // set the target PM + fpThread = new boost::thread(WEReadThreadRunner(*this)); + } + catch (std::exception& ex) + { + // cout << ex.what() << endl; + // throw runtime_error("Exception occured in WEFileReadThread\n"); + throw runtime_error(ex.what()); + } - if (fpThread) - { - // Need to send a all clear?? - } + if (fpThread) + { + // Need to send a all clear?? + } } //------------------------------------------------------------------------------ bool WEFileReadThread::chkForListOfFiles(std::string& FileName) { - //cout << "Inside chkForListOfFiles("<< FileName << ")" << endl; - std::string aFileName = FileName; + // cout << "Inside chkForListOfFiles("<< FileName << ")" << endl; + std::string aFileName = FileName; - istringstream iss(aFileName); - ostringstream oss; - size_t start = 0, end = 0; - const char* sep = " ,|"; - ms3_status_st ms3status; + istringstream iss(aFileName); + ostringstream oss; + size_t start = 0, end = 0; + const char* sep = " ,|"; + ms3_status_st ms3status; - do + do + { + end = aFileName.find_first_of(sep, start); + std::string aFile = aFileName.substr(start, end - start); + if (aFile == "STDIN" || aFile == "stdin") + aFile = "/dev/stdin"; + + if (fSdh.getDebugLvl() > 1) + cout << "Next Input File " << aFile << endl; + + if ((!doS3Import && access(aFile.c_str(), O_RDONLY) != 0) || + (doS3Import && ms3_status(s3Connection, s3Bucket.c_str(), aFile.c_str(), &ms3status) != 0)) { - end = aFileName.find_first_of(sep, start); - std::string aFile = aFileName.substr(start, end - start); - if (aFile == "STDIN" || aFile == "stdin") - aFile = "/dev/stdin"; - - if (fSdh.getDebugLvl() > 1) - cout << "Next Input File " << aFile << endl; - - if ((!doS3Import && access(aFile.c_str(), O_RDONLY) != 0) || - (doS3Import && ms3_status(s3Connection, s3Bucket.c_str(), - aFile.c_str(), &ms3status) != 0)) - { - oss << "Could not access " << aFile; - throw runtime_error(oss.str()); - } - - fInfileList.push_back(aFile); - start = end + 1; + oss << "Could not access " << aFile; + throw runtime_error(oss.str()); } - while (end != string::npos); - //cout << "Going out chkForListOfFiles("<< FileName << ")" << endl; + fInfileList.push_back(aFile); + start = end + 1; + } while (end != string::npos); - return false; + // cout << "Going out chkForListOfFiles("<< FileName << ")" << endl; + + return false; } //------------------------------------------------------------------------------ std::string WEFileReadThread::getNextInputDataFile() { - std::string aNextFile; + std::string aNextFile; - if (fInfileList.size() > 0) - { - aNextFile = fInfileList.front(); - fInfileList.pop_front(); - } + if (fInfileList.size() > 0) + { + aNextFile = fInfileList.front(); + fInfileList.pop_front(); + } - //cout << "Next Input DataFile = " << aNextFile << endl; + // cout << "Next Input DataFile = " << aNextFile << endl; - return aNextFile; + return aNextFile; } //------------------------------------------------------------------------------ void WEFileReadThread::add2InputDataFileList(std::string& FileName) { - fInfileList.push_front(FileName); + fInfileList.push_front(FileName); } //------------------------------------------------------------------------------ void WEFileReadThread::shutdown() { - this->setContinue(false); - boost::mutex::scoped_lock aLock(fFileMutex); //wait till readDataFile() finish + this->setContinue(false); + boost::mutex::scoped_lock aLock(fFileMutex); // wait till readDataFile() finish - //if(fInFile.is_open()) fInFile.close(); //@BUG 4326 - if (fIfFile.is_open()) fIfFile.close(); - if (buf) - { - s3Stream.reset(); - arrSource.reset(); - free(buf); - buf = NULL; - } + // if(fInFile.is_open()) fInFile.close(); //@BUG 4326 + if (fIfFile.is_open()) + fIfFile.close(); + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + buf = NULL; + } } //------------------------------------------------------------------------------ void WEFileReadThread::feedData() { - unsigned int aRowCnt = 0; - const unsigned int c10mSec = 10000; + unsigned int aRowCnt = 0; + const unsigned int c10mSec = 10000; - while (isContinue()) + while (isContinue()) + { + unsigned int TgtPmId = getTgtPmId(); + + if (TgtPmId == 0) { - unsigned int TgtPmId = getTgtPmId(); - - if (TgtPmId == 0) - { - setTgtPmId(fSdh.getNextPm2Feed()); - TgtPmId = getTgtPmId(); - } - - if ((TgtPmId > 0) && (fInFile.good())) - { - try - { - messageqcpp::SBS aSbs(new messageqcpp::ByteStream); - - if (fSdh.getImportDataMode() == IMPORT_DATA_TEXT) - aRowCnt = readDataFile(aSbs); - else - aRowCnt = readBinaryDataFile(aSbs, - fSdh.getTableRecLen() ); - - //cout << "Length " << aSbs->length() < 0) && (fInFile.good())) + { + try + { + messageqcpp::SBS aSbs(new messageqcpp::ByteStream); + + if (fSdh.getImportDataMode() == IMPORT_DATA_TEXT) + aRowCnt = readDataFile(aSbs); + else + aRowCnt = readBinaryDataFile(aSbs, fSdh.getTableRecLen()); + + // cout << "Length " << aSbs->length() < 0)) + { + fBuff[aLen - 1] = '\n'; + fBuff[aLen] = 0; - while ((!fInFile.eof()) && (aIdx < getBatchQty())) - { - if (fEnclEsc) - { - //pStart = aBuff; - aLen = getNextRow(fInFile, fBuff, fBuffSize - 1); - } - else - { - fInFile.getline(fBuff, fBuffSize - 1); - aLen = fInFile.gcount(); - } + if (fSdh.getDebugLvl() > 3) + cout << "Data Read " << fBuff << endl; - ////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it - //cout << "Data Length " << aLen < 0)) - { - fBuff[aLen - 1] = '\n'; - fBuff[aLen] = 0; + (*Sbs).append(reinterpret_cast(fBuff), aLen); + aIdx++; - if (fSdh.getDebugLvl() > 3) cout << "Data Read " << fBuff << endl; + if (fSdh.getDebugLvl() > 2) + cout << "File data line = " << aIdx << endl; + } + else if (aLen >= fBuffSize - 2) // Didn't hit delim; BIG ROW + { + cout << "Bad Row data " << endl; + cout << fBuff << endl; + throw runtime_error("Data Row too BIG to handle!!"); + } - (*Sbs).append(reinterpret_cast(fBuff), aLen); - aIdx++; + // for debug + // if(fSdh.getDebugLvl()>3) cout << aIdx << endl; + } // while - if (fSdh.getDebugLvl() > 2) cout << "File data line = " << aIdx << endl; - } - else if (aLen >= fBuffSize - 2) //Didn't hit delim; BIG ROW - { - cout << "Bad Row data " << endl; - cout << fBuff << endl; - throw runtime_error("Data Row too BIG to handle!!"); - } + return aIdx; + } // if - //for debug - //if(fSdh.getDebugLvl()>3) cout << aIdx << endl; - }// while - - return aIdx; - }// if - - return 0; + return 0; } //------------------------------------------------------------------------------ // Read input data as binary data //------------------------------------------------------------------------------ -unsigned int WEFileReadThread::readBinaryDataFile(messageqcpp::SBS& Sbs, - unsigned int recLen) +unsigned int WEFileReadThread::readBinaryDataFile(messageqcpp::SBS& Sbs, unsigned int recLen) { - boost::mutex::scoped_lock aLock(fFileMutex); + boost::mutex::scoped_lock aLock(fFileMutex); - if ((fInFile.good()) && (!fInFile.eof())) + if ((fInFile.good()) && (!fInFile.eof())) + { + unsigned int aIdx = 0; + unsigned int aLen = 0; + *Sbs << (ByteStream::byte)(WE_CLT_SRV_DATA); + + while ((!fInFile.eof()) && (aIdx < getBatchQty())) { - unsigned int aIdx = 0; - unsigned int aLen = 0; - *Sbs << (ByteStream::byte)(WE_CLT_SRV_DATA); + fInFile.read(fBuff, recLen); + aLen = fInFile.gcount(); - while ( (!fInFile.eof()) && (aIdx < getBatchQty()) ) + if (aLen > 0) + { + (*Sbs).append(reinterpret_cast(fBuff), aLen); + aIdx++; + + if (fSdh.getDebugLvl() > 2) + cout << "Binary input data line = " << aIdx << endl; + + if (aLen != recLen) { - fInFile.read(fBuff, recLen); - aLen = fInFile.gcount(); + cout << "Binary input data does not end on record boundary;" + " Last record is " + << aLen << " bytes long." + << " Expected record length is: " << recLen << endl; + } + } + } // while - if (aLen > 0) - { - (*Sbs).append(reinterpret_cast(fBuff), aLen); - aIdx++; + return aIdx; + } // if - if (fSdh.getDebugLvl() > 2) - cout << "Binary input data line = " << aIdx << endl; - - if (aLen != recLen) - { - cout << "Binary input data does not end on record boundary;" - " Last record is " << aLen << " bytes long." << - " Expected record length is: " << recLen << endl; - } - } - } // while - - return aIdx; - } // if - - return 0; + return 0; } //------------------------------------------------------------------------------ void WEFileReadThread::openInFile() { - try + try + { + /* If an S3 transfer + use ms3 lib to d/l data into mem + use boost::iostreams to wrap the mem in a stream interface + point infile's stream buffer to it. + */ + + if (fSdh.getDebugLvl()) + cout << "Input Filename: " << fInFileName << endl; + + if (doS3Import) { - /* If an S3 transfer - use ms3 lib to d/l data into mem - use boost::iostreams to wrap the mem in a stream interface - point infile's stream buffer to it. - */ - - if (fSdh.getDebugLvl()) cout << "Input Filename: " << fInFileName << endl; + size_t bufLen = 0; + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + buf = NULL; + } + if (fSdh.getDebugLvl()) + cout << "Downloading " << fInFileName << endl; + int err = ms3_get(s3Connection, s3Bucket.c_str(), fInFileName.c_str(), &buf, &bufLen); + if (fSdh.getDebugLvl()) + cout << "Download complete." << endl; + if (err) + { + ostringstream os; + if (ms3_server_error(s3Connection)) + os << "Download of '" << fInFileName + << "' failed. Error from the server: " << ms3_server_error(s3Connection); + else + os << "Download of '" << fInFileName << "' failed. Got '" << ms3_error(err) << "'."; + throw runtime_error(os.str()); + } - if (doS3Import) - { - size_t bufLen = 0; - if (buf) - { - s3Stream.reset(); - arrSource.reset(); - free(buf); - buf = NULL; - } - if (fSdh.getDebugLvl()) - cout << "Downloading " << fInFileName << endl; - int err = ms3_get(s3Connection, s3Bucket.c_str(), fInFileName.c_str(), - &buf, &bufLen); - if (fSdh.getDebugLvl()) - cout << "Download complete." << endl; - if (err) - { - ostringstream os; - if (ms3_server_error(s3Connection)) - os << "Download of '" << fInFileName << "' failed. Error from the server: " - << ms3_server_error(s3Connection); - else - os << "Download of '" << fInFileName << "' failed. Got '" << ms3_error(err) - << "'."; - throw runtime_error(os.str()); - } - - arrSource.reset(new boost::iostreams::array_source((char *) buf, bufLen)); - s3Stream.reset(new boost::iostreams::stream(*arrSource)); - fInFile.rdbuf(s3Stream->rdbuf()); - } + arrSource.reset(new boost::iostreams::array_source((char*)buf, bufLen)); + s3Stream.reset(new boost::iostreams::stream(*arrSource)); + fInFile.rdbuf(s3Stream->rdbuf()); + } - else if (fInFileName == "/dev/stdin") - { - char aDefCon[16], aGreenCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m"); + else if (fInFileName == "/dev/stdin") + { + char aDefCon[16], aGreenCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m"); - if (fSdh.getDebugLvl()) // BUG 4195 - cout << aGreenCol - << "trying to read from STDIN... " - << aDefCon << endl; - fInFile.rdbuf(cin.rdbuf()); - } + if (fSdh.getDebugLvl()) // BUG 4195 + cout << aGreenCol << "trying to read from STDIN... " << aDefCon << endl; + fInFile.rdbuf(cin.rdbuf()); + } - //@BUG 4326 - else if (fInFileName != "/dev/stdin") - { - if (!fIfFile.is_open()) - { - if (fSdh.getImportDataMode() == IMPORT_DATA_TEXT) - fIfFile.open(fInFileName.c_str()); - else // @bug 5193: binary import - fIfFile.open(fInFileName.c_str(), - std::ios_base::in | std::ios_base::binary); - } + //@BUG 4326 + else if (fInFileName != "/dev/stdin") + { + if (!fIfFile.is_open()) + { + if (fSdh.getImportDataMode() == IMPORT_DATA_TEXT) + fIfFile.open(fInFileName.c_str()); + else // @bug 5193: binary import + fIfFile.open(fInFileName.c_str(), std::ios_base::in | std::ios_base::binary); + } - if (!fIfFile.good()) - throw runtime_error("Could not open Input file " + fInFileName); + if (!fIfFile.good()) + throw runtime_error("Could not open Input file " + fInFileName); - fInFile.rdbuf(fIfFile.rdbuf()); //@BUG 4326 - } + fInFile.rdbuf(fIfFile.rdbuf()); //@BUG 4326 + } #ifdef _MSC_VER - else // @bug 5193: binary import + else // @bug 5193: binary import + { + if (fSdh.getImportDataMode() != IMPORT_DATA_TEXT) + { + if (_setmode(_fileno(stdin), _O_BINARY) == -1) { - if (fSdh.getImportDataMode() != IMPORT_DATA_TEXT) - { - if (_setmode(_fileno(stdin), _O_BINARY) == -1) - { - throw runtime_error("Could not change stdin to binary"); - } - } + throw runtime_error("Could not change stdin to binary"); } + } + } #endif - //@BUG 4326 -below three lines commented out - // if (!fInFile.is_open()) fInFile.open(fInFileName.c_str()); - // if (!fInFile.good()) - // throw runtime_error("Could not open Input file "+fInFileName); - - } - catch (std::exception& ex) - { - cout << "Error in Opening input data file " << fInFileName << endl; - throw runtime_error(ex.what()); //BUG 4201 FIX - } + //@BUG 4326 -below three lines commented out + // if (!fInFile.is_open()) fInFile.open(fInFileName.c_str()); + // if (!fInFile.good()) + // throw runtime_error("Could not open Input file "+fInFileName); + } + catch (std::exception& ex) + { + cout << "Error in Opening input data file " << fInFileName << endl; + throw runtime_error(ex.what()); // BUG 4201 FIX + } } //------------------------------------------------------------------------------ int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen) { - //const char ENCL ='\"'; //TODO for time being - //const char ESC = '\0'; //TODO for time being - const char ENCL = fEncl; - const char ESC = fEsc; - bool aTrailEsc = false; - char* pEnd = pBuf; - char aCh = ifs.get(); + // const char ENCL ='\"'; //TODO for time being + // const char ESC = '\0'; //TODO for time being + const char ENCL = fEncl; + const char ESC = fEsc; + bool aTrailEsc = false; + char* pEnd = pBuf; + char aCh = ifs.get(); - while (ifs.good()) + while (ifs.good()) + { + if (aCh == ENCL) { - if (aCh == ENCL) - { - // we got the first enclosedBy char. - *pEnd++ = aCh; - aCh = ifs.get(); + // we got the first enclosedBy char. + *pEnd++ = aCh; + aCh = ifs.get(); - //cout << "aCh 1 = " << aCh << endl; - while (aCh != ENCL) //Loop thru till we hit another one - { - if (aCh == ESC) //check spl cond ESC inside ENCL of '\n' here - { - *pEnd++ = aCh; - aCh = ifs.get(); - *pEnd++ = aCh; - aCh = ifs.get(); // get the next char for while loop - //cout << "aCh 2 = " << aCh << endl; - }// case ESC - else - { - *pEnd++ = aCh; - aCh = ifs.get(); - //cout << "aCh 3 = " << aCh << endl; - } - } - - *pEnd++ = aCh; // ENCL char got - aTrailEsc = true; //@BUG 4641 - }//case ENCL - else if (aCh == ESC) + // cout << "aCh 1 = " << aCh << endl; + while (aCh != ENCL) // Loop thru till we hit another one + { + if (aCh == ESC) // check spl cond ESC inside ENCL of '\n' here { - *pEnd++ = aCh; - aCh = ifs.get(); - *pEnd++ = aCh; - //cout << "aCh 4 = " << aCh << endl; - }// case ESC + *pEnd++ = aCh; + aCh = ifs.get(); + *pEnd++ = aCh; + aCh = ifs.get(); // get the next char for while loop + // cout << "aCh 2 = " << aCh << endl; + } // case ESC else { - *pEnd++ = aCh; - //cout << "aCh 5 = " << aCh << endl; + *pEnd++ = aCh; + aCh = ifs.get(); + // cout << "aCh 3 = " << aCh << endl; } + } - //cout << "pBuf1 " << pBuf << endl; - if ((aCh == '\n') || ((pEnd - pBuf) == MaxLen)) break; // we got a full row + *pEnd++ = aCh; // ENCL char got + aTrailEsc = true; //@BUG 4641 + } // case ENCL + else if (aCh == ESC) + { + *pEnd++ = aCh; + aCh = ifs.get(); + *pEnd++ = aCh; + // cout << "aCh 4 = " << aCh << endl; + } // case ESC + else + { + *pEnd++ = aCh; + // cout << "aCh 5 = " << aCh << endl; + } + // cout << "pBuf1 " << pBuf << endl; + if ((aCh == '\n') || ((pEnd - pBuf) == MaxLen)) + break; // we got a full row + + aCh = ifs.get(); + + // BUG 4641 To avoid seg fault when a wrong/no ESC char provided. + while (aTrailEsc) + { + // BUG 4903 EOF, to handle files ending w/ EOF and w/o '\n' + if ((aCh == '\n') || (aCh == EOF) || (aCh == fDelim)) + { + aTrailEsc = false; + break; + } + else + { + *pEnd++ = aCh; aCh = ifs.get(); + } + } - // BUG 4641 To avoid seg fault when a wrong/no ESC char provided. - while (aTrailEsc) - { - // BUG 4903 EOF, to handle files ending w/ EOF and w/o '\n' - if ((aCh == '\n') || (aCh == EOF) || (aCh == fDelim)) - { - aTrailEsc = false; - break; - } - else - { - *pEnd++ = aCh; - aCh = ifs.get(); - } - } + } // end of while loop - }// end of while loop - - return pEnd - pBuf; + return pEnd - pBuf; } -void WEFileReadThread::initS3Connection(const WECmdArgs &args) +void WEFileReadThread::initS3Connection(const WECmdArgs& args) { - doS3Import = args.isS3Import(); - if (doS3Import) - { - s3Key = args.getS3Key(); - s3Secret = args.getS3Secret(); - s3Bucket = args.getS3Bucket(); - s3Region = args.getS3Region(); - s3Host = args.getS3Host(); - ms3_library_init(); - s3Connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str())); - if (!s3Connection) - throw runtime_error("failed to get an S3 connection"); - } - else - s3Connection = NULL; - buf = NULL; + doS3Import = args.isS3Import(); + if (doS3Import) + { + s3Key = args.getS3Key(); + s3Secret = args.getS3Secret(); + s3Bucket = args.getS3Bucket(); + s3Region = args.getS3Region(); + s3Host = args.getS3Host(); + ms3_library_init(); + s3Connection = + ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str())); + if (!s3Connection) + throw runtime_error("failed to get an S3 connection"); + } + else + s3Connection = NULL; + buf = NULL; } //------------------------------------------------------------------------------ - } /* namespace WriteEngine */ - diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h index f81f5d492..a9a1c4e25 100644 --- a/writeengine/splitter/we_filereadthread.h +++ b/writeengine/splitter/we_filereadthread.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_filereadthread.h @@ -37,140 +37,137 @@ namespace WriteEngine { - class WESDHandler; class WEFileReadThread; class WEReadThreadRunner { -public: - WEReadThreadRunner(WEFileReadThread& Owner): fRef(Owner) - { - // ctor - } - ~WEReadThreadRunner() - { - } + public: + WEReadThreadRunner(WEFileReadThread& Owner) : fRef(Owner) + { + // ctor + } + ~WEReadThreadRunner() + { + } - void operator()(); // Thread function - -private: - WEFileReadThread& fRef; + void operator()(); // Thread function + private: + WEFileReadThread& fRef; }; - - - //------------------------------------------------------------------------------ - class WEFileReadThread { -public: - WEFileReadThread(WESDHandler& aSdh); - virtual ~WEFileReadThread(); + public: + WEFileReadThread(WESDHandler& aSdh); + virtual ~WEFileReadThread(); - void reset(); - void setup(std::string FileName); - void shutdown(); - void feedData(); - unsigned int readDataFile(messageqcpp::SBS& Sbs); - unsigned int readBinaryDataFile(messageqcpp::SBS& Sbs, unsigned int recLen); - void openInFile(); + void reset(); + void setup(std::string FileName); + void shutdown(); + void feedData(); + unsigned int readDataFile(messageqcpp::SBS& Sbs); + unsigned int readBinaryDataFile(messageqcpp::SBS& Sbs, unsigned int recLen); + void openInFile(); - int getNextRow(std::istream& ifs, char* pBuf, int MaxLen); + int getNextRow(std::istream& ifs, char* pBuf, int MaxLen); - boost::thread* getFpThread() const - { - return fpThread; - } - bool isContinue() const - { - return fContinue; - } - void setContinue(bool fContinue) - { - this->fContinue = fContinue; - } - std::string getInFileName() const - { - return fInFileName; - } - unsigned int getTgtPmId() const - { - return fTgtPmId; - } - unsigned int getBatchQty() const - { - return fBatchQty; - } - void setFpThread(boost::thread* fpThread) - { - this->fpThread = fpThread; - } - void setInFileName(std::string fInFileName) - { - if ((0 == fInFileName.compare("STDIN")) || (0 == fInFileName.compare("stdin"))) - this->fInFileName = "/dev/stdin"; - else - this->fInFileName = fInFileName; - } - //@BUG 4326 - const std::istream& getInFile() const - { - return fInFile; - } - void setBatchQty(unsigned int BatchQty) - { - fBatchQty = BatchQty; - } + boost::thread* getFpThread() const + { + return fpThread; + } + bool isContinue() const + { + return fContinue; + } + void setContinue(bool fContinue) + { + this->fContinue = fContinue; + } + std::string getInFileName() const + { + return fInFileName; + } + unsigned int getTgtPmId() const + { + return fTgtPmId; + } + unsigned int getBatchQty() const + { + return fBatchQty; + } + void setFpThread(boost::thread* fpThread) + { + this->fpThread = fpThread; + } + void setInFileName(std::string fInFileName) + { + if ((0 == fInFileName.compare("STDIN")) || (0 == fInFileName.compare("stdin"))) + this->fInFileName = "/dev/stdin"; + else + this->fInFileName = fInFileName; + } + //@BUG 4326 + const std::istream& getInFile() const + { + return fInFile; + } + void setBatchQty(unsigned int BatchQty) + { + fBatchQty = BatchQty; + } - bool chkForListOfFiles(std::string& FileName); - std::string getNextInputDataFile(); - void add2InputDataFileList(std::string& FileName); + bool chkForListOfFiles(std::string& FileName); + std::string getNextInputDataFile(); + void add2InputDataFileList(std::string& FileName); -private: - enum { DEFAULTBUFFSIZE = 1024 * 1024 }; + private: + enum + { + DEFAULTBUFFSIZE = 1024 * 1024 + }; - // don't allow anyone else to set - void setTgtPmId(unsigned int fTgtPmId) - { - this->fTgtPmId = fTgtPmId; - } + // don't allow anyone else to set + void setTgtPmId(unsigned int fTgtPmId) + { + this->fTgtPmId = fTgtPmId; + } - WESDHandler& fSdh; - boost::thread* fpThread; - boost::mutex fFileMutex; - bool fContinue; - std::string fInFileName; - std::istream fInFile; //@BUG 4326 - std::ifstream fIfFile; //@BUG 4326 + WESDHandler& fSdh; + boost::thread* fpThread; + boost::mutex fFileMutex; + bool fContinue; + std::string fInFileName; + std::istream fInFile; //@BUG 4326 + std::ifstream fIfFile; //@BUG 4326 - typedef std::list strList; - strList fInfileList; + typedef std::list strList; + strList fInfileList; - unsigned int fTgtPmId; - unsigned int fBatchQty; - bool fEnclEsc; // Encl/Esc char is set - char fEncl; // Encl char - char fEsc; // Esc char - char fDelim; // Column Delimit char - char* fBuff; // main data buffer - int fBuffSize; - - /* To support mode 1 imports from objects on S3 */ - void initS3Connection(const WECmdArgs &); - bool doS3Import; - std::string s3Key; - std::string s3Secret; - std::string s3Bucket; - std::string s3Region; - std::string s3Host; - ms3_st *s3Connection; - uint8_t *buf; - std::unique_ptr arrSource; - std::unique_ptr > s3Stream; + unsigned int fTgtPmId; + unsigned int fBatchQty; + bool fEnclEsc; // Encl/Esc char is set + char fEncl; // Encl char + char fEsc; // Esc char + char fDelim; // Column Delimit char + char* fBuff; // main data buffer + int fBuffSize; + + /* To support mode 1 imports from objects on S3 */ + void initS3Connection(const WECmdArgs&); + bool doS3Import; + std::string s3Key; + std::string s3Secret; + std::string s3Bucket; + std::string s3Region; + std::string s3Host; + ms3_st* s3Connection; + uint8_t* buf; + std::unique_ptr arrSource; + std::unique_ptr > s3Stream; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_respreadthread.cpp b/writeengine/splitter/we_respreadthread.cpp index 3f1f58360..df424769d 100644 --- a/writeengine/splitter/we_respreadthread.cpp +++ b/writeengine/splitter/we_respreadthread.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_respreadthread.cpp @@ -37,27 +37,23 @@ using namespace boost; namespace WriteEngine { - -WERespReadThread::WERespReadThread(WESDHandler& aSdh): - fSdh(aSdh) +WERespReadThread::WERespReadThread(WESDHandler& aSdh) : fSdh(aSdh) { - // ctor + // ctor } -WERespReadThread::WERespReadThread(const WERespReadThread& rhs): - fSdh(rhs.fSdh) +WERespReadThread::WERespReadThread(const WERespReadThread& rhs) : fSdh(rhs.fSdh) { - // copy ctor + // copy ctor } WERespReadThread::~WERespReadThread() { - // dtor + // dtor } void WERespReadThread::operator()() { - //call datahandler checkForAllPmMsgs() - fSdh.checkForRespMsgs(); + // call datahandler checkForAllPmMsgs() + fSdh.checkForRespMsgs(); } - } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_respreadthread.h b/writeengine/splitter/we_respreadthread.h index b14c92b37..eed130345 100644 --- a/writeengine/splitter/we_respreadthread.h +++ b/writeengine/splitter/we_respreadthread.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_respreadthread.h @@ -32,19 +32,17 @@ namespace WriteEngine { - class WERespReadThread { -public: - WERespReadThread(WESDHandler& aSdh); - WERespReadThread(const WERespReadThread& rhs); - virtual ~WERespReadThread(); + public: + WERespReadThread(WESDHandler& aSdh); + WERespReadThread(const WERespReadThread& rhs); + virtual ~WERespReadThread(); - void operator()(); - -private: - WESDHandler& fSdh; + void operator()(); + private: + WESDHandler& fSdh; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 31f6c73d2..b076c91c2 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -82,33 +82,33 @@ namespace WriteEngine void WEPmList::addPm2List(int PmId) { - boost::mutex::scoped_lock aLock(fListMutex); - fPmList.push_back(PmId); - aLock.unlock(); + boost::mutex::scoped_lock aLock(fListMutex); + fPmList.push_back(PmId); + aLock.unlock(); } //------------------------------------------------------------------------------ void WEPmList::addPriorityPm2List(int PmId) { - boost::mutex::scoped_lock aLock(fListMutex); - fPmList.push_front(PmId); - aLock.unlock(); + boost::mutex::scoped_lock aLock(fListMutex); + fPmList.push_front(PmId); + aLock.unlock(); } //------------------------------------------------------------------------------ int WEPmList::getNextPm() { - boost::mutex::scoped_lock aLock(fListMutex); - int aPmId = 0; + boost::mutex::scoped_lock aLock(fListMutex); + int aPmId = 0; - if (!fPmList.empty()) - { - aPmId = fPmList.front(); - fPmList.pop_front(); - } + if (!fPmList.empty()) + { + aPmId = fPmList.front(); + fPmList.pop_front(); + } - return aPmId; + return aPmId; } //------------------------------------------------------------------------------ @@ -116,35 +116,35 @@ int WEPmList::getNextPm() // so that the sendingthreads will not keep sending data. void WEPmList::clearPmList() { - boost::mutex::scoped_lock aLock(fListMutex); + boost::mutex::scoped_lock aLock(fListMutex); - if (!fPmList.empty()) - fPmList.clear(); + if (!fPmList.empty()) + fPmList.clear(); - aLock.unlock(); + aLock.unlock(); } //------------------------------------------------------------------------------ bool WEPmList::check4Pm(int PmId) { - boost::mutex::scoped_lock aLock(fListMutex); - WePmList::iterator aIt = fPmList.begin(); - bool aFound = false; + boost::mutex::scoped_lock aLock(fListMutex); + WePmList::iterator aIt = fPmList.begin(); + bool aFound = false; - while (aIt != fPmList.end()) + while (aIt != fPmList.end()) + { + if ((*aIt) == PmId) { - if ((*aIt) == PmId) - { - aFound = true; - fPmList.erase(aIt); - break; - } - - ++aIt; + aFound = true; + fPmList.erase(aIt); + break; } - return aFound; + ++aIt; + } + + return aFound; } //------------------------------------------------------------------------------ @@ -153,144 +153,140 @@ bool WEPmList::check4Pm(int PmId) // //------------------------------------------------------------------------------ -WESDHandler::WESDHandler(WESplitterApp& Ref) : - fRef(Ref), - fLog(), - fQId(101), // 101 - took it from air - fOam(), - fModuleTypeConfig(), - fDebugLvl(0), - fPmCount(0), - fTableLock(0), - fTableOId(0), - fFixedBinaryRecLen(0), - fRespMutex(), - fRespCond(), - fSendMutex(), - fRespList(), - fpRespThread(0), - fDataFeedList(), - fFileReadThread(*this), - fDisconnectFailure(false), - fForcedFailure(false), - fAllCpiStarted(false), - fFirstDataSent(false), - fFirstPmToSend(0), - fSelectOtherPm(false), - fContinue(true), - fWeSplClients(MAX_PMS), - fBrmRptVec(), - fpBatchLoader(0) +WESDHandler::WESDHandler(WESplitterApp& Ref) + : fRef(Ref) + , fLog() + , fQId(101) + , // 101 - took it from air + fOam() + , fModuleTypeConfig() + , fDebugLvl(0) + , fPmCount(0) + , fTableLock(0) + , fTableOId(0) + , fFixedBinaryRecLen(0) + , fRespMutex() + , fRespCond() + , fSendMutex() + , fRespList() + , fpRespThread(0) + , fDataFeedList() + , fFileReadThread(*this) + , fDisconnectFailure(false) + , fForcedFailure(false) + , fAllCpiStarted(false) + , fFirstDataSent(false) + , fFirstPmToSend(0) + , fSelectOtherPm(false) + , fContinue(true) + , fWeSplClients(MAX_PMS) + , fBrmRptVec() + , fpBatchLoader(0) { - fRm = joblist::ResourceManager::instance(); + fRm = joblist::ResourceManager::instance(); } //------------------------------------------------------------------------------ WESDHandler::~WESDHandler() { - try + try + { + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) { - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) - { - if (fWeSplClients[aCnt] != 0) - { - delete fWeSplClients[aCnt]; - fWeSplClients[aCnt] = 0; - } - } - - delete fpRespThread; - fpRespThread = 0; - delete fpBatchLoader; - fpBatchLoader = 0; - } - catch (...) - { - std::string aStr = "Handled an error in ~WESDHandler"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, - logging::M0000); - cout << aStr << endl; + if (fWeSplClients[aCnt] != 0) + { + delete fWeSplClients[aCnt]; + fWeSplClients[aCnt] = 0; + } } + delete fpRespThread; + fpRespThread = 0; + delete fpBatchLoader; + fpBatchLoader = 0; + } + catch (...) + { + std::string aStr = "Handled an error in ~WESDHandler"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + cout << aStr << endl; + } } //------------------------------------------------------------------------------ void WESDHandler::reset() { - fTableLock = 0; - fTableOId = 0; - fFixedBinaryRecLen = 0; - //fpRespThread = 0; - //fFileReadThread(*this); - fForcedFailure = false; - fAllCpiStarted = false; - fFirstDataSent = false; - fFirstPmToSend = 0; - fSelectOtherPm = false; - fContinue = true; - //fWeSplClients(); - //fpBatchLoader = 0; - fImportRslt.reset(); - fLog.closeLog(); + fTableLock = 0; + fTableOId = 0; + fFixedBinaryRecLen = 0; + // fpRespThread = 0; + // fFileReadThread(*this); + fForcedFailure = false; + fAllCpiStarted = false; + fFirstDataSent = false; + fFirstPmToSend = 0; + fSelectOtherPm = false; + fContinue = true; + // fWeSplClients(); + // fpBatchLoader = 0; + fImportRslt.reset(); + fLog.closeLog(); - try + try + { + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) { - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) - { - if (fWeSplClients[aCnt] != 0) - { - delete fWeSplClients[aCnt]; - fWeSplClients[aCnt] = 0; - } - } - - delete fpRespThread; - fpRespThread = 0; - delete fpBatchLoader; - fpBatchLoader = 0; - } - catch (...) - { - std::string aStr = "Handled an error in ~WESDHandler"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, - logging::M0000); - cout << aStr << endl; + if (fWeSplClients[aCnt] != 0) + { + delete fWeSplClients[aCnt]; + fWeSplClients[aCnt] = 0; + } } - + delete fpRespThread; + fpRespThread = 0; + delete fpBatchLoader; + fpBatchLoader = 0; + } + catch (...) + { + std::string aStr = "Handled an error in ~WESDHandler"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + cout << aStr << endl; + } } //------------------------------------------------------------------------------ -//BP 10/24/2011 14:22 +// BP 10/24/2011 14:22 //------------------------------------------------------------------------------ void WESDHandler::send2Pm(ByteStream& Bs, unsigned int PmId) { - //mutex::scoped_lock aLock(fSendMutex); + // mutex::scoped_lock aLock(fSendMutex); - if (PmId == 0) // send it to everyone + if (PmId == 0) // send it to everyone + { + for (int aIdx = 1; aIdx <= getPmCount(); ++aIdx) { - for (int aIdx = 1; aIdx <= getPmCount(); ++aIdx) - { - if (fWeSplClients[aIdx] != 0) - { - boost::mutex::scoped_lock aLock(fWeSplClients[aIdx]->fWriteMutex); - fWeSplClients[aIdx]->write(Bs); - aLock.unlock(); - } - } - } - else - { - boost::mutex::scoped_lock aLock(fWeSplClients[PmId]->fWriteMutex); - fWeSplClients[PmId]->write(Bs); + if (fWeSplClients[aIdx] != 0) + { + boost::mutex::scoped_lock aLock(fWeSplClients[aIdx]->fWriteMutex); + fWeSplClients[aIdx]->write(Bs); aLock.unlock(); + } } + } + else + { + boost::mutex::scoped_lock aLock(fWeSplClients[PmId]->fWriteMutex); + fWeSplClients[PmId]->write(Bs); + aLock.unlock(); + } - //aLock.unlock(); + // aLock.unlock(); } //------------------------------------------------------------------------------ @@ -298,31 +294,28 @@ void WESDHandler::send2Pm(ByteStream& Bs, unsigned int PmId) //------------------------------------------------------------------------------ void WESDHandler::send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId) { + // mutex::scoped_lock aLock(fSendMutex); - //mutex::scoped_lock aLock(fSendMutex); - - if (PmId == 0) // send it to everyone + if (PmId == 0) // send it to everyone + { + for (int aIdx = 1; aIdx <= getPmCount(); ++aIdx) { - for (int aIdx = 1; aIdx <= getPmCount(); ++aIdx) - { - if (fWeSplClients[aIdx] != 0) - { - boost::mutex::scoped_lock aLock(fWeSplClients[aIdx]->fSentQMutex); - fWeSplClients[aIdx]->add2SendQueue(Sbs); - aLock.unlock(); - - } - } - } - else - { - boost::mutex::scoped_lock aLock(fWeSplClients[PmId]->fSentQMutex); - fWeSplClients[PmId]->add2SendQueue(Sbs); + if (fWeSplClients[aIdx] != 0) + { + boost::mutex::scoped_lock aLock(fWeSplClients[aIdx]->fSentQMutex); + fWeSplClients[aIdx]->add2SendQueue(Sbs); aLock.unlock(); + } } + } + else + { + boost::mutex::scoped_lock aLock(fWeSplClients[PmId]->fSentQMutex); + fWeSplClients[PmId]->add2SendQueue(Sbs); + aLock.unlock(); + } - //aLock.unlock(); - + // aLock.unlock(); } //------------------------------------------------------------------------------ @@ -330,21 +323,21 @@ void WESDHandler::send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId) void WESDHandler::sendEODMsg() { - // BUG 5035 Sending multiple EOD so that to avoid 'silly window syndrome' - for (int idx = 0; idx < 3; idx++) - { - messageqcpp::SBS aSbs(new messageqcpp::ByteStream); - *aSbs << (ByteStream::byte) WE_CLT_SRV_EOD; - send2Pm(aSbs); - } + // BUG 5035 Sending multiple EOD so that to avoid 'silly window syndrome' + for (int idx = 0; idx < 3; idx++) + { + messageqcpp::SBS aSbs(new messageqcpp::ByteStream); + *aSbs << (ByteStream::byte)WE_CLT_SRV_EOD; + send2Pm(aSbs); + } - { - std::stringstream aStrStr; - aStrStr << "Send EOD message to All PMs"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - } + { + std::stringstream aStrStr; + aStrStr << "Send EOD message to All PMs"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + } } //------------------------------------------------------------------------------ @@ -352,751 +345,710 @@ void WESDHandler::sendEODMsg() //------------------------------------------------------------------------------ void WESDHandler::checkForRespMsgs() { - ByteStream aBs; - ByteStream::byte aPmId; - ByteStream::byte aMsgId; - //boost::shared_ptr aSbs; - messageqcpp::SBS aSbs; + ByteStream aBs; + ByteStream::byte aPmId; + ByteStream::byte aMsgId; + // boost::shared_ptr aSbs; + messageqcpp::SBS aSbs; - while (isContinue()) + while (isContinue()) + { + boost::mutex::scoped_lock aLock(fRespMutex); + + // NOTE - if isContinue is not checked thread will hang on shutdown + // by locking again on fRespList.empty() + while ((fRespList.empty()) && (isContinue())) + fRespCond.wait(aLock); + + // if(!isContinue()) { aLock.unlock(); break; } testing for rare hanging + + // cout <<"wait signaled checkForRespMsgs" << endl; + while (!fRespList.empty()) { - boost::mutex::scoped_lock aLock(fRespMutex); + // mutex::scoped_lock aLock (fRespMutex); + aSbs = fRespList.front(); + fRespList.pop_front(); + // aLock.unlock(); + *aSbs >> aMsgId; + *aSbs >> aPmId; - //NOTE - if isContinue is not checked thread will hang on shutdown - // by locking again on fRespList.empty() - while ((fRespList.empty()) && (isContinue())) - fRespCond.wait(aLock); + // Debugging + // cout << "aMsgid = " << static_cast(aMsgId) << endl; + switch (aMsgId) + { + case WE_CLT_SRV_KEEPALIVE: onKeepAliveMessage(static_cast(aPmId)); break; - //if(!isContinue()) { aLock.unlock(); break; } testing for rare hanging + case WE_CLT_SRV_ACK: onAckResponse(static_cast(aPmId)); break; - //cout <<"wait signaled checkForRespMsgs" << endl; - while (!fRespList.empty()) - { - //mutex::scoped_lock aLock (fRespMutex); - aSbs = fRespList.front(); - fRespList.pop_front(); - //aLock.unlock(); - *aSbs >> aMsgId; - *aSbs >> aPmId; + case WE_CLT_SRV_DATARQST: onDataRqstResponse(static_cast(aPmId)); break; - //Debugging - //cout << "aMsgid = " << static_cast(aMsgId) << endl; - switch (aMsgId) - { - case WE_CLT_SRV_KEEPALIVE: - onKeepAliveMessage(static_cast(aPmId)); - break; + case WE_CLT_SRV_EOD: onEodResponse(static_cast(aPmId)); break; - case WE_CLT_SRV_ACK: - onAckResponse(static_cast(aPmId)); - break; + case WE_CLT_SRV_STARTCPI: onStartCpiResponse(static_cast(aPmId)); break; - case WE_CLT_SRV_DATARQST: - onDataRqstResponse(static_cast(aPmId)); - break; + case WE_CLT_SRV_CPIPASS: onCpimportPass(static_cast(aPmId)); break; - case WE_CLT_SRV_EOD: - onEodResponse(static_cast(aPmId)); - break; + case WE_CLT_SRV_CPIFAIL: onCpimportFail(static_cast(aPmId)); break; - case WE_CLT_SRV_STARTCPI: - onStartCpiResponse(static_cast(aPmId)); - break; + case WE_CLT_SRV_BRMRPT: onBrmReport(static_cast(aPmId), aSbs); break; - case WE_CLT_SRV_CPIPASS: - onCpimportPass(static_cast(aPmId)); - break; + case WE_CLT_SRV_ROLLBACK: onRollbackResult(static_cast(aPmId), aSbs); break; - case WE_CLT_SRV_CPIFAIL: - onCpimportFail(static_cast(aPmId)); - break; + case WE_CLT_SRV_CLEANUP: onCleanupResult(static_cast(aPmId), aSbs); break; - case WE_CLT_SRV_BRMRPT: - onBrmReport(static_cast(aPmId), aSbs); - break; + case WE_CLT_SRV_DBRCNT: onDBRootCount(static_cast(aPmId), aSbs); break; - case WE_CLT_SRV_ROLLBACK: - onRollbackResult(static_cast(aPmId), aSbs); - break; + case WE_CLT_SRV_ERRLOG: onErrorFile(static_cast(aPmId), aSbs); break; - case WE_CLT_SRV_CLEANUP: - onCleanupResult(static_cast(aPmId), aSbs); - break; + case WE_CLT_SRV_BADLOG: onBadFile(static_cast(aPmId), aSbs); break; - case WE_CLT_SRV_DBRCNT: - onDBRootCount(static_cast(aPmId), aSbs); - break; + case WE_CLT_SRV_IMPFILEERROR: onImpFileError(static_cast(aPmId)); break; - case WE_CLT_SRV_ERRLOG: - onErrorFile(static_cast(aPmId), aSbs); - break; + default: break; + } // switch - case WE_CLT_SRV_BADLOG: - onBadFile(static_cast(aPmId), aSbs); - break; - - case WE_CLT_SRV_IMPFILEERROR: - onImpFileError(static_cast(aPmId)); - break; - - default: - break; - } // switch - - aSbs.reset(); - } // while not empty() - - aLock.unlock(); - // yield here so that other threads get slice - } //while + aSbs.reset(); + } // while not empty() + aLock.unlock(); + // yield here so that other threads get slice + } // while } //------------------------------------------------------------------------------ void WESDHandler::add2RespQueue(const messageqcpp::SBS& Sbs) { - boost::mutex::scoped_lock aLock(fRespMutex); - fRespList.push_back(Sbs); - aLock.unlock(); - //cout <<"Notifing from add2RespQueue" << endl; - fRespCond.notify_one(); + boost::mutex::scoped_lock aLock(fRespMutex); + fRespList.push_back(Sbs); + aLock.unlock(); + // cout <<"Notifing from add2RespQueue" << endl; + fRespCond.notify_one(); } //------------------------------------------------------------------------------ void WESDHandler::setup() { - std::stringstream aPid; - bool bRollback; - bool bForce; - aPid << getpid(); - std::string aTimeStamp = getTime2Str(); - std::string aLogName; - std::string aErrLogName; + std::stringstream aPid; + bool bRollback; + bool bForce; + aPid << getpid(); + std::string aTimeStamp = getTime2Str(); + std::string aLogName; + std::string aErrLogName; - if (fRef.fCmdArgs.isJobLogOnly()) - aLogName = std::string(MCSLOGDIR) + "/cpimport/" + - "cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".log"; - else - aLogName = std::string(MCSLOGDIR) + "/cpimport/" + - "cpimport_" + aTimeStamp + "_" + aPid.str() + ".log"; + if (fRef.fCmdArgs.isJobLogOnly()) + aLogName = std::string(MCSLOGDIR) + "/cpimport/" + "cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".log"; + else + aLogName = std::string(MCSLOGDIR) + "/cpimport/" + "cpimport_" + aTimeStamp + "_" + aPid.str() + ".log"; + if (getDebugLvl() > 1) + cout << "LogName : " << aLogName << endl; - if (getDebugLvl() > 1) cout << "LogName : " << aLogName << endl; + if (fRef.fCmdArgs.isJobLogOnly()) + aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" + "cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".err"; + else + aErrLogName = + std::string(MCSLOGDIR) + "/cpimport/" + "cpimport_" + aTimeStamp + "_" + aPid.str() + ".err"; - if (fRef.fCmdArgs.isJobLogOnly()) - aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" + - "cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".err"; - else - aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" + - "cpimport_" + aTimeStamp + "_" + aPid.str() + ".err"; + if (getDebugLvl() > 1) + cout << "ErrLogName : " << aErrLogName << endl; + // consoleFlag false will only output only MSGLOG_LVL1 to console + // and MSGLOG_LVL2 to log file without writing to console. + // fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), false); + fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), getConsoleLog()); - if (getDebugLvl() > 1) cout << "ErrLogName : " << aErrLogName << endl; + // In mode 0 and Mode 1, we need to construct the input file list and check availability + if (0 == fRef.fCmdArgs.getMode() || 1 == fRef.fCmdArgs.getMode()) + setInputFileList(fRef.getLocFile()); - // consoleFlag false will only output only MSGLOG_LVL1 to console - // and MSGLOG_LVL2 to log file without writing to console. - //fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), false); - fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), getConsoleLog()); + fImportRslt.startTimer(); - // In mode 0 and Mode 1, we need to construct the input file list and check availability - if (0 == fRef.fCmdArgs.getMode() || 1 == fRef.fCmdArgs.getMode()) - setInputFileList(fRef.getLocFile()); + if (fPmCount == 0) // Should have already set in cpimport invoke check + { + throw(runtime_error("Configuration Error. PM Count = 0")); + } - fImportRslt.startTimer(); + if (fRef.fCmdArgs.getPmVecSize() == 0) // No Pms listed in Cmd line + { + // BUG 4668 - Added this code to find the PM's realtime from OAM + // fOam.getSystemConfig("pm", fModuleTypeConfig); //commented out since- + // - we are calling that function in check4CpiInvokeMode() + oam::DeviceNetworkList::iterator pt = fModuleTypeConfig.ModuleNetworkList.begin(); - if (fPmCount == 0) // Should have already set in cpimport invoke check + for (; pt != fModuleTypeConfig.ModuleNetworkList.end(); pt++) { - throw (runtime_error("Configuration Error. PM Count = 0")); + int moduleID = + atoi((*pt).DeviceName.substr(oam::MAX_MODULE_TYPE_SIZE, oam::MAX_MODULE_ID_SIZE).c_str()); + + if (getDebugLvl() > 1) + cout << "Adding PmId - " << moduleID << endl; + + fRef.fCmdArgs.add2PmVec(moduleID); } - if (fRef.fCmdArgs.getPmVecSize() == 0) //No Pms listed in Cmd line - { - //BUG 4668 - Added this code to find the PM's realtime from OAM - //fOam.getSystemConfig("pm", fModuleTypeConfig); //commented out since- - // - we are calling that function in check4CpiInvokeMode() - oam::DeviceNetworkList::iterator pt = - fModuleTypeConfig.ModuleNetworkList.begin(); + /* + for (int PmId = 1; ((PmId <= fPmCount) && (PmId < MAX_PMS)); ++PmId) + { + if(getDebugLvl()>1) cout<<"Adding PmId - "<& aVec = fRef.fCmdArgs.getPmVec(); - if (getDebugLvl() > 1) - cout << "Adding PmId - " << moduleID << endl; + for (unsigned int PmId = 1; (PmId <= static_cast(fPmCount)); ++PmId) + { + int opState = oam::ACTIVE; + bool aDegraded = false; + ostringstream aOss; + aOss << "pm" << PmId; + std::string aModName = aOss.str(); - fRef.fCmdArgs.add2PmVec(moduleID); - } + if (getDebugLvl()) + cout << "getModuleStatus() ModName = " << aModName << endl; - /* - for (int PmId = 1; ((PmId <= fPmCount) && (PmId < MAX_PMS)); ++PmId) - { - if(getDebugLvl()>1) cout<<"Adding PmId - "<& aVec = fRef.fCmdArgs.getPmVec(); + if (getDebugLvl()) + cout << "ModName = " << aModName << " opState = " << opState << endl; + } + catch (std::exception& ex) + {} - for (unsigned int PmId = 1; (PmId <= static_cast(fPmCount)); ++PmId) - { - int opState = oam::ACTIVE; - bool aDegraded = false; - ostringstream aOss; - aOss << "pm" << PmId; - std::string aModName = aOss.str(); + if (opState != oam::ACTIVE ) // BUG 4668 + { + aVec.erase(std::remove(aVec.begin(), aVec.end(), PmId), aVec.end()); + } + } + } + */ - if (getDebugLvl()) - cout << "getModuleStatus() ModName = " << aModName << endl; + int rtn = fDbrm.getSystemReady(); - try - { - fOam.getModuleStatus(aModName, opState, aDegraded); + if (rtn < 1) + { + ostringstream oss; + oss << "System is not ready (" << rtn << "). Verify that Columnstore is up and ready "; + // fLog.logMsg( oss.str(), MSGLVL_ERROR ); + setContinue(false); + throw runtime_error(oss.str()); + } + else if (BRM::ERR_OK != fDbrm.isReadWrite()) + { + ostringstream oss; + oss << "Error: System in ReadOnly state."; + // fLog.logMsg(oss.str(), MSGLVL_ERROR); + setContinue(false); + throw runtime_error(oss.str()); + } + else if (fDbrm.getSystemShutdownPending(bRollback, bForce) > 0) + { + ostringstream oss; + oss << "System is being shutdown. Can't start a new import"; + setContinue(false); + throw runtime_error(oss.str()); + } + else if (fDbrm.getSystemSuspendPending(bRollback) > 0 || fDbrm.getSystemSuspended()) + { + ostringstream oss; + oss << "System is in write disabled state. Can't start a new import"; + setContinue(false); + throw runtime_error(oss.str()); + } - if (getDebugLvl()) - cout << "ModName = " << aModName << " opState = " << opState << endl; - } - catch (std::exception& ex) - {} - - if (opState != oam::ACTIVE ) // BUG 4668 - { - aVec.erase(std::remove(aVec.begin(), aVec.end(), PmId), aVec.end()); - } - } - } -*/ - - int rtn = fDbrm.getSystemReady(); - - if (rtn < 1) - { - ostringstream oss; - oss << "System is not ready (" << rtn << "). Verify that Columnstore is up and ready "; - //fLog.logMsg( oss.str(), MSGLVL_ERROR ); - setContinue(false); - throw runtime_error( oss.str() ); - } - else if (BRM::ERR_OK != fDbrm.isReadWrite()) - { - ostringstream oss; - oss << "Error: System in ReadOnly state."; - //fLog.logMsg(oss.str(), MSGLVL_ERROR); - setContinue(false); - throw runtime_error(oss.str()); - } - else if (fDbrm.getSystemShutdownPending(bRollback, bForce) > 0) - { - ostringstream oss; - oss << "System is being shutdown. Can't start a new import"; - setContinue(false); - throw runtime_error(oss.str()); - } - else if (fDbrm.getSystemSuspendPending(bRollback) > 0 || fDbrm.getSystemSuspended()) - { - ostringstream oss; - oss << "System is in write disabled state. Can't start a new import"; - setContinue(false); - throw runtime_error(oss.str()); - } - - if ((fRef.fCmdArgs.getMode() == 1) || (fRef.fCmdArgs.getMode() == 2)) - { - fTableOId = 0; - fFixedBinaryRecLen = 0; - - try - { - int32_t tblOid = getTableOID(fRef.fCmdArgs.getSchemaName(), - fRef.fCmdArgs.getTableName()); - fTableOId = tblOid; - - if (getDebugLvl()) - cout << "Table OID = " << fTableOId << endl; - - if (fRef.fCmdArgs.getImportDataMode() != IMPORT_DATA_TEXT) - { - fFixedBinaryRecLen = calcTableRecLen( - fRef.fCmdArgs.getSchemaName(), - fRef.fCmdArgs.getTableName()); - } - } - catch (std::exception& ex) - { - std::string aDetails = fRef.fCmdArgs.getSchemaName() + "." - + fRef.fCmdArgs.getTableName() + " ERROR : "; - std::string aStr = aDetails + ex.what(); - logging::Message::Args errMsgArgs; - //BUG 4152 - errMsgArgs.add(fRef.fCmdArgs.getSchemaName()); - errMsgArgs.add(fRef.fCmdArgs.getTableName()); - errMsgArgs.add(ex.what()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0097); - fLog.logMsg( aStr, MSGLVL_ERROR ); - setContinue(false); - - if (fTableOId == 0) // error getting table OID - throw runtime_error( - "Please make sure both schema and table exists!!"); - else // error getting fixed binary record length - throw runtime_error( ex.what() ); - } - - int aWaitIntvl = 10; // In seconds - - try - { - string aWaitPeriod = config::Config::makeConfig()->getConfig( - "SystemConfig", - "WaitPeriod"); - - if (!aWaitPeriod.empty()) aWaitIntvl = atoi(aWaitPeriod.c_str()); - - if (getDebugLvl()) cout << "aWaitPeriod = " << aWaitPeriod << endl; - } - catch (std::exception&) - { - aWaitIntvl = 10; - } - - std::vector aPmVec = fRef.fCmdArgs.getPmVec(); - WETableLockGrabber aTLG(*this); - string errMsg; - - for (int aIdx = 0; aIdx < (aWaitIntvl * 10); aIdx++) - { - try - { - fTableLock = aTLG.grabTableLock(aPmVec, fTableOId); - - if (getDebugLvl() > 1) - cout << "Table Lock = " << fTableLock << endl; - } - catch (std::exception& ex) - { - errMsg = ex.what(); - } - - if (fTableLock != 0) break; - - usleep(100000); - } - - if (fTableLock == 0) - { - ostringstream oss; - oss << "Failed to acquire Table Lock of "; - oss << fRef.fCmdArgs.getSchemaName() << "."; - oss << fRef.fCmdArgs.getTableName() << "; " << errMsg; - //fLog.logMsg( oss.str(), MSGLVL_ERROR ); - setContinue(false); - throw runtime_error(oss.str()); - } - - if (0 != fTableOId) - { - try - { - if (getDebugLvl() > 1) - { - for (unsigned int idx = 0; idx < aPmVec.size(); idx++) - cout << "PmId = " << aPmVec[idx] << std::endl; - } - - fpBatchLoader = new BatchLoader(fTableOId, 0, aPmVec); - //int aRet=fpBatchLoader->selectFirstPM(fFirstPmToSend, fSelectOtherPm); - //if (aRet != 0) throw runtime_error("BatchLoader error.. exiting"); - } - catch (std::exception& ex) - { - releaseTableLocks(); - ostringstream oss; - oss << ex.what() << " ... import exiting"; - //fLog.logMsg( oss.str(), MSGLVL_ERROR ); - setContinue(false); - throw runtime_error(oss.str()); - } - } - } - - for (int PmId = 1; ((PmId <= fPmCount) && (PmId < MAX_PMS)); ++PmId) - { - try - { - if (fRef.getPmStatus(PmId)) - { - fWeSplClients.at(PmId) = new WESplClient(*this, PmId); - - if (fWeSplClients[PmId] != NULL) - { - fWeSplClients[PmId]->setup(); - - if (2 == fRef.fCmdArgs.getMode()) - fWeSplClients[PmId]->setRdSecTo(fPmCount); //Set Rd T/O to 1 sec - } - else - { - std::string aStr; - aStr = std::string("Encountered NULL WESplClient : ") + std::to_string(PmId); - cout << aStr << endl; - fLog.logMsg( aStr, MSGLVL_ERROR ); - throw WESdHandlerException(aStr); - } - } - - } - catch (const std::exception& ex) - { - std::string aStr = ex.what(); - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - releaseTableLocks(); //BUG 4295 - release table lock as connection fails - //cout << aStr << endl; - fLog.logMsg( aStr, MSGLVL_ERROR ); - throw runtime_error("Error in connection setup."); - } - } - - // to initiate adding data to the SendQueue, that many BS - // BUG 5031 - Initial Data request count set to 100. This will be the max - // Q size on the WES side. Here after a batch send for every data rqst - for (int aIdx = 0; aIdx < MAX_WES_QSIZE; aIdx++) - { - for (int PmId = 1; PmId <= fPmCount; PmId++) - { - if (fWeSplClients[PmId] != 0) - fWeSplClients[PmId]->incDataRqstCount(); - - //fDataFeedList.addPm2List(PmId); - } - } - - fpRespThread = new boost::thread(WERespReadThread(*this)); + if ((fRef.fCmdArgs.getMode() == 1) || (fRef.fCmdArgs.getMode() == 2)) + { + fTableOId = 0; + fFixedBinaryRecLen = 0; try { - // start the File Read thread - if ((fRef.fCmdArgs.getMode() == 0) || (fRef.fCmdArgs.getMode() == 1)) - { - if (fRef.getLocFile() == "STDIN") - { - ostringstream oss; - oss << "Reading input from STDIN to import into table "; - oss << fRef.fCmdArgs.getSchemaName() << "."; - oss << fRef.fCmdArgs.getTableName() << "..."; + int32_t tblOid = getTableOID(fRef.fCmdArgs.getSchemaName(), fRef.fCmdArgs.getTableName()); + fTableOId = tblOid; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - } + if (getDebugLvl()) + cout << "Table OID = " << fTableOId << endl; - if (getDebugLvl()) - cout << "BatchQuantity=" << fRef.fCmdArgs.getBatchQuantity() << endl; - - fFileReadThread.setBatchQty(fRef.fCmdArgs.getBatchQuantity()); - fFileReadThread.setup(fRef.getLocFile()); - } + if (fRef.fCmdArgs.getImportDataMode() != IMPORT_DATA_TEXT) + { + fFixedBinaryRecLen = calcTableRecLen(fRef.fCmdArgs.getSchemaName(), fRef.fCmdArgs.getTableName()); + } } catch (std::exception& ex) { - releaseTableLocks(); //BUG 4295 + std::string aDetails = fRef.fCmdArgs.getSchemaName() + "." + fRef.fCmdArgs.getTableName() + " ERROR : "; + std::string aStr = aDetails + ex.what(); + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(fRef.fCmdArgs.getSchemaName()); + errMsgArgs.add(fRef.fCmdArgs.getTableName()); + errMsgArgs.add(ex.what()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0097); + fLog.logMsg(aStr, MSGLVL_ERROR); + setContinue(false); + + if (fTableOId == 0) // error getting table OID + throw runtime_error("Please make sure both schema and table exists!!"); + else // error getting fixed binary record length throw runtime_error(ex.what()); } - //Output "Running distributed import (Mode{x}) on [all/] PMs" + int aWaitIntvl = 10; // In seconds + + try { - ostringstream oss; - oss << "Running distributed import (mode "; - oss << fRef.fCmdArgs.getMode() << ") on "; + string aWaitPeriod = config::Config::makeConfig()->getConfig("SystemConfig", "WaitPeriod"); - if (fRef.fCmdArgs.getPmVecSize() == fPmCount) - oss << "all PMs..."; - else - { - oss << " PM "; - std::vector aPmVec = fRef.fCmdArgs.getPmVec(); - unsigned int aIdx = 0; + if (!aWaitPeriod.empty()) + aWaitIntvl = atoi(aWaitPeriod.c_str()); - while (aIdx < aPmVec.size()) - { - oss << aPmVec[aIdx]; - aIdx++; - - if (aIdx != aPmVec.size()) oss << ","; - } - - oss << " ..."; - } - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); + if (getDebugLvl()) + cout << "aWaitPeriod = " << aWaitPeriod << endl; + } + catch (std::exception&) + { + aWaitIntvl = 10; } + std::vector aPmVec = fRef.fCmdArgs.getPmVec(); + WETableLockGrabber aTLG(*this); + string errMsg; + for (int aIdx = 0; aIdx < (aWaitIntvl * 10); aIdx++) + { + try + { + fTableLock = aTLG.grabTableLock(aPmVec, fTableOId); + if (getDebugLvl() > 1) + cout << "Table Lock = " << fTableLock << endl; + } + catch (std::exception& ex) + { + errMsg = ex.what(); + } + + if (fTableLock != 0) + break; + + usleep(100000); + } + + if (fTableLock == 0) + { + ostringstream oss; + oss << "Failed to acquire Table Lock of "; + oss << fRef.fCmdArgs.getSchemaName() << "."; + oss << fRef.fCmdArgs.getTableName() << "; " << errMsg; + // fLog.logMsg( oss.str(), MSGLVL_ERROR ); + setContinue(false); + throw runtime_error(oss.str()); + } + + if (0 != fTableOId) + { + try + { + if (getDebugLvl() > 1) + { + for (unsigned int idx = 0; idx < aPmVec.size(); idx++) + cout << "PmId = " << aPmVec[idx] << std::endl; + } + + fpBatchLoader = new BatchLoader(fTableOId, 0, aPmVec); + // int aRet=fpBatchLoader->selectFirstPM(fFirstPmToSend, fSelectOtherPm); + // if (aRet != 0) throw runtime_error("BatchLoader error.. exiting"); + } + catch (std::exception& ex) + { + releaseTableLocks(); + ostringstream oss; + oss << ex.what() << " ... import exiting"; + // fLog.logMsg( oss.str(), MSGLVL_ERROR ); + setContinue(false); + throw runtime_error(oss.str()); + } + } + } + + for (int PmId = 1; ((PmId <= fPmCount) && (PmId < MAX_PMS)); ++PmId) + { + try + { + if (fRef.getPmStatus(PmId)) + { + fWeSplClients.at(PmId) = new WESplClient(*this, PmId); + + if (fWeSplClients[PmId] != NULL) + { + fWeSplClients[PmId]->setup(); + + if (2 == fRef.fCmdArgs.getMode()) + fWeSplClients[PmId]->setRdSecTo(fPmCount); // Set Rd T/O to 1 sec + } + else + { + std::string aStr; + aStr = std::string("Encountered NULL WESplClient : ") + std::to_string(PmId); + cout << aStr << endl; + fLog.logMsg(aStr, MSGLVL_ERROR); + throw WESdHandlerException(aStr); + } + } + } + catch (const std::exception& ex) + { + std::string aStr = ex.what(); + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + releaseTableLocks(); // BUG 4295 - release table lock as connection fails + // cout << aStr << endl; + fLog.logMsg(aStr, MSGLVL_ERROR); + throw runtime_error("Error in connection setup."); + } + } + + // to initiate adding data to the SendQueue, that many BS + // BUG 5031 - Initial Data request count set to 100. This will be the max + // Q size on the WES side. Here after a batch send for every data rqst + for (int aIdx = 0; aIdx < MAX_WES_QSIZE; aIdx++) + { + for (int PmId = 1; PmId <= fPmCount; PmId++) + { + if (fWeSplClients[PmId] != 0) + fWeSplClients[PmId]->incDataRqstCount(); + + // fDataFeedList.addPm2List(PmId); + } + } + + fpRespThread = new boost::thread(WERespReadThread(*this)); + + try + { + // start the File Read thread + if ((fRef.fCmdArgs.getMode() == 0) || (fRef.fCmdArgs.getMode() == 1)) + { + if (fRef.getLocFile() == "STDIN") + { + ostringstream oss; + oss << "Reading input from STDIN to import into table "; + oss << fRef.fCmdArgs.getSchemaName() << "."; + oss << fRef.fCmdArgs.getTableName() << "..."; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss.str(), MSGLVL_INFO1); + } + + if (getDebugLvl()) + cout << "BatchQuantity=" << fRef.fCmdArgs.getBatchQuantity() << endl; + + fFileReadThread.setBatchQty(fRef.fCmdArgs.getBatchQuantity()); + fFileReadThread.setup(fRef.getLocFile()); + } + } + catch (std::exception& ex) + { + releaseTableLocks(); // BUG 4295 + throw runtime_error(ex.what()); + } + + // Output "Running distributed import (Mode{x}) on [all/] PMs" + { + ostringstream oss; + oss << "Running distributed import (mode "; + oss << fRef.fCmdArgs.getMode() << ") on "; + + if (fRef.fCmdArgs.getPmVecSize() == fPmCount) + oss << "all PMs..."; + else + { + oss << " PM "; + std::vector aPmVec = fRef.fCmdArgs.getPmVec(); + unsigned int aIdx = 0; + + while (aIdx < aPmVec.size()) + { + oss << aPmVec[aIdx]; + aIdx++; + + if (aIdx != aPmVec.size()) + oss << ","; + } + + oss << " ..."; + } + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss.str(), MSGLVL_INFO1); + } } //----------------------------------------------------------------------------- bool WESDHandler::updateCPAndHWMInBRM() { - if (getDebugLvl()) - cout << "Inside updateCPAndHWMInBRM()" << endl; + if (getDebugLvl()) + cout << "Inside updateCPAndHWMInBRM()" << endl; - WEBrmUpdater aBrmUpdater(*this); - bool aRslt = aBrmUpdater.updateCasualPartitionAndHighWaterMarkInBRM(); - return aRslt; + WEBrmUpdater aBrmUpdater(*this); + bool aRslt = aBrmUpdater.updateCasualPartitionAndHighWaterMarkInBRM(); + return aRslt; } //----------------------------------------------------------------------------- void WESDHandler::cancelOutstandingCpimports() { - std::string aStr = "Canceling outstanding cpimports"; + std::string aStr = "Canceling outstanding cpimports"; - if (getDebugLvl()) cout << aStr << endl; + if (getDebugLvl()) + cout << aStr << endl; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStr, MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStr, MSGLVL_INFO1); - fFileReadThread.shutdown(); - bool aSetFail = false; + fFileReadThread.shutdown(); + bool aSetFail = false; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) + if ((!fWeSplClients[aCnt]->isCpiFailed()) && (!fWeSplClients[aCnt]->isCpiPassed())) + { + if (getDebugLvl()) + cout << "Canceling Cpimport in " << aCnt << endl; + + // clear the sendQ + fWeSplClients[aCnt]->clearSendQueue(); + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_EOD; + + if (fWeSplClients[aCnt]->isConnected()) { - if ((!fWeSplClients[aCnt]->isCpiFailed()) - && (!fWeSplClients[aCnt]->isCpiPassed())) - { - if (getDebugLvl()) - cout << "Canceling Cpimport in " << aCnt << endl; - - // clear the sendQ - fWeSplClients[aCnt]->clearSendQueue(); - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_EOD; - - if (fWeSplClients[aCnt]->isConnected()) - { - boost::mutex::scoped_lock aLock(fWeSplClients[aCnt]->fWriteMutex); - fWeSplClients[aCnt]->write(aBs); - aLock.unlock(); - } - else - { - fWeSplClients[aCnt]->setCpiFailed(true); //Setting as FAILED - } - } - // if it is passed already set it as canceled so that we can rollback - else if (fWeSplClients[aCnt]->isCpiPassed()) - { - fWeSplClients[aCnt]->setCpiPassed(false); - fWeSplClients[aCnt]->setCpiFailed(true); //Setting as FAILED - aSetFail = true; - } + boost::mutex::scoped_lock aLock(fWeSplClients[aCnt]->fWriteMutex); + fWeSplClients[aCnt]->write(aBs); + aLock.unlock(); } + else + { + fWeSplClients[aCnt]->setCpiFailed(true); // Setting as FAILED + } + } + // if it is passed already set it as canceled so that we can rollback + else if (fWeSplClients[aCnt]->isCpiPassed()) + { + fWeSplClients[aCnt]->setCpiPassed(false); + fWeSplClients[aCnt]->setCpiFailed(true); // Setting as FAILED + aSetFail = true; + } } + } - // setting Manual Failed caused a Rollback. Warrented if it the last PM - if (aSetFail) - { - if (checkAllCpiFailStatus()) - doRollback(); - } + // setting Manual Failed caused a Rollback. Warrented if it the last PM + if (aSetFail) + { + if (checkAllCpiFailStatus()) + doRollback(); + } - if (getDebugLvl()) - cout << "Canceled all outstanding cpimports!!" << endl; + if (getDebugLvl()) + cout << "Canceled all outstanding cpimports!!" << endl; } //----------------------------------------------------------------------------- bool WESDHandler::checkForRollbackAndCleanup() { - bool aRetStatus = true; + bool aRetStatus = true; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - // If Anyone of the client is - if ((!fWeSplClients[aCnt]->isCpiFailed()) && //NOT Failed - (!fWeSplClients[aCnt]->isCpiPassed()) && //NOT Passed - (fWeSplClients[aCnt]->isConnected())) //NOT Disconnected - { - aRetStatus = false; // then its not time to Rollback/Cleanup - break; - } - } + // If Anyone of the client is + if ((!fWeSplClients[aCnt]->isCpiFailed()) && // NOT Failed + (!fWeSplClients[aCnt]->isCpiPassed()) && // NOT Passed + (fWeSplClients[aCnt]->isConnected())) // NOT Disconnected + { + aRetStatus = false; // then its not time to Rollback/Cleanup + break; + } } + } - return aRetStatus; + return aRetStatus; } //----------------------------------------------------------------------------- bool WESDHandler::checkForCpiFailStatus() { - bool aRetStatus = false; + bool aRetStatus = false; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - // If Anyone of the client is - if (fWeSplClients[aCnt]->isCpiFailed()) // Failed - { - aRetStatus = true; // encounterd a Failure - break; - } - } + // If Anyone of the client is + if (fWeSplClients[aCnt]->isCpiFailed()) // Failed + { + aRetStatus = true; // encounterd a Failure + break; + } } + } - //if no CPI failed but a Forced Failure simulated, return Status as true - if ((!aRetStatus) && (fForcedFailure)) aRetStatus = true; + // if no CPI failed but a Forced Failure simulated, return Status as true + if ((!aRetStatus) && (fForcedFailure)) + aRetStatus = true; - return aRetStatus; + return aRetStatus; } //----------------------------------------------------------------------------- void WESDHandler::checkForConnections() { - time_t aNow = time(0); + time_t aNow = time(0); - for (int PmId = 1; PmId <= fPmCount; ++PmId) + for (int PmId = 1; PmId <= fPmCount; ++PmId) + { + if (fWeSplClients[PmId] != 0) { - if (fWeSplClients[PmId] != 0) - { - if (aNow - fWeSplClients[PmId]->getLastInTime() > 180) - { - std::string aStr; - aStr = std::string("Heartbeats missed - Non Responsive PM") + std::to_string(PmId); - fLog.logMsg( aStr, MSGLVL_ERROR ); - fWeSplClients[PmId]->onDisconnect(); - exit(1); //Otherwise; have to wait till write() comes out - } - } + if (aNow - fWeSplClients[PmId]->getLastInTime() > 180) + { + std::string aStr; + aStr = std::string("Heartbeats missed - Non Responsive PM") + std::to_string(PmId); + fLog.logMsg(aStr, MSGLVL_ERROR); + fWeSplClients[PmId]->onDisconnect(); + exit(1); // Otherwise; have to wait till write() comes out + } } + } } //----------------------------------------------------------------------------- void WESDHandler::sendHeartbeats() { - messageqcpp::SBS aSbs(new messageqcpp::ByteStream); - *aSbs << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE; - send2Pm(aSbs); + messageqcpp::SBS aSbs(new messageqcpp::ByteStream); + *aSbs << (ByteStream::byte)WE_CLT_SRV_KEEPALIVE; + send2Pm(aSbs); } //----------------------------------------------------------------------------- void WESDHandler::shutdown() { - if (fDisconnectFailure) + if (fDisconnectFailure) + { + onDisconnectFailure(); + } + + if (fRef.fSigHup) + { + onHandlingSigHup(); + } + + if (fRef.fSignaled) + { + onHandlingSignal(); + } + + fDataFeedList.clearPmList(); + + fFileReadThread.shutdown(); + + if (fFileReadThread.getFpThread()) + fFileReadThread.getFpThread()->join(); + + if (getDebugLvl()) + cout << "cleaning up Client threads " << endl; + + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - onDisconnectFailure(); + fWeSplClients[aCnt]->setContinue(false); + fWeSplClients[aCnt]->setConnected(false); + + if (fWeSplClients[aCnt]->getFpThread() != NULL) + fWeSplClients[aCnt]->getFpThread()->join(); + + if (getDebugLvl()) + fWeSplClients[aCnt]->printStats(); } + } - if (fRef.fSigHup) - { - onHandlingSigHup(); - } + boost::mutex::scoped_lock aLock(fRespMutex); + this->setContinue(false); + usleep(100000); // so that response thread get updated. + fRespCond.notify_all(); + aLock.unlock(); - if (fRef.fSignaled) - { - onHandlingSignal(); - } + if (fpRespThread) + fpRespThread->join(); - - fDataFeedList.clearPmList(); - - fFileReadThread.shutdown(); - - if (fFileReadThread.getFpThread()) - fFileReadThread.getFpThread()->join(); - - if (getDebugLvl()) cout << "cleaning up Client threads " << endl; - - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) - { - if (fWeSplClients[aCnt] != 0) - { - fWeSplClients[aCnt]->setContinue(false); - fWeSplClients[aCnt]->setConnected(false); - - if (fWeSplClients[aCnt]->getFpThread() != NULL) - fWeSplClients[aCnt]->getFpThread()->join(); - - if (getDebugLvl()) fWeSplClients[aCnt]->printStats(); - } - } - - - boost::mutex::scoped_lock aLock(fRespMutex); - this->setContinue(false); - usleep(100000); // so that response thread get updated. - fRespCond.notify_all(); - aLock.unlock(); - - if (fpRespThread) fpRespThread->join(); - - fLog.logMsg("Shutdown of all child threads Finished!!", MSGLVL_INFO2); + fLog.logMsg("Shutdown of all child threads Finished!!", MSGLVL_INFO2); } //------------------------------------------------------------------------------ void WESDHandler::onStartCpiResponse(int PmId) { - if (getDebugLvl()) - cout << "On Start CPI response arrived " << PmId << endl; + if (getDebugLvl()) + cout << "On Start CPI response arrived " << PmId << endl; - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_STARTCPI; - send2Pm(aBs, PmId); - fWeSplClients[PmId]->setCpiStarted(true); + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_STARTCPI; + send2Pm(aBs, PmId); + fWeSplClients[PmId]->setCpiStarted(true); } //------------------------------------------------------------------------------ void WESDHandler::onDataRqstResponse(int PmId) { - // may be we should add this pmid to the queue - //cout << "Received an DataRqst from "<< PmId << endl; - for debug - //BUG 5031 - Don't allow to accumulate RqstCnt uncontrollably - // since that will end up sending too many messages to WES. - // - Walt don't want this checking since we are going FIXED Q size - int aCnt = fWeSplClients[PmId]->getDataRqstCount(); + // may be we should add this pmid to the queue + // cout << "Received an DataRqst from "<< PmId << endl; - for debug + // BUG 5031 - Don't allow to accumulate RqstCnt uncontrollably + // since that will end up sending too many messages to WES. + // - Walt don't want this checking since we are going FIXED Q size + int aCnt = fWeSplClients[PmId]->getDataRqstCount(); - if (aCnt < MAX_WES_QSIZE) // 100 - { - if (getDebugLvl() > 2) - cout << "DataReqst [" << PmId << "] = " << - fWeSplClients[PmId]->getDataRqstCount() << endl; + if (aCnt < MAX_WES_QSIZE) // 100 + { + if (getDebugLvl() > 2) + cout << "DataReqst [" << PmId << "] = " << fWeSplClients[PmId]->getDataRqstCount() << endl; - fWeSplClients[PmId]->incDataRqstCount(); - } + fWeSplClients[PmId]->incDataRqstCount(); + } - //fDataFeedList.addPm2List(PmId); + // fDataFeedList.addPm2List(PmId); } //------------------------------------------------------------------------------ void WESDHandler::onAckResponse(int PmId) { - // may be we should add this pmid to the queue - if (getDebugLvl()) - cout << "Received an ACK from " << PmId << endl; + // may be we should add this pmid to the queue + if (getDebugLvl()) + cout << "Received an ACK from " << PmId << endl; - //fDataFeedList.addPm2List(PmId); + // fDataFeedList.addPm2List(PmId); } //------------------------------------------------------------------------------ void WESDHandler::onNakResponse(int PmId) { - if (getDebugLvl()) - cout << "Received a NAK from " << PmId << endl; + if (getDebugLvl()) + cout << "Received a NAK from " << PmId << endl; } //------------------------------------------------------------------------------ // Eod response means we are do not send anymore data to this PM @@ -1104,60 +1056,60 @@ void WESDHandler::onNakResponse(int PmId) // consume too much CPU void WESDHandler::onEodResponse(int PmId) { - if (getDebugLvl() > 2) - cout << "Received a EOD from " << PmId << endl; + if (getDebugLvl() > 2) + cout << "Received a EOD from " << PmId << endl; - if (fRef.fCmdArgs.getMode() == 0) + if (fRef.fCmdArgs.getMode() == 0) + { + // This is when one PM fail on Mode 0 + if (checkForCpiFailStatus()) // someone else failed, { - // This is when one PM fail on Mode 0 - if (checkForCpiFailStatus()) // someone else failed, - { - // so set this as failed for rollback. - if (getDebugLvl()) - cout << "Setting CPI failed on " << PmId << endl; + // so set this as failed for rollback. + if (getDebugLvl()) + cout << "Setting CPI failed on " << PmId << endl; - fWeSplClients[PmId]->setCpiPassed(false); - fWeSplClients[PmId]->setCpiFailed(true); + fWeSplClients[PmId]->setCpiPassed(false); + fWeSplClients[PmId]->setCpiFailed(true); - if (getDebugLvl()) - cout << "Calling onSigInterrupt from onEodResponse() " << endl; + if (getDebugLvl()) + cout << "Calling onSigInterrupt from onEodResponse() " << endl; - if (checkAllCpiFailStatus()) fRef.onSigInterrupt(1); - } - else - { - if (getDebugLvl()) - cout << "Calling onCpimportPass() from onEodResponse() " << endl; - - onCpimportPass(PmId); // set dummy Cpimport Pass - - if (checkAllCpiPassStatus()) // Mode 0 won't don't have BRM report - { - fImportRslt.stopTimer(); - ostringstream oss1; - //Bulk load completed, total run time : 2.98625 seconds - oss1 << "Load file distribution completed, total run time : "; - oss1 << fImportRslt.getTotalRunTime() << " seconds" << endl; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss1.str(), MSGLVL_INFO1 ); - - fRef.onSigInterrupt(0); // 1 for the sake of it - } - } + if (checkAllCpiFailStatus()) + fRef.onSigInterrupt(1); } - else if (fRef.fCmdArgs.getMode() == 1) + else { - // Only time when we get a EOD on Mode 1 is when there is failure - // in one of the PMs. Stop sending data and Send EOD to all PMS. - // Make sure fileread thread is not working anymore. - if (fFileReadThread.isContinue()) //check already stopped running - { - sendEODMsg(); - fFileReadThread.shutdown(); - } - } + if (getDebugLvl()) + cout << "Calling onCpimportPass() from onEodResponse() " << endl; + onCpimportPass(PmId); // set dummy Cpimport Pass + + if (checkAllCpiPassStatus()) // Mode 0 won't don't have BRM report + { + fImportRslt.stopTimer(); + ostringstream oss1; + // Bulk load completed, total run time : 2.98625 seconds + oss1 << "Load file distribution completed, total run time : "; + oss1 << fImportRslt.getTotalRunTime() << " seconds" << endl; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss1.str(), MSGLVL_INFO1); + + fRef.onSigInterrupt(0); // 1 for the sake of it + } + } + } + else if (fRef.fCmdArgs.getMode() == 1) + { + // Only time when we get a EOD on Mode 1 is when there is failure + // in one of the PMs. Stop sending data and Send EOD to all PMS. + // Make sure fileread thread is not working anymore. + if (fFileReadThread.isContinue()) // check already stopped running + { + sendEODMsg(); + fFileReadThread.shutdown(); + } + } } //------------------------------------------------------------------------------ @@ -1167,434 +1119,430 @@ void WESDHandler::onEodResponse(int PmId) void WESDHandler::onImpFileError(int PmId) { - if (fRef.fCmdArgs.getMode() != 0) - { - cout << "ERROR : we should not be here. Mode Non-Zero" << endl; - } + if (fRef.fCmdArgs.getMode() != 0) + { + cout << "ERROR : we should not be here. Mode Non-Zero" << endl; + } - std::stringstream aStrStr; - aStrStr << "Target file Error from PM" << PmId - << " - File already exists or path doesn't exist"; - char aDefCon[16], aRedCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); - cout << aRedCol << aStrStr.str() << aDefCon << endl; - fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); + std::stringstream aStrStr; + aStrStr << "Target file Error from PM" << PmId << " - File already exists or path doesn't exist"; + char aDefCon[16], aRedCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); + cout << aRedCol << aStrStr.str() << aDefCon << endl; + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); - if (!fWeSplClients[PmId]->isCpiFailed()) - fWeSplClients[PmId]->setCpiFailed(true); - else - return; // this failure already reported so get out - - // If IMPFILE open Failed, then stop sending data to other PMs too - if (checkAllCpiFailStatus()) - { - // If all are failed we will get out; no rollback for mode 0 - fRef.onSigInterrupt(1); // 1 for the sake of it - } - else - { - // Stop sending data to all PMs. Also send EOF to all PMs - cancelOutstandingCpimports(); - } + if (!fWeSplClients[PmId]->isCpiFailed()) + fWeSplClients[PmId]->setCpiFailed(true); + else + return; // this failure already reported so get out + // If IMPFILE open Failed, then stop sending data to other PMs too + if (checkAllCpiFailStatus()) + { + // If all are failed we will get out; no rollback for mode 0 + fRef.onSigInterrupt(1); // 1 for the sake of it + } + else + { + // Stop sending data to all PMs. Also send EOF to all PMs + cancelOutstandingCpimports(); + } } //------------------------------------------------------------------------------ void WESDHandler::onPmErrorResponse(int PmId) { - if (getDebugLvl()) - cout << "Received a NAK from " << PmId << endl; + if (getDebugLvl()) + cout << "Received a NAK from " << PmId << endl; } //------------------------------------------------------------------------------ void WESDHandler::onKeepAliveMessage(int PmId) { - if (getDebugLvl()) - cout << "Received a Keep Alive from " << PmId << endl; + if (getDebugLvl()) + cout << "Received a Keep Alive from " << PmId << endl; } //------------------------------------------------------------------------------ void WESDHandler::onCpimportPass(int PmId) { - std::stringstream aStrStr; - aStrStr << "Received a Cpimport Pass from PM" << PmId; - logging::Message::Args errMsgArgs; - //BUG 4152 - errMsgArgs.add(PmId); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0098); + std::stringstream aStrStr; + aStrStr << "Received a Cpimport Pass from PM" << PmId; + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(PmId); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0098); - if (getDebugLvl()) - cout << aStrStr.str() << endl; + if (getDebugLvl()) + cout << aStrStr.str() << endl; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); - fWeSplClients[PmId]->setCpiPassed(true); + fWeSplClients[PmId]->setCpiPassed(true); - // Every CPI passed, BRM report will be send to us b4 this Msg. - if (checkForCpiFailStatus()) // someone else failed, - { - // so set this as failed for rollback. - fWeSplClients[PmId]->setCpiPassed(false); - fWeSplClients[PmId]->setCpiFailed(true); - - if (checkAllCpiFailStatus()) - doRollback(); - } + // Every CPI passed, BRM report will be send to us b4 this Msg. + if (checkForCpiFailStatus()) // someone else failed, + { + // so set this as failed for rollback. + fWeSplClients[PmId]->setCpiPassed(false); + fWeSplClients[PmId]->setCpiFailed(true); + if (checkAllCpiFailStatus()) + doRollback(); + } } //------------------------------------------------------------------------------ void WESDHandler::onCpimportFail(int PmId, bool SigHandle) { - std::stringstream aStrStr; - char aDefCon[16], aRedCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); - aStrStr << aRedCol << "Received a Cpimport Failure from PM" << PmId << aDefCon; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); + std::stringstream aStrStr; + char aDefCon[16], aRedCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); + aStrStr << aRedCol << "Received a Cpimport Failure from PM" << PmId << aDefCon; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); - if (0 != PmId) - { - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - //cout << aRedCol << aStrStr.str() << aDefCon << endl; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); - std::stringstream aStrStr2; - aStrStr2 << "Please verify error log files in PM" << PmId; + if (0 != PmId) + { + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + // cout << aRedCol << aStrStr.str() << aDefCon << endl; + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); + std::stringstream aStrStr2; + aStrStr2 << "Please verify error log files in PM" << PmId; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr2.str(), MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr2.str(), MSGLVL_INFO1); - if (!fWeSplClients[PmId]->isCpiFailed()) - fWeSplClients[PmId]->setCpiFailed(true); - else - return; // this failure already reported so get out - } - - if ((SigHandle) && (0 == PmId)) - { - fForcedFailure = true; - } - -// if (!fWeSplClients[PmId]->isCpiFailed()) -// fWeSplClients[PmId]->setCpiFailed(true); -// else -// return; // this failure already reported so get out - - // If Any CPI Failed, then stop other CPIMPORTS too - // cancelOutstandingCpimports() - called below - // TODO - later do a total rollback and release locks. - if (checkAllCpiFailStatus()) - { - doRollback(); - //fRef.onSigInterrupt(1); // 1 for the sake of it - } + if (!fWeSplClients[PmId]->isCpiFailed()) + fWeSplClients[PmId]->setCpiFailed(true); else - { - // Stop sending data to all PMs. Also send EOF to all PMs - // so that all cpimports will finish bulk upload - cancelOutstandingCpimports(); - } + return; // this failure already reported so get out + } + if ((SigHandle) && (0 == PmId)) + { + fForcedFailure = true; + } + + // if (!fWeSplClients[PmId]->isCpiFailed()) + // fWeSplClients[PmId]->setCpiFailed(true); + // else + // return; // this failure already reported so get out + + // If Any CPI Failed, then stop other CPIMPORTS too + // cancelOutstandingCpimports() - called below + // TODO - later do a total rollback and release locks. + if (checkAllCpiFailStatus()) + { + doRollback(); + // fRef.onSigInterrupt(1); // 1 for the sake of it + } + else + { + // Stop sending data to all PMs. Also send EOF to all PMs + // so that all cpimports will finish bulk upload + cancelOutstandingCpimports(); + } } //------------------------------------------------------------------------------ void WESDHandler::onBrmReport(int PmId, messageqcpp::SBS& Sbs) { - std::stringstream aStrStr; - aStrStr << "Received a BRM-Report from " << PmId; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); + std::stringstream aStrStr; + aStrStr << "Received a BRM-Report from " << PmId; + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); - if (getDebugLvl()) - cout << aStrStr.str() << endl; + if (getDebugLvl()) + cout << aStrStr.str() << endl; - fWeSplClients[PmId]->setBrmRptRcvd(true); + fWeSplClients[PmId]->setBrmRptRcvd(true); - std::string aStr; - int64_t aTotRows = 0; - int64_t aInsRows = 0; - int aColNum = 0; - CalpontSystemCatalog::ColDataType aColType = CalpontSystemCatalog::INT; - int aOorVal = 0; - std::string aBadFileName; - std::string aErrFileName; - std::string aColName; + std::string aStr; + int64_t aTotRows = 0; + int64_t aInsRows = 0; + int aColNum = 0; + CalpontSystemCatalog::ColDataType aColType = CalpontSystemCatalog::INT; + int aOorVal = 0; + std::string aBadFileName; + std::string aErrFileName; + std::string aColName; + + if (getDebugLvl() > 2) + cout << "BRM Report length : " << (*Sbs).length() << endl; + + while ((*Sbs).length() > 0) + { + (*Sbs) >> aStr; if (getDebugLvl() > 2) - cout << "BRM Report length : " << (*Sbs).length() << endl; + cout << "BRM Report value : " << aStr << endl; - while ((*Sbs).length() > 0) + bool aRet = WEBrmUpdater::prepareRowsInsertedInfo(aStr, aTotRows, aInsRows); + + if (aRet) { - (*Sbs) >> aStr; - - if (getDebugLvl() > 2) - cout << "BRM Report value : " << aStr << endl; - - bool aRet = WEBrmUpdater::prepareRowsInsertedInfo(aStr, aTotRows, aInsRows); - - if (aRet) - { - setRowsUploadInfo(PmId, aTotRows, aInsRows); - fImportRslt.updateRowsProcessed(aTotRows); - fImportRslt.updateRowsInserted(aInsRows); - } - - aRet = WEBrmUpdater::prepareColumnOutOfRangeInfo(aStr, aColNum, aColType, - aColName, aOorVal); - - if (aRet) - { - add2ColOutOfRangeInfo(PmId, aColNum, aColType, aColName, aOorVal); - fImportRslt.updateColOutOfRangeInfo(aColNum, aColType, aColName, aOorVal); - } - - aRet = WEBrmUpdater::prepareBadDataFileInfo(aStr, aBadFileName); - - if (aRet) - { - setBadFileName(PmId, aBadFileName); - - // BUG 4324 - Mode 2 bad/err files left in PM(s). - if (1 == fRef.fCmdArgs.getMode()) getBadLog(PmId, aBadFileName); - else if (2 == fRef.fCmdArgs.getMode()) - { - std::stringstream aOss; - aOss << "Bad File : " << aBadFileName << " @ PM" << PmId; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aOss.str(), MSGLVL_INFO1 ); - } - } - - aRet = WEBrmUpdater::prepareErrorFileInfo(aStr, aErrFileName); - - if (aRet) - { - setErrorFileName(PmId, aErrFileName); - - // BUG 4324 - Mode 2 bad/err files left in PM(s). - if (1 == fRef.fCmdArgs.getMode()) getErrorLog(PmId, aErrFileName); - else if (2 == fRef.fCmdArgs.getMode()) - { - std::stringstream aOss; - aOss << "Err File : " << aErrFileName << " @ PM" << PmId; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aOss.str(), MSGLVL_INFO1 ); - } - } - - aRet = check4CriticalErrMsgs(aStr); // check 4 Critical msg from .bin - - if (aRet) - { - std::stringstream aOss; - aOss << "PM" << PmId << " : " << aStr; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aOss.str()); - - if (!fRef.fCmdArgs.getConsoleOutput()) - { - ostringstream oss; - oss << startup::StartUp::tmpDir() << fTableOId << ".txt"; - ofstream dmlFile(oss.str().c_str(), std::ofstream::app); - - if (dmlFile.is_open()) - { - dmlFile << aOss.str(); - dmlFile << endl; - } - - dmlFile.close(); - } - - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - - if (getDebugLvl()) cout << aOss.str() << endl; - - fLog.logMsg( aOss.str(), MSGLVL_ERROR ); - } - else // do not add Crit Msgs to BRMRpt vector - fBrmRptVec.push_back(aStr); + setRowsUploadInfo(PmId, aTotRows, aInsRows); + fImportRslt.updateRowsProcessed(aTotRows); + fImportRslt.updateRowsInserted(aInsRows); } - // Even when CPI fail, we get BRMRpt to get the Err/Bad file. - if (checkForCpiFailStatus()) - return; // if a PM failed, don't update BRM. + aRet = WEBrmUpdater::prepareColumnOutOfRangeInfo(aStr, aColNum, aColType, aColName, aOorVal); - //cout << "Checking clients for BRM Reports" << endl; - //TODO we should update BRM with the report we got. - if (check4AllBrmReports()) + if (aRet) { - bool aRslt = updateCPAndHWMInBRM(); + add2ColOutOfRangeInfo(PmId, aColNum, aColType, aColName, aOorVal); + fImportRslt.updateColOutOfRangeInfo(aColNum, aColType, aColName, aOorVal); + } - if (aRslt) + aRet = WEBrmUpdater::prepareBadDataFileInfo(aStr, aBadFileName); + + if (aRet) + { + setBadFileName(PmId, aBadFileName); + + // BUG 4324 - Mode 2 bad/err files left in PM(s). + if (1 == fRef.fCmdArgs.getMode()) + getBadLog(PmId, aBadFileName); + else if (2 == fRef.fCmdArgs.getMode()) + { + std::stringstream aOss; + aOss << "Bad File : " << aBadFileName << " @ PM" << PmId; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aOss.str(), MSGLVL_INFO1); + } + } + + aRet = WEBrmUpdater::prepareErrorFileInfo(aStr, aErrFileName); + + if (aRet) + { + setErrorFileName(PmId, aErrFileName); + + // BUG 4324 - Mode 2 bad/err files left in PM(s). + if (1 == fRef.fCmdArgs.getMode()) + getErrorLog(PmId, aErrFileName); + else if (2 == fRef.fCmdArgs.getMode()) + { + std::stringstream aOss; + aOss << "Err File : " << aErrFileName << " @ PM" << PmId; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aOss.str(), MSGLVL_INFO1); + } + } + + aRet = check4CriticalErrMsgs(aStr); // check 4 Critical msg from .bin + + if (aRet) + { + std::stringstream aOss; + aOss << "PM" << PmId << " : " << aStr; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aOss.str()); + + if (!fRef.fCmdArgs.getConsoleOutput()) + { + ostringstream oss; + oss << startup::StartUp::tmpDir() << fTableOId << ".txt"; + ofstream dmlFile(oss.str().c_str(), std::ofstream::app); + + if (dmlFile.is_open()) { - std::stringstream aStrStr; - aStrStr << "BRM updated successfully "; + dmlFile << aOss.str(); + dmlFile << endl; + } - if (getDebugLvl()) cout << aStrStr.str() << endl; + dmlFile.close(); + } - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - if (fTableLock != 0) - { + if (getDebugLvl()) + cout << aOss.str() << endl; - WETableLockGrabber aTLG(*this); - bool aRet = aTLG.changeTableLockState(fTableLock); + fLog.logMsg(aOss.str(), MSGLVL_ERROR); + } + else // do not add Crit Msgs to BRMRpt vector + fBrmRptVec.push_back(aStr); + } - if (aRet) - { - if (getDebugLvl()) - cout << "\tSuccessfully changed TableLock State" << endl; + // Even when CPI fail, we get BRMRpt to get the Err/Bad file. + if (checkForCpiFailStatus()) + return; // if a PM failed, don't update BRM. - doCleanup(true); - } - else - { - std::stringstream aStrStr; - aStrStr << "Failed to change TableLock state to cleanup"; - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); + // cout << "Checking clients for BRM Reports" << endl; + // TODO we should update BRM with the report we got. + if (check4AllBrmReports()) + { + bool aRslt = updateCPAndHWMInBRM(); - if (getDebugLvl()) cout << aStrStr.str() << endl; - } + if (aRslt) + { + std::stringstream aStrStr; + aStrStr << "BRM updated successfully "; - } + if (getDebugLvl()) + cout << aStrStr.str() << endl; + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); + + if (fTableLock != 0) + { + WETableLockGrabber aTLG(*this); + bool aRet = aTLG.changeTableLockState(fTableLock); + + if (aRet) + { + if (getDebugLvl()) + cout << "\tSuccessfully changed TableLock State" << endl; + + doCleanup(true); } else { - std::stringstream aStrStr; - aStrStr << "\tBRM update Failed : Need to Manually release the table locks"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + std::stringstream aStrStr; + aStrStr << "Failed to change TableLock state to cleanup"; + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); - if (getDebugLvl()) cout << aStrStr.str() << endl; - - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); - fRef.onSigInterrupt(1); //BUG 4701 //failure in BRM update & process, ie. 1 + if (getDebugLvl()) + cout << aStrStr.str() << endl; } - + } } else { - if (getDebugLvl()) cout << "Still Brm Reports to come in!!" << endl; - } + std::stringstream aStrStr; + aStrStr << "\tBRM update Failed : Need to Manually release the table locks"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + if (getDebugLvl()) + cout << aStrStr.str() << endl; + + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); + fRef.onSigInterrupt(1); // BUG 4701 //failure in BRM update & process, ie. 1 + } + } + else + { + if (getDebugLvl()) + cout << "Still Brm Reports to come in!!" << endl; + } } //------------------------------------------------------------------------------ bool WESDHandler::check4CriticalErrMsgs(std::string& Entry) { - bool aFound = false; + bool aFound = false; - if ((!Entry.empty()) && (Entry.at(0) == 'M')) - { - aFound = true; - // start from after "MERR: " - std::string aTmp(Entry.begin() + 6, Entry.end()); - Entry = aTmp; - } + if ((!Entry.empty()) && (Entry.at(0) == 'M')) + { + aFound = true; + // start from after "MERR: " + std::string aTmp(Entry.begin() + 6, Entry.end()); + Entry = aTmp; + } - return aFound; + return aFound; } //------------------------------------------------------------------------------ void WESDHandler::onErrorFile(int PmId, messageqcpp::SBS& Sbs) { - std::stringstream aStrStr; - aStrStr << "Received ErrReport from " << PmId; - logging::Message::Args errMsgArgs; - // BUG 4152 - errMsgArgs.add(PmId); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0099); + std::stringstream aStrStr; + aStrStr << "Received ErrReport from " << PmId; + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(PmId); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0099); - if (getDebugLvl()) cout << aStrStr.str() << endl; + if (getDebugLvl()) + cout << aStrStr.str() << endl; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); - //TODO - Open the ERROR log file and append to it - ofstream aErrFile; - std::string aErrFileName; - std::string aTmpFileName; - std::string aData; - (*Sbs) >> aTmpFileName; + // TODO - Open the ERROR log file and append to it + ofstream aErrFile; + std::string aErrFileName; + std::string aTmpFileName; + std::string aData; + (*Sbs) >> aTmpFileName; - // BUG 4324 - Mode 1 bad/err files stored in datafile loc or CWD - size_t aPos = aTmpFileName.rfind('/'); + // BUG 4324 - Mode 1 bad/err files stored in datafile loc or CWD + size_t aPos = aTmpFileName.rfind('/'); - if (aPos != std::string::npos) + if (aPos != std::string::npos) + { + std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/' + + std::string aInFile = fRef.fCmdArgs.getErrorDir(); + + if (aInFile != "/dev/stdin") { - std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/' + size_t aPos2 = aInFile.rfind('/'); - std::string aInFile = fRef.fCmdArgs.getErrorDir(); + if (aPos2 != std::string::npos) + { + std::string aStr = aInFile.substr(0, aPos2 + 1); + // std::cout << "Point 1 " << aStr << std::endl; + std::stringstream aStrStr1; + aStrStr1 << aStr << aFile; + aTmpFileName = aStrStr1.str(); + // std::cout << "Point 2 " << aTmpFileName << std::endl; + } + else + aTmpFileName = aFile; + } + else + aTmpFileName = aFile; - if (aInFile != "/dev/stdin") - { - size_t aPos2 = aInFile.rfind('/'); + if (getDebugLvl()) + std::cout << "Prep ErrFile " << aTmpFileName << std::endl; + } - if (aPos2 != std::string::npos) - { - std::string aStr = aInFile.substr(0, aPos2 + 1); - //std::cout << "Point 1 " << aStr << std::endl; - std::stringstream aStrStr1; - aStrStr1 << aStr << aFile; - aTmpFileName = aStrStr1.str(); - //std::cout << "Point 2 " << aTmpFileName << std::endl; - } - else - aTmpFileName = aFile; - } - else - aTmpFileName = aFile; + aStrStr.str(std::string()); + aStrStr << aTmpFileName << "_" << PmId; + aErrFileName = aStrStr.str(); // PmId+"_"+aTmpFileName; - if (getDebugLvl()) - std::cout << "Prep ErrFile " << aTmpFileName << std::endl; + if (getDebugLvl()) + cout << "Error File Name: " << aErrFileName << endl; + + if (getDebugLvl()) + cout << "Error Data: " << endl; + + try + { + aErrFile.open(aErrFileName.c_str()); + + while ((*Sbs).length() > 0) + { + (*Sbs) >> aData; + + if (getDebugLvl() > 1) + cout << aData << endl; + + aErrFile << aData; + aErrFile << endl; } + aErrFile.close(); + setErrorFileName(PmId, aErrFileName); aStrStr.str(std::string()); - aStrStr << aTmpFileName << "_" << PmId; - aErrFileName = aStrStr.str(); //PmId+"_"+aTmpFileName; - - if (getDebugLvl()) - cout << "Error File Name: " << aErrFileName << endl; - - if (getDebugLvl()) - cout << "Error Data: " << endl; - - try - { - aErrFile.open(aErrFileName.c_str()); - - while ((*Sbs).length() > 0) - { - (*Sbs) >> aData; - - if (getDebugLvl() > 1) cout << aData << endl; - - aErrFile << aData; - aErrFile << endl; - } - - aErrFile.close(); - setErrorFileName(PmId, aErrFileName); - aStrStr.str(std::string()); - aStrStr << "Row numbers with error reasons are listed in file : " << aErrFileName; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr.str(), MSGLVL_INFO1 ); - - } - catch (std::exception&) - { - cout << "Error in opening the ERROR file!!" << aErrFileName << endl; - cout << "Error in opening the ERROR file!!" << aTmpFileName << endl; - cout << "Check for ErrorFile " << aTmpFileName << "in Pm " << PmId - << endl; - } + aStrStr << "Row numbers with error reasons are listed in file : " << aErrFileName; + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr.str(), MSGLVL_INFO1); + } + catch (std::exception&) + { + cout << "Error in opening the ERROR file!!" << aErrFileName << endl; + cout << "Error in opening the ERROR file!!" << aTmpFileName << endl; + cout << "Check for ErrorFile " << aTmpFileName << "in Pm " << PmId << endl; + } } //------------------------------------------------------------------------------ @@ -1602,1368 +1550,1328 @@ void WESDHandler::onErrorFile(int PmId, messageqcpp::SBS& Sbs) //------------------------------------------------------------------------------ void WESDHandler::onBadFile(int PmId, messageqcpp::SBS& Sbs) { - std::stringstream aStrStr; - aStrStr << "Received BadData Report from " << PmId; - logging::Message::Args errMsgArgs; - //BUG 4152 - errMsgArgs.add(PmId); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0100); + std::stringstream aStrStr; + aStrStr << "Received BadData Report from " << PmId; + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(PmId); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0100); - if (getDebugLvl()) cout << aStrStr.str() << endl; + if (getDebugLvl()) + cout << aStrStr.str() << endl; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); - //TODO - Open the ERROR log file and append to it - ofstream aBadFile; - std::string aBadFileName; - std::string aTmpFileName; - std::string aData; - (*Sbs) >> aTmpFileName; + // TODO - Open the ERROR log file and append to it + ofstream aBadFile; + std::string aBadFileName; + std::string aTmpFileName; + std::string aData; + (*Sbs) >> aTmpFileName; - // BUG 4324 - Mode 1 bad/err files stored in datafile loc or CWD - size_t aPos = aTmpFileName.rfind('/'); + // BUG 4324 - Mode 1 bad/err files stored in datafile loc or CWD + size_t aPos = aTmpFileName.rfind('/'); - if (aPos != std::string::npos) + if (aPos != std::string::npos) + { + std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/' + + std::string aInFile = fRef.fCmdArgs.getErrorDir(); + + if (aInFile != "/dev/stdin") { - std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/' + size_t aPos2 = aInFile.rfind('/'); - std::string aInFile = fRef.fCmdArgs.getErrorDir(); - - if (aInFile != "/dev/stdin") - { - size_t aPos2 = aInFile.rfind('/'); - - if (aPos2 != std::string::npos) - { - std::string aStr = aInFile.substr(0, aPos2 + 1); - //std::cout << "Point 1 " << aStr << std::endl; - std::stringstream aStrStr1; - aStrStr1 << aStr << aFile; - aTmpFileName = aStrStr1.str(); - //std::cout << "Point 2 " << aTmpFileName << std::endl; - } - else - aTmpFileName = aFile; - } - else - aTmpFileName = aFile; - - if (getDebugLvl() > 1) - std::cout << "Prep BadFile " << aTmpFileName << std::endl; + if (aPos2 != std::string::npos) + { + std::string aStr = aInFile.substr(0, aPos2 + 1); + // std::cout << "Point 1 " << aStr << std::endl; + std::stringstream aStrStr1; + aStrStr1 << aStr << aFile; + aTmpFileName = aStrStr1.str(); + // std::cout << "Point 2 " << aTmpFileName << std::endl; + } + else + aTmpFileName = aFile; } - - aStrStr.str(std::string()); - aStrStr << aTmpFileName << "_" << PmId; - aBadFileName = aStrStr.str(); //PmId+"_"+aTmpFileName; + else + aTmpFileName = aFile; if (getDebugLvl() > 1) - cout << "Bad File Name: " << aBadFileName << endl; + std::cout << "Prep BadFile " << aTmpFileName << std::endl; + } - if ( (getDebugLvl() > 1) && - (fRef.fCmdArgs.getImportDataMode() == IMPORT_DATA_TEXT) ) - cout << "Bad Data: " << endl; + aStrStr.str(std::string()); + aStrStr << aTmpFileName << "_" << PmId; + aBadFileName = aStrStr.str(); // PmId+"_"+aTmpFileName; - std::string task; + if (getDebugLvl() > 1) + cout << "Bad File Name: " << aBadFileName << endl; - try + if ((getDebugLvl() > 1) && (fRef.fCmdArgs.getImportDataMode() == IMPORT_DATA_TEXT)) + cout << "Bad Data: " << endl; + + std::string task; + + try + { + task = "opening"; + aBadFile.open(aBadFileName.c_str()); + + task = "copying rejected rows to"; + + while ((*Sbs).length() > 0) { - task = "opening"; - aBadFile.open(aBadFileName.c_str()); + (*Sbs) >> aData; - task = "copying rejected rows to"; + if ((getDebugLvl() > 1) && (fRef.fCmdArgs.getImportDataMode() == IMPORT_DATA_TEXT)) + { + cout << aData; + } - while ((*Sbs).length() > 0) - { - (*Sbs) >> aData; - - if ( (getDebugLvl() > 1) && - (fRef.fCmdArgs.getImportDataMode() == IMPORT_DATA_TEXT) ) - { - cout << aData; - } - - aBadFile.write( aData.c_str(), aData.length() ); - } - - aBadFile.close(); - setBadFileName(PmId, aBadFileName); - - aStrStr.str(std::string()); - aStrStr << "Exact error rows are listed in file : " << aBadFileName; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr.str(), MSGLVL_INFO1 ); - } - catch (std::exception& ex) - { - cout << "Error in " << task << " the bad file " << - aBadFileName << "; " << ex.what() << endl; - cout << "Check for Bad File " << aTmpFileName << " on Pm " << PmId << endl; + aBadFile.write(aData.c_str(), aData.length()); } + + aBadFile.close(); + setBadFileName(PmId, aBadFileName); + + aStrStr.str(std::string()); + aStrStr << "Exact error rows are listed in file : " << aBadFileName; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr.str(), MSGLVL_INFO1); + } + catch (std::exception& ex) + { + cout << "Error in " << task << " the bad file " << aBadFileName << "; " << ex.what() << endl; + cout << "Check for Bad File " << aTmpFileName << " on Pm " << PmId << endl; + } } //------------------------------------------------------------------------------ void WESDHandler::getErrorLog(int PmId, const std::string& ErrFileName) { - if (getDebugLvl()) - cout << "Requesting Error Log" << endl; - - //TODO code appropriately for the message - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_ERRLOG; - aBs << ErrFileName; - send2Pm(aBs, PmId); - //fWeSplClients[PmId]->setErrLogRqst(true); + if (getDebugLvl()) + cout << "Requesting Error Log" << endl; + // TODO code appropriately for the message + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_ERRLOG; + aBs << ErrFileName; + send2Pm(aBs, PmId); + // fWeSplClients[PmId]->setErrLogRqst(true); } //------------------------------------------------------------------------------ void WESDHandler::getBadLog(int PmId, const std::string& BadFileName) { - if (getDebugLvl()) - cout << "Requesting Bad Log" << endl; - - //TODO code appropriately for the message - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_BADLOG; - aBs << BadFileName; - send2Pm(aBs, PmId); - //fWeSplClients[PmId]->setBadLogRqst(true); + if (getDebugLvl()) + cout << "Requesting Bad Log" << endl; + // TODO code appropriately for the message + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_BADLOG; + aBs << BadFileName; + send2Pm(aBs, PmId); + // fWeSplClients[PmId]->setBadLogRqst(true); } //------------------------------------------------------------------------------ void WESDHandler::onRollbackResult(int PmId, messageqcpp::SBS& Sbs) { - ByteStream::byte aRslt = 0; + ByteStream::byte aRslt = 0; - (*Sbs) >> aRslt; + (*Sbs) >> aRslt; + + if (getDebugLvl()) + cout << "Rollback rslt arrived PmId = " << PmId << " Rslt = " << (int)aRslt << endl; + + if (aRslt) + fWeSplClients[PmId]->setRollbackRslt(1); + else + { + fWeSplClients[PmId]->setRollbackRslt(-1); + std::stringstream aStrStr; + aStrStr << "Rollback Failed on PM : " << PmId; + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(PmId); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0101); + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); if (getDebugLvl()) - cout << "Rollback rslt arrived PmId = " << PmId << " Rslt = " - << (int) aRslt << endl; + cout << aStrStr.str() << endl; + } - if (aRslt) - fWeSplClients[PmId]->setRollbackRslt(1); - else + int aStatus = check4RollbackRslts(); + + if (aStatus == -1) + { + std::stringstream aStrStr; + aStrStr << "Rollback Failed on one or more PMs"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); + + if (getDebugLvl()) + cout << aStrStr.str() << endl; + + // fRef.onSigInterrupt(1); // 1 for the sake of it + if (check4AllRollbackStatus()) { - fWeSplClients[PmId]->setRollbackRslt(-1); - std::stringstream aStrStr; - aStrStr << "Rollback Failed on PM : " << PmId; - logging::Message::Args errMsgArgs; - //BUG 4152 - errMsgArgs.add(PmId); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, - logging::M0101); - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); - - if (getDebugLvl()) - cout << aStrStr.str() << endl; + fRef.onSigInterrupt(1); // process altogether is a failure } + } + else if (aStatus == 1) + { + std::stringstream aStrStr; + aStrStr << "Rollback succeed on all PMs"; + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); - int aStatus = check4RollbackRslts(); - - if (aStatus == -1) - { - std::stringstream aStrStr; - aStrStr << "Rollback Failed on one or more PMs"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, - logging::M0000); - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); - - if (getDebugLvl()) - cout << aStrStr.str() << endl; - - //fRef.onSigInterrupt(1); // 1 for the sake of it - if (check4AllRollbackStatus()) - { - fRef.onSigInterrupt(1); // process altogether is a failure - } - } - else if (aStatus == 1) - { - std::stringstream aStrStr; - aStrStr << "Rollback succeed on all PMs"; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); - - if (getDebugLvl()) cout << aStrStr.str() << endl; - - // false flag sent to doCleanup says to not delete HDFS temp db files, - // because the bulk rollback will have already deleted them. We still - // call doCleanup for other file cleanup (like deleting meta file). - doCleanup(false); - } + if (getDebugLvl()) + cout << aStrStr.str() << endl; + // false flag sent to doCleanup says to not delete HDFS temp db files, + // because the bulk rollback will have already deleted them. We still + // call doCleanup for other file cleanup (like deleting meta file). + doCleanup(false); + } } //------------------------------------------------------------------------------ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) { - ByteStream::byte aRslt = 0; - (*Sbs) >> aRslt; + ByteStream::byte aRslt = 0; + (*Sbs) >> aRslt; + + if (getDebugLvl()) + cout << "Cleanup rslt arrived PmId = " << PmId << " Rslt = " << (int)aRslt << endl; + + if (aRslt) + fWeSplClients[PmId]->setCleanupRslt(1); + else + { + fWeSplClients[PmId]->setCleanupRslt(-1); + std::stringstream aStrStr; + aStrStr << "ERROR: Cleanup Failed on PM : " << PmId; + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(PmId); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0102); if (getDebugLvl()) - cout << "Cleanup rslt arrived PmId = " << PmId << " Rslt = " - << (int) aRslt << endl; + cout << aStrStr.str() << endl; - if (aRslt) - fWeSplClients[PmId]->setCleanupRslt(1); + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); + } + + int aStatus = check4CleanupRslts(); + + if (aStatus == -1) + { + std::stringstream aStrStr; + aStrStr << "Cleanup Failed on one or more PMs"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + fLog.logMsg(aStrStr.str(), MSGLVL_ERROR); + + if (getDebugLvl()) + cout << aStrStr.str() << endl; + + // fRef.onSigInterrupt(1); // 1 for the sake of it - + // We need to wait till all the results comes back. + if (check4AllCleanupStatus()) + { + fRef.onSigInterrupt(1); // failure in cleanup & process, ie. 1 + } + } + else if (aStatus == 1) + { + releaseTableLocks(); + + std::stringstream aStrStr; + aStrStr << "Cleanup succeed on all PMs"; + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); + + if (getDebugLvl()) + cout << aStrStr.str() << endl; + + if (checkAllCpiPassStatus()) // Cleanup and entire process success. + { + ostringstream oss; + // For table walt.abc: 1000 rows processed and 1000 rows inserted. + oss << "For table "; + oss << fRef.fCmdArgs.getSchemaName() << "."; + oss << fRef.fCmdArgs.getTableName() << ": "; + oss << fImportRslt.fRowsPro << " rows processed and "; + oss << fImportRslt.fRowsIns << " rows inserted."; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss.str(), MSGLVL_INFO1); + + // BUG 4399 Print out WARN messages for out of range counts + WEColOorVec::iterator aIt = fImportRslt.fColOorVec.begin(); + ofstream dmlFile; + + if (!fRef.fCmdArgs.getConsoleOutput()) // for DML to use file + { + ostringstream oss; + oss << startup::StartUp::tmpDir() << fTableOId << ".txt"; + dmlFile.open(oss.str().c_str()); + } + + while (aIt != fImportRslt.fColOorVec.end()) + { + if ((*aIt).fNoOfOORs > 0) + { + ostringstream ossSatCnt; + ossSatCnt << "Column " << (*aIt).fColName << "; Number of "; + + switch ((*aIt).fColType) + { + case CalpontSystemCatalog::DATE: ossSatCnt << "invalid dates replaced with zero value: "; break; + + case CalpontSystemCatalog::DATETIME: + ossSatCnt << "invalid date/times replaced with zero value: "; + break; + + case CalpontSystemCatalog::TIMESTAMP: + ossSatCnt << "invalid timestamps replaced with zero value: "; + break; + + case CalpontSystemCatalog::TIME: ossSatCnt << "invalid times replaced with zero value: "; break; + + case CalpontSystemCatalog::CHAR: ossSatCnt << "character strings truncated: "; break; + + case CalpontSystemCatalog::VARCHAR: ossSatCnt << "varchar strings truncated: "; break; + + default: ossSatCnt << "rows inserted with saturated values: "; break; + } + + ossSatCnt << (*aIt).fNoOfOORs; + fLog.logMsg(ossSatCnt.str(), MSGLVL_WARNING); + + if (!fRef.fCmdArgs.getConsoleOutput()) // for DML to use + { + if (dmlFile.is_open()) + { + dmlFile << (*aIt).fNoOfOORs; + dmlFile << endl; + } + } + } + + aIt++; + } + + dmlFile.close(); + fImportRslt.stopTimer(); + ostringstream oss1; + // Bulk load completed, total run time : 2.98625 seconds + oss1 << "Bulk load completed, total run time : "; + oss1 << fImportRslt.getTotalRunTime() << " seconds"; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss1.str(), MSGLVL_INFO1); + + fRef.onSigInterrupt(0); // 0 for entire success + } else { - fWeSplClients[PmId]->setCleanupRslt(-1); - std::stringstream aStrStr; - aStrStr << "ERROR: Cleanup Failed on PM : " << PmId; - logging::Message::Args errMsgArgs; - //BUG 4152 - errMsgArgs.add(PmId); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0102); + ostringstream oss; + oss << "Table " << fRef.fCmdArgs.getSchemaName() << "."; + oss << fRef.fCmdArgs.getTableName() << ": (OID-"; + oss << this->getTableOID() << ") was NOT successfully loaded."; - if (getDebugLvl()) cout << aStrStr.str() << endl; + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss.str(), MSGLVL_INFO1); - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); + fImportRslt.stopTimer(); + ostringstream oss1; + // Bulk load completed, total run time : 2.98625 seconds + oss1 << "Bulk load completed, total run time : "; + oss1 << fImportRslt.getTotalRunTime() << " seconds"; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(oss1.str(), MSGLVL_INFO1); + + // Even though cleanup is success, entire process is failure + fRef.onSigInterrupt(1); // therefore 1 } - - int aStatus = check4CleanupRslts(); - - if (aStatus == -1) - { - std::stringstream aStrStr; - aStrStr << "Cleanup Failed on one or more PMs"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - fLog.logMsg( aStrStr.str(), MSGLVL_ERROR ); - - if (getDebugLvl()) cout << aStrStr.str() << endl; - - //fRef.onSigInterrupt(1); // 1 for the sake of it - - // We need to wait till all the results comes back. - if (check4AllCleanupStatus()) - { - fRef.onSigInterrupt(1); // failure in cleanup & process, ie. 1 - } - } - else if (aStatus == 1) - { - releaseTableLocks(); - - std::stringstream aStrStr; - aStrStr << "Cleanup succeed on all PMs"; - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); - - if (getDebugLvl()) - cout << aStrStr.str() << endl; - - if (checkAllCpiPassStatus()) //Cleanup and entire process success. - { - ostringstream oss; - //For table walt.abc: 1000 rows processed and 1000 rows inserted. - oss << "For table "; - oss << fRef.fCmdArgs.getSchemaName() << "."; - oss << fRef.fCmdArgs.getTableName() << ": "; - oss << fImportRslt.fRowsPro << " rows processed and "; - oss << fImportRslt.fRowsIns << " rows inserted."; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - - // BUG 4399 Print out WARN messages for out of range counts - WEColOorVec::iterator aIt = fImportRslt.fColOorVec.begin(); - ofstream dmlFile; - - if (!fRef.fCmdArgs.getConsoleOutput()) //for DML to use file - { - ostringstream oss; - oss << startup::StartUp::tmpDir() << fTableOId << ".txt"; - dmlFile.open(oss.str().c_str()); - } - - while (aIt != fImportRslt.fColOorVec.end()) - { - if ((*aIt).fNoOfOORs > 0) - { - ostringstream ossSatCnt; - ossSatCnt << "Column " << (*aIt).fColName << "; Number of "; - - switch ((*aIt).fColType) - { - case CalpontSystemCatalog::DATE: - ossSatCnt << "invalid dates replaced with zero value: "; - break; - - case CalpontSystemCatalog::DATETIME: - ossSatCnt << "invalid date/times replaced with zero value: "; - break; - - case CalpontSystemCatalog::TIMESTAMP: - ossSatCnt << "invalid timestamps replaced with zero value: "; - break; - - case CalpontSystemCatalog::TIME: - ossSatCnt << "invalid times replaced with zero value: "; - break; - - case CalpontSystemCatalog::CHAR: - ossSatCnt << "character strings truncated: "; - break; - - case CalpontSystemCatalog::VARCHAR: - ossSatCnt << "varchar strings truncated: "; - break; - - default: - ossSatCnt << "rows inserted with saturated values: "; - break; - } - - ossSatCnt << (*aIt).fNoOfOORs; - fLog.logMsg(ossSatCnt.str(), MSGLVL_WARNING); - - if (!fRef.fCmdArgs.getConsoleOutput()) //for DML to use - { - if (dmlFile.is_open()) - { - dmlFile << (*aIt).fNoOfOORs; - dmlFile << endl; - } - } - } - - aIt++; - } - - dmlFile.close(); - fImportRslt.stopTimer(); - ostringstream oss1; - //Bulk load completed, total run time : 2.98625 seconds - oss1 << "Bulk load completed, total run time : "; - oss1 << fImportRslt.getTotalRunTime() << " seconds"; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss1.str(), MSGLVL_INFO1 ); - - fRef.onSigInterrupt(0); // 0 for entire success - } - else - { - ostringstream oss; - oss << "Table " << fRef.fCmdArgs.getSchemaName() << "."; - oss << fRef.fCmdArgs.getTableName() << ": (OID-"; - oss << this->getTableOID() << ") was NOT successfully loaded."; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss.str(), MSGLVL_INFO1 ); - - fImportRslt.stopTimer(); - ostringstream oss1; - //Bulk load completed, total run time : 2.98625 seconds - oss1 << "Bulk load completed, total run time : "; - oss1 << fImportRslt.getTotalRunTime() << " seconds"; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( oss1.str(), MSGLVL_INFO1 ); - - // Even though cleanup is success, entire process is failure - fRef.onSigInterrupt(1); // therefore 1 - } - } - + } } //------------------------------------------------------------------------------ void WESDHandler::onDBRootCount(int PmId, messageqcpp::SBS& Sbs) { - ByteStream::byte aDbrCnt = 0; - (*Sbs) >> aDbrCnt; + ByteStream::byte aDbrCnt = 0; + (*Sbs) >> aDbrCnt; - if (getDebugLvl()) - cout << "No of DBRoots in PM" << PmId << " = " << (int) aDbrCnt << endl; - - if (aDbrCnt > 0) - { - fWeSplClients[PmId]->setDbRootCnt(static_cast(aDbrCnt)); - fWeSplClients[PmId]->resetDbRootVar(); - } + if (getDebugLvl()) + cout << "No of DBRoots in PM" << PmId << " = " << (int)aDbrCnt << endl; + if (aDbrCnt > 0) + { + fWeSplClients[PmId]->setDbRootCnt(static_cast(aDbrCnt)); + fWeSplClients[PmId]->resetDbRootVar(); + } } //------------------------------------------------------------------------------ void WESDHandler::doRollback() { - - std::string aAppName = "cpimport"; - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_ROLLBACK; - aBs << (ByteStream::octbyte) fTableLock; - aBs << (ByteStream::quadbyte) fTableOId; - aBs << fRef.fCmdArgs.getTableName(); - aBs << aAppName; - boost::mutex::scoped_lock aLock(fSendMutex); - send2Pm(aBs); - aLock.unlock(); - + std::string aAppName = "cpimport"; + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_ROLLBACK; + aBs << (ByteStream::octbyte)fTableLock; + aBs << (ByteStream::quadbyte)fTableOId; + aBs << fRef.fCmdArgs.getTableName(); + aBs << aAppName; + boost::mutex::scoped_lock aLock(fSendMutex); + send2Pm(aBs); + aLock.unlock(); } //------------------------------------------------------------------------------ void WESDHandler::doCleanup(bool deleteHdfsTempDbFiles) { - if (getDebugLvl()) - cout << "A cleanup is called!!" << endl; + if (getDebugLvl()) + cout << "A cleanup is called!!" << endl; - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_CLEANUP; - aBs << (ByteStream::quadbyte) fTableOId; - aBs << (ByteStream::byte) deleteHdfsTempDbFiles; - boost::mutex::scoped_lock aLock(fSendMutex); - send2Pm(aBs); - aLock.unlock(); + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_CLEANUP; + aBs << (ByteStream::quadbyte)fTableOId; + aBs << (ByteStream::byte)deleteHdfsTempDbFiles; + boost::mutex::scoped_lock aLock(fSendMutex); + send2Pm(aBs); + aLock.unlock(); } //------------------------------------------------------------------------------ bool WESDHandler::releaseTableLocks() { - if (fTableLock != 0) + if (fTableLock != 0) + { + WETableLockGrabber aTLG(*this); + + // BUG 4398. Move the call to takeSnapshot() from cpimport.bin to here + fDbrm.takeSnapshot(); + + bool aRet = aTLG.releaseTableLock(fTableLock); + + if (aRet) { - WETableLockGrabber aTLG(*this); + std::stringstream aStrStr; + aStrStr << "Released Table Lock"; + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - // BUG 4398. Move the call to takeSnapshot() from cpimport.bin to here - fDbrm.takeSnapshot(); + if (getDebugLvl()) + cout << aStrStr.str() << endl; - bool aRet = aTLG.releaseTableLock(fTableLock); + fLog.logMsg(aStrStr.str(), MSGLVL_INFO2); - if (aRet) - { - std::stringstream aStrStr; - aStrStr << "Released Table Lock"; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, - logging::M0000); - - if (getDebugLvl()) - cout << aStrStr.str() << endl; - - fLog.logMsg( aStrStr.str(), MSGLVL_INFO2 ); - - return true; - } + return true; } + } - return false; + return false; } //------------------------------------------------------------------------------ int WESDHandler::check4RollbackRslts() { - int aStatus = 1; + int aStatus = 1; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - int aRslt = fWeSplClients[aCnt]->getRollbackRslt(); + int aRslt = fWeSplClients[aCnt]->getRollbackRslt(); - if (aRslt == -1) - { - //cout << "Rollback Failed in PM - " << aCnt << endl; - aStatus = -1; - break; - } - else if (aRslt == 0) // not all results available yet - { - aStatus = 0; - break; - } - } + if (aRslt == -1) + { + // cout << "Rollback Failed in PM - " << aCnt << endl; + aStatus = -1; + break; + } + else if (aRslt == 0) // not all results available yet + { + aStatus = 0; + break; + } } + } - return aStatus; + return aStatus; } //------------------------------------------------------------------------------ int WESDHandler::check4CleanupRslts() { - int aStatus = 1; + int aStatus = 1; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - int aRslt = fWeSplClients[aCnt]->getCleanupRslt(); + int aRslt = fWeSplClients[aCnt]->getCleanupRslt(); - if (aRslt == -1) - { - //cout << "Cleanup Failed in PM - " << aCnt << endl; - aStatus = -1; - break; - } - else if (aRslt == 0) // not all results available yet - { - aStatus = 0; - break; - } - } + if (aRslt == -1) + { + // cout << "Cleanup Failed in PM - " << aCnt << endl; + aStatus = -1; + break; + } + else if (aRslt == 0) // not all results available yet + { + aStatus = 0; + break; + } } + } - return aStatus; - + return aStatus; } //------------------------------------------------------------------------------ bool WESDHandler::check4AllRollbackStatus() { - bool aStatus = true; + bool aStatus = true; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - int aRslt = fWeSplClients[aCnt]->getRollbackRslt(); + int aRslt = fWeSplClients[aCnt]->getRollbackRslt(); - if (aRslt == 0) // not all results available yet; either -1/1 - { - aStatus = false; - break; - } - } + if (aRslt == 0) // not all results available yet; either -1/1 + { + aStatus = false; + break; + } } + } - return aStatus; + return aStatus; } //------------------------------------------------------------------------------ bool WESDHandler::check4AllCleanupStatus() { - bool aStatus = true; + bool aStatus = true; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - int aRslt = fWeSplClients[aCnt]->getCleanupRslt(); + int aRslt = fWeSplClients[aCnt]->getCleanupRslt(); - if (aRslt == 0) // not all results available yet; either -1/1 - { - aStatus = false; - break; - } - } + if (aRslt == 0) // not all results available yet; either -1/1 + { + aStatus = false; + break; + } } + } - return aStatus; + return aStatus; } //------------------------------------------------------------------------------ bool WESDHandler::checkAllCpiPassStatus() { - bool aStatus = true; + bool aStatus = true; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - if (!fWeSplClients[aCnt]->isCpiPassed()) - { - if (getDebugLvl()) - cout << "CPI Pass status still False in " << aCnt << endl; + if (!fWeSplClients[aCnt]->isCpiPassed()) + { + if (getDebugLvl()) + cout << "CPI Pass status still False in " << aCnt << endl; - aStatus = false; - break; - } - } + aStatus = false; + break; + } } + } - return aStatus; + return aStatus; } // //------------------------------------------------------------------------------ bool WESDHandler::checkAllCpiFailStatus() { - bool aStatus = true; + bool aStatus = true; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - if (!fWeSplClients[aCnt]->isCpiFailed()) - { - if (getDebugLvl()) - cout << "CPI Fail status still False in " << aCnt << endl; + if (!fWeSplClients[aCnt]->isCpiFailed()) + { + if (getDebugLvl()) + cout << "CPI Fail status still False in " << aCnt << endl; - aStatus = false; - break; - } - } + aStatus = false; + break; + } } + } - return aStatus; + return aStatus; } //----------------------------------------------------------------------------- bool WESDHandler::check4AllBrmReports() { - for (int PmId = 1; PmId <= fPmCount; ++PmId) + for (int PmId = 1; PmId <= fPmCount; ++PmId) + { + if (fWeSplClients[PmId] != NULL) { - if (fWeSplClients[PmId] != NULL) - { - if (!fWeSplClients[PmId]->isBrmRptRcvd()) - { - if (getDebugLvl()) - cout << "BRMReport from " << PmId << " still not received" - << endl; + if (!fWeSplClients[PmId]->isBrmRptRcvd()) + { + if (getDebugLvl()) + cout << "BRMReport from " << PmId << " still not received" << endl; - return false; - } - } + return false; + } } + } - return true; + return true; } - //------------------------------------------------------------------------------ int WESDHandler::getNextPm2Feed() { - //int aLdPm = leastDataSendPm(); - int aLdPm = getNextDbrPm2Send(); + // int aLdPm = leastDataSendPm(); + int aLdPm = getNextDbrPm2Send(); - if (fWeSplClients[aLdPm] != 0) + if (fWeSplClients[aLdPm] != 0) + { + // Balancing the total bytes sent and Q Size + int aSz = fWeSplClients[aLdPm]->getSendQSize(); + + if (aSz > MAX_QSIZE) { - // Balancing the total bytes sent and Q Size - int aSz = fWeSplClients[aLdPm]->getSendQSize(); + // cout << "Queue Size = " << aSz << endl; + if (fpBatchLoader) + fpBatchLoader->reverseSequence(); - if (aSz > MAX_QSIZE) - { - //cout << "Queue Size = " << aSz << endl; - if (fpBatchLoader) fpBatchLoader->reverseSequence(); + aLdPm = 0; // Filled Q + } // Check enough DataRqst to send Data - aLdPm = 0; //Filled Q - } // Check enough DataRqst to send Data + if (getDebugLvl() > 2) + cout << "NextPm2Feed " << aLdPm << endl; - if (getDebugLvl() > 2) cout << "NextPm2Feed " << aLdPm << endl; + // will work only specific to mode 0 (fpBatchLoader==0) + if ((aLdPm > 0) && (!fpBatchLoader)) + fWeSplClients[aLdPm]->decDbRootVar(); + } + else if (aLdPm != 0) + { + cout << "Next PMid Error : PmId = " << aLdPm << endl; + aLdPm = 0; + } - // will work only specific to mode 0 (fpBatchLoader==0) - if ((aLdPm > 0) && (!fpBatchLoader)) fWeSplClients[aLdPm]->decDbRootVar(); - } - else if (aLdPm != 0) - { - cout << "Next PMid Error : PmId = " << aLdPm << endl; - aLdPm = 0; - } - - return aLdPm; + return aLdPm; } //------------------------------------------------------------------------------ int WESDHandler::getNextDbrPm2Send() { - unsigned int aDbrVar = 0; - unsigned int aPmId = 0; + unsigned int aDbrVar = 0; + unsigned int aPmId = 0; - if ((!fAllCpiStarted) && (1 == fRef.fCmdArgs.getMode())) + if ((!fAllCpiStarted) && (1 == fRef.fCmdArgs.getMode())) + { + check4AllCpiStarts(); + return aPmId; // Not all Cpi started. + } + + // NOTE : Implementing BatchLoader, which will be used to select the + // FIRST PM to send data and subsequent PM's to send data + if (fpBatchLoader) // for mode 1 and 2 only, since mode 0 don't have tableOID + { + if ((!fFirstDataSent) && (1 == fRef.fCmdArgs.getMode())) { - check4AllCpiStarts(); - return aPmId; // Not all Cpi started. + try + { + aPmId = fFirstPmToSend; + fFirstDataSent = true; + + if (fSelectOtherPm) + aPmId = 0; // Select the other method + + if (getDebugLvl()) + cout << "1st PM for data = " << aPmId << endl; + + fpBatchLoader->prepareForSecondPM(); + } + catch (std::exception& ex) + { + fLog.logMsg(ex.what(), MSGLVL_ERROR); + logging::Message::Args errMsgArgs; + errMsgArgs.add(ex.what()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + } } - - //NOTE : Implementing BatchLoader, which will be used to select the - // FIRST PM to send data and subsequent PM's to send data - if (fpBatchLoader) //for mode 1 and 2 only, since mode 0 don't have tableOID + else { - if ((!fFirstDataSent) && (1 == fRef.fCmdArgs.getMode())) - { - - try - { - aPmId = fFirstPmToSend; - fFirstDataSent = true; - - if (fSelectOtherPm) aPmId = 0; //Select the other method - - if (getDebugLvl()) cout << "1st PM for data = " << aPmId << endl; - - fpBatchLoader->prepareForSecondPM(); - } - catch (std::exception& ex) - { - fLog.logMsg(ex.what(), MSGLVL_ERROR ); - logging::Message::Args errMsgArgs; - errMsgArgs.add(ex.what()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - } - - } - else - { - aPmId = fpBatchLoader->selectNextPM(); - } - - if (getDebugLvl() > 2) cout << "Next PM to get data = " << aPmId << endl; - - return aPmId; - } - - //--------- The part below is for mode 0, which don't have table relation - - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) - { - - if (fWeSplClients[aCnt] != 0) - { - if (aPmId == 0) // init - { - aDbrVar = fWeSplClients[aCnt]->getDbRootVar(); - aPmId = aCnt; - } - else if (fWeSplClients[aCnt]->getDbRootVar() > aDbrVar) - { - aPmId = aCnt; - aDbrVar = fWeSplClients[aCnt]->getDbRootVar(); - } - } - - } - - if (aDbrVar == 0) - { - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) - { - if (fWeSplClients[aCnt] != 0) - { - fWeSplClients[aCnt]->resetDbRootVar(); - } - } - - aPmId = 0; + aPmId = fpBatchLoader->selectNextPM(); } if (getDebugLvl() > 2) - cout << "DbrPM2Send [" << aPmId << "] = " << aDbrVar << endl; + cout << "Next PM to get data = " << aPmId << endl; return aPmId; + } + //--------- The part below is for mode 0, which don't have table relation + + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) + { + if (aPmId == 0) // init + { + aDbrVar = fWeSplClients[aCnt]->getDbRootVar(); + aPmId = aCnt; + } + else if (fWeSplClients[aCnt]->getDbRootVar() > aDbrVar) + { + aPmId = aCnt; + aDbrVar = fWeSplClients[aCnt]->getDbRootVar(); + } + } + } + + if (aDbrVar == 0) + { + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) + { + fWeSplClients[aCnt]->resetDbRootVar(); + } + } + + aPmId = 0; + } + + if (getDebugLvl() > 2) + cout << "DbrPM2Send [" << aPmId << "] = " << aDbrVar << endl; + + return aPmId; } - //------------------------------------------------------------------------------ int WESDHandler::leastDataSendPm() { - unsigned int aTx = 0; - int aPmId = 0; + unsigned int aTx = 0; + int aPmId = 0; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - if (aPmId == 0) // init - { - aTx = fWeSplClients[aCnt]->getBytesTx(); - aPmId = aCnt; - } - else if (fWeSplClients[aCnt]->getBytesTx() < aTx) - { - aPmId = aCnt; - aTx = fWeSplClients[aCnt]->getBytesTx(); - } - } + if (aPmId == 0) // init + { + aTx = fWeSplClients[aCnt]->getBytesTx(); + aPmId = aCnt; + } + else if (fWeSplClients[aCnt]->getBytesTx() < aTx) + { + aPmId = aCnt; + aTx = fWeSplClients[aCnt]->getBytesTx(); + } } + } - return aPmId; + return aPmId; } //------------------------------------------------------------------------------ int WESDHandler::getTableOID(std::string Schema, std::string Table) { - execplan::CalpontSystemCatalog::ROPair roPair; - CalpontSystemCatalog::TableName tableName(Schema, Table); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(); - roPair = systemCatalogPtr->tableRID(tableName); + execplan::CalpontSystemCatalog::ROPair roPair; + CalpontSystemCatalog::TableName tableName(Schema, Table); + boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(); + roPair = systemCatalogPtr->tableRID(tableName); - return roPair.objnum; + return roPair.objnum; } //------------------------------------------------------------------------------ // Get the expected import binary fixed record length for the specified table. //------------------------------------------------------------------------------ -unsigned int WESDHandler::calcTableRecLen( - const std::string& schema, const std::string table) +unsigned int WESDHandler::calcTableRecLen(const std::string& schema, const std::string table) { - unsigned int recLen = 0; + unsigned int recLen = 0; - CalpontSystemCatalog::TableName tableName(schema, table); - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog(); - CalpontSystemCatalog::RIDList colRidList = - systemCatalogPtr->columnRIDs(tableName, true); - CalpontSystemCatalog::RIDList::const_iterator rid_iterator = - colRidList.begin(); + CalpontSystemCatalog::TableName tableName(schema, table); + boost::shared_ptr systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(); + CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(tableName, true); + CalpontSystemCatalog::RIDList::const_iterator rid_iterator = colRidList.begin(); - std::set colListInJobFile; - fRef.fCmdArgs.getColumnList( colListInJobFile ); - std::set::const_iterator setIter; + std::set colListInJobFile; + fRef.fCmdArgs.getColumnList(colListInJobFile); + std::set::const_iterator setIter; - // Add up the column widths to get the total record length - while (rid_iterator != colRidList.end()) + // Add up the column widths to get the total record length + while (rid_iterator != colRidList.end()) + { + CalpontSystemCatalog::ROPair roPair = *rid_iterator; + CalpontSystemCatalog::OID oid = roPair.objnum; + CalpontSystemCatalog::ColType colType; + + colType = systemCatalogPtr->colType(oid); + + // If we have a list of column names taken from an XML job file, + // then we filter against that list + if (colListInJobFile.size() > 0) { - CalpontSystemCatalog::ROPair roPair = *rid_iterator; - CalpontSystemCatalog::OID oid = roPair.objnum; - CalpontSystemCatalog::ColType colType; + CalpontSystemCatalog::TableColName colName; + colName = systemCatalogPtr->colName(oid); - colType = systemCatalogPtr->colType(oid); + setIter = colListInJobFile.find(colName.column); - // If we have a list of column names taken from an XML job file, - // then we filter against that list - if (colListInJobFile.size() > 0) - { - CalpontSystemCatalog::TableColName colName; - colName = systemCatalogPtr->colName(oid); - - setIter = colListInJobFile.find( colName.column ); - - if (setIter != colListInJobFile.end()) - { - recLen += colType.colWidth; - } - } - else - { - recLen += colType.colWidth; - } - - ++rid_iterator; + if (setIter != colListInJobFile.end()) + { + recLen += colType.colWidth; + } + } + else + { + recLen += colType.colWidth; } - if (getDebugLvl()) - cout << "Binary record length for " << schema << '.' << table << - " is: " << recLen << endl; + ++rid_iterator; + } - return recLen; + if (getDebugLvl()) + cout << "Binary record length for " << schema << '.' << table << " is: " << recLen << endl; + + return recLen; } //------------------------------------------------------------------------------ void WESDHandler::check4CpiInvokeMode() { - try + try + { + oam::oamModuleInfo_t aModInfo = fOam.getModuleInfo(); + string aModuleType = boost::get<1>(aModInfo); + + if (getDebugLvl()) + cout << "ModuleType " << aModuleType << endl; + + int aInstallType = boost::get<5>(aModInfo); + + if (getDebugLvl()) + cout << "InstallType " << aInstallType << endl; + + fOam.getSystemConfig("pm", fModuleTypeConfig); + fPmCount = fModuleTypeConfig.ModuleCount; + + if (getDebugLvl()) + cout << "PM Count " << fPmCount << endl; + + // oam::INSTALL_COMBINE_* 2,3,4 ie UM+PM in 1 Machine + if ((aInstallType > 1) && (fPmCount == 1)) { - oam::oamModuleInfo_t aModInfo = fOam.getModuleInfo(); - string aModuleType = boost::get < 1 > (aModInfo); - - if (getDebugLvl()) - cout << "ModuleType " << aModuleType << endl; - - int aInstallType = boost::get < 5 > (aModInfo); - - if (getDebugLvl()) - cout << "InstallType " << aInstallType << endl; - - fOam.getSystemConfig("pm", fModuleTypeConfig); - fPmCount = fModuleTypeConfig.ModuleCount; - - if (getDebugLvl()) - cout << "PM Count " << fPmCount << endl; - - //oam::INSTALL_COMBINE_* 2,3,4 ie UM+PM in 1 Machine - if ((aInstallType > 1) && (fPmCount == 1)) - { - //Mode arg was NOT provided; set to to default Mode 3 - if (fRef.fCmdArgs.getArgMode() == -1) - { - fRef.fCmdArgs.setMode(3); - fRef.fCmdArgs.setCpiInvoke(); - } - else if (fRef.fCmdArgs.getArgMode() == 3) //BUG 4210 - { - fRef.fCmdArgs.setCpiInvoke(); - } - else if (fRef.fCmdArgs.getArgMode() == 0) - { - throw runtime_error("Mode 0 allowed only in Multi-Nodes."); - } - - } - else if ((aInstallType == oam::INSTALL_NORMAL) - && (aModuleType == "um")) - { - //BUG 4165 - if (fRef.fCmdArgs.getMode() == 3) - { - throw runtime_error("Mode 3 imports can only be run on a PM."); - } - else if (fRef.fCmdArgs.getMode() == -1) //default mode //BUG 4210 - { - fRef.fCmdArgs.setMode(1); - } - } - else if (aModuleType == "pm") //BUG 4210 - { - //BUG 4210 - if ((fPmCount >= 1) && (fRef.fCmdArgs.getMode() == 3)) - { - fRef.fCmdArgs.setCpiInvoke(); - } - // Single node default without argument option m - else if ((fPmCount == 1) && (fRef.fCmdArgs.getArgMode() == -1)) - { - fRef.fCmdArgs.setMode(3); - fRef.fCmdArgs.setCpiInvoke(); - } // Multi-node default without argument option m - else if ((fPmCount > 1) && (fRef.fCmdArgs.getArgMode() == -1)) - { - fRef.fCmdArgs.setMode(1); - } - } - + // Mode arg was NOT provided; set to to default Mode 3 + if (fRef.fCmdArgs.getArgMode() == -1) + { + fRef.fCmdArgs.setMode(3); + fRef.fCmdArgs.setCpiInvoke(); + } + else if (fRef.fCmdArgs.getArgMode() == 3) // BUG 4210 + { + fRef.fCmdArgs.setCpiInvoke(); + } + else if (fRef.fCmdArgs.getArgMode() == 0) + { + throw runtime_error("Mode 0 allowed only in Multi-Nodes."); + } } - catch (runtime_error& exp) + else if ((aInstallType == oam::INSTALL_NORMAL) && (aModuleType == "um")) { - throw runtime_error(exp.what()); + // BUG 4165 + if (fRef.fCmdArgs.getMode() == 3) + { + throw runtime_error("Mode 3 imports can only be run on a PM."); + } + else if (fRef.fCmdArgs.getMode() == -1) // default mode //BUG 4210 + { + fRef.fCmdArgs.setMode(1); + } } - catch (...) + else if (aModuleType == "pm") // BUG 4210 { - std::string aStr = - "oam.getModuleInfo/getSystemConfig error : WESDHandler check4CpiInvoke()"; - throw runtime_error(aStr); + // BUG 4210 + if ((fPmCount >= 1) && (fRef.fCmdArgs.getMode() == 3)) + { + fRef.fCmdArgs.setCpiInvoke(); + } + // Single node default without argument option m + else if ((fPmCount == 1) && (fRef.fCmdArgs.getArgMode() == -1)) + { + fRef.fCmdArgs.setMode(3); + fRef.fCmdArgs.setCpiInvoke(); + } // Multi-node default without argument option m + else if ((fPmCount > 1) && (fRef.fCmdArgs.getArgMode() == -1)) + { + fRef.fCmdArgs.setMode(1); + } } + } + catch (runtime_error& exp) + { + throw runtime_error(exp.what()); + } + catch (...) + { + std::string aStr = "oam.getModuleInfo/getSystemConfig error : WESDHandler check4CpiInvoke()"; + throw runtime_error(aStr); + } - check4PmArguments(); - + check4PmArguments(); } //------------------------------------------------------------------------------ bool WESDHandler::check4PmArguments() { - if (fRef.fCmdArgs.getPmVecSize() > 0) + if (fRef.fCmdArgs.getPmVecSize() > 0) + { + std::vector aPmVec = fRef.fCmdArgs.getPmVec(); + int aSize = aPmVec.size(); + + for (int aIdx = 0; aIdx < aSize; aIdx++) { - std::vector aPmVec = fRef.fCmdArgs.getPmVec(); - int aSize = aPmVec.size(); - - for (int aIdx = 0; aIdx < aSize; aIdx++ ) - { - if ((fPmCount < static_cast(aPmVec[aIdx])) || (0 == aPmVec[aIdx])) - { - std::stringstream aStrStr; - aStrStr << "Invalid argument PMid " << aPmVec[aIdx] << endl; - throw runtime_error(aStrStr.str()); - } - } - + if ((fPmCount < static_cast(aPmVec[aIdx])) || (0 == aPmVec[aIdx])) + { + std::stringstream aStrStr; + aStrStr << "Invalid argument PMid " << aPmVec[aIdx] << endl; + throw runtime_error(aStrStr.str()); + } } + } - return true; + return true; } - //------------------------------------------------------------------------------ bool WESDHandler::check4AllCpiStarts() { - bool aStarted = true; + bool aStarted = true; - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) { - if (fWeSplClients[aCnt] != 0) - { - if (!fWeSplClients[aCnt]->isCpiStarted()) - aStarted = false; - } + if (!fWeSplClients[aCnt]->isCpiStarted()) + aStarted = false; } + } - if (aStarted) - fAllCpiStarted = aStarted; + if (aStarted) + fAllCpiStarted = aStarted; - return aStarted; + return aStarted; } //------------------------------------------------------------------------------ void WESDHandler::exportJobFile(std::string& JobId, std::string& JobFileName) { + if (getDebugLvl()) + cout << "JobFile Name is " << JobFileName << endl; + + ifstream aInFile; + aInFile.open(JobFileName.c_str()); + + if ((aInFile.is_open()) && (!aInFile.eof())) + { + std::stringstream aSs; + aSs << fRef.fCmdArgs.getTmpFileDir(); + aSs << "/Job_"; + aSs << JobId; + aSs << ".xml"; + + messageqcpp::ByteStream aBs; + aBs << (ByteStream::byte)WE_CLT_SRV_JOBID; + aBs << aSs.str(); + send2Pm(aBs); if (getDebugLvl()) - cout << "JobFile Name is " << JobFileName << endl; + cout << "exportJobFile::Send RmtFileName " << aSs.str() << endl; - ifstream aInFile; - aInFile.open(JobFileName.c_str()); + // Read everything to a String + std::string aData((std::istreambuf_iterator(aInFile)), std::istreambuf_iterator()); - if ((aInFile.is_open()) && (!aInFile.eof())) - { - std::stringstream aSs; - aSs << fRef.fCmdArgs.getTmpFileDir(); - aSs << "/Job_"; - aSs << JobId; - aSs << ".xml"; - - messageqcpp::ByteStream aBs; - aBs << (ByteStream::byte) WE_CLT_SRV_JOBID; - aBs << aSs.str(); - send2Pm(aBs); - - if (getDebugLvl()) - cout << "exportJobFile::Send RmtFileName " << aSs.str() << endl; - - // Read everything to a String - std::string aData((std::istreambuf_iterator(aInFile)), - std::istreambuf_iterator()); - - if (getDebugLvl()) - cout << "Sending XML FileData " << aData << endl; - - aBs.restart(); - aBs << (ByteStream::byte) WE_CLT_SRV_JOBDATA; - aBs << aData; - send2Pm(aBs); - - } - else - { - throw runtime_error("unable to open Job File"); - } + if (getDebugLvl()) + cout << "Sending XML FileData " << aData << endl; + aBs.restart(); + aBs << (ByteStream::byte)WE_CLT_SRV_JOBDATA; + aBs << aData; + send2Pm(aBs); + } + else + { + throw runtime_error("unable to open Job File"); + } } //------------------------------------------------------------------------------ bool WESDHandler::getConsoleLog() { - return fRef.fCmdArgs.getConsoleLog(); + return fRef.fCmdArgs.getConsoleLog(); } //------------------------------------------------------------------------------ char WESDHandler::getEnclChar() { - return fRef.fCmdArgs.getEnclChar(); + return fRef.fCmdArgs.getEnclChar(); } //------------------------------------------------------------------------------ char WESDHandler::getEscChar() { - return fRef.fCmdArgs.getEscChar(); + return fRef.fCmdArgs.getEscChar(); } //------------------------------------------------------------------------------ int WESDHandler::getReadBufSize() { - return fRef.fCmdArgs.getReadBufSize(); + return fRef.fCmdArgs.getReadBufSize(); } //------------------------------------------------------------------------------ char WESDHandler::getDelimChar() { - return fRef.fCmdArgs.getDelimChar(); + return fRef.fCmdArgs.getDelimChar(); } //------------------------------------------------------------------------------ std::string WESDHandler::getTableName() const { - return fRef.fCmdArgs.getTableName(); + return fRef.fCmdArgs.getTableName(); } //------------------------------------------------------------------------------ std::string WESDHandler::getSchemaName() const { - return fRef.fCmdArgs.getSchemaName(); + return fRef.fCmdArgs.getSchemaName(); } ImportDataMode WESDHandler::getImportDataMode() const { - return fRef.fCmdArgs.getImportDataMode(); + return fRef.fCmdArgs.getImportDataMode(); } //------------------------------------------------------------------------------ -void WESDHandler::sysLog(const logging::Message::Args& msgArgs, - logging::LOG_TYPE logType, logging::Message::MessageID msgId) +void WESDHandler::sysLog(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType, + logging::Message::MessageID msgId) { - fRef.fpSysLog->logMsg(msgArgs, logType, msgId); + fRef.fpSysLog->logMsg(msgArgs, logType, msgId); } //------------------------------------------------------------------------------ std::string WESDHandler::getTime2Str() const { - char aBuff[64]; - time_t aTime; - struct tm pTm; - time(&aTime); - localtime_r(&aTime, &pTm); + char aBuff[64]; + time_t aTime; + struct tm pTm; + time(&aTime); + localtime_r(&aTime, &pTm); - // M D H M S - snprintf(aBuff, sizeof(aBuff), "%02d%02d%02d%02d%02d", - pTm.tm_mon + 1, pTm.tm_mday, pTm.tm_hour, - pTm.tm_min, pTm.tm_sec); + // M D H M S + snprintf(aBuff, sizeof(aBuff), "%02d%02d%02d%02d%02d", pTm.tm_mon + 1, pTm.tm_mday, pTm.tm_hour, pTm.tm_min, + pTm.tm_sec); - return aBuff; + return aBuff; } //------------------------------------------------------------------------------ void WESDHandler::setInputFileList(std::string InFileName) { - fFileReadThread.chkForListOfFiles(InFileName); + fFileReadThread.chkForListOfFiles(InFileName); } //------------------------------------------------------------------------------ void WESDHandler::onHandlingSignal() { - std::stringstream aStrStr; - char aDefCon[16], aRedCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); - aStrStr << aRedCol << "Received signal to terminate the process." << aDefCon; + std::stringstream aStrStr; + char aDefCon[16], aRedCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); + aStrStr << aRedCol << "Received signal to terminate the process." << aDefCon; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr.str(), MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr.str(), MSGLVL_INFO1); - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - std::stringstream aStrStr1; - aStrStr1 << "Handling signal ......"; + std::stringstream aStrStr1; + aStrStr1 << "Handling signal ......"; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr1.str(), MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr1.str(), MSGLVL_INFO1); + + fRef.fSignaled = false; + bool aTblLockReleased = false; + bool aRollbackSuccess = false; + onCpimportFail(0, true); + usleep(2000000 * fPmCount); + + // BUG 4649 - Some systems taking too long to finish the process. + // So we have to wait some more time. + std::stringstream aStrStr2; + aStrStr2 << "Rolling back .........."; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr2.str(), MSGLVL_INFO1); + + for (int aIdx = 0; aIdx < 60; aIdx++) + { + int aStatus = check4RollbackRslts(); + + if (1 == aStatus) + { + if (getDebugLvl()) + cout << "Rollback Successful... " << endl; + + aRollbackSuccess = true; + break; + } + else if (-1 == aStatus) + { + if (getDebugLvl()) + cout << "Rollback Failed... " << endl; + + break; + } - fRef.fSignaled = false; - bool aTblLockReleased = false; - bool aRollbackSuccess = false; - onCpimportFail(0, true); usleep(2000000 * fPmCount); + } - //BUG 4649 - Some systems taking too long to finish the process. - // So we have to wait some more time. - std::stringstream aStrStr2; - aStrStr2 << "Rolling back .........."; + // Bug 5774 - if rollback failed, leave the tablelock + if (!aRollbackSuccess) + { + std::stringstream aStrStr2a; + aStrStr2a << "Rollback Failed; Leaving Tablelock ... "; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr2.str(), MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr2a.str(), MSGLVL_INFO1); - for (int aIdx = 0; aIdx < 60; aIdx++) + return; + } + + std::stringstream aStrStr3; + aStrStr3 << "Cleaning up .........."; + + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr3.str(), MSGLVL_INFO1); + + for (int aIdx = 0; aIdx < 60; aIdx++) + { + int aStatus = check4CleanupRslts(); + + if (aStatus == 1) { - int aStatus = check4RollbackRslts(); + if (getDebugLvl()) + cout << "Cleanup Successful... " << endl; - if (1 == aStatus) - { - if (getDebugLvl()) cout << "Rollback Successful... " << endl; - - aRollbackSuccess = true; - break; - } - else if (-1 == aStatus) - { - if (getDebugLvl()) cout << "Rollback Failed... " << endl; - - break; - } - - usleep(2000000 * fPmCount); + releaseTableLocks(); + aTblLockReleased = true; + break; } - //Bug 5774 - if rollback failed, leave the tablelock - if (!aRollbackSuccess) - { - std::stringstream aStrStr2a; - aStrStr2a << "Rollback Failed; Leaving Tablelock ... "; + usleep(2000000 * fPmCount); + } - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr2a.str(), MSGLVL_INFO1 ); - - return; - } - - std::stringstream aStrStr3; - aStrStr3 << "Cleaning up .........."; - - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr3.str(), MSGLVL_INFO1 ); - - for (int aIdx = 0; aIdx < 60; aIdx++) - { - int aStatus = check4CleanupRslts(); - - if (aStatus == 1) - { - if (getDebugLvl()) cout << "Cleanup Successful... " << endl; - - releaseTableLocks(); - aTblLockReleased = true; - break; - } - - usleep(2000000 * fPmCount); - } - - if ((!aTblLockReleased) && (aRollbackSuccess)) - { - releaseTableLocks(); - } + if ((!aTblLockReleased) && (aRollbackSuccess)) + { + releaseTableLocks(); + } } //------------------------------------------------------------------------------ - void WESDHandler::onHandlingSigHup() { - std::stringstream aStrStr; - char aDefCon[16], aRedCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); - aStrStr << aRedCol << "Interrupt received .... Program exiting." << aDefCon; + std::stringstream aStrStr; + char aDefCon[16], aRedCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); + aStrStr << aRedCol << "Interrupt received .... Program exiting." << aDefCon; - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr.str(), MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr.str(), MSGLVL_INFO1); - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStrStr.str()); - fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - - - fRef.fSigHup = false; - //onCpimportFail(0, true); - cancelOutstandingCpimports will hang on send() - //cancelOutstandingCpimports(); - exit(1); // Hard exit on SIGHUP signal + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStrStr.str()); + fRef.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + fRef.fSigHup = false; + // onCpimportFail(0, true); - cancelOutstandingCpimports will hang on send() + // cancelOutstandingCpimports(); + exit(1); // Hard exit on SIGHUP signal } //------------------------------------------------------------------------------ void WESDHandler::onDisconnectFailure() { - string aStr("Trying to reconnect and rollback"); + string aStr("Trying to reconnect and rollback"); - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStr, MSGLVL_INFO1 ); + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStr, MSGLVL_INFO1); - for (int aSec = 0; aSec < 15; aSec++ ) + for (int aSec = 0; aSec < 15; aSec++) + { + bool aDisconnect = false; + usleep(1000000); + + for (int PmId = 1; PmId <= fPmCount; ++PmId) { - bool aDisconnect = false; - usleep(1000000); - - for (int PmId = 1; PmId <= fPmCount; ++PmId) + if (fWeSplClients[PmId] != 0) + { + if (!fWeSplClients[PmId]->isConnected()) { - if (fWeSplClients[PmId] != 0) - { - if (!fWeSplClients[PmId]->isConnected()) - { - aDisconnect = true; + aDisconnect = true; - try - { - fWeSplClients[PmId]->setup(); - } - catch (runtime_error&) - { - cout << "Unable to connect to PM" << - PmId << "; Trying again..." << endl; - } - } - } + try + { + fWeSplClients[PmId]->setup(); + } + catch (runtime_error&) + { + cout << "Unable to connect to PM" << PmId << "; Trying again..." << endl; + } } - - if (!aDisconnect) break; + } } - doRollback(); + if (!aDisconnect) + break; + } - bool aTblLockReleased = false; - bool aRollbackSuccess = true; - //BUG 4649 - Some systems taking too long to finish the process. - // So we have to wait some more time. - std::stringstream aStrStr2; - aStrStr2 << "Rolling back .........."; + doRollback(); - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr2.str(), MSGLVL_INFO1 ); + bool aTblLockReleased = false; + bool aRollbackSuccess = true; + // BUG 4649 - Some systems taking too long to finish the process. + // So we have to wait some more time. + std::stringstream aStrStr2; + aStrStr2 << "Rolling back .........."; - for (int aIdx = 0; aIdx < 10; aIdx++) + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr2.str(), MSGLVL_INFO1); + + for (int aIdx = 0; aIdx < 10; aIdx++) + { + int aStatus = check4RollbackRslts(); + + if (1 == aStatus) { - int aStatus = check4RollbackRslts(); + if (getDebugLvl()) + cout << "Rollback Successful... " << endl; - if (1 == aStatus) - { - if (getDebugLvl()) cout << "Rollback Successful... " << endl; + break; + } + else if (-1 == aStatus) + { + if (getDebugLvl()) + cout << "Rollback Failed... " << endl; - break; - } - else if (-1 == aStatus) - { - if (getDebugLvl()) cout << "Rollback Failed... " << endl; - - aRollbackSuccess = false; - break; - } - - usleep(2000000 * fPmCount); + aRollbackSuccess = false; + break; } - std::stringstream aStrStr3; - aStrStr3 << "Cleaning up .........."; + usleep(2000000 * fPmCount); + } - if ( fRef.fCmdArgs.getConsoleOutput()) - fLog.logMsg( aStrStr3.str(), MSGLVL_INFO1 ); + std::stringstream aStrStr3; + aStrStr3 << "Cleaning up .........."; - for (int aIdx = 0; aIdx < 10; aIdx++) + if (fRef.fCmdArgs.getConsoleOutput()) + fLog.logMsg(aStrStr3.str(), MSGLVL_INFO1); + + for (int aIdx = 0; aIdx < 10; aIdx++) + { + int aStatus = check4CleanupRslts(); + + if (aStatus == 1) { - int aStatus = check4CleanupRslts(); + if (getDebugLvl()) + cout << "Cleanup Successful... " << endl; - if (aStatus == 1) - { - if (getDebugLvl()) cout << "Cleanup Successful... " << endl; - - if (aRollbackSuccess) - { - releaseTableLocks(); - aTblLockReleased = true; - } - - break; - } - - usleep(2000000 * fPmCount); - } - - if ((!aTblLockReleased) && (aRollbackSuccess)) - { + if (aRollbackSuccess) + { releaseTableLocks(); + aTblLockReleased = true; + } + + break; } + usleep(2000000 * fPmCount); + } + + if ((!aTblLockReleased) && (aRollbackSuccess)) + { + releaseTableLocks(); + } } //------------------------------------------------------------------------------ void WESDHandler::setDisconnectFailure(bool Flag) { - if (fFileReadThread.isContinue()) //check already stopped running - { - sendEODMsg(); - fFileReadThread.shutdown(); - } + if (fFileReadThread.isContinue()) // check already stopped running + { + sendEODMsg(); + fFileReadThread.shutdown(); + } - fDisconnectFailure = Flag; - fRef.onSigInterrupt(1); // process altogether is a failure + fDisconnectFailure = Flag; + fRef.onSigInterrupt(1); // process altogether is a failure } //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_sdhandler.h b/writeengine/splitter/we_sdhandler.h index e8d624e1b..318b199c6 100644 --- a/writeengine/splitter/we_sdhandler.h +++ b/writeengine/splitter/we_sdhandler.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_spltrdatahandler.h @@ -30,7 +30,6 @@ #ifndef WE_SPLITTERDATAHANDLER_H_ #define WE_SPLITTERDATAHANDLER_H_ - #include "liboamcpp.h" #include "resourcemanager.h" #include "threadsafequeue.h" @@ -42,11 +41,9 @@ #include "we_filereadthread.h" #include "we_splclient.h" - namespace WriteEngine { - -class WESplitterApp; //forward declaration +class WESplitterApp; // forward declaration class WESplClient; class WEFileReadThread; @@ -56,302 +53,308 @@ class WEFileReadThread; class WEPmList { -public: - WEPmList(): fPmList(), fListMutex() {} - virtual ~WEPmList() - { - fPmList.clear(); - } + public: + WEPmList() : fPmList(), fListMutex() + { + } + virtual ~WEPmList() + { + fPmList.clear(); + } - void addPm2List(int PmId); - void addPriorityPm2List(int PmId); - int getNextPm(); - void clearPmList(); - bool check4Pm(int PmId); - -private: - typedef std::list WePmList; // List to add in front - WePmList fPmList; - boost::mutex fListMutex; //mutex controls add/remove + void addPm2List(int PmId); + void addPriorityPm2List(int PmId); + int getNextPm(); + void clearPmList(); + bool check4Pm(int PmId); + private: + typedef std::list WePmList; // List to add in front + WePmList fPmList; + boost::mutex fListMutex; // mutex controls add/remove }; - //------------------------------------------------------------------------------ class WESDHandler { + public: + WESDHandler(WESplitterApp& Ref); + WESDHandler(const WESDHandler& rhs); + virtual ~WESDHandler(); -public: - WESDHandler(WESplitterApp& Ref); - WESDHandler(const WESDHandler& rhs); - virtual ~WESDHandler(); + void setup(); + void shutdown(); + void reset(); + void send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId = 0); + void send2Pm(messageqcpp::ByteStream& Bs, unsigned int PmId = 0); + void sendEODMsg(); + void checkForRespMsgs(); + void add2RespQueue(const messageqcpp::SBS& Sbs); + void exportJobFile(std::string& JobId, std::string& JobFileName); + int leastDataSendPm(); + bool check4AllBrmReports(); + bool updateCPAndHWMInBRM(); + void cancelOutstandingCpimports(); + void doRollback(); + void doCleanup(bool deleteHdfsTempDbFiles); + void getErrorLog(int PmId, const std::string& ErrFileName); + void getBadLog(int PmId, const std::string& BadFileName); + int check4RollbackRslts(); + bool check4AllRollbackStatus(); + int check4CleanupRslts(); + bool check4AllCleanupStatus(); + bool check4AllCpiStarts(); + bool releaseTableLocks(); + void check4CpiInvokeMode(); + bool check4PmArguments(); + void setInputFileList(std::string InFileName); + bool check4CriticalErrMsgs(std::string& Entry); - void setup(); - void shutdown(); - void reset(); - void send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId = 0); - void send2Pm(messageqcpp::ByteStream& Bs, unsigned int PmId = 0); - void sendEODMsg(); - void checkForRespMsgs(); - void add2RespQueue(const messageqcpp::SBS& Sbs); - void exportJobFile(std::string& JobId, std::string& JobFileName); - int leastDataSendPm(); - bool check4AllBrmReports(); - bool updateCPAndHWMInBRM(); - void cancelOutstandingCpimports(); - void doRollback(); - void doCleanup(bool deleteHdfsTempDbFiles); - void getErrorLog(int PmId, const std::string& ErrFileName); - void getBadLog(int PmId, const std::string& BadFileName); - int check4RollbackRslts(); - bool check4AllRollbackStatus(); - int check4CleanupRslts(); - bool check4AllCleanupStatus(); - bool check4AllCpiStarts(); - bool releaseTableLocks(); - void check4CpiInvokeMode(); - bool check4PmArguments(); - void setInputFileList(std::string InFileName); - bool check4CriticalErrMsgs(std::string& Entry); + void onStartCpiResponse(int PmId); + void onDataRqstResponse(int PmId); + void onAckResponse(int PmId); + void onNakResponse(int PmId); + void onEodResponse(int Pmid); + void onPmErrorResponse(int PmId); + void onKeepAliveMessage(int PmId); + void onCpimportPass(int PmId); + void onCpimportFail(int PmId, bool SigHandle = false); + void onImpFileError(int PmId); + void onBrmReport(int PmId, messageqcpp::SBS& Sbs); + void onErrorFile(int PmId, messageqcpp::SBS& Sbs); + void onBadFile(int PmId, messageqcpp::SBS& Sbs); + void onRollbackResult(int PmId, messageqcpp::SBS& Sbs); + void onCleanupResult(int PmId, messageqcpp::SBS& Sbs); + void onDBRootCount(int PmId, messageqcpp::SBS& Sbs); + void onHandlingSignal(); + void onHandlingSigHup(); + void onDisconnectFailure(); - void onStartCpiResponse(int PmId); - void onDataRqstResponse(int PmId); - void onAckResponse(int PmId); - void onNakResponse(int PmId); - void onEodResponse(int Pmid); - void onPmErrorResponse(int PmId); - void onKeepAliveMessage(int PmId); - void onCpimportPass(int PmId); - void onCpimportFail(int PmId, bool SigHandle = false); - void onImpFileError(int PmId); - void onBrmReport(int PmId, messageqcpp::SBS& Sbs); - void onErrorFile(int PmId, messageqcpp::SBS& Sbs); - void onBadFile(int PmId, messageqcpp::SBS& Sbs); - void onRollbackResult(int PmId, messageqcpp::SBS& Sbs); - void onCleanupResult(int PmId, messageqcpp::SBS& Sbs); - void onDBRootCount(int PmId, messageqcpp::SBS& Sbs); - void onHandlingSignal(); - void onHandlingSigHup(); - void onDisconnectFailure(); + int getNextPm2Feed(); + int getNextDbrPm2Send(); + int getTableOID(std::string Schema, std::string Table); + std::string getTime2Str() const; - int getNextPm2Feed(); - int getNextDbrPm2Send(); - int getTableOID(std::string Schema, std::string Table); - std::string getTime2Str() const; + bool checkAllCpiPassStatus(); + bool checkAllCpiFailStatus(); + bool checkForRollbackAndCleanup(); + bool checkForCpiFailStatus(); - bool checkAllCpiPassStatus(); - bool checkAllCpiFailStatus(); - bool checkForRollbackAndCleanup(); - bool checkForCpiFailStatus(); + void checkForConnections(); + void sendHeartbeats(); + std::string getTableName() const; + std::string getSchemaName() const; + char getEnclChar(); + char getEscChar(); + char getDelimChar(); + bool getConsoleLog(); + int getReadBufSize(); + ImportDataMode getImportDataMode() const; + void sysLog(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType, + logging::Message::MessageID msgId); - void checkForConnections(); - void sendHeartbeats(); - std::string getTableName() const; - std::string getSchemaName() const; - char getEnclChar(); - char getEscChar(); - char getDelimChar(); - bool getConsoleLog(); - int getReadBufSize(); - ImportDataMode getImportDataMode() const; - void sysLog(const logging::Message::Args& msgArgs, - logging::LOG_TYPE logType, logging::Message::MessageID msgId); + boost::thread* getFpRespThread() const + { + return fpRespThread; + } + unsigned int getQId() const + { + return fQId; + } + void setFpRespThread(boost::thread* pRespThread) + { + fpRespThread = pRespThread; + } + void setQId(unsigned int QId) + { + fQId = QId; + } + bool isContinue() const + { + return fContinue; + } + void setContinue(bool Continue) + { + fContinue = Continue; + } + int getPmCount() const + { + return fPmCount; + } + void setPmCount(int PmCount) + { + fPmCount = PmCount; + } + int getNextPm2Send() + { + return fDataFeedList.getNextPm(); + } + bool check4Ack(unsigned int PmId) + { + return fDataFeedList.check4Pm(PmId); + } + int getTableOID() + { + return fTableOId; + } + void setDebugLvl(int DebugLvl) + { + fDebugLvl = DebugLvl; + } + int getDebugLvl() + { + return fDebugLvl; + } + unsigned int getTableRecLen() const + { + return fFixedBinaryRecLen; + } + void updateRowTx(unsigned int RowCnt, int CIdx) + { + fWeSplClients[CIdx]->updateRowTx(RowCnt); + } + void resetRowTx() + { + for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + { + if (fWeSplClients[aCnt] != 0) + { + fWeSplClients[aCnt]->resetRowTx(); + } + } + } + void setRowsUploadInfo(int PmId, int64_t RowsRead, int64_t RowsInserted) + { + fWeSplClients[PmId]->setRowsUploadInfo(RowsRead, RowsInserted); + } + void add2ColOutOfRangeInfo(int PmId, int ColNum, execplan::CalpontSystemCatalog::ColDataType ColType, + std::string& ColName, int NoOfOors) + { + fWeSplClients[PmId]->add2ColOutOfRangeInfo(ColNum, ColType, ColName, NoOfOors); + } + void setErrorFileName(int PmId, const std::string& ErrFileName) + { + fWeSplClients[PmId]->setErrInfoFile(ErrFileName); + } + void setBadFileName(int PmId, const std::string& BadFileName) + { + fWeSplClients[PmId]->setBadDataFile(BadFileName); + } + void setDisconnectFailure(bool Flag); + bool getDisconnectFailure() + { + return fDisconnectFailure; + } - boost::thread* getFpRespThread() const + public: // for multi-table support + WESplitterApp& fRef; + Log fLog; // logger + + private: + unsigned int fQId; + joblist::ResourceManager* fRm; + oam::Oam fOam; + oam::ModuleTypeConfig fModuleTypeConfig; + int fDebugLvl; + int fPmCount; + + int64_t fTableLock; + int32_t fTableOId; + uint32_t fFixedBinaryRecLen; + + boost::mutex fRespMutex; + boost::condition fRespCond; + + boost::mutex fSendMutex; + + // It could be a queue too. Stores all the responses from PMs + typedef std::list WESRespList; + WESRespList fRespList; + // Other member variables + boost::thread* fpRespThread; + + WEPmList fDataFeedList; + WEFileReadThread fFileReadThread; + + bool fDisconnectFailure; // Failure due to disconnect from PM + bool fForcedFailure; + bool fAllCpiStarted; + bool fFirstDataSent; + unsigned int fFirstPmToSend; + bool fSelectOtherPm; // Don't send first data to First PM + bool fContinue; + // set of PM specific vector entries + typedef std::vector WESplClients; + WESplClients fWeSplClients; + enum + { + MAX_PMS = 512, + MAX_QSIZE = 10, + MAX_WES_QSIZE = 100 + }; + + typedef std::vector StrVec; + StrVec fBrmRptVec; + + BRM::DBRM fDbrm; + + batchloader::BatchLoader* fpBatchLoader; + + unsigned int calcTableRecLen(const std::string& schema, const std::string table); + + class WEImportRslt + { + public: + WEImportRslt() : fRowsPro(0), fRowsIns(0), fStartTime(), fEndTime(), fTotTime(0) { - return fpRespThread; } - unsigned int getQId() const + ~WEImportRslt() { - return fQId; } - void setFpRespThread(boost::thread* pRespThread) + + public: + void reset() { - fpRespThread = pRespThread; + fRowsPro = 0; + fRowsIns = 0; + fTotTime = 0; + fColOorVec.clear(); } - void setQId(unsigned int QId) + void updateRowsProcessed(int64_t Rows) { - fQId = QId; + fRowsPro += Rows; } - bool isContinue() const + void updateRowsInserted(int64_t Rows) { - return fContinue; + fRowsIns += Rows; } - void setContinue(bool Continue) + void updateColOutOfRangeInfo(int aColNum, execplan::CalpontSystemCatalog::ColDataType aColType, + std::string aColName, int aNoOfOors) { - fContinue = Continue; - } - int getPmCount() const - { - return fPmCount; - } - void setPmCount(int PmCount) - { - fPmCount = PmCount; - } - int getNextPm2Send() - { - return fDataFeedList.getNextPm(); - } - bool check4Ack(unsigned int PmId) - { - return fDataFeedList.check4Pm(PmId); - } - int getTableOID() - { - return fTableOId; - } - void setDebugLvl(int DebugLvl) - { - fDebugLvl = DebugLvl; - } - int getDebugLvl() - { - return fDebugLvl; - } - unsigned int getTableRecLen() const - { - return fFixedBinaryRecLen; - } - void updateRowTx(unsigned int RowCnt, int CIdx) - { - fWeSplClients[CIdx]->updateRowTx(RowCnt); - } - void resetRowTx() - { - for (int aCnt = 1; aCnt <= fPmCount; aCnt++) + WEColOorVec::iterator aIt = fColOorVec.begin(); + + while (aIt != fColOorVec.end()) + { + if ((*aIt).fColNum == aColNum) { - if (fWeSplClients[aCnt] != 0) - { - fWeSplClients[aCnt]->resetRowTx(); - } + (*aIt).fNoOfOORs += aNoOfOors; + break; } - } - void setRowsUploadInfo(int PmId, int64_t RowsRead, int64_t RowsInserted) - { - fWeSplClients[PmId]->setRowsUploadInfo(RowsRead, RowsInserted); - } - void add2ColOutOfRangeInfo(int PmId, int ColNum, - execplan::CalpontSystemCatalog::ColDataType ColType, - std::string& ColName, int NoOfOors) - { - fWeSplClients[PmId]->add2ColOutOfRangeInfo(ColNum, ColType, ColName, NoOfOors); - } - void setErrorFileName(int PmId, const std::string& ErrFileName) - { - fWeSplClients[PmId]->setErrInfoFile(ErrFileName); - } - void setBadFileName(int PmId, const std::string& BadFileName) - { - fWeSplClients[PmId]->setBadDataFile(BadFileName); - } - void setDisconnectFailure(bool Flag); - bool getDisconnectFailure() - { - return fDisconnectFailure; - } + aIt++; + } -public: // for multi-table support - WESplitterApp& fRef; - Log fLog; // logger - -private: - unsigned int fQId; - joblist::ResourceManager* fRm; - oam::Oam fOam; - oam::ModuleTypeConfig fModuleTypeConfig; - int fDebugLvl; - int fPmCount; - - int64_t fTableLock; - int32_t fTableOId; - uint32_t fFixedBinaryRecLen; - - boost::mutex fRespMutex; - boost::condition fRespCond; - - boost::mutex fSendMutex; - - // It could be a queue too. Stores all the responses from PMs - typedef std::list WESRespList; - WESRespList fRespList; - // Other member variables - boost::thread* fpRespThread; - - WEPmList fDataFeedList; - WEFileReadThread fFileReadThread; - - bool fDisconnectFailure; //Failure due to disconnect from PM - bool fForcedFailure; - bool fAllCpiStarted; - bool fFirstDataSent; - unsigned int fFirstPmToSend; - bool fSelectOtherPm; // Don't send first data to First PM - bool fContinue; - // set of PM specific vector entries - typedef std::vector WESplClients; - WESplClients fWeSplClients; - enum { MAX_PMS = 512, MAX_QSIZE = 10, MAX_WES_QSIZE = 100}; - - typedef std::vector StrVec; - StrVec fBrmRptVec; - - BRM::DBRM fDbrm; - - batchloader::BatchLoader* fpBatchLoader; - - unsigned int calcTableRecLen(const std::string& schema, - const std::string table); - - class WEImportRslt - { - public: - WEImportRslt(): fRowsPro(0), fRowsIns(0), fStartTime(), fEndTime(), fTotTime(0) {} - ~WEImportRslt() {} - public: - void reset() - { - fRowsPro = 0; - fRowsIns = 0; - fTotTime = 0; - fColOorVec.clear(); - } - void updateRowsProcessed(int64_t Rows) - { - fRowsPro += Rows; - } - void updateRowsInserted(int64_t Rows) - { - fRowsIns += Rows; - } - void updateColOutOfRangeInfo(int aColNum, execplan::CalpontSystemCatalog::ColDataType aColType, - std::string aColName, int aNoOfOors) - { - WEColOorVec::iterator aIt = fColOorVec.begin(); - - while (aIt != fColOorVec.end()) - { - if ((*aIt).fColNum == aColNum) - { - (*aIt).fNoOfOORs += aNoOfOors; - break; - } - - aIt++; - } - - if (aIt == fColOorVec.end()) - { - // First time for aColNum to have out of range count - WEColOORInfo aColOorInfo; - aColOorInfo.fColNum = aColNum; - aColOorInfo.fColType = aColType; - aColOorInfo.fColName = aColName; - aColOorInfo.fNoOfOORs = aNoOfOors; - fColOorVec.push_back(aColOorInfo); - } + if (aIt == fColOorVec.end()) + { + // First time for aColNum to have out of range count + WEColOORInfo aColOorInfo; + aColOorInfo.fColNum = aColNum; + aColOorInfo.fColType = aColType; + aColOorInfo.fColName = aColName; + aColOorInfo.fNoOfOORs = aNoOfOors; + fColOorVec.push_back(aColOorInfo); + } #if 0 @@ -370,40 +373,39 @@ private: } #endif - } - void startTimer() - { - gettimeofday( &fStartTime, 0 ); - } - void stopTimer() - { - gettimeofday( &fEndTime, 0 ); - } - float getTotalRunTime() - { - //fTotTime = (fEndTime>0)?(fEndTime-fStartTime):0; - fTotTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) - - (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); - return fTotTime; - } + } + void startTimer() + { + gettimeofday(&fStartTime, 0); + } + void stopTimer() + { + gettimeofday(&fEndTime, 0); + } + float getTotalRunTime() + { + // fTotTime = (fEndTime>0)?(fEndTime-fStartTime):0; + fTotTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) - + (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); + return fTotTime; + } - public: - int64_t fRowsPro; //Rows processed - int64_t fRowsIns; //Rows inserted - timeval fStartTime; //StartTime - timeval fEndTime; //EndTime - float fTotTime; //TotalTime - // A vector containing a list of rows and counts of Out of Range values - WEColOorVec fColOorVec; - }; - WEImportRslt fImportRslt; - - friend class WESplClient; - friend class WEBrmUpdater; - friend class WESplitterApp; - friend class WEFileReadThread; - friend class WETableLockGrabber; + public: + int64_t fRowsPro; // Rows processed + int64_t fRowsIns; // Rows inserted + timeval fStartTime; // StartTime + timeval fEndTime; // EndTime + float fTotTime; // TotalTime + // A vector containing a list of rows and counts of Out of Range values + WEColOorVec fColOorVec; + }; + WEImportRslt fImportRslt; + friend class WESplClient; + friend class WEBrmUpdater; + friend class WESplitterApp; + friend class WEFileReadThread; + friend class WETableLockGrabber; }; //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_splclient.cpp b/writeengine/splitter/we_splclient.cpp index bd51a042c..8bcbdcd56 100644 --- a/writeengine/splitter/we_splclient.cpp +++ b/writeengine/splitter/we_splclient.cpp @@ -17,9 +17,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_splclient.cpp @@ -55,428 +55,413 @@ using namespace oam; namespace WriteEngine { - //------------------------------------------------------------------------------ -//BP 10/24/2011 14:25 +// BP 10/24/2011 14:25 //------------------------------------------------------------------------------ void WESplClientRunner::operator()() { - fOwner.sendAndRecv(); + fOwner.sendAndRecv(); } //------------------------------------------------------------------------------ -//BP 10/24/2011 14:25 +// BP 10/24/2011 14:25 //------------------------------------------------------------------------------ - -WESplClient::WESplClient(WESDHandler& Sdh, int PmId): - fContinue(true), - fConnected(false), - fPmId(PmId), - fDbrCnt(0), - fDbrVar(0), - fDataRqstCnt(0), - fRdSecTo(0), - fRowTx(0), - fBytesTx(0), - fBytesRcv(0), - fLastInTime(0), - fStartTime(time(0)), - fSend(true), - fCpiStarted(false), - fCpiPassed(false), - fCpiFailed(false), - fBrmRptRcvd(false), - fRollbackRslt(0), - fCleanupRslt(0), - fServer(), - fClnt(), - fpThread(0), - fOwner(Sdh) +WESplClient::WESplClient(WESDHandler& Sdh, int PmId) + : fContinue(true) + , fConnected(false) + , fPmId(PmId) + , fDbrCnt(0) + , fDbrVar(0) + , fDataRqstCnt(0) + , fRdSecTo(0) + , fRowTx(0) + , fBytesTx(0) + , fBytesRcv(0) + , fLastInTime(0) + , fStartTime(time(0)) + , fSend(true) + , fCpiStarted(false) + , fCpiPassed(false) + , fCpiFailed(false) + , fBrmRptRcvd(false) + , fRollbackRslt(0) + , fCleanupRslt(0) + , fServer() + , fClnt() + , fpThread(0) + , fOwner(Sdh) { - // TODO ctor + // TODO ctor } //------------------------------------------------------------------------------ WESplClient::~WESplClient() { - delete fpThread; - fpThread = 0; + delete fpThread; + fpThread = 0; } //------------------------------------------------------------------------------ void WESplClient::setup() { - // do the setup stuffs here - if (fOwner.getDebugLvl()) - cout << "setting connection to moduleid " << getPmId() << endl; + // do the setup stuffs here + if (fOwner.getDebugLvl()) + cout << "setting connection to moduleid " << getPmId() << endl; - char buff[32]; - snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", getPmId()); - fServer = buff; + char buff[32]; + snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", getPmId()); + fServer = buff; - fClnt.reset(new MessageQueueClient(fServer)); + fClnt.reset(new MessageQueueClient(fServer)); - if (fOwner.getDebugLvl()) - cout << "WEServer : " << fServer << " " << fClnt->addr2String() << endl; + if (fOwner.getDebugLvl()) + cout << "WEServer : " << fServer << " " << fClnt->addr2String() << endl; - try + try + { + if (fClnt->connect()) { - if (fClnt->connect()) - { - onConnect(); - startClientThread(); - } - else - { - throw runtime_error("Connection refused"); - } + onConnect(); + startClientThread(); } - catch (std::exception& ex) + else { - cerr << "Could not connect to " << fServer << ": " << ex.what() << endl; - throw runtime_error("Problem in connecting to PM"); + throw runtime_error("Connection refused"); } - catch (...) - { - cerr << "Could not connect to " << fServer << endl; - throw runtime_error("Problem in connecting to PM"); - } - - + } + catch (std::exception& ex) + { + cerr << "Could not connect to " << fServer << ": " << ex.what() << endl; + throw runtime_error("Problem in connecting to PM"); + } + catch (...) + { + cerr << "Could not connect to " << fServer << endl; + throw runtime_error("Problem in connecting to PM"); + } } //------------------------------------------------------------------------------ void WESplClient::startClientThread() { - this->fpThread = new boost::thread(WESplClientRunner(*this)); + this->fpThread = new boost::thread(WESplClientRunner(*this)); } //------------------------------------------------------------------------------ void WESplClient::sendAndRecv() { - while (fContinue) + while (fContinue) + { + try { - try - { - // Send messages if out queue has something - send(); - // Recv messages if there is something in socket or timeout - recv(); - } - catch (runtime_error&) - { - //setCpiFailed(true); - done in onDisconnect() BUG - setConnected(false); - - if (fOwner.getDebugLvl()) - cout << "Disconnect from PM - " << getPmId() << endl; - - onDisconnect(); - } + // Send messages if out queue has something + send(); + // Recv messages if there is something in socket or timeout + recv(); } - - if (this->fCpiFailed) + catch (runtime_error&) { - // NOTE : commented out to avoid sending rollback twice. - //fOwner.onCpimportFail(this->getPmId()); - char aDefCon[16], aRedCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); + // setCpiFailed(true); - done in onDisconnect() BUG + setConnected(false); - if (fOwner.getDebugLvl()) - cout << aRedCol << "Bulk load FAILED on PM " - << getPmId() << aDefCon << endl; - } - else if (this->fCpiPassed) - { - //if(fOwner.getDebugLvl()) - //BUG 4195 - char aDefCon[16], aGreenCol[16]; - snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); - snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m"); + if (fOwner.getDebugLvl()) + cout << "Disconnect from PM - " << getPmId() << endl; - if (fOwner.getDebugLvl()) - cout << aGreenCol << "Bulk load Finished Successfully on PM " - << getPmId() << aDefCon << endl; - } - else if (!this->fCpiStarted) - { - if (fOwner.getDebugLvl()) - cout << "Cpimport Failed to Start!!!" << this->getPmId() << endl; + onDisconnect(); } + } + + if (this->fCpiFailed) + { + // NOTE : commented out to avoid sending rollback twice. + // fOwner.onCpimportFail(this->getPmId()); + char aDefCon[16], aRedCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m"); + + if (fOwner.getDebugLvl()) + cout << aRedCol << "Bulk load FAILED on PM " << getPmId() << aDefCon << endl; + } + else if (this->fCpiPassed) + { + // if(fOwner.getDebugLvl()) + // BUG 4195 + char aDefCon[16], aGreenCol[16]; + snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); + snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m"); + + if (fOwner.getDebugLvl()) + cout << aGreenCol << "Bulk load Finished Successfully on PM " << getPmId() << aDefCon << endl; + } + else if (!this->fCpiStarted) + { + if (fOwner.getDebugLvl()) + cout << "Cpimport Failed to Start!!!" << this->getPmId() << endl; + } } //------------------------------------------------------------------------------ void WESplClient::send() { + if ((!fSendQueue.empty()) && (getDataRqstCount() > 0)) + { + if (fOwner.getDebugLvl() > 2) + cout << "DataRqstCnt [" << getPmId() << "] = " << getDataRqstCount() << endl; - if ((!fSendQueue.empty()) && (getDataRqstCount() > 0)) + boost::mutex::scoped_lock aLock(fSentQMutex); + messageqcpp::SBS aSbs = fSendQueue.front(); + fSendQueue.pop(); + aLock.unlock(); + int aLen = (*aSbs).length(); + + if (aLen > 0) { - if (fOwner.getDebugLvl() > 2) - cout << "DataRqstCnt [" << getPmId() << "] = " - << getDataRqstCount() << endl; + boost::mutex::scoped_lock aLock(fWriteMutex); + setBytesTx(getBytesTx() + aLen); - boost::mutex::scoped_lock aLock(fSentQMutex); - messageqcpp::SBS aSbs = fSendQueue.front(); - fSendQueue.pop(); - aLock.unlock(); - int aLen = (*aSbs).length(); + try + { + if (isConnected()) + fClnt->write(aSbs); + } + catch (...) + { + } - if (aLen > 0) - { - boost::mutex::scoped_lock aLock(fWriteMutex); - setBytesTx(getBytesTx() + aLen); - - try - { - if (isConnected()) - fClnt->write(aSbs); - } - catch (...) - { - } - - aLock.unlock(); - } - - decDataRqstCount(); - //decDbRootVar(); + aLock.unlock(); } - //setSendFlag(fOwner.check4Ack(fPmId)); + decDataRqstCount(); + // decDbRootVar(); + } + // setSendFlag(fOwner.check4Ack(fPmId)); } //------------------------------------------------------------------------------ void WESplClient::recv() { - messageqcpp::SBS aSbs; - struct timespec rm_ts; - rm_ts.tv_sec = fRdSecTo; //0 when data sending otherwise 1- second - rm_ts.tv_nsec = 20000000; // 20 milliSec - bool isTimeOut = false; - int aLen = 0; + messageqcpp::SBS aSbs; + struct timespec rm_ts; + rm_ts.tv_sec = fRdSecTo; // 0 when data sending otherwise 1- second + rm_ts.tv_nsec = 20000000; // 20 milliSec + bool isTimeOut = false; + int aLen = 0; - try - { - if (isConnected()) - aSbs = fClnt->read(&rm_ts, &isTimeOut); - } - catch (std::exception& ex) - { - setConnected(false); - cout << ex.what() << endl; - cout << "fClnt read error on " << getPmId() << endl; - throw runtime_error("fClnt read error"); - } + try + { + if (isConnected()) + aSbs = fClnt->read(&rm_ts, &isTimeOut); + } + catch (std::exception& ex) + { + setConnected(false); + cout << ex.what() << endl; + cout << "fClnt read error on " << getPmId() << endl; + throw runtime_error("fClnt read error"); + } - // - aSbs->length()>0 add to the sdh.fWesMsgQueue - try - { - if (aSbs) - aLen = aSbs->length(); - } - catch (...) - { - aLen = 0; - } - - if (aLen > 0) - { - setLastInTime(time(0)); //added back for BUG 4535 / BUG 4195 - setBytesRcv( getBytesRcv() + aLen); - fOwner.add2RespQueue(aSbs); - } - else if ((aLen <= 0) && (!isTimeOut)) //disconnect - { - cout << "Disconnect from PM - " << getPmId() << " IP " << endl; - onDisconnect(); - } + // - aSbs->length()>0 add to the sdh.fWesMsgQueue + try + { + if (aSbs) + aLen = aSbs->length(); + } + catch (...) + { + aLen = 0; + } + if (aLen > 0) + { + setLastInTime(time(0)); // added back for BUG 4535 / BUG 4195 + setBytesRcv(getBytesRcv() + aLen); + fOwner.add2RespQueue(aSbs); + } + else if ((aLen <= 0) && (!isTimeOut)) // disconnect + { + cout << "Disconnect from PM - " << getPmId() << " IP " << endl; + onDisconnect(); + } } //------------------------------------------------------------------------------ void WESplClient::write(const messageqcpp::ByteStream& Msg) { - setBytesTx(getBytesTx() + Msg.length()); + setBytesTx(getBytesTx() + Msg.length()); - try - { - if (Msg.length() > 0) - fClnt->write(Msg); - } - catch (...) - { - //ignore it - } + try + { + if (Msg.length() > 0) + fClnt->write(Msg); + } + catch (...) + { + // ignore it + } } //------------------------------------------------------------------------------ void WESplClient::read(messageqcpp::SBS& Sbs) { - // read from the WEServerMsgQueue - // if Key is needed give that constant here + // read from the WEServerMsgQueue + // if Key is needed give that constant here } //------------------------------------------------------------------------------ -//TODO - We may need to make it much more efficient by incorporating file read +// TODO - We may need to make it much more efficient by incorporating file read void WESplClient::add2SendQueue(const messageqcpp::SBS& Sbs) { - this->fSendQueue.push(Sbs); + this->fSendQueue.push(Sbs); } - void WESplClient::clearSendQueue() { - boost::mutex::scoped_lock aLock(fSentQMutex); + boost::mutex::scoped_lock aLock(fSentQMutex); - while (!fSendQueue.empty()) - fSendQueue.pop(); + while (!fSendQueue.empty()) + fSendQueue.pop(); - aLock.unlock(); + aLock.unlock(); } int WESplClient::getSendQSize() { - int aQSize = 0; - boost::mutex::scoped_lock aLock(fSentQMutex); - aQSize = fSendQueue.size(); - aLock.unlock(); - return aQSize; + int aQSize = 0; + boost::mutex::scoped_lock aLock(fSentQMutex); + aQSize = fSendQueue.size(); + aLock.unlock(); + return aQSize; } - //------------------------------------------------------------------------------ void WESplClient::printStats() { - if (fOwner.getDebugLvl()) + if (fOwner.getDebugLvl()) + { + cout << "\tPMid \t" << getPmId() << endl; + cout << "\tTx Rows \t" << getRowTx() << endl; + // if(fOwner.getDebugLvl()) + cout << "\tTx Bytes \t" << getBytesTx() << endl; + // if(fOwner.getDebugLvl()) + cout << "\tRcv Bytes \t" << getBytesRcv() << endl; + cout << "\tInserted/Read Rows " << fRowsUploadInfo.fRowsInserted << "/" << fRowsUploadInfo.fRowsRead + << endl; + + if (fColOorVec.size() > 0) + cout << "\tCol Id\tColName\t\t\tout-of-range count" << endl; + + WEColOorVec::iterator aIt = fColOorVec.begin(); + + while (aIt != fColOorVec.end()) { - cout << "\tPMid \t" << getPmId() << endl; - cout << "\tTx Rows \t" << getRowTx() << endl; - //if(fOwner.getDebugLvl()) - cout << "\tTx Bytes \t" << getBytesTx() << endl; - //if(fOwner.getDebugLvl()) - cout << "\tRcv Bytes \t" << getBytesRcv() << endl; - cout << "\tInserted/Read Rows " << fRowsUploadInfo.fRowsInserted << "/" - << fRowsUploadInfo.fRowsRead << endl; - - if (fColOorVec.size() > 0) - cout << "\tCol Id\tColName\t\t\tout-of-range count" << endl; - - WEColOorVec::iterator aIt = fColOorVec.begin(); - - while (aIt != fColOorVec.end()) - { - cout << "\t" << (*aIt).fColNum << "\t" << (*aIt).fColName << "\t\t" << (*aIt).fNoOfOORs << endl; - aIt++; - } - - if (!fBadDataFile.empty())cout << "\tBad Data Filename " << fBadDataFile << endl; - - if (!fErrInfoFile.empty())cout << "\tError Filename " << fErrInfoFile << endl; - - cout << "\t(" << getLastInTime() - getStartTime() << "sec)" << endl; - cout << "\t" << endl; + cout << "\t" << (*aIt).fColNum << "\t" << (*aIt).fColName << "\t\t" << (*aIt).fNoOfOORs << endl; + aIt++; } + + if (!fBadDataFile.empty()) + cout << "\tBad Data Filename " << fBadDataFile << endl; + + if (!fErrInfoFile.empty()) + cout << "\tError Filename " << fErrInfoFile << endl; + + cout << "\t(" << getLastInTime() - getStartTime() << "sec)" << endl; + cout << "\t" << endl; + } } //------------------------------------------------------------------------------ void WESplClient::onConnect() { - //TODO - // update all the flags on Connect. - // alert data can be send now - // do not allow to connect back again. + // TODO + // update all the flags on Connect. + // alert data can be send now + // do not allow to connect back again. - // when reconnect happens, reset the variables - setRollbackRslt(0); - setCleanupRslt(0); - setCpiPassed(false); - setCpiFailed(false); + // when reconnect happens, reset the variables + setRollbackRslt(0); + setCleanupRslt(0); + setCpiPassed(false); + setCpiFailed(false); - setContinue(true); - setConnected(true); - ByteStream bsWrite; - bsWrite << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE; + setContinue(true); + setConnected(true); + ByteStream bsWrite; + bsWrite << (ByteStream::byte)WE_CLT_SRV_KEEPALIVE; - try - { - this->write(bsWrite); // send the keep init keep alive - } - catch (...) - { - } - - // need to send Alarm - fIpAddress = fClnt->addr2String(); + try + { + this->write(bsWrite); // send the keep init keep alive + } + catch (...) + { + } + // need to send Alarm + fIpAddress = fClnt->addr2String(); } //------------------------------------------------------------------------------ void WESplClient::onDisconnect() { - //TODO - // - set fContinue to false - set the thread free - setContinue(false); - setConnected(false); - setRollbackRslt(-1); - setCleanupRslt(-1); + // TODO + // - set fContinue to false - set the thread free + setContinue(false); + setConnected(false); + setRollbackRslt(-1); + setCleanupRslt(-1); - if ((!fCpiPassed) && (!fCpiFailed)) //a hard disconnection - { - fOwner.onCpimportFail(fPmId); - fOwner.setDisconnectFailure(true); - } + if ((!fCpiPassed) && (!fCpiFailed)) // a hard disconnection + { + fOwner.onCpimportFail(fPmId); + fOwner.setDisconnectFailure(true); + } - // update all the flags of disconnect. - // alert on roll back - // do not allow to connect back again. + // update all the flags of disconnect. + // alert on roll back + // do not allow to connect back again. - try - { - // BT Should log this probably instead - // ALARMManager alarmMgr; - //std::string alarmItem = sin_addr2String(fClnt->serv_addr().sin_addr); - std::string alarmItem = fClnt->addr2String(); - alarmItem.append(" WriteEngineServer"); - //alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET); - } - catch (...) - { - // just ignore it for time being. - } + try + { + // BT Should log this probably instead + // ALARMManager alarmMgr; + // std::string alarmItem = sin_addr2String(fClnt->serv_addr().sin_addr); + std::string alarmItem = fClnt->addr2String(); + alarmItem.append(" WriteEngineServer"); + // alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET); + } + catch (...) + { + // just ignore it for time being. + } } //------------------------------------------------------------------------------ void WESplClient::setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted) { - fRowsUploadInfo.fRowsRead = RowsRead; - fRowsUploadInfo.fRowsInserted = RowsInserted; + fRowsUploadInfo.fRowsRead = RowsRead; + fRowsUploadInfo.fRowsInserted = RowsInserted; } - //------------------------------------------------------------------------------ -void WESplClient::add2ColOutOfRangeInfo(int ColNum, - execplan::CalpontSystemCatalog::ColDataType ColType, - std::string& ColName, int NoOfOors) +void WESplClient::add2ColOutOfRangeInfo(int ColNum, execplan::CalpontSystemCatalog::ColDataType ColType, + std::string& ColName, int NoOfOors) { - WEColOORInfo aColOorInfo; - aColOorInfo.fColNum = ColNum; - aColOorInfo.fColType = ColType; - aColOorInfo.fColName = ColName; - aColOorInfo.fNoOfOORs = NoOfOors; - fColOorVec.push_back(aColOorInfo); + WEColOORInfo aColOorInfo; + aColOorInfo.fColNum = ColNum; + aColOorInfo.fColType = ColType; + aColOorInfo.fColName = ColName; + aColOorInfo.fNoOfOORs = NoOfOors; + fColOorVec.push_back(aColOorInfo); } //------------------------------------------------------------------------------ void WESplClient::setBadDataFile(const std::string& BadDataFile) { - fBadDataFile = BadDataFile; + fBadDataFile = BadDataFile; } //------------------------------------------------------------------------------ - void WESplClient::setErrInfoFile(const std::string& ErrInfoFile) { - fErrInfoFile = ErrInfoFile; + fErrInfoFile = ErrInfoFile; } //------------------------------------------------------------------------------ - } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_splclient.h b/writeengine/splitter/we_splclient.h index 0a33aadec..cf419d575 100644 --- a/writeengine/splitter/we_splclient.h +++ b/writeengine/splitter/we_splclient.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_splclient.h @@ -38,364 +38,379 @@ namespace WriteEngine { - -class WESplClient; //forward decleration +class WESplClient; // forward decleration // Structure for holding the Out of Range data from the BRMReport // This class is also used by we_sdhandler to hold the agregate info. -class WEColOORInfo // Column Out-Of-Range Info +class WEColOORInfo // Column Out-Of-Range Info { -public: - WEColOORInfo(): fColNum(0), fColType(execplan::CalpontSystemCatalog::INT), fNoOfOORs(0) {} - ~WEColOORInfo() {} -public: - int fColNum; - execplan::CalpontSystemCatalog::ColDataType fColType; - std::string fColName; - int fNoOfOORs; + public: + WEColOORInfo() : fColNum(0), fColType(execplan::CalpontSystemCatalog::INT), fNoOfOORs(0) + { + } + ~WEColOORInfo() + { + } + + public: + int fColNum; + execplan::CalpontSystemCatalog::ColDataType fColType; + std::string fColName; + int fNoOfOORs; }; typedef std::vector WEColOorVec; //------------------------------------------------------------------------------ -class WESdHandlerException: public std::exception +class WESdHandlerException : public std::exception { -public: - std::string fWhat; - WESdHandlerException(std::string& What) throw() - { - fWhat = What; - } - virtual ~WESdHandlerException() throw() {} - virtual const char* what() const throw() - { - return fWhat.c_str(); - } + public: + std::string fWhat; + WESdHandlerException(std::string& What) throw() + { + fWhat = What; + } + virtual ~WESdHandlerException() throw() + { + } + virtual const char* what() const throw() + { + return fWhat.c_str(); + } }; //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ - class WESplClientRunner { -public: - WESplClientRunner(WESplClient& Sc): fOwner(Sc) { /* ctor */ } - virtual ~WESplClientRunner() {/* dtor */} - void operator()(); + public: + WESplClientRunner(WESplClient& Sc) : fOwner(Sc) + { /* ctor */ + } + virtual ~WESplClientRunner() + { /* dtor */ + } + void operator()(); -public: - WESplClient& fOwner; + public: + WESplClient& fOwner; }; //------------------------------------------------------------------------------ class WESplClient { -public: - WESplClient(WESDHandler& Sdh, int PmId); - virtual ~WESplClient(); + public: + WESplClient(WESDHandler& Sdh, int PmId); + virtual ~WESplClient(); - void setup(); - void startClientThread(); - void sendAndRecv(); - void send(); - void recv(); - void write(const messageqcpp::ByteStream& Msg); - void read(messageqcpp::SBS& Sbs); - void add2SendQueue(const messageqcpp::SBS& Sbs); - void clearSendQueue(); - int getSendQSize(); + void setup(); + void startClientThread(); + void sendAndRecv(); + void send(); + void recv(); + void write(const messageqcpp::ByteStream& Msg); + void read(messageqcpp::SBS& Sbs); + void add2SendQueue(const messageqcpp::SBS& Sbs); + void clearSendQueue(); + int getSendQSize(); - void printStats(); - void onConnect(); - void onDisconnect(); + void printStats(); + void onConnect(); + void onDisconnect(); - unsigned int getRowTx() const - { - return fRowTx; - } - uint32_t getBytesRcv() const - { - return fBytesRcv; - } - uint32_t getBytesTx() - { - boost::mutex::scoped_lock aLock(fTxMutex); - return fBytesTx; - } - boost::thread* getFpThread() const - { - return fpThread; - } - time_t getLastInTime() - { - boost::mutex::scoped_lock aLock(fLastInMutex); - return (fLastInTime > 0) ? fLastInTime : fStartTime; //BUG 4309 - } - time_t getStartTime() const - { - return fStartTime; - } - time_t getElapsedTime() - { - return (getLastInTime() - getStartTime()); - } - bool isCpiStarted() const - { - return fCpiStarted; - } - bool isCpiPassed() const - { - return fCpiPassed; - } - bool isCpiFailed() const - { - return fCpiFailed; - } - bool isBrmRptRcvd() const - { - return fBrmRptRcvd; - } - int getRollbackRslt() const - { - return fRollbackRslt; - } - int getCleanupRslt() const - { - return fCleanupRslt; - } - bool getSendFlag() const - { - return fSend; - } - unsigned int getPmId() const - { - return fPmId; - } - unsigned int getDbRootCnt() const - { - return fDbrCnt; - } - unsigned int getDbRootVar() - { - boost::mutex::scoped_lock aLock(fDataRqstMutex); - return fDbrVar; - } - int getDataRqstCount() - { - boost::mutex::scoped_lock aLock(fDataRqstMutex); - return fDataRqstCnt; - } - long getRdSecTo() const - { - return fRdSecTo; - } - bool isConnected() const - { - return fConnected; - } - bool isContinue() const - { - return fContinue; - } - const std::string& getServer() const - { - return fServer; - } - const std::string& getIpAddress() const - { - return fIpAddress; - } - void setBytesRcv(uint32_t BytesRcv) - { - fBytesRcv = BytesRcv; - } - void setBytesTx(uint32_t BytesTx) - { - boost::mutex::scoped_lock aLock(fTxMutex); - fBytesTx = BytesTx; - aLock.unlock(); - } - void updateBytesTx(uint32_t fBytes) - { - boost::mutex::scoped_lock aLock(fTxMutex); - fBytesTx += fBytes; - aLock.unlock(); - } - void setConnected(bool Connected) - { - fConnected = Connected; - } - void setContinue(bool Continue) - { - fContinue = Continue; - } - void setFpThread(boost::thread* pThread) - { - fpThread = pThread; - } - void setLastInTime(time_t LastInTime) - { - fLastInTime = LastInTime; - } - void setStartTime(time_t StartTime) - { - boost::mutex::scoped_lock aLock(fLastInMutex); - fStartTime = StartTime; - aLock.lock(); - } - void setSendFlag(bool Send) - { - fSend = Send; - } - void setCpiStarted(bool Start) - { - fCpiStarted = Start; - } - void setCpiPassed(bool Pass) - { - setLastInTime(time(0)); - fCpiPassed = Pass; - } - void setCpiFailed(bool Fail) - { - setLastInTime(time(0)); - fCpiFailed = Fail; - fRowsUploadInfo.fRowsRead = 0; - fRowsUploadInfo.fRowsInserted = 0; - } - void setBrmRptRcvd(bool Rcvd) - { - fBrmRptRcvd = Rcvd; - } - void setRollbackRslt(int Rslt) - { - fRollbackRslt = Rslt; - } - void setCleanupRslt(int Rslt) - { - fCleanupRslt = Rslt; - } - void setPmId(unsigned int PmId) - { - fPmId = PmId; - } - void setDbRootCnt(unsigned int DbrCnt) - { - fDbrCnt = DbrCnt; - } - void resetDbRootVar() - { - boost::mutex::scoped_lock aLock(fDataRqstMutex); - fDbrVar = fDbrCnt; - aLock.unlock(); - } - void decDbRootVar() - { - boost::mutex::scoped_lock aLock(fDataRqstMutex); + unsigned int getRowTx() const + { + return fRowTx; + } + uint32_t getBytesRcv() const + { + return fBytesRcv; + } + uint32_t getBytesTx() + { + boost::mutex::scoped_lock aLock(fTxMutex); + return fBytesTx; + } + boost::thread* getFpThread() const + { + return fpThread; + } + time_t getLastInTime() + { + boost::mutex::scoped_lock aLock(fLastInMutex); + return (fLastInTime > 0) ? fLastInTime : fStartTime; // BUG 4309 + } + time_t getStartTime() const + { + return fStartTime; + } + time_t getElapsedTime() + { + return (getLastInTime() - getStartTime()); + } + bool isCpiStarted() const + { + return fCpiStarted; + } + bool isCpiPassed() const + { + return fCpiPassed; + } + bool isCpiFailed() const + { + return fCpiFailed; + } + bool isBrmRptRcvd() const + { + return fBrmRptRcvd; + } + int getRollbackRslt() const + { + return fRollbackRslt; + } + int getCleanupRslt() const + { + return fCleanupRslt; + } + bool getSendFlag() const + { + return fSend; + } + unsigned int getPmId() const + { + return fPmId; + } + unsigned int getDbRootCnt() const + { + return fDbrCnt; + } + unsigned int getDbRootVar() + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); + return fDbrVar; + } + int getDataRqstCount() + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); + return fDataRqstCnt; + } + long getRdSecTo() const + { + return fRdSecTo; + } + bool isConnected() const + { + return fConnected; + } + bool isContinue() const + { + return fContinue; + } + const std::string& getServer() const + { + return fServer; + } + const std::string& getIpAddress() const + { + return fIpAddress; + } + void setBytesRcv(uint32_t BytesRcv) + { + fBytesRcv = BytesRcv; + } + void setBytesTx(uint32_t BytesTx) + { + boost::mutex::scoped_lock aLock(fTxMutex); + fBytesTx = BytesTx; + aLock.unlock(); + } + void updateBytesTx(uint32_t fBytes) + { + boost::mutex::scoped_lock aLock(fTxMutex); + fBytesTx += fBytes; + aLock.unlock(); + } + void setConnected(bool Connected) + { + fConnected = Connected; + } + void setContinue(bool Continue) + { + fContinue = Continue; + } + void setFpThread(boost::thread* pThread) + { + fpThread = pThread; + } + void setLastInTime(time_t LastInTime) + { + fLastInTime = LastInTime; + } + void setStartTime(time_t StartTime) + { + boost::mutex::scoped_lock aLock(fLastInMutex); + fStartTime = StartTime; + aLock.lock(); + } + void setSendFlag(bool Send) + { + fSend = Send; + } + void setCpiStarted(bool Start) + { + fCpiStarted = Start; + } + void setCpiPassed(bool Pass) + { + setLastInTime(time(0)); + fCpiPassed = Pass; + } + void setCpiFailed(bool Fail) + { + setLastInTime(time(0)); + fCpiFailed = Fail; + fRowsUploadInfo.fRowsRead = 0; + fRowsUploadInfo.fRowsInserted = 0; + } + void setBrmRptRcvd(bool Rcvd) + { + fBrmRptRcvd = Rcvd; + } + void setRollbackRslt(int Rslt) + { + fRollbackRslt = Rslt; + } + void setCleanupRslt(int Rslt) + { + fCleanupRslt = Rslt; + } + void setPmId(unsigned int PmId) + { + fPmId = PmId; + } + void setDbRootCnt(unsigned int DbrCnt) + { + fDbrCnt = DbrCnt; + } + void resetDbRootVar() + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); + fDbrVar = fDbrCnt; + aLock.unlock(); + } + void decDbRootVar() + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); - if (fDbrVar > 0) --fDbrVar; + if (fDbrVar > 0) + --fDbrVar; - aLock.unlock(); - } - void setRdSecTo(long RdSecTo) - { - fRdSecTo = RdSecTo; - } - void setDataRqstCount(int DataRqstCnt) - { - boost::mutex::scoped_lock aLock(fDataRqstMutex); - fDataRqstCnt = DataRqstCnt; - aLock.unlock(); - } - void decDataRqstCount() - { - boost::mutex::scoped_lock aLock(fDataRqstMutex); + aLock.unlock(); + } + void setRdSecTo(long RdSecTo) + { + fRdSecTo = RdSecTo; + } + void setDataRqstCount(int DataRqstCnt) + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); + fDataRqstCnt = DataRqstCnt; + aLock.unlock(); + } + void decDataRqstCount() + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); - if (fDataRqstCnt > 0) --fDataRqstCnt; + if (fDataRqstCnt > 0) + --fDataRqstCnt; - aLock.unlock(); - } - void incDataRqstCount() + aLock.unlock(); + } + void incDataRqstCount() + { + boost::mutex::scoped_lock aLock(fDataRqstMutex); + ++fDataRqstCnt; + aLock.unlock(); + } + void setServer(const std::string& Server) + { + fServer = Server; + } + void setIpAddress(const std::string& IpAddr) + { + fIpAddress = IpAddr; + } + void updateRowTx(unsigned int aCnt) + { + fRowTx += aCnt; + } + void resetRowTx() + { + fRowTx = 0; + } + + private: + bool fContinue; + bool fConnected; + unsigned int fPmId; + unsigned int fDbrCnt; + unsigned int fDbrVar; // Var to keep track next PM to send. + int fDataRqstCnt; // Data request count + long fRdSecTo; // read timeout sec + unsigned int fRowTx; // No. Of Rows Transmitted + uint32_t fBytesTx; + uint32_t fBytesRcv; + time_t fLastInTime; + time_t fStartTime; + bool fSend; + bool fCpiStarted; + bool fCpiPassed; + bool fCpiFailed; + bool fBrmRptRcvd; + int fRollbackRslt; + int fCleanupRslt; + + boost::mutex fTxMutex; // mutex for TxBytes + boost::mutex fDataRqstMutex; + boost::mutex fWriteMutex; + boost::mutex fSentQMutex; + boost::mutex fLastInMutex; + typedef std::queue WESendQueue; + WESendQueue fSendQueue; + + std::string fServer; + std::string fIpAddress; + boost::shared_ptr fClnt; + boost::thread* fpThread; + WESDHandler& fOwner; + + class WERowsUploadInfo + { + public: + WERowsUploadInfo() : fRowsRead(0), fRowsInserted(0) { - boost::mutex::scoped_lock aLock(fDataRqstMutex); - ++fDataRqstCnt; - aLock.unlock(); } - void setServer(const std::string& Server) + ~WERowsUploadInfo() { - fServer = Server; - } - void setIpAddress(const std::string& IpAddr) - { - fIpAddress = IpAddr; - } - void updateRowTx(unsigned int aCnt) - { - fRowTx += aCnt; - } - void resetRowTx() - { - fRowTx = 0; } -private: - bool fContinue; - bool fConnected; - unsigned int fPmId; - unsigned int fDbrCnt; - unsigned int fDbrVar; // Var to keep track next PM to send. - int fDataRqstCnt; // Data request count - long fRdSecTo; // read timeout sec - unsigned int fRowTx; // No. Of Rows Transmitted - uint32_t fBytesTx; - uint32_t fBytesRcv; - time_t fLastInTime; - time_t fStartTime; - bool fSend; - bool fCpiStarted; - bool fCpiPassed; - bool fCpiFailed; - bool fBrmRptRcvd; - int fRollbackRslt; - int fCleanupRslt; + public: + int64_t fRowsRead; + int64_t fRowsInserted; + }; + WERowsUploadInfo fRowsUploadInfo; + WEColOorVec fColOorVec; + std::string fBadDataFile; + std::string fErrInfoFile; - boost::mutex fTxMutex; //mutex for TxBytes - boost::mutex fDataRqstMutex; - boost::mutex fWriteMutex; - boost::mutex fSentQMutex; - boost::mutex fLastInMutex; - typedef std::queue WESendQueue; - WESendQueue fSendQueue; - - std::string fServer; - std::string fIpAddress; - boost::shared_ptr fClnt; - boost::thread* fpThread; - WESDHandler& fOwner; - - class WERowsUploadInfo - { - public: - WERowsUploadInfo(): fRowsRead(0), fRowsInserted(0) {} - ~WERowsUploadInfo() {} - public: - int64_t fRowsRead; - int64_t fRowsInserted; - }; - WERowsUploadInfo fRowsUploadInfo; - WEColOorVec fColOorVec; - std::string fBadDataFile; - std::string fErrInfoFile; - - void setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted); - void add2ColOutOfRangeInfo(int ColNum, execplan::CalpontSystemCatalog::ColDataType ColType, - std::string& ColName, int NoOfOors); - void setBadDataFile(const std::string& BadDataFile); - void setErrInfoFile(const std::string& ErrInfoFile); - - friend class WESDHandler; + void setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted); + void add2ColOutOfRangeInfo(int ColNum, execplan::CalpontSystemCatalog::ColDataType ColType, + std::string& ColName, int NoOfOors); + void setBadDataFile(const std::string& BadDataFile); + void setErrInfoFile(const std::string& ErrInfoFile); + friend class WESDHandler; }; //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 57cfb9e15..366c64ad7 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -49,7 +49,6 @@ using namespace batchloader; static int SPLTR_EXIT_STATUS = 0; - namespace WriteEngine { bool WESplitterApp::fContinue = true; @@ -58,94 +57,92 @@ bool WESplitterApp::fSigHup = false; SimpleSysLog* WESplitterApp::fpSysLog = 0; - -//WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) : +// WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) : // fCmdArgs(CmdArgs), fDh(*this), fpSysLog(0) -WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) : - fCmdArgs(CmdArgs), fDh(*this) +WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) : fCmdArgs(CmdArgs), fDh(*this) { - fpSysLog = SimpleSysLog::instance(); - fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SPLIT)); - setupSignalHandlers(); - std::string err; - fDh.setDebugLvl(fCmdArgs.getDebugLvl()); + fpSysLog = SimpleSysLog::instance(); + fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SPLIT)); + setupSignalHandlers(); + std::string err; + fDh.setDebugLvl(fCmdArgs.getDebugLvl()); - fDh.check4CpiInvokeMode(); + fDh.check4CpiInvokeMode(); - fCmdArgs.checkForCornerCases(); + fCmdArgs.checkForCornerCases(); - if (fCmdArgs.isCpimportInvokeMode()) + if (fCmdArgs.isCpimportInvokeMode()) + { + try { - try - { - invokeCpimport(); - } - catch (std::exception& ex) - { - cout << "Invoking Mode 3" << endl; - cout << ex.what() << endl; - SPLTR_EXIT_STATUS = 1; - exit(SPLTR_EXIT_STATUS); - } - - exit(SPLTR_EXIT_STATUS); + invokeCpimport(); } - else + catch (std::exception& ex) { - if (fCmdArgs.isHelpMode()) fCmdArgs.usage(); - - if (fCmdArgs.getMultiTableCount() <= 1) - { - try - { - fDh.setup(); - } - catch (std::exception& ex) - { - //err = string("Error in constructing WESplitterApp") + ex.what(); - err = ex.what(); //cleaning up for BUG 4298 - logging::Message::Args errMsgArgs; - errMsgArgs.add(err); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - - if (!fCmdArgs.getConsoleOutput()) - { - ofstream dmlFile; - ostringstream oss; - oss << startup::StartUp::tmpDir() << fDh.getTableOID() << ".txt"; - dmlFile.open(oss.str().c_str()); - - if (dmlFile.is_open()) - { - dmlFile << err; - dmlFile << endl; - dmlFile.close(); - } - - SPLTR_EXIT_STATUS = 2; - } - else - SPLTR_EXIT_STATUS = 1; - - //cout << err << endl; - fDh.fLog.logMsg( err, MSGLVL_ERROR ); - fContinue = false; - //throw runtime_error(err); BUG 4298 - } - } + cout << "Invoking Mode 3" << endl; + cout << ex.what() << endl; + SPLTR_EXIT_STATUS = 1; + exit(SPLTR_EXIT_STATUS); } + exit(SPLTR_EXIT_STATUS); + } + else + { + if (fCmdArgs.isHelpMode()) + fCmdArgs.usage(); + + if (fCmdArgs.getMultiTableCount() <= 1) + { + try + { + fDh.setup(); + } + catch (std::exception& ex) + { + // err = string("Error in constructing WESplitterApp") + ex.what(); + err = ex.what(); // cleaning up for BUG 4298 + logging::Message::Args errMsgArgs; + errMsgArgs.add(err); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + + if (!fCmdArgs.getConsoleOutput()) + { + ofstream dmlFile; + ostringstream oss; + oss << startup::StartUp::tmpDir() << fDh.getTableOID() << ".txt"; + dmlFile.open(oss.str().c_str()); + + if (dmlFile.is_open()) + { + dmlFile << err; + dmlFile << endl; + dmlFile.close(); + } + + SPLTR_EXIT_STATUS = 2; + } + else + SPLTR_EXIT_STATUS = 1; + + // cout << err << endl; + fDh.fLog.logMsg(err, MSGLVL_ERROR); + fContinue = false; + // throw runtime_error(err); BUG 4298 + } + } + } } WESplitterApp::~WESplitterApp() { - //fDh.shutdown(); - usleep(1000); //1 millisec just checking + // fDh.shutdown(); + usleep(1000); // 1 millisec just checking - std::string aStr = "Calling WESplitterApp Destructor\n"; - - if (fDh.getDebugLvl()) cout << aStr << endl; + std::string aStr = "Calling WESplitterApp Destructor\n"; + if (fDh.getDebugLvl()) + cout << aStr << endl; } //------------------------------------------------------------------------------ @@ -155,25 +152,25 @@ WESplitterApp::~WESplitterApp() void WESplitterApp::setupSignalHandlers() { #ifdef _MSC_VER - //FIXME + // FIXME #else - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = WESplitterApp::onSigInterrupt; - sigaction(SIGINT, &sa, 0); - sa.sa_handler = WESplitterApp::onSigTerminate; - sigaction(SIGTERM, &sa, 0); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, 0); - sa.sa_handler = WESplitterApp::onSigHup; - sigaction(SIGHUP, &sa, 0); - sa.sa_handler = WESplitterApp::onSigInterrupt; - sigaction(SIGUSR1, &sa, 0); - /* - signal(SIGPIPE, SIG_IGN); - signal(SIGINT, WESplitterApp::onSigInterrupt); - signal(SIGTERM, WESplitterApp::onSigTerminate); - signal(SIGHUP, WESplitterApp::onSigHup); */ + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = WESplitterApp::onSigInterrupt; + sigaction(SIGINT, &sa, 0); + sa.sa_handler = WESplitterApp::onSigTerminate; + sigaction(SIGTERM, &sa, 0); + sa.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sa, 0); + sa.sa_handler = WESplitterApp::onSigHup; + sigaction(SIGHUP, &sa, 0); + sa.sa_handler = WESplitterApp::onSigInterrupt; + sigaction(SIGUSR1, &sa, 0); + /* + signal(SIGPIPE, SIG_IGN); + signal(SIGINT, WESplitterApp::onSigInterrupt); + signal(SIGTERM, WESplitterApp::onSigTerminate); + signal(SIGHUP, WESplitterApp::onSigHup); */ #endif } //------------------------------------------------------------------------------ @@ -181,17 +178,17 @@ void WESplitterApp::setupSignalHandlers() //------------------------------------------------------------------------------ void WESplitterApp::onSigTerminate(int aInt) { - cout << "onSigTerminate received signal " << aInt << endl; + cout << "onSigTerminate received signal " << aInt << endl; - if (15 == aInt) - { - fSignaled = true; - } + if (15 == aInt) + { + fSignaled = true; + } - fContinue = false; //force to call destructor - - if (aInt == 1) SPLTR_EXIT_STATUS = 1; + fContinue = false; // force to call destructor + if (aInt == 1) + SPLTR_EXIT_STATUS = 1; } //------------------------------------------------------------------------------ @@ -199,16 +196,17 @@ void WESplitterApp::onSigTerminate(int aInt) //------------------------------------------------------------------------------ void WESplitterApp::onSigInterrupt(int aInt) { - //cout << "onSigInterrupt received signal " << aInt << endl; - if ((2 == aInt) || (10 == aInt)) - { - fSignaled = true; - //cout << "ctrl-c received" << endl; - } + // cout << "onSigInterrupt received signal " << aInt << endl; + if ((2 == aInt) || (10 == aInt)) + { + fSignaled = true; + // cout << "ctrl-c received" << endl; + } - fContinue = false; //force to call destructor + fContinue = false; // force to call destructor - if (aInt == 1) SPLTR_EXIT_STATUS = 1; + if (aInt == 1) + SPLTR_EXIT_STATUS = 1; } //------------------------------------------------------------------------------ @@ -216,18 +214,19 @@ void WESplitterApp::onSigInterrupt(int aInt) //------------------------------------------------------------------------------ void WESplitterApp::onSigHup(int aInt) { - fSigHup = true; - fContinue = false; - std::string aStr = "Interrupt received...Program Exiting..."; - cout << aStr << endl; + fSigHup = true; + fContinue = false; + std::string aStr = "Interrupt received...Program Exiting..."; + cout << aStr << endl; - if (aInt == 1) SPLTR_EXIT_STATUS = 1; + if (aInt == 1) + SPLTR_EXIT_STATUS = 1; - logging::Message::Args errMsgArgs; - errMsgArgs.add(aStr); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); + logging::Message::Args errMsgArgs; + errMsgArgs.add(aStr); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000); - exit(SPLTR_EXIT_STATUS); //BUG 4534 - exit w/o writing to log + exit(SPLTR_EXIT_STATUS); // BUG 4534 - exit w/o writing to log } //------------------------------------------------------------------------------ @@ -235,282 +234,282 @@ void WESplitterApp::onSigHup(int aInt) //------------------------------------------------------------------------------ void WESplitterApp::processMessages() { - boost::uuids::uuid u = boost::uuids::random_generator()(); - fCmdArgs.setJobUUID(u); + boost::uuids::uuid u = boost::uuids::random_generator()(); + fCmdArgs.setJobUUID(u); - messageqcpp::ByteStream aBs; - unsigned int aRollCount = 0; + messageqcpp::ByteStream aBs; + unsigned int aRollCount = 0; - if (fDh.getDebugLvl()) - cout << "Inside WESplitterApp::processMessages() " - << "Mode = " << fCmdArgs.getMode() << endl; + if (fDh.getDebugLvl()) + cout << "Inside WESplitterApp::processMessages() " + << "Mode = " << fCmdArgs.getMode() << endl; - //TODO - handle all the messages here - if (fCmdArgs.getMode() == 2) + // TODO - handle all the messages here + if (fCmdArgs.getMode() == 2) + { + try { - try - { - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_MODE; - aBs << (messageqcpp::ByteStream::quadbyte) fCmdArgs.getMode(); - fDh.send2Pm(aBs); + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_MODE; + aBs << (messageqcpp::ByteStream::quadbyte)fCmdArgs.getMode(); + fDh.send2Pm(aBs); - std::string aJobId = fCmdArgs.getJobId(); + std::string aJobId = fCmdArgs.getJobId(); - if ((aJobId.length() > 0) && (!fCmdArgs.isJobLogOnly())) // Export jobFile NOW - { - std::string aJobFileName = fCmdArgs.getJobFileName(); - fDh.exportJobFile(aJobId, aJobFileName ); - } + if ((aJobId.length() > 0) && (!fCmdArgs.isJobLogOnly())) // Export jobFile NOW + { + std::string aJobFileName = fCmdArgs.getJobFileName(); + fDh.exportJobFile(aJobId, aJobFileName); + } - aBs.restart(); - std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine(); - fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 ); + aBs.restart(); + std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine(); + fDh.fLog.logMsg(aCpImpCmd, MSGLVL_INFO2); - if (fDh.getDebugLvl()) - cout << "CPImport cmd line - " << aCpImpCmd << endl; + if (fDh.getDebugLvl()) + cout << "CPImport cmd line - " << aCpImpCmd << endl; - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_CMDLINEARGS; - aBs << aCpImpCmd; - fDh.send2Pm(aBs); + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_CMDLINEARGS; + aBs << aCpImpCmd; + fDh.send2Pm(aBs); - aBs.restart(); - std::string aBrmRpt = fCmdArgs.getBrmRptFileName(); + aBs.restart(); + std::string aBrmRpt = fCmdArgs.getBrmRptFileName(); - if (fDh.getDebugLvl()) - cout << "BrmReport FileName - " << aBrmRpt << endl; + if (fDh.getDebugLvl()) + cout << "BrmReport FileName - " << aBrmRpt << endl; - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_BRMRPT; - aBs << aBrmRpt; - fDh.send2Pm(aBs); - - } - catch (std::exception& exp) - { - //cout << exp.what() << endl; - SPLTR_EXIT_STATUS = 1; - //exit(SPLTR_EXIT_STATUS); - throw runtime_error(exp.what()); - } + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_BRMRPT; + aBs << aBrmRpt; + fDh.send2Pm(aBs); } - else if (fCmdArgs.getMode() == 1) + catch (std::exception& exp) { - try - { - // In this mode we ignore almost all cmd lines args which - // are usually send to cpimport - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_MODE; - aBs << (messageqcpp::ByteStream::quadbyte) fCmdArgs.getMode(); - fDh.send2Pm(aBs); - - std::string aJobId = fCmdArgs.getJobId(); - - if (fDh.getDebugLvl()) cout << "ProcessMsgs aJobId " << aJobId << endl; - - if ((aJobId.length() > 0) && (!fCmdArgs.isJobLogOnly())) // Export jobFile NOW - { - std::string aJobFileName = fCmdArgs.getJobFileName(); - - if (fDh.getDebugLvl()) cout << "ProcessMsgs Calling exportJobFile " << endl; - - fDh.exportJobFile(aJobId, aJobFileName ); - - if (fDh.getDebugLvl()) cout << "ProcessMsgs Calling exportJobFile " << endl; - } - - aBs.restart(); - std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine(); - fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 ); - - if (fDh.getDebugLvl()) - cout << "CPImport cmd line - " << aCpImpCmd << endl; - - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_CMDLINEARGS; - aBs << aCpImpCmd; - fDh.send2Pm(aBs); - - aBs.restart(); - std::string aBrmRpt = fCmdArgs.getBrmRptFileName(); - - if (fDh.getDebugLvl()) - cout << "BrmReport FileName - " << aBrmRpt << endl; - - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_BRMRPT; - aBs << aBrmRpt; - fDh.send2Pm(aBs); - - } - catch (std::exception& exp) - { - //cout << exp.what() << endl; - SPLTR_EXIT_STATUS = 1; - //exit(SPLTR_EXIT_STATUS); - throw runtime_error(exp.what()); - } + // cout << exp.what() << endl; + SPLTR_EXIT_STATUS = 1; + // exit(SPLTR_EXIT_STATUS); + throw runtime_error(exp.what()); } - else if (fCmdArgs.getMode() == 0) + } + else if (fCmdArgs.getMode() == 1) + { + try { - try + // In this mode we ignore almost all cmd lines args which + // are usually send to cpimport + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_MODE; + aBs << (messageqcpp::ByteStream::quadbyte)fCmdArgs.getMode(); + fDh.send2Pm(aBs); + + std::string aJobId = fCmdArgs.getJobId(); + + if (fDh.getDebugLvl()) + cout << "ProcessMsgs aJobId " << aJobId << endl; + + if ((aJobId.length() > 0) && (!fCmdArgs.isJobLogOnly())) // Export jobFile NOW + { + std::string aJobFileName = fCmdArgs.getJobFileName(); + + if (fDh.getDebugLvl()) + cout << "ProcessMsgs Calling exportJobFile " << endl; + + fDh.exportJobFile(aJobId, aJobFileName); + + if (fDh.getDebugLvl()) + cout << "ProcessMsgs Calling exportJobFile " << endl; + } + + aBs.restart(); + std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine(); + fDh.fLog.logMsg(aCpImpCmd, MSGLVL_INFO2); + + if (fDh.getDebugLvl()) + cout << "CPImport cmd line - " << aCpImpCmd << endl; + + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_CMDLINEARGS; + aBs << aCpImpCmd; + fDh.send2Pm(aBs); + + aBs.restart(); + std::string aBrmRpt = fCmdArgs.getBrmRptFileName(); + + if (fDh.getDebugLvl()) + cout << "BrmReport FileName - " << aBrmRpt << endl; + + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_BRMRPT; + aBs << aBrmRpt; + fDh.send2Pm(aBs); + } + catch (std::exception& exp) + { + // cout << exp.what() << endl; + SPLTR_EXIT_STATUS = 1; + // exit(SPLTR_EXIT_STATUS); + throw runtime_error(exp.what()); + } + } + else if (fCmdArgs.getMode() == 0) + { + try + { + // In this mode we ignore almost all cmd lines args which + // are usually send to cpimport + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_MODE; + aBs << (messageqcpp::ByteStream::quadbyte)fCmdArgs.getMode(); + fDh.send2Pm(aBs); + + aBs.restart(); + std::string aCpImpFileName = fCmdArgs.getPmFile(); + + if (aCpImpFileName.length() == 0) + { + fCmdArgs.setPmFile(fCmdArgs.getLocFile()); + aCpImpFileName = fCmdArgs.getPmFile(); + + if ((aCpImpFileName.length() == 0) || (aCpImpFileName == "STDIN")) { - // In this mode we ignore almost all cmd lines args which - // are usually send to cpimport - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_MODE; - aBs << (messageqcpp::ByteStream::quadbyte) fCmdArgs.getMode(); - fDh.send2Pm(aBs); - - aBs.restart(); - std::string aCpImpFileName = fCmdArgs.getPmFile(); - - if (aCpImpFileName.length() == 0) - { - fCmdArgs.setPmFile(fCmdArgs.getLocFile()); - aCpImpFileName = fCmdArgs.getPmFile(); - - if ((aCpImpFileName.length() == 0) || (aCpImpFileName == "STDIN")) - { - throw (runtime_error("PM Remote filename not specified")); - } - } - - if (fDh.getDebugLvl()) - cout << "CPImport FileName - " << aCpImpFileName << endl; - - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_IMPFILENAME; - aBs << aCpImpFileName; - fDh.send2Pm(aBs); + throw(runtime_error("PM Remote filename not specified")); } - catch (std::exception& exp) + } + + if (fDh.getDebugLvl()) + cout << "CPImport FileName - " << aCpImpFileName << endl; + + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_IMPFILENAME; + aBs << aCpImpFileName; + fDh.send2Pm(aBs); + } + catch (std::exception& exp) + { + // cout << exp.what() << endl; + SPLTR_EXIT_STATUS = 1; + // exit(SPLTR_EXIT_STATUS); + throw runtime_error(exp.what()); + } + } + + int aNoSec = 2; + bool bRollback = false; + bool bForce = false; + int iShutdown; + + // TODO - this is for just time being.... + // we need to process message of main thread here.. + // here we need to cont check the status of different things + while (fContinue) + { + ++aRollCount; + usleep(1000000); + // Check to see if someone has ordered a shutdown with rollback or force. + iShutdown = fDh.fDbrm.getSystemShutdownPending(bRollback, bForce); + + if (iShutdown >= 0) + { + if (bRollback) + { + if (iShutdown > 0) // Means a shutdown, stop or restart { - //cout << exp.what() << endl; - SPLTR_EXIT_STATUS = 1; - //exit(SPLTR_EXIT_STATUS); - throw runtime_error(exp.what()); + cout << "System stop has been ordered. Rollback" << endl; } + else + { + cout << "Database writes have been suspended. Rollback" << endl; + } + + fSignaled = true; + fContinue = false; + } + else if (bForce) + { + // BUG 5012 - added to avoid rollback + fContinue = false; + ostringstream oss; + oss << "Table " << fCmdArgs.getSchemaName() << "."; + oss << fCmdArgs.getTableName() << ": (OID-"; + oss << fDh.getTableOID() << ") was NOT successfully loaded."; + cout << oss.str() << endl; + logging::Message::Args errMsgArgs; + // BUG 4152 + errMsgArgs.add(fCmdArgs.getSchemaName()); + errMsgArgs.add(fCmdArgs.getTableName()); + errMsgArgs.add(fDh.getTableOID()); + std::string aStr = "Immediate system stop has been ordered, rollback deferred"; + cout << aStr << endl; + SPLTR_EXIT_STATUS = 1; + errMsgArgs.add(aStr); + fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0096); + exit(SPLTR_EXIT_STATUS); + // BUG 5012 - commented out to avoid rollback + // cout << "Immediate system stop has been ordered. No rollback" << endl; + // fSignaled = true; + // fContinue = false; + } } - int aNoSec = 2; - bool bRollback = false; - bool bForce = false; - int iShutdown; - - // TODO - this is for just time being.... - // we need to process message of main thread here.. - // here we need to cont check the status of different things - while (fContinue) + // Send out a heartbeat to the WriteEnginServers every 10 seconds + if ((0 == (aRollCount % aNoSec)) && (!fSignaled)) // Debugging - every 10 seconds { - ++aRollCount; - usleep(1000000); - // Check to see if someone has ordered a shutdown with rollback or force. - iShutdown = fDh.fDbrm.getSystemShutdownPending(bRollback, bForce); + if (aNoSec < 10) + aNoSec++; // progressively go up to 10Sec interval - if (iShutdown >= 0) - { - if (bRollback) - { - if (iShutdown > 0) // Means a shutdown, stop or restart - { - cout << "System stop has been ordered. Rollback" << endl; - } - else - { - cout << "Database writes have been suspended. Rollback" << endl; - } - - fSignaled = true; - fContinue = false; - } - else if (bForce) - { - //BUG 5012 - added to avoid rollback - fContinue = false; - ostringstream oss; - oss << "Table " << fCmdArgs.getSchemaName() << "."; - oss << fCmdArgs.getTableName() << ": (OID-"; - oss << fDh.getTableOID() << ") was NOT successfully loaded."; - cout << oss.str() << endl; - logging::Message::Args errMsgArgs; - //BUG 4152 - errMsgArgs.add(fCmdArgs.getSchemaName()); - errMsgArgs.add(fCmdArgs.getTableName()); - errMsgArgs.add(fDh.getTableOID()); - std::string aStr = "Immediate system stop has been ordered, rollback deferred"; - cout << aStr << endl; - SPLTR_EXIT_STATUS = 1; - errMsgArgs.add(aStr); - fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0096); - exit(SPLTR_EXIT_STATUS); - //BUG 5012 - commented out to avoid rollback - //cout << "Immediate system stop has been ordered. No rollback" << endl; - //fSignaled = true; - //fContinue = false; - } - } - - // Send out a heartbeat to the WriteEnginServers every 10 seconds - if ((0 == (aRollCount % aNoSec)) && (!fSignaled)) // Debugging - every 10 seconds - { - if (aNoSec < 10) aNoSec++; //progressively go up to 10Sec interval - - aBs.restart(); - aBs << (messageqcpp::ByteStream::byte) WE_CLT_SRV_KEEPALIVE; - boost::mutex::scoped_lock aLock(fDh.fSendMutex); - fDh.send2Pm(aBs); - aLock.unlock(); - //fDh.sendHeartbeats(); - //fDh.checkForConnections(); - decided to recv SIGHUP from OAM instead of this - } + aBs.restart(); + aBs << (messageqcpp::ByteStream::byte)WE_CLT_SRV_KEEPALIVE; + boost::mutex::scoped_lock aLock(fDh.fSendMutex); + fDh.send2Pm(aBs); + aLock.unlock(); + // fDh.sendHeartbeats(); + // fDh.checkForConnections(); - decided to recv SIGHUP from OAM instead of this } + } - fDh.shutdown(); + fDh.shutdown(); -} // processMessages +} // processMessages void WESplitterApp::invokeCpimport() { - boost::uuids::uuid u = boost::uuids::random_generator()(); - fCmdArgs.setJobUUID(u); + boost::uuids::uuid u = boost::uuids::random_generator()(); + fCmdArgs.setJobUUID(u); - fCmdArgs.setMode(3); - std::string aCmdLineStr = fCmdArgs.getCpImportCmdLine(); + fCmdArgs.setMode(3); + std::string aCmdLineStr = fCmdArgs.getCpImportCmdLine(); - if (fDh.getDebugLvl()) - cout << "CPI CmdLineArgs : " << aCmdLineStr << endl; + if (fDh.getDebugLvl()) + cout << "CPI CmdLineArgs : " << aCmdLineStr << endl; - std::vector Cmds; + std::vector Cmds; - std::istringstream ss(aCmdLineStr); - std::string arg; - std::vector v2; - v2.reserve(50); + std::istringstream ss(aCmdLineStr); + std::string arg; + std::vector v2; + v2.reserve(50); - while (ss >> arg) - { - //we need something that works on Windows as well as linux - v2.push_back(arg); - } + while (ss >> arg) + { + // we need something that works on Windows as well as linux + v2.push_back(arg); + } - for (unsigned int j = 0; j < v2.size(); ++j) - { - Cmds.push_back(const_cast(v2[j].c_str())); - } + for (unsigned int j = 0; j < v2.size(); ++j) + { + Cmds.push_back(const_cast(v2[j].c_str())); + } - Cmds.push_back(0); //null terminate + Cmds.push_back(0); // null terminate - int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works with full Path - - if (fDh.getDebugLvl()) - cout << "Return status of cpimport is " << aRet << endl; + int aRet = execvp(Cmds[0], &Cmds[0]); // NOTE - works with full Path + if (fDh.getDebugLvl()) + cout << "Return status of cpimport is " << aRet << endl; } //----------------------------------------------------------------------------- void WESplitterApp::updateWithJobFile(int aIdx) { - fCmdArgs.updateWithJobFile(aIdx); + fCmdArgs.updateWithJobFile(aIdx); } //----------------------------------------------------------------------------- - } /* namespace WriteEngine */ //------------------------------------------------------------------------------ @@ -519,58 +518,58 @@ void WESplitterApp::updateWithJobFile(int aIdx) int main(int argc, char** argv) { - std::string err; - std::cin.sync_with_stdio(false); + std::string err; + std::cin.sync_with_stdio(false); - try + try + { + WriteEngine::WECmdArgs aWeCmdArgs(argc, argv); + WriteEngine::WESplitterApp aWESplitterApp(aWeCmdArgs); + int aTblCnt = aWESplitterApp.fCmdArgs.getMultiTableCount(); + + if (aTblCnt > 1) { - WriteEngine::WECmdArgs aWeCmdArgs(argc, argv); - WriteEngine::WESplitterApp aWESplitterApp(aWeCmdArgs); - int aTblCnt = aWESplitterApp.fCmdArgs.getMultiTableCount(); + for (int idx = 0; idx < aTblCnt; idx++) + { + aWESplitterApp.fDh.reset(); + aWESplitterApp.fContinue = true; + aWESplitterApp.updateWithJobFile(idx); - if (aTblCnt > 1) + try { - for (int idx = 0; idx < aTblCnt; idx++) - { - aWESplitterApp.fDh.reset(); - aWESplitterApp.fContinue = true; - aWESplitterApp.updateWithJobFile(idx); - - try - { - aWESplitterApp.fDh.setup(); - } - catch (std::exception& ex) - { - //err = string("Error in constructing WESplitterApp") + ex.what(); - err = ex.what(); //cleaning up for BUG 4298 - logging::Message::Args errMsgArgs; - errMsgArgs.add(err); - aWESplitterApp.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); - SPLTR_EXIT_STATUS = 1; - aWESplitterApp.fDh.fLog.logMsg( err, WriteEngine::MSGLVL_ERROR ); - aWESplitterApp.fContinue = false; - //throw runtime_error(err); BUG 4298 - } - - aWESplitterApp.processMessages(); - - if (SPLTR_EXIT_STATUS == 1) break; - } + aWESplitterApp.fDh.setup(); } - else + catch (std::exception& ex) { - aWESplitterApp.processMessages(); + // err = string("Error in constructing WESplitterApp") + ex.what(); + err = ex.what(); // cleaning up for BUG 4298 + logging::Message::Args errMsgArgs; + errMsgArgs.add(err); + aWESplitterApp.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000); + SPLTR_EXIT_STATUS = 1; + aWESplitterApp.fDh.fLog.logMsg(err, WriteEngine::MSGLVL_ERROR); + aWESplitterApp.fContinue = false; + // throw runtime_error(err); BUG 4298 } + + aWESplitterApp.processMessages(); + + if (SPLTR_EXIT_STATUS == 1) + break; + } } - catch (std::exception& exp) + else { - cerr << exp.what() << endl; - SPLTR_EXIT_STATUS = 1; - exit(SPLTR_EXIT_STATUS); // exit with an error + aWESplitterApp.processMessages(); } + } + catch (std::exception& exp) + { + cerr << exp.what() << endl; + SPLTR_EXIT_STATUS = 1; + exit(SPLTR_EXIT_STATUS); // exit with an error + } - return SPLTR_EXIT_STATUS; + return SPLTR_EXIT_STATUS; } // vim:ts=4 sw=4: - diff --git a/writeengine/splitter/we_splitterapp.h b/writeengine/splitter/we_splitterapp.h index e3e1d1b92..efb7ae71a 100644 --- a/writeengine/splitter/we_splitterapp.h +++ b/writeengine/splitter/we_splitterapp.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_splitterapp.h * @@ -41,60 +41,54 @@ namespace WriteEngine { - - class WESplitterApp { -public: - WESplitterApp(WECmdArgs& CmdArgs); - virtual ~WESplitterApp(); + public: + WESplitterApp(WECmdArgs& CmdArgs); + virtual ~WESplitterApp(); + void processMessages(); + int getMode() + { + return fCmdArgs.getMode(); + } + bool getPmStatus(int Id) + { + return fCmdArgs.getPmStatus(Id); + } + std::string getLocFile() + { + return fCmdArgs.getLocFile(); + } + std::string getPmFile() + { + return fCmdArgs.getPmFile(); + } + void updateWithJobFile(int aIdx); + // setup the signal handlers for the main app + void setupSignalHandlers(); + static void onSigTerminate(int aInt); + static void onSigInterrupt(int aInt); + static void onSigHup(int aInt); - void processMessages(); - int getMode() - { - return fCmdArgs.getMode(); - } - bool getPmStatus(int Id) - { - return fCmdArgs.getPmStatus(Id); - } - std::string getLocFile() - { - return fCmdArgs.getLocFile(); - } - std::string getPmFile() - { - return fCmdArgs.getPmFile(); - } - void updateWithJobFile(int aIdx); + void invokeCpimport(); + private: + public: // for multi table support + WECmdArgs& fCmdArgs; + WESDHandler fDh; + static bool fContinue; - // setup the signal handlers for the main app - void setupSignalHandlers(); - static void onSigTerminate(int aInt); - static void onSigInterrupt(int aInt); - static void onSigHup(int aInt); + public: + static bool fSignaled; + static bool fSigHup; - void invokeCpimport(); - -private: -public: // for multi table support - WECmdArgs& fCmdArgs; - WESDHandler fDh; - static bool fContinue; - -public: - static bool fSignaled; - static bool fSigHup; - -public: - static SimpleSysLog* fpSysLog; - -public: - friend class WESDHandler; + public: + static SimpleSysLog* fpSysLog; + public: + friend class WESDHandler; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_tablelockgrabber.cpp b/writeengine/splitter/we_tablelockgrabber.cpp index e87e25bfe..f894586b0 100644 --- a/writeengine/splitter/we_tablelockgrabber.cpp +++ b/writeengine/splitter/we_tablelockgrabber.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_tablelockgrabber.cpp @@ -37,93 +37,78 @@ using namespace std; #include "we_sdhandler.h" #include "we_tablelockgrabber.h" - - namespace WriteEngine { - -uint64_t WETableLockGrabber::grabTableLock(std::vector& PmList, - uint32_t tableOID) +uint64_t WETableLockGrabber::grabTableLock(std::vector& PmList, uint32_t tableOID) { - uint64_t aLockId; - std::string aProcName = "cpimport"; - uint32_t aProcId = getpid(); - int32_t aSessId = -1; - int32_t aTxnId = -1; + uint64_t aLockId; + std::string aProcName = "cpimport"; + uint32_t aProcId = getpid(); + int32_t aSessId = -1; + int32_t aTxnId = -1; + try + { + aLockId = + fRef.fDbrm.getTableLock(PmList, tableOID, &aProcName, &aProcId, &aSessId, &aTxnId, BRM::LOADING); + } + catch (std::exception& e) + { + cout << "ERROR: Failed to get Table Lock " << e.what() << endl; + throw runtime_error(e.what()); + } + if (aLockId == 0) + { + ostringstream oss; + oss << " Table currently locked by process-" << aProcName << "; pid-" << aProcId << "; session-" + << aSessId << "; txn-" << aTxnId; + throw runtime_error(oss.str()); + } - try - { - aLockId = fRef.fDbrm.getTableLock(PmList, tableOID, &aProcName, - &aProcId, &aSessId, &aTxnId, BRM::LOADING); - } - catch (std::exception& e) - { + // cout << "lock ID = " << aLockId << endl; + // if (aLockId == 0) + // cout << " existing owner name = " << aProcName << " pid = " << + // aProcId << " session = " << aSessId << endl; - cout << "ERROR: Failed to get Table Lock " << e.what() << endl; - throw runtime_error(e.what()); - } - - if (aLockId == 0) - { - ostringstream oss; - oss << " Table currently locked by process-" << - aProcName << "; pid-" << aProcId << - "; session-" << aSessId << - "; txn-" << aTxnId; - throw runtime_error(oss.str()); - } - - //cout << "lock ID = " << aLockId << endl; - //if (aLockId == 0) - // cout << " existing owner name = " << aProcName << " pid = " << - // aProcId << " session = " << aSessId << endl; - - return aLockId; + return aLockId; } bool WETableLockGrabber::releaseTableLock(uint64_t LockId) { - bool aRet; + bool aRet; - //cout << "releasing lock " << LockId << endl; - try - { - aRet = fRef.fDbrm.releaseTableLock(LockId); - } - catch (std::exception& e) - { - cout << "caught an exception: " << e.what() << endl; - throw runtime_error(e.what()); - } + // cout << "releasing lock " << LockId << endl; + try + { + aRet = fRef.fDbrm.releaseTableLock(LockId); + } + catch (std::exception& e) + { + cout << "caught an exception: " << e.what() << endl; + throw runtime_error(e.what()); + } - return aRet; + return aRet; } bool WETableLockGrabber::changeTableLockState(uint64_t LockId) { - bool aRet; + bool aRet; - //cout << "changing state of lock " << LockId << endl; + // cout << "changing state of lock " << LockId << endl; - try - { - aRet = fRef.fDbrm.changeState(LockId, BRM::CLEANUP); - } - catch (std::exception& e) - { - cout << "caught an exception: " << e.what() << endl; - throw runtime_error(e.what()); - } + try + { + aRet = fRef.fDbrm.changeState(LockId, BRM::CLEANUP); + } + catch (std::exception& e) + { + cout << "caught an exception: " << e.what() << endl; + throw runtime_error(e.what()); + } - return aRet; + return aRet; } - - - - - - } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_tablelockgrabber.h b/writeengine/splitter/we_tablelockgrabber.h index cc253f9cd..1f6d144d4 100644 --- a/writeengine/splitter/we_tablelockgrabber.h +++ b/writeengine/splitter/we_tablelockgrabber.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_tableLockgrabber.h @@ -30,34 +30,30 @@ #ifndef WE_TABLELOCKGRABBER_H_ #define WE_TABLELOCKGRABBER_H_ - - namespace WriteEngine { - -class WESDHandler; // forward deceleration +class WESDHandler; // forward deceleration class WETableLockGrabber { -public: - WETableLockGrabber(WESDHandler& Ref): fRef(Ref) { } - virtual ~WETableLockGrabber() { } + public: + WETableLockGrabber(WESDHandler& Ref) : fRef(Ref) + { + } + virtual ~WETableLockGrabber() + { + } -public: + public: + uint64_t grabTableLock(std::vector& PmList, uint32_t tableOID); - uint64_t grabTableLock(std::vector& PmList, - uint32_t tableOID); + bool releaseTableLock(uint64_t LockId); - bool releaseTableLock(uint64_t LockId); - - bool changeTableLockState(uint64_t LockId); - - -private: - WESDHandler& fRef; + bool changeTableLockState(uint64_t LockId); + private: + WESDHandler& fRef; }; - } /* namespace WriteEngine */ #endif /* WE_TABLELOCKGRABBER_H_ */ diff --git a/writeengine/splitter/we_xmlgetter.cpp b/writeengine/splitter/we_xmlgetter.cpp index 8de145531..222e4fd72 100644 --- a/writeengine/splitter/we_xmlgetter.cpp +++ b/writeengine/splitter/we_xmlgetter.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_xmlgetter.cpp @@ -27,7 +27,6 @@ * Author: bpaul */ - #include #include #include @@ -44,29 +43,25 @@ using namespace std; namespace WriteEngine { - //------------------------------------------------------------------------------ // WEXmlgetter constructor //------------------------------------------------------------------------------ -WEXmlgetter::WEXmlgetter(std::string& ConfigName): - fConfigName(ConfigName), - fDoc( NULL ), - fpRoot( NULL ) +WEXmlgetter::WEXmlgetter(std::string& ConfigName) : fConfigName(ConfigName), fDoc(NULL), fpRoot(NULL) { - // xmlNodePtr curPtr; - fDoc = xmlParseFile( ConfigName.c_str() ); + // xmlNodePtr curPtr; + fDoc = xmlParseFile(ConfigName.c_str()); - if (fDoc == NULL ) - throw runtime_error("WEXmlgetter::getConfig(): no XML document!"); + if (fDoc == NULL) + throw runtime_error("WEXmlgetter::getConfig(): no XML document!"); - fpRoot = xmlDocGetRootElement( fDoc ); + fpRoot = xmlDocGetRootElement(fDoc); - if ( fpRoot == NULL ) - { - xmlFreeDoc( fDoc ); - fDoc = NULL; - throw runtime_error("WEXmlgetter::getConfig(): no XML Root Tag!"); - } + if (fpRoot == NULL) + { + xmlFreeDoc(fDoc); + fDoc = NULL; + throw runtime_error("WEXmlgetter::getConfig(): no XML Root Tag!"); + } } //------------------------------------------------------------------------------ @@ -74,66 +69,64 @@ WEXmlgetter::WEXmlgetter(std::string& ConfigName): //------------------------------------------------------------------------------ WEXmlgetter::~WEXmlgetter() { - xmlFreeDoc( fDoc ); - fDoc = NULL; + xmlFreeDoc(fDoc); + fDoc = NULL; } //------------------------------------------------------------------------------ // Get/return the property or attribute value (strVal) for the specified xml tag // (pNode) and property/attribute (pTag) //------------------------------------------------------------------------------ -bool WEXmlgetter::getNodeAttribute(const xmlNode* pNode, - const char* pTag, std::string& strVal) const +bool WEXmlgetter::getNodeAttribute(const xmlNode* pNode, const char* pTag, std::string& strVal) const { - xmlChar* pTmp = NULL; - bool bFound = false; + xmlChar* pTmp = NULL; + bool bFound = false; - pTmp = xmlGetProp( const_cast(pNode), (xmlChar*) pTag ); + pTmp = xmlGetProp(const_cast(pNode), (xmlChar*)pTag); - if ( pTmp ) - { - bFound = true; - strVal = (char*)pTmp; - xmlFree( pTmp ); - } - else - { - strVal.clear(); - } // end if + if (pTmp) + { + bFound = true; + strVal = (char*)pTmp; + xmlFree(pTmp); + } + else + { + strVal.clear(); + } // end if - return bFound; + return bFound; } //------------------------------------------------------------------------------ // Get/return the node content (strVal) for the specified xml tag (pNode) //------------------------------------------------------------------------------ -bool WEXmlgetter::getNodeContent( const xmlNode* pNode, - std::string& strVal) const +bool WEXmlgetter::getNodeContent(const xmlNode* pNode, std::string& strVal) const { - xmlChar* pTmp = NULL; - bool bFound = false; + xmlChar* pTmp = NULL; + bool bFound = false; - if ( pNode->children != NULL ) + if (pNode->children != NULL) + { + pTmp = xmlNodeGetContent(pNode->children); + + if (pTmp) { - pTmp = xmlNodeGetContent( pNode->children ); - - if ( pTmp ) - { - bFound = true; - strVal = (char*)pTmp; - xmlFree( pTmp ); - } - else - { - strVal.clear(); - } + bFound = true; + strVal = (char*)pTmp; + xmlFree(pTmp); } else { - strVal.clear(); + strVal.clear(); } + } + else + { + strVal.clear(); + } - return bFound; + return bFound; } //------------------------------------------------------------------------------ @@ -155,47 +148,46 @@ bool WEXmlgetter::getNodeContent( const xmlNode* pNode, // always returning the text node content inside each rather than // any node that might be within each tag. //------------------------------------------------------------------------------ -void WEXmlgetter::getConfig(const string& section, - const string& name, vector& values) const +void WEXmlgetter::getConfig(const string& section, const string& name, vector& values) const { - string res; + string res; - if (section.length() == 0) - throw invalid_argument("Config::getConfig: section must have a length"); + if (section.length() == 0) + throw invalid_argument("Config::getConfig: section must have a length"); - xmlNode* pPtr = fpRoot->xmlChildrenNode; + xmlNode* pPtr = fpRoot->xmlChildrenNode; - while (pPtr != NULL) + while (pPtr != NULL) + { + // cout << "pPtr->name: " << + // (const xmlChar*)pPtr->name << std::endl; + + if ((!xmlStrcmp(pPtr->name, (const xmlChar*)section.c_str()))) { - //cout << "pPtr->name: " << - // (const xmlChar*)pPtr->name << std::endl; + xmlNodePtr pPtr2 = pPtr->xmlChildrenNode; - if ((!xmlStrcmp(pPtr->name, (const xmlChar*)section.c_str()))) + while (pPtr2 != NULL) + { + // cout << " pPtr2->name: " << + // (const xmlChar*)pPtr2->name << std::endl; + + if ((!xmlStrcmp(pPtr2->name, (const xmlChar*)name.c_str()))) { - xmlNodePtr pPtr2 = pPtr->xmlChildrenNode; + xmlNodePtr pPtr3 = pPtr2->xmlChildrenNode; + values.push_back((const char*)pPtr3->content); - while (pPtr2 != NULL) - { - //cout << " pPtr2->name: " << - // (const xmlChar*)pPtr2->name << std::endl; - - if ((!xmlStrcmp(pPtr2->name, (const xmlChar*)name.c_str()))) - { - xmlNodePtr pPtr3 = pPtr2->xmlChildrenNode; - values.push_back((const char*)pPtr3->content); - - //cout << " pPtr3->name: " << - // (const xmlChar*)pPtr3->name << - // "; content: " << (const xmlChar*)pPtr3->content << - // "; len: " << strlen((char*)pPtr3->content) << std::endl; - } - - pPtr2 = pPtr2->next; - } + // cout << " pPtr3->name: " << + // (const xmlChar*)pPtr3->name << + // "; content: " << (const xmlChar*)pPtr3->content << + // "; len: " << strlen((char*)pPtr3->content) << std::endl; } - pPtr = pPtr->next; + pPtr2 = pPtr2->next; + } } + + pPtr = pPtr->next; + } } //------------------------------------------------------------------------------ @@ -210,35 +202,37 @@ void WEXmlgetter::getConfig(const string& section, //------------------------------------------------------------------------------ std::string WEXmlgetter::getValue(const vector& sections) const { - std::string aRet; - const xmlNode* pPtr = fpRoot; - int aSize = sections.size(); - int aIdx = 0; + std::string aRet; + const xmlNode* pPtr = fpRoot; + int aSize = sections.size(); + int aIdx = 0; - //cout << aSize << endl; - while (aIdx < aSize) + // cout << aSize << endl; + while (aIdx < aSize) + { + // cout << aIdx <<" "<< sections[aIdx] << endl; + pPtr = getNode(pPtr, sections[aIdx]); + + if ((pPtr == NULL) || (aIdx == aSize - 1)) + break; + else { - //cout << aIdx <<" "<< sections[aIdx] << endl; - pPtr = getNode(pPtr, sections[aIdx]); - - if ((pPtr == NULL) || (aIdx == aSize - 1)) break; - else - { - //cout << "getValue Name " << (const char*)pPtr->name << endl; - pPtr = pPtr->xmlChildrenNode; - aIdx++; - } + // cout << "getValue Name " << (const char*)pPtr->name << endl; + pPtr = pPtr->xmlChildrenNode; + aIdx++; } + } - if (pPtr != NULL) - { - //aRet = (const char*)pPtr->content; - std::string aBuff; + if (pPtr != NULL) + { + // aRet = (const char*)pPtr->content; + std::string aBuff; - if (getNodeContent(pPtr, aBuff)) aRet = aBuff; - } + if (getNodeContent(pPtr, aBuff)) + aRet = aBuff; + } - return aRet; + return aRet; } //------------------------------------------------------------------------------ @@ -246,23 +240,23 @@ std::string WEXmlgetter::getValue(const vector& sections) const // a node with the specified name (section). The xmlNode (if found) is // returned. //------------------------------------------------------------------------------ -const xmlNode* WEXmlgetter::getNode(const xmlNode* pParent, - const string& section)const +const xmlNode* WEXmlgetter::getNode(const xmlNode* pParent, const string& section) const { - if (pParent == NULL) return NULL; + if (pParent == NULL) + return NULL; - const xmlNode* pPtr = pParent; + const xmlNode* pPtr = pParent; - while (pPtr != NULL ) - { - //cout << "getNode Name " << (const char*)pPtr->name << endl; - if (!xmlStrcmp(pPtr->name, (const xmlChar*)section.c_str())) - return pPtr; - else - pPtr = pPtr->next; - } + while (pPtr != NULL) + { + // cout << "getNode Name " << (const char*)pPtr->name << endl; + if (!xmlStrcmp(pPtr->name, (const xmlChar*)section.c_str())) + return pPtr; + else + pPtr = pPtr->next; + } - return pPtr; + return pPtr; } //------------------------------------------------------------------------------ @@ -270,46 +264,46 @@ const xmlNode* WEXmlgetter::getNode(const xmlNode* pParent, // In the last child of this tree, we look for the specified attribute tag, // and return its value. //------------------------------------------------------------------------------ -std::string WEXmlgetter::getAttribute(const vector& sections, - const string& Tag) const +std::string WEXmlgetter::getAttribute(const vector& sections, const string& Tag) const { - std::string aRet; - const xmlNode* pPtr = fpRoot; - int aSize = sections.size(); + std::string aRet; + const xmlNode* pPtr = fpRoot; + int aSize = sections.size(); - if (aSize == 0) - throw invalid_argument("WEXmlgetter::getAttribute(): section must be valid"); + if (aSize == 0) + throw invalid_argument("WEXmlgetter::getAttribute(): section must be valid"); - int aIdx = 0; + int aIdx = 0; - //cout << aSize << endl; - while (aIdx < aSize) + // cout << aSize << endl; + while (aIdx < aSize) + { + // cout << aIdx <<" "<< sections[aIdx] << endl; + pPtr = getNode(pPtr, sections[aIdx]); + + if ((pPtr == NULL) || (aIdx == aSize - 1)) + break; + else { - //cout << aIdx <<" "<< sections[aIdx] << endl; - pPtr = getNode(pPtr, sections[aIdx]); - - if ((pPtr == NULL) || (aIdx == aSize - 1)) break; - else - { - //cout << "getValue Name " << (const char*)pPtr->name << endl; - pPtr = pPtr->xmlChildrenNode; - aIdx++; - } + // cout << "getValue Name " << (const char*)pPtr->name << endl; + pPtr = pPtr->xmlChildrenNode; + aIdx++; } + } - if (pPtr != NULL) - { - std::string aBuff; + if (pPtr != NULL) + { + std::string aBuff; - //cout << "attrTagNode Name " << (const char*)pPtr->name << endl; - if (getNodeAttribute(pPtr, Tag.c_str(), aBuff)) - aRet = aBuff; + // cout << "attrTagNode Name " << (const char*)pPtr->name << endl; + if (getNodeAttribute(pPtr, Tag.c_str(), aBuff)) + aRet = aBuff; - //aRet = (const char*)pPtr->content; - //cout << "Attribute("<content; + // cout << "Attribute("<& sections, // attribute tag, and return its value. Hence a vector of attribute values // is returned. //------------------------------------------------------------------------------ -void WEXmlgetter::getAttributeListForAllChildren( - const vector& sections, - const string& attributeTag, - vector& attributeValues) +void WEXmlgetter::getAttributeListForAllChildren(const vector& sections, const string& attributeTag, + vector& attributeValues) { - const xmlNode* pPtr = fpRoot; - int aSize = sections.size(); + const xmlNode* pPtr = fpRoot; + int aSize = sections.size(); - if (aSize == 0) + if (aSize == 0) + { + throw invalid_argument( + "WEXmlgetter::getAttributeListForAllChildren():" + " No XML nodes specified in section search list"); + } + + // Step down the branch that has the nodes of interest + int aIdx = 0; + + while (aIdx < aSize) + { + pPtr = getNode(pPtr, sections[aIdx]); + + if ((pPtr == NULL) || (aIdx == aSize - 1)) { - throw invalid_argument("WEXmlgetter::getAttributeListForAllChildren():" - " No XML nodes specified in section search list"); + break; } - - // Step down the branch that has the nodes of interest - int aIdx = 0; - - while (aIdx < aSize) + else { - pPtr = getNode(pPtr, sections[aIdx]); - - if ((pPtr == NULL) || (aIdx == aSize - 1)) - { - break; - } - else - { - pPtr = pPtr->xmlChildrenNode; - aIdx++; - } + pPtr = pPtr->xmlChildrenNode; + aIdx++; } + } - // Look for all the "matching" nodes at the end of the branch, and - // get the requested attribute value for each matching node. - if (pPtr != NULL) + // Look for all the "matching" nodes at the end of the branch, and + // get the requested attribute value for each matching node. + if (pPtr != NULL) + { + while (pPtr != NULL) { - while (pPtr != NULL ) - { - std::string attrib; + std::string attrib; - if (getNodeAttribute(pPtr, attributeTag.c_str(), attrib)) - { - attributeValues.push_back(attrib); - } + if (getNodeAttribute(pPtr, attributeTag.c_str(), attrib)) + { + attributeValues.push_back(attrib); + } - pPtr = pPtr->next; - } + pPtr = pPtr->next; } + } } } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_xmlgetter.h b/writeengine/splitter/we_xmlgetter.h index 935cd4d27..55a6a8bef 100644 --- a/writeengine/splitter/we_xmlgetter.h +++ b/writeengine/splitter/we_xmlgetter.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id$ -* -*******************************************************************************/ + * $Id$ + * + *******************************************************************************/ /* * we_xmlgetter.h @@ -34,37 +34,31 @@ namespace WriteEngine { - class WEXmlgetter { -public: - WEXmlgetter(std::string& ConfigName); - virtual ~WEXmlgetter(); + public: + WEXmlgetter(std::string& ConfigName); + virtual ~WEXmlgetter(); -public: - //..Public methods - std::string getValue(const std::vector& section) const; - std::string getAttribute(const std::vector& sections, - const std::string& Tag) const; - void getConfig(const std::string& section, - const std::string& name, std::vector& values ) const; - void getAttributeListForAllChildren( - const std::vector& sections, - const std::string& attributeTag, - std::vector& attributeValues); + public: + //..Public methods + std::string getValue(const std::vector& section) const; + std::string getAttribute(const std::vector& sections, const std::string& Tag) const; + void getConfig(const std::string& section, const std::string& name, std::vector& values) const; + void getAttributeListForAllChildren(const std::vector& sections, + const std::string& attributeTag, + std::vector& attributeValues); -private: - //..Private methods - const xmlNode* getNode(const xmlNode* pParent, - const std::string& section)const; - bool getNodeAttribute(const xmlNode* pNode, - const char* pTag, std::string& strVal ) const; - bool getNodeContent( const xmlNode* pNode, std::string& strVal) const; + private: + //..Private methods + const xmlNode* getNode(const xmlNode* pParent, const std::string& section) const; + bool getNodeAttribute(const xmlNode* pNode, const char* pTag, std::string& strVal) const; + bool getNodeContent(const xmlNode* pNode, std::string& strVal) const; - //..Private data members - std::string fConfigName; // xml filename - xmlDocPtr fDoc; // xml document pointer - xmlNode* fpRoot; // root element + //..Private data members + std::string fConfigName; // xml filename + xmlDocPtr fDoc; // xml document pointer + xmlNode* fpRoot; // root element }; } /* namespace WriteEngine */ diff --git a/writeengine/wrapper/tdriver.cpp b/writeengine/wrapper/tdriver.cpp index f2044e3e8..825185eae 100644 --- a/writeengine/wrapper/tdriver.cpp +++ b/writeengine/wrapper/tdriver.cpp @@ -36,2908 +36,2902 @@ extern WriteEngine::BRMWrapper* brmWrapperPtr; class WriteEngineTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(WriteEngineTest); - CPPUNIT_TEST_SUITE( WriteEngineTest ); + // Column related operation testing + /* + CPPUNIT_TEST( testCreateColumnFiles ); + CPPUNIT_TEST( testWriteColumnFiles ); + CPPUNIT_TEST( testReadColumnFiles ); -// Column related operation testing + CPPUNIT_TEST( testInsertRowInt ); + CPPUNIT_TEST( testInsertRowLonglong ); + CPPUNIT_TEST( testInsertRowDouble ); + CPPUNIT_TEST( testInsertRowChar ); + CPPUNIT_TEST( testInsertBulk ); + + // Table related operation testing + CPPUNIT_TEST( testTableAllocateRowId ); + + // Wrapper interface testing + CPPUNIT_TEST( testInterfaceCreateColumnFile ); + CPPUNIT_TEST( testInterfaceInsertUpdateDeleteRow ); + CPPUNIT_TEST( testInterfaceInsertRowMedIntDouble ); + + CPPUNIT_TEST( testInterfaceInsertRowChar ); + + CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth4 ); + CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth3 ); + CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth8 ); + CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth1 ); + + + CPPUNIT_TEST( testInterfaceInsertRowLongLong ); + CPPUNIT_TEST( testInterfaceInsertRowByte ); + CPPUNIT_TEST( testInterfaceInsertRowToken ); + CPPUNIT_TEST( testInterfaceInsertRowSingleChar ); + CPPUNIT_TEST( testInterfaceInsertRowDoubleChar ); + + CPPUNIT_TEST( testInterfaceCreateIndex ); + CPPUNIT_TEST( testInterfaceDropIndex ); + CPPUNIT_TEST( testInterfaceUpdateIndexChar ); + CPPUNIT_TEST( testInterfaceUpdateMultiColIndex); + + CPPUNIT_TEST( testInterfaceDctnryToken ); + CPPUNIT_TEST( testInterfaceDctnryTokenRollBack ); + + CPPUNIT_TEST( testInsertCommitRollback ); + CPPUNIT_TEST( testInterfaceInsertRowHwm ); + + CPPUNIT_TEST( testInterfaceCombineIndexLoad ); // note: this test case must be the last one + //CPPUNIT_TEST( testTmpBulkPerformance ); // note: this test case must be the last one + */ + // CPPUNIT_TEST(testBoostFloat); + /* + CPPUNIT_TEST(testCreateColumnFiles1); + CPPUNIT_TEST(testCreateColumnFiles2); + CPPUNIT_TEST(testCreateColumnFiles3); + CPPUNIT_TEST(testCreateColumnFiles4); + */ + + // CPPUNIT_TEST(testCreateFileMultipleIONBF); + // CPPUNIT_TEST(testCreateFile64MBIONBF); + // CPPUNIT_TEST(testCreateFileMultipleBF); + // CPPUNIT_TEST(testCreateFilelargerBF); + CPPUNIT_TEST(testSoloman); + /* + CPPUNIT_TEST(testCreateFileNoBRM); + CPPUNIT_TEST(testCreateFileBRM); + */ + CPPUNIT_TEST_SUITE_END(); + + private: + WriteEngineWrapper m_wrapper; + Session m_session; + + public: + void setUp() + { + m_wrapper.setDebugLevel(DEBUG_3); + m_session.txnid = 10; + BRMWrapper::setUseBrm(true); + } + + void tearDown() + { + } + + void SetColumnStruct(ColStruct& column, OID dataOid, int colWidth, + CalpontSystemCatalog::ColDataType colDataType) + { + column.dataOid = dataOid; + column.colWidth = colWidth; + column.colDataType = colDataType; + column.tokenFlag = false; + } + + void CreateColumnFile(FID fid, int width, CalpontSystemCatalog::ColDataType colDataType, ColType colType) + { + int rc; + ColumnOp colOp; + Column curCol; + + colOp.deleteFile(fid); + CPPUNIT_ASSERT(colOp.exists(fid) == false); + + // create column files + rc = colOp.createColumn(curCol, 2, width, colDataType, colType, fid); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testCreateColumnFiles() + { + ColumnOp colOp; + Column curCol; + int rc; + + colOp.initColumn(curCol); + + // file opertaions + colOp.deleteFile(100); + CPPUNIT_ASSERT(colOp.exists(100) == false); + + colOp.deleteFile(101); + CPPUNIT_ASSERT(colOp.exists(101) == false); + + colOp.deleteFile(103); + CPPUNIT_ASSERT(colOp.exists(103) == false); + + rc = colOp.createColumn(curCol, 2, 50, WriteEngine::CHAR, WriteEngine::WR_CHAR, 101); + CPPUNIT_ASSERT(rc == NO_ERROR); + + colOp.deleteFile(101); + rc = colOp.createColumn(curCol, 2, 3, WriteEngine::CHAR, WriteEngine::WR_CHAR, 101); + CPPUNIT_ASSERT(rc == NO_ERROR); + + colOp.deleteFile(101); + rc = colOp.createColumn(curCol, 2, 5, WriteEngine::CHAR, WriteEngine::WR_CHAR, 101); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // create column files + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR, 100); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = colOp.createColumn(curCol, 2, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR, 100); + CPPUNIT_ASSERT(rc == ERR_FILE_EXIST); + + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + colOp.clearColumn(curCol); + + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == ERR_INVALID_PARAM); + + rc = colOp.createTable(); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testWriteColumnFiles() + { + ColumnOp colOp; + Column curCol; + int rc; + unsigned char buf[BYTE_PER_BLOCK]; + + colOp.initColumn(curCol); + CPPUNIT_ASSERT(curCol.colNo == 0); + CPPUNIT_ASSERT(curCol.colWidth == 0); + CPPUNIT_ASSERT(curCol.dataFile.pFile == NULL); + // CPPUNIT_ASSERT( curCol.bitmapFile.pFile == NULL ); + CPPUNIT_ASSERT(curCol.dataFile.fid == 0); + // CPPUNIT_ASSERT( curCol.bitmapFile.fid == 0 ); + + colOp.setColParam(curCol, 2, 4, INT, WriteEngine::WR_INT, 100); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + int testVal = 60; + memset(buf, 0, BYTE_PER_BLOCK); + memcpy(buf + 8, &testVal, 4); + + rc = colOp.writeDBFileFbo(curCol.dataFile.pFile, buf, 1, 1); // block 2 + CPPUNIT_ASSERT(rc == NO_ERROR); + + colOp.clearColumn(curCol); + + CPPUNIT_ASSERT(curCol.colNo == 0); + CPPUNIT_ASSERT(curCol.colWidth == 0); + CPPUNIT_ASSERT(curCol.dataFile.pFile == NULL); + // CPPUNIT_ASSERT( curCol.bitmapFile.pFile == NULL ); + CPPUNIT_ASSERT(curCol.dataFile.fid == 0); + // CPPUNIT_ASSERT( curCol.bitmapFile.fid == 0 ); + + CPPUNIT_ASSERT(colOp.isValid(curCol) == false); + } + + void testReadColumnFiles() + { + ColumnOp colOp; + Column curCol, errCol; + int rc; + unsigned char buf[BYTE_PER_BLOCK]; + CalpontSystemCatalog::ColDataType colDataType; + + // colOp.setUseBrm(false); + CPPUNIT_ASSERT(colOp.getColDataType("integer", colDataType) == true); + CPPUNIT_ASSERT(colDataType == INT); + CPPUNIT_ASSERT(colOp.getColDataType("Int1", colDataType) == false); + + // check error situation + colOp.initColumn(errCol); + colOp.setColParam(errCol, 2, 8, CHAR, WriteEngine::WR_CHAR, 103); + rc = colOp.openColumnFile(errCol); + CPPUNIT_ASSERT(rc == ERR_FILE_READ); + colOp.clearColumn(errCol); + + colOp.initColumn(errCol); + colOp.setColParam(errCol, 2, 8, CHAR, WriteEngine::WR_CHAR, 103); + rc = colOp.openColumnFile(errCol); + CPPUNIT_ASSERT(rc == ERR_FILE_READ); + colOp.clearColumn(errCol); + + // check normal situation + colOp.initColumn(curCol); + + colOp.setColParam(curCol, 2, 8, CHAR, WriteEngine::WR_CHAR, 100); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + memset(buf, 0, BYTE_PER_BLOCK); + + rc = colOp.readDBFile(curCol.dataFile.pFile, buf, 1, true); // block 2 + CPPUNIT_ASSERT(rc == NO_ERROR); + + int testVal = 0; + memcpy(&testVal, buf + 8, 8); + printf("\nread the test value : %d", testVal); + CPPUNIT_ASSERT(testVal == 60); + + colOp.clearColumn(curCol); + } + + void testInsertRowInt() + { + ColumnOp colOp; + Column curCol; + int rc, valArray[5], oldValArray[5]; + RID rowIdArray[5]; + + rowIdArray[0] = 1; + rowIdArray[1] = 3; + rowIdArray[2] = 4; + rowIdArray[3] = 7; + rowIdArray[4] = 8; + + valArray[0] = 8; + valArray[1] = 5; + valArray[2] = 3; + valArray[3] = 0; + valArray[4] = 16; + + CreateColumnFile(100, 4, WriteEngine::INT, WriteEngine::WR_INT); + + colOp.initColumn(curCol); + colOp.setColParam(curCol, 2, 4, INT, WriteEngine::WR_INT, 100); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = colOp.writeRow(curCol, 5, (RID*)rowIdArray, valArray, oldValArray); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (int i = 0; i < 5; i++) + CPPUNIT_ASSERT(oldValArray[i] == (int)0x80000001); + + colOp.clearColumn(curCol); + } + + void testInsertRowLonglong() + { + ColumnOp colOp; + Column curCol; + int rc; + RID rowIdArray[3]; + int width = 8, totalRow = 3; + FID fid = 100; + long long valArray[3]; + uint64_t oldValArray[3], verifyArray[3]; + CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::BIGINT; + ColType colType = WriteEngine::WR_LONGLONG; + uint64_t emptyVal = 0x8000000000000001LL; + + rowIdArray[0] = 1; + rowIdArray[1] = 3; + rowIdArray[2] = 4; + + verifyArray[0] = valArray[0] = 32111238; + verifyArray[1] = valArray[1] = 1231235; + verifyArray[2] = valArray[2] = 67731233; + + CreateColumnFile(fid, width, colDataType, colType); + + colOp.initColumn(curCol); + colOp.setColParam(curCol, 2, width, colDataType, colType, fid); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = colOp.writeRow(curCol, totalRow, (RID*)rowIdArray, valArray, oldValArray); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (int i = 0; i < totalRow; i++) + CPPUNIT_ASSERT(oldValArray[i] == emptyVal); + + valArray[0] = 1900003; + valArray[1] = 2349000; + valArray[2] = 78900123; + + rc = colOp.writeRow(curCol, totalRow, (RID*)rowIdArray, valArray, oldValArray); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (int i = 0; i < totalRow; i++) + CPPUNIT_ASSERT(oldValArray[i] == verifyArray[i]); + + colOp.clearColumn(curCol); + } + + void testInsertRowDouble() + { + ColumnOp colOp; + Column curCol; + int rc; + RID rowIdArray[3]; + int width = 8, totalRow = 3; + FID fid = 100; + double valArray[3]; + int64_t oldValArray[3]; + CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::DOUBLE; + ColType colType = WriteEngine::WR_DOUBLE; + int64_t emptyVal = 0xFFFAAAAAAAAAAAABLL; + + rowIdArray[0] = 1; + rowIdArray[1] = 3; + rowIdArray[2] = 4; + + valArray[0] = 8; + valArray[1] = 5; + valArray[2] = 3; + + CreateColumnFile(fid, width, colDataType, colType); + + colOp.initColumn(curCol); + colOp.setColParam(curCol, 2, width, colDataType, colType, fid); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = colOp.writeRow(curCol, totalRow, (RID*)rowIdArray, valArray, oldValArray); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (int i = 0; i < totalRow; i++) + CPPUNIT_ASSERT(oldValArray[i] == emptyVal); + + colOp.clearColumn(curCol); + } + + void testInsertRowChar() + { + ColumnOp colOp; + Column curCol; + int rc; + RID rowIdArray[3]; + int width = 8, totalRow = 3; + FID fid = 100; + char valArray[24], buf[8]; + char testValue[3][8] = {"abc", "eDFFF", "GHK"}; + uint64_t oldValArray[3]; + CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::CHAR; + ColType colType = WriteEngine::WR_CHAR; + uint64_t emptyVal = 0xFFFFFFFFFFFFFFFFLL; + + rowIdArray[0] = 1; + rowIdArray[1] = 3; + rowIdArray[2] = 4; + + memset(valArray, 0, 24); + colOp.writeBufValue((unsigned char*)buf, testValue[0], 8, true); + memcpy(valArray, buf, 8); + colOp.writeBufValue((unsigned char*)buf, testValue[1], 8, true); + memcpy(valArray + 8, buf, 8); + colOp.writeBufValue((unsigned char*)buf, testValue[2], 8, true); + memcpy(valArray + 16, buf, 8); + + CreateColumnFile(fid, width, colDataType, colType); + + colOp.initColumn(curCol); + colOp.setColParam(curCol, 2, width, colDataType, colType, fid); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = colOp.writeRow(curCol, totalRow, (RID*)rowIdArray, valArray, oldValArray); + CPPUNIT_ASSERT(rc == NO_ERROR); + + for (int i = 0; i < totalRow; i++) + CPPUNIT_ASSERT(oldValArray[i] == emptyVal); + + colOp.clearColumn(curCol); + } + + void testInsertBulk() + { + FILE* pSourceFile = NULL; + ColumnOp colOp; + Column curCol; + int rc, width = 8; + int hwm; + FID fid = 999; + CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::CHAR; + ColType colType = WriteEngine::WR_CHAR; + + colOp.deleteFile(999); + colOp.deleteFile(998); + CPPUNIT_ASSERT(colOp.exists(999) == false); + CPPUNIT_ASSERT(colOp.exists(998) == false); + + int allocSize = 0; + CPPUNIT_ASSERT(colOp.createFile(998, 20, allocSize, 0xEEEE, 2) == NO_ERROR); + CreateColumnFile(fid, width, colDataType, colType); + + colOp.initColumn(curCol); + colOp.setColParam(curCol, 2, width, colDataType, colType, fid); + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + pSourceFile = colOp.openFile(998); + CPPUNIT_ASSERT(pSourceFile != NULL); + + hwm = colOp.getFileSize(curCol.dataFile.pFile) / BYTE_PER_BLOCK - 5; + rc = colOp.insertBulk(curCol, pSourceFile, hwm, 20); + CPPUNIT_ASSERT(rc == NO_ERROR); + + colOp.clearColumn(curCol); + colOp.closeFile(pSourceFile); + } + + void testTableAllocateRowId() + { + ColumnOp colOp; + Column curCol; + RID rowIdArray[3]; + int rc; + int width = 4, hwm = 0; + FID fid = 100; + CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::INT; + ColType colType = WriteEngine::WR_INT; + + CreateColumnFile(fid, width, colDataType, colType); + + colOp.initColumn(curCol); + colOp.setColParam(curCol, 2, width, colDataType, colType, fid); + + rc = colOp.openColumnFile(curCol); + CPPUNIT_ASSERT(rc == NO_ERROR); + + colOp.allocRowId(curCol, 3, rowIdArray, hwm); + CPPUNIT_ASSERT(rowIdArray[0] == 0); + CPPUNIT_ASSERT(rowIdArray[1] == 1); + CPPUNIT_ASSERT(rowIdArray[2] == 2); + } + + void testInterfaceCreateColumnFile() + { + ColumnOp colOp; + FID fid = 100; + + if (colOp.exists(fid)) + { + CPPUNIT_ASSERT(m_wrapper.dropColumn(m_session.txnid, fid) == NO_ERROR); + CPPUNIT_ASSERT(m_wrapper.dropColumn(m_session.txnid, fid) == ERR_FILE_NOT_EXIST); + } + + CPPUNIT_ASSERT(m_wrapper.createColumn(m_session.txnid, fid, WriteEngine::DOUBLE, 8) == NO_ERROR); + CPPUNIT_ASSERT(colOp.exists(100) == true); + } + + void testInterfaceInsertUpdateDeleteRow() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + int rc, total, valArray[3]; + float dVal; + // ColumnOp colOp; + + // colOp = m_wrapper.getColumnOp(); + + CreateColumnFile(100, 4, WriteEngine::INT, WriteEngine::WR_INT); + CreateColumnFile(200, 4, WriteEngine::FLOAT, WriteEngine::WR_FLOAT); + + // test column struct list + SetColumnStruct(curColStruct, 100, 4, WriteEngine::INT); + colStructList.push_back(curColStruct); + + SetColumnStruct(curColStruct, 200, 4, WriteEngine::FLOAT); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 4); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::INT); + + testColStruct = colStructList[1]; + CPPUNIT_ASSERT(testColStruct.dataOid == 200); + CPPUNIT_ASSERT(testColStruct.colWidth == 4); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::FLOAT); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // test column values + // add values for the first column + curTuple.data = 3; + curTupleList.push_back(curTuple); + + curTuple.data = 0; + curTupleList.push_back(curTuple); + + curTuple.data = 100; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + m_wrapper.convertValArray(3, WriteEngine::WR_INT, curTupleList, valArray); + CPPUNIT_ASSERT(valArray[0] == 3); + CPPUNIT_ASSERT(valArray[1] == 0); + CPPUNIT_ASSERT(valArray[2] == 100); + + curTupleList.clear(); + + // add values for the second column + curTuple.data = 0.0f; + curTupleList.push_back(curTuple); + + curTuple.data = 1234.78f; + curTupleList.push_back(curTuple); + + curTuple.data = 999.98f; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + float valArray1[3]; + m_wrapper.convertValArray(3, WriteEngine::WR_FLOAT, curTupleList, valArray1); + CPPUNIT_ASSERT(valArray1[0] == 0.0f); + CPPUNIT_ASSERT(valArray1[1] == 1234.78f); + CPPUNIT_ASSERT(valArray1[2] == 999.98f); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 3); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 0); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 100); + + // second column + testTupleList = static_cast(colValueList[1]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + dVal = boost::any_cast(testTuple.data); + CPPUNIT_ASSERT(dVal == 0.0f); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 1234.78f); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 999.98f); + + // m_wrapper.printInputValue( colStructList, colValueList, ridList ); + + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 3); + CPPUNIT_ASSERT(ridList[1] == 4); + CPPUNIT_ASSERT(ridList[2] == 5); + + // try to update rows + colValueList.clear(); + curTupleList.clear(); + + curTuple.data = 9; + curTupleList.push_back(curTuple); + + curTuple.data = 15; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + + // add values for the second column + curTupleList.clear(); + + curTuple.data = 1.99f; + curTupleList.push_back(curTuple); + + curTuple.data = 3000.15f; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + + ridList.clear(); + ridList.push_back((RID)2); + ridList.push_back((RID)3); + + rc = m_wrapper.updateColumnRec(m_session.txnid, colStructList, colValueList, colOldValList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(colOldValList.size() == 2); + + m_wrapper.printInputValue(colStructList, colOldValList, ridList); + curTupleList = colOldValList[0]; + curTuple = curTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 100); + curTuple = curTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 3); + + curTupleList = colOldValList[1]; + curTuple = curTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 999.98f); + curTuple = curTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 0.0f); + + ridList[0] = 1; + ridList[1] = 2; + + rc = m_wrapper.deleteRow(m_session.txnid, colStructList, colOldValList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(colOldValList.size() == 2); + + m_wrapper.printInputValue(colStructList, colOldValList, ridList); + curTupleList = colOldValList[0]; + curTuple = curTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 0); + curTuple = curTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 9); + + curTupleList = colOldValList[1]; + curTuple = curTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 1234.78f); + curTuple = curTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(curTuple.data) == 1.99f); + } + + void testInterfaceInsertRowMedIntDouble() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + int total, rc; + double dVal; + + CreateColumnFile(100, 4, WriteEngine::MEDINT, WriteEngine::WR_INT); + CreateColumnFile(200, 8, WriteEngine::DOUBLE, WriteEngine::WR_DOUBLE); + + // test column struct list + SetColumnStruct(curColStruct, 100, 4, WriteEngine::MEDINT); + colStructList.push_back(curColStruct); + + SetColumnStruct(curColStruct, 200, 8, WriteEngine::DOUBLE); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 4); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::MEDINT); + + testColStruct = colStructList[1]; + CPPUNIT_ASSERT(testColStruct.dataOid == 200); + CPPUNIT_ASSERT(testColStruct.colWidth == 8); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::DOUBLE); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // test column values + // add values for the first column + curTuple.data = 102; + curTupleList.push_back(curTuple); + + curTuple.data = 0; + curTupleList.push_back(curTuple); + + curTuple.data = 200; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add values for the second column + curTuple.data = 0.0; + curTupleList.push_back(curTuple); + + curTuple.data = 1234.78; + curTupleList.push_back(curTuple); + + curTuple.data = 999.98; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)9); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 102); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 0); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 200); + + // second column + testTupleList = static_cast(colValueList[1]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + dVal = boost::any_cast(testTuple.data); + CPPUNIT_ASSERT(dVal == 0.0); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 1234.78); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 999.98); + + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 3); + CPPUNIT_ASSERT(ridList[1] == 4); + CPPUNIT_ASSERT(ridList[2] == 5); + } + + void testInterfaceInsertRowChar() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + std::string curStr, testStr; + int total, rc, width = 5; + + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc != NO_ERROR); + + rc = m_wrapper.updateColumnRec(m_session.txnid, colStructList, colValueList, colOldValList, ridList); + CPPUNIT_ASSERT(rc != NO_ERROR); + + // test column struct list + CreateColumnFile(100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, width, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == width); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // test column values + // add values for the first column + curStr = "aaaaa"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "bbbbb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "ccccc"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)9); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "aaaaa"); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "bbbbb"); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "ccccc"); + + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 3); + CPPUNIT_ASSERT(ridList[1] == 4); + CPPUNIT_ASSERT(ridList[2] == 5); + } + + void testInterfaceInsertRowCharQAWidth4() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + std::string curStr, testStr; + int rc, width = 4; + + // test column struct list + CreateColumnFile(100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, width, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == width); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + // test column values + // add values for the first column + curStr = "aaaa"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "aaa"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "aa"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "a"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + CPPUNIT_ASSERT(ridList[3] == 3); + + // separate inserts + colValueList.clear(); + curStr = "bbbb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 4); + + colValueList.clear(); + curStr = "bbb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 5); + + colValueList.clear(); + curStr = "bb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 6); + + colValueList.clear(); + curStr = "b"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 7); + } + + void testInterfaceInsertRowCharQAWidth3() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + std::string curStr, testStr; + int rc, width = 3; + + // test column struct list + CreateColumnFile(100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, width, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == width); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + // test column values + // add values for the first column + curStr = "aaa"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "aa"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "a"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // separate inserts + colValueList.clear(); + curStr = "bbb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 3); + + colValueList.clear(); + curStr = "bb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 4); + + colValueList.clear(); + curStr = "b"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 5); + } + + void testInterfaceInsertRowCharQAWidth8() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + std::string curStr, testStr; + int rc, width = 8; + + // test column struct list + CreateColumnFile(100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, width, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == width); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + // test column values + // add values for the first column + curStr = "12345678"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "123456"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "1234"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // separate inserts + colValueList.clear(); + curStr = "bbbb1234"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 3); + + colValueList.clear(); + curStr = "bb"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 4); + + colValueList.clear(); + curStr = "b"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 5); + } + + void testInterfaceInsertRowCharQAWidth1() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + std::string curStr, testStr; + int rc, width = 1; + + // test column struct list + CreateColumnFile(100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, width, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == width); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + // test column values + // add values for the first column + curStr = "a"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "b"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + curStr = "c"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // separate inserts + colValueList.clear(); + curStr = "1"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 3); + + colValueList.clear(); + curStr = "2"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 4); + + colValueList.clear(); + curStr = "3"; + curTuple.data = curStr; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); + curTupleList.clear(); + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 5); + } + + void testInterfaceInsertRowSingleChar() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + string curString; + int total, rc; + ColumnOp colOp; + + // colOp = m_wrapper.getColumnOp(); + + colOp.deleteFile(100); + + // test column struct list + CreateColumnFile(100, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, 1, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 1); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + // test column values + // add values for the first column + curString = "\376"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + curString = "a"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + curString = "c"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + curString = "G"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "a"); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "c"); + + testTuple = testTupleList[3]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "G"); + + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + printf("\nrc=%d", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + CPPUNIT_ASSERT(ridList[3] == 3); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 4); + CPPUNIT_ASSERT(ridList[1] == 5); + CPPUNIT_ASSERT(ridList[2] == 6); + CPPUNIT_ASSERT(ridList[3] == 7); + } + + void testInterfaceInsertRowDoubleChar() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + string curString; + int total, rc; + ColumnOp colOp; + + // colOp = m_wrapper.getColumnOp(); + + colOp.deleteFile(100); + + // test column struct list + CreateColumnFile(100, 2, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, 2, WriteEngine::CHAR); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 2); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + // test column values + // add values for the first column + curString = "\377\376"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + curString = "ab"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + curString = "c"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + curString = "Ge"; + curTuple.data = curString; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "ab"); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "c"); + + testTuple = testTupleList[3]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == "Ge"); + + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + printf("\nrc=%d", rc); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + CPPUNIT_ASSERT(ridList[3] == 3); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 4); + CPPUNIT_ASSERT(ridList[1] == 5); + CPPUNIT_ASSERT(ridList[2] == 6); + CPPUNIT_ASSERT(ridList[3] == 7); + } + + void testInterfaceInsertRowLongLong() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + int total, rc; + + CreateColumnFile(100, 8, WriteEngine::BIGINT, WriteEngine::WR_LONGLONG); + CreateColumnFile(200, 2, WriteEngine::SMALLINT, WriteEngine::WR_SHORT); + + // test column struct list + SetColumnStruct(curColStruct, 100, 8, WriteEngine::BIGINT); + colStructList.push_back(curColStruct); + + SetColumnStruct(curColStruct, 200, 2, WriteEngine::SMALLINT); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 8); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::BIGINT); + + testColStruct = colStructList[1]; + CPPUNIT_ASSERT(testColStruct.dataOid == 200); + CPPUNIT_ASSERT(testColStruct.colWidth == 2); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::SMALLINT); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // test column values + // add values for the first column + curTuple.data = (long long)1021231; + curTupleList.push_back(curTuple); + + curTuple.data = (long long)0; + curTupleList.push_back(curTuple); + + curTuple.data = (long long)93232200; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add values for the second column + curTuple.data = (short)1000; + curTupleList.push_back(curTuple); + + curTuple.data = (short)5678; + curTupleList.push_back(curTuple); + + curTuple.data = (short)9; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)9); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 1021231); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 0); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 93232200); + + // second column + testTupleList = static_cast(colValueList[1]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 1000); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 5678); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 9); + + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 3); + CPPUNIT_ASSERT(ridList[1] == 4); + CPPUNIT_ASSERT(ridList[2] == 5); + } + + void testInterfaceInsertRowByte() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + int total, rc; + ColumnOp colOp; + + CreateColumnFile(100, 1, WriteEngine::TINYINT, WriteEngine::WR_BYTE); + CreateColumnFile(200, 4, WriteEngine::INT, WriteEngine::WR_INT); + + // test column struct list + SetColumnStruct(curColStruct, 100, 1, WriteEngine::TINYINT); + colStructList.push_back(curColStruct); + + SetColumnStruct(curColStruct, 200, 4, WriteEngine::INT); + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 1); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::TINYINT); + + testColStruct = colStructList[1]; + CPPUNIT_ASSERT(testColStruct.dataOid == 200); + CPPUNIT_ASSERT(testColStruct.colWidth == 4); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::INT); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // test column values + // add values for the first column + curTuple.data = (char)21; + curTupleList.push_back(curTuple); + + curTuple.data = (char)-30; + curTupleList.push_back(curTuple); + + curTuple.data = (char)127; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add values for the second column + curTuple.data = 1000; + curTupleList.push_back(curTuple); + + curTuple.data = 5678; + curTupleList.push_back(curTuple); + + curTuple.data = 9; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)9); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(colValueList[0]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 21); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == -30); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 127); + + // second column + testTupleList = static_cast(colValueList[1]); + total = testTupleList.size(); + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 1000); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 5678); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 9); + + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 3); + CPPUNIT_ASSERT(ridList[1] == 4); + CPPUNIT_ASSERT(ridList[2] == 5); + } + + void testInterfaceInsertRowToken() + { + ColStruct curColStruct, testColStruct; + ColTuple curTuple, testTuple; + ColTupleList curTupleList, testTupleList; + ColStructList colStructList; + ColValueList colValueList, colOldValList; + RIDList ridList; + std::string curStr, testStr; + int rc; + ColumnOp colOp; + OID dctnryOID = 800, treeOID = 801, listOID = 802; + DctnryTuple dctnryTuple; + + // no matter what happened, drop stores + // BRMWrapper::setUseBrm(false); + m_wrapper.dropStore(m_session.txnid, dctnryOID, treeOID, listOID); + rc = m_wrapper.createStore(m_session.txnid, dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = m_wrapper.openStore(m_session.txnid, dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // colOp = m_wrapper.getColumnOp(); + + // test column struct list + CreateColumnFile(100, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR); + + SetColumnStruct(curColStruct, 100, 8, WriteEngine::CHAR); + curColStruct.tokenFlag = true; + colStructList.push_back(curColStruct); + + testColStruct = colStructList[0]; + CPPUNIT_ASSERT(testColStruct.dataOid == 100); + CPPUNIT_ASSERT(testColStruct.colWidth == 8); + CPPUNIT_ASSERT(testColStruct.colDataType == WriteEngine::CHAR); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // test column values + // add values for the first column + strcpy((char*)dctnryTuple.sigValue, "ABCD 12345"); + dctnryTuple.sigSize = 10; + rc = m_wrapper.tokenize(m_session.txnid, dctnryTuple); + + if (rc != NO_ERROR) + printf("1197 Tokenize failed, rc %i\n", rc); + + CPPUNIT_ASSERT(rc == NO_ERROR); + curTuple.data = dctnryTuple.token; + curTupleList.push_back(curTuple); + + Token test1; + test1 = boost::any_cast(curTuple.data); + // printf( "\ncurTuple.datatype = %d\n", curTuple.data.type() ); + + strcpy((char*)dctnryTuple.sigValue, "CBED 1334678"); + dctnryTuple.sigSize = 12; + rc = m_wrapper.tokenize(m_session.txnid, dctnryTuple); + + if (rc != NO_ERROR) + printf("1208 Tokenize failed, rc %i\n", rc); + + CPPUNIT_ASSERT(rc == NO_ERROR); + curTuple.data = dctnryTuple.token; + curTupleList.push_back(curTuple); + + strcpy((char*)dctnryTuple.sigValue, "GHED 2334"); + dctnryTuple.sigSize = 9; + rc = m_wrapper.tokenize(m_session.txnid, dctnryTuple); + CPPUNIT_ASSERT(rc == NO_ERROR); + curTuple.data = dctnryTuple.token; + curTupleList.push_back(curTuple); + + colValueList.push_back(curTupleList); + + curTupleList.clear(); + + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)9); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkValid(m_session.txnid, colStructList, colValueList, ridList) == NO_ERROR); + + // m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); + CPPUNIT_ASSERT(m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList) == + NO_ERROR); + CPPUNIT_ASSERT(ridList[0] == 0); + CPPUNIT_ASSERT(ridList[1] == 1); + CPPUNIT_ASSERT(ridList[2] == 2); + + // try to insert more rows and row id should increase + rc = m_wrapper.insertColumnRec(m_session.txnid, colStructList, colValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + CPPUNIT_ASSERT(ridList[0] == 3); + CPPUNIT_ASSERT(ridList[1] == 4); + CPPUNIT_ASSERT(ridList[2] == 5); + + m_wrapper.dropStore(m_session.txnid, dctnryOID, treeOID, listOID); + } + + void testInterfaceCreateIndex() + { + int rc; + ColumnOp colOp; + + if (colOp.exists(900)) + { + rc = m_wrapper.dropIndex(m_session.txnid, 900, 901); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + rc = m_wrapper.createIndex(m_session.txnid, 900, 901); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testInterfaceDropIndex() + { + int rc; + + rc = m_wrapper.dropIndex(m_session.txnid, 900, 901); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void SetIndexStruct(IdxStruct& curStruct, OID treeOid, OID listOid, int width, + CalpontSystemCatalog::ColDataType dataType) + { + curStruct.treeOid = treeOid; + curStruct.listOid = listOid; + curStruct.idxWidth = width; + curStruct.idxDataType = dataType; + curStruct.tokenFlag = false; + } + + void testInterfaceUpdateIndexChar() + { + IdxStruct curStruct, testStruct; + IdxTuple curTuple, testTuple; + IdxTupleList curTupleList, testTupleList; + IdxStructList idxStructList; + IdxValueList idxValueList; + RIDList ridList; + std::string curStr, testStr; + int total, rc; + + testInterfaceCreateIndex(); + + CPPUNIT_ASSERT(m_wrapper.updateIndexRec(m_session.txnid, idxStructList, idxValueList, ridList) != + NO_ERROR); + + // test column struct list + SetIndexStruct(curStruct, 900, 901, 4, WriteEngine::MEDINT); + idxStructList.push_back(curStruct); + + testStruct = idxStructList[0]; + CPPUNIT_ASSERT(testStruct.treeOid == 900); + CPPUNIT_ASSERT(testStruct.listOid == 901); + CPPUNIT_ASSERT(testStruct.idxWidth == 4); + CPPUNIT_ASSERT(testStruct.idxDataType == WriteEngine::MEDINT); + + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // add values for the first index + curTuple.data = 102; + curTupleList.push_back(curTuple); + + curTuple.data = 102; + curTupleList.push_back(curTuple); + + curTuple.data = 102; + curTupleList.push_back(curTuple); + + curTuple.data = 0; + curTupleList.push_back(curTuple); + + curTuple.data = 200; + curTupleList.push_back(curTuple); + + idxValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)15); + ridList.push_back((RID)16); + ridList.push_back((RID)17); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(idxValueList[0]); + total = testTupleList.size(); + + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 102); + + testTuple = testTupleList[3]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 0); + + testTuple = testTupleList[4]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 200); + + rc = m_wrapper.updateIndexRec(m_session.txnid, idxStructList, idxValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + + // CPPUNIT_ASSERT( m_wrapper.deleteIndexRec( idxStructList, idxValueList, ridList ) == NO_ERROR ); + } + + void testInterfaceUpdateMultiColIndex() + { + IdxStruct curStruct, testStruct; + IdxTuple curTuple, testTuple; + IdxTupleList curTupleList, testTupleList; + IdxStructList idxStructList; + IdxValueList idxValueList; + RIDList ridList; + std::string curStr, testStr; + int total, rc; + + testInterfaceCreateIndex(); + + CPPUNIT_ASSERT(m_wrapper.updateIndexRec(m_session.txnid, idxStructList, idxValueList, ridList) != + NO_ERROR); + + // test column struct list + SetIndexStruct(curStruct, 900, 901, 4, WriteEngine::MEDINT); + idxStructList.push_back(curStruct); + + SetIndexStruct(curStruct, 900, 901, 2, WriteEngine::SMALLINT); + idxStructList.push_back(curStruct); + + testStruct = idxStructList[0]; + CPPUNIT_ASSERT(testStruct.treeOid == 900); + CPPUNIT_ASSERT(testStruct.listOid == 901); + CPPUNIT_ASSERT(testStruct.idxWidth == 4); + CPPUNIT_ASSERT(testStruct.idxDataType == WriteEngine::MEDINT); + + testStruct = idxStructList[1]; + CPPUNIT_ASSERT(testStruct.treeOid == 900); + CPPUNIT_ASSERT(testStruct.listOid == 901); + CPPUNIT_ASSERT(testStruct.idxWidth == 2); + CPPUNIT_ASSERT(testStruct.idxDataType == WriteEngine::SMALLINT); + + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // add values for the first index + curTuple.data = 102; + curTupleList.push_back(curTuple); + + curTuple.data = 0; + curTupleList.push_back(curTuple); + + curTuple.data = 200; + curTupleList.push_back(curTuple); + + idxValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add values for the second index + curTuple.data = (short)1; + curTupleList.push_back(curTuple); + + curTuple.data = (short)2; + curTupleList.push_back(curTuple); + + curTuple.data = (short)3; + curTupleList.push_back(curTuple); + + idxValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + ERR_ROWID_VALUE_NOT_MATCH); + ridList.push_back((RID)9); + ridList.push_back((RID)10); + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(idxValueList[0]); + total = testTupleList.size(); + + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 102); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 0); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 200); + + // second column + testTupleList = static_cast(idxValueList[1]); + total = testTupleList.size(); + + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 1); + + testTuple = testTupleList[1]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 2); + + testTuple = testTupleList[2]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 3); + + rc = m_wrapper.processMultiColIndexRec(m_session.txnid, idxStructList, idxValueList, ridList); + CPPUNIT_ASSERT(rc == NO_ERROR); + } + + void testInterfaceDeleteIndexChar() + { + IdxStruct curStruct, testStruct; + IdxTuple curTuple, testTuple; + IdxTupleList curTupleList, testTupleList; + std::vector idxStructList; + std::vector idxValueList; + std::vector ridList; + std::string curStr, testStr; + int total; + + // test column struct list + SetIndexStruct(curStruct, 900, 901, 4, WriteEngine::MEDINT); + idxStructList.push_back(curStruct); + + testStruct = idxStructList[0]; + CPPUNIT_ASSERT(testStruct.treeOid == 900); + CPPUNIT_ASSERT(testStruct.listOid == 901); + CPPUNIT_ASSERT(testStruct.idxWidth == 4); + CPPUNIT_ASSERT(testStruct.idxDataType == WriteEngine::MEDINT); + + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + ERR_STRUCT_VALUE_NOT_MATCH); + + // add values for the first index + curTuple.data = 102; + curTupleList.push_back(curTuple); + + idxValueList.push_back(curTupleList); + + curTupleList.clear(); + + // add one rowId + ridList.push_back((RID)5); + CPPUNIT_ASSERT(m_wrapper.checkIndexValid(m_session.txnid, idxStructList, idxValueList, ridList) == + NO_ERROR); + + // retrieve the values back + // first column + testTupleList = static_cast(idxValueList[0]); + total = testTupleList.size(); + + testTuple = testTupleList[0]; + CPPUNIT_ASSERT(boost::any_cast(testTuple.data) == 102); + + CPPUNIT_ASSERT(m_wrapper.deleteIndexRec(m_session.txnid, idxStructList, idxValueList, ridList) == + NO_ERROR); + } + + void testInterfaceDctnryToken() + { /* - CPPUNIT_TEST( testCreateColumnFiles ); - CPPUNIT_TEST( testWriteColumnFiles ); - CPPUNIT_TEST( testReadColumnFiles ); - - CPPUNIT_TEST( testInsertRowInt ); - CPPUNIT_TEST( testInsertRowLonglong ); - CPPUNIT_TEST( testInsertRowDouble ); - CPPUNIT_TEST( testInsertRowChar ); - CPPUNIT_TEST( testInsertBulk ); - - // Table related operation testing - CPPUNIT_TEST( testTableAllocateRowId ); - - // Wrapper interface testing - CPPUNIT_TEST( testInterfaceCreateColumnFile ); - CPPUNIT_TEST( testInterfaceInsertUpdateDeleteRow ); - CPPUNIT_TEST( testInterfaceInsertRowMedIntDouble ); - - CPPUNIT_TEST( testInterfaceInsertRowChar ); - - CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth4 ); - CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth3 ); - CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth8 ); - CPPUNIT_TEST( testInterfaceInsertRowCharQAWidth1 ); - - - CPPUNIT_TEST( testInterfaceInsertRowLongLong ); - CPPUNIT_TEST( testInterfaceInsertRowByte ); - CPPUNIT_TEST( testInterfaceInsertRowToken ); - CPPUNIT_TEST( testInterfaceInsertRowSingleChar ); - CPPUNIT_TEST( testInterfaceInsertRowDoubleChar ); - - CPPUNIT_TEST( testInterfaceCreateIndex ); - CPPUNIT_TEST( testInterfaceDropIndex ); - CPPUNIT_TEST( testInterfaceUpdateIndexChar ); - CPPUNIT_TEST( testInterfaceUpdateMultiColIndex); - - CPPUNIT_TEST( testInterfaceDctnryToken ); - CPPUNIT_TEST( testInterfaceDctnryTokenRollBack ); - - CPPUNIT_TEST( testInsertCommitRollback ); - CPPUNIT_TEST( testInterfaceInsertRowHwm ); - - CPPUNIT_TEST( testInterfaceCombineIndexLoad ); // note: this test case must be the last one - //CPPUNIT_TEST( testTmpBulkPerformance ); // note: this test case must be the last one - */ -//CPPUNIT_TEST(testBoostFloat); - /* - CPPUNIT_TEST(testCreateColumnFiles1); - CPPUNIT_TEST(testCreateColumnFiles2); - CPPUNIT_TEST(testCreateColumnFiles3); - CPPUNIT_TEST(testCreateColumnFiles4); - */ - -//CPPUNIT_TEST(testCreateFileMultipleIONBF); -//CPPUNIT_TEST(testCreateFile64MBIONBF); -//CPPUNIT_TEST(testCreateFileMultipleBF); -//CPPUNIT_TEST(testCreateFilelargerBF); - CPPUNIT_TEST(testSoloman); - /* - CPPUNIT_TEST(testCreateFileNoBRM); - CPPUNIT_TEST(testCreateFileBRM); - */ - CPPUNIT_TEST_SUITE_END(); - -private: - WriteEngineWrapper m_wrapper; - Session m_session; - -public: - void setUp() - { - m_wrapper.setDebugLevel( DEBUG_3 ); - m_session.txnid = 10; - BRMWrapper::setUseBrm( true ); - } - - void tearDown() - { - - } - - void SetColumnStruct( ColStruct& column, OID dataOid, int colWidth, CalpontSystemCatalog::ColDataType colDataType ) - { - column.dataOid = dataOid; - column.colWidth = colWidth; - column.colDataType = colDataType; - column.tokenFlag = false; - } - - void CreateColumnFile( FID fid, int width, CalpontSystemCatalog::ColDataType colDataType, ColType colType ) - { - int rc; - ColumnOp colOp; - Column curCol; - - colOp.deleteFile( fid ); - CPPUNIT_ASSERT( colOp.exists( fid ) == false ); - - // create column files - rc = colOp.createColumn( curCol, 2, width, colDataType, colType, fid ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - void testCreateColumnFiles() - { - ColumnOp colOp; - Column curCol; - int rc; - - - colOp.initColumn( curCol ); - - // file opertaions - colOp.deleteFile( 100 ); - CPPUNIT_ASSERT( colOp.exists( 100 ) == false ); - - colOp.deleteFile( 101 ); - CPPUNIT_ASSERT( colOp.exists( 101 ) == false ); - - colOp.deleteFile( 103 ); - CPPUNIT_ASSERT( colOp.exists( 103 ) == false ); - - rc = colOp.createColumn( curCol, 2, 50, WriteEngine::CHAR, WriteEngine::WR_CHAR, 101 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - colOp.deleteFile( 101 ); - rc = colOp.createColumn( curCol, 2, 3, WriteEngine::CHAR, WriteEngine::WR_CHAR, 101 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - colOp.deleteFile( 101 ); - rc = colOp.createColumn( curCol, 2, 5, WriteEngine::CHAR, WriteEngine::WR_CHAR, 101 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - // create column files - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR, 100 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = colOp.createColumn( curCol, 2, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR, 100 ); - CPPUNIT_ASSERT( rc == ERR_FILE_EXIST ); - - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - colOp.clearColumn( curCol ); - - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == ERR_INVALID_PARAM ); - - rc = colOp.createTable(); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - void testWriteColumnFiles() - { - ColumnOp colOp; - Column curCol; - int rc; - unsigned char buf[BYTE_PER_BLOCK]; - - - colOp.initColumn( curCol ); - CPPUNIT_ASSERT( curCol.colNo == 0 ); - CPPUNIT_ASSERT( curCol.colWidth == 0 ); - CPPUNIT_ASSERT( curCol.dataFile.pFile == NULL ); -// CPPUNIT_ASSERT( curCol.bitmapFile.pFile == NULL ); - CPPUNIT_ASSERT( curCol.dataFile.fid == 0 ); -// CPPUNIT_ASSERT( curCol.bitmapFile.fid == 0 ); - - colOp.setColParam( curCol, 2, 4, INT, WriteEngine::WR_INT, 100 ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - int testVal = 60; - memset( buf, 0, BYTE_PER_BLOCK ); - memcpy( buf + 8, &testVal, 4 ); - - rc = colOp.writeDBFileFbo( curCol.dataFile.pFile, buf, 1, 1 ); // block 2 - CPPUNIT_ASSERT( rc == NO_ERROR ); - - colOp.clearColumn( curCol ); - - CPPUNIT_ASSERT( curCol.colNo == 0 ); - CPPUNIT_ASSERT( curCol.colWidth == 0 ); - CPPUNIT_ASSERT( curCol.dataFile.pFile == NULL ); -// CPPUNIT_ASSERT( curCol.bitmapFile.pFile == NULL ); - CPPUNIT_ASSERT( curCol.dataFile.fid == 0 ); -// CPPUNIT_ASSERT( curCol.bitmapFile.fid == 0 ); - - CPPUNIT_ASSERT( colOp.isValid( curCol ) == false ); - - } - - void testReadColumnFiles() - { - ColumnOp colOp; - Column curCol, errCol; - int rc; - unsigned char buf[BYTE_PER_BLOCK]; - CalpontSystemCatalog::ColDataType colDataType; - -// colOp.setUseBrm(false); - CPPUNIT_ASSERT( colOp.getColDataType( "integer", colDataType ) == true ); - CPPUNIT_ASSERT( colDataType == INT ); - CPPUNIT_ASSERT( colOp.getColDataType( "Int1", colDataType ) == false ); - - // check error situation - colOp.initColumn( errCol ); - colOp.setColParam( errCol, 2, 8, CHAR, WriteEngine::WR_CHAR, 103 ); - rc = colOp.openColumnFile( errCol ); - CPPUNIT_ASSERT( rc == ERR_FILE_READ ); - colOp.clearColumn( errCol ); - - - colOp.initColumn( errCol ); - colOp.setColParam( errCol, 2, 8, CHAR, WriteEngine::WR_CHAR, 103 ); - rc = colOp.openColumnFile( errCol ); - CPPUNIT_ASSERT( rc == ERR_FILE_READ ); - colOp.clearColumn( errCol ); - - // check normal situation - colOp.initColumn( curCol ); - - colOp.setColParam( curCol, 2, 8, CHAR, WriteEngine::WR_CHAR, 100 ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - memset( buf, 0, BYTE_PER_BLOCK ); - - rc = colOp.readDBFile( curCol.dataFile.pFile, buf, 1, true ); // block 2 - CPPUNIT_ASSERT( rc == NO_ERROR ); - - int testVal = 0; - memcpy( &testVal, buf + 8, 8 ); - printf( "\nread the test value : %d", testVal ); - CPPUNIT_ASSERT( testVal == 60 ); - - colOp.clearColumn( curCol ); - } - - - void testInsertRowInt() - { - ColumnOp colOp; - Column curCol; - int rc, valArray[5], oldValArray[5]; - RID rowIdArray[5]; - - - rowIdArray[0] = 1; - rowIdArray[1] = 3; - rowIdArray[2] = 4; - rowIdArray[3] = 7; - rowIdArray[4] = 8; - - valArray[0] = 8; - valArray[1] = 5; - valArray[2] = 3; - valArray[3] = 0; - valArray[4] = 16; - - CreateColumnFile( 100, 4, WriteEngine::INT, WriteEngine::WR_INT ); - - colOp.initColumn( curCol ); - colOp.setColParam( curCol, 2, 4, INT, WriteEngine::WR_INT, 100 ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = colOp.writeRow( curCol, 5, (RID*) rowIdArray, valArray, oldValArray ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( int i = 0; i < 5; i++ ) - CPPUNIT_ASSERT( oldValArray[i] == (int)0x80000001 ); - - colOp.clearColumn( curCol ); - } - - void testInsertRowLonglong() - { - ColumnOp colOp; - Column curCol; - int rc; - RID rowIdArray[3]; - int width = 8, totalRow = 3; - FID fid = 100; - long long valArray[3]; - uint64_t oldValArray[3], verifyArray[3]; - CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::BIGINT; - ColType colType = WriteEngine::WR_LONGLONG; - uint64_t emptyVal = 0x8000000000000001LL; - - - rowIdArray[0] = 1; - rowIdArray[1] = 3; - rowIdArray[2] = 4; - - verifyArray[0] = valArray[0] = 32111238; - verifyArray[1] = valArray[1] = 1231235; - verifyArray[2] = valArray[2] = 67731233; - - CreateColumnFile( fid, width, colDataType, colType ); - - colOp.initColumn( curCol ); - colOp.setColParam( curCol, 2, width, colDataType, colType, fid ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = colOp.writeRow( curCol, totalRow, (RID*) rowIdArray, valArray, oldValArray ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( int i = 0; i < totalRow; i++ ) - CPPUNIT_ASSERT( oldValArray[i] == emptyVal ); - - valArray[0] = 1900003; - valArray[1] = 2349000; - valArray[2] = 78900123; - - rc = colOp.writeRow( curCol, totalRow, (RID*) rowIdArray, valArray, oldValArray ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( int i = 0; i < totalRow; i++ ) - CPPUNIT_ASSERT( oldValArray[i] == verifyArray[i] ); - - colOp.clearColumn( curCol ); - } - - - void testInsertRowDouble() - { - ColumnOp colOp; - Column curCol; - int rc; - RID rowIdArray[3]; - int width = 8, totalRow = 3; - FID fid = 100; - double valArray[3]; - int64_t oldValArray[3]; - CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::DOUBLE; - ColType colType = WriteEngine::WR_DOUBLE; - int64_t emptyVal = 0xFFFAAAAAAAAAAAABLL; - - - rowIdArray[0] = 1; - rowIdArray[1] = 3; - rowIdArray[2] = 4; - - valArray[0] = 8; - valArray[1] = 5; - valArray[2] = 3; - - CreateColumnFile( fid, width, colDataType, colType ); - - colOp.initColumn( curCol ); - colOp.setColParam( curCol, 2, width, colDataType, colType, fid ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = colOp.writeRow( curCol, totalRow, (RID*) rowIdArray, valArray, oldValArray ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( int i = 0; i < totalRow; i++ ) - CPPUNIT_ASSERT( oldValArray[i] == emptyVal ); - - colOp.clearColumn( curCol ); - } - - void testInsertRowChar() - { - ColumnOp colOp; - Column curCol; - int rc; - RID rowIdArray[3]; - int width = 8, totalRow = 3; - FID fid = 100; - char valArray[24], buf[8]; - char testValue[3][8] = { "abc", "eDFFF", "GHK" }; - uint64_t oldValArray[3]; - CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::CHAR; - ColType colType = WriteEngine::WR_CHAR; - uint64_t emptyVal = 0xFFFFFFFFFFFFFFFFLL; - - - rowIdArray[0] = 1; - rowIdArray[1] = 3; - rowIdArray[2] = 4; - - memset( valArray, 0, 24 ); - colOp.writeBufValue( (unsigned char*)buf, testValue[0], 8, true ); - memcpy( valArray, buf, 8 ); - colOp.writeBufValue( (unsigned char*)buf, testValue[1], 8, true ); - memcpy( valArray + 8, buf, 8 ); - colOp.writeBufValue( (unsigned char*)buf, testValue[2], 8, true ); - memcpy( valArray + 16, buf, 8 ); - - CreateColumnFile( fid, width, colDataType, colType ); - - colOp.initColumn( curCol ); - colOp.setColParam( curCol, 2, width, colDataType, colType, fid ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - rc = colOp.writeRow( curCol, totalRow, (RID*) rowIdArray, valArray, oldValArray ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - for ( int i = 0; i < totalRow; i++ ) - CPPUNIT_ASSERT( oldValArray[i] == emptyVal ); - - colOp.clearColumn( curCol ); - } - - void testInsertBulk() - { - FILE* pSourceFile = NULL; - ColumnOp colOp; - Column curCol; - int rc, width = 8; - int hwm; - FID fid = 999; - CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::CHAR; - ColType colType = WriteEngine::WR_CHAR; - - colOp.deleteFile( 999 ); - colOp.deleteFile( 998 ); - CPPUNIT_ASSERT( colOp.exists( 999 ) == false ); - CPPUNIT_ASSERT( colOp.exists( 998 ) == false ); - - int allocSize = 0; - CPPUNIT_ASSERT( colOp.createFile( 998, 20, allocSize, 0xEEEE, 2 ) == NO_ERROR ); - CreateColumnFile( fid, width, colDataType, colType ); - - colOp.initColumn( curCol ); - colOp.setColParam( curCol, 2, width, colDataType, colType, fid ); - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - pSourceFile = colOp.openFile( 998 ); - CPPUNIT_ASSERT( pSourceFile != NULL ); - - hwm = colOp.getFileSize( curCol.dataFile.pFile ) / BYTE_PER_BLOCK - 5 ; - rc = colOp.insertBulk( curCol, pSourceFile, hwm, 20 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - colOp.clearColumn( curCol ); - colOp.closeFile( pSourceFile ); - } - - void testTableAllocateRowId() - { - ColumnOp colOp; - Column curCol; - RID rowIdArray[3]; - int rc; - int width = 4, hwm = 0; - FID fid = 100; - CalpontSystemCatalog::ColDataType colDataType = CalpontSystemCatalog::INT; - ColType colType = WriteEngine::WR_INT; - - - CreateColumnFile( fid, width, colDataType, colType ); - - colOp.initColumn( curCol ); - colOp.setColParam( curCol, 2, width, colDataType, colType, fid ); - - rc = colOp.openColumnFile( curCol ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - colOp.allocRowId( curCol, 3, rowIdArray, hwm ); - CPPUNIT_ASSERT( rowIdArray[0] == 0 ); - CPPUNIT_ASSERT( rowIdArray[1] == 1 ); - CPPUNIT_ASSERT( rowIdArray[2] == 2 ); - } - - - void testInterfaceCreateColumnFile() - { - ColumnOp colOp; - FID fid = 100; - - - if ( colOp.exists( fid ) ) - { - CPPUNIT_ASSERT( m_wrapper.dropColumn( m_session.txnid, fid ) == NO_ERROR ); - CPPUNIT_ASSERT( m_wrapper.dropColumn( m_session.txnid, fid ) == ERR_FILE_NOT_EXIST ); - } - - CPPUNIT_ASSERT( m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::DOUBLE, 8 ) == NO_ERROR ); - CPPUNIT_ASSERT( colOp.exists( 100 ) == true ); - } - - void testInterfaceInsertUpdateDeleteRow() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - int rc, total, valArray[3]; - float dVal; -// ColumnOp colOp; - -// colOp = m_wrapper.getColumnOp(); - - - CreateColumnFile( 100, 4, WriteEngine::INT, WriteEngine::WR_INT ); - CreateColumnFile( 200, 4, WriteEngine::FLOAT, WriteEngine::WR_FLOAT ); - - // test column struct list - SetColumnStruct( curColStruct, 100, 4, WriteEngine::INT ); - colStructList.push_back( curColStruct ); - - SetColumnStruct( curColStruct, 200, 4, WriteEngine::FLOAT ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 4 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::INT ); - - testColStruct = colStructList[1]; - CPPUNIT_ASSERT( testColStruct.dataOid == 200 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 4 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::FLOAT ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // test column values - // add values for the first column - curTuple.data = 3; - curTupleList.push_back( curTuple ); - - curTuple.data = 0; - curTupleList.push_back( curTuple ); - - curTuple.data = 100; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - m_wrapper.convertValArray( 3, WriteEngine::WR_INT, curTupleList, valArray ); - CPPUNIT_ASSERT( valArray[0] == 3 ); - CPPUNIT_ASSERT( valArray[1] == 0 ); - CPPUNIT_ASSERT( valArray[2] == 100 ); - - curTupleList.clear(); - - // add values for the second column - curTuple.data = 0.0f; - curTupleList.push_back( curTuple ); - - curTuple.data = 1234.78f; - curTupleList.push_back( curTuple ); - - curTuple.data = 999.98f; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - float valArray1[3]; - m_wrapper.convertValArray( 3, WriteEngine::WR_FLOAT, curTupleList, valArray1 ); - CPPUNIT_ASSERT( valArray1[0] == 0.0f ); - CPPUNIT_ASSERT( valArray1[1] == 1234.78f); - CPPUNIT_ASSERT( valArray1[2] == 999.98f ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 3 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 0 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 100 ); - - // second column - testTupleList = static_cast(colValueList[1]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - dVal = boost::any_cast( testTuple.data ); - CPPUNIT_ASSERT( dVal == 0.0f ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 1234.78f ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 999.98f ); - -// m_wrapper.printInputValue( colStructList, colValueList, ridList ); - - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 3 ); - CPPUNIT_ASSERT( ridList[1] == 4 ); - CPPUNIT_ASSERT( ridList[2] == 5 ); - - // try to update rows - colValueList.clear(); - curTupleList.clear(); - - curTuple.data = 9; - curTupleList.push_back( curTuple ); - - curTuple.data = 15; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - - // add values for the second column - curTupleList.clear(); - - curTuple.data = 1.99f; - curTupleList.push_back( curTuple ); - - curTuple.data = 3000.15f; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - - ridList.clear(); - ridList.push_back( (RID) 2 ); - ridList.push_back( (RID) 3 ); - - rc = m_wrapper.updateColumnRec( m_session.txnid, colStructList, colValueList, colOldValList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( colOldValList.size() == 2 ); - - m_wrapper.printInputValue( colStructList, colOldValList, ridList ); - curTupleList = colOldValList[0]; - curTuple = curTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 100 ); - curTuple = curTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 3 ); - - curTupleList = colOldValList[1]; - curTuple = curTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 999.98f ); - curTuple = curTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 0.0f ); - - ridList[0] = 1; - ridList[1] = 2; - - rc = m_wrapper.deleteRow( m_session.txnid, colStructList, colOldValList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( colOldValList.size() == 2 ); - - m_wrapper.printInputValue( colStructList, colOldValList, ridList ); - curTupleList = colOldValList[0]; - curTuple = curTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 0 ); - curTuple = curTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 9 ); - - curTupleList = colOldValList[1]; - curTuple = curTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 1234.78f ); - curTuple = curTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( curTuple.data ) == 1.99f ); - - } - - void testInterfaceInsertRowMedIntDouble() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - int total, rc; - double dVal; - - - - CreateColumnFile( 100, 4, WriteEngine::MEDINT, WriteEngine::WR_INT ); - CreateColumnFile( 200, 8, WriteEngine::DOUBLE, WriteEngine::WR_DOUBLE ); - - // test column struct list - SetColumnStruct( curColStruct, 100, 4, WriteEngine::MEDINT ); - colStructList.push_back( curColStruct ); - - SetColumnStruct( curColStruct, 200, 8, WriteEngine::DOUBLE ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 4 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::MEDINT ); - - testColStruct = colStructList[1]; - CPPUNIT_ASSERT( testColStruct.dataOid == 200 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 8 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::DOUBLE ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // test column values - // add values for the first column - curTuple.data = 102; - curTupleList.push_back( curTuple ); - - curTuple.data = 0; - curTupleList.push_back( curTuple ); - - curTuple.data = 200; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add values for the second column - curTuple.data = 0.0; - curTupleList.push_back( curTuple ); - - curTuple.data = 1234.78; - curTupleList.push_back( curTuple ); - - curTuple.data = 999.98; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 9 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 102 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 0 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 200 ); - - // second column - testTupleList = static_cast(colValueList[1]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - dVal = boost::any_cast( testTuple.data ); - CPPUNIT_ASSERT( dVal == 0.0 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 1234.78 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 999.98 ); - -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 3 ); - CPPUNIT_ASSERT( ridList[1] == 4 ); - CPPUNIT_ASSERT( ridList[2] == 5 ); - } - - - void testInterfaceInsertRowChar() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - std::string curStr, testStr; - int total, rc, width = 5; - - - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - - rc = m_wrapper.updateColumnRec( m_session.txnid, colStructList, colValueList, colOldValList, ridList ); - CPPUNIT_ASSERT( rc != NO_ERROR ); - - // test column struct list - CreateColumnFile( 100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, width, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == width ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // test column values - // add values for the first column - curStr = "aaaaa"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "bbbbb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "ccccc"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 9 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "aaaaa" ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "bbbbb" ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "ccccc" ); - -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 3 ); - CPPUNIT_ASSERT( ridList[1] == 4 ); - CPPUNIT_ASSERT( ridList[2] == 5 ); - } - - - void testInterfaceInsertRowCharQAWidth4() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - std::string curStr, testStr; - int rc, width = 4; - - // test column struct list - CreateColumnFile( 100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, width, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == width ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - // test column values - // add values for the first column - curStr = "aaaa"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "aaa"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "aa"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "a"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - CPPUNIT_ASSERT( ridList[3] == 3 ); - - // separate inserts - colValueList.clear(); - curStr = "bbbb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 4 ); - - colValueList.clear(); - curStr = "bbb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 5 ); - - colValueList.clear(); - curStr = "bb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 6 ); - - colValueList.clear(); - curStr = "b"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 7 ); - } - - void testInterfaceInsertRowCharQAWidth3() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - std::string curStr, testStr; - int rc, width = 3; - - // test column struct list - CreateColumnFile( 100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, width, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == width ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - // test column values - // add values for the first column - curStr = "aaa"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "aa"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "a"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // separate inserts - colValueList.clear(); - curStr = "bbb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 3 ); - - colValueList.clear(); - curStr = "bb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 4 ); - - colValueList.clear(); - curStr = "b"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 5 ); - } - - void testInterfaceInsertRowCharQAWidth8() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - std::string curStr, testStr; - int rc, width = 8; - - // test column struct list - CreateColumnFile( 100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, width, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == width ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - // test column values - // add values for the first column - curStr = "12345678"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "123456"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "1234"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // separate inserts - colValueList.clear(); - curStr = "bbbb1234"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 3 ); - - colValueList.clear(); - curStr = "bb"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 4 ); - - colValueList.clear(); - curStr = "b"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 5 ); - } - - void testInterfaceInsertRowCharQAWidth1() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - std::string curStr, testStr; - int rc, width = 1; - - // test column struct list - CreateColumnFile( 100, width, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, width, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == width ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - // test column values - // add values for the first column - curStr = "a"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "b"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - curStr = "c"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // separate inserts - colValueList.clear(); - curStr = "1"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 3 ); - - colValueList.clear(); - curStr = "2"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 4 ); - - colValueList.clear(); - curStr = "3"; - curTuple.data = curStr; - curTupleList.push_back( curTuple ); - colValueList.push_back( curTupleList ); - curTupleList.clear(); - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 5 ); - } - - - void testInterfaceInsertRowSingleChar() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - string curString; - int total, rc; - ColumnOp colOp; - -// colOp = m_wrapper.getColumnOp(); - - colOp.deleteFile( 100 ); - - // test column struct list - CreateColumnFile( 100, 1, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, 1, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 1 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - // test column values - // add values for the first column - curString = "\376"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - curString = "a"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - curString = "c"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - curString = "G"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "a" ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "c" ); - - testTuple = testTupleList[3]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "G" ); - - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - printf( "\nrc=%d", rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - CPPUNIT_ASSERT( ridList[3] == 3 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 4 ); - CPPUNIT_ASSERT( ridList[1] == 5 ); - CPPUNIT_ASSERT( ridList[2] == 6 ); - CPPUNIT_ASSERT( ridList[3] == 7 ); - } - - void testInterfaceInsertRowDoubleChar() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - string curString; - int total, rc; - ColumnOp colOp; - -// colOp = m_wrapper.getColumnOp(); - - colOp.deleteFile( 100 ); - - // test column struct list - CreateColumnFile( 100, 2, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, 2, WriteEngine::CHAR ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 2 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - // test column values - // add values for the first column - curString = "\377\376"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - curString = "ab"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - curString = "c"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - curString = "Ge"; - curTuple.data = curString; - curTupleList.push_back( curTuple ); - - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "ab" ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "c" ); - - testTuple = testTupleList[3]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == "Ge" ); - - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - printf( "\nrc=%d", rc ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - CPPUNIT_ASSERT( ridList[3] == 3 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 4 ); - CPPUNIT_ASSERT( ridList[1] == 5 ); - CPPUNIT_ASSERT( ridList[2] == 6 ); - CPPUNIT_ASSERT( ridList[3] == 7 ); - } - - void testInterfaceInsertRowLongLong() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - int total, rc; - - - - CreateColumnFile( 100, 8, WriteEngine::BIGINT, WriteEngine::WR_LONGLONG ); - CreateColumnFile( 200, 2, WriteEngine::SMALLINT, WriteEngine::WR_SHORT ); - - // test column struct list - SetColumnStruct( curColStruct, 100, 8, WriteEngine::BIGINT ); - colStructList.push_back( curColStruct ); - - SetColumnStruct( curColStruct, 200, 2, WriteEngine::SMALLINT ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 8 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::BIGINT ); - - testColStruct = colStructList[1]; - CPPUNIT_ASSERT( testColStruct.dataOid == 200 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 2 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::SMALLINT ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // test column values - // add values for the first column - curTuple.data = (long long)1021231; - curTupleList.push_back( curTuple ); - - curTuple.data = (long long)0; - curTupleList.push_back( curTuple ); - - curTuple.data = (long long)93232200; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add values for the second column - curTuple.data = (short)1000; - curTupleList.push_back( curTuple ); - - curTuple.data = (short)5678; - curTupleList.push_back( curTuple ); - - curTuple.data = (short)9; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 9 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 1021231 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 0 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 93232200 ); - - // second column - testTupleList = static_cast(colValueList[1]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 1000 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 5678 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 9 ); - -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 3 ); - CPPUNIT_ASSERT( ridList[1] == 4 ); - CPPUNIT_ASSERT( ridList[2] == 5 ); - } - - - void testInterfaceInsertRowByte() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - int total, rc; - ColumnOp colOp; - - CreateColumnFile( 100, 1, WriteEngine::TINYINT, WriteEngine::WR_BYTE ); - CreateColumnFile( 200, 4, WriteEngine::INT, WriteEngine::WR_INT ); - - // test column struct list - SetColumnStruct( curColStruct, 100, 1, WriteEngine::TINYINT ); - colStructList.push_back( curColStruct ); - - SetColumnStruct( curColStruct, 200, 4, WriteEngine::INT ); - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 1 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::TINYINT ); - - testColStruct = colStructList[1]; - CPPUNIT_ASSERT( testColStruct.dataOid == 200 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 4 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::INT ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // test column values - // add values for the first column - curTuple.data = (char)21; - curTupleList.push_back( curTuple ); - - curTuple.data = (char) - 30; - curTupleList.push_back( curTuple ); - - curTuple.data = (char)127; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add values for the second column - curTuple.data = 1000; - curTupleList.push_back( curTuple ); - - curTuple.data = 5678; - curTupleList.push_back( curTuple ); - - curTuple.data = 9; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 9 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(colValueList[0]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 21 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == -30 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 127 ); - - // second column - testTupleList = static_cast(colValueList[1]); - total = testTupleList.size(); - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 1000 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 5678 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 9 ); - -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 3 ); - CPPUNIT_ASSERT( ridList[1] == 4 ); - CPPUNIT_ASSERT( ridList[2] == 5 ); - - } - - - void testInterfaceInsertRowToken() - { - ColStruct curColStruct, testColStruct; - ColTuple curTuple, testTuple; - ColTupleList curTupleList, testTupleList; - ColStructList colStructList; - ColValueList colValueList, colOldValList; - RIDList ridList; - std::string curStr, testStr; - int rc; - ColumnOp colOp; - OID dctnryOID = 800, treeOID = 801, listOID = 802; - DctnryTuple dctnryTuple; - - // no matter what happened, drop stores -// BRMWrapper::setUseBrm(false); - m_wrapper.dropStore( m_session.txnid, dctnryOID, treeOID, listOID); - rc = m_wrapper.createStore( m_session.txnid, dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_wrapper.openStore( m_session.txnid, dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - //colOp = m_wrapper.getColumnOp(); - - // test column struct list - CreateColumnFile( 100, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR ); - - SetColumnStruct( curColStruct, 100, 8, WriteEngine::CHAR ); - curColStruct.tokenFlag = true; - colStructList.push_back( curColStruct ); - - testColStruct = colStructList[0]; - CPPUNIT_ASSERT( testColStruct.dataOid == 100 ); - CPPUNIT_ASSERT( testColStruct.colWidth == 8 ); - CPPUNIT_ASSERT( testColStruct.colDataType == WriteEngine::CHAR ); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // test column values - // add values for the first column - strcpy( (char*)dctnryTuple.sigValue, "ABCD 12345" ); - dctnryTuple.sigSize = 10; - rc = m_wrapper.tokenize( m_session.txnid, dctnryTuple ); - - if (rc != NO_ERROR) - printf("1197 Tokenize failed, rc %i\n", rc); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - curTuple.data = dctnryTuple.token; - curTupleList.push_back( curTuple ); - - Token test1; - test1 = boost::any_cast( curTuple.data ); -// printf( "\ncurTuple.datatype = %d\n", curTuple.data.type() ); - - strcpy( (char*)dctnryTuple.sigValue, "CBED 1334678" ); - dctnryTuple.sigSize = 12; - rc = m_wrapper.tokenize( m_session.txnid, dctnryTuple ); - - if (rc != NO_ERROR) - printf("1208 Tokenize failed, rc %i\n", rc); - - CPPUNIT_ASSERT( rc == NO_ERROR ); - curTuple.data = dctnryTuple.token; - curTupleList.push_back( curTuple ); - - strcpy( (char*)dctnryTuple.sigValue, "GHED 2334" ); - dctnryTuple.sigSize = 9; - rc = m_wrapper.tokenize( m_session.txnid, dctnryTuple ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - curTuple.data = dctnryTuple.token; - curTupleList.push_back( curTuple ); - - colValueList.push_back( curTupleList ); - - curTupleList.clear(); - - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 9 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkValid( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - -// m_wrapper.printInputValue( (OID)103, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ) == NO_ERROR ); - CPPUNIT_ASSERT( ridList[0] == 0 ); - CPPUNIT_ASSERT( ridList[1] == 1 ); - CPPUNIT_ASSERT( ridList[2] == 2 ); - - // try to insert more rows and row id should increase - rc = m_wrapper.insertColumnRec( m_session.txnid, colStructList, colValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - CPPUNIT_ASSERT( ridList[0] == 3 ); - CPPUNIT_ASSERT( ridList[1] == 4 ); - CPPUNIT_ASSERT( ridList[2] == 5 ); - - m_wrapper.dropStore( m_session.txnid, dctnryOID, treeOID, listOID); - } - - void testInterfaceCreateIndex() - { - int rc; - ColumnOp colOp; - - if ( colOp.exists( 900 ) ) - { - rc = m_wrapper.dropIndex( m_session.txnid, 900, 901 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - rc = m_wrapper.createIndex( m_session.txnid, 900, 901 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - void testInterfaceDropIndex() - { - int rc; - - rc = m_wrapper.dropIndex( m_session.txnid, 900, 901 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - } - - void SetIndexStruct( IdxStruct& curStruct, OID treeOid, OID listOid, int width, CalpontSystemCatalog::ColDataType dataType ) - { - curStruct.treeOid = treeOid; - curStruct.listOid = listOid; - curStruct.idxWidth = width; - curStruct.idxDataType = dataType; - curStruct.tokenFlag = false; - } - - void testInterfaceUpdateIndexChar() - { - IdxStruct curStruct, testStruct; - IdxTuple curTuple, testTuple; - IdxTupleList curTupleList, testTupleList; - IdxStructList idxStructList; - IdxValueList idxValueList; - RIDList ridList; - std::string curStr, testStr; - int total, rc; - - testInterfaceCreateIndex(); - - CPPUNIT_ASSERT( m_wrapper.updateIndexRec( m_session.txnid, idxStructList, idxValueList, ridList ) != NO_ERROR ); - - // test column struct list - SetIndexStruct( curStruct, 900, 901, 4, WriteEngine::MEDINT ); - idxStructList.push_back( curStruct ); - - testStruct = idxStructList[0]; - CPPUNIT_ASSERT( testStruct.treeOid == 900 ); - CPPUNIT_ASSERT( testStruct.listOid == 901 ); - CPPUNIT_ASSERT( testStruct.idxWidth == 4 ); - CPPUNIT_ASSERT( testStruct.idxDataType == WriteEngine::MEDINT ); - - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // add values for the first index - curTuple.data = 102; - curTupleList.push_back( curTuple ); - - curTuple.data = 102; - curTupleList.push_back( curTuple ); - - curTuple.data = 102; - curTupleList.push_back( curTuple ); - - curTuple.data = 0; - curTupleList.push_back( curTuple ); - - curTuple.data = 200; - curTupleList.push_back( curTuple ); - - idxValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 15 ); - ridList.push_back( (RID) 16 ); - ridList.push_back( (RID) 17 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(idxValueList[0]); - total = testTupleList.size(); - - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 102 ); - - testTuple = testTupleList[3]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 0 ); - - testTuple = testTupleList[4]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 200 ); - - rc = m_wrapper.updateIndexRec( m_session.txnid, idxStructList, idxValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - -// CPPUNIT_ASSERT( m_wrapper.deleteIndexRec( idxStructList, idxValueList, ridList ) == NO_ERROR ); - } - - void testInterfaceUpdateMultiColIndex() - { - IdxStruct curStruct, testStruct; - IdxTuple curTuple, testTuple; - IdxTupleList curTupleList, testTupleList; - IdxStructList idxStructList; - IdxValueList idxValueList; - RIDList ridList; - std::string curStr, testStr; - int total, rc; - - testInterfaceCreateIndex(); - - CPPUNIT_ASSERT( m_wrapper.updateIndexRec( m_session.txnid, idxStructList, idxValueList, ridList ) != NO_ERROR ); - - // test column struct list - SetIndexStruct( curStruct, 900, 901, 4, WriteEngine::MEDINT ); - idxStructList.push_back( curStruct ); - - SetIndexStruct( curStruct, 900, 901, 2, WriteEngine::SMALLINT ); - idxStructList.push_back( curStruct ); - - testStruct = idxStructList[0]; - CPPUNIT_ASSERT( testStruct.treeOid == 900 ); - CPPUNIT_ASSERT( testStruct.listOid == 901 ); - CPPUNIT_ASSERT( testStruct.idxWidth == 4 ); - CPPUNIT_ASSERT( testStruct.idxDataType == WriteEngine::MEDINT ); - - testStruct = idxStructList[1]; - CPPUNIT_ASSERT( testStruct.treeOid == 900 ); - CPPUNIT_ASSERT( testStruct.listOid == 901 ); - CPPUNIT_ASSERT( testStruct.idxWidth == 2 ); - CPPUNIT_ASSERT( testStruct.idxDataType == WriteEngine::SMALLINT ); - - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // add values for the first index - curTuple.data = 102; - curTupleList.push_back( curTuple ); - - curTuple.data = 0; - curTupleList.push_back( curTuple ); - - curTuple.data = 200; - curTupleList.push_back( curTuple ); - - idxValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add values for the second index - curTuple.data = (short)1; - curTupleList.push_back( curTuple ); - - curTuple.data = (short)2; - curTupleList.push_back( curTuple ); - - curTuple.data = (short)3; - curTupleList.push_back( curTuple ); - - idxValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == ERR_ROWID_VALUE_NOT_MATCH ); - ridList.push_back( (RID) 9 ); - ridList.push_back( (RID) 10 ); - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(idxValueList[0]); - total = testTupleList.size(); - - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 102 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 0 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 200 ); - - // second column - testTupleList = static_cast(idxValueList[1]); - total = testTupleList.size(); - - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 1 ); - - testTuple = testTupleList[1]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 2 ); - - testTuple = testTupleList[2]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 3 ); - - rc = m_wrapper.processMultiColIndexRec( m_session.txnid, idxStructList, idxValueList, ridList ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - } - - void testInterfaceDeleteIndexChar() - { - IdxStruct curStruct, testStruct; - IdxTuple curTuple, testTuple; - IdxTupleList curTupleList, testTupleList; - std::vector idxStructList; - std::vector idxValueList; - std::vector ridList; - std::string curStr, testStr; - int total; - - // test column struct list - SetIndexStruct( curStruct, 900, 901, 4, WriteEngine::MEDINT ); - idxStructList.push_back( curStruct ); - - testStruct = idxStructList[0]; - CPPUNIT_ASSERT( testStruct.treeOid == 900 ); - CPPUNIT_ASSERT( testStruct.listOid == 901 ); - CPPUNIT_ASSERT( testStruct.idxWidth == 4 ); - CPPUNIT_ASSERT( testStruct.idxDataType == WriteEngine::MEDINT ); - - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == ERR_STRUCT_VALUE_NOT_MATCH ); - - // add values for the first index - curTuple.data = 102; - curTupleList.push_back( curTuple ); - - idxValueList.push_back( curTupleList ); - - curTupleList.clear(); - - // add one rowId - ridList.push_back( (RID) 5 ); - CPPUNIT_ASSERT( m_wrapper.checkIndexValid( m_session.txnid, idxStructList, idxValueList, ridList ) == NO_ERROR ); - - // retrieve the values back - // first column - testTupleList = static_cast(idxValueList[0]); - total = testTupleList.size(); - - testTuple = testTupleList[0]; - CPPUNIT_ASSERT( boost::any_cast( testTuple.data ) == 102 ); - - - CPPUNIT_ASSERT( m_wrapper.deleteIndexRec( m_session.txnid, idxStructList, idxValueList, ridList ) == NO_ERROR ); - } - - void testInterfaceDctnryToken() - { - /* - int rc; - OID dctnryOID=880, treeOID=881, listOID=882; - DctnryTuple dctnryTuple; - DctnryStruct dctnryStruct; - - printf("Running testInterfaceDctnryToken\n"); - memset(&dctnryTuple, 0, sizeof(dctnryTuple)); - memset(&dctnryStruct,0, sizeof(dctnryStruct)); - - dctnryStruct.dctnryOid = dctnryOID; - dctnryStruct.treeOid = treeOID; - dctnryStruct.listOid = listOID; - - BRMWrapper::setUseBrm(true); - m_wrapper.dropStore( m_session.txnid, dctnryOID, treeOID, listOID); - rc = m_wrapper.createStore( m_session.txnid, dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = m_wrapper.openStore( m_session.txnid, dctnryOID, treeOID, listOID); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - Token token[17], token2[17]; - memset(token, 0, 17*sizeof(Token)); - memset(token2, 0, 17*sizeof(Token)); - int i,j,k,smallSize=1000; + int rc; + OID dctnryOID=880, treeOID=881, listOID=882; + DctnryTuple dctnryTuple; + DctnryStruct dctnryStruct; + + printf("Running testInterfaceDctnryToken\n"); + memset(&dctnryTuple, 0, sizeof(dctnryTuple)); + memset(&dctnryStruct,0, sizeof(dctnryStruct)); + + dctnryStruct.dctnryOid = dctnryOID; + dctnryStruct.treeOid = treeOID; + dctnryStruct.listOid = listOID; + + BRMWrapper::setUseBrm(true); + m_wrapper.dropStore( m_session.txnid, dctnryOID, treeOID, listOID); + rc = m_wrapper.createStore( m_session.txnid, dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT( rc == NO_ERROR ); + rc = m_wrapper.openStore( m_session.txnid, dctnryOID, treeOID, listOID); + CPPUNIT_ASSERT( rc == NO_ERROR ); + + Token token[17], token2[17]; + memset(token, 0, 17*sizeof(Token)); + memset(token2, 0, 17*sizeof(Token)); + int i,j,k,smallSize=1000; + j=255; + for (k=1; k<17; k++) + { + for (i=0; i " << colOp.getTotalfRunTime() << " msec"<< endl; - colOp.deleteFile( fid); - sleep(3); + iSize = fwrite((char*)charBuf, 4, 1, pFile); + printf("\niSize=%d \n", iSize); + fclose(pFile); } - void testCreateColumnFiles2() + unsigned char charBuf2[100]; + memset(charBuf2, '\0', 100); + pFile = fopen("test", "r"); + iSize = fread(charBuf2, 4, 1, pFile); + memcpy(&testfValue12, charBuf2, 4); + + cout << " Using Float, NULL value=" << testfValue12 << endl; + + anyVal = fValue; + testfValue = boost::any_cast(anyVal); + memcpy(charBuf, &testfValue, 4); + pFile = fopen("test", "w+b"); + + if (pFile != NULL) { - ColumnOp colOp; - Column curCol; - int rc; - OID fid = 20; - - m_session.txnid++; - // file opertaions - - //colOp.startfTimer(); - colOp.initColumn( curCol ); - rc = colOp.createColumn( curCol, 0, 4, WriteEngine::INT, WriteEngine::WR_CHAR, (FID)fid ); - //colOp.stopfTimer(); - colOp.clearColumn( curCol ); - //rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::INT, 4); - //colOp.stopfTimer(); - //cout<< "total run time for 2th column we_int size=4 -> " << colOp.getTotalfRunTime() << " msec"<< endl; - CPPUNIT_ASSERT( rc == NO_ERROR ); - colOp.deleteFile( fid); - sleep(5); + iSize = fwrite((char*)charBuf, 4, 1, pFile); + printf("\niSize=%d \n", iSize); + fclose(pFile); } - void testCreateColumnFiles3() + + memset(charBuf2, '\0', 100); + pFile = fopen("test", "r"); + iSize = fread(charBuf2, 4, 1, pFile); + memcpy(&testfValue2, charBuf2, 4); + cout << " Using unsinged int, NULL value=" << testfValue2 << endl; + + // Other method + + anyVal = fValue; + testfValue = boost::any_cast(anyVal); + ofstream fout("file.dat", ios::binary); + fout.write((char*)(&testfValue), sizeof(testfValue)); + fout.close(); + + ifstream fin("file.dat", ios::binary); + fin.read((char*)(&testfValue2), sizeof(testfValue2)); + cout << "Using unsigned int NULL value=" << testfValue2 << endl; + } + void testCreateColumnFiles1() + { + ColumnOp colOp; + Column curCol; + int rc; + OID fid = 10; + + m_session.txnid++; + // file opertaions + + // colOp.startfTimer(); + colOp.initColumn(curCol); + rc = colOp.createColumn(curCol, 0, 8, WriteEngine::CHAR, WriteEngine::WR_CHAR, (FID)fid); + // colOp.stopfTimer(); + colOp.clearColumn(curCol); + // rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::CHAR, 8); + + // cout<< "total run time for 1th column we_char size=8 -> " << colOp.getTotalfRunTime() << " msec"<< + // endl; + colOp.deleteFile(fid); + sleep(3); + } + + void testCreateColumnFiles2() + { + ColumnOp colOp; + Column curCol; + int rc; + OID fid = 20; + + m_session.txnid++; + // file opertaions + + // colOp.startfTimer(); + colOp.initColumn(curCol); + rc = colOp.createColumn(curCol, 0, 4, WriteEngine::INT, WriteEngine::WR_CHAR, (FID)fid); + // colOp.stopfTimer(); + colOp.clearColumn(curCol); + // rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::INT, 4); + // colOp.stopfTimer(); + // cout<< "total run time for 2th column we_int size=4 -> " << colOp.getTotalfRunTime() << " msec"<< endl; + CPPUNIT_ASSERT(rc == NO_ERROR); + colOp.deleteFile(fid); + sleep(5); + } + void testCreateColumnFiles3() + { + ColumnOp colOp; + Column curCol; + int rc; + OID fid = 30; + + m_session.txnid++; + // file opertaions + + // colOp.startfTimer(); + colOp.initColumn(curCol); + rc = colOp.createColumn(curCol, 0, 4, WriteEngine::CHAR, WriteEngine::WR_CHAR, (FID)fid); + colOp.clearColumn(curCol); + // colOp.stopfTimer(); + // rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::CHAR, 4); + // colOp.stopfTimer(); + // cout<< "total run time for 3th column we_char size=4 -> " << colOp.getTotalfRunTime() << " msec"<< + // endl; + CPPUNIT_ASSERT(rc == NO_ERROR); + colOp.deleteFile(fid); + sleep(3); + } + void testCreateColumnFiles4() + { + ColumnOp colOp; + Column curCol; + int rc; + OID fid = 40; + + m_session.txnid++; + // file opertaions + + // colOp.startfTimer(); + colOp.initColumn(curCol); + rc = colOp.createColumn(curCol, 0, 2, WriteEngine::SMALLINT, WriteEngine::WR_CHAR, (FID)fid); + colOp.clearColumn(curCol); + // rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::SMALLINT, 2 ); + // colOp.stopfTimer(); + // cout<< "total run time for 4th column we_smallint size=2 -> " << colOp.getTotalfRunTime() << " msec"<< + // endl; + CPPUNIT_ASSERT(rc == NO_ERROR); + colOp.deleteFile(fid); + } + void testCreateFileMultipleBF() + { + FileOp fileop; + ColumnOp colOp; + char fileName[FILE_NAME_SIZE]; + int rc, numOfBlock = 8192, allocSize = 8192; + FILE* pFile; + int t_diff; + struct timeb t_start, t_current; + + OID fid = 600; + uint64_t emptyVal = 0; + int width = 4; + + rc = fileop.oid2FileName(fid, fileName, true); + emptyVal = colOp.getEmptyRowValue(WriteEngine::INT, width); + int multiple = 1; + int writeSize = multiple * BYTE_PER_BLOCK; + unsigned char writeBuf[writeSize]; + BRMWrapper::getInstance()->allocateExtent((const OID)fid, numOfBlock, allocSize); + + pFile = fopen(fileName, "w+b"); + + if (pFile != NULL) { - ColumnOp colOp; - Column curCol; - int rc; - OID fid = 30; + fileop.setEmptyBuf(writeBuf, writeSize, emptyVal, width); + // setvbuf(pFile , NULL, _IONBF, 0 ); + int loopSize = numOfBlock / multiple; + ftime(&t_start); - m_session.txnid++; - // file opertaions + for (int i = 0; i < loopSize; i++) + { + fwrite(writeBuf, writeSize, 1, pFile); + } - //colOp.startfTimer(); - colOp.initColumn( curCol ); - rc = colOp.createColumn( curCol, 0, 4, WriteEngine::CHAR, WriteEngine::WR_CHAR, (FID)fid ); - colOp.clearColumn( curCol ); - //colOp.stopfTimer(); - //rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::CHAR, 4); - //colOp.stopfTimer(); - //cout<< "total run time for 3th column we_char size=4 -> " << colOp.getTotalfRunTime() << " msec"<< endl; - CPPUNIT_ASSERT( rc == NO_ERROR ); - colOp.deleteFile( fid); - sleep(3); + ftime(&t_current); + t_diff = (int)(1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); + cout << "total run time for create file 64MB with writesize= " << writeSize << " byte" + << "loop size=" << loopSize << " times" + << " Time for Multiple Write 64MB with Buffer->we_int size=4 -> " << t_diff << " msec" << endl; + + fileop.closeFile(pFile); } - void testCreateColumnFiles4() + + colOp.deleteFile(fid); + } + void testCreateFilelargerBF() + { + FileOp fileop; + ColumnOp colOp; + char fileName[FILE_NAME_SIZE]; + int rc, numOfBlock = 8192, allocSize = 8192; + FILE* pFile; + + int t_diff; + struct timeb t_start, t_current; + + OID fid = 1600; + uint64_t emptyVal = 0; + int width = 4; + + rc = fileop.oid2FileName(fid, fileName, true); + emptyVal = colOp.getEmptyRowValue(WriteEngine::INT, width); + int multiple = 1; + int writeSize = multiple * BYTE_PER_BLOCK; + unsigned char writeBuf[writeSize]; + BRMWrapper::getInstance()->allocateExtent((const OID)fid, numOfBlock, allocSize); + + pFile = fopen(fileName, "w+b"); + + if (pFile != NULL) { - ColumnOp colOp; - Column curCol; - int rc; - OID fid = 40; + fileop.setEmptyBuf(writeBuf, writeSize, emptyVal, width); - m_session.txnid++; - // file opertaions + setvbuf(pFile, NULL, _IOFBF, DEFAULT_BUFSIZ); + // setvbuf(pFile , (char*)writeBuf, _IOFBF, writeSize/*DEFAULT_WRITE_BUFSIZE*/); + int loopSize = allocSize / multiple; + ftime(&t_start); - //colOp.startfTimer(); - colOp.initColumn( curCol ); - rc = colOp.createColumn( curCol, 0, 2, WriteEngine::SMALLINT, WriteEngine::WR_CHAR, (FID)fid ); - colOp.clearColumn( curCol ); - //rc = m_wrapper.createColumn( m_session.txnid, fid, WriteEngine::SMALLINT, 2 ); - //colOp.stopfTimer(); - //cout<< "total run time for 4th column we_smallint size=2 -> " << colOp.getTotalfRunTime() << " msec"<< endl; - CPPUNIT_ASSERT( rc == NO_ERROR ); - colOp.deleteFile( fid); + for (int i = 0; i < loopSize; i++) + { + fwrite(writeBuf, writeSize, 1, pFile); + } + + ftime(&t_current); + t_diff = (int)(1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); + cout << "Buffer size =" << DEFAULT_BUFSIZ << endl; + cout << "total run time for create file 64MB with writesize= " << writeSize << " byte" + << "loop size=" << loopSize << " times" + << " Time for Multiple Write 64MB with Buffer->we_int size=4 -> " << t_diff << " msec" << endl; + + fileop.closeFile(pFile); } - void testCreateFileMultipleBF() + + colOp.deleteFile(fid); + } + + void testCreateFileMultipleIONBF() + { + FileOp fileop; + ColumnOp colOp; + char fileName[FILE_NAME_SIZE]; + int rc, numOfBlock = 8192, allocSize = 8192; + FILE* pFile; + int t_diff; + struct timeb t_start, t_current; + + OID fid = 400; + uint64_t emptyVal = 0; + int width = 4; + + rc = fileop.oid2FileName(fid, fileName, true); + emptyVal = colOp.getEmptyRowValue(WriteEngine::INT, width); + int multiple = 128; + int writeSize = multiple * BYTE_PER_BLOCK; + unsigned char writeBuf[writeSize]; + BRMWrapper::getInstance()->allocateExtent((const OID)fid, numOfBlock, allocSize); + + pFile = fopen(fileName, "w+b"); + + if (pFile != NULL) { - FileOp fileop; - ColumnOp colOp; - char fileName[FILE_NAME_SIZE]; - int rc, numOfBlock = 8192, allocSize = 8192; - FILE* pFile; - int t_diff; - struct timeb t_start, t_current; + fileop.setEmptyBuf(writeBuf, writeSize, emptyVal, width); + setvbuf(pFile, NULL, _IONBF, 0); + int loopSize = numOfBlock / multiple; + ftime(&t_start); + for (int i = 0; i < loopSize; i++) + { + fwrite(writeBuf, writeSize, 1, pFile); + } - OID fid = 600; - uint64_t emptyVal = 0; - int width = 4; + ftime(&t_current); + t_diff = (int)(1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); + cout << "total run time for create file 64MB with writesize= " << writeSize << " byte" + << "loop size=" << loopSize << " times" + << " Time for Multiple Write 64MB NO Buffer->we_int size=4 -> " << t_diff << " msec" << endl; - rc = fileop.oid2FileName( fid, fileName, true ) ; - emptyVal = colOp.getEmptyRowValue( WriteEngine::INT, width); - int multiple = 1; - int writeSize = multiple * BYTE_PER_BLOCK; - unsigned char writeBuf[writeSize]; - BRMWrapper::getInstance()->allocateExtent( (const OID)fid, numOfBlock, allocSize ); - - pFile = fopen( fileName, "w+b" ); - - if ( pFile != NULL ) - { - fileop.setEmptyBuf( writeBuf, writeSize, emptyVal, width ); - //setvbuf(pFile , NULL, _IONBF, 0 ); - int loopSize = numOfBlock / multiple; - ftime(&t_start); - - for ( int i = 0; i < loopSize; i++ ) - { - fwrite( writeBuf, writeSize, 1, pFile ); - } - - ftime(&t_current); - t_diff = (int) (1000.0 * (t_current.time - t_start.time) + - (t_current.millitm - t_start.millitm)); - cout << "total run time for create file 64MB with writesize= " << writeSize << " byte" << "loop size=" - << loopSize << " times" << " Time for Multiple Write 64MB with Buffer->we_int size=4 -> " << t_diff << " msec" << endl; - - fileop.closeFile( pFile ); - } - - colOp.deleteFile( fid); + fileop.closeFile(pFile); } - void testCreateFilelargerBF() + + colOp.deleteFile(fid); + } + void testCreateFile64MBIONBF() + { + FileOp fileop; + ColumnOp colOp; + char fileName[FILE_NAME_SIZE]; + int rc, numOfBlock = 8192, allocSize = 8192; + FILE* pFile; + int t_diff; + struct timeb t_start, t_current; + + OID fid = 500; + uint64_t emptyVal = 0; + int width = 4; + + rc = fileop.oid2FileName(fid, fileName, true); + emptyVal = colOp.getEmptyRowValue(WriteEngine::INT, width); + int multiple = 8192; + int writeSize = multiple * BYTE_PER_BLOCK; + unsigned char* writeBuf; + BRMWrapper::getInstance()->allocateExtent((const OID)fid, numOfBlock, allocSize); + + pFile = fopen(fileName, "w+b"); + + if (pFile != NULL) { - FileOp fileop; - ColumnOp colOp; - char fileName[FILE_NAME_SIZE]; - int rc, numOfBlock = 8192, allocSize = 8192; - FILE* pFile; + writeBuf = new unsigned char[writeSize]; + fileop.setEmptyBuf(writeBuf, writeSize, emptyVal, width); + setvbuf(pFile, NULL, _IONBF, 0); - int t_diff; - struct timeb t_start, t_current; + ftime(&t_start); + fwrite(writeBuf, writeSize, 1, pFile); + ftime(&t_current); + t_diff = (int)(1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); - - OID fid = 1600; - uint64_t emptyVal = 0; - int width = 4; - - rc = fileop.oid2FileName( fid, fileName, true ) ; - emptyVal = colOp.getEmptyRowValue( WriteEngine::INT, width); - int multiple = 1; - int writeSize = multiple * BYTE_PER_BLOCK; - unsigned char writeBuf[writeSize]; - BRMWrapper::getInstance()->allocateExtent( (const OID)fid, numOfBlock, allocSize ); - - pFile = fopen( fileName, "w+b" ); - - if ( pFile != NULL ) - { - fileop.setEmptyBuf( writeBuf, writeSize, emptyVal, width ); - - setvbuf(pFile, NULL, _IOFBF, DEFAULT_BUFSIZ); - //setvbuf(pFile , (char*)writeBuf, _IOFBF, writeSize/*DEFAULT_WRITE_BUFSIZE*/); - int loopSize = allocSize / multiple; - ftime(&t_start); - - for ( int i = 0; i < loopSize; i++ ) - { - fwrite( writeBuf, writeSize, 1, pFile ); - } - - ftime(&t_current); - t_diff = (int) (1000.0 * (t_current.time - t_start.time) + - (t_current.millitm - t_start.millitm)); - cout << "Buffer size =" << DEFAULT_BUFSIZ << endl; - cout << "total run time for create file 64MB with writesize= " << writeSize << " byte" << "loop size=" - << loopSize << " times" << " Time for Multiple Write 64MB with Buffer->we_int size=4 -> " << t_diff << " msec" << endl; - - fileop.closeFile( pFile ); - } - - colOp.deleteFile( fid); + cout << "total run time for create file Single Write 64MB NO Buffer->we_int size=4 -> " << t_diff + << " msec" << endl; + fileop.closeFile(pFile); } - void testCreateFileMultipleIONBF() + colOp.deleteFile(fid); + } + void testCreateFileNoBRM() + { + FileOp fileop; + ColumnOp colOp; + char fileName[FILE_NAME_SIZE]; + int rc, allocSize = 8192; + + OID fid = 401; + uint64_t emptyVal = 0; + + // colOp.deleteFile( fid); + + // cout << "start time=" << colOp.t_start << " msec" << endl; + + rc = fileop.oid2FileName(fid, fileName, true); + emptyVal = colOp.getEmptyRowValue(WriteEngine::INT, 4); + // colOp.startfTimer(); + + fileop.createFile(fileName, allocSize, emptyVal, 4); + // colOp.stopfTimer(); + // cout<<"stop time=" << colOp.t_current << " msec"<< endl; + // cout<< "total run time for column NO BRM we_int size=4 -> " << colOp.getTotalfRunTime() << " msec"<< + // endl; + colOp.deleteFile(fid); + } + + void testSoloman() + { + char fileName[80] = "/usr/local/mariadb/columnstore/data1/test.dat"; + char outfile[80] = "/usr/local/mariadb/columnstore/data1/out.dat"; + int numOfBlock = 8192, blockSize = 8192; + FILE* testFile; + FILE* outFile; + int t_diff; + struct timeb t_start, t_current; + int multiple = numOfBlock; + int writeSize = multiple * blockSize; + unsigned char* writeBuf; + writeBuf = new unsigned char[writeSize]; + memset(writeBuf, 0, writeSize); + testFile = fopen(fileName, "w+b"); + outFile = fopen(outfile, "w"); + + if (testFile != NULL) { - FileOp fileop; - ColumnOp colOp; - char fileName[FILE_NAME_SIZE]; - int rc, numOfBlock = 8192, allocSize = 8192; - FILE* pFile; - int t_diff; - struct timeb t_start, t_current; + setvbuf(testFile, (char*)writeBuf, _IOFBF, writeSize); + int loopSize = numOfBlock / multiple; + ftime(&t_start); + for (int i = 0; i < loopSize; i++) + { + fwrite(writeBuf, writeSize, 1, testFile); + } - OID fid = 400; - uint64_t emptyVal = 0; - int width = 4; - - rc = fileop.oid2FileName( fid, fileName, true ) ; - emptyVal = colOp.getEmptyRowValue( WriteEngine::INT, width); - int multiple = 128; - int writeSize = multiple * BYTE_PER_BLOCK; - unsigned char writeBuf[writeSize]; - BRMWrapper::getInstance()->allocateExtent( (const OID)fid, numOfBlock, allocSize ); - - pFile = fopen( fileName, "w+b" ); - - if ( pFile != NULL ) - { - fileop.setEmptyBuf( writeBuf, writeSize, emptyVal, width ); - setvbuf(pFile, NULL, _IONBF, 0 ); - int loopSize = numOfBlock / multiple; - ftime(&t_start); - - for ( int i = 0; i < loopSize; i++ ) - { - fwrite( writeBuf, writeSize, 1, pFile ); - } - - ftime(&t_current); - t_diff = (int) (1000.0 * (t_current.time - t_start.time) + - (t_current.millitm - t_start.millitm)); - cout << "total run time for create file 64MB with writesize= " << writeSize << " byte" << "loop size=" - << loopSize << " times" << " Time for Multiple Write 64MB NO Buffer->we_int size=4 -> " << t_diff << " msec" << endl; - - fileop.closeFile( pFile ); - } - - colOp.deleteFile( fid); - } - void testCreateFile64MBIONBF() - { - FileOp fileop; - ColumnOp colOp; - char fileName[FILE_NAME_SIZE]; - int rc, numOfBlock = 8192, allocSize = 8192; - FILE* pFile; - int t_diff; - struct timeb t_start, t_current; - - OID fid = 500; - uint64_t emptyVal = 0; - int width = 4; - - rc = fileop.oid2FileName( fid, fileName, true ) ; - emptyVal = colOp.getEmptyRowValue( WriteEngine::INT, width); - int multiple = 8192; - int writeSize = multiple * BYTE_PER_BLOCK; - unsigned char* writeBuf; - BRMWrapper::getInstance()->allocateExtent( (const OID)fid, numOfBlock, allocSize ); - - pFile = fopen( fileName, "w+b" ); - - if ( pFile != NULL ) - { - writeBuf = new unsigned char[writeSize]; - fileop.setEmptyBuf( writeBuf, writeSize, emptyVal, width ); - setvbuf(pFile, NULL, _IONBF, 0 ); - - ftime(&t_start); - fwrite( writeBuf, writeSize, 1, pFile ); - ftime(&t_current); - t_diff = (int) (1000.0 * (t_current.time - t_start.time) + - (t_current.millitm - t_start.millitm)); - - cout << "total run time for create file Single Write 64MB NO Buffer->we_int size=4 -> " << t_diff << " msec" << endl; - fileop.closeFile( pFile ); - } - - colOp.deleteFile( fid); - } - void testCreateFileNoBRM() - { - FileOp fileop; - ColumnOp colOp; - char fileName[FILE_NAME_SIZE]; - int rc, allocSize = 8192; - - OID fid = 401; - uint64_t emptyVal = 0; - - - //colOp.deleteFile( fid); - - //cout << "start time=" << colOp.t_start << " msec" << endl; - - rc = fileop.oid2FileName( fid, fileName, true ) ; - emptyVal = colOp.getEmptyRowValue( WriteEngine::INT, 4); - //colOp.startfTimer(); - - fileop.createFile( fileName, allocSize, emptyVal, 4 ); - //colOp.stopfTimer(); - //cout<<"stop time=" << colOp.t_current << " msec"<< endl; - //cout<< "total run time for column NO BRM we_int size=4 -> " << colOp.getTotalfRunTime() << " msec"<< endl; - colOp.deleteFile( fid); - - } - - void testSoloman() - { - char fileName[80] = "/usr/local/mariadb/columnstore/data1/test.dat"; - char outfile[80] = "/usr/local/mariadb/columnstore/data1/out.dat"; - int numOfBlock = 8192, blockSize = 8192; - FILE* testFile; - FILE* outFile; - int t_diff; - struct timeb t_start, t_current; - int multiple = numOfBlock; - int writeSize = multiple * blockSize; - unsigned char* writeBuf; - writeBuf = new unsigned char[writeSize]; - memset(writeBuf, 0, writeSize); - testFile = fopen( fileName, "w+b" ); - outFile = fopen(outfile, "w"); - - if ( testFile != NULL ) - { - setvbuf(testFile, (char*)writeBuf, _IOFBF, writeSize); - int loopSize = numOfBlock / multiple; - ftime(&t_start); - - for ( int i = 0; i < loopSize; i++ ) - { - fwrite( writeBuf, writeSize, 1, testFile ); - } - - ftime(&t_current); - t_diff = (int) (1000.0 * (t_current.time - t_start.time) + - (t_current.millitm - t_start.millitm)); - char buff[256]; - sprintf(buff, " create a 64MB file with writesize= %d byte", writeSize); - fwrite(buff, strlen(buff), 1, outFile); - sprintf(buff, " loop size = %d", loopSize); - fwrite(buff, strlen(buff), 1, outFile); - sprintf(buff, " Time for writing %d = %d msec", writeSize, t_diff); - fwrite(buff, strlen(buff), 1, outFile); - fclose( testFile ); - fclose( outFile); - } - else - fprintf(outFile, "FILE DOES NOT EXIST: %s", fileName); - - delete [] writeBuf; - + ftime(&t_current); + t_diff = (int)(1000.0 * (t_current.time - t_start.time) + (t_current.millitm - t_start.millitm)); + char buff[256]; + sprintf(buff, " create a 64MB file with writesize= %d byte", writeSize); + fwrite(buff, strlen(buff), 1, outFile); + sprintf(buff, " loop size = %d", loopSize); + fwrite(buff, strlen(buff), 1, outFile); + sprintf(buff, " Time for writing %d = %d msec", writeSize, t_diff); + fwrite(buff, strlen(buff), 1, outFile); + fclose(testFile); + fclose(outFile); } + else + fprintf(outFile, "FILE DOES NOT EXIST: %s", fileName); + delete[] writeBuf; + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( WriteEngineTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(WriteEngineTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 5a351c260..25bed1ebe 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -51,29 +51,29 @@ namespace WriteEngine { struct RefcolInfo { - int localHwm; - unsigned numExtents; + int localHwm; + unsigned numExtents; }; - /** * Constructor */ ColumnOp::ColumnOp() { - //memset(m_workBlock.data, 0, BYTE_PER_BLOCK); + // memset(m_workBlock.data, 0, BYTE_PER_BLOCK); } ColumnOp::ColumnOp(Log* logger) { - setDebugLevel(logger->getDebugLevel()); - setLogger (logger); + setDebugLevel(logger->getDebugLevel()); + setLogger(logger); } /** * Default Destructor */ ColumnOp::~ColumnOp() -{} +{ +} /*********************************************************** * DESCRIPTION: @@ -88,504 +88,509 @@ ColumnOp::~ColumnOp() * NO_ERROR if success * rowIdArray - allocation of the row id left here ***********************************************************/ -int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, - Column& column, uint64_t totalRow, RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm, - bool& newFile, ColStructList& newColStructList, DctnryStructList& newDctnryStructList, std::vector >& dbRootExtentTrackers, - bool insertSelect, bool isBatchInsert, OID tableOid, bool isFirstBatchPm, std::vector* newExtents) +int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, Column& column, uint64_t totalRow, + RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm, + bool& newFile, ColStructList& newColStructList, + DctnryStructList& newDctnryStructList, + std::vector >& dbRootExtentTrackers, + bool insertSelect, bool isBatchInsert, OID tableOid, bool isFirstBatchPm, + std::vector* newExtents) { - //MultiFiles per OID: always append the rows to the end for now. - // See if the current HWM block might be in an abbreviated extent that - // needs to be expanded, if we end up adding enough rows. - bool bCheckAbbrevExtent = false; - uint64_t numBlksPerInitialExtent = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * column.colWidth; - int counter = 0; - uint64_t totalRowPerBlock = BYTE_PER_BLOCK / column.colWidth; - uint64_t extentRows = BRMWrapper::getInstance()->getExtentRows(); + // MultiFiles per OID: always append the rows to the end for now. + // See if the current HWM block might be in an abbreviated extent that + // needs to be expanded, if we end up adding enough rows. + bool bCheckAbbrevExtent = false; + uint64_t numBlksPerInitialExtent = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * column.colWidth; + int counter = 0; + uint64_t totalRowPerBlock = BYTE_PER_BLOCK / column.colWidth; + uint64_t extentRows = BRMWrapper::getInstance()->getExtentRows(); - if (useStartingExtent) + if (useStartingExtent) + { + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((column.dataFile.fPartition == 0) && (column.dataFile.fSegment == 0) && + ((hwm + 1) <= numBlksPerInitialExtent)) + bCheckAbbrevExtent = abbreviatedExtent(column.dataFile.pFile, column.colWidth); + + // The current existed rows upto hwm + uint64_t currentRows = totalRowPerBlock * hwm; + uint64_t numExtentsFilled = currentRows / extentRows; + uint64_t rowsAvailable = extentRows - (numExtentsFilled * extentRows); + rowsLeft = totalRow < rowsAvailable ? 0 : totalRow - rowsAvailable; + } + else + { + rowsLeft = totalRow; + } + + newExtent = false; + uint32_t j = 0, i = 0, rowsallocated = 0; + int rc = 0; + newFile = false; + Column newCol; + unsigned char buf[BYTE_PER_BLOCK]; + unsigned char* curVal; + const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + if (useStartingExtent) + { + // ZZ. For insert select, skip the hwm block and start inserting from the next block + // to avoid self insert issue. + // For batch insert: if not first batch, use the saved last rid to start adding rows. + + if (!insertSelect || !isFirstBatchPm) { - // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((column.dataFile.fPartition == 0) && - (column.dataFile.fSegment == 0) && - ((hwm + 1) <= numBlksPerInitialExtent)) - bCheckAbbrevExtent = abbreviatedExtent(column.dataFile.pFile, column.colWidth); + //..Search the HWM block for empty rows + rc = readBlock(column.dataFile.pFile, buf, hwm); - //The current existed rows upto hwm - uint64_t currentRows = totalRowPerBlock * hwm; - uint64_t numExtentsFilled = currentRows / extentRows; - uint64_t rowsAvailable = extentRows - (numExtentsFilled * extentRows); - rowsLeft = totalRow < rowsAvailable ? 0 : totalRow - rowsAvailable; - } - else - { - rowsLeft = totalRow; - } + if (rc != NO_ERROR) + return rc; - newExtent = false; - uint32_t j = 0, i = 0, rowsallocated = 0; - int rc = 0; - newFile = false; - Column newCol; - unsigned char buf[BYTE_PER_BLOCK]; - unsigned char* curVal; - const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, - column.colWidth); - if (useStartingExtent) - { - // ZZ. For insert select, skip the hwm block and start inserting from the next block - // to avoid self insert issue. - //For batch insert: if not first batch, use the saved last rid to start adding rows. - - if (!insertSelect || !isFirstBatchPm) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) + { + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { - //..Search the HWM block for empty rows - rc = readBlock(column.dataFile.pFile, buf, hwm); + rowIdArray[counter] = getRowId(hwm, column.colWidth, j); + rowsallocated++; + counter++; - if ( rc != NO_ERROR) - return rc; - - for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) - { - if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) - { - rowIdArray[counter] = getRowId(hwm, column.colWidth, j); - rowsallocated++; - counter++; - - if (rowsallocated >= totalRow) - break; - } - } + if (rowsallocated >= totalRow) + break; } + } + } + } + + if (rowsallocated < totalRow) + { + if (useStartingExtent) + { + //..Search remaining blks in current extent (after HWM) for empty rows + // Need go to next block + // need check whether this block is the last block for this extent + while (((totalRowPerBlock * (hwm + 1)) % extentRows) > 0) + { + hwm++; + + // Expand abbreviated initial extent on disk if needed. + if (bCheckAbbrevExtent) + { + if ((hwm + 1) > numBlksPerInitialExtent) + { + RETURN_ON_ERROR(expandAbbrevExtent(column)); + bCheckAbbrevExtent = false; + } + } + + rc = readBlock(column.dataFile.pFile, buf, hwm); + + // MCOL-498 add a block. + // DRRTUY TODO Given there is no more hwm pre-allocated we + // could extend the file once to accomodate all records. + if (rc != NO_ERROR) + { + if (rc == ERR_FILE_EOF) + { + setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); + RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, buf, hwm)); + } + else + { + return rc; + } + } + + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) + { + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) + { + rowIdArray[counter] = getRowId(hwm, column.colWidth, j); + rowsallocated++; + counter++; + + if (rowsallocated >= totalRow) + break; + } + } + + if (rowsallocated >= totalRow) + break; + } } + if ((rowsallocated == 0) && isFirstBatchPm) + { + TableMetaData::removeTableMetaData(tableOid); + } + + // Check if a new extent is needed if (rowsallocated < totalRow) { - if (useStartingExtent) + // Create another extent + uint16_t dbRoot; + uint32_t partition = 0; + uint16_t segment; + IDBDataFile* pFile = NULL; + std::string segFile; + rowsLeft = 0; + int allocSize = 0; + newExtent = true; + + if ((column.dataFile.fid < 3000) || (!isBatchInsert)) // systables or single insert + { + dbRoot = column.dataFile.fDbRoot; + } + else + { + // Find out where the rest rows go + BRM::LBID_t startLbid; + // need to put in a loop until newExtent is true + newExtent = + dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); + + while (!newExtent) { - //..Search remaining blks in current extent (after HWM) for empty rows - //Need go to next block - //need check whether this block is the last block for this extent - while (((totalRowPerBlock * (hwm + 1)) % extentRows) > 0) + /*partially filled extent encountered due to user moved dbroot. Set hwm to the end of the extent. + If compressed,fill the rest eith empty values. + */ + unsigned int BLKS_PER_EXTENT = 0; + unsigned int nBlks = 0; + unsigned int nRem = 0; + FileOp fileOp; + long long fileSizeBytes = 0; + + for (i = 0; i < dbRootExtentTrackers.size(); i++) + { + uint32_t colNo = column.colNo; + if (i != colNo) + dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + + // Round up HWM to the end of the current extent + BLKS_PER_EXTENT = + (BRMWrapper::getInstance()->getExtentRows() * newColStructList[i].colWidth) / BYTE_PER_BLOCK; + nBlks = newHwm + 1; + nRem = nBlks % BLKS_PER_EXTENT; + + if (nRem > 0) + newHwm = nBlks - nRem + BLKS_PER_EXTENT - 1; + else + newHwm = nBlks - 1; + + // save it to set in the end + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtInfo aExt; + aExt.dbRoot = dbRoot; + aExt.partNum = partition; + aExt.segNum = segment; + aExt.hwm = newHwm; + aExt.isNewExt = false; + aExt.current = false; + aColExtsInfo.push_back(aExt); + + if (newColStructList[i].fCompressionType > 0) { - hwm++; + string errorInfo; + rc = fileOp.fillCompColumnExtentEmptyChunks( + newColStructList[i].dataOid, newColStructList[i].colWidth, emptyVal, dbRoot, partition, + segment, newColStructList[i].colDataType, newHwm, segFile, errorInfo); - // Expand abbreviated initial extent on disk if needed. - if (bCheckAbbrevExtent) - { - if ((hwm + 1) > numBlksPerInitialExtent) - { - RETURN_ON_ERROR(expandAbbrevExtent(column)); - bCheckAbbrevExtent = false; - } - } - - rc = readBlock(column.dataFile.pFile, buf, hwm); - - // MCOL-498 add a block. - // DRRTUY TODO Given there is no more hwm pre-allocated we - // could extend the file once to accomodate all records. - if (rc != NO_ERROR) - { - if (rc == ERR_FILE_EOF) - { - setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); - RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, buf, hwm)); - } - else - { - return rc; - } - } - - for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) - { - if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) - { - rowIdArray[counter] = getRowId(hwm, column.colWidth, j); - rowsallocated++; - counter++; - - if (rowsallocated >= totalRow) - break; - } - } - - if (rowsallocated >= totalRow) - break; + if (rc != NO_ERROR) + return rc; } - } - - if ((rowsallocated == 0) && isFirstBatchPm) - { - TableMetaData::removeTableMetaData(tableOid); - } - - //Check if a new extent is needed - if (rowsallocated < totalRow) - { - //Create another extent - uint16_t dbRoot; - uint32_t partition = 0; - uint16_t segment; - IDBDataFile* pFile = NULL; - std::string segFile; - rowsLeft = 0; - int allocSize = 0; - newExtent = true; - - if ((column.dataFile.fid < 3000) || (!isBatchInsert)) //systables or single insert + //@Bug 4758. Check whether this is a abbreviated extent + else if (newColStructList[i].fCompressionType == 0) { - dbRoot = column.dataFile.fDbRoot; + rc = fileOp.getFileSize(newColStructList[i].dataOid, dbRoot, partition, segment, fileSizeBytes); + + if (rc != NO_ERROR) + return rc; + + if (fileSizeBytes == (long long)INITIAL_EXTENT_ROWS_TO_DISK * newColStructList[i].colWidth) + { + IDBDataFile* pFile = + fileOp.openFile(newColStructList[i].dataOid, dbRoot, partition, segment, segFile); + + if (!pFile) + { + rc = ERR_FILE_OPEN; + return rc; + } + + rc = fileOp.expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, newColStructList[i].colWidth, + newColStructList[i].colDataType); + + // set hwm for this extent. + fileOp.closeFile(pFile); + + if (rc != NO_ERROR) + return rc; + } + } + + tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + newExtent = + dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + } + } + + std::vector extents; + + if (newExtent) + { + // extend all columns together + std::vector cols; + BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; + + for (i = 0; i < newColStructList.size(); i++) + { + createStripeColumnExtentsArgIn.oid = newColStructList[i].dataOid; + createStripeColumnExtentsArgIn.width = newColStructList[i].colWidth; + createStripeColumnExtentsArgIn.colDataType = newColStructList[i].colDataType; + cols.push_back(createStripeColumnExtentsArgIn); + } + + rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partition, segment, extents); + newHwm = extents[column.colNo].startBlkOffset; + + if (rc != NO_ERROR) + return rc; + + // Create column files + vector lbids; + vector colDataTypes; + + for (i = 0; i < extents.size(); i++) + { + setColParam(newCol, 0, newColStructList[i].colWidth, newColStructList[i].colDataType, + newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, dbRoot, partition, segment); + + compressionType(newColStructList[i].fCompressionType); + rc = extendColumn(newCol, false, extents[i].startBlkOffset, extents[i].startLbid, + extents[i].allocSize, dbRoot, partition, segment, segFile, pFile, newFile); + + if (rc != NO_ERROR) + return rc; + + newColStructList[i].fColPartition = partition; + newColStructList[i].fColSegment = segment; + newColStructList[i].fColDbRoot = dbRoot; + newDctnryStructList[i].fColPartition = partition; + newDctnryStructList[i].fColSegment = segment; + newDctnryStructList[i].fColDbRoot = dbRoot; + lbids.push_back(extents[i].startLbid); + if (newExtents) + { + (*newExtents).push_back(extents[i].startLbid); + } + colDataTypes.push_back(newColStructList[i].colDataType); + } + + // mark the extents to updating + rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); + + if (rc != NO_ERROR) + return rc; + + // create corresponding dictionary files + if (newFile) + { + boost::scoped_ptr we(new WriteEngineWrapper()); + we->setTransId(txnid); + we->setBulkFlag(true); + std::map columnOids; + + for (i = 0; i < newDctnryStructList.size(); i++) + { + if (newDctnryStructList[i].dctnryOid > 0) + { + rc = we->createDctnry(txnid, newDctnryStructList[i].dctnryOid, newDctnryStructList[i].colWidth, + dbRoot, partition, segment, newDctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + return rc; + + columnOids[newDctnryStructList[i].dctnryOid] = newDctnryStructList[i].dctnryOid; + } + } + + we->flushDataFiles(rc, txnid, columnOids); + } + } + + // save the extent info for batch insert + if (isBatchInsert && newExtent) + { + TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); + + for (i = 0; i < newColStructList.size(); i++) + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && + (it->partNum == newColStructList[i].fColPartition) && + (it->segNum == newColStructList[i].fColSegment)) + break; + + it++; + } + + ColExtInfo aExt; + aExt.dbRoot = newColStructList[i].fColDbRoot; + aExt.partNum = newColStructList[i].fColPartition; + aExt.segNum = newColStructList[i].fColSegment; + aExt.hwm = extents[i].startBlkOffset; + aExt.isNewExt = true; + aExt.current = true; + aExt.isDict = false; + aColExtsInfo.push_back(aExt); + tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + for (i = 0; i < newDctnryStructList.size(); i++) + { + if (newDctnryStructList[i].dctnryOid > 0) + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(newDctnryStructList[i].dctnryOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newDctnryStructList[i].fColDbRoot) && + (it->partNum == newDctnryStructList[i].fColPartition) && + (it->segNum == newDctnryStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = newDctnryStructList[i].fColDbRoot; + aExt.partNum = newDctnryStructList[i].fColPartition; + aExt.segNum = newDctnryStructList[i].fColSegment; + aExt.compType = newDctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + } + + tableMetaData->setColExtsInfo(newDctnryStructList[i].dctnryOid, aColExtsInfo); + } + } + } + + setColParam(newCol, 0, column.colWidth, column.colDataType, column.colType, column.dataFile.fid, + column.compressionType, dbRoot, partition, segment); + rc = openColumnFile(newCol, segFile, false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + return rc; + + //@Bug 3164 update compressed extent + updateColumnExtent(newCol.dataFile.pFile, allocSize, /*lbid=*/0); + //..Search first block of new extent for empty rows + rc = readBlock(newCol.dataFile.pFile, buf, newHwm); + + // MCOL-498 add a block. + // DRRTUY TODO Given there is no more hwm pre-allocated we + // could extend the file once to accomodate all records. + if (rc != NO_ERROR) + { + if (rc == ERR_FILE_EOF) + { + setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); + RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); + } + else + { + return rc; + } + } + + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) + { + if (isEmptyRow((uint64_t*)curVal, emptyVal, + column.colWidth)) // Why to check it if beacause line 483 is always true ? + { + rowIdArray[counter] = getRowId(newHwm, column.colWidth, j); + rowsallocated++; + rowsLeft++; + counter++; + + if (rowsallocated >= totalRow) + { + break; + } + } + } + + if (rowsallocated < totalRow) + { + //..Search remaining blks in new extent for empty rows + newHwm++; + + while (((totalRowPerBlock * newHwm) % extentRows) > 0) + { + rc = readBlock(newCol.dataFile.pFile, buf, newHwm); + + // MCOL-498 add a block. + // DRRTUY TODO Given there is no more hwm pre-allocated we + // could extend the file once to accomodate all records. + if (rc != NO_ERROR) + { + if (rc == ERR_FILE_EOF) + { + setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); + RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); } else { - //Find out where the rest rows go - BRM::LBID_t startLbid; - //need to put in a loop until newExtent is true - newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); - TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); - - while (!newExtent) - { - /*partially filled extent encountered due to user moved dbroot. Set hwm to the end of the extent. - If compressed,fill the rest eith empty values. - */ - unsigned int BLKS_PER_EXTENT = 0; - unsigned int nBlks = 0; - unsigned int nRem = 0; - FileOp fileOp; - long long fileSizeBytes = 0; - - for (i = 0; i < dbRootExtentTrackers.size(); i++) - { - uint32_t colNo = column.colNo; - if (i != colNo) - dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); - - // Round up HWM to the end of the current extent - BLKS_PER_EXTENT = (BRMWrapper::getInstance()->getExtentRows() * newColStructList[i].colWidth) / BYTE_PER_BLOCK; - nBlks = newHwm + 1; - nRem = nBlks % BLKS_PER_EXTENT; - - if (nRem > 0) - newHwm = nBlks - nRem + BLKS_PER_EXTENT - 1; - else - newHwm = nBlks - 1; - - //save it to set in the end - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(newColStructList[i].dataOid); - ColExtInfo aExt; - aExt.dbRoot = dbRoot; - aExt.partNum = partition; - aExt.segNum = segment; - aExt.hwm = newHwm; - aExt.isNewExt = false; - aExt.current = false; - aColExtsInfo.push_back(aExt); - - if (newColStructList[i].fCompressionType > 0) - { - string errorInfo; - rc = fileOp.fillCompColumnExtentEmptyChunks( - newColStructList[i].dataOid, - newColStructList[i].colWidth, emptyVal, dbRoot, - partition, segment, - newColStructList[i].colDataType, newHwm, - segFile, errorInfo); - - if (rc != NO_ERROR) - return rc; - } - //@Bug 4758. Check whether this is a abbreviated extent - else if (newColStructList[i].fCompressionType == 0) - { - rc = fileOp.getFileSize(newColStructList[i].dataOid, dbRoot, partition, segment, fileSizeBytes); - - if (rc != NO_ERROR) - return rc; - - if (fileSizeBytes == (long long) INITIAL_EXTENT_ROWS_TO_DISK * newColStructList[i].colWidth) - { - IDBDataFile* pFile = fileOp.openFile( newColStructList[i].dataOid, dbRoot, partition, segment, segFile ); - - if ( !pFile ) - { - rc = ERR_FILE_OPEN; - return rc; - } - - rc = fileOp.expandAbbrevColumnExtent( - pFile, dbRoot, emptyVal, - newColStructList[i].colWidth, - newColStructList[i].colDataType); - - //set hwm for this extent. - fileOp.closeFile(pFile); - - if (rc != NO_ERROR) - return rc; - } - } - - tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); - } - - newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); - } + return rc; } + } - std::vector extents; - - if (newExtent) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) + { + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { - //extend all columns together - std::vector cols; - BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; + rowIdArray[counter] = getRowId(newHwm, newCol.colWidth, j); + rowsallocated++; + rowsLeft++; + counter++; - for (i = 0; i < newColStructList.size(); i++) - { - createStripeColumnExtentsArgIn.oid = newColStructList[i].dataOid; - createStripeColumnExtentsArgIn.width = newColStructList[i].colWidth; - createStripeColumnExtentsArgIn.colDataType = newColStructList[i].colDataType; - cols.push_back(createStripeColumnExtentsArgIn); - } - - rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partition, segment, extents); - newHwm = extents[column.colNo].startBlkOffset; - - if (rc != NO_ERROR) - return rc; - - //Create column files - vector lbids; - vector colDataTypes; - - for ( i = 0; i < extents.size(); i++) - { - setColParam(newCol, 0, newColStructList[i].colWidth, newColStructList[i].colDataType, newColStructList[i].colType, - newColStructList[i].dataOid, newColStructList[i].fCompressionType, dbRoot, partition, segment); - - compressionType(newColStructList[i].fCompressionType); - rc = extendColumn(newCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, - dbRoot, partition, segment, segFile, pFile, newFile); - - if (rc != NO_ERROR) - return rc; - - newColStructList[i].fColPartition = partition; - newColStructList[i].fColSegment = segment; - newColStructList[i].fColDbRoot = dbRoot; - newDctnryStructList[i].fColPartition = partition; - newDctnryStructList[i].fColSegment = segment; - newDctnryStructList[i].fColDbRoot = dbRoot; - lbids.push_back(extents[i].startLbid); - if (newExtents) - { - (*newExtents).push_back(extents[i].startLbid); - } - colDataTypes.push_back(newColStructList[i].colDataType); - } - - //mark the extents to updating - rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); - - if (rc != NO_ERROR) - return rc; - - //create corresponding dictionary files - if (newFile) - { - boost::scoped_ptr we (new WriteEngineWrapper()); - we->setTransId(txnid); - we->setBulkFlag(true); - std::map columnOids; - - for (i = 0; i < newDctnryStructList.size(); i++) - { - if (newDctnryStructList[i].dctnryOid > 0) - { - rc = we->createDctnry(txnid, newDctnryStructList[i].dctnryOid, newDctnryStructList[i].colWidth, dbRoot, partition, - segment, newDctnryStructList[i].fCompressionType); - - if ( rc != NO_ERROR) - return rc; - - columnOids[newDctnryStructList[i].dctnryOid] = newDctnryStructList[i].dctnryOid ; - } - } - - we->flushDataFiles(rc, txnid, columnOids); - } + if (rowsallocated >= totalRow) + { + break; + } } + } - //save the extent info for batch insert - if (isBatchInsert && newExtent) - { - TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); - - for (i = 0; i < newColStructList.size(); i++) - { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(newColStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == newColStructList[i].fColSegment)) - break; - - it++; - } - - ColExtInfo aExt; - aExt.dbRoot = newColStructList[i].fColDbRoot; - aExt.partNum = newColStructList[i].fColPartition; - aExt.segNum = newColStructList[i].fColSegment; - aExt.hwm = extents[i].startBlkOffset; - aExt.isNewExt = true; - aExt.current = true; - aExt.isDict = false; - aColExtsInfo.push_back(aExt); - tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); - } - - for (i = 0; i < newDctnryStructList.size(); i++) - { - if (newDctnryStructList[i].dctnryOid > 0) - { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(newDctnryStructList[i].dctnryOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newDctnryStructList[i].fColDbRoot) && (it->partNum == newDctnryStructList[i].fColPartition) && (it->segNum == newDctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = newDctnryStructList[i].fColDbRoot; - aExt.partNum = newDctnryStructList[i].fColPartition; - aExt.segNum = newDctnryStructList[i].fColSegment; - aExt.compType = newDctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - } - - tableMetaData->setColExtsInfo(newDctnryStructList[i].dctnryOid, aColExtsInfo); - } - } - } - - setColParam(newCol, 0, column.colWidth, column.colDataType, column.colType, - column.dataFile.fid, column.compressionType, dbRoot, partition, segment); - rc = openColumnFile(newCol, segFile, false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - //@Bug 3164 update compressed extent - updateColumnExtent(newCol.dataFile.pFile, allocSize, /*lbid=*/0); - //..Search first block of new extent for empty rows - rc = readBlock(newCol.dataFile.pFile, buf, newHwm); - - // MCOL-498 add a block. - // DRRTUY TODO Given there is no more hwm pre-allocated we - // could extend the file once to accomodate all records. - if (rc != NO_ERROR) - { - if (rc == ERR_FILE_EOF) - { - setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); - RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); - } - else - { - return rc; - } - } - - for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) - { - if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) // Why to check it if beacause line 483 is always true ? - { - rowIdArray[counter] = getRowId(newHwm, column.colWidth, j); - rowsallocated++; - rowsLeft++; - counter++; - - if (rowsallocated >= totalRow) - { - break; - } - } - } - - if (rowsallocated < totalRow) - { - //..Search remaining blks in new extent for empty rows - newHwm++; - - while (((totalRowPerBlock * newHwm) % extentRows) > 0) - { - rc = readBlock(newCol.dataFile.pFile, buf, newHwm); - - // MCOL-498 add a block. - // DRRTUY TODO Given there is no more hwm pre-allocated we - // could extend the file once to accomodate all records. - if (rc != NO_ERROR) - { - if (rc == ERR_FILE_EOF) - { - setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); - RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); - } - else - { - return rc; - } - } - - for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) - { - if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) - { - rowIdArray[counter] = getRowId(newHwm, newCol.colWidth, j); - rowsallocated++; - rowsLeft++; - counter++; - - if (rowsallocated >= totalRow) - { - break; - } - } - } - - if ((rowsallocated < totalRow)) - { - newHwm++; - } - else - break; - } - } + if ((rowsallocated < totalRow)) + { + newHwm++; + } + else + break; } + } } + } - if (rowsallocated < totalRow) - { - return 1; - } + if (rowsallocated < totalRow) + { + return 1; + } - if (!newExtent) - rowsLeft = 0; + if (!newExtent) + rowsLeft = 0; - - return NO_ERROR; + return NO_ERROR; } /*********************************************************** @@ -598,14 +603,14 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, ***********************************************************/ void ColumnOp::clearColumn(Column& column) const { - if (column.dataFile.pFile) - { - column.dataFile.pFile->flush(); - } + if (column.dataFile.pFile) + { + column.dataFile.pFile->flush(); + } - //setColParam(column); - closeColumnFile(column); - setColParam(column); + // setColParam(column); + closeColumnFile(column); + setColParam(column); } /*********************************************************** @@ -618,10 +623,10 @@ void ColumnOp::clearColumn(Column& column) const ***********************************************************/ void ColumnOp::closeColumnFile(Column& column) const { - if (column.dataFile.pFile != NULL) - closeFile(column.dataFile.pFile); + if (column.dataFile.pFile != NULL) + closeFile(column.dataFile.pFile); - column.dataFile.pFile = NULL; + column.dataFile.pFile = NULL; } /*********************************************************** @@ -640,548 +645,540 @@ void ColumnOp::closeColumnFile(Column& column) const * ERR_FILE_EXIST if file exists * ERR_FILE_CREATE if something wrong in creating the file ***********************************************************/ -int ColumnOp::createColumn(Column& column, - int colNo, - int colWidth, - CalpontSystemCatalog::ColDataType colDataType, - ColType colType, - FID dataFid, - uint16_t dbRoot, - uint32_t partition) +int ColumnOp::createColumn(Column& column, int colNo, int colWidth, + CalpontSystemCatalog::ColDataType colDataType, ColType colType, FID dataFid, + uint16_t dbRoot, uint32_t partition) { - int rc, newWidth, allocSize; - int compressionType = column.compressionType; - setColParam(column, colNo, colWidth, colDataType, colType); - const uint8_t* emptyVal = getEmptyRowValue(colDataType, - colWidth); - newWidth = getCorrectRowWidth(colDataType, colWidth); - column.dataFile.fid = dataFid; - column.dataFile.fDbRoot = dbRoot; - column.dataFile.fPartition = partition; - column.dataFile.fSegment = 0; - column.compressionType = compressionType; - rc = createFile(column.dataFile.fid, allocSize, dbRoot, partition, colDataType, emptyVal, newWidth); + int rc, newWidth, allocSize; + int compressionType = column.compressionType; + setColParam(column, colNo, colWidth, colDataType, colType); + const uint8_t* emptyVal = getEmptyRowValue(colDataType, colWidth); + newWidth = getCorrectRowWidth(colDataType, colWidth); + column.dataFile.fid = dataFid; + column.dataFile.fDbRoot = dbRoot; + column.dataFile.fPartition = partition; + column.dataFile.fSegment = 0; + column.compressionType = compressionType; + rc = createFile(column.dataFile.fid, allocSize, dbRoot, partition, colDataType, emptyVal, newWidth); - if (rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; - return NO_ERROR; + return NO_ERROR; } /* BUG931 - * @brief Fills up a column with null/default values in all non-empty rows as the reference column. The reference column - * would typically be another column of the same table. + * @brief Fills up a column with null/default values in all non-empty rows as the reference column. The + * reference column would typically be another column of the same table. * * @param * * @return */ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, void* defaultVal, Dctnry* dctnry, - ColumnOp* refColOp, const OID dictOid, - const int dictColWidth, const string defaultValStr, bool autoincrement) + ColumnOp* refColOp, const OID dictOid, const int dictColWidth, + const string defaultValStr, bool autoincrement) { - unsigned char refColBuf[BYTE_PER_BLOCK]; //Refernce column buffer - unsigned char colBuf[BYTE_PER_BLOCK]; - bool dirty = false; - HWM colHwm = 0; - RID maxRowId = 0; - int size = sizeof(Token); + unsigned char refColBuf[BYTE_PER_BLOCK]; // Refernce column buffer + unsigned char colBuf[BYTE_PER_BLOCK]; + bool dirty = false; + HWM colHwm = 0; + RID maxRowId = 0; + int size = sizeof(Token); - long long startColFbo = 0; - long long startRefColFbo = 0; + long long startColFbo = 0; + long long startRefColFbo = 0; - int refBufOffset = 0; - int colBufOffset = 0; - uint64_t nexValNeeded = 0; - uint64_t nextVal; - uint32_t partition; - uint16_t segment; - HWM lastRefHwm; - int rc = 0; - std::string segFile, errorMsg; - BRM::LBID_t startLbid; - bool newFile = true; - int allocSize = 0; - /* - boost::scoped_ptr dctnry; - if (m_compressionType == 0) - dctnry.reset(new DctnryCompress0); - else - dctnry.reset(new DctnryCompress1); + int refBufOffset = 0; + int colBufOffset = 0; + uint64_t nexValNeeded = 0; + uint64_t nextVal; + uint32_t partition; + uint16_t segment; + HWM lastRefHwm; + int rc = 0; + std::string segFile, errorMsg; + BRM::LBID_t startLbid; + bool newFile = true; + int allocSize = 0; + /* + boost::scoped_ptr dctnry; + if (m_compressionType == 0) + dctnry.reset(new DctnryCompress0); + else + dctnry.reset(new DctnryCompress1); - boost::scoped_ptr refColOp; - if (refCol.compressionType != 0) - refColOp.reset(new ColumnOpCompress1); - else - refColOp.reset(new ColumnOpCompress0); - */ - //get dbroots from config - Config config; - config.initConfigCache(); - std::vector rootList; - config.getRootIdList( rootList ); - const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, - column.colWidth); - // Set TypeHandler to get empty value ptr for the ref column - findTypeHandler(refCol.colWidth, refCol.colDataType); - const uint8_t* refEmptyVal = getEmptyRowValue(refCol.colDataType, - refCol.colWidth); - findTypeHandler(column.colWidth, column.colDataType); - //find the dbroots which have rows for refrence column - unsigned int i = 0, k = 0; + boost::scoped_ptr refColOp; + if (refCol.compressionType != 0) + refColOp.reset(new ColumnOpCompress1); + else + refColOp.reset(new ColumnOpCompress0); + */ + // get dbroots from config + Config config; + config.initConfigCache(); + std::vector rootList; + config.getRootIdList(rootList); + const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + // Set TypeHandler to get empty value ptr for the ref column + findTypeHandler(refCol.colWidth, refCol.colDataType); + const uint8_t* refEmptyVal = getEmptyRowValue(refCol.colDataType, refCol.colWidth); + findTypeHandler(column.colWidth, column.colDataType); + // find the dbroots which have rows for refrence column + unsigned int i = 0, k = 0; - for (i = 0; i < rootList.size(); i++) + for (i = 0; i < rootList.size(); i++) + { + std::vector refEntries; + rc = BRMWrapper::getInstance()->getExtents_dbroot(refCol.dataFile.fid, refEntries, rootList[i]); + std::vector::const_iterator iter = refEntries.begin(); + + while (iter != refEntries.end()) { - std::vector refEntries; - rc = BRMWrapper::getInstance()->getExtents_dbroot(refCol.dataFile.fid, refEntries, rootList[i]); - std::vector::const_iterator iter = refEntries.begin(); + // fill in for the new column for each extent in the reference column + // organize the extents into a file + std::vector fileExtents; + fileExtents.push_back(refEntries[0]); - while ( iter != refEntries.end() ) + // cout << "Get extent for ref oid:dbroot:part:seg = " << + // refCol.dataFile.fid<<":"< fileExtents; - fileExtents.push_back(refEntries[0]); + fileExtents.push_back(refEntries[k]); + } + } - //cout << "Get extent for ref oid:dbroot:part:seg = " << refCol.dataFile.fid<<":"< lastRefHwm) + lastRefHwm = fileExtents[k].HWM; + } + + // create extents for the new column + // If we are processing the first extent in the first segment + // file, we check to see if we have enough rows (256K) to re- + // quire just create the initial abbrev extent for the new column. + std::vector newEntries; + + if ((refEntries.size() == 1) && (refEntries[0].partitionNum == 0) && (refEntries[0].segmentNum == 0)) + { + //@Bug3565 use ref colwidth to calculate. + unsigned int numBlksForFirstExtent = (INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK) * refCol.colWidth; + + if ((lastRefHwm + 1) < numBlksForFirstExtent) + { + rc = createColumn(column, 0, column.colWidth, column.colDataType, WriteEngine::WR_CHAR, + column.dataFile.fid, rootList[i], 0); + + if (rc != NO_ERROR) + return rc; + + // cout << "createColumn for oid " << column.dataFile.fid << endl; + BRM::EMEntry aEntry; + aEntry.partitionNum = partition = 0; + aEntry.segmentNum = segment = 0; + aEntry.dbRoot = rootList[i]; + newEntries.push_back(aEntry); + + if (dictOid > 3000) // Create dictionary file if needed + { + rc = dctnry->createDctnry(dictOid, dictColWidth, rootList[i], partition, segment, startLbid, + newFile); + + if (rc != NO_ERROR) + return rc; + + //@Bug 5652. + std::map oids1; + oids1[dictOid] = dictOid; + dctnry->flushFile(rc, oids1); + dctnry->closeDctnry(); + + // tokenize default value if needed + if (defaultValStr.length() > 0) { - if ((refEntries[0].partitionNum == refEntries[k].partitionNum) && (refEntries[0].segmentNum == refEntries[k].segmentNum)) //already the same dbroot - { - fileExtents.push_back(refEntries[k]); - } + DctnryStruct dctnryStruct; + dctnryStruct.dctnryOid = dictOid; + dctnryStruct.columnOid = column.dataFile.fid; + dctnryStruct.fColPartition = partition; + dctnryStruct.fColSegment = segment; + dctnryStruct.fColDbRoot = rootList[i]; + dctnryStruct.colWidth = dictColWidth; + dctnryStruct.fCompressionType = column.compressionType; + DctnryTuple dctnryTuple; + dctnryTuple.sigValue = (unsigned char*)defaultValStr.c_str(); + dctnryTuple.sigSize = defaultValStr.length(); + + rc = dctnry->openDctnry(dctnryStruct.dctnryOid, dctnryStruct.fColDbRoot, + dctnryStruct.fColPartition, dctnryStruct.fColSegment, + false); // @bug 5572 HDFS tmp file + rc = dctnry->updateDctnry(dctnryTuple.sigValue, dctnryTuple.sigSize, dctnryTuple.token); + + if (dctnryStruct.fCompressionType > 0) + dctnry->closeDctnry(false); + else + dctnry->closeDctnry(true); + + if (rc != NO_ERROR) + return rc; + + memcpy(defaultVal, &dctnryTuple.token, size); + //@Bug 5652. + std::map oids1; + oids1[dictOid] = dictOid; + dctnry->flushFile(rc, oids1); } + } + } + } - //Process this file - lastRefHwm = fileExtents[0].HWM; + if (newEntries.size() == 0) + { + for (k = 0; k < fileExtents.size(); k++) + { + uint16_t dbroot = rootList[i]; + partition = fileExtents[k].partitionNum; + segment = fileExtents[k].segmentNum; - for (k = 1; k < fileExtents.size(); k++) + if (k == 0) + { + rc = addExtent(column, dbroot, partition, segment, segFile, startLbid, newFile, allocSize); + + if (rc != NO_ERROR) + return rc; // Clean up will be done throgh DDLProc + + // cout << "extendColumn for oid " << column.dataFile.fid << endl; + BRM::EMEntry aEntry; + aEntry.partitionNum = partition; + aEntry.segmentNum = segment; + aEntry.dbRoot = rootList[i]; + newEntries.push_back(aEntry); + + if ((dictOid > 3000) && newFile) // Create dictionary file if needed { - //Find the hwm of this file - if (fileExtents[k].HWM > lastRefHwm) - lastRefHwm = fileExtents[k].HWM; - } + rc = dctnry->createDctnry(dictOid, dictColWidth, rootList[i], partition, segment, startLbid, + newFile); - //create extents for the new column - // If we are processing the first extent in the first segment - // file, we check to see if we have enough rows (256K) to re- - // quire just create the initial abbrev extent for the new column. - std::vector newEntries; + if (rc != NO_ERROR) + return rc; - if (( refEntries.size() == 1) && (refEntries[0].partitionNum == 0) && (refEntries[0].segmentNum == 0)) - { - //@Bug3565 use ref colwidth to calculate. - unsigned int numBlksForFirstExtent = - (INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK) * refCol.colWidth; + //@Bug 5652. + std::map oids1; + oids1[dictOid] = dictOid; + dctnry->flushFile(rc, oids1); + dctnry->closeDctnry(); - if ((lastRefHwm + 1) < numBlksForFirstExtent) - { - rc = createColumn(column, 0, column.colWidth, column.colDataType, - WriteEngine::WR_CHAR, column.dataFile.fid, rootList[i], 0); + // tokenize default value if needed + if (defaultValStr.length() > 0) + { + DctnryStruct dctnryStruct; + dctnryStruct.dctnryOid = dictOid; + dctnryStruct.columnOid = column.dataFile.fid; + dctnryStruct.fColPartition = partition; + dctnryStruct.fColSegment = segment; + dctnryStruct.fColDbRoot = rootList[i]; + dctnryStruct.colWidth = dictColWidth; + dctnryStruct.fCompressionType = column.compressionType; + DctnryTuple dctnryTuple; + dctnryTuple.sigValue = (unsigned char*)defaultValStr.c_str(); + // WriteEngineWrapper wrapper; + dctnryTuple.sigSize = defaultValStr.length(); + // rc = wrapper.tokenize(txnid, dctnryStruct, dctnryTuple); + rc = dctnry->openDctnry(dctnryStruct.dctnryOid, dctnryStruct.fColDbRoot, + dctnryStruct.fColPartition, dctnryStruct.fColSegment, + false); // @bug 5572 HDFS tmp file + rc = dctnry->updateDctnry(dctnryTuple.sigValue, dctnryTuple.sigSize, dctnryTuple.token); - if (rc != NO_ERROR) - return rc; - - //cout << "createColumn for oid " << column.dataFile.fid << endl; - BRM::EMEntry aEntry; - aEntry.partitionNum = partition = 0; - aEntry.segmentNum = segment = 0; - aEntry.dbRoot = rootList[i]; - newEntries.push_back(aEntry); - - if (dictOid > 3000) //Create dictionary file if needed - { - rc = dctnry->createDctnry(dictOid, dictColWidth, - rootList[i], partition, segment, startLbid, newFile); - - if (rc != NO_ERROR) - return rc; - - //@Bug 5652. - std::map oids1; - oids1[dictOid] = dictOid; - dctnry->flushFile(rc, oids1); - dctnry->closeDctnry(); - - //tokenize default value if needed - if (defaultValStr.length() > 0) - { - DctnryStruct dctnryStruct; - dctnryStruct.dctnryOid = dictOid; - dctnryStruct.columnOid = column.dataFile.fid; - dctnryStruct.fColPartition = partition; - dctnryStruct.fColSegment = segment; - dctnryStruct.fColDbRoot = rootList[i]; - dctnryStruct.colWidth = dictColWidth; - dctnryStruct.fCompressionType = column.compressionType; - DctnryTuple dctnryTuple; - dctnryTuple.sigValue = (unsigned char*)defaultValStr.c_str(); - dctnryTuple.sigSize = defaultValStr.length(); - - rc = dctnry->openDctnry(dctnryStruct.dctnryOid, - dctnryStruct.fColDbRoot, dctnryStruct.fColPartition, - dctnryStruct.fColSegment, - false); // @bug 5572 HDFS tmp file - rc = dctnry->updateDctnry(dctnryTuple.sigValue, dctnryTuple.sigSize, dctnryTuple.token); - - if (dctnryStruct.fCompressionType > 0) - dctnry->closeDctnry(false); - else - dctnry->closeDctnry(true); - - if (rc != NO_ERROR) - return rc; - - memcpy(defaultVal, &dctnryTuple.token, size); - //@Bug 5652. - std::map oids1; - oids1[dictOid] = dictOid; - dctnry->flushFile(rc, oids1); - } - } - } - } - - if (newEntries.size() == 0) - { - for (k = 0; k < fileExtents.size(); k++) - { - uint16_t dbroot = rootList[i]; - partition = fileExtents[k].partitionNum; - segment = fileExtents[k].segmentNum; - - if ( k == 0) - { - rc = addExtent(column, dbroot, partition, segment, - segFile, startLbid, newFile, allocSize) ; - - if (rc != NO_ERROR) - return rc; //Clean up will be done throgh DDLProc - - //cout << "extendColumn for oid " << column.dataFile.fid << endl; - BRM::EMEntry aEntry; - aEntry.partitionNum = partition; - aEntry.segmentNum = segment; - aEntry.dbRoot = rootList[i]; - newEntries.push_back(aEntry); - - if ((dictOid > 3000) && newFile) //Create dictionary file if needed - { - rc = dctnry->createDctnry(dictOid, dictColWidth, - rootList[i], partition, segment, startLbid, newFile); - - if (rc != NO_ERROR) - return rc; - - //@Bug 5652. - std::map oids1; - oids1[dictOid] = dictOid; - dctnry->flushFile(rc, oids1); - dctnry->closeDctnry(); - - //tokenize default value if needed - if (defaultValStr.length() > 0) - { - DctnryStruct dctnryStruct; - dctnryStruct.dctnryOid = dictOid; - dctnryStruct.columnOid = column.dataFile.fid; - dctnryStruct.fColPartition = partition; - dctnryStruct.fColSegment = segment; - dctnryStruct.fColDbRoot = rootList[i]; - dctnryStruct.colWidth = dictColWidth; - dctnryStruct.fCompressionType = column.compressionType; - DctnryTuple dctnryTuple; - dctnryTuple.sigValue = (unsigned char*)defaultValStr.c_str(); - //WriteEngineWrapper wrapper; - dctnryTuple.sigSize = defaultValStr.length(); - //rc = wrapper.tokenize(txnid, dctnryStruct, dctnryTuple); - rc = dctnry->openDctnry(dctnryStruct.dctnryOid, - dctnryStruct.fColDbRoot, dctnryStruct.fColPartition, - dctnryStruct.fColSegment, - false); // @bug 5572 HDFS tmp file - rc = dctnry->updateDctnry(dctnryTuple.sigValue, dctnryTuple.sigSize, dctnryTuple.token); - - if (dctnryStruct.fCompressionType > 0) - dctnry->closeDctnry(false); - else - dctnry->closeDctnry(true); - - if (rc != NO_ERROR) - return rc; - - memcpy(defaultVal, &dctnryTuple.token, size); - //@Bug 5652. - std::map oids1; - oids1[dictOid] = dictOid; - dctnry->flushFile(rc, oids1); - - } - } - } - else //just add a extent to the file - { - rc = addExtent(column, dbroot, partition, segment, - segFile, startLbid, newFile, allocSize) ; - - if (rc != NO_ERROR) - return rc; //Clean up will be done throgh DDLProc - } - } - } - - //Fill the new file with values - //Open new column file and reference column file - column.dataFile.fDbRoot = rootList[i]; - column.dataFile.fPartition = newEntries[0].partitionNum; - column.dataFile.fSegment = newEntries[0].segmentNum; - RETURN_ON_ERROR(openColumnFile(column, segFile, false)); // @bug 5572 HDFS tmp file - //cout << "Processing new col file " << segFile << endl; - refCol.dataFile.fDbRoot = rootList[i]; - refCol.dataFile.fPartition = newEntries[0].partitionNum; - refCol.dataFile.fSegment = newEntries[0].segmentNum; - std::string segFileRef; - RETURN_ON_ERROR(refColOp->openColumnFile(refCol, segFileRef, false)); // @bug 5572 HDFS tmp file - //cout << "Processing ref file " << segFileRef << " and hwm is " << lastRefHwm << endl; - RETURN_ON_ERROR(refColOp->readBlock(refCol.dataFile.pFile, refColBuf, lastRefHwm)); - - refBufOffset = BYTE_PER_BLOCK - refCol.colWidth; - maxRowId = (lastRefHwm * BYTE_PER_BLOCK) / refCol.colWidth; //Local maxRowId - - while (refBufOffset > 0) - { - if (memcmp(&refColBuf[refBufOffset], refEmptyVal, refCol.colWidth) != 0) - { - maxRowId = maxRowId + (refBufOffset / refCol.colWidth); - break; - } - - refBufOffset -= refCol.colWidth; - } - - //Compute local hwm for the new column - colHwm = (maxRowId * column.colWidth) / BYTE_PER_BLOCK; - //cout << " new col hwm is " << colHwm << endl; - startRefColFbo = 0; - startColFbo = 0; - //Initizliaing to BYTE_PER_BLOCK to force read the first time - refBufOffset = BYTE_PER_BLOCK; - colBufOffset = BYTE_PER_BLOCK; - dirty = false; - ExtCPInfo cpInfo(column.colDataType, column.colWidth); - - if (autoincrement) - { - uint64_t nextValStart = 0; - - while (startRefColFbo <= lastRefHwm || startColFbo <= colHwm) - { - //nexValNeeded = 0; - //cout << "current startRefColFbo:startColFbo:refBufOffset:colBufOffset = " << startRefColFbo <<":"<< startColFbo <<":"< BYTE_PER_BLOCK) - { - //If current reference column block is fully processed get to the next one - //cout << "reading from ref " << endl; - RETURN_ON_ERROR(refColOp->readBlock(refCol.dataFile.pFile, refColBuf, startRefColFbo)); - startRefColFbo++; - refBufOffset = 0; - nexValNeeded = 0; - } - - if ((colBufOffset + column.colWidth) > BYTE_PER_BLOCK) - { - //Current block of the new colum is full. Write it if dirty and then get the next block - if (dirty) - { - //cout << " writing to new col " << endl; - RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, colBuf, startColFbo - 1)); - dirty = false; - } - - //cout << "reading from new col " << endl; - RETURN_ON_ERROR(readBlock(column.dataFile.pFile, colBuf, startColFbo)); - startColFbo++; - colBufOffset = 0; - } - - if (nexValNeeded == 0) - { - int tmpBufOffset = 0; - - while ((tmpBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) - - { - if (memcmp(refColBuf + tmpBufOffset, refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. - { - nexValNeeded++; - //memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); - //dirty = true; - } - - tmpBufOffset += refCol.colWidth; - } - - //reserve the next value, should have a AI sequence in controller from DDLProc - if (nexValNeeded > 0) - { - rc = BRMWrapper::getInstance()->getAutoIncrementRange(column.dataFile.fid, nexValNeeded, nextVal, errorMsg); - - if (rc != NO_ERROR) - return rc; - } - - nextValStart = nextVal; - } - - //write the values to column - - - //colBufOffset = 0; @Bug 5436 ref column coud have different column width - while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && - ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) - { - if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. - { - memcpy(defaultVal, &nextVal, 8); - nextVal++; - memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); - dirty = true; - } - - refBufOffset += refCol.colWidth; - colBufOffset += column.colWidth; - } - } - - if (!cpInfo.isBinaryColumn()) - { - cpInfo.fCPInfo.max = nextValStart + nexValNeeded - 1; - cpInfo.fCPInfo.min = nextValStart; - } + if (dctnryStruct.fCompressionType > 0) + dctnry->closeDctnry(false); else - { - cpInfo.fCPInfo.bigMax = nextValStart + nexValNeeded - 1; - cpInfo.fCPInfo.bigMin = nextValStart; - } - cpInfo.fCPInfo.seqNum = 0; + dctnry->closeDctnry(true); + if (rc != NO_ERROR) + return rc; + + memcpy(defaultVal, &dctnryTuple.token, size); + //@Bug 5652. + std::map oids1; + oids1[dictOid] = dictOid; + dctnry->flushFile(rc, oids1); + } } - else - { - while (startRefColFbo <= lastRefHwm || startColFbo <= colHwm) - { - if ((refBufOffset + refCol.colWidth) > BYTE_PER_BLOCK) - { - //If current reference column block is fully processed get to the next one - RETURN_ON_ERROR(refColOp->readBlock(refCol.dataFile.pFile, refColBuf, startRefColFbo)); - startRefColFbo++; - refBufOffset = 0; - } + } + else // just add a extent to the file + { + rc = addExtent(column, dbroot, partition, segment, segFile, startLbid, newFile, allocSize); - if ((colBufOffset + column.colWidth) > BYTE_PER_BLOCK) - { - //Current block of the new colum is full. Write it if dirty and then get the next block - if (dirty) - { - RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, colBuf, startColFbo - 1)); - dirty = false; - } + if (rc != NO_ERROR) + return rc; // Clean up will be done throgh DDLProc + } + } + } - RETURN_ON_ERROR(readBlock(column.dataFile.pFile, colBuf, startColFbo)); - startColFbo++; - colBufOffset = 0; - } - - while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && - ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) - { - if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) - { - /*if (autoincrement) - { - memcpy(defaultVal, &nextVal, 8); - nextVal++; - } */ - memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); - dirty = true; - } - else if (column.compressionType != 0) //@Bug 3866, fill the empty row value for compressed chunk - { - memcpy(colBuf + colBufOffset, emptyVal, column.colWidth); - dirty = true; - } + // Fill the new file with values + // Open new column file and reference column file + column.dataFile.fDbRoot = rootList[i]; + column.dataFile.fPartition = newEntries[0].partitionNum; + column.dataFile.fSegment = newEntries[0].segmentNum; + RETURN_ON_ERROR(openColumnFile(column, segFile, false)); // @bug 5572 HDFS tmp file + // cout << "Processing new col file " << segFile << endl; + refCol.dataFile.fDbRoot = rootList[i]; + refCol.dataFile.fPartition = newEntries[0].partitionNum; + refCol.dataFile.fSegment = newEntries[0].segmentNum; + std::string segFileRef; + RETURN_ON_ERROR(refColOp->openColumnFile(refCol, segFileRef, false)); // @bug 5572 HDFS tmp file + // cout << "Processing ref file " << segFileRef << " and hwm is " << lastRefHwm << endl; + RETURN_ON_ERROR(refColOp->readBlock(refCol.dataFile.pFile, refColBuf, lastRefHwm)); - refBufOffset += refCol.colWidth; - colBufOffset += column.colWidth; - } - } + refBufOffset = BYTE_PER_BLOCK - refCol.colWidth; + maxRowId = (lastRefHwm * BYTE_PER_BLOCK) / refCol.colWidth; // Local maxRowId - cpInfo.toInvalid(); + while (refBufOffset > 0) + { + if (memcmp(&refColBuf[refBufOffset], refEmptyVal, refCol.colWidth) != 0) + { + maxRowId = maxRowId + (refBufOffset / refCol.colWidth); + break; + } - cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID; - } + refBufOffset -= refCol.colWidth; + } + // Compute local hwm for the new column + colHwm = (maxRowId * column.colWidth) / BYTE_PER_BLOCK; + // cout << " new col hwm is " << colHwm << endl; + startRefColFbo = 0; + startColFbo = 0; + // Initizliaing to BYTE_PER_BLOCK to force read the first time + refBufOffset = BYTE_PER_BLOCK; + colBufOffset = BYTE_PER_BLOCK; + dirty = false; + ExtCPInfo cpInfo(column.colDataType, column.colWidth); + + if (autoincrement) + { + uint64_t nextValStart = 0; + + while (startRefColFbo <= lastRefHwm || startColFbo <= colHwm) + { + // nexValNeeded = 0; + // cout << "current startRefColFbo:startColFbo:refBufOffset:colBufOffset = " << startRefColFbo + // <<":"<< startColFbo <<":"< BYTE_PER_BLOCK) + { + // If current reference column block is fully processed get to the next one + // cout << "reading from ref " << endl; + RETURN_ON_ERROR(refColOp->readBlock(refCol.dataFile.pFile, refColBuf, startRefColFbo)); + startRefColFbo++; + refBufOffset = 0; + nexValNeeded = 0; + } + + if ((colBufOffset + column.colWidth) > BYTE_PER_BLOCK) + { + // Current block of the new colum is full. Write it if dirty and then get the next block if (dirty) { - RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, colBuf, startColFbo - 1)); - dirty = false; + // cout << " writing to new col " << endl; + RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, colBuf, startColFbo - 1)); + dirty = false; } - std::map oids; - oids[column.dataFile.fid] = column.dataFile.fid; - oids[refCol.dataFile.fid] = refCol.dataFile.fid; - rc = flushFile(rc, oids); - closeColumnFile(column); - refColOp->closeColumnFile(refCol); - oids.clear(); + // cout << "reading from new col " << endl; + RETURN_ON_ERROR(readBlock(column.dataFile.pFile, colBuf, startColFbo)); + startColFbo++; + colBufOffset = 0; + } - //Mark extents invalid first - BRM::LBID_t startLbid; - rc = BRMWrapper::getInstance()->getStartLbid(column.dataFile.fid, column.dataFile.fPartition, column.dataFile.fSegment, colHwm, startLbid); + if (nexValNeeded == 0) + { + int tmpBufOffset = 0; + + while ((tmpBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) - if (autoincrement) //@Bug 4074. Mark it invalid first to set later { - ExtCPInfo cpInfo1(column.colDataType, column.colWidth); - cpInfo1.toInvalid(); - cpInfo1.fCPInfo.seqNum = SEQNUM_MARK_INVALID; - cpInfo1.fCPInfo.firstLbid = startLbid; - ExtCPInfoList cpinfoList1; - cpinfoList1.push_back(cpInfo1); - rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList1); + if (memcmp(refColBuf + tmpBufOffset, refEmptyVal, refCol.colWidth) != + 0) // Find the number of nextVal needed. + { + nexValNeeded++; + // memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); + // dirty = true; + } - if ( rc != NO_ERROR) - return rc; + tmpBufOffset += refCol.colWidth; } - ExtCPInfoList cpinfoList; - cpInfo.fCPInfo.firstLbid = startLbid; - cpinfoList.push_back(cpInfo); - rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); - - if ( rc != NO_ERROR) - return rc; - - rc = BRMWrapper::getInstance()->setLocalHWM((OID)column.dataFile.fid, column.dataFile.fPartition, - column.dataFile.fSegment, colHwm); - - if ( rc != NO_ERROR) - return rc; - - //erase the entries from this dbroot. - std::vector refEntriesTrimed; - - for (uint32_t m = 0; m < refEntries.size(); m++) + // reserve the next value, should have a AI sequence in controller from DDLProc + if (nexValNeeded > 0) { - if ((refEntries[0].partitionNum != refEntries[m].partitionNum) || (refEntries[0].segmentNum != refEntries[m].segmentNum)) - refEntriesTrimed.push_back(refEntries[m]); + rc = BRMWrapper::getInstance()->getAutoIncrementRange(column.dataFile.fid, nexValNeeded, + nextVal, errorMsg); + + if (rc != NO_ERROR) + return rc; } - refEntriesTrimed.swap(refEntries); - iter = refEntries.begin(); + nextValStart = nextVal; + } + + // write the values to column + + // colBufOffset = 0; @Bug 5436 ref column coud have different column width + while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && + ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) + { + if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != + 0) // Find the number of nextVal needed. + { + memcpy(defaultVal, &nextVal, 8); + nextVal++; + memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); + dirty = true; + } + + refBufOffset += refCol.colWidth; + colBufOffset += column.colWidth; + } } - } - return rc; + if (!cpInfo.isBinaryColumn()) + { + cpInfo.fCPInfo.max = nextValStart + nexValNeeded - 1; + cpInfo.fCPInfo.min = nextValStart; + } + else + { + cpInfo.fCPInfo.bigMax = nextValStart + nexValNeeded - 1; + cpInfo.fCPInfo.bigMin = nextValStart; + } + cpInfo.fCPInfo.seqNum = 0; + } + else + { + while (startRefColFbo <= lastRefHwm || startColFbo <= colHwm) + { + if ((refBufOffset + refCol.colWidth) > BYTE_PER_BLOCK) + { + // If current reference column block is fully processed get to the next one + RETURN_ON_ERROR(refColOp->readBlock(refCol.dataFile.pFile, refColBuf, startRefColFbo)); + startRefColFbo++; + refBufOffset = 0; + } + + if ((colBufOffset + column.colWidth) > BYTE_PER_BLOCK) + { + // Current block of the new colum is full. Write it if dirty and then get the next block + if (dirty) + { + RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, colBuf, startColFbo - 1)); + dirty = false; + } + + RETURN_ON_ERROR(readBlock(column.dataFile.pFile, colBuf, startColFbo)); + startColFbo++; + colBufOffset = 0; + } + + while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && + ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) + { + if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) + { + /*if (autoincrement) + { + memcpy(defaultVal, &nextVal, 8); + nextVal++; + } */ + memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); + dirty = true; + } + else if (column.compressionType != 0) //@Bug 3866, fill the empty row value for compressed chunk + { + memcpy(colBuf + colBufOffset, emptyVal, column.colWidth); + dirty = true; + } + + refBufOffset += refCol.colWidth; + colBufOffset += column.colWidth; + } + } + + cpInfo.toInvalid(); + + cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID; + } + + if (dirty) + { + RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, colBuf, startColFbo - 1)); + dirty = false; + } + + std::map oids; + oids[column.dataFile.fid] = column.dataFile.fid; + oids[refCol.dataFile.fid] = refCol.dataFile.fid; + rc = flushFile(rc, oids); + closeColumnFile(column); + refColOp->closeColumnFile(refCol); + oids.clear(); + + // Mark extents invalid first + BRM::LBID_t startLbid; + rc = BRMWrapper::getInstance()->getStartLbid(column.dataFile.fid, column.dataFile.fPartition, + column.dataFile.fSegment, colHwm, startLbid); + + if (autoincrement) //@Bug 4074. Mark it invalid first to set later + { + ExtCPInfo cpInfo1(column.colDataType, column.colWidth); + cpInfo1.toInvalid(); + cpInfo1.fCPInfo.seqNum = SEQNUM_MARK_INVALID; + cpInfo1.fCPInfo.firstLbid = startLbid; + ExtCPInfoList cpinfoList1; + cpinfoList1.push_back(cpInfo1); + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList1); + + if (rc != NO_ERROR) + return rc; + } + + ExtCPInfoList cpinfoList; + cpInfo.fCPInfo.firstLbid = startLbid; + cpinfoList.push_back(cpInfo); + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); + + if (rc != NO_ERROR) + return rc; + + rc = BRMWrapper::getInstance()->setLocalHWM((OID)column.dataFile.fid, column.dataFile.fPartition, + column.dataFile.fSegment, colHwm); + + if (rc != NO_ERROR) + return rc; + + // erase the entries from this dbroot. + std::vector refEntriesTrimed; + + for (uint32_t m = 0; m < refEntries.size(); m++) + { + if ((refEntries[0].partitionNum != refEntries[m].partitionNum) || + (refEntries[0].segmentNum != refEntries[m].segmentNum)) + refEntriesTrimed.push_back(refEntries[m]); + } + + refEntriesTrimed.swap(refEntries); + iter = refEntries.begin(); + } + } + + return rc; } /*********************************************************** @@ -1196,11 +1193,10 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi ***********************************************************/ int ColumnOp::createTable(/*const FID tableFid*/) const { -// return createFile(tableFid, DEFAULT_TOTAL_BLOCK ); - return NO_ERROR; + // return createFile(tableFid, DEFAULT_TOTAL_BLOCK ); + return NO_ERROR; } - /*********************************************************** * DESCRIPTION: * Drop column related files @@ -1213,10 +1209,9 @@ int ColumnOp::createTable(/*const FID tableFid*/) const ***********************************************************/ int ColumnOp::dropColumn(const FID dataFid) { - return deleteFile(dataFid); + return deleteFile(dataFid); } - /*********************************************************** * DESCRIPTION: * Drop column and dictionary related files @@ -1228,20 +1223,20 @@ int ColumnOp::dropColumn(const FID dataFid) ***********************************************************/ int ColumnOp::dropFiles(const std::vector& dataFids) { - return deleteFiles(dataFids); + return deleteFiles(dataFids); } int ColumnOp::dropPartitions(const std::vector& dataFids, const std::vector& partitions) { - return deletePartitions(dataFids, partitions); + return deletePartitions(dataFids, partitions); } int ColumnOp::deleteOIDsFromExtentMap(const std::vector& dataFids) { - int rc = 0; - rc = BRMWrapper::getInstance()->deleteOIDsFromExtentMap(dataFids); - return rc; + int rc = 0; + rc = BRMWrapper::getInstance()->deleteOIDsFromExtentMap(dataFids); + return rc; } /************************************************************** @@ -1267,78 +1262,37 @@ int ColumnOp::deleteOIDsFromExtentMap(const std::vector& dataFids) * NO_ERROR if success * other number if fail **************************************************************/ -int ColumnOp::extendColumn( - const Column& column, - bool leaveFileOpen, - HWM hwm, - BRM::LBID_t startLbid, - int allocSize, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - IDBDataFile*& pFile, - bool& newFile, - char* hdrs) +int ColumnOp::extendColumn(const Column& column, bool leaveFileOpen, HWM hwm, BRM::LBID_t startLbid, + int allocSize, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, IDBDataFile*& pFile, bool& newFile, char* hdrs) { - const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, - column.colWidth); - int rc = extendFile(column.dataFile.fid, - emptyVal, - column.colWidth, - column.colDataType, - hwm, - startLbid, - allocSize, - dbRoot, - partition, - segment, - segFile, - pFile, - newFile, - hdrs); + const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + int rc = extendFile(column.dataFile.fid, emptyVal, column.colWidth, column.colDataType, hwm, startLbid, + allocSize, dbRoot, partition, segment, segFile, pFile, newFile, hdrs); - if (rc != NO_ERROR) - { - if ((!leaveFileOpen) && (pFile)) - closeFile( pFile ); - - return rc; - } - - // Only close file for DML/DDL; leave file open for bulkload - if (!leaveFileOpen) - closeFile( pFile ); + if (rc != NO_ERROR) + { + if ((!leaveFileOpen) && (pFile)) + closeFile(pFile); return rc; + } + + // Only close file for DML/DDL; leave file open for bulkload + if (!leaveFileOpen) + closeFile(pFile); + + return rc; } -int ColumnOp::addExtent( - const Column& column, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - BRM::LBID_t& startLbid, - bool& newFile, - int& allocSize, - char* hdrs) +int ColumnOp::addExtent(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, BRM::LBID_t& startLbid, bool& newFile, int& allocSize, + char* hdrs) { - const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, - column.colWidth); - int rc = addExtentExactFile(column.dataFile.fid, - emptyVal, - column.colWidth, - allocSize, - dbRoot, - partition, - segment, - column.colDataType, - segFile, - startLbid, - newFile, - hdrs); - return rc; + const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + int rc = addExtentExactFile(column.dataFile.fid, emptyVal, column.colWidth, allocSize, dbRoot, partition, + segment, column.colDataType, segFile, startLbid, newFile, hdrs); + return rc; } /*********************************************************** @@ -1353,15 +1307,11 @@ int ColumnOp::addExtent( ***********************************************************/ int ColumnOp::expandAbbrevExtent(const Column& column) { - const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, - column.colWidth); - int rc = expandAbbrevColumnExtent(column.dataFile.pFile, - column.dataFile.fDbRoot, - emptyVal, - column.colWidth, - column.colDataType); + const uint8_t* emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + int rc = expandAbbrevColumnExtent(column.dataFile.pFile, column.dataFile.fDbRoot, emptyVal, column.colWidth, + column.colDataType); - return rc; + return rc; } /*********************************************************** @@ -1374,17 +1324,17 @@ int ColumnOp::expandAbbrevExtent(const Column& column) ***********************************************************/ bool ColumnOp::getColDataType(const char* name, CalpontSystemCatalog::ColDataType& colDataType) const { - bool bFound = false; + bool bFound = false; - for (int i = 0; i < CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE; i++) - if (strcmp(name, ColDataTypeStr[i]) == 0) - { - colDataType = static_cast(i); - bFound = true; - break; - } + for (int i = 0; i < CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE; i++) + if (strcmp(name, ColDataTypeStr[i]) == 0) + { + colDataType = static_cast(i); + bFound = true; + break; + } - return bFound; + return bFound; } /*********************************************************** @@ -1397,8 +1347,8 @@ bool ColumnOp::getColDataType(const char* name, CalpontSystemCatalog::ColDataTyp ***********************************************************/ void ColumnOp::initColumn(Column& column) const { - setColParam(column); - column.dataFile.pFile = NULL; + setColParam(column); + column.dataFile.pFile = NULL; } /*********************************************************** @@ -1411,30 +1361,23 @@ void ColumnOp::initColumn(Column& column) const * RETURN: * true if success, false otherwise ***********************************************************/ -inline bool ColumnOp::isEmptyRow(uint64_t* curVal, - const uint8_t* emptyVal, - const int colWidth) +inline bool ColumnOp::isEmptyRow(uint64_t* curVal, const uint8_t* emptyVal, const int colWidth) { - // colWidth is either 1, 2, 4, 8, or 16 (Convertor::getCorrectRowWidth) - switch(colWidth){ - case 1: - return *(uint8_t*)curVal == *(uint8_t*)emptyVal; + // colWidth is either 1, 2, 4, 8, or 16 (Convertor::getCorrectRowWidth) + switch (colWidth) + { + case 1: return *(uint8_t*)curVal == *(uint8_t*)emptyVal; - case 2: - return *(uint16_t*)curVal == *(uint16_t*)emptyVal; - - case 4: - return *(uint32_t*)curVal == *(uint32_t*)emptyVal; - - case 8: - return *(uint64_t*)curVal == *(uint64_t*)emptyVal; - - case 16: - return *(uint128_t*)curVal == *(uint128_t*)emptyVal; - - } + case 2: return *(uint16_t*)curVal == *(uint16_t*)emptyVal; - return false; + case 4: return *(uint32_t*)curVal == *(uint32_t*)emptyVal; + + case 8: return *(uint64_t*)curVal == *(uint64_t*)emptyVal; + + case 16: return *(uint128_t*)curVal == *(uint128_t*)emptyVal; + } + + return false; } /*********************************************************** @@ -1447,7 +1390,7 @@ inline bool ColumnOp::isEmptyRow(uint64_t* curVal, ***********************************************************/ bool ColumnOp::isValid(Column& column) const { - return /*column.colNo > 0 && */ column.colWidth > 0 ; + return /*column.colNo > 0 && */ column.colWidth > 0; } /*********************************************************** @@ -1463,52 +1406,44 @@ bool ColumnOp::isValid(Column& column) const * ERR_FILE_READ if something wrong in reading the file ***********************************************************/ // @bug 5572 - HDFS usage: add *.tmp file backup flag -int ColumnOp::openColumnFile(Column& column, - std::string& segFile, - bool useTmpSuffix, - int ioBuffSize) const +int ColumnOp::openColumnFile(Column& column, std::string& segFile, bool useTmpSuffix, int ioBuffSize) const { - if (!isValid(column)) - return ERR_INVALID_PARAM; + if (!isValid(column)) + return ERR_INVALID_PARAM; - // open column data file - column.dataFile.pFile = openFile(column, - column.dataFile.fDbRoot, - column.dataFile.fPartition, - column.dataFile.fSegment, - column.dataFile.fSegFileName, - useTmpSuffix, - "r+b", ioBuffSize); - segFile = column.dataFile.fSegFileName; + // open column data file + column.dataFile.pFile = + openFile(column, column.dataFile.fDbRoot, column.dataFile.fPartition, column.dataFile.fSegment, + column.dataFile.fSegFileName, useTmpSuffix, "r+b", ioBuffSize); + segFile = column.dataFile.fSegFileName; - if (column.dataFile.pFile == NULL) - { - ostringstream oss; - oss << "oid: " << column.dataFile.fid << " with path " << segFile; - logging::Message::Args args; - logging::Message message(1); - args.add("Error opening file "); - args.add(oss.str()); - args.add(""); - args.add(""); - message.format(args); - logging::LoggingID lid(21); - logging::MessageLog ml(lid); + if (column.dataFile.pFile == NULL) + { + ostringstream oss; + oss << "oid: " << column.dataFile.fid << " with path " << segFile; + logging::Message::Args args; + logging::Message message(1); + args.add("Error opening file "); + args.add(oss.str()); + args.add(""); + args.add(""); + message.format(args); + logging::LoggingID lid(21); + logging::MessageLog ml(lid); - ml.logErrorMessage( message ); - return ERR_FILE_OPEN; + ml.logErrorMessage(message); + return ERR_FILE_OPEN; + } - } - - // open column bitmap file - /* column.bitmapFile.pFile = openFile(column.bitmapFile.fid); - if (column.bitmapFile.pFile == NULL) { - closeFile(column.dataFile.pFile ); // clear previous one - column.dataFile.pFile = NULL; - return ERR_FILE_OPEN; - } - */ - return NO_ERROR; + // open column bitmap file + /* column.bitmapFile.pFile = openFile(column.bitmapFile.fid); + if (column.bitmapFile.pFile == NULL) { + closeFile(column.dataFile.pFile ); // clear previous one + column.dataFile.pFile = NULL; + return ERR_FILE_OPEN; + } + */ + return NO_ERROR; } /*********************************************************** @@ -1537,28 +1472,21 @@ int ColumnOp::openColumnFile(Column& column, * RETURN: * none ***********************************************************/ -void ColumnOp::setColParam(Column& column, - int colNo, - int colWidth, - CalpontSystemCatalog::ColDataType colDataType, - ColType colType, - FID dataFid, - int compressionType, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment) const +void ColumnOp::setColParam(Column& column, int colNo, int colWidth, + CalpontSystemCatalog::ColDataType colDataType, ColType colType, FID dataFid, + int compressionType, uint16_t dbRoot, uint32_t partition, uint16_t segment) const { - column.colNo = colNo; - column.colWidth = colWidth; - column.colType = colType; - column.colDataType = colDataType; - - column.dataFile.fid = dataFid; - column.dataFile.fDbRoot = dbRoot; - column.dataFile.fPartition = partition; - column.dataFile.fSegment = segment; + column.colNo = colNo; + column.colWidth = colWidth; + column.colType = colType; + column.colDataType = colDataType; - column.compressionType = compressionType; + column.dataFile.fid = dataFid; + column.dataFile.fDbRoot = dbRoot; + column.dataFile.fPartition = partition; + column.dataFile.fSegment = segment; + + column.compressionType = compressionType; } /*********************************************************** @@ -1575,151 +1503,162 @@ void ColumnOp::setColParam(Column& column, * RETURN: * NO_ERROR if success, other number otherwise ***********************************************************/ -int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, const void* valArray, void* oldValArray, bool bDelete ) +int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, const void* valArray, + void* oldValArray, bool bDelete) { - uint64_t i = 0, curRowId; - int dataFbo, dataBio, curDataFbo = -1; - unsigned char dataBuf[BYTE_PER_BLOCK]; - bool bExit = false, bDataDirty = false; - const void* pVal = 0; - char charTmpBuf[8]; - int rc = NO_ERROR; - uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; + uint64_t i = 0, curRowId; + int dataFbo, dataBio, curDataFbo = -1; + unsigned char dataBuf[BYTE_PER_BLOCK]; + bool bExit = false, bDataDirty = false; + const void* pVal = 0; + char charTmpBuf[8]; + int rc = NO_ERROR; + uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; - while (!bExit) - { - curRowId = rowIdArray[i]; - - calculateRowId(curRowId, rowsInBlock, curCol.colWidth, dataFbo, dataBio); - - // load another data block if necessary - if (curDataFbo != dataFbo) - { - if (bDataDirty) - { - rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - - if ( rc != NO_ERROR) - return rc; - - bDataDirty = false; - } - - curDataFbo = dataFbo; - rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - - if ( rc != NO_ERROR) - return rc; - - bDataDirty = true; - } - - switch (curCol.colType) - { - case WriteEngine::WR_FLOAT : - if (!bDelete) pVal = &((float*) valArray)[i]; - - break; - - case WriteEngine::WR_DOUBLE : - if (!bDelete) pVal = &((double*) valArray)[i]; - - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - case WriteEngine::WR_CHAR : - if (!bDelete) - { - memcpy(charTmpBuf, (char*)valArray + i * 8, 8); - pVal = charTmpBuf; - } - break; - - case WriteEngine::WR_SHORT : - if (!bDelete) pVal = &((short*) valArray)[i]; - - break; - - case WriteEngine::WR_BYTE : - if (!bDelete) pVal = &((char*) valArray)[i]; - break; - - case WriteEngine::WR_LONGLONG: - if (!bDelete) pVal = &((long long*) valArray)[i]; - break; - - case WriteEngine::WR_TOKEN: - if (!bDelete) pVal = &((Token*) valArray)[i]; - break; - - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - if (!bDelete) pVal = &((int*) valArray)[i]; - break; - - case WriteEngine::WR_USHORT: - if (!bDelete) pVal = &((uint16_t*) valArray)[i]; - break; - - case WriteEngine::WR_UBYTE : - if (!bDelete) pVal = &((uint8_t*) valArray)[i]; - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - if (!bDelete) pVal = &((uint32_t*) valArray)[i]; - break; - - case WriteEngine::WR_ULONGLONG: - if (!bDelete) pVal = &((uint64_t*) valArray)[i]; - break; - - case WriteEngine::WR_BINARY: - if (!bDelete) - { - if (curCol.colWidth == datatypes::MAXDECIMALWIDTH) - pVal = &((int128_t*) valArray)[i]; - } - break; - - default : - if (!bDelete) pVal = &((int*) valArray)[i]; - break; - } - - if (bDelete) - { - pVal = getEmptyRowValue(curCol.colDataType, - curCol.colWidth); - } - - // This is the write stuff - if (oldValArray) - { - uint8_t* p = static_cast(oldValArray); - memcpy(p + curCol.colWidth * i, dataBuf + dataBio, curCol.colWidth); - } - - writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); - - i++; - - if (i >= totalRow) - bExit = true; - } - - // take care of the cleanup - if (bDataDirty && curDataFbo >= 0) + while (!bExit) + { + curRowId = rowIdArray[i]; + + calculateRowId(curRowId, rowsInBlock, curCol.colWidth, dataFbo, dataBio); + + // load another data block if necessary + if (curDataFbo != dataFbo) { + if (bDataDirty) + { rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - if ( rc != NO_ERROR) - return rc; + if (rc != NO_ERROR) + return rc; + bDataDirty = false; + } + + curDataFbo = dataFbo; + rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if (rc != NO_ERROR) + return rc; + + bDataDirty = true; } - return rc; + + switch (curCol.colType) + { + case WriteEngine::WR_FLOAT: + if (!bDelete) + pVal = &((float*)valArray)[i]; + + break; + + case WriteEngine::WR_DOUBLE: + if (!bDelete) + pVal = &((double*)valArray)[i]; + + break; + + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + case WriteEngine::WR_CHAR: + if (!bDelete) + { + memcpy(charTmpBuf, (char*)valArray + i * 8, 8); + pVal = charTmpBuf; + } + break; + + case WriteEngine::WR_SHORT: + if (!bDelete) + pVal = &((short*)valArray)[i]; + + break; + + case WriteEngine::WR_BYTE: + if (!bDelete) + pVal = &((char*)valArray)[i]; + break; + + case WriteEngine::WR_LONGLONG: + if (!bDelete) + pVal = &((long long*)valArray)[i]; + break; + + case WriteEngine::WR_TOKEN: + if (!bDelete) + pVal = &((Token*)valArray)[i]; + break; + + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + if (!bDelete) + pVal = &((int*)valArray)[i]; + break; + + case WriteEngine::WR_USHORT: + if (!bDelete) + pVal = &((uint16_t*)valArray)[i]; + break; + + case WriteEngine::WR_UBYTE: + if (!bDelete) + pVal = &((uint8_t*)valArray)[i]; + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: + if (!bDelete) + pVal = &((uint32_t*)valArray)[i]; + break; + + case WriteEngine::WR_ULONGLONG: + if (!bDelete) + pVal = &((uint64_t*)valArray)[i]; + break; + + case WriteEngine::WR_BINARY: + if (!bDelete) + { + if (curCol.colWidth == datatypes::MAXDECIMALWIDTH) + pVal = &((int128_t*)valArray)[i]; + } + break; + + default: + if (!bDelete) + pVal = &((int*)valArray)[i]; + break; + } + + if (bDelete) + { + pVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + } + + // This is the write stuff + if (oldValArray) + { + uint8_t* p = static_cast(oldValArray); + memcpy(p + curCol.colWidth * i, dataBuf + dataBio, curCol.colWidth); + } + + writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); + + i++; + + if (i >= totalRow) + bExit = true; + } + + // take care of the cleanup + if (bDataDirty && curDataFbo >= 0) + { + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if (rc != NO_ERROR) + return rc; + } + return rc; } /*********************************************************** @@ -1734,292 +1673,241 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, * RETURN: * NO_ERROR if success, other number otherwise ***********************************************************/ -int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, void* oldValArray, bool bDelete ) +int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, + void* oldValArray, bool bDelete) { - uint64_t i = 0, curRowId; - int dataFbo, dataBio, curDataFbo = -1; - unsigned char dataBuf[BYTE_PER_BLOCK]; - bool bExit = false, bDataDirty = false; - const void* pVal = 0; - //void* pOldVal; - char charTmpBuf[8]; - int rc = NO_ERROR; + uint64_t i = 0, curRowId; + int dataFbo, dataBio, curDataFbo = -1; + unsigned char dataBuf[BYTE_PER_BLOCK]; + bool bExit = false, bDataDirty = false; + const void* pVal = 0; + // void* pOldVal; + char charTmpBuf[8]; + int rc = NO_ERROR; - while (!bExit) + while (!bExit) + { + curRowId = ridList[i]; + + calculateRowId(curRowId, BYTE_PER_BLOCK / curCol.colWidth, curCol.colWidth, dataFbo, dataBio); + + // load another data block if necessary + if (curDataFbo != dataFbo) { - curRowId = ridList[i]; - - calculateRowId(curRowId, BYTE_PER_BLOCK / curCol.colWidth, curCol.colWidth, dataFbo, dataBio); - - // load another data block if necessary - if (curDataFbo != dataFbo) - { - if (bDataDirty) - { - rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - - if ( rc != NO_ERROR) - return rc; - - curCol.dataFile.pFile->flush(); - bDataDirty = false; - } - - curDataFbo = dataFbo; - //@Bug 4849. need to check error code to prevent disk error - rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - - if ( rc != NO_ERROR) - return rc; - - bDataDirty = true; - } - - // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that - // How about pVal = valArray? You're always getting the 0'th element here anyways. - // TODO MCOL-641 add support here - // This branch does not seem to be called from anywhere - if (!bDelete) - { - switch (curCol.colType) - { - case WriteEngine::WR_FLOAT : - pVal = &((float*) valArray)[0]; - break; - - case WriteEngine::WR_DOUBLE : - pVal = &((double*) valArray)[0]; - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - case WriteEngine::WR_CHAR : - memcpy(charTmpBuf, (char*)valArray, 8); - pVal = charTmpBuf; - break; - - // case WriteEngine::WR_BIT : pVal = &((bool *) valArray)[i]; break; - case WriteEngine::WR_SHORT : - pVal = &((short*) valArray)[0]; - break; - - case WriteEngine::WR_BYTE : - pVal = &((char*) valArray)[0]; - break; - - case WriteEngine::WR_LONGLONG: - pVal = &((long long*) valArray)[0]; - break; - - case WriteEngine::WR_TOKEN: - pVal = &((Token*) valArray)[0]; - break; - - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - pVal = &((int*) valArray)[0]; - break; - - case WriteEngine::WR_USHORT : - pVal = &((uint16_t*) valArray)[0]; - break; - - case WriteEngine::WR_UBYTE : - pVal = &((uint8_t*) valArray)[0]; - break; - - case WriteEngine::WR_ULONGLONG: - pVal = &((uint64_t*) valArray)[0]; - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - pVal = &((uint32_t*) valArray)[0]; - break; - - default : - pVal = &((int*) valArray)[0]; - break; - } - } - else - { - pVal = getEmptyRowValue(curCol.colDataType, - curCol.colWidth); - } - - // This is the write stuff - if (oldValArray) - { - uint8_t* p = static_cast(oldValArray); - memcpy(p + i * curCol.colWidth, dataBuf + dataBio, curCol.colWidth); - } - - writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); - - i++; - - if (i >= totalRow) - bExit = true; - } - - // take care of the cleanup - if (bDataDirty && curDataFbo >= 0) - { - //@Bug 4849. need to check error code to prevent disk error + if (bDataDirty) + { rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if (rc != NO_ERROR) + return rc; + + curCol.dataFile.pFile->flush(); + bDataDirty = false; + } + + curDataFbo = dataFbo; + //@Bug 4849. need to check error code to prevent disk error + rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if (rc != NO_ERROR) + return rc; + + bDataDirty = true; } - curCol.dataFile.pFile->flush(); + // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that + // How about pVal = valArray? You're always getting the 0'th element here anyways. + // TODO MCOL-641 add support here + // This branch does not seem to be called from anywhere + if (!bDelete) + { + switch (curCol.colType) + { + case WriteEngine::WR_FLOAT: pVal = &((float*)valArray)[0]; break; - return rc; + case WriteEngine::WR_DOUBLE: pVal = &((double*)valArray)[0]; break; + + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + case WriteEngine::WR_CHAR: + memcpy(charTmpBuf, (char*)valArray, 8); + pVal = charTmpBuf; + break; + + // case WriteEngine::WR_BIT : pVal = &((bool *) valArray)[i]; break; + case WriteEngine::WR_SHORT: pVal = &((short*)valArray)[0]; break; + + case WriteEngine::WR_BYTE: pVal = &((char*)valArray)[0]; break; + + case WriteEngine::WR_LONGLONG: pVal = &((long long*)valArray)[0]; break; + + case WriteEngine::WR_TOKEN: pVal = &((Token*)valArray)[0]; break; + + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: pVal = &((int*)valArray)[0]; break; + + case WriteEngine::WR_USHORT: pVal = &((uint16_t*)valArray)[0]; break; + + case WriteEngine::WR_UBYTE: pVal = &((uint8_t*)valArray)[0]; break; + + case WriteEngine::WR_ULONGLONG: pVal = &((uint64_t*)valArray)[0]; break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: pVal = &((uint32_t*)valArray)[0]; break; + + default: pVal = &((int*)valArray)[0]; break; + } + } + else + { + pVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + } + + // This is the write stuff + if (oldValArray) + { + uint8_t* p = static_cast(oldValArray); + memcpy(p + i * curCol.colWidth, dataBuf + dataBio, curCol.colWidth); + } + + writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); + + i++; + + if (i >= totalRow) + bExit = true; + } + + // take care of the cleanup + if (bDataDirty && curDataFbo >= 0) + { + //@Bug 4849. need to check error code to prevent disk error + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + } + + curCol.dataFile.pFile->flush(); + + return rc; } /*********************************************************** - * DESCRIPTION: - * Write rows - * PARAMETERS: - * curCol - column information - * totalRow - the total number of rows need to be inserted - * ridList - the vector of row id - * valArray - the array of one row value - * oldValArray - the array of old value - * RETURN: - * NO_ERROR if success, other number otherwise - ***********************************************************/ -int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, void* oldValArray) + * DESCRIPTION: + * Write rows + * PARAMETERS: + * curCol - column information + * totalRow - the total number of rows need to be inserted + * ridList - the vector of row id + * valArray - the array of one row value + * oldValArray - the array of old value + * RETURN: + * NO_ERROR if success, other number otherwise + ***********************************************************/ +int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, + void* oldValArray) { - uint64_t i = 0, curRowId; - int dataFbo, dataBio, curDataFbo = -1; - unsigned char dataBuf[BYTE_PER_BLOCK]; - bool bExit = false, bDataDirty = false; - void* pVal = 0; - //void* pOldVal; - char charTmpBuf[8]; - int rc = NO_ERROR; + uint64_t i = 0, curRowId; + int dataFbo, dataBio, curDataFbo = -1; + unsigned char dataBuf[BYTE_PER_BLOCK]; + bool bExit = false, bDataDirty = false; + void* pVal = 0; + // void* pOldVal; + char charTmpBuf[8]; + int rc = NO_ERROR; - while (!bExit) - { - curRowId = ridList[i]; - - calculateRowId(curRowId, BYTE_PER_BLOCK / curCol.colWidth, curCol.colWidth, dataFbo, dataBio); - - // load another data block if necessary - if (curDataFbo != dataFbo) - { - if (bDataDirty) - { - rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - - if ( rc != NO_ERROR) - return rc; - - bDataDirty = false; - } - - curDataFbo = dataFbo; - rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - - if ( rc != NO_ERROR) - return rc; - - bDataDirty = true; - } - - // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that - switch (curCol.colType) - { - case WriteEngine::WR_FLOAT : - pVal = &((float*) valArray)[i]; - break; - - case WriteEngine::WR_DOUBLE : - pVal = &((double*) valArray)[i]; - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - case WriteEngine::WR_CHAR : - { - memcpy(charTmpBuf, (char*)valArray + i * 8, 8); - pVal = charTmpBuf; - } - break; - - case WriteEngine::WR_SHORT : - pVal = &((short*) valArray)[i]; - break; - - case WriteEngine::WR_BYTE : - pVal = &((char*) valArray)[i]; - break; - - case WriteEngine::WR_LONGLONG: - pVal = &((long long*) valArray)[i]; - break; - - case WriteEngine::WR_TOKEN: - pVal = &((Token*) valArray)[i]; - break; - - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - pVal = &((int*) valArray)[i]; - break; - - case WriteEngine::WR_USHORT : - pVal = &((uint16_t*) valArray)[i]; - break; - - case WriteEngine::WR_UBYTE : - pVal = &((uint8_t*) valArray)[i]; - break; - - case WriteEngine::WR_ULONGLONG: - pVal = &((uint64_t*) valArray)[i]; - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - pVal = &((uint32_t*) valArray)[i]; - break; - - case WriteEngine::WR_BINARY: - pVal = &((int128_t*) valArray)[i]; - break; - - default : - pVal = &((int*) valArray)[i]; - break; - } - - // This is the write stuff - if (oldValArray) - { - uint8_t* p = static_cast(oldValArray); - memcpy(p + curCol.colWidth * i, dataBuf + dataBio, curCol.colWidth); - } - - writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); - - i++; - - if (i >= totalRow) - bExit = true; - } - - // take care of the cleanup - if (bDataDirty && curDataFbo >= 0) + while (!bExit) + { + curRowId = ridList[i]; + + calculateRowId(curRowId, BYTE_PER_BLOCK / curCol.colWidth, curCol.colWidth, dataFbo, dataBio); + + // load another data block if necessary + if (curDataFbo != dataFbo) { + if (bDataDirty) + { rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if (rc != NO_ERROR) + return rc; + + bDataDirty = false; + } + + curDataFbo = dataFbo; + rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if (rc != NO_ERROR) + return rc; + + bDataDirty = true; } - return rc; + // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that + switch (curCol.colType) + { + case WriteEngine::WR_FLOAT: pVal = &((float*)valArray)[i]; break; + + case WriteEngine::WR_DOUBLE: pVal = &((double*)valArray)[i]; break; + + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + case WriteEngine::WR_CHAR: + { + memcpy(charTmpBuf, (char*)valArray + i * 8, 8); + pVal = charTmpBuf; + } + break; + + case WriteEngine::WR_SHORT: pVal = &((short*)valArray)[i]; break; + + case WriteEngine::WR_BYTE: pVal = &((char*)valArray)[i]; break; + + case WriteEngine::WR_LONGLONG: pVal = &((long long*)valArray)[i]; break; + + case WriteEngine::WR_TOKEN: pVal = &((Token*)valArray)[i]; break; + + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: pVal = &((int*)valArray)[i]; break; + + case WriteEngine::WR_USHORT: pVal = &((uint16_t*)valArray)[i]; break; + + case WriteEngine::WR_UBYTE: pVal = &((uint8_t*)valArray)[i]; break; + + case WriteEngine::WR_ULONGLONG: pVal = &((uint64_t*)valArray)[i]; break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: pVal = &((uint32_t*)valArray)[i]; break; + + case WriteEngine::WR_BINARY: pVal = &((int128_t*)valArray)[i]; break; + + default: pVal = &((int*)valArray)[i]; break; + } + + // This is the write stuff + if (oldValArray) + { + uint8_t* p = static_cast(oldValArray); + memcpy(p + curCol.colWidth * i, dataBuf + dataBio, curCol.colWidth); + } + + writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); + + i++; + + if (i >= totalRow) + bExit = true; + } + + // take care of the cleanup + if (bDataDirty && curDataFbo >= 0) + { + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + } + + return rc; } - -} //end of namespace +} // namespace WriteEngine // vim:ts=4 sw=4: - diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index 87afd1eea..1862981e9 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -17,7 +17,6 @@ // $Id: we_colop.h 4728 2013-08-08 19:26:10Z chao $ - /** @file */ #ifndef _WE_COLOP_H_ @@ -41,311 +40,255 @@ namespace WriteEngine { class Log; -//const int COL_DATATYPE_NUM = 15; -//const char ColDataTypeStr[COL_DATATYPE_NUM][20] = { "bit", "tinyint", "char", "smallint", "decimal", "medint", "integer", "float", "date", "bigint", +// const int COL_DATATYPE_NUM = 15; +// const char ColDataTypeStr[COL_DATATYPE_NUM][20] = { "bit", "tinyint", "char", "smallint", "decimal", +// "medint", "integer", "float", "date", "bigint", // "double", "datetime", "varchar", "clob", "blob" }; /** Class ColumnOp */ class ColumnOp : public DbFileOp { -public: - /** - * @brief Constructor - */ - EXPORT ColumnOp(); - EXPORT ColumnOp(Log* logger); + public: + /** + * @brief Constructor + */ + EXPORT ColumnOp(); + EXPORT ColumnOp(Log* logger); - /** - * @brief Default Destructor - */ - EXPORT virtual ~ColumnOp(); + /** + * @brief Default Destructor + */ + EXPORT virtual ~ColumnOp(); - EXPORT virtual int allocRowId(const TxnID& txnid, - bool useStartingExtent, - Column& column, - uint64_t totalRow, - RID* rowIdArray, - HWM& hwm, - bool& newExtent, - uint64_t& rowsLeft, - HWM& newHwm, - bool& newFile, - ColStructList& newColStructList, - DctnryStructList& newDctnryStructList, - std::vector >& dbRootExtentTrackers, - bool insertSelect, - bool isBatchInsert, - OID tableOid, bool isFirstBatchPm, - std::vector* newEtents); + EXPORT virtual int allocRowId(const TxnID& txnid, bool useStartingExtent, Column& column, uint64_t totalRow, + RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm, + bool& newFile, ColStructList& newColStructList, + DctnryStructList& newDctnryStructList, + std::vector >& dbRootExtentTrackers, + bool insertSelect, bool isBatchInsert, OID tableOid, bool isFirstBatchPm, + std::vector* newEtents); - /** - * @brief Create column file(s) - */ - EXPORT virtual int createColumn(Column& column, - int colNo, - int colWidth, - execplan::CalpontSystemCatalog::ColDataType colDataType, - ColType colType, - FID dataFid, - uint16_t dbRoot, - uint32_t partition); + /** + * @brief Create column file(s) + */ + EXPORT virtual int createColumn(Column& column, int colNo, int colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType, ColType colType, + FID dataFid, uint16_t dbRoot, uint32_t partition); - /** - * @brief Fills up a column with null/default values in all non-empty rows as the reference column. The reference column - * would typically be another column of the same table. - * - * @param column The new colum - * @param refCol The reference column for identifying valid rows - * @param defaultVal The default value of the new column - * @param dictOid The dictionary store OID for a dictionary column - * @param dictColWidth The dictionary string width for a dictionary column - */ - //BUG931 - EXPORT virtual int fillColumn(const TxnID& txnid, - Column& column, - Column& refCol, - void* defaultVal, - Dctnry* dctnry, - ColumnOp* refColOp, - const OID dictOid = 0, - const int dictColWidth = 0, - const std::string defaultValStr = "", - bool autoincrement = false); + /** + * @brief Fills up a column with null/default values in all non-empty rows as the reference column. The + * reference column would typically be another column of the same table. + * + * @param column The new colum + * @param refCol The reference column for identifying valid rows + * @param defaultVal The default value of the new column + * @param dictOid The dictionary store OID for a dictionary column + * @param dictColWidth The dictionary string width for a dictionary column + */ + // BUG931 + EXPORT virtual int fillColumn(const TxnID& txnid, Column& column, Column& refCol, void* defaultVal, + Dctnry* dctnry, ColumnOp* refColOp, const OID dictOid = 0, + const int dictColWidth = 0, const std::string defaultValStr = "", + bool autoincrement = false); - /** - * @brief Create a table file - */ - EXPORT virtual int createTable() const; + /** + * @brief Create a table file + */ + EXPORT virtual int createTable() const; - /** - * @brief Drop column file(s) - */ - EXPORT virtual int dropColumn(const FID dataFid); + /** + * @brief Drop column file(s) + */ + EXPORT virtual int dropColumn(const FID dataFid); - /** - * @brief Drop column and dictionary file(s) - */ - EXPORT virtual int dropFiles(const std::vector& dataFids); + /** + * @brief Drop column and dictionary file(s) + */ + EXPORT virtual int dropFiles(const std::vector& dataFids); - /** - * @brief Delete file(s) for the given partition - */ - EXPORT virtual int dropPartitions(const std::vector& dataFids, - const std::vector& partitions); + /** + * @brief Delete file(s) for the given partition + */ + EXPORT virtual int dropPartitions(const std::vector& dataFids, + const std::vector& partitions); + EXPORT virtual int deleteOIDsFromExtentMap(const std::vector& dataFids); + /** + * @brief Drop a table file + */ + int dropTable() const + { + return NO_ERROR; + } - EXPORT virtual int deleteOIDsFromExtentMap(const std::vector& dataFids); - /** - * @brief Drop a table file - */ - int dropTable() const - { - return NO_ERROR; - } + /** + * @brief Expand the abbreviated extent to a full extent for this column. + */ + EXPORT virtual int expandAbbrevExtent(const Column& column); - /** - * @brief Expand the abbreviated extent to a full extent for this column. - */ - EXPORT virtual int expandAbbrevExtent(const Column& column); + /** + * @brief Add an extent to the specified column OID and DBRoot. + * When this function returns, the file position will be located at the + * end of the file. If the applicable column segment file does not exist, + * extendColumn() will create the new segment file. + * The extent must already exist in the extentmap prior to calling this fctn. + * + * @param column Column struct with input column attributes. + * @param leaveFileOpen Leave the db file open when leaving this function + * @param hwm The fbo of the column segment file where the new extent begins + * @param startLbid The starting LBID for the new extent. + * @param allocSize Number of blocks to be written for an extent + * @param dbRoot The DBRoot of the file with the new extent. + * @param partition Partition num of the file with the new extent. + * @param segment The segment number of the file with the new extent. + * @param segFile (out) Name of segment file to which the extent is added. + * @param pFile (out) FILE ptr to the file where the extent is added. + * @param newFile (out) Indicates if extent was added to new or existing file + * @param hdrs (out) Contents of headers if file is compressed. + * @return returns NO_ERROR if success. + */ + EXPORT int extendColumn(const Column& column, bool leaveFileOpen, HWM hwm, BRM::LBID_t startLbid, + int allocSize, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, IDBDataFile*& pFile, bool& newFile, char* hdrs = NULL); - /** - * @brief Add an extent to the specified column OID and DBRoot. - * When this function returns, the file position will be located at the - * end of the file. If the applicable column segment file does not exist, - * extendColumn() will create the new segment file. - * The extent must already exist in the extentmap prior to calling this fctn. - * - * @param column Column struct with input column attributes. - * @param leaveFileOpen Leave the db file open when leaving this function - * @param hwm The fbo of the column segment file where the new extent begins - * @param startLbid The starting LBID for the new extent. - * @param allocSize Number of blocks to be written for an extent - * @param dbRoot The DBRoot of the file with the new extent. - * @param partition Partition num of the file with the new extent. - * @param segment The segment number of the file with the new extent. - * @param segFile (out) Name of segment file to which the extent is added. - * @param pFile (out) FILE ptr to the file where the extent is added. - * @param newFile (out) Indicates if extent was added to new or existing file - * @param hdrs (out) Contents of headers if file is compressed. - * @return returns NO_ERROR if success. - */ - EXPORT int extendColumn(const Column& column, - bool leaveFileOpen, - HWM hwm, - BRM::LBID_t startLbid, - int allocSize, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - IDBDataFile*& pFile, - bool& newFile, - char* hdrs = NULL); + /** + * @brief Add an extent to the OID specified in the column argument. + * When this function returns, the file position will be located at the + * end of the file. + * + * @param column Column struct with input column attributes. + * @param dbRoot (in) The DBRoot of the file with the new extent. + * @param partition (in) The partition num of the file with the new extent. + * @param segment (in) The segment number of the file with the new extent. + * @param segFile (out) Name of segment file to which the extent is added. + * @param startLbid (out) The starting LBID for the new extent. + * @param newFile (out) Indicates if extent was added to new or existing file + * @param hdsr (out) Contents of headers if file is compressed. + * @param allocSize (out) number of blocks to be written for an extent + * @return returns NO_ERROR if success. + */ + EXPORT int addExtent(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, BRM::LBID_t& startLbid, bool& newFile, int& allocSize, + char* hdrs = NULL); - /** - * @brief Add an extent to the OID specified in the column argument. - * When this function returns, the file position will be located at the - * end of the file. - * - * @param column Column struct with input column attributes. - * @param dbRoot (in) The DBRoot of the file with the new extent. - * @param partition (in) The partition num of the file with the new extent. - * @param segment (in) The segment number of the file with the new extent. - * @param segFile (out) Name of segment file to which the extent is added. - * @param startLbid (out) The starting LBID for the new extent. - * @param newFile (out) Indicates if extent was added to new or existing file - * @param hdsr (out) Contents of headers if file is compressed. - * @param allocSize (out) number of blocks to be written for an extent - * @return returns NO_ERROR if success. - */ - EXPORT int addExtent(const Column& column, - uint16_t dbRoot, - uint32_t partition, - uint16_t segment, - std::string& segFile, - BRM::LBID_t& startLbid, - bool& newFile, - int& allocSize, - char* hdrs = NULL); + /** + * @brief Get columne data type + */ + EXPORT virtual bool getColDataType(const char* name, + execplan::CalpontSystemCatalog::ColDataType& colDataType) const; - /** - * @brief Get columne data type - */ - EXPORT virtual bool getColDataType(const char* name, execplan::CalpontSystemCatalog::ColDataType& colDataType) const; + /** + * @brief Initialize the column + */ + EXPORT virtual void initColumn(Column& column) const; - /** - * @brief Initialize the column - */ - EXPORT virtual void initColumn(Column& column) const; + /** + * @brief Check whether it is an empty row + */ + inline bool isEmptyRow(uint64_t* curVal, const uint8_t* emptyVal, const int colWidth); - /** - * @brief Check whether it is an empty row - */ - inline bool isEmptyRow(uint64_t* curVal, - const uint8_t* emptyVal, - const int colWidth); + /** + * @brief Check whether it is a valid column + */ + EXPORT virtual bool isValid(Column& column) const; - /** - * @brief Check whether it is a valid column - */ - EXPORT virtual bool isValid(Column& column) const; + /** + * @brief Open column file, segFile is set to the name of the column + * segment file that is opened. + */ + EXPORT virtual int openColumnFile(Column& column, std::string& segFile, bool useTmpSuffix, + int ioBuffSize = DEFAULT_BUFSIZ) const; - /** - * @brief Open column file, segFile is set to the name of the column - * segment file that is opened. - */ - EXPORT virtual int openColumnFile(Column& column, - std::string& segFile, - bool useTmpSuffix, - int ioBuffSize = DEFAULT_BUFSIZ) const; + /** + * @brief Open table file + */ + int openTableFile() const + { + return NO_ERROR; + } - /** - * @brief Open table file - */ - int openTableFile() const - { - return NO_ERROR; - } + /** + * @brief Delete a file + */ + EXPORT virtual void setColParam( + Column& column, int colNo = 0, int colWidth = 0, + execplan::CalpontSystemCatalog::ColDataType colDataType = execplan::CalpontSystemCatalog::INT, + ColType colType = WR_INT, FID dataFid = 0, int comppre = 0, uint16_t dbRoot = 0, uint32_t partition = 0, + uint16_t segment = 0) const; - /** - * @brief Delete a file - */ - EXPORT virtual void setColParam(Column& column, int colNo = 0, - int colWidth = 0, - execplan::CalpontSystemCatalog::ColDataType colDataType = execplan::CalpontSystemCatalog::INT, - ColType colType = WR_INT, - FID dataFid = 0, - int comppre = 0, - uint16_t dbRoot = 0, - uint32_t partition = 0, - uint16_t segment = 0) const; + /** + * @brief Write row(s) + */ + EXPORT virtual int writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, const void* valArray, + void* oldValArray = 0, bool bDelete = false); - /** - * @brief Write row(s) - */ - EXPORT virtual int writeRow(Column& curCol, - uint64_t totalRow, - const RID* rowIdArray, - const void* valArray, - void* oldValArray =0, - bool bDelete = false); + /** + * @brief Write row(s) for delete @Bug 1886,2870 + */ + EXPORT virtual int writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridList, + const void* valArray, void* oldValArray = 0, bool bDelete = false); - /** - * @brief Write row(s) for delete @Bug 1886,2870 - */ - EXPORT virtual int writeRows(Column& curCol, - uint64_t totalRow, - const RIDList& ridList, - const void* valArray, - void* oldValArray = 0, - bool bDelete = false); + /** + * @brief Write row(s) for update @Bug 1886,2870 + */ + EXPORT virtual int writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& ridList, + const void* valArray, void* oldValArray = 0); - /** - * @brief Write row(s) for update @Bug 1886,2870 - */ - EXPORT virtual int writeRowsValues(Column& curCol, - uint64_t totalRow, - const RIDList& ridList, - const void* valArray, - void* oldValArray = 0); + /** + * @brief Test if the pFile is an abbreviated extent. + */ + virtual bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const = 0; - /** - * @brief Test if the pFile is an abbreviated extent. - */ - virtual bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const = 0; + /** + * @brief Caculate the number of blocks in file. + */ + virtual int blocksInFile(IDBDataFile* pFile) const = 0; - /** - * @brief Caculate the number of blocks in file. - */ - virtual int blocksInFile(IDBDataFile* pFile) const = 0; + /** + * @brief Clear a column + */ + EXPORT void clearColumn(Column& column) const; - /** - * @brief Clear a column - */ - EXPORT void clearColumn(Column& column) const; + /** + * @brief open a data file of column + */ + virtual IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", + int ioBuffSize = DEFAULT_BUFSIZ) const = 0; - /** - * @brief open a data file of column - */ - virtual IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, - uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const = 0; + /** + * @brief backup blocks to version buffer + */ + int writeVB(IDBDataFile* pSource, const OID sourceOid, IDBDataFile* pTarget, const OID targetOid, + const std::vector& fboList, const BRM::VBRange& freeList, size_t& nBlocksProcessed, + const size_t fboCurrentOffset); + /** + * @brief restore blocks from version buffer + */ + int copyVB(IDBDataFile* pSource, const BRM::VER_t txnD, const OID oid, std::vector& fboList, + std::vector& rangeList); + /** + * @brief close column file + */ + EXPORT virtual void closeColumnFile(Column& column) const; - /** - * @brief backup blocks to version buffer - */ - int writeVB(IDBDataFile* pSource, const OID sourceOid, IDBDataFile* pTarget, const OID targetOid, - const std::vector& fboList, const BRM::VBRange& freeList, - size_t& nBlocksProcessed, const size_t fboCurrentOffset); + protected: + /** + * @brief populate readBuf with data in block #lbid + */ + virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) = 0; - /** - * @brief restore blocks from version buffer - */ - int copyVB(IDBDataFile* pSource, const BRM::VER_t txnD, const OID oid, std::vector& fboList, - std::vector& rangeList); - /** - * @brief close column file - */ - EXPORT virtual void closeColumnFile(Column& column) const; + /** + * @brief output writeBuf to pFile starting at position fbo + */ + virtual int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) = 0; -protected: - - /** - * @brief populate readBuf with data in block #lbid - */ - virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) = 0; - - /** - * @brief output writeBuf to pFile starting at position fbo - */ - virtual int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) = 0; - -private: + private: }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_COLOP_H_ +#endif // _WE_COLOP_H_ diff --git a/writeengine/wrapper/we_colopcompress.cpp b/writeengine/wrapper/we_colopcompress.cpp index cdf186e43..5a47fe37b 100644 --- a/writeengine/wrapper/we_colopcompress.cpp +++ b/writeengine/wrapper/we_colopcompress.cpp @@ -17,7 +17,6 @@ // $Id: we_colopcompress.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ - /** @file */ #include @@ -32,7 +31,6 @@ namespace WriteEngine { - class ChunkManager; // -------------------------------------------------------------------------------------------- @@ -44,74 +42,69 @@ class ChunkManager; */ ColumnOpCompress0::ColumnOpCompress0() { - m_compressionType = 0; + m_compressionType = 0; } - ColumnOpCompress0::ColumnOpCompress0(Log* logger) { - m_compressionType = 0; - setDebugLevel( logger->getDebugLevel() ); - setLogger ( logger ); + m_compressionType = 0; + setDebugLevel(logger->getDebugLevel()); + setLogger(logger); } /** * Default Destructor */ ColumnOpCompress0::~ColumnOpCompress0() -{} - - -// @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* ColumnOpCompress0::openFile( - const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment, - std::string& segFile, bool useTmpSuffix, const char* mode, const int ioBuffSize) const { - return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, - mode, column.colWidth, useTmpSuffix); } +// @bug 5572 - HDFS usage: add *.tmp file backup flag +IDBDataFile* ColumnOpCompress0::openFile(const Column& column, const uint16_t dbRoot, + const uint32_t partition, const uint16_t segment, + std::string& segFile, bool useTmpSuffix, const char* mode, + const int ioBuffSize) const +{ + return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, mode, column.colWidth, + useTmpSuffix); +} bool ColumnOpCompress0::abbreviatedExtent(IDBDataFile* pFile, int colWidth) const { - long long fsize; + long long fsize; - if (getFileSize(pFile, fsize) == NO_ERROR) - { - return (fsize == INITIAL_EXTENT_ROWS_TO_DISK * colWidth); - } + if (getFileSize(pFile, fsize) == NO_ERROR) + { + return (fsize == INITIAL_EXTENT_ROWS_TO_DISK * colWidth); + } - // TODO: Log error - return false; + // TODO: Log error + return false; } - int ColumnOpCompress0::blocksInFile(IDBDataFile* pFile) const { - long long fsize; + long long fsize; - if (getFileSize(pFile, fsize) == NO_ERROR) - { - return (fsize / BYTE_PER_BLOCK); - } + if (getFileSize(pFile, fsize) == NO_ERROR) + { + return (fsize / BYTE_PER_BLOCK); + } - // TODO: Log error - return 0; + // TODO: Log error + return 0; } - int ColumnOpCompress0::readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) { - return readDBFile(pFile, readBuf, fbo, true); + return readDBFile(pFile, readBuf, fbo, true); } - int ColumnOpCompress0::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) { - return writeDBFileFbo(pFile, writeBuf, fbo, 1); + return writeDBFileFbo(pFile, writeBuf, fbo, 1); } - // -------------------------------------------------------------------------------------------- // ColumnOp with compression type 1 // -------------------------------------------------------------------------------------------- @@ -123,16 +116,16 @@ int ColumnOpCompress0::saveBlock(IDBDataFile* pFile, const unsigned char* writeB ColumnOpCompress1::ColumnOpCompress1(uint32_t compressionType, Log* logger) { - m_compressionType = compressionType; - m_chunkManager = new ChunkManager(); + m_compressionType = compressionType; + m_chunkManager = new ChunkManager(); - if (logger) - { - setDebugLevel( logger->getDebugLevel() ); - setLogger ( logger ); - } + if (logger) + { + setDebugLevel(logger->getDebugLevel()); + setLogger(logger); + } - m_chunkManager->fileOp(this); + m_chunkManager->fileOp(this); } /** @@ -140,83 +133,74 @@ ColumnOpCompress1::ColumnOpCompress1(uint32_t compressionType, Log* logger) */ ColumnOpCompress1::~ColumnOpCompress1() { - if (m_chunkManager) - { - delete m_chunkManager; - } + if (m_chunkManager) + { + delete m_chunkManager; + } } // @bug 5572 - HDFS usage: add *.tmp file backup flag -IDBDataFile* ColumnOpCompress1::openFile( - const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment, - std::string& segFile, bool useTmpSuffix, const char* mode, const int ioBuffSize) const +IDBDataFile* ColumnOpCompress1::openFile(const Column& column, const uint16_t dbRoot, + const uint32_t partition, const uint16_t segment, + std::string& segFile, bool useTmpSuffix, const char* mode, + const int ioBuffSize) const { - return m_chunkManager->getFilePtr(column, dbRoot, partition, segment, segFile, - mode, ioBuffSize, useTmpSuffix); + return m_chunkManager->getFilePtr(column, dbRoot, partition, segment, segFile, mode, ioBuffSize, + useTmpSuffix); } - bool ColumnOpCompress1::abbreviatedExtent(IDBDataFile* pFile, int colWidth) const { - return (blocksInFile(pFile) == INITIAL_EXTENT_ROWS_TO_DISK * colWidth / BYTE_PER_BLOCK); + return (blocksInFile(pFile) == INITIAL_EXTENT_ROWS_TO_DISK * colWidth / BYTE_PER_BLOCK); } - int ColumnOpCompress1::blocksInFile(IDBDataFile* pFile) const { - return m_chunkManager->getBlockCount(pFile); + return m_chunkManager->getBlockCount(pFile); } - int ColumnOpCompress1::readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) { - return m_chunkManager->readBlock(pFile, readBuf, fbo); + return m_chunkManager->readBlock(pFile, readBuf, fbo); } - int ColumnOpCompress1::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) { - return m_chunkManager->saveBlock(pFile, writeBuf, fbo); + return m_chunkManager->saveBlock(pFile, writeBuf, fbo); } - int ColumnOpCompress1::flushFile(int rc, std::map& columnOids) { - return m_chunkManager->flushChunks(rc, columnOids); + return m_chunkManager->flushChunks(rc, columnOids); } - -int ColumnOpCompress1::expandAbbrevColumnExtent( - IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, int width, - execplan::CalpontSystemCatalog::ColDataType colDataType ) +int ColumnOpCompress1::expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, + int width, + execplan::CalpontSystemCatalog::ColDataType colDataType) { - // update the uncompressed initial chunk to full chunk - int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width); + // update the uncompressed initial chunk to full chunk + int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width); - // ERR_COMP_FILE_NOT_FOUND is acceptable here. It just means that the - // file hasn't been loaded into the chunk manager yet. No big deal. - if (rc != NO_ERROR && rc != ERR_COMP_FILE_NOT_FOUND) - { - return rc; - } + // ERR_COMP_FILE_NOT_FOUND is acceptable here. It just means that the + // file hasn't been loaded into the chunk manager yet. No big deal. + if (rc != NO_ERROR && rc != ERR_COMP_FILE_NOT_FOUND) + { + return rc; + } - // let the base to physically expand extent. - return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width, - colDataType); + // let the base to physically expand extent. + return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width, colDataType); } - int ColumnOpCompress1::updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { - return m_chunkManager->updateColumnExtent(pFile, nBlocks, lbid); + return m_chunkManager->updateColumnExtent(pFile, nBlocks, lbid); } - void ColumnOpCompress1::closeColumnFile(Column& column) const { - // Leave file closing to chunk manager. - column.dataFile.pFile = NULL; + // Leave file closing to chunk manager. + column.dataFile.pFile = NULL; } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/wrapper/we_colopcompress.h b/writeengine/wrapper/we_colopcompress.h index 33da38646..6a9170fdf 100644 --- a/writeengine/wrapper/we_colopcompress.h +++ b/writeengine/wrapper/we_colopcompress.h @@ -17,7 +17,6 @@ // $Id: we_colopcompress.h 4726 2013-08-07 03:38:36Z bwilkinson $ - /** @file */ #ifndef _WE_COLOP_COMPRESS_H_ @@ -38,160 +37,152 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class ColumnOpCompress0 */ class ColumnOpCompress0 : public ColumnOp { -public: - /** - * @brief Constructor - */ - EXPORT ColumnOpCompress0(); - EXPORT ColumnOpCompress0(Log* logger); + public: + /** + * @brief Constructor + */ + EXPORT ColumnOpCompress0(); + EXPORT ColumnOpCompress0(Log* logger); - /** - * @brief Default Destructor - */ - EXPORT virtual ~ColumnOpCompress0(); + /** + * @brief Default Destructor + */ + EXPORT virtual ~ColumnOpCompress0(); - /** - * @brief virtual method in ColumnOp - */ - IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, - uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const; + /** + * @brief virtual method in ColumnOp + */ + IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", + int ioBuffSize = DEFAULT_BUFSIZ) const; - /** - * @brief virtual method in ColumnOp - */ - bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const; + /** + * @brief virtual method in ColumnOp + */ + bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const; - /** - * @brief virtual method in ColumnOp - */ - int blocksInFile(IDBDataFile* pFile) const; + /** + * @brief virtual method in ColumnOp + */ + int blocksInFile(IDBDataFile* pFile) const; + protected: + /** + * @brief virtual method in ColumnOp + */ + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); -protected: + /** + * @brief virtual method in ColumnOp + */ + int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo); - /** - * @brief virtual method in ColumnOp - */ - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); - - /** - * @brief virtual method in ColumnOp - */ - int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo); - - -private: + private: }; - - /** Class ColumnOpCompress1 */ class ColumnOpCompress1 : public ColumnOp { -public: - /** - * @brief Constructor - */ - EXPORT ColumnOpCompress1(uint32_t compressionType, Log* logger = 0); + public: + /** + * @brief Constructor + */ + EXPORT ColumnOpCompress1(uint32_t compressionType, Log* logger = 0); - /** - * @brief Default Destructor - */ - EXPORT virtual ~ColumnOpCompress1(); + /** + * @brief Default Destructor + */ + EXPORT virtual ~ColumnOpCompress1(); - /** - * @brief virtual method in FileOp - */ - EXPORT int flushFile(int rc, std::map& columnOids); + /** + * @brief virtual method in FileOp + */ + EXPORT int flushFile(int rc, std::map& columnOids); - /** - * @brief virtual method in FileOp - */ - int expandAbbrevColumnExtent( - IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, - int width, execplan::CalpontSystemCatalog::ColDataType colDataType); + /** + * @brief virtual method in FileOp + */ + int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, int width, + execplan::CalpontSystemCatalog::ColDataType colDataType); - /** - * @brief virtual method in ColumnOp - */ - IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, - uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const; + /** + * @brief virtual method in ColumnOp + */ + IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, + std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", + int ioBuffSize = DEFAULT_BUFSIZ) const; - /** - * @brief virtual method in ColumnOp - */ - bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const; + /** + * @brief virtual method in ColumnOp + */ + bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const; - /** - * @brief virtual method in ColumnOp - */ - int blocksInFile(IDBDataFile* pFile) const; + /** + * @brief virtual method in ColumnOp + */ + int blocksInFile(IDBDataFile* pFile) const; -// void chunkManager(ChunkManager* cm); + // void chunkManager(ChunkManager* cm); - /** - * @brief virtual method in FileOp - */ - void setTransId(const TxnID& transId) - { - ColumnOp::setTransId(transId); + /** + * @brief virtual method in FileOp + */ + void setTransId(const TxnID& transId) + { + ColumnOp::setTransId(transId); - if (m_chunkManager) m_chunkManager->setTransId(transId); - } + if (m_chunkManager) + m_chunkManager->setTransId(transId); + } - void setBulkFlag(bool isBulkLoad) - { - m_chunkManager->setBulkFlag(isBulkLoad); - }; + void setBulkFlag(bool isBulkLoad) + { + m_chunkManager->setBulkFlag(isBulkLoad); + }; - void setFixFlag(bool isFix) - { - m_chunkManager->setFixFlag(isFix); - }; + void setFixFlag(bool isFix) + { + m_chunkManager->setFixFlag(isFix); + }; -protected: + protected: + /** + * @brief virtual method in FileOp + */ + int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); - /** - * @brief virtual method in FileOp - */ - int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); + /** + * @brief virtual method in ColumnOp + */ + void closeColumnFile(Column& column) const; - /** - * @brief virtual method in ColumnOp - */ - void closeColumnFile(Column& column) const; + /** + * @brief virtual method in ColumnOp + */ + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); - /** - * @brief virtual method in ColumnOp - */ - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); + /** + * @brief virtual method in ColumnOp + */ + int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo); - /** - * @brief virtual method in ColumnOp - */ - int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo); + /** + * @brief Set the IsInsert flag in the ChunkManager. + * This forces flush at end of statement. Used only for bulk insert. + */ + void setIsInsert(bool isInsert) + { + m_chunkManager->setIsInsert(isInsert); + } - /** - * @brief Set the IsInsert flag in the ChunkManager. - * This forces flush at end of statement. Used only for bulk insert. - */ - void setIsInsert(bool isInsert) - { - m_chunkManager->setIsInsert(isInsert); - } - -private: + private: }; - -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_COLOP_COMPRESS_H_ +#endif // _WE_COLOP_COMPRESS_H_ diff --git a/writeengine/wrapper/we_dctnrycompress.cpp b/writeengine/wrapper/we_dctnrycompress.cpp index e352165ce..1011a5ee8 100644 --- a/writeengine/wrapper/we_dctnrycompress.cpp +++ b/writeengine/wrapper/we_dctnrycompress.cpp @@ -17,7 +17,6 @@ // $Id: we_dctnrycompress.cpp 4726 2013-08-07 03:38:36Z bwilkinson $ - /** @file */ #include @@ -31,7 +30,6 @@ using namespace std; #include "we_dctnrycompress.h" - namespace WriteEngine { class ChunkManager; @@ -44,21 +42,22 @@ class ChunkManager; */ DctnryCompress0::DctnryCompress0() { - m_compressionType = 0; + m_compressionType = 0; } DctnryCompress0::DctnryCompress0(Log* logger) { - m_compressionType = 0; - setDebugLevel( logger->getDebugLevel() ); - setLogger ( logger ); + m_compressionType = 0; + setDebugLevel(logger->getDebugLevel()); + setLogger(logger); } /** * Default Destructor */ DctnryCompress0::~DctnryCompress0() -{} +{ +} // ----------------------------------------------------------------------------- // Dctnry with compression type 1 @@ -69,16 +68,16 @@ DctnryCompress0::~DctnryCompress0() */ DctnryCompress1::DctnryCompress1(uint32_t compressionType, Log* logger) { - m_compressionType = compressionType; - m_chunkManager = new ChunkManager(); + m_compressionType = compressionType; + m_chunkManager = new ChunkManager(); - if (logger) - { - setDebugLevel( logger->getDebugLevel() ); - setLogger ( logger ); - } + if (logger) + { + setDebugLevel(logger->getDebugLevel()); + setLogger(logger); + } - m_chunkManager->fileOp(this); + m_chunkManager->fileOp(this); } /** @@ -86,103 +85,90 @@ DctnryCompress1::DctnryCompress1(uint32_t compressionType, Log* logger) */ DctnryCompress1::~DctnryCompress1() { - if (m_chunkManager) - delete m_chunkManager; + if (m_chunkManager) + delete m_chunkManager; } -int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks, - int64_t lbid) +int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { - return m_chunkManager->updateDctnryExtent(pFile, nBlocks, lbid); + return m_chunkManager->updateDctnryExtent(pFile, nBlocks, lbid); } -IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width, - const char* mode, - int ioBuffSize, +IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize, BRM::LBID_t lbid) { - return m_chunkManager->createDctnryFile(m_dctnryOID, width, m_dbRoot, - m_partition, m_segment, name, mode, - ioBuffSize, lbid); + return m_chunkManager->createDctnryFile(m_dctnryOID, width, m_dbRoot, m_partition, m_segment, name, mode, + ioBuffSize, lbid); } - // @bug 5572 - HDFS usage: add *.tmp file backup flag IDBDataFile* DctnryCompress1::openDctnryFile(bool useTmpSuffix) { - return m_chunkManager->getFilePtr( - m_dctnryOID, m_dbRoot, m_partition, m_segment, m_segFileName, "r+b", DEFAULT_BUFSIZ, useTmpSuffix); + return m_chunkManager->getFilePtr(m_dctnryOID, m_dbRoot, m_partition, m_segment, m_segFileName, "r+b", + DEFAULT_BUFSIZ, useTmpSuffix); } - void DctnryCompress1::closeDctnryFile(bool doFlush, std::map& columnOids) { - if (doFlush) - m_chunkManager->flushChunks(NO_ERROR, columnOids); - else - m_chunkManager->cleanUp(columnOids); + if (doFlush) + m_chunkManager->flushChunks(NO_ERROR, columnOids); + else + m_chunkManager->cleanUp(columnOids); - m_dFile = NULL; + m_dFile = NULL; } - int DctnryCompress1::numOfBlocksInFile() { - return m_chunkManager->getBlockCount(m_dFile); + return m_chunkManager->getBlockCount(m_dFile); } - int DctnryCompress1::readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, const bool isFbo) { - int fbo = lbid; + int fbo = lbid; - if (!isFbo) - RETURN_ON_ERROR(lbidToFbo(lbid, fbo)); + if (!isFbo) + RETURN_ON_ERROR(lbidToFbo(lbid, fbo)); - return m_chunkManager->readBlock(pFile, readBuf, fbo); + return m_chunkManager->readBlock(pFile, readBuf, fbo); } - int DctnryCompress1::writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid, const int numOfBlock) { - int fbo = 0; - RETURN_ON_ERROR(lbidToFbo(lbid, fbo)); + int fbo = 0; + RETURN_ON_ERROR(lbidToFbo(lbid, fbo)); - for (int i = 0; i < numOfBlock; i++) - RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo + i)); + for (int i = 0; i < numOfBlock; i++) + RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo + i)); - return NO_ERROR; + return NO_ERROR; } -int DctnryCompress1::writeDBFileNoVBCache(IDBDataFile* pFile, - const unsigned char* writeBuf, const int fbo, - const int numOfBlock) +int DctnryCompress1::writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo, + const int numOfBlock) { - //int fbo = 0; - //RETURN_ON_ERROR(lbidToFbo(lbid, fbo)); + // int fbo = 0; + // RETURN_ON_ERROR(lbidToFbo(lbid, fbo)); - for (int i = 0; i < numOfBlock; i++) - RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo + i)); + for (int i = 0; i < numOfBlock; i++) + RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo + i)); - return NO_ERROR; + return NO_ERROR; } int DctnryCompress1::flushFile(int rc, std::map& columnOids) { - return m_chunkManager->flushChunks(rc, columnOids); + return m_chunkManager->flushChunks(rc, columnOids); } - int DctnryCompress1::lbidToFbo(const uint64_t lbid, int& fbo) { - uint16_t dbRoot; - uint16_t segment; - uint32_t partition; - return BRMWrapper::getInstance()->getFboOffset(lbid, dbRoot, partition, segment, fbo); + uint16_t dbRoot; + uint16_t segment; + uint32_t partition; + return BRMWrapper::getInstance()->getFboOffset(lbid, dbRoot, partition, segment, fbo); } - -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/wrapper/we_dctnrycompress.h b/writeengine/wrapper/we_dctnrycompress.h index 8ec3ad3ac..e75174eaa 100644 --- a/writeengine/wrapper/we_dctnrycompress.h +++ b/writeengine/wrapper/we_dctnrycompress.h @@ -17,7 +17,6 @@ // $Id: we_dctnrycompress.h 4726 2013-08-07 03:38:36Z bwilkinson $ - /** @file */ #ifndef _WE_DCTNRY_COMPRESS_H_ @@ -36,142 +35,134 @@ /** Namespace WriteEngine */ namespace WriteEngine { - /** Class DctnryCompress */ class DctnryCompress0 : public Dctnry { -public: - /** - * @brief Constructor - */ - EXPORT DctnryCompress0(); - EXPORT DctnryCompress0(Log* logger); + public: + /** + * @brief Constructor + */ + EXPORT DctnryCompress0(); + EXPORT DctnryCompress0(Log* logger); - /** - * @brief Default Destructor - */ - EXPORT virtual ~DctnryCompress0(); + /** + * @brief Default Destructor + */ + EXPORT virtual ~DctnryCompress0(); }; - - /** Class DctnryCompress1 */ class DctnryCompress1 : public Dctnry { -public: - /** - * @brief Constructor - */ - EXPORT DctnryCompress1(uint32_t compressionType, Log* logger = 0); + public: + /** + * @brief Constructor + */ + EXPORT DctnryCompress1(uint32_t compressionType, Log* logger = 0); - /** - * @brief Default Destructor - */ - EXPORT virtual ~DctnryCompress1(); + /** + * @brief Default Destructor + */ + EXPORT virtual ~DctnryCompress1(); - /** - * @brief virtual method in FileOp - */ - EXPORT int flushFile(int rc, std::map& columnOids); + /** + * @brief virtual method in FileOp + */ + EXPORT int flushFile(int rc, std::map& columnOids); - /** - * @brief virtual method in DBFileOp - */ - EXPORT int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, - const bool isFbo = false ); + /** + * @brief virtual method in DBFileOp + */ + EXPORT int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, + const bool isFbo = false); - /** - * @brief virtual method in DBFileOp - */ - EXPORT int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid, - const int numOfBlock = 1); + /** + * @brief virtual method in DBFileOp + */ + EXPORT int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid, + const int numOfBlock = 1); - /** - * @brief virtual method in DBFileOp - */ - EXPORT int writeDBFileNoVBCache(IDBDataFile* pFile, - const unsigned char* writeBuf, const int fbo, - const int numOfBlock = 1); + /** + * @brief virtual method in DBFileOp + */ + EXPORT int writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo, + const int numOfBlock = 1); + /** + * @brief virtual method in Dctnry + */ + IDBDataFile* createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize, int64_t lbid); - /** - * @brief virtual method in Dctnry - */ - IDBDataFile* createDctnryFile(const char* name, int width, - const char* mode, int ioBuffSize, - int64_t lbid); + /** + * @brief virtual method in Dctnry + */ + IDBDataFile* openDctnryFile(bool useTmpSuffix); - /** - * @brief virtual method in Dctnry - */ - IDBDataFile* openDctnryFile(bool useTmpSuffix); + /** + * @brief virtual method in Dctnry + */ + void closeDctnryFile(bool doFlush, std::map& columnOids); - /** - * @brief virtual method in Dctnry - */ - void closeDctnryFile(bool doFlush, std::map& columnOids); + /** + * @brief virtual method in Dctnry + */ + int numOfBlocksInFile(); - /** - * @brief virtual method in Dctnry - */ - int numOfBlocksInFile(); + /** + * @brief For bulkload to use + */ + void setMaxActiveChunkNum(unsigned int maxActiveChunkNum) + { + m_chunkManager->setMaxActiveChunkNum(maxActiveChunkNum); + }; + void setBulkFlag(bool isBulkLoad) + { + m_chunkManager->setBulkFlag(isBulkLoad); + }; + void setFixFlag(bool isFix) + { + m_chunkManager->setFixFlag(isFix); + }; + int checkFixLastDictChunk() + { + return m_chunkManager->checkFixLastDictChunk(m_dctnryOID, m_dbRoot, m_partition, m_segment); + }; + // void chunkManager(ChunkManager* cm); - /** - * @brief For bulkload to use - */ - void setMaxActiveChunkNum(unsigned int maxActiveChunkNum) - { - m_chunkManager->setMaxActiveChunkNum(maxActiveChunkNum); - }; - void setBulkFlag(bool isBulkLoad) - { - m_chunkManager->setBulkFlag(isBulkLoad); - }; - void setFixFlag(bool isFix) - { - m_chunkManager->setFixFlag(isFix); - }; - int checkFixLastDictChunk () - { - return m_chunkManager->checkFixLastDictChunk(m_dctnryOID, m_dbRoot, m_partition, m_segment); - }; -// void chunkManager(ChunkManager* cm); + /** + * @brief virtual method in FileOp + */ + void setTransId(const TxnID& transId) + { + Dctnry::setTransId(transId); - /** - * @brief virtual method in FileOp - */ - void setTransId(const TxnID& transId) - { - Dctnry::setTransId(transId); + if (m_chunkManager) + m_chunkManager->setTransId(transId); + } - if (m_chunkManager) m_chunkManager->setTransId(transId); - } + /** + * @brief Set the IsInsert flag in the ChunkManager. + * This forces flush at end of statement. Used only for bulk insert. + */ + void setIsInsert(bool isInsert) + { + m_chunkManager->setIsInsert(isInsert); + } - /** - * @brief Set the IsInsert flag in the ChunkManager. - * This forces flush at end of statement. Used only for bulk insert. - */ - void setIsInsert(bool isInsert) - { - m_chunkManager->setIsInsert(isInsert); - } + protected: + /** + * @brief virtual method in FileOp + */ + int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); -protected: - - /** - * @brief virtual method in FileOp - */ - int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); - - /** - * @brief convert lbid to fbo - */ - int lbidToFbo(const uint64_t lbid, int& fbo); + /** + * @brief convert lbid to fbo + */ + int lbidToFbo(const uint64_t lbid, int& fbo); }; - -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_DCTNRY_COMPRESS_H_ +#endif // _WE_DCTNRY_COMPRESS_H_ diff --git a/writeengine/wrapper/we_tablemetadata.cpp b/writeengine/wrapper/we_tablemetadata.cpp index 0bc22aa42..64bfff06d 100644 --- a/writeengine/wrapper/we_tablemetadata.cpp +++ b/writeengine/wrapper/we_tablemetadata.cpp @@ -35,34 +35,33 @@ boost::mutex TableMetaData::map_mutex; /*static*/ TableMetaData::TableMetaDataMap TableMetaData::fTableMetaDataMap; - TableMetaData* TableMetaData::makeTableMetaData(uint32_t tableOid) { - boost::mutex::scoped_lock lock(map_mutex); - TableMetaData* instance; - TableMetaDataMap::const_iterator it = fTableMetaDataMap.find(tableOid); + boost::mutex::scoped_lock lock(map_mutex); + TableMetaData* instance; + TableMetaDataMap::const_iterator it = fTableMetaDataMap.find(tableOid); - if (it == fTableMetaDataMap.end()) - { - instance = new TableMetaData(); - fTableMetaDataMap[tableOid] = instance; - return instance; - } + if (it == fTableMetaDataMap.end()) + { + instance = new TableMetaData(); + fTableMetaDataMap[tableOid] = instance; + return instance; + } - return it->second; + return it->second; } /* static */ void TableMetaData::removeTableMetaData(uint32_t tableOid) { - boost::mutex::scoped_lock lock(map_mutex); - TableMetaDataMap::iterator it = fTableMetaDataMap.find(tableOid); + boost::mutex::scoped_lock lock(map_mutex); + TableMetaDataMap::iterator it = fTableMetaDataMap.find(tableOid); - if (it != fTableMetaDataMap.end()) - { - delete (*it).second; - fTableMetaDataMap.erase(it); - } + if (it != fTableMetaDataMap.end()) + { + delete (*it).second; + fTableMetaDataMap.erase(it); + } } TableMetaData::TableMetaData() @@ -72,42 +71,42 @@ TableMetaData::~TableMetaData() { } -ColExtsInfo& TableMetaData::getColExtsInfo (OID columnOid) +ColExtsInfo& TableMetaData::getColExtsInfo(OID columnOid) { - boost::mutex::scoped_lock lock(fColsExtsInfoLock); - ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid); + boost::mutex::scoped_lock lock(fColsExtsInfoLock); + ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid); - if (it != fColsExtsInfoMap.end()) - { - return it->second; - } - else - { - ColExtsInfo colExtsInfo; - fColsExtsInfoMap[columnOid] = colExtsInfo; - return fColsExtsInfoMap[columnOid]; - } + if (it != fColsExtsInfoMap.end()) + { + return it->second; + } + else + { + ColExtsInfo colExtsInfo; + fColsExtsInfoMap[columnOid] = colExtsInfo; + return fColsExtsInfoMap[columnOid]; + } } -void TableMetaData::setColExtsInfo (OID columnOid, ColExtsInfo colExtsInfo) +void TableMetaData::setColExtsInfo(OID columnOid, ColExtsInfo colExtsInfo) { - boost::mutex::scoped_lock lock(fColsExtsInfoLock); - ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid); + boost::mutex::scoped_lock lock(fColsExtsInfoLock); + ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid); - if (it != fColsExtsInfoMap.end()) - { - it->second = colExtsInfo; - } - else - { - fColsExtsInfoMap[columnOid] = colExtsInfo; - } + if (it != fColsExtsInfoMap.end()) + { + it->second = colExtsInfo; + } + else + { + fColsExtsInfoMap[columnOid] = colExtsInfo; + } } ColsExtsInfoMap& TableMetaData::getColsExtsInfoMap() { - boost::mutex::scoped_lock lock(fColsExtsInfoLock); - return fColsExtsInfoMap; + boost::mutex::scoped_lock lock(fColsExtsInfoLock); + return fColsExtsInfoMap; } -} //end of namespace +} // namespace WriteEngine // vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_tablemetadata.h b/writeengine/wrapper/we_tablemetadata.h index 2a6f8a60b..2e4f114a7 100644 --- a/writeengine/wrapper/we_tablemetadata.h +++ b/writeengine/wrapper/we_tablemetadata.h @@ -36,34 +36,34 @@ namespace WriteEngine { - struct ColExtInfo { - uint16_t dbRoot; - uint32_t partNum; - uint16_t segNum; - HWM hwm; - RID lastRid; - int64_t max; - int64_t min; - bool isNewExt; - bool current; - uint16_t compType; - bool isDict; + uint16_t dbRoot; + uint32_t partNum; + uint16_t segNum; + HWM hwm; + RID lastRid; + int64_t max; + int64_t min; + bool isNewExt; + bool current; + uint16_t compType; + bool isDict; - ColExtInfo() : - dbRoot(0), - partNum(0), - segNum(0), - hwm(0), - lastRid(0), - max(0), - min(0), - isNewExt(false), - current(true), - compType(2), - isDict(false) - {} + ColExtInfo() + : dbRoot(0) + , partNum(0) + , segNum(0) + , hwm(0) + , lastRid(0) + , max(0) + , min(0) + , isNewExt(false) + , current(true) + , compType(2) + , isDict(false) + { + } }; typedef std::vector ColExtsInfo; @@ -72,33 +72,32 @@ typedef std::map ColsExtsInfoMap; class TableMetaData { -public: - typedef std::map TableMetaDataMap; - EXPORT static TableMetaData* makeTableMetaData(uint32_t tableOid); - /** remove and delete the instance map to the tableOid - * @param tableOid - */ - EXPORT static void removeTableMetaData(uint32_t tableOid); + public: + typedef std::map TableMetaDataMap; + EXPORT static TableMetaData* makeTableMetaData(uint32_t tableOid); + /** remove and delete the instance map to the tableOid + * @param tableOid + */ + EXPORT static void removeTableMetaData(uint32_t tableOid); - EXPORT ColExtsInfo& getColExtsInfo (OID columnOid); - EXPORT void setColExtsInfo (OID columnOid, ColExtsInfo colExtsInfo); - EXPORT ColsExtsInfoMap& getColsExtsInfoMap(); + EXPORT ColExtsInfo& getColExtsInfo(OID columnOid); + EXPORT void setColExtsInfo(OID columnOid, ColExtsInfo colExtsInfo); + EXPORT ColsExtsInfoMap& getColsExtsInfoMap(); -private: - /** Constuctors */ - explicit TableMetaData(); - explicit TableMetaData(const TableMetaData& rhs); - ~TableMetaData(); - static boost::mutex map_mutex; - static TableMetaDataMap fTableMetaDataMap; - boost::mutex fColsExtsInfoLock; - ColsExtsInfoMap fColsExtsInfoMap; + private: + /** Constuctors */ + explicit TableMetaData(); + explicit TableMetaData(const TableMetaData& rhs); + ~TableMetaData(); + static boost::mutex map_mutex; + static TableMetaDataMap fTableMetaDataMap; + boost::mutex fColsExtsInfoLock; + ColsExtsInfoMap fColsExtsInfoMap; }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT #endif // vim:ts=4 sw=4: - diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index f5c9c431e..815204120 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -67,92 +67,102 @@ using namespace idbdatafile; namespace WriteEngine //#define PROFILE 1 -#define RETURN_ON_ERROR_REPORT(statement) do { int rc = (statement); if (rc != NO_ERROR) { cerr << "failed at " << __LINE__ << "function " << __FUNCTION__ << ", statement '" << #statement << "', rc " << rc << endl; return rc; } } while (0) +#define RETURN_ON_ERROR_REPORT(statement) \ + do \ + { \ + int rc = (statement); \ + if (rc != NO_ERROR) \ + { \ + cerr << "failed at " << __LINE__ << "function " << __FUNCTION__ << ", statement '" << #statement \ + << "', rc " << rc << endl; \ + return rc; \ + } \ + } while (0) { StopWatch timer; using OidToIdxMap = std::unordered_map; /**@brief WriteEngineWrapper Constructor -*/ -WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP) + */ +WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP) { - m_colOp[UN_COMPRESSED_OP] = new ColumnOpCompress0; - m_dctnry[UN_COMPRESSED_OP] = new DctnryCompress0; + m_colOp[UN_COMPRESSED_OP] = new ColumnOpCompress0; + m_dctnry[UN_COMPRESSED_OP] = new DctnryCompress0; - m_colOp[COMPRESSED_OP_1] = new ColumnOpCompress1(/*comressionType=*/1); - m_dctnry[COMPRESSED_OP_1] = new DctnryCompress1(/*compressionType=*/1); + m_colOp[COMPRESSED_OP_1] = new ColumnOpCompress1(/*comressionType=*/1); + m_dctnry[COMPRESSED_OP_1] = new DctnryCompress1(/*compressionType=*/1); - m_colOp[COMPRESSED_OP_2] = new ColumnOpCompress1(/*comressionType=*/3); - m_dctnry[COMPRESSED_OP_2] = new DctnryCompress1(/*compressionType=*/3); + m_colOp[COMPRESSED_OP_2] = new ColumnOpCompress1(/*comressionType=*/3); + m_dctnry[COMPRESSED_OP_2] = new DctnryCompress1(/*compressionType=*/3); } -WriteEngineWrapper::WriteEngineWrapper(const WriteEngineWrapper& rhs) : m_opType(rhs.m_opType) +WriteEngineWrapper::WriteEngineWrapper(const WriteEngineWrapper& rhs) : m_opType(rhs.m_opType) { - m_colOp[UN_COMPRESSED_OP] = new ColumnOpCompress0; - m_dctnry[UN_COMPRESSED_OP] = new DctnryCompress0; + m_colOp[UN_COMPRESSED_OP] = new ColumnOpCompress0; + m_dctnry[UN_COMPRESSED_OP] = new DctnryCompress0; - m_colOp[COMPRESSED_OP_1] = new ColumnOpCompress1(/*compressionType=*/1); - m_dctnry[COMPRESSED_OP_1] = new DctnryCompress1(/*compressionType=*/1); + m_colOp[COMPRESSED_OP_1] = new ColumnOpCompress1(/*compressionType=*/1); + m_dctnry[COMPRESSED_OP_1] = new DctnryCompress1(/*compressionType=*/1); - m_colOp[COMPRESSED_OP_2] = new ColumnOpCompress1(/*compressionType=*/3); - m_dctnry[COMPRESSED_OP_2] = new DctnryCompress1(/*compressionType=*/3); + m_colOp[COMPRESSED_OP_2] = new ColumnOpCompress1(/*compressionType=*/3); + m_dctnry[COMPRESSED_OP_2] = new DctnryCompress1(/*compressionType=*/3); } /**@brief WriteEngineWrapper Constructor -*/ + */ WriteEngineWrapper::~WriteEngineWrapper() { - delete m_colOp[UN_COMPRESSED_OP]; - delete m_dctnry[UN_COMPRESSED_OP]; + delete m_colOp[UN_COMPRESSED_OP]; + delete m_dctnry[UN_COMPRESSED_OP]; - delete m_colOp[COMPRESSED_OP_1]; - delete m_dctnry[COMPRESSED_OP_1]; + delete m_colOp[COMPRESSED_OP_1]; + delete m_dctnry[COMPRESSED_OP_1]; - delete m_colOp[COMPRESSED_OP_2]; - delete m_dctnry[COMPRESSED_OP_2]; + delete m_colOp[COMPRESSED_OP_2]; + delete m_dctnry[COMPRESSED_OP_2]; } /**@brief Perform upfront initialization -*/ + */ /* static */ void WriteEngineWrapper::init(unsigned subSystemID) { - SimpleSysLog::instance()->setLoggingID(logging::LoggingID(subSystemID)); - Config::initConfigCache(); - BRMWrapper::getInstance(); + SimpleSysLog::instance()->setLoggingID(logging::LoggingID(subSystemID)); + Config::initConfigCache(); + BRMWrapper::getInstance(); - // Bug 5415 Add HDFS MemBuffer vs. FileBuffer decision logic. - config::Config* cf = config::Config::makeConfig(); - //-------------------------------------------------------------------------- - // Memory overload protection. This setting will cause the process to die should - // it, by itself, consume maxPct of total memory. Monitored in MonitorProcMem. - // Only used at the express direction of Field Support. - //-------------------------------------------------------------------------- - int maxPct = 0; //disable by default - string strMaxPct = cf->getConfig("WriteEngine", "MaxPct"); + // Bug 5415 Add HDFS MemBuffer vs. FileBuffer decision logic. + config::Config* cf = config::Config::makeConfig(); + //-------------------------------------------------------------------------- + // Memory overload protection. This setting will cause the process to die should + // it, by itself, consume maxPct of total memory. Monitored in MonitorProcMem. + // Only used at the express direction of Field Support. + //-------------------------------------------------------------------------- + int maxPct = 0; // disable by default + string strMaxPct = cf->getConfig("WriteEngine", "MaxPct"); - if ( strMaxPct.length() != 0 ) - maxPct = cf->uFromText(strMaxPct); + if (strMaxPct.length() != 0) + maxPct = cf->uFromText(strMaxPct); - //-------------------------------------------------------------------------- - // MemoryCheckPercent. This controls at what percent of total memory be consumed - // by all processes before we switch from HdfsRdwrMemBuffer to HdfsRdwrFileBuffer. - // This is only used in Hdfs installations. - //-------------------------------------------------------------------------- - int checkPct = 95; - string strCheckPct = cf->getConfig("SystemConfig", "MemoryCheckPercent"); + //-------------------------------------------------------------------------- + // MemoryCheckPercent. This controls at what percent of total memory be consumed + // by all processes before we switch from HdfsRdwrMemBuffer to HdfsRdwrFileBuffer. + // This is only used in Hdfs installations. + //-------------------------------------------------------------------------- + int checkPct = 95; + string strCheckPct = cf->getConfig("SystemConfig", "MemoryCheckPercent"); - if ( strCheckPct.length() != 0 ) - checkPct = cf->uFromText(strCheckPct); + if (strCheckPct.length() != 0) + checkPct = cf->uFromText(strCheckPct); - //-------------------------------------------------------------------------- - // If we're either HDFS, or maxPct is turned on, start the monitor thread. - // Otherwise, we don't need it, so don't waste the resources. - //-------------------------------------------------------------------------- - if (maxPct > 0 || IDBPolicy::useHdfs()) - { - new boost::thread(utils::MonitorProcMem(maxPct, checkPct, subSystemID)); - } + //-------------------------------------------------------------------------- + // If we're either HDFS, or maxPct is turned on, start the monitor thread. + // Otherwise, we don't need it, so don't waste the resources. + //-------------------------------------------------------------------------- + if (maxPct > 0 || IDBPolicy::useHdfs()) + { + new boost::thread(utils::MonitorProcMem(maxPct, checkPct, subSystemID)); + } } /*@brief checkValid --Check input parameters are valid @@ -168,37 +178,37 @@ WriteEngineWrapper::~WriteEngineWrapper() * NO_ERROR if success * others if something wrong in the checking process ***********************************************************/ -int WriteEngineWrapper::checkValid(const TxnID& txnid, const ColStructList& colStructList, const ColValueList& colValueList, const RIDList& ridList) const +int WriteEngineWrapper::checkValid(const TxnID& txnid, const ColStructList& colStructList, + const ColValueList& colValueList, const RIDList& ridList) const { - ColTupleList curTupleList; - ColStructList::size_type structListSize; - ColValueList::size_type valListSize; - ColTupleList::size_type totalRow; + ColTupleList curTupleList; + ColStructList::size_type structListSize; + ColValueList::size_type valListSize; + ColTupleList::size_type totalRow; - if (colStructList.size() == 0) - return ERR_STRUCT_EMPTY; + if (colStructList.size() == 0) + return ERR_STRUCT_EMPTY; - structListSize = colStructList.size() ; - valListSize = colValueList.size(); + structListSize = colStructList.size(); + valListSize = colValueList.size(); - if (structListSize != valListSize) - return ERR_STRUCT_VALUE_NOT_MATCH; + if (structListSize != valListSize) + return ERR_STRUCT_VALUE_NOT_MATCH; - for (ColValueList::size_type i = 0; i < valListSize; i++) + for (ColValueList::size_type i = 0; i < valListSize; i++) + { + curTupleList = static_cast(colValueList[i]); + totalRow = curTupleList.size(); + + if (ridList.size() > 0) { + if (totalRow != ridList.size()) + return ERR_ROWID_VALUE_NOT_MATCH; + } - curTupleList = static_cast(colValueList[i]); - totalRow = curTupleList.size(); + } // end of for (int i = 0; - if (ridList.size() > 0) - { - if (totalRow != ridList.size()) - return ERR_ROWID_VALUE_NOT_MATCH; - } - - } // end of for (int i = 0; - - return NO_ERROR; + return NO_ERROR; } /*@brief findSmallestColumn --Find the smallest column for this table @@ -217,19 +227,19 @@ void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colSt // MCOL-1675: find the smallest column width to calculate the RowID from so // that all HWMs will be incremented by this operation { - int32_t lowColLen = 8192; - for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++) - { - if (colStructList[colIt].colWidth < lowColLen) - { - colId = colIt; - lowColLen = colStructList[colId].colWidth; - if ( lowColLen == 1 ) - { - break; - } - } + int32_t lowColLen = 8192; + for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++) + { + if (colStructList[colIt].colWidth < lowColLen) + { + colId = colIt; + lowColLen = colStructList[colId].colWidth; + if (lowColLen == 1) + { + break; + } } + } } /** @brief Fetch values from arrays into VT-types references casting arrays to (element type) ET-typed arrays. @@ -239,84 +249,90 @@ void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colSt * values written before and deletion does not have values to write to. */ template -void fetchNewOldValues(VT& value, VT& oldValue, const void* array, const void* oldArray, size_t i, size_t totalNewRow) +void fetchNewOldValues(VT& value, VT& oldValue, const void* array, const void* oldArray, size_t i, + size_t totalNewRow) { - const ET* eArray = static_cast(array); - const ET* oldEArray = static_cast(oldArray); - if (eArray) - { - value = eArray[i < totalNewRow ? i : 0]; - } - if (oldEArray) - { - oldValue = oldEArray[i]; - } + const ET* eArray = static_cast(array); + const ET* oldEArray = static_cast(oldArray); + if (eArray) + { + value = eArray[i < totalNewRow ? i : 0]; + } + if (oldEArray) + { + oldValue = oldEArray[i]; + } } -static bool updateBigRangeCheckForInvalidity(ExtCPInfo* maxMin, int128_t value, int128_t oldValue, const void* valArrayVoid, const void* oldValArrayVoid) +static bool updateBigRangeCheckForInvalidity(ExtCPInfo* maxMin, int128_t value, int128_t oldValue, + const void* valArrayVoid, const void* oldValArrayVoid) { - if (!oldValArrayVoid) - { // insertion. we can update range directly. range will not become invalid here. - maxMin->fCPInfo.bigMax = std::max(maxMin->fCPInfo.bigMax, value); // we update big range because int columns can be associated with decimals. - maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, value); + if (!oldValArrayVoid) + { // insertion. we can update range directly. range will not become invalid here. + maxMin->fCPInfo.bigMax = + std::max(maxMin->fCPInfo.bigMax, + value); // we update big range because int columns can be associated with decimals. + maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, value); + } + else if (!valArrayVoid) + { // deletion. we need to check old value only, is it on (or outside) range boundary. + if (oldValue >= maxMin->fCPInfo.bigMax || oldValue <= maxMin->fCPInfo.bigMin) + { + maxMin->toInvalid(); + return true; // no point working further. } - else if (!valArrayVoid) - { // deletion. we need to check old value only, is it on (or outside) range boundary. - if (oldValue >= maxMin->fCPInfo.bigMax || oldValue <= maxMin->fCPInfo.bigMin) - { - maxMin->toInvalid(); - return true; // no point working further. - } + } + else if (valArrayVoid && oldValArrayVoid) + { // update. we need to check boundaries as in deletion and extend as in insertion. + // check boundaries as in deletion accounting for possible extension. + if ((oldValue <= maxMin->fCPInfo.bigMin && value > oldValue) || + (oldValue >= maxMin->fCPInfo.bigMax && value < oldValue)) + { // we are overwriting value on the boundary with value that does not preserve or extend range. + maxMin->toInvalid(); + return true; } - else if (valArrayVoid && oldValArrayVoid) - { // update. we need to check boundaries as in deletion and extend as in insertion. - // check boundaries as in deletion accounting for possible extension. - if ( (oldValue <= maxMin->fCPInfo.bigMin && value > oldValue) - || (oldValue >= maxMin->fCPInfo.bigMax && value < oldValue)) - { // we are overwriting value on the boundary with value that does not preserve or extend range. - maxMin->toInvalid(); - return true; - } - maxMin->fCPInfo.bigMax = std::max(maxMin->fCPInfo.bigMax, value); - maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, value); - } - return false; + maxMin->fCPInfo.bigMax = std::max(maxMin->fCPInfo.bigMax, value); + maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, value); + } + return false; } - -template -bool updateRangeCheckForInvalidity(ExtCPInfo* maxMin, InternalType value, InternalType oldValue, const void* valArrayVoid, const void* oldValArrayVoid) +template +bool updateRangeCheckForInvalidity(ExtCPInfo* maxMin, InternalType value, InternalType oldValue, + const void* valArrayVoid, const void* oldValArrayVoid) { - if (!oldValArrayVoid) - { // insertion. we can update range directly. range will not become invalid here. - maxMin->fCPInfo.bigMax = std::max(maxMin->fCPInfo.bigMax, (int128_t)value); // we update big range because int columns can be associated with decimals. - maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, (int128_t)value); - maxMin->fCPInfo.max = std::max((InternalType)maxMin->fCPInfo.max, value); - maxMin->fCPInfo.min = std::min((InternalType)maxMin->fCPInfo.min, value); + if (!oldValArrayVoid) + { // insertion. we can update range directly. range will not become invalid here. + maxMin->fCPInfo.bigMax = std::max( + maxMin->fCPInfo.bigMax, + (int128_t)value); // we update big range because int columns can be associated with decimals. + maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, (int128_t)value); + maxMin->fCPInfo.max = std::max((InternalType)maxMin->fCPInfo.max, value); + maxMin->fCPInfo.min = std::min((InternalType)maxMin->fCPInfo.min, value); + } + else if (!valArrayVoid) + { // deletion. we need to check old value only, is it on (or outside) range boundary. + if (oldValue >= (InternalType)maxMin->fCPInfo.max || oldValue <= (InternalType)maxMin->fCPInfo.min) + { + maxMin->toInvalid(); + return true; // no point working further. } - else if (!valArrayVoid) - { // deletion. we need to check old value only, is it on (or outside) range boundary. - if (oldValue >= (InternalType)maxMin->fCPInfo.max || oldValue <= (InternalType)maxMin->fCPInfo.min) - { - maxMin->toInvalid(); - return true; // no point working further. - } + } + else if (valArrayVoid && oldValArrayVoid) + { // update. we need to check boundaries as in deletion and extend as in insertion. + // check boundaries as in deletion accounting for possible extension. + if ((oldValue <= (InternalType)maxMin->fCPInfo.min && value > oldValue) || + (oldValue >= (InternalType)maxMin->fCPInfo.max && value < oldValue)) + { // we are overwriting value on the boundary with value that does not preserve or extend range. + maxMin->toInvalid(); + return true; } - else if (valArrayVoid && oldValArrayVoid) - { // update. we need to check boundaries as in deletion and extend as in insertion. - // check boundaries as in deletion accounting for possible extension. - if ( (oldValue <= (InternalType)maxMin->fCPInfo.min && value > oldValue) - || (oldValue >= (InternalType)maxMin->fCPInfo.max && value < oldValue)) - { // we are overwriting value on the boundary with value that does not preserve or extend range. - maxMin->toInvalid(); - return true; - } - maxMin->fCPInfo.bigMax = std::max(maxMin->fCPInfo.bigMax, (int128_t)value); - maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, (int128_t)value); - maxMin->fCPInfo.max = std::max((InternalType)maxMin->fCPInfo.max, value); - maxMin->fCPInfo.min = std::min((InternalType)maxMin->fCPInfo.min, value); - } - return false; + maxMin->fCPInfo.bigMax = std::max(maxMin->fCPInfo.bigMax, (int128_t)value); + maxMin->fCPInfo.bigMin = std::min(maxMin->fCPInfo.bigMin, (int128_t)value); + maxMin->fCPInfo.max = std::max((InternalType)maxMin->fCPInfo.max, value); + maxMin->fCPInfo.min = std::min((InternalType)maxMin->fCPInfo.min, value); + } + return false; } /** @@ -324,149 +340,154 @@ bool updateRangeCheckForInvalidity(ExtCPInfo* maxMin, InternalType value, Intern * both arrays are present (update). */ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_t totalOldRow, - const execplan::CalpontSystemCatalog::ColType& cscColType, - const ColType colType, - const void* valArrayVoid, const void* oldValArrayVoid, - ExtCPInfo* maxMin, bool canStartWithInvalidRange) + const execplan::CalpontSystemCatalog::ColType& cscColType, + const ColType colType, const void* valArrayVoid, + const void* oldValArrayVoid, ExtCPInfo* maxMin, + bool canStartWithInvalidRange) { - if (!maxMin) + if (!maxMin) + { + return; + } + if (colType == WR_CHAR) + { + maxMin->toInvalid(); // simple and wrong solution. + return; + } + bool isUnsigned = false; // TODO: should change with type. + switch (colType) + { + case WR_UBYTE: + case WR_USHORT: + case WR_UINT: + case WR_ULONGLONG: + case WR_CHAR: { - return ; + isUnsigned = true; + break; } - if (colType == WR_CHAR) + default: // WR_BINARY is signed. + break; + } + if (!canStartWithInvalidRange) + { + // check if range is invalid, we can't update it. + if (maxMin->isInvalid()) { - maxMin->toInvalid(); // simple and wrong solution. - return; + return; } - bool isUnsigned = false; // TODO: should change with type. + } + if (colType == WR_CHAR) + { + idbassert(MAX_COLUMN_BOUNDARY == sizeof(uint64_t)); // have to check that - code below depends on that. + if (!maxMin->isInvalid()) + { + maxMin->fromToChars(); + } + } + size_t i; + for (i = 0; i < totalOldRow; i++) + { + int64_t value = 0, oldValue = 0; + uint64_t uvalue = 0, oldUValue = 0; + int128_t bvalue = 0, oldBValue = 0; + + // fetching. May not assign value or oldValue variables when array is not present. switch (colType) { - case WR_UBYTE: - case WR_USHORT: - case WR_UINT: - case WR_ULONGLONG: - case WR_CHAR: - { - isUnsigned = true; - break; - } - default: // WR_BINARY is signed. - break; + case WR_BYTE: + { + fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); + break; + } + case WR_UBYTE: + { + fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, + totalNewRow); + break; + } + case WR_SHORT: + { + fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); + break; + } + case WR_USHORT: + { + fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, + totalNewRow); + break; + } + case WR_MEDINT: + case WR_INT: + { + fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); + break; + } + case WR_UMEDINT: + case WR_UINT: + { + fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, + totalNewRow); + break; + } + case WR_LONGLONG: + { + fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); + break; + } + case WR_ULONGLONG: + { + fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, + totalNewRow); + break; + } + case WR_BINARY: + { + fetchNewOldValues(bvalue, oldBValue, valArrayVoid, oldValArrayVoid, i, + totalNewRow); + break; + } + case WR_CHAR: + { + fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, + totalNewRow); + // for characters (strings, actually), we fetched then in LSB order, on x86, at the very least. + // this means most significant byte of the string, which is first, is now in LSB of uvalue/oldValue. + // we must perform a conversion. + uvalue = uint64ToStr(uvalue); + oldValue = uint64ToStr(oldValue); + break; + } + default: idbassert_s(0, "unknown WR type tag"); return; } - if (!canStartWithInvalidRange) + if (maxMin->isBinaryColumn()) + { // special case of wide decimals. They fit into int128_t range so we do not care about signedness. + if (updateBigRangeCheckForInvalidity(maxMin, bvalue, oldBValue, valArrayVoid, oldValArrayVoid)) + { + return; + } + } + else if (isUnsigned) { - // check if range is invalid, we can't update it. - if (maxMin->isInvalid()) - { - return ; - } + if (updateRangeCheckForInvalidity(maxMin, uvalue, oldUValue, valArrayVoid, oldValArrayVoid)) + { + return; + } } - if (colType == WR_CHAR) + else { - idbassert(MAX_COLUMN_BOUNDARY == sizeof(uint64_t)); // have to check that - code below depends on that. - if (!maxMin->isInvalid()) - { - maxMin->fromToChars(); - } - } - size_t i; - for (i = 0; i < totalOldRow; i++) { - int64_t value = 0, oldValue = 0; - uint64_t uvalue = 0, oldUValue = 0; - int128_t bvalue = 0, oldBValue = 0; - - // fetching. May not assign value or oldValue variables when array is not present. - switch (colType) - { - case WR_BYTE: - { - fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_UBYTE: - { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_SHORT: - { - fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_USHORT: - { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_MEDINT: - case WR_INT: - { - fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_UMEDINT: - case WR_UINT: - { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_LONGLONG: - { - fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_ULONGLONG: - { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_BINARY: - { - fetchNewOldValues(bvalue, oldBValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - break; - } - case WR_CHAR: - { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); - // for characters (strings, actually), we fetched then in LSB order, on x86, at the very least. - // this means most significant byte of the string, which is first, is now in LSB of uvalue/oldValue. - // we must perform a conversion. - uvalue = uint64ToStr(uvalue); - oldValue = uint64ToStr(oldValue); - break; - } - default: - idbassert_s(0, "unknown WR type tag"); - return; - } - if (maxMin->isBinaryColumn()) - { // special case of wide decimals. They fit into int128_t range so we do not care about signedness. - if (updateBigRangeCheckForInvalidity(maxMin, bvalue, oldBValue, valArrayVoid, oldValArrayVoid)) - { - return ; - } - } - else if (isUnsigned) - { - if (updateRangeCheckForInvalidity(maxMin, uvalue, oldUValue, valArrayVoid, oldValArrayVoid)) - { - return ; - } - } - else - { - if (updateRangeCheckForInvalidity(maxMin, value, oldValue, valArrayVoid, oldValArrayVoid)) - { - return ; - } - } - } - // the range will be kept. - // to handle character columns properly we need to convert range values back to strings. - if (colType == WR_CHAR) - { - maxMin->fromToChars(); + if (updateRangeCheckForInvalidity(maxMin, value, oldValue, valArrayVoid, oldValArrayVoid)) + { + return; + } } + } + // the range will be kept. + // to handle character columns properly we need to convert range values back to strings. + if (colType == WR_CHAR) + { + maxMin->fromToChars(); + } } /*@convertValArray - Convert interface values to internal values */ @@ -482,184 +503,188 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ * valArray - output value array * nullArray - output null flag array ***********************************************************/ -void WriteEngineWrapper::convertValArray(const size_t totalRow, const CalpontSystemCatalog::ColType& cscColType, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) +void WriteEngineWrapper::convertValArray(const size_t totalRow, + const CalpontSystemCatalog::ColType& cscColType, + const ColType colType, ColTupleList& curTupleList, void* valArray, + bool bFromList) { - ColTuple curTuple; - ColTupleList::size_type i; + ColTuple curTuple; + ColTupleList::size_type i; - if (bFromList) + if (bFromList) + { + for (i = 0; i < curTupleList.size(); i++) { - for (i = 0; i < curTupleList.size(); i++) - { - curTuple = curTupleList[i]; - convertValue(cscColType, colType, valArray, i, curTuple.data, true); - } - } - else - { - for (i = 0; i < totalRow; i++) - { - convertValue(cscColType, colType, valArray, i, curTuple.data, false); - curTupleList.push_back(curTuple); - } + curTuple = curTupleList[i]; + convertValue(cscColType, colType, valArray, i, curTuple.data, true); } + } + else + { + for (i = 0; i < totalRow; i++) + { + convertValue(cscColType, colType, valArray, i, curTuple.data, false); + curTupleList.push_back(curTuple); + } + } } /* * @brief Convert column value to its internal representation */ -void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, void* value, boost::any& data) +void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, + ColType colType, void* value, boost::any& data) { - string curStr; - int size; + string curStr; + int size; - switch (colType) + switch (colType) + { + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + if (data.type() == typeid(int)) + { + int val = boost::any_cast(data); + size = sizeof(int); + memcpy(value, &val, size); + } + else + { + uint32_t val = boost::any_cast(data); + size = sizeof(uint32_t); + memcpy(value, &val, size); + } + + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: { - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - if (data.type() == typeid(int)) - { - int val = boost::any_cast(data); - size = sizeof(int); - memcpy(value, &val, size); - } - else - { - uint32_t val = boost::any_cast(data); - size = sizeof(uint32_t); - memcpy(value, &val, size); - } + uint32_t val = boost::any_cast(data); + size = sizeof(uint32_t); + memcpy(value, &val, size); + } + break; - break; + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + curStr = boost::any_cast(data); - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - { - uint32_t val = boost::any_cast(data); - size = sizeof(uint32_t); - memcpy(value, &val, size); - } - break; + if ((int)curStr.length() > MAX_COLUMN_BOUNDARY) + curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR : - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - curStr = boost::any_cast(data); + memcpy(value, curStr.c_str(), curStr.length()); + break; - if ((int) curStr.length() > MAX_COLUMN_BOUNDARY) - curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); + case WriteEngine::WR_FLOAT: + { + float val = boost::any_cast(data); - memcpy(value, curStr.c_str(), curStr.length()); - break; + // N.B.There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, + // but not necessarily the same bits that you put in. This only seems to be for float (double seems + // to work). + if (isnan(val)) + { + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + val = *tfp; + } - case WriteEngine::WR_FLOAT: - { - float val = boost::any_cast(data); + size = sizeof(float); + memcpy(value, &val, size); + } + break; -//N.B.There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, -// but not necessarily the same bits that you put in. This only seems to be for float (double seems -// to work). - if (isnan(val)) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - val = *tfp; - } + case WriteEngine::WR_DOUBLE: + { + double val = boost::any_cast(data); + size = sizeof(double); + memcpy(value, &val, size); + } + break; - size = sizeof(float); - memcpy(value, &val, size); - } - break; + case WriteEngine::WR_SHORT: + { + short val = boost::any_cast(data); + size = sizeof(short); + memcpy(value, &val, size); + } + break; - case WriteEngine::WR_DOUBLE: - { - double val = boost::any_cast(data); - size = sizeof(double); - memcpy(value, &val, size); - } - break; + case WriteEngine::WR_USHORT: + { + uint16_t val = boost::any_cast(data); + size = sizeof(uint16_t); + memcpy(value, &val, size); + } + break; - case WriteEngine::WR_SHORT: - { - short val = boost::any_cast(data); - size = sizeof(short); - memcpy(value, &val, size); - } - break; + case WriteEngine::WR_BYTE: + { + char val = boost::any_cast(data); + size = sizeof(char); + memcpy(value, &val, size); + } + break; - case WriteEngine::WR_USHORT: - { - uint16_t val = boost::any_cast(data); - size = sizeof(uint16_t); - memcpy(value, &val, size); - } - break; + case WriteEngine::WR_UBYTE: + { + uint8_t val = boost::any_cast(data); + size = sizeof(uint8_t); + memcpy(value, &val, size); + } + break; - case WriteEngine::WR_BYTE: - { - char val = boost::any_cast(data); - size = sizeof(char); - memcpy(value, &val, size); - } - break; + case WriteEngine::WR_LONGLONG: + if (data.type() == typeid(long long)) + { + long long val = boost::any_cast(data); + size = sizeof(long long); + memcpy(value, &val, size); + } + else if (data.type() == typeid(long)) + { + long val = boost::any_cast(data); + size = sizeof(long); + memcpy(value, &val, size); + } + else + { + uint64_t val = boost::any_cast(data); + size = sizeof(uint64_t); + memcpy(value, &val, size); + } - case WriteEngine::WR_UBYTE: - { - uint8_t val = boost::any_cast(data); - size = sizeof(uint8_t); - memcpy(value, &val, size); - } - break; + break; - case WriteEngine::WR_LONGLONG: - if (data.type() == typeid(long long)) - { - long long val = boost::any_cast(data); - size = sizeof(long long); - memcpy(value, &val, size); - } - else if (data.type() == typeid(long)) - { - long val = boost::any_cast(data); - size = sizeof(long); - memcpy(value, &val, size); - } - else - { - uint64_t val = boost::any_cast(data); - size = sizeof(uint64_t); - memcpy(value, &val, size); - } + case WriteEngine::WR_ULONGLONG: + { + uint64_t val = boost::any_cast(data); + size = sizeof(uint64_t); + memcpy(value, &val, size); + } + break; - break; + case WriteEngine::WR_TOKEN: + { + Token val = boost::any_cast(data); + size = sizeof(Token); + memcpy(value, &val, size); + } + break; - case WriteEngine::WR_ULONGLONG: - { - uint64_t val = boost::any_cast(data); - size = sizeof(uint64_t); - memcpy(value, &val, size); - } - break; + case WriteEngine::WR_BINARY: + { + size = cscColType.colWidth; + int128_t val = boost::any_cast(data); + memcpy(value, &val, size); + } + break; - case WriteEngine::WR_TOKEN: - { - Token val = boost::any_cast(data); - size = sizeof(Token); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_BINARY: - { - size = cscColType.colWidth; - int128_t val = boost::any_cast(data); - memcpy(value, &val, size); - } - break; - - } // end of switch (colType) -} /*@convertValue - The base for converting values */ + } // end of switch (colType) +} /*@convertValue - The base for converting values */ /*********************************************************** * DESCRIPTION: @@ -671,177 +696,144 @@ void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColT * RETURN: * none ***********************************************************/ -void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscColType, const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) +void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscColType, const ColType colType, + void* valArray, const size_t pos, boost::any& data, bool fromList) { - string curStr; + string curStr; - if (fromList) + if (fromList) + { + switch (colType) { - switch (colType) + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + // here we assign value to an array element and update range. + if (data.type() == typeid(long)) + ((int*)valArray)[pos] = static_cast(boost::any_cast(data)); + else if (data.type() == typeid(int)) + ((int*)valArray)[pos] = boost::any_cast(data); + else + { // this interesting part is for magic values like NULL or EMPTY (marks deleted elements). + // we will not put these into range. + ((int*)valArray)[pos] = boost::any_cast(data); + } + + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: ((uint32_t*)valArray)[pos] = boost::any_cast(data); break; + + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + curStr = boost::any_cast(data); + + if ((int)curStr.length() > MAX_COLUMN_BOUNDARY) + curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); + + memcpy((char*)valArray + pos * MAX_COLUMN_BOUNDARY, curStr.c_str(), curStr.length()); + break; + + // case WriteEngine::WR_LONG : ((long*)valArray)[pos] = + // boost::any_cast(curTuple.data); + // break; + case WriteEngine::WR_FLOAT: + ((float*)valArray)[pos] = boost::any_cast(data); + + if (isnan(((float*)valArray)[pos])) { - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - // here we assign value to an array element and update range. - if (data.type() == typeid(long)) - ((int*)valArray)[pos] = static_cast(boost::any_cast(data)); - else if (data.type() == typeid(int)) - ((int*)valArray)[pos] = boost::any_cast(data); - else - { // this interesting part is for magic values like NULL or EMPTY (marks deleted elements). - // we will not put these into range. - ((int*)valArray)[pos] = boost::any_cast(data); - } + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + ((float*)valArray)[pos] = *tfp; + } - break; + break; - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - ((uint32_t*)valArray)[pos] = boost::any_cast(data); - break; + case WriteEngine::WR_DOUBLE: ((double*)valArray)[pos] = boost::any_cast(data); break; - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR : - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - curStr = boost::any_cast(data); + case WriteEngine::WR_SHORT: ((short*)valArray)[pos] = boost::any_cast(data); break; - if ((int) curStr.length() > MAX_COLUMN_BOUNDARY) - curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); + case WriteEngine::WR_USHORT: + ((uint16_t*)valArray)[pos] = boost::any_cast(data); + break; - memcpy((char*)valArray + pos * MAX_COLUMN_BOUNDARY, curStr.c_str(), curStr.length()); - break; + // case WriteEngine::WR_BIT: ((bool*)valArray)[pos] = boost::any_cast(data); + // break; + case WriteEngine::WR_BYTE: ((char*)valArray)[pos] = boost::any_cast(data); break; -// case WriteEngine::WR_LONG : ((long*)valArray)[pos] = boost::any_cast(curTuple.data); -// break; - case WriteEngine::WR_FLOAT: - ((float*)valArray)[pos] = boost::any_cast(data); + case WriteEngine::WR_UBYTE: ((uint8_t*)valArray)[pos] = boost::any_cast(data); break; - if (isnan(((float*)valArray)[pos])) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - ((float*)valArray)[pos] = *tfp; - } + case WriteEngine::WR_LONGLONG: + if (data.type() == typeid(long long)) + ((long long*)valArray)[pos] = boost::any_cast(data); + else if (data.type() == typeid(long)) + ((long long*)valArray)[pos] = (long long)boost::any_cast(data); + else + ((long long*)valArray)[pos] = boost::any_cast(data); - break; + break; - case WriteEngine::WR_DOUBLE: - ((double*)valArray)[pos] = boost::any_cast(data); - break; + case WriteEngine::WR_ULONGLONG: ((uint64_t*)valArray)[pos] = boost::any_cast(data); break; - case WriteEngine::WR_SHORT: - ((short*)valArray)[pos] = boost::any_cast(data); - break; + case WriteEngine::WR_TOKEN: ((Token*)valArray)[pos] = boost::any_cast(data); break; - case WriteEngine::WR_USHORT: - ((uint16_t*)valArray)[pos] = boost::any_cast(data); - break; - -// case WriteEngine::WR_BIT: ((bool*)valArray)[pos] = boost::any_cast(data); -// break; - case WriteEngine::WR_BYTE: - ((char*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_UBYTE: - ((uint8_t*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_LONGLONG: - if (data.type() == typeid(long long)) - ((long long*)valArray)[pos] = boost::any_cast(data); - else if (data.type() == typeid(long)) - ((long long*)valArray)[pos] = (long long)boost::any_cast(data); - else - ((long long*)valArray)[pos] = boost::any_cast(data); - - break; - - case WriteEngine::WR_ULONGLONG: - ((uint64_t*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_TOKEN: - ((Token*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_BINARY: - size_t size = cscColType.colWidth; - int128_t val = boost::any_cast(data); - memcpy((uint8_t*)valArray + pos * size, &val, size); - break; - } // end of switch (colType) - } - else + case WriteEngine::WR_BINARY: + size_t size = cscColType.colWidth; + int128_t val = boost::any_cast(data); + memcpy((uint8_t*)valArray + pos * size, &val, size); + break; + } // end of switch (colType) + } + else + { + switch (colType) { - switch (colType) - { - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - data = ((int*)valArray)[pos]; - break; + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: data = ((int*)valArray)[pos]; break; - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - data = ((uint64_t*)valArray)[pos]; - break; + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: data = ((uint64_t*)valArray)[pos]; break; - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR : - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - char tmp[10]; - memcpy(tmp, (char*)valArray + pos * 8, 8); - curStr = tmp; - data = curStr; - break; + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + char tmp[10]; + memcpy(tmp, (char*)valArray + pos * 8, 8); + curStr = tmp; + data = curStr; + break; -// case WriteEngine::WR_LONG : ((long*)valArray)[pos] = boost::any_cast(curTuple.data); -// break; - case WriteEngine::WR_FLOAT: - data = ((float*)valArray)[pos]; - break; + // case WriteEngine::WR_LONG : ((long*)valArray)[pos] = + // boost::any_cast(curTuple.data); + // break; + case WriteEngine::WR_FLOAT: data = ((float*)valArray)[pos]; break; - case WriteEngine::WR_DOUBLE: - data = ((double*)valArray)[pos]; - break; + case WriteEngine::WR_DOUBLE: data = ((double*)valArray)[pos]; break; - case WriteEngine::WR_SHORT: - data = ((short*)valArray)[pos]; - break; + case WriteEngine::WR_SHORT: data = ((short*)valArray)[pos]; break; - case WriteEngine::WR_USHORT: - data = ((uint16_t*)valArray)[pos]; - break; + case WriteEngine::WR_USHORT: + data = ((uint16_t*)valArray)[pos]; + break; -// case WriteEngine::WR_BIT: data = ((bool*)valArray)[pos]; -// break; - case WriteEngine::WR_BYTE: - data = ((char*)valArray)[pos]; - break; + // case WriteEngine::WR_BIT: data = ((bool*)valArray)[pos]; + // break; + case WriteEngine::WR_BYTE: data = ((char*)valArray)[pos]; break; - case WriteEngine::WR_UBYTE: - data = ((uint8_t*)valArray)[pos]; - break; + case WriteEngine::WR_UBYTE: data = ((uint8_t*)valArray)[pos]; break; - case WriteEngine::WR_LONGLONG: - data = ((long long*)valArray)[pos]; - break; + case WriteEngine::WR_LONGLONG: data = ((long long*)valArray)[pos]; break; - case WriteEngine::WR_ULONGLONG: - data = ((uint64_t*)valArray)[pos]; - break; + case WriteEngine::WR_ULONGLONG: data = ((uint64_t*)valArray)[pos]; break; - case WriteEngine::WR_TOKEN: - data = ((Token*)valArray)[pos]; - break; + case WriteEngine::WR_TOKEN: data = ((Token*)valArray)[pos]; break; - case WriteEngine::WR_BINARY: - data = ((int128_t*)valArray)[pos]; - break; + case WriteEngine::WR_BINARY: data = ((int128_t*)valArray)[pos]; break; - } // end of switch (colType) - } // end of if + } // end of switch (colType) + } // end of if } /*@createColumn - Create column files, including data and bitmap files @@ -861,317 +853,298 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscCo * ERR_FILE_EXIST if file exists * ERR_FILE_CREATE if something wrong in creating the file ***********************************************************/ -int WriteEngineWrapper::createColumn( - const TxnID& txnid, - const OID& dataOid, - const CalpontSystemCatalog::ColDataType dataType, - int dataWidth, - uint16_t dbRoot, - uint32_t partition, - int compressionType) +int WriteEngineWrapper::createColumn(const TxnID& txnid, const OID& dataOid, + const CalpontSystemCatalog::ColDataType dataType, int dataWidth, + uint16_t dbRoot, uint32_t partition, int compressionType) { - int rc; - Column curCol; + int rc; + Column curCol; - int compress_op = op(compressionType); - m_colOp[compress_op]->initColumn(curCol); - m_colOp[compress_op]->findTypeHandler(dataWidth, dataType); + int compress_op = op(compressionType); + m_colOp[compress_op]->initColumn(curCol); + m_colOp[compress_op]->findTypeHandler(dataWidth, dataType); - - rc = m_colOp[compress_op]->createColumn(curCol, 0, dataWidth, dataType, - WriteEngine::WR_CHAR, (FID)dataOid, dbRoot, partition); + rc = m_colOp[compress_op]->createColumn(curCol, 0, dataWidth, dataType, WriteEngine::WR_CHAR, (FID)dataOid, + dbRoot, partition); - // This is optional, however, it's recommended to do so to free heap - // memory if assigned in the future - m_colOp[compress_op]->clearColumn(curCol); - std::map oids; + // This is optional, however, it's recommended to do so to free heap + // memory if assigned in the future + m_colOp[compress_op]->clearColumn(curCol); + std::map oids; - if (rc == NO_ERROR) - rc = flushDataFiles(NO_ERROR, txnid, oids); + if (rc == NO_ERROR) + rc = flushDataFiles(NO_ERROR, txnid, oids); - if (rc != NO_ERROR) - { - return rc; - } - - RETURN_ON_ERROR(BRMWrapper::getInstance()->setLocalHWM(dataOid, partition, 0, 0)); - // @bug 281 : fix for bug 281 - Add flush VM cache to clear all write buffer - //flushVMCache(); + if (rc != NO_ERROR) + { return rc; + } + + RETURN_ON_ERROR(BRMWrapper::getInstance()->setLocalHWM(dataOid, partition, 0, 0)); + // @bug 281 : fix for bug 281 - Add flush VM cache to clear all write buffer + // flushVMCache(); + return rc; } -//BUG931 +// BUG931 /** * @brief Fill column with default values */ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, - const CalpontSystemCatalog::ColType& colType, - ColTuple defaultVal, const OID& refColOID, - const CalpontSystemCatalog::ColDataType refColDataType, - int refColWidth, int refCompressionType, - bool isNULL, int compressionType, - const string& defaultValStr, - const OID& dictOid, bool autoincrement) + const CalpontSystemCatalog::ColType& colType, ColTuple defaultVal, + const OID& refColOID, + const CalpontSystemCatalog::ColDataType refColDataType, int refColWidth, + int refCompressionType, bool isNULL, int compressionType, + const string& defaultValStr, const OID& dictOid, bool autoincrement) { - int rc = NO_ERROR; - Column newCol; - Column refCol; - ColType newColType; - ColType refColType; - boost::scoped_array defVal(new char[MAX_COLUMN_BOUNDARY]); - ColumnOp* colOpNewCol = m_colOp[op(compressionType)]; - ColumnOp* refColOp = m_colOp[op(refCompressionType)]; - Dctnry* dctnry = m_dctnry[op(compressionType)]; - colOpNewCol->initColumn(newCol); - refColOp->initColumn(refCol); - uint16_t dbRoot = 1; //not to be used - int newDataWidth = colType.colWidth; - //Convert HWM of the reference column for the new column - //Bug 1703,1705 - bool isToken = false; + int rc = NO_ERROR; + Column newCol; + Column refCol; + ColType newColType; + ColType refColType; + boost::scoped_array defVal(new char[MAX_COLUMN_BOUNDARY]); + ColumnOp* colOpNewCol = m_colOp[op(compressionType)]; + ColumnOp* refColOp = m_colOp[op(refCompressionType)]; + Dctnry* dctnry = m_dctnry[op(compressionType)]; + colOpNewCol->initColumn(newCol); + refColOp->initColumn(refCol); + uint16_t dbRoot = 1; // not to be used + int newDataWidth = colType.colWidth; + // Convert HWM of the reference column for the new column + // Bug 1703,1705 + bool isToken = false; - if (((colType.colDataType == CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || - ((colType.colDataType == CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || - (colType.colDataType == CalpontSystemCatalog::VARBINARY) || - (colType.colDataType == CalpontSystemCatalog::BLOB) || - (colType.colDataType == CalpontSystemCatalog::TEXT)) + if (((colType.colDataType == CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || + ((colType.colDataType == CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || + (colType.colDataType == CalpontSystemCatalog::VARBINARY) || + (colType.colDataType == CalpontSystemCatalog::BLOB) || + (colType.colDataType == CalpontSystemCatalog::TEXT)) + { + isToken = true; + } + + Convertor::convertColType(colType.colDataType, colType.colWidth, newColType, isToken); + + // WIP + // replace with isDictCol + if (((refColDataType == CalpontSystemCatalog::VARCHAR) && (refColWidth > 7)) || + ((refColDataType == CalpontSystemCatalog::CHAR) && (refColWidth > 8)) || + (refColDataType == CalpontSystemCatalog::VARBINARY) || + (colType.colDataType == CalpontSystemCatalog::BLOB) || + (colType.colDataType == CalpontSystemCatalog::TEXT)) + { + isToken = true; + } + + newDataWidth = colOpNewCol->getCorrectRowWidth(colType.colDataType, colType.colWidth); + // MCOL-1347 CS doubles the width for ALTER TABLE..ADD COLUMN + if (colType.colWidth < 4 && colType.colDataType == CalpontSystemCatalog::VARCHAR) + { + newDataWidth >>= 1; + } + + Convertor::convertColType(refColDataType, refColWidth, refColType, isToken); + refColOp->setColParam(refCol, 0, refColOp->getCorrectRowWidth(refColDataType, refColWidth), refColDataType, + refColType, (FID)refColOID, refCompressionType, dbRoot); + colOpNewCol->setColParam(newCol, 0, newDataWidth, colType.colDataType, newColType, (FID)dataOid, + compressionType, dbRoot); + // refColOp and colOpNewCol are the same ptr. + colOpNewCol->findTypeHandler(newDataWidth, colType.colDataType); + + int size = sizeof(Token); + + if (newColType == WriteEngine::WR_TOKEN) + { + if (isNULL) { - isToken = true; + Token nullToken; + memcpy(defVal.get(), &nullToken, size); } - - Convertor::convertColType(colType.colDataType, colType.colWidth, newColType, isToken); - + // Tokenization is done when we create dictionary file + } + else + { // WIP - // replace with isDictCol - if (((refColDataType == CalpontSystemCatalog::VARCHAR) && (refColWidth > 7)) || - ((refColDataType == CalpontSystemCatalog::CHAR) && (refColWidth > 8)) || - (refColDataType == CalpontSystemCatalog::VARBINARY) || - (colType.colDataType == CalpontSystemCatalog::BLOB) || - (colType.colDataType == CalpontSystemCatalog::TEXT)) - { - isToken = true; - } + convertValue(colType, newColType, defVal.get(), defaultVal.data); + } - newDataWidth = colOpNewCol->getCorrectRowWidth(colType.colDataType, colType.colWidth); - // MCOL-1347 CS doubles the width for ALTER TABLE..ADD COLUMN - if ( colType.colWidth < 4 && colType.colDataType == CalpontSystemCatalog::VARCHAR ) - { - newDataWidth >>= 1; - } + if (rc == NO_ERROR) + rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, + colType.colWidth, defaultValStr, autoincrement); - Convertor::convertColType(refColDataType, refColWidth, refColType, isToken); - refColOp->setColParam(refCol, 0, refColOp->getCorrectRowWidth(refColDataType, refColWidth), - refColDataType, refColType, (FID)refColOID, refCompressionType, dbRoot); - colOpNewCol->setColParam(newCol, 0, newDataWidth, - colType.colDataType, newColType, (FID)dataOid, compressionType, dbRoot); - // refColOp and colOpNewCol are the same ptr. - colOpNewCol->findTypeHandler(newDataWidth, colType.colDataType); + // flushing files is in colOp->fillColumn() - - int size = sizeof(Token); - - if (newColType == WriteEngine::WR_TOKEN) - { - if (isNULL) - { - Token nullToken; - memcpy(defVal.get(), &nullToken, size); - } - //Tokenization is done when we create dictionary file - } - else - { - // WIP - convertValue(colType, newColType, defVal.get(), defaultVal.data); - } - - if (rc == NO_ERROR) - rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, colType.colWidth, defaultValStr, autoincrement); - -// flushing files is in colOp->fillColumn() - - return rc; + return rc; } int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector& colExtentsColType, vector& colExtentsStruct, vector& colOldValueList, vector& ridLists, const int32_t tableOid) { - ColTuple curTuple; - ColStruct curColStruct; - CalpontSystemCatalog::ColType cscColType; - DctnryStruct dctnryStruct; - ColValueList colValueList; - ColTupleList curTupleList; - DctnryStructList dctnryStructList; - DctnryValueList dctnryValueList; - ColStructList colStructList; - CSCTypesList cscColTypeList; - int rc; - string tmpStr(""); - vector dctnryExtentsStruct; + ColTuple curTuple; + ColStruct curColStruct; + CalpontSystemCatalog::ColType cscColType; + DctnryStruct dctnryStruct; + ColValueList colValueList; + ColTupleList curTupleList; + DctnryStructList dctnryStructList; + DctnryValueList dctnryValueList; + ColStructList colStructList; + CSCTypesList cscColTypeList; + int rc; + string tmpStr(""); + vector dctnryExtentsStruct; - if (colExtentsStruct.size() == 0 || ridLists.size() == 0) - return ERR_STRUCT_EMPTY; + if (colExtentsStruct.size() == 0 || ridLists.size() == 0) + return ERR_STRUCT_EMPTY; - // set transaction id - setTransId(txnid); - unsigned numExtents = colExtentsStruct.size(); + // set transaction id + setTransId(txnid); + unsigned numExtents = colExtentsStruct.size(); - for (unsigned extent = 0; extent < numExtents; extent++) + for (unsigned extent = 0; extent < numExtents; extent++) + { + colStructList = colExtentsStruct[extent]; + cscColTypeList = colExtentsColType[extent]; + + for (ColStructList::size_type i = 0; i < colStructList.size(); i++) { - colStructList = colExtentsStruct[extent]; - cscColTypeList = colExtentsColType[extent]; + curTupleList.clear(); + curColStruct = colStructList[i]; + cscColType = cscColTypeList[i]; + Convertor::convertColType(&curColStruct); - for (ColStructList::size_type i = 0; i < colStructList.size(); i++) - { - curTupleList.clear(); - curColStruct = colStructList[i]; - cscColType = cscColTypeList[i]; - Convertor::convertColType(&curColStruct); + const uint8_t* emptyVal = m_colOp[op(curColStruct.fCompressionType)]->getEmptyRowValue( + curColStruct.colDataType, curColStruct.colWidth); - const uint8_t* emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); + if (curColStruct.colWidth == datatypes::MAXDECIMALWIDTH) + curTuple.data = *(int128_t*)emptyVal; + else + curTuple.data = *(int64_t*)emptyVal; - if (curColStruct.colWidth == datatypes::MAXDECIMALWIDTH) - curTuple.data = *(int128_t*)emptyVal; - else - curTuple.data = *(int64_t*)emptyVal; + curTupleList.push_back(curTuple); + colValueList.push_back(curTupleList); - curTupleList.push_back(curTuple); - colValueList.push_back(curTupleList); + dctnryStruct.dctnryOid = 0; + dctnryStruct.fColPartition = curColStruct.fColPartition; + dctnryStruct.fColSegment = curColStruct.fColSegment; + dctnryStruct.fColDbRoot = curColStruct.fColDbRoot; + dctnryStruct.columnOid = colStructList[i].dataOid; + dctnryStructList.push_back(dctnryStruct); - dctnryStruct.dctnryOid = 0; - dctnryStruct.fColPartition = curColStruct.fColPartition; - dctnryStruct.fColSegment = curColStruct.fColSegment; - dctnryStruct.fColDbRoot = curColStruct.fColDbRoot; - dctnryStruct.columnOid = colStructList[i].dataOid; - dctnryStructList.push_back(dctnryStruct); - - DctnryTuple dctnryTuple; - DctColTupleList dctColTuples; - dctnryTuple.sigValue = (unsigned char*)tmpStr.c_str(); - dctnryTuple.sigSize = tmpStr.length(); - dctnryTuple.isNull = true; - dctColTuples.push_back (dctnryTuple); - dctnryValueList.push_back (dctColTuples); - } - - dctnryExtentsStruct.push_back(dctnryStructList); + DctnryTuple dctnryTuple; + DctColTupleList dctColTuples; + dctnryTuple.sigValue = (unsigned char*)tmpStr.c_str(); + dctnryTuple.sigSize = tmpStr.length(); + dctnryTuple.isNull = true; + dctColTuples.push_back(dctnryTuple); + dctnryValueList.push_back(dctColTuples); } - // unfortunately I don't have a better way to instruct without passing too many parameters - m_opType = DELETE; - rc = updateColumnRec(txnid, colExtentsColType, colExtentsStruct, colValueList, colOldValueList, ridLists, dctnryExtentsStruct, dctnryValueList, tableOid); - m_opType = NOOP; + dctnryExtentsStruct.push_back(dctnryStructList); + } - return rc; + // unfortunately I don't have a better way to instruct without passing too many parameters + m_opType = DELETE; + rc = updateColumnRec(txnid, colExtentsColType, colExtentsStruct, colValueList, colOldValueList, ridLists, + dctnryExtentsStruct, dctnryValueList, tableOid); + m_opType = NOOP; + + return rc; } -inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, - ColType colType, int colWidth) +inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, ColType colType, int colWidth) { - // MCS allocates 8 bytes even for CHARs smaller then 8 bytes. - switch (colType) - { - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); - break; - case WriteEngine::WR_TOKEN: - valArray = calloc(sizeof(Token), totalRow); - break; - default: - valArray = calloc(totalRow, colWidth); - break; - } + // MCS allocates 8 bytes even for CHARs smaller then 8 bytes. + switch (colType) + { + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: valArray = calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); break; + case WriteEngine::WR_TOKEN: valArray = calloc(sizeof(Token), totalRow); break; + default: valArray = calloc(totalRow, colWidth); break; + } } -int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStructs, - RIDList& ridList, DctnryStructList& dctnryStructList) +int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStructs, RIDList& ridList, + DctnryStructList& dctnryStructList) { - /* Need to scan all files including dictionary store files to check whether there is any bad chunks - * - */ - int rc = 0; - Column curCol; - void* valArray = NULL; + /* Need to scan all files including dictionary store files to check whether there is any bad chunks + * + */ + int rc = 0; + Column curCol; + void* valArray = NULL; - m_opType = DELETE; + m_opType = DELETE; - for (unsigned i = 0; i < colStructs.size(); i++) + for (unsigned i = 0; i < colStructs.size(); i++) + { + ColumnOp* colOp = m_colOp[op(colStructs[i].fCompressionType)]; + unsigned needFixFiles = colStructs[i].tokenFlag ? 2 : 1; + colOp->initColumn(curCol); + + for (unsigned j = 0; j < needFixFiles; j++) { - ColumnOp* colOp = m_colOp[op(colStructs[i].fCompressionType)]; - unsigned needFixFiles = colStructs[i].tokenFlag ? 2 : 1; - colOp->initColumn(curCol); + if (j == 0) + { + colOp->setColParam(curCol, 0, colStructs[i].colWidth, colStructs[i].colDataType, + colStructs[i].colType, colStructs[i].dataOid, colStructs[i].fCompressionType, + colStructs[i].fColDbRoot, colStructs[i].fColPartition, colStructs[i].fColSegment); + colOp->findTypeHandler(colStructs[i].colWidth, colStructs[i].colDataType); - for (unsigned j = 0; j < needFixFiles; j++) + string segFile; + rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) // If openFile fails, disk error or header error is assumed. { - if (j == 0) - { - colOp->setColParam(curCol, 0, colStructs[i].colWidth, - colStructs[i].colDataType, colStructs[i].colType, colStructs[i].dataOid, - colStructs[i].fCompressionType, colStructs[i].fColDbRoot, - colStructs[i].fColPartition, colStructs[i].fColSegment); - colOp->findTypeHandler(colStructs[i].colWidth, colStructs[i].colDataType); - - string segFile; - rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) //If openFile fails, disk error or header error is assumed. - { - //report error and return. - std::ostringstream oss; - WErrorCodes ec; - string err = ec.errorString(rc); - oss << "Error opening file oid:dbroot:partition:segment = " << colStructs[i].dataOid << ":" << colStructs[i].fColDbRoot - << ":" << colStructs[i].fColPartition << ":" << colStructs[i].fColSegment << " and error code is " << rc << " with message " << err; - throw std::runtime_error(oss.str()); - } - - allocateValArray(valArray, 1, colStructs[i].colType, colStructs[i].colWidth); - - rc = colOp->writeRows(curCol, ridList.size(), ridList, valArray, 0, true); - - if ( rc != NO_ERROR) - { - //read error is fixed in place - if (rc == ERR_COMP_COMPRESS) //write error - { - - } - - } - - //flush files will be done in the end of fix. - colOp->clearColumn(curCol); - - if (valArray != NULL) - free(valArray); - } - else //dictionary file. How to fix - { - //read headers out, uncompress the last chunk, if error, replace it with empty chunk. - Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; - rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, - dctnryStructList[i].fColDbRoot, dctnryStructList[i].fColPartition, - dctnryStructList[i].fColSegment, - false); - - rc = dctnry->checkFixLastDictChunk(); - rc = dctnry->closeDctnry(true); - - } + // report error and return. + std::ostringstream oss; + WErrorCodes ec; + string err = ec.errorString(rc); + oss << "Error opening file oid:dbroot:partition:segment = " << colStructs[i].dataOid << ":" + << colStructs[i].fColDbRoot << ":" << colStructs[i].fColPartition << ":" + << colStructs[i].fColSegment << " and error code is " << rc << " with message " << err; + throw std::runtime_error(oss.str()); } - } - return rc; + allocateValArray(valArray, 1, colStructs[i].colType, colStructs[i].colWidth); + + rc = colOp->writeRows(curCol, ridList.size(), ridList, valArray, 0, true); + + if (rc != NO_ERROR) + { + // read error is fixed in place + if (rc == ERR_COMP_COMPRESS) // write error + { + } + } + + // flush files will be done in the end of fix. + colOp->clearColumn(curCol); + + if (valArray != NULL) + free(valArray); + } + else // dictionary file. How to fix + { + // read headers out, uncompress the last chunk, if error, replace it with empty chunk. + Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; + rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, dctnryStructList[i].fColDbRoot, + dctnryStructList[i].fColPartition, dctnryStructList[i].fColSegment, false); + + rc = dctnry->checkFixLastDictChunk(); + rc = dctnry->closeDctnry(true); + } + } + } + + return rc; } /*@flushVMCache - Flush VM cache -*/ + */ /*********************************************************** * DESCRIPTION: * Flush sytem VM cache @@ -1182,10 +1155,9 @@ int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStru ***********************************************************/ void WriteEngineWrapper::flushVMCache() const { -// int fd = open("/proc/sys/vm/drop_caches", O_WRONLY); -// write(fd, "3", 1); -// close(fd); - + // int fd = open("/proc/sys/vm/drop_caches", O_WRONLY); + // write(fd, "3", 1); + // close(fd); } #if 0 @@ -1207,35 +1179,33 @@ static void log_this(const char *message, #endif /** @brief Determine whether we may update a column's ranges (by type) and return nullptr if we can't */ -static ExtCPInfo* -getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* currentCPInfo) +static ExtCPInfo* getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* currentCPInfo) { - if (colStruct.tokenFlag) + if (colStruct.tokenFlag) + { + return nullptr; + } + switch (colStruct.colType) + { + // here we enumerate all supported types. + case WR_BYTE: + case WR_SHORT: + case WR_INT: + case WR_LONGLONG: + case WR_CHAR: + case WR_UBYTE: + case WR_USHORT: + case WR_UINT: + case WR_ULONGLONG: + case WR_MEDINT: + case WR_UMEDINT: + case WR_BINARY: { - return nullptr; - } - switch(colStruct.colType) - { - // here we enumerate all supported types. - case WR_BYTE: - case WR_SHORT: - case WR_INT: - case WR_LONGLONG: - case WR_CHAR: - case WR_UBYTE: - case WR_USHORT: - case WR_UINT: - case WR_ULONGLONG: - case WR_MEDINT: - case WR_UMEDINT: - case WR_BINARY: - { - return currentCPInfo; - } - // all unsupported types must not be supported. - default: - return nullptr; // safe choice for everything we can't do. + return currentCPInfo; } + // all unsupported types must not be supported. + default: return nullptr; // safe choice for everything we can't do. + } } /** @brief Let only valid ranges to be present. @@ -1247,20 +1217,20 @@ getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* current * * To achieve that, we filter these invalid ranges here. */ -static void -setInvalidCPInfosSpecialMarks(std::vector& cpInfos) +static void setInvalidCPInfosSpecialMarks(std::vector& cpInfos) { - size_t i; - for(i=0; i < cpInfos.size(); i++) { - if (cpInfos[i].isInvalid()) - { - cpInfos[i].fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; - } + size_t i; + for (i = 0; i < cpInfos.size(); i++) + { + if (cpInfos[i].isInvalid()) + { + cpInfos[i].fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; } + } } /*@insertColumnRecs - Insert value(s) into a column -*/ + */ /*********************************************************** * DESCRIPTION: * Insert values into columns (batchinsert) @@ -1272,2911 +1242,2833 @@ setInvalidCPInfosSpecialMarks(std::vector& cpInfos) * others if something wrong in inserting the value ***********************************************************/ -int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - ColStructList& colStructList, - ColValueList& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - std::vector >& dbRootExtentTrackers, - RBMetaWriter* fRBMetaWriter, - bool bFirstExtentOnThisPM, - bool insertSelect, - bool isAutoCommitOn, - OID tableOid, - bool isFirstBatchPm) +int WriteEngineWrapper::insertColumnRecs( + const TxnID& txnid, const CSCTypesList& cscColTypeList, ColStructList& colStructList, + ColValueList& colValueList, DctnryStructList& dctnryStructList, DictStrList& dictStrList, + std::vector >& dbRootExtentTrackers, RBMetaWriter* fRBMetaWriter, + bool bFirstExtentOnThisPM, bool insertSelect, bool isAutoCommitOn, OID tableOid, bool isFirstBatchPm) { - int rc; - RID* rowIdArray = NULL; - ColTupleList curTupleList; - Column curCol; - ColStruct curColStruct; - ColValueList colOldValueList; - ColValueList colNewValueList; - ColStructList newColStructList; - DctnryStructList newDctnryStructList; - HWM hwm = 0; - HWM oldHwm = 0; - HWM newHwm = 0; - ColTupleList::size_type totalRow; - ColStructList::size_type totalColumns; - uint64_t rowsLeft = 0; - bool newExtent = false; - RIDList ridList; - ColumnOp* colOp = NULL; - ColSplitMaxMinInfoList maxMins; + int rc; + RID* rowIdArray = NULL; + ColTupleList curTupleList; + Column curCol; + ColStruct curColStruct; + ColValueList colOldValueList; + ColValueList colNewValueList; + ColStructList newColStructList; + DctnryStructList newDctnryStructList; + HWM hwm = 0; + HWM oldHwm = 0; + HWM newHwm = 0; + ColTupleList::size_type totalRow; + ColStructList::size_type totalColumns; + uint64_t rowsLeft = 0; + bool newExtent = false; + RIDList ridList; + ColumnOp* colOp = NULL; + ColSplitMaxMinInfoList maxMins; - // Set tmp file suffix to modify HDFS db file - bool useTmpSuffix = false; + // Set tmp file suffix to modify HDFS db file + bool useTmpSuffix = false; - m_opType = INSERT; + m_opType = INSERT; - if (idbdatafile::IDBPolicy::useHdfs()) - { - if (!bFirstExtentOnThisPM) - useTmpSuffix = true; - } + if (idbdatafile::IDBPolicy::useHdfs()) + { + if (!bFirstExtentOnThisPM) + useTmpSuffix = true; + } - unsigned i = 0; + unsigned i = 0; #ifdef PROFILE - StopWatch timer; + StopWatch timer; #endif - // debug information for testing - if (isDebug(DEBUG_2)) + // debug information for testing + if (isDebug(DEBUG_2)) + { + printInputValue(colStructList, colValueList, ridList, dctnryStructList, dictStrList); + } + + // end + + // Convert data type and column width to write engine specific + for (i = 0; i < colStructList.size(); i++) + Convertor::convertColType(&colStructList[i]); + + for (const auto& colStruct : colStructList) + { + ColSplitMaxMinInfo tmp(colStruct.colDataType, colStruct.colWidth); + maxMins.push_back(tmp); + } + + uint32_t colId = 0; + // MCOL-1675: find the smallest column width to calculate the RowID from so + // that all HWMs will be incremented by this operation + findSmallestColumn(colId, colStructList); + + // rc = checkValid(txnid, colStructList, colValueList, ridList); + // if (rc != NO_ERROR) + // return rc; + + setTransId(txnid); + uint16_t dbRoot, segmentNum; + uint32_t partitionNum; + string segFile; + bool newFile; + TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); + // populate colStructList with file information + IDBDataFile* pFile = NULL; + std::vector extentInfo; + int currentDBrootIdx = 0; + std::vector + extents; // this structure valid only when isFirstOnBatchPm is true. + std::vector newExtentsStartingLbids; // we keep column-indexed LBIDs here for **new** extents. + // It is set and valid and used only when rowsLeft > 0. + + //-------------------------------------------------------------------------- + // For first batch on this PM: + // o get starting extent from ExtentTracker, and allocate extent if needed + // o construct colStructList and dctnryStructList accordingly + // o save extent information in tableMetaData for future use + // If not first batch on this PM: + // o construct colStructList and dctnryStructList from tableMetaData + //-------------------------------------------------------------------------- + if (isFirstBatchPm) + { + currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); + dbRoot = extentInfo[currentDBrootIdx].fDbRoot; + partitionNum = extentInfo[currentDBrootIdx].fPartition; + + //---------------------------------------------------------------------- + // check whether this extent is the first on this PM + //---------------------------------------------------------------------- + if (bFirstExtentOnThisPM) { - printInputValue(colStructList, colValueList, ridList, dctnryStructList, dictStrList); + std::vector cols; + BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; + + for (i = 0; i < colStructList.size(); i++) + { + createStripeColumnExtentsArgIn.oid = colStructList[i].dataOid; + createStripeColumnExtentsArgIn.width = colStructList[i].colWidth; + createStripeColumnExtentsArgIn.colDataType = colStructList[i].colDataType; + cols.push_back(createStripeColumnExtentsArgIn); + } + + rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partitionNum, segmentNum, + extents); + + if (rc != NO_ERROR) + return rc; + + // Create column files + ExtCPInfoList cpinfoList; + + for (i = 0; i < extents.size(); i++) + { + ExtCPInfo cpInfo(colStructList[i].colDataType, colStructList[i].colWidth); + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + colOp->initColumn(curCol); + colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType, + colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, dbRoot, partitionNum, segmentNum); + colOp->findTypeHandler(colStructList[i].colWidth, colStructList[i].colDataType); + rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, + extents[i].allocSize, dbRoot, partitionNum, segmentNum, segFile, pFile, + newFile); + + if (rc != NO_ERROR) + return rc; + + cpInfo.toInvalid(); + + cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; + + // mark the extents to invalid + cpInfo.fCPInfo.firstLbid = extents[i].startLbid; + cpinfoList.push_back(cpInfo); + colStructList[i].fColPartition = partitionNum; + colStructList[i].fColSegment = segmentNum; + colStructList[i].fColDbRoot = dbRoot; + dctnryStructList[i].fColPartition = partitionNum; + dctnryStructList[i].fColSegment = segmentNum; + dctnryStructList[i].fColDbRoot = dbRoot; + } + + // mark the extents to invalid + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); + + if (rc != NO_ERROR) + return rc; + + // create corresponding dictionary files + for (i = 0; i < dctnryStructList.size(); i++) + { + if (dctnryStructList[i].dctnryOid > 0) + { + rc = createDctnry(txnid, dctnryStructList[i].dctnryOid, dctnryStructList[i].colWidth, dbRoot, + partitionNum, segmentNum, dctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + return rc; + } + } + } // if ( bFirstExtentOnThisPM) + else // if (!bFirstExtentOnThisPM) + { + std::vector tmpExtentInfo; + + for (i = 0; i < dbRootExtentTrackers.size(); i++) + { + tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); + colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; + colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; + colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; + dctnryStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; + dctnryStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; + dctnryStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; + } } - // end - - //Convert data type and column width to write engine specific + //---------------------------------------------------------------------- + // Save the extents info in tableMetaData + //---------------------------------------------------------------------- for (i = 0; i < colStructList.size(); i++) - Convertor::convertColType(&colStructList[i]); - - for (const auto& colStruct : colStructList) { - ColSplitMaxMinInfo tmp(colStruct.colDataType, colStruct.colWidth); - maxMins.push_back(tmp); - } + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); - uint32_t colId = 0; - // MCOL-1675: find the smallest column width to calculate the RowID from so - // that all HWMs will be incremented by this operation - findSmallestColumn(colId, colStructList); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment)) + break; - // rc = checkValid(txnid, colStructList, colValueList, ridList); - // if (rc != NO_ERROR) - // return rc; + it++; + } - setTransId(txnid); - uint16_t dbRoot, segmentNum; - uint32_t partitionNum; - string segFile; - bool newFile; - TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); - //populate colStructList with file information - IDBDataFile* pFile = NULL; - std::vector extentInfo; - int currentDBrootIdx = 0; - std::vector extents; // this structure valid only when isFirstOnBatchPm is true. - std::vector newExtentsStartingLbids; // we keep column-indexed LBIDs here for **new** extents. It is set and valid and used only when rowsLeft > 0. + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aExt.isDict = false; - //-------------------------------------------------------------------------- - // For first batch on this PM: - // o get starting extent from ExtentTracker, and allocate extent if needed - // o construct colStructList and dctnryStructList accordingly - // o save extent information in tableMetaData for future use - // If not first batch on this PM: - // o construct colStructList and dctnryStructList from tableMetaData - //-------------------------------------------------------------------------- - if (isFirstBatchPm) - { - currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); - extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); - dbRoot = extentInfo[currentDBrootIdx].fDbRoot; - partitionNum = extentInfo[currentDBrootIdx].fPartition; - - //---------------------------------------------------------------------- - // check whether this extent is the first on this PM - //---------------------------------------------------------------------- if (bFirstExtentOnThisPM) { - std::vector cols; - BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; - - for (i = 0; i < colStructList.size(); i++) - { - createStripeColumnExtentsArgIn.oid = colStructList[i].dataOid; - createStripeColumnExtentsArgIn.width = colStructList[i].colWidth; - createStripeColumnExtentsArgIn.colDataType = colStructList[i].colDataType; - cols.push_back(createStripeColumnExtentsArgIn); - } - - rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partitionNum, segmentNum, extents); - - if (rc != NO_ERROR) - return rc; - - //Create column files - ExtCPInfoList cpinfoList; - - for ( i = 0; i < extents.size(); i++) - { - ExtCPInfo cpInfo(colStructList[i].colDataType, colStructList[i].colWidth); - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - colOp->initColumn(curCol); - colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType, - colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType, - dbRoot, partitionNum, segmentNum); - colOp->findTypeHandler(colStructList[i].colWidth, - colStructList[i].colDataType); - rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot, - partitionNum, segmentNum, segFile, pFile, newFile); - - if (rc != NO_ERROR) - return rc; - - cpInfo.toInvalid(); - - cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; - - //mark the extents to invalid - cpInfo.fCPInfo.firstLbid = extents[i].startLbid; - cpinfoList.push_back(cpInfo); - colStructList[i].fColPartition = partitionNum; - colStructList[i].fColSegment = segmentNum; - colStructList[i].fColDbRoot = dbRoot; - dctnryStructList[i].fColPartition = partitionNum; - dctnryStructList[i].fColSegment = segmentNum; - dctnryStructList[i].fColDbRoot = dbRoot; - } - - //mark the extents to invalid - rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); - - if (rc != NO_ERROR) - return rc; - - //create corresponding dictionary files - for (i = 0; i < dctnryStructList.size(); i++) - { - if (dctnryStructList[i].dctnryOid > 0) - { - rc = createDctnry(txnid, dctnryStructList[i].dctnryOid, dctnryStructList[i].colWidth, dbRoot, partitionNum, - segmentNum, dctnryStructList[i].fCompressionType); - - if ( rc != NO_ERROR) - return rc; - } - } - } // if ( bFirstExtentOnThisPM) - else // if (!bFirstExtentOnThisPM) + aExt.hwm = extents[i].startBlkOffset; + aExt.isNewExt = true; + } + else { - std::vector tmpExtentInfo; - - for (i = 0; i < dbRootExtentTrackers.size(); i++) - { - tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); - colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; - colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; - colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; - dctnryStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; - dctnryStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; - dctnryStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; - } + std::vector tmpExtentInfo; + tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); + aExt.isNewExt = false; + aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; } - //---------------------------------------------------------------------- - // Save the extents info in tableMetaData - //---------------------------------------------------------------------- - for (i = 0; i < colStructList.size(); i++) - { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); + aExt.current = true; + aColExtsInfo.push_back(aExt); + } - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; + tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aExt.isDict = false; - - if (bFirstExtentOnThisPM) - { - aExt.hwm = extents[i].startBlkOffset; - aExt.isNewExt = true; - } - else - { - std::vector tmpExtentInfo; - tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); - aExt.isNewExt = false; - aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; - } - - aExt.current = true; - aColExtsInfo.push_back(aExt); - } - - tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - for (i = 0; i < dctnryStructList.size(); i++) - { - if (dctnryStructList[i].dctnryOid > 0) - { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && (it->partNum == dctnryStructList[i].fColPartition) && (it->segNum == dctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = dctnryStructList[i].fColDbRoot; - aExt.partNum = dctnryStructList[i].fColPartition; - aExt.segNum = dctnryStructList[i].fColSegment; - aExt.compType = dctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - } - - tableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); - } - } - - } // if (isFirstBatchPm) - else //get the extent info from tableMetaData + for (i = 0; i < dctnryStructList.size(); i++) { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); + if (dctnryStructList[i].dctnryOid > 0) + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if (it->current) - break; + if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && + (it->partNum == dctnryStructList[i].fColPartition) && + (it->segNum == dctnryStructList[i].fColSegment)) + break; - it++; + it++; } - if (it == aColExtsInfo.end()) - return 1; - - for (i = 0; i < colStructList.size(); i++) + if (it == aColExtsInfo.end()) // add this one to the list { - colStructList[i].fColPartition = it->partNum; - colStructList[i].fColSegment = it->segNum; - colStructList[i].fColDbRoot = it->dbRoot; - dctnryStructList[i].fColPartition = it->partNum; - dctnryStructList[i].fColSegment = it->segNum; - dctnryStructList[i].fColDbRoot = it->dbRoot; + ColExtInfo aExt; + aExt.dbRoot = dctnryStructList[i].fColDbRoot; + aExt.partNum = dctnryStructList[i].fColPartition; + aExt.segNum = dctnryStructList[i].fColSegment; + aExt.compType = dctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); } + + tableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); + } } - curTupleList = static_cast(colValueList[0]); - totalRow = curTupleList.size(); - totalColumns = colStructList.size(); - rowIdArray = new RID[totalRow]; - // use scoped_array to ensure ptr deletion regardless of where we return - boost::scoped_array rowIdArrayPtr(rowIdArray); - memset(rowIdArray, 0, (sizeof(RID)*totalRow)); - - //-------------------------------------------------------------------------- - // allocate row id(s) - //-------------------------------------------------------------------------- - curColStruct = colStructList[colId]; - colOp = m_colOp[op(curColStruct.fCompressionType)]; - - colOp->initColumn(curCol); - - //Get the correct segment, partition, column file - vector colExtentInfo; //Save those empty extents in case of failure to rollback - vector dictExtentInfo; //Save those empty extents in case of failure to rollback - vector fileInfo; - dbRoot = curColStruct.fColDbRoot; - //use the first column to calculate row id + } // if (isFirstBatchPm) + else // get the extent info from tableMetaData + { ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if ((it->dbRoot == colStructList[colId].fColDbRoot) && - (it->partNum == colStructList[colId].fColPartition) && - (it->segNum == colStructList[colId].fColSegment) && it->current ) - { - break; - } - it++; + if (it->current) + break; + + it++; } - if (it != aColExtsInfo.end()) - { - hwm = it->hwm; - } - - oldHwm = hwm; //Save this info for rollback - //need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, - curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, - curColStruct.fColDbRoot, curColStruct.fColPartition, curColStruct.fColSegment); - colOp->findTypeHandler(curColStruct.colWidth, - curColStruct.colDataType); - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - { - return rc; - } - - //get hwm first - // @bug 286 : fix for bug 286 - correct the typo in getHWM - //RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); - - Column newCol; - -#ifdef PROFILE - timer.start("allocRowId"); -#endif - newColStructList = colStructList; - newDctnryStructList = dctnryStructList; - bool bUseStartExtent = true; - - if (idbdatafile::IDBPolicy::useHdfs()) - insertSelect = true; - - rc = colOp->allocRowId(txnid, bUseStartExtent, - curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, - newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm, &newExtentsStartingLbids); - - if (rc != NO_ERROR) //Clean up is already done - return rc; - -#ifdef PROFILE - timer.stop("allocRowId"); -#endif - - //-------------------------------------------------------------------------- - // Expand initial abbreviated extent if any RID in 1st extent is > 256K. - // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. - //-------------------------------------------------------------------------- - // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((curCol.dataFile.fPartition == 0) && - (curCol.dataFile.fSegment == 0) && - ((totalRow - rowsLeft) > 0) && - (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) - { - for (unsigned k=0; ksetColParam(expandCol, 0, - colStructList[k].colWidth, - colStructList[k].colDataType, - colStructList[k].colType, - colStructList[k].dataOid, - colStructList[k].fCompressionType, - colStructList[k].fColDbRoot, - colStructList[k].fColPartition, - colStructList[k].fColSegment); - colOp->findTypeHandler(colStructList[k].colWidth, - colStructList[k].colDataType); - rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file - - if (rc == NO_ERROR) - { - if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) - { - rc = colOp->expandAbbrevExtent(expandCol); - } - } - - if (rc != NO_ERROR) - { - return rc; - } - - colOp->clearColumn(expandCol); // closes the file (if uncompressed) - } - } - - //-------------------------------------------------------------------------- - // Tokenize data if needed - //-------------------------------------------------------------------------- - if (insertSelect && isAutoCommitOn) - BRMWrapper::setUseVb( false ); - else - BRMWrapper::setUseVb( true ); - - dictStr::iterator dctStr_iter; - ColTupleList::iterator col_iter; + if (it == aColExtsInfo.end()) + return 1; for (i = 0; i < colStructList.size(); i++) { - if (colStructList[i].tokenFlag) + colStructList[i].fColPartition = it->partNum; + colStructList[i].fColSegment = it->segNum; + colStructList[i].fColDbRoot = it->dbRoot; + dctnryStructList[i].fColPartition = it->partNum; + dctnryStructList[i].fColSegment = it->segNum; + dctnryStructList[i].fColDbRoot = it->dbRoot; + } + } + + curTupleList = static_cast(colValueList[0]); + totalRow = curTupleList.size(); + totalColumns = colStructList.size(); + rowIdArray = new RID[totalRow]; + // use scoped_array to ensure ptr deletion regardless of where we return + boost::scoped_array rowIdArrayPtr(rowIdArray); + memset(rowIdArray, 0, (sizeof(RID) * totalRow)); + + //-------------------------------------------------------------------------- + // allocate row id(s) + //-------------------------------------------------------------------------- + curColStruct = colStructList[colId]; + colOp = m_colOp[op(curColStruct.fCompressionType)]; + + colOp->initColumn(curCol); + + // Get the correct segment, partition, column file + vector colExtentInfo; // Save those empty extents in case of failure to rollback + vector dictExtentInfo; // Save those empty extents in case of failure to rollback + vector fileInfo; + dbRoot = curColStruct.fColDbRoot; + // use the first column to calculate row id + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[colId].fColDbRoot) && + (it->partNum == colStructList[colId].fColPartition) && + (it->segNum == colStructList[colId].fColSegment) && it->current) + { + break; + } + it++; + } + + if (it != aColExtsInfo.end()) + { + hwm = it->hwm; + } + + oldHwm = hwm; // Save this info for rollback + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, + curColStruct.dataOid, curColStruct.fCompressionType, curColStruct.fColDbRoot, + curColStruct.fColPartition, curColStruct.fColSegment); + colOp->findTypeHandler(curColStruct.colWidth, curColStruct.colDataType); + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + { + return rc; + } + + // get hwm first + // @bug 286 : fix for bug 286 - correct the typo in getHWM + // RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); + + Column newCol; + +#ifdef PROFILE + timer.start("allocRowId"); +#endif + newColStructList = colStructList; + newDctnryStructList = dctnryStructList; + bool bUseStartExtent = true; + + if (idbdatafile::IDBPolicy::useHdfs()) + insertSelect = true; + + rc = colOp->allocRowId(txnid, bUseStartExtent, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, + rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, + dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm, + &newExtentsStartingLbids); + + if (rc != NO_ERROR) // Clean up is already done + return rc; + +#ifdef PROFILE + timer.stop("allocRowId"); +#endif + + //-------------------------------------------------------------------------- + // Expand initial abbreviated extent if any RID in 1st extent is > 256K. + // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. + //-------------------------------------------------------------------------- + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((curCol.dataFile.fPartition == 0) && (curCol.dataFile.fSegment == 0) && ((totalRow - rowsLeft) > 0) && + (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + { + for (unsigned k = 0; k < colStructList.size(); k++) + { + // Skip the selected column + if (k == colId) + continue; + Column expandCol; + colOp = m_colOp[op(colStructList[k].fCompressionType)]; + colOp->setColParam(expandCol, 0, colStructList[k].colWidth, colStructList[k].colDataType, + colStructList[k].colType, colStructList[k].dataOid, + colStructList[k].fCompressionType, colStructList[k].fColDbRoot, + colStructList[k].fColPartition, colStructList[k].fColSegment); + colOp->findTypeHandler(colStructList[k].colWidth, colStructList[k].colDataType); + rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file + + if (rc == NO_ERROR) + { + if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) { - dctStr_iter = dictStrList[i].begin(); - col_iter = colValueList[i].begin(); - Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; - rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, - dctnryStructList[i].fColDbRoot, dctnryStructList[i].fColPartition, - dctnryStructList[i].fColSegment, - useTmpSuffix); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - { - //cout << "Error opening dctnry file " << dctnryStructList[i].dctnryOid << endl; - return rc; - } - - for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) - { - if (dctStr_iter->length() == 0) - { - Token nullToken; - col_iter->data = nullToken; - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - col_iter->data = dctTuple.token; - } - - dctStr_iter++; - col_iter++; - - } - - //close dictionary files - rc = dctnry->closeDctnry(false); - - if (rc != NO_ERROR) - return rc; - - if (newExtent) - { - //@Bug 4854 back up hwm chunk for the file to be modified - if (fRBMetaWriter) - fRBMetaWriter->backupDctnryHWMChunk(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment); - - rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, - newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, - newDctnryStructList[i].fColSegment, - false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - for (uint32_t rows = 0; rows < rowsLeft; rows++) - { - if (dctStr_iter->length() == 0) - { - Token nullToken; - col_iter->data = nullToken; - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - col_iter->data = dctTuple.token; - } - - dctStr_iter++; - col_iter++; - } - - //close dictionary files - rc = dctnry->closeDctnry(false); - - if (rc != NO_ERROR) - return rc; - } + rc = colOp->expandAbbrevExtent(expandCol); } + } + + if (rc != NO_ERROR) + { + return rc; + } + + colOp->clearColumn(expandCol); // closes the file (if uncompressed) + } + } + + //-------------------------------------------------------------------------- + // Tokenize data if needed + //-------------------------------------------------------------------------- + if (insertSelect && isAutoCommitOn) + BRMWrapper::setUseVb(false); + else + BRMWrapper::setUseVb(true); + + dictStr::iterator dctStr_iter; + ColTupleList::iterator col_iter; + + for (i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) + { + dctStr_iter = dictStrList[i].begin(); + col_iter = colValueList[i].begin(); + Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; + rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, dctnryStructList[i].fColDbRoot, + dctnryStructList[i].fColPartition, dctnryStructList[i].fColSegment, + useTmpSuffix); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + { + // cout << "Error opening dctnry file " << dctnryStructList[i].dctnryOid << endl; + return rc; + } + + for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) + { + if (dctStr_iter->length() == 0) + { + Token nullToken; + col_iter->data = nullToken; + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } + +#ifdef PROFILE + timer.stop("tokenize"); +#endif + col_iter->data = dctTuple.token; + } + + dctStr_iter++; + col_iter++; + } + + // close dictionary files + rc = dctnry->closeDctnry(false); + + if (rc != NO_ERROR) + return rc; + + if (newExtent) + { + //@Bug 4854 back up hwm chunk for the file to be modified + if (fRBMetaWriter) + fRBMetaWriter->backupDctnryHWMChunk( + newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, + newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment); + + rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, + newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment, + false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + return rc; + + for (uint32_t rows = 0; rows < rowsLeft; rows++) + { + if (dctStr_iter->length() == 0) + { + Token nullToken; + col_iter->data = nullToken; + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } + +#ifdef PROFILE + timer.stop("tokenize"); +#endif + col_iter->data = dctTuple.token; + } + + dctStr_iter++; + col_iter++; + } + + // close dictionary files + rc = dctnry->closeDctnry(false); + + if (rc != NO_ERROR) + return rc; + } + } + } + + if (insertSelect && isAutoCommitOn) + BRMWrapper::setUseVb(false); + else + BRMWrapper::setUseVb(true); + + //-------------------------------------------------------------------------- + // Update column info structure @Bug 1862 set hwm, and + // Prepare ValueList for new extent (if applicable) + //-------------------------------------------------------------------------- + //@Bug 2205 Check whether all rows go to the new extent + RID lastRid = 0; + RID lastRidNew = 0; + + if (totalRow - rowsLeft > 0) + { + lastRid = rowIdArray[totalRow - rowsLeft - 1]; + lastRidNew = rowIdArray[totalRow - 1]; + } + else + { + lastRid = 0; + lastRidNew = rowIdArray[totalRow - 1]; + } + + // if a new extent is created, all the columns in this table should have their own new extent + // First column already processed + + //@Bug 1701. Close the file (if uncompressed) + m_colOp[op(curCol.compressionType)]->clearColumn(curCol); + // Update hwm to set them in the end + bool succFlag = false; + unsigned colWidth = 0; + int curFbo = 0, curBio; + + for (i = 0; i < totalColumns; i++) + { + // shoud be obtained from saved hwm + aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); + it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment) && it->current) + break; + + it++; } - if (insertSelect && isAutoCommitOn) - BRMWrapper::setUseVb( false ); - else - BRMWrapper::setUseVb( true ); - - //-------------------------------------------------------------------------- - // Update column info structure @Bug 1862 set hwm, and - // Prepare ValueList for new extent (if applicable) - //-------------------------------------------------------------------------- - //@Bug 2205 Check whether all rows go to the new extent - RID lastRid = 0; - RID lastRidNew = 0; - - if (totalRow - rowsLeft > 0) + if (it != aColExtsInfo.end()) // update hwm info { - lastRid = rowIdArray[totalRow - rowsLeft - 1]; - lastRidNew = rowIdArray[totalRow - 1]; + oldHwm = it->hwm; + } + + // save hwm for the old extent + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + if (succFlag) + { + if ((HWM)curFbo >= oldHwm) + { + it->hwm = (HWM)curFbo; + } + + //@Bug 4947. set current to false for old extent. + if (newExtent) + { + it->current = false; + } + } + else + return ERR_INVALID_PARAM; + + // update hwm for the new extent + if (newExtent) + { + it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && + (it->partNum == newColStructList[i].fColPartition) && + (it->segNum == newColStructList[i].fColSegment) && it->current) + break; + + it++; + } + + succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + if (succFlag) + { + if (it != aColExtsInfo.end()) + { + it->hwm = (HWM)curFbo; + // cout << "setting hwm to " << (int)curFbo <<" for seg " <segNum << endl; + it->current = true; + } + } + else + return ERR_INVALID_PARAM; + } + + tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + //-------------------------------------------------------------------------- + // Prepare the valuelist for the new extent + //-------------------------------------------------------------------------- + ColTupleList colTupleList; + ColTupleList newColTupleList; + ColTupleList firstPartTupleList; + + for (unsigned i = 0; i < totalColumns; i++) + { + colTupleList = static_cast(colValueList[i]); + + for (uint64_t j = rowsLeft; j > 0; j--) + { + newColTupleList.push_back(colTupleList[totalRow - j]); + } + + colNewValueList.push_back(newColTupleList); + newColTupleList.clear(); + + // update the oldvalue list for the old extent + for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) + { + firstPartTupleList.push_back(colTupleList[j]); + } + + colOldValueList.push_back(firstPartTupleList); + firstPartTupleList.clear(); + } + + // end of allocate row id + +#ifdef PROFILE + timer.start("writeColumnRec"); +#endif + + if (rc == NO_ERROR) + { + //---------------------------------------------------------------------- + // Mark extents invalid + //---------------------------------------------------------------------- + bool successFlag = true; + unsigned width = 0; + int curFbo = 0, curBio, lastFbo = -1; + + if (isFirstBatchPm && (totalRow == rowsLeft)) + { + // in this particular case we already marked extents as invalid up there. } else { - lastRid = 0; - lastRidNew = rowIdArray[totalRow - 1]; + int firstHalfCount = totalRow - rowsLeft; + for (unsigned i = 0; i < colStructList.size(); i++) + { + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + width = colStructList[i].colWidth; + if (firstHalfCount) + { + ExtCPInfo* cpInfoP = + getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[0]); + RID thisRid = rowsLeft ? lastRid : lastRidNew; + successFlag = colOp->calculateRowId(thisRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, colStructList[i], curFbo, cpInfoP)); + } + } + maxMins[i].fSplitMaxMinInfoPtrs[0] = cpInfoP; + } + if (rowsLeft) + { + ExtCPInfo* cpInfoP = + getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[1]); + if (cpInfoP) + { + RETURN_ON_ERROR(GetLBIDRange(newExtentsStartingLbids[i], colStructList[i], *cpInfoP)); + } + + maxMins[i].fSplitMaxMinInfoPtrs[1] = cpInfoP; + } + } } - //if a new extent is created, all the columns in this table should have their own new extent - //First column already processed + markTxnExtentsAsInvalid(txnid); - //@Bug 1701. Close the file (if uncompressed) - m_colOp[op(curCol.compressionType)]->clearColumn(curCol); - //Update hwm to set them in the end - bool succFlag = false; - unsigned colWidth = 0; - int curFbo = 0, curBio; - - for (i = 0; i < totalColumns; i++) + //---------------------------------------------------------------------- + // Write row(s) to database file(s) + //---------------------------------------------------------------------- + std::vector cpinfoList; + for (auto& splitCPInfo : maxMins) { - //shoud be obtained from saved hwm - aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); + for (i = 0; i < 2; i++) + { + ExtCPInfo* cpInfo = splitCPInfo.fSplitMaxMinInfoPtrs[i]; + if (cpInfo) + { + cpinfoList.push_back(*cpInfo); + cpinfoList[cpinfoList.size() - 1].fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; + } + } + } + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); + if (rc != NO_ERROR) + { + return rc; + } + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, + colNewValueList, tableOid, useTmpSuffix, true, &maxMins); // @bug 5572 HDFS tmp file + + if (rc == NO_ERROR) + { + if (dctnryStructList.size() > 0) + { + vector oids{static_cast(tableOid)}; + for (const DctnryStruct& dctnryStruct : dctnryStructList) + { + oids.push_back(dctnryStruct.dctnryOid); + } + + rc = flushOIDsFromCache(oids); + + if (rc != 0) + rc = ERR_BLKCACHE_FLUSH_LIST; // translate to WE error + } + + if (rc == NO_ERROR) + { + int index = 0; + for (auto& splitCPInfo : maxMins) + { + for (i = 0; i < 2; i++) + { + ExtCPInfo* cpInfo = splitCPInfo.fSplitMaxMinInfoPtrs[i]; + if (cpInfo) + { + cpinfoList[index] = *cpInfo; + cpinfoList[index].fCPInfo.seqNum++; + index++; + } + } + } + setInvalidCPInfosSpecialMarks(cpinfoList); + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); + } + } + } + + return rc; +} + +int WriteEngineWrapper::insertColumnRecsBinary( + const TxnID& txnid, ColStructList& colStructList, std::vector& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + std::vector >& dbRootExtentTrackers, RBMetaWriter* fRBMetaWriter, + bool bFirstExtentOnThisPM, bool insertSelect, bool isAutoCommitOn, OID tableOid, bool isFirstBatchPm) +{ + int rc; + RID* rowIdArray = NULL; + Column curCol; + ColStruct curColStruct; + ColStructList newColStructList; + std::vector colNewValueList; + DctnryStructList newDctnryStructList; + HWM hwm = 0; + HWM oldHwm = 0; + HWM newHwm = 0; + size_t totalRow; + ColStructList::size_type totalColumns; + uint64_t rowsLeft = 0; + bool newExtent = false; + RIDList ridList; + ColumnOp* colOp = NULL; + std::vector dictLbids; + + // Set tmp file suffix to modify HDFS db file + bool useTmpSuffix = false; + + m_opType = INSERT; + + if (idbdatafile::IDBPolicy::useHdfs()) + { + if (!bFirstExtentOnThisPM) + useTmpSuffix = true; + } + + unsigned i = 0; +#ifdef PROFILE + StopWatch timer; +#endif + + // Convert data type and column width to write engine specific + for (i = 0; i < colStructList.size(); i++) + Convertor::convertColType(&colStructList[i]); + + uint32_t colId = 0; + // MCOL-1675: find the smallest column width to calculate the RowID from so + // that all HWMs will be incremented by this operation + findSmallestColumn(colId, colStructList); + + // rc = checkValid(txnid, colStructList, colValueList, ridList); + // if (rc != NO_ERROR) + // return rc; + + setTransId(txnid); + uint16_t dbRoot, segmentNum; + uint32_t partitionNum; + string segFile; + bool newFile; + TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); + // populate colStructList with file information + IDBDataFile* pFile = NULL; + std::vector extentInfo; + int currentDBrootIdx = 0; + std::vector extents; + + //-------------------------------------------------------------------------- + // For first batch on this PM: + // o get starting extent from ExtentTracker, and allocate extent if needed + // o construct colStructList and dctnryStructList accordingly + // o save extent information in tableMetaData for future use + // If not first batch on this PM: + // o construct colStructList and dctnryStructList from tableMetaData + //-------------------------------------------------------------------------- + if (isFirstBatchPm) + { + currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); + dbRoot = extentInfo[currentDBrootIdx].fDbRoot; + partitionNum = extentInfo[currentDBrootIdx].fPartition; + + //---------------------------------------------------------------------- + // check whether this extent is the first on this PM + //---------------------------------------------------------------------- + if (bFirstExtentOnThisPM) + { + // cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl; + std::vector cols; + BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; + + for (i = 0; i < colStructList.size(); i++) + { + createStripeColumnExtentsArgIn.oid = colStructList[i].dataOid; + createStripeColumnExtentsArgIn.width = colStructList[i].colWidth; + createStripeColumnExtentsArgIn.colDataType = colStructList[i].colDataType; + cols.push_back(createStripeColumnExtentsArgIn); + } + + rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partitionNum, segmentNum, + extents); + + if (rc != NO_ERROR) + return rc; + + // Create column files + ExtCPInfoList cpinfoList; + + for (i = 0; i < extents.size(); i++) + { + ExtCPInfo cpInfo(colStructList[i].colDataType, colStructList[i].colWidth); + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType, + colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, dbRoot, partitionNum, segmentNum); + colOp->findTypeHandler(colStructList[i].colWidth, colStructList[i].colDataType); + rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, + extents[i].allocSize, dbRoot, partitionNum, segmentNum, segFile, pFile, + newFile); + + if (rc != NO_ERROR) + return rc; + + cpInfo.toInvalid(); + + cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; + + // mark the extents to invalid + cpInfo.fCPInfo.firstLbid = extents[i].startLbid; + cpinfoList.push_back(cpInfo); + colStructList[i].fColPartition = partitionNum; + colStructList[i].fColSegment = segmentNum; + colStructList[i].fColDbRoot = dbRoot; + dctnryStructList[i].fColPartition = partitionNum; + dctnryStructList[i].fColSegment = segmentNum; + dctnryStructList[i].fColDbRoot = dbRoot; + } + + // mark the extents to invalid + rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); + + if (rc != NO_ERROR) + return rc; + + // create corresponding dictionary files + for (i = 0; i < dctnryStructList.size(); i++) + { + if (dctnryStructList[i].dctnryOid > 0) + { + rc = createDctnry(txnid, dctnryStructList[i].dctnryOid, dctnryStructList[i].colWidth, dbRoot, + partitionNum, segmentNum, dctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + return rc; + } + } + } // if ( bFirstExtentOnThisPM) + else // if (!bFirstExtentOnThisPM) + { + std::vector tmpExtentInfo; + + for (i = 0; i < dbRootExtentTrackers.size(); i++) + { + tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); + colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; + colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; + colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; + // cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aExt.isDict = false; + + if (bFirstExtentOnThisPM) + { + aExt.hwm = extents[i].startBlkOffset; + aExt.isNewExt = true; + // cout << "adding a ext to metadata" << endl; + } + else + { + std::vector tmpExtentInfo; + tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); + aExt.isNewExt = false; + aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; + // cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl; + } + + aExt.current = true; + aColExtsInfo.push_back(aExt); + // cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl; + } + + tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + for (i = 0; i < dctnryStructList.size(); i++) + { + if (dctnryStructList[i].dctnryOid > 0) + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && + (it->partNum == dctnryStructList[i].fColPartition) && + (it->segNum == dctnryStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = dctnryStructList[i].fColDbRoot; + aExt.partNum = dctnryStructList[i].fColPartition; + aExt.segNum = dctnryStructList[i].fColSegment; + aExt.compType = dctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + } + + tableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); + } + } + + } // if (isFirstBatchPm) + else // get the extent info from tableMetaData + { + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if (it->current) + break; + + it++; + } + + if (it == aColExtsInfo.end()) + return 1; + + for (i = 0; i < colStructList.size(); i++) + { + colStructList[i].fColPartition = it->partNum; + colStructList[i].fColSegment = it->segNum; + colStructList[i].fColDbRoot = it->dbRoot; + dctnryStructList[i].fColPartition = it->partNum; + dctnryStructList[i].fColSegment = it->segNum; + dctnryStructList[i].fColDbRoot = it->dbRoot; + } + } + + totalColumns = colStructList.size(); + totalRow = colValueList.size() / totalColumns; + rowIdArray = new RID[totalRow]; + // use scoped_array to ensure ptr deletion regardless of where we return + boost::scoped_array rowIdArrayPtr(rowIdArray); + memset(rowIdArray, 0, (sizeof(RID) * totalRow)); + + //-------------------------------------------------------------------------- + // allocate row id(s) + //-------------------------------------------------------------------------- + + curColStruct = colStructList[colId]; + + colOp = m_colOp[op(curColStruct.fCompressionType)]; + + colOp->initColumn(curCol); + + // Get the correct segment, partition, column file + vector colExtentInfo; // Save those empty extents in case of failure to rollback + vector dictExtentInfo; // Save those empty extents in case of failure to rollback + vector fileInfo; + dbRoot = curColStruct.fColDbRoot; + // use the smallest column to calculate row id + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[colId].fColDbRoot) && + (it->partNum == colStructList[colId].fColPartition) && + (it->segNum == colStructList[colId].fColSegment) && it->current) + break; + + it++; + } + + if (it != aColExtsInfo.end()) + { + hwm = it->hwm; + // cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and + // seg is " << colStructList[colId].fColSegment << endl; + } + + oldHwm = hwm; // Save this info for rollback + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, + curColStruct.dataOid, curColStruct.fCompressionType, curColStruct.fColDbRoot, + curColStruct.fColPartition, curColStruct.fColSegment); + colOp->findTypeHandler(curColStruct.colWidth, curColStruct.colDataType); + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + { + return rc; + } + + // get hwm first + // @bug 286 : fix for bug 286 - correct the typo in getHWM + // RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); + + Column newCol; + +#ifdef PROFILE + timer.start("allocRowId"); +#endif + newColStructList = colStructList; + newDctnryStructList = dctnryStructList; + bool bUseStartExtent = true; + + if (idbdatafile::IDBPolicy::useHdfs()) + insertSelect = true; + + rc = colOp->allocRowId(txnid, bUseStartExtent, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, + rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, + dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm, NULL); + + // cout << "after allocrowid, total row = " < 256K. + // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. + //-------------------------------------------------------------------------- + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((curCol.dataFile.fPartition == 0) && (curCol.dataFile.fSegment == 0) && ((totalRow - rowsLeft) > 0) && + (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + { + for (size_t k = 0; k < colStructList.size(); k++) + { + // Skip the selected column + if (k == (size_t)colId) + continue; + + Column expandCol; + colOp = m_colOp[op(colStructList[k].fCompressionType)]; + // Shouldn't we change 0 to colId here? + colOp->setColParam(expandCol, 0, colStructList[k].colWidth, colStructList[k].colDataType, + colStructList[k].colType, colStructList[k].dataOid, + colStructList[k].fCompressionType, colStructList[k].fColDbRoot, + colStructList[k].fColPartition, colStructList[k].fColSegment); + colOp->findTypeHandler(colStructList[k].colWidth, colStructList[k].colDataType); + rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file + + if (rc == NO_ERROR) + { + if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) + { + rc = colOp->expandAbbrevExtent(expandCol); + } + } + + if (rc != NO_ERROR) + { + return rc; + } + + colOp->closeColumnFile(expandCol); + } + } + + //-------------------------------------------------------------------------- + // Tokenize data if needed + //-------------------------------------------------------------------------- + if (insertSelect && isAutoCommitOn) + BRMWrapper::setUseVb(false); + else + BRMWrapper::setUseVb(true); + + dictStr::iterator dctStr_iter; + uint64_t* colValPtr; + size_t rowsPerColumn = colValueList.size() / colStructList.size(); + + for (i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) + { + dctStr_iter = dictStrList[i].begin(); + Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; + rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, dctnryStructList[i].fColDbRoot, + dctnryStructList[i].fColPartition, dctnryStructList[i].fColSegment, + useTmpSuffix); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + { + return rc; + } + + for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) + { + colValPtr = &colValueList[(i * rowsPerColumn) + rows]; + + if (dctStr_iter->length() == 0) + { + Token nullToken; + memcpy(colValPtr, &nullToken, 8); + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } + +#ifdef PROFILE + timer.stop("tokenize"); +#endif + memcpy(colValPtr, &dctTuple.token, 8); + dictLbids.push_back(dctTuple.token.fbo); + } + + dctStr_iter++; + } + + // close dictionary files + rc = dctnry->closeDctnry(false); + + if (rc != NO_ERROR) + return rc; + + if (newExtent) + { + //@Bug 4854 back up hwm chunk for the file to be modified + if (fRBMetaWriter) + fRBMetaWriter->backupDctnryHWMChunk( + newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, + newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment); + + rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, + newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment, + false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + return rc; + + for (uint32_t rows = 0; rows < rowsLeft; rows++) + { + colValPtr = &colValueList[(i * rowsPerColumn) + rows]; + + if (dctStr_iter->length() == 0) + { + Token nullToken; + memcpy(colValPtr, &nullToken, 8); + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } + +#ifdef PROFILE + timer.stop("tokenize"); +#endif + memcpy(colValPtr, &dctTuple.token, 8); + dictLbids.push_back(dctTuple.token.fbo); + } + + dctStr_iter++; + } + + // close dictionary files + rc = dctnry->closeDctnry(false); + + if (rc != NO_ERROR) + return rc; + } + } + } + + if (insertSelect && isAutoCommitOn) + BRMWrapper::setUseVb(false); + else + BRMWrapper::setUseVb(true); + + //-------------------------------------------------------------------------- + // Update column info structure @Bug 1862 set hwm, and + // Prepare ValueList for new extent (if applicable) + //-------------------------------------------------------------------------- + //@Bug 2205 Check whether all rows go to the new extent + RID lastRid = 0; + RID lastRidNew = 0; + + if (totalRow - rowsLeft > 0) + { + lastRid = rowIdArray[totalRow - rowsLeft - 1]; + lastRidNew = rowIdArray[totalRow - 1]; + } + else + { + lastRid = 0; + lastRidNew = rowIdArray[totalRow - 1]; + } + + // cout << "rowid allocated is " << lastRid << endl; + // if a new extent is created, all the columns in this table should have their own new extent + // First column already processed + + //@Bug 1701. Close the file (if uncompressed) + m_colOp[op(curCol.compressionType)]->closeColumnFile(curCol); + // cout << "Saving hwm info for new ext batch" << endl; + // Update hwm to set them in the end + bool succFlag = false; + unsigned colWidth = 0; + int curFbo = 0, curBio; + + for (i = 0; i < totalColumns; i++) + { + // shoud be obtained from saved hwm + aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); + it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment) && it->current) + break; + + it++; + } + + if (it != aColExtsInfo.end()) // update hwm info + { + oldHwm = it->hwm; + + // save hwm for the old extent + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + // cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << + // ":" << curFbo << ":" << oldHwm << endl; + if (succFlag) + { + if ((HWM)curFbo >= oldHwm) + { + it->hwm = (HWM)curFbo; + } + + //@Bug 4947. set current to false for old extent. + if (newExtent) + { + it->current = false; + } + + // cout << "updated old ext info for oid " << colStructList[i].dataOid << " + // dbroot:part:seg:hwm:current = " + //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent + //is " << newExtent << endl; + } + else + return ERR_INVALID_PARAM; + } + + // update hwm for the new extent + if (newExtent) + { + it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && + (it->partNum == newColStructList[i].fColPartition) && + (it->segNum == newColStructList[i].fColSegment) && it->current) + break; + + it++; + } + + colWidth = newColStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + if (succFlag) + { + if (it != aColExtsInfo.end()) + { + it->hwm = (HWM)curFbo; + // cout << "setting hwm to " << (int)curFbo <<" for seg " <segNum << endl; + it->current = true; + } + } + else + return ERR_INVALID_PARAM; + } + + tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + //-------------------------------------------------------------------------- + // Prepare the valuelist for the new extent + //-------------------------------------------------------------------------- + + for (unsigned i = 1; i <= totalColumns; i++) + { + // Copy values to second value list + for (uint64_t j = rowsLeft; j > 0; j--) + { + colNewValueList.push_back(colValueList[(totalRow * i) - j]); + } + } + + // end of allocate row id + +#ifdef PROFILE + timer.start("writeColumnRec"); +#endif + // cout << "Writing column record" << endl; + + if (rc == NO_ERROR) + { + //---------------------------------------------------------------------- + // Mark extents invalid + //---------------------------------------------------------------------- + bool successFlag = true; + unsigned width = 0; + int curFbo = 0, curBio, lastFbo = -1; + + if (isFirstBatchPm && (totalRow == rowsLeft)) + { + } + else + { + for (unsigned i = 0; i < colStructList.size(); i++) + { + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + width = colStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, colStructList[i], curFbo)); + } + } + else + return ERR_INVALID_PARAM; + } + } + + // If we create a new extent for this batch + for (unsigned i = 0; i < newColStructList.size(); i++) + { + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + width = newColStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, newColStructList[i], curFbo)); + } + } + else + return ERR_INVALID_PARAM; + } + + markTxnExtentsAsInvalid(txnid); + + //---------------------------------------------------------------------- + // Write row(s) to database file(s) + //---------------------------------------------------------------------- + bool versioning = !(isAutoCommitOn && insertSelect); + AddDictToList(txnid, dictLbids); + rc = + writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, + colNewValueList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file + } + + return rc; +} + +int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, const CSCTypesList& cscColTypeList, + ColStructList& colStructList, ColValueList& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + const int32_t tableOid) +{ + int rc; + RID* rowIdArray = NULL; + ColTupleList curTupleList; + Column curCol; + ColStruct curColStruct; + ColValueList colOldValueList; + ColValueList colNewValueList; + ColStructList newColStructList; + DctnryStructList newDctnryStructList; + HWM hwm = 0; + HWM newHwm = 0; + HWM oldHwm = 0; + ColTupleList::size_type totalRow; + ColStructList::size_type totalColumns; + uint64_t rowsLeft = 0; + bool newExtent = false; + RIDList ridList; + ColumnOp* colOp = NULL; + uint32_t i = 0; +#ifdef PROFILE + StopWatch timer; +#endif + + m_opType = INSERT; + + // debug information for testing + if (isDebug(DEBUG_2)) + { + printInputValue(colStructList, colValueList, ridList, dctnryStructList, dictStrList); + } + + // end + + // Convert data type and column width to write engine specific + for (i = 0; i < colStructList.size(); i++) + Convertor::convertColType(&colStructList[i]); + + rc = checkValid(txnid, colStructList, colValueList, ridList); + + if (rc != NO_ERROR) + return rc; + + setTransId(txnid); + + curTupleList = static_cast(colValueList[0]); + totalRow = curTupleList.size(); + totalColumns = colStructList.size(); + rowIdArray = new RID[totalRow]; + // use scoped_array to ensure ptr deletion regardless of where we return + boost::scoped_array rowIdArrayPtr(rowIdArray); + memset(rowIdArray, 0, (sizeof(RID) * totalRow)); + + // allocate row id(s) + curColStruct = colStructList[0]; + colOp = m_colOp[op(curColStruct.fCompressionType)]; + + colOp->initColumn(curCol); + + // Get the correct segment, partition, column file + uint16_t dbRoot, segmentNum; + uint32_t partitionNum; + vector colExtentInfo; // Save those empty extents in case of failure to rollback + vector dictExtentInfo; // Save those empty extents in case of failure to rollback + vector fileInfo; + ExtentInfo info; + // Don't search for empty space, always append to the end. May need to revisit this part + dbRoot = curColStruct.fColDbRoot; + int extState; + bool extFound; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot(curColStruct.dataOid, dbRoot, partitionNum, + segmentNum, hwm, extState, extFound)); + + for (i = 0; i < colStructList.size(); i++) + { + colStructList[i].fColPartition = partitionNum; + colStructList[i].fColSegment = segmentNum; + colStructList[i].fColDbRoot = dbRoot; + } + + oldHwm = hwm; // Save this info for rollback + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, + curColStruct.dataOid, curColStruct.fCompressionType, dbRoot, partitionNum, segmentNum); + colOp->findTypeHandler(curColStruct.colWidth, curColStruct.colDataType); + string segFile; + rc = colOp->openColumnFile(curCol, segFile, false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + { + return rc; + } + + // get hwm first + // @bug 286 : fix for bug 286 - correct the typo in getHWM + // RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); + + //...Note that we are casting totalRow to int to be in sync with + //...allocRowId(). So we are assuming that totalRow + //...(curTupleList.size()) will fit into an int. We arleady made + //...that assumption earlier in this method when we used totalRow + //...in the call to calloc() to allocate rowIdArray. + Column newCol; + bool newFile; + +#ifdef PROFILE + timer.start("allocRowId"); +#endif + + newColStructList = colStructList; + newDctnryStructList = dctnryStructList; + std::vector > dbRootExtentTrackers; + bool bUseStartExtent = true; + rc = colOp->allocRowId(txnid, bUseStartExtent, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, + rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, + dbRootExtentTrackers, false, false, 0, false, NULL); + + if ((rc == ERR_FILE_DISK_SPACE) && newExtent) + { + for (i = 0; i < newColStructList.size(); i++) + { + info.oid = newColStructList[i].dataOid; + info.partitionNum = newColStructList[i].fColPartition; + info.segmentNum = newColStructList[i].fColSegment; + info.dbRoot = newColStructList[i].fColDbRoot; + + if (newFile) + fileInfo.push_back(info); + + colExtentInfo.push_back(info); + } + + int rc1 = BRMWrapper::getInstance()->deleteEmptyColExtents(colExtentInfo); + + if ((rc1 == 0) && newFile) + { + rc1 = colOp->deleteFile(fileInfo[0].oid, fileInfo[0].dbRoot, fileInfo[0].partitionNum, + fileInfo[0].segmentNum); + + if (rc1 != NO_ERROR) + return rc; + + FileOp fileOp; + + for (i = 0; i < newDctnryStructList.size(); i++) + { + if (newDctnryStructList[i].dctnryOid > 0) + { + info.oid = newDctnryStructList[i].dctnryOid; + info.partitionNum = newDctnryStructList[i].fColPartition; + info.segmentNum = newDctnryStructList[i].fColSegment; + info.dbRoot = newDctnryStructList[i].fColDbRoot; + info.newFile = true; + fileInfo.push_back(info); + dictExtentInfo.push_back(info); + } + } + + if (dictExtentInfo.size() > 0) + { + rc1 = BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(dictExtentInfo); + + if (rc1 != NO_ERROR) + return rc; + + for (unsigned j = 0; j < fileInfo.size(); j++) + { + rc1 = fileOp.deleteFile(fileInfo[j].oid, fileInfo[j].dbRoot, fileInfo[j].partitionNum, + fileInfo[j].segmentNum); + } + } + } + } + + TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tableOid); + + //..Expand initial abbreviated extent if any RID in 1st extent is > 256K + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((partitionNum == 0) && (segmentNum == 0) && ((totalRow - rowsLeft) > 0) && + (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + { + for (size_t k = 1; k < colStructList.size(); k++) + { + Column expandCol; + colOp = m_colOp[op(colStructList[k].fCompressionType)]; + colOp->setColParam(expandCol, 0, colStructList[k].colWidth, colStructList[k].colDataType, + colStructList[k].colType, colStructList[k].dataOid, + colStructList[k].fCompressionType, dbRoot, partitionNum, segmentNum); + colOp->findTypeHandler(colStructList[k].colWidth, colStructList[k].colDataType); + rc = colOp->openColumnFile(expandCol, segFile, false); // @bug 5572 HDFS tmp file + + if (rc == NO_ERROR) + { + if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) + { + rc = colOp->expandAbbrevExtent(expandCol); + } + } + + if (rc != NO_ERROR) + { + if (newExtent) + { + // Remove the empty extent added to the first column + int rc1 = BRMWrapper::getInstance()->deleteEmptyColExtents(colExtentInfo); + + if ((rc1 == 0) && newFile) + { + rc1 = colOp->deleteFile(fileInfo[0].oid, fileInfo[0].dbRoot, fileInfo[0].partitionNum, + fileInfo[0].segmentNum); + } + } + + colOp->clearColumn(expandCol); // closes the file + return rc; + } + + colOp->clearColumn(expandCol); // closes the file + } + } + + BRMWrapper::setUseVb(true); + // Tokenize data if needed + dictStr::iterator dctStr_iter; + ColTupleList::iterator col_iter; + + for (i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) + { + dctStr_iter = dictStrList[i].begin(); + col_iter = colValueList[i].begin(); + Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; + + dctnryStructList[i].fColPartition = partitionNum; + dctnryStructList[i].fColSegment = segmentNum; + dctnryStructList[i].fColDbRoot = dbRoot; + rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, dctnryStructList[i].fColDbRoot, + dctnryStructList[i].fColPartition, dctnryStructList[i].fColSegment, + false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + return rc; + + ColExtsInfo aColExtsInfo = aTableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && + (it->partNum == dctnryStructList[i].fColPartition) && + (it->segNum == dctnryStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = dctnryStructList[i].fColDbRoot; + aExt.partNum = dctnryStructList[i].fColPartition; + aExt.segNum = dctnryStructList[i].fColSegment; + aExt.compType = dctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + aTableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); + } + + for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) + { + if (dctStr_iter->length() == 0) + { + Token nullToken; + col_iter->data = nullToken; + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } + +#ifdef PROFILE + timer.stop("tokenize"); +#endif + col_iter->data = dctTuple.token; + } + + dctStr_iter++; + col_iter++; + } + + // close dictionary files + rc = dctnry->closeDctnry(); + + if (rc != NO_ERROR) + return rc; + + if (newExtent) + { + rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, + newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment, + false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + return rc; + + aColExtsInfo = aTableMetaData->getColExtsInfo(newDctnryStructList[i].dctnryOid); it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) - && (it->segNum == colStructList[i].fColSegment) && it->current) - break; + if ((it->dbRoot == newDctnryStructList[i].fColDbRoot) && + (it->partNum == newDctnryStructList[i].fColPartition) && + (it->segNum == newDctnryStructList[i].fColSegment)) + break; - it++; + it++; } - if (it != aColExtsInfo.end()) //update hwm info + if (it == aColExtsInfo.end()) // add this one to the list { - oldHwm = it->hwm; + ColExtInfo aExt; + aExt.dbRoot = newDctnryStructList[i].fColDbRoot; + aExt.partNum = newDctnryStructList[i].fColPartition; + aExt.segNum = newDctnryStructList[i].fColSegment; + aExt.compType = newDctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + aTableMetaData->setColExtsInfo(newDctnryStructList[i].dctnryOid, aColExtsInfo); } - // save hwm for the old extent - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - if (succFlag) + for (uint32_t rows = 0; rows < rowsLeft; rows++) { - if ((HWM)curFbo >= oldHwm) + if (dctStr_iter->length() == 0) + { + Token nullToken; + col_iter->data = nullToken; + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) { - it->hwm = (HWM)curFbo; + dctnry->closeDctnry(); + return rc; } - //@Bug 4947. set current to false for old extent. - if (newExtent) - { - it->current = false; - } +#ifdef PROFILE + timer.stop("tokenize"); +#endif + col_iter->data = dctTuple.token; + } + dctStr_iter++; + col_iter++; } - else - return ERR_INVALID_PARAM; - //update hwm for the new extent - if (newExtent) + // close dictionary files + rc = dctnry->closeDctnry(); + + if (rc != NO_ERROR) + return rc; + } + } + } + + // Update column info structure @Bug 1862 set hwm + //@Bug 2205 Check whether all rows go to the new extent + RID lastRid = 0; + RID lastRidNew = 0; + + if (totalRow - rowsLeft > 0) + { + lastRid = rowIdArray[totalRow - rowsLeft - 1]; + lastRidNew = rowIdArray[totalRow - 1]; + } + else + { + lastRid = 0; + lastRidNew = rowIdArray[totalRow - 1]; + } + + // cout << "rowid allocated is " << lastRid << endl; + // if a new extent is created, all the columns in this table should have their own new extent + + //@Bug 1701. Close the file + m_colOp[op(curCol.compressionType)]->clearColumn(curCol); + std::vector hwmVecNewext; + std::vector hwmVecOldext; + + if (newExtent) // Save all hwms to set them later. + { + BulkSetHWMArg aHwmEntryNew; + BulkSetHWMArg aHwmEntryOld; + bool succFlag = false; + unsigned colWidth = 0; + int extState; + bool extFound; + int curFbo = 0, curBio; + + for (i = 0; i < totalColumns; i++) + { + Column curColLocal; + colOp->initColumn(curColLocal); + + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + colOp->setColParam(curColLocal, 0, newColStructList[i].colWidth, newColStructList[i].colDataType, + newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, dbRoot, partitionNum, segmentNum); + colOp->findTypeHandler(newColStructList[i].colWidth, newColStructList[i].colDataType); + + rc = BRMWrapper::getInstance()->getLastHWM_DBroot(curColLocal.dataFile.fid, dbRoot, partitionNum, + segmentNum, oldHwm, extState, extFound); + + info.oid = curColLocal.dataFile.fid; + info.partitionNum = partitionNum; + info.segmentNum = segmentNum; + info.dbRoot = dbRoot; + info.hwm = oldHwm; + colExtentInfo.push_back(info); + // @Bug 2714 need to set hwm for the old extent + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + // cout << "insertcolumnrec oid:rid:fbo:hwm = " << colStructList[i].dataOid << ":" << lastRid << ":" + // << curFbo << ":" << hwm << endl; + if (succFlag) + { + if ((HWM)curFbo > oldHwm) { - it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) - && (it->segNum == newColStructList[i].fColSegment) && it->current) - break; - - it++; - } - - succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - if (succFlag) - { - if (it != aColExtsInfo.end()) - { - it->hwm = (HWM)curFbo; - //cout << "setting hwm to " << (int)curFbo <<" for seg " <segNum << endl; - it->current = true; - } - } - else - return ERR_INVALID_PARAM; + aHwmEntryOld.oid = colStructList[i].dataOid; + aHwmEntryOld.partNum = partitionNum; + aHwmEntryOld.segNum = segmentNum; + aHwmEntryOld.hwm = curFbo; + hwmVecOldext.push_back(aHwmEntryOld); } + } + else + return ERR_INVALID_PARAM; - tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + colWidth = newColStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + if (succFlag) + { + aHwmEntryNew.oid = newColStructList[i].dataOid; + aHwmEntryNew.partNum = newColStructList[i].fColPartition; + aHwmEntryNew.segNum = newColStructList[i].fColSegment; + aHwmEntryNew.hwm = curFbo; + hwmVecNewext.push_back(aHwmEntryNew); + } + + m_colOp[op(curColLocal.compressionType)]->clearColumn(curColLocal); } - //-------------------------------------------------------------------------- - //Prepare the valuelist for the new extent - //-------------------------------------------------------------------------- + // Prepare the valuelist for the new extent ColTupleList colTupleList; ColTupleList newColTupleList; ColTupleList firstPartTupleList; for (unsigned i = 0; i < totalColumns; i++) { - colTupleList = static_cast(colValueList[i]); + colTupleList = static_cast(colValueList[i]); - for (uint64_t j = rowsLeft; j > 0; j--) - { - newColTupleList.push_back(colTupleList[totalRow - j]); - } + for (uint64_t j = rowsLeft; j > 0; j--) + { + newColTupleList.push_back(colTupleList[totalRow - j]); + } - colNewValueList.push_back(newColTupleList); - newColTupleList.clear(); + colNewValueList.push_back(newColTupleList); + newColTupleList.clear(); - //update the oldvalue list for the old extent - for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) - { - firstPartTupleList.push_back(colTupleList[j]); - } + // upate the oldvalue list for the old extent + for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) + { + firstPartTupleList.push_back(colTupleList[j]); + } - colOldValueList.push_back(firstPartTupleList); - firstPartTupleList.clear(); + colOldValueList.push_back(firstPartTupleList); + firstPartTupleList.clear(); } + } - // end of allocate row id + // Mark extents invalid + bool successFlag = true; + unsigned width = 0; + int curFbo = 0, curBio, lastFbo = -1; + + if (totalRow - rowsLeft > 0) + { + for (unsigned i = 0; i < colStructList.size(); i++) + { + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + width = colStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, colStructList[i], curFbo)); + } + } + } + } + + lastRid = rowIdArray[totalRow - 1]; + + for (unsigned i = 0; i < newColStructList.size(); i++) + { + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + width = newColStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, newColStructList[i], curFbo)); + } + } + } + + // cout << "lbids size = " << lbids.size()<< endl; + markTxnExtentsAsInvalid(txnid); + + if (rc == NO_ERROR) + { + // MCOL-66 The DBRM can't handle concurrent transactions to sys tables + static boost::mutex dbrmMutex; + boost::mutex::scoped_lock lk(dbrmMutex); + + if (newExtent) + { + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, + colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file + } + else + { + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, + colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file + } + } #ifdef PROFILE - timer.start("writeColumnRec"); + timer.stop("writeColumnRec"); #endif + // for (ColTupleList::size_type i = 0; i < totalRow; i++) + // ridList.push_back((RID) rowIdArray[i]); - if (rc == NO_ERROR) + // if (rc == NO_ERROR) + // rc = flushDataFiles(NO_ERROR); + + if (!newExtent) + { + // flushVMCache(); + bool succFlag = false; + unsigned colWidth = 0; + int extState; + bool extFound; + int curFbo = 0, curBio; + std::vector hwmVec; + + for (unsigned i = 0; i < totalColumns; i++) { - //---------------------------------------------------------------------- - //Mark extents invalid - //---------------------------------------------------------------------- - bool successFlag = true; - unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; + // colOp = m_colOp[op(colStructList[i].fCompressionType)]; + // Set all columns hwm together + BulkSetHWMArg aHwmEntry; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( + colStructList[i].dataOid, dbRoot, partitionNum, segmentNum, hwm, extState, extFound)); + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - if (isFirstBatchPm && (totalRow == rowsLeft)) + // cout << "insertcolumnrec oid:rid:fbo:hwm = " << colStructList[i].dataOid << ":" << lastRid << ":" + // << curFbo << ":" << hwm << endl; + if (succFlag) + { + if ((HWM)curFbo > hwm) { - // in this particular case we already marked extents as invalid up there. - } - else - { - int firstHalfCount = totalRow - rowsLeft; - for (unsigned i = 0; i < colStructList.size(); i++) - { - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - width = colStructList[i].colWidth; - if (firstHalfCount) - { - ExtCPInfo* cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[0]); - RID thisRid = rowsLeft ? lastRid : lastRidNew; - successFlag = colOp->calculateRowId(thisRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - colStructList[i], - curFbo, cpInfoP)); - } - } - maxMins[i].fSplitMaxMinInfoPtrs[0] = cpInfoP; - } - if (rowsLeft) - { - ExtCPInfo* cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[1]); - if (cpInfoP) - { - RETURN_ON_ERROR(GetLBIDRange(newExtentsStartingLbids[i], colStructList[i], *cpInfoP)); - } - - maxMins[i].fSplitMaxMinInfoPtrs[1] = cpInfoP; - } - } - } - - markTxnExtentsAsInvalid(txnid); - - //---------------------------------------------------------------------- - // Write row(s) to database file(s) - //---------------------------------------------------------------------- - std::vector cpinfoList; - for (auto& splitCPInfo : maxMins) - { - for (i = 0; i < 2; i ++) - { - ExtCPInfo* cpInfo = splitCPInfo.fSplitMaxMinInfoPtrs[i]; - if (cpInfo) - { - cpinfoList.push_back(*cpInfo); - cpinfoList[cpinfoList.size() - 1].fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; - } - } - } - rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); - if (rc != NO_ERROR) - { - return rc; - } - rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix, true, &maxMins); // @bug 5572 HDFS tmp file - - if (rc == NO_ERROR) - { - if (dctnryStructList.size() > 0) - { - vector oids {static_cast(tableOid)}; - for (const DctnryStruct &dctnryStruct : dctnryStructList) - { - oids.push_back(dctnryStruct.dctnryOid); - } - - rc = flushOIDsFromCache(oids); - - if (rc != 0) - rc = ERR_BLKCACHE_FLUSH_LIST; // translate to WE error - } - - if (rc == NO_ERROR) - { - int index = 0; - for (auto& splitCPInfo : maxMins) - { - for (i = 0; i < 2; i ++) - { - ExtCPInfo* cpInfo = splitCPInfo.fSplitMaxMinInfoPtrs[i]; - if (cpInfo) - { - cpinfoList[index] = *cpInfo; - cpinfoList[index].fCPInfo.seqNum ++; - index ++; - } - } - } - setInvalidCPInfosSpecialMarks(cpinfoList); - rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); - } - + aHwmEntry.oid = colStructList[i].dataOid; + aHwmEntry.partNum = partitionNum; + aHwmEntry.segNum = segmentNum; + aHwmEntry.hwm = curFbo; + hwmVec.push_back(aHwmEntry); } + } + else + return ERR_INVALID_PARAM; } - return rc; + if (hwmVec.size() > 0) + { + std::vector mergeCPDataArgs; + RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP(hwmVec, mergeCPDataArgs)); + } + } + + if (newExtent) + { +#ifdef PROFILE + timer.start("flushVMCache"); +#endif + std::vector mergeCPDataArgs; + RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP(hwmVecNewext, mergeCPDataArgs)); + RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP(hwmVecOldext, mergeCPDataArgs)); + // flushVMCache(); +#ifdef PROFILE + timer.stop("flushVMCache"); +#endif + } + +#ifdef PROFILE + timer.finish(); +#endif + return rc; } -int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, - ColStructList& colStructList, - std::vector& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - std::vector >& dbRootExtentTrackers, - RBMetaWriter* fRBMetaWriter, - bool bFirstExtentOnThisPM, - bool insertSelect, - bool isAutoCommitOn, - OID tableOid, - bool isFirstBatchPm) +int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, const CSCTypesList& cscColTypeList, + ColStructList& colStructList, ColValueList& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + const int32_t tableOid) { - int rc; - RID* rowIdArray = NULL; - Column curCol; - ColStruct curColStruct; - ColStructList newColStructList; - std::vector colNewValueList; - DctnryStructList newDctnryStructList; - HWM hwm = 0; - HWM oldHwm = 0; - HWM newHwm = 0; - size_t totalRow; - ColStructList::size_type totalColumns; - uint64_t rowsLeft = 0; - bool newExtent = false; - RIDList ridList; - ColumnOp* colOp = NULL; - std::vector dictLbids; + int rc; + RID* rowIdArray = NULL; + ColTupleList curTupleList; + Column curCol; + ColStruct curColStruct; + ColValueList colOldValueList; + ColValueList colNewValueList; + ColStructList newColStructList; + DctnryStructList newDctnryStructList; + HWM hwm = 0; + HWM newHwm = 0; + HWM oldHwm = 0; + ColTupleList::size_type totalRow; + ColStructList::size_type totalColumns; + uint64_t rowsLeft = 0; + bool newExtent = false; + RIDList ridList; + ColumnOp* colOp = NULL; + uint32_t i = 0; - // Set tmp file suffix to modify HDFS db file - bool useTmpSuffix = false; - - m_opType = INSERT; - - if (idbdatafile::IDBPolicy::useHdfs()) - { - if (!bFirstExtentOnThisPM) - useTmpSuffix = true; - } - - unsigned i = 0; #ifdef PROFILE - StopWatch timer; + StopWatch timer; #endif - //Convert data type and column width to write engine specific - for (i = 0; i < colStructList.size(); i++) - Convertor::convertColType(&colStructList[i]); + m_opType = INSERT; - uint32_t colId = 0; - // MCOL-1675: find the smallest column width to calculate the RowID from so - // that all HWMs will be incremented by this operation - findSmallestColumn(colId, colStructList); + // debug information for testing + if (isDebug(DEBUG_2)) + { + printInputValue(colStructList, colValueList, ridList, dctnryStructList, dictStrList); + } + // Convert data type and column width to write engine specific + for (i = 0; i < colStructList.size(); i++) + Convertor::convertColType(&colStructList[i]); - // rc = checkValid(txnid, colStructList, colValueList, ridList); - // if (rc != NO_ERROR) - // return rc; + uint32_t colId = 0; + // MCOL-1675: find the smallest column width to calculate the RowID from so + // that all HWMs will be incremented by this operation + findSmallestColumn(colId, colStructList); - setTransId(txnid); - uint16_t dbRoot, segmentNum; - uint32_t partitionNum; - string segFile; - bool newFile; - TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); - //populate colStructList with file information - IDBDataFile* pFile = NULL; - std::vector extentInfo; - int currentDBrootIdx = 0; - std::vector extents; + rc = checkValid(txnid, colStructList, colValueList, ridList); - //-------------------------------------------------------------------------- - // For first batch on this PM: - // o get starting extent from ExtentTracker, and allocate extent if needed - // o construct colStructList and dctnryStructList accordingly - // o save extent information in tableMetaData for future use - // If not first batch on this PM: - // o construct colStructList and dctnryStructList from tableMetaData - //-------------------------------------------------------------------------- - if (isFirstBatchPm) - { - currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); - extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); - dbRoot = extentInfo[currentDBrootIdx].fDbRoot; - partitionNum = extentInfo[currentDBrootIdx].fPartition; + if (rc != NO_ERROR) + return rc; - //---------------------------------------------------------------------- - // check whether this extent is the first on this PM - //---------------------------------------------------------------------- - if (bFirstExtentOnThisPM) - { - //cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl; - std::vector cols; - BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; + setTransId(txnid); - for (i = 0; i < colStructList.size(); i++) - { - createStripeColumnExtentsArgIn.oid = colStructList[i].dataOid; - createStripeColumnExtentsArgIn.width = colStructList[i].colWidth; - createStripeColumnExtentsArgIn.colDataType = colStructList[i].colDataType; - cols.push_back(createStripeColumnExtentsArgIn); - } + curTupleList = static_cast(colValueList[0]); + totalRow = curTupleList.size(); + totalColumns = colStructList.size(); + rowIdArray = new RID[totalRow]; + // use scoped_array to ensure ptr deletion regardless of where we return + boost::scoped_array rowIdArrayPtr(rowIdArray); + memset(rowIdArray, 0, (sizeof(RID) * totalRow)); - rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partitionNum, segmentNum, extents); + //-------------------------------------------------------------------------- + // allocate row id(s) + //-------------------------------------------------------------------------- + curColStruct = colStructList[colId]; + colOp = m_colOp[op(curColStruct.fCompressionType)]; - if (rc != NO_ERROR) - return rc; + colOp->initColumn(curCol); - //Create column files - ExtCPInfoList cpinfoList; + // Get the correct segment, partition, column file + uint16_t dbRoot; + uint16_t segmentNum = 0; + uint32_t partitionNum = 0; + // Don't search for empty space, always append to the end. May revisit later + dbRoot = curColStruct.fColDbRoot; + int extState; + bool bStartExtFound; + bool bUseStartExtent = false; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot(curColStruct.dataOid, dbRoot, partitionNum, + segmentNum, hwm, extState, bStartExtFound)); - for ( i = 0; i < extents.size(); i++) - { - ExtCPInfo cpInfo(colStructList[i].colDataType, colStructList[i].colWidth); - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType, - colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType, - dbRoot, partitionNum, segmentNum); - colOp->findTypeHandler(colStructList[i].colWidth, - colStructList[i].colDataType); - rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot, - partitionNum, segmentNum, segFile, pFile, newFile); + if ((bStartExtFound) && (extState == BRM::EXTENTAVAILABLE)) + bUseStartExtent = true; - if (rc != NO_ERROR) - return rc; + for (i = 0; i < colStructList.size(); i++) + { + colStructList[i].fColPartition = partitionNum; + colStructList[i].fColSegment = segmentNum; + colStructList[i].fColDbRoot = dbRoot; + } - cpInfo.toInvalid(); + for (i = 0; i < dctnryStructList.size(); i++) + { + dctnryStructList[i].fColPartition = partitionNum; + dctnryStructList[i].fColSegment = segmentNum; + dctnryStructList[i].fColDbRoot = dbRoot; + } - cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; + oldHwm = hwm; // Save this info for rollback + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, + curColStruct.dataOid, curColStruct.fCompressionType, dbRoot, partitionNum, segmentNum); + colOp->findTypeHandler(curColStruct.colWidth, curColStruct.colDataType); + string segFile; - //mark the extents to invalid - cpInfo.fCPInfo.firstLbid = extents[i].startLbid; - cpinfoList.push_back(cpInfo); - colStructList[i].fColPartition = partitionNum; - colStructList[i].fColSegment = segmentNum; - colStructList[i].fColDbRoot = dbRoot; - dctnryStructList[i].fColPartition = partitionNum; - dctnryStructList[i].fColSegment = segmentNum; - dctnryStructList[i].fColDbRoot = dbRoot; - } - - //mark the extents to invalid - rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); - - if (rc != NO_ERROR) - return rc; - - //create corresponding dictionary files - for (i = 0; i < dctnryStructList.size(); i++) - { - if (dctnryStructList[i].dctnryOid > 0) - { - rc = createDctnry(txnid, dctnryStructList[i].dctnryOid, dctnryStructList[i].colWidth, dbRoot, partitionNum, - segmentNum, dctnryStructList[i].fCompressionType); - - if ( rc != NO_ERROR) - return rc; - } - } - } // if ( bFirstExtentOnThisPM) - else // if (!bFirstExtentOnThisPM) - { - std::vector tmpExtentInfo; - - for (i = 0; i < dbRootExtentTrackers.size(); i++) - { - tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); - colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; - colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; - colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; - //cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aExt.isDict = false; - - if (bFirstExtentOnThisPM) - { - aExt.hwm = extents[i].startBlkOffset; - aExt.isNewExt = true; - //cout << "adding a ext to metadata" << endl; - } - else - { - std::vector tmpExtentInfo; - tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); - aExt.isNewExt = false; - aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; - //cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl; - } - - aExt.current = true; - aColExtsInfo.push_back(aExt); - //cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl; - } - - tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - for (i = 0; i < dctnryStructList.size(); i++) - { - if (dctnryStructList[i].dctnryOid > 0) - { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && (it->partNum == dctnryStructList[i].fColPartition) && (it->segNum == dctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = dctnryStructList[i].fColDbRoot; - aExt.partNum = dctnryStructList[i].fColPartition; - aExt.segNum = dctnryStructList[i].fColSegment; - aExt.compType = dctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - } - - tableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); - } - } - - } // if (isFirstBatchPm) - else //get the extent info from tableMetaData - { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if (it->current) - break; - - it++; - } - - if (it == aColExtsInfo.end()) - return 1; - - for (i = 0; i < colStructList.size(); i++) - { - colStructList[i].fColPartition = it->partNum; - colStructList[i].fColSegment = it->segNum; - colStructList[i].fColDbRoot = it->dbRoot; - dctnryStructList[i].fColPartition = it->partNum; - dctnryStructList[i].fColSegment = it->segNum; - dctnryStructList[i].fColDbRoot = it->dbRoot; - } - } - - totalColumns = colStructList.size(); - totalRow = colValueList.size() / totalColumns; - rowIdArray = new RID[totalRow]; - // use scoped_array to ensure ptr deletion regardless of where we return - boost::scoped_array rowIdArrayPtr(rowIdArray); - memset(rowIdArray, 0, (sizeof(RID)*totalRow)); - - //-------------------------------------------------------------------------- - // allocate row id(s) - //-------------------------------------------------------------------------- - - curColStruct = colStructList[colId]; - - colOp = m_colOp[op(curColStruct.fCompressionType)]; - - colOp->initColumn(curCol); - - //Get the correct segment, partition, column file - vector colExtentInfo; //Save those empty extents in case of failure to rollback - vector dictExtentInfo; //Save those empty extents in case of failure to rollback - vector fileInfo; - dbRoot = curColStruct.fColDbRoot; - //use the smallest column to calculate row id - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[colId].fColDbRoot) && (it->partNum == colStructList[colId].fColPartition) && (it->segNum == colStructList[colId].fColSegment) && it->current ) - break; - - it++; - } - - if (it != aColExtsInfo.end()) - { - hwm = it->hwm; - //cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[colId].fColSegment << endl; - } - - oldHwm = hwm; //Save this info for rollback - //need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, - curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, - curColStruct.fColDbRoot, curColStruct.fColPartition, curColStruct.fColSegment); - colOp->findTypeHandler(curColStruct.colWidth, - curColStruct.colDataType); - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file + if (bUseStartExtent) + { + rc = colOp->openColumnFile(curCol, segFile, true); // @bug 5572 HDFS tmp file if (rc != NO_ERROR) { - return rc; + return rc; } + } - //get hwm first - // @bug 286 : fix for bug 286 - correct the typo in getHWM - //RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); - - Column newCol; + bool newFile; #ifdef PROFILE - timer.start("allocRowId"); + timer.start("allocRowId"); #endif - newColStructList = colStructList; - newDctnryStructList = dctnryStructList; - bool bUseStartExtent = true; + newColStructList = colStructList; + newDctnryStructList = dctnryStructList; + std::vector > dbRootExtentTrackers; + rc = colOp->allocRowId(txnid, bUseStartExtent, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, + rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, + dbRootExtentTrackers, false, false, 0, false, NULL); - if (idbdatafile::IDBPolicy::useHdfs()) - insertSelect = true; - - rc = colOp->allocRowId(txnid, bUseStartExtent, - curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, - newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm, NULL); - - //cout << "after allocrowid, total row = " < 256K. - // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. - //-------------------------------------------------------------------------- - // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((curCol.dataFile.fPartition == 0) && - (curCol.dataFile.fSegment == 0) && - ((totalRow - rowsLeft) > 0) && - (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + //-------------------------------------------------------------------------- + // Handle case where we ran out of disk space allocating a new extent. + // Rollback extentmap and delete any db files that were created. + //-------------------------------------------------------------------------- + if (rc != NO_ERROR) + { + if ((rc == ERR_FILE_DISK_SPACE) && newExtent) { - for (size_t k = 0; k < colStructList.size(); k++) + vector colExtentInfo; + vector dictExtentInfo; + vector fileInfo; + ExtentInfo info; + + for (i = 0; i < newColStructList.size(); i++) + { + info.oid = newColStructList[i].dataOid; + info.partitionNum = newColStructList[i].fColPartition; + info.segmentNum = newColStructList[i].fColSegment; + info.dbRoot = newColStructList[i].fColDbRoot; + + if (newFile) + fileInfo.push_back(info); + + colExtentInfo.push_back(info); + } + + int rc1 = BRMWrapper::getInstance()->deleteEmptyColExtents(colExtentInfo); + + // Only rollback dictionary extents "if" store file is new + if ((rc1 == 0) && newFile) + { + for (unsigned int j = 0; j < fileInfo.size(); j++) { - // Skip the selected column - if (k == (size_t)colId) - continue; - - Column expandCol; - colOp = m_colOp[op(colStructList[k].fCompressionType)]; - // Shouldn't we change 0 to colId here? - colOp->setColParam(expandCol, 0, - colStructList[k].colWidth, - colStructList[k].colDataType, - colStructList[k].colType, - colStructList[k].dataOid, - colStructList[k].fCompressionType, - colStructList[k].fColDbRoot, - colStructList[k].fColPartition, - colStructList[k].fColSegment); - colOp->findTypeHandler(colStructList[k].colWidth, - colStructList[k].colDataType); - rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file - - if (rc == NO_ERROR) - { - if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) - { - rc = colOp->expandAbbrevExtent(expandCol); - } - } - - if (rc != NO_ERROR) - { - return rc; - } - - colOp->closeColumnFile(expandCol); + // ignore return code and delete what we can + rc1 = colOp->deleteFile(fileInfo[j].oid, fileInfo[j].dbRoot, fileInfo[j].partitionNum, + fileInfo[j].segmentNum); } - } - //-------------------------------------------------------------------------- - // Tokenize data if needed - //-------------------------------------------------------------------------- - if (insertSelect && isAutoCommitOn) - BRMWrapper::setUseVb( false ); - else - BRMWrapper::setUseVb( true ); + fileInfo.clear(); - dictStr::iterator dctStr_iter; - uint64_t* colValPtr; - size_t rowsPerColumn = colValueList.size() / colStructList.size(); - - for (i = 0; i < colStructList.size(); i++) - { - if (colStructList[i].tokenFlag) + for (i = 0; i < newDctnryStructList.size(); i++) { - dctStr_iter = dictStrList[i].begin(); - Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; - rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, - dctnryStructList[i].fColDbRoot, dctnryStructList[i].fColPartition, - dctnryStructList[i].fColSegment, - useTmpSuffix); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - { - return rc; - } - - for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) - { - colValPtr = &colValueList[(i * rowsPerColumn) + rows]; - - if (dctStr_iter->length() == 0) - { - Token nullToken; - memcpy(colValPtr, &nullToken, 8); - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - memcpy(colValPtr, &dctTuple.token, 8); - dictLbids.push_back(dctTuple.token.fbo); - } - - dctStr_iter++; - - } - - //close dictionary files - rc = dctnry->closeDctnry(false); - - if (rc != NO_ERROR) - return rc; - - if (newExtent) - { - //@Bug 4854 back up hwm chunk for the file to be modified - if (fRBMetaWriter) - fRBMetaWriter->backupDctnryHWMChunk(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment); - - rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, - newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, - newDctnryStructList[i].fColSegment, - false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - for (uint32_t rows = 0; rows < rowsLeft; rows++) - { - colValPtr = &colValueList[(i * rowsPerColumn) + rows]; - - if (dctStr_iter->length() == 0) - { - Token nullToken; - memcpy(colValPtr, &nullToken, 8); - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - memcpy(colValPtr, &dctTuple.token, 8); - dictLbids.push_back(dctTuple.token.fbo); - } - - dctStr_iter++; - } - - //close dictionary files - rc = dctnry->closeDctnry(false); - - if (rc != NO_ERROR) - return rc; - } + if (newDctnryStructList[i].dctnryOid > 0) + { + info.oid = newDctnryStructList[i].dctnryOid; + info.partitionNum = newDctnryStructList[i].fColPartition; + info.segmentNum = newDctnryStructList[i].fColSegment; + info.dbRoot = newDctnryStructList[i].fColDbRoot; + info.newFile = true; + fileInfo.push_back(info); + dictExtentInfo.push_back(info); + } } - } - if (insertSelect && isAutoCommitOn) - BRMWrapper::setUseVb( false ); - else - BRMWrapper::setUseVb( true ); + if (dictExtentInfo.size() > 0) + { + FileOp fileOp; + rc1 = BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(dictExtentInfo); - //-------------------------------------------------------------------------- - // Update column info structure @Bug 1862 set hwm, and - // Prepare ValueList for new extent (if applicable) - //-------------------------------------------------------------------------- - //@Bug 2205 Check whether all rows go to the new extent - RID lastRid = 0; - RID lastRidNew = 0; + if (rc1 != NO_ERROR) + return rc; - if (totalRow - rowsLeft > 0) + for (unsigned j = 0; j < fileInfo.size(); j++) + { + rc1 = fileOp.deleteFile(fileInfo[j].oid, fileInfo[j].dbRoot, fileInfo[j].partitionNum, + fileInfo[j].segmentNum); + } + } + } + } // disk space error allocating new extent + + return rc; + } // rc != NO_ERROR from call to allocRowID() + +#ifdef PROFILE + timer.stop("allocRowId"); +#endif + + TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tableOid); + + //-------------------------------------------------------------------------- + // Expand initial abbreviated extent if any RID in 1st extent is > 256K. + // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. + //-------------------------------------------------------------------------- + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((colStructList[colId].fColPartition == 0) && (colStructList[colId].fColSegment == 0) && + ((totalRow - rowsLeft) > 0) && + (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + { + for (unsigned k = 0; k < colStructList.size(); k++) { - lastRid = rowIdArray[totalRow - rowsLeft - 1]; - lastRidNew = rowIdArray[totalRow - 1]; - } - else + if (k == colId) + continue; + Column expandCol; + colOp = m_colOp[op(colStructList[k].fCompressionType)]; + colOp->setColParam(expandCol, 0, colStructList[k].colWidth, colStructList[k].colDataType, + colStructList[k].colType, colStructList[k].dataOid, + colStructList[k].fCompressionType, colStructList[k].fColDbRoot, + colStructList[k].fColPartition, colStructList[k].fColSegment); + colOp->findTypeHandler(colStructList[k].colWidth, colStructList[k].colDataType); + rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file + + if (rc == NO_ERROR) + { + if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) + { + rc = colOp->expandAbbrevExtent(expandCol); + } + } + + colOp->clearColumn(expandCol); // closes the file + + if (rc != NO_ERROR) + { + return rc; + } + } // loop through columns + } // if starting extent needs to be expanded + + //-------------------------------------------------------------------------- + // Tokenize data if needed + //-------------------------------------------------------------------------- + dictStr::iterator dctStr_iter; + ColTupleList::iterator col_iter; + + for (unsigned i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) { - lastRid = 0; - lastRidNew = rowIdArray[totalRow - 1]; - } + dctStr_iter = dictStrList[i].begin(); + col_iter = colValueList[i].begin(); + Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; - //cout << "rowid allocated is " << lastRid << endl; - //if a new extent is created, all the columns in this table should have their own new extent - //First column already processed + ColExtsInfo aColExtsInfo = aTableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); - //@Bug 1701. Close the file (if uncompressed) - m_colOp[op(curCol.compressionType)]->closeColumnFile(curCol); - //cout << "Saving hwm info for new ext batch" << endl; - //Update hwm to set them in the end - bool succFlag = false; - unsigned colWidth = 0; - int curFbo = 0, curBio; + if (bUseStartExtent) + { + rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, dctnryStructList[i].fColDbRoot, + dctnryStructList[i].fColPartition, dctnryStructList[i].fColSegment, + true); // @bug 5572 HDFS tmp file - for (i = 0; i < totalColumns; i++) - { - //shoud be obtained from saved hwm - aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[i].dataOid); + if (rc != NO_ERROR) + return rc; + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && + (it->partNum == dctnryStructList[i].fColPartition) && + (it->segNum == dctnryStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = dctnryStructList[i].fColDbRoot; + aExt.partNum = dctnryStructList[i].fColPartition; + aExt.segNum = dctnryStructList[i].fColSegment; + aExt.compType = dctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + aTableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); + } + + for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) + { + if (dctStr_iter->length() == 0) + { + Token nullToken; + col_iter->data = nullToken; + } + else + { +#ifdef PROFILE + timer.start("tokenize"); +#endif + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); + + if (rc != NO_ERROR) + { + dctnry->closeDctnry(); + return rc; + } + +#ifdef PROFILE + timer.stop("tokenize"); +#endif + col_iter->data = dctTuple.token; + } + + dctStr_iter++; + col_iter++; + } + + // close dictionary files + rc = dctnry->closeDctnry(); + + if (rc != NO_ERROR) + return rc; + } // tokenize dictionary rows in 1st extent + + if (newExtent) + { + rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, newDctnryStructList[i].fColDbRoot, + newDctnryStructList[i].fColPartition, newDctnryStructList[i].fColSegment, + false); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + return rc; + + aColExtsInfo = aTableMetaData->getColExtsInfo(newDctnryStructList[i].dctnryOid); it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) - && (it->segNum == colStructList[i].fColSegment) && it->current) - break; + if ((it->dbRoot == newDctnryStructList[i].fColDbRoot) && + (it->partNum == newDctnryStructList[i].fColPartition) && + (it->segNum == newDctnryStructList[i].fColSegment)) + break; - it++; + it++; } - if (it != aColExtsInfo.end()) //update hwm info + if (it == aColExtsInfo.end()) // add this one to the list { - oldHwm = it->hwm; - - // save hwm for the old extent - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - //cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl; - if (succFlag) - { - if ((HWM)curFbo >= oldHwm) - { - it->hwm = (HWM)curFbo; - } - - //@Bug 4947. set current to false for old extent. - if (newExtent) - { - it->current = false; - } - - //cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = " - //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl; - } - else - return ERR_INVALID_PARAM; - + ColExtInfo aExt; + aExt.dbRoot = newDctnryStructList[i].fColDbRoot; + aExt.partNum = newDctnryStructList[i].fColPartition; + aExt.segNum = newDctnryStructList[i].fColSegment; + aExt.compType = newDctnryStructList[i].fCompressionType; + aExt.isDict = true; + aColExtsInfo.push_back(aExt); + aTableMetaData->setColExtsInfo(newDctnryStructList[i].dctnryOid, aColExtsInfo); } - //update hwm for the new extent - if (newExtent) + for (uint32_t rows = 0; rows < rowsLeft; rows++) { - it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) - && (it->segNum == newColStructList[i].fColSegment) && it->current) - break; - - it++; - } - - colWidth = newColStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - if (succFlag) - { - if (it != aColExtsInfo.end()) - { - it->hwm = (HWM)curFbo; - //cout << "setting hwm to " << (int)curFbo <<" for seg " <segNum << endl; - it->current = true; - } - } - else - return ERR_INVALID_PARAM; - } - - tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - //-------------------------------------------------------------------------- - //Prepare the valuelist for the new extent - //-------------------------------------------------------------------------- - - for (unsigned i = 1; i <= totalColumns; i++) - { - // Copy values to second value list - for (uint64_t j = rowsLeft; j > 0; j--) - { - colNewValueList.push_back(colValueList[(totalRow * i) - j]); - } - } - - // end of allocate row id - + if (dctStr_iter->length() == 0) + { + Token nullToken; + col_iter->data = nullToken; + } + else + { #ifdef PROFILE - timer.start("writeColumnRec"); + timer.start("tokenize"); #endif -//cout << "Writing column record" << endl; - - if (rc == NO_ERROR) - { - //---------------------------------------------------------------------- - //Mark extents invalid - //---------------------------------------------------------------------- - bool successFlag = true; - unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; - - if (isFirstBatchPm && (totalRow == rowsLeft)) - {} - else - { - for (unsigned i = 0; i < colStructList.size(); i++) - { - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - width = colStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - colStructList[i], - curFbo)); - } - } - else - return ERR_INVALID_PARAM; - } - } - - // If we create a new extent for this batch - for (unsigned i = 0; i < newColStructList.size(); i++) - { - colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - width = newColStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - newColStructList[i], - curFbo)); - } - } - else - return ERR_INVALID_PARAM; - } - - markTxnExtentsAsInvalid(txnid); - - //---------------------------------------------------------------------- - // Write row(s) to database file(s) - //---------------------------------------------------------------------- - bool versioning = !(isAutoCommitOn && insertSelect); - AddDictToList(txnid, dictLbids); - rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file - } - - return rc; -} - - -int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - ColStructList& colStructList, - ColValueList& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - const int32_t tableOid) -{ - int rc; - RID* rowIdArray = NULL; - ColTupleList curTupleList; - Column curCol; - ColStruct curColStruct; - ColValueList colOldValueList; - ColValueList colNewValueList; - ColStructList newColStructList; - DctnryStructList newDctnryStructList; - HWM hwm = 0; - HWM newHwm = 0; - HWM oldHwm = 0; - ColTupleList::size_type totalRow; - ColStructList::size_type totalColumns; - uint64_t rowsLeft = 0; - bool newExtent = false; - RIDList ridList; - ColumnOp* colOp = NULL; - uint32_t i = 0; -#ifdef PROFILE - StopWatch timer; -#endif - - m_opType = INSERT; - - // debug information for testing - if (isDebug(DEBUG_2)) - { - printInputValue(colStructList, colValueList, ridList, dctnryStructList, dictStrList); - } - - // end - - //Convert data type and column width to write engine specific - for (i = 0; i < colStructList.size(); i++) - Convertor::convertColType(&colStructList[i]); - - rc = checkValid(txnid, colStructList, colValueList, ridList); - - if (rc != NO_ERROR) - return rc; - - setTransId(txnid); - - curTupleList = static_cast(colValueList[0]); - totalRow = curTupleList.size(); - totalColumns = colStructList.size(); - rowIdArray = new RID[totalRow]; - // use scoped_array to ensure ptr deletion regardless of where we return - boost::scoped_array rowIdArrayPtr(rowIdArray); - memset(rowIdArray, 0, (sizeof(RID)*totalRow)); - - // allocate row id(s) - curColStruct = colStructList[0]; - colOp = m_colOp[op(curColStruct.fCompressionType)]; - - colOp->initColumn(curCol); - - //Get the correct segment, partition, column file - uint16_t dbRoot, segmentNum; - uint32_t partitionNum; - vector colExtentInfo; //Save those empty extents in case of failure to rollback - vector dictExtentInfo; //Save those empty extents in case of failure to rollback - vector fileInfo; - ExtentInfo info; - //Don't search for empty space, always append to the end. May need to revisit this part - dbRoot = curColStruct.fColDbRoot; - int extState; - bool extFound; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( - curColStruct.dataOid, dbRoot, partitionNum, segmentNum, hwm, - extState, extFound)); - - for (i = 0; i < colStructList.size(); i++) - { - colStructList[i].fColPartition = partitionNum; - colStructList[i].fColSegment = segmentNum; - colStructList[i].fColDbRoot = dbRoot; - } - - oldHwm = hwm; //Save this info for rollback - //need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType, - curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, - dbRoot, partitionNum, segmentNum); - colOp->findTypeHandler(curColStruct.colWidth, - curColStruct.colDataType); - string segFile; - rc = colOp->openColumnFile(curCol, segFile, false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - { - return rc; - } - - //get hwm first - // @bug 286 : fix for bug 286 - correct the typo in getHWM - //RETURN_ON_ERROR(BRMWrapper::getInstance()->getHWM(curColStruct.dataOid, hwm)); - - //...Note that we are casting totalRow to int to be in sync with - //...allocRowId(). So we are assuming that totalRow - //...(curTupleList.size()) will fit into an int. We arleady made - //...that assumption earlier in this method when we used totalRow - //...in the call to calloc() to allocate rowIdArray. - Column newCol; - bool newFile; - -#ifdef PROFILE - timer.start("allocRowId"); -#endif - - newColStructList = colStructList; - newDctnryStructList = dctnryStructList; - std::vector > dbRootExtentTrackers; - bool bUseStartExtent = true; - rc = colOp->allocRowId(txnid, bUseStartExtent, - curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, - dbRootExtentTrackers, false, false, 0, false, NULL); - - if ((rc == ERR_FILE_DISK_SPACE) && newExtent) - { - for (i = 0; i < newColStructList.size(); i++) - { - info.oid = newColStructList[i].dataOid; - info.partitionNum = newColStructList[i].fColPartition; - info.segmentNum = newColStructList[i].fColSegment; - info.dbRoot = newColStructList[i].fColDbRoot; - - if (newFile) - fileInfo.push_back (info); - - colExtentInfo.push_back (info); - } - - int rc1 = BRMWrapper::getInstance()->deleteEmptyColExtents(colExtentInfo); - - if ((rc1 == 0) && newFile) - { - rc1 = colOp->deleteFile(fileInfo[0].oid, fileInfo[0].dbRoot, fileInfo[0].partitionNum, fileInfo[0].segmentNum); - - if ( rc1 != NO_ERROR) - return rc; - - FileOp fileOp; - - for (i = 0; i < newDctnryStructList.size(); i++) - { - if (newDctnryStructList[i].dctnryOid > 0) - { - info.oid = newDctnryStructList[i].dctnryOid; - info.partitionNum = newDctnryStructList[i].fColPartition; - info.segmentNum = newDctnryStructList[i].fColSegment; - info.dbRoot = newDctnryStructList[i].fColDbRoot; - info.newFile = true; - fileInfo.push_back (info); - dictExtentInfo.push_back (info); - } - } - - if (dictExtentInfo.size() > 0) - { - rc1 = BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(dictExtentInfo); - - if ( rc1 != NO_ERROR) - return rc; - - for (unsigned j = 0; j < fileInfo.size(); j++) - { - rc1 = fileOp.deleteFile(fileInfo[j].oid, fileInfo[j].dbRoot, - fileInfo[j].partitionNum, fileInfo[j].segmentNum); - } - } - } - } - - TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tableOid); - - //..Expand initial abbreviated extent if any RID in 1st extent is > 256K -// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((partitionNum == 0) && - (segmentNum == 0) && - ((totalRow - rowsLeft) > 0) && - (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) - { - for (size_t k = 1; k < colStructList.size(); k++) - { - Column expandCol; - colOp = m_colOp[op(colStructList[k].fCompressionType)]; - colOp->setColParam(expandCol, 0, - colStructList[k].colWidth, - colStructList[k].colDataType, - colStructList[k].colType, - colStructList[k].dataOid, - colStructList[k].fCompressionType, - dbRoot, - partitionNum, - segmentNum); - colOp->findTypeHandler(colStructList[k].colWidth, - colStructList[k].colDataType); - rc = colOp->openColumnFile(expandCol, segFile, false); // @bug 5572 HDFS tmp file - - if (rc == NO_ERROR) - { - if (colOp->abbreviatedExtent(expandCol.dataFile.pFile, colStructList[k].colWidth)) - { - rc = colOp->expandAbbrevExtent(expandCol); - } - } + DctnryTuple dctTuple; + dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); + dctTuple.sigSize = dctStr_iter->length(); + dctTuple.isNull = false; + rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); if (rc != NO_ERROR) { - if (newExtent) - { - //Remove the empty extent added to the first column - int rc1 = BRMWrapper::getInstance()-> - deleteEmptyColExtents(colExtentInfo); - - if ((rc1 == 0) && newFile) - { - rc1 = colOp->deleteFile(fileInfo[0].oid, - fileInfo[0].dbRoot, - fileInfo[0].partitionNum, - fileInfo[0].segmentNum); - } - } - - colOp->clearColumn(expandCol); // closes the file - return rc; + dctnry->closeDctnry(); + return rc; } - colOp->clearColumn(expandCol); // closes the file - } - } - - BRMWrapper::setUseVb(true); - //Tokenize data if needed - dictStr::iterator dctStr_iter; - ColTupleList::iterator col_iter; - - for (i = 0; i < colStructList.size(); i++) - { - if (colStructList[i].tokenFlag) - { - dctStr_iter = dictStrList[i].begin(); - col_iter = colValueList[i].begin(); - Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; - - dctnryStructList[i].fColPartition = partitionNum; - dctnryStructList[i].fColSegment = segmentNum; - dctnryStructList[i].fColDbRoot = dbRoot; - rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, - dctnryStructList[i].fColDbRoot, dctnryStructList[i].fColPartition, - dctnryStructList[i].fColSegment, - false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - ColExtsInfo aColExtsInfo = aTableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && (it->partNum == dctnryStructList[i].fColPartition) && (it->segNum == dctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = dctnryStructList[i].fColDbRoot; - aExt.partNum = dctnryStructList[i].fColPartition; - aExt.segNum = dctnryStructList[i].fColSegment; - aExt.compType = dctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - aTableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); - } - - for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) - { - if (dctStr_iter->length() == 0) - { - Token nullToken; - col_iter->data = nullToken; - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - #ifdef PROFILE - timer.stop("tokenize"); + timer.stop("tokenize"); #endif - col_iter->data = dctTuple.token; - } + col_iter->data = dctTuple.token; + } - dctStr_iter++; - col_iter++; - - } - - //close dictionary files - rc = dctnry->closeDctnry(); - - if (rc != NO_ERROR) - return rc; - - if (newExtent) - { - rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, - newDctnryStructList[i].fColDbRoot, newDctnryStructList[i].fColPartition, - newDctnryStructList[i].fColSegment, - false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - aColExtsInfo = aTableMetaData->getColExtsInfo(newDctnryStructList[i].dctnryOid); - it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newDctnryStructList[i].fColDbRoot) && (it->partNum == newDctnryStructList[i].fColPartition) && (it->segNum == newDctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = newDctnryStructList[i].fColDbRoot; - aExt.partNum = newDctnryStructList[i].fColPartition; - aExt.segNum = newDctnryStructList[i].fColSegment; - aExt.compType = newDctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - aTableMetaData->setColExtsInfo(newDctnryStructList[i].dctnryOid, aColExtsInfo); - } - - for (uint32_t rows = 0; rows < rowsLeft; rows++) - { - if (dctStr_iter->length() == 0) - { - Token nullToken; - col_iter->data = nullToken; - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - col_iter->data = dctTuple.token; - } - - dctStr_iter++; - col_iter++; - } - - //close dictionary files - rc = dctnry->closeDctnry(); - - if (rc != NO_ERROR) - return rc; - } - } - } - - - //Update column info structure @Bug 1862 set hwm - //@Bug 2205 Check whether all rows go to the new extent - RID lastRid = 0; - RID lastRidNew = 0; - - if (totalRow - rowsLeft > 0) - { - lastRid = rowIdArray[totalRow - rowsLeft - 1]; - lastRidNew = rowIdArray[totalRow - 1]; - } - else - { - lastRid = 0; - lastRidNew = rowIdArray[totalRow - 1]; - } - - //cout << "rowid allocated is " << lastRid << endl; - //if a new extent is created, all the columns in this table should have their own new extent - - //@Bug 1701. Close the file - m_colOp[op(curCol.compressionType)]->clearColumn(curCol); - std::vector hwmVecNewext; - std::vector hwmVecOldext; - - if (newExtent) //Save all hwms to set them later. - { - BulkSetHWMArg aHwmEntryNew; - BulkSetHWMArg aHwmEntryOld; - bool succFlag = false; - unsigned colWidth = 0; - int extState; - bool extFound; - int curFbo = 0, curBio; - - for (i = 0; i < totalColumns; i++) - { - Column curColLocal; - colOp->initColumn(curColLocal); - - colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - colOp->setColParam(curColLocal, 0, - newColStructList[i].colWidth, newColStructList[i].colDataType, - newColStructList[i].colType, newColStructList[i].dataOid, - newColStructList[i].fCompressionType, dbRoot, partitionNum, segmentNum); - colOp->findTypeHandler(newColStructList[i].colWidth, - newColStructList[i].colDataType); - - rc = BRMWrapper::getInstance()->getLastHWM_DBroot( - curColLocal.dataFile.fid, dbRoot, partitionNum, segmentNum, oldHwm, - extState, extFound); - - info.oid = curColLocal.dataFile.fid; - info.partitionNum = partitionNum; - info.segmentNum = segmentNum; - info.dbRoot = dbRoot; - info.hwm = oldHwm; - colExtentInfo.push_back(info); - // @Bug 2714 need to set hwm for the old extent - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - //cout << "insertcolumnrec oid:rid:fbo:hwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << hwm << endl; - if (succFlag) - { - if ((HWM)curFbo > oldHwm) - { - aHwmEntryOld.oid = colStructList[i].dataOid; - aHwmEntryOld.partNum = partitionNum; - aHwmEntryOld.segNum = segmentNum; - aHwmEntryOld.hwm = curFbo; - hwmVecOldext.push_back(aHwmEntryOld); - } - } - else - return ERR_INVALID_PARAM; - - colWidth = newColStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - if (succFlag) - { - aHwmEntryNew.oid = newColStructList[i].dataOid; - aHwmEntryNew.partNum = newColStructList[i].fColPartition; - aHwmEntryNew.segNum = newColStructList[i].fColSegment; - aHwmEntryNew.hwm = curFbo; - hwmVecNewext.push_back(aHwmEntryNew); - } - - m_colOp[op(curColLocal.compressionType)]->clearColumn(curColLocal); + dctStr_iter++; + col_iter++; } - //Prepare the valuelist for the new extent - ColTupleList colTupleList; - ColTupleList newColTupleList; - ColTupleList firstPartTupleList; - - for (unsigned i = 0; i < totalColumns; i++) - { - colTupleList = static_cast(colValueList[i]); - - for (uint64_t j = rowsLeft; j > 0; j--) - { - newColTupleList.push_back(colTupleList[totalRow - j]); - } - - colNewValueList.push_back(newColTupleList); - newColTupleList.clear(); - - //upate the oldvalue list for the old extent - for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) - { - firstPartTupleList.push_back(colTupleList[j]); - } - - colOldValueList.push_back(firstPartTupleList); - firstPartTupleList.clear(); - } - } - -//Mark extents invalid - bool successFlag = true; - unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; - - if (totalRow - rowsLeft > 0) - { - for (unsigned i = 0; i < colStructList.size(); i++) - { - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - width = colStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - colStructList[i], - curFbo)); - } - } - } - } - - lastRid = rowIdArray[totalRow - 1]; - - for (unsigned i = 0; i < newColStructList.size(); i++) - { - colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - width = newColStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - newColStructList[i], - curFbo)); - } - } - } - - //cout << "lbids size = " << lbids.size()<< endl; - markTxnExtentsAsInvalid(txnid); - - if (rc == NO_ERROR) - { - // MCOL-66 The DBRM can't handle concurrent transactions to sys tables - static boost::mutex dbrmMutex; - boost::mutex::scoped_lock lk(dbrmMutex); - - if (newExtent) - { - rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file - } - else - { - rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file - } - } - -#ifdef PROFILE - timer.stop("writeColumnRec"); -#endif -// for (ColTupleList::size_type i = 0; i < totalRow; i++) -// ridList.push_back((RID) rowIdArray[i]); - - // if (rc == NO_ERROR) - // rc = flushDataFiles(NO_ERROR); - - if ( !newExtent ) - { - //flushVMCache(); - bool succFlag = false; - unsigned colWidth = 0; - int extState; - bool extFound; - int curFbo = 0, curBio; - std::vector hwmVec; - - for (unsigned i = 0; i < totalColumns; i++) - { - //colOp = m_colOp[op(colStructList[i].fCompressionType)]; - //Set all columns hwm together - BulkSetHWMArg aHwmEntry; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot(colStructList[i].dataOid, dbRoot, partitionNum, segmentNum, hwm, - extState, extFound)); - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - //cout << "insertcolumnrec oid:rid:fbo:hwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << hwm << endl; - if (succFlag) - { - if ((HWM)curFbo > hwm) - { - aHwmEntry.oid = colStructList[i].dataOid; - aHwmEntry.partNum = partitionNum; - aHwmEntry.segNum = segmentNum; - aHwmEntry.hwm = curFbo; - hwmVec.push_back(aHwmEntry); - } - } - else - return ERR_INVALID_PARAM; - } - - if (hwmVec.size() > 0 ) - { - std::vector mergeCPDataArgs; - RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( hwmVec, mergeCPDataArgs)); - } - } - - if (newExtent) - { -#ifdef PROFILE - timer.start("flushVMCache"); -#endif - std::vector mergeCPDataArgs; - RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( hwmVecNewext, mergeCPDataArgs)); - RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( hwmVecOldext, mergeCPDataArgs)); - //flushVMCache(); -#ifdef PROFILE - timer.stop("flushVMCache"); -#endif - } - -#ifdef PROFILE - timer.finish(); -#endif - return rc; -} - -int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - ColStructList& colStructList, - ColValueList& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - const int32_t tableOid) -{ - int rc; - RID* rowIdArray = NULL; - ColTupleList curTupleList; - Column curCol; - ColStruct curColStruct; - ColValueList colOldValueList; - ColValueList colNewValueList; - ColStructList newColStructList; - DctnryStructList newDctnryStructList; - HWM hwm = 0; - HWM newHwm = 0; - HWM oldHwm = 0; - ColTupleList::size_type totalRow; - ColStructList::size_type totalColumns; - uint64_t rowsLeft = 0; - bool newExtent = false; - RIDList ridList; - ColumnOp* colOp = NULL; - uint32_t i = 0; - -#ifdef PROFILE - StopWatch timer; -#endif - - m_opType = INSERT; - - // debug information for testing - if (isDebug(DEBUG_2)) - { - printInputValue(colStructList, colValueList, ridList, dctnryStructList, dictStrList); - } - - //Convert data type and column width to write engine specific - for (i = 0; i < colStructList.size(); i++) - Convertor::convertColType(&colStructList[i]); - - uint32_t colId = 0; - // MCOL-1675: find the smallest column width to calculate the RowID from so - // that all HWMs will be incremented by this operation - findSmallestColumn(colId, colStructList); - - rc = checkValid(txnid, colStructList, colValueList, ridList); - - if (rc != NO_ERROR) - return rc; - - setTransId(txnid); - - curTupleList = static_cast(colValueList[0]); - totalRow = curTupleList.size(); - totalColumns = colStructList.size(); - rowIdArray = new RID[totalRow]; - // use scoped_array to ensure ptr deletion regardless of where we return - boost::scoped_array rowIdArrayPtr(rowIdArray); - memset(rowIdArray, 0, (sizeof(RID)*totalRow)); - - //-------------------------------------------------------------------------- - // allocate row id(s) - //-------------------------------------------------------------------------- - curColStruct = colStructList[colId]; - colOp = m_colOp[op(curColStruct.fCompressionType)]; - - colOp->initColumn(curCol); - - //Get the correct segment, partition, column file - uint16_t dbRoot; - uint16_t segmentNum = 0; - uint32_t partitionNum = 0; - //Don't search for empty space, always append to the end. May revisit later - dbRoot = curColStruct.fColDbRoot; - int extState; - bool bStartExtFound; - bool bUseStartExtent = false; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( - curColStruct.dataOid, dbRoot, partitionNum, segmentNum, hwm, - extState, bStartExtFound)); - - if ((bStartExtFound) && (extState == BRM::EXTENTAVAILABLE)) - bUseStartExtent = true; - - for (i = 0; i < colStructList.size(); i++) - { - colStructList[i].fColPartition = partitionNum; - colStructList[i].fColSegment = segmentNum; - colStructList[i].fColDbRoot = dbRoot; - } - - for (i = 0; i < dctnryStructList.size(); i++) - { - dctnryStructList[i].fColPartition = partitionNum; - dctnryStructList[i].fColSegment = segmentNum; - dctnryStructList[i].fColDbRoot = dbRoot; - } - - oldHwm = hwm; //Save this info for rollback - //need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, - curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, - dbRoot, partitionNum, segmentNum); - colOp->findTypeHandler(curColStruct.colWidth, - curColStruct.colDataType); - string segFile; - - if (bUseStartExtent) - { - rc = colOp->openColumnFile(curCol, segFile, true); // @bug 5572 HDFS tmp file + // close dictionary files + rc = dctnry->closeDctnry(); if (rc != NO_ERROR) + return rc; + } // tokenize dictionary rows in second extent + } // tokenize dictionary columns + } // loop through columns to see which ones need tokenizing + + //---------------------------------------------------------------------- + // Update column info structure @Bug 1862 set hwm, and + // Prepare ValueList for new extent (if applicable) + //---------------------------------------------------------------------- + //@Bug 2205 Check whether all rows go to the new extent + RID lastRid = 0; + RID lastRidNew = 0; + + if (totalRow - rowsLeft > 0) + { + lastRid = rowIdArray[totalRow - rowsLeft - 1]; + lastRidNew = rowIdArray[totalRow - 1]; + } + else + { + lastRid = 0; + lastRidNew = rowIdArray[totalRow - 1]; + } + + // if a new extent is created, all the columns in this table should + // have their own new extent + + //@Bug 1701. Close the file + if (bUseStartExtent) + { + m_colOp[op(curCol.compressionType)]->clearColumn(curCol); + } + + std::vector hwmVecNewext; + std::vector hwmVecOldext; + + if (newExtent) // Save all hwms to set them later. + { + BulkSetHWMArg aHwmEntryNew; + BulkSetHWMArg aHwmEntryOld; + + bool succFlag = false; + unsigned colWidth = 0; + int curFbo = 0, curBio; + + for (i = 0; i < totalColumns; i++) + { + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + + // @Bug 2714 need to set hwm for the old extent + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + if (succFlag) + { + if ((HWM)curFbo > oldHwm) { - return rc; + aHwmEntryOld.oid = colStructList[i].dataOid; + aHwmEntryOld.partNum = colStructList[i].fColPartition; + aHwmEntryOld.segNum = colStructList[i].fColSegment; + aHwmEntryOld.hwm = curFbo; + hwmVecOldext.push_back(aHwmEntryOld); } + } + else + return ERR_INVALID_PARAM; + + colWidth = newColStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + if (succFlag) + { + aHwmEntryNew.oid = newColStructList[i].dataOid; + aHwmEntryNew.partNum = newColStructList[i].fColPartition; + aHwmEntryNew.segNum = newColStructList[i].fColSegment; + aHwmEntryNew.hwm = curFbo; + hwmVecNewext.push_back(aHwmEntryNew); + } } - bool newFile; - -#ifdef PROFILE - timer.start("allocRowId"); -#endif - newColStructList = colStructList; - newDctnryStructList = dctnryStructList; - std::vector > dbRootExtentTrackers; - rc = colOp->allocRowId(txnid, bUseStartExtent, - curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, - rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, - dbRootExtentTrackers, false, false, 0, false, NULL); - - //-------------------------------------------------------------------------- - // Handle case where we ran out of disk space allocating a new extent. - // Rollback extentmap and delete any db files that were created. - //-------------------------------------------------------------------------- - if (rc != NO_ERROR) - { - if ((rc == ERR_FILE_DISK_SPACE) && newExtent) - { - vector colExtentInfo; - vector dictExtentInfo; - vector fileInfo; - ExtentInfo info; - - for (i = 0; i < newColStructList.size(); i++) - { - info.oid = newColStructList[i].dataOid; - info.partitionNum = newColStructList[i].fColPartition; - info.segmentNum = newColStructList[i].fColSegment; - info.dbRoot = newColStructList[i].fColDbRoot; - - if (newFile) - fileInfo.push_back (info); - - colExtentInfo.push_back (info); - } - - int rc1 = BRMWrapper::getInstance()->deleteEmptyColExtents(colExtentInfo); - - // Only rollback dictionary extents "if" store file is new - if ((rc1 == 0) && newFile) - { - for (unsigned int j = 0; j < fileInfo.size(); j++) - { - // ignore return code and delete what we can - rc1 = colOp->deleteFile(fileInfo[j].oid, - fileInfo[j].dbRoot, - fileInfo[j].partitionNum, - fileInfo[j].segmentNum); - } - - fileInfo.clear(); - - for (i = 0; i < newDctnryStructList.size(); i++) - { - if (newDctnryStructList[i].dctnryOid > 0) - { - info.oid = newDctnryStructList[i].dctnryOid; - info.partitionNum = newDctnryStructList[i].fColPartition; - info.segmentNum = newDctnryStructList[i].fColSegment; - info.dbRoot = newDctnryStructList[i].fColDbRoot; - info.newFile = true; - fileInfo.push_back (info); - dictExtentInfo.push_back (info); - } - } - - if (dictExtentInfo.size() > 0) - { - FileOp fileOp; - rc1 = BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(dictExtentInfo); - - if ( rc1 != NO_ERROR) - return rc; - - for (unsigned j = 0; j < fileInfo.size(); j++) - { - rc1 = fileOp.deleteFile(fileInfo[j].oid, - fileInfo[j].dbRoot, - fileInfo[j].partitionNum, - fileInfo[j].segmentNum); - } - } - } - } // disk space error allocating new extent - - return rc; - } // rc != NO_ERROR from call to allocRowID() - -#ifdef PROFILE - timer.stop("allocRowId"); -#endif - - TableMetaData* aTableMetaData = TableMetaData::makeTableMetaData(tableOid); - - //-------------------------------------------------------------------------- - // Expand initial abbreviated extent if any RID in 1st extent is > 256K. - // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. - //-------------------------------------------------------------------------- - // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((colStructList[colId].fColPartition == 0) && - (colStructList[colId].fColSegment == 0) && - ((totalRow - rowsLeft) > 0) && - (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) - { - for (unsigned k=0; ksetColParam(expandCol, 0, - colStructList[k].colWidth, - colStructList[k].colDataType, - colStructList[k].colType, - colStructList[k].dataOid, - colStructList[k].fCompressionType, - colStructList[k].fColDbRoot, - colStructList[k].fColPartition, - colStructList[k].fColSegment); - colOp->findTypeHandler(colStructList[k].colWidth, - colStructList[k].colDataType); - rc = colOp->openColumnFile(expandCol, segFile, true); // @bug 5572 HDFS tmp file - - if (rc == NO_ERROR) - { - if (colOp->abbreviatedExtent( - expandCol.dataFile.pFile, colStructList[k].colWidth)) - { - rc = colOp->expandAbbrevExtent(expandCol); - } - } - - colOp->clearColumn(expandCol); // closes the file - - if (rc != NO_ERROR) - { - return rc; - } - } // loop through columns - } // if starting extent needs to be expanded - - //-------------------------------------------------------------------------- - // Tokenize data if needed - //-------------------------------------------------------------------------- - dictStr::iterator dctStr_iter; - ColTupleList::iterator col_iter; - - for (unsigned i = 0; i < colStructList.size(); i++) - { - if (colStructList[i].tokenFlag) - { - dctStr_iter = dictStrList[i].begin(); - col_iter = colValueList[i].begin(); - Dctnry* dctnry = m_dctnry[op(dctnryStructList[i].fCompressionType)]; - - ColExtsInfo aColExtsInfo = aTableMetaData->getColExtsInfo(dctnryStructList[i].dctnryOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - if (bUseStartExtent) - { - rc = dctnry->openDctnry(dctnryStructList[i].dctnryOid, - dctnryStructList[i].fColDbRoot, - dctnryStructList[i].fColPartition, - dctnryStructList[i].fColSegment, - true); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == dctnryStructList[i].fColDbRoot) && (it->partNum == dctnryStructList[i].fColPartition) && (it->segNum == dctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = dctnryStructList[i].fColDbRoot; - aExt.partNum = dctnryStructList[i].fColPartition; - aExt.segNum = dctnryStructList[i].fColSegment; - aExt.compType = dctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - aTableMetaData->setColExtsInfo(dctnryStructList[i].dctnryOid, aColExtsInfo); - } - - - for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) - { - if (dctStr_iter->length() == 0) - { - Token nullToken; - col_iter->data = nullToken; - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, - dctTuple, - dctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - col_iter->data = dctTuple.token; - } - - dctStr_iter++; - col_iter++; - } - - //close dictionary files - rc = dctnry->closeDctnry(); - - if (rc != NO_ERROR) - return rc; - } // tokenize dictionary rows in 1st extent - - if (newExtent) - { - rc = dctnry->openDctnry(newDctnryStructList[i].dctnryOid, - newDctnryStructList[i].fColDbRoot, - newDctnryStructList[i].fColPartition, - newDctnryStructList[i].fColSegment, - false); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - return rc; - - aColExtsInfo = aTableMetaData->getColExtsInfo(newDctnryStructList[i].dctnryOid); - it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newDctnryStructList[i].fColDbRoot) && (it->partNum == newDctnryStructList[i].fColPartition) && (it->segNum == newDctnryStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = newDctnryStructList[i].fColDbRoot; - aExt.partNum = newDctnryStructList[i].fColPartition; - aExt.segNum = newDctnryStructList[i].fColSegment; - aExt.compType = newDctnryStructList[i].fCompressionType; - aExt.isDict = true; - aColExtsInfo.push_back(aExt); - aTableMetaData->setColExtsInfo(newDctnryStructList[i].dctnryOid, aColExtsInfo); - } - - for (uint32_t rows = 0; rows < rowsLeft; rows++) - { - if (dctStr_iter->length() == 0) - { - Token nullToken; - col_iter->data = nullToken; - } - else - { -#ifdef PROFILE - timer.start("tokenize"); -#endif - DctnryTuple dctTuple; - dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); - dctTuple.sigSize = dctStr_iter->length(); - dctTuple.isNull = false; - rc = tokenize(txnid, - dctTuple, - newDctnryStructList[i].fCompressionType); - - if (rc != NO_ERROR) - { - dctnry->closeDctnry(); - return rc; - } - -#ifdef PROFILE - timer.stop("tokenize"); -#endif - col_iter->data = dctTuple.token; - } - - dctStr_iter++; - col_iter++; - } - - //close dictionary files - rc = dctnry->closeDctnry(); - - if (rc != NO_ERROR) - return rc; - } // tokenize dictionary rows in second extent - } // tokenize dictionary columns - } // loop through columns to see which ones need tokenizing - //---------------------------------------------------------------------- - // Update column info structure @Bug 1862 set hwm, and - // Prepare ValueList for new extent (if applicable) + // Prepare the valuelist for the new extent //---------------------------------------------------------------------- - //@Bug 2205 Check whether all rows go to the new extent - RID lastRid = 0; - RID lastRidNew = 0; + ColTupleList colTupleList; + ColTupleList newColTupleList; + ColTupleList firstPartTupleList; - if (totalRow - rowsLeft > 0) + for (unsigned i = 0; i < totalColumns; i++) { - lastRid = rowIdArray[totalRow - rowsLeft - 1]; - lastRidNew = rowIdArray[totalRow - 1]; + colTupleList = static_cast(colValueList[i]); + + for (uint64_t j = rowsLeft; j > 0; j--) + { + newColTupleList.push_back(colTupleList[totalRow - j]); + } + + colNewValueList.push_back(newColTupleList); + + newColTupleList.clear(); + + // upate the oldvalue list for the old extent + for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) + { + firstPartTupleList.push_back(colTupleList[j]); + } + + colOldValueList.push_back(firstPartTupleList); + firstPartTupleList.clear(); + } + } + + //-------------------------------------------------------------------------- + // Mark extents invalid + //-------------------------------------------------------------------------- + // WIP + // Set min/max in dmlprocprocessor if aplicable + vector lbids; + vector colDataTypes; + bool successFlag = true; + unsigned width = 0; + int curFbo = 0, curBio, lastFbo = -1; + lastRid = rowIdArray[totalRow - 1]; + + for (unsigned i = 0; i < colStructList.size(); i++) + { + colOp = m_colOp[op(colStructList[i].fCompressionType)]; + width = colStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + colDataTypes.push_back(colStructList[i].colDataType); + RETURN_ON_ERROR(AddLBIDtoList(txnid, colStructList[i], curFbo)); + } + } + } + + markTxnExtentsAsInvalid(txnid); + + //-------------------------------------------------------------------------- + // Write row(s) to database file(s) + //-------------------------------------------------------------------------- +#ifdef PROFILE + timer.start("writeColumnRec"); +#endif + + if (rc == NO_ERROR) + { + if (newExtent) + { + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, + colNewValueList, tableOid, + false); // @bug 5572 HDFS tmp file } else { - lastRid = 0; - lastRidNew = rowIdArray[totalRow - 1]; - } - - //if a new extent is created, all the columns in this table should - //have their own new extent - - //@Bug 1701. Close the file - if (bUseStartExtent) - { - m_colOp[op(curCol.compressionType)]->clearColumn(curCol); - } - - std::vector hwmVecNewext; - std::vector hwmVecOldext; - - if (newExtent) //Save all hwms to set them later. - { - BulkSetHWMArg aHwmEntryNew; - BulkSetHWMArg aHwmEntryOld; - - bool succFlag = false; - unsigned colWidth = 0; - int curFbo = 0, curBio; - - for (i = 0; i < totalColumns; i++) - { - colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - - // @Bug 2714 need to set hwm for the old extent - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, - BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - if (succFlag) - { - if ((HWM)curFbo > oldHwm) - { - aHwmEntryOld.oid = colStructList[i].dataOid; - aHwmEntryOld.partNum = colStructList[i].fColPartition; - aHwmEntryOld.segNum = colStructList[i].fColSegment; - aHwmEntryOld.hwm = curFbo; - hwmVecOldext.push_back(aHwmEntryOld); - } - } - else - return ERR_INVALID_PARAM; - - colWidth = newColStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRidNew, - BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - if (succFlag) - { - aHwmEntryNew.oid = newColStructList[i].dataOid; - aHwmEntryNew.partNum = newColStructList[i].fColPartition; - aHwmEntryNew.segNum = newColStructList[i].fColSegment; - aHwmEntryNew.hwm = curFbo; - hwmVecNewext.push_back(aHwmEntryNew); - } - } - - //---------------------------------------------------------------------- - // Prepare the valuelist for the new extent - //---------------------------------------------------------------------- - ColTupleList colTupleList; - ColTupleList newColTupleList; - ColTupleList firstPartTupleList; - - for (unsigned i = 0; i < totalColumns; i++) - { - colTupleList = static_cast(colValueList[i]); - - for (uint64_t j = rowsLeft; j > 0; j--) - { - newColTupleList.push_back(colTupleList[totalRow - j]); - } - - colNewValueList.push_back(newColTupleList); - - newColTupleList.clear(); - - //upate the oldvalue list for the old extent - for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) - { - firstPartTupleList.push_back(colTupleList[j]); - } - - colOldValueList.push_back(firstPartTupleList); - firstPartTupleList.clear(); - } - } - - //-------------------------------------------------------------------------- - //Mark extents invalid - //-------------------------------------------------------------------------- - // WIP - // Set min/max in dmlprocprocessor if aplicable - vector lbids; - vector colDataTypes; - bool successFlag = true; - unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; - lastRid = rowIdArray[totalRow - 1]; - - for (unsigned i = 0; i < colStructList.size(); i++) - { - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - width = colStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRid, - BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - colDataTypes.push_back(colStructList[i].colDataType); - RETURN_ON_ERROR(AddLBIDtoList(txnid, - colStructList[i], - curFbo)); - } - } - } - - markTxnExtentsAsInvalid(txnid); - - //-------------------------------------------------------------------------- - // Write row(s) to database file(s) - //-------------------------------------------------------------------------- -#ifdef PROFILE - timer.start("writeColumnRec"); -#endif - - if (rc == NO_ERROR) - { - if (newExtent) - { - rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, - rowIdArray, newColStructList, colNewValueList, tableOid, - false); // @bug 5572 HDFS tmp file - } - else - { - rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, - rowIdArray, newColStructList, colNewValueList, tableOid, - true); // @bug 5572 HDFS tmp file - } + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, + colNewValueList, tableOid, + true); // @bug 5572 HDFS tmp file } + } #ifdef PROFILE - timer.stop("writeColumnRec"); + timer.stop("writeColumnRec"); #endif - //-------------------------------------------------------------------------- - // Update BRM - //-------------------------------------------------------------------------- - if ( !newExtent ) - { - bool succFlag = false; - unsigned colWidth = 0; - int extState; - bool extFound; - int curFbo = 0, curBio; - std::vector hwmVec; + //-------------------------------------------------------------------------- + // Update BRM + //-------------------------------------------------------------------------- + if (!newExtent) + { + bool succFlag = false; + unsigned colWidth = 0; + int extState; + bool extFound; + int curFbo = 0, curBio; + std::vector hwmVec; - for (unsigned i = 0; i < totalColumns; i++) + for (unsigned i = 0; i < totalColumns; i++) + { + // Set all columns hwm together + BulkSetHWMArg aHwmEntry; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( + colStructList[i].dataOid, colStructList[i].fColDbRoot, colStructList[i].fColPartition, + colStructList[i].fColSegment, hwm, extState, extFound)); + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + + // cout << "insertcolumnrec oid:rid:fbo:hwm = " << + // colStructList[i].dataOid << ":" << lastRid << ":" << + // curFbo << ":" << hwm << endl; + if (succFlag) + { + if ((HWM)curFbo > hwm) { - //Set all columns hwm together - BulkSetHWMArg aHwmEntry; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( - colStructList[i].dataOid, - colStructList[i].fColDbRoot, - colStructList[i].fColPartition, - colStructList[i].fColSegment, - hwm, - extState, extFound)); - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, - BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - - //cout << "insertcolumnrec oid:rid:fbo:hwm = " << - //colStructList[i].dataOid << ":" << lastRid << ":" << - //curFbo << ":" << hwm << endl; - if (succFlag) - { - if ((HWM)curFbo > hwm) - { - aHwmEntry.oid = colStructList[i].dataOid; - aHwmEntry.partNum = colStructList[i].fColPartition; - aHwmEntry.segNum = colStructList[i].fColSegment; - aHwmEntry.hwm = curFbo; - hwmVec.push_back(aHwmEntry); - } - } - else - return ERR_INVALID_PARAM; + aHwmEntry.oid = colStructList[i].dataOid; + aHwmEntry.partNum = colStructList[i].fColPartition; + aHwmEntry.segNum = colStructList[i].fColSegment; + aHwmEntry.hwm = curFbo; + hwmVec.push_back(aHwmEntry); } - - std::vector mergeCPDataArgs; - RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( - hwmVec, mergeCPDataArgs)); - } - else // if (newExtent) - { -#ifdef PROFILE - timer.start("flushVMCache"); -#endif - std::vector mergeCPDataArgs; - - if (hwmVecNewext.size() > 0) - RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( - hwmVecNewext, mergeCPDataArgs)); - - if (hwmVecOldext.size() > 0) - RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( - hwmVecOldext, mergeCPDataArgs)); - -#ifdef PROFILE - timer.stop("flushVMCache"); -#endif + } + else + return ERR_INVALID_PARAM; } + std::vector mergeCPDataArgs; + RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP(hwmVec, mergeCPDataArgs)); + } + else // if (newExtent) + { #ifdef PROFILE - timer.finish(); + timer.start("flushVMCache"); #endif - return rc; + std::vector mergeCPDataArgs; + + if (hwmVecNewext.size() > 0) + RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP(hwmVecNewext, mergeCPDataArgs)); + + if (hwmVecOldext.size() > 0) + RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP(hwmVecOldext, mergeCPDataArgs)); + +#ifdef PROFILE + timer.stop("flushVMCache"); +#endif + } + +#ifdef PROFILE + timer.finish(); +#endif + return rc; } /*@brief printInputValue - Print input value -*/ + */ /*********************************************************** * DESCRIPTION: * Print input value @@ -4188,108 +4080,110 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, * RETURN: * none ***********************************************************/ -void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, - const ColValueList& colValueList, - const RIDList& ridList, - const DctnryStructList& dctnryStructList, - const DictStrList& dictStrList) const +void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, const ColValueList& colValueList, + const RIDList& ridList, const DctnryStructList& dctnryStructList, + const DictStrList& dictStrList) const { - ColTupleList curTupleList; - ColStruct curColStruct; - ColTuple curTuple; - string curStr; - ColStructList::size_type i; - size_t j; - OidToIdxMap oidToIdxMap; + ColTupleList curTupleList; + ColStruct curColStruct; + ColTuple curTuple; + string curStr; + ColStructList::size_type i; + size_t j; + OidToIdxMap oidToIdxMap; - std::cerr << std::endl << "=========================" << std::endl; + std::cerr << std::endl << "=========================" << std::endl; - std::cerr << "Total RIDs: " << ridList.size() << std::endl; + std::cerr << "Total RIDs: " << ridList.size() << std::endl; - for (i = 0; i < ridList.size(); i++) - std::cerr << "RID[" << i << "] : " << ridList[i] << std::endl; + for (i = 0; i < ridList.size(); i++) + std::cerr << "RID[" << i << "] : " << ridList[i] << std::endl; - std::cerr << "Total Columns: " << colStructList.size() << std::endl; + std::cerr << "Total Columns: " << colStructList.size() << std::endl; - for (i = 0; i < colStructList.size(); i++) + for (i = 0; i < colStructList.size(); i++) + { + curColStruct = colStructList[i]; + curTupleList = colValueList[i]; + if (curColStruct.tokenFlag) { - curColStruct = colStructList[i]; - curTupleList = colValueList[i]; - if (curColStruct.tokenFlag) - { - oidToIdxMap.insert({curColStruct.dataOid, i}); - continue; - } - - std::cerr << "Column[" << i << "]"; - std::cerr << "Data file OID : " << curColStruct.dataOid << "\t"; - std::cerr << "Width : " << curColStruct.colWidth << "\t" << " Type: " << curColStruct.colDataType << std::endl; - std::cerr << "Total values : " << curTupleList.size() << std::endl; - - for (j = 0; j < curTupleList.size(); j++) - { - curTuple = curTupleList[j]; - - try - { - if (curTuple.data.type() == typeid(int)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(unsigned int)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(float)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(long long)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(unsigned long long)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(int128_t)) - curStr = datatypes::TSInt128(boost::any_cast(curTuple.data)).toString(); - else if (curTuple.data.type() == typeid(double)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(short)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(unsigned short)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else if (curTuple.data.type() == typeid(char)) - curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); - else - curStr = boost::any_cast(curTuple.data); - } - catch (...) - { - } - - if (isDebug(DEBUG_3)) - std::cerr << "Value[" << j << "]: " << curStr.c_str() << std::endl; - } - } - for (i = 0; i < dctnryStructList.size(); ++i) - { - if (dctnryStructList[i].dctnryOid == 0) - continue; - std::cerr << "Dict[" << i << "]"; - std::cerr << " file OID : " << dctnryStructList[i].dctnryOid << " Token file OID: " << dctnryStructList[i].columnOid << "\t"; - std::cerr << "Width : " << dctnryStructList[i].colWidth << "\t" << " Type: " << dctnryStructList[i].fCompressionType << std::endl; - std::cerr << "Total values : " << dictStrList.size() << std::endl; - if (isDebug(DEBUG_3)) - { - for (j = 0; j < dictStrList[i].size(); ++j) - { - // We presume there will be a value. - auto tokenOidIdx = oidToIdxMap[dctnryStructList[i].columnOid]; - std::cerr << "string [" << dictStrList[i][j] << "]" << std::endl; - bool isToken = colStructList[tokenOidIdx].colType == WriteEngine::WR_TOKEN && - colStructList[tokenOidIdx].tokenFlag; - if (isToken && !colValueList[tokenOidIdx][j].data.empty()) - { - Token t = boost::any_cast(colValueList[tokenOidIdx][j].data); - std::cerr << "Token: block pos:[" << t.op << "] fbo: [" << t.fbo << "] bc: [" << t.bc << "]" << std::endl; - } - } - } + oidToIdxMap.insert({curColStruct.dataOid, i}); + continue; } - std::cerr << "=========================" << std::endl; + std::cerr << "Column[" << i << "]"; + std::cerr << "Data file OID : " << curColStruct.dataOid << "\t"; + std::cerr << "Width : " << curColStruct.colWidth << "\t" + << " Type: " << curColStruct.colDataType << std::endl; + std::cerr << "Total values : " << curTupleList.size() << std::endl; + + for (j = 0; j < curTupleList.size(); j++) + { + curTuple = curTupleList[j]; + + try + { + if (curTuple.data.type() == typeid(int)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(unsigned int)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(float)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(long long)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(unsigned long long)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(int128_t)) + curStr = datatypes::TSInt128(boost::any_cast(curTuple.data)).toString(); + else if (curTuple.data.type() == typeid(double)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(short)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(unsigned short)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(char)) + curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else + curStr = boost::any_cast(curTuple.data); + } + catch (...) + { + } + + if (isDebug(DEBUG_3)) + std::cerr << "Value[" << j << "]: " << curStr.c_str() << std::endl; + } + } + for (i = 0; i < dctnryStructList.size(); ++i) + { + if (dctnryStructList[i].dctnryOid == 0) + continue; + std::cerr << "Dict[" << i << "]"; + std::cerr << " file OID : " << dctnryStructList[i].dctnryOid + << " Token file OID: " << dctnryStructList[i].columnOid << "\t"; + std::cerr << "Width : " << dctnryStructList[i].colWidth << "\t" + << " Type: " << dctnryStructList[i].fCompressionType << std::endl; + std::cerr << "Total values : " << dictStrList.size() << std::endl; + if (isDebug(DEBUG_3)) + { + for (j = 0; j < dictStrList[i].size(); ++j) + { + // We presume there will be a value. + auto tokenOidIdx = oidToIdxMap[dctnryStructList[i].columnOid]; + std::cerr << "string [" << dictStrList[i][j] << "]" << std::endl; + bool isToken = colStructList[tokenOidIdx].colType == WriteEngine::WR_TOKEN && + colStructList[tokenOidIdx].tokenFlag; + if (isToken && !colValueList[tokenOidIdx][j].data.empty()) + { + Token t = boost::any_cast(colValueList[tokenOidIdx][j].data); + std::cerr << "Token: block pos:[" << t.op << "] fbo: [" << t.fbo << "] bc: [" << t.bc << "]" + << std::endl; + } + } + } + } + + std::cerr << "=========================" << std::endl; } /*********************************************************** @@ -4305,172 +4199,176 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, * others if something wrong in inserting the value ***********************************************************/ int WriteEngineWrapper::processVersionBuffer(IDBDataFile* pFile, const TxnID& txnid, - const ColStruct& colStruct, int width, - int totalRow, const RID* rowIdArray, vector& rangeList) + const ColStruct& colStruct, int width, int totalRow, + const RID* rowIdArray, vector& rangeList) { - if (idbdatafile::IDBPolicy::useHdfs()) - return 0; + if (idbdatafile::IDBPolicy::useHdfs()) + return 0; - RID curRowId; - int rc = NO_ERROR; - int curFbo = 0, curBio, lastFbo = -1; - bool successFlag; - BRM::LBID_t lbid; - BRM::VER_t verId = (BRM::VER_t) txnid; - vector fboList; - LBIDRange range; - ColumnOp* colOp = m_colOp[op(colStruct.fCompressionType)]; + RID curRowId; + int rc = NO_ERROR; + int curFbo = 0, curBio, lastFbo = -1; + bool successFlag; + BRM::LBID_t lbid; + BRM::VER_t verId = (BRM::VER_t)txnid; + vector fboList; + LBIDRange range; + ColumnOp* colOp = m_colOp[op(colStruct.fCompressionType)]; - for (int i = 0; i < totalRow; i++) + for (int i = 0; i < totalRow; i++) + { + curRowId = rowIdArray[i]; + // cout << "processVersionBuffer got rid " << curRowId << endl; + successFlag = colOp->calculateRowId(curRowId, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) { - curRowId = rowIdArray[i]; - //cout << "processVersionBuffer got rid " << curRowId << endl; - successFlag = colOp->calculateRowId(curRowId, BYTE_PER_BLOCK / width, width, curFbo, curBio); + if (curFbo != lastFbo) + { + // cout << "processVersionBuffer is processing lbid " << lbid << endl; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo(colStruct.dataOid, colStruct.fColPartition, + colStruct.fColSegment, curFbo, lbid)); + // cout << "processVersionBuffer is processing lbid " << lbid << endl; + fboList.push_back((uint32_t)curFbo); + range.start = lbid; + range.size = 1; + rangeList.push_back(range); + } - if (successFlag) - { - if (curFbo != lastFbo) - { - //cout << "processVersionBuffer is processing lbid " << lbid << endl; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo( - colStruct.dataOid, colStruct.fColPartition, colStruct.fColSegment, curFbo, lbid)); - //cout << "processVersionBuffer is processing lbid " << lbid << endl; - fboList.push_back((uint32_t)curFbo); - range.start = lbid; - range.size = 1; - rangeList.push_back(range); - } - - lastFbo = curFbo; - } + lastFbo = curFbo; } + } - std::vector freeList; - rc = BRMWrapper::getInstance()-> - writeVB(pFile, verId, colStruct.dataOid, fboList, rangeList, colOp, freeList, colStruct.fColDbRoot); + std::vector freeList; + rc = BRMWrapper::getInstance()->writeVB(pFile, verId, colStruct.dataOid, fboList, rangeList, colOp, + freeList, colStruct.fColDbRoot); - return rc; + return rc; } int WriteEngineWrapper::processVersionBuffers(IDBDataFile* pFile, const TxnID& txnid, - const ColStruct& colStruct, int width, - int totalRow, const RIDList& ridList, - vector& rangeList) + const ColStruct& colStruct, int width, int totalRow, + const RIDList& ridList, vector& rangeList) { - if (idbdatafile::IDBPolicy::useHdfs()) - return 0; + if (idbdatafile::IDBPolicy::useHdfs()) + return 0; - RID curRowId; - int rc = NO_ERROR; - int curFbo = 0, curBio, lastFbo = -1; - bool successFlag; - BRM::LBID_t lbid; - BRM::VER_t verId = (BRM::VER_t) txnid; - LBIDRange range; - vector fboList; - //vector rangeList; - ColumnOp* colOp = m_colOp[op(colStruct.fCompressionType)]; + RID curRowId; + int rc = NO_ERROR; + int curFbo = 0, curBio, lastFbo = -1; + bool successFlag; + BRM::LBID_t lbid; + BRM::VER_t verId = (BRM::VER_t)txnid; + LBIDRange range; + vector fboList; + // vector rangeList; + ColumnOp* colOp = m_colOp[op(colStruct.fCompressionType)]; - for (int i = 0; i < totalRow; i++) + for (int i = 0; i < totalRow; i++) + { + curRowId = ridList[i]; + // cout << "processVersionBuffer got rid " << curRowId << endl; + successFlag = colOp->calculateRowId(curRowId, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) { - curRowId = ridList[i]; - //cout << "processVersionBuffer got rid " << curRowId << endl; - successFlag = colOp->calculateRowId(curRowId, BYTE_PER_BLOCK / width, width, curFbo, curBio); + if (curFbo != lastFbo) + { + // cout << "processVersionBuffer is processing lbid " << lbid << endl; + RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo(colStruct.dataOid, colStruct.fColPartition, + colStruct.fColSegment, curFbo, lbid)); + // cout << "processVersionBuffer is processing lbid " << lbid << endl; + fboList.push_back((uint32_t)curFbo); + range.start = lbid; + range.size = 1; + rangeList.push_back(range); + } - if (successFlag) - { - if (curFbo != lastFbo) - { - //cout << "processVersionBuffer is processing lbid " << lbid << endl; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo( - colStruct.dataOid, colStruct.fColPartition, colStruct.fColSegment, curFbo, lbid)); - //cout << "processVersionBuffer is processing lbid " << lbid << endl; - fboList.push_back((uint32_t)curFbo); - range.start = lbid; - range.size = 1; - rangeList.push_back(range); - } - - lastFbo = curFbo; - } + lastFbo = curFbo; } + } -//cout << "calling writeVB with blocks " << rangeList.size() << endl; - std::vector freeList; - rc = BRMWrapper::getInstance()-> - writeVB(pFile, verId, colStruct.dataOid, fboList, rangeList, colOp, freeList, colStruct.fColDbRoot); + // cout << "calling writeVB with blocks " << rangeList.size() << endl; + std::vector freeList; + rc = BRMWrapper::getInstance()->writeVB(pFile, verId, colStruct.dataOid, fboList, rangeList, colOp, + freeList, colStruct.fColDbRoot); - return rc; + return rc; } -int WriteEngineWrapper::processBeginVBCopy(const TxnID& txnid, const vector& colStructList, const RIDList& ridList, - std::vector& freeList, vector >& fboLists, vector >& rangeLists, - vector& rangeListTot) +int WriteEngineWrapper::processBeginVBCopy(const TxnID& txnid, const vector& colStructList, + const RIDList& ridList, std::vector& freeList, + vector >& fboLists, + vector >& rangeLists, + vector& rangeListTot) { - if (idbdatafile::IDBPolicy::useHdfs()) - return 0; + if (idbdatafile::IDBPolicy::useHdfs()) + return 0; - RID curRowId; - int rc = NO_ERROR; - int curFbo = 0, curBio, lastFbo = -1; - bool successFlag; - BRM::LBID_t lbid; - LBIDRange range; + RID curRowId; + int rc = NO_ERROR; + int curFbo = 0, curBio, lastFbo = -1; + bool successFlag; + BRM::LBID_t lbid; + LBIDRange range; -//StopWatch timer; -// timer.start("calculation"); - for (uint32_t j = 0; j < colStructList.size(); j++) + // StopWatch timer; + // timer.start("calculation"); + for (uint32_t j = 0; j < colStructList.size(); j++) + { + vector fboList; + vector rangeList; + lastFbo = -1; + ColumnOp* colOp = m_colOp[op(colStructList[j].fCompressionType)]; + + ColStruct curColStruct = colStructList[j]; + Convertor::convertColType(&curColStruct); + + for (uint32_t i = 0; i < ridList.size(); i++) { - vector fboList; - vector rangeList; - lastFbo = -1; - ColumnOp* colOp = m_colOp[op(colStructList[j].fCompressionType)]; + curRowId = ridList[i]; + // cout << "processVersionBuffer got rid " << curRowId << endl; + successFlag = colOp->calculateRowId(curRowId, BYTE_PER_BLOCK / curColStruct.colWidth, + curColStruct.colWidth, curFbo, curBio); - ColStruct curColStruct = colStructList[j]; - Convertor::convertColType(&curColStruct); - - for (uint32_t i = 0; i < ridList.size(); i++) + if (successFlag) + { + if (curFbo != lastFbo) { - curRowId = ridList[i]; - //cout << "processVersionBuffer got rid " << curRowId << endl; - successFlag = colOp->calculateRowId(curRowId, BYTE_PER_BLOCK / curColStruct.colWidth, curColStruct.colWidth, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - //cout << "processVersionBuffer is processing curFbo " << curFbo << endl; - RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo( - colStructList[j].dataOid, colStructList[j].fColPartition, colStructList[j].fColSegment, curFbo, lbid)); - //cout << "beginVBCopy is processing lbid:transaction " << lbid <<":"<getBrmInfo(colStructList[j].dataOid, + colStructList[j].fColPartition, + colStructList[j].fColSegment, curFbo, lbid)); + // cout << "beginVBCopy is processing lbid:transaction " << lbid <<":"<pruneLBIDList(txnid, &rangeList, &fboList); - rangeLists.push_back(rangeList); - - fboLists.push_back(fboList); - rangeListTot.insert(rangeListTot.end(), rangeList.begin(), rangeList.end()); + lastFbo = curFbo; + } } - if (rangeListTot.size() > 0) - rc = BRMWrapper::getInstance()->getDbrmObject()->beginVBCopy(txnid, colStructList[0].fColDbRoot, rangeListTot, freeList); + BRMWrapper::getInstance()->pruneLBIDList(txnid, &rangeList, &fboList); + rangeLists.push_back(rangeList); -//timer.stop("beginVBCopy"); -//timer.finish(); - return rc; + fboLists.push_back(fboList); + rangeListTot.insert(rangeListTot.end(), rangeList.begin(), rangeList.end()); + } + + if (rangeListTot.size() > 0) + rc = BRMWrapper::getInstance()->getDbrmObject()->beginVBCopy(txnid, colStructList[0].fColDbRoot, + rangeListTot, freeList); + + // timer.stop("beginVBCopy"); + // timer.finish(); + return rc; } /** -* @brief Process versioning for batch insert - only version the hwm block. -*/ + * @brief Process versioning for batch insert - only version the hwm block. + */ #if 0 int WriteEngineWrapper::processBatchVersions(const TxnID& txnid, std::vector columns, std::vector& rangeList) { @@ -4526,429 +4424,421 @@ int WriteEngineWrapper::processBatchVersions(const TxnID& txnid, std::vector& rangeList) +void WriteEngineWrapper::writeVBEnd(const TxnID& txnid, std::vector& rangeList) { - if (idbdatafile::IDBPolicy::useHdfs()) - return; + if (idbdatafile::IDBPolicy::useHdfs()) + return; - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); } -int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, - const vector& colExtentsColType, - vector& colExtentsStruct, - ColValueList& colValueList, - vector& colOldValueList, - vector& ridLists, +int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const vector& colExtentsColType, + vector& colExtentsStruct, ColValueList& colValueList, + vector& colOldValueList, vector& ridLists, vector& dctnryExtentsStruct, - DctnryValueList& dctnryValueList, - const int32_t tableOid) + DctnryValueList& dctnryValueList, const int32_t tableOid) { - int rc = 0; - unsigned numExtents = colExtentsStruct.size(); - ColStructList colStructList; - DctnryStructList dctnryStructList; - WriteEngine::CSCTypesList cscColTypeList; - ColumnOp* colOp = NULL; - ExtCPInfoList infosToUpdate; + int rc = 0; + unsigned numExtents = colExtentsStruct.size(); + ColStructList colStructList; + DctnryStructList dctnryStructList; + WriteEngine::CSCTypesList cscColTypeList; + ColumnOp* colOp = NULL; + ExtCPInfoList infosToUpdate; + + if (m_opType != DELETE) + { + m_opType = UPDATE; + } + + for (unsigned extent = 0; extent < numExtents; extent++) + { + colStructList = colExtentsStruct[extent]; + dctnryStructList = dctnryExtentsStruct[extent]; + cscColTypeList = colExtentsColType[extent]; if (m_opType != DELETE) { - m_opType = UPDATE; - } + // Tokenize data if needed + vector tokenList; - for (unsigned extent = 0; extent < numExtents; extent++) - { - colStructList = colExtentsStruct[extent]; - dctnryStructList = dctnryExtentsStruct[extent]; - cscColTypeList = colExtentsColType[extent]; + DctColTupleList::iterator dctCol_iter; + ColTupleList::iterator col_iter; - if (m_opType != DELETE) + for (unsigned i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) { - //Tokenize data if needed - vector tokenList; + // only need to tokenize once + dctCol_iter = dctnryValueList[i].begin(); + Token token; - DctColTupleList::iterator dctCol_iter; - ColTupleList::iterator col_iter; - - for (unsigned i = 0; i < colStructList.size(); i++) - { - if (colStructList[i].tokenFlag) - { - // only need to tokenize once - dctCol_iter = dctnryValueList[i].begin(); - Token token; - - if (!dctCol_iter->isNull) - { - RETURN_ON_ERROR_REPORT(tokenize( - txnid, dctnryStructList[i], *dctCol_iter, true)); // @bug 5572 HDFS tmp file - token = dctCol_iter->token; + if (!dctCol_iter->isNull) + { + RETURN_ON_ERROR_REPORT( + tokenize(txnid, dctnryStructList[i], *dctCol_iter, true)); // @bug 5572 HDFS tmp file + token = dctCol_iter->token; #ifdef PROFILE -//timer.stop("tokenize"); +// timer.stop("tokenize"); #endif - } + } - tokenList.push_back(token); - } - } - - int dicPos = 0; - - for (unsigned i = 0; i < colStructList.size(); i++) - { - if (colStructList[i].tokenFlag) - { - // only need to tokenize once - col_iter = colValueList[i].begin(); - - while (col_iter != colValueList[i].end()) - { - col_iter->data = tokenList[dicPos]; - col_iter++; - } - - dicPos++; - } - } + tokenList.push_back(token); } + } - RIDList::iterator rid_iter; - //Mark extents invalid - bool successFlag = true; - unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; - rid_iter = ridLists[extent].begin(); - RID aRid = *rid_iter; + int dicPos = 0; - ExtCPInfoList currentExtentRanges; - for (const auto& colStruct : colStructList) + for (unsigned i = 0; i < colStructList.size(); i++) + { + if (colStructList[i].tokenFlag) { - currentExtentRanges.push_back(ExtCPInfo(colStruct.colDataType, colStruct.colWidth)); // temporary for each extent. - } - std::vector currentExtentRangesPtrs(colStructList.size(), NULL); // pointers for each extent. - - for (unsigned j = 0; j < colStructList.size(); j++) - { - colOp = m_colOp[op(colStructList[j].fCompressionType)]; - ExtCPInfo* cpInfoP = &(currentExtentRanges[j]); - cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[j], cpInfoP); - currentExtentRangesPtrs[j] = cpInfoP; - - if (colStructList[j].tokenFlag) - continue; - - width = colOp->getCorrectRowWidth(colStructList[j].colDataType, colStructList[j].colWidth); - successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) - { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR_REPORT(AddLBIDtoList(txnid, - colStructList[j], - curFbo, cpInfoP)); - } - } - } - -//#ifdef PROFILE -//timer.start("markExtentsInvalid"); -//#endif - - if (m_opType != DELETE) - m_opType = UPDATE; - - rc = writeColumnRecUpdate(txnid, cscColTypeList, colStructList, colValueList, colOldValueList, - ridLists[extent], tableOid, true, ridLists[extent].size(), ¤tExtentRangesPtrs); - - if (rc != NO_ERROR) - break; - - // copy updated ranges into bulk update vector. - for(auto cpInfoPtr : currentExtentRangesPtrs) - { - if (cpInfoPtr) - { - cpInfoPtr->fCPInfo.seqNum ++; - infosToUpdate.push_back(*cpInfoPtr); - } + // only need to tokenize once + col_iter = colValueList[i].begin(); + + while (col_iter != colValueList[i].end()) + { + col_iter->data = tokenList[dicPos]; + col_iter++; + } + + dicPos++; } + } } - markTxnExtentsAsInvalid(txnid); - if (rc == NO_ERROR) - { - ExtCPInfoList infosToDrop = infosToUpdate; - for (auto& cpInfo : infosToDrop) - { - cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; - } - rc = BRMWrapper::getInstance()->setExtentsMaxMin(infosToDrop); - setInvalidCPInfosSpecialMarks(infosToUpdate); - rc = BRMWrapper::getInstance()->setExtentsMaxMin(infosToUpdate); - } - return rc; -} -int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - vector& colExtentsStruct, - ColValueList& colValueList, - const RIDList& ridLists, - const int32_t tableOid) -{ - //Mark extents invalid - ColumnOp* colOp = NULL; + RIDList::iterator rid_iter; + // Mark extents invalid bool successFlag = true; unsigned width = 0; int curFbo = 0, curBio, lastFbo = -1; - RID aRid = ridLists[0]; - int rc = 0; - ExtCPInfoList infosToUpdate; - for (const auto& colStruct : colExtentsStruct) + rid_iter = ridLists[extent].begin(); + RID aRid = *rid_iter; + + ExtCPInfoList currentExtentRanges; + for (const auto& colStruct : colStructList) { - infosToUpdate.push_back(ExtCPInfo(colStruct.colDataType, colStruct.colWidth)); + currentExtentRanges.push_back( + ExtCPInfo(colStruct.colDataType, colStruct.colWidth)); // temporary for each extent. } - ExtCPInfoList bulkUpdateInfos; - std::vector pointersToInfos; // pointersToInfos[i] points to infosToUpdate[i] and may be NULL. + std::vector currentExtentRangesPtrs(colStructList.size(), NULL); // pointers for each extent. - m_opType = UPDATE; - - for (unsigned j = 0; j < colExtentsStruct.size(); j++) + for (unsigned j = 0; j < colStructList.size(); j++) { - colOp = m_colOp[op(colExtentsStruct[j].fCompressionType)]; + colOp = m_colOp[op(colStructList[j].fCompressionType)]; + ExtCPInfo* cpInfoP = &(currentExtentRanges[j]); + cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[j], cpInfoP); + currentExtentRangesPtrs[j] = cpInfoP; - ExtCPInfo* cpInfoP = &(infosToUpdate[j]); - cpInfoP = getCPInfoToUpdateForUpdatableType(colExtentsStruct[j], cpInfoP); - pointersToInfos.push_back(cpInfoP); + if (colStructList[j].tokenFlag) + continue; - if (colExtentsStruct[j].tokenFlag) - continue; + width = colOp->getCorrectRowWidth(colStructList[j].colDataType, colStructList[j].colWidth); + successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - width = colOp->getCorrectRowWidth(colExtentsStruct[j].colDataType, colExtentsStruct[j].colWidth); - successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) + if (successFlag) + { + if (curFbo != lastFbo) { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - colExtentsStruct[j], - curFbo, cpInfoP)); - } + RETURN_ON_ERROR_REPORT(AddLBIDtoList(txnid, colStructList[j], curFbo, cpInfoP)); } + } } - markTxnExtentsAsInvalid(txnid); + //#ifdef PROFILE + // timer.start("markExtentsInvalid"); + //#endif if (m_opType != DELETE) - m_opType = UPDATE; + m_opType = UPDATE; + rc = writeColumnRecUpdate(txnid, cscColTypeList, colStructList, colValueList, colOldValueList, + ridLists[extent], tableOid, true, ridLists[extent].size(), + ¤tExtentRangesPtrs); + + if (rc != NO_ERROR) + break; + + // copy updated ranges into bulk update vector. + for (auto cpInfoPtr : currentExtentRangesPtrs) + { + if (cpInfoPtr) + { + cpInfoPtr->fCPInfo.seqNum++; + infosToUpdate.push_back(*cpInfoPtr); + } + } + } + markTxnExtentsAsInvalid(txnid); + if (rc == NO_ERROR) + { + ExtCPInfoList infosToDrop = infosToUpdate; + for (auto& cpInfo : infosToDrop) + { + cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; + } + rc = BRMWrapper::getInstance()->setExtentsMaxMin(infosToDrop); + setInvalidCPInfosSpecialMarks(infosToUpdate); + rc = BRMWrapper::getInstance()->setExtentsMaxMin(infosToUpdate); + } + return rc; +} + +int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, const CSCTypesList& cscColTypeList, + vector& colExtentsStruct, ColValueList& colValueList, + const RIDList& ridLists, const int32_t tableOid) +{ + // Mark extents invalid + ColumnOp* colOp = NULL; + bool successFlag = true; + unsigned width = 0; + int curFbo = 0, curBio, lastFbo = -1; + RID aRid = ridLists[0]; + int rc = 0; + ExtCPInfoList infosToUpdate; + for (const auto& colStruct : colExtentsStruct) + { + infosToUpdate.push_back(ExtCPInfo(colStruct.colDataType, colStruct.colWidth)); + } + ExtCPInfoList bulkUpdateInfos; + std::vector pointersToInfos; // pointersToInfos[i] points to infosToUpdate[i] and may be NULL. + + m_opType = UPDATE; + + for (unsigned j = 0; j < colExtentsStruct.size(); j++) + { + colOp = m_colOp[op(colExtentsStruct[j].fCompressionType)]; + + ExtCPInfo* cpInfoP = &(infosToUpdate[j]); + cpInfoP = getCPInfoToUpdateForUpdatableType(colExtentsStruct[j], cpInfoP); + pointersToInfos.push_back(cpInfoP); + + if (colExtentsStruct[j].tokenFlag) + continue; + + width = colOp->getCorrectRowWidth(colExtentsStruct[j].colDataType, colExtentsStruct[j].colWidth); + successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, colExtentsStruct[j], curFbo, cpInfoP)); + } + } + } + + markTxnExtentsAsInvalid(txnid); + + if (m_opType != DELETE) + m_opType = UPDATE; + + for (auto cpInfoP : pointersToInfos) + { + if (cpInfoP) + { + auto tmp = *cpInfoP; + tmp.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; + bulkUpdateInfos.push_back(tmp); + } + } + if (!bulkUpdateInfos.empty()) + { + rc = BRMWrapper::getInstance()->setExtentsMaxMin(bulkUpdateInfos); + } + + rc = writeColumnRecords(txnid, cscColTypeList, colExtentsStruct, colValueList, ridLists, tableOid, true, + &pointersToInfos); + + if (rc == NO_ERROR) + { + bulkUpdateInfos.clear(); for (auto cpInfoP : pointersToInfos) { - if (cpInfoP) - { - auto tmp = *cpInfoP; - tmp.fCPInfo.seqNum = SEQNUM_MARK_INVALID_SET_RANGE; - bulkUpdateInfos.push_back(tmp); - } + if (cpInfoP) + { + cpInfoP->fCPInfo.seqNum++; + bulkUpdateInfos.push_back(*cpInfoP); + } } if (!bulkUpdateInfos.empty()) { - rc = BRMWrapper::getInstance()->setExtentsMaxMin(bulkUpdateInfos); + setInvalidCPInfosSpecialMarks(bulkUpdateInfos); + rc = BRMWrapper::getInstance()->setExtentsMaxMin(bulkUpdateInfos); } - - rc = writeColumnRecords(txnid, cscColTypeList, colExtentsStruct, colValueList, ridLists, tableOid, true, &pointersToInfos); - - if (rc == NO_ERROR) - { - bulkUpdateInfos.clear(); - for (auto cpInfoP : pointersToInfos) - { - if (cpInfoP) - { - cpInfoP->fCPInfo.seqNum ++; - bulkUpdateInfos.push_back(*cpInfoP); - } - } - if (!bulkUpdateInfos.empty()) - { - setInvalidCPInfosSpecialMarks(bulkUpdateInfos); - rc = BRMWrapper::getInstance()->setExtentsMaxMin(bulkUpdateInfos); - } - } - m_opType = NOOP; - return rc; + } + m_opType = NOOP; + return rc; } -int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - vector& colStructList, - ColValueList& colValueList, - const RIDList& ridLists, const int32_t tableOid, bool versioning, std::vector* cpInfos) +int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList, + vector& colStructList, ColValueList& colValueList, + const RIDList& ridLists, const int32_t tableOid, bool versioning, + std::vector* cpInfos) { - bool bExcp; - int rc = 0; - void* valArray = NULL; - void* oldValArray = NULL; - Column curCol; - ColStruct curColStruct; - CalpontSystemCatalog::ColType curColType; - ColTupleList curTupleList; - ColStructList::size_type totalColumn; - ColStructList::size_type i; - ColTupleList::size_type totalRow; - setTransId(txnid); - totalColumn = colStructList.size(); - totalRow = ridLists.size(); + bool bExcp; + int rc = 0; + void* valArray = NULL; + void* oldValArray = NULL; + Column curCol; + ColStruct curColStruct; + CalpontSystemCatalog::ColType curColType; + ColTupleList curTupleList; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + ColTupleList::size_type totalRow; + setTransId(txnid); + totalColumn = colStructList.size(); + totalRow = ridLists.size(); - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - for (i = 0; i < totalColumn; i++) + for (i = 0; i < totalColumn; i++) + { + ExtCPInfo* cpInfo = NULL; + if (cpInfos) { - ExtCPInfo* cpInfo = NULL; - if (cpInfos) - { - cpInfo = (*cpInfos)[i]; - } - valArray = NULL; - oldValArray = NULL; - - curColStruct = colStructList[i]; - curColType = cscColTypeList[i]; - curTupleList = colValueList[i]; - ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; + cpInfo = (*cpInfos)[i]; + } + valArray = NULL; + oldValArray = NULL; - Convertor::convertColType(&curColStruct); + curColStruct = colStructList[i]; + curColType = cscColTypeList[i]; + curTupleList = colValueList[i]; + ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; - // set params - colOp->initColumn(curCol); + Convertor::convertColType(&curColStruct); - colOp->setColParam(curCol, 0, curColStruct.colWidth, - curColStruct.colDataType, curColStruct.colType, curColStruct.dataOid, - curColStruct.fCompressionType, - curColStruct.fColDbRoot, curColStruct.fColPartition, curColStruct.fColSegment); - colOp->findTypeHandler(curColStruct.colWidth, - curColStruct.colDataType); - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(curColStruct.dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); + // set params + colOp->initColumn(curCol); - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == curColStruct.fColDbRoot) && (it->partNum == curColStruct.fColPartition) && (it->segNum == curColStruct.fColSegment)) - break; + colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, + curColStruct.dataOid, curColStruct.fCompressionType, curColStruct.fColDbRoot, + curColStruct.fColPartition, curColStruct.fColSegment); + colOp->findTypeHandler(curColStruct.colWidth, curColStruct.colDataType); + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(curColStruct.dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); - it++; - } + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == curColStruct.fColDbRoot) && (it->partNum == curColStruct.fColPartition) && + (it->segNum == curColStruct.fColSegment)) + break; - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = curColStruct.fColDbRoot; - aExt.partNum = curColStruct.fColPartition; - aExt.segNum = curColStruct.fColSegment; - aExt.compType = curColStruct.fCompressionType; - aExt.isDict = false; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - string segFile; - rc = colOp->openColumnFile(curCol, segFile, true); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - vector rangeList; - - if (versioning) - { - rc = processVersionBuffers(curCol.dataFile.pFile, txnid, curColStruct, - curColStruct.colWidth, totalRow, ridLists, rangeList); - } - - if (rc != NO_ERROR) - { - if (curColStruct.fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - - allocateValArray(valArray, totalRow, curColStruct.colType, curColStruct.colWidth); - - if (m_opType != INSERT && cpInfo) - { - allocateValArray(oldValArray, totalRow, curColStruct.colType, curColStruct.colWidth); - } - - // convert values to valArray - bExcp = false; - - try - { - convertValArray(totalRow, cscColTypeList[i], curColStruct.colType, curTupleList, valArray); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRowsValues(curCol, totalRow, ridLists, valArray, oldValArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - colOp->clearColumn(curCol); - - updateMaxMinRange(totalRow, totalRow, cscColTypeList[i], curColStruct.colType, valArray, oldValArray, cpInfo, false); - - if (curColStruct.fCompressionType == 0) - { - std::vector files; - BRM::FileInfo aFile; - aFile.partitionNum = curColStruct.fColPartition; - aFile.dbRoot = curColStruct.fColDbRoot;; - aFile.segmentNum = curColStruct.fColSegment; - aFile.compType = curColStruct.fCompressionType; - files.push_back(aFile); - - if (idbdatafile::IDBPolicy::useHdfs()) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - if (oldValArray != NULL) - { - free(oldValArray); - } - - // check error - if (rc != NO_ERROR) - break; + it++; } - return rc; + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = curColStruct.fColDbRoot; + aExt.partNum = curColStruct.fColPartition; + aExt.segNum = curColStruct.fColSegment; + aExt.compType = curColStruct.fCompressionType; + aExt.isDict = false; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + string segFile; + rc = colOp->openColumnFile(curCol, segFile, true); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + vector rangeList; + + if (versioning) + { + rc = processVersionBuffers(curCol.dataFile.pFile, txnid, curColStruct, curColStruct.colWidth, totalRow, + ridLists, rangeList); + } + + if (rc != NO_ERROR) + { + if (curColStruct.fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + + allocateValArray(valArray, totalRow, curColStruct.colType, curColStruct.colWidth); + + if (m_opType != INSERT && cpInfo) + { + allocateValArray(oldValArray, totalRow, curColStruct.colType, curColStruct.colWidth); + } + + // convert values to valArray + bExcp = false; + + try + { + convertValArray(totalRow, cscColTypeList[i], curColStruct.colType, curTupleList, valArray); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRowsValues(curCol, totalRow, ridLists, valArray, oldValArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + colOp->clearColumn(curCol); + + updateMaxMinRange(totalRow, totalRow, cscColTypeList[i], curColStruct.colType, valArray, oldValArray, + cpInfo, false); + + if (curColStruct.fCompressionType == 0) + { + std::vector files; + BRM::FileInfo aFile; + aFile.partitionNum = curColStruct.fColPartition; + aFile.dbRoot = curColStruct.fColDbRoot; + ; + aFile.segmentNum = curColStruct.fColSegment; + aFile.compType = curColStruct.fCompressionType; + files.push_back(aFile); + + if (idbdatafile::IDBPolicy::useHdfs()) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + if (valArray != NULL) + free(valArray); + + if (oldValArray != NULL) + { + free(oldValArray); + } + + // check error + if (rc != NO_ERROR) + break; + } + + return rc; } /*@brief writeColumnRec - Write values to a column -*/ + */ /*********************************************************** * DESCRIPTION: * Write values to a column @@ -4965,1103 +4855,1076 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, * NO_ERROR if success * others if something wrong in inserting the value ***********************************************************/ -int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - const ColStructList& colStructList, - ColValueList& colValueList, - RID* rowIdArray, - const ColStructList& newColStructList, - ColValueList& newColValueList, - const int32_t tableOid, - bool useTmpSuffix, - bool versioning, - ColSplitMaxMinInfoList* maxMins) +int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& cscColTypeList, + const ColStructList& colStructList, ColValueList& colValueList, + RID* rowIdArray, const ColStructList& newColStructList, + ColValueList& newColValueList, const int32_t tableOid, + bool useTmpSuffix, bool versioning, ColSplitMaxMinInfoList* maxMins) { - bool bExcp; - int rc = 0; - void* valArray; - void* oldValArray; - string segFile; - Column curCol; - ColTupleList oldTupleList; - ColStructList::size_type totalColumn; - ColStructList::size_type i; - ColTupleList::size_type totalRow1, totalRow2; + bool bExcp; + int rc = 0; + void* valArray; + void* oldValArray; + string segFile; + Column curCol; + ColTupleList oldTupleList; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + ColTupleList::size_type totalRow1, totalRow2; - setTransId(txnid); + setTransId(txnid); - totalColumn = colStructList.size(); + totalColumn = colStructList.size(); #ifdef PROFILE - StopWatch timer; + StopWatch timer; #endif - totalRow1 = colValueList[0].size(); - totalRow2 = 0; + totalRow1 = colValueList[0].size(); + totalRow2 = 0; - if (newColValueList.size() > 0) - totalRow2 = newColValueList[0].size(); + if (newColValueList.size() > 0) + totalRow2 = newColValueList[0].size(); - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - for (i = 0; i < totalColumn; i++) + for (i = 0; i < totalColumn; i++) + { + if (totalRow2 > 0) { - if (totalRow2 > 0) - { - RID* secondPart = rowIdArray + totalRow1; + RID* secondPart = rowIdArray + totalRow1; - //@Bug 2205 Check if all rows go to the new extent - if (totalRow1 > 0) - { - //Write the first batch - valArray = NULL; - oldValArray = NULL; - RID* firstPart = rowIdArray; - ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - // need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, 0, colStructList[i].colWidth, - colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, - colStructList[i].fCompressionType, colStructList[i].fColDbRoot, - colStructList[i].fColPartition, colStructList[i].fColSegment); - colOp->findTypeHandler(colStructList[i].colWidth, - colStructList[i].colDataType); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow1, firstPart, rangeList); - - if (rc != NO_ERROR) - { - if (colStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - // WIP We can allocate based on column size and not colType - // have to init the size here - allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); - - ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType(colStructList[i], - maxMins ? - ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); - - if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. - { - allocateValArray(oldValArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); - } - else - { - oldValArray = NULL; - } - - // convert values to valArray - // WIP Is m_opType ever set to DELETE? - if (m_opType != DELETE) - { - bExcp = false; - - try - { - // WIP We convert values twice!? - // dmlcommandproc converts strings to boost::any and this converts - // into actual type value masked by *void - // It is not clear why we need to convert to boost::any b/c we can convert from the original string here - convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray, oldValArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, oldValArray, true); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - - colOp->clearColumn(curCol); - - updateMaxMinRange(totalRow1, totalRow1, cscColTypeList[i], colStructList[i].colType, valArray, oldValArray, cpInfo, rowIdArray[0] == 0 && m_opType == INSERT); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - if (oldValArray != NULL) - free(oldValArray); - - // check error - if (rc != NO_ERROR) - break; - } - - //Process the second batch - valArray = NULL; - oldValArray = NULL; - - ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, newColStructList[i].colWidth, - newColStructList[i].colDataType, newColStructList[i].colType, newColStructList[i].dataOid, - newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, - newColStructList[i].fColPartition, newColStructList[i].fColSegment); - colOp->findTypeHandler(newColStructList[i].colWidth, - newColStructList[i].colDataType); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == newColStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = newColStructList[i].fColDbRoot; - aExt.partNum = newColStructList[i].fColPartition; - aExt.segNum = newColStructList[i].fColSegment; - aExt.compType = newColStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); - } - - // Pass "false" for hdfs tmp file flag. Since we only allow 1 - // extent per segment file (with HDFS), we can assume a second - // extent is going to a new file (and won't need tmp file). - rc = colOp->openColumnFile(curCol, segFile, false, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], - newColStructList[i].colWidth, totalRow2, secondPart, rangeList); - - if (rc != NO_ERROR) - { - if (newColStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType(newColStructList[i], - maxMins ? - ((*maxMins)[i]).fSplitMaxMinInfoPtrs[1] : NULL); - allocateValArray(valArray, totalRow2, newColStructList[i].colType, newColStructList[i].colWidth); - - if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. - { - allocateValArray(oldValArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); - } - else - { - oldValArray = NULL; - } - - // convert values to valArray - if (m_opType != DELETE) - { - bExcp = false; - - try - { - convertValArray(totalRow2, cscColTypeList[i], newColStructList[i].colType, newColValueList[i], valArray); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray, oldValArray); // XXX: here we use secondPart array and just below we use rowIdArray. WHY??? -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow2, rowIdArray, valArray, oldValArray, true); // XXX: BUG: here we use rowIdArray and just above we use secondPart array. WHY??? -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - - - colOp->clearColumn(curCol); - - updateMaxMinRange(totalRow2, totalRow2, cscColTypeList[i], newColStructList[i].colType, valArray, oldValArray, cpInfo, secondPart[0] == 0 && m_opType == INSERT); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - // check error - if (rc != NO_ERROR) - break; - } - else - { - valArray = NULL; - oldValArray = NULL; - - ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; - - ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType(colStructList[i], - maxMins ? - ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); - - // set params - colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, colStructList[i].colWidth, - colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, - colStructList[i].fCompressionType, colStructList[i].fColDbRoot, - colStructList[i].fColPartition, colStructList[i].fColSegment); - colOp->findTypeHandler(colStructList[i].colWidth, - colStructList[i].colDataType); - - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - //cout << " Opened file oid " << curCol.dataFile.pFile << endl; - if (rc != NO_ERROR) - break; - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow1, rowIdArray, rangeList); - - if (rc != NO_ERROR) - { - if (colStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); - - if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. - { - allocateValArray(oldValArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); - } - else - { - oldValArray = NULL; - } - - // convert values to valArray - if (m_opType != DELETE) - { - bExcp = false; - - try - { - convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray, true); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, oldValArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, oldValArray, true); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - - colOp->clearColumn(curCol); - - updateMaxMinRange(totalRow1, totalRow1, cscColTypeList[i], colStructList[i].colType, valArray, oldValArray, cpInfo, rowIdArray[0] == 0 && m_opType == INSERT); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - if (oldValArray != NULL) - free(oldValArray); - - // check error - if (rc != NO_ERROR) - break; - } - } // end of for (i = 0 - -#ifdef PROFILE - timer.finish(); -#endif - return rc; -} - -int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, - const ColStructList& colStructList, - std::vector& colValueList, - RID* rowIdArray, - const ColStructList& newColStructList, - std::vector& newColValueList, - const int32_t tableOid, - bool useTmpSuffix, - bool versioning) -{ - int rc = 0; - void* valArray = NULL; - string segFile; - Column curCol; - ColStructList::size_type totalColumn; - ColStructList::size_type i; - size_t totalRow1, totalRow2; - - setTransId(txnid); - - totalColumn = colStructList.size(); -#ifdef PROFILE - StopWatch timer; -#endif - - totalRow1 = colValueList.size() / totalColumn; - - if (newColValueList.size() > 0) - { - totalRow2 = newColValueList.size() / newColStructList.size(); - totalRow1 -= totalRow2; - } - else - { - totalRow2 = 0; - } - - // It is possible totalRow1 is zero but totalRow2 has values - if ((totalRow1 == 0) && (totalRow2 == 0)) - return rc; - - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - - if (totalRow1) - { - valArray = malloc(sizeof(uint64_t) * totalRow1); - - for (i = 0; i < totalColumn; i++) - { - //@Bug 2205 Check if all rows go to the new extent - //Write the first batch - RID* firstPart = rowIdArray; - ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - // need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, 0, colStructList[i].colWidth, - colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, - colStructList[i].fCompressionType, colStructList[i].fColDbRoot, - colStructList[i].fColPartition, colStructList[i].fColSegment); - colOp->findTypeHandler(colStructList[i].colWidth, - colStructList[i].colDataType); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow1, firstPart, rangeList); - - if (rc != NO_ERROR) - { - if (colStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - //totalRow1 -= totalRow2; - // have to init the size here - // nullArray = (bool*) malloc(sizeof(bool) * totalRow); - uint8_t tmp8; - uint16_t tmp16; - uint32_t tmp32; - - for (size_t j = 0; j < totalRow1; j++) - { - uint64_t curValue = colValueList[((totalRow1 + totalRow2) * i) + j]; - - switch (colStructList[i].colType) - { - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - ((uint64_t*)valArray)[j] = curValue; - break; - - case WriteEngine::WR_INT: - case WriteEngine::WR_UINT: - case WriteEngine::WR_MEDINT: - case WriteEngine::WR_UMEDINT: - case WriteEngine::WR_FLOAT: - tmp32 = curValue; - ((uint32_t*)valArray)[j] = tmp32; - break; - - case WriteEngine::WR_ULONGLONG: - case WriteEngine::WR_LONGLONG: - case WriteEngine::WR_DOUBLE: - case WriteEngine::WR_TOKEN: - ((uint64_t*)valArray)[j] = curValue; - break; - - case WriteEngine::WR_BYTE: - case WriteEngine::WR_UBYTE: - tmp8 = curValue; - ((uint8_t*)valArray)[j] = tmp8; - break; - - case WriteEngine::WR_SHORT: - case WriteEngine::WR_USHORT: - tmp16 = curValue; - ((uint16_t*)valArray)[j] = tmp16; - break; - - case WriteEngine::WR_BINARY: - ((uint64_t*)valArray)[j] = curValue; - break; - - } - } - - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - colOp->closeColumnFile(curCol); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - // check error - if (rc != NO_ERROR) - break; - - } // end of for (i = 0 - - if (valArray != NULL) - { - free(valArray); - valArray = NULL; - } - } - - // MCOL-1176 - Write second extent - if (totalRow2) - { - valArray = malloc(sizeof(uint64_t) * totalRow2); - - for (i = 0; i < newColStructList.size(); i++) - { - //@Bug 2205 Check if all rows go to the new extent - //Write the first batch - RID* secondPart = rowIdArray + totalRow1; - ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - // need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, 0, newColStructList[i].colWidth, - newColStructList[i].colDataType, newColStructList[i].colType, newColStructList[i].dataOid, - newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, - newColStructList[i].fColPartition, newColStructList[i].fColSegment); - colOp->findTypeHandler(newColStructList[i].colWidth, - newColStructList[i].colDataType); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = newColStructList[i].fColDbRoot; - aExt.partNum = newColStructList[i].fColPartition; - aExt.segNum = newColStructList[i].fColSegment; - aExt.compType = newColStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); - } - - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], - newColStructList[i].colWidth, totalRow2, secondPart, rangeList); - - if (rc != NO_ERROR) - { - if (newColStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - //totalRow1 -= totalRow2; - // have to init the size here - // nullArray = (bool*) malloc(sizeof(bool) * totalRow); - uint8_t tmp8; - uint16_t tmp16; - uint32_t tmp32; - - for (size_t j = 0; j < totalRow2; j++) - { - uint64_t curValue = newColValueList[(totalRow2 * i) + j]; - - switch (newColStructList[i].colType) - { - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - ((uint64_t*)valArray)[j] = curValue; - break; - - case WriteEngine::WR_INT: - case WriteEngine::WR_UINT: - case WriteEngine::WR_MEDINT: - case WriteEngine::WR_UMEDINT: - case WriteEngine::WR_FLOAT: - tmp32 = curValue; - ((uint32_t*)valArray)[j] = tmp32; - break; - - case WriteEngine::WR_ULONGLONG: - case WriteEngine::WR_LONGLONG: - case WriteEngine::WR_DOUBLE: - case WriteEngine::WR_TOKEN: - ((uint64_t*)valArray)[j] = curValue; - break; - - case WriteEngine::WR_BYTE: - case WriteEngine::WR_UBYTE: - tmp8 = curValue; - ((uint8_t*)valArray)[j] = tmp8; - break; - - case WriteEngine::WR_SHORT: - case WriteEngine::WR_USHORT: - tmp16 = curValue; - ((uint16_t*)valArray)[j] = tmp16; - break; - - case WriteEngine::WR_BINARY: - ((uint64_t*)valArray)[j] = curValue; - break; - } - } - - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - colOp->closeColumnFile(curCol); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - // check error - if (rc != NO_ERROR) - break; - - } // end of for (i = 0 - } - - if (valArray != NULL) - free(valArray); - - -#ifdef PROFILE - timer.finish(); -#endif - return rc; -} - -int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - const ColStructList& colStructList, - const ColValueList& colValueList, - vector& colOldValueList, - const RIDList& ridList, - const int32_t tableOid, - bool convertStructFlag, - ColTupleList::size_type nRows, - std::vector* cpInfos) -{ - bool bExcp; - int rc = 0; - void* valArray = NULL; - void* oldValArray = NULL; - Column curCol; - ColStruct curColStruct; - ColTupleList curTupleList, oldTupleList; - ColStructList::size_type totalColumn; - ColStructList::size_type i; - ColTupleList::size_type totalRow; - - setTransId(txnid); - colOldValueList.clear(); - totalColumn = colStructList.size(); - totalRow = nRows; - -#ifdef PROFILE - StopWatch timer; -#endif - - vector rangeListTot; - std::vector freeList; - vector > fboLists; - vector > rangeLists; - rc = processBeginVBCopy(txnid, colStructList, ridList, freeList, fboLists, rangeLists, rangeListTot); - - if (rc != NO_ERROR) - { - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); - - switch (rc) - { - case BRM::ERR_DEADLOCK: - return ERR_BRM_DEAD_LOCK; - - case BRM::ERR_VBBM_OVERFLOW: - return ERR_BRM_VB_OVERFLOW; - - case BRM::ERR_NETWORK: - return ERR_BRM_NETWORK; - - case BRM::ERR_READONLY: - return ERR_BRM_READONLY; - - default: - return ERR_BRM_BEGIN_COPY; - } - } - - VBRange aRange; - uint32_t blocksProcessedThisOid = 0; - uint32_t blocksProcessed = 0; - std::vector files; - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - - for (i = 0; i < totalColumn; i++) - { + //@Bug 2205 Check if all rows go to the new extent + if (totalRow1 > 0) + { + // Write the first batch valArray = NULL; oldValArray = NULL; - curColStruct = colStructList[i]; - curTupleList = colValueList[i]; //same value for all rows - ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; - - // convert column data type - if (convertStructFlag) - Convertor::convertColType(&curColStruct); + RID* firstPart = rowIdArray; + ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; // set params colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, curColStruct.colWidth, - curColStruct.colDataType, curColStruct.colType, curColStruct.dataOid, - curColStruct.fCompressionType, curColStruct.fColDbRoot, - curColStruct.fColPartition, curColStruct.fColSegment); - colOp->findTypeHandler(curColStruct.colWidth, - curColStruct.colDataType); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType, + colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); + colOp->findTypeHandler(colStructList[i].colWidth, colStructList[i].colDataType); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(curColStruct.dataOid); + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if ((it->dbRoot == curColStruct.fColDbRoot) && (it->partNum == curColStruct.fColPartition) && (it->segNum == curColStruct.fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = curColStruct.fColDbRoot; - aExt.partNum = curColStruct.fColPartition; - aExt.segNum = curColStruct.fColSegment; - aExt.compType = curColStruct.fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - string segFile; - rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) + if ((it->dbRoot == colStructList[i].fColDbRoot) && + (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) break; - if (curColStruct.fCompressionType == 0) - { - BRM::FileInfo aFile; - aFile.oid = curColStruct.dataOid; - aFile.partitionNum = curColStruct.fColPartition; - aFile.dbRoot = curColStruct.fColDbRoot;; - aFile.segmentNum = curColStruct.fColSegment; - aFile.compType = curColStruct.fCompressionType; - files.push_back(aFile); + it++; } + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + // handling versioning - std::vector curFreeList; - uint32_t blockUsed = 0; + vector rangeList; - if (!idbdatafile::IDBPolicy::useHdfs()) + if (versioning) { - if (rangeListTot.size() > 0) + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], colStructList[i].colWidth, + totalRow1, firstPart, rangeList); + + if (rc != NO_ERROR) + { + if (colStructList[i].fCompressionType == 0) { - if (freeList[0].size >= (blocksProcessed + rangeLists[i].size())) - { - aRange.vbOID = freeList[0].vbOID; - aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; - aRange.size = rangeLists[i].size(); - curFreeList.push_back(aRange); - } - else - { - aRange.vbOID = freeList[0].vbOID; - aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; - aRange.size = freeList[0].size - blocksProcessed; - blockUsed = aRange.size; - curFreeList.push_back(aRange); - - if (freeList.size() > 1) - { - aRange.vbOID = freeList[1].vbOID; - aRange.vbFBO = freeList[1].vbFBO + blocksProcessedThisOid; - aRange.size = rangeLists[i].size() - blockUsed; - curFreeList.push_back(aRange); - blocksProcessedThisOid += aRange.size; - } - else - { - rc = 1; - break; - } - } - - blocksProcessed += rangeLists[i].size(); - - rc = BRMWrapper::getInstance()-> - writeVB(curCol.dataFile.pFile, (BRM::VER_t)txnid, - curColStruct.dataOid, fboLists[i], rangeLists[i], - colOp, curFreeList, curColStruct.fColDbRoot, true); - } - } - - if (rc != NO_ERROR) - { - if (curColStruct.fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); + curCol.dataFile.pFile->flush(); } - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); - + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); break; + } } - allocateValArray(valArray, 1, curColStruct.colType, curColStruct.colWidth); + // WIP We can allocate based on column size and not colType + // have to init the size here + allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); - ExtCPInfo* cpInfo; - cpInfo = cpInfos ? ((*cpInfos)[i]) : NULL; + ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( + colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); - if (cpInfo) + if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. { - allocateValArray(oldValArray, totalRow, curColStruct.colType, curColStruct.colWidth); + allocateValArray(oldValArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); + } + else + { + oldValArray = NULL; } // convert values to valArray + // WIP Is m_opType ever set to DELETE? if (m_opType != DELETE) { - bExcp = false; - ColTuple curTuple; - curTuple = curTupleList[0]; + bExcp = false; - try - { - convertValue(cscColTypeList[i], curColStruct.colType, valArray, curTuple.data); - } - catch (...) - { - bExcp = true; - } + try + { + // WIP We convert values twice!? + // dmlcommandproc converts strings to boost::any and this converts + // into actual type value masked by *void + // It is not clear why we need to convert to boost::any b/c we can convert from the original + // string here + convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], + valArray); + } + catch (...) + { + bExcp = true; + } - if (bExcp) - { - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + if (bExcp) + { + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - return ERR_PARSING; - } + return ERR_PARSING; + } #ifdef PROFILE - timer.start("writeRow "); + timer.start("writeRow "); #endif - rc = colOp->writeRows(curCol, totalRow, ridList, valArray, oldValArray); + rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray, oldValArray); #ifdef PROFILE - timer.stop("writeRow "); + timer.stop("writeRow "); #endif } else { #ifdef PROFILE - timer.start("writeRows "); + timer.start("writeRow "); #endif - rc = colOp->writeRows(curCol, totalRow, ridList, valArray, oldValArray, true); + rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, oldValArray, true); #ifdef PROFILE - timer.stop("writeRows "); + timer.stop("writeRow "); #endif } - updateMaxMinRange(1, totalRow, cscColTypeList[i], curColStruct.colType, m_opType == DELETE ? NULL : valArray, oldValArray, cpInfo, false); -//timer.start("Delete:closefile"); colOp->clearColumn(curCol); -//timer.stop("Delete:closefile"); + updateMaxMinRange(totalRow1, totalRow1, cscColTypeList[i], colStructList[i].colType, valArray, + oldValArray, cpInfo, rowIdArray[0] == 0 && m_opType == INSERT); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + if (valArray != NULL) - { - free(valArray); - valArray = NULL; - } + free(valArray); if (oldValArray != NULL) - { - free(oldValArray); - oldValArray = NULL; - } + free(oldValArray); // check error if (rc != NO_ERROR) + break; + } + + // Process the second batch + valArray = NULL; + oldValArray = NULL; + + ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, newColStructList[i].colWidth, newColStructList[i].colDataType, + newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, + newColStructList[i].fColPartition, newColStructList[i].fColSegment); + colOp->findTypeHandler(newColStructList[i].colWidth, newColStructList[i].colDataType); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && + (it->partNum == newColStructList[i].fColPartition) && + (it->segNum == newColStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = newColStructList[i].fColDbRoot; + aExt.partNum = newColStructList[i].fColPartition; + aExt.segNum = newColStructList[i].fColSegment; + aExt.compType = newColStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + // Pass "false" for hdfs tmp file flag. Since we only allow 1 + // extent per segment file (with HDFS), we can assume a second + // extent is going to a new file (and won't need tmp file). + rc = colOp->openColumnFile(curCol, segFile, false, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], + newColStructList[i].colWidth, totalRow2, secondPart, rangeList); + + if (rc != NO_ERROR) + { + if (newColStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( + newColStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[1] : NULL); + allocateValArray(valArray, totalRow2, newColStructList[i].colType, newColStructList[i].colWidth); + + if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. + { + allocateValArray(oldValArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); + } + else + { + oldValArray = NULL; + } + + // convert values to valArray + if (m_opType != DELETE) + { + bExcp = false; + + try + { + convertValArray(totalRow2, cscColTypeList[i], newColStructList[i].colType, newColValueList[i], + valArray); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow( + curCol, totalRow2, secondPart, valArray, + oldValArray); // XXX: here we use secondPart array and just below we use rowIdArray. WHY??? +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow( + curCol, totalRow2, rowIdArray, valArray, oldValArray, + true); // XXX: BUG: here we use rowIdArray and just above we use secondPart array. WHY??? +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + + colOp->clearColumn(curCol); + + updateMaxMinRange(totalRow2, totalRow2, cscColTypeList[i], newColStructList[i].colType, valArray, + oldValArray, cpInfo, secondPart[0] == 0 && m_opType == INSERT); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + if (valArray != NULL) + free(valArray); + + // check error + if (rc != NO_ERROR) + break; + } + else + { + valArray = NULL; + oldValArray = NULL; + + ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; + + ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( + colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); + + // set params + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType, + colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); + colOp->findTypeHandler(colStructList[i].colWidth, colStructList[i].colDataType); + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + // cout << " Opened file oid " << curCol.dataFile.pFile << endl; + if (rc != NO_ERROR) + break; + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], colStructList[i].colWidth, + totalRow1, rowIdArray, rangeList); + + if (rc != NO_ERROR) + { + if (colStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); + + if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. + { + allocateValArray(oldValArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); + } + else + { + oldValArray = NULL; + } + + // convert values to valArray + if (m_opType != DELETE) + { + bExcp = false; + + try + { + convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray, + true); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, oldValArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, oldValArray, true); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + + colOp->clearColumn(curCol); + + updateMaxMinRange(totalRow1, totalRow1, cscColTypeList[i], colStructList[i].colType, valArray, + oldValArray, cpInfo, rowIdArray[0] == 0 && m_opType == INSERT); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + if (valArray != NULL) + free(valArray); + + if (oldValArray != NULL) + free(oldValArray); + + // check error + if (rc != NO_ERROR) + break; + } + } // end of for (i = 0 + +#ifdef PROFILE + timer.finish(); +#endif + return rc; +} + +int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, + std::vector& colValueList, RID* rowIdArray, + const ColStructList& newColStructList, + std::vector& newColValueList, const int32_t tableOid, + bool useTmpSuffix, bool versioning) +{ + int rc = 0; + void* valArray = NULL; + string segFile; + Column curCol; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + size_t totalRow1, totalRow2; + + setTransId(txnid); + + totalColumn = colStructList.size(); +#ifdef PROFILE + StopWatch timer; +#endif + + totalRow1 = colValueList.size() / totalColumn; + + if (newColValueList.size() > 0) + { + totalRow2 = newColValueList.size() / newColStructList.size(); + totalRow1 -= totalRow2; + } + else + { + totalRow2 = 0; + } + + // It is possible totalRow1 is zero but totalRow2 has values + if ((totalRow1 == 0) && (totalRow2 == 0)) + return rc; + + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + + if (totalRow1) + { + valArray = malloc(sizeof(uint64_t) * totalRow1); + + for (i = 0; i < totalColumn; i++) + { + //@Bug 2205 Check if all rows go to the new extent + // Write the first batch + RID* firstPart = rowIdArray; + ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType, + colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); + colOp->findTypeHandler(colStructList[i].colWidth, colStructList[i].colDataType); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], colStructList[i].colWidth, + totalRow1, firstPart, rangeList); + + if (rc != NO_ERROR) + { + if (colStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + // totalRow1 -= totalRow2; + // have to init the size here + // nullArray = (bool*) malloc(sizeof(bool) * totalRow); + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + + for (size_t j = 0; j < totalRow1; j++) + { + uint64_t curValue = colValueList[((totalRow1 + totalRow2) * i) + j]; + + switch (colStructList[i].colType) + { + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: ((uint64_t*)valArray)[j] = curValue; break; + + case WriteEngine::WR_INT: + case WriteEngine::WR_UINT: + case WriteEngine::WR_MEDINT: + case WriteEngine::WR_UMEDINT: + case WriteEngine::WR_FLOAT: + tmp32 = curValue; + ((uint32_t*)valArray)[j] = tmp32; break; - } // end of for (i = 0) + case WriteEngine::WR_ULONGLONG: + case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_DOUBLE: + case WriteEngine::WR_TOKEN: ((uint64_t*)valArray)[j] = curValue; break; -// timer.start("Delete:purgePrimProcFdCache"); - if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + case WriteEngine::WR_BYTE: + case WriteEngine::WR_UBYTE: + tmp8 = curValue; + ((uint8_t*)valArray)[j] = tmp8; + break; -//timer.stop("Delete:purgePrimProcFdCache"); + case WriteEngine::WR_SHORT: + case WriteEngine::WR_USHORT: + tmp16 = curValue; + ((uint16_t*)valArray)[j] = tmp16; + break; + + case WriteEngine::WR_BINARY: ((uint64_t*)valArray)[j] = curValue; break; + } + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + colOp->closeColumnFile(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0 + + if (valArray != NULL) + { + free(valArray); + valArray = NULL; + } + } + + // MCOL-1176 - Write second extent + if (totalRow2) + { + valArray = malloc(sizeof(uint64_t) * totalRow2); + + for (i = 0; i < newColStructList.size(); i++) + { + //@Bug 2205 Check if all rows go to the new extent + // Write the first batch + RID* secondPart = rowIdArray + totalRow1; + ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, newColStructList[i].colWidth, newColStructList[i].colDataType, + newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, + newColStructList[i].fColPartition, newColStructList[i].fColSegment); + colOp->findTypeHandler(newColStructList[i].colWidth, newColStructList[i].colDataType); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && + (it->partNum == newColStructList[i].fColPartition) && + (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = newColStructList[i].fColDbRoot; + aExt.partNum = newColStructList[i].fColPartition; + aExt.segNum = newColStructList[i].fColSegment; + aExt.compType = newColStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], + newColStructList[i].colWidth, totalRow2, secondPart, rangeList); + + if (rc != NO_ERROR) + { + if (newColStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + // totalRow1 -= totalRow2; + // have to init the size here + // nullArray = (bool*) malloc(sizeof(bool) * totalRow); + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + + for (size_t j = 0; j < totalRow2; j++) + { + uint64_t curValue = newColValueList[(totalRow2 * i) + j]; + + switch (newColStructList[i].colType) + { + case WriteEngine::WR_VARBINARY: // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: ((uint64_t*)valArray)[j] = curValue; break; + + case WriteEngine::WR_INT: + case WriteEngine::WR_UINT: + case WriteEngine::WR_MEDINT: + case WriteEngine::WR_UMEDINT: + case WriteEngine::WR_FLOAT: + tmp32 = curValue; + ((uint32_t*)valArray)[j] = tmp32; + break; + + case WriteEngine::WR_ULONGLONG: + case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_DOUBLE: + case WriteEngine::WR_TOKEN: ((uint64_t*)valArray)[j] = curValue; break; + + case WriteEngine::WR_BYTE: + case WriteEngine::WR_UBYTE: + tmp8 = curValue; + ((uint8_t*)valArray)[j] = tmp8; + break; + + case WriteEngine::WR_SHORT: + case WriteEngine::WR_USHORT: + tmp16 = curValue; + ((uint16_t*)valArray)[j] = tmp16; + break; + + case WriteEngine::WR_BINARY: ((uint64_t*)valArray)[j] = curValue; break; + } + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + colOp->closeColumnFile(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0 + } + + if (valArray != NULL) + free(valArray); + +#ifdef PROFILE + timer.finish(); +#endif + return rc; +} + +int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesList& cscColTypeList, + const ColStructList& colStructList, + const ColValueList& colValueList, vector& colOldValueList, + const RIDList& ridList, const int32_t tableOid, + bool convertStructFlag, ColTupleList::size_type nRows, + std::vector* cpInfos) +{ + bool bExcp; + int rc = 0; + void* valArray = NULL; + void* oldValArray = NULL; + Column curCol; + ColStruct curColStruct; + ColTupleList curTupleList, oldTupleList; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + ColTupleList::size_type totalRow; + + setTransId(txnid); + colOldValueList.clear(); + totalColumn = colStructList.size(); + totalRow = nRows; + +#ifdef PROFILE + StopWatch timer; +#endif + + vector rangeListTot; + std::vector freeList; + vector > fboLists; + vector > rangeLists; + rc = processBeginVBCopy(txnid, colStructList, ridList, freeList, fboLists, rangeLists, rangeListTot); + + if (rc != NO_ERROR) + { if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + + switch (rc) + { + case BRM::ERR_DEADLOCK: return ERR_BRM_DEAD_LOCK; + + case BRM::ERR_VBBM_OVERFLOW: return ERR_BRM_VB_OVERFLOW; + + case BRM::ERR_NETWORK: return ERR_BRM_NETWORK; + + case BRM::ERR_READONLY: return ERR_BRM_READONLY; + + default: return ERR_BRM_BEGIN_COPY; + } + } + + VBRange aRange; + uint32_t blocksProcessedThisOid = 0; + uint32_t blocksProcessed = 0; + std::vector files; + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + + for (i = 0; i < totalColumn; i++) + { + valArray = NULL; + oldValArray = NULL; + curColStruct = colStructList[i]; + curTupleList = colValueList[i]; // same value for all rows + ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; + + // convert column data type + if (convertStructFlag) + Convertor::convertColType(&curColStruct); + + // set params + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, + curColStruct.dataOid, curColStruct.fCompressionType, curColStruct.fColDbRoot, + curColStruct.fColPartition, curColStruct.fColSegment); + colOp->findTypeHandler(curColStruct.colWidth, curColStruct.colDataType); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(curColStruct.dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == curColStruct.fColDbRoot) && (it->partNum == curColStruct.fColPartition) && + (it->segNum == curColStruct.fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) // add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = curColStruct.fColDbRoot; + aExt.partNum = curColStruct.fColPartition; + aExt.segNum = curColStruct.fColSegment; + aExt.compType = curColStruct.fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + string segFile; + rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + if (curColStruct.fCompressionType == 0) + { + BRM::FileInfo aFile; + aFile.oid = curColStruct.dataOid; + aFile.partitionNum = curColStruct.fColPartition; + aFile.dbRoot = curColStruct.fColDbRoot; + ; + aFile.segmentNum = curColStruct.fColSegment; + aFile.compType = curColStruct.fCompressionType; + files.push_back(aFile); + } + + // handling versioning + std::vector curFreeList; + uint32_t blockUsed = 0; + + if (!idbdatafile::IDBPolicy::useHdfs()) + { + if (rangeListTot.size() > 0) + { + if (freeList[0].size >= (blocksProcessed + rangeLists[i].size())) + { + aRange.vbOID = freeList[0].vbOID; + aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; + aRange.size = rangeLists[i].size(); + curFreeList.push_back(aRange); + } + else + { + aRange.vbOID = freeList[0].vbOID; + aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; + aRange.size = freeList[0].size - blocksProcessed; + blockUsed = aRange.size; + curFreeList.push_back(aRange); + + if (freeList.size() > 1) + { + aRange.vbOID = freeList[1].vbOID; + aRange.vbFBO = freeList[1].vbFBO + blocksProcessedThisOid; + aRange.size = rangeLists[i].size() - blockUsed; + curFreeList.push_back(aRange); + blocksProcessedThisOid += aRange.size; + } + else + { + rc = 1; + break; + } + } + + blocksProcessed += rangeLists[i].size(); + + rc = BRMWrapper::getInstance()->writeVB(curCol.dataFile.pFile, (BRM::VER_t)txnid, + curColStruct.dataOid, fboLists[i], rangeLists[i], colOp, + curFreeList, curColStruct.fColDbRoot, true); + } + } + + if (rc != NO_ERROR) + { + if (curColStruct.fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + if (rangeListTot.size() > 0) BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); -//timer.stop("Delete:writecolrec"); -//#ifdef PROFILE -//timer.finish(); -//#endif - return rc; + break; + } + + allocateValArray(valArray, 1, curColStruct.colType, curColStruct.colWidth); + + ExtCPInfo* cpInfo; + cpInfo = cpInfos ? ((*cpInfos)[i]) : NULL; + + if (cpInfo) + { + allocateValArray(oldValArray, totalRow, curColStruct.colType, curColStruct.colWidth); + } + + // convert values to valArray + if (m_opType != DELETE) + { + bExcp = false; + ColTuple curTuple; + curTuple = curTupleList[0]; + + try + { + convertValue(cscColTypeList[i], curColStruct.colType, valArray, curTuple.data); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRows(curCol, totalRow, ridList, valArray, oldValArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRows "); +#endif + rc = colOp->writeRows(curCol, totalRow, ridList, valArray, oldValArray, true); +#ifdef PROFILE + timer.stop("writeRows "); +#endif + } + + updateMaxMinRange(1, totalRow, cscColTypeList[i], curColStruct.colType, + m_opType == DELETE ? NULL : valArray, oldValArray, cpInfo, false); + // timer.start("Delete:closefile"); + colOp->clearColumn(curCol); + + // timer.stop("Delete:closefile"); + if (valArray != NULL) + { + free(valArray); + valArray = NULL; + } + + if (oldValArray != NULL) + { + free(oldValArray); + oldValArray = NULL; + } + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0) + + // timer.start("Delete:purgePrimProcFdCache"); + if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + + // timer.stop("Delete:purgePrimProcFdCache"); + if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + + // timer.stop("Delete:writecolrec"); + //#ifdef PROFILE + // timer.finish(); + //#endif + return rc; } /*@brief tokenize - return a token for a given signature and size -*/ + */ /*********************************************************** * DESCRIPTION: * return a token for a given signature and size @@ -6079,14 +5942,14 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, ***********************************************************/ int WriteEngineWrapper::tokenize(const TxnID& txnid, DctnryTuple& dctnryTuple, int ct) { - int cop = op(ct); - m_dctnry[cop]->setTransId(txnid); - return m_dctnry[cop]->updateDctnry(dctnryTuple.sigValue, dctnryTuple.sigSize, dctnryTuple.token); + int cop = op(ct); + m_dctnry[cop]->setTransId(txnid); + return m_dctnry[cop]->updateDctnry(dctnryTuple.sigValue, dctnryTuple.sigSize, dctnryTuple.token); } /*@brief tokenize - return a token for a given signature and size * accept OIDs as input -*/ + */ /*********************************************************** * DESCRIPTION: * Token for a given signature and size @@ -6102,30 +5965,27 @@ int WriteEngineWrapper::tokenize(const TxnID& txnid, DctnryTuple& dctnryTuple, i * NO_ERROR if success * others if something wrong in inserting the value ***********************************************************/ -int WriteEngineWrapper::tokenize(const TxnID& txnid, - DctnryStruct& dctnryStruct, - DctnryTuple& dctnryTuple, - bool useTmpSuffix) // @bug 5572 HDFS tmp file +int WriteEngineWrapper::tokenize(const TxnID& txnid, DctnryStruct& dctnryStruct, DctnryTuple& dctnryTuple, + bool useTmpSuffix) // @bug 5572 HDFS tmp file { - //find the corresponding column segment file the token is going to be inserted. + // find the corresponding column segment file the token is going to be inserted. - Dctnry* dctnry = m_dctnry[op(dctnryStruct.fCompressionType)]; - int rc = dctnry->openDctnry(dctnryStruct.dctnryOid, - dctnryStruct.fColDbRoot, dctnryStruct.fColPartition, - dctnryStruct.fColSegment, - useTmpSuffix); // @bug 5572 TBD - - if (rc != NO_ERROR) - return rc; - - rc = tokenize(txnid, dctnryTuple, dctnryStruct.fCompressionType); - - int rc2 = dctnry->closeDctnry(true); // close file, even if tokenize() fails - - if ((rc == NO_ERROR) && (rc2 != NO_ERROR)) - rc = rc2; + Dctnry* dctnry = m_dctnry[op(dctnryStruct.fCompressionType)]; + int rc = dctnry->openDctnry(dctnryStruct.dctnryOid, dctnryStruct.fColDbRoot, dctnryStruct.fColPartition, + dctnryStruct.fColSegment, + useTmpSuffix); // @bug 5572 TBD + if (rc != NO_ERROR) return rc; + + rc = tokenize(txnid, dctnryTuple, dctnryStruct.fCompressionType); + + int rc2 = dctnry->closeDctnry(true); // close file, even if tokenize() fails + + if ((rc == NO_ERROR) && (rc2 != NO_ERROR)) + rc = rc2; + + return rc; } /*********************************************************** @@ -6144,39 +6004,35 @@ int WriteEngineWrapper::tokenize(const TxnID& txnid, * ERR_FILE_EXIST if file exists * ERR_FILE_CREATE if something wrong in creating the file ***********************************************************/ -int WriteEngineWrapper::createDctnry(const TxnID& txnid, - const OID& dctnryOid, - int colWidth, - uint16_t dbRoot, - uint32_t partiotion, - uint16_t segment, - int compressionType) +int WriteEngineWrapper::createDctnry(const TxnID& txnid, const OID& dctnryOid, int colWidth, uint16_t dbRoot, + uint32_t partiotion, uint16_t segment, int compressionType) { - BRM::LBID_t startLbid; - return m_dctnry[op(compressionType)]-> - createDctnry( dctnryOid, colWidth, dbRoot, partiotion, segment, startLbid); + BRM::LBID_t startLbid; + return m_dctnry[op(compressionType)]->createDctnry(dctnryOid, colWidth, dbRoot, partiotion, segment, + startLbid); } -int WriteEngineWrapper::convertRidToColumn (RID& rid, uint16_t& dbRoot, uint32_t& partition, - uint16_t& segment, RID filesPerColumnPartition, - RID extentsPerSegmentFile, RID extentRows, - uint16_t startDBRoot, unsigned dbrootCnt) +int WriteEngineWrapper::convertRidToColumn(RID& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, + RID filesPerColumnPartition, RID extentsPerSegmentFile, + RID extentRows, uint16_t startDBRoot, unsigned dbrootCnt) { - int rc = 0; - partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); + int rc = 0; + partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows); - segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % filesPerColumnPartition; + segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % + filesPerColumnPartition; - dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; + dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1; - //Calculate the relative rid for this segment file - RID relRidInPartition = rid - ((RID)partition * (RID)filesPerColumnPartition * (RID)extentsPerSegmentFile * (RID)extentRows); - assert (relRidInPartition <= (RID)filesPerColumnPartition * (RID)extentsPerSegmentFile * (RID)extentRows); - uint32_t numExtentsInThisPart = relRidInPartition / extentRows; - unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; - RID relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; - rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; - return rc; + // Calculate the relative rid for this segment file + RID relRidInPartition = + rid - ((RID)partition * (RID)filesPerColumnPartition * (RID)extentsPerSegmentFile * (RID)extentRows); + assert(relRidInPartition <= (RID)filesPerColumnPartition * (RID)extentsPerSegmentFile * (RID)extentRows); + uint32_t numExtentsInThisPart = relRidInPartition / extentRows; + unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition; + RID relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows; + rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows; + return rc; } /*********************************************************** @@ -6188,16 +6044,13 @@ int WriteEngineWrapper::convertRidToColumn (RID& rid, uint16_t& dbRoot, uint32_t * RETURN: * NO_ERROR if operation is successful ***********************************************************/ -int WriteEngineWrapper::clearTableLockOnly( - uint64_t lockID, - std::string& errMsg) +int WriteEngineWrapper::clearTableLockOnly(uint64_t lockID, std::string& errMsg) { - bool bReleased; + bool bReleased; - int rc = BRMWrapper::getInstance()->releaseTableLock( lockID, - bReleased, errMsg); + int rc = BRMWrapper::getInstance()->releaseTableLock(lockID, bReleased, errMsg); - return rc; + return rc; } /*********************************************************** @@ -6215,383 +6068,372 @@ int WriteEngineWrapper::clearTableLockOnly( * RETURN: * NO_ERROR if rollback completed succesfully ***********************************************************/ -int WriteEngineWrapper::bulkRollback(OID tableOid, - uint64_t lockID, - const std::string& tableName, - const std::string& applName, - bool debugConsole, string& errorMsg) +int WriteEngineWrapper::bulkRollback(OID tableOid, uint64_t lockID, const std::string& tableName, + const std::string& applName, bool debugConsole, string& errorMsg) { - errorMsg.clear(); + errorMsg.clear(); - BulkRollbackMgr rollbackMgr(tableOid, lockID, tableName, applName); + BulkRollbackMgr rollbackMgr(tableOid, lockID, tableName, applName); - if (debugConsole) - rollbackMgr.setDebugConsole(true); + if (debugConsole) + rollbackMgr.setDebugConsole(true); - // We used to pass "false" to not keep (delete) the metafiles at the end of - // the rollback. But after the transition to sharedNothing, we pass "true" - // to initially keep these files. The metafiles are deleted later, only - // after all the distributed bulk rollbacks are successfully completed. - int rc = rollbackMgr.rollback( true ); + // We used to pass "false" to not keep (delete) the metafiles at the end of + // the rollback. But after the transition to sharedNothing, we pass "true" + // to initially keep these files. The metafiles are deleted later, only + // after all the distributed bulk rollbacks are successfully completed. + int rc = rollbackMgr.rollback(true); - if (rc != NO_ERROR) - errorMsg = rollbackMgr.getErrorMsg(); + if (rc != NO_ERROR) + errorMsg = rollbackMgr.getErrorMsg(); - // Ignore the return code for now; more important to base rc on the - // success or failure of the previous work - BRMWrapper::getInstance()->takeSnapshot(); + // Ignore the return code for now; more important to base rc on the + // success or failure of the previous work + BRMWrapper::getInstance()->takeSnapshot(); - return rc; + return rc; } int WriteEngineWrapper::rollbackCommon(const TxnID& txnid, int sessionId) { - //Remove the unwanted tmp files and recover compressed chunks. - string prefix; + // Remove the unwanted tmp files and recover compressed chunks. + string prefix; - // BUG 4312 - RemoveTxnFromLBIDMap(txnid); - RemoveTxnFromDictMap(txnid); + // BUG 4312 + RemoveTxnFromLBIDMap(txnid); + RemoveTxnFromDictMap(txnid); - config::Config* config = config::Config::makeConfig(); - prefix = config->getConfig("SystemConfig", "DBRMRoot"); + config::Config* config = config::Config::makeConfig(); + prefix = config->getConfig("SystemConfig", "DBRMRoot"); - if (prefix.length() == 0) + if (prefix.length() == 0) + { + cerr << "Need a valid DBRMRoot entry in Calpont configuation file"; + return ERR_INVALID_PARAM; + } + + uint64_t pos = prefix.find_last_of("/"); + std::string aDMLLogFileName; + + if (pos != string::npos) + { + aDMLLogFileName = prefix.substr(0, pos + 1); // Get the file path + } + else + { + logging::Message::Args args; + args.add("RollbackTran cannot find the dbrm directory for the DML log file"); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_CRITICAL, logging::M0007); + return ERR_OPEN_DML_LOG; + } + + std::ostringstream oss; + oss << txnid << "_" << Config::getLocalModuleID(); + aDMLLogFileName += "DMLLog_" + oss.str(); + + if (IDBPolicy::exists(aDMLLogFileName.c_str())) + { + // TODO-for now the DML log file will always be in a local + // filesystem since IDBDataFile doesn't have any support for + // a cpp iostream interface. need to decide if this is ok. + boost::scoped_ptr aDMLLogFile(IDBDataFile::open( + IDBPolicy::getType(aDMLLogFileName.c_str(), IDBPolicy::WRITEENG), aDMLLogFileName.c_str(), "r", 0)); + + if (aDMLLogFile) // need recover { - cerr << "Need a valid DBRMRoot entry in Calpont configuation file"; - return ERR_INVALID_PARAM; - } + ssize_t fileSize = aDMLLogFile->size(); + boost::scoped_array buf(new char[fileSize]); - uint64_t pos = prefix.find_last_of ("/") ; - std::string aDMLLogFileName; + if (aDMLLogFile->read(buf.get(), fileSize) != fileSize) + return ERR_FILE_READ; - if (pos != string::npos) - { - aDMLLogFileName = prefix.substr(0, pos + 1); //Get the file path - } - else - { + std::istringstream strstream(string(buf.get(), fileSize)); + std::string backUpFileType; + std::string filename; + int64_t size; + int64_t offset; + + while (strstream >> backUpFileType >> filename >> size >> offset) + { + // cout << "Found: " << backUpFileType << " name " << filename << "size: " << size << " offset: " << + // offset << endl; + std::ostringstream oss; + oss << "RollbackTran found " << backUpFileType << " name " << filename << " size: " << size + << " offset: " << offset; logging::Message::Args args; - args.add("RollbackTran cannot find the dbrm directory for the DML log file"); - SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_CRITICAL, logging::M0007); - return ERR_OPEN_DML_LOG; + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, logging::M0007); - } - - std::ostringstream oss; - oss << txnid << "_" << Config::getLocalModuleID(); - aDMLLogFileName += "DMLLog_" + oss.str(); - - if (IDBPolicy::exists(aDMLLogFileName.c_str())) - { - // TODO-for now the DML log file will always be in a local - // filesystem since IDBDataFile doesn't have any support for - // a cpp iostream interface. need to decide if this is ok. - boost::scoped_ptr aDMLLogFile(IDBDataFile::open( - IDBPolicy::getType(aDMLLogFileName.c_str(), - IDBPolicy::WRITEENG), - aDMLLogFileName.c_str(), "r", 0)); - - if (aDMLLogFile) //need recover + if (backUpFileType.compare("rlc") == 0) { - ssize_t fileSize = aDMLLogFile->size(); - boost::scoped_array buf(new char[fileSize]); - - if (aDMLLogFile->read(buf.get(), fileSize) != fileSize) - return ERR_FILE_READ; - - std::istringstream strstream(string(buf.get(), fileSize)); - std::string backUpFileType; - std::string filename; - int64_t size; - int64_t offset; - - while (strstream >> backUpFileType >> filename >> size >> offset) - { - //cout << "Found: " << backUpFileType << " name " << filename << "size: " << size << " offset: " << offset << endl; - std::ostringstream oss; - oss << "RollbackTran found " << backUpFileType << " name " << filename << " size: " << size << " offset: " << offset; - logging::Message::Args args; - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, logging::M0007); - - if (backUpFileType.compare("rlc") == 0) - { - //remove the rlc file - filename += ".rlc"; - //cout << " File removed: " << filename << endl; - IDBPolicy::remove(filename.c_str()); - logging::Message::Args args1; - args1.add(filename); - args1.add(" is removed."); - SimpleSysLog::instance()->logMsg(args1, logging::LOG_TYPE_INFO, logging::M0007); - } - else if (backUpFileType.compare("tmp") == 0) - { - int rc = NO_ERROR; - string orig(filename + ".orig"); - - // restore the orig file - if (IDBPolicy::exists(orig.c_str())) - { - // not likely both cdf and tmp exist - if (IDBPolicy::exists(filename.c_str()) && - IDBPolicy::remove(filename.c_str()) != 0) - rc = ERR_COMP_REMOVE_FILE; - - if (rc == NO_ERROR && IDBPolicy::rename(orig.c_str(), filename.c_str()) != 0) - rc = ERR_COMP_RENAME_FILE; - } - - // remove the tmp file - string tmp(filename + ".tmp"); - - if (rc == NO_ERROR && IDBPolicy::exists(tmp.c_str()) && - IDBPolicy::remove(tmp.c_str()) != 0) - rc = ERR_COMP_REMOVE_FILE; - - // remove the chunk shifting helper - string rlc(filename + ".rlc"); - - if (rc == NO_ERROR && IDBPolicy::exists(rlc.c_str()) && - IDBPolicy::remove(rlc.c_str()) != 0) - rc = ERR_COMP_REMOVE_FILE; - - logging::Message::Args args1; - args1.add(filename); - - if (rc == NO_ERROR) - { - args1.add(" is restored."); - SimpleSysLog::instance()->logMsg(args1, - logging::LOG_TYPE_INFO, logging::M0007); - } - else - { - args1.add(" may not restored: "); - args1.add(rc); - SimpleSysLog::instance()->logMsg(args1, - logging::LOG_TYPE_CRITICAL, logging::M0007); - - return rc; - } - } - else - { - //copy back to the data file - std::string backFileName(filename); - - if (backUpFileType.compare("chk") == 0 ) - backFileName += ".chk"; - else - backFileName += ".hdr"; - - //cout << "Rollback found file " << backFileName << endl; - IDBDataFile* sourceFile = IDBDataFile::open( - IDBPolicy::getType(backFileName.c_str(), IDBPolicy::WRITEENG), - backFileName.c_str(), "r", 0); - IDBDataFile* targetFile = IDBDataFile::open( - IDBPolicy::getType(filename.c_str(), IDBPolicy::WRITEENG), - filename.c_str(), "r+", 0); - - size_t byteRead; - unsigned char* readBuf = new unsigned char[size]; - boost::scoped_array readBufPtr( readBuf ); - - if ( sourceFile != NULL ) - { - int rc = sourceFile->seek( 0, 0 ); - - if (rc) - return ERR_FILE_SEEK; - - byteRead = sourceFile->read( readBuf, size ); - - if ( (int) byteRead != size ) - { - logging::Message::Args args6; - args6.add("Rollback cannot read backup file "); - args6.add(backFileName); - SimpleSysLog::instance()->logMsg(args6, logging::LOG_TYPE_ERROR, logging::M0007); - return ERR_FILE_READ; - } - } - else - { - logging::Message::Args args5; - args5.add("Rollback cannot open backup file "); - args5.add(backFileName); - SimpleSysLog::instance()->logMsg(args5, logging::LOG_TYPE_ERROR, logging::M0007); - return ERR_FILE_NULL; - } - - size_t byteWrite; - - if ( targetFile != NULL ) - { - int rc = targetFile->seek( offset, 0 ); - - if (rc) - return ERR_FILE_SEEK; - - byteWrite = targetFile->write( readBuf, size ); - - if ( (int) byteWrite != size ) - { - logging::Message::Args args3; - args3.add("Rollback cannot copy to file "); - args3.add(filename); - args3.add( "from file "); - args3.add(backFileName); - SimpleSysLog::instance()->logMsg(args3, logging::LOG_TYPE_ERROR, logging::M0007); - - return ERR_FILE_WRITE; - } - } - else - { - logging::Message::Args args4; - args4.add("Rollback cannot open target file "); - args4.add(filename); - SimpleSysLog::instance()->logMsg(args4, logging::LOG_TYPE_ERROR, logging::M0007); - return ERR_FILE_NULL; - } - - //cout << "Rollback copied to file " << filename << " from file " << backFileName << endl; - - delete targetFile; - delete sourceFile; - IDBPolicy::remove( backFileName.c_str() ); - logging::Message::Args arg1; - arg1.add("Rollback copied to file "); - arg1.add(filename); - arg1.add( "from file "); - arg1.add(backFileName); - SimpleSysLog::instance()->logMsg(arg1, logging::LOG_TYPE_INFO, logging::M0007); - } - } + // remove the rlc file + filename += ".rlc"; + // cout << " File removed: " << filename << endl; + IDBPolicy::remove(filename.c_str()); + logging::Message::Args args1; + args1.add(filename); + args1.add(" is removed."); + SimpleSysLog::instance()->logMsg(args1, logging::LOG_TYPE_INFO, logging::M0007); } + else if (backUpFileType.compare("tmp") == 0) + { + int rc = NO_ERROR; + string orig(filename + ".orig"); - IDBPolicy::remove(aDMLLogFileName.c_str()); + // restore the orig file + if (IDBPolicy::exists(orig.c_str())) + { + // not likely both cdf and tmp exist + if (IDBPolicy::exists(filename.c_str()) && IDBPolicy::remove(filename.c_str()) != 0) + rc = ERR_COMP_REMOVE_FILE; + + if (rc == NO_ERROR && IDBPolicy::rename(orig.c_str(), filename.c_str()) != 0) + rc = ERR_COMP_RENAME_FILE; + } + + // remove the tmp file + string tmp(filename + ".tmp"); + + if (rc == NO_ERROR && IDBPolicy::exists(tmp.c_str()) && IDBPolicy::remove(tmp.c_str()) != 0) + rc = ERR_COMP_REMOVE_FILE; + + // remove the chunk shifting helper + string rlc(filename + ".rlc"); + + if (rc == NO_ERROR && IDBPolicy::exists(rlc.c_str()) && IDBPolicy::remove(rlc.c_str()) != 0) + rc = ERR_COMP_REMOVE_FILE; + + logging::Message::Args args1; + args1.add(filename); + + if (rc == NO_ERROR) + { + args1.add(" is restored."); + SimpleSysLog::instance()->logMsg(args1, logging::LOG_TYPE_INFO, logging::M0007); + } + else + { + args1.add(" may not restored: "); + args1.add(rc); + SimpleSysLog::instance()->logMsg(args1, logging::LOG_TYPE_CRITICAL, logging::M0007); + + return rc; + } + } + else + { + // copy back to the data file + std::string backFileName(filename); + + if (backUpFileType.compare("chk") == 0) + backFileName += ".chk"; + else + backFileName += ".hdr"; + + // cout << "Rollback found file " << backFileName << endl; + IDBDataFile* sourceFile = IDBDataFile::open( + IDBPolicy::getType(backFileName.c_str(), IDBPolicy::WRITEENG), backFileName.c_str(), "r", 0); + IDBDataFile* targetFile = IDBDataFile::open( + IDBPolicy::getType(filename.c_str(), IDBPolicy::WRITEENG), filename.c_str(), "r+", 0); + + size_t byteRead; + unsigned char* readBuf = new unsigned char[size]; + boost::scoped_array readBufPtr(readBuf); + + if (sourceFile != NULL) + { + int rc = sourceFile->seek(0, 0); + + if (rc) + return ERR_FILE_SEEK; + + byteRead = sourceFile->read(readBuf, size); + + if ((int)byteRead != size) + { + logging::Message::Args args6; + args6.add("Rollback cannot read backup file "); + args6.add(backFileName); + SimpleSysLog::instance()->logMsg(args6, logging::LOG_TYPE_ERROR, logging::M0007); + return ERR_FILE_READ; + } + } + else + { + logging::Message::Args args5; + args5.add("Rollback cannot open backup file "); + args5.add(backFileName); + SimpleSysLog::instance()->logMsg(args5, logging::LOG_TYPE_ERROR, logging::M0007); + return ERR_FILE_NULL; + } + + size_t byteWrite; + + if (targetFile != NULL) + { + int rc = targetFile->seek(offset, 0); + + if (rc) + return ERR_FILE_SEEK; + + byteWrite = targetFile->write(readBuf, size); + + if ((int)byteWrite != size) + { + logging::Message::Args args3; + args3.add("Rollback cannot copy to file "); + args3.add(filename); + args3.add("from file "); + args3.add(backFileName); + SimpleSysLog::instance()->logMsg(args3, logging::LOG_TYPE_ERROR, logging::M0007); + + return ERR_FILE_WRITE; + } + } + else + { + logging::Message::Args args4; + args4.add("Rollback cannot open target file "); + args4.add(filename); + SimpleSysLog::instance()->logMsg(args4, logging::LOG_TYPE_ERROR, logging::M0007); + return ERR_FILE_NULL; + } + + // cout << "Rollback copied to file " << filename << " from file " << backFileName << endl; + + delete targetFile; + delete sourceFile; + IDBPolicy::remove(backFileName.c_str()); + logging::Message::Args arg1; + arg1.add("Rollback copied to file "); + arg1.add(filename); + arg1.add("from file "); + arg1.add(backFileName); + SimpleSysLog::instance()->logMsg(arg1, logging::LOG_TYPE_INFO, logging::M0007); + } + } } - return 0; + IDBPolicy::remove(aDMLLogFileName.c_str()); + } + return 0; } int WriteEngineWrapper::rollbackTran(const TxnID& txnid, int sessionId) { - if ( rollbackCommon( txnid, sessionId ) != 0 ) - return -1; + if (rollbackCommon(txnid, sessionId) != 0) + return -1; - return BRMWrapper::getInstance()->rollBack(txnid, sessionId); + return BRMWrapper::getInstance()->rollBack(txnid, sessionId); } int WriteEngineWrapper::rollbackBlocks(const TxnID& txnid, int sessionId) { - if ( rollbackCommon( txnid, sessionId ) != 0 ) - return -1; + if (rollbackCommon(txnid, sessionId) != 0) + return -1; - return BRMWrapper::getInstance()->rollBackBlocks(txnid, sessionId); + return BRMWrapper::getInstance()->rollBackBlocks(txnid, sessionId); } int WriteEngineWrapper::rollbackVersion(const TxnID& txnid, int sessionId) { - // BUG 4312 - RemoveTxnFromLBIDMap(txnid); - RemoveTxnFromDictMap(txnid); + // BUG 4312 + RemoveTxnFromLBIDMap(txnid); + RemoveTxnFromDictMap(txnid); - return BRMWrapper::getInstance()->rollBackVersion(txnid, sessionId); + return BRMWrapper::getInstance()->rollBackVersion(txnid, sessionId); } -int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, const uint64_t nextVal, const uint32_t sessionID, const uint16_t dbRoot) +int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, const uint64_t nextVal, + const uint32_t sessionID, const uint16_t dbRoot) { - int rc = NO_ERROR; - boost::shared_ptr systemCatalogPtr; - RIDList ridList; - ColValueList colValueList; - WriteEngine::ColTupleList colTuples; - ColStructList colStructList; - WriteEngine::CSCTypesList cscColTypeList; - WriteEngine::ColStruct colStruct; - CalpontSystemCatalog::ColType colType; - colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_NEXTVALUE; - colType.colWidth = colStruct.colWidth = 8; - colStruct.tokenFlag = false; - colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::UBIGINT; - colStruct.fColDbRoot = dbRoot; + int rc = NO_ERROR; + boost::shared_ptr systemCatalogPtr; + RIDList ridList; + ColValueList colValueList; + WriteEngine::ColTupleList colTuples; + ColStructList colStructList; + WriteEngine::CSCTypesList cscColTypeList; + WriteEngine::ColStruct colStruct; + CalpontSystemCatalog::ColType colType; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_NEXTVALUE; + colType.colWidth = colStruct.colWidth = 8; + colStruct.tokenFlag = false; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::UBIGINT; + colStruct.fColDbRoot = dbRoot; - m_opType = UPDATE; + m_opType = UPDATE; - if (idbdatafile::IDBPolicy::useHdfs()) - colStruct.fCompressionType = 2; + if (idbdatafile::IDBPolicy::useHdfs()) + colStruct.fCompressionType = 2; - colStructList.push_back(colStruct); - cscColTypeList.push_back(colType); - ColTuple colTuple; - systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); - CalpontSystemCatalog::ROPair ropair; + colStructList.push_back(colStruct); + cscColTypeList.push_back(colType); + ColTuple colTuple; + systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair ropair; - try - { - ropair = systemCatalogPtr->nextAutoIncrRid(columnoid); - } - catch (...) - { - rc = ERR_AUTOINC_RID; - } - - if (rc != NO_ERROR) - return rc; - - ridList.push_back(ropair.rid); - colTuple.data = nextVal; - colTuples.push_back(colTuple); - colValueList.push_back(colTuples); - rc = writeColumnRecords(txnId, cscColTypeList, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); - - if (rc != NO_ERROR) - return rc; - - //flush PrimProc cache - vector blockList; - BRM::LBIDRange_v lbidRanges; - rc = BRMWrapper::getInstance()->lookupLbidRanges(OID_SYSCOLUMN_NEXTVALUE, - lbidRanges); - - if (rc != NO_ERROR) - return rc; - - LBIDRange_v::iterator it; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - for (LBID_t lbid = it->start; lbid < (it->start + it->size); lbid++) - { - blockList.push_back(lbid); - } - } - - //Bug 5459 Flush FD cache - std::vector files; - BRM::FileInfo aFile; - aFile.oid = colStruct.dataOid; - aFile.partitionNum = colStruct.fColPartition; - aFile.dbRoot = colStruct.fColDbRoot;; - aFile.segmentNum = colStruct.fColSegment; - aFile.compType = colStruct.fCompressionType; - files.push_back(aFile); - - if (idbdatafile::IDBPolicy::useHdfs()) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - - rc = cacheutils::flushPrimProcAllverBlocks (blockList); - - if (rc != 0) - rc = ERR_BLKCACHE_FLUSH_LIST; // translate to WE error + try + { + ropair = systemCatalogPtr->nextAutoIncrRid(columnoid); + } + catch (...) + { + rc = ERR_AUTOINC_RID; + } + if (rc != NO_ERROR) return rc; + + ridList.push_back(ropair.rid); + colTuple.data = nextVal; + colTuples.push_back(colTuple); + colValueList.push_back(colTuples); + rc = writeColumnRecords(txnId, cscColTypeList, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); + + if (rc != NO_ERROR) + return rc; + + // flush PrimProc cache + vector blockList; + BRM::LBIDRange_v lbidRanges; + rc = BRMWrapper::getInstance()->lookupLbidRanges(OID_SYSCOLUMN_NEXTVALUE, lbidRanges); + + if (rc != NO_ERROR) + return rc; + + LBIDRange_v::iterator it; + + for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) + { + for (LBID_t lbid = it->start; lbid < (it->start + it->size); lbid++) + { + blockList.push_back(lbid); + } + } + + // Bug 5459 Flush FD cache + std::vector files; + BRM::FileInfo aFile; + aFile.oid = colStruct.dataOid; + aFile.partitionNum = colStruct.fColPartition; + aFile.dbRoot = colStruct.fColDbRoot; + ; + aFile.segmentNum = colStruct.fColSegment; + aFile.compType = colStruct.fCompressionType; + files.push_back(aFile); + + if (idbdatafile::IDBPolicy::useHdfs()) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + + rc = cacheutils::flushPrimProcAllverBlocks(blockList); + + if (rc != 0) + rc = ERR_BLKCACHE_FLUSH_LIST; // translate to WE error + + return rc; } /*********************************************************** @@ -6604,82 +6446,80 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, ***********************************************************/ int WriteEngineWrapper::flushDataFiles(int rc, const TxnID txnId, std::map& columnOids) { - RemoveTxnFromLBIDMap(txnId); - RemoveTxnFromDictMap(txnId); + RemoveTxnFromLBIDMap(txnId); + RemoveTxnFromDictMap(txnId); - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) + { + int rc1 = m_colOp[i]->flushFile(rc, columnOids); + int rc2 = m_dctnry[i]->flushFile(rc, columnOids); + + if (rc == NO_ERROR) { - int rc1 = m_colOp[i]->flushFile(rc, columnOids); - int rc2 = m_dctnry[i]->flushFile(rc, columnOids); - - if (rc == NO_ERROR) - { - rc = (rc1 != NO_ERROR) ? rc1 : rc2; - } + rc = (rc1 != NO_ERROR) ? rc1 : rc2; } + } - return rc; + return rc; } -void WriteEngineWrapper::AddDictToList(const TxnID txnid, - std::vector& lbids) +void WriteEngineWrapper::AddDictToList(const TxnID txnid, std::vector& lbids) { - std::tr1::unordered_map::iterator mapIter; + std::tr1::unordered_map::iterator mapIter; - mapIter = m_dictLBIDMap.find(txnid); - - if (mapIter == m_dictLBIDMap.end()) - { - dictLBIDRec_t tempRecord; - tempRecord.insert(lbids.begin(), lbids.end()); - m_dictLBIDMap[txnid] = tempRecord; - return; - } - else - { - dictLBIDRec_t& txnRecord = mapIter->second; - txnRecord.insert(lbids.begin(), lbids.end()); - } + mapIter = m_dictLBIDMap.find(txnid); + if (mapIter == m_dictLBIDMap.end()) + { + dictLBIDRec_t tempRecord; + tempRecord.insert(lbids.begin(), lbids.end()); + m_dictLBIDMap[txnid] = tempRecord; + return; + } + else + { + dictLBIDRec_t& txnRecord = mapIter->second; + txnRecord.insert(lbids.begin(), lbids.end()); + } } // Get CPInfo for given starting LBID and column description structure. -int WriteEngineWrapper::GetLBIDRange(const BRM::LBID_t startingLBID, const ColStruct& colStruct, ExtCPInfo& cpInfo) +int WriteEngineWrapper::GetLBIDRange(const BRM::LBID_t startingLBID, const ColStruct& colStruct, + ExtCPInfo& cpInfo) { - int rtn; - BRM::CPMaxMin maxMin; - rtn = BRMWrapper::getInstance()->getExtentCPMaxMin(startingLBID, maxMin); - bool isBinary = cpInfo.isBinaryColumn(); - maxMin.isBinaryColumn = isBinary; - cpInfo.fCPInfo.firstLbid = startingLBID; - if (rtn) - { - cpInfo.toInvalid(); - return rtn; - } - // if we are provided with CPInfo pointer to update, we record current range there. - // please note that we may fail here for unknown extents - e.g., newly allocated ones. - // for these we mark CPInfo as invalid (above) and proceed as usual. - // XXX With this logic we may end with invalid ranges for extents that were - // allocated and recorded, yet not in our copy of extentmap. - // As we update only part of that extent, the recorded range will be for - // that part of the extent. - // It should be investigated whether such situation is possible. - // XXX Please note that most if not all calls to AddLBIDToList are enclosed into - // RETURN_ON_ERROR() macro. - // If we have failed to obtain information above we will abort execution of the function - // that called us. - // This is potential source of bugs. - cpInfo.fCPInfo.bigMax = maxMin.bigMax; - cpInfo.fCPInfo.bigMin = maxMin.bigMin; - cpInfo.fCPInfo.max = maxMin.max; - cpInfo.fCPInfo.min = maxMin.min; - cpInfo.fCPInfo.seqNum = maxMin.seqNum; - cpInfo.fCPInfo.isBinaryColumn = maxMin.isBinaryColumn; + int rtn; + BRM::CPMaxMin maxMin; + rtn = BRMWrapper::getInstance()->getExtentCPMaxMin(startingLBID, maxMin); + bool isBinary = cpInfo.isBinaryColumn(); + maxMin.isBinaryColumn = isBinary; + cpInfo.fCPInfo.firstLbid = startingLBID; + if (rtn) + { + cpInfo.toInvalid(); return rtn; + } + // if we are provided with CPInfo pointer to update, we record current range there. + // please note that we may fail here for unknown extents - e.g., newly allocated ones. + // for these we mark CPInfo as invalid (above) and proceed as usual. + // XXX With this logic we may end with invalid ranges for extents that were + // allocated and recorded, yet not in our copy of extentmap. + // As we update only part of that extent, the recorded range will be for + // that part of the extent. + // It should be investigated whether such situation is possible. + // XXX Please note that most if not all calls to AddLBIDToList are enclosed into + // RETURN_ON_ERROR() macro. + // If we have failed to obtain information above we will abort execution of the function + // that called us. + // This is potential source of bugs. + cpInfo.fCPInfo.bigMax = maxMin.bigMax; + cpInfo.fCPInfo.bigMin = maxMin.bigMin; + cpInfo.fCPInfo.max = maxMin.max; + cpInfo.fCPInfo.min = maxMin.min; + cpInfo.fCPInfo.seqNum = maxMin.seqNum; + cpInfo.fCPInfo.isBinaryColumn = maxMin.isBinaryColumn; + return rtn; } - /*********************************************************** * DESCRIPTION: * Add an lbid to a list of lbids for sending to markExtentsInvalid. @@ -6695,83 +6535,82 @@ int WriteEngineWrapper::GetLBIDRange(const BRM::LBID_t startingLBID, const ColSt * cpInfo - a CPInfo to collect if we need that. * RETURN: 0 => OK. -1 => error ***********************************************************/ -int WriteEngineWrapper::AddLBIDtoList(const TxnID txnid, - const ColStruct& colStruct, - const int fbo, - ExtCPInfo* cpInfo) +int WriteEngineWrapper::AddLBIDtoList(const TxnID txnid, const ColStruct& colStruct, const int fbo, + ExtCPInfo* cpInfo) { - int rtn = 0; + int rtn = 0; - BRM::LBID_t startingLBID; - SP_TxnLBIDRec_t spTxnLBIDRec; - std::tr1::unordered_map::iterator mapIter; + BRM::LBID_t startingLBID; + SP_TxnLBIDRec_t spTxnLBIDRec; + std::tr1::unordered_map::iterator mapIter; - // Find the set of extent starting LBIDs for this transaction. If not found, then create it. - mapIter = m_txnLBIDMap.find(txnid); + // Find the set of extent starting LBIDs for this transaction. If not found, then create it. + mapIter = m_txnLBIDMap.find(txnid); - if (mapIter == m_txnLBIDMap.end()) - { - // This is a new transaction. - SP_TxnLBIDRec_t sptemp(new TxnLBIDRec); - spTxnLBIDRec = sptemp; - m_txnLBIDMap[txnid] = spTxnLBIDRec; -// cout << "New transaction entry " << txnid << " transaction count " << m_txnLBIDMap.size() << endl; - } - else - { - spTxnLBIDRec = (*mapIter).second; - } + if (mapIter == m_txnLBIDMap.end()) + { + // This is a new transaction. + SP_TxnLBIDRec_t sptemp(new TxnLBIDRec); + spTxnLBIDRec = sptemp; + m_txnLBIDMap[txnid] = spTxnLBIDRec; + // cout << "New transaction entry " << txnid << " transaction count " << m_txnLBIDMap.size() << + // endl; + } + else + { + spTxnLBIDRec = (*mapIter).second; + } - // Get the extent starting lbid given all these values (startingLBID is an out parameter). - rtn = BRMWrapper::getInstance()->getStartLbid(colStruct.dataOid, colStruct.fColPartition, - colStruct.fColSegment, fbo, startingLBID); + // Get the extent starting lbid given all these values (startingLBID is an out parameter). + rtn = BRMWrapper::getInstance()->getStartLbid(colStruct.dataOid, colStruct.fColPartition, + colStruct.fColSegment, fbo, startingLBID); - if (rtn != 0) - return -1; + if (rtn != 0) + return -1; - // if we are given the cpInfo (column's ranges can be traced), we should update it. - if (cpInfo) - { - rtn = GetLBIDRange(startingLBID, colStruct, *cpInfo); - } - else - { - spTxnLBIDRec->AddLBID(startingLBID, colStruct.colDataType); - } + // if we are given the cpInfo (column's ranges can be traced), we should update it. + if (cpInfo) + { + rtn = GetLBIDRange(startingLBID, colStruct, *cpInfo); + } + else + { + spTxnLBIDRec->AddLBID(startingLBID, colStruct.colDataType); + } - return rtn; + return rtn; } void WriteEngineWrapper::RemoveTxnFromDictMap(const TxnID txnid) { - std::tr1::unordered_map::iterator mapIter; + std::tr1::unordered_map::iterator mapIter; - mapIter = m_dictLBIDMap.find(txnid); + mapIter = m_dictLBIDMap.find(txnid); - if (mapIter != m_dictLBIDMap.end()) - { - m_dictLBIDMap.erase(txnid); - } + if (mapIter != m_dictLBIDMap.end()) + { + m_dictLBIDMap.erase(txnid); + } } int WriteEngineWrapper::markTxnExtentsAsInvalid(const TxnID txnid, bool erase) { - int rc = 0; - std::tr1::unordered_map::iterator mapIter = m_txnLBIDMap.find(txnid); - if (mapIter != m_txnLBIDMap.end()) + int rc = 0; + std::tr1::unordered_map::iterator mapIter = m_txnLBIDMap.find(txnid); + if (mapIter != m_txnLBIDMap.end()) + { + SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; + if (!spTxnLBIDRec->m_LBIDs.empty()) { - SP_TxnLBIDRec_t spTxnLBIDRec = (*mapIter).second; - if (!spTxnLBIDRec->m_LBIDs.empty()) - { - rc = BRMWrapper::getInstance()->markExtentsInvalid(spTxnLBIDRec->m_LBIDs, spTxnLBIDRec->m_ColDataTypes); - } - - if (erase) - { - m_txnLBIDMap.erase(txnid); // spTxnLBIDRec is auto-destroyed - } + rc = BRMWrapper::getInstance()->markExtentsInvalid(spTxnLBIDRec->m_LBIDs, spTxnLBIDRec->m_ColDataTypes); } - return rc; + + if (erase) + { + m_txnLBIDMap.erase(txnid); // spTxnLBIDRec is auto-destroyed + } + } + return rc; } /*********************************************************** @@ -6785,10 +6624,8 @@ int WriteEngineWrapper::markTxnExtentsAsInvalid(const TxnID txnid, bool erase) ***********************************************************/ int WriteEngineWrapper::RemoveTxnFromLBIDMap(const TxnID txnid) { - return markTxnExtentsAsInvalid(txnid, true); + return markTxnExtentsAsInvalid(txnid, true); } - -} //end of namespace +} // end of namespace // vim:ts=4 sw=4: - diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index a3f9634fc..a8eb864fd 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -17,7 +17,6 @@ // $Id: writeengine.h 4726 2013-08-07 03:38:36Z bwilkinson $ - /** @file */ #ifndef _WRITE_ENGINE_H_ @@ -56,7 +55,6 @@ /** Namespace WriteEngine */ namespace WriteEngine { - //... Total compression operation: un_compresssed, compressed const int UN_COMPRESSED_OP = 0; const int COMPRESSED_OP_1 = 1; @@ -73,763 +71,720 @@ class Log; // for truly long running transactions. struct TxnLBIDRec { - std::tr1::unordered_set m_LBIDSet; - std::vector m_LBIDs; - std::vector m_ColDataTypes; + std::tr1::unordered_set m_LBIDSet; + std::vector m_LBIDs; + std::vector m_ColDataTypes; - TxnLBIDRec() {}; - ~TxnLBIDRec() {} - void AddLBID(BRM::LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType& colDataType) + TxnLBIDRec(){}; + ~TxnLBIDRec() + { + } + void AddLBID(BRM::LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType& colDataType) + { + if (m_LBIDSet.insert(lbid).second) { - if ( m_LBIDSet.insert(lbid).second) - { - m_LBIDs.push_back(lbid); - m_ColDataTypes.push_back(colDataType); - } + m_LBIDs.push_back(lbid); + m_ColDataTypes.push_back(colDataType); } + } }; -typedef boost::shared_ptr SP_TxnLBIDRec_t; +typedef boost::shared_ptr SP_TxnLBIDRec_t; typedef std::set dictLBIDRec_t; /** @brief Range information for 1 or 2 extents changed by DML operation. */ -struct ColSplitMaxMinInfo { - ExtCPInfo fSplitMaxMinInfo[2]; /** @brief internal to write engine: min/max ranges for data in one and, possible, second extent. */ - ExtCPInfo* fSplitMaxMinInfoPtrs[2]; /** @brief pointers to CPInfos in fSplitMaxMinInfo above */ - ColSplitMaxMinInfo(execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) - : fSplitMaxMinInfo { ExtCPInfo(colDataType, colWidth), ExtCPInfo(colDataType, colWidth) } - { - fSplitMaxMinInfoPtrs[0] = fSplitMaxMinInfoPtrs[1] = NULL; // disable by default. - } +struct ColSplitMaxMinInfo +{ + ExtCPInfo fSplitMaxMinInfo[2]; /** @brief internal to write engine: min/max ranges for data in one and, + possible, second extent. */ + ExtCPInfo* fSplitMaxMinInfoPtrs[2]; /** @brief pointers to CPInfos in fSplitMaxMinInfo above */ + ColSplitMaxMinInfo(execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) + : fSplitMaxMinInfo{ExtCPInfo(colDataType, colWidth), ExtCPInfo(colDataType, colWidth)} + { + fSplitMaxMinInfoPtrs[0] = fSplitMaxMinInfoPtrs[1] = NULL; // disable by default. + } }; typedef std::vector ColSplitMaxMinInfoList; - /** Class WriteEngineWrapper */ class WriteEngineWrapper : public WEObj { -public: - /** - * @brief Constructor - */ - EXPORT WriteEngineWrapper(); + public: + /** + * @brief Constructor + */ + EXPORT WriteEngineWrapper(); - EXPORT WriteEngineWrapper(const WriteEngineWrapper& rhs); - /** - * @brief Default Destructor - */ - EXPORT ~WriteEngineWrapper(); + EXPORT WriteEngineWrapper(const WriteEngineWrapper& rhs); + /** + * @brief Default Destructor + */ + EXPORT ~WriteEngineWrapper(); - /************************************************************************ - * Interface definitions - ************************************************************************/ - /** - * @brief Performs static/global initialization for BRMWrapper. - * Should be called once from the main thread. - */ - EXPORT static void init(unsigned subSystemID); + /************************************************************************ + * Interface definitions + ************************************************************************/ + /** + * @brief Performs static/global initialization for BRMWrapper. + * Should be called once from the main thread. + */ + EXPORT static void init(unsigned subSystemID); - /** - * @brief Build a index from an oid file (NOTE: this is write engine internal used function, just for test purpose and not for generic use - */ - int buildIndex(const OID& colOid, const OID& treeOid, const OID& listOid, - execplan::CalpontSystemCatalog::ColDataType colDataType, int width, int hwm, - bool resetFile, uint64_t& totalRows, int maxRow = IDX_DEFAULT_READ_ROW) - { - return -1; - } + /** + * @brief Build a index from an oid file (NOTE: this is write engine internal used function, just for test + * purpose and not for generic use + */ + int buildIndex(const OID& colOid, const OID& treeOid, const OID& listOid, + execplan::CalpontSystemCatalog::ColDataType colDataType, int width, int hwm, bool resetFile, + uint64_t& totalRows, int maxRow = IDX_DEFAULT_READ_ROW) + { + return -1; + } - /** - * @brief Build a index from a file - */ - int buildIndex(const std::string& sourceFileName, const OID& treeOid, const OID& listOid, - execplan::CalpontSystemCatalog::ColDataType colDataType, int width, int hwm, bool resetFile, - uint64_t& totalRows, const std::string& indexName, Log* pLogger, - int maxRow = IDX_DEFAULT_READ_ROW) - { - return -1; - } + /** + * @brief Build a index from a file + */ + int buildIndex(const std::string& sourceFileName, const OID& treeOid, const OID& listOid, + execplan::CalpontSystemCatalog::ColDataType colDataType, int width, int hwm, bool resetFile, + uint64_t& totalRows, const std::string& indexName, Log* pLogger, + int maxRow = IDX_DEFAULT_READ_ROW) + { + return -1; + } - /** - * @brief Close a index file - */ - void closeIndex() { } + /** + * @brief Close a index file + */ + void closeIndex() + { + } - /** - * @brief Close a dictionary - */ - int closeDctnry(const TxnID& txnid, int i, bool realClose = true) - { - return m_dctnry[op(i)]->closeDctnry(realClose); - } + /** + * @brief Close a dictionary + */ + int closeDctnry(const TxnID& txnid, int i, bool realClose = true) + { + return m_dctnry[op(i)]->closeDctnry(realClose); + } - /** - * @brief Commit transaction - */ - int commit(const TxnID& txnid) - { - m_txnLBIDMap.erase(txnid); - return BRMWrapper::getInstance()->commit(txnid); - } + /** + * @brief Commit transaction + */ + int commit(const TxnID& txnid) + { + m_txnLBIDMap.erase(txnid); + return BRMWrapper::getInstance()->commit(txnid); + } - /** - * @brief Convert interface value list to internal value array - */ - EXPORT void convertValArray(const size_t totalRow, + /** + * @brief Convert interface value list to internal value array + */ + EXPORT void convertValArray(const size_t totalRow, + const execplan::CalpontSystemCatalog::ColType& cscColType, + const ColType colType, ColTupleList& curTupleList, void* valArray, + bool bFromList = true); + /** + * @brief Updates range information given old range information, old values, new values and column + * information. + */ + EXPORT void updateMaxMinRange(const size_t totalNewRow, const size_t totalOldRow, const execplan::CalpontSystemCatalog::ColType& cscColType, - const ColType colType, - ColTupleList& curTupleList, void* valArray, - bool bFromList = true) ; - /** - * @brief Updates range information given old range information, old values, new values and column information. - */ - EXPORT void updateMaxMinRange(const size_t totalNewRow, const size_t totalOldRow, - const execplan::CalpontSystemCatalog::ColType& cscColType, - const ColType colType, - const void* valArray, const void* oldValArray, + const ColType colType, const void* valArray, const void* oldValArray, ExtCPInfo* maxMin, bool canStartWithInvalidRange); + /** + * @brief Create a column, include object ids for column data and bitmap files + * @param dataOid column datafile object id + * @param dataType column data type + * @param dataWidth column width + * @param dbRoot DBRoot under which file is to be located (1-based) + * @param partition Starting partition number for segment file path (0-based). + * @param compressionType compression type + */ + EXPORT int createColumn(const TxnID& txnid, const OID& dataOid, + execplan::CalpontSystemCatalog::ColDataType dataType, int dataWidth, + uint16_t dbRoot, uint32_t partition = 0, int compressionType = 0); - /** - * @brief Create a column, include object ids for column data and bitmap files - * @param dataOid column datafile object id - * @param dataType column data type - * @param dataWidth column width - * @param dbRoot DBRoot under which file is to be located (1-based) - * @param partition Starting partition number for segment file path (0-based). - * @param compressionType compression type - */ - EXPORT int createColumn(const TxnID& txnid, const OID& dataOid, - execplan::CalpontSystemCatalog::ColDataType dataType, int dataWidth, - uint16_t dbRoot, uint32_t partition = 0, int compressionType = 0); + // BUG931 + /** + * @brief Fill a new column with default value using row-ids from a reference column + * + * @param txnid Transaction id + * @param dataOid OID of the new column + * @param dataType Data-type of the new column + * @param dataWidth Width of the new column + * @param defaultVal Default value to be filled in the new column + * @param refColOID OID of the reference column + * @param refColDataType Data-type of the referecne column + * @param refColWidth Width of the reference column + */ + EXPORT int fillColumn(const TxnID& txnid, const OID& dataOid, + const execplan::CalpontSystemCatalog::ColType& colType, ColTuple defaultVal, + const OID& refColOID, execplan::CalpontSystemCatalog::ColDataType refColDataType, + int refColWidth, int refCompressionType, bool isNULL, int compressionType, + const std::string& defaultValStr, const OID& dictOid = 0, bool autoincrement = false); - //BUG931 - /** - * @brief Fill a new column with default value using row-ids from a reference column - * - * @param txnid Transaction id - * @param dataOid OID of the new column - * @param dataType Data-type of the new column - * @param dataWidth Width of the new column - * @param defaultVal Default value to be filled in the new column - * @param refColOID OID of the reference column - * @param refColDataType Data-type of the referecne column - * @param refColWidth Width of the reference column - */ - EXPORT int fillColumn(const TxnID& txnid, const OID& dataOid, const execplan::CalpontSystemCatalog::ColType& colType, - ColTuple defaultVal, - const OID& refColOID, execplan::CalpontSystemCatalog::ColDataType refColDataType, - int refColWidth, int refCompressionType, bool isNULL, int compressionType, - const std::string& defaultValStr, const OID& dictOid = 0, bool autoincrement = false); + /** + * @brief Create a index related files, include object ids for index tree and list files - /** - * @brief Create a index related files, include object ids for index tree and list files + * @param treeOid index tree file object id + * @param listOid index list file object id + */ + int createIndex(const TxnID& txnid, const OID& treeOid, const OID& listOid) + { + int rc = -1; + return rc; + } - * @param treeOid index tree file object id - * @param listOid index list file object id - */ - int createIndex(const TxnID& txnid, const OID& treeOid, const OID& listOid) + /** + * @brief Create dictionary + * @param dctnryOid dictionary signature file object id + * @param partition Starting partition number for segment file path (0-based). + * @param segment segment number + * @param compressionType compression type + */ + EXPORT int createDctnry(const TxnID& txnid, const OID& dctnryOid, int colWidth, uint16_t dbRoot, + uint32_t partiotion = 0, uint16_t segment = 0, int compressionType = 0); + + /** + * @brief Delete a list of rows from a table + * @param colStructList column struct list + * @param colOldValueList column old values list (return value) + * @param rowIdList row id list + */ + EXPORT int deleteRow(const TxnID& txnid, const std::vector& colExtentsColType, + std::vector& colExtentsStruct, std::vector& colOldValueList, + std::vector& ridLists, const int32_t tableOid); + + /** + * @brief Delete a list of rows from a table + * @param colStructList column struct list + * @param rowIdList row id list + */ + + EXPORT int deleteBadRows(const TxnID& txnid, ColStructList& colStructs, RIDList& ridList, + DctnryStructList& dctnryStructList); + + /** + * @brief delete a dictionary signature and its token + * @param dctnryStruct dictionary structure + * @param dctnryTuple dictionary tuple + */ + // ITER17_Obsolete + // int deleteToken(const TxnID& txnid, Token& token); // Files need already open + // int deleteToken(const TxnID& txnid, DctnryStruct& dctnryStruct, Token& token); + + /** + * @brief Drop a column, include object ids for column data file + * @param dataOid column datafile object id + */ + int dropColumn(const TxnID& txnid, const OID dataOid) + { + return m_colOp[0]->dropColumn((FID)dataOid); + } + + /** + * @brief Drop files + * @param dataOids column and dictionary datafile object id + */ + int dropFiles(const TxnID& txnid, const std::vector& dataOids) + { + return m_colOp[0]->dropFiles(dataOids); + } + + /** + * @brief Delete files for one partition + * @param dataOids column and dictionary datafile object id + */ + int deletePartitions(const std::vector& dataOids, const std::vector& partitions) + { + return m_colOp[0]->dropPartitions(dataOids, partitions); + } + + int deleteOIDsFromExtentMap(const TxnID& txnid, const std::vector& dataOids) + { + return m_colOp[0]->deleteOIDsFromExtentMap(dataOids); + } + + /** + * @brief Create a index related files, include object ids for index tree and list files + * @param treeOid index tree file object id + * @param listOid index list file object id + */ + int dropIndex(const TxnID& txnid, const OID& treeOid, const OID& listOid) + { + return -1; + } + + /** + * @brief Drop a dictionary + * @param dctnryOid dictionary signature file object id + * @param treeOid dictionary tree file object id + * @param listOid index list file object id + */ + int dropDctnry(const TxnID& txnid, const OID& dctnryOid, const OID& treeOid, const OID& listOid) + { + return m_dctnry[0]->dropDctnry(dctnryOid); + } + + /** + * @brief Flush VM write cache + * @param None + */ + EXPORT void flushVMCache() const; + + /** + * @brief Insert values into a table + * @param colStructList column structure list + * @param colValueList column value list + * @param dicStringListt dictionary values list + * @param dbRootExtentTrackers dbrootTrackers + * @param bFirstExtentOnThisPM true when there is no extent on this PM + * @param insertSelect if insert with select, the hwm block is skipped + * @param isAutoCommitOn if autocommit on, only the hwm block is versioned, + * else eveything is versioned + * @param tableOid used to get table meta data + * @param isFirstBatchPm to track if this batch is first batch for this PM. + */ + EXPORT int insertColumnRecs(const TxnID& txnid, const CSCTypesList& cscColTypeList, + ColStructList& colStructList, ColValueList& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + std::vector >& dbRootExtentTrackers, + RBMetaWriter* fRBMetaWriter, bool bFirstExtentOnThisPM, + bool insertSelect = false, bool isAutoCommitOn = false, OID tableOid = 0, + bool isFirstBatchPm = false); + + EXPORT int insertColumnRecsBinary( + const TxnID& txnid, ColStructList& colStructList, std::vector& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + std::vector >& dbRootExtentTrackers, RBMetaWriter* fRBMetaWriter, + bool bFirstExtentOnThisPM, bool insertSelect = false, bool isAutoCommitOn = false, OID tableOid = 0, + bool isFirstBatchPm = false); + + /** + * @brief Insert values into systables + * @param colStructList column structure list + * @param colValueList column value list + * @param dicStringListt dictionary values list + */ + EXPORT int insertColumnRec_SYS(const TxnID& txnid, const CSCTypesList& cscColTypeList, + ColStructList& colStructList, ColValueList& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + const int32_t tableOid); + + /** + * @brief Insert a row + * @param colStructList column structure list + * @param colValueList column value list + * @param dicStringListt dictionary values list + */ + EXPORT int insertColumnRec_Single(const TxnID& txnid, const CSCTypesList& cscColTypeList, + ColStructList& colStructList, ColValueList& colValueList, + DctnryStructList& dctnryStructList, DictStrList& dictStrList, + const int32_t tableOid); + /** + * @brief Open dictionary + * @param txnid relevant transaction + * @param dctnryStruct dictionary column to open + * @param useTmpSuffix Bulk HDFS usage: use *.tmp file suffix + */ + // @bug 5572 - HDFS usage: add *.tmp file backup flag + int openDctnry(const TxnID& txnid, DctnryStruct dctnryStruct, bool useTmpSuffix) + { + int compress_op = op(dctnryStruct.fCompressionType); + m_dctnry[compress_op]->setTransId(txnid); + return m_dctnry[compress_op]->openDctnry(dctnryStruct.dctnryOid, dctnryStruct.fColDbRoot, + dctnryStruct.fColPartition, dctnryStruct.fColSegment, + useTmpSuffix); + } + + /** + * @brief Rollback transaction (common portion) + */ + EXPORT int rollbackCommon(const TxnID& txnid, int sessionId); + + /** + * @brief Rollback transaction + */ + EXPORT int rollbackTran(const TxnID& txnid, int sessionId); + + /** + * @brief Rollback transaction + */ + EXPORT int rollbackBlocks(const TxnID& txnid, int sessionId); + + /** + * @brief Rollback transaction + */ + EXPORT int rollbackVersion(const TxnID& txnid, int sessionId); + + /** + * @brief Set the IsInsert flag in the ChunkManagers. + * This forces flush at end of block. Used only for bulk insert. + */ + void setIsInsert(bool bIsInsert) + { + m_colOp[COMPRESSED_OP_1]->chunkManager()->setIsInsert(bIsInsert); + m_dctnry[COMPRESSED_OP_1]->chunkManager()->setIsInsert(true); + m_colOp[COMPRESSED_OP_2]->chunkManager()->setIsInsert(bIsInsert); + m_dctnry[COMPRESSED_OP_2]->chunkManager()->setIsInsert(true); + } + + /** + * @brief Get the IsInsert flag as set in the ChunkManagers. + * Since both chunk managers are supposed to be in lockstep as regards the + * isInsert flag, we need only grab one. + * + */ + bool getIsInsert() + { + return m_colOp[COMPRESSED_OP_1]->chunkManager()->getIsInsert(); + } + + std::tr1::unordered_map& getTxnMap() + { + return m_txnLBIDMap; + }; + std::tr1::unordered_map& getDictMap() + { + return m_dictLBIDMap; + }; + /** + * @brief Flush the ChunkManagers. + */ + int flushChunks(int rc, const std::map& columOids) + { + std::vector compressedOpIds = {COMPRESSED_OP_1, COMPRESSED_OP_2}; + + for (const auto compressedOpId : compressedOpIds) { - int rc = -1; - return rc; + auto rtn = m_colOp[compressedOpId]->chunkManager()->flushChunks(rc, columOids); + if (rtn != NO_ERROR) + return rtn; + + rtn = m_dctnry[compressedOpId]->chunkManager()->flushChunks(rc, columOids); + if (rtn != NO_ERROR) + return rtn; } - /** - * @brief Create dictionary - * @param dctnryOid dictionary signature file object id - * @param partition Starting partition number for segment file path (0-based). - * @param segment segment number - * @param compressionType compression type - */ - EXPORT int createDctnry(const TxnID& txnid, const OID& dctnryOid, - int colWidth, uint16_t dbRoot, - uint32_t partiotion = 0, uint16_t segment = 0, int compressionType = 0); + return NO_ERROR; + } - /** - * @brief Delete a list of rows from a table - * @param colStructList column struct list - * @param colOldValueList column old values list (return value) - * @param rowIdList row id list - */ - EXPORT int deleteRow(const TxnID& txnid, const std::vector& colExtentsColType, std::vector& colExtentsStruct, - std::vector& colOldValueList, std::vector& ridLists, const int32_t tableOid); - - /** - * @brief Delete a list of rows from a table - * @param colStructList column struct list - * @param rowIdList row id list - */ - - EXPORT int deleteBadRows(const TxnID& txnid, ColStructList& colStructs, - RIDList& ridList, DctnryStructList& dctnryStructList); - - - /** - * @brief delete a dictionary signature and its token - * @param dctnryStruct dictionary structure - * @param dctnryTuple dictionary tuple - */ - //ITER17_Obsolete - // int deleteToken(const TxnID& txnid, Token& token); // Files need already open - // int deleteToken(const TxnID& txnid, DctnryStruct& dctnryStruct, Token& token); - - /** - * @brief Drop a column, include object ids for column data file - * @param dataOid column datafile object id - */ - int dropColumn(const TxnID& txnid, const OID dataOid) + /** + * @brief Set the transaction id into all fileops + */ + void setTransId(const TxnID& txnid) + { + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) { - return m_colOp[0]->dropColumn((FID) dataOid); + m_colOp[i]->setTransId(txnid); + m_dctnry[i]->setTransId(txnid); } + } - /** - * @brief Drop files - * @param dataOids column and dictionary datafile object id - */ - int dropFiles(const TxnID& txnid, const std::vector& dataOids) + /** + * @brief Set the fIsBulk id into all fileops + */ + void setBulkFlag(bool isBulk) + { + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) { - return m_colOp[0]->dropFiles(dataOids); + m_colOp[i]->setBulkFlag(isBulk); + m_dctnry[i]->setBulkFlag(isBulk); } + } - /** - * @brief Delete files for one partition - * @param dataOids column and dictionary datafile object id - */ - int deletePartitions(const std::vector& dataOids, - const std::vector& partitions) + /** + * @brief Set the fIsFix into all fileops + */ + void setFixFlag(bool isFix = false) + { + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) { - return m_colOp[0]->dropPartitions(dataOids, partitions); + m_colOp[i]->setFixFlag(isFix); + m_dctnry[i]->setFixFlag(isFix); } + } - int deleteOIDsFromExtentMap (const TxnID& txnid, const std::vector& dataOids) + /** + * @brief let chunkmanager start transaction. + * + */ + int startTransaction(const TxnID& txnid) + { + int rc = 0; + rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->startTransaction(txnid); + // if ( rc == 0) + // rc = m_dctnry[COMPRESSED_OP]->chunkManager()->startTransaction(txnid); + return rc; + } + + /** + * @brief let chunkmanager confirm transaction. + * + */ + int confirmTransaction(const TxnID& txnid) + { + int rc = 0; + rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->confirmTransaction(txnid); + return rc; + } + + /** + * @brief let chunkmanager end transaction. + * + */ + int endTransaction(const TxnID& txnid, bool success) + { + int rc = 0; + rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->endTransaction(txnid, success); + // if ( rc == 0) + // rc = m_dctnry[COMPRESSED_OP]->chunkManager()->endTransaction(txnid, success); + return rc; + } + + /** + * @brief Tokenize a dictionary signature into a token + * @param dctnryStruct dictionary structure + * @param dctnryTuple dictionary tuple + * @param useTmpSuffix Bulk HDFS usage: use *.tmp file suffix + */ + EXPORT int tokenize(const TxnID& txnid, DctnryTuple&, int compType); // Files need open first + EXPORT int tokenize(const TxnID& txnid, DctnryStruct& dctnryStruct, DctnryTuple& dctnryTuple, + bool useTmpSuffix); + + /** + * @brief Update values into a column (New one) + * @param colStructList column structure list + * @param colValueList column value list + * @param colOldValueList column old values list (return value) + * @param ridList row id list + */ + EXPORT int updateColumnRec(const TxnID& txnid, const std::vector& colExtentsColType, + std::vector& colExtentsStruct, ColValueList& colValueList, + std::vector& colOldValueList, std::vector& ridLists, + std::vector& dctnryExtentsStruct, + DctnryValueList& dctnryValueList, const int32_t tableOid); + + /** + * @brief Update values into columns + * @param colStructList column structure list + * @param colValueList column value list + * @param ridList row id list + */ + + EXPORT int updateColumnRecs(const TxnID& txnid, const CSCTypesList& cscColTypeList, + std::vector& colStructList, ColValueList& colValueList, + const RIDList& ridLists, const int32_t tableOid); + + /** + * @brief Release specified table lock. + * @param lockID Table lock id to be released. + * @param errorMsg Return error message + */ + EXPORT int clearTableLockOnly(uint64_t lockID, std::string& errorMsg); + + /** + * @brief Rollback the specified table + * @param tableOid Table to be rolled back + * @param lockID Table lock id of the table to be rolled back. + * Currently used for logging only. + * @param tableName Name of table associated with tableOid. + * Currently used for logging only. + * @param applName Application that is driving this bulk rollback. + * Currently used for logging only. + * @param debugConsole Enable logging to console + * @param errorMsg Return error message + */ + EXPORT int bulkRollback(OID tableOid, uint64_t lockID, const std::string& tableName, + const std::string& applName, bool debugConsole, std::string& errorMsg); + + /** + * @brief update SYSCOLUMN next value + * @param oidValueMap + */ + EXPORT int updateNextValue(const TxnID txnId, const OID& columnoid, const uint64_t nextVal, + const uint32_t sessionID, const uint16_t dbRoot); + + /** + * @brief write active datafiles to disk + * + */ + EXPORT int flushDataFiles(int rc, const TxnID txnId, std::map& columnOids); + + /** + * @brief Process versioning for batch insert - only version the hwm block. + */ + EXPORT int processBatchVersions(const TxnID& txnid, std::vector columns, + std::vector& rangeList); + + EXPORT void writeVBEnd(const TxnID& txnid, std::vector& rangeList); + + /************************************************************************ + * Future implementations + ************************************************************************/ + /** + * @brief Begin transaction + */ + // todo: add implementation when we work on version control + // int beginTran(const TransID transOid) { return NO_ERROR; } + + /** + * @brief End transaction + */ + // todo: add implementation when we work on version control + // int endTran(const TransID transOid) { return NO_ERROR; } + // WIP + void setDebugLevel(const DebugLevel level) + { + WEObj::setDebugLevel(level); + + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) { - return m_colOp[0]->deleteOIDsFromExtentMap(dataOids); + m_colOp[i]->setDebugLevel(level); + m_dctnry[i]->setDebugLevel(level); } + } // todo: cleanup - /** - * @brief Create a index related files, include object ids for index tree and list files - * @param treeOid index tree file object id - * @param listOid index list file object id - */ - int dropIndex(const TxnID& txnid, const OID& treeOid, const OID& listOid) - { - return -1; - } + /************************************************************************ + * Internal use definitions + ************************************************************************/ + private: + /** + * @brief Check whether the passing parameters are valid + */ + int checkValid(const TxnID& txnid, const ColStructList& colStructList, const ColValueList& colValueList, + const RIDList& ridList) const; - /** - * @brief Drop a dictionary - * @param dctnryOid dictionary signature file object id - * @param treeOid dictionary tree file object id - * @param listOid index list file object id - */ - int dropDctnry(const TxnID& txnid, const OID& dctnryOid, const OID& treeOid, const OID& listOid) - { - return m_dctnry[0]->dropDctnry(dctnryOid); - } - - /** - * @brief Flush VM write cache - * @param None - */ - EXPORT void flushVMCache() const; - - /** - * @brief Insert values into a table - * @param colStructList column structure list - * @param colValueList column value list - * @param dicStringListt dictionary values list - * @param dbRootExtentTrackers dbrootTrackers - * @param bFirstExtentOnThisPM true when there is no extent on this PM - * @param insertSelect if insert with select, the hwm block is skipped - * @param isAutoCommitOn if autocommit on, only the hwm block is versioned, - * else eveything is versioned - * @param tableOid used to get table meta data - * @param isFirstBatchPm to track if this batch is first batch for this PM. - */ - EXPORT int insertColumnRecs(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - ColStructList& colStructList, - ColValueList& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - std::vector >& dbRootExtentTrackers, - RBMetaWriter* fRBMetaWriter, - bool bFirstExtentOnThisPM, - bool insertSelect = false, - bool isAutoCommitOn = false, - OID tableOid = 0, - bool isFirstBatchPm = false); - - EXPORT int insertColumnRecsBinary(const TxnID& txnid, - ColStructList& colStructList, - std::vector& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - std::vector >& dbRootExtentTrackers, - RBMetaWriter* fRBMetaWriter, - bool bFirstExtentOnThisPM, - bool insertSelect = false, - bool isAutoCommitOn = false, - OID tableOid = 0, - bool isFirstBatchPm = false); - - - /** - * @brief Insert values into systables - * @param colStructList column structure list - * @param colValueList column value list - * @param dicStringListt dictionary values list - */ - EXPORT int insertColumnRec_SYS(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - ColStructList& colStructList, - ColValueList& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - const int32_t tableOid); - - /** - * @brief Insert a row - * @param colStructList column structure list - * @param colValueList column value list - * @param dicStringListt dictionary values list - */ - EXPORT int insertColumnRec_Single(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - ColStructList& colStructList, - ColValueList& colValueList, - DctnryStructList& dctnryStructList, - DictStrList& dictStrList, - const int32_t tableOid); - /** - * @brief Open dictionary - * @param txnid relevant transaction - * @param dctnryStruct dictionary column to open - * @param useTmpSuffix Bulk HDFS usage: use *.tmp file suffix - */ - // @bug 5572 - HDFS usage: add *.tmp file backup flag - int openDctnry(const TxnID& txnid, DctnryStruct dctnryStruct, bool useTmpSuffix) - { - int compress_op = op(dctnryStruct.fCompressionType); - m_dctnry[compress_op]->setTransId(txnid); - return m_dctnry[compress_op]->openDctnry( - dctnryStruct.dctnryOid, - dctnryStruct.fColDbRoot, - dctnryStruct.fColPartition, - dctnryStruct.fColSegment, - useTmpSuffix); - } - - /** - * @brief Rollback transaction (common portion) - */ - EXPORT int rollbackCommon(const TxnID& txnid, int sessionId); - - /** - * @brief Rollback transaction - */ - EXPORT int rollbackTran(const TxnID& txnid, int sessionId); - - /** - * @brief Rollback transaction - */ - EXPORT int rollbackBlocks(const TxnID& txnid, int sessionId); - - /** - * @brief Rollback transaction - */ - EXPORT int rollbackVersion(const TxnID& txnid, int sessionId); - - /** - * @brief Set the IsInsert flag in the ChunkManagers. - * This forces flush at end of block. Used only for bulk insert. - */ - void setIsInsert(bool bIsInsert) - { - m_colOp[COMPRESSED_OP_1]->chunkManager()->setIsInsert(bIsInsert); - m_dctnry[COMPRESSED_OP_1]->chunkManager()->setIsInsert(true); - m_colOp[COMPRESSED_OP_2]->chunkManager()->setIsInsert(bIsInsert); - m_dctnry[COMPRESSED_OP_2]->chunkManager()->setIsInsert(true); - } - - /** - * @brief Get the IsInsert flag as set in the ChunkManagers. - * Since both chunk managers are supposed to be in lockstep as regards the - * isInsert flag, we need only grab one. - * - */ - bool getIsInsert() - { - return m_colOp[COMPRESSED_OP_1]->chunkManager()->getIsInsert(); - } - - std::tr1::unordered_map& getTxnMap() - { - - return m_txnLBIDMap; - }; - std::tr1::unordered_map& getDictMap() - { - return m_dictLBIDMap; - }; - /** - * @brief Flush the ChunkManagers. - */ - int flushChunks(int rc, const std::map& columOids) - { - std::vector compressedOpIds = {COMPRESSED_OP_1, - COMPRESSED_OP_2}; - - for (const auto compressedOpId : compressedOpIds) - { - auto rtn = m_colOp[compressedOpId]->chunkManager()->flushChunks( - rc, columOids); - if (rtn != NO_ERROR) - return rtn; - - rtn = m_dctnry[compressedOpId]->chunkManager()->flushChunks( - rc, columOids); - if (rtn != NO_ERROR) - return rtn; - } - - return NO_ERROR; - } - - /** - * @brief Set the transaction id into all fileops - */ - void setTransId(const TxnID& txnid) - { - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setTransId(txnid); - m_dctnry[i]->setTransId(txnid); - } - } - - /** - * @brief Set the fIsBulk id into all fileops - */ - void setBulkFlag(bool isBulk) - { - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setBulkFlag(isBulk); - m_dctnry[i]->setBulkFlag(isBulk); - } - } - - /** - * @brief Set the fIsFix into all fileops - */ - void setFixFlag(bool isFix = false) - { - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setFixFlag(isFix); - m_dctnry[i]->setFixFlag(isFix); - } - } - - /** - * @brief let chunkmanager start transaction. - * - */ - int startTransaction(const TxnID& txnid) - { - int rc = 0; - rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->startTransaction(txnid); - //if ( rc == 0) - // rc = m_dctnry[COMPRESSED_OP]->chunkManager()->startTransaction(txnid); - return rc; - } - - /** - * @brief let chunkmanager confirm transaction. - * - */ - int confirmTransaction (const TxnID& txnid) - { - int rc = 0; - rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->confirmTransaction( - txnid); - return rc; - } - - - /** - * @brief let chunkmanager end transaction. - * - */ - int endTransaction(const TxnID& txnid, bool success) - { - int rc = 0; - rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->endTransaction(txnid, - success); - //if ( rc == 0) - // rc = m_dctnry[COMPRESSED_OP]->chunkManager()->endTransaction(txnid, success); - return rc; - } - - /** - * @brief Tokenize a dictionary signature into a token - * @param dctnryStruct dictionary structure - * @param dctnryTuple dictionary tuple - * @param useTmpSuffix Bulk HDFS usage: use *.tmp file suffix - */ - EXPORT int tokenize(const TxnID& txnid, DctnryTuple&, int compType ); // Files need open first - EXPORT int tokenize(const TxnID& txnid, DctnryStruct& dctnryStruct, DctnryTuple& dctnryTuple, - bool useTmpSuffix); - - /** - * @brief Update values into a column (New one) - * @param colStructList column structure list - * @param colValueList column value list - * @param colOldValueList column old values list (return value) - * @param ridList row id list - */ - EXPORT int updateColumnRec(const TxnID& txnid, - const std::vector& colExtentsColType, - std::vector& colExtentsStruct, - ColValueList& colValueList, - std::vector& colOldValueList, - std::vector& ridLists, - std::vector& dctnryExtentsStruct, - DctnryValueList& dctnryValueList, - const int32_t tableOid); - - /** - * @brief Update values into columns - * @param colStructList column structure list - * @param colValueList column value list - * @param ridList row id list - */ - - EXPORT int updateColumnRecs(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - std::vector& colStructList, - ColValueList& colValueList, - const RIDList& ridLists, - const int32_t tableOid); - - /** - * @brief Release specified table lock. - * @param lockID Table lock id to be released. - * @param errorMsg Return error message - */ - EXPORT int clearTableLockOnly(uint64_t lockID, - std::string& errorMsg); - - /** - * @brief Rollback the specified table - * @param tableOid Table to be rolled back - * @param lockID Table lock id of the table to be rolled back. - * Currently used for logging only. - * @param tableName Name of table associated with tableOid. - * Currently used for logging only. - * @param applName Application that is driving this bulk rollback. - * Currently used for logging only. - * @param debugConsole Enable logging to console - * @param errorMsg Return error message - */ - EXPORT int bulkRollback(OID tableOid, - uint64_t lockID, - const std::string& tableName, - const std::string& applName, - bool debugConsole, std::string& errorMsg); - - /** - * @brief update SYSCOLUMN next value - * @param oidValueMap - */ - EXPORT int updateNextValue(const TxnID txnId, const OID& columnoid, const uint64_t nextVal, const uint32_t sessionID, const uint16_t dbRoot); - - /** - * @brief write active datafiles to disk - * - */ - EXPORT int flushDataFiles(int rc, const TxnID txnId, std::map& columnOids); - - /** - * @brief Process versioning for batch insert - only version the hwm block. - */ - EXPORT int processBatchVersions(const TxnID& txnid, std::vector columns, std::vector& rangeList); - - EXPORT void writeVBEnd(const TxnID& txnid, std::vector& rangeList); - - /************************************************************************ - * Future implementations - ************************************************************************/ - /** - * @brief Begin transaction - */ - // todo: add implementation when we work on version control - // int beginTran(const TransID transOid) { return NO_ERROR; } - - /** - * @brief End transaction - */ - // todo: add implementation when we work on version control - // int endTran(const TransID transOid) { return NO_ERROR; } - // WIP - void setDebugLevel(const DebugLevel level) - { - WEObj::setDebugLevel(level); - - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setDebugLevel(level); - m_dctnry[i]->setDebugLevel(level); - } - } // todo: cleanup - - /************************************************************************ - * Internal use definitions - ************************************************************************/ -private: - /** - * @brief Check whether the passing parameters are valid - */ - int checkValid(const TxnID& txnid, const ColStructList& colStructList, const ColValueList& colValueList, const RIDList& ridList) const; - - /** + /** * @brief Find the smallest column for this table */ - void findSmallestColumn(uint32_t &colId, ColStructList colStructList); + void findSmallestColumn(uint32_t& colId, ColStructList colStructList); - /** - * @brief Convert interface column type to an internal column type - */ - void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); + /** + * @brief Convert interface column type to an internal column type + */ + void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, + void* valArray, size_t pos, boost::any& data, bool fromList = true); - /** - * @brief Convert column value to its internal representation - * - * @param colType Column data-type - * @param value Memory pointer for storing output value. Should be pre-allocated - * @param data Column data - */ - void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, const ColType colType, void* value, boost::any& data); + /** + * @brief Convert column value to its internal representation + * + * @param colType Column data-type + * @param value Memory pointer for storing output value. Should be pre-allocated + * @param data Column data + */ + void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, const ColType colType, + void* value, boost::any& data); - /** - * @brief Print input value from DDL/DML processors - */ - void printInputValue(const ColStructList& colStructList, const ColValueList& colValueList, const RIDList& ridList, const DctnryStructList& dctnryStructList, const DictStrList& dictStrList) const; + /** + * @brief Print input value from DDL/DML processors + */ + void printInputValue(const ColStructList& colStructList, const ColValueList& colValueList, + const RIDList& ridList, const DctnryStructList& dctnryStructList, + const DictStrList& dictStrList) const; - /** - * @brief Process version buffer - */ - int processVersionBuffer(IDBDataFile* pFile, const TxnID& txnid, const ColStruct& colStruct, - int width, int totalRow, const RID* rowIdArray, - std::vector& rangeList); + /** + * @brief Process version buffer + */ + int processVersionBuffer(IDBDataFile* pFile, const TxnID& txnid, const ColStruct& colStruct, int width, + int totalRow, const RID* rowIdArray, std::vector& rangeList); - /** - * @brief Process version buffers for update and delete @Bug 1886,2870 - */ - int processVersionBuffers(IDBDataFile* pFile, const TxnID& txnid, const ColStruct& colStruct, - int width, int totalRow, const RIDList& ridList, - std::vector& rangeList); + /** + * @brief Process version buffers for update and delete @Bug 1886,2870 + */ + int processVersionBuffers(IDBDataFile* pFile, const TxnID& txnid, const ColStruct& colStruct, int width, + int totalRow, const RIDList& ridList, std::vector& rangeList); - int processBeginVBCopy(const TxnID& txnid, const std::vector& colStructList, const RIDList& ridList, - std::vector& freeList, std::vector >& fboLists, - std::vector >& rangeLists, std::vector& rangeListTot); + int processBeginVBCopy(const TxnID& txnid, const std::vector& colStructList, + const RIDList& ridList, std::vector& freeList, + std::vector >& fboLists, + std::vector >& rangeLists, + std::vector& rangeListTot); - - /** - * @brief Common methods to write values to a column - */ - int writeColumnRec(const TxnID& txnid, - const CSCTypesList& cscColTypes, - const ColStructList& colStructList, - ColValueList& colValueList, - RID* rowIdArray, const ColStructList& newColStructList, - ColValueList& newColValueList, const int32_t tableOid, - bool useTmpSuffix, bool versioning = true, - ColSplitMaxMinInfoList* maxMins = NULL); + /** + * @brief Common methods to write values to a column + */ + int writeColumnRec(const TxnID& txnid, const CSCTypesList& cscColTypes, const ColStructList& colStructList, + ColValueList& colValueList, RID* rowIdArray, const ColStructList& newColStructList, + ColValueList& newColValueList, const int32_t tableOid, bool useTmpSuffix, + bool versioning = true, ColSplitMaxMinInfoList* maxMins = NULL); - int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, - std::vector& colValueList, - RID* rowIdArray, const ColStructList& newColStructList, - std::vector& newColValueList, - const int32_t tableOid, - bool useTmpSuffix, bool versioning = true); + int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, + std::vector& colValueList, RID* rowIdArray, + const ColStructList& newColStructList, std::vector& newColValueList, + const int32_t tableOid, bool useTmpSuffix, bool versioning = true); - //@Bug 1886,2870 pass the address of ridList vector - int writeColumnRecUpdate(const TxnID& txnid, - const CSCTypesList& cscColTypeList, - const ColStructList& colStructList, - const ColValueList& colValueList, std::vector& colOldValueList, - const RIDList& ridList, const int32_t tableOid, - bool convertStructFlag = true, ColTupleList::size_type nRows = 0, std::vector* cpInfos = NULL); + //@Bug 1886,2870 pass the address of ridList vector + int writeColumnRecUpdate(const TxnID& txnid, const CSCTypesList& cscColTypeList, + const ColStructList& colStructList, const ColValueList& colValueList, + std::vector& colOldValueList, const RIDList& ridList, + const int32_t tableOid, bool convertStructFlag = true, + ColTupleList::size_type nRows = 0, std::vector* cpInfos = NULL); - //For update column from column to use - int writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList, - std::vector& colStructList, - ColValueList& colValueList, const RIDList& ridLists, - const int32_t tableOid, bool versioning = true, std::vector* cpInfos = NULL); + // For update column from column to use + int writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList, + std::vector& colStructList, ColValueList& colValueList, + const RIDList& ridLists, const int32_t tableOid, bool versioning = true, + std::vector* cpInfos = NULL); - /** - * @brief util method to convert rowid to a column file - * - */ - int convertRidToColumn(RID& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, - const RID filesPerColumnPartition, const RID extentsPerSegmentFile, - const RID extentRows, uint16_t startDBRoot, unsigned dbrootCnt); + /** + * @brief util method to convert rowid to a column file + * + */ + int convertRidToColumn(RID& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, + const RID filesPerColumnPartition, const RID extentsPerSegmentFile, + const RID extentRows, uint16_t startDBRoot, unsigned dbrootCnt); - void AddDictToList(const TxnID txnid, std::vector& lbids); - void RemoveTxnFromDictMap(const TxnID txnid); + void AddDictToList(const TxnID txnid, std::vector& lbids); + void RemoveTxnFromDictMap(const TxnID txnid); - // Bug 4312: We use a hash map to hold the set of starting LBIDS for a given - // txn so that we don't waste time marking the same extent as invalid. This - // list should be trimmed if it gets too big. - int AddLBIDtoList(const TxnID txnid, - const ColStruct& colStruct, - const int fbo, - ExtCPInfo* cpInfo = NULL // provide CPInfo pointer if you want max/min updated. - ); + // Bug 4312: We use a hash map to hold the set of starting LBIDS for a given + // txn so that we don't waste time marking the same extent as invalid. This + // list should be trimmed if it gets too big. + int AddLBIDtoList(const TxnID txnid, const ColStruct& colStruct, const int fbo, + ExtCPInfo* cpInfo = NULL // provide CPInfo pointer if you want max/min updated. + ); - // Get CPInfo for given starting LBID and column description structure. - int GetLBIDRange(const BRM::LBID_t startingLBID, const ColStruct& colStruct, ExtCPInfo& cpInfo); + // Get CPInfo for given starting LBID and column description structure. + int GetLBIDRange(const BRM::LBID_t startingLBID, const ColStruct& colStruct, ExtCPInfo& cpInfo); - // mark extents of the transaction as invalid. erase transaction from txn->lbidsrec map if requested. - int markTxnExtentsAsInvalid(const TxnID txnid, bool erase = false); + // mark extents of the transaction as invalid. erase transaction from txn->lbidsrec map if requested. + int markTxnExtentsAsInvalid(const TxnID txnid, bool erase = false); - // write LBID's new ranges. - int setExtentsNewMaxMins(const ColSplitMaxMinInfoList& maxMins, bool haveSplit); - - int RemoveTxnFromLBIDMap(const TxnID txnid); + // write LBID's new ranges. + int setExtentsNewMaxMins(const ColSplitMaxMinInfoList& maxMins, bool haveSplit); - int op(int compressionType) + int RemoveTxnFromLBIDMap(const TxnID txnid); + + int op(int compressionType) + { + switch (compressionType) { - switch (compressionType) - { - case 1: - case 2: - return COMPRESSED_OP_1; - case 3: - return COMPRESSED_OP_2; - } - - return 0; + case 1: + case 2: return COMPRESSED_OP_1; + case 3: return COMPRESSED_OP_2; } + return 0; + } - // This is a Map of sets of LBIDS for each transaction. A Transaction's list will be removed upon commit or rollback. - std::tr1::unordered_map m_txnLBIDMap; + // This is a Map of sets of LBIDS for each transaction. A Transaction's list will be removed upon commit or + // rollback. + std::tr1::unordered_map m_txnLBIDMap; - // MCOL-1160: We need to track dictionary LBIDs so we can tell PrimProc - // to flush the blocks after an API bulk-write. - std::tr1::unordered_map m_dictLBIDMap; + // MCOL-1160: We need to track dictionary LBIDs so we can tell PrimProc + // to flush the blocks after an API bulk-write. + std::tr1::unordered_map m_dictLBIDMap; - ColumnOp* m_colOp[TOTAL_COMPRESS_OP]; // column operations - Dctnry* m_dctnry[TOTAL_COMPRESS_OP]; // dictionary operations - OpType m_opType; // operation type + ColumnOp* m_colOp[TOTAL_COMPRESS_OP]; // column operations + Dctnry* m_dctnry[TOTAL_COMPRESS_OP]; // dictionary operations + OpType m_opType; // operation type }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WRITE_ENGINE_H_ +#endif // _WRITE_ENGINE_H_ diff --git a/writeengine/xml/txml.cpp b/writeengine/xml/txml.cpp index e4ddbe7a5..fdd6d467e 100644 --- a/writeengine/xml/txml.cpp +++ b/writeengine/xml/txml.cpp @@ -32,61 +32,54 @@ using namespace WriteEngine; class XmlTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(XmlTest); + CPPUNIT_TEST(test1); - CPPUNIT_TEST_SUITE( XmlTest ); + // XML basic testing + CPPUNIT_TEST(testBasicXMLRead); + /*CPPUNIT_TEST( testAddTreeNode ); + // Index tree testing + CPPUNIT_TEST( testTreeGetTestbitValue ); + */ + CPPUNIT_TEST_SUITE_END(); - CPPUNIT_TEST( test1 ); + private: + public: + void setUp() + { + } -// XML basic testing - CPPUNIT_TEST( testBasicXMLRead ); - /*CPPUNIT_TEST( testAddTreeNode ); + void tearDown() + { + } - // Index tree testing - CPPUNIT_TEST( testTreeGetTestbitValue ); - */ - CPPUNIT_TEST_SUITE_END(); + void test1() + { + } + void testBasicXMLRead() + { + int rc; + XMLJob myJob; -private: - -public: - void setUp() - { - } - - void tearDown() - { - } - - void test1() - { - } - void testBasicXMLRead() - { - int rc; - XMLJob myJob; - - rc = myJob.loadJobXmlFile( "../test/bulk/job/Job_127.xml" ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - - myJob.printJobInfo(); - } + rc = myJob.loadJobXmlFile("../test/bulk/job/Job_127.xml"); + CPPUNIT_ASSERT(rc == NO_ERROR); + myJob.printJobInfo(); + } }; -CPPUNIT_TEST_SUITE_REGISTRATION( XmlTest ); +CPPUNIT_TEST_SUITE_REGISTRATION(XmlTest); #include #include -int main( int argc, char** argv) +int main(int argc, char** argv) { - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest( registry.makeTest() ); - bool wasSuccessful = runner.run( "", false ); - return (wasSuccessful ? 0 : 1); + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); } - - diff --git a/writeengine/xml/we_xmlgendata.cpp b/writeengine/xml/we_xmlgendata.cpp index b3a200615..e8ce4c1f6 100644 --- a/writeengine/xml/we_xmlgendata.cpp +++ b/writeengine/xml/we_xmlgendata.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_xmlgendata.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_xmlgendata.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ #include "we_xmlgendata.h" @@ -33,71 +33,69 @@ const std::string JOBDIR("job"); namespace WriteEngine { -/* static */ const std::string XMLGenData::DELIMITER ("-d"); -/* static */ const std::string XMLGenData::DESCRIPTION ("-s"); +/* static */ const std::string XMLGenData::DELIMITER("-d"); +/* static */ const std::string XMLGenData::DESCRIPTION("-s"); #ifdef _MSC_VER __declspec(dllexport) #endif -/* static */ const std::string XMLGenData::ENCLOSED_BY_CHAR ("-E"); + /* static */ const std::string XMLGenData::ENCLOSED_BY_CHAR("-E"); #ifdef _MSC_VER __declspec(dllexport) #endif -/* static */ const std::string XMLGenData::ESCAPE_CHAR ("-C"); + /* static */ const std::string XMLGenData::ESCAPE_CHAR("-C"); #ifdef _MSC_VER __declspec(dllexport) #endif -/* static */ const std::string XMLGenData::JOBID ("-j"); -/* static */ const std::string XMLGenData::MAXERROR ("-e"); -/* static */ const std::string XMLGenData::NAME ("-n"); -/* static */ const std::string XMLGenData::PATH ("-p"); + /* static */ const std::string XMLGenData::JOBID("-j"); +/* static */ const std::string XMLGenData::MAXERROR("-e"); +/* static */ const std::string XMLGenData::NAME("-n"); +/* static */ const std::string XMLGenData::PATH("-p"); #ifdef _MSC_VER __declspec(dllexport) #endif -/* static */ const std::string XMLGenData::RPT_DEBUG ("-b"); -/* static */ const std::string XMLGenData::USER ("-u"); -/* static */ const std::string XMLGenData::NO_OF_READ_BUFFER ("-r"); + /* static */ const std::string XMLGenData::RPT_DEBUG("-b"); +/* static */ const std::string XMLGenData::USER("-u"); +/* static */ const std::string XMLGenData::NO_OF_READ_BUFFER("-r"); /* static */ const std::string XMLGenData::READ_BUFFER_CAPACITY("-c"); -/* static */ const std::string XMLGenData::WRITE_BUFFER_SIZE ("-w"); -/* static */ const std::string XMLGenData::EXT ("-x"); +/* static */ const std::string XMLGenData::WRITE_BUFFER_SIZE("-w"); +/* static */ const std::string XMLGenData::EXT("-x"); //------------------------------------------------------------------------------ // XMLGenData constructor // Omit inserting JOBID; derived class is required to insert //------------------------------------------------------------------------------ -XMLGenData::XMLGenData( ) +XMLGenData::XMLGenData() { - fParms.insert(ParmList::value_type(DELIMITER, std::string("|"))); - fParms.insert(ParmList::value_type(DESCRIPTION, std::string())); - fParms.insert(ParmList::value_type(ENCLOSED_BY_CHAR, std::string(""))); - fParms.insert(ParmList::value_type(ESCAPE_CHAR, std::string("\\"))); - fParms.insert(ParmList::value_type(JOBID, std::string("299"))); - fParms.insert(ParmList::value_type(MAXERROR, std::string("10"))); - fParms.insert(ParmList::value_type(NAME, std::string())); + fParms.insert(ParmList::value_type(DELIMITER, std::string("|"))); + fParms.insert(ParmList::value_type(DESCRIPTION, std::string())); + fParms.insert(ParmList::value_type(ENCLOSED_BY_CHAR, std::string(""))); + fParms.insert(ParmList::value_type(ESCAPE_CHAR, std::string("\\"))); + fParms.insert(ParmList::value_type(JOBID, std::string("299"))); + fParms.insert(ParmList::value_type(MAXERROR, std::string("10"))); + fParms.insert(ParmList::value_type(NAME, std::string())); #ifdef _MSC_VER - std::string br; - br = Config::getBulkRoot(); - boost::filesystem::path p(br); + std::string br; + br = Config::getBulkRoot(); + boost::filesystem::path p(br); #else - boost::filesystem::path p{ std::string(Config::getBulkRoot()) }; + boost::filesystem::path p{std::string(Config::getBulkRoot())}; #endif - p /= JOBDIR; - fParms.insert(ParmList::value_type(PATH, p.string())); + p /= JOBDIR; + fParms.insert(ParmList::value_type(PATH, p.string())); - fParms.insert(ParmList::value_type(RPT_DEBUG, std::string("0"))); - fParms.insert(ParmList::value_type(USER, std::string())); - fParms.insert(ParmList::value_type(NO_OF_READ_BUFFER, std::string("5"))); - fParms.insert(ParmList::value_type(READ_BUFFER_CAPACITY, - std::string("1048576"))); - fParms.insert(ParmList::value_type(WRITE_BUFFER_SIZE, - std::string("10485760"))); - fParms.insert(ParmList::value_type(EXT, std::string("tbl"))); + fParms.insert(ParmList::value_type(RPT_DEBUG, std::string("0"))); + fParms.insert(ParmList::value_type(USER, std::string())); + fParms.insert(ParmList::value_type(NO_OF_READ_BUFFER, std::string("5"))); + fParms.insert(ParmList::value_type(READ_BUFFER_CAPACITY, std::string("1048576"))); + fParms.insert(ParmList::value_type(WRITE_BUFFER_SIZE, std::string("10485760"))); + fParms.insert(ParmList::value_type(EXT, std::string("tbl"))); } //------------------------------------------------------------------------------ // XMLGenData destructor //------------------------------------------------------------------------------ /* virtual */ -XMLGenData::~XMLGenData( ) +XMLGenData::~XMLGenData() { } @@ -106,12 +104,12 @@ XMLGenData::~XMLGenData( ) //------------------------------------------------------------------------------ std::string XMLGenData::getParm(const std::string& key) const { - ParmList::const_iterator p = fParms.find(key); + ParmList::const_iterator p = fParms.find(key); - if (fParms.end() != p) - return p->second; - else - return ""; + if (fParms.end() != p) + return p->second; + else + return ""; } //------------------------------------------------------------------------------ @@ -122,4 +120,4 @@ void XMLGenData::print(std::ostream& /* os */) const { } -} //namespace WriteEngine +} // namespace WriteEngine diff --git a/writeengine/xml/we_xmlgendata.h b/writeengine/xml/we_xmlgendata.h index e68eca04d..ef0fe81c9 100644 --- a/writeengine/xml/we_xmlgendata.h +++ b/writeengine/xml/we_xmlgendata.h @@ -37,7 +37,6 @@ namespace WriteEngine { - /** @brief Base class for storing input data used to generate a Job XML file. * * This class represents common code refactored out of inputmgr.h, under @@ -46,77 +45,77 @@ namespace WriteEngine */ class XMLGenData { -public: - typedef std::vector TableList; - typedef std::map ParmList; - typedef std::vector LoadNames; + public: + typedef std::vector TableList; + typedef std::map ParmList; + typedef std::vector LoadNames; - // Valid parms that can be stored and retrieved from XMLGenData - EXPORT const static std::string DELIMITER; - EXPORT const static std::string DESCRIPTION; + // Valid parms that can be stored and retrieved from XMLGenData + EXPORT const static std::string DELIMITER; + EXPORT const static std::string DESCRIPTION; #if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT) - __declspec(dllimport) + __declspec(dllimport) #endif - EXPORT const static std::string ENCLOSED_BY_CHAR; + EXPORT const static std::string ENCLOSED_BY_CHAR; #if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT) - __declspec(dllimport) + __declspec(dllimport) #endif - EXPORT const static std::string ESCAPE_CHAR; + EXPORT const static std::string ESCAPE_CHAR; #if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT) - __declspec(dllimport) + __declspec(dllimport) #endif - EXPORT const static std::string JOBID; - EXPORT const static std::string MAXERROR; - EXPORT const static std::string NAME; - EXPORT const static std::string PATH; + EXPORT const static std::string JOBID; + EXPORT const static std::string MAXERROR; + EXPORT const static std::string NAME; + EXPORT const static std::string PATH; #if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT) - __declspec(dllimport) + __declspec(dllimport) #endif - EXPORT const static std::string RPT_DEBUG; - EXPORT const static std::string USER; - EXPORT const static std::string NO_OF_READ_BUFFER; - EXPORT const static std::string READ_BUFFER_CAPACITY; - EXPORT const static std::string WRITE_BUFFER_SIZE; - EXPORT const static std::string EXT; + EXPORT const static std::string RPT_DEBUG; + EXPORT const static std::string USER; + EXPORT const static std::string NO_OF_READ_BUFFER; + EXPORT const static std::string READ_BUFFER_CAPACITY; + EXPORT const static std::string WRITE_BUFFER_SIZE; + EXPORT const static std::string EXT; - /** @brief XMLGenData constructor - */ - EXPORT XMLGenData(); + /** @brief XMLGenData constructor + */ + EXPORT XMLGenData(); - /** @brief XMLGenData destructor - */ - EXPORT virtual ~XMLGenData(); + /** @brief XMLGenData destructor + */ + EXPORT virtual ~XMLGenData(); - /** @brief Print contents of this object to the specified stream. - */ - EXPORT virtual void print(std::ostream& os) const; + /** @brief Print contents of this object to the specified stream. + */ + EXPORT virtual void print(std::ostream& os) const; - EXPORT std::string getParm(const std::string& key) const; - const TableList& getTables() const - { - return fTables; - } - const std::string& getSchema() const - { - return fSchema; - } - const LoadNames& getLoadNames() const - { - return fLoadNames; - } + EXPORT std::string getParm(const std::string& key) const; + const TableList& getTables() const + { + return fTables; + } + const std::string& getSchema() const + { + return fSchema; + } + const LoadNames& getLoadNames() const + { + return fLoadNames; + } -protected: - TableList fTables; - ParmList fParms; - std::string fSchema; - LoadNames fLoadNames; + protected: + TableList fTables; + ParmList fParms; + std::string fSchema; + LoadNames fLoadNames; -private: - XMLGenData(const XMLGenData&); // disable default copy ctor - XMLGenData& operator=(const XMLGenData&); // disable default assignment + private: + XMLGenData(const XMLGenData&); // disable default copy ctor + XMLGenData& operator=(const XMLGenData&); // disable default assignment }; -} +} // namespace WriteEngine #undef EXPORT diff --git a/writeengine/xml/we_xmlgenproc.cpp b/writeengine/xml/we_xmlgenproc.cpp index a50fa3ecd..3fca53e50 100644 --- a/writeengine/xml/we_xmlgenproc.cpp +++ b/writeengine/xml/we_xmlgenproc.cpp @@ -37,42 +37,42 @@ using namespace execplan; namespace { -const char* DICT_TYPE("D"); -const char* ENCODING("UTF-8"); -const char* LOGNAME("Jobxml_"); -const char* JOBNAME("Job_"); -} +const char* DICT_TYPE("D"); +const char* ENCODING("UTF-8"); +const char* LOGNAME("Jobxml_"); +const char* JOBNAME("Job_"); +} // namespace namespace WriteEngine { - //------------------------------------------------------------------------------ // XMLGen constructor //------------------------------------------------------------------------------ -XMLGenProc::XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt) : - fLog(), - fDoc(), - fWriter(), - fErrorString("XMLGen encountered exception, abnormal exit " - "and file not created.\nCheck error log at:\t"), - fDebugLevel(0), - fInputMgr(mgr), - fSysCatRpt(bSysCatRpt), - fUseXmlLogFile(bUseXmlLogFile) +XMLGenProc::XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt) + : fLog() + , fDoc() + , fWriter() + , fErrorString( + "XMLGen encountered exception, abnormal exit " + "and file not created.\nCheck error log at:\t") + , fDebugLevel(0) + , fInputMgr(mgr) + , fSysCatRpt(bSysCatRpt) + , fUseXmlLogFile(bUseXmlLogFile) { - std::string logFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME + - fInputMgr->getParm(XMLGenData::JOBID) + ".log" ); - std::string errFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME + - fInputMgr->getParm(XMLGenData::JOBID) + ".err" ); - fErrorString.append(errFile + "\n"); + std::string logFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME + + fInputMgr->getParm(XMLGenData::JOBID) + ".log"); + std::string errFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME + + fInputMgr->getParm(XMLGenData::JOBID) + ".err"); + fErrorString.append(errFile + "\n"); - if (fUseXmlLogFile) - { - fLog.setLogFileName( logFile.c_str(), errFile.c_str() ); - std::ostringstream ss; - fInputMgr->print( ss ); - fLog.logMsg(ss.str(), MSGLVL_INFO1 ); - } + if (fUseXmlLogFile) + { + fLog.setLogFileName(logFile.c_str(), errFile.c_str()); + std::ostringstream ss; + fInputMgr->print(ss); + fLog.logMsg(ss.str(), MSGLVL_INFO1); + } } //------------------------------------------------------------------------------ @@ -80,303 +80,267 @@ XMLGenProc::XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt) : //------------------------------------------------------------------------------ XMLGenProc::~XMLGenProc() { - xmlFreeDoc(fDoc); + xmlFreeDoc(fDoc); } - + //------------------------------------------------------------------------------ // startXMLFile // Creates xmlDocPtr (fDoc) that we will use in generating our XML file. // Create XML header section. //------------------------------------------------------------------------------ -void XMLGenProc::startXMLFile( ) +void XMLGenProc::startXMLFile() { - fWriter = xmlNewTextWriterDoc(&fDoc, 0); + fWriter = xmlNewTextWriterDoc(&fDoc, 0); - if (fWriter == NULL) + if (fWriter == NULL) + { + throw std::runtime_error( + "Error creating the xml fWriter: " + "bad return from xmlNewTextWriter"); + } + + /* Start the fDocument with the xml default for the version, + * encoding UTF-8 and the default for the standalone + * declaration. */ + int rc = xmlTextWriterStartDocument(fWriter, NULL, ENCODING, NULL); + + if (rc < 0) + { + throw std::runtime_error( + "Error at xmlTextWriterStartfDocument: " + "bad return from xmlTextWriterStartDocument"); + } + + if (!fSysCatRpt) // skip non-syscat tags if we are writing a syscat dump + { + xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_BULK_JOB]); + xmlTextWriterWriteFormatElement(fWriter, BAD_CAST xmlTagTable[TAG_ID], "%d", + atoi(fInputMgr->getParm(XMLGenData::JOBID).c_str())); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_NAME], + BAD_CAST fInputMgr->getParm(XMLGenData::NAME).c_str()); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_DESC], + BAD_CAST fInputMgr->getParm(XMLGenData::DESCRIPTION).c_str()); + + std::string now(boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_CREATE_DATE], + BAD_CAST now.substr(0, 8).c_str()); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_CREATE_TIME], + BAD_CAST now.substr(9, 4).c_str()); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_USER], + BAD_CAST fInputMgr->getParm(XMLGenData::USER).c_str()); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_DELIMITER], + BAD_CAST fInputMgr->getParm(XMLGenData::DELIMITER).c_str()); + + // Only include enclosedBy and escape chars if enclosedBy was specified + std::string enclosedByChar = fInputMgr->getParm(XMLGenData::ENCLOSED_BY_CHAR); + + if (enclosedByChar.length() > 0) { - throw std::runtime_error("Error creating the xml fWriter: " - "bad return from xmlNewTextWriter"); + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_ENCLOSED_BY_CHAR], + BAD_CAST fInputMgr->getParm(XMLGenData::ENCLOSED_BY_CHAR).c_str()); } - /* Start the fDocument with the xml default for the version, - * encoding UTF-8 and the default for the standalone - * declaration. */ - int rc = xmlTextWriterStartDocument(fWriter, NULL, ENCODING, NULL); + // Include escape character regardless of whether the "enclosed by" + // character is given, because the escape character can still be used + // to override the default NULL escape sequence '\N', to be something + // else like '#N'. + xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_ESCAPE_CHAR], + BAD_CAST fInputMgr->getParm(XMLGenData::ESCAPE_CHAR).c_str()); - if (rc < 0) - { - throw std::runtime_error("Error at xmlTextWriterStartfDocument: " - "bad return from xmlTextWriterStartDocument"); - } + // Added new tags for configurable parameters + xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_READ_BUFFERS]); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_NO_OF_READ_BUFFERS], "%d", + atoi(fInputMgr->getParm(XMLGenData::NO_OF_READ_BUFFER).c_str())); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_READ_BUFFER_SIZE], "%d", + atoi(fInputMgr->getParm(XMLGenData::READ_BUFFER_CAPACITY).c_str())); + xmlTextWriterEndElement(fWriter); + xmlTextWriterWriteFormatElement(fWriter, BAD_CAST xmlTagTable[TAG_WRITE_BUFFER_SIZE], "%d", + atoi(fInputMgr->getParm(XMLGenData::WRITE_BUFFER_SIZE).c_str())); + // End of additions + } - if (!fSysCatRpt) // skip non-syscat tags if we are writing a syscat dump - { - xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_BULK_JOB]); - xmlTextWriterWriteFormatElement(fWriter, BAD_CAST xmlTagTable[TAG_ID], - "%d", atoi(fInputMgr->getParm(XMLGenData::JOBID).c_str())); - xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_NAME], - BAD_CAST fInputMgr->getParm(XMLGenData::NAME).c_str() ); - xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_DESC], - BAD_CAST fInputMgr->getParm(XMLGenData::DESCRIPTION).c_str() ); - - std::string now(boost::posix_time::to_iso_string( - boost::posix_time::second_clock::local_time())); - xmlTextWriterWriteElement(fWriter, - BAD_CAST xmlTagTable[TAG_CREATE_DATE], - BAD_CAST now.substr(0, 8).c_str() ); - xmlTextWriterWriteElement(fWriter, - BAD_CAST xmlTagTable[TAG_CREATE_TIME], - BAD_CAST now.substr(9, 4).c_str() ); - xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_USER], - BAD_CAST fInputMgr->getParm(XMLGenData::USER).c_str() ); - xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_DELIMITER], - BAD_CAST fInputMgr->getParm(XMLGenData::DELIMITER).c_str() ); - - // Only include enclosedBy and escape chars if enclosedBy was specified - std::string enclosedByChar = fInputMgr->getParm( - XMLGenData::ENCLOSED_BY_CHAR); - - if (enclosedByChar.length() > 0) - { - xmlTextWriterWriteElement(fWriter, - BAD_CAST xmlTagTable[TAG_ENCLOSED_BY_CHAR], - BAD_CAST fInputMgr->getParm( - XMLGenData::ENCLOSED_BY_CHAR).c_str() ); - } - - // Include escape character regardless of whether the "enclosed by" - // character is given, because the escape character can still be used - // to override the default NULL escape sequence '\N', to be something - // else like '#N'. - xmlTextWriterWriteElement(fWriter, - BAD_CAST xmlTagTable[TAG_ESCAPE_CHAR], - BAD_CAST fInputMgr->getParm(XMLGenData::ESCAPE_CHAR).c_str() ); - - // Added new tags for configurable parameters - xmlTextWriterStartElement(fWriter, - BAD_CAST xmlTagTable[TAG_READ_BUFFERS]); - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_NO_OF_READ_BUFFERS], "%d", - atoi(fInputMgr->getParm(XMLGenData::NO_OF_READ_BUFFER).c_str())); - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_READ_BUFFER_SIZE], "%d", - atoi(fInputMgr->getParm(XMLGenData::READ_BUFFER_CAPACITY).c_str())); - xmlTextWriterEndElement(fWriter); - xmlTextWriterWriteFormatElement(fWriter, - BAD_CAST xmlTagTable[TAG_WRITE_BUFFER_SIZE], "%d", - atoi( fInputMgr->getParm(XMLGenData::WRITE_BUFFER_SIZE).c_str())); - // End of additions - } - - xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_SCHEMA]); - xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_NAME], - BAD_CAST fInputMgr->getSchema().c_str() ); + xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_SCHEMA]); + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_NAME], + BAD_CAST fInputMgr->getSchema().c_str()); } - + //------------------------------------------------------------------------------ // makeTableData // Create XML tag for a table. //------------------------------------------------------------------------------ void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table) { - static unsigned kount; + static unsigned kount; - xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_TABLE]); - std::string tmp(table.schema + "." + table.table); - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_TBL_NAME], BAD_CAST tmp.c_str() ); + xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_TABLE]); + std::string tmp(table.schema + "." + table.table); + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_NAME], BAD_CAST tmp.c_str()); - if (fSysCatRpt) // Write full schema information for syscat rpt + if (fSysCatRpt) // Write full schema information for syscat rpt + { + try { - try - { - boost::shared_ptr cat = - CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - cat->identity(CalpontSystemCatalog::EC); - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_TBL_OID], "%d", - cat->tableRID(table).objnum); - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Error getting OID for table " << - table.schema << '.' << table.table << ": " << ex.what(); - throw std::runtime_error( oss.str() ); - } - catch (...) - { - std::ostringstream oss; - oss << "Unknown error getting OID for table " << - table.schema << '.' << table.table; - throw std::runtime_error( oss.str() ); - } + boost::shared_ptr cat = + CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(CalpontSystemCatalog::EC); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%d", + cat->tableRID(table).objnum); + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error getting OID for table " << table.schema << '.' << table.table << ": " << ex.what(); + throw std::runtime_error(oss.str()); + } + catch (...) + { + std::ostringstream oss; + oss << "Unknown error getting OID for table " << table.schema << '.' << table.table; + throw std::runtime_error(oss.str()); + } + } + + if (!fSysCatRpt) // skip non-syscat tags if we are writing a syscat dump + { + const XMLGenData::LoadNames& loadNames = fInputMgr->getLoadNames(); + + if (loadNames.size() > kount) + { + tmp = loadNames[kount]; + } + else + { + tmp = (table.table + "." + fInputMgr->getParm(XMLGenData::EXT).c_str()); } - if (!fSysCatRpt) // skip non-syscat tags if we are writing a syscat dump - { - const XMLGenData::LoadNames& loadNames = fInputMgr->getLoadNames(); + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_LOAD_NAME], BAD_CAST tmp.c_str()); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_MAX_ERR_ROW], "%d", + atoi(fInputMgr->getParm(XMLGenData::MAXERROR).c_str())); + } - if ( loadNames.size() > kount ) - { - tmp = loadNames[kount]; - } - else - { - tmp = (table.table + "." + fInputMgr->getParm( - XMLGenData::EXT).c_str()); - } - - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_LOAD_NAME], BAD_CAST tmp.c_str() ); - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_MAX_ERR_ROW], "%d", - atoi( fInputMgr->getParm(XMLGenData::MAXERROR).c_str()) ); - } - - kount++; + kount++; } - + //------------------------------------------------------------------------------ // sortColumnsByPosition // Sort list of columns by column position. //------------------------------------------------------------------------------ void XMLGenProc::sortColumnsByPosition(SysCatColumnList& columns) { - std::map tempCols; + std::map tempCols; - SysCatColumnList::const_iterator cend = columns.end(); + SysCatColumnList::const_iterator cend = columns.end(); - for (SysCatColumnList::const_iterator col = columns.begin(); - col != cend; ++col) - { - tempCols[col->colType.colPosition] = *col ; - } + for (SysCatColumnList::const_iterator col = columns.begin(); col != cend; ++col) + { + tempCols[col->colType.colPosition] = *col; + } - columns.clear(); + columns.clear(); - std::map::iterator pos; + std::map::iterator pos; - for (pos = tempCols.begin(); pos != tempCols.end(); ++pos) - { - columns.push_back(pos->second); - } + for (pos = tempCols.begin(); pos != tempCols.end(); ++pos) + { + columns.push_back(pos->second); + } - tempCols.clear(); + tempCols.clear(); } - + //------------------------------------------------------------------------------ // makeColumnData // Create XML tag for the columns in a table. //------------------------------------------------------------------------------ bool XMLGenProc::makeColumnData(const CalpontSystemCatalog::TableName& table) { - SysCatColumnList columns; - getColumnsForTable(table.schema, table.table, columns); - sortColumnsByPosition(columns); + SysCatColumnList columns; + getColumnsForTable(table.schema, table.table, columns); + sortColumnsByPosition(columns); - if (columns.empty()) + if (columns.empty()) + { + if (fUseXmlLogFile) { - if (fUseXmlLogFile) + fLog.logMsg("No columns for " + table.table + ", or table does not exist", MSGLVL_ERROR); + } + + return false; + } + + SysCatColumnList::const_iterator cend = columns.end(); + + for (SysCatColumnList::const_iterator col = columns.begin(); col != cend; ++col) + { + xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_COLUMN]); + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_COL_NAME], + BAD_CAST col->tableColName.column.c_str()); + + if (fSysCatRpt) // Write full schema information for syscat rpt + { + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_COL_OID], "%d", col->oid); + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_DATA_TYPE], + BAD_CAST ColDataTypeStr[col->colType.colDataType]); + + if (col->colType.compressionType != CalpontSystemCatalog::NO_COMPRESSION) + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_COMPRESS_TYPE], "%d", + col->colType.compressionType); + + // Old logic went by scale > 0; New logic checks for "decimal" type + if ((0 < col->colType.scale) || (col->colType.colDataType == CalpontSystemCatalog::DECIMAL) || + (col->colType.colDataType == CalpontSystemCatalog::UDECIMAL)) + { + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_PRECISION], "%d", + col->colType.precision); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_SCALE], "%d", col->colType.scale); + } + + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_WIDTH], "%d", + col->colType.colWidth); + + if (col->colType.autoincrement) + { + int autoInc = 1; + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_AUTOINCREMENT_FLAG], "%d", + autoInc); + } + + // need dictionary and decimal stuff + if (col->colType.ddn.dictOID > 0) + { + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_COL_TYPE], BAD_CAST DICT_TYPE); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_DVAL_OID], "%d", + col->colType.ddn.dictOID); + } + + // Include NotNull and Default value + const std::string col_defaultValue(col->colType.defaultValue); + + if (col->colType.constraintType == execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + int notNull = 1; + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_NOT_NULL], "%d", notNull); + + if (!col_defaultValue.empty()) { - fLog.logMsg("No columns for " + table.table + - ", or table does not exist", MSGLVL_ERROR ); + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_DEFAULT_VALUE], + BAD_CAST col_defaultValue.c_str()); } + } + else if (col->colType.constraintType == execplan::CalpontSystemCatalog::DEFAULT_CONSTRAINT) + { + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_DEFAULT_VALUE], + BAD_CAST col_defaultValue.c_str()); + } + } // end of "if fSysCatRpt" - return false; - } + xmlTextWriterEndElement(fWriter); + } - SysCatColumnList::const_iterator cend = columns.end(); - - for (SysCatColumnList::const_iterator col = columns.begin(); - col != cend; ++col) - { - xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_COLUMN]); - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_COL_NAME], - BAD_CAST col->tableColName.column.c_str()); - - if (fSysCatRpt) // Write full schema information for syscat rpt - { - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_COL_OID], "%d", col->oid); - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_DATA_TYPE], - BAD_CAST ColDataTypeStr[ - col->colType.colDataType]); - - if (col->colType.compressionType != - CalpontSystemCatalog::NO_COMPRESSION) - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_COMPRESS_TYPE], "%d", - col->colType.compressionType); - - // Old logic went by scale > 0; New logic checks for "decimal" type - if ( (0 < col->colType.scale ) || - (col->colType.colDataType == CalpontSystemCatalog::DECIMAL) || - (col->colType.colDataType == CalpontSystemCatalog::UDECIMAL) ) - { - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_PRECISION], "%d", - col->colType.precision); - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_SCALE], "%d", col->colType.scale); - } - - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_WIDTH], "%d", col->colType.colWidth); - - if (col->colType.autoincrement) - { - int autoInc = 1; - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_AUTOINCREMENT_FLAG], "%d", autoInc); - } - - //need dictionary and decimal stuff - if (col->colType.ddn.dictOID > 0) - { - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_COL_TYPE], BAD_CAST DICT_TYPE ); - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_DVAL_OID], "%d", - col->colType.ddn.dictOID ); - } - - // Include NotNull and Default value - const std::string col_defaultValue(col->colType.defaultValue); - - if (col->colType.constraintType == - execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) - { - int notNull = 1; - xmlTextWriterWriteFormatAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_NOT_NULL], "%d", notNull); - - if (!col_defaultValue.empty()) - { - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_DEFAULT_VALUE], - BAD_CAST col_defaultValue.c_str()); - } - } - else if (col->colType.constraintType == - execplan::CalpontSystemCatalog::DEFAULT_CONSTRAINT) - { - xmlTextWriterWriteAttribute(fWriter, - BAD_CAST xmlTagTable[TAG_DEFAULT_VALUE], - BAD_CAST col_defaultValue.c_str()); - } - } // end of "if fSysCatRpt" - - xmlTextWriterEndElement(fWriter); - } - - xmlTextWriterEndElement(fWriter); //table - return true; + xmlTextWriterEndElement(fWriter); // table + return true; } - + //------------------------------------------------------------------------------ // getColumnsForTable // Access the system catalog in order to get the OID, column type, and column @@ -387,112 +351,104 @@ bool XMLGenProc::makeColumnData(const CalpontSystemCatalog::TableName& table) // decided to implement the functionality in XMLGen in order to eliminate the // dependency on ddlpackageprocessor. //------------------------------------------------------------------------------ -void XMLGenProc::getColumnsForTable( - const std::string& schema, - const std::string& table, - SysCatColumnList& colList) +void XMLGenProc::getColumnsForTable(const std::string& schema, const std::string& table, + SysCatColumnList& colList) { - CalpontSystemCatalog::TableName tableName; - tableName.schema = schema; - tableName.table = table; + CalpontSystemCatalog::TableName tableName; + tableName.schema = schema; + tableName.table = table; - try + try + { + boost::shared_ptr systemCatalogPtr = + CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + systemCatalogPtr->identity(CalpontSystemCatalog::EC); + + const CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true); + + CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin(); + + while (rid_iterator != ridList.end()) { - boost::shared_ptr systemCatalogPtr = - CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - systemCatalogPtr->identity(CalpontSystemCatalog::EC); + CalpontSystemCatalog::ROPair roPair = *rid_iterator; - const CalpontSystemCatalog::RIDList ridList = - systemCatalogPtr->columnRIDs(tableName, true); + SysCatColumn column; + column.oid = roPair.objnum; + column.colType = systemCatalogPtr->colType(column.oid); + column.tableColName = systemCatalogPtr->colName(column.oid); - CalpontSystemCatalog::RIDList::const_iterator rid_iterator = - ridList.begin(); + colList.push_back(column); - while (rid_iterator != ridList.end()) - { - CalpontSystemCatalog::ROPair roPair = *rid_iterator; - - SysCatColumn column; - column.oid = roPair.objnum; - column.colType = systemCatalogPtr->colType(column.oid); - column.tableColName = systemCatalogPtr->colName(column.oid); - - colList.push_back(column); - - ++rid_iterator; - } - } - catch (std::exception& ex) - { - std::ostringstream oss; - oss << "Error reading columns for table " << - schema << '.' << table << ": " << ex.what(); - throw std::runtime_error( oss.str() ); - } - catch (...) - { - std::ostringstream oss; - oss << "Unknown error reading columns for table " << - schema << '.' << table; - throw std::runtime_error( oss.str() ); + ++rid_iterator; } + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error reading columns for table " << schema << '.' << table << ": " << ex.what(); + throw std::runtime_error(oss.str()); + } + catch (...) + { + std::ostringstream oss; + oss << "Unknown error reading columns for table " << schema << '.' << table; + throw std::runtime_error(oss.str()); + } } //------------------------------------------------------------------------------ // Generate Job XML File Name //------------------------------------------------------------------------------ -std::string XMLGenProc::genJobXMLFileName( ) const +std::string XMLGenProc::genJobXMLFileName() const { - std::string xmlFileName; - boost::filesystem::path p(std::string( - fInputMgr->getParm(XMLGenData::PATH))); + std::string xmlFileName; + boost::filesystem::path p(std::string(fInputMgr->getParm(XMLGenData::PATH))); - //Append the jobname, jobid & file extension - std::string fileName( JOBNAME ); - fileName += fInputMgr->getParm(XMLGenData::JOBID); - fileName += ".xml"; - p /= fileName; + // Append the jobname, jobid & file extension + std::string fileName(JOBNAME); + fileName += fInputMgr->getParm(XMLGenData::JOBID); + fileName += ".xml"; + p /= fileName; - //If the filespec doesn't begin with a '/' (i.e. it's not an absolute path), - // attempt to make it absolute so that we can log the full pathname. + // If the filespec doesn't begin with a '/' (i.e. it's not an absolute path), + // attempt to make it absolute so that we can log the full pathname. #ifdef _MSC_VER - //We won't worry about being so fancy in Windows, just print a relative - // path if so given - xmlFileName = p.string(); + // We won't worry about being so fancy in Windows, just print a relative + // path if so given + xmlFileName = p.string(); #else - if (!p.has_root_path()) - { - char cwdPath[4096]; - char *buf; - buf = getcwd(cwdPath, sizeof(cwdPath)); - if (buf == NULL) - throw std::runtime_error("Failed to get the current working directory!"); - boost::filesystem::path p2(cwdPath); - p2 /= p; - xmlFileName = p2.string(); - } - else - { - xmlFileName = p.string(); - } + if (!p.has_root_path()) + { + char cwdPath[4096]; + char* buf; + buf = getcwd(cwdPath, sizeof(cwdPath)); + if (buf == NULL) + throw std::runtime_error("Failed to get the current working directory!"); + boost::filesystem::path p2(cwdPath); + p2 /= p; + xmlFileName = p2.string(); + } + else + { + xmlFileName = p.string(); + } #endif - return xmlFileName; + return xmlFileName; } //------------------------------------------------------------------------------ // writeXMLFile //------------------------------------------------------------------------------ -void XMLGenProc::writeXMLFile( const std::string& xmlFileName ) +void XMLGenProc::writeXMLFile(const std::string& xmlFileName) { - xmlTextWriterEndDocument(fWriter); - xmlFreeTextWriter(fWriter); + xmlTextWriterEndDocument(fWriter); + xmlFreeTextWriter(fWriter); - xmlSaveFormatFile(xmlFileName.c_str(), fDoc, 1); + xmlSaveFormatFile(xmlFileName.c_str(), fDoc, 1); } //------------------------------------------------------------------------------ @@ -500,10 +456,10 @@ void XMLGenProc::writeXMLFile( const std::string& xmlFileName ) //------------------------------------------------------------------------------ void XMLGenProc::logErrorMessage(const std::string& msg) { - if (fUseXmlLogFile) - { - fLog.logMsg( msg, MSGLVL_ERROR ); - } + if (fUseXmlLogFile) + { + fLog.logMsg(msg, MSGLVL_ERROR); + } } -} // namespace bulkloadxml +} // namespace WriteEngine diff --git a/writeengine/xml/we_xmlgenproc.h b/writeengine/xml/we_xmlgenproc.h index 4b7ec4ed5..241ca763a 100644 --- a/writeengine/xml/we_xmlgenproc.h +++ b/writeengine/xml/we_xmlgenproc.h @@ -42,9 +42,9 @@ namespace WriteEngine { struct SysCatColumn { - execplan::CalpontSystemCatalog::OID oid; - execplan::CalpontSystemCatalog::ColType colType; - execplan::CalpontSystemCatalog::TableColName tableColName; + execplan::CalpontSystemCatalog::OID oid; + execplan::CalpontSystemCatalog::ColType colType; + execplan::CalpontSystemCatalog::TableColName tableColName; }; typedef std::vector SysCatColumnList; @@ -58,83 +58,78 @@ class XMLGenData; */ class XMLGenProc { -public: + public: + /** @brief XMLGenProc constructor + * + * @param mgr The input data used to generate a Job XML file. + * @param bUseXmlLogFile Log info/errors to Job XML log file. + * @param bSysCatRpt Generating SysCat report (true) or XML file (false) + */ + EXPORT XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt); + EXPORT ~XMLGenProc(); - /** @brief XMLGenProc constructor - * - * @param mgr The input data used to generate a Job XML file. - * @param bUseXmlLogFile Log info/errors to Job XML log file. - * @param bSysCatRpt Generating SysCat report (true) or XML file (false) - */ - EXPORT XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt); - EXPORT ~XMLGenProc(); + /** @brief start constructing XML file document. + */ + EXPORT void startXMLFile(); - /** @brief start constructing XML file document. - */ - EXPORT void startXMLFile( ); + /** @brief Creates table tag for the specified table. + * + * @param table Name of table for which the table tag is to be generated. + */ + EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table); - /** @brief Creates table tag for the specified table. - * - * @param table Name of table for which the table tag is to be generated. - */ - EXPORT void makeTableData( - const execplan::CalpontSystemCatalog::TableName& table); + /** @brief Creates column tags for the specified table. + * + * @param table Name of table for which the column tags are to be generated. + * @return true means column tags created; else false is returned + */ + EXPORT bool makeColumnData(const execplan::CalpontSystemCatalog::TableName& table); - /** @brief Creates column tags for the specified table. - * - * @param table Name of table for which the column tags are to be generated. - * @return true means column tags created; else false is returned - */ - EXPORT bool makeColumnData( - const execplan::CalpontSystemCatalog::TableName& table); + /** @brief Generate Job XML file name + */ + EXPORT std::string genJobXMLFileName() const; - /** @brief Generate Job XML file name - */ - EXPORT std::string genJobXMLFileName( ) const; + /** @brief Write xml file document to the destination Job XML file. + * + * @param xmlFileName Name of XML file to be generated. + */ + EXPORT void writeXMLFile(const std::string& xmlFileName); - /** @brief Write xml file document to the destination Job XML file. - * - * @param xmlFileName Name of XML file to be generated. - */ - EXPORT void writeXMLFile( const std::string& xmlFileName ); + /** @brief log a message. + * + * @param msg The message to be logged to the error log file. + */ + EXPORT void logErrorMessage(const std::string& msg); + std::string errorString() + { + return fErrorString; + } - /** @brief log a message. - * - * @param msg The message to be logged to the error log file. - */ - EXPORT void logErrorMessage(const std::string& msg); - std::string errorString() - { - return fErrorString; - } + /** @brief set debug level + */ + void setDebugLevel(int dbg) + { + fDebugLevel = dbg; + } - /** @brief set debug level - */ - void setDebugLevel( int dbg ) - { - fDebugLevel = dbg; - } + protected: + private: + XMLGenProc(const XMLGenProc&); // disable default copy ctor + XMLGenProc& operator=(const XMLGenProc&); // disable default assignment + void getColumnsForTable(const std::string& schema, const std::string& table, SysCatColumnList& colList); + void sortColumnsByPosition(SysCatColumnList& columns); -protected: - -private: - XMLGenProc(const XMLGenProc&); // disable default copy ctor - XMLGenProc& operator=(const XMLGenProc&); // disable default assignment - void getColumnsForTable(const std::string& schema, - const std::string& table, SysCatColumnList& colList); - void sortColumnsByPosition(SysCatColumnList& columns); - - Log fLog; - xmlDocPtr fDoc; - xmlTextWriterPtr fWriter; - std::string fErrorString; - int fDebugLevel; - XMLGenData* fInputMgr; // Input data used to generate Job XML file - bool fSysCatRpt; // True colxml output or a syscat report - bool fUseXmlLogFile;//Log info/errors to Job XML log file + Log fLog; + xmlDocPtr fDoc; + xmlTextWriterPtr fWriter; + std::string fErrorString; + int fDebugLevel; + XMLGenData* fInputMgr; // Input data used to generate Job XML file + bool fSysCatRpt; // True colxml output or a syscat report + bool fUseXmlLogFile; // Log info/errors to Job XML log file }; -} // namespace WriteEngine +} // namespace WriteEngine #undef EXPORT diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 257b50cf7..0cd44b458 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_xmljob.cpp 4579 2013-03-19 23:16:54Z dhall $ -* -*******************************************************************************/ + * $Id: we_xmljob.cpp 4579 2013-03-19 23:16:54Z dhall $ + * + *******************************************************************************/ /** @file */ #define WRITEENGINEXMLJOB_DLLEXPORT @@ -44,41 +44,34 @@ using namespace std; using namespace execplan; - namespace WriteEngine { // Maximum saturation value for DECIMAL types based on precision // TODO MCOL-641 add support here. see dataconvert.cpp -const long long columnstore_precision[19] = -{ - 0, - 9, - 99, - 999, - 9999, - 99999, - 999999, - 9999999, - 99999999, - 999999999, - 9999999999LL, - 99999999999LL, - 999999999999LL, - 9999999999999LL, - 99999999999999LL, - 999999999999999LL, - 9999999999999999LL, - 99999999999999999LL, - 999999999999999999LL -}; - +const long long columnstore_precision[19] = {0, + 9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999LL, + 99999999999LL, + 999999999999LL, + 9999999999999LL, + 99999999999999LL, + 999999999999999LL, + 9999999999999999LL, + 99999999999999999LL, + 999999999999999999LL}; + //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -XMLJob::XMLJob( ) : fDebugLevel( DEBUG_0 ), - fDeleteTempFile(false), - fValidateColList(true), - fTimeZone("SYSTEM") +XMLJob::XMLJob() : fDebugLevel(DEBUG_0), fDeleteTempFile(false), fValidateColList(true), fTimeZone("SYSTEM") { } @@ -88,12 +81,12 @@ XMLJob::XMLJob( ) : fDebugLevel( DEBUG_0 ), //------------------------------------------------------------------------------ XMLJob::~XMLJob() { - if ((fDeleteTempFile) && (!fJobFileName.empty())) - { - unlink( fJobFileName.c_str() ); - } + if ((fDeleteTempFile) && (!fJobFileName.empty())) + { + unlink(fJobFileName.c_str()); + } } - + //------------------------------------------------------------------------------ // Load a job xml file // fileName - name of file to load @@ -101,281 +94,270 @@ XMLJob::~XMLJob() // bValidateColumnList - validate that all db columns have an XML tag // returns NO_ERROR if success; other if fail //------------------------------------------------------------------------------ -int XMLJob::loadJobXmlFile( const string& fileName, - bool bTempFile, - bool bValidateColumnList, - string& errMsg ) +int XMLJob::loadJobXmlFile(const string& fileName, bool bTempFile, bool bValidateColumnList, string& errMsg) { - int rc; + int rc; - fDeleteTempFile = bTempFile; - fJobFileName = fileName; - fValidateColList = bValidateColumnList; + fDeleteTempFile = bTempFile; + fJobFileName = fileName; + fValidateColList = bValidateColumnList; - try - { - rc = parseDoc( fileName.c_str() ); + try + { + rc = parseDoc(fileName.c_str()); - if (rc != NO_ERROR) - return rc; - } - catch (exception& ex) - { - errMsg = ex.what(); - return ERR_XML_PARSE; - } + if (rc != NO_ERROR) + return rc; + } + catch (exception& ex) + { + errMsg = ex.what(); + return ERR_XML_PARSE; + } - return rc; + return rc; } - + //------------------------------------------------------------------------------ // Print contents of fJob to the specified logger object. // logger - Log object to use in logging //------------------------------------------------------------------------------ -void XMLJob::printJobInfo( Log& logger ) const +void XMLJob::printJobInfo(Log& logger) const { - const Job& job = fJob; + const Job& job = fJob; - ostringstream oss1; - oss1 << "Job " << job.id << " input\n"; - oss1 << "===============================================" << endl; - oss1 << "Name : " << job.name << endl; - oss1 << "Desc : " << job.desc << endl; - oss1 << "User : " << job.userName << endl; - oss1 << "Delim: " << job.fDelimiter << endl; - oss1 << "Enclosed By : "; + ostringstream oss1; + oss1 << "Job " << job.id << " input\n"; + oss1 << "===============================================" << endl; + oss1 << "Name : " << job.name << endl; + oss1 << "Desc : " << job.desc << endl; + oss1 << "User : " << job.userName << endl; + oss1 << "Delim: " << job.fDelimiter << endl; + oss1 << "Enclosed By : "; - if (job.fEnclosedByChar) - oss1 << job.fEnclosedByChar << endl; - else - oss1 << "n/a" << endl; + if (job.fEnclosedByChar) + oss1 << job.fEnclosedByChar << endl; + else + oss1 << "n/a" << endl; - oss1 << "Escape Char : "; + oss1 << "Escape Char : "; - if (job.fEscapeChar) - oss1 << job.fEscapeChar << endl; - else - oss1 << "n/a" << endl; + if (job.fEscapeChar) + oss1 << job.fEscapeChar << endl; + else + oss1 << "n/a" << endl; - oss1 << "Read Buffers: " << job.numberOfReadBuffers << endl; - oss1 << "Read Buffer Size: " << job.readBufferSize << endl; - oss1 << "setvbuf Size: " << job.writeBufferSize << endl; - oss1 << "Create Date : " << job.createDate << endl; - oss1 << "Create Time : " << job.createTime << endl; - oss1 << "Schema Name : " << job.schema << endl; + oss1 << "Read Buffers: " << job.numberOfReadBuffers << endl; + oss1 << "Read Buffer Size: " << job.readBufferSize << endl; + oss1 << "setvbuf Size: " << job.writeBufferSize << endl; + oss1 << "Create Date : " << job.createDate << endl; + oss1 << "Create Time : " << job.createTime << endl; + oss1 << "Schema Name : " << job.schema << endl; - oss1 << "Num Tables : " << job.jobTableList.size() << endl; - logger.logMsg( oss1.str(), MSGLVL_INFO2 ); + oss1 << "Num Tables : " << job.jobTableList.size() << endl; + logger.logMsg(oss1.str(), MSGLVL_INFO2); - for ( unsigned int i = 0; i < job.jobTableList.size(); i++ ) + for (unsigned int i = 0; i < job.jobTableList.size(); i++) + { + const JobTable& jobTable = job.jobTableList[i]; + ostringstream oss2; + oss2 << "\n-------------------------------------------------" << endl; + oss2 << "\tTable Name : " << jobTable.tblName << endl; + oss2 << "\tTable OID : " << jobTable.mapOid << endl; + oss2 << "\tTable Load Name : " << jobTable.loadFileName << endl; + oss2 << "\tMax Err Num : " << jobTable.maxErrNum << endl; + + const JobColList& colList = jobTable.colList; + + oss2 << "\tNum of Columns : " << colList.size() << endl; + logger.logMsg(oss2.str(), MSGLVL_INFO2); + + // Note that we don't print JobColumn.dataType because it is not carried + // in the XML file. dataType is assigned/used internally by bulkload. + for (unsigned int j = 0; j < jobTable.fFldRefs.size(); j++) { - const JobTable& jobTable = job.jobTableList[i]; - ostringstream oss2; - oss2 << "\n-------------------------------------------------" << endl; - oss2 << "\tTable Name : " << jobTable.tblName << endl; - oss2 << "\tTable OID : " << jobTable.mapOid << endl; - oss2 << "\tTable Load Name : " << jobTable.loadFileName << - endl; - oss2 << "\tMax Err Num : " << jobTable.maxErrNum << endl; + unsigned idx = jobTable.fFldRefs[j].fArrayIndex; + BulkFldColRel fldColType = jobTable.fFldRefs[j].fFldColType; + const JobColumn& jobCol = + ((fldColType == BULK_FLDCOL_IGNORE_FIELD) ? jobTable.fIgnoredFields[idx] : jobTable.colList[idx]); + ostringstream oss3; + oss3 << "\n\t****************************************" << endl; - const JobColList& colList = jobTable.colList; + if (fldColType == BULK_FLDCOL_COLUMN_DEFAULT) + oss3 << "\t\tDefaultColumn Name: " << jobCol.colName << endl; + else + oss3 << "\t\tColumn Name : " << jobCol.colName << endl; - oss2 << "\tNum of Columns : " << colList.size() << endl; - logger.logMsg( oss2.str(), MSGLVL_INFO2 ); + oss3 << "\t\tColumn OID : " << jobCol.mapOid << endl; + oss3 << "\t\tColumn type name : " << jobCol.typeName << endl; + oss3 << "\t\tColumn width : " << jobCol.width << endl; + oss3 << "\t\tColumn Not Null : " << jobCol.fNotNull << endl; + oss3 << "\t\tColumn WithDefault: " << jobCol.fWithDefault << endl; + oss3 << "\t\tColumn type : " << jobCol.colType << endl; + oss3 << "\t\tColumn comp type : " << jobCol.compressionType << endl; + oss3 << "\t\tColumn autoInc : " << jobCol.autoIncFlag << endl; - // Note that we don't print JobColumn.dataType because it is not carried - // in the XML file. dataType is assigned/used internally by bulkload. - for ( unsigned int j = 0; j < jobTable.fFldRefs.size(); j++ ) - { - unsigned idx = jobTable.fFldRefs[j].fArrayIndex; - BulkFldColRel fldColType = jobTable.fFldRefs[j].fFldColType; - const JobColumn& jobCol = ((fldColType == BULK_FLDCOL_IGNORE_FIELD) ? - jobTable.fIgnoredFields[idx] : - jobTable.colList[idx] ); - ostringstream oss3; - oss3 << "\n\t****************************************" << endl; + if (jobCol.typeName == ColDataTypeStr[CalpontSystemCatalog::DECIMAL]) + { + oss3 << "\t\tColumn Precision : " << jobCol.precision << endl; + oss3 << "\t\tColumn Scale : " << jobCol.scale << endl; + } - if (fldColType == BULK_FLDCOL_COLUMN_DEFAULT) - oss3 << "\t\tDefaultColumn Name: " << jobCol.colName << endl; - else - oss3 << "\t\tColumn Name : " << jobCol.colName << endl; + if (jobCol.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL]) + { + oss3 << "\t\tColumn Precision : " << jobCol.precision << endl; + oss3 << "\t\tColumn Scale : " << jobCol.scale << endl; + } - oss3 << "\t\tColumn OID : " << jobCol.mapOid << endl; - oss3 << "\t\tColumn type name : " << jobCol.typeName << endl; - oss3 << "\t\tColumn width : " << jobCol.width << endl; - oss3 << "\t\tColumn Not Null : " << jobCol.fNotNull << endl; - oss3 << "\t\tColumn WithDefault: " << jobCol.fWithDefault << endl; - oss3 << "\t\tColumn type : " << jobCol.colType << endl; - oss3 << "\t\tColumn comp type : " << jobCol.compressionType << endl; - oss3 << "\t\tColumn autoInc : " << jobCol.autoIncFlag << endl; + if (jobCol.colType == 'D') + { + oss3 << "\t\tDictionary Oid : " << jobCol.dctnry.dctnryOid << endl; + } - if ( jobCol.typeName == ColDataTypeStr[CalpontSystemCatalog::DECIMAL] ) - { - oss3 << "\t\tColumn Precision : " << jobCol.precision << endl; - oss3 << "\t\tColumn Scale : " << jobCol.scale << endl; - } - - if ( jobCol.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL] ) - { - oss3 << "\t\tColumn Precision : " << jobCol.precision << endl; - oss3 << "\t\tColumn Scale : " << jobCol.scale << endl; - } - - if ( jobCol.colType == 'D' ) - { - oss3 << "\t\tDictionary Oid : " << - jobCol.dctnry.dctnryOid << endl; - } - - logger.logMsg( oss3.str(), MSGLVL_INFO2 ); - } // end of loop through columns in a table - } // end of loop through tables + logger.logMsg(oss3.str(), MSGLVL_INFO2); + } // end of loop through columns in a table + } // end of loop through tables } - + //------------------------------------------------------------------------------ // Print brief contents of specified Job to specified logger object. // logger - Log object to use in logging //------------------------------------------------------------------------------ -void XMLJob::printJobInfoBrief( Log& logger ) const +void XMLJob::printJobInfoBrief(Log& logger) const { - const Job& job = fJob; + const Job& job = fJob; - ostringstream oss1; - oss1 << "XMLJobFile: Delim(" << job.fDelimiter << "); EnclosedBy("; + ostringstream oss1; + oss1 << "XMLJobFile: Delim(" << job.fDelimiter << "); EnclosedBy("; - if (job.fEnclosedByChar) - oss1 << job.fEnclosedByChar; - else - oss1 << "n/a"; + if (job.fEnclosedByChar) + oss1 << job.fEnclosedByChar; + else + oss1 << "n/a"; - oss1 << "); EscapeChar("; + oss1 << "); EscapeChar("; - if (job.fEscapeChar) - oss1 << job.fEscapeChar; - else - oss1 << "n/a"; + if (job.fEscapeChar) + oss1 << job.fEscapeChar; + else + oss1 << "n/a"; - oss1 << "); ReadBufs(" << job.numberOfReadBuffers << - "); ReadBufSize(" << job.readBufferSize << - "); setvbufSize(" << job.writeBufferSize << ')'; - logger.logMsg( oss1.str(), MSGLVL_INFO2 ); + oss1 << "); ReadBufs(" << job.numberOfReadBuffers << "); ReadBufSize(" << job.readBufferSize + << "); setvbufSize(" << job.writeBufferSize << ')'; + logger.logMsg(oss1.str(), MSGLVL_INFO2); - for ( unsigned int i = 0; i < job.jobTableList.size(); i++ ) + for (unsigned int i = 0; i < job.jobTableList.size(); i++) + { + const JobTable& jobTable = job.jobTableList[i]; + ostringstream oss2; + oss2 << " Table(" << jobTable.tblName << "); OID(" << jobTable.mapOid << ')' << "; MaxErrNum(" + << jobTable.maxErrNum << ')'; + logger.logMsg(oss2.str(), MSGLVL_INFO2); + + for (unsigned int j = 0; j < jobTable.fFldRefs.size(); j++) { - const JobTable& jobTable = job.jobTableList[i]; - ostringstream oss2; - oss2 << " Table(" << jobTable.tblName << - "); OID(" << jobTable.mapOid << ')' << - "; MaxErrNum(" << jobTable.maxErrNum << ')'; - logger.logMsg( oss2.str(), MSGLVL_INFO2 ); + unsigned idx = jobTable.fFldRefs[j].fArrayIndex; + BulkFldColRel fldColType = jobTable.fFldRefs[j].fFldColType; + const JobColumn& jobCol = + ((fldColType == BULK_FLDCOL_IGNORE_FIELD) ? jobTable.fIgnoredFields[idx] : jobTable.colList[idx]); + ostringstream oss3; - for ( unsigned int j = 0; j < jobTable.fFldRefs.size(); j++ ) - { - unsigned idx = jobTable.fFldRefs[j].fArrayIndex; - BulkFldColRel fldColType = jobTable.fFldRefs[j].fFldColType; - const JobColumn& jobCol = ((fldColType == BULK_FLDCOL_IGNORE_FIELD) ? - jobTable.fIgnoredFields[idx] : - jobTable.colList[idx]); - ostringstream oss3; + if (fldColType == BULK_FLDCOL_COLUMN_DEFAULT) + oss3 << " DefaultColumn(" << jobCol.colName; + else + oss3 << " Column(" << jobCol.colName; - if (fldColType == BULK_FLDCOL_COLUMN_DEFAULT) - oss3 << " DefaultColumn(" << jobCol.colName; - else - oss3 << " Column(" << jobCol.colName; + oss3 << "); OID(" << jobCol.mapOid << "); Type(" << jobCol.typeName << "); Width(" << jobCol.width + << "); Comp(" << jobCol.compressionType; - oss3 << "); OID(" << jobCol.mapOid << - "); Type(" << jobCol.typeName << - "); Width(" << jobCol.width << - "); Comp(" << jobCol.compressionType; + if (jobCol.colType == 'D') + oss3 << "); DctnryOid(" << jobCol.dctnry.dctnryOid; - if ( jobCol.colType == 'D' ) - oss3 << "); DctnryOid(" << jobCol.dctnry.dctnryOid; + oss3 << ')'; - oss3 << ')'; + if (jobCol.autoIncFlag) + oss3 << "; autoInc"; - if (jobCol.autoIncFlag) - oss3 << "; autoInc"; + if (jobCol.fNotNull) + oss3 << "; NotNull"; - if (jobCol.fNotNull) - oss3 << "; NotNull"; + if (jobCol.fWithDefault) + oss3 << "; WithDefault"; - if (jobCol.fWithDefault) - oss3 << "; WithDefault"; - - logger.logMsg( oss3.str(), MSGLVL_INFO2 ); - } - } // end of for( int i + logger.logMsg(oss3.str(), MSGLVL_INFO2); + } + } // end of for( int i } - + //------------------------------------------------------------------------------ // Process a node // pNode - current node // returns TRUE if success, FALSE otherwise //------------------------------------------------------------------------------ -bool XMLJob::processNode( xmlNode* pNode ) +bool XMLJob::processNode(xmlNode* pNode) { - if ( isTag( pNode, TAG_BULK_JOB )) - { - // no work for the BulkJob tag - } - else if ( isTag( pNode, TAG_CREATE_DATE )) - setJobData( pNode, TAG_CREATE_DATE, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_CREATE_TIME )) - setJobData( pNode, TAG_CREATE_TIME, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_COLUMN )) - setJobData( pNode, TAG_COLUMN, false, TYPE_EMPTY ); - else if ( isTag( pNode, TAG_DEFAULT_COLUMN )) - setJobData( pNode, TAG_DEFAULT_COLUMN, false, TYPE_EMPTY ); - else if ( isTag( pNode, TAG_DESC )) - setJobData( pNode, TAG_DESC, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_ID )) - setJobData( pNode, TAG_ID, true, TYPE_INT ); - else if ( isTag( pNode, TAG_IGNORE_FIELD )) - setJobData( pNode, TAG_IGNORE_FIELD, false, TYPE_EMPTY ); - else if ( isTag( pNode, TAG_NAME )) - setJobData( pNode, TAG_NAME, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_PATH )) - setJobData( pNode, TAG_PATH, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_TABLE )) - setJobData( pNode, TAG_TABLE, false, TYPE_EMPTY ); - else if ( isTag( pNode, TAG_TYPE )) - setJobData( pNode, TAG_TYPE, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_USER )) - setJobData( pNode, TAG_USER, true, TYPE_CHAR ); - else if ( isTag( pNode, TAG_SCHEMA)) - setJobData( pNode, TAG_SCHEMA, false, TYPE_EMPTY ); - else if ( isTag( pNode, TAG_READ_BUFFERS)) - setJobData( pNode, TAG_READ_BUFFERS, false, TYPE_EMPTY ); - else if ( isTag( pNode, TAG_WRITE_BUFFER_SIZE)) - setJobData( pNode, TAG_WRITE_BUFFER_SIZE, true, TYPE_INT); - else if ( isTag( pNode, TAG_DELIMITER)) - setJobData( pNode, TAG_DELIMITER, true, TYPE_CHAR); - else if ( isTag( pNode, TAG_ENCLOSED_BY_CHAR)) - setJobData( pNode, TAG_ENCLOSED_BY_CHAR, true, TYPE_CHAR); - else if ( isTag( pNode, TAG_ESCAPE_CHAR)) - setJobData( pNode, TAG_ESCAPE_CHAR, true, TYPE_CHAR); - else - { - ostringstream oss; - oss << "Unrecognized TAG in Job XML file: <" << pNode->name << ">"; - throw runtime_error( oss.str() ); - } + if (isTag(pNode, TAG_BULK_JOB)) + { + // no work for the BulkJob tag + } + else if (isTag(pNode, TAG_CREATE_DATE)) + setJobData(pNode, TAG_CREATE_DATE, true, TYPE_CHAR); + else if (isTag(pNode, TAG_CREATE_TIME)) + setJobData(pNode, TAG_CREATE_TIME, true, TYPE_CHAR); + else if (isTag(pNode, TAG_COLUMN)) + setJobData(pNode, TAG_COLUMN, false, TYPE_EMPTY); + else if (isTag(pNode, TAG_DEFAULT_COLUMN)) + setJobData(pNode, TAG_DEFAULT_COLUMN, false, TYPE_EMPTY); + else if (isTag(pNode, TAG_DESC)) + setJobData(pNode, TAG_DESC, true, TYPE_CHAR); + else if (isTag(pNode, TAG_ID)) + setJobData(pNode, TAG_ID, true, TYPE_INT); + else if (isTag(pNode, TAG_IGNORE_FIELD)) + setJobData(pNode, TAG_IGNORE_FIELD, false, TYPE_EMPTY); + else if (isTag(pNode, TAG_NAME)) + setJobData(pNode, TAG_NAME, true, TYPE_CHAR); + else if (isTag(pNode, TAG_PATH)) + setJobData(pNode, TAG_PATH, true, TYPE_CHAR); + else if (isTag(pNode, TAG_TABLE)) + setJobData(pNode, TAG_TABLE, false, TYPE_EMPTY); + else if (isTag(pNode, TAG_TYPE)) + setJobData(pNode, TAG_TYPE, true, TYPE_CHAR); + else if (isTag(pNode, TAG_USER)) + setJobData(pNode, TAG_USER, true, TYPE_CHAR); + else if (isTag(pNode, TAG_SCHEMA)) + setJobData(pNode, TAG_SCHEMA, false, TYPE_EMPTY); + else if (isTag(pNode, TAG_READ_BUFFERS)) + setJobData(pNode, TAG_READ_BUFFERS, false, TYPE_EMPTY); + else if (isTag(pNode, TAG_WRITE_BUFFER_SIZE)) + setJobData(pNode, TAG_WRITE_BUFFER_SIZE, true, TYPE_INT); + else if (isTag(pNode, TAG_DELIMITER)) + setJobData(pNode, TAG_DELIMITER, true, TYPE_CHAR); + else if (isTag(pNode, TAG_ENCLOSED_BY_CHAR)) + setJobData(pNode, TAG_ENCLOSED_BY_CHAR, true, TYPE_CHAR); + else if (isTag(pNode, TAG_ESCAPE_CHAR)) + setJobData(pNode, TAG_ESCAPE_CHAR, true, TYPE_CHAR); + else + { + ostringstream oss; + oss << "Unrecognized TAG in Job XML file: <" << pNode->name << ">"; + throw runtime_error(oss.str()); + } - if (XMLOp::processNode( pNode )) + if (XMLOp::processNode(pNode)) + { + if (isTag(pNode, TAG_TABLE)) { - if ( isTag( pNode, TAG_TABLE )) - { - postProcessTableNode(); - } - } - else - { - return false; + postProcessTableNode(); } + } + else + { + return false; + } - return true; + return true; } - + //------------------------------------------------------------------------------ // Generic setter // pNode - current node @@ -383,163 +365,130 @@ bool XMLJob::processNode( xmlNode* pNode ) // bExpectContent - should node content be present to process // tagType - data type //------------------------------------------------------------------------------ -void XMLJob::setJobData( xmlNode* pNode, - const xmlTag tag, - bool bExpectContent, - XML_DTYPE tagType ) +void XMLJob::setJobData(xmlNode* pNode, const xmlTag tag, bool bExpectContent, XML_DTYPE tagType) { - int intVal = 0; - long long llVal = 0; - std::string bufString; - bool bSuccess = false; + int intVal = 0; + long long llVal = 0; + std::string bufString; + bool bSuccess = false; - if (bExpectContent) + if (bExpectContent) + { + if (tagType == TYPE_INT) + bSuccess = getNodeContent(pNode, &intVal, TYPE_INT); + else // longlong + if (tagType == TYPE_LONGLONG) + bSuccess = getNodeContent(pNode, &llVal, TYPE_LONGLONG); + else // char + if (tagType == TYPE_CHAR) + bSuccess = getNodeContentStr(pNode, bufString); + + if (!bSuccess) + return; + } + + // process tag content and attributes + switch (tag) + { + case TAG_READ_BUFFERS: setReadBuffers(pNode); break; + + case TAG_COLUMN: setJobDataColumn(pNode, false); break; + + case TAG_CREATE_DATE: fJob.createDate = bufString; break; + + case TAG_CREATE_TIME: fJob.createTime = bufString; break; + + case TAG_DEFAULT_COLUMN: setJobDataColumn(pNode, true); break; + + case TAG_DESC: fJob.desc = bufString; break; + + case TAG_ID: fJob.id = intVal; break; + + case TAG_IGNORE_FIELD: setJobDataIgnoreField(); break; + + case TAG_NAME: fJob.name = bufString; break; + + case TAG_PATH: + // no action necessary, but keep for backwards compatability + break; + + case TAG_TABLE: setJobDataTable(pNode); break; + + case TAG_TYPE: + // no action necessary, but keep for backwards compatability + break; + + case TAG_USER: fJob.userName = bufString; break; + + case TAG_SCHEMA: setSchema(pNode); break; + + case TAG_WRITE_BUFFER_SIZE: fJob.writeBufferSize = intVal; break; + + case TAG_DELIMITER: { - if ( tagType == TYPE_INT ) - bSuccess = getNodeContent( pNode, &intVal, TYPE_INT ); - else // longlong - if ( tagType == TYPE_LONGLONG ) - bSuccess = getNodeContent( pNode, &llVal, TYPE_LONGLONG ); - else // char - if ( tagType == TYPE_CHAR ) - bSuccess = getNodeContentStr( pNode, bufString ); + const char* buf = bufString.c_str(); - if (!bSuccess) - return; + if ((!strcmp(buf, "\\t")) || (!strcmp(buf, "'\\t'"))) + { + fJob.fDelimiter = '\t'; + } + else + { + fJob.fDelimiter = bufString[0]; + } + + break; } - // process tag content and attributes - switch ( tag ) + case TAG_ENCLOSED_BY_CHAR: { - case TAG_READ_BUFFERS: - setReadBuffers( pNode ); - break; - - case TAG_COLUMN: - setJobDataColumn( pNode, false ); - break; - - case TAG_CREATE_DATE: - fJob.createDate = bufString; - break; - - case TAG_CREATE_TIME: - fJob.createTime = bufString; - break; - - case TAG_DEFAULT_COLUMN: - setJobDataColumn( pNode, true ); - break; - - case TAG_DESC: - fJob.desc = bufString; - break; - - case TAG_ID: - fJob.id = intVal; - break; - - case TAG_IGNORE_FIELD: - setJobDataIgnoreField( ); - break; - - case TAG_NAME: - fJob.name = bufString; - break; - - case TAG_PATH: - // no action necessary, but keep for backwards compatability - break; - - case TAG_TABLE: - setJobDataTable( pNode ); - break; - - case TAG_TYPE: - // no action necessary, but keep for backwards compatability - break; - - case TAG_USER: - fJob.userName = bufString; - break; - - case TAG_SCHEMA: - setSchema( pNode ); - break; - - case TAG_WRITE_BUFFER_SIZE: - fJob.writeBufferSize = intVal; - break; - - case TAG_DELIMITER: - { - const char* buf = bufString.c_str(); - - if ((!strcmp(buf, "\\t")) || - (!strcmp(buf, "'\\t'"))) - { - fJob.fDelimiter = '\t'; - } - else - { - fJob.fDelimiter = bufString[0]; - } - - break; - } - - case TAG_ENCLOSED_BY_CHAR: - { - fJob.fEnclosedByChar = bufString[0]; - break; - } - - case TAG_ESCAPE_CHAR: - { - fJob.fEscapeChar = bufString[0]; - break; - } - - default: - break; + fJob.fEnclosedByChar = bufString[0]; + break; } + + case TAG_ESCAPE_CHAR: + { + fJob.fEscapeChar = bufString[0]; + break; + } + + default: break; + } } - + //------------------------------------------------------------------------------ // Set table information parms. // pNode - current node //------------------------------------------------------------------------------ -void XMLJob::setJobDataTable( xmlNode* pNode ) +void XMLJob::setJobDataTable(xmlNode* pNode) { - int intVal; - std::string bufString; - JobTable curTable; + int intVal; + std::string bufString; + JobTable curTable; - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_ORIG_NAME], bufString ) ) - curTable.tblName = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_ORIG_NAME], bufString)) + curTable.tblName = bufString; - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_TBL_NAME], bufString ) ) - curTable.tblName = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_TBL_NAME], bufString)) + curTable.tblName = bufString; - if (curTable.tblName.empty()) - { - throw runtime_error( - "Required table name attribute (tblName) missing from Table tag"); - } + if (curTable.tblName.empty()) + { + throw runtime_error("Required table name attribute (tblName) missing from Table tag"); + } - if ( getNodeAttribute( pNode, xmlTagTable[TAG_TBL_OID], &intVal, TYPE_INT ) ) - curTable.mapOid = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_TBL_OID], &intVal, TYPE_INT)) + curTable.mapOid = intVal; - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_LOAD_NAME], bufString ) ) - curTable.loadFileName = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_LOAD_NAME], bufString)) + curTable.loadFileName = bufString; - if ( getNodeAttribute( pNode, xmlTagTable[TAG_MAX_ERR_ROW], &intVal, - TYPE_INT)) - curTable.maxErrNum = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_MAX_ERR_ROW], &intVal, TYPE_INT)) + curTable.maxErrNum = intVal; - fJob.jobTableList.push_back( curTable ); + fJob.jobTableList.push_back(curTable); } - + //------------------------------------------------------------------------------ // Set column information parms. // pNode - current node @@ -555,281 +504,250 @@ void XMLJob::setJobDataTable( xmlNode* pNode ) // Any other new tags probably don't need adding to setJobDataColumn() either, // for the same reason. //------------------------------------------------------------------------------ -void XMLJob::setJobDataColumn( xmlNode* pNode, bool bDefaultCol ) +void XMLJob::setJobDataColumn(xmlNode* pNode, bool bDefaultCol) { - int intVal; - std::string bufString; - JobColumn curColumn; + int intVal; + std::string bufString; + JobColumn curColumn; - if ( fJob.jobTableList.size() == 0 ) - return; + if (fJob.jobTableList.size() == 0) + return; - int tableNo = fJob.jobTableList.size() - 1; + int tableNo = fJob.jobTableList.size() - 1; - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_ORIG_NAME], bufString ) ) - curColumn.colName = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_ORIG_NAME], bufString)) + curColumn.colName = bufString; - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_COL_NAME], bufString ) ) - curColumn.colName = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_COL_NAME], bufString)) + curColumn.colName = bufString; - if (curColumn.colName.empty()) - { - ostringstream oss; - oss << "Required column name attribute (colName) missing from " - "Column tag for table " << - fJob.jobTableList[tableNo].tblName; - throw runtime_error( oss.str() ); - } + if (curColumn.colName.empty()) + { + ostringstream oss; + oss << "Required column name attribute (colName) missing from " + "Column tag for table " + << fJob.jobTableList[tableNo].tblName; + throw runtime_error(oss.str()); + } - if ( getNodeAttribute( pNode, xmlTagTable[TAG_COL_OID], &intVal, TYPE_INT ) ) - curColumn.mapOid = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_COL_OID], &intVal, TYPE_INT)) + curColumn.mapOid = intVal; - if ( getNodeAttribute( pNode, xmlTagTable[TAG_WIDTH], &intVal, TYPE_INT ) ) - { - curColumn.width = intVal; - curColumn.definedWidth = intVal; //@Bug 3040 - } + if (getNodeAttribute(pNode, xmlTagTable[TAG_WIDTH], &intVal, TYPE_INT)) + { + curColumn.width = intVal; + curColumn.definedWidth = intVal; //@Bug 3040 + } - if ( getNodeAttribute( pNode, xmlTagTable[TAG_PRECISION], &intVal, TYPE_INT)) - curColumn.precision = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_PRECISION], &intVal, TYPE_INT)) + curColumn.precision = intVal; - if ( getNodeAttribute( pNode, xmlTagTable[TAG_SCALE], &intVal, TYPE_INT ) ) - curColumn.scale = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_SCALE], &intVal, TYPE_INT)) + curColumn.scale = intVal; - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_DATA_TYPE], bufString ) ) - curColumn.typeName = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_DATA_TYPE], bufString)) + curColumn.typeName = bufString; - if ( getNodeAttribute( pNode, xmlTagTable[TAG_COMPRESS_TYPE], &intVal, - TYPE_INT)) - { - curColumn.compressionType = intVal; - curColumn.dctnry.fCompressionType = intVal; - } + if (getNodeAttribute(pNode, xmlTagTable[TAG_COMPRESS_TYPE], &intVal, TYPE_INT)) + { + curColumn.compressionType = intVal; + curColumn.dctnry.fCompressionType = intVal; + } - if ( getNodeAttribute( pNode, xmlTagTable[TAG_AUTOINCREMENT_FLAG], - &intVal, TYPE_INT)) - { - if (intVal) - curColumn.autoIncFlag = true; - else - curColumn.autoIncFlag = false; - } - - if ( getNodeAttributeStr( pNode, xmlTagTable[TAG_COL_TYPE], bufString ) ) - { - const char* buf = bufString.c_str(); - - if ( !strcmp( buf, "D" ) ) - { - curColumn.colType = 'D'; - - // @Bug 2565: Retain dictionary width to use in truncating strings, - // since BulkLoad eventually stores column token width in 'width'. - curColumn.dctnryWidth = curColumn.width; - - if ( getNodeAttribute( pNode, - xmlTagTable[TAG_DVAL_OID], - &intVal, - TYPE_INT ) ) - curColumn.dctnry.dctnryOid = intVal; - } - } - - // This is a workaround that DBBuilder can not pass decimal type to XML file - if ( ( curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::INT] || - curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::BIGINT] || - curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::SMALLINT] || - curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::TINYINT]) && - curColumn.scale > 0 ) - curColumn.typeName = ColDataTypeStr[CalpontSystemCatalog::DECIMAL]; - - // end of workaround - - // Initialize the saturation limits for this column - initSatLimits( curColumn ); - - // Save default columns in separate list, so that we can intentionally - // add/keep them at the "end" of colList later, after all other columns. - if (bDefaultCol) // temporarily save in separate list - { - curColumn.fFldColRelation = BULK_FLDCOL_COLUMN_DEFAULT; - fDefaultColumns.push_back ( curColumn ); - } + if (getNodeAttribute(pNode, xmlTagTable[TAG_AUTOINCREMENT_FLAG], &intVal, TYPE_INT)) + { + if (intVal) + curColumn.autoIncFlag = true; else - { - // Add to list of db columns to be loaded - curColumn.fFldColRelation = BULK_FLDCOL_COLUMN_FIELD; - fJob.jobTableList[tableNo].colList.push_back ( curColumn ); + curColumn.autoIncFlag = false; + } - // Add to combined field list of columns and ignored fields - JobFieldRef fieldRef( BULK_FLDCOL_COLUMN_FIELD, - fJob.jobTableList[tableNo].colList.size() - 1 ); - fJob.jobTableList[tableNo].fFldRefs.push_back( fieldRef ); + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_COL_TYPE], bufString)) + { + const char* buf = bufString.c_str(); + + if (!strcmp(buf, "D")) + { + curColumn.colType = 'D'; + + // @Bug 2565: Retain dictionary width to use in truncating strings, + // since BulkLoad eventually stores column token width in 'width'. + curColumn.dctnryWidth = curColumn.width; + + if (getNodeAttribute(pNode, xmlTagTable[TAG_DVAL_OID], &intVal, TYPE_INT)) + curColumn.dctnry.dctnryOid = intVal; } + } + + // This is a workaround that DBBuilder can not pass decimal type to XML file + if ((curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::INT] || + curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::BIGINT] || + curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::SMALLINT] || + curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::TINYINT]) && + curColumn.scale > 0) + curColumn.typeName = ColDataTypeStr[CalpontSystemCatalog::DECIMAL]; + + // end of workaround + + // Initialize the saturation limits for this column + initSatLimits(curColumn); + + // Save default columns in separate list, so that we can intentionally + // add/keep them at the "end" of colList later, after all other columns. + if (bDefaultCol) // temporarily save in separate list + { + curColumn.fFldColRelation = BULK_FLDCOL_COLUMN_DEFAULT; + fDefaultColumns.push_back(curColumn); + } + else + { + // Add to list of db columns to be loaded + curColumn.fFldColRelation = BULK_FLDCOL_COLUMN_FIELD; + fJob.jobTableList[tableNo].colList.push_back(curColumn); + + // Add to combined field list of columns and ignored fields + JobFieldRef fieldRef(BULK_FLDCOL_COLUMN_FIELD, fJob.jobTableList[tableNo].colList.size() - 1); + fJob.jobTableList[tableNo].fFldRefs.push_back(fieldRef); + } } - + //------------------------------------------------------------------------------ // Set column information parms for an input field that is to be ignored //------------------------------------------------------------------------------ -void XMLJob::setJobDataIgnoreField( ) +void XMLJob::setJobDataIgnoreField() { - JobColumn curColumn; + JobColumn curColumn; - int tableNo = fJob.jobTableList.size() - 1; - ostringstream oss; - oss << "IgnoreField" << fJob.jobTableList[tableNo].fFldRefs.size() + 1; - curColumn.colName = oss.str(); + int tableNo = fJob.jobTableList.size() - 1; + ostringstream oss; + oss << "IgnoreField" << fJob.jobTableList[tableNo].fFldRefs.size() + 1; + curColumn.colName = oss.str(); - // Add to list of ignored fields - curColumn.fFldColRelation = BULK_FLDCOL_IGNORE_FIELD; - fJob.jobTableList[tableNo].fIgnoredFields.push_back( curColumn ); + // Add to list of ignored fields + curColumn.fFldColRelation = BULK_FLDCOL_IGNORE_FIELD; + fJob.jobTableList[tableNo].fIgnoredFields.push_back(curColumn); - // Add to combined field list of columns and ignored fields - JobFieldRef fieldRef( BULK_FLDCOL_IGNORE_FIELD, - fJob.jobTableList[tableNo].fIgnoredFields.size() - 1 ); - fJob.jobTableList[tableNo].fFldRefs.push_back ( fieldRef ); + // Add to combined field list of columns and ignored fields + JobFieldRef fieldRef(BULK_FLDCOL_IGNORE_FIELD, fJob.jobTableList[tableNo].fIgnoredFields.size() - 1); + fJob.jobTableList[tableNo].fFldRefs.push_back(fieldRef); } //------------------------------------------------------------------------------ // Initialize the saturation limits for the specified column. //------------------------------------------------------------------------------ -void XMLJob::initSatLimits( JobColumn& curColumn ) const +void XMLJob::initSatLimits(JobColumn& curColumn) const { - // If one of the integer types, we set the min/max saturation value. - // For DECIMAL columns this will vary with the precision. - if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::INT] ) - { - curColumn.fMinIntSat = MIN_INT; - curColumn.fMaxIntSat = MAX_INT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UINT] ) - { - curColumn.fMinIntSat = MIN_UINT; - curColumn.fMaxIntSat = MAX_UINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::BIGINT] ) - { - curColumn.fMinIntSat = MIN_BIGINT; - curColumn.fMaxIntSat = MAX_BIGINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UBIGINT] ) - { - curColumn.fMinIntSat = MIN_UBIGINT; - curColumn.fMaxIntSat = MAX_UBIGINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::MEDINT] ) - { - curColumn.fMinIntSat = MIN_MEDINT; - curColumn.fMaxIntSat = MAX_MEDINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UMEDINT] ) - { - curColumn.fMinIntSat = MIN_UMEDINT; - curColumn.fMaxIntSat = MAX_UMEDINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::SMALLINT] ) - { - curColumn.fMinIntSat = MIN_SMALLINT; - curColumn.fMaxIntSat = MAX_SMALLINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::USMALLINT] ) - { - curColumn.fMinIntSat = MIN_USMALLINT; - curColumn.fMaxIntSat = MAX_USMALLINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::TINYINT] ) - { - curColumn.fMinIntSat = MIN_TINYINT; - curColumn.fMaxIntSat = MAX_TINYINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UTINYINT] ) - { - curColumn.fMinIntSat = MIN_UTINYINT; - curColumn.fMaxIntSat = MAX_UTINYINT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::DECIMAL] ) - { - curColumn.fMinIntSat = -columnstore_precision[curColumn.precision]; - curColumn.fMaxIntSat = columnstore_precision[curColumn.precision]; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UDECIMAL] ) - { - curColumn.fMinIntSat = 0; - curColumn.fMaxIntSat = columnstore_precision[curColumn.precision]; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::FLOAT] ) - { - curColumn.fMinDblSat = MIN_FLOAT; - curColumn.fMaxDblSat = MAX_FLOAT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UFLOAT] ) - { - curColumn.fMinDblSat = 0.0; - curColumn.fMaxDblSat = MAX_FLOAT; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::DOUBLE] ) - { - curColumn.fMinDblSat = MIN_DOUBLE; - curColumn.fMaxDblSat = MAX_DOUBLE; - } - else if ( curColumn.typeName == - ColDataTypeStr[CalpontSystemCatalog::UDOUBLE] ) - { - curColumn.fMinDblSat = 0.0; - curColumn.fMaxDblSat = MAX_DOUBLE; - } + // If one of the integer types, we set the min/max saturation value. + // For DECIMAL columns this will vary with the precision. + if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::INT]) + { + curColumn.fMinIntSat = MIN_INT; + curColumn.fMaxIntSat = MAX_INT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UINT]) + { + curColumn.fMinIntSat = MIN_UINT; + curColumn.fMaxIntSat = MAX_UINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::BIGINT]) + { + curColumn.fMinIntSat = MIN_BIGINT; + curColumn.fMaxIntSat = MAX_BIGINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UBIGINT]) + { + curColumn.fMinIntSat = MIN_UBIGINT; + curColumn.fMaxIntSat = MAX_UBIGINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::MEDINT]) + { + curColumn.fMinIntSat = MIN_MEDINT; + curColumn.fMaxIntSat = MAX_MEDINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UMEDINT]) + { + curColumn.fMinIntSat = MIN_UMEDINT; + curColumn.fMaxIntSat = MAX_UMEDINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::SMALLINT]) + { + curColumn.fMinIntSat = MIN_SMALLINT; + curColumn.fMaxIntSat = MAX_SMALLINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::USMALLINT]) + { + curColumn.fMinIntSat = MIN_USMALLINT; + curColumn.fMaxIntSat = MAX_USMALLINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::TINYINT]) + { + curColumn.fMinIntSat = MIN_TINYINT; + curColumn.fMaxIntSat = MAX_TINYINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UTINYINT]) + { + curColumn.fMinIntSat = MIN_UTINYINT; + curColumn.fMaxIntSat = MAX_UTINYINT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::DECIMAL]) + { + curColumn.fMinIntSat = -columnstore_precision[curColumn.precision]; + curColumn.fMaxIntSat = columnstore_precision[curColumn.precision]; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL]) + { + curColumn.fMinIntSat = 0; + curColumn.fMaxIntSat = columnstore_precision[curColumn.precision]; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::FLOAT]) + { + curColumn.fMinDblSat = MIN_FLOAT; + curColumn.fMaxDblSat = MAX_FLOAT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UFLOAT]) + { + curColumn.fMinDblSat = 0.0; + curColumn.fMaxDblSat = MAX_FLOAT; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::DOUBLE]) + { + curColumn.fMinDblSat = MIN_DOUBLE; + curColumn.fMaxDblSat = MAX_DOUBLE; + } + else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UDOUBLE]) + { + curColumn.fMinDblSat = 0.0; + curColumn.fMaxDblSat = MAX_DOUBLE; + } } //------------------------------------------------------------------------------ // Set Read Buffers attributes // pNode - current node //------------------------------------------------------------------------------ -void XMLJob::setReadBuffers( xmlNode* pNode ) +void XMLJob::setReadBuffers(xmlNode* pNode) { - int intVal = 0; + int intVal = 0; - if (getNodeAttribute(pNode, - xmlTagTable[TAG_NO_OF_READ_BUFFERS], - &intVal, - TYPE_INT )) - fJob.numberOfReadBuffers = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_NO_OF_READ_BUFFERS], &intVal, TYPE_INT)) + fJob.numberOfReadBuffers = intVal; - if (getNodeAttribute(pNode, - xmlTagTable[TAG_READ_BUFFER_SIZE], - &intVal, - TYPE_INT )) - fJob.readBufferSize = intVal; + if (getNodeAttribute(pNode, xmlTagTable[TAG_READ_BUFFER_SIZE], &intVal, TYPE_INT)) + fJob.readBufferSize = intVal; } - + //------------------------------------------------------------------------------ // Set Schema attributes // pNode - current node //------------------------------------------------------------------------------ -void XMLJob::setSchema( xmlNode* pNode ) +void XMLJob::setSchema(xmlNode* pNode) { - std::string bufString; + std::string bufString; - if ( getNodeAttributeStr( pNode, - xmlTagTable[TAG_SCHEMA_NAME], - bufString ) ) - fJob.schema = bufString; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_SCHEMA_NAME], bufString)) + fJob.schema = bufString; } - + //------------------------------------------------------------------------------ // Transfer any/all columns from temporary fDefaultColumns, to // the end of the column/field lists. @@ -839,465 +757,425 @@ void XMLJob::setSchema( xmlNode* pNode ) //------------------------------------------------------------------------------ void XMLJob::postProcessTableNode() { - bool bValidateNoDefColWithoutDefValue = false; + bool bValidateNoDefColWithoutDefValue = false; - if (fDefaultColumns.size() > 0) + if (fDefaultColumns.size() > 0) + { + bValidateNoDefColWithoutDefValue = true; + int tableNo = fJob.jobTableList.size() - 1; + + for (unsigned k = 0; k < fDefaultColumns.size(); k++) { - bValidateNoDefColWithoutDefValue = true; - int tableNo = fJob.jobTableList.size() - 1; + // Add to list of db columns to be loaded + fJob.jobTableList[tableNo].colList.push_back(fDefaultColumns[k]); - for (unsigned k = 0; k < fDefaultColumns.size(); k++) - { - // Add to list of db columns to be loaded - fJob.jobTableList[tableNo].colList.push_back( fDefaultColumns[k] ); - - // Add to combined list of columns and ignored fields - JobFieldRef fieldRef( BULK_FLDCOL_COLUMN_DEFAULT, - fJob.jobTableList[tableNo].colList.size() - 1 ); - fJob.jobTableList[tableNo].fFldRefs.push_back( fieldRef ); - } - - fDefaultColumns.clear(); + // Add to combined list of columns and ignored fields + JobFieldRef fieldRef(BULK_FLDCOL_COLUMN_DEFAULT, fJob.jobTableList[tableNo].colList.size() - 1); + fJob.jobTableList[tableNo].fFldRefs.push_back(fieldRef); } - // Supplement xml file contents with information from syscat - execplan::CalpontSystemCatalog::RIDList colRidList; - fillInXMLDataAsLoaded( colRidList ); + fDefaultColumns.clear(); + } - // After getting all the system catalog information... - // Validate that if there are any tags for a NotNull - // column, that the column is defined as NotNull With Default. - if (bValidateNoDefColWithoutDefValue) + // Supplement xml file contents with information from syscat + execplan::CalpontSystemCatalog::RIDList colRidList; + fillInXMLDataAsLoaded(colRidList); + + // After getting all the system catalog information... + // Validate that if there are any tags for a NotNull + // column, that the column is defined as NotNull With Default. + if (bValidateNoDefColWithoutDefValue) + { + int tableNo = fJob.jobTableList.size() - 1; + + for (unsigned int iCol = 0; iCol < fJob.jobTableList[tableNo].colList.size(); iCol++) { - int tableNo = fJob.jobTableList.size() - 1; + JobColumn& col = fJob.jobTableList[tableNo].colList[iCol]; - for (unsigned int iCol = 0; - iCol < fJob.jobTableList[tableNo].colList.size(); iCol++) + if (col.fFldColRelation == BULK_FLDCOL_COLUMN_DEFAULT) + { + if ((col.fNotNull) && (!col.fWithDefault)) { - JobColumn& col = fJob.jobTableList[tableNo].colList[iCol]; - - if (col.fFldColRelation == BULK_FLDCOL_COLUMN_DEFAULT) - { - if ( (col.fNotNull) && (!col.fWithDefault) ) - { - std::ostringstream oss; - oss << "Column " << col.colName << " in table " << - fJob.jobTableList[tableNo].tblName << " is NotNull " - "w/o default; cannot be used with "; - throw std::runtime_error( oss.str() ); - } - } + std::ostringstream oss; + oss << "Column " << col.colName << " in table " << fJob.jobTableList[tableNo].tblName + << " is NotNull " + "w/o default; cannot be used with "; + throw std::runtime_error(oss.str()); } + } } + } - // Make sure all Columns in the DB are counted for with or - // tags (unless validate is disabled) - if (fValidateColList) - validateAllColumnsHaveTags( colRidList ); + // Make sure all Columns in the DB are counted for with or + // tags (unless validate is disabled) + if (fValidateColList) + validateAllColumnsHaveTags(colRidList); } - + //------------------------------------------------------------------------------ // Use the table and column names from the last
just loaded, to // collect the remaining information from the system catalog, in order to // populate the JobColumn structure. //------------------------------------------------------------------------------ -void XMLJob::fillInXMLDataAsLoaded( - execplan::CalpontSystemCatalog::RIDList& colRidList) +void XMLJob::fillInXMLDataAsLoaded(execplan::CalpontSystemCatalog::RIDList& colRidList) { - boost::shared_ptr cat = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - cat->identity(execplan::CalpontSystemCatalog::EC); + boost::shared_ptr cat = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(execplan::CalpontSystemCatalog::EC); - // Get the table and column attributes for the last
processed - unsigned int iTbl = fJob.jobTableList.size() - 1; - JobTable& tbl = fJob.jobTableList[iTbl]; + // Get the table and column attributes for the last
processed + unsigned int iTbl = fJob.jobTableList.size() - 1; + JobTable& tbl = fJob.jobTableList[iTbl]; - std::string tblName; - string::size_type startName = tbl.tblName.rfind('.'); + std::string tblName; + string::size_type startName = tbl.tblName.rfind('.'); - if (startName == string::npos) - tblName.assign( tbl.tblName ); - else - tblName.assign( tbl.tblName.substr(startName + 1) ); + if (startName == string::npos) + tblName.assign(tbl.tblName); + else + tblName.assign(tbl.tblName.substr(startName + 1)); - execplan::CalpontSystemCatalog::TableName table( - fJob.schema, tblName ); + execplan::CalpontSystemCatalog::TableName table(fJob.schema, tblName); - if (fJob.jobTableList[iTbl].mapOid == 0) + if (fJob.jobTableList[iTbl].mapOid == 0) + { + execplan::CalpontSystemCatalog::OID tblOid = cat->tableRID(table).objnum; + tbl.mapOid = tblOid; + } + + // This call is made to improve performance. + // The call forces all the column information for this table to be + // cached at one time, instead of doing it piece-meal through repeated + // calls to lookupOID(). + colRidList = cat->columnRIDs(table, true); + + // Loop through the columns to get the column attributes + for (unsigned int iCol = 0; iCol < fJob.jobTableList[iTbl].colList.size(); iCol++) + { + JobColumn& col = fJob.jobTableList[iTbl].colList[iCol]; + + if (col.mapOid == 0) { - execplan::CalpontSystemCatalog::OID tblOid = - cat->tableRID(table).objnum; - tbl.mapOid = tblOid; + execplan::CalpontSystemCatalog::TableColName column; + column.schema = fJob.schema; + column.table = tblName; + column.column = col.colName; + execplan::CalpontSystemCatalog::OID colOid = cat->lookupOID(column); + + if (colOid < 0) + { + ostringstream oss; + oss << "Column OID lookup failed for: " << column; + throw runtime_error(oss.str()); + } + + col.mapOid = colOid; + + execplan::CalpontSystemCatalog::ColType colType = cat->colType(col.mapOid); + + col.width = colType.colWidth; + col.definedWidth = colType.colWidth; + + if ((colType.scale > 0) || (colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) || + (colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL)) + { + col.precision = colType.precision; + col.scale = colType.scale; + } + + col.typeName = ColDataTypeStr[colType.colDataType]; + col.compressionType = colType.compressionType; + col.dctnry.fCompressionType = colType.compressionType; + + if (colType.autoincrement) + col.autoIncFlag = true; + else + col.autoIncFlag = false; + + // Initialize NotNull and Default Value (based on data type) + fillInXMLDataNotNullDefault(tbl.tblName, colType, col); + + if (colType.ddn.dictOID > 0) + { + col.colType = 'D'; + col.dctnryWidth = colType.colWidth; + col.dctnry.dctnryOid = colType.ddn.dictOID; + } + + // @bug3801: For backwards compatability, we treat + // integer types with nonzero 0 scale as decimal if scale > 0 + if (((col.typeName == ColDataTypeStr[CalpontSystemCatalog::INT]) || + (col.typeName == ColDataTypeStr[CalpontSystemCatalog::BIGINT]) || + (col.typeName == ColDataTypeStr[CalpontSystemCatalog::SMALLINT]) || + (col.typeName == ColDataTypeStr[CalpontSystemCatalog::TINYINT])) && + (col.scale > 0)) + { + col.typeName = ColDataTypeStr[CalpontSystemCatalog::DECIMAL]; + } + + // Initialize the saturation limits for this column + initSatLimits(col); } - - // This call is made to improve performance. - // The call forces all the column information for this table to be - // cached at one time, instead of doing it piece-meal through repeated - // calls to lookupOID(). - colRidList = cat->columnRIDs(table, true); - - // Loop through the columns to get the column attributes - for (unsigned int iCol = 0; - iCol < fJob.jobTableList[iTbl].colList.size(); iCol++) - { - JobColumn& col = fJob.jobTableList[iTbl].colList[iCol]; - - if (col.mapOid == 0) - { - execplan::CalpontSystemCatalog::TableColName column; - column.schema = fJob.schema; - column.table = tblName; - column.column = col.colName; - execplan::CalpontSystemCatalog::OID colOid = - cat->lookupOID( column ); - - if (colOid < 0) - { - ostringstream oss; - oss << "Column OID lookup failed for: " << column; - throw runtime_error( oss.str() ); - } - - col.mapOid = colOid; - - execplan::CalpontSystemCatalog::ColType colType = - cat->colType( col.mapOid ); - - col.width = colType.colWidth; - col.definedWidth = colType.colWidth; - - if ((colType.scale > 0) || - (colType.colDataType == - execplan::CalpontSystemCatalog::DECIMAL) || - (colType.colDataType == - execplan::CalpontSystemCatalog::UDECIMAL)) - { - col.precision = colType.precision; - col.scale = colType.scale; - } - - col.typeName = ColDataTypeStr[colType.colDataType]; - col.compressionType = colType.compressionType; - col.dctnry.fCompressionType = colType.compressionType; - - if (colType.autoincrement) - col.autoIncFlag = true; - else - col.autoIncFlag = false; - - // Initialize NotNull and Default Value (based on data type) - fillInXMLDataNotNullDefault( tbl.tblName, colType, col ); - - if (colType.ddn.dictOID > 0) - { - col.colType = 'D'; - col.dctnryWidth = colType.colWidth; - col.dctnry.dctnryOid = colType.ddn.dictOID; - } - - // @bug3801: For backwards compatability, we treat - // integer types with nonzero 0 scale as decimal if scale > 0 - if ( ((col.typeName == - ColDataTypeStr[CalpontSystemCatalog::INT]) || - (col.typeName == - ColDataTypeStr[CalpontSystemCatalog::BIGINT]) || - (col.typeName == - ColDataTypeStr[CalpontSystemCatalog::SMALLINT]) || - (col.typeName == - ColDataTypeStr[CalpontSystemCatalog::TINYINT])) && - (col.scale > 0) ) - { - col.typeName = ColDataTypeStr[CalpontSystemCatalog::DECIMAL]; - } - - // Initialize the saturation limits for this column - initSatLimits( col ); - } - } // end of loop through columns + } // end of loop through columns } - + //------------------------------------------------------------------------------ // Using information from the system catalog (in colType), fill in the // applicable NotNull Default values into the specified JobColumn. //------------------------------------------------------------------------------ -void XMLJob::fillInXMLDataNotNullDefault( - const std::string& fullTblName, - execplan::CalpontSystemCatalog::ColType& colType, - JobColumn& col ) +void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName, + execplan::CalpontSystemCatalog::ColType& colType, JobColumn& col) { - const std::string col_defaultValue(colType.defaultValue); + const std::string col_defaultValue(colType.defaultValue); - if (colType.constraintType == - execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + if (colType.constraintType == execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT) + { + col.fNotNull = true; + + if (!col_defaultValue.empty()) + col.fWithDefault = true; + } + else if (colType.constraintType == execplan::CalpontSystemCatalog::DEFAULT_CONSTRAINT) + { + col.fWithDefault = true; + } + + if (col.fWithDefault) + { + bool bDefaultConvertError = false; + + // Convert Default Value. + // We go ahead and report basic format conversion error; + // but we don't do complete validation (like checking to see + // if the default is too large for the given integer type), + // because we assume DDL is fully validating the default value. + switch (colType.colDataType) { - col.fNotNull = true; + case execplan::CalpontSystemCatalog::BIT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::BIGINT: + { + errno = 0; + col.fDefaultInt = strtoll(col_defaultValue.c_str(), 0, 10); - if (!col_defaultValue.empty()) - col.fWithDefault = true; - } - else if (colType.constraintType == - execplan::CalpontSystemCatalog::DEFAULT_CONSTRAINT) - { - col.fWithDefault = true; - } + if (errno == ERANGE) + bDefaultConvertError = true; - if (col.fWithDefault) - { - bool bDefaultConvertError = false; + break; + } - // Convert Default Value. - // We go ahead and report basic format conversion error; - // but we don't do complete validation (like checking to see - // if the default is too large for the given integer type), - // because we assume DDL is fully validating the default value. - switch (colType.colDataType) + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UBIGINT: + { + errno = 0; + col.fDefaultUInt = strtoull(col_defaultValue.c_str(), 0, 10); + + if (errno == ERANGE) + bDefaultConvertError = true; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH)) { - case execplan::CalpontSystemCatalog::BIT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - { - errno = 0; - col.fDefaultInt = strtoll(col_defaultValue.c_str(), 0, 10); + col.fDefaultWideDecimal = colType.decimal128FromString(col_defaultValue, &bDefaultConvertError); + } + else + { + col.fDefaultInt = Convertor::convertDecimalString(col_defaultValue.c_str(), + col_defaultValue.length(), colType.scale); - if (errno == ERANGE) - bDefaultConvertError = true; - - break; - } - - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - { - errno = 0; - col.fDefaultUInt = strtoull(col_defaultValue.c_str(), 0, 10); - - if (errno == ERANGE) - bDefaultConvertError = true; - - break; - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH)) - { - col.fDefaultWideDecimal = colType.decimal128FromString( - col_defaultValue, &bDefaultConvertError); - } - else - { - col.fDefaultInt = Convertor::convertDecimalString( - col_defaultValue.c_str(), - col_defaultValue.length(), - colType.scale); - - if (errno == ERANGE) - bDefaultConvertError = true; - } - - break; - } - - case execplan::CalpontSystemCatalog::DATE: - { - int convertStatus; - int32_t dt = - dataconvert::DataConvert::convertColumnDate( - col_defaultValue.c_str(), - dataconvert::CALPONTDATE_ENUM, convertStatus, - col_defaultValue.length() ); - - if (convertStatus != 0) - bDefaultConvertError = true; - - col.fDefaultInt = dt; - break; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - int convertStatus; - int64_t dt = - dataconvert::DataConvert::convertColumnDatetime( - col_defaultValue.c_str(), - dataconvert::CALPONTDATETIME_ENUM, convertStatus, - col_defaultValue.length() ); - - if (convertStatus != 0) - bDefaultConvertError = true; - - col.fDefaultInt = dt; - break; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - int convertStatus; - int64_t dt = - dataconvert::DataConvert::convertColumnTimestamp( - col_defaultValue.c_str(), - dataconvert::CALPONTDATETIME_ENUM, convertStatus, - col_defaultValue.length(), fTimeZone ); - - if (convertStatus != 0) - bDefaultConvertError = true; - - col.fDefaultInt = dt; - break; - } - - case execplan::CalpontSystemCatalog::TIME: - { - int convertStatus; - int64_t dt = - dataconvert::DataConvert::convertColumnTime( - col_defaultValue.c_str(), - dataconvert::CALPONTTIME_ENUM, convertStatus, - col_defaultValue.length() ); - - if (convertStatus != 0) - bDefaultConvertError = true; - - col.fDefaultInt = dt; - break; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - errno = 0; - col.fDefaultDbl = strtod(col_defaultValue.c_str(), 0); - - if (errno == ERANGE) - bDefaultConvertError = true; - - break; - } - - default: - { - col.fDefaultChr = col_defaultValue; - break; - } + if (errno == ERANGE) + bDefaultConvertError = true; } - if (bDefaultConvertError) - { - std::ostringstream oss; - oss << "Column " << col.colName << " in table " << fullTblName << - " has an invalid default value in system catalog."; - throw std::runtime_error( oss.str() ); - } + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + int convertStatus; + int32_t dt = dataconvert::DataConvert::convertColumnDate(col_defaultValue.c_str(), + dataconvert::CALPONTDATE_ENUM, convertStatus, + col_defaultValue.length()); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + int convertStatus; + int64_t dt = dataconvert::DataConvert::convertColumnDatetime( + col_defaultValue.c_str(), dataconvert::CALPONTDATETIME_ENUM, convertStatus, + col_defaultValue.length()); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int convertStatus; + int64_t dt = dataconvert::DataConvert::convertColumnTimestamp( + col_defaultValue.c_str(), dataconvert::CALPONTDATETIME_ENUM, convertStatus, + col_defaultValue.length(), fTimeZone); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int convertStatus; + int64_t dt = dataconvert::DataConvert::convertColumnTime(col_defaultValue.c_str(), + dataconvert::CALPONTTIME_ENUM, convertStatus, + col_defaultValue.length()); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + errno = 0; + col.fDefaultDbl = strtod(col_defaultValue.c_str(), 0); + + if (errno == ERANGE) + bDefaultConvertError = true; + + break; + } + + default: + { + col.fDefaultChr = col_defaultValue; + break; + } } + + if (bDefaultConvertError) + { + std::ostringstream oss; + oss << "Column " << col.colName << " in table " << fullTblName + << " has an invalid default value in system catalog."; + throw std::runtime_error(oss.str()); + } + } } - + //------------------------------------------------------------------------------ // Use the table and column names from the last
just loaded, to // validate that all the columns have a or tag // present in the job XML file. //------------------------------------------------------------------------------ -void XMLJob::validateAllColumnsHaveTags( - const execplan::CalpontSystemCatalog::RIDList& colRidList) const +void XMLJob::validateAllColumnsHaveTags(const execplan::CalpontSystemCatalog::RIDList& colRidList) const { - // Validate column list for the last
processed - unsigned int iTbl = fJob.jobTableList.size() - 1; - const JobTable& tbl = fJob.jobTableList[iTbl]; + // Validate column list for the last
processed + unsigned int iTbl = fJob.jobTableList.size() - 1; + const JobTable& tbl = fJob.jobTableList[iTbl]; - std::string tblName; - string::size_type startName = tbl.tblName.rfind('.'); + std::string tblName; + string::size_type startName = tbl.tblName.rfind('.'); - if (startName == string::npos) - tblName.assign( tbl.tblName ); - else - tblName.assign( tbl.tblName.substr(startName + 1) ); + if (startName == string::npos) + tblName.assign(tbl.tblName); + else + tblName.assign(tbl.tblName.substr(startName + 1)); - try + try + { + // Loop through column tags, saving col OIDs to a std::set for lookups + std::set colOIDList; + typedef std::set::iterator SetIter; + std::pair retVal; + + for (unsigned int iCol = 0; iCol < fJob.jobTableList[iTbl].colList.size(); iCol++) { - // Loop through column tags, saving col OIDs to a std::set for lookups - std::set colOIDList; - typedef std::set::iterator SetIter; - std::pair retVal; + const JobColumn& col = fJob.jobTableList[iTbl].colList[iCol]; + retVal = colOIDList.insert(col.mapOid); - for (unsigned int iCol = 0; - iCol < fJob.jobTableList[iTbl].colList.size(); iCol++) - { - const JobColumn& col = fJob.jobTableList[iTbl].colList[iCol]; - retVal = colOIDList.insert( col.mapOid ); + if (!retVal.second) + { + boost::shared_ptr cat = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(execplan::CalpontSystemCatalog::EC); - if (!retVal.second) - { - boost::shared_ptr cat = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - cat->identity(execplan::CalpontSystemCatalog::EC); - - execplan::CalpontSystemCatalog::TableColName dbColName = - cat->colName( col.mapOid ); - std::ostringstream oss; - oss << "Column " << dbColName.column << " referenced in Job XML" - " file more than once."; - throw std::runtime_error( oss.str() ); - } - } - - SetIter pos; - - // Loop thru cols in system catalog and verify that each one has a tag - execplan::CalpontSystemCatalog::RIDList::const_iterator rid_iterator = - colRidList.begin(); - - while (rid_iterator != colRidList.end()) - { - pos = colOIDList.find( rid_iterator->objnum ); - - if (pos != colOIDList.end()) - { - colOIDList.erase( pos ); // through with this column, so delete - } - else - { - boost::shared_ptr cat = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog( - BULK_SYSCAT_SESSION_ID); - cat->identity(execplan::CalpontSystemCatalog::EC); - - execplan::CalpontSystemCatalog::TableColName dbColName = - cat->colName( rid_iterator->objnum ); - std::ostringstream oss; - oss << "No tag present in Job XML file for DB column: " << - dbColName.column; - throw std::runtime_error( oss.str() ); - } - - ++rid_iterator; - } - } - catch (std::exception& ex) - { + execplan::CalpontSystemCatalog::TableColName dbColName = cat->colName(col.mapOid); std::ostringstream oss; - oss << "Error validating column list for table " << - fJob.schema << '.' << tblName << "; " << ex.what(); - throw std::runtime_error( oss.str() ); + oss << "Column " << dbColName.column + << " referenced in Job XML" + " file more than once."; + throw std::runtime_error(oss.str()); + } } - catch (...) + + SetIter pos; + + // Loop thru cols in system catalog and verify that each one has a tag + execplan::CalpontSystemCatalog::RIDList::const_iterator rid_iterator = colRidList.begin(); + + while (rid_iterator != colRidList.end()) { + pos = colOIDList.find(rid_iterator->objnum); + + if (pos != colOIDList.end()) + { + colOIDList.erase(pos); // through with this column, so delete + } + else + { + boost::shared_ptr cat = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(execplan::CalpontSystemCatalog::EC); + + execplan::CalpontSystemCatalog::TableColName dbColName = cat->colName(rid_iterator->objnum); std::ostringstream oss; - oss << "Unknown Error validating column list for table " << - fJob.schema << '.' << tblName; - throw std::runtime_error( oss.str() ); + oss << "No tag present in Job XML file for DB column: " << dbColName.column; + throw std::runtime_error(oss.str()); + } + + ++rid_iterator; } + } + catch (std::exception& ex) + { + std::ostringstream oss; + oss << "Error validating column list for table " << fJob.schema << '.' << tblName << "; " << ex.what(); + throw std::runtime_error(oss.str()); + } + catch (...) + { + std::ostringstream oss; + oss << "Unknown Error validating column list for table " << fJob.schema << '.' << tblName; + throw std::runtime_error(oss.str()); + } } - + //------------------------------------------------------------------------------ // Generate a permanent or temporary Job XML file name path. // sXMLJobDir Command line override for complete Job directory path @@ -1310,148 +1188,137 @@ void XMLJob::validateAllColumnsHaveTags( // errMsg Relevant error message if return value is not NO_ERROR. //------------------------------------------------------------------------------ /* static */ -int XMLJob::genJobXMLFileName( - const string& sXMLJobDir, - const string& jobDir, - const string& jobId, - bool bTempFile, - const string& schemaName, - const string& tableName, - boost::filesystem::path& xmlFilePath, - string& errMsg, - std::string& tableOIDStr ) +int XMLJob::genJobXMLFileName(const string& sXMLJobDir, const string& jobDir, const string& jobId, + bool bTempFile, const string& schemaName, const string& tableName, + boost::filesystem::path& xmlFilePath, string& errMsg, std::string& tableOIDStr) { - // get full file directory path for XML job description file - if (sXMLJobDir.empty()) - { - xmlFilePath = Config::getBulkRoot(); - xmlFilePath /= jobDir; - } - else - { - xmlFilePath = sXMLJobDir; + // get full file directory path for XML job description file + if (sXMLJobDir.empty()) + { + xmlFilePath = Config::getBulkRoot(); + xmlFilePath /= jobDir; + } + else + { + xmlFilePath = sXMLJobDir; - //If filespec doesn't begin with a '/' (i.e. it's not an absolute path), - // attempt to make it absolute so that we can log the full pathname. - if (!xmlFilePath.has_root_path()) - { + // If filespec doesn't begin with a '/' (i.e. it's not an absolute path), + // attempt to make it absolute so that we can log the full pathname. + if (!xmlFilePath.has_root_path()) + { #ifdef _MSC_VER - // nothing else to do + // nothing else to do #else - char cwdPath[4096]; - char *err; - err = getcwd(cwdPath, sizeof(cwdPath)); - if (err == NULL) - { - errMsg = "Failed to get the current working directory."; - return -1; - } - string trailingPath(xmlFilePath.string()); - xmlFilePath = cwdPath; - xmlFilePath /= trailingPath; + char cwdPath[4096]; + char* err; + err = getcwd(cwdPath, sizeof(cwdPath)); + if (err == NULL) + { + errMsg = "Failed to get the current working directory."; + return -1; + } + string trailingPath(xmlFilePath.string()); + xmlFilePath = cwdPath; + xmlFilePath /= trailingPath; #endif - } } + } - // Append the file name to the directory path - string jobFileName; + // Append the file name to the directory path + string jobFileName; - if (bTempFile) - { - // Create tmp directory if does not exist - RETURN_ON_ERROR( createTempJobDir( xmlFilePath.string(), errMsg ) ); - jobFileName += tableOIDStr; - //jobFileName += schemaName; - // jobFileName += '_'; - // jobFileName += tableName; - jobFileName += "_D"; + if (bTempFile) + { + // Create tmp directory if does not exist + RETURN_ON_ERROR(createTempJobDir(xmlFilePath.string(), errMsg)); + jobFileName += tableOIDStr; + // jobFileName += schemaName; + // jobFileName += '_'; + // jobFileName += tableName; + jobFileName += "_D"; - string now(boost::posix_time::to_iso_string( boost::posix_time::second_clock::local_time())); + string now(boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())); - // microseconds - struct timeval tp; - gettimeofday(&tp, 0); - ostringstream usec; - usec << setfill('0') << setw(6) << tp.tv_usec; + // microseconds + struct timeval tp; + gettimeofday(&tp, 0); + ostringstream usec; + usec << setfill('0') << setw(6) << tp.tv_usec; - jobFileName += now.substr(0, 8); - jobFileName += "_T"; - jobFileName += now.substr(9, 6); - jobFileName += "_S"; - jobFileName += usec.str(); - jobFileName += '_'; - } + jobFileName += now.substr(0, 8); + jobFileName += "_T"; + jobFileName += now.substr(9, 6); + jobFileName += "_S"; + jobFileName += usec.str(); + jobFileName += '_'; + } - jobFileName += "Job_"; - jobFileName += jobId; - jobFileName += ".xml"; + jobFileName += "Job_"; + jobFileName += jobId; + jobFileName += ".xml"; - xmlFilePath /= jobFileName; + xmlFilePath /= jobFileName; - return NO_ERROR; + return NO_ERROR; } - + //------------------------------------------------------------------------------ // Create directory for temporary XML job description files. // OAM restart should delete any/all files in this directory. //------------------------------------------------------------------------------ /* static */ -int XMLJob::createTempJobDir( const string& xmlFilePath, - string& errMsg ) +int XMLJob::createTempJobDir(const string& xmlFilePath, string& errMsg) { - boost::filesystem::path pathDir(xmlFilePath); + boost::filesystem::path pathDir(xmlFilePath); - // create temp directory for XML job file if it does not exist - try + // create temp directory for XML job file if it does not exist + try + { + if (!boost::filesystem::exists(xmlFilePath)) { - if ( !boost::filesystem::exists( xmlFilePath ) ) - { - string boostErrString; + string boostErrString; - try - { - boost::filesystem::create_directories(pathDir); - } - catch (exception& ex) - { - // ignore exception for now; we may have just had a - // race condition where 2 jobs were creating dirs. - boostErrString = ex.what(); - } + try + { + boost::filesystem::create_directories(pathDir); + } + catch (exception& ex) + { + // ignore exception for now; we may have just had a + // race condition where 2 jobs were creating dirs. + boostErrString = ex.what(); + } - if ( !boost::filesystem::exists( xmlFilePath ) ) - { - ostringstream oss; - oss << "Error creating XML temp job file directory(1) " << - xmlFilePath << "; " << boostErrString; - errMsg = oss.str(); - - return ERR_DIR_CREATE; - } - } - } - catch (exception& ex) - { + if (!boost::filesystem::exists(xmlFilePath)) + { ostringstream oss; - oss << "Error creating XML temp job file directory(2) " << - xmlFilePath << "; " << ex.what(); + oss << "Error creating XML temp job file directory(1) " << xmlFilePath << "; " << boostErrString; errMsg = oss.str(); return ERR_DIR_CREATE; + } } + } + catch (exception& ex) + { + ostringstream oss; + oss << "Error creating XML temp job file directory(2) " << xmlFilePath << "; " << ex.what(); + errMsg = oss.str(); - if (!boost::filesystem::is_directory(pathDir) ) - { - ostringstream oss; - oss << "Error creating XML temp job file directory " << - xmlFilePath << "; path already exists as non-directory" << endl; - errMsg = oss.str(); + return ERR_DIR_CREATE; + } - return ERR_DIR_CREATE; - } + if (!boost::filesystem::is_directory(pathDir)) + { + ostringstream oss; + oss << "Error creating XML temp job file directory " << xmlFilePath + << "; path already exists as non-directory" << endl; + errMsg = oss.str(); - return NO_ERROR; + return ERR_DIR_CREATE; + } + + return NO_ERROR; } -} //end of namespace - +} // namespace WriteEngine diff --git a/writeengine/xml/we_xmljob.h b/writeengine/xml/we_xmljob.h index 3f144975b..dac83aa33 100644 --- a/writeengine/xml/we_xmljob.h +++ b/writeengine/xml/we_xmljob.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_xmljob.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_xmljob.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_XMLJOB_H_ @@ -50,123 +50,106 @@ struct JobColumn; */ class XMLJob : public XMLOp { + public: + /** + * @brief Constructor + */ + EXPORT XMLJob(); -public: - /** - * @brief Constructor - */ - EXPORT XMLJob(); + /** + * @brief Default Destructor + */ + EXPORT ~XMLJob(); - /** - * @brief Default Destructor - */ - EXPORT ~XMLJob(); + /** + * @brief Utility to generate a full path name for a Job XML file name + * @param sXMLJobDir Command line override for complete Job directory path + * @param jobDIr Job subdirectory under default path + * @param jobId Job ID + * @param bTempFile Are we creating a temporary Job XML File name + * @param schmaName If temp file, this is schema name to use + * @param tableName If temp file, this is the table name to use + * @param xmlDirPath The complete Job XML file path that is constructed + * @param errMsg Error message if return value is not NO_ERROR + */ + EXPORT static int genJobXMLFileName(const std::string& sXMLJobDir, const std::string& jobDir, + const std::string& jobId, bool bTempFile, const std::string& schemaName, + const std::string& tableName, boost::filesystem::path& xmlDirPath, + std::string& errMsg, std::string& tableOIDStr); - /** - * @brief Utility to generate a full path name for a Job XML file name - * @param sXMLJobDir Command line override for complete Job directory path - * @param jobDIr Job subdirectory under default path - * @param jobId Job ID - * @param bTempFile Are we creating a temporary Job XML File name - * @param schmaName If temp file, this is schema name to use - * @param tableName If temp file, this is the table name to use - * @param xmlDirPath The complete Job XML file path that is constructed - * @param errMsg Error message if return value is not NO_ERROR - */ - EXPORT static int genJobXMLFileName( - const std::string& sXMLJobDir, - const std::string& jobDir, - const std::string& jobId, - bool bTempFile, - const std::string& schemaName, - const std::string& tableName, - boost::filesystem::path& xmlDirPath, - std::string& errMsg, - std::string& tableOIDStr ); + /** + * @brief Get job structure + */ + const Job& getJob() const + { + return fJob; + } - /** - * @brief Get job structure - */ - const Job& getJob() const - { - return fJob; - } + /** + * @brief Load job information + * @param fileName Name of Job XML file to be read + * @param bTempFile Are we creating a temporary Job XML File that will be + * deleted by the destructor + * @param bValidateColumnList Validate that all columns have an XML tag + * @param errMsg Error message if return value is not NO_ERROR + */ + EXPORT int loadJobXmlFile(const std::string& fileName, bool bTempFile, bool bValidateColumnList, + std::string& errMsg); - /** - * @brief Load job information - * @param fileName Name of Job XML file to be read - * @param bTempFile Are we creating a temporary Job XML File that will be - * deleted by the destructor - * @param bValidateColumnList Validate that all columns have an XML tag - * @param errMsg Error message if return value is not NO_ERROR - */ - EXPORT int loadJobXmlFile( const std::string& fileName, - bool bTempFile, - bool bValidateColumnList, - std::string& errMsg ); + /** + * @brief Print job related information + * @param logger Log object that is to receive the print output + */ + EXPORT void printJobInfo(Log& logger) const; - /** - * @brief Print job related information - * @param logger Log object that is to receive the print output - */ - EXPORT void printJobInfo(Log& logger) const; + /** + * @brief Print abbreviated job related information + * @param logger Log object that is to receive the print output + */ + EXPORT void printJobInfoBrief(Log& logger) const; - /** - * @brief Print abbreviated job related information - * @param logger Log object that is to receive the print output - */ - EXPORT void printJobInfoBrief(Log& logger) const; + /** + * @brief Process node + * @param pParentNode Node to be parsed from XML + */ + EXPORT bool processNode(xmlNode* pParentNode); - /** - * @brief Process node - * @param pParentNode Node to be parsed from XML - */ - EXPORT bool processNode( xmlNode* pParentNode ); + /** + * @brief Set timezone + */ + void setTimeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } - /** - * @brief Set timezone - */ - void setTimeZone(const std::string& timeZone) - { - fTimeZone = timeZone; - } + private: + void setJobData(xmlNode* pNode, const xmlTag tag, bool bExpectContent, XML_DTYPE tagType); + void setJobDataColumn(xmlNode* pNode, bool bDefaultCol); + void setJobDataIgnoreField(); + void setJobDataTable(xmlNode* pNode); + void setReadBuffers(xmlNode* pNode); + void setSchema(xmlNode* pNode); + void initSatLimits(JobColumn& column) const; + void fillInXMLDataAsLoaded(execplan::CalpontSystemCatalog::RIDList& colRidList); + void fillInXMLDataNotNullDefault(const std::string& fullTblName, + execplan::CalpontSystemCatalog::ColType& colType, JobColumn& col); + void validateAllColumnsHaveTags(const execplan::CalpontSystemCatalog::RIDList& colRidList) const; + void postProcessTableNode(); + static int createTempJobDir(const std::string& xmlFilePath, std::string& errMsg); -private: - void setJobData( xmlNode* pNode, - const xmlTag tag, - bool bExpectContent, - XML_DTYPE tagType ); - void setJobDataColumn( xmlNode* pNode, bool bDefaultCol ); - void setJobDataIgnoreField ( ); - void setJobDataTable ( xmlNode* pNode ); - void setReadBuffers ( xmlNode* pNode ); - void setSchema ( xmlNode* pNode ); - void initSatLimits( JobColumn& column ) const; - void fillInXMLDataAsLoaded( - execplan::CalpontSystemCatalog::RIDList& colRidList); - void fillInXMLDataNotNullDefault( - const std::string& fullTblName, - execplan::CalpontSystemCatalog::ColType& colType, - JobColumn& col ); - void validateAllColumnsHaveTags( const - execplan::CalpontSystemCatalog::RIDList& colRidList) const; - void postProcessTableNode( ); - static int createTempJobDir( const std::string& xmlFilePath, - std::string& errMsg ); + Job fJob; // current job xml - Job fJob; // current job xml - - DebugLevel fDebugLevel; // internal use debug level - bool fDeleteTempFile; // delete tmp jobfile in destructor - std::string fJobFileName; // job file name - JobColList fDefaultColumns; // temporary list of default cols - // for table node being processed - bool fValidateColList; // Validate all cols have XML tag - std::string fTimeZone; // Timezone used for TIMESTAMP datatype + DebugLevel fDebugLevel; // internal use debug level + bool fDeleteTempFile; // delete tmp jobfile in destructor + std::string fJobFileName; // job file name + JobColList fDefaultColumns; // temporary list of default cols + // for table node being processed + bool fValidateColList; // Validate all cols have XML tag + std::string fTimeZone; // Timezone used for TIMESTAMP datatype }; -} //end of namespace +} // namespace WriteEngine #undef EXPORT -#endif // _WE_XMLJOB_H_ +#endif // _WE_XMLJOB_H_ diff --git a/writeengine/xml/we_xmlop.cpp b/writeengine/xml/we_xmlop.cpp index 16106feeb..f85cc71fc 100644 --- a/writeengine/xml/we_xmlop.cpp +++ b/writeengine/xml/we_xmlop.cpp @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_xmlop.cpp 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_xmlop.cpp 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #include @@ -29,11 +29,10 @@ namespace WriteEngine { - //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -XMLOp::XMLOp( ) : m_fDoc( NULL ), m_pRoot( NULL ) +XMLOp::XMLOp() : m_fDoc(NULL), m_pRoot(NULL) { } @@ -42,8 +41,8 @@ XMLOp::XMLOp( ) : m_fDoc( NULL ), m_pRoot( NULL ) //------------------------------------------------------------------------------ XMLOp::~XMLOp() { - closeDoc(); - xmlCleanupParser(); + closeDoc(); + xmlCleanupParser(); } //------------------------------------------------------------------------------ @@ -51,11 +50,11 @@ XMLOp::~XMLOp() //------------------------------------------------------------------------------ void XMLOp::closeDoc() { - if ( m_fDoc != NULL ) - { - xmlFreeDoc( m_fDoc ); - m_fDoc = NULL; - } + if (m_fDoc != NULL) + { + xmlFreeDoc(m_fDoc); + m_fDoc = NULL; + } } //------------------------------------------------------------------------------ @@ -63,29 +62,20 @@ void XMLOp::closeDoc() // buf - data buffer // dataType - data type //------------------------------------------------------------------------------ -void XMLOp::convertNodeValue( void* pVal, const char* buf, - XML_DTYPE dataType ) const +void XMLOp::convertNodeValue(void* pVal, const char* buf, XML_DTYPE dataType) const { - switch ( dataType ) - { - case TYPE_CHAR : - strcpy( (char*) pVal, buf ); - break; + switch (dataType) + { + case TYPE_CHAR: strcpy((char*)pVal, buf); break; - case TYPE_DOUBLE : - case TYPE_FLOAT : - *((float*)pVal) = (float) atof( buf ); - break; + case TYPE_DOUBLE: + case TYPE_FLOAT: *((float*)pVal) = (float)atof(buf); break; - case TYPE_LONGLONG : - *((long long*)pVal) = (long long) atoll( buf ); - break; + case TYPE_LONGLONG: *((long long*)pVal) = (long long)atoll(buf); break; - case TYPE_INT : - default : - *((int*)pVal) = (int) atoi( buf ); - break; - } + case TYPE_INT: + default: *((int*)pVal) = (int)atoi(buf); break; + } } //------------------------------------------------------------------------------ @@ -96,25 +86,24 @@ void XMLOp::convertNodeValue( void* pVal, const char* buf, // pVal (output) - return value buffer // returns TRUE if found, FALSE otherwise //------------------------------------------------------------------------------ -bool XMLOp::getNodeAttribute( xmlNode* pNode, const char* pTag, - void* pVal, XML_DTYPE dataType ) const +bool XMLOp::getNodeAttribute(xmlNode* pNode, const char* pTag, void* pVal, XML_DTYPE dataType) const { - char buf[XML_NODE_BUF_SIZE]; - xmlChar* pTmp = NULL; - bool bFound = false; + char buf[XML_NODE_BUF_SIZE]; + xmlChar* pTmp = NULL; + bool bFound = false; - pTmp = xmlGetProp( pNode, (xmlChar*) pTag ); + pTmp = xmlGetProp(pNode, (xmlChar*)pTag); - if ( pTmp ) - { - bFound = true; - strcpy( buf, (char*) pTmp ); - xmlFree( pTmp ); + if (pTmp) + { + bFound = true; + strcpy(buf, (char*)pTmp); + xmlFree(pTmp); - convertNodeValue( pVal, buf, dataType ); - } // end if + convertNodeValue(pVal, buf, dataType); + } // end if - return bFound; + return bFound; } //------------------------------------------------------------------------------ @@ -125,22 +114,21 @@ bool XMLOp::getNodeAttribute( xmlNode* pNode, const char* pTag, // pVal (output) - return value buffer // returns TRUE if found, FALSE otherwise //------------------------------------------------------------------------------ -bool XMLOp::getNodeAttributeStr( xmlNode* pNode, const char* pTag, - std::string& strVal ) const +bool XMLOp::getNodeAttributeStr(xmlNode* pNode, const char* pTag, std::string& strVal) const { - xmlChar* pTmp = NULL; - bool bFound = false; + xmlChar* pTmp = NULL; + bool bFound = false; - pTmp = xmlGetProp( pNode, (xmlChar*) pTag ); + pTmp = xmlGetProp(pNode, (xmlChar*)pTag); - if ( pTmp ) - { - bFound = true; - strVal = (char*)pTmp; - xmlFree( pTmp ); - } // end if + if (pTmp) + { + bFound = true; + strVal = (char*)pTmp; + xmlFree(pTmp); + } // end if - return bFound; + return bFound; } //------------------------------------------------------------------------------ @@ -150,27 +138,26 @@ bool XMLOp::getNodeAttributeStr( xmlNode* pNode, const char* pTag, // dataType - column data type // returns TRUE if found, FALSE otherwise //------------------------------------------------------------------------------ -bool XMLOp::getNodeContent( const xmlNode* pNode, void* pVal, - XML_DTYPE dataType ) +bool XMLOp::getNodeContent(const xmlNode* pNode, void* pVal, XML_DTYPE dataType) { - char buf[XML_NODE_BUF_SIZE]; - xmlChar* pTmp = NULL; - bool bFound = false; + char buf[XML_NODE_BUF_SIZE]; + xmlChar* pTmp = NULL; + bool bFound = false; - if ( pNode->children != NULL ) + if (pNode->children != NULL) + { + pTmp = xmlNodeGetContent(pNode->children); + + if (pTmp) { - pTmp = xmlNodeGetContent( pNode->children ); - - if ( pTmp ) - { - bFound = true; - strcpy( buf, (char*) pTmp ); - xmlFree( pTmp ); - convertNodeValue( pVal, buf, dataType ); - } + bFound = true; + strcpy(buf, (char*)pTmp); + xmlFree(pTmp); + convertNodeValue(pVal, buf, dataType); } + } - return bFound; + return bFound; } //------------------------------------------------------------------------------ @@ -179,24 +166,24 @@ bool XMLOp::getNodeContent( const xmlNode* pNode, void* pVal, // strVal - return value // returns TRUE if found, FALSE otherwise //------------------------------------------------------------------------------ -bool XMLOp::getNodeContentStr( const xmlNode* pNode, std::string& strVal) +bool XMLOp::getNodeContentStr(const xmlNode* pNode, std::string& strVal) { - xmlChar* pTmp = NULL; - bool bFound = false; + xmlChar* pTmp = NULL; + bool bFound = false; - if ( pNode->children != NULL ) + if (pNode->children != NULL) + { + pTmp = xmlNodeGetContent(pNode->children); + + if (pTmp) { - pTmp = xmlNodeGetContent( pNode->children ); - - if ( pTmp ) - { - bFound = true; - strVal = (char*)pTmp; - xmlFree( pTmp ); - } + bFound = true; + strVal = (char*)pTmp; + xmlFree(pTmp); } + } - return bFound; + return bFound; } //------------------------------------------------------------------------------ @@ -204,16 +191,16 @@ bool XMLOp::getNodeContentStr( const xmlNode* pNode, std::string& strVal) // xmlFileName - xml file name // returns NO_ERROR if success; other if failure //------------------------------------------------------------------------------ -int XMLOp::parseDoc( const char* xmlFileName ) +int XMLOp::parseDoc(const char* xmlFileName) { - int rc; + int rc; - rc = readDoc( xmlFileName ); + rc = readDoc(xmlFileName); - if ( rc != NO_ERROR ) - return rc; + if (rc != NO_ERROR) + return rc; - return processNode( m_pRoot ) ? NO_ERROR : ERR_XML_PARSE; + return processNode(m_pRoot) ? NO_ERROR : ERR_XML_PARSE; } //------------------------------------------------------------------------------ @@ -221,23 +208,23 @@ int XMLOp::parseDoc( const char* xmlFileName ) // xmlFileName - xml file name // returns NO_ERROR if success; other if failure //------------------------------------------------------------------------------ -int XMLOp::readDoc( const char* xmlFileName ) +int XMLOp::readDoc(const char* xmlFileName) { -// xmlNodePtr curPtr; - m_fDoc = xmlParseFile( xmlFileName ); + // xmlNodePtr curPtr; + m_fDoc = xmlParseFile(xmlFileName); - if ( m_fDoc == NULL ) - return ERR_XML_FILE; + if (m_fDoc == NULL) + return ERR_XML_FILE; - m_pRoot = xmlDocGetRootElement( m_fDoc ); + m_pRoot = xmlDocGetRootElement(m_fDoc); - if ( m_pRoot == NULL ) - { - closeDoc(); - return ERR_XML_ROOT_ELEM; - } + if (m_pRoot == NULL) + { + closeDoc(); + return ERR_XML_ROOT_ELEM; + } - return NO_ERROR; + return NO_ERROR; } //------------------------------------------------------------------------------ @@ -245,16 +232,16 @@ int XMLOp::readDoc( const char* xmlFileName ) // pParentNode - parent node // returns true if more nodes, else returns false. //------------------------------------------------------------------------------ -bool XMLOp::processNode( xmlNode* pParentNode ) +bool XMLOp::processNode(xmlNode* pParentNode) { - xmlNode* pCurNode = NULL; - bool bContinue = true; + xmlNode* pCurNode = NULL; + bool bContinue = true; - for ( pCurNode = pParentNode->children; pCurNode; pCurNode = pCurNode->next) - if ( pCurNode->type == XML_ELEMENT_NODE) - bContinue = processNode( pCurNode ); + for (pCurNode = pParentNode->children; pCurNode; pCurNode = pCurNode->next) + if (pCurNode->type == XML_ELEMENT_NODE) + bContinue = processNode(pCurNode); - return bContinue; + return bContinue; } -} //end of namespace +} // namespace WriteEngine diff --git a/writeengine/xml/we_xmlop.h b/writeengine/xml/we_xmlop.h index a7f8dcd69..2fe80e9b7 100644 --- a/writeengine/xml/we_xmlop.h +++ b/writeengine/xml/we_xmlop.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_xmlop.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_xmlop.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_XMLOP_H_ @@ -39,81 +39,76 @@ const int XML_NODE_BUF_SIZE = 256; */ class XMLOp { -protected: - /** @brief XML types used in parsing XML content and attributes - * - */ - enum XML_DTYPE - { - TYPE_EMPTY = 1, - TYPE_CHAR = 2, - TYPE_DOUBLE = 3, - TYPE_FLOAT = 4, - TYPE_LONGLONG = 5, - TYPE_INT = 6 - }; + protected: + /** @brief XML types used in parsing XML content and attributes + * + */ + enum XML_DTYPE + { + TYPE_EMPTY = 1, + TYPE_CHAR = 2, + TYPE_DOUBLE = 3, + TYPE_FLOAT = 4, + TYPE_LONGLONG = 5, + TYPE_INT = 6 + }; - /** - * @brief Constructor - */ - XMLOp(); + /** + * @brief Constructor + */ + XMLOp(); - /** - * @brief Default Destructor - */ - virtual ~XMLOp(); + /** + * @brief Default Destructor + */ + virtual ~XMLOp(); - /** - * @brief Get node attribute for non-strings - */ - bool getNodeAttribute( xmlNode* pNode, const char* pTag, - void* pVal, XML_DTYPE dataType ) const; + /** + * @brief Get node attribute for non-strings + */ + bool getNodeAttribute(xmlNode* pNode, const char* pTag, void* pVal, XML_DTYPE dataType) const; - /** - * @brief Get node attribute for strings - */ - bool getNodeAttributeStr( xmlNode* pNode, const char* pTag, - std::string& strVal ) const; + /** + * @brief Get node attribute for strings + */ + bool getNodeAttributeStr(xmlNode* pNode, const char* pTag, std::string& strVal) const; - /** - * @brief Get node content for non-strings - */ - bool getNodeContent( const xmlNode* pNode, void* pVal, - XML_DTYPE dataType ); + /** + * @brief Get node content for non-strings + */ + bool getNodeContent(const xmlNode* pNode, void* pVal, XML_DTYPE dataType); - /** - * @brief Get node content for strings - */ - bool getNodeContentStr( const xmlNode* pNode, std::string& strVal); + /** + * @brief Get node content for strings + */ + bool getNodeContentStr(const xmlNode* pNode, std::string& strVal); - /** - * @brief Check whether it is certain tag - */ - bool isTag( const xmlNode* pNode, const xmlTag tag ) - { - return !xmlStrcmp( pNode->name, (const xmlChar*)xmlTagTable[tag] ); - } + /** + * @brief Check whether it is certain tag + */ + bool isTag(const xmlNode* pNode, const xmlTag tag) + { + return !xmlStrcmp(pNode->name, (const xmlChar*)xmlTagTable[tag]); + } - /** - * @brief Parse xml document - */ - int parseDoc( const char* xmlFileName ); + /** + * @brief Parse xml document + */ + int parseDoc(const char* xmlFileName); - /** - * @brief Process node (recursion) - */ - virtual bool processNode( xmlNode* pParentNode ); + /** + * @brief Process node (recursion) + */ + virtual bool processNode(xmlNode* pParentNode); - xmlDocPtr m_fDoc; // xml document pointer - xmlNode* m_pRoot; // root element + xmlDocPtr m_fDoc; // xml document pointer + xmlNode* m_pRoot; // root element -private: - void closeDoc(); - void convertNodeValue( void* pVal, const char* buf, - XML_DTYPE dataType ) const; - int readDoc( const char* xmlFileName ); + private: + void closeDoc(); + void convertNodeValue(void* pVal, const char* buf, XML_DTYPE dataType) const; + int readDoc(const char* xmlFileName); }; - -} //end of namespace -#endif // _WE_XMLOP_H_ +} // namespace WriteEngine +#endif // _WE_XMLOP_H_ diff --git a/writeengine/xml/we_xmltag.h b/writeengine/xml/we_xmltag.h index c9e991179..8cd9d1c96 100644 --- a/writeengine/xml/we_xmltag.h +++ b/writeengine/xml/we_xmltag.h @@ -16,9 +16,9 @@ MA 02110-1301, USA. */ /******************************************************************************* -* $Id: we_xmltag.h 4450 2013-01-21 14:13:24Z rdempsey $ -* -*******************************************************************************/ + * $Id: we_xmltag.h 4450 2013-01-21 14:13:24Z rdempsey $ + * + *******************************************************************************/ /** @file */ #ifndef _WE_XMLTAG_H_ @@ -28,100 +28,73 @@ /** Namespace WriteEngine */ namespace WriteEngine { -const int MAX_XML_TAG_NAME_SIZE = 30; /** @brief Max size of xml tag name */ +const int MAX_XML_TAG_NAME_SIZE = 30; /** @brief Max size of xml tag name */ enum xmlTag { - // Element - TAG_BULK_JOB, - TAG_COLUMN, - TAG_CREATE_DATE, - TAG_CREATE_TIME, - TAG_DEFAULT_COLUMN, - TAG_DELIMITER, - TAG_DESC, - TAG_ENCLOSED_BY_CHAR, - TAG_ESCAPE_CHAR, - TAG_ID, - TAG_IGNORE_FIELD, - TAG_NAME, - TAG_PATH, // obsolete, but keep to be backwards compatible with old files - TAG_SCHEMA, - TAG_TABLE, - TAG_TYPE, // obsolete, but kept to be backwards compatible with old files - TAG_USER, - TAG_READ_BUFFERS, - TAG_WRITE_BUFFER_SIZE, + // Element + TAG_BULK_JOB, + TAG_COLUMN, + TAG_CREATE_DATE, + TAG_CREATE_TIME, + TAG_DEFAULT_COLUMN, + TAG_DELIMITER, + TAG_DESC, + TAG_ENCLOSED_BY_CHAR, + TAG_ESCAPE_CHAR, + TAG_ID, + TAG_IGNORE_FIELD, + TAG_NAME, + TAG_PATH, // obsolete, but keep to be backwards compatible with old files + TAG_SCHEMA, + TAG_TABLE, + TAG_TYPE, // obsolete, but kept to be backwards compatible with old files + TAG_USER, + TAG_READ_BUFFERS, + TAG_WRITE_BUFFER_SIZE, - // Attributes - TAG_NO_OF_READ_BUFFERS, - TAG_READ_BUFFER_SIZE, - TAG_COL_NAME, - TAG_COL_OID, - TAG_COL_TYPE, - TAG_COMPRESS_TYPE, - TAG_DATA_TYPE, - TAG_AUTOINCREMENT_FLAG, - TAG_DVAL_OID, - TAG_LOAD_NAME, - TAG_MAX_ERR_ROW, - TAG_NOT_NULL, - TAG_DEFAULT_VALUE, - TAG_ORIG_NAME, //@bug 3599: deprecated; kept for backwards compatibility - TAG_PRECISION, - TAG_SCALE, - TAG_TBL_NAME, - TAG_TBL_OID, - TAG_WIDTH, - TAG_SCHEMA_NAME, - NUM_OF_XML_TAGS + // Attributes + TAG_NO_OF_READ_BUFFERS, + TAG_READ_BUFFER_SIZE, + TAG_COL_NAME, + TAG_COL_OID, + TAG_COL_TYPE, + TAG_COMPRESS_TYPE, + TAG_DATA_TYPE, + TAG_AUTOINCREMENT_FLAG, + TAG_DVAL_OID, + TAG_LOAD_NAME, + TAG_MAX_ERR_ROW, + TAG_NOT_NULL, + TAG_DEFAULT_VALUE, + TAG_ORIG_NAME, //@bug 3599: deprecated; kept for backwards compatibility + TAG_PRECISION, + TAG_SCALE, + TAG_TBL_NAME, + TAG_TBL_OID, + TAG_WIDTH, + TAG_SCHEMA_NAME, + NUM_OF_XML_TAGS }; -const char xmlTagTable[NUM_OF_XML_TAGS + 1][MAX_XML_TAG_NAME_SIZE] = -{ +const char xmlTagTable[NUM_OF_XML_TAGS + 1][MAX_XML_TAG_NAME_SIZE] = { // Elements - "BulkJob", - "Column", - "CreateDate", - "CreateTime", - "DefaultColumn", - "Delimiter", - "Desc", - "EnclosedByChar", - "EscapeChar", - "Id", - "IgnoreField", - "Name", - "Path", //@bug 3777: keep obsolete tag - "Schema", - "Table", - "Type", //@bug 3777: keep obsolete tag - "User", - "ReadBuffers", - "WriteBufferSize", + "BulkJob", "Column", "CreateDate", "CreateTime", "DefaultColumn", "Delimiter", "Desc", "EnclosedByChar", + "EscapeChar", "Id", "IgnoreField", "Name", + "Path", //@bug 3777: keep obsolete tag + "Schema", "Table", + "Type", //@bug 3777: keep obsolete tag + "User", "ReadBuffers", "WriteBufferSize", // Attributes - "count", - "size", + "count", "size", "colName", //@bug 3599: replaces origName - "colOid", - "colType", - "compressType", - "dataType", - "autoincrement", - "dValOid", - "loadName", - "maxErrRow", - "notnull", - "defaultValue", - "origName", //@bug 3599: deprecated; kept for backwards compatibility - "precision", - "scale", + "colOid", "colType", "compressType", "dataType", "autoincrement", "dValOid", "loadName", "maxErrRow", + "notnull", "defaultValue", + "origName", //@bug 3599: deprecated; kept for backwards compatibility + "precision", "scale", "tblName", //@bug 3599: replaces origName - "tblOid", - "width", - "Name" -}; + "tblOid", "width", "Name"}; -} //end of namespace -#endif // _WE_XMLTAG_H_ +} // namespace WriteEngine +#endif // _WE_XMLTAG_H_